From 15f3b920ad7eb7fcca3afee14d16049db2046d4b Mon Sep 17 00:00:00 2001
From: Nathan Luehr <nluehr@nvidia.com>
Date: Wed, 14 Feb 2018 16:27:23 -0800
Subject: [PATCH 001/960] Fix __shared__ types with non-empty constructor

std::complex<T> has a non-empty constructor (zero assignment) that is not
compatible with CUDA __shared__ memory. This fixes current reliance on
undefined behavior. (and removes an unnecessary run-time initialization).
---
 .../core/kernels/reduction_gpu_kernels.cu.h   | 37 +++++++++++++++++--
 1 file changed, 33 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
index 15ae4c1fc5..95a3e222b5 100644
--- a/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
+++ b/tensorflow/core/kernels/reduction_gpu_kernels.cu.h
@@ -244,6 +244,33 @@ __global__ void RowReduceKernel(
   if (row < num_rows && lane == 0) out[row] = sum;
 }
 
+template <typename T1>
+struct storage_type {
+  T1 val;
+  __host__ __device__ storage_type() {}
+  __host__ __device__ operator T1() { return val; }
+  __host__ __device__ storage_type<T1>& operator=(const T1& in) {
+    val = in;
+    return *this;
+  }
+};
+
+template <typename T2>
+struct storage_type<std::complex<T2>> {
+  T2 real;
+  T2 imag;
+  __host__ __device__ storage_type() {}
+  __host__ __device__ operator std::complex<T2>() {
+    return std::complex<T2>(real, imag);
+  }
+  __host__ __device__ storage_type<std::complex<T2>>& operator=(
+      const std::complex<T2>& in) {
+    real = in.real();
+    imag = in.imag();
+    return *this;
+  }
+};
+
 // Works only if there are <= 16 columns
 // each warps sums over multiple rows at once
 template <typename T, typename outT, typename Op>
@@ -268,7 +295,7 @@ __global__ void ColumnReduceMax16ColumnsKernel(
 
   // 1D array necessary due to bug in CUDA 9 compiler.
   // TODO(nluehr) revert to 2D array when compiler is ready.
-  __shared__ value_type partial_sums[32 * 33];
+  __shared__ storage_type<value_type> partial_sums[32 * 33];
 
   row += rows_per_warp * gridDim.y * blockDim.y;
   for (; row < num_rows; row += rows_per_warp * gridDim.y * blockDim.y) {
@@ -294,7 +321,8 @@ __global__ void ColumnReduceMax16ColumnsKernel(
 
     if (blockDim.y > 1) {
       for (int row = 1; row < blockDim.y; ++row) {
-        s = op(s, partial_sums[threadIdx.x * 33 + row]);
+        value_type t = partial_sums[threadIdx.x * 33 + row];
+        s = op(s, t);
       }
     }
 
@@ -316,7 +344,7 @@ __global__ void ColumnReduceKernel(
 
   // 1D array necessary due to bug in CUDA 9 compiler.
   // TODO(nluehr) revert to 2D array when compiler is ready.
-  __shared__ value_type partial_sums[32 * 33];
+  __shared__ storage_type<value_type> partial_sums[32 * 33];
 
   row += gridDim.y * blockDim.y;
 
@@ -347,7 +375,8 @@ __global__ void ColumnReduceKernel(
         min(blockDim.y, num_rows - blockIdx.y * blockDim.y);
 
     for (int row = 1; row < numRowsThisBlock; ++row) {
-      s = op(s, partial_sums[threadIdx.x * 33 + row]);
+      value_type t = partial_sums[threadIdx.x * 33 + row];
+      s = op(s, t);
     }
 
     out[col * gridDim.y + blockIdx.y] = s;
-- 
GitLab


From 08a3509b2ecbd9fdfdb4f50b81e11f491291647e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20Thom=C3=A9?= <carlthome@gmail.com>
Date: Sun, 18 Feb 2018 17:02:13 +0100
Subject: [PATCH 002/960] Add NumPy style warning when casting complex to float

---
 tensorflow/python/ops/math_ops.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index da9957aa2a..2c422ebca4 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -169,6 +169,7 @@ from tensorflow.python.ops import gen_sparse_ops
 from tensorflow.python.ops import gen_spectral_ops
 from tensorflow.python.ops import gen_state_ops
 from tensorflow.python.ops import state_ops
+from tensorflow.python.platform import tf_logging as logging
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_math_ops import *
@@ -830,6 +831,8 @@ def to_float(x, name="ToFloat"):
   Raises:
     TypeError: If `x` cannot be cast to the `float32`.
   """
+  if x.dtype.is_complex:
+    logging.warn('Casting complex to real discards imaginary part.')
   return cast(x, dtypes.float32, name=name)
 
 
-- 
GitLab


From b808636c795e7a96a1e7264076a95d3e9343f430 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20Thom=C3=A9?= <carlthome@gmail.com>
Date: Sun, 18 Feb 2018 20:46:07 +0100
Subject: [PATCH 003/960] Fix quotes

---
 tensorflow/python/ops/math_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 2c422ebca4..4c7dc9559f 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -832,7 +832,7 @@ def to_float(x, name="ToFloat"):
     TypeError: If `x` cannot be cast to the `float32`.
   """
   if x.dtype.is_complex:
-    logging.warn('Casting complex to real discards imaginary part.')
+    logging.warn("Casting complex to real discards imaginary part.")
   return cast(x, dtypes.float32, name=name)
 
 
-- 
GitLab


From 62a05fe71ba5157e7abeb291f4b8b6ac7abf97fb Mon Sep 17 00:00:00 2001
From: DavidNorman <davidn@graphcore.ai>
Date: Tue, 27 Feb 2018 11:51:05 +0000
Subject: [PATCH 004/960] Ensure that the backend_deps is a non-frozen object

---
 tensorflow/compiler/xla/tests/build_defs.bzl | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/tests/build_defs.bzl b/tensorflow/compiler/xla/tests/build_defs.bzl
index 610302ac12..eac2eb286c 100644
--- a/tensorflow/compiler/xla/tests/build_defs.bzl
+++ b/tensorflow/compiler/xla/tests/build_defs.bzl
@@ -137,7 +137,8 @@ def xla_test(name,
       backend_deps += ["//tensorflow/compiler/xla/tests:test_macros_gpu"]
       this_backend_tags += ["requires-gpu-sm35"]
     elif backend in plugins:
-      backend_deps = plugins[backend]["deps"]
+      backend_deps = []
+      backend_deps += plugins[backend]["deps"]
       this_backend_copts += plugins[backend]["copts"]
       this_backend_tags += plugins[backend]["tags"]
       this_backend_args += plugins[backend]["args"]
-- 
GitLab


From 2e98952221bfe83fadc3054e66b2ff3c23c44a24 Mon Sep 17 00:00:00 2001
From: DavidNorman <davidn@graphcore.ai>
Date: Tue, 27 Feb 2018 13:52:13 +0000
Subject: [PATCH 005/960] Allow the large R1 slice tests to be disabled

---
 tensorflow/compiler/xla/tests/slice_test.cc | 35 +++++++++++++++++----
 1 file changed, 29 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc
index fe36df160d..50cd56d2d4 100644
--- a/tensorflow/compiler/xla/tests/slice_test.cc
+++ b/tensorflow/compiler/xla/tests/slice_test.cc
@@ -211,6 +211,9 @@ class SliceR1Test : public ClientLibraryTestBase,
   }
 };
 
+// A version of SliceR1Test used to label and disable 'large' tests
+class SliceR1LargeTest : public SliceR1Test {};
+
 string SliceR1TestDataToString(const ::testing::TestParamInfo<R1Spec>& data) {
   const R1Spec& spec = data.param;
   return ::tensorflow::strings::Printf("%lld_%lld_%lld_%lld", spec.input_dim0,
@@ -230,6 +233,18 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run<uint64>(GetParam()); }
 
 XLA_TEST_P(SliceR1Test, DoIt_S64) { Run<int64>(GetParam()); }
 
+XLA_TEST_P(SliceR1LargeTest, DoIt_F32) { Run<float>(GetParam()); }
+
+XLA_TEST_P(SliceR1LargeTest, DoIt_F64) { Run<double>(GetParam()); }
+
+XLA_TEST_P(SliceR1LargeTest, DoIt_U32) { Run<uint32>(GetParam()); }
+
+XLA_TEST_P(SliceR1LargeTest, DoIt_S32) { Run<int32>(GetParam()); }
+
+XLA_TEST_P(SliceR1LargeTest, DoIt_U64) { Run<uint64>(GetParam()); }
+
+XLA_TEST_P(SliceR1LargeTest, DoIt_S64) { Run<int64>(GetParam()); }
+
 // Tests for R1 slice ops.
 // The format for each testcase is {input size, start, limit, stride}.
 // clang-format off
@@ -237,12 +252,6 @@ INSTANTIATE_TEST_CASE_P(
     SliceR1TestInstantiation,
     SliceR1Test,
     ::testing::Values(
-// TODO(b/69425338): This uses too much memory on GPU.
-#ifndef XLA_TEST_BACKEND_GPU
-        R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1},
-        R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1},
-        R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1},
-#endif
         R1Spec{10, 0, 0, 1},
         R1Spec{10, 7, 7, 1},
         R1Spec{10, 0, 5, 1},
@@ -278,6 +287,20 @@ INSTANTIATE_TEST_CASE_P(
     SliceR1TestDataToString
 );
 
+// TODO(b/69425338): This uses too much memory on GPU.
+#ifndef XLA_TEST_BACKEND_GPU
+INSTANTIATE_TEST_CASE_P(
+    SliceR1TestBigSlicesInstantiation,
+    SliceR1LargeTest,
+    ::testing::Values(
+          R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024, 12 * 1024 * 1024, 1},
+          R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 + 1, 12 * 1024 * 1024 - 1, 1},
+          R1Spec{16 * 1024 * 1024, 4 * 1024 * 1024 - 1, 12 * 1024 * 1024 + 1, 1}
+    ),
+    SliceR1TestDataToString
+);
+#endif
+
 INSTANTIATE_TEST_CASE_P(
     SliceStridedR1TestInstantiation,
     SliceR1Test,
-- 
GitLab


From 1e60c2ff7f5537bbaeb9a7cdadbf2d4e7aa441e2 Mon Sep 17 00:00:00 2001
From: Peter Lee <peterlee0127@gmail.com>
Date: Sat, 3 Mar 2018 13:51:25 +0800
Subject: [PATCH 006/960] add support for other cpu tyoe(aarch64)  for tensorRT

---
 configure.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 97f46757ee..711dee2d65 100644
--- a/configure.py
+++ b/configure.py
@@ -40,7 +40,7 @@ _DEFAULT_CUDA_PATH = '/usr/local/cuda'
 _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
 _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing '
                           'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION)
-_DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/x86_64-linux-gnu'
+_DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/%s-linux-gnu' % platform.machine()
 _TF_OPENCL_VERSION = '1.2'
 _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
 _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
-- 
GitLab


From 70b60d9cce9a7879fbff396f283f19bed3b39793 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20Thom=C3=A9?= <carlthome@gmail.com>
Date: Mon, 5 Mar 2018 14:08:19 +0100
Subject: [PATCH 007/960] Move complex->float warning into tf.cast

---
 tensorflow/python/ops/math_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 56d58016b8..1608393c16 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -762,6 +762,8 @@ def cast(x, dtype, name=None):
   Raises:
     TypeError: If `x` cannot be cast to the `dtype`.
   """
+  if x.dtype.is_complex and dtype.is_floating:
+    logging.warn("Casting complex to real discards imaginary part.")
   base_type = dtypes.as_dtype(dtype).base_dtype
   with ops.name_scope(name, "Cast", [x]) as name:
     if isinstance(x, sparse_tensor.SparseTensor):
@@ -826,8 +828,6 @@ def to_float(x, name="ToFloat"):
   Raises:
     TypeError: If `x` cannot be cast to the `float32`.
   """
-  if x.dtype.is_complex:
-    logging.warn("Casting complex to real discards imaginary part.")
   return cast(x, dtypes.float32, name=name)
 
 
-- 
GitLab


From cf897725ab8c3f09d973c5f242b05ca7eb258801 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20Thom=C3=A9?= <carlthome@gmail.com>
Date: Mon, 5 Mar 2018 18:07:22 +0100
Subject: [PATCH 008/960] Check dtype after convert_to_tensor

---
 tensorflow/python/ops/math_ops.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 1608393c16..e315a09ea9 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -762,22 +762,22 @@ def cast(x, dtype, name=None):
   Raises:
     TypeError: If `x` cannot be cast to the `dtype`.
   """
-  if x.dtype.is_complex and dtype.is_floating:
-    logging.warn("Casting complex to real discards imaginary part.")
   base_type = dtypes.as_dtype(dtype).base_dtype
   with ops.name_scope(name, "Cast", [x]) as name:
     if isinstance(x, sparse_tensor.SparseTensor):
       values_cast = cast(x.values, base_type, name=name)
-      return sparse_tensor.SparseTensor(x.indices, values_cast, x.dense_shape)
+      x = sparse_tensor.SparseTensor(x.indices, values_cast, x.dense_shape)
     else:
       # TODO(josh11b): If x is not already a Tensor, we could return
       # ops.convert_to_tensor(x, dtype=dtype, ...)  here, but that
       # allows some conversions that cast() can't do, e.g. casting numbers to
       # strings.
       x = ops.convert_to_tensor(x, name="x")
-      if x.dtype.base_dtype == base_type:
-        return x
-      return gen_math_ops.cast(x, base_type, name=name)
+      if x.dtype.base_dtype != base_type:
+        x = gen_math_ops.cast(x, base_type, name=name)
+    if x.dtype.is_complex and dtype.is_floating:
+      logging.warn("Casting complex to real discards imaginary part.")
+    return x
 
 
 @tf_export("saturate_cast")
-- 
GitLab


From f7a04228e0368f3c9bad22a66fe7267e41ecb128 Mon Sep 17 00:00:00 2001
From: DavidNorman <davidn@graphcore.ai>
Date: Thu, 8 Mar 2018 07:05:53 +0000
Subject: [PATCH 009/960] Register half in some ops which support all floating
 point types

---
 tensorflow/core/ops/nn_ops.cc | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 910fbaca9e..6d4a3fda51 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -472,7 +472,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropInput")
     .Input("filter: T")
     .Input("out_backprop: T")
     .Output("output: T")
-    .Attr("T: {bfloat16, float, double}")
+    .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
@@ -490,7 +490,7 @@ REGISTER_OP("DepthwiseConv2dNativeBackpropFilter")
     .Input("filter_sizes: int32")
     .Input("out_backprop: T")
     .Output("output: T")
-    .Attr("T: {bfloat16, float, double}")
+    .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
@@ -589,7 +589,7 @@ REGISTER_OP("AvgPool3D")
     .Attr("strides: list(int) >= 5")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnet3dDataFormatAttrString())
-    .Attr("T: {bfloat16, float, double}")
+    .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn(shape_inference::Pool3DShape);
 
 REGISTER_OP("AvgPool3DGrad")
@@ -600,7 +600,7 @@ REGISTER_OP("AvgPool3DGrad")
     .Attr("strides: list(int) >= 5")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnet3dDataFormatAttrString())
-    .Attr("T: {bfloat16, float, double}")
+    .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
@@ -618,7 +618,7 @@ REGISTER_OP("MaxPool3D")
     .Attr("strides: list(int) >= 5")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnet3dDataFormatAttrString())
-    .Attr("T: {bfloat16, float}")
+    .Attr("T: {half, bfloat16, float}")
     .SetShapeFn(shape_inference::Pool3DShape);
 
 REGISTER_OP("MaxPool3DGrad")
@@ -630,8 +630,8 @@ REGISTER_OP("MaxPool3DGrad")
     .Attr("strides: list(int) >= 5")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnet3dDataFormatAttrString())
-    .Attr("T: {bfloat16, float} = DT_FLOAT")
-    .Attr("TInput: {bfloat16, float} = DT_FLOAT")
+    .Attr("T: {half, bfloat16, float} = DT_FLOAT")
+    .Attr("TInput: {half, bfloat16, float} = DT_FLOAT")
     .SetShapeFn([](InferenceContext* c) {
       return UnchangedShapeWithRank(c, 5);
     });
-- 
GitLab


From 6834f2ffcfd67b0fb198a3202341137a98fb9983 Mon Sep 17 00:00:00 2001
From: Luke Iwanski <luke@codeplay.com>
Date: Tue, 14 Nov 2017 15:12:42 +0000
Subject: [PATCH 010/960] Fixes automerge

---
 third_party/sycl/sycl/BUILD.tpl | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/third_party/sycl/sycl/BUILD.tpl b/third_party/sycl/sycl/BUILD.tpl
index 21b1a2bbf7..b7e9aa8edb 100755
--- a/third_party/sycl/sycl/BUILD.tpl
+++ b/third_party/sycl/sycl/BUILD.tpl
@@ -21,7 +21,7 @@ config_setting(
     name = "using_sycl_trisycl",
     define_values = {
         "using_sycl": "true",
-        "using_trisycl": "false",
+        "using_trisycl": "true",
     },
 )
 
-- 
GitLab


From 3bed12b81fe5ffc04e14ccaaf1b25ace4222f505 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Thu, 8 Mar 2018 13:56:11 -0800
Subject: [PATCH 011/960] Update version string to 1.7.0rc0 everywhere.

---
 tensorflow/core/public/version.h              |  4 ++--
 tensorflow/docs_src/install/install_c.md      |  2 +-
 tensorflow/docs_src/install/install_go.md     |  2 +-
 tensorflow/docs_src/install/install_java.md   | 22 +++++++++----------
 tensorflow/docs_src/install/install_linux.md  | 22 +++++++++----------
 tensorflow/docs_src/install/install_mac.md    | 10 ++++-----
 .../docs_src/install/install_sources.md       |  9 ++++++--
 tensorflow/tools/docker/Dockerfile.devel      |  2 +-
 .../tools/docker/Dockerfile.devel-cpu-mkl     |  2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |  2 +-
 tensorflow/tools/pip_package/setup.py         |  2 +-
 11 files changed, 42 insertions(+), 37 deletions(-)

diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 22f2c02b78..15082bb337 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 6
+#define TF_MINOR_VERSION 7
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc0"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 0481c97885..733c7a6625 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 8f89898c92..421215f367 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 0ee9c849e1..7758520c50 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc0</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0</version>
+                 <version>1.7.0-rc0</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc0</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc0</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc0.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.7.0-rc0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 3e8744bf9d..f4d4e65548 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -189,7 +189,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -294,7 +294,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -480,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -647,14 +647,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -666,14 +666,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -685,14 +685,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -704,14 +704,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 94defcd18c..055a463718 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -523,7 +523,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py2-none-any.whl
 </pre>
 
 
@@ -531,5 +531,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index c09c9c2c0c..10840295f9 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -359,10 +359,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0 on Linux:
+for TensorFlow 1.7.0rc0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -459,6 +459,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -478,6 +480,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -492,6 +495,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 22c73c3fe1..11f476d12c 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -70,7 +70,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
 # more difficult to experiment with local changes. Instead, just add
diff --git a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
index 3690e7dfe5..037d13116e 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
@@ -3,7 +3,7 @@ FROM tensorflow/tensorflow:latest-devel
 LABEL maintainer="Clayne Robison<clayne.b.robison@intel.com>"
 
 # These arguments are parameterized. Use --build-args to override.
-ARG TF_BRANCH=r1.6
+ARG TF_BRANCH=r1.7
 ARG WHL_DIR=/whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 69ba340f92..1fcb6428b2 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -79,7 +79,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 815ea8157d..69825a0d7c 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0'
+_VERSION = '1.7.0-rc0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
-- 
GitLab


From 2a849d5c1fda91c7cbb16786354d5143519da650 Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Thu, 8 Mar 2018 17:37:33 -0800
Subject: [PATCH 012/960] Disable checkpointable_utils_test

failed http://ci.tensorflow.org/view/Release/job/release-debian-cpu/99/consoleFull
---
 tensorflow/contrib/eager/python/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 5a6251b871..fad833dd2d 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -266,7 +266,10 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
+    tags = [
+        "no_oss",  # b/74395663
+        "no_windows",  # TODO: needs investigation on Windows
+    ],
 )
 
 filegroup(
-- 
GitLab


From b006115403f4a6592dee630132b0cf9c6519a922 Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Thu, 8 Mar 2018 19:54:14 -0800
Subject: [PATCH 013/960] Make spinn_test less flaky (#17580)

---
 tensorflow/contrib/eager/python/examples/spinn/spinn_test.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 081b0af14f..3f9a7818a5 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -417,7 +417,6 @@ class SpinnTest(test_util.TensorFlowTestCase):
                     if event.summary.value
                     and event.summary.value[0].tag == "train/loss"]
     self.assertEqual(config.epochs, len(train_losses))
-    self.assertLess(train_losses[-1], train_losses[0])
 
     # 5. Verify that checkpoints exist and contains all the expected variables.
     self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*")))
-- 
GitLab


From 32584800fe9032396713baf413914ddd391152dc Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 9 Mar 2018 00:14:49 -0800
Subject: [PATCH 014/960] Hide os from docs generator (#17577)

Hide os so we don't generate api_docs for it
---
 tensorflow/contrib/__init__.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index bcf0d7b48b..669d611b01 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -95,6 +95,7 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
+del os
 del LazyLoader
 
 del absolute_import
-- 
GitLab


From 4638dd1923055b9aa80ec643c1ccc3a78e41069a Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Fri, 9 Mar 2018 10:19:13 -0800
Subject: [PATCH 015/960] Fix pylint error in single_return.py

---
 tensorflow/contrib/py2tf/converters/single_return.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/py2tf/converters/single_return.py b/tensorflow/contrib/py2tf/converters/single_return.py
index 90bc22008f..1194b98f5e 100644
--- a/tensorflow/contrib/py2tf/converters/single_return.py
+++ b/tensorflow/contrib/py2tf/converters/single_return.py
@@ -212,7 +212,7 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
 
   def __init__(self):
     self.cant_return = False
-    super(gast.NodeVisitor, self).__init__()
+    super(DetectReturnInUnsupportedControlFlow, self).__init__()
 
   def visit_While(self, node):
     self.cant_return = True
-- 
GitLab


From b8f4e763171dcab40defcee1a981c3d2d32aaeca Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Fri, 9 Mar 2018 10:54:37 -0800
Subject: [PATCH 016/960] Adding the new variables to path rather than
 overriding them.

---
 tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
index e1b56b9a25..7d471b4703 100755
--- a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
+++ b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
@@ -31,5 +31,5 @@ export TF_NEED_OPENCL_SYCL=0
 export TF_NEED_MKL=0
 export COMPUTECPP_PATH="/usr/local"
 
-export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+export PATH="$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
 build_libtensorflow_tarball "-cpu-darwin-$(uname -m)"
-- 
GitLab


From 41b5fd15e72756dd6ee3a3395db306f107f1e628 Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Fri, 9 Mar 2018 11:09:13 -0800
Subject: [PATCH 017/960] Disable tensorflow/contrib/learn:monitors_test for
 pip gpu

---
 tensorflow/contrib/learn/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 895f70eecf..cc69678a2d 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -227,6 +227,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/monitors_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_pip_gpu"],  # b/74437598
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
-- 
GitLab


From 8ed55f4d54fbc85e2cd605aa6540b2fb5909500d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Carl=20Thom=C3=A9?= <carlthome@gmail.com>
Date: Mon, 12 Mar 2018 10:43:32 +0100
Subject: [PATCH 018/960] Change to tf.DType

---
 tensorflow/python/ops/math_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index e315a09ea9..c095be2aaf 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -775,7 +775,7 @@ def cast(x, dtype, name=None):
       x = ops.convert_to_tensor(x, name="x")
       if x.dtype.base_dtype != base_type:
         x = gen_math_ops.cast(x, base_type, name=name)
-    if x.dtype.is_complex and dtype.is_floating:
+    if x.dtype.is_complex and base_type.is_floating:
       logging.warn("Casting complex to real discards imaginary part.")
     return x
 
-- 
GitLab


From 3abebb0618cb6f830f5afaf2cd0b8c938e584aad Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Mon, 12 Mar 2018 11:36:17 -0700
Subject: [PATCH 019/960] Update RELEASE.md for r1.7 (#17583)

---
 RELEASE.md | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 60 insertions(+)

diff --git a/RELEASE.md b/RELEASE.md
index 6f54dee58f..c63d9f20c9 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,3 +1,63 @@
+# Release 1.7.0
+
+## Major Features And Improvements
+* Eager mode is moving out of contrib, try `tf.enable_eager_execution()`.
+* Graph rewrites emulating fixed-point quantization compatible with TensorFlow Lite, supported by new `tf.contrib.quantize` package.
+* Easily customize gradient computation with `tf.custom_gradient`.
+* [TensorBoard Debugger Plugin](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md), the graphical user interface (GUI) of TensorFlow Debugger (tfdbg), is now in alpha.
+* Experimental support for reading a sqlite database as a `Dataset` with new `tf.contrib.data.SqlDataset`.
+* Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`.
+* Better text processing with `tf.regex_replace`.
+* Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length`
+
+## Bug Fixes and Other Changes
+* Accelerated Linear Algebra (XLA):
+  * Add `MaxPoolGradGrad` support for XLA
+  * CSE pass from Tensorflow is now disabled in XLA.
+* `tf.data`:
+  * `tf.data.Dataset`
+    * Add support for building C++ Dataset op kernels as external libraries, using the `tf.load_op_library()` mechanism.
+    * `Dataset.list_files()` now shuffles its output by default.
+    * `Dataset.shuffle(..., seed=tf.constant(0, dtype=tf.int64))` now yields the same sequence of elements as `Dataset.shuffle(..., seed=0)`.
+  * Add `num_parallel_reads` argument to `tf.data.TFRecordDataset`.
+* `tf.contrib`:
+  * `tf.contrib.bayesflow.halton_sequence` now supports randomization.
+  * Add support for scalars in `tf.contrib.all_reduce`.
+  * Add `effective_sample_size` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `potential_scale_reduction` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `BatchNormalization`, `Kumaraswamy` bijectors.
+  * Deprecate `tf.contrib.learn`. Please check contrib/learn/README.md for instructions on how to convert existing code.
+  * `tf.contrib.data`
+    * Remove deprecated `tf.contrib.data.Dataset`, `tf.contrib.data.Iterator`, `tf.contrib.data.FixedLengthRecordDataset`, `tf.contrib.data.TextLineDataset`, and `tf.contrib.data.TFRecordDataset` classes.
+    * Added `bucket_by_sequence_length`, `sliding_window_batch`, and `make_batched_features_dataset`
+  * Remove unmaintained `tf.contrib.ndlstm`. You can find it externally at https://github.com/tmbarchive/tfndlstm.
+  * Moved most of `tf.contrib.bayesflow` to its own repo: `tfp`
+* Other:
+  * tf.py_func now reports the full stack trace if an exception occurs.
+  * Integrate `TPUClusterResolver` with GKE's integration for Cloud TPUs.
+  * Add a library for statistical testing of samplers.
+  * Add Helpers to stream data from the GCE VM to a Cloud TPU.
+  * Integrate ClusterResolvers with TPUEstimator.
+  * Unify metropolis_hastings interface with HMC kernel.
+  * Move LIBXSMM convolutions to a separate --define flag so that they are disabled by default.
+  * Fix `MomentumOptimizer` lambda.
+  * Reduce `tfp.layers` boilerplate via programmable docstrings.
+  * Add `auc_with_confidence_intervals`, a method for computing the AUC and confidence interval with linearithmic time complexity.
+  * `regression_head` now accepts customized link function, to satisfy the usage that user can define their own link function if the `array_ops.identity` does not meet the requirement.
+  * Fix `initialized_value` and `initial_value` behaviors for `ResourceVariables` created from `VariableDef` protos.
+  * Add TensorSpec to represent the specification of Tensors.
+  * Constant folding pass is now deterministic.
+  * Support `float16` `dtype` in `tf.linalg.*`.
+  * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abe, Alistair Low, Andy Kernahan, Appledore, Ben, Ben Barsdell, Boris Pfahringer, Brad Wannow, Brett Koonce, Carl Thomé, cclauss, Chengzhi Chen, Chris Drake, Christopher Yeh, Clayne Robison, Codrut Grosu, Daniel Trebbien, Danny Goodman, David Goodwin, David Norman, Deron Eriksson, Donggeon Lim, Donny Viszneki, DosLin, DylanDmitri, Francisco Guerrero, Fred Reiss, gdh1995, Giuseppe, Glenn Weidner, gracehoney, Guozhong Zhuang, Haichen "Hc" Li, Harald Husum, harumitsu.nobuta, Henry Spivey, hsm207, Jekyll Song, Jerome, Jiongyan Zhang, jjsjann123, John Sungjin Park, Johnson145, JoshVarty, Julian Wolff, Jun Wang, June-One, Kamil Sindi, Kb Sriram, Kdavis-Mozilla, Kenji, lazypanda1, Liang-Chi Hsieh, Loo Rong Jie, Mahesh Bhosale, MandarJKulkarni, ManHyuk, Marcus Ong, Marshal Hayes, Martin Pool, matthieudelaro, mdfaijul, mholzel, Michael Zhou, Ming Li, Minmin Sun, Myungjoo Ham, MyungsungKwak, Naman Kamra, Peng Yu, Penghao Cen, Phil, Raghuraman-K, resec, Rohin Mohanadas, Sandeep N Gupta, Scott Tseng, seaotterman, Seo Sanghyeon, Sergei Lebedev, Ted Chang, terrytangyuan, Tim H, tkunic, Tod, vihanjain, Yan Facai (颜发才), Yin Li, Yong Tang, Yukun Chen, Yusuke Yamada
+
+
+
 # Release 1.6.0
 
 ## Breaking Changes
-- 
GitLab


From d392b1c9ebf131b9ac64ff289d26e43afea21c10 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Fri, 9 Mar 2018 18:17:43 -0800
Subject: [PATCH 020/960] Fix the windows build

---
 tensorflow/core/kernels/snapshot_op.cc        | 30 +++++++++++++++++++
 tensorflow/core/kernels/snapshot_op.h         | 26 +++++-----------
 tensorflow/core/kernels/snapshot_op_gpu.cu.cc | 10 +++----
 3 files changed, 42 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/snapshot_op.cc b/tensorflow/core/kernels/snapshot_op.cc
index 50157d5d48..fe04dcf72e 100644
--- a/tensorflow/core/kernels/snapshot_op.cc
+++ b/tensorflow/core/kernels/snapshot_op.cc
@@ -22,6 +22,26 @@ limitations under the License.
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename Device, typename Scalar>
+class SnapshotOp : public OpKernel {
+ public:
+  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& input = context->input(0);
+    Tensor* output = nullptr;
+    // Try to use buffer forwarding to avoid an explicit copy.
+    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                {0}, 0, input.shape(), &output));
+    if (!output->SharesBufferWith(input)) {
+      functor::Snapshot<Device, Scalar> functor;
+      functor(context->eigen_device<Device>(), input.flat<Scalar>(),
+              output->flat<Scalar>());
+    }
+  }
+};
 
 #define REGISTER_KERNEL(TYPE)                                        \
   REGISTER_KERNEL_BUILDER(                                           \
@@ -31,6 +51,16 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
 
+#if GOOGLE_CUDA
+#define REGISTER_KERNEL(TYPE)                                        \
+  REGISTER_KERNEL_BUILDER(                                           \
+      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
+      SnapshotOp<GPUDevice, TYPE>);
+
+TF_CALL_POD_TYPES(REGISTER_KERNEL);
+#undef REGISTER_KERNEL
+#endif
+
 #if TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SyclDevice;
 #define REGISTER_SYCL_KERNEL(TYPE)                                    \
diff --git a/tensorflow/core/kernels/snapshot_op.h b/tensorflow/core/kernels/snapshot_op.h
index b94834f159..a18065d42b 100644
--- a/tensorflow/core/kernels/snapshot_op.h
+++ b/tensorflow/core/kernels/snapshot_op.h
@@ -26,29 +26,19 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
+namespace functor {
 
+// Functor used by SnapshotOp.
 template <typename Device, typename Scalar>
-class SnapshotOp : public OpKernel {
- public:
-  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input = context->input(0);
-    Tensor* output = nullptr;
-    // Try to use buffer forwarding to avoid an explicit copy.
-    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 0, input.shape(), &output));
-    if (!output->SharesBufferWith(input)) {
-      // We had to allocate a new buffer since the refcount on the input was
-      // greater than 1. Copy the input to the new buffer.
-      const Device& device = context->eigen_device<Device>();
-      device.memcpy(output->template flat<Scalar>().data(),
-                    input.template flat<Scalar>().data(),
-                    input.NumElements() * sizeof(Scalar));
-    }
+struct Snapshot {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar>::ConstTensor input,
+                  typename TTypes<Scalar>::Tensor output) {
+    device.memcpy(output.data(), input.data(), input.size() * sizeof(Scalar));
   }
 };
 
+}  // namespace functor
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_KERNELS_SNAPSHOT_OP_H_
diff --git a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
index 52070be838..f1c0ed2eae 100644
--- a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
@@ -24,13 +24,11 @@ limitations under the License.
 namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_KERNEL(TYPE)                                        \
-  REGISTER_KERNEL_BUILDER(                                           \
-      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
-      SnapshotOp<GPUDevice, TYPE>);
+// Definition of the GPU implementations declared in softsign_op.cc.
+#define DEFINE_GPU_KERNELS(T)                      \
+  template struct functor::Snapshot<GPUDevice, T>;
 
-TF_CALL_POD_TYPES(REGISTER_KERNEL);
-#undef REGISTER_KERNEL
+TF_CALL_POD_TYPES(DEFINE_GPU_KERNELS);
 
 }  // namespace tensorflow
 
-- 
GitLab


From 73f2da07577330648cd294d321545d089b600748 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Sat, 10 Mar 2018 22:24:27 -0800
Subject: [PATCH 021/960] Disable keras:convolutional_test.

---
 tensorflow/contrib/cmake/tf_tests.cmake | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 1c4ebd7f0c..e2ed5f6c73 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -208,6 +208,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py"
     # Test is flaky on Windows GPU builds (b/38283730).
     "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/gmm_test.py"
+    # Disable following manual tag in BUILD.
+    "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/layers/convolutional_test.py"
+
   )
   if (WIN32)
     set(tf_test_src_py_exclude
-- 
GitLab


From df4cbfa33d711c1fad107bfaea0862bfdc8c3fd8 Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Mon, 12 Mar 2018 16:49:40 -0700
Subject: [PATCH 022/960] Make tensorflow/python:framework_importer_test large

tensorflow/python:framework_importer_test sometime times out during release builds
---
 tensorflow/python/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 04e926ff16..6dd53ffdf6 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1048,7 +1048,7 @@ py_test(
 
 py_test(
     name = "framework_importer_test",
-    size = "medium",
+    size = "large",
     srcs = ["framework/importer_test.py"],
     main = "framework/importer_test.py",
     srcs_version = "PY2AND3",
-- 
GitLab


From 4b0bb6d5d01c0b2aa89df11b45ff8d728d4c2059 Mon Sep 17 00:00:00 2001
From: Siby Jose Plathottam <sibyjackgrove@gmail.com>
Date: Mon, 12 Mar 2018 22:48:49 -0500
Subject: [PATCH 023/960] Fix to 'Model' object has no attribute
 '_container_nodes' error when using tf.keras.utils.plot_model().

Fix to #17633. Duplicate of #17658
'Model' object has no attribute '_container_nodes' error when using tf.keras.utils.plot_model().
Replaced
if node_key in model._container_nodes:
with
if node_key in model._network_nodes: # pylint: disable=protected-access

in tensorflow\python\keras_impl\keras\utils\vis_utils.py.
---
 tensorflow/python/keras/_impl/keras/utils/vis_utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
index 45c1b92075..4761cece82 100644
--- a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
@@ -120,7 +120,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
     layer_id = str(id(layer))
     for i, node in enumerate(layer._inbound_nodes):
       node_key = layer.name + '_ib-' + str(i)
-      if node_key in model._container_nodes:
+      if node_key in model._network_nodes:  # pylint: disable=protected-access
         for inbound_layer in node.inbound_layers:
           inbound_layer_id = str(id(inbound_layer))
           layer_id = str(id(layer))
-- 
GitLab


From fa5c66ba74a505e4a4b8472332918798bb17bb39 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Sun, 11 Mar 2018 15:38:16 -0700
Subject: [PATCH 024/960] Fixes a race condition in function instantiation.

Previously, if the same function was being concurrently instantiated
and released:

1. Thread one could begin to instantiate the function, determine
   that it already existed in the runtime, then be preempted.
2. Thread two could release the handle on the function, causing it to
   be freed and removed from the `FunctionLibraryRuntime::items_` map.
3. Thread one could then incorrectly assume that the function still
   existed, and fail to find it in the `FunctionLibraryRuntime::items_`
   map, causing a segfault when it attempted to increment the refcount
   on an uninitialized object.

PiperOrigin-RevId: 188661500
---
 tensorflow/core/common_runtime/function.cc    | 24 +++++++++++++++----
 .../kernel_tests/filter_dataset_op_test.py    |  8 +++++++
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 3e937ceb64..7174a876f6 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -479,11 +479,26 @@ Status FunctionLibraryRuntimeImpl::Instantiate(
   InstantiateOptions options_copy(options);
   options_copy.target = device_name_;
   const string key = Canonicalize(function_name, attrs, options_copy);
-  *handle = parent_->GetHandle(key);
-  if (*handle != kInvalidHandle) {
+
+  {
     mutex_lock l(mu_);
-    items_[parent_->GetHandleOnDevice(device_name_, *handle)]->Ref();
-    return Status::OK();
+    *handle = parent_->GetHandle(key);
+    if (*handle != kInvalidHandle) {
+      FunctionLibraryRuntime::LocalHandle handle_on_device =
+          parent_->GetHandleOnDevice(device_name_, *handle);
+      if (handle_on_device == kInvalidLocalHandle) {
+        return errors::Internal("LocalHandle not found for handle ", *handle,
+                                ".");
+      }
+      auto item_handle = items_.find(handle_on_device);
+      if (item_handle == items_.end()) {
+        return errors::Internal("LocalHandle ", handle_on_device,
+                                " for handle ", *handle,
+                                " not found in items.");
+      }
+      item_handle->second->Ref();
+      return Status::OK();
+    }
   }
 
   Status s;
@@ -536,6 +551,7 @@ Status FunctionLibraryRuntimeImpl::ReleaseHandle(Handle handle) {
   }
 
   LocalHandle h = parent_->GetHandleOnDevice(device_name_, handle);
+  CHECK_NE(h, kInvalidLocalHandle);
   mutex_lock l(mu_);
   CHECK_EQ(1, items_.count(h));
   Item* item = items_[h];
diff --git a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
index 2c71723167..4f2216f0a3 100644
--- a/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/filter_dataset_op_test.py
@@ -176,6 +176,14 @@ class FilterDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
+  def testParallelFilters(self):
+    dataset = dataset_ops.Dataset.range(10).filter(
+        lambda x: math_ops.equal(x % 2, 0))
+    iterators = [dataset.make_one_shot_iterator() for _ in range(10)]
+    next_elements = [iterator.get_next() for iterator in iterators]
+    with self.test_session() as sess:
+      self.assertEqual([0 for _ in range(10)], sess.run(next_elements))
+
 
 class FilterDatasetBenchmark(test.Benchmark):
 
-- 
GitLab


From 8198a84e1a584cd3d14acd0bd52e04cf2d66f341 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Tue, 13 Mar 2018 16:38:08 -0700
Subject: [PATCH 025/960] Update documentation

---
 tensorflow/contrib/tensorrt/README.md | 46 +++++++++++++++++++++++----
 1 file changed, 40 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index 461e627e99..6eafc1754c 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -1,15 +1,15 @@
-Using TensorRT in TensorFlow
-============================
+# Using TensorRT in TensorFlow
+
 
 This module provides necessary bindings and introduces TRT_engine_op
 operator that wraps a subgraph in TensorRT. This is still a work in progress
 but should be useable with most common graphs.
 
-Compilation
------------
+## Compilation
+
 
 In order to compile the module, you need to have a local TensorRT
-installation (libnvinfer.so and respective include files). During the
+installation ( libnvinfer.so and respective include files ). During the
 configuration step, TensorRT should be enabled and installation path
 should be set. If installed through package managers (deb,rpm),
 configure script should find the necessary components from the system
@@ -22,4 +22,38 @@ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use can be found in test/test_tftrt.py directory
+will be available. An example use can be found in test/test_tftrt.py script
+
+## Installing TensorRT 3.0.4
+
+In order to make use of TensorRT integration, you will need a local installation of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). Due to compiler compatibility, you will need to download and install the TensorRT 3.0.4 tarball for _Ubuntu 14.04_, i.e., **_TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz_**, even if you are using Ubuntu 16.04 or later.
+
+### Preparing TensorRT installation
+
+Once you have downloaded TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz, you will need to unpack it to an installation directory, which will be referred to as <install_dir>. Please replace <install_dir> with the full path of actual installation directory you choose in commands below.
+
+```shell
+cd <install_dir> && tar -zxf /path/to/TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz
+```
+
+After unpacking the binaries, you have several options to use them:
+
+#### To run TensorFlow as a user without superuser privileges
+
+For a regular user without any sudo rights, you should add TensorRT to your `$LD_LIBRARY_PATH`:
+
+  ```shell
+   export LD_LIBRARY_PATH=<install_dir>/TensorRT-3.0.4/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+  ```
+
+Then you are ready to use TensorFlow-TensorRT integration. `$LD_LIBRARY_PATH` must contain the path to TensorRT installation for TensorFlow-TensorRT integration to work. If you are using a VirtualEnv-like setup, you can add the command above to your `bin/activate` script or to your `.bashrc` script.
+
+#### To run TensorFlow as a superuser
+
+ When running as a superuser, such as in a container or via sudo, the `$LD_LIBRARY_PATH` approach above may not work. The following is preferred when the user has superuser privileges:
+
+  ```shell
+  echo "<install_dir>/TensorRT-3.0.4/lib" | sudo tee /etc/ld.so.conf.d/tensorrt304.conf && sudo ldconfig
+  ```
+
+  Please ensure that any existing deb package installation of TensorRT is removed before following these instructions to avoid package conflicts.
\ No newline at end of file
-- 
GitLab


From 6db713c99b2f5f929f94a86997c0462a2c7a5486 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 14 Mar 2018 13:10:27 -0700
Subject: [PATCH 026/960] Fix tensorflow/python/eager:benchmarks_test.

PiperOrigin-RevId: 189073615
---
 tensorflow/python/eager/benchmarks_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 4255677a68..8c6e7e5758 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -55,7 +55,7 @@ def c_tfe_py_fastpath_execute(a,
                               transpose_b=False,
                               name=None):
   ctx = context.context()
-  assert ctx.in_eager_mode(
+  assert ctx.executing_eagerly(
   ), "The prototype doesn't contain C code for graph construction"
   try:
     return pywrap_tensorflow.TFE_Py_FastPathExecute(
-- 
GitLab


From 15d97a4bb76fd613b5c8f8eb3178396c66f0529e Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 14 Mar 2018 13:11:57 -0700
Subject: [PATCH 027/960] Removing underscore prefix from _scalar_summary and
 _histogram_summary.

PiperOrigin-RevId: 189073787
---
 tensorflow/python/framework/python_op_gen.cc | 3 +--
 tensorflow/python/ops/logging_ops.py         | 4 ++--
 tensorflow/python/summary/summary.py         | 6 ++----
 3 files changed, 5 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 00a63650b7..271597c459 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -100,8 +100,7 @@ bool IsOpWithUnderscorePrefix(const string& s) {
        "fused_batch_norm", "histogram_fixed_width", "stack",
        "batch_norm_with_global_normalization",
        // TODO(annarev): replace these ops in the next change.
-       "broadcast_gradient_args", "enter", "histogram_summary", "ref_enter",
-       "ref_identity", "scalar_summary"});
+       "broadcast_gradient_args", "enter", "ref_enter", "ref_identity"});
   return kUnderscoreOps->count(s) > 0;
 }
 
diff --git a/tensorflow/python/ops/logging_ops.py b/tensorflow/python/ops/logging_ops.py
index a7ea7dc6e1..222b8ebc9d 100644
--- a/tensorflow/python/ops/logging_ops.py
+++ b/tensorflow/python/ops/logging_ops.py
@@ -109,7 +109,7 @@ def histogram_summary(tag, values, collections=None, name=None):
     buffer.
   """
   with ops.name_scope(name, "HistogramSummary", [tag, values]) as scope:
-    val = gen_logging_ops._histogram_summary(
+    val = gen_logging_ops.histogram_summary(
         tag=tag, values=values, name=scope)
     _Collect(val, collections, [ops.GraphKeys.SUMMARIES])
   return val
@@ -346,7 +346,7 @@ def scalar_summary(tags, values, collections=None, name=None):
     buffer.
   """
   with ops.name_scope(name, "ScalarSummary", [tags, values]) as scope:
-    val = gen_logging_ops._scalar_summary(tags=tags, values=values, name=scope)
+    val = gen_logging_ops.scalar_summary(tags=tags, values=values, name=scope)
     _Collect(val, collections, [ops.GraphKeys.SUMMARIES])
   return val
 
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 2a3918b9b4..f1b2be0a1a 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -98,8 +98,7 @@ def scalar(name, tensor, collections=None, family=None):
   """
   with _summary_op_util.summary_scope(
       name, family, values=[tensor]) as (tag, scope):
-    # pylint: disable=protected-access
-    val = _gen_logging_ops._scalar_summary(tags=tag, values=tensor, name=scope)
+    val = _gen_logging_ops.scalar_summary(tags=tag, values=tensor, name=scope)
     _summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
   return val
 
@@ -191,8 +190,7 @@ def histogram(name, values, collections=None, family=None):
   with _summary_op_util.summary_scope(
       name, family, values=[values],
       default_name='HistogramSummary') as (tag, scope):
-    # pylint: disable=protected-access
-    val = _gen_logging_ops._histogram_summary(
+    val = _gen_logging_ops.histogram_summary(
         tag=tag, values=values, name=scope)
     _summary_op_util.collect(val, collections, [_ops.GraphKeys.SUMMARIES])
   return val
-- 
GitLab


From 4ff81a7d891c2affadb9c9068cf37fdb109c0e77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 13:46:57 -0700
Subject: [PATCH 028/960] Set shard count to 4 for
 tensorflow/contrib/linalg:linear_operator_block_diag_test to avoid timeouts

This was getting flaky timeouts because it was taking over 300s about 25% of the time.
This change increases the shard count to 4 to reduce the total elapsed time.
With 4 shards, the maximum time per shard is about 180s.

PiperOrigin-RevId: 189079453
---
 tensorflow/contrib/linalg/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD
index d4f2e70631..3bc1427bd2 100644
--- a/tensorflow/contrib/linalg/BUILD
+++ b/tensorflow/contrib/linalg/BUILD
@@ -58,6 +58,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    shard_count = 4,
 )
 
 filegroup(
-- 
GitLab


From 16632df29d09b3bbe5cc75df39f5ec720225e6d1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 13:47:22 -0700
Subject: [PATCH 029/960] Remove nsync header files from
 tensorflow/core/platform/default/mutex.h

The implementation of tensorflow/core/platform/default/mutex.h uses the nsync
library, so mutex.h has included nsync header files.

This has been awkward, because each TensorFlow build (bazel, cmake, make, plus
the instructions for compiling individual custom ops, on all the various
platforms) has needed to handle the include paths correctly, reaching into a
package that is downloaded separately from TensorFlow itself.

This change avoids that awkwardness, instead taking on two different
irritations:
- mutex.h now defines two structs that are large enough and aligned enough to
  contain an nsync_mu and an nsync_cv.  This is an abstraction violation,
  because TensorFlow's source should not need to know how big these data
  structures are.  However, this is unlikely to cause problems because:
  1) this is checked by a static assertion in mutex.cc, so we will notice
     immediately should a change be needed, and
  2) this will likely never fail because we have no intent of allowing nsync's
     data strcutures to get bigger.

- The methods of mutex and condition_variable can no longer be inlined, because
  that too would require mutex.h to include the nsync header files.  (Or we'd
  need to declare the nsync functions directly in mutex.h, which would be
  another abstraction violation.)   However, this is a small imposition
  because the overhead of a procedure call is typically small.

The assumption behind this CL is that these irritations are less important than
the ongoing frustration of maintaining the complex include path in multiple build
systems.

PiperOrigin-RevId: 189079523
---
 .../compiler/xla/service/interpreter/BUILD    |  5 +-
 .../contrib/android/cmake/CMakeLists.txt      |  1 -
 tensorflow/contrib/cmake/tf_shared_lib.cmake  |  7 +-
 .../contrib/makefile/proto_text_cc_files.txt  |  1 +
 tensorflow/core/BUILD                         |  1 -
 tensorflow/core/platform/default/mutex.cc     | 89 +++++++++++++++++++
 tensorflow/core/platform/default/mutex.h      | 54 +++++------
 tensorflow/python/platform/sysconfig.py       |  1 -
 tensorflow/tensorflow.bzl                     | 17 ----
 tensorflow/tools/pip_package/setup.py         |  3 +-
 10 files changed, 121 insertions(+), 58 deletions(-)
 create mode 100644 tensorflow/core/platform/default/mutex.cc

diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 0819ab3b90..0db3863f24 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -63,10 +63,7 @@ cc_library(
     name = "platform_id",
     srcs = ["platform_id.cc"],
     hdrs = ["platform_id.h"],
-    deps = [
-        "@nsync//:nsync_headers",
-        "//tensorflow/core:stream_executor_headers_lib",
-    ] + if_static(
+    deps = ["//tensorflow/core:stream_executor_headers_lib"] + if_static(
         ["@protobuf_archive//:protobuf"],
         ["@protobuf_archive//:protobuf_headers"],
     ),
diff --git a/tensorflow/contrib/android/cmake/CMakeLists.txt b/tensorflow/contrib/android/cmake/CMakeLists.txt
index a115d1610e..ecf1a103d2 100644
--- a/tensorflow/contrib/android/cmake/CMakeLists.txt
+++ b/tensorflow/contrib/android/cmake/CMakeLists.txt
@@ -75,7 +75,6 @@ target_link_libraries(tensorflow_inference
 include_directories(
     ${PREBUILT_DIR}/proto
     ${PREBUILT_DIR}/protobuf/include
-    ${PREBUILT_DIR}/nsync/public
     ${TENSORFLOW_ROOT_DIR}/tensorflow/contrib/makefile/downloads/eigen
     ${TENSORFLOW_ROOT_DIR}
     ${CMAKE_CURRENT_SOURCE_DIR}/..)
diff --git a/tensorflow/contrib/cmake/tf_shared_lib.cmake b/tensorflow/contrib/cmake/tf_shared_lib.cmake
index 6d36d5fc5c..9738bbeb9a 100644
--- a/tensorflow/contrib/cmake/tf_shared_lib.cmake
+++ b/tensorflow/contrib/cmake/tf_shared_lib.cmake
@@ -100,8 +100,7 @@ if(WIN32)
 endif(WIN32)
 
 target_include_directories(tensorflow PUBLIC 
-    $<INSTALL_INTERFACE:include/>
-    $<INSTALL_INTERFACE:include/external/nsync/public>)
+    $<INSTALL_INTERFACE:include/>)
 
 install(TARGETS tensorflow EXPORT tensorflow_export
         RUNTIME DESTINATION bin
@@ -133,10 +132,6 @@ install(DIRECTORY ${tensorflow_source_dir}/tensorflow/stream_executor/
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src/google/
         DESTINATION include/google
         FILES_MATCHING PATTERN "*.h")
-# nsync headers
-install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/
-        DESTINATION include/external/nsync
-        FILES_MATCHING PATTERN "*.h")
 # Eigen directory
 install(DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/eigen/src/eigen/Eigen/
         DESTINATION include/Eigen)
diff --git a/tensorflow/contrib/makefile/proto_text_cc_files.txt b/tensorflow/contrib/makefile/proto_text_cc_files.txt
index d56e388477..77c936d8c5 100644
--- a/tensorflow/contrib/makefile/proto_text_cc_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_cc_files.txt
@@ -17,6 +17,7 @@ tensorflow/core/platform/env_time.cc
 tensorflow/core/platform/setround.cc
 tensorflow/core/platform/denormal.cc
 tensorflow/core/platform/default/tracing.cc
+tensorflow/core/platform/default/mutex.cc
 tensorflow/core/platform/default/logging.cc
 tensorflow/core/platform/cpu_info.cc
 tensorflow/core/lib/wav/wav_io.cc
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 9f5b8027f3..213315f40e 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1958,7 +1958,6 @@ cc_header_only_library(
     deps = [
         ":framework",
         ":reader_base",
-        "@nsync//:nsync_headers",
     ],
 )
 
diff --git a/tensorflow/core/platform/default/mutex.cc b/tensorflow/core/platform/default/mutex.cc
new file mode 100644
index 0000000000..79830a4738
--- /dev/null
+++ b/tensorflow/core/platform/default/mutex.cc
@@ -0,0 +1,89 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/mutex.h"
+#include <chrono>
+#include <condition_variable>
+#include "nsync_cv.h"
+#include "nsync_mu.h"
+
+namespace tensorflow {
+
+// Check that the external_mu_space struct used to reserve space for the mutex
+// in tensorflow::mutex is big enough.
+static_assert(sizeof(nsync::nsync_mu) <= sizeof(mutex::external_mu_space),
+              "tensorflow::mutex::external_mu_space needs to be bigger");
+
+// Cast a pointer to mutex::external_mu_space to a pointer to the mutex mutex
+// representation.  This is done so that the header files for nsync_mu do not
+// need to be included in every file that uses tensorflow's mutex.
+static inline nsync::nsync_mu *mu_cast(mutex::external_mu_space *mu) {
+  return reinterpret_cast<nsync::nsync_mu *>(mu);
+}
+
+mutex::mutex() { nsync::nsync_mu_init(mu_cast(&mu_)); }
+
+void mutex::lock() { nsync::nsync_mu_lock(mu_cast(&mu_)); }
+
+bool mutex::try_lock() { return nsync::nsync_mu_trylock(mu_cast(&mu_)) != 0; };
+
+void mutex::unlock() { nsync::nsync_mu_unlock(mu_cast(&mu_)); }
+
+void mutex::lock_shared() { nsync::nsync_mu_rlock(mu_cast(&mu_)); }
+
+bool mutex::try_lock_shared() {
+  return nsync::nsync_mu_rtrylock(mu_cast(&mu_)) != 0;
+};
+
+void mutex::unlock_shared() { nsync::nsync_mu_runlock(mu_cast(&mu_)); }
+
+// Check that the external_cv_space struct used to reserve space for the
+// condition variable in tensorflow::condition_variable is big enough.
+static_assert(
+    sizeof(nsync::nsync_cv) <= sizeof(condition_variable::external_cv_space),
+    "tensorflow::condition_variable::external_cv_space needs to be bigger");
+
+// Cast a pointer to mutex::external_cv_space to a pointer to the condition
+// variable representation.  This is done so that the header files for nsync_mu
+// do not need to be included in every file that uses tensorflow's
+// condition_variable.
+static inline nsync::nsync_cv *cv_cast(
+    condition_variable::external_cv_space *cv) {
+  return reinterpret_cast<nsync::nsync_cv *>(cv);
+}
+
+condition_variable::condition_variable() {
+  nsync::nsync_cv_init(cv_cast(&cv_));
+}
+
+void condition_variable::wait(mutex_lock &lock) {
+  nsync::nsync_cv_wait(cv_cast(&cv_), mu_cast(&lock.mutex()->mu_));
+}
+
+std::cv_status condition_variable::wait_until_system_clock(
+    mutex_lock &lock,
+    const std::chrono::system_clock::time_point timeout_time) {
+  int r = nsync::nsync_cv_wait_with_deadline(
+      cv_cast(&cv_), mu_cast(&lock.mutex()->mu_), timeout_time, nullptr);
+  return r ? std::cv_status::timeout : std::cv_status::no_timeout;
+}
+
+void condition_variable::notify_one() { nsync::nsync_cv_signal(cv_cast(&cv_)); }
+
+void condition_variable::notify_all() {
+  nsync::nsync_cv_broadcast(cv_cast(&cv_));
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/default/mutex.h b/tensorflow/core/platform/default/mutex.h
index 044c754e80..a12d92795e 100644
--- a/tensorflow/core/platform/default/mutex.h
+++ b/tensorflow/core/platform/default/mutex.h
@@ -22,9 +22,8 @@ limitations under the License.
 #include <chrono>
 #include <condition_variable>
 #include <mutex>
-#include "nsync_cv.h"
-#include "nsync_mu.h"
 #include "tensorflow/core/platform/thread_annotations.h"
+
 namespace tensorflow {
 
 #undef mutex_lock
@@ -38,26 +37,26 @@ class condition_variable;
 // lock.
 class LOCKABLE mutex {
  public:
-  mutex() { nsync::nsync_mu_init(&mu_); }
-  // The default implementation of nsync_mutex is safe to use after the linker
-  // initializations
+  mutex();
+  // The default implementation of the underlying mutex is safe to use after
+  // the linker initialization to zero.
   explicit mutex(LinkerInitialized x) {}
 
-  void lock() EXCLUSIVE_LOCK_FUNCTION() { nsync::nsync_mu_lock(&mu_); }
-  bool try_lock() EXCLUSIVE_TRYLOCK_FUNCTION(true) {
-    return nsync::nsync_mu_trylock(&mu_) != 0;
-  };
-  void unlock() UNLOCK_FUNCTION() { nsync::nsync_mu_unlock(&mu_); }
+  void lock() EXCLUSIVE_LOCK_FUNCTION();
+  bool try_lock() EXCLUSIVE_TRYLOCK_FUNCTION(true);
+  void unlock() UNLOCK_FUNCTION();
+
+  void lock_shared() SHARED_LOCK_FUNCTION();
+  bool try_lock_shared() SHARED_TRYLOCK_FUNCTION(true);
+  void unlock_shared() UNLOCK_FUNCTION();
 
-  void lock_shared() SHARED_LOCK_FUNCTION() { nsync::nsync_mu_rlock(&mu_); }
-  bool try_lock_shared() SHARED_TRYLOCK_FUNCTION(true) {
-    return nsync::nsync_mu_rtrylock(&mu_) != 0;
+  struct external_mu_space {
+    void* space[2];
   };
-  void unlock_shared() UNLOCK_FUNCTION() { nsync::nsync_mu_runlock(&mu_); }
 
  private:
   friend class condition_variable;
-  nsync::nsync_mu mu_;
+  external_mu_space mu_;
 };
 
 // Mimic a subset of the std::unique_lock<tensorflow::mutex> functionality.
@@ -139,26 +138,29 @@ class SCOPED_LOCKABLE tf_shared_lock {
 // Mimic std::condition_variable.
 class condition_variable {
  public:
-  condition_variable() { nsync::nsync_cv_init(&cv_); }
+  condition_variable();
 
-  void wait(mutex_lock& lock) {
-    nsync::nsync_cv_wait(&cv_, &lock.mutex()->mu_);
-  }
+  void wait(mutex_lock& lock);
   template <class Rep, class Period>
   std::cv_status wait_for(mutex_lock& lock,
                           std::chrono::duration<Rep, Period> dur) {
-    int r = nsync::nsync_cv_wait_with_deadline(
-        &cv_, &lock.mutex()->mu_, std::chrono::system_clock::now() + dur,
-        nullptr);
-    return r ? std::cv_status::timeout : std::cv_status::no_timeout;
+    return wait_until_system_clock(lock,
+                                   std::chrono::system_clock::now() + dur);
   }
-  void notify_one() { nsync::nsync_cv_signal(&cv_); }
-  void notify_all() { nsync::nsync_cv_broadcast(&cv_); }
+  void notify_one();
+  void notify_all();
+
+  struct external_cv_space {
+    void* space[2];
+  };
 
  private:
   friend ConditionResult WaitForMilliseconds(mutex_lock* mu,
                                              condition_variable* cv, int64 ms);
-  nsync::nsync_cv cv_;
+  std::cv_status wait_until_system_clock(
+      mutex_lock& lock,
+      const std::chrono::system_clock::time_point timeout_time);
+  external_cv_space cv_;
 };
 
 inline ConditionResult WaitForMilliseconds(mutex_lock* mu,
diff --git a/tensorflow/python/platform/sysconfig.py b/tensorflow/python/platform/sysconfig.py
index 5c50fa023d..fdd2b903fc 100644
--- a/tensorflow/python/platform/sysconfig.py
+++ b/tensorflow/python/platform/sysconfig.py
@@ -68,7 +68,6 @@ def get_compile_flags():
   """
   flags = []
   flags.append('-I%s' % get_include())
-  flags.append('-I%s/external/nsync/public' % get_include())
   flags.append('-D_GLIBCXX_USE_CXX11_ABI=%d' % _CXX11_ABI_FLAG)
   return flags
 
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 818d67f7b5..ae305a28e0 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1158,22 +1158,6 @@ def transitive_hdrs(name, deps=[], **kwargs):
 # the libraries in deps.
 def cc_header_only_library(name, deps=[], includes=[], **kwargs):
   _transitive_hdrs(name=name + "_gather", deps=deps)
-
-  # We could generalize the following, but rather than complicate things
-  # here, we'll do the minimal use case for now, and hope bazel comes up
-  # with a better solution before too long.  We'd expect it to compute
-  # the right include path by itself, but it doesn't, possibly because
-  # _transitive_hdrs lost some information about the include path.
-  if "@nsync//:nsync_headers" in deps:
-    # Buiding tensorflow from @org_tensorflow finds this two up.
-    nsynch = "../../external/nsync/public"
-    # Building tensorflow from elsewhere finds it four up.
-    # Note that native.repository_name() is not yet available in TF's Kokoro.
-    if REPOSITORY_NAME != "@":
-      nsynch = "../../" + nsynch
-    includes = includes[:]
-    includes.append(nsynch)
-
   native.cc_library(name=name,
                     hdrs=[":" + name + "_gather"],
                     includes=includes,
@@ -1182,7 +1166,6 @@ def cc_header_only_library(name, deps=[], includes=[], **kwargs):
 def tf_custom_op_library_additional_deps():
   return [
       "@protobuf_archive//:protobuf_headers",
-      "@nsync//:nsync_headers",
       clean_dep("//third_party/eigen3"),
       clean_dep("//tensorflow/core:framework_headers_lib"),
   ]
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 4b6f123daa..e1a5f091ba 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -200,8 +200,7 @@ headers = (list(find_files('*.h', 'tensorflow/core')) +
            list(find_files('*.h', 'tensorflow/stream_executor')) +
            list(find_files('*.h', 'google/protobuf_archive/src')) +
            list(find_files('*', 'third_party/eigen3')) +
-           list(find_files('*', 'external/eigen_archive')) +
-           list(find_files('*.h', 'external/nsync/public')))
+           list(find_files('*', 'external/eigen_archive')))
 
 setup(
     name=project_name,
-- 
GitLab


From 61eab3f8c4ed8bdf4324e99508a104307483da2a Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 14 Mar 2018 14:07:43 -0700
Subject: [PATCH 030/960] Removing underscore prefix from _enter and
 _ref_enter.

PiperOrigin-RevId: 189082871
---
 tensorflow/python/client/session_test.py      |  6 ++--
 tensorflow/python/framework/python_op_gen.cc  |  2 +-
 .../kernel_tests/control_flow_ops_py_test.py  | 30 +++++++++----------
 .../kernel_tests/control_flow_util_test.py    | 10 +++----
 tensorflow/python/ops/control_flow_ops.py     | 10 +++----
 5 files changed, 28 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index ccd7a5117a..781725d63b 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -190,12 +190,10 @@ class SessionTest(test_util.TensorFlowTestCase):
       a = constant_op.constant(0.0, shape=[2, 3])
       # NOTE(mrry): The original_op is nonsense, but used here to test that the
       #   errors are reported correctly.
-      # pylint: disable=protected-access
       with sess.graph._original_op(a.op):
         b = array_ops.identity(a, name='id')
       with sess.graph._original_op(b.op):
         c = array_ops.placeholder(dtypes.float32)
-      # pylint: enable=protected-access
 
       def exc_predicate(e):
         return (e.op == c.op and e.op._original_op == b.op and
@@ -1785,8 +1783,8 @@ class SessionTest(test_util.TensorFlowTestCase):
     # Ensure that errors from building the graph get propagated.
     data = array_ops.placeholder(dtypes.float32, shape=[])
     # pylint: disable=protected-access
-    enter_1 = gen_control_flow_ops._enter(data, 'foo_1', False)
-    enter_2 = gen_control_flow_ops._enter(data, 'foo_2', False)
+    enter_1 = gen_control_flow_ops.enter(data, 'foo_1', False)
+    enter_2 = gen_control_flow_ops.enter(data, 'foo_2', False)
     # pylint: enable=protected-access
     res = math_ops.add(enter_1, enter_2)
     with self.assertRaisesOpError('has inputs from different frames'):
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 271597c459..6ee8e554de 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -100,7 +100,7 @@ bool IsOpWithUnderscorePrefix(const string& s) {
        "fused_batch_norm", "histogram_fixed_width", "stack",
        "batch_norm_with_global_normalization",
        // TODO(annarev): replace these ops in the next change.
-       "broadcast_gradient_args", "enter", "ref_enter", "ref_identity"});
+       "broadcast_gradient_args", "ref_identity"});
   return kUnderscoreOps->count(s) > 0;
 }
 
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index b429fa5c42..d47b030fa1 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -144,7 +144,7 @@ class ControlFlowTest(test.TestCase):
 
       enter_v = control_flow_ops._Enter(v, "foo_1", is_constant=True)
       nine = constant_op.constant(9)
-      enter_nine = gen_control_flow_ops._enter(nine, "foo_1")
+      enter_nine = gen_control_flow_ops.enter(nine, "foo_1")
       op = state_ops.assign(enter_v, enter_nine)
       v2 = control_flow_ops.with_dependencies([op], enter_v)
       v3 = control_flow_ops.exit(v2)
@@ -164,9 +164,9 @@ class ControlFlowTest(test.TestCase):
   def testEnterMulExit(self):
     with self.test_session():
       data = constant_op.constant([1, 2, 3, 4, 5, 6], name="data")
-      enter_data = gen_control_flow_ops._enter(data, "foo_1", False)
+      enter_data = gen_control_flow_ops.enter(data, "foo_1", False)
       five = constant_op.constant(5)
-      enter_five = gen_control_flow_ops._enter(five, "foo_1", False)
+      enter_five = gen_control_flow_ops.enter(five, "foo_1", False)
       mul_op = math_ops.multiply(enter_data, enter_five)
       exit_op = control_flow_ops.exit(mul_op)
 
@@ -178,12 +178,12 @@ class ControlFlowTest(test.TestCase):
       v = variables.Variable([0.0, 0.0], dtype=dtypes.float32)
 
       # If is_constant=True, the shape information should be propagated.
-      enter_v_constant = gen_control_flow_ops._enter(
+      enter_v_constant = gen_control_flow_ops.enter(
           v, "frame1", is_constant=True)
       self.assertEqual(enter_v_constant.shape, [2])
 
       # Otherwise, the shape should be unknown.
-      enter_v_non_constant = gen_control_flow_ops._enter(
+      enter_v_non_constant = gen_control_flow_ops.enter(
           v, "frame2", is_constant=False)
       self.assertEqual(enter_v_non_constant.shape, None)
 
@@ -257,8 +257,8 @@ class ControlFlowTest(test.TestCase):
       false = ops.convert_to_tensor(False)
       n = constant_op.constant(10)
 
-      enter_false = gen_control_flow_ops._enter(false, "foo_1", False)
-      enter_n = gen_control_flow_ops._enter(n, "foo_1", False)
+      enter_false = gen_control_flow_ops.enter(false, "foo_1", False)
+      enter_n = gen_control_flow_ops.enter(n, "foo_1", False)
 
       merge_n = control_flow_ops.merge([enter_n, enter_n], name="merge_n")[0]
       switch_n = control_flow_ops.switch(merge_n, enter_false)
@@ -275,9 +275,9 @@ class ControlFlowTest(test.TestCase):
       one = constant_op.constant(1)
       n = constant_op.constant(10)
 
-      enter_i = gen_control_flow_ops._enter(zero, "foo", False)
-      enter_one = gen_control_flow_ops._enter(one, "foo", True)
-      enter_n = gen_control_flow_ops._enter(n, "foo", True)
+      enter_i = gen_control_flow_ops.enter(zero, "foo", False)
+      enter_one = gen_control_flow_ops.enter(one, "foo", True)
+      enter_n = gen_control_flow_ops.enter(n, "foo", True)
 
       with ops.device(test.gpu_device_name()):
         merge_i = control_flow_ops.merge([enter_i, enter_i])[0]
@@ -301,9 +301,9 @@ class ControlFlowTest(test.TestCase):
       one = constant_op.constant(1)
       n = constant_op.constant(10)
 
-      enter_i = gen_control_flow_ops._enter(zero, "foo", False)
-      enter_one = gen_control_flow_ops._enter(one, "foo", True)
-      enter_n = gen_control_flow_ops._enter(n, "foo", True)
+      enter_i = gen_control_flow_ops.enter(zero, "foo", False)
+      enter_one = gen_control_flow_ops.enter(one, "foo", True)
+      enter_n = gen_control_flow_ops.enter(n, "foo", True)
 
       merge_i = control_flow_ops.merge([enter_i, enter_i])[0]
 
@@ -324,8 +324,8 @@ class ControlFlowTest(test.TestCase):
   def testDifferentFrame(self):
     with self.test_session():
       data = array_ops.placeholder(dtypes.float32, shape=[])
-      enter_1 = gen_control_flow_ops._enter(data, "foo_1", False)
-      enter_2 = gen_control_flow_ops._enter(data, "foo_2", False)
+      enter_1 = gen_control_flow_ops.enter(data, "foo_1", False)
+      enter_2 = gen_control_flow_ops.enter(data, "foo_2", False)
       res = math_ops.add(enter_1, enter_2)
       with self.assertRaisesOpError("has inputs from different frames"):
         res.eval(feed_dict={data: 1.0})
diff --git a/tensorflow/python/kernel_tests/control_flow_util_test.py b/tensorflow/python/kernel_tests/control_flow_util_test.py
index 23185eaeec..39e96f74b0 100644
--- a/tensorflow/python/kernel_tests/control_flow_util_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_util_test.py
@@ -41,17 +41,17 @@ class ControlFlowUtilTest(test.TestCase):
     self.assertFalse(control_flow_util.IsSwitch(test_ops.int_output().op))
 
   def testIsLoopEnter(self):
-    enter = gen_control_flow_ops._enter(1, frame_name="name").op
+    enter = gen_control_flow_ops.enter(1, frame_name="name").op
     self.assertTrue(control_flow_util.IsLoopEnter(enter))
     self.assertFalse(control_flow_util.IsLoopConstantEnter(enter))
 
-    ref_enter = gen_control_flow_ops._ref_enter(test_ops.ref_output(),
-                                                frame_name="name").op
+    ref_enter = gen_control_flow_ops.ref_enter(test_ops.ref_output(),
+                                               frame_name="name").op
     self.assertTrue(control_flow_util.IsLoopEnter(ref_enter))
     self.assertFalse(control_flow_util.IsLoopConstantEnter(ref_enter))
 
-    const_enter = gen_control_flow_ops._enter(1, frame_name="name",
-                                              is_constant=True).op
+    const_enter = gen_control_flow_ops.enter(1, frame_name="name",
+                                             is_constant=True).op
     self.assertTrue(control_flow_util.IsLoopEnter(const_enter))
     self.assertTrue(control_flow_util.IsLoopConstantEnter(const_enter))
 
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index ff4f452bed..24c30802b5 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -264,10 +264,10 @@ def _Enter(data,
   data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype and use_ref:  # pylint: disable=protected-access
-      result = gen_control_flow_ops._ref_enter(
+      result = gen_control_flow_ops.ref_enter(
           data, frame_name, is_constant, parallel_iterations, name=name)
     else:
-      result = gen_control_flow_ops._enter(
+      result = gen_control_flow_ops.enter(
           data, frame_name, is_constant, parallel_iterations, name=name)
     if use_input_shape:
       result.set_shape(data.get_shape())
@@ -282,7 +282,7 @@ def _Enter(data,
         parallel_iterations=parallel_iterations,
         use_input_shape=use_input_shape,
         name=name)
-    indices = gen_control_flow_ops._enter(
+    indices = gen_control_flow_ops.enter(
         data.indices,
         frame_name,
         is_constant,
@@ -293,7 +293,7 @@ def _Enter(data,
     if isinstance(data, ops.IndexedSlices):
       dense_shape = data.dense_shape
       if dense_shape is not None:
-        dense_shape = gen_control_flow_ops._enter(
+        dense_shape = gen_control_flow_ops.enter(
             dense_shape,
             frame_name,
             is_constant,
@@ -303,7 +303,7 @@ def _Enter(data,
           dense_shape.set_shape(data.dense_shape.get_shape())
       return ops.IndexedSlices(values, indices, dense_shape)
     else:
-      dense_shape = gen_control_flow_ops._enter(
+      dense_shape = gen_control_flow_ops.enter(
           data.dense_shape,
           frame_name,
           is_constant,
-- 
GitLab


From ac8ce1fe760efff6585d790b784ec67255198879 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 14:34:15 -0700
Subject: [PATCH 031/960] Rename KMeans _parse_tensor_or_dict to
 _parse_features_if_necessary and add a unit test.

PiperOrigin-RevId: 189087384
---
 .../factorization/python/ops/kmeans.py        |  4 ++--
 .../factorization/python/ops/kmeans_test.py   | 23 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
index 7319eaa7de..c092f85d35 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -105,7 +105,7 @@ class _InitializeClustersHook(session_run_hook.SessionRunHook):
         logging.info(e)
 
 
-def _parse_tensor_or_dict(features):
+def _parse_features_if_necessary(features):
   """Helper function to convert the input points into a usable format.
 
   Args:
@@ -166,7 +166,7 @@ class _ModelFn(object):
     # input_points is a single Tensor. Therefore, the sharding functionality
     # in clustering_ops is unused, and some of the values below are lists of a
     # single item.
-    input_points = _parse_tensor_or_dict(features)
+    input_points = _parse_features_if_necessary(features)
 
     # Let N = the number of input_points.
     # all_distances: A list of one matrix of shape (N, num_clusters). Each value
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index f9598bfc08..06a2c52c11 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -226,6 +226,28 @@ class KMeansTest(KMeansTestBase):
     self._infer_helper(kmeans, clusters, 10)
     self._infer_helper(kmeans, clusters, 1)
 
+  def test_parse_features(self):
+    """Tests the various behaviours of kmeans._parse_features_if_necessary."""
+
+    # No-op if a tensor is passed in.
+    features = constant_op.constant(self.points)
+    parsed_features = kmeans_lib._parse_features_if_necessary(features)
+    self.assertAllEqual(features, parsed_features)
+
+    # A dict is transformed into a tensor.
+    feature_dict = {
+        'x': [[point[0]] for point in self.points],
+        'y': [[point[1]] for point in self.points]
+    }
+    parsed_feature_dict = kmeans_lib._parse_features_if_necessary(feature_dict)
+    # Perform a sanity check.
+    self.assertEqual(features.shape, parsed_feature_dict.shape)
+    self.assertEqual(features.dtype, parsed_feature_dict.dtype)
+    # Then check that running the tensor yields the original list of points.
+    with self.test_session() as sess:
+      parsed_points = sess.run(parsed_feature_dict)
+      self.assertAllEqual(self.points, parsed_points)
+
 
 class KMeansTestMultiStageInit(KMeansTestBase):
 
@@ -394,7 +416,6 @@ class KMeansCosineDistanceTest(KMeansTestBase):
     true_assignments = [0] * 2 + [1] * 2 + [2] * 8
     true_score = len(points) - np.tensordot(
         normalize(points), true_centers[true_assignments])
-
     kmeans = kmeans_lib.KMeansClustering(
         3,
         initial_clusters=self.initial_clusters,
-- 
GitLab


From 5ac329bd86e400d47155e0c890669f4ee688771d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 14:47:40 -0700
Subject: [PATCH 032/960] Adding non-linear image warping ops to
 tf.contrib.image

New ops are:
tf.contrib.image.sparse_image_warp, tf.contrib.image.dense_image_warp, and tf.contrib.image.interpolate_spline.

PiperOrigin-RevId: 189089672
---
 tensorflow/contrib/image/BUILD                | 109 +++++++
 tensorflow/contrib/image/__init__.py          |   7 +
 .../kernel_tests/dense_image_warp_test.py     | 264 ++++++++++++++++
 .../kernel_tests/interpolate_spline_test.py   | 261 ++++++++++++++++
 .../kernel_tests/sparse_image_warp_test.py    | 251 +++++++++++++++
 .../test_data/Yellow_Smiley_Face.png          | Bin 0 -> 14060 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-0.png | Bin 0 -> 18537 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-1.png | Bin 0 -> 19086 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-4.png | Bin 0 -> 18884 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-0.png | Bin 0 -> 18109 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-1.png | Bin 0 -> 19251 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-4.png | Bin 0 -> 19132 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-0.png | Bin 0 -> 17500 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-1.png | Bin 0 -> 18058 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-4.png | Bin 0 -> 19313 bytes
 .../image/python/ops/dense_image_warp.py      | 196 ++++++++++++
 .../image/python/ops/interpolate_spline.py    | 285 ++++++++++++++++++
 .../image/python/ops/sparse_image_warp.py     | 192 ++++++++++++
 18 files changed, 1565 insertions(+)
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/ops/dense_image_warp.py
 create mode 100644 tensorflow/contrib/image/python/ops/interpolate_spline.py
 create mode 100644 tensorflow/contrib/image/python/ops/sparse_image_warp.py

diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 3ff02e085e..760ed70fbb 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -78,7 +78,10 @@ tf_custom_op_py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":dense_image_warp_py",
         ":image_ops",
+        ":interpolate_spline_py",
+        ":sparse_image_warp_py",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:common_shapes",
@@ -194,6 +197,112 @@ cuda_py_test(
     ],
 )
 
+py_library(
+    name = "dense_image_warp_py",
+    srcs = [
+        "python/ops/dense_image_warp.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_library(
+    name = "interpolate_spline_py",
+    srcs = [
+        "python/ops/interpolate_spline.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+    ],
+)
+
+py_library(
+    name = "sparse_image_warp_py",
+    srcs = [
+        "python/ops/sparse_image_warp.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dense_image_warp_py",
+        ":interpolate_spline_py",
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+    ],
+)
+
+cuda_py_test(
+    name = "sparse_image_warp_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/sparse_image_warp_test.py"],
+    additional_deps = [
+        ":sparse_image_warp_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+    ],
+    data = glob(["python/kernel_tests/test_data/*.png"]),
+)
+
+cuda_py_test(
+    name = "dense_image_warp_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/dense_image_warp_test.py"],
+    additional_deps = [
+        ":dense_image_warp_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
+cuda_py_test(
+    name = "interpolate_spline_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/interpolate_spline_test.py"],
+    additional_deps = [
+        ":interpolate_spline_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+        "//third_party/py/scipy",
+    ],
+)
+
 tf_py_test(
     name = "segmentation_test",
     size = "medium",
diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index cc8ed117ba..e982030bc8 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -30,6 +30,9 @@ projective transforms (including rotation) are supported.
 @@transform
 @@translate
 @@translations_to_projective_transforms
+@@dense_image_warp
+@@interpolate_spline
+@@sparse_image_warp
 
 ## Image Segmentation `Ops`
 
@@ -47,6 +50,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.image.python.ops.dense_image_warp import dense_image_warp
+
 from tensorflow.contrib.image.python.ops.distort_image_ops import adjust_hsv_in_yiq
 from tensorflow.contrib.image.python.ops.distort_image_ops import random_hsv_in_yiq
 
@@ -57,7 +62,9 @@ from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
 from tensorflow.contrib.image.python.ops.image_ops import translate
 from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
+from tensorflow.contrib.image.python.ops.interpolate_spline import interpolate_spline
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
+from tensorflow.contrib.image.python.ops.sparse_image_warp import sparse_image_warp
 
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
new file mode 100644
index 0000000000..24d99ccaa6
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
@@ -0,0 +1,264 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for dense_image_warp."""
+
+import math
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import dense_image_warp
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+from tensorflow.python.training import adam
+
+
+class DenseImageWarpTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    np.random.seed(0)
+
+  def test_interpolate_small_grid_ij(self):
+    grid = constant_op.constant(
+        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
+    query_points = constant_op.constant(
+        [[0., 0.], [1., 0.], [2., 0.5], [1.5, 1.5]], shape=[1, 4, 2])
+    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def test_interpolate_small_grid_xy(self):
+    grid = constant_op.constant(
+        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
+    query_points = constant_op.constant(
+        [[0., 0.], [0., 1.], [0.5, 2.0], [1.5, 1.5]], shape=[1, 4, 2])
+    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(
+        grid, query_points, indexing='xy')
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def test_interpolate_small_grid_batched(self):
+    grid = constant_op.constant(
+        [[[0., 1.], [3., 4.]], [[5., 6.], [7., 8.]]], shape=[2, 2, 2, 1])
+    query_points = constant_op.constant([[[0., 0.], [1., 0.], [0.5, 0.5]],
+                                         [[0.5, 0.], [1., 0.], [1., 1.]]])
+    expected_results = np.reshape(
+        np.array([[0., 3., 2.], [6., 7., 8.]]), [2, 3, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def get_image_and_flow_placeholders(self, shape, image_type, flow_type):
+    batch_size, height, width, numchannels = shape
+    image_shape = [batch_size, height, width, numchannels]
+    flow_shape = [batch_size, height, width, 2]
+
+    tf_type = {
+        'float16': dtypes.half,
+        'float32': dtypes.float32,
+        'float64': dtypes.float64
+    }
+
+    image = array_ops.placeholder(dtype=tf_type[image_type], shape=image_shape)
+
+    flows = array_ops.placeholder(dtype=tf_type[flow_type], shape=flow_shape)
+    return image, flows
+
+  def get_random_image_and_flows(self, shape, image_type, flow_type):
+    batch_size, height, width, numchannels = shape
+    image_shape = [batch_size, height, width, numchannels]
+    image = np.random.normal(size=image_shape)
+    flow_shape = [batch_size, height, width, 2]
+    flows = np.random.normal(size=flow_shape) * 3
+    return image.astype(image_type), flows.astype(flow_type)
+
+  def assert_correct_interpolation_value(self,
+                                         image,
+                                         flows,
+                                         pred_interpolation,
+                                         batch_index,
+                                         y_index,
+                                         x_index,
+                                         low_precision=False):
+    """Assert that the tf interpolation matches hand-computed value."""
+
+    height = image.shape[1]
+    width = image.shape[2]
+    displacement = flows[batch_index, y_index, x_index, :]
+    float_y = y_index - displacement[0]
+    float_x = x_index - displacement[1]
+    floor_y = max(min(height - 2, math.floor(float_y)), 0)
+    floor_x = max(min(width - 2, math.floor(float_x)), 0)
+    ceil_y = floor_y + 1
+    ceil_x = floor_x + 1
+
+    alpha_y = min(max(0.0, float_y - floor_y), 1.0)
+    alpha_x = min(max(0.0, float_x - floor_x), 1.0)
+
+    floor_y = int(floor_y)
+    floor_x = int(floor_x)
+    ceil_y = int(ceil_y)
+    ceil_x = int(ceil_x)
+
+    top_left = image[batch_index, floor_y, floor_x, :]
+    top_right = image[batch_index, floor_y, ceil_x, :]
+    bottom_left = image[batch_index, ceil_y, floor_x, :]
+    bottom_right = image[batch_index, ceil_y, ceil_x, :]
+
+    interp_top = alpha_x * (top_right - top_left) + top_left
+    interp_bottom = alpha_x * (bottom_right - bottom_left) + bottom_left
+    interp = alpha_y * (interp_bottom - interp_top) + interp_top
+    atol = 1e-6
+    rtol = 1e-6
+    if low_precision:
+      atol = 1e-2
+      rtol = 1e-3
+    self.assertAllClose(
+        interp,
+        pred_interpolation[batch_index, y_index, x_index, :],
+        atol=atol,
+        rtol=rtol)
+
+  def check_zero_flow_correctness(self, shape, image_type, flow_type):
+    """Assert using zero flows doesn't change the input image."""
+
+    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
+                                                        flow_type)
+    interp = dense_image_warp.dense_image_warp(image, flows)
+
+    with self.test_session() as sess:
+      rand_image, rand_flows = self.get_random_image_and_flows(
+          shape, image_type, flow_type)
+      rand_flows *= 0
+
+      predicted_interpolation = sess.run(
+          interp, feed_dict={
+              image: rand_image,
+              flows: rand_flows
+          })
+      self.assertAllClose(rand_image, predicted_interpolation)
+
+  def test_zero_flows(self):
+    """Apply check_zero_flow_correctness() for a few sizes and types."""
+
+    shapes_to_try = [[3, 4, 5, 6], [1, 2, 2, 1]]
+    for shape in shapes_to_try:
+      self.check_zero_flow_correctness(
+          shape, image_type='float32', flow_type='float32')
+
+  def check_interpolation_correctness(self,
+                                      shape,
+                                      image_type,
+                                      flow_type,
+                                      num_probes=5):
+    """Interpolate, and then assert correctness for a few query locations."""
+
+    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
+                                                        flow_type)
+    interp = dense_image_warp.dense_image_warp(image, flows)
+    low_precision = image_type == 'float16' or flow_type == 'float16'
+    with self.test_session() as sess:
+      rand_image, rand_flows = self.get_random_image_and_flows(
+          shape, image_type, flow_type)
+
+      pred_interpolation = sess.run(
+          interp, feed_dict={
+              image: rand_image,
+              flows: rand_flows
+          })
+
+      for _ in range(num_probes):
+        batch_index = np.random.randint(0, shape[0])
+        y_index = np.random.randint(0, shape[1])
+        x_index = np.random.randint(0, shape[2])
+
+        self.assert_correct_interpolation_value(
+            rand_image,
+            rand_flows,
+            pred_interpolation,
+            batch_index,
+            y_index,
+            x_index,
+            low_precision=low_precision)
+
+  def test_interpolation(self):
+    """Apply check_interpolation_correctness() for a few sizes and types."""
+
+    shapes_to_try = [[3, 4, 5, 6], [1, 5, 5, 3], [1, 2, 2, 1]]
+    for im_type in ['float32', 'float64', 'float16']:
+      for flow_type in ['float32', 'float64', 'float16']:
+        for shape in shapes_to_try:
+          self.check_interpolation_correctness(shape, im_type, flow_type)
+
+  def test_gradients_exist(self):
+    """Check that backprop can run.
+
+    The correctness of the gradients is assumed, since the forward propagation
+    is tested to be correct and we only use built-in tf ops.
+    However, we perform a simple test to make sure that backprop can actually
+    run. We treat the flows as a tf.Variable and optimize them to minimize
+    the difference between the interpolated image and the input image.
+    """
+
+    batch_size, height, width, numchannels = [4, 5, 6, 7]
+    image_shape = [batch_size, height, width, numchannels]
+    image = random_ops.random_normal(image_shape)
+    flow_shape = [batch_size, height, width, 2]
+    init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25)
+    flows = variables.Variable(init_flows)
+
+    interp = dense_image_warp.dense_image_warp(image, flows)
+    loss = math_ops.reduce_mean(math_ops.square(interp - image))
+
+    optimizer = adam.AdamOptimizer(1.0)
+    grad = gradients.gradients(loss, [flows])
+    opt_func = optimizer.apply_gradients(zip(grad, [flows]))
+    init_op = variables.global_variables_initializer()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(10):
+        sess.run(opt_func)
+
+  def test_size_exception(self):
+    """Make sure it throws an exception for images that are too small."""
+
+    shape = [1, 2, 1, 1]
+    msg = 'Should have raised an exception for invalid image size'
+    with self.assertRaises(ValueError, msg=msg):
+      self.check_interpolation_correctness(shape, 'float32', 'float32')
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
new file mode 100644
index 0000000000..1cba46e17e
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
@@ -0,0 +1,261 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for interpolate_spline."""
+
+import numpy as np
+from scipy import interpolate as sc_interpolate
+
+from tensorflow.contrib.image.python.ops import interpolate_spline
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+from tensorflow.python.training import momentum
+
+
+class _InterpolationProblem(object):
+  """Abstract class for interpolation problem descriptions."""
+
+  def get_problem(self, optimizable=False, extrapolate=True, dtype='float32'):
+    """Make data for an interpolation problem where all x vectors are n-d.
+
+    Args:
+      optimizable: If True, then make train_points a tf.Variable.
+      extrapolate: If False, then clamp the query_points values to be within
+      the max and min of train_points.
+      dtype: The data type to use.
+
+    Returns:
+      query_points, query_values, train_points, train_values: training and
+      test tensors for interpolation problem
+    """
+
+    # The values generated here depend on a seed of 0.
+    np.random.seed(0)
+
+    batch_size = 1
+    num_training_points = 10
+    num_query_points = 4
+
+    init_points = np.random.uniform(
+        size=[batch_size, num_training_points, self.DATA_DIM])
+
+    init_points = init_points.astype(dtype)
+    train_points = (
+        variables.Variable(init_points)
+        if optimizable else constant_op.constant(init_points))
+    train_values = self.tf_function(train_points)
+
+    query_points_np = np.random.uniform(
+        size=[batch_size, num_query_points, self.DATA_DIM])
+    query_points_np = query_points_np.astype(dtype)
+    if not extrapolate:
+      query_points_np = np.clip(query_points_np, np.min(init_points),
+                                np.max(init_points))
+
+    query_points = constant_op.constant(query_points_np)
+    query_values = self.np_function(query_points_np)
+
+    return query_points, query_values, train_points, train_values
+
+
+class _QuadraticPlusSinProblem1D(_InterpolationProblem):
+  """1D interpolation problem used for regression testing."""
+  DATA_DIM = 1
+  HARDCODED_QUERY_VALUES = {
+      (1.0, 0.0): [6.2647187603, -7.84362604077, -5.63690142322, 1.42928896387],
+      (1.0,
+       0.01): [6.77688289946, -8.02163669853, -5.79491157027, 1.4063285693],
+      (2.0,
+       0.0): [8.67110264937, -8.41281390883, -5.80190044693, 1.50155606059],
+      (2.0,
+       0.01): [6.70797816797, -7.49709587663, -5.28965776238, 1.52284731741],
+      (3.0,
+       0.0): [9.37691802935, -8.50390141515, -5.80786417426, 1.63467762122],
+      (3.0,
+       0.01): [4.47106304758, -5.71266128361, -3.92529303296, 1.86755293857],
+      (4.0,
+       0.0): [9.58172461111, -8.51432104771, -5.80967675388, 1.63361164256],
+      (4.0, 0.01): [
+          -3.87902711352, -0.0253462273846, 1.79857618022, -0.769339675725
+      ]
+  }
+
+  def np_function(self, x):
+    """Takes np array, evaluates the test function, and returns np array."""
+    return np.sum(
+        np.power((x - 0.5), 3) - 0.25 * x + 10 * np.sin(x * 10),
+        axis=2,
+        keepdims=True)
+
+  def tf_function(self, x):
+    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
+    return math_ops.reduce_mean(
+        math_ops.pow((x - 0.5), 3) - 0.25 * x + 10 * math_ops.sin(x * 10),
+        2,
+        keepdims=True)
+
+
+class _QuadraticPlusSinProblemND(_InterpolationProblem):
+  """3D interpolation problem used for regression testing."""
+
+  DATA_DIM = 3
+  HARDCODED_QUERY_VALUES = {
+      (1.0, 0.0): [1.06609663962, 1.28894849357, 1.10882405595, 1.63966936885],
+      (1.0, 0.01): [1.03123780748, 1.2952930985, 1.10366822954, 1.65265118569],
+      (2.0, 0.0): [0.627787735064, 1.43802857251, 1.00194632358, 1.91667538215],
+      (2.0, 0.01): [0.730159985046, 1.41702471595, 1.0065827217, 1.85758519312],
+      (3.0, 0.0): [0.350460417862, 1.67223539464, 1.00475331246, 2.31580322491],
+      (3.0,
+       0.01): [0.624557250556, 1.63138876667, 0.976588193162, 2.12511237866],
+      (4.0,
+       0.0): [0.898129669986, 1.24434133638, -0.938056116931, 1.59910338833],
+      (4.0,
+       0.01): [0.0930360338179, -3.38791305538, -1.00969032567, 0.745535080382],
+  }
+
+  def np_function(self, x):
+    """Takes np array, evaluates the test function, and returns np array."""
+    return np.sum(
+        np.square(x - 0.5) + 0.25 * x + 1 * np.sin(x * 15),
+        axis=2,
+        keepdims=True)
+
+  def tf_function(self, x):
+    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
+    return math_ops.reduce_sum(
+        math_ops.square(x - 0.5) + 0.25 * x + 1 * math_ops.sin(x * 15),
+        2,
+        keepdims=True)
+
+
+class InterpolateSplineTest(test_util.TensorFlowTestCase):
+
+  def test_1d_linear_interpolation(self):
+    """For 1d linear interpolation, we can compare directly to scipy."""
+
+    tp = _QuadraticPlusSinProblem1D()
+    (query_points, _, train_points, train_values) = tp.get_problem(
+        extrapolate=False, dtype='float64')
+    interpolation_order = 1
+
+    with ops.name_scope('interpolator'):
+      interpolator = interpolate_spline.interpolate_spline(
+          train_points, train_values, query_points, interpolation_order)
+      with self.test_session() as sess:
+        fetches = [query_points, train_points, train_values, interpolator]
+        query_points_, train_points_, train_values_, interp_ = sess.run(fetches)
+
+        # Just look at the first element of the minibatch.
+        # Also, trim the final singleton dimension.
+        interp_ = interp_[0, :, 0]
+        query_points_ = query_points_[0, :, 0]
+        train_points_ = train_points_[0, :, 0]
+        train_values_ = train_values_[0, :, 0]
+
+        # Compute scipy interpolation.
+        scipy_interp_function = sc_interpolate.interp1d(
+            train_points_, train_values_, kind='linear')
+
+        scipy_interpolation = scipy_interp_function(query_points_)
+        scipy_interpolation_on_train = scipy_interp_function(train_points_)
+
+        # Even with float64 precision, the interpolants disagree with scipy a
+        # bit due to the fact that we add the EPSILON to prevent sqrt(0), etc.
+        tol = 1e-3
+
+        self.assertAllClose(
+            train_values_, scipy_interpolation_on_train, atol=tol, rtol=tol)
+        self.assertAllClose(interp_, scipy_interpolation, atol=tol, rtol=tol)
+
+  def test_1d_interpolation(self):
+    """Regression test for interpolation with 1-D points."""
+
+    tp = _QuadraticPlusSinProblem1D()
+    (query_points, _, train_points,
+     train_values) = tp.get_problem(dtype='float64')
+
+    for order in (1, 2, 3):
+      for reg_weight in (0, 0.01):
+        interpolator = interpolate_spline.interpolate_spline(
+            train_points, train_values, query_points, order, reg_weight)
+
+        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
+        target_interpolation = np.array(target_interpolation)
+        with self.test_session() as sess:
+          interp_val = sess.run(interpolator)
+          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
+
+  def test_nd_linear_interpolation(self):
+    """Regression test for interpolation with N-D points."""
+
+    tp = _QuadraticPlusSinProblemND()
+    (query_points, _, train_points,
+     train_values) = tp.get_problem(dtype='float64')
+
+    for order in (1, 2, 3):
+      for reg_weight in (0, 0.01):
+        interpolator = interpolate_spline.interpolate_spline(
+            train_points, train_values, query_points, order, reg_weight)
+
+        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
+        target_interpolation = np.array(target_interpolation)
+        with self.test_session() as sess:
+          interp_val = sess.run(interpolator)
+          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
+
+  def test_interpolation_gradient(self):
+    """Make sure that backprop can run. Correctness of gradients is assumed.
+
+    Here, we create a use a small 'training' set and a more densely-sampled
+    set of query points, for which we know the true value in advance. The goal
+    is to choose x locations for the training data such that interpolating using
+    this training data yields the best reconstruction for the function
+    values at the query points. The training data locations are optimized
+    iteratively using gradient descent.
+    """
+    tp = _QuadraticPlusSinProblemND()
+    (query_points, query_values, train_points,
+     train_values) = tp.get_problem(optimizable=True)
+
+    regularization = 0.001
+    for interpolation_order in (1, 2, 3, 4):
+      interpolator = interpolate_spline.interpolate_spline(
+          train_points, train_values, query_points, interpolation_order,
+          regularization)
+
+      loss = math_ops.reduce_mean(math_ops.square(query_values - interpolator))
+
+      optimizer = momentum.MomentumOptimizer(0.001, 0.9)
+      grad = gradients.gradients(loss, [train_points])
+      grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
+      opt_func = optimizer.apply_gradients(zip(grad, [train_points]))
+      init_op = variables.global_variables_initializer()
+
+      with self.test_session() as sess:
+        sess.run(init_op)
+        for _ in range(100):
+          sess.run([loss, opt_func])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
new file mode 100644
index 0000000000..017969d230
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
@@ -0,0 +1,251 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for sparse_image_warp."""
+
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import sparse_image_warp
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import image_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.platform import test
+
+from tensorflow.python.training import momentum
+
+
+class SparseImageWarpTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    np.random.seed(0)
+
+  def testGetBoundaryLocations(self):
+    image_height = 11
+    image_width = 11
+    num_points_per_edge = 4
+    locs = sparse_image_warp._get_boundary_locations(image_height, image_width,
+                                                     num_points_per_edge)
+    num_points = locs.shape[0]
+    self.assertEqual(num_points, 4 + 4 * num_points_per_edge)
+    locs = [(locs[i, 0], locs[i, 1]) for i in range(num_points)]
+    for i in (0, image_height - 1):
+      for j in (0, image_width - 1):
+        self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+      for i in (2, 4, 6, 8):
+        for j in (0, image_width - 1):
+          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+      for i in (0, image_height - 1):
+        for j in (2, 4, 6, 8):
+          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+  def testGetGridLocations(self):
+    image_height = 5
+    image_width = 3
+    grid = sparse_image_warp._get_grid_locations(image_height, image_width)
+    for i in range(image_height):
+      for j in range(image_width):
+        self.assertEqual(grid[i, j, 0], i)
+        self.assertEqual(grid[i, j, 1], j)
+
+  def testZeroShift(self):
+    """Run assertZeroShift for various hyperparameters."""
+    for order in (1, 2):
+      for regularization in (0, 0.01):
+        for num_boundary_points in (0, 1):
+          self.assertZeroShift(order, regularization, num_boundary_points)
+
+  def assertZeroShift(self, order, regularization, num_boundary_points):
+    """Check that warping with zero displacements doesn't change the image."""
+    batch_size = 1
+    image_height = 4
+    image_width = 4
+    channels = 3
+
+    image = np.random.uniform(
+        size=[batch_size, image_height, image_width, channels])
+
+    input_image_op = constant_op.constant(np.float32(image))
+
+    control_point_locations = [[1., 1.], [2., 2.], [2., 1.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)))
+
+    control_point_displacements = np.zeros(
+        control_point_locations.shape.as_list())
+    control_point_displacements = constant_op.constant(
+        np.float32(control_point_displacements))
+
+    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
+        input_image_op,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        interpolation_order=order,
+        regularization_weight=regularization,
+        num_boundary_points=num_boundary_points)
+
+    with self.test_session() as sess:
+      warped_image, input_image, _ = sess.run(
+          [warped_image_op, input_image_op, flow_field])
+
+      self.assertAllClose(warped_image, input_image)
+
+  def testMoveSinglePixel(self):
+    """Run assertMoveSinglePixel for various hyperparameters and data types."""
+    for order in (1, 2):
+      for num_boundary_points in (1, 2):
+        for type_to_use in (dtypes.float32, dtypes.float64):
+          self.assertMoveSinglePixel(order, num_boundary_points, type_to_use)
+
+  def assertMoveSinglePixel(self, order, num_boundary_points, type_to_use):
+    """Move a single block in a small grid using warping."""
+    batch_size = 1
+    image_height = 7
+    image_width = 7
+    channels = 3
+
+    image = np.zeros([batch_size, image_height, image_width, channels])
+    image[:, 3, 3, :] = 1.0
+    input_image_op = constant_op.constant(image, dtype=type_to_use)
+
+    # Place a control point at the one white pixel.
+    control_point_locations = [[3., 3.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)),
+        dtype=type_to_use)
+    # Shift it one pixel to the right.
+    control_point_displacements = [[0., 1.0]]
+    control_point_displacements = constant_op.constant(
+        np.float32(np.expand_dims(control_point_displacements, 0)),
+        dtype=type_to_use)
+
+    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
+        input_image_op,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        interpolation_order=order,
+        num_boundary_points=num_boundary_points)
+
+    with self.test_session() as sess:
+      warped_image, input_image, flow = sess.run(
+          [warped_image_op, input_image_op, flow_field])
+      # Check that it moved the pixel correctly.
+      self.assertAllClose(
+          warped_image[0, 4, 5, :],
+          input_image[0, 4, 4, :],
+          atol=1e-5,
+          rtol=1e-5)
+
+      # Test that there is no flow at the corners.
+      for i in (0, image_height - 1):
+        for j in (0, image_width - 1):
+          self.assertAllClose(
+              flow[0, i, j, :], np.zeros([2]), atol=1e-5, rtol=1e-5)
+
+  def load_image(self, image_file, sess):
+    image_op = image_ops.decode_png(
+        io_ops.read_file(image_file), dtype=dtypes.uint8, channels=4)[:, :, 0:3]
+    return sess.run(image_op)
+
+  def testSmileyFace(self):
+    """Check warping accuracy by comparing to hardcoded warped images."""
+
+    test_data_dir = test.test_src_dir_path('contrib/image/python/'
+                                           'kernel_tests/test_data/')
+    input_file = test_data_dir + 'Yellow_Smiley_Face.png'
+    with self.test_session() as sess:
+      input_image = self.load_image(input_file, sess)
+    control_points = np.asarray([[64, 59], [180 - 64, 59], [39, 111],
+                                 [180 - 39, 111], [90, 143], [58, 134],
+                                 [180 - 58, 134]])  # pyformat: disable
+    control_point_displacements = np.asarray(
+        [[-10.5, 10.5], [10.5, 10.5], [0, 0], [0, 0], [0, -10], [-20, 10.25],
+         [10, 10.75]])
+    control_points_op = constant_op.constant(
+        np.expand_dims(np.float32(control_points[:, [1, 0]]), 0))
+    control_point_displacements_op = constant_op.constant(
+        np.expand_dims(np.float32(control_point_displacements[:, [1, 0]]), 0))
+    float_image = np.expand_dims(np.float32(input_image) / 255, 0)
+    input_image_op = constant_op.constant(float_image)
+
+    for interpolation_order in (1, 2, 3):
+      for num_boundary_points in (0, 1, 4):
+        warp_op, _ = sparse_image_warp.sparse_image_warp(
+            input_image_op,
+            control_points_op,
+            control_points_op + control_point_displacements_op,
+            interpolation_order=interpolation_order,
+            num_boundary_points=num_boundary_points)
+        with self.test_session() as sess:
+          warped_image = sess.run(warp_op)
+          out_image = np.uint8(warped_image[0, :, :, :] * 255)
+          target_file = (
+              test_data_dir +
+              'Yellow_Smiley_Face_Warp-interp' + '-{}-clamp-{}.png'.format(
+                  interpolation_order, num_boundary_points))
+
+          target_image = self.load_image(target_file, sess)
+
+          # Check that the target_image and out_image difference is no
+          # bigger than 1 (on a scale of 0-255). Due to differences in
+          # floating point computation on different devices, the float
+          # output in warped_image may get rounded to a different int
+          # than that in the saved png file loaded into target_image.
+          self.assertAllClose(target_image, out_image, atol=1, rtol=1e-3)
+
+  def testThatBackpropRuns(self):
+    """Run optimization to ensure that gradients can be computed."""
+
+    batch_size = 1
+    image_height = 9
+    image_width = 12
+    image = variables.Variable(
+        np.float32(
+            np.random.uniform(size=[batch_size, image_height, image_width, 3])))
+    control_point_locations = [[3., 3.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)))
+    control_point_displacements = [[0.25, -0.5]]
+    control_point_displacements = constant_op.constant(
+        np.float32(np.expand_dims(control_point_displacements, 0)))
+    warped_image, _ = sparse_image_warp.sparse_image_warp(
+        image,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        num_boundary_points=3)
+
+    loss = math_ops.reduce_mean(math_ops.abs(warped_image - image))
+    optimizer = momentum.MomentumOptimizer(0.001, 0.9)
+    grad = gradients.gradients(loss, [image])
+    grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
+    opt_func = optimizer.apply_gradients(zip(grad, [image]))
+    init_op = variables.global_variables_initializer()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(5):
+        sess.run([loss, opt_func])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
new file mode 100644
index 0000000000000000000000000000000000000000..7e303881e213a82e412d18de9d9d86f368726f06
GIT binary patch
literal 14060
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}9Bd2>47O+4j2IXg*pj^6T^Rm@;DWu&Co?cG
zu$OrHy0YJ7<rh|!=$WbelYv2j!PCVtq+-t7xs??nuK$kRpLvq=NrQ@tq6}xagTWmM
zmYxQMu0<;zG<K|Jy0`n)sa3nReidK+8Wnpz_I#|qO@XMO0DnkGKp>yv!2(t%1$ISo
z&l83--`$s=x@`68)xTz)^St-`<?`LDS5@z-w);GH`>NHe8~I9CILp2YXt%nukZ-GS
zrPOjh(~VYon`(bDd|{e!Yq|e&MNZDSZY^zX-}w3YU&=^I3f)^7>=EJ7p|7OG6y(f&
zC^hn%i+gp7=cK^6dk;QVC{*v|-Fo%vr0x6mE!?(k+ro9r)B?hzU$39G<iNa1LjSk@
zsrm26z|Qc5kz>n~liN&78FJTnFf4B8duwHDXIB-oJui0Bq)EGuELrB(vV2)vnWx{k
z7IhVmj_%2VLQ0O)R2hsIIOY0z6c{4aUaQy^D9xV|x;-;vLO_IuqJ6wmUfzPV_xIXP
z&5^11E!?<&(V|1~5wWrVo&Ed{M={K1xU%KM)ePoW3{k6R9654A^5sj;>1wK_Z|Bdt
zwe|K4!@!MyfB#kr3QB2d)pXFB;Kg=Yh;_4FJmW0Z7*FNbPuK&awG-dRao*m>x@zCg
zWAU@*zt>;6BICy7>CgYihl<)U2VBevUomwxbB03ln;T15Yk#Tu+dZA~?qm8KLFwu9
z-hC{6sdVu1Ld7SIuX)m}j4lVxKDT)BUa>OHNh%AMxE$iQXL%L2jz{L?$xMw^tGH@5
zmaqS!)(}7YPuJF@xoaHwXIgChKA(;6jp@HL#?{CA<zxl#E@t1z$>njxXohQ|;FBtk
zpA%kResN;0OXgcE-aWhK&DBl5x~gsQ{Tg8n9ldGS-rsIdXMB@$Y4r`RQX|H@xo2iP
zu{JjP6>pRoH2=uQ$I6QyJ<8y?nyPAadXr0Iu1Mikk5#>|ju~VxeCnQT6*T$98U58d
z-jiDH#Bk>CRTZuO+#6HyMdO}*g}{H7fLkfy+eBC1Wu7}l#4z*Djz<1IX>)_Wdp-+2
zs{S^`mg_*3x8o&|A6II&HZAYHQx$#SnRFz#ScF)b;Dayy&eKlTRbGsqrkZn;>E+Vt
z6TaWhzY!YH(0=;#$5P?AcVgkytX3&6F6>}UK0fc$@0*+3zepLYtqHEM;9hrvee=X9
zZh^nb3EE|kYU-{m`|f){q%t?Ra;wg6{!Q|0b0;z~goYl7&R_WG@Ar-GPYBvn|1h}c
zV4WkBlkd>pD}O)x<jdvhXXYBGI}}yzG7n+A{+RJg-1Oc{w>r*bUsMo`;aU0WaHT=^
z;w{@=_T8Ot_UY5<*Vk1&rGK83$#^2w$nfE$`ijbru5ZqtKYr;>j1A+0du#O<EWH}n
zz<2hH&(F(KG?T^7%{E`)?$veN-62g`PkL%9n}gtkG`U%RkE6Ng**=Wq*>^ll*<!!Y
z%CA@7{1cs#Dmm+^e0|IRjmI}Uzo7Uh{ePVe^MZrD`ioVsK5LY-Hmkh&<Ye;uov&6M
z)ZANLU2AgW+M-Q`yEkQ7Z+V>)`M2MRW%ItPP9`508_T}^`SzyPpQ;5ncj_>|zP|X~
zN%f89_VWGm77f>%{slz^<chtxa^=P4v@;fOr><TvXPodMAzp$(!acR2HK01&Uu5Rj
z*CpB<>wAtL-Y)jX<Lviyi#+mD%#WO{`qcUQm|Lb*?o%(B>UWlZW=(FsA0Co0`SAK>
zu2;nx`{vqKKZ*YQ+<#WVF-gNfyZKjc{9@2}kQJ@oqtkAoB)wf*I+}Z5zggFTHRWlx
z8)h9hd%W@|<5{z+3!x2DQ|F1UT+cqQM#$anzfAn!U)o{|d+)a91UvLgPv4aB?ak(Q
z`#+s(JUQ2T^?|IltarN>em|2XV#+qV_n3*yqHSq!w>>(c-FIitiBsD$GU`LFDQywU
z&`x;%oVomNYwYf_hYSDy{<v&!{;H=}!W#VcJey@4cBkg^l!q^0CO*_UV$%>gq0-!I
z%UTJgcpavo-Y<V<%l!HKlF@vVP}t=o4x6WXM7~%$HC67;>FJ9fwTgc@d~NLv_60Y-
znO{HiY3mop8JUM#rmTLq^LZD4-tOF(s-z^3j^mFTk3VL8(0Kgu#WPbo5BNPhWyCNc
zmyv1XBz@!g8o{?G)}+SzhS!N?*(97^G5O3-o+|lQ`+Mr7BftO5nq?RN`*GUoW-sIU
z+EO+Wg3sr+|7+)ed}x=I$h7I(-`#$>R`SJ(oyEuX;x|1x6@A`HThq)Wgj<h4O)BAh
zXMS7jP3PK5oe6Sh#ZUb+c%eV{pW};#m%G_5_>*)O*c_-lzKC_^z3o+Uzy3;{efhM5
z<H4TK0@qgxtI6B-1oz7vl-hDhZC_ZXWl->9w%FpcJ)zxVbFb-Zonl@a9=6Y<Gb&xT
zZN;bKtG{&C7{~q;e)@WQ_`g>_?QX9)dG5rdj@d6>*enhc3_N-v=UmjpW_HE9WzyTy
z(jRYnl=Q~*k5aV5T0N)q)TRI4ov)ww|J<HR=U^qJJq%MaR=6aZ_V(_(#qoOKsgH})
z-{;50yX~%@=eqFptGAr`pH>OZn!kLjaGGHBHJQzJTG@+t@9Wu{AuG&$eof?yiPPI9
zuU^&A(LKs{kTp|qtJ(tF=`lusma@yusK4IAd3W*NIDY0$S;{Bout^8}&&g=Gn!5J>
z)hKQ&U;9dvfQ;|?W*`4-|5R!7<7Lq?hiu)pZ|nztWh__te1MH%KK~3W(Ys2@g=vnA
zLfu^-B%?Z`xGyAji~0U~^RRvW4g*tDBc2aM4q@t6N}CxjE4;p3S?M*4X^os$ZRJek
zjkizCs$KQ_!PXNQ2UE5`-1;PgF?{;Z1ncz@T?^P6dU#6gzJ~Ptcx%q1uW_JV!EU+h
zRjGz|Rp0M!-uM4>eA+xQ;m4mpMH!xWV%jjf_tGB?pOie^7bcU|_3zsASL(0Ek&Dy*
zD?faieelWFvKa@nV+@}xm1KPTk-^iG<LoJ?U+ecjI<v<{hdto<ZmY9P#aONM^o*WP
zbbpq==jk-#OUlZ7<!-5O7Mzv)Fk!JS%UW@UY}GCIdF;JDN9p!7ENy&J`}0@FnrTnn
zE7v}~P@VNlVvED)oX%qT+-YnK7cYL8SIigm{msFfKY#u>@!nmunst`9E?dd|x{H71
zD;|`;UcW0V<^6>K{VjqK@(Z{Ai`*(@vLPk%*^=g4Vccu9k~?~JTi4v|x9AsLzedcf
z?ueLc>b$cBY+L4<FmUj*r=4;5^zQDAu&_4&RjUK@3Lg0`6JxYeRxW&D`#t9C(>a#T
zK{a-)ai!~ePk3!vmKe5p71O3k%btC_mGV^h_R7jXD^9s3?45Hsi*L8aqHcw2a?9_f
zMX=xUoTW6M-{QH9%DY>wD^pW{EZQWb+9aG~WNLk(*I@nL4~HB*<JL26<kz~Q;wwEf
zb4O)hRBOoI6TdjtKFdyg;OhGM#c6MDHLcq>*`@cMRW`l-DKRx~+A%%mhU?sOYR!a-
ziVChxRQ|DP6_-1&|B+Nv?F)Z@AOEuZ_q)pYv$IS!PQDIh2z|@e7}<Nl=KdP?ncIZa
zgETc>7QHgkUEBNMq*O{?Ll%F9_O-)S2mBnK{jILFIq>9U$Njz1lfAq)>{_+zXVn{1
z=?iyvxqge;l=APFx4!HwC#B6?8^zMzcczM_SSfwZTh#G}O?K(mdo#~IW=>BLpQUAF
z{W~yY?bN><Z&i9@!=8L)3SO?1_`A&K<mtnoo~_cTn*G?B;mDox`%|U$_S-C8o@Jad
zW9!AJua`{j+<Lpk%q8Fd%Yn;htKUq2kf=8C%qhv-Z1*?MXUK|gHhu1XdiIo>ILpl@
z+Vh2lm~4LQ_{}w7oX^lvSYw&9!m@$y_vv`OxcaYG7gznR={mnuc?)}kKl6G!uk7b*
zmaAPll(R*zZQ}HFd5w*08eU!NcjD2T`&{hAqT04r#{YlR+fvh#EE^22`}r+c#&)at
ztf_GL|9{_u+gjSS3pBT{VKh6m>6`QGjeFHkWhie53|XeUQE<gUrz?9F+?pCVrA#>G
zV*1e|4FCU`pOY|M^-^9WBZ$BDr0U`SpI<C4*6--#{P@yZ;^kt)lj^#1Q9CZ~kmX*f
zwJYej{^LIXt=BJIlv4X8rFq>m_W4=Y({D_-K3=(a(F0i<iN9rU9DEWL?XFKRaQ9{V
z^5o>o@cZ@u|JW~G=GK<|{@2XIYks9aJ@q15m*JOFcJ`lh3Axz|qHN7;_lZtednn74
zpJCEuL8a3k=T9DNymV<<WxxW@20!0ksgmol;d^ZhlahLv^j^>YTF#O=-)Fmf_F^vX
zb#iH8*Y>~IW^QRK|9Afk*$HzU)_z%j?u5hgH#t2z8V6pz%9?U8)wH#}Z_l?|pTp)>
z=e@6J`2XFxJ3cX3r~O3Si|>y2U2QC%oZGo8kTK<R&ox`Q2M^mX=3h|IVHS9J$}^xy
z_Qh#^`OoJLwI=`mS#wOTx8tw&lIcGWuluOn5cni(ebIZHyDJ!Gt-Tx^A@Si#u*AHH
zjpDJ^oo+j`EdA_lpT1J{es=HH>hKDU1-)~AP2<T8xf;f>b>ECNm;G~?-|!!}rQ*n4
z>mA*8Eg^Ku>$CGWpDz2<b15n5!N;E$iofSQ+41wi+S!6uA78CLUtjX#g5N|IvpPk&
zmaNE?g|WZa%==X{Pf09IebX!V+l%J>O^)0s=cm)A@wQ7_z`Vvnj%`B1XO`9?-WS>H
z&;I%_$@`b;C9#I<^PAH3w>&<S$sQJXJ#|%2;FXPXGrPL`mCd&^ZQbg|Z?K*BLw0tm
zjD^C(7lHRS-mvyt{a{ukU!c_7e=inG7$mXW+fb4q_Jr%g<6}#8cD-D7Qu&0$nO`r~
zFs$H9X1!q6@T7a*vb03A&AeN_#3v*sioV<PIrz(~uOhq3>zP)rJg=nniZdlu_voh7
zo3oZLdVBH>`xCn#n|Sv8&eK)%({T<7;k<O|)Wit^jraDh-cV7vmmx1Z>(mP$|JdD3
z+~P{cy1e05b`igJ#NKefzI@)S{mB;>IiCCyYrE8Y#lO7y`_g4!ul1L?)7jzoENaG4
zW$DgkrrDbckBYuuv}aFPg;DVGZwEJ}8Vg8EbG`k*<aSsnV&<A@rc!<3cikf+nV+4}
zJa}vCBlUSn$+i{0-wOE5=X)lzrrUhhq^YH=Gbgz``}{oly88SFCM=fI;w}jOy5#M?
z*S7rKfs}7fneqFaZ?ly3@Q4N6yCL!BwW3Qx_u?nr`gupr%*>ZK#KSPZeqJ}H@V%o8
zo%fteOuKhwEk{m`US%bt&zy)H4r7KX(@$Icud$D-yZLkT>bwY^K&iRkUo7^Jw-V8}
zu)E;K%36|@^(FK3GheH{-r{eYXRqp2$gKIVEw`t2t6Y`#p(dV;h0E;2kBP^>IoNAn
z*V4w89sj0JR&~dRLsM6*Pv5&$^mfA0{nqUJ_r)Ds6M6K;#>4q5Yn+)2{(fbw`Pgdx
z?Qq83JgL3+7PA}VN}cQJJ<4&My-T5So?Y#i)57X*?kfeSf4!iSlB05!Ig7_w`sB;M
z4V(EyL}U)%+4){UV1xKlvwXWI>vs|Sb~RxOOeb%yy;JBeX_mk3J6op%!wl<iCr8H*
z0lNj~y1v|gx9!LllRu|FIx2OPz1WyMdHw$H_dc)p@%i%Wg<0y~27_CPY+v5a`O>`B
zOD;P&BB1hHGyBdK`#%qr*TmN!xM=zB%Vois0!OJylav;+_lF;R`jp)`^ZUlqY}s|Y
z&)pREy?WKbmDQkl+2XVQCo*?$4&0?QpTDEe?acc9j|wU+w^>@+Ef=&Z`SfICIlt5s
z?qGK5=e#XSpU&h+-S!dB@ZooenVFP$?LYUMd$l6#_wPIUP?KpB&z<u7{SOY8Et=xF
z@8$A{d*AL=pKi)%WsIv8y&-)hqvW8;dC8@he;TJNX<15kGqzY*y16@~r!D)Fd2i1_
z%~Opl=SAkLiun6JpSXo#i>STrouq2VZP(omb6u*ROw`x?oi;IS-I=4`-Yzv@Kc{hD
z@5`@0YooI_R(&muK7MZ1LdJ?O7Z;Qz?yfwiGxy%|N#)07U(BD?6+K60zVL#>%Y7s_
zZ}692@6WHj{;AV!zkFpWl{*qSfky*&$AmvW&Mv>>^R3t0@7+4QWX8cv_L%J3d%sSz
z?tilPzg>X%lN)Xa-O}Ua_Sz&e9jg9%h{>Yj&{4AskB@)fRex^}qwN;)yC?n&UYN2l
z{;2X@X<p~mxy@Ib*Y2OaBhA;WPqsKcWL5e7vzaoS_cIoys-L&nbb3?I_j{|$p2nOy
z-RN)6TW5bTis#<H7e)1dlz%N>_NZ+4@l^2_6AB+2JzKV<!f%gVZ{LYq8f#jk*1qVB
zHtU)F{EX6;Z=b`~{W<$?tH7%Rjb1tT<Cu&;ta-ZmrP9jVIk&^Q@5FpRCjS40^7S%S
zg&!PeyD!b0$?^84aQLBR)3>pHnV@*L&OH5GL&{aNjNBHfqg!4^WiNViQSj!;BfA$b
zdsqMY+2Jz3vmvw1wVN3_l~&I!kxKcTvA1&b?#J?Z_uibVd-E$r`M^}8lAGD<9QVuF
z8k|38c$4YZm&^WpYcD9|YcL8W>}UC>ees4zj?B`tie~-Swr;I^bEELcxs@|tJ)b|7
zW37FDvctiX&5rZzvhwe~(z{??#`1o#=*^EO-G1)Ts6A)d;o)<{x2*5D(S^=V!QiyB
zQg_<73Gcam^OntD-JIMMbEfInQ;z>$s38}1_y4)Mu8!ODHr(3QoBJ~1;j{#vm60K0
z`QJaEf48se_Ivg+vxCC5VKtlPYP|i-I6bE6-^Ps=mtPr1u>JaSnO#rP+Ds=%M6~k#
z=fw|&e?LjcVVD21*!`~jlI1&gh*fdil<z%ZXVLp4MN9Lc{a)c7uNWCO!-$67MPHsy
zS5~u>ERVJoTkTM8{r5{F^WnocrPaP{GRu<QKPRYhMsD}E$McU?K7DE$cK-gRQ>R`f
z-u*Adn4>;Pc<S`szw?<HQrFqfdnJ>w_;2Yu9v-QdMuuHcmTcY*{gRWnR2=JRVG(uP
z`}V@VP2sQSZoPeTtE5{w_s{=gqN4AZW8?goeVsL%Zd9>|y>IvNWh~iwda*e>_l6he
z79MBTiD<Z2`|ClP_VGl{7d<_U;$PorR)2|KvCD^z{Yho=)}C)u)6*Jm&#Wlto$d5x
z-_~hfIj(D53Ing?R~K<@Si!^4BgL8@Bcr$R)6cL5IjhW|`6{*tY!ZG7&dhzb`BPH)
zYr#nSqephww@AJGkk6(wSFP#pzUh6vrS)4nZ$w-#moz!Br&jZ+x}S?tgJat5nQ<#S
zg{$@D?Cq4iIuET$zxFG!R!+WP)46>SllJTGJ-1ozjUlUmv~*rs)V8Zzvn94X_}CEI
z=Q&YUTd8foX2ZLzlM}5bpRl{3>z`kKGu7k8efjOPYPY?Nozci#-P_+CtYoy{AKQ{+
zsh*SF<Tg)o*wv#|T*k!k_QST4U2#Un;nRLjP*K@oSH~$Nbm(CF<gJ!&7jLc;RqX06
zmVfc1;(9q}gG0s@<q7QDr+%va&17geBjel8wD7DKOFV5Kzts1zwquJHoYGR}^!04n
zyu+Jc&EJ~Y%wF#5(Xm^AM_SB#uDW#Z>{DG6HXUwn-&LizaN&Um2iLs|(%2o=e77&o
z{xOdVqeRk{xevF`n>lk!#+Mfz8E?fkjL&mlI`C-?+kK}Mf9Kw9oN=$-rA6Gi?TBQ&
z;cO{GC9xWXv)7upD=EA0;{I}B;ghwYzN)(!q$5SDel~Fn%%7E2qkTu<Mcm#==Y0I0
zxvCm&F}dNjKi&35n%&H~J$BRYif4Y5u-&!$;>w%XRs<Sea6aA~7+HS4YX|R_6BF0H
zJGZIg!L5}Fd{QhWFPl!frS>-TSw5X|<lP>=U%&L*+0}V18E?PHTe$V)3-_gG?f?G?
z+59>sGBa~-;j5Jw=VT{{EM&|+;Iu)$RLV?;W#!6uy|>B^vaWmUrJ??y>)YZpd*c*z
zdG9io2u>Aenma?HGA}GF$NXyC3h@}}*0TwZm+pSL*mbR(&@%t`y)QD#XT8_kyG8wo
zea^S}?c3j(SWPzRcR%*#=DU!R<mAT8%P}{nt~Tnr_oYAa7>|mpn^Ogw(}K&Fh4o}D
zE(kAT@Y&TS<(_(v!{b?h=teVV7QMOoVk=c!+N%prs@9%ezogSf?!l+i!3lh7IqPn3
zRtk@K$ai6Ly7}6Av)~QelK)h+e1Fr~?fR5QspL9i?5+<{CO6EIi+9}FDE&?OTWmQm
z|E{gc&(5Tzq-c~K)_66`HAQaAwY4WcJhYorUsv$oLQ(O+{C^@>^m0z>-gbPyzPfxt
zu=+<{v5VX9YU-}dmZ@cKF}Z&6Vj%yfZSAtsoogHNm-cWZ{oTgQ=JI&e>R-2arb$)W
z9AM_>X)@fCcCDGs#6tVigXW2jjvsg5XZ-ui^8P;6BiUa)yqsiMnwNGr$k~?Vg!EkV
z%FwraJ9lfX#S4u*wiQ$R+~3sQ$c>n=<#+vyg@GPEJzL7YO582-w&#%+<JB$MbAHKX
zM#BXMBPTvQT)X3?T6&$LCTGP5NB7xgJ6+6NuGCDQ+M37U_W9dU)~Xj5)?NPf{kv<%
zew&$}F6vCllHB!p;txY@^@%qkUHtv}4qRF)yl{{5#Z||8UUnUqw{QJysbtOQXZd~Y
ziOK$MNnbLmb>-~jKU|Auc{OcH@c%PePdr?{zB>B;%XwBSJG)<B<ZOHPZoU;eg<Vws
z&$BP*zijlKIjKcF&VpA_@!Gn3E0?<^|NOL(gPVEH!5lWDy*!gkleLA!)E*w@S2j%P
z*t*LxL*v6G?*mIbr*-Yt+x}<sCVd7Ee|9Bh!@lsg>d;-KuP4cfEn{`LeJWAcs;m0y
z&Fqa2{zt62C2AB~ool})s#Gm0xmP>v^0Lz7+wb?edwFU6n4f*r{j(0c{;bc}&aU<F
z>gwT@K4e(B%J9m@jR#hR9<kmoUTNd7+|R*<Wm4tOJ=%}FeclM2ZCd@M{Mpv93lFlY
zFLWyHu;}Vuo}j#~W%GHzBlB#ZCw+Y6_~P#F%bmh}ykC{HwdLOLHD{Y&GpTcig#4F=
zPYY5PZwj1}I=vTUp%2f8m&+GMY&+AwMu*>N<?PPA)y(C0S>N2eY;}9iTMv<k$w>m-
zeEav<H(fa{cly?rg2d+*sSjMAw$6Oh(DRD>1>@JL$KQBAam;nEUlRD<%J?K(@!}V1
zfyI)ls--JdeNr|w<Pj8%>sYvn$yonxM&!oAu<O0ej=5`F7AY<87FRfTH&4g0TlUMH
zoPd&xOL+S9P9K<CzUaZ$>jq*66n|~5xa%Gq9PGS%0{7B}l{GiN?7CyYAMkU*r!DUv
zUn>_a<<6MtY5s5bvW0!!+^#Wm`n{gJx(RMk`rw-SCW^m0Oht+5#pUB~c5UtH`LlWM
zLG_pGR=#}sf=Pjw=fbWDcR}+$Cr{2RxEHVV`_6*0yDvhy(ykroT3e81|KhRnW*Zyn
zge|?1BIQZ8H{QO9oH*I_?=M9yJGotc8+LKbogiRor0CdWq1Eh_tll}9|J){?x)ign
zYkTHzIy3Y0B8lDSnigzTy0&<;<e5bszRPFV>dii{UH{3W^HS|EhK(B~%BD!_MYES=
zWJK)WVexd)Cmr^==Vm8t;5#RCW!J^&$?O@%2hOf(&^W7i#_hB~%{*c4qE9Q``57AC
z?OV?GqNC%&IYmXTEw2`bZd$YUjpTafERki7rk~D*?J@cN^y7il>#lKEI5JYL@9mPE
zaaL*HRQZORzfP+6x0h_-;&c?5y`0&~=#seGue=q3Gh4%>HJ2PT?K-ok)PcvsKgVU+
zWmkvK3s{es9p-S@Y%}rb2C0p28)`l=t<})+v9H_0_u|8c2?bvtX`U@AO5Qy84#%z?
zsm*^F4*X$Y*jJvmc>BzVjF+kRc5!4m@C(gN6Ir{~C3CgiEuOfD2>Z$-b`8=}tKKhI
z!fd5;Mta@XlU^!ei?&|5A)_An_WYz}dv|_(%sVY{lg=c;3#abn%w*pbDI#+4dE4@`
zsCBicPhEca@S&JeNyZBG5ZQaDZ|_Jhbe+J!=R1qZ{I{9TtAd%9yLBdUBw3y1$S|I8
z@XXqOe`hnk+w$nfzqM^EdT)9C-g15Q^XbLOuG3DgR^HRQefv>S$HLQWv4$*@**-LG
z(>7l4vecMcY1W;NYhhY_X$H-k?|n-t_u<=I`?IF1!NQ@s=1FL7Sfbe09EPeJ2R+NR
zv;}8Pf0nW-OK?p?*0mOPN6TF~w<q6ZF!574{Y>zd`#kf<kvz4h+vJKkroHqyJKNey
z_FzopHZGHd{YMXHm543<m>|A&%7zQ&9)T<?g?>3iX)+WjZTXh#&k~np-xF<iVp~r1
zmboY1ERmh%XUrVMVyck$alz*|F0T``yWbdY&0T$QPEsS^m1|xLW<TXv;&o(MV%W5^
zi#Cfcc&U-6vfDx0HS<7ZcYE~3TV1vbKU*DHwM~pcbS`6F3!ljf@t`NuRsEtj?^E)q
z<7tp967#XX)_gZC#WrP`4u3?T6|eA&osvh=3_ibUk35q9yXdw(bHj1wySv|&IQ;(0
z*l&2oWW^rq)&{P%6VtcdPFe2pM7{Sw*fX2m2ffXkZ}&Yl=sFOlJ>{+QqJJ0szKe3a
z(C1^%IO)HuRV(+Do33=#ZN<|i)m!dVT*z5s7G-8DIyrUswZrd@G@XquPjbFd@-61p
z@*b;yvppy2xV*S^tIf*P>2tK%k=zRpLbzVm{PMWV78RMkc$V-2#?NB>+j{PP%UsOi
zUw`bRruKyER~^6avzEP3TFSdM>$yr%bL%>uNxM7TrrHZ$*f1f@HEfZQ^>fEF>ozNB
zpMG;kC+hQpZA=yHXVQ`ue)}DGdfG+C*kIum;rgrgr8-AThFnTzVc!yblkpZu!GWl5
z_E<lWnUMjFd^2Xdef|1jbEeGchwHO4CJ0EK`jsAdn(Zgk6|)Rhu3bFmvrO18q+M$k
zpX>TF?7*_ibE{tR&Gejha`jwkse+!fmu-VJJR56WUA?XZsDIp1nl|}zBF7w`gLQpV
zizgqfuM0}E+t2)L*_DTf85(MHcJaNKB=rCM&!15fdK38y*rse`YWv$Bw|bS-hfk;7
z142U5v#pa~WG;BqbUyF&y3hCK-!Mzi(_x9LlU%VT=cV#Rua6gZ?sUl1tV{b`>hS#e
z?{YZ{ffbB%_V`~oCmsLxfd8^?y;BF9SSu^u?M&+a9dN0+^G@1v^{#7gw&k*?G5olF
zzk!vzp<O=jrSd~<b@r)Cm;UJWecn5#B;ozNHZN17&GBN^E)xH{)o!y(JwLcVCwP)d
zO8R84-23|!)t<g((9!b#FHvFe$wAaL`&!?H+gts3rQM$8?_ZlAyNf4{lYv333A7eq
zgZ5^|ls9kOFRqJS!Bo(odzjfO|GZrN+E1>Z-nQ^aRj@`>2_1f{6IJ&+SM}4UKOG$(
z9xUo!#PWN~tH$0%`ur(kF&8CN(k{-r&UDGD^rGt$^XNRi4T;W|>!!utNR_;?U*44A
z(@h(>3yqh8>StP6Bs;LLuG87QZ2gzNkL6Vv8o2d)oJve$?wG%qKcV#{Y_X~S`ciEx
zsTr1&b?pC^oR)3pcz4QcO0%%&j>F+$AwRlby*d+Zc4VsKa^>z%%k%fGJjN~`qGWQV
zS=IYdPLA}MLYB*q4*oNUcyYUJnvUZ7jTQg@F6+M(rh59o$Hxw(rd$<2V-~Dl{ks0f
zX7@iwweH>Ei}<!(@$$xG-W?y0IWZhq<Z9^QFa9%K-T8}IX*c`sJKIbu3^rs4M#gR7
zSg~r;etvT+rHj9r8e(@h%#!+ZK>IMOmHP6B-MP1!LicE_X%C9`Oj+A$c;b;_NvdVT
z&(GW&lih8FglcyD|0i+x`OW!JtBx+tef976$?AjUZy27P=jZb=*J@+fkjky~`O^nc
zeSedrZ~ph|Uh9YM6Y&g+*5+97r)z=E>%_uayA)3g<?Vi}VfSZ)l%k^Cp5J*)j~?6h
zeN(NQzp#+IPtHqSS1$K>(uXM%81{VTt6Y1OU7n%*&O^T~si|yMwqM?b$1fB)kan-m
z#o_vsr$4<_J9AHNs1j~vJFu^R?Rx%%dwY^>KJ)C{{PW4=jWbL%&pi4T7Pmad)OLng
z|2FBVk-jsTZ2nn9{`-C3bIH2vUyHM)J}OlN3VJcr9#=W=z_xYjRNmcQUT0=K=ur%;
z=UKxm=sbOHZ2!s0-5W|WP4~n-$gl4{xORKlvzl!eZ*OqPH2QH|Un_t4(ZzpVQs?FE
z=B;=lc;GSPy^3asec97Ln&sW#E~{^Cb;#8%>pI2Jc<%i7{r?~L2kqrr_A`E0?f*4?
zQ>QZOL<Th`9`4$Z$o%uCx6p=?mxX?ni!=03aWC`loZFT3`)YLlN{4mv{(1XsJ~aHj
z{n$?-{8Y_%*{si>J~XbF6F7bQjBfqDKTm%?uiZOk;*Y&?TiNAottRdAInVgCQ`{_8
zvS7)UkYnHP|K3<tDtoVHGuzaue~h*D-u3;c__qDHwsvrF<b-q2PB)dBozKa-ucfD!
zU-(qhN@~X^uLpOE8w(#d<lWse&wK{wfzI$i521pyrYB<eN;xwY+>WVOza$`JV|~5z
zM4jp?O@Yb$6H1-y1vkdJUVYA$DtTkCb;I&`s~i&-ww&^kY;6_#_J(tgMMgjS3@cNf
z?f0ZQ7b!_zk6r$a=gb+MgZK8n>2|I-aOrhegIUuI-Q&zVYTZ+!7SFsM^Y6L+l0!ev
z7#|RiXZZh1e5Q?D!UG2dL&1(NCHp4(e}WPQ2WH+)sFz^pUnIaZgE4tQV4&a+D{JeN
zFAol~_<Xh1{H4C<DtpGkDT)(+?*G5`p!WI~wSOKQw3|~~CXu&8k%^gomHw1H@2s|-
z*;mW>=TEV18~<|t>pU58l44<He$84<Q77#yx0GrZd{|)STN!$%^0}b?-#?!Z9G7R_
zT|T+=R{Zg!E9cH>+9+*Z#&GrOk4r1oMC$)uSbk6P*VRWy?^PU)tJu5hD#L+|aw*%_
zT-DH;v>|7vyj(x~#fuI>lf;vcw<#$$)^q6Zk7;M+US*nDS8v`jC*hXX1)CL>uaz?V
z<0dd3c<=t0+aq-EH>rZFp?^LdJ$j(ue$uIESJsW3(_`I~w#BYqc!Njv?fQDxkH+T}
zxG#L)6rA$whDX}6@6$!KYn(gl-ITA?yytnb*TnO2b<W$_8@EhKJ@8uh`V!j@4lM6>
z`%4PlI1zN=Wmv<uBy&GM-AE^`x-BZ6YuP`hu!iiOtiouV_F%g2><Nh_dzQVwuB$ig
z+Vf|xqV=w<Te!z&b6mMs<VS8ZTRG0x*GjL+oc{a$deZMQ`!`#|PBfLS{%uj2cyCuI
z|L5)ZqndB(++X!H;MAMj>{F&alHQzb##(jX&gS<y>q~*6PEsCkW1S2S7%ksp=@S_#
z*CVym_DH92|Kl5m<@a=MGj56y6+O_xd27oVpW0a-ehZ$u7P)#)f8^oEH_!BUjCQ`m
z{hfJp=lJhmb1&=5R;CR%>!#^lmD-VfJVhrWKwZP%Jy|{Q&y%)@i8{>JW1OqDUab0C
z`uT0or#T@kQ$O9WZD(Ro+g`MWvG-Dt&i!4YlP*qH&$-W+e1W;CXpQZJ3G?{wl+X7s
zwmChG^+SLCmBVw{3;gr?&s@IjKe<kNi(Ey6$=;bKwRVK9X`l4%vi}4x?~P|pZH$)n
zUhsNN<c&{<xILEU_hqOTF74mDa?yUq#oPKmgv9gDDe98{{Vnr+)ln|_v~wxjABGCJ
zPCdrA@z)FG2Tj~tGGD$tpueAC;lf96*M$8#^Qm&`!B*~xyOUVkm^H-Z92V=iW-z4Q
zym=!wH1*5s%eMrLXD1%_Zg2IhcrI)EW^+3GhQefbkuq;-b0+clHU9Ctu1KUsrCq$4
z$FhDE)3-MV^EWUUJGj|zU(&`V)75uuP2l1y;h7(^cJFy{)~s>j^}`Db{g%Hztrgra
zXJjMCe{F5By|w^D*qRB22U(3y+|IXNP`WC1cf-oyuXBIM*G$-=EbYm%LbRe+$&%sE
z_x&32o7t{kzpi}#ROJn+eHlkMa(?|t?)midLPfJH%La?{I`=&q8D8ptxS4+I%M_6Z
zr}a<jUgx_RUS?)cX%uk4=88$J)rKn5pX+*`T#1u->Mith-*W%s({gVcoO-o0Vyg^8
z!y+Y{?P<I(+p|gn($sU+`=^~}-}6)L!s_W}e?KICYJGDr_j!&LOGo0nyR%D#w{ma3
zY7=O;_s>c76_!;Z=WkfbM>{-!{<!!5Kk?r$n~NV!b*T(n;OD_`ubThG70=mqKMV{q
zg*Mdg{`%SS=GIj1ooPB}=2}h6mXXijv-f7XwpRWTp4fLkpKtvCA#qRU+nkOZM&1Pv
zA2{y$`MCeX?7yWS_^!QU6?nPj@(BhFyJc(>=KD+iv;TYLp6A@2Uq6x+w7b9TT=4Ys
z#BA%bHD7Bjf99QC^tt^0!ha_O_izQR-mY_cs*|gz=#7dO0?*FOY-UmRKjSqeeM;Vv
zFC{{2!`EM1cK!O~^aoX67~J}{l;^2;&#RZ4Vcy^K;N+{F&pqy^8Q*NZ;dNl~>A(NJ
z-`HAfz0NI|nO8wTsOiJO=EAi1_p-hRD+j!F@L4|7(Kd}|rA^J@I}fL;Bs}%{6mC_j
zwL-2+eS21R<LmY8|K9)q5pcI$zTo-XXTJp_U01){W3%hbuF@61@Bjbc;mW#idv%{*
zgIlCzomGWEvEFW;yNq>H(o#i}%+B(>@|^5(v^DQs@RmyB&IiHECfMy%-(2=yZLL`L
zwGW@=>m+u3kNf<$qapRQ#m4Dg_vHUIypgpokj-&RKcIMlVabJnkka>ko7s4sEY>%O
z?pV9zV)4XHn;K4dB)pBPjZ0Mga7ug0qT*Yb&wgj+Pg41`<l7v}0}Gvh)%3GhJmO6F
znDn{!<16oyI0G@4lm8zuhpdZg-Q6d9*z9$>=iL6uy&}C5jqmM$am;-9@X4VIxtANh
znq_EbhJ882ec-G)Gw-f)Wx-x)_B~&O3{pBi7w383+SqLK^D}dKk7CjH_w)a=i0N(F
zX0Of57<TDK$x`9^T-}0$tT#54YU_M^dHHl^3me0gD+$@Pwt9067-Q%BN?B~OxOARd
zzgnzJ_<FmK(*k?>WSQb>1U0mjF3!sRm91}8HKpPqEBBXvKh0Oycq2A4&9J?F%dS^J
zbgJOe&Trp;Jl6R2r#R->_r)h}1-?w(e$#Jp7gx^DFTpia=f8i)IeV>i<}H&4yWbzN
zJuD@4;_w`1HU@t?&MQ|UZs$i|`1tsU9gl>=xxg>}Q|I|D`Zh)L+PCa)cddS|yQyg}
z|LNA&i0Ihf1@BZ?L>DqXI3c*kwt!)?-X4LE4fpge6e(LpcV7AUF{NbwotOvL<A0`R
zWuMgjGCf{xdxWu3LqT5dg0HV{7(Q6Jyy3JyyWO7&|KBfpaZ1~Brkw4Xx3}{)Y!qJl
za*uT0fpzkA2U;8c-EA%tF}c9-iivSb`Rg{#<#QI5#RNT>c60-GX@$j0%WK>9E6>E4
zl!TU=uJqaWfhi!4Z^_b$`yX#xQ2V>#xSaBYb#cDuT$jAa2&#|RbmSNRb-$1@silk$
z_I?l8m?bJUefsC)t2ZlOfAvS%E#a*7`-H0N>$Em_*)yn%I$X5co#R^lohdwq@z0mZ
zm67YG{}I)XaA5nBmm9cy+q5r{buU#vId4rmdaO@%vE0|!>3@sY9LS!M9;O+xY?D#K
zx15?ywbti0)|DjfyxB3e;G%29|M&ID3b!96ir-q+uxnxC9oF67<6b+5ySg(!pX03b
zcjt4axz@AuFN>^Nr?!37{rdT5-pc%M=4Ud$oAUnIBx!@quS=wCO%5Jvy|Fim)sH<O
z=`Y98qOGB3y1d1Cdqw}xvKBA&oBifR=|!X4cDd|t4!55?(ENUH`ujUdN*>%^VPZ3l
zj6H9c2d`Mh_Tyi55qrs9t4km6oe=+QlFnB-Ma%N$FEh6bUDM_s(mj1(_4-9mTE#ac
z+_%%&US5{8vu?JcGV}WX&+TX2-fh0eW<_)FF}KrYU*2v%F#o^LGd~e2uV3-}vR-Vu
zZ|=|ka{Ap(VGplM{O1m>+Bfgcmy@TJ=dSwpM$wkF{_oT~S3kdw`<3k*8F?W7KUa@@
ze!jK3!R6Cs4%1Q<)Y%(Zxj!7!-~XWe+_@?G*BjXcCQfbr>ODOvBtk;|#Wf~VhpK0H
ztJj;>=eo!47P>7ey!hW;bCVT|Ij2tD`MWatI1|6E%dd4eLW>W_#Pi2gX`1EUXy9b9
zh+Q&gwPu6c_jfPMYpv{R{}g^%tld#|pp<D-Y`pL5+4&9W^HqJL7Ja*YZP^5YPrtTC
z=WVo$W78IMh?^xlqq1^K`n1#sebyWDuWHQ+jayfLK4C-A;+uZUUu8`R4PUIF!!x;!
z*I9kBnz?$VO-Apwt&8kxJKpd4m1UImY@zcz_rUE}_S>&~vG*fe!P~7*UJC8r#Jo}C
zewN(dnd|p;JrdO}$aZA>ztsNVDo>>?iyj{Sc;hht<7pQc3nr#nyPS3lys@H5!e>j^
zmO|y1(R*twcY0o)^y)JE4AbsAHXAShoTo0hVUNvroemx5*=7sBz1#iq%(mPe+B=K|
zR(m$GH3Tl+@aF0C!ay&s8oz~R(PmzYpC8Qlo>#=2_BP6>s<yE3!*7!yCnkeJrGjgb
zDVvN=bkAJlbjG`wYn##2XG#T+dw<;bw?Ftq^3#JWPaK#zPM<lVp1-Hj)6nplsEhS!
zw@kNU^O7G1A3mQ?xwWm=<-;DEOCLQq-RQplT2)tRZ}nvM`gUICHh%SBudeFjSu94@
z<=i<{t5z^PJ9{|X{!hb71?O1PnX{NP7&<I}=hXZ?r7fYB^^@Ta>rIX{^Mh$mzfXR)
zL)raAQRviLj9NxQFD|v3{49O%m$1#Mp<DPS<IK72(Sb{%P1fy`vR@n9y^=vI#@^HI
zEUSrCx!a{JnLpm_d|r^{(P6ye?J~);H=lk}*`}0cR{m(Y>(4JwzB8NO?|D1FzHen{
zsKujqk)N)rtd+6(`FF*t6$g&*E_?W9W$?rU((e!Q7-+AR4!bIMP;9Htjk&u8=SoX8
z+|ECKj#s)ME>rW<U2E@TQ|Su~0ims3lZ0w^*4r=hnb}mKJmHPm<(&(+ahUPPRqeFQ
zW#|1ITdw$f_4<NBN5*e#_0z6hUE{#)5GvXyCZ7B@e}CeqEt$VOWO+{Hg|S6j-*Nj8
z<y)L&U6Hxw?2f->Zz8tyiHnLTSnt@pWqlgE6oZy_V9^(iYkxj*tIe-96S~QmyrwBf
z{6>OOtv9n@^T~DW%$qXu)O}~Zdesn}cW@i8bU~dX<F}jkrypEZa$t5SH9f^IcclIH
zw!}|QPI`!awa(Mqofz(0%j3qd`L!?O79;bf8FBjB%<J~c&Az_&@Xn=6Epp!K+zsz$
z<ekv<=-%Fn^v`KM()MzD7cD+%-1lbs@)MnRH|&<*ezb1di#rjl+$MW<W@(=;JD7Hz
zU&i4<oAigTyGm!g`@Z<fElIQ9D3KR#SD!vjd^t^5qUe;ShqqVLx>PfX-4ENA7A#h6
zohG`XGWCsvkG^&9wKOs3X^p<K1B#!|jo4SHoW{7}X7=sct!xVz9ZsD;UVVG};?>8S
z8zm(L_zWJX#Vo2d-9C3ilpD)BjrE#2?6!NqNp=1ISH5RQskYd`gBtF>3$HPmFf3TX
zG5_fkh4SlqJvN_xN|Lp^-@4{8oLy&bdf`h|tn7vg!*6S2I-|?qnbcp6DqeL}F*vGA
z_UF$ZKVGjlv@;g=G&KCMAwI>R)-1}ob<@s<Wz*j}=`mTUvp2eSZ%F-HHK+bZ!Q~Q0
zUbnS;84MP6GE3R{AARkWp7`qX!sjcOXX$oa?F!9qn7(`(%dF{XqA#w;yPtICf4J9o
zc0pk7T5JBTyVizrmMoU)>e^8AJ?}}*trkIHH3dDTLsn<D++n+Y>ff1DGv>D6cDQ`H
zEY~ekvf#AtjlF5Sd1ucZyMN(`?X#x9Y&I!|fVjT7H*Z#?A7)D_dDqk8*nH(+&I|q1
zdRxMN7ny`kDHmG$cFk_lnbUQhkJ^5JaC-gzV?R?<ZzR=S(On(JU$VGQT<p<VS?huq
zxAP63Y`H8TFS=BBL5<+SLpNj0mYRm}OihhTId{Ij=5g<hty!Y)&YU}D%P>P+b-Ulz
zpy`<_X7upb*r#%--7jR{rG1#SW7k}**I(|uGRlvglD6F^`@_XOCt`w`ZT^~c+5HvC
zE`N7~mqCXw`?A#)u|_tABS#WS?(eI3wrsY+mp`8cbFa;hirl{WOtkpYo4S{!qO2lT
zOxJgR`uY6h>33pc;=|hF&-8Ejy7Da3S`A&^a8>o>k8igx{(q#Cqx{pAQzto0_U_4(
zeKoI6s`}HX58IvF-|Tf;yLIJR=DRB<TnV{zXSKN4zmNSd=FjX5Sl*w!lW|L^{JS!5
z_T73+5i{$*%f6al!BqYE^T$8`{{Ap;Ff&r>jCjkM;hMN|<qlg*$<JbXF1NnC><D>h
z7PXi2a$JMhS!TwBr(Rd~eBvtVlR3Eh_xHz=41DZz*|k~u4%(lVE=h>{{%*(9N8K0S
zcWDb;-nsd&b&qeVxya63>z-$!p$BH?FM4yBe`0V%$9{2<2f14&{S(hywU?>ZY2(Hd
zUVg`Ho7oZ{G_k(0%@UDx5!!HFC~3;GB|H!KE~K3;w*Sil+RL#$oxe_7<G|t6)B+2e
z)Y}2O*=Ge^-(qs-&Q@)`nFfp}r^olT?5ln2eD{(=bYtaxyM&iPGB<Y!{&aP%u-E55
zzeuu8AcNOxX2UYm9rg8hgrvJW)8{lUnQ1I&Us3R3s_WvCi7o-RKc3p;)AlfSr_j;Y
z*B5WtBNPArc6&Y3jnwxUxAyskEy(13d0YJRu9AaGwwS2QE9RSYtlz!j`@46QS=O7f
z47s=)d8Gq(7pZdb%efS$s$P?`=`d$}Q}Vsz)-<bzYpc9dRF-{v(|L2Qb#g+G(4+Gf
z%y-@TTtcdL{faxhjNvxNlgGy!V|O?B&JOtc^}2?p-m#0Tv{Ifwe_VgOKUue7)dRb|
ze^)SPB)c+6++8hxuBXQ(Ek$Kga<bs$X>BLIl$5-@EgrbHdUD~53jUwBDp^*q>*IMi
zdpi4*CksApzaKDlcK57LMIkcweNDHbYUWRGU(fi5V>&x$zW*zRTl@AfR55tGy~ew|
z<aOIahF`yGI=Hx<cNn#{IDGpl@b3FzX=(1rX^mZ66;95ItV}ENGZ<dI4qWcj$e_je
zL_|E<+m&?_qYOiVL__PU<aVnodp<2tt|(c7*b659*Z$F-e%F>eDjOLX7#KWV{an^L
HB{Ts5wR(V`

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..7fd9e4e6d69f3120428d1d778846d495cea1a989
GIT binary patch
literal 18537
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3Fz|G745^rtlpw*nn1P*v
zfq~)we<{|*4C+IOdE|U2Z!?1cv!lKG#DB9{InVitat0sNyU+JyzGR)99iR84jLX+}
zR6RFvE@<tQWU8H~rr?++zT0=x?Yr;PGH1U&Rrzh-?{9bfYZyVkzjMivtHpq)_2H2v
zx=Yt_-TU9rbgflmYvs&V_ZdeI*!&Z=*=K)b;i=CuL6;;<-OfgOPYhZXRT}BEcGl-9
zmqeKMcW%(?oUmJXCr9f87n3%|b8pU6q%wTlB*DD*vCYQ!+tzK%&EP%W`1(-o;__;_
zEo__P>p$ML1Bd&SOSW7s1_BNb3(wZKDMZ|?_#fR_sueh?bHYsBZ4nXQn9BOY>-&!<
zp3!EJZ7TFTp0UhRzdJVE{JdD;(G`NBURp~T1^2MOzZoUzF3HB2z`*#U?ZkY}1NQ$P
zFE&+7QqpYoei-$;wm7?Ho^ATQZTsHODc-NA_UQ48t%o-6*aP;v)TGYl{{G(S)0^Ad
zoBR8n9UFrk8@-*Hf*l&>&YvIa(-Z2^F|)tFarSR};mh`J$Nozg_D<#PtoyU+|Aio)
zO~?NKU-;<%zrVl#{{H{{{r`QnzrX)9-lFsK%gg(3Z`bR`|11ChZ~yoA|Ns8^`Tpzc
z|NHBBODu#yi4m0Oj5?d=&yNk5;O)>59W%$<yF1*k&pl*HuuDsD&?H}vj)~o`?YX%P
zI~gYTs_rXPnO?ubm3x*^1yhr_)BguY&%D3?e_#Fo?@v$vm$&~{`}^Dbo16bH_y51Q
z>g)Z<>i_L)e*AxQ)PY@(51Ov8bT-eP9U3tsI%0-@&?I;F_V~z|;Z9A_UR}{HEuGWF
zFBvr`MQ}EnvI=MzZ|FL(Sf>6_!*Yf#EliJ-Jgxr!|NsB*@B97o|Lgz$`u_6r{@>sK
zzrXwY{VCD5@bwb+_ND&#khp+ZmJ@1MNhkBAIa4MGOqd{W;hNZ`Yhn{7KAbw`iI{Nm
z!>6eUiH_&LTWIkeI(qE?{~1&Hik7)wZVHo2Sew0}TY>YR>)Pgje}A*TWc>U4|NsAw
zkMp?q8N9iF{=*Bw!?VqK<n4|A{5tyKzy0C=|Cz<aA@-e_wvo@lL4dWr&FxW>z}Kmd
z{%iOCzt9zQe%&LXox1~q-{*ZzQ-AMX5E6Q?^w_N`nc0aNx3|6ceP@QrX_1N|XMx-;
z50ol<{{MTnVX~W4$GSzwwB%It^#mP4ZLTnV-!XBOO6Tp62R1j$mvvUAMcy#GCMwwX
znAuAYoP?!*965OYyq>zchQ5AYqT<ShiVEt-wG|H@JbwJdnKKQ$Cme|LGcZ~d%Kl$c
za^sT<3ueog39pwYT+{p?(0gcBl|U)iwnr?9xAxZ;TwZXpqmgkU)4#vTJL=~@Ji^(Q
zd|V=KU&@Zs?uVDv+v4|2{QI*o{tCp)QX7sOJaR-s{rK`lO1XJ@$w^9T+NZS@51u@H
z_{5nWeNzJ?ql9ej2kXN=Y%w-i&A%q=!fi2&NqjbQ4|a>Ov3=RnI;m%Zv=R@`{|B1P
zakKyYKmXw|`{BR8d3Ki@-`RP%;I$ZtV_$Dn^WU%E0+e9?|CeewqNsCb;S!Z)i<RF$
zQd+;}jJoE@lgE!AI&_FZGqx~9?8a)L8B9sDXGVw~@DeX6?AdW-y2XUliCTOWzYhB(
z{(or77bMBdt5k8Sq4@v*^on0o9v)(C;})05yPITD-uLixdfR+^$$x(rC;BjggLYB;
zqlA<c0nw*du7;Kr1Z>?fAt>aDm`F-`a<hq=M1Zn%lSILChBX)Y`!hW>{x`eZtl8z*
z9U@}7!<(a`vLWnJg1^j(T)sQ8(iMS?cYd^TI58Fli_8oA_dor|hsKBBa@)GaCHrKP
zLE-zexJ^GkLo^i<xQl`x2?#&lx;3=4__DdF=%gu6o;*oON^Crlpeem!s+fi$m$x=g
z#A*q*WBXSG@l+^@x-&NK&uLigt|?@2>3;l`i9%1Fv`Sbx>lQ6|%5Y7I#j^9yzq18j
zwGQ9i&C||jILFR+wVeUPi<OT80tL5hxl&Mib=%gdkDjL|rndT|DvI;)+z?`kn%lH@
zPJ)Ap*8hb?f)_g%@9(mm^-)FSk8JpoHiI-}=67;QUVZ<0cp_Gbrm*b^`1!H<;YH;(
z?Qn@c`E-kl2@gLeYw@3FhFFyN=)seeuiwr4It+|WL<B`r(vp%AT5h$4*%%}o(JoRs
znEFtYGp^`?md%CL{2sB*fsO|zE3A9cWWc*~*TGvBM_;fl4%fcXcKV-UdC5c(VLurj
z56>sB!`rT}m(07H6tSh_;k($j=jSE&)h_nqfh3qkb&nF#(xyy*zI|ih=PxGZrBhSW
z(-V@Kz8FoI%=B#=tIi|I3y<0u_ul--@5?3krR5Z-j#`45hoRN#8HN38<aA{hh)PFf
zE&c!h|3bcwg-a~|WHQa{zVrU?*Y1aRt=npUOKi(e2PK@d(poO65MQP}N=Qt6^5{`P
zK>#<8$=1!m6DK6NdJAvpaG$~(QLV8fm|4lIfHzMm++@X7uA-QvX#zW<R3fB>Up(b<
z%WPTNaPY;}MJpX&KQ=HBm}v1J#^TrC*$<z3w{6XqG|Nw`_|)?7Fnd?qIcO^J16d{{
z^yt!+DIdO?e*bV`!qlgUsja)_G&V9?`YNc3yJ&Vzsl8aFzRY6Up6)3<8(eJKbc((m
zopoh{it?5~zdc|1u03hGx2>TxY2y^12}c}jP9-?#x`NW@=k|v`3fr!(meh+&*-_4K
z#pBNgwbbd+gNKRNFHI^ay!i2h!L`f6X{o6#OV}c;8g|WO+^Xf+RhrVWl%Zkr4q?W$
z&Zhk~4#r|j7dEV9%T5*!KXSPD(2Lm$rxbHSIipr7X%#-m>;`51H<E|{{pGPPGniw2
z{KG$6%iK_?WhR9lo}3oe*}r~BL`G<IcWrXyZ50#cS@D&F*-gmNQ$S*~dO@MiPr2hK
zL`^nbl;YjdwsV4)xX!Z#ZQTo31big_C+2dmS)B4eK$zXAnX9w8vGL*W^0wXOl3{Ba
z&OQJ8$1~~)B&jIb7jD?NF(b>%-DSq^-LEw?IX7-JG&DHyTlvJvCPuNjJQ<lxuVXm3
z{bKe1+Njx~%2jBd_A%S#<hvWIM9(Z@)xCH?m~q_{#;tr?r#YHfUr>^h`!2wytbDFf
zV8<`}dI`Ilgoyp~5?_b4tqzy8t5r%25Cx|fCH2A$8w@->K14@l9XS#)Yu2gHA2%8r
z8X6cW^*F5yNYHF(Y-D_-@kW*9-Z_qcuO7%xaywJR6wc*vxk<A%Ec?%K)zb|Z_cTlw
z^C;|(3o%%*-8jIDZ;P2`)UN|teeHk#&wqHFzwK~4DDha-%>kt(dHas83`poEDH<jk
zeE6{8)vHsLRZ>5HNv&AbrJ&Be<jY5S%~}Ux_AKF+#>PfYF%zE+I})2z6lPfV7`!cb
zSMTQLHiN_HWQR%9p$08(L;h|tm9H5LzN&Xh)^th!`|o~EJVR|kW=hNR(v;Ny|NkF1
zV6=9)#;0i+5Zk23#4aPl<LAedl*DuC61cG6fBwTy<HXkxCn@O`dU$-eaXb6>&zTXC
zn%&)>4UCKo3>2=beF$Ol7vtH`@`-<%)Pq*WF0m^a4olc(SQJG#d%C%?g$TAt^*lM+
zAa-9uWnIXj2Cde2Y%1rK4*dVmv%#h3s*lP313Uj09@SIvJ+SnHI(J2B^Rn)lGadc=
z9UU4RothF4vn^UB0d<2};f9Tdr%!xfXTSa9$BXQ&uFoGf7#bNV9%pg;<|utN!D(&4
zj0qC|6AI5aq$ot4OLoz!yO1n$HEqquvmMr~3fc({svlPz&tfY3{{R2^j#&*GmNW0*
zC`x9U&aq6-ry<nKukYaP*^VJo5-%!&%F26t4{vv1fg}g5!h*t$85tffZdSj3zV!6?
zuxXQ#u@NW;4jOArYZAS9#lcTlpkbzhfop(~Sk_5Ke^m}vP;LnN+cZ^0^V-B*N8|Pd
zVr*<zUi|ql@9o*Vg!Q3&#U=+2;n?L5G}9y`C0kl0-@K7<agk8-O|mF%dw7_A&Qnk)
z<o|!AnCpcF8)nQ}rJ`a{QJwwy(<Wmhr%No#%F4o5tlZpkME}>bO_AeT_mB13yc6~^
z$?+$G{wJ*D_5RM-9_Lg2KlyUg-m9IyALAe8IEuTxeX(e3?!&jiE{_&WICAR8&&dy8
z2De=cYjbXwu&X;F_Z1q{b1e4k^_?{Nt*q?Sn>Qm9GMVPtJ?NC($;xo@=KqOFe6N38
zwU0<qJ|wu7xydA{aS>0s&w-mfM~(`6da_^En4-sTc3i;6uwL?l^{yA4%`+4O*LX7a
zE3o|idHBOiVdqfM!v~vr^6w{gRza<sV6kXX&ZNn?zP?W*BW5Ioata4+N@$a43fXUP
zfQkFufj8Ys{mF+O%h^Ug;LBj+(NuaYt$XUDyR@OF!2ZA^YJ3*rrCXZ0Tv^v0VR5hj
zZ?vcK@P+&5H&mM+-kkno@d}6~hi*Q5@Gy~|)9~!Mve0nx<kZ&d${*Ooh38FT4%`2y
z`KaoW{;r!2jixdI+O9baYekvZ|8HD%jB&#3?FGl`d5ljo{5SL{Vwfl!RkGXAO0B@r
z$4n|AElof?-NQ5S!GnYwJ10MU=se|{COE+ynq)C!hD2&A>z}{8#l^Nqj_!<%keGO<
z_s*e&maVD}b5i90HGeJoe>jM(DEYI!kE)Zv%B!9BH5-pHUYqu^o`*G~O<+qaAH&VW
zj)DfBx&$|yYfBi|S*rI0WCu*1*u~X6UHtI(_dLJ98{1SidB{UN=;SYR=1kA3RZ4Yr
zZk(LwgoI+woas<v%>2NxR<LVYjKWbJVdlf}OYB&S9y!RRrXP^z{r@nkKt(F4N}lDc
zdInP{gYu;Ee~Y=2P73|tW;PN0b641B>I}8nvy5tfAN%lD`f$HI&xf}&;g)^+<aF=e
zvBbnMF0Pmxt9V*!4cKCMg?s+;hZ!vRA!#hDkeTjqsas~&haJbIDpf?E9Z?aSw4{Um
z#L1=>t)3Z-JWnEyuVM{m{Ir5|lUQ_gn|-}R-Jb-D0)|`lkXD|Pr_7>7Hg$E&7A{ni
zl4AOC_jtpV;?@rjRMr{Vg{+O>uC&o?kZ}C-?7{U8`3q~i4)T4tqR%7Y%DRkWo*BDQ
zLR#8}O-<9})%}fXejR~0I;qjqfQzfh#U-MpRWc*XDb45KVWr>pRbB}R4yVpe_{Y)m
zmw#(T!*t0E)?@V{Z+TiIB6hMcuJmjv(`?|HDjxOAp)Y?;b8<%WDwg(m#fI+ggLmeD
zS{&l~5_<7z5IdUG_~hjH?Cl?Sa0r{43WkNTRcz*)X7u19XX;`1j)}k4W!U{c-XI~J
z>X6l#q-*dmxnr@af(e`4-V>X6ZY=r0d@Iy&$+W%~4B^%y;nht3P6tmlFc=!LRadw9
z&zHEjHyL6}Qer}afWU)ITS7cNHUtI>KY7x!gp)1emqVFMn##fIRrP`1;uVEW4lBI(
z2ZTK0Tl}Ag=YReGf4_f!muG9M&&lyQbLPZTfmzl6l8xsxKJ)X5^h)lw`SAVy|M>m?
z>i_?n-}}~{o&8CKAY0B)M(zWb6;cm`{hna;nTfHeh{w!~=lpzQs5=A&61Hy(^YYtv
z^{Qw}a&nV8caPFT`J8P$JIhb8<U1`|_q-vyaXHVLkL|`~_5c5Ue|Pt~b$Oh9&5wsi
zyU)+J|NrCncX?~;%^M8bySf^7?P}=Z;de`T$QAY`lds$-Vd_zxT^nZg$U3yG){p&F
z`|Qm6+uQ%|sr>x?dp=uO7+d1Q|EEs%xViPTwe`frs7Q2gt!up98o4lGn%E4jhYd4l
zI(l~>yf_h(oRThHnez4H#qAp>CM72~xT+NW7q~aiL9oU0qkPV}9>%RR8nvC*#awXu
zBR?r=O30^APp2mzmwR)w`1zHU>f7`7{{8gy`u_U=k`fX(Zq96Nu~@RS($usyCwHl`
z(iIEKnGda>7aZ5P=Agdm-oa$4Fw=+w-{1fDpZ{-X#>MT$&+GQq{Ct17z5YY3YEBN%
zp+h~NKJ_eH*5m2vbL$q*hSr-cj0e9oo=H8BQTafqk%5noEjSq3tV_zt=jY(PI(f>I
z2Hsxf^Q#S&dZ!yNERCG_YaNe7b3(_AW7{5yUspN0e}Db|xII5gPEPt9z3tA9^7nV|
z@4w+Zr@PxVYS!@!0c(~V@dyrXm_567;zxa*Q~?>a|NT~pZKrcjXmx&MvikY|KPY+Z
zt^1pPtVe%!c>Uh;_vg>g*Vt-ZWb*lgK}Lqbv**T@RYn|~f0b9p9k$xqrrB3o#rk@N
z)f{ecd6#5UxhcfEdwwTpu2pa6#ECN-8yPiI4HoiEOZJq^cqx9{De1=<j%f=2zP)|^
z^Rxe|)#dZ-{yw?A{r(T<mWdM^XU`4|oVb43u_I@F!yFn!qvs^LnawM42)gswkjty?
z@2~eSFW(mxecvbZ^XIRx_J8grCL|;zB?*Y8zIc)F<y)G7sH$S&fya|7*yeP5pWR>2
z^Z!3AeFla;O-yP!Kg-BaX_?)>L(-BP)-cRpmVUrYsZXdl!<02^gHpG1@yUMq{r+<*
zBEq-7esQtgh3)us{r&p!b=IY?o|@<HJ9=zYVWFY1v2nr`S(evQY&_c+zC4s?-xjD5
zR`d7Q{~H_SC#%=n)cknf&fgNu#>{+pe?5<WyirK#-{%h!K731i@Kn`JSR|p$Uqa$S
zw%D06Hs)=!SZ_K&b5NtXX+mAI!9T^XQIZ!XF>Tve%yuQKl~Fm&U@`~S<2N^t2QQbi
zFMK4(>3HbCfddB$UR*G)s|zzU<SQ?ijEfV?+1UM(?aY)@4XY+Gddkdfw125!GSB8`
z^4C}EXPZy1Id<yvbDrX71|3~S78V8u#*>$c7f<PFWaJiq?ZZ3Ei$fX`eTfb{Q3eWA
z<pokFsPwPMo)E;DG)r!$pQG%rBU0KCmyW-_9&eVvZ(r_h-k4sVKM#&{PCs*|=hiKz
zO`BqDY|dCH@t-y1S-}&d@-K*G>4)Zq`2GL(SADI1du#n{vxz(BR{pPJi;iY9HD!y8
z6e>w{)1A|;*?lcNG2uYa90LPThnM01|E6_+(o&A>QWeN8cqGqU@{m>P*Xa{K#J8C|
z)IS!)wx(NJpy&Vp|M@qIa&K>wwy&E|dFSfu>-Kqfer%}xd|2IopOX`V+oP451uGK|
z1RQ8kalP^6`}_a7H$HrQa#CJDep02(q5btd(c27ScDbBeY1`AIp}ahUXH8Rqt^nI6
z3rHw9=_dStctg5F^slS8p24wrm6lj;qhDWM{=T@FKkrV3YmtXS<MZ?Tbz>@izr3tJ
zA^1kY5t)OFni!pJnfdn4cwo<b7@UhLKBiw>q+k9npk&?cr>A+!-%qNP<1Ug(?yc})
z3lo-*fVO9wObiUWYrjlsELp&yBKzm)RF?@Qe~(5_aM-EtZ?~uJ@74GBm!u1;eE9fy
z{q61caeIILJJ@`_!kdFlCfA>B$)Pv$W}1IE`MT!W{H%R+<a@Vxecd0AiRE?wQY`A{
zJ$xDLbw(keXu--OvtpnL?NG#o@A8Z58pEwlq%s#}eiV<6ZQi=%;3T_q^XtDoIqBZj
z#K^WT{(r%LyOho6>T@pkOv@+;)tIsA#3P;^rG|I*{{DSu=ll@MD}O$vemb9UQ1oVn
zm*5&Th~Z5p9+C;VzGu|bPig5d@#7NqQ7#H+UB}~Z_>bT1UwXoi|Ig3wZ|A?izs|RH
zamK&Dzv~|#JI>B;XKc`s+#P!+PH2|(#BN@;B^}3Sosihjb!X4d-={=h%_*;cEygS?
z49ZYGJ}sijOlxYRw$$m&O4`A+!wq6{lB3_ho}Ld=7}w`LxU>I6CTp37bd3Oy_<^WO
zWl*}Ee}Dh~e}AnT74?%2?5_uni2izXv^;04%C#uA?5_tKqPem&8z((`cmMz2ABLrV
z6aW61`S371Xi&|<;>5+ioAQm!XFl@BoMeS05l20(2?8t!es8J2=p}fAE%1wL(u_VA
z)r*IJU0(kF|B8|+&;I}Ck+=I(_5YvxyG@N1E&<$|L<^!N=Gp%K{8G4d&*76NKm506
z<~~_Hb0Nd)9x3K+ElpcrbAzfVP##d{Qws_HXJoWUNAkD?OY(tfMiXvkgqo(gexBd>
z@cw-JdWG-lfvh#ZzWlzsd%ncS_G1dm`V@@5E|B=~^z{AX{r_wJc1`3x$>ik3V_!F6
zzJ-KDff(lu=F}ZsVvx{k3fG=+vG7NNICD_vp@wM|6TVG7%=-WEj)I0~dT(0Ug@u`|
zts|#unH@NB;=||X@rxEuV|4EpUE6YCvBb^U|L^br@13a5D#^h)(Z;EZJ3)-6LQASc
z#|7m2|NoO3eRuzF&o@m-`O_VE^niNggwy<NJzT8*CmVf~oTg2m9vU*GyV0v*_UxIR
zoe!U%-~Z-KkHk^7YgLTzY;2x#ZEBV*koeu-_z*N!{qOJZ^QTVzc+M}%#jmN!CMf8|
z+c-t|$7CjE<}y=6>E8I0S7=rb56^!^SB@a=W<kxJ3X?d3Ubfts?zmB;Gv&aE-2aji
z|F}6etXjo2;c1~xLw9%U)Tu9TZ@>TMO+me(Kv(seT>mX$+<I>{PTAG}JAUev$J#gR
z)}2#Obd+7(vBt~b_6BIU>0E8>sZZ`(kSza9DJg_G*F^Sp#!Gv)w)#7F&iwkdC@1F%
zXCNbEn55*n`2GJD9J_DNHfQ25dF6Ni6TO9hOy*0f@d%wdbI&|=H3Oq00ii`rmKm8%
zjJGY=uRSzkV}sW9PV#o{vUi)7NUsU}C!W0FkNmL}me*Jsjdq+n@gv^fKO#PU3fI8{
z2M(M*y=d`a_T4L56RIECXEizW=O|6$^Llyc^yvlb&VBl<F8WHu&#!HP0-K!NDc>fh
zCs!JHE}Y`u5?KxLOH!k`>&2+K9xv>fw{$YDO!{~D&4D$(8@5?qkjOY{%FNCloUq{2
z;dc4>y^CH538tmFm6fR}E7#@arCsrGO<o)-z$3vW@nO{w6Jake4}()D41(_&K7XFx
z(WR7Y%U;)TC--2?md5OT|NhrQ{gvdTt#yHI=l}ozyQS8gXgwHzMW;1PD`A>aePPDW
z=g-vAQ&l|^1w1?pGqOH?`e;~OV6f=eeftpaLQ&}*E*FkS7O2eq-_+D2B-C^4*cuTt
z>z*D7CudC_o)U)jb{}|FCxb?c82<lHa_ZJS!FuMy6Q=bs3ZFk2=%sbDgwHx~Rmri_
zI5zg+9vj=TvM#1juP$XBpIOb#g4?uM{pXAM&t~rDC=zm<{P*~Q0|!q0i1+Qid9<~6
z(rRuC3yDjYBy@B&IXNdWuA90R5_(Nv0tGj=n*HvQS!|$rCX96+OJih0Lgu1P8x5^2
zJeDrc=H%Eg@wsheq@S`fqq1_Hot;>ZhKn};zeE?~b)T5SE1O0CrY8JYzwB5_%8{I$
zGd=lkjOGdH=^b5399&ApC+37S#LP($)8*mefmoR26!olm=H~y8G}JFh>|psAVpwoY
zzT)$?ZPzr+!g_l?HFtJ8dUtPM;W2aWTtQ{o2Tl=EygM#)%}Zk2x$Ecg10U|6>atX8
zo7=M@DLtKoFYU#Pgal>dW&wtbi~VdHwn0bKlbraBmQ=CqU6GLThyC@tN9#>LoJd;6
z|3u@Dw$h=ns8!J+OAa4&bo!N$o?cWUa`sHflxb<tA0!+IG)M^3h!Ezw_KA73f}@(`
zg`l1t7K@viz5V#C)zvu-kDog?Vcxlb`11;0bL*tV8Y0Udyj<n51M0IsAC(z1i$o7K
zoZo!l*k!@hE)&_$U1V4{>5F}kHD9E_OqT`n_W$(aYL+j4d}(iW%X_`Y4->a-7WVX=
z`u(F|^V=zZ*t?C-%s6&`&jHnrt*wpI>;hasRm;am^DT?(bYp-0zO4Sn)XGReBqbv!
z#ls`v!NY?$&6dbKxYf~+&J1n1{`sgpud$KQ-O9m_uV=$8j$FG|#>Lv*k`fWZ>}kIm
zjQ;)qU;pyb?##=1I<Zy%Z*SkfgDE_0nz4q5k!h%3>|E(-M*oDpgexv_G55JNsMkL5
z(yssi=jZ<?C-3)4|M!{w?fJdE|971G_2faqhfhfwT1gwW9b4DX#qi%xeGRN){_|0p
zEpsXxvruPbl){lf#)DV-Ol1`|Z(K6<{{H{>c6==S_~`fZb9;B!{yx1r+`jJ54^T=y
zb7sY|V~<~Iipl6?W@>i4&Ei}0l3{78L&>j(1^aK9JThcI_4n)R|7T~{o9F!4oqBrz
z>+6#HY8uZS;^r{W&@jlzHa>FPc*cx?Vh?6}1vN*7&Co*p&qw81jH<B-YuzVw^QJV|
zM<$&8#3FpWKIPqf`~P-zf4)9EeBQqP-`}5~`&(NbgC@<LJzMf;UXoKPv!0n@D$l`F
z9R9b(4*vcB|Niam|K;sce*Bx;ICEy><jIZ^GaO^)gys6jG|V{Ccm`UZ{`siPWcKA`
z!{Qqa0sjrtk{%Y9zQ{Z9*d|0MT+8wCq0c7F+dPg=wfL}vzoOD1rPA3aD?!2OAOmv~
zCzG!Lq)Ptz|G#{YF_ZYo1{2lO4T^kQrKV5bp!uU;vCp*D!NtIuwV+VRo6qQ!P?D5P
zMwY~ckV88^yvljN_>9TXMDNoR8D?h4Oh{7W{432{-t+EEOKh3X(0TF30Ut3I@!ttX
z1xtC>ys4K6Vv&}~sGOc4^H|gPg9`T_H(s%Z{qMUpp+5Qf|G$3L$;Ley%&Yh^pFgfw
z`lS}}=Fl=B<4A?#iQe3g1P@BF*Sk7z@NSxUYeIir!hvNu1&mFcOp=hy(DdHs*cum!
z0B@!k?WzwuS!Q!wQ)21u4YN^LF43gP94vWIOygR_@x}8B>{+h{H;X^;*M?|JYHVfw
z*xAgumE&iB<c$X&VkaK`H$1X&n*5S!GCz{%F3|bdE7bHd`NJ}vN8J1nElw(Z|Nl!q
z_$#=id#S|4QlVQN2RZ~bB6daybEVB}3>DA~k~DBBaJ6C9?B$6OJkOB);Qc$kvW7Vv
zAC`cI#~J?rKNMADV3gp`b;R6ggVtdm;T<j#29G+IG_^$Sl~~hNC@S5&S4+{NtA8b^
z5Cb(0<_L9`M~bggy!c-tM<m)nV#g8#VU|rIb3XDrGtT>=tkErYkjLdqlELX2^()kP
z?oDB2JR%}-LN*HG;H1X;N7~HTx*~G~%+<fhn~P^_IeN3qXo@(^!`4%gs-Pr0LEWjQ
zx@-gU8g386>&p(`gy?Gu*XEg#8?@x<qKwR^_I=aX*DYx{c5GII!DaTehmE3!0uyy4
zIyCq<6mfd-9{Z;i){xE5{mhTm0@5owG}Xdj_B0-z7jp%-*l{znO=mnj;lD-7|77Kl
z;?gXF*B-I>PFT{?q<xns8e*MO-h@MoWh52&TMZLD79Q18c4IuyFuT`uVZt(gwkt0j
zA3ySxJaSqn{xr|tRV=~}vMLicL2_MF{*r(E;;y+d0+%lwka9bsBDlu&M7>PRnIj+h
zmFIof?)1ULw?kgu*{uEqgVJs-*gS#bSM_Nd^>b3$c3s`!x<O6QLOfisDY9<CQQLJ&
zjf~1xotl0pG(6hm4$P@i<$yXdyIA4%fi>~}lsl7s1$Icu%xGm#YjSw}NK5yFI^Q1-
zf29P4UExMg7@t)+rf8;^fwP!XcJTuL)rZoHBqbjnUKVkC)ee^pYOFqAXDnr9&PqvG
z#?!R)L_N>c9j|8nKf=3>@8^`pty>Q4su5s<+9|Er>S27S%A)7PY2OLAlp>lrbsv2c
zk>i*OSv_EHY^;1~<CPskEX;i?4o&7OP!(YdIm#k@<bX<d{eOvne-eNEpWoQqdnxpI
z#U&O!M-E7Sb($TWn4n;D>#O|AUkBp*A9yRhzxUU+;KAvm-S_wG{dwlhj)2mIsvK-R
z6^DddfBe?`arvNvmhQVdi8pplb#!PrczJSGy5R$+9YUa0LI3|Ztzk)auv*j3n9a}X
zDW>b2P?o_7nvVYQ;bD33<<H*J>o#mQ&dA*O>4Sly=I{Pha=N|-3P)8pDI82`tXB}-
z{6Ny9;EjZHxOnp<q2`&A%{wfvZqpISX;*^g(?daP1vchL7#A*B%FM<iDDn>E`Tze<
zpFDZu)F}m}Lq`sM;&<j<`IOz}(ZwSlr*rKPI?K@f|Gy*;ujIYG$v^(jKeJfPkXI#0
z09s%;tqd#BXK^tUR%@N9Fk$Ea!U;S)JpX6Tkno%H<Ja5U_N=UCdU|Ypeg8K+{U0IB
z!ko25(fGvd-ouO2HZ<JdFUi9LT0(GRbGu`3al##FMBPsPFMnI}>i-S0A--qqs$!y7
z|F^CE_4?f0`PbL|oiZt@tni`B)z+1QuO(mAPdM`b@SK+Ve*0^G_nw$1rPDBRqGRC1
zgC{y3{w!|0y<JkyPNi6cySUKOh}l;a+D%MKa(J`mf4|JNi1TH?{>$rL6llJ1uKw5e
z&(H6#4&Q%%_V<act%h1%?oLgD&bGG#9cOHs5%$1;&5z4p6uiU=6m-rcq#svMI<z5A
z_HZjV&;5N9XWA(+gx8DKJ3;bM(joaH|NlD+ANl`3gO$_sC37OE`qq#Ab?@@$e{XN^
zkBghb$CvZ}-@>JAe>8bsCmKae=3d9r62`(8_u%#Eg#7a#p79<&(#aFP&fw2q@4#OT
z%v+RL`k_PKNrx;ZJhl(o^rJf@VZs#mi&y?Ps&xPV|NrgX-&^<o{kyZ+J~T8kDQU-(
zCo^Wu_>gg==ly?=O((8$&D-3<R;}8^y<xNQk|hQbl7@5a&tJHIzTk_-;rI7dmhew9
zQ%bn|9x~C?<iF?ORl`H!^#}5G+|Af8r|{@iMSlXP6yrPl{#KuzwO-o1Zhzt9?Hde&
zqolUambSLGzBWx^7DrLcKlevMIYmitr}9W1ICUx^=gfpjhb~+@wxLw}@b-M3$VeVD
zGZh&X_YFA*;-PKXq{dLmj>IC9sXQ;Nm;PI<rFg;i=l}oz`{n<Cd;i}s|K4?3>pBO&
z`hS0ZzMpBl-^h5U!IorGuI&1j#wjgEM<%h#eE4j<r~dqhN34ghuji@$Ze&w?YQn^W
z51t=a(DI5BpT6f8sMPuY|InPNM>BcWtp6<^s(S9o$M5g|>&O3_YyCa@VAJ|`zP~je
zAHDCF4_LZp4Nq2<&;Gi<smX_T&1STGchO~o(vBn(v$UBeM-IHd&m(VV;Be^eZS%I`
z=MtMXfoAzVyplF-Nf8iIN-W?B4;Jy?1x*Ku4!iuN%8sxq|JmTEa;*MJ;OGAplUSJ#
zgSxQycYiPc`YJ#Bdfon_r{=d9MVXj-^yB^{Cm&AbU|DcgZAR<ydX>~2L1!+oy}xI8
zXWv-{%f`Ou=EnB+hi9eR-rkms+n2(@o$TS4_Tb5pA{VQV+-E;tw}$4R$T!TMEdT#I
zm!C8;FgTDc(~)UlHqWT};pgY|*Vp~Mx98{aHIec6_qE2&IM`9u^nboR&#PBH@%#Th
zewKD1O@ukiD&c_G>mL^l7-eL5s=qnhw68N@VF5MRZ0gPw{4j8ilsvr1l}AvJhmVhE
z)hZq)rWaM5$82XMndriM#ZD$36Wu)iD_@tIkuJaZnm*5mmaosx|1W?4Z*A1q+QV({
zpP&2t@596W9UP0Nh<u)H{(qjWb4XlI56`b(XC_VRnKLItli{w<p0&IcOO0(NvobRu
z{{Nq+{JoKfxAC4@Zqdr$(-v-;^ze5%TX1k&>S>8}@#!;6Iv*b4WD5^xla^*<Wo6UT
zoAdnO#EWMkk#Q&5mb=-3r}jos(}Kfq^>*#Jt7O5k?q8Cn>j!a*)Y8%~>>9Vc#ko!2
zUf7v`n*F5Er+X9b%#__{qr@>M`iW{(>ipD70omI9k-LRt3*TJb>NRQhwYb}PF5<Ql
zEHl)^+isLU|NY|u-^0S|D-wh!ehyDKW|1_{KR8>1HBE%I$1!x_>jd34M&D)~=Xr~D
z5_v9(1gfmw8$bWQg|<JWa*KJ)njh-$NbX*7<5km)%txR|0ZoH^dh-5gH)xGs>5mWV
zkN5xg@-S#^-^|Xw`Sb~ckDrb8qP{%7vGKl>Q_uT*ieKvfo)Qo^aOlJj`Ni_xGk<el
zQ{y?jW{U&c{r$#&{+<2sPIk)gce2eJ4Ywpbc#vRVk`fS{{NZbA#O`T{*Y(<@%|Y`=
z5xb{3#>{baYjX^p>gd|)=-PVl#tg>@iKjvGXB-V;I3exKJ9F1^H77{4&hN}HsBsnK
znYQ)rfdd+B`UhY1N~xa^5H!*0x?!sMFE??AyjjHkm-@@+d}x_o{H|J?|LwNfuTQ@g
z-;>OKVO#dBbIUK6*!jB}nx9RQ2ohMaZ@o<arc}>OJ|e!g`>pz9T=Ok<-S=MAm1@Xy
z`r5Hyo)R@IP5CE(r1vVPb_$fpHY^Ibe&d@kgGujU#>+=aJ%t|4(0N?6aMg+87KiJc
z&NlqF|2_OV)iMtp!gppFay2^$xbAl|%Vgo+C~)M`A?-JsHm$wWuejMXwDvxD*L60E
zBdjyIt3$=BsKLkfu8rNEhxuB&zwawEtW-~)cYUAwQZdC9xzl&mg<Os0*cJD%GEF3Q
z)0=53XTJL$`><=ywIesOZo386OgJd9WQoBMlZmOaQJv344vJg+tM5=yJyxj9>>!}f
z*Pq?Sc%Ak61DQ;&r4ycKNR`O8F-9%cd1xUmlV1DxFH1EOc-8AHTdw8+ftKUT9R(aW
zh>IMVuN)Ck5!auzoLwNgQq13_^T=J3PEVdmcG0}iXQur&JhS*s@s1tEFD!~~=cQKW
z&RYB4Q!}h<S?6@tzQs>>28vAAF}@VCJ=gd8%a`YtopdUQy+6rikMZ$cN`0=^110MA
z_Zt^_SpTb^;mD#H7`pzc&EfBkC9`rP#F`k^btfJ$aPZP#J?|`1Yw`Y(df6TD;?yUx
zyE$6|1m@hWZTu+Q!{fqZVjys2-d4ViY}^|d3<cjmS=)I+R!i)`6$73#>}loN&v`<o
zGQ8cDAFJ>CIkD}1ut34ASvr$f8M!JJ)|lUC+I()aws+?3tDg&w#4F8E>s69soBq7U
zm32?BR-Ge@;FrJmlPe{XWaQ<ldK$NG5c6bWSXCYSJz~v+E<H#~xoO^;&+;$#7~EKO
z5H!O2_~Fm|j1wxlXZ2K$sHh#&&^xE0cTPq7^pW%DIo3NWgegQHKXKy2+uQYJ-~Oxn
z|9kkrA+z`2-_r`3Cl8$XF@NSy`<zC8=01~y227J#Wk3brcfC`QQ_h^>xpnJ=GM@!w
zk&Ln7L^I$1IWCY=w<+crv%b~?F=HcxKhGZ}UAUGuWlGA!2MJG}rapO`oRHMiUy!g(
zCQxiL7q{`9-QUAqS{~js7TNB=^YGcf<7du%c*?EJ&%e&?SO?qwI>SA6XFt3WJ-j+R
z05o|U7|52F2O15&v2pJ1e#xg5Ts%p~j{p0g4z*cMf<3eSkh`nmQVwnH<_3nv3lx}{
znO&wa>T#Ph-ZV)$_WJt&B}*me+5Od5Zho=S*uY`>?f-vHe)#>p-ptHq?v06jar@IO
z3Yi2e-@M_`(&G95KiOZB$HMHQvB3g$rWmdbUXVhx>D;M9(vlw*`v3T+`h$c0m4Mcv
zf{xo0j@JMGcmC|z4?jNIGctmf+|8IN`QZEIn%~C`9Q*eFv^ZN?8E9fXBJzK-aq*I7
z^L&XuxzrzDxdjt#Y(TSQD^@+a(zLURdF8|gr6&7Y1#Qrf-T(iGV%{{{^tbM8`XK*i
zO`*wAjZ+bk|MypYoj!f~!*6f@uUW%$!d$$t&@g8Au>)t%e)#nCzkU6L_lsp%13;@q
z54ZF5%Ng3#^$7g=+<x#t!^8XbOrng-I5sJKV&Ryj6P66|lGFZU-1DRqcp`!t?(c0{
z_Q1Z#t7-Yv`v3n<o-nYf|L5=Y=)d@tZo|4i2@|HLSA2bC&wu{^|NoM8|JAhixbgk?
z-v98rew+V%3BNfB4*R?V{Q4XN1mf+*_zZ%X4=&SKw~8HH_#`z=6*%#fMQPUm|Nq&&
zB{Mo7$X|M@He>1O|K|DsI=lYWf9YtPFMm{X%^IHl^~N=SPks2NdsWPpPyN7=BOhK0
zAMTUoxxY_w+UJ5IqZ6l%KYVzwC!X<ohlQaj2Xj_uQx2$$^Z&n7TrJ<;WiS8#|L?K#
zi9Pc#-?;x1UrFg4Wn(_P-@g9e#}9w1|Nh#)!y>7L`BBIIddaxGNeL+_760Zuyspot
zrp9*a{Vf^42any`Hm6IzyPI6`TTIYMMuw-R=E%FagI9kXh-!Y|>5Qo0cOHBFzkbRl
zMom+P)1psO8Y9mfxVdJ{|9g9Wo<Dy4!$;*`Q}~62na#})uZ`x3iQ!ovXIS&M>qSR@
zle|4>Np{7Tj)z||+veK~KKlRPSVF>J!=_1M0WM<FCtO*hq9Boz)L8EOKl#kj|L$}4
zA7Hp&$p3z`Lhi2Pk2owX+vM#fT3Rp8{qbJ@@N$2iH*a{pzcbX(HU=%fe#|~s&-n=d
zy}ijh%6cEZ4rjBp%#c$}+tJze;C(!sn_HWCzQngTi9ddcu6&f96cCUwVUkk8N%47g
zI?UTXH8V!Pa=Z*3mpdrp6s^o5-2N?p;;ZHVEAISyE8RRp;_&b97tBQN$kzWe`1AKP
zD6w^mgH|JSbSMb1+bTak&i}&U+{?>6udf@wxpVTvGv1P}IdkVWwzeL;KVQ)jw65&o
z`*^k+MW(ly($d<j%O&_^5-NU)yt3g)NJ^S8DJdW%B_Z|bIzi*e2U7nIO|`qgCB*UT
z$3#e&Y?-r=tHD9QCH{h%M%lp{Pl5f%<AwjQtlpnsD3NohQNpL(af^gsz?H)(_cy%Z
zvEFR3=5vYap1>zx{g?9k8JAyI<$1sE@(Q2J3M_{m1SY6t)TSu@f7Tx4BYgUhMbPyS
z(V$JaYhG)9c(zB2>(nG2<@27wHS;b0Z8UW7`ZaIc($_oO@@)API~Mit*vXp~{qgy%
zndxQVB)(?uMy_TDo;G_Mc6kRYtsRz~oBbbT-+!Rck=Hc2`9alo*?C^=UeT+AG|q}R
zG1>p&<ZKl9?r}`h`M20+&2SehzGDw|-N~!jlRhuKN0H-&)uN6<7mi6R$pTF0tJmh$
zO;bNnni^Xuvu5ck3AQPVay7Y56&<cL<?|PAnBw6%Ytr*0hKU~2as%rv>*`NzS|Ab?
zeBFfi+Pi?CcAiboW3N37fBf5~u*CYT`r4P(@4yaTGRK;)$w8paeiA3I+X@eh#v{fR
zJb5P+-@I92zG35*s1-U(9GUYHH{axNOn!9#+~OJERVGht-KGEaaq#g-*GUtWbxcSt
zx!aSXTYKJQp{bIKpzq^PMKU+@HcnG;ICArZ|C;Xj-E#Lrzgs7Ea7^dkE<Ky~SU~8U
z^F7MfwWExAj1R^y5b@f0d*l7!X)=5wt(WDG{}0=p`RM19j}J-~t;>uSoBpfv-M1T|
zd$zxcf7bcF@sVWD!-^-YptixB+55Sg5=4&Hch&t=cj2%zkZze4w_y2;n^g;zJ!31(
zC_3fKImKm*Lq}WFD{bjrmuJ2!*eP>v=bUT}8G|*JY%iZ=$FA==?z8;LRw-NdQ<EMG
zo>%s~P<m*Bje3_r)2c(?cg6@XU0?cogHFqW%ni9b&C47w7s&Xj-<qbdK*K`Dz}HRr
z(F~o%?=`(Awdfo!w3t7C;>`WpzMjELQZ^^<P1~BZ@jz*$-o#1UQjID(*4*FpvGhu8
zduC8*&h|Ioj~Jdh8{qY<IBm1}<>e-S_6pBG_C!=$?J3{8{{p*Z&A|~fXO=DZAqSo|
z`R+##BCjVs?BdhoitYL@A2{LBwUx~m4mz*3SfN{WXl3)G?M-btCnxYvGFQ}$`x^K9
z)AF)UF4J0bm08w#%h=D>X36ZjR$}$#_m$Tce1R^TJ5wh<uaLW~``b2XQ^baw3aXzi
zSBv%?E;*IrwDnY;yUgYoB^ED3sn`Y`&pf%c*WVxhB|rE45-&m59eQiUjw|2Qb8Ij)
zk=k;5iDu9u({|>mr+&7y7)m&=Y`He&bc$r8+uCCZ4M8o6hZQUTH5_VX+SdENzUEW)
z?61`?Ur+j!8n*XSh0prqImb9bQMhK7A$L=NNUOb^63e#@2UHcA7B19Osi@x_|3G5O
zYHw#3g^ZUI_9!eo*d`aKcJabrX7P8HHFubgSA~9GY4tYY(y^2JI-*S<CC+EhW<C@9
zYL-Tlo%;EmF*E!k*9tT}obXui`qJx<3l`irzZmY7GyARB=|hDljLxMmc~a$?9jt!i
ztySLl@0A`imMzijJDk=mpd)6&yI3&s#M`#_#<L5p1X<e3CUgF_o##F;_&N`p&h!jl
znb~3Sk^6EprKUKo%)jrJ9`W|bNp&Zy-XIYb&ex`)M`c02S~A0tyU9VIb^q5*?9Hi;
zI-I*$oZRcT?O(K7ou5ti7kgEPq2R%omk-oL0-X-Wv?b~t{%~*hc8}s3ou907i5?;w
z;})A9+q0!*L1squ6s7YQrOq$bxbgOdRi1fM!>%-m54Al9KTo)ncp&A#XO^~`jen14
z-p#pi>*T{-W%_3h&FFkk;o{Yz;A<)KqB>0b(Tvwq|H>&!xNnPI#kEw!d-+*?@ik9t
zp4HVqzi*-Dd8f$k*mwS?yTfXwXI_f(Q{QxE?n`EnALhi^a<>@p)H*w}7rshzEV%4>
z!Tw!;K}msD-l82GO&tq59uypIRhoC=ZmAY;Yt6?=)18h!Kbt;b$A->y4o@K$!A)=C
zjy^tM(#TPJUN+<Mhu?MA4K*Xe%Vy4%xV7OgJCBpWZZ#(vWuAw>1GxGQ&$r5xZeuLv
zO%4cENc!~k>?SX%!?QQF1f3|}q1SR?lLE(uZ5O0|_f@t{k#Ohxe^W{GVWmfK#^24o
z+a(L8#RVLf+vi;zSkV0q+-FESc<X_}GP%a>DIZn%PMk6;E^eEwe(|x;p}*hxr55~8
zG~V*TOFT{}U}@QdhmB@G_SdHy7=XKh;pqWEP48UO($W%=nv^=O*8elGsXkWll<V-v
z$2`x^J3s#V)VpX++nexa75%g{(6ZziGbP{MbLHVTGEUZ9@$X-1LHY5@<F@md7_`M#
zGnpK`zhAOXF4dw+Bycyh)|$1JtJOikjlY9StygTer>@%KgW(@Nd(|##vCVhlsXpAJ
z^pEwSpH#$+#;4KGY7XCV_|R>%&P40u68`(@z9)^^9-etv&s(_v*n`R=hK>!Xf^90C
zmFv0%4|T5Ve!PM)?edFVn{-?RQ$1rZn`~M8>tW@x_gjy&yQy8eZ{J|^AX9|phFPd<
z;GWp8ERx4-;`w*ynjcqAo}sp@ukeZ_{|9iyuYowRb-$@*Q<BJ$G?DL&EcX-Suc{kx
zOV=>1K5KDsnfaO+$8Y~w{AY7cw&OEy-&xjY-}qp2cSY#5ExOn4HScHtdd!65L~$Rp
z;47)Mnp}q_=qz4!VU~s6#(-1v-Ly+LJyQBytKMb6mOMl4nWW&f*1{uElUOFUuUA}d
zvh;Pvto!d5-52LuUwKDuN73yU)oqQXdw=PJi^Mgvb@^Hxcv|->GKz3=GWqXtG@a)B
zV5fSLNQR17;}iGIO`MEvb5ka-&6@T6)|T3{s~2v1rDR*~a_7PIUmeM(gw{&7?H6?y
zkVu=cY}rHkD+*l!T{_M!HtDx5;$ELSezErW=e$XI+hffg1zjZEr?u`{mV0qlW>V}+
zt7nq;&8xT9UjKRW(1b+`R(VWXFJb+9+kdO1E3u{($7Z+ftB-5813NLP<~yHZwUa1Y
z*oSAl&R$%HkM&xJF4)2OjM2<2Lr`6bNtU@mXVc%0&5q&I9i5sUKKExcG-P9DW@BY-
z(~n=URYF}Iw6?dW$7If%)-^7PiHV>UL{Gih($d-{t4sRLO-e{jt@tzJ;W=4QXXyEP
z$$x*76Ox^0)~IvLm?5!dt>mj$5^vsYP)|B@i)C}s!Nm-|fBx6Mu&IodJ-jZqqjo1~
zqK@JJ|21>GA?=joUg26I{R>`*+Blwn<Z9zw&E>yHditf*!iNp-j!fi>J>#&jtK{OF
zPYMD;|5$_`N&K^T8vSr%mh}18M;{;jd+zNqn|i;#vjqw)>$-!=EGA6SNsbg~GTHh=
z{<w?a>#XosY%P;o47Tvq|DW=;e3qJ@`jkbP1+l;8?Q+vxvLttB%nK`q<B1X<YL89)
zJTrBpHq+XVHe1v<P84@Fy)NDVS~mRHx`U0^VwPn2Mh1o2%)j68oefkv?U`-J)#AY8
zI)84f5=RuLlE#hb3cvLa3~gG&_#$p}%#Gq%Y!H#NG5F=XO*tD^2B|Fh@o@5df$Ye*
z#dYQTIv@JhGdw@J>r!cms8`{<9_7T2TxHe3miL?AFVS>NP>G+mF_(ogGL6AQsJQ3H
zdC9-*mv7}*{E|EEl-+XRvp`eju|f$p<@lvXN@uD)l2{6I>n4-)S>?MK{>Zf37az8L
z?IeBN@p8cRExJK*+K^;%WzI&f1_K_K`#~>M_#Oqlc>XK9u5od9^zIvF1p!=qXHx=8
zzMPn1di?1FrxqC-zq$Q?9>h5{%$cbcwCMfJ^pjy;B}ux93rvq~SeL}pUB70&^k1gq
zjX^ommfWJV*q*!SBp->|v;1{N^u!5LTc6fMXyngY`fYA%qJ)I|&OJLHuz<oWspfNA
zfuX}BCKDm?AHT(|9`wq5uc)#;%rEdhvPqOhjhSD%FK$o542y}0r^Q|*<@-Q-)k#j*
z&Nr+TIi%X>%_!B$ZD8!&+}6q(sKt|W>cIPiRm`uJ@uVF6p)Jnw`>A)^*VmHY-X>Rk
z=c_)IkdOd1z>!<9V)+;Qtn=(Q)pv;eIWCbb+oH<2tp4MPB&`L;2SOJbBrM^1qQn?E
zjWIf|_V+Z<rmEcA5^?*E%rt^8&^u(gok={=MV8aGuXN2z=3v#s3pkTZ1lKSNorrU|
z>}?)@pqVe5E8)OgqX)MNL$!H$c;fdP#q0);xSyS^(H&|CDS?w3dz6d?rt=kaE^ajn
zi?nLdDROf8Y&=hiG5ke>8h1=v!1eO~f!4}AEtgor*qHmKeEh6_`0Q++=<P-__I?$N
zB9L5O`EQP6l+-PzAOGc5MLIm?GZ<DYF!#Oc6l!=Tu_ow;H*3-*mawA{n_1E-o%c+?
z$Rw=$OY(%T(hSy<^X+-c-x=iGI<g^G`jmksXn!hby^LjfUt@Q-qX);W@&i%954?D1
z=<=j!+PIp}Qp~WFGqx65!`jiGY13txdXbAKMem=@fd=#5q+1<p*v{72cs4SYpK`dy
zt=}bQpBAxyUgBM=HgSCp3r<jv7-afPlg`HG=7XoWcJWU>@ugAy-Nz+tXIPFj=n8B!
zdQkQ8xcR>{mQ5T7LrynL4A>wR_~Wdi(c}mIGCWVz1XI4e5<R>sRAmVd8*FV<($1Qh
ziMLGH^z_(*gI}butbEDr%b0dyLI~qSCKh9*e=II3Txv#<kGIRNo_Rz)eoKVAu!>!$
z!eV}T2{GNI8ygy1rW`zQ0Gc{dQc@l~N!hR=;l$Zg1H(qWlEX5$4$0hgntVvaQP@2#
z;i(Ykg;xrHPJMWx8YEulF7$=z+MKpb!L&yS2c~(Q&^FswpFYE&G4Z)y8#}+mzIycs
z2`NyMe?oS4X@^UsolQzeNH8#M^<xtXmV2ac^KbIKHuZfLiV+hYOjHfxz7}yLXTxL0
z5AQT)Jv=f+hNnW9o%!O?WB2#-+`S9he`QnSaqEu(d~HHvLIP;S<7+l(h~U||)Pw`m
z>?Z8(+PdsOaN&dZ?~<4%y^>1W@<5;G!kiX&<*4i_3Hy0gTka5ImrHe-6cK#v_jjJN
zXD{64Pnk0rI(y2=Z)nI?Tidoog-u#I<L{w{-9-mQ4vM%7yWHqn>~b*l+oSa^99oAb
zGkuuR_RPS<G~?dES8Qu?l4Lq1|MJ}E;_?ydgKS7%xPAJ=AB8VY2^m2G!AVlF_G{O}
zH<6%$f_?Sr1!ZXu9yr|Z>fkx?ZucYhh}UX1l6=$knzBAfwDNEqQ%$?ucva)ckHf*c
z1cLV``0AbLnQt$7ZeHpPv)+eCIJ?3OK`HA0f2a6_KksA@uMFmi-)B(s=fs1jM-QCm
zzx#)eea2NT!(O4JB-U_^WHy<DX0mGH{R(=iJ0`Ktd&l;%lYcsA#VdtBo-BI*J}Mtx
z7t6yf=I}iw7-TAFCBvp)f1dldeSR)!SC>-pcb1|`i=$IhV=L=kg;oZo-|LPn=V1%7
zQeauD5aIZsT=ZJX!3HVMBNo!DHy(My=yP;OmY7A=myX24Y@mWD@9v^QU9pg0I26w8
z?QL99ZQRmkv|^=WKSP2`)ThG@+s(VDcI$M!<GXS&EQY~%T0(X+qcZP9_KHUm^_!b3
z8dl$IoM};DSa4PC_jlu*n}-Ts2pqn={KvDXPSAK3!~g$E59|X2*@A=F^7Gl!)7cmq
zUkF&QGvtqnF)=e@)-99t;3-*it25B8XJ&yzoYH}0yUI@HaNAGs<q!Y<&ciKcAY*^7
z;HS~y<NYDl&7k!l|Nkq!$+xp(Q&nZ-<_4vP!om#gsf@=PB$)eJnq;2-5N2P=U~DKD
zf9SE1faHYd1~#UeX^#$QH9ZrG>Fv>vmpC^!IbzR@hi4?)%=10uP96Xm3UcSE*WkSl
zplK>eNuF1)cor?{XinSkEnm_gh+*9(mYpx#@7AV#t^KgwtRvHS(iAn0vZn<%n?hTD
zOnUst6*STI?S1-=I@!NA3adat3NllXCn8c(L_{JcMnXnb(!^BK#YMpT?+2b70Uwzk
z!EDTey#8w%-Y_xj{~00MaFfB^to_6>qluYJhkktNOnev9_V~DD-aSw>Tj}yZD;uS?
z7g9i*$x|mxOZ)I8Wy7|#3DeUBL{%?vHD(_<rJ!7BATaT(V#AVU8ey?4cbh{6_A77P
zX1QUTy}@pk39n~w%T2zqq4D9T)VBBcIp?(bDT18(|G(1aRpO$l1_lWmwxnFR-u}+9
z)!b#mV)@OF4&)RBs1^zY9~b5<nAyC1dP_XZ^y&*j>{06_b{=TR@v8pbm-s!e?d<G@
z>U%GM>JN~&_bZ+|dQ?H<#DWC}9=tlWV9Ak$gaZm{ZJXC}ZSDEO#?B%jEXm_CUE>Gi
z#J!?1yoqb{j`*<ithm^p#uo$KiXFEn;l{@1hgUS)Zf{>%utV}A#F>k>Oh0z&l!DF~
z1?|%dmLE^ZIFXRTVtbS=%jz8?|C&zb?5AfN1m8utN_<e``}9iLhI4vCV@n#x$6g+e
z$0vnB18DE=B<`q~2s)JD`+LcKbxXfn!i@G4pEq?XXyKP*qhs7WN5{s4cjr1rOILLX
zJ-NdjbCUI_ro>x|KMdJl9=u=vAl$9=gZz6FF6OMPAHSzJ&XlZDx&HAnPyRiFn4O0{
zypaS=FyG&I@$l9eP=`PJ!KbawY;4RdF5bMwgjrs`dA4+OC+C+mqZ5(&KW?uo(3`lH
z#pYCo`1c*FoKlWI$jn_(`2SC0#b?mg%C`IaLFwSee)%e{dykLv)c<$7J)0RcS<UeO
z|Dsn9<nIKywf+AO+LXz{Dw&cZaptUKN~&Z_i^P(pk|LrDSF+R#9Tah2x9a1S*GKr%
zCa`p5K1$G4dGM<KU&@bfy`UC$US8Y!c#tJKY9x0}dUj5>t@^vfz8c4z2w9cuJrGZv
znQP6};K1V|f2b~irTV=>_5bdOn>V^!?M0Z)8n*@<bTTOAXyj>;Jn=-hV9sHQuU$Ic
zThl(z6Ih|OsVDWI<)=9&dk;9i`Q#S+U}DUkL&u*kab2_{h3DEa6Txl1XWiyrUj09F
z(R)cY7Y(j8OLH@&j!jfaoVPOjuE=Yz>l@!kJ((}R8gvH8|Nm$1T7ynoIrecsf0G3B
z!LS^b_w}0Do;CJd+h;g_W8$`E=g@59(R}v(?;gv0r;6uVdpgAii&(f{zVN-`{O1JS
zT<`0Fp*@R}*4;~9`^w$0DAsF9y>^^ok<6}{xje25PqGJHKC)>;?i=gpD=%_<TEad3
zgV~~n!Xp3VF9Mu@Bf^~&So)exmg*GpB~S5^W83=oD9ePM3bVgP9<wRFKTA}xzIXc4
z-=BYhTS0r~Zsa<kz~geBOVm-IMepzb_rgakUMR@wdHiPlYNrsXC#m;g(|MWs>(+1V
z6Fhe?YR>YyxvFa)?3yqsPh7C4@pqWolQ}CymudtZs%}|OX2`bJnPsaJ%UVs&O4oBo
zYK5jb73-W)7H(}dS=h(CS9@;JLI#bn(~;4_o+oD6O#kfnGq`4_BdCD;|DTyb`c>Br
TwZ<3qpvd%e^>bP0l+XkKzc=NG

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..86d225e5d2158804f88dca881f69ed3ab287d866
GIT binary patch
literal 19086
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s33iWhx45^rtlpw*nn1P*v
zfq~)we<{|*4C=IH9_imGctC;YNKY@PEbs4r59i6&x8IB0WGs<#(Av&?&M;m2apu~t
z4XaiZt<?!~(P-+}RDNp7ch(7KUiQTs&p2qy@aAo{>$A5rm21E*zH`ZutHptb<$y}g
zh1S5pgWdoC&(=`l<7k@v&i0LdOO?}JWB1#?Y?d$kxh$kS*0gVa;)cvk`?gOx5*u>V
zC*(heL(4m-PnAtev^*FXnph45DW$)$o~C+D^tRY@$?KxuWBV3A`Ey9$9&Fc@kd0i;
z4kFd%&%dq<3K0+w{P6$(iK9o)dOFs2zh8Q|G~>HdX4`}{iYM!*AC~Y^Ubiaxx~OmP
zm9LB6zd!uo_O+=?&l)}6+k5eu<<SlSfhH^V<S863`fPSNLJj*eJ2W;My{>#@(Gq!S
zZqDp;oBI~~OkVSP>-$YQhyJ|I*w+Y(mH+>*Ots}|P7rV@zrLVGLdBJ->=EOU-|+{e
zE0_+({c*f7cO#eKy#sk|;pdun&X-vw<oe9dc;##VdHcH`g@&HrRFg9E*v8g`%vFjW
zlOAXvEd3G8cCx-?V)OG=T%Xz*)@U!|eB4p_t@gOVg515C7ryV)d0~|^dj_ZGcI(B=
zpww|CM4PX{Kt%hdyUq^D3&A_?{^uy*6>7TMG$Z2R1cwvC7EZ3N(N|6Wp0${g)f32Y
z>)Np!?H7)|Xg^rRzjkVIZ?VC%N=~8f#{$NW7SDNfc2|;$cj7spPL=DW^2e=wOXuy>
zxnahj;8+*)%v%N=-coNmn`h3PICpOA)Txm%bG)6MgWTP(9dQW=ojP;&?1{a-oz2aS
zQ$?jJZ*XyL`NW_tzW0oSuXcf6QPIJ3ZR&zMES3cP{Q3F*a{vG5=KU>yaNvLScZqv@
zlPi8tdwAcz&Ay(kT8j~4(Vouc$&+U`H^+udarX7~2@k(_E6T~axwreZy|J;f^zHhz
zv<i`b?p~7r|NZ}e{6xoAxr3KZIB2Ckc+?@p<ji9E$nf>t`UzpWL6J=te*gZyU*7)T
z-+zDqUta!S-QVcVos%EFXdYf2&Xa%N=+EENA6{Q~XgB49X0x77XNQKlvuAtz_QnSX
zA3NuF^U9Rz^Z(l`Kdu)OV{<$9zo%zLU|&b+44on_hxNV>CUQ-0>NGGg_;WOPv%*rI
zH!)APw1)gXaC&O}|9^k~f9(c&t^B=&S#FX=)s%-{v)gu;OaA+lDAAG|0kQGSwAF$K
z6a-w#jb_Y95D1(p!6TcXIg@WiquwLq6&nOO^yV{ND9^k9ZqLRC25Hx-KGv1y@h5H$
z({ydPtit(2w0mAgv0LNm)lY5rZ!;|7c@lc)<DZ|6iz9U_Ys*{rELk>zB|zKbO4k$t
z<vwQ1#UgW_JKU=RryeO0P;3VJ^hC$RoH*ok>O#QDqpndtJrgGi?!5S2e(ffQM~l??
zY<zj2gsJT5Ip`r%KhgBiX1kzWE`9<hHbrR5o_WThQt9wpf5N>Vz0y3duNmIif4<<O
z(&2V~p7?zZ$DXoDKs+e*;K;!<XEc-!tzDy&m#LYZvLqo_S5xT_2WPpPn}=opG`kPi
z)m}V`;O1G=$&g*u<YFIno5@|ZHFEg^5&ILJJo79R7|!i2Zcn_f*Oq-<LN6|*;#<$d
z+uIrQHiC-J|No^<965OW_`=oNxk*atxocCBlr+_jad<L*Xk@=~iAk?xui3%SwF%lz
z1utEfNLK_lu5~)NChUUr<{7RWHnj!@e-8VI73isS%2*vKc*Jt}>S`YU`FttSERdu)
z17!BHH9C1oi(Wigl9ZyNs(O@5Q$2x|f!StWL4VTU!xLKF_?J9!_e#|0WZtRcxJubT
zdf|yq=4#32sK*XBSNO_2xx}=#Co$oGhgiYH{q>UTVv{Wjn;!lsY~vP}xVKMsh6>ah
z4M!BUwV%9Lk(a+dDN9RR|2%7Ek&%Ic!gK)-Q;us^tm4e;p6M4Ya?IVQP@=e8dX2AN
zkJVHAv?<CrJ0g?*CMW#h&R(sg-dE2fag~eNP3Y&x=7+CB+YYx&`prwRDDQjtx4P=M
z2-sJPvL6YEKfiQ6yrksHhV9{Dp`s5SCAA#=_5c6>i?ilfL`$4#7XC0vCi=fvw$!1}
z%?X}XD>~Vh`Z>NT|8vyGaE2+j+tyF=_dX>Y2;6x<Qr^xWX2*eo9|ni#+47v9XIS&g
zW&aO|mlnl8x^QL6hmVFEwufK7EG#aRl$`SC_=yt@S7tcmr7Te4u8?L@`z`Lw%f)Q^
zb-~x{4Os`L2V7`7y5XC9fG|tcBZmS-Q;|LU4$rdKFsc5((VmKfAZr?#dEEO9_Efnp
z*I<Tt$NiCjNXqsN7dUuKHf#!-H1T0tYHEwC1an^(^V*#awz3JY7bZMEbW802MJ4Hz
zEN?PdlQ+nC|7@ISVX*X{lBo2F*qDqT5Bz0#HYhzg&flgTF5x#PA!1|G!=J@%=jUh2
z6+#m7qRK}B;o{%EnSK9o;nL;F51ytrs&l&?`_IF}<6$Xn+_^wR?nA=SP@w}-V*d@I
z)GJ;|P5IJj6~yN$@Pc`n#8eI^nVm1YX07_CwAJ#$DG43fs1zY1uBM)Q`%*0mn-Xst
zfh>;Ovmo~lq!e6~`)I?Kke=@14I2ZeO@E%8nCRdq<{^0gz=03_jg5?!Nd{@Q8@4M3
zOkJp|_T(J*yp|^0ix0AL6|#>@DE@7C=Bk|`dNPHnlUc9z4%g4m?GGQhw!OVAxh*Gg
zM|pqSMNkXl|Nkws_VTtG@U-mjR$^PQAdy*xE8^i*okJEp$E_A_$Wd@w?{LVq<X@+-
zruy;L&*dwpr$$}BZ}BbnwgqRC%(Xt%bhk-d9Fv5PIxyW)@=Qn!Z(!6cDD?i=t9s~6
zby>4Q;rYs{ckHoOPig#DmDdYf8OE>GsT8>OVb?}Ah4g1L*Ck8bF#GoRuk_#P;1X$3
z=pzrmvQuYD0>Z@7($iBC5*m)k1kL;?ztXJHRa=QgQEOo!hqo}#hk&${BclE%Rx;X#
zO%N6fYD#eF(%Q1(Z-16fm*xNe|J{qsX2o{2eEM&1^Kg;mp(6~k9BMK%CZsej3ksXo
zIDful%$$W=r#?I*d8lmyD5x3!|6f$J_0W+c2hN|LIQ_Z2WTuMl+4E<5J_zl3(qL{m
z@z(7Bk_Bb%hOY&ul(B7N`mcZNj`Th+mSC~1Z#!f4KgmdE$$k;f-6xm1hG%n;;;J;q
z)$tRuXU?~me0Mkb#ft=sl8%SJ%eDBoLlbM-qlCo7ppYkL&+~u&d|62BSyEEdQyq=J
z^CwPZ43jvJ%(u4H(PuJKw?)(<$J{l4jvBB`?2*{edwlWPqrx|p8ePA|&(P@#{IE^e
z?KiVv!wFt)8>JIXANunA`42C2<~ctvu=p6LGW-93QPiV^<YY6WN#{;|DlZLv`uusr
zls_9D&2417b|oPp!NE)Thd0NUl*U;{Cr)NHnKe;K&Ob>hKt({yNkHsis-$G#FH;M(
z%>PQuwU#uoq(3~^FiXV1Gv&)W*~3px^6V}*wy8Mq;kS9|BWRi8H@msH`RE}hLw)@V
z*TUw_o;`Kq#7@SjnZ}A&r+k?5vA?lV(Cp<Ymd8d01_p|zMh_m%D7D;k_TZ5NAH>tt
zyqp>1!Va1&+M4o^VXbO6(}ry_fx1pYsVPTFY*w9?ytwDXfBRX<7jloBa?M_M$R}Aq
zC`m&v?ZM;Z8(TUbzKd;}Z*LguX9UToYoZ+a4hRUyEPD8W|E>O=KMVI4d@()$an@VM
za82K;0+qumn<tyjj!58JJGHpFXZiPbL+S55VdWdFRO>w(x|(Eo-YdFxXlZX`zBlWf
zP}0%h!(SEJrTX<&{=O1#@pcUpL!h{ErRI;XET79<M86q#eOh^*arp)TN3V!(fdwK#
zPUpDm<l`Xal1br)O-9lZC&b0rZr|44uzB+#FK(A#4HYaa`I++Tj1B~GGTS&hoN@7E
zToKFZqRck$?SxALUb7rT-HlnYHJQR2G`nNi5_}6Xx(qMtEowh>;J}an_R2-(qV<i9
zJR4TEGpnhw`T4c=@UW$&wLLv8`R?A*Eq_5t`v3n$8igA+7-VLhnlhEu%1Ud~rcDY<
zdDJRSHHgYxu;I-%H&8S+eXvSkDr-@=u&LLBxz1;rq~7VpWHP<uaZ_PSD>^83IV0-d
z;+?#jx<&^c$*p>7uXI*<4%bK4O|cQ}<^@ZcV?ud^g_)(LovW;vnVAnavGVlG2Vbd&
z*sf#`wmtLYl&O{0*4mpl86_{_QFA%u5Temh#-{Qk;iwLCkZo_wJY{t$w<Rf!Icj&f
zVpJKUraaa(kl3Kg#Ktz|O2cuklyJo$HY4%F2X~xFak$idPEF(Vp+gg9ocQos{P3rz
zJpFPL=i5QzT*(-0VU|}~npIWx>rERKuPc9;a<Iii>cXy{_CdFq<i1-J+uO`f3^5nj
zJ)xGT-^QfD^i<(gu87B+*|!d{ZV(CIAjZZ#hf(<WJcR{v$7Db5*pcxlCWfc8ljr|G
z!!=PyKfDl}`IQ?SuSrVI{M_7=XU&Qz+7#>2!D(&HTy$Ib+9U@(n;8l`5mOJg$Q=J5
z(88dicgM_ai4e!Ep9z(M>lm~ZRxm_P6P&Fer`uy8!6Ol~;&)=Ndy1LWArTj`#DoLu
zc&6`YU~qO}X<o12JVk_A*z)5UsP7_$g@qTdS>q5fW9R(&oaW{#*A{MM^=J3cY?hp6
zG($pi=7eAdB?ZAyek1W?&6$m?+SjJBuq2$l$ugNwu0ir+&yLiIk`cLyF5Z#_u_-Ft
zJ2+?Tn!vhrspOwO5;?h&Ev<!SMIS*{G5r6pWLe0;QLt;ri(kKIdw6V67H9S-<xti+
zsxV!w;}i4N7KbIGChrwn71Gu>GP=K+6cY8okVSWyq+y8AWR{J~gm(A~s&f7N)XW%d
z)i7VaF?1TEra@_`<e$HiJUkK>79UzU!ax@N|F5+6?7@>Kv+~vn2_`OFb*kTVy>h5t
z&ju!DrN8{H^8{^oJd8?M?EE2+<;MlqDAm96?9#jfZ`LHa8SlHsuu973Xv9(p!`n(9
z%FMI0rLVqhd6sYJI$^c~&xREnHz)u2G4bKs=(f|-J*90}!BM89Sh!)cv9!dABgbY*
zOM5##^EFt_z~<7(*q@+wMbKbNKym|roq>_j0f~sk?8a&n+V}iRK4MiT%(0B&5o1x)
zfh`<&_ZzS&zp^-6AJlIC;d}f9Mq}gVZtlaUr}M<`Gq|(+_=o5G8|5HHypm#}hZh$M
ztK{ult2S*kJh144{hqn3<%-*4`d+^H?-8tfg6je2Yx%HZg9G(FOB~|X2ruT|%%HHZ
zC-K0vvcOA*%EuJg#ln-MxgCT1O;taBFxauf094q1QayaU-{r^D#>U1*Xg=F^_Tcg3
znfdEqzFMWK(&FJiVNaink~PQ1Su$Hy98@%CC`M#S1~H^1HY{g4^dyKoVVj>spx~?|
zoqwwP)SEQ>)-*6`Dm`fO^5WUIkLTDio;PofRB=KAOslY<Xj4RlMoYW2mgdJpTs{9i
zmMSq{+F=nhr;*W$t;_6l=QN`WrbY$^4?65kSdWNki7aDoI_ndYbzzA?Fb`w$M-}ch
zFPj;ovmB<icFdk(z`|m%Vzu!Gf$&6V0?;b-@Z|jU>y@OGmyz+}YKEO>J9dTqkY6*w
zVP<i=xz2=JV*fiG3L2*RZuoFO*+WiYhKqr;(vB&D(OnO;6bqJ$`W!iSEFt@JLh8{2
zXMLxrLqlYa1q+LXtjv|%+^Ii)%$RUnlIO+Gd5p?#{}tbw|LNZKj@wdIk@?ILhpb-*
zxIUZ^x}2vVXP`8(;Pd~(QPLS&oHKfsuW-<o_|UO<rn2hM2XD_NWO;==g*0cCCRp6K
zG2_dZls|vBii%bmc$H3xW6QCT4vbQmAJ&ljpD``8Y4w(bhU_B~!#;#~PEp}bac~oD
z-hbx6E8m<lfvm&_nr0iau3t|tC{xS20jV~X0)BRMe6X}${rY9+<_!n3Sza9C-g1oH
zr;9Tz^1$-Ai&_d>mM*TiV9&<J_G4GO|NOez-`^(Bojb9;ed^Syj9b?vHQel2&DtD!
zXd?TDZvh<Nk|ZQ0{{8=V_QTK5|L5EPFD^Fz@X^q~z~I0&Ho>WOAHD}9EK^(}AyIIu
zsKCI&(r}Lb`45j+8~@&f#L1!Q7CUxE>gqg^l@)DmRXn8-$h}cm?)!vS&73PYHLQ8i
zbbh1ZfkWJuh7Y?u*erhkuW#F5|KG0WN8QCm#q4}{>c79U78Yh!SMT0qv%j@<Nptg>
z#zu}aHIux!n_NW>^7!TmHM;!YS-k(-yT6r>j;wd@|5pd<!q>AUCbGG^Kl@RkR$0lW
zrnX4UuESG4L+Jxc<IJ8XPbF<^B;MUm-%-GLe*>sf`Tzf+=@u#~EU__Hu3TkJSgx07
zCcfcG>5@atsi)>OGVXlyvp-kyx%~{z|NsB~{{H9Z@ulA9&(AD=_V#xEeEa`@{z_Z%
zR=>S9n_E0i#`f2ZyRnK|UUMf;?(FPjEKgBB#M19yxZ?Qywhc@!Dt~?-|M2$qe^%~&
z@9zA3d~2)y`nbP;Ke1{ZJ$0(!>#O~u+7iM-Gm?^LE?KJSB|jsGheyM+MKJR`w@X)N
z=fP854{sQ@WncfG3Tlrr{QrMwrbR{NO&#5*XU;6yxl>Y7z&Y-JqFBN*p(m;~JqAK@
zDH*ks>u+%JYyA5D{{G9${pvmzo3gLJ{r-OcfA`*w^YiO#|NVJ=w0pgNTutr&f7L~s
z%<S#C*W|b5uonGn;o%VDIl|zkQf%OP?Ee1$`|JPb|N5dIy}fRK-QV-m^)<}?{X6^N
zpY7px{{Pig|KfIhSihj*<e{dCot<ZXv7S4&CUU~gjMSb90?qC0pg!<&|Hm@Mkh1g8
zObZoNR#}-TA|g93aPi!DSD~g-SkcJ1-u*%G7ak8yX5GSs_3{7y|1W=e>GP~vvF`nL
zfBwm)ulo1p<@KG#>+Amg`TYO?{xv$Xd3j<H50s?4*&k^Axo~9y^S{5zKR!LZe`RI7
zS<a8T*Vo?5+ZXJ3V1M}ccb@2Nf9k)!%8!>{yil>~Pa^AyPKM372OHKt0rfd0Bn@A@
zFo@aHVGQk=9g3Ut@#Dtm=+iT2coY;Q^aVd$6?4H{(!l6nYiZJdrK?giG?^`H4ffRi
zHPz35ywLgk$0a-*KmPqazcu^)ww#^&OJ6tp&)=|ao0`Pmg47)h>r7hQxgYN?|8JJ}
zXJ78^@3*)Af8ZtQu`j`*YKo%vYkN<2?F7E!hc6!`@pU*Zsi=^ssg?ZlRdPc6dT@W@
z|NletERG!cv3Bj#)Ktqt6^<`wl|S@p*PP(K*MGEZ%O(4sDIfW_J${^@{CM5t<MMxh
zxn8-nH2?m;nEn6qUS685A6N7D-(UW$GYdlvc`Q6kQjfUUE43u@w9Jg(|F8PbkMoOM
z<MZzS`~25BF!lU=d!F?3MkXc(H9rr1n9m`b@pxXw-Y~Wvl|u~^y&i1(^^51#E1orL
zZs<ZvutSqT`P;`wg_q~Z%^L=nS<*5aSr2w|Jz2;nlgX6$>HYos?{9W)E_vB%U+)p|
z<;S<T=cBjV<=iOvf4E)VLTNUm`<=-lT@SqCUOYHEdw=$|n!VNEzaQ^c`SIwye4D7Y
zMA>__GJD2#R~oc^9~iYvo_z4~WXD+9>TYPRRZ`PAlb)%mt$)5Rrtko}`h~K7$6xCN
z9JefK$ZeKNi)vif^Z)<<`@6rFU0R|nZ!e_PVR+#4^ZVO!tABiY`h9(Tz2Z{KH4M6u
z0+C%0m|kp}Z}+$Q%Zu;V*51FjN3k;SeLFwT<}|~Yy<Q;?4>qh#J2)ljKwz4J_J%Df
z0U<~1ZbO2=$zNv88lR}BMQ(1BWMp_Gc)6!U3SGJKL0;4M!7AT>hrS6Vd8*d`|7ThG
z>EXk}{2p8Z96$d2{C;}+{eJoSkMCrQPB$k6ItD2nP}!JK^YzvHjmh@+_Wk|+R=Uuz
zhs$qHf<?uIIY$r72<+Q`#^KtDg9jhHJEx%7=qd)TRh|50oSb6B#DY$plF69ZySC_{
zj^YJw{zn(;+1l#)<@ep+`T5#%e-&|Qfe#N4pU=H5SO5PXd)Xr~ITpqUX(b+s!v2$P
zVLd$m{{KJ!;pyr9y3zX<md13l%i9~hxp}zYIUi?KfvLUW)dyN05)u+NHWE{&Y*?ZK
zDcYU<Wfm=}v9c1fvFVXW?7Ftd#d-txQXxg9&IQ*k#Pw`!%in2ERMk?e`2X+v-QDx!
z_Wk+x@v;Amhdetz|FX}y*fX>3U$RERfoVnyqVDbg|M|_$_w4-tYX12Y&1EuuVSo7P
zX`bVK6PIz<&lXwB*%WFmF5Wy%Y}!f-sO6qAH8nY6Vvnq>YI=J98)%8V$z<IrbkIY%
z{$rSGhsvk_|NnQ3&$q93`zM&OzyAO4zrWwVy*=Nq?vJa>%!3VQgHHrSTus(;@YHyJ
z@9*yF@89q57s++1`P%jHiD%Mm*%J(Fa}KC>pK-k=Bz8<ezkeO1o$ll<b7$8sp`@e_
zokCB<*cNR$U?I=A)YGwKW1~X-zdyC_?p!`UU-Opazvt)wXJ5Zp|C1^I-D6Ls8DF`h
zekC}Bx*5DQFfcHv`Ss=fudneUa`QiZRzLjlvC8{KH|e(xn+tiX6JA}tk}_eUqd7Eo
zJ!KXxiiwI+($V>Em?T(zwyAT1ctL`J!O{iWY^uL~ytueO<+`T8hqt%qYlq+a|KlUO
zdsf1Mi7o}A6L<_vXRz`xPtswk{r&Czr>FPr>w}*0vbDAIOTN3W?ymfAnZ~?hnl5`6
zD>I9WcY#9n|9>YHzPh?HX=zSV)3PTAoZglsh#$N*=|}fOsSiu0*2~-1DKBR4W~=-2
z<M+3>{!1>;Ggv&QF`)Nw!$R)JIgFu(yJr6X|G)n4uk-KkPf_dhpD%H5pK7Fd)NJO+
zIS(2nSSqR)i_1Y{PlYcpk1sV<u<qag4Q%XD)t~iWHQDR*^5o?5HHDtBXKSmUZ~y<t
z$L1HTpDhlokN<C1_lI@2H+OcHgEqHh!4jc0rhM0o*dh%N|NqZ3-~Rvq&+V6v96aD~
z(eU|?_A`ZBJtZY2mn^v;3>lM3YVzjhE`Ifb!_ZK1yKzCN_W!`kEG9b-_?an1^e^bH
z`Ty^?{89J0e}8>_e|h=+`u|B*65((D9iGj%;gu`fi%p<m_dh@7L;GTs-j^iVC@=i+
zTEt?*cK66Q(Dd8?|4nLqd3j;Q#e%7+W>*S+woCQ>pD=sQjI-P_%}+C~@=jaT-rL(b
zbLN`0`y?DcyqBN%rf)0T&vtX(g)5KiOp}-})zH}ZQ779Y#&sw8CNUiR7<`7$ASGNE
zG(*Dh|9_Jkr?vI$HER|uP>5RjVLs#YGm>7ROxA`HYdsE}`q!Ox3zW~9nAX_XOlf16
zmKHWQ=f0#Mafgd7CzUPj=_h_=--IwFo{A77qkqpIC#R$?b$aE{rx?U&nEYx-NoS`a
z2d5$nq@CNO#wRBi#>&blv%KxUyEm6X;g?fPrCIZ4oMO$IvOr<?1_Sk<4-XyyfkTH*
zoIl@n>siAQnOX5G`X0ROcl7Asy#B@fV?%$xqf3j=UKV$Y53iFF6E9qIJM#e2he=9G
zNJvOc4GMa6@#@n<9F97IoQj>&YR64EzKQpW2tN)8d!}w|lhDxH>lrz-y|-5|+OmDW
zs-szwgW0sXa~)k<TYGz*>J2=+lO>B3I3Hw9ND6JR;W~8kWI`HS?Ef<d4nUF?*qY?z
zut32HlagH4HcvdhZ^N>w|0~+vH!kCU)5@-{zIo|V6?S%4_9=#jhFRI4KYrRIFr7of
zAWJ@GC40Zf9St_Nw*P<r@a)>P=FAz9Q0+5kc&1I``6Kc~#OAMH)8@iAZ+Jd^;<2#_
z&;u91VEYmi#e^R}eei>Q_a$*%*#%1_*7SDn-w`dH@wR8b#Ib*=DSy)Q^cF5T(ti1>
zOT&Eo|EpFMbarkO3iNoeGF75uh06x+t`37$D-DA!&M9i0dho>MnWN1V29K`9q@)KA
z97>=QJ55t$d^T;`^TADZTWIb7gz_1Gox3jXUm?m8mHCl<w!%4)#Y<Hmyj|<C?f#9M
zGmjj#Oi8tLb-7WX!nEZ{-|9;TL;Viy<k=zA)xoiNk<#WF62c#?ry4fyv~2F>{dwzQ
z!wJo56I~u2o;!DVii!dRVWF9lA|R4-`N9K-b&emV$Y+>eQ_%f(FlzY&FZmbBFKlei
zR905S#H3u)iHSM$@S)M7ML8?l7YMKMRZ#kIS|TRqOUjWGKjJ+)47IwN`}@z#YMnb*
zQM1!AY?{qo4hDbWh;)^L-U&_&_V&%Qq_%v3*_oIa6#8`Xv^1A*T6<a?qJ_Icmj5|^
z@zS5}B@F(5o92WqXfiUI^YiDflonQLX=Qo&?yXxF%#mT=7R9`|=tz7>f|`DlCg&>G
zs96^;PW1KY=-iW*86fcWwa6dO#zw|kZ*F^g&}hh!zcBlfl0?LwiHW#O5&ocjXJRm?
z%t5s@#l?vgTHG<BT-QD^pVH91T3#|m$+Tq0jv2<rqF1k0?$|lA;D~KnZj*MWz?%3C
zV$wVUdg9{EU0lg-Y`Z52oNR4xVR5dvUu5VZ>aZc|pib0{o01k53-(4L*(vfQDe+*a
z{QtuboP-3FHf_AB`9t_@e9qDH@oa7NO-(v`_oi*zCiLO81qV}HoLF^rpt-rBr6nH^
zd$O+U2NjMOPex0Y4cAOLl2uhCY;7ZRb0Sz+Jw4=$ig-RhGwkRVUf;~cQ2AJRPe7xL
zE4$<(NZUIpDJe-r_^}J0mz&!bGZwF45j&o)Id%!U0YTn^I-7p|ViH!L_4~&Ofy*-<
zJxK`*ew>+i?ZV~9iD}1tuFc4lHNN}Dwd?)<9l`?4>sGONzLIA?T%Y!=e$}d;yLacb
zwfVS+F;`Xb1TQz*v1^jl&M8bWqP*tjTh2m?n53k%v?)_l4|6!WI5lqA_}2Mnd*sW6
zCCn);p<=qcf76qGtzUoMzI<J3;YPK8zgqkIkDl|ZxM#y08mgC;cIFaaHf!U4i-Wh$
z9f&%ekmWeR$LIgjCI8OPKK}5@ku#^e&vX0u@H95^e12|RQ7Kq`i%+#-#zH<eHa11Y
zj7QL5fA~0g;*^&8)_jS7{{NSM+YrdnEI5-<IQ<v@einx3E9dffY!dh&Kechre+dbR
zf88A$%F3nL*iVaz&fQ)2_wjW7`Uk-e*R1JbVp^l6b!XnZM<-8idh{sg%o&~?%F>o)
zk_IfC%Qzp2E|_uV*thtwZ~rH$u9>WEoOkEP_h)DS$M0L9BcGCzA|RGF!>IM(`ToYq
zmu51*Ver2#{C}5LVj`%^8WzR|b?L-Oj|4<h6B2&3mzdaoI@`nZ|NsBPHWOwsR(YwU
z$m>MSa=2&Td-SY8@I()`D~qJh{moBQ+>t;3a4YxwKDpZB@9(DT$JcG%Xz1y&Av@!P
zg!E@^&BBcv4Ijk4?ot=qc+5nu#XdDbH{_q8;os*E6IZNA;NtjE|L4bkZt;Kib!R^O
zwPv=qZeFC++|1m(T6^m3`5Ft)bl9@~`QNXf>QKVTZOp-8U|?(vb?n2($>KtXd?ZhN
z?BzC0Ft`xJ92Ir&QAmy}2Ul}*^T9(-7Y;c+xDW0_{{8-5URqjNSeW_P*Q-a4*dDyp
z`a`6_T`bVWF)BYQiYF?nr~d!H_B->A9CPcJV$oJ8Qd~Ac094Qa|7QrbE-^j*>9c?Q
z-U2;5JSWNp3>9M?T4t(UaFH;$#GV#)utfISA?8C4>o_E4%=ou?v+*IGgQDwMxY?el
zsTgKHm0wl;LF0fjcjk&k>n|)ZR;W{!)z;p^2AQ2rN=#2rOlg@enK6}vxk$;uOM{1p
z=YQc8c^(g*HL1tyb9(vT9A)ckV$|i6@X%M`n8xBF<*}o%Yl;jz&x7?|8=CglOaA+x
zj&P@2ZovV;WgP3QSSO!tFt7dNTx@giQ}c`?jvML@PFCw!6Yt?=m7&HXp`aY;aNz!a
z(9keUbF#BtAA8XpgFj-0Z%#68U6<G(`jw@dcLzsNCZqVz{*Y9I1#Grb(`S@4sGsT#
zbO^e1P)B`&Br|h|k64DS)Q)NO|0VwYO@^4(q|#UKv6aoMv9ZdXk2&jiGh?}C)3F1O
zDz2z;q&;jj?fi3C;D5U_@4p^DHzp;umps#^G;%+fAK}Qhzux%I|ML)2k`j}Coj>v8
z{r~zCKkIo`aI#igixjRGVrCA~X4!nR(Y`PtRpGd;(PMd`X2x=nBrYQXFV!O=LPwVH
zpZRwD0CNm0vt#Up<^?dr5*o~%Bn_nIu~=*JNaV2X$!uZ_Wh`n+I1pB6kl-T0oRwnm
zpu;9+<7o%S#zt+`#AW@Cj0L=<0xT1fnqlLWNr|bcEmxKQPvomo-Zc4;L)aq+wiz#(
zXC9el*Wq~NOHzksg@+WM<c~g1nF-U46m+M~w_kV~md6^;hC2owvzIx_mKHpPQSFEP
zv9trzbP80b{J*hdM;9+!k3^Ef75{_6A2S5cG(<m*S6SWp<`-XR!<~k|KZUC-G$9Ru
zpP!%0uX%D{zR`y54in$`raJg?v7F{#^OE_T+6=o1i>EF5;+V2ue6?GOG<)^u$sP?i
z*Z=%Cm(gejq)U<1cy>*azrc;EwE_(L=kzeDo$HWtP*LSuqsl&~TSk?qV)9XYm9Iy>
z2S{u<`AC2A1Fi?R*u@)MER7B=Wf1=Jhf`J&TCO*){e5V@k;H{l{5i%Oc~<bS^?Yf3
zDNvDOyr{^~Go?kdNquU9FFOl2XA%p0eq-ZIBZW=||0qYc`}>`%wV_e+^E>}09p+o^
zsv!#v4xRgz-f}mcVSk<^Uq}9gMN@eT1lPE7{JV0%!R2>z<C-N08rHHoZY$4VXiDXj
z`SXL())o}5|Nkfb{4E|=<k04|aJr>a7VjB1wl`V+8BGmS6(WRLKALi0GihvhG?;22
zaJT>HERF(I*)0#JE#xg&I`#eig`Lp0;?M8z-~azV!P@ZO-~asYuQ@$C^3l|_bG282
z6_2!dcGN*1F&>LW0`(gm1G*Yt#GMJyGU#Cc@a5xQ>n+<Mo${og-}(0y2(3NXX!DD6
zzw|Ul_eVeamDdVL_U#a2|CiLvXsYz_h%Iwp85?sO$H7BsGn(uFOYEy_-DU|7iudpL
z*Z&X5=Sz24rK9jdo|*ZR6+hda+VCgWRMi@~oTp9x^W$TGGxH{6*|l2|RFn!6L{)B_
zR`4_|5_HOEm}#D#rk49ys`bHf{<iY>3-*da%>|9bE%*PQF!x72Pe<zE{}WjY6=G%w
ze0X2}>-d2W{S!al-<UjKTYLT-tFMylt74A*J09>&{A+>4nsrAHA6+GvC$;zB-`_m8
zwmi%I3<GMO2y#C7U*G0GUt*pelH33MeBaLhpM_O&Vw>Iu6VnO1w|YEfXZGPWdThT+
zfq7NZ4(osUcYf6U`%^9>!^6gwGiQ#E!PFnx;x?&l%$>R1O|LfYo@1Br*sbl9SKAB;
zHgE4;w|>8qJ$$&G=l(uJ0|Nu7Un)O8d;k0Ue-Y7#-6uGCI+Y9s4lmxo##MABX>E+a
zo$dMm|NZ{{|N8p>lO~<{|Np=9BqwiSo)sTkv}+T#DRyK!Jn#}=zQ5<f_4I_))QXQy
z4{sW?Wo8PQB*{KeNSnvH=-E~6w&&*;UWN3sl79aB`v3R$`iaN1{w1sNT{)(?N9QO1
z>udjx2Wl=7HPGOz`Ty_x&(HUd9-Vpg$P5ch!^h0o&CD~qZ~7g{nJIMnpF?&iPtB%;
zgoGde=Q}zyIEGGrcviYi+T6oKrb3AEp9pjM{+dpU7ebIc{p;J?|I7VPSl@Wy_^rS3
z;s5{t1wMkjaMM7rNz`nX6wB}b|M&m@_jhl}%ijl^?=M*zIcKh=;kP@B-j@IBS8P39
zW^;SRhF!cr3k07$N^f9PR&HLR(!5gha4R>@^>v0df4vTgXnRiHn6dK7|N6Fm`Guui
z%*@cF|Lf=H`}6JVANmGLG%aKEn2_kp>k_lP$#`nazCVlCzn8VPPWIvHZ($K$^+vvj
zrE#X(2D5}tPxcH={>_#~5e+*mn)li^uh4Lgmp{BU+a*)6=)92Jhv)o<pP%nIycOCG
z`}Oy?J@euF`|AsjnOtg=`CxJQlIDTEra}{~|NsC0`Tzg>*5!3RbASE2wKZN(PiEo5
zn3iAGJ1()z>z$S%_3&c9_s0a5o*o`YN1nDe9{qTuFJA<zJ^4TC@~o-4zdya=uT<eo
z4QM!ie*S*C{{MZozuTi0{sFBhTJ1PtGHY7akKh0Q$LDA=*P83N++6Byujcz}N9JX7
zNy%qlO5_|J+4fk<6&vfyO|7^jb}cHcKvZ-_Lx2CmWo?epvlH*zwRujK(2G}F810=k
zTSV#i55vQEcXwp3g8HuN|G)Q-kN@9S?|%N*irFs&Uox+}$*9`7r{Vea_4Vrhb@`b;
zH{{$5XKVW(8F}a1x12R=W(0OedI%`{^}LzjDORvx{rL$K4<0ywenX9=bGG*3iOL;p
zjCBp^`YP+5eEM&Hcx^OKzntQ?FR+s4@9c-?=I-BL_ct{mq3NsFgzs+7JPvYi6C8vZ
zrTQP=x%b~7`oDeszw)1-@;9f|R{r>)pQ3p3^Yi-3%8l2)MC{i6Z+PQ>fa`}6w+UDI
z4G)|?ec<%z58reTPuJ&}ZEn1$>ga?C2S8(fYHf840yjfBi?k~L&3SlPy+sEc##iR<
z=4~+GagpCIxQY4jMuFWI3>cIzED+hE$zgIs#DhhME9y>U*~@2h{dWn?c)nMICBWAH
z)1=htua0uAHvg$DWZ<;AaLwATgtzxfqi_F8<cL%Wb&Y+#>Qh;W=D}qevz>BGrlvff
zuG&<r!+l`arSCq%k3B9Yoax?aQ<Pt$AG+uGPWP&xjG)z{Nr?#wKYo4HSJ9d|L!epm
z#N&lMAn<?Ux3vzv&1+sIG-NrHzrMf!zkbXQ+w^nWWv%P%s=j>xU(Y5c_M(T6o&E9V
z^!mi`$Yfc*gZ6t4Ht0@GaCMgc^SgaVUYu*IqgU6%m%(l8^x7<oB_>UloHJK)%NB{2
zmV$dyJ}I-AmA*dV1eH%GwyQw%QDQ<u#rJpT54ZnUQF-9KUt+?tM8TtMH8c5r^lSF=
zcct&E{r&&J!TXcd>(|HsPrkozzKl%Iu3b9Ywyk-8@9*txyth1)l9DD&PyhA#xjZ+w
zvYS;x!^T2U2_9v|NQVRS?Rid5bG*4`4UeiSPyIgwpLu6MOa1Qb<Z0(OmXI)zkTP1a
z%BZ8$kb}eE!^el#`F#HsZcNqYDSy9c3T$L&ip+oiIX|-V&b4eY`5>>Nsc?SZA+{3>
zIa!VRxmQ>HtN-!g|JT>`<?sL5R)0HxbMyZ7aeqI*xEL=g`fP(inY+80n_Jr%pCh({
zMn*nKNjw``-}CGcW{+XnDf)AMKy&hs2624}7Z(Wy^O^T|9!|(Uec;%!1&a?Ce9<_(
zEte<#oWUH6g9i>BQcyjrpmj<?<;a3XhbGKCsi5!w#zn74sPfN@hi{{&h#<o9$M5M6
zKR({S;n=hKB~uUgtY~H5Q^=+Gzy9~P|DT@jpKboX>dTAwo74Z#v-|t|$4AiYVqu}0
zt?jfXCO&ufFjdtwKEAe{#q|oefBpJuucyb>^TH)LA-CN~VXL}?LDtMf)_;GKK?_=s
za%-5|%z60Ko2|5zEi{x(Q<E(>x9#t5Nj~|s9Yw7VZyLAV-YyvtDaj|7^5A*;iPOms
z9xASt|NW5p@NDxJ<)@)#Kw?5dLQ2YypP%0!@AqM1>gmxi*7<)?Y=Tp7(>5~+y_g@y
zrLX#%+3V|nAN%n6`TzU-B`Yc=b#)?QVk2#BBBoBAX=y2`kl8VL@<aPc%%-L@>;E>%
zTO4E9Jdsg(?E^2>!`oIku+{%Js`+!`!#mk2r;oK+Y??QF_Q5Mt4qlnEa7D+%qug!E
z?h<l#DHVTbJ-lts799NMDBHitsn+EZdH3CV3LKzSC@9bTo&E6X>Hqxl|AYk`^2!Vr
zE$jckLiB9Cn;U4_jg!N`!fMl}&&D1e1}|T1kdQRg)-pOM!`^#Q-u|C`^|$Z8N?p<?
z&7SQTGw0#`{q;URGdh{i9rcl!vGrsDn|-~}96R3B;zC0MqokeYvu4h844QQCaP!00
z;cfZ%CF=gA2#BUWc%X16)X`gYd!FH)-Nzfs&O+vqpUk!8YH{Fc-QTk=f=8JnEY8uU
z|9ktyOjbb_)95n~_)nYNey4r8wdKjg=@Vpbm>v7bk@&t^xb@oRgA7YzKYy;!RPEP_
zRge6p#kFzTs!7WXQ#F<P4OS<u^_#qHZN>ba(z7=5=InNsFU{Fb*)L&T;Kgyh{8eeH
zM3Bz=R!5fP0>0;U6JIJ%V|_0#+}B|aF6o-)`F(wP`G5ZXe}8`-K5(jQ&#QLEpbCYx
zQh)eoA52e3Nf8iA+OjdRX{(Q<gh2bU#XJ6;o&A5!n*R@{S8U#BcxTsF|Hzq)Mmv}|
zxg<KbwjVNZ>f%cI@LnD?Ir;sa<IbQU9{YNuKfl{Q+Ze7{*_aU3*kNCDqTsjLw2k4A
z*lPM#ZC9UG@#n|?@9$4Mk6e6a#(`yT{wJT=_UE`n-v53Zwy*#H*DEgkzufO{qj=>1
ze});^#`ku&+h}$zE6?b!3}!VjG`zF(@P|*TQ?xZt9yr>nnwr4Z_*m1pKvZ{ozH!Wc
zzPn7FJW#vT`t;)^&dvLKzM&*qy5<{8=`UVYvB2;O1$99e8Dpb=yK8^1U!kFFllbYA
z(Vw4(51jiaUq0(c{}G8uRyG;X!q#6VB^Q{2$}=((!#xY6Soweapa1Z8d0YB<fwOTb
z5KErSHRNMZ<Y2iz?bP`n|I=?|CU4nh>bNe@(_;4O{46i0vYRYhM1z?&=j1N55;<`C
zvfdoq<Mm}pQHObyGJUURy#EmA@=PMHUVMkt<o`zMMQ&@o6}uEz1a3Aw`jD9*;pN!#
z-R3>GZgApRUi;@n!84vCI$JLNvv1=lD*Y_J@`J<c#DB>NKaQXI^Zw=K`ZH&EK18dr
zuu9t1{`&v?e0|up`W31yM%6COprwy@5_eRKpL+T@dBW7AGmRuBJk#sQY+1B)Dm%Z#
zxp_zDfo3om{{Me7SC{XA0#ECHmfnZ|o;Nx_;N+Xou}nBQ=jMq?KW)PUzhAok<-3Zz
zW{ZIu8{hfH=-|3Ex$vElwGyELf~9>DrD73TOUru#1#2%nVt2@{@b@~rMeO;#bF$|r
ze8@bUdVR{A>$8}0z^O6GDU54ALkqK;+l!wUJx;R=p5?xwajkyIC05;i{|${m6RY~4
zY>)Hs@Vt5R|Nh?J|9^ka*mQuY?r+kL>gf*;v9_G}{+=f+EJDrqL%IKkRW88{k`9O8
z9C3ogj8nvq@A`-T{^t4qZlOePqOcd2L#!i*ZpejOllIsB{r&&H{im59=FH*gmpA6%
z{-2&E=PMj>;Qf6b^L!(lx*mZ>pOp{tD;`a3y6!$h^V!$zw&Ld(+|NFQnyu1jUoY|Q
zPGZIXc{8fGs}&D6yxkYc*Y|(E-QWNJWG8<3|0nUmv$TpY9S^VT3$bOn@m2hv_wYmF
zk$((!c5JMyY<_-Bp=^e^K_(NyNi_AxAIaTfY;2(JAH)CuPMWLs)h6$#lfCGa=AhfR
zA*xfYp|kGKkMoBP9XQdk>EFla{%z&&CF=e!d#!gtk#S#LDrhbFpUNX1X2!<NJv`GI
zi(?qpJ~|*=cz>T^&W(o62M@vq_nkatKr@p!8229PWYny6IHkH}S6kzbXZ%Yhe|pI5
z^yGU!XvuR$g+$#S2l+Gd4<EdbZ_|&LI5#)B;tR*b={Igj*w}8IukU?^g^RiG$*1@7
zhrhn=_{0hA{yOD9v8g{-@R;qK7_ZI32dfSn%>DhG|M2#Fo~9;__Zv4Dcz8Si&TL*4
z1*#oynI?sQxWwepW`1st<E^kGM-uY<u1K)WIo)6$@{mhHQgWJ(V@bybP#X~xCi4BY
zADa?yo3Ziox_nu@H1dfOOC;~QI16p<<~2Hp7rS%Bf@+h0XFmM1Jv`f-M@i|&uJ<Oa
z_5TbE3=M039obN%dU%>H&vZS*n%~Dh{5EH{wq`arcMcMAE;D0(BBZw~wx3B-bZ>>A
z;iR)CPbw%kr&=1SIe0rnu<x6&^ZI(@nEic4l_F}8_}?>kFK252Ppf?ilOyZK0%`RE
zmhg=Qk_Q=|oJ-U;J0!~D`myQ9toHwV<XQYhgXRbnd2p~4ovEIs!Szb&bke+wTkiT;
zb|<XvIW5(%9Q3I2$xl6(`@x|z)cQ7mvT0bh@b!h#O+6}(98;WDelL98q@mQu$#h8I
z`cZ?tztjJ-=6Xjeu>`F<mw4d&#<%trb~$>{*1r3<@IGJ#wIQASW$OPKfL8y%muv3e
zxT5f<u>Tp8kEx8z1?z<m8YWJ33=r5=`Ru;^i?d-SyY|YyTf#5DpmrlWzeJx*;*CwM
z5ARwZX-fBDh)my|zzAAG`tKi4T^)~?7tg6vAv^S>RxAJipZ??5)Q3knzl6@Yc8%xJ
zBcA&Ii}V%UvK<m~8y~GqlCw*(C}yk8<c9{<nz^=Itp+>|_A((K81$U@A6yaUxVOXn
z!c9jJ**SMuN(;m@Wku%ys<3ZXt@ir0)bm!`y@v*Ei3>6(6s=#|f1@z0*x^I}@#t$$
zb9}RpO{`qDj+4oTKlxjHkK7&mWt_K<6?zDLnzL{DYmF}N`!Cgv@|-oNDD_PgQtVJV
zRIuB+CSl&}(CMu@i*;763fFqIi=**<eRa&a(x2zod}uqT7BzX<mbdR)^1)%ZWcFUJ
zW&@E{`xx%71rJ_5y0Fl$r`i8SHjB8s^-d3Uc54pND+lFWtTnD~^YX4<R-jPs!S{0q
z=gk9|kxDG<ezQ#3t8X%S|Gg{a?@k!CIZo0^yI#1rXt$tihE$i+yv3^+YVD2d=APbr
za7*7Lmr!5*nkAZrKC!`}EsBl~w=bEfoZLTmq5#Lij~&kwZ(H!~I`^<4)sQEC?i&57
z3A;3Q#8}M#SgVrB>3+M&Dsk3r;rh!K=e943{I>I)EGXXR%-qj)$bsir{jEiJ8e4Z+
z+}zhRCCU2NqJ7;89_>xt5nju^mM_-Y)~(QSnDd{tBTwX})|gLEPnWkn+Y$3_`L<^#
z=eM7{ef#qf*B+K{Kd<}Fo+8t)BUT}ITJ^D^qeJ8)3!ZivK8{9#ExJw#x99kAIIeFG
z`tqBH?TO{IqZuxf7Kj9Gy1g+j_<H%u?HU_&cHLe7>3;pTyO&;0NImoXOq!3{4!uMV
zna%ef|7N)U+L`ZwiLTX^r>nGPwV9-DeN<D&xcTK24Ws2B+1TdyW$w?o;UQ!;d%oLK
zHn7L$?B{MO5IAa|@UwaX3u}8pN5-|C3;%EZzhI@~FJZ1likV#s(k+JsesSHJkTTKv
z>yDSp%<tT-E;znb`lCsznKH+{DNm=~WS;-?+PlKo9BI$2J%#g%_deQn=6Q?aa^b==
z8Aq~O7d`m>=6i-z-{CUv?cJ#xZwH31&?@wql{8C+Z_RZHw(Zihd0iV53+&9l><j6Q
z4-EZkq}0`+#KycV*rxxZMC|hW=|&GLj(iRXZCP-AvBrxM4eu6*KmjICrPCflEeRV}
z{4cQKKg$@IX0c}VTl-0SUY}Oh6QAZi>B;Jk#q3}|&h_VN3J^F}KSj4l@spB);6>NB
zR*m0ZyCrH{H{3gLqE)X+?0nP?gQ~4B93{jq<+LZ79X|8@to$;2i(Au7mTE+-y?3r^
zB^Ps4;%3Qt*AsmgPdwK4-MX;nal-AK?Hr9k9-WT^u1`_zW(^7wsgMgQvM`umyt~jU
zQKIMgT<*31C$*$L{LOK&q$_BaX~@-p&(~+A&)a-oxNU~oY~H{lj~icaiYwkL#PTcq
zR?*3i9RgESeHZuhpG%+Aa(ZcSu$u6q7uf}W+UDDDi9D9~-7@a4nm@RDpEGMecT<4C
zar><Eo0c40rodH{B`EXffAkXDqRDL>iz<9=1$=6FIj7l6?RvRG`dyoT`Th5Wdxh+L
zZ{|g=7ktDm`F7(+$(J&6QJW(k?9vEzJtcJcNtNpPLYYY|FGY?&EK0OVGjZ~2Ik4%*
zTM3>EyZ$Y+nXC2u=bv^r4#&U+rk^bx52PI~I+=4UCFt!w9cQmgQ~vH-q_gerN2lu>
z%)i|CjS_2Tp4XncSS0bp+qU<{GLur*gv~i$As6(iVu8qU<u=E_D<=$ini@1-X<QLl
zx@EzI>8$U{AN_b8_4hVpE5VvshFmQM0<QC$GByaV$$5LkY_Iu;UDvp{<qkOCkaTou
z+7zwA5*EARhNVN-$>(S1pZ_CreA~tES4<<z?A_PhsHj(0TGz5b=lr(AcTV{Udp-Q!
zv-s<_J{CUadP82%r1y%8I5a0tTBLBA!`WLQIiTV31B(?}-{vMh__`=JQmlKmx`~-n
z;mMNgf-Gh3y~$hDd;_@@FU%@+?I_lHUFqRDZ+`r~f8W=J?U2?`i_4OF_A~P`XkGsQ
z|4ZgrbF~_XxcDzj&|LYZsdd|}o!s8rpNG9+vDUuZ)Vq{Jtn%nohd%-j=d!klm><3)
zdE#|}414W)oB31EADvb=op+&*=cMf8%JUrF@%kBmSD$oOvN`C9rI#Q}ilnP!=t9%m
zZH6t1flp?*cmMf6cbN(AWtZ!QJUZO3Pgzto9kmX7=yLvO^o{fa{x@5ft?PW;zWCm;
zL)vn`{cc-XFP<(BPC19njsASoJ$z*)&$n+J*&F&Q4GbQvE>UoBX;D;Ycz8}$>$~iP
z+;s3_hzXNGiz0sfnR)Qiq=P3q9R2zldwU&2rZ_q?-%@X}kh_$7j9KBXYDZOUESs%u
z+jM<Nzj-Zu^6~r5Y{->n78c$j?i+jH)TxBLa|>2?`|}q4c)Tj(!lIWi0^2sHd!BBD
zRQGe{_;NV}h_vpXGCzndi0xR<ncTxS%ybU+ow;|%NX0rpP37bR&3P}5UYDKAdgt8U
zxqZic+7v@w=Os&cO^e8TFk7S4w5aBiiAd|D#Gk#X8*|ra2exDy^{H@Xc0EtLeZ(-b
z?Y*(@j(6MdZ=P1U?0uwI8>3=hL&f!r+bsGX_0<crT>0v3zV=&9t#st4z-b%{&Hcm6
zz~QuJrXp8sfPgE%8mn*Ayn|{pSZ-@Ko!)pmw$IyuJFHMtX-A``ln#I3p9kMRr!zAj
zo_lz%HNXAR)P<QEU8h&oCj0F?=$w6O(yqF+PlaEs`XsjpbtFU{*_b(TS&QQBwxASA
z#|Dml{vro8J%3l<H1kZ-nEU*U@#7AT%P!YvPDz#cU~}!O_}U$Mp<fSn-LpNSUARTJ
z_P?Lx0X|TXwr7qZSF?kFt2~EPAb*0e;E_YtH?w92FZrNy&-Kc^LrytIlw1$!s!S5u
zEfC49=)3IJ&$~tjf4+-<t6r5pD`x$YmA@9&?w9VH{dh%o=rj@5$0s&z$nBC<(LA%+
zOM1TlApyk?HjgAuA2R4rs+s?L?&p~*fo5To?(KS&_x-#o%ju@a7q;;*w>e(EP&z}c
zwD+|V%jK5b;@x{!`nElq6zr!i$KTXoqkh`X=2Pb3d)iUEuRmS*erL>zRrB9}WdgbW
z%*>5^&H*BA_RZEEYDzqMYWfLARSuKOKUk@`O%7FA*UFg5e#xwNS?2P8>kIbHvA=IQ
z|KSCbNi8<&U2?(o;o8d!_9!e833a`HdD|N8>D;mHuZ7ya*akha=uzI+pJ{adhFROe
zO$tXC59S^EKkfD9$^IP6I+rPOXn0Ray1gP>r+Zzp1YiF5J&)I|dVRz&vuld_g8z5a
zez%=A>UzZOx$`-=VpPhqYnVP=QM-HLf`-Oc*6K|Mrg{}jmH$xC(j#hQ>|9+kA;)F=
zSK$(d3*HYSCG6@zgFTRq-E47jZRYtBEG!cD_NNO7H1xf25DQLXvbo4Vv30t@iI|S7
z|Nk4mxp(%$%}%5F?lzE2prj>dSD$ucL*v1lGY=kM*t?aHpXpDeWbq+ZrUyG&jI>WX
z9G`0Xg>ix$FEjI&WddStU8heN7#KXb-NnQh5zHLL|IPm0xwqZopdr~idpw^RK*pJr
zww3Iwot=1yrS1IujbD;CRxxj8G_7D*rd)WEPv<}hXWFI%m%aY}o16I9txeY2Bd`4s
zRG(SlpWnwf<jWsk6M3Qg+(TE%c~=@jZ#popVeB<ckt{fVkikkKyy5Yu>-YEbsQVk8
znc;XS!4ldeRazMP{W1IDGc$SE`2?N@__VC)-o<d!A?D<i#(>7gOtpVX6WmU+%7vXc
z*t*)j$u!d8!eV!xzkhlB=LbEJfCk6fw*Ma*7jB>K7$;|VuYpbBbliml|NmbQ)81r0
zVd_Kqs^ccS#hsi{a}K_BDprva-~6#<B~S4^2IJa<xq(VT3o6+y6_bn#RP(mT{CFvR
zc#0;Exc;O!5++d7ce+=6R64xG^Fo;Uze$p7Lt2!5j<OW9{WD_NfAvSPM4j`8*=~DQ
zf6!=P=Tp7s;#U9PNGIyZhID?>Z_uSqN;~sGa~A7jd5nyBSXp^wWCFq$rRs0_^w|Dh
z&_nxed%RT+EESKG>Ho>H_FsQgOG0S>)+tFYvFrP<TKV*zh%ph;jWl@RA@l#AVb0xC
z8)__1?NZzX4f^GefBdrr?e==~h^MfS=ggT4bvo-r8Z`gOAK#_;=~N0^&m{TjMr#<R
z&o-PYI60s|)$57J(Wz<|Bq~lBu-(yo{G+ff^Rk4T&4P0x=HP-%sW{}}&tj0^kDro%
z{svxRH~P_k@ruLbqcZkORXQhx@Hj9S7WS;<IdoJ<VuI*`qr6dTl)E2m+J7)M2tGDd
zn`iFa3!;-lK+Xegm#SB05EgEpBjfDN-Mrbjq=dh&LG;!QOEu=+lmpu)_VGAmnL74(
z`J|p;6%BK~@TX~c#o<#&eYkj5a5gb2*Z%o^yx_Z@bD`nkqup2ToP+GWS(GuILs_|b
ziiop2d-G;vW^L_hip6In#hzR&S!1mK!9is=N8{|~<*rU;znZiE^shbK+TOeR-Qm#R
z6GGPB*?l}9<Ve9Uld0y*Aj83n4v8rl8n#^vW8>#%D=Rx<!_B-+iA9^Uls&6!p+TX7
zyP_n6|Lw^Vx-(q4=cRN_ZJg@jE%QO|-r8uMr%!p@`-PsGPJ$$qMd$t*1qQZlGGeQ*
zXJcf1^Rl_|#gWFT-kb+S4PC;tT~-~B(*5_pe_p@@Va7CfHvi4dj9Z^Hl-KIVrDWu!
zSd{gg$!P#px(uMbsVW>?MlLP}OO_a%IQ3{vyu=1m(+d(D(=S@oPMBiBVbwNc60>ok
z!2>@ro*R=mqE6k|)|>b$q^<aQM(KT9a6B%G<1sM!@WEilY~v|Y9M<1TOK`|sWR#%7
z8pFxj>+LDb$mh|U^T#)Q%PA)Q=`)tGG547?dLFts-=634a^p2oM+<(Ntqo#>P2?>~
zdz6rnkdT}_VM2n2ZfZecqJWs%oi_(gIau^HePdTs*teP~yx&{$+LZ$iE4CbXC3l8R
zlX)9gD`QxKfq}uEy0aJVo!d}r-B?_y0b0fi@?z1W1_nk(w)ptA6CP~L%o&1C8=fBQ
zFtl|{IdwqzR|MlM-4up1*Yy4!IoPoMi9$rng0s4BZXPzMwK{xq@)rw3$by4Ko*;KU
zc#t3<p6=n9_~BF1goy{^_CI)e%IJblp#bxT62l3y%`?wkaQPr2x1(wUkHkynxUN0b
z#|<)t4xgKQA$Lg$DDpt=34PSSpr^-{oy}HW-nKyD#VYkPQEadH?o8#@bKD_vZ651C
z5$BYxP4UW=oy^(xt&Gt#8T}>W_Zz*rcXmSp<KdacCALZYkf6^6IVvG3DIq<*pdg{3
zJe`BPc^%V{t1hVuwuK3-KitJTOeQ4cPkVEu@v7JvHue<%pKou2!aZikfe(KyeSfX4
z0J-Y_|3#IL8kpna*iuv5dU)Cz7}(f)lvvKsU=I_tVc6{JFfWtE+Tw>W`@b>`De=t*
z8s@CuQl@+O;^L0Ik4xB~Cd5B#U{F+K<KzUTN^5Jj$VfIvN0)BVUKa~N(=QuOJ$ltS
z!SGNckHPFYM^>mC6)q4}|1jm_1IEKIFN4bJH}}qdc*tB@c5xys-pU^}F!=eg2@0|?
zGqZ7Xvx$nbm6oz`a=I)QeQ2@gDCZ`Nd3jm~Z!z*KEY5D8Q^UE<d7|*C@9%lU^}(y~
zf180!j^EFhvZ(>&M27$WrDhyqWIS_*XU`s<KYw_vtavV6;!#uMId_gHCg#H7&kvR_
zd8p~}Kq;g1vHh_mL7Q7Ex&J+mJ=$^R-}nB9$NAgp|4YoXO}??M7j$Su`gsXC+oT_V
zXNRQ=O$1H0gUmm1gpqy1-k=HEd`p)~>gY<Al!7KmcI;dj%6~0S_unfvU&%^e%b2tz
zGn>M%{RXW6|0Y-boAdCVZQJ|%lI!A9E$YA=^L(~XPf}t)uKxdDN~DvS*P&1E=@Zbj
zLqIra-d{i{>B6O?3zwSaO+9?<7-;hI`0)eB+B$ykuNSoL*uXT&&4uHd?+zXpZA-hl
zlpmj1Uu~FeF6lQv4K%Ov*c~+7XIGm98qs+jF4b2b%mK0L%pF6nRs#Xoi>p_-Fm+Bi
zr1jumdCJn>*47&=E+z?v%>Tvp4s35VKNC{C&qC(Eq1*3uT22ereVixpvij+bA~m<R
zMu7~ezUC=k47TcCdn9u&`K|v3k@XQ1^i~F(nmqr=_3%)^JLf0Q(d6nmXnAVVYbDjt
ziSK8xJ1lW+YGF#G$ica4&t}%9-!8H3koo=>)B^ec|IA%Ot_B01)`#^Ty=(9NpTH*@
z<Knc^xk<^Q>&LkXs*N@8-yi-cv48HIzb1Q?581HaeG=z?Zbh2`+w{b{w{qD(`q!67
zZPTsW>gv%sB|tUZYvGgt?O2PoQ%h4NW~dz#G|ozmyzza4_?bn%$5?muzZcGb3eH$k
zlRBF#4jf~9tF<_HrBj-*0`C@wE&+}tmszGQQ?%{t3~K)K$u4dYF`kgD`R||Xk9n-R
fkPyUvQWP^oPUPjdAN#Ie1F7<K^>bP0l+XkKdlJbi

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..37e8ffae114625d0cc6a07ab2b8dbbb7413a3829
GIT binary patch
literal 18884
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3?DlkV45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpNBR#6GAQ!2#4P?6zh3S1-gT_08Vp~o9>@jF-t4Zi$UURypSEXE
zwD*B~HyX|>UY@yb(J8Y~UEzw_qHcbW1xL;s3NZ-sI9!xSlk{9({(o|&XbaQHdrjp{
z|L?wTsP2e4HzTOyqjO0|--)VxX9fmSwhjlEXltcaRWr`dsqVS^JMmueyQpx@?#Q_H
zF`t-0M&7w%&DCHaz<NNOSCFYmUO=?ye=;ZIL8l2KW@($xABw(lj`7*{;-2KqI=R<>
zCVu$*#<@W8M(NsRC%@RvNshf7#gxV4$tt4Yuz-Q(i?iU_$ocN)H^13^OyhP~(wXCm
znkQ?Yzn(PZ-^`QTV9PFT<Z5;hU<sJ?x$n&}FV@}KfB(<lC~AMeN$|XLetIwqYqjEp
zm2AtT@BM#}aPHNw*lCraPwy39-m_=BYhuT|Gi@&%mz-O<)xeFPD_%Ij_IIk)#T81b
zInqfk91&uij@LvFE5uH}74|D`ndh^}V}if`z1Gh=#SXIh$|Y;A0}3FU4>cT#_;FDB
z&0l`y3YN7Z(yL>GSyh*K3mSd(G2V0kaY0{pt;A+#4u-t9YO2O<np?RpO1?gAWc|XQ
zscq8E%9A?`rG%BTdW)7#F;HD2n(Xr4kVUICYH`rMy63TekW{d$)7h&lJYq(;SC@PI
z{P?(e;W2Zf9UFt4nmT9ywigyYSJDtJVC;Kf=M7=@po^dQgSOQC`SJeV-umr%e`~+L
zdw+TP|8w*IfB*AyfBt<5yE>T{AC$PEsVb?{Ib@2rM~A;p&*?)>E`ELiv9TVWouLsw
z?1hC*_DM=kTr6Y!>*Es9Gg@qW|2U-nZ+-puzx=$61xoe*{#5_@ale(jKL75o-=CiT
zH%|Yzzv}3R&+3P#>odzXEdkj8O2bl}&66ibM$8EJ=(v5z$tN__$=^T3ySsP(fBP&Y
zhrHNFD}4nn_RN~}Z=tAAh469vC1yG*g$qoQA9nr!`}_N+r~7Mv|BK!G>;KQ<Hg0hV
zyV|55ADbUuU*E_VtOT;+|9`0~oy`*`Hcp=&8!#c-*Z05)k1&7#TZfwBL%!JC$SSDv
zO?i=?((=vv-~a#o-WnZ`Smcg>IMO&v=K`0|8&7uKg$uNtRrsDbJ^hy3wl-Qa?{3nL
z>gf+3I=8*QFIo35MdHk2HK;vHI-6(CoY>wT>(LP%AAjab$jy_Ue!jgkdwJ8d9K;;8
zjQ{=r|DS&=hogydg61>(ET8}B2LdY;#6@phI$mG#uyNLse@>U!l^@ri_;B<4dmeT^
zgEu!07JSz`ytkTXzP<6EU#1?DIUrG90=BBTIW%U@@jEg1j=9~v8R-->Nl<8dPY=(D
ztL%MWK8h=`_Ip=6ayb3#Llb-UdX}~3-Alxc1ez8Jntom2t++k$|NrzK|K~rvZ{PO+
zzocGlQpDD-hmTy_+W95z>Qa7uPQRck2=$mtXLD!g#MV|%pPt*7f^MGnK7P#2#i60I
zw^wlS#!d};11Yf&iaKYesZCR2+$`j{I>sPbHoCLv+N~e$cVb*~SN}^+Xjs<p|G#A2
z-=rTuCqMj{+_p7al20a~;#14R$L?$%uOwCARylih9Y5rB^>Xm-OF=F!EmJ2??(BSN
zzoz9tQ0juO-XFI)eC$X_UtAh#D3G?XaqF%|^}hWEg)Wa8ChV^_uK9K3!w1E~*Vgh>
ze>bwJK4wti<_<Mjq|-TI!rdDYCy%?IKjq~VHEVKv`^=e=6V(E;1-2~HNL%A9av+)e
zl<Qhefv&*Yt2)}4bg~4ea$M|utRLm%n7wI%Dj!>f=BIDEhv!=J+}>{d=HA&4pHvU`
z%QH(V3qbAoaU?N8;ql|-7cVrHuQ{WyaNy9PLk&^Q3gPk*ml?$z&#Yp}WMW;)cj&6p
z!4;YTDTcq)Ui1cEDQQ+aeV}2Yg~F5b^LftCGmP1HW<#Cr;hn`i=jZcBEOCGuej+kb
zRh7}(JGZ``H#XLglantjj4eWuk<q88=YK*@hEiHeN=xCxp8o=qCQkhLzkY{;@R>RP
zSDbqA!v2hj8e3bvM^gruf?@j_9?1iu&LKud1_nwy1%f3X|Nr09(_?X4<jotNt}dQO
zk9drXc*OOMbfS-a_@a4uwz&$s&Q_=!HcUEv%FEQ`%85HMVZOcdCr{>_rY|{xtMh+^
z^g@ZB5`kt9^5Pb}JgSpiFx57~?bro3qYbN$ygA`ub=AQ1hBqGvr=z^R(HxtT1wRZ9
ze|yW*FVF8I!w5;y4M!HP(n`o%CoZ!lHAO{R>Clk_2O6%ta>;8<>dmRx%v_z#yj8W)
zQmAX`y94vXF1R@vzUDjQ(yUzg$Mq5`>xq+1jGCzjZf?&roMU--Lx$kt$H$oiS|HBe
zGj$_xg8>iA{SHO}W)~5U4USUJPHYH?RLlM#a!hT>B8MGCap$aqmh9iZQRwlMpgY?t
zW83#FPtBPuY;e(ew=zHT*|p5`E}4qFb6YY~uwmB!s}1qrcZCF9R94}Y<bM~kE$FB7
z_f`JIUnC{Aec`<)ahf3^Qh@20pk?y?z*FY?KWu^&(~IIC35Y)3vgJx&m(ivT0b-)5
zi76=!OF7w=Y-43!Q`T<q?=aiGfGFt=8`{*FGMXDEezafn$UWOu!JWq|s`2RedXHuX
zPt~HJ<c6h+J`C5p*RWYmU7#-8({kbfgSX_0HV0P=OGz6WiIh}H5mCvoHQ>y3S^bjp
zd>u$IEOLL8kdkuYib!#>@u#m>jg2NHrKdMc74F&a<zP!lzkz{~(kz7&t?X+&IT8&F
z4Gatn{y66@Qkul7!nkmm=$1WwF>?$K-D#e1s3Xec>xq@G>{ofEL`x*R4LGxi)z;L3
z!9`%_pM->j3l|e7OidLKOS`e9^Wi1cHgWwIhGA^rNM2MqyScUX<dqORl_Q6)hPr!p
zPVekw46AiGP#>YqxO48u`HaUGh~4U7jp}Au_Eg`j@XIvKoE)ypN>45prgTp@9(Pgw
z-UL}@dD(*!PaK{tGBNE?1QlYxr$4-)*miff<hS?f6(1TNzK_3kOcr8c?#YAa&L!oq
z7nIJ<Okb|4b?O*HZuKE<9*%@2b5jXP$pqHJ^-H=MI2X74;Lqk#;L5rZBpY`oA+3DM
zHr6w?Cm0f0UwIYlTxHyo*A(jcKXI!xvzs*Q-aiV;HZx~R%E(9@IV!25B4L)3u%oc~
z;dyzZ9z`)|U?)C$@GPyo__DXpmTlX@9zJOi|N8%-VuzGN#$u@nEC(BIWjN$}Ch&w9
zC`s~}bNMQ7iu<wQNc|Oy<Rf8`3ru1Xdj3mrJ@Dmqxzw~o&Z2XU#fED-1r1wu7cD)i
zpmgZK`STl+SPwrvZ4x5|Y9s#tzsM65xdOtEi_6VB`nQ`IPE1a1HNP%7;g!gieXs4W
zur&H8x!V_6H!y0-E||`{AyBiz>VOWj#KU#2LhWkYA%!WCPoABqzhcIs9K)6Az%%dM
zh8Cf)22;lcjE0773l!Mo<l2^cOX|g?{P@(W<_j(H7S%mUNK2bCUHsEm)9$`)S1t)9
zr@4Jpa?@%wx0yLXh?lu&X@gYdhi_^{HOk@I9U)D3_*_j{j%qBq#B{TuVlJz>@KN~{
ztpd!<LFMApL?)c(EZMH8?r6lt&&I|U7snPK&lVTgrs^$mZcbvw_r8bs?Pc61K$|9u
ziXJ5-CI$rybMu(+b8ij^6iiBLx~Aq)*L+-Q&qPK}oeFDaDftZvu9{{6t9jV2xHPDA
z#cf?YRXgJ>-;^S`vjsUp8CyB`96og5zySt-ffY+x!;UAMe8UpVX_SzhTu_$w;6cKJ
zXK5B?JrA#HYncZ^im63uAX_h8n#9AsnUBLDEJ(;@-Ia&?oV4U76x(}!Is944&9+Hs
zM~{Z%S9wnsUDgAqMW6iYV6@EIIF*;j!&Q)Zl@}Y^lohUWPYxt2d^2ejdwL=!MwM%m
zg|TsSFE6vUHnX(!;f2mT`f&lqwa}Cl1q$PkU}0|l?R<P@lP5oJHWihepd`)oAeT=q
zVs`V#|Mfb51#?Q-nX^td2DF8+8Z0>ZNJD5t*u@ik2YgsEv;?^(RBPY3WgehubCo?!
zC^6x{bgv1nGiFHW=tx9FNJvP6tKy%NvR*Pna*ZD-Elrpz%E!H#i^pW*<j2V=DNR>6
zHeAf!aee~(zJJC>|9=0U&uMn4A*gvlU%OIzA={dnLYqqz*KJB@xtbAib-{9{C5L#z
z-=(*d2PNnxu~mF(HBarx?|P&eexYHbVaUOEcX{&f2Q9A!b$R~(U*z{lK<pVe_hv3m
z!{7jch9x`^zZ$+vC~_^F%8~Pf|6Z-@xw0OQm^o%jrD6gbJEFEJUs2*ra<gR4y7O35
z{omqOP8lxm92Ae*I*H$!ps`1vIdl4h^>!D6c=V3?`0yM%#`FK5Va@+@8`Akp%}zi=
z)#=fL=g+w~4g0$`1%!w+sj^m_N;srp`R}lr@4v&RIN5r7_)gB{@!(|5?m9TD!65ip
zJ)4du^K`v4M$FSi$|7YrMY`iZta+=j?7#t$l%ymB)6^fI+aKOGZ*y*6v0)kyBp^)+
z3yL@Y{3WHL60v#(hp{23R8?HcllEjvW6Z(>&wT^98Qf;L{Wzk@?4v)$Ex7H8fLqSQ
zcD3&1@m-Nx1_>FCuZ0tJTLN}?eC1GHXU4uKAtB+wrPQNQM;~Z{;_u8EiGB6yKfZQ9
z{8898-+m)2+eN7FGzvFt-n?UXwz}%g%*;+>W8*_ok~}L?*^0I@v1d%>`1FIFxu-j9
zt8&}sk0ItWwh06^toA!N`J4pDQl2MMy6#<Zn0wIuqW6Zajy}H<Qd2i<Yg^WDoUS2d
z&kFXJl6~QZ4F<PvX&pU&TUzquh7AS=3ZX0>b0gFuPBr{sh|4ihI=jhe5#JnTABWUr
z*{v<Fo7fGng{c%hNz_&Oz_3PZVlV@<7IXI0mdGCux5$P+bgcRH;7Q7ct*I5ir#<|f
z-j;p+#QQxEuPE6Uu2|9G;&Nl=Z0!vj4HL`sCY<6?TcX5x$*@uXgG=^qN5)OzhCzqx
zdoCQG_1MsP{*PzYQB(dO-@p;Rv2nSD<BT@R%Rj1D&0q-s<#0_^;lP78XFfdPJpB7R
zkN^BhH!C68I!Vc0n3<WKJv(E?*2$Bl<mI`8W+gbh+Pz4W;X_m#=c*>b%_mPX2&XtC
zD>F_>VPXDXc-;1y#Gz9MSPfD)u1Y_$mZ4j;hoj+x<{TF>#m>evQzs^Ja3vX-rza#I
zer<mT>J#Ha78Z$?7KyIz;sbYCm^%*oKACdx1V{dgB(V(w4G~Wobt4mc>tZsS5}rI{
zbkTAS?y*l@urV;p$1P&A>^!xCrJ`%N46ZulCZ&Mt@e2tZy{QJKsR>Hj^B}%Za-TeP
zs*h9C-bG6GZRQ56*($UWorKz#b5^|h=l)HU?LfR*(llO`Om#QLy3Z4rN)#4PWWD%F
z_*=yBP?ZE<`5Dg)jEoEn4J9NESy&BMteW)5RRZdA^+FdnD{0Bj%^Uu-n+h2g8z^0y
zxL98E(Zi~Xd)igsbTD@ab(HLV^72qrPQp}I=ZKI;o?<S`_+yUpu(LO>)^1*^%4}?W
z(vAg^dUOj73=NMRjr{#%MNzTw!C0QJO3KW8vKl%M-~7N@!%^t-RzTQ5NNirl;Wy0-
zHhwhdU`bI4-l1o+lCe5UF{0c+V4|JDSwZ1s53eKxbM>3jP)F((&Y0n`Z0XJJjtvLB
zxt}cLxDskpF!BG2Zu5m4^BN>1FMeC~fbqZKBFXSy3LGc26CN;ZzTuGfFiBOiV2P-V
zn85Wb4U>5!JXvmPh>Eto$Y67IRnlz-)vgTx|0^jL78Gn?VYkl8{%l~lkk?YdOh93)
z_C#}qgO}M;x&;?aIB@ZhgEz~LS2`BrlmE&qe+`%+rN9t=MJVUR15dGtQ~Vap_1se$
z8RvGd<Zg<HkhpV4qNLPwbqv()W`!Fz8U6h6!qs)<=8XpzPh(W(dk}NPtMJOy`x@z{
zilP!c9=9$EFt2%YaI%Ysmi-BDRhMrK7raDb${uDm85lfR>v$ohb4{d#q~wpEl3H34
zTef(-*a<Z|Y~h?a4lXTQ8yGhI`hTLZ$7hjOqOaut3n4;=0ga)Co@x^mxfy$o6n-=h
zoOqP;`obNT*w?u<OzmD47&!6Z*<MEnhW@P@P_wiO3ko+{SiQ2a;yUaf+&Ga(BK1K1
zl3ERm?+cPvxx4<~z;=Io{(iffA0PkzeqaCJpquN-BJO&rMPdg+rx_%z`uTf0sEFj3
z|5yLtZ<pzkF98#_CR}qobB4##l4sSbBYfS^l$`Y=COBBh=n})eVBVEGKlD$WD7luy
z(M$5gaWR9WB}?i(rm|k+dTP&n`2YU;|9dMxKYepky<e{O$CsC&{<ESYpQoo@Wo4YC
zq@11Ik8NITOLiQ;<&&78ptbgar$f(;B@dcrZ0h;{|9}1efBzS|*UQ=d`uyqXeSZ0W
zhK7HiKmM(+e?BASh=Rt66K8xvl3HDK*JWBR;T6zOir5(w!=tImqpQoqFE6xB9Fm~t
zSX5NpShLo&q-2L7lenFCQ|E)(^A?(F?@?~P8FO^v-T%f0fBygYczkEEf7$zewJ$C-
zYlp|x{rNGuv9WRH%)sE_8;6}wp7wSO5040#&^vP`<JH^~4B20ljX#9D&B*$oF4%J7
z{{H{*dw-RGd$V5Fy6)Z0&!4}&-9O>#LnbDkU%!0%`h5KSdVc-lsj0De<8I6s{VT!2
zi*bh1j&JvW{W4k;bMnI@)`y(X_Jq=%%|>R^-oE)_WGtMTl$4f~l#-B;mXeZ`kkF)i
zy}@9{18qx%O*}jr#+nAx6`$vPIop%*;^yZ5-R1YAHavK9aWQ}W|6`k8-8<s4XVIZ6
zHzTL@_p_F*xMSaw(w=%<g-v{VQqq*fwYLBN|NZ^`?QQ$l*Xw6neLem3bVS+GjT;Sb
z+%Qm8HMFue_VD;~d|QYh_W_&5jN0O>IgD;?ZeO^l>ET6Xrfbkd5K#2t<HqR7l`B?$
zG%$2j51DAbRq3kc{{+qk&SS3czW+~e=$+*Frz0?J!F2ij`*pGV{v3L8^0=JcpN7-9
z)Ajb|Wm^9H{XJe^ztYZ5OlB+FwW3G%Y||M%RXrcrx0M|O#r}^E`A56n2QRPNSuOs%
z?r8jeiEnRzmw(qg{QA02VIhx`lS`!AhDQo%PXGVcvvG5?<>t19ujg1J25BcLT~G)O
z68iXM%Jav`t^JXP3${sbkX8P}|FUG>{t3EAC8B2<t`==kw6FVfZ^Of^*t>Z)n$3-k
z55K>!H_zKM+xq*dPT}+aoUZF6EaPYEi8wwr#vn9F^dL)yax72%f1{k+-)e7eTF)={
z=ivTnj~+U=ZQ4{Hz3q>B=1H-6a$MJrX)Y_%(`#Fy!6qokBnAnwL(?sG?1<pueL7`o
zWn%ho$M&5C=HeIn^OI9rlycHPCF(!_|L<@4n;XrWHs#*mC%9wVlmGQ?@%#VHv-(<e
zeVzMsy}buLBzYvx{q9#@DBz<S;VrwSVPa$B!v_cDtH1r(Q}oo_aP@|||7jXJf9gLd
zZ1n!c@hxCN=p!c|A09R~o>{Ya-n{wo22^7H|9@z@#gZi~GBQhATW=;Dj?@V3Hfw9_
z+%R`x!)meg27`Zpe^);~cKX}5zwhr0Uw!`jd;i1R+waddzrVlq_2J*&?F(J(|9Q3a
zd|_nNn5s6X=HsLHKR@sP_Wu9Bui6{GUitl<=k8se>H5OE=L^<P7d4)mVCv}5aPV|@
z<9zw~Mv$WF&`gVnNXe~RRJLwqP3+{}Y38W!I>FtvfT!l6lSb=>-{0?FTer97$A{WS
z;d+06e*XXO@A>2X`zt;?5Z?BQhi}6)Rh}Cnpqi#&n!LE)ADiN5&mSLG(K-A3|NMt9
z0^6o)OWxbB9>)F8h&fnh^M{1wWCPRGfS{y=gLmIRYPmx*Em~T3=<2Gfs9G92%d9=f
znD1|3WOQIve|^O9bB2>xy*fYr|G)om`}?(##Xq0>UlOx5GBkX5<Kv5$m-Xk{{XNuC
zeMaJ_YWK<+jQ&aow0=K3dp~-6oxHW%r)3TA?@OBHr`_1t-14=if^DC{YjK|gM~*1y
zoqO=&#DTNDmqD#+hX4N$%>>24*Do#=6%q+g8PmiC*H(U%FQ0Wl$L2wmVD7~D{dM_w
zD&Ahxoy33k_s`GoH>cmvyZ`UYtE=X0Y-~YNG8HM3212~c+)7sNF0a48_xJ22Nz8oj
z?j}d<ob=>y#0PcOKU4Yi^_x36n>QHv9fl^sc@{f%Mt=S3`t+%yZN!s_+GjMwToyU*
z-ouo?v2khC&Vq+G8=1e?{Zo0qMoLWVn6mr4`X3*J%M!S*O=+|*Jisa$@K|oOVvmHI
z;h+EKKm7Q3pIf|c;&!$7-}BpSt0m6OQFP=!Ske%x8q?C*=@>QZ;DLt5X6D^Lp>h6>
zpOG;yIXSSfP;7}nqnpZ`D(NS?*i>Z`=BV+#U!|~kzl7(L`{(AG%iAdIy5`Dyf8Srr
z@^@#iuiwAF_IK}<6RvDWrU?I-t-OQ5M6&kJkLPc1@1L$Waev^G-{oyzUwdA5+VHb;
zKew{t!Gy%d&1rBCySVNQ4Q;(~GjI}X7I(8lKC5QupQ9eyCs^5>Qm(JNy@ZuV=*Q2`
z_ut?DKhOUEw};G2+&i>2A82tFnEDzlHQ;URvakR5{p)LcdAmQ0p9oBKZkO0stL)3Y
zPEFmgH%w2DO;wf6(D02Kq|iAu$zsQjh_7E;MMZaBh!S0KHfT-nsd|sOaW<aM?cIca
z{{Q^^{p#@eiDFtS8m3Ntd9Ydk{=UBlW0yZzsUuMku2In7EyHzh-{0SVe#VD|dD$E+
zcrA8#ZL~_h@xl9rN0+iL^7iXHcws{0WZ`ZpXrRurxUprctBZ?7;#%H6cLQUB>K`6n
z>|$d5Z?4_nCy(8`%KrWP`}_U3xBdI;{~w*m^zUI`_#aU&y~r&gM?bv0yq}%FZu5o-
za)BNX4zsrfFPAv6dz*y+f7bqvB`VErY~L;#K&_c%QBisG>Xnv=$cwA^J<c}W;c$ID
z`_J*)e6D^PGc=$5KRbK9rcKbJ-{1H9&#Q6u<@#T6<b{3D6Q`QiLk)>iB7c8<{eN=u
zevw~~{@1rjo1eI;(0_A-!{np4)KU`?64KKrOg(D16B2|@{xWOU`0U!%q@^`SLXvND
zl;g_02boNDhJqhoyuWXMeqODLrA@%W_3{7r)&D<z=upQ*sS84kQD2g$X=KQyAGmwP
zf$hJfq@-QlpWpKjT{(E*!yCz&XZhF`O=e{d@@{QxbPS#9=*@kaO9A3DCwaN#w6vfw
zF(L8vlmv&oFvlKu!Ne4o2?l~smQF1ABH!84@P9>5ef_^bmA|JQ`EY<?!;ymxYtJaG
z<UG{iEaNNSeem<g$MvWBW2(OQJvqY2*wgc3^Hv@njkDr0kBp3Xii&u2bY4V%f~3bK
zF@{_R6nOUNFkWxk{Qv&~zJ;0Wt5t4U-4I}6Q*2)DV*8F&MpO38tFI@lEgu_2*)$|q
z9=u?&=6mMhws(P{rSB^b&+6!!6KZ!`f8n;M_;fZkfi;XQ^DEn2G@f1$T^e=Hde;$`
z)n?Ofeg8X0-HD?^@QkuXKDZU+v_9;}QOlCj%8jA&C#G^ddm0%Ly{vv?;YJgS88iMB
z7fjfv$SrDMY`kfsqQ1FeM9~8w_sK~I7S3b5TEHeI#>U4t=XveKM}jq<Wq&ZP3*?RI
zddS1e%N7_|0_t-z{QvLdFQcX9<mIK~QsOq@w&Z_9mkyVJULBqPtt|yTmjrol+?-ia
z5fSj1|E8Ep;hIdQ&C3!RIE@5!l4s16WMQ3ovM=1?!NY_1HmY!N^kpe{D{7}DBwV<X
z5)j~U|0<-;S`ur>b-;jQPeQwZmZI~WPyfT+j6_tHIC{ORDC(AZ^zwlg-^4kaXLa7a
zDRB7hv1{}Fs}yt07v?+PUOt7%tL56e$yGjr%7*c);;WjQ4~O<&&+G{B`jMH+c%^H~
zxtIIYbf!hRu$<A(hJ=>CjFA!3nKK{!*rV1mo{ihl#eSoA_Kry$`{H_*E>&q|H8MBv
zo+BfEYlq#c$AMgyIV)N8jy4GXopG|2(etJ~TigGb7@k$DbVT@?3=Ir2G7~%1l^C!6
z)B1O?lp`TAQ9!^UoD*V^r;LvekDJ>CHufAr@u(Sx^v}1N+ieIFYYAqKda`o$=hLS@
zPn1;Q0f9Sr&RALf$;s)F&}86u3O?r062}&!#KhEd>(-i#^u3e0SXd-{d`{T-G4ow`
z#kYnx^hu-@$A#<e+ac{>r-v(7a;{impzJP}^|NuB?8QkZ;+H&a_-_!zc5N2x&aR*N
z>FU0o9R|k6uU>xq^x?rx-Nvb>8v++OFf21VQ_`HYY=y?+Ra&3l-@mb_DPzIPlM7aQ
zb~658xOP>pV8Pn63aYJ2ka%+Plwo4J^XO5AVrxvQ!}=GhI~)X}zx;R3t^echtM^}0
z;$Lb?#^krTa&lsVf@YPKesOVPIak@2aa@y{oWRJ;cw=g3hk=%cL3fwY!}uAU@-A*h
z9$u5y%W5CE{~)13RPNNN7mpL5dDm0s&z~3{pAL(|Wv#|Vfgk1F*E&9#!ewP-EWCN!
zG==m3gMwu2?1avo;gOhr_pv+A5m95Fif&$w=7}HoPY_6UVPTh-pJ~c%Z|_`XRT;IB
z`O)mco*tf<m>0&-QF$j%87C*1|Nj<NNZ+ntt2&ZwV$HGco<sIThn;86{D1x0S4Gt`
zZ{_BVhMirTDl0lZe%_qmqMLTJAxtA_8QYpvb({5xzBK`+moD}A`K{Trr{_s&Mt1f|
z`#Zdn0x?e-mkIG*xF!uP_B>@gR#*Q!s3qva_2d74{a3DK^^y~+G?w%fEclxL|M1Rl
z--IF~qwd^U!n$^kO}a}<&khS|c6Mbq#rCk;gk@}No-$5k<uBUx^ZbDi^V{1UBWG$V
zHpWKG*m9`3v-9BLW<}jz72g|-A2XSX#oYY(Tp{fsClx+3Gqu1#HMe8`H~etRk1#Mg
z6h7&ntJLQo{mQ2<z7n?y-c<ka<iy@yMdM&)MaQXAr{4IstAk_q%$W%Z3s~gkjSF3w
z=BYRtq%P1*IIzVpXp*5q!_^BDW}fifz{A7i<HK|98c%HOlkW+Ij9U)!{Fz)(|IgsW
z>BT(fpo!&3;{t`*9URKS!d?}MU&9nSpV=R)`5WJ1(YNY}zMHDkf@6Y0Nx3;DCr$(?
ztFeFhv}wjn&TU&Vx;i&1`tDRzKJ)0IeBX>?k&yzEQXRz7lYg&Yb?VBcLn{{__IVO9
zZL#v<7EYd9Pt1=OGX1$LZ1Um1ee)KRQ^)e4$w!5ck58<wjx8<iOis%Kr(Zf5f!-^e
zes;I~OVADvSS8_htp3oUHNxt%wr!VwWWTthzyIjPz?5S=wUUx$VPR}NERFk%4u(#B
zpjFi285E$&*Z1GbipMy;XvR##$B{qIA2@K}$PoeY;|A$`%?lNmCQD~9gl8SFw^UbZ
zy7U54A0#<VN={1tef|)Wrhx%B^BQkQ!_%Gh7F}F6SGmL@-Hz2ieLCxS|L)ndxHfKF
z$S%#zy||03JZ##dNvds^F0Ikg`Cn+wuN(R2xLQZ#u?IoTGgoa;mn!@Af7Y!3=6NaI
zor%Y`I!*5HPrR(w#>y=rB61?~c{}@q`3sM-&X_4_VG&>giG?J`Ny$y6p$p6#55H?l
zNN|w(#lO>@t<GLt_|+8~Vdlf}TW;1|3H>u8k)iywy@YsR!0cIEGFDGY9vzwe_V)bv
zou58FJ|2JOOwXiAGHPl}=gujyvDNVW;N$$oZY;J|=U}MK1IHOl1)0s911_-}f3A<a
z_y6)T!{f(|b8dfo|Kj3*|9J*~ejPn<_UwYSX9Yx#Jb3H-*Nv^N!TzX$k<p)@&69kf
zVdXgO_xFCrXT>L$mi0&!_H1)`ET8r0NB=^GnJ(o#KBwz%WGPf^YUnmDC@`?F;IOvf
z`1|+w`{Vul|Nr~@_(_UWZk$0vw@u8%F7<sMH&0}hyl{zeT@<I$|Afqw37IE<JU{<`
zeY~WEgv5>=5*qXC5AvUg>-qLs-Rmu6cFf6BrvCq-&8(o{aC1vrqFpO!o^XJRzaq17
zPTquWE)z4q{R@mHC^H;PO-pN-)8p-O?pTkONg>;wPUe2jAD4Nrg>i3rk(8*oA$#Vv
zh8ZHw+k_y)&`v6S|Nl?)?)&et@}YdnTJcx09RDY3$+G3V-giNWi@ER1NBOCL<)xb?
zTvAem52rb->XGQU)~BB;&?hFvye9jA!O{ytpcO~||2wJl)&KwZ_vh#S#)tp^|NpOG
z{#0SAz>HMpTS9?bmNe`LWbI6GNLp<9&Hjvx)-wJrE0PXOy!&v|k>zr0ns#taYV?=n
zxnZRr!98idePCZBERXc<umAu5C$pzy$C8G{PZ{^iFz+mK4ErFj9Qj~X*o8|h%%(!@
zMYS#|#VtQBu?BP5mj^I!QTg;){nS>_%qpm9urBQ1;VSJ2>4pC%{!;r;_QL*#(wF}$
zx+ZoDSUC8}|2OOsNMTDmmEh1T*TJz)s`12?hLx=KcU#nVJ_xdzz;Hf^FNPsE(HK%+
zg3OXy^5LEKmPoEn?<GMkf-7SjwT(BFrAth3*}(MYC>wK@i&26Lk4^ui5V?dbm({f&
zLd+v3GtGMAz;q&j{oj;E&EL=Yy|%(r{*V8!#ZxpmFFi;|O8LW`J<aIAF@ptPPyGLX
zq55vZTcHly8J;W;qhw+#lnw;%eBdv}qAX_|C&?kI6){<4lUqz1Pgfb6p(eBTkAJdD
zKR^R5bHcxWyk8Y-PH>9}oLO;IJ<9rv{kul}Kb|an2A5fww|!#vKXo8}&*Z2%38Lu^
zz11-%lNy5ASOk0|kC=EsQoGa2u)6>My06SjXt*ld16o8-%2hP0AzW`l8hcdhp@xI0
z4x-0PWQ&9j`l$X7Jo@ptwq$3UilEKXa|f6|1s(hUpJ#vFLYP_JG7~Z*ez;!r@mL(!
zvw=^`h`BhYnbWkPU@5;&Uh~8mvQcML4mDilI(YTjfyq@H%seCmxS84Zq@GE5Gh;!h
zhDxa3{`$pJWT3wDl(DZ@nl5v~dC>>!Cdmu0M3z{JdpvB6`CuThD(s)KB*$~31w|Xx
z_=3Cz*Cg}r*z&-im&c;3_aN(sImNv@6NTqB?5lU52x>U~|L^22GeNWT*#G~6$8R~i
zIsN~ie!xnE&0F$>^(sjNFA<-Hn9U1X-_+-*tYc|ZH#HEL9@8P1av*RHPtE$4kcQU_
zN;rOeWL7PNx=i14LYw=-r|imG!hZHo(<$I-H#l&EN22M+ot{$;&xC^OCQM<vaEjk%
zuY}D}{!Pm5{2<M1;hO*b{eAw(7k<lMf$ioG*7C9~2@GV*>E&k&VGGk^&Jsvo#+hQ4
zR&r=3&x)Tj_9QhYOf;6bJx_(v=57D~|C0ayB>wm(>&gpBr!y;le|-FYd;b55O2OkA
z2WBlgu~vZX#+2LfIeR177D+EW!F7R)Cq-}zQ!``v9wP&Rj^rdMnH@(AcW-s(5y+Xj
zum1dp&+1*!(GsVb(Ut%Iy+7XnzoK%Y^z0K$kJk5Cim+Yz)Xcb;L-Fk26+9-BW*l+!
zIWM+fD#=Hps$s^$OV7_s-rM7tSq$}@O5gqc|LyDl{l2?<zmbv81U7b^)C1O<Jv<#V
zt17q-#QZwy;F1fPZT;~5{r~Ih|8sC}-mu9?iPg%8d2(+g!-N(NGfT+=?Zw%f8hWe@
zjExP9gLidIIC1aIWOW|>_(dAfLei(_@Bi~3K0f|`ef>qVT?-XdG}m}L{#QPk^e)iB
z-Mz0SFRyN2@pJZ^==6^n?s*bI{r{BSHXeADlVbko_=Kh9jfKzooNH|lw{nXt<w;_@
z_>k+}{d9-j^Fb5Z|NlGPwEOb*_I~sHe+C8{{`^1BGSjm0q=C{@)r`l>c}~Re%zyCZ
z=KudZ=J|E;`~Lj>|NsA+Hx@sCC}^L(=*68PxQ553H`qk4vX!ey;ezOzh=T^1LWlR)
z^Bg;-vV?!jN1ik1wB&mE<rkel1t}VwK*{{y|4&c(FZpm4$$p&AXyt#Jfm{E{m4@sG
z+{?G`tNZ)?>FN3F<Ntek85tWZ_n%(i%DL`H<9=U<6x*hiZH!xQNlAcO9Mi-O-`~%p
zA8)j$uJ?0GVoa5xFw4B(_Z~br%+9vu@;hh*dCJuP`}6<f<M;RX|Br}}NO<``e$9ad
z2idm?2RsBKF0-UfXW^PL^Z)xBA0s_FBHh|rySo_|zi{)GS+a~jCY71FaOsHy%!Y<+
zm6dJQ<r4F3lLbUo%ff{!y}K3P&OSf?;#O!g^QO?J=jZqL%m1(W+jXa^;mxF@RTs|N
zFA-xf{A%m;<o*5n+uLgIZGJwxg{8ZlUHx8h?#G-JNA5IuUXY17%eP}Wo9n5Bl$3yg
zgbUZxE$ZexeB`<)uRy8#sHMx}&FMVz?G{SIL+}2+ztulJyl>~9(HoG|7xIJK+j&M|
z^RhSZmwMk%KmYH*$&)jt9MRT1*)oyi!}|uO;Lq~TGOy#TV%|xXXfiV`St2oKj>L~2
z5?0od=jN$yyX#Q)u<*0m;m5~ctb`^9P>pP4Q}yNj+vqPDZKBc|2e|@2|Nmd#;U!Z$
z$3COB_y7O@clZ7-KG@X1-0yEe!Gn&TCx<5LbZB~SQ0u5)u~h2dmaq8%OF1nP($YG5
zQZ8IiulUlDcuJ%#`?^O*>7QI~lc@rq-bA*YpMNnGS}5Po{PFGW{q^zxlaBnYmk{p@
z&=RrP8=vy!{r&r^!|Q!!eJMQJr9WxXnVB<fnwofAp5&}4>|ZYN!})~jiMJOHa<i~V
zNJ&eEgi4mZNtj{M`S6)<Qa;BKt`Ey+Ecf3yrw6on;{Sgqd(Owdzt@8@>+k6kXG%`W
zn{fPu;9apRnW1b;981_#*fue>`Op6+Yxk%4VAJ~D<^L1X(oURwxNUp7^Q+|ves_!o
zvbvOtC%$g2w^+g!SUuSzA?M75i3cxSJ-VSp^YH2EJgKQS*6&~NWY5?C|0V6}k6n^V
zO@Q^N{AJ?z|EvD@=l%QpJ!j7FL_B{Hc)i%*fXJWr;{VR;9omZj|Nrsv|GByL(cAyU
zY<zTkZS;OcW!?Ap{yu$rR5bL&qZ4oHB{G{BmETDgsB|b!(@|)cKHV{9&cnaeZR_Jf
z-mdsFLs75q;Q4+>Kfb>XkxH)xO&9{pp7VX11WBNWd}RLX$NUf%_|VUIt335U#P`;F
z{jSkx8cjYinil;1_5Snodh3!G;(Mge9y+vO*|9(W|J%=SJL9J#85_xVCo}1VDmTZE
z@BI%SySM4bOGrpa)cs4bsGs-nFne3}b@0SqiG+%ZL`6lxyt@u5Q9u9Be|Sc+O<aG&
z3TPDj%hdlfh}r*d`b<eXzGGkh+qX2@IGtippU7taPg(lYySx8SPya7t@ge(Y*Zc49
z|37`28WNQB?|c8lcX#*m@bL8X@bt_5ZCtDVt6#Y=K};@Uu}pT5gI;o*K|)H(jzZ=t
z^Zt#7&8xHyH?#9(Uo&`f@9cvYClazxC*+(_&^&qI+_?j%yb{FN{AWKt$o%HJ38Zwm
z5@QHjDLtoKXf2DiF3Z9FKSUfq$lEz*OWQw`f8%m7$dBc*>Gn9TW{;$<DKoQH-%UzS
zoDuu&(eG;}xlTFDcFH%%9zNS}JF~g|1UFkkWWe<++qSIBns?P8$JaU5z1BZSiRXP_
z3Rj~^?Hg5|W=lThpjl;k56!nWYRf%n=9=I1_`Ra)rZ=L|Vv|@NcWiPAU8phP`Gryz
z#u)K!yyu-|4u7vdv0lW2{rUTQFZw`B-v9qUG|A%Lo}c9(AKA0ByItXoU~@aR-{ZB7
zgo0-ac>UwC&FS_2^Zq>k_jmvOeSiP|`T76$_WzX?5>uzlSiI<ltV{$iPee=WOg@<(
zr9X|XaOULjY*1ddpwC4~GBB8dxsT;w!$h^VGTtUpoBDGK3I{%XQe|dlKHSa^>TmM!
z@WjROgs(R`GwbMvVs+30-R*gXcXl5?aOB8=GiMYuP6Pxtrs^t$F$nIjoBQyd?HloS
zNaTZDz4v$hrzi2%)oM5LR9{Tui)d=_UwOb>_0Rs&*Wd5%uK)Kx{mH{b5%Kg*8xt>G
zNii}?+PE=MRIrh6pMCAG@YuO8pPjAe;o*tj_oqGQR-1)TXUc(#VF%(nc@C!>JaFK{
zfBVB17xS!7xUR&;W>oW=&Gw#i^z4NjCoWvj@bG1D+v(|&ar;t!{F(XizCD||dRzT}
zi5W9E&a5>sP!NgPf9}H@$!~t3M#2C8cjkhVgzNottCmP?h`QjFySL^0{<hY+t*v6G
z-Yw%;(>X&%VgKxv@`u|S|K~ptNR&BT7$9QEqMT%Bo8}YI;bL&lm?!M(#MCKEGo@~q
zzK_fljF`6KR5#CpRXtl*UVm<`H2vf3+drHu&)dvr{l04X(i6pV4qM)uwl6*QX1UXX
zb?a+=x2G%XZtu<9RrhN0JaDV$P~D^FyUYK_?M<38tt~J7=l}oudmOp+btM+u7L=GV
zQ!+GE^6Hg{1orEOhqUTg9z0IoQSq_AgX8gR^Z$|(ptTn}cS`>I^Yi$jCYirHD^@l(
z1{|K8xI>Mb1GI?dil#=iS65>@`{}C}KZH0HToM&_SoZS$or5ziAerRQCEj1(`ybx3
zZL9ts&}R4l|Ns7S6%nb#72?9qOF@gfe}1+vEnUVhzo75$@97VJmrq%JE>UG-hK0C9
z!G~9(hp(;<_;!Ck&$ewW=Qvc<Oc#V|m%lT(v#a&<HP|3{Tv5%R6CXY*Z>i*XdFl|C
z_LHx6AHFFwugPSRXKZHnSUk0{_38hK%JnyHEO@u0{QdkL7JH%+4_q`%ba+)|&@+LB
zk-h%E(HzUeAHHZ#(N;R7pwqLC%~JEg%^h95K1c2AL2IHvd{p)`hx*5^;nQpJ!=Il|
z5Yzco!lqPwxNq07dG9AKS@0<T{=dgh|MI`|F*Y*F$TG66`qJO|(ted&)0>Htj1(T6
zlWjB4Uyvg#+&16d(~Qyo@2CIvpmP8E`p3IFg&<Ki&%&-g?Z#H_suy?v|K~Z;$zUhi
z{x*qs$|+XOyMNNt|NsB+u>J(&ivRzOYkq%=5BOl;=62@iHo;js5)u;k_9R-A@mT#i
zaP+8x#t8+5hIMS`btNkpY}VS;o+@}JvyZzEnx86j-d$eKbAO-Wb^j9|+4sKSza((5
zC%k8><Ntg6{~tfvy5e?Ai^RSC|G$6LPE%|Q^Xu7R!pOcoU-{LP`27+kB@z_|c?X^K
zgdU_93EUL<`*-%kui0(e^FL}r>x!8cAg9#H9JR2R-1t!bQd9r`6%)DiEdvD1zyH?`
z|L1IVxu=Kc`}_aTpQI$DACKzn`2x;`Z>0}^ea+)P&td1GLp-~7{W!Ksf#dTrS)0Gp
z&(D|SlR0qxK?1BsJT%i{UrpkU3c;fmJzqLByaR5_N6b>_m-{O$`j_9_-J#*(-QEA~
z>;4?SV9EumZtUvQEb3$o|14eB_H<d~nzI~_w5QA4OZ3UR+eZt5ItQSszjCSHo6}#c
zZZ_BzG(n9c?Kkt%8wn|Y^mYEczh~Q`SpVOs=HH+H_xIN?KFK8f=jY*q_i{-s&7GZ%
z{r!$nQm4Oa967TiNmeU0A;F@w>)~Z}COv3y&$RgWFLg&D^Q|ea?_`p1vA=$*q|ULW
zu&=)S{lE74_kaJlKin<OvuDr$|NoX|WL_>{wXZjd*>T{*FViW%-^({o6l~sMAvW~{
z``JZD1&t2KSp5G#Kk=zoi|7t$ikxZj?|*v5uPF~-haXZEHDlG_Y@PVgUR-=p=$Ru&
z6jY8B{QsA6>Hy!peW@0elO7&s*9ecA<rpn}`iR6jJ}tS(9ZkE-1=q&fKurr@yQltq
z!5fJyE?c6dKJ?#6?%H3`_+r(c-^V|E)jqu3pQrv`z|Tn&6B+i^r|+l|y(pECk|H3Q
zI$?rCezCy)Z3zd$igNDzg*<{9CvJIvpP`I3TW`pgl{P-=O8vb_T92Qn|M(@U_~|h_
zv#{{t)6*^7zq+e|_Er2YZ%aNd;Cu1^f60gl2_D{!^Zk8&zBaCWeb>6JTfCF25@M**
zgyQ@AQY*gmWogWs{b9?ri#4C#%a?F!vlti|ZP;XFQ*q$HIlm~r1*ShfFkY;d5%}<&
z|M1(}JmPu^Zn}{X5;e6OV-6ndD7kI&?_klHj@>mgAD)wav%2~M%yf%&@##N4vD%s@
zlr8(%yuV&@U%k7@N`;?~pZmAD_e=cyC%fUx$LIW@4upNZkxtYR0ii<+793cx?AU_U
zrxg?rE?9f^!u8`HUW<qD-aT-8J7{OX9IMtBiC&(G`;T|__;Il~MD6yUFC_W~V&j&%
zhI|eVA};gWqLiu`*bC&@3%GiPbJt(6XgVT##Lac<&#tQ)Zzf%s!Tl%x?ELvBQ{ud=
zTqWE;*~}@4{cd_}rb^1p<-M*Ord6gzp4+A*ZdF#F7<<!fQO7>_>|K0<-|hz291FN^
z!kg+Dx=<wZSM};iTP2)tSIMWz1dH(d)i$_W?VsFL>?3^m!`E5z;BsioEL*Nt2c8!D
zn|>cy%DSF!*e<Z=o?J@)f~n#H(t_TKU1yub#O%KP)eKw4tzs}C(WvFl1ZS4y6Pu>@
z?G*fH-J|^Xr-@}!tl+`#_h(F5l)G_SZt^_NrsZ>1GAlb>Ht73eo7!pUBkUE+)x0&L
z`&x!?;DqNcf`vX3c1)K(M~JQJTK9WV=B!C7>R-ci9^~AuFXLq{tu6jtv=O`;Zq3|<
zTx|{lE&GccdD^no470X$=kMWvpd8WSy!)ktfxOZoc@bB>kPr9wB_@`Yl}X-s{(ax!
z0~SKZr+a;<`g<dzhADt+sosbA>4zrlQ}?|n^_-*W?wxNF?ae03B<MPb1YS4cy=LSq
zak#L%dAqVRi{PrRMGBr8-k;`_{yqQwh3>T0b%!NxeY^8vtJ>*zoANF^=ng($=~5>6
zQ>_jhc1z~ka<v5TwAfE=kBIJwd$92N$Buc<^Vl!A<=8ad>R6!k<FHtl-syiF(m||F
z9L6n~hyJL!$6nsH<DGQWo#REZ`mFoR-vw>g_+TT`UU+2NMU!VgFT|^#c|K+Hrgg`?
z&zmT+?DD&8z+=Ie>N)T7wlB7oev=efP8StCogcupbiyPJB@PZ}8~)QyUfbTTD}P-)
zr<#{J)OD87afxHDe3y@WuQZhZTPI=HdD(V%pU&=Tvj)g0<(#>O+)V)j$LseTacoJ?
z=RLUdpkhngG`s9Cfhrt@LM#UzrS2&#bN1uAdLi1#^VP9t>30_I>g(V2&GzqGrg%DH
z{qoM&9Pw(*Un(}lY>EmDbyH@Eb)O`_a>Q_-zeJFTM(dO-ra?)!x9EB)vP3<u)Der(
zv*4SS`fYm5bQ|;C*RN}5dfiWs%<#SZ=W{_ySQnGRc}tlad7VmAk~SyqU%u|F;;p}S
zhFm|VgfGmEZfpEJ^K$3yx4gk+){8O@6}W`$H(9>;QK3!HG<%ERETFA~TjttxHzf!h
ztLKthnzYNr<Ng7|jP3as+P?^!n3{HMVadO*;1wq$CB(uRzw)5M;(C*k$GXRDZOU!R
z&F_3yS=Dv4>8H%)nhV0QbtQ9cJVSgBZ<70v)$M#eQg5c;L?J~E=WE%s+7uMd-|nwZ
zk$i64%oynwdFlP;`<rh+{4K)j^q`U<XwmEKt9`|1*+*JWTdtXTs=`O+v4xA^I&T4k
z*@ZSnQi->(n9getnxW>~93;V3>bw5<OP9a}BCADj>(1s4y64Q(?xQw8<iqaJL*B<?
z^kn!{PV(n6?_dIXZqDr8T+InQt@cYEd|RyG)!yW{%hLK+g5Bh3c6V{}<4t->k5Yxa
z9N8Ur{n7B4xUI7J$&;cYk9{W!UR2)_JASj1RYvmI&kjMSe_wLHRRvzZVdmK+Ai-9s
zGf|IAroD2R&d#`*YHf^)5n{&zpS=i6ls@^_zb8D@rvEl?tb6EEmCL0za=&kW-)YL*
zuxP^b8EUiKL<HMn^diLeoi{mIS`u3ryNc`bl1vr9$YTZt`#;w#(%3CN%T1u>UNGOK
zqvmImV?Bdcy_$aEfHcT2Q|9<`H#rEj%`Xa^`0bkE3Fl`glOOPZt1o$b?O^hZdqtXJ
zLaj=Z_@rwiTK0ZVWP6snZ^q2HCnt(eZMt2sKg`NLsq<iomqgIC{70p}^B!B|m_>*M
zEi!ddU~z2}h*_UeoBptwrM_@K^ZCQucE)Vbc{D>qiQ~iH=XZB){8N**Th%vmZN&Qb
zhj-nNt~$2TYoW+tMbD<!Mp7H^Z;mLiF`B8M>U%inSl;o9GY=2EbP1i9y5;Td;9mzD
zEYj@WXT4OJ;$Oonr&g&u{ru9y^L8+T{Ig_^Emw<yK&$<h2}`1MkKQ^Ib};P2ih0fg
zYpz&VaMTN^J`!uYTN)U7qrCB?&pn?>mCkia?)N9B9{G6tUid_dwW{+Y!sYs%({iNS
z4qCk0wkyc$*Xc#u|1UlIr{3y@*}V2CGt?X%B!padto!%k@+K4I?_b~B^50kYotge>
zmd)~2U#fr9m1G@%_t+%U$(+~W_979d01*qm*mePpqXr639ithRsCb^68lm@2=6i{z
z_pE0{r~YvHgUc-?twI9>gP471K0IJdGU4$*+hqE+qr*T#+BoM{i;~CP$jBX6=gi^x
z^M~jDKEn+g3?d>8GqQ|0I1LL54K*|jUc3a6e}1#e&OCZwyp^YK<tif;6@xhz2SFRx
zetzbO-*0qh@9BcSRwvIjyYQ&&bg;_%keRh*%E1#S7A!yh;Av|fQxuoNva^hi>*EY%
zEF3DnvOzbADw!49R2?b!XS3zz!G>FZj@(i>>Z5e;9*?c<6AMA{g!FbXv!Ws%`+B3A
z|K}32PAyo$5i7J{`SFA_HeZ%Qhj^5fIC_12TzYqI+-P_p+$B2FFe1{>rshP!SFOXl
z%X$3g8OYe2DEMQ+tgnCg_V$o@b}8xUAHKO28b}?O%CTY9pYQz-f0ed1vwvJ`bR0B1
z&hY>LoH>S|p1<q;B*lmoQ#Y*W-P#*|QM*F*e3D{BWF-rKg8GdYVoI}}4A~}aIJy4c
z8I%1IKWZe7nat*PoEID#acw4dOT{E>`xBvY0bPq0tn-~3EBC}=jrP298F3}m#1oqY
zSe9t6>%OMK8F_5mLes*Y_lr$`=ZR?6AKhi-d&BIo!)>F!Gsdyy%QA%T|LoIP^nT)}
zr9U$j-_6gs?>BXkPUfuoDWKNe|NmQN8}hjZ2)Ochdff|tA+ksy$#7b#GFNBoiWj0K
zTvJ&&=Ed6x+J98-syEV}vGb%)-m`rMc{cr<RQmQG`@f)nv)`*(CbmY8!?&0x+}1d{
zNq{9sy07`Yv2TlF;E{dGs-8s^_X>Li+&4u0pRx9J{E`pccWW=F$Vjp+S+XiaYRaPb
znb9o^UT@IhX|Al>^lJSMy~?`%!ShNBR!c0^;M(-&tL7@uBDDYi*UYu$Y7P)+vFEOA
zQfkt)X%G`LyLX8BiGth#Wsh^JH=;V4Cb&s=PL298`I)0ii$hbeZgh_E;Y06q-ta#X
zXnH->WNPluZiRc3+^?8!m#*}?og{I?tkz#3VHQW@>qizj((~GvZCv+b-<DQ=@t<G6
z99jCcGeYc}Z0J&*K#^mCa*g-ihln}_h!i=@P8VricX*QoTglwx%%_uPaX3$E(Wug|
zOOcJcBmcPnSBBK189Ki&N7sSN;VZKt+wzxtDrn8ni?!PLVOP_Gt2r*0mhrP4JS6gn
zf6M2&39BEz)%orBGfcGB|FTWuV<D~II^3Uo1bl>p)EXCx1TCA*Z5{mYa?a;=hn^>v
zhaILZRX-$A*taP~#*pp(#aW9EM7{c@VfJp@Bx4nei>fTQrmg#Z?{gR=!>(1>SKgoa
zyX-}PmGZW=|4!tb=wVjrn|PVs*tjHyF;|s=nR!v1#3jvU#%FbWd>JXLR~s7`7@Rm|
zWN<*O?ZHa<An8X^ObG{O8%>a2m?1k8w4*sD;l{41i{v29QYBrXfB#Y=_RM&AQCY|D
z;L|*t+?S24Kb&Wl@wNF#9tyTsv41dYvXb=P>f;9aa)*1RMJz=jF0*X6t3MYI&As<M
zr*Tc8Xsv)tdV|=(9hdZG>4bg#!aik5&;9+9)AUkr?4A82P#-#+rnLOWz1`Cj?^<Ql
z2(RIrcvT^Lq27U64w42+68;-|PCNJ*Kjg|_JmJBWQT6|CvPDr#;=7nHCv`vrFQDB9
z%RhhrZGL!)CQn-%kCarv@rScy+D)t<Jg_f(ZGYj}w+%)Y7&htr%TTrp`gpq|M`T@L
z;v>c=Pqz&QZkM}z-rY%j@%E@uoC9da1Z3LghoB|RPfqg4+fSN&>7gputx%@|HkUb#
zj7#_XwYf$xi!*Vw*)8*m>5?q?t~tTh^}??IH%>(<*|J{$`i$4P-v02`>?@JtpswZr
z|4PL-Ufj0_?Y0aFmHhN6;MTnv5B}Fr`C4Bf^!kdd)NgyYX-7p4<}=mB?l`h>npj1b
zfX%l>qG>;BTV`I7C|K^t)}!+2ul3<AnLNM01uk>~PXdD+S?}}lFngP8H)#KAQ&Ye>
zvz8@nDGgye4@_OiY94jh#X!w~QF7%g`)#xMEUg~qITvevJa>4#(Al#t)66rJ{{J)l
z^YgGlGTY%@r7Ajts-SK14FCTx$~f@zWpLX(IW|Q_Hg<M4Hnt-Z*#2nvJP~?iFZg+T
z#NsX+hoF_s@z&RtI4sF5NS)F4KvU<VIv-oll#k!_56`jW@tz)(xDpmGi%hg;%#aWf
zmHhQf;>=mW;v$|6UCi?=F5C#%9k}yE(~OsGAq`hq)=XvfR0}xTI%BCCv!*J$(f9lI
zZS(CV#dH#8nDPF8_-zU(MS>jXWS5$fGGSWUi<gNTwzuoeRGeXCaGAYm=0T5BOfEJI
z;Zqym6a_JET7AN}kx{vKkt+8Ng@bOOdDr-T24`j-HmJ2a+|D23C&|MDn&$_ZcuUV%
zP_XS+6<d8h8z0{kP4hLH%({gMn;t5KtjU<bcyPthtE(2M6#j9Kb?eOGtJLieUoVk%
zE+t|E!)e}_5KtE%WJ=nB&mRq!EHOB8+?hY}SX7-sf{1j4<coDGi&U1|NKTAmpO~n8
zFpb@?llfBi-inEd_iWmHXP<~X>I~h)zo=;4lLrYKHYR%brf%5o9`nC(tKY$?0TKl+
z_C;ZB%SBIdi#Fx*C51d<kKpD>(eBW<QFw7)zU}I2$*{Fa7PYe$J-x;Pa?Ah!i|Qtx
zJ#$7u_w0lB=O;`!prG9R;?M(6C64P4GRhJ{oqZlD+<1N@pDC(Hh5rDzjm=$QHW^iR
zX1CxkH;vmmg~5{nKPNrA9105Y|Nj>`)}A{60_V;xSanK3_UwbFtt(O`3QR0MglJb-
zaUXG!Sf|r{ye7rL^N~VC!hx#~{KX26^~>|TzGkQscc$Q<&D=CYsKY`ZH86yQvDMbH
zm6x|YNqHf;`p0hFA2&}lxol(qsLt1PYh_UJLw}hJt_{;9L8JY9PF%QpwBV=FOzwXt
zKtoHQaL#?yz|6|p_8<YY(Y3OYO;2x1R2!q_(LewDRWBwiV-6B#X9-x%>!@-dezrpK
z#}L)~`wU}t9odj0dHCJkD>A=SgrNQkew2`qkdT@xAea~soc!TiT0nTZfvIc!^QP6^
z-0VlV9|bx5QD8s#t3%;HaKeH@!^ubY*Mr&$F?&vINMt@d)0ij!Uc&yg?rTu<;vY3I
z^YO7UGO|fZvUz*6>Fcwxva<2<WfUH2FjRL>(H9p#pw+$q%E90p6Id=tiSbCRWb9@1
zml4-D%DI28;ETrLx3^6sZz(~`Ua5v7jXW-{k`)yaKE9Gurb?QaO7ieZ9yy|syHQBl
zbqUj-UePtF%wa44uXy})hU?LOd7k)v1~T^N42snbPgW0^r=$qo<0bU~y!@3XCkM3f
zYtkehCMF&^Ii6p?c$}Pgf`WLOnlv7(o`~A8$<X&$zJ8rU)}sSU#lr&`EgAm*DsAf)
zm*kUckvnBj0r97lNGCJzhK+_FJ{WLt7_3-faN>kPM~A_VorW2ihJSvs`nsJkFI8SH
zedZji;bYYwjMqdO_V;&Pvv!@<RO0ymPvVZU-o)#Apb_<V_fji9Ha&cPzA-cO0jLuS
zI@qMAlbP?q>?l@N$(9ySsn*jY5fTDg`_s}Y`Q+(Bb=O2zl~9KYABBCVj$gf(l92G@
zQ!8kM-haMC+@1uBiV2|8IK=fO?(LQC;bfA8y7@{cv+#xu4*rpXf{pLG9zRODa3#gS
zAYsC^ws*@~=Bzq!u(7yYrS<UKil{aJ|2uzw?bK(wyS)G5o5;4`-z7ogj74Be&d-;$
ztC28j-v>&J|Nl!RbuvpHcr6|l_M$IRPmj&ekgc?o&DE7nP%z`;xl^YOoIQKsG`Fjb
zjmB0X=8KuEn_h^<&2XH=`eel!^ZW(s8#in)sQGuM;IGx;mzR0g$AJpt51&*`c0Smt
zn|%Q^UkeJ5L!He00zyr4Cz&$$Ib^8@2C^k4zUV!7<_wRH4$qo38Y=AUCAK;)j4X~z
zA|fUK{Y|d;KkwmP>$dds67y^lEB?z>ZTP>xo+tml)A^VdnSH-sUQSrA-vL@?2r_+M
zC$q4Dp<zLh)AmFoBd289rU%?N6mFO7uWx_n|L6bwhwtNGXy$(?eJF2d@aIqGJ`b@X
nfo~r`-UsO>bdnO%QA^AWsab}<kG%G~0*Vq(S3j3^P6<r_-pxi^

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..e49b5816120d43a669264915f1b6747606e080e0
GIT binary patch
literal 18109
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjYw>h(45^rtlpw*nn1P*v
zfq~)we<{|*4C<rI|3~^a3LY>JIHd9|{-!_I@()a%p#=`C_vAjvcg<QL$Hwa8!4P#T
zc9HeB3)daeBUT%m``4ag1_j`qN!ENV20Sjw?8o&cX6*R?bAkZxgdf{o`!*c)vYz!!
z^s>#_|8tlH1r~;>Fe!<w{C)2BwQE~DLnfZ;IdenG=<z0nK*d>?Sv1!A`dl^OZeW_Q
zv?X%a@imb%TC;i{?tNbRv+`HrkIzqPpZ`84kvT0*QgH9`SN^><bNRuscV*H>zGed+
z*2w~@UMo^2PVD{vU$<q(3=S2KZ?7kWmAnbNy6fSai@WT*owKiUZMbG}z4e0c<z$&;
zuf`L?mP*qn$+9`8{E9o;A@IO<P3yG)t^?lpI6myW_g(G4TV6qf#bGN&mp-sjea<#r
zY3Yf=6Wcy+`e1W<*QtHpuhr6b>K*;_TD{KJAMB7bQ>?if3<O+EXL&Sfx;S#Y_}iZ{
z(Kte7!N!7XZhsWAR-T@ATI93b?{dpaQwryCO!PR}J27|lY&CuF(9limjFVjgk3_k9
z2_{<v@4r3wizD}*pM`rZZ*F!KT$``V(xY72FYqj}p*ZF3oze|rwf&h&&ma7~{@3mt
z69Xs-_jERQcTb%>xp(^XSeKSycXtQBKL4Oe?oLhKZf&tHEuHP{jZ?qcD=S}naYXoX
z#FYi2iV~V_4qm_h%g28E|G%DxN8m`qk_YlCNB{i%{Qvv=`u+9)|9^gde{=f(^>Kg8
zpPjj%ef?kUj}QOPO1JULvzcAl2+A-Zn{_&y+uNu1_j?CSi1z9_ej~!eu`xbsmV4B!
zV3(Gevwz!jbDQvSOy&J=FvZbDV!~sg>{$oq>J&_qUtxOAe#;Z4hl|t})Mse@|NsB{
z|NsBZ^Z&)}`0)S7$NKQ~MtkbdfB2$#_;^20{ePoBzmI?T|KGt4l)eA|mpXCe;Hgt5
z&Y#!RJEyLxnUSWpXrZEl_GxXsbBZcQ&K&$Wzmrq$1y96cnPaI52@OXWnYX1hsw!O!
zl8EX_y8Mq{_vM1DpY5yIbayIf|6^a@kf8NnQS-<5_xJy+`%9dgo4liL?!%M9ZM(}Q
z<?Pdb{F?gkx_+B|J)1%k$lL$_OEnxhc<7LZ(jiS<-Bs)LGjnuSEj_BPbxKv|%%PK<
zFMX24`2SD1%osCgqTt>Y?$<h2FFbI9L)Rl<_M#6eKlnSlCoxTk5{;0SpTQfkV_N-*
zrW08YT)FS~c<BFpAK&IbU*g@J#2w}R55MKMb&E^p-Aw|y;IjIzV<O;`wkZ3Nfbiq6
zFtMu_CzcjX3J!lRB>MEhqokzNzvoY!Xqd9xu$$}59A5Fn16TcyXm?B#dGKw~RE>;O
zwr`sl&(=KYXpd-#nVsOR@jqbepZ0Lo35iqv=S#%xO{(}a<Kd^&Hs9G2_x2`N{GRsk
zIDgd<afp@fj}p?-LV|_OOhO7vL$_@Z5EM^OO-*f3=cout>{T>r@#Q_Yq&aHBpTh?1
z*(}Bjj>%th;@Ir>vws%T1uv#2ZCcZ`UjF~Te}f+X72XM78mD>H{ZISxZ|=iKu5DLW
zOX|fYRs5Xv@UHbOHyMb1xsL?Io?W><ecQ&1U%!V>nx6jnL4pI9I9pFsV`C#@xH-#i
z1?9*8|NsAg@RFq0tOH?d6|}gw9BFhmoEF65%oxSnw70c!>kWssF&B<VN(4;ih;rIl
zIXUrtTpKexD0C}6w>^CBKg&fPVw>lqgruY?Q=e|$5Kvfpb=%gdQ>UdRB_uSsax!nb
z)WjIRMq#<A(k0$H){0f?Ov@6p8<Tt`BZPUc$?+CF7Fox$uA!09)KOuwz_I=HJpS{I
zWNc3rd{jDoZ!gdL`^GhYy`Gk7L+nX=l#r70;8BvO$dk`suND<vG%^qn7EW$d;aD=~
zRKr?pu9IhZPP}!zvZP_JmZM&^!o%)a|2AyVo_NzDhG~8L1Yfs}tkH^%p-<%76s8?w
zUg_t!jn8`~WBP%m|Nf=^_|W+9j%C|qbxA(iq#d=hAHEJ><*EU(H|o)YM@hlq;^n2G
z+c#gna9vzbAfffEtE5E0uDKtK8W@ca8ALDe=;isN%A&+nplX-H$?YmrI)~YK{o_sI
z!Mw~u-}SexYGIrfFyZ#X3{UpzM+ct#`+xq!dwEdi?3XiK6L+@Yi^k!}>OAuHL5hBi
z5L=xdJ$RlT93ZfHTiEx{moHtN`s8tP(^n742}~0ix6VpGAlxnC$zr?vaI=Ka!3}O~
zHrFO{^VFQ=aOU!MI+Bt5z-Q%+DpieFhOhcS@tm5Fa3E~$ANP`|fXS@ajxaL*|7ZB;
z_wj;HDu-`v<@x>1(5BWa^9-n}|NnoHN#TlBT|0NbuBdqN;#C(n_h#ouoq}0fMas;Z
z4>+7Y5|Pa3{k1XV?t@j?AGT;4a!ar+S;gWVmEs`6JmXtZ?vy`=C&oOuv!6$!Ir;wv
z!v%TD5jhHzKeT*$DSWt*nJ4?2LCn50A6|(v-<*6HQXVK77aABEa&Q*jxbdQ`T{=4I
z6gTHaV`Jw}hRQ`v4!SlEJXz8O4nGpznV~o#yRfHK;PSa2?Ye<U$4ttK4R$GNT;)o;
zEv&mXfiHi7ne~P%Iu$`|%*;xu2efD1-Y!}8Hp!x9#=}3AM%j{3>(mPi3O2lW?cLID
zU0u2I^oauJYbOp|V!X04Lz6k&B;lHJgtVkbC!1}e;}vCdafL3<(%cW`$`dwQ>mAMb
z%DT)gg>^z@)tcDIHLC0n)7Y=2Fg|y>A#Z08v-i}7a{a@*N_o!DSKcDP3AN3#kb}Fp
zy83lW%8!hU4}}E=i`BSYTA7qD2C-FT9SC0dAggQ9RjD7+8aEzIV$9BUU9suFI}Z!-
zQWZJXWgC}@rbN1|Y@TNE!QA@8gcCjX^`N50qGm?oEt9s{=8_&!+u6X$MM<|XBg;!w
z?bgvF5uZOA8X6iN3SwhcJ|yA6koYb3Vzb=hmknxta=%VIlJMj*=1t6<B@q4fL0Yn@
z@Y=wpFoOk4jVm0P{3Sa?n$I-wHa_r?i3n!C*7E7U{o(cTJmz@@TecX?u{`|YtM*1$
z4v2+jg&#f?+_;^srh3!bdbKm#eo0A5#an^eB5V<}zu5Z-gxfZ4Z8;F?k)Z2gFqu0|
z#ZW-}<_u5v{Y-0CvaDTnFuZ7ismlXR=6jzGSlwz|u61y}g~Hki&(BM$`K4G?OnCS*
zxGjD^hr1QTvN;wJkuya^UH|^|b#^|xWy_J7Gb9q9vM2{N9&89%_{V*&xuc)9hGJic
z#Dc9t3O)O8cyVmm(yE%&(d`{{bB9Dkww{irXt5!?nPbK5DFr7Rw`v`nl{ewyY;zv%
zaHE`iJhFO+<6EE+Ho>Bz;)aiph_jQ<-Mghrmj0BGl$4N|@!%zsYN^5{7Ms)uJ+4bW
zaz3dz8a3^YyRX#`B?jgD$!G8Iw1^0suutFnW9NreEeVetJRasxc=UclLg>%;@`soE
z^HhH~(uq1!@Ym|_@%|rCiV)jsqIU{581PKC;9Aj;%w)Epg>{poTC%~kg=}2Mk2x&(
zIQOY~|J3y!KTk?6di_e^n8fl*xm(+mGS6%<KKfhQ_?>ycj!Rx_i485>o3cN}URGPl
z|9G{m`sQWk?_^hm{o9r5Yj06f!vD2m=TBGF%QLjp?<;t4sfYLq?O0pHtb0Lurk+u&
zIO{An#Rcy->O_R++<yMA5Zoj>G|ytj3<*}&o6XI-%1S!CJWDJsA|xeeCd^B8$;w?Y
zMTTd^XSEGX4V-R93ZgtG)D)7sCr<qMzrKURCelD@<_CWFCI@e^o)v}7GM+svPDy+?
zW*%miR<rnK)A6!546~g6E#7`_0%O#j!^+bA9AUiz2M!-jNIZDp{P_zP4ir4&Irk0J
zyZHbA(4-qPW=Q<}VId?mXVK!brY1)sB4$WR%t)}5o@XR6ag!Z0^Q#qXZmpmA*H$)a
z9y-wxwm-p4`QJj(b343?VnkUCLz(qbGtz`E>-}s$+f{!<<j>KZSq~)Hm2Vt6pv5QN
zve>>JRG(>RCG99+{9U34PHahsL>@hR=HdVS)QJt(FHK5FOl(`qtfs;ytgI|NhtZLT
z$A^dKjK`(`-4ic4)ebS_Z*-`#?TO%WaZ_nA^-WfaImM=aAkOK4*+aua<&PEl#SVC?
zsj*1=a?NZE73+(LkPs1-ymd<=Cr9GO&5ah2-0QSH%*CZ5I5>9FWL*`NA2Sy4@;tFf
zbJB7Z*c-WIQnTR75C4y!;K+A#m~^;)Mxe%{9opUhB{p_1*IU3Mp!X*;Q(3uf`v1eX
zB{L-1>$#bIWp?gmT3<dvbi-!j6|0P%JT=^~!9d3P*oVK?UUyl*esuDbnKbE-p<&p{
zmDiGzY|fm~C}q#eW?ov?c#~Z}Q1anisU;!}8O{f8NoL%XvTTs(SiiwD&};*P<=V&P
z22Na+r}c_fGlg?A^E-HJ6dX8vcEQ@S0&?d*{4_p%w42AiUdadQFBLvj)rTD%-Cs)7
z6&3kz%(1>Ug)!g5=#ZD7+mdapUWp9yf{wmQJsu02r!^Ls@*8%G`AqHEakSH9+Oh_v
z-50F%XBhVGKXX7N#X-x7g;jFRT1mV5w2J@p9$wRJf##s5ZI&USu1?Or!a_2vEH`G%
zT)0@mV<{7}56dApKNFt~O)Qq03fYat7t`Vl7(X6#4{}KUrvHNTUvj}i4&Q_IA*nAI
zvL7?A6>Zx3lrcRax6)xk2S;-^x3dpV^A?jsZjfx6l$4M#VdBH`vTMBj+ow%PNKQ?4
zD&pF-`~P7s<|qF+mdO3=wvzfNAC{^V#Azfx{rSWr4-?rVzcZM!O_%C<;8w_KW_l!J
zq3n<I2M&Ph{rC6(*Z)iT@wuHbnN`Y3QmUavCCbVnd(I+NZnmCl8k-b0?Py_X-fG&s
z+q!v%L<pqSo+K#nV9VyqU7Z_4LZ2q5rMVb=w_nrKaD7U_a_6X7{}Y+}PTc#~KP6zo
zWr1}W4ub#x&wqG*y}i7BUHzvgk3T=3zpv)!$pate&z<WTGsoMnFWjq(`)2qKrwIZI
zZl~+-bS%h>IQt*uK2T@k{r}i~e}3QD8GnBM|G)qK%BOgEMa^;yof;TC*)imcJvTRV
z-fjVgow`k{(;RQ~E;i+w-YUIx@!<y#4itRVI=niZhlhs;8ghk&7khe4E?*XI)%lr{
z(&BEjXF@^)f4ZU3BA#@|yPT_a44&V=IIZu|$G7wC|HtkBSM=k<-mkCM&$amY<L2h`
z`|JLuCL|;zB|UkX`uKVJ^eIowOhxr|OoW6VKX{UomX>zFK(ar!$0LYGBJ0DB-kvEf
zc@OUI=h<IpXjAni{qL{*)6?sBSAFFd7B11&VcD?R*u%phE6b>@-FWA2V=phEMD-x;
z7RG39W?fBz^#=|%KD??8>O$n*OMyB+W{1Je<L=GP%`+zo-g0++y=l=l+nEz)P4nSd
zp(z-arr<yE&v7%QmbQQOe}8@c^0NK%vbi^Qe*X6SynW%;!!u_}9zC+-&)=zg_B^q*
zeL7=?#EcmRXWji5R7mc)al~iqf|E(<fByb{e`@Od?CW)RcYOSPcX$21x}(!??bsoq
zq$II<^UR(e1Mk!~4Cil%t4UvPkdXTRF1C%8Tf!_4=D?b7Yv#_M&wJ@-i%dJq>otca
zvnuDjk+DoY;JM|<NBOO14w!~@cu3yg_t&QGk7LxWUG4n$|Nr&2SiWhqv9{*NpTD)+
zS~Xw4=2m{Y{g`P^mwl(e{D2P60*-f}KJ4G$+jDMCH_hJn=FZQ*-}4t)&F|^qNlfH%
zb?rHJ?ElP}ot|<(l4e~gVZM3&|9|_#|Nio9PBVlC+{5STZK0C?B_$;O|8M6@KbS4x
z=Ek<=q0W+jT8qAStSve?^{0U9hxzvP^KE`g$nSsk^Yi)t{{(~EjQ{-k`Tf_|@7LGw
zumAcgI%3A8&6`cu%u937y&*ZlWrLf2702RL=?4PZu5$k`ocZC+&G~b!_n({d^W~RJ
zrD=W#dVieXkjTtz|G!>Nj!joLsWdS+%isVPr(4fW>p!O4%*@SOO%MP43`?DkS+Pb3
z7Rle**08kyzsKjLGdAaac=xVeg>9Zrw}<xs|Nnn~eeLgiwvS(apYTkh2?t+YEziIA
z=fKI6Yu4&2=$z@&J#<0hN3v8f<18<e5AvE9E{NLH{z^aIxBltV_xoxTSCu)i+1DGb
zi9DQ;bVxz{_<_UDO>S!f&oxNBE)d)yw&qo2q~xN-(1z)psNGyG1_E<#{C@wF%X{{f
zf6fmMT=H8t(Zp_+vSjy*jk~z+wdR(YEs<(dG5%CGvw){`lBRU*ZtZE>@7w}Ka#<!B
z25z~(GAuU1`1XsK*MF<${pUR2|M>XtgSWyje^GIH^vUn1qS=EJoLXicF{{t1{x^E;
zcxmU-Rm%h}O+GL+PI1Ayi>3DU&9A<{;@rvX-!8&>N(j>NN@^4p`Y`E%{hmJ#TE-F&
zlw&u1V$R}YKDB}8+M3SD@6N)f-ruk1m)-UK$4B-{Obnbge?cK|{LLGmW5=8xJ$hqf
zbLK|Xb)JgH626S<R9dxPE~x(Y`ucwT_<uJyJ$?Q0vB=vzyNU@9FRQoR-!FONMn_Qp
z>KhHa`11oBLzTtFnZXrS6MwjZnAxGzoHwf0?q9)r%>IdHT16(~<$vPL3#VFjC_VZ0
zb^81J_5VLMFFL#b{{H{A<?qh^`pPeF^P@>5HzL}S$D)hn%q{lmaW>DK4_;ngZ<hP3
z>hZDl^7bBQzOHDPI5F{kTv9OGCcQI>dTc#HtgH2w`0%AdQgYLF%Ll$(Kl&=zI!@HT
zkU3p{rA&i4=x1Y&#x<eYjM20I-+zC9|NFbu-m~6sY*hE(_wL3=mzX)R?(Nf$ur%(s
zX#CA_`6G|TnLqWvzrBy$y>D-o>&wPTK0Z9<?<cJk55GN`QS;r#Q%*MuOd-}KIYpi5
zoVNd>S5y6pT<#VT<;U?oZk)3k>kXFv`~QFW0)>L7r@W1r88~ZxfBXLIZ2$9fdk>05
zpV-QLP3fV1mT!wrN95)N-#87^i+ij0=iU3$TW`}W@!_jBv$i&~WO`tdtd9G%rp+78
zd?#FmTG&)Ax=r?{e%7XjaKns8^@*ZfX;O#Q#7sE+OMi{sum6vZs+*b$PO9za-BbJf
z^P8LH8)iL~cl)!@U9r#NU;B|(2Af^~7O!eIdUyZ-=Vxc*Po3ZM@Be&9mzFt44mK=y
za`fiPC|j7>4(dxY{QsZibna14&xymc3MSU`bf|u=pV-A)v8ZveJr7U4!7Q~Itz1qq
z&dxJr?k6|QxBnlz=f}Tez5Nms+gI;%xRPhFWx>nT1Ir|g?;6<D{P_L5+^d_HLqI(J
z!6T)3>uZyk>oXgVK^h)OKmW_`<6yqDr-481Pr8H84&z1dTpi@4j;>fTQ8MEucOwhS
z4N=jbC%@YYB_t(1c#=}};{$h}#bF(W3#+8hv@|_S@QF`Qn^K>D@6X>)PvxzxAJ4a+
zvW!12jx8~fjggV7*HQk!fm3a(U9x9E0<ozib)xtuhagVoOPBm~7RMPVKgcmzy&>!>
zPlltW?%B0#b(M6^{QUN|yy#Ca%fCNA&!0a1>(kTnJ(kkpcapd18gvUc_5A1I;mN=E
z=jY$w_HXvob~Q{BYhI-E^OodBUD3n$YmdHp2kD<DIek)@wu~*SnR)5fM#Cqqc4}?A
zbGs*8y(YG8)1<ToYytuaPah`!{P2)HtG6N2{m<cL5A0V>Hw<PA(>=j@-kz<ket+HH
z@W8~x#Df=8Rahe;0;X6fFx1~>T3wZ*3-wKoh7-5=4vteE8f(`65)Wmpif3Eg`jOpP
zMsOAv*B(YjttoAc%*@Qphv(b>-?Fvx|G&TOQj2#;as4-xY2Iq&Wc*N`k2OF~UcPyo
z8TZpzQ8l%;2?A`KoLxMQueS*p7j=PR36vU|Ow1COuU+V(oAX1)Tk^wydzG~bWu6>j
ziW-mp{8_Vc<E*f-pbw%MPZ;_5mdV@y<K*7V#kq0OF`1}6|6E0nhcwJwvSFLQq3@65
z2R_7yO$+qv>uhJQTF<|;F7d<1#0}fh7I=Ma5f!ks2Nn0A#JASs-@!>7NnWhST8#p#
zH_X`hk3V{rgSJxumrz((+oL39XXik#n$u^_NJwd&J$keykn53SglK}V<cq89>q-uv
z`4b;C>*V$6&6C!x*=XqO#&+(a&nCu+t{g^?fxe`~gdgnhCmmpOemGgD!0gb{|Njp!
zi}<kfpS{*kd1nF33;!#tPJR4nSY4g}?`!uQht$Nxkg#Xhu09P7e|{{?^g*J5f~x`Z
zwutlt9(pW2SFfgS+mv*3@9dW!lZ*ZwI(jr9^yq^pE^T2t3_Bya;<q&NeCUS6SCUhn
zn?ky^M<i2z*P(8)E1#HOAGj{xv+a3(i0KE$V2wvlQc8*@y?eX$(ggtpWsZOU|I3T1
z<)*72Idi5VOSb4^o2lN$NjsJpc#G?l3X5N_FE5{VBSJ{sRRRQDTqLBVCHL&v@ZMgT
zKS505h&Zg}eP*qJk-?vKQBDolqyPVF2yZAj_WwVRhxFWs5A6Ba(j1#&>I@7F{-h`U
znm=n*VuHeysii+YJ^g;9dR~9Os-B;of`e1c9A4$Dk5|+pyv1W&Io2^TU!G9U!}C8i
zMdI(Di1{`@x7YmCf8_ruI5bs5C&j}@<=$%-#x<$ThOQjU%+Q)4T>HbS7D2PGOp?66
z+owoPSoUh;70v&J1^*XslAO>a#I)e!!E@&ll9hKQw^#d~z3Dq!Zf~u3#i{6!DY|NI
z+cpGvhE0>rk5gZ^?7;k_4<WV@m)d7aDqCA?vmgIIRl8oec+&ipnuo8f<Y8ko$!Sqy
zUb9#s<H7%WSPK60&z3t=-FVU2##<W_j?6jls%1Rk@jO9imd!UD9xqy)fB#?1-d}mg
zdS-Kr-#c0yJ8zp=e%`vIm^nP`?8^Txb8)JQ|Iut-AQiLe|GwJa=Jo&Pdrtm$XIHDO
zmV~6_m8+6QMiNO$5-lr@CpNG?i1hBM-w0}7{r{iz=btUJrmRALi_rqLS^o>fn3<V#
z!j%s(&hT?eJ>bk6%(BzMQE-3#|Ig><p6`>dFaG>&`R(od_f&uHZftb)>N<SP?Z)Nc
zGw1v~0w+#pe7>+@hT(As4Yy7mdyZxNETGAR^7sGl?)iCtlInlGxRy9xO@#vs79Co!
z?%adNhabE-qu>?j-fM7RzSM^$pqV6w|NpndIC3#4a>yL)zF__R|MrCkZaBY5JLolC
z)W6K2V1~oomOUx$N50jZ`21VPso<pCG3~;0lRmh|_wSQg%x<j0v1I$RRR)t6KC{&*
zJI8K&N1gG4Wm4IvispB7SvZZSG(|16I-2!Aa$n)R!%x{jlQwIj9Qh6?aA+5c3;q9p
z=V52Wfeqb9X0$1X1v$lsi!L;*V0jaD#AE5BE5;U?%Wo?^OR-bBeQurBDt^(w-;bNS
zWX72!e)+U0VC|)+%sD(~cJdw;e?HMp<rh<A){?beu@^xF;s5_hjg_4DY!eewT9z}j
zT`6hE(+|3m<aZ`X*M36k|HWFr+f$_ebsRtQH$#bM$7AX2SqjTGD6U|u7d8ISkigFx
z3>nF7Qs-lfXl`Z{yF5YcxP8u9KDM4K45so^8>e=={X3qpc&YTC+@}8)pS1reG3s(j
z7KpITP)pd#k^RUa%pigxO<l|$Qn)m!@!eqJ<49-;m4DGH#JpvK!*&sA@$gdzMBKf3
zBZ4`vNgR@|-=V~x9i<o~c~FjPj$s5B58IyN%`<AFQlLW{i3vZtP0lqYsks=oiLATR
zpy}uHtAYEUbI@mgXRi99gYpxPX#Z<ZR94NsbdYI7U)4fW#|`T^q4mzs|MEXCusoT@
z&OGTs{SJj7!6v0{<<tkB@)a&0)LGfszGNt9E))p%7d(9Q@(GEah9z9ALZ=}GR8yFC
z#@o;>D^1<pwlwmd$@wThTV&mrMxB5B&RoV*1#~=4XsC4fXs3i|H;W6*w^PW0YTIrp
z@xYgNO|S0_w(KoVt&HxQ8j1})*>W!S>vU|reaInflfvR0g{>zXv!C)<IRrH}s<DEb
zi%E^KYg>6Pv>muu`cU*>%HoMTSPVrMTvg!Vv3SfgU1a6NUU$YQrG{em{7ntaTg=!G
zGBNW(tz2VwT!+6Ps>S(<{kCK5>y|Vu6AZU>6nL0l|G-N!qS~ZllHfAk8J%ey3`UnA
zqf|+bpXK)a?z&cUG^2e|NYn$R!or6+YeIR7S^`%uIyl+F;IdehmE4|&A5Ki51~tR~
z|4yDVH_rU(k6fgn`|QBvuoHgrA5soB2-?ng$!7e?U};BxDcho}JTsY^8Kccbi#24m
zrNO>x>Mu@U{cpI_A}#bn(8>B7r5zqIo-CVn9KBgScpNenaJqDY{a;Gs%T<aD5|S6C
zWq4LF$6LH&Ujd3gP;J}zQ~vi3^-s5qaxz)78y$l*ol=ApIVaBLkmv|7Q(-*l#3XEB
zWCRI^Bu6bR$%OCclHaaPaI;i<Y_I5;bcubI#kw^1s3k)4Uv`SS2+Joi)SFM0fVPvJ
z-ko|B)T+d1AILXbC+c0g!_!Be!U7F9&m|m4NIrY&$QHegS?bHS91u2YmYr~m$u$^M
zaQy%8q-j}@dGOfLS!x$<@T_>)ee!@xp{QKOrVPQ84d#vyy!3YPq&S#XW-$LaC2P^C
z(|6JhTJ$=-KjkF)|M;a3N3MA$9`KNyp|bDFhxhmYPuKr{XV=$Q_jc!q8Iqn{Yt0<R
z9A6tf$g+Ll#khk-lFeTgG!^mnHP4<sJXu*hIu`36D)T1HtvkGm2bu<wnq>AgZb^Qs
ze`)RVj$F+X(aMS9TK^AkyUQCXbNyW;%g<L=*MIt?zHVLla{v0judm0mebFt9aNW@L
z-}$au^UHbrkL*aFAR;l_%&?{1_{$fAJu2THDBHBMd~$O50}b6nVi5r?yS57GMkZWu
zN-owZd-wl;{m~;c*2Vlhci=<3Q`67y@9lR?7MYj3z2I0q&)W!LmUSuS8k1OAR;)7G
zQ+aqpv3m0iiNKq4c^Y;mi3meXJ+yVF$ho7l6gEr~tGL8orl<UP|L5oX|NZ?P@6_ZP
zGNrS-n=xAK_|)r#OP&5?GELvi?!74?A>qgW`HpUFj$U1jGiN?L!ucs&LHPcm1EG&0
zW+ojHZ2C~dWVu#BT#IEg>$<boG(PSveZ5><|6fF8p_Igig|j&Chjqrt2~1AfaI0JW
zD9@r4M#i2Vo;PoJ=FQ`Y-)HdVe*fQlZcz+pf*@m$Nl6a*)BiS4{dwcFWLj*q;I9+E
zlRvsX|M~eopZvdL$Bt#BA6L}x*PfZQR_)(o&w?9L@)5$97kG;mD5xAs$U1f4@L|xj
z)!ErguIi|i#urXi5`dWN^d(F}Qc_^LlA@c@1uL}zwsn1+dZ{mZ^6&rCjr;TQ%}x2T
zG8sX^7#^PbiC_LX2lXtQuF&-QZ~H}O$Fc(p*nE6={QP*LqIiz?8^5`Ec)@}L2hRF7
zm94qTBAE*+fI->zNqA4&(fSI@#wullb-QHzJDbk^`2YX?mdy9z>;LI!pPn@Tyt>Mf
zhQ&Mw;!PJ^Rm*rAyhioDfQO`jg5tpmGfo^hbjYAs?eJ3XCoZ~0)ssEGW<bKYsVvoS
z_M8o33{j=cckcggS7}xEUBKbg$1=<O{o^+`=g*iSaq05Tw{IdgUggLTOME-)(H2%^
zN#2<zjO;!>JooPLG&b^V&oeA3G1yb(TDQIR$3AT-h{=b7E+6jybg=os_AKGpia)JO
zdcME^?=$bu%4N?E?=Js;>GICSi!5f&EL2rnqslmO>6HuGS_X{#{LX=b&Aq&bpPl8g
zwB&hzU-)+0fmJ;{Um*2aQlq|Y`+6bQT$2Uc?E{W`e%xYtA({s?%lP&6{WCM?tNZ`^
z^yz85zCK@VZ5SgX8(Ukw$F6^g60J8s{b+CQURiO1xwMomGqdggf5~fW5)+b>SFBGD
z2vRag^D+<=m~RJhxs$Y=d**~g(^~$x3m?C><8Y6k$hUs^|LfxZ-o5eT<B5s$-{1eA
zo{+Gk;NkWC^>gg&|FyoAI(p?qgYq@bZEo%SKmN>ocz{u(`g0nanOR#C6I*TVqA!p9
zS&m$Rb+enyEhHrW8!ouFyV)e=-+acWe=KZdqkO*n|NVXc?d|pF&)4rSea-K2Dj^}E
z;{U(<6P4}b_xxy1Zt0z&#k_f?^_>HIDtYAX9B$6HH`a+e6A*Q3L%luo$MQb{K1w_m
zRpy8R(1T&K4HB*hzIZg@5sRntx1`7K@BbIq`!mn%?de6X_YK&K{pS6dJMrMTh83$e
zh)HC`JgH(oB-fV7#1ywb?ZyViiF`V`l6Ex-H+D}44UzQlv^6k%IlR8GH2&nf4bZTP
ze#0EzvMqzz+nPC9heg%u|KH#LKR(_+*Sh}9oS$ERe2hPHhG*HbHU9Jdw9eo7{jD^!
zwe{g^Yxf%&-vq@ytI5QA4<+Z_8$AU;bHZOV4zG{z5Da2wW)l~0Q+AiQwkGk!$wZ5?
z9z~-`2M;tjIxy5$%F7%GXL+QrIDv6f6~qxudfy!53c0L>ni(@^t1t)6Yz%17={xh|
z^Yj0!!|QL~-oGa9@7#$K6>a7mzA@wJnVI&-`xnd1-<EwnKR$li@&5lOP8l7##Ln#Y
z=tsY<;~)8Zn$3-k58p<&c~6(<lWi_iRgw7jH@ToR^}@x(2TxOH*vw44Yt=SgUs6d)
zV$U9lipq=O%|dUsIWSy1eDFaPBc#53H?>v&!GUe3UfZX95Sf;AL1e-59(MkJI+34h
ze|=eRU-xJG#zbG=ry_!hH6I?lKRJ2-lP5FeY@Mz!&6?%2J@0R7Qc_ydp*JjvYJK&}
zioOm9?(gTBZ*Oc<*CX)oQIdgi@`I<T51yuG<fIf7B+M{uPP}i|wzpdH&Rt0*Wl0;G
z3+{o1OZnHV?lTlMFff3GlGD4jf7(wfh8zB0$jjrhxB1!?hbub#a(`~_{r!APW_<qr
zf6BUNWyFsQN}gP{;J}}^xA)in{x@UB2A$}(OMSa)f6s3|{NH=}zm9I>#4nK>)cOQj
z_SL1{*v@}&Z+X9C=+uX&#b4ZXm^O`viHXO_iRboq<D6SZHq_fU@3n27E!{j-w0WIg
z2zSxkru{7kKrJOup>|LE_k6+Hg}>^TSa&Qu#B-rEsl(?FD2?j>n`isG`tdRO>hJ%m
zetlWL-8?!=t9`PtHaj~r^Wp#6;r}=|3}kG6%x-Rec$~jQw5F!Vz5n0WkBPs2ezv!^
zW_~xtP|1ti;s<CVLci_l>A+{nfB#7At8<;gD<i|x*2c4RDbKE5Jl@j`=2#poc))PD
zSDNSbHNzV>3@j`RX3S9Vo93jab>W-pgD4jcP=tb-Fb89Pvg<!g{9Lf3N|opTKf{{8
zzuuplYd_oU@10FgpD*>^-`g8GYgWag#Wxe#cm1vZ^+jG^|M0`Z^&%oO=Gpx{edGv(
zz4)Kc&;LJq)Z;(zPxaql_VV(~s~##FMjkl$_BK!ceIuJ{wx^#y7<~Agyfe|+vr|#K
z`{4Qh#IsUuywVbW^V2H6^?=qk6+f5g>XP{KwR3I81W|?c0-(VB|3AsGuc&9~;{O+S
z2(T^D`T6(v|L5obKR^3>U*6q*Vf8v`34`t1i;o}K(A{;&pEdQz*Vpq~SPtLYTYu%s
zjBjs$pFMl_!&mJ?noUhS_xJz*|NHy>`}=z~FcmLR=VoJTv#*!fSC_h@kU6mE&;Rou
zewmhpN<6zIpPAV<MT9LoyUn&*!f#&6j+&W?r^VO;1KIri*qE6`re?UZF0+t=v>%_u
zT5~l!2-Lo|QqhRMaU=Rv!o%L2J+tQ>a^~R9e#jVhFw618agl5X6-D+FSB$UP+_Q=M
zcx?VuS+RWIs(WV@AEde;v#^_$mU}z&b@epX(%QAFxQiEv?3wv&d52l@yug-qyIU7p
z*eOjEx~6pA@civf3ciIw2KCYB5+w|L6`K5ym+_@?W=~X+|JCXs(=6$=P$Pk7OWTwe
z)dtNkv_t<anE<M0|NnQ&D>hIt)eHzx`kvD-_jhaR>GMmy>usvP`3Fq6cyFGm#f-Cu
zn?wEj1k3bnnVFfFZ2a{9>+Apab$@Q}{A_M)e0X#E|B0`s?5~%ssF2uK^Yi~VT~Kj1
z-~NC7-(UW(WOpCt|MT<khi|%v-`?h_|7UREyzlhs=|BF<SD$KN_VZ&iHDz;kW#i>#
zV`gsq{$6s+R>^&}$q$~puTZ>k%R$=-l8vv#D)NDb>dmg3gnVG+`M~tA;fY|4Nb|vo
z4?H<InRwWEGBc~&96Vq7-Cc2Z#@SgnB{$wYugjL_R@(G5)Ms*@RAlI;bx&=imrhu=
zMW-@wgU)V$F1gDN9DRp_vTl8Qa;I!Ar*qEjY&j=`z7xef&7Der{^fnXpQyX#*si3=
z3*W^(HJ7vq2prlU7Guz+;wY1Rf1yazfy+#GXFq(32N!*3=Gt<#I`Fi(vser6X?{_;
z#&%uD+=`~PI+t=E^RJtILuZ}F`$Sd0nOb*Kmi>5?-xu8f?(!RpqU*W7o3CvRJSDX8
z+N9@+p{}WtqOH~OwZ$yvfv3)G{$i`n+4gYP1rx>74l37nO}a5-`iT$sLj)2c9a^>>
zo%m+P!5)*=Yx4?!d%LT2K6bBtAwO#YM;X{Xcjj*7YEY19@#o-dZ<{7}Y<kA4z}il;
z<qJXuTG$+&mTuD3nJ{bj$NPC!`yRLNzWQ8p`GK7#eSsYsPnTWE^Z8^~828mymhD#Q
zk_J)n6DryKYx^^AB_G*jBcIC2xb%bEYc2+_MnjXc-81X8SuENs-C~tfW(($WXXi5p
z2CaPXt8e|!k1U{JgC}$Mg0eu1y}#?hxDDGrSgv7-zuf*I)rGG+_|Hob6`5+y=1CL2
z+*FsCk;<MX@y%EEQk41mOWXRFS#F({ImsrxRCY>gL}=oys=_m-47>B$c04XPcEX~|
z@XOxY->sSMt8+wzPhwfC$+hNal*_*_6&(viZXerpp;W(ZN|MA0qeU;e*Hm2J?_+;X
zom;d5ywt)e@;sxY+Wx}F=XaIve|GNg!6Qc&tUM_rcP=aI)S5+ymaI9WqtH-O$r^Q@
zDRAMRzo$QZe7t{s{QuZ}fBqkAu77_oaid(&gpO|GJvAp6tUdeTpY0YuQDNn4;fHw)
z3=M1koB%BXeR-KDCx_?5vETdac}|_Wz*wrNSfeH|$0YH<><ojYkR%N%99jYdP8VNo
zpIXHrE$IJv=HAAs3AxG1${q44-}gqmShc%D?trGTkEM@?M3UTH(@B;8JTI5Wu2$Lj
z=!dAQn?#p_fsKF7%xC*Ww)y5)T)$X2BQ-}#jCIM8AG5W54sBa#x{TAyF*`)GMt<FH
zts|$UY*)20=5}vl>D+h!<<2PrNxqRgY72H*Xo#Q4>^{N+imfwq4MEG3UE^oD@N~5H
zGUgpP_Hiepw0S}j1E-;)hm+L>rKy)LZIt_;5P400-+rCwx7~402EK9G@gm#bvZhVz
zR1rL{ykxg}*+HJ?9M196UT<0x`t<RPZMOQZ*=>zB%QHQKclBu;l4xSk+$gqm-PfP}
zg(s4E&MB$}{bS{x>#TWf;=bkU;#rQ{B)?7W`+KRk+4`Sn${ywAvr3%bfs^%?xxQS7
z3<O&DOEn91i9}@|T)SMLs=nYgqfW~-r71!I8dD_1*E+u9&#O7j8F^;5;>p>KXMW6n
z;Cy!1**2A#Pb(kj_Md8hfA-Tmt%shUb}TB0Ew`4rBk(Vr!%$8lv+G)lWN2&99KSc;
z=bT>?=Fur|z{Go>u!#<DBfFmX8duFFOLSId@9I-X*tz7rVcW8WxjW+u`+mwS?MU7h
zU1ohPn@v*n$bIz$4#&tvrpJ{NI}CpahIX?Cwrq{~eX;QTnfLy@{m)}3yX*w7Ltis@
zV;7SnPn&&NdO=Ld*6UHyHxGTUXOK_u7If7SiD4^y>|<;key5;(g-p2267D4@<?BAF
ztGwz~3-UT<Ah%zytFAvPuFrm{dJ4m}WBdGbXSX%3JAJZZ(P06DHUWV|BS)5OOPL!v
zH{M39-x(93BjBi+W-Q0Q<=oSyBIg#GdMP@EKD0d-c>Rm*cInTSsh+o!bW7(heQ~)n
z@qmQOjyTKtmU18Gty-j^;k{*<#-`Q-87`MEnB;6<6L#c8$*H1~`%jn5Z+K_ra^wBS
zdB5k*&u30PvXf_GeAA!1U&XlcXHSQWHLRJno9mE+fNTFVM<=bDBAP90FW+GKZ&RO;
zD%#Q8t7tc&fm2iGh{^`BbwXj<g{NjFc2&l{Gx+wq`Pp9kV}%lb8)~=q{eE2fBPey{
zthURY+m3GY6<RHFS@Uqi`{JDns=kZ0x7;<Fzd24>HL`6=%=$U!ITqUXyZ@5YJXvhR
ze|nL|hgz9-o9mu;-rsy{VlY*8Hm}ozU3c=XY@NAMeWF>4(ZNkCv@VGxY*g^<NKP|e
z)bVtZ!IIbe=FF%QSi0l>M;ku&SF^5mB^sO-W8qlH{xV*H``M8?EyyC=DRXT><(k`k
zr=}L)gqv@3*F0j}<FGDk?ky{Ej%;129G4C;Gd7ms9Rl?p3g!{dr1zO`esf0em}2-X
z>-qhr*S4obFYt-nbico!-PbwN$W-(Gvxeo5Y&bN$Yvhk_oOWTGhv&3HyU8jZnTMB~
zZl8Vian;k3Me8)VmTZ|5*0t}r1RuLi|MSV$PpAFaAFX$+urMdtpzZNQ0~gIlGc+C*
zB}xc1DQsx_AO6-zeZlLZ{lTG2G@~Xz=WdM=SA1brwmUxJ=QDxm>KXfg{q|jC4$fI~
zW^Dv{$Yp+lfRCU^a`LR-e5?Ot{yTClPPgTV(~+Pl++u;}L<Ey3aX04dkJdT-?#&yW
zPdnWdYws%CPh|A*y1325Gt#K9YVwxE1tNzJ<hfsdP&q}_X~J_B#@W2rwVfs`v*3$u
z&rFJp(W{Wl^^Se~d(L?t=54-#3q)?~Mw*GoN4x!tK74D!B#UDvH{W+E`KkB0^Rdgc
z8yqR!d6%O}{qDu0#hFicRCheB^$%54^>n%)Eb^<2({TTVEk7!D8%;Zs-*n^onG2V$
zT7$i_X0|PNvjb0?{2z(DmkVV0WaqZ7yZh(#U%{iUf(yI5O5P+~<Xh@txr0w`?Uq}5
zXD28Ib13FdGB0$fpVFe~`J2`Eo=Mc@(jRuAscI7=ce%U{&(P`Kcl^7jkeT!;p@glA
z)=ixBS59;1eM`B>^`1fuiXOo&26rFr;IG*B{l8bMYGBu8&fCj!HMtgbC?srG?&Eac
z)bmzsX+f+_f1*L%t!<n~JC^qye%mH6=X{J_%=#ymzK0(sY&#abQRnbfp2L5vzxrPa
zJs<l^C-Tda!`~P|Me`9zf@`%;acy>ST;rsr&9C_6PE#-Yh6zWVjwpz2THDdu>p9W2
zIl*aF%!5$=XMfKaL_7WRYuY~P+iR1X3#PYT?$mMSxFx1&@$2}XiiF6X#n&GC9{%bT
z`Ka=Yu~*`(E2fSP^Rl;UEM=>?Ip5Ucr+4i3C{E3lS~c@k9+o{WQuOUimTCVyQ)AI<
z4(B$;(}zrEs^vt_{`Ol|afQ~ed5au06VCG}a2W8!eE)Fa-(kNOd3}GTR5Czzo6MPO
z2ufV8_jegKMQ|uJ&E4B;{^5$*yaP@w{R<Stx)eHgv@VJCoTy>=pW(1bzx|)ge~(YB
zmTL0n<I6bbRen$Vo%Q7wfga`a&ow0*5;x@DF#Fsy<%U^pf9l0;7JObW)vInN&AKJp
zvO}uEJN^FQ|4T$_`=>mybZmG%DRqgalYr|H!&J{n7fN^8m&Jaxc_np!xkiN8s;<NZ
znfs3at5oADjLo&&|Kic6JC7c-f&zQVTu^4<Y1^OWe^5G><)G`Gn)LZcTx;0n9p}o(
ze6g{$nxm_f#C>*-Zv7;E$w}%n&PzSM8zAR?Yr}k}^>+Dl&a#OHa4Ej?yCl*4+Q`>}
zFY5ivr01d9;<t_Znts^Nacp?LG53sdlyIAiU}DD<j*ZC^R3B~IJ0ttu*=v*5Eq*W9
zv`FFnO07#%ESBk<G&(j>#Zj{-F;ar<+SFx<^Hyfp{Fl0TNkr4|*>0mev3UDU*H6BM
zxO$c$SE~U}+y1q;mMT0-apiyTaBpj|`&=Chjiw9SKia-^uK2>MAG}a<)+fP9e|Dzx
zy)R~LeQlYvPis#49A}O-Pq+E~`SRRa@=Go&i^91jfB6NTv#WAODW?jQoSL+2*{T!8
zQ=C>F*f@V?Rcy)a8EVG_lOv`n{Muu-*WbENpw&?0sKEsj4#)e8O=I0}C+WH#+{Z3<
zeX7aPozp9(I(co<+4fR6@zL|($Kj9?6colS1_CbglNPs@bvZG8tK0Kmu|r;A+4hME
zQo$|SldSnZTJ+z}+U=WQbo0!FCvQ&eQ+}{qxm2dzJ0~t9OKQq5yG_%X%d@)waqd62
zX6d?@R;#*H6weDP)d=aa{61I6a<n!5@DtbQ`4QcQXN(`8Flc#hxb5w%pY~_{vKMhI
z^$4Dm9I3(eOHOm9+NnvUypQG?UY|Yp+7{iQNvY-C^FCQ$*!e{Cc?F~}nPbS+V8G)l
zzi1uTwWE$K>Mt4hc=PM@%{zFN`Gvw!MORl>>lvEKliKD#F<05M;85)UBR#P#$<tU>
z&gXxR&-(waY~JM5Gsa#Qx0&!}zG^$aYuPG>HL0<lLQM~DdK~OeJs7p+n8dEWed!W2
z)XMF??Q8t-Zo!uGhjzG3oc-+BL<>8H$A%(N!P7)q-wRg+E!DrPz5L8Qu*0^@-pkeE
zz|&%{<MQX`-LMT?H{ZDCcK=@3{6p;q(J_LKk?)UK?^qekY9yI9siJ@G=kEMSm#s}(
zE!sKG$;v<a?>tYKDYW%_g!imVrC)4I&u$A8G2xBeCaxWNC;EZbgwh~q1y2dKV-quz
z=0%3@Nss)N5X&DR=QMkrmY2%5lywGd$r;h7J}rLmBe#D~>coy+E*0RS>QJ6~nvmkM
zKR<p?cl7Lh_%0T-h=Y~2?e}-d3E?8af(|{R!pRRFB>ec)dhpgPNB{oD>C+uuS`J>A
z;OO1`@Hl_l{~KXXCRu&s=@jyq;Al2EcJHYVuSA=Bc@J+)zHqSgL=J~!LqgA{O-3(X
z8fs`7dU!uJVT{}I_+{IJ_3j%s{r{8r;6VbUjlE_zWS5(K9hYKQSlO*?g{dpQ+_`t?
zp4wK8ldLS=9T6)cQuH?HaOananeCkZ@UGXtAKo&LWkQeID6^FI8a5U$PInMdRL%4X
zyzt%Q<i6l(&(#;ShdIQ?KDUh8tP^qC;6iDS^6g{0jC|ECoHb8pNKILk8}Ix^nPH!K
zzy0<Q`;24@NJPznIAp%OK+cv;92%J}NA$!`nA)slKf$=CEhAYvCuy>mqtXkR#96hw
zjdQBzD~Rt^>fe-de`WTPl*k)qYyKYFr1G&a{JJ4eiezs0H4)Zx7fs|8=KS898WC#8
zChb1uki@mA`_gB1DD^3qtgj51r_2(ySVZ90b4`I)&VI6mXKaeWy`MF+?71BRM2^<G
z&TE}9_24(PpPRMk-~9eTFd{zfyb2qe*UppqdoQoO{I={|;nV41EBY<eJT<(JO{}zQ
zZay!Q7Bx%AHAmXF`I<)SH4)a+P0#PHIUMhFI$WmRASL!>@ur@A$D>wttvejGtTWMJ
zWpui`pTL9PTXain_4M3-OtAA?|H+6g`35+ZY=Op2yx<1Iq{r(|2nOBcXi}=k%YVQc
zH0#QwYv+{0nliYabDTc3AwTWt+3oX+=Dv46U;JuTZTh^8(>AGiE}AJcAyu4-(_~ks
zg!>dHjxB1wmVMLh=kFI~GSie~TjQF2Yg+2T>Yl_{O|CiJ2G=f_t<qZ7xo`Pvy%oQf
z=zq`Bk>l0glv{D06I9~wnQI8jNv`vGEHYBn;~a91o;t-`;h3M4`8?#CnVf@wH@nCj
zT{p3<5o^t^<_k~ybAZ2d@z>U{eX|t5v*(L{IJ{<#S)YdM{mUji%%=~T*v|jBZPk}u
zLdPYXB_^f*uq*TlOyHU4E+T99XSwGd^R<FamCIfi$V^$3E7-K|aB072;BCKi#k+Vv
z?GtA4@SHT^`4Y|4&MD_xiu_uOzwT!@&*8c478@K|15Qk5W<%Y2ZCaQ41__aFO`Ems
z8#js`5st{rRGg;F_ee}yL^9&qIgyo2nR1GM{&2`xMC|za|LozYz{Rhl-kA9|8=l#0
z6EuHcmVoz+tu2bb{bWvCSI(Q9`oqp^qKbs_qa(8)%N&Sr-0}KF^|h(nQg6>tD|F^#
z4m`3+fJLcPqxJbh)6A}GB1cPq{lD1#yr$s!0bOgpV-r^`()kH?;FdWM2i{*K=C`<M
zLAk&|%MIT@2;Dl_(8@gTfRjk7+jhxqh1}hDL;f>(a@gxv*8Z7UJ$c9L=SQ=2k8OCb
zanxdwNN%?wACpj1Wtqh3U6WGtw#J4&v2;>c`g?!mv<kW7AFJ)&-`|+qq8PRN^pSOk
zze}<$>r`>fzBDCOkz)>Hl!%L9kkIk|v{&t(1>*6h1@mIWuU-0n#nkt(G$^vS%(3Na
zFc4|pzw=*=gT~&5+2&uW&NW@w*er7J+}j1t++4D2I`YrROcFS>`-I~&HOq6SkBI-D
zd-UYA;xopD5xdrH$<0z;qQdfO);@QRS#60PoA1q6I(m1Z_lN%Hwu%9vI^0ePK0*@N
zfnB+aUk5Gg^UdhAnik>KXFutg#f|c1orNyhe&BFw3e!Fz8}&zpxigVH(&_t8<HNVN
zUucbff6ws4C!;+zCl4Gtq@a5A!#mmL2?EVMJS8`zjf|P(FE3F#kW=%}OJ;{i0AJmo
zhI8&t(<JKtaJ>EMQ1!s&+O3G+>t+O=b6CN4@XC+x`iD<X=P^z*SQCBx!!usU3LYm-
zO9_svYAGUe|0k;5P<WgtZ*Nrd|D1x>sShuO55K?9lazE}qkzY|g;n0d+08Q3MI)B7
zg`O5)n~<Qe*X*Dct7H8?gFO`o17gn>JYqTA&d&oWY@J@7O6r^P-~H{Qf9k0YJWLO?
z6nAq;NGuS3%EQwU#k_OQf+U{>s@!KnoC3t}T<2+4=uG&^S7h0^hHdSK16h36CKY{7
zYg6}^*jJza<Ntin05oWT_j2=0jty!YtDfh-Vp{XC)tk|=n9W(p@v~r&VT1RC87Yl1
zD+~+_?(95V@K);Z=jRtB9U-nsN<0u2Vvx{XcP6MMQQ(O>8(WF`LDq{`92n}@7r#98
zMRSg?|NF(RTFcB0MJLSC`S(Bl$KTnY4W85W1-UOl%y61t(Y$%%HRTygPaF_a%m{4N
z@lcuW@p<DW$rrPo3q*M$qUC-ZHSC`A|G(tDy~#VudLMqvJ+iP4qB-f1=$ait+Ut&Z
zr#DP{wUH~SsMAC&`@_PGhO!PzM2|3U*>Su?ed4AWk7fk%@bIwn8N}>3@Zp_o(jv&V
z*+X$_6LbwGyp_1|iTQFe%Vdi=-I0ufTUuvKVvcOla%_y$GB8?HV0hJFft&rT%!toI
zGBLZ46}%QZyg6N?Jr(MNgFcKqF0r#LSSrM}Ys>Ww#~8y*f9&k3cz)o*k_nUTVpuNA
zm>3&geenK%BAfNH>JKh9^QJVeZgEZq4R}B0K74jIXe(OHpN=F*EtaHsW7~ny90P*|
z%e73*LK(vD3xzaydk4s@mf8~1B-t8a%#(J-=z&Mv46TXWoF<byHH{5ci^eGZ|8M-~
z_wf%e1P_0G%@e+Uk^X$pl7j#Lo%Ag?9MfAD#di5{1xu%oNrhLP%~3YyZ3RbOvHl5D
zW?rMp9+v9TB@qAVp%$~j&WOtbdYUP$1*!4-jbe5k*^thEcy;&-El5gn+G4q38IR3P
zCjYuaJghO9tj3xXLdp}k9_hPw<QEtuEa6PqZ}8x&_&TSi1$&P?VOKiMzvks*d$*%s
z{@a7rr?c}L?WyTZb%ahV9DFRlM$YXs^EEEk@QQ~YbvG4XRFcS+7T`^gYC5K>7C9lX
z=s}TOn()*aLDLq*|4V+Yq}csfEB{||$w7st^7lqDI}Q|lR{Qnypa3K)oL;Vd;3W{j
z&9h~LLvr#jne5r~6L=3b2);YA;>cpTehazO9gl?0mh-f{(qH3x;{X2}r4Re_*{*aP
zH<`ewoN8dCbmRYj<C;Gw3Ldi^-dW6ZeqQ4JPSB3W|NotMnhX;}BxWd8%z8cJsur^w
zo6~0%DQ08tq-U{bA`T{7L{$hfe-M|8@a9|7cQDxCK@oqNjB$cD`;3Jx+G`WkWS>m=
z_?Z3h-fEuo^PpgDs1(xxP4I(ff1V24?3G~aIpDN9%uso%M1$$kgj0)HS?X9%^esH%
zS&^mM#2XZ|X$P0YAG@PH)h4MCsT05cYxg}n-|FawV)aL1;yw`XA2Q-N*0A`1!m_vn
zPZQ7BaW9LDVaSd;xQb!5)T3bLWnO>I^8NYAa#rRH3uC`WlW({Qf6`gk8JSF*|1@mg
zU!QJKIVtflTbsK7hqqRcGVIX4Uw`b~OdWl^xi07b*j;d<=?AxTlUCtU&MisI(|?}W
zG-IzZn@p-=T0??p%k?i2(V`KPxo3&an3g0aQ88uV(w_bGl6>-MH@0*>{Fck~+Yp+F
zW^Ot@-%di}!eqgJe?Xzbnjv>MbhmNB5xuM3yj!01sdcPga^#lR(f37bnkr;?_<fRQ
z$?~kwWY+urUH|aLWS-q+hB5nmD(nTp<$KaW(ZvzcGb%puuJhws78vnaEbOo$YoVyY
zjvXqd76#nXXIh%BDy6J{=%vRa@v&>U+M)QmKk8TB99Z@2e|=m1f6(6k8{7LIUeR1s
zCIoR|(!mn*G>rr?iHb=~>kcTX9Tz@RQ*mQP%o?Q~4D0^%ZvAi|bb>(w%Zd6&S)omO
z$7eAWbUyZyyuQxx&CSCFKMW3ke(o}NAuNsaaBtah{FcA->69HxqL(N0?C?(W`CP~<
z`L>JKr*{Vz{~AlVbv*uD#~w5MS=er0`@mCg!j!`Ia)-aY<@x>H_|AU+H$RnN@w=Ug
zea2d?KYzK`^z!UD8DZOLDk~tliDl}<z?_7qkJg{=cv=5&RZ>Byd<3`jmI7yCM{eFH
zQy5?FSQi1Fiagw|^5!$NRCs?%DZ2m7&;L7SEAKePA9GYQtsvoQANK=);RtSqq#(|9
zER9!vY>vt;GpfjE^|dWpcff=_e3xTzgSX6~wl~&)zV#-)3TX>oE+J=|^y5=&i*PW+
zkxnzC4?Z<qb2mKZQN<~Ki|HqX=FL!X-D5O~;ol+V%kNKY+94(LV{VuI!sJaGr)ivc
z<`8u3@o}EZ%Z)*M`Cf@0zP^5mY&$d>5)XK=S8(yLvFYTn<w!DVA6Vngu`TL@il$9W
z%uhM~15x4+CaS(LSYye4nd_15(T0f%20KqY0u@|44s5701?|j!f6wsGFV~_xh>3@G
z&STE5Y>141;HA35X<EWE_s?d_ypp<jm9ka;Fw{Rh*zmC2<me7A{vC=ZuJSJ1`y)E}
zf1!!>gD8dNGH>pkE%;$@@;lQ3s98qZ-7}YWo;m9>P5uOfzo>J{)#jHfO1=u0tZaD2
z*Q{g-e=at?cSqN>{}Q=AKF9w5e-Xp+_)jG$$ok~dEox`YnH=~45@yT{nzKFPPMUgU
Og7}`UelF{r5}E*j-wS^L

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..df3cf2004312ed0ed0ebf1f0340cbfec7fd9ac46
GIT binary patch
literal 19251
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3`sL~37*a7ODM5mDF#|gT
z0|UeV|5B`r8Po?o^It<}XJcpP0$D>7MzFJw^gD932JkS?zW(ZXYsYu{r3xDR%zkp)
zHa(D0n0)X$`@NLNwnCl-o^G4Eu7`ep^ZM89WsC*i#Tq-<7!1l&qo&>wO<Zzr@+3y1
z<oD-2maUo*+0eIm+uCn)w_m&)#hfsy>i@QXwg132UAc6Ux5Yq!<$z@JHJND@5}kiO
zZ`NVq*Yp2w*U==gYTtC_mS<PHE}s8v;ahwCuT#Nch0~Y5?ai0Eziq#SvsbQ{o?Fl2
zQrSK2lJnWqm;@9&jjwTq7WS#EKkf448G{0o;EJHBt1SHASzi}jx<KSv&h56xE0);$
zpOxIYN|N!OdH>_ju~2tR+Q`-HAmCD-Z(5{uf>ZJ3yZ`@BI%yv~<g~?Nv9YF*lgCuO
zlrGJ<pB4WF-6NkVRu~ue8HSb0uDh;v?DEq`d%{DX*Vft1pXKJ95O~0n;f~n^k%d}k
zGTcHt0$P+RY=1J||2cVr%0!{yLvO!n@T@vvlqSi@$GnR#xBARyKc%I~G8eyZ(z#*g
zCwwjRj7#XZzt_$7-;)KUpfi(fxj_yK7cV-*>9+C1;*FpG|CfyB-nvOxXV1~d+1&!3
zIg{5uGYx*Tr+!MaWq_)~X`^+kOit$n+Ptzl_jyU~d|zQH<#Q7a7j6}CJ=Jr1%l&V4
z?(5HWuCJN+Y_tEW^lvBLSkFDYDPfjDAA`c8!wu`4?<e0bndjNU5EXvwd;I@j-}t~0
zb>`Y$zGed+w~P92T!LC2IxL->oxR`XE5&ErXckC0kj8N9OZtmzRo@>~v;7zE)43e=
z+GOdjZN9<V-mcYnWaGc;RK)6@iMh!R9dawDRIRGc|K@qP`tgLL+bx{LuK)VjG)JS^
zl`SRmhULe3i#wjr-0k-(?waV~hQgNO8EO^$N(TRb{L|me3=WW~hFlE>Jg)T@0yE9U
z9ba6?{_-z*VMY6+3$6u5g)5V~&&|?wHrjjqfBnS65^J|^+a`Vdprx^|lZ)5Z+I}O6
ztPmH;V_R)beq83E=q{;}zx43b`lpP>>GAn%r^LHd++Ug*(UN?`COL9~&>nY=i0~yW
zf;qRZ8=SKQhl13JBL|Ni`}6(1y{#>0X{n{KaH6KBA~SQKq-5Lu{r@*^_>-QLl#uX)
z`~2CSp8o;HRz*CHlQ_QJVl-Cb+-ehW#?e^J`0v6_hgA|Dlf~xQJ?Lb1IuvvI!+ZI|
z%l&!Q#~J?lb+q88(cyP@dF<<rV)mZ;@LB!v?d{AG#sSbmWX6$$$B!R7e}3&k#hk2F
ziFtZTYRA?tIB@pxVFqu;55JhQB^$O%HcoOk{Icjtg-N6guZYf3&TCfOYlRw@GkXZR
znEm-*U-7G9_Q?<b?U`Ht{G8l4ckaVyzHQ9x68H8bTGY*X_|&`Y`FY8If2B7p;{^M8
zQTd~U#KcS2rd_%sVs0K@STI3c;K%vXr;i<Mw9QM1{3E~T(Sd0$|D1}niq0GiP0BVu
zFk5rRQ3hr=VHUxOC+3RBsIt%D)n7DWGT)MyU8|-yGHTX39QgmAM?cQs&fe1>UI-qZ
zXUlVay>ZQt10P-rGyk<d2J!2n_(ut8X;&^!zH&W$)7Gixh7-eslKy{xuWxM3T(riy
zD@MkZ_1ltW-E#%k>{fI#R9|wu+!`2_aLxJzCsSaBf~Ip=XV^{$W_IJ+KmX5vcqx2%
zdp^%}ePbDm0|j5T4mY#&yuWAo=a1)hiDamK?vEZkOZ)qOzM;~gJ6A(}V`D=+JEu<-
zjWbNv{J(K4=fA{mr=(TVZ%%T@RX*&fI5UYMJhye8MS!ncAdkP;iq1)mjHZzalmGuW
z{`2?rhu7kVR|fN>pEH<af4<-u&*64{o}QjBFK$8Ys(dtIqTq&2lRkghVqz*PD*X7-
zv$T{1hbv4gUNYM*Gn87Gak2SMk3*+KMQuxt&b&hIEk_z}1vrHF<XA-6HYwj+IHh1k
z=VN(hw^I(u2ZT@k{?4;K-*}GY;SI^`haVs3neb6)7N}}s`2T-V;v)f(l<@HBpFdyz
z_}yGoIQiL=l%^i%#i5KFZZYZxUHoKwO^eO{WaFLrF5Mn6t-@)i9M**euw1-llEKSu
z(*E~9zvt8Xhfmpc79Q{x$>=zEnNeAJno>(+<HP&*ZT#{Qa<)k~c1?Zwqp;1oT*9t?
zS)vf65?K`b=)who!Xl%R^6T3+PJI3#AtAxRPwa(ebK^t%HP^dN%sQdiH!-7QW$Feu
z%OcK})d2^i?j?xjd|2Tse%wsp6Mr0+lBxH@muylQ6+-MW2mk-)ssCs2=ik|aM=XcG
zz2#Z%Z){U>;KNsKUnN0ELahTOm`&TldO8a?YzPPndX$>f)ULy9W3GH%?Sv_l^5b~6
zDH`tn!XA&=BaTc2`D2OSv01Y(u1PpBc|!s38b#jl1xs@7bTaRC<eJ{ZE*ll(Al9fN
z{v_(zefzfK{gQn0X*c%IfA}J>jbDDneJ+SM*UU8JJ7mD)dVhOD;>xJbox#(J`tB`j
zf4#7(i;uIp)3aM*!ij{VeSd{-$A6k|y8hv=cV_4NQ)dS6`u<n+16P<C$EH8~=kTye
zvV`r<o@%J&y3UK~YJqy>ij9g(R2(*0U5sx#rhe*wXMvM7>nTR&_#zYQWuM!$mL7RK
zFJ$V&HP3A49cYwayVuq<Sw@m=S?839@NK=d|M#Zfzb*_aEf+<dJb37kimvXP=jn6i
zzE#mYdG-_+Cm&md-_aFe{}NvxTr}mzjZ^$jcClSN<(%87q@8>qs^URLvBGri7gGhd
zDTe4Q6+Gs_)-`)#+M5Zfik%WtlXm>tenRO)kA1zwy}ijdHn%4}^=exkE@@YPY~HpL
zpyUlobbgN>JW08BaUwV8#-2ViagmhtloqvOHa51LlT3S+4jgpI&PrG&aVCvL$tsZj
z%Em3y6J#2~#2>6t_;5x4$7_)*EL~>~%(U3BjMF6b1n<@R`z8BilOwivJv_qM_Wb-x
z;}xKm*#G~FoVFf1d^jy}p^#YG)5nX|6%L%^@D?|a3aU6XnS<Hr<cS~e|JS=!tn3ci
za9EIe&9a1)hHd{O6oL%8Cv>N}Es=U|e@AgeCBx3PB$rDk8Uha~himw3&~U%Tu`=?3
z=#hFEl_LT!vRsJ{t0dN#avL?6TTJ*Q-NZdHDJdx-IeEk8WC5Wh0rB)3Te}|~WY+qU
z1oHg<|4U{b<ZV#kajlo;uQE{NNqp2O*z@jS&zU0{Ujq&{RZU`6yl}xuzDDTZAw%E8
z`_>!g9p7kS{wnM3R>lp2)3wg{EP30fsFXBO>sVIV<{%@*xAW|Ne+=VZH+lJs37kSl
z*EF+T^R|vk->p<pk@I6kgmCGfm#@8Y-tF&lU2>R}!Px15&Ee`LEpz_YeVE4pP5~N)
z8#Wt9M{268Sw%%_fBvkzoG0VN;aSQ56ICsRbrwvN&il~euI(i7lT+!_e|wvhi(C>O
z_8fAdKa@D*?j4X}D`#f*;oYOko$sAA(bahV#E<rEdna7eP+r5>t#Kg1kwwzp{i9CW
zl*5cMT>e6dDJcN~2^~Et4;~~$Z0~#cDYa|X3P`vt(kRTxKAn<Qb@PVC$B%^uMn(&d
z=^a^nVOMh_2yli~90+D+&N|@O)wSQFuxCYpL_#8${xm~juAoGQje?u|8q`JDX6*ex
zG38;TOx%_Qk&QV_ChsnCsYgt4KKuW_eD-~=-h&NqwAl7|vIr-H86G;sQ&`B8lLIO>
zV)l5fv77;Mhmw7vhIaS#>Fm}P8pVYh4<72^j4L{Dhf81g;Z?3BTn!8!=Av>pKg4hd
zwrp7`@buLG|2`YqRAdfxGfq)(^D5@?n0Tr|N}DGmlhxGM!08m9O6rSOigv9l6I%)e
zpPsFM;Z)|PQrOZT;H@?%myazclQr9!g>S|Ti5oX17cG{&aYJI6Z}N^x;r^RHpzbjS
zxo7J1-<1_BHg0m_;!$&RQ(Mx*C!f>McUplr*+9WmUqP|^j-In#vgE``Qp!PG4U#82
z#9nI@c{H7PYR^3BVL|GLZ(N5MGS)VA_^rw87S!D7(9N@?O?Q@*&teW0ZVyJ6tY{X#
zhzJQLX32<1Nj0B@9i`nH6k!3PUbtdK$CBmNtt}FpHW?kdCHa9P&8f?b?ZykSTM~lT
zp0O4QH8MQmdTPH$u`}l4f+LbeO%1Eu`1Q{i$?U!1pr>dcrTSte`^ubzhRF6gc@yUL
zX4EWLmDr;0Sg~0o+b7w~!hoCKr&mUXXWl%XGiP}I{^couH(}2QS#bJPvMfAt>XWNW
z#PK5$MFj_L&->t!nv~#>S7~q{OV~wOWb1?m^LH0&TIc8_eCp_84rW=a^q*h4G1VcI
z;d)=#Y=Z@>jx6Q5$--9gMLC-Hu~L@ml{W5K!59CWX83M&OMyM*4u{dI#;wk$8>WO^
zV46I8_Q9)DA0FjyJK8NNXQ#5Zo*Uwb+{Fs~{Hr%_t_zr8Xm8KGN3ccasRxgmM^xh!
zRf$w)uZNAx=1vq06I&Q=&QtJJqLWj2fvD4rR}()QsvR=n-5(esaAdB)dxt|@o-C(Q
zlbn8tN*=tD{P5AlBp0c=mg33^2_Yc~85xO|){U*gdC+)Y_#`1I>B^-^{9Hz1Ay3?S
znL2_6o-zha{%o&P|G)if*o3VH3nZIVe#mFrIh;K{BT_)`oI^}tBPWN5=!!=!QXKkc
zBqSvyCM*kASjzqFm|9)KR*r*9KmM<0<Kt_4lhf8OFIo03<;Ry!j&NfG0|Ntytvxdw
zxfm2VT=(ZSg>B~9)3&|yU45_E!w9!4dOX^1e+$H)bYz*hJ}h+Vl!{+>H%4gvRzLnC
zC~<E`veKc1Z59$4l84XhDY7TV6mq88zIExItoH48x$e7FMco_CME=f|e<p2Nc<bwT
z!5M*EQO#OKbsj<t3&72GC9}c}n~bU}Gk5OF`uuUDk%56hg8aqgBO3MstjudlT)o9I
zQrQ+wY*b!WI92QfkBy~t_$x)V(1%4l|4Q1HKX6pxmXMA=BHSq;cGzXc3<(KI$%qJv
zEn6h^?D1F;1M!-YS>cJ3oh&R7Cr%U?8X6u5VvFj3VE^oZK~bFJRHtv8%0&#0Jxz)+
zH>D&KbiE}yv_BV0cC^jdV96Yw*nHJC!drHWPm_kFM2Cjl1kXRYf;+tI?AX-R+2Z5b
z?CcgXf!b65|0`(~c61e~s@{x<@;V^aZG515#s&)q$u~{sgN%$8i5v9v@MJWxybcd(
z*dVw_x?Re|!AC6pfPmr*CC*hx*2EiZU0~h3#EEI=9n~3j5|WZjmPp*WE9v7aIb&vF
zH6I()BU*(wZmeKsf4zB=(sBk1huIU`mrML;v1XnmQY2t-XqBQ*!}5Y-s~E!5j>bO;
zV4V<Hp`h*Ht<<qetbS#~@>h*g?as=^#?2E2n_F3%CkQ0&hqP3b=2%Rb>iXwzZ%Rsp
z!bU!S`4gE;$2UE!KfyIYTXsg^|BX+J)-bF#@;UgwBDIrc$0cn&wmQ8cD}`k{oF*NZ
z_?wd}DfO^tSy|f-3%2ZRwz9G{*OZ{{onWzJN5q@Arfb%!CNljG6bw69-_ui{m>uzN
z(YZDSb^VxS51Ck$=e@b&pr{aGktkL)<IxV&!;eHStvhmg$$>RV8&)NzqzH(leE6O|
zVS++}DI`x$uvoIBMMg%XwdKZzQ*29iF-QK9Uvt1gVT-Pc%M`JSN!)sZNnAXiCh>W0
z6-vuxN>AQ4!NXu?0(WyGqw3uSOBLDJN{;gI@Z{z3<mItwi9_6aXu5@kMMO@nC=ajd
zjhm8*s}E0%7CkXL{GllOr&oLTd#rq7udMv5je(7gt>+N)(%z=UQ=F0?u!rdQNf<7Y
z4;O6?ZDBmA{A5eZhwUbXQ&o2eiD_#)7Z^0=>4F`7XuidbGiRqRS!x+DTYSROi5a|n
zUznJK(iBRg|2_0$<hgN)otgRYe|h`5`d?o@AMdyS|8L<`UY6`djjNIr7_ys1xc>c5
z|MU0v|LOYw_t&Q@9G<&m8UGGFww@<T*PMCq`m};V!(~0FPt0rb^7L}EwND>A)(~Rg
z@L05{LF!J{5>1m6hwPqM20gjbu<e?-x6J=vU%#K8E<f8W*1GJ?#p~<sD=Tg+Sz3AJ
zDyyk!>yxLIKYv67x(9#Q66KL~;lSLk`MoP8e8qwdU0Oc=|Nq~<?oa*SU-z5Y>+|ma
z`}^VHeSZ0Wh6aDagOa{{PM$JZ`JsnT#hS-a(rkgv=}~-#o<2)6FipL&W8%Zh>Tjy!
zpbnjBQBiT@&D*c3DG`YiPjRRxh8<FttGs!qeu<*;3aKS8MYl}7^Ww$R)92gy?`P-T
ztN;IR_x=6<|Ni;O{)jg*(N9^q(BEH`pTAO6)b7N|>=nFxNw+rt5K?V33Szuwa^%1P
zhMNsv{y#lk-_G~<&YqvgCo1osXY-T2lc%hVEibQao!+q?o@EOZ+*DPCj%~X*iS?F^
z$TJn)XEF9EXRchF`0!_OTmJnG5s*A`Xr{%NFH4%5p3I#2KlzI0iOeR!m46!a+0q&&
zf2h;%ak(AWAQXCje*HYt*GIm*RFB{Hr%7x_{=J&}TVEf1dHFoQOvQ}3Yx6TT4<0?r
zBDwnLMWux@0+VNOs3{+-|Nrm*zq18DKfRCKyf5eGr^i2wr^x*Jtp3EK^?z?~z5l$F
zGw0H_Y)*dgQ1Ok0+nv*P5)v1VPUPm<QPv^F$;nn)%BHCJrjr?x%I1MG91qWv))t8k
ztO7;3pZK@g$m=yl)N@ClY6#oBKvi(X#`t}I=2(Bf(j)molUZ_4{r}?QeeC>lH98sw
zU%pKf7fd{SS6gja$bqA3LIz4x#WKD*_5S)l|KaKB|9^k~Kf~nZ<(0wv_thNCJi5Q0
z=l1sh|G&Nc|NlRao?eT)g2aSN?AJo=GqfZPR>|!U<K}Mjoh>oX=3o*l%ymbO{?ySu
z8WE{@@qELUvNpl3azFW3yGu-v_`t-x@7f$e<!2YZbL8Hws=c|1TR&dthTxyi&+lhn
zzjyVz{Qu9-``g>s^zamE_DDKs@R&@ExLI(ZBTGsyHX};l!P(jSkN5x2$op6E_SXCN
z_c>Me{r@l7CzmQ9kRTwOY+!gWNGpNq%$I`=%9;iXrpXk%<>BF3wTfrgE}r!BNt!m$
z6nTjyDJ^Zvbn&2|N6Cpl`)_6cJ>O7O!T8Fs|3aYt4UQZ2SJvrBIeBWFpI;xh``f3x
zyYv75R~KmF;o;d||G)av6YuorJ8i4w_Sf@UK4;TiwLwg=LW}i`N^9hwgw*6^mMlMC
zUfzFsdHvPv^$KdK2lgD>P;Gv=PnPHXec{D(8X4W~TN#fxNQ9aC`0&)#@hB-p@PNjP
z|NlQU$s!_h=GLvPOPB7HxbS_m@-c%6?7KH?3Uohg5X^b!%>V!Y|KH#L|J=>Z{`G&A
z%DC_EujeU$R}-`G(YJSZ`vcp3E_kp#QkqxMkjZrJOX0US=TlGbi`n<5`@|GR#`yh4
zId@JJJY+igJN9Ewzu|&qiYBH!JUmubJexK}oMC}P-6V^D|I;sAnIa&P(lS*tqw=qQ
zQBfm*gOSl8;e+u@9u}`jWs7-Q&(>By-@bl-+1sa!-Q_*FdDz<aR?pA7@!`R_x&Hg>
z{<;<^hBc^1Ca5W|;h4TLLxsh_uIkJ02M6E3zE-Q6eE!u`ZJz0RhBXIoemTUqfv+<u
zb#Z8fz`Et?(=LJr))~OzFlFk`-X513Gaf`wo~S*geuebyhoPKl4;!c16^P3HkKg;N
z=I^i7_VpV|w|s1}um2an=SSMhOSk3i{<J(bR$43;xcFgc^Gdl$naw6i2?-5)%=fmx
zKYnqsy@!_Zg9p$3+a{|^#_d&d<i6&`GB4(V_mqjVl}{+?=<rBMMO0mcCXz`OCZ<nA
zLn0<diC*Yvk#}k`Yix>|s^Gs<adExJllqin@9)>=-K}yC`f%{z-s<_`Yj>5uzqfq8
z{r?2*4~DZvcqBIQ%n<BRY&|W&7V@s1t?hsP|9`(fJ$-+1vAy41rT=&KpPufiXDRVv
zoACy%W@a@tHak1E$jC3QauEBS{AGN6dVc+SBqQ_RaI=Mtpn|+M!(*paLVuRBrq%sD
zec;3VsZ(DbWL_klu;Rh-{{7|e>#T~NJU!eVUtq#<;etI+Bpc7v{|09chBQY`b8y{P
z^RxWbmG9Nx>uP_w_-$ll{Qlnf&kx6_&CIVSSgMH|FLZDb5dpQRA_A<}!(GtT)%5Gv
zn+czv8_t@Rq4eJ{n4!;#b*GM_w_g4I{r|r`<^H+5w*KFr@>f@0OPkvz95G3gN%l4T
zZxF=5+@^P?h$+&V>&Ks;_dh<ikK4E8MNLoseTjMY>XG8lY`E3L*ZMVnFZ%gY^2g7M
zypZ(g<S+B*&z+hY9f|PC4?2ssOlgdqnotyY>G*Vgdk;Rgkb;+&b{9Xt_b4O%fbxtb
zM$L@QV&R2NX+Qq+%m0(J{`UE*wimzu&%d)*EaVGfTsNtC;zUP({@MaDaN>6IlsR*z
zCn@PoQ<DsjhP#fElF-)`vQNrh#9t8>`M6v~Waa+)|6Cje0goP<=hxla_xI)N>-Q_P
z1=*CcB-vCx)^7-UXy10Fo~N+oVf_Ao)n8tG-=6>Ex~71`w~L$^y}oQRqFibQhKT|q
zE`pHC#K~L6M@!3Q(V{aOcDQqEx?DMAwVK0h62~(Ii8Loi5yhAook^4Kq_nXxGcz+E
zetv%c`q<w;zPvQ=(K*+kEBoj0ghR8u6d2YX_|YH!$DWz_@N)nE^X&fq{;a;Vvv~&V
z0T#xn)+U{fn@OUguJQMvR(i_pulp-1@Zj;oMlt>g)6|%~f<<iCCP=?I^k!kg+JFTf
zF>|~F0~;nO{P_I*{@dI0C)`@X<B=(~c}s$WuM&@epv_aRFlUK?s}9{W{pbIy`~R>1
zkn&C!SIHwsPhRX;UU49&Mj<;fCFRGrUPpiarM{4H7$;Aem>3-|FD8$JT``@1k{u#t
zxcv3%4JUET^I}QOm1t2^JGOASx|g4a!k)7y4j=yX=coCd)Wi0B0{h~{vMpP-*8Mzw
z;KTpp{r~su@ezp&)6zbzpz2zdAt3S9NzDD-V*^9O6|0Rud{VjsO_(2BriiScC&!*P
z>p<AfKhCdq|2ryhFe)Trn(BWEiGLqIZL+XTl-{E%_2=*J`O~L={rXz|h$sWkil<!H
zrZmQ^dtfeXYxwuqSAU0w&O?XgN=rD_|2xWm!ENt_B^#K^%GkujL?WRUdCD9*WOL`v
zen|;~-CFg9Q+Pi}J=ZVNX6gLX9CiB7;RpYc6Mj5@pm6y3aW~Hh4$h56hK37{Nfw>@
z*RR#5{YY|xM8>WE4?P45iWfA!dBYPFbY_Z-+L?m~A3Sq=v&^K4VeR`aOLJ~^A0M9j
ze+#yh!Y#G2c~k%IPpZS5c|XO&$`zWVFU7kCG4#It|KHHv>Ejl&f6Cq)HyNd*yx6#T
z@_n}C<m747pa1&tQT*L9wtrl#;Z_O}*B6}j@-#eoI{3<rD_2uDY+M*+FLqWYMw5Aw
zk6T+K^UB8@usreS&xFO{3(j&pnR?hGeW&wpwGFdZ8AuC9bhq!9*rah%Rn6`05tkmD
z|C=`${QL3I-MP7a?p#54mc<(ly;*Ixa?WdMiZuTl9G)&Fn3%Hgfx^BcE^L*RpSBu5
zc(*%gno@>CoLp)fq+#Hs!l$PvR#=#(r}v{s?#a(z`y~W>H0%{TSt`TC+&6w~Z+ys~
zwXxB_H#&O5k|PuL9CCJO2o0YO8gjC>R^Dd9sQBM#feOcy5GS8q2PZ!eX!`&E|MV#-
zAz^9PuBjQ9c61x7s83!lDw`lC#Nw@-nJE$r4Nw(6H#fJuJh8mI75Njh#qLaAH^X^;
z#BnnY^N^V{{;{&0SiU@ak|3A7{lBtuaZ%x9XQ>&A5y}r%CVdFE+Hp&M!hxUXPn>x0
z<jAt+?tYI1gp+63%xvskbWNO@nfcL;>_3O&BcxZYS;GMuhGzKx-$|wK{{H`l1{*eP
zR9wWnQp-`WrMD_^u4kjUQ{htPlrQYMs;1_%ckS|;EG6dY`AJLb&Y?p(8KI)@zOfXQ
zHbmAbi02$p5iGfEVk#*s6LIE@hqpWv6AvpZPhH)TUvkz5td=zL7>G;%|9}3&KU=SG
zXyK;P7r+1CrcFf&B0?F8Kc+M8{mbY&VH<~8%htb*r6OPT<>fE$sodu1_(@0S$hPA-
zIW4iVu3x|YG)NNHJ``;5z$+p`a)z6t$e;iJ?KL{C-ks~}*B23AP-G+`YM7EPymiYA
z1qVH?4_h1zxu?q82UK}NgEn(QSJ$RQ6OISG{uP@X5*I96V$bv9D*HM!&8ntmWl71t
zo153JT2;c?r)+3w_~PY9RrS+9e|K-*w8&CAdoq)+KsICZ)`|nc2@W@SPI$IY{5ZdX
z!MV_|IznRh4vU9h4jwsTP^#q|DS36*`htczMjbN~6H`(iJaoLG1GQhIR!67j&><ck
zp8pfJsTev){QqxkWN2vQaDA$=p~0WjloTi1FY_6XHEfuuEH=IPpR(bE!_yr-I<_u!
zynicd$;y|PPEOAM|L5SOAFVABbLY4`dXzGEu4_cZhwai56BbQoj4o1K7T7ek+x>&Q
z{r~-|w89b}{`a2tXJ74a`)i#hX=!Y=wQcj{*xcP?KDr7X&~f?@Za8NSPfkupZ#mS3
zDtv)~WqI%H|Nr}&+Sb?nCpjUZA-?#_e|ur&$MH87ZtgMYURW(?Fp1+1XGAdDr!s!F
zGe-^{_%MIs$NjTs@0dCB@5jgMudl1!Q~!VU@&5gP|NWic+36TG>GCnR6Zhi}9dwL{
znKO0bL`L_Y{r0~)JFAmK{u~x?>EPh{r>~W?ZNrcKxwr2tyZ^hphjZ`Rc{67^#>_c*
zWs0J~goCHM9j7ch$(+QvXKI%wAA3y=Py9ZGbDJPpb<12^&@5}~{G{yTJd45$i{JfS
zugEpm#pJx>k)+T91Lw9iKPyg7m@mxtyk{%7XOV5*Ud4M^Dv#O4d;YAgh}qzw+4<9<
ziG#6A<evGx>kP{?c@yt7UrUSsRK8l$zv*M=?C*Upj8cnZr*F-bV>%Z2TW*zrW2hs`
zxxYqRCf*Vjo4zY}nnc^@iRalsL)b}<L8pGazaOu*tllH^u>F=#5B}Fbxx9Mf_ale8
z1=yC9G?-3Zuv?)1SA#Mi9~&QESWM*<saD31r#?&-%Xlks!a<4sxIJIr|D+_ISFe63
zPgHA2Xk=vX>EW4gue<{qNq#as_xCwUmH7Ps|35h;#ldRsOwLZ;l}Qo--M#-;oa9K9
zWou^CG(E8M#L^S>A%a^X5*sG&Ffu$Sk$T`Ri(_7_&}6j_D%>f0o356y9tlxB@!-0?
zh$hr*mA?A_|LT8#;}=Vrz%4&xL#X5jr547QPx9Gvc-Z!=4Rn)Wp6TK(#v@_q5_Hp{
zMrLw~#k!EzYabocE=xb~664u0rP1>Fr~mecSBGn)8^R6UU;qE_kAn>UG8@9wmEXJ)
zR@?IB;DLr3MK<P5dJ@vTE`>ZooZ>(GPq{hs+(>F&5~04#LQy|7QH?KU(mjU%O^j<k
z{}9l-CHewxAf(SNuUTkdXz-``?wq4XPW<4%X6ra7li!nfgB<UljAFJKBFyh5H8U=r
zaa2c3B0#yJ@r8!^zJSC7%NV9xtbEBF=gDhxz&QAP!<2w~dm3dp89}KDR3v)J*w_6@
zPWbU%zUs3Gv&}n6i3g!PCNB^83%0Iy*%vEr+WA2K`H%jCM^8C)8=tXITfa#mRK~x3
z8snxVLZMj-jGsPO&eLLxeDyYZO6ysui~VHk_t*YzzasLt{|aC0{q#1o4F^v!czbx5
zt3J>&zTjdonRQNgWKKfY1gW!6mh!MYa<ecv-NVS%v*P;fWY!r$@9!_3A_Fx`)AHY+
z#D+?KZ)=gT7K24<d~8dWsd~<hXi*OI^wDeSRlnBRnWMCOlgi6OSJh+_4xBjg;k|rg
zN@M7^tJ-by9=kiB=BV`9*Z=$b?=Qb*>VbT-lp`KWAGT?IQebCZ=%(SfJZS<?#c3WJ
z!IKU#+04FN!Qvk~d9wX08W{hvX)e==dGgeI$`NQg?%gS)n!jBBGCu$R8!w3c(H$10
zuvDlimF-^|`<&j@XB_&R*m^V)9lE5qlyP3$$h)aqzG>^31q>-75(PKS^8-Yn?ujh^
z@R+?sfM@3)2R-8pzcdBdZrz<}5SjG1>C~~Qb_HAAI{TxZ%<-+UY>bzTl9JV7xOb~z
zIoqF;Oicg&I@|w*tN=JP%i>>hPga7j<cz5to=4fPD6imbVw~C?X7k|Xh6nwAY>%c)
zoco4N-@#92hHr?)@Bj5}+w&#&Es{G48qoaz-^o+PzW!h9UFCioo35>_U3adhHk`d<
zw1BU}aKWZRR;9E@Kl*#S=XiDWcn4iIm@uz_P18Z!?89j%iTry8HUIk-@j%)t+Lr$g
z+?H9P8(gtoG;XTFg3{E7lYe_WGc+>zqrU4&!T}9q34=w9{tp{hA2?wDr*(gEcl_A|
z7M9)XBd$6#q&<QLiN6dukGJ&oD!JM(`rkA~;X;<!8XgPr%X=eS4Nmj0^=LSD+cybz
z?qAuc{_=rQfd#`BlP~Y(51*d?BKkKpde(;}C^l_9c(7q{QKG`4t&QqFotl;s9o^wJ
zPwhW;Nft;O`7oqi{Pa~DG#j7s8#)5(<SDbi{{PS4-{o@xWHxJdNxV6f5od6ytNp)$
zAe+k9h&k-#PnY}G|Noo(;Bj(O?vdZ^OBgJ@{w2D|t>Z{ru%@(grUKi7*}tCiAAWwG
z=g*%b^QQT$9Ef`P=l}T+?_>|pwSI9G-X8byskg8Bp{~T#pQCnTh2=Fi%{xMFds5kU
zc~7u>fA8;1<L7sOf3Kf3xiuv8=7cK?RQV?D+7#h#EzyxTe^I0f!-N0zZMxACda+3z
z-N^v~3;G&=+*Sgw*yxU13k^umY4`X4ul>wsH08_t|NsAQkj?*}_}KgZg>7mIy*hsn
z@AYOC|1QsbI9_ep|IO+9Z|?f~`1Ex9jW1nq*L0WOwp(_ho=5oKLaEkFyEz}_ug+;?
z)Q>k>6LWGy3g_Wwc9rdV8<qa-l8}&icmH_N4pm43@RWIf|9|b@U-Ha{>vPiW5_{Py
zyqfZN{%e=ow?j!tO1@wIf5n#<;>O0!`|PYQ`7PhSLR08+){|WQ8J!I0)3nxjid}AH
z=CQDl`0+zR&i+{4Oh!kCtW!V!&wu!@b_#2g0L)`D_5Y3bRDKqJtKw<WqS8G_K|%Fw
zz0AUX<ApkLfBxOKXS24}i;Fv>(kl6JMx^}ejTMua>P#Y{=GN5kJbJ|A<HHlZU1&Fx
zq<@yeGMCDqLaBCQpveP<|NoPmjGsO}ZqF0ZSN@CNoBh9}#6RbAZOvD#e*_EtQ+odE
zt9<mfnuiY)we-XeM4Kh3iL5E~KVzolbD3=i+d-}wGb9cjlBoNaa$|G*!)v;?Zhif4
ze|UX7PyYQy<)IQ15>V%R%k=#J@7!H-W^zWND$6t%XSY*7K0ZD_Mf1I!{lB?WkLv56
z?^%1`N4uBO>3{BC<}BWVYtjzgwfAV;X676sa(Ht(&+4#+I(H9qb>vS^xccLZ=Hb`Z
zBl13hyzu|Olef(On>YXe{mstlSi#wt;i>Sp$z?UGz)F_#$fl0v|Ns9#H}`kp+gtit
zT5B>h@7!3#8?0{=nc&MiLzVs76$6$@tSo>2pBIonf8f}$4K<dBPfljJmTn+$QT`zF
zy7=^p4-DS0eCjO&>dXEA#oC|qgzug`+YQqO33mra<v0K9*Sr<|67&22|NW(}?fhqb
zX`DE5;Zj$JuxXspiXX*GpX+wHR%nZ_QRUtAu-P`!fk9ZfxslPi#N_Z?Yo6L#9{qTs
z)e{r)St~d3I?c1`TxxpizyWB`Nk0Ga>+Ah?{{R2~{blzw>FN2u@!UmY0|TYliyb!K
zzscX`VBd84-~XSV>pwnP-`gv1U-RSojT7%*ojP`=$7UA;`-zPmHerr`0@$BLToYhE
zbcn~$5VTGu=gx@(M~)ce%l))o`|<pP`}_Ns9y|!Q)KljEzQ6UKp2)umsbO0(m%T&N
zWQnOAk3?{GgfKg^O<vsIU)H&|rb`;{+f(&bU$@uVc5e4vS<6oof|=8T5*)mo|4(rF
zx0owwks{BTGZLFNN#xy0h}b>t;c@=9mBAb5aJZH=GcWpnTD+}aexo!r7UgHxetRP?
zEL>#$*nUkiQ)+_9Ck{b3H?|vdbu@Ud{{R2K_V>5%7Z%DdT`I5U_vg>2r~Ci^mpnKB
z|ItHD`)#{#wjEgGXrL{Xa@5&6)xf~u&;RoUk622!Bp>Hvv$t;xT^$hi>EKjB`)!SD
zx*HoGo|A10U%w$6T6Ah!Mnp(NME?Kx?{9xEuTj}Q{;(p4O*(t#IB4qrbIzaN%(ue1
z&(106{D;TK|35vw-+$((uaA!27jQQ)GCFa}Xm8C=_b91(Ru9iAo=IwEbf4I;+DX#x
z!Eye!^7jiC->Ux)S~qv$!hsF-_Ra0=&HL<{m#CaGUew{E{r|u59IK-r*p8Q)L!%os
zAjA_B^WU!iAHVgAX2!)U8S`@u4rNKQxw(1FW?q{6@xA=v>+5;=<^EU}J-Pnq=>79^
ze(GwRSh4>6rza=xo9F-g|Nnf!&rkFH=l}EaJjh;oA&c?MD#o3epX8@@$L!?ME_lRp
z_~qq*GA}M>dHKW7&hm7N8^`QFmymQQA>+h?B}WdNKY!rt*#oCfA2{6n{)c|d($imG
zM|3HH`sR>UQ~iIVn&034UtV7C<kYeD%L;YjMOzIN8YWH@oK)~2mF@6s^Z#Y<{%o!N
zeO}qU?(d(U_8J|w1{41N`@4VY)ObmW1-t(~?3KRqsFAU=_u2!8u(q3TB=R2Iw{O#r
z7ck!!6>U6YmXU_G@tYe5LCX$LPUhKPZ>*tZ)X{Cs!DW<@Wwc~@^8Vxe;-C`pws{-B
ze1RBfnhWZ*e+D+Srw$zY#J_UR5B7L_9(|{P@ETBM{Qq<N!>6a~zrXui`|ZvB!|nC@
z@&Ae+9eK~r|BsWyKuW@ZoqhA}of|nh4VNwdpOD;~_w?Dbw3=^k);BR7k$Ixj$e5p9
zVo~DU!SesV@td2>vR9`~OMCDz@xryV3)j*<d{2+q){}Tvs?9iE;@#ckiZ30Ft*sBQ
zXtHT)wyn{*Bqj6xo#CG9_P<KiY;2(R2gtSdAK$&cUVrJ*|B2sjDSb~XEc1DH_xI)N
z-yeT_YybY<-}>L*{?9i5Ur`|;D;v3Z(G6y%8!j#qk`fYmcYhr|+&n|1?&~Xg>D%=;
za-YxWY+zIt<XN4^Vff&xcAM|)1v!C%F7~cLlN6Pk6-}l*JjmSk`MIQ=eOkr;c@MAa
zvxSCEnWq)?wQ=jm!mnC~7rWp1y^)6p)Sv;m%U>oZ2Q;Rqul!lQxRHO&y9RNU3ClDd
zzr9^=ZM}be#K+g)^Vt{~-8ebjyuHJurNfGg#cFHQ^z_&~YU=;}d4El}MZ2VQ=8T#D
z|Ns2l-^%(f<;<t#7By}*wzmKOCG+mP#jGuTFV{RvYRcIoM?lH!^l1gvqYGA@`tXeR
z@N|8i{dI;bRyxjVc+u{Yq$arI$K~ZOw9idsgw(eFpf-E`rzh)sc!byVp5(aA!X5ja
z`Mv>fbl{)=pP$zY3H5MtzEM)@;o|bywrx#Umyd_L+P~z4gde}Z|DUX0AHT0<8rPgT
zAWI60|EDJ%nx!D&R<x<1VSl~ky*-Y1zK97l6xHakY}jPv;c4jMHECzh<Hzo8`tb!~
z(#S!yzs~SLS5-y*6Zt(889gN@gmDO8u4iLA_W%F?(<clnDmD}p8GZU}%*AQQ#c8-{
z)1t4k|BMX{Ykq$E|JYsQ|L<}(F)=pA#{V1K^q2*Y9y`HQ|KCW)o-elH@WF!%R&><<
z>V5N5gpI9D-Ctr|+|lqNNZIftW;fRX1)e<tcjlIS`2YXd$@+r8m=e*;3!{QsF71_M
zzS;NW(=J(=HKyyY?%5-k>gmy0k!oPXc1>>4;+yFYPt0_Fk?gZPsDN*2kLu-}oBjTX
zi80KWD70t(`A;q@zAn_QnHh5TN5`by3m7tZ4fnaug@m}g;@=+=AO8IO|HDW3C8E+7
zRyav%wJMzmkvXRO?f=BBHNU>R|MYafyq!bt!^g=F9{kYP;@Zyfdc|uFUr7N_jK05r
zVaD%&a}OS2Iqj9aMU0J^*(>TG*MpZYzvpLcZ34|PffAj*<-h;w6`!8Sui5g0UH)a$
zRGA3}d0tFlYhir-T4}e?Ba@7kp#IH|kN2g`{~eA!cKGm$b>|M8`}h97eaHNb-(ubM
z5>iqsev4gvFkxESgJ)?!{z`jqX5OjN&|+UxB)|4~6WhOksXNMgA6{0kIR^>NH**d7
z8VopG<DGn7KVq%=zq{o9h3JwM-Q5S5PGaj-|B%nZ)+YWV@!(-ar=Dk4hH7r-+nSe7
zm7a3_^Bkt%>z2+@%F<ot!XeY1C(UnEchP$5rCS`=MFs!s-FmXD^|op8O`nsSCVZa$
zkO4F`{{O$zc7aLn65by9zkxI2$O<-<)B~5;xBO{{?w@7%&zYtE-=F_KKJGu>zi^R^
z3}`-Y`h*|+yT7NJL@sFf|6j81Z&JlCk&FF5KD9df^{qI%<j58qg{*zG$rXQRHF!b1
zbLi4Di$bQO8aH&bmwaW|>&aZ#dHMhUj-}6MtmS2E``^>^KW_iO@*f}Se=Iy{UuWRq
zVbIlSSWuwwzD|NQ02H=gwGYp=?qId}^oeK98WG>K$xX?}Ij(3TQoTN?lH9RF;@=;~
z)%X4#K7Me*v)d995+CG`{GZ?0`0#yvTetZCzdt_8&*PslQ&L1k;@{uj|DT_)_gc4G
zYTbkD`k>{idG}m8Hf#z?yf1z0!}Yn=EyAD#@c+M4zl@q%+kAV;ef7sLeM@XOd(PNE
z;g$>!56_9mFFYQL1-SqDaqz?2+xxSx3vi{RO5V8f&)X}ga`F8S3=@7-e`HF{>FVuu
zbZFRO%Qb^xr|zeh!iR5f?_mB=2r4!}>C0bcf4%XY{pSzx?zWRzE7>Ua{e)kWwqx;y
z>+ApX$^GenbH~13LPb?FB64Lx{M8T46Re(IRA#FX?wvZ-F;Fo6l!G_diCQre7JH9(
zkKJ`<%Nv3W{{P?U{GR{+jrUZu=dMaHF_`$<BZHTFjq_blBhj|z$N#(5F5a_e&TQ=N
zcJ%9e_&)xSW;sifeZ7QTU5Z7$+|uZS2OHN<X+1wl>3i(Pu<!pDp57!;_djh%Y4?Xd
zP`H3*9fO|#|9$+!FVp577Kg%2vJG<n_CJxDlJe)h{NelidHCfNrcK_k$>_vs<3B$R
z9ysc{{I-h0!TI)}^@STY7#J8juD%$)USeNetBy*7fUxtOwOudEts2DjCGPEae<E{2
zRL05a#i<KwS0AwVA4=UArY9h8TkV$cgB#Q&WBC7n&s<wRhX8>Vd#**nd!1GmvrRSL
z-4>*z?=_XF*RAC$N9Dgo0`~>YCZBz$GI^(f`}Mb5uPU{r$*XOZ+{?bjOlX&qkMO?p
zHquXz+h5pb!h5)Z!_{%#V$)R5(1*KL2k3C$R^#YUdNf1DQS+GK`Q2M3_bq+DGJ8`F
zzq%D);*NbB)4S(Pb~z`r?xIQU)Y1rWakOQ&Enl+%PpiEulbGIt1LqVYA}lqgpA_0?
zXuLr{lUaDd1;v0Xr&jIy!1qA@W#qKKIf{=h7{jl=edEm}rm6I8LGIz5{RLB%d^No%
zwPf4=5?NIH{$=$t&i9gRYmQ1xV_kY-mOxAW!onjGyZQt&q;5^yrFL4QE84l9>G!!^
zNpC%`RWwh2p1$_q&gTLj)y@_goVRGdw*SkG4$wlL|NpnlT+7w!Akb=WIEz(K$F8C5
z;2Y`udykkongS1Zom_NK;Sg{2M|O9e@AC{251o^|VQYPO;?%`DCqH|ioI7)cfKvYK
z7_l;M)`HkW0?8dp2R(mrUYlBZ?DLB3Ez2@zsO{=owW}xLTa?XQhWp!Bi+Bkh7h#Qb
z6PU2<$G`YGnd1v{CE1RBe6TH7rE`bPz6W{tZ)fn-{=e~CQXU*|Yvyj`Y%maUk#CAT
zESRGcqq2Et`_xwD9SyEJtg0MhW}1sy7iEYjS3JEs|M&Nu&Gp7IP6@ilWb$?NFP@ux
zx$tM<l3ac}4i4unZXM<3UpQBL_z0Kw8fv%(281$1*>79@_o7LPWToFMAy*a7Ag0?!
zzqk)OTo0VK{8+w>@%cEd*HuB^Cv2Ryubwlx95hT0o=>m)n`BYWr@C^{RkecXi&Z;k
zylm^`P0irnq%ya&^Wpb=HcrmA{q+m4+NeIbZJxoovt0l1-Q7I;aRzfN4i<dTU{+T@
ze13jQb+$_5a(~Hd>ry97OS33yN&H*YcEW?r+4;pMW-dk*EBl%Ri_)&fR#w5JGiP{g
zY<QTMEP~mJHZyPb?aE|jW<LD&^$|Jf7|NR2kRj&%H}Cp>V7PbiBy&Y<Vc`cMB`uAk
zt_6BaUj)3)XBRY9=bv=oG`C{jRMS2l=EMqdndg6>t~8jJdrfMoo=T!$_h+Bm_dedA
zz0gJTrIk(prX?CPmgSa7I6VLSTwjgF>4Hg$WTw}o2fvSeR1>jVviZjDn$U~iuYA22
zzP3}T>40pLYV+qS1&`kIc};wt-@TcMf#LuEDKl;PTmuAJ?WgV#?42fhqlq_SdFl!c
z7sZR&d5%tt1V3JmIjQYYQ8}+**GZG$nwgf9i+PV5C_I(>7W1hv@}HNc39lkwi0|Z8
zi{3|yE#o}B%XnvcZgpww%ygb+lcnokS}p3><f6I6W?7ZC;)SoaeN{Vh%>2dMR5<tf
z%Lubn$Tj5dZ;rbx;XbR)M*Z}@bB(VbS=1l?HCz2D&yk(cB3s^GK4=F{W<4`Eay1)>
zxbo|<9XWV%;o|7O4A1MD@4r%87aPl>*`dc*re&ql(a4vX>zZBrSUT^)#0kI8{jId*
zS$O-7<G$^8{%ZcP>)Tv#w6;HJit2RkprlB7soi#Y|1zdJajfE6yYs$h@UFgHYNrL5
zH2Xhoxw+`J*>hvZ120>i?=>o18`yQcdE3)6<@5hnNUmJAFq6Z1Qj3LJS3&Hej>0Fh
z*6*i2oO=1lSFb<=9`4l{4G)){vzisZzw7<JMvm0Vj)ECgkdU5Z$k*f`;5vWP5iULl
zS7%3^90iq_!+p$^I#-;8rKO!5-}LsHusAYhJ#2fhs`DJH+Rd4jyFVLS&!4<F^G05$
zlBO$9cl8M&SJsERyQXtbTlz}!xN_>rDqn#I79y<6Jg*n-S8lPHb+Kcc>t&0%0!=pR
zo;&W|cpI@+qW`4P$z$;xnk_yDmnrgmw22YFeas|&*OBMD*XKSh$e*;~zVPum=U@D4
zU+dxQ9n8~ww8<dFDs^+B-SSlhGQ0Xtw$wdY>b_uE;PnvE>)NN6{#m|c%GUSBvsdn$
zGU5BRc*FY|LDyyY>fcCKgJY^??q04#1|qHY>Ki>2SA>3GcQ=oX;j(iqzkhyu#=>ar
zA52`YUOZaV5!IEs@X>{UtzVz>D5mWGAehJZeZu*IB^%GY7FGPT_SoxF*5<BZCcSm@
z<ND7&tCVQlq4#xHQERkrpZhG|NiDzUW=;Jo7yG;>ZM!o^&HT4wnvptJzPH{KITN2L
z)#kX=X>ntw(M$t5ZGqtET`}kG-w14(QnH(e&1bpJ^c{MAhu<#ylz3l+Wr|mY+-lLe
zeA{|cA6E9bt%^AOplD&(;}WOqmNFg5OOAYxTtD;dr;SI-W<UJ!xq1JM;y)W)Hv38#
z&HumfPbfdwzf<NKayKOiw8<+9IQ@EcdaXjiiC7MvmIZ5f?!S5K(mRDm0V*Dooi8XY
z<rU+-G{Ggp?QYfKg9o|a|K=&*dLw^-`tGBJSr?yM%lv+^qI&PE(9YAXWwWn#-R^xX
zsJT(6%(`Umq6Y;FZmV{A2z`<{|F}fyeCHygXMZBriXB#*%h$}f?QBX@;uMSh7k{v&
zhl+-}a&$Glp89gi&7Plgo(f%8J>Bs*>Z9$q4;<&us_i>paVDwpv4U#QrT4{l$+LFo
zIc>N(VcDANEr~BH<}KCWIGEA0EOfbszW%O#pD%csbNss?TEGr2CFa<2Hx&pRtM}B+
z>=sxU{osniLlMm@cf{XHTd+KQB<8Es!F%>%(xLl~N2I32>Au|dc*pdI59jTcey45G
z)ueEsQ10hE&v%=1YgSE-5Zf%d`|fF{YO6qzHP?NXzp$EjyuxRa0?S{!M342ypB}2~
zUsYsxecxK~y{nue)9j8LTqyMszP(P}Wx?&gkGb?bO>a(D?Pfi#x~}_qp^eaS7RT=D
zX%}B^xN9Wy=y!zJb?wE1&5TY9GF=3_4n1rTSn|@^-b-k;=wwglC;J%+&7WG%_C37$
zqI?l}q4uHcOU%sEBlgZ}oGfhklA+LK$-5Tg3pX-4nIA2iBJ$;6jhh>roLq)=j9=fw
zlfrC*f}oye+@6F7kCQ8YPI~yieo<S6B7;%)x7Xs$^W~eTiZ=J~G|!P~Ze;w*o*2I`
z#iDA;!+*7H-QtpFc_}|Wv8I1>W&04qJyR($HTB1znT{?kj7IT)|61?j)Y0L2_Kauh
zRGzrF8_$CkbdA@1{v^<%_VvB|;kD5`l9D{(>n1#igtW-l%vR)Uao}-{_iA!-x)AYZ
zhGD`74wmE2#gak>9C?lgEQh-!%WiI%AQ*jh$tvI9zh`grFE!!$VjE?>v3GObx8vUf
zyLK(h+;=YW``lE=meN$2k2cS4_Sxr`l%C`fT-1@Pd|Jc%(i8;+&$<O4<5l`rYAxej
zdZPH&wYupx1%iD)B)<Ee7T<X1vz2AHK${3_)ctiEcO36uee#L>_1becWuLM#fZCVS
zE&lyUw5XhP@DNiL<F$)?XQr_)t9Nmcu(6fYi%qg9;kbKug3O2i_FMG5&NSE?1q6dO
z7)_W48uBP8Pyg|my()rrZR0Gd<~2IaU0ltxr4Ju!<+;7h@Xw!<1^;X~)gFA*6c%pY
zXy{yIG%d4g4Z~U|$GpZvto!TJElRr%L>2o$!s^g;i+}%8Eh;7`noez;EnRy;X6MJo
z<u@7_!ot|p)n6o@eSSW8*Rl4gQ$fuFUS783WHv)Xw&dit0}dH`+qO7Pf7UR0vZGrY
zXn^rm7Mrzon{&H_kFVsnH;EPBc&v<F%?irXSFBH;FyX*G&W7a-Ho81)R}MY?U*Be)
zFVQD+u+oDY)K6ph|6j?V=lJQ<2GwTGlZ6-EY`l5M;c-%KC1^nL&&drnmWTiR<avI6
zvgI~il_Lj^98u6Y^WgE}goFbMDn}A>&M4^lB`EY=V@Uel&d(#RXHfI&$c9Ye5|wA)
za)oaB@bK{H=zw<roIJ@RD9E#H*^h7f2WJZw=&d`(?(*k-eA_a$w#>^O-=x3=)|QzY
zIa?e=T;%n1BqTYw#5VYHh$SQ_vXvz(iYcgN<*gB&F=K~XQ%u3EHPz)3cU~vh9X2@g
zzxImGVx9B8hYFel19=zUxoP%l)~ZFXXDlnu|Izn)(|Z>|FGUtlr&&s>`}`&B!hYvJ
zlRUmK_ht38)@6>_Thd|$4++hDe`o3>g#({8xE@I?eZ&9D|L)~ioa=T~$@C~!o-?qS
z_8DAFE}3<n%Y{Lp-QLo&gV}tiWk<KQ|AU)0Oz#!_6)G8?K2cL$I%U!~huybQ|8FUt
zV|i|Uyz%`r>4g$(g+0%0>z3Y=I5ycLPIc3|E2gQQ^CmB|R?3N1;dD%x^(}u^;gM~E
zt|F}WkDCOBHYKzd)%%IMyUu&Zw0x<4vh(~}|Im%=QY3v38w7~FunKL}QRX;fJj?C+
zLyHKpb>0$-Uaxqi+PqW#M&I6c@8>rropzce;JRP1PZzYq<^TUBvvs)|90XeTCkdA&
z&OW-y{DaHAgUS`UyU!+Vh~$~ZdRXd+t1g@1|621I4?pg0oVhc<;kn@PB`*ut%t<}P
zYTuA}Kw^#R`An%}6BjvnwH%OWYA`rrxkat(MfDV?Q`fh?E%9FI{FhTF`(y2@u62hk
zPYEfm$X>>|^xF3qm2R)UCVkIYx!aVHUtrT$uiKMUQf^KU%x45;!zHkGr{jxv3VKVE
z`ClxoYQ1-yo1^!_tAOQQKf({@pPS%tsG8;8@{3!KS*}@IZ_!uud7r$<Rx!txed#it
zg2^+|uT5RoT+~r-5xIHp@?-O|>Tb=N63~^oC^Mi-AR~I((s}21SY3GkXp-?gW1g>1
z9_vbjQ<>7W<B5qpwY43p+%r5|8X13QNc{WK$hqRiE|+&#0-09KaPeS0q1nv1RqJ5R
z%tgIDj%o`ZJ(BR7t8}>xl5>^HdT&2F%aeY7(U-XwE@|pH#@On`G<>>ne2eVm+!?Pn
z&T4qcI_Zy-qQmBi2?~o>{`@^1v^jHg`bp?o6eY95n*Zl6Ts-*UIsdt5{B=hh3cnp?
zzTni$7+tbxsuIg&(`6RRes1|Jvh49mhw^s@YhpZ|;-!S4mM50J*;&kEoSx(<`Qz^a
zhtms=$md4PZNI;D$0Sxg?Fl<WzK9s+9or%m`Qf^LTj*+uclVZ_08QM1GP+{npFbxz
zR2Uv^<&L=H!s8TO<`CDA<R&^xTB2YHzmJ}rc7ZET*x{9v7(Yn#slR>n>G%KtJkQS?
z=iGF9`6~<RfZW)>&Fnmr)eLKXwaDG9*weDdj5FKmpzyLEfBkFjDQTJ-FIZ+{qse@^
zX3edO5*hmrKdt+-?ScFLdgCu&4Q;C3(!5$h>*qo4(JlP*`*^_zg~NL)BecYoq?Kl5
zvg*$0;8b|zEX>qVD3NA#=+w~?v%1fmRg8LlcTOMR@o(DaV77ek+=q88g~DncLObnB
zmCP@u>+^7n8@+gCWMOG&uwdQ4q$Atdq$-{I9?W2uYnaKoA+7trMUmDVZ=ZaQNPXp3
zLh2mSmzU2_%3$bcN+~?!a`-@lDr>?ro{CAgwsa<5R%81yas3pKk3sHCFaNMPooDOT
z3wMv5ko*0;zkucO%-6^H|1DmpoS1MxD4pj9lhkFINep4p9~j)0wCcM=^}eh)qtdim
zrfGHik{<1FiD&22E539{%ok(>nfd>}QgKLQXQyNQ{KR`UY?+xg$}f0$UgVfeV*LKC
z#)d&x;9?R}_{_!}&3PHeq}Km9esIEHrwb8lo@$4&%}Z)#T+H$Fdp~IZ+^?#(x3@j#
z%{mTU;jt*=#L3g$3)lBMM$BkzWULlSOgJ#fXv4Xafio;5K1`E5amr&W({yV|!>}ul
zwln!ZN^tPy<*}H=%AB>a(d_)Er`*o*@`wNYRLSvHgwB2}(kL`AHa>F1z{JF$p!l(j
z#c``Xzm#7GLMI8wS5D^LAeJ`cDf=cPC1Lp})s8uvI48|G%BIr!=jY)K35?F^>Wx>s
zmOvwE(F+%$6AW+n*Ylh`%M%mBBDH)*P==znn`&>oO!ixcnnefK^mA-0Du`@fz4eGg
z$)*GcEwc&F-zhfDHZ$~@;qY|I!2{3<!bOK{gv7<0=gJ=5lF4IZ)A7`vhbN*-fGtPW
z;95|N!hx%PN9NmI$T@gRMsWrA>fnzY|BhedIsPi`%qiCJM+psAb^iZ1)`>e~P^k2%
z!BPw6ymKnb?Ci~be23>)@=W0V%VB0MQLselQS`DMLhQ`j*7$UrvCq1iqTcs^MXYp;
zE{}z{CCHlwnL>v@J>}t-pY*`=5oCyL(KDVc`|O;HjhlB`PWzg0M4gZAQIeo;Scj+l
z5;eZ0TQU_s$0jQjT=jC<W~uPvvU=OfU`a7OmGn;WiIBvx=+h?_R@SyZHEkz6rql*h
zOyXePw!|UmVWSNDj7>9+3FQ<&2x?o+;pi<h!&4?ZwCU>H9jEx&dfuJ9yBm}TbRrzK
z#%YH^&H8ss)X%SNgF)M?tQQM+pK#@1-nLHXW^L4;PG(=0=^HD&L~L#qW;*(@L<IB8
zxhq}QugIEgUoYV|KP_TIL*jG4N#7Sv;DVZU@{pRD8Cz&*+cLEmb~$T2S!NzNQ{%#9
z(p%9q?+Tkfvzv_YW1BUd%&TY4xXN|y*Hinf`(N)`x7GfZNIR3TqjEA=usRDgj65GT
zFeoarxx2Tuv9SdPzL}j_sLHWPn4S6EqS<?nIJ%umP*5sNc={+m>(9@_1&`Sd&obqS
z-ezD^=d+_|3Cul-j}j6R5)u<9Oi1wXO5)&66%ccqQ`>FYnQ-k6OY)QxM;vt<?%VDV
zV*hxROG^BncICf04^IfTrJk11i(mFIGJOWrU%8JO806&GJU!c<q_9<2vt?$o@$ntH
zE5^3z*L;Bn?OK7$3P**zBfJIIycIfgsxP5gWGzP{_rB`mpfe?w`SQ5;8`;$ReTqwh
zo<^|99b%xNA!uC2*qBXJlub}@PH%A2V|Ers!$9vziWRTK&$#qx7aA-O<2j+pwpp(7
zd*8!1k!{TE65rmZSA1%b)8&D-6c@!mYGBaQdvVV+IG9aVmQ7an$i#aYoUE_xiex=J
z|2p}+Zj>;c!F9ol!_93<5tEZ${kei428Ul>=4t0Q-c!^0^b6lEP^g1W1So$5ZeZ%^
zu~k;G#m2H(T7q^F%F4dz^AI_nc*P<@S}9{;|MDyaW`Ue^f#AP?kDKwFYIt>Mf4!vN
z{InZ;XFt4W+je>R!sB{;pg}Fr;Yu@(Fm9bKxNB>SPmiNlm!ntL!DDR)kFZqTX5;5i
z+QhbB(d1Obj$P6fQt~|(X}2!;G&IET2Q8oY@xA}y-|DtrX~{mB#EO4&9zHzmSR%{|
zuA`64-OJhLAmB1z@6p`l5e*ZiZq>_&C9wZ%T$J>Qb$d-?T9XB9j>$>BGqd&27Eb4I
zZQL3n>gzmDxGhC8)id_7%;hC(bhAB!djgMLx5@6jB(2w1*r#zx>uKSJbve?%&sDy<
z*X3^4mEm~jFl(=(=ej%R{~JrP_+MLEvqN)H`LB8Fk|QPDZ-uSgc=$urqOT72U^kt)
zyOFEKfalo9{T;1gQ2~Nq|K2}&_NcEoyLa@2L%e4X+BO_c<()Zc&HnfM`yOxKBKCdZ
znzcG>Eb1R?lnX0=_vO%N-73)>^r(5M$~Q-jB`Sv6HTAnTG^x5SU9}`SZL3@MEwSvv
zzIBV=OR`N_w2re;;^fR_2Q8n?%oGy4SY32XVp{9-hc=dcd&FlH+>Z?JbiQG>&U?;|
h^3reRufaV=W`?;7RzxxUV7w1X)Sj+>F6*2UngG!74J-fv

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..e1799a87c8542d7e515b6185d7e8f6f75fe73f3e
GIT binary patch
literal 19132
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3YW8$-45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpM<#3(WKiH~dGLL{Wq#sgmWfTX4&P^7t--ZIMJwyj-w6z7wy;_`
zd*428|M<Hl*vKcB0=XCzIanQ3wGY03SAW>9$>H$!w~n)n&p2D2IVF*GEo<sM*R3nW
zCSH<aJmQm9qQ7>k%MqUM`by^;t(Alh+?!#1H{S9SGsty!E;;hGD)6u#Nb>&AZ|HVY
z;(fjO-4}->d|2eZ%AJ#Zw%Kx-XZu6>L+js{%9y&%OW&^}lq~MF{FTdz>f=6V($db&
z>=2ARXJEW2Ogk%g>4GWKjMs7h?khdI?V&}%ydCeq*?4ej)^0w{4Yn)Pk*mc(V7A!l
z5?v#us*kV!|9`TPqxF@e$a3TA7j}t>iOzbQmaPu|(q%f#Siaobx>h-Bn(CR)tKZvB
zly^7xJ!>GL^w!E%B>QzGM}w5|DyD$crA#58g@rGEo-#*O_2xDq<>g0?AN+mJ@YAko
zbH46-TKQ$ymMQV&A654-g6x<2)Y&|D?#$NKnNz2F22BbM4fP0_67SaLA3ohZaAIW8
zq{hjg^_3sjFX?bkR$e6U$)eXcFCgK~e;%It|NsAg|Nj2}`T76v?*0A#)YSdY&;PIc
z`|Er=fBih0pP&EP9=^Z-|Np<qKR%}~aBeq(M)$4G=I(A+cXyw#un=G06GvLYJv%$w
z+Z$V3C(fNaaq3rj?$h;dZckVmwFMVUms}B4f01kI|NnxjXYcR-U-$pt_s`Gw%iI6^
z`|t1fzrX9BpZj}v`}_Nc+w1q${r&#o;r{ynY*sxQAge&BY*MFlzyxPs-w;Pf7k76D
z@9yA`DZTyup#c+Sf^1~9ZQ3X*eIuCjS`*9KV~xd$FWF|rC<NKp|EvD~?tZuU|9v$-
ze?L0<f3o`jxP5=Ve|`O5Ki=riFVhPP-K;=X{QocI1-8M>Eg&q+!@WJ;x!K>V%iXOl
zIAqGiW@fqL9~Lo%KT=TWzWKvJ>|of%1uA?h2kYhS{_L;+|Gt_1f83rQ<sTpI&%Q2k
zZ(r(<9}^!wKi|mbzK9W=prx2VG2<CBB|Kz`zrTNkx3`P0uS@J)@0d9gr;1Kgn=otI
ze@TgdhDJ%tm~}cCG*uO+OU~$Ip3EE0ANuG0{rD{|%!Y6O|F55K|9^Mg-}|lH|K;pb
zc2rJ&cuuzM{CvrM_3})*LQrp9>1=LqpE`ZIci_bH2OPZI-H%-gity{}oGL1%Izfb?
zQ}Ig1v8zn31^@rIt1GP0<yj-joA$`TKiWf+c_P=tr>Bnh|M#DlVo@>S;X~)P-`^$U
z_9XoHnjH{s4mE!XDAK3Sog3=ze&c|{(c|uSPjrZ0+Le;jq{6y}mAPtW<JyXb<;&|&
zJd%(V(^2Irn%ksYc)<0ak-?w#m%qBF*!}ze^YDlF@`q=e^JHH)dUN~uhgYJ9A0OxW
z|IhHxKidn6eY{YcN;;co&YalXeDsjhy+cliuZDX0^|j8PJ+=9<y|6H|P2Pm6<q=*S
zZf;ZBn1UFag?4{PX^AOZFjew{+42AX{~K_NZc*dN+SznZpmFBs`HhVa-^aJ{%S-I5
zOSPz)l6cdo?fH30Is3GP1c{QvpP^R9fYQL>3jycO`JFxH785ngJ7PxfWZ^g!g9UDi
z*H*FU{xf(ja%K}_Fk?|?)6A*~r`zf+rYu}CyZ-+LEh#oOwi9c3LH;^F-}uhHvm4TQ
z58vI*<3FEooe~Qq9q4pAhfE3f_P%z^?c|*pr^rYLr>42RZ|!gW5OvFKS}omVdhKF2
zOUzV`O-WB2gVu0ztPF7oZd)4AIFW~EgA<d}zWVb8uf-03ea%z-%^+s~xevcg5BJM6
zONtjjJ@Dhm!Q;o5t<uWOU!RnvwRX9>mg>=Sr%o~St6W*9dG1K#&0Q)Xeh;+F0#5O;
z<@oYmnbJ7NMImbfx9E<{`iWhVY;0GKFe={Po4li__2D7bw&ddyda+4AJ|sTaZ49;H
z!I6clw6gNn<z%nUNK{nUKY#4dA%^bE3}?p8Z#`Gc=9;hFxcayb&%eWyM819b^51?>
zeAm`94l7+Alqqg?3%s>TVzI*ihq>HaR6hN;Km7YU&vHLQ8QW6@pHvS2{mmn9&lhsW
zz`(!&YT1S(ikc@EFHm^;e0gI2`jv|hpFPqN!(gCr)MVW$ChsGUSIJ(OwZdaleW8i>
ze?`^4{Sn$~SM+$Qwl)QRQ23sj!0k3+5=WF)<>$7<*I{j&(<S?45<z+M^mN8fB{rxX
z4Mz^1II&=%;>$N{UcFh9p00lKAR~Vjn@uLu)&mD7c^R~An4m5DW^Sm=-bfMC6{0LQ
zYaNb<Z{Y3z>pbh(-|m>l3;OyE6Q;>$cs_X<+*bKnVqILSMg6>ohuMF*i!g(ueNpzK
zgrp=9@#ld-lX&>ezkRwiX-Z1lQqD6hALXN0Hmpwk$e;X5G~MXXGKPK*FVmJVg9N|R
ziPG8bFHb3TE>K}j(UVHPZ`Z~vEx{*~@Z(eK!`I<eZsJh0;~yoYrw0ZKmX==q^3~MP
zOe`_&U%$yBR)2wvtL##pF;gG(RP11iX->G|#q>s#|Mmg#Wjbo&dz}&+`sORCy^@gp
zpS;aea>G>l`u|2Xe@__{s~zqX=9#X4c`FB`@LJ^lC?PHF$>YaYE=}s|+VtVmB{89-
zw8X@e2EN!o?p<I1drWLmP}F7X`Ny%xveQ&`p{Q0yVOuEIAvZbAgH{`x!VD7pwPysE
zoLHP<V4k39c7a9l&Hj4Hy8mewWj%>sv)Yv1CGPECe))|oB(N$UB_t<LoAzw`)~h|e
zTg?mv9zRG(O8V75M<>9Wui`-Bx%G#VMXI?Rx}@2j@bq3${n;;;Yr!Sj!nvZ8Ig+tR
zv+16U<K$oe|I07=-E5e_=F7U{m3Ydh|MhM2?IrK+PmkEpkoZ2Xjh$b@u6EJS4lRhK
zxsM(^Nx6Du3I~q~KbO&zNsp3}lUvleC77+gbehcWTKeQbVDo|{5?6jktTE--Wf2uF
zw)tY$t>XvpeLAql{Q{TJ7Dc0`vq3(3dpLf6Zhv^&yiL|x!Yn6YM@i?yui0#~^wdDD
zsQ>>LB|duaFwxv}s=rT3SwVo9Ktgh2V#`v_j`jjmqYLSm3QaivJq-BCzw>P4#0?jG
z4o@!GaQ?w#KP!z)*0o)Za*D3Lwk_FWY**S&?62p!zu!1!_puE{s)zsm<?)|ybRq8~
z#5Y@JgBl_{uJ`%#-W*G{ZJK@W;L|(DCRb|sPl_yAwej&aAuk2{hk;D3(|Gi+&Rw=;
z-RW($UMC-0%qZN}nsT={n%(ojt&7*Eg&IeN$5sUdTm4xUxT?NQ#a7j+qK&JQOO%60
z&Lb(r@pHo$_v)F|{LlKXyytzt?Tklf(21FqY4<Jrw%4WXgM@3{$%ALkXs8@{{W3Ek
z;>EfJ2hN{6$I@NxVR?`tKk>k4$!*VAmE4uCikv8AoY%#6?(gXX2O3HxL{=&@YaY8G
z-gU_6dwq!J998zL#q8()aYsLCTv2Xt>}*Hwv_m%qSZ3b3yJu$NL#MXg<(WpZko>S_
z_D0?Y1p(Ll*N+_YlaglNSjZWBaP6Gau5WC)TNIL>hdyztvncxSSZqIeaptPTg3mVV
zUSIj18>{_o-*-a+y(ukXbv$d0tQ{A&SNsa?JhV1_nzYrPV7o;Xl3Fhxd2%?Uc}O0s
zc-yndLu)R(T}0BfuqtNd?xaZuyRMh&aG%?l`^NhC$Is$Vm_QA(MQM*7JV*!(7Ut*Q
z-rK!dK<H6QQd1T8GutOoEu5O228IR(2X1t{a8%KrA|!1YHo-fs;)$aY=h|b9s~x@B
z|8G3$)UJ>al)=e;s=-xy(WBr0<zLmQun99CwqN3MnB%j8($_!#`#0A)vPEP|s;RN%
z<*}KYv&qS|?JSl&H$T1N_q16rA%U_e3Y5GrUY^|Bv4NZ4Tu9(S%M`8^^BTjoKO`|r
zt>NM6;o<S&;i;eKB(U>?g9~R+%fY1^R=95yk3M(!%&8NPF1-2o|9`!YX2^F1Yvy?x
ziBmP!Bs44)-BG<DlU1$bozRvkjc1PfDEavCym`a3Z5xRE{M>ks&B+h1M6X@~brk;p
zU*rcWsIFa|+S^^s!)5g3d3t(EO3PB7430nl|I3TDet7YxJ!biV&n7_+8-;jJs2aEk
zggr}E<96%W;ga^p{c4lY!4+<fNfQ}^d4p0}%vpBwG;CHZWuIon?aQ6k*f^8#LdwLY
z5fKtPIua3)k`WOSeR4~qenHCkMNXh1?8=oXeSO<_c}yOF>~7iOFjGPx^l*JfT1tz>
zOitaDiAf@=ZvwazQyh1cZDC?t#HO2IU~KCoWBM#0Wumh5SzaEGr!12!=J0+LD!Sx;
z&3nNL1&!9NuM(4z49wcN-rYa{;hn5#+X=Xp0YQ^Gx;F8083_qINJvRZN=RsO(aKuP
zcxgwn(<@&CLnEc_VuFe7Yd@bn(6F@WKuG^Jivz2;^VHVmsYgj3YY62Hn%MA6ZpFzD
zZQQfGB*f!ZI4te(RGPt)a^T33grq~DHq`m~JiosMCQjpn<TjJS6Q{V`T_f9DBsjT?
z4`vCwMVw^RWt&jFp{M8ngjWeu4l!))Y>!b@R##x_F=-6jm5^4R(qQ{u>EyrTZb=he
zb>9lC4b0gkVE604{4CW4S9P)s8P<ClWD6+$HuCY|NlW9Y|7TG1|6IWjgQdaHI(m^t
zVMo`ew$_>L?Xx?&KPPxgo(N^LIro5-jXCIO<C$Y`nh!8b*}dpvDwwKqVkw`G%;vw1
zq7Sr$3Nl@?BmTH+iz;LV%w7>;d!j+7dwJ@q2J4FkhK4g{8`u0l_u;kp;f={6YfT}o
zCnfvB51)%!*rnawW}Q6AX<&Hpvz1D!f#UPJh@L~*B2T2+8Fd>9MAv52`t_`0iTox%
zNu;Rc-|>ZVavuwX{7ZY)?<z=0m@0?G`uzX@|FDO11jD*z8tc9sJasA|`S6Fw?1#_K
z=UMJI@%|4;3@RBHZrE&GU6JAL7PV?+r<0$zQ&#$c(>zbiG;8cy1-JfiH3%>kwh2}2
zHc$!;IuUfjmv_xy{~fVMnAdc&S;{7eDQ4(778(RaHf>&^5Hw-NmxCuyCZr$#@QnBH
z@qQljya|u;Rp2(gc<I^NdNU*2d(#F7|7%Wuk{h;J3GeZ1G<B7nAhj?<vw=}lBd}`r
zhdf3f)oasO!U}B|bniY`pW0!_r7ix>*X6bJ%B_v6I~TZ`>9HBUY@R>=;N7_o4>GrH
z&zFqbuXa2cVo{QkGrzfcwNFoTYpaa7xbd162a^j4n#>bj5;$2cbrvwJj68neL`P7_
z0fr@#zvWAPWO);=iJUm~FL^?RNHUYnQ+A`yXQn1*lL|Zbuzb}oTH?fVamfVMiz$Y@
zj$%`$q;zyA8yF=eq#b)}e+Octl4W5*(WWEEqksL%;^x?}u(F3SY^Q?Zj08{iz!eU+
z6?;B>X_=!VVAo<CxLDF(NKt4(=8>o<f%!=nCb2FPa|<!6@;e~)G2$}gGQBmoE{bSt
z8BLjH^x}oVhE0#SMb^OWxN%EMP0gyKtH{9Uk<<i-)$sy%qbd$Igg7Zo7vs=B$PguZ
zxIyq&#gY@uCocAMUI<BgmFf`MxmCeYqK~UUjI*MPm+gwc;VVbaoLR8)<b)X~6x7{|
zBA}-07It)hR#mB3vGU`AF3pVlLR%ADRu|5E@a`Ck$Jvhm5>MIxi5;kS>iNH-tMDpQ
zMxmR)gH@3QD^m_k)48yt<H*4Vb5nx^Z}SLNMUG89ZBwTzTFf|jXHMhn*&P2BK`ktX
z|NoT~3pu!ojvR@otU9H*_N9}B)XlXUwyAG<f9k}EhFeE}@E6<eNpN_5=Mm$7Lm|c}
znS`^CmWXMkh&W|zd#rDBAxo)3=wNu7LBbOLrdHl7Ms93nWo@&h*pidk{QQLWKy!~`
z;fz^YYAO*2zL^^@;5AcNFP!GEgy)BoT*}k{Hk%9suAculx_Gx7V`qM+;UazMKory8
z<EM{qk(#b>?GSVJDTirCLqiii*?A-c`<L~cIOXBeqG&V6(T8WfdmzXg|NkqQfr_7B
zzooS`Hy(T>r`O!U=^@&z@<YD3QS4HH6?2d|OLmq6b4FIjO5Y1xf;l!EGg-zIlgjq(
zO2b`+J1?32gPDC9XNfQ~H_w)Co-51D&u{3LkdTl7jR&p5g2Ig!*4dR+T1E<!Sob72
zoL>BYV$fmxC#qFFJ^ur=W!WN<4z1Drw^%8qVdV_Q*-}1^YqSbAo0*$>17da=WDCDp
z%Nz5uErIvYfd;*fQ1MNTH8nhmi9D-T@#q}k&4+qItMJ6>?y9OR4(`oL?&?9_Oo#rp
z&q<PyoEUs8J|*&s6$jrxWl@6+2IGYfG|fC#>7UT^D=_uuyr$2yWSTsWMBupw-P#39
zg?uzUeuwz>9=tj8#xzZE>8v!z;>XXMu`ySAdR=GElr&f*78iWL{rJUH!)t$&mhsrA
zvdru0TYcjImxq^O1f!>K$QsU8p5g*CSIG^k+-yBY%)WCP8yy`Q4jyeicw<ImZ|{b;
z(2$v9v1Cb$tn5@5mm3pii+H$lFbBCTntPNv-mY(3nasq1f-A}u0?b*P99CbEn6Tu6
zi^0w(jB8Sv=P9*Jv{?O7Rmx_qADav(Q~2eU&Q3?y)`O?I8k?IZbW2D;&7WYgV`t>t
zxuT_|t`QLu8$=~;TxQvJrC}#)BV%+bqw9yxBlh+GYJYwC{P?(h{l7oXvtm`2xi;x|
za69-&HpH#{^ZPicbUohBbASKhl`3yM{-m-oyBRWzr#Scq)zt8qnei-J)^hv`Gy*Od
z35q@2ye&*bG_|>sVIJRvofE8D1v3|lGe5gx^G=!JV)C#1`~Tb5|Jzsa@aDh2<?(xd
zoH=z$Md{GWB}dk;(o)brudQ*S&FX5?w%e_Y*;&ea?UGtF9nu_b^sK0w$of#F>)8MQ
z|Ns9u_~F^v{q6k!@9h5m{N&{Q@%#TJr~O;M=+J}Lr&Saj-e~a8GCCtFCtw<}{OJVW
zfbjGS*V8S^c#L}&fdYl$|NleNEg~Xk-nk>v+j}%Y{fzOVzBq$JNv)h=(;U|P=Z`<n
zc;0Y@tnisxtn&{2`~U0f^NWkijnn7Z6g;@MIeq^9JwLy_z8>%9#^&hA$IZR+LjiAn
zeQI1>+7c^1@dH;o4n{dAhB^xvJPK=@Wc2S(;*P4X=VzJTH_!c5`1x7<iC6J)acq{B
z$3j<22n)}&v60xhBOpxp(UwU&N>7}4G$WH^)53>GINOqsf4IpCPJ4%@TU1n9`uI*g
zawH-!yRqKD{(xkLV9%Xm=X1g{I+=U_{(pYn{&>Iri&s_^kB&U%m*1!S{!7*OcgMHo
ze)pSKQ}*G(;(d13a&lr7%rgUO9y75lmN&M2Xe6`wgF{FC|9|zDm&xC~JOBJ#=bKx9
z{4_p%b@l)Jdw=R*U;FPpT_QMGQdPBcmR;(R84KLCB`zGf=*8C|<?G8<TH1Db`N!O)
zkn-u!^=AgAQ@3oIl#+H#s9R#T!2gA%Vc<crf30slUGArKoYiQvum3mC`umb)Y1jAF
zmfzq1|KERmG4;LG-;+N-+dXq;^?dvPzkZv4>e6R(66{<vNy0Iu;ij@{A4|>3&wqda
z_nuz&{>`77mqOn>etd3!`0efg%gg@Gw)uJg^71D+ng?#BJ5F;7W1jx>Ny>+hi4&$D
zPX~>2{{Meyrp1jLGuEuVnwl~rFgzq-8PA!u?>fvhgHDTV_V1pTBxh3*9=`qcmzV1H
z^^fkD|NC!$`1W>tKG~{ES4ti|J}$rF`pgHK_7WR3orC$;vKojyHs|F}`1ASs|7&aS
zi|hO-dV4GXPqyX#J&7~y|Np<U^Z)(5%I7jb*-@P}*qWKSd8T9|qai5Kft~W?>Cd@y
zL=>*iOH|?hQp4{rzrp0=bjHOQj!%`icseZa@Bcr``umkm;r_b6N(GA!?63bHzwb}k
zhX<FJdgt$}`zveuuRv6BN85FY2_=b}Cc1Knc}L8eQvLnkon2pl|NOka)AH}Xv(jy$
zt0lJOtLL7!Sr^E=W>ZE^Ss9zXJ)5KBoHj^@;?PWsoSZABrjZl8Ikq%%dDydVzR@sU
z>_!)FjO&U2-qYvlMwdN2ExzcTz>m+*|1Wl*FRr_*@c+N#`tfy%N5qO|IWYeZl>Be_
zXdYwvm&Z&elqESnot$jX&R27P^Yh=|-@8bD{cnHx?QI_IFaw(!k1a<f{zj}(WnboH
z@$%(K1>N4uHPG;wXVKDPk&`QW^TviWyDdx%dv6?&d39i#PJq_3{q_I#;%eUg{Oo_E
zjf?qkzkI#C{l85)H{UMz-~Z_B<r$W8^;er$i#U2S&bYR4x=i)A8vDXW&u5#fm?-`G
zmugWxE%CD2r6{?Kmko^3iw>@uG;w0%<jDum^fY#J?>-5)e#=(Xw{L%5blYXX&D?XP
zVJ6=OF$n>oz^^vd-wMyqyY0d2v*<uO|Nd+1_t*dacKrMMdLgaPk{_;dADN<5QDncX
z$-Z7f&hAg`rzicJ(>?EneEDU1cwH<{{(Yg<a@RLVq;<1|#|jG`R8VTV3mxG*G|%G8
z*QYM76NOo}++^JSlWnbNQ{2=CD*`T9>31-_ys?q}{QP<~Gn<45-`|6Vr>#m}+-zo_
zpv1oO)B&B(7Z?oB3Z7ZT6)fg$dE~^2f*&9Ht;_4`|M}G1WnumQ&+yKUg9XocIHNXf
zGygb6EW=mW*LRLKbR6i=B#R?QcdlJ?#Nd$ly$1)PoD`JgG*kbyF05Z68}+5>$(NVv
z_xDdOF1olrzJ7a7<(CHs({J=fu?Vs(II7L8XnG{v6IAY=KXRnt(UJb_>nCiwU%riQ
z3tsMVxvA7aHaxR&*~FPM4_=$rI9<GZBQ$^~SxlMY^7YFNNdf1_{{x=8{7XKjH_0fb
zlVS5pM%Vg(e|DF?{(67EO1cpnTU-48e|L9%{rK!GzyG|NgMzyY%noL;t~T;%xXRV?
z>;3)zvHSmhKG<xpA2*?L!tL+xdBWFS_#k1y*0Uuk`S5}B{jQRbl5&zoNT@3d%Zz~j
z_bu1;PxSJf`NX{T%z>y~3)H7<&)>hN^7GsK_C-SM%*>!Uk@t6g);#CilcqR>uV5<6
z4llu^E@74*A0Pj3<+iu4QTXq6bAP?0oUPKrn~aPVv!r(H(k)<p^ytQK9f${={ACs`
zn&ali^ybZl!on?wZxw9!{J*&9;syB}Q2~?MnxCH@+h4mFu;}33-ShMA|7+Q<bRbGO
z!Ix!5)&{S{qJ^*O|NW`|`DuOmdyP{+K!M+5x^5dQb6*zgxu;K44Ga%nN{3XcPV#cX
zLXQGNpC%+GCM5h|e?8C8aaEApvHA_no)QVmO&l+YDKnP{vF@8FQ+bWIa?vD4<BtD-
zex5$&cCAfEqT<Jg({FFf2P*%T@9Q`|RiQ&CXT!F*K$CyJzP>-!8=rqaC`$XmeEp-$
z4<_eE?BwytWU_NTe)#Z$wPz0;Jow?QbXOqMxB8YYE)^{;Hx#zdPtaD}ASS)zmEa-)
z;mh@Wd|g7vkN5ANXZQEp-{10g1kX2^%O<Q1o^aYbr6Ka}f`a=0|NcKa8?VH^uYgfa
z&1k*I(Wr$9=XNb*3=3mROw{4-hx*e~CMU;7OY063)0!KfxUQ*ulHX=0ve~!m+M$N;
zF%KT{g&sJ3_|*CHetsSY4qRVfFK7F!%kUN>Bg<u$c~$Jb)<sS~+npc(e}Dh~-m0(f
zkMnm4v8$^yTfcN{GJPajP#!8VVN&-BjpimMX5p+zsFe>te%P>bWg$0*!6I3;{#&OV
z_DQfV-I9>v@_I+o+av#6jVH{XAL`iH|73y#6DYoZeYri`+&<tj=UMI<OF7b57IP+r
zHcb08VQoT#N*}-czn~k}1q2V?t5x9m(37=*bJ5mlRaMC=S1$&ufD@=wX7Ps2o4Yy+
z3JMh3`Gnt0+5UgS6c$dYifX1h9jQMO7A8j;aszpk`ilw;U%q%Cu}GQEz}UF%|G(#F
zXUp^O)VPGG{x?vXG2=?ZX$O<{fBt_2&88?D8@oRLdGh4MnHCBRJ3|{aTep5afBw9J
zN=sZ3H151*bpHH#<KuJUUej{U9~(Dm_y$Jsb7h}u*v0<d=0`mn8(Vbroyqn`7#SJO
zu4J_)Fn-`KuSs^Z`RCG>^sit3|Bs)N_B9EA{?7h0%hkWXWvx34W6{ILnJJ1VTU%Qj
zXG_=ql7skfb=Zt0ODrDL9kZAubMeiAYugT-a?M_sl)wGd$wF<dN%5Qi{QBx2GUe~{
z^ZRdHVvN!{7@V_!^Ph8QzxqMN*@r*;|6lLv*>&r>jSUYU-;sCLiv$$TU(nK6@E(>G
zRQS@;+(JX;3=P#P794ngw6R^}*wQ;Q&UXKoaDFx;IX>gf{xfHIK7NdP^X5tW$`%%O
zdwXp+t^@JD3!c_(xx&Eca?OIn`Nt0l6O)L#ze@XW7?`C6gsVrJGcyb723R<)OA-;4
zOi2xdj{Q4nTC%WOva()$^j@Di>Ek(jw`2c%dOnDJ;NSVZ@#4gv-3Q&Y?E|8g9XxpI
z*RLnrZ6YEir%v7Z@6S(f7ZIiDtt?5i0zNjY-u3V@I(fn%_|gLpFQs^W=6817jtaWH
z`({F#cTOsO|Nl$w*lBs=hJf+Q71v)L;^z1G)v%mx&0N7Vr&ui=7d*XE&%^V7%NCZ)
zm$ybnp0sT9^68n|-|rebS32{gVbqmHiaw4pp`2+_Ne2|UO~c~mc}C6hjG86WZ(v}s
zVue9V`{d=KvJS^~82na?U{qFS78kE-7J<aqhkt*6pFe-TbI)#;=-JFqccm_IZOrIo
z2>*3pRt#v?FFEO#xY)8MPgULBG(5bEEv&oi{{4}@-CB2IBL}a^qHeJ>Q$F$^_9;p2
z>G>ZTG9xCY<3?@=V@s>#s#PZ>A0FVk5X79O&N{E^!K;gY@1cp&Q)|+sH#dqzKkY3m
zI2Lcwxp=|>AF&UoUhtdg{;#N*k$z@_sGwrfd^sTjk(8-ZpPHD9KX{gQsKX}4lxbbj
zQ~NfBgSVc@>)cd)V`PwEW|A^-y87A=32A8^{mcC}Hb#0Ryxo`NxAnw{1szcPDjMd_
z_4Mju-7N6$zx-NT#{H8d?>+d@E%JIt+nx>6<o{m`@$%T9AN@>0-Tlx<mosO2e*Qeg
z#H6!9S0p2_ySsO5!hxwA8?>CS_1y{lb^P$*2QN=PdhC3NGuOw5M^KQbwbdg0%|w<0
zRo0kLo;j<Rv|RFrOcyw*@OgUj85*)}DQJl;;@tW9ufFnQ`yacKdd%3<HXXSC??BuW
z3ro*4^YgP46kH5AX3UUqak+8oR7hyZjvqf}7#4lDzoMbCEa5=#oIi&Xb-uYQUUuxs
z+q178A9hOVH__=?xLf*uR6G~s#I+%kl2Z;sr+A%I_~hiu;_lf?EM(y2zEZ?w99KA1
zOyCRq*4B^mi)T1~iJ39;zqM7@+Q?*aF}42V-aS1$j~>Y^U8=Qk;iOHQ-t0)`Gv1l-
zUE!S*lW}6ARJZG?v%Qwe&1X+{v;Hp-Gd6b4)I7YWl4sJSC*>v^+W6<WX<VBa%*MtR
z7&wKG8<M6}a@5q)%F3ohRxEifHB(|j-ts2t4wDKldA0N-GZwp-Dco{1G%-`@kd%=8
zudQ9YJ|a<8e*J_gC7+&p>y+P?l;*y2b>+@oT?GXONiO{JR;aptlV@gTez%Inzsu26
z{>+*GckW1tibg&=pB}OE)A`58BOW<>b|zj_V$;_4`5(ygfLS8qwRlW-`oI6_51zP8
zgyk~6w6taO?f?J#&%b~3d-<ZQ2AwNE{?F%pTg4#3=^wO%?VRCcjyt0N13fjGCw|;N
zefp8T)%V!>_T1U~`{>%}_<c1$4<9?WV#SHneEpnsbsgoyebw7V|FyB!ef{vIaZbR7
zu7s5V6MU0X|E^zp^v2brGJ*#;q@BGlYyHo@?#zJ`ClpkUB;=hFkUaU|fy1qjbt;TO
z!EDOPhu6nzw1TG0|NnQ=vy|9h|Nk^YZlb}#X>S)y<@)fYvF)F-DyxmK_6g&H4=POl
zi6>q%pL+Hv>B}df@1KNze|~=d{r!4L$^V--8X6fIKIoJCwBv{1L{0u|;pg(okL{Bm
z|37xD=j_@4imNks?8vaQ@(ViE(ER>>pb%tKf6d&DTn7|*T;@->`Tu`V;U?LZf{Rz2
zGTAs8UEj?+;(1uEdVW_m`^EkhU7r|Lm7G#qb8K4Xd$;r`a;9Cc`5#iEZu~f*K1*v8
z(<Gg_O%|^^IoH4WJ|od>ZNAqZ!TTo|qPPn{1Bw6tJE`>5|NnPTn4S4>JsVq^o7<Dy
z4l@EybE@5N?U=#K$0jkIu~de2?ZL*dodyf0N=mSD7Hu&U=&JKE{XfA+IpXC5q2uvs
z3A#2Dg6}mj#^?r^tbBhza0fJZtMow@r1UpFy#N3Igw+QBl&&iHcyO{l4O2S7yY7Uz
z-a(HiJKVJ`H|V|p(SIcEfSBP7&-NL!nod}xDV%n&d3yYTm*kGNqZfqEZqL^!hE6&-
zsr1da|6lv_)A#@NCm3xrba|dkYG!=7OONfw)PpCk-!y!%B6vfX-igdd_I=0LV_3rN
znZoxNB^(i3rpM;Q)!`|><JFW#e@UJl;y)6&>i%{Wy%mRMj=uQ)|LT6TeODI_I>?Zj
z_{aU#?n5_wcrIL?_uzQ_LoKI(7;fVh#*-#bcBCAb>r${(=+mNU3q%a3@?E>cWZB5a
zaA2y!1BTUazQ5O)y%HJ(dY1ocezWBl8yFZktTXz<pKN?+5{FIgpX1jQQXNEH3JtWR
zEqxbEmFMwLiE45<J#oh^d6lIMMpu&6nAR*k(ZFaF#Kz2gczeEvDkNPbIZpcV`JZjd
zl1VTB|F8G(d?;_T@Be@IEoKw61fLWduQBD8au9A=&g^!@;HZv)N0*I_W?SP9GtIi*
zAKd*1xi|QocRKL<yGFDU)O{*_@9+QLSO1^=nc<c@{dYngr)UeXtvQx>z(X=ZGJ|35
zhXYY92}*21x8y616sn3(3X;3_>}>+e|Np`ofp)b{nxKVc4FCT-y<7XI`a7R-p#m5G
zl7svnKL1^{{1o2uu{qgBm<XCof8;62vqJE&LlRfQfn^M9O;0uGu1)Z5IwAG{(2WyS
z>_QK+s)b~p|M)-u;bZqH&dZ^WlVqv>|Ia)}k^hsyi~18Q*mP3=xXPNP9=P_&;oi2!
z`bg$IR}MxMCHRV+C_HwcLG@avr%45u+$Qe+gG>sZjFN(!7tCHj3w%$R`F4MQ{rRb{
zuP+>wFLo=vu{84G$Htg6hd!tF_^!^d%?m7f!<(5`b6or4$fGpl@kbFGma7V<{~a#)
zpTIc7lSR9!x&FVzzB*S!X!F!lX1@LZy6^A!_4Qx>|NsBr|MZsk!A-GmPjz%jOz;x_
zp}Uf2iHXI6eGal>zxxlW+0{Nc|CGnY#q!#gqYbOLckbs^XXjSfx}~ANt@hWE56q?i
zq#>!$Qzm}jpSthw`uErv?@4A_>GjX~G)qM_Q}%b!nWr00Da^2t2xykD^fg%UER8K^
zV#}F?1F<;*KD;x!Bn`eSS}7CLuxGQ5!Y;cy7k?ROIC{$5-~Yez@2~Uw>(`t)qfr}v
z=5$kFesJ1>i=K|sHx9fsb7Gdu_}HT3?V~?sfzg?VAxRes7(N(1yKmnXzW&1YZ;-JG
zCr_FEb$@IB{P^B4pL6F<$3E>fiH!2M9p0TgAlbvC5z0Kzk*~(@s9RF%0#}=57KTfi
zZf008Y+3dEIDcF9_Y3EbLreSBVgK&!{QUd#^ZGk?{wJ>9AF%A%!@XfWQY@GB5)%><
z9C9lYa_fu|CMP-gGR{!i@u>6154B~2R*xpJx;*)u-WI)G!fvtLOAbhAtq%Klf6vd?
zXJ^+xdURnUZ_eu~34<H|KZa?{h++Bh<Kz2xcjxciIa5Za!ot$<_m&x*&5Zd^5_3z9
z0;W|x=v%10qp<nmDN#0YaW+QAR}vlj7yf!D3mPA>4?OP+b#DBGFQ1;yU+(`uE7LGR
zg`16y&Er4o+GM8G1J7C?s@orAR9<)B@BjG^pPr7NI`#aMClODd&fL1?L3Z@TU+o#2
z3zvvVv&@#|*&%%SKtuX@iBqQ~=h-Aqn4WH6s95&UaAItjPb?1)kAA$7O>LJR)JlJu
z_xJzTetKe_*2>hMd?oeRgk5cojg5^H1$Tb!V%rd|*VDsufB*m5e}A^OvCW<!@acl&
zwWMao@;8&B<Pzo{mX0yyFgFKzV}JeRZ7Q<<n-0tmh})Y~@n0@h01|BeGWK<Us(*hw
z|Np<vh7)co6aW90l$207?fqX_oo~i#!`*8#N_oZH*T($(_dTDjwA3sxP;H426Vrps
z;WKv5VN`x6S)kr}Fs?a)$<UBZP_Qk2zr?q9s%5!W^Byr~D>GXz1dUBUWUjKG*4PNm
zFY>qleSQ7^_V)UJe;Uu+{h!<*ySrhFppc@g$)P%(<7=bs!`A*fa_pFd`0>+cdR9J_
zzZ51O^XNnF*BM&>6IcA{j=q^NQ?ijkOpJ}6zwQ2h$!)pH4~1Ai9r*vB=l;ILm&c7@
zYnz<>W$OR^ss8xr`|0Ts!Uy9+4*vJp_*Fhq)sXYW@&5gj)%RC_dg31=vwoePc2-H!
z-S=O_XDckTSQpZ2dh~#;(Tj$?w#~DpK?}@gn+xx2Y`qc~n9|F`!*hO~;ho*>OP@gV
zl)udWdgD90zCORG{3YS<|Nr_qetdOd?6a6WUj6J=PF;|4=l}ozdNDr|uddQ>YVz^*
zW!g~t>&If@nIGjZ`z+a8a%4&8!Asi?uv%D3PMRdKuRc9uOUJ`kp-GSDEm@`^b7$}A
zg6DkGR3NGRkWuQdhllrDm;W<RnESZ7@(91+(ZlsuG#oGPD11Eq-#`BQ`~J>0_m7B}
z(b&P#7#Wh_%UEx4CBe6>*(%b3L0P$Zs;F~-z~Pz3JagyrxWv706@B1kSn%nd?VIJ9
zki30p)Anb_`~S<^|7*?hYb=ijm1c9fjyD)SV9cNI^etWK{8^&~u7~H_|DUe+_tu3E
zAD^Dy-`mSHZ{CF5bv!G&F3xJt*gLh};G2DjqYbY=TS#;pLqbZ*gb4{AJ}DN(ZHY&@
z+UoyHEb~be5K?;e%Oi<lj|*r$&)@1Vf_xx9g2w%WivIll{r|T43A5J!JUssupR%V+
zx6PW^py8dsUh3ew!PNJkp~0W(e}BqDul`=N=sUmszq2PEXzD&acCq9@ytd+WZI;^$
z3nngPpD|N%&05K<Y)KXtiLmu)9v%rhDuqwya;2=l*YKkDbKAqG-f!5|pdODavZ?v;
z{qFAiIdd-5X(=oJIZ^C(<r0%o<oO1Z<>wC^Xt-mVo|2I8<NN#n)Aj#v&;4zfdg}V;
z=k@i!zk$|1URrwp|G&S^R_Po!{b$66vF%yP6vb)S(Wb4zvStm>^K*{hrE+f{D|pJ~
zTx;v>!qS**<018LSFQBv8ycYDd62&+S%4-Y3LoE}ZT^4BQppF0B|Nw*A27%Fcs!Q*
zANYFue;yvt8cwHkRr|``o<BF&zWmLP!sqAWZ*)jDuhP1Fe*XUn+XNS@@}1e(UT+|j
z9d&R`SR3<?pOYURW_Q_E_o-#!sxDCYA8=sP)@BP0UGy#Sael#_eP=(sk^E)?a>W1t
zhxX0?{@A^umrwHdFLr6+<c1{-H%?ryujool`E&jRhvLH7YWMd3-d+Fy{oC91d3S$3
zejm@Kt1A{8JMHrFdi%OR@Bi1gou2-G!-hZ3UOpMg0U3oo*Y-Q)l_kiYmQs6g&$f+Q
ze1YNgNrx14&u*x*J-pDFr(fPU=gtWQ_2UZ`98gd=aNy`s1-)|#Szd2;9{(`q<8%IL
zsWOOx*}8v!zW+2{vh`p~h=IbhNC`<vg_jI*2{nIz{lBzyzq<dw%J1*)-`@WJ-TnXN
z&(GaA&;M8Z^HY4-G)<E!ckj>tdvEXl*Vq20BqSuHr3nZn{rT~6eFw*<i8udmROO2)
z<tmza(BRJ`R+c}%juyO?nv$Dv;K8d?3l<+Pc)|f%vA5jM@Xf8G3Tnp=96x^G<Vgj+
za{&=29=!31n*I2xcU$`Tf_oPrMZ%$Z7IFLk)xW$HFD?Bf*j(a4D%+bZ(GHO>5Be@I
z|L-&B$KPjXL4o!E*Vq5Y`~S1DN}fF%$;5m!Cido&r!#NdkeH_T_uSdDAHHfIzQ3QR
zrl#l38y=6qZqG%kef29IWo$iT!1DjUam;?c*z1})XAT@YwqO|uCFGyqP_BRYU^9<-
zp242l)1WEA=jVCm+Z*56aqz(#o~s@k&N2$te`t94DfLTs*aL`T@60jeYH{Fko&Va5
zi`gvrcBEzn_j&{NFD^;R+8j|Fw>UOQ=}uw1d6PTmyCO@FjmLxfM{UXXSFX}_)ADD3
zx8bJz>w|H+zMo%O1#+$3tMTdakEzS*=I!5nCj3<M;`gdNw$h2Q;gMoHz4UZ`G#x&9
z<o+baj(};~`Isg>xfYeobIjoU71P|)CL41_Se5$o{mW`C`1ulKB-`dQ&jD9Zhbr4n
zOxORvuQvI~v$Q||<tryJ2evrOS}|eSvGeo)_x8-#x%1}D8xk$8l2@-pELl49hJbtF
zzx0HJAOHT|KiJ&iXyNOtmYBHa;`xSCuG%(jiHDiQD?Tz6O8lIx&U5S53$BVv$(|mG
zEn62xKen`xNZ89RlA7^5?cvMEufyBQ-xru!NWdm(56!cXkd(Bm{q_IRQF~R@3RR~3
zb|-IcwPpW5{H!V~b5mA!V`dg}b7NCeV+#v&yV2;w!G3q&U-7z$4J|B-S7<!$m#?p^
zkO(}@@S}kx?U96(^6U_no*teLlaJ)&O4`+>2#C4u_iJr#Zk#JyeY3$PvR`yh?dgI)
z7W+1aL!#=?JPU<n<@t7wO3e4tQVw}2O_1YaR`K1i;lOE=CljSs<Xc!stc(B8{#{i1
zcKwag7d#eOVrPyRHi4?fetF}Vy{8TwIB?*|5l|97+xvG0@22Os>;9+hsFj`%nur4x
zLBVScHf(fU%w=}qEC0(iegAoQd?sYECNEW(CI8RZz@X;enGIE6_4)YP^y4KYBqZ!=
zf90nf;V_raS5E%1U3Wv%{(4EhxE49tZ;<8ZO-&I~9GiaKx1ZGR{{L@s#ot*EAG_CN
zLA|px{NwNPir}a6%wIyHT-Nd@D*K)|7~P#wTokbK`T75LRbRv>8O@j>@#f8pgmRC?
zyc-uAx&?4E^BNc%*Zl43u*uDxqI)ib;pdgWHD9^YW;aiq`0#`v)As9-mj0oc7IrlW
z7DX(H*Z%*XaD|~a@JB0`$HpK0(=sOH?{Z%tYrWtrleu~G4vWPN4F6aRA4N%IJUGtZ
zmVJGJvF4gJJUKZx-m^M;b7in!UbK|+iOA>Y{D*IDX3^7tw9O99w3sndG9n^iS>?a~
z?)#d5*efefTm9DXR1atza9{oZ?U|S56>l_Ny}#c$BlGX{DS!HP9Tl?WFEVU0;Vx=E
z^!`4NjLeJa3lo^OgfzKJGBY1OJ)KA1P9S0dXoJxI|A%fqOGrtn_}=&My!?^QnHv@c
zas0N=Y1NFGAUJ)tUCodGFD}|wtd0ns>KG%lcCt~?YW|o9_wC!Z=P&%6IA;z|P>_YP
z&}&ZrAIJAeNJyNUlUVV&tpU`r0LAi5a79zv-6;C<$^ZWy&M*J{caORHQNGeac3B&z
z<+7VM|L5QN!EfiLt#n93<A=OvXs;S~3ByZWr(Ie+JUnO4@a(T!cz;Usni&$z6Qkuo
z(^~sMYx5ZQLSweP;@E6+3t{m!^P5(PE1VU)(Zao`>Bsl?_xoh)A9`Pih?Kl><KOC)
zh6P1^FTSbOefX??_;7nibmhL9#D?%w1r5t{4#+%o&@R2VC$Zuy*Xc_FCQyT?TPWCS
ztjUd4ajcl-@N(XPi{Ib>m$&`(|E2IDA6r{D$Ak4A*BAFNl^i&6;=^<PBmpOf2FDPQ
zZYj2$$1)j@GE&)E{{FK)+{$gSJOo<)9hz=oSD%)U>?}FabycJRyYU3;EgE;a#s43^
zd+_-21<OE{d%-5zlMim2w|#ytnEbq^#zNiiV$}v758Kd~2mk9qTOQ)}D!t)>I<egD
z_xbr3vN+!{$!Id)TzlfIj~iRt|Lu8(cXl#QO-)LgFgY1CN0h$&R@8}y><SaVKjl_(
z{PXv;f>zhPXAY~a70w3zaeh{C@bmMF{uPHni2)R4;YBrnP89qy>DoEtauv@CPS&fw
z|Ns9_|M7*>@Y1LE@}P?1(xsAJf#Uo>zVlnHvdB1iu({)iqQ(gYy}r%A5)OpzRAAmB
z*jN8wf=|Zben7t{)V1Q?Yj)^sHZKX@@%w{F())XgQCo9zc$%6b^6ZvLSNxv#@U%Fa
zn;R&*>%}I4_ADIZ1+VO`Yw;^HHZ<($GSbjCe(~Da!oom8!eO<&S&?3~iZEye<3F2{
zE>WRvO-&JQ2NXW`{$G;x#EF&f-JQfAe`bCVJkG-d4X`b<thpKt1YG1h)~;od<`=P9
zDZWuqw@LHJ1mPQz9UZL)`MU&<tlcIvE%BSV|C1SNTgnvH&3cw=c)%hkba$mGhez;p
z@6Yo}_8n)t6xFlv{l;m#mT6?n`da-)<%tDDscfp}JmEHu#wjV1P6Dn+KE9vPSeYh~
zaa&k%(vyAX?#+EVOQY3reMm8Qf7hYu7XSXISA1%D_}rgOPfzEp%o9=79YS2f!iVSE
zYdHS@tlr$faQ5`)_xB}d%#@Unl-!n^Y*9V!;Zg23b$<!HSSQQW2lwsYc(42auhr_u
z?{d)Ap{i=hs%puY7>PZ5B(`joeDYLtulF2<2A`M%uSL|<*pibMnIx_K@{l1Yq~`ar
z4=)7IJ=2wf#EnwG&;S3<et68ztgilRfh}|2Ef(JozsuXo-wRkaOq|#_ckaXQ`LpiC
zvTpqKG5N)TM}4w9)@24Ye@}gQ$jtogZ`P67oQ&1q4Bp&4oRD@bA@k&dRi_?2V2Dj>
z&Q9pc`S2Fpv(b-NT6WXI;==U82jyDNYPgt#Hb4Gd-qtP7S)QZ_amJdt)?5t^JTCHD
zC#K0AxpnDu#Ua~cT^wfLZogz*^q{G=DM`@Yg55(jj)~LQ{OrTOjlHKQeLP~W)M6`{
zwR(e&MDvOB$*J=?m43}zmMD=Cy+*Z<)3Lhvm%{UcKAYoFrzWjZ@;tGrTuf@$r749=
z*74uz3J`fAIN7T6yv0nlU4GUd=4}Np^;Vhy%BL2EO$X2MCGy_-<?<&th>?-a&(G!L
z!9#~W{ILu%Y`w|1=<iA4w*31N_x2>-*wFa!S83bz^^$&b7gf$J{b_Xg>};O!^+r0e
zrzXrgm5_2oLHF!}H9iXx{Zw{TO?~*lvCV(JKx10cAqAbDJc~~$I<x8vJDrzYTbEk#
zcUHu4NN6mXtH{;hAmA!5y-jb%5#b1PX`43ok7Z?xPo@ZLTe0BalS5qx7c6oVxc1Wc
z!$adU|BVggpA@{a(fr{2kLBdobB86ixN)4Hbxle2SU~6*<5bT;1=Yh3lGkXuN-Vmv
zQ#f91S?9k0C!)f4^%bg|_Y9uYVxe~WhS~qViXRtqy<_iN1#>LwQ261!yIJGfC6o7)
zuVudhx00sJRpf3C5NWkHYd4ED(u@i}^639fdy@#C!+ISvOms7a&%UbW_AkgWxSzMD
z;Jj{oG5?$b{u7EUxw{wd*T1|`VcXun8$&;>lX++IvF1@_#56to5DS~WFSgH(J)I(r
zc%J-RHo<C(fiF*U<+CEizMXeDm|~{Co_e{n)RkjeYvGdD8PQ!$tA4SrU|M{<d65E-
zXYjjn8TRvsx0ULz*V5d+eQu=N<u|p89ZEI-EB-|5+TCfJV!~Vcw|x_24@1vvL#{&(
z0<QCqummlcDB@(bTg1^LG4ZjjO`P5W%SN3gzJeT2xS~2bbXLqR)#V9XEL6Ys=(NN8
z{?@1c>VACC{&wB2SLc3T<Mg+l<#X@A{<}&p#(KXqbZVce>XtJsogj4EjpJMOs;;YC
z$v56+{5IVy)&J3kjrq}xpL1^ZC@fgFQAeiz;f9T7+oin<_ZICeuq(Ei^gPFG&Gj96
zrGGz9uD6<)`ok{sRFS8IkgJZE$H|PAYqo_O3^UF?beI$%lE0hj^2^CD*ME;cAOG=V
zX7rY2I{&@SM&AV&C_Qs@`I`y^j@8$jNwoShNvAE5kP-T-EU==E$$r}X0}tM?G#f_>
zZ1!#9l-b32m|^k*?wDN#yScx=xw-f4F88UPhyI>RxS;e{-0KJ1#E%lU^U6IUk9|Fr
z*8E=CbA{F{x5%K-x4D6H*<2NQP8Od_FSEWhMdMMCW5>QD8lDz>hZWlzUmMLgo@@3+
z!R2_&Y7LKIo$jMePbYnyrJ!)W^YQ%pvOM#agsn*?hfHqfeY8oLd9&xaaBTjM_pv!w
zy{0W()uPzW{M&YU@aurU89p1{>TmoPE<OL)fyW0w-q`NH@+m852gaN^w%lMJ##m*v
zIP$&a<>mEl+}-Tx-7J65|5<p(v;&$_O)*l!tVho-+?w&h-}B;Y<GXhY3J%{bz9aj6
z=ZBkSvwK%fR_f4rSXT0H&9;flSQrCKwnvDqJDqZ<sB_)wR`1y2odyyUR5}7gR5+Il
z-_|{@{M<P8=hcAqCj(>C?b5AMEmBO9%!2RR{C`n2tG-TY&Gj{5Z@zbx+;e8Rd}Ldp
zj1KpC=gSiPFG~!3f7QO471wXG{C4X0#{~*~ok~aZczK!)j(nG&yM5-O+NeLL?wtrd
zn&dd`ufT6nknfhv-pJLIAkbhhvS^{etMUyOWA8f^lxM#Tlv0hVk($3SPGN3|P@#ah
zU{qh5impn}zLz`R_T^i?%L`k*sN-*)LGq<Z{S{|^p85ErY~2cxb={epZYSQ}qU&|h
z<cZ}jzga=kIwq+6b+Kx{+VxdRu!-TmyXLmr553>`$Auj=oX#D2t=_o)dFJNS!0Q!q
zN)~*lRof1xIfrhHDA;|mG-kaB>-XOY`<E436j=G{yiT_=Jl7%6=5jGV)E-<WY?-r>
zvjtRAt?t+`QL1C(PTq)yjkXaDwi$-k5(K9yXdId~Q$ljZ!Mq~w<GDq`eg-`A&mZ_>
z-y>`}t8L%$?{7Ij<`zXfnUVha_ZcR=(*jMORn7-?y;kD6S>94G^@GW;K$k!T)l|>e
z$;*!Q{|J_0zuXekqkOpG`QNbMlKHnP`;+IjD>!&fYgI|S-O#_)Z}p;B_t2Y5V>l9@
zzMoyXH23V$9o2il4pq`BjEFE;6M6W+xxPsv1xxukq$U6SJ-wmQ*f~gOT5`ZcQ&wT&
zZ&$DGV33yHqAMwIV8QzH3aUpFGEXXKd++$_uKVGW>fz_-d8X?dpP6^|!xzoNmzT3l
z`?;90qkd!9l%5`*OP6?-F6Hs`e32agL5<Z@_4xjJp4DN7TecbPsc@Vn4;hwJ%6j+b
z7;jtkcL@`d4dxybrH_3GWj)c#ZfxA#%<P<}cX)~Ci_;tr9G!hmHAtL0dUV0clLt<m
zc<}P1fWUzR=gukUol{V1+R$z15HrUyW{zXjEJcF}i64{NK0lYtyPI@l*VM+jb60#b
zI<h}vcEF)SJb{5cA3s{SyBbz2W%ve~99!<s!_F@h=hwptsb`e7RZg3KJ|L7eH|xPF
zuMZ+}|Nk5BsW`ZyK+(D0K69@oM?*>0j2R0y$4{EXqo>DHQ*+_utb+{m>{53Ww?BLt
z+_uykG`w8#Z_dN7*^|1R#l+Y$GuzJiwEd~kY0ix3=BoI3pkb{=qlk1npJ7V+WMNP}
z0$L|;RwyB9m~*RTbK1eFE*B0Q@8>z*Z|pPYM8OLIP#bf9eKNnb2j{`_{CCeJryf0U
z{(M3PC=fuCr%Hzooc7*iZ^&45?yyynGuw|ZosCncCSKQTn{6)n?r!ppEuD?CXFohA
zTk}_^LquA-d5=x=OiAY|tLC-3e(aCESat|;rJOOho}GAJE@^flbeqo_(D;ObfNOke
zvf@h?R_TLIEPES!kAG5-bJ^RLAT7POfy3#|Hjf*J-gGqu`x#H*IsE)@;yb%$0TbTT
z%!-Sixd-069aNIP)g{e#Yg(n7*ODXOZ<bF{>WT<|G(+WJ;fdm|rpF4W<2EG!$Pc)A
z;MlqQ`U^dK(jsfmZ!_%E>Au-7*p#Wn6UcR5nWG?fRhP-$UAMpd{TeJBlM!TVwySTK
z*{tuGptX|!|0^B*e()gA-@iODF*mdYpWI?u`{97p5zuy?zo$SQoay>J=jS_%Ti5<R
zwqWt$4<D78rKOuU7?iA-UGzadc>@PufvbJgPsg}OCPUEdM*V+@x<3gOKPEJ`w?90_
z%ciN>R{mb1qGG}Gu2ZLtH*9n)meo7{SX1kR$`tc_iLNe*vUe(L%bG!hnV@()c=^yF
zp3~C|B_st`gl$l&V%jpzC`MqqkxxSy_kpQ_?<W4YP?P_#B3)sMQ9-D7{XYYp$U_(I
z_#CofhP6Hy&Zzr6E%9a0ivV-U4nBrFK`G{J0b_xv<o44hUJOw|3#v{zTr2vlc6g5E
zljS^YY|!1lN^AM<@0j@ToNSA-!SM{vT{Ctugda)v6+RKw;eYCj|CfxT4F4LAcK!RC
z{Nvx;h3or27}(cAle}(W&7Tti5ul#i6;>XXcGFugl~UR@(}EHXgxMsBs%W^TJUTY3
zYQkK1PKEQK8J_>|+qYR3OMH8?@QN+eh@D^SKPoxb+beC7oZ!SBZZEyI;ovOM3C(Pi
z8`Z4}Q*~^1Jz$t_Y;aY0<_rmgNqo$Lnr%Ns8r1sse0^qr$>h|#yFA?Dlgt>qMWN<Z
zmS0>S4+_VKD5D!U9Og2;T=?gB#Gmi-rM`*>r@gGdaX@?CmL|<UZ*IBR6AWiPGj175
zZ!3sZNj>%7<M8o}M+J^P(;s^(@mO4Xy}zDET+iUkcj34*ZqTle(&oRpNrw!o&6>BE
zcx`WOWV|>v)xox#V>07bxoBoy{~eDf9g<o(<JH9-Dqrn5tY+IxHazX&BNQ%psKJDX
zU(@W}z0?)!+-}}<fE60Xg$9O(6%__Pb59*O>$^j^yYvk6%#ZT<{e1p>$7b<8xGCZ*
z6DiJo@5kXSeE(;#>k0n(mvkU7Dq&K^D{;0ZQ$BuBbdHrh+{nzrSF8kDECxzR`=6gX
zc1%FvfI%V?v$i(pNv6pz7OjHSTORt$@U)zgws|U0WZ9q2(Dfig;@XkMXJ$5E#hJ6R
z5*&QRcz7a$nA1*Gd}vs>Z{94;X4njnQoZ}<_3=EfuN(jQef+}h(+5s=<|jypvmKgz
zbmK9Nbw?WAxg9l|%qHF1b0AA$-7(>?$TgbGlN;}x+fg@n;jUQ^pLiyjFA;$Hc~Qxk
z{)rP0-kh2E$mNJ6<A-nT|B{*+qfa(wTKwy9JolYtu|W3jNh%HLE~3FL65A8iq++ge
zsWI<k`28ZV?NydnyU7O7coD<@|BKF@*w)DCoUY!yP?4FruT%6)n&y?NM_FPi4L8pz
z+~{$=)}14+*KpM##_wpp2y@>N#|=>jC)+)Eaaz1>Yqn%n)d$%;f0(OoZE0^|adu;K
zPEsmK*4=aXP{TyE2P!tRI#+J)OcCbXaIK_b7V8c!%O+3B>2D);9@*~s|8Me+>gkD(
zTwbiv?}a44MbG9)XlpmOvN~sKHqVqy>J!X9b>NATj7v<Lo3}v968@%Zh7bCxpB!n_
z>0Vyi`mAD&ZUCForQ@Kfu>HnsA`Wh-Fg(0A`pU~qk6?cM*RpbgK=WQ(=Tg(=9-c-m
zwpUD-vc=ePRx)n>a_qi<qT24qT{$tG%>9->Tt6)5w0Qj3y{-7UMB14Ii(09wE*oD+
zZMmrKQ3HdaA!rJ2f&iPc@{yIrGjjRZx`H{ERhdjKT*@Exn#rg4h<Sg(Rb?KFQ#@=v
ztDZdfZ?mnIFw0M~C}>#3-QfvyQs^Vl9;BqC2al64TuAuvEv=w1F(61O;HbhKmld2&
zHU}o#dFakGFio=J^lj-W(@^{Q-~RBiUY_FTMsw_ZpUslp1a(v5BL}t#Z;wu#$YW*I
zvG+8O#Vm!2NlczkEz`C<oTTb6o|fO5z%03W;=u-G-<rRt3Ldk8hV`|>jclsj4u`2j
z6a1p!M-9x3jBJL6Y?YO4?(S^a*=&rAY;tmJa&mK0IS(qb*f0c(g|o9vw|t_?XX2h@
z%I>fH_+M??+Gt5WIq;z&Z=<#PvnPUPR6xn9{E-7&Qxi{44v&@=kCPKmRTa;*YdkeI
zJbU)=JbI+@cx}eUj+nFwvn)J98|qgyH+EF9thRV&aP0PWp6&UdQQHlb#-O7c*2nXA
zIoCk^Ds|#WBU?p<goTB~kt3kd@0J#cHEShHN+kaMNt`e#NkH6vOV&dM|3!|>KdNPZ
zF#Zeb`r*bV#az_#@cw?D+uMzE?wtXh3bHkuXMg?Vf=yhYqX8KH|Ce&<Waizl$*7>n
zsG!KHKO;3c`NE~74V#+g9aK^~rl7_qTabL<8jE5k3+sNV7+wV%*~}+P8f;d5{Aql6
zZ8T5$dn21#@KF<IXY<J01tf;@L6fr7o=#@%2M-e;JWG4<EKNW(b;5*%gruYg&(jTz
zLDBx;LBfx}(pAS8_4S#}%}cUX4!St=9EdMW*ygPG<oWpvvtR75OSPz*^YBMuo4CHj
zJp1$?UpgP2mv6hj|D@#B#jx{{k~*0s9r&Dne(!IbI<;~B{Ko0xRa%Sp*fg)vX>MRB
zd9IdH-LXKSd4U46y}i;!2Zo79IoCz}`7K^0bm9O13rcT)7Po1KORS4&T-N#ey5u}t
zP$l*7>+6H_L<}GSe+9Z6OMaV>$g4;H>*KeJiLkS8Y~H@Dx_`~hzLctRd$Z&1ea!D~
zmhVw_3sOq|c<r=vn%1$X<TgpgjMF|U&zOET8j3_$#NOGyt7>lY?H6-DUs;~f8>Q(Q
zo_OxM#De%2;GUz@yiR6e10$!)jLKi4QrN`A*v!l{n3b3;C;t6i{-QJBMEdJP`Fno<
zk=zyd{<?mfef<L0>(lj}tpy)Ivz3%)XEV=%j*Y8&8yTNQEKB5v6e$D`lVWDr?eejJ
SmEWitq`}kG&t;ucLK6V<ok-6B

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..2c346e0ce5487e21d41aa4e6306fd83a7b4ffdb4
GIT binary patch
literal 17500
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjQTB9k45^rtlpw*nn1P*v
zfq~)we<{|*4C<pyI)}rN{)4;+9C!{X>`eLJDEwk^kok&p`cK54y=As{TBNmmvDU87
zGV_FIPCThq=B}osk$BzAgMm$_^4RWsS9{X8yq2sDI<$M~?TPiDz;SqG(nhWp0|D0Z
z-il)aEgox*{Qtj6M0$f@(~8~Itw$TTDtxavdgbt+a+zZf3`{=1S$4ATvHCH!7OR$B
z@3r6eexF)=qclTzi>{JZ!wM~i7(s?RKgzF4A7)jG>Uws@#p#G<p_jVg^E0Q5)@e^y
zx~=wG@A$$q>t|M`#DguDV(M&eZjN+madv13_UMR?nd2Tf(K~FKbJVO*pPtBo2@_|}
zZft&RudU5&{JH*5;J^R&yA+g_^`_2<`j;5Y>6`Wc|9|IeKc1ieZ=V0}-rnEezrEdm
zef|HP#n0bgUS4ls`|J1L-}~p=|NsBb*5IHA8#JkwbT-eO8|l#C95Tf{Xi~hV=ZO;@
z4zY86otir5|F^fcW){9|pEL1PgVa-=86SI>u|>-4oN!<%_cAui&OgqVIT(3({s#!(
zv}bGk|NsC0-~a#rU+(|^-Mzo%&(6%hy!`*Y9UtGny<LBQ|Nq|~n;)K+-_>qZ2(|df
zk%K2r95{ViP3h3;1qvw%3h&<NtXp+TN#n$UQ(T@b6_2EY=P+98DQ0ryact9K-o4S{
z%3<FB?Ppii?{NM1w|SfIKgG=d-~a#r|NQ)adFyZG_xJ4&U-zf__qYGc{dw%`cpO9?
zfO0G-XKpxh@Z?D)y>rW#sbytrzka2)Y{7v;Cp$IXDt`R+-`=Lyz`)>7dP<Af;~5`&
zcZ(dH&op7_h7cJEUH*zk4#AysVkQK0Hu3rkZ)kcVzv)-~|9`bVKD>W)^ndiWKi0Lg
z9^SQX`~F_C?(d@fDG4CIG5r6(sQl4`=g$KI1PslmfBibu)bQccXKqIX(oz|{RW&9a
zIdP&vM)E{dfw~minnx!z`E{)%CuGT7dG%P{Z1F$E%*Jr@{{g2ZrX)2p?!EMr{qm3Z
z|NsB@pKl~%b7Die{^6&mdDh1p*Zg*S9?=B#;i6!$ePLq71p!?A;US@^>4}LAuEte8
zNzY2x1omllERtKf=iodJ18-KFjtNFnZm}4fE-;JwuXx?`geQw;>YwJRCOrTD8P@zb
zQ1DUd@V~!2%l(aG_W8Wu!~#y6i*g@5c#tq{vT$K(C^u)|)$7mGQd1pH{rzuv^Twlj
zjg1fGwZ!FAC;WOUzi<;{B=<U%ul%zWq8>SzGitt07Ig7r@pv0qQS`GtrbuCB%W{4;
zHc)z5_~Nv9+w1F+d~zu_c1?YF&(_M+7Zia1|1a`<l#r4lApH2^#fklW*NO`#3J5gx
z7%GU17O*L;3fTF7!!qdz-R67gi775Bf@k*dZJzsN6Qg_eWJX_s#im`;q$XqwgA(MT
zY1$m7A3ia!+|yXW*;@EP;qarQJj?wq?ui8j-2eZJiXJ5-CQh38aQnuKr_Py$g+5J4
zPj`50WMGhBqVD-fi8Cp)?7$?BD9MJ+XAb!EoyaoC=DNeCB*fx&m{oUQ;3*ys>9}17
zR?9_PJanL8mfeJ#=J`fC5eGIDs~=t+&SN5F%?Js-s7DFO$zef4Wo6g=y)RFnoSdAR
zns8vQ+5=Z6W#vUr`vsr=mz11ny;P&;|Ns9F_4zj_O%i#$GT;Ks#8sRrkw-Y#-);HC
zzw%yVxW<X2oXpJ3UYi`A{rh{m;IrD{W4%1DuNm4@IINxF0t!4(>T`OOkeWJas_2(*
zW?o)KVF3xLsi`eTZJAYM7fk#Ag4gYi>rwljIcFPYHUyL|oGx=>nc#e#2d_GFdW{y8
zId4#vI#Z{!=iyKOP#N8+4$P9_KMx)_@Zr1u;lu4budf^D+&WV5n2lL<c{Rwo|Nj@6
z6moDE|NikpRsD8WcJb!T#!Bj}ZYyq{n0UNln$!g)e&zJV%lUI6<yX7uaI(5f9a=9K
z`P*J5m2;k%W>mIF+}8!_Ok0{*R{lTKFvo5~m`TCM>+2=ed=hSK>wWmt`&G{_NK`4=
z7k>D(DKbh+Mcvxk!sF8i17oF`Ew`Se9PnU&ku2&_c<jJ|5Bw|F{bbkHl9(XGSY#}a
zXEBkLSuS;ktQgZ)PNC_Ha>rL_vhA$?Bp=CJv?(Fs2Y=?$q%WC@yb*6Qne><v|NLeb
zwXd3zcuJ)0@o~w0_3GAn3n5-qGA``s_#h&_yS?pIXZL1fV`pB+)xHlBdshn`T;Vuj
z@@e}J%_$uR!y*NQUmsD9iQ2G?$3*Yimqyi{3aTCdCDsZ&Imd1COOiuX#NyYVnTZFP
z+EPzT$k`+;NJ*0bClDn|kmp$0Ua6{DN=bBV+Gu!i605R~?1H5{U%oVkZAdOKynMu-
z?G5W;dzruv`+pw30!Km|B7`-TDR^-<P7L_B@S5g{Qr>TTy8l8xeb+y{)SJgR-6-e2
z-#-Rzhy}WZ9$s7_VO4A_FS<K6D64X_EtztVp}VN!=_h`*4&(cWmjq}DK51!EHtOgU
zdNi47QPY9d65`V7O$QV%&QhN7_QL)X$!}-8WSGbFVY>W;g`1zAmQ?dis`xo+&Jjo&
zS28Qyu*s;h;>EM4J0qh`85tg&e6&8|k;A&+3Dp}uXt?`{{WqA!myybrrgVTOHRfXT
zwUPtzjuQgc2?TFmV7NhT;{O#ZpU5*;{T7MaxnMb`4_n%)hT{G8={L4^FWfov;X~(1
z!k~uM|Nm>kt@#=hcxp}cUZmaF(KUPV@`A@Y_j<e|+m}bQItt%+n6)ol;+JRU{f@SU
z-Gx!p3|r=XRhg82>}%oH>NZW*gT*-$ST`8XIsEyKGiyfIn^TpK;x6>QtO%MJsvKsM
zsK175RaN$%raOMq+qY?jgg!Z8$g8!r;=WZe$3vNClJ}BhCtlhvn!*lh@hQa=X=ry(
zomSP_B2ipuXl!6$WMt%&G-cvMM#~oyqojF0%xSq5*1#YA_wb9VKh9^r>^POUMpC4H
zYX546154YMxUm`L#Kax`qP*u|W7uf{Ju{AL`v3nM+ti&2h(2BL!{G4q^GDS9ph<9o
zMNf~2w)WpUcT6H8AD(Oy42v=_Qc$hjax-KJ8@o-U!gaL*vui=({UY;<q>ZyU)dZ6+
z3AA=T*5{M{pZG(NTRMBIQ;n9R^L%?_5iz4T_x*EzK}<drxpu=Qqno$0r%(ME5vgfl
zXz0XXQ|NG+St->a<I7DhH@6B_#>D^02?<Flf7n;^NlesYTA|}+`cb|)#o>|B0iTA=
zk^x(d7jzjU1h;V(T$NpNP4oZ9BF^mgDHlFyw6_>X9c^6q+`dSaHI!e{?CHbA6>C!y
z($a41p8oK%`k`yg;PiQDo<&7PL`>{gZtmKi-la2VEXZt{IFVCG_{Jsns?0{4@N*|P
zUVc=%#Lj#;e#sO@e}NA#(p|cWnm#i6i_I`(bf2LVAuPO1%!j#?kwx|o|KY?RUzC<}
zvh{>~dM|G!E^J_E_~-ZW14oW1=${XWI|nMv&duequNV3NY6kuP-!jvdtHFS$HJ+)p
zse3Yef!O+sCwEt{=|3-&>GtSt4hiBd&Gq)W|9jTIBMpmlYdId<_#eJtD11PYJ1nK3
zbNRa7iAL5d-FwgR&SG}%tNHm$vdlZ&S9%%a(U7ne%TKGkkzRQ2>}G!jrQdp*o|hk3
zs2rQ@v8TQGe)F0bmAx|^!a8?0JbUJ_d+}|?FSgTIZ?~2H{h7KS94=1z#T*<36%{Y0
zPGw!SlGD)G_;4_rN?D<Z+=rCL6_X}L&E26DzT-svjw|=~*O$NFSN-P3V(sua+lmj3
z)2I9T`-cQgijJIl{<L@F+_@8H&TO1Hld=4@=;l-E`z)KhTYj{0>mA!4zaP|~`F?J0
zytG-({oUW6?=JtpulD!%hllG=Wbp_JvibW@yAY5jF7C$1_e9|Eyxa-FY$;b{^!TP9
z)#7vi|Ig5->WF~!>4L{>`=Diklc$W0%^E+ysL)Wys;WsnJw7}<J|3A&lUxK;mK^T1
z`pMT@U6DLF;8a7`|9^j<A86bytS)C=_vhp9@Ao&BUH$R%=GLvPXU?jgI>lpQIrHa_
z2*ap<|2GyID$CWkvt|b!xZI#R$7NmI-<_G4uUCHFXH)Uv`F;C2o0pwA!y_ftW1P0(
z#tnt+<9r{??F*N3rhNIV4r<Ww^753wf8-+rN%c-Dd|_c~-rifMO$$g)j(l>9Wt!20
zn6!wkOqw<mj<5N*urSb=aoUuR@9*zt=f7uS{yuv1vs2UcUDjp^iKUenPd<D8dT{7d
zVWFg>(~cT0miKLG+;NC)x|q#L?xh~PtG{3W_}E@b>Yjbom-Z81Efy`}k(BgFP35t%
zu@DmEdEn2!<H$tr{`~tAY3EWRHZ(l^S^Opp?k-*3ne*mpL`SEJiMjO{F>8xon(E3T
zsT29f_0d1)SEmy-jq~o;#m4MOJw5I9j$8L47>$jU&CR`oCb<=D+SSFy&(5y=?n=XI
zl><E?Y)V|rO{{FpEg!$X-+y{~{h8U{Gq0^VzrXIU`pNP+b9f>nd3bqwY;0IU7OAp&
z%xszIwIR$<;NboJl19doW$%yKwQ_@#Vbi%enp#~~uKkLR*50(iz`$Yl@tzA8dE8vA
zIh3#cKNQ8I5qR!I$4(WAb8Zda-|yFtC`h}$E?T8kmYMnR{r&&#Yk%!cICyr(jK9&_
z?S6e`ue4-kp5&UbZShg1S-<L6bh24G8YIm)+>j+Whuc^3{QUnb*Vk{a`g(o7y{ES%
z4-e1%{l;rzPj4txewb=KVMXI{K4D=e7T+$xy}6vCDQRgd)}&0Bcrecn?9`;BgoFuG
zMOUo&P*{4kVJ%;KwWzW6A{B18WA+|7Tf`37_dG~*;QMG_|Ie=GN8YC=$@cY=ELZ+{
zczAuc_<g@Q6{UyU+WqI(9gg*T;KZ{c%AmG)<~)fBE*JK6dK84O{};38$K`pp_v7~{
zT+Zf~GswB+awUf&O-TWiH9NYEj~qAd=uS2SC9?njlZ1pHdwCa^6<>b(sOkKE;{|Ce
z|MR?<%XenVM{(1s3EP}6dMAj99pjZ=S5P2ey0e*mj?K@iUteDDF0*aqOz>riILfIe
zo6<Xjb1vibhsXQ=-q`l`_V#?0X%Z_EKBuv<vCUDAJ;9Kz=$O)#n2_+`Vd94`DF#M|
zf_6btQqrgI=HI_u5f@HQc93OipTZ(x$m_}eKVj?1c#|V1e(-Ny=crvOaOe8^{TntJ
z?XCMeb&^0zLc)))ukW{VzYkkqSMmOydXA`Mlx*W_&ZgA|ezc$U+0xRKcW$P!ecrt#
zC8rx27!J4bMkw)!JFK%dko-}4@{|z^i-Csb!qt$98svbG@aO3*>;ASZV_VW{b|R3w
z|F^wCi-wtm<in@O?XO&E4twHobI~H1@9+0Nex9ywx2K?yU%u|`{r^eF`<6!pEB3iG
zNPf)V<vSy$9VOa4EnorHJiEWO|Nflkm*?0j&Qaaf*w}dE>Y<cy?GINTyf~qt-X0eK
zX^tdq+Bh*a?ce!^h_)~D8yPn?GOgU=pr<wAG>?mxqwD_wVXi2xpZ}kqx7XF(=s%x>
z_n{EuzrV=|X=ye!Kd#QVuQz-&i7`K(*;sRdD)*mOmT5&#UtYF9Kkx6-8qOcz-}5Y5
zr1B=`aQw~$Q5Qd@J$raouH@n3(ny0eRg;oZ|MIVP|F@VoPVuYPg-1X5b#^BBiq&si
zs<<WS$@~5F^*%l-M~>{!<aK2OfwZ(^(cA6jTYO}5&-}>W?3gmK!Iby!;e-FqANcV8
z>}>m*njRmYDZe}-W;jO7XzcE0(cNIlt|4w5`Cw9NLPA1Xnt@T0fqD9aCoU2kkUZdY
zSE^;2TEHWTc?u8C`d5TD1#VfODy!po^4PHh$B(~ywaUfsL}J7F`TtLxD%xB7oB6QP
znxAr;-40x7-TL8xPO`*>NqowW>qSMyPHkWC@`b^NPm`9hW*%YO`@3tuC{v)0%8>_8
zjwmQL3P2;;>D<{9);berOJz*_``?~ReCO(>iL3r4N;GM>Ub)&TEBn;N)$^q6j|26t
zuIuFO|0Rg9ReZAgC&dyTbU?&?hK~C2C2P+3>`c$em7Frw^IGd1JBbh1Oe3P@!oql*
zoH#B+)3cNB*9j_3L3Igb{t}Y^ldk9#eB$30(xmyLzwx2Gy87a&Q%hW0)L0WaSe=}5
z)~xCI^GD~+n;waaZF~P5fADB!+k;K@6K_Oht^WW2e`cnkx3_U;hrr#l2iSJ(l>GBo
zQbkphUqXi={326kWlNh;M~4F|G$kcD=v@BAzB(*m7T^E>&dK5{4;-*M)OhWm!!xDn
z@*1r>XGzsZ&g@*a%;00K*_sE(`|TSWC)wEeY)HEt-&4}CT<d4IV$<@9112mB<u2tF
z7bb4rp00dbzov#~<w_nlwv;UOP1Q|z3>&$N3Jo>1m4#FxQJCZqwdF*EQ44Et_!T9V
zs0j;H6&49b{rxn(^X2{?p7Q$rdzY%Z-c-@@3Uh9DkC?$4z3g~Y%Y&@sB;O-DPANW_
zCBE?3jTtA8wuX9lpE-5(bhn~vYh&xCi;jP0@@B^}GnSR{bah>5Gk{o@nDB!=*8bmo
z#(fK9@;LHN|1(~|H|rql#2qJ84bt8-Omx3{H?_4TA}g!!{q65h&d>kPqrzluov5#0
zsH|*vWljIo#ux*Ggk)vy<c6c>Vf+6aPW)8Q)>hxyxyFCqpPwHd-rt<AQqewhreoyH
z#9t+BjEq5t8Dt+wMY-x(D&*|&;`n~soQ<8mjhS6SLITtyV)*|*$!V3Z!Drs6WZ~ZK
zoBe;=PyJ|LyXC-^r}oT;<AsGMzP$XltLw|m86GPVA|fQ-zG>m+n)3JW(S&5<dc!mB
zryp^68?F-CBK1UnmCm(A9Mz16kA2&}u+dhnt@p<<c4=v6501mLOhY^*D_)5_IlBDA
zI{l*F!$*%USiq2522E~%{_SNn`^gu0sw0PSPVbQ){L@=ymoaSRc2hDgm@0Y0_{)6z
z`to-*duo2p{rXzI?zdaQ;vF3vv**uu@$TN$!Xo`HIl|zx`8pxiU}ffuS{pZjhR0rC
z|IaI3_x;<yeFcpNZf<F6;?dRRadgzM_4Hymc+{@qTBS|pp$oT8ZAj&Yw2PDe{M*~t
z*vL4~PD1kGoBtb@F!}NdG-x&LnIJLK#chVtk40RYs(;+tTFxzg@8;g$H?OXi|Nl4n
z$9MifFC!zPj?RtN*4@jNpT2#ALs6Gs!ASN&S9sH=3X#Qz6DFSkjRl^a_c#6Ir26o6
z3v!}UQ&JL=lLbUmJv<XVJQ5~MU+%YQflc9`|K|e&4{j*aMYxDf?DB<B?hC#g9ZMQa
zeH~afP7|B4k}+5&+hK9~wfFb`+f;x1dUyBy^mBV^|NYs$!(#U`wf<>h{SySVwY8a_
z&GVAE*twkNfc#$<%Sk&nJ&|WV{Qvv=|8MV~FZgM6c(FUroH;z-zVUQ*fzY8tSA4`y
zykxG^WBzwFDdSY#pM)FR`xh>0fo9G>|Mqfe{+c-XXuZbSAM(BerQXUm{>i?QJRXcK
zC)?P~4@x+7r~L_RTE5aFrDxLI9)rosMx7mIT)W>LVfd5E{4b>Fm58$=Bh+t=KUr84
z#3Xn+Oc=LHxh+!Vdt%b4_CsFt!Gy!ymexiI*@{L627ivKwa(nbVa8g(X(X_G?hKKh
zmEyK7FDDp1V9*sz*ebiPZ-U^qmyONMj&5yGS2ng<%P4HCKH&3{e>>Z)=YP*L%wFmo
z)-cV;fYp$BO5+rrZfzzF9h(p83R_%!YO<QE1-$PqR(J5@I?>9qOzO#%#mkN@Sa1NE
zFde<vS6Bt?(0Iu(&E=nRWRJ`&9uJkUBMocg9a4@R6@C4~l|xO0?TS!GV?fkF<<>9-
zUaN*_A_l4d|0XxQ;fGXFP19xmC*GbcA-N!xd6R+>Gt&fVnHeXW8X0Y0Y+S~}w#TvG
z{vGEjp-A@!UMw@h3<T0yH@$CDNk~cg@wxrrrAZKPHT6tzV!6o0%%-BJB$&`)u)xiR
z+3i76*8<PPYkw;iebzUN-66#IF_)DsXH$VQ$1bk3hR&i2+bwtOl!ST(RPK1PoY>eG
z6YmfuArbTBQtOW?2ODN>oN$@t<6PDzQ;us(oNHz`&v?=Pn{~6RQ43?y=EtwYL7g^+
z|Nos#JOYIonVF}})0)HR>2N~uOoPiKPs#tC4o6&sil!tSvr5ZS-Y4ypW#p8`@8(eO
z*GhTY;wA|Ri78W|4sd#R>R<b6wJB;Bf|!|uUL^SP&Umy#t3v5<Go$+^h1tF-n+_~Z
z`@rOOMfc|nhTKBM7#Gh@h~JYAevtc8;_UWfUcJjt#_T4A=R#*%KFP0F*u-&7PibQ1
zlo=_F_RBoihJ8?(zL2qCs(?U)=GS+!&~%oV@T1$LmwW$LrX(2^Uxn#nCno1+EOf3p
zbs(oqU~;fz!0rASf89)Cc`Q;+Y8(7ZHt}OG-Q*zsLn(;c=*Q2=Feh~Qt?=rZr!~df
zE4Z1_UE}d|r5RoZt{pwqMyJ?fI8#qB{PAS5d&ZdP(B`F>%42R0vE3;|J41QihR5=P
ziH*C=XEC~)7(Yl5o^RygrL^Rlsf$+HfwhGmOfjA;O6N{=T>8|IvrAAU1rlLSdBqDR
zXI#!P5Ylb>=ppDM%yUDDVYPys;G|n(N0tb+Dc;?dkZ|Dow<A+s3>dYwq4v#-{@H)!
zNlTZ4yM0tPbEx4#5r-ays2sxu*Eo12xC|aoO2|CepzQh3U(%U>!j*3>!Y$$|5c`_S
zQ~xji>Z{O{@sfERPqd8j0;7M48V9D96dYBWcysQXhdO^aneH8yuVVQBhSP;BSj_hc
zB6X<n{W!w4Eh7DZVxQ?F$qk{QGddWR)jqh0u8a8JzxalN8t+3-?Hz|pPcWVF<^1v6
zXn(yi#8xM5%YO^cE&tQ*nfP*FQinjKU&1l2kPr61YB&5iZZF}H*=zK`*SUG!mIW!M
z0(yIAyte!xE_db(4@7rTWBjhpI}Hu8C;1d7a#zNba_Q+YZ{u(X{Qv&`f9vwPnjatB
zotj+z`dB@=HXn4{kmC^6W^<x#`N^g?4-PWr-A&q2+<x%F1V^W)#va?7JHGu$)_Z(l
zvYmw`G|eVC-MYx)Sby<ce8c3Pp8pBYiw`WlP`+Vd){(92Ib9E4U48%4XZhtzY^<uk
zar<y>TFG~oG2+XjodVmUAIV?RTWo)0%YlmtKA>*Z@jgQrcjGg&Tqi0XDfclbmohLi
z+OPrCBmmXFhorWy=w#S*`n`PCg2hYaVix)<bpHAO|NX&c`SkO3zrVd*zf#kBl}^~=
z3qi~(O@^1x+V?zZz3Ta&E%8)BLc)ekNk4u~O}wtxr6nHIyrg2<&jpc?)}qrE%L}iV
zdc6Nkh|a2r68gb<@#6RQ|L^Vn{rB73cwSyJMMbwOQj9k>)|_>lBqcT@jW6!Np+gIn
z9Qp8C{P5r3JYFI)L5@#2tY%4Bh=Ut1PI{LA4xi#Y<9xqB^ku|nrFllodpoNB|NDMw
zs=S1R1Q!=eWQ0MnW!l+K;$c;Pj~4J8RAPU3_8^~&tK^)ylJjhnEvlz+ElX4rRC*o*
zjX@_aql(rhVa3=Pzu*6_Ke4ac<|Ie9iSnXX*VlvQm~QO*yLzc=dnc!~^zC|?uOc>%
z2af)KU?%x`-5JHnItmT_{Ra;<Jv=YpHd(!M*Ka01582xX_+dudO7p2Me9%?#<oA|B
z5l)M4t{qMP|NpnE{k3}6FHv^(<V3~&OH`Em=5E>Ha^Vu=GW`i#d1j_Cvd^2xla<Az
z9cExt<59Ha5#tf*qbE*4%xsdo$7CJBILl~)SI7Ppjy;n$CqCcxpi^kh(i6-5>(kEt
zDR_71{DTBNc6PI{us<ugHs?8M2O6C2SvX5a!NH|PQLF3W-|Dt+Wo`5AJv&00=8CN0
z4P684L^|;%xs*6r>3q2Caq-)R$Lni9h%sD9O1``6eDSlIkbs1hYoE58x^hl4uUK>7
zign0jRu%&zqn1{~7q7ux_G0zGC=cBPj#ZGjZ<_z6d22+QxXOu5KTa|C`1DMB`QOOk
z&)@(5?>{>mUsYvSP-NscM`OXNQ!LguAKI_ttnlJqQ+qvelFkDMmzIUQW<9*F-?q)H
zO<cd@lG7_G`9E6FqOeJw`-yVe!|hgw^3JH4nm_h2`Tzg_-F<(v9S`pIpa0L)?9<O*
zn~V(uGoBgvh!s3|dp02QWWh5Y=R!kfPjwNY`7Q?6Cme#9b*QT2TK$z@toLMovU|Jz
z4{$X&Ep(=F(|#VF|Ns8}E`M|5`=m+V`Q`ouhNS%a_V)h!`xR0PD}Kq&=q%l#%FXfP
zYj@)Ry0-fN5@G97GIEcWf3*0wjP2h+lNe1-aB^)bN&Ua*s?Lenu$~icE<8`R<;rYq
zjeGY0=jZ&RUHTIz=C6<c|M$QB;jP*AGiLmY-}#9<e_})F3nt%!3N>yHi~4yFujzuO
z&+7k!29Iv+p1z{mYy#uIrO#(*O@w;j<}(MYB!Q<DH9s_u|NsBr-HJ^<Qa|eZLwV-I
z`_Iq+zb@|Y)<usVKR#ZsD7L@;fBvs8>&^53)&2g)ALOL6@82)$J(F3PnGfII&ZDZ@
zA-pbrPXY&DTEsq?=WF|p^iO}R<uPIPjQ{`PrIEIQ!p81GJ+3s#zA(v$fuG;s-d?|U
z?fj&qdk2Hrn3<W4jSqK=|6dpTyZHaV`Rn63zt#O;w5&66o{ofs#J_*7aSK@dWp?f2
z*<Ww`<-2jrj)u?I3fFA(D|pMV7Yj+>PBIehB?fkTCOAx(5z1Ml&0_wm{^O(dD^_g3
zdw2f-zsU}Ex4wVRZ<DwGw?6Xo=VQJ0@%#V%{wZ9!N7VnPp;8bVGxOnIX`bk926uL~
zu5hoZk(f1G(yk_fgEKK=uav7{)MBCBv+{bWERfJ^n&-?V`O(F0tytK?1^zLKTignl
z_1M?{lePY~c+H!~({%0Myp244YDe84htBKz@e&r6lKpaji{IaqzoN!BZ;Iicga7~Y
z%(pX)*?sK8Bi2W8g`%Qtb#-kY3fkV}fa2{}Rhwn8MBM&k=^T26YubwwSRuYlYFz)c
zajoh9_9Jd3aS7h;{}nqI=<+*z7#jTf|Mm6%UtjNMXWRe%`Fa1=skd*;NIgAm{+l;E
zYu0q=Nh&KJ-dS8<St<GL?eFtv&ZMLrYsum`+@YnVz#vqqA7>D==S0CT6Q`K4X^yU~
zj-gW(jVCvDb}n4hl=v!y&CINAfdX4z-WOw)i##rD8Cn;w{a#QA4GRAG&R(of_}<!A
zEM&^ARyz0g_y7O$$CmIc>iPNOV}9Vp_Li3KLP9>l%RacdY$z@^HZV3mal$~c{M!Hj
z=?Q6RJ4#-@Kid6&p55Q__xI-K-*?N%WMrS<Dtdjr@t3cP%kJ#h2`YXsTu)C(NO<rd
z;l#<rh&?kN9_Md6<kYs$uFbxFgAVu6%Rd}~J3*xfs7`Xqs#6M=`7bFc7_OIOVmD#y
zg2lGgbvJJAjokNV_h#ehaPjTa#k+fW4sXx@|Kq1*pZx#kvW(aF_w&fe@SLCj-@5+a
z``6e1@2mg+{j++}E1Av*msHzitpna&{_(k8QLW9<h2_@2gtWAR($o`YQw2m*GqRF8
zx|4TQ2;LRga#F;n;ilJ7AAd-EIawe4VPAB!VG7@cs?8dv95H|X{@yRFUgzoe@57gu
z_m`;j-P~NMqccNN;-7uppXvSY|1Wm0zk4U5?w?Dcrn2(k&(C@M=l|PZ`}_Yy<^OT}
z{(OIYygp)c`H{x@{}S`;({~iLK0L_W7Vk0rh^MFFi4z8Yel~a5*VVOkaI{?sVN+CW
zD}60->9XWBy`!6^a5EW5uX5QI1c|n!#!zh@9*ypO3wk-PtggCn^TdlsN8>lA)kg3B
z#?E~BzvtwNMT>4IsMY`c<R3Th;h&%TSFQ5-{r!KE^Il(uN1OQNCAQ`M)>k@I@%h<*
z`+A8dPbGKk)U=hXJJI<6zhvFtq>67n56{WARexVlzQ?hlhv)G5`8;#xXas7g90`ay
zqo8ysAoOU#Un}Q){ljyuc`jb;n40_{LOm_kFbtm5x2i}OB+E`q`j@zbhwX{ji|zUU
z`DA`rrk~q>cX$2G+nY<ui?g$goR|Fj`}_Ravjy+(#pmU<DZBsc=r+E$yZvaayuFc!
zr{SF)ALqBWI=Zwxylc%?R@Sz^Uh>Au0_z787<+nn?(Z|)Q+aqpt#wI=2-A)o694`r
zUbve2;8@w?$QqmGIWo@4%7;6JFKn4Q=}<xni|K?JpTzxzMZraM(xEk`6D~2<BrvZ%
z<G|x)Tm3ElXxIC*XXDLse?`We*;u5yewmu^9>dcoe!PEsd;gj>|Mg;iDC?g6@aLyJ
zBcl#zPXF+1a~^ShBb(Y&1s@a+UtJAavGM1J<GDvm7joOxrBr-qNc>&a_Wk|E?aC`w
z8y6HjkdfHBEp3LyL`8*$hu?DB_Et-(`8e!cD`9!fuz!lnYTpZHcOk9mGjnwLnjJ)1
z>^%f$^Ex^>Sum`7{y?naT&rWE$A>wLosMgnd=+(fi!R~a61w)~oChm^o;-c#<>a6Z
z-b&S}iT)C=vv$3U5}9r@f9VpPyDEh(4<t*>gE*Ojudkl+Wv^IM;)8Ey75jg#IlEM!
zJF(;1obx`yfi2f=eDk-RZ{Kcs#`v+v?L{J9vT_<u4|cJ0e_}eW*mu&XX8x*0nJcs|
ziJbraOh_(!^0cF}p!v}|bFI0W0|Z>pPu38W7Cd<DnCOk0H+NfEiznpjN9<~24|{hY
zNbZMYe3yiQ{iWseiu8*4izT0Z@|iwo=9G4?dwdrIvZfd5a)pKlPm8R4*RpVfPHkMH
zPIsA%?$d4WcfB`HkO-JRkB9r9jd9-^?S1~2C#XoU1+qM-%sBh?*Zf~Qz9-KL=vucp
zH-2&DWf#q+2dhtVHRj5PJ=%4}G}TjP@-p|Y>cw-^Pt6R;1IPNExrSWL4k9kc@2=>4
zac!HKlH7sdn-x6yj}+hN94U)d+}pZlkyzKeWgZ>|8GI8g4;$G0`!MtLKhDDPmsY2H
zwoOg7T>kRNPpRYERd()L=l1fPq?hL50N>5Mi}&2$zRx{&btaP{``$;Fmj^j{74V&B
zPL^QfV7%@s(IjBNcKVRbVUzgCPx6;cN@QCc0vD`ucv+smN#{dtKv(Hs)|Y+F8cpjC
zZ=3l1gb~ZZ^tHV1HC1kF@@>J+zB1F0ugO8cb-tob22bnOEAtw!SyyO8^*xN_U}eAH
zlBD6I&~PB{k)D{;;zb*`&CPmt{><qo=X+&rVi!E)Hs)FIb;9%MZ^PUI)maiPZs#0H
zGW=-~<eKQGJ~{PSh0M3#3o`|p_WfTqF({+gaL)N7hN+&@1X*PG)`Z<KJFR+Iv-W&V
z-xP~;v0kTlP5%`ivr{$Zx(To6p*;87L81C<H|5sIYX-jhZns0vvElK9$`aYkQ&oTe
zWyt<sn7sP2;7%q`DmgRPmb<Aypnd+8gW@eEg}I{Bm#F^As_jbK-z0xDkp2DvO^xP`
z1rCSmo4drW%G@ctQ?&ni-Qk0Uk8@wSJeb&XFg!}YS9#T|jkoVMJ=hkfeS7V`n<ti?
zy!~iT_vxj!^KTzsc(!=s{q9X7t4_WAa??`c%;MXfOEtS$16y7TwN=O!p2>2Z^uXe8
z{^_Ta7A4pn-?lQFhxzi8D%Z;sp30fJO746)<%fKg72~_|G~-i~9#6Dbrjapg`ah4L
z{?^1lP03b&_w}!voEmYh`c?EvbM8v8kG9O&%hePha=czFD%@xmQ(<->M~y`MQuP;e
zXEkSJW(7paHZ2I~RNAHRB=>_*t!MD<hXodgAD;hd_x$7UKYQa$d1}k!jhBWL#`rCt
zesocfQnry`;*D1&KN?(@PT3WA^<%_bztBl57Oi*`antO!?y{Q}*>&5(d6g9-^gh<6
z88<bYWjnw4cB;XF(wXNadrjsYOXg!g6*T?FZ`-|cjt+qnQYF|9E3WIF@?__^pT5Gk
zj}_LLg#3Hbo3%CQQr_mc&9^s3O<Z<A+)#)6xTBYB&}<Pq;eGuAck<?(|MFXZQvAkC
zPn_O?Q{I%hySokr2(;}#a&(4-$PsO&j@G5;r@DWPWYLz^mRD$9+S=PXK~2DPhD86f
zIl7Y;w53kZt!HC<_UXKVu|d46%%1ZH?JS%%4+X3XJ6R*g+E^lUxv=M?i~g)0{#zcw
zUn?~3PY=?%K6QF`QS8#!KkPEQg6idsCgy+se5lYOA#%-ipXG1LN_Jm)Ez^G{EypZm
zH4mGPSccSN!_%tQwGS({ISLf)znphDLxJVFu~We6i!Yx<t`O}%yQ#;-*GKsFvYSh;
zF1o(>{_~$IeV#%kvbXE@e)<=FVM>c}pUvC}lm5z`Pu_j>^Nzb6k2lq7iRyG~ICb1#
zW;g5GrAa?4Oq6(vAL(>X75Kq;W<owREH-u>au8_Se|pxF4k;DEB$wH%Yn1OhZ20D?
zQ!C&dot>?%E0iPTa#GbP;(J%DQ^4%oS!Jsl=coRwPn-60(x>Gw*S~U=d>f*)^wj6)
zX~FBR?=Q^J6}$g=r?J$>dF~EZw=zv#=6^o;_S!Q>TW_zld0wP=x}m=3#H$}BIbIJd
zZRFfuTl4MnpJy!bVAm4O$A;T{Cq4Mh!uZ|#y7u9P3)MWEQU%+-RFzm=F?BuoHtop5
z*HZ5;y-m6uA?9@9y)cW<@*{?mIRjhXOWR4bok`<Job@uw^Q9@X)DN|1WoI3Jzm`(e
z@Ltum>QT)yjT`SRWh~}e$k{(oVF^5vb+oAS@t&r;cD>g!q=RO$e|VJ7E&TP{(_K6N
zF@o}T&n!c(LjgQ(`!6j|&}CJB8DbS89?qVwuKz%HPmB6TrM$gvnj&8EyEHVWKKg9=
zC2+~othNb?x8*14D^3eaysxoRD^?_H|9)A%r&+3HyDz3G+pJ%26z4P{HOt^ZMM&kk
z#}V_-dt}~wwLj<8{|H`f-ZSQn0?!IQc^d0(`pNWU>$GX99!IyG^<6r%Fv@tgVA~J7
zb*Dd{OmPx1oF6<*!&N|l#p7hbi4w296MK)}dA~z1^yuTp#|1kdRL=2h+p$!|_oUII
z8Ly|R7@pmf>a-zd)7?#Rn<M^Iyw6^Fn|Ifz?@zD%;{|zg%8Y|tO%4KW_8MZW2ff8F
zPj+Nqf48db-cs`qAD#FG%g%T|S!p$8!YtO^vR$rTt<Ow&j5nPAbLP)y{>(^Kt}B1_
zCO?sE+kW-c#)yummu8%2Z#yV{_=16uQ|in$TB65`_J`kIYc=0;u7%vg`7<x3zE8_O
zwMbTXrRl{FrgN%O!sgy|ZY$f}Kkx0X8)iX8I|W#Lmd{WNdi2{z?TO{Iqd9KV6j|oE
z&oYt<3>9JRa~64MvF195<D`ITIc8T(+nFyk@f<KYS-dX%*yD50FGgsuwSQ|FG3)96
ziIB0CJ2UojH6@6&$=5I1>&cZV(2_5(ha)_Hq4W!@T}t}O8AX#FJRF4`+Yc=~cYT7(
z1efCXRo4!Gp7U8s>Gk%r0vDo+_j*m@*?o6u=DR|h`+3hD6QacCyB|BPmaFZ$@O03(
zYD>A3Mz^NzdR8Vae`Wv4VCTaP99|#S)J{EA`m$Pv{ruwB5jqiiE49M3&!y)`-+oqd
z$mC>kgNbW4U;9Mu$%Z_KCY){4IP`ty*}A6VODtw29sYii85ART=GbyII|#JyH%Vg_
z(h=WysXMZAR?LCj9;<AfSx(<*IikxFn0VD!+0b5%{X}`gO1aMo&ptk5d)#hx?4sG(
zDa+nIyzuV3%1OPxxdwMOq^TyfAN=+=rAN1G<$Lv|Igv$Hf!9MskDXb*w*Jx2;}f6f
zZ09)mvE#X-s+S;($H}KoB8n`LZSRZilvSgWd2&yG^}1fT`=P~^uStFRJM^3ue7&Um
zr}zF$J-ctmU$MlwmVuX&-I)z)d<eM8b0~6TYhL@-RQ{1i?x=8ul)r+QSafcB_QDmu
zVPYrqx^yy=6qcNcNtCO(SMx0N=5Js9(|cFtm9G74qj5SrFy_&WV4XFk=XU09PJD6e
znp}IXlzE@W$$gvG{rmg<{D}(rWP@Yn?Vo!DJWd)Yu+;ibTWIPf=vpG1*%hS1dD%ta
z!5sV2Ig=PO-k$yQ^0xhoujW2AT(dx(umAs_%!0UZ{-!1cv98|b8^0LD9mtN1z8M{P
z^JYZG=1n;<K}T|=1UcFGC#<Zq<T?D|fAW!=b$yRl>&S<GJ!7<9<ByGa;)(v|k79@X
zCif;sU!NVcE%S}d<?q(djMK_r)(0K0*?O#WqR`S0HoN*%9J6cY=Pu6eRBCHfa5yFX
zf!TsD^VF_QIueJko1aN#p0oMIUB8$`8=ULG#cxvM6;1oPKh~wM!fXFd|NQ)aMdik)
z&x#_Vjgxx{|2CT&MMVC$um0BG+M4+L+y3+O{@Pc5`hR*l&-eGnd#aCr_^MsfQ&X^v
zr)jo@2K$F?#ww5Q?@3HZYg;C&rq;GSUos*>!mcg_H2d+j`@kx0J-rvNSi{0<c=^~C
z-CmM$SaF6YS0rSebkfiN|L<?l|8Ji6=i;qjU*F!|uc(-F^QO$NUlz0HPv|?jIUO|A
zSO56f{@LdL<M;iMH@f)e5$oak_Df{d9IYE{nvb41ap3sz1ILbicrSnW{QL`g8n%on
zdmetz&p6_sC3xn=L6-S;h8h|M8#Wl!{5}R66McW5XU&?HwTJC5wQdzUxC*jhXV2`7
zT+IPIuJ4=G-xdqZiT!JE@cu{sKdSYP?Bb%bES#){8=Dv2*tgJWVL5+_MazWh<ms-b
zJ_U+Ak=gWS-t+AyrpLTn_nt_&_dD)ZnLOJ{txTyarpq{E*#+-9{9pe$QMpfX5sOOa
zth;AsaT%~Bc4#yeb1+443$9u1<lItu%wpePf0@1){kxi))*qHvWHR3NmRnN3`JtE0
zh5VN>Us*tTG$}LpuT|!ya8c1`M_l?kIKnhF=g3aJzyE*z&rk2)-~T^d|NrlAZ}0#6
z`+vS&>W;$ZhnLmc%=6h6oaXw@mZErfeLT;mO*~9YJofcQd#a8uSbu)OnllP2EV5Hl
z5*zt?xj76zd@|aw(J^^$W8=f;{%!L13l>kauQRCmdFaDC*_4~cC2Bq0+*Vt-?kBmJ
z^E*9DR@4-=^YY%7nNiBMjLAe3(#hWwV-4C5Qp+N~g~LtIW!vVj4ELOD+R8urY>PYM
z)zLbkL*>}bLsL#a+O**^-(_35J^aF7_O5%q;Ps(pl`5R~4jr;oQqnwcu%yst`&${d
zT}?-7kMH^yJtHRXS)l`mi{KWv=PR`CGd+~a?cUTPU~o)*Z)BT|d!Rz!LXjz--YqZQ
zAa<;xfH{FDDtVrw>avs9YkNQ5JGQCj`@y%<6`Qtt1RFfN`u=m}_iuLHH4%Hk8DY&V
zTabfW<tM3j+qd>EHEW8fZkcoZabz`bcS7RA)-tYxN*9xyx_EczJWQT7(^F7_<6-UL
zmWr)!qdCM}e|$=qp>V+FwTbKXm$mzU{fi3s=zP54{(1ZINO{4F`bz_*J&?KGw#+eB
zxi67t6HjRB-rPmsTnY~hF!VHDn|4m6bFZ0bY(SLA0~xPG8AF~4LYG@UyUa_DT$A`p
z|Cx=RlWR|;%J1g26YeO~IRAY-Bfezi>w92l@0qn1RA9FpKV%&sl2hBcSy-A|K1N5U
zhCy1|Rfl!!1mTZ$K@}{DYAc0q<gIGuFS{YRTH}vJ-QVunH_Kf)cKO{h`xMi+SkK$m
zSAtEad*|hPg~v0?Sr}Uy1QK|rG!;9yJwC8WpshnmIN6Kyy6MyY*PnhRZ+!ek{&?aU
zWyJ*|pL=$>U0(2hV{SzF7Tr}`UX20;|Fm8FL?oQ2Df&)S`LIX7_fNU}rRbMtaXY<0
zL;3&z@0n%K-IO44q&{oCl`2PLvjxXa(Q|oqKeqj1XzYH>&864I^=pfW=3nWiydC}p
zcRt#buh!W-<&meDu*dbz#~y(`x!T7chfFK7@(f*QT6)&vQtAJIw$!`-njH@qJox>l
z%z|&5@2pMtm79WG1U-wqGtO5VJ~8iM-`|}1Xx9$CJHHh!r)*X{-<f<aeao`<H}g21
z`PlWuYvk9)n=9Kpv7Ardd(0r9Ygy!(e|8VDXPxm)s(ZR=%Vfx6@GY|qxmp}}T<^c&
z5)*4u=s4O?!ML~nt*DGNx8p%S>E)7hb>}#BvnH1RXLWj@{_wxbdT!&qRiW9t-<L`B
zWp46%o14A6uP`NYM!L=U+{piDdlU{x)XcYW&t6h;rXzV(*RHf!PUo&ey`qX8i{`gJ
zo3voc63yE8XWnd5Ih`TZqv#pXmAiXyXMN{od)xCheu8yxo92I55+-kD>jYYk`~Ux*
z*_PbR0Rk=dDitM7WpZ=HxXk|jzjiKYgNR6U_wq-K`$Q^PngxP9mfcI;om!UmtuWMR
z$&}Mdtrl&L*S=mXb&ou?ZE5C=^snFc=l)la+;whW@~j_rmt7`JNR1HdbDk%B{M1{X
zrv7Qwh4)ORxz4)uox@S$Q6;GLw=3zk2<xo2M1wwao~c2RRe>#;t5U8{H@LwaF7@$o
zYPHz2`>CMeKTzx&a<v$Uu(HQ^2y(jyOMKy5X|sRn1Hm^sqPaPXSdu5V-erx{j{m>w
z=lt+9uiJT^H=Rmd`+irlO#im8mtF2ZmN_jX_K!J*<KRBw<5N^$cNt1FPtkCV4Hn6f
z_Nu*m+1P8>wS%IkiZ4ykNtRflwPZ==1R=#4&%L9+`TdC&yfjNiu}gp{JMfkN$H@)9
z<sWH*_8tEJ@8l^n!EoL2{{MzX|L*VqfB*LO|9vt)|GyMI{Pi`Dj}MQ$z0>lDm(z@X
z{8(`Mu8*%|Pme@JrKF3iq>ip63yVbE|1<%ywr`Fv_<#IdDE;ZmRmmAMCH3OdethY4
zjG5Ee+xzfM<cnoG9<}ae(JY(%*)N@V@GR}e@98(#*!3E&Z9Cv&HDe`c*W~~IM`qh{
zwIuMk%vbthx+tP%X=??mV1HirW7RDmnf@tcC~xH4wfk(7!ZVJ;2AusTQ~Pp-1z+yE
zTexCXt<2JOsj->SE;E+tzxe%P&N&aEXEPTmoaba(vs5Q^rA6J(%eN#%RZbVg{+efU
zb=M~Yfi@G~ef`hx-ZVOvp<flbVVXt|QyFv9-_&^f<1f>{LDpe5srBXG|7U0j8Zi8R
zef|IHZ-1))|NDQi`Tzevi4|WscHI%bv~{t+_2cXMY;kdHmX>Yj=S%A7NSrw%@#BZY
zks}go)+`8J-@xE2);gIhB4WY9l}lJ+Ltni)bKuOGf`?3p*T?hd=zxa%K74j&zZntP
z8KuD)&&w$}SyVK2!nCx6#6(a){S<zBfiI?ENyef3N|1rtBXc16Q2ud~(bU$p993+#
z4eHmmBPyFWZYm5An3-dk*sYs#Gs#@t<D<g-`SWcY_V6D&VbYpAIg#VF)?*DN9q!`?
zOII@eXOs+l8|@wW$Trve`o!m6Zqo!0RbTtJ|7+5gRRw&>FMj{f2wdJ!Y{L85XV$96
zmC18mZ>>qazcQOwL5<V(U}atL@2bn6)w8EVQc=>+KR@rUkN=;4@6Y#VXa65=|G%&9
z@BcHBpel6o`~Uv}6VG*eFhuemJ$Ft){rHEk+RYOMn`cTkFHmS+q0u~5lv!N-@csQE
zua{k7>gWJ9QWO*pB%~cH_^MUH^Y%sHi$zb{`6Z7Wk(g)Kx~!r8|AK$5DUm;}*?O$z
z+cZORlfS!1hoetVV}F0+RMDmWjobof4p)dQm?{sckk8D4R224ob6PEJK1exk*eYoK
z@vn9KfiM9U-M^0O0}NZ8f?7-(Csej1&i+|eo_Dw)<mx>Wk)!uY-6M^BuYcSpuzz;X
zWm&}zrD?4us;58H_9&j;dH?3N@2sN#l{NLZ%vV{X{kY?r40~y>;Thv~-BY&c&U3&2
z*dqL$_4H$2iMmnAx``cASQxzok6Z8^OL+Ua@L6-5@XP5p9)o+lXXY4kwL0*)$o~*b
zR^HUw+dB0H<F|k28>X}7+BDpXc=<2sa%YmVg*ykcq?++h%k9?BcfXsVX3ukrb=LFK
zA&;dwy02BQ>dIZ5yHPCpMDdzX-EW0yEJrr-t<98h-}|>~#j_XHT~2$A1h0uEn?&x=
zaS@Dc<4E<K^rCw4E;Zk+8$P=fu-tJmx7fMJ{LN2rU+Bv0)m#k@0<Qi7iG~s(26}dN
z4hBE}-&Ha*)a|g}pta)QL6xix6`xx=i)R$i-F0tP+S^@cCE6ShJP~ewn0iihYuYRi
zp;MDI4t>|;YI0aNVgHi_oJ%{B*DPH%BmG$57NP9Gu5GG3FRL>z>T*XV^F*rYvRqg7
z{b=(^M&RRvkFU#~Tza~becI{D1aLR!%<Q#X4F)2u_KrE+w*_RlvQKO8`)@qsn(O9J
zo^wT$S0o6YH<mfa|NLH&#J4G*HW{23+G#H#?SEqCy5!p#(ZA*`bJR>aVrjYjhqHMl
zck+>0$ttQ_ragIR8+vrtBGbz)-+Pp=EfTrrWzNa?d`0%p3hgfOO$V<g9kSJxTTu*d
zXk3|N$km`A(Q3cJp@1Vtcw$@F@m>Ep_cXU(5L?J5+q7cag7T*Nio@q0*RJPIJU>%n
z;-;y|`%B#Rt_fQC_{63J9<PVCm#@6I_TR5C#OYjA(K3;1qMFi&V=I>@E_i=&))a%b
zDFuDGhpSr><~2_1nm2iM`rdz%;rH*ngj;?<L4c)wLIaEE;#M|o?UT%Z{^#?s1aYox
z5xK|hGqv_$+39;a=GV27Uqso**05js8>#(QH{B^A){-wssPDt;`AxF(7!8BCqAp8)
z@7=0#bXy|N^;W|<>c7u@elVvbd~KxIp7dEPdw7p~U3b{A2W)FnN<u@~nwL8h*!fsk
z7Id$Ym6bFxkziqw`0`b9&05L2KM4uR&3<$F`9D16-jZDOn2lLo9W*;7Z)Z?YU|>^o
zLP7QDhc}YU($dWh3?+A8&$kDyM6j<jIC08Je4%3>s{-4CtI90XOCAV9dWuOcbEX<A
zXg1HD-8gyj!}IcN($Z|XxonJ#Y_YLy;o)uX?=MVlDq%RWZ&~SSwoQj09A<Bmx0h&X
zkysa#_~3DJ#s7H^zvsW0q*Pnm_WQf!k)x9T{-+Cww5*G5)oE1Qly`KF_Ce9V5lb)J
zfD{Q*8>Y!wTbE2ba-zf0zu(cJ!O^?>;9XfOC6}Pz-4YF@KRCku``s9VQjHA_Ykn}?
zmH+*iy?Kes;kUPW{`}#YZ*Sbu>3Dsi<0=V{9|zc9cCzg7;xpnBf|@<!n4GY1$&ujL
zxsJZPx7Kdo)WjxV8qD@@oz2r%o$Md(@JH^`Qag6w^l8xEBF&QrjvZUD?%aVBCloX|
zzS^byw_9@3b?O(dGfoyJkuVdF96ZgvHpbL+VRq%2vyz~hssHH$0t;s8p1A0fve9vg
z&P|3%b_%&1Y;0_Hb{V`(mb;`bWw!fh3*A3)fcb$dpJ66bFfTY}r5cVLJbn7Zd->)D
zhUPv#W^wW67MA8t&Jtc0(dB*``92r62(rDba`+ZuVBp~X^TtiV_LBwyd&)Q~WlgVL
z<ZC(zSqr$R{8596ab%?Am#+&C+qt>%Xle1(|1&5k3d)@*zpkRyeCq^eZhu>+b~h`P
zsSeYw8yf^HlymeJII?B+hE*vk(4}FEvL7{w%(JgIl8`jy;4=E~(eTAfLkUU8*{?%h
z&PzCEvM0S?(kCOgdd)+R?S3**Ef-hB_T=cy6BO*w0efaq{G*0B<=_5VgFGPO5k33h
z9hqHvZCbv(F_*cH9RGD-8NU{j(Vd=|GLXu1QShULw6uiO)E^(49mA)CmP@s^I>yRY
z9k5hZeptHs2t(FU?YC9hUl^BI$(<L|hXntk;71KQ+@T>U6DB4mBqRicr5PAE%k6V}
zxGR!F!PJFgm3wf5<qi`bhHKN<H$ffd{^-GjhPd_x3l4xHR{wlL)+q(;(;uGmH%|}<
zWcPc(Zql`O;S9SEC3g!okKJ-<2H6DKg<JWkA*SfS!GjAH9XfF8l!Dr^gq$-A*7C;2
z`#n#sk6`coxcc}}*)1=bnF@N_l};~WJo5!IhOj92QG-tPpTE+(uFq_5KX{_!;0@5M
zI9p!ct}u;^mkgWd8=X*|EzwXvW$u-C=l}nQt~*<l2-?dG8KiJ@Z*QDFy>aGD$N2dN
zZ%9;CO8*NE?_@c0L?i9UWRq2fTe$d*(z-vKfF^*@M+r$ydE3GQ69t5le*B&7=-z(t
zhJ@8+4NXs0?T*ZlNAkB7F8L@;esG}a{|&R<Nug})5tCVqR)gkq|NmbU3bOde*KS8I
zF00Ec79LE<IisM-Ik#X@%|f=7cX^dcCuV(WSk7soU{Tqk9r@rR)K<?&4R78(dY(RE
zQqqs_{f=SN4&IpI7%RK0SFngfO0}U(lX<$=mYtmkEyLVisW7Cya+LlI3flkw7u7v#
zc$1lsmNsEJXqmx-$H@j}X%{Xxm-(-_bmaj53ZdeN%f}!12xl<tuP)eXAdzUEuK=1n
z1w~5HqlWp1m@PNF{$J0QoXp0|-1h%JC=)(;x-x-PvX{-rbjo%82<_>NJ{m{F`6EP=
zuYxvU{{O!y?NNh|;s4+4cPD&)%H6!t@XIN~vNASFNv$<vYWk@OvNsfx=Oh;}-LeD~
c9smC`Gq{>WCzfxqH3hlZ)78&qol`;+09#ZQZ~y=R

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..6f8b65451cc08a463e4305ddc4be0dbe2879fae9
GIT binary patch
literal 18058
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qj3-okx45^rtlpw*nn1P*v
zfq~)we<{|*4C<pylyG2bm^g9635^A;kZ`$j!JDtyfXDTsRbf{Xm!#6h|8ET$I8T24
zy!^DoUTcf*56>>$({aM~`q#Tr)_v9Y?N1wZD=^(Q%StVGiFhw8xVyb0B}#MZOl6J&
z!M*F=SDunsTdt;+mO0IVVV%?}z9XA2)xTi}Ma7v*o_S1$0<0HSZM&Vx%)LP*rBn91
zz0`_KHvZJzb!I!AmG&y{*?&^7>3_|WDqcmIZ@)E<y_ZOT@5`a+w!$oVpPu;pn48<!
zZhXEu&u_=wPrFuaeOB<`@kEzbfA7ihi$2VJl=|>)U)B1p+os&w_Vs!2{d`|e&A@X4
zj+#-cyLcRW9kaC-S@z#!Z17<G(r3uv@Ku~4G__63>*Dq7DNOGx=cy;(kVsQg5>QN!
z4KMAR-nv#bc}3V1Cc#hhp3SUw_i}MLcTvKGP4J@0#DAxE*@F|;kx2*h4jBlz953Wh
z>=NGbBtlQ&<A3=hf{iUqLO&9$=e<<Sy!y|<I;z)gck%7-<pp2<-rAYATbIAR|ME)N
zwucpKR<FA8er~ZPOX=K|TFYFrLq+DqvkE9Ikl)vG|N2q|PbG!qeb%RIa%UJjC3y7&
zYHF(b2^Y5=naS*V?1V(5c6R8rh_xoZ9E|r~f4%hE_r0XDXPD@0x7_YU0X@$?UU)k#
zcv0~itBDfsbI-5J{<ha6R&P(-*4vsaci&66=a^O42`cYtXJNRfzvIl8Khy0PL2<Na
z%0{jh1D>}3Nvy0NSuTDp*-<WV|G%M3$OT^+ku9%G%|pCjzjbPR;_syJ^Mr80lgc^E
z93$&Cy$Kb`zi<EO2Dh^n-^rhE|D5{yDnR(ajBh6&99BB);L;w}b>*G@Q?0{mFS)!r
ze@?8rr76g8t>{X*Nw4QE5-7TT;`5Tn7i=zDc%9SiT9J97)I(^C?kg#m6<R#Z*F<ak
zGY{=^=Fs7msQz;N4>;VnOtIx^b`WUYuY1Tz;b_RpoS>y*U;Zt>z^w6x(Pg87vF4xX
zSe?0_KBiTCuzTia;W97s+o{j_fqPDWQf@kNxH^o(&u2;3!S@_Te)_xAEV1g`v2|Tc
z`TnXMn_MS$Jht4fomRDH!{Zlsn3mq;`Mak{@?4_Z{l~Vk?zb0t#R@a|I)@tav>dQl
zbjsvz_+6uVu!|OXK6>yl(cEZ~kwHLV`Sq~Chl0Y7Q<9sXRlB*l9Q^SAzrEYB`VT@(
z?9!?`4UCjr9<U{?jW;;Ozs00+|AZgM8M=+m9BEm_n$6U-%wUH51GYn3{~Xr1y+D-9
z=*NF~tsj4_53i5s$-ifCXV1wGKMW85{LFKHo?*=HV;|njAKsoHVG_g*$;*q<9wnrt
zJb03F@!GV_8v{RmHl8@`*`z5?8m4Ud;KeAnd%}zMrTiIYPC*S}`xU(SwrpydC}=60
zpvL{iQerDl!U55<Gd$T(mT5FiXIT+upzxC^`(Rs4*Mbny4=3FEN;!VNk8k_`Uovig
zTE(Z9hmTy_R)<UK#U|~jo!vNb;=|wNTE|pG;0~EEUA(C5T2cA+&6|UtJxfbTYh#m9
z`S)OB_r!n;lbDP;BftJR?jiqwq7d)2jE0+iCw91)Xl_v9TOx9R!TZh&wPZ2574Lqw
zXGujxr6(jfgy<DCObb(ZvcI0Er-z4M-e^z#`3;52ho7G2`TpK`PxWzzmt29cQ1E+{
zkeX^{G>M<v`0GbQL9w)lkCNPYMcA$sG0hX%u+8$p(s%Y1GkaooDoi!W@MPJokhW&=
zHs^<3k{3i4FXM5sY&`5!F{|Nv3C{|-Bai%gLi$t-4HC4yADEuoU(fUZpJB}2Qw862
z4xgONbG*-RPp$XXXB?1Z-~>tr#>T;?&z5f782Ip6n*_&O^Fxmw=&vbk2rG2Bc|w=d
z&8?!FC9oo)By@sw$&00|Zo;D3D;t(eHm){Mh&ZsolKG(9s;YkrRRx#aVh`Mzpldnd
zX2wdEro4M87KKfThuPYs%_Z#WmN@NagIK9ixM70<C)cK*KO>JHiFozeyQsjx*y!K?
z|MMA*JQ(u(4rYnzsIss*t@btW7N63?;cvL%%4hk|SN1vwA9`x@NG#=J&MG=M)o#Ny
z!;UYFAvYdqz54g}e}BxS1tEGF4B49!9JHJyBqaX*Oa1Y={oxtOw%pqie6mRuzotC=
zobI(~IwQ;(D^_%{u|>JKN3UANWo&eaol8<;!B-BOJr}fCx43)|*Zlg&)l2NZVTe=v
z@(DX7rgB8R_@90tOk6<tyTnsQW;1OA?s@emwiy^X1xX*;nh_gWq2;J&GeKMYPbg2(
zqlOLt|0e(V(wX?!txeWiLN7My$A`v;-}8+gi-YY~GA=YQGV16qK6=dR=<(YpPk-LL
z!N9SnX*qvIm|IrLfyZ*{PPvN~dH!|J-K#WRiD!i?ca)0Z0#V}~Tnu+Qn;F}sDy)(Z
z_!!=p^wQqVjV&U`!Skj^T(g_8(Sgv-3px+GDtJEmRoZ4*EYT;ESn+4Z87)ZgD;XDZ
zaDH@gzrA$XtyL>Hj13G751ryYQKP9-|KGjq`hK1ZhyLs5tYBWNb#Q9lzsF*Z$qGrV
zrm}yM9Wt36yBY8J3ZMDXs4l5E-MJ&BWy%ymzo!2Ci$$e*I?{9&r6nGirncc*%!M5v
zLO#8hKYX~IXLY!-PVA`<e=IFG@__w4XXa_%W&@tK{r9JE?qXoqo4ZhGaY2FwTQdJ6
zxgOpx*W3b^oLGN<qw@2QKioN%z1*tvzOcrNt!v|oiIRo?7@qdsbPQs*max!BoxQv8
z-%<G`S2VwS?0ezJtJ&zOD|&2K`@BQRf@UYzDNdR4!SBKJh-u1qB-eZ^S6V5g@WJ5i
zJI)^mCX|Myh8umK=EvH+?qr~@?XP_&j%>-AaI-h-n1R9hFSf5wZTtHt<{P+nnKEOq
zV1oirYy5{B2dv-Pwl63VlbT*+RAwmlvDHIe)?TsY`@}mRrTgYA58C$G=3DJHVWwm4
z0dMAI>UMu}^Qn96;5zBJ+}Y0VRccwu&z#~D6yLAeFsJmEkK#VRg~~OGZVOI{KL{1G
zlP-3<b1*$WuXOXOvQIx8|IG^vW$@K3QjlBu`JuxRGoEAXij8}8xU-J#dbI1pt^NBq
zftx2&W*G7{DDb$}zdrZs)eF%(bN6z5{}goZ(A^gSb)Od-UTkd?$vFL5|Dn0e;~Uql
z{4ZZ9?P#0yCVlPZ<tG0p{p#ottuflk*tl{ILtfdEB8F#jH4Q4QP8piBd(LrZ8=J6g
zUhn%OB&ngg(QajuLS|ah;w_bX_P!_x=IC0gHRq&Se%^-%{CA#2l}~9(J-9D9c4Mx6
zoueK&il>C%<~*RlW5cTXWxa7p(?84R2mhQSUa&2bpC!KZ!mpVQ9iH#>ewxeN-gc$x
z$)4pYAI-Itnwj<`AAMaX=%gN0!!6HqH}MO*f)L}2w#hlNs!ToCA|_4H3`{ZJ?ri1D
zvR>r!i_Lv|Hd#CljyGXwm2g}mA7FMVkKgR+*}GEe>*6LAI)DCh%iBHXbyrY9YzFW1
z&i?mopti!4u#LP26nKggUNYyzTwvDv$Xc!c);;zYQ;EaE!o-Z33PCKMJ#Xb7e25E*
zD!%6w@k_h1sQ0t9UDvnO4CgM{Wlu1kyY)%q^lLp25*#9KZJ5p1YSp{BMr7^kbNh69
z<1cGp5ZSzHbLG0^*UDM$Ete0nPEy<&wpiYt>CiD(0p)M9&pRI<DDA!KdyN<Dn3-V5
zu*B<3*doF+L2I`G!{W^@R<o8USg@~iYG-_@YyaQvkAizo^ttDSZ_-vz`x*0w|8~X&
z6Q3hZjJnYcM<+Na8K*tmy7GWZrjmPkUUz`gmJN){ue`U_Ik}%{rEqt>pxnnh8h@DP
zHKl)Z*R5Rr`hQH#1n;_NFD=)tuXbIT<$byroQzt+9r+F@aAcnj-*$KJ?7tcR_rH6f
zaN>*`hav9`kIulq%XiFue(krOZA!CLX2|3lO1&bPFLe(UUjD(pyg|85vr4?=&z6j3
z?n}9L7<0B={U9oL=4pkZO6a6o?g>)_WM>PCeX!gn@w52ht;Ss*uddWGY-xLasPMp>
z_>a<{U{EqE)X?r`Ws5p-sz{;VY>`-AM?~XN&WbMz+dTwaHFSamMR|?H<!U@qg}&`@
z>CN<En^nceqZifiAduhPM}kY>RKR2%8v%pK;yUsI6FDvfF*7so;t11=F}yOh2UPJ!
zMe$s__Ci!R3{uT2nHAo+p&=o`VQge{U=piMr-1@*pmPiZzx9+4PN&YExGKOLb;`l(
zN5o~uS@SOlF{Uvk9SG>pR9JVVEtD}V^+b#h8}qWy7bbBtzdO;EUfW^O;u}=u<Qx>l
zV`s+`6~(h=&5p;j8yg!NA+?N>Rv`yx;q4nQIJk=!@EREz7#JA+YxdcEFzd_z4S|6!
zlHD>98zwa{nx-yj)m(StFi-6tPsW?GD~jy@nY4uRUOUpIE|~nvHBou0#tB!oho6`u
zdFON+$S%IoV9?VeQBxzq!cwq%B9eV`EFvN#a&tod{F!p|rldrHPDVqoyzzp*4rMEj
z4^1MS-Rj~-55!UqOl&SpjcE!qNJ@UZQT%e{jxO^Uy(1=!t=-z14?GPcSU;{vOilgp
zJ>9@KIU)V{^F(Oq&9V6R?{9vN&f!C!#KmrYU^{LfVwM!go4JX1(=mBXh6$G}w(U@i
zka(bfd!yE2o%Rh{#~!n-c`6ph{83&1kKfS=3fjLWZdIO~(bmyn@Zz;`LD8dPNpP8=
zG{?fkwDs(nCpT|OPIz`Ikc<CEm9EFEgp@zr{u&<^25qV1)jZguz4k#kr})7v3&WSp
zt3Mm`s>ZQI%J^%#DojmK6K7+qS;?-~%~nw<IcKiqlBI<^nveo!f<;8c3=@+nJiM(F
z#Ig+)Omzy*&id#2^q=#p{t18i<!>C|QD`!3Ide45E_w1HttQb2>r+omJ2YRX;MGwd
z_K4d{B1DZ7yv=!dc&w~=`ucuE$w7l8rYIt!W8>!k?y;$##aEtf49YPu_~W`Z=0Xr#
zQL<R&n>InUPx7HMwjCUtkF<<COt@AX#kf86oH*mrk%;#D?|FD6M0uH+#muf%u}oL=
z^W&+hvG6W|nmxfnMYS>{<Va-1gQZ97BOW%EzGY34=L&k-bjiw*i{puaqh{d0zrVl#
z{(isyzkxHCo>Nw3gLZ8|MAw9@m-T0OKK!?5b~`a)noGgPjjP=KJb8F{goJo@?Xp<@
z37S3)YSt}MO3l*Z@s{k_u(N2zPYI<DsdgO~T6*gh$_xG}b}Ae=Kfm6t_Se3oqerX1
z-zzNKc;eJ2DXFf^EUu3qH!is5@`F+NSPd65+jK$Yxq>EQmmO@HcBBaA{r>s+{^R5S
z<!pX@{`U6%{`&tbR~fx{VX$HI|MVoKW2y#^B_t&z0+xjsEDwLsRG?d+r>xw($EJCX
z470fSH(Opv034cbF=K{=sR@gv#lM5g{2xwU@&DoEr7OI-9t5lOY*=!*A*A!bYXvqH
zHj~af`~T;io|b)i`TRYVpLvy)58vN^fo;vEO%40^XT{8!Hf@>$H#hT{vNJ*)kzC<l
zThx_0{c9>1)|GG?|FS>)`uhKEIX~^Hzisap|L-s@Y{N#wf+C|o|Ia^ocu++0WJcaO
z1x?R6sXY;C4<3jqY9}Wgu*>}Z-=0}pySbhH@G{>U!tCHwd}zLfL`o{_jG2-XCb7<G
z+;@p(5%a0!l!miMoaP=paG+sV9>3zh|NsA&zu%`9SyXqdNBI1_g*wy!{h$Bv++6;>
zdzVkw+bboybmEjHYuBASf9Ozy;NJ=^PPUwrtl3-&G4mc9%+dJp@$vto-TQrJe){^%
zcgk0h{dJDd4}`7%SN@F0IYuTqORKqsg;`ke@Z=LOna>~5Va{r7ToxNR(b1*l;ZbfK
z>tB#~KlGfXuux7|xH2s4$t#9pUG+uDpZ@=kS9uqobK2>@^285~0&U61<-UEBbMuxw
z@!)88{$%y|j^4Y5_5Zf+um7)jcB|0?4K0psY#lkd3A$Y$%+&%e{Qu9hzs~T@y}!z8
z&tgNbs&ARnm~^DRt@^t}Rn<TJh!6j-YCno;zt!oQwm{S2*`lpBGiFNm^lp^qg9Kf8
z5eMhTQz!nkuKk<hkd<O!bf9&?wLk7cKhGa%uqo=hBiuMe^2F5B`>(Ix?=z?3$K&Jj
zKb#+ae_#Ln{C+(ho08w(e$SDSwY1Dz;?uKrZa14jGM}%^>Z&yjEvueCJ-t7Ad)?cY
zKP#S}lTVtKQ2X<cLA6=)O3mgD4rcDc*P0TBV$+NiroNugJDFqofA?@Tl_LiZH~&=-
zfF{H6A`XrX2fie{lW@@YmXQ2^z^i-zhIL$Nf=v3B3wr+l|1WP>^Zw%Eeiv3|=EK+5
z*YDbWzPx<?{>snHL4Oj(R8GXO&5&C2lvheYqi@<O1ICy?|GUNO&2xUtn)U13G2TT%
zY%Dev2MlVhGLLV2@T$ikVT#O*hM6-R<L5iZ$aI4`sQ>>Tnr?CANQ;Y0L_tngs|wSU
zciL;%yCoD|edRNh{!2*w`}_Cz_v8Kd`Q`ViE2>CHi0SQ#jI4Zgb@luDe+yJCEFAhg
z*09=19Z=<C&rmkp!KEni`v3p+@&9$>{_I@-TzO^oK1m6QvNs7kO1o2Hc`ROuH%;x^
zXZOuh3>pG6K^ak0gk|T>i_g3SW-m-k<y3k!F^ID$>%b(n2ZpB(9V&QwsylwahxOeT
zpP%24-fowbX<7bOs%VvyAMXsMBg;AeXfjOa)X3Ub|9^ks<LkQ7`|j@bKDR7QO|5Nh
zwCA<ftv3#+cAWY9bFyQMjJG5+BuzxF-MG=v+uQix@9*<xN^ZJ2mGRn@hILb0O{YHm
z?W?eO#RpM6o)yi_{{uZ;wtjkZQ#~!s&Ex*miVv@^?{DSKFDx|F)_z@FXqecuY=Nsp
zk$Cfoj^k5ZE}T+H`N|{U|M&Oz|7U008>iJ+mA_N>oaZ#-t9;cOe#2TZjsv&k59YHf
z=VWTh%Qvsk5PJ&^h0LG*^7S)jL`X^+^7|eLGWhfVzj*ZAPvV}2QQoY(n7<wGkAL*&
z&J8wa&SwAlb@_LG6ddnU&%ggqWy;R~i7r|!hKpAHJl?RhEy>|h)70*Hd2;suwq{?y
z{`z`-?O(6H{Sp!q`|8{CDpffjtPCj7Y%_3i-S|Eonqy~Lu&~?^6=9iZwm#{Y$*mf$
zs7pVaSN+ppWt7pfaiihM)1M~?YC9iT8!bOwZ*TQOrWC`k^;b$7EYr?6NKW)<s<`sN
zi~YpmDvpBZ=ib-;-dF$cPk&<!sDdyzKPeq^(QO{bCdmZPmd;McP*Lyakl}$tGcA7n
zv^;ubfnbl-@2;>Gg{Z~<|NnQ5y}>yBZ}Td-iIZl`{O{@gd4d?5$AkO!ZI_qde{=6|
z&x;o>-t7MaE(?U2X1rv`-r00XZ(s5A?>me4*G-CC>cA*3&nzyU8TCMwy^eK8<j4E=
zY^kYie0(K^&`=IvTl4c1x7h0subB5XcE(IKF#6{z#eP`FZ-X%l%Z;Z`et2-RseJhL
zwLf<Eyu17VODg*w_-s<-_P@XPbi<N2@>y92Zt)+QJnx^e!Jqnnf6l+YK1EJJOQl71
z8wb-9$DlQ`!ak85)8y?p%(()!aHd7w-(N@1oaw5P<1JY?b)w+J--;($w{C6Rd!0+z
zo$q@+56}N8Q-AvUcFy)Yz*hI~Px;$h$CcgV61%_FugHmU<zDB~uw3ZitP=}*($D`3
z2wHAnV6b6>fri$k#e)01S$lcePi!jCJ#&WV{e9yEu~N9LDylzIQX&kJzT3abkVvyS
zkdyf5s9LK=duqy#yLV4=aY{URkYH8*PWRp?hH#(cWfC!uKA1FWo^Cky<KJI-Y3YfF
zKi_EFWi_*td9!U}_I6#VfRH0W-SFfhAvtsA%=*P;h6@6_V@@kfRXWjn%08sU@?q1Y
zl9-q$5fL6Y;#%1vB4)h1`}@<Ur{y_@y{1-$SGji_nV71`u-T`n>(8&R{f&){lP51a
zl9+l_LEU{>+@HysJUlOyUkE6uwJjBcW>ZPk`}_Y_{rn{UWQS<ZO2)Y^zN!UZ+8ZD8
zX9+gljQOjse}3}Bh3;j)e*F8peu2X6#qRM1+a(NEv#oi=v*R_lO{nOmb@lQ4|4p8d
z@aVC!eIW;@Aq(roJ5sI8O|GmqI~DwI)cx&RdKT(2N%{Bp>yy*c+UDN)E`B@HXM&hU
z%>j|k>$@Wp6u6ZZP3U}CU$tx9ym=gz2S5J(9bZ=`=I57|bFsg>SDp7Dt7L&Xx6R78
z^LJWmtE#rI)!k@)X#-P?E9<&X%t~Kh2u?GDR^&4+R8)4but)?p^H1WKpgp~whv$PF
zR}|NQsDuYu6aQSkD43IBpsMe}!W#Mi&(Gy+&o);T7|sh~37_?&dm&FY*N^tIH7ly(
z;@FIh!}Rpr{zN+!Fp7z_wXw0q#T9)~Vm`BoDN-QU*^HU_@N)l%ZDG)my!<TT;lrQJ
zA_DdQ|GU4Nx?rl<i(dYwm5h@gK2?>MU3=_^OA(WxczSt(z|Su)&5u3fj4|afY~`Al
z(s<I<AR^*lals#ct|W0m#hueq($YM<k}h0U4wbi3hzLnIRaBfTxeDq%^K&16d@Nt`
z@0C5<mCouXe>G364Pk4m-;l(dJ-5m9>z~AgAE(d!S-<_dk&@HQC7+rnPmc8LjCOCI
zK7YRC;yKRep1yEZSZKshfAh2aVT~3eN5^9m1j<rV)AI6!Yz`eb@Ziaj3)kEZ)o_J1
zT$|m>t;Bq$kH^}28m}KT>0f^K<KHvhqT^rn)3!EEvHPdA`QtL?Cs&&9{Sy#Qo^5bY
z$t<jO=FA%`i&mVNIzvNUyL(%|xUu7dRTdkx4*R%8WJ|vJ%BTI>UQ@H}oSz>fqnnbj
zm9TL0YHeqK{^kt^6aO9KoG`uPkYvR#9UdMY`#OU^e>ziHIU#k*WQ#fnHRD8Iu8apx
zQVRtfuY6;l7s$J&^?3Xhj^<NcU7OC$$qbEO@9(ogOQYcFGc8t@j?W(ql8%_nvry|O
zP?c>`bN19*_dhDiCwO_$$`uBJn-^$nbR^y~VT+ADq}9nXBm3|LmWhnU#)lst=dm}q
z_6VA8XIiwh-rTW6V&Wz#rKZ5^xqqB>1OF*r)y!DRDkW~YP(fVfzomslcJ|kAZ;MS$
zMUxT_&U34%jEsqOy?x6hDQU@sb=AMRrHV6J|HtpS%CgRiBbduywngCK<KORJp4?Jp
z+P+FldEVrp88ank&6X6?PYVcC+~3~BX>i=lP<mkj8#njix3?pBppMx-@A3Ql?|FD)
zcrFxma$Y;rxaM|3)7lR|`gh4Po=a$0x~Kj`o?DqN^Wl0PGo@8BGA`fWmD*K(x!T6-
z-zQU9`~Keb_xJxB85wbKZj6lJXlwDPtnBRU*x)d2QiArMkLG5Dn#l@&f9?0=Xq-Iu
zE#AF7@q6AkRqwj|+uz<lJNtiqyz}!YOHfl;(#J<))oRHtTMJ#8Tp6;n9MY7z>wYvG
zWq~A=DY1L`S`0Y;Jt&&Dc$UQ%*&06f`4^@aoAdRqJ1Z))M=CeyvNDeY2Tu(%v%Tk<
zYhTl4cJEt1V`j(U!o}AMyPq+*2XH2&wUn~#`cOQLL&ngKVb1*OJ=aT1&(+R;7AsP6
zf6;0Edz%f9D^At8IdMhT_f5H*bd1ht>7EkG<xWoUc_;EQehJsa`wISjIl66(mqm`>
z1dj?HdbuV4{y&@AU)-kmjSUU{xLSSZJ(SvF>MM~@eOKax8s8JwNCAfJOuDsyk{i~g
z{y%=;z=1P={3B*amK{*`{5j*ti~sD!{856R_)jMO-=KMHmdgSa<;V77)9Mc$`rq36
zKRfGDZ>SbyQz{$Ff$*lgdrp4%sx4Lw36VoHE$aUL`TgZ(KHH7RRU0<XRY>PPEIGln
z=|k32`)x_hjJNG3{+Gz9s4)=e3S`?;%0BTb*FT}w#vMxR+P~$^gz|LGGEUTFmb|m`
z@Q0_|V(=<=#on5q{M)LgG&1fw{nlPt*_ipujPe(P6aW8LJ~y#-?i3H7txePHCY<Jp
z;bg6kJAO4oVQZA)ZJ9T=;#JFdQW{KC4U*=BpFD7&;J4YgBhW-P-NLT6sZ9Rg|NrNY
zuw<qf{QdvGed0t(P1!%`4l*(Q7V<*J8W`5pDm`IpW_16hpt<b=muS+WX+MNmSBuVI
zWxJMQctEq6aiw|po-YFHo;`Ga(*_NL=@$Fy{<7Y`&lWxZhkRL;d5oyA52LW#>}JN&
zmzHAs3r;01<F~mfWxV-OXKaO{OO|#^LSUo))IZMO4PzPBNwI|Me*R}$_7!Sce9@np
z@9(xx5YT3RGa>H8GM0$BpTu|0J;<@wjVZge>E^SBT>_@H3CnocdL}h9Mo*r=sJwjA
zETa>z_aun<{Wo|c%g|7Ig?louV*K7imA6bGrOb4TeKkM-zPcK(sCc4qGtY{*JU`}2
zbUf}z{2NiB>Qu9tdGncs1A?AMj%fVrP}wAMFiVPAP<AQfYOf0nhPs#k|1aF;CMh8S
zb>V!Ay1mukzyGgKt7vTkrK*I6E1EnWj}v%qM9FFB9Y1z7v?#%ijd`0**H&fcASNZ#
zm4*r025mv75)u-26th3KZG@Pl6i~FM_P4m$i=GXCbN^3B`fGp5f_;x)qC&{*gpb@=
zSqTTEqcbKRY0$kMG(+Ko_A%2ph7prkKWa8NHa>h8J4Y2-d)!y}%j_xh|G)DoHV?%{
zi(-juEQw8XKQ)(qVv0J|P(0CDpto0M--(00(>NqPq-AV*#FCw)q`dRN#*6?JZnmBm
z>S-*0pZU(=jexl7&~#Ay>-)dI@=Mx8Yt&fl+9icA9ni3;HAo22s5r&{B*9hNSi+-U
z-A0M0MMzBUvn0n%`-mVmsU~hGw^J9dI55oH%?L?zN(P($KRhfSvHFqypK10>q@LJc
z%CJ(Nq{F<;F-W98VZ%gW7OCk!Bo0L-2wWCUJdnuBY<9F+FEWAY$v=);KGH>&AOBkK
z>+XaaHpk-M|NqAsUUwz<%vE4JC?RnC?}6jTB?7iS;Br5ca<CyPY;sglfQGA$)DnjC
zD=cSSSsNWOt?&R;_qHbveUJbD4_w#Qu!Ltx8wYdHX%RkyY_Tnen6n!h(-&Xm;JGnN
zydp_9`*(A~Ny9IbSeY+^L;8Du8@KoeRZzc};s1Z72^Mzs|Co0c|NdY9Lv@AE2I)T+
zq$GP*Jmr!}D9iZ&IN?CDtX_v{*aa&N9*avX|CThE-ZfY-t26ZP0=6T|d;b5Iw5v_B
zsF%B+x(E{7N)tfYhu2$pf?&ysHPXzVI3$>vnccJ+8Ej56iHmRMI4Gl^eqf@`hn`uF
z{q>Ga5pK4PG)Qo<ubcY+zww@mg9Sg0_8mSBH>>Xdzt4Yv+snySnEZFn@;7igX7lfV
z`Vq6$hZ@SOSX+O8er~U(_T#AIjEzm)4<%n8QEk=i%RCyOrS+g8!LpI_-96QUUQ2MM
zQQE-s?|A?I_xJxd^D-UUuJ)(B`@zljrXxp195=-8{dM#1-#-T${db7ZTx59Rjnayz
z@|TiAeRS&$1@4qH&os)rlW=1L<7qvUB!+`6OoBV><{o(H2zTYZ{r^Az`@8@D|B2r<
z7A)b3d358DtoNxO|Nr0b5`AA)6(?)+gVS06+L8wC+6R%3`=(WWSQ3@8pGW1#&&dmS
z&Rn>%^WibxMOxfj40+htzP!`kBxUjap|j5K<cZM84p{r=*Vq4t**%uFT)Wp&JW-t6
zXu&nj6L-VidV+r0ANdo$zV82zkN0oPczl0<y^l{sca@3WziZ6%s#+M=A9G}>`<Egh
zmbRl>{BK934MX;?16B%biU0qd{qVz((HLsQ1dACnCHL1f-q$<FkeQ;;+kR*+-_t`k
zd2Z;7x%$ia&;Mst`)hU+)9NlRZsE)EGc2yL9Bfd2wDH(?dzmP<>P-y{d3kJwg>CWs
zCC<%FHZW1SFB9|i^#A`n>*EA73JeSkpkcM+Yv0HJ^-r=S0-GJKwI<)L?cbW)7`M>C
z;Lq>R&)=V%eBW>GFFAn&i`Sg#Stt>n+qB1@FPQh77Eg4nu|zN%GxHalB0s;j(A6Fm
z!4rPm<6>q$yfwQ8IzFZ}0W^$IQL*6R@Bj9H)bum%I0Uiz%rW`$|Ns8w{`F^Pe|L0h
zI(ojpyStk)f6}3=OAag*GIr!yq{eD^;L;>V-`<C>Lfa;*bINFUn1wg#^m*GdGaufV
z%pxVj!vmUL1Z9_qqCd}{y5{XY+|4ybP3w}JMa9ZK^`n3O{{Mgf`}_K5XMPsEy3)VF
zATK#NOy(%_IyU9BN`~kE+NWwnnBV4YOkrgF^M_~VOrH66hHIjafB0wnFo<#IsRKUR
zGmN*0>j{)W8tO_DEdKrZ`Te!Ha?uoLKQ-4^m;brj{$LOk_IRYR@!$XdpaIXkJ3mt2
z-SNM6?M-Lroi}ePl;px)o3EK3z4g-d-CRD~2XD?KWSrPgs(pB&v&+ikK8!Q|igWyY
z&VR0S@d;>DOt84O@9+0VtV)k2o?FwZCh{ct;>T%XCypNY@7%s`>$ZK*pQr0+Kb|@{
z`OrkE4X=*&RIg#>EZUv2g2~XZZHGnM`}>k={>QezV$wdaN@AW}+@6FVKPGNagd2Y2
zhQ$4Se@`D^xS9Q*zwGmu`Xy$XTNHl&m#@<|Ra(1XTgrc9gFnB&zrX+R@cp8qd;6-t
zZ{IcR{H<BNot)Xh%zgh7CM&t6aHw;0BqS#n6eoZ9kYG{NvT)a|hc}INobS%q{NYGL
zx47iJJ&J2UaRV;NcI=R7ZTWXlN^*i3d&@`ut(;9uMGkss{NGpqe|Oc_=}(?)H_!W1
zSp1lS|9L{{-}Y5AX1Fxmtai#?!1FVOk$ux9p1L|7CMF(fGlP&2gM_^YInQ%QgdP3=
z-&n@7>FES@AE@69&K*2<OhcJ@rzztmhQIQcOuzhpm~El3dv<{O>;K9tfBye(f4Ez`
ze!BjDyV6(Fe}Dh~@BjIN|8^xyIe7jwzdU4muF>Ye(S|8}Y+pNATR|fmv&|*<)g=D-
z-k*3?t8II}WK@*Im#>l^=DcjMRO!0uqWWnTEc<{H#_=;}7`i3gmP}#1+ug9m^^twe
zd#*ck3qBogub(@2{&Jt6T$&P+l9D2#Gv)06fpU__O`aP<>`_ezO#hhL9Q^*CN8a8k
z{c8Q+B#V*`MfLWBw`U(b+Un@u{-JP6#BUvW`=tA3y`cUKD8WwvwFv+J`DtHU>%sX^
zK91MW$Z2|?)4NB>DS!G)w5$)g+SeJ}+4nd7c;9^Ew3_NKFV;^I`uzU>e|ei9-~Zbm
zKG?iJD*9rZkz|0fv@z=f)#(rC=qNn+nB4aD^};tjJZ<OaOMZKkIK#aE;hRV{QPCGu
zH@tbn<L1V5>J&>))B8!ae`Y*9%DpHD(l$_<V6kK8Obg3}N7-un<T7Iw{(O@KburuW
z@Bg!}`}6(A#{1Ug|Ey|$h)YX5`|?hoCFLD=FTKN=l|^EI%};LW26u1C6Rqs?ju`$q
zxZJ-(a>9oKw!AzxOUpK9cF-!08ygxOqh>YEpWirpwxY!hMN`qJ2?w+V-FjYED%@G_
zFL`gT@~JXtD!B11;lib+o)m{Szx($rl9*>8At9k~^3lV?_0ik@RQ>qyey(+Wzx@Bd
z|Nhns3c7iFUwe^JCM|vK^z{EVzgSFPotJOBzW)Ep6$XditxY{J&Ei2T`zEF*(@qtz
z)&Dm-Glw(R^3-YLj!wgjOv4v14J|ATbmGnwJYqTg{5;S4I70(N!<ZchK0IXpC91pQ
zmdt#6;oU+|cTKR6ko;f&pD(lh(09-O&R$|X6GE6gm-oy6pKbB+_r1OQ?Q4Ji|M(a*
zJp1~!vASxZi`yn{Zet^ZKk>W2^=~kE{Q3ERNePL#y}!PHeQlqXrk2C&zs5JPZDMeO
z!+~aYp5^||H>091ELOOALqbMY^2`~DdwUXh6f{0ODcqKSU&6$6;c<baX<``<&dY<!
zh>kt0KrZ|L|H+(<yv+^*EYH_G7VLPStmLF5)zC5N<cb!y7mSK5k1E+uaOfyof4^q1
z{O*IY{RbAFGT3o_-nm58e^=(&I<3gRy)*Nf#WbDCyRNOewCT#C8_zAaXMRdN;m0=N
zIse4m$T^00j1Sr`IoDs*w{QQP>RC)30xS{UlM-|vpV%ejD)IUCbNB0a!|vSs!UCEX
zelmA0Z-WDmTl~W#je9<`oMg~B;J|ZI$wOuui-*P|y+2%R4~k7qYdwE2_597~J}q$C
z^+P!gSuQS*j2#%KxvuuQb<IlMb>fw+AOC&W!WFSPcKb8S%Pky-H=HewIQ3ys<L?E(
z^mUqCB;0@5wfTPjqL$h9-J<=Te(k}{{Vc5)W*9$T4oXD-|38^)$aO%0$F=^*LC23=
zUDH48i1WX5Raf>_m+Qj2vxQSQ{>-0#pkA`CanFpGJ6FZbeCYI`Z$aC4k)|uNdK8x$
z%w|qIH*>D=@+Ko^uT5nRp*q_h_gfi-zxUwW%G6-kQTX}mX4_*)Ler;KZfOhJ_pJcZ
z44Pn(k|I&}@6Y+epXW~$EZlSX|9>7Ho)5|r3wT4#%-mR6J))o3t9(tc_<sM|+Wm@K
zWrc;C7bqN_ZT>%@*>XpegtEw=|K|&yaxFT1^7!!&zs;F1UD9G>%h6<ze2`URQ|omo
z!Wv@H4TV$zkw5JDSqJ)}e)L~i+p5BPBthdyxrfS@qMZc~`EzP55AUz%(b3_VH0l4$
zng10v<un(IvmCs?pJ#o%aui$rKLZbs1j`b!>~zuWxlG|dA26O<9AyA)uhei@zp=Yt
zzv6S&1heDy7ShVE|2seZ_xwr9jyl<>IR%A=9-e>ye|`P``udDugT<Q6Tyi!E6+eYe
zO`Dh)5Oin}+h(2vp<fROdM<wlsdEp77a15Ezj$Hr=jY)Ar+)F9^)hwFKKaz#$hhsL
zT-d?|*8Po*4?jQuU;XV*<9QAPBcnfmPain@SH8^Ubw*<&(>$9*i)t~e6KBtyS+J%j
z|Fxjefv7nL7?qU|pP%pIyA#?}JT%?n-{0gPzos^}vRY~MZJGANKIl}#t*$WHMLU1o
z-Cdu5?@xRF=|8`wI{Nf9wm-MGwr0+fHQ@rSHmmqK>ETiCjHZSo=JRHnSsb{<;^?h?
zeci*`|2sfSuNc5{IRE~qKX~kHCN)p(L8|bU0}WfJN<5UBxct6^q~wg5|KH#E$Uo1*
z#YLi`LZa^Pum7K)*MG`0kvhgONlh=VrRe|PWC4*B0U@O|g0CePXFPd)yk&adAxLl^
z3TG`WGFq`J>7E%!tRYus`T;+)wr6!63PHk8<e3luumAr~-lii+XVXSQ1LObV!oT|?
z-TRvI?@N4p=jt<O(?-J;tA$sDaZHn(Q1I{9)W(@JTP|ONnEWJWCoh8{k1?-M@P_|c
zE>~6-tm`Oo@hHFWm@{a}2lfB9?3SFLrFZS>W8L#4ap8l<MQKl$q%HFj>|QrVE6dgE
z#Lq7myd4&qcAWXKSfpp8;hz0Q+fs`sxxAmPSAB#Tl-HFmRERdl-r(?`UGRIuw*Uh@
z;~(cw{CIzTeZ9EepZ4zyA|fT9JeAzBQ?l-_(!vgD#Ruo*+1%XPgw+LHS&faGcUYWT
zDm~+4lTK^c@(-Vk?(Fw>dITwH4}~Auu-W*-XXl$9r%e~|+)&Y-(yi0_#)*xsEqePu
z`?@m+&h&gR;O9Slem+l3%#}TLZ5|(<avz>=&*S66BO}9e=8TK!mlvFMe-bPzCOrI{
z?y^l#AVEMl`NwzuYW*n_1*LBOcrC*5`=qc?5F0ZyB$e)&yPK=QfX9`8QS;>P$sG^g
z{r}(Zyx?uZ+^**Jju%&Hlo)0ku@njK;Pl*g;NhLuyRIIbvG?Yim0xt1b3VWD>(Sn8
zB1f;e@P9B3`BRe>y(U$nLurZ=i(t>f*WVULcsk9Rl={Rn)id_8Nr>p@GZsR}>kni^
zzNlW~`lRQ@H;rGq-Sh5TU9W0>L_^dev_TXUxLfAfa<v!;wA%AVczAu}k#o>~pP#F4
zwyl_f`#;A`rRLXK8ISFjc}PA}?f>ch?!?A0<!0f!<;?PV^&*<Wzi0Xibv2oU>i%rM
zJoj*Qk7LU=*Oo$;mc#|`Us`SBP2MtDQs}5f7<cyb6Go>dty;A1<Y$8gEnDBeFFU@=
z$k&AT^rxq#^W^89jx&9{LiKpo%J1d#ZDK!y2hrBdu;prX5NNjFawjc1np@jH;bo$2
z``)Puug^24T*wqDc__-AFL>lQbMSHrkG2+@nTIW}eRcbn{rN{?U+wZ-=LHqk(uX${
z$h<#(s#2-RkcYX{_xi$9HkZHmJZ^n?=d$!s!&N@Qp{|nxrirjdwryRad2L_KwstMw
zW{(>y7j@XUt4y))U;kwL<m<W17;b-yKlvL{BpPrWJJzQ9YvN88@4oZ%CF}k;xE%if
zpXbdRp7-|*KYVg}U3ZGnW=7uqv>$(FK0GbnW?L=6CzJ5wQ|rUm;cdtJJ+-|RAH0uu
zX{!4*CGlfY+tzH!H*Y0-dL^Gckq8Nul##hm%`&Cgf4)TA9*6ls>1k;X9v&28jd&g)
zv0<z8!zV|(Z@e!$4Bh6SbU~rjXX@Yo{hM13OwLnya9#gMf;>yj?_&xYCq6u8Z=NC|
zx=PgJ^S_tDZQj!*?CR5Y6f`E@HfsxCuW)$Nkv=P7=6$uvH}=hac!ZNpS((k#vn~0!
z#Gbv9CMFVf|56@2UVNp%<22i*FR!m^w`~6ZKmEeRgLT!G2VLVo{9--SwuUh|Svm2C
z3^-Wk%+lp*0IgqF+P0Q$V|9mZ+ji0Bz|9Ajs7>A=<`{6nplN@{YwH6BJo0Cnzu$Xc
z(6;XO-siSP`|^t~ZZhCm&eSL}^T<*&UdIpZ{5Ml(Nw~*Mx46gj&PM%ofy}d?*}~_-
zj!jVcVdv^NOUO0D_qI^m|8%d=<*PKjU;Sd)@_gZy&xx+?E^+_5EjH}ie&Tf_I9<<~
zwUM{MfQKcYVMarsKygZt8^hAYOFcX%a=B(|csM@vU}<uAn7-kD{kHpEi`V&2QC)m9
z`cuV}n5s+sW{M*3mDlD<hkmu49mIJh^7OvV>%Q22?%CRtT+^$0D(1h+lFd3i%w88u
zC7f>`6VrDxcxl_HwpM=|qlNqB4;6_gwl7*4I^pbeff>s-odJ!Y{{O#bt~FPK1CNV*
zhunc%+8kB9{x4O($TW5IUTr$j)jD?(hnV)i{SQP|P8RSm;yL;EZE?$i(&>9Mwrw_l
z&0#6+c&kjmWqtOnwnYaF%*=PnzjjXbz1*SH)~KS$F=3L8`ssVEzvX|;Tjg~9W9=o8
z<BEMB_q>!|kr`(s>^?;`)?GsB$iKzYa+4$1S3HsCZP$|0v^NoH6$It}DRXVPS{!&<
z?OR+^1vVCXN%ueS+_PGI!v#OVY)1j^?v)`a8*V10u-=R{3BB}$J??>^$0vT7evZfM
z73c2EN{v)s(YoOM&GJ=UDhCVsrd56XVA#7!=Z4v3m)OOou8z|ZBY6(r{kyiiLN2l6
z{o`HkRZ+(0HA=Hj-ss3YvMEKTQ)yP4iL2KVmA;#1EkEL>H}+L3trw~EyMA%op1Aa4
zYraGe0UN!yheCail-AbF{s37RIoFoEDM8?P{f`d{8XhesCYj9_glZ?+ZPRmV>Q0v8
zeW+k6;vaVFLi5Gh7wc`_S?qWm_phM9-s-_|{cHM}yC0{%C~-P|G4+b0W{X1K$J#EY
z%PA&{*2!O9-hKD0*JTCO*}Tn+flI!>HBj+1=s9lbW8ZQ?TqXB(ij%;D%AVsq?Ga)@
zS^dj}k1Ow%y{)@iH1gY5<9pWEvR8H4sQX6h6vU<(KPx^ZbpLt5fo(fu1e&%jT_@P|
zS59;5$s$jQB0Kx7wM%FC)n9&Ar*->f#I^bm)*ayDZpth}?xp~N<MnO%1zd|3D)zJ;
zxq0&JzVzbY`3IJ+Y%(h>(CAPQmbudLz*NIjLnJlK{O&>D<B!Y#?=Fu&5x6@y_rZT_
z`%iziri!s1SKf85=w!i(yyIWJ)*pWv^0NOnui^=#5K&D|=UchUf@|h~o_Vy%V%gPS
zoDxnUK3ZHd?57WzTr9PcZP8f2Se}2ck2ZU|O5$ckj)?Ui_i*K(&3abk=q1v6lI3>V
z)+C!Wlce28H(AI?@OT7^u=*zNUw->&>BLDldo))4^fBl0nOwJj*@^S#)K`8KI`_PO
zj@a}v@G{aVbL_dB0z{71pYT2+<8*SVp^Q*gm)icO+IaPY3s$x{aL5?VD&kOz`r~Z!
zMMKzg*XPH2?+e~pKDW(dFIMs7cDyPWkS>@Z>AYBQn(O}mrPKS_fA77cvms)_^AyR;
znwh5(6uvG?6+E6I8R{xhXi>0#bKGK$qNvhonO-qfa`TSY=o$Zx<M(wA4GeAh@#6TZ
zMLL>XpJZ+yEBsJfW*w$|Xu{`6wH*7J3*Mr!>sc5d8wMUJ>Rh+^uF>p`Hw|VM*d$5Z
z$(!RBqi6KhYf@UYq6A-ItmHlOReas@>$Tn|ujhAh`jh@>9=I%=GS`r+IY7WA{-g4Z
zl^F`T0++Qa80#l?KjULlX`LFrLEwtfjTIU*)D&CiI|wzWtZIEb?dIIn#2cbTCcN8k
z`#*eA+g0-+?O0m#^V)d}-<2Pm_*qg@l8ui&X1Y!K{_y`Va%QV6%$=!rDI}M<bnTWj
z(YD6ag_%F>0$Zl!Y|mPiJ!?~+?iDA6mQ!|erzF)>k1Kl~dg?SS@pjC5?YPT3=XNp0
zJ}*(?@Cc3(Kd#t!^X-akpUIn+ynku6ic3-KiXu<p9M-fyyBccOORZ0Mx4r%&3%Hb+
zqsZM{AaJz)&vMu0hi8^L8g^uCp4IdFg0LXV=8ixewZ+F;clKJguaQ0e@l@e^+qfNf
zkN-IFi7oL%v+KT@&eJyP*z_;^`Qp6su5)kQB#in}B=`A8DzS7q-PSX73oNkOdtcy<
z(6NUHo;D>W+*aYt?0T;Iywv93!}YBWp${cD#jbY|O!Sy{`TG~!+}->}z8sFBiL<Vl
z_HEYKy6j@<obx<~lZ3*Xlss7&3wvJY9mq=)U0=E;P41{kU}$~1eK|N*=FC~k)e<1k
zYVRnRv#r~aRsKN2g_IAmHDQwzHslEG$qL~~D{OxBuBbJ({Lm6k;d66-CZ_FcNS2uK
z{JNrCzE7#;hk%)fH*q*OF=*QVNWR_IwMM7X=7Y^DE=2>LS5n6lBPHBR{<1x3Jy5za
zcT$UvvM;~ETh^ELMaQ<hpZuJK(Y0|3$H&X^+wV4JI$h3?@|j$^^t~a^u6MUq*UnYw
z>s-`5XVIqCRiU%D?Y-DE_Y*kKl_ppyyfq38Tp@cRl=IUWrhkE4mX>U~x^0`&H=ZeU
zNOQNZPuo#7wXwPR;n(ao`+A8TI~VR%Ik7;0qC@<A$GCZpetidzwSk6(A|&pwu=ixR
z$)c{#mYLbsExyqAXs@(n-5-Tjyeyo073o4jAql$75|W^*KPQ*-y5@|>2@$%f2?-Uy
zrp%cMooiK^U=a~1nRh>J!?t6Vd~Mehj^29n;89XXXQD;joQGetb^Kbo4@9r|z}o84
z_qD6bXvQp~6Q_+=tTw)J%Sb|Up+(OthuFEGWro*u*?fK3Dl6NjYD>oLP5SYj-^$M0
zG9WN<!<LkSqC@*uNNi3xpxQd8WL>=ab0%olNU7pJ7ifFZ#*Htwnigp~ZMb>x&>;n#
zGXiqw3jW(2zPwz;TF~CWFwwv;F(4%6!j+T_8y$HhFKVPZG;A<v?&LhY*qz7Ai>IrL
z$9uZbo*GVBg@r5z#>OX37)VGOrlc=EfAKNnhai^gHmAS8?}#-}0d;LagJ(9p1%Is$
zvGPc`vZk>e6-i7<nJ_JF!`9Rt<^72_jgs8gePB2dAf&F|+|LhMFWAG=yiBcmp4`%7
zlUO1mC8tc4ythA{gEuuGAmPRaMynsSKaEZ`c|2ldXJ@vzZ*FCE4iaka;P`gZW6L>?
zs96VciX9~K?;F`vIBcqxfKBfy&9R8v3)+(A=*;|(?@XP}9n*?Wt9My7PZmDByPU_m
zEMSV=jvW&J{w80zoc!Ti+J(!>0-~uGuBc>1v+Fi2Pyp?;J9myJD2Ruh-^ix+l!Er@
z4e9*M;^O<XEiW<%3p4ZcGh17OR!2lgG;cHeq9PW<;QzHT^v_*uqiz#OPf2Nxg<V~W
zMU{x5DzCqMhM%LJ-vrkO4b9C5FAJZttDCc8or08rNXmmpNk4v1SJdb@czLp;PmiMu
z%Wt)yDy9=F_HQ$5o*{8~sW*?97>|?`kF>dwhL({{70cU+iE}?Dwdi&|ew_T^aWV&Q
zs)4ENO38~1>*REp-A+05E%^Um^4c23;$NWQ%>Vzl%vI!WP7r95H|gM7)bUbV?5%u~
zsE*qIR{Luy7j-n!3};-~v|vYXt_9ofs)rWuo<4XOw<EhSM>^Cs@J@GaQ^Y*|#`OzV
zrATh;eXPJzde$ON=b>%(tQN&tZSN;PXJ5SQUA^qw`^WFCc|K8vhdD?|HTC4C3npJ|
z+Z?aUK3iFNY*R#diL7FP$f6DtRgR9w+}Wp<m(SU3;rMc`%d*aGe?Q9J0Jl-*%mLK_
zJZ<)}HBqcfA6z(il(|Fw!rkr)r<bxaRvI3Hx3}(or1HsX#Rs;HN9POtGi7l)H2=x^
zZMqy9TzT6=djkF@X@%#S+xR~I`>gKy&nF)jtTNzvHgli5mrrr*`meRiIujK<Ih=E5
zYbc%GkZY1#Z~4lEkLw<1kFwJRlRa@StWrI%Cq{13bt>5T;io`b%z6vH%&zBx$9KHD
z*rK|9wTju=*}8wZ!{`6@czWGGcJlMzOt1WF!0ix_%UcZuT;dajIb1eu40rjUb??~k
z3RZui3)h0y<n;6|{V)EkzDYuK`JJqpvfF0Je$x~#^^Lq$vrW4rIP#6_yN-8OTaHO=
zNt2i$lq}&siDl^wwM}QDPB;aKELr*0HBQ~X_Ra4<ZP$PNS@+~~PtX?K(@r86zk75F
ztkJIYlMrG#@-Z~uLi^xW^M8+zukXoSoVjAva{mK-KbS!EkdhgLjEuxI-R5&Ci<a^S
zar38q_^e*y$R;npMMg}#T}+~@i$_a~XTH5N-)Dv6`2`JI_x8+G*x<(hl=bBM`#fvb
z@Z8^L2paV{dGf<UW@c{g<_3nt-``uLS6+xY+F+I66uEPOD&LL=&(HIG{K(VJKlx=5
z1Gx53y7pQ|hR3?xC?hk`Q*y^7R+~Zrp8NLo5_Nx@-u<Zl-q$#DX5;+%5AWNz-QT})
z{RCw>0n47Yl|q}%{X`v(U65JFa<CyVAwf(!{hYy(qZ5q__+aJBwuXNnn-edqP2o)P
zNlz@>*wG^`CUzt)KvD6<p-?rot{^tUZONr(vqU~F6J*x#lsuutBe7C)bu+)bT4f`@
z5!87Lzunu^`tY>)n@)!5+8<l&-ZISdI6Ae@Ftu=|n}5UBGY3Qte&Kd`a$dfzS6VZw
zl^^0M#X_5^BO8)f#nc}a=|!zt?tJ2rm)H@NfFvU&=C54>O$^453RO8`Hf|_V{iS9f
z1ak8K|4NF54q{sz!;&6(ip`KaJU@OVQ^y*n0@b85Px|hs%vvV;sF^Q&C0k<6j{_U3
z%`Kz<PJrrL8(i^O?eG*$jo+LS1r=<Qdza0NYFKb}MvP6{gP?wOsR@!IOlKY(I(1<F
zcZC;+Slhg(Ym`DAs#{p||J;Tw(JwZvHc#c4-F7W>op(4(WS+;-68?1x$3pt=yRA_Y
z`Nf(obcDymC*8on;LqRF8wwOpp6)b++Foh#@?C73WwFG+KZ;sYHfkj(Xm>u)RJ<T!
z@Sr2)k1G4J*a|PcKbF$Lw##%Y{xqq3T%6M$dUJn0&ulZpId;CE#MFGCVQpFX=lAgq
zC7Oq?uD)S?(?w~`!zM|Sh{LRw4IgGSFXNf%ZQz@{>bOa7!kb5l2X5}L@i3VB!14V(
z!!_~#tx3)BjX&uMzurc-T?=cA-`~*{7~?57JzO%NS!i|U4j;7)?yv^UR{MzyE&262
zPAaJRCYPGcGjYG5yt~YBj%9N!<4UMoil;p~A=tJxyVL7Lg#dG3#}QRO8)-fRWr@|s
z2DvW6F3*xoc~}3OVRtA?{3E051^xALhG%9t9(wug2Q*kd`>eaaU((82QbbfzN9V$0
z=1rE<DjL+~9|$>et0nm;b1FzTZS!jteD7(M(JlGG`kDQ<*4he>Ce4?(&D-APy*OpN
z3gi?}P5D1%Zu)tNTel?Q_8gdWOtQdSx#Nts%QKZTdpMpiVl+B)B-Kdo@Wx+K{s9T4
zqRg6_%s0Q){|8UmZ0S64Lm6}s2E+gVi(cGVI$2%v)-8#Bb*?+6_JnD%2xxkwI3#DO
zb4<9JaJA!j;*Vf9W@bC~WTOwuV`E;;w@I8~IeF1c5or7@(kT4%AH0$J(o&6T^M8+L
zII3AoKQv_27VmAmn$lF3`0D5rp~*%cRK&dmW^um1Z|pPUK*2X1)rp<Zag{}1WG=kF
zFIiPB`QxXglk>vpITN>UPhkD;IcM836Lwz~`{@fZS*9kMSxXoQ$uF~z>z)wIIV))k
z%aI1<P(uaPzWe(n#dH#G?3??`T?!J+iw-Gne1Ffdq|E5V34@48$LPfo!N=;C6z6vK
zD5mVq*!5w##F^BOM<N>83f8VYo~5Se|1k6XBd^O9lbB}x`2U}${=d<iyQeoKvHt8f
z`v48^MYpcWb8@!*t7B7F*U7Gzn9x_|7$?`ZQ-tIHuN0pK(@uf?0uwy;&Ukk)CahPd
zb=r&zJjW*=`6w=(agr&h^ZuTb6J~VOosroJGVcHXMbGvbu(Gywak16czcBLOa-?ym
z0gpIyw#vg(TbGDAMKH=adqi1uyz%(M?X<B=AaeNu;o~cVdA@$-d4BF;34bfhl$($F
z<qWP|H~#bQtbkNk{!2?11#ux2SIaO>6GyJqqA}jSANwl>*8KR;k*vmT!j_g>^iAjR
z5>KAz=O2F;bA_7nuP;bavTct|+l2tOygV1><|Ut)&#9^%n!tEat8-5hoBu<{Ak~g#
ztj4MjGs_O#{QuAJ&+p>}U$va8tq)IB-eMar3@+amoh(asbZpyV(sn22$jci?wpebs
z_B!HqKF0#D)~5RoMq-N}>-$M?nA#Ow)j84m_>^edx3V_p_7w*2yX2uRxf)keR|jfy
z|EqgZ=6XV@l#OkYFgx?hPz%FU9?6Coj~J)V<_t4DqSE*OzhvG2G>e)U3-``__{_Iz
z&QWfNBc3|4^6;!$#q;tdPhujEj7&!g<NZl3`_mNjby_51cZfXha$1x2Xyw-r5nZce
zoqQfueD4G8iD%`$z<*(pEYuyjj}j6R1OyU3d{6J_Nii^S(X`=E<CrAGzLml7V{fF9
z;7ldXYic|<gpTj82c;I9niCt+cn|L?<>8l`P<C5L6=DuELlTbzt8nneZ6Lm<tDnm{
Hr-UW|I2X+)

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..8e78146d955ae8f02230121e6314f3285e87611e
GIT binary patch
literal 19313
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3w(xXu45^rtlpw*nn1P*v
zfq~)we<{|*4C+KOkMtkpJ)ppI<ekvLf4`lQ>_XicPBY)AlP%~sv`J5MYpKt@$#nBn
zh<WIw2`hvy{N;ReHmYR5-Oi(oAd8Ni-w4vo8pzXkP*n7{zDK*jrVso#mG*GH`%!kF
zKC3n=I8^ZAoM25)6+`!D3@sA<T8!@{6u4aWUAuYvYyz8c`SCd~YS)+Mt^f0c8D!v{
zD~?<Z3IePbB#-O#FfDW~`M;Z0w<StqQT?*&lJ?`jpUa&4Sa_szUJ}n~qh(KmC)Iwx
zt@iqq;)d@!PD*cN3}??2x@a(C)m5%EWv6VejQ1~Mqn$NRM4sOqFJ5%*_qCt^?R(<q
zKl|&+gM;bLR70*70}&VDB(vT16Fpeo?)|?~XA4K>qSguXe~Kk`?`;aUnep#nb%)Hc
zj75Dfp3YHyb?l<vY4aBU`@0h>{3p*%7jF6{cdvVE0M}YsFURbTEx+9b|7b82xp@gC
z-~GNT*VOIY%=4R<9C;np@9ZU*wsW1>GW9oXAb(!DwveyIK!D}K+q|QkCt8bDc)1S$
z|Ia4XuXNCCMOm3&!NVV{cYajPJhQoQ2J2M?p;>Cnt}OfV^7nU(ZI{1DwOuIfnW$tU
z96D>K*4cBv^x5?eN%z(7=vI`qXm43sp~=VapfDk}HqzjH>HX|;rMI)r?H4}W&VNcA
zWVO_-PUpah{y{+=US09Qlifq7#zxGT+TA^~zrS_<{Lc3F#<_pxm6aE@IA}{u@Zxy0
zDMOPjXoAA#|NsC0|NVVGzx=<w)!*MA?cRTX|Np;#e!ib;U9TVW<NK47``5?+-&_0p
z{qg=rK4nmv1*OEQ&gRa}z_@ww0TcYAW<|%(_jhWF4w>Ta*%=%(Y2xhP^1{MZZyOG}
z`e}cd(zx8>XLs0j1$C*EZ~N>2@2ma&{l&%or>Fn7tNZi*aQpu6@Bi;Ee}BK7|Ns7)
z#2^1^7clpgKvTI<XLE0F=luDTXU}$aYVwYc4+#tmiI4Y)of{e<p?j^+&FtNlPyCTG
zD-{j~Jw9-Xe@YX}J%>gHwzhg7pa1vw|1W=dXuq=izkB=se!sf<Kfk<D&Hr;Bp7S4G
zAI~g1#f2Ga%Nvkq8mCU3+1cqCG$}ed`qqVjb7y=UJUeH0cX!U6J8|yZ#+g6m*Tnvm
zcVClSRH&-paq57NOhi}L|NntpQ~&?}?-V2&^LM`e|9v$-zrVZ;3NhQdKl#V|>YtxC
z-cx`6!xv3vVd2C7|2r>m1?BPo|E1n^Iy*FkIy(nBJ0H6g<PteEIATVyOG|4z`$V+~
zR<aj5j(DiPaAj>XTBG^cKI_qe&?p5V-k@EMf&4FD-r4#2{n^?7`{j+_+&*6L-|q13
z?L71C40%3%G=chPPp7kAUwB;Hr87QG{{8_$K@PEVy&WFdtGFee^@%yk(CEY@)ZNoF
z<F4=@QQo3Q4*u*LzA#?@J-xA!@t$OxaooNXi}JpQ7nR#?Z<o}IOR4xb=i%ey2cs`S
zjW6kJo;x=(EX={r@5l)cA8+qNCp>)o`esg@I^)U*`JMSp-i$LenNPXuS#IbnULdNT
z@s<19HO@)1_IpVESDLE*Us6J%?tj{j%E=EuB(_PLOXS^8tN76H@U(cF<Bj!<;7ly#
z(%C$Dva3r=xU1`pBQDpjgxtFl5)d)t;s1IMb{0$42ieXBMvFukCLC!L{C-1QDyKKB
zLq}zk;xfYoUzruWd?Jw^ryA7z{{R1PY+&%`=iv|E^$#!i=h<Cm_~!2E506+6zrD?~
zzm9jEA1}l@5s-DQ?d`##p%<=%Ts`jY8#{OM<jE6zdl~QPyZQ=bm~z`(P&|C^|HIDa
ziGn-dxp$UUNH*W<IWWuULJ<F!1PAT*3sN#3nIb!*8nY7<5)yv=pa1Z_eOtf0<i6VE
z9d&aPkMXpXzn6G-FZIXw{)f-cH}(o>K!fwak%OmB9XWPv)k4LLymgP?uV1v_fTG5U
zb0<$8IPjr=;zUNvTN}0-Z<xmZ!;9n2t_@P3_?1%^m?TUTQu6wu+^t~hE}+Ra$!3DJ
zj=Y_LP5rro-)4uqM0viyH@2xf^x^mS1Lon-Fg<bP;Gsh*sz={GNqO*m`N}nCR5eeY
zJ=(g$F`!!bzu_WD#nxGQ1)|I+*7{XkJ029Gu=;pM(UTzdoeO`7pG=w4$?PdJQRe|?
zYh&ZXyVh;&{1WTp({F6+efY=~<g<VOWKYQHLqc=Ik%K2szIwX!^|NKE2?|Q;$4{R=
zec-@>2CG}W)A$6gvfTUB@S~+^^B)J<GY(<ROOH9~CM(?8KZ8xl_pxJ$(_u!-LWfh6
zw&zRc-A%Hno%QelW83O*NxK?}85~Io2@DWB8jc)1dp0Xe>-~cTx#{Yw*XXF|oaqTE
zG*DpI-nPX_A#39Dh(PPr8XNn3589sz<A|zfiGFn8)Pbm@3r-ptESM&rvGU1N@3zm+
zCC|-Cys^3c;Z5VVe)*U3fjkiN7nMJnFj4T+=gZ|KAr~(TPoMH6A+60tnz=|PDYtIX
zQQ<UsQEl;Mg2uc5r>7i=+Pq-9!Gt7{R~{Xo1ELpbJY}DEr193jqd_|ry8rw={o#w|
z;f2mT>i$MDHYYwjVx9ZA3lbxX;vWeJJ~S~7E-bsYX>)K;*t691^n`>4y)BKV`VV%>
ztLhm_$!kPP>Z<<z|6ly7#++n{6U-|mk|q8dK9cbM+jOP2^}5;&ss9I$*#x+qRM6$;
zI56Mk-``}5>S+(JXtwc6OU$!LtoS`G&+r)3SHX`00tLT*H!ms(FgKg_<Z*IZ%Afuz
z3jd5B%#l3U<2NPZ&;R)+lvuKNw<IiPu}oc{;jqem!es6$Y1ZN$vMVEh$!9-2*f7yT
z;m!JZ$+9;I7G*t&_wCx2`@a;<gausXqX$n@O-)02ISdL4E(C-;NlQy>5MQ<9wV=yg
zM$Wu5DUGJu%3qZy?7R`omh+5f9oy=Y9Zsy7HoSS<(~fkR@*cc-V)bbWiGPL$3(QRw
z7_y_E{I7587MJv!pBAx4!s?x0HpDMmq71nj40!gujwvV>`OUC!$1jG1$G;iYPAGh3
z`s$8q%afv(33sM!<Ue-WXzug6*xQ9`4BN~mUMN#y2nuJqHf`z>nWa{3AuQ|eOr9We
z`)$zOOoqAbdvBiETzlf`Y{~z&`zNv_xJ|m}W%k4Ry#IyzDNlQ*7@XH&-O4w80z0U}
zU6goo;bLV;$;`)(7Oh>Yd*sLwhVu(HT-42&*;uV5sw#Z=*Txx3S<eOfZTZOG^N`CX
zlxG?n`?WmXX9sMqJ>$+%U|D;XQQz_7RQU{EK@aDzS2Ww2*(J^L({|L(E)zTf@#3P;
zlZmNICrx{nk-b_^^W;H>+zKNFQRW^V9s#{2e;Qhj*GEiC`on+n_KTxz%52ja&$Xt;
zOnJmOllOGrhezx?B^r2a7QA%QoF5?DA7i7yWfQTo;lZIfEFblkENRJ3Z{aT&(6N`8
z&?S(T#>UOfR#?cUs@i5*ETI>pAR_q$R8ukh|G%i{Q9?>eLRwl;;l<OZiZ*Qudh{r%
z<)~rJHd!H=4z0F{f<YJmIP2}5sLfZgM!`uau;H+<pv^u-S6ih*#bdJtqBcw_FqSl!
z#Iec3>s8Cn-bO}S*26ao3YY!=|9{~zb0@|ly&v9&ewflY$K}GL?ny$;69k$&I1XP~
z$>ZLCDdnpGIJYlKdz6rt_T>5VFQ1M5y*8N{Oh`=p*+1uNV&D(?opT!}%1->O8Y6a-
z<%vY9LzgsLj~Z)Q<G}{0-31MEWCS<;|9{w9+$8oyDl_vW5#==vE?bNrylXE`JS*ZB
zVdu!O##g}R5fc;7v}rs}PCUWOjbtnie0VE;`2GDGi`#6VF%dPpdFoV8uP%Fw8(Mnj
zyj)#dW=dX_%9z~5c!}}g3-<+Eh5jw_TDaBH;J3~UUv`~bzKE~=m*NfvG+di|@SbiH
z_o8#(le!%KHr>4CkgYCw`B?+^qSC~Fe-bM`wIrSrdEp=_nqC4e6#PyeJauZxA|)}Q
zBw68PJ#B4<lNU}LRaSm%AHkR0v{6z*pp{Wr=g7jD5^>E5PYm2Lz8%S^U!Zw7u+~6f
zs@sHV0%_kIR<wx9oe1g>;s}yT2$AWqVtwYvwnVF+aixW$iS-6Co(zV#uL&%6&3kN|
zD=nGz^$*|M%fl}hkb4&z-cF$K4ha?Q?%LGfw@pC!aa)C4l&VAUv3fQ(H8wW3h}J_a
z+=X9mwFt_Eb$pWHO8DzG$FcLuQ|G*i4^$Mx6qZI=aGE~+|NsArP$8?%H!&ga4Ga!k
zVpREfP$FY)fy;I=<41*C%{GJ?`uOlDDe<VP^3>Gu%r-N;v;X{ux6;!TltiK7Zc=Dq
zWOU+mcS~E;(IXKZT}8%*h6jV$bS^x2$mh>ELpPw}Kw@8_fl{kZ(b|5Q;{Sn?JRX|@
z^ftL&31ZPZ$KWqq(7kbMpg@zyQ3mF1M~-s@H(Z*~q^iDU9{02~O=h>C1CzaG%#hfz
zL!ziyG9ps)+S=qDh0PDo%g0P~(}6poqvJzsYvj=*5nY{y#zsa5_wcR`7pVMbuj9MG
z)bjt~MJ$mM1a+?jBnx^6^J(r>Na6?;U$@LgIqFiQdG$o8*&1?=FH%~}JtQaW<q>w7
z(H6pS@aq5LQEUlJKB+q<@d*nvD=RyvsWD4SA8umhDSszmCSwn^Tcfa`cyoJu^s!@6
zp5C7iFY4voDv{8z<&A`-q=IJi#E<v?|L@tbk+))7^D*;z2b^R-J8&pxAHULa^O?gW
zC+!;-Oq8ax^Q?$5Nq)#RAzCzJpR?b)7pa`Hm^)q`ciJ(*h|RZa=1fJ+&c@Ep#9Jn9
z>iz|7>V|NuTUutQsYGaNa2OaE7#JBXoF>x|sJY}X3y+5=yYiYhfBVI1Rzyz-v}(!V
zYdESG)^JI%HB{^bD}!>@oyT5rC&CnV%Xk-yW@gM0^tD}csv%tOe_|y2JHa^(OTMu)
zGc$8@JC~Ro-e1pidAYGoz2CngXNb*nV!+)`V}nA6{DilhwvCO_72V%08g*qgPE5Jf
zcva9TVp?)#La@v+jrZpz^q=#s6<shpG@~Jz{e!_F%jWXqANHtqm!AzSXZiR;QGA=2
zV8FAt!E0O@c%lS9iJBBxE!gq=g;V17hti>`39)r2{BQkmv9jo2<I17Is;QX%<}X(v
zxYL@X<jk+FJ-N3x$E|I5UmuIH@t$?ADTNV2iB00_6GWK(#XJI8-FZIn8Spr;K2(nF
z={fP2X@$rUlO-1o+<cq^-X3DUAo5tqQS-?JnT7&ppE-gjC!ToM@+tATpW<p4M~{0h
z;x1e=Z?sxlB>w!7xN}#srL{BbP6D*pRZ=h1(Bw={VUd*b+O)~&pc0R6>H-zPD<Tat
zM|f6j?33BVG5@eFf4Z{w!cv8eqC7n}cFZW`Pfzf5j+oP{c#eZTBA7o)si9iNSo6T%
z9&N`5UV0ghD%z(7<jy4|95`^Y^P!>=B)F9<3p=_#xwu>J+;O6~&`_D3G3!)<lb>Wy
zk57PTkB^6A=ZRk{_Bp-tOsbODJjXr6)+4Cm6lYAXf6S|mY6?E)35tQiHaa{L7^J4p
zIK(`&d;Zx22M(M$vtaF60r~R_)}3oGOY?=c4V2cNJ$U~7ySHoS&&%WUl#~cO-Ekzr
zN$5Ab^p9WMf{wEmx?b!!<(#dTz~t65r7@KE+CL7*10B|p3naU{U5wdP#RYF}Q#he!
zA$$1Pv4q^SA6^K88ujJxCyB8@{i0j=;?=36$D%%eGBQ+n7Ou!}^Z$#PZH<kcjSu;4
zVkR7VVP7Q8lqIms$frT(%YVaNLKzJHy3H}d3$2;i6H*S`b2&Q8=s}E$ARAl6j|0+6
zdM>y++>mXk_Z9ebL6Ujfo+C$(K6rAZ;4$0b#qLYao`DqDN(-YVP7~wdF)=d{as2eK
z{>nzRCxPB>71_-bKiWs#WLO_I;jq+7)sOO~*Cv)K&TiIzBxxw};B|=r<A)IQ6S`U^
zEo&wUYD=$;ZM}KvQ2a%O)sDuzzPp>4oE;dN7b<4nV*^)#N@j%xg&V6XGAgT9ZQ5+C
zsLqr!-@w4&z;D41H;zvGJ@M*+y^aRYHcS)aS#i;!=5FJ1sYA^Y1xL(h85@KeC2SCE
zI`NjDt;eKM$7zzz9PY#!vy58Wj82>okagnWfd-ygp@%1DW%X+h&qC#;Y%VzmkC{(g
z{r|s5_kV$gi68B=n2!61Rwzkqwm7(od$VYh<l>CLZqbC^kUyt3-0Da(H4<-3c-h?8
z7;vz`bm{{y`2s;RGd6Q`Hd|YvsAW(qv_PI{YnPUi_TI3`=s*{5MOFiUkfJ;1uUi~F
zK`fJ8IyCe-7;FrCP1BP%eatwREX!=hHcKaL>WOU$VN-Z`ZUnI%JjEJmC?ziG>0uxu
zI`Kj=G!nE5B_uqmD>Aq_Hy&hp@bZB@oA|~fY?+4-)-UPR3Rs?OqO)O}@dU;&%`;9-
zKNhu0I2cqebk)6fq)T7(!4`?da}o}CnC}Ql%B^&`F>4l&o*qw4jmu7`*>fy@{J2q5
z%WGm%F>^*hnbN_V9)kT5s;yfJUh7}lrJvHvbE2sw#?ePfLI2lE)(v8@JQAABn{OPr
zwW(3EaWO~YfsWD`Q@%}sJv}^Y*6<uX$}?*gkBm&qUQnw2|6gg2#hNuDfB$Z^v{d}I
zz+6ATe4V5DA!YrS&fexLG@1E2^qmTpa{3r=)KX(h(B=7{aX7R>AfGjEX~l6LPBT*j
zL3Va$7nbG)3W=+s?ww%a;<|Is-rm&InFh1OBLrSe3*Qp*Y5qjPTz?&wgk{WY+7@d%
zBpW;kXmD?_Y1kikz(;k8*-C*l!HISf1ydw>Bsh<r>~xHuU+@rappUQU-n~x~-W`~y
zbKzW$OIC5hy05Bij*L0xno+;o7>}t+Pgp1(9@(`2gTv#bTi#KZ9R1{ZEN-nm(e+V9
z#L<y0KmU*}KO_+xnr^XVNkvWV*CR(BRQp^IV)?Yp@XOvL364lE<9&ik_5c6X|NnP<
zbNc-K_5c6Ay1IRP{{H_zn{Dp*OzZh@MRN*63@=BM*2jz%BZb#T@9$0i@!_F;it7FI
z^N;B5Ja9SsplY}1#!W^Nl7=&87&x4>I=~9G)x2iiQq|`V6^}AVe{xzYB4N1g<l(Ca
zgk_ZPTwF1+YuXHjiNE?AAO7E-zh6J5qUz0!!?xA$=2(4gnLXRrug~A#|H{3%YnOt8
z{QBA(87(a~h*|zG^bQm>NRsGQYr2>C^Z)<<`|El1<Nxih`T70l=l$zqe}De+5;PX1
zscFc@X7u`n!HLtx1%-m?=4TTAaPxkAQ<#*JGGSukhK-39Wjy!8K-rw(|NlcXEm~SF
z=gghz>hj<yo3fkgArAG4b2#*!Bqb#zHo88se{!Ya4$IFUACI^3hELPoUGU(*-}wFi
zj0_C+)c=2ca<c#P^ZPAoer)XSuE|)jYUWG}Zf={c57~FjGCC2nWA^`K4P}YHJB~;i
z`u_Vr|KauZ|F`G;jokX`_O-S4`FDOWTLw+)?0opdv#s>?zp5&u)^=knYvYbCp-*B*
zVwUVJ<k|3~X{Vhe|AQwf1_lWiE+$&k$vh8IfQIl)3lY(yZ{HqG6lGV=n&sdv@8hB5
zWAK&7gWL6CV{6QQ0|oaR{qp;_=j^<9vGBvQv)3~`|NZ`MAHIIy&iem{Te<a5pYnP0
zX335nGbWU$33ITma6OSTY0CqxMhU|s8<)@f|6g){-QV(OXV%Y}<)423pVPe%rj@>R
z|59%3_;`P=HBbIM12;D#4o<<oC^oh^Sr>$OZ44v>-`6Lmqy&VfUbvoaQ7e`H2<p~*
z&mKHU*|ufM<mt~-9D3Rh8Qb1Uw0hAoQ82e$>RE>7Grf)fr|a3~XWgoP%vN=(p*jEl
zKN;(9Yj*s2FirQnUG1;dUCiI?*$xS17!*#L)p#V_Gh((Q&;S3%H9tP&AMca*^nC9>
z?@w#??u-Bb^MtQ6@R|9k{&3rWmwCb6TW>aM_ATFi`uOpLv||d|-It+*s)y!TsHn2$
z<R0C*V@AMZq2e!}<&~BHOt6o!7Bk%Wf1=Q-dbYOu`}_ar<lirU&etg1TlfE8`R{M-
zpFS<F{$}&*1LMwBN8(!^_`98$*cB=KkiErvb^N|R_5}~zU5~c6^ZysR<5Be6>~It7
zk4;9qzTS|U$K%n;6;&)clh;vAjc3*@i?g74hX4N$O|qCVLn1Y`b?Vfg5fK+I9jV_^
zaa2R`-$M_L9aj1QB_AH_Jv)2-|KIKwv;Y15{r}?P^Rlw({4y1v-b$zBG9BEJt{`&u
zhqhGA$&P}h-#$P8&&*!uGvmY0cX#cNd~yHxH`$_YPU7#fHuHQA?KXxrk0odD#55}%
zI>eKe#gmi669Fl44ow0TxpU{Ls;JxuY<?hrY2TCj6?gt;gmNBp)UN-yWRiXTKfl>!
zf8NJG;x1_0oF2bErt<lfmD4S_inmxCzWU%mPwbkFePLz~j3n;~xi2_9y*~HuueTQz
zg|@Bx_xJRMT<OET(o5zpZ;*WLl9ia0^x;!dK+vIepezUu02Y=VrY0^sc1lVF_`P<z
zUD~&GS;G>ZHJ*&CAAn+^{N0|NH8Y<Ci<G~s*;Dj%Zut6r|Ns8xk2Y~;n75}d)Nol`
z%uAknrDV_`;NRc-t;=d|@0t0;ceQ!GQOquvLuuh>1cTU|SlNnRUA%H+=Lu$T1@Gi9
z<Kxp)RJ6y->yO9urGMT1R5OyFY%qGP?-u$=e44_)BsGpF(~jTV?5`jjKIuR^|NcIi
z%8iAOo6pa$PhP^nwugr;=VRCEl0*m5<01#Q%xY|GY@9gp<BN;($NT?ReCtVB|M%C?
zf;SR}m-{byc__3f!OZl+4jmnyZQE|NM?i9glfO((&Y5Y`gf3k=^I@NMkntn?ZFl;M
z`dPA?esr%r;9#l#<^TWs_3`yTe@>obyTAN>-7K4*uMW4*|NlSzK%kStamk{<Zf{;G
zhn1m)(`{cIJqem!`2T-@^6~A(&rj@0dwxpPs6X-Ge362cUp^*Im~tdZ3g&egnKh?Q
z9h$)W%254Mfr;7wLs^seSKMA$Z|M8`^Yi!n>;K!=Dm_0i@%{b(ar^${-QTy|e_qYO
zSObHE?pU@QPNtn6heJIRQxCf?cyW7sefYXRI|?3N?-oyqv#)v1XIXvaLx|Fkokm7H
zmo7!{Kx$nle;FgAJ5f<gH@G+}G@I`1amZ^`aQ_o8)5>1u`04w5c{Q~*`}&TIQ;!*r
z_wC(R@=}(WnfYE4bGB{M&AbCk+4`^8hp_VS@cg&0{Z;<sL%(_ciTyKv9p|6aKhfp&
zkw<~eCsR{WHf&6M@XRe7RPp`)@8mDDYuBPrpL8UaY7{9Qh%`9R#k(ey^V+xL_I!M5
zadBannixf6_WgPH;^K7sItMMaH7A)?bI1tH=aUH7njFQLXuPNT`}YeA-@m^XSd#to
z`g)b@gI1T0HU#cppxMR{9nDr(S7Hi}$WNb?YHBR1l_X5Ly{y0duYXd+l-<m1S?kdI
z^wFY><^MlFzuzxc`}M^|@tljD`wb49`8z{u;?~D2O9H;^tN;J})z$d;eF<T<Dh#Xc
zJZ~-iBa$8M5+vjeEk2z*Wo&G8qN18sBsN|<;oz^aVI5ag{!{&BUOaCvsBy^4uWn!v
z%)VC8us;5O%<gX=pPlVjQR8?rrIDu~RMJJv3>0qn_y7O?>gxUY{U`iooSZD!mvyj2
zbaOp<8DpXu7RDAD`la<a)H9wkMMZzAs^&;UvMjZ2y!OVyPwD^v<ZB8UynH%MMMZmF
zy>d!vKXiXTC@N}SUaD5&VqWuzXUAn;=50)m<@uNeALZZs^Y_C;`O4*T4h=0V%{we+
zR&D$izHybKp6iYs5<I*YtwA%e4FCT-dCFY6)HG=l&kDt+&AObDZ=Bv8e<b(sP}ETe
z=8&jaw~x3qotcmT2C1ogik}<5(>NX_p5XQOu-M1{9U8_LT<z-r9Y5xFk;ze2)kv8A
zV9LRU#TQQ<JgA`Dyzd*NfOYbe5fb|I=FR^JX+Z*8nQLw^mxer+SAHD7#H2CO?}3zs
zy9X!d#_DSANy0(~2M!(j^5y0B<^KC0^`)%|?33BLW0vfRsDFvxyb+f!_sjqP@#BQ$
zZ6>~+UP%|16IZ-9tT<q0aQ@7h0|y)TtrCIcu8+;Dw8USXxmkbZ^{ENck{vVJuBdHL
zX|m+_Ccf3lF?-{`)Px_;UumhUxYTWa{Qmy`HEVqO<!X=4=KG)`^<>K-hb>7pQ$^(E
zf2y+U>o+e~Z)C0c!FX#I?;nPFGk*M(Y-tT#0*xpYK2A=*)YNPB|Nk96#koa^Me6AU
zZiWf$`zA~}Vc+wmvAa=*u|6_FqN}T?qT<Ax%XL3Lxx2Lqe+^=JvzAdQ=zRR1lT155
z9GF>>k~&jG#lxPLneEIOiJqR0SvH5c9;BaqsHJ{EN~VA{DT(LPC!RGP6XEv8#qk*$
zx;?qZ5XIZn_F&TlBj&9|KfBjXI54YZ!PDT7DYG^kze`oGU<8e|Jg$$7<Xgh}dd5!4
z`o7~m)i#+g^xHT)9DaX(K7XAaw_$W!LE|2qDH2Me4?IJE2+!%@XkMVeEF2UI>YD%m
z?_^+UapT5~e~Zi97xw)B|Nr2-YPXq*ic7crIDX&*`}d5O^{lM(u7wqUI%f0a=}uOb
zh^0#{4IT+Cb#n3T%V=epx1>4J&@jQ<JYdqypZm9&ZSLWjJVV0rEMImuo4&r$mxnAY
z(b5H}ckb}`_-JH7J?JN+^Z&oGk%C{I`*%wn%|ETV*}uEjR)1oTb~-pKBS1>2r|18c
zt*p0hpFR1(Uifl-eSK&C{kr6y;}6ycbC@X?T-C1+6kUAd#EBpOkN5vqRFrx2NJX8m
zw6txT8JpOxd-GcP9vtNq7WUeB@W276l^-lDC1=i5^iEk=o}D~5;qZz`gKE_sv*!Qb
z$hm07%xTL4BQqT?HCU;uPu{-0WrKnCCDX(V-oE-9J7#Qd$k?^v+vW#fMXo8aXGy6)
zug}g-<K`9<6cm!;6B6Q?I+f?zwIi<&aBc{b|Iy9Mp{dfc2-F4r|39h8d-H}rj~^r)
z3H!5%XM*!8iJZG3F>iTdjvlk`x$*1t!Gi(8i|^maaH;v5l&1FNrRKT?2RbJ{-ni)h
z4JC=SXAa~9CCus!TldHL>ev3>xAmT!yO*gMPdxsku-JIZHX|3;iN$NA)eiWuXRHm0
zaaOo=Nke@Zr0#7};|mK@b97Ym^JCk=ADDmQFZ*h*4Q#e)+#;))o~Sgf{`j%KyL;n?
zfa23)>WPU=b1l=-+?197Z85177f&=ZW4j`8$cHyVm@8~~lCN-t>x7#;C%A6h_@}L5
z;5YBg%2lUaHk6(`eR@NUrLz;0>)U`o491!Xx*jPh0acqJ;hof^{q#Y?qeo8N8}>_x
zUt74<_``JhdP#|Y2ci_~CrbREkg~BX?w_Sq*L;hIfiY_owNCY!_HuJ?Zf)_{wl$ND
zt?RIuQ?jn&g-4(4%~CT|86AraN)tWHa&rEgno5f4KKk$}>Dkkxa_MW<@Z7t{!_IH?
z<JY3=-$j^gjxsZEo5C2VsoQH*2(72yMDXxUn{O98bE;@nj>el00cvh;YHDgHzL~1s
zcp2*BaXiBCK%Y%|gYH%LxDEZ!thPLjIKlJe57*OwKR>@e+Wr3acKhgUZyw&<{NBF)
z-{BJ{6x5Erd!QgBto-<y+Non~;XPuPcifs?!Saxm!SL0_Ywj~%zWKku9#paKuK$1k
z?d|_^w!i*AbZ)EvFCif*siG<w6C?5Hk;Iv^k`)yloq3lI{QuAM|DWQzQYW{HBk_9@
zK7327_#=_U0V(Z{OxwQw&?Hr^4Jij(=5RbX$)wg-cTB5=(OAH^FhMQh$1{U9Dp98!
z8XF(p-(UaVuKwTO*Vp6g>cqmr+<J0<yST~KIVPoWs&P*_(ik)C&)?tgKR^E;zu)M?
z=gTUN3e0OX8NxR;oW8gJynxiH4=)6-fCkJ#6~~s}|NbQY`2S!2k5a;ojWZ@J7Wx1G
z|9O_~K(z>~1Sc={6Yj!~JtaSU{e6OAC&xh_`G6xwzRaILUvk@Bq54w}-CArxT>NY*
zY<00Gvi>D{Bzo$dIrE>DMS_LpL|1MQ!?kZL|Mv0;-1MIxkOdvLb5iMpbd{&ip3SHk
z^pNKXH^0L7MUHw52@R?f_pf-xIZfvOM3##}EP7^9cMJqh9bxHaj43?t;K8_VOVi1}
z+{NlPGx->P_<vcuXukd80!YK#Nzd}%zWV>|-QA2uBBpDKezNNu1k4usFPR|j=<rJ9
ziuGCpPm2ROHvbk+ijb^On!)Ivr=}*GU^-bPwnB(~)|CSclg!*}f3*|^L8C*Z@BRJ%
zd+Ps(N6qSNYy{2UA8hat-VogJuzcl$slpjb5uPRK$`gA$RyH!ahZqWUm9F7rHU76@
zR`XRKpQ$^}xOmI_H@IWQz~JA=xYF7n)xOR}@;M~WIH~mM$Jgzz{QUfXy;wvoqrSw9
zfM90ktX~IwRC!v0*rFabPE(TzI4lyL*{EGCd{ahIEmh!3-_17%eB_xKE=*!gYZtp@
zbd*Kje$kuhQ17est&gwYU;A79+O~#ik`vN2*O;E-*sVR8^PYdBtjXz)kmUvmCe{VL
zO3LDLCt}u|Y+_V?{O_=uN7*Tbh|>Z-mH#)W%2qM2xo)7ad1K#vrvuCVH98?>jFX;a
z-QN2D{f!UxnVE&N!dY)^Xy`OPBqXrpTVexGk;2_`XB)Pg9ek;&a?D=EY^7kE_N<R4
z_8+e(pQw`DAlbUQ?%*`TmrNVZ?K|{)^^b$g{dx9JFnbAib-#SQZT&y?XSPRjcV_ek
zM07QNFE!k4#v{?ha<D!`XlA#V;s3yHE;gki<9QwG`?efdFB_Z{z@T)QMP^S`Ymtit
zB=e~B#qa-TTg@hUzU}_0Z~U663seQ!Dzp@<DwyZ%Z<v^o$rOIZFri!bk*kEq`s@PJ
zDz;7GO>7=s!UZ4R$$I63##;XWcbfU<&xeP<kN3~{^M_}`Gu9x>gC`nFig|*%g=X<3
z1o23?axk+=-xXhx^CYKzrufEV0{^-iPJGj3ewSxp_%dyS3b#j-f^y`a|K~rv6#nI>
z1&wV_nfLeqSN;3LZ*R}Mq_WZQ(7^^1p;JdroM5n?!EJT!V1v=&CY}$|W&Q_BX1r`*
z)cn-ZuluLz#Kq2d$&X^ZVhxsheQF({s?5xw1v44$)1YDJEn`35=I7dV=Z-X7y5*qf
zq;QjaXHlcL77q{4hlK`gGeo4Acb5kUGcsotB)h3hEIi+kJf&$#i9=6sv+M#@4mP$o
zQ_sD={z4i$edpvUGvEGy-Jc)g%*@)%hyVZoFL+q`!^D7?WN8V_<`XH5r3PBe*(V)S
zwiqUKJH=!&MGE-pGiP-jNj{Xo{=-}FNyw*Prd21Qj`Ng>-}k5T#|QJ8U%X+P53FCV
zoEajb&Aj_};S|Y?Uf(SdKl^XR9GHKab6d2hYqRWxTmC=)&wu#T`^yoy0r&U+-&_9v
z`2GDh6LyN`_y)2?h<=vOR~MVyUBD(Mo+Kj2BT>r5eD8*8s7&@s<B-X$*V<SoO4a{M
zsrbh8*Fq5<`1Sw()PH(%eYW|36_pEYo{|Zj43`xC9zVb!|Fy}2WoP3d2I0oZKV*N)
zPwky4GC?w<S*GHVTy7TgeyKlhf?p0zU|%0+Xj9QJ&jM;<{PEvkUar5r{eMTdv9W<e
z+o_4+`G49)B{EVunU^p%^`2Ffuk_m?{QLFw|HA6~v@|!KJh353g@Y~VlgYn_Tx>yE
z!faPcobx|?*FXICH_!inh8o(z=_d{47&O!5>{52rO8=cu1G8$!k6&NsZ%(hT{r``D
z(*cQXKR?>%JYs12XJqi_@Bjb$Ifhm-ALRQ?U(7h|C#WQCzOTQpE+F{xo6F1Z&)Al*
zVWU)QX|D0CEAc$V23r@eZIat?>Cvmuwl_I#|LTPJ=BV%=&|WaxTymbR^ZI^>jfXa^
z`g&^W{P*|&|NhQDOYOmJ!Gx1f<+GX`ShRIi74tg|^%#gu{Qv)d-S2PT54Z1s^yo~*
zVtM~xzuN^5Mg*%bvy8ZFcHkrv6KH<;V^iYiG@&rzN0I;ASMmhEU+NvfHw)4dJv7PU
z-TnW6KRwN7)7dTi(6CEx(+8tx|LbF{b2g^G`~JRuzV-LsnKK>z`+M8l8Ql#VRwo>o
za!5yX!!ov*r)Lu~BQmm#Hf%E5Q+KxDkA<kuVIPedcdEZj81$z>Ix2@IS=`(I|L-&2
znrtSo8wXGEoba5Tv6SQ5h2ygV{vG7qU;qF4w%qx>y_P+_H)qT!nAxyZ;y@Hbf^WqC
zgsV#HrkHn3Vr3~PGP1BX-muB&&7DpoUoW8p!P6EjEPpRyS9c@{?u&hOe}8{^I)8ut
zL~qt9E)2I>JJla?F4_-Tn)v7E`9rPp_xAmL{r>#gB}bZ+6BE}4Z<xmD5@yo8am4|a
z!a|-sdw7ob8Q$6Lz9ZXfgV{l5(Au)v*$+>P>wuQlGJpqk_SOHd{{HSgzx;!P?UPPk
zI3?6`r*lrjzs1M&x-9Ge|GT^4;pb0J=jY^D{P}Bn^Tq}-`yE|MHe31sUHL42a%;1c
zhqv*M9|m)*k8j8p|8+w|&*5Ip?d|zI>F161)b+ZGfy>TAQAL0D*Ze$xdism6tZBSW
z+b&B5EpnK(PJ)v|(EtDc|9^jff4|WAepJ-G&JKg;Pd=PFVX#n*<IE+d@LdO^<~-1>
zW2>Ikz#JFH79Y=MXV*5<SW+)mY1M}Wh1DFD|K~kC!ue*ZIK-lT?C(!buJ@n!C$XV4
zRADXenH>jjeQU`%f4tDcF64Tr@O^FV`*r{RtY6V_{o1tNnUd94nU()1XJ;MQSzmBP
zKjZ0<37%pC2hN;H$UXbu{rQ6bcFx(_hu_}*kvHMUe5L|bagN82-P_vvH)cadN)GLt
z`S<hl{qg($w7&6AOi2MXaxNVZyEQ?B`Iv#mrwy9StMuCH|Nnb``}^Vz9}aI$ulMr)
zdFYUZrR7Dp`3txbcXh~IPij!rOgX4HP2obr1_S39nZvEzJki?>3JMKNN(@e%NU+@7
zyyq6nBJQG2&2<+=pqVJH=+CdO|4&cfzrXHaw(uFf`}H|FE~-tXD;^(iw+~)c<L6OP
z{q<G8zW%u_CVF*sY&ofH-;~%x|Je8ZX%DI6IVr`$Z(x+v(VP0<Sz5&AwuM`!JbV{>
z;+V^uu9GuDfBu>I@K<Tu_xBr5L(^egQO)09@88|6kKg|<J^laj0}Y#2NSNCGZJ#bb
z;peaa{}q*ZdU_`K@aWpq{r^}0?alkh&HM6heR=%$c0CUd&-ZtKA3jZW=yEj?D$ZiA
zY?#B?*=t!~R|9IShOamJ^Y1LEq}o-=Q~u7t#ME%d?#I^mCDwjyy#HZ)zNFvWLzU;O
zAhA2oqV8|fj)I5x`{nB=hL;|=V9)mD=Z*gxrpo-6lvKPu|H19FfB*lS{P5`L{l)I}
zXXpNY``}>wgs#aGCpv~t|9N+}{f5mGF9<O*yZt&4x;a5@aUyHozmy;UWN$_N`QHEV
zRcPBJp|-EDCHL%+ICDlqMKv%>^2e^?nA3YNgq0c?7}WebQ}EyJoC!4QD;v(;-u~ah
zQt|n5!$aa_3Txjb{^6gyu2p@?{}r1z@z~cJ*;ITezrQcOTda0}$;<64G$t=qef{@$
z{e0Ws-~ZbmetCI+qAELcWM}e?X<{cHoRe)6R$nktT>kul<Htdxux-3N*Vh|sXd0e4
zZQRjmSWpN;GiDo~IOW6_mXe^bsPoi;10Q~yAO8LQ#qZ>akV@^)OpAT>|G&SyT+hh(
zgHN)8_Yg~DHP7tz?o*WF_y5~n_xJtgbWp?o|EsI}rOp4{+4c4Ny}kSO<Ns~mnyRb+
zyz1|-`j|WFY;0_4X>5^^%lzm6D=Kn|+NGk(cV^j(4XpM5jqdDXO^uE;jEFMQ&@_~>
zIZ^P_=<v~Q9>`)98ylWGcX;&Uja=NF`M)YHd&(&Ja7WeDhlkncWI<EzOpAH8zsnyU
zinp{>dvdeV&5h0N*#3;OH~wGHp73AZ`djs*Bk%L?|NH;v=l|E&K~B|IJ(`kw^wHzP
zX~~DR)Q*M3{+&8iG^%;i27`ivKgXvTeOTU~cu821%^^>ywM6yDPe}_)$&ykKVq%gI
z(@(P~Xn6S0xoy6^<h}jr760cwJj4o`dR2UsC(*EE-h6w>b8`;bnQ?)_4^$w|w3snd
zQqJ}ld$x~5Uv6T<7W0X(q}YBiT`GS5@7tFe>#8sO#>ULj(#rb!i+5UfZ#K>j5SYEk
z##s3B|M&O*|Nrw-KGX8#e}4IYoE!#^Jh*ryIGY%!+Wt9sem+n1Hiw%|P94@Vsi~4(
zT@pn_67y`6cT`M#cviYi-d^I$)xx+>9o!CKy$@iHnQkE=@$c`?!v{{C`tjfX^8f!I
z^0-r8>l8IJKl=4QEbNS$n$Dj;JVixkmMzmUGV<BLyzHN$fq_lUkMGBN_ZNKC((*}4
z+Hs1Lt>;n`W2a%zf!XFf9}aW8zn>lu+PZ1xOvxQP!Dxp>ORHqv-=rVk`K#90ddA*d
z?*H+sCJ$`(`^g;8azX(Xb@q({N7Ov*9{Ja-?f=NwSQ55qUaL@`iQNI+^|z8Nb|l{n
zw^-I$cqKJ3TQo?+;3$i}y<zC9S?e|{O<7v#x9U{K9KX=cLfJNEzW2S~&wSoD^M{f$
z?|;>an;Tc=%I{+9Zd%58A!q63b+_j-eX&)1!jl8Z^)oF%)mHIyd+FQu5%N#$-(2AP
z@hPYxdrITPiGdy+fe|w}uSK=p(dlH8v-|V=m1xL>nKK(_&TO13>pwer#`i$ZruzR9
z@9re-C~Srd{5!g`F85M;o-<L@+2!5c<b<?i=30>c{h@NPH?z%oHrVF=f1oUrA^ldd
zVd6x_vR&MV>%S;7NJvUZ+}r!R{?`}zu&|8je-F3+U$(60&6~w1nbbi`m%rs^WPbc_
zfB5zF8}C`$7>wL5%(qSc@wNLv;7@4cFaMz`FQ1Yix^#9!NsoNWEZ(ZWZ~p)1F<?C*
ze#ywd;16h0;Q9Ige)IltOLiuwr4^K=rKSGm-<>H_d8anztNRCrw6wPT`vTboDk>5M
zvT0sg4NI0tn@h;qs}*botxo#?|Il=cd3LSpC;xw#(pa{Z^Nkqy7H_8ii`5p?hp+!<
zSt}J)_wkd_i4z7NKK^x|)ziAXbHB$MLEe3}$vbML3%_nypm0)rhBEV$TP>hb0DI8#
z<$L?p1D-%VFx{f=Ps6gX8G`>O=$Sa4IhX7!aH46x-CygPANn<G-^{n?*|m#DNa%l{
zH+PEeVuq?R={?&&y%u+^7MZZR|Nnn*BKtM<!=*B~d7y^eKbfN$Pqdg<zH`v@PI$!g
zVbRmKx9hj(|8Jf?Vb`u6BO{(SZ~iB)K00A*KNHK34~$avIyyW_NfCKH$5=1Snm^-B
z{C<gbajJ8m&F4eYEhHo*>;4{+>1$0ZuV`($a^=VI6C9C@M;bze|JHwdbN^uTg=M<J
z!cV?@{r~^p!O0ghHXUHPw>P=shrle|$Vkb5e-E2a>13O*JbZ)S`J!UuIkw&vFO49H
z@p|;nZ@rFAOo@`rUj9$)iwc`=6(_pLt^M_Tdg9|_^&f&AfBckmagkWERPx4+2bmcn
z3!3-WOWxb>erNWVzt)Gpzvuap|NMZC?uVGOC7;!%^>WOBhD3PLpFbxHp7ZUh_2~Kk
zq11N6YO@R0i~j%r4|4Ei_5Tl4Z%o(c`SpvZrsn_58E&)1Ee`(w&!eNmv)oV7MMQ)n
zLop)ifNxqt!j3ZD`zbmyP-CZC{QK7`_woFJhV_x%SAID7%lGvB-(UN?{GH4pW-BY6
zUAzA8+~GEv)$YS<@xxoQc_vK?(36)xyxjkVbmT(@%dC!_Zjax`>pb6*0P4Vk3dr!a
z8#XBT6&_Djz0_hihr3+o@&5Y%{BnQlzw1pof9%+Tb>}|3m){p?Y1L5wU*g}t*0{{J
zdXbC;3@Z2cS291X`^Pia^B|YPRrY5uES4F0Y^eX;R(DTB58~WCbFH}=3<O-{Jvv%@
zJp^(d|MgFN%c_&3!=mcyw5!AHAJ;^YM{^P+<W3(D`^?r}Tq0X4JN3cAH+OGk>DKzs
zdY9_u)v`dxUT$9J6_bZDr<<lo95yiMJM;OSFUO{yMGID4kXrkY<5zvdp5?jgqZWu%
z`f=pnZctY$x3vqK)UvE|my9$xmbc7R<ZCeCag|p-Eh58kRP}=Kop?PTpNAaY0w*|G
zT6zS<SVF?Lob|u_&n%Zu<~NIq@T?syJ2yUK2yNXuUAolQIre#&!v0B@=M^5Y+>;(z
zz?W^<mgwL$#p=oumyW~tH@R@!nzqh4``0`HjjsFlOXdhMT|TmFS?>$s^McEJmvG$O
z?AkP=^4Rb2X^;|d%Z!B}=eWerICxIaG4Y4c6}y7@CXLcU%v?wLdr~?CIe0jj&TV^i
zDNcW<U8nNB@?Bp|_OuivvMCzySZv?_vt@x!{?sp`d*XjabZ`A)d)ejs!#j@4?%j`C
zEzqOv+iWPp8Wnt9qjieLSMj&Jk#6%YZ~J}c+00c|+hdnax?*|h)ogzEBTS%$-G`=I
z?5j)NQQUs;Ebm<-3x$Xxo9eb5ljnAJItER8cwL{(%&e{ce@2no$@}|x`sIxcjE(2m
zo!yYiefaBZp8NZaZECw#?CCUr{yzSN!GT_B9`ihdGcyi8czpQ5n==bmoLI2xl!Cf@
z?eW8)AsqL9qne*hE5ep6-Kfz#ook1bWQMPR#rKEKN$oz6#IgpKSR9YoP2cd9FKpfw
zVUN6o_q;(Jlk_HRx#$$wBQG+i=w_Bg+g#nU`9~%$bIIR+Ui^ChL$kbdn@W#!wKRNJ
z;k@_%&sq(ZmIbe0RBu_9nGxM`;MQ%<|JrjFuM6zjv_#`X@v5w=fA@>+*kHob^Ehsf
z+LAqKKlIkS&D(g;vZr$E6NWT!ia9jh0#uNFVA%W2Sx;qx=bBsdZ5&1J`uKpny<ziW
zbqg<sluJK9Cb#|lEh%T0Vo^UY@wVBE8yD~IZ`nGT%dRFNVsqQWGm>n&x^2$w5>C#N
zKE9Iw{;3++xC$jCB~^TEdiWyn$OqLW|NCdo{m`Fcuwa@@0cbtSA=aeTqLA<ax#6$#
zj8F1q*}Ba4J~1E75My(5V{>$TQE~eJe^6hLUk=oWRnYNiO*`H#4q8{dI?V9R{qr9_
zsUCiLndkjI$GN`2A099sZf55>b&6+qx$%+X#w~3|28M<wP6r?Vu-sqr+}z|H#q5F4
z3JVNYtP=98-@&%;fo%i_Xdl!)+ay`Yc;1}Z8#x;scwFZ<ttjZ-yR_S)S$k(sb;Ql+
zD7mDC*E<sm#k!U&3H@2Z*H|-Y{)C!|hd<9)Cavhl#PwN<W!?SgY6Iuj-{<>teq;D0
zw^ovERoAvunY)&E&E=+=b$JLqlDMB9J2Sm!;<P*KoHb8P+O<r>Blwfd5BsIjr#B_=
z_!iqvQ{Xviq<;6wJ-^+0PGzRbN5DG}=ghI?b14uxQs3gIDYQsntwkHN#bWIX){LE9
zVIhkqPWa)Kv+;kD_|4er#_MNNp6xyPq_!#Jex}s3NR#mMn?6>pjk)A{d4tZxgUnq@
zQeR@**Dcog{v|nfV(O(S%M{OdDmm_pmw7T%L6^I&@%`gnGt}A|mvyeXV`A9%#WvSF
zHh5ad)#)=2H>e067h!dDm?qeE!)(>B^Sk!uo-p!snzr}hbBl_58y%`X{PI3%(Jph^
z>Gr(}(J$b%)iY-!SF;08n?0YOIr~F4_T_r9bD~~7{*h+aa>4V(rG?I{)}`-NSlROf
z9=hCG7<BO3Ez=f<N%kr~IU`rx-zBG&xoO`y)l#K$nJH2S6M0-6XKjjGY`SiDPtWxY
zD@}8~BPXdG&YAd}W8<6UQ#4w0lW)H%Yf*G;c+Y>Ob&B$i-}BZTmgrgdddK~pF(-`p
z*k`dR?sNVi`sH@7McVGC6)Ig3Yd_db`Q@7C-(w?}aklRK%OhW{o?YE^BWIudgmWJk
z&j0&#ZDi=2^Z)lXG0uZ{XYPLPrT~HC_6q#$(M1L&9O8;CYVA@|f5ne%tl#*Hokjd$
z%$-?L(sQR&B$)kJ5Fd57;&gSv{O@~@7vA~X@VIJe<_sS>(`n{^E5!KiT&`{_jO}xG
zjksR8v%qTOZ9&%%(YD5Q+DsKn>+Kh<Gvw)F(mYwr$Np;8rjTBi`@S4D{FjduMy$P&
z!@+bb?5<14#iJi<>t>$+So<jA?6sTkUs^4Z`FrYa-V@7du9G(0-4wAQW>ef{6QAW<
zbZ43US+vB3<6Qct9u-C3_$Kq&f^CaiKRX)jtN3Ns@c%6zsD$j9wV$ggK;&4x*fCia
zLB*VmH>`QhhhuA)r?=f(<6cnwic`s{<%6VCL4lE|Y>d>dfG_9I{m^Y^wzs(d(?Yr6
zis|(3fA#fw-wIFf@wc1*__xbN=apK1>MaX0Z{+QaE0odcUgykF`{G=h2<v3dO21h_
z*Jb)sBtPUQJ^Z~UOh?S4Ge~DTf0_H?$je8zZOnap+_v8!WwT<Rr_ilwHvhZQB8}#5
zx~-_Xt~-c@v8{2+7u$8G4H9O}3Ou}El|Ylp)0$@m3H$3yFMDl_tAF|<?)<k$?My12
z-`><;;{|zc%IuBYO$h==>*pt3eDdUph^47!<AspfCg19}nYwf<=!vTBu&BHE=z@}7
zSo4Jm8lHiBKOQ{zv7(^g`dzo>sY$>0o;CF^Ih5J{G9tw))#zbG&P*G14h^m)M@kKS
zf7Ry9wr~r*XmUwJ(L;!jy(MA0bI1F3rO>6T3S!fY-IUv$W!Mi_y!@%J;kDOjy_Oia
zP5;%dvuv6ZC+Sqjf1P6&*s^Z({mplc<Tl3{&oADa6gfq8*0F>GKYiA_OBC&V@OzHm
z*SzhSH#SA+-Hu<z*83^{@-g*iWvf&F%R-v|N-}AZl03e?9hN?BZY<JzO~>5W*2f$B
z%szJEW@jC@K+3<S-d;zqF2}fe53g#oadNih-v{+71VmDHR0vw_@jbJN*`wS!Y?`9Y
zoP+238vFYl11CC0%xDQvkh7^cu%TG}@YU5k;p>gw+;)$!6&5a8_wlqiTXuHa_I$}J
zS0(LglM)gWLA{7esu^hx-Dz%aprNiSA#Ja+<|xYuuxMufOU}u1dggTF`8mUs6a$+I
zho*E;Kk@(nIdcs8Tml5z>`yAIIEnl!=S&aC3eo#;?its*h8LbDi>`LD+)Rq!IPJ}?
zzrSaGn*GPP{@Lz@GHlaWk6W&>vP*cJyQxjl(<wGsq`mg(zRTY^9B;qseqoXrX*H=n
zcv<HZ3ARkHX%8!R=s6Z_|HJRu85ANK>n<|!dC%f=3^7^vBgNisnJ?dYTIhCK?9B9*
z9cAa71deO8zK`Etd-T{$`Sgk$Y2VM%W(|;mQ>8f;3Ubcx^o$QgEt_FG{r`=$2PUSH
zixx}9?N75PX0z2%xHI2=;mfuq7xO%f<6`F?yg%PDdiKNr^;vy;AF->c@n~uBSXuGp
z<?(Qf8N9i7_QQYs!>6aWm^%Gh@j*){`&2?|YC&P*iZv+{CLa!xe&oef5hcgN!xO*X
zXpV*B6%$)fGoIo9e<iI#o0<~~*7Z#ixe%pz;wW3%>^XDiDk?NQJkH-%`B|b*UOj6j
zr`~~+oO5q|{hAsOmUiJvih;TMeG{*S28QMahQr7EdD7B&q|J?TZXWvZ%k=O<XOW+M
z6DKy#oT;eRws6A)$5>hK+me>73q<8+XidE7Ki_DMmFuTO==L?GRdd4v4+ey^+<SB2
z=z&|G8XBD2C7zv0ys=k0HLO#iVT(y?tK^iaps|dmCJ7r`NePLL*jATphL`R3m4^zx
zXdJ%0oF{s_QOu452M#zy?Qdvo1f@8amW9jO4jyiH^x-*eAeF|+m|y(hmMgE||Ih7;
z*0U4uT20yuS;(iPRr2<YMA-T^HNK{kOy`br&9hIxv8nap1;w_~*Ag>kNbIXUY$wVz
z(?(b*>A|zK2TxK8ijxBZ6DLef{qQaA$M5NlQ>UJ}SkB>aI^D%jRFGNDHYp(`<;L#m
ziT~@`&d-<RlS}#WU;ftP<fKE7f<3l}1xz@2Ws0M7Gbmu3n9|dwv%e(!@@Cu#UM{g|
zQ^(vy2avNE{{NpdSCOyTLBMtXfd+?$+srzgERDwstll-=C@x?Uk`ZTMVLINF7<|#!
z+4-{5ei_T3KWFawlcKlQkz<OIXO7{UjHcoQnX?}YPaiYr%jgXPw{>rJ#EC8Hs6GDu
zq*08xrY6_6vrXsre-)nfu`W_xefyU+DcQcaA6Z=4T3dA4q($*Cg8<7J<HDZjimH`;
z0!v<+zo<JZnZ8)!@UEa~Ay;_~eqCY)6@^NtzU%0K_GYoNUg!*qv5oKu7Fp}xZC`VL
zPvVbXQybgcL1Rw!|3L#Q_x2`Fn3D3~S=x`kvmISnr1YkQ$Y*TTH#(52v#j@tCBvpo
zFE6XNP1l!nah24IO?vP+`NzMx51;$9nVPoo%S*VpNE|uBp~H7HYW4xvk0IIu2Ol4o
z%*v8j7o(_Ey#Q1KfYS4|%g2uKoStUL!6_*5@PG>s^G1j1jQO(;%)cdJI4v<Lb@D^G
zPg@>)i5<D3GvCgzq}*61!od%*rbVf&>F2A^w#UaeTJxQsX>{V>Len4pw_1}O-kwh6
zjY(mfWm&<a$UkACk-=dxJwJmL|Np1|_&xpM5ze;K*C%`=;X_4A2T!hA#Z&#wprBAt
z&rIPfciKb8#Y!yMZyu#jekHLZ$%jps;UJ5bPxIrO#%(tu+D=bDSq_T&|NoT|Gf%y}
z&10PAFx&D+u;IZ?Elv&>3Sx4asy4bYFmIdG=H{wgcU&OY!+FZzzo#$U^b9<e0Ne4b
zlzaD56DyB--h>Gi4>dVws+g{wFe~#zyZ)?=5l1BhvIUy>4y3Y9WT=;T<i8^#_5a`G
z8Rq?ox6LNSHi0Y$g`aMrP5rqG_s@TL%+9%T#sV+4oT=I?+XQ-aV@`cIA|MxgWcL&M
zd!N|OHb-n0m?)Q?;E-{D=j6n9F_Y4R3_x-4|G(1CXHTA{YG@>!IO!-UaQLd{!JOg;
zS$~pcJCDm$ID|FFvjhwDdMD0i4xT2MIaBE41N%$A)Xq$4QR~odHB?yIBdjj5YPICL
z*u{sOpxbzrDi?hGoDM3Eo;;PTsg;y4m{#}qxY@_HWetp(J2-WH7ic<*9hn?f!1+k<
zxN7T5xtM86>Z~T^Li`tkn8O^;m>g`-t#t@GwlbKfx0mPo`rvd<J8%<BY5Cc+Cr&I_
zapJ<&qY2sFOaCM!I6OBpSTJ3l=SElW{|a}e6W%fvOE{k8`+Q%pQ*!Mbg=6A7Vp^w3
zewZ%Fb3^F({rx=d{YECHhB14+-Y``)LHZv`o1KrHIH90^{KD<i3CYZND|<{M6W*SB
zBpK$!9e-v~t6*uM>AEI%-Us?B9V`*S#?1GQG#*zzk#~4an1sign7Y47H@5XIT-*Eb
zqH@-|XONkPMJu-JsU5p;<;a8Arw<%!+kEz5gL0~Z@rGw+F;C587F-ROA!R74y!s%s
zf$xE-9fr0|%NTrB_1YIs1-X-bozdgZ>7Z@9%X}9*af^TsG+_Aue^H4>dwaX0@nl8Q
zsf<O&+}Bi^+Ff=uPcf2YDp2KXiZaM`FJ_ySWKwwM5RYEp)}>B9Nk`+?8XVa6QBhs6
z<HX)-p0{s#qPGXF)pCTUo<)Z&<h8Y%Hyb;bnKjRpWL8$L5<Tn?)69I-w<khro`hK?
zL%7qysgpW9_wDeqHkmB>AxxX6rK|U7w`AH`r#si2OCce+=$68UEhf$D^+D$BuxOmc
zI=Ne?<MP54KPM&FI`JIHSB^U9utYD$Ez$Q#L^l`PotMmUT`{|lUAT0p;GN95ZrPQf
zPy>1K6HntD8RrTM=P0R?RGojnSYI*SQD)!4aC2p2h4zZo-Al4q{bzoB%noXkXorLA
zuz$9`enPNvc+tr(6MJ}^0|cB444S*Rl1?i|X*EXmFOKYr+&n>Cdk2H#<$|U5XWTgD
z7ITP6@K~JUVe4Ufd|kgS`MAWjHHj8gQy%^-wwk*GQt2=9d<0q-nwIw9adJR-`iCzm
z1!ZXgp+~R(JK!T$A=NV@(8=wRVuaB_-(!mfS9^cRR$nRm>%aZs`};wscf7fKdP9X_
zQn=<_Xefm~a$uXlyZYTb9yd2od!VT3!q<X`-CH&Kl&1tpeAveGA;k8`moN!~M?$N0
zuJQ3HAKRe0jDxvPiRI9bpOX`x`?XD0m)w@us<&VXY%{6*qXuR(Gd4XvHbq4?Z*MkX
zVYbXnHdR%&$jB)!4O=Y|6-54~gjgNfV$$g;!?WU~h_-{C^o3t)JRT<>J@<ccvn+a>
z!JFI1KYUXC`MP8UEEcjs4n2ENMuta8iAP9?M@@}q&Kw>oDIPbs3wu3lYF@C%F4}0w
zE&6CO$3&@*>U?Z_UNUF<OK(2OnRsXC;SHI>ho@@uOxHKwQ+xWu-`@=LA9I6>8qg^-
z8;&%xJ^1PA(9k${uA@(nW5kTc=4MCF&VzU6G`6=lPMqi%C%21j?nJ?=6^xt8)+|f(
zO5}Q!ANc4}t7f;izS`sa_Mlo?FFtKY{d~|V6^Gj;?dsKjo^S@uw1Fx(DV<Jc{s#{d
z64KKXl9CeA(*=Z*5>ityTu8WZxp`gLv!|&OCMQprl=R@4TPCNFrCp<XYQkEb4_6c_
zcB$Oo4|efe>BF<ldD{7nWvq{Z&UyI#{X(wwl=9P=pb0dF|NpPdHRNJY<Y8g=(aHVu
zzxq$3!oP!FTefygm1%!^w?^@Fp7~El3%SC)>z!JP0?MmyuT2Y6zwE`RRIxu-S1G@?
z_^{ya$LrGcx7Ocw&|KoOw#V_@mSc0i{OdfqBasp8g;kx*{0S*30>a4&$;k=n=?@+y
z35cdnn3xE1`sCyZ)6xV)Qy)BPnpeSTFT$Fg%oO?IzI~g0{eosc9i0n1+D#Pa+oxB2
zYI*oExlP`Ffw<6}U97fJ??0!%@^8B+3`vZ4<{EMxP!MsJ=W1F}@n86q{J#eef?Tab
ztV&;o&C+D&(2xFfxWMLARFINe&rO}3dUKLhSmr3F2i>~;{NCd`&(7xdHAe-ywgd$%
zHNBSP=klUf+~s}G$GgiamI{OYpxMdH-=M-FCiddv51tLeArH@Tb0npt{P;Qf;cfG(
jeivQ^h&u?LfW^#kCBJ6FvfZ0pK^i<={an^LB{Ts5;0bTK

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
new file mode 100644
index 0000000000..9403003be1
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -0,0 +1,196 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Image warping using per-pixel flow vectors."""
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def _interpolate_bilinear(grid,
+                          query_points,
+                          name='interpolate_bilinear',
+                          indexing='ij'):
+  """Similar to Matlab's interp2 function.
+
+  Finds values for query points on a grid using bilinear interpolation.
+
+  Args:
+    grid: a 4-D float `Tensor` of shape `[batch, height, width, channels]`.
+    query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 2]`.
+    name: a name for the operation (optional).
+    indexing: whether the query points are specified as row and column (ij),
+      or Cartesian coordinates (xy).
+
+  Returns:
+    values: a 3-D `Tensor` with shape `[batch, N, channels]`
+
+  Raises:
+    ValueError: if the indexing mode is invalid, or if the shape of the inputs
+      invalid.
+  """
+  if indexing != 'ij' and indexing != 'xy':
+    raise ValueError('Indexing mode must be \'ij\' or \'xy\'')
+
+  with ops.name_scope(name):
+    shape = grid.get_shape().as_list()
+    if len(shape) != 4:
+      msg = 'Grid must be 4 dimensional. Received size: '
+      raise ValueError(msg + str(grid.get_shape()))
+
+    batch_size, height, width, channels = shape
+    query_type = query_points.dtype
+    grid_type = grid.dtype
+
+    if (len(query_points.get_shape()) != 3 or
+        query_points.get_shape()[2].value != 2):
+      msg = ('Query points must be 3 dimensional and size 2 in dim 2. Received '
+             'size: ')
+      raise ValueError(msg + str(query_points.get_shape()))
+
+    _, num_queries, _ = query_points.get_shape().as_list()
+
+    if height < 2 or width < 2:
+      msg = 'Grid must be at least batch_size x 2 x 2 in size. Received size: '
+      raise ValueError(msg + str(grid.get_shape()))
+
+    alphas = []
+    floors = []
+    ceils = []
+
+    index_order = [0, 1] if indexing == 'ij' else [1, 0]
+    unstacked_query_points = array_ops.unstack(query_points, axis=2)
+
+    for dim in index_order:
+      with ops.name_scope('dim-' + str(dim)):
+        queries = unstacked_query_points[dim]
+
+        size_in_indexing_dimension = shape[dim + 1]
+
+        # max_floor is size_in_indexing_dimension - 2 so that max_floor + 1
+        # is still a valid index into the grid.
+        max_floor = math_ops.cast(size_in_indexing_dimension - 2, query_type)
+        min_floor = constant_op.constant(0.0, dtype=query_type)
+        floor = math_ops.minimum(
+            math_ops.maximum(min_floor, math_ops.floor(queries)), max_floor)
+        int_floor = math_ops.cast(floor, dtypes.int32)
+        floors.append(int_floor)
+        ceil = int_floor + 1
+        ceils.append(ceil)
+
+        # alpha has the same type as the grid, as we will directly use alpha
+        # when taking linear combinations of pixel values from the image.
+        alpha = math_ops.cast(queries - floor, grid_type)
+        min_alpha = constant_op.constant(0.0, dtype=grid_type)
+        max_alpha = constant_op.constant(1.0, dtype=grid_type)
+        alpha = math_ops.minimum(math_ops.maximum(min_alpha, alpha), max_alpha)
+
+        # Expand alpha to [b, n, 1] so we can use broadcasting
+        # (since the alpha values don't depend on the channel).
+        alpha = array_ops.expand_dims(alpha, 2)
+        alphas.append(alpha)
+
+    if batch_size * height * width > np.iinfo(np.int32).max / 8:
+      error_msg = """The image size or batch size is sufficiently large
+                     that the linearized addresses used by array_ops.gather
+                     may exceed the int32 limit."""
+      raise ValueError(error_msg)
+
+    flattened_grid = array_ops.reshape(grid,
+                                       [batch_size * height * width, channels])
+    batch_offsets = array_ops.reshape(
+        math_ops.range(batch_size) * height * width, [batch_size, 1])
+
+    # This wraps array_ops.gather. We reshape the image data such that the
+    # batch, y, and x coordinates are pulled into the first dimension.
+    # Then we gather. Finally, we reshape the output back. It's possible this
+    # code would be made simpler by using array_ops.gather_nd.
+    def gather(y_coords, x_coords, name):
+      with ops.name_scope('gather-' + name):
+        linear_coordinates = batch_offsets + y_coords * width + x_coords
+        gathered_values = array_ops.gather(flattened_grid, linear_coordinates)
+        return array_ops.reshape(gathered_values,
+                                 [batch_size, num_queries, channels])
+
+    # grab the pixel values in the 4 corners around each query point
+    top_left = gather(floors[0], floors[1], 'top_left')
+    top_right = gather(floors[0], ceils[1], 'top_right')
+    bottom_left = gather(ceils[0], floors[1], 'bottom_left')
+    bottom_right = gather(ceils[0], ceils[1], 'bottom_right')
+
+    # now, do the actual interpolation
+    with ops.name_scope('interpolate'):
+      interp_top = alphas[1] * (top_right - top_left) + top_left
+      interp_bottom = alphas[1] * (bottom_right - bottom_left) + bottom_left
+      interp = alphas[0] * (interp_bottom - interp_top) + interp_top
+
+    return interp
+
+
+def dense_image_warp(image, flow, name='dense_image_warp'):
+  """Image warping using per-pixel flow vectors.
+
+  Apply a non-linear warp to the image, where the warp is specified by a dense
+  flow field of offset vectors that define the correspondences of pixel values
+  in the output image back to locations in the  source image. Specifically, the
+  pixel value at output[b, j, i, c] is
+  images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c].
+
+  The locations specified by this formula do not necessarily map to an int
+  index. Therefore, the pixel value is obtained by bilinear
+  interpolation of the 4 nearest pixels around
+  (b, j - flow[b, j, i, 0], i - flow[b, j, i, 1]). For locations outside
+  of the image, we use the nearest pixel values at the image boundary.
+
+
+  Args:
+    image: 4-D float `Tensor` with shape `[batch, height, width, channels]`.
+    flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`.
+    name: A name for the operation (optional).
+
+    Note that image and flow can be of type tf.half, tf.float32, or tf.float64,
+    and do not necessarily have to be the same type.
+
+  Returns:
+    A 4-D float `Tensor` with shape`[batch, height, width, channels]`
+      and same type as input image.
+
+  Raises:
+    ValueError: if height < 2 or width < 2 or the inputs have the wrong number
+                of dimensions.
+  """
+  with ops.name_scope(name):
+    batch_size, height, width, channels = image.get_shape().as_list()
+    # The flow is defined on the image grid. Turn the flow into a list of query
+    # points in the grid space.
+    grid_x, grid_y = array_ops.meshgrid(
+        math_ops.range(width), math_ops.range(height))
+    stacked_grid = math_ops.cast(
+        array_ops.stack([grid_y, grid_x], axis=2), flow.dtype)
+    batched_grid = array_ops.expand_dims(stacked_grid, axis=0)
+    query_points_on_grid = batched_grid - flow
+    query_points_flattened = array_ops.reshape(query_points_on_grid,
+                                               [batch_size, height * width, 2])
+    # Compute values at the query points, then reshape the result back to the
+    # image grid.
+    interpolated = _interpolate_bilinear(image, query_points_flattened)
+    interpolated = array_ops.reshape(interpolated,
+                                     [batch_size, height, width, channels])
+    return interpolated
diff --git a/tensorflow/contrib/image/python/ops/interpolate_spline.py b/tensorflow/contrib/image/python/ops/interpolate_spline.py
new file mode 100644
index 0000000000..ad17921991
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/interpolate_spline.py
@@ -0,0 +1,285 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Polyharmonic spline interpolation."""
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+
+EPSILON = 0.0000000001
+
+
+def _cross_squared_distance_matrix(x, y):
+  """Pairwise squared distance between two (batch) matrices' rows (2nd dim).
+
+  Computes the pairwise distances between rows of x and rows of y
+  Args:
+    x: [batch_size, n, d] float `Tensor`
+    y: [batch_size, m, d] float `Tensor`
+
+  Returns:
+    squared_dists: [batch_size, n, m] float `Tensor`, where
+    squared_dists[b,i,j] = ||x[b,i,:] - y[b,j,:]||^2
+  """
+  x_norm_squared = math_ops.reduce_sum(math_ops.square(x), 2)
+  y_norm_squared = math_ops.reduce_sum(math_ops.square(y), 2)
+
+  # Expand so that we can broadcast.
+  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
+  y_norm_squared_tile = array_ops.expand_dims(y_norm_squared, 1)
+
+  x_y_transpose = math_ops.matmul(x, y, adjoint_b=True)
+
+  # squared_dists[b,i,j] = ||x_bi - y_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
+  squared_dists = x_norm_squared_tile - 2 * x_y_transpose + y_norm_squared_tile
+
+  return squared_dists
+
+
+def _pairwise_squared_distance_matrix(x):
+  """Pairwise squared distance among a (batch) matrix's rows (2nd dim).
+
+  This saves a bit of computation vs. using _cross_squared_distance_matrix(x,x)
+
+  Args:
+    x: `[batch_size, n, d]` float `Tensor`
+
+  Returns:
+    squared_dists: `[batch_size, n, n]` float `Tensor`, where
+    squared_dists[b,i,j] = ||x[b,i,:] - x[b,j,:]||^2
+  """
+
+  x_x_transpose = math_ops.matmul(x, x, adjoint_b=True)
+  x_norm_squared = array_ops.matrix_diag_part(x_x_transpose)
+  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
+
+  # squared_dists[b,i,j] = ||x_bi - x_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
+  squared_dists = x_norm_squared_tile - 2 * x_x_transpose + array_ops.transpose(
+      x_norm_squared_tile, [0, 2, 1])
+
+  return squared_dists
+
+
+def _solve_interpolation(train_points, train_values, order,
+                         regularization_weight):
+  """Solve for interpolation coefficients.
+
+  Computes the coefficients of the polyharmonic interpolant for the 'training'
+  data defined by (train_points, train_values) using the kernel phi.
+
+  Args:
+    train_points: `[b, n, d]` interpolation centers
+    train_values: `[b, n, k]` function values
+    order: order of the interpolation
+    regularization_weight: weight to place on smoothness regularization term
+
+  Returns:
+    w: `[b, n, k]` weights on each interpolation center
+    v: `[b, d, k]` weights on each input dimension
+  """
+
+  b, n, d = train_points.get_shape().as_list()
+  _, _, k = train_values.get_shape().as_list()
+
+  # First, rename variables so that the notation (c, f, w, v, A, B, etc.)
+  # follows https://en.wikipedia.org/wiki/Polyharmonic_spline.
+  # To account for python style guidelines we use
+  # matrix_a for A and matrix_b for B.
+
+  c = train_points
+  f = train_values
+
+  # Next, construct the linear system.
+  with ops.name_scope('construct_linear_system'):
+
+    matrix_a = _phi(_pairwise_squared_distance_matrix(c), order)  # [b, n, n]
+    if regularization_weight > 0:
+      batch_identity_matrix = np.expand_dims(np.eye(n), 0)
+      batch_identity_matrix = constant_op.constant(
+          batch_identity_matrix, dtype=train_points.dtype)
+
+      matrix_a += regularization_weight * batch_identity_matrix
+
+    # Append ones to the feature values for the bias term in the linear model.
+    ones = array_ops.ones([b, n, 1], train_points.dtype)
+    matrix_b = array_ops.concat([c, ones], 2)  # [b, n, d + 1]
+
+    # [b, n + d + 1, n]
+    left_block = array_ops.concat(
+        [matrix_a, array_ops.transpose(matrix_b, [0, 2, 1])], 1)
+
+    num_b_cols = matrix_b.get_shape()[2]  # d + 1
+    lhs_zeros = array_ops.zeros([b, num_b_cols, num_b_cols], train_points.dtype)
+    right_block = array_ops.concat([matrix_b, lhs_zeros],
+                                   1)  # [b, n + d + 1, d + 1]
+    lhs = array_ops.concat([left_block, right_block],
+                           2)  # [b, n + d + 1, n + d + 1]
+
+    rhs_zeros = array_ops.zeros([b, d + 1, k], train_points.dtype)
+    rhs = array_ops.concat([f, rhs_zeros], 1)  # [b, n + d + 1, k]
+
+  # Then, solve the linear system and unpack the results.
+  with ops.name_scope('solve_linear_system'):
+    w_v = linalg_ops.matrix_solve(lhs, rhs)
+    w = w_v[:, :n, :]
+    v = w_v[:, n:, :]
+
+  return w, v
+
+
+def _apply_interpolation(query_points, train_points, w, v, order):
+  """Apply polyharmonic interpolation model to data.
+
+  Given coefficients w and v for the interpolation model, we evaluate
+  interpolated function values at query_points.
+
+  Args:
+    query_points: `[b, m, d]` x values to evaluate the interpolation at
+    train_points: `[b, n, d]` x values that act as the interpolation centers
+                    ( the c variables in the wikipedia article)
+    w: `[b, n, k]` weights on each interpolation center
+    v: `[b, d, k]` weights on each input dimension
+    order: order of the interpolation
+
+  Returns:
+    Polyharmonic interpolation evaluated at points defined in query_points.
+  """
+
+  batch_size = train_points.get_shape()[0].value
+  num_query_points = query_points.get_shape()[1].value
+
+  # First, compute the contribution from the rbf term.
+  pairwise_dists = _cross_squared_distance_matrix(query_points, train_points)
+  phi_pairwise_dists = _phi(pairwise_dists, order)
+
+  rbf_term = math_ops.matmul(phi_pairwise_dists, w)
+
+  # Then, compute the contribution from the linear term.
+  # Pad query_points with ones, for the bias term in the linear model.
+  query_points_pad = array_ops.concat([
+      query_points,
+      array_ops.ones([batch_size, num_query_points, 1], train_points.dtype)
+  ], 2)
+  linear_term = math_ops.matmul(query_points_pad, v)
+
+  return rbf_term + linear_term
+
+
+def _phi(r, order):
+  """Coordinate-wise nonlinearity used to define the order of the interpolation.
+
+  See https://en.wikipedia.org/wiki/Polyharmonic_spline for the definition.
+
+  Args:
+    r: input op
+    order: interpolation order
+
+  Returns:
+    phi_k evaluated coordinate-wise on r, for k = r
+  """
+
+  # using EPSILON prevents log(0), sqrt0), etc.
+  # sqrt(0) is well-defined, but its gradient is not
+  with ops.name_scope('phi'):
+    if order == 1:
+      r = math_ops.maximum(r, EPSILON)
+      r = math_ops.sqrt(r)
+      return r
+    elif order == 2:
+      return 0.5 * r * math_ops.log(math_ops.maximum(r, EPSILON))
+    elif order == 4:
+      return 0.5 * math_ops.square(r) * math_ops.log(
+          math_ops.maximum(r, EPSILON))
+    elif order % 2 == 0:
+      r = math_ops.maximum(r, EPSILON)
+      return 0.5 * math_ops.pow(r, 0.5 * order) * math_ops.log(r)
+    else:
+      r = math_ops.maximum(r, EPSILON)
+      return math_ops.pow(r, 0.5 * order)
+
+
+def interpolate_spline(train_points,
+                       train_values,
+                       query_points,
+                       order,
+                       regularization_weight=0.0,
+                       name='interpolate_spline'):
+  r"""Interpolate signal using polyharmonic interpolation.
+
+  The interpolant has the form
+  $$f(x) = \sum_{i = 1}^n w_i \phi(||x - c_i||) + v^T x + b.$$
+
+  This is a sum of two terms: (1) a weighted sum of radial basis function (RBF)
+  terms, with the centers \\(c_1, ... c_n\\), and (2) a linear term with a bias.
+  The \\(c_i\\) vectors are 'training' points. In the code, b is absorbed into v
+  by appending 1 as a final dimension to x. The coefficients w and v are
+  estimated such that the interpolant exactly fits the value of the function at
+  the \\(c_i\\) points, the vector w is orthogonal to each \\(c_i\\), and the
+  vector w sums to 0. With these constraints, the coefficients can be obtained
+  by solving a linear system.
+
+  \\(\phi\\) is an RBF, parametrized by an interpolation
+  order. Using order=2 produces the well-known thin-plate spline.
+
+  We also provide the option to perform regularized interpolation. Here, the
+  interpolant is selected to trade off between the squared loss on the training
+  data and a certain measure of its curvature
+  ([details](https://en.wikipedia.org/wiki/Polyharmonic_spline)).
+  Using a regularization weight greater than zero has the effect that the
+  interpolant will no longer exactly fit the training data. However, it may be
+  less vulnerable to overfitting, particularly for high-order interpolation.
+
+  Note the interpolation procedure is differentiable with respect to all inputs
+  besides the order parameter.
+
+  Args:
+    train_points: `[batch_size, n, d]` float `Tensor` of n d-dimensional
+      locations. These do not need to be regularly-spaced.
+    train_values: `[batch_size, n, k]` float `Tensor` of n c-dimensional values
+      evaluated at train_points.
+    query_points: `[batch_size, m, d]` `Tensor` of m d-dimensional locations
+      where we will output the interpolant's values.
+    order: order of the interpolation. Common values are 1 for
+      \\(\phi(r) = r\\), 2 for \\(\phi(r) = r^2 * log(r)\\) (thin-plate spline),
+       or 3 for \\(\phi(r) = r^3\\).
+    regularization_weight: weight placed on the regularization term.
+      This will depend substantially on the problem, and it should always be
+      tuned. For many problems, it is reasonable to use no regularization.
+      If using a non-zero value, we recommend a small value like 0.001.
+    name: name prefix for ops created by this function
+
+  Returns:
+    `[b, m, k]` float `Tensor` of query values. We use train_points and
+    train_values to perform polyharmonic interpolation. The query values are
+    the values of the interpolant evaluated at the locations specified in
+    query_points.
+  """
+  with ops.name_scope(name):
+
+    # First, fit the spline to the observed data.
+    with ops.name_scope('solve'):
+      w, v = _solve_interpolation(train_points, train_values, order,
+                                  regularization_weight)
+
+    # Then, evaluate the spline at the query locations.
+    with ops.name_scope('predict'):
+      query_values = _apply_interpolation(query_points, train_points, w, v,
+                                          order)
+
+  return query_values
diff --git a/tensorflow/contrib/image/python/ops/sparse_image_warp.py b/tensorflow/contrib/image/python/ops/sparse_image_warp.py
new file mode 100644
index 0000000000..9f50503d8f
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/sparse_image_warp.py
@@ -0,0 +1,192 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Image warping using sparse flow defined at control points."""
+
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import dense_image_warp
+from tensorflow.contrib.image.python.ops import interpolate_spline
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+
+
+def _get_grid_locations(image_height, image_width):
+  """Wrapper for np.meshgrid."""
+
+  y_range = np.linspace(0, image_height - 1, image_height)
+  x_range = np.linspace(0, image_width - 1, image_width)
+  y_grid, x_grid = np.meshgrid(y_range, x_range, indexing='ij')
+  return np.stack((y_grid, x_grid), -1)
+
+
+def _expand_to_minibatch(np_array, batch_size):
+  """Tile arbitrarily-sized np_array to include new batch dimension."""
+  tiles = [batch_size] + [1] * np_array.ndim
+  return np.tile(np.expand_dims(np_array, 0), tiles)
+
+
+def _get_boundary_locations(image_height, image_width, num_points_per_edge):
+  """Compute evenly-spaced indices along edge of image."""
+  y_range = np.linspace(0, image_height - 1, num_points_per_edge + 2)
+  x_range = np.linspace(0, image_width - 1, num_points_per_edge + 2)
+  ys, xs = np.meshgrid(y_range, x_range, indexing='ij')
+  is_boundary = np.logical_or(
+      np.logical_or(xs == 0, xs == image_width - 1),
+      np.logical_or(ys == 0, ys == image_height - 1))
+  return np.stack([ys[is_boundary], xs[is_boundary]], axis=-1)
+
+
+def _add_zero_flow_controls_at_boundary(control_point_locations,
+                                        control_point_flows, image_height,
+                                        image_width, boundary_points_per_edge):
+  """Add control points for zero-flow boundary conditions.
+
+   Augment the set of control points with extra points on the
+   boundary of the image that have zero flow.
+
+  Args:
+    control_point_locations: input control points
+    control_point_flows: their flows
+    image_height: image height
+    image_width: image width
+    boundary_points_per_edge: number of points to add in the middle of each
+                           edge (not including the corners).
+                           The total number of points added is
+                           4 + 4*(boundary_points_per_edge).
+
+  Returns:
+    merged_control_point_locations: augmented set of control point locations
+    merged_control_point_flows: augmented set of control point flows
+  """
+
+  batch_size = control_point_locations.get_shape()[0].value
+
+  boundary_point_locations = _get_boundary_locations(image_height, image_width,
+                                                     boundary_points_per_edge)
+
+  boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2])
+
+  type_to_use = control_point_locations.dtype
+  boundary_point_locations = constant_op.constant(
+      _expand_to_minibatch(boundary_point_locations, batch_size),
+      dtype=type_to_use)
+
+  boundary_point_flows = constant_op.constant(
+      _expand_to_minibatch(boundary_point_flows, batch_size), dtype=type_to_use)
+
+  merged_control_point_locations = array_ops.concat(
+      [control_point_locations, boundary_point_locations], 1)
+
+  merged_control_point_flows = array_ops.concat(
+      [control_point_flows, boundary_point_flows], 1)
+
+  return merged_control_point_locations, merged_control_point_flows
+
+
+def sparse_image_warp(image,
+                      source_control_point_locations,
+                      dest_control_point_locations,
+                      interpolation_order=2,
+                      regularization_weight=0.0,
+                      num_boundary_points=0,
+                      name='sparse_image_warp'):
+  """Image warping using correspondences between sparse control points.
+
+  Apply a non-linear warp to the image, where the warp is specified by
+  the source and destination locations of a (potentially small) number of
+  control points. First, we use a polyharmonic spline
+  (@{tf.contrib.image.interpolate_spline}) to interpolate the displacements
+  between the corresponding control points to a dense flow field.
+  Then, we warp the image using this dense flow field
+  (@{tf.contrib.image.dense_image_warp}).
+
+  Let t index our control points. For regularization_weight=0, we have:
+  warped_image[b, dest_control_point_locations[b, t, 0],
+                  dest_control_point_locations[b, t, 1], :] =
+  image[b, source_control_point_locations[b, t, 0],
+           source_control_point_locations[b, t, 1], :].
+
+  For regularization_weight > 0, this condition is met approximately, since
+  regularized interpolation trades off smoothness of the interpolant vs.
+  reconstruction of the interpolant at the control points.
+  See @{tf.contrib.image.interpolate_spline} for further documentation of the
+  interpolation_order and regularization_weight arguments.
+
+
+  Args:
+    image: `[batch, height, width, channels]` float `Tensor`
+    source_control_point_locations: `[batch, num_control_points, 2]` float
+      `Tensor`
+    dest_control_point_locations: `[batch, num_control_points, 2]` float
+      `Tensor`
+    interpolation_order: polynomial order used by the spline interpolation
+    regularization_weight: weight on smoothness regularizer in interpolation
+    num_boundary_points: How many zero-flow boundary points to include at
+      each image edge.Usage:
+        num_boundary_points=0: don't add zero-flow points
+        num_boundary_points=1: 4 corners of the image
+        num_boundary_points=2: 4 corners and one in the middle of each edge
+          (8 points total)
+        num_boundary_points=n: 4 corners and n-1 along each edge
+    name: A name for the operation (optional).
+
+    Note that image and offsets can be of type tf.half, tf.float32, or
+    tf.float64, and do not necessarily have to be the same type.
+
+  Returns:
+    warped_image: `[batch, height, width, channels]` float `Tensor` with same
+      type as input image.
+    flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense
+      flow field produced by the interpolation.
+  """
+
+  control_point_flows = (
+      dest_control_point_locations - source_control_point_locations)
+
+  clamp_boundaries = num_boundary_points > 0
+  boundary_points_per_edge = num_boundary_points - 1
+
+  with ops.name_scope(name):
+
+    batch_size, image_height, image_width, _ = image.get_shape().as_list()
+
+    # This generates the dense locations where the interpolant
+    # will be evaluated.
+    grid_locations = _get_grid_locations(image_height, image_width)
+
+    flattened_grid_locations = np.reshape(grid_locations,
+                                          [image_height * image_width, 2])
+
+    flattened_grid_locations = constant_op.constant(
+        _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype)
+
+    if clamp_boundaries:
+      (dest_control_point_locations,
+       control_point_flows) = _add_zero_flow_controls_at_boundary(
+           dest_control_point_locations, control_point_flows, image_height,
+           image_width, boundary_points_per_edge)
+
+    flattened_flows = interpolate_spline.interpolate_spline(
+        dest_control_point_locations, control_point_flows,
+        flattened_grid_locations, interpolation_order, regularization_weight)
+
+    dense_flows = array_ops.reshape(flattened_flows,
+                                    [batch_size, image_height, image_width, 2])
+
+    warped_image = dense_image_warp.dense_image_warp(image, dense_flows)
+
+    return warped_image, dense_flows
-- 
GitLab


From a12d5cc7d5aa3d159312424a2b84d33a7648775d Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Wed, 14 Mar 2018 15:02:20 -0700
Subject: [PATCH 033/960] Ensure zeros constant created is dominated by Switch
 node of cond context.

All values entering control flow context either has to be via a switch or dominated by one.

PiperOrigin-RevId: 189092017
---
 .../python/kernel_tests/control_flow_ops_py_test.py | 13 ++++++++++---
 tensorflow/python/ops/control_flow_ops.py           |  5 ++++-
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index d47b030fa1..5257826ebd 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -633,7 +633,8 @@ class ControlFlowTest(test.TestCase):
       sess.run(r)
 
   def testCondGrad_1(self):
-    with self.test_session():
+    graph = ops.Graph()
+    with graph.as_default():
       x = constant_op.constant(10.0, name="x")
       pred = math_ops.less(1, 2)
       fn1 = lambda: array_ops.identity(x)
@@ -641,8 +642,14 @@ class ControlFlowTest(test.TestCase):
       r = control_flow_ops.cond(pred, fn1, fn2)
 
       grad = gradients_impl.gradients(r, [x])[0]
-      result = grad.eval()
-    self.assertAllEqual(1.0, result)
+      with self.test_session():
+        self.assertAllEqual(1.0, grad.eval())
+    # The gradients computation creates a tensor with zeros by broadcasting a
+    # zeros constant to the required shape. Verify that the zero constant
+    # feeding into the fill is dominated by a Switch.
+    zero = graph.get_operation_by_name("gradients/zeros/Const")
+    self.assertEqual(len(zero.control_inputs), 1)
+    self.assertEqual(zero.control_inputs[0].type, "Switch")
 
   def testCondGrad_2(self):
     with self.test_session():
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 24c30802b5..20da66c303 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -1467,7 +1467,10 @@ def ZerosLikeOutsideLoop(op, index):
       branch = op_ctxt.branch
       switch_val = switch(op.inputs[0], pred)[1 - branch]
       zeros_shape = array_ops.shape_internal(switch_val, optimize=False)
-      return array_ops.zeros(zeros_shape, dtype=val.dtype)
+      # Ensure ops created within array_ops.zeros are dominated by switch in
+      # cond context.
+      with ops.control_dependencies([switch_val]):
+        return array_ops.zeros(zeros_shape, dtype=val.dtype)
     else:
       return array_ops.zeros_like(val, optimize=False)
 
-- 
GitLab


From c6dbbdc339ab351700793885417d12fd7172d14c Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 14 Mar 2018 15:25:53 -0700
Subject: [PATCH 034/960] Remove underscore prefix from ref_identity op.

PiperOrigin-RevId: 189096108
---
 tensorflow/python/framework/python_op_gen.cc        |  2 +-
 tensorflow/python/grappler/item_test.py             |  2 +-
 .../python/kernel_tests/control_flow_ops_py_test.py | 13 +++++--------
 .../python/kernel_tests/identity_op_py_test.py      |  2 +-
 tensorflow/python/ops/control_flow_ops.py           |  2 +-
 5 files changed, 9 insertions(+), 12 deletions(-)

diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 6ee8e554de..846e0c356c 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -100,7 +100,7 @@ bool IsOpWithUnderscorePrefix(const string& s) {
        "fused_batch_norm", "histogram_fixed_width", "stack",
        "batch_norm_with_global_normalization",
        // TODO(annarev): replace these ops in the next change.
-       "broadcast_gradient_args", "ref_identity"});
+       "broadcast_gradient_args"});
   return kUnderscoreOps->count(s) > 0;
 }
 
diff --git a/tensorflow/python/grappler/item_test.py b/tensorflow/python/grappler/item_test.py
index 7c3efd6249..c40de9da0a 100644
--- a/tensorflow/python/grappler/item_test.py
+++ b/tensorflow/python/grappler/item_test.py
@@ -111,7 +111,7 @@ class ItemTest(test.TestCase):
     with ops.Graph().as_default() as g:
       c = constant_op.constant([10])
       v = variables.Variable([3], dtype=dtypes.int32)
-      i = gen_array_ops._ref_identity(v)
+      i = gen_array_ops.ref_identity(v)
       a = state_ops.assign(i, c)
       train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
       train_op.append(a)
diff --git a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
index 5257826ebd..75f8644f69 100644
--- a/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
+++ b/tensorflow/python/kernel_tests/control_flow_ops_py_test.py
@@ -591,10 +591,10 @@ class ControlFlowTest(test.TestCase):
       # Both v_f and v_t are uninitialized references. However, an actual use
       # of the reference in the 'true' branch in the 'tf.identity' op will
       # not 'fire' when v is uninitialized, so this is a valid construction.
-      # This test tests that _ref_identity allows uninitialized ref as input
+      # This test tests that ref_identity allows uninitialized ref as input
       # so that this construction is allowed.
-      v_f_op = gen_array_ops._ref_identity(v_f)
-      v_t_op = gen_array_ops._ref_identity(v_t)
+      v_f_op = gen_array_ops.ref_identity(v_f)
+      v_t_op = gen_array_ops.ref_identity(v_t)
       with ops.control_dependencies([v_f_op]):
         assign_v = state_ops.assign(v, [1.0])
       with ops.control_dependencies([v_t_op]):
@@ -751,7 +751,7 @@ class ControlFlowTest(test.TestCase):
 
       def b(i, x):
         self.assertEqual(x.dtype, dtypes.int32_ref)
-        return (i + 1, gen_array_ops._ref_identity(x))
+        return (i + 1, gen_array_ops.ref_identity(x))
 
       r = control_flow_ops.while_loop(c, b, [i, x], parallel_iterations=5)
 
@@ -2212,12 +2212,9 @@ class ControlFlowTest(test.TestCase):
 
       self.assertEqual(x.dtype, dtypes.int32_ref)
 
-      # pylint: disable=protected-access
       def body(i, x):
         self.assertEqual(x.dtype, dtypes.int32_ref)
-        return [i + 1, gen_array_ops._ref_identity(x)]
-
-      # pylint: enable=protected-access
+        return [i + 1, gen_array_ops.ref_identity(x)]
 
       r = control_flow_ops.while_loop(c, body, [i, x], parallel_iterations=5)
 
diff --git a/tensorflow/python/kernel_tests/identity_op_py_test.py b/tensorflow/python/kernel_tests/identity_op_py_test.py
index 2cfe420bd4..49fb76d5b4 100644
--- a/tensorflow/python/kernel_tests/identity_op_py_test.py
+++ b/tensorflow/python/kernel_tests/identity_op_py_test.py
@@ -65,7 +65,7 @@ class IdentityOpTest(test.TestCase):
           constant_op.constant(
               [[1, 2, 3], [6, 5, 4]], dtype=dtypes.int32))
       self.assertEquals(shape, tensor.get_shape())
-      self.assertEquals(shape, gen_array_ops._ref_identity(tensor).get_shape())
+      self.assertEquals(shape, gen_array_ops.ref_identity(tensor).get_shape())
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 20da66c303..1278768d8b 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -196,7 +196,7 @@ def _Identity(data, name=None):
   data = ops.internal_convert_to_tensor_or_indexed_slices(data, as_ref=True)
   if isinstance(data, ops.Tensor):
     if data.dtype._is_ref_dtype:  # pylint: disable=protected-access
-      return gen_array_ops._ref_identity(data, name=name)
+      return gen_array_ops.ref_identity(data, name=name)
     else:
       return array_ops.identity(data, name=name)
   else:
-- 
GitLab


From c77422ff3fe8b8cab831488f449fb1a3c64127b1 Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Wed, 14 Mar 2018 15:30:20 -0700
Subject: [PATCH 035/960] Re-phrase some tests in replicate_model_fn using
 parameterized tests.

PiperOrigin-RevId: 189096779
---
 tensorflow/contrib/estimator/BUILD            |  1 +
 .../estimator/replicate_model_fn_test.py      | 26 +++++++------------
 2 files changed, 11 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 773c6ab6c7..26d6bc5ae6 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -354,6 +354,7 @@ cuda_py_test(
     size = "medium",
     srcs = ["python/estimator/replicate_model_fn_test.py"],
     additional_deps = [
+        "//third_party/py/absl/testing:parameterized",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:dnn",
         "//tensorflow/python/estimator:export_export",
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py
index d46a18aacf..144b45982c 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import re
 import shutil
 import tempfile
+from absl.testing import parameterized
 import numpy as np
 import six
 
@@ -57,26 +58,19 @@ from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import training
 
 
-# TODO(isaprykin):  Parametrize all the tests on
-#   replicate_model_fn._VariableDistributionMode when it's supported.
-class DNNClassifierIntegrationTest(test_util.TensorFlowTestCase):
+class DNNClassifierIntegrationTest(test_util.TensorFlowTestCase,
+                                   parameterized.TestCase):
 
   def setUp(self):
     self._model_dir = tempfile.mkdtemp()
 
-  def test_complete_flow_with_public_version(self):
-    return self._complete_flow_with_mode(mode=None)
-
-  def test_complete_flow_with_mode_local_ps_server(self):
-    return self._complete_flow_with_mode(
-        replicate_model_fn._VariableDistributionMode.
-        SHARED_LOCAL_PARAMETER_SERVER)
-
-  def test_complete_flow_with_mode_round_robin(self):
-    return self._complete_flow_with_mode(
-        replicate_model_fn._VariableDistributionMode.SHARED_ROUND_ROBIN)
-
-  def _complete_flow_with_mode(self, mode):
+  @parameterized.named_parameters(
+      ('PublicInterface', None),
+      ('ParameterServerMode', replicate_model_fn._VariableDistributionMode.
+       SHARED_LOCAL_PARAMETER_SERVER),
+      ('RoundRobinMode',
+       replicate_model_fn._VariableDistributionMode.SHARED_ROUND_ROBIN))
+  def test_complete_flow_with_mode(self, mode):
     n_classes = 3
     input_dimension = 2
     batch_size = 12
-- 
GitLab


From 124a1835637fb71d84087430f79fe166b394f791 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Wed, 14 Mar 2018 15:36:32 -0700
Subject: [PATCH 036/960] Automated g4 rollback of changelist 189071037

PiperOrigin-RevId: 189097692
---
 tensorflow/core/lib/wav/wav_io.cc      |  74 +++---
 tensorflow/core/lib/wav/wav_io.h       |  33 ---
 tensorflow/core/lib/wav/wav_io_test.cc | 319 -------------------------
 3 files changed, 32 insertions(+), 394 deletions(-)

diff --git a/tensorflow/core/lib/wav/wav_io.cc b/tensorflow/core/lib/wav/wav_io.cc
index 2165415ba5..77d3c88998 100644
--- a/tensorflow/core/lib/wav/wav_io.cc
+++ b/tensorflow/core/lib/wav/wav_io.cc
@@ -81,42 +81,13 @@ inline float Int16SampleToFloat(int16 data) {
   return data * kMultiplier;
 }
 
-}  // namespace
-
-// Handles moving the data index forward, validating the arguments, and avoiding
-// overflow or underflow.
-Status IncrementOffset(int old_offset, size_t increment, size_t max_size,
-                       int* new_offset) {
-  if (old_offset < 0) {
-    return errors::InvalidArgument("Negative offsets are not allowed: ",
-                                   old_offset);
-  }
-  if (old_offset > max_size) {
-    return errors::InvalidArgument("Initial offset is outside data range: ",
-                                   old_offset);
-  }
-  if (increment < 0) {
-    return errors::InvalidArgument("Negative increments are not allowed: ",
-                                   old_offset);
-  }
-  *new_offset = old_offset + increment;
-  if (*new_offset > max_size) {
-    return errors::InvalidArgument("Data too short when trying to read string");
-  }
-  // See above for the check that the input offset is positive. If it's negative
-  // here then it means that there's been an overflow in the arithmetic.
-  if (*new_offset < 0) {
-    return errors::InvalidArgument("Offset too large, overflowed: ",
-                                   *new_offset);
-  }
-  return Status::OK();
-}
-
 Status ExpectText(const string& data, const string& expected_text,
                   int* offset) {
-  int new_offset;
-  TF_RETURN_IF_ERROR(
-      IncrementOffset(*offset, expected_text.size(), data.size(), &new_offset));
+  const int new_offset = *offset + expected_text.size();
+  if (new_offset > data.size()) {
+    return errors::InvalidArgument("Data too short when trying to read ",
+                                   expected_text);
+  }
   const string found_text(data.begin() + *offset, data.begin() + new_offset);
   if (found_text != expected_text) {
     return errors::InvalidArgument("Header mismatch: Expected ", expected_text,
@@ -126,16 +97,40 @@ Status ExpectText(const string& data, const string& expected_text,
   return Status::OK();
 }
 
+template <class T>
+Status ReadValue(const string& data, T* value, int* offset) {
+  const int new_offset = *offset + sizeof(T);
+  if (new_offset > data.size()) {
+    return errors::InvalidArgument("Data too short when trying to read value");
+  }
+  if (port::kLittleEndian) {
+    memcpy(value, data.data() + *offset, sizeof(T));
+  } else {
+    *value = 0;
+    const uint8* data_buf =
+        reinterpret_cast<const uint8*>(data.data() + *offset);
+    int shift = 0;
+    for (int i = 0; i < sizeof(T); ++i, shift += 8) {
+      *value = *value | (data_buf[i] << shift);
+    }
+  }
+  *offset = new_offset;
+  return Status::OK();
+}
+
 Status ReadString(const string& data, int expected_length, string* value,
                   int* offset) {
-  int new_offset;
-  TF_RETURN_IF_ERROR(
-      IncrementOffset(*offset, expected_length, data.size(), &new_offset));
+  const int new_offset = *offset + expected_length;
+  if (new_offset > data.size()) {
+    return errors::InvalidArgument("Data too short when trying to read string");
+  }
   *value = string(data.begin() + *offset, data.begin() + new_offset);
   *offset = new_offset;
   return Status::OK();
 }
 
+}  // namespace
+
 Status EncodeAudioAsS16LEWav(const float* audio, size_t sample_rate,
                              size_t num_channels, size_t num_frames,
                              string* wav_string) {
@@ -277,11 +272,6 @@ Status DecodeLin16WaveAsFloatVector(const string& wav_string,
     TF_RETURN_IF_ERROR(ReadString(wav_string, 4, &chunk_id, &offset));
     uint32 chunk_size;
     TF_RETURN_IF_ERROR(ReadValue<uint32>(wav_string, &chunk_size, &offset));
-    if (chunk_size > std::numeric_limits<int32>::max()) {
-      return errors::InvalidArgument(
-          "WAV data chunk '", chunk_id, "' is too large: ", chunk_size,
-          " bytes, but the limit is ", std::numeric_limits<int32>::max());
-    }
     if (chunk_id == kDataChunkId) {
       if (was_data_found) {
         return errors::InvalidArgument("More than one data chunk found in WAV");
diff --git a/tensorflow/core/lib/wav/wav_io.h b/tensorflow/core/lib/wav/wav_io.h
index f004524177..adca0ee303 100644
--- a/tensorflow/core/lib/wav/wav_io.h
+++ b/tensorflow/core/lib/wav/wav_io.h
@@ -21,9 +21,6 @@ limitations under the License.
 #include <string>
 #include <vector>
 
-#include "tensorflow/core/lib/core/casts.h"
-#include "tensorflow/core/lib/core/coding.h"
-#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -58,36 +55,6 @@ Status DecodeLin16WaveAsFloatVector(const string& wav_string,
                                     uint32* sample_count, uint16* channel_count,
                                     uint32* sample_rate);
 
-// Everything below here is only exposed publicly for testing purposes.
-
-// Handles moving the data index forward, validating the arguments, and avoiding
-// overflow or underflow.
-Status IncrementOffset(int old_offset, size_t increment, size_t max_size,
-                       int* new_offset);
-
-// This function is only exposed in the header for testing purposes, as a
-// template that needs to be instantiated. Reads a typed numeric value from a
-// stream of data.
-template <class T>
-Status ReadValue(const string& data, T* value, int* offset) {
-  int new_offset;
-  TF_RETURN_IF_ERROR(
-      IncrementOffset(*offset, sizeof(T), data.size(), &new_offset));
-  if (port::kLittleEndian) {
-    memcpy(value, data.data() + *offset, sizeof(T));
-  } else {
-    *value = 0;
-    const uint8* data_buf =
-        reinterpret_cast<const uint8*>(data.data() + *offset);
-    int shift = 0;
-    for (int i = 0; i < sizeof(T); ++i, shift += 8) {
-      *value = *value | (data_buf[i] << shift);
-    }
-  }
-  *offset = new_offset;
-  return Status::OK();
-}
-
 }  // namespace wav
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/lib/wav/wav_io_test.cc b/tensorflow/core/lib/wav/wav_io_test.cc
index d8a83fc464..40ddd94abe 100644
--- a/tensorflow/core/lib/wav/wav_io_test.cc
+++ b/tensorflow/core/lib/wav/wav_io_test.cc
@@ -25,12 +25,6 @@ limitations under the License.
 namespace tensorflow {
 namespace wav {
 
-// These are defined in wav_io.cc, and the signatures are here so we don't have
-// to expose them in the public header.
-Status ExpectText(const string& data, const string& expected_text, int* offset);
-Status ReadString(const string& data, int expected_length, string* value,
-                  int* offset);
-
 TEST(WavIO, BadArguments) {
   float audio[] = {0.0f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f};
   string result;
@@ -161,318 +155,5 @@ TEST(WavIO, BasicStereo) {
   EXPECT_EQ(expected, result);
 }
 
-// Test how chunk sizes larger than 2GB are handled, since they're stored as
-// unsigned int32s, so there are lots of ways for conversions to confuse the
-// decoding logic. The expected behavior is to fail with an error, since such
-// large WAV files are not common, and are unsupported by many readers.
-// See b/72655902.
-TEST(WavIO, ChunkSizeOverflow) {
-  std::vector<uint8> wav_data = {
-      'R', 'I', 'F', 'F',      // ChunkID
-      60, 0, 0, 0,             // ChunkSize: 36 + SubChunk2Size
-      'W', 'A', 'V', 'E',      // Format
-      'f', 'm', 't', ' ',      // Subchunk1ID
-      16, 0, 0, 0,             // Subchunk1Size
-      1, 0,                    // AudioFormat: 1=PCM
-      1, 0,                    // NumChannels
-      0x44, 0xac, 0, 0,        // SampleRate: 44100
-      0x88, 0x58, 0x1, 0,      // BytesPerSecond: SampleRate * NumChannels *
-                               //                 BitsPerSample/8
-      2, 0,                    // BytesPerSample: NumChannels * BitsPerSample/8
-      16, 0,                   // BitsPerSample
-      'd', 'a', 't', 'a',      // Subchunk2ID
-      8, 0, 0, 0,              // Subchunk2Size: NumSamples * NumChannels *
-                               //                BitsPerSample/8
-      0, 0,                    // Sample 1: 0
-      0xff, 0x7f,              // Sample 2: 32767 (saturated)
-      0, 0,                    // Sample 3: 0
-      0x00, 0x80,              // Sample 4: -32768 (saturated)
-      'f', 'o', 'o', 'o',      // Subchunk2ID
-      0xff, 0xff, 0xff, 0xf8,  // Chunk size that could cause an infinite loop.
-      0, 0,                    // Sample 1: 0
-      0xff, 0x7f,              // Sample 2: 32767 (saturated)
-      0, 0,                    // Sample 3: 0
-      0x00, 0x80,              // Sample 4: -32768 (saturated)
-  };
-  string wav_data_string(wav_data.begin(), wav_data.end());
-  std::vector<float> decoded_audio;
-  uint32 decoded_sample_count;
-  uint16 decoded_channel_count;
-  uint32 decoded_sample_rate;
-  Status decode_status = DecodeLin16WaveAsFloatVector(
-      wav_data_string, &decoded_audio, &decoded_sample_count,
-      &decoded_channel_count, &decoded_sample_rate);
-  EXPECT_FALSE(decode_status.ok());
-  EXPECT_TRUE(StringPiece(decode_status.error_message()).contains("too large"))
-      << decode_status.error_message();
-}
-
-TEST(WavIO, IncrementOffset) {
-  int new_offset = -1;
-  TF_EXPECT_OK(IncrementOffset(0, 10, 20, &new_offset));
-  EXPECT_EQ(10, new_offset);
-
-  new_offset = -1;
-  TF_EXPECT_OK(IncrementOffset(10, 4, 20, &new_offset));
-  EXPECT_EQ(14, new_offset);
-
-  new_offset = -1;
-  TF_EXPECT_OK(IncrementOffset(99, 1, 100, &new_offset));
-  EXPECT_EQ(100, new_offset);
-
-  new_offset = -1;
-  EXPECT_FALSE(IncrementOffset(-1, 1, 100, &new_offset).ok());
-
-  new_offset = -1;
-  EXPECT_FALSE(IncrementOffset(0, -1, 100, &new_offset).ok());
-
-  new_offset = -1;
-  EXPECT_FALSE(IncrementOffset(std::numeric_limits<int>::max(), 1,
-                               std::numeric_limits<int>::max(), &new_offset)
-                   .ok());
-
-  new_offset = -1;
-  EXPECT_FALSE(IncrementOffset(101, 1, 100, &new_offset).ok());
-}
-
-TEST(WavIO, ExpectText) {
-  std::vector<uint8> test_data = {
-      'E', 'x', 'p', 'e', 'c', 't', 'e', 'd',
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  TF_EXPECT_OK(ExpectText(test_string, "Expected", &offset));
-  EXPECT_EQ(8, offset);
-
-  offset = 0;
-  Status expect_status = ExpectText(test_string, "Unexpected", &offset);
-  EXPECT_FALSE(expect_status.ok());
-
-  offset = 0;
-  TF_EXPECT_OK(ExpectText(test_string, "Exp", &offset));
-  EXPECT_EQ(3, offset);
-  TF_EXPECT_OK(ExpectText(test_string, "ected", &offset));
-  EXPECT_EQ(8, offset);
-  expect_status = ExpectText(test_string, "foo", &offset);
-  EXPECT_FALSE(expect_status.ok());
-}
-
-TEST(WavIO, ReadString) {
-  std::vector<uint8> test_data = {
-      'E', 'x', 'p', 'e', 'c', 't', 'e', 'd',
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  string read_value;
-  TF_EXPECT_OK(ReadString(test_string, 2, &read_value, &offset));
-  EXPECT_EQ("Ex", read_value);
-  EXPECT_EQ(2, offset);
-
-  TF_EXPECT_OK(ReadString(test_string, 6, &read_value, &offset));
-  EXPECT_EQ("pected", read_value);
-  EXPECT_EQ(8, offset);
-
-  Status read_status = ReadString(test_string, 3, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueInt8) {
-  std::vector<uint8> test_data = {0x00, 0x05, 0xff, 0x80};
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  int8 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(1, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(5, read_value);
-  EXPECT_EQ(2, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(-1, read_value);
-  EXPECT_EQ(3, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(-128, read_value);
-  EXPECT_EQ(4, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueUInt8) {
-  std::vector<uint8> test_data = {0x00, 0x05, 0xff, 0x80};
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  uint8 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(1, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(5, read_value);
-  EXPECT_EQ(2, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(255, read_value);
-  EXPECT_EQ(3, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(128, read_value);
-  EXPECT_EQ(4, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueInt16) {
-  std::vector<uint8> test_data = {
-      0x00, 0x00,  // 0
-      0xff, 0x00,  // 255
-      0x00, 0x01,  // 256
-      0xff, 0xff,  // -1
-      0x00, 0x80,  // -32768
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  int16 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(2, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(255, read_value);
-  EXPECT_EQ(4, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(256, read_value);
-  EXPECT_EQ(6, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(-1, read_value);
-  EXPECT_EQ(8, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(-32768, read_value);
-  EXPECT_EQ(10, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueUInt16) {
-  std::vector<uint8> test_data = {
-      0x00, 0x00,  // 0
-      0xff, 0x00,  // 255
-      0x00, 0x01,  // 256
-      0xff, 0xff,  // 65535
-      0x00, 0x80,  // 32768
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  uint16 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(2, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(255, read_value);
-  EXPECT_EQ(4, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(256, read_value);
-  EXPECT_EQ(6, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(65535, read_value);
-  EXPECT_EQ(8, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(32768, read_value);
-  EXPECT_EQ(10, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueInt32) {
-  std::vector<uint8> test_data = {
-      0x00, 0x00, 0x00, 0x00,  // 0
-      0xff, 0x00, 0x00, 0x00,  // 255
-      0x00, 0xff, 0x00, 0x00,  // 65280
-      0x00, 0x00, 0xff, 0x00,  // 16,711,680
-      0xff, 0xff, 0xff, 0xff,  // -1
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  int32 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(4, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(255, read_value);
-  EXPECT_EQ(8, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(65280, read_value);
-  EXPECT_EQ(12, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(16711680, read_value);
-  EXPECT_EQ(16, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(-1, read_value);
-  EXPECT_EQ(20, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
-TEST(WavIO, ReadValueUInt32) {
-  std::vector<uint8> test_data = {
-      0x00, 0x00, 0x00, 0x00,  // 0
-      0xff, 0x00, 0x00, 0x00,  // 255
-      0x00, 0xff, 0x00, 0x00,  // 65280
-      0x00, 0x00, 0xff, 0x00,  // 16,711,680
-      0xff, 0xff, 0xff, 0xff,  // 4,294,967,295
-  };
-  string test_string(test_data.begin(), test_data.end());
-
-  int offset = 0;
-  uint32 read_value;
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(0, read_value);
-  EXPECT_EQ(4, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(255, read_value);
-  EXPECT_EQ(8, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(65280, read_value);
-  EXPECT_EQ(12, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(16711680, read_value);
-  EXPECT_EQ(16, offset);
-
-  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
-  EXPECT_EQ(4294967295, read_value);
-  EXPECT_EQ(20, offset);
-
-  Status read_status = ReadValue(test_string, &read_value, &offset);
-  EXPECT_FALSE(read_status.ok());
-}
-
 }  // namespace wav
 }  // namespace tensorflow
-- 
GitLab


From 33792456dbe0600b5c23f8cbffea0e74a69386c1 Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Wed, 14 Mar 2018 15:58:15 -0700
Subject: [PATCH 037/960] Automated g4 rollback of changelist 188525171

PiperOrigin-RevId: 189100846
---
 tensorflow/contrib/lite/kernels/BUILD         |  31 +++-
 .../contrib/lite/kernels/audio_spectrogram.cc | 165 ++++++++++++++++++
 .../lite/kernels/audio_spectrogram_test.cc    | 122 +++++++++++++
 .../lite/kernels/internal/spectrogram.cc      |   1 -
 tensorflow/contrib/lite/kernels/mfcc.cc       | 154 ++++++++++++++++
 tensorflow/contrib/lite/kernels/mfcc_test.cc  | 104 +++++++++++
 tensorflow/contrib/lite/kernels/register.cc   |  14 ++
 7 files changed, 589 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/audio_spectrogram.cc
 create mode 100644 tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc
 create mode 100644 tensorflow/contrib/lite/kernels/mfcc.cc
 create mode 100644 tensorflow/contrib/lite/kernels/mfcc_test.cc

diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index b8ab6d96a0..9c63269324 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -135,6 +135,7 @@ cc_library(
     srcs = [
         "activations.cc",
         "add.cc",
+        "audio_spectrogram.cc",
         "basic_rnn.cc",
         "batch_to_space_nd.cc",
         "bidirectional_sequence_lstm.cc",
@@ -156,6 +157,7 @@ cc_library(
         "lsh_projection.cc",
         "lstm.cc",
         "mean.cc",
+        "mfcc.cc",
         "mul.cc",
         "pad.cc",
         "pooling.cc",
@@ -196,15 +198,42 @@ cc_library(
         "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite:string_util",
         "//tensorflow/contrib/lite/kernels:gemm_support",
+        "//tensorflow/contrib/lite/kernels/internal:audio_utils",
         "//tensorflow/contrib/lite/kernels/internal:kernel_utils",
         "//tensorflow/contrib/lite/kernels/internal:optimized",
         "//tensorflow/contrib/lite/kernels/internal:optimized_base",
         "//tensorflow/contrib/lite/kernels/internal:quantization_util",
         "//tensorflow/contrib/lite/kernels/internal:reference",
         "//tensorflow/contrib/lite/kernels/internal:reference_base",
-        "//tensorflow/contrib/lite/kernels/internal:round",
         "//tensorflow/contrib/lite/kernels/internal:tensor_utils",
         "@farmhash_archive//:farmhash",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "audio_spectrogram_test",
+    size = "small",
+    srcs = ["audio_spectrogram_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
+    ],
+)
+
+tf_cc_test(
+    name = "mfcc_test",
+    size = "small",
+    srcs = ["mfcc_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+        "@flatbuffers",
     ],
 )
 
diff --git a/tensorflow/contrib/lite/kernels/audio_spectrogram.cc b/tensorflow/contrib/lite/kernels/audio_spectrogram.cc
new file mode 100644
index 0000000000..602f3888c1
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/audio_spectrogram.cc
@@ -0,0 +1,165 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/spectrogram.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+#include "tensorflow/contrib/lite/kernels/op_macros.h"
+
+#include "flatbuffers/flexbuffers.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace audio_spectrogram {
+
+constexpr int kInputTensor = 0;
+constexpr int kOutputTensor = 0;
+
+enum KernelType {
+  kReference,
+};
+
+typedef struct {
+  int window_size;
+  int stride;
+  bool magnitude_squared;
+  int output_height;
+  internal::Spectrogram* spectrogram;
+} TfLiteAudioSpectrogramParams;
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new TfLiteAudioSpectrogramParams;
+
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+  data->window_size = m["window_size"].AsInt64();
+  data->stride = m["stride"].AsInt64();
+  data->magnitude_squared = m["magnitude_squared"].AsBool();
+
+  data->spectrogram = new internal::Spectrogram;
+
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  auto* params = reinterpret_cast<TfLiteAudioSpectrogramParams*>(buffer);
+  delete params->spectrogram;
+  delete params;
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  auto* params =
+      reinterpret_cast<TfLiteAudioSpectrogramParams*>(node->user_data);
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 1);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  TF_LITE_ENSURE_EQ(context, NumDimensions(input), 2);
+
+  TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32);
+  TF_LITE_ENSURE_EQ(context, input->type, output->type);
+
+  TF_LITE_ENSURE(context, params->spectrogram->Initialize(params->window_size,
+                                                          params->stride));
+  const int64_t sample_count = input->dims->data[0];
+  const int64_t length_minus_window = (sample_count - params->window_size);
+  if (length_minus_window < 0) {
+    params->output_height = 0;
+  } else {
+    params->output_height = 1 + (length_minus_window / params->stride);
+  }
+  TfLiteIntArray* output_size = TfLiteIntArrayCreate(3);
+  output_size->data[0] = input->dims->data[1];
+  output_size->data[1] = params->output_height;
+  output_size->data[2] = params->spectrogram->output_frequency_channels();
+
+  return context->ResizeTensor(context, output, output_size);
+}
+
+template <KernelType kernel_type>
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  auto* params =
+      reinterpret_cast<TfLiteAudioSpectrogramParams*>(node->user_data);
+
+  TfLiteTensor* input = GetInput(context, node, kInputTensor);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  TF_LITE_ENSURE(context, params->spectrogram->Initialize(params->window_size,
+                                                          params->stride));
+
+  const float* input_data = GetTensorData<float>(input);
+
+  const int64_t sample_count = input->dims->data[0];
+  const int64_t channel_count = input->dims->data[1];
+
+  const int64_t output_width = params->spectrogram->output_frequency_channels();
+
+  float* output_flat = GetTensorData<float>(output);
+
+  std::vector<float> input_for_channel(sample_count);
+  for (int64_t channel = 0; channel < channel_count; ++channel) {
+    float* output_slice =
+        output_flat + (channel * params->output_height * output_width);
+    for (int i = 0; i < sample_count; ++i) {
+      input_for_channel[i] = input_data[i * channel_count + channel];
+    }
+    std::vector<std::vector<float>> spectrogram_output;
+    TF_LITE_ENSURE(context,
+                   params->spectrogram->ComputeSquaredMagnitudeSpectrogram(
+                       input_for_channel, &spectrogram_output));
+    TF_LITE_ENSURE_EQ(context, spectrogram_output.size(),
+                      params->output_height);
+    TF_LITE_ENSURE(context, spectrogram_output.empty() ||
+                                (spectrogram_output[0].size() == output_width));
+    for (int row_index = 0; row_index < params->output_height; ++row_index) {
+      const std::vector<float>& spectrogram_row = spectrogram_output[row_index];
+      TF_LITE_ENSURE_EQ(context, spectrogram_row.size(), output_width);
+      float* output_row = output_slice + (row_index * output_width);
+      if (params->magnitude_squared) {
+        for (int i = 0; i < output_width; ++i) {
+          output_row[i] = spectrogram_row[i];
+        }
+      } else {
+        for (int i = 0; i < output_width; ++i) {
+          output_row[i] = sqrtf(spectrogram_row[i]);
+        }
+      }
+    }
+  }
+  return kTfLiteOk;
+}
+
+}  // namespace audio_spectrogram
+
+TfLiteRegistration* Register_AUDIO_SPECTROGRAM() {
+  static TfLiteRegistration r = {
+      audio_spectrogram::Init, audio_spectrogram::Free,
+      audio_spectrogram::Prepare,
+      audio_spectrogram::Eval<audio_spectrogram::kReference>};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc b/tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc
new file mode 100644
index 0000000000..8d460fdfc6
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/audio_spectrogram_test.cc
@@ -0,0 +1,122 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
+
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
+
+class BaseAudioSpectrogramOpModel : public SingleOpModel {
+ public:
+  BaseAudioSpectrogramOpModel(const TensorData& input1,
+                              const TensorData& output, int window_size,
+                              int stride, bool magnitude_squared) {
+    input1_ = AddInput(input1);
+    output_ = AddOutput(output);
+
+    flexbuffers::Builder fbb;
+    fbb.Map([&]() {
+      fbb.Int("window_size", window_size);
+      fbb.Int("stride", stride);
+      fbb.Bool("magnitude_squared", magnitude_squared);
+    });
+    fbb.Finish();
+    SetCustomOp("AudioSpectrogram", fbb.GetBuffer(),
+                Register_AUDIO_SPECTROGRAM);
+    BuildInterpreter({GetShape(input1_)});
+  }
+
+  int input1() { return input1_; }
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ protected:
+  int input1_;
+  int output_;
+};
+
+TEST(BaseAudioSpectrogramOpModel, NonSquaredTest) {
+  BaseAudioSpectrogramOpModel m({TensorType_FLOAT32, {8, 1}},
+                                {TensorType_FLOAT32, {}}, 8, 1, false);
+  m.PopulateTensor<float>(m.input1(),
+                          {-1.0f, 0.0f, 1.0f, 0.0f, -1.0f, 0.0f, 1.0f, 0.0f});
+
+  m.Invoke();
+
+  std::vector<int> output_shape = m.GetOutputShape();
+  EXPECT_EQ(3, output_shape.size());
+  EXPECT_THAT(output_shape, ElementsAre(1, 1, 5));
+
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear(
+                                 {0.0f, 1.0f, 2.0f, 1.0f, 0.0f}, 1e-3)));
+}
+
+TEST(SpectrogramOpTest, SquaredTest) {
+  BaseAudioSpectrogramOpModel m({TensorType_FLOAT32, {8, 1}},
+                                {TensorType_FLOAT32, {}}, 8, 1, true);
+  m.PopulateTensor<float>(m.input1(),
+                          {-1.0f, 0.0f, 1.0f, 0.0f, -1.0f, 0.0f, 1.0f, 0.0f});
+
+  m.Invoke();
+
+  std::vector<int> output_shape = m.GetOutputShape();
+  EXPECT_EQ(3, output_shape.size());
+  EXPECT_THAT(output_shape, ElementsAre(1, 1, 5));
+
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear(
+                                 {0.f, 1.f, 4.f, 1.f, 0.f}, 1e-3)));
+}
+
+TEST(SpectrogramOpTest, StrideTest) {
+  BaseAudioSpectrogramOpModel m({TensorType_FLOAT32, {10, 1}},
+                                {TensorType_FLOAT32, {}}, 8, 2, true);
+  m.PopulateTensor<float>(m.input1(), {-1.0f, 0.0f, 1.0f, 0.0f, -1.0f, 0.0f,
+                                       1.0f, 0.0f, 1.0f, 0.0f});
+
+  m.Invoke();
+
+  std::vector<int> output_shape = m.GetOutputShape();
+  EXPECT_THAT(output_shape, ElementsAre(1, 2, 5));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray(ArrayFloatNear(
+                                 {0, 1, 4, 1, 0, 1, 2, 1, 2, 1}, 1e-3)));
+}
+
+}  // namespace
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/internal/spectrogram.cc b/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
index 66ca694dc4..0e481a9d40 100644
--- a/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
+++ b/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
@@ -54,7 +54,6 @@ inline int Log2Floor(uint n) {
       log += shift;
     }
   }
-  assert(value == 1);
   return log;
 }
 
diff --git a/tensorflow/contrib/lite/kernels/mfcc.cc b/tensorflow/contrib/lite/kernels/mfcc.cc
new file mode 100644
index 0000000000..018db0dc54
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/mfcc.cc
@@ -0,0 +1,154 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/contrib/lite/kernels/internal/mfcc.h"
+#include "flatbuffers/flexbuffers.h"
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/mfcc_dct.h"
+#include "tensorflow/contrib/lite/kernels/internal/mfcc_mel_filterbank.h"
+#include "tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+#include "tensorflow/contrib/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+namespace mfcc {
+
+enum KernelType {
+  kReference,
+};
+
+typedef struct {
+  float upper_frequency_limit;
+  float lower_frequency_limit;
+  int filterbank_channel_count;
+  int dct_coefficient_count;
+} TfLiteMfccParams;
+
+constexpr int kInputTensorWav = 0;
+constexpr int kInputTensorRate = 1;
+constexpr int kOutputTensor = 0;
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new TfLiteMfccParams;
+
+  const uint8_t* buffer_t = reinterpret_cast<const uint8_t*>(buffer);
+
+  const flexbuffers::Map& m = flexbuffers::GetRoot(buffer_t, length).AsMap();
+  data->upper_frequency_limit = m["upper_frequency_limit"].AsInt64();
+  data->lower_frequency_limit = m["lower_frequency_limit"].AsInt64();
+  data->filterbank_channel_count = m["filterbank_channel_count"].AsInt64();
+  data->dct_coefficient_count = m["dct_coefficient_count"].AsInt64();
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<TfLiteMfccParams*>(buffer);
+}
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  auto* params = reinterpret_cast<TfLiteMfccParams*>(node->user_data);
+
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  TfLiteTensor* inputWav = GetInput(context, node, kInputTensorWav);
+  TfLiteTensor* inputRate = GetInput(context, node, kInputTensorRate);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  TF_LITE_ENSURE_EQ(context, NumDimensions(inputWav), 3);
+  TF_LITE_ENSURE_EQ(context, NumDimensions(inputRate), 1);
+
+  TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32);
+  TF_LITE_ENSURE_EQ(context, inputWav->type, output->type);
+
+  TfLiteIntArray* output_size = TfLiteIntArrayCreate(3);
+  output_size->data[0] = inputWav->dims->data[0];
+  output_size->data[1] = inputWav->dims->data[1];
+  output_size->data[2] = params->dct_coefficient_count;
+
+  return context->ResizeTensor(context, output, output_size);
+}
+
+// Input is a single squared-magnitude spectrogram frame. The input spectrum
+// is converted to linear magnitude and weighted into bands using a
+// triangular mel filterbank, and a discrete cosine transform (DCT) of the
+// values is taken. Output is populated with the lowest dct_coefficient_count
+// of these values.
+template <KernelType kernel_type>
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  auto* params = reinterpret_cast<TfLiteMfccParams*>(node->user_data);
+
+  TfLiteTensor* inputWav = GetInput(context, node, kInputTensorWav);
+  TfLiteTensor* inputRate = GetInput(context, node, kInputTensorRate);
+  TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  const int32 sample_rate = *GetTensorData<int>(inputRate);
+
+  const int spectrogram_channels = inputWav->dims->data[2];
+  const int spectrogram_samples = inputWav->dims->data[1];
+  const int audio_channels = inputWav->dims->data[0];
+
+  internal::Mfcc mfcc;
+  mfcc.set_upper_frequency_limit(params->upper_frequency_limit);
+  mfcc.set_lower_frequency_limit(params->lower_frequency_limit);
+  mfcc.set_filterbank_channel_count(params->filterbank_channel_count);
+  mfcc.set_dct_coefficient_count(params->dct_coefficient_count);
+
+  mfcc.Initialize(spectrogram_channels, sample_rate);
+
+  const float* spectrogram_flat = GetTensorData<float>(inputWav);
+  float* output_flat = GetTensorData<float>(output);
+
+  for (int audio_channel = 0; audio_channel < audio_channels; ++audio_channel) {
+    for (int spectrogram_sample = 0; spectrogram_sample < spectrogram_samples;
+         ++spectrogram_sample) {
+      const float* sample_data =
+          spectrogram_flat +
+          (audio_channel * spectrogram_samples * spectrogram_channels) +
+          (spectrogram_sample * spectrogram_channels);
+      std::vector<double> mfcc_input(sample_data,
+                                     sample_data + spectrogram_channels);
+      std::vector<double> mfcc_output;
+      mfcc.Compute(mfcc_input, &mfcc_output);
+      TF_LITE_ENSURE_EQ(context, params->dct_coefficient_count,
+                        mfcc_output.size());
+      float* output_data = output_flat +
+                           (audio_channel * spectrogram_samples *
+                            params->dct_coefficient_count) +
+                           (spectrogram_sample * params->dct_coefficient_count);
+      for (int i = 0; i < params->dct_coefficient_count; ++i) {
+        output_data[i] = mfcc_output[i];
+      }
+    }
+  }
+
+  return kTfLiteOk;
+}
+
+}  // namespace mfcc
+
+TfLiteRegistration* Register_MFCC() {
+  static TfLiteRegistration r = {mfcc::Init, mfcc::Free, mfcc::Prepare,
+                                 mfcc::Eval<mfcc::kReference>};
+  return &r;
+}
+
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/mfcc_test.cc b/tensorflow/contrib/lite/kernels/mfcc_test.cc
new file mode 100644
index 0000000000..0291ca8c1c
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/mfcc_test.cc
@@ -0,0 +1,104 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <functional>
+#include <memory>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "flatbuffers/flexbuffers.h"
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace ops {
+namespace custom {
+
+TfLiteRegistration* Register_MFCC();
+
+namespace {
+
+using ::testing::ElementsAre;
+using ::testing::ElementsAreArray;
+
+class BaseMfccOpModel : public SingleOpModel {
+ public:
+  BaseMfccOpModel(const TensorData& input1, const TensorData& input2,
+                  const TensorData& output) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+
+    flexbuffers::Builder fbb;
+    fbb.Map([&]() {
+      fbb.Int("upper_frequency_limit", 4000);
+      fbb.Int("lower_frequency_limit", 20);
+      fbb.Int("filterbank_channel_count", 40);
+      fbb.Int("dct_coefficient_count", 13);
+    });
+    fbb.Finish();
+    SetCustomOp("Mfcc", fbb.GetBuffer(), Register_MFCC);
+
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  int input1() { return input1_; }
+  int input2() { return input2_; }
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+TEST(MfccOpTest, SimpleTest) {
+  BaseMfccOpModel m({TensorType_FLOAT32, {1, 1, 513}}, {TensorType_INT32, {1}},
+                    {TensorType_FLOAT32, {}});
+
+  std::vector<float> data(513);
+  for (int i = 0; i < data.size(); ++i) {
+    data[i] = i + 1;
+  }
+  m.PopulateTensor<float>(m.input1(), 0, data.data(),
+                          data.data() + data.size());
+  m.PopulateTensor<int>(m.input2(), {22050});
+
+  m.Invoke();
+
+  std::vector<int> output_shape = m.GetOutputShape();
+  EXPECT_THAT(output_shape, ElementsAre(1, 1, 13));
+  EXPECT_THAT(
+      m.GetOutput(),
+      ElementsAreArray(ArrayFloatNear(
+          {29.13970072, -6.41568601, -0.61903012, -0.96778652, -0.26819878,
+           -0.40907028, -0.15614748, -0.23203119, -0.10481487, -0.1543029,
+           -0.0769791, -0.10806114, -0.06047613},
+          1e-3)));
+}
+
+}  // namespace
+}  // namespace custom
+}  // namespace ops
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 9537b79a9a..369d3b9886 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -17,6 +17,14 @@ limitations under the License.
 
 namespace tflite {
 namespace ops {
+
+namespace custom {
+
+TfLiteRegistration* Register_AUDIO_SPECTROGRAM();
+TfLiteRegistration* Register_MFCC();
+
+}  // namespace custom
+
 namespace builtin {
 
 TfLiteRegistration* Register_RELU();
@@ -123,6 +131,12 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LOG_SOFTMAX, Register_LOG_SOFTMAX());
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
   AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
+
+  // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
+  // custom ops aren't always included by default.
+  AddCustom("Mfcc", tflite::ops::custom::Register_MFCC());
+  AddCustom("AudioSpectrogram",
+            tflite::ops::custom::Register_AUDIO_SPECTROGRAM());
 }
 
 TfLiteRegistration* BuiltinOpResolver::FindOp(
-- 
GitLab


From e33ef644f42235432027dd7ec2afd74dc3441dd9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 16:03:54 -0700
Subject: [PATCH 038/960] Enable async eager mode in python. Add some
 benchmarks.

PiperOrigin-RevId: 189101670
---
 .../python/examples/resnet50/resnet50_test.py | 127 ++++++++++++------
 tensorflow/contrib/eager/python/tfe.py        |  12 ++
 tensorflow/python/eager/benchmarks_test.py    |  97 +++++++++++--
 tensorflow/python/eager/context.py            | 107 +++++++++++++--
 tensorflow/python/eager/core_test.py          |  50 +++++++
 tensorflow/python/framework/ops.py            |  50 +++++--
 tensorflow/python/framework/ops_test.py       |   3 +
 tensorflow/python/pywrap_tfe.i                |   4 +
 tensorflow/tools/api/golden/tensorflow.pbtxt  |   2 +-
 9 files changed, 371 insertions(+), 81 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py
index 65dcc53aab..d6923293a3 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py
+++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50_test.py
@@ -64,22 +64,29 @@ def train_one_step(model, images, labels, optimizer):
 
 class ResNet50Test(tf.test.TestCase):
 
-  def _apply(self, defun=False):
+  def _apply(self, defun=False, execution_mode=None):
     device, data_format = device_and_data_format()
     model = resnet50.ResNet50(data_format)
     if defun:
       model.call = tfe.defun(model.call)
-    with tf.device(device):
+    with tf.device(device), tfe.execution_mode(execution_mode):
       images, _ = random_batch(2)
       output = model(images, training=False)
+      tfe.async_wait()
     self.assertEqual((2, 1000), output.shape)
 
   def test_apply(self):
     self._apply(defun=False)
 
+  def test_apply_async(self):
+    self._apply(defun=False, execution_mode=tfe.ASYNC)
+
   def test_apply_with_defun(self):
     self._apply(defun=True)
 
+  def test_apply_with_defun_async(self):
+    self._apply(defun=True, execution_mode=tfe.ASYNC)
+
   def test_apply_no_top(self):
     device, data_format = device_and_data_format()
     model = resnet50.ResNet50(data_format, include_top=False)
@@ -98,7 +105,7 @@ class ResNet50Test(tf.test.TestCase):
       output = model(images, training=False)
     self.assertEqual((2, 2048), output.shape)
 
-  def test_train(self):
+  def _test_train(self, execution_mode=None):
     device, data_format = device_and_data_format()
     model = resnet50.ResNet50(data_format)
     tf.train.get_or_create_global_step()
@@ -106,15 +113,22 @@ class ResNet50Test(tf.test.TestCase):
     with tf.contrib.summary.create_file_writer(
         logdir, max_queue=0,
         name='t0').as_default(), tf.contrib.summary.always_record_summaries():
-      with tf.device(device):
+      with tf.device(device), tfe.execution_mode(execution_mode):
         optimizer = tf.train.GradientDescentOptimizer(0.1)
         images, labels = random_batch(2)
         train_one_step(model, images, labels, optimizer)
         self.assertEqual(320, len(model.variables))
+        tfe.async_wait()
     events = summary_test_util.events_from_logdir(logdir)
     self.assertEqual(len(events), 2)
     self.assertEqual(events[1].summary.value[0].tag, 'loss')
 
+  def test_train(self):
+    self._test_train()
+
+  def test_train_async(self):
+    self._test_train(execution_mode=tfe.ASYNC)
+
   def test_no_garbage(self):
     device, data_format = device_and_data_format()
     model = resnet50.ResNet50(data_format)
@@ -183,59 +197,84 @@ class ResNet50Benchmarks(tf.test.Benchmark):
     # a sync. This is a roundabout way, yes.
     tf.constant(1.).cpu()
 
-  def _benchmark_eager_apply(self, label, defun=False):
-    device, data_format = device_and_data_format()
-    model = resnet50.ResNet50(data_format)
-    if defun:
-      model.call = tfe.defun(model.call)
-    batch_size = 64
-    num_burn = 5
-    num_iters = 30
-    with tf.device(device):
-      images, _ = random_batch(batch_size)
-      for _ in xrange(num_burn):
-        model(images, training=False).cpu()
-      gc.collect()
-      start = time.time()
-      for _ in xrange(num_iters):
-        model(images, training=False).cpu()
-      self._report(label, start, num_iters, device, batch_size, data_format)
-
-  def benchmark_eager_apply(self):
-    self._benchmark_eager_apply('eager_apply', defun=False)
-
-  def benchmark_eager_apply_with_defun(self):
-    self._benchmark_eager_apply('eager_apply_with_defun', defun=True)
-
-  def _benchmark_eager_train(self, label, make_iterator, defun=False):
-    device, data_format = device_and_data_format()
-    for batch_size in self._train_batch_sizes():
-      (images, labels) = random_batch(batch_size)
-      num_burn = 3
-      num_iters = 10
+  def _benchmark_eager_apply(self, label, defun=False, execution_mode=None):
+    with tfe.execution_mode(execution_mode):
+      device, data_format = device_and_data_format()
       model = resnet50.ResNet50(data_format)
       if defun:
         model.call = tfe.defun(model.call)
-      optimizer = tf.train.GradientDescentOptimizer(0.1)
-
+      batch_size = 64
+      num_burn = 5
+      num_iters = 30
       with tf.device(device):
-        iterator = make_iterator((images, labels))
+        images, _ = random_batch(batch_size)
         for _ in xrange(num_burn):
-          (images, labels) = iterator.next()
-          train_one_step(model, images, labels, optimizer)
-        self._force_gpu_sync()
+          model(images, training=False).cpu()
+        if execution_mode:
+          tfe.async_wait()
         gc.collect()
-
         start = time.time()
         for _ in xrange(num_iters):
-          (images, labels) = iterator.next()
-          train_one_step(model, images, labels, optimizer)
-        self._force_gpu_sync()
+          model(images, training=False).cpu()
+        if execution_mode:
+          tfe.async_wait()
         self._report(label, start, num_iters, device, batch_size, data_format)
 
+  def benchmark_eager_apply(self):
+    self._benchmark_eager_apply('eager_apply', defun=False)
+
+  def benchmark_eager_apply_async(self):
+    self._benchmark_eager_apply(
+        'eager_apply_async', defun=False, execution_mode=tfe.ASYNC)
+
+  def benchmark_eager_apply_with_defun(self):
+    self._benchmark_eager_apply('eager_apply_with_defun', defun=True)
+
+  def _benchmark_eager_train(self,
+                             label,
+                             make_iterator,
+                             defun=False,
+                             execution_mode=None):
+    with tfe.execution_mode(execution_mode):
+      device, data_format = device_and_data_format()
+      for batch_size in self._train_batch_sizes():
+        (images, labels) = random_batch(batch_size)
+        num_burn = 3
+        num_iters = 10
+        model = resnet50.ResNet50(data_format)
+        if defun:
+          model.call = tfe.defun(model.call)
+        optimizer = tf.train.GradientDescentOptimizer(0.1)
+
+        with tf.device(device):
+          iterator = make_iterator((images, labels))
+          for _ in xrange(num_burn):
+            (images, labels) = iterator.next()
+            train_one_step(model, images, labels, optimizer)
+          if execution_mode:
+            tfe.async_wait()
+          self._force_gpu_sync()
+          gc.collect()
+
+          start = time.time()
+          for _ in xrange(num_iters):
+            (images, labels) = iterator.next()
+            train_one_step(model, images, labels, optimizer)
+          if execution_mode:
+            tfe.async_wait()
+          self._force_gpu_sync()
+          self._report(label, start, num_iters, device, batch_size, data_format)
+
   def benchmark_eager_train(self):
     self._benchmark_eager_train('eager_train', MockIterator, defun=False)
 
+  def benchmark_eager_train_async(self):
+    self._benchmark_eager_train(
+        'eager_train_async',
+        MockIterator,
+        defun=False,
+        execution_mode=tfe.ASYNC)
+
   def benchmark_eager_train_with_defun(self):
     self._benchmark_eager_train(
         'eager_train_with_defun', MockIterator, defun=True)
diff --git a/tensorflow/contrib/eager/python/tfe.py b/tensorflow/contrib/eager/python/tfe.py
index 5aabc9aae8..c6f3f20e78 100644
--- a/tensorflow/contrib/eager/python/tfe.py
+++ b/tensorflow/contrib/eager/python/tfe.py
@@ -62,12 +62,18 @@ To use, at program startup, call `tfe.enable_eager_execution()`.
 
 @@executing_eagerly
 @@in_eager_mode
+@@set_execution_mode
+@@execution_mode
+@@async_wait
+@@async_clear_error
 
 @@run_test_in_graph_and_eager_modes
 
 @@DEVICE_PLACEMENT_EXPLICIT
 @@DEVICE_PLACEMENT_WARN
 @@DEVICE_PLACEMENT_SILENT
+@@SYNC
+@@ASYNC
 """
 
 from __future__ import absolute_import
@@ -95,6 +101,12 @@ from tensorflow.python.eager.context import DEVICE_PLACEMENT_WARN
 from tensorflow.python.eager.context import DEVICE_PLACEMENT_SILENT
 from tensorflow.python.eager.context import executing_eagerly
 from tensorflow.python.eager.context import list_devices
+from tensorflow.python.eager.context import set_execution_mode
+from tensorflow.python.eager.context import execution_mode
+from tensorflow.python.eager.context import async_wait
+from tensorflow.python.eager.context import async_clear_error
+from tensorflow.python.eager.context import SYNC
+from tensorflow.python.eager.context import ASYNC
 from tensorflow.python.eager.context import num_gpus
 from tensorflow.python.eager.execution_callbacks import add_execution_callback
 from tensorflow.python.eager.execution_callbacks import clear_execution_callbacks
diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 8c6e7e5758..9ca5041c38 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -82,16 +82,24 @@ class MicroBenchmarks(test.Benchmark):
     self._num_iters_2_by_2 = 30000
     self._num_iters_100_by_784 = 1000
 
-  def _run(self, func, num_iters):
+  def _run(self, func, num_iters, execution_mode=None):
     # call func to maybe warm up the GPU
-    func()
-    start = time.time()
-    for _ in xrange(num_iters):
+    ctx = context.context()
+    with ctx.execution_mode(execution_mode):
       func()
-    end = time.time()
-    mean_us = (end - start) * 1e6 / num_iters
-    self.report_benchmark(iters=num_iters, wall_time=mean_us,
-                          extras={"examples_per_sec": num_iters/(end-start)})
+      if execution_mode == context.ASYNC:
+        ctx.async_wait()
+      start = time.time()
+      for _ in xrange(num_iters):
+        func()
+      if execution_mode == context.ASYNC:
+        ctx.async_wait()
+      end = time.time()
+      mean_us = (end - start) * 1e6 / num_iters
+      self.report_benchmark(
+          iters=num_iters,
+          wall_time=mean_us,
+          extras={"examples_per_sec": num_iters / (end - start)})
 
   def benchmark_create_np_array(self):
     func = lambda: np.array([3.0])
@@ -236,9 +244,10 @@ class MicroBenchmarks(test.Benchmark):
     func = lambda: np.dot(a, b)
     self._run(func, num_iters)
 
-  def _benchmark_tf_matmul(self, m, transpose_b, num_iters):
+  def _benchmark_tf_matmul(self, m, transpose_b, num_iters,
+                           execution_mode=None):
     func = lambda: math_ops.matmul(m, m, transpose_b=transpose_b)
-    self._run(func, num_iters)
+    self._run(func, num_iters, execution_mode=execution_mode)
 
   def _benchmark_gen_math_ops_matmul(self, m, transpose_b, num_iters):
     def func():
@@ -267,10 +276,14 @@ class MicroBenchmarks(test.Benchmark):
 
     self._run(func, num_iters)
 
-  def _benchmark_defun_matmul(self, m, transpose_b, num_iters):
+  def _benchmark_defun_matmul(self,
+                              m,
+                              transpose_b,
+                              num_iters,
+                              execution_mode=None):
     f = function.defun(math_ops.matmul)
     func = lambda: f(m, m, transpose_b)
-    self._run(func, num_iters)
+    self._run(func, num_iters, execution_mode=execution_mode)
 
   def _benchmark_read_variable(self, m, num_iters):
     self._run(m.value, num_iters)
@@ -301,6 +314,15 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_tf_matmul(
           m, transpose_b=False, num_iters=self._num_iters_2_by_2)
 
+  def benchmark_tf_matmul_2_by_2_CPU_async(self):
+    with context.device(CPU):
+      m = self._m_2_by_2.cpu()
+      self._benchmark_tf_matmul(
+          m,
+          transpose_b=False,
+          num_iters=self._num_iters_2_by_2,
+          execution_mode=context.ASYNC)
+
   def benchmark_gen_math_ops_matmul_2_by_2_CPU(self):
     with context.device(CPU):
       m = self._m_2_by_2.cpu()
@@ -325,6 +347,15 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_defun_matmul(
           m, transpose_b=False, num_iters=self._num_iters_2_by_2)
 
+  def benchmark_defun_matmul_2_by_2_CPU_async(self):
+    with context.device(CPU):
+      m = self._m_2_by_2.cpu()
+      self._benchmark_defun_matmul(
+          m,
+          transpose_b=False,
+          num_iters=self._num_iters_2_by_2,
+          execution_mode=context.ASYNC)
+
   def benchmark_tf_matmul_2_by_2_GPU(self):
     if not context.num_gpus():
       return
@@ -333,6 +364,17 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_tf_matmul(
           m, transpose_b=False, num_iters=self._num_iters_2_by_2)
 
+  def benchmark_tf_matmul_2_by_2_GPU_async(self):
+    if not context.num_gpus():
+      return
+    with context.device(GPU):
+      m = self._m_2_by_2.gpu()
+      self._benchmark_tf_matmul(
+          m,
+          transpose_b=False,
+          num_iters=self._num_iters_2_by_2,
+          execution_mode=context.ASYNC)
+
   def benchmark_gen_math_ops_matmul_2_by_2_GPU(self):
     if not context.num_gpus():
       return
@@ -357,6 +399,17 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_defun_matmul(
           m, transpose_b=False, num_iters=self._num_iters_2_by_2)
 
+  def benchmark_defun_matmul_2_by_2_GPU_async(self):
+    if not context.num_gpus():
+      return
+    with context.device(GPU):
+      m = self._m_2_by_2.gpu()
+      self._benchmark_defun_matmul(
+          m,
+          transpose_b=False,
+          num_iters=self._num_iters_2_by_2,
+          execution_mode=context.ASYNC)
+
   # Benchmarks for AA.T, A of dimension 100 by 784.
   def benchmark_np_matmul_100_by_784(self):
     self._benchmark_np_matmul(
@@ -370,6 +423,15 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_tf_matmul(
           m, transpose_b=True, num_iters=self._num_iters_100_by_784)
 
+  def benchmark_tf_matmul_100_by_784_CPU_async(self):
+    with context.device(CPU):
+      m = self._m_100_by_784.cpu()
+      self._benchmark_tf_matmul(
+          m,
+          transpose_b=True,
+          num_iters=self._num_iters_100_by_784,
+          execution_mode=context.ASYNC)
+
   def benchmark_gen_math_ops_matmul_100_by_784_CPU(self):
     with context.device(CPU):
       m = self._m_100_by_784.cpu()
@@ -402,6 +464,17 @@ class MicroBenchmarks(test.Benchmark):
       self._benchmark_tf_matmul(
           m, transpose_b=True, num_iters=self._num_iters_100_by_784)
 
+  def benchmark_tf_matmul_100_by_784_GPU_async(self):
+    if not context.num_gpus():
+      return
+    with context.device(GPU):
+      m = self._m_100_by_784.gpu()
+      self._benchmark_tf_matmul(
+          m,
+          transpose_b=True,
+          num_iters=self._num_iters_100_by_784,
+          execution_mode=context.ASYNC)
+
   def benchmark_gen_math_ops_matmul_100_by_784_GPU(self):
     if not context.num_gpus():
       return
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 87d3ed880a..7953d10a89 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -53,6 +53,8 @@ DEVICE_PLACEMENT_WARN = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_WARN
 DEVICE_PLACEMENT_SILENT = pywrap_tensorflow.TFE_DEVICE_PLACEMENT_SILENT
 DEVICE_PLACEMENT_SILENT_FOR_INT32 = (
     pywrap_tensorflow.TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32)
+SYNC = 0
+ASYNC = 1
 
 
 class _TensorCache(object):
@@ -89,6 +91,7 @@ class _EagerContext(threading.local):
     self.summary_writer_resource = None
     self.scalar_cache = {}
     self.ones_rank_cache = _TensorCache()
+    self.execution_mode = None
 
 
 ContextStackEntry = collections.namedtuple(
@@ -131,24 +134,41 @@ context_stack = ContextStack()
 class Context(object):
   """Environment in which eager operations execute."""
 
-  def __init__(self, config=None, device_policy=None):
+  # TODO(agarwal): create and link in some documentation for `execution_mode`.
+  # pylint: disable=redefined-outer-name
+  def __init__(self, config=None, device_policy=None, execution_mode=None):
     """Creates a new Context.
 
     Args:
       config: (Optional.) A `ConfigProto` protocol buffer with configuration
-       options for the Context. Note that a lot of these options may be
-       currently unimplemented or irrelevant when eager execution is enabled.
+        options for the Context. Note that a lot of these options may be
+        currently unimplemented or irrelevant when eager execution is enabled.
       device_policy: (Optional.) What policy to use when trying to run an
-       operation on a device with inputs which are not on that device.
-       Valid values:
-         tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is not
-           correct.
-         tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the
+         operation on a device with inputs which are not on that device.
+         When set to None, an appropriate value will be picked automatically.
+         The value picked may change between TensorFlow releases.
+
+         Defaults to tf.contrib.eager.DEVICE_PLACEMENT_SILENT_FOR_INT32.
+         Valid values:
+         - tfe.DEVICE_PLACEMENT_EXPLICIT: raises an error if the placement is
+           not correct.
+         - tfe.DEVICE_PLACEMENT_WARN: copies the tensors which are not on the
            right device but raises a warning.
-         tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might
+         - tfe.DEVICE_PLACEMENT_SILENT: silently copies the tensors. This might
            hide performance problems.
-         tfe.DEVICE_PLACEMENT_SILENT_FOR_INT32: silently copies int32 tensors,
+         - tfe.DEVICE_PLACEMENT_SILENT_FOR_INT32: silently copies int32 tensors,
            raising errors on the other ones.
+      execution_mode: (Optional.) Policy controlling how operations dispatched
+        are actually executed. When set to None, an appropriate value will be
+        picked automatically. The value picked may change between TensorFlow
+        releases.
+        Valid values:
+        - tf.contrib.eager.SYNC: executes each operation synchronously.
+        - tf.contrib.eager.ASYNC: executes each operation asynchronously. These
+          operations may return "non-ready" handles.
+
+    Raises:
+     ValueError: If execution_mode is not valid.
     """
     self._eager_context = _EagerContext()
     self._context_handle = None
@@ -158,6 +178,14 @@ class Context(object):
     self._seed = None
     self._initialize_lock = threading.Lock()
     self._device_policy = device_policy
+    if execution_mode not in (None, SYNC, ASYNC):
+      raise ValueError(
+          "execution_mode should be None/SYNC/ASYNC. Got %s" % execution_mode)
+    if execution_mode is None:
+      execution_mode = SYNC
+    self._execution_mode = execution_mode
+
+  # pylint: enable=redefined-outer-name
 
   def _set_global_seed(self, seed):
     """Set a global eager mode seed for random ops."""
@@ -195,6 +223,8 @@ class Context(object):
           if self._device_policy is not None:
             pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy(
                 opts, self._device_policy)
+          if self._execution_mode == ASYNC:
+            pywrap_tensorflow.TFE_ContextOptionsSetAsync(True)
           self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
       finally:
         pywrap_tensorflow.TFE_DeleteContextOptions(opts)
@@ -356,6 +386,43 @@ class Context(object):
     """List of the names of devices available to execute operations."""
     return self._devices
 
+  def get_execution_mode(self):
+    mode = self._eager_context.execution_mode
+    if mode is None:
+      mode = self._execution_mode
+    return mode
+
+  def set_execution_mode(self, mode):
+    """Sets execution mode for current thread."""
+    if mode not in (None, SYNC, ASYNC):
+      raise ValueError(
+          "Execution mode should be None/SYNC/ASYNC. Got %s" % mode)
+    if mode is None:
+      mode = SYNC
+    self._eager_context.execution_mode = mode
+    with errors.raise_exception_on_not_ok_status() as status:
+      pywrap_tensorflow.TFE_ContextSetAsyncForThread(self._handle,
+                                                     mode == ASYNC, status)
+
+  @tf_contextlib.contextmanager
+  def execution_mode(self, mode):
+    """Context manager for setting execution mode for current thread."""
+    old_mode = self.get_execution_mode()
+    try:
+      self.set_execution_mode(mode)
+      yield
+    finally:
+      self.set_execution_mode(old_mode)
+
+  def async_wait(self):
+    """Waits for ops dispatched in ASYNC mode to finish."""
+    with errors.raise_exception_on_not_ok_status() as status:
+      pywrap_tensorflow.TFE_ContextAsyncWait(self._handle, status)
+
+  def async_clear_error(self):
+    """Clears errors raised during ASYNC execution."""
+    pywrap_tensorflow.TFE_ContextAsyncClearError(self._handle)
+
   def num_gpus(self):
     """The number of GPUs available to execute operations."""
     self._initialize_handle_and_devices()
@@ -595,6 +662,26 @@ def list_devices():
   return context().devices()
 
 
+def set_execution_mode(mode):
+  """Sets execution mode for the current thread."""
+  context().set_execution_mode(mode)
+
+
+def execution_mode(mode):
+  """Context manager for setting execution mode for current thread."""
+  return context().execution_mode(mode)
+
+
+def async_wait():
+  """Waits for ops dispatched in ASYNC mode to finish."""
+  return context().async_wait()
+
+
+def async_clear_error():
+  """Clears errors raised during ASYNC execution mode."""
+  return context().async_clear_error()
+
+
 def num_gpus():
   """Get the number of available GPU devices.
 
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 61c5526d48..6dfd8d1afa 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -63,6 +63,16 @@ class TFETest(test_util.TensorFlowTestCase):
     ctx.scope_name = 'foo'
     self.assertEqual('foo', ctx.scope_name)
 
+    self.assertEqual(context.SYNC, ctx.get_execution_mode())
+    ctx.set_execution_mode(context.ASYNC)
+    self.assertEqual(context.ASYNC, ctx.get_execution_mode())
+    ctx.set_execution_mode(context.SYNC)
+    self.assertEqual(context.SYNC, ctx.get_execution_mode())
+    with ctx.execution_mode(context.ASYNC):
+      self.assertEqual(context.ASYNC, ctx.get_execution_mode())
+    ctx.set_execution_mode(context.SYNC)
+    self.assertEqual(context.SYNC, ctx.get_execution_mode())
+
     self.assertIsNone(ctx.summary_writer_resource)
     ctx.summary_writer_resource = 'mock'
     self.assertEqual('mock', ctx.summary_writer_resource)
@@ -208,6 +218,23 @@ class TFETest(test_util.TensorFlowTestCase):
     with self.assertRaises(RuntimeError):
       x.gpu(context.context().num_gpus() + 1)
 
+  def testCopyBetweenDevicesAsync(self):
+    if not context.context().num_gpus():
+      self.skipTest('No GPUs found')
+    with context.execution_mode(context.ASYNC):
+      x = constant_op.constant([[1., 2.], [3., 4.]])
+      x = x.cpu()
+      x = x.gpu()
+      x = x.gpu()
+      x = x.cpu()
+      context.async_wait()
+
+    # Invalid device
+    with self.assertRaises(RuntimeError):
+      x.gpu(context.context().num_gpus() + 1)
+      context.async_wait()
+    context.async_clear_error()
+
   def testCopyScope(self):
     if not context.context().num_gpus():
       self.skipTest('No GPUs found')
@@ -248,6 +275,29 @@ class TFETest(test_util.TensorFlowTestCase):
         attrs=('T', three.dtype.as_datatype_enum))[0]
     self.assertAllEqual(15, product)
 
+  def testExecuteBasicAsync(self):
+    with context.execution_mode(context.ASYNC):
+      three = constant_op.constant(3)
+      five = constant_op.constant(5)
+      product = execute(
+          b'Mul',
+          num_outputs=1,
+          inputs=[three, five],
+          attrs=('T', three.dtype.as_datatype_enum))[0]
+      self.assertAllEqual(15, product)
+    # Error: Invalid arguments
+    context.set_execution_mode(context.ASYNC)
+    with self.assertRaises(errors.InvalidArgumentError):
+      execute(
+          b'MatMul',
+          num_outputs=1,
+          inputs=[three, five],
+          attrs=('transpose_a', False, 'transpose_b', False, 'T',
+                 three.dtype.as_datatype_enum))
+      context.async_wait()
+    context.async_clear_error()
+    context.set_execution_mode(context.SYNC)
+
   def testExecuteTooManyNumOutputs(self):
     # num_outputs provided is 50, but only one output is produced.
     product = execute(
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 44df22cf58..b2f43773fe 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5184,7 +5184,8 @@ def init_scope():
 
 
 @tf_export("enable_eager_execution")
-def enable_eager_execution(config=None, device_policy=None):
+def enable_eager_execution(config=None, device_policy=None,
+                           execution_mode=None):
   """Enables eager execution for the lifetime of this program.
 
   Eager execution provides an imperative interface to TensorFlow. With eager
@@ -5210,13 +5211,15 @@ def enable_eager_execution(config=None, device_policy=None):
 
   Args:
     config: (Optional.) A @{tf.ConfigProto} to use to configure the environment
-     in which operations are executed. Note that @{tf.ConfigProto} is also
-     used to configure graph execution (via @{tf.Session}) and many options
-     within `tf.ConfigProto` are not implemented (or are irrelevant) when
+      in which operations are executed. Note that @{tf.ConfigProto} is also
+      used to configure graph execution (via @{tf.Session}) and many options
+      within `tf.ConfigProto` are not implemented (or are irrelevant) when
      eager execution is enabled.
     device_policy: (Optional.) Policy controlling how operations requiring
      inputs on a specific device (e.g., a GPU 0) handle inputs on a different
-     device  (e.g. GPU 1 or CPU).
+     device  (e.g. GPU 1 or CPU). When set to None, an appropriate value will be
+     picked automatically. The value picked may change between TensorFlow
+     releases.
      Valid values:
 
       - tf.contrib.eager.DEVICE_PLACEMENT_EXPLICIT: raises an error if the
@@ -5232,6 +5235,15 @@ def enable_eager_execution(config=None, device_policy=None):
 
       - tf.contrib.eager.DEVICE_PLACEMENT_SILENT_FOR_INT32: silently copies
         int32 tensors, raising errors on the other ones.
+    execution_mode: (Optional.) Policy controlling how operations dispatched are
+      actually executed. When set to None, an appropriate value will be picked
+      automatically. The value picked may change between TensorFlow releases.
+      Valid values:
+
+        - tf.contrib.eager.SYNC: executes each operation synchronously.
+
+        - tf.contrib.eager.ASYNC: executes each operation asynchronously. These
+          operations may return "non-ready" handles.
 
   Raises:
     ValueError: If eager execution is enabled after creating/executing a
@@ -5248,6 +5260,10 @@ def enable_eager_execution(config=None, device_policy=None):
     raise ValueError(
         "device_policy must be one of None, tf.contrib.eager.DEVICE_PLACEMENT_*"
     )
+  if execution_mode not in (None, context.SYNC, context.ASYNC):
+    raise ValueError(
+        "execution_mode must be one of None, tf.contrib.eager.SYNC, "
+        "tf.contrib.eager.ASYNC")
   # pylint: disable=protected-access
   if context._default_mode == context.GRAPH_MODE:
     graph_mode_has_been_used = (
@@ -5258,8 +5274,10 @@ def enable_eager_execution(config=None, device_policy=None):
           "tf.enable_eager_execution must be called at program startup.")
   context._default_mode = context.EAGER_MODE
   if context._context is None:
-    context._context = context.Context(config=config,
-                                       device_policy=device_policy)
+    context._context = context.Context(
+        config=config,
+        device_policy=device_policy,
+        execution_mode=execution_mode)
     if context.context_stack.stack:
       raise AssertionError("Invariant violated: The context stack must "
                            "be empty when eager execution is enabled.")
@@ -5267,15 +5285,19 @@ def enable_eager_execution(config=None, device_policy=None):
     # context stack; this entry won't ever be popped, as it's impossible to
     # disable eager execution
     context.context_stack.push(False, context.eager_mode)
-  elif ((config is not None and config is not context._context._config)
-        or (device_policy is not None
-            and device_policy is not context._context._device_policy)):
+  elif ((config is not None and config is not context._context._config) or
+        (device_policy is not None and
+         device_policy is not context._context._device_policy) or
+        (execution_mode is not None and
+         execution_mode is not context._context._execution_mode)):
     raise ValueError("Trying to change the options of an active eager"
                      " execution. Context config: %s, specified config:"
-                     " %s. Context device policy: %s; specified device"
-                     " policy: %s." % (config, context._context._config,
-                                       device_policy,
-                                       context._context._device_policy))
+                     " %s. Context device policy: %s, specified device"
+                     " policy: %s. Context execution mode: %s, "
+                     " specified execution mode %s." %
+                     (context._context._config, config,
+                      context._context._device_policy, device_policy,
+                      context._context._execution_mode, execution_mode))
   else:
     raise ValueError(
         "tf.enable_eager_execution must be called at program startup.")
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index 6daab80408..d96e0708f8 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -2919,6 +2919,9 @@ class EnableEagerExecutionTest(test_util.TensorFlowTestCase):
     with self.assertRaisesRegexp(ValueError, "device_policy must be one of"):
       c = config_pb2.ConfigProto()
       ops.enable_eager_execution(c, c)
+    with self.assertRaisesRegexp(ValueError, "execution_mode must be one of"):
+      c = config_pb2.ConfigProto()
+      ops.enable_eager_execution(c, execution_mode=c)
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index b481ddf5d4..39fabb9c1b 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -26,6 +26,9 @@ limitations under the License.
 %rename("%s") TFE_ContextClearCaches;
 %rename("%s") TFE_ContextGetDevicePlacementPolicy;
 %rename("%s") TFE_ContextSetThreadLocalDevicePlacementPolicy;
+%rename("%s") TFE_ContextSetAsyncForThread;
+%rename("%s") TFE_ContextAsyncWait;
+%rename("%s") TFE_ContextAsyncClearError;
 %rename("%s") TFE_OpNameGetAttrType;
 %rename("%s") TFE_Py_InitEagerTensor;
 %rename("%s") TFE_Py_RegisterExceptionClass;
@@ -51,6 +54,7 @@ limitations under the License.
 %rename("%s") TFE_NewContextOptions;
 %rename("%s") TFE_ContextOptionsSetConfig;
 %rename("%s") TFE_ContextOptionsSetDevicePlacementPolicy;
+%rename("%s") TFE_ContextOptionsSetAsync;
 %rename("%s") TFE_DeleteContextOptions;
 %rename("%s") TFE_Py_TensorShapeSlice;
 
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 80735cea5d..99e09c3759 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -974,7 +974,7 @@ tf_module {
   }
   member_method {
     name: "enable_eager_execution"
-    argspec: "args=[\'config\', \'device_policy\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'config\', \'device_policy\', \'execution_mode\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "encode_base64"
-- 
GitLab


From d7efbfa0c68be25c05de99c1eb099fa9ba3dddb4 Mon Sep 17 00:00:00 2001
From: Yifei Feng <yifeif@google.com>
Date: Wed, 14 Mar 2018 16:20:13 -0700
Subject: [PATCH 039/960] Automated g4 rollback of changelist 189089672

PiperOrigin-RevId: 189104013
---
 tensorflow/contrib/image/BUILD                | 109 -------
 tensorflow/contrib/image/__init__.py          |   7 -
 .../kernel_tests/dense_image_warp_test.py     | 264 ----------------
 .../kernel_tests/interpolate_spline_test.py   | 261 ----------------
 .../kernel_tests/sparse_image_warp_test.py    | 251 ---------------
 .../test_data/Yellow_Smiley_Face.png          | Bin 14060 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-0.png | Bin 18537 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-1.png | Bin 19086 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-4.png | Bin 18884 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-0.png | Bin 18109 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-1.png | Bin 19251 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-4.png | Bin 19132 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-0.png | Bin 17500 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-1.png | Bin 18058 -> 0 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-4.png | Bin 19313 -> 0 bytes
 .../image/python/ops/dense_image_warp.py      | 196 ------------
 .../image/python/ops/interpolate_spline.py    | 285 ------------------
 .../image/python/ops/sparse_image_warp.py     | 192 ------------
 18 files changed, 1565 deletions(-)
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
 delete mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
 delete mode 100644 tensorflow/contrib/image/python/ops/dense_image_warp.py
 delete mode 100644 tensorflow/contrib/image/python/ops/interpolate_spline.py
 delete mode 100644 tensorflow/contrib/image/python/ops/sparse_image_warp.py

diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 760ed70fbb..3ff02e085e 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -78,10 +78,7 @@ tf_custom_op_py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        ":dense_image_warp_py",
         ":image_ops",
-        ":interpolate_spline_py",
-        ":sparse_image_warp_py",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:common_shapes",
@@ -197,112 +194,6 @@ cuda_py_test(
     ],
 )
 
-py_library(
-    name = "dense_image_warp_py",
-    srcs = [
-        "python/ops/dense_image_warp.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:util",
-        "//third_party/py/numpy",
-    ],
-)
-
-py_library(
-    name = "interpolate_spline_py",
-    srcs = [
-        "python/ops/interpolate_spline.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:util",
-    ],
-)
-
-py_library(
-    name = "sparse_image_warp_py",
-    srcs = [
-        "python/ops/sparse_image_warp.py",
-    ],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":dense_image_warp_py",
-        ":interpolate_spline_py",
-        "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:platform",
-        "//tensorflow/python:util",
-    ],
-)
-
-cuda_py_test(
-    name = "sparse_image_warp_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/sparse_image_warp_test.py"],
-    additional_deps = [
-        ":sparse_image_warp_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:clip_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:image_ops",
-        "//tensorflow/python:variables",
-        "//tensorflow/core:protos_all_py",
-    ],
-    data = glob(["python/kernel_tests/test_data/*.png"]),
-)
-
-cuda_py_test(
-    name = "dense_image_warp_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/dense_image_warp_test.py"],
-    additional_deps = [
-        ":dense_image_warp_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:clip_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:image_ops",
-        "//tensorflow/python:variables",
-        "//tensorflow/core:protos_all_py",
-    ],
-)
-
-cuda_py_test(
-    name = "interpolate_spline_test",
-    size = "medium",
-    srcs = ["python/kernel_tests/interpolate_spline_test.py"],
-    additional_deps = [
-        ":interpolate_spline_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:client",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:clip_ops",
-        "//tensorflow/python:io_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:image_ops",
-        "//tensorflow/python:variables",
-        "//tensorflow/core:protos_all_py",
-        "//third_party/py/scipy",
-    ],
-)
-
 tf_py_test(
     name = "segmentation_test",
     size = "medium",
diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index e982030bc8..cc8ed117ba 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -30,9 +30,6 @@ projective transforms (including rotation) are supported.
 @@transform
 @@translate
 @@translations_to_projective_transforms
-@@dense_image_warp
-@@interpolate_spline
-@@sparse_image_warp
 
 ## Image Segmentation `Ops`
 
@@ -50,8 +47,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.image.python.ops.dense_image_warp import dense_image_warp
-
 from tensorflow.contrib.image.python.ops.distort_image_ops import adjust_hsv_in_yiq
 from tensorflow.contrib.image.python.ops.distort_image_ops import random_hsv_in_yiq
 
@@ -62,9 +57,7 @@ from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
 from tensorflow.contrib.image.python.ops.image_ops import translate
 from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
-from tensorflow.contrib.image.python.ops.interpolate_spline import interpolate_spline
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
-from tensorflow.contrib.image.python.ops.sparse_image_warp import sparse_image_warp
 
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
deleted file mode 100644
index 24d99ccaa6..0000000000
--- a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
+++ /dev/null
@@ -1,264 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for dense_image_warp."""
-
-import math
-import numpy as np
-
-from tensorflow.contrib.image.python.ops import dense_image_warp
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gradients
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import googletest
-
-from tensorflow.python.training import adam
-
-
-class DenseImageWarpTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    np.random.seed(0)
-
-  def test_interpolate_small_grid_ij(self):
-    grid = constant_op.constant(
-        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
-    query_points = constant_op.constant(
-        [[0., 0.], [1., 0.], [2., 0.5], [1.5, 1.5]], shape=[1, 4, 2])
-    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
-
-    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
-
-    with self.test_session() as sess:
-      predicted = sess.run(interp)
-      self.assertAllClose(expected_results, predicted)
-
-  def test_interpolate_small_grid_xy(self):
-    grid = constant_op.constant(
-        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
-    query_points = constant_op.constant(
-        [[0., 0.], [0., 1.], [0.5, 2.0], [1.5, 1.5]], shape=[1, 4, 2])
-    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
-
-    interp = dense_image_warp._interpolate_bilinear(
-        grid, query_points, indexing='xy')
-
-    with self.test_session() as sess:
-      predicted = sess.run(interp)
-      self.assertAllClose(expected_results, predicted)
-
-  def test_interpolate_small_grid_batched(self):
-    grid = constant_op.constant(
-        [[[0., 1.], [3., 4.]], [[5., 6.], [7., 8.]]], shape=[2, 2, 2, 1])
-    query_points = constant_op.constant([[[0., 0.], [1., 0.], [0.5, 0.5]],
-                                         [[0.5, 0.], [1., 0.], [1., 1.]]])
-    expected_results = np.reshape(
-        np.array([[0., 3., 2.], [6., 7., 8.]]), [2, 3, 1])
-
-    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
-
-    with self.test_session() as sess:
-      predicted = sess.run(interp)
-      self.assertAllClose(expected_results, predicted)
-
-  def get_image_and_flow_placeholders(self, shape, image_type, flow_type):
-    batch_size, height, width, numchannels = shape
-    image_shape = [batch_size, height, width, numchannels]
-    flow_shape = [batch_size, height, width, 2]
-
-    tf_type = {
-        'float16': dtypes.half,
-        'float32': dtypes.float32,
-        'float64': dtypes.float64
-    }
-
-    image = array_ops.placeholder(dtype=tf_type[image_type], shape=image_shape)
-
-    flows = array_ops.placeholder(dtype=tf_type[flow_type], shape=flow_shape)
-    return image, flows
-
-  def get_random_image_and_flows(self, shape, image_type, flow_type):
-    batch_size, height, width, numchannels = shape
-    image_shape = [batch_size, height, width, numchannels]
-    image = np.random.normal(size=image_shape)
-    flow_shape = [batch_size, height, width, 2]
-    flows = np.random.normal(size=flow_shape) * 3
-    return image.astype(image_type), flows.astype(flow_type)
-
-  def assert_correct_interpolation_value(self,
-                                         image,
-                                         flows,
-                                         pred_interpolation,
-                                         batch_index,
-                                         y_index,
-                                         x_index,
-                                         low_precision=False):
-    """Assert that the tf interpolation matches hand-computed value."""
-
-    height = image.shape[1]
-    width = image.shape[2]
-    displacement = flows[batch_index, y_index, x_index, :]
-    float_y = y_index - displacement[0]
-    float_x = x_index - displacement[1]
-    floor_y = max(min(height - 2, math.floor(float_y)), 0)
-    floor_x = max(min(width - 2, math.floor(float_x)), 0)
-    ceil_y = floor_y + 1
-    ceil_x = floor_x + 1
-
-    alpha_y = min(max(0.0, float_y - floor_y), 1.0)
-    alpha_x = min(max(0.0, float_x - floor_x), 1.0)
-
-    floor_y = int(floor_y)
-    floor_x = int(floor_x)
-    ceil_y = int(ceil_y)
-    ceil_x = int(ceil_x)
-
-    top_left = image[batch_index, floor_y, floor_x, :]
-    top_right = image[batch_index, floor_y, ceil_x, :]
-    bottom_left = image[batch_index, ceil_y, floor_x, :]
-    bottom_right = image[batch_index, ceil_y, ceil_x, :]
-
-    interp_top = alpha_x * (top_right - top_left) + top_left
-    interp_bottom = alpha_x * (bottom_right - bottom_left) + bottom_left
-    interp = alpha_y * (interp_bottom - interp_top) + interp_top
-    atol = 1e-6
-    rtol = 1e-6
-    if low_precision:
-      atol = 1e-2
-      rtol = 1e-3
-    self.assertAllClose(
-        interp,
-        pred_interpolation[batch_index, y_index, x_index, :],
-        atol=atol,
-        rtol=rtol)
-
-  def check_zero_flow_correctness(self, shape, image_type, flow_type):
-    """Assert using zero flows doesn't change the input image."""
-
-    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
-                                                        flow_type)
-    interp = dense_image_warp.dense_image_warp(image, flows)
-
-    with self.test_session() as sess:
-      rand_image, rand_flows = self.get_random_image_and_flows(
-          shape, image_type, flow_type)
-      rand_flows *= 0
-
-      predicted_interpolation = sess.run(
-          interp, feed_dict={
-              image: rand_image,
-              flows: rand_flows
-          })
-      self.assertAllClose(rand_image, predicted_interpolation)
-
-  def test_zero_flows(self):
-    """Apply check_zero_flow_correctness() for a few sizes and types."""
-
-    shapes_to_try = [[3, 4, 5, 6], [1, 2, 2, 1]]
-    for shape in shapes_to_try:
-      self.check_zero_flow_correctness(
-          shape, image_type='float32', flow_type='float32')
-
-  def check_interpolation_correctness(self,
-                                      shape,
-                                      image_type,
-                                      flow_type,
-                                      num_probes=5):
-    """Interpolate, and then assert correctness for a few query locations."""
-
-    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
-                                                        flow_type)
-    interp = dense_image_warp.dense_image_warp(image, flows)
-    low_precision = image_type == 'float16' or flow_type == 'float16'
-    with self.test_session() as sess:
-      rand_image, rand_flows = self.get_random_image_and_flows(
-          shape, image_type, flow_type)
-
-      pred_interpolation = sess.run(
-          interp, feed_dict={
-              image: rand_image,
-              flows: rand_flows
-          })
-
-      for _ in range(num_probes):
-        batch_index = np.random.randint(0, shape[0])
-        y_index = np.random.randint(0, shape[1])
-        x_index = np.random.randint(0, shape[2])
-
-        self.assert_correct_interpolation_value(
-            rand_image,
-            rand_flows,
-            pred_interpolation,
-            batch_index,
-            y_index,
-            x_index,
-            low_precision=low_precision)
-
-  def test_interpolation(self):
-    """Apply check_interpolation_correctness() for a few sizes and types."""
-
-    shapes_to_try = [[3, 4, 5, 6], [1, 5, 5, 3], [1, 2, 2, 1]]
-    for im_type in ['float32', 'float64', 'float16']:
-      for flow_type in ['float32', 'float64', 'float16']:
-        for shape in shapes_to_try:
-          self.check_interpolation_correctness(shape, im_type, flow_type)
-
-  def test_gradients_exist(self):
-    """Check that backprop can run.
-
-    The correctness of the gradients is assumed, since the forward propagation
-    is tested to be correct and we only use built-in tf ops.
-    However, we perform a simple test to make sure that backprop can actually
-    run. We treat the flows as a tf.Variable and optimize them to minimize
-    the difference between the interpolated image and the input image.
-    """
-
-    batch_size, height, width, numchannels = [4, 5, 6, 7]
-    image_shape = [batch_size, height, width, numchannels]
-    image = random_ops.random_normal(image_shape)
-    flow_shape = [batch_size, height, width, 2]
-    init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25)
-    flows = variables.Variable(init_flows)
-
-    interp = dense_image_warp.dense_image_warp(image, flows)
-    loss = math_ops.reduce_mean(math_ops.square(interp - image))
-
-    optimizer = adam.AdamOptimizer(1.0)
-    grad = gradients.gradients(loss, [flows])
-    opt_func = optimizer.apply_gradients(zip(grad, [flows]))
-    init_op = variables.global_variables_initializer()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      for _ in range(10):
-        sess.run(opt_func)
-
-  def test_size_exception(self):
-    """Make sure it throws an exception for images that are too small."""
-
-    shape = [1, 2, 1, 1]
-    msg = 'Should have raised an exception for invalid image size'
-    with self.assertRaises(ValueError, msg=msg):
-      self.check_interpolation_correctness(shape, 'float32', 'float32')
-
-
-if __name__ == '__main__':
-  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
deleted file mode 100644
index 1cba46e17e..0000000000
--- a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
+++ /dev/null
@@ -1,261 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for interpolate_spline."""
-
-import numpy as np
-from scipy import interpolate as sc_interpolate
-
-from tensorflow.contrib.image.python.ops import interpolate_spline
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import test_util
-
-from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import gradients
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import googletest
-
-from tensorflow.python.training import momentum
-
-
-class _InterpolationProblem(object):
-  """Abstract class for interpolation problem descriptions."""
-
-  def get_problem(self, optimizable=False, extrapolate=True, dtype='float32'):
-    """Make data for an interpolation problem where all x vectors are n-d.
-
-    Args:
-      optimizable: If True, then make train_points a tf.Variable.
-      extrapolate: If False, then clamp the query_points values to be within
-      the max and min of train_points.
-      dtype: The data type to use.
-
-    Returns:
-      query_points, query_values, train_points, train_values: training and
-      test tensors for interpolation problem
-    """
-
-    # The values generated here depend on a seed of 0.
-    np.random.seed(0)
-
-    batch_size = 1
-    num_training_points = 10
-    num_query_points = 4
-
-    init_points = np.random.uniform(
-        size=[batch_size, num_training_points, self.DATA_DIM])
-
-    init_points = init_points.astype(dtype)
-    train_points = (
-        variables.Variable(init_points)
-        if optimizable else constant_op.constant(init_points))
-    train_values = self.tf_function(train_points)
-
-    query_points_np = np.random.uniform(
-        size=[batch_size, num_query_points, self.DATA_DIM])
-    query_points_np = query_points_np.astype(dtype)
-    if not extrapolate:
-      query_points_np = np.clip(query_points_np, np.min(init_points),
-                                np.max(init_points))
-
-    query_points = constant_op.constant(query_points_np)
-    query_values = self.np_function(query_points_np)
-
-    return query_points, query_values, train_points, train_values
-
-
-class _QuadraticPlusSinProblem1D(_InterpolationProblem):
-  """1D interpolation problem used for regression testing."""
-  DATA_DIM = 1
-  HARDCODED_QUERY_VALUES = {
-      (1.0, 0.0): [6.2647187603, -7.84362604077, -5.63690142322, 1.42928896387],
-      (1.0,
-       0.01): [6.77688289946, -8.02163669853, -5.79491157027, 1.4063285693],
-      (2.0,
-       0.0): [8.67110264937, -8.41281390883, -5.80190044693, 1.50155606059],
-      (2.0,
-       0.01): [6.70797816797, -7.49709587663, -5.28965776238, 1.52284731741],
-      (3.0,
-       0.0): [9.37691802935, -8.50390141515, -5.80786417426, 1.63467762122],
-      (3.0,
-       0.01): [4.47106304758, -5.71266128361, -3.92529303296, 1.86755293857],
-      (4.0,
-       0.0): [9.58172461111, -8.51432104771, -5.80967675388, 1.63361164256],
-      (4.0, 0.01): [
-          -3.87902711352, -0.0253462273846, 1.79857618022, -0.769339675725
-      ]
-  }
-
-  def np_function(self, x):
-    """Takes np array, evaluates the test function, and returns np array."""
-    return np.sum(
-        np.power((x - 0.5), 3) - 0.25 * x + 10 * np.sin(x * 10),
-        axis=2,
-        keepdims=True)
-
-  def tf_function(self, x):
-    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
-    return math_ops.reduce_mean(
-        math_ops.pow((x - 0.5), 3) - 0.25 * x + 10 * math_ops.sin(x * 10),
-        2,
-        keepdims=True)
-
-
-class _QuadraticPlusSinProblemND(_InterpolationProblem):
-  """3D interpolation problem used for regression testing."""
-
-  DATA_DIM = 3
-  HARDCODED_QUERY_VALUES = {
-      (1.0, 0.0): [1.06609663962, 1.28894849357, 1.10882405595, 1.63966936885],
-      (1.0, 0.01): [1.03123780748, 1.2952930985, 1.10366822954, 1.65265118569],
-      (2.0, 0.0): [0.627787735064, 1.43802857251, 1.00194632358, 1.91667538215],
-      (2.0, 0.01): [0.730159985046, 1.41702471595, 1.0065827217, 1.85758519312],
-      (3.0, 0.0): [0.350460417862, 1.67223539464, 1.00475331246, 2.31580322491],
-      (3.0,
-       0.01): [0.624557250556, 1.63138876667, 0.976588193162, 2.12511237866],
-      (4.0,
-       0.0): [0.898129669986, 1.24434133638, -0.938056116931, 1.59910338833],
-      (4.0,
-       0.01): [0.0930360338179, -3.38791305538, -1.00969032567, 0.745535080382],
-  }
-
-  def np_function(self, x):
-    """Takes np array, evaluates the test function, and returns np array."""
-    return np.sum(
-        np.square(x - 0.5) + 0.25 * x + 1 * np.sin(x * 15),
-        axis=2,
-        keepdims=True)
-
-  def tf_function(self, x):
-    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
-    return math_ops.reduce_sum(
-        math_ops.square(x - 0.5) + 0.25 * x + 1 * math_ops.sin(x * 15),
-        2,
-        keepdims=True)
-
-
-class InterpolateSplineTest(test_util.TensorFlowTestCase):
-
-  def test_1d_linear_interpolation(self):
-    """For 1d linear interpolation, we can compare directly to scipy."""
-
-    tp = _QuadraticPlusSinProblem1D()
-    (query_points, _, train_points, train_values) = tp.get_problem(
-        extrapolate=False, dtype='float64')
-    interpolation_order = 1
-
-    with ops.name_scope('interpolator'):
-      interpolator = interpolate_spline.interpolate_spline(
-          train_points, train_values, query_points, interpolation_order)
-      with self.test_session() as sess:
-        fetches = [query_points, train_points, train_values, interpolator]
-        query_points_, train_points_, train_values_, interp_ = sess.run(fetches)
-
-        # Just look at the first element of the minibatch.
-        # Also, trim the final singleton dimension.
-        interp_ = interp_[0, :, 0]
-        query_points_ = query_points_[0, :, 0]
-        train_points_ = train_points_[0, :, 0]
-        train_values_ = train_values_[0, :, 0]
-
-        # Compute scipy interpolation.
-        scipy_interp_function = sc_interpolate.interp1d(
-            train_points_, train_values_, kind='linear')
-
-        scipy_interpolation = scipy_interp_function(query_points_)
-        scipy_interpolation_on_train = scipy_interp_function(train_points_)
-
-        # Even with float64 precision, the interpolants disagree with scipy a
-        # bit due to the fact that we add the EPSILON to prevent sqrt(0), etc.
-        tol = 1e-3
-
-        self.assertAllClose(
-            train_values_, scipy_interpolation_on_train, atol=tol, rtol=tol)
-        self.assertAllClose(interp_, scipy_interpolation, atol=tol, rtol=tol)
-
-  def test_1d_interpolation(self):
-    """Regression test for interpolation with 1-D points."""
-
-    tp = _QuadraticPlusSinProblem1D()
-    (query_points, _, train_points,
-     train_values) = tp.get_problem(dtype='float64')
-
-    for order in (1, 2, 3):
-      for reg_weight in (0, 0.01):
-        interpolator = interpolate_spline.interpolate_spline(
-            train_points, train_values, query_points, order, reg_weight)
-
-        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
-        target_interpolation = np.array(target_interpolation)
-        with self.test_session() as sess:
-          interp_val = sess.run(interpolator)
-          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
-
-  def test_nd_linear_interpolation(self):
-    """Regression test for interpolation with N-D points."""
-
-    tp = _QuadraticPlusSinProblemND()
-    (query_points, _, train_points,
-     train_values) = tp.get_problem(dtype='float64')
-
-    for order in (1, 2, 3):
-      for reg_weight in (0, 0.01):
-        interpolator = interpolate_spline.interpolate_spline(
-            train_points, train_values, query_points, order, reg_weight)
-
-        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
-        target_interpolation = np.array(target_interpolation)
-        with self.test_session() as sess:
-          interp_val = sess.run(interpolator)
-          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
-
-  def test_interpolation_gradient(self):
-    """Make sure that backprop can run. Correctness of gradients is assumed.
-
-    Here, we create a use a small 'training' set and a more densely-sampled
-    set of query points, for which we know the true value in advance. The goal
-    is to choose x locations for the training data such that interpolating using
-    this training data yields the best reconstruction for the function
-    values at the query points. The training data locations are optimized
-    iteratively using gradient descent.
-    """
-    tp = _QuadraticPlusSinProblemND()
-    (query_points, query_values, train_points,
-     train_values) = tp.get_problem(optimizable=True)
-
-    regularization = 0.001
-    for interpolation_order in (1, 2, 3, 4):
-      interpolator = interpolate_spline.interpolate_spline(
-          train_points, train_values, query_points, interpolation_order,
-          regularization)
-
-      loss = math_ops.reduce_mean(math_ops.square(query_values - interpolator))
-
-      optimizer = momentum.MomentumOptimizer(0.001, 0.9)
-      grad = gradients.gradients(loss, [train_points])
-      grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
-      opt_func = optimizer.apply_gradients(zip(grad, [train_points]))
-      init_op = variables.global_variables_initializer()
-
-      with self.test_session() as sess:
-        sess.run(init_op)
-        for _ in range(100):
-          sess.run([loss, opt_func])
-
-
-if __name__ == '__main__':
-  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
deleted file mode 100644
index 017969d230..0000000000
--- a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
+++ /dev/null
@@ -1,251 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for sparse_image_warp."""
-
-import numpy as np
-
-from tensorflow.contrib.image.python.ops import sparse_image_warp
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import test_util
-from tensorflow.python.ops import clip_ops
-from tensorflow.python.ops import gradients
-from tensorflow.python.ops import image_ops
-from tensorflow.python.ops import io_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import variables
-from tensorflow.python.platform import googletest
-from tensorflow.python.platform import test
-
-from tensorflow.python.training import momentum
-
-
-class SparseImageWarpTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    np.random.seed(0)
-
-  def testGetBoundaryLocations(self):
-    image_height = 11
-    image_width = 11
-    num_points_per_edge = 4
-    locs = sparse_image_warp._get_boundary_locations(image_height, image_width,
-                                                     num_points_per_edge)
-    num_points = locs.shape[0]
-    self.assertEqual(num_points, 4 + 4 * num_points_per_edge)
-    locs = [(locs[i, 0], locs[i, 1]) for i in range(num_points)]
-    for i in (0, image_height - 1):
-      for j in (0, image_width - 1):
-        self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
-
-      for i in (2, 4, 6, 8):
-        for j in (0, image_width - 1):
-          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
-
-      for i in (0, image_height - 1):
-        for j in (2, 4, 6, 8):
-          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
-
-  def testGetGridLocations(self):
-    image_height = 5
-    image_width = 3
-    grid = sparse_image_warp._get_grid_locations(image_height, image_width)
-    for i in range(image_height):
-      for j in range(image_width):
-        self.assertEqual(grid[i, j, 0], i)
-        self.assertEqual(grid[i, j, 1], j)
-
-  def testZeroShift(self):
-    """Run assertZeroShift for various hyperparameters."""
-    for order in (1, 2):
-      for regularization in (0, 0.01):
-        for num_boundary_points in (0, 1):
-          self.assertZeroShift(order, regularization, num_boundary_points)
-
-  def assertZeroShift(self, order, regularization, num_boundary_points):
-    """Check that warping with zero displacements doesn't change the image."""
-    batch_size = 1
-    image_height = 4
-    image_width = 4
-    channels = 3
-
-    image = np.random.uniform(
-        size=[batch_size, image_height, image_width, channels])
-
-    input_image_op = constant_op.constant(np.float32(image))
-
-    control_point_locations = [[1., 1.], [2., 2.], [2., 1.]]
-    control_point_locations = constant_op.constant(
-        np.float32(np.expand_dims(control_point_locations, 0)))
-
-    control_point_displacements = np.zeros(
-        control_point_locations.shape.as_list())
-    control_point_displacements = constant_op.constant(
-        np.float32(control_point_displacements))
-
-    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
-        input_image_op,
-        control_point_locations,
-        control_point_locations + control_point_displacements,
-        interpolation_order=order,
-        regularization_weight=regularization,
-        num_boundary_points=num_boundary_points)
-
-    with self.test_session() as sess:
-      warped_image, input_image, _ = sess.run(
-          [warped_image_op, input_image_op, flow_field])
-
-      self.assertAllClose(warped_image, input_image)
-
-  def testMoveSinglePixel(self):
-    """Run assertMoveSinglePixel for various hyperparameters and data types."""
-    for order in (1, 2):
-      for num_boundary_points in (1, 2):
-        for type_to_use in (dtypes.float32, dtypes.float64):
-          self.assertMoveSinglePixel(order, num_boundary_points, type_to_use)
-
-  def assertMoveSinglePixel(self, order, num_boundary_points, type_to_use):
-    """Move a single block in a small grid using warping."""
-    batch_size = 1
-    image_height = 7
-    image_width = 7
-    channels = 3
-
-    image = np.zeros([batch_size, image_height, image_width, channels])
-    image[:, 3, 3, :] = 1.0
-    input_image_op = constant_op.constant(image, dtype=type_to_use)
-
-    # Place a control point at the one white pixel.
-    control_point_locations = [[3., 3.]]
-    control_point_locations = constant_op.constant(
-        np.float32(np.expand_dims(control_point_locations, 0)),
-        dtype=type_to_use)
-    # Shift it one pixel to the right.
-    control_point_displacements = [[0., 1.0]]
-    control_point_displacements = constant_op.constant(
-        np.float32(np.expand_dims(control_point_displacements, 0)),
-        dtype=type_to_use)
-
-    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
-        input_image_op,
-        control_point_locations,
-        control_point_locations + control_point_displacements,
-        interpolation_order=order,
-        num_boundary_points=num_boundary_points)
-
-    with self.test_session() as sess:
-      warped_image, input_image, flow = sess.run(
-          [warped_image_op, input_image_op, flow_field])
-      # Check that it moved the pixel correctly.
-      self.assertAllClose(
-          warped_image[0, 4, 5, :],
-          input_image[0, 4, 4, :],
-          atol=1e-5,
-          rtol=1e-5)
-
-      # Test that there is no flow at the corners.
-      for i in (0, image_height - 1):
-        for j in (0, image_width - 1):
-          self.assertAllClose(
-              flow[0, i, j, :], np.zeros([2]), atol=1e-5, rtol=1e-5)
-
-  def load_image(self, image_file, sess):
-    image_op = image_ops.decode_png(
-        io_ops.read_file(image_file), dtype=dtypes.uint8, channels=4)[:, :, 0:3]
-    return sess.run(image_op)
-
-  def testSmileyFace(self):
-    """Check warping accuracy by comparing to hardcoded warped images."""
-
-    test_data_dir = test.test_src_dir_path('contrib/image/python/'
-                                           'kernel_tests/test_data/')
-    input_file = test_data_dir + 'Yellow_Smiley_Face.png'
-    with self.test_session() as sess:
-      input_image = self.load_image(input_file, sess)
-    control_points = np.asarray([[64, 59], [180 - 64, 59], [39, 111],
-                                 [180 - 39, 111], [90, 143], [58, 134],
-                                 [180 - 58, 134]])  # pyformat: disable
-    control_point_displacements = np.asarray(
-        [[-10.5, 10.5], [10.5, 10.5], [0, 0], [0, 0], [0, -10], [-20, 10.25],
-         [10, 10.75]])
-    control_points_op = constant_op.constant(
-        np.expand_dims(np.float32(control_points[:, [1, 0]]), 0))
-    control_point_displacements_op = constant_op.constant(
-        np.expand_dims(np.float32(control_point_displacements[:, [1, 0]]), 0))
-    float_image = np.expand_dims(np.float32(input_image) / 255, 0)
-    input_image_op = constant_op.constant(float_image)
-
-    for interpolation_order in (1, 2, 3):
-      for num_boundary_points in (0, 1, 4):
-        warp_op, _ = sparse_image_warp.sparse_image_warp(
-            input_image_op,
-            control_points_op,
-            control_points_op + control_point_displacements_op,
-            interpolation_order=interpolation_order,
-            num_boundary_points=num_boundary_points)
-        with self.test_session() as sess:
-          warped_image = sess.run(warp_op)
-          out_image = np.uint8(warped_image[0, :, :, :] * 255)
-          target_file = (
-              test_data_dir +
-              'Yellow_Smiley_Face_Warp-interp' + '-{}-clamp-{}.png'.format(
-                  interpolation_order, num_boundary_points))
-
-          target_image = self.load_image(target_file, sess)
-
-          # Check that the target_image and out_image difference is no
-          # bigger than 1 (on a scale of 0-255). Due to differences in
-          # floating point computation on different devices, the float
-          # output in warped_image may get rounded to a different int
-          # than that in the saved png file loaded into target_image.
-          self.assertAllClose(target_image, out_image, atol=1, rtol=1e-3)
-
-  def testThatBackpropRuns(self):
-    """Run optimization to ensure that gradients can be computed."""
-
-    batch_size = 1
-    image_height = 9
-    image_width = 12
-    image = variables.Variable(
-        np.float32(
-            np.random.uniform(size=[batch_size, image_height, image_width, 3])))
-    control_point_locations = [[3., 3.]]
-    control_point_locations = constant_op.constant(
-        np.float32(np.expand_dims(control_point_locations, 0)))
-    control_point_displacements = [[0.25, -0.5]]
-    control_point_displacements = constant_op.constant(
-        np.float32(np.expand_dims(control_point_displacements, 0)))
-    warped_image, _ = sparse_image_warp.sparse_image_warp(
-        image,
-        control_point_locations,
-        control_point_locations + control_point_displacements,
-        num_boundary_points=3)
-
-    loss = math_ops.reduce_mean(math_ops.abs(warped_image - image))
-    optimizer = momentum.MomentumOptimizer(0.001, 0.9)
-    grad = gradients.gradients(loss, [image])
-    grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
-    opt_func = optimizer.apply_gradients(zip(grad, [image]))
-    init_op = variables.global_variables_initializer()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      for _ in range(5):
-        sess.run([loss, opt_func])
-
-
-if __name__ == '__main__':
-  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
deleted file mode 100644
index 7e303881e213a82e412d18de9d9d86f368726f06..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 14060
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}9Bd2>47O+4j2IXg*pj^6T^Rm@;DWu&Co?cG
zu$OrHy0YJ7<rh|!=$WbelYv2j!PCVtq+-t7xs??nuK$kRpLvq=NrQ@tq6}xagTWmM
zmYxQMu0<;zG<K|Jy0`n)sa3nReidK+8Wnpz_I#|qO@XMO0DnkGKp>yv!2(t%1$ISo
z&l83--`$s=x@`68)xTz)^St-`<?`LDS5@z-w);GH`>NHe8~I9CILp2YXt%nukZ-GS
zrPOjh(~VYon`(bDd|{e!Yq|e&MNZDSZY^zX-}w3YU&=^I3f)^7>=EJ7p|7OG6y(f&
zC^hn%i+gp7=cK^6dk;QVC{*v|-Fo%vr0x6mE!?(k+ro9r)B?hzU$39G<iNa1LjSk@
zsrm26z|Qc5kz>n~liN&78FJTnFf4B8duwHDXIB-oJui0Bq)EGuELrB(vV2)vnWx{k
z7IhVmj_%2VLQ0O)R2hsIIOY0z6c{4aUaQy^D9xV|x;-;vLO_IuqJ6wmUfzPV_xIXP
z&5^11E!?<&(V|1~5wWrVo&Ed{M={K1xU%KM)ePoW3{k6R9654A^5sj;>1wK_Z|Bdt
zwe|K4!@!MyfB#kr3QB2d)pXFB;Kg=Yh;_4FJmW0Z7*FNbPuK&awG-dRao*m>x@zCg
zWAU@*zt>;6BICy7>CgYihl<)U2VBevUomwxbB03ln;T15Yk#Tu+dZA~?qm8KLFwu9
z-hC{6sdVu1Ld7SIuX)m}j4lVxKDT)BUa>OHNh%AMxE$iQXL%L2jz{L?$xMw^tGH@5
zmaqS!)(}7YPuJF@xoaHwXIgChKA(;6jp@HL#?{CA<zxl#E@t1z$>njxXohQ|;FBtk
zpA%kResN;0OXgcE-aWhK&DBl5x~gsQ{Tg8n9ldGS-rsIdXMB@$Y4r`RQX|H@xo2iP
zu{JjP6>pRoH2=uQ$I6QyJ<8y?nyPAadXr0Iu1Mikk5#>|ju~VxeCnQT6*T$98U58d
z-jiDH#Bk>CRTZuO+#6HyMdO}*g}{H7fLkfy+eBC1Wu7}l#4z*Djz<1IX>)_Wdp-+2
zs{S^`mg_*3x8o&|A6II&HZAYHQx$#SnRFz#ScF)b;Dayy&eKlTRbGsqrkZn;>E+Vt
z6TaWhzY!YH(0=;#$5P?AcVgkytX3&6F6>}UK0fc$@0*+3zepLYtqHEM;9hrvee=X9
zZh^nb3EE|kYU-{m`|f){q%t?Ra;wg6{!Q|0b0;z~goYl7&R_WG@Ar-GPYBvn|1h}c
zV4WkBlkd>pD}O)x<jdvhXXYBGI}}yzG7n+A{+RJg-1Oc{w>r*bUsMo`;aU0WaHT=^
z;w{@=_T8Ot_UY5<*Vk1&rGK83$#^2w$nfE$`ijbru5ZqtKYr;>j1A+0du#O<EWH}n
zz<2hH&(F(KG?T^7%{E`)?$veN-62g`PkL%9n}gtkG`U%RkE6Ng**=Wq*>^ll*<!!Y
z%CA@7{1cs#Dmm+^e0|IRjmI}Uzo7Uh{ePVe^MZrD`ioVsK5LY-Hmkh&<Ye;uov&6M
z)ZANLU2AgW+M-Q`yEkQ7Z+V>)`M2MRW%ItPP9`508_T}^`SzyPpQ;5ncj_>|zP|X~
zN%f89_VWGm77f>%{slz^<chtxa^=P4v@;fOr><TvXPodMAzp$(!acR2HK01&Uu5Rj
z*CpB<>wAtL-Y)jX<Lviyi#+mD%#WO{`qcUQm|Lb*?o%(B>UWlZW=(FsA0Co0`SAK>
zu2;nx`{vqKKZ*YQ+<#WVF-gNfyZKjc{9@2}kQJ@oqtkAoB)wf*I+}Z5zggFTHRWlx
z8)h9hd%W@|<5{z+3!x2DQ|F1UT+cqQM#$anzfAn!U)o{|d+)a91UvLgPv4aB?ak(Q
z`#+s(JUQ2T^?|IltarN>em|2XV#+qV_n3*yqHSq!w>>(c-FIitiBsD$GU`LFDQywU
z&`x;%oVomNYwYf_hYSDy{<v&!{;H=}!W#VcJey@4cBkg^l!q^0CO*_UV$%>gq0-!I
z%UTJgcpavo-Y<V<%l!HKlF@vVP}t=o4x6WXM7~%$HC67;>FJ9fwTgc@d~NLv_60Y-
znO{HiY3mop8JUM#rmTLq^LZD4-tOF(s-z^3j^mFTk3VL8(0Kgu#WPbo5BNPhWyCNc
zmyv1XBz@!g8o{?G)}+SzhS!N?*(97^G5O3-o+|lQ`+Mr7BftO5nq?RN`*GUoW-sIU
z+EO+Wg3sr+|7+)ed}x=I$h7I(-`#$>R`SJ(oyEuX;x|1x6@A`HThq)Wgj<h4O)BAh
zXMS7jP3PK5oe6Sh#ZUb+c%eV{pW};#m%G_5_>*)O*c_-lzKC_^z3o+Uzy3;{efhM5
z<H4TK0@qgxtI6B-1oz7vl-hDhZC_ZXWl->9w%FpcJ)zxVbFb-Zonl@a9=6Y<Gb&xT
zZN;bKtG{&C7{~q;e)@WQ_`g>_?QX9)dG5rdj@d6>*enhc3_N-v=UmjpW_HE9WzyTy
z(jRYnl=Q~*k5aV5T0N)q)TRI4ov)ww|J<HR=U^qJJq%MaR=6aZ_V(_(#qoOKsgH})
z-{;50yX~%@=eqFptGAr`pH>OZn!kLjaGGHBHJQzJTG@+t@9Wu{AuG&$eof?yiPPI9
zuU^&A(LKs{kTp|qtJ(tF=`lusma@yusK4IAd3W*NIDY0$S;{Bout^8}&&g=Gn!5J>
z)hKQ&U;9dvfQ;|?W*`4-|5R!7<7Lq?hiu)pZ|nztWh__te1MH%KK~3W(Ys2@g=vnA
zLfu^-B%?Z`xGyAji~0U~^RRvW4g*tDBc2aM4q@t6N}CxjE4;p3S?M*4X^os$ZRJek
zjkizCs$KQ_!PXNQ2UE5`-1;PgF?{;Z1ncz@T?^P6dU#6gzJ~Ptcx%q1uW_JV!EU+h
zRjGz|Rp0M!-uM4>eA+xQ;m4mpMH!xWV%jjf_tGB?pOie^7bcU|_3zsASL(0Ek&Dy*
zD?faieelWFvKa@nV+@}xm1KPTk-^iG<LoJ?U+ecjI<v<{hdto<ZmY9P#aONM^o*WP
zbbpq==jk-#OUlZ7<!-5O7Mzv)Fk!JS%UW@UY}GCIdF;JDN9p!7ENy&J`}0@FnrTnn
zE7v}~P@VNlVvED)oX%qT+-YnK7cYL8SIigm{msFfKY#u>@!nmunst`9E?dd|x{H71
zD;|`;UcW0V<^6>K{VjqK@(Z{Ai`*(@vLPk%*^=g4Vccu9k~?~JTi4v|x9AsLzedcf
z?ueLc>b$cBY+L4<FmUj*r=4;5^zQDAu&_4&RjUK@3Lg0`6JxYeRxW&D`#t9C(>a#T
zK{a-)ai!~ePk3!vmKe5p71O3k%btC_mGV^h_R7jXD^9s3?45Hsi*L8aqHcw2a?9_f
zMX=xUoTW6M-{QH9%DY>wD^pW{EZQWb+9aG~WNLk(*I@nL4~HB*<JL26<kz~Q;wwEf
zb4O)hRBOoI6TdjtKFdyg;OhGM#c6MDHLcq>*`@cMRW`l-DKRx~+A%%mhU?sOYR!a-
ziVChxRQ|DP6_-1&|B+Nv?F)Z@AOEuZ_q)pYv$IS!PQDIh2z|@e7}<Nl=KdP?ncIZa
zgETc>7QHgkUEBNMq*O{?Ll%F9_O-)S2mBnK{jILFIq>9U$Njz1lfAq)>{_+zXVn{1
z=?iyvxqge;l=APFx4!HwC#B6?8^zMzcczM_SSfwZTh#G}O?K(mdo#~IW=>BLpQUAF
z{W~yY?bN><Z&i9@!=8L)3SO?1_`A&K<mtnoo~_cTn*G?B;mDox`%|U$_S-C8o@Jad
zW9!AJua`{j+<Lpk%q8Fd%Yn;htKUq2kf=8C%qhv-Z1*?MXUK|gHhu1XdiIo>ILpl@
z+Vh2lm~4LQ_{}w7oX^lvSYw&9!m@$y_vv`OxcaYG7gznR={mnuc?)}kKl6G!uk7b*
zmaAPll(R*zZQ}HFd5w*08eU!NcjD2T`&{hAqT04r#{YlR+fvh#EE^22`}r+c#&)at
ztf_GL|9{_u+gjSS3pBT{VKh6m>6`QGjeFHkWhie53|XeUQE<gUrz?9F+?pCVrA#>G
zV*1e|4FCU`pOY|M^-^9WBZ$BDr0U`SpI<C4*6--#{P@yZ;^kt)lj^#1Q9CZ~kmX*f
zwJYej{^LIXt=BJIlv4X8rFq>m_W4=Y({D_-K3=(a(F0i<iN9rU9DEWL?XFKRaQ9{V
z^5o>o@cZ@u|JW~G=GK<|{@2XIYks9aJ@q15m*JOFcJ`lh3Axz|qHN7;_lZtednn74
zpJCEuL8a3k=T9DNymV<<WxxW@20!0ksgmol;d^ZhlahLv^j^>YTF#O=-)Fmf_F^vX
zb#iH8*Y>~IW^QRK|9Afk*$HzU)_z%j?u5hgH#t2z8V6pz%9?U8)wH#}Z_l?|pTp)>
z=e@6J`2XFxJ3cX3r~O3Si|>y2U2QC%oZGo8kTK<R&ox`Q2M^mX=3h|IVHS9J$}^xy
z_Qh#^`OoJLwI=`mS#wOTx8tw&lIcGWuluOn5cni(ebIZHyDJ!Gt-Tx^A@Si#u*AHH
zjpDJ^oo+j`EdA_lpT1J{es=HH>hKDU1-)~AP2<T8xf;f>b>ECNm;G~?-|!!}rQ*n4
z>mA*8Eg^Ku>$CGWpDz2<b15n5!N;E$iofSQ+41wi+S!6uA78CLUtjX#g5N|IvpPk&
zmaNE?g|WZa%==X{Pf09IebX!V+l%J>O^)0s=cm)A@wQ7_z`Vvnj%`B1XO`9?-WS>H
z&;I%_$@`b;C9#I<^PAH3w>&<S$sQJXJ#|%2;FXPXGrPL`mCd&^ZQbg|Z?K*BLw0tm
zjD^C(7lHRS-mvyt{a{ukU!c_7e=inG7$mXW+fb4q_Jr%g<6}#8cD-D7Qu&0$nO`r~
zFs$H9X1!q6@T7a*vb03A&AeN_#3v*sioV<PIrz(~uOhq3>zP)rJg=nniZdlu_voh7
zo3oZLdVBH>`xCn#n|Sv8&eK)%({T<7;k<O|)Wit^jraDh-cV7vmmx1Z>(mP$|JdD3
z+~P{cy1e05b`igJ#NKefzI@)S{mB;>IiCCyYrE8Y#lO7y`_g4!ul1L?)7jzoENaG4
zW$DgkrrDbckBYuuv}aFPg;DVGZwEJ}8Vg8EbG`k*<aSsnV&<A@rc!<3cikf+nV+4}
zJa}vCBlUSn$+i{0-wOE5=X)lzrrUhhq^YH=Gbgz``}{oly88SFCM=fI;w}jOy5#M?
z*S7rKfs}7fneqFaZ?ly3@Q4N6yCL!BwW3Qx_u?nr`gupr%*>ZK#KSPZeqJ}H@V%o8
zo%fteOuKhwEk{m`US%bt&zy)H4r7KX(@$Icud$D-yZLkT>bwY^K&iRkUo7^Jw-V8}
zu)E;K%36|@^(FK3GheH{-r{eYXRqp2$gKIVEw`t2t6Y`#p(dV;h0E;2kBP^>IoNAn
z*V4w89sj0JR&~dRLsM6*Pv5&$^mfA0{nqUJ_r)Ds6M6K;#>4q5Yn+)2{(fbw`Pgdx
z?Qq83JgL3+7PA}VN}cQJJ<4&My-T5So?Y#i)57X*?kfeSf4!iSlB05!Ig7_w`sB;M
z4V(EyL}U)%+4){UV1xKlvwXWI>vs|Sb~RxOOeb%yy;JBeX_mk3J6op%!wl<iCr8H*
z0lNj~y1v|gx9!LllRu|FIx2OPz1WyMdHw$H_dc)p@%i%Wg<0y~27_CPY+v5a`O>`B
zOD;P&BB1hHGyBdK`#%qr*TmN!xM=zB%Vois0!OJylav;+_lF;R`jp)`^ZUlqY}s|Y
z&)pREy?WKbmDQkl+2XVQCo*?$4&0?QpTDEe?acc9j|wU+w^>@+Ef=&Z`SfICIlt5s
z?qGK5=e#XSpU&h+-S!dB@ZooenVFP$?LYUMd$l6#_wPIUP?KpB&z<u7{SOY8Et=xF
z@8$A{d*AL=pKi)%WsIv8y&-)hqvW8;dC8@he;TJNX<15kGqzY*y16@~r!D)Fd2i1_
z%~Opl=SAkLiun6JpSXo#i>STrouq2VZP(omb6u*ROw`x?oi;IS-I=4`-Yzv@Kc{hD
z@5`@0YooI_R(&muK7MZ1LdJ?O7Z;Qz?yfwiGxy%|N#)07U(BD?6+K60zVL#>%Y7s_
zZ}692@6WHj{;AV!zkFpWl{*qSfky*&$AmvW&Mv>>^R3t0@7+4QWX8cv_L%J3d%sSz
z?tilPzg>X%lN)Xa-O}Ua_Sz&e9jg9%h{>Yj&{4AskB@)fRex^}qwN;)yC?n&UYN2l
z{;2X@X<p~mxy@Ib*Y2OaBhA;WPqsKcWL5e7vzaoS_cIoys-L&nbb3?I_j{|$p2nOy
z-RN)6TW5bTis#<H7e)1dlz%N>_NZ+4@l^2_6AB+2JzKV<!f%gVZ{LYq8f#jk*1qVB
zHtU)F{EX6;Z=b`~{W<$?tH7%Rjb1tT<Cu&;ta-ZmrP9jVIk&^Q@5FpRCjS40^7S%S
zg&!PeyD!b0$?^84aQLBR)3>pHnV@*L&OH5GL&{aNjNBHfqg!4^WiNViQSj!;BfA$b
zdsqMY+2Jz3vmvw1wVN3_l~&I!kxKcTvA1&b?#J?Z_uibVd-E$r`M^}8lAGD<9QVuF
z8k|38c$4YZm&^WpYcD9|YcL8W>}UC>ees4zj?B`tie~-Swr;I^bEELcxs@|tJ)b|7
zW37FDvctiX&5rZzvhwe~(z{??#`1o#=*^EO-G1)Ts6A)d;o)<{x2*5D(S^=V!QiyB
zQg_<73Gcam^OntD-JIMMbEfInQ;z>$s38}1_y4)Mu8!ODHr(3QoBJ~1;j{#vm60K0
z`QJaEf48se_Ivg+vxCC5VKtlPYP|i-I6bE6-^Ps=mtPr1u>JaSnO#rP+Ds=%M6~k#
z=fw|&e?LjcVVD21*!`~jlI1&gh*fdil<z%ZXVLp4MN9Lc{a)c7uNWCO!-$67MPHsy
zS5~u>ERVJoTkTM8{r5{F^WnocrPaP{GRu<QKPRYhMsD}E$McU?K7DE$cK-gRQ>R`f
z-u*Adn4>;Pc<S`szw?<HQrFqfdnJ>w_;2Yu9v-QdMuuHcmTcY*{gRWnR2=JRVG(uP
z`}V@VP2sQSZoPeTtE5{w_s{=gqN4AZW8?goeVsL%Zd9>|y>IvNWh~iwda*e>_l6he
z79MBTiD<Z2`|ClP_VGl{7d<_U;$PorR)2|KvCD^z{Yho=)}C)u)6*Jm&#Wlto$d5x
z-_~hfIj(D53Ing?R~K<@Si!^4BgL8@Bcr$R)6cL5IjhW|`6{*tY!ZG7&dhzb`BPH)
zYr#nSqephww@AJGkk6(wSFP#pzUh6vrS)4nZ$w-#moz!Br&jZ+x}S?tgJat5nQ<#S
zg{$@D?Cq4iIuET$zxFG!R!+WP)46>SllJTGJ-1ozjUlUmv~*rs)V8Zzvn94X_}CEI
z=Q&YUTd8foX2ZLzlM}5bpRl{3>z`kKGu7k8efjOPYPY?Nozci#-P_+CtYoy{AKQ{+
zsh*SF<Tg)o*wv#|T*k!k_QST4U2#Un;nRLjP*K@oSH~$Nbm(CF<gJ!&7jLc;RqX06
zmVfc1;(9q}gG0s@<q7QDr+%va&17geBjel8wD7DKOFV5Kzts1zwquJHoYGR}^!04n
zyu+Jc&EJ~Y%wF#5(Xm^AM_SB#uDW#Z>{DG6HXUwn-&LizaN&Um2iLs|(%2o=e77&o
z{xOdVqeRk{xevF`n>lk!#+Mfz8E?fkjL&mlI`C-?+kK}Mf9Kw9oN=$-rA6Gi?TBQ&
z;cO{GC9xWXv)7upD=EA0;{I}B;ghwYzN)(!q$5SDel~Fn%%7E2qkTu<Mcm#==Y0I0
zxvCm&F}dNjKi&35n%&H~J$BRYif4Y5u-&!$;>w%XRs<Sea6aA~7+HS4YX|R_6BF0H
zJGZIg!L5}Fd{QhWFPl!frS>-TSw5X|<lP>=U%&L*+0}V18E?PHTe$V)3-_gG?f?G?
z+59>sGBa~-;j5Jw=VT{{EM&|+;Iu)$RLV?;W#!6uy|>B^vaWmUrJ??y>)YZpd*c*z
zdG9io2u>Aenma?HGA}GF$NXyC3h@}}*0TwZm+pSL*mbR(&@%t`y)QD#XT8_kyG8wo
zea^S}?c3j(SWPzRcR%*#=DU!R<mAT8%P}{nt~Tnr_oYAa7>|mpn^Ogw(}K&Fh4o}D
zE(kAT@Y&TS<(_(v!{b?h=teVV7QMOoVk=c!+N%prs@9%ezogSf?!l+i!3lh7IqPn3
zRtk@K$ai6Ly7}6Av)~QelK)h+e1Fr~?fR5QspL9i?5+<{CO6EIi+9}FDE&?OTWmQm
z|E{gc&(5Tzq-c~K)_66`HAQaAwY4WcJhYorUsv$oLQ(O+{C^@>^m0z>-gbPyzPfxt
zu=+<{v5VX9YU-}dmZ@cKF}Z&6Vj%yfZSAtsoogHNm-cWZ{oTgQ=JI&e>R-2arb$)W
z9AM_>X)@fCcCDGs#6tVigXW2jjvsg5XZ-ui^8P;6BiUa)yqsiMnwNGr$k~?Vg!EkV
z%FwraJ9lfX#S4u*wiQ$R+~3sQ$c>n=<#+vyg@GPEJzL7YO582-w&#%+<JB$MbAHKX
zM#BXMBPTvQT)X3?T6&$LCTGP5NB7xgJ6+6NuGCDQ+M37U_W9dU)~Xj5)?NPf{kv<%
zew&$}F6vCllHB!p;txY@^@%qkUHtv}4qRF)yl{{5#Z||8UUnUqw{QJysbtOQXZd~Y
ziOK$MNnbLmb>-~jKU|Auc{OcH@c%PePdr?{zB>B;%XwBSJG)<B<ZOHPZoU;eg<Vws
z&$BP*zijlKIjKcF&VpA_@!Gn3E0?<^|NOL(gPVEH!5lWDy*!gkleLA!)E*w@S2j%P
z*t*LxL*v6G?*mIbr*-Yt+x}<sCVd7Ee|9Bh!@lsg>d;-KuP4cfEn{`LeJWAcs;m0y
z&Fqa2{zt62C2AB~ool})s#Gm0xmP>v^0Lz7+wb?edwFU6n4f*r{j(0c{;bc}&aU<F
z>gwT@K4e(B%J9m@jR#hR9<kmoUTNd7+|R*<Wm4tOJ=%}FeclM2ZCd@M{Mpv93lFlY
zFLWyHu;}Vuo}j#~W%GHzBlB#ZCw+Y6_~P#F%bmh}ykC{HwdLOLHD{Y&GpTcig#4F=
zPYY5PZwj1}I=vTUp%2f8m&+GMY&+AwMu*>N<?PPA)y(C0S>N2eY;}9iTMv<k$w>m-
zeEav<H(fa{cly?rg2d+*sSjMAw$6Oh(DRD>1>@JL$KQBAam;nEUlRD<%J?K(@!}V1
zfyI)ls--JdeNr|w<Pj8%>sYvn$yonxM&!oAu<O0ej=5`F7AY<87FRfTH&4g0TlUMH
zoPd&xOL+S9P9K<CzUaZ$>jq*66n|~5xa%Gq9PGS%0{7B}l{GiN?7CyYAMkU*r!DUv
zUn>_a<<6MtY5s5bvW0!!+^#Wm`n{gJx(RMk`rw-SCW^m0Oht+5#pUB~c5UtH`LlWM
zLG_pGR=#}sf=Pjw=fbWDcR}+$Cr{2RxEHVV`_6*0yDvhy(ykroT3e81|KhRnW*Zyn
zge|?1BIQZ8H{QO9oH*I_?=M9yJGotc8+LKbogiRor0CdWq1Eh_tll}9|J){?x)ign
zYkTHzIy3Y0B8lDSnigzTy0&<;<e5bszRPFV>dii{UH{3W^HS|EhK(B~%BD!_MYES=
zWJK)WVexd)Cmr^==Vm8t;5#RCW!J^&$?O@%2hOf(&^W7i#_hB~%{*c4qE9Q``57AC
z?OV?GqNC%&IYmXTEw2`bZd$YUjpTafERki7rk~D*?J@cN^y7il>#lKEI5JYL@9mPE
zaaL*HRQZORzfP+6x0h_-;&c?5y`0&~=#seGue=q3Gh4%>HJ2PT?K-ok)PcvsKgVU+
zWmkvK3s{es9p-S@Y%}rb2C0p28)`l=t<})+v9H_0_u|8c2?bvtX`U@AO5Qy84#%z?
zsm*^F4*X$Y*jJvmc>BzVjF+kRc5!4m@C(gN6Ir{~C3CgiEuOfD2>Z$-b`8=}tKKhI
z!fd5;Mta@XlU^!ei?&|5A)_An_WYz}dv|_(%sVY{lg=c;3#abn%w*pbDI#+4dE4@`
zsCBicPhEca@S&JeNyZBG5ZQaDZ|_Jhbe+J!=R1qZ{I{9TtAd%9yLBdUBw3y1$S|I8
z@XXqOe`hnk+w$nfzqM^EdT)9C-g15Q^XbLOuG3DgR^HRQefv>S$HLQWv4$*@**-LG
z(>7l4vecMcY1W;NYhhY_X$H-k?|n-t_u<=I`?IF1!NQ@s=1FL7Sfbe09EPeJ2R+NR
zv;}8Pf0nW-OK?p?*0mOPN6TF~w<q6ZF!574{Y>zd`#kf<kvz4h+vJKkroHqyJKNey
z_FzopHZGHd{YMXHm543<m>|A&%7zQ&9)T<?g?>3iX)+WjZTXh#&k~np-xF<iVp~r1
zmboY1ERmh%XUrVMVyck$alz*|F0T``yWbdY&0T$QPEsS^m1|xLW<TXv;&o(MV%W5^
zi#Cfcc&U-6vfDx0HS<7ZcYE~3TV1vbKU*DHwM~pcbS`6F3!ljf@t`NuRsEtj?^E)q
z<7tp967#XX)_gZC#WrP`4u3?T6|eA&osvh=3_ibUk35q9yXdw(bHj1wySv|&IQ;(0
z*l&2oWW^rq)&{P%6VtcdPFe2pM7{Sw*fX2m2ffXkZ}&Yl=sFOlJ>{+QqJJ0szKe3a
z(C1^%IO)HuRV(+Do33=#ZN<|i)m!dVT*z5s7G-8DIyrUswZrd@G@XquPjbFd@-61p
z@*b;yvppy2xV*S^tIf*P>2tK%k=zRpLbzVm{PMWV78RMkc$V-2#?NB>+j{PP%UsOi
zUw`bRruKyER~^6avzEP3TFSdM>$yr%bL%>uNxM7TrrHZ$*f1f@HEfZQ^>fEF>ozNB
zpMG;kC+hQpZA=yHXVQ`ue)}DGdfG+C*kIum;rgrgr8-AThFnTzVc!yblkpZu!GWl5
z_E<lWnUMjFd^2Xdef|1jbEeGchwHO4CJ0EK`jsAdn(Zgk6|)Rhu3bFmvrO18q+M$k
zpX>TF?7*_ibE{tR&Gejha`jwkse+!fmu-VJJR56WUA?XZsDIp1nl|}zBF7w`gLQpV
zizgqfuM0}E+t2)L*_DTf85(MHcJaNKB=rCM&!15fdK38y*rse`YWv$Bw|bS-hfk;7
z142U5v#pa~WG;BqbUyF&y3hCK-!Mzi(_x9LlU%VT=cV#Rua6gZ?sUl1tV{b`>hS#e
z?{YZ{ffbB%_V`~oCmsLxfd8^?y;BF9SSu^u?M&+a9dN0+^G@1v^{#7gw&k*?G5olF
zzk!vzp<O=jrSd~<b@r)Cm;UJWecn5#B;ozNHZN17&GBN^E)xH{)o!y(JwLcVCwP)d
zO8R84-23|!)t<g((9!b#FHvFe$wAaL`&!?H+gts3rQM$8?_ZlAyNf4{lYv333A7eq
zgZ5^|ls9kOFRqJS!Bo(odzjfO|GZrN+E1>Z-nQ^aRj@`>2_1f{6IJ&+SM}4UKOG$(
z9xUo!#PWN~tH$0%`ur(kF&8CN(k{-r&UDGD^rGt$^XNRi4T;W|>!!utNR_;?U*44A
z(@h(>3yqh8>StP6Bs;LLuG87QZ2gzNkL6Vv8o2d)oJve$?wG%qKcV#{Y_X~S`ciEx
zsTr1&b?pC^oR)3pcz4QcO0%%&j>F+$AwRlby*d+Zc4VsKa^>z%%k%fGJjN~`qGWQV
zS=IYdPLA}MLYB*q4*oNUcyYUJnvUZ7jTQg@F6+M(rh59o$Hxw(rd$<2V-~Dl{ks0f
zX7@iwweH>Ei}<!(@$$xG-W?y0IWZhq<Z9^QFa9%K-T8}IX*c`sJKIbu3^rs4M#gR7
zSg~r;etvT+rHj9r8e(@h%#!+ZK>IMOmHP6B-MP1!LicE_X%C9`Oj+A$c;b;_NvdVT
z&(GW&lih8FglcyD|0i+x`OW!JtBx+tef976$?AjUZy27P=jZb=*J@+fkjky~`O^nc
zeSedrZ~ph|Uh9YM6Y&g+*5+97r)z=E>%_uayA)3g<?Vi}VfSZ)l%k^Cp5J*)j~?6h
zeN(NQzp#+IPtHqSS1$K>(uXM%81{VTt6Y1OU7n%*&O^T~si|yMwqM?b$1fB)kan-m
z#o_vsr$4<_J9AHNs1j~vJFu^R?Rx%%dwY^>KJ)C{{PW4=jWbL%&pi4T7Pmad)OLng
z|2FBVk-jsTZ2nn9{`-C3bIH2vUyHM)J}OlN3VJcr9#=W=z_xYjRNmcQUT0=K=ur%;
z=UKxm=sbOHZ2!s0-5W|WP4~n-$gl4{xORKlvzl!eZ*OqPH2QH|Un_t4(ZzpVQs?FE
z=B;=lc;GSPy^3asec97Ln&sW#E~{^Cb;#8%>pI2Jc<%i7{r?~L2kqrr_A`E0?f*4?
zQ>QZOL<Th`9`4$Z$o%uCx6p=?mxX?ni!=03aWC`loZFT3`)YLlN{4mv{(1XsJ~aHj
z{n$?-{8Y_%*{si>J~XbF6F7bQjBfqDKTm%?uiZOk;*Y&?TiNAottRdAInVgCQ`{_8
zvS7)UkYnHP|K3<tDtoVHGuzaue~h*D-u3;c__qDHwsvrF<b-q2PB)dBozKa-ucfD!
zU-(qhN@~X^uLpOE8w(#d<lWse&wK{wfzI$i521pyrYB<eN;xwY+>WVOza$`JV|~5z
zM4jp?O@Yb$6H1-y1vkdJUVYA$DtTkCb;I&`s~i&-ww&^kY;6_#_J(tgMMgjS3@cNf
z?f0ZQ7b!_zk6r$a=gb+MgZK8n>2|I-aOrhegIUuI-Q&zVYTZ+!7SFsM^Y6L+l0!ev
z7#|RiXZZh1e5Q?D!UG2dL&1(NCHp4(e}WPQ2WH+)sFz^pUnIaZgE4tQV4&a+D{JeN
zFAol~_<Xh1{H4C<DtpGkDT)(+?*G5`p!WI~wSOKQw3|~~CXu&8k%^gomHw1H@2s|-
z*;mW>=TEV18~<|t>pU58l44<He$84<Q77#yx0GrZd{|)STN!$%^0}b?-#?!Z9G7R_
zT|T+=R{Zg!E9cH>+9+*Z#&GrOk4r1oMC$)uSbk6P*VRWy?^PU)tJu5hD#L+|aw*%_
zT-DH;v>|7vyj(x~#fuI>lf;vcw<#$$)^q6Zk7;M+US*nDS8v`jC*hXX1)CL>uaz?V
z<0dd3c<=t0+aq-EH>rZFp?^LdJ$j(ue$uIESJsW3(_`I~w#BYqc!Njv?fQDxkH+T}
zxG#L)6rA$whDX}6@6$!KYn(gl-ITA?yytnb*TnO2b<W$_8@EhKJ@8uh`V!j@4lM6>
z`%4PlI1zN=Wmv<uBy&GM-AE^`x-BZ6YuP`hu!iiOtiouV_F%g2><Nh_dzQVwuB$ig
z+Vf|xqV=w<Te!z&b6mMs<VS8ZTRG0x*GjL+oc{a$deZMQ`!`#|PBfLS{%uj2cyCuI
z|L5)ZqndB(++X!H;MAMj>{F&alHQzb##(jX&gS<y>q~*6PEsCkW1S2S7%ksp=@S_#
z*CVym_DH92|Kl5m<@a=MGj56y6+O_xd27oVpW0a-ehZ$u7P)#)f8^oEH_!BUjCQ`m
z{hfJp=lJhmb1&=5R;CR%>!#^lmD-VfJVhrWKwZP%Jy|{Q&y%)@i8{>JW1OqDUab0C
z`uT0or#T@kQ$O9WZD(Ro+g`MWvG-Dt&i!4YlP*qH&$-W+e1W;CXpQZJ3G?{wl+X7s
zwmChG^+SLCmBVw{3;gr?&s@IjKe<kNi(Ey6$=;bKwRVK9X`l4%vi}4x?~P|pZH$)n
zUhsNN<c&{<xILEU_hqOTF74mDa?yUq#oPKmgv9gDDe98{{Vnr+)ln|_v~wxjABGCJ
zPCdrA@z)FG2Tj~tGGD$tpueAC;lf96*M$8#^Qm&`!B*~xyOUVkm^H-Z92V=iW-z4Q
zym=!wH1*5s%eMrLXD1%_Zg2IhcrI)EW^+3GhQefbkuq;-b0+clHU9Ctu1KUsrCq$4
z$FhDE)3-MV^EWUUJGj|zU(&`V)75uuP2l1y;h7(^cJFy{)~s>j^}`Db{g%Hztrgra
zXJjMCe{F5By|w^D*qRB22U(3y+|IXNP`WC1cf-oyuXBIM*G$-=EbYm%LbRe+$&%sE
z_x&32o7t{kzpi}#ROJn+eHlkMa(?|t?)midLPfJH%La?{I`=&q8D8ptxS4+I%M_6Z
zr}a<jUgx_RUS?)cX%uk4=88$J)rKn5pX+*`T#1u->Mith-*W%s({gVcoO-o0Vyg^8
z!y+Y{?P<I(+p|gn($sU+`=^~}-}6)L!s_W}e?KICYJGDr_j!&LOGo0nyR%D#w{ma3
zY7=O;_s>c76_!;Z=WkfbM>{-!{<!!5Kk?r$n~NV!b*T(n;OD_`ubThG70=mqKMV{q
zg*Mdg{`%SS=GIj1ooPB}=2}h6mXXijv-f7XwpRWTp4fLkpKtvCA#qRU+nkOZM&1Pv
zA2{y$`MCeX?7yWS_^!QU6?nPj@(BhFyJc(>=KD+iv;TYLp6A@2Uq6x+w7b9TT=4Ys
z#BA%bHD7Bjf99QC^tt^0!ha_O_izQR-mY_cs*|gz=#7dO0?*FOY-UmRKjSqeeM;Vv
zFC{{2!`EM1cK!O~^aoX67~J}{l;^2;&#RZ4Vcy^K;N+{F&pqy^8Q*NZ;dNl~>A(NJ
z-`HAfz0NI|nO8wTsOiJO=EAi1_p-hRD+j!F@L4|7(Kd}|rA^J@I}fL;Bs}%{6mC_j
zwL-2+eS21R<LmY8|K9)q5pcI$zTo-XXTJp_U01){W3%hbuF@61@Bjbc;mW#idv%{*
zgIlCzomGWEvEFW;yNq>H(o#i}%+B(>@|^5(v^DQs@RmyB&IiHECfMy%-(2=yZLL`L
zwGW@=>m+u3kNf<$qapRQ#m4Dg_vHUIypgpokj-&RKcIMlVabJnkka>ko7s4sEY>%O
z?pV9zV)4XHn;K4dB)pBPjZ0Mga7ug0qT*Yb&wgj+Pg41`<l7v}0}Gvh)%3GhJmO6F
znDn{!<16oyI0G@4lm8zuhpdZg-Q6d9*z9$>=iL6uy&}C5jqmM$am;-9@X4VIxtANh
znq_EbhJ882ec-G)Gw-f)Wx-x)_B~&O3{pBi7w383+SqLK^D}dKk7CjH_w)a=i0N(F
zX0Of57<TDK$x`9^T-}0$tT#54YU_M^dHHl^3me0gD+$@Pwt9067-Q%BN?B~OxOARd
zzgnzJ_<FmK(*k?>WSQb>1U0mjF3!sRm91}8HKpPqEBBXvKh0Oycq2A4&9J?F%dS^J
zbgJOe&Trp;Jl6R2r#R->_r)h}1-?w(e$#Jp7gx^DFTpia=f8i)IeV>i<}H&4yWbzN
zJuD@4;_w`1HU@t?&MQ|UZs$i|`1tsU9gl>=xxg>}Q|I|D`Zh)L+PCa)cddS|yQyg}
z|LNA&i0Ihf1@BZ?L>DqXI3c*kwt!)?-X4LE4fpge6e(LpcV7AUF{NbwotOvL<A0`R
zWuMgjGCf{xdxWu3LqT5dg0HV{7(Q6Jyy3JyyWO7&|KBfpaZ1~Brkw4Xx3}{)Y!qJl
za*uT0fpzkA2U;8c-EA%tF}c9-iivSb`Rg{#<#QI5#RNT>c60-GX@$j0%WK>9E6>E4
zl!TU=uJqaWfhi!4Z^_b$`yX#xQ2V>#xSaBYb#cDuT$jAa2&#|RbmSNRb-$1@silk$
z_I?l8m?bJUefsC)t2ZlOfAvS%E#a*7`-H0N>$Em_*)yn%I$X5co#R^lohdwq@z0mZ
zm67YG{}I)XaA5nBmm9cy+q5r{buU#vId4rmdaO@%vE0|!>3@sY9LS!M9;O+xY?D#K
zx15?ywbti0)|DjfyxB3e;G%29|M&ID3b!96ir-q+uxnxC9oF67<6b+5ySg(!pX03b
zcjt4axz@AuFN>^Nr?!37{rdT5-pc%M=4Ud$oAUnIBx!@quS=wCO%5Jvy|Fim)sH<O
z=`Y98qOGB3y1d1Cdqw}xvKBA&oBifR=|!X4cDd|t4!55?(ENUH`ujUdN*>%^VPZ3l
zj6H9c2d`Mh_Tyi55qrs9t4km6oe=+QlFnB-Ma%N$FEh6bUDM_s(mj1(_4-9mTE#ac
z+_%%&US5{8vu?JcGV}WX&+TX2-fh0eW<_)FF}KrYU*2v%F#o^LGd~e2uV3-}vR-Vu
zZ|=|ka{Ap(VGplM{O1m>+Bfgcmy@TJ=dSwpM$wkF{_oT~S3kdw`<3k*8F?W7KUa@@
ze!jK3!R6Cs4%1Q<)Y%(Zxj!7!-~XWe+_@?G*BjXcCQfbr>ODOvBtk;|#Wf~VhpK0H
ztJj;>=eo!47P>7ey!hW;bCVT|Ij2tD`MWatI1|6E%dd4eLW>W_#Pi2gX`1EUXy9b9
zh+Q&gwPu6c_jfPMYpv{R{}g^%tld#|pp<D-Y`pL5+4&9W^HqJL7Ja*YZP^5YPrtTC
z=WVo$W78IMh?^xlqq1^K`n1#sebyWDuWHQ+jayfLK4C-A;+uZUUu8`R4PUIF!!x;!
z*I9kBnz?$VO-Apwt&8kxJKpd4m1UImY@zcz_rUE}_S>&~vG*fe!P~7*UJC8r#Jo}C
zewN(dnd|p;JrdO}$aZA>ztsNVDo>>?iyj{Sc;hht<7pQc3nr#nyPS3lys@H5!e>j^
zmO|y1(R*twcY0o)^y)JE4AbsAHXAShoTo0hVUNvroemx5*=7sBz1#iq%(mPe+B=K|
zR(m$GH3Tl+@aF0C!ay&s8oz~R(PmzYpC8Qlo>#=2_BP6>s<yE3!*7!yCnkeJrGjgb
zDVvN=bkAJlbjG`wYn##2XG#T+dw<;bw?Ftq^3#JWPaK#zPM<lVp1-Hj)6nplsEhS!
zw@kNU^O7G1A3mQ?xwWm=<-;DEOCLQq-RQplT2)tRZ}nvM`gUICHh%SBudeFjSu94@
z<=i<{t5z^PJ9{|X{!hb71?O1PnX{NP7&<I}=hXZ?r7fYB^^@Ta>rIX{^Mh$mzfXR)
zL)raAQRviLj9NxQFD|v3{49O%m$1#Mp<DPS<IK72(Sb{%P1fy`vR@n9y^=vI#@^HI
zEUSrCx!a{JnLpm_d|r^{(P6ye?J~);H=lk}*`}0cR{m(Y>(4JwzB8NO?|D1FzHen{
zsKujqk)N)rtd+6(`FF*t6$g&*E_?W9W$?rU((e!Q7-+AR4!bIMP;9Htjk&u8=SoX8
z+|ECKj#s)ME>rW<U2E@TQ|Su~0ims3lZ0w^*4r=hnb}mKJmHPm<(&(+ahUPPRqeFQ
zW#|1ITdw$f_4<NBN5*e#_0z6hUE{#)5GvXyCZ7B@e}CeqEt$VOWO+{Hg|S6j-*Nj8
z<y)L&U6Hxw?2f->Zz8tyiHnLTSnt@pWqlgE6oZy_V9^(iYkxj*tIe-96S~QmyrwBf
z{6>OOtv9n@^T~DW%$qXu)O}~Zdesn}cW@i8bU~dX<F}jkrypEZa$t5SH9f^IcclIH
zw!}|QPI`!awa(Mqofz(0%j3qd`L!?O79;bf8FBjB%<J~c&Az_&@Xn=6Epp!K+zsz$
z<ekv<=-%Fn^v`KM()MzD7cD+%-1lbs@)MnRH|&<*ezb1di#rjl+$MW<W@(=;JD7Hz
zU&i4<oAigTyGm!g`@Z<fElIQ9D3KR#SD!vjd^t^5qUe;ShqqVLx>PfX-4ENA7A#h6
zohG`XGWCsvkG^&9wKOs3X^p<K1B#!|jo4SHoW{7}X7=sct!xVz9ZsD;UVVG};?>8S
z8zm(L_zWJX#Vo2d-9C3ilpD)BjrE#2?6!NqNp=1ISH5RQskYd`gBtF>3$HPmFf3TX
zG5_fkh4SlqJvN_xN|Lp^-@4{8oLy&bdf`h|tn7vg!*6S2I-|?qnbcp6DqeL}F*vGA
z_UF$ZKVGjlv@;g=G&KCMAwI>R)-1}ob<@s<Wz*j}=`mTUvp2eSZ%F-HHK+bZ!Q~Q0
zUbnS;84MP6GE3R{AARkWp7`qX!sjcOXX$oa?F!9qn7(`(%dF{XqA#w;yPtICf4J9o
zc0pk7T5JBTyVizrmMoU)>e^8AJ?}}*trkIHH3dDTLsn<D++n+Y>ff1DGv>D6cDQ`H
zEY~ekvf#AtjlF5Sd1ucZyMN(`?X#x9Y&I!|fVjT7H*Z#?A7)D_dDqk8*nH(+&I|q1
zdRxMN7ny`kDHmG$cFk_lnbUQhkJ^5JaC-gzV?R?<ZzR=S(On(JU$VGQT<p<VS?huq
zxAP63Y`H8TFS=BBL5<+SLpNj0mYRm}OihhTId{Ij=5g<hty!Y)&YU}D%P>P+b-Ulz
zpy`<_X7upb*r#%--7jR{rG1#SW7k}**I(|uGRlvglD6F^`@_XOCt`w`ZT^~c+5HvC
zE`N7~mqCXw`?A#)u|_tABS#WS?(eI3wrsY+mp`8cbFa;hirl{WOtkpYo4S{!qO2lT
zOxJgR`uY6h>33pc;=|hF&-8Ejy7Da3S`A&^a8>o>k8igx{(q#Cqx{pAQzto0_U_4(
zeKoI6s`}HX58IvF-|Tf;yLIJR=DRB<TnV{zXSKN4zmNSd=FjX5Sl*w!lW|L^{JS!5
z_T73+5i{$*%f6al!BqYE^T$8`{{Ap;Ff&r>jCjkM;hMN|<qlg*$<JbXF1NnC><D>h
z7PXi2a$JMhS!TwBr(Rd~eBvtVlR3Eh_xHz=41DZz*|k~u4%(lVE=h>{{%*(9N8K0S
zcWDb;-nsd&b&qeVxya63>z-$!p$BH?FM4yBe`0V%$9{2<2f14&{S(hywU?>ZY2(Hd
zUVg`Ho7oZ{G_k(0%@UDx5!!HFC~3;GB|H!KE~K3;w*Sil+RL#$oxe_7<G|t6)B+2e
z)Y}2O*=Ge^-(qs-&Q@)`nFfp}r^olT?5ln2eD{(=bYtaxyM&iPGB<Y!{&aP%u-E55
zzeuu8AcNOxX2UYm9rg8hgrvJW)8{lUnQ1I&Us3R3s_WvCi7o-RKc3p;)AlfSr_j;Y
z*B5WtBNPArc6&Y3jnwxUxAyskEy(13d0YJRu9AaGwwS2QE9RSYtlz!j`@46QS=O7f
z47s=)d8Gq(7pZdb%efS$s$P?`=`d$}Q}Vsz)-<bzYpc9dRF-{v(|L2Qb#g+G(4+Gf
z%y-@TTtcdL{faxhjNvxNlgGy!V|O?B&JOtc^}2?p-m#0Tv{Ifwe_VgOKUue7)dRb|
ze^)SPB)c+6++8hxuBXQ(Ek$Kga<bs$X>BLIl$5-@EgrbHdUD~53jUwBDp^*q>*IMi
zdpi4*CksApzaKDlcK57LMIkcweNDHbYUWRGU(fi5V>&x$zW*zRTl@AfR55tGy~ew|
z<aOIahF`yGI=Hx<cNn#{IDGpl@b3FzX=(1rX^mZ66;95ItV}ENGZ<dI4qWcj$e_je
zL_|E<+m&?_qYOiVL__PU<aVnodp<2tt|(c7*b659*Z$F-e%F>eDjOLX7#KWV{an^L
HB{Ts5wR(V`

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
deleted file mode 100644
index 7fd9e4e6d69f3120428d1d778846d495cea1a989..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18537
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3Fz|G745^rtlpw*nn1P*v
zfq~)we<{|*4C+IOdE|U2Z!?1cv!lKG#DB9{InVitat0sNyU+JyzGR)99iR84jLX+}
zR6RFvE@<tQWU8H~rr?++zT0=x?Yr;PGH1U&Rrzh-?{9bfYZyVkzjMivtHpq)_2H2v
zx=Yt_-TU9rbgflmYvs&V_ZdeI*!&Z=*=K)b;i=CuL6;;<-OfgOPYhZXRT}BEcGl-9
zmqeKMcW%(?oUmJXCr9f87n3%|b8pU6q%wTlB*DD*vCYQ!+tzK%&EP%W`1(-o;__;_
zEo__P>p$ML1Bd&SOSW7s1_BNb3(wZKDMZ|?_#fR_sueh?bHYsBZ4nXQn9BOY>-&!<
zp3!EJZ7TFTp0UhRzdJVE{JdD;(G`NBURp~T1^2MOzZoUzF3HB2z`*#U?ZkY}1NQ$P
zFE&+7QqpYoei-$;wm7?Ho^ATQZTsHODc-NA_UQ48t%o-6*aP;v)TGYl{{G(S)0^Ad
zoBR8n9UFrk8@-*Hf*l&>&YvIa(-Z2^F|)tFarSR};mh`J$Nozg_D<#PtoyU+|Aio)
zO~?NKU-;<%zrVl#{{H{{{r`QnzrX)9-lFsK%gg(3Z`bR`|11ChZ~yoA|Ns8^`Tpzc
z|NHBBODu#yi4m0Oj5?d=&yNk5;O)>59W%$<yF1*k&pl*HuuDsD&?H}vj)~o`?YX%P
zI~gYTs_rXPnO?ubm3x*^1yhr_)BguY&%D3?e_#Fo?@v$vm$&~{`}^Dbo16bH_y51Q
z>g)Z<>i_L)e*AxQ)PY@(51Ov8bT-eP9U3tsI%0-@&?I;F_V~z|;Z9A_UR}{HEuGWF
zFBvr`MQ}EnvI=MzZ|FL(Sf>6_!*Yf#EliJ-Jgxr!|NsB*@B97o|Lgz$`u_6r{@>sK
zzrXwY{VCD5@bwb+_ND&#khp+ZmJ@1MNhkBAIa4MGOqd{W;hNZ`Yhn{7KAbw`iI{Nm
z!>6eUiH_&LTWIkeI(qE?{~1&Hik7)wZVHo2Sew0}TY>YR>)Pgje}A*TWc>U4|NsAw
zkMp?q8N9iF{=*Bw!?VqK<n4|A{5tyKzy0C=|Cz<aA@-e_wvo@lL4dWr&FxW>z}Kmd
z{%iOCzt9zQe%&LXox1~q-{*ZzQ-AMX5E6Q?^w_N`nc0aNx3|6ceP@QrX_1N|XMx-;
z50ol<{{MTnVX~W4$GSzwwB%It^#mP4ZLTnV-!XBOO6Tp62R1j$mvvUAMcy#GCMwwX
znAuAYoP?!*965OYyq>zchQ5AYqT<ShiVEt-wG|H@JbwJdnKKQ$Cme|LGcZ~d%Kl$c
za^sT<3ueog39pwYT+{p?(0gcBl|U)iwnr?9xAxZ;TwZXpqmgkU)4#vTJL=~@Ji^(Q
zd|V=KU&@Zs?uVDv+v4|2{QI*o{tCp)QX7sOJaR-s{rK`lO1XJ@$w^9T+NZS@51u@H
z_{5nWeNzJ?ql9ej2kXN=Y%w-i&A%q=!fi2&NqjbQ4|a>Ov3=RnI;m%Zv=R@`{|B1P
zakKyYKmXw|`{BR8d3Ki@-`RP%;I$ZtV_$Dn^WU%E0+e9?|CeewqNsCb;S!Z)i<RF$
zQd+;}jJoE@lgE!AI&_FZGqx~9?8a)L8B9sDXGVw~@DeX6?AdW-y2XUliCTOWzYhB(
z{(or77bMBdt5k8Sq4@v*^on0o9v)(C;})05yPITD-uLixdfR+^$$x(rC;BjggLYB;
zqlA<c0nw*du7;Kr1Z>?fAt>aDm`F-`a<hq=M1Zn%lSILChBX)Y`!hW>{x`eZtl8z*
z9U@}7!<(a`vLWnJg1^j(T)sQ8(iMS?cYd^TI58Fli_8oA_dor|hsKBBa@)GaCHrKP
zLE-zexJ^GkLo^i<xQl`x2?#&lx;3=4__DdF=%gu6o;*oON^Crlpeem!s+fi$m$x=g
z#A*q*WBXSG@l+^@x-&NK&uLigt|?@2>3;l`i9%1Fv`Sbx>lQ6|%5Y7I#j^9yzq18j
zwGQ9i&C||jILFR+wVeUPi<OT80tL5hxl&Mib=%gdkDjL|rndT|DvI;)+z?`kn%lH@
zPJ)Ap*8hb?f)_g%@9(mm^-)FSk8JpoHiI-}=67;QUVZ<0cp_Gbrm*b^`1!H<;YH;(
z?Qn@c`E-kl2@gLeYw@3FhFFyN=)seeuiwr4It+|WL<B`r(vp%AT5h$4*%%}o(JoRs
znEFtYGp^`?md%CL{2sB*fsO|zE3A9cWWc*~*TGvBM_;fl4%fcXcKV-UdC5c(VLurj
z56>sB!`rT}m(07H6tSh_;k($j=jSE&)h_nqfh3qkb&nF#(xyy*zI|ih=PxGZrBhSW
z(-V@Kz8FoI%=B#=tIi|I3y<0u_ul--@5?3krR5Z-j#`45hoRN#8HN38<aA{hh)PFf
zE&c!h|3bcwg-a~|WHQa{zVrU?*Y1aRt=npUOKi(e2PK@d(poO65MQP}N=Qt6^5{`P
zK>#<8$=1!m6DK6NdJAvpaG$~(QLV8fm|4lIfHzMm++@X7uA-QvX#zW<R3fB>Up(b<
z%WPTNaPY;}MJpX&KQ=HBm}v1J#^TrC*$<z3w{6XqG|Nw`_|)?7Fnd?qIcO^J16d{{
z^yt!+DIdO?e*bV`!qlgUsja)_G&V9?`YNc3yJ&Vzsl8aFzRY6Up6)3<8(eJKbc((m
zopoh{it?5~zdc|1u03hGx2>TxY2y^12}c}jP9-?#x`NW@=k|v`3fr!(meh+&*-_4K
z#pBNgwbbd+gNKRNFHI^ay!i2h!L`f6X{o6#OV}c;8g|WO+^Xf+RhrVWl%Zkr4q?W$
z&Zhk~4#r|j7dEV9%T5*!KXSPD(2Lm$rxbHSIipr7X%#-m>;`51H<E|{{pGPPGniw2
z{KG$6%iK_?WhR9lo}3oe*}r~BL`G<IcWrXyZ50#cS@D&F*-gmNQ$S*~dO@MiPr2hK
zL`^nbl;YjdwsV4)xX!Z#ZQTo31big_C+2dmS)B4eK$zXAnX9w8vGL*W^0wXOl3{Ba
z&OQJ8$1~~)B&jIb7jD?NF(b>%-DSq^-LEw?IX7-JG&DHyTlvJvCPuNjJQ<lxuVXm3
z{bKe1+Njx~%2jBd_A%S#<hvWIM9(Z@)xCH?m~q_{#;tr?r#YHfUr>^h`!2wytbDFf
zV8<`}dI`Ilgoyp~5?_b4tqzy8t5r%25Cx|fCH2A$8w@->K14@l9XS#)Yu2gHA2%8r
z8X6cW^*F5yNYHF(Y-D_-@kW*9-Z_qcuO7%xaywJR6wc*vxk<A%Ec?%K)zb|Z_cTlw
z^C;|(3o%%*-8jIDZ;P2`)UN|teeHk#&wqHFzwK~4DDha-%>kt(dHas83`poEDH<jk
zeE6{8)vHsLRZ>5HNv&AbrJ&Be<jY5S%~}Ux_AKF+#>PfYF%zE+I})2z6lPfV7`!cb
zSMTQLHiN_HWQR%9p$08(L;h|tm9H5LzN&Xh)^th!`|o~EJVR|kW=hNR(v;Ny|NkF1
zV6=9)#;0i+5Zk23#4aPl<LAedl*DuC61cG6fBwTy<HXkxCn@O`dU$-eaXb6>&zTXC
zn%&)>4UCKo3>2=beF$Ol7vtH`@`-<%)Pq*WF0m^a4olc(SQJG#d%C%?g$TAt^*lM+
zAa-9uWnIXj2Cde2Y%1rK4*dVmv%#h3s*lP313Uj09@SIvJ+SnHI(J2B^Rn)lGadc=
z9UU4RothF4vn^UB0d<2};f9Tdr%!xfXTSa9$BXQ&uFoGf7#bNV9%pg;<|utN!D(&4
zj0qC|6AI5aq$ot4OLoz!yO1n$HEqquvmMr~3fc({svlPz&tfY3{{R2^j#&*GmNW0*
zC`x9U&aq6-ry<nKukYaP*^VJo5-%!&%F26t4{vv1fg}g5!h*t$85tffZdSj3zV!6?
zuxXQ#u@NW;4jOArYZAS9#lcTlpkbzhfop(~Sk_5Ke^m}vP;LnN+cZ^0^V-B*N8|Pd
zVr*<zUi|ql@9o*Vg!Q3&#U=+2;n?L5G}9y`C0kl0-@K7<agk8-O|mF%dw7_A&Qnk)
z<o|!AnCpcF8)nQ}rJ`a{QJwwy(<Wmhr%No#%F4o5tlZpkME}>bO_AeT_mB13yc6~^
z$?+$G{wJ*D_5RM-9_Lg2KlyUg-m9IyALAe8IEuTxeX(e3?!&jiE{_&WICAR8&&dy8
z2De=cYjbXwu&X;F_Z1q{b1e4k^_?{Nt*q?Sn>Qm9GMVPtJ?NC($;xo@=KqOFe6N38
zwU0<qJ|wu7xydA{aS>0s&w-mfM~(`6da_^En4-sTc3i;6uwL?l^{yA4%`+4O*LX7a
zE3o|idHBOiVdqfM!v~vr^6w{gRza<sV6kXX&ZNn?zP?W*BW5Ioata4+N@$a43fXUP
zfQkFufj8Ys{mF+O%h^Ug;LBj+(NuaYt$XUDyR@OF!2ZA^YJ3*rrCXZ0Tv^v0VR5hj
zZ?vcK@P+&5H&mM+-kkno@d}6~hi*Q5@Gy~|)9~!Mve0nx<kZ&d${*Ooh38FT4%`2y
z`KaoW{;r!2jixdI+O9baYekvZ|8HD%jB&#3?FGl`d5ljo{5SL{Vwfl!RkGXAO0B@r
z$4n|AElof?-NQ5S!GnYwJ10MU=se|{COE+ynq)C!hD2&A>z}{8#l^Nqj_!<%keGO<
z_s*e&maVD}b5i90HGeJoe>jM(DEYI!kE)Zv%B!9BH5-pHUYqu^o`*G~O<+qaAH&VW
zj)DfBx&$|yYfBi|S*rI0WCu*1*u~X6UHtI(_dLJ98{1SidB{UN=;SYR=1kA3RZ4Yr
zZk(LwgoI+woas<v%>2NxR<LVYjKWbJVdlf}OYB&S9y!RRrXP^z{r@nkKt(F4N}lDc
zdInP{gYu;Ee~Y=2P73|tW;PN0b641B>I}8nvy5tfAN%lD`f$HI&xf}&;g)^+<aF=e
zvBbnMF0Pmxt9V*!4cKCMg?s+;hZ!vRA!#hDkeTjqsas~&haJbIDpf?E9Z?aSw4{Um
z#L1=>t)3Z-JWnEyuVM{m{Ir5|lUQ_gn|-}R-Jb-D0)|`lkXD|Pr_7>7Hg$E&7A{ni
zl4AOC_jtpV;?@rjRMr{Vg{+O>uC&o?kZ}C-?7{U8`3q~i4)T4tqR%7Y%DRkWo*BDQ
zLR#8}O-<9})%}fXejR~0I;qjqfQzfh#U-MpRWc*XDb45KVWr>pRbB}R4yVpe_{Y)m
zmw#(T!*t0E)?@V{Z+TiIB6hMcuJmjv(`?|HDjxOAp)Y?;b8<%WDwg(m#fI+ggLmeD
zS{&l~5_<7z5IdUG_~hjH?Cl?Sa0r{43WkNTRcz*)X7u19XX;`1j)}k4W!U{c-XI~J
z>X6l#q-*dmxnr@af(e`4-V>X6ZY=r0d@Iy&$+W%~4B^%y;nht3P6tmlFc=!LRadw9
z&zHEjHyL6}Qer}afWU)ITS7cNHUtI>KY7x!gp)1emqVFMn##fIRrP`1;uVEW4lBI(
z2ZTK0Tl}Ag=YReGf4_f!muG9M&&lyQbLPZTfmzl6l8xsxKJ)X5^h)lw`SAVy|M>m?
z>i_?n-}}~{o&8CKAY0B)M(zWb6;cm`{hna;nTfHeh{w!~=lpzQs5=A&61Hy(^YYtv
z^{Qw}a&nV8caPFT`J8P$JIhb8<U1`|_q-vyaXHVLkL|`~_5c5Ue|Pt~b$Oh9&5wsi
zyU)+J|NrCncX?~;%^M8bySf^7?P}=Z;de`T$QAY`lds$-Vd_zxT^nZg$U3yG){p&F
z`|Qm6+uQ%|sr>x?dp=uO7+d1Q|EEs%xViPTwe`frs7Q2gt!up98o4lGn%E4jhYd4l
zI(l~>yf_h(oRThHnez4H#qAp>CM72~xT+NW7q~aiL9oU0qkPV}9>%RR8nvC*#awXu
zBR?r=O30^APp2mzmwR)w`1zHU>f7`7{{8gy`u_U=k`fX(Zq96Nu~@RS($usyCwHl`
z(iIEKnGda>7aZ5P=Agdm-oa$4Fw=+w-{1fDpZ{-X#>MT$&+GQq{Ct17z5YY3YEBN%
zp+h~NKJ_eH*5m2vbL$q*hSr-cj0e9oo=H8BQTafqk%5noEjSq3tV_zt=jY(PI(f>I
z2Hsxf^Q#S&dZ!yNERCG_YaNe7b3(_AW7{5yUspN0e}Db|xII5gPEPt9z3tA9^7nV|
z@4w+Zr@PxVYS!@!0c(~V@dyrXm_567;zxa*Q~?>a|NT~pZKrcjXmx&MvikY|KPY+Z
zt^1pPtVe%!c>Uh;_vg>g*Vt-ZWb*lgK}Lqbv**T@RYn|~f0b9p9k$xqrrB3o#rk@N
z)f{ecd6#5UxhcfEdwwTpu2pa6#ECN-8yPiI4HoiEOZJq^cqx9{De1=<j%f=2zP)|^
z^Rxe|)#dZ-{yw?A{r(T<mWdM^XU`4|oVb43u_I@F!yFn!qvs^LnawM42)gswkjty?
z@2~eSFW(mxecvbZ^XIRx_J8grCL|;zB?*Y8zIc)F<y)G7sH$S&fya|7*yeP5pWR>2
z^Z!3AeFla;O-yP!Kg-BaX_?)>L(-BP)-cRpmVUrYsZXdl!<02^gHpG1@yUMq{r+<*
zBEq-7esQtgh3)us{r&p!b=IY?o|@<HJ9=zYVWFY1v2nr`S(evQY&_c+zC4s?-xjD5
zR`d7Q{~H_SC#%=n)cknf&fgNu#>{+pe?5<WyirK#-{%h!K731i@Kn`JSR|p$Uqa$S
zw%D06Hs)=!SZ_K&b5NtXX+mAI!9T^XQIZ!XF>Tve%yuQKl~Fm&U@`~S<2N^t2QQbi
zFMK4(>3HbCfddB$UR*G)s|zzU<SQ?ijEfV?+1UM(?aY)@4XY+Gddkdfw125!GSB8`
z^4C}EXPZy1Id<yvbDrX71|3~S78V8u#*>$c7f<PFWaJiq?ZZ3Ei$fX`eTfb{Q3eWA
z<pokFsPwPMo)E;DG)r!$pQG%rBU0KCmyW-_9&eVvZ(r_h-k4sVKM#&{PCs*|=hiKz
zO`BqDY|dCH@t-y1S-}&d@-K*G>4)Zq`2GL(SADI1du#n{vxz(BR{pPJi;iY9HD!y8
z6e>w{)1A|;*?lcNG2uYa90LPThnM01|E6_+(o&A>QWeN8cqGqU@{m>P*Xa{K#J8C|
z)IS!)wx(NJpy&Vp|M@qIa&K>wwy&E|dFSfu>-Kqfer%}xd|2IopOX`V+oP451uGK|
z1RQ8kalP^6`}_a7H$HrQa#CJDep02(q5btd(c27ScDbBeY1`AIp}ahUXH8Rqt^nI6
z3rHw9=_dStctg5F^slS8p24wrm6lj;qhDWM{=T@FKkrV3YmtXS<MZ?Tbz>@izr3tJ
zA^1kY5t)OFni!pJnfdn4cwo<b7@UhLKBiw>q+k9npk&?cr>A+!-%qNP<1Ug(?yc})
z3lo-*fVO9wObiUWYrjlsELp&yBKzm)RF?@Qe~(5_aM-EtZ?~uJ@74GBm!u1;eE9fy
z{q61caeIILJJ@`_!kdFlCfA>B$)Pv$W}1IE`MT!W{H%R+<a@Vxecd0AiRE?wQY`A{
zJ$xDLbw(keXu--OvtpnL?NG#o@A8Z58pEwlq%s#}eiV<6ZQi=%;3T_q^XtDoIqBZj
z#K^WT{(r%LyOho6>T@pkOv@+;)tIsA#3P;^rG|I*{{DSu=ll@MD}O$vemb9UQ1oVn
zm*5&Th~Z5p9+C;VzGu|bPig5d@#7NqQ7#H+UB}~Z_>bT1UwXoi|Ig3wZ|A?izs|RH
zamK&Dzv~|#JI>B;XKc`s+#P!+PH2|(#BN@;B^}3Sosihjb!X4d-={=h%_*;cEygS?
z49ZYGJ}sijOlxYRw$$m&O4`A+!wq6{lB3_ho}Ld=7}w`LxU>I6CTp37bd3Oy_<^WO
zWl*}Ee}Dh~e}AnT74?%2?5_uni2izXv^;04%C#uA?5_tKqPem&8z((`cmMz2ABLrV
z6aW61`S371Xi&|<;>5+ioAQm!XFl@BoMeS05l20(2?8t!es8J2=p}fAE%1wL(u_VA
z)r*IJU0(kF|B8|+&;I}Ck+=I(_5YvxyG@N1E&<$|L<^!N=Gp%K{8G4d&*76NKm506
z<~~_Hb0Nd)9x3K+ElpcrbAzfVP##d{Qws_HXJoWUNAkD?OY(tfMiXvkgqo(gexBd>
z@cw-JdWG-lfvh#ZzWlzsd%ncS_G1dm`V@@5E|B=~^z{AX{r_wJc1`3x$>ik3V_!F6
zzJ-KDff(lu=F}ZsVvx{k3fG=+vG7NNICD_vp@wM|6TVG7%=-WEj)I0~dT(0Ug@u`|
zts|#unH@NB;=||X@rxEuV|4EpUE6YCvBb^U|L^br@13a5D#^h)(Z;EZJ3)-6LQASc
z#|7m2|NoO3eRuzF&o@m-`O_VE^niNggwy<NJzT8*CmVf~oTg2m9vU*GyV0v*_UxIR
zoe!U%-~Z-KkHk^7YgLTzY;2x#ZEBV*koeu-_z*N!{qOJZ^QTVzc+M}%#jmN!CMf8|
z+c-t|$7CjE<}y=6>E8I0S7=rb56^!^SB@a=W<kxJ3X?d3Ubfts?zmB;Gv&aE-2aji
z|F}6etXjo2;c1~xLw9%U)Tu9TZ@>TMO+me(Kv(seT>mX$+<I>{PTAG}JAUev$J#gR
z)}2#Obd+7(vBt~b_6BIU>0E8>sZZ`(kSza9DJg_G*F^Sp#!Gv)w)#7F&iwkdC@1F%
zXCNbEn55*n`2GJD9J_DNHfQ25dF6Ni6TO9hOy*0f@d%wdbI&|=H3Oq00ii`rmKm8%
zjJGY=uRSzkV}sW9PV#o{vUi)7NUsU}C!W0FkNmL}me*Jsjdq+n@gv^fKO#PU3fI8{
z2M(M*y=d`a_T4L56RIECXEizW=O|6$^Llyc^yvlb&VBl<F8WHu&#!HP0-K!NDc>fh
zCs!JHE}Y`u5?KxLOH!k`>&2+K9xv>fw{$YDO!{~D&4D$(8@5?qkjOY{%FNCloUq{2
z;dc4>y^CH538tmFm6fR}E7#@arCsrGO<o)-z$3vW@nO{w6Jake4}()D41(_&K7XFx
z(WR7Y%U;)TC--2?md5OT|NhrQ{gvdTt#yHI=l}ozyQS8gXgwHzMW;1PD`A>aePPDW
z=g-vAQ&l|^1w1?pGqOH?`e;~OV6f=eeftpaLQ&}*E*FkS7O2eq-_+D2B-C^4*cuTt
z>z*D7CudC_o)U)jb{}|FCxb?c82<lHa_ZJS!FuMy6Q=bs3ZFk2=%sbDgwHx~Rmri_
zI5zg+9vj=TvM#1juP$XBpIOb#g4?uM{pXAM&t~rDC=zm<{P*~Q0|!q0i1+Qid9<~6
z(rRuC3yDjYBy@B&IXNdWuA90R5_(Nv0tGj=n*HvQS!|$rCX96+OJih0Lgu1P8x5^2
zJeDrc=H%Eg@wsheq@S`fqq1_Hot;>ZhKn};zeE?~b)T5SE1O0CrY8JYzwB5_%8{I$
zGd=lkjOGdH=^b5399&ApC+37S#LP($)8*mefmoR26!olm=H~y8G}JFh>|psAVpwoY
zzT)$?ZPzr+!g_l?HFtJ8dUtPM;W2aWTtQ{o2Tl=EygM#)%}Zk2x$Ecg10U|6>atX8
zo7=M@DLtKoFYU#Pgal>dW&wtbi~VdHwn0bKlbraBmQ=CqU6GLThyC@tN9#>LoJd;6
z|3u@Dw$h=ns8!J+OAa4&bo!N$o?cWUa`sHflxb<tA0!+IG)M^3h!Ezw_KA73f}@(`
zg`l1t7K@viz5V#C)zvu-kDog?Vcxlb`11;0bL*tV8Y0Udyj<n51M0IsAC(z1i$o7K
zoZo!l*k!@hE)&_$U1V4{>5F}kHD9E_OqT`n_W$(aYL+j4d}(iW%X_`Y4->a-7WVX=
z`u(F|^V=zZ*t?C-%s6&`&jHnrt*wpI>;hasRm;am^DT?(bYp-0zO4Sn)XGReBqbv!
z#ls`v!NY?$&6dbKxYf~+&J1n1{`sgpud$KQ-O9m_uV=$8j$FG|#>Lv*k`fWZ>}kIm
zjQ;)qU;pyb?##=1I<Zy%Z*SkfgDE_0nz4q5k!h%3>|E(-M*oDpgexv_G55JNsMkL5
z(yssi=jZ<?C-3)4|M!{w?fJdE|971G_2faqhfhfwT1gwW9b4DX#qi%xeGRN){_|0p
zEpsXxvruPbl){lf#)DV-Ol1`|Z(K6<{{H{>c6==S_~`fZb9;B!{yx1r+`jJ54^T=y
zb7sY|V~<~Iipl6?W@>i4&Ei}0l3{78L&>j(1^aK9JThcI_4n)R|7T~{o9F!4oqBrz
z>+6#HY8uZS;^r{W&@jlzHa>FPc*cx?Vh?6}1vN*7&Co*p&qw81jH<B-YuzVw^QJV|
zM<$&8#3FpWKIPqf`~P-zf4)9EeBQqP-`}5~`&(NbgC@<LJzMf;UXoKPv!0n@D$l`F
z9R9b(4*vcB|Niam|K;sce*Bx;ICEy><jIZ^GaO^)gys6jG|V{Ccm`UZ{`siPWcKA`
z!{Qqa0sjrtk{%Y9zQ{Z9*d|0MT+8wCq0c7F+dPg=wfL}vzoOD1rPA3aD?!2OAOmv~
zCzG!Lq)Ptz|G#{YF_ZYo1{2lO4T^kQrKV5bp!uU;vCp*D!NtIuwV+VRo6qQ!P?D5P
zMwY~ckV88^yvljN_>9TXMDNoR8D?h4Oh{7W{432{-t+EEOKh3X(0TF30Ut3I@!ttX
z1xtC>ys4K6Vv&}~sGOc4^H|gPg9`T_H(s%Z{qMUpp+5Qf|G$3L$;Ley%&Yh^pFgfw
z`lS}}=Fl=B<4A?#iQe3g1P@BF*Sk7z@NSxUYeIir!hvNu1&mFcOp=hy(DdHs*cum!
z0B@!k?WzwuS!Q!wQ)21u4YN^LF43gP94vWIOygR_@x}8B>{+h{H;X^;*M?|JYHVfw
z*xAgumE&iB<c$X&VkaK`H$1X&n*5S!GCz{%F3|bdE7bHd`NJ}vN8J1nElw(Z|Nl!q
z_$#=id#S|4QlVQN2RZ~bB6daybEVB}3>DA~k~DBBaJ6C9?B$6OJkOB);Qc$kvW7Vv
zAC`cI#~J?rKNMADV3gp`b;R6ggVtdm;T<j#29G+IG_^$Sl~~hNC@S5&S4+{NtA8b^
z5Cb(0<_L9`M~bggy!c-tM<m)nV#g8#VU|rIb3XDrGtT>=tkErYkjLdqlELX2^()kP
z?oDB2JR%}-LN*HG;H1X;N7~HTx*~G~%+<fhn~P^_IeN3qXo@(^!`4%gs-Pr0LEWjQ
zx@-gU8g386>&p(`gy?Gu*XEg#8?@x<qKwR^_I=aX*DYx{c5GII!DaTehmE3!0uyy4
zIyCq<6mfd-9{Z;i){xE5{mhTm0@5owG}Xdj_B0-z7jp%-*l{znO=mnj;lD-7|77Kl
z;?gXF*B-I>PFT{?q<xns8e*MO-h@MoWh52&TMZLD79Q18c4IuyFuT`uVZt(gwkt0j
zA3ySxJaSqn{xr|tRV=~}vMLicL2_MF{*r(E;;y+d0+%lwka9bsBDlu&M7>PRnIj+h
zmFIof?)1ULw?kgu*{uEqgVJs-*gS#bSM_Nd^>b3$c3s`!x<O6QLOfisDY9<CQQLJ&
zjf~1xotl0pG(6hm4$P@i<$yXdyIA4%fi>~}lsl7s1$Icu%xGm#YjSw}NK5yFI^Q1-
zf29P4UExMg7@t)+rf8;^fwP!XcJTuL)rZoHBqbjnUKVkC)ee^pYOFqAXDnr9&PqvG
z#?!R)L_N>c9j|8nKf=3>@8^`pty>Q4su5s<+9|Er>S27S%A)7PY2OLAlp>lrbsv2c
zk>i*OSv_EHY^;1~<CPskEX;i?4o&7OP!(YdIm#k@<bX<d{eOvne-eNEpWoQqdnxpI
z#U&O!M-E7Sb($TWn4n;D>#O|AUkBp*A9yRhzxUU+;KAvm-S_wG{dwlhj)2mIsvK-R
z6^DddfBe?`arvNvmhQVdi8pplb#!PrczJSGy5R$+9YUa0LI3|Ztzk)auv*j3n9a}X
zDW>b2P?o_7nvVYQ;bD33<<H*J>o#mQ&dA*O>4Sly=I{Pha=N|-3P)8pDI82`tXB}-
z{6Ny9;EjZHxOnp<q2`&A%{wfvZqpISX;*^g(?daP1vchL7#A*B%FM<iDDn>E`Tze<
zpFDZu)F}m}Lq`sM;&<j<`IOz}(ZwSlr*rKPI?K@f|Gy*;ujIYG$v^(jKeJfPkXI#0
z09s%;tqd#BXK^tUR%@N9Fk$Ea!U;S)JpX6Tkno%H<Ja5U_N=UCdU|Ypeg8K+{U0IB
z!ko25(fGvd-ouO2HZ<JdFUi9LT0(GRbGu`3al##FMBPsPFMnI}>i-S0A--qqs$!y7
z|F^CE_4?f0`PbL|oiZt@tni`B)z+1QuO(mAPdM`b@SK+Ve*0^G_nw$1rPDBRqGRC1
zgC{y3{w!|0y<JkyPNi6cySUKOh}l;a+D%MKa(J`mf4|JNi1TH?{>$rL6llJ1uKw5e
z&(H6#4&Q%%_V<act%h1%?oLgD&bGG#9cOHs5%$1;&5z4p6uiU=6m-rcq#svMI<z5A
z_HZjV&;5N9XWA(+gx8DKJ3;bM(joaH|NlD+ANl`3gO$_sC37OE`qq#Ab?@@$e{XN^
zkBghb$CvZ}-@>JAe>8bsCmKae=3d9r62`(8_u%#Eg#7a#p79<&(#aFP&fw2q@4#OT
z%v+RL`k_PKNrx;ZJhl(o^rJf@VZs#mi&y?Ps&xPV|NrgX-&^<o{kyZ+J~T8kDQU-(
zCo^Wu_>gg==ly?=O((8$&D-3<R;}8^y<xNQk|hQbl7@5a&tJHIzTk_-;rI7dmhew9
zQ%bn|9x~C?<iF?ORl`H!^#}5G+|Af8r|{@iMSlXP6yrPl{#KuzwO-o1Zhzt9?Hde&
zqolUambSLGzBWx^7DrLcKlevMIYmitr}9W1ICUx^=gfpjhb~+@wxLw}@b-M3$VeVD
zGZh&X_YFA*;-PKXq{dLmj>IC9sXQ;Nm;PI<rFg;i=l}oz`{n<Cd;i}s|K4?3>pBO&
z`hS0ZzMpBl-^h5U!IorGuI&1j#wjgEM<%h#eE4j<r~dqhN34ghuji@$Ze&w?YQn^W
z51t=a(DI5BpT6f8sMPuY|InPNM>BcWtp6<^s(S9o$M5g|>&O3_YyCa@VAJ|`zP~je
zAHDCF4_LZp4Nq2<&;Gi<smX_T&1STGchO~o(vBn(v$UBeM-IHd&m(VV;Be^eZS%I`
z=MtMXfoAzVyplF-Nf8iIN-W?B4;Jy?1x*Ku4!iuN%8sxq|JmTEa;*MJ;OGAplUSJ#
zgSxQycYiPc`YJ#Bdfon_r{=d9MVXj-^yB^{Cm&AbU|DcgZAR<ydX>~2L1!+oy}xI8
zXWv-{%f`Ou=EnB+hi9eR-rkms+n2(@o$TS4_Tb5pA{VQV+-E;tw}$4R$T!TMEdT#I
zm!C8;FgTDc(~)UlHqWT};pgY|*Vp~Mx98{aHIec6_qE2&IM`9u^nboR&#PBH@%#Th
zewKD1O@ukiD&c_G>mL^l7-eL5s=qnhw68N@VF5MRZ0gPw{4j8ilsvr1l}AvJhmVhE
z)hZq)rWaM5$82XMndriM#ZD$36Wu)iD_@tIkuJaZnm*5mmaosx|1W?4Z*A1q+QV({
zpP&2t@596W9UP0Nh<u)H{(qjWb4XlI56`b(XC_VRnKLItli{w<p0&IcOO0(NvobRu
z{{Nq+{JoKfxAC4@Zqdr$(-v-;^ze5%TX1k&>S>8}@#!;6Iv*b4WD5^xla^*<Wo6UT
zoAdnO#EWMkk#Q&5mb=-3r}jos(}Kfq^>*#Jt7O5k?q8Cn>j!a*)Y8%~>>9Vc#ko!2
zUf7v`n*F5Er+X9b%#__{qr@>M`iW{(>ipD70omI9k-LRt3*TJb>NRQhwYb}PF5<Ql
zEHl)^+isLU|NY|u-^0S|D-wh!ehyDKW|1_{KR8>1HBE%I$1!x_>jd34M&D)~=Xr~D
z5_v9(1gfmw8$bWQg|<JWa*KJ)njh-$NbX*7<5km)%txR|0ZoH^dh-5gH)xGs>5mWV
zkN5xg@-S#^-^|Xw`Sb~ckDrb8qP{%7vGKl>Q_uT*ieKvfo)Qo^aOlJj`Ni_xGk<el
zQ{y?jW{U&c{r$#&{+<2sPIk)gce2eJ4Ywpbc#vRVk`fS{{NZbA#O`T{*Y(<@%|Y`=
z5xb{3#>{baYjX^p>gd|)=-PVl#tg>@iKjvGXB-V;I3exKJ9F1^H77{4&hN}HsBsnK
znYQ)rfdd+B`UhY1N~xa^5H!*0x?!sMFE??AyjjHkm-@@+d}x_o{H|J?|LwNfuTQ@g
z-;>OKVO#dBbIUK6*!jB}nx9RQ2ohMaZ@o<arc}>OJ|e!g`>pz9T=Ok<-S=MAm1@Xy
z`r5Hyo)R@IP5CE(r1vVPb_$fpHY^Ibe&d@kgGujU#>+=aJ%t|4(0N?6aMg+87KiJc
z&NlqF|2_OV)iMtp!gppFay2^$xbAl|%Vgo+C~)M`A?-JsHm$wWuejMXwDvxD*L60E
zBdjyIt3$=BsKLkfu8rNEhxuB&zwawEtW-~)cYUAwQZdC9xzl&mg<Os0*cJD%GEF3Q
z)0=53XTJL$`><=ywIesOZo386OgJd9WQoBMlZmOaQJv344vJg+tM5=yJyxj9>>!}f
z*Pq?Sc%Ak61DQ;&r4ycKNR`O8F-9%cd1xUmlV1DxFH1EOc-8AHTdw8+ftKUT9R(aW
zh>IMVuN)Ck5!auzoLwNgQq13_^T=J3PEVdmcG0}iXQur&JhS*s@s1tEFD!~~=cQKW
z&RYB4Q!}h<S?6@tzQs>>28vAAF}@VCJ=gd8%a`YtopdUQy+6rikMZ$cN`0=^110MA
z_Zt^_SpTb^;mD#H7`pzc&EfBkC9`rP#F`k^btfJ$aPZP#J?|`1Yw`Y(df6TD;?yUx
zyE$6|1m@hWZTu+Q!{fqZVjys2-d4ViY}^|d3<cjmS=)I+R!i)`6$73#>}loN&v`<o
zGQ8cDAFJ>CIkD}1ut34ASvr$f8M!JJ)|lUC+I()aws+?3tDg&w#4F8E>s69soBq7U
zm32?BR-Ge@;FrJmlPe{XWaQ<ldK$NG5c6bWSXCYSJz~v+E<H#~xoO^;&+;$#7~EKO
z5H!O2_~Fm|j1wxlXZ2K$sHh#&&^xE0cTPq7^pW%DIo3NWgegQHKXKy2+uQYJ-~Oxn
z|9kkrA+z`2-_r`3Cl8$XF@NSy`<zC8=01~y227J#Wk3brcfC`QQ_h^>xpnJ=GM@!w
zk&Ln7L^I$1IWCY=w<+crv%b~?F=HcxKhGZ}UAUGuWlGA!2MJG}rapO`oRHMiUy!g(
zCQxiL7q{`9-QUAqS{~js7TNB=^YGcf<7du%c*?EJ&%e&?SO?qwI>SA6XFt3WJ-j+R
z05o|U7|52F2O15&v2pJ1e#xg5Ts%p~j{p0g4z*cMf<3eSkh`nmQVwnH<_3nv3lx}{
znO&wa>T#Ph-ZV)$_WJt&B}*me+5Od5Zho=S*uY`>?f-vHe)#>p-ptHq?v06jar@IO
z3Yi2e-@M_`(&G95KiOZB$HMHQvB3g$rWmdbUXVhx>D;M9(vlw*`v3T+`h$c0m4Mcv
zf{xo0j@JMGcmC|z4?jNIGctmf+|8IN`QZEIn%~C`9Q*eFv^ZN?8E9fXBJzK-aq*I7
z^L&XuxzrzDxdjt#Y(TSQD^@+a(zLURdF8|gr6&7Y1#Qrf-T(iGV%{{{^tbM8`XK*i
zO`*wAjZ+bk|MypYoj!f~!*6f@uUW%$!d$$t&@g8Au>)t%e)#nCzkU6L_lsp%13;@q
z54ZF5%Ng3#^$7g=+<x#t!^8XbOrng-I5sJKV&Ryj6P66|lGFZU-1DRqcp`!t?(c0{
z_Q1Z#t7-Yv`v3n<o-nYf|L5=Y=)d@tZo|4i2@|HLSA2bC&wu{^|NoM8|JAhixbgk?
z-v98rew+V%3BNfB4*R?V{Q4XN1mf+*_zZ%X4=&SKw~8HH_#`z=6*%#fMQPUm|Nq&&
zB{Mo7$X|M@He>1O|K|DsI=lYWf9YtPFMm{X%^IHl^~N=SPks2NdsWPpPyN7=BOhK0
zAMTUoxxY_w+UJ5IqZ6l%KYVzwC!X<ohlQaj2Xj_uQx2$$^Z&n7TrJ<;WiS8#|L?K#
zi9Pc#-?;x1UrFg4Wn(_P-@g9e#}9w1|Nh#)!y>7L`BBIIddaxGNeL+_760Zuyspot
zrp9*a{Vf^42any`Hm6IzyPI6`TTIYMMuw-R=E%FagI9kXh-!Y|>5Qo0cOHBFzkbRl
zMom+P)1psO8Y9mfxVdJ{|9g9Wo<Dy4!$;*`Q}~62na#})uZ`x3iQ!ovXIS&M>qSR@
zle|4>Np{7Tj)z||+veK~KKlRPSVF>J!=_1M0WM<FCtO*hq9Boz)L8EOKl#kj|L$}4
zA7Hp&$p3z`Lhi2Pk2owX+vM#fT3Rp8{qbJ@@N$2iH*a{pzcbX(HU=%fe#|~s&-n=d
zy}ijh%6cEZ4rjBp%#c$}+tJze;C(!sn_HWCzQngTi9ddcu6&f96cCUwVUkk8N%47g
zI?UTXH8V!Pa=Z*3mpdrp6s^o5-2N?p;;ZHVEAISyE8RRp;_&b97tBQN$kzWe`1AKP
zD6w^mgH|JSbSMb1+bTak&i}&U+{?>6udf@wxpVTvGv1P}IdkVWwzeL;KVQ)jw65&o
z`*^k+MW(ly($d<j%O&_^5-NU)yt3g)NJ^S8DJdW%B_Z|bIzi*e2U7nIO|`qgCB*UT
z$3#e&Y?-r=tHD9QCH{h%M%lp{Pl5f%<AwjQtlpnsD3NohQNpL(af^gsz?H)(_cy%Z
zvEFR3=5vYap1>zx{g?9k8JAyI<$1sE@(Q2J3M_{m1SY6t)TSu@f7Tx4BYgUhMbPyS
z(V$JaYhG)9c(zB2>(nG2<@27wHS;b0Z8UW7`ZaIc($_oO@@)API~Mit*vXp~{qgy%
zndxQVB)(?uMy_TDo;G_Mc6kRYtsRz~oBbbT-+!Rck=Hc2`9alo*?C^=UeT+AG|q}R
zG1>p&<ZKl9?r}`h`M20+&2SehzGDw|-N~!jlRhuKN0H-&)uN6<7mi6R$pTF0tJmh$
zO;bNnni^Xuvu5ck3AQPVay7Y56&<cL<?|PAnBw6%Ytr*0hKU~2as%rv>*`NzS|Ab?
zeBFfi+Pi?CcAiboW3N37fBf5~u*CYT`r4P(@4yaTGRK;)$w8paeiA3I+X@eh#v{fR
zJb5P+-@I92zG35*s1-U(9GUYHH{axNOn!9#+~OJERVGht-KGEaaq#g-*GUtWbxcSt
zx!aSXTYKJQp{bIKpzq^PMKU+@HcnG;ICArZ|C;Xj-E#Lrzgs7Ea7^dkE<Ky~SU~8U
z^F7MfwWExAj1R^y5b@f0d*l7!X)=5wt(WDG{}0=p`RM19j}J-~t;>uSoBpfv-M1T|
zd$zxcf7bcF@sVWD!-^-YptixB+55Sg5=4&Hch&t=cj2%zkZze4w_y2;n^g;zJ!31(
zC_3fKImKm*Lq}WFD{bjrmuJ2!*eP>v=bUT}8G|*JY%iZ=$FA==?z8;LRw-NdQ<EMG
zo>%s~P<m*Bje3_r)2c(?cg6@XU0?cogHFqW%ni9b&C47w7s&Xj-<qbdK*K`Dz}HRr
z(F~o%?=`(Awdfo!w3t7C;>`WpzMjELQZ^^<P1~BZ@jz*$-o#1UQjID(*4*FpvGhu8
zduC8*&h|Ioj~Jdh8{qY<IBm1}<>e-S_6pBG_C!=$?J3{8{{p*Z&A|~fXO=DZAqSo|
z`R+##BCjVs?BdhoitYL@A2{LBwUx~m4mz*3SfN{WXl3)G?M-btCnxYvGFQ}$`x^K9
z)AF)UF4J0bm08w#%h=D>X36ZjR$}$#_m$Tce1R^TJ5wh<uaLW~``b2XQ^baw3aXzi
zSBv%?E;*IrwDnY;yUgYoB^ED3sn`Y`&pf%c*WVxhB|rE45-&m59eQiUjw|2Qb8Ij)
zk=k;5iDu9u({|>mr+&7y7)m&=Y`He&bc$r8+uCCZ4M8o6hZQUTH5_VX+SdENzUEW)
z?61`?Ur+j!8n*XSh0prqImb9bQMhK7A$L=NNUOb^63e#@2UHcA7B19Osi@x_|3G5O
zYHw#3g^ZUI_9!eo*d`aKcJabrX7P8HHFubgSA~9GY4tYY(y^2JI-*S<CC+EhW<C@9
zYL-Tlo%;EmF*E!k*9tT}obXui`qJx<3l`irzZmY7GyARB=|hDljLxMmc~a$?9jt!i
ztySLl@0A`imMzijJDk=mpd)6&yI3&s#M`#_#<L5p1X<e3CUgF_o##F;_&N`p&h!jl
znb~3Sk^6EprKUKo%)jrJ9`W|bNp&Zy-XIYb&ex`)M`c02S~A0tyU9VIb^q5*?9Hi;
zI-I*$oZRcT?O(K7ou5ti7kgEPq2R%omk-oL0-X-Wv?b~t{%~*hc8}s3ou907i5?;w
z;})A9+q0!*L1squ6s7YQrOq$bxbgOdRi1fM!>%-m54Al9KTo)ncp&A#XO^~`jen14
z-p#pi>*T{-W%_3h&FFkk;o{Yz;A<)KqB>0b(Tvwq|H>&!xNnPI#kEw!d-+*?@ik9t
zp4HVqzi*-Dd8f$k*mwS?yTfXwXI_f(Q{QxE?n`EnALhi^a<>@p)H*w}7rshzEV%4>
z!Tw!;K}msD-l82GO&tq59uypIRhoC=ZmAY;Yt6?=)18h!Kbt;b$A->y4o@K$!A)=C
zjy^tM(#TPJUN+<Mhu?MA4K*Xe%Vy4%xV7OgJCBpWZZ#(vWuAw>1GxGQ&$r5xZeuLv
zO%4cENc!~k>?SX%!?QQF1f3|}q1SR?lLE(uZ5O0|_f@t{k#Ohxe^W{GVWmfK#^24o
z+a(L8#RVLf+vi;zSkV0q+-FESc<X_}GP%a>DIZn%PMk6;E^eEwe(|x;p}*hxr55~8
zG~V*TOFT{}U}@QdhmB@G_SdHy7=XKh;pqWEP48UO($W%=nv^=O*8elGsXkWll<V-v
z$2`x^J3s#V)VpX++nexa75%g{(6ZziGbP{MbLHVTGEUZ9@$X-1LHY5@<F@md7_`M#
zGnpK`zhAOXF4dw+Bycyh)|$1JtJOikjlY9StygTer>@%KgW(@Nd(|##vCVhlsXpAJ
z^pEwSpH#$+#;4KGY7XCV_|R>%&P40u68`(@z9)^^9-etv&s(_v*n`R=hK>!Xf^90C
zmFv0%4|T5Ve!PM)?edFVn{-?RQ$1rZn`~M8>tW@x_gjy&yQy8eZ{J|^AX9|phFPd<
z;GWp8ERx4-;`w*ynjcqAo}sp@ukeZ_{|9iyuYowRb-$@*Q<BJ$G?DL&EcX-Suc{kx
zOV=>1K5KDsnfaO+$8Y~w{AY7cw&OEy-&xjY-}qp2cSY#5ExOn4HScHtdd!65L~$Rp
z;47)Mnp}q_=qz4!VU~s6#(-1v-Ly+LJyQBytKMb6mOMl4nWW&f*1{uElUOFUuUA}d
zvh;Pvto!d5-52LuUwKDuN73yU)oqQXdw=PJi^Mgvb@^Hxcv|->GKz3=GWqXtG@a)B
zV5fSLNQR17;}iGIO`MEvb5ka-&6@T6)|T3{s~2v1rDR*~a_7PIUmeM(gw{&7?H6?y
zkVu=cY}rHkD+*l!T{_M!HtDx5;$ELSezErW=e$XI+hffg1zjZEr?u`{mV0qlW>V}+
zt7nq;&8xT9UjKRW(1b+`R(VWXFJb+9+kdO1E3u{($7Z+ftB-5813NLP<~yHZwUa1Y
z*oSAl&R$%HkM&xJF4)2OjM2<2Lr`6bNtU@mXVc%0&5q&I9i5sUKKExcG-P9DW@BY-
z(~n=URYF}Iw6?dW$7If%)-^7PiHV>UL{Gih($d-{t4sRLO-e{jt@tzJ;W=4QXXyEP
z$$x*76Ox^0)~IvLm?5!dt>mj$5^vsYP)|B@i)C}s!Nm-|fBx6Mu&IodJ-jZqqjo1~
zqK@JJ|21>GA?=joUg26I{R>`*+Blwn<Z9zw&E>yHditf*!iNp-j!fi>J>#&jtK{OF
zPYMD;|5$_`N&K^T8vSr%mh}18M;{;jd+zNqn|i;#vjqw)>$-!=EGA6SNsbg~GTHh=
z{<w?a>#XosY%P;o47Tvq|DW=;e3qJ@`jkbP1+l;8?Q+vxvLttB%nK`q<B1X<YL89)
zJTrBpHq+XVHe1v<P84@Fy)NDVS~mRHx`U0^VwPn2Mh1o2%)j68oefkv?U`-J)#AY8
zI)84f5=RuLlE#hb3cvLa3~gG&_#$p}%#Gq%Y!H#NG5F=XO*tD^2B|Fh@o@5df$Ye*
z#dYQTIv@JhGdw@J>r!cms8`{<9_7T2TxHe3miL?AFVS>NP>G+mF_(ogGL6AQsJQ3H
zdC9-*mv7}*{E|EEl-+XRvp`eju|f$p<@lvXN@uD)l2{6I>n4-)S>?MK{>Zf37az8L
z?IeBN@p8cRExJK*+K^;%WzI&f1_K_K`#~>M_#Oqlc>XK9u5od9^zIvF1p!=qXHx=8
zzMPn1di?1FrxqC-zq$Q?9>h5{%$cbcwCMfJ^pjy;B}ux93rvq~SeL}pUB70&^k1gq
zjX^ommfWJV*q*!SBp->|v;1{N^u!5LTc6fMXyngY`fYA%qJ)I|&OJLHuz<oWspfNA
zfuX}BCKDm?AHT(|9`wq5uc)#;%rEdhvPqOhjhSD%FK$o542y}0r^Q|*<@-Q-)k#j*
z&Nr+TIi%X>%_!B$ZD8!&+}6q(sKt|W>cIPiRm`uJ@uVF6p)Jnw`>A)^*VmHY-X>Rk
z=c_)IkdOd1z>!<9V)+;Qtn=(Q)pv;eIWCbb+oH<2tp4MPB&`L;2SOJbBrM^1qQn?E
zjWIf|_V+Z<rmEcA5^?*E%rt^8&^u(gok={=MV8aGuXN2z=3v#s3pkTZ1lKSNorrU|
z>}?)@pqVe5E8)OgqX)MNL$!H$c;fdP#q0);xSyS^(H&|CDS?w3dz6d?rt=kaE^ajn
zi?nLdDROf8Y&=hiG5ke>8h1=v!1eO~f!4}AEtgor*qHmKeEh6_`0Q++=<P-__I?$N
zB9L5O`EQP6l+-PzAOGc5MLIm?GZ<DYF!#Oc6l!=Tu_ow;H*3-*mawA{n_1E-o%c+?
z$Rw=$OY(%T(hSy<^X+-c-x=iGI<g^G`jmksXn!hby^LjfUt@Q-qX);W@&i%954?D1
z=<=j!+PIp}Qp~WFGqx65!`jiGY13txdXbAKMem=@fd=#5q+1<p*v{72cs4SYpK`dy
zt=}bQpBAxyUgBM=HgSCp3r<jv7-afPlg`HG=7XoWcJWU>@ugAy-Nz+tXIPFj=n8B!
zdQkQ8xcR>{mQ5T7LrynL4A>wR_~Wdi(c}mIGCWVz1XI4e5<R>sRAmVd8*FV<($1Qh
ziMLGH^z_(*gI}butbEDr%b0dyLI~qSCKh9*e=II3Txv#<kGIRNo_Rz)eoKVAu!>!$
z!eV}T2{GNI8ygy1rW`zQ0Gc{dQc@l~N!hR=;l$Zg1H(qWlEX5$4$0hgntVvaQP@2#
z;i(Ykg;xrHPJMWx8YEulF7$=z+MKpb!L&yS2c~(Q&^FswpFYE&G4Z)y8#}+mzIycs
z2`NyMe?oS4X@^UsolQzeNH8#M^<xtXmV2ac^KbIKHuZfLiV+hYOjHfxz7}yLXTxL0
z5AQT)Jv=f+hNnW9o%!O?WB2#-+`S9he`QnSaqEu(d~HHvLIP;S<7+l(h~U||)Pw`m
z>?Z8(+PdsOaN&dZ?~<4%y^>1W@<5;G!kiX&<*4i_3Hy0gTka5ImrHe-6cK#v_jjJN
zXD{64Pnk0rI(y2=Z)nI?Tidoog-u#I<L{w{-9-mQ4vM%7yWHqn>~b*l+oSa^99oAb
zGkuuR_RPS<G~?dES8Qu?l4Lq1|MJ}E;_?ydgKS7%xPAJ=AB8VY2^m2G!AVlF_G{O}
zH<6%$f_?Sr1!ZXu9yr|Z>fkx?ZucYhh}UX1l6=$knzBAfwDNEqQ%$?ucva)ckHf*c
z1cLV``0AbLnQt$7ZeHpPv)+eCIJ?3OK`HA0f2a6_KksA@uMFmi-)B(s=fs1jM-QCm
zzx#)eea2NT!(O4JB-U_^WHy<DX0mGH{R(=iJ0`Ktd&l;%lYcsA#VdtBo-BI*J}Mtx
z7t6yf=I}iw7-TAFCBvp)f1dldeSR)!SC>-pcb1|`i=$IhV=L=kg;oZo-|LPn=V1%7
zQeauD5aIZsT=ZJX!3HVMBNo!DHy(My=yP;OmY7A=myX24Y@mWD@9v^QU9pg0I26w8
z?QL99ZQRmkv|^=WKSP2`)ThG@+s(VDcI$M!<GXS&EQY~%T0(X+qcZP9_KHUm^_!b3
z8dl$IoM};DSa4PC_jlu*n}-Ts2pqn={KvDXPSAK3!~g$E59|X2*@A=F^7Gl!)7cmq
zUkF&QGvtqnF)=e@)-99t;3-*it25B8XJ&yzoYH}0yUI@HaNAGs<q!Y<&ciKcAY*^7
z;HS~y<NYDl&7k!l|Nkq!$+xp(Q&nZ-<_4vP!om#gsf@=PB$)eJnq;2-5N2P=U~DKD
zf9SE1faHYd1~#UeX^#$QH9ZrG>Fv>vmpC^!IbzR@hi4?)%=10uP96Xm3UcSE*WkSl
zplK>eNuF1)cor?{XinSkEnm_gh+*9(mYpx#@7AV#t^KgwtRvHS(iAn0vZn<%n?hTD
zOnUst6*STI?S1-=I@!NA3adat3NllXCn8c(L_{JcMnXnb(!^BK#YMpT?+2b70Uwzk
z!EDTey#8w%-Y_xj{~00MaFfB^to_6>qluYJhkktNOnev9_V~DD-aSw>Tj}yZD;uS?
z7g9i*$x|mxOZ)I8Wy7|#3DeUBL{%?vHD(_<rJ!7BATaT(V#AVU8ey?4cbh{6_A77P
zX1QUTy}@pk39n~w%T2zqq4D9T)VBBcIp?(bDT18(|G(1aRpO$l1_lWmwxnFR-u}+9
z)!b#mV)@OF4&)RBs1^zY9~b5<nAyC1dP_XZ^y&*j>{06_b{=TR@v8pbm-s!e?d<G@
z>U%GM>JN~&_bZ+|dQ?H<#DWC}9=tlWV9Ak$gaZm{ZJXC}ZSDEO#?B%jEXm_CUE>Gi
z#J!?1yoqb{j`*<ithm^p#uo$KiXFEn;l{@1hgUS)Zf{>%utV}A#F>k>Oh0z&l!DF~
z1?|%dmLE^ZIFXRTVtbS=%jz8?|C&zb?5AfN1m8utN_<e``}9iLhI4vCV@n#x$6g+e
z$0vnB18DE=B<`q~2s)JD`+LcKbxXfn!i@G4pEq?XXyKP*qhs7WN5{s4cjr1rOILLX
zJ-NdjbCUI_ro>x|KMdJl9=u=vAl$9=gZz6FF6OMPAHSzJ&XlZDx&HAnPyRiFn4O0{
zypaS=FyG&I@$l9eP=`PJ!KbawY;4RdF5bMwgjrs`dA4+OC+C+mqZ5(&KW?uo(3`lH
z#pYCo`1c*FoKlWI$jn_(`2SC0#b?mg%C`IaLFwSee)%e{dykLv)c<$7J)0RcS<UeO
z|Dsn9<nIKywf+AO+LXz{Dw&cZaptUKN~&Z_i^P(pk|LrDSF+R#9Tah2x9a1S*GKr%
zCa`p5K1$G4dGM<KU&@bfy`UC$US8Y!c#tJKY9x0}dUj5>t@^vfz8c4z2w9cuJrGZv
znQP6};K1V|f2b~irTV=>_5bdOn>V^!?M0Z)8n*@<bTTOAXyj>;Jn=-hV9sHQuU$Ic
zThl(z6Ih|OsVDWI<)=9&dk;9i`Q#S+U}DUkL&u*kab2_{h3DEa6Txl1XWiyrUj09F
z(R)cY7Y(j8OLH@&j!jfaoVPOjuE=Yz>l@!kJ((}R8gvH8|Nm$1T7ynoIrecsf0G3B
z!LS^b_w}0Do;CJd+h;g_W8$`E=g@59(R}v(?;gv0r;6uVdpgAii&(f{zVN-`{O1JS
zT<`0Fp*@R}*4;~9`^w$0DAsF9y>^^ok<6}{xje25PqGJHKC)>;?i=gpD=%_<TEad3
zgV~~n!Xp3VF9Mu@Bf^~&So)exmg*GpB~S5^W83=oD9ePM3bVgP9<wRFKTA}xzIXc4
z-=BYhTS0r~Zsa<kz~geBOVm-IMepzb_rgakUMR@wdHiPlYNrsXC#m;g(|MWs>(+1V
z6Fhe?YR>YyxvFa)?3yqsPh7C4@pqWolQ}CymudtZs%}|OX2`bJnPsaJ%UVs&O4oBo
zYK5jb73-W)7H(}dS=h(CS9@;JLI#bn(~;4_o+oD6O#kfnGq`4_BdCD;|DTyb`c>Br
TwZ<3qpvd%e^>bP0l+XkKzc=NG

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
deleted file mode 100644
index 86d225e5d2158804f88dca881f69ed3ab287d866..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 19086
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s33iWhx45^rtlpw*nn1P*v
zfq~)we<{|*4C=IH9_imGctC;YNKY@PEbs4r59i6&x8IB0WGs<#(Av&?&M;m2apu~t
z4XaiZt<?!~(P-+}RDNp7ch(7KUiQTs&p2qy@aAo{>$A5rm21E*zH`ZutHptb<$y}g
zh1S5pgWdoC&(=`l<7k@v&i0LdOO?}JWB1#?Y?d$kxh$kS*0gVa;)cvk`?gOx5*u>V
zC*(heL(4m-PnAtev^*FXnph45DW$)$o~C+D^tRY@$?KxuWBV3A`Ey9$9&Fc@kd0i;
z4kFd%&%dq<3K0+w{P6$(iK9o)dOFs2zh8Q|G~>HdX4`}{iYM!*AC~Y^Ubiaxx~OmP
zm9LB6zd!uo_O+=?&l)}6+k5eu<<SlSfhH^V<S863`fPSNLJj*eJ2W;My{>#@(Gq!S
zZqDp;oBI~~OkVSP>-$YQhyJ|I*w+Y(mH+>*Ots}|P7rV@zrLVGLdBJ->=EOU-|+{e
zE0_+({c*f7cO#eKy#sk|;pdun&X-vw<oe9dc;##VdHcH`g@&HrRFg9E*v8g`%vFjW
zlOAXvEd3G8cCx-?V)OG=T%Xz*)@U!|eB4p_t@gOVg515C7ryV)d0~|^dj_ZGcI(B=
zpww|CM4PX{Kt%hdyUq^D3&A_?{^uy*6>7TMG$Z2R1cwvC7EZ3N(N|6Wp0${g)f32Y
z>)Np!?H7)|Xg^rRzjkVIZ?VC%N=~8f#{$NW7SDNfc2|;$cj7spPL=DW^2e=wOXuy>
zxnahj;8+*)%v%N=-coNmn`h3PICpOA)Txm%bG)6MgWTP(9dQW=ojP;&?1{a-oz2aS
zQ$?jJZ*XyL`NW_tzW0oSuXcf6QPIJ3ZR&zMES3cP{Q3F*a{vG5=KU>yaNvLScZqv@
zlPi8tdwAcz&Ay(kT8j~4(Vouc$&+U`H^+udarX7~2@k(_E6T~axwreZy|J;f^zHhz
zv<i`b?p~7r|NZ}e{6xoAxr3KZIB2Ckc+?@p<ji9E$nf>t`UzpWL6J=te*gZyU*7)T
z-+zDqUta!S-QVcVos%EFXdYf2&Xa%N=+EENA6{Q~XgB49X0x77XNQKlvuAtz_QnSX
zA3NuF^U9Rz^Z(l`Kdu)OV{<$9zo%zLU|&b+44on_hxNV>CUQ-0>NGGg_;WOPv%*rI
zH!)APw1)gXaC&O}|9^k~f9(c&t^B=&S#FX=)s%-{v)gu;OaA+lDAAG|0kQGSwAF$K
z6a-w#jb_Y95D1(p!6TcXIg@WiquwLq6&nOO^yV{ND9^k9ZqLRC25Hx-KGv1y@h5H$
z({ydPtit(2w0mAgv0LNm)lY5rZ!;|7c@lc)<DZ|6iz9U_Ys*{rELk>zB|zKbO4k$t
z<vwQ1#UgW_JKU=RryeO0P;3VJ^hC$RoH*ok>O#QDqpndtJrgGi?!5S2e(ffQM~l??
zY<zj2gsJT5Ip`r%KhgBiX1kzWE`9<hHbrR5o_WThQt9wpf5N>Vz0y3duNmIif4<<O
z(&2V~p7?zZ$DXoDKs+e*;K;!<XEc-!tzDy&m#LYZvLqo_S5xT_2WPpPn}=opG`kPi
z)m}V`;O1G=$&g*u<YFIno5@|ZHFEg^5&ILJJo79R7|!i2Zcn_f*Oq-<LN6|*;#<$d
z+uIrQHiC-J|No^<965OW_`=oNxk*atxocCBlr+_jad<L*Xk@=~iAk?xui3%SwF%lz
z1utEfNLK_lu5~)NChUUr<{7RWHnj!@e-8VI73isS%2*vKc*Jt}>S`YU`FttSERdu)
z17!BHH9C1oi(Wigl9ZyNs(O@5Q$2x|f!StWL4VTU!xLKF_?J9!_e#|0WZtRcxJubT
zdf|yq=4#32sK*XBSNO_2xx}=#Co$oGhgiYH{q>UTVv{Wjn;!lsY~vP}xVKMsh6>ah
z4M!BUwV%9Lk(a+dDN9RR|2%7Ek&%Ic!gK)-Q;us^tm4e;p6M4Ya?IVQP@=e8dX2AN
zkJVHAv?<CrJ0g?*CMW#h&R(sg-dE2fag~eNP3Y&x=7+CB+YYx&`prwRDDQjtx4P=M
z2-sJPvL6YEKfiQ6yrksHhV9{Dp`s5SCAA#=_5c6>i?ilfL`$4#7XC0vCi=fvw$!1}
z%?X}XD>~Vh`Z>NT|8vyGaE2+j+tyF=_dX>Y2;6x<Qr^xWX2*eo9|ni#+47v9XIS&g
zW&aO|mlnl8x^QL6hmVFEwufK7EG#aRl$`SC_=yt@S7tcmr7Te4u8?L@`z`Lw%f)Q^
zb-~x{4Os`L2V7`7y5XC9fG|tcBZmS-Q;|LU4$rdKFsc5((VmKfAZr?#dEEO9_Efnp
z*I<Tt$NiCjNXqsN7dUuKHf#!-H1T0tYHEwC1an^(^V*#awz3JY7bZMEbW802MJ4Hz
zEN?PdlQ+nC|7@ISVX*X{lBo2F*qDqT5Bz0#HYhzg&flgTF5x#PA!1|G!=J@%=jUh2
z6+#m7qRK}B;o{%EnSK9o;nL;F51ytrs&l&?`_IF}<6$Xn+_^wR?nA=SP@w}-V*d@I
z)GJ;|P5IJj6~yN$@Pc`n#8eI^nVm1YX07_CwAJ#$DG43fs1zY1uBM)Q`%*0mn-Xst
zfh>;Ovmo~lq!e6~`)I?Kke=@14I2ZeO@E%8nCRdq<{^0gz=03_jg5?!Nd{@Q8@4M3
zOkJp|_T(J*yp|^0ix0AL6|#>@DE@7C=Bk|`dNPHnlUc9z4%g4m?GGQhw!OVAxh*Gg
zM|pqSMNkXl|Nkws_VTtG@U-mjR$^PQAdy*xE8^i*okJEp$E_A_$Wd@w?{LVq<X@+-
zruy;L&*dwpr$$}BZ}BbnwgqRC%(Xt%bhk-d9Fv5PIxyW)@=Qn!Z(!6cDD?i=t9s~6
zby>4Q;rYs{ckHoOPig#DmDdYf8OE>GsT8>OVb?}Ah4g1L*Ck8bF#GoRuk_#P;1X$3
z=pzrmvQuYD0>Z@7($iBC5*m)k1kL;?ztXJHRa=QgQEOo!hqo}#hk&${BclE%Rx;X#
zO%N6fYD#eF(%Q1(Z-16fm*xNe|J{qsX2o{2eEM&1^Kg;mp(6~k9BMK%CZsej3ksXo
zIDful%$$W=r#?I*d8lmyD5x3!|6f$J_0W+c2hN|LIQ_Z2WTuMl+4E<5J_zl3(qL{m
z@z(7Bk_Bb%hOY&ul(B7N`mcZNj`Th+mSC~1Z#!f4KgmdE$$k;f-6xm1hG%n;;;J;q
z)$tRuXU?~me0Mkb#ft=sl8%SJ%eDBoLlbM-qlCo7ppYkL&+~u&d|62BSyEEdQyq=J
z^CwPZ43jvJ%(u4H(PuJKw?)(<$J{l4jvBB`?2*{edwlWPqrx|p8ePA|&(P@#{IE^e
z?KiVv!wFt)8>JIXANunA`42C2<~ctvu=p6LGW-93QPiV^<YY6WN#{;|DlZLv`uusr
zls_9D&2417b|oPp!NE)Thd0NUl*U;{Cr)NHnKe;K&Ob>hKt({yNkHsis-$G#FH;M(
z%>PQuwU#uoq(3~^FiXV1Gv&)W*~3px^6V}*wy8Mq;kS9|BWRi8H@msH`RE}hLw)@V
z*TUw_o;`Kq#7@SjnZ}A&r+k?5vA?lV(Cp<Ymd8d01_p|zMh_m%D7D;k_TZ5NAH>tt
zyqp>1!Va1&+M4o^VXbO6(}ry_fx1pYsVPTFY*w9?ytwDXfBRX<7jloBa?M_M$R}Aq
zC`m&v?ZM;Z8(TUbzKd;}Z*LguX9UToYoZ+a4hRUyEPD8W|E>O=KMVI4d@()$an@VM
za82K;0+qumn<tyjj!58JJGHpFXZiPbL+S55VdWdFRO>w(x|(Eo-YdFxXlZX`zBlWf
zP}0%h!(SEJrTX<&{=O1#@pcUpL!h{ErRI;XET79<M86q#eOh^*arp)TN3V!(fdwK#
zPUpDm<l`Xal1br)O-9lZC&b0rZr|44uzB+#FK(A#4HYaa`I++Tj1B~GGTS&hoN@7E
zToKFZqRck$?SxALUb7rT-HlnYHJQR2G`nNi5_}6Xx(qMtEowh>;J}an_R2-(qV<i9
zJR4TEGpnhw`T4c=@UW$&wLLv8`R?A*Eq_5t`v3n$8igA+7-VLhnlhEu%1Ud~rcDY<
zdDJRSHHgYxu;I-%H&8S+eXvSkDr-@=u&LLBxz1;rq~7VpWHP<uaZ_PSD>^83IV0-d
z;+?#jx<&^c$*p>7uXI*<4%bK4O|cQ}<^@ZcV?ud^g_)(LovW;vnVAnavGVlG2Vbd&
z*sf#`wmtLYl&O{0*4mpl86_{_QFA%u5Temh#-{Qk;iwLCkZo_wJY{t$w<Rf!Icj&f
zVpJKUraaa(kl3Kg#Ktz|O2cuklyJo$HY4%F2X~xFak$idPEF(Vp+gg9ocQos{P3rz
zJpFPL=i5QzT*(-0VU|}~npIWx>rERKuPc9;a<Iii>cXy{_CdFq<i1-J+uO`f3^5nj
zJ)xGT-^QfD^i<(gu87B+*|!d{ZV(CIAjZZ#hf(<WJcR{v$7Db5*pcxlCWfc8ljr|G
z!!=PyKfDl}`IQ?SuSrVI{M_7=XU&Qz+7#>2!D(&HTy$Ib+9U@(n;8l`5mOJg$Q=J5
z(88dicgM_ai4e!Ep9z(M>lm~ZRxm_P6P&Fer`uy8!6Ol~;&)=Ndy1LWArTj`#DoLu
zc&6`YU~qO}X<o12JVk_A*z)5UsP7_$g@qTdS>q5fW9R(&oaW{#*A{MM^=J3cY?hp6
zG($pi=7eAdB?ZAyek1W?&6$m?+SjJBuq2$l$ugNwu0ir+&yLiIk`cLyF5Z#_u_-Ft
zJ2+?Tn!vhrspOwO5;?h&Ev<!SMIS*{G5r6pWLe0;QLt;ri(kKIdw6V67H9S-<xti+
zsxV!w;}i4N7KbIGChrwn71Gu>GP=K+6cY8okVSWyq+y8AWR{J~gm(A~s&f7N)XW%d
z)i7VaF?1TEra@_`<e$HiJUkK>79UzU!ax@N|F5+6?7@>Kv+~vn2_`OFb*kTVy>h5t
z&ju!DrN8{H^8{^oJd8?M?EE2+<;MlqDAm96?9#jfZ`LHa8SlHsuu973Xv9(p!`n(9
z%FMI0rLVqhd6sYJI$^c~&xREnHz)u2G4bKs=(f|-J*90}!BM89Sh!)cv9!dABgbY*
zOM5##^EFt_z~<7(*q@+wMbKbNKym|roq>_j0f~sk?8a&n+V}iRK4MiT%(0B&5o1x)
zfh`<&_ZzS&zp^-6AJlIC;d}f9Mq}gVZtlaUr}M<`Gq|(+_=o5G8|5HHypm#}hZh$M
ztK{ult2S*kJh144{hqn3<%-*4`d+^H?-8tfg6je2Yx%HZg9G(FOB~|X2ruT|%%HHZ
zC-K0vvcOA*%EuJg#ln-MxgCT1O;taBFxauf094q1QayaU-{r^D#>U1*Xg=F^_Tcg3
znfdEqzFMWK(&FJiVNaink~PQ1Su$Hy98@%CC`M#S1~H^1HY{g4^dyKoVVj>spx~?|
zoqwwP)SEQ>)-*6`Dm`fO^5WUIkLTDio;PofRB=KAOslY<Xj4RlMoYW2mgdJpTs{9i
zmMSq{+F=nhr;*W$t;_6l=QN`WrbY$^4?65kSdWNki7aDoI_ndYbzzA?Fb`w$M-}ch
zFPj;ovmB<icFdk(z`|m%Vzu!Gf$&6V0?;b-@Z|jU>y@OGmyz+}YKEO>J9dTqkY6*w
zVP<i=xz2=JV*fiG3L2*RZuoFO*+WiYhKqr;(vB&D(OnO;6bqJ$`W!iSEFt@JLh8{2
zXMLxrLqlYa1q+LXtjv|%+^Ii)%$RUnlIO+Gd5p?#{}tbw|LNZKj@wdIk@?ILhpb-*
zxIUZ^x}2vVXP`8(;Pd~(QPLS&oHKfsuW-<o_|UO<rn2hM2XD_NWO;==g*0cCCRp6K
zG2_dZls|vBii%bmc$H3xW6QCT4vbQmAJ&ljpD``8Y4w(bhU_B~!#;#~PEp}bac~oD
z-hbx6E8m<lfvm&_nr0iau3t|tC{xS20jV~X0)BRMe6X}${rY9+<_!n3Sza9C-g1oH
zr;9Tz^1$-Ai&_d>mM*TiV9&<J_G4GO|NOez-`^(Bojb9;ed^Syj9b?vHQel2&DtD!
zXd?TDZvh<Nk|ZQ0{{8=V_QTK5|L5EPFD^Fz@X^q~z~I0&Ho>WOAHD}9EK^(}AyIIu
zsKCI&(r}Lb`45j+8~@&f#L1!Q7CUxE>gqg^l@)DmRXn8-$h}cm?)!vS&73PYHLQ8i
zbbh1ZfkWJuh7Y?u*erhkuW#F5|KG0WN8QCm#q4}{>c79U78Yh!SMT0qv%j@<Nptg>
z#zu}aHIux!n_NW>^7!TmHM;!YS-k(-yT6r>j;wd@|5pd<!q>AUCbGG^Kl@RkR$0lW
zrnX4UuESG4L+Jxc<IJ8XPbF<^B;MUm-%-GLe*>sf`Tzf+=@u#~EU__Hu3TkJSgx07
zCcfcG>5@atsi)>OGVXlyvp-kyx%~{z|NsB~{{H9Z@ulA9&(AD=_V#xEeEa`@{z_Z%
zR=>S9n_E0i#`f2ZyRnK|UUMf;?(FPjEKgBB#M19yxZ?Qywhc@!Dt~?-|M2$qe^%~&
z@9zA3d~2)y`nbP;Ke1{ZJ$0(!>#O~u+7iM-Gm?^LE?KJSB|jsGheyM+MKJR`w@X)N
z=fP854{sQ@WncfG3Tlrr{QrMwrbR{NO&#5*XU;6yxl>Y7z&Y-JqFBN*p(m;~JqAK@
zDH*ks>u+%JYyA5D{{G9${pvmzo3gLJ{r-OcfA`*w^YiO#|NVJ=w0pgNTutr&f7L~s
z%<S#C*W|b5uonGn;o%VDIl|zkQf%OP?Ee1$`|JPb|N5dIy}fRK-QV-m^)<}?{X6^N
zpY7px{{Pig|KfIhSihj*<e{dCot<ZXv7S4&CUU~gjMSb90?qC0pg!<&|Hm@Mkh1g8
zObZoNR#}-TA|g93aPi!DSD~g-SkcJ1-u*%G7ak8yX5GSs_3{7y|1W=e>GP~vvF`nL
zfBwm)ulo1p<@KG#>+Amg`TYO?{xv$Xd3j<H50s?4*&k^Axo~9y^S{5zKR!LZe`RI7
zS<a8T*Vo?5+ZXJ3V1M}ccb@2Nf9k)!%8!>{yil>~Pa^AyPKM372OHKt0rfd0Bn@A@
zFo@aHVGQk=9g3Ut@#Dtm=+iT2coY;Q^aVd$6?4H{(!l6nYiZJdrK?giG?^`H4ffRi
zHPz35ywLgk$0a-*KmPqazcu^)ww#^&OJ6tp&)=|ao0`Pmg47)h>r7hQxgYN?|8JJ}
zXJ78^@3*)Af8ZtQu`j`*YKo%vYkN<2?F7E!hc6!`@pU*Zsi=^ssg?ZlRdPc6dT@W@
z|NletERG!cv3Bj#)Ktqt6^<`wl|S@p*PP(K*MGEZ%O(4sDIfW_J${^@{CM5t<MMxh
zxn8-nH2?m;nEn6qUS685A6N7D-(UW$GYdlvc`Q6kQjfUUE43u@w9Jg(|F8PbkMoOM
z<MZzS`~25BF!lU=d!F?3MkXc(H9rr1n9m`b@pxXw-Y~Wvl|u~^y&i1(^^51#E1orL
zZs<ZvutSqT`P;`wg_q~Z%^L=nS<*5aSr2w|Jz2;nlgX6$>HYos?{9W)E_vB%U+)p|
z<;S<T=cBjV<=iOvf4E)VLTNUm`<=-lT@SqCUOYHEdw=$|n!VNEzaQ^c`SIwye4D7Y
zMA>__GJD2#R~oc^9~iYvo_z4~WXD+9>TYPRRZ`PAlb)%mt$)5Rrtko}`h~K7$6xCN
z9JefK$ZeKNi)vif^Z)<<`@6rFU0R|nZ!e_PVR+#4^ZVO!tABiY`h9(Tz2Z{KH4M6u
z0+C%0m|kp}Z}+$Q%Zu;V*51FjN3k;SeLFwT<}|~Yy<Q;?4>qh#J2)ljKwz4J_J%Df
z0U<~1ZbO2=$zNv88lR}BMQ(1BWMp_Gc)6!U3SGJKL0;4M!7AT>hrS6Vd8*d`|7ThG
z>EXk}{2p8Z96$d2{C;}+{eJoSkMCrQPB$k6ItD2nP}!JK^YzvHjmh@+_Wk|+R=Uuz
zhs$qHf<?uIIY$r72<+Q`#^KtDg9jhHJEx%7=qd)TRh|50oSb6B#DY$plF69ZySC_{
zj^YJw{zn(;+1l#)<@ep+`T5#%e-&|Qfe#N4pU=H5SO5PXd)Xr~ITpqUX(b+s!v2$P
zVLd$m{{KJ!;pyr9y3zX<md13l%i9~hxp}zYIUi?KfvLUW)dyN05)u+NHWE{&Y*?ZK
zDcYU<Wfm=}v9c1fvFVXW?7Ftd#d-txQXxg9&IQ*k#Pw`!%in2ERMk?e`2X+v-QDx!
z_Wk+x@v;Amhdetz|FX}y*fX>3U$RERfoVnyqVDbg|M|_$_w4-tYX12Y&1EuuVSo7P
zX`bVK6PIz<&lXwB*%WFmF5Wy%Y}!f-sO6qAH8nY6Vvnq>YI=J98)%8V$z<IrbkIY%
z{$rSGhsvk_|NnQ3&$q93`zM&OzyAO4zrWwVy*=Nq?vJa>%!3VQgHHrSTus(;@YHyJ
z@9*yF@89q57s++1`P%jHiD%Mm*%J(Fa}KC>pK-k=Bz8<ezkeO1o$ll<b7$8sp`@e_
zokCB<*cNR$U?I=A)YGwKW1~X-zdyC_?p!`UU-Opazvt)wXJ5Zp|C1^I-D6Ls8DF`h
zekC}Bx*5DQFfcHv`Ss=fudneUa`QiZRzLjlvC8{KH|e(xn+tiX6JA}tk}_eUqd7Eo
zJ!KXxiiwI+($V>Em?T(zwyAT1ctL`J!O{iWY^uL~ytueO<+`T8hqt%qYlq+a|KlUO
zdsf1Mi7o}A6L<_vXRz`xPtswk{r&Czr>FPr>w}*0vbDAIOTN3W?ymfAnZ~?hnl5`6
zD>I9WcY#9n|9>YHzPh?HX=zSV)3PTAoZglsh#$N*=|}fOsSiu0*2~-1DKBR4W~=-2
z<M+3>{!1>;Ggv&QF`)Nw!$R)JIgFu(yJr6X|G)n4uk-KkPf_dhpD%H5pK7Fd)NJO+
zIS(2nSSqR)i_1Y{PlYcpk1sV<u<qag4Q%XD)t~iWHQDR*^5o?5HHDtBXKSmUZ~y<t
z$L1HTpDhlokN<C1_lI@2H+OcHgEqHh!4jc0rhM0o*dh%N|NqZ3-~Rvq&+V6v96aD~
z(eU|?_A`ZBJtZY2mn^v;3>lM3YVzjhE`Ifb!_ZK1yKzCN_W!`kEG9b-_?an1^e^bH
z`Ty^?{89J0e}8>_e|h=+`u|B*65((D9iGj%;gu`fi%p<m_dh@7L;GTs-j^iVC@=i+
zTEt?*cK66Q(Dd8?|4nLqd3j;Q#e%7+W>*S+woCQ>pD=sQjI-P_%}+C~@=jaT-rL(b
zbLN`0`y?DcyqBN%rf)0T&vtX(g)5KiOp}-})zH}ZQ779Y#&sw8CNUiR7<`7$ASGNE
zG(*Dh|9_Jkr?vI$HER|uP>5RjVLs#YGm>7ROxA`HYdsE}`q!Ox3zW~9nAX_XOlf16
zmKHWQ=f0#Mafgd7CzUPj=_h_=--IwFo{A77qkqpIC#R$?b$aE{rx?U&nEYx-NoS`a
z2d5$nq@CNO#wRBi#>&blv%KxUyEm6X;g?fPrCIZ4oMO$IvOr<?1_Sk<4-XyyfkTH*
zoIl@n>siAQnOX5G`X0ROcl7Asy#B@fV?%$xqf3j=UKV$Y53iFF6E9qIJM#e2he=9G
zNJvOc4GMa6@#@n<9F97IoQj>&YR64EzKQpW2tN)8d!}w|lhDxH>lrz-y|-5|+OmDW
zs-szwgW0sXa~)k<TYGz*>J2=+lO>B3I3Hw9ND6JR;W~8kWI`HS?Ef<d4nUF?*qY?z
zut32HlagH4HcvdhZ^N>w|0~+vH!kCU)5@-{zIo|V6?S%4_9=#jhFRI4KYrRIFr7of
zAWJ@GC40Zf9St_Nw*P<r@a)>P=FAz9Q0+5kc&1I``6Kc~#OAMH)8@iAZ+Jd^;<2#_
z&;u91VEYmi#e^R}eei>Q_a$*%*#%1_*7SDn-w`dH@wR8b#Ib*=DSy)Q^cF5T(ti1>
zOT&Eo|EpFMbarkO3iNoeGF75uh06x+t`37$D-DA!&M9i0dho>MnWN1V29K`9q@)KA
z97>=QJ55t$d^T;`^TADZTWIb7gz_1Gox3jXUm?m8mHCl<w!%4)#Y<Hmyj|<C?f#9M
zGmjj#Oi8tLb-7WX!nEZ{-|9;TL;Viy<k=zA)xoiNk<#WF62c#?ry4fyv~2F>{dwzQ
z!wJo56I~u2o;!DVii!dRVWF9lA|R4-`N9K-b&emV$Y+>eQ_%f(FlzY&FZmbBFKlei
zR905S#H3u)iHSM$@S)M7ML8?l7YMKMRZ#kIS|TRqOUjWGKjJ+)47IwN`}@z#YMnb*
zQM1!AY?{qo4hDbWh;)^L-U&_&_V&%Qq_%v3*_oIa6#8`Xv^1A*T6<a?qJ_Icmj5|^
z@zS5}B@F(5o92WqXfiUI^YiDflonQLX=Qo&?yXxF%#mT=7R9`|=tz7>f|`DlCg&>G
zs96^;PW1KY=-iW*86fcWwa6dO#zw|kZ*F^g&}hh!zcBlfl0?LwiHW#O5&ocjXJRm?
z%t5s@#l?vgTHG<BT-QD^pVH91T3#|m$+Tq0jv2<rqF1k0?$|lA;D~KnZj*MWz?%3C
zV$wVUdg9{EU0lg-Y`Z52oNR4xVR5dvUu5VZ>aZc|pib0{o01k53-(4L*(vfQDe+*a
z{QtuboP-3FHf_AB`9t_@e9qDH@oa7NO-(v`_oi*zCiLO81qV}HoLF^rpt-rBr6nH^
zd$O+U2NjMOPex0Y4cAOLl2uhCY;7ZRb0Sz+Jw4=$ig-RhGwkRVUf;~cQ2AJRPe7xL
zE4$<(NZUIpDJe-r_^}J0mz&!bGZwF45j&o)Id%!U0YTn^I-7p|ViH!L_4~&Ofy*-<
zJxK`*ew>+i?ZV~9iD}1tuFc4lHNN}Dwd?)<9l`?4>sGONzLIA?T%Y!=e$}d;yLacb
zwfVS+F;`Xb1TQz*v1^jl&M8bWqP*tjTh2m?n53k%v?)_l4|6!WI5lqA_}2Mnd*sW6
zCCn);p<=qcf76qGtzUoMzI<J3;YPK8zgqkIkDl|ZxM#y08mgC;cIFaaHf!U4i-Wh$
z9f&%ekmWeR$LIgjCI8OPKK}5@ku#^e&vX0u@H95^e12|RQ7Kq`i%+#-#zH<eHa11Y
zj7QL5fA~0g;*^&8)_jS7{{NSM+YrdnEI5-<IQ<v@einx3E9dffY!dh&Kechre+dbR
zf88A$%F3nL*iVaz&fQ)2_wjW7`Uk-e*R1JbVp^l6b!XnZM<-8idh{sg%o&~?%F>o)
zk_IfC%Qzp2E|_uV*thtwZ~rH$u9>WEoOkEP_h)DS$M0L9BcGCzA|RGF!>IM(`ToYq
zmu51*Ver2#{C}5LVj`%^8WzR|b?L-Oj|4<h6B2&3mzdaoI@`nZ|NsBPHWOwsR(YwU
z$m>MSa=2&Td-SY8@I()`D~qJh{moBQ+>t;3a4YxwKDpZB@9(DT$JcG%Xz1y&Av@!P
zg!E@^&BBcv4Ijk4?ot=qc+5nu#XdDbH{_q8;os*E6IZNA;NtjE|L4bkZt;Kib!R^O
zwPv=qZeFC++|1m(T6^m3`5Ft)bl9@~`QNXf>QKVTZOp-8U|?(vb?n2($>KtXd?ZhN
z?BzC0Ft`xJ92Ir&QAmy}2Ul}*^T9(-7Y;c+xDW0_{{8-5URqjNSeW_P*Q-a4*dDyp
z`a`6_T`bVWF)BYQiYF?nr~d!H_B->A9CPcJV$oJ8Qd~Ac094Qa|7QrbE-^j*>9c?Q
z-U2;5JSWNp3>9M?T4t(UaFH;$#GV#)utfISA?8C4>o_E4%=ou?v+*IGgQDwMxY?el
zsTgKHm0wl;LF0fjcjk&k>n|)ZR;W{!)z;p^2AQ2rN=#2rOlg@enK6}vxk$;uOM{1p
z=YQc8c^(g*HL1tyb9(vT9A)ckV$|i6@X%M`n8xBF<*}o%Yl;jz&x7?|8=CglOaA+x
zj&P@2ZovV;WgP3QSSO!tFt7dNTx@giQ}c`?jvML@PFCw!6Yt?=m7&HXp`aY;aNz!a
z(9keUbF#BtAA8XpgFj-0Z%#68U6<G(`jw@dcLzsNCZqVz{*Y9I1#Grb(`S@4sGsT#
zbO^e1P)B`&Br|h|k64DS)Q)NO|0VwYO@^4(q|#UKv6aoMv9ZdXk2&jiGh?}C)3F1O
zDz2z;q&;jj?fi3C;D5U_@4p^DHzp;umps#^G;%+fAK}Qhzux%I|ML)2k`j}Coj>v8
z{r~zCKkIo`aI#igixjRGVrCA~X4!nR(Y`PtRpGd;(PMd`X2x=nBrYQXFV!O=LPwVH
zpZRwD0CNm0vt#Up<^?dr5*o~%Bn_nIu~=*JNaV2X$!uZ_Wh`n+I1pB6kl-T0oRwnm
zpu;9+<7o%S#zt+`#AW@Cj0L=<0xT1fnqlLWNr|bcEmxKQPvomo-Zc4;L)aq+wiz#(
zXC9el*Wq~NOHzksg@+WM<c~g1nF-U46m+M~w_kV~md6^;hC2owvzIx_mKHpPQSFEP
zv9trzbP80b{J*hdM;9+!k3^Ef75{_6A2S5cG(<m*S6SWp<`-XR!<~k|KZUC-G$9Ru
zpP!%0uX%D{zR`y54in$`raJg?v7F{#^OE_T+6=o1i>EF5;+V2ue6?GOG<)^u$sP?i
z*Z=%Cm(gejq)U<1cy>*azrc;EwE_(L=kzeDo$HWtP*LSuqsl&~TSk?qV)9XYm9Iy>
z2S{u<`AC2A1Fi?R*u@)MER7B=Wf1=Jhf`J&TCO*){e5V@k;H{l{5i%Oc~<bS^?Yf3
zDNvDOyr{^~Go?kdNquU9FFOl2XA%p0eq-ZIBZW=||0qYc`}>`%wV_e+^E>}09p+o^
zsv!#v4xRgz-f}mcVSk<^Uq}9gMN@eT1lPE7{JV0%!R2>z<C-N08rHHoZY$4VXiDXj
z`SXL())o}5|Nkfb{4E|=<k04|aJr>a7VjB1wl`V+8BGmS6(WRLKALi0GihvhG?;22
zaJT>HERF(I*)0#JE#xg&I`#eig`Lp0;?M8z-~azV!P@ZO-~asYuQ@$C^3l|_bG282
z6_2!dcGN*1F&>LW0`(gm1G*Yt#GMJyGU#Cc@a5xQ>n+<Mo${og-}(0y2(3NXX!DD6
zzw|Ul_eVeamDdVL_U#a2|CiLvXsYz_h%Iwp85?sO$H7BsGn(uFOYEy_-DU|7iudpL
z*Z&X5=Sz24rK9jdo|*ZR6+hda+VCgWRMi@~oTp9x^W$TGGxH{6*|l2|RFn!6L{)B_
zR`4_|5_HOEm}#D#rk49ys`bHf{<iY>3-*da%>|9bE%*PQF!x72Pe<zE{}WjY6=G%w
ze0X2}>-d2W{S!al-<UjKTYLT-tFMylt74A*J09>&{A+>4nsrAHA6+GvC$;zB-`_m8
zwmi%I3<GMO2y#C7U*G0GUt*pelH33MeBaLhpM_O&Vw>Iu6VnO1w|YEfXZGPWdThT+
zfq7NZ4(osUcYf6U`%^9>!^6gwGiQ#E!PFnx;x?&l%$>R1O|LfYo@1Br*sbl9SKAB;
zHgE4;w|>8qJ$$&G=l(uJ0|Nu7Un)O8d;k0Ue-Y7#-6uGCI+Y9s4lmxo##MABX>E+a
zo$dMm|NZ{{|N8p>lO~<{|Np=9BqwiSo)sTkv}+T#DRyK!Jn#}=zQ5<f_4I_))QXQy
z4{sW?Wo8PQB*{KeNSnvH=-E~6w&&*;UWN3sl79aB`v3R$`iaN1{w1sNT{)(?N9QO1
z>udjx2Wl=7HPGOz`Ty_x&(HUd9-Vpg$P5ch!^h0o&CD~qZ~7g{nJIMnpF?&iPtB%;
zgoGde=Q}zyIEGGrcviYi+T6oKrb3AEp9pjM{+dpU7ebIc{p;J?|I7VPSl@Wy_^rS3
z;s5{t1wMkjaMM7rNz`nX6wB}b|M&m@_jhl}%ijl^?=M*zIcKh=;kP@B-j@IBS8P39
zW^;SRhF!cr3k07$N^f9PR&HLR(!5gha4R>@^>v0df4vTgXnRiHn6dK7|N6Fm`Guui
z%*@cF|Lf=H`}6JVANmGLG%aKEn2_kp>k_lP$#`nazCVlCzn8VPPWIvHZ($K$^+vvj
zrE#X(2D5}tPxcH={>_#~5e+*mn)li^uh4Lgmp{BU+a*)6=)92Jhv)o<pP%nIycOCG
z`}Oy?J@euF`|AsjnOtg=`CxJQlIDTEra}{~|NsC0`Tzg>*5!3RbASE2wKZN(PiEo5
zn3iAGJ1()z>z$S%_3&c9_s0a5o*o`YN1nDe9{qTuFJA<zJ^4TC@~o-4zdya=uT<eo
z4QM!ie*S*C{{MZozuTi0{sFBhTJ1PtGHY7akKh0Q$LDA=*P83N++6Byujcz}N9JX7
zNy%qlO5_|J+4fk<6&vfyO|7^jb}cHcKvZ-_Lx2CmWo?epvlH*zwRujK(2G}F810=k
zTSV#i55vQEcXwp3g8HuN|G)Q-kN@9S?|%N*irFs&Uox+}$*9`7r{Vea_4Vrhb@`b;
zH{{$5XKVW(8F}a1x12R=W(0OedI%`{^}LzjDORvx{rL$K4<0ywenX9=bGG*3iOL;p
zjCBp^`YP+5eEM&Hcx^OKzntQ?FR+s4@9c-?=I-BL_ct{mq3NsFgzs+7JPvYi6C8vZ
zrTQP=x%b~7`oDeszw)1-@;9f|R{r>)pQ3p3^Yi-3%8l2)MC{i6Z+PQ>fa`}6w+UDI
z4G)|?ec<%z58reTPuJ&}ZEn1$>ga?C2S8(fYHf840yjfBi?k~L&3SlPy+sEc##iR<
z=4~+GagpCIxQY4jMuFWI3>cIzED+hE$zgIs#DhhME9y>U*~@2h{dWn?c)nMICBWAH
z)1=htua0uAHvg$DWZ<;AaLwATgtzxfqi_F8<cL%Wb&Y+#>Qh;W=D}qevz>BGrlvff
zuG&<r!+l`arSCq%k3B9Yoax?aQ<Pt$AG+uGPWP&xjG)z{Nr?#wKYo4HSJ9d|L!epm
z#N&lMAn<?Ux3vzv&1+sIG-NrHzrMf!zkbXQ+w^nWWv%P%s=j>xU(Y5c_M(T6o&E9V
z^!mi`$Yfc*gZ6t4Ht0@GaCMgc^SgaVUYu*IqgU6%m%(l8^x7<oB_>UloHJK)%NB{2
zmV$dyJ}I-AmA*dV1eH%GwyQw%QDQ<u#rJpT54ZnUQF-9KUt+?tM8TtMH8c5r^lSF=
zcct&E{r&&J!TXcd>(|HsPrkozzKl%Iu3b9Ywyk-8@9*txyth1)l9DD&PyhA#xjZ+w
zvYS;x!^T2U2_9v|NQVRS?Rid5bG*4`4UeiSPyIgwpLu6MOa1Qb<Z0(OmXI)zkTP1a
z%BZ8$kb}eE!^el#`F#HsZcNqYDSy9c3T$L&ip+oiIX|-V&b4eY`5>>Nsc?SZA+{3>
zIa!VRxmQ>HtN-!g|JT>`<?sL5R)0HxbMyZ7aeqI*xEL=g`fP(inY+80n_Jr%pCh({
zMn*nKNjw``-}CGcW{+XnDf)AMKy&hs2624}7Z(Wy^O^T|9!|(Uec;%!1&a?Ce9<_(
zEte<#oWUH6g9i>BQcyjrpmj<?<;a3XhbGKCsi5!w#zn74sPfN@hi{{&h#<o9$M5M6
zKR({S;n=hKB~uUgtY~H5Q^=+Gzy9~P|DT@jpKboX>dTAwo74Z#v-|t|$4AiYVqu}0
zt?jfXCO&ufFjdtwKEAe{#q|oefBpJuucyb>^TH)LA-CN~VXL}?LDtMf)_;GKK?_=s
za%-5|%z60Ko2|5zEi{x(Q<E(>x9#t5Nj~|s9Yw7VZyLAV-YyvtDaj|7^5A*;iPOms
z9xASt|NW5p@NDxJ<)@)#Kw?5dLQ2YypP%0!@AqM1>gmxi*7<)?Y=Tp7(>5~+y_g@y
zrLX#%+3V|nAN%n6`TzU-B`Yc=b#)?QVk2#BBBoBAX=y2`kl8VL@<aPc%%-L@>;E>%
zTO4E9Jdsg(?E^2>!`oIku+{%Js`+!`!#mk2r;oK+Y??QF_Q5Mt4qlnEa7D+%qug!E
z?h<l#DHVTbJ-lts799NMDBHitsn+EZdH3CV3LKzSC@9bTo&E6X>Hqxl|AYk`^2!Vr
zE$jckLiB9Cn;U4_jg!N`!fMl}&&D1e1}|T1kdQRg)-pOM!`^#Q-u|C`^|$Z8N?p<?
z&7SQTGw0#`{q;URGdh{i9rcl!vGrsDn|-~}96R3B;zC0MqokeYvu4h844QQCaP!00
z;cfZ%CF=gA2#BUWc%X16)X`gYd!FH)-Nzfs&O+vqpUk!8YH{Fc-QTk=f=8JnEY8uU
z|9ktyOjbb_)95n~_)nYNey4r8wdKjg=@Vpbm>v7bk@&t^xb@oRgA7YzKYy;!RPEP_
zRge6p#kFzTs!7WXQ#F<P4OS<u^_#qHZN>ba(z7=5=InNsFU{Fb*)L&T;Kgyh{8eeH
zM3Bz=R!5fP0>0;U6JIJ%V|_0#+}B|aF6o-)`F(wP`G5ZXe}8`-K5(jQ&#QLEpbCYx
zQh)eoA52e3Nf8iA+OjdRX{(Q<gh2bU#XJ6;o&A5!n*R@{S8U#BcxTsF|Hzq)Mmv}|
zxg<KbwjVNZ>f%cI@LnD?Ir;sa<IbQU9{YNuKfl{Q+Ze7{*_aU3*kNCDqTsjLw2k4A
z*lPM#ZC9UG@#n|?@9$4Mk6e6a#(`yT{wJT=_UE`n-v53Zwy*#H*DEgkzufO{qj=>1
ze});^#`ku&+h}$zE6?b!3}!VjG`zF(@P|*TQ?xZt9yr>nnwr4Z_*m1pKvZ{ozH!Wc
zzPn7FJW#vT`t;)^&dvLKzM&*qy5<{8=`UVYvB2;O1$99e8Dpb=yK8^1U!kFFllbYA
z(Vw4(51jiaUq0(c{}G8uRyG;X!q#6VB^Q{2$}=((!#xY6Soweapa1Z8d0YB<fwOTb
z5KErSHRNMZ<Y2iz?bP`n|I=?|CU4nh>bNe@(_;4O{46i0vYRYhM1z?&=j1N55;<`C
zvfdoq<Mm}pQHObyGJUURy#EmA@=PMHUVMkt<o`zMMQ&@o6}uEz1a3Aw`jD9*;pN!#
z-R3>GZgApRUi;@n!84vCI$JLNvv1=lD*Y_J@`J<c#DB>NKaQXI^Zw=K`ZH&EK18dr
zuu9t1{`&v?e0|up`W31yM%6COprwy@5_eRKpL+T@dBW7AGmRuBJk#sQY+1B)Dm%Z#
zxp_zDfo3om{{Me7SC{XA0#ECHmfnZ|o;Nx_;N+Xou}nBQ=jMq?KW)PUzhAok<-3Zz
zW{ZIu8{hfH=-|3Ex$vElwGyELf~9>DrD73TOUru#1#2%nVt2@{@b@~rMeO;#bF$|r
ze8@bUdVR{A>$8}0z^O6GDU54ALkqK;+l!wUJx;R=p5?xwajkyIC05;i{|${m6RY~4
zY>)Hs@Vt5R|Nh?J|9^ka*mQuY?r+kL>gf*;v9_G}{+=f+EJDrqL%IKkRW88{k`9O8
z9C3ogj8nvq@A`-T{^t4qZlOePqOcd2L#!i*ZpejOllIsB{r&&H{im59=FH*gmpA6%
z{-2&E=PMj>;Qf6b^L!(lx*mZ>pOp{tD;`a3y6!$h^V!$zw&Ld(+|NFQnyu1jUoY|Q
zPGZIXc{8fGs}&D6yxkYc*Y|(E-QWNJWG8<3|0nUmv$TpY9S^VT3$bOn@m2hv_wYmF
zk$((!c5JMyY<_-Bp=^e^K_(NyNi_AxAIaTfY;2(JAH)CuPMWLs)h6$#lfCGa=AhfR
zA*xfYp|kGKkMoBP9XQdk>EFla{%z&&CF=e!d#!gtk#S#LDrhbFpUNX1X2!<NJv`GI
zi(?qpJ~|*=cz>T^&W(o62M@vq_nkatKr@p!8229PWYny6IHkH}S6kzbXZ%Yhe|pI5
z^yGU!XvuR$g+$#S2l+Gd4<EdbZ_|&LI5#)B;tR*b={Igj*w}8IukU?^g^RiG$*1@7
zhrhn=_{0hA{yOD9v8g{-@R;qK7_ZI32dfSn%>DhG|M2#Fo~9;__Zv4Dcz8Si&TL*4
z1*#oynI?sQxWwepW`1st<E^kGM-uY<u1K)WIo)6$@{mhHQgWJ(V@bybP#X~xCi4BY
zADa?yo3Ziox_nu@H1dfOOC;~QI16p<<~2Hp7rS%Bf@+h0XFmM1Jv`f-M@i|&uJ<Oa
z_5TbE3=M039obN%dU%>H&vZS*n%~Dh{5EH{wq`arcMcMAE;D0(BBZw~wx3B-bZ>>A
z;iR)CPbw%kr&=1SIe0rnu<x6&^ZI(@nEic4l_F}8_}?>kFK252Ppf?ilOyZK0%`RE
zmhg=Qk_Q=|oJ-U;J0!~D`myQ9toHwV<XQYhgXRbnd2p~4ovEIs!Szb&bke+wTkiT;
zb|<XvIW5(%9Q3I2$xl6(`@x|z)cQ7mvT0bh@b!h#O+6}(98;WDelL98q@mQu$#h8I
z`cZ?tztjJ-=6Xjeu>`F<mw4d&#<%trb~$>{*1r3<@IGJ#wIQASW$OPKfL8y%muv3e
zxT5f<u>Tp8kEx8z1?z<m8YWJ33=r5=`Ru;^i?d-SyY|YyTf#5DpmrlWzeJx*;*CwM
z5ARwZX-fBDh)my|zzAAG`tKi4T^)~?7tg6vAv^S>RxAJipZ??5)Q3knzl6@Yc8%xJ
zBcA&Ii}V%UvK<m~8y~GqlCw*(C}yk8<c9{<nz^=Itp+>|_A((K81$U@A6yaUxVOXn
z!c9jJ**SMuN(;m@Wku%ys<3ZXt@ir0)bm!`y@v*Ei3>6(6s=#|f1@z0*x^I}@#t$$
zb9}RpO{`qDj+4oTKlxjHkK7&mWt_K<6?zDLnzL{DYmF}N`!Cgv@|-oNDD_PgQtVJV
zRIuB+CSl&}(CMu@i*;763fFqIi=**<eRa&a(x2zod}uqT7BzX<mbdR)^1)%ZWcFUJ
zW&@E{`xx%71rJ_5y0Fl$r`i8SHjB8s^-d3Uc54pND+lFWtTnD~^YX4<R-jPs!S{0q
z=gk9|kxDG<ezQ#3t8X%S|Gg{a?@k!CIZo0^yI#1rXt$tihE$i+yv3^+YVD2d=APbr
za7*7Lmr!5*nkAZrKC!`}EsBl~w=bEfoZLTmq5#Lij~&kwZ(H!~I`^<4)sQEC?i&57
z3A;3Q#8}M#SgVrB>3+M&Dsk3r;rh!K=e943{I>I)EGXXR%-qj)$bsir{jEiJ8e4Z+
z+}zhRCCU2NqJ7;89_>xt5nju^mM_-Y)~(QSnDd{tBTwX})|gLEPnWkn+Y$3_`L<^#
z=eM7{ef#qf*B+K{Kd<}Fo+8t)BUT}ITJ^D^qeJ8)3!ZivK8{9#ExJw#x99kAIIeFG
z`tqBH?TO{IqZuxf7Kj9Gy1g+j_<H%u?HU_&cHLe7>3;pTyO&;0NImoXOq!3{4!uMV
zna%ef|7N)U+L`ZwiLTX^r>nGPwV9-DeN<D&xcTK24Ws2B+1TdyW$w?o;UQ!;d%oLK
zHn7L$?B{MO5IAa|@UwaX3u}8pN5-|C3;%EZzhI@~FJZ1likV#s(k+JsesSHJkTTKv
z>yDSp%<tT-E;znb`lCsznKH+{DNm=~WS;-?+PlKo9BI$2J%#g%_deQn=6Q?aa^b==
z8Aq~O7d`m>=6i-z-{CUv?cJ#xZwH31&?@wql{8C+Z_RZHw(Zihd0iV53+&9l><j6Q
z4-EZkq}0`+#KycV*rxxZMC|hW=|&GLj(iRXZCP-AvBrxM4eu6*KmjICrPCflEeRV}
z{4cQKKg$@IX0c}VTl-0SUY}Oh6QAZi>B;Jk#q3}|&h_VN3J^F}KSj4l@spB);6>NB
zR*m0ZyCrH{H{3gLqE)X+?0nP?gQ~4B93{jq<+LZ79X|8@to$;2i(Au7mTE+-y?3r^
zB^Ps4;%3Qt*AsmgPdwK4-MX;nal-AK?Hr9k9-WT^u1`_zW(^7wsgMgQvM`umyt~jU
zQKIMgT<*31C$*$L{LOK&q$_BaX~@-p&(~+A&)a-oxNU~oY~H{lj~icaiYwkL#PTcq
zR?*3i9RgESeHZuhpG%+Aa(ZcSu$u6q7uf}W+UDDDi9D9~-7@a4nm@RDpEGMecT<4C
zar><Eo0c40rodH{B`EXffAkXDqRDL>iz<9=1$=6FIj7l6?RvRG`dyoT`Th5Wdxh+L
zZ{|g=7ktDm`F7(+$(J&6QJW(k?9vEzJtcJcNtNpPLYYY|FGY?&EK0OVGjZ~2Ik4%*
zTM3>EyZ$Y+nXC2u=bv^r4#&U+rk^bx52PI~I+=4UCFt!w9cQmgQ~vH-q_gerN2lu>
z%)i|CjS_2Tp4XncSS0bp+qU<{GLur*gv~i$As6(iVu8qU<u=E_D<=$ini@1-X<QLl
zx@EzI>8$U{AN_b8_4hVpE5VvshFmQM0<QC$GByaV$$5LkY_Iu;UDvp{<qkOCkaTou
z+7zwA5*EARhNVN-$>(S1pZ_CreA~tES4<<z?A_PhsHj(0TGz5b=lr(AcTV{Udp-Q!
zv-s<_J{CUadP82%r1y%8I5a0tTBLBA!`WLQIiTV31B(?}-{vMh__`=JQmlKmx`~-n
z;mMNgf-Gh3y~$hDd;_@@FU%@+?I_lHUFqRDZ+`r~f8W=J?U2?`i_4OF_A~P`XkGsQ
z|4ZgrbF~_XxcDzj&|LYZsdd|}o!s8rpNG9+vDUuZ)Vq{Jtn%nohd%-j=d!klm><3)
zdE#|}414W)oB31EADvb=op+&*=cMf8%JUrF@%kBmSD$oOvN`C9rI#Q}ilnP!=t9%m
zZH6t1flp?*cmMf6cbN(AWtZ!QJUZO3Pgzto9kmX7=yLvO^o{fa{x@5ft?PW;zWCm;
zL)vn`{cc-XFP<(BPC19njsASoJ$z*)&$n+J*&F&Q4GbQvE>UoBX;D;Ycz8}$>$~iP
z+;s3_hzXNGiz0sfnR)Qiq=P3q9R2zldwU&2rZ_q?-%@X}kh_$7j9KBXYDZOUESs%u
z+jM<Nzj-Zu^6~r5Y{->n78c$j?i+jH)TxBLa|>2?`|}q4c)Tj(!lIWi0^2sHd!BBD
zRQGe{_;NV}h_vpXGCzndi0xR<ncTxS%ybU+ow;|%NX0rpP37bR&3P}5UYDKAdgt8U
zxqZic+7v@w=Os&cO^e8TFk7S4w5aBiiAd|D#Gk#X8*|ra2exDy^{H@Xc0EtLeZ(-b
z?Y*(@j(6MdZ=P1U?0uwI8>3=hL&f!r+bsGX_0<crT>0v3zV=&9t#st4z-b%{&Hcm6
zz~QuJrXp8sfPgE%8mn*Ayn|{pSZ-@Ko!)pmw$IyuJFHMtX-A``ln#I3p9kMRr!zAj
zo_lz%HNXAR)P<QEU8h&oCj0F?=$w6O(yqF+PlaEs`XsjpbtFU{*_b(TS&QQBwxASA
z#|Dml{vro8J%3l<H1kZ-nEU*U@#7AT%P!YvPDz#cU~}!O_}U$Mp<fSn-LpNSUARTJ
z_P?Lx0X|TXwr7qZSF?kFt2~EPAb*0e;E_YtH?w92FZrNy&-Kc^LrytIlw1$!s!S5u
zEfC49=)3IJ&$~tjf4+-<t6r5pD`x$YmA@9&?w9VH{dh%o=rj@5$0s&z$nBC<(LA%+
zOM1TlApyk?HjgAuA2R4rs+s?L?&p~*fo5To?(KS&_x-#o%ju@a7q;;*w>e(EP&z}c
zwD+|V%jK5b;@x{!`nElq6zr!i$KTXoqkh`X=2Pb3d)iUEuRmS*erL>zRrB9}WdgbW
z%*>5^&H*BA_RZEEYDzqMYWfLARSuKOKUk@`O%7FA*UFg5e#xwNS?2P8>kIbHvA=IQ
z|KSCbNi8<&U2?(o;o8d!_9!e833a`HdD|N8>D;mHuZ7ya*akha=uzI+pJ{adhFROe
zO$tXC59S^EKkfD9$^IP6I+rPOXn0Ray1gP>r+Zzp1YiF5J&)I|dVRz&vuld_g8z5a
zez%=A>UzZOx$`-=VpPhqYnVP=QM-HLf`-Oc*6K|Mrg{}jmH$xC(j#hQ>|9+kA;)F=
zSK$(d3*HYSCG6@zgFTRq-E47jZRYtBEG!cD_NNO7H1xf25DQLXvbo4Vv30t@iI|S7
z|Nk4mxp(%$%}%5F?lzE2prj>dSD$ucL*v1lGY=kM*t?aHpXpDeWbq+ZrUyG&jI>WX
z9G`0Xg>ix$FEjI&WddStU8heN7#KXb-NnQh5zHLL|IPm0xwqZopdr~idpw^RK*pJr
zww3Iwot=1yrS1IujbD;CRxxj8G_7D*rd)WEPv<}hXWFI%m%aY}o16I9txeY2Bd`4s
zRG(SlpWnwf<jWsk6M3Qg+(TE%c~=@jZ#popVeB<ckt{fVkikkKyy5Yu>-YEbsQVk8
znc;XS!4ldeRazMP{W1IDGc$SE`2?N@__VC)-o<d!A?D<i#(>7gOtpVX6WmU+%7vXc
z*t*)j$u!d8!eV!xzkhlB=LbEJfCk6fw*Ma*7jB>K7$;|VuYpbBbliml|NmbQ)81r0
zVd_Kqs^ccS#hsi{a}K_BDprva-~6#<B~S4^2IJa<xq(VT3o6+y6_bn#RP(mT{CFvR
zc#0;Exc;O!5++d7ce+=6R64xG^Fo;Uze$p7Lt2!5j<OW9{WD_NfAvSPM4j`8*=~DQ
zf6!=P=Tp7s;#U9PNGIyZhID?>Z_uSqN;~sGa~A7jd5nyBSXp^wWCFq$rRs0_^w|Dh
z&_nxed%RT+EESKG>Ho>H_FsQgOG0S>)+tFYvFrP<TKV*zh%ph;jWl@RA@l#AVb0xC
z8)__1?NZzX4f^GefBdrr?e==~h^MfS=ggT4bvo-r8Z`gOAK#_;=~N0^&m{TjMr#<R
z&o-PYI60s|)$57J(Wz<|Bq~lBu-(yo{G+ff^Rk4T&4P0x=HP-%sW{}}&tj0^kDro%
z{svxRH~P_k@ruLbqcZkORXQhx@Hj9S7WS;<IdoJ<VuI*`qr6dTl)E2m+J7)M2tGDd
zn`iFa3!;-lK+Xegm#SB05EgEpBjfDN-Mrbjq=dh&LG;!QOEu=+lmpu)_VGAmnL74(
z`J|p;6%BK~@TX~c#o<#&eYkj5a5gb2*Z%o^yx_Z@bD`nkqup2ToP+GWS(GuILs_|b
ziiop2d-G;vW^L_hip6In#hzR&S!1mK!9is=N8{|~<*rU;znZiE^shbK+TOeR-Qm#R
z6GGPB*?l}9<Ve9Uld0y*Aj83n4v8rl8n#^vW8>#%D=Rx<!_B-+iA9^Uls&6!p+TX7
zyP_n6|Lw^Vx-(q4=cRN_ZJg@jE%QO|-r8uMr%!p@`-PsGPJ$$qMd$t*1qQZlGGeQ*
zXJcf1^Rl_|#gWFT-kb+S4PC;tT~-~B(*5_pe_p@@Va7CfHvi4dj9Z^Hl-KIVrDWu!
zSd{gg$!P#px(uMbsVW>?MlLP}OO_a%IQ3{vyu=1m(+d(D(=S@oPMBiBVbwNc60>ok
z!2>@ro*R=mqE6k|)|>b$q^<aQM(KT9a6B%G<1sM!@WEilY~v|Y9M<1TOK`|sWR#%7
z8pFxj>+LDb$mh|U^T#)Q%PA)Q=`)tGG547?dLFts-=634a^p2oM+<(Ntqo#>P2?>~
zdz6rnkdT}_VM2n2ZfZecqJWs%oi_(gIau^HePdTs*teP~yx&{$+LZ$iE4CbXC3l8R
zlX)9gD`QxKfq}uEy0aJVo!d}r-B?_y0b0fi@?z1W1_nk(w)ptA6CP~L%o&1C8=fBQ
zFtl|{IdwqzR|MlM-4up1*Yy4!IoPoMi9$rng0s4BZXPzMwK{xq@)rw3$by4Ko*;KU
zc#t3<p6=n9_~BF1goy{^_CI)e%IJblp#bxT62l3y%`?wkaQPr2x1(wUkHkynxUN0b
z#|<)t4xgKQA$Lg$DDpt=34PSSpr^-{oy}HW-nKyD#VYkPQEadH?o8#@bKD_vZ651C
z5$BYxP4UW=oy^(xt&Gt#8T}>W_Zz*rcXmSp<KdacCALZYkf6^6IVvG3DIq<*pdg{3
zJe`BPc^%V{t1hVuwuK3-KitJTOeQ4cPkVEu@v7JvHue<%pKou2!aZikfe(KyeSfX4
z0J-Y_|3#IL8kpna*iuv5dU)Cz7}(f)lvvKsU=I_tVc6{JFfWtE+Tw>W`@b>`De=t*
z8s@CuQl@+O;^L0Ik4xB~Cd5B#U{F+K<KzUTN^5Jj$VfIvN0)BVUKa~N(=QuOJ$ltS
z!SGNckHPFYM^>mC6)q4}|1jm_1IEKIFN4bJH}}qdc*tB@c5xys-pU^}F!=eg2@0|?
zGqZ7Xvx$nbm6oz`a=I)QeQ2@gDCZ`Nd3jm~Z!z*KEY5D8Q^UE<d7|*C@9%lU^}(y~
zf180!j^EFhvZ(>&M27$WrDhyqWIS_*XU`s<KYw_vtavV6;!#uMId_gHCg#H7&kvR_
zd8p~}Kq;g1vHh_mL7Q7Ex&J+mJ=$^R-}nB9$NAgp|4YoXO}??M7j$Su`gsXC+oT_V
zXNRQ=O$1H0gUmm1gpqy1-k=HEd`p)~>gY<Al!7KmcI;dj%6~0S_unfvU&%^e%b2tz
zGn>M%{RXW6|0Y-boAdCVZQJ|%lI!A9E$YA=^L(~XPf}t)uKxdDN~DvS*P&1E=@Zbj
zLqIra-d{i{>B6O?3zwSaO+9?<7-;hI`0)eB+B$ykuNSoL*uXT&&4uHd?+zXpZA-hl
zlpmj1Uu~FeF6lQv4K%Ov*c~+7XIGm98qs+jF4b2b%mK0L%pF6nRs#Xoi>p_-Fm+Bi
zr1jumdCJn>*47&=E+z?v%>Tvp4s35VKNC{C&qC(Eq1*3uT22ereVixpvij+bA~m<R
zMu7~ezUC=k47TcCdn9u&`K|v3k@XQ1^i~F(nmqr=_3%)^JLf0Q(d6nmXnAVVYbDjt
ziSK8xJ1lW+YGF#G$ica4&t}%9-!8H3koo=>)B^ec|IA%Ot_B01)`#^Ty=(9NpTH*@
z<Knc^xk<^Q>&LkXs*N@8-yi-cv48HIzb1Q?581HaeG=z?Zbh2`+w{b{w{qD(`q!67
zZPTsW>gv%sB|tUZYvGgt?O2PoQ%h4NW~dz#G|ozmyzza4_?bn%$5?muzZcGb3eH$k
zlRBF#4jf~9tF<_HrBj-*0`C@wE&+}tmszGQQ?%{t3~K)K$u4dYF`kgD`R||Xk9n-R
fkPyUvQWP^oPUPjdAN#Ie1F7<K^>bP0l+XkKdlJbi

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
deleted file mode 100644
index 37e8ffae114625d0cc6a07ab2b8dbbb7413a3829..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18884
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3?DlkV45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpNBR#6GAQ!2#4P?6zh3S1-gT_08Vp~o9>@jF-t4Zi$UURypSEXE
zwD*B~HyX|>UY@yb(J8Y~UEzw_qHcbW1xL;s3NZ-sI9!xSlk{9({(o|&XbaQHdrjp{
z|L?wTsP2e4HzTOyqjO0|--)VxX9fmSwhjlEXltcaRWr`dsqVS^JMmueyQpx@?#Q_H
zF`t-0M&7w%&DCHaz<NNOSCFYmUO=?ye=;ZIL8l2KW@($xABw(lj`7*{;-2KqI=R<>
zCVu$*#<@W8M(NsRC%@RvNshf7#gxV4$tt4Yuz-Q(i?iU_$ocN)H^13^OyhP~(wXCm
znkQ?Yzn(PZ-^`QTV9PFT<Z5;hU<sJ?x$n&}FV@}KfB(<lC~AMeN$|XLetIwqYqjEp
zm2AtT@BM#}aPHNw*lCraPwy39-m_=BYhuT|Gi@&%mz-O<)xeFPD_%Ij_IIk)#T81b
zInqfk91&uij@LvFE5uH}74|D`ndh^}V}if`z1Gh=#SXIh$|Y;A0}3FU4>cT#_;FDB
z&0l`y3YN7Z(yL>GSyh*K3mSd(G2V0kaY0{pt;A+#4u-t9YO2O<np?RpO1?gAWc|XQ
zscq8E%9A?`rG%BTdW)7#F;HD2n(Xr4kVUICYH`rMy63TekW{d$)7h&lJYq(;SC@PI
z{P?(e;W2Zf9UFt4nmT9ywigyYSJDtJVC;Kf=M7=@po^dQgSOQC`SJeV-umr%e`~+L
zdw+TP|8w*IfB*AyfBt<5yE>T{AC$PEsVb?{Ib@2rM~A;p&*?)>E`ELiv9TVWouLsw
z?1hC*_DM=kTr6Y!>*Es9Gg@qW|2U-nZ+-puzx=$61xoe*{#5_@ale(jKL75o-=CiT
zH%|Yzzv}3R&+3P#>odzXEdkj8O2bl}&66ibM$8EJ=(v5z$tN__$=^T3ySsP(fBP&Y
zhrHNFD}4nn_RN~}Z=tAAh469vC1yG*g$qoQA9nr!`}_N+r~7Mv|BK!G>;KQ<Hg0hV
zyV|55ADbUuU*E_VtOT;+|9`0~oy`*`Hcp=&8!#c-*Z05)k1&7#TZfwBL%!JC$SSDv
zO?i=?((=vv-~a#o-WnZ`Smcg>IMO&v=K`0|8&7uKg$uNtRrsDbJ^hy3wl-Qa?{3nL
z>gf+3I=8*QFIo35MdHk2HK;vHI-6(CoY>wT>(LP%AAjab$jy_Ue!jgkdwJ8d9K;;8
zjQ{=r|DS&=hogydg61>(ET8}B2LdY;#6@phI$mG#uyNLse@>U!l^@ri_;B<4dmeT^
zgEu!07JSz`ytkTXzP<6EU#1?DIUrG90=BBTIW%U@@jEg1j=9~v8R-->Nl<8dPY=(D
ztL%MWK8h=`_Ip=6ayb3#Llb-UdX}~3-Alxc1ez8Jntom2t++k$|NrzK|K~rvZ{PO+
zzocGlQpDD-hmTy_+W95z>Qa7uPQRck2=$mtXLD!g#MV|%pPt*7f^MGnK7P#2#i60I
zw^wlS#!d};11Yf&iaKYesZCR2+$`j{I>sPbHoCLv+N~e$cVb*~SN}^+Xjs<p|G#A2
z-=rTuCqMj{+_p7al20a~;#14R$L?$%uOwCARylih9Y5rB^>Xm-OF=F!EmJ2??(BSN
zzoz9tQ0juO-XFI)eC$X_UtAh#D3G?XaqF%|^}hWEg)Wa8ChV^_uK9K3!w1E~*Vgh>
ze>bwJK4wti<_<Mjq|-TI!rdDYCy%?IKjq~VHEVKv`^=e=6V(E;1-2~HNL%A9av+)e
zl<Qhefv&*Yt2)}4bg~4ea$M|utRLm%n7wI%Dj!>f=BIDEhv!=J+}>{d=HA&4pHvU`
z%QH(V3qbAoaU?N8;ql|-7cVrHuQ{WyaNy9PLk&^Q3gPk*ml?$z&#Yp}WMW;)cj&6p
z!4;YTDTcq)Ui1cEDQQ+aeV}2Yg~F5b^LftCGmP1HW<#Cr;hn`i=jZcBEOCGuej+kb
zRh7}(JGZ``H#XLglantjj4eWuk<q88=YK*@hEiHeN=xCxp8o=qCQkhLzkY{;@R>RP
zSDbqA!v2hj8e3bvM^gruf?@j_9?1iu&LKud1_nwy1%f3X|Nr09(_?X4<jotNt}dQO
zk9drXc*OOMbfS-a_@a4uwz&$s&Q_=!HcUEv%FEQ`%85HMVZOcdCr{>_rY|{xtMh+^
z^g@ZB5`kt9^5Pb}JgSpiFx57~?bro3qYbN$ygA`ub=AQ1hBqGvr=z^R(HxtT1wRZ9
ze|yW*FVF8I!w5;y4M!HP(n`o%CoZ!lHAO{R>Clk_2O6%ta>;8<>dmRx%v_z#yj8W)
zQmAX`y94vXF1R@vzUDjQ(yUzg$Mq5`>xq+1jGCzjZf?&roMU--Lx$kt$H$oiS|HBe
zGj$_xg8>iA{SHO}W)~5U4USUJPHYH?RLlM#a!hT>B8MGCap$aqmh9iZQRwlMpgY?t
zW83#FPtBPuY;e(ew=zHT*|p5`E}4qFb6YY~uwmB!s}1qrcZCF9R94}Y<bM~kE$FB7
z_f`JIUnC{Aec`<)ahf3^Qh@20pk?y?z*FY?KWu^&(~IIC35Y)3vgJx&m(ivT0b-)5
zi76=!OF7w=Y-43!Q`T<q?=aiGfGFt=8`{*FGMXDEezafn$UWOu!JWq|s`2RedXHuX
zPt~HJ<c6h+J`C5p*RWYmU7#-8({kbfgSX_0HV0P=OGz6WiIh}H5mCvoHQ>y3S^bjp
zd>u$IEOLL8kdkuYib!#>@u#m>jg2NHrKdMc74F&a<zP!lzkz{~(kz7&t?X+&IT8&F
z4Gatn{y66@Qkul7!nkmm=$1WwF>?$K-D#e1s3Xec>xq@G>{ofEL`x*R4LGxi)z;L3
z!9`%_pM->j3l|e7OidLKOS`e9^Wi1cHgWwIhGA^rNM2MqyScUX<dqORl_Q6)hPr!p
zPVekw46AiGP#>YqxO48u`HaUGh~4U7jp}Au_Eg`j@XIvKoE)ypN>45prgTp@9(Pgw
z-UL}@dD(*!PaK{tGBNE?1QlYxr$4-)*miff<hS?f6(1TNzK_3kOcr8c?#YAa&L!oq
z7nIJ<Okb|4b?O*HZuKE<9*%@2b5jXP$pqHJ^-H=MI2X74;Lqk#;L5rZBpY`oA+3DM
zHr6w?Cm0f0UwIYlTxHyo*A(jcKXI!xvzs*Q-aiV;HZx~R%E(9@IV!25B4L)3u%oc~
z;dyzZ9z`)|U?)C$@GPyo__DXpmTlX@9zJOi|N8%-VuzGN#$u@nEC(BIWjN$}Ch&w9
zC`s~}bNMQ7iu<wQNc|Oy<Rf8`3ru1Xdj3mrJ@Dmqxzw~o&Z2XU#fED-1r1wu7cD)i
zpmgZK`STl+SPwrvZ4x5|Y9s#tzsM65xdOtEi_6VB`nQ`IPE1a1HNP%7;g!gieXs4W
zur&H8x!V_6H!y0-E||`{AyBiz>VOWj#KU#2LhWkYA%!WCPoABqzhcIs9K)6Az%%dM
zh8Cf)22;lcjE0773l!Mo<l2^cOX|g?{P@(W<_j(H7S%mUNK2bCUHsEm)9$`)S1t)9
zr@4Jpa?@%wx0yLXh?lu&X@gYdhi_^{HOk@I9U)D3_*_j{j%qBq#B{TuVlJz>@KN~{
ztpd!<LFMApL?)c(EZMH8?r6lt&&I|U7snPK&lVTgrs^$mZcbvw_r8bs?Pc61K$|9u
ziXJ5-CI$rybMu(+b8ij^6iiBLx~Aq)*L+-Q&qPK}oeFDaDftZvu9{{6t9jV2xHPDA
z#cf?YRXgJ>-;^S`vjsUp8CyB`96og5zySt-ffY+x!;UAMe8UpVX_SzhTu_$w;6cKJ
zXK5B?JrA#HYncZ^im63uAX_h8n#9AsnUBLDEJ(;@-Ia&?oV4U76x(}!Is944&9+Hs
zM~{Z%S9wnsUDgAqMW6iYV6@EIIF*;j!&Q)Zl@}Y^lohUWPYxt2d^2ejdwL=!MwM%m
zg|TsSFE6vUHnX(!;f2mT`f&lqwa}Cl1q$PkU}0|l?R<P@lP5oJHWihepd`)oAeT=q
zVs`V#|Mfb51#?Q-nX^td2DF8+8Z0>ZNJD5t*u@ik2YgsEv;?^(RBPY3WgehubCo?!
zC^6x{bgv1nGiFHW=tx9FNJvP6tKy%NvR*Pna*ZD-Elrpz%E!H#i^pW*<j2V=DNR>6
zHeAf!aee~(zJJC>|9=0U&uMn4A*gvlU%OIzA={dnLYqqz*KJB@xtbAib-{9{C5L#z
z-=(*d2PNnxu~mF(HBarx?|P&eexYHbVaUOEcX{&f2Q9A!b$R~(U*z{lK<pVe_hv3m
z!{7jch9x`^zZ$+vC~_^F%8~Pf|6Z-@xw0OQm^o%jrD6gbJEFEJUs2*ra<gR4y7O35
z{omqOP8lxm92Ae*I*H$!ps`1vIdl4h^>!D6c=V3?`0yM%#`FK5Va@+@8`Akp%}zi=
z)#=fL=g+w~4g0$`1%!w+sj^m_N;srp`R}lr@4v&RIN5r7_)gB{@!(|5?m9TD!65ip
zJ)4du^K`v4M$FSi$|7YrMY`iZta+=j?7#t$l%ymB)6^fI+aKOGZ*y*6v0)kyBp^)+
z3yL@Y{3WHL60v#(hp{23R8?HcllEjvW6Z(>&wT^98Qf;L{Wzk@?4v)$Ex7H8fLqSQ
zcD3&1@m-Nx1_>FCuZ0tJTLN}?eC1GHXU4uKAtB+wrPQNQM;~Z{;_u8EiGB6yKfZQ9
z{8898-+m)2+eN7FGzvFt-n?UXwz}%g%*;+>W8*_ok~}L?*^0I@v1d%>`1FIFxu-j9
zt8&}sk0ItWwh06^toA!N`J4pDQl2MMy6#<Zn0wIuqW6Zajy}H<Qd2i<Yg^WDoUS2d
z&kFXJl6~QZ4F<PvX&pU&TUzquh7AS=3ZX0>b0gFuPBr{sh|4ihI=jhe5#JnTABWUr
z*{v<Fo7fGng{c%hNz_&Oz_3PZVlV@<7IXI0mdGCux5$P+bgcRH;7Q7ct*I5ir#<|f
z-j;p+#QQxEuPE6Uu2|9G;&Nl=Z0!vj4HL`sCY<6?TcX5x$*@uXgG=^qN5)OzhCzqx
zdoCQG_1MsP{*PzYQB(dO-@p;Rv2nSD<BT@R%Rj1D&0q-s<#0_^;lP78XFfdPJpB7R
zkN^BhH!C68I!Vc0n3<WKJv(E?*2$Bl<mI`8W+gbh+Pz4W;X_m#=c*>b%_mPX2&XtC
zD>F_>VPXDXc-;1y#Gz9MSPfD)u1Y_$mZ4j;hoj+x<{TF>#m>evQzs^Ja3vX-rza#I
zer<mT>J#Ha78Z$?7KyIz;sbYCm^%*oKACdx1V{dgB(V(w4G~Wobt4mc>tZsS5}rI{
zbkTAS?y*l@urV;p$1P&A>^!xCrJ`%N46ZulCZ&Mt@e2tZy{QJKsR>Hj^B}%Za-TeP
zs*h9C-bG6GZRQ56*($UWorKz#b5^|h=l)HU?LfR*(llO`Om#QLy3Z4rN)#4PWWD%F
z_*=yBP?ZE<`5Dg)jEoEn4J9NESy&BMteW)5RRZdA^+FdnD{0Bj%^Uu-n+h2g8z^0y
zxL98E(Zi~Xd)igsbTD@ab(HLV^72qrPQp}I=ZKI;o?<S`_+yUpu(LO>)^1*^%4}?W
z(vAg^dUOj73=NMRjr{#%MNzTw!C0QJO3KW8vKl%M-~7N@!%^t-RzTQ5NNirl;Wy0-
zHhwhdU`bI4-l1o+lCe5UF{0c+V4|JDSwZ1s53eKxbM>3jP)F((&Y0n`Z0XJJjtvLB
zxt}cLxDskpF!BG2Zu5m4^BN>1FMeC~fbqZKBFXSy3LGc26CN;ZzTuGfFiBOiV2P-V
zn85Wb4U>5!JXvmPh>Eto$Y67IRnlz-)vgTx|0^jL78Gn?VYkl8{%l~lkk?YdOh93)
z_C#}qgO}M;x&;?aIB@ZhgEz~LS2`BrlmE&qe+`%+rN9t=MJVUR15dGtQ~Vap_1se$
z8RvGd<Zg<HkhpV4qNLPwbqv()W`!Fz8U6h6!qs)<=8XpzPh(W(dk}NPtMJOy`x@z{
zilP!c9=9$EFt2%YaI%Ysmi-BDRhMrK7raDb${uDm85lfR>v$ohb4{d#q~wpEl3H34
zTef(-*a<Z|Y~h?a4lXTQ8yGhI`hTLZ$7hjOqOaut3n4;=0ga)Co@x^mxfy$o6n-=h
zoOqP;`obNT*w?u<OzmD47&!6Z*<MEnhW@P@P_wiO3ko+{SiQ2a;yUaf+&Ga(BK1K1
zl3ERm?+cPvxx4<~z;=Io{(iffA0PkzeqaCJpquN-BJO&rMPdg+rx_%z`uTf0sEFj3
z|5yLtZ<pzkF98#_CR}qobB4##l4sSbBYfS^l$`Y=COBBh=n})eVBVEGKlD$WD7luy
z(M$5gaWR9WB}?i(rm|k+dTP&n`2YU;|9dMxKYepky<e{O$CsC&{<ESYpQoo@Wo4YC
zq@11Ik8NITOLiQ;<&&78ptbgar$f(;B@dcrZ0h;{|9}1efBzS|*UQ=d`uyqXeSZ0W
zhK7HiKmM(+e?BASh=Rt66K8xvl3HDK*JWBR;T6zOir5(w!=tImqpQoqFE6xB9Fm~t
zSX5NpShLo&q-2L7lenFCQ|E)(^A?(F?@?~P8FO^v-T%f0fBygYczkEEf7$zewJ$C-
zYlp|x{rNGuv9WRH%)sE_8;6}wp7wSO5040#&^vP`<JH^~4B20ljX#9D&B*$oF4%J7
z{{H{*dw-RGd$V5Fy6)Z0&!4}&-9O>#LnbDkU%!0%`h5KSdVc-lsj0De<8I6s{VT!2
zi*bh1j&JvW{W4k;bMnI@)`y(X_Jq=%%|>R^-oE)_WGtMTl$4f~l#-B;mXeZ`kkF)i
zy}@9{18qx%O*}jr#+nAx6`$vPIop%*;^yZ5-R1YAHavK9aWQ}W|6`k8-8<s4XVIZ6
zHzTL@_p_F*xMSaw(w=%<g-v{VQqq*fwYLBN|NZ^`?QQ$l*Xw6neLem3bVS+GjT;Sb
z+%Qm8HMFue_VD;~d|QYh_W_&5jN0O>IgD;?ZeO^l>ET6Xrfbkd5K#2t<HqR7l`B?$
zG%$2j51DAbRq3kc{{+qk&SS3czW+~e=$+*Frz0?J!F2ij`*pGV{v3L8^0=JcpN7-9
z)Ajb|Wm^9H{XJe^ztYZ5OlB+FwW3G%Y||M%RXrcrx0M|O#r}^E`A56n2QRPNSuOs%
z?r8jeiEnRzmw(qg{QA02VIhx`lS`!AhDQo%PXGVcvvG5?<>t19ujg1J25BcLT~G)O
z68iXM%Jav`t^JXP3${sbkX8P}|FUG>{t3EAC8B2<t`==kw6FVfZ^Of^*t>Z)n$3-k
z55K>!H_zKM+xq*dPT}+aoUZF6EaPYEi8wwr#vn9F^dL)yax72%f1{k+-)e7eTF)={
z=ivTnj~+U=ZQ4{Hz3q>B=1H-6a$MJrX)Y_%(`#Fy!6qokBnAnwL(?sG?1<pueL7`o
zWn%ho$M&5C=HeIn^OI9rlycHPCF(!_|L<@4n;XrWHs#*mC%9wVlmGQ?@%#VHv-(<e
zeVzMsy}buLBzYvx{q9#@DBz<S;VrwSVPa$B!v_cDtH1r(Q}oo_aP@|||7jXJf9gLd
zZ1n!c@hxCN=p!c|A09R~o>{Ya-n{wo22^7H|9@z@#gZi~GBQhATW=;Dj?@V3Hfw9_
z+%R`x!)meg27`Zpe^);~cKX}5zwhr0Uw!`jd;i1R+waddzrVlq_2J*&?F(J(|9Q3a
zd|_nNn5s6X=HsLHKR@sP_Wu9Bui6{GUitl<=k8se>H5OE=L^<P7d4)mVCv}5aPV|@
z<9zw~Mv$WF&`gVnNXe~RRJLwqP3+{}Y38W!I>FtvfT!l6lSb=>-{0?FTer97$A{WS
z;d+06e*XXO@A>2X`zt;?5Z?BQhi}6)Rh}Cnpqi#&n!LE)ADiN5&mSLG(K-A3|NMt9
z0^6o)OWxbB9>)F8h&fnh^M{1wWCPRGfS{y=gLmIRYPmx*Em~T3=<2Gfs9G92%d9=f
znD1|3WOQIve|^O9bB2>xy*fYr|G)om`}?(##Xq0>UlOx5GBkX5<Kv5$m-Xk{{XNuC
zeMaJ_YWK<+jQ&aow0=K3dp~-6oxHW%r)3TA?@OBHr`_1t-14=if^DC{YjK|gM~*1y
zoqO=&#DTNDmqD#+hX4N$%>>24*Do#=6%q+g8PmiC*H(U%FQ0Wl$L2wmVD7~D{dM_w
zD&Ahxoy33k_s`GoH>cmvyZ`UYtE=X0Y-~YNG8HM3212~c+)7sNF0a48_xJ22Nz8oj
z?j}d<ob=>y#0PcOKU4Yi^_x36n>QHv9fl^sc@{f%Mt=S3`t+%yZN!s_+GjMwToyU*
z-ouo?v2khC&Vq+G8=1e?{Zo0qMoLWVn6mr4`X3*J%M!S*O=+|*Jisa$@K|oOVvmHI
z;h+EKKm7Q3pIf|c;&!$7-}BpSt0m6OQFP=!Ske%x8q?C*=@>QZ;DLt5X6D^Lp>h6>
zpOG;yIXSSfP;7}nqnpZ`D(NS?*i>Z`=BV+#U!|~kzl7(L`{(AG%iAdIy5`Dyf8Srr
z@^@#iuiwAF_IK}<6RvDWrU?I-t-OQ5M6&kJkLPc1@1L$Waev^G-{oyzUwdA5+VHb;
zKew{t!Gy%d&1rBCySVNQ4Q;(~GjI}X7I(8lKC5QupQ9eyCs^5>Qm(JNy@ZuV=*Q2`
z_ut?DKhOUEw};G2+&i>2A82tFnEDzlHQ;URvakR5{p)LcdAmQ0p9oBKZkO0stL)3Y
zPEFmgH%w2DO;wf6(D02Kq|iAu$zsQjh_7E;MMZaBh!S0KHfT-nsd|sOaW<aM?cIca
z{{Q^^{p#@eiDFtS8m3Ntd9Ydk{=UBlW0yZzsUuMku2In7EyHzh-{0SVe#VD|dD$E+
zcrA8#ZL~_h@xl9rN0+iL^7iXHcws{0WZ`ZpXrRurxUprctBZ?7;#%H6cLQUB>K`6n
z>|$d5Z?4_nCy(8`%KrWP`}_U3xBdI;{~w*m^zUI`_#aU&y~r&gM?bv0yq}%FZu5o-
za)BNX4zsrfFPAv6dz*y+f7bqvB`VErY~L;#K&_c%QBisG>Xnv=$cwA^J<c}W;c$ID
z`_J*)e6D^PGc=$5KRbK9rcKbJ-{1H9&#Q6u<@#T6<b{3D6Q`QiLk)>iB7c8<{eN=u
zevw~~{@1rjo1eI;(0_A-!{np4)KU`?64KKrOg(D16B2|@{xWOU`0U!%q@^`SLXvND
zl;g_02boNDhJqhoyuWXMeqODLrA@%W_3{7r)&D<z=upQ*sS84kQD2g$X=KQyAGmwP
zf$hJfq@-QlpWpKjT{(E*!yCz&XZhF`O=e{d@@{QxbPS#9=*@kaO9A3DCwaN#w6vfw
zF(L8vlmv&oFvlKu!Ne4o2?l~smQF1ABH!84@P9>5ef_^bmA|JQ`EY<?!;ymxYtJaG
z<UG{iEaNNSeem<g$MvWBW2(OQJvqY2*wgc3^Hv@njkDr0kBp3Xii&u2bY4V%f~3bK
zF@{_R6nOUNFkWxk{Qv&~zJ;0Wt5t4U-4I}6Q*2)DV*8F&MpO38tFI@lEgu_2*)$|q
z9=u?&=6mMhws(P{rSB^b&+6!!6KZ!`f8n;M_;fZkfi;XQ^DEn2G@f1$T^e=Hde;$`
z)n?Ofeg8X0-HD?^@QkuXKDZU+v_9;}QOlCj%8jA&C#G^ddm0%Ly{vv?;YJgS88iMB
z7fjfv$SrDMY`kfsqQ1FeM9~8w_sK~I7S3b5TEHeI#>U4t=XveKM}jq<Wq&ZP3*?RI
zddS1e%N7_|0_t-z{QvLdFQcX9<mIK~QsOq@w&Z_9mkyVJULBqPtt|yTmjrol+?-ia
z5fSj1|E8Ep;hIdQ&C3!RIE@5!l4s16WMQ3ovM=1?!NY_1HmY!N^kpe{D{7}DBwV<X
z5)j~U|0<-;S`ur>b-;jQPeQwZmZI~WPyfT+j6_tHIC{ORDC(AZ^zwlg-^4kaXLa7a
zDRB7hv1{}Fs}yt07v?+PUOt7%tL56e$yGjr%7*c);;WjQ4~O<&&+G{B`jMH+c%^H~
zxtIIYbf!hRu$<A(hJ=>CjFA!3nKK{!*rV1mo{ihl#eSoA_Kry$`{H_*E>&q|H8MBv
zo+BfEYlq#c$AMgyIV)N8jy4GXopG|2(etJ~TigGb7@k$DbVT@?3=Ir2G7~%1l^C!6
z)B1O?lp`TAQ9!^UoD*V^r;LvekDJ>CHufAr@u(Sx^v}1N+ieIFYYAqKda`o$=hLS@
zPn1;Q0f9Sr&RALf$;s)F&}86u3O?r062}&!#KhEd>(-i#^u3e0SXd-{d`{T-G4ow`
z#kYnx^hu-@$A#<e+ac{>r-v(7a;{impzJP}^|NuB?8QkZ;+H&a_-_!zc5N2x&aR*N
z>FU0o9R|k6uU>xq^x?rx-Nvb>8v++OFf21VQ_`HYY=y?+Ra&3l-@mb_DPzIPlM7aQ
zb~658xOP>pV8Pn63aYJ2ka%+Plwo4J^XO5AVrxvQ!}=GhI~)X}zx;R3t^echtM^}0
z;$Lb?#^krTa&lsVf@YPKesOVPIak@2aa@y{oWRJ;cw=g3hk=%cL3fwY!}uAU@-A*h
z9$u5y%W5CE{~)13RPNNN7mpL5dDm0s&z~3{pAL(|Wv#|Vfgk1F*E&9#!ewP-EWCN!
zG==m3gMwu2?1avo;gOhr_pv+A5m95Fif&$w=7}HoPY_6UVPTh-pJ~c%Z|_`XRT;IB
z`O)mco*tf<m>0&-QF$j%87C*1|Nj<NNZ+ntt2&ZwV$HGco<sIThn;86{D1x0S4Gt`
zZ{_BVhMirTDl0lZe%_qmqMLTJAxtA_8QYpvb({5xzBK`+moD}A`K{Trr{_s&Mt1f|
z`#Zdn0x?e-mkIG*xF!uP_B>@gR#*Q!s3qva_2d74{a3DK^^y~+G?w%fEclxL|M1Rl
z--IF~qwd^U!n$^kO}a}<&khS|c6Mbq#rCk;gk@}No-$5k<uBUx^ZbDi^V{1UBWG$V
zHpWKG*m9`3v-9BLW<}jz72g|-A2XSX#oYY(Tp{fsClx+3Gqu1#HMe8`H~etRk1#Mg
z6h7&ntJLQo{mQ2<z7n?y-c<ka<iy@yMdM&)MaQXAr{4IstAk_q%$W%Z3s~gkjSF3w
z=BYRtq%P1*IIzVpXp*5q!_^BDW}fifz{A7i<HK|98c%HOlkW+Ij9U)!{Fz)(|IgsW
z>BT(fpo!&3;{t`*9URKS!d?}MU&9nSpV=R)`5WJ1(YNY}zMHDkf@6Y0Nx3;DCr$(?
ztFeFhv}wjn&TU&Vx;i&1`tDRzKJ)0IeBX>?k&yzEQXRz7lYg&Yb?VBcLn{{__IVO9
zZL#v<7EYd9Pt1=OGX1$LZ1Um1ee)KRQ^)e4$w!5ck58<wjx8<iOis%Kr(Zf5f!-^e
zes;I~OVADvSS8_htp3oUHNxt%wr!VwWWTthzyIjPz?5S=wUUx$VPR}NERFk%4u(#B
zpjFi285E$&*Z1GbipMy;XvR##$B{qIA2@K}$PoeY;|A$`%?lNmCQD~9gl8SFw^UbZ
zy7U54A0#<VN={1tef|)Wrhx%B^BQkQ!_%Gh7F}F6SGmL@-Hz2ieLCxS|L)ndxHfKF
z$S%#zy||03JZ##dNvds^F0Ikg`Cn+wuN(R2xLQZ#u?IoTGgoa;mn!@Af7Y!3=6NaI
zor%Y`I!*5HPrR(w#>y=rB61?~c{}@q`3sM-&X_4_VG&>giG?J`Ny$y6p$p6#55H?l
zNN|w(#lO>@t<GLt_|+8~Vdlf}TW;1|3H>u8k)iywy@YsR!0cIEGFDGY9vzwe_V)bv
zou58FJ|2JOOwXiAGHPl}=gujyvDNVW;N$$oZY;J|=U}MK1IHOl1)0s911_-}f3A<a
z_y6)T!{f(|b8dfo|Kj3*|9J*~ejPn<_UwYSX9Yx#Jb3H-*Nv^N!TzX$k<p)@&69kf
zVdXgO_xFCrXT>L$mi0&!_H1)`ET8r0NB=^GnJ(o#KBwz%WGPf^YUnmDC@`?F;IOvf
z`1|+w`{Vul|Nr~@_(_UWZk$0vw@u8%F7<sMH&0}hyl{zeT@<I$|Afqw37IE<JU{<`
zeY~WEgv5>=5*qXC5AvUg>-qLs-Rmu6cFf6BrvCq-&8(o{aC1vrqFpO!o^XJRzaq17
zPTquWE)z4q{R@mHC^H;PO-pN-)8p-O?pTkONg>;wPUe2jAD4Nrg>i3rk(8*oA$#Vv
zh8ZHw+k_y)&`v6S|Nl?)?)&et@}YdnTJcx09RDY3$+G3V-giNWi@ER1NBOCL<)xb?
zTvAem52rb->XGQU)~BB;&?hFvye9jA!O{ytpcO~||2wJl)&KwZ_vh#S#)tp^|NpOG
z{#0SAz>HMpTS9?bmNe`LWbI6GNLp<9&Hjvx)-wJrE0PXOy!&v|k>zr0ns#taYV?=n
zxnZRr!98idePCZBERXc<umAu5C$pzy$C8G{PZ{^iFz+mK4ErFj9Qj~X*o8|h%%(!@
zMYS#|#VtQBu?BP5mj^I!QTg;){nS>_%qpm9urBQ1;VSJ2>4pC%{!;r;_QL*#(wF}$
zx+ZoDSUC8}|2OOsNMTDmmEh1T*TJz)s`12?hLx=KcU#nVJ_xdzz;Hf^FNPsE(HK%+
zg3OXy^5LEKmPoEn?<GMkf-7SjwT(BFrAth3*}(MYC>wK@i&26Lk4^ui5V?dbm({f&
zLd+v3GtGMAz;q&j{oj;E&EL=Yy|%(r{*V8!#ZxpmFFi;|O8LW`J<aIAF@ptPPyGLX
zq55vZTcHly8J;W;qhw+#lnw;%eBdv}qAX_|C&?kI6){<4lUqz1Pgfb6p(eBTkAJdD
zKR^R5bHcxWyk8Y-PH>9}oLO;IJ<9rv{kul}Kb|an2A5fww|!#vKXo8}&*Z2%38Lu^
zz11-%lNy5ASOk0|kC=EsQoGa2u)6>My06SjXt*ld16o8-%2hP0AzW`l8hcdhp@xI0
z4x-0PWQ&9j`l$X7Jo@ptwq$3UilEKXa|f6|1s(hUpJ#vFLYP_JG7~Z*ez;!r@mL(!
zvw=^`h`BhYnbWkPU@5;&Uh~8mvQcML4mDilI(YTjfyq@H%seCmxS84Zq@GE5Gh;!h
zhDxa3{`$pJWT3wDl(DZ@nl5v~dC>>!Cdmu0M3z{JdpvB6`CuThD(s)KB*$~31w|Xx
z_=3Cz*Cg}r*z&-im&c;3_aN(sImNv@6NTqB?5lU52x>U~|L^22GeNWT*#G~6$8R~i
zIsN~ie!xnE&0F$>^(sjNFA<-Hn9U1X-_+-*tYc|ZH#HEL9@8P1av*RHPtE$4kcQU_
zN;rOeWL7PNx=i14LYw=-r|imG!hZHo(<$I-H#l&EN22M+ot{$;&xC^OCQM<vaEjk%
zuY}D}{!Pm5{2<M1;hO*b{eAw(7k<lMf$ioG*7C9~2@GV*>E&k&VGGk^&Jsvo#+hQ4
zR&r=3&x)Tj_9QhYOf;6bJx_(v=57D~|C0ayB>wm(>&gpBr!y;le|-FYd;b55O2OkA
z2WBlgu~vZX#+2LfIeR177D+EW!F7R)Cq-}zQ!``v9wP&Rj^rdMnH@(AcW-s(5y+Xj
zum1dp&+1*!(GsVb(Ut%Iy+7XnzoK%Y^z0K$kJk5Cim+Yz)Xcb;L-Fk26+9-BW*l+!
zIWM+fD#=Hps$s^$OV7_s-rM7tSq$}@O5gqc|LyDl{l2?<zmbv81U7b^)C1O<Jv<#V
zt17q-#QZwy;F1fPZT;~5{r~Ih|8sC}-mu9?iPg%8d2(+g!-N(NGfT+=?Zw%f8hWe@
zjExP9gLidIIC1aIWOW|>_(dAfLei(_@Bi~3K0f|`ef>qVT?-XdG}m}L{#QPk^e)iB
z-Mz0SFRyN2@pJZ^==6^n?s*bI{r{BSHXeADlVbko_=Kh9jfKzooNH|lw{nXt<w;_@
z_>k+}{d9-j^Fb5Z|NlGPwEOb*_I~sHe+C8{{`^1BGSjm0q=C{@)r`l>c}~Re%zyCZ
z=KudZ=J|E;`~Lj>|NsA+Hx@sCC}^L(=*68PxQ553H`qk4vX!ey;ezOzh=T^1LWlR)
z^Bg;-vV?!jN1ik1wB&mE<rkel1t}VwK*{{y|4&c(FZpm4$$p&AXyt#Jfm{E{m4@sG
z+{?G`tNZ)?>FN3F<Ntek85tWZ_n%(i%DL`H<9=U<6x*hiZH!xQNlAcO9Mi-O-`~%p
zA8)j$uJ?0GVoa5xFw4B(_Z~br%+9vu@;hh*dCJuP`}6<f<M;RX|Br}}NO<``e$9ad
z2idm?2RsBKF0-UfXW^PL^Z)xBA0s_FBHh|rySo_|zi{)GS+a~jCY71FaOsHy%!Y<+
zm6dJQ<r4F3lLbUo%ff{!y}K3P&OSf?;#O!g^QO?J=jZqL%m1(W+jXa^;mxF@RTs|N
zFA-xf{A%m;<o*5n+uLgIZGJwxg{8ZlUHx8h?#G-JNA5IuUXY17%eP}Wo9n5Bl$3yg
zgbUZxE$ZexeB`<)uRy8#sHMx}&FMVz?G{SIL+}2+ztulJyl>~9(HoG|7xIJK+j&M|
z^RhSZmwMk%KmYH*$&)jt9MRT1*)oyi!}|uO;Lq~TGOy#TV%|xXXfiV`St2oKj>L~2
z5?0od=jN$yyX#Q)u<*0m;m5~ctb`^9P>pP4Q}yNj+vqPDZKBc|2e|@2|Nmd#;U!Z$
z$3COB_y7O@clZ7-KG@X1-0yEe!Gn&TCx<5LbZB~SQ0u5)u~h2dmaq8%OF1nP($YG5
zQZ8IiulUlDcuJ%#`?^O*>7QI~lc@rq-bA*YpMNnGS}5Po{PFGW{q^zxlaBnYmk{p@
z&=RrP8=vy!{r&r^!|Q!!eJMQJr9WxXnVB<fnwofAp5&}4>|ZYN!})~jiMJOHa<i~V
zNJ&eEgi4mZNtj{M`S6)<Qa;BKt`Ey+Ecf3yrw6on;{Sgqd(Owdzt@8@>+k6kXG%`W
zn{fPu;9apRnW1b;981_#*fue>`Op6+Yxk%4VAJ~D<^L1X(oURwxNUp7^Q+|ves_!o
zvbvOtC%$g2w^+g!SUuSzA?M75i3cxSJ-VSp^YH2EJgKQS*6&~NWY5?C|0V6}k6n^V
zO@Q^N{AJ?z|EvD@=l%QpJ!j7FL_B{Hc)i%*fXJWr;{VR;9omZj|Nrsv|GByL(cAyU
zY<zTkZS;OcW!?Ap{yu$rR5bL&qZ4oHB{G{BmETDgsB|b!(@|)cKHV{9&cnaeZR_Jf
z-mdsFLs75q;Q4+>Kfb>XkxH)xO&9{pp7VX11WBNWd}RLX$NUf%_|VUIt335U#P`;F
z{jSkx8cjYinil;1_5Snodh3!G;(Mge9y+vO*|9(W|J%=SJL9J#85_xVCo}1VDmTZE
z@BI%SySM4bOGrpa)cs4bsGs-nFne3}b@0SqiG+%ZL`6lxyt@u5Q9u9Be|Sc+O<aG&
z3TPDj%hdlfh}r*d`b<eXzGGkh+qX2@IGtippU7taPg(lYySx8SPya7t@ge(Y*Zc49
z|37`28WNQB?|c8lcX#*m@bL8X@bt_5ZCtDVt6#Y=K};@Uu}pT5gI;o*K|)H(jzZ=t
z^Zt#7&8xHyH?#9(Uo&`f@9cvYClazxC*+(_&^&qI+_?j%yb{FN{AWKt$o%HJ38Zwm
z5@QHjDLtoKXf2DiF3Z9FKSUfq$lEz*OWQw`f8%m7$dBc*>Gn9TW{;$<DKoQH-%UzS
zoDuu&(eG;}xlTFDcFH%%9zNS}JF~g|1UFkkWWe<++qSIBns?P8$JaU5z1BZSiRXP_
z3Rj~^?Hg5|W=lThpjl;k56!nWYRf%n=9=I1_`Ra)rZ=L|Vv|@NcWiPAU8phP`Gryz
z#u)K!yyu-|4u7vdv0lW2{rUTQFZw`B-v9qUG|A%Lo}c9(AKA0ByItXoU~@aR-{ZB7
zgo0-ac>UwC&FS_2^Zq>k_jmvOeSiP|`T76$_WzX?5>uzlSiI<ltV{$iPee=WOg@<(
zr9X|XaOULjY*1ddpwC4~GBB8dxsT;w!$h^VGTtUpoBDGK3I{%XQe|dlKHSa^>TmM!
z@WjROgs(R`GwbMvVs+30-R*gXcXl5?aOB8=GiMYuP6Pxtrs^t$F$nIjoBQyd?HloS
zNaTZDz4v$hrzi2%)oM5LR9{Tui)d=_UwOb>_0Rs&*Wd5%uK)Kx{mH{b5%Kg*8xt>G
zNii}?+PE=MRIrh6pMCAG@YuO8pPjAe;o*tj_oqGQR-1)TXUc(#VF%(nc@C!>JaFK{
zfBVB17xS!7xUR&;W>oW=&Gw#i^z4NjCoWvj@bG1D+v(|&ar;t!{F(XizCD||dRzT}
zi5W9E&a5>sP!NgPf9}H@$!~t3M#2C8cjkhVgzNottCmP?h`QjFySL^0{<hY+t*v6G
z-Yw%;(>X&%VgKxv@`u|S|K~ptNR&BT7$9QEqMT%Bo8}YI;bL&lm?!M(#MCKEGo@~q
zzK_fljF`6KR5#CpRXtl*UVm<`H2vf3+drHu&)dvr{l04X(i6pV4qM)uwl6*QX1UXX
zb?a+=x2G%XZtu<9RrhN0JaDV$P~D^FyUYK_?M<38tt~J7=l}oudmOp+btM+u7L=GV
zQ!+GE^6Hg{1orEOhqUTg9z0IoQSq_AgX8gR^Z$|(ptTn}cS`>I^Yi$jCYirHD^@l(
z1{|K8xI>Mb1GI?dil#=iS65>@`{}C}KZH0HToM&_SoZS$or5ziAerRQCEj1(`ybx3
zZL9ts&}R4l|Ns7S6%nb#72?9qOF@gfe}1+vEnUVhzo75$@97VJmrq%JE>UG-hK0C9
z!G~9(hp(;<_;!Ck&$ewW=Qvc<Oc#V|m%lT(v#a&<HP|3{Tv5%R6CXY*Z>i*XdFl|C
z_LHx6AHFFwugPSRXKZHnSUk0{_38hK%JnyHEO@u0{QdkL7JH%+4_q`%ba+)|&@+LB
zk-h%E(HzUeAHHZ#(N;R7pwqLC%~JEg%^h95K1c2AL2IHvd{p)`hx*5^;nQpJ!=Il|
z5Yzco!lqPwxNq07dG9AKS@0<T{=dgh|MI`|F*Y*F$TG66`qJO|(ted&)0>Htj1(T6
zlWjB4Uyvg#+&16d(~Qyo@2CIvpmP8E`p3IFg&<Ki&%&-g?Z#H_suy?v|K~Z;$zUhi
z{x*qs$|+XOyMNNt|NsB+u>J(&ivRzOYkq%=5BOl;=62@iHo;js5)u;k_9R-A@mT#i
zaP+8x#t8+5hIMS`btNkpY}VS;o+@}JvyZzEnx86j-d$eKbAO-Wb^j9|+4sKSza((5
zC%k8><Ntg6{~tfvy5e?Ai^RSC|G$6LPE%|Q^Xu7R!pOcoU-{LP`27+kB@z_|c?X^K
zgdU_93EUL<`*-%kui0(e^FL}r>x!8cAg9#H9JR2R-1t!bQd9r`6%)DiEdvD1zyH?`
z|L1IVxu=Kc`}_aTpQI$DACKzn`2x;`Z>0}^ea+)P&td1GLp-~7{W!Ksf#dTrS)0Gp
z&(D|SlR0qxK?1BsJT%i{UrpkU3c;fmJzqLByaR5_N6b>_m-{O$`j_9_-J#*(-QEA~
z>;4?SV9EumZtUvQEb3$o|14eB_H<d~nzI~_w5QA4OZ3UR+eZt5ItQSszjCSHo6}#c
zZZ_BzG(n9c?Kkt%8wn|Y^mYEczh~Q`SpVOs=HH+H_xIN?KFK8f=jY*q_i{-s&7GZ%
z{r!$nQm4Oa967TiNmeU0A;F@w>)~Z}COv3y&$RgWFLg&D^Q|ea?_`p1vA=$*q|ULW
zu&=)S{lE74_kaJlKin<OvuDr$|NoX|WL_>{wXZjd*>T{*FViW%-^({o6l~sMAvW~{
z``JZD1&t2KSp5G#Kk=zoi|7t$ikxZj?|*v5uPF~-haXZEHDlG_Y@PVgUR-=p=$Ru&
z6jY8B{QsA6>Hy!peW@0elO7&s*9ecA<rpn}`iR6jJ}tS(9ZkE-1=q&fKurr@yQltq
z!5fJyE?c6dKJ?#6?%H3`_+r(c-^V|E)jqu3pQrv`z|Tn&6B+i^r|+l|y(pECk|H3Q
zI$?rCezCy)Z3zd$igNDzg*<{9CvJIvpP`I3TW`pgl{P-=O8vb_T92Qn|M(@U_~|h_
zv#{{t)6*^7zq+e|_Er2YZ%aNd;Cu1^f60gl2_D{!^Zk8&zBaCWeb>6JTfCF25@M**
zgyQ@AQY*gmWogWs{b9?ri#4C#%a?F!vlti|ZP;XFQ*q$HIlm~r1*ShfFkY;d5%}<&
z|M1(}JmPu^Zn}{X5;e6OV-6ndD7kI&?_klHj@>mgAD)wav%2~M%yf%&@##N4vD%s@
zlr8(%yuV&@U%k7@N`;?~pZmAD_e=cyC%fUx$LIW@4upNZkxtYR0ii<+793cx?AU_U
zrxg?rE?9f^!u8`HUW<qD-aT-8J7{OX9IMtBiC&(G`;T|__;Il~MD6yUFC_W~V&j&%
zhI|eVA};gWqLiu`*bC&@3%GiPbJt(6XgVT##Lac<&#tQ)Zzf%s!Tl%x?ELvBQ{ud=
zTqWE;*~}@4{cd_}rb^1p<-M*Ord6gzp4+A*ZdF#F7<<!fQO7>_>|K0<-|hz291FN^
z!kg+Dx=<wZSM};iTP2)tSIMWz1dH(d)i$_W?VsFL>?3^m!`E5z;BsioEL*Nt2c8!D
zn|>cy%DSF!*e<Z=o?J@)f~n#H(t_TKU1yub#O%KP)eKw4tzs}C(WvFl1ZS4y6Pu>@
z?G*fH-J|^Xr-@}!tl+`#_h(F5l)G_SZt^_NrsZ>1GAlb>Ht73eo7!pUBkUE+)x0&L
z`&x!?;DqNcf`vX3c1)K(M~JQJTK9WV=B!C7>R-ci9^~AuFXLq{tu6jtv=O`;Zq3|<
zTx|{lE&GccdD^no470X$=kMWvpd8WSy!)ktfxOZoc@bB>kPr9wB_@`Yl}X-s{(ax!
z0~SKZr+a;<`g<dzhADt+sosbA>4zrlQ}?|n^_-*W?wxNF?ae03B<MPb1YS4cy=LSq
zak#L%dAqVRi{PrRMGBr8-k;`_{yqQwh3>T0b%!NxeY^8vtJ>*zoANF^=ng($=~5>6
zQ>_jhc1z~ka<v5TwAfE=kBIJwd$92N$Buc<^Vl!A<=8ad>R6!k<FHtl-syiF(m||F
z9L6n~hyJL!$6nsH<DGQWo#REZ`mFoR-vw>g_+TT`UU+2NMU!VgFT|^#c|K+Hrgg`?
z&zmT+?DD&8z+=Ie>N)T7wlB7oev=efP8StCogcupbiyPJB@PZ}8~)QyUfbTTD}P-)
zr<#{J)OD87afxHDe3y@WuQZhZTPI=HdD(V%pU&=Tvj)g0<(#>O+)V)j$LseTacoJ?
z=RLUdpkhngG`s9Cfhrt@LM#UzrS2&#bN1uAdLi1#^VP9t>30_I>g(V2&GzqGrg%DH
z{qoM&9Pw(*Un(}lY>EmDbyH@Eb)O`_a>Q_-zeJFTM(dO-ra?)!x9EB)vP3<u)Der(
zv*4SS`fYm5bQ|;C*RN}5dfiWs%<#SZ=W{_ySQnGRc}tlad7VmAk~SyqU%u|F;;p}S
zhFm|VgfGmEZfpEJ^K$3yx4gk+){8O@6}W`$H(9>;QK3!HG<%ERETFA~TjttxHzf!h
ztLKthnzYNr<Ng7|jP3as+P?^!n3{HMVadO*;1wq$CB(uRzw)5M;(C*k$GXRDZOU!R
z&F_3yS=Dv4>8H%)nhV0QbtQ9cJVSgBZ<70v)$M#eQg5c;L?J~E=WE%s+7uMd-|nwZ
zk$i64%oynwdFlP;`<rh+{4K)j^q`U<XwmEKt9`|1*+*JWTdtXTs=`O+v4xA^I&T4k
z*@ZSnQi->(n9getnxW>~93;V3>bw5<OP9a}BCADj>(1s4y64Q(?xQw8<iqaJL*B<?
z^kn!{PV(n6?_dIXZqDr8T+InQt@cYEd|RyG)!yW{%hLK+g5Bh3c6V{}<4t->k5Yxa
z9N8Ur{n7B4xUI7J$&;cYk9{W!UR2)_JASj1RYvmI&kjMSe_wLHRRvzZVdmK+Ai-9s
zGf|IAroD2R&d#`*YHf^)5n{&zpS=i6ls@^_zb8D@rvEl?tb6EEmCL0za=&kW-)YL*
zuxP^b8EUiKL<HMn^diLeoi{mIS`u3ryNc`bl1vr9$YTZt`#;w#(%3CN%T1u>UNGOK
zqvmImV?Bdcy_$aEfHcT2Q|9<`H#rEj%`Xa^`0bkE3Fl`glOOPZt1o$b?O^hZdqtXJ
zLaj=Z_@rwiTK0ZVWP6snZ^q2HCnt(eZMt2sKg`NLsq<iomqgIC{70p}^B!B|m_>*M
zEi!ddU~z2}h*_UeoBptwrM_@K^ZCQucE)Vbc{D>qiQ~iH=XZB){8N**Th%vmZN&Qb
zhj-nNt~$2TYoW+tMbD<!Mp7H^Z;mLiF`B8M>U%inSl;o9GY=2EbP1i9y5;Td;9mzD
zEYj@WXT4OJ;$Oonr&g&u{ru9y^L8+T{Ig_^Emw<yK&$<h2}`1MkKQ^Ib};P2ih0fg
zYpz&VaMTN^J`!uYTN)U7qrCB?&pn?>mCkia?)N9B9{G6tUid_dwW{+Y!sYs%({iNS
z4qCk0wkyc$*Xc#u|1UlIr{3y@*}V2CGt?X%B!padto!%k@+K4I?_b~B^50kYotge>
zmd)~2U#fr9m1G@%_t+%U$(+~W_979d01*qm*mePpqXr639ithRsCb^68lm@2=6i{z
z_pE0{r~YvHgUc-?twI9>gP471K0IJdGU4$*+hqE+qr*T#+BoM{i;~CP$jBX6=gi^x
z^M~jDKEn+g3?d>8GqQ|0I1LL54K*|jUc3a6e}1#e&OCZwyp^YK<tif;6@xhz2SFRx
zetzbO-*0qh@9BcSRwvIjyYQ&&bg;_%keRh*%E1#S7A!yh;Av|fQxuoNva^hi>*EY%
zEF3DnvOzbADw!49R2?b!XS3zz!G>FZj@(i>>Z5e;9*?c<6AMA{g!FbXv!Ws%`+B3A
z|K}32PAyo$5i7J{`SFA_HeZ%Qhj^5fIC_12TzYqI+-P_p+$B2FFe1{>rshP!SFOXl
z%X$3g8OYe2DEMQ+tgnCg_V$o@b}8xUAHKO28b}?O%CTY9pYQz-f0ed1vwvJ`bR0B1
z&hY>LoH>S|p1<q;B*lmoQ#Y*W-P#*|QM*F*e3D{BWF-rKg8GdYVoI}}4A~}aIJy4c
z8I%1IKWZe7nat*PoEID#acw4dOT{E>`xBvY0bPq0tn-~3EBC}=jrP298F3}m#1oqY
zSe9t6>%OMK8F_5mLes*Y_lr$`=ZR?6AKhi-d&BIo!)>F!Gsdyy%QA%T|LoIP^nT)}
zr9U$j-_6gs?>BXkPUfuoDWKNe|NmQN8}hjZ2)Ochdff|tA+ksy$#7b#GFNBoiWj0K
zTvJ&&=Ed6x+J98-syEV}vGb%)-m`rMc{cr<RQmQG`@f)nv)`*(CbmY8!?&0x+}1d{
zNq{9sy07`Yv2TlF;E{dGs-8s^_X>Li+&4u0pRx9J{E`pccWW=F$Vjp+S+XiaYRaPb
znb9o^UT@IhX|Al>^lJSMy~?`%!ShNBR!c0^;M(-&tL7@uBDDYi*UYu$Y7P)+vFEOA
zQfkt)X%G`LyLX8BiGth#Wsh^JH=;V4Cb&s=PL298`I)0ii$hbeZgh_E;Y06q-ta#X
zXnH->WNPluZiRc3+^?8!m#*}?og{I?tkz#3VHQW@>qizj((~GvZCv+b-<DQ=@t<G6
z99jCcGeYc}Z0J&*K#^mCa*g-ihln}_h!i=@P8VricX*QoTglwx%%_uPaX3$E(Wug|
zOOcJcBmcPnSBBK189Ki&N7sSN;VZKt+wzxtDrn8ni?!PLVOP_Gt2r*0mhrP4JS6gn
zf6M2&39BEz)%orBGfcGB|FTWuV<D~II^3Uo1bl>p)EXCx1TCA*Z5{mYa?a;=hn^>v
zhaILZRX-$A*taP~#*pp(#aW9EM7{c@VfJp@Bx4nei>fTQrmg#Z?{gR=!>(1>SKgoa
zyX-}PmGZW=|4!tb=wVjrn|PVs*tjHyF;|s=nR!v1#3jvU#%FbWd>JXLR~s7`7@Rm|
zWN<*O?ZHa<An8X^ObG{O8%>a2m?1k8w4*sD;l{41i{v29QYBrXfB#Y=_RM&AQCY|D
z;L|*t+?S24Kb&Wl@wNF#9tyTsv41dYvXb=P>f;9aa)*1RMJz=jF0*X6t3MYI&As<M
zr*Tc8Xsv)tdV|=(9hdZG>4bg#!aik5&;9+9)AUkr?4A82P#-#+rnLOWz1`Cj?^<Ql
z2(RIrcvT^Lq27U64w42+68;-|PCNJ*Kjg|_JmJBWQT6|CvPDr#;=7nHCv`vrFQDB9
z%RhhrZGL!)CQn-%kCarv@rScy+D)t<Jg_f(ZGYj}w+%)Y7&htr%TTrp`gpq|M`T@L
z;v>c=Pqz&QZkM}z-rY%j@%E@uoC9da1Z3LghoB|RPfqg4+fSN&>7gputx%@|HkUb#
zj7#_XwYf$xi!*Vw*)8*m>5?q?t~tTh^}??IH%>(<*|J{$`i$4P-v02`>?@JtpswZr
z|4PL-Ufj0_?Y0aFmHhN6;MTnv5B}Fr`C4Bf^!kdd)NgyYX-7p4<}=mB?l`h>npj1b
zfX%l>qG>;BTV`I7C|K^t)}!+2ul3<AnLNM01uk>~PXdD+S?}}lFngP8H)#KAQ&Ye>
zvz8@nDGgye4@_OiY94jh#X!w~QF7%g`)#xMEUg~qITvevJa>4#(Al#t)66rJ{{J)l
z^YgGlGTY%@r7Ajts-SK14FCTx$~f@zWpLX(IW|Q_Hg<M4Hnt-Z*#2nvJP~?iFZg+T
z#NsX+hoF_s@z&RtI4sF5NS)F4KvU<VIv-oll#k!_56`jW@tz)(xDpmGi%hg;%#aWf
zmHhQf;>=mW;v$|6UCi?=F5C#%9k}yE(~OsGAq`hq)=XvfR0}xTI%BCCv!*J$(f9lI
zZS(CV#dH#8nDPF8_-zU(MS>jXWS5$fGGSWUi<gNTwzuoeRGeXCaGAYm=0T5BOfEJI
z;Zqym6a_JET7AN}kx{vKkt+8Ng@bOOdDr-T24`j-HmJ2a+|D23C&|MDn&$_ZcuUV%
zP_XS+6<d8h8z0{kP4hLH%({gMn;t5KtjU<bcyPthtE(2M6#j9Kb?eOGtJLieUoVk%
zE+t|E!)e}_5KtE%WJ=nB&mRq!EHOB8+?hY}SX7-sf{1j4<coDGi&U1|NKTAmpO~n8
zFpb@?llfBi-inEd_iWmHXP<~X>I~h)zo=;4lLrYKHYR%brf%5o9`nC(tKY$?0TKl+
z_C;ZB%SBIdi#Fx*C51d<kKpD>(eBW<QFw7)zU}I2$*{Fa7PYe$J-x;Pa?Ah!i|Qtx
zJ#$7u_w0lB=O;`!prG9R;?M(6C64P4GRhJ{oqZlD+<1N@pDC(Hh5rDzjm=$QHW^iR
zX1CxkH;vmmg~5{nKPNrA9105Y|Nj>`)}A{60_V;xSanK3_UwbFtt(O`3QR0MglJb-
zaUXG!Sf|r{ye7rL^N~VC!hx#~{KX26^~>|TzGkQscc$Q<&D=CYsKY`ZH86yQvDMbH
zm6x|YNqHf;`p0hFA2&}lxol(qsLt1PYh_UJLw}hJt_{;9L8JY9PF%QpwBV=FOzwXt
zKtoHQaL#?yz|6|p_8<YY(Y3OYO;2x1R2!q_(LewDRWBwiV-6B#X9-x%>!@-dezrpK
z#}L)~`wU}t9odj0dHCJkD>A=SgrNQkew2`qkdT@xAea~soc!TiT0nTZfvIc!^QP6^
z-0VlV9|bx5QD8s#t3%;HaKeH@!^ubY*Mr&$F?&vINMt@d)0ij!Uc&yg?rTu<;vY3I
z^YO7UGO|fZvUz*6>Fcwxva<2<WfUH2FjRL>(H9p#pw+$q%E90p6Id=tiSbCRWb9@1
zml4-D%DI28;ETrLx3^6sZz(~`Ua5v7jXW-{k`)yaKE9Gurb?QaO7ieZ9yy|syHQBl
zbqUj-UePtF%wa44uXy})hU?LOd7k)v1~T^N42snbPgW0^r=$qo<0bU~y!@3XCkM3f
zYtkehCMF&^Ii6p?c$}Pgf`WLOnlv7(o`~A8$<X&$zJ8rU)}sSU#lr&`EgAm*DsAf)
zm*kUckvnBj0r97lNGCJzhK+_FJ{WLt7_3-faN>kPM~A_VorW2ihJSvs`nsJkFI8SH
zedZji;bYYwjMqdO_V;&Pvv!@<RO0ymPvVZU-o)#Apb_<V_fji9Ha&cPzA-cO0jLuS
zI@qMAlbP?q>?l@N$(9ySsn*jY5fTDg`_s}Y`Q+(Bb=O2zl~9KYABBCVj$gf(l92G@
zQ!8kM-haMC+@1uBiV2|8IK=fO?(LQC;bfA8y7@{cv+#xu4*rpXf{pLG9zRODa3#gS
zAYsC^ws*@~=Bzq!u(7yYrS<UKil{aJ|2uzw?bK(wyS)G5o5;4`-z7ogj74Be&d-;$
ztC28j-v>&J|Nl!RbuvpHcr6|l_M$IRPmj&ekgc?o&DE7nP%z`;xl^YOoIQKsG`Fjb
zjmB0X=8KuEn_h^<&2XH=`eel!^ZW(s8#in)sQGuM;IGx;mzR0g$AJpt51&*`c0Smt
zn|%Q^UkeJ5L!He00zyr4Cz&$$Ib^8@2C^k4zUV!7<_wRH4$qo38Y=AUCAK;)j4X~z
zA|fUK{Y|d;KkwmP>$dds67y^lEB?z>ZTP>xo+tml)A^VdnSH-sUQSrA-vL@?2r_+M
zC$q4Dp<zLh)AmFoBd289rU%?N6mFO7uWx_n|L6bwhwtNGXy$(?eJF2d@aIqGJ`b@X
nfo~r`-UsO>bdnO%QA^AWsab}<kG%G~0*Vq(S3j3^P6<r_-pxi^

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
deleted file mode 100644
index e49b5816120d43a669264915f1b6747606e080e0..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18109
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjYw>h(45^rtlpw*nn1P*v
zfq~)we<{|*4C<rI|3~^a3LY>JIHd9|{-!_I@()a%p#=`C_vAjvcg<QL$Hwa8!4P#T
zc9HeB3)daeBUT%m``4ag1_j`qN!ENV20Sjw?8o&cX6*R?bAkZxgdf{o`!*c)vYz!!
z^s>#_|8tlH1r~;>Fe!<w{C)2BwQE~DLnfZ;IdenG=<z0nK*d>?Sv1!A`dl^OZeW_Q
zv?X%a@imb%TC;i{?tNbRv+`HrkIzqPpZ`84kvT0*QgH9`SN^><bNRuscV*H>zGed+
z*2w~@UMo^2PVD{vU$<q(3=S2KZ?7kWmAnbNy6fSai@WT*owKiUZMbG}z4e0c<z$&;
zuf`L?mP*qn$+9`8{E9o;A@IO<P3yG)t^?lpI6myW_g(G4TV6qf#bGN&mp-sjea<#r
zY3Yf=6Wcy+`e1W<*QtHpuhr6b>K*;_TD{KJAMB7bQ>?if3<O+EXL&Sfx;S#Y_}iZ{
z(Kte7!N!7XZhsWAR-T@ATI93b?{dpaQwryCO!PR}J27|lY&CuF(9limjFVjgk3_k9
z2_{<v@4r3wizD}*pM`rZZ*F!KT$``V(xY72FYqj}p*ZF3oze|rwf&h&&ma7~{@3mt
z69Xs-_jERQcTb%>xp(^XSeKSycXtQBKL4Oe?oLhKZf&tHEuHP{jZ?qcD=S}naYXoX
z#FYi2iV~V_4qm_h%g28E|G%DxN8m`qk_YlCNB{i%{Qvv=`u+9)|9^gde{=f(^>Kg8
zpPjj%ef?kUj}QOPO1JULvzcAl2+A-Zn{_&y+uNu1_j?CSi1z9_ej~!eu`xbsmV4B!
zV3(Gevwz!jbDQvSOy&J=FvZbDV!~sg>{$oq>J&_qUtxOAe#;Z4hl|t})Mse@|NsB{
z|NsBZ^Z&)}`0)S7$NKQ~MtkbdfB2$#_;^20{ePoBzmI?T|KGt4l)eA|mpXCe;Hgt5
z&Y#!RJEyLxnUSWpXrZEl_GxXsbBZcQ&K&$Wzmrq$1y96cnPaI52@OXWnYX1hsw!O!
zl8EX_y8Mq{_vM1DpY5yIbayIf|6^a@kf8NnQS-<5_xJy+`%9dgo4liL?!%M9ZM(}Q
z<?Pdb{F?gkx_+B|J)1%k$lL$_OEnxhc<7LZ(jiS<-Bs)LGjnuSEj_BPbxKv|%%PK<
zFMX24`2SD1%osCgqTt>Y?$<h2FFbI9L)Rl<_M#6eKlnSlCoxTk5{;0SpTQfkV_N-*
zrW08YT)FS~c<BFpAK&IbU*g@J#2w}R55MKMb&E^p-Aw|y;IjIzV<O;`wkZ3Nfbiq6
zFtMu_CzcjX3J!lRB>MEhqokzNzvoY!Xqd9xu$$}59A5Fn16TcyXm?B#dGKw~RE>;O
zwr`sl&(=KYXpd-#nVsOR@jqbepZ0Lo35iqv=S#%xO{(}a<Kd^&Hs9G2_x2`N{GRsk
zIDgd<afp@fj}p?-LV|_OOhO7vL$_@Z5EM^OO-*f3=cout>{T>r@#Q_Yq&aHBpTh?1
z*(}Bjj>%th;@Ir>vws%T1uv#2ZCcZ`UjF~Te}f+X72XM78mD>H{ZISxZ|=iKu5DLW
zOX|fYRs5Xv@UHbOHyMb1xsL?Io?W><ecQ&1U%!V>nx6jnL4pI9I9pFsV`C#@xH-#i
z1?9*8|NsAg@RFq0tOH?d6|}gw9BFhmoEF65%oxSnw70c!>kWssF&B<VN(4;ih;rIl
zIXUrtTpKexD0C}6w>^CBKg&fPVw>lqgruY?Q=e|$5Kvfpb=%gdQ>UdRB_uSsax!nb
z)WjIRMq#<A(k0$H){0f?Ov@6p8<Tt`BZPUc$?+CF7Fox$uA!09)KOuwz_I=HJpS{I
zWNc3rd{jDoZ!gdL`^GhYy`Gk7L+nX=l#r70;8BvO$dk`suND<vG%^qn7EW$d;aD=~
zRKr?pu9IhZPP}!zvZP_JmZM&^!o%)a|2AyVo_NzDhG~8L1Yfs}tkH^%p-<%76s8?w
zUg_t!jn8`~WBP%m|Nf=^_|W+9j%C|qbxA(iq#d=hAHEJ><*EU(H|o)YM@hlq;^n2G
z+c#gna9vzbAfffEtE5E0uDKtK8W@ca8ALDe=;isN%A&+nplX-H$?YmrI)~YK{o_sI
z!Mw~u-}SexYGIrfFyZ#X3{UpzM+ct#`+xq!dwEdi?3XiK6L+@Yi^k!}>OAuHL5hBi
z5L=xdJ$RlT93ZfHTiEx{moHtN`s8tP(^n742}~0ix6VpGAlxnC$zr?vaI=Ka!3}O~
zHrFO{^VFQ=aOU!MI+Bt5z-Q%+DpieFhOhcS@tm5Fa3E~$ANP`|fXS@ajxaL*|7ZB;
z_wj;HDu-`v<@x>1(5BWa^9-n}|NnoHN#TlBT|0NbuBdqN;#C(n_h#ouoq}0fMas;Z
z4>+7Y5|Pa3{k1XV?t@j?AGT;4a!ar+S;gWVmEs`6JmXtZ?vy`=C&oOuv!6$!Ir;wv
z!v%TD5jhHzKeT*$DSWt*nJ4?2LCn50A6|(v-<*6HQXVK77aABEa&Q*jxbdQ`T{=4I
z6gTHaV`Jw}hRQ`v4!SlEJXz8O4nGpznV~o#yRfHK;PSa2?Ye<U$4ttK4R$GNT;)o;
zEv&mXfiHi7ne~P%Iu$`|%*;xu2efD1-Y!}8Hp!x9#=}3AM%j{3>(mPi3O2lW?cLID
zU0u2I^oauJYbOp|V!X04Lz6k&B;lHJgtVkbC!1}e;}vCdafL3<(%cW`$`dwQ>mAMb
z%DT)gg>^z@)tcDIHLC0n)7Y=2Fg|y>A#Z08v-i}7a{a@*N_o!DSKcDP3AN3#kb}Fp
zy83lW%8!hU4}}E=i`BSYTA7qD2C-FT9SC0dAggQ9RjD7+8aEzIV$9BUU9suFI}Z!-
zQWZJXWgC}@rbN1|Y@TNE!QA@8gcCjX^`N50qGm?oEt9s{=8_&!+u6X$MM<|XBg;!w
z?bgvF5uZOA8X6iN3SwhcJ|yA6koYb3Vzb=hmknxta=%VIlJMj*=1t6<B@q4fL0Yn@
z@Y=wpFoOk4jVm0P{3Sa?n$I-wHa_r?i3n!C*7E7U{o(cTJmz@@TecX?u{`|YtM*1$
z4v2+jg&#f?+_;^srh3!bdbKm#eo0A5#an^eB5V<}zu5Z-gxfZ4Z8;F?k)Z2gFqu0|
z#ZW-}<_u5v{Y-0CvaDTnFuZ7ismlXR=6jzGSlwz|u61y}g~Hki&(BM$`K4G?OnCS*
zxGjD^hr1QTvN;wJkuya^UH|^|b#^|xWy_J7Gb9q9vM2{N9&89%_{V*&xuc)9hGJic
z#Dc9t3O)O8cyVmm(yE%&(d`{{bB9Dkww{irXt5!?nPbK5DFr7Rw`v`nl{ewyY;zv%
zaHE`iJhFO+<6EE+Ho>Bz;)aiph_jQ<-Mghrmj0BGl$4N|@!%zsYN^5{7Ms)uJ+4bW
zaz3dz8a3^YyRX#`B?jgD$!G8Iw1^0suutFnW9NreEeVetJRasxc=UclLg>%;@`soE
z^HhH~(uq1!@Ym|_@%|rCiV)jsqIU{581PKC;9Aj;%w)Epg>{poTC%~kg=}2Mk2x&(
zIQOY~|J3y!KTk?6di_e^n8fl*xm(+mGS6%<KKfhQ_?>ycj!Rx_i485>o3cN}URGPl
z|9G{m`sQWk?_^hm{o9r5Yj06f!vD2m=TBGF%QLjp?<;t4sfYLq?O0pHtb0Lurk+u&
zIO{An#Rcy->O_R++<yMA5Zoj>G|ytj3<*}&o6XI-%1S!CJWDJsA|xeeCd^B8$;w?Y
zMTTd^XSEGX4V-R93ZgtG)D)7sCr<qMzrKURCelD@<_CWFCI@e^o)v}7GM+svPDy+?
zW*%miR<rnK)A6!546~g6E#7`_0%O#j!^+bA9AUiz2M!-jNIZDp{P_zP4ir4&Irk0J
zyZHbA(4-qPW=Q<}VId?mXVK!brY1)sB4$WR%t)}5o@XR6ag!Z0^Q#qXZmpmA*H$)a
z9y-wxwm-p4`QJj(b343?VnkUCLz(qbGtz`E>-}s$+f{!<<j>KZSq~)Hm2Vt6pv5QN
zve>>JRG(>RCG99+{9U34PHahsL>@hR=HdVS)QJt(FHK5FOl(`qtfs;ytgI|NhtZLT
z$A^dKjK`(`-4ic4)ebS_Z*-`#?TO%WaZ_nA^-WfaImM=aAkOK4*+aua<&PEl#SVC?
zsj*1=a?NZE73+(LkPs1-ymd<=Cr9GO&5ah2-0QSH%*CZ5I5>9FWL*`NA2Sy4@;tFf
zbJB7Z*c-WIQnTR75C4y!;K+A#m~^;)Mxe%{9opUhB{p_1*IU3Mp!X*;Q(3uf`v1eX
zB{L-1>$#bIWp?gmT3<dvbi-!j6|0P%JT=^~!9d3P*oVK?UUyl*esuDbnKbE-p<&p{
zmDiGzY|fm~C}q#eW?ov?c#~Z}Q1anisU;!}8O{f8NoL%XvTTs(SiiwD&};*P<=V&P
z22Na+r}c_fGlg?A^E-HJ6dX8vcEQ@S0&?d*{4_p%w42AiUdadQFBLvj)rTD%-Cs)7
z6&3kz%(1>Ug)!g5=#ZD7+mdapUWp9yf{wmQJsu02r!^Ls@*8%G`AqHEakSH9+Oh_v
z-50F%XBhVGKXX7N#X-x7g;jFRT1mV5w2J@p9$wRJf##s5ZI&USu1?Or!a_2vEH`G%
zT)0@mV<{7}56dApKNFt~O)Qq03fYat7t`Vl7(X6#4{}KUrvHNTUvj}i4&Q_IA*nAI
zvL7?A6>Zx3lrcRax6)xk2S;-^x3dpV^A?jsZjfx6l$4M#VdBH`vTMBj+ow%PNKQ?4
zD&pF-`~P7s<|qF+mdO3=wvzfNAC{^V#Azfx{rSWr4-?rVzcZM!O_%C<;8w_KW_l!J
zq3n<I2M&Ph{rC6(*Z)iT@wuHbnN`Y3QmUavCCbVnd(I+NZnmCl8k-b0?Py_X-fG&s
z+q!v%L<pqSo+K#nV9VyqU7Z_4LZ2q5rMVb=w_nrKaD7U_a_6X7{}Y+}PTc#~KP6zo
zWr1}W4ub#x&wqG*y}i7BUHzvgk3T=3zpv)!$pate&z<WTGsoMnFWjq(`)2qKrwIZI
zZl~+-bS%h>IQt*uK2T@k{r}i~e}3QD8GnBM|G)qK%BOgEMa^;yof;TC*)imcJvTRV
z-fjVgow`k{(;RQ~E;i+w-YUIx@!<y#4itRVI=niZhlhs;8ghk&7khe4E?*XI)%lr{
z(&BEjXF@^)f4ZU3BA#@|yPT_a44&V=IIZu|$G7wC|HtkBSM=k<-mkCM&$amY<L2h`
z`|JLuCL|;zB|UkX`uKVJ^eIowOhxr|OoW6VKX{UomX>zFK(ar!$0LYGBJ0DB-kvEf
zc@OUI=h<IpXjAni{qL{*)6?sBSAFFd7B11&VcD?R*u%phE6b>@-FWA2V=phEMD-x;
z7RG39W?fBz^#=|%KD??8>O$n*OMyB+W{1Je<L=GP%`+zo-g0++y=l=l+nEz)P4nSd
zp(z-arr<yE&v7%QmbQQOe}8@c^0NK%vbi^Qe*X6SynW%;!!u_}9zC+-&)=zg_B^q*
zeL7=?#EcmRXWji5R7mc)al~iqf|E(<fByb{e`@Od?CW)RcYOSPcX$21x}(!??bsoq
zq$II<^UR(e1Mk!~4Cil%t4UvPkdXTRF1C%8Tf!_4=D?b7Yv#_M&wJ@-i%dJq>otca
zvnuDjk+DoY;JM|<NBOO14w!~@cu3yg_t&QGk7LxWUG4n$|Nr&2SiWhqv9{*NpTD)+
zS~Xw4=2m{Y{g`P^mwl(e{D2P60*-f}KJ4G$+jDMCH_hJn=FZQ*-}4t)&F|^qNlfH%
zb?rHJ?ElP}ot|<(l4e~gVZM3&|9|_#|Nio9PBVlC+{5STZK0C?B_$;O|8M6@KbS4x
z=Ek<=q0W+jT8qAStSve?^{0U9hxzvP^KE`g$nSsk^Yi)t{{(~EjQ{-k`Tf_|@7LGw
zumAcgI%3A8&6`cu%u937y&*ZlWrLf2702RL=?4PZu5$k`ocZC+&G~b!_n({d^W~RJ
zrD=W#dVieXkjTtz|G!>Nj!joLsWdS+%isVPr(4fW>p!O4%*@SOO%MP43`?DkS+Pb3
z7Rle**08kyzsKjLGdAaac=xVeg>9Zrw}<xs|Nnn~eeLgiwvS(apYTkh2?t+YEziIA
z=fKI6Yu4&2=$z@&J#<0hN3v8f<18<e5AvE9E{NLH{z^aIxBltV_xoxTSCu)i+1DGb
zi9DQ;bVxz{_<_UDO>S!f&oxNBE)d)yw&qo2q~xN-(1z)psNGyG1_E<#{C@wF%X{{f
zf6fmMT=H8t(Zp_+vSjy*jk~z+wdR(YEs<(dG5%CGvw){`lBRU*ZtZE>@7w}Ka#<!B
z25z~(GAuU1`1XsK*MF<${pUR2|M>XtgSWyje^GIH^vUn1qS=EJoLXicF{{t1{x^E;
zcxmU-Rm%h}O+GL+PI1Ayi>3DU&9A<{;@rvX-!8&>N(j>NN@^4p`Y`E%{hmJ#TE-F&
zlw&u1V$R}YKDB}8+M3SD@6N)f-ruk1m)-UK$4B-{Obnbge?cK|{LLGmW5=8xJ$hqf
zbLK|Xb)JgH626S<R9dxPE~x(Y`ucwT_<uJyJ$?Q0vB=vzyNU@9FRQoR-!FONMn_Qp
z>KhHa`11oBLzTtFnZXrS6MwjZnAxGzoHwf0?q9)r%>IdHT16(~<$vPL3#VFjC_VZ0
zb^81J_5VLMFFL#b{{H{A<?qh^`pPeF^P@>5HzL}S$D)hn%q{lmaW>DK4_;ngZ<hP3
z>hZDl^7bBQzOHDPI5F{kTv9OGCcQI>dTc#HtgH2w`0%AdQgYLF%Ll$(Kl&=zI!@HT
zkU3p{rA&i4=x1Y&#x<eYjM20I-+zC9|NFbu-m~6sY*hE(_wL3=mzX)R?(Nf$ur%(s
zX#CA_`6G|TnLqWvzrBy$y>D-o>&wPTK0Z9<?<cJk55GN`QS;r#Q%*MuOd-}KIYpi5
zoVNd>S5y6pT<#VT<;U?oZk)3k>kXFv`~QFW0)>L7r@W1r88~ZxfBXLIZ2$9fdk>05
zpV-QLP3fV1mT!wrN95)N-#87^i+ij0=iU3$TW`}W@!_jBv$i&~WO`tdtd9G%rp+78
zd?#FmTG&)Ax=r?{e%7XjaKns8^@*ZfX;O#Q#7sE+OMi{sum6vZs+*b$PO9za-BbJf
z^P8LH8)iL~cl)!@U9r#NU;B|(2Af^~7O!eIdUyZ-=Vxc*Po3ZM@Be&9mzFt44mK=y
za`fiPC|j7>4(dxY{QsZibna14&xymc3MSU`bf|u=pV-A)v8ZveJr7U4!7Q~Itz1qq
z&dxJr?k6|QxBnlz=f}Tez5Nms+gI;%xRPhFWx>nT1Ir|g?;6<D{P_L5+^d_HLqI(J
z!6T)3>uZyk>oXgVK^h)OKmW_`<6yqDr-481Pr8H84&z1dTpi@4j;>fTQ8MEucOwhS
z4N=jbC%@YYB_t(1c#=}};{$h}#bF(W3#+8hv@|_S@QF`Qn^K>D@6X>)PvxzxAJ4a+
zvW!12jx8~fjggV7*HQk!fm3a(U9x9E0<ozib)xtuhagVoOPBm~7RMPVKgcmzy&>!>
zPlltW?%B0#b(M6^{QUN|yy#Ca%fCNA&!0a1>(kTnJ(kkpcapd18gvUc_5A1I;mN=E
z=jY$w_HXvob~Q{BYhI-E^OodBUD3n$YmdHp2kD<DIek)@wu~*SnR)5fM#Cqqc4}?A
zbGs*8y(YG8)1<ToYytuaPah`!{P2)HtG6N2{m<cL5A0V>Hw<PA(>=j@-kz<ket+HH
z@W8~x#Df=8Rahe;0;X6fFx1~>T3wZ*3-wKoh7-5=4vteE8f(`65)Wmpif3Eg`jOpP
zMsOAv*B(YjttoAc%*@Qphv(b>-?Fvx|G&TOQj2#;as4-xY2Iq&Wc*N`k2OF~UcPyo
z8TZpzQ8l%;2?A`KoLxMQueS*p7j=PR36vU|Ow1COuU+V(oAX1)Tk^wydzG~bWu6>j
ziW-mp{8_Vc<E*f-pbw%MPZ;_5mdV@y<K*7V#kq0OF`1}6|6E0nhcwJwvSFLQq3@65
z2R_7yO$+qv>uhJQTF<|;F7d<1#0}fh7I=Ma5f!ks2Nn0A#JASs-@!>7NnWhST8#p#
zH_X`hk3V{rgSJxumrz((+oL39XXik#n$u^_NJwd&J$keykn53SglK}V<cq89>q-uv
z`4b;C>*V$6&6C!x*=XqO#&+(a&nCu+t{g^?fxe`~gdgnhCmmpOemGgD!0gb{|Njp!
zi}<kfpS{*kd1nF33;!#tPJR4nSY4g}?`!uQht$Nxkg#Xhu09P7e|{{?^g*J5f~x`Z
zwutlt9(pW2SFfgS+mv*3@9dW!lZ*ZwI(jr9^yq^pE^T2t3_Bya;<q&NeCUS6SCUhn
zn?ky^M<i2z*P(8)E1#HOAGj{xv+a3(i0KE$V2wvlQc8*@y?eX$(ggtpWsZOU|I3T1
z<)*72Idi5VOSb4^o2lN$NjsJpc#G?l3X5N_FE5{VBSJ{sRRRQDTqLBVCHL&v@ZMgT
zKS505h&Zg}eP*qJk-?vKQBDolqyPVF2yZAj_WwVRhxFWs5A6Ba(j1#&>I@7F{-h`U
znm=n*VuHeysii+YJ^g;9dR~9Os-B;of`e1c9A4$Dk5|+pyv1W&Io2^TU!G9U!}C8i
zMdI(Di1{`@x7YmCf8_ruI5bs5C&j}@<=$%-#x<$ThOQjU%+Q)4T>HbS7D2PGOp?66
z+owoPSoUh;70v&J1^*XslAO>a#I)e!!E@&ll9hKQw^#d~z3Dq!Zf~u3#i{6!DY|NI
z+cpGvhE0>rk5gZ^?7;k_4<WV@m)d7aDqCA?vmgIIRl8oec+&ipnuo8f<Y8ko$!Sqy
zUb9#s<H7%WSPK60&z3t=-FVU2##<W_j?6jls%1Rk@jO9imd!UD9xqy)fB#?1-d}mg
zdS-Kr-#c0yJ8zp=e%`vIm^nP`?8^Txb8)JQ|Iut-AQiLe|GwJa=Jo&Pdrtm$XIHDO
zmV~6_m8+6QMiNO$5-lr@CpNG?i1hBM-w0}7{r{iz=btUJrmRALi_rqLS^o>fn3<V#
z!j%s(&hT?eJ>bk6%(BzMQE-3#|Ig><p6`>dFaG>&`R(od_f&uHZftb)>N<SP?Z)Nc
zGw1v~0w+#pe7>+@hT(As4Yy7mdyZxNETGAR^7sGl?)iCtlInlGxRy9xO@#vs79Co!
z?%adNhabE-qu>?j-fM7RzSM^$pqV6w|NpndIC3#4a>yL)zF__R|MrCkZaBY5JLolC
z)W6K2V1~oomOUx$N50jZ`21VPso<pCG3~;0lRmh|_wSQg%x<j0v1I$RRR)t6KC{&*
zJI8K&N1gG4Wm4IvispB7SvZZSG(|16I-2!Aa$n)R!%x{jlQwIj9Qh6?aA+5c3;q9p
z=V52Wfeqb9X0$1X1v$lsi!L;*V0jaD#AE5BE5;U?%Wo?^OR-bBeQurBDt^(w-;bNS
zWX72!e)+U0VC|)+%sD(~cJdw;e?HMp<rh<A){?beu@^xF;s5_hjg_4DY!eewT9z}j
zT`6hE(+|3m<aZ`X*M36k|HWFr+f$_ebsRtQH$#bM$7AX2SqjTGD6U|u7d8ISkigFx
z3>nF7Qs-lfXl`Z{yF5YcxP8u9KDM4K45so^8>e=={X3qpc&YTC+@}8)pS1reG3s(j
z7KpITP)pd#k^RUa%pigxO<l|$Qn)m!@!eqJ<49-;m4DGH#JpvK!*&sA@$gdzMBKf3
zBZ4`vNgR@|-=V~x9i<o~c~FjPj$s5B58IyN%`<AFQlLW{i3vZtP0lqYsks=oiLATR
zpy}uHtAYEUbI@mgXRi99gYpxPX#Z<ZR94NsbdYI7U)4fW#|`T^q4mzs|MEXCusoT@
z&OGTs{SJj7!6v0{<<tkB@)a&0)LGfszGNt9E))p%7d(9Q@(GEah9z9ALZ=}GR8yFC
z#@o;>D^1<pwlwmd$@wThTV&mrMxB5B&RoV*1#~=4XsC4fXs3i|H;W6*w^PW0YTIrp
z@xYgNO|S0_w(KoVt&HxQ8j1})*>W!S>vU|reaInflfvR0g{>zXv!C)<IRrH}s<DEb
zi%E^KYg>6Pv>muu`cU*>%HoMTSPVrMTvg!Vv3SfgU1a6NUU$YQrG{em{7ntaTg=!G
zGBNW(tz2VwT!+6Ps>S(<{kCK5>y|Vu6AZU>6nL0l|G-N!qS~ZllHfAk8J%ey3`UnA
zqf|+bpXK)a?z&cUG^2e|NYn$R!or6+YeIR7S^`%uIyl+F;IdehmE4|&A5Ki51~tR~
z|4yDVH_rU(k6fgn`|QBvuoHgrA5soB2-?ng$!7e?U};BxDcho}JTsY^8Kccbi#24m
zrNO>x>Mu@U{cpI_A}#bn(8>B7r5zqIo-CVn9KBgScpNenaJqDY{a;Gs%T<aD5|S6C
zWq4LF$6LH&Ujd3gP;J}zQ~vi3^-s5qaxz)78y$l*ol=ApIVaBLkmv|7Q(-*l#3XEB
zWCRI^Bu6bR$%OCclHaaPaI;i<Y_I5;bcubI#kw^1s3k)4Uv`SS2+Joi)SFM0fVPvJ
z-ko|B)T+d1AILXbC+c0g!_!Be!U7F9&m|m4NIrY&$QHegS?bHS91u2YmYr~m$u$^M
zaQy%8q-j}@dGOfLS!x$<@T_>)ee!@xp{QKOrVPQ84d#vyy!3YPq&S#XW-$LaC2P^C
z(|6JhTJ$=-KjkF)|M;a3N3MA$9`KNyp|bDFhxhmYPuKr{XV=$Q_jc!q8Iqn{Yt0<R
z9A6tf$g+Ll#khk-lFeTgG!^mnHP4<sJXu*hIu`36D)T1HtvkGm2bu<wnq>AgZb^Qs
ze`)RVj$F+X(aMS9TK^AkyUQCXbNyW;%g<L=*MIt?zHVLla{v0judm0mebFt9aNW@L
z-}$au^UHbrkL*aFAR;l_%&?{1_{$fAJu2THDBHBMd~$O50}b6nVi5r?yS57GMkZWu
zN-owZd-wl;{m~;c*2Vlhci=<3Q`67y@9lR?7MYj3z2I0q&)W!LmUSuS8k1OAR;)7G
zQ+aqpv3m0iiNKq4c^Y;mi3meXJ+yVF$ho7l6gEr~tGL8orl<UP|L5oX|NZ?P@6_ZP
zGNrS-n=xAK_|)r#OP&5?GELvi?!74?A>qgW`HpUFj$U1jGiN?L!ucs&LHPcm1EG&0
zW+ojHZ2C~dWVu#BT#IEg>$<boG(PSveZ5><|6fF8p_Igig|j&Chjqrt2~1AfaI0JW
zD9@r4M#i2Vo;PoJ=FQ`Y-)HdVe*fQlZcz+pf*@m$Nl6a*)BiS4{dwcFWLj*q;I9+E
zlRvsX|M~eopZvdL$Bt#BA6L}x*PfZQR_)(o&w?9L@)5$97kG;mD5xAs$U1f4@L|xj
z)!ErguIi|i#urXi5`dWN^d(F}Qc_^LlA@c@1uL}zwsn1+dZ{mZ^6&rCjr;TQ%}x2T
zG8sX^7#^PbiC_LX2lXtQuF&-QZ~H}O$Fc(p*nE6={QP*LqIiz?8^5`Ec)@}L2hRF7
zm94qTBAE*+fI->zNqA4&(fSI@#wullb-QHzJDbk^`2YX?mdy9z>;LI!pPn@Tyt>Mf
zhQ&Mw;!PJ^Rm*rAyhioDfQO`jg5tpmGfo^hbjYAs?eJ3XCoZ~0)ssEGW<bKYsVvoS
z_M8o33{j=cckcggS7}xEUBKbg$1=<O{o^+`=g*iSaq05Tw{IdgUggLTOME-)(H2%^
zN#2<zjO;!>JooPLG&b^V&oeA3G1yb(TDQIR$3AT-h{=b7E+6jybg=os_AKGpia)JO
zdcME^?=$bu%4N?E?=Js;>GICSi!5f&EL2rnqslmO>6HuGS_X{#{LX=b&Aq&bpPl8g
zwB&hzU-)+0fmJ;{Um*2aQlq|Y`+6bQT$2Uc?E{W`e%xYtA({s?%lP&6{WCM?tNZ`^
z^yz85zCK@VZ5SgX8(Ukw$F6^g60J8s{b+CQURiO1xwMomGqdggf5~fW5)+b>SFBGD
z2vRag^D+<=m~RJhxs$Y=d**~g(^~$x3m?C><8Y6k$hUs^|LfxZ-o5eT<B5s$-{1eA
zo{+Gk;NkWC^>gg&|FyoAI(p?qgYq@bZEo%SKmN>ocz{u(`g0nanOR#C6I*TVqA!p9
zS&m$Rb+enyEhHrW8!ouFyV)e=-+acWe=KZdqkO*n|NVXc?d|pF&)4rSea-K2Dj^}E
z;{U(<6P4}b_xxy1Zt0z&#k_f?^_>HIDtYAX9B$6HH`a+e6A*Q3L%luo$MQb{K1w_m
zRpy8R(1T&K4HB*hzIZg@5sRntx1`7K@BbIq`!mn%?de6X_YK&K{pS6dJMrMTh83$e
zh)HC`JgH(oB-fV7#1ywb?ZyViiF`V`l6Ex-H+D}44UzQlv^6k%IlR8GH2&nf4bZTP
ze#0EzvMqzz+nPC9heg%u|KH#LKR(_+*Sh}9oS$ERe2hPHhG*HbHU9Jdw9eo7{jD^!
zwe{g^Yxf%&-vq@ytI5QA4<+Z_8$AU;bHZOV4zG{z5Da2wW)l~0Q+AiQwkGk!$wZ5?
z9z~-`2M;tjIxy5$%F7%GXL+QrIDv6f6~qxudfy!53c0L>ni(@^t1t)6Yz%17={xh|
z^Yj0!!|QL~-oGa9@7#$K6>a7mzA@wJnVI&-`xnd1-<EwnKR$li@&5lOP8l7##Ln#Y
z=tsY<;~)8Zn$3-k58p<&c~6(<lWi_iRgw7jH@ToR^}@x(2TxOH*vw44Yt=SgUs6d)
zV$U9lipq=O%|dUsIWSy1eDFaPBc#53H?>v&!GUe3UfZX95Sf;AL1e-59(MkJI+34h
ze|=eRU-xJG#zbG=ry_!hH6I?lKRJ2-lP5FeY@Mz!&6?%2J@0R7Qc_ydp*JjvYJK&}
zioOm9?(gTBZ*Oc<*CX)oQIdgi@`I<T51yuG<fIf7B+M{uPP}i|wzpdH&Rt0*Wl0;G
z3+{o1OZnHV?lTlMFff3GlGD4jf7(wfh8zB0$jjrhxB1!?hbub#a(`~_{r!APW_<qr
zf6BUNWyFsQN}gP{;J}}^xA)in{x@UB2A$}(OMSa)f6s3|{NH=}zm9I>#4nK>)cOQj
z_SL1{*v@}&Z+X9C=+uX&#b4ZXm^O`viHXO_iRboq<D6SZHq_fU@3n27E!{j-w0WIg
z2zSxkru{7kKrJOup>|LE_k6+Hg}>^TSa&Qu#B-rEsl(?FD2?j>n`isG`tdRO>hJ%m
zetlWL-8?!=t9`PtHaj~r^Wp#6;r}=|3}kG6%x-Rec$~jQw5F!Vz5n0WkBPs2ezv!^
zW_~xtP|1ti;s<CVLci_l>A+{nfB#7At8<;gD<i|x*2c4RDbKE5Jl@j`=2#poc))PD
zSDNSbHNzV>3@j`RX3S9Vo93jab>W-pgD4jcP=tb-Fb89Pvg<!g{9Lf3N|opTKf{{8
zzuuplYd_oU@10FgpD*>^-`g8GYgWag#Wxe#cm1vZ^+jG^|M0`Z^&%oO=Gpx{edGv(
zz4)Kc&;LJq)Z;(zPxaql_VV(~s~##FMjkl$_BK!ceIuJ{wx^#y7<~Agyfe|+vr|#K
z`{4Qh#IsUuywVbW^V2H6^?=qk6+f5g>XP{KwR3I81W|?c0-(VB|3AsGuc&9~;{O+S
z2(T^D`T6(v|L5obKR^3>U*6q*Vf8v`34`t1i;o}K(A{;&pEdQz*Vpq~SPtLYTYu%s
zjBjs$pFMl_!&mJ?noUhS_xJz*|NHy>`}=z~FcmLR=VoJTv#*!fSC_h@kU6mE&;Rou
zewmhpN<6zIpPAV<MT9LoyUn&*!f#&6j+&W?r^VO;1KIri*qE6`re?UZF0+t=v>%_u
zT5~l!2-Lo|QqhRMaU=Rv!o%L2J+tQ>a^~R9e#jVhFw618agl5X6-D+FSB$UP+_Q=M
zcx?VuS+RWIs(WV@AEde;v#^_$mU}z&b@epX(%QAFxQiEv?3wv&d52l@yug-qyIU7p
z*eOjEx~6pA@civf3ciIw2KCYB5+w|L6`K5ym+_@?W=~X+|JCXs(=6$=P$Pk7OWTwe
z)dtNkv_t<anE<M0|NnQ&D>hIt)eHzx`kvD-_jhaR>GMmy>usvP`3Fq6cyFGm#f-Cu
zn?wEj1k3bnnVFfFZ2a{9>+Apab$@Q}{A_M)e0X#E|B0`s?5~%ssF2uK^Yi~VT~Kj1
z-~NC7-(UW(WOpCt|MT<khi|%v-`?h_|7UREyzlhs=|BF<SD$KN_VZ&iHDz;kW#i>#
zV`gsq{$6s+R>^&}$q$~puTZ>k%R$=-l8vv#D)NDb>dmg3gnVG+`M~tA;fY|4Nb|vo
z4?H<InRwWEGBc~&96Vq7-Cc2Z#@SgnB{$wYugjL_R@(G5)Ms*@RAlI;bx&=imrhu=
zMW-@wgU)V$F1gDN9DRp_vTl8Qa;I!Ar*qEjY&j=`z7xef&7Der{^fnXpQyX#*si3=
z3*W^(HJ7vq2prlU7Guz+;wY1Rf1yazfy+#GXFq(32N!*3=Gt<#I`Fi(vser6X?{_;
z#&%uD+=`~PI+t=E^RJtILuZ}F`$Sd0nOb*Kmi>5?-xu8f?(!RpqU*W7o3CvRJSDX8
z+N9@+p{}WtqOH~OwZ$yvfv3)G{$i`n+4gYP1rx>74l37nO}a5-`iT$sLj)2c9a^>>
zo%m+P!5)*=Yx4?!d%LT2K6bBtAwO#YM;X{Xcjj*7YEY19@#o-dZ<{7}Y<kA4z}il;
z<qJXuTG$+&mTuD3nJ{bj$NPC!`yRLNzWQ8p`GK7#eSsYsPnTWE^Z8^~828mymhD#Q
zk_J)n6DryKYx^^AB_G*jBcIC2xb%bEYc2+_MnjXc-81X8SuENs-C~tfW(($WXXi5p
z2CaPXt8e|!k1U{JgC}$Mg0eu1y}#?hxDDGrSgv7-zuf*I)rGG+_|Hob6`5+y=1CL2
z+*FsCk;<MX@y%EEQk41mOWXRFS#F({ImsrxRCY>gL}=oys=_m-47>B$c04XPcEX~|
z@XOxY->sSMt8+wzPhwfC$+hNal*_*_6&(viZXerpp;W(ZN|MA0qeU;e*Hm2J?_+;X
zom;d5ywt)e@;sxY+Wx}F=XaIve|GNg!6Qc&tUM_rcP=aI)S5+ymaI9WqtH-O$r^Q@
zDRAMRzo$QZe7t{s{QuZ}fBqkAu77_oaid(&gpO|GJvAp6tUdeTpY0YuQDNn4;fHw)
z3=M1koB%BXeR-KDCx_?5vETdac}|_Wz*wrNSfeH|$0YH<><ojYkR%N%99jYdP8VNo
zpIXHrE$IJv=HAAs3AxG1${q44-}gqmShc%D?trGTkEM@?M3UTH(@B;8JTI5Wu2$Lj
z=!dAQn?#p_fsKF7%xC*Ww)y5)T)$X2BQ-}#jCIM8AG5W54sBa#x{TAyF*`)GMt<FH
zts|$UY*)20=5}vl>D+h!<<2PrNxqRgY72H*Xo#Q4>^{N+imfwq4MEG3UE^oD@N~5H
zGUgpP_Hiepw0S}j1E-;)hm+L>rKy)LZIt_;5P400-+rCwx7~402EK9G@gm#bvZhVz
zR1rL{ykxg}*+HJ?9M196UT<0x`t<RPZMOQZ*=>zB%QHQKclBu;l4xSk+$gqm-PfP}
zg(s4E&MB$}{bS{x>#TWf;=bkU;#rQ{B)?7W`+KRk+4`Sn${ywAvr3%bfs^%?xxQS7
z3<O&DOEn91i9}@|T)SMLs=nYgqfW~-r71!I8dD_1*E+u9&#O7j8F^;5;>p>KXMW6n
z;Cy!1**2A#Pb(kj_Md8hfA-Tmt%shUb}TB0Ew`4rBk(Vr!%$8lv+G)lWN2&99KSc;
z=bT>?=Fur|z{Go>u!#<DBfFmX8duFFOLSId@9I-X*tz7rVcW8WxjW+u`+mwS?MU7h
zU1ohPn@v*n$bIz$4#&tvrpJ{NI}CpahIX?Cwrq{~eX;QTnfLy@{m)}3yX*w7Ltis@
zV;7SnPn&&NdO=Ld*6UHyHxGTUXOK_u7If7SiD4^y>|<;key5;(g-p2267D4@<?BAF
ztGwz~3-UT<Ah%zytFAvPuFrm{dJ4m}WBdGbXSX%3JAJZZ(P06DHUWV|BS)5OOPL!v
zH{M39-x(93BjBi+W-Q0Q<=oSyBIg#GdMP@EKD0d-c>Rm*cInTSsh+o!bW7(heQ~)n
z@qmQOjyTKtmU18Gty-j^;k{*<#-`Q-87`MEnB;6<6L#c8$*H1~`%jn5Z+K_ra^wBS
zdB5k*&u30PvXf_GeAA!1U&XlcXHSQWHLRJno9mE+fNTFVM<=bDBAP90FW+GKZ&RO;
zD%#Q8t7tc&fm2iGh{^`BbwXj<g{NjFc2&l{Gx+wq`Pp9kV}%lb8)~=q{eE2fBPey{
zthURY+m3GY6<RHFS@Uqi`{JDns=kZ0x7;<Fzd24>HL`6=%=$U!ITqUXyZ@5YJXvhR
ze|nL|hgz9-o9mu;-rsy{VlY*8Hm}ozU3c=XY@NAMeWF>4(ZNkCv@VGxY*g^<NKP|e
z)bVtZ!IIbe=FF%QSi0l>M;ku&SF^5mB^sO-W8qlH{xV*H``M8?EyyC=DRXT><(k`k
zr=}L)gqv@3*F0j}<FGDk?ky{Ej%;129G4C;Gd7ms9Rl?p3g!{dr1zO`esf0em}2-X
z>-qhr*S4obFYt-nbico!-PbwN$W-(Gvxeo5Y&bN$Yvhk_oOWTGhv&3HyU8jZnTMB~
zZl8Vian;k3Me8)VmTZ|5*0t}r1RuLi|MSV$PpAFaAFX$+urMdtpzZNQ0~gIlGc+C*
zB}xc1DQsx_AO6-zeZlLZ{lTG2G@~Xz=WdM=SA1brwmUxJ=QDxm>KXfg{q|jC4$fI~
zW^Dv{$Yp+lfRCU^a`LR-e5?Ot{yTClPPgTV(~+Pl++u;}L<Ey3aX04dkJdT-?#&yW
zPdnWdYws%CPh|A*y1325Gt#K9YVwxE1tNzJ<hfsdP&q}_X~J_B#@W2rwVfs`v*3$u
z&rFJp(W{Wl^^Se~d(L?t=54-#3q)?~Mw*GoN4x!tK74D!B#UDvH{W+E`KkB0^Rdgc
z8yqR!d6%O}{qDu0#hFicRCheB^$%54^>n%)Eb^<2({TTVEk7!D8%;Zs-*n^onG2V$
zT7$i_X0|PNvjb0?{2z(DmkVV0WaqZ7yZh(#U%{iUf(yI5O5P+~<Xh@txr0w`?Uq}5
zXD28Ib13FdGB0$fpVFe~`J2`Eo=Mc@(jRuAscI7=ce%U{&(P`Kcl^7jkeT!;p@glA
z)=ixBS59;1eM`B>^`1fuiXOo&26rFr;IG*B{l8bMYGBu8&fCj!HMtgbC?srG?&Eac
z)bmzsX+f+_f1*L%t!<n~JC^qye%mH6=X{J_%=#ymzK0(sY&#abQRnbfp2L5vzxrPa
zJs<l^C-Tda!`~P|Me`9zf@`%;acy>ST;rsr&9C_6PE#-Yh6zWVjwpz2THDdu>p9W2
zIl*aF%!5$=XMfKaL_7WRYuY~P+iR1X3#PYT?$mMSxFx1&@$2}XiiF6X#n&GC9{%bT
z`Ka=Yu~*`(E2fSP^Rl;UEM=>?Ip5Ucr+4i3C{E3lS~c@k9+o{WQuOUimTCVyQ)AI<
z4(B$;(}zrEs^vt_{`Ol|afQ~ed5au06VCG}a2W8!eE)Fa-(kNOd3}GTR5Czzo6MPO
z2ufV8_jegKMQ|uJ&E4B;{^5$*yaP@w{R<Stx)eHgv@VJCoTy>=pW(1bzx|)ge~(YB
zmTL0n<I6bbRen$Vo%Q7wfga`a&ow0*5;x@DF#Fsy<%U^pf9l0;7JObW)vInN&AKJp
zvO}uEJN^FQ|4T$_`=>mybZmG%DRqgalYr|H!&J{n7fN^8m&Jaxc_np!xkiN8s;<NZ
znfs3at5oADjLo&&|Kic6JC7c-f&zQVTu^4<Y1^OWe^5G><)G`Gn)LZcTx;0n9p}o(
ze6g{$nxm_f#C>*-Zv7;E$w}%n&PzSM8zAR?Yr}k}^>+Dl&a#OHa4Ej?yCl*4+Q`>}
zFY5ivr01d9;<t_Znts^Nacp?LG53sdlyIAiU}DD<j*ZC^R3B~IJ0ttu*=v*5Eq*W9
zv`FFnO07#%ESBk<G&(j>#Zj{-F;ar<+SFx<^Hyfp{Fl0TNkr4|*>0mev3UDU*H6BM
zxO$c$SE~U}+y1q;mMT0-apiyTaBpj|`&=Chjiw9SKia-^uK2>MAG}a<)+fP9e|Dzx
zy)R~LeQlYvPis#49A}O-Pq+E~`SRRa@=Go&i^91jfB6NTv#WAODW?jQoSL+2*{T!8
zQ=C>F*f@V?Rcy)a8EVG_lOv`n{Muu-*WbENpw&?0sKEsj4#)e8O=I0}C+WH#+{Z3<
zeX7aPozp9(I(co<+4fR6@zL|($Kj9?6colS1_CbglNPs@bvZG8tK0Kmu|r;A+4hME
zQo$|SldSnZTJ+z}+U=WQbo0!FCvQ&eQ+}{qxm2dzJ0~t9OKQq5yG_%X%d@)waqd62
zX6d?@R;#*H6weDP)d=aa{61I6a<n!5@DtbQ`4QcQXN(`8Flc#hxb5w%pY~_{vKMhI
z^$4Dm9I3(eOHOm9+NnvUypQG?UY|Yp+7{iQNvY-C^FCQ$*!e{Cc?F~}nPbS+V8G)l
zzi1uTwWE$K>Mt4hc=PM@%{zFN`Gvw!MORl>>lvEKliKD#F<05M;85)UBR#P#$<tU>
z&gXxR&-(waY~JM5Gsa#Qx0&!}zG^$aYuPG>HL0<lLQM~DdK~OeJs7p+n8dEWed!W2
z)XMF??Q8t-Zo!uGhjzG3oc-+BL<>8H$A%(N!P7)q-wRg+E!DrPz5L8Qu*0^@-pkeE
zz|&%{<MQX`-LMT?H{ZDCcK=@3{6p;q(J_LKk?)UK?^qekY9yI9siJ@G=kEMSm#s}(
zE!sKG$;v<a?>tYKDYW%_g!imVrC)4I&u$A8G2xBeCaxWNC;EZbgwh~q1y2dKV-quz
z=0%3@Nss)N5X&DR=QMkrmY2%5lywGd$r;h7J}rLmBe#D~>coy+E*0RS>QJ6~nvmkM
zKR<p?cl7Lh_%0T-h=Y~2?e}-d3E?8af(|{R!pRRFB>ec)dhpgPNB{oD>C+uuS`J>A
z;OO1`@Hl_l{~KXXCRu&s=@jyq;Al2EcJHYVuSA=Bc@J+)zHqSgL=J~!LqgA{O-3(X
z8fs`7dU!uJVT{}I_+{IJ_3j%s{r{8r;6VbUjlE_zWS5(K9hYKQSlO*?g{dpQ+_`t?
zp4wK8ldLS=9T6)cQuH?HaOananeCkZ@UGXtAKo&LWkQeID6^FI8a5U$PInMdRL%4X
zyzt%Q<i6l(&(#;ShdIQ?KDUh8tP^qC;6iDS^6g{0jC|ECoHb8pNKILk8}Ix^nPH!K
zzy0<Q`;24@NJPznIAp%OK+cv;92%J}NA$!`nA)slKf$=CEhAYvCuy>mqtXkR#96hw
zjdQBzD~Rt^>fe-de`WTPl*k)qYyKYFr1G&a{JJ4eiezs0H4)Zx7fs|8=KS898WC#8
zChb1uki@mA`_gB1DD^3qtgj51r_2(ySVZ90b4`I)&VI6mXKaeWy`MF+?71BRM2^<G
z&TE}9_24(PpPRMk-~9eTFd{zfyb2qe*UppqdoQoO{I={|;nV41EBY<eJT<(JO{}zQ
zZay!Q7Bx%AHAmXF`I<)SH4)a+P0#PHIUMhFI$WmRASL!>@ur@A$D>wttvejGtTWMJ
zWpui`pTL9PTXain_4M3-OtAA?|H+6g`35+ZY=Op2yx<1Iq{r(|2nOBcXi}=k%YVQc
zH0#QwYv+{0nliYabDTc3AwTWt+3oX+=Dv46U;JuTZTh^8(>AGiE}AJcAyu4-(_~ks
zg!>dHjxB1wmVMLh=kFI~GSie~TjQF2Yg+2T>Yl_{O|CiJ2G=f_t<qZ7xo`Pvy%oQf
z=zq`Bk>l0glv{D06I9~wnQI8jNv`vGEHYBn;~a91o;t-`;h3M4`8?#CnVf@wH@nCj
zT{p3<5o^t^<_k~ybAZ2d@z>U{eX|t5v*(L{IJ{<#S)YdM{mUji%%=~T*v|jBZPk}u
zLdPYXB_^f*uq*TlOyHU4E+T99XSwGd^R<FamCIfi$V^$3E7-K|aB072;BCKi#k+Vv
z?GtA4@SHT^`4Y|4&MD_xiu_uOzwT!@&*8c478@K|15Qk5W<%Y2ZCaQ41__aFO`Ems
z8#js`5st{rRGg;F_ee}yL^9&qIgyo2nR1GM{&2`xMC|za|LozYz{Rhl-kA9|8=l#0
z6EuHcmVoz+tu2bb{bWvCSI(Q9`oqp^qKbs_qa(8)%N&Sr-0}KF^|h(nQg6>tD|F^#
z4m`3+fJLcPqxJbh)6A}GB1cPq{lD1#yr$s!0bOgpV-r^`()kH?;FdWM2i{*K=C`<M
zLAk&|%MIT@2;Dl_(8@gTfRjk7+jhxqh1}hDL;f>(a@gxv*8Z7UJ$c9L=SQ=2k8OCb
zanxdwNN%?wACpj1Wtqh3U6WGtw#J4&v2;>c`g?!mv<kW7AFJ)&-`|+qq8PRN^pSOk
zze}<$>r`>fzBDCOkz)>Hl!%L9kkIk|v{&t(1>*6h1@mIWuU-0n#nkt(G$^vS%(3Na
zFc4|pzw=*=gT~&5+2&uW&NW@w*er7J+}j1t++4D2I`YrROcFS>`-I~&HOq6SkBI-D
zd-UYA;xopD5xdrH$<0z;qQdfO);@QRS#60PoA1q6I(m1Z_lN%Hwu%9vI^0ePK0*@N
zfnB+aUk5Gg^UdhAnik>KXFutg#f|c1orNyhe&BFw3e!Fz8}&zpxigVH(&_t8<HNVN
zUucbff6ws4C!;+zCl4Gtq@a5A!#mmL2?EVMJS8`zjf|P(FE3F#kW=%}OJ;{i0AJmo
zhI8&t(<JKtaJ>EMQ1!s&+O3G+>t+O=b6CN4@XC+x`iD<X=P^z*SQCBx!!usU3LYm-
zO9_svYAGUe|0k;5P<WgtZ*Nrd|D1x>sShuO55K?9lazE}qkzY|g;n0d+08Q3MI)B7
zg`O5)n~<Qe*X*Dct7H8?gFO`o17gn>JYqTA&d&oWY@J@7O6r^P-~H{Qf9k0YJWLO?
z6nAq;NGuS3%EQwU#k_OQf+U{>s@!KnoC3t}T<2+4=uG&^S7h0^hHdSK16h36CKY{7
zYg6}^*jJza<Ntin05oWT_j2=0jty!YtDfh-Vp{XC)tk|=n9W(p@v~r&VT1RC87Yl1
zD+~+_?(95V@K);Z=jRtB9U-nsN<0u2Vvx{XcP6MMQQ(O>8(WF`LDq{`92n}@7r#98
zMRSg?|NF(RTFcB0MJLSC`S(Bl$KTnY4W85W1-UOl%y61t(Y$%%HRTygPaF_a%m{4N
z@lcuW@p<DW$rrPo3q*M$qUC-ZHSC`A|G(tDy~#VudLMqvJ+iP4qB-f1=$ait+Ut&Z
zr#DP{wUH~SsMAC&`@_PGhO!PzM2|3U*>Su?ed4AWk7fk%@bIwn8N}>3@Zp_o(jv&V
z*+X$_6LbwGyp_1|iTQFe%Vdi=-I0ufTUuvKVvcOla%_y$GB8?HV0hJFft&rT%!toI
zGBLZ46}%QZyg6N?Jr(MNgFcKqF0r#LSSrM}Ys>Ww#~8y*f9&k3cz)o*k_nUTVpuNA
zm>3&geenK%BAfNH>JKh9^QJVeZgEZq4R}B0K74jIXe(OHpN=F*EtaHsW7~ny90P*|
z%e73*LK(vD3xzaydk4s@mf8~1B-t8a%#(J-=z&Mv46TXWoF<byHH{5ci^eGZ|8M-~
z_wf%e1P_0G%@e+Uk^X$pl7j#Lo%Ag?9MfAD#di5{1xu%oNrhLP%~3YyZ3RbOvHl5D
zW?rMp9+v9TB@qAVp%$~j&WOtbdYUP$1*!4-jbe5k*^thEcy;&-El5gn+G4q38IR3P
zCjYuaJghO9tj3xXLdp}k9_hPw<QEtuEa6PqZ}8x&_&TSi1$&P?VOKiMzvks*d$*%s
z{@a7rr?c}L?WyTZb%ahV9DFRlM$YXs^EEEk@QQ~YbvG4XRFcS+7T`^gYC5K>7C9lX
z=s}TOn()*aLDLq*|4V+Yq}csfEB{||$w7st^7lqDI}Q|lR{Qnypa3K)oL;Vd;3W{j
z&9h~LLvr#jne5r~6L=3b2);YA;>cpTehazO9gl?0mh-f{(qH3x;{X2}r4Re_*{*aP
zH<`ewoN8dCbmRYj<C;Gw3Ldi^-dW6ZeqQ4JPSB3W|NotMnhX;}BxWd8%z8cJsur^w
zo6~0%DQ08tq-U{bA`T{7L{$hfe-M|8@a9|7cQDxCK@oqNjB$cD`;3Jx+G`WkWS>m=
z_?Z3h-fEuo^PpgDs1(xxP4I(ff1V24?3G~aIpDN9%uso%M1$$kgj0)HS?X9%^esH%
zS&^mM#2XZ|X$P0YAG@PH)h4MCsT05cYxg}n-|FawV)aL1;yw`XA2Q-N*0A`1!m_vn
zPZQ7BaW9LDVaSd;xQb!5)T3bLWnO>I^8NYAa#rRH3uC`WlW({Qf6`gk8JSF*|1@mg
zU!QJKIVtflTbsK7hqqRcGVIX4Uw`b~OdWl^xi07b*j;d<=?AxTlUCtU&MisI(|?}W
zG-IzZn@p-=T0??p%k?i2(V`KPxo3&an3g0aQ88uV(w_bGl6>-MH@0*>{Fck~+Yp+F
zW^Ot@-%di}!eqgJe?Xzbnjv>MbhmNB5xuM3yj!01sdcPga^#lR(f37bnkr;?_<fRQ
z$?~kwWY+urUH|aLWS-q+hB5nmD(nTp<$KaW(ZvzcGb%puuJhws78vnaEbOo$YoVyY
zjvXqd76#nXXIh%BDy6J{=%vRa@v&>U+M)QmKk8TB99Z@2e|=m1f6(6k8{7LIUeR1s
zCIoR|(!mn*G>rr?iHb=~>kcTX9Tz@RQ*mQP%o?Q~4D0^%ZvAi|bb>(w%Zd6&S)omO
z$7eAWbUyZyyuQxx&CSCFKMW3ke(o}NAuNsaaBtah{FcA->69HxqL(N0?C?(W`CP~<
z`L>JKr*{Vz{~AlVbv*uD#~w5MS=er0`@mCg!j!`Ia)-aY<@x>H_|AU+H$RnN@w=Ug
zea2d?KYzK`^z!UD8DZOLDk~tliDl}<z?_7qkJg{=cv=5&RZ>Byd<3`jmI7yCM{eFH
zQy5?FSQi1Fiagw|^5!$NRCs?%DZ2m7&;L7SEAKePA9GYQtsvoQANK=);RtSqq#(|9
zER9!vY>vt;GpfjE^|dWpcff=_e3xTzgSX6~wl~&)zV#-)3TX>oE+J=|^y5=&i*PW+
zkxnzC4?Z<qb2mKZQN<~Ki|HqX=FL!X-D5O~;ol+V%kNKY+94(LV{VuI!sJaGr)ivc
z<`8u3@o}EZ%Z)*M`Cf@0zP^5mY&$d>5)XK=S8(yLvFYTn<w!DVA6Vngu`TL@il$9W
z%uhM~15x4+CaS(LSYye4nd_15(T0f%20KqY0u@|44s5701?|j!f6wsGFV~_xh>3@G
z&STE5Y>141;HA35X<EWE_s?d_ypp<jm9ka;Fw{Rh*zmC2<me7A{vC=ZuJSJ1`y)E}
zf1!!>gD8dNGH>pkE%;$@@;lQ3s98qZ-7}YWo;m9>P5uOfzo>J{)#jHfO1=u0tZaD2
z*Q{g-e=at?cSqN>{}Q=AKF9w5e-Xp+_)jG$$ok~dEox`YnH=~45@yT{nzKFPPMUgU
Og7}`UelF{r5}E*j-wS^L

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
deleted file mode 100644
index df3cf2004312ed0ed0ebf1f0340cbfec7fd9ac46..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 19251
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3`sL~37*a7ODM5mDF#|gT
z0|UeV|5B`r8Po?o^It<}XJcpP0$D>7MzFJw^gD932JkS?zW(ZXYsYu{r3xDR%zkp)
zHa(D0n0)X$`@NLNwnCl-o^G4Eu7`ep^ZM89WsC*i#Tq-<7!1l&qo&>wO<Zzr@+3y1
z<oD-2maUo*+0eIm+uCn)w_m&)#hfsy>i@QXwg132UAc6Ux5Yq!<$z@JHJND@5}kiO
zZ`NVq*Yp2w*U==gYTtC_mS<PHE}s8v;ahwCuT#Nch0~Y5?ai0Eziq#SvsbQ{o?Fl2
zQrSK2lJnWqm;@9&jjwTq7WS#EKkf448G{0o;EJHBt1SHASzi}jx<KSv&h56xE0);$
zpOxIYN|N!OdH>_ju~2tR+Q`-HAmCD-Z(5{uf>ZJ3yZ`@BI%yv~<g~?Nv9YF*lgCuO
zlrGJ<pB4WF-6NkVRu~ue8HSb0uDh;v?DEq`d%{DX*Vft1pXKJ95O~0n;f~n^k%d}k
zGTcHt0$P+RY=1J||2cVr%0!{yLvO!n@T@vvlqSi@$GnR#xBARyKc%I~G8eyZ(z#*g
zCwwjRj7#XZzt_$7-;)KUpfi(fxj_yK7cV-*>9+C1;*FpG|CfyB-nvOxXV1~d+1&!3
zIg{5uGYx*Tr+!MaWq_)~X`^+kOit$n+Ptzl_jyU~d|zQH<#Q7a7j6}CJ=Jr1%l&V4
z?(5HWuCJN+Y_tEW^lvBLSkFDYDPfjDAA`c8!wu`4?<e0bndjNU5EXvwd;I@j-}t~0
zb>`Y$zGed+w~P92T!LC2IxL->oxR`XE5&ErXckC0kj8N9OZtmzRo@>~v;7zE)43e=
z+GOdjZN9<V-mcYnWaGc;RK)6@iMh!R9dawDRIRGc|K@qP`tgLL+bx{LuK)VjG)JS^
zl`SRmhULe3i#wjr-0k-(?waV~hQgNO8EO^$N(TRb{L|me3=WW~hFlE>Jg)T@0yE9U
z9ba6?{_-z*VMY6+3$6u5g)5V~&&|?wHrjjqfBnS65^J|^+a`Vdprx^|lZ)5Z+I}O6
ztPmH;V_R)beq83E=q{;}zx43b`lpP>>GAn%r^LHd++Ug*(UN?`COL9~&>nY=i0~yW
zf;qRZ8=SKQhl13JBL|Ni`}6(1y{#>0X{n{KaH6KBA~SQKq-5Lu{r@*^_>-QLl#uX)
z`~2CSp8o;HRz*CHlQ_QJVl-Cb+-ehW#?e^J`0v6_hgA|Dlf~xQJ?Lb1IuvvI!+ZI|
z%l&!Q#~J?lb+q88(cyP@dF<<rV)mZ;@LB!v?d{AG#sSbmWX6$$$B!R7e}3&k#hk2F
ziFtZTYRA?tIB@pxVFqu;55JhQB^$O%HcoOk{Icjtg-N6guZYf3&TCfOYlRw@GkXZR
znEm-*U-7G9_Q?<b?U`Ht{G8l4ckaVyzHQ9x68H8bTGY*X_|&`Y`FY8If2B7p;{^M8
zQTd~U#KcS2rd_%sVs0K@STI3c;K%vXr;i<Mw9QM1{3E~T(Sd0$|D1}niq0GiP0BVu
zFk5rRQ3hr=VHUxOC+3RBsIt%D)n7DWGT)MyU8|-yGHTX39QgmAM?cQs&fe1>UI-qZ
zXUlVay>ZQt10P-rGyk<d2J!2n_(ut8X;&^!zH&W$)7Gixh7-eslKy{xuWxM3T(riy
zD@MkZ_1ltW-E#%k>{fI#R9|wu+!`2_aLxJzCsSaBf~Ip=XV^{$W_IJ+KmX5vcqx2%
zdp^%}ePbDm0|j5T4mY#&yuWAo=a1)hiDamK?vEZkOZ)qOzM;~gJ6A(}V`D=+JEu<-
zjWbNv{J(K4=fA{mr=(TVZ%%T@RX*&fI5UYMJhye8MS!ncAdkP;iq1)mjHZzalmGuW
z{`2?rhu7kVR|fN>pEH<af4<-u&*64{o}QjBFK$8Ys(dtIqTq&2lRkghVqz*PD*X7-
zv$T{1hbv4gUNYM*Gn87Gak2SMk3*+KMQuxt&b&hIEk_z}1vrHF<XA-6HYwj+IHh1k
z=VN(hw^I(u2ZT@k{?4;K-*}GY;SI^`haVs3neb6)7N}}s`2T-V;v)f(l<@HBpFdyz
z_}yGoIQiL=l%^i%#i5KFZZYZxUHoKwO^eO{WaFLrF5Mn6t-@)i9M**euw1-llEKSu
z(*E~9zvt8Xhfmpc79Q{x$>=zEnNeAJno>(+<HP&*ZT#{Qa<)k~c1?Zwqp;1oT*9t?
zS)vf65?K`b=)who!Xl%R^6T3+PJI3#AtAxRPwa(ebK^t%HP^dN%sQdiH!-7QW$Feu
z%OcK})d2^i?j?xjd|2Tse%wsp6Mr0+lBxH@muylQ6+-MW2mk-)ssCs2=ik|aM=XcG
zz2#Z%Z){U>;KNsKUnN0ELahTOm`&TldO8a?YzPPndX$>f)ULy9W3GH%?Sv_l^5b~6
zDH`tn!XA&=BaTc2`D2OSv01Y(u1PpBc|!s38b#jl1xs@7bTaRC<eJ{ZE*ll(Al9fN
z{v_(zefzfK{gQn0X*c%IfA}J>jbDDneJ+SM*UU8JJ7mD)dVhOD;>xJbox#(J`tB`j
zf4#7(i;uIp)3aM*!ij{VeSd{-$A6k|y8hv=cV_4NQ)dS6`u<n+16P<C$EH8~=kTye
zvV`r<o@%J&y3UK~YJqy>ij9g(R2(*0U5sx#rhe*wXMvM7>nTR&_#zYQWuM!$mL7RK
zFJ$V&HP3A49cYwayVuq<Sw@m=S?839@NK=d|M#Zfzb*_aEf+<dJb37kimvXP=jn6i
zzE#mYdG-_+Cm&md-_aFe{}NvxTr}mzjZ^$jcClSN<(%87q@8>qs^URLvBGri7gGhd
zDTe4Q6+Gs_)-`)#+M5Zfik%WtlXm>tenRO)kA1zwy}ijdHn%4}^=exkE@@YPY~HpL
zpyUlobbgN>JW08BaUwV8#-2ViagmhtloqvOHa51LlT3S+4jgpI&PrG&aVCvL$tsZj
z%Em3y6J#2~#2>6t_;5x4$7_)*EL~>~%(U3BjMF6b1n<@R`z8BilOwivJv_qM_Wb-x
z;}xKm*#G~FoVFf1d^jy}p^#YG)5nX|6%L%^@D?|a3aU6XnS<Hr<cS~e|JS=!tn3ci
za9EIe&9a1)hHd{O6oL%8Cv>N}Es=U|e@AgeCBx3PB$rDk8Uha~himw3&~U%Tu`=?3
z=#hFEl_LT!vRsJ{t0dN#avL?6TTJ*Q-NZdHDJdx-IeEk8WC5Wh0rB)3Te}|~WY+qU
z1oHg<|4U{b<ZV#kajlo;uQE{NNqp2O*z@jS&zU0{Ujq&{RZU`6yl}xuzDDTZAw%E8
z`_>!g9p7kS{wnM3R>lp2)3wg{EP30fsFXBO>sVIV<{%@*xAW|Ne+=VZH+lJs37kSl
z*EF+T^R|vk->p<pk@I6kgmCGfm#@8Y-tF&lU2>R}!Px15&Ee`LEpz_YeVE4pP5~N)
z8#Wt9M{268Sw%%_fBvkzoG0VN;aSQ56ICsRbrwvN&il~euI(i7lT+!_e|wvhi(C>O
z_8fAdKa@D*?j4X}D`#f*;oYOko$sAA(bahV#E<rEdna7eP+r5>t#Kg1kwwzp{i9CW
zl*5cMT>e6dDJcN~2^~Et4;~~$Z0~#cDYa|X3P`vt(kRTxKAn<Qb@PVC$B%^uMn(&d
z=^a^nVOMh_2yli~90+D+&N|@O)wSQFuxCYpL_#8${xm~juAoGQje?u|8q`JDX6*ex
zG38;TOx%_Qk&QV_ChsnCsYgt4KKuW_eD-~=-h&NqwAl7|vIr-H86G;sQ&`B8lLIO>
zV)l5fv77;Mhmw7vhIaS#>Fm}P8pVYh4<72^j4L{Dhf81g;Z?3BTn!8!=Av>pKg4hd
zwrp7`@buLG|2`YqRAdfxGfq)(^D5@?n0Tr|N}DGmlhxGM!08m9O6rSOigv9l6I%)e
zpPsFM;Z)|PQrOZT;H@?%myazclQr9!g>S|Ti5oX17cG{&aYJI6Z}N^x;r^RHpzbjS
zxo7J1-<1_BHg0m_;!$&RQ(Mx*C!f>McUplr*+9WmUqP|^j-In#vgE``Qp!PG4U#82
z#9nI@c{H7PYR^3BVL|GLZ(N5MGS)VA_^rw87S!D7(9N@?O?Q@*&teW0ZVyJ6tY{X#
zhzJQLX32<1Nj0B@9i`nH6k!3PUbtdK$CBmNtt}FpHW?kdCHa9P&8f?b?ZykSTM~lT
zp0O4QH8MQmdTPH$u`}l4f+LbeO%1Eu`1Q{i$?U!1pr>dcrTSte`^ubzhRF6gc@yUL
zX4EWLmDr;0Sg~0o+b7w~!hoCKr&mUXXWl%XGiP}I{^couH(}2QS#bJPvMfAt>XWNW
z#PK5$MFj_L&->t!nv~#>S7~q{OV~wOWb1?m^LH0&TIc8_eCp_84rW=a^q*h4G1VcI
z;d)=#Y=Z@>jx6Q5$--9gMLC-Hu~L@ml{W5K!59CWX83M&OMyM*4u{dI#;wk$8>WO^
zV46I8_Q9)DA0FjyJK8NNXQ#5Zo*Uwb+{Fs~{Hr%_t_zr8Xm8KGN3ccasRxgmM^xh!
zRf$w)uZNAx=1vq06I&Q=&QtJJqLWj2fvD4rR}()QsvR=n-5(esaAdB)dxt|@o-C(Q
zlbn8tN*=tD{P5AlBp0c=mg33^2_Yc~85xO|){U*gdC+)Y_#`1I>B^-^{9Hz1Ay3?S
znL2_6o-zha{%o&P|G)if*o3VH3nZIVe#mFrIh;K{BT_)`oI^}tBPWN5=!!=!QXKkc
zBqSvyCM*kASjzqFm|9)KR*r*9KmM<0<Kt_4lhf8OFIo03<;Ry!j&NfG0|Ntytvxdw
zxfm2VT=(ZSg>B~9)3&|yU45_E!w9!4dOX^1e+$H)bYz*hJ}h+Vl!{+>H%4gvRzLnC
zC~<E`veKc1Z59$4l84XhDY7TV6mq88zIExItoH48x$e7FMco_CME=f|e<p2Nc<bwT
z!5M*EQO#OKbsj<t3&72GC9}c}n~bU}Gk5OF`uuUDk%56hg8aqgBO3MstjudlT)o9I
zQrQ+wY*b!WI92QfkBy~t_$x)V(1%4l|4Q1HKX6pxmXMA=BHSq;cGzXc3<(KI$%qJv
zEn6h^?D1F;1M!-YS>cJ3oh&R7Cr%U?8X6u5VvFj3VE^oZK~bFJRHtv8%0&#0Jxz)+
zH>D&KbiE}yv_BV0cC^jdV96Yw*nHJC!drHWPm_kFM2Cjl1kXRYf;+tI?AX-R+2Z5b
z?CcgXf!b65|0`(~c61e~s@{x<@;V^aZG515#s&)q$u~{sgN%$8i5v9v@MJWxybcd(
z*dVw_x?Re|!AC6pfPmr*CC*hx*2EiZU0~h3#EEI=9n~3j5|WZjmPp*WE9v7aIb&vF
zH6I()BU*(wZmeKsf4zB=(sBk1huIU`mrML;v1XnmQY2t-XqBQ*!}5Y-s~E!5j>bO;
zV4V<Hp`h*Ht<<qetbS#~@>h*g?as=^#?2E2n_F3%CkQ0&hqP3b=2%Rb>iXwzZ%Rsp
z!bU!S`4gE;$2UE!KfyIYTXsg^|BX+J)-bF#@;UgwBDIrc$0cn&wmQ8cD}`k{oF*NZ
z_?wd}DfO^tSy|f-3%2ZRwz9G{*OZ{{onWzJN5q@Arfb%!CNljG6bw69-_ui{m>uzN
z(YZDSb^VxS51Ck$=e@b&pr{aGktkL)<IxV&!;eHStvhmg$$>RV8&)NzqzH(leE6O|
zVS++}DI`x$uvoIBMMg%XwdKZzQ*29iF-QK9Uvt1gVT-Pc%M`JSN!)sZNnAXiCh>W0
z6-vuxN>AQ4!NXu?0(WyGqw3uSOBLDJN{;gI@Z{z3<mItwi9_6aXu5@kMMO@nC=ajd
zjhm8*s}E0%7CkXL{GllOr&oLTd#rq7udMv5je(7gt>+N)(%z=UQ=F0?u!rdQNf<7Y
z4;O6?ZDBmA{A5eZhwUbXQ&o2eiD_#)7Z^0=>4F`7XuidbGiRqRS!x+DTYSROi5a|n
zUznJK(iBRg|2_0$<hgN)otgRYe|h`5`d?o@AMdyS|8L<`UY6`djjNIr7_ys1xc>c5
z|MU0v|LOYw_t&Q@9G<&m8UGGFww@<T*PMCq`m};V!(~0FPt0rb^7L}EwND>A)(~Rg
z@L05{LF!J{5>1m6hwPqM20gjbu<e?-x6J=vU%#K8E<f8W*1GJ?#p~<sD=Tg+Sz3AJ
zDyyk!>yxLIKYv67x(9#Q66KL~;lSLk`MoP8e8qwdU0Oc=|Nq~<?oa*SU-z5Y>+|ma
z`}^VHeSZ0Wh6aDagOa{{PM$JZ`JsnT#hS-a(rkgv=}~-#o<2)6FipL&W8%Zh>Tjy!
zpbnjBQBiT@&D*c3DG`YiPjRRxh8<FttGs!qeu<*;3aKS8MYl}7^Ww$R)92gy?`P-T
ztN;IR_x=6<|Ni;O{)jg*(N9^q(BEH`pTAO6)b7N|>=nFxNw+rt5K?V33Szuwa^%1P
zhMNsv{y#lk-_G~<&YqvgCo1osXY-T2lc%hVEibQao!+q?o@EOZ+*DPCj%~X*iS?F^
z$TJn)XEF9EXRchF`0!_OTmJnG5s*A`Xr{%NFH4%5p3I#2KlzI0iOeR!m46!a+0q&&
zf2h;%ak(AWAQXCje*HYt*GIm*RFB{Hr%7x_{=J&}TVEf1dHFoQOvQ}3Yx6TT4<0?r
zBDwnLMWux@0+VNOs3{+-|Nrm*zq18DKfRCKyf5eGr^i2wr^x*Jtp3EK^?z?~z5l$F
zGw0H_Y)*dgQ1Ok0+nv*P5)v1VPUPm<QPv^F$;nn)%BHCJrjr?x%I1MG91qWv))t8k
ztO7;3pZK@g$m=yl)N@ClY6#oBKvi(X#`t}I=2(Bf(j)molUZ_4{r}?QeeC>lH98sw
zU%pKf7fd{SS6gja$bqA3LIz4x#WKD*_5S)l|KaKB|9^k~Kf~nZ<(0wv_thNCJi5Q0
z=l1sh|G&Nc|NlRao?eT)g2aSN?AJo=GqfZPR>|!U<K}Mjoh>oX=3o*l%ymbO{?ySu
z8WE{@@qELUvNpl3azFW3yGu-v_`t-x@7f$e<!2YZbL8Hws=c|1TR&dthTxyi&+lhn
zzjyVz{Qu9-``g>s^zamE_DDKs@R&@ExLI(ZBTGsyHX};l!P(jSkN5x2$op6E_SXCN
z_c>Me{r@l7CzmQ9kRTwOY+!gWNGpNq%$I`=%9;iXrpXk%<>BF3wTfrgE}r!BNt!m$
z6nTjyDJ^Zvbn&2|N6Cpl`)_6cJ>O7O!T8Fs|3aYt4UQZ2SJvrBIeBWFpI;xh``f3x
zyYv75R~KmF;o;d||G)av6YuorJ8i4w_Sf@UK4;TiwLwg=LW}i`N^9hwgw*6^mMlMC
zUfzFsdHvPv^$KdK2lgD>P;Gv=PnPHXec{D(8X4W~TN#fxNQ9aC`0&)#@hB-p@PNjP
z|NlQU$s!_h=GLvPOPB7HxbS_m@-c%6?7KH?3Uohg5X^b!%>V!Y|KH#L|J=>Z{`G&A
z%DC_EujeU$R}-`G(YJSZ`vcp3E_kp#QkqxMkjZrJOX0US=TlGbi`n<5`@|GR#`yh4
zId@JJJY+igJN9Ewzu|&qiYBH!JUmubJexK}oMC}P-6V^D|I;sAnIa&P(lS*tqw=qQ
zQBfm*gOSl8;e+u@9u}`jWs7-Q&(>By-@bl-+1sa!-Q_*FdDz<aR?pA7@!`R_x&Hg>
z{<;<^hBc^1Ca5W|;h4TLLxsh_uIkJ02M6E3zE-Q6eE!u`ZJz0RhBXIoemTUqfv+<u
zb#Z8fz`Et?(=LJr))~OzFlFk`-X513Gaf`wo~S*geuebyhoPKl4;!c16^P3HkKg;N
z=I^i7_VpV|w|s1}um2an=SSMhOSk3i{<J(bR$43;xcFgc^Gdl$naw6i2?-5)%=fmx
zKYnqsy@!_Zg9p$3+a{|^#_d&d<i6&`GB4(V_mqjVl}{+?=<rBMMO0mcCXz`OCZ<nA
zLn0<diC*Yvk#}k`Yix>|s^Gs<adExJllqin@9)>=-K}yC`f%{z-s<_`Yj>5uzqfq8
z{r?2*4~DZvcqBIQ%n<BRY&|W&7V@s1t?hsP|9`(fJ$-+1vAy41rT=&KpPufiXDRVv
zoACy%W@a@tHak1E$jC3QauEBS{AGN6dVc+SBqQ_RaI=Mtpn|+M!(*paLVuRBrq%sD
zec;3VsZ(DbWL_klu;Rh-{{7|e>#T~NJU!eVUtq#<;etI+Bpc7v{|09chBQY`b8y{P
z^RxWbmG9Nx>uP_w_-$ll{Qlnf&kx6_&CIVSSgMH|FLZDb5dpQRA_A<}!(GtT)%5Gv
zn+czv8_t@Rq4eJ{n4!;#b*GM_w_g4I{r|r`<^H+5w*KFr@>f@0OPkvz95G3gN%l4T
zZxF=5+@^P?h$+&V>&Ks;_dh<ikK4E8MNLoseTjMY>XG8lY`E3L*ZMVnFZ%gY^2g7M
zypZ(g<S+B*&z+hY9f|PC4?2ssOlgdqnotyY>G*Vgdk;Rgkb;+&b{9Xt_b4O%fbxtb
zM$L@QV&R2NX+Qq+%m0(J{`UE*wimzu&%d)*EaVGfTsNtC;zUP({@MaDaN>6IlsR*z
zCn@PoQ<DsjhP#fElF-)`vQNrh#9t8>`M6v~Waa+)|6Cje0goP<=hxla_xI)N>-Q_P
z1=*CcB-vCx)^7-UXy10Fo~N+oVf_Ao)n8tG-=6>Ex~71`w~L$^y}oQRqFibQhKT|q
zE`pHC#K~L6M@!3Q(V{aOcDQqEx?DMAwVK0h62~(Ii8Loi5yhAook^4Kq_nXxGcz+E
zetv%c`q<w;zPvQ=(K*+kEBoj0ghR8u6d2YX_|YH!$DWz_@N)nE^X&fq{;a;Vvv~&V
z0T#xn)+U{fn@OUguJQMvR(i_pulp-1@Zj;oMlt>g)6|%~f<<iCCP=?I^k!kg+JFTf
zF>|~F0~;nO{P_I*{@dI0C)`@X<B=(~c}s$WuM&@epv_aRFlUK?s}9{W{pbIy`~R>1
zkn&C!SIHwsPhRX;UU49&Mj<;fCFRGrUPpiarM{4H7$;Aem>3-|FD8$JT``@1k{u#t
zxcv3%4JUET^I}QOm1t2^JGOASx|g4a!k)7y4j=yX=coCd)Wi0B0{h~{vMpP-*8Mzw
z;KTpp{r~su@ezp&)6zbzpz2zdAt3S9NzDD-V*^9O6|0Rud{VjsO_(2BriiScC&!*P
z>p<AfKhCdq|2ryhFe)Trn(BWEiGLqIZL+XTl-{E%_2=*J`O~L={rXz|h$sWkil<!H
zrZmQ^dtfeXYxwuqSAU0w&O?XgN=rD_|2xWm!ENt_B^#K^%GkujL?WRUdCD9*WOL`v
zen|;~-CFg9Q+Pi}J=ZVNX6gLX9CiB7;RpYc6Mj5@pm6y3aW~Hh4$h56hK37{Nfw>@
z*RR#5{YY|xM8>WE4?P45iWfA!dBYPFbY_Z-+L?m~A3Sq=v&^K4VeR`aOLJ~^A0M9j
ze+#yh!Y#G2c~k%IPpZS5c|XO&$`zWVFU7kCG4#It|KHHv>Ejl&f6Cq)HyNd*yx6#T
z@_n}C<m747pa1&tQT*L9wtrl#;Z_O}*B6}j@-#eoI{3<rD_2uDY+M*+FLqWYMw5Aw
zk6T+K^UB8@usreS&xFO{3(j&pnR?hGeW&wpwGFdZ8AuC9bhq!9*rah%Rn6`05tkmD
z|C=`${QL3I-MP7a?p#54mc<(ly;*Ixa?WdMiZuTl9G)&Fn3%Hgfx^BcE^L*RpSBu5
zc(*%gno@>CoLp)fq+#Hs!l$PvR#=#(r}v{s?#a(z`y~W>H0%{TSt`TC+&6w~Z+ys~
zwXxB_H#&O5k|PuL9CCJO2o0YO8gjC>R^Dd9sQBM#feOcy5GS8q2PZ!eX!`&E|MV#-
zAz^9PuBjQ9c61x7s83!lDw`lC#Nw@-nJE$r4Nw(6H#fJuJh8mI75Njh#qLaAH^X^;
z#BnnY^N^V{{;{&0SiU@ak|3A7{lBtuaZ%x9XQ>&A5y}r%CVdFE+Hp&M!hxUXPn>x0
z<jAt+?tYI1gp+63%xvskbWNO@nfcL;>_3O&BcxZYS;GMuhGzKx-$|wK{{H`l1{*eP
zR9wWnQp-`WrMD_^u4kjUQ{htPlrQYMs;1_%ckS|;EG6dY`AJLb&Y?p(8KI)@zOfXQ
zHbmAbi02$p5iGfEVk#*s6LIE@hqpWv6AvpZPhH)TUvkz5td=zL7>G;%|9}3&KU=SG
zXyK;P7r+1CrcFf&B0?F8Kc+M8{mbY&VH<~8%htb*r6OPT<>fE$sodu1_(@0S$hPA-
zIW4iVu3x|YG)NNHJ``;5z$+p`a)z6t$e;iJ?KL{C-ks~}*B23AP-G+`YM7EPymiYA
z1qVH?4_h1zxu?q82UK}NgEn(QSJ$RQ6OISG{uP@X5*I96V$bv9D*HM!&8ntmWl71t
zo153JT2;c?r)+3w_~PY9RrS+9e|K-*w8&CAdoq)+KsICZ)`|nc2@W@SPI$IY{5ZdX
z!MV_|IznRh4vU9h4jwsTP^#q|DS36*`htczMjbN~6H`(iJaoLG1GQhIR!67j&><ck
zp8pfJsTev){QqxkWN2vQaDA$=p~0WjloTi1FY_6XHEfuuEH=IPpR(bE!_yr-I<_u!
zynicd$;y|PPEOAM|L5SOAFVABbLY4`dXzGEu4_cZhwai56BbQoj4o1K7T7ek+x>&Q
z{r~-|w89b}{`a2tXJ74a`)i#hX=!Y=wQcj{*xcP?KDr7X&~f?@Za8NSPfkupZ#mS3
zDtv)~WqI%H|Nr}&+Sb?nCpjUZA-?#_e|ur&$MH87ZtgMYURW(?Fp1+1XGAdDr!s!F
zGe-^{_%MIs$NjTs@0dCB@5jgMudl1!Q~!VU@&5gP|NWic+36TG>GCnR6Zhi}9dwL{
znKO0bL`L_Y{r0~)JFAmK{u~x?>EPh{r>~W?ZNrcKxwr2tyZ^hphjZ`Rc{67^#>_c*
zWs0J~goCHM9j7ch$(+QvXKI%wAA3y=Py9ZGbDJPpb<12^&@5}~{G{yTJd45$i{JfS
zugEpm#pJx>k)+T91Lw9iKPyg7m@mxtyk{%7XOV5*Ud4M^Dv#O4d;YAgh}qzw+4<9<
ziG#6A<evGx>kP{?c@yt7UrUSsRK8l$zv*M=?C*Upj8cnZr*F-bV>%Z2TW*zrW2hs`
zxxYqRCf*Vjo4zY}nnc^@iRalsL)b}<L8pGazaOu*tllH^u>F=#5B}Fbxx9Mf_ale8
z1=yC9G?-3Zuv?)1SA#Mi9~&QESWM*<saD31r#?&-%Xlks!a<4sxIJIr|D+_ISFe63
zPgHA2Xk=vX>EW4gue<{qNq#as_xCwUmH7Ps|35h;#ldRsOwLZ;l}Qo--M#-;oa9K9
zWou^CG(E8M#L^S>A%a^X5*sG&Ffu$Sk$T`Ri(_7_&}6j_D%>f0o356y9tlxB@!-0?
zh$hr*mA?A_|LT8#;}=Vrz%4&xL#X5jr547QPx9Gvc-Z!=4Rn)Wp6TK(#v@_q5_Hp{
zMrLw~#k!EzYabocE=xb~664u0rP1>Fr~mecSBGn)8^R6UU;qE_kAn>UG8@9wmEXJ)
zR@?IB;DLr3MK<P5dJ@vTE`>ZooZ>(GPq{hs+(>F&5~04#LQy|7QH?KU(mjU%O^j<k
z{}9l-CHewxAf(SNuUTkdXz-``?wq4XPW<4%X6ra7li!nfgB<UljAFJKBFyh5H8U=r
zaa2c3B0#yJ@r8!^zJSC7%NV9xtbEBF=gDhxz&QAP!<2w~dm3dp89}KDR3v)J*w_6@
zPWbU%zUs3Gv&}n6i3g!PCNB^83%0Iy*%vEr+WA2K`H%jCM^8C)8=tXITfa#mRK~x3
z8snxVLZMj-jGsPO&eLLxeDyYZO6ysui~VHk_t*YzzasLt{|aC0{q#1o4F^v!czbx5
zt3J>&zTjdonRQNgWKKfY1gW!6mh!MYa<ecv-NVS%v*P;fWY!r$@9!_3A_Fx`)AHY+
z#D+?KZ)=gT7K24<d~8dWsd~<hXi*OI^wDeSRlnBRnWMCOlgi6OSJh+_4xBjg;k|rg
zN@M7^tJ-by9=kiB=BV`9*Z=$b?=Qb*>VbT-lp`KWAGT?IQebCZ=%(SfJZS<?#c3WJ
z!IKU#+04FN!Qvk~d9wX08W{hvX)e==dGgeI$`NQg?%gS)n!jBBGCu$R8!w3c(H$10
zuvDlimF-^|`<&j@XB_&R*m^V)9lE5qlyP3$$h)aqzG>^31q>-75(PKS^8-Yn?ujh^
z@R+?sfM@3)2R-8pzcdBdZrz<}5SjG1>C~~Qb_HAAI{TxZ%<-+UY>bzTl9JV7xOb~z
zIoqF;Oicg&I@|w*tN=JP%i>>hPga7j<cz5to=4fPD6imbVw~C?X7k|Xh6nwAY>%c)
zoco4N-@#92hHr?)@Bj5}+w&#&Es{G48qoaz-^o+PzW!h9UFCioo35>_U3adhHk`d<
zw1BU}aKWZRR;9E@Kl*#S=XiDWcn4iIm@uz_P18Z!?89j%iTry8HUIk-@j%)t+Lr$g
z+?H9P8(gtoG;XTFg3{E7lYe_WGc+>zqrU4&!T}9q34=w9{tp{hA2?wDr*(gEcl_A|
z7M9)XBd$6#q&<QLiN6dukGJ&oD!JM(`rkA~;X;<!8XgPr%X=eS4Nmj0^=LSD+cybz
z?qAuc{_=rQfd#`BlP~Y(51*d?BKkKpde(;}C^l_9c(7q{QKG`4t&QqFotl;s9o^wJ
zPwhW;Nft;O`7oqi{Pa~DG#j7s8#)5(<SDbi{{PS4-{o@xWHxJdNxV6f5od6ytNp)$
zAe+k9h&k-#PnY}G|Noo(;Bj(O?vdZ^OBgJ@{w2D|t>Z{ru%@(grUKi7*}tCiAAWwG
z=g*%b^QQT$9Ef`P=l}T+?_>|pwSI9G-X8byskg8Bp{~T#pQCnTh2=Fi%{xMFds5kU
zc~7u>fA8;1<L7sOf3Kf3xiuv8=7cK?RQV?D+7#h#EzyxTe^I0f!-N0zZMxACda+3z
z-N^v~3;G&=+*Sgw*yxU13k^umY4`X4ul>wsH08_t|NsAQkj?*}_}KgZg>7mIy*hsn
z@AYOC|1QsbI9_ep|IO+9Z|?f~`1Ex9jW1nq*L0WOwp(_ho=5oKLaEkFyEz}_ug+;?
z)Q>k>6LWGy3g_Wwc9rdV8<qa-l8}&icmH_N4pm43@RWIf|9|b@U-Ha{>vPiW5_{Py
zyqfZN{%e=ow?j!tO1@wIf5n#<;>O0!`|PYQ`7PhSLR08+){|WQ8J!I0)3nxjid}AH
z=CQDl`0+zR&i+{4Oh!kCtW!V!&wu!@b_#2g0L)`D_5Y3bRDKqJtKw<WqS8G_K|%Fw
zz0AUX<ApkLfBxOKXS24}i;Fv>(kl6JMx^}ejTMua>P#Y{=GN5kJbJ|A<HHlZU1&Fx
zq<@yeGMCDqLaBCQpveP<|NoPmjGsO}ZqF0ZSN@CNoBh9}#6RbAZOvD#e*_EtQ+odE
zt9<mfnuiY)we-XeM4Kh3iL5E~KVzolbD3=i+d-}wGb9cjlBoNaa$|G*!)v;?Zhif4
ze|UX7PyYQy<)IQ15>V%R%k=#J@7!H-W^zWND$6t%XSY*7K0ZD_Mf1I!{lB?WkLv56
z?^%1`N4uBO>3{BC<}BWVYtjzgwfAV;X676sa(Ht(&+4#+I(H9qb>vS^xccLZ=Hb`Z
zBl13hyzu|Olef(On>YXe{mstlSi#wt;i>Sp$z?UGz)F_#$fl0v|Ns9#H}`kp+gtit
zT5B>h@7!3#8?0{=nc&MiLzVs76$6$@tSo>2pBIonf8f}$4K<dBPfljJmTn+$QT`zF
zy7=^p4-DS0eCjO&>dXEA#oC|qgzug`+YQqO33mra<v0K9*Sr<|67&22|NW(}?fhqb
zX`DE5;Zj$JuxXspiXX*GpX+wHR%nZ_QRUtAu-P`!fk9ZfxslPi#N_Z?Yo6L#9{qTs
z)e{r)St~d3I?c1`TxxpizyWB`Nk0Ga>+Ah?{{R2~{blzw>FN2u@!UmY0|TYliyb!K
zzscX`VBd84-~XSV>pwnP-`gv1U-RSojT7%*ojP`=$7UA;`-zPmHerr`0@$BLToYhE
zbcn~$5VTGu=gx@(M~)ce%l))o`|<pP`}_Ns9y|!Q)KljEzQ6UKp2)umsbO0(m%T&N
zWQnOAk3?{GgfKg^O<vsIU)H&|rb`;{+f(&bU$@uVc5e4vS<6oof|=8T5*)mo|4(rF
zx0owwks{BTGZLFNN#xy0h}b>t;c@=9mBAb5aJZH=GcWpnTD+}aexo!r7UgHxetRP?
zEL>#$*nUkiQ)+_9Ck{b3H?|vdbu@Ud{{R2K_V>5%7Z%DdT`I5U_vg>2r~Ci^mpnKB
z|ItHD`)#{#wjEgGXrL{Xa@5&6)xf~u&;RoUk622!Bp>Hvv$t;xT^$hi>EKjB`)!SD
zx*HoGo|A10U%w$6T6Ah!Mnp(NME?Kx?{9xEuTj}Q{;(p4O*(t#IB4qrbIzaN%(ue1
z&(106{D;TK|35vw-+$((uaA!27jQQ)GCFa}Xm8C=_b91(Ru9iAo=IwEbf4I;+DX#x
z!Eye!^7jiC->Ux)S~qv$!hsF-_Ra0=&HL<{m#CaGUew{E{r|u59IK-r*p8Q)L!%os
zAjA_B^WU!iAHVgAX2!)U8S`@u4rNKQxw(1FW?q{6@xA=v>+5;=<^EU}J-Pnq=>79^
ze(GwRSh4>6rza=xo9F-g|Nnf!&rkFH=l}EaJjh;oA&c?MD#o3epX8@@$L!?ME_lRp
z_~qq*GA}M>dHKW7&hm7N8^`QFmymQQA>+h?B}WdNKY!rt*#oCfA2{6n{)c|d($imG
zM|3HH`sR>UQ~iIVn&034UtV7C<kYeD%L;YjMOzIN8YWH@oK)~2mF@6s^Z#Y<{%o!N
zeO}qU?(d(U_8J|w1{41N`@4VY)ObmW1-t(~?3KRqsFAU=_u2!8u(q3TB=R2Iw{O#r
z7ck!!6>U6YmXU_G@tYe5LCX$LPUhKPZ>*tZ)X{Cs!DW<@Wwc~@^8Vxe;-C`pws{-B
ze1RBfnhWZ*e+D+Srw$zY#J_UR5B7L_9(|{P@ETBM{Qq<N!>6a~zrXui`|ZvB!|nC@
z@&Ae+9eK~r|BsWyKuW@ZoqhA}of|nh4VNwdpOD;~_w?Dbw3=^k);BR7k$Ixj$e5p9
zVo~DU!SesV@td2>vR9`~OMCDz@xryV3)j*<d{2+q){}Tvs?9iE;@#ckiZ30Ft*sBQ
zXtHT)wyn{*Bqj6xo#CG9_P<KiY;2(R2gtSdAK$&cUVrJ*|B2sjDSb~XEc1DH_xI)N
z-yeT_YybY<-}>L*{?9i5Ur`|;D;v3Z(G6y%8!j#qk`fYmcYhr|+&n|1?&~Xg>D%=;
za-YxWY+zIt<XN4^Vff&xcAM|)1v!C%F7~cLlN6Pk6-}l*JjmSk`MIQ=eOkr;c@MAa
zvxSCEnWq)?wQ=jm!mnC~7rWp1y^)6p)Sv;m%U>oZ2Q;Rqul!lQxRHO&y9RNU3ClDd
zzr9^=ZM}be#K+g)^Vt{~-8ebjyuHJurNfGg#cFHQ^z_&~YU=;}d4El}MZ2VQ=8T#D
z|Ns2l-^%(f<;<t#7By}*wzmKOCG+mP#jGuTFV{RvYRcIoM?lH!^l1gvqYGA@`tXeR
z@N|8i{dI;bRyxjVc+u{Yq$arI$K~ZOw9idsgw(eFpf-E`rzh)sc!byVp5(aA!X5ja
z`Mv>fbl{)=pP$zY3H5MtzEM)@;o|bywrx#Umyd_L+P~z4gde}Z|DUX0AHT0<8rPgT
zAWI60|EDJ%nx!D&R<x<1VSl~ky*-Y1zK97l6xHakY}jPv;c4jMHECzh<Hzo8`tb!~
z(#S!yzs~SLS5-y*6Zt(889gN@gmDO8u4iLA_W%F?(<clnDmD}p8GZU}%*AQQ#c8-{
z)1t4k|BMX{Ykq$E|JYsQ|L<}(F)=pA#{V1K^q2*Y9y`HQ|KCW)o-elH@WF!%R&><<
z>V5N5gpI9D-Ctr|+|lqNNZIftW;fRX1)e<tcjlIS`2YXd$@+r8m=e*;3!{QsF71_M
zzS;NW(=J(=HKyyY?%5-k>gmy0k!oPXc1>>4;+yFYPt0_Fk?gZPsDN*2kLu-}oBjTX
zi80KWD70t(`A;q@zAn_QnHh5TN5`by3m7tZ4fnaug@m}g;@=+=AO8IO|HDW3C8E+7
zRyav%wJMzmkvXRO?f=BBHNU>R|MYafyq!bt!^g=F9{kYP;@Zyfdc|uFUr7N_jK05r
zVaD%&a}OS2Iqj9aMU0J^*(>TG*MpZYzvpLcZ34|PffAj*<-h;w6`!8Sui5g0UH)a$
zRGA3}d0tFlYhir-T4}e?Ba@7kp#IH|kN2g`{~eA!cKGm$b>|M8`}h97eaHNb-(ubM
z5>iqsev4gvFkxESgJ)?!{z`jqX5OjN&|+UxB)|4~6WhOksXNMgA6{0kIR^>NH**d7
z8VopG<DGn7KVq%=zq{o9h3JwM-Q5S5PGaj-|B%nZ)+YWV@!(-ar=Dk4hH7r-+nSe7
zm7a3_^Bkt%>z2+@%F<ot!XeY1C(UnEchP$5rCS`=MFs!s-FmXD^|op8O`nsSCVZa$
zkO4F`{{O$zc7aLn65by9zkxI2$O<-<)B~5;xBO{{?w@7%&zYtE-=F_KKJGu>zi^R^
z3}`-Y`h*|+yT7NJL@sFf|6j81Z&JlCk&FF5KD9df^{qI%<j58qg{*zG$rXQRHF!b1
zbLi4Di$bQO8aH&bmwaW|>&aZ#dHMhUj-}6MtmS2E``^>^KW_iO@*f}Se=Iy{UuWRq
zVbIlSSWuwwzD|NQ02H=gwGYp=?qId}^oeK98WG>K$xX?}Ij(3TQoTN?lH9RF;@=;~
z)%X4#K7Me*v)d995+CG`{GZ?0`0#yvTetZCzdt_8&*PslQ&L1k;@{uj|DT_)_gc4G
zYTbkD`k>{idG}m8Hf#z?yf1z0!}Yn=EyAD#@c+M4zl@q%+kAV;ef7sLeM@XOd(PNE
z;g$>!56_9mFFYQL1-SqDaqz?2+xxSx3vi{RO5V8f&)X}ga`F8S3=@7-e`HF{>FVuu
zbZFRO%Qb^xr|zeh!iR5f?_mB=2r4!}>C0bcf4%XY{pSzx?zWRzE7>Ua{e)kWwqx;y
z>+ApX$^GenbH~13LPb?FB64Lx{M8T46Re(IRA#FX?wvZ-F;Fo6l!G_diCQre7JH9(
zkKJ`<%Nv3W{{P?U{GR{+jrUZu=dMaHF_`$<BZHTFjq_blBhj|z$N#(5F5a_e&TQ=N
zcJ%9e_&)xSW;sifeZ7QTU5Z7$+|uZS2OHN<X+1wl>3i(Pu<!pDp57!;_djh%Y4?Xd
zP`H3*9fO|#|9$+!FVp577Kg%2vJG<n_CJxDlJe)h{NelidHCfNrcK_k$>_vs<3B$R
z9ysc{{I-h0!TI)}^@STY7#J8juD%$)USeNetBy*7fUxtOwOudEts2DjCGPEae<E{2
zRL05a#i<KwS0AwVA4=UArY9h8TkV$cgB#Q&WBC7n&s<wRhX8>Vd#**nd!1GmvrRSL
z-4>*z?=_XF*RAC$N9Dgo0`~>YCZBz$GI^(f`}Mb5uPU{r$*XOZ+{?bjOlX&qkMO?p
zHquXz+h5pb!h5)Z!_{%#V$)R5(1*KL2k3C$R^#YUdNf1DQS+GK`Q2M3_bq+DGJ8`F
zzq%D);*NbB)4S(Pb~z`r?xIQU)Y1rWakOQ&Enl+%PpiEulbGIt1LqVYA}lqgpA_0?
zXuLr{lUaDd1;v0Xr&jIy!1qA@W#qKKIf{=h7{jl=edEm}rm6I8LGIz5{RLB%d^No%
zwPf4=5?NIH{$=$t&i9gRYmQ1xV_kY-mOxAW!onjGyZQt&q;5^yrFL4QE84l9>G!!^
zNpC%`RWwh2p1$_q&gTLj)y@_goVRGdw*SkG4$wlL|NpnlT+7w!Akb=WIEz(K$F8C5
z;2Y`udykkongS1Zom_NK;Sg{2M|O9e@AC{251o^|VQYPO;?%`DCqH|ioI7)cfKvYK
z7_l;M)`HkW0?8dp2R(mrUYlBZ?DLB3Ez2@zsO{=owW}xLTa?XQhWp!Bi+Bkh7h#Qb
z6PU2<$G`YGnd1v{CE1RBe6TH7rE`bPz6W{tZ)fn-{=e~CQXU*|Yvyj`Y%maUk#CAT
zESRGcqq2Et`_xwD9SyEJtg0MhW}1sy7iEYjS3JEs|M&Nu&Gp7IP6@ilWb$?NFP@ux
zx$tM<l3ac}4i4unZXM<3UpQBL_z0Kw8fv%(281$1*>79@_o7LPWToFMAy*a7Ag0?!
zzqk)OTo0VK{8+w>@%cEd*HuB^Cv2Ryubwlx95hT0o=>m)n`BYWr@C^{RkecXi&Z;k
zylm^`P0irnq%ya&^Wpb=HcrmA{q+m4+NeIbZJxoovt0l1-Q7I;aRzfN4i<dTU{+T@
ze13jQb+$_5a(~Hd>ry97OS33yN&H*YcEW?r+4;pMW-dk*EBl%Ri_)&fR#w5JGiP{g
zY<QTMEP~mJHZyPb?aE|jW<LD&^$|Jf7|NR2kRj&%H}Cp>V7PbiBy&Y<Vc`cMB`uAk
zt_6BaUj)3)XBRY9=bv=oG`C{jRMS2l=EMqdndg6>t~8jJdrfMoo=T!$_h+Bm_dedA
zz0gJTrIk(prX?CPmgSa7I6VLSTwjgF>4Hg$WTw}o2fvSeR1>jVviZjDn$U~iuYA22
zzP3}T>40pLYV+qS1&`kIc};wt-@TcMf#LuEDKl;PTmuAJ?WgV#?42fhqlq_SdFl!c
z7sZR&d5%tt1V3JmIjQYYQ8}+**GZG$nwgf9i+PV5C_I(>7W1hv@}HNc39lkwi0|Z8
zi{3|yE#o}B%XnvcZgpww%ygb+lcnokS}p3><f6I6W?7ZC;)SoaeN{Vh%>2dMR5<tf
z%Lubn$Tj5dZ;rbx;XbR)M*Z}@bB(VbS=1l?HCz2D&yk(cB3s^GK4=F{W<4`Eay1)>
zxbo|<9XWV%;o|7O4A1MD@4r%87aPl>*`dc*re&ql(a4vX>zZBrSUT^)#0kI8{jId*
zS$O-7<G$^8{%ZcP>)Tv#w6;HJit2RkprlB7soi#Y|1zdJajfE6yYs$h@UFgHYNrL5
zH2Xhoxw+`J*>hvZ120>i?=>o18`yQcdE3)6<@5hnNUmJAFq6Z1Qj3LJS3&Hej>0Fh
z*6*i2oO=1lSFb<=9`4l{4G)){vzisZzw7<JMvm0Vj)ECgkdU5Z$k*f`;5vWP5iULl
zS7%3^90iq_!+p$^I#-;8rKO!5-}LsHusAYhJ#2fhs`DJH+Rd4jyFVLS&!4<F^G05$
zlBO$9cl8M&SJsERyQXtbTlz}!xN_>rDqn#I79y<6Jg*n-S8lPHb+Kcc>t&0%0!=pR
zo;&W|cpI@+qW`4P$z$;xnk_yDmnrgmw22YFeas|&*OBMD*XKSh$e*;~zVPum=U@D4
zU+dxQ9n8~ww8<dFDs^+B-SSlhGQ0Xtw$wdY>b_uE;PnvE>)NN6{#m|c%GUSBvsdn$
zGU5BRc*FY|LDyyY>fcCKgJY^??q04#1|qHY>Ki>2SA>3GcQ=oX;j(iqzkhyu#=>ar
zA52`YUOZaV5!IEs@X>{UtzVz>D5mWGAehJZeZu*IB^%GY7FGPT_SoxF*5<BZCcSm@
z<ND7&tCVQlq4#xHQERkrpZhG|NiDzUW=;Jo7yG;>ZM!o^&HT4wnvptJzPH{KITN2L
z)#kX=X>ntw(M$t5ZGqtET`}kG-w14(QnH(e&1bpJ^c{MAhu<#ylz3l+Wr|mY+-lLe
zeA{|cA6E9bt%^AOplD&(;}WOqmNFg5OOAYxTtD;dr;SI-W<UJ!xq1JM;y)W)Hv38#
z&HumfPbfdwzf<NKayKOiw8<+9IQ@EcdaXjiiC7MvmIZ5f?!S5K(mRDm0V*Dooi8XY
z<rU+-G{Ggp?QYfKg9o|a|K=&*dLw^-`tGBJSr?yM%lv+^qI&PE(9YAXWwWn#-R^xX
zsJT(6%(`Umq6Y;FZmV{A2z`<{|F}fyeCHygXMZBriXB#*%h$}f?QBX@;uMSh7k{v&
zhl+-}a&$Glp89gi&7Plgo(f%8J>Bs*>Z9$q4;<&us_i>paVDwpv4U#QrT4{l$+LFo
zIc>N(VcDANEr~BH<}KCWIGEA0EOfbszW%O#pD%csbNss?TEGr2CFa<2Hx&pRtM}B+
z>=sxU{osniLlMm@cf{XHTd+KQB<8Es!F%>%(xLl~N2I32>Au|dc*pdI59jTcey45G
z)ueEsQ10hE&v%=1YgSE-5Zf%d`|fF{YO6qzHP?NXzp$EjyuxRa0?S{!M342ypB}2~
zUsYsxecxK~y{nue)9j8LTqyMszP(P}Wx?&gkGb?bO>a(D?Pfi#x~}_qp^eaS7RT=D
zX%}B^xN9Wy=y!zJb?wE1&5TY9GF=3_4n1rTSn|@^-b-k;=wwglC;J%+&7WG%_C37$
zqI?l}q4uHcOU%sEBlgZ}oGfhklA+LK$-5Tg3pX-4nIA2iBJ$;6jhh>roLq)=j9=fw
zlfrC*f}oye+@6F7kCQ8YPI~yieo<S6B7;%)x7Xs$^W~eTiZ=J~G|!P~Ze;w*o*2I`
z#iDA;!+*7H-QtpFc_}|Wv8I1>W&04qJyR($HTB1znT{?kj7IT)|61?j)Y0L2_Kauh
zRGzrF8_$CkbdA@1{v^<%_VvB|;kD5`l9D{(>n1#igtW-l%vR)Uao}-{_iA!-x)AYZ
zhGD`74wmE2#gak>9C?lgEQh-!%WiI%AQ*jh$tvI9zh`grFE!!$VjE?>v3GObx8vUf
zyLK(h+;=YW``lE=meN$2k2cS4_Sxr`l%C`fT-1@Pd|Jc%(i8;+&$<O4<5l`rYAxej
zdZPH&wYupx1%iD)B)<Ee7T<X1vz2AHK${3_)ctiEcO36uee#L>_1becWuLM#fZCVS
zE&lyUw5XhP@DNiL<F$)?XQr_)t9Nmcu(6fYi%qg9;kbKug3O2i_FMG5&NSE?1q6dO
z7)_W48uBP8Pyg|my()rrZR0Gd<~2IaU0ltxr4Ju!<+;7h@Xw!<1^;X~)gFA*6c%pY
zXy{yIG%d4g4Z~U|$GpZvto!TJElRr%L>2o$!s^g;i+}%8Eh;7`noez;EnRy;X6MJo
z<u@7_!ot|p)n6o@eSSW8*Rl4gQ$fuFUS783WHv)Xw&dit0}dH`+qO7Pf7UR0vZGrY
zXn^rm7Mrzon{&H_kFVsnH;EPBc&v<F%?irXSFBH;FyX*G&W7a-Ho81)R}MY?U*Be)
zFVQD+u+oDY)K6ph|6j?V=lJQ<2GwTGlZ6-EY`l5M;c-%KC1^nL&&drnmWTiR<avI6
zvgI~il_Lj^98u6Y^WgE}goFbMDn}A>&M4^lB`EY=V@Uel&d(#RXHfI&$c9Ye5|wA)
za)oaB@bK{H=zw<roIJ@RD9E#H*^h7f2WJZw=&d`(?(*k-eA_a$w#>^O-=x3=)|QzY
zIa?e=T;%n1BqTYw#5VYHh$SQ_vXvz(iYcgN<*gB&F=K~XQ%u3EHPz)3cU~vh9X2@g
zzxImGVx9B8hYFel19=zUxoP%l)~ZFXXDlnu|Izn)(|Z>|FGUtlr&&s>`}`&B!hYvJ
zlRUmK_ht38)@6>_Thd|$4++hDe`o3>g#({8xE@I?eZ&9D|L)~ioa=T~$@C~!o-?qS
z_8DAFE}3<n%Y{Lp-QLo&gV}tiWk<KQ|AU)0Oz#!_6)G8?K2cL$I%U!~huybQ|8FUt
zV|i|Uyz%`r>4g$(g+0%0>z3Y=I5ycLPIc3|E2gQQ^CmB|R?3N1;dD%x^(}u^;gM~E
zt|F}WkDCOBHYKzd)%%IMyUu&Zw0x<4vh(~}|Im%=QY3v38w7~FunKL}QRX;fJj?C+
zLyHKpb>0$-Uaxqi+PqW#M&I6c@8>rropzce;JRP1PZzYq<^TUBvvs)|90XeTCkdA&
z&OW-y{DaHAgUS`UyU!+Vh~$~ZdRXd+t1g@1|621I4?pg0oVhc<;kn@PB`*ut%t<}P
zYTuA}Kw^#R`An%}6BjvnwH%OWYA`rrxkat(MfDV?Q`fh?E%9FI{FhTF`(y2@u62hk
zPYEfm$X>>|^xF3qm2R)UCVkIYx!aVHUtrT$uiKMUQf^KU%x45;!zHkGr{jxv3VKVE
z`ClxoYQ1-yo1^!_tAOQQKf({@pPS%tsG8;8@{3!KS*}@IZ_!uud7r$<Rx!txed#it
zg2^+|uT5RoT+~r-5xIHp@?-O|>Tb=N63~^oC^Mi-AR~I((s}21SY3GkXp-?gW1g>1
z9_vbjQ<>7W<B5qpwY43p+%r5|8X13QNc{WK$hqRiE|+&#0-09KaPeS0q1nv1RqJ5R
z%tgIDj%o`ZJ(BR7t8}>xl5>^HdT&2F%aeY7(U-XwE@|pH#@On`G<>>ne2eVm+!?Pn
z&T4qcI_Zy-qQmBi2?~o>{`@^1v^jHg`bp?o6eY95n*Zl6Ts-*UIsdt5{B=hh3cnp?
zzTni$7+tbxsuIg&(`6RRes1|Jvh49mhw^s@YhpZ|;-!S4mM50J*;&kEoSx(<`Qz^a
zhtms=$md4PZNI;D$0Sxg?Fl<WzK9s+9or%m`Qf^LTj*+uclVZ_08QM1GP+{npFbxz
zR2Uv^<&L=H!s8TO<`CDA<R&^xTB2YHzmJ}rc7ZET*x{9v7(Yn#slR>n>G%KtJkQS?
z=iGF9`6~<RfZW)>&Fnmr)eLKXwaDG9*weDdj5FKmpzyLEfBkFjDQTJ-FIZ+{qse@^
zX3edO5*hmrKdt+-?ScFLdgCu&4Q;C3(!5$h>*qo4(JlP*`*^_zg~NL)BecYoq?Kl5
zvg*$0;8b|zEX>qVD3NA#=+w~?v%1fmRg8LlcTOMR@o(DaV77ek+=q88g~DncLObnB
zmCP@u>+^7n8@+gCWMOG&uwdQ4q$Atdq$-{I9?W2uYnaKoA+7trMUmDVZ=ZaQNPXp3
zLh2mSmzU2_%3$bcN+~?!a`-@lDr>?ro{CAgwsa<5R%81yas3pKk3sHCFaNMPooDOT
z3wMv5ko*0;zkucO%-6^H|1DmpoS1MxD4pj9lhkFINep4p9~j)0wCcM=^}eh)qtdim
zrfGHik{<1FiD&22E539{%ok(>nfd>}QgKLQXQyNQ{KR`UY?+xg$}f0$UgVfeV*LKC
z#)d&x;9?R}_{_!}&3PHeq}Km9esIEHrwb8lo@$4&%}Z)#T+H$Fdp~IZ+^?#(x3@j#
z%{mTU;jt*=#L3g$3)lBMM$BkzWULlSOgJ#fXv4Xafio;5K1`E5amr&W({yV|!>}ul
zwln!ZN^tPy<*}H=%AB>a(d_)Er`*o*@`wNYRLSvHgwB2}(kL`AHa>F1z{JF$p!l(j
z#c``Xzm#7GLMI8wS5D^LAeJ`cDf=cPC1Lp})s8uvI48|G%BIr!=jY)K35?F^>Wx>s
zmOvwE(F+%$6AW+n*Ylh`%M%mBBDH)*P==znn`&>oO!ixcnnefK^mA-0Du`@fz4eGg
z$)*GcEwc&F-zhfDHZ$~@;qY|I!2{3<!bOK{gv7<0=gJ=5lF4IZ)A7`vhbN*-fGtPW
z;95|N!hx%PN9NmI$T@gRMsWrA>fnzY|BhedIsPi`%qiCJM+psAb^iZ1)`>e~P^k2%
z!BPw6ymKnb?Ci~be23>)@=W0V%VB0MQLselQS`DMLhQ`j*7$UrvCq1iqTcs^MXYp;
zE{}z{CCHlwnL>v@J>}t-pY*`=5oCyL(KDVc`|O;HjhlB`PWzg0M4gZAQIeo;Scj+l
z5;eZ0TQU_s$0jQjT=jC<W~uPvvU=OfU`a7OmGn;WiIBvx=+h?_R@SyZHEkz6rql*h
zOyXePw!|UmVWSNDj7>9+3FQ<&2x?o+;pi<h!&4?ZwCU>H9jEx&dfuJ9yBm}TbRrzK
z#%YH^&H8ss)X%SNgF)M?tQQM+pK#@1-nLHXW^L4;PG(=0=^HD&L~L#qW;*(@L<IB8
zxhq}QugIEgUoYV|KP_TIL*jG4N#7Sv;DVZU@{pRD8Cz&*+cLEmb~$T2S!NzNQ{%#9
z(p%9q?+Tkfvzv_YW1BUd%&TY4xXN|y*Hinf`(N)`x7GfZNIR3TqjEA=usRDgj65GT
zFeoarxx2Tuv9SdPzL}j_sLHWPn4S6EqS<?nIJ%umP*5sNc={+m>(9@_1&`Sd&obqS
z-ezD^=d+_|3Cul-j}j6R5)u<9Oi1wXO5)&66%ccqQ`>FYnQ-k6OY)QxM;vt<?%VDV
zV*hxROG^BncICf04^IfTrJk11i(mFIGJOWrU%8JO806&GJU!c<q_9<2vt?$o@$ntH
zE5^3z*L;Bn?OK7$3P**zBfJIIycIfgsxP5gWGzP{_rB`mpfe?w`SQ5;8`;$ReTqwh
zo<^|99b%xNA!uC2*qBXJlub}@PH%A2V|Ers!$9vziWRTK&$#qx7aA-O<2j+pwpp(7
zd*8!1k!{TE65rmZSA1%b)8&D-6c@!mYGBaQdvVV+IG9aVmQ7an$i#aYoUE_xiex=J
z|2p}+Zj>;c!F9ol!_93<5tEZ${kei428Ul>=4t0Q-c!^0^b6lEP^g1W1So$5ZeZ%^
zu~k;G#m2H(T7q^F%F4dz^AI_nc*P<@S}9{;|MDyaW`Ue^f#AP?kDKwFYIt>Mf4!vN
z{InZ;XFt4W+je>R!sB{;pg}Fr;Yu@(Fm9bKxNB>SPmiNlm!ntL!DDR)kFZqTX5;5i
z+QhbB(d1Obj$P6fQt~|(X}2!;G&IET2Q8oY@xA}y-|DtrX~{mB#EO4&9zHzmSR%{|
zuA`64-OJhLAmB1z@6p`l5e*ZiZq>_&C9wZ%T$J>Qb$d-?T9XB9j>$>BGqd&27Eb4I
zZQL3n>gzmDxGhC8)id_7%;hC(bhAB!djgMLx5@6jB(2w1*r#zx>uKSJbve?%&sDy<
z*X3^4mEm~jFl(=(=ej%R{~JrP_+MLEvqN)H`LB8Fk|QPDZ-uSgc=$urqOT72U^kt)
zyOFEKfalo9{T;1gQ2~Nq|K2}&_NcEoyLa@2L%e4X+BO_c<()Zc&HnfM`yOxKBKCdZ
znzcG>Eb1R?lnX0=_vO%N-73)>^r(5M$~Q-jB`Sv6HTAnTG^x5SU9}`SZL3@MEwSvv
zzIBV=OR`N_w2re;;^fR_2Q8n?%oGy4SY32XVp{9-hc=dcd&FlH+>Z?JbiQG>&U?;|
h^3reRufaV=W`?;7RzxxUV7w1X)Sj+>F6*2UngG!74J-fv

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
deleted file mode 100644
index e1799a87c8542d7e515b6185d7e8f6f75fe73f3e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 19132
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3YW8$-45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpM<#3(WKiH~dGLL{Wq#sgmWfTX4&P^7t--ZIMJwyj-w6z7wy;_`
zd*428|M<Hl*vKcB0=XCzIanQ3wGY03SAW>9$>H$!w~n)n&p2D2IVF*GEo<sM*R3nW
zCSH<aJmQm9qQ7>k%MqUM`by^;t(Alh+?!#1H{S9SGsty!E;;hGD)6u#Nb>&AZ|HVY
z;(fjO-4}->d|2eZ%AJ#Zw%Kx-XZu6>L+js{%9y&%OW&^}lq~MF{FTdz>f=6V($db&
z>=2ARXJEW2Ogk%g>4GWKjMs7h?khdI?V&}%ydCeq*?4ej)^0w{4Yn)Pk*mc(V7A!l
z5?v#us*kV!|9`TPqxF@e$a3TA7j}t>iOzbQmaPu|(q%f#Siaobx>h-Bn(CR)tKZvB
zly^7xJ!>GL^w!E%B>QzGM}w5|DyD$crA#58g@rGEo-#*O_2xDq<>g0?AN+mJ@YAko
zbH46-TKQ$ymMQV&A654-g6x<2)Y&|D?#$NKnNz2F22BbM4fP0_67SaLA3ohZaAIW8
zq{hjg^_3sjFX?bkR$e6U$)eXcFCgK~e;%It|NsAg|Nj2}`T76v?*0A#)YSdY&;PIc
z`|Er=fBih0pP&EP9=^Z-|Np<qKR%}~aBeq(M)$4G=I(A+cXyw#un=G06GvLYJv%$w
z+Z$V3C(fNaaq3rj?$h;dZckVmwFMVUms}B4f01kI|NnxjXYcR-U-$pt_s`Gw%iI6^
z`|t1fzrX9BpZj}v`}_Nc+w1q${r&#o;r{ynY*sxQAge&BY*MFlzyxPs-w;Pf7k76D
z@9yA`DZTyup#c+Sf^1~9ZQ3X*eIuCjS`*9KV~xd$FWF|rC<NKp|EvD~?tZuU|9v$-
ze?L0<f3o`jxP5=Ve|`O5Ki=riFVhPP-K;=X{QocI1-8M>Eg&q+!@WJ;x!K>V%iXOl
zIAqGiW@fqL9~Lo%KT=TWzWKvJ>|of%1uA?h2kYhS{_L;+|Gt_1f83rQ<sTpI&%Q2k
zZ(r(<9}^!wKi|mbzK9W=prx2VG2<CBB|Kz`zrTNkx3`P0uS@J)@0d9gr;1Kgn=otI
ze@TgdhDJ%tm~}cCG*uO+OU~$Ip3EE0ANuG0{rD{|%!Y6O|F55K|9^Mg-}|lH|K;pb
zc2rJ&cuuzM{CvrM_3})*LQrp9>1=LqpE`ZIci_bH2OPZI-H%-gity{}oGL1%Izfb?
zQ}Ig1v8zn31^@rIt1GP0<yj-joA$`TKiWf+c_P=tr>Bnh|M#DlVo@>S;X~)P-`^$U
z_9XoHnjH{s4mE!XDAK3Sog3=ze&c|{(c|uSPjrZ0+Le;jq{6y}mAPtW<JyXb<;&|&
zJd%(V(^2Irn%ksYc)<0ak-?w#m%qBF*!}ze^YDlF@`q=e^JHH)dUN~uhgYJ9A0OxW
z|IhHxKidn6eY{YcN;;co&YalXeDsjhy+cliuZDX0^|j8PJ+=9<y|6H|P2Pm6<q=*S
zZf;ZBn1UFag?4{PX^AOZFjew{+42AX{~K_NZc*dN+SznZpmFBs`HhVa-^aJ{%S-I5
zOSPz)l6cdo?fH30Is3GP1c{QvpP^R9fYQL>3jycO`JFxH785ngJ7PxfWZ^g!g9UDi
z*H*FU{xf(ja%K}_Fk?|?)6A*~r`zf+rYu}CyZ-+LEh#oOwi9c3LH;^F-}uhHvm4TQ
z58vI*<3FEooe~Qq9q4pAhfE3f_P%z^?c|*pr^rYLr>42RZ|!gW5OvFKS}omVdhKF2
zOUzV`O-WB2gVu0ztPF7oZd)4AIFW~EgA<d}zWVb8uf-03ea%z-%^+s~xevcg5BJM6
zONtjjJ@Dhm!Q;o5t<uWOU!RnvwRX9>mg>=Sr%o~St6W*9dG1K#&0Q)Xeh;+F0#5O;
z<@oYmnbJ7NMImbfx9E<{`iWhVY;0GKFe={Po4li__2D7bw&ddyda+4AJ|sTaZ49;H
z!I6clw6gNn<z%nUNK{nUKY#4dA%^bE3}?p8Z#`Gc=9;hFxcayb&%eWyM819b^51?>
zeAm`94l7+Alqqg?3%s>TVzI*ihq>HaR6hN;Km7YU&vHLQ8QW6@pHvS2{mmn9&lhsW
zz`(!&YT1S(ikc@EFHm^;e0gI2`jv|hpFPqN!(gCr)MVW$ChsGUSIJ(OwZdaleW8i>
ze?`^4{Sn$~SM+$Qwl)QRQ23sj!0k3+5=WF)<>$7<*I{j&(<S?45<z+M^mN8fB{rxX
z4Mz^1II&=%;>$N{UcFh9p00lKAR~Vjn@uLu)&mD7c^R~An4m5DW^Sm=-bfMC6{0LQ
zYaNb<Z{Y3z>pbh(-|m>l3;OyE6Q;>$cs_X<+*bKnVqILSMg6>ohuMF*i!g(ueNpzK
zgrp=9@#ld-lX&>ezkRwiX-Z1lQqD6hALXN0Hmpwk$e;X5G~MXXGKPK*FVmJVg9N|R
ziPG8bFHb3TE>K}j(UVHPZ`Z~vEx{*~@Z(eK!`I<eZsJh0;~yoYrw0ZKmX==q^3~MP
zOe`_&U%$yBR)2wvtL##pF;gG(RP11iX->G|#q>s#|Mmg#Wjbo&dz}&+`sORCy^@gp
zpS;aea>G>l`u|2Xe@__{s~zqX=9#X4c`FB`@LJ^lC?PHF$>YaYE=}s|+VtVmB{89-
zw8X@e2EN!o?p<I1drWLmP}F7X`Ny%xveQ&`p{Q0yVOuEIAvZbAgH{`x!VD7pwPysE
zoLHP<V4k39c7a9l&Hj4Hy8mewWj%>sv)Yv1CGPECe))|oB(N$UB_t<LoAzw`)~h|e
zTg?mv9zRG(O8V75M<>9Wui`-Bx%G#VMXI?Rx}@2j@bq3${n;;;Yr!Sj!nvZ8Ig+tR
zv+16U<K$oe|I07=-E5e_=F7U{m3Ydh|MhM2?IrK+PmkEpkoZ2Xjh$b@u6EJS4lRhK
zxsM(^Nx6Du3I~q~KbO&zNsp3}lUvleC77+gbehcWTKeQbVDo|{5?6jktTE--Wf2uF
zw)tY$t>XvpeLAql{Q{TJ7Dc0`vq3(3dpLf6Zhv^&yiL|x!Yn6YM@i?yui0#~^wdDD
zsQ>>LB|duaFwxv}s=rT3SwVo9Ktgh2V#`v_j`jjmqYLSm3QaivJq-BCzw>P4#0?jG
z4o@!GaQ?w#KP!z)*0o)Za*D3Lwk_FWY**S&?62p!zu!1!_puE{s)zsm<?)|ybRq8~
z#5Y@JgBl_{uJ`%#-W*G{ZJK@W;L|(DCRb|sPl_yAwej&aAuk2{hk;D3(|Gi+&Rw=;
z-RW($UMC-0%qZN}nsT={n%(ojt&7*Eg&IeN$5sUdTm4xUxT?NQ#a7j+qK&JQOO%60
z&Lb(r@pHo$_v)F|{LlKXyytzt?Tklf(21FqY4<Jrw%4WXgM@3{$%ALkXs8@{{W3Ek
z;>EfJ2hN{6$I@NxVR?`tKk>k4$!*VAmE4uCikv8AoY%#6?(gXX2O3HxL{=&@YaY8G
z-gU_6dwq!J998zL#q8()aYsLCTv2Xt>}*Hwv_m%qSZ3b3yJu$NL#MXg<(WpZko>S_
z_D0?Y1p(Ll*N+_YlaglNSjZWBaP6Gau5WC)TNIL>hdyztvncxSSZqIeaptPTg3mVV
zUSIj18>{_o-*-a+y(ukXbv$d0tQ{A&SNsa?JhV1_nzYrPV7o;Xl3Fhxd2%?Uc}O0s
zc-yndLu)R(T}0BfuqtNd?xaZuyRMh&aG%?l`^NhC$Is$Vm_QA(MQM*7JV*!(7Ut*Q
z-rK!dK<H6QQd1T8GutOoEu5O228IR(2X1t{a8%KrA|!1YHo-fs;)$aY=h|b9s~x@B
z|8G3$)UJ>al)=e;s=-xy(WBr0<zLmQun99CwqN3MnB%j8($_!#`#0A)vPEP|s;RN%
z<*}KYv&qS|?JSl&H$T1N_q16rA%U_e3Y5GrUY^|Bv4NZ4Tu9(S%M`8^^BTjoKO`|r
zt>NM6;o<S&;i;eKB(U>?g9~R+%fY1^R=95yk3M(!%&8NPF1-2o|9`!YX2^F1Yvy?x
ziBmP!Bs44)-BG<DlU1$bozRvkjc1PfDEavCym`a3Z5xRE{M>ks&B+h1M6X@~brk;p
zU*rcWsIFa|+S^^s!)5g3d3t(EO3PB7430nl|I3TDet7YxJ!biV&n7_+8-;jJs2aEk
zggr}E<96%W;ga^p{c4lY!4+<fNfQ}^d4p0}%vpBwG;CHZWuIon?aQ6k*f^8#LdwLY
z5fKtPIua3)k`WOSeR4~qenHCkMNXh1?8=oXeSO<_c}yOF>~7iOFjGPx^l*JfT1tz>
zOitaDiAf@=ZvwazQyh1cZDC?t#HO2IU~KCoWBM#0Wumh5SzaEGr!12!=J0+LD!Sx;
z&3nNL1&!9NuM(4z49wcN-rYa{;hn5#+X=Xp0YQ^Gx;F8083_qINJvRZN=RsO(aKuP
zcxgwn(<@&CLnEc_VuFe7Yd@bn(6F@WKuG^Jivz2;^VHVmsYgj3YY62Hn%MA6ZpFzD
zZQQfGB*f!ZI4te(RGPt)a^T33grq~DHq`m~JiosMCQjpn<TjJS6Q{V`T_f9DBsjT?
z4`vCwMVw^RWt&jFp{M8ngjWeu4l!))Y>!b@R##x_F=-6jm5^4R(qQ{u>EyrTZb=he
zb>9lC4b0gkVE604{4CW4S9P)s8P<ClWD6+$HuCY|NlW9Y|7TG1|6IWjgQdaHI(m^t
zVMo`ew$_>L?Xx?&KPPxgo(N^LIro5-jXCIO<C$Y`nh!8b*}dpvDwwKqVkw`G%;vw1
zq7Sr$3Nl@?BmTH+iz;LV%w7>;d!j+7dwJ@q2J4FkhK4g{8`u0l_u;kp;f={6YfT}o
zCnfvB51)%!*rnawW}Q6AX<&Hpvz1D!f#UPJh@L~*B2T2+8Fd>9MAv52`t_`0iTox%
zNu;Rc-|>ZVavuwX{7ZY)?<z=0m@0?G`uzX@|FDO11jD*z8tc9sJasA|`S6Fw?1#_K
z=UMJI@%|4;3@RBHZrE&GU6JAL7PV?+r<0$zQ&#$c(>zbiG;8cy1-JfiH3%>kwh2}2
zHc$!;IuUfjmv_xy{~fVMnAdc&S;{7eDQ4(778(RaHf>&^5Hw-NmxCuyCZr$#@QnBH
z@qQljya|u;Rp2(gc<I^NdNU*2d(#F7|7%Wuk{h;J3GeZ1G<B7nAhj?<vw=}lBd}`r
zhdf3f)oasO!U}B|bniY`pW0!_r7ix>*X6bJ%B_v6I~TZ`>9HBUY@R>=;N7_o4>GrH
z&zFqbuXa2cVo{QkGrzfcwNFoTYpaa7xbd162a^j4n#>bj5;$2cbrvwJj68neL`P7_
z0fr@#zvWAPWO);=iJUm~FL^?RNHUYnQ+A`yXQn1*lL|Zbuzb}oTH?fVamfVMiz$Y@
zj$%`$q;zyA8yF=eq#b)}e+Octl4W5*(WWEEqksL%;^x?}u(F3SY^Q?Zj08{iz!eU+
z6?;B>X_=!VVAo<CxLDF(NKt4(=8>o<f%!=nCb2FPa|<!6@;e~)G2$}gGQBmoE{bSt
z8BLjH^x}oVhE0#SMb^OWxN%EMP0gyKtH{9Uk<<i-)$sy%qbd$Igg7Zo7vs=B$PguZ
zxIyq&#gY@uCocAMUI<BgmFf`MxmCeYqK~UUjI*MPm+gwc;VVbaoLR8)<b)X~6x7{|
zBA}-07It)hR#mB3vGU`AF3pVlLR%ADRu|5E@a`Ck$Jvhm5>MIxi5;kS>iNH-tMDpQ
zMxmR)gH@3QD^m_k)48yt<H*4Vb5nx^Z}SLNMUG89ZBwTzTFf|jXHMhn*&P2BK`ktX
z|NoT~3pu!ojvR@otU9H*_N9}B)XlXUwyAG<f9k}EhFeE}@E6<eNpN_5=Mm$7Lm|c}
znS`^CmWXMkh&W|zd#rDBAxo)3=wNu7LBbOLrdHl7Ms93nWo@&h*pidk{QQLWKy!~`
z;fz^YYAO*2zL^^@;5AcNFP!GEgy)BoT*}k{Hk%9suAculx_Gx7V`qM+;UazMKory8
z<EM{qk(#b>?GSVJDTirCLqiii*?A-c`<L~cIOXBeqG&V6(T8WfdmzXg|NkqQfr_7B
zzooS`Hy(T>r`O!U=^@&z@<YD3QS4HH6?2d|OLmq6b4FIjO5Y1xf;l!EGg-zIlgjq(
zO2b`+J1?32gPDC9XNfQ~H_w)Co-51D&u{3LkdTl7jR&p5g2Ig!*4dR+T1E<!Sob72
zoL>BYV$fmxC#qFFJ^ur=W!WN<4z1Drw^%8qVdV_Q*-}1^YqSbAo0*$>17da=WDCDp
z%Nz5uErIvYfd;*fQ1MNTH8nhmi9D-T@#q}k&4+qItMJ6>?y9OR4(`oL?&?9_Oo#rp
z&q<PyoEUs8J|*&s6$jrxWl@6+2IGYfG|fC#>7UT^D=_uuyr$2yWSTsWMBupw-P#39
zg?uzUeuwz>9=tj8#xzZE>8v!z;>XXMu`ySAdR=GElr&f*78iWL{rJUH!)t$&mhsrA
zvdru0TYcjImxq^O1f!>K$QsU8p5g*CSIG^k+-yBY%)WCP8yy`Q4jyeicw<ImZ|{b;
z(2$v9v1Cb$tn5@5mm3pii+H$lFbBCTntPNv-mY(3nasq1f-A}u0?b*P99CbEn6Tu6
zi^0w(jB8Sv=P9*Jv{?O7Rmx_qADav(Q~2eU&Q3?y)`O?I8k?IZbW2D;&7WYgV`t>t
zxuT_|t`QLu8$=~;TxQvJrC}#)BV%+bqw9yxBlh+GYJYwC{P?(h{l7oXvtm`2xi;x|
za69-&HpH#{^ZPicbUohBbASKhl`3yM{-m-oyBRWzr#Scq)zt8qnei-J)^hv`Gy*Od
z35q@2ye&*bG_|>sVIJRvofE8D1v3|lGe5gx^G=!JV)C#1`~Tb5|Jzsa@aDh2<?(xd
zoH=z$Md{GWB}dk;(o)brudQ*S&FX5?w%e_Y*;&ea?UGtF9nu_b^sK0w$of#F>)8MQ
z|Ns9u_~F^v{q6k!@9h5m{N&{Q@%#TJr~O;M=+J}Lr&Saj-e~a8GCCtFCtw<}{OJVW
zfbjGS*V8S^c#L}&fdYl$|NleNEg~Xk-nk>v+j}%Y{fzOVzBq$JNv)h=(;U|P=Z`<n
zc;0Y@tnisxtn&{2`~U0f^NWkijnn7Z6g;@MIeq^9JwLy_z8>%9#^&hA$IZR+LjiAn
zeQI1>+7c^1@dH;o4n{dAhB^xvJPK=@Wc2S(;*P4X=VzJTH_!c5`1x7<iC6J)acq{B
z$3j<22n)}&v60xhBOpxp(UwU&N>7}4G$WH^)53>GINOqsf4IpCPJ4%@TU1n9`uI*g
zawH-!yRqKD{(xkLV9%Xm=X1g{I+=U_{(pYn{&>Iri&s_^kB&U%m*1!S{!7*OcgMHo
ze)pSKQ}*G(;(d13a&lr7%rgUO9y75lmN&M2Xe6`wgF{FC|9|zDm&xC~JOBJ#=bKx9
z{4_p%b@l)Jdw=R*U;FPpT_QMGQdPBcmR;(R84KLCB`zGf=*8C|<?G8<TH1Db`N!O)
zkn-u!^=AgAQ@3oIl#+H#s9R#T!2gA%Vc<crf30slUGArKoYiQvum3mC`umb)Y1jAF
zmfzq1|KERmG4;LG-;+N-+dXq;^?dvPzkZv4>e6R(66{<vNy0Iu;ij@{A4|>3&wqda
z_nuz&{>`77mqOn>etd3!`0efg%gg@Gw)uJg^71D+ng?#BJ5F;7W1jx>Ny>+hi4&$D
zPX~>2{{Meyrp1jLGuEuVnwl~rFgzq-8PA!u?>fvhgHDTV_V1pTBxh3*9=`qcmzV1H
z^^fkD|NC!$`1W>tKG~{ES4ti|J}$rF`pgHK_7WR3orC$;vKojyHs|F}`1ASs|7&aS
zi|hO-dV4GXPqyX#J&7~y|Np<U^Z)(5%I7jb*-@P}*qWKSd8T9|qai5Kft~W?>Cd@y
zL=>*iOH|?hQp4{rzrp0=bjHOQj!%`icseZa@Bcr``umkm;r_b6N(GA!?63bHzwb}k
zhX<FJdgt$}`zveuuRv6BN85FY2_=b}Cc1Knc}L8eQvLnkon2pl|NOka)AH}Xv(jy$
zt0lJOtLL7!Sr^E=W>ZE^Ss9zXJ)5KBoHj^@;?PWsoSZABrjZl8Ikq%%dDydVzR@sU
z>_!)FjO&U2-qYvlMwdN2ExzcTz>m+*|1Wl*FRr_*@c+N#`tfy%N5qO|IWYeZl>Be_
zXdYwvm&Z&elqESnot$jX&R27P^Yh=|-@8bD{cnHx?QI_IFaw(!k1a<f{zj}(WnboH
z@$%(K1>N4uHPG;wXVKDPk&`QW^TviWyDdx%dv6?&d39i#PJq_3{q_I#;%eUg{Oo_E
zjf?qkzkI#C{l85)H{UMz-~Z_B<r$W8^;er$i#U2S&bYR4x=i)A8vDXW&u5#fm?-`G
zmugWxE%CD2r6{?Kmko^3iw>@uG;w0%<jDum^fY#J?>-5)e#=(Xw{L%5blYXX&D?XP
zVJ6=OF$n>oz^^vd-wMyqyY0d2v*<uO|Nd+1_t*dacKrMMdLgaPk{_;dADN<5QDncX
z$-Z7f&hAg`rzicJ(>?EneEDU1cwH<{{(Yg<a@RLVq;<1|#|jG`R8VTV3mxG*G|%G8
z*QYM76NOo}++^JSlWnbNQ{2=CD*`T9>31-_ys?q}{QP<~Gn<45-`|6Vr>#m}+-zo_
zpv1oO)B&B(7Z?oB3Z7ZT6)fg$dE~^2f*&9Ht;_4`|M}G1WnumQ&+yKUg9XocIHNXf
zGygb6EW=mW*LRLKbR6i=B#R?QcdlJ?#Nd$ly$1)PoD`JgG*kbyF05Z68}+5>$(NVv
z_xDdOF1olrzJ7a7<(CHs({J=fu?Vs(II7L8XnG{v6IAY=KXRnt(UJb_>nCiwU%riQ
z3tsMVxvA7aHaxR&*~FPM4_=$rI9<GZBQ$^~SxlMY^7YFNNdf1_{{x=8{7XKjH_0fb
zlVS5pM%Vg(e|DF?{(67EO1cpnTU-48e|L9%{rK!GzyG|NgMzyY%noL;t~T;%xXRV?
z>;3)zvHSmhKG<xpA2*?L!tL+xdBWFS_#k1y*0Uuk`S5}B{jQRbl5&zoNT@3d%Zz~j
z_bu1;PxSJf`NX{T%z>y~3)H7<&)>hN^7GsK_C-SM%*>!Uk@t6g);#CilcqR>uV5<6
z4llu^E@74*A0Pj3<+iu4QTXq6bAP?0oUPKrn~aPVv!r(H(k)<p^ytQK9f${={ACs`
zn&ali^ybZl!on?wZxw9!{J*&9;syB}Q2~?MnxCH@+h4mFu;}33-ShMA|7+Q<bRbGO
z!Ix!5)&{S{qJ^*O|NW`|`DuOmdyP{+K!M+5x^5dQb6*zgxu;K44Ga%nN{3XcPV#cX
zLXQGNpC%+GCM5h|e?8C8aaEApvHA_no)QVmO&l+YDKnP{vF@8FQ+bWIa?vD4<BtD-
zex5$&cCAfEqT<Jg({FFf2P*%T@9Q`|RiQ&CXT!F*K$CyJzP>-!8=rqaC`$XmeEp-$
z4<_eE?BwytWU_NTe)#Z$wPz0;Jow?QbXOqMxB8YYE)^{;Hx#zdPtaD}ASS)zmEa-)
z;mh@Wd|g7vkN5ANXZQEp-{10g1kX2^%O<Q1o^aYbr6Ka}f`a=0|NcKa8?VH^uYgfa
z&1k*I(Wr$9=XNb*3=3mROw{4-hx*e~CMU;7OY063)0!KfxUQ*ulHX=0ve~!m+M$N;
zF%KT{g&sJ3_|*CHetsSY4qRVfFK7F!%kUN>Bg<u$c~$Jb)<sS~+npc(e}Dh~-m0(f
zkMnm4v8$^yTfcN{GJPajP#!8VVN&-BjpimMX5p+zsFe>te%P>bWg$0*!6I3;{#&OV
z_DQfV-I9>v@_I+o+av#6jVH{XAL`iH|73y#6DYoZeYri`+&<tj=UMI<OF7b57IP+r
zHcb08VQoT#N*}-czn~k}1q2V?t5x9m(37=*bJ5mlRaMC=S1$&ufD@=wX7Ps2o4Yy+
z3JMh3`Gnt0+5UgS6c$dYifX1h9jQMO7A8j;aszpk`ilw;U%q%Cu}GQEz}UF%|G(#F
zXUp^O)VPGG{x?vXG2=?ZX$O<{fBt_2&88?D8@oRLdGh4MnHCBRJ3|{aTep5afBw9J
zN=sZ3H151*bpHH#<KuJUUej{U9~(Dm_y$Jsb7h}u*v0<d=0`mn8(Vbroyqn`7#SJO
zu4J_)Fn-`KuSs^Z`RCG>^sit3|Bs)N_B9EA{?7h0%hkWXWvx34W6{ILnJJ1VTU%Qj
zXG_=ql7skfb=Zt0ODrDL9kZAubMeiAYugT-a?M_sl)wGd$wF<dN%5Qi{QBx2GUe~{
z^ZRdHVvN!{7@V_!^Ph8QzxqMN*@r*;|6lLv*>&r>jSUYU-;sCLiv$$TU(nK6@E(>G
zRQS@;+(JX;3=P#P794ngw6R^}*wQ;Q&UXKoaDFx;IX>gf{xfHIK7NdP^X5tW$`%%O
zdwXp+t^@JD3!c_(xx&Eca?OIn`Nt0l6O)L#ze@XW7?`C6gsVrJGcyb723R<)OA-;4
zOi2xdj{Q4nTC%WOva()$^j@Di>Ek(jw`2c%dOnDJ;NSVZ@#4gv-3Q&Y?E|8g9XxpI
z*RLnrZ6YEir%v7Z@6S(f7ZIiDtt?5i0zNjY-u3V@I(fn%_|gLpFQs^W=6817jtaWH
z`({F#cTOsO|Nl$w*lBs=hJf+Q71v)L;^z1G)v%mx&0N7Vr&ui=7d*XE&%^V7%NCZ)
zm$ybnp0sT9^68n|-|rebS32{gVbqmHiaw4pp`2+_Ne2|UO~c~mc}C6hjG86WZ(v}s
zVue9V`{d=KvJS^~82na?U{qFS78kE-7J<aqhkt*6pFe-TbI)#;=-JFqccm_IZOrIo
z2>*3pRt#v?FFEO#xY)8MPgULBG(5bEEv&oi{{4}@-CB2IBL}a^qHeJ>Q$F$^_9;p2
z>G>ZTG9xCY<3?@=V@s>#s#PZ>A0FVk5X79O&N{E^!K;gY@1cp&Q)|+sH#dqzKkY3m
zI2Lcwxp=|>AF&UoUhtdg{;#N*k$z@_sGwrfd^sTjk(8-ZpPHD9KX{gQsKX}4lxbbj
zQ~NfBgSVc@>)cd)V`PwEW|A^-y87A=32A8^{mcC}Hb#0Ryxo`NxAnw{1szcPDjMd_
z_4Mju-7N6$zx-NT#{H8d?>+d@E%JIt+nx>6<o{m`@$%T9AN@>0-Tlx<mosO2e*Qeg
z#H6!9S0p2_ySsO5!hxwA8?>CS_1y{lb^P$*2QN=PdhC3NGuOw5M^KQbwbdg0%|w<0
zRo0kLo;j<Rv|RFrOcyw*@OgUj85*)}DQJl;;@tW9ufFnQ`yacKdd%3<HXXSC??BuW
z3ro*4^YgP46kH5AX3UUqak+8oR7hyZjvqf}7#4lDzoMbCEa5=#oIi&Xb-uYQUUuxs
z+q178A9hOVH__=?xLf*uR6G~s#I+%kl2Z;sr+A%I_~hiu;_lf?EM(y2zEZ?w99KA1
zOyCRq*4B^mi)T1~iJ39;zqM7@+Q?*aF}42V-aS1$j~>Y^U8=Qk;iOHQ-t0)`Gv1l-
zUE!S*lW}6ARJZG?v%Qwe&1X+{v;Hp-Gd6b4)I7YWl4sJSC*>v^+W6<WX<VBa%*MtR
z7&wKG8<M6}a@5q)%F3ohRxEifHB(|j-ts2t4wDKldA0N-GZwp-Dco{1G%-`@kd%=8
zudQ9YJ|a<8e*J_gC7+&p>y+P?l;*y2b>+@oT?GXONiO{JR;aptlV@gTez%Inzsu26
z{>+*GckW1tibg&=pB}OE)A`58BOW<>b|zj_V$;_4`5(ygfLS8qwRlW-`oI6_51zP8
zgyk~6w6taO?f?J#&%b~3d-<ZQ2AwNE{?F%pTg4#3=^wO%?VRCcjyt0N13fjGCw|;N
zefp8T)%V!>_T1U~`{>%}_<c1$4<9?WV#SHneEpnsbsgoyebw7V|FyB!ef{vIaZbR7
zu7s5V6MU0X|E^zp^v2brGJ*#;q@BGlYyHo@?#zJ`ClpkUB;=hFkUaU|fy1qjbt;TO
z!EDOPhu6nzw1TG0|NnQ=vy|9h|Nk^YZlb}#X>S)y<@)fYvF)F-DyxmK_6g&H4=POl
zi6>q%pL+Hv>B}df@1KNze|~=d{r!4L$^V--8X6fIKIoJCwBv{1L{0u|;pg(okL{Bm
z|37xD=j_@4imNks?8vaQ@(ViE(ER>>pb%tKf6d&DTn7|*T;@->`Tu`V;U?LZf{Rz2
zGTAs8UEj?+;(1uEdVW_m`^EkhU7r|Lm7G#qb8K4Xd$;r`a;9Cc`5#iEZu~f*K1*v8
z(<Gg_O%|^^IoH4WJ|od>ZNAqZ!TTo|qPPn{1Bw6tJE`>5|NnPTn4S4>JsVq^o7<Dy
z4l@EybE@5N?U=#K$0jkIu~de2?ZL*dodyf0N=mSD7Hu&U=&JKE{XfA+IpXC5q2uvs
z3A#2Dg6}mj#^?r^tbBhza0fJZtMow@r1UpFy#N3Igw+QBl&&iHcyO{l4O2S7yY7Uz
z-a(HiJKVJ`H|V|p(SIcEfSBP7&-NL!nod}xDV%n&d3yYTm*kGNqZfqEZqL^!hE6&-
zsr1da|6lv_)A#@NCm3xrba|dkYG!=7OONfw)PpCk-!y!%B6vfX-igdd_I=0LV_3rN
znZoxNB^(i3rpM;Q)!`|><JFW#e@UJl;y)6&>i%{Wy%mRMj=uQ)|LT6TeODI_I>?Zj
z_{aU#?n5_wcrIL?_uzQ_LoKI(7;fVh#*-#bcBCAb>r${(=+mNU3q%a3@?E>cWZB5a
zaA2y!1BTUazQ5O)y%HJ(dY1ocezWBl8yFZktTXz<pKN?+5{FIgpX1jQQXNEH3JtWR
zEqxbEmFMwLiE45<J#oh^d6lIMMpu&6nAR*k(ZFaF#Kz2gczeEvDkNPbIZpcV`JZjd
zl1VTB|F8G(d?;_T@Be@IEoKw61fLWduQBD8au9A=&g^!@;HZv)N0*I_W?SP9GtIi*
zAKd*1xi|QocRKL<yGFDU)O{*_@9+QLSO1^=nc<c@{dYngr)UeXtvQx>z(X=ZGJ|35
zhXYY92}*21x8y616sn3(3X;3_>}>+e|Np`ofp)b{nxKVc4FCT-y<7XI`a7R-p#m5G
zl7svnKL1^{{1o2uu{qgBm<XCof8;62vqJE&LlRfQfn^M9O;0uGu1)Z5IwAG{(2WyS
z>_QK+s)b~p|M)-u;bZqH&dZ^WlVqv>|Ia)}k^hsyi~18Q*mP3=xXPNP9=P_&;oi2!
z`bg$IR}MxMCHRV+C_HwcLG@avr%45u+$Qe+gG>sZjFN(!7tCHj3w%$R`F4MQ{rRb{
zuP+>wFLo=vu{84G$Htg6hd!tF_^!^d%?m7f!<(5`b6or4$fGpl@kbFGma7V<{~a#)
zpTIc7lSR9!x&FVzzB*S!X!F!lX1@LZy6^A!_4Qx>|NsBr|MZsk!A-GmPjz%jOz;x_
zp}Uf2iHXI6eGal>zxxlW+0{Nc|CGnY#q!#gqYbOLckbs^XXjSfx}~ANt@hWE56q?i
zq#>!$Qzm}jpSthw`uErv?@4A_>GjX~G)qM_Q}%b!nWr00Da^2t2xykD^fg%UER8K^
zV#}F?1F<;*KD;x!Bn`eSS}7CLuxGQ5!Y;cy7k?ROIC{$5-~Yez@2~Uw>(`t)qfr}v
z=5$kFesJ1>i=K|sHx9fsb7Gdu_}HT3?V~?sfzg?VAxRes7(N(1yKmnXzW&1YZ;-JG
zCr_FEb$@IB{P^B4pL6F<$3E>fiH!2M9p0TgAlbvC5z0Kzk*~(@s9RF%0#}=57KTfi
zZf008Y+3dEIDcF9_Y3EbLreSBVgK&!{QUd#^ZGk?{wJ>9AF%A%!@XfWQY@GB5)%><
z9C9lYa_fu|CMP-gGR{!i@u>6154B~2R*xpJx;*)u-WI)G!fvtLOAbhAtq%Klf6vd?
zXJ^+xdURnUZ_eu~34<H|KZa?{h++Bh<Kz2xcjxciIa5Za!ot$<_m&x*&5Zd^5_3z9
z0;W|x=v%10qp<nmDN#0YaW+QAR}vlj7yf!D3mPA>4?OP+b#DBGFQ1;yU+(`uE7LGR
zg`16y&Er4o+GM8G1J7C?s@orAR9<)B@BjG^pPr7NI`#aMClODd&fL1?L3Z@TU+o#2
z3zvvVv&@#|*&%%SKtuX@iBqQ~=h-Aqn4WH6s95&UaAItjPb?1)kAA$7O>LJR)JlJu
z_xJzTetKe_*2>hMd?oeRgk5cojg5^H1$Tb!V%rd|*VDsufB*m5e}A^OvCW<!@acl&
zwWMao@;8&B<Pzo{mX0yyFgFKzV}JeRZ7Q<<n-0tmh})Y~@n0@h01|BeGWK<Us(*hw
z|Np<vh7)co6aW90l$207?fqX_oo~i#!`*8#N_oZH*T($(_dTDjwA3sxP;H426Vrps
z;WKv5VN`x6S)kr}Fs?a)$<UBZP_Qk2zr?q9s%5!W^Byr~D>GXz1dUBUWUjKG*4PNm
zFY>qleSQ7^_V)UJe;Uu+{h!<*ySrhFppc@g$)P%(<7=bs!`A*fa_pFd`0>+cdR9J_
zzZ51O^XNnF*BM&>6IcA{j=q^NQ?ijkOpJ}6zwQ2h$!)pH4~1Ai9r*vB=l;ILm&c7@
zYnz<>W$OR^ss8xr`|0Ts!Uy9+4*vJp_*Fhq)sXYW@&5gj)%RC_dg31=vwoePc2-H!
z-S=O_XDckTSQpZ2dh~#;(Tj$?w#~DpK?}@gn+xx2Y`qc~n9|F`!*hO~;ho*>OP@gV
zl)udWdgD90zCORG{3YS<|Nr_qetdOd?6a6WUj6J=PF;|4=l}ozdNDr|uddQ>YVz^*
zW!g~t>&If@nIGjZ`z+a8a%4&8!Asi?uv%D3PMRdKuRc9uOUJ`kp-GSDEm@`^b7$}A
zg6DkGR3NGRkWuQdhllrDm;W<RnESZ7@(91+(ZlsuG#oGPD11Eq-#`BQ`~J>0_m7B}
z(b&P#7#Wh_%UEx4CBe6>*(%b3L0P$Zs;F~-z~Pz3JagyrxWv706@B1kSn%nd?VIJ9
zki30p)Anb_`~S<^|7*?hYb=ijm1c9fjyD)SV9cNI^etWK{8^&~u7~H_|DUe+_tu3E
zAD^Dy-`mSHZ{CF5bv!G&F3xJt*gLh};G2DjqYbY=TS#;pLqbZ*gb4{AJ}DN(ZHY&@
z+UoyHEb~be5K?;e%Oi<lj|*r$&)@1Vf_xx9g2w%WivIll{r|T43A5J!JUssupR%V+
zx6PW^py8dsUh3ew!PNJkp~0W(e}BqDul`=N=sUmszq2PEXzD&acCq9@ytd+WZI;^$
z3nngPpD|N%&05K<Y)KXtiLmu)9v%rhDuqwya;2=l*YKkDbKAqG-f!5|pdODavZ?v;
z{qFAiIdd-5X(=oJIZ^C(<r0%o<oO1Z<>wC^Xt-mVo|2I8<NN#n)Aj#v&;4zfdg}V;
z=k@i!zk$|1URrwp|G&S^R_Po!{b$66vF%yP6vb)S(Wb4zvStm>^K*{hrE+f{D|pJ~
zTx;v>!qS**<018LSFQBv8ycYDd62&+S%4-Y3LoE}ZT^4BQppF0B|Nw*A27%Fcs!Q*
zANYFue;yvt8cwHkRr|``o<BF&zWmLP!sqAWZ*)jDuhP1Fe*XUn+XNS@@}1e(UT+|j
z9d&R`SR3<?pOYURW_Q_E_o-#!sxDCYA8=sP)@BP0UGy#Sael#_eP=(sk^E)?a>W1t
zhxX0?{@A^umrwHdFLr6+<c1{-H%?ryujool`E&jRhvLH7YWMd3-d+Fy{oC91d3S$3
zejm@Kt1A{8JMHrFdi%OR@Bi1gou2-G!-hZ3UOpMg0U3oo*Y-Q)l_kiYmQs6g&$f+Q
ze1YNgNrx14&u*x*J-pDFr(fPU=gtWQ_2UZ`98gd=aNy`s1-)|#Szd2;9{(`q<8%IL
zsWOOx*}8v!zW+2{vh`p~h=IbhNC`<vg_jI*2{nIz{lBzyzq<dw%J1*)-`@WJ-TnXN
z&(GaA&;M8Z^HY4-G)<E!ckj>tdvEXl*Vq20BqSuHr3nZn{rT~6eFw*<i8udmROO2)
z<tmza(BRJ`R+c}%juyO?nv$Dv;K8d?3l<+Pc)|f%vA5jM@Xf8G3Tnp=96x^G<Vgj+
za{&=29=!31n*I2xcU$`Tf_oPrMZ%$Z7IFLk)xW$HFD?Bf*j(a4D%+bZ(GHO>5Be@I
z|L-&B$KPjXL4o!E*Vq5Y`~S1DN}fF%$;5m!Cido&r!#NdkeH_T_uSdDAHHfIzQ3QR
zrl#l38y=6qZqG%kef29IWo$iT!1DjUam;?c*z1})XAT@YwqO|uCFGyqP_BRYU^9<-
zp242l)1WEA=jVCm+Z*56aqz(#o~s@k&N2$te`t94DfLTs*aL`T@60jeYH{Fko&Va5
zi`gvrcBEzn_j&{NFD^;R+8j|Fw>UOQ=}uw1d6PTmyCO@FjmLxfM{UXXSFX}_)ADD3
zx8bJz>w|H+zMo%O1#+$3tMTdakEzS*=I!5nCj3<M;`gdNw$h2Q;gMoHz4UZ`G#x&9
z<o+baj(};~`Isg>xfYeobIjoU71P|)CL41_Se5$o{mW`C`1ulKB-`dQ&jD9Zhbr4n
zOxORvuQvI~v$Q||<tryJ2evrOS}|eSvGeo)_x8-#x%1}D8xk$8l2@-pELl49hJbtF
zzx0HJAOHT|KiJ&iXyNOtmYBHa;`xSCuG%(jiHDiQD?Tz6O8lIx&U5S53$BVv$(|mG
zEn62xKen`xNZ89RlA7^5?cvMEufyBQ-xru!NWdm(56!cXkd(Bm{q_IRQF~R@3RR~3
zb|-IcwPpW5{H!V~b5mA!V`dg}b7NCeV+#v&yV2;w!G3q&U-7z$4J|B-S7<!$m#?p^
zkO(}@@S}kx?U96(^6U_no*teLlaJ)&O4`+>2#C4u_iJr#Zk#JyeY3$PvR`yh?dgI)
z7W+1aL!#=?JPU<n<@t7wO3e4tQVw}2O_1YaR`K1i;lOE=CljSs<Xc!stc(B8{#{i1
zcKwag7d#eOVrPyRHi4?fetF}Vy{8TwIB?*|5l|97+xvG0@22Os>;9+hsFj`%nur4x
zLBVScHf(fU%w=}qEC0(iegAoQd?sYECNEW(CI8RZz@X;enGIE6_4)YP^y4KYBqZ!=
zf90nf;V_raS5E%1U3Wv%{(4EhxE49tZ;<8ZO-&I~9GiaKx1ZGR{{L@s#ot*EAG_CN
zLA|px{NwNPir}a6%wIyHT-Nd@D*K)|7~P#wTokbK`T75LRbRv>8O@j>@#f8pgmRC?
zyc-uAx&?4E^BNc%*Zl43u*uDxqI)ib;pdgWHD9^YW;aiq`0#`v)As9-mj0oc7IrlW
z7DX(H*Z%*XaD|~a@JB0`$HpK0(=sOH?{Z%tYrWtrleu~G4vWPN4F6aRA4N%IJUGtZ
zmVJGJvF4gJJUKZx-m^M;b7in!UbK|+iOA>Y{D*IDX3^7tw9O99w3sndG9n^iS>?a~
z?)#d5*efefTm9DXR1atza9{oZ?U|S56>l_Ny}#c$BlGX{DS!HP9Tl?WFEVU0;Vx=E
z^!`4NjLeJa3lo^OgfzKJGBY1OJ)KA1P9S0dXoJxI|A%fqOGrtn_}=&My!?^QnHv@c
zas0N=Y1NFGAUJ)tUCodGFD}|wtd0ns>KG%lcCt~?YW|o9_wC!Z=P&%6IA;z|P>_YP
z&}&ZrAIJAeNJyNUlUVV&tpU`r0LAi5a79zv-6;C<$^ZWy&M*J{caORHQNGeac3B&z
z<+7VM|L5QN!EfiLt#n93<A=OvXs;S~3ByZWr(Ie+JUnO4@a(T!cz;Usni&$z6Qkuo
z(^~sMYx5ZQLSweP;@E6+3t{m!^P5(PE1VU)(Zao`>Bsl?_xoh)A9`Pih?Kl><KOC)
zh6P1^FTSbOefX??_;7nibmhL9#D?%w1r5t{4#+%o&@R2VC$Zuy*Xc_FCQyT?TPWCS
ztjUd4ajcl-@N(XPi{Ib>m$&`(|E2IDA6r{D$Ak4A*BAFNl^i&6;=^<PBmpOf2FDPQ
zZYj2$$1)j@GE&)E{{FK)+{$gSJOo<)9hz=oSD%)U>?}FabycJRyYU3;EgE;a#s43^
zd+_-21<OE{d%-5zlMim2w|#ytnEbq^#zNiiV$}v758Kd~2mk9qTOQ)}D!t)>I<egD
z_xbr3vN+!{$!Id)TzlfIj~iRt|Lu8(cXl#QO-)LgFgY1CN0h$&R@8}y><SaVKjl_(
z{PXv;f>zhPXAY~a70w3zaeh{C@bmMF{uPHni2)R4;YBrnP89qy>DoEtauv@CPS&fw
z|Ns9_|M7*>@Y1LE@}P?1(xsAJf#Uo>zVlnHvdB1iu({)iqQ(gYy}r%A5)OpzRAAmB
z*jN8wf=|Zben7t{)V1Q?Yj)^sHZKX@@%w{F())XgQCo9zc$%6b^6ZvLSNxv#@U%Fa
zn;R&*>%}I4_ADIZ1+VO`Yw;^HHZ<($GSbjCe(~Da!oom8!eO<&S&?3~iZEye<3F2{
zE>WRvO-&JQ2NXW`{$G;x#EF&f-JQfAe`bCVJkG-d4X`b<thpKt1YG1h)~;od<`=P9
zDZWuqw@LHJ1mPQz9UZL)`MU&<tlcIvE%BSV|C1SNTgnvH&3cw=c)%hkba$mGhez;p
z@6Yo}_8n)t6xFlv{l;m#mT6?n`da-)<%tDDscfp}JmEHu#wjV1P6Dn+KE9vPSeYh~
zaa&k%(vyAX?#+EVOQY3reMm8Qf7hYu7XSXISA1%D_}rgOPfzEp%o9=79YS2f!iVSE
zYdHS@tlr$faQ5`)_xB}d%#@Unl-!n^Y*9V!;Zg23b$<!HSSQQW2lwsYc(42auhr_u
z?{d)Ap{i=hs%puY7>PZ5B(`joeDYLtulF2<2A`M%uSL|<*pibMnIx_K@{l1Yq~`ar
z4=)7IJ=2wf#EnwG&;S3<et68ztgilRfh}|2Ef(JozsuXo-wRkaOq|#_ckaXQ`LpiC
zvTpqKG5N)TM}4w9)@24Ye@}gQ$jtogZ`P67oQ&1q4Bp&4oRD@bA@k&dRi_?2V2Dj>
z&Q9pc`S2Fpv(b-NT6WXI;==U82jyDNYPgt#Hb4Gd-qtP7S)QZ_amJdt)?5t^JTCHD
zC#K0AxpnDu#Ua~cT^wfLZogz*^q{G=DM`@Yg55(jj)~LQ{OrTOjlHKQeLP~W)M6`{
zwR(e&MDvOB$*J=?m43}zmMD=Cy+*Z<)3Lhvm%{UcKAYoFrzWjZ@;tGrTuf@$r749=
z*74uz3J`fAIN7T6yv0nlU4GUd=4}Np^;Vhy%BL2EO$X2MCGy_-<?<&th>?-a&(G!L
z!9#~W{ILu%Y`w|1=<iA4w*31N_x2>-*wFa!S83bz^^$&b7gf$J{b_Xg>};O!^+r0e
zrzXrgm5_2oLHF!}H9iXx{Zw{TO?~*lvCV(JKx10cAqAbDJc~~$I<x8vJDrzYTbEk#
zcUHu4NN6mXtH{;hAmA!5y-jb%5#b1PX`43ok7Z?xPo@ZLTe0BalS5qx7c6oVxc1Wc
z!$adU|BVggpA@{a(fr{2kLBdobB86ixN)4Hbxle2SU~6*<5bT;1=Yh3lGkXuN-Vmv
zQ#f91S?9k0C!)f4^%bg|_Y9uYVxe~WhS~qViXRtqy<_iN1#>LwQ261!yIJGfC6o7)
zuVudhx00sJRpf3C5NWkHYd4ED(u@i}^639fdy@#C!+ISvOms7a&%UbW_AkgWxSzMD
z;Jj{oG5?$b{u7EUxw{wd*T1|`VcXun8$&;>lX++IvF1@_#56to5DS~WFSgH(J)I(r
zc%J-RHo<C(fiF*U<+CEizMXeDm|~{Co_e{n)RkjeYvGdD8PQ!$tA4SrU|M{<d65E-
zXYjjn8TRvsx0ULz*V5d+eQu=N<u|p89ZEI-EB-|5+TCfJV!~Vcw|x_24@1vvL#{&(
z0<QCqummlcDB@(bTg1^LG4ZjjO`P5W%SN3gzJeT2xS~2bbXLqR)#V9XEL6Ys=(NN8
z{?@1c>VACC{&wB2SLc3T<Mg+l<#X@A{<}&p#(KXqbZVce>XtJsogj4EjpJMOs;;YC
z$v56+{5IVy)&J3kjrq}xpL1^ZC@fgFQAeiz;f9T7+oin<_ZICeuq(Ei^gPFG&Gj96
zrGGz9uD6<)`ok{sRFS8IkgJZE$H|PAYqo_O3^UF?beI$%lE0hj^2^CD*ME;cAOG=V
zX7rY2I{&@SM&AV&C_Qs@`I`y^j@8$jNwoShNvAE5kP-T-EU==E$$r}X0}tM?G#f_>
zZ1!#9l-b32m|^k*?wDN#yScx=xw-f4F88UPhyI>RxS;e{-0KJ1#E%lU^U6IUk9|Fr
z*8E=CbA{F{x5%K-x4D6H*<2NQP8Od_FSEWhMdMMCW5>QD8lDz>hZWlzUmMLgo@@3+
z!R2_&Y7LKIo$jMePbYnyrJ!)W^YQ%pvOM#agsn*?hfHqfeY8oLd9&xaaBTjM_pv!w
zy{0W()uPzW{M&YU@aurU89p1{>TmoPE<OL)fyW0w-q`NH@+m852gaN^w%lMJ##m*v
zIP$&a<>mEl+}-Tx-7J65|5<p(v;&$_O)*l!tVho-+?w&h-}B;Y<GXhY3J%{bz9aj6
z=ZBkSvwK%fR_f4rSXT0H&9;flSQrCKwnvDqJDqZ<sB_)wR`1y2odyyUR5}7gR5+Il
z-_|{@{M<P8=hcAqCj(>C?b5AMEmBO9%!2RR{C`n2tG-TY&Gj{5Z@zbx+;e8Rd}Ldp
zj1KpC=gSiPFG~!3f7QO471wXG{C4X0#{~*~ok~aZczK!)j(nG&yM5-O+NeLL?wtrd
zn&dd`ufT6nknfhv-pJLIAkbhhvS^{etMUyOWA8f^lxM#Tlv0hVk($3SPGN3|P@#ah
zU{qh5impn}zLz`R_T^i?%L`k*sN-*)LGq<Z{S{|^p85ErY~2cxb={epZYSQ}qU&|h
z<cZ}jzga=kIwq+6b+Kx{+VxdRu!-TmyXLmr553>`$Auj=oX#D2t=_o)dFJNS!0Q!q
zN)~*lRof1xIfrhHDA;|mG-kaB>-XOY`<E436j=G{yiT_=Jl7%6=5jGV)E-<WY?-r>
zvjtRAt?t+`QL1C(PTq)yjkXaDwi$-k5(K9yXdId~Q$ljZ!Mq~w<GDq`eg-`A&mZ_>
z-y>`}t8L%$?{7Ij<`zXfnUVha_ZcR=(*jMORn7-?y;kD6S>94G^@GW;K$k!T)l|>e
z$;*!Q{|J_0zuXekqkOpG`QNbMlKHnP`;+IjD>!&fYgI|S-O#_)Z}p;B_t2Y5V>l9@
zzMoyXH23V$9o2il4pq`BjEFE;6M6W+xxPsv1xxukq$U6SJ-wmQ*f~gOT5`ZcQ&wT&
zZ&$DGV33yHqAMwIV8QzH3aUpFGEXXKd++$_uKVGW>fz_-d8X?dpP6^|!xzoNmzT3l
z`?;90qkd!9l%5`*OP6?-F6Hs`e32agL5<Z@_4xjJp4DN7TecbPsc@Vn4;hwJ%6j+b
z7;jtkcL@`d4dxybrH_3GWj)c#ZfxA#%<P<}cX)~Ci_;tr9G!hmHAtL0dUV0clLt<m
zc<}P1fWUzR=gukUol{V1+R$z15HrUyW{zXjEJcF}i64{NK0lYtyPI@l*VM+jb60#b
zI<h}vcEF)SJb{5cA3s{SyBbz2W%ve~99!<s!_F@h=hwptsb`e7RZg3KJ|L7eH|xPF
zuMZ+}|Nk5BsW`ZyK+(D0K69@oM?*>0j2R0y$4{EXqo>DHQ*+_utb+{m>{53Ww?BLt
z+_uykG`w8#Z_dN7*^|1R#l+Y$GuzJiwEd~kY0ix3=BoI3pkb{=qlk1npJ7V+WMNP}
z0$L|;RwyB9m~*RTbK1eFE*B0Q@8>z*Z|pPYM8OLIP#bf9eKNnb2j{`_{CCeJryf0U
z{(M3PC=fuCr%Hzooc7*iZ^&45?yyynGuw|ZosCncCSKQTn{6)n?r!ppEuD?CXFohA
zTk}_^LquA-d5=x=OiAY|tLC-3e(aCESat|;rJOOho}GAJE@^flbeqo_(D;ObfNOke
zvf@h?R_TLIEPES!kAG5-bJ^RLAT7POfy3#|Hjf*J-gGqu`x#H*IsE)@;yb%$0TbTT
z%!-Sixd-069aNIP)g{e#Yg(n7*ODXOZ<bF{>WT<|G(+WJ;fdm|rpF4W<2EG!$Pc)A
z;MlqQ`U^dK(jsfmZ!_%E>Au-7*p#Wn6UcR5nWG?fRhP-$UAMpd{TeJBlM!TVwySTK
z*{tuGptX|!|0^B*e()gA-@iODF*mdYpWI?u`{97p5zuy?zo$SQoay>J=jS_%Ti5<R
zwqWt$4<D78rKOuU7?iA-UGzadc>@PufvbJgPsg}OCPUEdM*V+@x<3gOKPEJ`w?90_
z%ciN>R{mb1qGG}Gu2ZLtH*9n)meo7{SX1kR$`tc_iLNe*vUe(L%bG!hnV@()c=^yF
zp3~C|B_st`gl$l&V%jpzC`MqqkxxSy_kpQ_?<W4YP?P_#B3)sMQ9-D7{XYYp$U_(I
z_#CofhP6Hy&Zzr6E%9a0ivV-U4nBrFK`G{J0b_xv<o44hUJOw|3#v{zTr2vlc6g5E
zljS^YY|!1lN^AM<@0j@ToNSA-!SM{vT{Ctugda)v6+RKw;eYCj|CfxT4F4LAcK!RC
z{Nvx;h3or27}(cAle}(W&7Tti5ul#i6;>XXcGFugl~UR@(}EHXgxMsBs%W^TJUTY3
zYQkK1PKEQK8J_>|+qYR3OMH8?@QN+eh@D^SKPoxb+beC7oZ!SBZZEyI;ovOM3C(Pi
z8`Z4}Q*~^1Jz$t_Y;aY0<_rmgNqo$Lnr%Ns8r1sse0^qr$>h|#yFA?Dlgt>qMWN<Z
zmS0>S4+_VKD5D!U9Og2;T=?gB#Gmi-rM`*>r@gGdaX@?CmL|<UZ*IBR6AWiPGj175
zZ!3sZNj>%7<M8o}M+J^P(;s^(@mO4Xy}zDET+iUkcj34*ZqTle(&oRpNrw!o&6>BE
zcx`WOWV|>v)xox#V>07bxoBoy{~eDf9g<o(<JH9-Dqrn5tY+IxHazX&BNQ%psKJDX
zU(@W}z0?)!+-}}<fE60Xg$9O(6%__Pb59*O>$^j^yYvk6%#ZT<{e1p>$7b<8xGCZ*
z6DiJo@5kXSeE(;#>k0n(mvkU7Dq&K^D{;0ZQ$BuBbdHrh+{nzrSF8kDECxzR`=6gX
zc1%FvfI%V?v$i(pNv6pz7OjHSTORt$@U)zgws|U0WZ9q2(Dfig;@XkMXJ$5E#hJ6R
z5*&QRcz7a$nA1*Gd}vs>Z{94;X4njnQoZ}<_3=EfuN(jQef+}h(+5s=<|jypvmKgz
zbmK9Nbw?WAxg9l|%qHF1b0AA$-7(>?$TgbGlN;}x+fg@n;jUQ^pLiyjFA;$Hc~Qxk
z{)rP0-kh2E$mNJ6<A-nT|B{*+qfa(wTKwy9JolYtu|W3jNh%HLE~3FL65A8iq++ge
zsWI<k`28ZV?NydnyU7O7coD<@|BKF@*w)DCoUY!yP?4FruT%6)n&y?NM_FPi4L8pz
z+~{$=)}14+*KpM##_wpp2y@>N#|=>jC)+)Eaaz1>Yqn%n)d$%;f0(OoZE0^|adu;K
zPEsmK*4=aXP{TyE2P!tRI#+J)OcCbXaIK_b7V8c!%O+3B>2D);9@*~s|8Me+>gkD(
zTwbiv?}a44MbG9)XlpmOvN~sKHqVqy>J!X9b>NATj7v<Lo3}v968@%Zh7bCxpB!n_
z>0Vyi`mAD&ZUCForQ@Kfu>HnsA`Wh-Fg(0A`pU~qk6?cM*RpbgK=WQ(=Tg(=9-c-m
zwpUD-vc=ePRx)n>a_qi<qT24qT{$tG%>9->Tt6)5w0Qj3y{-7UMB14Ii(09wE*oD+
zZMmrKQ3HdaA!rJ2f&iPc@{yIrGjjRZx`H{ERhdjKT*@Exn#rg4h<Sg(Rb?KFQ#@=v
ztDZdfZ?mnIFw0M~C}>#3-QfvyQs^Vl9;BqC2al64TuAuvEv=w1F(61O;HbhKmld2&
zHU}o#dFakGFio=J^lj-W(@^{Q-~RBiUY_FTMsw_ZpUslp1a(v5BL}t#Z;wu#$YW*I
zvG+8O#Vm!2NlczkEz`C<oTTb6o|fO5z%03W;=u-G-<rRt3Ldk8hV`|>jclsj4u`2j
z6a1p!M-9x3jBJL6Y?YO4?(S^a*=&rAY;tmJa&mK0IS(qb*f0c(g|o9vw|t_?XX2h@
z%I>fH_+M??+Gt5WIq;z&Z=<#PvnPUPR6xn9{E-7&Qxi{44v&@=kCPKmRTa;*YdkeI
zJbU)=JbI+@cx}eUj+nFwvn)J98|qgyH+EF9thRV&aP0PWp6&UdQQHlb#-O7c*2nXA
zIoCk^Ds|#WBU?p<goTB~kt3kd@0J#cHEShHN+kaMNt`e#NkH6vOV&dM|3!|>KdNPZ
zF#Zeb`r*bV#az_#@cw?D+uMzE?wtXh3bHkuXMg?Vf=yhYqX8KH|Ce&<Waizl$*7>n
zsG!KHKO;3c`NE~74V#+g9aK^~rl7_qTabL<8jE5k3+sNV7+wV%*~}+P8f;d5{Aql6
zZ8T5$dn21#@KF<IXY<J01tf;@L6fr7o=#@%2M-e;JWG4<EKNW(b;5*%gruYg&(jTz
zLDBx;LBfx}(pAS8_4S#}%}cUX4!St=9EdMW*ygPG<oWpvvtR75OSPz*^YBMuo4CHj
zJp1$?UpgP2mv6hj|D@#B#jx{{k~*0s9r&Dne(!IbI<;~B{Ko0xRa%Sp*fg)vX>MRB
zd9IdH-LXKSd4U46y}i;!2Zo79IoCz}`7K^0bm9O13rcT)7Po1KORS4&T-N#ey5u}t
zP$l*7>+6H_L<}GSe+9Z6OMaV>$g4;H>*KeJiLkS8Y~H@Dx_`~hzLctRd$Z&1ea!D~
zmhVw_3sOq|c<r=vn%1$X<TgpgjMF|U&zOET8j3_$#NOGyt7>lY?H6-DUs;~f8>Q(Q
zo_OxM#De%2;GUz@yiR6e10$!)jLKi4QrN`A*v!l{n3b3;C;t6i{-QJBMEdJP`Fno<
zk=zyd{<?mfef<L0>(lj}tpy)Ivz3%)XEV=%j*Y8&8yTNQEKB5v6e$D`lVWDr?eejJ
SmEWitq`}kG&t;ucLK6V<ok-6B

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
deleted file mode 100644
index 2c346e0ce5487e21d41aa4e6306fd83a7b4ffdb4..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 17500
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjQTB9k45^rtlpw*nn1P*v
zfq~)we<{|*4C<pyI)}rN{)4;+9C!{X>`eLJDEwk^kok&p`cK54y=As{TBNmmvDU87
zGV_FIPCThq=B}osk$BzAgMm$_^4RWsS9{X8yq2sDI<$M~?TPiDz;SqG(nhWp0|D0Z
z-il)aEgox*{Qtj6M0$f@(~8~Itw$TTDtxavdgbt+a+zZf3`{=1S$4ATvHCH!7OR$B
z@3r6eexF)=qclTzi>{JZ!wM~i7(s?RKgzF4A7)jG>Uws@#p#G<p_jVg^E0Q5)@e^y
zx~=wG@A$$q>t|M`#DguDV(M&eZjN+madv13_UMR?nd2Tf(K~FKbJVO*pPtBo2@_|}
zZft&RudU5&{JH*5;J^R&yA+g_^`_2<`j;5Y>6`Wc|9|IeKc1ieZ=V0}-rnEezrEdm
zef|HP#n0bgUS4ls`|J1L-}~p=|NsBb*5IHA8#JkwbT-eO8|l#C95Tf{Xi~hV=ZO;@
z4zY86otir5|F^fcW){9|pEL1PgVa-=86SI>u|>-4oN!<%_cAui&OgqVIT(3({s#!(
zv}bGk|NsC0-~a#rU+(|^-Mzo%&(6%hy!`*Y9UtGny<LBQ|Nq|~n;)K+-_>qZ2(|df
zk%K2r95{ViP3h3;1qvw%3h&<NtXp+TN#n$UQ(T@b6_2EY=P+98DQ0ryact9K-o4S{
z%3<FB?Ppii?{NM1w|SfIKgG=d-~a#r|NQ)adFyZG_xJ4&U-zf__qYGc{dw%`cpO9?
zfO0G-XKpxh@Z?D)y>rW#sbytrzka2)Y{7v;Cp$IXDt`R+-`=Lyz`)>7dP<Af;~5`&
zcZ(dH&op7_h7cJEUH*zk4#AysVkQK0Hu3rkZ)kcVzv)-~|9`bVKD>W)^ndiWKi0Lg
z9^SQX`~F_C?(d@fDG4CIG5r6(sQl4`=g$KI1PslmfBibu)bQccXKqIX(oz|{RW&9a
zIdP&vM)E{dfw~minnx!z`E{)%CuGT7dG%P{Z1F$E%*Jr@{{g2ZrX)2p?!EMr{qm3Z
z|NsB@pKl~%b7Die{^6&mdDh1p*Zg*S9?=B#;i6!$ePLq71p!?A;US@^>4}LAuEte8
zNzY2x1omllERtKf=iodJ18-KFjtNFnZm}4fE-;JwuXx?`geQw;>YwJRCOrTD8P@zb
zQ1DUd@V~!2%l(aG_W8Wu!~#y6i*g@5c#tq{vT$K(C^u)|)$7mGQd1pH{rzuv^Twlj
zjg1fGwZ!FAC;WOUzi<;{B=<U%ul%zWq8>SzGitt07Ig7r@pv0qQS`GtrbuCB%W{4;
zHc)z5_~Nv9+w1F+d~zu_c1?YF&(_M+7Zia1|1a`<l#r4lApH2^#fklW*NO`#3J5gx
z7%GU17O*L;3fTF7!!qdz-R67gi775Bf@k*dZJzsN6Qg_eWJX_s#im`;q$XqwgA(MT
zY1$m7A3ia!+|yXW*;@EP;qarQJj?wq?ui8j-2eZJiXJ5-CQh38aQnuKr_Py$g+5J4
zPj`50WMGhBqVD-fi8Cp)?7$?BD9MJ+XAb!EoyaoC=DNeCB*fx&m{oUQ;3*ys>9}17
zR?9_PJanL8mfeJ#=J`fC5eGIDs~=t+&SN5F%?Js-s7DFO$zef4Wo6g=y)RFnoSdAR
zns8vQ+5=Z6W#vUr`vsr=mz11ny;P&;|Ns9F_4zj_O%i#$GT;Ks#8sRrkw-Y#-);HC
zzw%yVxW<X2oXpJ3UYi`A{rh{m;IrD{W4%1DuNm4@IINxF0t!4(>T`OOkeWJas_2(*
zW?o)KVF3xLsi`eTZJAYM7fk#Ag4gYi>rwljIcFPYHUyL|oGx=>nc#e#2d_GFdW{y8
zId4#vI#Z{!=iyKOP#N8+4$P9_KMx)_@Zr1u;lu4budf^D+&WV5n2lL<c{Rwo|Nj@6
z6moDE|NikpRsD8WcJb!T#!Bj}ZYyq{n0UNln$!g)e&zJV%lUI6<yX7uaI(5f9a=9K
z`P*J5m2;k%W>mIF+}8!_Ok0{*R{lTKFvo5~m`TCM>+2=ed=hSK>wWmt`&G{_NK`4=
z7k>D(DKbh+Mcvxk!sF8i17oF`Ew`Se9PnU&ku2&_c<jJ|5Bw|F{bbkHl9(XGSY#}a
zXEBkLSuS;ktQgZ)PNC_Ha>rL_vhA$?Bp=CJv?(Fs2Y=?$q%WC@yb*6Qne><v|NLeb
zwXd3zcuJ)0@o~w0_3GAn3n5-qGA``s_#h&_yS?pIXZL1fV`pB+)xHlBdshn`T;Vuj
z@@e}J%_$uR!y*NQUmsD9iQ2G?$3*Yimqyi{3aTCdCDsZ&Imd1COOiuX#NyYVnTZFP
z+EPzT$k`+;NJ*0bClDn|kmp$0Ua6{DN=bBV+Gu!i605R~?1H5{U%oVkZAdOKynMu-
z?G5W;dzruv`+pw30!Km|B7`-TDR^-<P7L_B@S5g{Qr>TTy8l8xeb+y{)SJgR-6-e2
z-#-Rzhy}WZ9$s7_VO4A_FS<K6D64X_EtztVp}VN!=_h`*4&(cWmjq}DK51!EHtOgU
zdNi47QPY9d65`V7O$QV%&QhN7_QL)X$!}-8WSGbFVY>W;g`1zAmQ?dis`xo+&Jjo&
zS28Qyu*s;h;>EM4J0qh`85tg&e6&8|k;A&+3Dp}uXt?`{{WqA!myybrrgVTOHRfXT
zwUPtzjuQgc2?TFmV7NhT;{O#ZpU5*;{T7MaxnMb`4_n%)hT{G8={L4^FWfov;X~(1
z!k~uM|Nm>kt@#=hcxp}cUZmaF(KUPV@`A@Y_j<e|+m}bQItt%+n6)ol;+JRU{f@SU
z-Gx!p3|r=XRhg82>}%oH>NZW*gT*-$ST`8XIsEyKGiyfIn^TpK;x6>QtO%MJsvKsM
zsK175RaN$%raOMq+qY?jgg!Z8$g8!r;=WZe$3vNClJ}BhCtlhvn!*lh@hQa=X=ry(
zomSP_B2ipuXl!6$WMt%&G-cvMM#~oyqojF0%xSq5*1#YA_wb9VKh9^r>^POUMpC4H
zYX546154YMxUm`L#Kax`qP*u|W7uf{Ju{AL`v3nM+ti&2h(2BL!{G4q^GDS9ph<9o
zMNf~2w)WpUcT6H8AD(Oy42v=_Qc$hjax-KJ8@o-U!gaL*vui=({UY;<q>ZyU)dZ6+
z3AA=T*5{M{pZG(NTRMBIQ;n9R^L%?_5iz4T_x*EzK}<drxpu=Qqno$0r%(ME5vgfl
zXz0XXQ|NG+St->a<I7DhH@6B_#>D^02?<Flf7n;^NlesYTA|}+`cb|)#o>|B0iTA=
zk^x(d7jzjU1h;V(T$NpNP4oZ9BF^mgDHlFyw6_>X9c^6q+`dSaHI!e{?CHbA6>C!y
z($a41p8oK%`k`yg;PiQDo<&7PL`>{gZtmKi-la2VEXZt{IFVCG_{Jsns?0{4@N*|P
zUVc=%#Lj#;e#sO@e}NA#(p|cWnm#i6i_I`(bf2LVAuPO1%!j#?kwx|o|KY?RUzC<}
zvh{>~dM|G!E^J_E_~-ZW14oW1=${XWI|nMv&duequNV3NY6kuP-!jvdtHFS$HJ+)p
zse3Yef!O+sCwEt{=|3-&>GtSt4hiBd&Gq)W|9jTIBMpmlYdId<_#eJtD11PYJ1nK3
zbNRa7iAL5d-FwgR&SG}%tNHm$vdlZ&S9%%a(U7ne%TKGkkzRQ2>}G!jrQdp*o|hk3
zs2rQ@v8TQGe)F0bmAx|^!a8?0JbUJ_d+}|?FSgTIZ?~2H{h7KS94=1z#T*<36%{Y0
zPGw!SlGD)G_;4_rN?D<Z+=rCL6_X}L&E26DzT-svjw|=~*O$NFSN-P3V(sua+lmj3
z)2I9T`-cQgijJIl{<L@F+_@8H&TO1Hld=4@=;l-E`z)KhTYj{0>mA!4zaP|~`F?J0
zytG-({oUW6?=JtpulD!%hllG=Wbp_JvibW@yAY5jF7C$1_e9|Eyxa-FY$;b{^!TP9
z)#7vi|Ig5->WF~!>4L{>`=Diklc$W0%^E+ysL)Wys;WsnJw7}<J|3A&lUxK;mK^T1
z`pMT@U6DLF;8a7`|9^j<A86bytS)C=_vhp9@Ao&BUH$R%=GLvPXU?jgI>lpQIrHa_
z2*ap<|2GyID$CWkvt|b!xZI#R$7NmI-<_G4uUCHFXH)Uv`F;C2o0pwA!y_ftW1P0(
z#tnt+<9r{??F*N3rhNIV4r<Ww^753wf8-+rN%c-Dd|_c~-rifMO$$g)j(l>9Wt!20
zn6!wkOqw<mj<5N*urSb=aoUuR@9*zt=f7uS{yuv1vs2UcUDjp^iKUenPd<D8dT{7d
zVWFg>(~cT0miKLG+;NC)x|q#L?xh~PtG{3W_}E@b>Yjbom-Z81Efy`}k(BgFP35t%
zu@DmEdEn2!<H$tr{`~tAY3EWRHZ(l^S^Opp?k-*3ne*mpL`SEJiMjO{F>8xon(E3T
zsT29f_0d1)SEmy-jq~o;#m4MOJw5I9j$8L47>$jU&CR`oCb<=D+SSFy&(5y=?n=XI
zl><E?Y)V|rO{{FpEg!$X-+y{~{h8U{Gq0^VzrXIU`pNP+b9f>nd3bqwY;0IU7OAp&
z%xszIwIR$<;NboJl19doW$%yKwQ_@#Vbi%enp#~~uKkLR*50(iz`$Yl@tzA8dE8vA
zIh3#cKNQ8I5qR!I$4(WAb8Zda-|yFtC`h}$E?T8kmYMnR{r&&#Yk%!cICyr(jK9&_
z?S6e`ue4-kp5&UbZShg1S-<L6bh24G8YIm)+>j+Whuc^3{QUnb*Vk{a`g(o7y{ES%
z4-e1%{l;rzPj4txewb=KVMXI{K4D=e7T+$xy}6vCDQRgd)}&0Bcrecn?9`;BgoFuG
zMOUo&P*{4kVJ%;KwWzW6A{B18WA+|7Tf`37_dG~*;QMG_|Ie=GN8YC=$@cY=ELZ+{
zczAuc_<g@Q6{UyU+WqI(9gg*T;KZ{c%AmG)<~)fBE*JK6dK84O{};38$K`pp_v7~{
zT+Zf~GswB+awUf&O-TWiH9NYEj~qAd=uS2SC9?njlZ1pHdwCa^6<>b(sOkKE;{|Ce
z|MR?<%XenVM{(1s3EP}6dMAj99pjZ=S5P2ey0e*mj?K@iUteDDF0*aqOz>riILfIe
zo6<Xjb1vibhsXQ=-q`l`_V#?0X%Z_EKBuv<vCUDAJ;9Kz=$O)#n2_+`Vd94`DF#M|
zf_6btQqrgI=HI_u5f@HQc93OipTZ(x$m_}eKVj?1c#|V1e(-Ny=crvOaOe8^{TntJ
z?XCMeb&^0zLc)))ukW{VzYkkqSMmOydXA`Mlx*W_&ZgA|ezc$U+0xRKcW$P!ecrt#
zC8rx27!J4bMkw)!JFK%dko-}4@{|z^i-Csb!qt$98svbG@aO3*>;ASZV_VW{b|R3w
z|F^wCi-wtm<in@O?XO&E4twHobI~H1@9+0Nex9ywx2K?yU%u|`{r^eF`<6!pEB3iG
zNPf)V<vSy$9VOa4EnorHJiEWO|Nflkm*?0j&Qaaf*w}dE>Y<cy?GINTyf~qt-X0eK
zX^tdq+Bh*a?ce!^h_)~D8yPn?GOgU=pr<wAG>?mxqwD_wVXi2xpZ}kqx7XF(=s%x>
z_n{EuzrV=|X=ye!Kd#QVuQz-&i7`K(*;sRdD)*mOmT5&#UtYF9Kkx6-8qOcz-}5Y5
zr1B=`aQw~$Q5Qd@J$raouH@n3(ny0eRg;oZ|MIVP|F@VoPVuYPg-1X5b#^BBiq&si
zs<<WS$@~5F^*%l-M~>{!<aK2OfwZ(^(cA6jTYO}5&-}>W?3gmK!Iby!;e-FqANcV8
z>}>m*njRmYDZe}-W;jO7XzcE0(cNIlt|4w5`Cw9NLPA1Xnt@T0fqD9aCoU2kkUZdY
zSE^;2TEHWTc?u8C`d5TD1#VfODy!po^4PHh$B(~ywaUfsL}J7F`TtLxD%xB7oB6QP
znxAr;-40x7-TL8xPO`*>NqowW>qSMyPHkWC@`b^NPm`9hW*%YO`@3tuC{v)0%8>_8
zjwmQL3P2;;>D<{9);berOJz*_``?~ReCO(>iL3r4N;GM>Ub)&TEBn;N)$^q6j|26t
zuIuFO|0Rg9ReZAgC&dyTbU?&?hK~C2C2P+3>`c$em7Frw^IGd1JBbh1Oe3P@!oql*
zoH#B+)3cNB*9j_3L3Igb{t}Y^ldk9#eB$30(xmyLzwx2Gy87a&Q%hW0)L0WaSe=}5
z)~xCI^GD~+n;waaZF~P5fADB!+k;K@6K_Oht^WW2e`cnkx3_U;hrr#l2iSJ(l>GBo
zQbkphUqXi={326kWlNh;M~4F|G$kcD=v@BAzB(*m7T^E>&dK5{4;-*M)OhWm!!xDn
z@*1r>XGzsZ&g@*a%;00K*_sE(`|TSWC)wEeY)HEt-&4}CT<d4IV$<@9112mB<u2tF
z7bb4rp00dbzov#~<w_nlwv;UOP1Q|z3>&$N3Jo>1m4#FxQJCZqwdF*EQ44Et_!T9V
zs0j;H6&49b{rxn(^X2{?p7Q$rdzY%Z-c-@@3Uh9DkC?$4z3g~Y%Y&@sB;O-DPANW_
zCBE?3jTtA8wuX9lpE-5(bhn~vYh&xCi;jP0@@B^}GnSR{bah>5Gk{o@nDB!=*8bmo
z#(fK9@;LHN|1(~|H|rql#2qJ84bt8-Omx3{H?_4TA}g!!{q65h&d>kPqrzluov5#0
zsH|*vWljIo#ux*Ggk)vy<c6c>Vf+6aPW)8Q)>hxyxyFCqpPwHd-rt<AQqewhreoyH
z#9t+BjEq5t8Dt+wMY-x(D&*|&;`n~soQ<8mjhS6SLITtyV)*|*$!V3Z!Drs6WZ~ZK
zoBe;=PyJ|LyXC-^r}oT;<AsGMzP$XltLw|m86GPVA|fQ-zG>m+n)3JW(S&5<dc!mB
zryp^68?F-CBK1UnmCm(A9Mz16kA2&}u+dhnt@p<<c4=v6501mLOhY^*D_)5_IlBDA
zI{l*F!$*%USiq2522E~%{_SNn`^gu0sw0PSPVbQ){L@=ymoaSRc2hDgm@0Y0_{)6z
z`to-*duo2p{rXzI?zdaQ;vF3vv**uu@$TN$!Xo`HIl|zx`8pxiU}ffuS{pZjhR0rC
z|IaI3_x;<yeFcpNZf<F6;?dRRadgzM_4Hymc+{@qTBS|pp$oT8ZAj&Yw2PDe{M*~t
z*vL4~PD1kGoBtb@F!}NdG-x&LnIJLK#chVtk40RYs(;+tTFxzg@8;g$H?OXi|Nl4n
z$9MifFC!zPj?RtN*4@jNpT2#ALs6Gs!ASN&S9sH=3X#Qz6DFSkjRl^a_c#6Ir26o6
z3v!}UQ&JL=lLbUmJv<XVJQ5~MU+%YQflc9`|K|e&4{j*aMYxDf?DB<B?hC#g9ZMQa
zeH~afP7|B4k}+5&+hK9~wfFb`+f;x1dUyBy^mBV^|NYs$!(#U`wf<>h{SySVwY8a_
z&GVAE*twkNfc#$<%Sk&nJ&|WV{Qvv=|8MV~FZgM6c(FUroH;z-zVUQ*fzY8tSA4`y
zykxG^WBzwFDdSY#pM)FR`xh>0fo9G>|Mqfe{+c-XXuZbSAM(BerQXUm{>i?QJRXcK
zC)?P~4@x+7r~L_RTE5aFrDxLI9)rosMx7mIT)W>LVfd5E{4b>Fm58$=Bh+t=KUr84
z#3Xn+Oc=LHxh+!Vdt%b4_CsFt!Gy!ymexiI*@{L627ivKwa(nbVa8g(X(X_G?hKKh
zmEyK7FDDp1V9*sz*ebiPZ-U^qmyONMj&5yGS2ng<%P4HCKH&3{e>>Z)=YP*L%wFmo
z)-cV;fYp$BO5+rrZfzzF9h(p83R_%!YO<QE1-$PqR(J5@I?>9qOzO#%#mkN@Sa1NE
zFde<vS6Bt?(0Iu(&E=nRWRJ`&9uJkUBMocg9a4@R6@C4~l|xO0?TS!GV?fkF<<>9-
zUaN*_A_l4d|0XxQ;fGXFP19xmC*GbcA-N!xd6R+>Gt&fVnHeXW8X0Y0Y+S~}w#TvG
z{vGEjp-A@!UMw@h3<T0yH@$CDNk~cg@wxrrrAZKPHT6tzV!6o0%%-BJB$&`)u)xiR
z+3i76*8<PPYkw;iebzUN-66#IF_)DsXH$VQ$1bk3hR&i2+bwtOl!ST(RPK1PoY>eG
z6YmfuArbTBQtOW?2ODN>oN$@t<6PDzQ;us(oNHz`&v?=Pn{~6RQ43?y=EtwYL7g^+
z|Nos#JOYIonVF}})0)HR>2N~uOoPiKPs#tC4o6&sil!tSvr5ZS-Y4ypW#p8`@8(eO
z*GhTY;wA|Ri78W|4sd#R>R<b6wJB;Bf|!|uUL^SP&Umy#t3v5<Go$+^h1tF-n+_~Z
z`@rOOMfc|nhTKBM7#Gh@h~JYAevtc8;_UWfUcJjt#_T4A=R#*%KFP0F*u-&7PibQ1
zlo=_F_RBoihJ8?(zL2qCs(?U)=GS+!&~%oV@T1$LmwW$LrX(2^Uxn#nCno1+EOf3p
zbs(oqU~;fz!0rASf89)Cc`Q;+Y8(7ZHt}OG-Q*zsLn(;c=*Q2=Feh~Qt?=rZr!~df
zE4Z1_UE}d|r5RoZt{pwqMyJ?fI8#qB{PAS5d&ZdP(B`F>%42R0vE3;|J41QihR5=P
ziH*C=XEC~)7(Yl5o^RygrL^Rlsf$+HfwhGmOfjA;O6N{=T>8|IvrAAU1rlLSdBqDR
zXI#!P5Ylb>=ppDM%yUDDVYPys;G|n(N0tb+Dc;?dkZ|Dow<A+s3>dYwq4v#-{@H)!
zNlTZ4yM0tPbEx4#5r-ays2sxu*Eo12xC|aoO2|CepzQh3U(%U>!j*3>!Y$$|5c`_S
zQ~xji>Z{O{@sfERPqd8j0;7M48V9D96dYBWcysQXhdO^aneH8yuVVQBhSP;BSj_hc
zB6X<n{W!w4Eh7DZVxQ?F$qk{QGddWR)jqh0u8a8JzxalN8t+3-?Hz|pPcWVF<^1v6
zXn(yi#8xM5%YO^cE&tQ*nfP*FQinjKU&1l2kPr61YB&5iZZF}H*=zK`*SUG!mIW!M
z0(yIAyte!xE_db(4@7rTWBjhpI}Hu8C;1d7a#zNba_Q+YZ{u(X{Qv&`f9vwPnjatB
zotj+z`dB@=HXn4{kmC^6W^<x#`N^g?4-PWr-A&q2+<x%F1V^W)#va?7JHGu$)_Z(l
zvYmw`G|eVC-MYx)Sby<ce8c3Pp8pBYiw`WlP`+Vd){(92Ib9E4U48%4XZhtzY^<uk
zar<y>TFG~oG2+XjodVmUAIV?RTWo)0%YlmtKA>*Z@jgQrcjGg&Tqi0XDfclbmohLi
z+OPrCBmmXFhorWy=w#S*`n`PCg2hYaVix)<bpHAO|NX&c`SkO3zrVd*zf#kBl}^~=
z3qi~(O@^1x+V?zZz3Ta&E%8)BLc)ekNk4u~O}wtxr6nHIyrg2<&jpc?)}qrE%L}iV
zdc6Nkh|a2r68gb<@#6RQ|L^Vn{rB73cwSyJMMbwOQj9k>)|_>lBqcT@jW6!Np+gIn
z9Qp8C{P5r3JYFI)L5@#2tY%4Bh=Ut1PI{LA4xi#Y<9xqB^ku|nrFllodpoNB|NDMw
zs=S1R1Q!=eWQ0MnW!l+K;$c;Pj~4J8RAPU3_8^~&tK^)ylJjhnEvlz+ElX4rRC*o*
zjX@_aql(rhVa3=Pzu*6_Ke4ac<|Ie9iSnXX*VlvQm~QO*yLzc=dnc!~^zC|?uOc>%
z2af)KU?%x`-5JHnItmT_{Ra;<Jv=YpHd(!M*Ka01582xX_+dudO7p2Me9%?#<oA|B
z5l)M4t{qMP|NpnE{k3}6FHv^(<V3~&OH`Em=5E>Ha^Vu=GW`i#d1j_Cvd^2xla<Az
z9cExt<59Ha5#tf*qbE*4%xsdo$7CJBILl~)SI7Ppjy;n$CqCcxpi^kh(i6-5>(kEt
zDR_71{DTBNc6PI{us<ugHs?8M2O6C2SvX5a!NH|PQLF3W-|Dt+Wo`5AJv&00=8CN0
z4P684L^|;%xs*6r>3q2Caq-)R$Lni9h%sD9O1``6eDSlIkbs1hYoE58x^hl4uUK>7
zign0jRu%&zqn1{~7q7ux_G0zGC=cBPj#ZGjZ<_z6d22+QxXOu5KTa|C`1DMB`QOOk
z&)@(5?>{>mUsYvSP-NscM`OXNQ!LguAKI_ttnlJqQ+qvelFkDMmzIUQW<9*F-?q)H
zO<cd@lG7_G`9E6FqOeJw`-yVe!|hgw^3JH4nm_h2`Tzg_-F<(v9S`pIpa0L)?9<O*
zn~V(uGoBgvh!s3|dp02QWWh5Y=R!kfPjwNY`7Q?6Cme#9b*QT2TK$z@toLMovU|Jz
z4{$X&Ep(=F(|#VF|Ns8}E`M|5`=m+V`Q`ouhNS%a_V)h!`xR0PD}Kq&=q%l#%FXfP
zYj@)Ry0-fN5@G97GIEcWf3*0wjP2h+lNe1-aB^)bN&Ua*s?Lenu$~icE<8`R<;rYq
zjeGY0=jZ&RUHTIz=C6<c|M$QB;jP*AGiLmY-}#9<e_})F3nt%!3N>yHi~4yFujzuO
z&+7k!29Iv+p1z{mYy#uIrO#(*O@w;j<}(MYB!Q<DH9s_u|NsBr-HJ^<Qa|eZLwV-I
z`_Iq+zb@|Y)<usVKR#ZsD7L@;fBvs8>&^53)&2g)ALOL6@82)$J(F3PnGfII&ZDZ@
zA-pbrPXY&DTEsq?=WF|p^iO}R<uPIPjQ{`PrIEIQ!p81GJ+3s#zA(v$fuG;s-d?|U
z?fj&qdk2Hrn3<W4jSqK=|6dpTyZHaV`Rn63zt#O;w5&66o{ofs#J_*7aSK@dWp?f2
z*<Ww`<-2jrj)u?I3fFA(D|pMV7Yj+>PBIehB?fkTCOAx(5z1Ml&0_wm{^O(dD^_g3
zdw2f-zsU}Ex4wVRZ<DwGw?6Xo=VQJ0@%#V%{wZ9!N7VnPp;8bVGxOnIX`bk926uL~
zu5hoZk(f1G(yk_fgEKK=uav7{)MBCBv+{bWERfJ^n&-?V`O(F0tytK?1^zLKTignl
z_1M?{lePY~c+H!~({%0Myp244YDe84htBKz@e&r6lKpaji{IaqzoN!BZ;Iicga7~Y
z%(pX)*?sK8Bi2W8g`%Qtb#-kY3fkV}fa2{}Rhwn8MBM&k=^T26YubwwSRuYlYFz)c
zajoh9_9Jd3aS7h;{}nqI=<+*z7#jTf|Mm6%UtjNMXWRe%`Fa1=skd*;NIgAm{+l;E
zYu0q=Nh&KJ-dS8<St<GL?eFtv&ZMLrYsum`+@YnVz#vqqA7>D==S0CT6Q`K4X^yU~
zj-gW(jVCvDb}n4hl=v!y&CINAfdX4z-WOw)i##rD8Cn;w{a#QA4GRAG&R(of_}<!A
zEM&^ARyz0g_y7O$$CmIc>iPNOV}9Vp_Li3KLP9>l%RacdY$z@^HZV3mal$~c{M!Hj
z=?Q6RJ4#-@Kid6&p55Q__xI-K-*?N%WMrS<Dtdjr@t3cP%kJ#h2`YXsTu)C(NO<rd
z;l#<rh&?kN9_Md6<kYs$uFbxFgAVu6%Rd}~J3*xfs7`Xqs#6M=`7bFc7_OIOVmD#y
zg2lGgbvJJAjokNV_h#ehaPjTa#k+fW4sXx@|Kq1*pZx#kvW(aF_w&fe@SLCj-@5+a
z``6e1@2mg+{j++}E1Av*msHzitpna&{_(k8QLW9<h2_@2gtWAR($o`YQw2m*GqRF8
zx|4TQ2;LRga#F;n;ilJ7AAd-EIawe4VPAB!VG7@cs?8dv95H|X{@yRFUgzoe@57gu
z_m`;j-P~NMqccNN;-7uppXvSY|1Wm0zk4U5?w?Dcrn2(k&(C@M=l|PZ`}_Yy<^OT}
z{(OIYygp)c`H{x@{}S`;({~iLK0L_W7Vk0rh^MFFi4z8Yel~a5*VVOkaI{?sVN+CW
zD}60->9XWBy`!6^a5EW5uX5QI1c|n!#!zh@9*ypO3wk-PtggCn^TdlsN8>lA)kg3B
z#?E~BzvtwNMT>4IsMY`c<R3Th;h&%TSFQ5-{r!KE^Il(uN1OQNCAQ`M)>k@I@%h<*
z`+A8dPbGKk)U=hXJJI<6zhvFtq>67n56{WARexVlzQ?hlhv)G5`8;#xXas7g90`ay
zqo8ysAoOU#Un}Q){ljyuc`jb;n40_{LOm_kFbtm5x2i}OB+E`q`j@zbhwX{ji|zUU
z`DA`rrk~q>cX$2G+nY<ui?g$goR|Fj`}_Ravjy+(#pmU<DZBsc=r+E$yZvaayuFc!
zr{SF)ALqBWI=Zwxylc%?R@Sz^Uh>Au0_z787<+nn?(Z|)Q+aqpt#wI=2-A)o694`r
zUbve2;8@w?$QqmGIWo@4%7;6JFKn4Q=}<xni|K?JpTzxzMZraM(xEk`6D~2<BrvZ%
z<G|x)Tm3ElXxIC*XXDLse?`We*;u5yewmu^9>dcoe!PEsd;gj>|Mg;iDC?g6@aLyJ
zBcl#zPXF+1a~^ShBb(Y&1s@a+UtJAavGM1J<GDvm7joOxrBr-qNc>&a_Wk|E?aC`w
z8y6HjkdfHBEp3LyL`8*$hu?DB_Et-(`8e!cD`9!fuz!lnYTpZHcOk9mGjnwLnjJ)1
z>^%f$^Ex^>Sum`7{y?naT&rWE$A>wLosMgnd=+(fi!R~a61w)~oChm^o;-c#<>a6Z
z-b&S}iT)C=vv$3U5}9r@f9VpPyDEh(4<t*>gE*Ojudkl+Wv^IM;)8Ey75jg#IlEM!
zJF(;1obx`yfi2f=eDk-RZ{Kcs#`v+v?L{J9vT_<u4|cJ0e_}eW*mu&XX8x*0nJcs|
ziJbraOh_(!^0cF}p!v}|bFI0W0|Z>pPu38W7Cd<DnCOk0H+NfEiznpjN9<~24|{hY
zNbZMYe3yiQ{iWseiu8*4izT0Z@|iwo=9G4?dwdrIvZfd5a)pKlPm8R4*RpVfPHkMH
zPIsA%?$d4WcfB`HkO-JRkB9r9jd9-^?S1~2C#XoU1+qM-%sBh?*Zf~Qz9-KL=vucp
zH-2&DWf#q+2dhtVHRj5PJ=%4}G}TjP@-p|Y>cw-^Pt6R;1IPNExrSWL4k9kc@2=>4
zac!HKlH7sdn-x6yj}+hN94U)d+}pZlkyzKeWgZ>|8GI8g4;$G0`!MtLKhDDPmsY2H
zwoOg7T>kRNPpRYERd()L=l1fPq?hL50N>5Mi}&2$zRx{&btaP{``$;Fmj^j{74V&B
zPL^QfV7%@s(IjBNcKVRbVUzgCPx6;cN@QCc0vD`ucv+smN#{dtKv(Hs)|Y+F8cpjC
zZ=3l1gb~ZZ^tHV1HC1kF@@>J+zB1F0ugO8cb-tob22bnOEAtw!SyyO8^*xN_U}eAH
zlBD6I&~PB{k)D{;;zb*`&CPmt{><qo=X+&rVi!E)Hs)FIb;9%MZ^PUI)maiPZs#0H
zGW=-~<eKQGJ~{PSh0M3#3o`|p_WfTqF({+gaL)N7hN+&@1X*PG)`Z<KJFR+Iv-W&V
z-xP~;v0kTlP5%`ivr{$Zx(To6p*;87L81C<H|5sIYX-jhZns0vvElK9$`aYkQ&oTe
zWyt<sn7sP2;7%q`DmgRPmb<Aypnd+8gW@eEg}I{Bm#F^As_jbK-z0xDkp2DvO^xP`
z1rCSmo4drW%G@ctQ?&ni-Qk0Uk8@wSJeb&XFg!}YS9#T|jkoVMJ=hkfeS7V`n<ti?
zy!~iT_vxj!^KTzsc(!=s{q9X7t4_WAa??`c%;MXfOEtS$16y7TwN=O!p2>2Z^uXe8
z{^_Ta7A4pn-?lQFhxzi8D%Z;sp30fJO746)<%fKg72~_|G~-i~9#6Dbrjapg`ah4L
z{?^1lP03b&_w}!voEmYh`c?EvbM8v8kG9O&%hePha=czFD%@xmQ(<->M~y`MQuP;e
zXEkSJW(7paHZ2I~RNAHRB=>_*t!MD<hXodgAD;hd_x$7UKYQa$d1}k!jhBWL#`rCt
zesocfQnry`;*D1&KN?(@PT3WA^<%_bztBl57Oi*`antO!?y{Q}*>&5(d6g9-^gh<6
z88<bYWjnw4cB;XF(wXNadrjsYOXg!g6*T?FZ`-|cjt+qnQYF|9E3WIF@?__^pT5Gk
zj}_LLg#3Hbo3%CQQr_mc&9^s3O<Z<A+)#)6xTBYB&}<Pq;eGuAck<?(|MFXZQvAkC
zPn_O?Q{I%hySokr2(;}#a&(4-$PsO&j@G5;r@DWPWYLz^mRD$9+S=PXK~2DPhD86f
zIl7Y;w53kZt!HC<_UXKVu|d46%%1ZH?JS%%4+X3XJ6R*g+E^lUxv=M?i~g)0{#zcw
zUn?~3PY=?%K6QF`QS8#!KkPEQg6idsCgy+se5lYOA#%-ipXG1LN_Jm)Ez^G{EypZm
zH4mGPSccSN!_%tQwGS({ISLf)znphDLxJVFu~We6i!Yx<t`O}%yQ#;-*GKsFvYSh;
zF1o(>{_~$IeV#%kvbXE@e)<=FVM>c}pUvC}lm5z`Pu_j>^Nzb6k2lq7iRyG~ICb1#
zW;g5GrAa?4Oq6(vAL(>X75Kq;W<owREH-u>au8_Se|pxF4k;DEB$wH%Yn1OhZ20D?
zQ!C&dot>?%E0iPTa#GbP;(J%DQ^4%oS!Jsl=coRwPn-60(x>Gw*S~U=d>f*)^wj6)
zX~FBR?=Q^J6}$g=r?J$>dF~EZw=zv#=6^o;_S!Q>TW_zld0wP=x}m=3#H$}BIbIJd
zZRFfuTl4MnpJy!bVAm4O$A;T{Cq4Mh!uZ|#y7u9P3)MWEQU%+-RFzm=F?BuoHtop5
z*HZ5;y-m6uA?9@9y)cW<@*{?mIRjhXOWR4bok`<Job@uw^Q9@X)DN|1WoI3Jzm`(e
z@Ltum>QT)yjT`SRWh~}e$k{(oVF^5vb+oAS@t&r;cD>g!q=RO$e|VJ7E&TP{(_K6N
zF@o}T&n!c(LjgQ(`!6j|&}CJB8DbS89?qVwuKz%HPmB6TrM$gvnj&8EyEHVWKKg9=
zC2+~othNb?x8*14D^3eaysxoRD^?_H|9)A%r&+3HyDz3G+pJ%26z4P{HOt^ZMM&kk
z#}V_-dt}~wwLj<8{|H`f-ZSQn0?!IQc^d0(`pNWU>$GX99!IyG^<6r%Fv@tgVA~J7
zb*Dd{OmPx1oF6<*!&N|l#p7hbi4w296MK)}dA~z1^yuTp#|1kdRL=2h+p$!|_oUII
z8Ly|R7@pmf>a-zd)7?#Rn<M^Iyw6^Fn|Ifz?@zD%;{|zg%8Y|tO%4KW_8MZW2ff8F
zPj+Nqf48db-cs`qAD#FG%g%T|S!p$8!YtO^vR$rTt<Ow&j5nPAbLP)y{>(^Kt}B1_
zCO?sE+kW-c#)yummu8%2Z#yV{_=16uQ|in$TB65`_J`kIYc=0;u7%vg`7<x3zE8_O
zwMbTXrRl{FrgN%O!sgy|ZY$f}Kkx0X8)iX8I|W#Lmd{WNdi2{z?TO{Iqd9KV6j|oE
z&oYt<3>9JRa~64MvF195<D`ITIc8T(+nFyk@f<KYS-dX%*yD50FGgsuwSQ|FG3)96
ziIB0CJ2UojH6@6&$=5I1>&cZV(2_5(ha)_Hq4W!@T}t}O8AX#FJRF4`+Yc=~cYT7(
z1efCXRo4!Gp7U8s>Gk%r0vDo+_j*m@*?o6u=DR|h`+3hD6QacCyB|BPmaFZ$@O03(
zYD>A3Mz^NzdR8Vae`Wv4VCTaP99|#S)J{EA`m$Pv{ruwB5jqiiE49M3&!y)`-+oqd
z$mC>kgNbW4U;9Mu$%Z_KCY){4IP`ty*}A6VODtw29sYii85ART=GbyII|#JyH%Vg_
z(h=WysXMZAR?LCj9;<AfSx(<*IikxFn0VD!+0b5%{X}`gO1aMo&ptk5d)#hx?4sG(
zDa+nIyzuV3%1OPxxdwMOq^TyfAN=+=rAN1G<$Lv|Igv$Hf!9MskDXb*w*Jx2;}f6f
zZ09)mvE#X-s+S;($H}KoB8n`LZSRZilvSgWd2&yG^}1fT`=P~^uStFRJM^3ue7&Um
zr}zF$J-ctmU$MlwmVuX&-I)z)d<eM8b0~6TYhL@-RQ{1i?x=8ul)r+QSafcB_QDmu
zVPYrqx^yy=6qcNcNtCO(SMx0N=5Js9(|cFtm9G74qj5SrFy_&WV4XFk=XU09PJD6e
znp}IXlzE@W$$gvG{rmg<{D}(rWP@Yn?Vo!DJWd)Yu+;ibTWIPf=vpG1*%hS1dD%ta
z!5sV2Ig=PO-k$yQ^0xhoujW2AT(dx(umAs_%!0UZ{-!1cv98|b8^0LD9mtN1z8M{P
z^JYZG=1n;<K}T|=1UcFGC#<Zq<T?D|fAW!=b$yRl>&S<GJ!7<9<ByGa;)(v|k79@X
zCif;sU!NVcE%S}d<?q(djMK_r)(0K0*?O#WqR`S0HoN*%9J6cY=Pu6eRBCHfa5yFX
zf!TsD^VF_QIueJko1aN#p0oMIUB8$`8=ULG#cxvM6;1oPKh~wM!fXFd|NQ)aMdik)
z&x#_Vjgxx{|2CT&MMVC$um0BG+M4+L+y3+O{@Pc5`hR*l&-eGnd#aCr_^MsfQ&X^v
zr)jo@2K$F?#ww5Q?@3HZYg;C&rq;GSUos*>!mcg_H2d+j`@kx0J-rvNSi{0<c=^~C
z-CmM$SaF6YS0rSebkfiN|L<?l|8Ji6=i;qjU*F!|uc(-F^QO$NUlz0HPv|?jIUO|A
zSO56f{@LdL<M;iMH@f)e5$oak_Df{d9IYE{nvb41ap3sz1ILbicrSnW{QL`g8n%on
zdmetz&p6_sC3xn=L6-S;h8h|M8#Wl!{5}R66McW5XU&?HwTJC5wQdzUxC*jhXV2`7
zT+IPIuJ4=G-xdqZiT!JE@cu{sKdSYP?Bb%bES#){8=Dv2*tgJWVL5+_MazWh<ms-b
zJ_U+Ak=gWS-t+AyrpLTn_nt_&_dD)ZnLOJ{txTyarpq{E*#+-9{9pe$QMpfX5sOOa
zth;AsaT%~Bc4#yeb1+443$9u1<lItu%wpePf0@1){kxi))*qHvWHR3NmRnN3`JtE0
zh5VN>Us*tTG$}LpuT|!ya8c1`M_l?kIKnhF=g3aJzyE*z&rk2)-~T^d|NrlAZ}0#6
z`+vS&>W;$ZhnLmc%=6h6oaXw@mZErfeLT;mO*~9YJofcQd#a8uSbu)OnllP2EV5Hl
z5*zt?xj76zd@|aw(J^^$W8=f;{%!L13l>kauQRCmdFaDC*_4~cC2Bq0+*Vt-?kBmJ
z^E*9DR@4-=^YY%7nNiBMjLAe3(#hWwV-4C5Qp+N~g~LtIW!vVj4ELOD+R8urY>PYM
z)zLbkL*>}bLsL#a+O**^-(_35J^aF7_O5%q;Ps(pl`5R~4jr;oQqnwcu%yst`&${d
zT}?-7kMH^yJtHRXS)l`mi{KWv=PR`CGd+~a?cUTPU~o)*Z)BT|d!Rz!LXjz--YqZQ
zAa<;xfH{FDDtVrw>avs9YkNQ5JGQCj`@y%<6`Qtt1RFfN`u=m}_iuLHH4%Hk8DY&V
zTabfW<tM3j+qd>EHEW8fZkcoZabz`bcS7RA)-tYxN*9xyx_EczJWQT7(^F7_<6-UL
zmWr)!qdCM}e|$=qp>V+FwTbKXm$mzU{fi3s=zP54{(1ZINO{4F`bz_*J&?KGw#+eB
zxi67t6HjRB-rPmsTnY~hF!VHDn|4m6bFZ0bY(SLA0~xPG8AF~4LYG@UyUa_DT$A`p
z|Cx=RlWR|;%J1g26YeO~IRAY-Bfezi>w92l@0qn1RA9FpKV%&sl2hBcSy-A|K1N5U
zhCy1|Rfl!!1mTZ$K@}{DYAc0q<gIGuFS{YRTH}vJ-QVunH_Kf)cKO{h`xMi+SkK$m
zSAtEad*|hPg~v0?Sr}Uy1QK|rG!;9yJwC8WpshnmIN6Kyy6MyY*PnhRZ+!ek{&?aU
zWyJ*|pL=$>U0(2hV{SzF7Tr}`UX20;|Fm8FL?oQ2Df&)S`LIX7_fNU}rRbMtaXY<0
zL;3&z@0n%K-IO44q&{oCl`2PLvjxXa(Q|oqKeqj1XzYH>&864I^=pfW=3nWiydC}p
zcRt#buh!W-<&meDu*dbz#~y(`x!T7chfFK7@(f*QT6)&vQtAJIw$!`-njH@qJox>l
z%z|&5@2pMtm79WG1U-wqGtO5VJ~8iM-`|}1Xx9$CJHHh!r)*X{-<f<aeao`<H}g21
z`PlWuYvk9)n=9Kpv7Ardd(0r9Ygy!(e|8VDXPxm)s(ZR=%Vfx6@GY|qxmp}}T<^c&
z5)*4u=s4O?!ML~nt*DGNx8p%S>E)7hb>}#BvnH1RXLWj@{_wxbdT!&qRiW9t-<L`B
zWp46%o14A6uP`NYM!L=U+{piDdlU{x)XcYW&t6h;rXzV(*RHf!PUo&ey`qX8i{`gJ
zo3voc63yE8XWnd5Ih`TZqv#pXmAiXyXMN{od)xCheu8yxo92I55+-kD>jYYk`~Ux*
z*_PbR0Rk=dDitM7WpZ=HxXk|jzjiKYgNR6U_wq-K`$Q^PngxP9mfcI;om!UmtuWMR
z$&}Mdtrl&L*S=mXb&ou?ZE5C=^snFc=l)la+;whW@~j_rmt7`JNR1HdbDk%B{M1{X
zrv7Qwh4)ORxz4)uox@S$Q6;GLw=3zk2<xo2M1wwao~c2RRe>#;t5U8{H@LwaF7@$o
zYPHz2`>CMeKTzx&a<v$Uu(HQ^2y(jyOMKy5X|sRn1Hm^sqPaPXSdu5V-erx{j{m>w
z=lt+9uiJT^H=Rmd`+irlO#im8mtF2ZmN_jX_K!J*<KRBw<5N^$cNt1FPtkCV4Hn6f
z_Nu*m+1P8>wS%IkiZ4ykNtRflwPZ==1R=#4&%L9+`TdC&yfjNiu}gp{JMfkN$H@)9
z<sWH*_8tEJ@8l^n!EoL2{{MzX|L*VqfB*LO|9vt)|GyMI{Pi`Dj}MQ$z0>lDm(z@X
z{8(`Mu8*%|Pme@JrKF3iq>ip63yVbE|1<%ywr`Fv_<#IdDE;ZmRmmAMCH3OdethY4
zjG5Ee+xzfM<cnoG9<}ae(JY(%*)N@V@GR}e@98(#*!3E&Z9Cv&HDe`c*W~~IM`qh{
zwIuMk%vbthx+tP%X=??mV1HirW7RDmnf@tcC~xH4wfk(7!ZVJ;2AusTQ~Pp-1z+yE
zTexCXt<2JOsj->SE;E+tzxe%P&N&aEXEPTmoaba(vs5Q^rA6J(%eN#%RZbVg{+efU
zb=M~Yfi@G~ef`hx-ZVOvp<flbVVXt|QyFv9-_&^f<1f>{LDpe5srBXG|7U0j8Zi8R
zef|IHZ-1))|NDQi`Tzevi4|WscHI%bv~{t+_2cXMY;kdHmX>Yj=S%A7NSrw%@#BZY
zks}go)+`8J-@xE2);gIhB4WY9l}lJ+Ltni)bKuOGf`?3p*T?hd=zxa%K74j&zZntP
z8KuD)&&w$}SyVK2!nCx6#6(a){S<zBfiI?ENyef3N|1rtBXc16Q2ud~(bU$p993+#
z4eHmmBPyFWZYm5An3-dk*sYs#Gs#@t<D<g-`SWcY_V6D&VbYpAIg#VF)?*DN9q!`?
zOII@eXOs+l8|@wW$Trve`o!m6Zqo!0RbTtJ|7+5gRRw&>FMj{f2wdJ!Y{L85XV$96
zmC18mZ>>qazcQOwL5<V(U}atL@2bn6)w8EVQc=>+KR@rUkN=;4@6Y#VXa65=|G%&9
z@BcHBpel6o`~Uv}6VG*eFhuemJ$Ft){rHEk+RYOMn`cTkFHmS+q0u~5lv!N-@csQE
zua{k7>gWJ9QWO*pB%~cH_^MUH^Y%sHi$zb{`6Z7Wk(g)Kx~!r8|AK$5DUm;}*?O$z
z+cZORlfS!1hoetVV}F0+RMDmWjobof4p)dQm?{sckk8D4R224ob6PEJK1exk*eYoK
z@vn9KfiM9U-M^0O0}NZ8f?7-(Csej1&i+|eo_Dw)<mx>Wk)!uY-6M^BuYcSpuzz;X
zWm&}zrD?4us;58H_9&j;dH?3N@2sN#l{NLZ%vV{X{kY?r40~y>;Thv~-BY&c&U3&2
z*dqL$_4H$2iMmnAx``cASQxzok6Z8^OL+Ua@L6-5@XP5p9)o+lXXY4kwL0*)$o~*b
zR^HUw+dB0H<F|k28>X}7+BDpXc=<2sa%YmVg*ykcq?++h%k9?BcfXsVX3ukrb=LFK
zA&;dwy02BQ>dIZ5yHPCpMDdzX-EW0yEJrr-t<98h-}|>~#j_XHT~2$A1h0uEn?&x=
zaS@Dc<4E<K^rCw4E;Zk+8$P=fu-tJmx7fMJ{LN2rU+Bv0)m#k@0<Qi7iG~s(26}dN
z4hBE}-&Ha*)a|g}pta)QL6xix6`xx=i)R$i-F0tP+S^@cCE6ShJP~ewn0iihYuYRi
zp;MDI4t>|;YI0aNVgHi_oJ%{B*DPH%BmG$57NP9Gu5GG3FRL>z>T*XV^F*rYvRqg7
z{b=(^M&RRvkFU#~Tza~becI{D1aLR!%<Q#X4F)2u_KrE+w*_RlvQKO8`)@qsn(O9J
zo^wT$S0o6YH<mfa|NLH&#J4G*HW{23+G#H#?SEqCy5!p#(ZA*`bJR>aVrjYjhqHMl
zck+>0$ttQ_ragIR8+vrtBGbz)-+Pp=EfTrrWzNa?d`0%p3hgfOO$V<g9kSJxTTu*d
zXk3|N$km`A(Q3cJp@1Vtcw$@F@m>Ep_cXU(5L?J5+q7cag7T*Nio@q0*RJPIJU>%n
z;-;y|`%B#Rt_fQC_{63J9<PVCm#@6I_TR5C#OYjA(K3;1qMFi&V=I>@E_i=&))a%b
zDFuDGhpSr><~2_1nm2iM`rdz%;rH*ngj;?<L4c)wLIaEE;#M|o?UT%Z{^#?s1aYox
z5xK|hGqv_$+39;a=GV27Uqso**05js8>#(QH{B^A){-wssPDt;`AxF(7!8BCqAp8)
z@7=0#bXy|N^;W|<>c7u@elVvbd~KxIp7dEPdw7p~U3b{A2W)FnN<u@~nwL8h*!fsk
z7Id$Ym6bFxkziqw`0`b9&05L2KM4uR&3<$F`9D16-jZDOn2lLo9W*;7Z)Z?YU|>^o
zLP7QDhc}YU($dWh3?+A8&$kDyM6j<jIC08Je4%3>s{-4CtI90XOCAV9dWuOcbEX<A
zXg1HD-8gyj!}IcN($Z|XxonJ#Y_YLy;o)uX?=MVlDq%RWZ&~SSwoQj09A<Bmx0h&X
zkysa#_~3DJ#s7H^zvsW0q*Pnm_WQf!k)x9T{-+Cww5*G5)oE1Qly`KF_Ce9V5lb)J
zfD{Q*8>Y!wTbE2ba-zf0zu(cJ!O^?>;9XfOC6}Pz-4YF@KRCku``s9VQjHA_Ykn}?
zmH+*iy?Kes;kUPW{`}#YZ*Sbu>3Dsi<0=V{9|zc9cCzg7;xpnBf|@<!n4GY1$&ujL
zxsJZPx7Kdo)WjxV8qD@@oz2r%o$Md(@JH^`Qag6w^l8xEBF&QrjvZUD?%aVBCloX|
zzS^byw_9@3b?O(dGfoyJkuVdF96ZgvHpbL+VRq%2vyz~hssHH$0t;s8p1A0fve9vg
z&P|3%b_%&1Y;0_Hb{V`(mb;`bWw!fh3*A3)fcb$dpJ66bFfTY}r5cVLJbn7Zd->)D
zhUPv#W^wW67MA8t&Jtc0(dB*``92r62(rDba`+ZuVBp~X^TtiV_LBwyd&)Q~WlgVL
z<ZC(zSqr$R{8596ab%?Am#+&C+qt>%Xle1(|1&5k3d)@*zpkRyeCq^eZhu>+b~h`P
zsSeYw8yf^HlymeJII?B+hE*vk(4}FEvL7{w%(JgIl8`jy;4=E~(eTAfLkUU8*{?%h
z&PzCEvM0S?(kCOgdd)+R?S3**Ef-hB_T=cy6BO*w0efaq{G*0B<=_5VgFGPO5k33h
z9hqHvZCbv(F_*cH9RGD-8NU{j(Vd=|GLXu1QShULw6uiO)E^(49mA)CmP@s^I>yRY
z9k5hZeptHs2t(FU?YC9hUl^BI$(<L|hXntk;71KQ+@T>U6DB4mBqRicr5PAE%k6V}
zxGR!F!PJFgm3wf5<qi`bhHKN<H$ffd{^-GjhPd_x3l4xHR{wlL)+q(;(;uGmH%|}<
zWcPc(Zql`O;S9SEC3g!okKJ-<2H6DKg<JWkA*SfS!GjAH9XfF8l!Dr^gq$-A*7C;2
z`#n#sk6`coxcc}}*)1=bnF@N_l};~WJo5!IhOj92QG-tPpTE+(uFq_5KX{_!;0@5M
zI9p!ct}u;^mkgWd8=X*|EzwXvW$u-C=l}nQt~*<l2-?dG8KiJ@Z*QDFy>aGD$N2dN
zZ%9;CO8*NE?_@c0L?i9UWRq2fTe$d*(z-vKfF^*@M+r$ydE3GQ69t5le*B&7=-z(t
zhJ@8+4NXs0?T*ZlNAkB7F8L@;esG}a{|&R<Nug})5tCVqR)gkq|NmbU3bOde*KS8I
zF00Ec79LE<IisM-Ik#X@%|f=7cX^dcCuV(WSk7soU{Tqk9r@rR)K<?&4R78(dY(RE
zQqqs_{f=SN4&IpI7%RK0SFngfO0}U(lX<$=mYtmkEyLVisW7Cya+LlI3flkw7u7v#
zc$1lsmNsEJXqmx-$H@j}X%{Xxm-(-_bmaj53ZdeN%f}!12xl<tuP)eXAdzUEuK=1n
z1w~5HqlWp1m@PNF{$J0QoXp0|-1h%JC=)(;x-x-PvX{-rbjo%82<_>NJ{m{F`6EP=
zuYxvU{{O!y?NNh|;s4+4cPD&)%H6!t@XIN~vNASFNv$<vYWk@OvNsfx=Oh;}-LeD~
c9smC`Gq{>WCzfxqH3hlZ)78&qol`;+09#ZQZ~y=R

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
deleted file mode 100644
index 6f8b65451cc08a463e4305ddc4be0dbe2879fae9..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 18058
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qj3-okx45^rtlpw*nn1P*v
zfq~)we<{|*4C<pylyG2bm^g9635^A;kZ`$j!JDtyfXDTsRbf{Xm!#6h|8ET$I8T24
zy!^DoUTcf*56>>$({aM~`q#Tr)_v9Y?N1wZD=^(Q%StVGiFhw8xVyb0B}#MZOl6J&
z!M*F=SDunsTdt;+mO0IVVV%?}z9XA2)xTi}Ma7v*o_S1$0<0HSZM&Vx%)LP*rBn91
zz0`_KHvZJzb!I!AmG&y{*?&^7>3_|WDqcmIZ@)E<y_ZOT@5`a+w!$oVpPu;pn48<!
zZhXEu&u_=wPrFuaeOB<`@kEzbfA7ihi$2VJl=|>)U)B1p+os&w_Vs!2{d`|e&A@X4
zj+#-cyLcRW9kaC-S@z#!Z17<G(r3uv@Ku~4G__63>*Dq7DNOGx=cy;(kVsQg5>QN!
z4KMAR-nv#bc}3V1Cc#hhp3SUw_i}MLcTvKGP4J@0#DAxE*@F|;kx2*h4jBlz953Wh
z>=NGbBtlQ&<A3=hf{iUqLO&9$=e<<Sy!y|<I;z)gck%7-<pp2<-rAYATbIAR|ME)N
zwucpKR<FA8er~ZPOX=K|TFYFrLq+DqvkE9Ikl)vG|N2q|PbG!qeb%RIa%UJjC3y7&
zYHF(b2^Y5=naS*V?1V(5c6R8rh_xoZ9E|r~f4%hE_r0XDXPD@0x7_YU0X@$?UU)k#
zcv0~itBDfsbI-5J{<ha6R&P(-*4vsaci&66=a^O42`cYtXJNRfzvIl8Khy0PL2<Na
z%0{jh1D>}3Nvy0NSuTDp*-<WV|G%M3$OT^+ku9%G%|pCjzjbPR;_syJ^Mr80lgc^E
z93$&Cy$Kb`zi<EO2Dh^n-^rhE|D5{yDnR(ajBh6&99BB);L;w}b>*G@Q?0{mFS)!r
ze@?8rr76g8t>{X*Nw4QE5-7TT;`5Tn7i=zDc%9SiT9J97)I(^C?kg#m6<R#Z*F<ak
zGY{=^=Fs7msQz;N4>;VnOtIx^b`WUYuY1Tz;b_RpoS>y*U;Zt>z^w6x(Pg87vF4xX
zSe?0_KBiTCuzTia;W97s+o{j_fqPDWQf@kNxH^o(&u2;3!S@_Te)_xAEV1g`v2|Tc
z`TnXMn_MS$Jht4fomRDH!{Zlsn3mq;`Mak{@?4_Z{l~Vk?zb0t#R@a|I)@tav>dQl
zbjsvz_+6uVu!|OXK6>yl(cEZ~kwHLV`Sq~Chl0Y7Q<9sXRlB*l9Q^SAzrEYB`VT@(
z?9!?`4UCjr9<U{?jW;;Ozs00+|AZgM8M=+m9BEm_n$6U-%wUH51GYn3{~Xr1y+D-9
z=*NF~tsj4_53i5s$-ifCXV1wGKMW85{LFKHo?*=HV;|njAKsoHVG_g*$;*q<9wnrt
zJb03F@!GV_8v{RmHl8@`*`z5?8m4Ud;KeAnd%}zMrTiIYPC*S}`xU(SwrpydC}=60
zpvL{iQerDl!U55<Gd$T(mT5FiXIT+upzxC^`(Rs4*Mbny4=3FEN;!VNk8k_`Uovig
zTE(Z9hmTy_R)<UK#U|~jo!vNb;=|wNTE|pG;0~EEUA(C5T2cA+&6|UtJxfbTYh#m9
z`S)OB_r!n;lbDP;BftJR?jiqwq7d)2jE0+iCw91)Xl_v9TOx9R!TZh&wPZ2574Lqw
zXGujxr6(jfgy<DCObb(ZvcI0Er-z4M-e^z#`3;52ho7G2`TpK`PxWzzmt29cQ1E+{
zkeX^{G>M<v`0GbQL9w)lkCNPYMcA$sG0hX%u+8$p(s%Y1GkaooDoi!W@MPJokhW&=
zHs^<3k{3i4FXM5sY&`5!F{|Nv3C{|-Bai%gLi$t-4HC4yADEuoU(fUZpJB}2Qw862
z4xgONbG*-RPp$XXXB?1Z-~>tr#>T;?&z5f782Ip6n*_&O^Fxmw=&vbk2rG2Bc|w=d
z&8?!FC9oo)By@sw$&00|Zo;D3D;t(eHm){Mh&ZsolKG(9s;YkrRRx#aVh`Mzpldnd
zX2wdEro4M87KKfThuPYs%_Z#WmN@NagIK9ixM70<C)cK*KO>JHiFozeyQsjx*y!K?
z|MMA*JQ(u(4rYnzsIss*t@btW7N63?;cvL%%4hk|SN1vwA9`x@NG#=J&MG=M)o#Ny
z!;UYFAvYdqz54g}e}BxS1tEGF4B49!9JHJyBqaX*Oa1Y={oxtOw%pqie6mRuzotC=
zobI(~IwQ;(D^_%{u|>JKN3UANWo&eaol8<;!B-BOJr}fCx43)|*Zlg&)l2NZVTe=v
z@(DX7rgB8R_@90tOk6<tyTnsQW;1OA?s@emwiy^X1xX*;nh_gWq2;J&GeKMYPbg2(
zqlOLt|0e(V(wX?!txeWiLN7My$A`v;-}8+gi-YY~GA=YQGV16qK6=dR=<(YpPk-LL
z!N9SnX*qvIm|IrLfyZ*{PPvN~dH!|J-K#WRiD!i?ca)0Z0#V}~Tnu+Qn;F}sDy)(Z
z_!!=p^wQqVjV&U`!Skj^T(g_8(Sgv-3px+GDtJEmRoZ4*EYT;ESn+4Z87)ZgD;XDZ
zaDH@gzrA$XtyL>Hj13G751ryYQKP9-|KGjq`hK1ZhyLs5tYBWNb#Q9lzsF*Z$qGrV
zrm}yM9Wt36yBY8J3ZMDXs4l5E-MJ&BWy%ymzo!2Ci$$e*I?{9&r6nGirncc*%!M5v
zLO#8hKYX~IXLY!-PVA`<e=IFG@__w4XXa_%W&@tK{r9JE?qXoqo4ZhGaY2FwTQdJ6
zxgOpx*W3b^oLGN<qw@2QKioN%z1*tvzOcrNt!v|oiIRo?7@qdsbPQs*max!BoxQv8
z-%<G`S2VwS?0ezJtJ&zOD|&2K`@BQRf@UYzDNdR4!SBKJh-u1qB-eZ^S6V5g@WJ5i
zJI)^mCX|Myh8umK=EvH+?qr~@?XP_&j%>-AaI-h-n1R9hFSf5wZTtHt<{P+nnKEOq
zV1oirYy5{B2dv-Pwl63VlbT*+RAwmlvDHIe)?TsY`@}mRrTgYA58C$G=3DJHVWwm4
z0dMAI>UMu}^Qn96;5zBJ+}Y0VRccwu&z#~D6yLAeFsJmEkK#VRg~~OGZVOI{KL{1G
zlP-3<b1*$WuXOXOvQIx8|IG^vW$@K3QjlBu`JuxRGoEAXij8}8xU-J#dbI1pt^NBq
zftx2&W*G7{DDb$}zdrZs)eF%(bN6z5{}goZ(A^gSb)Od-UTkd?$vFL5|Dn0e;~Uql
z{4ZZ9?P#0yCVlPZ<tG0p{p#ottuflk*tl{ILtfdEB8F#jH4Q4QP8piBd(LrZ8=J6g
zUhn%OB&ngg(QajuLS|ah;w_bX_P!_x=IC0gHRq&Se%^-%{CA#2l}~9(J-9D9c4Mx6
zoueK&il>C%<~*RlW5cTXWxa7p(?84R2mhQSUa&2bpC!KZ!mpVQ9iH#>ewxeN-gc$x
z$)4pYAI-Itnwj<`AAMaX=%gN0!!6HqH}MO*f)L}2w#hlNs!ToCA|_4H3`{ZJ?ri1D
zvR>r!i_Lv|Hd#CljyGXwm2g}mA7FMVkKgR+*}GEe>*6LAI)DCh%iBHXbyrY9YzFW1
z&i?mopti!4u#LP26nKggUNYyzTwvDv$Xc!c);;zYQ;EaE!o-Z33PCKMJ#Xb7e25E*
zD!%6w@k_h1sQ0t9UDvnO4CgM{Wlu1kyY)%q^lLp25*#9KZJ5p1YSp{BMr7^kbNh69
z<1cGp5ZSzHbLG0^*UDM$Ete0nPEy<&wpiYt>CiD(0p)M9&pRI<DDA!KdyN<Dn3-V5
zu*B<3*doF+L2I`G!{W^@R<o8USg@~iYG-_@YyaQvkAizo^ttDSZ_-vz`x*0w|8~X&
z6Q3hZjJnYcM<+Na8K*tmy7GWZrjmPkUUz`gmJN){ue`U_Ik}%{rEqt>pxnnh8h@DP
zHKl)Z*R5Rr`hQH#1n;_NFD=)tuXbIT<$byroQzt+9r+F@aAcnj-*$KJ?7tcR_rH6f
zaN>*`hav9`kIulq%XiFue(krOZA!CLX2|3lO1&bPFLe(UUjD(pyg|85vr4?=&z6j3
z?n}9L7<0B={U9oL=4pkZO6a6o?g>)_WM>PCeX!gn@w52ht;Ss*uddWGY-xLasPMp>
z_>a<{U{EqE)X?r`Ws5p-sz{;VY>`-AM?~XN&WbMz+dTwaHFSamMR|?H<!U@qg}&`@
z>CN<En^nceqZifiAduhPM}kY>RKR2%8v%pK;yUsI6FDvfF*7so;t11=F}yOh2UPJ!
zMe$s__Ci!R3{uT2nHAo+p&=o`VQge{U=piMr-1@*pmPiZzx9+4PN&YExGKOLb;`l(
zN5o~uS@SOlF{Uvk9SG>pR9JVVEtD}V^+b#h8}qWy7bbBtzdO;EUfW^O;u}=u<Qx>l
zV`s+`6~(h=&5p;j8yg!NA+?N>Rv`yx;q4nQIJk=!@EREz7#JA+YxdcEFzd_z4S|6!
zlHD>98zwa{nx-yj)m(StFi-6tPsW?GD~jy@nY4uRUOUpIE|~nvHBou0#tB!oho6`u
zdFON+$S%IoV9?VeQBxzq!cwq%B9eV`EFvN#a&tod{F!p|rldrHPDVqoyzzp*4rMEj
z4^1MS-Rj~-55!UqOl&SpjcE!qNJ@UZQT%e{jxO^Uy(1=!t=-z14?GPcSU;{vOilgp
zJ>9@KIU)V{^F(Oq&9V6R?{9vN&f!C!#KmrYU^{LfVwM!go4JX1(=mBXh6$G}w(U@i
zka(bfd!yE2o%Rh{#~!n-c`6ph{83&1kKfS=3fjLWZdIO~(bmyn@Zz;`LD8dPNpP8=
zG{?fkwDs(nCpT|OPIz`Ikc<CEm9EFEgp@zr{u&<^25qV1)jZguz4k#kr})7v3&WSp
zt3Mm`s>ZQI%J^%#DojmK6K7+qS;?-~%~nw<IcKiqlBI<^nveo!f<;8c3=@+nJiM(F
z#Ig+)Omzy*&id#2^q=#p{t18i<!>C|QD`!3Ide45E_w1HttQb2>r+omJ2YRX;MGwd
z_K4d{B1DZ7yv=!dc&w~=`ucuE$w7l8rYIt!W8>!k?y;$##aEtf49YPu_~W`Z=0Xr#
zQL<R&n>InUPx7HMwjCUtkF<<COt@AX#kf86oH*mrk%;#D?|FD6M0uH+#muf%u}oL=
z^W&+hvG6W|nmxfnMYS>{<Va-1gQZ97BOW%EzGY34=L&k-bjiw*i{puaqh{d0zrVl#
z{(isyzkxHCo>Nw3gLZ8|MAw9@m-T0OKK!?5b~`a)noGgPjjP=KJb8F{goJo@?Xp<@
z37S3)YSt}MO3l*Z@s{k_u(N2zPYI<DsdgO~T6*gh$_xG}b}Ae=Kfm6t_Se3oqerX1
z-zzNKc;eJ2DXFf^EUu3qH!is5@`F+NSPd65+jK$Yxq>EQmmO@HcBBaA{r>s+{^R5S
z<!pX@{`U6%{`&tbR~fx{VX$HI|MVoKW2y#^B_t&z0+xjsEDwLsRG?d+r>xw($EJCX
z470fSH(Opv034cbF=K{=sR@gv#lM5g{2xwU@&DoEr7OI-9t5lOY*=!*A*A!bYXvqH
zHj~af`~T;io|b)i`TRYVpLvy)58vN^fo;vEO%40^XT{8!Hf@>$H#hT{vNJ*)kzC<l
zThx_0{c9>1)|GG?|FS>)`uhKEIX~^Hzisap|L-s@Y{N#wf+C|o|Ia^ocu++0WJcaO
z1x?R6sXY;C4<3jqY9}Wgu*>}Z-=0}pySbhH@G{>U!tCHwd}zLfL`o{_jG2-XCb7<G
z+;@p(5%a0!l!miMoaP=paG+sV9>3zh|NsA&zu%`9SyXqdNBI1_g*wy!{h$Bv++6;>
zdzVkw+bboybmEjHYuBASf9Ozy;NJ=^PPUwrtl3-&G4mc9%+dJp@$vto-TQrJe){^%
zcgk0h{dJDd4}`7%SN@F0IYuTqORKqsg;`ke@Z=LOna>~5Va{r7ToxNR(b1*l;ZbfK
z>tB#~KlGfXuux7|xH2s4$t#9pUG+uDpZ@=kS9uqobK2>@^285~0&U61<-UEBbMuxw
z@!)88{$%y|j^4Y5_5Zf+um7)jcB|0?4K0psY#lkd3A$Y$%+&%e{Qu9hzs~T@y}!z8
z&tgNbs&ARnm~^DRt@^t}Rn<TJh!6j-YCno;zt!oQwm{S2*`lpBGiFNm^lp^qg9Kf8
z5eMhTQz!nkuKk<hkd<O!bf9&?wLk7cKhGa%uqo=hBiuMe^2F5B`>(Ix?=z?3$K&Jj
zKb#+ae_#Ln{C+(ho08w(e$SDSwY1Dz;?uKrZa14jGM}%^>Z&yjEvueCJ-t7Ad)?cY
zKP#S}lTVtKQ2X<cLA6=)O3mgD4rcDc*P0TBV$+NiroNugJDFqofA?@Tl_LiZH~&=-
zfF{H6A`XrX2fie{lW@@YmXQ2^z^i-zhIL$Nf=v3B3wr+l|1WP>^Zw%Eeiv3|=EK+5
z*YDbWzPx<?{>snHL4Oj(R8GXO&5&C2lvheYqi@<O1ICy?|GUNO&2xUtn)U13G2TT%
zY%Dev2MlVhGLLV2@T$ikVT#O*hM6-R<L5iZ$aI4`sQ>>Tnr?CANQ;Y0L_tngs|wSU
zciL;%yCoD|edRNh{!2*w`}_Cz_v8Kd`Q`ViE2>CHi0SQ#jI4Zgb@luDe+yJCEFAhg
z*09=19Z=<C&rmkp!KEni`v3p+@&9$>{_I@-TzO^oK1m6QvNs7kO1o2Hc`ROuH%;x^
zXZOuh3>pG6K^ak0gk|T>i_g3SW-m-k<y3k!F^ID$>%b(n2ZpB(9V&QwsylwahxOeT
zpP%24-fowbX<7bOs%VvyAMXsMBg;AeXfjOa)X3Ub|9^ks<LkQ7`|j@bKDR7QO|5Nh
zwCA<ftv3#+cAWY9bFyQMjJG5+BuzxF-MG=v+uQix@9*<xN^ZJ2mGRn@hILb0O{YHm
z?W?eO#RpM6o)yi_{{uZ;wtjkZQ#~!s&Ex*miVv@^?{DSKFDx|F)_z@FXqecuY=Nsp
zk$Cfoj^k5ZE}T+H`N|{U|M&Oz|7U008>iJ+mA_N>oaZ#-t9;cOe#2TZjsv&k59YHf
z=VWTh%Qvsk5PJ&^h0LG*^7S)jL`X^+^7|eLGWhfVzj*ZAPvV}2QQoY(n7<wGkAL*&
z&J8wa&SwAlb@_LG6ddnU&%ggqWy;R~i7r|!hKpAHJl?RhEy>|h)70*Hd2;suwq{?y
z{`z`-?O(6H{Sp!q`|8{CDpffjtPCj7Y%_3i-S|Eonqy~Lu&~?^6=9iZwm#{Y$*mf$
zs7pVaSN+ppWt7pfaiihM)1M~?YC9iT8!bOwZ*TQOrWC`k^;b$7EYr?6NKW)<s<`sN
zi~YpmDvpBZ=ib-;-dF$cPk&<!sDdyzKPeq^(QO{bCdmZPmd;McP*Lyakl}$tGcA7n
zv^;ubfnbl-@2;>Gg{Z~<|NnQ5y}>yBZ}Td-iIZl`{O{@gd4d?5$AkO!ZI_qde{=6|
z&x;o>-t7MaE(?U2X1rv`-r00XZ(s5A?>me4*G-CC>cA*3&nzyU8TCMwy^eK8<j4E=
zY^kYie0(K^&`=IvTl4c1x7h0subB5XcE(IKF#6{z#eP`FZ-X%l%Z;Z`et2-RseJhL
zwLf<Eyu17VODg*w_-s<-_P@XPbi<N2@>y92Zt)+QJnx^e!Jqnnf6l+YK1EJJOQl71
z8wb-9$DlQ`!ak85)8y?p%(()!aHd7w-(N@1oaw5P<1JY?b)w+J--;($w{C6Rd!0+z
zo$q@+56}N8Q-AvUcFy)Yz*hI~Px;$h$CcgV61%_FugHmU<zDB~uw3ZitP=}*($D`3
z2wHAnV6b6>fri$k#e)01S$lcePi!jCJ#&WV{e9yEu~N9LDylzIQX&kJzT3abkVvyS
zkdyf5s9LK=duqy#yLV4=aY{URkYH8*PWRp?hH#(cWfC!uKA1FWo^Cky<KJI-Y3YfF
zKi_EFWi_*td9!U}_I6#VfRH0W-SFfhAvtsA%=*P;h6@6_V@@kfRXWjn%08sU@?q1Y
zl9-q$5fL6Y;#%1vB4)h1`}@<Ur{y_@y{1-$SGji_nV71`u-T`n>(8&R{f&){lP51a
zl9+l_LEU{>+@HysJUlOyUkE6uwJjBcW>ZPk`}_Y_{rn{UWQS<ZO2)Y^zN!UZ+8ZD8
zX9+gljQOjse}3}Bh3;j)e*F8peu2X6#qRM1+a(NEv#oi=v*R_lO{nOmb@lQ4|4p8d
z@aVC!eIW;@Aq(roJ5sI8O|GmqI~DwI)cx&RdKT(2N%{Bp>yy*c+UDN)E`B@HXM&hU
z%>j|k>$@Wp6u6ZZP3U}CU$tx9ym=gz2S5J(9bZ=`=I57|bFsg>SDp7Dt7L&Xx6R78
z^LJWmtE#rI)!k@)X#-P?E9<&X%t~Kh2u?GDR^&4+R8)4but)?p^H1WKpgp~whv$PF
zR}|NQsDuYu6aQSkD43IBpsMe}!W#Mi&(Gy+&o);T7|sh~37_?&dm&FY*N^tIH7ly(
z;@FIh!}Rpr{zN+!Fp7z_wXw0q#T9)~Vm`BoDN-QU*^HU_@N)l%ZDG)my!<TT;lrQJ
zA_DdQ|GU4Nx?rl<i(dYwm5h@gK2?>MU3=_^OA(WxczSt(z|Su)&5u3fj4|afY~`Al
z(s<I<AR^*lals#ct|W0m#hueq($YM<k}h0U4wbi3hzLnIRaBfTxeDq%^K&16d@Nt`
z@0C5<mCouXe>G364Pk4m-;l(dJ-5m9>z~AgAE(d!S-<_dk&@HQC7+rnPmc8LjCOCI
zK7YRC;yKRep1yEZSZKshfAh2aVT~3eN5^9m1j<rV)AI6!Yz`eb@Ziaj3)kEZ)o_J1
zT$|m>t;Bq$kH^}28m}KT>0f^K<KHvhqT^rn)3!EEvHPdA`QtL?Cs&&9{Sy#Qo^5bY
z$t<jO=FA%`i&mVNIzvNUyL(%|xUu7dRTdkx4*R%8WJ|vJ%BTI>UQ@H}oSz>fqnnbj
zm9TL0YHeqK{^kt^6aO9KoG`uPkYvR#9UdMY`#OU^e>ziHIU#k*WQ#fnHRD8Iu8apx
zQVRtfuY6;l7s$J&^?3Xhj^<NcU7OC$$qbEO@9(ogOQYcFGc8t@j?W(ql8%_nvry|O
zP?c>`bN19*_dhDiCwO_$$`uBJn-^$nbR^y~VT+ADq}9nXBm3|LmWhnU#)lst=dm}q
z_6VA8XIiwh-rTW6V&Wz#rKZ5^xqqB>1OF*r)y!DRDkW~YP(fVfzomslcJ|kAZ;MS$
zMUxT_&U34%jEsqOy?x6hDQU@sb=AMRrHV6J|HtpS%CgRiBbduywngCK<KORJp4?Jp
z+P+FldEVrp88ank&6X6?PYVcC+~3~BX>i=lP<mkj8#njix3?pBppMx-@A3Ql?|FD)
zcrFxma$Y;rxaM|3)7lR|`gh4Po=a$0x~Kj`o?DqN^Wl0PGo@8BGA`fWmD*K(x!T6-
z-zQU9`~Keb_xJxB85wbKZj6lJXlwDPtnBRU*x)d2QiArMkLG5Dn#l@&f9?0=Xq-Iu
zE#AF7@q6AkRqwj|+uz<lJNtiqyz}!YOHfl;(#J<))oRHtTMJ#8Tp6;n9MY7z>wYvG
zWq~A=DY1L`S`0Y;Jt&&Dc$UQ%*&06f`4^@aoAdRqJ1Z))M=CeyvNDeY2Tu(%v%Tk<
zYhTl4cJEt1V`j(U!o}AMyPq+*2XH2&wUn~#`cOQLL&ngKVb1*OJ=aT1&(+R;7AsP6
zf6;0Edz%f9D^At8IdMhT_f5H*bd1ht>7EkG<xWoUc_;EQehJsa`wISjIl66(mqm`>
z1dj?HdbuV4{y&@AU)-kmjSUU{xLSSZJ(SvF>MM~@eOKax8s8JwNCAfJOuDsyk{i~g
z{y%=;z=1P={3B*amK{*`{5j*ti~sD!{856R_)jMO-=KMHmdgSa<;V77)9Mc$`rq36
zKRfGDZ>SbyQz{$Ff$*lgdrp4%sx4Lw36VoHE$aUL`TgZ(KHH7RRU0<XRY>PPEIGln
z=|k32`)x_hjJNG3{+Gz9s4)=e3S`?;%0BTb*FT}w#vMxR+P~$^gz|LGGEUTFmb|m`
z@Q0_|V(=<=#on5q{M)LgG&1fw{nlPt*_ipujPe(P6aW8LJ~y#-?i3H7txePHCY<Jp
z;bg6kJAO4oVQZA)ZJ9T=;#JFdQW{KC4U*=BpFD7&;J4YgBhW-P-NLT6sZ9Rg|NrNY
zuw<qf{QdvGed0t(P1!%`4l*(Q7V<*J8W`5pDm`IpW_16hpt<b=muS+WX+MNmSBuVI
zWxJMQctEq6aiw|po-YFHo;`Ga(*_NL=@$Fy{<7Y`&lWxZhkRL;d5oyA52LW#>}JN&
zmzHAs3r;01<F~mfWxV-OXKaO{OO|#^LSUo))IZMO4PzPBNwI|Me*R}$_7!Sce9@np
z@9(xx5YT3RGa>H8GM0$BpTu|0J;<@wjVZge>E^SBT>_@H3CnocdL}h9Mo*r=sJwjA
zETa>z_aun<{Wo|c%g|7Ig?louV*K7imA6bGrOb4TeKkM-zPcK(sCc4qGtY{*JU`}2
zbUf}z{2NiB>Qu9tdGncs1A?AMj%fVrP}wAMFiVPAP<AQfYOf0nhPs#k|1aF;CMh8S
zb>V!Ay1mukzyGgKt7vTkrK*I6E1EnWj}v%qM9FFB9Y1z7v?#%ijd`0**H&fcASNZ#
zm4*r025mv75)u-26th3KZG@Pl6i~FM_P4m$i=GXCbN^3B`fGp5f_;x)qC&{*gpb@=
zSqTTEqcbKRY0$kMG(+Ko_A%2ph7prkKWa8NHa>h8J4Y2-d)!y}%j_xh|G)DoHV?%{
zi(-juEQw8XKQ)(qVv0J|P(0CDpto0M--(00(>NqPq-AV*#FCw)q`dRN#*6?JZnmBm
z>S-*0pZU(=jexl7&~#Ay>-)dI@=Mx8Yt&fl+9icA9ni3;HAo22s5r&{B*9hNSi+-U
z-A0M0MMzBUvn0n%`-mVmsU~hGw^J9dI55oH%?L?zN(P($KRhfSvHFqypK10>q@LJc
z%CJ(Nq{F<;F-W98VZ%gW7OCk!Bo0L-2wWCUJdnuBY<9F+FEWAY$v=);KGH>&AOBkK
z>+XaaHpk-M|NqAsUUwz<%vE4JC?RnC?}6jTB?7iS;Br5ca<CyPY;sglfQGA$)DnjC
zD=cSSSsNWOt?&R;_qHbveUJbD4_w#Qu!Ltx8wYdHX%RkyY_Tnen6n!h(-&Xm;JGnN
zydp_9`*(A~Ny9IbSeY+^L;8Du8@KoeRZzc};s1Z72^Mzs|Co0c|NdY9Lv@AE2I)T+
zq$GP*Jmr!}D9iZ&IN?CDtX_v{*aa&N9*avX|CThE-ZfY-t26ZP0=6T|d;b5Iw5v_B
zsF%B+x(E{7N)tfYhu2$pf?&ysHPXzVI3$>vnccJ+8Ej56iHmRMI4Gl^eqf@`hn`uF
z{q>Ga5pK4PG)Qo<ubcY+zww@mg9Sg0_8mSBH>>Xdzt4Yv+snySnEZFn@;7igX7lfV
z`Vq6$hZ@SOSX+O8er~U(_T#AIjEzm)4<%n8QEk=i%RCyOrS+g8!LpI_-96QUUQ2MM
zQQE-s?|A?I_xJxd^D-UUuJ)(B`@zljrXxp195=-8{dM#1-#-T${db7ZTx59Rjnayz
z@|TiAeRS&$1@4qH&os)rlW=1L<7qvUB!+`6OoBV><{o(H2zTYZ{r^Az`@8@D|B2r<
z7A)b3d358DtoNxO|Nr0b5`AA)6(?)+gVS06+L8wC+6R%3`=(WWSQ3@8pGW1#&&dmS
z&Rn>%^WibxMOxfj40+htzP!`kBxUjap|j5K<cZM84p{r=*Vq4t**%uFT)Wp&JW-t6
zXu&nj6L-VidV+r0ANdo$zV82zkN0oPczl0<y^l{sca@3WziZ6%s#+M=A9G}>`<Egh
zmbRl>{BK934MX;?16B%biU0qd{qVz((HLsQ1dACnCHL1f-q$<FkeQ;;+kR*+-_t`k
zd2Z;7x%$ia&;Mst`)hU+)9NlRZsE)EGc2yL9Bfd2wDH(?dzmP<>P-y{d3kJwg>CWs
zCC<%FHZW1SFB9|i^#A`n>*EA73JeSkpkcM+Yv0HJ^-r=S0-GJKwI<)L?cbW)7`M>C
z;Lq>R&)=V%eBW>GFFAn&i`Sg#Stt>n+qB1@FPQh77Eg4nu|zN%GxHalB0s;j(A6Fm
z!4rPm<6>q$yfwQ8IzFZ}0W^$IQL*6R@Bj9H)bum%I0Uiz%rW`$|Ns8w{`F^Pe|L0h
zI(ojpyStk)f6}3=OAag*GIr!yq{eD^;L;>V-`<C>Lfa;*bINFUn1wg#^m*GdGaufV
z%pxVj!vmUL1Z9_qqCd}{y5{XY+|4ybP3w}JMa9ZK^`n3O{{Mgf`}_K5XMPsEy3)VF
zATK#NOy(%_IyU9BN`~kE+NWwnnBV4YOkrgF^M_~VOrH66hHIjafB0wnFo<#IsRKUR
zGmN*0>j{)W8tO_DEdKrZ`Te!Ha?uoLKQ-4^m;brj{$LOk_IRYR@!$XdpaIXkJ3mt2
z-SNM6?M-Lroi}ePl;px)o3EK3z4g-d-CRD~2XD?KWSrPgs(pB&v&+ikK8!Q|igWyY
z&VR0S@d;>DOt84O@9+0VtV)k2o?FwZCh{ct;>T%XCypNY@7%s`>$ZK*pQr0+Kb|@{
z`OrkE4X=*&RIg#>EZUv2g2~XZZHGnM`}>k={>QezV$wdaN@AW}+@6FVKPGNagd2Y2
zhQ$4Se@`D^xS9Q*zwGmu`Xy$XTNHl&m#@<|Ra(1XTgrc9gFnB&zrX+R@cp8qd;6-t
zZ{IcR{H<BNot)Xh%zgh7CM&t6aHw;0BqS#n6eoZ9kYG{NvT)a|hc}INobS%q{NYGL
zx47iJJ&J2UaRV;NcI=R7ZTWXlN^*i3d&@`ut(;9uMGkss{NGpqe|Oc_=}(?)H_!W1
zSp1lS|9L{{-}Y5AX1Fxmtai#?!1FVOk$ux9p1L|7CMF(fGlP&2gM_^YInQ%QgdP3=
z-&n@7>FES@AE@69&K*2<OhcJ@rzztmhQIQcOuzhpm~El3dv<{O>;K9tfBye(f4Ez`
ze!BjDyV6(Fe}Dh~@BjIN|8^xyIe7jwzdU4muF>Ye(S|8}Y+pNATR|fmv&|*<)g=D-
z-k*3?t8II}WK@*Im#>l^=DcjMRO!0uqWWnTEc<{H#_=;}7`i3gmP}#1+ug9m^^twe
zd#*ck3qBogub(@2{&Jt6T$&P+l9D2#Gv)06fpU__O`aP<>`_ezO#hhL9Q^*CN8a8k
z{c8Q+B#V*`MfLWBw`U(b+Un@u{-JP6#BUvW`=tA3y`cUKD8WwvwFv+J`DtHU>%sX^
zK91MW$Z2|?)4NB>DS!G)w5$)g+SeJ}+4nd7c;9^Ew3_NKFV;^I`uzU>e|ei9-~Zbm
zKG?iJD*9rZkz|0fv@z=f)#(rC=qNn+nB4aD^};tjJZ<OaOMZKkIK#aE;hRV{QPCGu
zH@tbn<L1V5>J&>))B8!ae`Y*9%DpHD(l$_<V6kK8Obg3}N7-un<T7Iw{(O@KburuW
z@Bg!}`}6(A#{1Ug|Ey|$h)YX5`|?hoCFLD=FTKN=l|^EI%};LW26u1C6Rqs?ju`$q
zxZJ-(a>9oKw!AzxOUpK9cF-!08ygxOqh>YEpWirpwxY!hMN`qJ2?w+V-FjYED%@G_
zFL`gT@~JXtD!B11;lib+o)m{Szx($rl9*>8At9k~^3lV?_0ik@RQ>qyey(+Wzx@Bd
z|Nhns3c7iFUwe^JCM|vK^z{EVzgSFPotJOBzW)Ep6$XditxY{J&Ei2T`zEF*(@qtz
z)&Dm-Glw(R^3-YLj!wgjOv4v14J|ATbmGnwJYqTg{5;S4I70(N!<ZchK0IXpC91pQ
zmdt#6;oU+|cTKR6ko;f&pD(lh(09-O&R$|X6GE6gm-oy6pKbB+_r1OQ?Q4Ji|M(a*
zJp1~!vASxZi`yn{Zet^ZKk>W2^=~kE{Q3ERNePL#y}!PHeQlqXrk2C&zs5JPZDMeO
z!+~aYp5^||H>091ELOOALqbMY^2`~DdwUXh6f{0ODcqKSU&6$6;c<baX<``<&dY<!
zh>kt0KrZ|L|H+(<yv+^*EYH_G7VLPStmLF5)zC5N<cb!y7mSK5k1E+uaOfyof4^q1
z{O*IY{RbAFGT3o_-nm58e^=(&I<3gRy)*Nf#WbDCyRNOewCT#C8_zAaXMRdN;m0=N
zIse4m$T^00j1Sr`IoDs*w{QQP>RC)30xS{UlM-|vpV%ejD)IUCbNB0a!|vSs!UCEX
zelmA0Z-WDmTl~W#je9<`oMg~B;J|ZI$wOuui-*P|y+2%R4~k7qYdwE2_597~J}q$C
z^+P!gSuQS*j2#%KxvuuQb<IlMb>fw+AOC&W!WFSPcKb8S%Pky-H=HewIQ3ys<L?E(
z^mUqCB;0@5wfTPjqL$h9-J<=Te(k}{{Vc5)W*9$T4oXD-|38^)$aO%0$F=^*LC23=
zUDH48i1WX5Raf>_m+Qj2vxQSQ{>-0#pkA`CanFpGJ6FZbeCYI`Z$aC4k)|uNdK8x$
z%w|qIH*>D=@+Ko^uT5nRp*q_h_gfi-zxUwW%G6-kQTX}mX4_*)Ler;KZfOhJ_pJcZ
z44Pn(k|I&}@6Y+epXW~$EZlSX|9>7Ho)5|r3wT4#%-mR6J))o3t9(tc_<sM|+Wm@K
zWrc;C7bqN_ZT>%@*>XpegtEw=|K|&yaxFT1^7!!&zs;F1UD9G>%h6<ze2`URQ|omo
z!Wv@H4TV$zkw5JDSqJ)}e)L~i+p5BPBthdyxrfS@qMZc~`EzP55AUz%(b3_VH0l4$
zng10v<un(IvmCs?pJ#o%aui$rKLZbs1j`b!>~zuWxlG|dA26O<9AyA)uhei@zp=Yt
zzv6S&1heDy7ShVE|2seZ_xwr9jyl<>IR%A=9-e>ye|`P``udDugT<Q6Tyi!E6+eYe
zO`Dh)5Oin}+h(2vp<fROdM<wlsdEp77a15Ezj$Hr=jY)Ar+)F9^)hwFKKaz#$hhsL
zT-d?|*8Po*4?jQuU;XV*<9QAPBcnfmPain@SH8^Ubw*<&(>$9*i)t~e6KBtyS+J%j
z|Fxjefv7nL7?qU|pP%pIyA#?}JT%?n-{0gPzos^}vRY~MZJGANKIl}#t*$WHMLU1o
z-Cdu5?@xRF=|8`wI{Nf9wm-MGwr0+fHQ@rSHmmqK>ETiCjHZSo=JRHnSsb{<;^?h?
zeci*`|2sfSuNc5{IRE~qKX~kHCN)p(L8|bU0}WfJN<5UBxct6^q~wg5|KH#E$Uo1*
z#YLi`LZa^Pum7K)*MG`0kvhgONlh=VrRe|PWC4*B0U@O|g0CePXFPd)yk&adAxLl^
z3TG`WGFq`J>7E%!tRYus`T;+)wr6!63PHk8<e3luumAr~-lii+XVXSQ1LObV!oT|?
z-TRvI?@N4p=jt<O(?-J;tA$sDaZHn(Q1I{9)W(@JTP|ONnEWJWCoh8{k1?-M@P_|c
zE>~6-tm`Oo@hHFWm@{a}2lfB9?3SFLrFZS>W8L#4ap8l<MQKl$q%HFj>|QrVE6dgE
z#Lq7myd4&qcAWXKSfpp8;hz0Q+fs`sxxAmPSAB#Tl-HFmRERdl-r(?`UGRIuw*Uh@
z;~(cw{CIzTeZ9EepZ4zyA|fT9JeAzBQ?l-_(!vgD#Ruo*+1%XPgw+LHS&faGcUYWT
zDm~+4lTK^c@(-Vk?(Fw>dITwH4}~Auu-W*-XXl$9r%e~|+)&Y-(yi0_#)*xsEqePu
z`?@m+&h&gR;O9Slem+l3%#}TLZ5|(<avz>=&*S66BO}9e=8TK!mlvFMe-bPzCOrI{
z?y^l#AVEMl`NwzuYW*n_1*LBOcrC*5`=qc?5F0ZyB$e)&yPK=QfX9`8QS;>P$sG^g
z{r}(Zyx?uZ+^**Jju%&Hlo)0ku@njK;Pl*g;NhLuyRIIbvG?Yim0xt1b3VWD>(Sn8
zB1f;e@P9B3`BRe>y(U$nLurZ=i(t>f*WVULcsk9Rl={Rn)id_8Nr>p@GZsR}>kni^
zzNlW~`lRQ@H;rGq-Sh5TU9W0>L_^dev_TXUxLfAfa<v!;wA%AVczAu}k#o>~pP#F4
zwyl_f`#;A`rRLXK8ISFjc}PA}?f>ch?!?A0<!0f!<;?PV^&*<Wzi0Xibv2oU>i%rM
zJoj*Qk7LU=*Oo$;mc#|`Us`SBP2MtDQs}5f7<cyb6Go>dty;A1<Y$8gEnDBeFFU@=
z$k&AT^rxq#^W^89jx&9{LiKpo%J1d#ZDK!y2hrBdu;prX5NNjFawjc1np@jH;bo$2
z``)Puug^24T*wqDc__-AFL>lQbMSHrkG2+@nTIW}eRcbn{rN{?U+wZ-=LHqk(uX${
z$h<#(s#2-RkcYX{_xi$9HkZHmJZ^n?=d$!s!&N@Qp{|nxrirjdwryRad2L_KwstMw
zW{(>y7j@XUt4y))U;kwL<m<W17;b-yKlvL{BpPrWJJzQ9YvN88@4oZ%CF}k;xE%if
zpXbdRp7-|*KYVg}U3ZGnW=7uqv>$(FK0GbnW?L=6CzJ5wQ|rUm;cdtJJ+-|RAH0uu
zX{!4*CGlfY+tzH!H*Y0-dL^Gckq8Nul##hm%`&Cgf4)TA9*6ls>1k;X9v&28jd&g)
zv0<z8!zV|(Z@e!$4Bh6SbU~rjXX@Yo{hM13OwLnya9#gMf;>yj?_&xYCq6u8Z=NC|
zx=PgJ^S_tDZQj!*?CR5Y6f`E@HfsxCuW)$Nkv=P7=6$uvH}=hac!ZNpS((k#vn~0!
z#Gbv9CMFVf|56@2UVNp%<22i*FR!m^w`~6ZKmEeRgLT!G2VLVo{9--SwuUh|Svm2C
z3^-Wk%+lp*0IgqF+P0Q$V|9mZ+ji0Bz|9Ajs7>A=<`{6nplN@{YwH6BJo0Cnzu$Xc
z(6;XO-siSP`|^t~ZZhCm&eSL}^T<*&UdIpZ{5Ml(Nw~*Mx46gj&PM%ofy}d?*}~_-
zj!jVcVdv^NOUO0D_qI^m|8%d=<*PKjU;Sd)@_gZy&xx+?E^+_5EjH}ie&Tf_I9<<~
zwUM{MfQKcYVMarsKygZt8^hAYOFcX%a=B(|csM@vU}<uAn7-kD{kHpEi`V&2QC)m9
z`cuV}n5s+sW{M*3mDlD<hkmu49mIJh^7OvV>%Q22?%CRtT+^$0D(1h+lFd3i%w88u
zC7f>`6VrDxcxl_HwpM=|qlNqB4;6_gwl7*4I^pbeff>s-odJ!Y{{O#bt~FPK1CNV*
zhunc%+8kB9{x4O($TW5IUTr$j)jD?(hnV)i{SQP|P8RSm;yL;EZE?$i(&>9Mwrw_l
z&0#6+c&kjmWqtOnwnYaF%*=PnzjjXbz1*SH)~KS$F=3L8`ssVEzvX|;Tjg~9W9=o8
z<BEMB_q>!|kr`(s>^?;`)?GsB$iKzYa+4$1S3HsCZP$|0v^NoH6$It}DRXVPS{!&<
z?OR+^1vVCXN%ueS+_PGI!v#OVY)1j^?v)`a8*V10u-=R{3BB}$J??>^$0vT7evZfM
z73c2EN{v)s(YoOM&GJ=UDhCVsrd56XVA#7!=Z4v3m)OOou8z|ZBY6(r{kyiiLN2l6
z{o`HkRZ+(0HA=Hj-ss3YvMEKTQ)yP4iL2KVmA;#1EkEL>H}+L3trw~EyMA%op1Aa4
zYraGe0UN!yheCail-AbF{s37RIoFoEDM8?P{f`d{8XhesCYj9_glZ?+ZPRmV>Q0v8
zeW+k6;vaVFLi5Gh7wc`_S?qWm_phM9-s-_|{cHM}yC0{%C~-P|G4+b0W{X1K$J#EY
z%PA&{*2!O9-hKD0*JTCO*}Tn+flI!>HBj+1=s9lbW8ZQ?TqXB(ij%;D%AVsq?Ga)@
zS^dj}k1Ow%y{)@iH1gY5<9pWEvR8H4sQX6h6vU<(KPx^ZbpLt5fo(fu1e&%jT_@P|
zS59;5$s$jQB0Kx7wM%FC)n9&Ar*->f#I^bm)*ayDZpth}?xp~N<MnO%1zd|3D)zJ;
zxq0&JzVzbY`3IJ+Y%(h>(CAPQmbudLz*NIjLnJlK{O&>D<B!Y#?=Fu&5x6@y_rZT_
z`%iziri!s1SKf85=w!i(yyIWJ)*pWv^0NOnui^=#5K&D|=UchUf@|h~o_Vy%V%gPS
zoDxnUK3ZHd?57WzTr9PcZP8f2Se}2ck2ZU|O5$ckj)?Ui_i*K(&3abk=q1v6lI3>V
z)+C!Wlce28H(AI?@OT7^u=*zNUw->&>BLDldo))4^fBl0nOwJj*@^S#)K`8KI`_PO
zj@a}v@G{aVbL_dB0z{71pYT2+<8*SVp^Q*gm)icO+IaPY3s$x{aL5?VD&kOz`r~Z!
zMMKzg*XPH2?+e~pKDW(dFIMs7cDyPWkS>@Z>AYBQn(O}mrPKS_fA77cvms)_^AyR;
znwh5(6uvG?6+E6I8R{xhXi>0#bKGK$qNvhonO-qfa`TSY=o$Zx<M(wA4GeAh@#6TZ
zMLL>XpJZ+yEBsJfW*w$|Xu{`6wH*7J3*Mr!>sc5d8wMUJ>Rh+^uF>p`Hw|VM*d$5Z
z$(!RBqi6KhYf@UYq6A-ItmHlOReas@>$Tn|ujhAh`jh@>9=I%=GS`r+IY7WA{-g4Z
zl^F`T0++Qa80#l?KjULlX`LFrLEwtfjTIU*)D&CiI|wzWtZIEb?dIIn#2cbTCcN8k
z`#*eA+g0-+?O0m#^V)d}-<2Pm_*qg@l8ui&X1Y!K{_y`Va%QV6%$=!rDI}M<bnTWj
z(YD6ag_%F>0$Zl!Y|mPiJ!?~+?iDA6mQ!|erzF)>k1Kl~dg?SS@pjC5?YPT3=XNp0
zJ}*(?@Cc3(Kd#t!^X-akpUIn+ynku6ic3-KiXu<p9M-fyyBccOORZ0Mx4r%&3%Hb+
zqsZM{AaJz)&vMu0hi8^L8g^uCp4IdFg0LXV=8ixewZ+F;clKJguaQ0e@l@e^+qfNf
zkN-IFi7oL%v+KT@&eJyP*z_;^`Qp6su5)kQB#in}B=`A8DzS7q-PSX73oNkOdtcy<
z(6NUHo;D>W+*aYt?0T;Iywv93!}YBWp${cD#jbY|O!Sy{`TG~!+}->}z8sFBiL<Vl
z_HEYKy6j@<obx<~lZ3*Xlss7&3wvJY9mq=)U0=E;P41{kU}$~1eK|N*=FC~k)e<1k
zYVRnRv#r~aRsKN2g_IAmHDQwzHslEG$qL~~D{OxBuBbJ({Lm6k;d66-CZ_FcNS2uK
z{JNrCzE7#;hk%)fH*q*OF=*QVNWR_IwMM7X=7Y^DE=2>LS5n6lBPHBR{<1x3Jy5za
zcT$UvvM;~ETh^ELMaQ<hpZuJK(Y0|3$H&X^+wV4JI$h3?@|j$^^t~a^u6MUq*UnYw
z>s-`5XVIqCRiU%D?Y-DE_Y*kKl_ppyyfq38Tp@cRl=IUWrhkE4mX>U~x^0`&H=ZeU
zNOQNZPuo#7wXwPR;n(ao`+A8TI~VR%Ik7;0qC@<A$GCZpetidzwSk6(A|&pwu=ixR
z$)c{#mYLbsExyqAXs@(n-5-Tjyeyo073o4jAql$75|W^*KPQ*-y5@|>2@$%f2?-Uy
zrp%cMooiK^U=a~1nRh>J!?t6Vd~Mehj^29n;89XXXQD;joQGetb^Kbo4@9r|z}o84
z_qD6bXvQp~6Q_+=tTw)J%Sb|Up+(OthuFEGWro*u*?fK3Dl6NjYD>oLP5SYj-^$M0
zG9WN<!<LkSqC@*uNNi3xpxQd8WL>=ab0%olNU7pJ7ifFZ#*Htwnigp~ZMb>x&>;n#
zGXiqw3jW(2zPwz;TF~CWFwwv;F(4%6!j+T_8y$HhFKVPZG;A<v?&LhY*qz7Ai>IrL
z$9uZbo*GVBg@r5z#>OX37)VGOrlc=EfAKNnhai^gHmAS8?}#-}0d;LagJ(9p1%Is$
zvGPc`vZk>e6-i7<nJ_JF!`9Rt<^72_jgs8gePB2dAf&F|+|LhMFWAG=yiBcmp4`%7
zlUO1mC8tc4ythA{gEuuGAmPRaMynsSKaEZ`c|2ldXJ@vzZ*FCE4iaka;P`gZW6L>?
zs96VciX9~K?;F`vIBcqxfKBfy&9R8v3)+(A=*;|(?@XP}9n*?Wt9My7PZmDByPU_m
zEMSV=jvW&J{w80zoc!Ti+J(!>0-~uGuBc>1v+Fi2Pyp?;J9myJD2Ruh-^ix+l!Er@
z4e9*M;^O<XEiW<%3p4ZcGh17OR!2lgG;cHeq9PW<;QzHT^v_*uqiz#OPf2Nxg<V~W
zMU{x5DzCqMhM%LJ-vrkO4b9C5FAJZttDCc8or08rNXmmpNk4v1SJdb@czLp;PmiMu
z%Wt)yDy9=F_HQ$5o*{8~sW*?97>|?`kF>dwhL({{70cU+iE}?Dwdi&|ew_T^aWV&Q
zs)4ENO38~1>*REp-A+05E%^Um^4c23;$NWQ%>Vzl%vI!WP7r95H|gM7)bUbV?5%u~
zsE*qIR{Luy7j-n!3};-~v|vYXt_9ofs)rWuo<4XOw<EhSM>^Cs@J@GaQ^Y*|#`OzV
zrATh;eXPJzde$ON=b>%(tQN&tZSN;PXJ5SQUA^qw`^WFCc|K8vhdD?|HTC4C3npJ|
z+Z?aUK3iFNY*R#diL7FP$f6DtRgR9w+}Wp<m(SU3;rMc`%d*aGe?Q9J0Jl-*%mLK_
zJZ<)}HBqcfA6z(il(|Fw!rkr)r<bxaRvI3Hx3}(or1HsX#Rs;HN9POtGi7l)H2=x^
zZMqy9TzT6=djkF@X@%#S+xR~I`>gKy&nF)jtTNzvHgli5mrrr*`meRiIujK<Ih=E5
zYbc%GkZY1#Z~4lEkLw<1kFwJRlRa@StWrI%Cq{13bt>5T;io`b%z6vH%&zBx$9KHD
z*rK|9wTju=*}8wZ!{`6@czWGGcJlMzOt1WF!0ix_%UcZuT;dajIb1eu40rjUb??~k
z3RZui3)h0y<n;6|{V)EkzDYuK`JJqpvfF0Je$x~#^^Lq$vrW4rIP#6_yN-8OTaHO=
zNt2i$lq}&siDl^wwM}QDPB;aKELr*0HBQ~X_Ra4<ZP$PNS@+~~PtX?K(@r86zk75F
ztkJIYlMrG#@-Z~uLi^xW^M8+zukXoSoVjAva{mK-KbS!EkdhgLjEuxI-R5&Ci<a^S
zar38q_^e*y$R;npMMg}#T}+~@i$_a~XTH5N-)Dv6`2`JI_x8+G*x<(hl=bBM`#fvb
z@Z8^L2paV{dGf<UW@c{g<_3nt-``uLS6+xY+F+I66uEPOD&LL=&(HIG{K(VJKlx=5
z1Gx53y7pQ|hR3?xC?hk`Q*y^7R+~Zrp8NLo5_Nx@-u<Zl-q$#DX5;+%5AWNz-QT})
z{RCw>0n47Yl|q}%{X`v(U65JFa<CyVAwf(!{hYy(qZ5q__+aJBwuXNnn-edqP2o)P
zNlz@>*wG^`CUzt)KvD6<p-?rot{^tUZONr(vqU~F6J*x#lsuutBe7C)bu+)bT4f`@
z5!87Lzunu^`tY>)n@)!5+8<l&-ZISdI6Ae@Ftu=|n}5UBGY3Qte&Kd`a$dfzS6VZw
zl^^0M#X_5^BO8)f#nc}a=|!zt?tJ2rm)H@NfFvU&=C54>O$^453RO8`Hf|_V{iS9f
z1ak8K|4NF54q{sz!;&6(ip`KaJU@OVQ^y*n0@b85Px|hs%vvV;sF^Q&C0k<6j{_U3
z%`Kz<PJrrL8(i^O?eG*$jo+LS1r=<Qdza0NYFKb}MvP6{gP?wOsR@!IOlKY(I(1<F
zcZC;+Slhg(Ym`DAs#{p||J;Tw(JwZvHc#c4-F7W>op(4(WS+;-68?1x$3pt=yRA_Y
z`Nf(obcDymC*8on;LqRF8wwOpp6)b++Foh#@?C73WwFG+KZ;sYHfkj(Xm>u)RJ<T!
z@Sr2)k1G4J*a|PcKbF$Lw##%Y{xqq3T%6M$dUJn0&ulZpId;CE#MFGCVQpFX=lAgq
zC7Oq?uD)S?(?w~`!zM|Sh{LRw4IgGSFXNf%ZQz@{>bOa7!kb5l2X5}L@i3VB!14V(
z!!_~#tx3)BjX&uMzurc-T?=cA-`~*{7~?57JzO%NS!i|U4j;7)?yv^UR{MzyE&262
zPAaJRCYPGcGjYG5yt~YBj%9N!<4UMoil;p~A=tJxyVL7Lg#dG3#}QRO8)-fRWr@|s
z2DvW6F3*xoc~}3OVRtA?{3E051^xALhG%9t9(wug2Q*kd`>eaaU((82QbbfzN9V$0
z=1rE<DjL+~9|$>et0nm;b1FzTZS!jteD7(M(JlGG`kDQ<*4he>Ce4?(&D-APy*OpN
z3gi?}P5D1%Zu)tNTel?Q_8gdWOtQdSx#Nts%QKZTdpMpiVl+B)B-Kdo@Wx+K{s9T4
zqRg6_%s0Q){|8UmZ0S64Lm6}s2E+gVi(cGVI$2%v)-8#Bb*?+6_JnD%2xxkwI3#DO
zb4<9JaJA!j;*Vf9W@bC~WTOwuV`E;;w@I8~IeF1c5or7@(kT4%AH0$J(o&6T^M8+L
zII3AoKQv_27VmAmn$lF3`0D5rp~*%cRK&dmW^um1Z|pPUK*2X1)rp<Zag{}1WG=kF
zFIiPB`QxXglk>vpITN>UPhkD;IcM836Lwz~`{@fZS*9kMSxXoQ$uF~z>z)wIIV))k
z%aI1<P(uaPzWe(n#dH#G?3??`T?!J+iw-Gne1Ffdq|E5V34@48$LPfo!N=;C6z6vK
zD5mVq*!5w##F^BOM<N>83f8VYo~5Se|1k6XBd^O9lbB}x`2U}${=d<iyQeoKvHt8f
z`v48^MYpcWb8@!*t7B7F*U7Gzn9x_|7$?`ZQ-tIHuN0pK(@uf?0uwy;&Ukk)CahPd
zb=r&zJjW*=`6w=(agr&h^ZuTb6J~VOosroJGVcHXMbGvbu(Gywak16czcBLOa-?ym
z0gpIyw#vg(TbGDAMKH=adqi1uyz%(M?X<B=AaeNu;o~cVdA@$-d4BF;34bfhl$($F
z<qWP|H~#bQtbkNk{!2?11#ux2SIaO>6GyJqqA}jSANwl>*8KR;k*vmT!j_g>^iAjR
z5>KAz=O2F;bA_7nuP;bavTct|+l2tOygV1><|Ut)&#9^%n!tEat8-5hoBu<{Ak~g#
ztj4MjGs_O#{QuAJ&+p>}U$va8tq)IB-eMar3@+amoh(asbZpyV(sn22$jci?wpebs
z_B!HqKF0#D)~5RoMq-N}>-$M?nA#Ow)j84m_>^edx3V_p_7w*2yX2uRxf)keR|jfy
z|EqgZ=6XV@l#OkYFgx?hPz%FU9?6Coj~J)V<_t4DqSE*OzhvG2G>e)U3-``__{_Iz
z&QWfNBc3|4^6;!$#q;tdPhujEj7&!g<NZl3`_mNjby_51cZfXha$1x2Xyw-r5nZce
zoqQfueD4G8iD%`$z<*(pEYuyjj}j6R1OyU3d{6J_Nii^S(X`=E<CrAGzLml7V{fF9
z;7ldXYic|<gpTj82c;I9niCt+cn|L?<>8l`P<C5L6=DuELlTbzt8nneZ6Lm<tDnm{
Hr-UW|I2X+)

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
deleted file mode 100644
index 8e78146d955ae8f02230121e6314f3285e87611e..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 19313
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3w(xXu45^rtlpw*nn1P*v
zfq~)we<{|*4C+KOkMtkpJ)ppI<ekvLf4`lQ>_XicPBY)AlP%~sv`J5MYpKt@$#nBn
zh<WIw2`hvy{N;ReHmYR5-Oi(oAd8Ni-w4vo8pzXkP*n7{zDK*jrVso#mG*GH`%!kF
zKC3n=I8^ZAoM25)6+`!D3@sA<T8!@{6u4aWUAuYvYyz8c`SCd~YS)+Mt^f0c8D!v{
zD~?<Z3IePbB#-O#FfDW~`M;Z0w<StqQT?*&lJ?`jpUa&4Sa_szUJ}n~qh(KmC)Iwx
zt@iqq;)d@!PD*cN3}??2x@a(C)m5%EWv6VejQ1~Mqn$NRM4sOqFJ5%*_qCt^?R(<q
zKl|&+gM;bLR70*70}&VDB(vT16Fpeo?)|?~XA4K>qSguXe~Kk`?`;aUnep#nb%)Hc
zj75Dfp3YHyb?l<vY4aBU`@0h>{3p*%7jF6{cdvVE0M}YsFURbTEx+9b|7b82xp@gC
z-~GNT*VOIY%=4R<9C;np@9ZU*wsW1>GW9oXAb(!DwveyIK!D}K+q|QkCt8bDc)1S$
z|Ia4XuXNCCMOm3&!NVV{cYajPJhQoQ2J2M?p;>Cnt}OfV^7nU(ZI{1DwOuIfnW$tU
z96D>K*4cBv^x5?eN%z(7=vI`qXm43sp~=VapfDk}HqzjH>HX|;rMI)r?H4}W&VNcA
zWVO_-PUpah{y{+=US09Qlifq7#zxGT+TA^~zrS_<{Lc3F#<_pxm6aE@IA}{u@Zxy0
zDMOPjXoAA#|NsC0|NVVGzx=<w)!*MA?cRTX|Np;#e!ib;U9TVW<NK47``5?+-&_0p
z{qg=rK4nmv1*OEQ&gRa}z_@ww0TcYAW<|%(_jhWF4w>Ta*%=%(Y2xhP^1{MZZyOG}
z`e}cd(zx8>XLs0j1$C*EZ~N>2@2ma&{l&%or>Fn7tNZi*aQpu6@Bi;Ee}BK7|Ns7)
z#2^1^7clpgKvTI<XLE0F=luDTXU}$aYVwYc4+#tmiI4Y)of{e<p?j^+&FtNlPyCTG
zD-{j~Jw9-Xe@YX}J%>gHwzhg7pa1vw|1W=dXuq=izkB=se!sf<Kfk<D&Hr;Bp7S4G
zAI~g1#f2Ga%Nvkq8mCU3+1cqCG$}ed`qqVjb7y=UJUeH0cX!U6J8|yZ#+g6m*Tnvm
zcVClSRH&-paq57NOhi}L|NntpQ~&?}?-V2&^LM`e|9v$-zrVZ;3NhQdKl#V|>YtxC
z-cx`6!xv3vVd2C7|2r>m1?BPo|E1n^Iy*FkIy(nBJ0H6g<PteEIATVyOG|4z`$V+~
zR<aj5j(DiPaAj>XTBG^cKI_qe&?p5V-k@EMf&4FD-r4#2{n^?7`{j+_+&*6L-|q13
z?L71C40%3%G=chPPp7kAUwB;Hr87QG{{8_$K@PEVy&WFdtGFee^@%yk(CEY@)ZNoF
z<F4=@QQo3Q4*u*LzA#?@J-xA!@t$OxaooNXi}JpQ7nR#?Z<o}IOR4xb=i%ey2cs`S
zjW6kJo;x=(EX={r@5l)cA8+qNCp>)o`esg@I^)U*`JMSp-i$LenNPXuS#IbnULdNT
z@s<19HO@)1_IpVESDLE*Us6J%?tj{j%E=EuB(_PLOXS^8tN76H@U(cF<Bj!<;7ly#
z(%C$Dva3r=xU1`pBQDpjgxtFl5)d)t;s1IMb{0$42ieXBMvFukCLC!L{C-1QDyKKB
zLq}zk;xfYoUzruWd?Jw^ryA7z{{R1PY+&%`=iv|E^$#!i=h<Cm_~!2E506+6zrD?~
zzm9jEA1}l@5s-DQ?d`##p%<=%Ts`jY8#{OM<jE6zdl~QPyZQ=bm~z`(P&|C^|HIDa
ziGn-dxp$UUNH*W<IWWuULJ<F!1PAT*3sN#3nIb!*8nY7<5)yv=pa1Z_eOtf0<i6VE
z9d&aPkMXpXzn6G-FZIXw{)f-cH}(o>K!fwak%OmB9XWPv)k4LLymgP?uV1v_fTG5U
zb0<$8IPjr=;zUNvTN}0-Z<xmZ!;9n2t_@P3_?1%^m?TUTQu6wu+^t~hE}+Ra$!3DJ
zj=Y_LP5rro-)4uqM0viyH@2xf^x^mS1Lon-Fg<bP;Gsh*sz={GNqO*m`N}nCR5eeY
zJ=(g$F`!!bzu_WD#nxGQ1)|I+*7{XkJ029Gu=;pM(UTzdoeO`7pG=w4$?PdJQRe|?
zYh&ZXyVh;&{1WTp({F6+efY=~<g<VOWKYQHLqc=Ik%K2szIwX!^|NKE2?|Q;$4{R=
zec-@>2CG}W)A$6gvfTUB@S~+^^B)J<GY(<ROOH9~CM(?8KZ8xl_pxJ$(_u!-LWfh6
zw&zRc-A%Hno%QelW83O*NxK?}85~Io2@DWB8jc)1dp0Xe>-~cTx#{Yw*XXF|oaqTE
zG*DpI-nPX_A#39Dh(PPr8XNn3589sz<A|zfiGFn8)Pbm@3r-ptESM&rvGU1N@3zm+
zCC|-Cys^3c;Z5VVe)*U3fjkiN7nMJnFj4T+=gZ|KAr~(TPoMH6A+60tnz=|PDYtIX
zQQ<UsQEl;Mg2uc5r>7i=+Pq-9!Gt7{R~{Xo1ELpbJY}DEr193jqd_|ry8rw={o#w|
z;f2mT>i$MDHYYwjVx9ZA3lbxX;vWeJJ~S~7E-bsYX>)K;*t691^n`>4y)BKV`VV%>
ztLhm_$!kPP>Z<<z|6ly7#++n{6U-|mk|q8dK9cbM+jOP2^}5;&ss9I$*#x+qRM6$;
zI56Mk-``}5>S+(JXtwc6OU$!LtoS`G&+r)3SHX`00tLT*H!ms(FgKg_<Z*IZ%Afuz
z3jd5B%#l3U<2NPZ&;R)+lvuKNw<IiPu}oc{;jqem!es6$Y1ZN$vMVEh$!9-2*f7yT
z;m!JZ$+9;I7G*t&_wCx2`@a;<gausXqX$n@O-)02ISdL4E(C-;NlQy>5MQ<9wV=yg
zM$Wu5DUGJu%3qZy?7R`omh+5f9oy=Y9Zsy7HoSS<(~fkR@*cc-V)bbWiGPL$3(QRw
z7_y_E{I7587MJv!pBAx4!s?x0HpDMmq71nj40!gujwvV>`OUC!$1jG1$G;iYPAGh3
z`s$8q%afv(33sM!<Ue-WXzug6*xQ9`4BN~mUMN#y2nuJqHf`z>nWa{3AuQ|eOr9We
z`)$zOOoqAbdvBiETzlf`Y{~z&`zNv_xJ|m}W%k4Ry#IyzDNlQ*7@XH&-O4w80z0U}
zU6goo;bLV;$;`)(7Oh>Yd*sLwhVu(HT-42&*;uV5sw#Z=*Txx3S<eOfZTZOG^N`CX
zlxG?n`?WmXX9sMqJ>$+%U|D;XQQz_7RQU{EK@aDzS2Ww2*(J^L({|L(E)zTf@#3P;
zlZmNICrx{nk-b_^^W;H>+zKNFQRW^V9s#{2e;Qhj*GEiC`on+n_KTxz%52ja&$Xt;
zOnJmOllOGrhezx?B^r2a7QA%QoF5?DA7i7yWfQTo;lZIfEFblkENRJ3Z{aT&(6N`8
z&?S(T#>UOfR#?cUs@i5*ETI>pAR_q$R8ukh|G%i{Q9?>eLRwl;;l<OZiZ*Qudh{r%
z<)~rJHd!H=4z0F{f<YJmIP2}5sLfZgM!`uau;H+<pv^u-S6ih*#bdJtqBcw_FqSl!
z#Iec3>s8Cn-bO}S*26ao3YY!=|9{~zb0@|ly&v9&ewflY$K}GL?ny$;69k$&I1XP~
z$>ZLCDdnpGIJYlKdz6rt_T>5VFQ1M5y*8N{Oh`=p*+1uNV&D(?opT!}%1->O8Y6a-
z<%vY9LzgsLj~Z)Q<G}{0-31MEWCS<;|9{w9+$8oyDl_vW5#==vE?bNrylXE`JS*ZB
zVdu!O##g}R5fc;7v}rs}PCUWOjbtnie0VE;`2GDGi`#6VF%dPpdFoV8uP%Fw8(Mnj
zyj)#dW=dX_%9z~5c!}}g3-<+Eh5jw_TDaBH;J3~UUv`~bzKE~=m*NfvG+di|@SbiH
z_o8#(le!%KHr>4CkgYCw`B?+^qSC~Fe-bM`wIrSrdEp=_nqC4e6#PyeJauZxA|)}Q
zBw68PJ#B4<lNU}LRaSm%AHkR0v{6z*pp{Wr=g7jD5^>E5PYm2Lz8%S^U!Zw7u+~6f
zs@sHV0%_kIR<wx9oe1g>;s}yT2$AWqVtwYvwnVF+aixW$iS-6Co(zV#uL&%6&3kN|
zD=nGz^$*|M%fl}hkb4&z-cF$K4ha?Q?%LGfw@pC!aa)C4l&VAUv3fQ(H8wW3h}J_a
z+=X9mwFt_Eb$pWHO8DzG$FcLuQ|G*i4^$Mx6qZI=aGE~+|NsArP$8?%H!&ga4Ga!k
zVpREfP$FY)fy;I=<41*C%{GJ?`uOlDDe<VP^3>Gu%r-N;v;X{ux6;!TltiK7Zc=Dq
zWOU+mcS~E;(IXKZT}8%*h6jV$bS^x2$mh>ELpPw}Kw@8_fl{kZ(b|5Q;{Sn?JRX|@
z^ftL&31ZPZ$KWqq(7kbMpg@zyQ3mF1M~-s@H(Z*~q^iDU9{02~O=h>C1CzaG%#hfz
zL!ziyG9ps)+S=qDh0PDo%g0P~(}6poqvJzsYvj=*5nY{y#zsa5_wcR`7pVMbuj9MG
z)bjt~MJ$mM1a+?jBnx^6^J(r>Na6?;U$@LgIqFiQdG$o8*&1?=FH%~}JtQaW<q>w7
z(H6pS@aq5LQEUlJKB+q<@d*nvD=RyvsWD4SA8umhDSszmCSwn^Tcfa`cyoJu^s!@6
zp5C7iFY4voDv{8z<&A`-q=IJi#E<v?|L@tbk+))7^D*;z2b^R-J8&pxAHULa^O?gW
zC+!;-Oq8ax^Q?$5Nq)#RAzCzJpR?b)7pa`Hm^)q`ciJ(*h|RZa=1fJ+&c@Ep#9Jn9
z>iz|7>V|NuTUutQsYGaNa2OaE7#JBXoF>x|sJY}X3y+5=yYiYhfBVI1Rzyz-v}(!V
zYdESG)^JI%HB{^bD}!>@oyT5rC&CnV%Xk-yW@gM0^tD}csv%tOe_|y2JHa^(OTMu)
zGc$8@JC~Ro-e1pidAYGoz2CngXNb*nV!+)`V}nA6{DilhwvCO_72V%08g*qgPE5Jf
zcva9TVp?)#La@v+jrZpz^q=#s6<shpG@~Jz{e!_F%jWXqANHtqm!AzSXZiR;QGA=2
zV8FAt!E0O@c%lS9iJBBxE!gq=g;V17hti>`39)r2{BQkmv9jo2<I17Is;QX%<}X(v
zxYL@X<jk+FJ-N3x$E|I5UmuIH@t$?ADTNV2iB00_6GWK(#XJI8-FZIn8Spr;K2(nF
z={fP2X@$rUlO-1o+<cq^-X3DUAo5tqQS-?JnT7&ppE-gjC!ToM@+tATpW<p4M~{0h
z;x1e=Z?sxlB>w!7xN}#srL{BbP6D*pRZ=h1(Bw={VUd*b+O)~&pc0R6>H-zPD<Tat
zM|f6j?33BVG5@eFf4Z{w!cv8eqC7n}cFZW`Pfzf5j+oP{c#eZTBA7o)si9iNSo6T%
z9&N`5UV0ghD%z(7<jy4|95`^Y^P!>=B)F9<3p=_#xwu>J+;O6~&`_D3G3!)<lb>Wy
zk57PTkB^6A=ZRk{_Bp-tOsbODJjXr6)+4Cm6lYAXf6S|mY6?E)35tQiHaa{L7^J4p
zIK(`&d;Zx22M(M$vtaF60r~R_)}3oGOY?=c4V2cNJ$U~7ySHoS&&%WUl#~cO-Ekzr
zN$5Ab^p9WMf{wEmx?b!!<(#dTz~t65r7@KE+CL7*10B|p3naU{U5wdP#RYF}Q#he!
zA$$1Pv4q^SA6^K88ujJxCyB8@{i0j=;?=36$D%%eGBQ+n7Ou!}^Z$#PZH<kcjSu;4
zVkR7VVP7Q8lqIms$frT(%YVaNLKzJHy3H}d3$2;i6H*S`b2&Q8=s}E$ARAl6j|0+6
zdM>y++>mXk_Z9ebL6Ujfo+C$(K6rAZ;4$0b#qLYao`DqDN(-YVP7~wdF)=d{as2eK
z{>nzRCxPB>71_-bKiWs#WLO_I;jq+7)sOO~*Cv)K&TiIzBxxw};B|=r<A)IQ6S`U^
zEo&wUYD=$;ZM}KvQ2a%O)sDuzzPp>4oE;dN7b<4nV*^)#N@j%xg&V6XGAgT9ZQ5+C
zsLqr!-@w4&z;D41H;zvGJ@M*+y^aRYHcS)aS#i;!=5FJ1sYA^Y1xL(h85@KeC2SCE
zI`NjDt;eKM$7zzz9PY#!vy58Wj82>okagnWfd-ygp@%1DW%X+h&qC#;Y%VzmkC{(g
z{r|s5_kV$gi68B=n2!61Rwzkqwm7(od$VYh<l>CLZqbC^kUyt3-0Da(H4<-3c-h?8
z7;vz`bm{{y`2s;RGd6Q`Hd|YvsAW(qv_PI{YnPUi_TI3`=s*{5MOFiUkfJ;1uUi~F
zK`fJ8IyCe-7;FrCP1BP%eatwREX!=hHcKaL>WOU$VN-Z`ZUnI%JjEJmC?ziG>0uxu
zI`Kj=G!nE5B_uqmD>Aq_Hy&hp@bZB@oA|~fY?+4-)-UPR3Rs?OqO)O}@dU;&%`;9-
zKNhu0I2cqebk)6fq)T7(!4`?da}o}CnC}Ql%B^&`F>4l&o*qw4jmu7`*>fy@{J2q5
z%WGm%F>^*hnbN_V9)kT5s;yfJUh7}lrJvHvbE2sw#?ePfLI2lE)(v8@JQAABn{OPr
zwW(3EaWO~YfsWD`Q@%}sJv}^Y*6<uX$}?*gkBm&qUQnw2|6gg2#hNuDfB$Z^v{d}I
zz+6ATe4V5DA!YrS&fexLG@1E2^qmTpa{3r=)KX(h(B=7{aX7R>AfGjEX~l6LPBT*j
zL3Va$7nbG)3W=+s?ww%a;<|Is-rm&InFh1OBLrSe3*Qp*Y5qjPTz?&wgk{WY+7@d%
zBpW;kXmD?_Y1kikz(;k8*-C*l!HISf1ydw>Bsh<r>~xHuU+@rappUQU-n~x~-W`~y
zbKzW$OIC5hy05Bij*L0xno+;o7>}t+Pgp1(9@(`2gTv#bTi#KZ9R1{ZEN-nm(e+V9
z#L<y0KmU*}KO_+xnr^XVNkvWV*CR(BRQp^IV)?Yp@XOvL364lE<9&ik_5c6X|NnP<
zbNc-K_5c6Ay1IRP{{H_zn{Dp*OzZh@MRN*63@=BM*2jz%BZb#T@9$0i@!_F;it7FI
z^N;B5Ja9SsplY}1#!W^Nl7=&87&x4>I=~9G)x2iiQq|`V6^}AVe{xzYB4N1g<l(Ca
zgk_ZPTwF1+YuXHjiNE?AAO7E-zh6J5qUz0!!?xA$=2(4gnLXRrug~A#|H{3%YnOt8
z{QBA(87(a~h*|zG^bQm>NRsGQYr2>C^Z)<<`|El1<Nxih`T70l=l$zqe}De+5;PX1
zscFc@X7u`n!HLtx1%-m?=4TTAaPxkAQ<#*JGGSukhK-39Wjy!8K-rw(|NlcXEm~SF
z=gghz>hj<yo3fkgArAG4b2#*!Bqb#zHo88se{!Ya4$IFUACI^3hELPoUGU(*-}wFi
zj0_C+)c=2ca<c#P^ZPAoer)XSuE|)jYUWG}Zf={c57~FjGCC2nWA^`K4P}YHJB~;i
z`u_Vr|KauZ|F`G;jokX`_O-S4`FDOWTLw+)?0opdv#s>?zp5&u)^=knYvYbCp-*B*
zVwUVJ<k|3~X{Vhe|AQwf1_lWiE+$&k$vh8IfQIl)3lY(yZ{HqG6lGV=n&sdv@8hB5
zWAK&7gWL6CV{6QQ0|oaR{qp;_=j^<9vGBvQv)3~`|NZ`MAHIIy&iem{Te<a5pYnP0
zX335nGbWU$33ITma6OSTY0CqxMhU|s8<)@f|6g){-QV(OXV%Y}<)423pVPe%rj@>R
z|59%3_;`P=HBbIM12;D#4o<<oC^oh^Sr>$OZ44v>-`6Lmqy&VfUbvoaQ7e`H2<p~*
z&mKHU*|ufM<mt~-9D3Rh8Qb1Uw0hAoQ82e$>RE>7Grf)fr|a3~XWgoP%vN=(p*jEl
zKN;(9Yj*s2FirQnUG1;dUCiI?*$xS17!*#L)p#V_Gh((Q&;S3%H9tP&AMca*^nC9>
z?@w#??u-Bb^MtQ6@R|9k{&3rWmwCb6TW>aM_ATFi`uOpLv||d|-It+*s)y!TsHn2$
z<R0C*V@AMZq2e!}<&~BHOt6o!7Bk%Wf1=Q-dbYOu`}_ar<lirU&etg1TlfE8`R{M-
zpFS<F{$}&*1LMwBN8(!^_`98$*cB=KkiErvb^N|R_5}~zU5~c6^ZysR<5Be6>~It7
zk4;9qzTS|U$K%n;6;&)clh;vAjc3*@i?g74hX4N$O|qCVLn1Y`b?Vfg5fK+I9jV_^
zaa2R`-$M_L9aj1QB_AH_Jv)2-|KIKwv;Y15{r}?P^Rlw({4y1v-b$zBG9BEJt{`&u
zhqhGA$&P}h-#$P8&&*!uGvmY0cX#cNd~yHxH`$_YPU7#fHuHQA?KXxrk0odD#55}%
zI>eKe#gmi669Fl44ow0TxpU{Ls;JxuY<?hrY2TCj6?gt;gmNBp)UN-yWRiXTKfl>!
zf8NJG;x1_0oF2bErt<lfmD4S_inmxCzWU%mPwbkFePLz~j3n;~xi2_9y*~HuueTQz
zg|@Bx_xJRMT<OET(o5zpZ;*WLl9ia0^x;!dK+vIepezUu02Y=VrY0^sc1lVF_`P<z
zUD~&GS;G>ZHJ*&CAAn+^{N0|NH8Y<Ci<G~s*;Dj%Zut6r|Ns8xk2Y~;n75}d)Nol`
z%uAknrDV_`;NRc-t;=d|@0t0;ceQ!GQOquvLuuh>1cTU|SlNnRUA%H+=Lu$T1@Gi9
z<Kxp)RJ6y->yO9urGMT1R5OyFY%qGP?-u$=e44_)BsGpF(~jTV?5`jjKIuR^|NcIi
z%8iAOo6pa$PhP^nwugr;=VRCEl0*m5<01#Q%xY|GY@9gp<BN;($NT?ReCtVB|M%C?
zf;SR}m-{byc__3f!OZl+4jmnyZQE|NM?i9glfO((&Y5Y`gf3k=^I@NMkntn?ZFl;M
z`dPA?esr%r;9#l#<^TWs_3`yTe@>obyTAN>-7K4*uMW4*|NlSzK%kStamk{<Zf{;G
zhn1m)(`{cIJqem!`2T-@^6~A(&rj@0dwxpPs6X-Ge362cUp^*Im~tdZ3g&egnKh?Q
z9h$)W%254Mfr;7wLs^seSKMA$Z|M8`^Yi!n>;K!=Dm_0i@%{b(ar^${-QTy|e_qYO
zSObHE?pU@QPNtn6heJIRQxCf?cyW7sefYXRI|?3N?-oyqv#)v1XIXvaLx|Fkokm7H
zmo7!{Kx$nle;FgAJ5f<gH@G+}G@I`1amZ^`aQ_o8)5>1u`04w5c{Q~*`}&TIQ;!*r
z_wC(R@=}(WnfYE4bGB{M&AbCk+4`^8hp_VS@cg&0{Z;<sL%(_ciTyKv9p|6aKhfp&
zkw<~eCsR{WHf&6M@XRe7RPp`)@8mDDYuBPrpL8UaY7{9Qh%`9R#k(ey^V+xL_I!M5
zadBannixf6_WgPH;^K7sItMMaH7A)?bI1tH=aUH7njFQLXuPNT`}YeA-@m^XSd#to
z`g)b@gI1T0HU#cppxMR{9nDr(S7Hi}$WNb?YHBR1l_X5Ly{y0duYXd+l-<m1S?kdI
z^wFY><^MlFzuzxc`}M^|@tljD`wb49`8z{u;?~D2O9H;^tN;J})z$d;eF<T<Dh#Xc
zJZ~-iBa$8M5+vjeEk2z*Wo&G8qN18sBsN|<;oz^aVI5ag{!{&BUOaCvsBy^4uWn!v
z%)VC8us;5O%<gX=pPlVjQR8?rrIDu~RMJJv3>0qn_y7O?>gxUY{U`iooSZD!mvyj2
zbaOp<8DpXu7RDAD`la<a)H9wkMMZzAs^&;UvMjZ2y!OVyPwD^v<ZB8UynH%MMMZmF
zy>d!vKXiXTC@N}SUaD5&VqWuzXUAn;=50)m<@uNeALZZs^Y_C;`O4*T4h=0V%{we+
zR&D$izHybKp6iYs5<I*YtwA%e4FCT-dCFY6)HG=l&kDt+&AObDZ=Bv8e<b(sP}ETe
z=8&jaw~x3qotcmT2C1ogik}<5(>NX_p5XQOu-M1{9U8_LT<z-r9Y5xFk;ze2)kv8A
zV9LRU#TQQ<JgA`Dyzd*NfOYbe5fb|I=FR^JX+Z*8nQLw^mxer+SAHD7#H2CO?}3zs
zy9X!d#_DSANy0(~2M!(j^5y0B<^KC0^`)%|?33BLW0vfRsDFvxyb+f!_sjqP@#BQ$
zZ6>~+UP%|16IZ-9tT<q0aQ@7h0|y)TtrCIcu8+;Dw8USXxmkbZ^{ENck{vVJuBdHL
zX|m+_Ccf3lF?-{`)Px_;UumhUxYTWa{Qmy`HEVqO<!X=4=KG)`^<>K-hb>7pQ$^(E
zf2y+U>o+e~Z)C0c!FX#I?;nPFGk*M(Y-tT#0*xpYK2A=*)YNPB|Nk96#koa^Me6AU
zZiWf$`zA~}Vc+wmvAa=*u|6_FqN}T?qT<Ax%XL3Lxx2Lqe+^=JvzAdQ=zRR1lT155
z9GF>>k~&jG#lxPLneEIOiJqR0SvH5c9;BaqsHJ{EN~VA{DT(LPC!RGP6XEv8#qk*$
zx;?qZ5XIZn_F&TlBj&9|KfBjXI54YZ!PDT7DYG^kze`oGU<8e|Jg$$7<Xgh}dd5!4
z`o7~m)i#+g^xHT)9DaX(K7XAaw_$W!LE|2qDH2Me4?IJE2+!%@XkMVeEF2UI>YD%m
z?_^+UapT5~e~Zi97xw)B|Nr2-YPXq*ic7crIDX&*`}d5O^{lM(u7wqUI%f0a=}uOb
zh^0#{4IT+Cb#n3T%V=epx1>4J&@jQ<JYdqypZm9&ZSLWjJVV0rEMImuo4&r$mxnAY
z(b5H}ckb}`_-JH7J?JN+^Z&oGk%C{I`*%wn%|ETV*}uEjR)1oTb~-pKBS1>2r|18c
zt*p0hpFR1(Uifl-eSK&C{kr6y;}6ycbC@X?T-C1+6kUAd#EBpOkN5vqRFrx2NJX8m
zw6txT8JpOxd-GcP9vtNq7WUeB@W276l^-lDC1=i5^iEk=o}D~5;qZz`gKE_sv*!Qb
z$hm07%xTL4BQqT?HCU;uPu{-0WrKnCCDX(V-oE-9J7#Qd$k?^v+vW#fMXo8aXGy6)
zug}g-<K`9<6cm!;6B6Q?I+f?zwIi<&aBc{b|Iy9Mp{dfc2-F4r|39h8d-H}rj~^r)
z3H!5%XM*!8iJZG3F>iTdjvlk`x$*1t!Gi(8i|^maaH;v5l&1FNrRKT?2RbJ{-ni)h
z4JC=SXAa~9CCus!TldHL>ev3>xAmT!yO*gMPdxsku-JIZHX|3;iN$NA)eiWuXRHm0
zaaOo=Nke@Zr0#7};|mK@b97Ym^JCk=ADDmQFZ*h*4Q#e)+#;))o~Sgf{`j%KyL;n?
zfa23)>WPU=b1l=-+?197Z85177f&=ZW4j`8$cHyVm@8~~lCN-t>x7#;C%A6h_@}L5
z;5YBg%2lUaHk6(`eR@NUrLz;0>)U`o491!Xx*jPh0acqJ;hof^{q#Y?qeo8N8}>_x
zUt74<_``JhdP#|Y2ci_~CrbREkg~BX?w_Sq*L;hIfiY_owNCY!_HuJ?Zf)_{wl$ND
zt?RIuQ?jn&g-4(4%~CT|86AraN)tWHa&rEgno5f4KKk$}>Dkkxa_MW<@Z7t{!_IH?
z<JY3=-$j^gjxsZEo5C2VsoQH*2(72yMDXxUn{O98bE;@nj>el00cvh;YHDgHzL~1s
zcp2*BaXiBCK%Y%|gYH%LxDEZ!thPLjIKlJe57*OwKR>@e+Wr3acKhgUZyw&<{NBF)
z-{BJ{6x5Erd!QgBto-<y+Non~;XPuPcifs?!Saxm!SL0_Ywj~%zWKku9#paKuK$1k
z?d|_^w!i*AbZ)EvFCif*siG<w6C?5Hk;Iv^k`)yloq3lI{QuAM|DWQzQYW{HBk_9@
zK7327_#=_U0V(Z{OxwQw&?Hr^4Jij(=5RbX$)wg-cTB5=(OAH^FhMQh$1{U9Dp98!
z8XF(p-(UaVuKwTO*Vp6g>cqmr+<J0<yST~KIVPoWs&P*_(ik)C&)?tgKR^E;zu)M?
z=gTUN3e0OX8NxR;oW8gJynxiH4=)6-fCkJ#6~~s}|NbQY`2S!2k5a;ojWZ@J7Wx1G
z|9O_~K(z>~1Sc={6Yj!~JtaSU{e6OAC&xh_`G6xwzRaILUvk@Bq54w}-CArxT>NY*
zY<00Gvi>D{Bzo$dIrE>DMS_LpL|1MQ!?kZL|Mv0;-1MIxkOdvLb5iMpbd{&ip3SHk
z^pNKXH^0L7MUHw52@R?f_pf-xIZfvOM3##}EP7^9cMJqh9bxHaj43?t;K8_VOVi1}
z+{NlPGx->P_<vcuXukd80!YK#Nzd}%zWV>|-QA2uBBpDKezNNu1k4usFPR|j=<rJ9
ziuGCpPm2ROHvbk+ijb^On!)Ivr=}*GU^-bPwnB(~)|CSclg!*}f3*|^L8C*Z@BRJ%
zd+Ps(N6qSNYy{2UA8hat-VogJuzcl$slpjb5uPRK$`gA$RyH!ahZqWUm9F7rHU76@
zR`XRKpQ$^}xOmI_H@IWQz~JA=xYF7n)xOR}@;M~WIH~mM$Jgzz{QUfXy;wvoqrSw9
zfM90ktX~IwRC!v0*rFabPE(TzI4lyL*{EGCd{ahIEmh!3-_17%eB_xKE=*!gYZtp@
zbd*Kje$kuhQ17est&gwYU;A79+O~#ik`vN2*O;E-*sVR8^PYdBtjXz)kmUvmCe{VL
zO3LDLCt}u|Y+_V?{O_=uN7*Tbh|>Z-mH#)W%2qM2xo)7ad1K#vrvuCVH98?>jFX;a
z-QN2D{f!UxnVE&N!dY)^Xy`OPBqXrpTVexGk;2_`XB)Pg9ek;&a?D=EY^7kE_N<R4
z_8+e(pQw`DAlbUQ?%*`TmrNVZ?K|{)^^b$g{dx9JFnbAib-#SQZT&y?XSPRjcV_ek
zM07QNFE!k4#v{?ha<D!`XlA#V;s3yHE;gki<9QwG`?efdFB_Z{z@T)QMP^S`Ymtit
zB=e~B#qa-TTg@hUzU}_0Z~U663seQ!Dzp@<DwyZ%Z<v^o$rOIZFri!bk*kEq`s@PJ
zDz;7GO>7=s!UZ4R$$I63##;XWcbfU<&xeP<kN3~{^M_}`Gu9x>gC`nFig|*%g=X<3
z1o23?axk+=-xXhx^CYKzrufEV0{^-iPJGj3ewSxp_%dyS3b#j-f^y`a|K~rv6#nI>
z1&wV_nfLeqSN;3LZ*R}Mq_WZQ(7^^1p;JdroM5n?!EJT!V1v=&CY}$|W&Q_BX1r`*
z)cn-ZuluLz#Kq2d$&X^ZVhxsheQF({s?5xw1v44$)1YDJEn`35=I7dV=Z-X7y5*qf
zq;QjaXHlcL77q{4hlK`gGeo4Acb5kUGcsotB)h3hEIi+kJf&$#i9=6sv+M#@4mP$o
zQ_sD={z4i$edpvUGvEGy-Jc)g%*@)%hyVZoFL+q`!^D7?WN8V_<`XH5r3PBe*(V)S
zwiqUKJH=!&MGE-pGiP-jNj{Xo{=-}FNyw*Prd21Qj`Ng>-}k5T#|QJ8U%X+P53FCV
zoEajb&Aj_};S|Y?Uf(SdKl^XR9GHKab6d2hYqRWxTmC=)&wu#T`^yoy0r&U+-&_9v
z`2GDh6LyN`_y)2?h<=vOR~MVyUBD(Mo+Kj2BT>r5eD8*8s7&@s<B-X$*V<SoO4a{M
zsrbh8*Fq5<`1Sw()PH(%eYW|36_pEYo{|Zj43`xC9zVb!|Fy}2WoP3d2I0oZKV*N)
zPwky4GC?w<S*GHVTy7TgeyKlhf?p0zU|%0+Xj9QJ&jM;<{PEvkUar5r{eMTdv9W<e
z+o_4+`G49)B{EVunU^p%^`2Ffuk_m?{QLFw|HA6~v@|!KJh353g@Y~VlgYn_Tx>yE
z!faPcobx|?*FXICH_!inh8o(z=_d{47&O!5>{52rO8=cu1G8$!k6&NsZ%(hT{r``D
z(*cQXKR?>%JYs12XJqi_@Bjb$Ifhm-ALRQ?U(7h|C#WQCzOTQpE+F{xo6F1Z&)Al*
zVWU)QX|D0CEAc$V23r@eZIat?>Cvmuwl_I#|LTPJ=BV%=&|WaxTymbR^ZI^>jfXa^
z`g&^W{P*|&|NhQDOYOmJ!Gx1f<+GX`ShRIi74tg|^%#gu{Qv)d-S2PT54Z1s^yo~*
zVtM~xzuN^5Mg*%bvy8ZFcHkrv6KH<;V^iYiG@&rzN0I;ASMmhEU+NvfHw)4dJv7PU
z-TnW6KRwN7)7dTi(6CEx(+8tx|LbF{b2g^G`~JRuzV-LsnKK>z`+M8l8Ql#VRwo>o
za!5yX!!ov*r)Lu~BQmm#Hf%E5Q+KxDkA<kuVIPedcdEZj81$z>Ix2@IS=`(I|L-&2
znrtSo8wXGEoba5Tv6SQ5h2ygV{vG7qU;qF4w%qx>y_P+_H)qT!nAxyZ;y@Hbf^WqC
zgsV#HrkHn3Vr3~PGP1BX-muB&&7DpoUoW8p!P6EjEPpRyS9c@{?u&hOe}8{^I)8ut
zL~qt9E)2I>JJla?F4_-Tn)v7E`9rPp_xAmL{r>#gB}bZ+6BE}4Z<xmD5@yo8am4|a
z!a|-sdw7ob8Q$6Lz9ZXfgV{l5(Au)v*$+>P>wuQlGJpqk_SOHd{{HSgzx;!P?UPPk
zI3?6`r*lrjzs1M&x-9Ge|GT^4;pb0J=jY^D{P}Bn^Tq}-`yE|MHe31sUHL42a%;1c
zhqv*M9|m)*k8j8p|8+w|&*5Ip?d|zI>F161)b+ZGfy>TAQAL0D*Ze$xdism6tZBSW
z+b&B5EpnK(PJ)v|(EtDc|9^jff4|WAepJ-G&JKg;Pd=PFVX#n*<IE+d@LdO^<~-1>
zW2>Ikz#JFH79Y=MXV*5<SW+)mY1M}Wh1DFD|K~kC!ue*ZIK-lT?C(!buJ@n!C$XV4
zRADXenH>jjeQU`%f4tDcF64Tr@O^FV`*r{RtY6V_{o1tNnUd94nU()1XJ;MQSzmBP
zKjZ0<37%pC2hN;H$UXbu{rQ6bcFx(_hu_}*kvHMUe5L|bagN82-P_vvH)cadN)GLt
z`S<hl{qg($w7&6AOi2MXaxNVZyEQ?B`Iv#mrwy9StMuCH|Nnb``}^Vz9}aI$ulMr)
zdFYUZrR7Dp`3txbcXh~IPij!rOgX4HP2obr1_S39nZvEzJki?>3JMKNN(@e%NU+@7
zyyq6nBJQG2&2<+=pqVJH=+CdO|4&cfzrXHaw(uFf`}H|FE~-tXD;^(iw+~)c<L6OP
z{q<G8zW%u_CVF*sY&ofH-;~%x|Je8ZX%DI6IVr`$Z(x+v(VP0<Sz5&AwuM`!JbV{>
z;+V^uu9GuDfBu>I@K<Tu_xBr5L(^egQO)09@88|6kKg|<J^laj0}Y#2NSNCGZJ#bb
z;peaa{}q*ZdU_`K@aWpq{r^}0?alkh&HM6heR=%$c0CUd&-ZtKA3jZW=yEj?D$ZiA
zY?#B?*=t!~R|9IShOamJ^Y1LEq}o-=Q~u7t#ME%d?#I^mCDwjyy#HZ)zNFvWLzU;O
zAhA2oqV8|fj)I5x`{nB=hL;|=V9)mD=Z*gxrpo-6lvKPu|H19FfB*lS{P5`L{l)I}
zXXpNY``}>wgs#aGCpv~t|9N+}{f5mGF9<O*yZt&4x;a5@aUyHozmy;UWN$_N`QHEV
zRcPBJp|-EDCHL%+ICDlqMKv%>^2e^?nA3YNgq0c?7}WebQ}EyJoC!4QD;v(;-u~ah
zQt|n5!$aa_3Txjb{^6gyu2p@?{}r1z@z~cJ*;ITezrQcOTda0}$;<64G$t=qef{@$
z{e0Ws-~ZbmetCI+qAELcWM}e?X<{cHoRe)6R$nktT>kul<Htdxux-3N*Vh|sXd0e4
zZQRjmSWpN;GiDo~IOW6_mXe^bsPoi;10Q~yAO8LQ#qZ>akV@^)OpAT>|G&SyT+hh(
zgHN)8_Yg~DHP7tz?o*WF_y5~n_xJtgbWp?o|EsI}rOp4{+4c4Ny}kSO<Ns~mnyRb+
zyz1|-`j|WFY;0_4X>5^^%lzm6D=Kn|+NGk(cV^j(4XpM5jqdDXO^uE;jEFMQ&@_~>
zIZ^P_=<v~Q9>`)98ylWGcX;&Uja=NF`M)YHd&(&Ja7WeDhlkncWI<EzOpAH8zsnyU
zinp{>dvdeV&5h0N*#3;OH~wGHp73AZ`djs*Bk%L?|NH;v=l|E&K~B|IJ(`kw^wHzP
zX~~DR)Q*M3{+&8iG^%;i27`ivKgXvTeOTU~cu821%^^>ywM6yDPe}_)$&ykKVq%gI
z(@(P~Xn6S0xoy6^<h}jr760cwJj4o`dR2UsC(*EE-h6w>b8`;bnQ?)_4^$w|w3snd
zQqJ}ld$x~5Uv6T<7W0X(q}YBiT`GS5@7tFe>#8sO#>ULj(#rb!i+5UfZ#K>j5SYEk
z##s3B|M&O*|Nrw-KGX8#e}4IYoE!#^Jh*ryIGY%!+Wt9sem+n1Hiw%|P94@Vsi~4(
zT@pn_67y`6cT`M#cviYi-d^I$)xx+>9o!CKy$@iHnQkE=@$c`?!v{{C`tjfX^8f!I
z^0-r8>l8IJKl=4QEbNS$n$Dj;JVixkmMzmUGV<BLyzHN$fq_lUkMGBN_ZNKC((*}4
z+Hs1Lt>;n`W2a%zf!XFf9}aW8zn>lu+PZ1xOvxQP!Dxp>ORHqv-=rVk`K#90ddA*d
z?*H+sCJ$`(`^g;8azX(Xb@q({N7Ov*9{Ja-?f=NwSQ55qUaL@`iQNI+^|z8Nb|l{n
zw^-I$cqKJ3TQo?+;3$i}y<zC9S?e|{O<7v#x9U{K9KX=cLfJNEzW2S~&wSoD^M{f$
z?|;>an;Tc=%I{+9Zd%58A!q63b+_j-eX&)1!jl8Z^)oF%)mHIyd+FQu5%N#$-(2AP
z@hPYxdrITPiGdy+fe|w}uSK=p(dlH8v-|V=m1xL>nKK(_&TO13>pwer#`i$ZruzR9
z@9re-C~Srd{5!g`F85M;o-<L@+2!5c<b<?i=30>c{h@NPH?z%oHrVF=f1oUrA^ldd
zVd6x_vR&MV>%S;7NJvUZ+}r!R{?`}zu&|8je-F3+U$(60&6~w1nbbi`m%rs^WPbc_
zfB5zF8}C`$7>wL5%(qSc@wNLv;7@4cFaMz`FQ1Yix^#9!NsoNWEZ(ZWZ~p)1F<?C*
ze#ywd;16h0;Q9Ige)IltOLiuwr4^K=rKSGm-<>H_d8anztNRCrw6wPT`vTboDk>5M
zvT0sg4NI0tn@h;qs}*botxo#?|Il=cd3LSpC;xw#(pa{Z^Nkqy7H_8ii`5p?hp+!<
zSt}J)_wkd_i4z7NKK^x|)ziAXbHB$MLEe3}$vbML3%_nypm0)rhBEV$TP>hb0DI8#
z<$L?p1D-%VFx{f=Ps6gX8G`>O=$Sa4IhX7!aH46x-CygPANn<G-^{n?*|m#DNa%l{
zH+PEeVuq?R={?&&y%u+^7MZZR|Nnn*BKtM<!=*B~d7y^eKbfN$Pqdg<zH`v@PI$!g
zVbRmKx9hj(|8Jf?Vb`u6BO{(SZ~iB)K00A*KNHK34~$avIyyW_NfCKH$5=1Snm^-B
z{C<gbajJ8m&F4eYEhHo*>;4{+>1$0ZuV`($a^=VI6C9C@M;bze|JHwdbN^uTg=M<J
z!cV?@{r~^p!O0ghHXUHPw>P=shrle|$Vkb5e-E2a>13O*JbZ)S`J!UuIkw&vFO49H
z@p|;nZ@rFAOo@`rUj9$)iwc`=6(_pLt^M_Tdg9|_^&f&AfBckmagkWERPx4+2bmcn
z3!3-WOWxb>erNWVzt)Gpzvuap|NMZC?uVGOC7;!%^>WOBhD3PLpFbxHp7ZUh_2~Kk
zq11N6YO@R0i~j%r4|4Ei_5Tl4Z%o(c`SpvZrsn_58E&)1Ee`(w&!eNmv)oV7MMQ)n
zLop)ifNxqt!j3ZD`zbmyP-CZC{QK7`_woFJhV_x%SAID7%lGvB-(UN?{GH4pW-BY6
zUAzA8+~GEv)$YS<@xxoQc_vK?(36)xyxjkVbmT(@%dC!_Zjax`>pb6*0P4Vk3dr!a
z8#XBT6&_Djz0_hihr3+o@&5Y%{BnQlzw1pof9%+Tb>}|3m){p?Y1L5wU*g}t*0{{J
zdXbC;3@Z2cS291X`^Pia^B|YPRrY5uES4F0Y^eX;R(DTB58~WCbFH}=3<O-{Jvv%@
zJp^(d|MgFN%c_&3!=mcyw5!AHAJ;^YM{^P+<W3(D`^?r}Tq0X4JN3cAH+OGk>DKzs
zdY9_u)v`dxUT$9J6_bZDr<<lo95yiMJM;OSFUO{yMGID4kXrkY<5zvdp5?jgqZWu%
z`f=pnZctY$x3vqK)UvE|my9$xmbc7R<ZCeCag|p-Eh58kRP}=Kop?PTpNAaY0w*|G
zT6zS<SVF?Lob|u_&n%Zu<~NIq@T?syJ2yUK2yNXuUAolQIre#&!v0B@=M^5Y+>;(z
zz?W^<mgwL$#p=oumyW~tH@R@!nzqh4``0`HjjsFlOXdhMT|TmFS?>$s^McEJmvG$O
z?AkP=^4Rb2X^;|d%Z!B}=eWerICxIaG4Y4c6}y7@CXLcU%v?wLdr~?CIe0jj&TV^i
zDNcW<U8nNB@?Bp|_OuivvMCzySZv?_vt@x!{?sp`d*XjabZ`A)d)ejs!#j@4?%j`C
zEzqOv+iWPp8Wnt9qjieLSMj&Jk#6%YZ~J}c+00c|+hdnax?*|h)ogzEBTS%$-G`=I
z?5j)NQQUs;Ebm<-3x$Xxo9eb5ljnAJItER8cwL{(%&e{ce@2no$@}|x`sIxcjE(2m
zo!yYiefaBZp8NZaZECw#?CCUr{yzSN!GT_B9`ihdGcyi8czpQ5n==bmoLI2xl!Cf@
z?eW8)AsqL9qne*hE5ep6-Kfz#ook1bWQMPR#rKEKN$oz6#IgpKSR9YoP2cd9FKpfw
zVUN6o_q;(Jlk_HRx#$$wBQG+i=w_Bg+g#nU`9~%$bIIR+Ui^ChL$kbdn@W#!wKRNJ
z;k@_%&sq(ZmIbe0RBu_9nGxM`;MQ%<|JrjFuM6zjv_#`X@v5w=fA@>+*kHob^Ehsf
z+LAqKKlIkS&D(g;vZr$E6NWT!ia9jh0#uNFVA%W2Sx;qx=bBsdZ5&1J`uKpny<ziW
zbqg<sluJK9Cb#|lEh%T0Vo^UY@wVBE8yD~IZ`nGT%dRFNVsqQWGm>n&x^2$w5>C#N
zKE9Iw{;3++xC$jCB~^TEdiWyn$OqLW|NCdo{m`Fcuwa@@0cbtSA=aeTqLA<ax#6$#
zj8F1q*}Ba4J~1E75My(5V{>$TQE~eJe^6hLUk=oWRnYNiO*`H#4q8{dI?V9R{qr9_
zsUCiLndkjI$GN`2A099sZf55>b&6+qx$%+X#w~3|28M<wP6r?Vu-sqr+}z|H#q5F4
z3JVNYtP=98-@&%;fo%i_Xdl!)+ay`Yc;1}Z8#x;scwFZ<ttjZ-yR_S)S$k(sb;Ql+
zD7mDC*E<sm#k!U&3H@2Z*H|-Y{)C!|hd<9)Cavhl#PwN<W!?SgY6Iuj-{<>teq;D0
zw^ovERoAvunY)&E&E=+=b$JLqlDMB9J2Sm!;<P*KoHb8P+O<r>Blwfd5BsIjr#B_=
z_!iqvQ{Xviq<;6wJ-^+0PGzRbN5DG}=ghI?b14uxQs3gIDYQsntwkHN#bWIX){LE9
zVIhkqPWa)Kv+;kD_|4er#_MNNp6xyPq_!#Jex}s3NR#mMn?6>pjk)A{d4tZxgUnq@
zQeR@**Dcog{v|nfV(O(S%M{OdDmm_pmw7T%L6^I&@%`gnGt}A|mvyeXV`A9%#WvSF
zHh5ad)#)=2H>e067h!dDm?qeE!)(>B^Sk!uo-p!snzr}hbBl_58y%`X{PI3%(Jph^
z>Gr(}(J$b%)iY-!SF;08n?0YOIr~F4_T_r9bD~~7{*h+aa>4V(rG?I{)}`-NSlROf
z9=hCG7<BO3Ez=f<N%kr~IU`rx-zBG&xoO`y)l#K$nJH2S6M0-6XKjjGY`SiDPtWxY
zD@}8~BPXdG&YAd}W8<6UQ#4w0lW)H%Yf*G;c+Y>Ob&B$i-}BZTmgrgdddK~pF(-`p
z*k`dR?sNVi`sH@7McVGC6)Ig3Yd_db`Q@7C-(w?}aklRK%OhW{o?YE^BWIudgmWJk
z&j0&#ZDi=2^Z)lXG0uZ{XYPLPrT~HC_6q#$(M1L&9O8;CYVA@|f5ne%tl#*Hokjd$
z%$-?L(sQR&B$)kJ5Fd57;&gSv{O@~@7vA~X@VIJe<_sS>(`n{^E5!KiT&`{_jO}xG
zjksR8v%qTOZ9&%%(YD5Q+DsKn>+Kh<Gvw)F(mYwr$Np;8rjTBi`@S4D{FjduMy$P&
z!@+bb?5<14#iJi<>t>$+So<jA?6sTkUs^4Z`FrYa-V@7du9G(0-4wAQW>ef{6QAW<
zbZ43US+vB3<6Qct9u-C3_$Kq&f^CaiKRX)jtN3Ns@c%6zsD$j9wV$ggK;&4x*fCia
zLB*VmH>`QhhhuA)r?=f(<6cnwic`s{<%6VCL4lE|Y>d>dfG_9I{m^Y^wzs(d(?Yr6
zis|(3fA#fw-wIFf@wc1*__xbN=apK1>MaX0Z{+QaE0odcUgykF`{G=h2<v3dO21h_
z*Jb)sBtPUQJ^Z~UOh?S4Ge~DTf0_H?$je8zZOnap+_v8!WwT<Rr_ilwHvhZQB8}#5
zx~-_Xt~-c@v8{2+7u$8G4H9O}3Ou}El|Ylp)0$@m3H$3yFMDl_tAF|<?)<k$?My12
z-`><;;{|zc%IuBYO$h==>*pt3eDdUph^47!<AspfCg19}nYwf<=!vTBu&BHE=z@}7
zSo4Jm8lHiBKOQ{zv7(^g`dzo>sY$>0o;CF^Ih5J{G9tw))#zbG&P*G14h^m)M@kKS
zf7Ry9wr~r*XmUwJ(L;!jy(MA0bI1F3rO>6T3S!fY-IUv$W!Mi_y!@%J;kDOjy_Oia
zP5;%dvuv6ZC+Sqjf1P6&*s^Z({mplc<Tl3{&oADa6gfq8*0F>GKYiA_OBC&V@OzHm
z*SzhSH#SA+-Hu<z*83^{@-g*iWvf&F%R-v|N-}AZl03e?9hN?BZY<JzO~>5W*2f$B
z%szJEW@jC@K+3<S-d;zqF2}fe53g#oadNih-v{+71VmDHR0vw_@jbJN*`wS!Y?`9Y
zoP+238vFYl11CC0%xDQvkh7^cu%TG}@YU5k;p>gw+;)$!6&5a8_wlqiTXuHa_I$}J
zS0(LglM)gWLA{7esu^hx-Dz%aprNiSA#Ja+<|xYuuxMufOU}u1dggTF`8mUs6a$+I
zho*E;Kk@(nIdcs8Tml5z>`yAIIEnl!=S&aC3eo#;?its*h8LbDi>`LD+)Rq!IPJ}?
zzrSaGn*GPP{@Lz@GHlaWk6W&>vP*cJyQxjl(<wGsq`mg(zRTY^9B;qseqoXrX*H=n
zcv<HZ3ARkHX%8!R=s6Z_|HJRu85ANK>n<|!dC%f=3^7^vBgNisnJ?dYTIhCK?9B9*
z9cAa71deO8zK`Etd-T{$`Sgk$Y2VM%W(|;mQ>8f;3Ubcx^o$QgEt_FG{r`=$2PUSH
zixx}9?N75PX0z2%xHI2=;mfuq7xO%f<6`F?yg%PDdiKNr^;vy;AF->c@n~uBSXuGp
z<?(Qf8N9i7_QQYs!>6aWm^%Gh@j*){`&2?|YC&P*iZv+{CLa!xe&oef5hcgN!xO*X
zXpV*B6%$)fGoIo9e<iI#o0<~~*7Z#ixe%pz;wW3%>^XDiDk?NQJkH-%`B|b*UOj6j
zr`~~+oO5q|{hAsOmUiJvih;TMeG{*S28QMahQr7EdD7B&q|J?TZXWvZ%k=O<XOW+M
z6DKy#oT;eRws6A)$5>hK+me>73q<8+XidE7Ki_DMmFuTO==L?GRdd4v4+ey^+<SB2
z=z&|G8XBD2C7zv0ys=k0HLO#iVT(y?tK^iaps|dmCJ7r`NePLL*jATphL`R3m4^zx
zXdJ%0oF{s_QOu452M#zy?Qdvo1f@8amW9jO4jyiH^x-*eAeF|+m|y(hmMgE||Ih7;
z*0U4uT20yuS;(iPRr2<YMA-T^HNK{kOy`br&9hIxv8nap1;w_~*Ag>kNbIXUY$wVz
z(?(b*>A|zK2TxK8ijxBZ6DLef{qQaA$M5NlQ>UJ}SkB>aI^D%jRFGNDHYp(`<;L#m
ziT~@`&d-<RlS}#WU;ftP<fKE7f<3l}1xz@2Ws0M7Gbmu3n9|dwv%e(!@@Cu#UM{g|
zQ^(vy2avNE{{NpdSCOyTLBMtXfd+?$+srzgERDwstll-=C@x?Uk`ZTMVLINF7<|#!
z+4-{5ei_T3KWFawlcKlQkz<OIXO7{UjHcoQnX?}YPaiYr%jgXPw{>rJ#EC8Hs6GDu
zq*08xrY6_6vrXsre-)nfu`W_xefyU+DcQcaA6Z=4T3dA4q($*Cg8<7J<HDZjimH`;
z0!v<+zo<JZnZ8)!@UEa~Ay;_~eqCY)6@^NtzU%0K_GYoNUg!*qv5oKu7Fp}xZC`VL
zPvVbXQybgcL1Rw!|3L#Q_x2`Fn3D3~S=x`kvmISnr1YkQ$Y*TTH#(52v#j@tCBvpo
zFE6XNP1l!nah24IO?vP+`NzMx51;$9nVPoo%S*VpNE|uBp~H7HYW4xvk0IIu2Ol4o
z%*v8j7o(_Ey#Q1KfYS4|%g2uKoStUL!6_*5@PG>s^G1j1jQO(;%)cdJI4v<Lb@D^G
zPg@>)i5<D3GvCgzq}*61!od%*rbVf&>F2A^w#UaeTJxQsX>{V>Len4pw_1}O-kwh6
zjY(mfWm&<a$UkACk-=dxJwJmL|Np1|_&xpM5ze;K*C%`=;X_4A2T!hA#Z&#wprBAt
z&rIPfciKb8#Y!yMZyu#jekHLZ$%jps;UJ5bPxIrO#%(tu+D=bDSq_T&|NoT|Gf%y}
z&10PAFx&D+u;IZ?Elv&>3Sx4asy4bYFmIdG=H{wgcU&OY!+FZzzo#$U^b9<e0Ne4b
zlzaD56DyB--h>Gi4>dVws+g{wFe~#zyZ)?=5l1BhvIUy>4y3Y9WT=;T<i8^#_5a`G
z8Rq?ox6LNSHi0Y$g`aMrP5rqG_s@TL%+9%T#sV+4oT=I?+XQ-aV@`cIA|MxgWcL&M
zd!N|OHb-n0m?)Q?;E-{D=j6n9F_Y4R3_x-4|G(1CXHTA{YG@>!IO!-UaQLd{!JOg;
zS$~pcJCDm$ID|FFvjhwDdMD0i4xT2MIaBE41N%$A)Xq$4QR~odHB?yIBdjj5YPICL
z*u{sOpxbzrDi?hGoDM3Eo;;PTsg;y4m{#}qxY@_HWetp(J2-WH7ic<*9hn?f!1+k<
zxN7T5xtM86>Z~T^Li`tkn8O^;m>g`-t#t@GwlbKfx0mPo`rvd<J8%<BY5Cc+Cr&I_
zapJ<&qY2sFOaCM!I6OBpSTJ3l=SElW{|a}e6W%fvOE{k8`+Q%pQ*!Mbg=6A7Vp^w3
zewZ%Fb3^F({rx=d{YECHhB14+-Y``)LHZv`o1KrHIH90^{KD<i3CYZND|<{M6W*SB
zBpK$!9e-v~t6*uM>AEI%-Us?B9V`*S#?1GQG#*zzk#~4an1sign7Y47H@5XIT-*Eb
zqH@-|XONkPMJu-JsU5p;<;a8Arw<%!+kEz5gL0~Z@rGw+F;C587F-ROA!R74y!s%s
zf$xE-9fr0|%NTrB_1YIs1-X-bozdgZ>7Z@9%X}9*af^TsG+_Aue^H4>dwaX0@nl8Q
zsf<O&+}Bi^+Ff=uPcf2YDp2KXiZaM`FJ_ySWKwwM5RYEp)}>B9Nk`+?8XVa6QBhs6
z<HX)-p0{s#qPGXF)pCTUo<)Z&<h8Y%Hyb;bnKjRpWL8$L5<Tn?)69I-w<khro`hK?
zL%7qysgpW9_wDeqHkmB>AxxX6rK|U7w`AH`r#si2OCce+=$68UEhf$D^+D$BuxOmc
zI=Ne?<MP54KPM&FI`JIHSB^U9utYD$Ez$Q#L^l`PotMmUT`{|lUAT0p;GN95ZrPQf
zPy>1K6HntD8RrTM=P0R?RGojnSYI*SQD)!4aC2p2h4zZo-Al4q{bzoB%noXkXorLA
zuz$9`enPNvc+tr(6MJ}^0|cB444S*Rl1?i|X*EXmFOKYr+&n>Cdk2H#<$|U5XWTgD
z7ITP6@K~JUVe4Ufd|kgS`MAWjHHj8gQy%^-wwk*GQt2=9d<0q-nwIw9adJR-`iCzm
z1!ZXgp+~R(JK!T$A=NV@(8=wRVuaB_-(!mfS9^cRR$nRm>%aZs`};wscf7fKdP9X_
zQn=<_Xefm~a$uXlyZYTb9yd2od!VT3!q<X`-CH&Kl&1tpeAveGA;k8`moN!~M?$N0
zuJQ3HAKRe0jDxvPiRI9bpOX`x`?XD0m)w@us<&VXY%{6*qXuR(Gd4XvHbq4?Z*MkX
zVYbXnHdR%&$jB)!4O=Y|6-54~gjgNfV$$g;!?WU~h_-{C^o3t)JRT<>J@<ccvn+a>
z!JFI1KYUXC`MP8UEEcjs4n2ENMuta8iAP9?M@@}q&Kw>oDIPbs3wu3lYF@C%F4}0w
zE&6CO$3&@*>U?Z_UNUF<OK(2OnRsXC;SHI>ho@@uOxHKwQ+xWu-`@=LA9I6>8qg^-
z8;&%xJ^1PA(9k${uA@(nW5kTc=4MCF&VzU6G`6=lPMqi%C%21j?nJ?=6^xt8)+|f(
zO5}Q!ANc4}t7f;izS`sa_Mlo?FFtKY{d~|V6^Gj;?dsKjo^S@uw1Fx(DV<Jc{s#{d
z64KKXl9CeA(*=Z*5>ityTu8WZxp`gLv!|&OCMQprl=R@4TPCNFrCp<XYQkEb4_6c_
zcB$Oo4|efe>BF<ldD{7nWvq{Z&UyI#{X(wwl=9P=pb0dF|NpPdHRNJY<Y8g=(aHVu
zzxq$3!oP!FTefygm1%!^w?^@Fp7~El3%SC)>z!JP0?MmyuT2Y6zwE`RRIxu-S1G@?
z_^{ya$LrGcx7Ocw&|KoOw#V_@mSc0i{OdfqBasp8g;kx*{0S*30>a4&$;k=n=?@+y
z35cdnn3xE1`sCyZ)6xV)Qy)BPnpeSTFT$Fg%oO?IzI~g0{eosc9i0n1+D#Pa+oxB2
zYI*oExlP`Ffw<6}U97fJ??0!%@^8B+3`vZ4<{EMxP!MsJ=W1F}@n86q{J#eef?Tab
ztV&;o&C+D&(2xFfxWMLARFINe&rO}3dUKLhSmr3F2i>~;{NCd`&(7xdHAe-ywgd$%
zHNBSP=klUf+~s}G$GgiamI{OYpxMdH-=M-FCiddv51tLeArH@Tb0npt{P;Qf;cfG(
jeivQ^h&u?LfW^#kCBJ6FvfZ0pK^i<={an^LB{Ts5;0bTK

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
deleted file mode 100644
index 9403003be1..0000000000
--- a/tensorflow/contrib/image/python/ops/dense_image_warp.py
+++ /dev/null
@@ -1,196 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Image warping using per-pixel flow vectors."""
-
-import numpy as np
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-
-
-def _interpolate_bilinear(grid,
-                          query_points,
-                          name='interpolate_bilinear',
-                          indexing='ij'):
-  """Similar to Matlab's interp2 function.
-
-  Finds values for query points on a grid using bilinear interpolation.
-
-  Args:
-    grid: a 4-D float `Tensor` of shape `[batch, height, width, channels]`.
-    query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 2]`.
-    name: a name for the operation (optional).
-    indexing: whether the query points are specified as row and column (ij),
-      or Cartesian coordinates (xy).
-
-  Returns:
-    values: a 3-D `Tensor` with shape `[batch, N, channels]`
-
-  Raises:
-    ValueError: if the indexing mode is invalid, or if the shape of the inputs
-      invalid.
-  """
-  if indexing != 'ij' and indexing != 'xy':
-    raise ValueError('Indexing mode must be \'ij\' or \'xy\'')
-
-  with ops.name_scope(name):
-    shape = grid.get_shape().as_list()
-    if len(shape) != 4:
-      msg = 'Grid must be 4 dimensional. Received size: '
-      raise ValueError(msg + str(grid.get_shape()))
-
-    batch_size, height, width, channels = shape
-    query_type = query_points.dtype
-    grid_type = grid.dtype
-
-    if (len(query_points.get_shape()) != 3 or
-        query_points.get_shape()[2].value != 2):
-      msg = ('Query points must be 3 dimensional and size 2 in dim 2. Received '
-             'size: ')
-      raise ValueError(msg + str(query_points.get_shape()))
-
-    _, num_queries, _ = query_points.get_shape().as_list()
-
-    if height < 2 or width < 2:
-      msg = 'Grid must be at least batch_size x 2 x 2 in size. Received size: '
-      raise ValueError(msg + str(grid.get_shape()))
-
-    alphas = []
-    floors = []
-    ceils = []
-
-    index_order = [0, 1] if indexing == 'ij' else [1, 0]
-    unstacked_query_points = array_ops.unstack(query_points, axis=2)
-
-    for dim in index_order:
-      with ops.name_scope('dim-' + str(dim)):
-        queries = unstacked_query_points[dim]
-
-        size_in_indexing_dimension = shape[dim + 1]
-
-        # max_floor is size_in_indexing_dimension - 2 so that max_floor + 1
-        # is still a valid index into the grid.
-        max_floor = math_ops.cast(size_in_indexing_dimension - 2, query_type)
-        min_floor = constant_op.constant(0.0, dtype=query_type)
-        floor = math_ops.minimum(
-            math_ops.maximum(min_floor, math_ops.floor(queries)), max_floor)
-        int_floor = math_ops.cast(floor, dtypes.int32)
-        floors.append(int_floor)
-        ceil = int_floor + 1
-        ceils.append(ceil)
-
-        # alpha has the same type as the grid, as we will directly use alpha
-        # when taking linear combinations of pixel values from the image.
-        alpha = math_ops.cast(queries - floor, grid_type)
-        min_alpha = constant_op.constant(0.0, dtype=grid_type)
-        max_alpha = constant_op.constant(1.0, dtype=grid_type)
-        alpha = math_ops.minimum(math_ops.maximum(min_alpha, alpha), max_alpha)
-
-        # Expand alpha to [b, n, 1] so we can use broadcasting
-        # (since the alpha values don't depend on the channel).
-        alpha = array_ops.expand_dims(alpha, 2)
-        alphas.append(alpha)
-
-    if batch_size * height * width > np.iinfo(np.int32).max / 8:
-      error_msg = """The image size or batch size is sufficiently large
-                     that the linearized addresses used by array_ops.gather
-                     may exceed the int32 limit."""
-      raise ValueError(error_msg)
-
-    flattened_grid = array_ops.reshape(grid,
-                                       [batch_size * height * width, channels])
-    batch_offsets = array_ops.reshape(
-        math_ops.range(batch_size) * height * width, [batch_size, 1])
-
-    # This wraps array_ops.gather. We reshape the image data such that the
-    # batch, y, and x coordinates are pulled into the first dimension.
-    # Then we gather. Finally, we reshape the output back. It's possible this
-    # code would be made simpler by using array_ops.gather_nd.
-    def gather(y_coords, x_coords, name):
-      with ops.name_scope('gather-' + name):
-        linear_coordinates = batch_offsets + y_coords * width + x_coords
-        gathered_values = array_ops.gather(flattened_grid, linear_coordinates)
-        return array_ops.reshape(gathered_values,
-                                 [batch_size, num_queries, channels])
-
-    # grab the pixel values in the 4 corners around each query point
-    top_left = gather(floors[0], floors[1], 'top_left')
-    top_right = gather(floors[0], ceils[1], 'top_right')
-    bottom_left = gather(ceils[0], floors[1], 'bottom_left')
-    bottom_right = gather(ceils[0], ceils[1], 'bottom_right')
-
-    # now, do the actual interpolation
-    with ops.name_scope('interpolate'):
-      interp_top = alphas[1] * (top_right - top_left) + top_left
-      interp_bottom = alphas[1] * (bottom_right - bottom_left) + bottom_left
-      interp = alphas[0] * (interp_bottom - interp_top) + interp_top
-
-    return interp
-
-
-def dense_image_warp(image, flow, name='dense_image_warp'):
-  """Image warping using per-pixel flow vectors.
-
-  Apply a non-linear warp to the image, where the warp is specified by a dense
-  flow field of offset vectors that define the correspondences of pixel values
-  in the output image back to locations in the  source image. Specifically, the
-  pixel value at output[b, j, i, c] is
-  images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c].
-
-  The locations specified by this formula do not necessarily map to an int
-  index. Therefore, the pixel value is obtained by bilinear
-  interpolation of the 4 nearest pixels around
-  (b, j - flow[b, j, i, 0], i - flow[b, j, i, 1]). For locations outside
-  of the image, we use the nearest pixel values at the image boundary.
-
-
-  Args:
-    image: 4-D float `Tensor` with shape `[batch, height, width, channels]`.
-    flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`.
-    name: A name for the operation (optional).
-
-    Note that image and flow can be of type tf.half, tf.float32, or tf.float64,
-    and do not necessarily have to be the same type.
-
-  Returns:
-    A 4-D float `Tensor` with shape`[batch, height, width, channels]`
-      and same type as input image.
-
-  Raises:
-    ValueError: if height < 2 or width < 2 or the inputs have the wrong number
-                of dimensions.
-  """
-  with ops.name_scope(name):
-    batch_size, height, width, channels = image.get_shape().as_list()
-    # The flow is defined on the image grid. Turn the flow into a list of query
-    # points in the grid space.
-    grid_x, grid_y = array_ops.meshgrid(
-        math_ops.range(width), math_ops.range(height))
-    stacked_grid = math_ops.cast(
-        array_ops.stack([grid_y, grid_x], axis=2), flow.dtype)
-    batched_grid = array_ops.expand_dims(stacked_grid, axis=0)
-    query_points_on_grid = batched_grid - flow
-    query_points_flattened = array_ops.reshape(query_points_on_grid,
-                                               [batch_size, height * width, 2])
-    # Compute values at the query points, then reshape the result back to the
-    # image grid.
-    interpolated = _interpolate_bilinear(image, query_points_flattened)
-    interpolated = array_ops.reshape(interpolated,
-                                     [batch_size, height, width, channels])
-    return interpolated
diff --git a/tensorflow/contrib/image/python/ops/interpolate_spline.py b/tensorflow/contrib/image/python/ops/interpolate_spline.py
deleted file mode 100644
index ad17921991..0000000000
--- a/tensorflow/contrib/image/python/ops/interpolate_spline.py
+++ /dev/null
@@ -1,285 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Polyharmonic spline interpolation."""
-
-import numpy as np
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import linalg_ops
-from tensorflow.python.ops import math_ops
-
-EPSILON = 0.0000000001
-
-
-def _cross_squared_distance_matrix(x, y):
-  """Pairwise squared distance between two (batch) matrices' rows (2nd dim).
-
-  Computes the pairwise distances between rows of x and rows of y
-  Args:
-    x: [batch_size, n, d] float `Tensor`
-    y: [batch_size, m, d] float `Tensor`
-
-  Returns:
-    squared_dists: [batch_size, n, m] float `Tensor`, where
-    squared_dists[b,i,j] = ||x[b,i,:] - y[b,j,:]||^2
-  """
-  x_norm_squared = math_ops.reduce_sum(math_ops.square(x), 2)
-  y_norm_squared = math_ops.reduce_sum(math_ops.square(y), 2)
-
-  # Expand so that we can broadcast.
-  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
-  y_norm_squared_tile = array_ops.expand_dims(y_norm_squared, 1)
-
-  x_y_transpose = math_ops.matmul(x, y, adjoint_b=True)
-
-  # squared_dists[b,i,j] = ||x_bi - y_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
-  squared_dists = x_norm_squared_tile - 2 * x_y_transpose + y_norm_squared_tile
-
-  return squared_dists
-
-
-def _pairwise_squared_distance_matrix(x):
-  """Pairwise squared distance among a (batch) matrix's rows (2nd dim).
-
-  This saves a bit of computation vs. using _cross_squared_distance_matrix(x,x)
-
-  Args:
-    x: `[batch_size, n, d]` float `Tensor`
-
-  Returns:
-    squared_dists: `[batch_size, n, n]` float `Tensor`, where
-    squared_dists[b,i,j] = ||x[b,i,:] - x[b,j,:]||^2
-  """
-
-  x_x_transpose = math_ops.matmul(x, x, adjoint_b=True)
-  x_norm_squared = array_ops.matrix_diag_part(x_x_transpose)
-  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
-
-  # squared_dists[b,i,j] = ||x_bi - x_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
-  squared_dists = x_norm_squared_tile - 2 * x_x_transpose + array_ops.transpose(
-      x_norm_squared_tile, [0, 2, 1])
-
-  return squared_dists
-
-
-def _solve_interpolation(train_points, train_values, order,
-                         regularization_weight):
-  """Solve for interpolation coefficients.
-
-  Computes the coefficients of the polyharmonic interpolant for the 'training'
-  data defined by (train_points, train_values) using the kernel phi.
-
-  Args:
-    train_points: `[b, n, d]` interpolation centers
-    train_values: `[b, n, k]` function values
-    order: order of the interpolation
-    regularization_weight: weight to place on smoothness regularization term
-
-  Returns:
-    w: `[b, n, k]` weights on each interpolation center
-    v: `[b, d, k]` weights on each input dimension
-  """
-
-  b, n, d = train_points.get_shape().as_list()
-  _, _, k = train_values.get_shape().as_list()
-
-  # First, rename variables so that the notation (c, f, w, v, A, B, etc.)
-  # follows https://en.wikipedia.org/wiki/Polyharmonic_spline.
-  # To account for python style guidelines we use
-  # matrix_a for A and matrix_b for B.
-
-  c = train_points
-  f = train_values
-
-  # Next, construct the linear system.
-  with ops.name_scope('construct_linear_system'):
-
-    matrix_a = _phi(_pairwise_squared_distance_matrix(c), order)  # [b, n, n]
-    if regularization_weight > 0:
-      batch_identity_matrix = np.expand_dims(np.eye(n), 0)
-      batch_identity_matrix = constant_op.constant(
-          batch_identity_matrix, dtype=train_points.dtype)
-
-      matrix_a += regularization_weight * batch_identity_matrix
-
-    # Append ones to the feature values for the bias term in the linear model.
-    ones = array_ops.ones([b, n, 1], train_points.dtype)
-    matrix_b = array_ops.concat([c, ones], 2)  # [b, n, d + 1]
-
-    # [b, n + d + 1, n]
-    left_block = array_ops.concat(
-        [matrix_a, array_ops.transpose(matrix_b, [0, 2, 1])], 1)
-
-    num_b_cols = matrix_b.get_shape()[2]  # d + 1
-    lhs_zeros = array_ops.zeros([b, num_b_cols, num_b_cols], train_points.dtype)
-    right_block = array_ops.concat([matrix_b, lhs_zeros],
-                                   1)  # [b, n + d + 1, d + 1]
-    lhs = array_ops.concat([left_block, right_block],
-                           2)  # [b, n + d + 1, n + d + 1]
-
-    rhs_zeros = array_ops.zeros([b, d + 1, k], train_points.dtype)
-    rhs = array_ops.concat([f, rhs_zeros], 1)  # [b, n + d + 1, k]
-
-  # Then, solve the linear system and unpack the results.
-  with ops.name_scope('solve_linear_system'):
-    w_v = linalg_ops.matrix_solve(lhs, rhs)
-    w = w_v[:, :n, :]
-    v = w_v[:, n:, :]
-
-  return w, v
-
-
-def _apply_interpolation(query_points, train_points, w, v, order):
-  """Apply polyharmonic interpolation model to data.
-
-  Given coefficients w and v for the interpolation model, we evaluate
-  interpolated function values at query_points.
-
-  Args:
-    query_points: `[b, m, d]` x values to evaluate the interpolation at
-    train_points: `[b, n, d]` x values that act as the interpolation centers
-                    ( the c variables in the wikipedia article)
-    w: `[b, n, k]` weights on each interpolation center
-    v: `[b, d, k]` weights on each input dimension
-    order: order of the interpolation
-
-  Returns:
-    Polyharmonic interpolation evaluated at points defined in query_points.
-  """
-
-  batch_size = train_points.get_shape()[0].value
-  num_query_points = query_points.get_shape()[1].value
-
-  # First, compute the contribution from the rbf term.
-  pairwise_dists = _cross_squared_distance_matrix(query_points, train_points)
-  phi_pairwise_dists = _phi(pairwise_dists, order)
-
-  rbf_term = math_ops.matmul(phi_pairwise_dists, w)
-
-  # Then, compute the contribution from the linear term.
-  # Pad query_points with ones, for the bias term in the linear model.
-  query_points_pad = array_ops.concat([
-      query_points,
-      array_ops.ones([batch_size, num_query_points, 1], train_points.dtype)
-  ], 2)
-  linear_term = math_ops.matmul(query_points_pad, v)
-
-  return rbf_term + linear_term
-
-
-def _phi(r, order):
-  """Coordinate-wise nonlinearity used to define the order of the interpolation.
-
-  See https://en.wikipedia.org/wiki/Polyharmonic_spline for the definition.
-
-  Args:
-    r: input op
-    order: interpolation order
-
-  Returns:
-    phi_k evaluated coordinate-wise on r, for k = r
-  """
-
-  # using EPSILON prevents log(0), sqrt0), etc.
-  # sqrt(0) is well-defined, but its gradient is not
-  with ops.name_scope('phi'):
-    if order == 1:
-      r = math_ops.maximum(r, EPSILON)
-      r = math_ops.sqrt(r)
-      return r
-    elif order == 2:
-      return 0.5 * r * math_ops.log(math_ops.maximum(r, EPSILON))
-    elif order == 4:
-      return 0.5 * math_ops.square(r) * math_ops.log(
-          math_ops.maximum(r, EPSILON))
-    elif order % 2 == 0:
-      r = math_ops.maximum(r, EPSILON)
-      return 0.5 * math_ops.pow(r, 0.5 * order) * math_ops.log(r)
-    else:
-      r = math_ops.maximum(r, EPSILON)
-      return math_ops.pow(r, 0.5 * order)
-
-
-def interpolate_spline(train_points,
-                       train_values,
-                       query_points,
-                       order,
-                       regularization_weight=0.0,
-                       name='interpolate_spline'):
-  r"""Interpolate signal using polyharmonic interpolation.
-
-  The interpolant has the form
-  $$f(x) = \sum_{i = 1}^n w_i \phi(||x - c_i||) + v^T x + b.$$
-
-  This is a sum of two terms: (1) a weighted sum of radial basis function (RBF)
-  terms, with the centers \\(c_1, ... c_n\\), and (2) a linear term with a bias.
-  The \\(c_i\\) vectors are 'training' points. In the code, b is absorbed into v
-  by appending 1 as a final dimension to x. The coefficients w and v are
-  estimated such that the interpolant exactly fits the value of the function at
-  the \\(c_i\\) points, the vector w is orthogonal to each \\(c_i\\), and the
-  vector w sums to 0. With these constraints, the coefficients can be obtained
-  by solving a linear system.
-
-  \\(\phi\\) is an RBF, parametrized by an interpolation
-  order. Using order=2 produces the well-known thin-plate spline.
-
-  We also provide the option to perform regularized interpolation. Here, the
-  interpolant is selected to trade off between the squared loss on the training
-  data and a certain measure of its curvature
-  ([details](https://en.wikipedia.org/wiki/Polyharmonic_spline)).
-  Using a regularization weight greater than zero has the effect that the
-  interpolant will no longer exactly fit the training data. However, it may be
-  less vulnerable to overfitting, particularly for high-order interpolation.
-
-  Note the interpolation procedure is differentiable with respect to all inputs
-  besides the order parameter.
-
-  Args:
-    train_points: `[batch_size, n, d]` float `Tensor` of n d-dimensional
-      locations. These do not need to be regularly-spaced.
-    train_values: `[batch_size, n, k]` float `Tensor` of n c-dimensional values
-      evaluated at train_points.
-    query_points: `[batch_size, m, d]` `Tensor` of m d-dimensional locations
-      where we will output the interpolant's values.
-    order: order of the interpolation. Common values are 1 for
-      \\(\phi(r) = r\\), 2 for \\(\phi(r) = r^2 * log(r)\\) (thin-plate spline),
-       or 3 for \\(\phi(r) = r^3\\).
-    regularization_weight: weight placed on the regularization term.
-      This will depend substantially on the problem, and it should always be
-      tuned. For many problems, it is reasonable to use no regularization.
-      If using a non-zero value, we recommend a small value like 0.001.
-    name: name prefix for ops created by this function
-
-  Returns:
-    `[b, m, k]` float `Tensor` of query values. We use train_points and
-    train_values to perform polyharmonic interpolation. The query values are
-    the values of the interpolant evaluated at the locations specified in
-    query_points.
-  """
-  with ops.name_scope(name):
-
-    # First, fit the spline to the observed data.
-    with ops.name_scope('solve'):
-      w, v = _solve_interpolation(train_points, train_values, order,
-                                  regularization_weight)
-
-    # Then, evaluate the spline at the query locations.
-    with ops.name_scope('predict'):
-      query_values = _apply_interpolation(query_points, train_points, w, v,
-                                          order)
-
-  return query_values
diff --git a/tensorflow/contrib/image/python/ops/sparse_image_warp.py b/tensorflow/contrib/image/python/ops/sparse_image_warp.py
deleted file mode 100644
index 9f50503d8f..0000000000
--- a/tensorflow/contrib/image/python/ops/sparse_image_warp.py
+++ /dev/null
@@ -1,192 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Image warping using sparse flow defined at control points."""
-
-import numpy as np
-
-from tensorflow.contrib.image.python.ops import dense_image_warp
-from tensorflow.contrib.image.python.ops import interpolate_spline
-
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-
-
-def _get_grid_locations(image_height, image_width):
-  """Wrapper for np.meshgrid."""
-
-  y_range = np.linspace(0, image_height - 1, image_height)
-  x_range = np.linspace(0, image_width - 1, image_width)
-  y_grid, x_grid = np.meshgrid(y_range, x_range, indexing='ij')
-  return np.stack((y_grid, x_grid), -1)
-
-
-def _expand_to_minibatch(np_array, batch_size):
-  """Tile arbitrarily-sized np_array to include new batch dimension."""
-  tiles = [batch_size] + [1] * np_array.ndim
-  return np.tile(np.expand_dims(np_array, 0), tiles)
-
-
-def _get_boundary_locations(image_height, image_width, num_points_per_edge):
-  """Compute evenly-spaced indices along edge of image."""
-  y_range = np.linspace(0, image_height - 1, num_points_per_edge + 2)
-  x_range = np.linspace(0, image_width - 1, num_points_per_edge + 2)
-  ys, xs = np.meshgrid(y_range, x_range, indexing='ij')
-  is_boundary = np.logical_or(
-      np.logical_or(xs == 0, xs == image_width - 1),
-      np.logical_or(ys == 0, ys == image_height - 1))
-  return np.stack([ys[is_boundary], xs[is_boundary]], axis=-1)
-
-
-def _add_zero_flow_controls_at_boundary(control_point_locations,
-                                        control_point_flows, image_height,
-                                        image_width, boundary_points_per_edge):
-  """Add control points for zero-flow boundary conditions.
-
-   Augment the set of control points with extra points on the
-   boundary of the image that have zero flow.
-
-  Args:
-    control_point_locations: input control points
-    control_point_flows: their flows
-    image_height: image height
-    image_width: image width
-    boundary_points_per_edge: number of points to add in the middle of each
-                           edge (not including the corners).
-                           The total number of points added is
-                           4 + 4*(boundary_points_per_edge).
-
-  Returns:
-    merged_control_point_locations: augmented set of control point locations
-    merged_control_point_flows: augmented set of control point flows
-  """
-
-  batch_size = control_point_locations.get_shape()[0].value
-
-  boundary_point_locations = _get_boundary_locations(image_height, image_width,
-                                                     boundary_points_per_edge)
-
-  boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2])
-
-  type_to_use = control_point_locations.dtype
-  boundary_point_locations = constant_op.constant(
-      _expand_to_minibatch(boundary_point_locations, batch_size),
-      dtype=type_to_use)
-
-  boundary_point_flows = constant_op.constant(
-      _expand_to_minibatch(boundary_point_flows, batch_size), dtype=type_to_use)
-
-  merged_control_point_locations = array_ops.concat(
-      [control_point_locations, boundary_point_locations], 1)
-
-  merged_control_point_flows = array_ops.concat(
-      [control_point_flows, boundary_point_flows], 1)
-
-  return merged_control_point_locations, merged_control_point_flows
-
-
-def sparse_image_warp(image,
-                      source_control_point_locations,
-                      dest_control_point_locations,
-                      interpolation_order=2,
-                      regularization_weight=0.0,
-                      num_boundary_points=0,
-                      name='sparse_image_warp'):
-  """Image warping using correspondences between sparse control points.
-
-  Apply a non-linear warp to the image, where the warp is specified by
-  the source and destination locations of a (potentially small) number of
-  control points. First, we use a polyharmonic spline
-  (@{tf.contrib.image.interpolate_spline}) to interpolate the displacements
-  between the corresponding control points to a dense flow field.
-  Then, we warp the image using this dense flow field
-  (@{tf.contrib.image.dense_image_warp}).
-
-  Let t index our control points. For regularization_weight=0, we have:
-  warped_image[b, dest_control_point_locations[b, t, 0],
-                  dest_control_point_locations[b, t, 1], :] =
-  image[b, source_control_point_locations[b, t, 0],
-           source_control_point_locations[b, t, 1], :].
-
-  For regularization_weight > 0, this condition is met approximately, since
-  regularized interpolation trades off smoothness of the interpolant vs.
-  reconstruction of the interpolant at the control points.
-  See @{tf.contrib.image.interpolate_spline} for further documentation of the
-  interpolation_order and regularization_weight arguments.
-
-
-  Args:
-    image: `[batch, height, width, channels]` float `Tensor`
-    source_control_point_locations: `[batch, num_control_points, 2]` float
-      `Tensor`
-    dest_control_point_locations: `[batch, num_control_points, 2]` float
-      `Tensor`
-    interpolation_order: polynomial order used by the spline interpolation
-    regularization_weight: weight on smoothness regularizer in interpolation
-    num_boundary_points: How many zero-flow boundary points to include at
-      each image edge.Usage:
-        num_boundary_points=0: don't add zero-flow points
-        num_boundary_points=1: 4 corners of the image
-        num_boundary_points=2: 4 corners and one in the middle of each edge
-          (8 points total)
-        num_boundary_points=n: 4 corners and n-1 along each edge
-    name: A name for the operation (optional).
-
-    Note that image and offsets can be of type tf.half, tf.float32, or
-    tf.float64, and do not necessarily have to be the same type.
-
-  Returns:
-    warped_image: `[batch, height, width, channels]` float `Tensor` with same
-      type as input image.
-    flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense
-      flow field produced by the interpolation.
-  """
-
-  control_point_flows = (
-      dest_control_point_locations - source_control_point_locations)
-
-  clamp_boundaries = num_boundary_points > 0
-  boundary_points_per_edge = num_boundary_points - 1
-
-  with ops.name_scope(name):
-
-    batch_size, image_height, image_width, _ = image.get_shape().as_list()
-
-    # This generates the dense locations where the interpolant
-    # will be evaluated.
-    grid_locations = _get_grid_locations(image_height, image_width)
-
-    flattened_grid_locations = np.reshape(grid_locations,
-                                          [image_height * image_width, 2])
-
-    flattened_grid_locations = constant_op.constant(
-        _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype)
-
-    if clamp_boundaries:
-      (dest_control_point_locations,
-       control_point_flows) = _add_zero_flow_controls_at_boundary(
-           dest_control_point_locations, control_point_flows, image_height,
-           image_width, boundary_points_per_edge)
-
-    flattened_flows = interpolate_spline.interpolate_spline(
-        dest_control_point_locations, control_point_flows,
-        flattened_grid_locations, interpolation_order, regularization_weight)
-
-    dense_flows = array_ops.reshape(flattened_flows,
-                                    [batch_size, image_height, image_width, 2])
-
-    warped_image = dense_image_warp.dense_image_warp(image, dense_flows)
-
-    return warped_image, dense_flows
-- 
GitLab


From cbce7fd25c5f1bd109692ac1d5250fe1942d5dc0 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 14 Mar 2018 17:05:57 -0700
Subject: [PATCH 040/960] Check for the value of the _XlaCompile boolean since
 xla often creates it with the default value of false.

PiperOrigin-RevId: 189110514
---
 tensorflow/core/grappler/optimizers/function_optimizer.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 87160f6b83..b17715e742 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -138,7 +138,8 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     }
     // Don't touch anything marked XLA to prevent XLA failures further down the
     // road.
-    if (func.attr().count("_XlaCompile") != 0) {
+    if (func.attr().count("_XlaCompile") > 0 &&
+        func.attr().at("_XlaCompile").b()) {
       continue;
     }
     // Can't create IdentityN nodes with no input or output: skip these
-- 
GitLab


From 53bad5a6eee61f7ac5f627640ff096266532f041 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 17:08:35 -0700
Subject: [PATCH 041/960] Automated g4 rollback of changelist 189060958

PiperOrigin-RevId: 189110935
---
 tensorflow/cc/framework/cc_op_gen.cc         |  1 -
 tensorflow/core/lib/core/stringpiece.cc      |  5 ++
 tensorflow/core/lib/core/stringpiece.h       |  6 ++
 tensorflow/core/lib/core/stringpiece_test.cc | 70 +++++++++++++++++++
 tensorflow/core/lib/hash/hash.h              |  1 -
 tensorflow/core/lib/hash/hash_test.cc        | 73 --------------------
 6 files changed, 81 insertions(+), 75 deletions(-)

diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc
index d73121c7b7..39893f5ccd 100644
--- a/tensorflow/cc/framework/cc_op_gen.cc
+++ b/tensorflow/cc/framework/cc_op_gen.cc
@@ -28,7 +28,6 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb_text.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
-#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 5bd79778a6..29b727fc44 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -17,9 +17,14 @@ limitations under the License.
 
 #include <algorithm>
 #include <iostream>
+#include "tensorflow/core/lib/hash/hash.h"
 
 namespace tensorflow {
 
+size_t StringPieceHasher::operator()(StringPiece s) const {
+  return Hash64(s.data(), s.size());
+}
+
 std::ostream& operator<<(std::ostream& o, StringPiece piece) {
   o.write(piece.data(), piece.size());
   return o;
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 910e4d9e2a..caa9642774 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -35,6 +35,8 @@ limitations under the License.
 
 namespace tensorflow {
 
+struct StringPieceHasher;
+
 class StringPiece {
  public:
   typedef size_t size_type;
@@ -129,6 +131,10 @@ class StringPiece {
   // Intentionally copyable
 };
 
+struct StringPieceHasher {
+  size_t operator()(StringPiece s) const;
+};
+
 inline bool operator==(StringPiece x, StringPiece y) {
   return ((x.size() == y.size()) &&
           (memcmp(x.data(), y.data(), x.size()) == 0));
diff --git a/tensorflow/core/lib/core/stringpiece_test.cc b/tensorflow/core/lib/core/stringpiece_test.cc
index d0dbeb6072..8f17b85b6d 100644
--- a/tensorflow/core/lib/core/stringpiece_test.cc
+++ b/tensorflow/core/lib/core/stringpiece_test.cc
@@ -65,4 +65,74 @@ TEST(StringPiece, Contains) {
   EXPECT_TRUE(!a.contains(d));
 }
 
+TEST(StringPieceHasher, Equality) {
+  StringPieceHasher hasher;
+
+  StringPiece s1("foo");
+  StringPiece s2("bar");
+  StringPiece s3("baz");
+  StringPiece s4("zot");
+
+  EXPECT_TRUE(hasher(s1) != hasher(s2));
+  EXPECT_TRUE(hasher(s1) != hasher(s3));
+  EXPECT_TRUE(hasher(s1) != hasher(s4));
+  EXPECT_TRUE(hasher(s2) != hasher(s3));
+  EXPECT_TRUE(hasher(s2) != hasher(s4));
+  EXPECT_TRUE(hasher(s3) != hasher(s4));
+
+  EXPECT_TRUE(hasher(s1) == hasher(s1));
+  EXPECT_TRUE(hasher(s2) == hasher(s2));
+  EXPECT_TRUE(hasher(s3) == hasher(s3));
+  EXPECT_TRUE(hasher(s4) == hasher(s4));
+}
+
+TEST(StringPieceHasher, HashMap) {
+  string s1("foo");
+  string s2("bar");
+  string s3("baz");
+
+  StringPiece p1(s1);
+  StringPiece p2(s2);
+  StringPiece p3(s3);
+
+  std::unordered_map<StringPiece, int, StringPieceHasher> map;
+
+  map.insert(std::make_pair(p1, 0));
+  map.insert(std::make_pair(p2, 1));
+  map.insert(std::make_pair(p3, 2));
+  EXPECT_EQ(map.size(), 3);
+
+  bool found[3] = {false, false, false};
+  for (auto const& val : map) {
+    int x = val.second;
+    EXPECT_TRUE(x >= 0 && x < 3);
+    EXPECT_TRUE(!found[x]);
+    found[x] = true;
+  }
+  EXPECT_EQ(found[0], true);
+  EXPECT_EQ(found[1], true);
+  EXPECT_EQ(found[2], true);
+
+  auto new_iter = map.find("zot");
+  EXPECT_TRUE(new_iter == map.end());
+
+  new_iter = map.find("bar");
+  EXPECT_TRUE(new_iter != map.end());
+
+  map.erase(new_iter);
+  EXPECT_EQ(map.size(), 2);
+
+  found[0] = false;
+  found[1] = false;
+  found[2] = false;
+  for (const auto& iter : map) {
+    int x = iter.second;
+    EXPECT_TRUE(x >= 0 && x < 3);
+    EXPECT_TRUE(!found[x]);
+    found[x] = true;
+  }
+  EXPECT_EQ(found[0], true);
+  EXPECT_EQ(found[1], false);
+  EXPECT_EQ(found[2], true);
+}
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/hash/hash.h b/tensorflow/core/lib/hash/hash.h
index ca05e6346e..77b8031598 100644
--- a/tensorflow/core/lib/hash/hash.h
+++ b/tensorflow/core/lib/hash/hash.h
@@ -76,7 +76,6 @@ struct hash<StringPiece> {
     return static_cast<size_t>(Hash64(sp.data(), sp.size()));
   }
 };
-using StringPieceHasher = ::tensorflow::hash<StringPiece>;
 
 template <typename T, typename U>
 struct hash<std::pair<T, U>> {
diff --git a/tensorflow/core/lib/hash/hash_test.cc b/tensorflow/core/lib/hash/hash_test.cc
index 7d58313132..0e5f6c6803 100644
--- a/tensorflow/core/lib/hash/hash_test.cc
+++ b/tensorflow/core/lib/hash/hash_test.cc
@@ -13,8 +13,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include <map>
-#include <unordered_map>
 #include <vector>
 
 #include "tensorflow/core/lib/hash/hash.h"
@@ -83,75 +81,4 @@ static void BM_Hash32(int iters, int len) {
 }
 BENCHMARK(BM_Hash32)->Range(1, 1024);
 
-TEST(StringPieceHasher, Equality) {
-  StringPieceHasher hasher;
-
-  StringPiece s1("foo");
-  StringPiece s2("bar");
-  StringPiece s3("baz");
-  StringPiece s4("zot");
-
-  EXPECT_TRUE(hasher(s1) != hasher(s2));
-  EXPECT_TRUE(hasher(s1) != hasher(s3));
-  EXPECT_TRUE(hasher(s1) != hasher(s4));
-  EXPECT_TRUE(hasher(s2) != hasher(s3));
-  EXPECT_TRUE(hasher(s2) != hasher(s4));
-  EXPECT_TRUE(hasher(s3) != hasher(s4));
-
-  EXPECT_TRUE(hasher(s1) == hasher(s1));
-  EXPECT_TRUE(hasher(s2) == hasher(s2));
-  EXPECT_TRUE(hasher(s3) == hasher(s3));
-  EXPECT_TRUE(hasher(s4) == hasher(s4));
-}
-
-TEST(StringPieceHasher, HashMap) {
-  string s1("foo");
-  string s2("bar");
-  string s3("baz");
-
-  StringPiece p1(s1);
-  StringPiece p2(s2);
-  StringPiece p3(s3);
-
-  std::unordered_map<StringPiece, int, StringPieceHasher> map;
-
-  map.insert(std::make_pair(p1, 0));
-  map.insert(std::make_pair(p2, 1));
-  map.insert(std::make_pair(p3, 2));
-  EXPECT_EQ(map.size(), 3);
-
-  bool found[3] = {false, false, false};
-  for (auto const& val : map) {
-    int x = val.second;
-    EXPECT_TRUE(x >= 0 && x < 3);
-    EXPECT_TRUE(!found[x]);
-    found[x] = true;
-  }
-  EXPECT_EQ(found[0], true);
-  EXPECT_EQ(found[1], true);
-  EXPECT_EQ(found[2], true);
-
-  auto new_iter = map.find("zot");
-  EXPECT_TRUE(new_iter == map.end());
-
-  new_iter = map.find("bar");
-  EXPECT_TRUE(new_iter != map.end());
-
-  map.erase(new_iter);
-  EXPECT_EQ(map.size(), 2);
-
-  found[0] = false;
-  found[1] = false;
-  found[2] = false;
-  for (const auto& iter : map) {
-    int x = iter.second;
-    EXPECT_TRUE(x >= 0 && x < 3);
-    EXPECT_TRUE(!found[x]);
-    found[x] = true;
-  }
-  EXPECT_EQ(found[0], true);
-  EXPECT_EQ(found[1], false);
-  EXPECT_EQ(found[2], true);
-}
-
 }  // namespace tensorflow
-- 
GitLab


From 6fdd2d6d912e1b918d0a89ec9a5e42991182bf69 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 18:06:25 -0700
Subject: [PATCH 042/960] Turn on strip_default_attrs in custom export
 strategy. Validated that the change WAI on a local pipeline.

PiperOrigin-RevId: 189118106
---
 .../boosted_trees/estimator_batch/custom_export_strategy.py     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
index 23ba76210b..d9b0d89a03 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/custom_export_strategy.py
@@ -54,7 +54,7 @@ def make_custom_export_strategy(name,
     An `ExportStrategy`.
   """
   base_strategy = saved_model_export_utils.make_export_strategy(
-      serving_input_fn=export_input_fn)
+      serving_input_fn=export_input_fn, strip_default_attrs=True)
   input_fn = export_input_fn()
   (sorted_feature_names, dense_floats, sparse_float_indices, _, _,
    sparse_int_indices, _, _) = gbdt_batch.extract_features(
-- 
GitLab


From 041cf60da44f4c423c8f1ef603ce74b3377eb2d3 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Mar 2018 18:39:25 -0700
Subject: [PATCH 043/960] [tf.data] Fix Python shape inference for
 `tf.contrib.data.map_and_batch()`.

Previously, it would incorrectly report that all batches have the same size, not accounting for the possibility of the last batch being partial.

Fixes #17720.

PiperOrigin-RevId: 189121488
---
 .../python/kernel_tests/batch_dataset_op_test.py   | 14 ++++++++++++++
 tensorflow/contrib/data/python/ops/batching.py     |  3 +--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 71dc1c1172..a2da953c7b 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -387,6 +387,20 @@ class BatchDatasetTest(test.TestCase):
   def testBatchAndMapDatasetWithParallelBatching(self):
     return self._testBatchAndMapDatasetHelper(num_parallel_batches=10)
 
+  def testMapAndBatchYieldsPartialBatch(self):
+    iterator = (dataset_ops.Dataset.range(10)
+                .apply(batching.map_and_batch(
+                    lambda x: array_ops.reshape(x * x, [1]), 4))
+                .make_one_shot_iterator())
+    self.assertEqual([None, 1], iterator.output_shapes.as_list())
+    next_element = iterator.get_next()
+    with self.test_session() as sess:
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index 6eb512dec6..6463d75750 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -374,8 +374,7 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
   @property
   def output_shapes(self):
     return nest.pack_sequence_as(self._output_shapes, [
-        tensor_shape.vector(tensor_util.constant_value(
-            self._batch_size)).concatenate(s)
+        tensor_shape.vector(None).concatenate(s)
         for s in nest.flatten(self._output_shapes)
     ])
 
-- 
GitLab


From 6f0c663f4c125a87519541cd6f11932e2cf4c039 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Wed, 14 Mar 2018 19:02:29 -0700
Subject: [PATCH 044/960] [XLA] Add a test for F32 -> U32 conversion

PiperOrigin-RevId: 189124355
---
 tensorflow/compiler/xla/tests/convert_test.cc | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/convert_test.cc b/tensorflow/compiler/xla/tests/convert_test.cc
index 59d6d7a415..7926767a4f 100644
--- a/tensorflow/compiler/xla/tests/convert_test.cc
+++ b/tensorflow/compiler/xla/tests/convert_test.cc
@@ -177,6 +177,24 @@ XLA_TEST_F(ConvertTest, ConvertR1U32ToR1F32) {
   ComputeAndCompareR1<float>(&builder, expected, {arg_data.get()});
 }
 
+XLA_TEST_F(ConvertTest, ConvertR1F32ToR1U32) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<float> arg{0.0f,        1.0f,          16777216.0f,
+                         16777218.0f, 2147483647.0f, 4294967040.0f};
+  std::unique_ptr<Literal> arg_literal = Literal::CreateR1<float>({arg});
+  auto arg_param = builder.Parameter(0, arg_literal->shape(), "arg_param");
+  std::unique_ptr<GlobalData> arg_data =
+      client_->TransferToServer(*arg_literal).ConsumeValueOrDie();
+
+  builder.ConvertElementType(arg_param, U32);
+
+  std::vector<uint32> expected(arg.size());
+  for (int64 i = 0; i < arg.size(); ++i) {
+    expected[i] = static_cast<uint32>(arg[i]);
+  }
+  ComputeAndCompareR1<uint32>(&builder, expected, {arg_data.get()});
+}
+
 XLA_TEST_F(ConvertTest, ConvertR1U32ToR1S64) {
   ComputationBuilder builder(client_, TestName());
   std::vector<uint32> arg{0, 1, 0x1000, 0x7fffffff, 0x80000082, 0xFFFFFFFF};
-- 
GitLab


From 6b2e5a4a1a11f3bed1260bf63cf78af3e7109a9d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 14 Mar 2018 19:03:28 -0700
Subject: [PATCH 045/960] Checkpointable: Add logic for late-naming of
 SaveableObjects

Previously SaveableObjects returned by _gather_saveables_for_checkpoint ran into a sanity check for naming, and would otherwise have been saved under the wrong key. This change has _gather_saveables_for_checkpoint returning factories for SaveableObjects instead, which are either passed the object-based saving key or are called with no arguments when using the global name.

Variables didn't run into this, since the Saver was generating their names.

PiperOrigin-RevId: 189124512
---
 .../eager/python/checkpointable_utils.py      |  6 +-
 .../eager/python/checkpointable_utils_test.py | 71 +++++++++++++++++++
 tensorflow/python/training/checkpointable.py  | 30 ++++++--
 tensorflow/python/training/saver.py           |  6 +-
 tensorflow/python/training/saver_test.py      | 14 ++--
 5 files changed, 113 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index d07121df63..677b56b7b8 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -220,12 +220,16 @@ def _serialize_checkpointables(
     object_proto = object_graph_proto.nodes.add()
     object_proto.slot_variables.extend(slot_variables.get(checkpointable, ()))
     object_name = object_names[checkpointable]
-    for name, saveable in (
+    for name, saveable_factory in (
         checkpointable._gather_saveables_for_checkpoint().items()):  # pylint: disable=protected-access
       attribute = object_proto.attributes.add()
       attribute.name = name
       attribute.checkpoint_key = "%s/%s/%s" % (
           object_name, _OBJECT_ATTRIBUTES_NAME, _escape_local_name(name))
+      if callable(saveable_factory):
+        saveable = saveable_factory(name=attribute.checkpoint_key)
+      else:
+        saveable = saveable_factory
       # Figure out the name-based Saver's name for this variable.
       saver_dict = saver_lib.BaseSaverBuilder.OpListToDict(
           [saveable], convert_variable_to_tensor=False)
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 2054878bf8..31f661e634 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras._impl.keras.engine import training
 from tensorflow.python.layers import core
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
@@ -152,6 +153,46 @@ class InterfaceTests(test.TestCase):
     self.assertAllEqual([1., 1., 1.], self.evaluate(v2))
 
 
+class _MirroringSaveable(
+    core_saver.BaseSaverBuilder.ResourceVariableSaveable):
+
+  def __init__(self, primary_variable, mirrored_variable, name):
+    self._primary_variable = primary_variable
+    self._mirrored_variable = mirrored_variable
+    super(_MirroringSaveable, self).__init__(
+        self._primary_variable, "", name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    """Restore the same value into both variables."""
+    tensor, = restored_tensors
+    return control_flow_ops.group(
+        self._primary_variable.assign(tensor),
+        self._mirrored_variable.assign(tensor))
+
+
+class _OwnsMirroredVariables(checkpointable.CheckpointableBase):
+  """A Checkpointable object which returns a more complex SaveableObject."""
+
+  def __init__(self):
+    self.non_dep_variable = variable_scope.get_variable(
+        name="non_dep_variable", initializer=6., use_resource=True)
+    self.mirrored = variable_scope.get_variable(
+        name="mirrored", initializer=15., use_resource=True)
+
+  def _gather_saveables_for_checkpoint(self):
+    def _saveable_factory(name=self.non_dep_variable.name):
+      return _MirroringSaveable(
+          primary_variable=self.non_dep_variable,
+          mirrored_variable=self.mirrored,
+          name=name)
+    return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory}
+
+  # The Saver sorts by name before parsing, so we need a name property.
+  @property
+  def name(self):
+    return self.non_dep_variable.name
+
+
 class CheckpointingTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
@@ -261,6 +302,36 @@ class CheckpointingTests(test.TestCase):
             optimizer_node.slot_variables[0]
             .slot_variable_node_id].attributes[0].checkpoint_key)
 
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testMoreComplexSaveableReturned(self):
+    v = _OwnsMirroredVariables()
+    checkpoint = checkpointable_utils.Checkpoint(v=v)
+    test_dir = self.get_temp_dir()
+    prefix = os.path.join(test_dir, "ckpt")
+    self.evaluate(v.non_dep_variable.assign(42.))
+    save_path = checkpoint.save(prefix)
+    self.evaluate(v.non_dep_variable.assign(43.))
+    self.evaluate(v.mirrored.assign(44.))
+    checkpoint.restore(save_path).assert_consumed().initialize_or_restore()
+    self.assertEqual(42., self.evaluate(v.non_dep_variable))
+    self.assertEqual(42., self.evaluate(v.mirrored))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testMoreComplexSaveableReturnedWithGlobalName(self):
+    # The same object can also be saved using the name-based saver.
+    v = _OwnsMirroredVariables()
+    saver = core_saver.Saver(var_list=[v])
+    test_dir = self.get_temp_dir()
+    prefix = os.path.join(test_dir, "ckpt")
+    self.evaluate(v.non_dep_variable.assign(42.))
+    with self.test_session() as sess:
+      save_path = saver.save(sess, prefix)
+      self.evaluate(v.non_dep_variable.assign(43.))
+      self.evaluate(v.mirrored.assign(44.))
+      saver.restore(sess, save_path)
+      self.assertEqual(42., self.evaluate(v.non_dep_variable))
+      self.assertEqual(42., self.evaluate(v.mirrored))
+
   @test_util.run_in_graph_and_eager_modes()
   def testSaveRestore(self):
     model = MyModel()
diff --git a/tensorflow/python/training/checkpointable.py b/tensorflow/python/training/checkpointable.py
index e49965703e..96e3c4821c 100644
--- a/tensorflow/python/training/checkpointable.py
+++ b/tensorflow/python/training/checkpointable.py
@@ -210,8 +210,8 @@ class _CheckpointPosition(object):
     restore_ops = []
     building_graph = not context.executing_eagerly()
     for serialized_tensor in self.object_proto.attributes:
-      saveable_object = saveables.get(serialized_tensor.name, None)
-      if saveable_object is None:
+      saveable_factory = saveables.get(serialized_tensor.name, None)
+      if saveable_factory is None:
         # Purposefully does not throw an exception if attributes have been added
         # or deleted. Stores unused attributes so an exception can be raised if
         # the user decides to check that everything in the checkpoint was
@@ -225,7 +225,11 @@ class _CheckpointPosition(object):
       else:
         existing_ops = None
       if existing_ops is None:
-        named_saveables[serialized_tensor.checkpoint_key] = saveable_object
+        if callable(saveable_factory):
+          saveable = saveable_factory(name=serialized_tensor.checkpoint_key)
+        else:
+          saveable = saveable_factory
+        named_saveables[serialized_tensor.checkpoint_key] = saveable
     if named_saveables:
       validated_saveables = (
           self._checkpoint.builder._ValidateAndSliceInputs(named_saveables))  # pylint: disable=protected-access
@@ -600,14 +604,30 @@ class CheckpointableBase(object):
     """Returns a dictionary of values to checkpoint with this object.
 
     Keys in the returned dictionary are local to this object and in a separate
-    namespace from dependencies. Values may either be `SaveableObject`s or
-    variables easily converted to `SaveableObject`s (as in `tf.train.Saver`'s
+    namespace from dependencies. Values may either be `SaveableObject` factories
+    or variables easily converted to `SaveableObject`s (as in `tf.train.Saver`'s
     `var_list` constructor argument).
 
+    `SaveableObjects` have a name set, which Checkpointable needs to generate
+    itself. So rather than returning `SaveableObjects` directly, this method
+    should return a dictionary of callables which take `name` arguments and
+    return `SaveableObjects` with that name.
+
+    If this object may also be passed to the global-name-based `tf.train.Saver`,
+    the returned callables should have a default value for their name argument
+    (i.e. be callable with no arguments).
+
     Returned values must be saved only by this object; if any value may be
     shared, it should instead be a dependency. For example, variable objects
     save their own values with the key `VARIABLE_VALUE_KEY`, but objects which
     reference variables simply add a dependency.
+
+    Returns:
+      The dictionary mapping attribute names to `SaveableObject` factories
+      described above. For example:
+      {VARIABLE_VALUE_KEY:
+       lambda name="global_name_for_this_object":
+       SaveableObject(name=name, ...)}
     """
     return {}
 
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 2ce57c4432..5f68eec6ce 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -578,9 +578,11 @@ class BaseSaverBuilder(object):
           names_to_saveables[name] = [var]
       elif (isinstance(var, checkpointable.CheckpointableBase)
             and not isinstance(var, variables.Variable)):
+        checkpointable_saveables = [
+            (factory() if callable(factory) else factory)
+            for factory in var._gather_saveables_for_checkpoint().values()]
         names_to_saveables.update(
-            BaseSaverBuilder.OpListToDict(
-                list(var._gather_saveables_for_checkpoint().values())))
+            BaseSaverBuilder.OpListToDict(checkpointable_saveables))
       else:
         if context.executing_eagerly():
           if not isinstance(var, resource_variable_ops.ResourceVariable):
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 46fe2735be..4e80fab9bd 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2873,11 +2873,11 @@ class _OwnsAVariableSimple(checkpointable.CheckpointableBase):
 class _MirroringSaveable(
     saver_module.BaseSaverBuilder.ResourceVariableSaveable):
 
-  def __init__(self, primary_variable, mirrored_variable):
+  def __init__(self, primary_variable, mirrored_variable, name):
     self._primary_variable = primary_variable
     self._mirrored_variable = mirrored_variable
     super(_MirroringSaveable, self).__init__(
-        self._primary_variable, "", self._primary_variable.name)
+        self._primary_variable, "", name)
 
   def restore(self, restored_tensors, restored_shapes):
     """Restore the same value into both variables."""
@@ -2897,10 +2897,12 @@ class _OwnsMirroredVariables(checkpointable.CheckpointableBase):
         name="mirrored", initializer=15., use_resource=True)
 
   def _gather_saveables_for_checkpoint(self):
-    saveable = _MirroringSaveable(
-        primary_variable=self.non_dep_variable,
-        mirrored_variable=self.mirrored)
-    return {checkpointable.VARIABLE_VALUE_KEY: saveable}
+    def _saveable_factory(name=self.non_dep_variable.name):
+      return _MirroringSaveable(
+          primary_variable=self.non_dep_variable,
+          mirrored_variable=self.mirrored,
+          name=name)
+    return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory}
 
   # The Saver sorts by name before parsing, so we need a name property.
   @property
-- 
GitLab


From 8b305c509353815e2086200b47d0af898fa9df35 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Wed, 14 Mar 2018 20:07:38 -0700
Subject: [PATCH 046/960] Add int64 to randomized tests.

PiperOrigin-RevId: 189129641
---
 tensorflow/compiler/tests/randomized_tests.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tests/randomized_tests.cc b/tensorflow/compiler/tests/randomized_tests.cc
index e72dd4eea9..e53efc3091 100644
--- a/tensorflow/compiler/tests/randomized_tests.cc
+++ b/tensorflow/compiler/tests/randomized_tests.cc
@@ -83,8 +83,8 @@ string LocalDeviceToFullDeviceName(const string& device) {
   return strings::StrCat("/job:localhost/replica:0/task:0/device:", device);
 }
 
-constexpr std::array<DataType, 4> kAllXlaTypes = {
-    {DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64}};
+constexpr std::array<DataType, 5> kAllXlaTypes = {
+    {DT_INT32, DT_FLOAT, DT_BOOL, DT_COMPLEX64, DT_INT64}};
 
 // An OpTestBuilder is a graph builder class that takes as input an operator to
 // test, its inputs and attributes, and builds a graph that executes the
-- 
GitLab


From d832e0e1b15302aa76ce6db121bdeb15d2dfd0f4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 20:23:36 -0700
Subject: [PATCH 047/960] Make hash.h a public header.

PiperOrigin-RevId: 189130768
---
 tensorflow/core/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 213315f40e..352e183104 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -313,6 +313,7 @@ cc_library(
         "lib/gtl/optional.h",
         "lib/gtl/priority_queue_util.h",
         "lib/hash/crc32c.h",
+        "lib/hash/hash.h",
         "lib/histogram/histogram.h",
         "lib/io/buffered_inputstream.h",
         "lib/io/compression.h",
-- 
GitLab


From 357cd4b8b2f960520fc57b6cfbf41117a2a20fc7 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Wed, 14 Mar 2018 20:35:42 -0700
Subject: [PATCH 048/960] Internal change.

PiperOrigin-RevId: 189131526
---
 tensorflow/c/BUILD                 |   3 +
 tensorflow/c/c_api_experimental.cc | 346 +++++++++++++++++++++++++++++
 tensorflow/c/c_api_experimental.h  |  26 +++
 3 files changed, 375 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 29ed957c9a..c178d7f81f 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -116,6 +116,9 @@ tf_cuda_library(
         ":c_api",
         ":c_api_internal",
         "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
     ],
 )
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index be7f85a5bb..f6d8949bb0 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -17,8 +17,297 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
+using tensorflow::Node;
+using tensorflow::NodeBuilder;
+using tensorflow::NodeDef;
+using tensorflow::Status;
+
+namespace {
+
+const char* const DEVICE_TPU_REPLICATED_CORE = "TPU_REPLICATED_CORE";
+const char* const DEVICE_TPU_SYSTEM = "TPU_SYSTEM";
+
+TF_Operation* ToTF_Operation(Node* node) {
+  return static_cast<TF_Operation*>(static_cast<void*>(node));
+}
+
+// Graph rewrite algorithm (modeled after the python TPU graph rewrite path):
+//
+// 1. For each input node I, feed it to a new TPUReplicatedInput node, which in
+// turn feeds a new Identity node N, and store the mapping I->N.
+//
+// 2. Rewrite all existing graph nodes by adding a attribute on TPU cluster. For
+// each node reading some input node I, rewire it to read from N instead based
+// on the I->N mapping in step #1.
+//
+// 3. For each output node O, feed it to a new Identity node, which in turn
+// feeds a new TPUReplicatedOutput node, which in turn feeds a new Identity node
+// M. Return the set of new output nodes (the "M" nodes) for caller to fetch
+// from.
+//
+// Limitations compared to the python TPU rewrite path:
+// - # replicas is always 1.
+// - Less error checking.
+//
+// TODO(hongm): Simplify the graph rewrite to generating fewer TPUReplicate
+// related nodes.
+class GraphRewriter {
+ public:
+  GraphRewriter(TF_Graph* graph, int num_input_nodes,
+                const TF_Output* input_nodes, int num_output_nodes,
+                const TF_Output* output_nodes)
+      EXCLUSIVE_LOCKS_REQUIRED(graph->mu)
+      : graph_(graph), input_nodes_(input_nodes) {
+    for (int i = 0; i < num_input_nodes; ++i) {
+      //  Will fill in the value part later when we create the associated new
+      //  input node.
+      input_node_map_[input_nodes[i].oper->node.name()] = nullptr;
+    }
+
+    // Grab all existing nodes for the upcoming rewrite, before mutating the
+    // graph.
+    for (Node* n : graph->graph.nodes()) {
+      nodes_to_rewrite_.push_back(n);
+    }
+
+    for (int i = 0; i < num_output_nodes; ++i) {
+      output_node_map_.emplace(output_nodes[i].oper->node.name(),
+                               PortIndexPair{output_nodes[i].index, i});
+    }
+  }
+
+  // On success, sets `config_op` and `shutdown_op` to the corresponding
+  // "ConfigureDistributedTPU" and "ShutdownDistributedTPU" nodes added to the
+  // graph.
+  tensorflow::Status Rewrite(TF_Output* new_output_nodes, TF_Output* config_op,
+                             TF_Output* shutdown_op)
+      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
+    TF_RETURN_IF_ERROR(ProcessInputNodes());
+
+    return RewriteGraphAndAddOutputNodes(new_output_nodes, config_op,
+                                         shutdown_op);
+  }
+
+ private:
+  // Synthensizes new nodes for the input nodes, and creates a replicated
+  // metadata node.
+  tensorflow::Status ProcessInputNodes() EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
+    Node* metadata_node;
+    TF_RETURN_IF_ERROR(
+        NodeBuilder(metadata_node_name_.c_str(), "TPUReplicateMetadata")
+            .Attr("num_replicas", 1)
+            .Attr("_tpu_replicate", cluster_name_.c_str())
+            .Finalize(&graph_->graph, &metadata_node));
+
+    for (int i = 0; i < input_node_map_.size(); ++i) {
+      VLOG(1) << "Handling input node " << input_nodes_[i].oper->node.name();
+      Node* replicated_input_node;
+      {
+        std::string replicated_input_name("TPUReplicate/input" +
+                                          std::to_string(i));
+        NodeBuilder::NodeOut input(&input_nodes_[i].oper->node,
+                                   input_nodes_[i].index);
+        std::vector<NodeBuilder::NodeOut> input_list;
+        input_list.push_back(input);
+        TF_RETURN_IF_ERROR(
+            NodeBuilder(replicated_input_name.c_str(), "TPUReplicatedInput")
+                // This op requires an input list.
+                .Input(input_list)
+                .Finalize(&graph_->graph, &replicated_input_node));
+      }
+
+      {
+        Node* new_input_node;
+        const std::string new_input_name("TPUReplicate/replicated_input_" +
+                                         std::to_string(i));
+        TF_RETURN_IF_ERROR(NodeBuilder(new_input_name.c_str(), "Identity")
+                               .Input(replicated_input_node, 0)
+                               .ControlInput(metadata_node)
+                               .Attr("_tpu_replicate", cluster_name_.c_str())
+                               .Finalize(&graph_->graph, &new_input_node));
+        DCHECK_GT(input_node_map_.count(input_nodes_[i].oper->node.name()), 0);
+        input_node_map_[input_nodes_[i].oper->node.name()] = new_input_node;
+      }
+    }
+    return Status::OK();
+  }
+
+  // On success, sets `config_op` and `shutdown_op` to the corresponding
+  // "ConfigureDistributedTPU" and "ShutdownDistributedTPU" nodes added to the
+  // graph.
+  tensorflow::Status RewriteGraphAndAddOutputNodes(TF_Output* new_output_nodes,
+                                                   TF_Output* config_op,
+                                                   TF_Output* shutdown_op)
+      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
+    tensorflow::Status s;
+    // For each non-input node in the input graph, place the node in a "TPU
+    // replicate cluster" via an attribute, and with the above metadata node
+    // as a control dependency.
+    //
+    // Although we have handled the input nodes in ProcessInputNodes(), some
+    // of those nodes may also serve as output nodes, which we will handle
+    // below.
+    for (Node* n : nodes_to_rewrite_) {
+      if (n->IsSource()) continue;
+      VLOG(1) << "Rewriting node " << n->name();
+
+      if (n->IsSink()) {
+        // TODO(hongm): Rewire SINK to be control dependent on the new input
+        // nodes created above?
+        continue;
+      }
+
+      const NodeDef& old_def = n->def();
+      Node* new_node;
+      if (input_node_map_.count(n->name())) {
+        new_node = input_node_map_[n->name()];
+      } else {
+        // This node is to replace `n` in the graph.
+        NodeDef new_def = n->def();
+        const std::string new_node_name = "TPUReplicate/" + n->name();
+        new_def.set_name(new_node_name);
+        new_def.clear_input();
+        for (int i = 0; i < old_def.input_size(); ++i) {
+          const std::string& old_input_name = old_def.input(i);
+          const std::string new_input_name =
+              input_node_map_.count(old_input_name) > 0
+                  ? std::string(input_node_map_[old_input_name]->name())
+                  : "TPUReplicate/" + old_input_name;
+          new_def.add_input(new_input_name);
+        }
+        if (old_def.input_size() == 0) {
+          // It is sufficient to only set control dependency of nodes without
+          // input. Other nodes with input(s) with inherit such control
+          // dependency.
+          // e.g. say the graph computes add(x, y). Once we make nodes x and y
+          // control-dependent on the metadata node, node add will inherit
+          // such control dependency indirectly.
+          new_def.add_input(
+              tensorflow::strings::StrCat("^", metadata_node_name_.c_str()));
+        }
+        tensorflow::AddNodeAttr("_tpu_replicate", cluster_name_.c_str(),
+                                &new_def);
+        new_node = graph_->graph.AddNode(new_def, &s);
+        if (!s.ok()) {
+          return s;
+        }
+        VLOG(1) << "The rewritten node node is " << new_node->DebugString();
+      }
+
+      if (output_node_map_.count(n->name()) > 0) {
+        VLOG(1) << "Handling output node " << n->name();
+        auto range_it = output_node_map_.equal_range(n->name());
+        for (auto it = range_it.first; it != range_it.second; ++it) {
+          const PortIndexPair& pair = it->second;
+          Node* out_identity_node;
+          {
+            VLOG(1) << "Handling its output port " << pair.port
+                    << " at output index " << pair.index;
+            std::string output_node_name = "TPUReplicate/Identity";
+            if (pair.index > 0) {
+              output_node_name += "_" + std::to_string(pair.index);
+            }
+            TF_RETURN_IF_ERROR(
+                NodeBuilder(output_node_name.c_str(), "Identity")
+                    .Input(new_node, pair.port)
+                    .Device(!old_def.device().empty()
+                                ? old_def.device()
+                                : tensorflow::strings::StrCat(
+                                      "/device:", DEVICE_TPU_REPLICATED_CORE))
+                    .Attr("_tpu_replicate", cluster_name_.c_str())
+                    .Finalize(&graph_->graph, &out_identity_node));
+            VLOG(1) << "out_identity_node: "
+                    << out_identity_node->DebugString();
+          }
+
+          Node* replicated_output_node;
+          {
+            const std::string replicated_output_node_name =
+                "TPUReplicate/output" + std::to_string(pair.index);
+            TF_RETURN_IF_ERROR(
+                NodeBuilder(replicated_output_node_name.c_str(),
+                            "TPUReplicatedOutput")
+                    .Input(out_identity_node, 0)
+                    .Attr("num_replicas", 1)
+                    .Finalize(&graph_->graph, &replicated_output_node));
+            VLOG(1) << "replicated_output_node: "
+                    << replicated_output_node->DebugString();
+          }
+
+          Node* final_output_node;
+          const std::string final_output_node_name =
+              "TPUReplicate/output_" + std::to_string(pair.index) + "_shard_" +
+              std::to_string(0);
+          TF_RETURN_IF_ERROR(
+              NodeBuilder(final_output_node_name.c_str(), "Identity")
+                  .Input(replicated_output_node, 0)
+                  .Finalize(&graph_->graph, &final_output_node));
+          VLOG(1) << "new_output_node: " << final_output_node->DebugString();
+          auto oper = ToTF_Operation(final_output_node);
+          new_output_nodes[pair.index] = {oper, 0};
+        }
+      }
+
+      if (input_node_map_.count(n->name()) == 0) {
+        graph_->graph.RemoveNode(n);
+      }
+    }
+
+    {
+      Node* config_node;
+      TF_RETURN_IF_ERROR(
+          NodeBuilder("ConfigureDistributedTPU", "ConfigureDistributedTPU")
+              .Device(DEVICE_TPU_SYSTEM)
+              .Finalize(&graph_->graph, &config_node));
+      *config_op = {ToTF_Operation(config_node), 0};
+    }
+
+    {
+      Node* shutdown_node;
+      TF_RETURN_IF_ERROR(
+          NodeBuilder("ShutdownDistributedTPU", "ShutdownDistributedTPU")
+              .Device(DEVICE_TPU_SYSTEM)
+              .Finalize(&graph_->graph, &shutdown_node));
+      *shutdown_op = {ToTF_Operation(shutdown_node), 0};
+    }
+
+    return Status::OK();
+  }
+
+  TF_Graph* const graph_;
+
+  const TF_Output* const input_nodes_;
+
+  const std::string cluster_name_ = "TPUReplicate/cluster";
+  const std::string metadata_node_name_ = "TPUReplicate/TPUReplicateMetadata";
+
+  // Keep mappings from the current input nodes to newly created input nodes,
+  // which we will use to rewrite existing nodes that read these
+  // inputs. e.g. A node that reads input node PlaceHolder could be rewired to
+  // read the created TPUReplicate/replicated_input_0 node.
+  std::unordered_map<std::string, Node*> input_node_map_;
+
+  std::vector<Node*> nodes_to_rewrite_;
+
+  // Map from name to set{(output port, output tensor idx)}.
+  // e.g. Say ther are 3 output tensors, respectively produced by (node 0,
+  // port 0), (node 0, port 1), (node 1, port 0). Then the mapping entries
+  // are: node 0 -> {(port 0, idx 0), (port 1, idx 1)} node 1 -> {(port 0, idx
+  // 2)} Based on these mappings, we will generated 3 new output nodes.
+  struct PortIndexPair {
+    int port;
+    int index;
+  };
+  std::multimap<std::string, PortIndexPair> output_node_map_;
+};
+
+}  // namespace
+
 void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
   tensorflow::ConfigProto& config = options->options.config;
   auto* optimizer_options =
@@ -37,3 +326,60 @@ void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
     optimizer_options->set_global_jit_level(tensorflow::OptimizerOptions::OFF);
   }
 }
+
+TF_Output TF_SetupTPUExecution(TF_Session* session, int num_input_nodes,
+                               const TF_Output* input_nodes,
+                               int num_output_nodes,
+                               const TF_Output* output_nodes,
+                               TF_Output* new_output_nodes, TF_Status* status) {
+  TF_Output config_op, shutdown_op;
+  {
+    auto graph = session->graph;
+    tensorflow::mutex_lock c(graph->mu);
+
+    VLOG(1) << "Graph before TPU rewrite: "
+            << graph->graph.ToGraphDefDebug().DebugString();
+    GraphRewriter rewriter(graph, num_input_nodes, input_nodes,
+                           num_output_nodes, output_nodes);
+    status->status =
+        rewriter.Rewrite(new_output_nodes, &config_op, &shutdown_op);
+    if (!status->status.ok()) {
+      return shutdown_op;
+    }
+    VLOG(1) << "Graph after TPU rewrite: "
+            << graph->graph.ToGraphDefDebug().DebugString();
+  }
+
+  VLOG(1) << "Initializing TPU";
+  TF_Tensor* dummy_output;
+  TF_SessionRun(session, /*run_options*/ nullptr,
+                // input related parameters
+                /*inputs*/ nullptr, /*input_values*/ nullptr, /*ninputs*/ 0,
+                // output related parameters
+                /*outputs*/ &config_op, /*output_values*/ &dummy_output,
+                /*noutputs*/ 1,
+                /*targets*/ nullptr, /*ntargets*/ 0,
+                /*run_metadata*/ nullptr, status);
+  if (status->status.ok()) {
+    TF_DeleteTensor(dummy_output);
+  }
+  return shutdown_op;
+}
+
+void TF_ShutdownTPUExecution(TF_Session* session, TF_Output shutdown_node,
+                             TF_Status* status) {
+  {
+    tensorflow::mutex_lock c(session->graph->mu);
+    VLOG(1) << "Shutting down TPU, with input graph: "
+            << session->graph->graph.ToGraphDefDebug().DebugString();
+  }
+
+  TF_SessionRun(session, /*run_options*/ nullptr,
+                // input related parameters
+                /*inputs*/ nullptr, /*input_values*/ nullptr, /*ninputs*/ 0,
+                // output related parameters
+                /*outputs*/ nullptr, /*output_values*/ nullptr,
+                /*noutputs*/ 0,
+                /*targets*/ &shutdown_node.oper, /*ntargets*/ 1,
+                /*run_metadata*/ nullptr, status);
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 5a7b007e40..af65123131 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -25,6 +25,7 @@ limitations under the License.
 // Experimental C API for TensorFlow.
 //
 // The API here is subject to changes in the future.
+// --------------------------------------------------------------------------
 
 // Macro to control visibility of exported symbols in the shared library (.so,
 // .dylib, .dll).
@@ -59,6 +60,31 @@ extern "C" {
 TF_CAPI_EXPORT extern void TF_EnableXLACompilation(TF_SessionOptions* options,
                                                    unsigned char enable);
 
+// Sets up TPU execution, by rewriting the graph accordingly, and initializing
+// TPU system.
+//
+// On success, returns a shutdown node to be used in a subsequent
+// TF_ShutdownTPUExecution(), and sets the new output nodes in
+// `new_output_nodes` for caller to fetch from. Must be called exactly once
+// before TF_SessionRun().
+//
+// The API and logic is modeled after the python counterparts
+// tpu.{initialize_system(), rewrite(), shutdown_system()}.
+//
+// TODO(b/74774824): Create separate APIs for initializing TPU system and graph
+// rewrite.
+TF_CAPI_EXPORT extern TF_Output TF_SetupTPUExecution(
+    TF_Session* session, int num_input_nodes, const TF_Output* input_nodes,
+    int num_output_nodes, const TF_Output* output_nodes,
+    TF_Output* new_output_nodes, TF_Status* status);
+
+// Shuts down TPU system. For any `session` where TF_SetupTPUExecution() has
+// been successfully called, this call must be made exactly once before the
+// session is closed.
+TF_CAPI_EXPORT extern void TF_ShutdownTPUExecution(TF_Session* session,
+                                                   TF_Output shutdown_node,
+                                                   TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
-- 
GitLab


From 9037e241de1e64044ff55ab539ccc1fb013c178a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 14 Mar 2018 20:39:10 -0700
Subject: [PATCH 049/960] Enable Add/AddN tree rewrite for symbolically equal
 shapes.

1) Rewrite a tree of Add/AddN ops with a single AddN,
   if all shapes are symbolically equal
2) Lookup shape properties using GraphProperties instead
   of direct access to Node attributes

PiperOrigin-RevId: 189131726
---
 .../optimizers/arithmetic_optimizer.cc        | 173 ++++++++++--------
 .../optimizers/arithmetic_optimizer.h         |   3 +
 .../optimizers/arithmetic_optimizer_test.cc   |  61 +++++-
 tensorflow/core/grappler/utils.cc             |  26 +++
 tensorflow/core/grappler/utils.h              |   4 +
 tensorflow/core/grappler/utils_test.cc        |  41 +++++
 6 files changed, 231 insertions(+), 77 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index c0fcfaf428..675cd8f072 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -197,35 +197,39 @@ bool IsNumberType(DataType dtype) { return kNumberTypes.Contains(dtype); }
 
 const char kOutputShapesAttr[] = "_output_shapes";
 
-PartialTensorShape GetInputShape(const string& input, const NodeMap& node_map) {
-  int output_pos;
-  string node_name = ParseNodeName(input, &output_pos);
-  const NodeDef* input_node = node_map.GetNode(node_name);
-  auto attr = input_node->attr();
-  if (attr.find(kOutputShapesAttr) == attr.end()) {
-    return PartialTensorShape();  // unknown shape
-  } else {
-    return attr.at(kOutputShapesAttr).list().shape(output_pos);
-  }
+// Shape is symbolically defined if it has a known rank, and each dimension is
+// defined, or is an unknown symbol (dim.size <= -2).
+bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
+  return !shape.unknown_rank() &&
+         std::all_of(
+             shape.dim().begin(), shape.dim().end(),
+             [](const TensorShapeProto::Dim& dim) { return dim.size() != -1; });
+}
+
+bool ShapeIsSymbolicallyDefined(const OpInfo::TensorProperties& properties) {
+  return ShapeIsSymbolicallyDefined(properties.shape());
 }
 
-bool ShapesEqual(const string& input_x, const string& input_y,
-                 const NodeMap& node_map) {
-  PartialTensorShape x_shape = GetInputShape(input_x, node_map);
-  PartialTensorShape y_shape = GetInputShape(input_y, node_map);
-  if (x_shape.unknown_rank() || y_shape.unknown_rank() ||
-      x_shape.dims() != y_shape.dims()) {
+bool ShapesSymbolicallyEqual(const TensorShapeProto& left,
+                             const TensorShapeProto& right) {
+  if (left.unknown_rank() || right.unknown_rank() ||
+      left.dim_size() != right.dim_size()) {
     return false;
   }
-  for (int i = 0; i < x_shape.dims(); ++i) {
-    if (x_shape.dim_size(i) == -1 || y_shape.dim_size(i) == -1 ||
-        x_shape.dim_size(i) != y_shape.dim_size(i)) {
+  for (int i = 0; i < left.dim_size(); ++i) {
+    if (left.dim(i).size() == -1 || right.dim(i).size() == -1 ||
+        left.dim(i).size() != right.dim(i).size()) {
       return false;
     }
   }
   return true;
 }
 
+bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
+                             const OpInfo::TensorProperties& right) {
+  return ShapesSymbolicallyEqual(left.shape(), right.shape());
+}
+
 // Returns whether `reshape` is an identity op. The tensor that `reshape`
 // reshapes is the `output_pos`-th output of node `input`.
 bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
@@ -290,16 +294,19 @@ NodeDef* GetTailOfValuePreservingChain(
 struct ArithmeticOptimizerContext {
   ArithmeticOptimizerContext(
       const std::unordered_set<string>* nodes_to_preserve,
-      GraphDef* optimized_graph, NodeMap* node_map, FrameMap* frame_map,
+      GraphDef* optimized_graph, GraphProperties* graph_properties,
+      NodeMap* node_map, FrameMap* frame_map,
       SetVector<NodeDef*>* nodes_to_simplify)
       : nodes_to_preserve(nodes_to_preserve),
         optimized_graph(optimized_graph),
+        graph_properties(graph_properties),
         node_map(node_map),
         frame_map(frame_map),
         nodes_to_simplify(nodes_to_simplify) {}
 
   const std::unordered_set<string>* nodes_to_preserve;
   GraphDef* optimized_graph;
+  GraphProperties* graph_properties;
   NodeMap* node_map;
   FrameMap* frame_map;
   SetVector<NodeDef*>* nodes_to_simplify;
@@ -388,7 +395,7 @@ class ArithmeticOptimizerStage {
     ctx_.nodes_to_simplify->PushBack(node);
   }
 
-  // Get a node by input name from a node map. Return a error if node was not
+  // Get a node by input name from a node map. Return an error if node was not
   // found.
   Status GetInputNode(const string& input, NodeDef** node) const {
     string node_name = NodeName(input);
@@ -401,22 +408,31 @@ class ArithmeticOptimizerStage {
     return Status::OK();
   }
 
-  // Get input shape from a node map. If node doesn't exists return unknown
-  // shape.
-  PartialTensorShape GetInputShape(const string& input) const {
-    int position;
-    string node_name = ParseNodeName(input, &position);
-    NodeDef* node;
-    Status node_status = GetInputNode(node_name, &node);
-    if (!node_status.ok()) {
-      return PartialTensorShape();  // unknown shape
+  // Lookup tensor properties by name. Tensor name might have non-zero port
+  // number. Return an error if tensor node doesn't exists in a graph, or it
+  // doesn't have properties defined for requested port.
+  Status GetTensorProperties(const string& tensor,
+                             OpInfo::TensorProperties* properties) const {
+    int port;
+    string tensor_node_name = ParseNodeName(tensor, &port);
+    if (port < 0) {
+      return errors::InvalidArgument(
+          "Can't get tensor properties of control dependency ", tensor);
     }
-    auto attr = node->attr();
-    if (attr.find(kOutputShapesAttr) == attr.end()) {
-      return PartialTensorShape();  // unknown shape
-    } else {
-      return attr.at(kOutputShapesAttr).list().shape(position);
+
+    const auto& output_properties =
+        ctx_.graph_properties->GetOutputProperties(tensor_node_name);
+    auto num_outputs = output_properties.size();
+
+    if (num_outputs == 0 || port > num_outputs - 1) {
+      return errors::InvalidArgument(
+          "Node ", tensor_node_name,
+          " is missing output properties at position :", port,
+          " (num_outputs=", num_outputs, ")");
     }
+
+    properties->CopyFrom(output_properties[port]);
+    return Status::OK();
   }
 
   NodeDef* AddCopyNode(const string& name, const NodeDef* node_to_copy) {
@@ -509,8 +525,8 @@ class ArithmeticOptimizerStage {
 // Rewrite a tree of Add/AddN with a single AddN operation, consuming all the
 // original inputs of absorbed nodes.
 //
-// All nodes in a Add/AddN subgraph must have fully specified and identical
-// shape. All nodes must have the same device placement.
+// All nodes in a Add/AddN subgraph must have symbolically equal shape. All
+// nodes must have the same device placement.
 //
 // Example:
 //                AddN_1
@@ -533,16 +549,12 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     if (!IsRewritable(node)) {
       return false;
     }
-    // and must have fully defined shape
-    // TODO(ezhulenev): support partially defined shapes, when we can prove that
-    // unknown dimensions in the rewritten subgraph are the same.
-    PartialTensorShape shape = GetInputShape(node->name());
-    if (!shape.IsFullyDefined()) {
-      return false;
-    }
-    // and must have inputs of fully defined shape identical to the output
-    // TODO(ezhulenev): relax this condition to support equal unknown dimensions
-    return HasAllInputsOfIdenticalShape(*node, shape);
+
+    // shape must be symbolically defined and all inputs compatible with it
+    OpInfo::TensorProperties properties;
+    Status has_properties = GetTensorProperties(node->name(), &properties);
+    return has_properties.ok() && ShapeIsSymbolicallyDefined(properties) &&
+           HasAllInputsOfSymbolicallyEqualShape(*node, properties);
   }
 
   Status TrySimplify(const NodeDef* node,
@@ -567,23 +579,26 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
   //   input_nodes: [x, y, z, w, q, e]
   struct AddOpsGroup {
     const NodeDef* root_node;
-    PartialTensorShape root_shape;
+    TensorShapeProto root_shape;
     // Add/AddN operations below the root level that were absorbed by this group
     std::vector<NodeDef*> absorbed_nodes;
     // Inputs of absorbed nodes that will be forwarded to rewritten AddN node
     std::vector<string> inputs;
   };
 
-  // Check if all inputs are fully defined and identical to expected shape
-  bool HasAllInputsOfIdenticalShape(const NodeDef& node,
-                                    const PartialTensorShape& shape) const {
+  // Check if all inputs have symbolically equal shapes
+  bool HasAllInputsOfSymbolicallyEqualShape(
+      const NodeDef& node, const OpInfo::TensorProperties& properties) const {
     const AddOpsRewriteStage* self = this;
-    return std::all_of(node.input().begin(), node.input().end(),
-                       [self, &shape](const string& input) {
-                         auto input_shape = self->GetInputShape(input);
-                         return input_shape.IsFullyDefined() &&
-                                input_shape.IsIdenticalTo(shape);
-                       });
+    return std::all_of(
+        node.input().begin(), node.input().end(),
+        [self, &properties](const string& input) {
+          OpInfo::TensorProperties input_properties;
+          Status has_input_properties =
+              self->GetTensorProperties(input, &input_properties);
+          return has_input_properties.ok() &&
+                 ShapesSymbolicallyEqual(properties, input_properties);
+        });
   }
 
   // TODO(ezhulenev): use GraphRewriter?
@@ -614,27 +629,25 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     if (!node_status.ok()) {
       return false;
     }
-
-    PartialTensorShape shape = GetInputShape(name);
-    CHECK(shape.IsIdenticalTo(group.root_shape))
-        << "Cannot absorb a node of incompatible shape";
-
     // check basic preconditions
     if (!IsRewritable(node)) {
       return false;
     }
-    // with a single output consumer (presumably if we reach this node from
+    // with a single output data consumer (presumably if we reach this node from
     // previously absorbed or a root node, it means that this node is not used
     // as an input to any other op, outside of the group)
-    if (ctx_.node_map->GetOutputs(node->name()).size() != 1) {
+    if (NumNonControlDataOutputs(*node, *ctx_.node_map) != 1) {
       return false;
     }
     // must be on the same device as a root node
     if (node->device() != group.root_node->device()) {
       return false;
     }
-    // All input shapes must be fully defined and equal to the node shape
-    return HasAllInputsOfIdenticalShape(*node, shape);
+    // All input shapes must be symbolically defined and equal to the node shape
+    OpInfo::TensorProperties properties;
+    Status has_properties = GetTensorProperties(name, &properties);
+    return has_properties.ok() &&
+           HasAllInputsOfSymbolicallyEqualShape(*node, properties);
   }
 
   // Node requirements both for a root node and an absorbed node
@@ -660,15 +673,19 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
   }
 
   // Check that optimized group node name doesn't exists. It might happen if
-  // graph optimized multiple times without pruning beween invocations.
+  // graph optimized multiple times without pruning between invocations.
   bool IsRewritten(const AddOpsGroup& group) const {
     return ctx_.node_map->NodeExists(AddOpsGroupName(group));
   }
 
   // Create an AddOpsGroup with a root in a given node
   Status CreateAddOpsGroup(const NodeDef* root_node, AddOpsGroup* group) {
+    OpInfo::TensorProperties root_node_output_properties;
+    TF_RETURN_IF_ERROR(
+        GetTensorProperties(root_node->name(), &root_node_output_properties));
+
     group->root_node = root_node;
-    group->root_shape = GetInputShape(root_node->name());
+    group->root_shape = root_node_output_properties.shape();
 
     group->absorbed_nodes.reserve(root_node->input_size());
     for (int i = 0; i < root_node->input_size(); ++i) {
@@ -737,6 +754,9 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
       added_node->add_input(input);
     }
 
+    // Add frame dependencies that the original node might have had.
+    AddFrameControlDeps(group.root_node, {added_node}, "", {});
+
     VLOG(1) << "Absorbed " << group.absorbed_nodes.size()
             << " Add/AddN nodes from the graph";
 
@@ -891,8 +911,11 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
           mul_node->input(0) == common_factor ? 1 : 0;
       unique_factors->push_back(mul_node->input(unique_factor_index));
       if (i > 0 && !IsAdd(*node)) {
-        *shapes_match = ShapesEqual(unique_factors->front(),
-                                    unique_factors->back(), *ctx_.node_map);
+        OpInfo::TensorProperties lhs;
+        OpInfo::TensorProperties rhs;
+        TF_RETURN_IF_ERROR(GetTensorProperties(unique_factors->front(), &lhs));
+        TF_RETURN_IF_ERROR(GetTensorProperties(unique_factors->back(), &rhs));
+        *shapes_match = ShapesSymbolicallyEqual(lhs, rhs);
       }
     }
     return Status::OK();
@@ -1627,8 +1650,8 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
   }
 
   const ArithmeticOptimizerContext ctx(&nodes_to_preserve_, optimized_graph_,
-                                       node_map_.get(), &frame_map_,
-                                       &nodes_to_simplify);
+                                       graph_properties_.get(), node_map_.get(),
+                                       &frame_map_, &nodes_to_simplify);
 
   std::vector<std::unique_ptr<ArithmeticOptimizerStage>> stages;
 
@@ -1660,8 +1683,10 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     const NodeDef* node = nodes_to_simplify.PopBack();
 
     // TODO(ezhulenev): move all rewrites into separate stages
-    string simplified_tensor =
-        TrySimplifyAndReplaceUses(node, &nodes_to_simplify);
+    string simplified_tensor = "";
+    if (options_.enable_try_simplify_and_replace) {
+      simplified_tensor = TrySimplifyAndReplaceUses(node, &nodes_to_simplify);
+    }
 
     // if it was not simplified try to run it through all configured stages
     if (simplified_tensor.empty()) {
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index d5a7af5ba6..2c6b52c072 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -55,6 +55,9 @@ class ArithmeticOptimizer : public GraphOptimizer {
 
   // Granular control for arithmetic optimizer stages
   struct ArithmeticOptimizerOptions {
+    // TODO(ezhulenev): flag do disable TrySimplifyAndReplaceUses in tests.
+    // Remove when all optimizers will be migrated to separate stages.
+    bool enable_try_simplify_and_replace = true;
     bool combine_add_to_addn = true;
     bool hoist_common_factor_out_of_aggregation = true;
     bool remove_inverse_transpose = true;
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index e1f47625c1..d677aee589 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -89,6 +89,7 @@ class ArithmeticOptimizerTest : public GrapplerTest {
   // should explicitly enable required optimization for tests isolation
   void DisableAllStages(ArithmeticOptimizer* optimizer) {
     ArithmeticOptimizer::ArithmeticOptimizerOptions options;
+    options.enable_try_simplify_and_replace = false;
     options.combine_add_to_addn = false;
     options.hoist_common_factor_out_of_aggregation = false;
     options.remove_inverse_transpose = false;
@@ -1270,7 +1271,7 @@ TEST_F(ArithmeticOptimizerTest, RemoveRedundantCast) {
   EXPECT_TRUE(IsNodesDirectlyConnected(node_map, "inputs", "outputs"));
 }
 
-TEST_F(ArithmeticOptimizerTest, AddOpsRewriteCollapseAddsOfIdenticalShape) {
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfIdenticalShape) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   tensorflow::Scope sx = s.NewSubScope("x");
   tensorflow::Scope sy = s.NewSubScope("y");
@@ -1322,7 +1323,7 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewriteCollapseAddsOfIdenticalShape) {
   EXPECT_EQ(collapsed_add->name(), updated_outputs->input(0));
 }
 
-TEST_F(ArithmeticOptimizerTest, AddOpsRewriteMultiplePasses) {
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MultiplePasses) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
 
   auto a = ops::Variable(s.WithOpName("a"), {2, 2}, DT_FLOAT);
@@ -1395,7 +1396,7 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewriteMultiplePasses) {
   EXPECT_EQ(collapsed_right->name(), updated_mul->input(1));
 }
 
-TEST_F(ArithmeticOptimizerTest, AddOpsRewriteAddInputThroughMultiplePaths) {
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddInputMultipleTimes) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
 
   auto a = ops::Variable(s.WithOpName("a"), {2, 2}, DT_FLOAT);
@@ -1440,5 +1441,59 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewriteAddInputThroughMultiplePaths) {
   EXPECT_EQ("c", collapsed_add->input(3));
 }
 
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfSymbolicallyEqualShape) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  // unknown input shape propagated symbolically through the graph
+  auto input = ops::Variable(s.WithOpName("input"), {-1, 2}, DT_FLOAT);
+
+  // [a, b, c] have symbolically equal shapes
+  auto a = ops::Sqrt(s.WithOpName("a"), input);
+  auto b = ops::Square(s.WithOpName("b"), input);
+  auto c = ops::Round(s.WithOpName("c"), input);
+
+  // [add_ab, add_abc] shape must be inferred from inputs
+  auto add_ab = ops::Add(s.WithOpName("Add_ab"), a, b);
+  auto add_abc = ops::Add(s.WithOpName("Add_abc"), add_ab, c);
+
+  auto outputs = ops::Identity(s.WithOpName("outputs"), add_abc);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyAddToAddNCombining(&optimizer);
+
+  OptimizeAndPrune(&optimizer, &item, &output);
+
+  // We expect the following rewrite(s) to occur:
+  //
+  //     +
+  //    / \
+  //   +   c      -->    AddN(a, b, c)
+  //  / \
+  // a   b
+  EXPECT_EQ(6, output.node_size());
+
+  NodeMap node_map(&output);
+
+  // check add tree was replaced with AddN
+  const NodeDef* collapsed_add =
+      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_abc_Add_ab");
+  ASSERT_TRUE(collapsed_add != nullptr);
+  EXPECT_EQ("AddN", collapsed_add->op());
+  EXPECT_EQ(3, collapsed_add->input_size());
+  EXPECT_EQ("a", collapsed_add->input(0));
+  EXPECT_EQ("b", collapsed_add->input(1));
+  EXPECT_EQ("c", collapsed_add->input(2));
+
+  // check output was re-wired to new node
+  const NodeDef* updated_outputs = node_map.GetNode("outputs");
+  ASSERT_TRUE(updated_outputs != nullptr);
+  EXPECT_EQ(collapsed_add->name(), updated_outputs->input(0));
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index eb1f882ff1..829bfe9e31 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -40,6 +40,16 @@ bool SafeSetScalarTensorValue(double value, Tensor* tensor) {
   tensor->flat<T>()(0) = static_cast<T>(value);
   return true;
 }
+
+// Is 'node' an operator that consumes only the shape of its input, not the
+// data itself?
+// TODO(ezhulenev): move to op_types.h. Requires to break circular dependency.
+// TODO(ezhulenev): what about Identity passing tensor to Shape consumer?
+bool IsShapeConsumer(const NodeDef& node) {
+  const string& op = node.op();
+  return op == "Shape" || op == "ShapeN" || op == "Rank" || op == "Size";
+}
+
 }  // namespace
 
 NodeMap::NodeMap(GraphDef* graph) {
@@ -270,6 +280,22 @@ int NumNonControlOutputs(const NodeDef& node, const NodeMap& node_map) {
   return num_outputs;
 }
 
+int NumNonControlDataOutputs(const NodeDef& node, const NodeMap& node_map) {
+  int num_data_outputs = 0;
+  for (const NodeDef* output : node_map.GetOutputs(node.name())) {
+    if (IsShapeConsumer(*output)) continue;
+
+    for (int i = 0; i < output->input_size(); ++i) {
+      const string& input = output->input(i);
+      if (!IsControlInput(input) && NodeName(input) == node.name()) {
+        ++num_data_outputs;
+        break;
+      }
+    }
+  }
+  return num_data_outputs;
+}
+
 // Returns the data type in attribute `attr_name` of `node`. If that attribute
 // doesn't exist, returns DT_INVALID.
 DataType GetDataTypeFromAttr(const NodeDef& node, const string& attr_name) {
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index fbd38c1531..7aa31939f5 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -144,6 +144,10 @@ int NumNonControlInputs(const NodeDef& node);
 // Number of connected non-control outputs.
 int NumNonControlOutputs(const NodeDef& node, const NodeMap& node_map);
 
+// Number of connected non-control data outputs (Ops that consume output tensor
+// data, not just it's shape).
+int NumNonControlDataOutputs(const NodeDef& node, const NodeMap& node_map);
+
 // Removes redundant control inputs from node.
 void DedupControlInputs(NodeDef* node);
 
diff --git a/tensorflow/core/grappler/utils_test.cc b/tensorflow/core/grappler/utils_test.cc
index eabce5b5ee..49a1996d25 100644
--- a/tensorflow/core/grappler/utils_test.cc
+++ b/tensorflow/core/grappler/utils_test.cc
@@ -292,6 +292,47 @@ TEST_F(UtilsTest, DedupControlInputs) {
   EXPECT_EQ("gnu", foo.input(1));
 }
 
+TEST_F(UtilsTest, NumNonControlOutputs) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  //  *) Round node has control dependency edge from Add, which
+  //     is not on this scheme (ASCII graphics limitation).
+  //
+  //   *Round    [Sqrt, Shape]
+  //      |           |
+  //      |   ctrl    |
+  //     Mul ------> Add
+  //     / \         / \
+  //    x   y       a   b
+  auto x = ops::Variable(s.WithOpName("x"), {1, 2}, DT_FLOAT);
+  auto y = ops::Variable(s.WithOpName("y"), {1, 2}, DT_FLOAT);
+  auto a = ops::Variable(s.WithOpName("a"), {1, 2}, DT_FLOAT);
+  auto b = ops::Variable(s.WithOpName("b"), {1, 2}, DT_FLOAT);
+
+  auto mul = ops::Multiply(s.WithOpName("mul"), x, y);
+  auto add = ops::Add(s.WithOpName("add").WithControlDependencies(mul), a, b);
+
+  auto shape = ops::Shape(s.WithOpName("shape"), add);
+  auto sqrt = ops::Sqrt(s.WithOpName("sqrt"), add);
+
+  auto round =
+      ops::Round(s.WithOpName("round").WithControlDependencies(add), mul);
+
+  GraphDef graph;
+  TF_CHECK_OK(s.ToGraphDef(&graph));
+  NodeMap node_map(&graph);
+
+  const NodeDef* add_node = node_map.GetNode("add");
+  ASSERT_TRUE(add_node != nullptr);
+
+  // [a, b] are only non-control inputs
+  EXPECT_EQ(2, NumNonControlInputs(*add_node));
+  // [sqrt, shape] are non control outputs
+  EXPECT_EQ(2, NumNonControlOutputs(*add_node, node_map));
+  // sqrt is the only data output
+  EXPECT_EQ(1, NumNonControlDataOutputs(*add_node, node_map));
+}
+
 TEST_F(UtilsTest, DeleteNodes) {}
 
 }  // namespace
-- 
GitLab


From b08c54271084b05ea822b3348a3a448a9fe3b898 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Mar 2018 02:22:17 -0700
Subject: [PATCH 050/960] [SE] [XLA:GPU] Inform --xla_hlo_profile of the GPU's
 memory bandwidth.

Add a memory_bandwidth() property to StreamExecutor's DeviceDescription,
and use this in the GPU's --xla_hlo_profile.

PiperOrigin-RevId: 189157407
---
 tensorflow/compiler/xla/service/gpu/gpu_compiler.cc |  2 ++
 tensorflow/stream_executor/cuda/cuda_driver.cc      | 13 +++++++++++++
 tensorflow/stream_executor/cuda/cuda_driver.h       | 10 +++++++++-
 .../stream_executor/cuda/cuda_gpu_executor.cc       | 12 ++++++++++++
 tensorflow/stream_executor/device_description.cc    |  3 +++
 tensorflow/stream_executor/device_description.h     |  9 +++++++++
 6 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
index c67b552abb..07be2a0cf9 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_compiler.cc
@@ -671,6 +671,8 @@ StatusOr<std::unique_ptr<Executable>> GpuCompiler::RunBackend(
 
   if (module->config().hlo_profiling_enabled()) {
     HloCostAnalysis cost_analysis(ShapeSizeBytesFunction());
+    cost_analysis.set_bytes_per_second(
+        stream_exec->GetDeviceDescription().memory_bandwidth());
     TF_RETURN_IF_ERROR(module->entry_computation()->Accept(&cost_analysis));
     profile_index_map = MakeUnique<HloProfileIndexMap>(*module);
     profile_printer =
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.cc b/tensorflow/stream_executor/cuda/cuda_driver.cc
index a017ff64d4..58e1e58c59 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.cc
+++ b/tensorflow/stream_executor/cuda/cuda_driver.cc
@@ -1503,6 +1503,19 @@ static port::StatusOr<T> GetSimpleAttribute(CUdevice device,
   return true;
 }
 
+/* static */ port::StatusOr<int> CUDADriver::GetDeviceAttribute(
+    CUdevice_attribute attribute, CUdevice device) {
+  int val;
+  CUresult res = cuDeviceGetAttribute(&val, attribute, device);
+  if (res != CUDA_SUCCESS) {
+    return port::Status{
+        port::error::INTERNAL,
+        port::Printf("failed to get device attribute %d for device %d: %s",
+                     attribute, device, ToString(res).c_str())};
+  }
+  return val;
+}
+
 /* static */ bool CUDADriver::IsEccEnabled(CUdevice device, bool *result) {
   int value = -1;
   CUresult res =
diff --git a/tensorflow/stream_executor/cuda/cuda_driver.h b/tensorflow/stream_executor/cuda/cuda_driver.h
index 4002ba2021..fa9172b3f0 100644
--- a/tensorflow/stream_executor/cuda/cuda_driver.h
+++ b/tensorflow/stream_executor/cuda/cuda_driver.h
@@ -400,12 +400,20 @@ class CUDADriver {
 
   // Returns a grab-bag of device properties in a caller-owned device_properties
   // structure for device_ordinal via cuDeviceGetProperties.
-  // This call is deprecated in the NVIDIA driver API.
+  //
+  // This call is deprecated in the NVIDIA driver API; its replacement is
+  // GetDeviceAttribute
   //
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE__DEPRECATED.html#group__CUDA__DEVICE__DEPRECATED_1g65a5b4e25186bd257df80b98c98cffe6
   static bool GetDeviceProperties(CUdevprop *device_properties,
                                   int device_ordinal);
 
+  // Gets a specific integer-valued property about the given device.
+  //
+  // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
+  static port::StatusOr<int> GetDeviceAttribute(CUdevice_attribute attribute,
+                                                CUdevice device);
+
   // Returns whether ECC is enabled for the given CUdevice via
   // cuDeviceGetattribute with CU_DEVICE_ATTRIBUTE_ECC_ENABLED.
   // http://docs.nvidia.com/cuda/cuda-driver-api/group__CUDA__DEVICE.html#group__CUDA__DEVICE_1g9c3e1414f0ad901d3278a4d6645fc266
diff --git a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
index 4bbd531e14..5ecaf46b8c 100644
--- a/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
+++ b/tensorflow/stream_executor/cuda/cuda_gpu_executor.cc
@@ -1103,6 +1103,18 @@ DeviceDescription *CUDAExecutor::PopulateDeviceDescription() const {
     builder.set_device_memory_size(device_memory_size);
   }
 
+  port::StatusOr<int> mem_clock_khz = CUDADriver::GetDeviceAttribute(
+      CU_DEVICE_ATTRIBUTE_MEMORY_CLOCK_RATE, device_ordinal_);
+  port::StatusOr<int> mem_bus_width_bits = CUDADriver::GetDeviceAttribute(
+      CU_DEVICE_ATTRIBUTE_GLOBAL_MEMORY_BUS_WIDTH, device_ordinal_);
+  if (mem_clock_khz.ok() && mem_bus_width_bits.ok()) {
+    // Times 2 because HBM is DDR memory; it gets two data bits per each data
+    // lane.
+    builder.set_memory_bandwidth(2 * int64_t{mem_clock_khz.ValueOrDie()} *
+                                 1000 *
+                                 int64_t{mem_bus_width_bits.ValueOrDie()} / 8);
+  }
+
   {
     BlockDim block_dim_limit;
     FillBlockDimLimit(&block_dim_limit);
diff --git a/tensorflow/stream_executor/device_description.cc b/tensorflow/stream_executor/device_description.cc
index a98143e34b..52f5319a3b 100644
--- a/tensorflow/stream_executor/device_description.cc
+++ b/tensorflow/stream_executor/device_description.cc
@@ -50,6 +50,7 @@ DeviceDescription::DeviceDescription()
       shared_memory_alloc_granularity_(1),
       device_address_bits_(kUninitializedUint64),
       device_memory_size_(kUninitializedUint64),
+      memory_bandwidth_(kUninitializedUint64),
       shared_memory_per_core_(kUninitializedUint64),
       shared_memory_per_block_(kUninitializedUint64),
       clock_rate_ghz_(-1.0),
@@ -85,6 +86,8 @@ std::unique_ptr<std::map<string, string>> DeviceDescription::ToMap() const {
   result["Device Address Bits"] = port::StrCat(device_address_bits());
   result["Device Memory Size"] =
       port::HumanReadableNumBytes::ToString(device_memory_size());
+  result["Memory Bandwidth"] = port::StrCat(
+      port::HumanReadableNumBytes::ToString(memory_bandwidth_), "/s");
 
   result["Shared Memory Per Core"] =
       port::HumanReadableNumBytes::ToString(shared_memory_per_core_);
diff --git a/tensorflow/stream_executor/device_description.h b/tensorflow/stream_executor/device_description.h
index f2b35bcb43..fcf0928096 100644
--- a/tensorflow/stream_executor/device_description.h
+++ b/tensorflow/stream_executor/device_description.h
@@ -140,6 +140,11 @@ class DeviceDescription {
   // Returns the device memory size in bytes.
   uint64 device_memory_size() const { return device_memory_size_; }
 
+  // Returns the device's memory bandwidth in bytes/sec.  (This is for
+  // reads/writes to/from the device's own memory, not for transfers between the
+  // host and device.)
+  uint64 memory_bandwidth() const { return memory_bandwidth_; }
+
   // Returns the device's core clock rate in GHz.
   float clock_rate_ghz() const { return clock_rate_ghz_; }
 
@@ -212,6 +217,7 @@ class DeviceDescription {
 
   uint64 device_address_bits_;
   uint64 device_memory_size_;
+  uint64 memory_bandwidth_;
 
   // Shared memory limits on a given device.
   uint64 shared_memory_per_core_;
@@ -305,6 +311,9 @@ class DeviceDescriptionBuilder {
   void set_device_memory_size(uint64 value) {
     device_description_->device_memory_size_ = value;
   }
+  void set_memory_bandwidth(uint64 value) {
+    device_description_->memory_bandwidth_ = value;
+  }
 
   void set_shared_memory_per_core(int64 value) {
     device_description_->shared_memory_per_core_ = value;
-- 
GitLab


From 10927e9f77d0bfebb597b5cc64fa3908db23361c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 06:56:23 -0700
Subject: [PATCH 051/960] Add ability to use feature_columns in KMeans
 Estimator.

PiperOrigin-RevId: 189179304
---
 tensorflow/contrib/factorization/BUILD        |  2 +
 .../factorization/python/ops/kmeans.py        | 49 +++++++++++++------
 .../factorization/python/ops/kmeans_test.py   | 37 ++++++++++----
 3 files changed, 62 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index 180f1b68f3..90f10f1fa8 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -66,6 +66,7 @@ tf_custom_op_py_library(
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:model_fn",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
     ],
 )
@@ -238,6 +239,7 @@ py_test(
         "//tensorflow/python:random_ops",
         "//tensorflow/python:training",
         "//tensorflow/python/estimator:run_config",
+        "//tensorflow/python/feature_column:feature_column_py",
         "//third_party/py/numpy",
     ],
 )
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
index c092f85d35..38faca119d 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -26,6 +26,7 @@ from tensorflow.contrib.factorization.python.ops import clustering_ops
 from tensorflow.python.estimator import estimator
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator.export import export_output
+from tensorflow.python.feature_column import feature_column as fc
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -105,24 +106,32 @@ class _InitializeClustersHook(session_run_hook.SessionRunHook):
         logging.info(e)
 
 
-def _parse_features_if_necessary(features):
+def _parse_features_if_necessary(features, feature_columns):
   """Helper function to convert the input points into a usable format.
 
   Args:
-    features: The input points.
+    features: The input features.
+    feature_columns: An optionable iterable containing all the feature columns
+      used by the model. All items in the set should be feature column instances
+      that can be passed to `tf.feature_column.input_layer`. If this is None,
+      all features will be used.
 
   Returns:
-    If `features` is a dict of `k` features, each of which is a vector of `n`
-    scalars, the return value is a Tensor of shape `(n, k)` representing `n`
-    input points, where the items in the `k` dimension are sorted
-    lexicographically by `features` key. If `features` is not a dict, it is
-    returned unmodified.
+    If `features` is a dict of `k` features (optionally filtered by
+    `feature_columns`), each of which is a vector of `n` scalars, the return
+    value is a Tensor of shape `(n, k)` representing `n` input points, where the
+    items in the `k` dimension are sorted lexicographically by `features` key.
+    If `features` is not a dict, it is returned unmodified.
   """
-  if isinstance(features, dict):
-    keys = sorted(features.keys())
-    with ops.colocate_with(features[keys[0]]):
-      features = array_ops.concat([features[k] for k in keys], axis=1)
-  return features
+  if not isinstance(features, dict):
+    return features
+
+  if feature_columns:
+    return fc.input_layer(features, feature_columns)
+
+  keys = sorted(features.keys())
+  with ops.colocate_with(features[keys[0]]):
+    return array_ops.concat([features[k] for k in keys], axis=1)
 
 
 class _ModelFn(object):
@@ -130,7 +139,8 @@ class _ModelFn(object):
 
   def __init__(self, num_clusters, initial_clusters, distance_metric,
                random_seed, use_mini_batch, mini_batch_steps_per_iteration,
-               kmeans_plus_plus_num_retries, relative_tolerance):
+               kmeans_plus_plus_num_retries, relative_tolerance,
+               feature_columns):
     self._num_clusters = num_clusters
     self._initial_clusters = initial_clusters
     self._distance_metric = distance_metric
@@ -139,6 +149,7 @@ class _ModelFn(object):
     self._mini_batch_steps_per_iteration = mini_batch_steps_per_iteration
     self._kmeans_plus_plus_num_retries = kmeans_plus_plus_num_retries
     self._relative_tolerance = relative_tolerance
+    self._feature_columns = feature_columns
 
   def model_fn(self, features, mode, config):
     """Model function for the estimator.
@@ -166,7 +177,7 @@ class _ModelFn(object):
     # input_points is a single Tensor. Therefore, the sharding functionality
     # in clustering_ops is unused, and some of the values below are lists of a
     # single item.
-    input_points = _parse_features_if_necessary(features)
+    input_points = _parse_features_if_necessary(features, self._feature_columns)
 
     # Let N = the number of input_points.
     # all_distances: A list of one matrix of shape (N, num_clusters). Each value
@@ -316,7 +327,8 @@ class KMeansClustering(estimator.Estimator):
                mini_batch_steps_per_iteration=1,
                kmeans_plus_plus_num_retries=2,
                relative_tolerance=None,
-               config=None):
+               config=None,
+               feature_columns=None):
     """Creates an Estimator for running KMeans training and inference.
 
     This Estimator implements the following variants of the K-means algorithm:
@@ -383,6 +395,10 @@ class KMeansClustering(estimator.Estimator):
         iterations. Stops learning if the loss changes less than this amount.
         This may not work correctly if `use_mini_batch=True`.
       config: See @{tf.estimator.Estimator}.
+      feature_columns: An optionable iterable containing all the feature columns
+        used by the model. All items in the set should be feature column
+        instances that can be passed to `tf.feature_column.input_layer`. If this
+        is None, all features will be used.
 
     Raises:
       ValueError: An invalid argument was passed to `initial_clusters` or
@@ -402,7 +418,8 @@ class KMeansClustering(estimator.Estimator):
         model_fn=_ModelFn(
             num_clusters, initial_clusters, distance_metric, random_seed,
             use_mini_batch, mini_batch_steps_per_iteration,
-            kmeans_plus_plus_num_retries, relative_tolerance).model_fn,
+            kmeans_plus_plus_num_retries, relative_tolerance,
+            feature_columns).model_fn,
         model_dir=model_dir,
         config=config)
 
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 06a2c52c11..0103cc4439 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -27,6 +27,7 @@ from sklearn.cluster import KMeans as SklearnKMeans
 # pylint: disable=g-import-not-at-top
 from tensorflow.contrib.factorization.python.ops import kmeans as kmeans_lib
 from tensorflow.python.estimator import run_config
+from tensorflow.python.feature_column import feature_column as fc
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -226,27 +227,43 @@ class KMeansTest(KMeansTestBase):
     self._infer_helper(kmeans, clusters, 10)
     self._infer_helper(kmeans, clusters, 1)
 
+  def _parse_feature_dict_helper(self, features, parsed_feature_dict):
+    # Perform a sanity check.
+    self.assertEqual(features.shape, parsed_feature_dict.shape)
+    self.assertEqual(features.dtype, parsed_feature_dict.dtype)
+    # Then check that running the tensor yields the original list of points.
+    with self.test_session() as sess:
+      parsed_points = sess.run(parsed_feature_dict)
+      self.assertAllEqual(self.points, parsed_points)
+
   def test_parse_features(self):
     """Tests the various behaviours of kmeans._parse_features_if_necessary."""
 
     # No-op if a tensor is passed in.
     features = constant_op.constant(self.points)
-    parsed_features = kmeans_lib._parse_features_if_necessary(features)
+    parsed_features = kmeans_lib._parse_features_if_necessary(features, None)
     self.assertAllEqual(features, parsed_features)
 
-    # A dict is transformed into a tensor.
+    # All values from a feature dict are transformed into a tensor.
     feature_dict = {
         'x': [[point[0]] for point in self.points],
         'y': [[point[1]] for point in self.points]
     }
-    parsed_feature_dict = kmeans_lib._parse_features_if_necessary(feature_dict)
-    # Perform a sanity check.
-    self.assertEqual(features.shape, parsed_feature_dict.shape)
-    self.assertEqual(features.dtype, parsed_feature_dict.dtype)
-    # Then check that running the tensor yields the original list of points.
-    with self.test_session() as sess:
-      parsed_points = sess.run(parsed_feature_dict)
-      self.assertAllEqual(self.points, parsed_points)
+    parsed_feature_dict = kmeans_lib._parse_features_if_necessary(
+        feature_dict, None)
+    self._parse_feature_dict_helper(features, parsed_feature_dict)
+
+    # Only the feature_columns of a feature dict are transformed into a tensor.
+    feature_dict_with_extras = {
+        'foo': 'bar',
+        'x': [[point[0]] for point in self.points],
+        'baz': {'fizz': 'buzz'},
+        'y': [[point[1]] for point in self.points]
+    }
+    feature_columns = [fc.numeric_column(key='x'), fc.numeric_column(key='y')]
+    parsed_feature_dict = kmeans_lib._parse_features_if_necessary(
+        feature_dict_with_extras, feature_columns)
+    self._parse_feature_dict_helper(features, parsed_feature_dict)
 
 
 class KMeansTestMultiStageInit(KMeansTestBase):
-- 
GitLab


From 1de48e49000c80d762052abb8173ee040e116293 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 08:09:08 -0700
Subject: [PATCH 052/960] Removed the pointer to model from interpreter.

PiperOrigin-RevId: 189186901
---
 tensorflow/contrib/lite/interpreter.h | 12 ------------
 tensorflow/contrib/lite/model.cc      |  2 --
 2 files changed, 14 deletions(-)

diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index 7c5a195815..af143370ee 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/error_reporter.h"
 #include "tensorflow/contrib/lite/memory_planner.h"
-#include "tensorflow/contrib/lite/schema/schema_generated.h"
 
 namespace tflite {
 
@@ -275,12 +274,6 @@ class Interpreter {
   // contain new nodes that replace 1 more nodes.
   TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
 
-  // WARNING: This is a deprecated interface and will be removed as soon as
-  // possible.  Please do not use it.
-  // TODO(impjdi): Remove this interface after resolving dependencies.
-  void set_model(const Model* model) { model_ = const_cast<Model*>(model); }
-  Model* model() const { return model_; }
-
   // Ensure the data in `tensor.data` is readable. In case delegate is used,
   // it might require to copy the data from delegate buffer to raw memory.
   TfLiteStatus EnsureTensorDataIsReadable(int tensor_index) {
@@ -509,11 +502,6 @@ class Interpreter {
   std::unique_ptr<NNAPIDelegate> nnapi_delegate_;
 
   std::unique_ptr<MemoryPlanner> memory_planner_;
-
-  // WARNING: This is a deprecated interface and will be removed as soon as
-  // possible.  Please do not use it.
-  // TODO(impjdi): Remove this interface after resolving dependencies.
-  Model* model_ = nullptr;
 };
 
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 8c456e70da..3cf6bcbfcd 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -794,8 +794,6 @@ TfLiteStatus InterpreterBuilder::operator()(
     return cleanup_and_error();
   }
 
-  (**interpreter).set_model(model_);
-
   // Parse inputs/outputs
   (**interpreter).SetInputs(FlatBufferIntArrayToVector(subgraph->inputs()));
   (**interpreter).SetOutputs(FlatBufferIntArrayToVector(subgraph->outputs()));
-- 
GitLab


From 1e1f1b8cec776bdfa85840a044e14a6461910dee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 08:37:21 -0700
Subject: [PATCH 053/960] Disable Add/AddN rewrite (temp).

PiperOrigin-RevId: 189189997
---
 tensorflow/core/grappler/optimizers/arithmetic_optimizer.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 2c6b52c072..bd2f42ee8c 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -58,7 +58,7 @@ class ArithmeticOptimizer : public GraphOptimizer {
     // TODO(ezhulenev): flag do disable TrySimplifyAndReplaceUses in tests.
     // Remove when all optimizers will be migrated to separate stages.
     bool enable_try_simplify_and_replace = true;
-    bool combine_add_to_addn = true;
+    bool combine_add_to_addn = false;
     bool hoist_common_factor_out_of_aggregation = true;
     bool remove_inverse_transpose = true;
     bool remove_redundant_bitcast = true;
-- 
GitLab


From de32f18ef3de465bc5391c4951a53f2a25b6ee3f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 09:07:41 -0700
Subject: [PATCH 054/960] Expose setNumThreads in the Java API.

PiperOrigin-RevId: 189193847
---
 .../tensorflow/lite/NativeInterpreterWrapper.java  |  6 ++++++
 .../main/native/nativeinterpreterwrapper_jni.cc    | 10 ++++++++++
 .../src/main/native/nativeinterpreterwrapper_jni.h | 11 +++++++++++
 .../java/org/tensorflow/lite/TestHelper.java       | 14 ++++++++++++++
 4 files changed, 41 insertions(+)

diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
index 014636ffff..518e8b3a96 100644
--- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
+++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
@@ -140,6 +140,10 @@ final class NativeInterpreterWrapper implements AutoCloseable {
     useNNAPI(interpreterHandle, useNNAPI);
   }
 
+  void setNumThreads(int numRecommendedThreads) {
+    numThreads(interpreterHandle, numRecommendedThreads);
+  }
+
   /** Gets index of an input given its name. */
   int getInputIndex(String name) {
     if (inputsIndexes == null) {
@@ -308,6 +312,8 @@ final class NativeInterpreterWrapper implements AutoCloseable {
 
   private static native void useNNAPI(long interpreterHandle, boolean state);
 
+  private static native void numThreads(long interpreterHandle, int numRecommendedThreads);
+
   private static native long createErrorReporter(int size);
 
   private static native long createModel(String modelPathOrBuffer, long errorHandle);
diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
index 2870ffe8eb..21bcff40bd 100644
--- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
+++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
@@ -316,6 +316,16 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_useNNAPI(JNIEnv* env,
   interpreter->UseNNAPI(static_cast<bool>(state));
 }
 
+JNIEXPORT void JNICALL
+Java_org_tensorflow_lite_NativeInterpreterWrapper_numThreads(JNIEnv* env,
+                                                             jclass clazz,
+                                                             jlong handle,
+                                                             jint num_threads) {
+  tflite::Interpreter* interpreter = convertLongToInterpreter(env, handle);
+  if (interpreter == nullptr) return;
+  interpreter->SetNumThreads(static_cast<int>(num_threads));
+}
+
 JNIEXPORT jlong JNICALL
 Java_org_tensorflow_lite_NativeInterpreterWrapper_createErrorReporter(
     JNIEnv* env, jclass clazz, jint size) {
diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
index d611ec7f38..fb76125471 100644
--- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
+++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
@@ -69,6 +69,17 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_useNNAPI(JNIEnv* env,
                                                            jlong handle,
                                                            jboolean state);
 
+/*
+ *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
+ *  Method:
+ *  Signature: (JI)
+ */
+JNIEXPORT void JNICALL
+Java_org_tensorflow_lite_NativeInterpreterWrapper_numThreads(JNIEnv* env,
+                                                             jclass clazz,
+                                                             jlong handle,
+                                                             jint num_threads);
+
 /*
  *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
  *  Method:
diff --git a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
index 3aef0c3bb6..3722e51b3b 100644
--- a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
+++ b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
@@ -33,6 +33,20 @@ public class TestHelper {
     }
   }
 
+  /**
+   * Sets the number of threads for an {@code Interpreter}.
+   *
+   * @param interpreter an instance of {@code Interpreter}. If it is not initialized, an {@code
+   *     IllegalArgumentException} will be thrown.
+   * @param numRecommendedThreads an integer value indicating the number of recommended threads.
+   */
+  public static void setNumThreads(Interpreter interpreter, int numRecommendedThreads) {
+    if (interpreter != null && interpreter.wrapper != null) {
+      interpreter.wrapper.setNumThreads(numRecommendedThreads);
+    } else {
+      throw new IllegalArgumentException("Interpreter has not initialized; Failed to setUseNNAPI.");
+    }
+  }
   /**
    * Gets the last inference duration in nanoseconds. It returns null if there is no previous
    * inference run or the last inference run failed.
-- 
GitLab


From 85297e839dbe4422b6f6e5ef3aff37af397773f9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 09:33:03 -0700
Subject: [PATCH 055/960] Enable constant folding optimizations in loop bodies
 by copying constants across Enter nodes.

PiperOrigin-RevId: 189197514
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/constant_folding.cc   | 135 +++++---
 .../optimizers/constant_folding_test.cc       | 323 ++++++++++--------
 .../core/grappler/utils/grappler_test.cc      |  21 +-
 .../core/grappler/utils/grappler_test.h       |  15 +-
 tensorflow/python/training/saver_test.py      |  11 +-
 6 files changed, 315 insertions(+), 191 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index fe095a725a..ffeaa3835e 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -554,5 +554,6 @@ tf_cc_test(
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
+        "//tensorflow/core/grappler/utils:grappler_test",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 77ccd4ffc8..263983584c 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1535,12 +1535,12 @@ Status ConstantFolding::ReplaceOperationWithConstant(
   return Status::OK();
 }
 
-Status ConstantFolding::SimplifyGraph(GraphDef* output,
+Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
                                       GraphProperties* properties,
                                       bool use_shape_info) {
   const bool is_aggressive = opt_level_ == RewriterConfig::AGGRESSIVE;
-  for (int i = 0; i < output->node_size(); ++i) {
-    NodeDef* node = output->mutable_node(i);
+  for (int i = 0; i < optimized_graph->node_size(); ++i) {
+    NodeDef* node = optimized_graph->mutable_node(i);
 
     // Remove Shuffle or Reverse op over scalar values.
     if (use_shape_info &&
@@ -1554,7 +1554,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         replaceable &= shape.dim(j).size() == 1;
       }
       if (replaceable) {
-        ReplaceOperationWithIdentity(0, node, output);
+        ReplaceOperationWithIdentity(0, node, optimized_graph);
         continue;
       }
     }
@@ -1595,7 +1595,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
           }
         }
         if (replaceable) {
-          ReplaceOperationWithIdentity(0, node, output);
+          ReplaceOperationWithIdentity(0, node, optimized_graph);
           continue;
         }
       }
@@ -1624,7 +1624,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
           }
         }
         if (replaceable) {
-          ReplaceOperationWithIdentity(0, node, output);
+          ReplaceOperationWithIdentity(0, node, optimized_graph);
           continue;
         }
       }
@@ -1648,7 +1648,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
           replaceable &= flatten(j) == 0;
         }
         if (replaceable) {
-          ReplaceOperationWithIdentity(0, node, output);
+          ReplaceOperationWithIdentity(0, node, optimized_graph);
           continue;
         }
       }
@@ -1668,7 +1668,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         replaceable &= shape.dim(j).size() > 1;
       }
       if (replaceable) {
-        ReplaceOperationWithIdentity(0, node, output);
+        ReplaceOperationWithIdentity(0, node, optimized_graph);
         continue;
       }
     }
@@ -1677,7 +1677,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         !OptimizedNodeExists(*node, "_const_axis")) {
       // Create constant axis node.
       Tensor axis_t(DT_INT32, TensorShape({}));
-      NodeDef* axis_node = output->add_node();
+      NodeDef* axis_node = optimized_graph->add_node();
       axis_node->set_name(OptimizedNodeName(*node, "_const_axis"));
       const int axis = node->attr().at("axis").i();
       if (!SetTensorValue(DT_INT32, axis, &axis_t).ok() ||
@@ -1685,7 +1685,6 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
                .ok()) {
         continue;
       }
-      VLOG(1) << "*** Rewriting trivial Pack node: " << node->DebugString();
       // Add a control dependency to make sure axis_node is in the right frame.
       const string ctrl_dep = ConstantFolding::AddControlDependency(
           node->input(0), graph_, node_map_.get());
@@ -1703,6 +1702,50 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       if (node->input_size() > 2) {
         node->mutable_input()->SwapElements(1, node->input_size() - 1);
       }
+      graph_modified_ = true;
+      continue;
+    }
+
+    // Move constants past Enter.
+    if (IsEnter(*node) && node->input_size() > 0) {
+      const string& node_name = node->name();
+      const NodeDef* input = node_map_->GetNode(node->input(0));
+      if (input != nullptr && IsReallyConstant(*input) &&
+          !OptimizedNodeExists(*input, "_enter")) {
+        auto fanouts = node_map_->GetOutputs(node_name);
+        // Find non-constant nodes that consume the output of *node.
+        std::vector<NodeDef*> consumers;
+        for (NodeDef* fanout : fanouts) {
+          if (!IsConstant(*fanout)) {
+            for (int i = 0; i < fanout->input_size(); ++i) {
+              if (fanout->input(i) == node_name) {
+                consumers.push_back(fanout);
+                break;
+              }
+            }
+          }
+        }
+        if (!consumers.empty()) {
+          NodeDef* new_node = optimized_graph->add_node();
+          *new_node = *input;
+          new_node->set_name(OptimizedNodeName(*input, "_enter"));
+          new_node->clear_input();
+          new_node->add_input(AsControlDependency(node_name));
+          node_map_->AddNode(new_node->name(), new_node);
+          node_map_->AddOutput(node_name, new_node->name());
+          for (NodeDef* consumer : consumers) {
+            for (int i = 0; i < consumer->input_size(); ++i) {
+              if (consumer->input(i) == node_name) {
+                node_map_->UpdateInput(consumer->name(), node_name,
+                                       new_node->name());
+                consumer->set_input(i, new_node->name());
+              }
+            }
+          }
+          graph_modified_ = true;
+          continue;
+        }
+      }
     }
 
     // Switch(x, x) will always feed false to its false branch and true to
@@ -1754,7 +1797,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
                     return n1->name() < n2->name();
                   });
         // Create constant false & true nodes.
-        NodeDef* false_node = output->add_node();
+        NodeDef* false_node = optimized_graph->add_node();
         false_node->set_name(OptimizedNodeName(*node, "_const_false"));
         if (!CreateNodeDef(false_node->name(), TensorValue(&false_t),
                            false_node)
@@ -1763,7 +1806,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         }
         false_node->set_device(node->device());
 
-        NodeDef* true_node = output->add_node();
+        NodeDef* true_node = optimized_graph->add_node();
         true_node->set_name(OptimizedNodeName(*node, "_const_true"));
         if (!CreateNodeDef(true_node->name(), TensorValue(&true_t), true_node)
                  .ok()) {
@@ -1776,10 +1819,10 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         const string false_port = node->name();
         const string true_port = strings::StrCat(node->name(), ":1");
         const string false_ctrl_dep =
-            AddControlDependency(false_port, output, node_map_.get());
+            AddControlDependency(false_port, optimized_graph, node_map_.get());
         false_node->add_input(false_ctrl_dep);
         const string true_ctrl_dep =
-            AddControlDependency(true_port, output, node_map_.get());
+            AddControlDependency(true_port, optimized_graph, node_map_.get());
         true_node->add_input(true_ctrl_dep);
 
         node_map_->AddNode(false_node->name(), false_node);
@@ -1861,13 +1904,13 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       if (y_matches_output_shape &&
           ((is_mul && x_is_one) || (is_add && x_is_zero))) {
         // 1 * y = y or 0 + y = y.
-        ReplaceOperationWithSnapshot(1, node, output);
+        ReplaceOperationWithSnapshot(1, node, optimized_graph);
         continue;
       }
 
       if (y_matches_output_shape && (is_sub && x_is_zero)) {
         // Replace 0 - y with Neg(y).
-        ReplaceSubtractionFromZeroByNegation(node, output);
+        ReplaceSubtractionFromZeroByNegation(node, optimized_graph);
         continue;
       }
 
@@ -1875,7 +1918,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       if (y_matches_output_shape && is_any_div && x_is_one) {
         DataType type = node->attr().at("T").type();
         if (DataTypeIsFloating(type) || DataTypeIsComplex(type)) {
-          ReplaceDivisionOfOnesByReciprocal(node, output);
+          ReplaceDivisionOfOnesByReciprocal(node, optimized_graph);
           continue;
         }
       }
@@ -1888,7 +1931,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       if (x_matches_output_shape && (((is_mul || is_any_div) && y_is_one) ||
                                      ((is_add || is_sub) && y_is_zero))) {
         // x * 1 = x or x / 1 = x or x +/- 0 = x
-        ReplaceOperationWithSnapshot(0, node, output);
+        ReplaceOperationWithSnapshot(0, node, optimized_graph);
         continue;
       }
 
@@ -1901,18 +1944,18 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
           (is_mul || is_matmul || optimize_zeros_divided_by_y)) {
         const PartialTensorShape shp(output_shape);
         if (shp.IsFullyDefined()) {
-          TF_RETURN_IF_ERROR(
-              ReplaceOperationWithConstant(0, output_shape, node, output));
+          TF_RETURN_IF_ERROR(ReplaceOperationWithConstant(0, output_shape, node,
+                                                          optimized_graph));
           continue;
         }
         // Even if an input shape is only partially known, we may known that it
         // matches the output shape and thus forward the corresponding zero
         // input.
         if ((is_mul || is_any_div) && x_is_zero && x_matches_output_shape) {
-          ReplaceOperationWithIdentity(0, node, output);
+          ReplaceOperationWithIdentity(0, node, optimized_graph);
           continue;
         } else if (is_mul && y_is_zero && y_matches_output_shape) {
-          ReplaceOperationWithIdentity(1, node, output);
+          ReplaceOperationWithIdentity(1, node, optimized_graph);
           continue;
         }
       }
@@ -1937,7 +1980,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         continue;
       }
       // Insert new reciprocal op and change node from Div to Mul.
-      NodeDef* reciprocal_node = output->add_node();
+      NodeDef* reciprocal_node = optimized_graph->add_node();
       reciprocal_node->set_name(OptimizedNodeName(*node, "_recip"));
       reciprocal_node->set_op("Reciprocal");
       reciprocal_node->set_device(node->device());
@@ -1950,6 +1993,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       node_map_->UpdateOutput(node->name(), const_input,
                               reciprocal_node->name());
       graph_modified_ = true;
+      continue;
     }
 
     // Consider the transformation
@@ -2042,6 +2086,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
     if (IsIdentityN(*node) && NumNonControlInputs(*node) > 0) {
       const std::set<NodeDef*>& tmp = node_map_->GetOutputs(node->name());
       const std::vector<NodeDef*> consumers(tmp.begin(), tmp.end());
+      bool updated_graph = false;
       for (int input_idx = 0; input_idx < node->input_size(); ++input_idx) {
         const string& input = node->input(input_idx);
         if (IsControlInput(input)) {
@@ -2072,7 +2117,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
               if (input_node_name == node->name() && output_idx == input_idx) {
                 consumer->set_input(consumer_input_idx, input);
                 // We will keep the input from IdentityN through a control
-                // dependendy, so we only need to add the consumer as an output
+                // dependency, so we only need to add the consumer as an output
                 // for the constant input node.
                 node_map_->AddOutput(NodeName(input), consumer->name());
                 add_dep = true;
@@ -2080,12 +2125,18 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
             }
             if (add_dep) {
               consumer->add_input(AsControlDependency(node->name()));
+              updated_graph = true;
             }
           }
         }
       }
-      for (NodeDef* consumer : consumers) {
-        DedupControlInputs(consumer);
+
+      if (updated_graph) {
+        for (NodeDef* consumer : consumers) {
+          DedupControlInputs(consumer);
+        }
+        graph_modified_ = true;
+        continue;
       }
     }
 
@@ -2126,7 +2177,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
       if (1 < const_inputs.size() &&
           const_inputs.size() < num_non_control_inputs &&
           !node_map_->NodeExists(new_node_name)) {
-        NodeDef* added_node = output->add_node();
+        NodeDef* added_node = optimized_graph->add_node();
         *added_node = *node;
         // Always use AddN for the constant node, since AccumulateNV2 is a fake
         // node that cannot be constant folded, since it does not have a kernel.
@@ -2230,7 +2281,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
         if (node_map_->NodeExists(new_node_name)) {
           break;
         }
-        NodeDef* added_node = output->add_node();
+        NodeDef* added_node = optimized_graph->add_node();
         *added_node = *node;
         added_node->set_name(new_node_name);
         node_map_->AddNode(added_node->name(), added_node);
@@ -2278,7 +2329,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* output,
 
 Status ConstantFolding::RunOptimizationPass(Cluster* cluster,
                                             const GrapplerItem& item,
-                                            GraphDef* output) {
+                                            GraphDef* optimized_graph) {
   node_map_.reset(new NodeMap(graph_));
   nodes_whitelist_.clear();
   // Fold fetch nodes iff it has a single fanout. Note that if a fetch node
@@ -2307,20 +2358,20 @@ Status ConstantFolding::RunOptimizationPass(Cluster* cluster,
     TF_RETURN_IF_ERROR(MaterializeShapes(properties));
     TF_RETURN_IF_ERROR(MaterializeConstants(properties));
   }
-  TF_RETURN_IF_ERROR(FoldGraph(output));
-  node_map_.reset(new NodeMap(output));
-  TF_RETURN_IF_ERROR(SimplifyGraph(output, &properties, can_use_shape_info));
+  TF_RETURN_IF_ERROR(FoldGraph(optimized_graph));
+  node_map_.reset(new NodeMap(optimized_graph));
+  TF_RETURN_IF_ERROR(
+      SimplifyGraph(optimized_graph, &properties, can_use_shape_info));
 
   return Status::OK();
 }
 
 Status ConstantFolding::Optimize(Cluster* cluster, const GrapplerItem& item,
-                                 GraphDef* output) {
+                                 GraphDef* optimized_graph) {
   // TensorFlow flushes denormals to zero and rounds to nearest, so we do
   // the same here.
   port::ScopedFlushDenormal flush;
   port::ScopedSetRound round(FE_TONEAREST);
-
   nodes_to_preserve_ = item.NodesToPreserve();
   for (const auto& feed : item.feed) {
     feed_nodes_.insert(NodeName(feed.first));
@@ -2332,20 +2383,20 @@ Status ConstantFolding::Optimize(Cluster* cluster, const GrapplerItem& item,
   }
 
   has_fetch_ = !item.fetch.empty();
-
   GrapplerItem item_to_optimize = item;
-  *output = item.graph;
+  *optimized_graph = item.graph;
   int64 node_count;
   do {
     graph_modified_ = false;
-    item_to_optimize.graph.Swap(output);
+    item_to_optimize.graph.Swap(optimized_graph);
     graph_ = &item_to_optimize.graph;
-    *output = GraphDef();
+    *optimized_graph = GraphDef();
     node_count = graph_->node_size();
-    TF_RETURN_IF_ERROR(RunOptimizationPass(cluster, item_to_optimize, output));
-  } while (graph_modified_ || output->node_size() != node_count);
-  *output->mutable_library() = item.graph.library();
-  *output->mutable_versions() = item.graph.versions();
+    TF_RETURN_IF_ERROR(
+        RunOptimizationPass(cluster, item_to_optimize, optimized_graph));
+  } while (graph_modified_ || optimized_graph->node_size() != node_count);
+  *optimized_graph->mutable_library() = item.graph.library();
+  *optimized_graph->mutable_versions() = item.graph.versions();
 
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 9050ccb053..aeb430b384 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -43,9 +43,9 @@ TEST_F(ConstantFoldingTest, SimpleFolding) {
   item.fetch.push_back("d");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(1, output.node_size());
@@ -89,9 +89,9 @@ TEST_F(ConstantFoldingTest, AddTree) {
   item.fetch = {"add_parent", "mul_parent", "addmul_parent"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // We expect the following rewrite(s) to occur:
@@ -525,9 +525,9 @@ TEST_F(ConstantFoldingTest, CreateConstNodes) {
 
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(24, output.node_size());
@@ -574,9 +574,9 @@ TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) {
   item.fetch.push_back("f");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(2, output.node_size());
@@ -615,9 +615,9 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
   item.fetch.push_back("e");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "e"};
@@ -658,9 +658,9 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "c",
@@ -715,9 +715,9 @@ TEST_F(ConstantFoldingTest, ControlDependenciesDeduplicate) {
   item.fetch.push_back("i2");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::vector<string> expected_nodes = {"dflt", "p1", "p2", "i2"};
@@ -789,9 +789,9 @@ TEST_F(ConstantFoldingTest, VariableNumberOfOutputs) {
   }
 
   item.fetch = outputs;
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int constant_folded = 0;
@@ -827,9 +827,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterialization) {
   item.fetch.push_back("p2");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -866,9 +866,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -920,9 +920,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
   int found = 0;
   for (const auto& node : output.node()) {
@@ -978,9 +978,9 @@ TEST_F(ConstantFoldingTest, ShapeMaterializationShapeN_MultipleOutputs) {
   item.fetch.push_back("ia");
   item.fetch.push_back("ib");
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1039,9 +1039,9 @@ TEST_F(ConstantFoldingTest, SwitchNodesEmptyFetch) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   std::set<string> present_nodes = {"v_in",     "v_ctrl",
@@ -1117,9 +1117,9 @@ TEST_F(ConstantFoldingTest, SwitchNodes) {
 
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
   std::set<string> present_nodes = {"v_in",     "v_ctrl",
                                     "switch",   "i",
@@ -1185,9 +1185,9 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   item.fetch = {"out1", "idx1", "out2", "idx2", "out3", "idx3"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found_nodes = 0;
@@ -1262,18 +1262,18 @@ TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) {
   item.fetch = {"out1", "out2"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef got;
-  Status status = fold.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
-  AddNode("in1", "VariableV2", {}, &want);
-  AddNode("in2", "VariableV2", {}, &want);
-  AddNode("s1", "Identity", {"in1"}, &want);
-  AddNode("s2", "Identity", {"in2", AsControlDependency("in1")}, &want);
-  AddNode("out1", "Add", {"s1", "s2"}, &want);
-  AddNode("out2", "Identity", {"s2"}, &want);
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("s1", "Identity", {"in1"}, {}, &want);
+  AddNode("s2", "Identity", {"in2", AsControlDependency("in1")}, {}, &want);
+  AddNode("out1", "Add", {"s1", "s2"}, {}, &want);
+  AddNode("out2", "Identity", {"s2"}, {}, &want);
 
   CompareGraphs(want, got);
 }
@@ -1295,21 +1295,21 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding fold(nullptr /* cpu_device */);
+    ConstantFolding optimizer(nullptr /* cpu_device */);
     GraphDef got;
-    Status status = fold.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
-    AddNode("in1", "VariableV2", {}, &want);
-    AddNode("in2", "VariableV2", {}, &want);
-    AddNode("begin", "Const", {}, &want);
-    AddNode("size", "Const", {}, &want);
+    AddNode("in1", "VariableV2", {}, {}, &want);
+    AddNode("in2", "VariableV2", {}, {}, &want);
+    AddNode("begin", "Const", {}, {}, &want);
+    AddNode("size", "Const", {}, {}, &want);
     AddNode("s1", "Identity",
             {"in1", AsControlDependency("begin"), AsControlDependency("size")},
-            &want);
-    AddNode("s2", "Slice", {"in2", "begin", "size"}, &want);
-    AddNode("out", "Add", {"s1", "s2"}, &want);
+            {}, &want);
+    AddNode("s2", "Slice", {"in2", "begin", "size"}, {}, &want);
+    AddNode("out", "Add", {"s1", "s2"}, {}, &want);
 
     CompareGraphs(want, got);
   }
@@ -1332,22 +1332,22 @@ TEST_F(ConstantFoldingTest, SliceWithSameDimensionRemoval) {
     item.fetch = {"out"};
     TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-    ConstantFolding fold(nullptr /* cpu_device */);
+    ConstantFolding optimizer(nullptr /* cpu_device */);
     GraphDef got;
-    Status status = fold.Optimize(nullptr, item, &got);
+    Status status = optimizer.Optimize(nullptr, item, &got);
     TF_EXPECT_OK(status);
 
     GraphDef want;
-    AddNode("in1", "VariableV2", {}, &want);
-    AddNode("in2", "VariableV2", {}, &want);
-    AddNode("begin1", "Const", {}, &want);
-    AddNode("begin2", "Const", {}, &want);
-    AddNode("size", "Const", {}, &want);
+    AddNode("in1", "VariableV2", {}, {}, &want);
+    AddNode("in2", "VariableV2", {}, {}, &want);
+    AddNode("begin1", "Const", {}, {}, &want);
+    AddNode("begin2", "Const", {}, {}, &want);
+    AddNode("size", "Const", {}, {}, &want);
     AddNode("s1", "Identity",
             {"in1", AsControlDependency("begin1"), AsControlDependency("size")},
-            &want);
-    AddNode("s2", "Slice", {"in2", "begin2", "size"}, &want);
-    AddNode("out", "Add", {"s1", "s2"}, &want);
+            {}, &want);
+    AddNode("s2", "Slice", {"in2", "begin2", "size"}, {}, &want);
+    AddNode("out", "Add", {"s1", "s2"}, {}, &want);
 
     CompareGraphs(want, got);
   }
@@ -1370,19 +1370,20 @@ TEST_F(ConstantFoldingTest, TileWithMultipliesBeingOne) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef got;
-  Status status = fold.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
-  AddNode("in1", "VariableV2", {}, &want);
-  AddNode("in2", "VariableV2", {}, &want);
-  AddNode("multiplies1", "Const", {}, &want);
-  AddNode("multiplies2", "Const", {}, &want);
-  AddNode("t1", "Identity", {"in1", AsControlDependency("multiplies1")}, &want);
-  AddNode("t2", "Tile", {"in2", "multiplies2"}, &want);
-  AddNode("out", "Add", {"t1", "t2"}, &want);
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("multiplies1", "Const", {}, {}, &want);
+  AddNode("multiplies2", "Const", {}, {}, &want);
+  AddNode("t1", "Identity", {"in1", AsControlDependency("multiplies1")}, {},
+          &want);
+  AddNode("t2", "Tile", {"in2", "multiplies2"}, {}, &want);
+  AddNode("out", "Add", {"t1", "t2"}, {}, &want);
 
   CompareGraphs(want, got);
 }
@@ -1408,23 +1409,23 @@ TEST_F(ConstantFoldingTest, PaddingWithZeroSize) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef got;
-  Status status = fold.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
-  AddNode("in1", "VariableV2", {}, &want);
-  AddNode("in2", "VariableV2", {}, &want);
-  AddNode("paddings1", "Const", {}, &want);
-  AddNode("paddings2", "Const", {}, &want);
-  AddNode("c1", "Const", {}, &want);
-  AddNode("c2", "Const", {}, &want);
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("paddings1", "Const", {}, {}, &want);
+  AddNode("paddings2", "Const", {}, {}, &want);
+  AddNode("c1", "Const", {}, {}, &want);
+  AddNode("c2", "Const", {}, {}, &want);
   AddNode("p1", "Identity",
           {"in1", AsControlDependency("paddings1"), AsControlDependency("c1")},
-          &want);
-  AddNode("p2", "PadV2", {"in2", "paddings2", "c2"}, &want);
-  AddNode("out", "Add", {"p1", "p2"}, &want);
+          {}, &want);
+  AddNode("p2", "PadV2", {"in2", "paddings2", "c2"}, {}, &want);
+  AddNode("out", "Add", {"p1", "p2"}, {}, &want);
 
   CompareGraphs(want, got);
 }
@@ -1444,17 +1445,17 @@ TEST_F(ConstantFoldingTest, SqueezeWithAllDimesionsGreaterThanOne) {
   item.fetch = {"out"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef got;
-  Status status = fold.Optimize(nullptr, item, &got);
+  Status status = optimizer.Optimize(nullptr, item, &got);
   TF_EXPECT_OK(status);
 
   GraphDef want;
-  AddNode("in1", "VariableV2", {}, &want);
-  AddNode("in2", "VariableV2", {}, &want);
-  AddNode("s1", "Identity", {"in1"}, &want);
-  AddNode("s2", "Squeeze", {"in2"}, &want);
-  AddNode("out", "Add", {"s1", "s2"}, &want);
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("s1", "Identity", {"in1"}, {}, &want);
+  AddNode("s2", "Squeeze", {"in2"}, {}, &want);
+  AddNode("out", "Add", {"s1", "s2"}, {}, &want);
 
   CompareGraphs(want, got);
 }
@@ -1475,9 +1476,9 @@ TEST_F(ConstantFoldingTest, NoOpReduction) {
   item.fetch.push_back("s");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   bool found = false;
@@ -1534,9 +1535,9 @@ TEST_F(ConstantFoldingTest, NoOpReshape) {
   item.fetch = {"s1", "s2", "s3", "s4"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1581,9 +1582,9 @@ TEST_F(ConstantFoldingTest, Packing) {
   GrapplerItem item;
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Make sure that the representation of the folded constant is space
@@ -1616,14 +1617,14 @@ TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Run a second time to make sure the optimization is idempotent.
   item.graph.Swap(&output);
-  status = fold.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1677,14 +1678,14 @@ TEST_F(ConstantFoldingTest, MaterializeReductionIndices) {
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   item.fetch.push_back("reshape");
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Run a second time to make sure the optimization is idempotent.
   item.graph.Swap(&output);
-  status = fold.Optimize(nullptr, item, &output);
+  status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   int found = 0;
@@ -1717,9 +1718,9 @@ TEST_F(ConstantFoldingTest, LargeConstant) {
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
   item.fetch.push_back("out");
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   // Make sure the diag node hasn't been folded, since it would use too much
@@ -1756,9 +1757,9 @@ TEST_F(ConstantFoldingTest, SwitchIdenticalInputs) {
   item.fetch.push_back("id_true");
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
   EXPECT_EQ(6, output.node_size());
@@ -1984,8 +1985,8 @@ TEST_F(ConstantFoldingTest, PartialFolding_IdentityN) {
   Output c1 = ops::Const(scope.WithOpName("c1"), 1.0f, {2, 2});
   Output c2 = ops::Const(scope.WithOpName("c2"), 2.0f, {2, 2});
   auto id_n = ops::IdentityN(scope.WithOpName("id_n"), {c1, x, c2});
-  auto id0 = ops::Identity(scope.WithOpName("id0"), id_n[1]);
-  auto id1 = ops::Identity(scope.WithOpName("id1"), id_n[0]);
+  auto id0 = ops::Identity(scope.WithOpName("id0"), id_n[0]);
+  auto id1 = ops::Identity(scope.WithOpName("id1"), id_n[1]);
   auto add0 = ops::Add(scope.WithOpName("add0"), id_n[0], id_n[1]);
   auto add1 = ops::Add(scope.WithOpName("add1"), id_n[0], id_n[2]);
 
@@ -1996,38 +1997,44 @@ TEST_F(ConstantFoldingTest, PartialFolding_IdentityN) {
   item.fetch.push_back("add0");
   item.fetch.push_back("add1");
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
-
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  LOG(INFO) << output.DebugString();
   TF_EXPECT_OK(status);
   EXPECT_EQ(8, output.node_size());
-  // id_n should remain unchanged.
-  EXPECT_EQ("id_n", output.node(3).name());
-  EXPECT_EQ(3, output.node(3).input_size());
-  EXPECT_EQ("c1", output.node(3).input(0));
-  EXPECT_EQ("x", output.node(3).input(1));
-  EXPECT_EQ("c2", output.node(3).input(2));
-  // id0 is unchanged.
-  EXPECT_EQ("id0", output.node(4).name());
-  EXPECT_EQ(1, output.node(4).input_size());
-  // id1 should have the constant input forwarded to it,
-  // and a control dependency from id_n.
-  EXPECT_EQ("id1", output.node(5).name());
-  EXPECT_EQ(2, output.node(5).input_size());
-  EXPECT_EQ("c1", output.node(5).input(0));
-  EXPECT_EQ("^id_n", output.node(5).input(1));
-
-  EXPECT_EQ("add0", output.node(6).name());
-  EXPECT_EQ(2, output.node(6).input_size());
-  EXPECT_EQ("c1", output.node(6).input(0));
-  EXPECT_EQ("id_n:1", output.node(6).input(1));
-
-  EXPECT_EQ("add1", output.node(7).name());
-  EXPECT_EQ(3, output.node(7).input_size());
-  EXPECT_EQ("c1", output.node(7).input(0));
-  EXPECT_EQ("c2", output.node(7).input(1));
-  EXPECT_EQ("^id_n", output.node(7).input(2));
+  for (const auto& node : output.node()) {
+    // id_n should remain unchanged.
+    if (node.name() == "id_n") {
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("c1", node.input(0));
+      EXPECT_EQ("x", node.input(1));
+      EXPECT_EQ("c2", node.input(2));
+    }
+    // id0 should be constant folded, and a control dependency from id_n.
+    if (node.name() == "id0") {
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^id_n", node.input(0));
+    }
+    // id1 is unchanged.
+    if ("id1" == node.name()) {
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("id_n:1", node.input(0));
+    }
+
+    if ("add0" == node.name()) {
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("c1", node.input(0));
+      EXPECT_EQ("id_n:1", node.input(1));
+    }
+    // add1 should bo constant folded and have a control dependency from id_n.
+    if ("add1" == node.name()) {
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^id_n", node.input(0));
+    }
+  }
 }
 
 TEST_F(ConstantFoldingTest, TrivialPack) {
@@ -2043,11 +2050,10 @@ TEST_F(ConstantFoldingTest, TrivialPack) {
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
   item.fetch.push_back("stack");
 
-  ConstantFolding fold(nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
-  Status status = fold.Optimize(nullptr, item, &output);
+  Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  LOG(INFO) << output.DebugString();
   EXPECT_EQ(5, output.node_size());
   for (const auto& node : output.node()) {
     if (node.name() == "stack") {
@@ -2072,6 +2078,55 @@ TEST_F(ConstantFoldingTest, TrivialPack) {
   EXPECT_EQ(tensors_expected[0].shape(), tensors[0].shape());
 }
 
+TEST_F(ConstantFoldingTest, Enter) {
+  GrapplerItem item;
+  AttrValue frame_name;
+  frame_name.set_s("foo");
+  AttrValue type;
+  type.set_type(DT_FLOAT);
+  AttrValue value;
+  Tensor value_tensor(DT_FLOAT, TensorShape({}));
+  value_tensor.flat<float>()(0) = 1;
+  value_tensor.AsProtoTensorContent(value.mutable_tensor());
+
+  GraphDef& graph = item.graph;
+  AddNode("x", "Placeholder", {}, {{"T", type}}, &graph);
+  AddNode("c1", "Const", {"^x"}, {{"value", value}, {"dtype", type}}, &graph);
+  AddNode("enter1", "Enter", {"x"}, {{"T", type}, {"frame_name", frame_name}},
+          &graph);
+  AddNode("enter2", "Enter", {"c1"}, {{"T", type}, {"frame_name", frame_name}},
+          &graph);
+  AddNode("id1", "Identity", {"enter1"}, {{"T", type}}, &graph);
+  AddNode("id2", "Identity", {"enter2"}, {{"T", type}}, &graph);
+  AddNode("id3", "Identity", {"enter2"}, {{"T", type}}, &graph);
+  item.fetch.push_back("id1");
+  item.fetch.push_back("id2");
+  item.fetch.push_back("id3");
+
+  ConstantFolding optimizer(nullptr /* cpu_device */);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+  // Run the optimizer twice to make sure the rewrite is idempotent.
+  item.graph.Swap(&output);
+  status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(7, output.node_size());
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "id1") {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("enter1", node.input(0));
+    }
+    if (node.name() == "id2" || node.name() == "id3") {
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^enter2", node.input(0));
+    }
+  }
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 89c3aa82bf..6b6cecebe1 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -23,7 +23,7 @@ namespace tensorflow {
 namespace grappler {
 
 std::vector<Tensor> GrapplerTest::EvaluateNodes(
-    const GraphDef& graph, const std::vector<string>& node_names) {
+    const GraphDef& graph, const std::vector<string>& node_names) const {
   SessionOptions options;
   std::unique_ptr<tensorflow::Session> session(NewSession(options));
   TF_CHECK_OK(session->Create(graph));
@@ -35,7 +35,8 @@ std::vector<Tensor> GrapplerTest::EvaluateNodes(
   return output_tensors;
 }
 
-std::vector<Tensor> GrapplerTest::EvaluateFetchNodes(const GrapplerItem& item) {
+std::vector<Tensor> GrapplerTest::EvaluateFetchNodes(
+    const GrapplerItem& item) const {
   SessionOptions options;
   std::unique_ptr<tensorflow::Session> session(NewSession(options));
   TF_CHECK_OK(session->Create(item.graph));
@@ -52,17 +53,23 @@ std::vector<Tensor> GrapplerTest::EvaluateFetchNodes(const GrapplerItem& item) {
   return output_tensors;
 }
 
-void GrapplerTest::AddNode(const string& name, const string& op,
-                           const std::vector<string>& inputs, GraphDef* graph) {
-  auto* node = graph->add_node();
+NodeDef* GrapplerTest::AddNode(
+    const string& name, const string& op, const std::vector<string>& inputs,
+    const std::vector<std::pair<string, AttrValue>>& attributes,
+    GraphDef* graph) const {
+  NodeDef* node = graph->add_node();
   node->set_name(name);
   node->set_op(op);
-  for (const auto& input : inputs) {
+  for (const string& input : inputs) {
     node->add_input(input);
   }
+  for (auto attr : attributes) {
+    (*node->mutable_attr())[attr.first] = attr.second;
+  }
+  return node;
 }
 
-void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) {
+void GrapplerTest::CompareGraphs(GraphDef want, GraphDef got) const {
   auto comparator = [](const NodeDef& n1, const NodeDef& n2) -> bool {
     return n1.name() < n2.name();
   };
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 3df6625d5c..c7f06557e7 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <vector>
 
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/graph.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -29,15 +30,17 @@ namespace grappler {
 
 class GrapplerTest : public ::testing::Test {
  protected:
-  std::vector<Tensor> EvaluateNodes(const GraphDef& graph,
-                                    const std::vector<string>& node_names);
+  std::vector<Tensor> EvaluateNodes(
+      const GraphDef& graph, const std::vector<string>& node_names) const;
 
-  std::vector<Tensor> EvaluateFetchNodes(const GrapplerItem& item);
+  std::vector<Tensor> EvaluateFetchNodes(const GrapplerItem& item) const;
 
-  void AddNode(const string& name, const string& op,
-               const std::vector<string>& inputs, GraphDef* graph);
+  NodeDef* AddNode(const string& name, const string& op,
+                   const std::vector<string>& inputs,
+                   const std::vector<std::pair<string, AttrValue>>& attributes,
+                   GraphDef* graph) const;
 
-  void CompareGraphs(GraphDef want, GraphDef got);
+  void CompareGraphs(GraphDef want, GraphDef got) const;
 
   // Check if node 'src' is directly connected to the input($position) of 'dst'.
   bool IsNodesDirectlyConnected(const NodeMap& node_map, const string& src,
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 4e80fab9bd..787582ae70 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -35,6 +35,7 @@ from google.protobuf import text_format
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.core.protobuf import meta_graph_pb2
 from tensorflow.core.protobuf import queue_runner_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.core.protobuf import saver_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.client import session
@@ -2165,7 +2166,13 @@ class MetaGraphTest(test.TestCase):
       # Build and run the gradients of the while loop. We use this below to
       # verify that the gradients are correct with an imported MetaGraphDef.
       grad = gradients_impl.gradients([output], [var])
-      with session.Session() as sess:
+      # Turn off constant folding to avoid breaking testNestedControlFlowSerDes.
+      # It appears that a missing control dependency in the gradient graph
+      # causes the fetch node to not be triggered.
+      no_constfold_config = config_pb2.ConfigProto()
+      no_constfold_config.graph_options.rewrite_options.constant_folding = (
+          rewriter_config_pb2.RewriterConfig.OFF)
+      with session.Session(config=no_constfold_config) as sess:
         sess.run(init_op)
         expected_grad_value = sess.run(grad)
 
@@ -2182,7 +2189,7 @@ class MetaGraphTest(test.TestCase):
 
       init_op = variables.global_variables_initializer()
 
-      with session.Session() as sess:
+      with session.Session(config=no_constfold_config) as sess:
         sess.run(init_op)
         actual_grad_value = sess.run(grad)
         self.assertEqual(expected_grad_value, actual_grad_value)
-- 
GitLab


From 9637651f144a04225279fa3f6ffe944ee35086d8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 09:40:28 -0700
Subject: [PATCH 056/960]   add grpc service stub for TPUProfilerAnalysis

PiperOrigin-RevId: 189198495
---
 tensorflow/contrib/tpu/profiler/BUILD         |  26 +++-
 .../tpu/profiler/tpu_profiler_analysis.proto  |  73 +++++++++
 .../tpu_profiler_analysis_pb2_grpc.py         | 138 ++++++++++++++++++
 3 files changed, 234 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto
 create mode 100644 tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py

diff --git a/tensorflow/contrib/tpu/profiler/BUILD b/tensorflow/contrib/tpu/profiler/BUILD
index 198da0203a..0a52d0b13b 100644
--- a/tensorflow/contrib/tpu/profiler/BUILD
+++ b/tensorflow/contrib/tpu/profiler/BUILD
@@ -18,7 +18,7 @@ filegroup(
     visibility = ["//tensorflow:__subpackages__"],
 )
 
-tf_proto_library_cc(
+tf_proto_library(
     name = "tpu_profiler_proto",
     srcs = ["tpu_profiler.proto"],
     has_services = 1,
@@ -98,16 +98,36 @@ tf_cc_test(
     ],
 )
 
-tf_proto_library_cc(
+tf_proto_library(
     name = "op_profile_proto",
     srcs = ["op_profile.proto"],
     cc_api_version = 2,
     visibility = ["//visibility:public"],
 )
 
-tf_proto_library_cc(
+tf_proto_library(
     name = "tf_op_stats_proto",
     srcs = ["tf_op_stats.proto"],
     cc_api_version = 2,
     visibility = ["//visibility:public"],
 )
+
+tf_proto_library(
+    name = "tpu_profiler_analysis_proto",
+    srcs = ["tpu_profiler_analysis.proto"],
+    has_services = 1,
+    cc_api_version = 2,
+    cc_grpc_version = 1,
+    protodeps = [":tpu_profiler_proto"] + tf_additional_all_protos(),
+    visibility = ["//visibility:public"],
+)
+
+py_library(
+    name = "tpu_profiler_analysis_pb2_grpc",
+    srcs = ["tpu_profiler_analysis_pb2_grpc.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":tpu_profiler_analysis_proto_py",
+    ],
+)
diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto
new file mode 100644
index 0000000000..a4fc8d4e87
--- /dev/null
+++ b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis.proto
@@ -0,0 +1,73 @@
+syntax = "proto3";
+package tensorflow;
+
+import "tensorflow/contrib/tpu/profiler/tpu_profiler.proto";
+
+message NewProfileSessionRequest {
+  ProfileRequest request = 1;
+  string repository_root = 2;
+  repeated string hosts = 3;
+}
+
+message NewProfileSessionResponse {
+  // Auxiliary error_message.
+  string error_message = 1;
+  // If success, return session identifier for future reference.
+  string session_id = 2;
+}
+
+message EnumProfileSessionsAndToolsRequest {
+  string repository_root = 1;
+}
+
+message ProfileSessionInfo {
+  string session_id = 1;
+  // Which tool data is available for consumption.
+  repeated string available_tools = 2;
+}
+
+message EnumProfileSessionsAndToolsResponse {
+  // Auxiliary error_message.
+  string error_message = 1;
+  // If success, the returned sessions information are stored here.
+  repeated ProfileSessionInfo sessions = 2;
+}
+
+message ProfileSessionDataRequest {
+  string repository_root = 1;
+  string session_id = 2;
+  // Which tool
+  string tool_name = 3;
+  // Tool's specific parameters. e.g. TraceViewer's viewport etc
+  map<string, string> parameters = 4;
+}
+
+message ProfileSessionDataResponse {
+  // Auxiliary error_message.
+  string error_message = 1;
+
+  // Output format. e.g. "json" or "proto" or "blob"
+  string output_format = 2;
+
+  // TODO(jiesun): figure out whether to put bytes or oneof tool specific proto.
+  bytes output = 3;
+}
+////////////////////////////////////////////////////////////////////////////////
+// TPUProfileAnalysis service provide entry point for profiling TPU and for
+// serving profiled data to Tensorboard through GRPC
+////////////////////////////////////////////////////////////////////////////////
+service TPUProfileAnalysis {
+  // Starts a profiling session, blocks until it completes.
+  // TPUProfileAnalysis service delegate this to TPUProfiler service.
+  // Populate the profiled data in repository, then return status to caller.
+  rpc NewSession(NewProfileSessionRequest) returns (NewProfileSessionResponse) {
+  }
+  // Enumerate existing sessions and return available profile tools.
+  rpc EnumSessions(EnumProfileSessionsAndToolsRequest)
+      returns (EnumProfileSessionsAndToolsResponse) {
+  }
+  // Retrieve specific tool's data for specific session.
+  rpc GetSessionToolData(ProfileSessionDataRequest)
+      returns (ProfileSessionDataResponse) {
+  }
+}
diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py
new file mode 100644
index 0000000000..c28fef22a9
--- /dev/null
+++ b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py
@@ -0,0 +1,138 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+# Generated by the gRPC Python protocol compiler plugin. DO NOT EDIT!
+#
+# Do not use pylint on generated code.
+# pylint: disable=missing-docstring,g-short-docstring-punctuation,g-no-space-after-docstring-summary,invalid-name,line-too-long,unused-argument,g-doc-args
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import grpc
+
+from third_party.tensorflow.contrib.tpu.profiler import tpu_profiler_analysis_pb2 as third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2
+
+
+class TPUProfileAnalysisStub(object):
+  """//////////////////////////////////////////////////////////////////////////////
+
+  TPUProfileAnalysis service provide entry point for profiling TPU and for
+  serving profiled data to Tensorboard through GRPC
+  //////////////////////////////////////////////////////////////////////////////
+  """
+
+  def __init__(self, channel):
+    """Constructor.
+
+    Args:
+      channel: A grpc.Channel.
+    """
+    self.NewSession = channel.unary_unary(
+        '/tensorflow.TPUProfileAnalysis/NewSession',
+        request_serializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        NewProfileSessionRequest.SerializeToString,
+        response_deserializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        NewProfileSessionResponse.FromString,
+    )
+    self.EnumSessions = channel.unary_unary(
+        '/tensorflow.TPUProfileAnalysis/EnumSessions',
+        request_serializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        EnumProfileSessionsAndToolsRequest.SerializeToString,
+        response_deserializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        EnumProfileSessionsAndToolsResponse.FromString,
+    )
+    self.GetSessionToolData = channel.unary_unary(
+        '/tensorflow.TPUProfileAnalysis/GetSessionToolData',
+        request_serializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        ProfileSessionDataRequest.SerializeToString,
+        response_deserializer=
+        third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+        ProfileSessionDataResponse.FromString,
+    )
+
+
+class TPUProfileAnalysisServicer(object):
+  """//////////////////////////////////////////////////////////////////////////////
+
+  TPUProfileAnalysis service provide entry point for profiling TPU and for
+  serving profiled data to Tensorboard through GRPC
+  //////////////////////////////////////////////////////////////////////////////
+  """
+
+  def NewSession(self, request, context):
+    """Starts a profiling session, blocks until it completes.
+    TPUProfileAnalysis service delegate this to TPUProfiler service.
+    Populate the profiled data in repository, then return status to caller.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def EnumSessions(self, request, context):
+    """Enumerate existing sessions and return available profile tools.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+  def GetSessionToolData(self, request, context):
+    """Retrieve specific tool's data for specific session.
+    """
+    context.set_code(grpc.StatusCode.UNIMPLEMENTED)
+    context.set_details('Method not implemented!')
+    raise NotImplementedError('Method not implemented!')
+
+
+def add_TPUProfileAnalysisServicer_to_server(servicer, server):
+  rpc_method_handlers = {
+      'NewSession':
+          grpc.unary_unary_rpc_method_handler(
+              servicer.NewSession,
+              request_deserializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              NewProfileSessionRequest.FromString,
+              response_serializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              NewProfileSessionResponse.SerializeToString,
+          ),
+      'EnumSessions':
+          grpc.unary_unary_rpc_method_handler(
+              servicer.EnumSessions,
+              request_deserializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              EnumProfileSessionsAndToolsRequest.FromString,
+              response_serializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              EnumProfileSessionsAndToolsResponse.SerializeToString,
+          ),
+      'GetSessionToolData':
+          grpc.unary_unary_rpc_method_handler(
+              servicer.GetSessionToolData,
+              request_deserializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              ProfileSessionDataRequest.FromString,
+              response_serializer=
+              third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2.
+              ProfileSessionDataResponse.SerializeToString,
+          ),
+  }
+  generic_handler = grpc.method_handlers_generic_handler(
+      'tensorflow.TPUProfileAnalysis', rpc_method_handlers)
+  server.add_generic_rpc_handlers((generic_handler,))
-- 
GitLab


From 2acc5b6c465832fc8c1fba2454d3dfd8f3aa2eb5 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 15 Mar 2018 10:26:13 -0700
Subject: [PATCH 057/960] TFE: Modify initialization of `ContextStack` to
 ensure eager context is kept.

When eager execution is enabled in the main thread, the fact that it was
enabled is propagated to subsequently created threads. This
change ...

(1) ensures that the fact that eager was enabled is also propagated to the `ContextStack`, which is renamed to `_ContextSwitchStack`, for clarity;
(2) adds a `_ContextSwitchStack` object to `Context` as a member, removing the global `context_stack`.

PiperOrigin-RevId: 189206207
---
 tensorflow/python/eager/context.py   | 40 +++++++++++++++++++---------
 tensorflow/python/eager/core_test.py | 15 +++++------
 tensorflow/python/eager/ops_test.py  | 17 ++++++++++++
 tensorflow/python/framework/ops.py   | 26 +++++++-----------
 4 files changed, 62 insertions(+), 36 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 7953d10a89..6c9a14730c 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -94,22 +94,32 @@ class _EagerContext(threading.local):
     self.execution_mode = None
 
 
-ContextStackEntry = collections.namedtuple(
-    "ContextStackEntry", ["is_building_function", "enter_context_fn"])
+ContextSwitch = collections.namedtuple(
+    "ContextSwitch", ["is_building_function", "enter_context_fn"])
 
 
-class ContextStack(threading.local):
+# `_ContextSwitchStack` is a `threading.local` to match the semantics of
+# ``DefaultGraphStack`, which is also a `threading.local`.
+class _ContextSwitchStack(threading.local):
   """A thread-local stack of context switches."""
 
-  def __init__(self):
-    super(ContextStack, self).__init__()
+  def __init__(self, eager):
+    super(_ContextSwitchStack, self).__init__()
     self.stack = []
+    if eager:
+      # Initialize the stack with a pointer to enter the eager context; this
+      # ensures that the fact that eager execution was enabled is propagated
+      # across threads, since (1) `enable_eager_execution` modifies a
+      # process-level flag (`_default_mode`) and (2) `__init__` is called each
+      # time a threading.local object is used in a separate thread.
+      self.push(is_building_function=False, enter_context_fn=eager_mode)
 
   def push(self, is_building_function, enter_context_fn):
     """Push metadata about a context switch onto the stack.
 
     A context switch can take one of two forms: installing a graph as the
-    default graph, or entering the eager context.
+    default graph, or entering the eager context. For each context switch,
+    we record whether or not the entered context is building a function.
 
     Args:
       is_building_function: (bool.) Whether the context is building a function.
@@ -118,7 +128,7 @@ class ContextStack(threading.local):
     """
 
     self.stack.append(
-        ContextStackEntry(is_building_function, enter_context_fn))
+        ContextSwitch(is_building_function, enter_context_fn))
 
   def pop(self):
     """Pop the stack."""
@@ -126,9 +136,6 @@ class ContextStack(threading.local):
     self.stack.pop()
 
 
-context_stack = ContextStack()
-
-
 # TODO(agarwal): rename to EagerContext / EagerRuntime ?
 # TODO(agarwal): consider keeping the corresponding Graph here.
 class Context(object):
@@ -171,6 +178,7 @@ class Context(object):
      ValueError: If execution_mode is not valid.
     """
     self._eager_context = _EagerContext()
+    self._context_switches = _ContextSwitchStack(self.executing_eagerly())
     self._context_handle = None
     self._context_devices = None
     self._post_execution_callbacks = []
@@ -283,13 +291,16 @@ class Context(object):
     old_mode = ctx.mode
     ctx.mode = mode
     if mode == EAGER_MODE:
-      context_stack.push(False, eager_mode)
+      # Entering graph mode does not provide us with sufficient information to
+      # record a context switch; graph-based context switches are only logged
+      # when a graph is registered as the default graph.
+      self.context_switches.push(False, eager_mode)
     try:
       yield
     finally:
       ctx.mode = old_mode
       if mode == EAGER_MODE:
-        context_stack.pop()
+        self.context_switches.pop()
 
   def executing_eagerly(self):
     """Returns True if current thread has eager executing enabled."""
@@ -545,6 +556,11 @@ class Context(object):
     run_metadata.ParseFromString(compat.as_bytes(proto_data))
     return run_metadata
 
+  @property
+  def context_switches(self):
+    """Returns a stack of context switches."""
+    return self._context_switches
+
 _context = None
 _context_lock = threading.Lock()
 
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 6dfd8d1afa..6ebf5b2481 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -123,19 +123,18 @@ class TFETest(test_util.TensorFlowTestCase):
     # available, when no device is explicitly provided)
     self.assertEqual(y.device, '/job:localhost/replica:0/task:0/device:CPU:0')
 
-  def testContextStackContainsEagerMode(self):
-    # Eager execution has been enabled, and no other context
-    # switch has occurred, so `context_stack` should contain
-    # exactly one entry.
-    self.assertEqual(len(context.context_stack.stack), 1)
-    stack_entry = context.context_stack.stack[0]
+  def testContextSwitchStackContainsEagerMode(self):
+    # Eager execution has been enabled, and no other context switch has
+    # occurred, so `context_switches` should contain exactly one entry.
+    self.assertEqual(len(context.context().context_switches.stack), 1)
+    switch = context.context().context_switches.stack[0]
 
     # The entry should log that eager mode was entered.
-    self.assertIs(stack_entry.enter_context_fn, context.eager_mode)
+    self.assertIs(switch.enter_context_fn, context.eager_mode)
 
     # It is not possible to build a graph function when eager execution
     # is enabled; the stack entry should reflect this fact.
-    self.assertFalse(stack_entry.is_building_function)
+    self.assertFalse(switch.is_building_function)
 
   def testInt32GPU(self):
     if not context.context().num_gpus():
diff --git a/tensorflow/python/eager/ops_test.py b/tensorflow/python/eager/ops_test.py
index f70c7544d6..fc76ede4c5 100644
--- a/tensorflow/python/eager/ops_test.py
+++ b/tensorflow/python/eager/ops_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import threading
 import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
@@ -376,6 +377,22 @@ class OpsTest(test_util.TensorFlowTestCase):
   def testNoOpIsNone(self):
     self.assertTrue(control_flow_ops.no_op() is None)
 
+  def testEagerContextPreservedAcrossThreads(self):
+    def init_fn():
+      self.assertTrue(context.executing_eagerly())
+      with ops.init_scope():
+        self.assertTrue(context.executing_eagerly())
+        context_switches = context.context().context_switches
+        self.assertEqual(len(context_switches.stack), 1)
+        self.assertFalse(context_switches.stack[0].is_building_function)
+        self.assertEqual(context_switches.stack[0].enter_context_fn,
+                         context.eager_mode)
+
+    self.assertTrue(context.executing_eagerly())
+    t1 = threading.Thread(target=init_fn)
+    t1.start()
+    t1.join()
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b2f43773fe..01a0e03be2 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5095,11 +5095,12 @@ class _DefaultGraphStack(_DefaultStack):  # pylint: disable=protected-access
   @tf_contextlib.contextmanager
   def get_controller(self, default):
     try:
-      context.context_stack.push(default.building_function, default.as_default)
+      context.context().context_switches.push(default.building_function,
+                                              default.as_default)
       with super(_DefaultGraphStack, self).get_controller(default) as g:
         yield g
     finally:
-      context.context_stack.pop()
+      context.context().context_switches.pop()
 
 
 _default_graph_stack = _DefaultGraphStack()
@@ -5125,13 +5126,13 @@ def init_scope():
         graph function. Here, a context is defined as either a graph or an eager
         context. Every context switch, i.e., every installation of a graph as
         the default graph and every switch into eager mode, is logged in a
-        thread-local stack called the `context_stack`; the log entry for a
+        thread-local stack called `context_switches`; the log entry for a
         context switch is popped from the stack when the context is exited.
-        Entering an `init_scope` is equivalent to crawling up the
-        `context_stack`, finding the first context that is not building a graph
-        function, and entering it. A caveat is that if graph mode is enabled
-        but the default graph stack is empty, then entering an `init_scope`
-        will simply install a fresh graph as the default one.
+        Entering an `init_scope` is equivalent to crawling up
+        `context_switches`, finding the first context that is not building a
+        graph function, and entering it. A caveat is that if graph mode is
+        enabled but the default graph stack is empty, then entering an
+        `init_scope` will simply install a fresh graph as the default one.
 
     (3) The gradient tape is paused while the scope is active.
   """
@@ -5161,7 +5162,7 @@ def init_scope():
       outer_context = default_graph.as_default
     else:
       # Find a context that is not building a function.
-      for stack_entry in reversed(context.context_stack.stack):
+      for stack_entry in reversed(context.context().context_switches.stack):
         if not stack_entry.is_building_function:
           outer_context = stack_entry.enter_context_fn
           break
@@ -5278,13 +5279,6 @@ def enable_eager_execution(config=None, device_policy=None,
         config=config,
         device_policy=device_policy,
         execution_mode=execution_mode)
-    if context.context_stack.stack:
-      raise AssertionError("Invariant violated: The context stack must "
-                           "be empty when eager execution is enabled.")
-    # Log that eager execution has been enabled by pushing an entry onto the
-    # context stack; this entry won't ever be popped, as it's impossible to
-    # disable eager execution
-    context.context_stack.push(False, context.eager_mode)
   elif ((config is not None and config is not context._context._config) or
         (device_policy is not None and
          device_policy is not context._context._device_policy) or
-- 
GitLab


From 4b0687d70c4bcab5ec2837345bd0115a0b356946 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 15 Mar 2018 10:39:55 -0700
Subject: [PATCH 058/960] [XLA] Add --xla_hlo_profile_last_run flag to
 replay_computation.

When using replay_computation for profiling, you usually only want to
do one or two warmup runs and then profile the last run of your model.
This flag makes that possible.

PiperOrigin-RevId: 189208924
---
 tensorflow/compiler/xla/tools/BUILD            |  1 +
 .../compiler/xla/tools/replay_computation.cc   | 18 ++++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 091fa0c3ec..2e55f609d1 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -75,6 +75,7 @@ cc_library(
     name = "replay_computation_library",
     srcs = ["replay_computation.cc"],
     deps = [
+        "//tensorflow/compiler/xla:execution_options_util",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/tools/replay_computation.cc b/tensorflow/compiler/xla/tools/replay_computation.cc
index eda5effbb9..62a353ad09 100644
--- a/tensorflow/compiler/xla/tools/replay_computation.cc
+++ b/tensorflow/compiler/xla/tools/replay_computation.cc
@@ -40,6 +40,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/lib/testing.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/execution_options_util.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/session.pb.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -66,6 +67,7 @@ struct Options {
   bool use_fake_data = false;
   bool print_result = true;
   int num_runs = 1;
+  bool xla_hlo_profile_last_run = false;
 };
 
 // Invokes the given computation passing arbitrary data for every (unbound)
@@ -122,16 +124,21 @@ StatusOr<std::unique_ptr<Literal>> ReplayComputation(
   std::unique_ptr<Literal> result;
   for (int i = 0; i < opts.num_runs; ++i) {
     ExecutionProfile profile;
+    ExecutionOptions execution_options = CreateDefaultExecutionOptions();
+    if (opts.xla_hlo_profile_last_run && i == opts.num_runs - 1) {
+      execution_options.mutable_debug_options()->set_xla_hlo_profile(true);
+    }
+
     if (opts.print_result) {
-      TF_ASSIGN_OR_RETURN(result, client->ExecuteAndTransfer(
-                                      computation, execute_arguments,
-                                      /*execution_options=*/nullptr, &profile));
+      TF_ASSIGN_OR_RETURN(
+          result, client->ExecuteAndTransfer(computation, execute_arguments,
+                                             &execution_options, &profile));
     } else {
       // If we're not printing the result, execute the computation but don't
       // bother retrieving the result.  This can be a significant speedup.
       TF_RETURN_IF_ERROR(client
                              ->Execute(computation, execute_arguments,
-                                       /*execution_options=*/nullptr, &profile)
+                                       &execution_options, &profile)
                              .status());
     }
     LOG(INFO) << "Execution took "
@@ -191,6 +198,9 @@ int main(int argc, char** argv) {
                        "Number of times to run each computation"),
       tensorflow::Flag("fake_infeed_shape", &opts.fake_infeed_shape,
                        "Shape of fake data to construct for (infinite) infeed"),
+      tensorflow::Flag(
+          "xla_hlo_profile_last_run", &opts.xla_hlo_profile_last_run,
+          "Pass --xla_hlo_profile the last time we run the computation."),
   };
   xla::string usage = tensorflow::Flags::Usage(argv[0], flag_list);
   bool parse_ok = tensorflow::Flags::Parse(&argc, argv, flag_list);
-- 
GitLab


From 39c5b0470e7e6b6f79c8f55d89ea46585168f2a8 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 15 Mar 2018 10:54:42 -0700
Subject: [PATCH 059/960] Really delete old checkpoints this time.

Follows up on cl/188187349, which fixed checkpoint management for tf.train.Saver
when executing eagerly. Except I was recreating the tf.train.Saver objects each
save, so tfe.Checkpoint and friends did not benefit from that change.

Keeps the same tf.train.Saver around when executing eagerly. This limits object
graph mutations just like when graph building; if there are complaints I can
assign to Saver._var_list instead, since eager tf.train.Saver is not specialized
to its var_list argument.

PiperOrigin-RevId: 189211552
---
 .../eager/python/checkpointable_utils.py      | 17 ++++++++--------
 .../eager/python/checkpointable_utils_test.py | 20 +++++++++++++++++++
 2 files changed, 28 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index 677b56b7b8..389d4a03c8 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -602,8 +602,7 @@ class CheckpointableSaver(object):
     """
     named_variables, graph_proto = _serialize_object_graph(
         self._root_checkpointable)
-    in_graph_mode = not context.executing_eagerly()
-    if in_graph_mode:
+    if not context.executing_eagerly():
       if session is None:
         session = ops.get_default_session()
       if self._object_graph_feed_tensor is None:
@@ -622,17 +621,17 @@ class CheckpointableSaver(object):
     named_variables[_OBJECT_GRAPH_PROTO_KEY] = _NoRestoreSaveable(
         tensor=object_graph_tensor,
         name=_OBJECT_GRAPH_PROTO_KEY)
-    if not in_graph_mode or self._last_save_object_graph != graph_proto:
-      if self._last_save_object_graph is not None and in_graph_mode:
+    if self._last_save_object_graph != graph_proto:
+      if self._last_save_object_graph is not None:
         raise NotImplementedError(
             "Using a single Saver to save a mutated object graph is not "
             "currently supported when graph building. Use a different Saver "
-            "when the object graph changes (save ops will be duplicated), or "
-            "file a feature request if this limitation bothers you.")
+            "when the object graph changes (save ops will be duplicated when "
+            "graph building), or file a feature request if this limitation "
+            "bothers you.")
       saver = saver_lib.Saver(var_list=named_variables)
-      if in_graph_mode:
-        self._last_save_saver = saver
-        self._last_save_object_graph = graph_proto
+      self._last_save_saver = saver
+      self._last_save_object_graph = graph_proto
     else:
       saver = self._last_save_saver
     with ops.device("/cpu:0"):
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 31f661e634..1ab94b88bd 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -849,6 +849,26 @@ class CheckpointingTests(test.TestCase):
         saver.save(checkpoint_prefix)
         self.assertEqual(before_ops, graph.get_operations())
 
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testCheckpointCleanup(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    obj = checkpointable.Checkpointable()
+    obj.var = variable_scope.get_variable(name="v", initializer=0.)
+    self.evaluate(checkpointable_utils.gather_initializers(obj))
+    saver = checkpointable_utils.Checkpoint(obj=obj)
+    for _ in range(10):
+      saver.save(checkpoint_prefix)
+    expected_filenames = ["checkpoint"]
+    for checkpoint_number in range(6, 11):
+      expected_filenames.append("ckpt-%d.index" % (checkpoint_number,))
+      expected_filenames.append(
+          "ckpt-%d.data-00000-of-00001" % (checkpoint_number,))
+    six.assertCountEqual(
+        self,
+        expected_filenames,
+        os.listdir(checkpoint_directory))
+
   def testManyRestoresGraph(self):
     """Restores after the first should not modify the graph."""
     with context.graph_mode():
-- 
GitLab


From 5e610ebf7f13b443dbaeef261f6c1b0429fb2592 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 15 Mar 2018 11:03:42 -0700
Subject: [PATCH 060/960] Internal change.

PiperOrigin-RevId: 189213372
---
 tensorflow/contrib/lite/python/BUILD         | 5 ++++-
 tensorflow/contrib/opt/BUILD                 | 3 +++
 tensorflow/examples/tutorials/word2vec/BUILD | 3 +++
 tensorflow/python/keras/BUILD                | 3 +++
 tensorflow/python/tools/BUILD                | 5 ++++-
 5 files changed, 17 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index 76607af079..d6f39219ed 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -69,7 +69,10 @@ py_test(
     name = "lite_test",
     srcs = ["lite_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_oss"],
+    tags = [
+        "no-internal-py3",
+        "no_oss",
+    ],
     deps = [
         ":lite",
         ":op_hint",
diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index 827279bd47..bacf15bbd6 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -52,6 +52,9 @@ py_test(
     name = "external_optimizer_test",
     srcs = ["python/training/external_optimizer_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no-internal-py3",
+    ],
     deps = [
         ":opt_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/examples/tutorials/word2vec/BUILD b/tensorflow/examples/tutorials/word2vec/BUILD
index 42d6355b4f..bfcf459269 100644
--- a/tensorflow/examples/tutorials/word2vec/BUILD
+++ b/tensorflow/examples/tutorials/word2vec/BUILD
@@ -13,6 +13,9 @@ py_binary(
         "word2vec_basic.py",
     ],
     srcs_version = "PY2AND3",
+    tags = [
+        "no-internal-py3",
+    ],
     deps = [
         "//tensorflow:tensorflow_py",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 8ace3e0968..eef91e9c5b 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -774,6 +774,9 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/engine/topology_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "no-internal-py3",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD
index 5415881cae..1de1adcfbc 100644
--- a/tensorflow/python/tools/BUILD
+++ b/tensorflow/python/tools/BUILD
@@ -249,7 +249,10 @@ py_test(
         "//tensorflow/cc/saved_model:saved_model_half_plus_two",
     ],
     srcs_version = "PY2AND3",
-    tags = ["manual"],
+    tags = [
+        "manual",
+        "no-internal-py3",
+    ],
     deps = [
         ":saved_model_cli",
         "//tensorflow/core:protos_all_py",
-- 
GitLab


From 4ce5beb9c5dab5d49c00f4a5f5dbd124b700cd56 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 11:16:21 -0700
Subject: [PATCH 061/960] Refactor and enable loop optimizer tests.

PiperOrigin-RevId: 189215781
---
 tensorflow/core/grappler/optimizers/BUILD     |   6 -
 .../optimizers/loop_optimizer_test.cc         | 550 ++++++++----------
 2 files changed, 230 insertions(+), 326 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ffeaa3835e..0df5307d9c 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -538,13 +538,7 @@ cc_library(
 
 tf_cc_test(
     name = "loop_optimizer_test",
-    size = "small",
     srcs = ["loop_optimizer_test.cc"],
-    tags = [
-        "manual",
-        "no_oss",  # b/74111495
-        "notap",
-    ],
     deps = [
         ":loop_optimizer",
         "//tensorflow/cc:cc_ops",
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
index 0bd202a2ab..0d45ba9b56 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/utils/grappler_test.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -26,72 +27,56 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-class LoopOptimizerTest : public ::testing::Test {
+class LoopOptimizerTest : public GrapplerTest {
  protected:
-  static NodeDef CreateNode(const string& name,
-                            const std::vector<string>& inputs) {
-    return CreateNode(name, "Identity", "", false, 0, inputs);
-  }
-  static NodeDef CreateNode(const string& name, const string& op,
-                            const std::vector<string>& inputs) {
-    return CreateNode(name, op, "", false, 0, inputs);
+  // These helpers always sets T=DT_FLOAT.
+  void AddEnterNode(const string& name, const string& frame,
+                    const bool is_constant, const int piterations,
+                    const std::vector<string>& inputs, GraphDef* graph) const {
+    std::vector<std::pair<string, AttrValue>> attributes;
+    AttrValue type;
+    type.set_type(DT_FLOAT);
+    attributes.emplace_back("T", type);
+    AttrValue frame_name;
+    frame_name.set_s(frame);
+    attributes.emplace_back("frame_name", frame_name);
+    AttrValue is_const;
+    is_const.set_b(is_constant);
+    attributes.emplace_back("is_constant", is_const);
+    AttrValue parallel_iterations;
+    parallel_iterations.set_i(piterations);
+    attributes.emplace_back("parallel_iterations", parallel_iterations);
+    AddNode(name, "Enter", inputs, attributes, graph);
   }
-  static NodeDef CreateNode(const string& name, const string& op,
-                            const string& frame,
-                            const bool is_constant,
-                            const int piterations,
-                            const std::vector<string>& inputs) {
-    NodeDef node;
-    node.set_name(name);
-    if (!op.empty()) {
-      node.set_op(op);
-    }
-    if (!frame.empty()) {
-      AttrValue frame_name;
-      frame_name.set_s(frame);
-      node.mutable_attr()->insert({"frame_name", frame_name});
-    }
-    if (op == "Enter") {
-      AttrValue is_const;
-      is_const.set_b(is_constant);
-      node.mutable_attr()->insert({"is_constant", is_const});
-      AttrValue parallel_iterations;
-      parallel_iterations.set_i(piterations);
-      node.mutable_attr()->insert(
-          {"parallel_iterations", parallel_iterations});
-    }
+
+  void AddSimpleNode(const string& name, const string& op,
+                     const std::vector<string>& inputs, GraphDef* graph) const {
+    std::vector<std::pair<string, AttrValue>> attributes;
     AttrValue type;
     type.set_type(DT_FLOAT);
-    node.mutable_attr()->insert({"T", type});
-    for (const string& input : inputs) {
-      node.add_input(input);
-    }
-    return node;
+    attributes.emplace_back("T", type);
+    AddNode(name, op, inputs, attributes, graph);
   }
 };
 
 TEST_F(LoopOptimizerTest, Basic) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"VariantAdd", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"VariantAdd"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"VariantAdd", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"VariantAdd"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -123,27 +108,22 @@ TEST_F(LoopOptimizerTest, Basic) {
 
 TEST_F(LoopOptimizerTest, Const) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode("Const", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "Const"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"VariantAdd", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"VariantAdd"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("Const", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "Const"}, &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"VariantAdd", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"VariantAdd"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -174,27 +154,23 @@ TEST_F(LoopOptimizerTest, Const) {
 
 TEST_F(LoopOptimizerTest, ControlOutput) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode(
-      "Less", "Less", {"VariantAdd", "less/y", "^InvariantAdd"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"VariantAdd"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"VariantAdd", "Less/y", "^InvariantAdd"},
+                &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"VariantAdd"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -223,47 +199,38 @@ TEST_F(LoopOptimizerTest, ControlOutput) {
 
 TEST_F(LoopOptimizerTest, NestedLoop1) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"Exit2", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"Exit2"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
-
-  *graph.add_node() = CreateNode(
-      "InvariantEnter2", "Enter", "while/while/while_context", true, 1,
-      {"VariantAdd"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd2", "Add", {"InvariantEnter2", "InvariantEnter2"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd2", "Add", {"InvariantAdd2", "Identity2"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter2", "Enter", "while/while/while_context", false, 1,
-      {"VariantEnter"});
-  *graph.add_node() = CreateNode(
-      "Merge2", "Merge", {"VariantEnter2", "NextIteration2"});
-  *graph.add_node() = CreateNode("Less2/y", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode("Less2", "Less", {"VariantAdd2", "less2/y"});
-  *graph.add_node() = CreateNode("LoopCond2", "LoopCond", {"Less2"});
-  *graph.add_node() = CreateNode("Switch2", "Switch", {"Merge2", "LoopCond2"});
-  *graph.add_node() = CreateNode("Identity2", {"Switch2:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration2", "NextIteration", {"VariantAdd2"});
-  *graph.add_node() = CreateNode("Exit2", "Exit", {"Switch2"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"Exit2", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"Exit2"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
+
+  AddEnterNode("InvariantEnter2", "while/while/while_context", true, 1,
+               {"VariantAdd"}, &graph);
+  AddSimpleNode("InvariantAdd2", "Add", {"InvariantEnter2", "InvariantEnter2"},
+                &graph);
+  AddSimpleNode("VariantAdd2", "Add", {"InvariantAdd2", "Identity2"}, &graph);
+  AddEnterNode("VariantEnter2", "while/while/while_context", false, 1,
+               {"VariantEnter"}, &graph);
+  AddSimpleNode("Merge2", "Merge", {"VariantEnter2", "NextIteration2"}, &graph);
+  AddSimpleNode("Less2/y", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("Less2", "Less", {"VariantAdd2", "Less2/y"}, &graph);
+  AddSimpleNode("LoopCond2", "LoopCond", {"Less2"}, &graph);
+  AddSimpleNode("Switch2", "Switch", {"Merge2", "LoopCond2"}, &graph);
+  AddSimpleNode("Identity2", "Identity", {"Switch2:1"}, &graph);
+  AddSimpleNode("NextIteration2", "NextIteration", {"VariantAdd2"}, &graph);
+  AddSimpleNode("Exit2", "Exit", {"Switch2"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -299,47 +266,38 @@ TEST_F(LoopOptimizerTest, NestedLoop1) {
 
 TEST_F(LoopOptimizerTest, NestedLoop2) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"Exit2", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"Exit2"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
-
-  *graph.add_node() = CreateNode(
-      "InvariantEnter2", "Enter", "while/while/while_context", true, 1,
-      {"InvariantAdd"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd2", "Add", {"InvariantEnter2", "InvariantEnter2"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd2", "Add", {"InvariantAdd2", "Identity2"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter2", "Enter", "while/while/while_context", false, 1,
-      {"VariantEnter"});
-  *graph.add_node() = CreateNode(
-      "Merge2", "Merge", {"VariantEnter2", "NextIteration2"});
-  *graph.add_node() = CreateNode("Less2/y", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode("Less2", "Less", {"VariantAdd2", "less2/y"});
-  *graph.add_node() = CreateNode("LoopCond2", "LoopCond", {"Less2"});
-  *graph.add_node() = CreateNode("Switch2", "Switch", {"Merge2", "LoopCond2"});
-  *graph.add_node() = CreateNode("Identity2", {"Switch2:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration2", "NextIteration", {"VariantAdd2"});
-  *graph.add_node() = CreateNode("Exit2", "Exit", {"Switch2"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"Exit2", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"Exit2"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
+
+  AddEnterNode("InvariantEnter2", "while/while/while_context", true, 1,
+               {"InvariantAdd"}, &graph);
+  AddSimpleNode("InvariantAdd2", "Add", {"InvariantEnter2", "InvariantEnter2"},
+                &graph);
+  AddSimpleNode("VariantAdd2", "Add", {"InvariantAdd2", "Identity2"}, &graph);
+  AddEnterNode("VariantEnter2", "while/while/while_context", false, 1,
+               {"VariantEnter"}, &graph);
+  AddSimpleNode("Merge2", "Merge", {"VariantEnter2", "NextIteration2"}, &graph);
+  AddSimpleNode("Less2/y", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("Less2", "Less", {"VariantAdd2", "Less2/y"}, &graph);
+  AddSimpleNode("LoopCond2", "LoopCond", {"Less2"}, &graph);
+  AddSimpleNode("Switch2", "Switch", {"Merge2", "LoopCond2"}, &graph);
+  AddSimpleNode("Identity2", "Identity", {"Switch2:1"}, &graph);
+  AddSimpleNode("NextIteration2", "NextIteration", {"VariantAdd2"}, &graph);
+  AddSimpleNode("Exit2", "Exit", {"Switch2"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -371,48 +329,38 @@ TEST_F(LoopOptimizerTest, NestedLoop2) {
 
 TEST_F(LoopOptimizerTest, NestedLoopConst1) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"Exit2", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"Exit2"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
-
-  *graph.add_node() = CreateNode(
-      "InvariantEnter2", "Enter", "while/while/while_context", true, 1,
-      {"VariantAdd"});
-  *graph.add_node() = CreateNode("Const2", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd2", "Add", {"InvariantEnter2", "Const2"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd2", "Add", {"InvariantAdd2", "Identity2"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter2", "Enter", "while/while/while_context", false, 1,
-      {"VariantEnter"});
-  *graph.add_node() = CreateNode(
-      "Merge2", "Merge", {"VariantEnter2", "NextIteration2"});
-  *graph.add_node() = CreateNode("Less2/y", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode("Less2", "Less", {"VariantAdd2", "less2/y"});
-  *graph.add_node() = CreateNode("LoopCond2", "LoopCond", {"Less2"});
-  *graph.add_node() = CreateNode("Switch2", "Switch", {"Merge2", "LoopCond2"});
-  *graph.add_node() = CreateNode("Identity2", {"Switch2:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration2", "NextIteration", {"VariantAdd2"});
-  *graph.add_node() = CreateNode("Exit2", "Exit", {"Switch2"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"Exit2", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"Exit2"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
+
+  AddEnterNode("InvariantEnter2", "while/while/while_context", true, 1,
+               {"VariantAdd"}, &graph);
+  AddSimpleNode("Const2", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("InvariantAdd2", "Add", {"InvariantEnter2", "Const2"}, &graph);
+  AddSimpleNode("VariantAdd2", "Add", {"InvariantAdd2", "Identity2"}, &graph);
+  AddEnterNode("VariantEnter2", "while/while/while_context", false, 1,
+               {"VariantEnter"}, &graph);
+  AddSimpleNode("Merge2", "Merge", {"VariantEnter2", "NextIteration2"}, &graph);
+  AddSimpleNode("Less2/y", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("Less2", "Less", {"VariantAdd2", "Less2/y"}, &graph);
+  AddSimpleNode("LoopCond2", "LoopCond", {"Less2"}, &graph);
+  AddSimpleNode("Switch2", "Switch", {"Merge2", "LoopCond2"}, &graph);
+  AddSimpleNode("Identity2", "Identity", {"Switch2:1"}, &graph);
+  AddSimpleNode("NextIteration2", "NextIteration", {"VariantAdd2"}, &graph);
+  AddSimpleNode("Exit2", "Exit", {"Switch2"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -445,48 +393,38 @@ TEST_F(LoopOptimizerTest, NestedLoopConst1) {
 
 TEST_F(LoopOptimizerTest, NestedLoopConst2) {
   GraphDef graph;
-  *graph.add_node() = CreateNode("0", {});
-  *graph.add_node() = CreateNode(
-      "InvariantEnter", "Enter", "while/while_context", true, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd", "Add", {"InvariantAdd", "Identity"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter", "Enter", "while/while_context", false, 1, {"0"});
-  *graph.add_node() = CreateNode(
-      "Merge", "Merge", {"VariantEnter", "NextIteration"});
-  *graph.add_node() = CreateNode("Less/y", "Const", {"^Identity"});
-  *graph.add_node() = CreateNode("Less", "Less", {"Exit2", "less/y"});
-  *graph.add_node() = CreateNode("LoopCond", "LoopCond", {"Less"});
-  *graph.add_node() = CreateNode("Switch", "Switch", {"Merge", "LoopCond"});
-  *graph.add_node() = CreateNode("Identity", {"Switch:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration", "NextIteration", {"Exit2"});
-  *graph.add_node() = CreateNode("Exit", "Exit", {"Switch"});
-  *graph.add_node() = CreateNode("1", {"Exit"});
-
-  *graph.add_node() = CreateNode(
-      "InvariantEnter2", "Enter", "while/while/while_context", true, 1,
-      {"InvariantAdd"});
-  *graph.add_node() = CreateNode("Const2", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode(
-      "InvariantAdd2", "Add", {"InvariantEnter2", "Const2"});
-  *graph.add_node() = CreateNode(
-      "VariantAdd2", "Add", {"InvariantAdd2", "Identity2"});
-  *graph.add_node() = CreateNode(
-      "VariantEnter2", "Enter", "while/while/while_context", false, 1,
-      {"VariantEnter"});
-  *graph.add_node() = CreateNode(
-      "Merge2", "Merge", {"VariantEnter2", "NextIteration2"});
-  *graph.add_node() = CreateNode("Less2/y", "Const", {"^Identity2"});
-  *graph.add_node() = CreateNode("Less2", "Less", {"VariantAdd2", "less2/y"});
-  *graph.add_node() = CreateNode("LoopCond2", "LoopCond", {"Less2"});
-  *graph.add_node() = CreateNode("Switch2", "Switch", {"Merge2", "LoopCond2"});
-  *graph.add_node() = CreateNode("Identity2", {"Switch2:1"});
-  *graph.add_node() = CreateNode(
-      "NextIteration2", "NextIteration", {"VariantAdd2"});
-  *graph.add_node() = CreateNode("Exit2", "Exit", {"Switch2"});
+  AddSimpleNode("In", "Identity", {}, &graph);
+  AddEnterNode("InvariantEnter", "while/while_context", true, 1, {"In"},
+               &graph);
+  AddSimpleNode("InvariantAdd", "Add", {"InvariantEnter", "InvariantEnter"},
+                &graph);
+  AddSimpleNode("VariantAdd", "Add", {"InvariantAdd", "Identity"}, &graph);
+  AddEnterNode("VariantEnter", "while/while_context", false, 1, {"In"}, &graph);
+  AddSimpleNode("Merge", "Merge", {"VariantEnter", "NextIteration"}, &graph);
+  AddSimpleNode("Less/y", "Const", {"^Identity"}, &graph);
+  AddSimpleNode("Less", "Less", {"Exit2", "Less/y"}, &graph);
+  AddSimpleNode("LoopCond", "LoopCond", {"Less"}, &graph);
+  AddSimpleNode("Switch", "Switch", {"Merge", "LoopCond"}, &graph);
+  AddSimpleNode("Identity", "Identity", {"Switch:1"}, &graph);
+  AddSimpleNode("NextIteration", "NextIteration", {"Exit2"}, &graph);
+  AddSimpleNode("Exit", "Exit", {"Switch"}, &graph);
+  AddSimpleNode("Out", "Identity", {"Exit"}, &graph);
+
+  AddEnterNode("InvariantEnter2", "while/while/while_context", true, 1,
+               {"InvariantAdd"}, &graph);
+  AddSimpleNode("Const2", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("InvariantAdd2", "Add", {"InvariantEnter2", "Const2"}, &graph);
+  AddSimpleNode("VariantAdd2", "Add", {"InvariantAdd2", "Identity2"}, &graph);
+  AddEnterNode("VariantEnter2", "while/while/while_context", false, 1,
+               {"VariantEnter"}, &graph);
+  AddSimpleNode("Merge2", "Merge", {"VariantEnter2", "NextIteration2"}, &graph);
+  AddSimpleNode("Less2/y", "Const", {"^Identity2"}, &graph);
+  AddSimpleNode("Less2", "Less", {"VariantAdd2", "Less2/y"}, &graph);
+  AddSimpleNode("LoopCond2", "LoopCond", {"Less2"}, &graph);
+  AddSimpleNode("Switch2", "Switch", {"Merge2", "LoopCond2"}, &graph);
+  AddSimpleNode("Identity2", "Identity", {"Switch2:1"}, &graph);
+  AddSimpleNode("NextIteration2", "NextIteration", {"VariantAdd2"}, &graph);
+  AddSimpleNode("Exit2", "Exit", {"Switch2"}, &graph);
 
   GrapplerItem item;
   item.graph = graph;
@@ -544,50 +482,26 @@ TEST_F(LoopOptimizerTest, NoOp) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
-namespace {
-NodeDef* AddNode(const string& name, const string& op,
-                 const std::vector<string>& inputs,
-                 const std::vector<std::pair<string, AttrValue>>& attributes,
-                 GraphDef* graph) {
-  NodeDef* node = graph->add_node();
-  node->set_name(name);
-  node->set_op(op);
-  for (const string& input : inputs) {
-    node->add_input(input);
-  }
-  for (auto attr : attributes) {
-    (*node->mutable_attr())[attr.first] = attr.second;
-  }
-  return node;
-}
-}  // namespace
-
 TEST_F(LoopOptimizerTest, RemovePush_NoOp) {
   GrapplerItem item;
-  AttrValue frame_name;
-  frame_name.set_s("foo");
-  AttrValue type;
-  type.set_type(DT_RESOURCE);
   GraphDef& graph = item.graph;
-  AddNode("c", "Const", {}, {}, &graph);
+  AddSimpleNode("c", "Const", {}, &graph);
   // Stack with corresponding push/pop.
-  AddNode("stack1", "StackV2", {}, {}, &graph);
-  AddNode("push1", "StackPushV2", {"stack1", "c"}, {}, &graph);
-  AddNode("pop1", "StackPopV2", {"stack1"}, {}, &graph);
-  AddNode("id1", "Identity", {"pop1"}, {}, &graph);
+  AddSimpleNode("stack1", "StackV2", {}, &graph);
+  AddSimpleNode("push1", "StackPushV2", {"stack1", "c"}, &graph);
+  AddSimpleNode("pop1", "StackPopV2", {"stack1"}, &graph);
+  AddSimpleNode("id1", "Identity", {"pop1"}, &graph);
   // Stack with corresponding push/pop behind Enter.
-  AddNode("stack2", "StackV2", {}, {}, &graph);
-  AddNode("push_enter", "Enter", {"stack2"},
-          {{"T", type}, {"frame_name", frame_name}}, &graph);
-  AddNode("push2", "StackPushV2", {"push_enter", "c"}, {}, &graph);
-  AddNode("pop_enter", "Enter", {"stack2"},
-          {{"T", type}, {"frame_name", frame_name}}, &graph);
-  AddNode("pop2", "StackPopV2", {"pop_enter"}, {}, &graph);
-  AddNode("id2", "Identity", {"pop2"}, {}, &graph);
+  AddSimpleNode("stack2", "StackV2", {}, &graph);
+  AddEnterNode("enter2_c", "frame_name", false, 1, {"c"}, &graph);
+  AddEnterNode("enter2_stack2", "frame_name", false, 1, {"stack2"}, &graph);
+  AddSimpleNode("push2", "StackPushV2", {"enter2_stack2", "enter2_c"}, &graph);
+  AddSimpleNode("pop2", "StackPopV2", {"enter2_stack2"}, &graph);
+  AddSimpleNode("id2", "Identity", {"pop2"}, &graph);
   // Stack with unexpected op type in fanout of Stack.
-  AddNode("stack3", "StackV2", {}, {}, &graph);
-  AddNode("push3", "StackPushV2", {"stack3", "c"}, {}, &graph);
-  AddNode("stop", "StopGradient", {"stack3"}, {}, &graph);
+  AddSimpleNode("stack3", "StackV2", {}, &graph);
+  AddSimpleNode("push3", "StackPushV2", {"stack3", "c"}, &graph);
+  AddSimpleNode("stop", "StopGradient", {"stack3"}, &graph);
   LoopOptimizer optimizer;
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -598,29 +512,25 @@ TEST_F(LoopOptimizerTest, RemovePush_NoOp) {
 TEST_F(LoopOptimizerTest, RemovePushWithoutMatchingPop) {
   GrapplerItem item;
   GraphDef& graph = item.graph;
-  AttrValue frame_name;
-  frame_name.set_s("foo");
-  AttrValue type;
-  type.set_type(DT_RESOURCE);
-  AddNode("c", "Const", {}, {}, &graph);
+  AddSimpleNode("c", "Const", {}, &graph);
   // Push without Pop.
-  AddNode("stack1", "StackV2", {}, {}, &graph);
-  AddNode("push1", "StackPushV2", {"stack1", "c"}, {}, &graph);
+  AddSimpleNode("stack1", "StackV2", {}, &graph);
+  AddSimpleNode("push1", "StackPushV2", {"stack1", "c"}, &graph);
   // Push without Pop behind Enter.
-  AddNode("stack2", "StackV2", {}, {}, &graph);
-  AddNode("push_enter", "Enter", {"stack2"},
-          {{"T", type}, {"frame_name", frame_name}}, &graph);
-  AddNode("push2", "StackPushV2", {"push_enter", "c"}, {}, &graph);
+  AddSimpleNode("stack2", "StackV2", {}, &graph);
+  AddEnterNode("enter_c", "frame_name", false, 1, {"c"}, &graph);
+  AddEnterNode("enter_stack2", "frame_name", false, 1, {"stack2"}, &graph);
+  AddSimpleNode("push2", "StackPushV2", {"enter_stack2", "enter_c"}, &graph);
   // Pop without consumer.
-  AddNode("stack3", "StackV2", {}, {}, &graph);
-  AddNode("push3", "StackPushV2", {"stack3", "c"}, {}, &graph);
-  AddNode("pop3", "StackPopV2", {"stack3"}, {}, &graph);
+  AddSimpleNode("stack3", "StackV2", {}, &graph);
+  AddSimpleNode("push3", "StackPushV2", {"stack3", "c"}, &graph);
+  AddSimpleNode("pop3", "StackPopV2", {"stack3"}, &graph);
 
   LoopOptimizer optimizer;
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  EXPECT_EQ(9, output.node_size());
+  EXPECT_EQ(10, output.node_size());
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     if (node.name() == "push1") {
@@ -631,8 +541,8 @@ TEST_F(LoopOptimizerTest, RemovePushWithoutMatchingPop) {
     } else if (node.name() == "push2") {
       EXPECT_EQ("Identity", node.op());
       EXPECT_EQ(2, node.input_size());
-      EXPECT_EQ("c", node.input(0));
-      EXPECT_EQ("^push_enter", node.input(1));
+      EXPECT_EQ("enter_c", node.input(0));
+      EXPECT_EQ("^enter_stack2", node.input(1));
     } else if (node.name() == "push3") {
       EXPECT_EQ("Identity", node.op());
       EXPECT_EQ(2, node.input_size());
-- 
GitLab


From 143f3585ca78380891cb862880c14c4ca9d7b9fd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 11:19:32 -0700
Subject: [PATCH 062/960] Broadcast Sub and Div from #17123 except for
 quantization.

PiperOrigin-RevId: 189216312
---
 tensorflow/contrib/lite/kernels/BUILD         |  32 +++++
 tensorflow/contrib/lite/kernels/div.cc        |  73 +++++++----
 tensorflow/contrib/lite/kernels/div_test.cc   | 118 ++++++++++++++++++
 .../internal/optimized/optimized_ops.h        |  91 ++++++++++++++
 .../internal/reference/reference_ops.h        |  82 ++++++++++++
 tensorflow/contrib/lite/kernels/sub.cc        |  77 ++++++++----
 tensorflow/contrib/lite/kernels/sub_test.cc   | 117 +++++++++++++++++
 .../testing/generated_examples_zip_test.cc    |  18 ++-
 8 files changed, 551 insertions(+), 57 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/div_test.cc
 create mode 100644 tensorflow/contrib/lite/kernels/sub_test.cc

diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 9c63269324..200cb3075b 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -269,6 +269,38 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "div_test",
+    size = "small",
+    srcs = ["div_test.cc"],
+    tags = [
+        "tflite_not_portable_ios_arm64",
+        "tflite_not_portable_ios_x86_64",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
+tf_cc_test(
+    name = "sub_test",
+    size = "small",
+    srcs = ["sub_test.cc"],
+    tags = [
+        "tflite_not_portable_ios_arm64",
+        "tflite_not_portable_ios_x86_64",
+    ],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "transpose_test",
     size = "small",
diff --git a/tensorflow/contrib/lite/kernels/div.cc b/tensorflow/contrib/lite/kernels/div.cc
index 44bd0dc85d..6dd243ad62 100644
--- a/tensorflow/contrib/lite/kernels/div.cc
+++ b/tensorflow/contrib/lite/kernels/div.cc
@@ -37,7 +37,23 @@ constexpr int kInputTensor1 = 0;
 constexpr int kInputTensor2 = 1;
 constexpr int kOutputTensor = 0;
 
+struct OpData {
+  bool requires_broadcast;
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new OpData;
+  data->requires_broadcast = false;
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
+
 TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
+
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
 
@@ -45,35 +61,47 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
-  TF_LITE_ENSURE_EQ(context, NumDimensions(input1), NumDimensions(input2));
-  for (int i = 0; i < NumDimensions(input1); ++i) {
-    TF_LITE_ENSURE_EQ(context, SizeOfDimension(input1, i),
-                      SizeOfDimension(input2, i));
-  }
+  TF_LITE_ENSURE_EQ(context, input1->type, input2->type);
+  output->type = input2->type;
+
+  data->requires_broadcast = !HaveSameShapes(input1, input2);
 
-  TF_LITE_ENSURE_EQ(context, input1->type, output->type);
-  TF_LITE_ENSURE_EQ(context, input2->type, output->type);
+  TfLiteIntArray* output_size = nullptr;
+  if (data->requires_broadcast) {
+    TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast(
+                                   context, input1, input2, &output_size));
+  } else {
+    output_size = TfLiteIntArrayCopy(input1->dims);
+  }
 
-  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input1->dims);
   return context->ResizeTensor(context, output, output_size);
 }
 
 template <KernelType kernel_type>
-void EvalDivFloat(TfLiteContext* context, TfLiteNode* node,
-                  TfLiteDivParams* params, TfLiteTensor* input1,
-                  TfLiteTensor* input2, TfLiteTensor* output) {
+void EvalFloat(TfLiteContext* context, TfLiteNode* node,
+               TfLiteDivParams* params, const OpData* data,
+               TfLiteTensor* input1, TfLiteTensor* input2,
+               TfLiteTensor* output) {
   float output_activation_min, output_activation_max;
   CalculateActivationRangeFloat(params->activation, &output_activation_min,
                                 &output_activation_max);
-#define TF_LITE_DIV(type)                                        \
-  type::Div(GetTensorData<float>(input1), GetTensorDims(input1), \
-            GetTensorData<float>(input2), GetTensorDims(input2), \
-            output_activation_min, output_activation_max,        \
-            GetTensorData<float>(output), GetTensorDims(output))
+#define TF_LITE_DIV(type, opname)                                   \
+  type::opname(GetTensorData<float>(input1), GetTensorDims(input1), \
+               GetTensorData<float>(input2), GetTensorDims(input2), \
+               output_activation_min, output_activation_max,        \
+               GetTensorData<float>(output), GetTensorDims(output))
   if (kernel_type == kReference) {
-    TF_LITE_DIV(reference_ops);
+    if (data->requires_broadcast) {
+      TF_LITE_DIV(reference_ops, BroadcastDiv);
+    } else {
+      TF_LITE_DIV(reference_ops, Div);
+    }
   } else {
-    TF_LITE_DIV(optimized_ops);
+    if (data->requires_broadcast) {
+      TF_LITE_DIV(optimized_ops, BroadcastDiv);
+    } else {
+      TF_LITE_DIV(optimized_ops, Div);
+    }
   }
 #undef TF_LITE_DIV
 }
@@ -81,13 +109,14 @@ void EvalDivFloat(TfLiteContext* context, TfLiteNode* node,
 template <KernelType kernel_type>
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params = reinterpret_cast<TfLiteDivParams*>(node->builtin_data);
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
 
   TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
   TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
   if (output->type == kTfLiteFloat32) {
-    EvalDivFloat<kernel_type>(context, node, params, input1, input2, output);
+    EvalFloat<kernel_type>(context, node, params, data, input1, input2, output);
   } else {
     context->ReportError(context, "Inputs and outputs not all float types.");
     return kTfLiteError;
@@ -99,19 +128,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 }  // namespace div
 
 TfLiteRegistration* Register_DIV_REF() {
-  static TfLiteRegistration r = {nullptr, nullptr, div::Prepare,
+  static TfLiteRegistration r = {div::Init, div::Free, div::Prepare,
                                  div::Eval<div::kReference>};
   return &r;
 }
 
 TfLiteRegistration* Register_DIV_GENERIC_OPT() {
-  static TfLiteRegistration r = {nullptr, nullptr, div::Prepare,
+  static TfLiteRegistration r = {div::Init, div::Free, div::Prepare,
                                  div::Eval<div::kGenericOptimized>};
   return &r;
 }
 
 TfLiteRegistration* Register_DIV_NEON_OPT() {
-  static TfLiteRegistration r = {nullptr, nullptr, div::Prepare,
+  static TfLiteRegistration r = {div::Init, div::Free, div::Prepare,
                                  div::Eval<div::kNeonOptimized>};
   return &r;
 }
diff --git a/tensorflow/contrib/lite/kernels/div_test.cc b/tensorflow/contrib/lite/kernels/div_test.cc
new file mode 100644
index 0000000000..276b8289fb
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/div_test.cc
@@ -0,0 +1,118 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class BaseDivOpModel : public SingleOpModel {
+ public:
+  BaseDivOpModel(const TensorData& input1, const TensorData& input2,
+                 const TensorData& output,
+                 ActivationFunctionType activation_type) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_DIV, BuiltinOptions_DivOptions,
+                 CreateDivOptions(builder_, activation_type).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  int input1() { return input1_; }
+  int input2() { return input2_; }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+class FloatDivOpModel : public BaseDivOpModel {
+ public:
+  using BaseDivOpModel::BaseDivOpModel;
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+};
+
+TEST(FloatDivOpTest, NoActivation) {
+  FloatDivOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+  m.PopulateTensor<float>(m.input1(), {-0.2, 0.2, -1.2, 0.8});
+  m.PopulateTensor<float>(m.input2(), {0.5, 0.2, -1.5, 0.5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({-0.4, 1.0, 0.8, 1.6})));
+}
+
+TEST(FloatDivOpTest, ActivationRELU_N1_TO_1) {
+  FloatDivOpModel m(
+      {TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {1, 2, 2, 1}},
+      {TensorType_FLOAT32, {}}, ActivationFunctionType_RELU_N1_TO_1);
+  m.PopulateTensor<float>(m.input1(), {-0.2, 0.2, -1.2, 0.8});
+  m.PopulateTensor<float>(m.input2(), {0.1, 0.2, -1.5, 0.5});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({-1.0, 1.0, 0.8, 1.0})));
+}
+
+TEST(FloatDivOpTest, VariousInputShapes) {
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 0.3, 0.8, 1.1, -2.0});
+    m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.6, 0.5, -1.1, -0.1});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({-20.0, 1.0, 0.5, 1.6, -1.0, 20.0})))
+        << "With shape number " << i;
+  }
+}
+
+TEST(FloatDivOpTest, WithBroadcast) {
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatDivOpModel m({TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, {}},  // always a scalar
+                      {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor<float>(m.input1(), {-0.2, 0.2, 0.07, 0.08, 0.11, -0.123});
+    m.PopulateTensor<float>(m.input2(), {0.1});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({-2.0, 2.0, 0.7, 0.8, 1.1, -1.23})))
+        << "With shape number " << i;
+  }
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index 6bbc213cc6..edd65c9170 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -2157,6 +2157,51 @@ inline void Div(const float* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary
+// dimensionality if the runtime code does a single loop over one dimension
+// that handles broadcasting as the base case. The code generator would then
+// generate max(D1, D2) nested for loops.
+// TODO(benoitjacob): BroadcastDiv is intentionally duplicated from
+// reference_ops.h. Once an optimized version is implemented and NdArrayDesc<T>
+// is no longer referenced in this file, move NdArrayDesc<T> from types.h to
+// reference_ops.h.
+template <typename T>
+void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastDiv");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          output_data[Offset(output_dims, c, x, y, b)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] /
+                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
+                  output_activation_min, output_activation_max);
+        }
+      }
+    }
+  }
+}
+
 // TODO(aselle): This is not actually optimized yet.
 inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
                 const float* input2_data, const Dims<4>& input2_dims,
@@ -2184,6 +2229,52 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
     }
   }
 }
+
+// TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary
+// dimensionality if the runtime code does a single loop over one dimension
+// that handles broadcasting as the base case. The code generator would then
+// generate max(D1, D2) nested for loops.
+// TODO(benoitjacob): BroadcastSub is intentionally duplicated from
+// reference_ops.h. Once an optimized version is implemented and NdArrayDesc<T>
+// is no longer referenced in this file, move NdArrayDesc<T> from types.h to
+// reference_ops.h.
+template <typename T>
+void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastSub");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          output_data[Offset(output_dims, c, x, y, b)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
+                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
+                  output_activation_min, output_activation_max);
+        }
+      }
+    }
+  }
+}
+
 template <FusedActivationFunctionType Ac, typename Scalar>
 void Concatenation(int concat_dim, const Scalar* const* input_data,
                    const Dims<4>* const* input_dims, int inputs_count,
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index d3d15edf4c..527276f7bd 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -1325,6 +1325,47 @@ inline void BroadcastMul(const uint8* input1_data, const Dims<4>& input1_dims,
                output_data, output_dims);
 }
 
+// TODO(jiawen): We can implement BroadcastDiv on buffers of arbitrary
+// dimensionality if the runtime code does a single loop over one dimension
+// that handles broadcasting as the base case. The code generator would then
+// generate max(D1, D2) nested for loops.
+template <typename T>
+void BroadcastDiv(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastDiv");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest
+  // stride, typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for
+  // the best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          output_data[Offset(output_dims, c, x, y, b)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] /
+                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
+                  output_activation_min, output_activation_max);
+        }
+      }
+    }
+  }
+}
+
 inline void Div(const float* input1_data, const Dims<4>& input1_dims,
                 const float* input2_data, const Dims<4>& input2_dims,
                 float output_activation_min, float output_activation_max,
@@ -1379,6 +1420,47 @@ inline void Sub(const float* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+// TODO(jiawen): We can implement BroadcastSub on buffers of arbitrary
+// dimensionality if the runtime code does a single loop over one dimension
+// that handles broadcasting as the base case. The code generator would then
+// generate max(D1, D2) nested for loops.
+template <typename T>
+void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
+                  const T* input2_data, const Dims<4>& input2_dims,
+                  T output_activation_min, T output_activation_max,
+                  T* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastSub");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          output_data[Offset(output_dims, c, x, y, b)] =
+              ActivationFunctionWithMinMax(
+                  input1_data[SubscriptToIndex(desc1, c, x, y, b)] -
+                      input2_data[SubscriptToIndex(desc2, c, x, y, b)],
+                  output_activation_min, output_activation_max);
+        }
+      }
+    }
+  }
+}
+
 template <FusedActivationFunctionType Ac, typename Scalar>
 void Concatenation(int concat_dim, const Scalar* const* input_data,
                    const Dims<4>* const* input_dims, int inputs_count,
diff --git a/tensorflow/contrib/lite/kernels/sub.cc b/tensorflow/contrib/lite/kernels/sub.cc
index ddaf498d5b..c15a7a50a4 100644
--- a/tensorflow/contrib/lite/kernels/sub.cc
+++ b/tensorflow/contrib/lite/kernels/sub.cc
@@ -26,7 +26,7 @@ namespace ops {
 namespace builtin {
 namespace sub {
 
-// This file has three implementation of Div.
+// This file has three implementation of Sub.
 enum KernelType {
   kReference,
   kGenericOptimized,  // Neon-free
@@ -37,7 +37,23 @@ constexpr int kInputTensor1 = 0;
 constexpr int kInputTensor2 = 1;
 constexpr int kOutputTensor = 0;
 
+struct OpData {
+  bool requires_broadcast;
+};
+
+void* Init(TfLiteContext* context, const char* buffer, size_t length) {
+  auto* data = new OpData;
+  data->requires_broadcast = false;
+  return data;
+}
+
+void Free(TfLiteContext* context, void* buffer) {
+  delete reinterpret_cast<OpData*>(buffer);
+}
+
 TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
+
   TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
 
@@ -45,49 +61,62 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
-  TF_LITE_ENSURE_EQ(context, NumDimensions(input1), NumDimensions(input2));
-  for (int i = 0; i < NumDimensions(input1); ++i) {
-    TF_LITE_ENSURE_EQ(context, SizeOfDimension(input1, i),
-                      SizeOfDimension(input2, i));
-  }
+  TF_LITE_ENSURE_EQ(context, input1->type, input2->type);
+  output->type = input2->type;
+
+  data->requires_broadcast = !HaveSameShapes(input1, input2);
 
-  TF_LITE_ENSURE_EQ(context, input1->type, output->type);
-  TF_LITE_ENSURE_EQ(context, input2->type, output->type);
+  TfLiteIntArray* output_size = nullptr;
+  if (data->requires_broadcast) {
+    TF_LITE_ENSURE_OK(context, CalculateShapeForBroadcast(
+                                   context, input1, input2, &output_size));
+  } else {
+    output_size = TfLiteIntArrayCopy(input1->dims);
+  }
 
-  TfLiteIntArray* output_size = TfLiteIntArrayCopy(input1->dims);
   return context->ResizeTensor(context, output, output_size);
 }
 
 template <KernelType kernel_type>
-void EvalSubFloat(TfLiteContext* context, TfLiteNode* node,
-                  TfLiteSubParams* params, TfLiteTensor* input1,
-                  TfLiteTensor* input2, TfLiteTensor* output) {
+void EvalFloat(TfLiteContext* context, TfLiteNode* node,
+               TfLiteSubParams* params, const OpData* data,
+               TfLiteTensor* input1, TfLiteTensor* input2,
+               TfLiteTensor* output) {
   float output_activation_min, output_activation_max;
   CalculateActivationRangeFloat(params->activation, &output_activation_min,
                                 &output_activation_max);
-#define TF_LITE_Sub(type)                                        \
-  type::Sub(GetTensorData<float>(input1), GetTensorDims(input1), \
-            GetTensorData<float>(input2), GetTensorDims(input2), \
-            output_activation_min, output_activation_max,        \
-            GetTensorData<float>(output), GetTensorDims(output))
+#define TF_LITE_SUB(type, opname)                                   \
+  type::opname(GetTensorData<float>(input1), GetTensorDims(input1), \
+               GetTensorData<float>(input2), GetTensorDims(input2), \
+               output_activation_min, output_activation_max,        \
+               GetTensorData<float>(output), GetTensorDims(output))
   if (kernel_type == kReference) {
-    TF_LITE_Sub(reference_ops);
+    if (data->requires_broadcast) {
+      TF_LITE_SUB(reference_ops, BroadcastSub);
+    } else {
+      TF_LITE_SUB(reference_ops, Sub);
+    }
   } else {
-    TF_LITE_Sub(optimized_ops);
+    if (data->requires_broadcast) {
+      TF_LITE_SUB(optimized_ops, BroadcastSub);
+    } else {
+      TF_LITE_SUB(optimized_ops, Sub);
+    }
   }
-#undef TF_LITE_Sub
+#undef TF_LITE_SUB
 }
 
 template <KernelType kernel_type>
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params = reinterpret_cast<TfLiteSubParams*>(node->builtin_data);
+  OpData* data = reinterpret_cast<OpData*>(node->user_data);
 
   TfLiteTensor* input1 = GetInput(context, node, kInputTensor1);
   TfLiteTensor* input2 = GetInput(context, node, kInputTensor2);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
 
   if (output->type == kTfLiteFloat32) {
-    EvalSubFloat<kernel_type>(context, node, params, input1, input2, output);
+    EvalFloat<kernel_type>(context, node, params, data, input1, input2, output);
   } else {
     context->ReportError(context, "Inputs and outputs not all float types.");
     return kTfLiteError;
@@ -99,19 +128,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 }  // namespace sub
 
 TfLiteRegistration* Register_SUB_REF() {
-  static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare,
+  static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare,
                                  sub::Eval<sub::kReference>};
   return &r;
 }
 
 TfLiteRegistration* Register_SUB_GENERIC_OPT() {
-  static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare,
+  static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare,
                                  sub::Eval<sub::kGenericOptimized>};
   return &r;
 }
 
 TfLiteRegistration* Register_SUB_NEON_OPT() {
-  static TfLiteRegistration r = {nullptr, nullptr, sub::Prepare,
+  static TfLiteRegistration r = {sub::Init, sub::Free, sub::Prepare,
                                  sub::Eval<sub::kNeonOptimized>};
   return &r;
 }
diff --git a/tensorflow/contrib/lite/kernels/sub_test.cc b/tensorflow/contrib/lite/kernels/sub_test.cc
new file mode 100644
index 0000000000..fdbb4243bb
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/sub_test.cc
@@ -0,0 +1,117 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class BaseSubOpModel : public SingleOpModel {
+ public:
+  BaseSubOpModel(const TensorData& input1, const TensorData& input2,
+                 const TensorData& output,
+                 ActivationFunctionType activation_type) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_SUB, BuiltinOptions_SubOptions,
+                 CreateSubOptions(builder_, activation_type).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  int input1() { return input1_; }
+  int input2() { return input2_; }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+class FloatSubOpModel : public BaseSubOpModel {
+ public:
+  using BaseSubOpModel::BaseSubOpModel;
+
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+};
+
+TEST(FloatSubOpModel, NoActivation) {
+  FloatSubOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {1, 2, 2, 1}},
+                    {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+  m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 1.7, 0.5});
+  m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.8});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({-2.1, 0.0, 1.4, -0.3})));
+}
+
+TEST(FloatSubOpModel, ActivationRELU_N1_TO_1) {
+  FloatSubOpModel m(
+      {TensorType_FLOAT32, {1, 2, 2, 1}}, {TensorType_FLOAT32, {1, 2, 2, 1}},
+      {TensorType_FLOAT32, {}}, ActivationFunctionType_RELU_N1_TO_1);
+  m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 1.7, 0.5});
+  m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.8});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(),
+              ElementsAreArray(ArrayFloatNear({-1.0, 0.0, 1.0, -0.3})));
+}
+
+TEST(FloatSubOpModel, VariousInputShapes) {
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatSubOpModel m({TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 1.7, 0.5, -1.1, 2.0});
+    m.PopulateTensor<float>(m.input2(), {0.1, 0.2, 0.3, 0.8, -1.1, 0.1});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({-2.1, 0.0, 1.4, -0.3, 0.0, 1.9})))
+        << "With shape number " << i;
+  }
+}
+
+TEST(FloatSubOpModel, WithBroadcast) {
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    FloatSubOpModel m({TensorType_FLOAT32, test_shapes[i]},
+                      {TensorType_FLOAT32, {}},  // always a scalar
+                      {TensorType_FLOAT32, {}}, ActivationFunctionType_NONE);
+    m.PopulateTensor<float>(m.input1(), {-2.0, 0.2, 1.7, 0.5, -1.1, 2.0});
+    m.PopulateTensor<float>(m.input2(), {0.5});
+    m.Invoke();
+    EXPECT_THAT(
+        m.GetOutput(),
+        ElementsAreArray(ArrayFloatNear({-2.5, -0.3, 1.2, 0.0, -1.6, 1.5})))
+        << "With shape number " << i;
+  }
+}
+
+}  // namespace
+}  // namespace tflite
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 86606d1239..88c5aaa099 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -47,10 +47,6 @@ tensorflow::Env* env = tensorflow::Env::Default();
 // Key is a substring of the test name and value is a bug number.
 // TODO(ahentz): make sure we clean this list up frequently.
 std::map<string, string> kBrokenTests = {
-    // Sub and Div don't support broadcasting.
-    {R"(^\/diva.*input_shape_1=\[1,3,4,3\],input_shape_2=\[3\])", "68500195"},
-    {R"(^\/suba.*input_shape_1=\[1,3,4,3\],input_shape_2=\[3\])", "68500195"},
-
     // Add only supports float32. (and "constant" tests use Add)
     {R"(^\/adda.*int32)", "68808744"},
     {R"(^\/constant.*int32)", "68808744"},
@@ -238,23 +234,25 @@ TEST_P(OpsTest, RunStuff) {
 
 INSTANTIATE_TESTS(add)
 INSTANTIATE_TESTS(avg_pool)
-INSTANTIATE_TESTS(space_to_batch_nd)
 INSTANTIATE_TESTS(batch_to_space_nd)
 INSTANTIATE_TESTS(concat)
 INSTANTIATE_TESTS(constant)
 INSTANTIATE_TESTS(control_dep)
 INSTANTIATE_TESTS(conv)
 INSTANTIATE_TESTS(depthwiseconv)
+INSTANTIATE_TESTS(div)
 INSTANTIATE_TESTS(exp)
 INSTANTIATE_TESTS(fully_connected)
 INSTANTIATE_TESTS(fused_batch_norm)
 INSTANTIATE_TESTS(gather)
 INSTANTIATE_TESTS(global_batch_norm)
-INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(l2_pool)
+INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
+INSTANTIATE_TESTS(lstm)
 INSTANTIATE_TESTS(max_pool)
+INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(mul)
 INSTANTIATE_TESTS(pad)
 INSTANTIATE_TESTS(relu)
@@ -264,15 +262,13 @@ INSTANTIATE_TESTS(reshape)
 INSTANTIATE_TESTS(resize_bilinear)
 INSTANTIATE_TESTS(sigmoid)
 INSTANTIATE_TESTS(softmax)
+INSTANTIATE_TESTS(space_to_batch_nd)
 INSTANTIATE_TESTS(space_to_depth)
-INSTANTIATE_TESTS(sub)
 INSTANTIATE_TESTS(split)
-INSTANTIATE_TESTS(div)
-INSTANTIATE_TESTS(transpose)
-INSTANTIATE_TESTS(lstm)
-INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(squeeze)
 INSTANTIATE_TESTS(strided_slice)
+INSTANTIATE_TESTS(sub)
+INSTANTIATE_TESTS(transpose)
 
 }  // namespace testing
 }  // namespace tflite
-- 
GitLab


From f465d7467ef5488aca1b4e620c27d4016c774ef8 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Thu, 15 Mar 2018 11:22:58 -0700
Subject: [PATCH 063/960] [XLA:CPU] Fix the parallel task assignment to not
 parallelize dot operations.

The IR emitter currently generates incorrect code for parallelized dot
operations.

Add test cases to check for dot operation parallelization.

PiperOrigin-RevId: 189216963
---
 tensorflow/compiler/xla/service/cpu/BUILD     | 25 ++++++
 .../xla/service/cpu/dot_op_emitter.cc         |  5 ++
 .../service/cpu/parallel_task_assignment.cc   | 19 ++---
 .../cpu/parallel_task_assignment_test.cc      | 84 +++++++++++++++++++
 4 files changed, 123 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 91ae66ece1..093db020c0 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -772,6 +772,31 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "parallel_task_assignment_test",
+    srcs = ["parallel_task_assignment_test.cc"],
+    deps = [
+        ":cpu_executable",
+        ":parallel_task_assignment",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla:shape_layout",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:algebraic_simplifier",
+        "//tensorflow/compiler/xla/service:computation_layout",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_verified_test_base",
+        "//tensorflow/compiler/xla/tests:test_utils",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "cpu_options",
     srcs = ["cpu_options.cc"],
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 6f06256e08..8b1e20d79e 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -715,6 +715,11 @@ tensorflow::Status DotOpEmitter::Emit() {
   // which performs the sum-of-products (the reduction loop) before storing
   // the result in the output buffer.
 
+  // This routine assumes that the dot operation is not in a parallelized
+  // enclosing computation.
+  CHECK(
+      dot_.parent()->root_instruction()->outer_dimension_partitions().empty());
+
   const Shape& lhs_shape = lhs_array_.GetShape();
   const Shape& rhs_shape = rhs_array_.GetShape();
 
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index deb21bf4ef..38f1668159 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -130,22 +130,21 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
   // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
   // *) Tuple-shaped.
   // TODO(b/27458679) Parallelize instructions which are skipped here.
-  if (instruction->opcode() == HloOpcode::kParameter ||
-      instruction->opcode() == HloOpcode::kConstant ||
-      instruction->opcode() == HloOpcode::kCall ||
-      instruction->opcode() == HloOpcode::kCustomCall ||
-      instruction->opcode() == HloOpcode::kSelectAndScatter ||
-      instruction->opcode() == HloOpcode::kGetTupleElement ||
-      instruction->opcode() == HloOpcode::kBitcast ||
-      instruction->opcode() == HloOpcode::kFft ||
-      (instruction->opcode() == HloOpcode::kConvolution &&
+  auto opcode = instruction->opcode();
+  if (opcode == HloOpcode::kParameter || opcode == HloOpcode::kConstant ||
+      opcode == HloOpcode::kCall || opcode == HloOpcode::kCustomCall ||
+      opcode == HloOpcode::kDot || opcode == HloOpcode::kSelectAndScatter ||
+      opcode == HloOpcode::kGetTupleElement || opcode == HloOpcode::kBitcast ||
+      opcode == HloOpcode::kFft ||
+      (opcode == HloOpcode::kConvolution &&
        PotentiallyImplementedAsEigenConvolution(*instruction)) ||
       PotentiallyImplementedAsEigenDot(*instruction) ||
-      (instruction->opcode() == HloOpcode::kFusion &&
+      (opcode == HloOpcode::kFusion &&
        instruction->fusion_kind() != HloInstruction::FusionKind::kLoop) ||
       ShapeUtil::IsTuple(instruction->shape())) {
     return 1;
   }
+
   // Consult 'cost_model_' to compute target parallel task count.
   return cost_model_->GetParallelTaskCount(instruction);
 }
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
new file mode 100644
index 0000000000..79b00135c6
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
@@ -0,0 +1,84 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/parallel_task_assignment.h"
+#include "tensorflow/compiler/xla/service/cpu/cpu_executable.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/hlo_verified_test_base.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+
+namespace xla {
+namespace {
+
+class ParallelTaskAssignmentTest : public HloVerifiedTestBase {
+ protected:
+  const HloCostAnalysis::ShapeSizeFunction shape_size_func_ =
+      cpu::CpuExecutable::ShapeSizeBytes;
+
+  // Use any value larger than 2 since we only test whether a module is
+  // parallelized or not
+  const int max_parallelism_ = 10;
+};
+
+TEST_F(ParallelTaskAssignmentTest, DotOperationNotParallelized) {
+  const string hlo_string = R"(
+    HloModule TestTaskParallel_Dot
+    ENTRY Dot {
+      dot_lhs = f32[196614,2]{1,0} parameter(0)
+      dot_rhs = f32[2,1]{1,0} parameter(1)
+      ROOT dot = f32[196614,1]{1,0} dot(dot_lhs, dot_rhs),
+        lhs_contracting_dims={1}, rhs_contracting_dims={0}
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string.c_str());
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
+                                            max_parallelism_, shape_size_func_)
+                                            .Run(&module()));
+  EXPECT_FALSE(changed);
+}
+
+TEST_F(ParallelTaskAssignmentTest,
+       FusedComputationWithDotOperationNotParallelized) {
+  const string hlo_string = R"(
+    HloModule TestTaskParallel_DotNestedInFusedComp
+    fused_computation.0 {
+      parameter.0 = f32[196614,2]{1,0} parameter(0)
+      parameter.0.1 = f32[2,1]{1,0} parameter(1)
+      parameter.0.2 = f32[196614,1]{1,0} parameter(2)
+      dot.0 = f32[196614,1]{1,0} dot(parameter.0, parameter.0.1),
+        lhs_contracting_dims={1}, rhs_contracting_dims={0}
+      ROOT add.0 = f32[196614,1]{1,0} add(dot.0, parameter.0.2)
+
+    }
+    ENTRY DotNestedInFusedComp {
+      parameter = f32[196614,2]{1,0} parameter(0)
+      parameter.1 = f32[2,1]{1,0} parameter(1)
+      parameter.2 = f32[196614,1]{1,0} parameter(2)
+      ROOT fusion = f32[196614,1]{1,0} fusion(parameter, parameter.1,
+        parameter.2), kind=kOutput, calls=fused_computation.0
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string.c_str());
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
+                                            max_parallelism_, shape_size_func_)
+                                            .Run(&module()));
+  EXPECT_FALSE(changed);
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From b4633736a9a2acd37cc6ec59644d40cb1a2b541b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 11:24:46 -0700
Subject: [PATCH 064/960] Prevent fusing activation functions that would remove
 non-discardable arrays.

PiperOrigin-RevId: 189217360
---
 .../fuse_activation_functions.cc              | 27 ++++++++++++-------
 1 file changed, 18 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/fuse_activation_functions.cc b/tensorflow/contrib/lite/toco/graph_transformations/fuse_activation_functions.cc
index ab943f72d1..c5ce3fcd95 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/fuse_activation_functions.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/fuse_activation_functions.cc
@@ -42,9 +42,9 @@ bool FuseActivationFunctions::Run(Model* model, std::size_t op_index) {
 
   if (CountTrueOutputs(*model, *op) > 1) {
     AddMessageF(
-        "Not fusing activation function into %s because it has more than one "
-        " consumed output",
-        LogName(*op));
+        "Not fusing activation function %s into %s because it has more than "
+        "one  consumed output",
+        LogName(*ac_op), LogName(*op));
     return false;
   }
 
@@ -56,22 +56,31 @@ bool FuseActivationFunctions::Run(Model* model, std::size_t op_index) {
     AddMessageF(
         "Not fusing activation function into %s because it is consumed by more "
         "than 1 other operator",
-        LogName(*op));
+        LogName(*ac_op), LogName(*op));
+    return false;
+  }
+
+  if (!IsDiscardableArray(*model, op->outputs[0])) {
+    AddMessageF(
+        "Not fusing activation function %s into %s because output %s it is not "
+        "discardable",
+        LogName(*ac_op), LogName(*op), op->outputs[0]);
     return false;
   }
 
   if (op->fused_activation_function != FusedActivationFunctionType::kNone) {
     AddMessageF(
-        "Not fusing activation function into %s because it already has a fused "
-        "activation function",
-        LogName(*op));
+        "Not fusing activation function %s into %s because it already has a "
+        "fused activation function",
+        LogName(*ac_op), LogName(*op));
     return false;
   }
 
   if (!OperatorSupportsFusedActivation(op->type)) {
     AddMessageF(
-        "Not fusing activation function because the %s op doesn't support it",
-        LogName(*op));
+        "Not fusing activation function %s because the %s op doesn't support "
+        "it",
+        LogName(*ac_op), LogName(*op));
     return false;
   }
 
-- 
GitLab


From 7e277054400e913af05672a2f6e3519f5b00873a Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Thu, 15 Mar 2018 11:53:00 -0700
Subject: [PATCH 065/960] [TF:XLA] Don't follow control dependencies for const
 analysis.

Those inputs are not actually required to be constants.

PiperOrigin-RevId: 189222308
---
 tensorflow/compiler/tf2xla/const_analysis.cc  |  6 ++++--
 .../compiler/tf2xla/const_analysis_test.cc    | 19 +++++++++++++++++++
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/const_analysis.cc b/tensorflow/compiler/tf2xla/const_analysis.cc
index 6f46532419..de1008803d 100644
--- a/tensorflow/compiler/tf2xla/const_analysis.cc
+++ b/tensorflow/compiler/tf2xla/const_analysis.cc
@@ -55,8 +55,10 @@ Status BackwardsConstAnalysis(const Graph& g,
         compile_time_const_args->at(index) = true;
         return;
       }
-      for (const Node* pred : node->in_nodes()) {
-        must_be_const.insert(pred);
+      for (const Edge* pred : node->in_edges()) {
+        if (!pred->IsControlEdge()) {
+          must_be_const.insert(pred->src());
+        }
       }
       return;
     }
diff --git a/tensorflow/compiler/tf2xla/const_analysis_test.cc b/tensorflow/compiler/tf2xla/const_analysis_test.cc
index 9d125f8d49..992b12c06d 100644
--- a/tensorflow/compiler/tf2xla/const_analysis_test.cc
+++ b/tensorflow/compiler/tf2xla/const_analysis_test.cc
@@ -79,5 +79,24 @@ TEST(ConstAnalysisTest, TopologicalOrder) {
   }
 }
 
+TEST(ConstAnalysisTest, DontFollowControlDependencies) {
+  Scope root = Scope::NewRootScope();
+
+  Output arg0 = ops::_Arg(root.WithOpName("Arg0"), DT_INT32, 0);
+  Output arg1 = ops::_Arg(root.WithOpName("Arg1"), DT_INT32, 1);
+  Output c1 =
+      ops::Const(root.WithOpName("c1").WithControlDependencies(arg0), 1, {1});
+  Output add = ops::Add(root, arg1, c1);
+  Output reshape = ops::Reshape(root, arg1, add);
+
+  Graph graph(OpRegistry::Global());
+  TF_ASSERT_OK(root.ToGraph(&graph));
+
+  std::vector<bool> const_args(2, false);
+  TF_ASSERT_OK(BackwardsConstAnalysis(graph, &const_args));
+
+  EXPECT_EQ(const_args, std::vector<bool>({false, true}));
+}
+
 }  // namespace
 }  // namespace tensorflow
-- 
GitLab


From 9d0640e68e07a65eb315e75e6aa73eb84d60dcf4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 12:06:24 -0700
Subject: [PATCH 066/960] Automated g4 rollback of changelist 189110935

PiperOrigin-RevId: 189224522
---
 tensorflow/cc/framework/cc_op_gen.cc         |  1 +
 tensorflow/core/lib/core/stringpiece.cc      |  5 --
 tensorflow/core/lib/core/stringpiece.h       |  6 --
 tensorflow/core/lib/core/stringpiece_test.cc | 70 -------------------
 tensorflow/core/lib/hash/hash.h              |  1 +
 tensorflow/core/lib/hash/hash_test.cc        | 73 ++++++++++++++++++++
 6 files changed, 75 insertions(+), 81 deletions(-)

diff --git a/tensorflow/cc/framework/cc_op_gen.cc b/tensorflow/cc/framework/cc_op_gen.cc
index 39893f5ccd..d73121c7b7 100644
--- a/tensorflow/cc/framework/cc_op_gen.cc
+++ b/tensorflow/cc/framework/cc_op_gen.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb_text.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 29b727fc44..5bd79778a6 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -17,14 +17,9 @@ limitations under the License.
 
 #include <algorithm>
 #include <iostream>
-#include "tensorflow/core/lib/hash/hash.h"
 
 namespace tensorflow {
 
-size_t StringPieceHasher::operator()(StringPiece s) const {
-  return Hash64(s.data(), s.size());
-}
-
 std::ostream& operator<<(std::ostream& o, StringPiece piece) {
   o.write(piece.data(), piece.size());
   return o;
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index caa9642774..910e4d9e2a 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -35,8 +35,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-struct StringPieceHasher;
-
 class StringPiece {
  public:
   typedef size_t size_type;
@@ -131,10 +129,6 @@ class StringPiece {
   // Intentionally copyable
 };
 
-struct StringPieceHasher {
-  size_t operator()(StringPiece s) const;
-};
-
 inline bool operator==(StringPiece x, StringPiece y) {
   return ((x.size() == y.size()) &&
           (memcmp(x.data(), y.data(), x.size()) == 0));
diff --git a/tensorflow/core/lib/core/stringpiece_test.cc b/tensorflow/core/lib/core/stringpiece_test.cc
index 8f17b85b6d..d0dbeb6072 100644
--- a/tensorflow/core/lib/core/stringpiece_test.cc
+++ b/tensorflow/core/lib/core/stringpiece_test.cc
@@ -65,74 +65,4 @@ TEST(StringPiece, Contains) {
   EXPECT_TRUE(!a.contains(d));
 }
 
-TEST(StringPieceHasher, Equality) {
-  StringPieceHasher hasher;
-
-  StringPiece s1("foo");
-  StringPiece s2("bar");
-  StringPiece s3("baz");
-  StringPiece s4("zot");
-
-  EXPECT_TRUE(hasher(s1) != hasher(s2));
-  EXPECT_TRUE(hasher(s1) != hasher(s3));
-  EXPECT_TRUE(hasher(s1) != hasher(s4));
-  EXPECT_TRUE(hasher(s2) != hasher(s3));
-  EXPECT_TRUE(hasher(s2) != hasher(s4));
-  EXPECT_TRUE(hasher(s3) != hasher(s4));
-
-  EXPECT_TRUE(hasher(s1) == hasher(s1));
-  EXPECT_TRUE(hasher(s2) == hasher(s2));
-  EXPECT_TRUE(hasher(s3) == hasher(s3));
-  EXPECT_TRUE(hasher(s4) == hasher(s4));
-}
-
-TEST(StringPieceHasher, HashMap) {
-  string s1("foo");
-  string s2("bar");
-  string s3("baz");
-
-  StringPiece p1(s1);
-  StringPiece p2(s2);
-  StringPiece p3(s3);
-
-  std::unordered_map<StringPiece, int, StringPieceHasher> map;
-
-  map.insert(std::make_pair(p1, 0));
-  map.insert(std::make_pair(p2, 1));
-  map.insert(std::make_pair(p3, 2));
-  EXPECT_EQ(map.size(), 3);
-
-  bool found[3] = {false, false, false};
-  for (auto const& val : map) {
-    int x = val.second;
-    EXPECT_TRUE(x >= 0 && x < 3);
-    EXPECT_TRUE(!found[x]);
-    found[x] = true;
-  }
-  EXPECT_EQ(found[0], true);
-  EXPECT_EQ(found[1], true);
-  EXPECT_EQ(found[2], true);
-
-  auto new_iter = map.find("zot");
-  EXPECT_TRUE(new_iter == map.end());
-
-  new_iter = map.find("bar");
-  EXPECT_TRUE(new_iter != map.end());
-
-  map.erase(new_iter);
-  EXPECT_EQ(map.size(), 2);
-
-  found[0] = false;
-  found[1] = false;
-  found[2] = false;
-  for (const auto& iter : map) {
-    int x = iter.second;
-    EXPECT_TRUE(x >= 0 && x < 3);
-    EXPECT_TRUE(!found[x]);
-    found[x] = true;
-  }
-  EXPECT_EQ(found[0], true);
-  EXPECT_EQ(found[1], false);
-  EXPECT_EQ(found[2], true);
-}
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/hash/hash.h b/tensorflow/core/lib/hash/hash.h
index 77b8031598..ca05e6346e 100644
--- a/tensorflow/core/lib/hash/hash.h
+++ b/tensorflow/core/lib/hash/hash.h
@@ -76,6 +76,7 @@ struct hash<StringPiece> {
     return static_cast<size_t>(Hash64(sp.data(), sp.size()));
   }
 };
+using StringPieceHasher = ::tensorflow::hash<StringPiece>;
 
 template <typename T, typename U>
 struct hash<std::pair<T, U>> {
diff --git a/tensorflow/core/lib/hash/hash_test.cc b/tensorflow/core/lib/hash/hash_test.cc
index 0e5f6c6803..7d58313132 100644
--- a/tensorflow/core/lib/hash/hash_test.cc
+++ b/tensorflow/core/lib/hash/hash_test.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <map>
+#include <unordered_map>
 #include <vector>
 
 #include "tensorflow/core/lib/hash/hash.h"
@@ -81,4 +83,75 @@ static void BM_Hash32(int iters, int len) {
 }
 BENCHMARK(BM_Hash32)->Range(1, 1024);
 
+TEST(StringPieceHasher, Equality) {
+  StringPieceHasher hasher;
+
+  StringPiece s1("foo");
+  StringPiece s2("bar");
+  StringPiece s3("baz");
+  StringPiece s4("zot");
+
+  EXPECT_TRUE(hasher(s1) != hasher(s2));
+  EXPECT_TRUE(hasher(s1) != hasher(s3));
+  EXPECT_TRUE(hasher(s1) != hasher(s4));
+  EXPECT_TRUE(hasher(s2) != hasher(s3));
+  EXPECT_TRUE(hasher(s2) != hasher(s4));
+  EXPECT_TRUE(hasher(s3) != hasher(s4));
+
+  EXPECT_TRUE(hasher(s1) == hasher(s1));
+  EXPECT_TRUE(hasher(s2) == hasher(s2));
+  EXPECT_TRUE(hasher(s3) == hasher(s3));
+  EXPECT_TRUE(hasher(s4) == hasher(s4));
+}
+
+TEST(StringPieceHasher, HashMap) {
+  string s1("foo");
+  string s2("bar");
+  string s3("baz");
+
+  StringPiece p1(s1);
+  StringPiece p2(s2);
+  StringPiece p3(s3);
+
+  std::unordered_map<StringPiece, int, StringPieceHasher> map;
+
+  map.insert(std::make_pair(p1, 0));
+  map.insert(std::make_pair(p2, 1));
+  map.insert(std::make_pair(p3, 2));
+  EXPECT_EQ(map.size(), 3);
+
+  bool found[3] = {false, false, false};
+  for (auto const& val : map) {
+    int x = val.second;
+    EXPECT_TRUE(x >= 0 && x < 3);
+    EXPECT_TRUE(!found[x]);
+    found[x] = true;
+  }
+  EXPECT_EQ(found[0], true);
+  EXPECT_EQ(found[1], true);
+  EXPECT_EQ(found[2], true);
+
+  auto new_iter = map.find("zot");
+  EXPECT_TRUE(new_iter == map.end());
+
+  new_iter = map.find("bar");
+  EXPECT_TRUE(new_iter != map.end());
+
+  map.erase(new_iter);
+  EXPECT_EQ(map.size(), 2);
+
+  found[0] = false;
+  found[1] = false;
+  found[2] = false;
+  for (const auto& iter : map) {
+    int x = iter.second;
+    EXPECT_TRUE(x >= 0 && x < 3);
+    EXPECT_TRUE(!found[x]);
+    found[x] = true;
+  }
+  EXPECT_EQ(found[0], true);
+  EXPECT_EQ(found[1], false);
+  EXPECT_EQ(found[2], true);
+}
+
 }  // namespace tensorflow
-- 
GitLab


From 5d624aa437d3541d35e42482ee649c005f7fd5b5 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Thu, 15 Mar 2018 12:15:07 -0700
Subject: [PATCH 067/960] Clarify that in_nodes and in_edges includes control
 edges.

PiperOrigin-RevId: 189225717
---
 tensorflow/core/graph/graph.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/graph/graph.h b/tensorflow/core/graph/graph.h
index cbd58b051a..f7ca7d0620 100644
--- a/tensorflow/core/graph/graph.h
+++ b/tensorflow/core/graph/graph.h
@@ -124,7 +124,8 @@ class Node {
   // Inputs requested by the NodeDef.  For the actual inputs, use in_edges.
   const protobuf::RepeatedPtrField<string>& requested_inputs() const;
 
-  // Get the neighboring nodes via edges either in or out of this node.
+  // Get the neighboring nodes via edges either in or out of this node.  This
+  // includes control edges.
   gtl::iterator_range<NeighborIter> in_nodes() const;
   gtl::iterator_range<NeighborIter> out_nodes() const;
   const EdgeSet& in_edges() const { return in_edges_; }
-- 
GitLab


From 0f508d4de379e800ad7f990de08959bbd6fcabb5 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 15 Mar 2018 12:33:10 -0700
Subject: [PATCH 068/960] Warn when creating a `tf.InteractiveSession` if
 another is active.

Fixes #13202 (as far as possible without breaking backwards compatibility).

PiperOrigin-RevId: 189228094
---
 tensorflow/python/client/session.py      | 19 +++++++++++++++
 tensorflow/python/client/session_test.py | 31 ++++++++++++++++++++++++
 2 files changed, 50 insertions(+)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 924d62992a..29f06c8f22 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import functools
 import re
 import threading
+import warnings
 
 import numpy as np
 
@@ -1624,6 +1625,9 @@ class InteractiveSession(BaseSession):
   ```
   """
 
+  _count_lock = threading.Lock()
+  _active_session_count = 0  # GUARDED_BY(_count_lock)
+
   def __init__(self, target='', graph=None, config=None):
     """Creates a new interactive TensorFlow session.
 
@@ -1652,6 +1656,15 @@ class InteractiveSession(BaseSession):
     config.graph_options.place_pruned_graph = True
 
     super(InteractiveSession, self).__init__(target, graph, config)
+    with InteractiveSession._count_lock:
+      if InteractiveSession._active_session_count > 0:
+        warnings.warn('An interactive session is already active. This can '
+                      'cause out-of-memory errors in some cases. You must '
+                      'explicitly call `InteractiveSession.close()` to release '
+                      'resources held by the other session(s).')
+      InteractiveSession._active_session_count += 1
+    self._closed = False
+
     self._default_session = self.as_default()
     self._default_session.enforce_nesting = False
     self._default_session.__enter__()
@@ -1664,6 +1677,12 @@ class InteractiveSession(BaseSession):
   def close(self):
     """Closes an `InteractiveSession`."""
     super(InteractiveSession, self).close()
+    with InteractiveSession._count_lock:
+      if not self._closed:
+        InteractiveSession._active_session_count -= 1
+        self._closed = True
+      else:
+        return
     if self._explicit_graph is not None:
       self._default_graph.__exit__(None, None, None)
     self._default_session.__exit__(None, None, None)
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 781725d63b..6c7339f3d8 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -22,6 +22,7 @@ import os
 import sys
 import threading
 import time
+import warnings
 
 import numpy as np
 import six
@@ -66,6 +67,10 @@ ops.RegisterShape('ConstructionFails')(common_shapes.unknown_shape)
 # @test_util.with_c_api
 class SessionTest(test_util.TensorFlowTestCase):
 
+  def setUp(self):
+    super(SessionTest, self).setUp()
+    warnings.simplefilter('always')
+
   def testUseExistingGraph(self):
     with ops.Graph().as_default() as g, ops.device('/cpu:0'):
       a = constant_op.constant(6.0, shape=[1, 1])
@@ -1191,6 +1196,32 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([[24.0]], e.eval())
       sess.close()
 
+  def testMultipleInteractiveSessionsWarning(self):
+    # Reinitialize the global state to ensure that the expected warnings will
+    # be emitted.
+    session.InteractiveSession._active_session_count = 0  # pylint: disable=protected-access
+
+    sess = session.InteractiveSession()
+    sess.close()
+    # Opening and closing interactive sessions serially should not warn.
+    with warnings.catch_warnings(record=True) as w:
+      sess = session.InteractiveSession()
+      sess.close()
+    self.assertEqual(0, len(w))
+
+    with warnings.catch_warnings(record=True) as w:
+      sess = session.InteractiveSession()
+    self.assertEqual(0, len(w))
+    with warnings.catch_warnings(record=True) as w:
+      sess2 = session.InteractiveSession()
+    self.assertEqual(1, len(w))
+    self.assertTrue('An interactive session is already active. This can cause '
+                    'out-of-memory errors in some cases. You must explicitly '
+                    'call `InteractiveSession.close()` to release resources '
+                    'held by the other session(s).' in str(w[0].message))
+    sess2.close()
+    sess.close()
+
   def testInteractivePlacePrunedGraph(self):
     sess = session.InteractiveSession()
 
-- 
GitLab


From d696214ab7d1565850702959983fa189d0bfe50c Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 15 Mar 2018 12:43:01 -0700
Subject: [PATCH 069/960] Remove underscore prefix from broadcast_gradient_args
 op.

PiperOrigin-RevId: 189229472
---
 .../contrib/nn/python/ops/scaled_softplus.py  |  4 +-
 tensorflow/python/framework/python_op_gen.cc  |  4 +-
 .../python/kernel_tests/basic_gpu_test.py     |  4 +-
 .../python/kernel_tests/bcast_ops_test.py     |  4 +-
 tensorflow/python/ops/math_grad.py            | 50 ++++++-------------
 5 files changed, 20 insertions(+), 46 deletions(-)

diff --git a/tensorflow/contrib/nn/python/ops/scaled_softplus.py b/tensorflow/contrib/nn/python/ops/scaled_softplus.py
index fcbfbc239c..7184ef2b66 100644
--- a/tensorflow/contrib/nn/python/ops/scaled_softplus.py
+++ b/tensorflow/contrib/nn/python/ops/scaled_softplus.py
@@ -30,9 +30,7 @@ def _reduce_and_reshape_grad(g, t):
   """Returns the gradient, sum-reduced and reshaped to `t`'s shape."""
   shape = array_ops.shape(t)
   g_shape = array_ops.shape(g)
-  # pylint: disable=protected-access
-  bcast_dims, _ = gen_array_ops._broadcast_gradient_args(shape, g_shape)
-  # pylint: enable=protected-access
+  bcast_dims, _ = gen_array_ops.broadcast_gradient_args(shape, g_shape)
   return array_ops.reshape(math_ops.reduce_sum(g, bcast_dims), shape)
 
 
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 846e0c356c..03721c9a68 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -98,9 +98,7 @@ bool IsOpWithUnderscorePrefix(const string& s) {
        // TODO(annarev): reduce usage of '*' imports and remove these from the
        // list.
        "fused_batch_norm", "histogram_fixed_width", "stack",
-       "batch_norm_with_global_normalization",
-       // TODO(annarev): replace these ops in the next change.
-       "broadcast_gradient_args"});
+       "batch_norm_with_global_normalization"});
   return kUnderscoreOps->count(s) > 0;
 }
 
diff --git a/tensorflow/python/kernel_tests/basic_gpu_test.py b/tensorflow/python/kernel_tests/basic_gpu_test.py
index 405651e8ae..987a6ffcd4 100644
--- a/tensorflow/python/kernel_tests/basic_gpu_test.py
+++ b/tensorflow/python/kernel_tests/basic_gpu_test.py
@@ -33,7 +33,7 @@ from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import variables
-from tensorflow.python.ops.gen_array_ops import _broadcast_gradient_args
+from tensorflow.python.ops.gen_array_ops import broadcast_gradient_args
 from tensorflow.python.platform import test
 
 
@@ -157,7 +157,7 @@ class BroadcastSimpleTest(test.TestCase):
 
   def _GetGradientArgs(self, xs, ys):
     with self.test_session(use_gpu=True) as sess:
-      return sess.run(_broadcast_gradient_args(xs, ys))
+      return sess.run(broadcast_gradient_args(xs, ys))
 
   def testBroadcast(self):
     r0, r1 = self._GetGradientArgs([2, 3, 5], [1])
diff --git a/tensorflow/python/kernel_tests/bcast_ops_test.py b/tensorflow/python/kernel_tests/bcast_ops_test.py
index cb46fcb007..3305e55c05 100644
--- a/tensorflow/python/kernel_tests/bcast_ops_test.py
+++ b/tensorflow/python/kernel_tests/bcast_ops_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.ops.gen_array_ops import _broadcast_gradient_args
 from tensorflow.python.ops.gen_array_ops import broadcast_args
+from tensorflow.python.ops.gen_array_ops import broadcast_gradient_args
 from tensorflow.python.platform import test
 
 
@@ -33,7 +33,7 @@ class BcastOpsTest(test.TestCase):
 
   def _GetGradientArgs(self, xs, ys):
     with self.test_session() as sess:
-      return sess.run(_broadcast_gradient_args(xs, ys))
+      return sess.run(broadcast_gradient_args(xs, ys))
 
   def testBasic(self):
     r = self._GetBroadcastShape([2, 3, 5], [1])
diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index d220fe3cce..eb33687cb5 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -620,9 +620,7 @@ def _IgammaGrad(op, grad):
   x = op.inputs[1]
   sa = array_ops.shape(a)
   sx = array_ops.shape(x)
-  # pylint: disable=protected-access
-  unused_ra, rx = gen_array_ops._broadcast_gradient_args(sa, sx)
-  # pylint: enable=protected-access
+  unused_ra, rx = gen_array_ops.broadcast_gradient_args(sa, sx)
 
   # Perform operations in log space before summing, because Gamma(a)
   # and Gamma'(a) can grow large.
@@ -649,9 +647,7 @@ def _BetaincGrad(op, grad):
   # versa; so its sufficient to check against shape(a).
   sa = array_ops.shape(a)
   sx = array_ops.shape(x)
-  # pylint: disable=protected-access
-  _, rx = gen_array_ops._broadcast_gradient_args(sa, sx)
-  # pylint: enable=protected-access
+  _, rx = gen_array_ops.broadcast_gradient_args(sa, sx)
 
   # Perform operations in log space before summing, because terms
   # can grow large.
@@ -677,9 +673,7 @@ def _ZetaGrad(op, grad):
   # Broadcast gradients
   sx = array_ops.shape(x)
   sq = array_ops.shape(q)
-  # pylint: disable=protected-access
-  unused_rx, rq = gen_array_ops._broadcast_gradient_args(sx, sq)
-  # pylint: enable=protected-access
+  unused_rx, rq = gen_array_ops.broadcast_gradient_args(sx, sq)
   # Evaluate gradient
   with ops.control_dependencies([grad]):
     x = math_ops.conj(x)
@@ -699,9 +693,7 @@ def _PolygammaGrad(op, grad):
   # Broadcast gradients
   sn = array_ops.shape(n)
   sx = array_ops.shape(x)
-  # pylint: disable=protected-access
-  unused_rn, rx = gen_array_ops._broadcast_gradient_args(sn, sx)
-  # pylint: enable=protected-access
+  unused_rn, rx = gen_array_ops.broadcast_gradient_args(sn, sx)
   # Evaluate gradient
   with ops.control_dependencies([grad]):
     n = math_ops.conj(n)
@@ -841,9 +833,7 @@ def _AddGrad(op, grad):
     return grad, grad
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   return (array_ops.reshape(math_ops.reduce_sum(grad, rx), sx),
           array_ops.reshape(math_ops.reduce_sum(grad, ry), sy))
 
@@ -858,9 +848,7 @@ def _SubGrad(op, grad):
     return grad, -grad
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   return (array_ops.reshape(math_ops.reduce_sum(grad, rx), sx),
           array_ops.reshape(-math_ops.reduce_sum(grad, ry), sy))
 
@@ -870,7 +858,6 @@ def _MulGrad(op, grad):
   """The gradient of scalar multiplication."""
   x = op.inputs[0]
   y = op.inputs[1]
-  # pylint: disable=protected-access
   if (isinstance(grad, ops.Tensor) and
       _ShapesFullySpecifiedAndEqual(x, y, grad) and
       grad.dtype in (dtypes.int32, dtypes.float32)):
@@ -878,14 +865,13 @@ def _MulGrad(op, grad):
   assert x.dtype.base_dtype == y.dtype.base_dtype, (x.dtype, " vs. ", y.dtype)
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   x = math_ops.conj(x)
   y = math_ops.conj(y)
   return (array_ops.reshape(
       math_ops.reduce_sum(gen_math_ops.mul(grad, y), rx), sx),
           array_ops.reshape(
               math_ops.reduce_sum(gen_math_ops.mul(x, grad), ry), sy))
-  # pylint: enable=protected-access
 
 
 @ops.RegisterGradient("Div")
@@ -895,9 +881,7 @@ def _DivGrad(op, grad):
   y = op.inputs[1]
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   x = math_ops.conj(x)
   y = math_ops.conj(y)
   return (array_ops.reshape(math_ops.reduce_sum(math_ops.div(grad, y), rx), sx),
@@ -920,9 +904,7 @@ def _FloorModGrad(op, grad):
 
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   floor_xy = math_ops.floor_div(x, y)
   gx = array_ops.reshape(math_ops.reduce_sum(grad, rx), sx)
   gy = array_ops.reshape(
@@ -942,9 +924,7 @@ def _RealDivGrad(op, grad):
   y = op.inputs[1]
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   x = math_ops.conj(x)
   y = math_ops.conj(y)
   return (array_ops.reshape(
@@ -962,7 +942,7 @@ def _PowGrad(op, grad):
   z = op.outputs[0]
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   x = math_ops.conj(x)
   y = math_ops.conj(y)
   z = math_ops.conj(z)
@@ -990,7 +970,7 @@ def _MaximumMinimumGrad(op, grad, selector_op):
   gradshape = array_ops.shape(grad)
   zeros = array_ops.zeros(gradshape, gdtype)
   xmask = selector_op(x, y)
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   xgrad = array_ops.where(xmask, grad, zeros)
   ygrad = array_ops.where(xmask, zeros, grad)
   gx = array_ops.reshape(math_ops.reduce_sum(xgrad, rx), sx)
@@ -1017,9 +997,7 @@ def _SquaredDifferenceGrad(op, grad):
   y = op.inputs[1]
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  # pylint: disable=protected-access
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
-  # pylint: enable=protected-access
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   with ops.control_dependencies([grad]):
     # The parens ensure that if grad is IndexedSlices, it'll get multiplied by
     # Tensor (not a number like 2.0) which causes it to convert to Tensor.
@@ -1183,7 +1161,7 @@ def _ComplexGrad(op, grad):
   y = op.inputs[1]
   sx = array_ops.shape(x)
   sy = array_ops.shape(y)
-  rx, ry = gen_array_ops._broadcast_gradient_args(sx, sy)
+  rx, ry = gen_array_ops.broadcast_gradient_args(sx, sy)
   return (array_ops.reshape(math_ops.reduce_sum(math_ops.real(grad), rx), sx),
           array_ops.reshape(math_ops.reduce_sum(math_ops.imag(grad), ry), sy))
 
-- 
GitLab


From 61032e9ca7bf9849cb65db9b646381d124080856 Mon Sep 17 00:00:00 2001
From: Mustafa Ispir <ispir@google.com>
Date: Thu, 15 Mar 2018 12:54:42 -0700
Subject: [PATCH 070/960] Small code readability improvement.

PiperOrigin-RevId: 189231130
---
 tensorflow/python/feature_column/feature_column.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 381153c66a..7d99fcb3e7 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -1889,12 +1889,12 @@ class _LazyBuilder(object):
       self._feature_tensors[key] = feature_tensor
       return feature_tensor
 
-    if not isinstance(key, (str, _FeatureColumn)):
-      raise TypeError('"key" must be either a "str" or "_FeatureColumn". '
-                      'Provided: {}'.format(key))
+    if isinstance(key, str):
+      raise ValueError('Feature {} is not in features dictionary.'.format(key))
 
     if not isinstance(key, _FeatureColumn):
-      raise ValueError('Feature {} is not in features dictionary.'.format(key))
+      raise TypeError('"key" must be either a "str" or "_FeatureColumn". '
+                      'Provided: {}'.format(key))
 
     column = key
     logging.debug('Transforming feature_column %s.', column)
-- 
GitLab


From ccd8079e579604547f4b4d8a6b061cfdc6ce49bf Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 15 Mar 2018 12:58:08 -0700
Subject: [PATCH 071/960] Merge changes from github.

PiperOrigin-RevId: 189231636
---
 README.md                                     |    4 +
 SECURITY.md                                   |   14 +-
 configure.py                                  |    7 +-
 .../xla/service/generic_transfer_manager.cc   |    9 +-
 .../compiler/xla/tests/convolution_test.cc    |    2 +-
 tensorflow/contrib/BUILD                      |   10 +-
 tensorflow/contrib/__init__.py                |    6 +-
 .../boosted_trees/lib/utils/batch_features.h  |    6 +-
 tensorflow/contrib/cmake/README.md            |   12 +-
 tensorflow/contrib/cmake/external/grpc.cmake  |    1 +
 .../contrib/cmake/external/protobuf.cmake     |    2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |    4 +
 tensorflow/contrib/data/__init__.py           |    4 +
 .../contrib/data/python/kernel_tests/BUILD    |   17 +
 .../kernel_tests/slide_dataset_op_test.py     |  242 +++
 tensorflow/contrib/data/python/ops/BUILD      |    1 +
 tensorflow/contrib/data/python/ops/sliding.py |  102 ++
 tensorflow/contrib/distributions/BUILD        |    2 +
 tensorflow/contrib/eager/python/BUILD         |    5 +-
 .../python/examples/linear_regression/BUILD   |    1 +
 tensorflow/contrib/factorization/BUILD        |    5 +-
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      |    9 +-
 tensorflow/contrib/gan/BUILD                  |    1 +
 tensorflow/contrib/kafka/BUILD                |  108 +-
 .../kafka/kernels/kafka_dataset_ops.cc        |    4 +-
 tensorflow/contrib/kafka/ops/dataset_ops.cc   |   44 +
 .../kafka/python/ops/kafka_dataset_ops.py     |    9 +-
 .../kafka/python/ops/kafka_op_loader.py       |   24 +
 .../contrib/kfac/python/kernel_tests/BUILD    |    1 +
 tensorflow/contrib/labeled_tensor/BUILD       |    1 +
 tensorflow/contrib/layers/BUILD               |    2 +
 .../layers/python/layers/embedding_ops.py     |    2 +-
 tensorflow/contrib/learn/BUILD                |   12 +-
 .../learn/python/learn/ops/embeddings_ops.py  |    2 +-
 tensorflow/contrib/lite/Makefile              |    9 +-
 tensorflow/contrib/lite/arena_planner.h       |    2 +-
 tensorflow/contrib/lite/build_rpi_lib.sh      |   22 +
 tensorflow/contrib/lite/builtin_ops.h         |    2 +-
 tensorflow/contrib/lite/error_reporter.h      |    2 +-
 tensorflow/contrib/lite/g3doc/rpi.md          |   50 +
 tensorflow/contrib/lite/interpreter.h         |    2 +-
 tensorflow/contrib/lite/interpreter_test.cc   |    2 +-
 tensorflow/contrib/lite/kernels/conv.cc       |    2 +-
 .../contrib/lite/kernels/depthwise_conv.cc    |    2 +-
 .../contrib/lite/kernels/fully_connected.cc   |    2 +-
 tensorflow/contrib/lite/kernels/kernel_util.h |    2 +-
 .../contrib/lite/kernels/lsh_projection.cc    |    2 +-
 tensorflow/contrib/lite/kernels/lstm.cc       |    6 +-
 tensorflow/contrib/lite/kernels/reshape.cc    |   12 +-
 .../contrib/lite/kernels/reshape_test.cc      |    2 +-
 tensorflow/contrib/lite/kernels/test_util.cc  |    4 +-
 .../kernels/unidirectional_sequence_lstm.cc   |    2 +-
 tensorflow/contrib/lite/memory_planner.h      |    4 +-
 tensorflow/contrib/lite/model.h               |    2 +-
 .../contrib/lite/nnapi/NeuralNetworksShim.h   |    2 +-
 tensorflow/contrib/lite/rpi_makefile.inc      |   33 +
 .../schema/builtin_ops_header/generator.cc    |    2 +-
 .../contrib/lite/simple_memory_arena.cc       |    6 +-
 tensorflow/contrib/lite/simple_memory_arena.h |    6 +-
 tensorflow/contrib/lookup/BUILD               |    1 +
 tensorflow/contrib/makefile/README.md         |    2 +
 tensorflow/contrib/makefile/build_all_ios.sh  |    3 +-
 tensorflow/contrib/mpi/mpi_utils.h            |    2 +
 .../contrib/predictor/predictor_factories.py  |    4 +-
 tensorflow/contrib/py2tf/converters/BUILD     |    2 +
 .../contrib/py2tf/converters/single_return.py |    2 +-
 tensorflow/contrib/py2tf/utils/BUILD          |    1 +
 .../quantize/python/fold_batch_norms.py       |    4 +-
 .../contrib/quantize/python/quant_ops.py      |    4 +-
 .../contrib/quantize/python/quantize.py       |    2 +-
 .../contrib/quantize/python/quantize_graph.py |    2 +-
 .../python/quantize_parameterized_test.py     |    8 +-
 .../contrib/quantize/python/quantize_test.py  |    2 +-
 .../contrib/remote_fused_graph/pylib/BUILD    |    1 -
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |    6 +-
 tensorflow/contrib/saved_model/BUILD          |    1 +
 .../seq2seq/python/ops/beam_search_decoder.py |    7 +-
 tensorflow/contrib/session_bundle/BUILD       |    1 +
 .../contrib/slim/python/slim/data/BUILD       |    1 +
 tensorflow/contrib/tensor_forest/BUILD        |    1 -
 tensorflow/contrib/tensorboard/BUILD          |    1 +
 tensorflow/contrib/tensorrt/BUILD             |    2 +
 tensorflow/contrib/tensorrt/README.md         |   23 +-
 tensorflow/contrib/tensorrt/__init__.py       |   18 +-
 .../contrib/tensorrt/convert/convert_graph.cc |  256 ++-
 .../contrib/tensorrt/convert/convert_graph.h  |    8 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 1481 ++++++++++++++---
 .../contrib/tensorrt/convert/convert_nodes.h  |   53 +-
 .../contrib/tensorrt/kernels/trt_calib_op.cc  |   11 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |   39 +-
 tensorflow/contrib/tensorrt/log/trt_logger.cc |    8 +-
 tensorflow/contrib/tensorrt/log/trt_logger.h  |    4 +-
 .../contrib/tensorrt/python/__init__.py       |    1 +
 .../contrib/tensorrt/python/trt_convert.py    |   68 +-
 .../tensorrt/resources/trt_int8_calibrator.cc |   56 +-
 .../tensorrt/resources/trt_int8_calibrator.h  |   15 +-
 .../contrib/tensorrt/test/test_tftrt.py       |   57 +-
 tensorflow/contrib/tensorrt/trt_conversion.i  |   63 +-
 tensorflow/contrib/timeseries/examples/BUILD  |    5 +-
 .../timeseries/python/timeseries/BUILD        |    5 +-
 .../timeseries/state_space_models/BUILD       |    1 +
 tensorflow/contrib/tpu/BUILD                  |    1 +
 tensorflow/contrib/util/loader.py             |    7 +-
 tensorflow/core/BUILD                         |    4 +
 .../base_api/api_def_SlideDataset.pbtxt       |   18 +
 .../core/distributed_runtime/tensor_coding.cc |    4 +-
 tensorflow/core/framework/dataset.h           |    4 +-
 tensorflow/core/graph/mkl_layout_pass.cc      |   31 +-
 .../grappler/optimizers/loop_optimizer.cc     |    8 +-
 tensorflow/core/kernels/BUILD                 |   12 +-
 tensorflow/core/kernels/data/BUILD            |   14 +
 .../core/kernels/data/slide_dataset_op.cc     |  252 +++
 tensorflow/core/kernels/depthtospace_op.cc    |    3 +
 .../core/kernels/depthtospace_op_gpu.cu.cc    |    6 +
 tensorflow/core/kernels/hexagon/BUILD         |    1 +
 .../core/kernels/mkl_conv_grad_filter_ops.cc  |   81 +-
 .../core/kernels/mkl_conv_grad_input_ops.cc   |   18 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  146 +-
 tensorflow/core/kernels/mkl_conv_ops.h        |  117 +-
 .../core/kernels/mkl_input_conversion_op.cc   |    7 +-
 tensorflow/core/kernels/mkl_relu_op.cc        |   23 +-
 .../core/kernels/segment_reduction_ops.h      |   14 +-
 tensorflow/core/kernels/spacetodepth_op.cc    |    3 +
 .../core/kernels/spacetodepth_op_gpu.cu.cc    |    6 +
 tensorflow/core/lib/core/stringpiece.cc       |    2 -
 tensorflow/core/lib/core/stringpiece.h        |    2 +-
 tensorflow/core/lib/io/record_reader.cc       |    2 +
 tensorflow/core/lib/io/record_reader.h        |    4 +-
 tensorflow/core/ops/dataset_ops.cc            |   12 +-
 tensorflow/core/ops/nn_ops.cc                 |    8 +
 tensorflow/core/platform/tracing.h            |    2 +-
 .../platform/windows/windows_file_system.cc   |    3 +-
 tensorflow/core/public/version.h              |    2 +-
 tensorflow/docs_src/community/welcome.md      |    4 +-
 tensorflow/docs_src/install/install_c.md      |    2 +-
 tensorflow/docs_src/install/install_go.md     |    2 +-
 tensorflow/docs_src/install/install_java.md   |   22 +-
 tensorflow/docs_src/install/install_linux.md  |   56 +-
 tensorflow/docs_src/install/install_mac.md    |   23 +-
 .../docs_src/install/install_sources.md       |    4 +-
 .../docs_src/install/install_windows.md       |    5 +-
 tensorflow/docs_src/performance/xla/jit.md    |    2 +-
 .../docs_src/programmers_guide/debugger.md    |    3 +-
 tensorflow/docs_src/programmers_guide/faq.md  |    4 +-
 .../summaries_and_tensorboard.md              |    2 +-
 .../docs_src/programmers_guide/using_tpu.md   |    7 +-
 tensorflow/docs_src/tutorials/layers.md       |   12 +-
 .../docs_src/tutorials/recurrent_quickdraw.md |    3 +-
 tensorflow/docs_src/tutorials/wide.md         |   16 +-
 .../examples/android/AndroidManifest.xml      |    4 +
 .../org/tensorflow/demo/CameraActivity.java   |    7 +-
 .../org/tensorflow/demo/StylizeActivity.java  |   60 +
 tensorflow/python/BUILD                       |   84 +-
 tensorflow/python/debug/BUILD                 |    1 +
 tensorflow/python/estimator/estimator.py      |   32 +-
 tensorflow/python/estimator/training.py       |   26 +-
 tensorflow/python/keras/BUILD                 |    5 +-
 tensorflow/python/kernel_tests/BUILD          |    4 -
 .../python/kernel_tests/conv_ops_test.py      |   20 +-
 .../kernel_tests/depthtospace_op_test.py      |   10 +-
 .../kernel_tests/spacetodepth_op_test.py      |   10 +-
 tensorflow/python/lib/io/file_io_test.py      |    5 +
 tensorflow/python/ops/nn_ops.py               |    2 +-
 tensorflow/python/ops/rnn.py                  |   17 +-
 tensorflow/python/ops/special_math_ops.py     |    4 +-
 .../python/ops/special_math_ops_test.py       |    5 +
 tensorflow/python/tools/freeze_graph.py       |   36 +-
 tensorflow/python/tools/saved_model_cli.py    |   60 +
 .../python/tools/saved_model_cli_test.py      |   22 +
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |   33 +-
 tensorflow/tensorflow.bzl                     |   18 +-
 .../tools/api/tests/api_compatibility_test.py |    2 +
 tensorflow/tools/ci_build/Dockerfile.cmake    |    5 +-
 .../windows/cpu/pip/build_tf_windows.sh       |    3 +-
 tensorflow/tools/def_file_filter/BUILD        |    0
 tensorflow/tools/def_file_filter/BUILD.tpl    |   15 +
 .../def_file_filter/def_file_filter.py.tpl    |  168 ++
 .../def_file_filter_configure.bzl             |   56 +
 tensorflow/tools/dist_test/README.md          |    8 +
 tensorflow/tools/dist_test/local_test.sh      |   22 +-
 tensorflow/tools/git/gen_git_source.py        |    7 +
 tensorflow/tools/graph_transforms/BUILD       |    1 +
 .../graph_transforms/fold_old_batch_norms.cc  |   67 +
 .../fold_old_batch_norms_test.cc              |   95 ++
 tensorflow/tools/pip_package/BUILD            |  129 +-
 tensorflow/tools/pip_package/setup.py         |    4 +-
 .../tools/test/upload_test_benchmarks.py      |    9 +-
 tensorflow/workspace.bzl                      |    8 +-
 third_party/jpeg/jpeg.BUILD                   |    4 +-
 third_party/kafka/BUILD                       |   13 +-
 third_party/py/BUILD.tpl                      |   22 +-
 191 files changed, 4250 insertions(+), 926 deletions(-)
 create mode 100644 tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
 create mode 100644 tensorflow/contrib/data/python/ops/sliding.py
 create mode 100644 tensorflow/contrib/kafka/ops/dataset_ops.cc
 create mode 100644 tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
 create mode 100755 tensorflow/contrib/lite/build_rpi_lib.sh
 create mode 100644 tensorflow/contrib/lite/g3doc/rpi.md
 create mode 100644 tensorflow/contrib/lite/rpi_makefile.inc
 create mode 100644 tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
 create mode 100644 tensorflow/core/kernels/data/slide_dataset_op.cc
 create mode 100644 tensorflow/tools/def_file_filter/BUILD
 create mode 100644 tensorflow/tools/def_file_filter/BUILD.tpl
 create mode 100644 tensorflow/tools/def_file_filter/def_file_filter.py.tpl
 create mode 100644 tensorflow/tools/def_file_filter/def_file_filter_configure.bzl

diff --git a/README.md b/README.md
index ef5bdc66ef..3cdb6e478d 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,10 @@ organization for the purposes of conducting machine learning and deep neural
 networks research.  The system is general enough to be applicable in a wide
 variety of other domains, as well.
 
+Keep up to date with release announcements and security updates by
+subscribing to
+[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
+
 ## Installation
 *See [Installing TensorFlow](https://www.tensorflow.org/get_started/os_setup.html) for instructions on how to install our release binaries or how to build from source.*
 
diff --git a/SECURITY.md b/SECURITY.md
index fea24b2739..2aaa9202d5 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -6,7 +6,7 @@ report vulnerabilities in TensorFlow.
 
 ## TensorFlow models are programs
 
-TensorFlow's runtime system interprets and executes programs. What machine 
+TensorFlow's runtime system interprets and executes programs. What machine
 learning practitioners term
 [**models**](https://developers.google.com/machine-learning/glossary/#model) are
 expressed as programs that TensorFlow executes.  TensorFlow programs are encoded
@@ -28,12 +28,12 @@ data you supply to TensorFlow to train a model, or to use a model to run
 inference on the data.
 
 **TensorFlow models are programs, and need to be treated as such from a security
-perspective.** 
+perspective.**
 
 ## Running untrusted models
 
 As a general rule: **Always** execute untrusted models inside a sandbox (e.g.,
-[nsjail](https://github.com/google/nsjail)). 
+[nsjail](https://github.com/google/nsjail)).
 
 There are several ways in which a model could become untrusted. Obviously, if an
 untrusted party supplies TensorFlow kernels, arbitrary code may be executed.
@@ -109,11 +109,11 @@ graphs known to the `ModelServer`. This means that an attacker may run
 graphs using untrusted inputs as described above, but they would not be able to
 execute arbitrary graphs. It is possible to safely expose a `ModelServer`
 directly to an untrusted network, **but only if the graphs it is configured to
-use have been carefully audited to be safe**. 
+use have been carefully audited to be safe**.
 
 Similar to best practices for other servers, we recommend running any
 `ModelServer` with appropriate privileges (i.e., using a separate user with
-reduced permisisons). In the spirit of defense in depth, we recommend
+reduced permissions). In the spirit of defense in depth, we recommend
 authenticating requests to any TensorFlow server connected to an untrusted
 network, as well as sandboxing the server to minimize the adverse effects of
 any breach.
@@ -133,7 +133,7 @@ which exhibit unexpected or unwanted behaviors. The fact that TensorFlow models
 can perform arbitrary computations means that they may read and write files,
 communicate via the network, produce deadlocks and infinite loops, or run out
 of memory. It is only when these behaviors are outside the specifications of the
-operations involved that such behavior is a vulnerability. 
+operations involved that such behavior is a vulnerability.
 
 A `FileWriter` writing a file is not unexpected behavior and therefore is not a
 vulnerability in TensorFlow. A `MatMul` allowing arbitrary binary code execution
@@ -168,7 +168,7 @@ below).
 
 Please use a descriptive subject line for your report email. After the initial
 reply to your report, the security team will endeavor to keep you informed of
-the progress being made towards a fix and announcement. 
+the progress being made towards a fix and announcement.
 
 If you believe that an existing (public) issue is security-related, please send
 an email to `security@tensorflow.org`. The email should include the issue ID and
diff --git a/configure.py b/configure.py
index 97f46757ee..d14edef1be 100644
--- a/configure.py
+++ b/configure.py
@@ -1048,7 +1048,10 @@ def set_tf_tensorrt_install_path(environ_cp):
 
     for lib_file in possible_files:
       if is_compatible(lib_file, cuda_ver, cudnn_ver):
-        ver_str = nvinfer_pattern.search(lib_file).group(1)
+        matches = nvinfer_pattern.search(lib_file)
+        if len(matches.groups()) == 0:
+          continue
+        ver_str = matches.group(1)
         ver = convert_version_to_int(ver_str) if len(ver_str) else 0
         if ver > highest_ver[0]:
           highest_ver = [ver, ver_str, lib_file]
@@ -1377,7 +1380,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.5.4')
+  check_bazel_version('0.10.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.cc b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
index 78dc0ad4fc..a99e2b7794 100644
--- a/tensorflow/compiler/xla/service/generic_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
@@ -38,14 +38,7 @@ namespace xla {
 
 GenericTransferManager::GenericTransferManager(se::Platform::Id platform_id,
                                                size_t pointer_size)
-    : platform_id_(platform_id), pointer_size_(pointer_size) {
-  // We currently only support kHostPlatformId for CPU, kCudaPlatformId for
-  // GPU and kInterpreterPlatformId for Interpreter. Before supporting other
-  // platforms, we need to test this transfer manager on them.
-  CHECK(platform_id_ == se::host::kHostPlatformId ||
-        platform_id_ == se::interpreter::kInterpreterPlatformId ||
-        platform_id_ == se::cuda::kCudaPlatformId);
-}
+    : platform_id_(platform_id), pointer_size_(pointer_size) {}
 
 se::Platform::Id GenericTransferManager::PlatformId() const {
   return platform_id_;
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 99640f5bb5..72715398de 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -723,7 +723,7 @@ INSTANTIATE_TEST_CASE_P(
 );
 #endif
 
-TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
+XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
   ComputationBuilder builder(client_, TestName());
   Shape input_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
   Shape filter_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index bab37e8906..986b61b3ea 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -8,6 +8,7 @@ package(default_visibility = ["//tensorflow:__subpackages__"])
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
+load("//tensorflow:tensorflow.bzl", "if_not_windows")
 
 py_library(
     name = "contrib_py",
@@ -51,7 +52,6 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
-        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -63,7 +63,6 @@ py_library(
         "//tensorflow/contrib/linalg:linalg_py",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
-        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
@@ -110,6 +109,10 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
+    ]) + if_not_windows([
+        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",  # unix dependency, need to fix code
+        "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
+        "//tensorflow/contrib/kafka",  # has some linking issue on opensssl.
     ]),
 )
 
@@ -121,6 +124,7 @@ cc_library(
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
+        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -147,7 +151,7 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:kafka_ops_op_lib",
+        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f6f539027..669d611b01 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
@@ -83,7 +85,8 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-from tensorflow.contrib.lite.python import lite
+if os.name != 'nt':
+  from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
@@ -92,6 +95,7 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
+del os
 del LazyLoader
 
 del absolute_import
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
index da5e744851..7815fa049a 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
@@ -48,9 +48,9 @@ class BatchFeatures {
   Status GetFeatureColumnSizes(int64* const num_dense_float_features,
                                int64* const num_sparse_float_features,
                                int64* const num_sparse_int_features) const {
-    QCHECK_NE(num_dense_float_features, nullptr);
-    QCHECK_NE(num_sparse_float_features, nullptr);
-    QCHECK_NE(num_sparse_int_features, nullptr);
+    QCHECK_NE(num_dense_float_features, (int64*) nullptr);
+    QCHECK_NE(num_sparse_float_features, (int64*) nullptr);
+    QCHECK_NE(num_sparse_int_features, (int64*) nullptr);
     *num_dense_float_features = dense_float_feature_columns_.size();
     *num_sparse_float_features = sparse_float_feature_columns_.size();
     *num_sparse_int_features = sparse_int_feature_columns_.size();
diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index 8f85a75ee4..fe83bb3204 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -26,7 +26,7 @@ The CMake files in this directory can build the core TensorFlow runtime, an
 example C++ binary, and a PIP package containing the runtime and Python
 bindings.
 
-### Pre-requisites
+### Prerequisites
 
 * CMake version 3.5 or later.
 
@@ -34,14 +34,16 @@ bindings.
 
 * [SWIG](http://www.swig.org/download.html)
 
-* Additional pre-requisites for Microsoft Windows:
+* Additional prerequisites for Microsoft Windows:
   - Visual Studio 2015
   - Python 3.5
-  - NumPy 1.11.0 or later
 
-* Additional pre-requisites for Linux:
+* Additional prerequisites for Linux:
   - Python 2.7 or later
   - [Docker](https://www.docker.com/) (for automated testing)
+
+* Python dependencies:
+  - wheel
   - NumPy 1.11.0 or later
 
 ### Known-good configurations
@@ -102,7 +104,7 @@ ops or APIs.
 Step-by-step Windows build
 ==========================
 
-1. Install the pre-requisites detailed above, and set up your environment.
+1. Install the prerequisites detailed above, and set up your environment.
 
    * The following commands assume that you are using the Windows Command
      Prompt (`cmd.exe`). You will need to set up your environment to use the
diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index a9f43a3ecb..17f65999fa 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -35,6 +35,7 @@ else()
   set(grpc_STATIC_LIBRARIES
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
+      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
 endif()
 
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index aba8a5244e..ab464bc99a 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
 set(PROTOBUF_URL https://github.com/google/protobuf.git)
-set(PROTOBUF_TAG 396336eb961b75f03b25824fe86cf6490fb75e3a)
+set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
 
 if(WIN32)
   if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 9f96a4b797..b3e5b30826 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -476,6 +476,10 @@ if (tensorflow_BUILD_CC_TESTS)
     "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*_test.cc"
   )
 
+  list(REMOVE_ITEM tf_test_src_simple
+    ${tf_core_profiler_test_srcs}
+  )
+
   set(tf_test_lib tf_test_lib)
   add_library(${tf_test_lib} STATIC ${tf_src_testlib})
 
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index f09d156832..9212b69700 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -40,6 +40,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@rejection_resample
 @@scan
 @@shuffle_and_repeat
+@@sliding_window_batch
 @@sloppy_interleave
 @@unbatch
 
@@ -72,6 +73,9 @@ from tensorflow.contrib.data.python.ops.readers import SqlDataset
 from tensorflow.contrib.data.python.ops.resampling import rejection_resample
 from tensorflow.contrib.data.python.ops.scan_ops import scan
 from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat
+from tensorflow.contrib.data.python.ops.sliding import sliding_window_batch
+from tensorflow.python.data.ops.iterator_ops import Iterator
+from tensorflow.python.ops.parsing_ops import parse_single_example_v2 as parse_single_example
 # pylint: enable=unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 22418b38e3..2c4d4adfda 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -498,6 +498,23 @@ py_test(
     ],
 )
 
+tf_py_test(
+    name = "slide_dataset_op_test",
+    size = "small",
+    srcs = ["slide_dataset_op_test.py"],
+    additional_deps = [
+        "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:transformation_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//third_party/py/numpy",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
new file mode 100644
index 0000000000..33c48e20be
--- /dev/null
+++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
@@ -0,0 +1,242 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the experimental input pipeline ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.data.python.ops import sliding
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+class SlideDatasetTest(test.TestCase):
+
+  def testSlideDataset(self):
+    """Test an dataset that maps a TF function across its input elements."""
+    components = (np.arange(7),
+                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
+                  np.array(37.0) * np.arange(7))
+
+    count = array_ops.placeholder(dtypes.int64, shape=[])
+    window_size = array_ops.placeholder(dtypes.int64, shape=[])
+    stride = array_ops.placeholder(dtypes.int64, shape=[])
+
+    def _map_fn(x, y, z):
+      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
+
+    # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
+    # RepeatDataset(count) -> _SlideDataset(window_size, stride).
+    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
+                .map(_map_fn)
+                .repeat(count)
+                .apply(sliding.sliding_window_batch(window_size, stride))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
+                     [t.shape.as_list() for t in get_next])
+
+    with self.test_session() as sess:
+      # Slide over a finite input, where the window_size divides the
+      # total number of elements.
+      sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 7})
+      # Same formula with convolution layer.
+      num_batches = (20 * 7 - 14) // 7 + 1
+      for i in range(num_batches):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(14):
+            self.assertAllEqual(component[(i*7 + j) % 7]**2,
+                                result_component[j])
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over a finite input, where the window_size does not
+      # divide the total number of elements.
+      sess.run(init_op, feed_dict={count: 20, window_size: 17, stride: 9})
+
+      num_batches = (20 * 7 - 17) // 9 + 1
+      for i in range(num_batches):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(17):
+            self.assertAllEqual(component[(i*9 + j) % 7]**2,
+                                result_component[j])
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over a finite input, which is less than window_size,
+      # should fail straight away.
+      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over an empty input should fail straight away.
+      sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Empty window_size should be an initialization time error.
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 0, stride: 0})
+
+      # Invalid stride should be an initialization time error.
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 0})
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 3})
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 5})
+
+  def assertSparseValuesEqual(self, a, b):
+    self.assertAllEqual(a.indices, b.indices)
+    self.assertAllEqual(a.values, b.values)
+    self.assertAllEqual(a.dense_shape, b.dense_shape)
+
+  def testSlideSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
+        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      num_batches = (10 - 5) // 3 + 1
+      for i in range(num_batches):
+        actual = sess.run(get_next)
+        expected = sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
+            values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
+            dense_shape=[5, 1])
+        self.assertTrue(sparse_tensor.is_sparse(actual))
+        self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testSlideSparseWithDifferentDenseShapes(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=array_ops.expand_dims(
+              math_ops.range(i, dtype=dtypes.int64), 1),
+          values=array_ops.fill([math_ops.to_int32(i)], i),
+          dense_shape=[i])
+
+    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
+        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      num_batches = (10 - 5) // 3 + 1
+      for i in range(num_batches):
+        actual = sess.run(get_next)
+        expected_indices = []
+        expected_values = []
+        for j in range(5):
+          for k in range(i * 3 + j):
+            expected_indices.append([j, k])
+            expected_values.append(i * 3 + j)
+        expected = sparse_tensor.SparseTensorValue(
+            indices=expected_indices,
+            values=expected_values,
+            dense_shape=[5, i * 3 + 5 - 1])
+        self.assertTrue(sparse_tensor.is_sparse(actual))
+        self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testNestedSlideSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    iterator = (dataset_ops.Dataset.range(10)
+                .map(_sparse)
+                .apply(sliding.sliding_window_batch(4, 2))
+                .apply(sliding.sliding_window_batch(3, 1))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      # Slide: 1st batch.
+      actual = sess.run(get_next)
+      expected = sparse_tensor.SparseTensorValue(
+          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
+                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
+                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
+          values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7],
+          dense_shape=[3, 4, 1])
+      self.assertTrue(sparse_tensor.is_sparse(actual))
+      self.assertSparseValuesEqual(actual, expected)
+      # Slide: 2nd batch.
+      actual = sess.run(get_next)
+      expected = sparse_tensor.SparseTensorValue(
+          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
+                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
+                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
+          values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9],
+          dense_shape=[3, 4, 1])
+      self.assertTrue(sparse_tensor.is_sparse(actual))
+      self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testSlideShapeError(self):
+
+    def generator():
+      yield [1.0, 2.0, 3.0]
+      yield [4.0, 5.0, 6.0]
+      yield [7.0, 8.0, 9.0, 10.0]
+
+    iterator = (dataset_ops.Dataset.from_generator(generator, dtypes.float32,
+                                                   output_shapes=[None])
+                .apply(sliding.sliding_window_batch(3, 1))
+                .make_initializable_iterator())
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(iterator.initializer)
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"Cannot batch tensors with different shapes in component 0. "
+          r"First element had shape \[3\] and element 2 had shape \[4\]."):
+        sess.run(next_element)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index f03430c5c5..c3331e9636 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -106,6 +106,7 @@ py_library(
         "interleave_ops.py",
         "resampling.py",
         "scan_ops.py",
+        "sliding.py",
         "stats_ops.py",
         "threadpool.py",
         "unique.py",
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
new file mode 100644
index 0000000000..19cc3cb89f
--- /dev/null
+++ b/tensorflow/contrib/data/python/ops/sliding.py
@@ -0,0 +1,102 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Sliding dataset transformations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import sparse
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_dataset_ops
+
+
+class _SlideDataset(dataset_ops.Dataset):
+  """A `Dataset` that passes a sliding window over its input."""
+
+  def __init__(self, input_dataset, window_size, stride=1):
+    """See `sliding_window_batch` for details."""
+    super(_SlideDataset, self).__init__()
+    self._input_dataset = input_dataset
+    self._window_size = ops.convert_to_tensor(
+        window_size, dtype=dtypes.int64, name="window_size")
+    self._stride = ops.convert_to_tensor(
+        stride, dtype=dtypes.int64, name="stride")
+
+  def _as_variant_tensor(self):
+    return gen_dataset_ops.slide_dataset(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        window_size=self._window_size,
+        stride=self._stride,
+        output_shapes=nest.flatten(
+            sparse.as_dense_shapes(self.output_shapes, self.output_classes)),
+        output_types=nest.flatten(
+            sparse.as_dense_types(self.output_types, self.output_classes)))
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    input_shapes = self._input_dataset.output_shapes
+    return nest.pack_sequence_as(input_shapes, [
+        tensor_shape.vector(None).concatenate(s)
+        for s in nest.flatten(self._input_dataset.output_shapes)
+    ])
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+
+
+def sliding_window_batch(window_size, stride=1):
+  """A sliding window with size of `window_size` and step of `stride`.
+
+  This transformation passes a sliding window over this dataset. The
+  window size is `window_size` and step size is `stride`. If the left
+  elements cannot fill up the sliding window, this transformation will
+  drop the final smaller element. For example:
+
+  ```python
+  # NOTE: The following examples use `{ ... }` to represent the
+  # contents of a dataset.
+  a = { [1], [2], [3], [4], [5], [6] }
+
+  a.apply(tf.contrib.data.sliding_window_batch(window_size=3, stride=2)) ==
+  {
+      [[1], [2], [3]],
+      [[3], [4], [5]],
+  }
+  ```
+
+  Args:
+    window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
+      elements in the sliding window.
+    stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the
+      steps moving the sliding window forward for one iteration. The default
+      is `1`. It must be in `[1, window_size)`.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return _SlideDataset(dataset, window_size, stride)
+
+  return _apply_fn
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 6bd3f5f09b..1bd73ee704 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -454,6 +454,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
@@ -1143,6 +1144,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 2f9bc68aaa..384ef7f963 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -266,7 +266,10 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = ["notsan"],
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
+    ],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index f86331af6f..2f6cfdf31e 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -22,6 +22,7 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index 90f10f1fa8..ad8568ad44 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -224,7 +224,10 @@ py_test(
     srcs = ["python/ops/kmeans_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67512932
+    tags = [
+        "nomac",  # b/73741358
+        "notsan",  # b/67512932
+    ],
     deps = [
         ":factorization_py",
         ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO",
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index e61221a6b0..35341406a0 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -256,6 +256,9 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
         if (p != std::string::npos) {
           string rgb24 = line.substr(p + 9, line.find(" ", p + 9));
           rgb24 = rgb24.substr(0, rgb24.find(","));
+          // Strip anything after " ", in case the format is
+          // `640x360 [SAR 1:1 DAR 16:9]`
+          rgb24 = rgb24.substr(0, rgb24.find(" "));
           string rgb24_width = rgb24.substr(0, rgb24.find("x"));
           string rgb24_height = rgb24.substr(rgb24_width.length() + 1);
           if (strings::safe_strtou32(rgb24_width, &width_value) &&
@@ -270,8 +273,10 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
       // We only look for the first stream mapping to have the number of the
       // frames.
       // Once processed we will not further process stream mapping section.
-      if (line.find("frame=  ") == 0) {
-        string number = line.substr(8, line.find(" ", 8));
+      if (line.find("frame=") == 0) {
+        // The format might be `frame=  166 ` or `frame=12488 `
+        string number = line.substr(6);
+        number = number.substr(number.find_first_not_of(" "));
         number = number.substr(0, number.find(" "));
         if (strings::safe_strtou32(number, &frames_value)) {
           in_mapping = false;
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 0eb0e3cbe2..ff6f3b7441 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -354,6 +354,7 @@ py_test(
     name = "classifier_metrics_test",
     srcs = ["python/eval/python/classifier_metrics_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":classifier_metrics",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/kafka/BUILD b/tensorflow/contrib/kafka/BUILD
index efb403462a..1c3974871c 100644
--- a/tensorflow/contrib/kafka/BUILD
+++ b/tensorflow/contrib/kafka/BUILD
@@ -1,66 +1,93 @@
-package(
-    default_visibility = ["//visibility:private"],
-)
+package(default_visibility = ["//tensorflow:internal"])
 
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
-load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_gen_op_wrapper_py",
+    "tf_kernel_library",
+    "tf_custom_op_library",
+    "tf_custom_op_py_library",
+    "tf_gen_op_libs",
+    "tf_py_test",
+)
 
-tf_kernel_library(
-    name = "kafka_kernels",
+py_library(
+    name = "kafka",
+    srcs = ["__init__.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dataset_ops",
+    ],
+)
+
+tf_custom_op_library(
+    name = "_dataset_ops.so",
+    srcs = ["ops/dataset_ops.cc"],
+    deps = [":dataset_kernels"],
+)
+
+tf_gen_op_libs(
+    op_lib_names = ["dataset_ops"],
+)
+
+cc_library(
+    name = "dataset_kernels",
     srcs = ["kernels/kafka_dataset_ops.cc"],
-    visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:bounds_check_lib",
-        "//tensorflow/core/kernels:dataset",
+        "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
         "@kafka",
+        "@protobuf_archive//:protobuf_headers",
     ],
+    alwayslink = 1,
 )
 
-tf_gen_op_libs(
-    op_lib_names = ["kafka_ops"],
+py_library(
+    name = "dataset_ops",
+    srcs = [
+        "python/ops/kafka_dataset_ops.py",
+    ],
+    srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/core:lib",
+        ":kafka_op_loader",
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
     ],
 )
 
 tf_gen_op_wrapper_py(
-    name = "gen_kafka_ops",
-    out = "python/ops/gen_kafka_ops.py",
-    require_shape_functions = True,
-    deps = [":kafka_ops_op_lib"],
+    name = "gen_dataset_ops",
+    out = "python/ops/gen_dataset_ops.py",
+    deps = ["//tensorflow/contrib/kafka:dataset_ops_op_lib"],
 )
 
-py_library(
-    name = "kafka",
-    srcs = [
-        "__init__.py",
-        "python/ops/kafka_dataset_ops.py",
+tf_kernel_library(
+    name = "dataset_ops_kernels",
+    deps = [
+        ":dataset_kernels",
+        "//tensorflow/core:framework",
+    ],
+    alwayslink = 1,
+)
+
+tf_custom_op_py_library(
+    name = "kafka_op_loader",
+    srcs = ["python/ops/kafka_op_loader.py"],
+    dso = ["//tensorflow/contrib/kafka:_dataset_ops.so"],
+    kernels = [
+        ":dataset_ops_kernels",
+        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
     ],
     srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
     deps = [
-        ":gen_kafka_ops",
+        ":gen_dataset_ops",
         "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
-        "//tensorflow/python/data/ops:readers",
     ],
 )
 
@@ -88,6 +115,7 @@ tf_py_test(
     ],
     tags = [
         "manual",
+        "no_windows",
         "notap",
     ],
 )
@@ -95,7 +123,9 @@ tf_py_test(
 filegroup(
     name = "all_files",
     srcs = glob(
-        ["**/*"],
+        include = [
+            "**/*",
+        ],
         exclude = [
             "**/METADATA",
             "**/OWNERS",
diff --git a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
index 88ef5f3571..a4cd4a2cc4 100644
--- a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
+++ b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
@@ -13,9 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/kernels/dataset.h"
-
-#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/dataset.h"
 
 #include "src-cpp/rdkafkacpp.h"
 
diff --git a/tensorflow/contrib/kafka/ops/dataset_ops.cc b/tensorflow/contrib/kafka/ops/dataset_ops.cc
new file mode 100644
index 0000000000..8cdf16103b
--- /dev/null
+++ b/tensorflow/contrib/kafka/ops/dataset_ops.cc
@@ -0,0 +1,44 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+REGISTER_OP("KafkaDataset")
+    .Input("topics: string")
+    .Input("servers: string")
+    .Input("group: string")
+    .Input("eof: bool")
+    .Input("timeout: int64")
+    .Output("handle: variant")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Creates a dataset that emits the messages of one or more Kafka topics.
+
+topics: A `tf.string` tensor containing one or more subscriptions,
+  in the format of [topic:partition:offset:length],
+  by default length is -1 for unlimited.
+servers: A list of bootstrap servers.
+group: The consumer group id.
+eof: If True, the kafka reader will stop on EOF.
+timeout: The timeout value for the Kafka Consumer to wait
+  (in millisecond).
+)doc");
+
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
index 8e51d27a34..a1624614d1 100644
--- a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
+++ b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
@@ -17,8 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.kafka.python.ops import gen_kafka_ops
-from tensorflow.python.data.ops.readers import Dataset
+from tensorflow.contrib.kafka.python.ops import kafka_op_loader  # pylint: disable=unused-import
+from tensorflow.contrib.kafka.python.ops import gen_dataset_ops
+from tensorflow.python.data.ops.dataset_ops import Dataset
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -58,8 +59,8 @@ class KafkaDataset(Dataset):
         timeout, dtype=dtypes.int64, name="timeout")
 
   def _as_variant_tensor(self):
-    return gen_kafka_ops.kafka_dataset(self._topics, self._servers, self._group,
-                                       self._eof, self._timeout)
+    return gen_dataset_ops.kafka_dataset(self._topics, self._servers,
+                                         self._group, self._eof, self._timeout)
 
   @property
   def output_classes(self):
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py b/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
new file mode 100644
index 0000000000..ec2fdea962
--- /dev/null
+++ b/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
@@ -0,0 +1,24 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Python helper for loading kafka ops and kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.util import loader
+from tensorflow.python.platform import resource_loader
+
+_dataset_ops = loader.load_op_library(
+    resource_loader.get_path_to_datafile("../../_dataset_ops.so"))
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 146ae8b7e2..d1c449402a 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -114,6 +114,7 @@ py_test(
     name = "utils_test",
     srcs = ["utils_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         "//tensorflow/contrib/kfac/python/ops:utils",
         "//tensorflow/contrib/tpu",
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 894e6f6946..544065dac6 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -70,6 +70,7 @@ py_test(
         "python/ops/core_test.py",
     ],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":_typecheck",
         ":core",
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index 852d06e1e3..cc7bbabf21 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -188,6 +188,7 @@ py_test(
     size = "small",
     srcs = ["python/layers/normalization_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/contrib/framework:framework_py",
@@ -353,6 +354,7 @@ py_test(
     size = "small",
     srcs = ["python/ops/sparse_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py
index b62e3050cd..ffa208540d 100644
--- a/tensorflow/contrib/layers/python/layers/embedding_ops.py
+++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py
@@ -470,7 +470,7 @@ def embedding_lookup_unique(params, ids, name=None):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops.shape(ids)
     ids_flat = array_ops.reshape(
-        ids, math_ops.reduce_prod(shape, keep_dims=True))
+        ids, math_ops.reduce_prod(shape, keepdims=True))
     unique_ids, idx = array_ops.unique(ids_flat)
     unique_embeddings = embedding_ops.embedding_lookup(params, unique_ids)
     embeds_flat = array_ops.gather(unique_embeddings, idx)
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index f837ca3265..b05f5eeaee 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -5,6 +5,8 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+load("//tensorflow:tensorflow.bzl", "py_test")
+
 package(default_visibility = [
     "//engedu/ml/tf_from_scratch:__pkg__",
     "//tensorflow:internal",
@@ -115,6 +117,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/data_feeder_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/python:client_testlib",
@@ -170,6 +173,7 @@ tf_py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
@@ -188,6 +192,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/graph_actions_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
@@ -426,7 +431,10 @@ py_test(
     size = "medium",
     srcs = ["python/learn/estimators/kmeans_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["noasan"],
+    tags = [
+        "noasan",  # b/73741358
+        "nomac",
+    ],
     deps = [
         ":learn",
         "//tensorflow/python:array_ops",
@@ -585,6 +593,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/io_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/learn/python/learn/datasets",
@@ -814,6 +823,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/utils/saved_model_export_utils_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
index b3b067b8e1..8f9811cf25 100644
--- a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
+++ b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
@@ -61,7 +61,7 @@ def embedding_lookup(params, ids, name='embedding_lookup'):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops_.shape(ids)
     ids_flat = array_ops_.reshape(
-        ids, math_ops.reduce_prod(shape, keep_dims=True))
+        ids, math_ops.reduce_prod(shape, keepdims=True))
     embeds_flat = nn.embedding_lookup(params, ids_flat, name)
     embed_shape = array_ops_.concat([shape, [-1]], 0)
     embeds = array_ops_.reshape(embeds_flat, embed_shape)
diff --git a/tensorflow/contrib/lite/Makefile b/tensorflow/contrib/lite/Makefile
index 7f31629272..b4504f246a 100644
--- a/tensorflow/contrib/lite/Makefile
+++ b/tensorflow/contrib/lite/Makefile
@@ -27,10 +27,10 @@ LIBDIR := $(MAKEFILE_DIR)/gen/lib/
 GENDIR := $(MAKEFILE_DIR)/gen/obj/
 
 # Settings for the host compiler.
-CXX := $(CC_PREFIX) gcc
+CXX := $(CC_PREFIX)gcc
 CXXFLAGS := --std=c++11 -O3 -DNDEBUG
-CC := $(CC_PREFIX) gcc
-CFLAGS :=
+CC := $(CC_PREFIX)gcc
+CFLAGS := -O3 -DNDEBUG
 LDOPTS :=
 LDOPTS += -L/usr/local/lib
 ARFLAGS := -r
@@ -57,10 +57,11 @@ LIBS := \
 
 # If we're on Linux, also link in the dl library.
 ifeq ($(HOST_OS),LINUX)
-	LIBS += -ldl -lpthread
+	LIBS += -ldl
 endif
 
 include $(MAKEFILE_DIR)/ios_makefile.inc
+include $(MAKEFILE_DIR)/rpi_makefile.inc
 
 # This library is the main target for this makefile. It will contain a minimal
 # runtime that can be linked in to other programs.
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index 58bc164619..f84b3dad95 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -33,7 +33,7 @@ class AllocationInfo;
 // each tensor needs to be allocated and deallocated, and preallocates all the
 // necessary memory (the PlanAllocations phase). It then assigns portions of
 // this memory buffer to each tensor (the ExecuteAllocations phase). Tensors may
-// share some of the bufer if a tensor B is to be allocated after another tensor
+// share some of the buffer if a tensor B is to be allocated after another tensor
 // A has been deallocated.
 //
 // If dynamic tensors are used the planning steps can be repeated during model
diff --git a/tensorflow/contrib/lite/build_rpi_lib.sh b/tensorflow/contrib/lite/build_rpi_lib.sh
new file mode 100755
index 0000000000..3824b16412
--- /dev/null
+++ b/tensorflow/contrib/lite/build_rpi_lib.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -x
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/../../.."
+
+CC_PREFIX=arm-linux-gnueabihf- make -j 3 -f tensorflow/contrib/lite/Makefile TARGET=RPI TARGET_ARCH=armv7
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 2218ea8eac..ea3ae3489e 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -24,7 +24,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real builtin
 // ops.
 typedef enum {
   kTfLiteBuiltinAdd = 0,
diff --git a/tensorflow/contrib/lite/error_reporter.h b/tensorflow/contrib/lite/error_reporter.h
index da193d2586..3c5f805f12 100644
--- a/tensorflow/contrib/lite/error_reporter.h
+++ b/tensorflow/contrib/lite/error_reporter.h
@@ -30,7 +30,7 @@ namespace tflite {
 //  va_list args;
 //  foo.Report("test %d", args); // where args is va_list
 //
-// Sublclass ErrorReporter to provide another reporting destination.
+// Subclass ErrorReporter to provide another reporting destination.
 // For example, if you have a GUI program, you might redirect to a buffer
 // that drives a GUI error log box.
 class ErrorReporter {
diff --git a/tensorflow/contrib/lite/g3doc/rpi.md b/tensorflow/contrib/lite/g3doc/rpi.md
new file mode 100644
index 0000000000..7a3a231626
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/rpi.md
@@ -0,0 +1,50 @@
+# TensorFlow Lite for Raspberry Pi
+
+## Cross compiling
+### Installing toolchian
+This has been tested on Ubuntu 16.04.3 64bit and Tensorflow devel docker image [tensorflow/tensorflow:nightly-devel](https://hub.docker.com/r/tensorflow/tensorflow/tags/).
+
+To cross compiling TensorFlow Lite. First you should install the toolchain and libs.
+```bash
+sudo apt-get update
+sudo apt-get install crossbuild-essential-armhf
+```
+> If you are using docker, you may not use `sudo`
+
+### Building
+Clone this Tensorflow repository, Run this script at the root of the repository to download all the dependencies:
+> The Tensorflow repository is in `/tensorflow` if you are using `tensorflow/tensorflow:nightly-devel` docker image, just try it.
+```bash
+./tensorflow/contrib/lite/download_dependencies.sh
+```
+Note than you only need to to this once.
+
+You should then be able to compile:
+```bash
+./tensorflow/contrib/lite/build_rpi_lib.sh
+```
+
+This should compile a static library in:
+`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
+
+## Native compiling
+This has been tested on Raspberry Pi 3b, Raspbian GNU/Linux 9.1 (stretch), gcc version 6.3.0 20170516 (Raspbian 6.3.0-18+rpi1).
+
+Log in to you RPI, install the toolchain.
+```bash
+sudo apt-get instal build-essential
+```
+
+First, clone this TensorFlow repository. Run this at the root of the repository:
+```bash
+./tensorflow/contrib/lite/download_dependencies.sh
+```
+Note than you only need to to this once.
+
+You should then be able to compile:
+```bash
+./tensorflow/contrib/lite/build_rpi_lib.sh
+```
+
+This should compile a static library in:
+`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index af143370ee..3749869f58 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -481,7 +481,7 @@ class Interpreter {
   // During Invoke(), Interpreter will allocate input tensors first, which are
   // known to be fixed size. Then it will allocate outputs from nodes as many
   // as possible. When there is a node that produces dynamic sized tensor.
-  // Intepreter will stop allocating tensors, set the value of next allocate
+  // Interpreter will stop allocating tensors, set the value of next allocate
   // node id, and execute the node to generate the output tensor before continue
   // to allocate successors. This process repeats until all nodes are executed.
   // NOTE: this relies on the order of nodes that is in topological order.
diff --git a/tensorflow/contrib/lite/interpreter_test.cc b/tensorflow/contrib/lite/interpreter_test.cc
index 7a029c7df8..72d4acedbe 100644
--- a/tensorflow/contrib/lite/interpreter_test.cc
+++ b/tensorflow/contrib/lite/interpreter_test.cc
@@ -40,7 +40,7 @@ TEST(BasicInterpreter, InvokeInvalidModel) {
   ASSERT_EQ(interpreter.Invoke(), kTfLiteOk);
 }
 
-// Test size accesser functions.
+// Test size accessor functions.
 TEST(BasicInterpreter, TestSizeFunctions) {
   Interpreter interpreter;
   int base_index;
diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc
index b91ba1a03d..e0cd12f1b4 100644
--- a/tensorflow/contrib/lite/kernels/conv.cc
+++ b/tensorflow/contrib/lite/kernels/conv.cc
@@ -64,7 +64,7 @@ struct OpData {
 
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/depthwise_conv.cc b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
index 15dbfe08c8..cad9ce114c 100644
--- a/tensorflow/contrib/lite/kernels/depthwise_conv.cc
+++ b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
@@ -52,7 +52,7 @@ enum KernelType {
 struct OpData {
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc
index a77fe94e49..888e67966c 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected.cc
@@ -48,7 +48,7 @@ enum KernelType {
 
 struct OpData {
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/kernel_util.h b/tensorflow/contrib/lite/kernels/kernel_util.h
index 28f53b9fbb..21da1daff7 100644
--- a/tensorflow/contrib/lite/kernels/kernel_util.h
+++ b/tensorflow/contrib/lite/kernels/kernel_util.h
@@ -58,7 +58,7 @@ inline bool IsConstantTensor(TfLiteTensor* tensor) {
 }
 
 // Determines whether tensor is dynamic. Note that a tensor can be non-const and
-// not dynamic. This function specificially checks for a dynamic tensor.
+// not dynamic. This function specifically checks for a dynamic tensor.
 inline bool IsDynamicTensor(TfLiteTensor* tensor) {
   return tensor->allocation_type == kTfLiteDynamic;
 }
diff --git a/tensorflow/contrib/lite/kernels/lsh_projection.cc b/tensorflow/contrib/lite/kernels/lsh_projection.cc
index 5f73b56ed9..0ee35775d5 100644
--- a/tensorflow/contrib/lite/kernels/lsh_projection.cc
+++ b/tensorflow/contrib/lite/kernels/lsh_projection.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// LSH Projection projects an input to a bit vector via locality senstive
+// LSH Projection projects an input to a bit vector via locality sensitive
 // hashing.
 //
 // Options:
diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc
index b9255b23a5..8cf1165135 100644
--- a/tensorflow/contrib/lite/kernels/lstm.cc
+++ b/tensorflow/contrib/lite/kernels/lstm.cc
@@ -213,9 +213,9 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
   // present.
   // 2) If projection weight is present, then projection bias is optional.
   // TODO(ghodrat): make sure this is correct.
-  const bool projecton_tensors_consistent =
+  const bool projection_tensors_consistent =
       ((projection_weights != nullptr) || (projection_bias == nullptr));
-  TF_LITE_ENSURE(context, projecton_tensors_consistent == true);
+  TF_LITE_ENSURE(context, projection_tensors_consistent == true);
 
   return kTfLiteOk;
 }
@@ -357,7 +357,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
+  // check the existence of only one to get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/kernels/reshape.cc b/tensorflow/contrib/lite/kernels/reshape.cc
index f3e6ddc9f4..438f70d311 100644
--- a/tensorflow/contrib/lite/kernels/reshape.cc
+++ b/tensorflow/contrib/lite/kernels/reshape.cc
@@ -49,20 +49,20 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   TfLiteIntArray* output_size = TfLiteIntArrayCreate(params->num_dimensions);
   int num_output_elements = 1;
-  int strech_dim = -1;
+  int stretch_dim = -1;
   for (int i = 0; i < params->num_dimensions; ++i) {
     int value = params->shape[i];
     if (value == -1) {
-      TF_LITE_ENSURE_EQ(context, strech_dim, -1);
-      strech_dim = i;
+      TF_LITE_ENSURE_EQ(context, stretch_dim, -1);
+      stretch_dim = i;
     } else {
       num_output_elements *= value;
       output_size->data[i] = value;
     }
   }
-  if (strech_dim != -1) {
-    output_size->data[strech_dim] = num_input_elements / num_output_elements;
-    num_output_elements *= output_size->data[strech_dim];
+  if (stretch_dim != -1) {
+    output_size->data[stretch_dim] = num_input_elements / num_output_elements;
+    num_output_elements *= output_size->data[stretch_dim];
   }
 
   TF_LITE_ENSURE_EQ(context, num_input_elements, num_output_elements);
diff --git a/tensorflow/contrib/lite/kernels/reshape_test.cc b/tensorflow/contrib/lite/kernels/reshape_test.cc
index 0fbcf6e6aa..aecbd0399f 100644
--- a/tensorflow/contrib/lite/kernels/reshape_test.cc
+++ b/tensorflow/contrib/lite/kernels/reshape_test.cc
@@ -60,7 +60,7 @@ TEST(ReshapeOpTest, TooManyDimensions) {
 
 TEST(ReshapeOpTest, TooManySpecialDimensions) {
   EXPECT_DEATH(ReshapeOpModel({1, 2, 4, 1}, {-1, -1, 2, 4}),
-               "strech_dim != -1");
+               "stretch_dim != -1");
 }
 
 TEST(ReshapeOpTest, SimpleTest) {
diff --git a/tensorflow/contrib/lite/kernels/test_util.cc b/tensorflow/contrib/lite/kernels/test_util.cc
index 373310bd87..0bb28b50b2 100644
--- a/tensorflow/contrib/lite/kernels/test_util.cc
+++ b/tensorflow/contrib/lite/kernels/test_util.cc
@@ -141,8 +141,8 @@ void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
 
 void SingleOpModel::SetCustomOp(
     const string& name, const std::vector<uint8_t>& custom_option,
-    const std::function<TfLiteRegistration*()>& registeration) {
-  custom_registrations_[name] = registeration;
+    const std::function<TfLiteRegistration*()>& registration) {
+  custom_registrations_[name] = registration;
   opcodes_.push_back(
       CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
   operators_.push_back(CreateOperator(
diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
index 508a570e2e..42941a97db 100644
--- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
+++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
@@ -360,7 +360,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
+  // check the existence of only one to get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/memory_planner.h b/tensorflow/contrib/lite/memory_planner.h
index 5cd6c20850..0294ec815c 100644
--- a/tensorflow/contrib/lite/memory_planner.h
+++ b/tensorflow/contrib/lite/memory_planner.h
@@ -34,8 +34,8 @@ class MemoryPlanner {
   // [first_node, last_node].
   virtual TfLiteStatus ExecuteAllocations(int first_node, int last_node) = 0;
 
-  // Invalidates allocations made earliers. This is called when tensors sizes
-  // have change. All planned allocations remain, but can't be used until
+  // Invalidates allocations made earlier. This is called when tensors sizes
+  // have changed. All planned allocations remain, but can't be used until
   // ExecuteAllocations() is called.
   virtual TfLiteStatus ResetAllocations() = 0;
 };
diff --git a/tensorflow/contrib/lite/model.h b/tensorflow/contrib/lite/model.h
index a467df5bb4..8dc1c794dc 100644
--- a/tensorflow/contrib/lite/model.h
+++ b/tensorflow/contrib/lite/model.h
@@ -64,7 +64,7 @@ class FlatBufferModel {
       const tflite::Model* model_spec,
       ErrorReporter* error_reporter = DefaultErrorReporter());
 
-  // Releases memory or unmaps mmaped meory.
+  // Releases memory or unmaps mmaped memory.
   ~FlatBufferModel();
 
   // Copying or assignment is disallowed to simplify ownership semantics.
diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
index 76032771af..bd49d327c9 100644
--- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
@@ -569,7 +569,7 @@ enum {
   ANEURALNETWORKS_LOGISTIC = 14,
 
   /**
-   * Projects an input to a bit vector via locality senstive hashing.
+   * Projects an input to a bit vector via locality sensitive hashing.
    *
    * Inputs:
    * * 0: Hash functions. Dim.size == 2, DataType: Float.
diff --git a/tensorflow/contrib/lite/rpi_makefile.inc b/tensorflow/contrib/lite/rpi_makefile.inc
new file mode 100644
index 0000000000..832ef5824b
--- /dev/null
+++ b/tensorflow/contrib/lite/rpi_makefile.inc
@@ -0,0 +1,33 @@
+# Settings for Raspberry Pi.
+ifeq ($(TARGET), RPI)
+	ifeq ($(TARGET_ARCH), armv7)
+		CXXFLAGS += \
+			-march=armv7-a \
+			-mfpu=neon-vfpv4 \
+			-funsafe-math-optimizations \
+			-ftree-vectorize
+
+		CCFLAGS += \
+			-march=armv7-a \
+			-mfpu=neon-vfpv4 \
+			-funsafe-math-optimizations \
+			-ftree-vectorize
+
+		LDFLAGS := \
+			-Wl,--no-export-dynamic \
+			-Wl,--exclude-libs,ALL \
+			-Wl,--gc-sections \
+			-Wl,--as-needed
+	endif
+
+	LIBS := \
+	-lstdc++ \
+	-lpthread \
+	-lm \
+	-ldl
+
+	OBJDIR := $(OBJDIR)rpi_$(TARGET_ARCH)/
+	LIBDIR := $(LIBDIR)rpi_$(TARGET_ARCH)/
+	BINDIR := $(BINDIR)rpi_$(TARGET_ARCH)/
+	DEPDIR := $(DEPDIR)rpi_$(TARGET_ARCH)/
+endif
diff --git a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
index 08bcfe4516..640972de77 100644
--- a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
+++ b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
@@ -46,7 +46,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real builtin
 // ops.
 typedef enum {
 )";
diff --git a/tensorflow/contrib/lite/simple_memory_arena.cc b/tensorflow/contrib/lite/simple_memory_arena.cc
index 4aab244989..2f2004f56b 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.cc
+++ b/tensorflow/contrib/lite/simple_memory_arena.cc
@@ -113,21 +113,21 @@ TfLiteStatus SimpleMemoryArena::Commit(TfLiteContext* context) {
     underlying_buffer_size_ = required_size;
     underlying_buffer_aligned_ptr_ = new_underlying_buffer_aligned_ptr;
   }
-  commited_ = true;
+  committed_ = true;
   return underlying_buffer_ != nullptr ? kTfLiteOk : kTfLiteError;
 }
 
 TfLiteStatus SimpleMemoryArena::ResolveAlloc(TfLiteContext* context,
                                              const ArenaAlloc& alloc,
                                              char** output_ptr) {
-  TF_LITE_ENSURE(context, commited_);
+  TF_LITE_ENSURE(context, committed_);
   TF_LITE_ENSURE(context, output_ptr != nullptr);
   *output_ptr = underlying_buffer_aligned_ptr_ + alloc.offset;
   return kTfLiteOk;
 }
 
 TfLiteStatus SimpleMemoryArena::Clear() {
-  commited_ = false;
+  committed_ = false;
   high_water_mark_ = 0;
   allocs_.clear();
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/simple_memory_arena.h b/tensorflow/contrib/lite/simple_memory_arena.h
index 0535522374..5faf78b59e 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.h
+++ b/tensorflow/contrib/lite/simple_memory_arena.h
@@ -22,7 +22,7 @@ limitations under the License.
 namespace tflite {
 
 // This little structure holds the offset and the size for a dynamic memory
-// allocation in the memory arena. When the arena is commited and the
+// allocation in the memory arena. When the arena is committed and the
 // underlying buffer is set, the alloc can be resolved into an actual memory
 // pointer.
 struct ArenaAlloc {
@@ -43,7 +43,7 @@ struct ArenaAlloc {
 class SimpleMemoryArena {
  public:
   explicit SimpleMemoryArena(size_t arena_alignment)
-      : commited_(false),
+      : committed_(false),
         arena_alignment_(arena_alignment),
         high_water_mark_(0),
         underlying_buffer_size_(0),
@@ -73,7 +73,7 @@ class SimpleMemoryArena {
   }
 
  private:
-  bool commited_;
+  bool committed_;
   size_t arena_alignment_;
   size_t high_water_mark_;
   std::unique_ptr<char[]> underlying_buffer_;
diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD
index 8ca03f4193..0a6edc33c5 100644
--- a/tensorflow/contrib/lookup/BUILD
+++ b/tensorflow/contrib/lookup/BUILD
@@ -46,6 +46,7 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     grpc_enabled = True,
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 filegroup(
diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md
index 995230dfa8..6c3b02e12b 100644
--- a/tensorflow/contrib/makefile/README.md
+++ b/tensorflow/contrib/makefile/README.md
@@ -194,6 +194,8 @@ with:
 srcs = glob(["libs/arm64-v8a/*.so"]),
 ```
 
+If you are building for Android TV (Shield TV devices), replace "portrait" with "landscape" for android:screenOrientation in all four activities in tensorflow/examples/android/AndroidManifest.xml
+
 Then run:
 ```bash
 # Create dir for native libs
diff --git a/tensorflow/contrib/makefile/build_all_ios.sh b/tensorflow/contrib/makefile/build_all_ios.sh
index 2d99791839..9b148688c4 100755
--- a/tensorflow/contrib/makefile/build_all_ios.sh
+++ b/tensorflow/contrib/makefile/build_all_ios.sh
@@ -80,10 +80,9 @@ if [[ ! -z "${OPTIMIZE_FOR_GRAPH}" ]]; then
         fi
     else
         echo "${PRNT_SLCTV_BIN} found. Using it"
-        ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
-
     fi
 
+    ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
 fi
 
 if [[ "${ONLY_MAKE_TENSORFLOW}" != "true" ]]; then
diff --git a/tensorflow/contrib/mpi/mpi_utils.h b/tensorflow/contrib/mpi/mpi_utils.h
index fa297c28cb..df055ff567 100644
--- a/tensorflow/contrib/mpi/mpi_utils.h
+++ b/tensorflow/contrib/mpi/mpi_utils.h
@@ -24,6 +24,8 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/str_util.h"
 
+// Skip MPI C++ bindings support, this matches the usage in other places
+#define OMPI_SKIP_MPICXX
 #include "third_party/mpi/mpi.h"
 #define MPI_CHECK(cmd)                                                \
   do {                                                                \
diff --git a/tensorflow/contrib/predictor/predictor_factories.py b/tensorflow/contrib/predictor/predictor_factories.py
index 04b5d5bdf1..6e77e934fe 100644
--- a/tensorflow/contrib/predictor/predictor_factories.py
+++ b/tensorflow/contrib/predictor/predictor_factories.py
@@ -53,7 +53,7 @@ def from_contrib_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, core_estimator.Estimator):
-    raise TypeError('Espected estimator to be of type '
+    raise TypeError('Expected estimator to be of type '
                     'tf.contrib.learn.Estimator, but got type '
                     'tf.python.estimator.Estimator. You likely want to call '
                     'from_estimator.')
@@ -88,7 +88,7 @@ def from_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, contrib_estimator.Estimator):
-    raise TypeError('Espected estimator to be of type '
+    raise TypeError('Expected estimator to be of type '
                     'tf.python.estimator.Estimator, but got type '
                     'tf.contrib.learn.Estimator. You likely want to call '
                     'from_contrib_estimator.')
diff --git a/tensorflow/contrib/py2tf/converters/BUILD b/tensorflow/contrib/py2tf/converters/BUILD
index f624c42686..4bb6f76019 100644
--- a/tensorflow/contrib/py2tf/converters/BUILD
+++ b/tensorflow/contrib/py2tf/converters/BUILD
@@ -81,6 +81,7 @@ py_test(
     name = "builtin_functions_test",
     srcs = ["builtin_functions_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/python:client_testlib",
@@ -91,6 +92,7 @@ py_test(
     name = "call_trees_test",
     srcs = ["call_trees_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/contrib/py2tf/impl",
diff --git a/tensorflow/contrib/py2tf/converters/single_return.py b/tensorflow/contrib/py2tf/converters/single_return.py
index 90bc22008f..1194b98f5e 100644
--- a/tensorflow/contrib/py2tf/converters/single_return.py
+++ b/tensorflow/contrib/py2tf/converters/single_return.py
@@ -212,7 +212,7 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
 
   def __init__(self):
     self.cant_return = False
-    super(gast.NodeVisitor, self).__init__()
+    super(DetectReturnInUnsupportedControlFlow, self).__init__()
 
   def visit_While(self, node):
     self.cant_return = True
diff --git a/tensorflow/contrib/py2tf/utils/BUILD b/tensorflow/contrib/py2tf/utils/BUILD
index d029289f5a..8bc338e801 100644
--- a/tensorflow/contrib/py2tf/utils/BUILD
+++ b/tensorflow/contrib/py2tf/utils/BUILD
@@ -83,6 +83,7 @@ py_test(
     name = "py_func_test",
     srcs = ["py_func_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":utils",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index b278265639..1afcbb8504 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -237,7 +237,7 @@ def _FindFusedBatchNorms(graph):
       # The batch variance used during forward and backward prop is biased,
       # i.e it is calculated as: V=sum(x(k)-mu)^2/N. For the moving average
       # calculation, the variance is corrected by the term N/N-1 (Bessel's
-      # correction). The variance tensor read from FuseBatchNorm has bessel's
+      # correction). The variance tensor read from FuseBatchNorm has Bessel's
       # correction applied, so we undo it here.
       scope, sep, _ = bn_op.name.rpartition('/')
       g = ops.get_default_graph()
@@ -306,7 +306,7 @@ def _ComputeBatchNormCorrections(context, match, freeze_batch_norm_delay,
 
   Args:
     context: The scope under which we look for batch norm params
-    match: Object containg required batch norm tensors for correction
+    match: Object containing required batch norm tensors for correction
       computation.
     freeze_batch_norm_delay: Delay in steps at which computation switches
       from regular batch norm to frozen mean and variance.
diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py
index 0a8e35080c..a4f7b1b221 100644
--- a/tensorflow/contrib/quantize/python/quant_ops.py
+++ b/tensorflow/contrib/quantize/python/quant_ops.py
@@ -282,8 +282,8 @@ def _FakeQuantWithMinMaxVars(inputs, min_var, max_var, per_channel, num_bits,
   Args:
     inputs: a tensor containing values to be quantized.
     min_var: a variable containing quantization range lower end(s).
-    max_var: a variable containing quantization range lupper end(s).
-    per_channel: a boolean specifying whether to use per-channel quantizatioh.
+    max_var: a variable containing quantization range upper end(s).
+    per_channel: a boolean specifying whether to use per-channel quantization.
     num_bits: Number of bits to use for quantization, must be between 2 and 8.
     narrow_range: Whether to use the narrow quantization range
       [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1].
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 0608ab9302..ec721afbc8 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -267,7 +267,7 @@ def _InsertQuantOp(context,
   """Inserts a quant op between a producer op and (multiple) consumer ops.
 
   Args:
-    context: Context w,here producer and consumer operations are nested.
+    context: Context where producer and consumer operations are nested.
     name: Name for the new quantization op within the context.
     producer: Producer operation of the pairs where quantization will be
       inserted.
diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index 5a3a74cec4..5abdcd2475 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -158,7 +158,7 @@ def experimental_create_training_graph(input_graph=None,
   often fail.
 
   Args:
-    input_graph: The tf.Graph to be transformed,if None then defaults to the
+    input_graph: The tf.Graph to be transformed, if None then defaults to the
       default graph.
     weight_bits: Number of bits to use for quantizing weights.
     activation_bits: Number of bits to use for quantizing activations.
diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
index 0624cc878b..db745aa562 100644
--- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
@@ -419,7 +419,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -470,7 +470,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -526,7 +526,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -565,7 +565,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initialzes with a truncated normal variable.
+      An initialized that initializes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index ef59475167..b2e5707a6d 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -144,7 +144,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initialzes with a truncated normal variable.
+      An initialized that initializes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
index 27f0a7f58f..54c66271cd 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD
+++ b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
@@ -38,7 +38,6 @@ py_test(
     size = "small",
     srcs = ["python/ops/remote_fused_graph_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":remote_fused_graph_ops_py",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 73f2607d84..358b2eb02b 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2133,7 +2133,7 @@ class Conv1DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_1d_lstm_cell", **kwargs):
     """Construct Conv1DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, **kwargs)
+    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, name=name, **kwargs)
 
 
 class Conv2DLSTMCell(ConvLSTMCell):
@@ -2144,7 +2144,7 @@ class Conv2DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_2d_lstm_cell", **kwargs):
     """Construct Conv2DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, **kwargs)
+    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, name=name, **kwargs)
 
 
 class Conv3DLSTMCell(ConvLSTMCell):
@@ -2155,7 +2155,7 @@ class Conv3DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_3d_lstm_cell", **kwargs):
     """Construct Conv3DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, **kwargs)
+    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, name=name, **kwargs)
 
 
 def _conv(args, filter_size, num_features, bias, bias_start=0.0):
diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index 245fe07f2b..b10757df47 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -53,6 +53,7 @@ py_test(
     size = "small",
     srcs = ["python/saved_model/reader_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":saved_model_py",
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 03fe31abf7..6adbb8be40 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -299,12 +299,13 @@ class BeamSearchDecoder(decoder.Decoder):
     """
     finished, start_inputs = self._finished, self._start_inputs
 
+    dtype = nest.flatten(self._initial_cell_state)[0].dtype
     log_probs = array_ops.one_hot(  # shape(batch_sz, beam_sz)
         array_ops.zeros([self._batch_size], dtype=dtypes.int32),
         depth=self._beam_width,
-        on_value=0.0,
-        off_value=-np.Inf,
-        dtype=nest.flatten(self._initial_cell_state)[0].dtype)
+        on_value=ops.convert_to_tensor(0.0, dtype=dtype),
+        off_value=ops.convert_to_tensor(-np.Inf, dtype=dtype),
+        dtype=dtype)
 
     initial_state = BeamSearchDecoderState(
         cell_state=self._initial_cell_state,
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 67011c8fef..3ad88a8a22 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -165,6 +165,7 @@ py_test(
     name = "gc_test",
     srcs = ["gc_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":gc",
diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD
index 5daabbd62e..7aa1684839 100644
--- a/tensorflow/contrib/slim/python/slim/data/BUILD
+++ b/tensorflow/contrib/slim/python/slim/data/BUILD
@@ -61,6 +61,7 @@ py_test(
     name = "dataset_data_provider_test",
     srcs = ["dataset_data_provider_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":dataset",
         ":dataset_data_provider",
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 1e4cc3f095..07b6b1f142 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -553,7 +553,6 @@ py_test(
     srcs = ["client/random_forest_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-        "no_windows",
         "nomac",  # b/63258195
         "notsan",
     ],
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index d833744d0c..db2e000ef8 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -9,6 +9,7 @@ exports_files(["LICENSE"])
 
 # For platform specific build config
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
+load("//tensorflow:tensorflow.bzl", "py_test")
 
 tf_proto_library(
     name = "protos_all",
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index c832c6f2e0..906cc3f034 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -83,6 +83,7 @@ cc_library(
         "kernels/trt_engine_op.h",
     ],
     copts = tf_copts(),
+    visibility = ["//visibility:public"],
     deps = [
         ":trt_logging",
         ":trt_resources",
@@ -154,6 +155,7 @@ py_library(
     deps = [
         ":trt_convert_py",
         ":trt_ops_py",
+        "//tensorflow/python:errors",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index dfcce0fd00..461e627e99 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -2,7 +2,8 @@ Using TensorRT in TensorFlow
 ============================
 
 This module provides necessary bindings and introduces TRT_engine_op
-operator that wraps a subgraph in TensorRT.
+operator that wraps a subgraph in TensorRT. This is still a work in progress
+but should be useable with most common graphs.
 
 Compilation
 -----------
@@ -15,26 +16,10 @@ configure script should find the necessary components from the system
 automatically. If installed from tar packages, user has to set path to
 location where the library is installed during configuration.
 
-
-```
+```shell
 bazel build --config=cuda --config=opt //tensorflow/tools/pip_package:build_pip_package
 bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use is shown below.
-
-```python
-import tensorflow as tf
-import tensorflow.contrib.tensorrt as trt
-#... create and train or load model
-gdef = sess.graph.as_graph_def()
-trt_gdef = trt.create_inference_graph(
-    gdef, #original graph_def
-    ["output"], #name of output node(s)
-    max_batch_size, #maximum batch size to run the inference
-    max_workspace_size_bytes) # max memory for TensorRT to use
-tf.reset_default_graph()
-tf.import_graph_def(graph_def=trt_gdef)
-#...... run inference
-```
+will be available. An example use can be found in test/test_tftrt.py directory
diff --git a/tensorflow/contrib/tensorrt/__init__.py b/tensorflow/contrib/tensorrt/__init__.py
index fd551d70b4..140ad48282 100644
--- a/tensorflow/contrib/tensorrt/__init__.py
+++ b/tensorflow/contrib/tensorrt/__init__.py
@@ -18,6 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-# pylint: disable=unused-import,wildcard-import
-from tensorflow.contrib.tensorrt.python import *
-# pylint: enable=unused-import,wildcard-import
+from tensorflow.python.framework import errors
+
+# pylint: disable=unused-import,wildcard-import,g-import-not-at-top
+try:
+  from tensorflow.contrib.tensorrt.python import *
+except errors.NotFoundError as e:
+  no_trt_message = (
+      '**** Failed to initialize TensorRT. This is either because the TensorRT'
+      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
+      ' it installed. If not installed, please go to'
+      ' https://developer.nvidia.com/tensorrt to download and install'
+      ' TensorRT ****')
+  print(no_trt_message)
+  raise e
+# pylint: enable=unused-import,wildcard-import,g-import-not-at-top
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 970f810473..eea8c8efa2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
 
+#include <list>
 #include <map>
 #include <set>
 #include <unordered_map>
@@ -48,13 +49,29 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-static bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
-      "Identity", "Const", "Conv2D", "MaxPool", "BiasAdd", "Relu",
-      "Add",      "Mul",   "Sub",    "Rsqrt",   "Pad"  // "Placeholder" ,"Mean"
+      "Identity",
+      "Const",
+      "Conv2D",
+      "MaxPool",
+      "BiasAdd",
+      "Relu",
+      "Add",
+      "Mul",
+      "Sub",
+      "Rsqrt",
+      "Pad",
+      "Mean",
+      "AvgPool",
+      "ConcatV2",
+      "DepthwiseConv2dNative",
+      "FusedBatchNorm",
+      "FusedBatchNormV2",
+      // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
   return candidate_ops.count(node_def.op());
@@ -69,6 +86,8 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
       if (!subgraph_node_ids.count(edge->src()->id()) &&
           !edge->src()->IsSource()) {
         incoming_edges->insert(edge);
+      } else {
+        VLOG(2) << edge->src()->name() << " N, ";
       }
     }
   }
@@ -82,7 +101,10 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
           !edge->dst()->IsSink()) {
+        VLOG(2) << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
+      } else {
+        VLOG(2) << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -109,74 +131,150 @@ std::unordered_map<string, std::vector<int>> BuildTensorNameMap(
   }
   return result;
 }
-
-tensorflow::Status ConvertSubGraphToTensorRT(
-    const std::vector<string>& output_names,
-    const std::set<int>& subgraph_node_ids,
-    size_t max_batch_size,  // Max batch size that engine will be created for
-    // Max amount of memory that engine will be allowed to consume, in bytes
-    size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_properties,
-    tensorflow::Graph* graph) {
-  tensorflow::EdgeSet subgraph_incoming_edges;
-  GetSubGraphIncomingEdges(*graph, subgraph_node_ids, &subgraph_incoming_edges);
-
+// TODO(sami): convert references to pointers
+struct ConvertGraphParams {
+  ConvertGraphParams(
+      tensorflow::Graph& inp_graph,
+      const std::vector<string>& output_node_names,
+      const std::set<int>& subgraph_node_id_numbers,
+      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
+      const tensorflow::grappler::GraphProperties& current_graph_properties,
+      std::unordered_map<string, std::pair<int, string>>* output_edges,
+      int engine_precision_mode)
+      : graph(inp_graph),
+        output_names(output_node_names),
+        subgraph_node_ids(subgraph_node_id_numbers),
+        max_batch_size(max_supported_batch_size),
+        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
+        graph_properties(current_graph_properties),
+        output_edge_map(output_edges),
+        precision_mode(engine_precision_mode) {}
+  tensorflow::Graph& graph;
+  const std::vector<string>& output_names;
+  const std::set<int>& subgraph_node_ids;
+  size_t max_batch_size;
+  size_t max_workspace_size_bytes;
+  const tensorflow::grappler::GraphProperties& graph_properties;
+  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
+  int precision_mode;
   std::vector<std::pair<int, int>> subgraph_inputs;
+  std::vector<std::pair<int, int>> subgraph_outputs;
+  tensorflow::EdgeSet subgraph_incoming_edges;
+  tensorflow::EdgeSet subgraph_outgoing_edges;
+};
 
-  // Collect inputs by looking for incoming edges
-  for (const tensorflow::Edge* edge : subgraph_incoming_edges) {
-    subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
+static tensorflow::Status FillSubGraphEdgeSets(ConvertGraphParams* p) {
+  GetSubGraphIncomingEdges(p->graph, p->subgraph_node_ids,
+                           &p->subgraph_incoming_edges);
+  for (const tensorflow::Edge* edge : p->subgraph_incoming_edges) {
+    p->subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
   }
+  auto output_name_to_index_map = BuildTensorNameMap(p->output_names);
   std::set<std::pair<int, int>> subgraph_outputs_set;
   // Collect outputs referenced from output_names
-  auto output_name_to_index_map = BuildTensorNameMap(output_names);
-  for (int node_id : subgraph_node_ids) {
-    tensorflow::Node* node = graph->FindNodeId(node_id);
+  for (int node_id : p->subgraph_node_ids) {
+    tensorflow::Node* node = p->graph.FindNodeId(node_id);
     if (output_name_to_index_map.count(node->name())) {
       for (int index : output_name_to_index_map.at(node->name())) {
         subgraph_outputs_set.insert({node_id, index});
       }
     }
   }
-  // Collect outputs referenced from outgoing edges
-  tensorflow::EdgeSet subgraph_outgoing_edges;
-  GetSubGraphOutgoingEdges(*graph, subgraph_node_ids, &subgraph_outgoing_edges);
-  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
+  GetSubGraphOutgoingEdges(p->graph, p->subgraph_node_ids,
+                           &p->subgraph_outgoing_edges);
+  for (const tensorflow::Edge* edge : p->subgraph_outgoing_edges) {
     subgraph_outputs_set.insert({edge->src()->id(), edge->src_output()});
   }
-  // Impose an ordering on the outputs
-  std::vector<std::pair<int, int>> subgraph_outputs(
-      subgraph_outputs_set.begin(), subgraph_outputs_set.end());
-  // Build TensorRT node and add it to the graph
+  p->subgraph_outputs.reserve(subgraph_outputs_set.size());
+  p->subgraph_outputs.insert(p->subgraph_outputs.begin(),
+                             subgraph_outputs_set.begin(),
+                             subgraph_outputs_set.end());
+  return tensorflow::Status::OK();
+};
+
+tensorflow::Status GetCalibNode(ConvertGraphParams* params) {
+  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
   tensorflow::NodeDef trt_node_def;
-  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(
-      *graph, subgraph_node_ids, subgraph_inputs, subgraph_outputs,
-      max_batch_size, max_workspace_size_bytes, graph_properties,
-      &trt_node_def));
+  SubGraphParams s(params->graph, params->subgraph_node_ids,
+                   params->subgraph_inputs, params->subgraph_outputs,
+                   params->max_batch_size, params->max_workspace_size_bytes,
+                   params->graph_properties, params->output_edge_map,
+                   &trt_node_def, params->precision_mode);
+  TF_RETURN_IF_ERROR(InjectCalibrationNode(s));
   tensorflow::Status status;
-  tensorflow::Node* trt_node = graph->AddNode(trt_node_def, &status);
+  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
+
+  TF_RETURN_IF_ERROR(status);
+
+  for (auto in_edge :
+       params->subgraph_incoming_edges) {  // loop over incoming edges and
+                                           // attach them to calib node
+    // tensorflow::Node* src_node = in_edge->src();
+    auto src_output = in_edge->src_output();
+    auto dst_node = in_edge->dst();
+    auto dst_input = in_edge->dst_input();
+    VLOG(1) << " update edge " << trt_node->name() << ":" << src_output
+            << " -> " << dst_node->name() << ":" << dst_input;
+    TF_RETURN_IF_ERROR(
+        params->graph.UpdateEdge(trt_node, src_output, dst_node, dst_input));
+  }
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertSubGraphToTensorRT(ConvertGraphParams* params) {
+  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
+  tensorflow::NodeDef trt_node_def;
+
+  SubGraphParams s(params->graph, params->subgraph_node_ids,
+                   params->subgraph_inputs, params->subgraph_outputs,
+                   params->max_batch_size, params->max_workspace_size_bytes,
+                   params->graph_properties, params->output_edge_map,
+                   &trt_node_def, params->precision_mode);
+  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(s));
+  tensorflow::Status status;
+  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
+
+  // AddNode does not wire edges.
+  // Re-map incoming edges to use the new TRT node instead of the orig subgraph
+  std::map<std::pair<int, int>, int> subgraph_edge_to_input_map;
+  for (size_t i = 0; i < params->subgraph_inputs.size(); ++i) {
+    subgraph_edge_to_input_map.insert({params->subgraph_inputs.at(i), i});
+  }
+  for (const tensorflow::Edge* edge : params->subgraph_incoming_edges) {
+    std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
+    int new_src_output = subgraph_edge_to_input_map.at(old_src);
+    params->graph.AddEdge(edge->src(), edge->src_output(), trt_node,
+                          new_src_output);
+    params->graph.RemoveEdge(edge);
+  }
+
+  VLOG(2) << "new wiring edges: " << trt_node->in_edges().size();
+  for (const tensorflow::Edge* edge : trt_node->in_edges()) {
+    VLOG(2) << edge->src()->name() << " port: " << edge->src_output();
+  }
+
   TF_RETURN_IF_ERROR(status);
 
   // Re-map outgoing edges to use the new TRT node instead of the orig subgraph
   std::map<std::pair<int, int>, int> subgraph_edge_to_output_map;
-  for (size_t i = 0; i < subgraph_outputs.size(); ++i) {
-    subgraph_edge_to_output_map.insert({subgraph_outputs.at(i), i});
+  for (size_t i = 0; i < params->subgraph_outputs.size(); ++i) {
+    subgraph_edge_to_output_map.insert({params->subgraph_outputs.at(i), i});
   }
   TF_RETURN_IF_ERROR(status);
-  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
+  for (const tensorflow::Edge* edge : params->subgraph_outgoing_edges) {
     std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
     int new_src_output = subgraph_edge_to_output_map.at(old_src);
-    TF_RETURN_IF_ERROR(graph->UpdateEdge(trt_node, new_src_output, edge->dst(),
-                                         edge->dst_input()));
+    TF_RETURN_IF_ERROR(params->graph.UpdateEdge(
+        trt_node, new_src_output, edge->dst(), edge->dst_input()));
   }
   // Remove the original subgraph
-  for (int node_id : subgraph_node_ids) {
-    tensorflow::Node* node = graph->FindNodeId(node_id);
+  for (int node_id : params->subgraph_node_ids) {
+    tensorflow::Node* node = params->graph.FindNodeId(node_id);
     // Don't remove the input placeholders
     if (node->type_string() == "Placeholder") {
       continue;
     }
-    graph->RemoveNode(node);
+    params->graph.RemoveNode(node);
   }
   return tensorflow::Status::OK();
 }
@@ -194,12 +292,39 @@ tensorflow::Status BuildNodeMap(
 }
 
 }  // namespace
+tensorflow::Status ConvertCalibGraphToInferGraph(
+    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph) {
+  VLOG(0) << "Starting Calib Conversion";
+  tensorflow::Graph graph(tensorflow::OpRegistry::Global());
+  TF_RETURN_IF_ERROR(tensorflow::ConvertGraphDefToGraph(
+      tensorflow::GraphConstructorOptions(), graph_def, &graph));
+  //  get calib nodes
+  std::vector<tensorflow::Node*> calib_nodes;
+  for (auto node : graph.op_nodes()) {
+    if (node->type_string() == "TRTCalibOp") {
+      VLOG(1) << "Found Calib Node";
+      calib_nodes.push_back(node);
+    }
+  }
+  VLOG(0) << "Num Calib nodes in graph= " << calib_nodes.size();
+  if (calib_nodes.size() == 0)
+    return tensorflow::errors::FailedPrecondition(
+        "Graph doesn't contain any calibration nodes!."
+        " Please generate calibration graph and run calibration first");
+  for (auto n : calib_nodes) {
+    TF_RETURN_IF_ERROR(
+        tensorrt::convert::ConvertCalibrationNodeToEngineNode(graph, n));
+  }
+  graph.ToGraphDef(infer_graph);
+  return tensorflow::Status::OK();
+}
 
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def) {
-  // Optimization pass
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
+    int precision_mode = FP32MODE, int minimum_segment_size = 3) {
+  // optimization pass
   tensorflow::grappler::GrapplerItem item;
   item.fetch = output_names;
   tensorflow::GraphDef gdef;
@@ -209,16 +334,23 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   tensorflow::grappler::LayoutOptimizer optimizer;
   tensorflow::grappler::Cluster* cluster;
 
-  // Virtual cluster
+  // virtual cluster
   tensorflow::DeviceProperties device_properties;
+
   device_properties.set_type("GPU");
   device_properties.mutable_environment()->insert({"architecture", "6"});
   cluster =
       new tensorflow::grappler::VirtualCluster({{"/GPU:0", device_properties}});
 
+  // single machine
+  int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
+  int num_gpus = tensorflow::grappler::GetNumAvailableGPUs();
+  VLOG(2) << "cpu_cores: " << num_cpu_cores;
+  VLOG(2) << "gpus: " << num_gpus;
+
   TF_RETURN_IF_ERROR(optimizer.Optimize(cluster, item, &gdef));
 
-  // Constant folding
+  // constant folding
   item.graph = gdef;
   tensorflow::grappler::ConstantFolding fold(nullptr);
   TF_RETURN_IF_ERROR(fold.Optimize(nullptr, item, &gdef));
@@ -226,7 +358,6 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   // AJ refactoring shape inference through grappler/GraphProperties.
   tensorflow::grappler::GraphProperties static_graph_properties(item);
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(false));
-
   // Build full graph
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
                                              gdef.library());
@@ -243,7 +374,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
 
   // TODO(sami): this should be passed as a knob!!!!
-  segment_options.minimum_segment_size = 2;
+  segment_options.minimum_segment_size = minimum_segment_size;
   tensorflow::tensorrt::segment::SegmentNodesVector segments;
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       gdef, IsTensorRTCandidate, segment_options, &segments));
@@ -252,14 +383,37 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
   std::unordered_map<string, tensorflow::Node*> node_map;
   TF_RETURN_IF_ERROR(BuildNodeMap(graph, &node_map));
+  std::unordered_map<string, std::pair<int, string>> output_edge_map;
+  int count = 0;
+  float total_num_nodes_in_segments = 0.;
+  for (auto s : segments) {
+    total_num_nodes_in_segments += s.size();
+  }
   for (const std::set<string>& subgraph_node_names : segments) {
     std::set<int> subgraph_node_ids;
+    size_t max_mem_per_engine =
+        max_workspace_size_bytes *
+        ((float)subgraph_node_names.size() / total_num_nodes_in_segments);
+    std::stringstream oss;
     for (const string& node_name : subgraph_node_names) {
+      oss << " " << node_name;
       subgraph_node_ids.insert(node_map.at(node_name)->id());
     }
-    TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRT(
-        output_names, subgraph_node_ids, max_batch_size,
-        max_workspace_size_bytes, static_graph_properties, &graph));
+    VLOG(2) << "Subgraph nodes" << oss.str();
+    ConvertGraphParams p(graph, output_names, subgraph_node_ids, max_batch_size,
+                         max_mem_per_engine, static_graph_properties,
+                         &output_edge_map, precision_mode);
+    if (precision_mode == INT8MODE) {
+      TF_RETURN_IF_ERROR(GetCalibNode(&p));
+    } else {
+      tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
+      if (status != tensorflow::Status::OK()) {
+        LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
+                     << " due to: \n"
+                     << status.ToString() << " SKIPPING......";
+      }
+      count++;
+    }
   }
   graph.ToGraphDef(new_graph_def);
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 154ad3f2e8..e1596e89e2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -28,6 +28,11 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
+// This method converts an already generated calibration graph which was used in
+// calibration runs to an inference graph
+tensorflow::Status ConvertCalibGraphToInferGraph(
+    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* new_graph_def);
+
 // max_batch_size: maximum batch size which can be used for inference for
 //                 optimization targets inference run with max batch size.
 // max_workspace_size_bytes: The upper bound of memory allowence for
@@ -35,7 +40,8 @@ namespace convert {
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def);
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
+    int precision_mode, int minimum_segment_size);
 
 }  // namespace convert
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 9ee717dd7f..75a3c3d034 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -24,6 +24,10 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"  // NOLINT
 #include "tensorflow/core/framework/types.h"
@@ -32,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tensor_coding.h"
@@ -39,7 +44,6 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorrt/include/NvInfer.h"
 
 //  Check if the types are equal. Cast to int first so that failure log message
@@ -49,6 +53,7 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrCat;
 
 namespace {
 
@@ -65,7 +70,8 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
       *trt_dtype = nvinfer1::DataType::kHALF;
       break;
     default:
-      return tensorflow::errors::InvalidArgument("Unsupported data type");
+      return tensorflow::errors::InvalidArgument(
+          "Unsupported data type " + tensorflow::DataTypeString(tf_dtype));
   }
   return tensorflow::Status::OK();
 }
@@ -112,6 +118,18 @@ static std::vector<std::pair<int, int>> CreateSamePadding(
   return padding;
 }
 
+string GetCommonNameScope(const string& op_name_a, const string& op_name_b) {
+  size_t last_scope_separator = 0;
+  for (size_t i = 0; i < std::min(op_name_a.size(), op_name_b.size()); ++i) {
+    if (op_name_a[i] != op_name_b[i]) {
+      break;
+    } else if (op_name_a[i] == '/') {
+      last_scope_separator = i + 1;
+    }
+  }
+  return op_name_a.substr(0, last_scope_separator);
+}
+
 class TRT_ShapedWeights {
  public:
   TRT_ShapedWeights(tensorflow::DataType type, const void* values,
@@ -244,6 +262,11 @@ std::vector<int> TFAttrs::get<std::vector<int>>(string key) const {
   return std::vector<int>(attr.begin(), attr.end());
 }
 
+template <>
+std::vector<string> TFAttrs::get<std::vector<string>>(string key) const {
+  auto attr = this->at(key)->list().s();
+  return std::vector<string>(attr.begin(), attr.end());
+}
 template <>
 nvinfer1::Dims TFAttrs::get<nvinfer1::Dims>(string key) const {
   auto values = this->get<std::vector<int>>(key);
@@ -266,6 +289,17 @@ tensorflow::DataType TFAttrs::get<tensorflow::DataType>(string key) const {
   return this->at(key)->type();
 }
 
+template <>
+float TFAttrs::get<float>(string key) const {
+  return this->at(key)->f();
+}
+
+template <>
+bool TFAttrs::get<bool>(string key) const {
+  return this->at(key)->b();
+}
+
+// TODO(jie): reorder4 & reorder2 should be merged?
 template <typename T>
 void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
               nvinfer1::DimsNCHW istrides, T* odata,
@@ -283,29 +317,87 @@ void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
   }
 }
 
+template <typename T>
+void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides,
+              T* odata, nvinfer1::DimsHW ostrides) {
+  for (int h = 0; h < shape.h(); ++h) {
+    for (int w = 0; w < shape.w(); ++w) {
+      odata[h * ostrides.h() + w * ostrides.w()] =
+          idata[h * ostrides.h() + w * ostrides.w()];
+    }
+  }
+}
+
+// TODO(jie): fallback to tensorflow!!
+void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
+                   TRT_ShapedWeights* oweights) {
+  int c = iweights.shape_.d[0];
+  int k = iweights.shape_.d[1];
+  oweights->shape_.d[0] = k;
+  oweights->shape_.d[1] = c;
+  nvinfer1::DimsHW istrides = {1, k};
+  nvinfer1::DimsHW ostrides = {c, 1};
+  switch (iweights.type_) {
+    case tensorflow::DataType::DT_FLOAT: {
+      Reorder2({k, c}, static_cast<float const*>(iweights.GetValues()),
+               istrides,
+               static_cast<float*>(const_cast<void*>(oweights->GetValues())),
+               ostrides);
+      break;
+    }
+    case tensorflow::DataType::DT_HALF: {
+      Reorder2(
+          {k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
+          istrides,
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
+    default:
+      LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
+                 << DataTypeString(iweights.type_);
+  }
+}
+
 void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights,
-                       TRT_ShapedWeights* oweights) {
+                       TRT_ShapedWeights* oweights, int num_groups) {
   CHECK_EQ(iweights.type_, oweights->type_);
   CHECK_EQ(iweights.size_bytes(), oweights->size_bytes());
   int r = iweights.shape_.d[0];
   int s = iweights.shape_.d[1];
-  int c = iweights.shape_.d[2];
-  int k = iweights.shape_.d[3];
-  oweights->shape_.d[0] = k;
-  oweights->shape_.d[1] = c;
+  // TRT requires GKcRS, while TF depthwise has RSCK
+  //   where c=1, C=G
+  VLOG(2) << "num_groups: " << num_groups;
+  int c = iweights.shape_.d[2] / num_groups;
+  VLOG(2) << "c" << iweights.shape_.d[2] << " then " << c;
+  int k = iweights.shape_.d[3] * num_groups;
+  VLOG(2) << "k" << iweights.shape_.d[3] << " then " << k;
+  oweights->shape_.d[0] = k / num_groups;
+  oweights->shape_.d[1] = c * num_groups;
   oweights->shape_.d[2] = r;
   oweights->shape_.d[3] = s;
   nvinfer1::DimsNCHW istrides = {1, k, s * k * c, c * k};
   nvinfer1::DimsNCHW ostrides = {c * r * s, r * s, s, 1};
   switch (iweights.type_) {
-    case tensorflow::DataType::DT_FLOAT:
+    case tensorflow::DataType::DT_FLOAT: {
       Reorder4({k, c, r, s}, static_cast<float const*>(iweights.GetValues()),
                istrides,
                static_cast<float*>(const_cast<void*>(oweights->GetValues())),
                ostrides);
       break;
+    }
+    case tensorflow::DataType::DT_HALF: {
+      Reorder4(
+          {k, c, r, s}, static_cast<Eigen::half const*>(iweights.GetValues()),
+          istrides,
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
+
     default:
-      LOG(FATAL) << "!!!!!!!!!!!!!!!!!!!!!!!!broke!!!!!!!!!!!!";
+      LOG(FATAL) << "Unsupported type, expected fp32 or fp16 but got "
+                 << DataTypeString(iweights.type_);
   }
 }
 
@@ -323,12 +415,11 @@ inline std::shared_ptr<T> infer_object(T* obj) {
   return std::shared_ptr<T>(obj, InferDeleter());
 }
 
-// Logger for GIE info/warning/errors
 class Converter;
 
 using OpConverter =
     std::function<tensorflow::Status(Converter&, const tensorflow::NodeDef&,
-                                     std::vector<TRT_TensorOrWeights> const&,
+                                     const std::vector<TRT_TensorOrWeights>&,
                                      std::vector<TRT_TensorOrWeights>*)>;
 
 class Converter {
@@ -336,34 +427,57 @@ class Converter {
   std::unordered_map<string, OpConverter> op_registry_;
   nvinfer1::INetworkDefinition* trt_network_;
   std::list<std::vector<uint8_t>> temp_bufs_;
-
+  tensorflow::tensorrt::TRTWeightStore* weight_store_;
+  bool fp16_;
   void register_op_converters();
-
   std::vector<TRT_TensorOrWeights> get_inputs(
       const tensorflow::NodeDef& node_def) {
     std::vector<TRT_TensorOrWeights> inputs;
-    for (const auto& input_name : node_def.input()) {
-      VLOG(2) << "Retrieve input: " << input_name;
-      inputs.push_back(trt_tensors_.at(input_name));
+    for (auto const& input_name : node_def.input()) {
+      /*************************************************************************
+       * TODO(jie) handle case 1) here
+       * Normalizes the inputs and extracts associated metadata:
+       * 1) Inputs can contain a colon followed by a suffix of characters.
+       *    That suffix may be a single number (e.g. inputName:1) or several
+       *    word characters separated from a number by a colon
+       *    (e.g. inputName:foo:1). The
+       *    latter case is used to denote inputs and outputs of functions.
+       * 2) Control dependency inputs contain caret at the beginning and we
+       *    remove this and annotate the edge as a control dependency.
+       ************************************************************************/
+      string name = input_name[0] == '^' ? input_name.substr(1) : input_name;
+      auto first = name.find_first_of(':');
+      if (first != string::npos && first + 2 == name.size() &&
+          name[first + 1] == '0')
+        name.erase(first);
+
+      VLOG(2) << "retrieve input: " << name;
+      if (trt_tensors_.count(name)) {
+        inputs.push_back(trt_tensors_.at(name));
+      } else {
+        LOG(FATAL) << "input: " << name << " not availabled for node at, "
+                   << node_def.name();
+      }
     }
     return inputs;
   }
 
  public:
-  explicit Converter(nvinfer1::INetworkDefinition* trt_network)
-      : trt_network_(trt_network) {
+  explicit Converter(nvinfer1::INetworkDefinition* trt_network,
+                     tensorflow::tensorrt::TRTWeightStore* ws, bool fp16)
+      : trt_network_(trt_network), weight_store_(ws), fp16_(fp16) {
     this->register_op_converters();
   }
-
+  tensorflow::tensorrt::TRTWeightStore* weight_store() { return weight_store_; }
   TRT_ShapedWeights get_temp_weights(tensorflow::DataType type,
                                      nvinfer1::Dims shape) {
     TRT_ShapedWeights weights(type, nullptr, shape);
     // TODO(jie): check weights size_bytes. 0 means type error
-    temp_bufs_.push_back(std::vector<uint8_t>(weights.size_bytes()));
-    weights.SetValues(temp_bufs_.back().data());
+    weight_store_->store_.push_back(std::vector<uint8_t>(weights.size_bytes()));
+    weights.SetValues(weight_store_->store_.back().data());
     return weights;
   }
-
+  bool isFP16() { return fp16_; };
   TRT_ShapedWeights get_temp_weights_like(const TRT_ShapedWeights& weights) {
     return this->get_temp_weights(weights.type_, weights.shape_);
   }
@@ -382,7 +496,7 @@ class Converter {
       TRT_TensorOrWeights output = outputs.at(i);
       // TODO(jie): tf protobuf seems to be omitting the :0 suffix
       string output_name = node_def.name();
-      if (i != 0) output_name = output_name + ":" + std::to_string(i);
+      if (i != 0) output_name = StrCat(output_name, ":", i);
       if (output.is_tensor()) {
         output.tensor()->setName(output_name.c_str());
       }
@@ -448,7 +562,7 @@ struct LambdaFactory {
     switch (op) {
       case OP_CATEGORY::RSQRT: {
         VLOG(2) << "RSQRT GETS DONE";
-        return [](T t) -> T { return 1.0 / std::sqrt(t); };
+        return [](T t) -> T { return 1.0 / sqrt(t); };
       }
       case OP_CATEGORY::NEG:
         return [](T t) -> T { return -t; };
@@ -534,6 +648,22 @@ struct LambdaFactory {
   }
 };
 
+template <>
+std::function<Eigen::half(Eigen::half)> LambdaFactory::unary<Eigen::half>() {
+  switch (op) {
+    case OP_CATEGORY::RSQRT: {
+      VLOG(2) << "RSQRT GETS DONE";
+      return [](Eigen::half t) -> Eigen::half {
+        return Eigen::half(1.0 / sqrt(float(t)));
+      };
+    }
+    case OP_CATEGORY::NEG:
+      return [](Eigen::half t) -> Eigen::half { return -t; };
+    default:
+      VLOG(2) << "Not supported op for unary: " << static_cast<int>(op);
+      return nullptr;
+  }
+}
 tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
                                 TRT_ShapedWeights* oweights,
                                 LambdaFactory unary_op) {
@@ -545,6 +675,14 @@ tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
       std::transform(inp, inp + iweights.count(), oup, unary_op.unary<float>());
       break;
     }
+    case tensorflow::DataType::DT_HALF: {
+      auto inp = static_cast<Eigen::half const*>(iweights.GetValues());
+      auto oup =
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
+      std::transform(inp, inp + iweights.count(), oup,
+                     unary_op.unary<Eigen::half>());
+      break;
+    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -588,6 +726,32 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
       }
       break;
     }
+    case tensorflow::DataType::DT_HALF: {
+      auto inp_l = static_cast<const Eigen::half*>(iweights_l.GetValues());
+      auto inp_r = static_cast<const Eigen::half*>(iweights_r.GetValues());
+      auto oup =
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
+
+      if (iweights_l.count() != iweights_r.count()) {
+        // We only supports broadcast of RankZero
+        if (iweights_l.count() == 1) {
+          VLOG(2) << "I bet it is not working!" << (*inp_l);
+          std::transform(inp_r, inp_r + iweights_r.count(), oup,
+                         binary_op.broadcast_l<Eigen::half>(*inp_l));
+        } else if (iweights_r.count() == 1) {
+          VLOG(2) << "I bet it is not working!" << (*inp_r);
+          std::transform(inp_l, inp_l + iweights_l.count(), oup,
+                         binary_op.broadcast_r<Eigen::half>(*inp_r));
+        } else {
+          return tensorflow::errors::Unimplemented(
+              "Binary op with non-rankZero broadcast not supported");
+        }
+      } else {
+        std::transform(inp_l, inp_l + iweights_l.count(), inp_r, oup,
+                       binary_op.binary<Eigen::half>());
+      }
+      break;
+    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -599,7 +763,7 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
 
 tensorflow::Status ConstantFoldUnary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input = inputs.at(0).weights();
 
@@ -613,13 +777,12 @@ tensorflow::Status ConstantFoldUnary(
   CHECK_EQ(weights_input.type_,
            TFAttrs(node_def).get<tensorflow::DataType>("T"));
 
-  // Maybe I should do a switch
   LambdaFactory unary_op;
   if (node_def.op() == "Rsqrt") {
     // Compute rsqrt
     unary_op.op = LambdaFactory::OP_CATEGORY::RSQRT;
     auto ret = UnaryCompute(weights_input, &weights_output, unary_op);
-    // PAss the output
+    // Pass the output
     if (ret == tensorflow::Status::OK()) {
       outputs->push_back(TRT_TensorOrWeights(weights_output));
     }
@@ -631,11 +794,11 @@ tensorflow::Status ConstantFoldUnary(
 }
 
 // TODO(jie,ben) broadcast is needed yet not implemented
-// Let's get the simple stuff working first. Maybe we should fall bakc to TF
+// Let's get the simple stuff working first. Maybe we should fall back to TF
 //   approach for constant folding
 tensorflow::Status ConstantFoldBinary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input_l = inputs.at(0).weights();
   TRT_ShapedWeights weights_input_r = inputs.at(1).weights();
@@ -648,12 +811,12 @@ tensorflow::Status ConstantFoldBinary(
         "Binary op implicit broadcast not supported: " + node_def.op());
 
   // TODO(jie): constant fold should really fall back to TF.
-  int nb_dims = weights_input_l.shape_.nbDims;
+  int num_dims = weights_input_l.shape_.nbDims;
   nvinfer1::Dims output_shape;
-  output_shape.nbDims = nb_dims;
-  VLOG(2) << "nb_dims: " << nb_dims
+  output_shape.nbDims = num_dims;
+  VLOG(2) << "nb_dims: " << num_dims
           << ", the other: " << weights_input_r.shape_.nbDims;
-  for (int i = 0; i < nb_dims; i++) {
+  for (int i = 0; i < num_dims; i++) {
     if (weights_input_l.shape_.d[i] == weights_input_r.shape_.d[i]) {
       output_shape.d[i] = weights_input_l.shape_.d[i];
     } else if (weights_input_l.shape_.d[i] == 1 ||
@@ -678,7 +841,6 @@ tensorflow::Status ConstantFoldBinary(
   // Allocate output weights
   TRT_ShapedWeights weights_output = ctx.get_temp_weights(dtype, output_shape);
 
-  // Maybe I should do a switch
   LambdaFactory binary_op;
   if (node_def.op() == "Sub") {
     binary_op.op = LambdaFactory::OP_CATEGORY::SUB;
@@ -712,48 +874,90 @@ tensorflow::Status BinaryTensorOpWeight(
   // Maybe this part has to be moved into the block of rsqrt later
 
   // Check type consistency
-  auto dtype = TFAttrs(node_def).get<nvinfer1::DataType>("T");
-  CHECK_EQ_TYPE(tensor->getType(), dtype);  // Cast to int for error messages
   nvinfer1::DataType ttype;
   TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
-  CHECK_EQ_TYPE(ttype, dtype);  // Cast to int for error message
 
   // Check scale mode
   auto dims_w = weights.shape_;
   auto dims_t = tensor->getDimensions();
 
-  // Default to channel-wise
+  // default to element-wise
   auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
 
+  // TODO(jie): maybe use a permuatation instead to support more cases;
+  bool permutation_flag = false;
+
   if (weights.count() == 1) {
     VLOG(2) << "UNIFORM";
     scale_mode = nvinfer1::ScaleMode::kUNIFORM;
   } else {
-    // No broadcasting on Batch dimension;
-    assert(dims_w.d[0] == 1);
-
-    // Broadcasting on Channel dimension only allowed in kUNIFORM
-    assert(dims_w.d[1] == dims_t.d[0]);
-    assert(dims_w.nbDims == dims_t.nbDims);
-
-    // Default is element;
-    for (int i = 2; i < dims_w.nbDims; i++) {
-      if (dims_w.d[i] != dims_t.d[i - 1]) {
-        scale_mode = nvinfer1::ScaleMode::kCHANNEL;
-        break;
+    // no broadcasting on Batch dimension;
+    VLOG(2) << "WEIGHTS DIM: " << dims_w.nbDims
+            << " tensor DIM: " << dims_t.nbDims;
+    if (dims_w.nbDims == dims_t.nbDims + 1) {
+      if (dims_w.d[0] == 1) {
+        for (int i = 1; i < dims_w.nbDims; i++) {
+          dims_w.d[i - 1] = dims_w.d[i];
+        }
+        dims_w.nbDims--;
+      } else {
+        return tensorflow::errors::InvalidArgument(
+            "Binary op cannot operate on batch, " + node_def.name());
       }
     }
-    if (scale_mode == nvinfer1::ScaleMode::kELEMENTWISE) {
+
+    if (dims_w.nbDims == dims_t.nbDims && dims_w.d[0] == dims_t.d[0]) {
       scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
-      for (int i = 2; i < dims_w.nbDims; i++) {
-        if (dims_w.d[i] != 1)
-          return tensorflow::errors::InvalidArgument(
-              "Weight shape not compatible at, " + node_def.name());
+      // default is element;
+      for (int i = 1; i < dims_w.nbDims; i++) {
+        if (dims_w.d[i] != dims_t.d[i]) {
+          // if dimension does not match, switch back to channel;
+          VLOG(2) << "channel";
+          scale_mode = nvinfer1::ScaleMode::kCHANNEL;
+          break;
+        }
+      }
+      // if channel as candidate, validate it
+      if (scale_mode == nvinfer1::ScaleMode::kCHANNEL) {
+        for (int i = 1; i < dims_w.nbDims; i++) {
+          if (dims_w.d[i] != 1)
+            return tensorflow::errors::InvalidArgument(
+                "Weight shape not compatible at, " + node_def.name());
+        }
+      } else {
+        VLOG(2) << "elementwise";
+      }
+    } else if (dims_w.nbDims == 1 &&
+               dims_w.d[0] == dims_t.d[dims_t.nbDims - 1]) {
+      // channel wise and broadcast required;
+      permutation_flag = true;
+      scale_mode = nvinfer1::ScaleMode::kCHANNEL;
+    } else {
+      return tensorflow::errors::InvalidArgument(
+          "Weight shape not compatible at, " + node_def.name());
+    }
+  }
+
+  // transpose last dimension
+  std::vector<int> permutation(dims_t.nbDims + 1);
+  if (permutation_flag) {
+    if (scale_mode == nvinfer1::ScaleMode::kCHANNEL && dims_t.nbDims > 1) {
+      // we swap the last dimension into channel for trt.
+      // because of tensorflow default broadcasting rules.
+      for (int i = 0; i < static_cast<int>(permutation.size()); i++) {
+        permutation[i] = i;
       }
+      permutation[1] = dims_t.nbDims;
+      permutation[dims_t.nbDims] = 1;
+      tensor = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor),
+                                   permutation);
+    } else {
+      return tensorflow::errors::InvalidArgument(
+          "Transpose cannot be applied, " + node_def.name());
     }
   }
 
-  // Prepare weights
+  // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
   TRT_ShapedWeights power_weights(weights.type_);
@@ -779,88 +983,26 @@ tensorflow::Status BinaryTensorOpWeight(
       scale_weights, power_weights);
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // transpose back dimension
+  if (permutation_flag) {
+    output_tensor = ctx.TransposeTensor(output_tensor, permutation);
+  }
 
   // Pass the output
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status BinaryTensorOpTensor(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
-      {"Add", nvinfer1::ElementWiseOperation::kSUM},
-      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
-      // {"max", nvinfer1::ElementWiseOperation::kMAX},
-      // {"min", nvinfer1::ElementWiseOperation::kMIN},
-      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
-      {"Div", nvinfer1::ElementWiseOperation::kDIV},
-  };
-
-  // FIXME assume type matches input weights
-  // Get trt type & shape
-  TFAttrs attrs(node_def);
-  // Maybe this part has to be moved into the block of rsqrt later
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
-
-  // Check type consistency
-  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
-  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
-  auto op_pair = ops.find(node_def.op());
-  if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
-
-  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
-      *const_cast<nvinfer1::ITensor*>(tensor_l),
-      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
-
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-
-  // Pass the output
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
+enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 
-tensorflow::Status ConvertPlaceholder(
+tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  VLOG(2) << "Placeholder should have been replace already";
-  return tensorflow::errors::Unimplemented(", cannot convert Placeholder op");
-  // OK this make sense since we are supposed to replace it with input
-  TFAttrs attrs(node_def);
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
-  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
-
-  dims.nbDims--;
-  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
-
-  nvinfer1::ITensor* output =
-      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
-  if (!output) {
-    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
-  }
-  outputs->push_back(TRT_TensorOrWeights(output));
-  return tensorflow::Status::OK();
-}
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs,
+    int group  // group ==0 specifies depthwise conv
+) {
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
-tensorflow::Status ConvertConv2D(Converter& ctx,
-                                 const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
-                                 std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  // TODO(jie): handle NHWC/NCHW transpose;
-  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
-  ReorderRSCKToKCRS(weights_rsck, &weights);
-  TRT_ShapedWeights biases(weights.type_);
-  int noutput = weights.shape_.d[0];
-  nvinfer1::DimsHW kernel_size;
-  kernel_size.h() = weights.shape_.d[2];
-  kernel_size.w() = weights.shape_.d[3];
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -874,11 +1016,31 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
     // TODO(jie): transpose it
   }
 
+  // tensor after transpose (NCHW)
+  auto tensor_dim = tensor->getDimensions();
+
+  int num_groups = group;
+  if (num_groups == 0)  // depthwise convolution
+    num_groups = tensor_dim.d[0];
+  VLOG(2) << "groups count: " << num_groups;
+
+  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
+  ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
+  TRT_ShapedWeights biases(weights.type_);
+  int noutput = weights.shape_.d[0] * num_groups;
+  nvinfer1::DimsHW kernel_size;
+  kernel_size.h() = weights.shape_.d[2];
+  kernel_size.w() = weights.shape_.d[3];
+  VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w();
+
   // TODO(jie): stride. (NHWC/NCHW)
   auto tf_stride = attrs.get<std::vector<int>>("strides");
+  VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index;
+  VLOG(2) << "stride!!!: " << tf_stride[0] << tf_stride[1] << tf_stride[2]
+          << tf_stride[3];
   nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
 
-  auto tensor_dim = tensor->getDimensions();
   std::vector<std::pair<int, int>> padding;
   // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
@@ -919,10 +1081,11 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
+  layer->setNbGroups(num_groups);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
   auto dim_after = output_tensor->getDimensions();
-  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1]
+  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1] << ", "
           << dim_after.d[2] << ", " << dim_after.d[3];
 
   if (data_format == "NHWC") {
@@ -935,11 +1098,101 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertConv2DHelper(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs, ConvolutionType type) {
+  switch (type) {
+    case ConvolutionType::DEFAULT:
+      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 1);
+    case ConvolutionType::DEPTHWISE_CONV:
+      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 0);
+  }
+  return tensorflow::errors::Unimplemented("unsupported convolution type at, " +
+                                           node_def.name());
+}
+
+tensorflow::Status BinaryTensorOpTensor(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
+      {"Add", nvinfer1::ElementWiseOperation::kSUM},
+      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
+      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
+      {"Div", nvinfer1::ElementWiseOperation::kDIV},
+  };
+
+  // FIXME assume type matches input weights
+  // get trt type & shape
+  TFAttrs attrs(node_def);
+  // maybe this part has to be moved into the block of rsqrt later
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+
+  // check type consistency
+  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
+  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
+  auto op_pair = ops.find(node_def.op());
+  if (op_pair == ops.end())
+    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
+                                             " not supported at: " +
+                                             node_def.name());
+
+  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
+      *const_cast<nvinfer1::ITensor*>(tensor_l),
+      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  // pass the output
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertPlaceholder(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  VLOG(2) << "Placeholder should have been replace already";
+  return tensorflow::errors::Unimplemented("cannot convert Placeholder op");
+  // OK this make sense since we are supposed to replace it with input
+  TFAttrs attrs(node_def);
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
+  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
+
+  dims.nbDims--;
+  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
+
+  nvinfer1::ITensor* output =
+      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
+  if (!output) {
+    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
+  }
+  outputs->push_back(TRT_TensorOrWeights(output));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertConv2D(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
+                             ConvolutionType::DEFAULT);
+}
+
+tensorflow::Status ConvertConv2DDepthwise(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
+                             ConvolutionType::DEPTHWISE_CONV);
+}
+
 tensorflow::Status ConvertPool(Converter& ctx,
                                const tensorflow::NodeDef& node_def,
-                               std::vector<TRT_TensorOrWeights> const& inputs,
+                               const std::vector<TRT_TensorOrWeights>& inputs,
                                std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -957,6 +1210,8 @@ tensorflow::Status ConvertPool(Converter& ctx,
   // TODO(jie): support other pooling type
   if (node_def.op() == "MaxPool")
     type = nvinfer1::PoolingType::kMAX;
+  else if (node_def.op() == "AvgPool")
+    type = nvinfer1::PoolingType::kAVERAGE;
   else
     return tensorflow::errors::Unimplemented("Only supports Max pool");
 
@@ -1019,9 +1274,9 @@ tensorflow::Status ConvertPool(Converter& ctx,
 
 tensorflow::Status ConvertActivation(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   nvinfer1::IActivationLayer* layer = ctx.network()->addActivation(
       *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ActivationType::kRELU);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
@@ -1031,14 +1286,14 @@ tensorflow::Status ConvertActivation(
 
 tensorflow::Status ConvertScale(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
     return tensorflow::errors::Unimplemented(
         "Only supports tensor op weight for now, at " + node_def.name());
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
@@ -1055,16 +1310,33 @@ tensorflow::Status ConvertScale(Converter& ctx,
   } else {
     VLOG(2) << "NCHW !!!!";
   }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      weights, empty_weights, empty_weights);
 
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  if (data_format == "NHWC") {
-    // TODO(jie): transpose it back!
-    output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1});
-  } else {
-    VLOG(2) << "NCHW !!!!";
+  auto dims = tensor->getDimensions();
+  VLOG(2) << "tensor dimensions: " << dims.nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    VLOG(2) << "i: " << dims.d[i];
+  }
+  dims = weights.shape_;
+  VLOG(2) << "tensor dimensions: " << dims.nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    VLOG(2) << "i: " << dims.d[i];
+  }
+
+  nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
+  if (weights.shape_.d[0] == 1) {
+    mode = nvinfer1::ScaleMode::kUNIFORM;
+  }
+
+  nvinfer1::IScaleLayer* layer =
+      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
+                              weights, empty_weights, empty_weights);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  if (data_format == "NHWC") {
+    // TODO(jie): transpose it back!
+    output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1});
+  } else {
+    VLOG(2) << "NCHW !!!!";
   }
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -1072,7 +1344,7 @@ tensorflow::Status ConvertScale(Converter& ctx,
 
 tensorflow::Status ConvertConst(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   const auto& weights_tensor = node_def.attr().at("value").tensor();
 
@@ -1091,20 +1363,144 @@ tensorflow::Status ConvertConst(Converter& ctx,
     VLOG(2) << "SCALAR!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
     if (tensor.dims() > 0) {
-      VLOG(2) << "Dimensions: " << tensor.dims();
-      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
-                                  GetTensorShape(tensor));
+      VLOG(2) << "dimensions: " << tensor.dims();
+      VLOG(2) << "size: " << weights_tensor.float_val_size();
+      scalar_shape = GetTensorShape(tensor);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        VLOG(2) << scalar_shape.d[i];
+      if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size()) {
+        if (weights_tensor.float_val_size() == 1 ||
+            scalar_shape.d[0] == weights_tensor.float_val_size()) {
+          scalar_shape.nbDims = 1;
+          // no dimension provided. flatten it
+          scalar_shape.d[0] = weights_tensor.float_val_size();
+          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+        } else {
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
+        }
+      }
     } else {
       VLOG(2) << "Dimensions: " << tensor.dims();
       scalar_shape.nbDims = 1;
-      scalar_shape.d[0] = 1;
+      // no dimension provided. flatten it
+      scalar_shape.d[0] = weights_tensor.float_val_size();
       scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
       for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
         scalar_shape.d[i] = 0;
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
-      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
-                                  scalar_shape);
+    }
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      auto half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      half_tensor.device(defd) =
+          tensor.flat<float>().template cast<Eigen::half>();
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<float> tensor_data(
+          weights_tensor.float_val().begin(),
+          weights_tensor.float_val()
+              .end());  //  make a local copy first to flatten
+      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    }
+  } else if (!weights_tensor.int_val().empty()) {
+    VLOG(2) << "int!!!" << node_def.name();
+    nvinfer1::Dims scalar_shape;
+    if (tensor.dims() > 0) {
+      VLOG(2) << "dimensions: " << tensor.dims();
+      scalar_shape = GetTensorShape(tensor);
+      if (GetShapeSize(scalar_shape) != weights_tensor.int_val_size()) {
+        if (weights_tensor.int_val_size() == 1 ||
+            scalar_shape.d[0] == weights_tensor.int_val_size()) {
+          scalar_shape.nbDims = 1;
+          // no dimension provided. flatten it
+          scalar_shape.d[0] = weights_tensor.int_val_size();
+          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+        } else {
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
+        }
+      }
+    } else {
+      VLOG(2) << "dimensions: " << tensor.dims();
+      scalar_shape.nbDims = 1;
+      // no dimension provided. flatten it
+      scalar_shape.d[0] = weights_tensor.int_val_size();
+      scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+      for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
+        scalar_shape.d[i] = 0;
+        scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
+      }
+    }
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      switch (dtype) {
+        case (tensorflow::DT_INT32): {
+          half_tensor.device(defd) =
+              tensor.flat<int32>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT16): {
+          half_tensor.device(defd) =
+              tensor.flat<int16>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT8): {
+          half_tensor.device(defd) =
+              tensor.flat<int8>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_UINT8): {
+          half_tensor.device(defd) =
+              tensor.flat<uint8>().template cast<Eigen::half>();
+          break;
+        }
+        default:
+          return tensorflow::errors::InvalidArgument(
+              "Datatype " + tensorflow::DataTypeString(dtype) +
+              " for FP16 conversion");
+          break;
+      };
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+      len_data = std::max(len_data, len_tensor);
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<int32> tensor_data(
+          weights_tensor.int_val().begin(),
+          weights_tensor.int_val()
+              .end());  //  make a local copy first to flatten
+                        //  doesn't have to be contigous
+      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
     }
   } else if (!weights_tensor.tensor_content().empty()) {
     VLOG(2) << "TENSOR!!!" << node_def.name();
@@ -1130,7 +1526,7 @@ tensorflow::Status ConvertConst(Converter& ctx,
 
 tensorflow::Status ConvertIdentity(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   outputs->push_back(inputs.at(0));
   return tensorflow::Status::OK();
@@ -1138,7 +1534,7 @@ tensorflow::Status ConvertIdentity(
 
 tensorflow::Status ConvertBinary(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 std::vector<TRT_TensorOrWeights> const& inputs,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2)
     return tensorflow::errors::FailedPrecondition(
@@ -1165,7 +1561,7 @@ tensorflow::Status ConvertBinary(Converter& ctx,
 
 tensorflow::Status ConvertUnary(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 1)
     return tensorflow::errors::FailedPrecondition(
@@ -1183,7 +1579,7 @@ tensorflow::Status ConvertUnary(Converter& ctx,
 
 tensorflow::Status ConvertReduce(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 std::vector<TRT_TensorOrWeights> const& inputs,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1191,7 +1587,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1229,6 +1625,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
       return tensorflow::errors::InvalidArgument("TRT cannot reduce at 0, at" +
                                                  node_def.name());
     if (index_list_data[i] == 1) permuted_index = 1;
+
     idx_set.emplace(index_list_data[i]);
   }
 
@@ -1236,7 +1633,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
   nvinfer1::DimsHW pool_kernel;
   if (permuted_index == 1) {
     for (int i = 2; i < nb_dims; i++) {
-      if (idx_set.count(i)) {
+      if (idx_set.count(i) == 0) {
         permuted_index = i;
         break;
       }
@@ -1271,12 +1668,13 @@ tensorflow::Status ConvertReduce(Converter& ctx,
     output_tensor = ctx.TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), permutation_order);
   }
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
 tensorflow::Status ConvertPad(Converter& ctx,
                               const tensorflow::NodeDef& node_def,
-                              std::vector<TRT_TensorOrWeights> const& inputs,
+                              const std::vector<TRT_TensorOrWeights>& inputs,
                               std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1284,7 +1682,7 @@ tensorflow::Status ConvertPad(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1371,19 +1769,287 @@ tensorflow::Status ConvertPad(Converter& ctx,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertConcat(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  // not including the last input (axis) here
+  int input_size = static_cast<int>(inputs.size()) - 1;
+
+  if (!inputs.at(0).is_tensor())
+    return tensorflow::errors::InvalidArgument(
+        "Concat in TRT support only Tensor input, at " + node_def.name());
+
+  // We are retrieving the axis
+  TRT_ShapedWeights axis = inputs.at(input_size).weights();
+
+  TFAttrs attrs(node_def);
+  // auto attr_size = attrs.at("N")->i();
+  // auto data_type = attrs.get<nvinfer1::DataType>("T");
+  auto index_type = attrs.get<tensorflow::DataType>("Tidx");
+
+  // TODO(jie): handle data type
+  // Only expect to handle INT32 as index attributes for now
+  if (index_type != tensorflow::DataType::DT_INT32)
+    return tensorflow::errors::Unimplemented(
+        "Tidx supports only DT_INT32, at " + node_def.name());
+
+  int index = *(static_cast<int*>(const_cast<void*>(axis.GetValues())));
+
+  // TODO(jie): early termination with no-op (attr_size==1)
+
+  auto dim = inputs.at(0).tensor()->getDimensions();
+  // dimension check
+  if (index > dim.nbDims + 1)
+    return tensorflow::errors::InvalidArgument(
+        "Concatenate on axis out of dimension range, at " + node_def.name());
+
+  if (index == 0)
+    return tensorflow::errors::InvalidArgument(
+        "Concatenate on batch dimension not supported, at " + node_def.name());
+
+  // incase we need permutation;
+  std::vector<int> permutation_order(dim.nbDims + 1);
+
+  for (int i = 0; i < dim.nbDims + 1; i++) permutation_order[i] = i;
+
+  if (index != 1) {
+    permutation_order[1] = index - 1;
+    permutation_order[index - 1] = 1;
+  }
+
+  std::vector<nvinfer1::ITensor const*> inputs_vec;
+  // Shap chack (all input tensor should have same shape)
+  // starting from 0 since we are probably also doing transpose here;
+  for (int i = 0; i < input_size; i++) {
+    auto tensor_i = inputs.at(i).tensor();
+    auto dim_i = tensor_i->getDimensions();
+    if (dim_i.nbDims != dim.nbDims)
+      return tensorflow::errors::InvalidArgument(
+          "Concatenate receives inputs with inconsistent dimensions, at " +
+          node_def.name());
+
+    for (int j = 0; j < dim.nbDims; j++) {
+      // check dimension consistency on non-concatenate axis
+      if (j != index - 1 && dim_i.d[j] != dim.d[j])
+        return tensorflow::errors::InvalidArgument(
+            "Concatenate receives inputs with inconsistent shape, at" +
+            node_def.name());
+    }
+
+    // TRT does concatenation only on channel!
+    if (index != 1)
+      tensor_i = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor_i),
+                                     permutation_order);
+
+    inputs_vec.push_back(tensor_i);
+  }
+
+  // nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  nvinfer1::IConcatenationLayer* layer = ctx.network()->addConcatenation(
+      const_cast<nvinfer1::ITensor* const*>(inputs_vec.data()),
+      inputs_vec.size());
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  if (index != 1) {
+    output_tensor = ctx.TransposeTensor(output_tensor, permutation_order);
+  }
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertFusedBatchNorm(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  TFAttrs attrs(node_def);
+  float epsilon = attrs.get<float>("epsilon");
+  auto data_format = attrs.get<string>("data_format");
+  if (data_format != "NCHW") {
+    return tensorflow::errors::Unimplemented(
+        "only data_format=NCHW is supported, at " + node_def.name());
+  }
+  bool is_training = attrs.get<bool>("is_training");
+  if (is_training) {
+    return tensorflow::errors::Unimplemented(
+        "only is_training=false is supported, at " + node_def.name());
+  }
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
+  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
+  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
+  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
+  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
+  TRT_ShapedWeights combined_scale_weights =
+      ctx.get_temp_weights_like(scale_weights);
+  TRT_ShapedWeights combined_offset_weights =
+      ctx.get_temp_weights_like(offset_weights);
+  size_t nweight = scale_weights.count();
+  if ((scale_weights.type_ == offset_weights.type_) &&
+      (mean_weights.type_ == variance_weights.type_) &&
+      (scale_weights.type_ == variance_weights.type_)) {
+    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
+        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+      return tensorflow::errors::Unimplemented(
+          "only float32 or float16 weight data type is supported, for node " +
+          node_def.name() + " got " +
+          tensorflow::DataTypeString(scale_weights.type_));
+    }
+    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
+        float offset =
+            (static_cast<float const*>(offset_weights.GetValues()))[i];
+        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
+        float variance =
+            (static_cast<float const*>(variance_weights.GetValues()))[i];
+        float& combined_scale_ref = const_cast<float*>(
+            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
+        float& combined_offset_ref = const_cast<float*>(
+            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
+        combined_scale_ref = scale / sqrtf(variance + epsilon);
+        combined_offset_ref = offset - mean * combined_scale_ref;
+      }
+    } else {
+      const Eigen::half* scale_vals =
+          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
+      const Eigen::half* off_vals =
+          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
+      const Eigen::half* mean_vals =
+          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
+      const Eigen::half* variance_vals =
+          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
+      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale(scale_vals[i]);
+        float offset(off_vals[i]);
+        float mean(mean_vals[i]);
+        float variance(variance_vals[i]);
+        float combined_scale_ref = scale / sqrtf(variance + epsilon);
+        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
+        float combined_offset_ref = offset - mean * combined_scale_ref;
+        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+      }
+    }
+  }
+  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
+      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
+      combined_offset_weights.GetWeightsForTRT(),
+      combined_scale_weights.GetWeightsForTRT(),
+      dummy_power_weights.GetWeightsForTRT());
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertMatMul(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+
+  // TODO(jie): transpose!
+  TFAttrs attrs(node_def);
+
+  TRT_ShapedWeights weights_ck = inputs.at(1).weights();
+  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_ck);
+  ReorderCKtoKC(weights_ck, &weights);
+  TRT_ShapedWeights biases(weights.type_);
+
+  int noutput = weights.shape_.d[0];
+
+  nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected(
+      *const_cast<nvinfer1::ITensor*>(tensor), noutput, weights, biases);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertReshape(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
+      !inputs.at(1).is_weights())
+    return tensorflow::errors::InvalidArgument(
+        "Input expects tensor and weights, at" + node_def.name());
+
+  // implement tensor binaryOp weight [channel wise] for now;
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  auto dims = tensor->getDimensions();
+  // restore implicit batch dimension
+
+  TRT_ShapedWeights shape = inputs.at(1).weights();
+
+  TFAttrs attrs(node_def);
+
+  auto padding_type = attrs.get<tensorflow::DataType>("Tshape");
+
+  if (shape.shape_.nbDims != 1)
+    return tensorflow::errors::InvalidArgument(
+        "reshape new shape is not 1 dimensional, at " + node_def.name());
+
+  // Only expect to handle INT32 as attributes for now
+  if (padding_type != tensorflow::DataType::DT_INT32)
+    return tensorflow::errors::Unimplemented(
+        "reshape new shape supports only DT_INT32, at " + node_def.name());
+
+  auto shape_data = static_cast<int*>(const_cast<void*>(shape.GetValues()));
+
+  if (shape_data[0] != -1)
+    return tensorflow::errors::InvalidArgument(
+        "reshape new shape first dimension is not -1, at " + node_def.name());
+
+  auto shape_num_dims = shape.shape_.d[0];
+  VLOG(2) << "shape dimensions: " << shape_num_dims;
+  int volume_w = 1;
+  for (int i = 1; i < shape.shape_.d[0]; i++) volume_w *= shape_data[i];
+
+  int volume_t = 1;
+  for (int i = 0; i < dims.nbDims; i++) volume_t *= dims.d[i];
+
+  VLOG(2) << "volume: " << volume_t << " volume weights: " << volume_w;
+  if (volume_w != volume_t)
+    return tensorflow::errors::InvalidArgument(
+        "volume does not agree between tensor and new shape, at " +
+        node_def.name());
+
+  nvinfer1::IShuffleLayer* layer =
+      ctx.network()->addShuffle(*const_cast<nvinfer1::ITensor*>(tensor));
+
+  nvinfer1::Dims reshape_dims;
+  VLOG(2) << "new dimension: " << shape_num_dims - 1;
+  reshape_dims.nbDims = shape_num_dims - 1;
+  for (int32_t i = 0; i < reshape_dims.nbDims; ++i) {
+    reshape_dims.d[i] = shape_data[i + 1];
+  }
+  layer->setReshapeDimensions(reshape_dims);
+  VLOG(2) << "new dimension: " << shape_num_dims - 1;
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  auto dims_output = output_tensor->getDimensions();
+  VLOG(2) << "output tensor dimension:" << dims_output.nbDims;
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
 void Converter::register_op_converters() {
   // vgg_16 slim implementation
   op_registry_["Placeholder"] = ConvertPlaceholder;
   op_registry_["Conv2D"] = ConvertConv2D;
+  op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
+  op_registry_["AvgPool"] = ConvertPool;
   // This could be really handled as ConvertBinary
   op_registry_["BiasAdd"] = ConvertScale;
   op_registry_["Const"] = ConvertConst;
-  // op_registry_["MatMul"] = ConvertFullyConnected;  // Not used in vgg
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
-  // op_registry_["AvgPool"] = ConvertPool;
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -1393,26 +2059,364 @@ void Converter::register_op_converters() {
   op_registry_["Mean"] = ConvertReduce;
   op_registry_["Pad"] = ConvertPad;
   // TODO(ben,jie): Add more ops
+
+  op_registry_["ConcatV2"] = ConvertConcat;
+  op_registry_["MatMul"] = ConvertMatMul;
+  op_registry_["Reshape"] = ConvertReshape;
+  op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm;
+  op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm;
 }
 
 }  // namespace
+tensorflow::Status GetTensorRTGraph(tensorrt::convert::SubGraphParams& s) {
+  return tensorflow::errors::Unimplemented("Not implemented yet");
+}
+tensorflow::Status ConvertCalibrationNodeToEngineNode(
+    tensorflow::Graph& graph, tensorflow::Node* c_node) {
+  const auto ndef = c_node->def();
+
+  TFAttrs attrs(ndef);
+  std::vector<string> segment_nodes(
+      attrs.get<std::vector<string>>("segment_nodes"));
+  std::vector<string> output_nodes(
+      attrs.get<std::vector<string>>("segment_output_names"));
+  std::vector<string> input_names(
+      attrs.get<std::vector<string>>("input_names"));
+  string res_name = attrs.get<string>("resource_name");
+  VLOG(1) << "Node name " << c_node->name() << " res_name " << res_name;
+  string engine_name = "my_trt_op";
+  {
+    const auto node_id = tensorflow::str_util::Split(res_name, "_");
+    engine_name += node_id.back();
+  }
+  std::map<string, tensorflow::Node*> node_maps;
+
+  for (auto n : graph.op_nodes()) {
+    node_maps.insert({n->name(), n});
+  }
+  VLOG(1) << "Output Nodes:";
+  std::vector<tensorflow::DataType> out_types;
+  std::vector<const tensorflow::Edge*> out_edges;
+  for (auto& i : output_nodes) {
+    auto node_port = tensorflow::str_util::Split(i, ":");
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+    auto out_node_name = node_port.at(0);
+    if (node_port.size() > 1) {
+      VLOG(1) << "Multi port output" << node_port.at(0) << " "
+              << node_port.at(1) << " size=" << node_port.size();
+    }
+    auto node_it = node_maps.find(out_node_name);
+    if (node_it != node_maps.end()) {
+      tensorflow::Node* out_node = node_it->second;
+      int port = 0;
+      if (node_port.size() == 2) {
+        port = std::strtoul(node_port.at(1).c_str(), nullptr, 10);
+        out_types.push_back(out_node->output_type(port));
+      } else {
+        out_types.push_back(out_node->output_type(0));
+      }
+      for (auto out_edge : out_node->out_edges()) {
+        if (out_edge->src_output() == port) {
+          out_edges.push_back(out_edge);
+          break;
+        }
+      }
+    } else {
+      LOG(WARNING) << " couldn't find output node " << out_node_name;
+    }
+  }
+  VLOG(1) << "Input Nodes:";
+  for (auto& i : input_names) {
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+  }
+  auto trt_rm = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto resmgr = trt_rm->getManager("TRTCalibOps");
+  tensorflow::tensorrt::TRTCalibrationResource* calib_res = nullptr;
+  auto status = resmgr->Lookup(res_name, res_name, &calib_res);
+  if (!status.ok() || !calib_res->calibrator_) {
+    return tensorflow::errors::FailedPrecondition(
+        "You must run calibration"
+        " and inference conversion in the same proces");
+  }
+
+  calib_res->calibrator_->setDone();
+  calib_res->thr_->join();
+  delete calib_res->thr_;
+  if (!calib_res->engine_) {
+    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+                  "calibration graph?";
+  }
+  auto weight_rmgr = trt_rm->getManager("WeightStore");
+  TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
+      res_name, res_name));
+  auto engine_plan = calib_res->engine_->serialize();
+  calib_res->engine_->destroy();
+  calib_res->network_->destroy();
+  calib_res->builder_->destroy();
+  calib_res->thr_ = nullptr;
+  calib_res->engine_ = nullptr;
+  calib_res->builder_ = nullptr;
+  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
+  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  for (const auto in_edge : c_node->in_edges()) {
+    auto src = in_edge->src();
+    int dest_port = in_edge->dst_input();
+    income_edges.emplace_back(src->name(), in_edge->src_output(),
+                              c_node->input_type(dest_port));
+  }
+  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
+      income_edges);
+  op_builder.Input(input_list);
+  tensorflow::NodeDef engine_node;
+  const char* engine_plan_data = static_cast<const char*>(engine_plan->data());
+  string engine_plan_string(engine_plan_data,
+                            engine_plan_data + engine_plan->size());
+  status = op_builder.Attr("serialized_engine", engine_plan_string)
+               .Attr("input_nodes", input_names)
+               .Attr("output_nodes", output_nodes)
+               .Attr("OutT", out_types)
+               .Finalize(&engine_node);
+  if (!status.ok()) {
+    LOG(ERROR) << "Engine Node creation failed";
+    return status;
+  }
+  auto trt_engine_node = graph.AddNode(engine_node, &status);
+  TF_CHECK_OK(status);
+  for (size_t i = 0; i < out_edges.size(); i++) {
+    VLOG(1) << "Connecting trt_engine_node output " << i << " with "
+            << out_edges.at(i)->dst()->name() << " port "
+            << out_edges.at(i)->dst_input();
+    TF_RETURN_IF_ERROR(graph.UpdateEdge(trt_engine_node, i,
+                                        out_edges.at(i)->dst(),
+                                        out_edges.at(i)->dst_input()));
+  }
+  VLOG(1) << "Segment nodes:";
+  for (auto& i : segment_nodes) {
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+    auto it = node_maps.find(i);
+    if (it != node_maps.end()) {
+      graph.RemoveNode(it->second);
+    }
+  }
+  graph.RemoveNode(c_node);
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
+  // Visit nodes in reverse topological order and construct the TRT network.
+
+  // Toposort
+  std::vector<tensorflow::Node*> order_vec;
+  tensorflow::GetPostOrder(s.graph, &order_vec);
+  // Select just the subgraph
+  std::list<tensorflow::Node*> order;
+  for (tensorflow::Node* node : order_vec) {
+    if (s.subgraph_node_ids.count(node->id())) {
+      order.push_front(node);  // we want topological order to contstruct the
+      // network layer by layer
+    }
+  }
+  // topological order is needed to build TRT network
+  static int static_id = 0;
+  string subgraph_name_scope;
+  if (!order.empty()) {
+    subgraph_name_scope = order.front()->name();
+  }
+  for (const tensorflow::Node* node : order) {
+    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
+  }
+  // TODO(sami,ben,jie): proper naming!
+  string calib_op_name =
+      StrCat(subgraph_name_scope, "my_trt_calib_op_", static_id);
+  string engine_name = StrCat(subgraph_name_scope, "my_trt_op", static_id);
+  static_id++;
+  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto op_rmgr = trt_rmgr->getManager("TRTCalibOps");
+  auto op_res = new tensorflow::tensorrt::TRTCalibrationResource();
+  TF_CHECK_OK(op_rmgr->Create(calib_op_name, calib_op_name, op_res));
+  op_res->logger_ = new tensorflow::tensorrt::Logger();
+  op_res->builder_ = nvinfer1::createInferBuilder(*(op_res->logger_));
+
+  if (!op_res->builder_) {
+    return tensorflow::errors::Internal(
+        "failed to create TensorRT builder object");
+  }
+
+  op_res->network_ = op_res->builder_->createNetwork();
+  if (!op_res->network_) {
+    return tensorflow::errors::Internal(
+        "failed to create TensorRT network object");
+  }
+
+  // Build the network
+  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
+  auto ws = new tensorflow::tensorrt::TRTWeightStore();
+  TF_CHECK_OK(weight_rmgr->Create(calib_op_name, calib_op_name, ws));
+  Converter converter(op_res->network_, ws, s.precision_mode == FP16MODE);
+  std::vector<string> input_names;
+  std::vector<tensorflow::DataType> input_dtypes;
+  for (const std::pair<int, int>& input : s.input_inds) {
+    VLOG(2) << "parsing input. Node id= " << input.first;
+    int node_id = input.first;
+    int output_idx = input.second;
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    auto node_name = node->name();
+    input_names.push_back(node_name);  // insert original node name without port
+    // TODO(jie): alternative :)
+    if (!s.graph_properties.HasOutputProperties(node_name))
+      return tensorflow::errors::Internal("failed to find input node: " +
+                                          node_name);
+
+    auto op_info_vec = s.graph_properties.GetOutputProperties(node_name);
+    if (static_cast<int>(op_info_vec.size()) < output_idx)
+      return tensorflow::errors::Internal(
+          "accessing output index of: ", output_idx, ", at node: ", node_name,
+          "with output entry from shape_map: ", op_info_vec.size());
+
+    auto op_info = op_info_vec.at(output_idx);
+
+    tensorflow::DataType tf_dtype = op_info.dtype();
+    input_dtypes.push_back(tf_dtype);
+
+    nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+
+    VLOG(2) << "accessing output index of: " << output_idx
+            << ", at node: " << node_name
+            << "with output entry from shape_map: " << op_info_vec.size();
+
+    // TODO(ben,jie): update TRT input format/dimension
+    nvinfer1::DimsCHW input_dim_psuedo_chw;
+    for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
+
+    for (int i = 1; i < op_info.shape().dim_size(); i++) {
+      VLOG(2) << "dimension: " << i
+              << " , size: " << op_info.shape().dim(i).size();
+      input_dim_psuedo_chw.d[i - 1] = op_info.shape().dim(i).size();
+    }
+
+    // TODO(ben,jie): proper way to restore input tensor name?
+    auto input_tensor_name = node_name;
+    if (output_idx != 0) input_tensor_name = StrCat(node_name, ":", output_idx);
+
+    nvinfer1::ITensor* input_tensor = converter.network()->addInput(
+        input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
+
+    if (!input_tensor)
+      return tensorflow::errors::InvalidArgument(
+          "Failed to create Input layer");
+    VLOG(2) << "input tensor name :" << input_tensor_name;
+
+    if (!converter.insert_input_tensor(input_tensor_name, input_tensor))
+      return tensorflow::errors::AlreadyExists(
+          "output tensor already exists for op: " + input_tensor_name);
+  }
+
+  VLOG(2) << "finished sorting";
+
+  for (const tensorflow::Node* node : order) {
+    const tensorflow::NodeDef& node_def = node->def();
+    VLOG(2) << "converting node: " << node_def.name() << " , " << node_def.op();
+    TF_RETURN_IF_ERROR(converter.convert_node(node_def));
+  }
+
+  VLOG(2) << "finished conversion";
+
+  // Gather output metadata
+  std::vector<string> output_names;
+  std::vector<tensorflow::DataType> output_dtypes;
+  int trt_engine_op_output_idx = 0;
+  for (const std::pair<int, int>& output : s.output_inds) {
+    int node_id = output.first;
+    int output_idx = output.second;
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    string op_name = node->name();
+    string tensor_name = op_name;
+
+    s.output_edge_map->insert(
+        {trt_engine_op_output_idx == 0
+             ? engine_name
+             : StrCat(engine_name, ":", trt_engine_op_output_idx),
+         {output_idx, tensor_name}});
+    trt_engine_op_output_idx++;
+    if (output_idx != 0) {
+      tensor_name = StrCat(tensor_name, ":", output_idx);
+    }
+    VLOG(1) << "output tensor name: " << tensor_name;
+    output_names.push_back(tensor_name);
+    auto tensor_or_weights = converter.get_tensor(tensor_name);
+    if (!tensor_or_weights.is_tensor()) {
+      return tensorflow::errors::InvalidArgument(
+          "Output node is weights not tensor");
+    }
+    nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
+    if (!tensor) {
+      return tensorflow::errors::NotFound("Output tensor not found: " +
+                                          tensor_name);
+    }
+    converter.network()->markOutput(*tensor);
+    tensorflow::DataType tf_dtype = node->output_type(output_idx);
+    output_dtypes.push_back(tf_dtype);
+    nvinfer1::DataType trt_dtype = nvinfer1::DataType::kFLOAT;
+    TF_RETURN_IF_ERROR(ConvertDType(tf_dtype, &trt_dtype));
+    tensor->setType(trt_dtype);
+  }
+
+  VLOG(2) << "finished output";
+
+  // Build the engine
+  op_res->builder_->setMaxBatchSize(s.max_batch_size);
+  op_res->builder_->setMaxWorkspaceSize(s.max_workspace_size_bytes);
+
+  // Build the TRT op
+  // TODO(sami,ben,jie): proper naming!
+  tensorflow::NodeDefBuilder op_builder(calib_op_name, "TRTCalibOp");
+  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  for (size_t i = 0; i < input_names.size(); ++i) {
+    int output_idx = s.input_inds.at(i).second;
+    // we wired up the input here already, it is redundant to do it again in
+    //  ConvertSubGraphToTensorRT(convert_graph.cc)
+    auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
+        input_names.at(i), output_idx, input_dtypes.at(i));
+    VLOG(1) << calib_op_name << " input " << i << " = " << input_names.at(i)
+            << ":" << output_idx
+            << " dType= " << tensorflow::DataTypeString(input_dtypes.at(i));
+    income_edges.push_back(incoming_edge);
+  }
+  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
+      income_edges);
+  op_builder.Input(input_list);
+  std::vector<string> segment_names;
+  segment_names.reserve(s.subgraph_node_ids.size());
+  for (int i : s.subgraph_node_ids) {
+    auto node = s.graph.FindNodeId(i);
+    segment_names.push_back(node->name());
+  }
+  LOG(INFO) << "finished op preparation";
+
+  auto status = op_builder.Attr("segment_nodes", segment_names)
+                    .Attr("input_names", input_names)
+                    .Attr("segment_output_names", output_names)
+                    .Attr("resource_name", calib_op_name)
+                    .Finalize(s.trt_node);
+
+  LOG(INFO) << status.ToString();
+  LOG(INFO) << "finished op building";
+
+  return tensorflow::Status::OK();
+}
 
 tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
-    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
-    const std::vector<std::pair<int, int>>& input_inds,
-    const std::vector<std::pair<int, int>>& output_inds, size_t max_batch_size,
-    size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_properties,
-    tensorflow::NodeDef* trt_node) {
+    tensorrt::convert::SubGraphParams& s) {
   // Visit nodes in reverse topological order and construct the TRT network.
 
   // Toposort
   std::vector<tensorflow::Node*> order_vec;
-  tensorflow::GetPostOrder(graph, &order_vec);
+  tensorflow::GetPostOrder(s.graph, &order_vec);
   // Select just the subgraph
   std::list<tensorflow::Node*> order;
   for (tensorflow::Node* node : order_vec) {
-    if (subgraph_node_ids.count(node->id())) {
+    if (s.subgraph_node_ids.count(node->id())) {
       // We want topological order to contstruct the
       // network layer by layer
       order.push_front(node);
@@ -1434,46 +2438,86 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
         "Failed to create TensorRT network object");
   }
 
+  string subgraph_name_scope;
+  if (!order.empty()) {
+    subgraph_name_scope = order.front()->name();
+  }
+  for (const tensorflow::Node* node : order) {
+    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
+  }
+  static int static_id = 0;
+  // TODO(sami,ben,jie): proper naming!
+  string engine_name = StrCat(subgraph_name_scope, "my_trt_op");
+  engine_name = StrCat(engine_name, static_id++);
+  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
+  auto ws = new tensorflow::tensorrt::TRTWeightStore();
+  TF_CHECK_OK(weight_rmgr->Create(engine_name, engine_name, ws));
+
   // Build the network
-  Converter converter(trt_network.get());
+  Converter converter(trt_network.get(), ws, s.precision_mode == FP16MODE);
 
   std::vector<string> input_names;
   std::vector<tensorflow::DataType> input_dtypes;
-  for (std::pair<int, int> const& input : input_inds) {
+  for (const std::pair<int, int>& input : s.input_inds) {
+    VLOG(2) << "parsing input!!!!!";
     int node_id = input.first;
     int output_idx = input.second;
-    tensorflow::Node* node = graph.FindNodeId(node_id);
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
     auto node_name = node->name();
-    input_names.push_back(node_name);  // Insert original node name without port
-    // TODO(jie): alternative :)
-    if (!graph_properties.HasOutputProperties(node_name))
-      return tensorflow::errors::Internal("Failed to find input node: " +
-                                          node_name);
+    // input_names should use the node name in the graph
+    // here it should be the input tensor name -> matching the binding
+    // insert original node name without port
+    auto tensor_name = node_name;
+    if (output_idx != 0) {
+      tensor_name = StrCat(tensor_name, ":", output_idx);
+    }
 
-    auto op_info_vec = graph_properties.GetOutputProperties(node_name);
-    if (static_cast<int>(op_info_vec.size()) < output_idx)
-      return tensorflow::errors::Internal(
-          "Accessing output index of: " + std::to_string(output_idx) +
-          ", at node: " + node_name + " with output entry from shape_map: " +
-          std::to_string(op_info_vec.size()));
+    VLOG(2) << "input name: " << node_name << " tensor_name: " << tensor_name
+            << " idx: " << output_idx;
 
-    auto op_info = op_info_vec.at(output_idx);
+    auto shape_inference_node_name = node_name;
+    auto shape_inference_output_idx = output_idx;
+    // rewire the shape inference to original node in the graph
+    if (s.output_edge_map->count(tensor_name)) {
+      shape_inference_node_name = s.output_edge_map->at(tensor_name).second;
+      shape_inference_output_idx = s.output_edge_map->at(tensor_name).first;
+    }
+    if (shape_inference_output_idx < 0) continue;
+    VLOG(2) << "shapeinference name: " << shape_inference_node_name
+            << " idx: " << shape_inference_output_idx;
+
+    if (!s.graph_properties.HasOutputProperties(shape_inference_node_name))
+      return tensorflow::errors::Internal("failed to find input node: " +
+                                          shape_inference_node_name);
+
+    auto op_info_vec =
+        s.graph_properties.GetOutputProperties(shape_inference_node_name);
+    if (static_cast<int>(op_info_vec.size()) <= shape_inference_output_idx)
+      return tensorflow::errors::Internal(
+          "accessing output index of: ", shape_inference_output_idx,
+          ", at node: ", shape_inference_node_name,
+          " with output entry from shape_map: ", op_info_vec.size());
 
+    auto op_info = op_info_vec.at(shape_inference_output_idx);
     tensorflow::DataType tf_dtype = op_info.dtype();
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
-    VLOG(2) << "Accessing output index of: " << std::to_string(output_idx)
+    VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
-            << " with output entry from shape_map: "
-            << std::to_string(op_info_vec.size());
-
+            << " with output entry from shape_map: " << op_info_vec.size();
     // TODO(ben,jie): update TRT input format/dimension
     nvinfer1::DimsCHW input_dim_psuedo_chw;
     for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
 
+    // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
+    //            update the code once TRT 4.0 comes out.
+    if (op_info.shape().dim_size() != 4)
+      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
               << " , size: " << op_info.shape().dim(i).size();
@@ -1482,9 +2526,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(ben,jie): proper way to restore input tensor name?
     auto input_tensor_name = node_name;
-    if (output_idx != 0)
-      input_tensor_name = node_name + ":" + std::to_string(output_idx);
+    if (output_idx != 0) {
+      input_tensor_name = StrCat(node_name, ":", output_idx);
+    }
 
+    input_names.push_back(input_tensor_name);
     nvinfer1::ITensor* input_tensor = converter.network()->addInput(
         input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
 
@@ -1511,14 +2557,22 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   // Gather output metadata
   std::vector<string> output_names;
   std::vector<tensorflow::DataType> output_dtypes;
-  for (std::pair<int, int> const& output : output_inds) {
+  int trt_engine_op_output_idx = 0;
+  for (const std::pair<int, int>& output : s.output_inds) {
     int node_id = output.first;
     int output_idx = output.second;
-    tensorflow::Node* node = graph.FindNodeId(node_id);
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
     string op_name = node->name();
     string tensor_name = op_name;
+
+    s.output_edge_map->insert(
+        {trt_engine_op_output_idx == 0
+             ? engine_name
+             : StrCat(engine_name, ":", trt_engine_op_output_idx),
+         {output_idx, tensor_name}});
+    trt_engine_op_output_idx++;
     if (output_idx != 0)
-      tensor_name = tensor_name + ":" + std::to_string(output_idx);
+      tensorflow::strings::StrAppend(&tensor_name, ":", output_idx);
     VLOG(2) << "Output tensor name: " << tensor_name;
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
@@ -1540,19 +2594,25 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
 
   VLOG(2) << "Finished output";
-  // TODO(jie): static_id is not thread safe.
-  static int static_id = 0;
 
   // Build the engine
-  trt_builder->setMaxBatchSize(max_batch_size);
-  trt_builder->setMaxWorkspaceSize(max_workspace_size_bytes);
-  VLOG(0) << "Starting build engine " << static_id;
-  // TODO(ben,jie): half2 and int8 mode support
+  trt_builder->setMaxBatchSize(s.max_batch_size);
+  trt_builder->setMaxWorkspaceSize(s.max_workspace_size_bytes);
+  VLOG(0) << "Max batch size= " << s.max_batch_size
+          << " max workspace size= " << s.max_workspace_size_bytes;
+  if (s.precision_mode == FP16MODE) {
+    trt_builder->setHalf2Mode(true);
+    VLOG(0) << "Using FP16 precision mode";
+  }
+  LOG(INFO) << "starting build engine";
   string engine_plan_string;
   {
     auto trt_engine =
         infer_object(trt_builder->buildCudaEngine(*converter.network()));
     VLOG(0) << "Built network";
+    if (trt_engine.get() == nullptr) {
+      return tensorflow::errors::Internal("Engine building failure");
+    }
     auto engine_plan = infer_object(trt_engine->serialize());
     VLOG(0) << "Serialized engine";
     const char* engine_plan_data =
@@ -1560,18 +2620,19 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     engine_plan_string =
         string(engine_plan_data, engine_plan_data + engine_plan->size());
   }
-
-  VLOG(0) << "Finished engine";
+  TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
+      engine_name, engine_name));
+  LOG(INFO) << "finished engine " << engine_name;
 
   // Build the TRT op
-  // TODO(sami,ben,jie): proper naming!
-  tensorflow::NodeDefBuilder op_builder(
-      tensorflow::strings::StrCat("my_trt_op", static_id++), "TRTEngineOp");
+  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
   std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  VLOG(2) << "input edge size: " << input_names.size();
   for (size_t i = 0; i < input_names.size(); ++i) {
-    int output_idx = input_inds.at(i).second;
-    // We wired up the input here already, it is redundant to do it again in
-    // ConvertSubGraphToTensorRT(convert_graph.cc)
+    VLOG(2) << "input edges: " << i << " " << input_names.at(i);
+    int output_idx = s.input_inds.at(i).second;
+    // we wired up the input here already, it is redundant to do it again in
+    //  ConvertSubGraphToTensorRT(convert_graph.cc)
     auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
         input_names.at(i), output_idx, input_dtypes.at(i));
     income_edges.push_back(incoming_edge);
@@ -1586,7 +2647,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
                     .Attr("input_nodes", input_names)
                     .Attr("output_nodes", output_names)
                     .Attr("OutT", output_dtypes)
-                    .Finalize(trt_node);
+                    .Finalize(s.trt_node);
 
   VLOG(0) << status.ToString() << " finished op building";
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 2e7fd19566..954a1e72f8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -17,6 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
 
 #include <set>
+#include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -32,16 +34,49 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
-    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
-    const std::vector<std::pair<int, int>>&
-        input_inds,  // {node_id, output_idx}
-    const std::vector<std::pair<int, int>>&
-        output_inds,  // {node_id, output_idx}
-    size_t max_batch_size, size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_prop,
-    tensorflow::NodeDef* trt_node);
+const int FP32MODE = 0;
+const int FP16MODE = 1;
+const int INT8MODE = 2;
 
+struct SubGraphParams {
+  SubGraphParams(
+      tensorflow::Graph& inp_graph,
+      const std::set<int>& subgraph_node_id_numbers,
+      const std::vector<std::pair<int, int>>& input_indices,
+      const std::vector<std::pair<int, int>>& output_indices,
+      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
+      const tensorflow::grappler::GraphProperties& current_graph_properties,
+      std::unordered_map<string, std::pair<int, string>>* output_edges,
+      tensorflow::NodeDef* constructed_trt_node,
+      int engine_precision_mode = FP32MODE)
+      : graph(inp_graph),
+        subgraph_node_ids(subgraph_node_id_numbers),
+        input_inds(input_indices),
+        output_inds(output_indices),
+        max_batch_size(max_supported_batch_size),
+        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
+        graph_properties(current_graph_properties),
+        output_edge_map(output_edges),
+        trt_node(constructed_trt_node),
+        precision_mode(engine_precision_mode) {}
+
+  tensorflow::Graph& graph;
+  const std::set<int>& subgraph_node_ids;
+  const std::vector<std::pair<int, int>>& input_inds;   // {node_id, output_idx}
+  const std::vector<std::pair<int, int>>& output_inds;  // {node_id, output_idx}
+  size_t max_batch_size;
+  size_t max_workspace_size_bytes;
+  const tensorflow::grappler::GraphProperties& graph_properties;
+  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
+  tensorflow::NodeDef* trt_node;
+  const int precision_mode;
+};
+
+// TODO(sami): Replace references with const reference or pointers
+tensorflow::Status ConvertSubGraphToTensorRTNodeDef(SubGraphParams& params);
+tensorflow::Status InjectCalibrationNode(SubGraphParams& params);
+tensorflow::Status ConvertCalibrationNodeToEngineNode(tensorflow::Graph& graph,
+                                                      tensorflow::Node* c_node);
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
index 1dcb87e768..aea44fd8a2 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
@@ -21,10 +21,11 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/stream_executor.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda_runtime_api.h"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
@@ -113,7 +114,13 @@ void TRTCalibOp::Compute(tensorflow::OpKernelContext* ctx) {
     ctx->set_output(i, t);
   }
   VLOG(2) << "Filled map for sending";
-  calib_res->calibrator_->setBatch(input_data);
+  // copied from cuda_kernel_helper since it seems only valid in *.cu.cc files
+  const cudaStream_t* stream = CHECK_NOTNULL(
+      reinterpret_cast<const cudaStream_t*>(ctx->op_device_context()
+                                                ->stream()
+                                                ->implementation()
+                                                ->CudaStreamMemberHack()));
+  calib_res->calibrator_->setBatch(input_data, *stream);
   VLOG(2) << "Passed calibration data";
   // TODO(aaroey): make sure we wait for the completion of calibration on the
   // last batch in future PR.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 8efdf63ebe..b32371b642 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -24,8 +24,12 @@ limitations under the License.
 #include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
-namespace tensorrt {
 static ::tensorflow::tensorrt::Logger logger;
+namespace gpu = ::perftools::gputools;
+using IRuntime = nvinfer1::IRuntime;
+using Dims = nvinfer1::Dims;
+
+namespace tensorrt {
 
 TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // read serialized_engine
@@ -40,10 +44,21 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // TODO(samikama) runtime should be taken from a resourcemanager as well.
   // Only engine should be in the op and context and runtime should be taken
   // from resourcemanager
-  nvinfer1::IRuntime* infer = nvinfer1::createInferRuntime(logger);
+  // TODO(jie): cudaSetDevice make sure trt engine is allocated on the same
+  // gpu where the input/output is also located.
+  int gpu_id = context->device()->tensorflow_gpu_device_info()->gpu_id;
+  cudaSetDevice(gpu_id);
+  int device;
+  cudaGetDevice(&device);
+  if (gpu_id != device) LOG(FATAL) << "set device failed!";
+
+  // TODO(samikama) runtime should be taken from a resourcemanager as well.
+  // Only engine should be in the op and context and runtime should be taken
+  // from resourcemanager
+
+  IRuntime* infer = nvinfer1::createInferRuntime(logger);
   trt_engine_ptr_.reset(infer->deserializeCudaEngine(
       serialized_engine.c_str(), serialized_engine.size(), nullptr));
-
   trt_execution_context_ptr_.reset(trt_engine_ptr_->createExecutionContext());
   // Runtime is safe to delete after engine creation
   infer->destroy();
@@ -55,7 +70,6 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
 
   size_t binding_index;
   int num_batch = 0;
-  bool valid = true;
   for (int i = 0; i < context->num_inputs(); i++) {
     // Grab the input tensor
     binding_index = trt_engine_ptr_->getBindingIndex(input_nodes_[i].c_str());
@@ -64,8 +78,12 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     const TensorShape& input_shape = input_tensor.shape();
     if (i == 0) {
       num_batch = input_shape.dim_size(0);
+      if (num_batch > trt_engine_ptr_->getMaxBatchSize()) {
+        LOG(FATAL) << "input tensor batch larger than max_batch_size: "
+                   << trt_engine_ptr_->getMaxBatchSize();
+      }
     } else if (num_batch != input_shape.dim_size(0)) {
-      valid = false;
+      LOG(FATAL) << "input data inconsistent batch size";
       break;
     }
     switch (trt_engine_ptr_->getBindingDataType(binding_index)) {
@@ -81,9 +99,6 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     }
   }
 
-  // Might want a different way to inform the user of batch size inconsistency
-  if (!valid) LOG(WARNING) << "input data inconsistent batch size";
-
   for (int i = 0; i < static_cast<int>(output_nodes_.size()); i++) {
     // This is bad that we have to reallocate output buffer every run.
     // Create an output tensor
@@ -126,9 +141,11 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
                                                 ->implementation()
                                                 ->CudaStreamMemberHack()));
 
-  // execution handled by TF since we are getting stream from TF.
-  // it is safe for CPU pointer array (buffers) to go out of scope after enqueue
-  trt_execution_context_ptr_->enqueue(num_batch, &buffers[0], *stream, nullptr);
+  // TODO(jie): trt enqueue does not return error
+  auto ret = trt_execution_context_ptr_->enqueue(num_batch, &buffers[0],
+                                                 *stream, nullptr);
+  VLOG(2) << "enqueue returns: " << ret;
+  // sync should be done by TF.
 }
 
 REGISTER_KERNEL_BUILDER(Name("TRTEngineOp").Device(DEVICE_GPU), TRTEngineOp);
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.cc b/tensorflow/contrib/tensorrt/log/trt_logger.cc
index 7add8cb8b3..dda0dc9e71 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.cc
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.cc
@@ -27,19 +27,19 @@ void Logger::log(Severity severity, const char* msg) {
   // Suppress info-level messages
   switch (severity) {
     case Severity::kINFO: {  // Mark TRT info messages as debug!
-      VLOG(2) << msg;
+      VLOG(2) << name_ << " " << msg;
       break;
     }
     case Severity::kWARNING: {
-      LOG(WARNING) << msg;
+      LOG(WARNING) << name_ << " " << msg;
       break;
     }
     case Severity::kERROR: {
-      LOG(ERROR) << msg;
+      LOG(ERROR) << name_ << " " << msg;
       break;
     }
     case Severity::kINTERNAL_ERROR: {
-      LOG(FATAL) << msg;
+      LOG(FATAL) << name_ << " " << msg;
       break;
     }
     // This is useless for now. But would catch it in future if enum changes. It
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/contrib/tensorrt/log/trt_logger.h
index d71f66b933..7f3544f8cf 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.h
@@ -27,9 +27,11 @@ namespace tensorrt {
 
 // Logger for GIE info/warning/errors
 class Logger : public nvinfer1::ILogger {
- private:
+ public:
+  Logger(string name = "DefaultLogger") : name_(name){};
   void log(nvinfer1::ILogger::Severity severity, const char* msg) override;
 
+ private:
   string name_;
 };
 
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 7e050a768c..0b2321b5fc 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -20,5 +20,6 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 9454862f85..666220d78c 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -20,11 +20,17 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 import six as _six
+from tensorflow.contrib.tensorrt.wrap_conversion import calib_convert
 from tensorflow.contrib.tensorrt.wrap_conversion import trt_convert
 from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import errors_impl as _impl
+from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.grappler import tf_optimizer
+from tensorflow.python.util import compat
+# pylint: enable=unused-import,line-too-long
 
 
 # TODO(skama): get outputs from session when implemented as c++
@@ -32,22 +38,33 @@ from tensorflow.python.framework import ops
 def create_inference_graph(input_graph_def,
                            outputs,
                            max_batch_size=1,
-                           max_workspace_size_bytes=2 << 20):
+                           max_workspace_size_bytes=2 << 20,
+                           precision_mode="FP32",
+                           minimum_segment_size=3):
   """Python wrapper for the TRT transormation.
 
-
   Args:
     input_graph_def: GraphDef object containing a model to be transformed.
-    outputs: List of tensors or node names for the model outputs.
+    outputs: list of tensors or node names for the model outputs.
     max_batch_size: max size for the input batch
     max_workspace_size_bytes: parameter to control memory allocation (in Bytes)
+    precision_mode: one of 'FP32', 'FP16' and 'INT8'
+    minimum_segment_size: the minimum number of nodes required for a subgraph to
+      be replaced by TRTEngineOp.
 
   Returns:
     New GraphDef with TRTEngineOps placed in graph replacing subgraphs.
 
   Raises:
+    ValueError: if the provided precision mode is invalid.
     RuntimeError: if the returned status message is malformed.
   """
+  supported_precision_modes = {"FP32": 0, "FP16": 1, "INT8": 2}
+  if precision_mode.upper() not in supported_precision_modes:
+    raise ValueError(("precision mode '{}' is not supported."
+                      "It should be one of {}").format(
+                          precision_mode, "{'FP32', 'FP16', 'INT8'}"))
+  mode = supported_precision_modes[precision_mode.upper()]
 
   def py2bytes(inp):
     return inp
@@ -83,7 +100,7 @@ def create_inference_graph(input_graph_def,
   # pair or strings where first one is encoded status and the second
   # one is the transformed graphs protobuf string.
   out = trt_convert(input_graph_def_str, out_names, max_batch_size,
-                    max_workspace_size_bytes)
+                    max_workspace_size_bytes, mode, minimum_segment_size)
   status = to_string(out[0])
   output_graph_def_string = out[1]
   del input_graph_def_str  # Save some memory
@@ -101,3 +118,46 @@ def create_inference_graph(input_graph_def,
   output_graph_def.ParseFromString(output_graph_def_string)
   del output_graph_def_string  # Save some memory
   return output_graph_def
+
+
+def calib_graph_to_infer_graph(calibration_graph_def):
+  """Convert an existing calibration graph to inference graph.
+
+  Args:
+    calibration_graph_def: the calibration GraphDef object with calibration data
+  Returns:
+    New GraphDef with TRTEngineOps placed in graph replacing calibration nodes.
+  Raises:
+    RuntimeError: if the returned status message is malformed.
+  """
+
+  def py2string(inp):
+    return inp
+
+  def py3string(inp):
+    return inp.decode("utf-8")
+
+  if _six.PY2:
+    to_string = py2string
+  else:
+    to_string = py3string
+
+  graph_str = calibration_graph_def.SerializeToString()
+  out = calib_convert(graph_str)
+  status = to_string(out[0])
+  output_graph_def_string = out[1]
+  del graph_str  # Save some memory
+  if len(status) < 2:
+    raise _impl.UnknownError(None, None, status)
+  if status[:2] != "OK":
+    msg = status.split(";")
+    if len(msg) == 1:
+      raise RuntimeError("Status message is malformed {}".format(status))
+    # pylint: disable=protected-access
+    raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
+                                         int(msg[0]))
+    # pylint: enable=protected-access
+  output_graph_def = graph_pb2.GraphDef()
+  output_graph_def.ParseFromString(output_graph_def_string)
+  del output_graph_def_string  # Save some memory
+  return output_graph_def
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
index 3d5cc76c42..74df75902e 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda_runtime_api.h"
+#include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -38,22 +38,18 @@ TRTInt8Calibrator::TRTInt8Calibrator(
       done_(false),
       dev_buffers_(dev_buffers),
       calib_running_(false),
+      batch_is_set_(false),
       engine_name_(engine_name) {}
 
-bool TRTInt8Calibrator::setBatch(
-    const std::unordered_map<string, void*>& data) {
-  // TODO(aaroey): make sure that in future PR:
-  // 1. the mutex_lock is outside of the loop
-  // 2. wait() is used instead of wait_for()
-  // 3. done_ is to be protected by the mutex
-  // 4. the first batch is not missed
-  if (done_) return false;
-  while (calib_running_.load(
-      std::memory_order_acquire)) {  // wait while calibration is running
-    tensorflow::mutex_lock l(cond_mtx_);
-    cond_.wait_for(l, std::chrono::milliseconds(50));
-    if (done_) return false;
+bool TRTInt8Calibrator::setBatch(const std::unordered_map<string, void*>& data,
+                                 const cudaStream_t stream) {
+  tensorflow::mutex_lock lock(cond_mtx_);
+  while ((calib_running_ || batch_is_set_) &&
+         !done_) {  // wait while calibration is running
+    cond_.wait(lock);
   }
+  if (done_) return false;
+  CHECK(!calib_running_ && !batch_is_set_);
   VLOG(1) << "Set Batch Waiting finished";
   for (const auto it : data) {
     auto devptr = dev_buffers_.find(it.first);
@@ -65,27 +61,32 @@ bool TRTInt8Calibrator::setBatch(
 
     // TODO(aaroey): we should not use sync copy on default stream. Make sure
     // stream->ThenMemcpy() is used in future PRs.
-    auto status =
-        cudaMemcpy(d.first, it.second, d.second, cudaMemcpyDeviceToDevice);
+    // TODO(sami,aaroey): Need to figureout a way to ensure synchronization
+    // between stream, perhaps using a tensor?
+    auto status = cudaMemcpyAsync(d.first, it.second, d.second,
+                                  cudaMemcpyDeviceToDevice, stream);
     if (status != cudaSuccess) {
       LOG(FATAL) << "cudaMemcpy " << engine_name_ << " for '" << it.first
                  << "' failed with " << status;
     }
   }
-  calib_running_.store(true, std::memory_order_release);  // release builder
+
+  // TODO(Sami, aaorey): Find an alternative way!
+  cudaStreamSynchronize(
+      stream);  // we have to wait for the stream before returning!
+  batch_is_set_ = true;
   cond_.notify_all();
   return true;
 }
 
 bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
                                  int num_bindings) {
-  calib_running_.store(false, std::memory_order_release);  // wait for new batch
+  tensorflow::mutex_lock lock(cond_mtx_);
+  calib_running_ = false;
   cond_.notify_all();
-  while (!calib_running_.load(
-      std::memory_order_acquire)) {  // wait until new batch arrives
-    tensorflow::mutex_lock l(cond_mtx_);
-    cond_.wait_for(l, std::chrono::milliseconds(50));
-    if (done_) return false;
+  while ((!batch_is_set_ && !done_)) {  // wait until new batch arrives
+    cond_.wait(lock);
+
   }
   if (done_) {
     return false;
@@ -100,6 +101,8 @@ bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
 
     bindings[i] = it->second.first;
   }
+  batch_is_set_ = false;
+  calib_running_ = true;
   return true;
 }
 
@@ -107,6 +110,12 @@ const void* TRTInt8Calibrator::readCalibrationCache(std::size_t& length) {
   return nullptr;
 }
 
+void TRTInt8Calibrator::setDone() {
+  tensorflow::mutex_lock lock(cond_mtx_);
+  done_ = true;
+  cond_.notify_all();
+}
+
 void TRTInt8Calibrator::writeCalibrationCache(const void* ptr,
                                               std::size_t length) {}
 TRTInt8Calibrator::~TRTInt8Calibrator() {
@@ -115,5 +124,6 @@ TRTInt8Calibrator::~TRTInt8Calibrator() {
 
 }  // namespace tensorrt
 }  // namespace tensorflow
+
 #endif
 #endif
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
index 8830f7efe7..d77aa2c5ab 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
@@ -24,7 +24,10 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
+
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
+
 namespace tensorflow {
 namespace tensorrt {
 // This class provides a 1 element queue to match TFs push model to
@@ -39,8 +42,9 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   int getBatchSize() const override;
   bool getBatch(void* bindings[], const char* names[],
                 int num_bindings) override;
-  bool setBatch(const std::unordered_map<string, void*>& data);
-  void setDone() { done_ = true; }
+  bool setBatch(const std::unordered_map<string, void*>& data,
+                const cudaStream_t stream);
+  void setDone();
   const void* readCalibrationCache(std::size_t& length) override;
   void writeCalibrationCache(const void* ptr, std::size_t length) override;
   ~TRTInt8Calibrator();
@@ -55,11 +59,14 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   const std::unordered_map<string, std::pair<void*, size_t>>
       dev_buffers_;  // map to keep tensorrt input buffers and sizes keyed with
                      // buffer names
-  std::atomic_bool calib_running_;
+  bool calib_running_;
+  bool batch_is_set_;
   string engine_name_;
 };
+
 }  // namespace tensorrt
 }  // namespace tensorflow
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+
 #endif
 #endif
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index c78f6f2224..0b661bd536 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -60,6 +60,7 @@ def get_simple_graph_def():
 
 
 def run_graph(gdef, dumm_inp):
+  """Run given graphdef once."""
   gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
   ops.reset_default_graph()
   g = ops.Graph()
@@ -74,15 +75,65 @@ def run_graph(gdef, dumm_inp):
   return val
 
 
+# Use real data that is representatitive of the inference dataset
+# for calibration. For this test script it is random data.
+def run_calibration(gdef, dumm_inp):
+  """Run given calibration graph multiple times."""
+  gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
+  ops.reset_default_graph()
+  g = ops.Graph()
+  with g.as_default():
+    inp, out = importer.import_graph_def(
+        graph_def=gdef, return_elements=["input", "output"])
+    inp = inp.outputs[0]
+    out = out.outputs[0]
+  with csess.Session(
+      config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess:
+    # run over real calibration data here, we are mimicking a calibration set of
+    # 30 different batches. Use as much calibration data as you want
+    for _ in range(30):
+      val = sess.run(out, {inp: dumm_inp})
+  return val
+
+
 if "__main__" in __name__:
   inp_dims = (100, 24, 24, 2)
   dummy_input = np.random.random_sample(inp_dims)
-  gdef = get_simple_graph_def()
+  orig_graph = get_simple_graph_def()  # use a frozen graph for inference
   # Get optimized graph
-  trt_graph = trt.create_inference_graph(gdef, ["output"], inp_dims[0])
-  o1 = run_graph(gdef, dummy_input)
+  trt_graph = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="FP32",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  o1 = run_graph(orig_graph, dummy_input)
   o2 = run_graph(trt_graph, dummy_input)
   o3 = run_graph(trt_graph, dummy_input)
   assert np.array_equal(o1, o2)
   assert np.array_equal(o3, o2)  # sanity check
+  fp16_graph = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="FP16",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  int8_calib_gdef = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="INT8",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  o4 = run_graph(fp16_graph, dummy_input)
+  _ = run_calibration(int8_calib_gdef, dummy_input)
+  int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
+  o5 = run_graph(int8_graph, dummy_input)
+  assert np.allclose(o1, o4)
+  assert np.allclose(o1, o5)
   print("Pass")
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index d679945d56..46480e99a1 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -64,13 +64,17 @@ PyObject* pair_helper(std::pair<string, string>* in) {
 %ignoreall
 %unignore tensorflow;
 %unignore trt_convert;
+%unignore calib_convert;
 
 %{
+
 std::pair<string, string> trt_convert(
     string graph_def_string,  // The serialized GraphDef string.
     std::vector<string> output_names,
     size_t max_batch_size,
-    size_t max_workspace_size_bytes
+    size_t max_workspace_size_bytes,
+    int precision_mode,
+    int minimum_segment_size
     // Unfortunately we can't use TF_Status here since it
     // is in c/c_api and brings in a lot of other libraries
     // which in turn declare ops. These ops are included
@@ -90,16 +94,64 @@ std::pair<string, string> trt_convert(
     return std::pair<string, string>{out_status, ""};
   }
 
+  if(precision_mode < 0 || precision_mode > 2){
+    out_status = "InvalidArgument;Invalid precision_mode";
+    return std::pair<string, string>{out_status, ""};
+  }
   if (!output_names.size()) {
     out_status = "InvalidArgument;Size of the output_names vector is 0";
     return std::pair<string, string>{out_status, ""};
-    // return "";
   }
   tensorflow::GraphDef outGraph;
   tensorflow::Status conversion_status =
       tensorflow::tensorrt::convert::ConvertGraphDefToTensorRT(
           graph_def, output_names, max_batch_size, max_workspace_size_bytes,
-          &outGraph);
+          &outGraph, precision_mode, minimum_segment_size);
+  if (!conversion_status.ok()) {
+    auto retCode = (int)conversion_status.code();
+    char buff[2000];
+    snprintf(buff, 2000, "%d;%s", retCode,
+             conversion_status.error_message().c_str());
+    out_status = buff;
+    return std::pair<string, string>{out_status, ""};
+  }
+  string result;
+  if (!outGraph.SerializeToString(&result)) {
+    out_status = "InvalidArgument;Couldn't serialize output as a GraphDef";
+    return std::pair<string, string>{out_status, ""};
+  }
+  out_status = "OK;All good!";
+  return std::pair<string, string>{out_status, result};
+#else
+  // Returns FAILED_PRECONDITION.
+  return std::pair<string, string>{"9;TensorRT is not enabled!", ""};
+#endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
+}
+
+std::pair<string, string> calib_convert(string graph_def_string  //  const tensorflow::GraphDef&
+    // unfortunately we can't use TF_Status here since it
+    // is in c/c_api and brings in a lot of other libraries
+    // which in turn declare ops. These ops are included
+    // statically in our library and cause an abort when
+    // module is loaded due to double registration
+    // until Tensorflow properly exposes these headers
+    // we have to work around this by returning a string
+    // and converting it to exception on python side.
+    //,TF_Status* out_status) {
+) {
+#if GOOGLE_CUDA && GOOGLE_TENSORRT
+  string out_status;
+
+  tensorflow::GraphDef graph_def;
+  if (!graph_def.ParseFromString(graph_def_string)) {
+    out_status = "InvalidArgument;Couldn't interpret input as a GraphDef";
+    return std::pair<string, string>{out_status, ""};
+  }
+
+  tensorflow::GraphDef outGraph;
+  tensorflow::Status conversion_status =
+      tensorflow::tensorrt::convert::ConvertCalibGraphToInferGraph(graph_def,
+                                                                   &outGraph);
   if (!conversion_status.ok()) {
     auto retCode = (int)conversion_status.code();
     char buff[2000];
@@ -122,10 +174,13 @@ std::pair<string, string> trt_convert(
 }
 %}
 
+std::pair<string, string> calib_convert(string graph_def_string);
+
 std::pair<string, string> trt_convert(string graph_def_string,
                                       std::vector<string> output_names,
                                       size_t max_batch_size,
-                                      size_t max_workspace_size_bytes);
+                                      size_t max_workspace_size_bytes,
+                                      int precision_mode, int minimum_segment_size);
 
 
 %unignoreall
diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index bb86ecb220..70bf67c779 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -25,7 +25,10 @@ py_test(
     srcs = ["predict_test.py"],
     data = ["data/period_trend.csv"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67513579
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",  # b/67513579
+    ],
     deps = [
         ":predict",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index ed3ed4c0e1..64f5cd8357 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -156,9 +156,7 @@ py_test(
         "head_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip_gpu",  # b/63391119
-    ],
+    tags = ["no_pip_gpu"],  # b/63391119
     deps = [
         ":feature_keys",
         ":head",
@@ -427,6 +425,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip_gpu",  # b/63391119
+        "no_windows",  # TODO: needs investigation on Windows
     ],
     deps = [
         ":feature_keys",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index c86d06e923..07df7bc9a5 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -40,6 +40,7 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["state_space_model_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":state_space_model",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index ed930e44e8..f9d433a45b 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -225,6 +225,7 @@ tf_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:layers",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 tf_py_test(
diff --git a/tensorflow/contrib/util/loader.py b/tensorflow/contrib/util/loader.py
index f4283cd9ed..dca01d26f4 100644
--- a/tensorflow/contrib/util/loader.py
+++ b/tensorflow/contrib/util/loader.py
@@ -42,9 +42,10 @@ def load_op_library(path):
     plugin.
   """
   if os.name == 'nt':
-    # To avoid makeing every user_ops aware of windows, re-write
-    # the file extension from .so to .dll.
-    path = re.sub(r'\.so$', '.dll', path)
+    # To avoid making every user_ops aware of windows, re-write
+    # the file extension from .so to .dll if .so file doesn't exist.
+    if not os.path.exists(path):
+      path = re.sub(r'\.so$', '.dll', path)
 
     # Currently we have only some user_ops as dlls on windows - don't try
     # to load them if the dll is not found.
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 352e183104..5c9fd2f406 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3327,6 +3327,10 @@ tf_cc_test(
     size = "small",
     srcs = ["common_runtime/function_test.cc"],
     linkstatic = tf_kernel_tests_linkstatic(),
+    tags = [
+        "manual",
+        "no_oss",
+    ],
     deps = [
         ":core",
         ":core_cpu",
diff --git a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
new file mode 100644
index 0000000000..9fabe7863e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
@@ -0,0 +1,18 @@
+op {
+  graph_op_name: "SlideDataset"
+  in_arg {
+    name: "window_size"
+    description: <<END
+A scalar representing the number of elements in the
+sliding window.
+END
+  }
+  in_arg {
+    name: "stride"
+    description: <<END
+A scalar representing the steps moving the sliding window
+forward in one iteration. It must be in `[1, window_size)`.
+END
+  }
+  summary: "Creates a dataset that passes a sliding window over `input_dataset`."
+}
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.cc b/tensorflow/core/distributed_runtime/tensor_coding.cc
index 34a4013547..fe2d1a1293 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.cc
+++ b/tensorflow/core/distributed_runtime/tensor_coding.cc
@@ -81,7 +81,7 @@ void TensorResponse::InitPartial(const RecvTensorResponse& response) {
 Status TensorResponse::ParseFrom(Source* source) {
   if (!on_host_) {
     protobuf::io::CodedInputStream input(source->contents());
-    input.SetTotalBytesLimit(INT_MAX);  // Unlimited
+    input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
 
     // Pre-parse into local storage, then delegate to device.
     if (!meta_.ParseFromCodedStream(&input) || !input.ConsumedEntireMessage()) {
@@ -217,7 +217,7 @@ bool TensorResponse::ParseTensorSubmessage(
 
 bool TensorResponse::ParseFast(Source* source) {
   protobuf::io::CodedInputStream input(source->contents());
-  input.SetTotalBytesLimit(INT_MAX);  // Unlimited
+  input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
   while (true) {
     auto p = input.ReadTagWithCutoff(127);
     int tag = GetTagFieldNumber(p.first);
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index beaf0adbc5..cfe23d1ffe 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -474,11 +474,11 @@ class GraphDatasetBase : public DatasetBase {
   }
 
   // Key for storing the Dataset graph in the serialized format.
-  static const char kDatasetGraphKey[];
+  TF_EXPORT static const char kDatasetGraphKey[];
 
   // Key for storing the output node of the Dataset graph in the serialized
   // format.
-  static const char kDatasetGraphOutputNodeKey[];
+  TF_EXPORT static const char kDatasetGraphOutputNodeKey[];
 
  private:
   Status Serialize(OpKernelContext* ctx, string* serialized_graph_def,
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 02038c5d77..1507b6eae2 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2492,10 +2492,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       mkl_op_registry::GetMklOpName(csinfo_.identity),
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.lrn, mkl_op_registry::GetMklOpName(csinfo_.lrn),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.lrn_grad,
                       mkl_op_registry::GetMklOpName(csinfo_.lrn_grad),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.max_pool,
                       mkl_op_registry::GetMklOpName(csinfo_.max_pool),
                       CopyAttrsPooling, NonDepthBatchWisePoolRewrite});
@@ -2865,6 +2865,28 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return false;
   }
 
+  // If the depth_radius of LRN is not 2, then MKL DNN takes unoptimized 
+  // path. The unoptimized path is slow. Thus we dont rewrite the node 
+  // and use default Eigen. But for depth_radius=2, MKL DNN optimized 
+  // path is taken, i.e., eigen node is rewritten by MKl DNN node.
+  static bool LrnRewrite(const Node* n) {
+    CHECK_NOTNULL(n);
+
+    int depth_radius;
+    CHECK_EQ(GetNodeAttr(n->def(), "depth_radius", &depth_radius).ok(), true);
+
+    // if the depth_radius of LRN is not 2, don't rewrite the node by MKL DNN
+    // and use eigen node instead 
+    if (depth_radius == 2) {
+      return true;
+    }
+    VLOG(1) << "LrnRewrite: The model sets depth_radius as not 2 which"
+            << "case is not optimized by Intel MKL, thus using Eigen op"
+            << "for LRN " ; 
+
+    return false;
+  }
+
   static bool AddNRewrite(const Node* n) {
     CHECK_NOTNULL(n);
 
@@ -3528,11 +3550,13 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   string data_format;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
   TF_CHECK_OK(
@@ -3541,6 +3565,7 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
+  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
@@ -3778,12 +3803,14 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   DataType T_pred, T_succ;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   string data_format_pred, data_format_succ;
   bool use_cudnn_on_gnu;
   TF_CHECK_OK(GetNodeAttr(pred->def(), "T", &T_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "T", &T_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(pred->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "data_format", &data_format_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 244653504d..8f13c4a702 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -408,7 +409,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
       frame_children_[frame_ids[0]].insert(frame_ids[1]);
       frame_parent_[frame_ids.back()] = frame_ids[frame_ids.size() - 2];
     }
-    if (!frame_ids.empty()) {
+    if (frame_ids.size() >= 1) {
       frame_children_.insert(std::make_pair(frame_ids.back(), empty_set_));
       if (node->op() == "LoopCond") {
         if (loop_cond_.count(frame_ids.back())) {
@@ -427,7 +428,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
   }
 
   for (auto it = frame_children_.begin(); it != frame_children_.end(); ++it) {
-    if (it->second.empty()) {
+    if (it->second.size() == 0) {
       worklist.push_back(it->first);
     }
   }
@@ -440,7 +441,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
     if (parent_it != frame_parent_.end()) {
       int parent_id = parent_it->second;
       frame_children_[parent_id].erase(frame_id);
-      if (frame_children_[parent_id].empty()) {
+      if (frame_children_[parent_id].size() == 0) {
         worklist.push_back(parent_id);
       }
     }
@@ -464,7 +465,6 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
 Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                GraphDef* optimized_graph) {
   TF_RETURN_IF_ERROR(RemoveStackOps(item.graph, optimized_graph));
-
   optimized_graph_ = optimized_graph;
 
   // Set up helper data structures.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 48d5955ad1..2e39f25fc1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5157,7 +5157,6 @@ tf_kernel_library(
     ],
     hdrs = [
         "meta_support.h",
-        "quantization_utils.h",
         "reference_gemm.h",
     ],
     deps = [
@@ -5232,6 +5231,7 @@ tf_cc_test(
     name = "quantization_utils_test",
     srcs = ["quantization_utils_test.cc"],
     deps = [
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5294,6 +5294,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5355,6 +5356,7 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5377,6 +5379,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5441,6 +5444,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5461,6 +5465,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5500,6 +5505,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5556,6 +5562,7 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5578,6 +5585,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5614,6 +5622,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5635,6 +5644,7 @@ tf_cc_test(
     deps = [
         ":batch_norm_op",
         ":ops_testutil",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 484d4f88d6..01754ec21a 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -113,6 +113,19 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "slide_dataset_op",
+    srcs = ["slide_dataset_op.cc"],
+    deps = [
+        ":dataset",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:batch_util",
+    ],
+)
+
 tf_kernel_library(
     name = "padded_batch_dataset_op",
     srcs = ["padded_batch_dataset_op.cc"],
@@ -538,6 +551,7 @@ tf_kernel_library(
         ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
+        ":slide_dataset_op",
         ":sparse_tensor_slice_dataset_op",
         ":sql_dataset_ops",
         ":stats_aggregator_ops",
diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/slide_dataset_op.cc
new file mode 100644
index 0000000000..4f3537b691
--- /dev/null
+++ b/tensorflow/core/kernels/data/slide_dataset_op.cc
@@ -0,0 +1,252 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/batch_util.h"
+#include "tensorflow/core/kernels/data/dataset.h"
+
+namespace tensorflow {
+
+namespace {
+
+// See documentation in ../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class SlideDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit SlideDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 window_size = 0;
+    int64 stride = 1;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "window_size", &window_size));
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "stride", &stride));
+    OP_REQUIRES(
+        ctx, window_size > 0,
+        errors::InvalidArgument("Window size must be greater than zero."));
+    OP_REQUIRES(
+        ctx, stride > 0 && stride < window_size,
+        errors::InvalidArgument("Stride must be in [1, window_size)."));
+
+    *output = new Dataset(ctx, window_size, stride, input);
+  }
+
+ private:
+  class Dataset : public GraphDatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, int64 window_size, int64 stride, const DatasetBase* input)
+        : GraphDatasetBase(ctx), window_size_(window_size), stride_(stride), input_(input) {
+      input_->Ref();
+
+      const auto& input_shapes = input_->output_shapes();
+      output_shapes_.reserve(input_shapes.size());
+      for (const auto& input_shape : input_shapes) {
+        output_shapes_.emplace_back(
+            PartialTensorShape({-1}).Concatenate(input_shape));
+      }
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIterator(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          Iterator::Params{this, strings::StrCat(prefix, "::Slide")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() override {
+      return strings::StrCat("SlideDatasetOp(", window_size_, ", ", stride_, ")::Dataset");
+    }
+
+   protected:
+    Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node));
+      Node* window_size = nullptr;
+      Node* stride = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(window_size_, &window_size));
+      TF_RETURN_IF_ERROR(b->AddScalar(stride_, &stride));
+      TF_RETURN_IF_ERROR(
+          b->AddDataset(this, {input_graph_node, window_size, stride}, output));
+      return Status::OK();
+    }
+
+   private:
+
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            input_impl_(params.dataset->input_->MakeIterator(params.prefix)) {}
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        const int64 window_size = dataset()->window_size_;
+        const int64 stride = dataset()->stride_;
+        std::vector<std::vector<Tensor>> batch_elements;
+        {
+          mutex_lock l(mu_);
+          if (!input_impl_) {
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+          batch_elements.reserve(window_size);
+          const bool first_call = cache_.empty();
+          if (first_call) {
+            cache_.reserve(window_size);
+          } else {
+            // Reuse cache in the previous iteration.
+            cache_.swap(batch_elements);
+          }
+          // Fill up with new elements.
+          *end_of_sequence = false;
+          for (size_t i = batch_elements.size(); i < window_size && !*end_of_sequence;
+              ++i) {
+            std::vector<Tensor> batch_element_tuple;
+            TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &batch_element_tuple,
+                                                    end_of_sequence));
+            if (!*end_of_sequence) {
+              batch_elements.push_back(std::move(batch_element_tuple));
+            } else {
+              input_impl_.reset();
+            }
+          }
+          // Drop the final smaller blocks.
+          if (batch_elements.size() < window_size) {
+            DCHECK(*end_of_sequence);
+            return Status::OK();
+          }
+          // Cache the data used for the next iteration.
+          for (size_t i = stride; i < window_size; ++i) {
+            cache_.emplace_back(batch_elements[i]);
+          }
+        }
+
+        // Construct output tensors.
+        // Those codes below are copied from batch_dataset_op.cc.
+        const size_t num_tuple_components = batch_elements[0].size();
+        const int64 num_batch_elements = batch_elements.size();
+        for (size_t component_index = 0; component_index < num_tuple_components;
+             ++component_index) {
+          const Tensor& first_element = batch_elements[0][component_index];
+          TensorShape batch_component_shape({num_batch_elements});
+          batch_component_shape.AppendShape(first_element.shape());
+          Tensor batch_component(cpu_allocator(), first_element.dtype(),
+                                 batch_component_shape);
+          // Build the output tuple component by copying one slice
+          // from each input element in the batch.
+          for (size_t i = 0; i < num_batch_elements; ++i) {
+            if (batch_elements[i][component_index].shape() !=
+                first_element.shape()) {
+              return errors::InvalidArgument(
+                  "Cannot batch tensors with different shapes in component ",
+                  component_index, ". First element had shape ",
+                  first_element.shape().DebugString(), " and element ", i,
+                  " had shape ",
+                  batch_elements[i][component_index].shape().DebugString(),
+                  ".");
+            }
+            TF_RETURN_IF_ERROR(batch_util::CopyElementToSlice(
+                std::move(batch_elements[i][component_index]), &batch_component,
+                i));
+          }
+          out_tensors->emplace_back(std::move(batch_component));
+        }
+        *end_of_sequence = false;
+        return Status::OK();
+      }
+
+     protected:
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (!input_impl_) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impl_empty"), ""));
+        } else {
+          TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_));
+        }
+        // Save cache.
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(strings::StrCat("cache_size"), cache_.size()));
+        for (int64 i = 0; i < cache_.size(); i++) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              strings::StrCat("cache[", i, "]_size"), cache_[i].size()));
+          for (int64 j = 0; j < cache_[i].size(); j++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                strings::StrCat("cache[", i, "][", j, "]"), cache_[i][j]));
+          }
+        }
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (!reader->Contains(full_name("input_impl_empty"))) {
+          TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_));
+        } else {
+          input_impl_.reset();
+        }
+        // Restore cache.
+        int64 cache_size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(strings::StrCat("cache_size"), &cache_size));
+        cache_.resize(cache_size);
+        for (int64 i = 0; i < cache_size; i++) {
+          int64 vector_size;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              strings::StrCat("cache[", i, "]_size"), &vector_size));
+          cache_[i].resize(vector_size);
+          for (int64 j = 0; j < vector_size; j++) {
+            TF_RETURN_IF_ERROR(reader->ReadTensor(
+                strings::StrCat("cache[", i, "][", j, "]"), &cache_[i][j]));
+          }
+        }
+        return Status::OK();
+      }
+
+     private:
+      mutex mu_;
+      std::vector<std::vector<Tensor>> cache_ GUARDED_BY(mu_);
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+    };
+
+    const int64 window_size_;
+    const int64 stride_;
+    const DatasetBase* const input_;
+    std::vector<PartialTensorShape> output_shapes_;
+  };
+};
+
+REGISTER_KERNEL_BUILDER(Name("SlideDataset").Device(DEVICE_CPU),
+                        SlideDatasetOp);
+
+}  // namespace
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/depthtospace_op.cc b/tensorflow/core/kernels/depthtospace_op.cc
index 39aa3e9eb0..b74a09e2cb 100644
--- a/tensorflow/core/kernels/depthtospace_op.cc
+++ b/tensorflow/core/kernels/depthtospace_op.cc
@@ -187,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     DepthToSpaceOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
+    DepthToSpaceOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     DepthToSpaceOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
index 184c703599..0656081177 100644
--- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
@@ -238,6 +238,12 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// Instantiate the GPU implementations for Eigen::half.
+template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NCHW>;
+template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NHWC>;
+
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 108d59db2c..7688305019 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -45,6 +45,7 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/kernels:cwise_op",
+        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/kernels:reduction_ops",
         "//tensorflow/core/kernels:remote_fused_graph_execute_utils",
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index 1401bc65a4..e0706568b1 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -444,6 +444,7 @@ class MklConv2DCustomBackpropFilterOp
   ~MklConv2DCustomBackpropFilterOp() {}
 
  private:
+  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -492,7 +493,9 @@ class MklConv2DCustomBackpropFilterOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor, const memory::dims& strides,
+                       Tensor** output_tensor,
+                       const memory::dims& strides,
+                       const memory::dims& dilations,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -518,31 +521,32 @@ class MklConv2DCustomBackpropFilterOp
       bias_grad->SetOpMemDesc(bias_grad_dims, memory::format::x);
     }
 
-    // Create convolution backward weights primitive.
-    auto bwd_desc =
-        (biasEnabled && (bias_grad != nullptr))
-            ? convolution_backward_weights::desc(
-                  convolution_direct, input->GetOpMemDesc(),
-                  output->GetOpMemDesc(), bias_grad->GetOpMemDesc(),
-                  outbackprop->GetOpMemDesc(), strides, padding_l, padding_r,
-                  padding)
-            : convolution_backward_weights::desc(
-                  convolution_direct, input->GetOpMemDesc(),
-                  output->GetOpMemDesc(), outbackprop->GetOpMemDesc(), strides,
-                  padding_l, padding_r, padding);
-
-    auto bwd_pd = convolution_backward_weights::primitive_desc(
-        bwd_desc, cpu_engine, conv_fwd_pd);
-
-    // Allocate output tensor.
-    AllocateOutputTensor(context, bwd_pd, bwd_output_dims, bwd_output_format,
-                         output_tensor);
-
-    CHECK_NOTNULL(*output_tensor);
-    // Set buffer handle using allocated output tensor.
-    output->SetUsrMemDataHandle(*output_tensor);
-
     if (biasEnabled && (bias_grad != nullptr)) {
+      // Create convolution backward weights with bias primitive.
+      // Use dilated convolution in case dilate rates are greater than zero.
+      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  bias_grad->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(), strides,
+                                  dilations, padding_l, padding_r, padding) :
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  bias_grad->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(),
+                                  strides, padding_l, padding_r, padding);
+      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
+                                                            cpu_engine,
+                                                            conv_fwd_pd);
+
+      // Allocate output tensor.
+      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
+                           bwd_output_format, output_tensor);
+
+      CHECK_NOTNULL(*output_tensor);
+      // Set buffer handle using allocated output tensor.
+      output->SetUsrMemDataHandle(*output_tensor);
+
       // Allocate bias_grad tensor
       TensorShape bias_grad_shape({depth});
       Tensor* bias_grad_tensor = nullptr;
@@ -553,11 +557,32 @@ class MklConv2DCustomBackpropFilterOp
           memory::desc({bias_grad_dims}, MklDnnType<T>(), memory::format::x);
       bias_grad->SetUsrMem(bias_grad_md, bias_grad_tensor);
       bias_grad->SetUsrMemDataHandle(bias_grad_tensor);
-    }
 
-    if (biasEnabled && (bias_grad != nullptr)) {
-      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output, bias_grad);
+      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output,
+                                  bias_grad);
     } else {
+      // Create convolution backward weights primitive.
+      // Use dilated convolution in case dilate rates are greater than zero.
+      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(), strides,
+                                  dilations, padding_l, padding_r, padding) :
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(),
+                                  strides, padding_l, padding_r, padding);
+      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
+                                                            cpu_engine,
+                                                            conv_fwd_pd);
+
+      // Allocate output tensor.
+      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
+                           bwd_output_format, output_tensor);
+
+      CHECK_NOTNULL(*output_tensor);
+      // Set buffer handle using allocated output tensor.
+      output->SetUsrMemDataHandle(*output_tensor);
       PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output);
     }
   }
diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
index eeed009531..d203c04934 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
@@ -369,6 +369,7 @@ class MklConv2DCustomBackpropInputOp
  private:
   const int kInputIndex_Filter = 1, kInputIndex_InputSizes = 0,
             kInputIndex_OutBackProp = 2;
+  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -419,7 +420,9 @@ class MklConv2DCustomBackpropInputOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor, const memory::dims& strides,
+                       Tensor** output_tensor,
+                       const memory::dims& strides,
+                       const memory::dims& dilations,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -432,9 +435,16 @@ class MklConv2DCustomBackpropInputOp
     CHECK_NOTNULL(output_tensor);
 
     // Create convolution backward data primitive.
-    auto bwd_desc = convolution_backward_data::desc(
-        convolution_direct, output->GetOpMemDesc(), filter->GetOpMemDesc(),
-        outbackprop->GetOpMemDesc(), strides, padding_l, padding_r, padding);
+    // Use dilated convolution in case dilate rates are greater than zero.
+    auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_data::desc(convolution_direct,
+                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
+                      outbackprop->GetOpMemDesc(), strides,
+                      dilations, padding_l, padding_r, padding):
+        convolution_backward_data::desc(convolution_direct,
+                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
+                      outbackprop->GetOpMemDesc(),
+                      strides, padding_l, padding_r, padding);
 
     auto bwd_pd = convolution_backward_data::primitive_desc(
         bwd_desc, cpu_engine, conv_fwd_pd);
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 1440da8f82..f0818eb96d 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -493,6 +493,7 @@ class MklConv2DOp : public OpKernel {
   ~MklConv2DOp() {}
 
   explicit MklConv2DOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
     string data_format;
     OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
@@ -509,6 +510,20 @@ class MklConv2DOp : public OpKernel {
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(context, dilations_.size() == 4,
+                errors::InvalidArgument("Sliding window dilations field must "
+                                        "specify 4 dimensions"));
+    const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N');
+    const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C');
+    const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H');
+    const int64 dilation_w = GetTensorDim(dilations_, data_format_, 'W');
+    OP_REQUIRES(context, dilation_n == 1 && dilation_c == 1,
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "dilations in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, dilation_h > 0 && dilation_w > 0,
+        errors::InvalidArgument("Dilated rates should be larger than 0."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -530,17 +545,19 @@ class MklConv2DOp : public OpKernel {
       MklDnnData<T> filter(&cpu_engine);
       MklDnnData<T> output(&cpu_engine);
 
-      memory::dims src_dims, filter_dims, padding_l, padding_r, strides;
+      memory::dims src_dims, filter_dims, padding_l, padding_r,
+                   dilations, strides;
       memory::dims output_dims_tf_order, output_dims_mkl_order;
 
       // Get shapes of input tensors in MKL-DNN order
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
+                             dilations_);
       auto src_tf_shape = GetTfShape(context, kInputIndex_Src);
       auto filter_tf_shape = GetTfShape(context, kInputIndex_Filter);
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides,
-          &output_dims_tf_order, &output_dims_mkl_order, &padding_l,
-          &padding_r);
+          &dilations, &output_dims_tf_order, &output_dims_mkl_order,
+          &padding_l, &padding_r);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -553,6 +570,7 @@ class MklConv2DOp : public OpKernel {
         //               Need semantics for Null MKL tensor
         MklDnnShape output_mkl_shape;
         output_mkl_shape.SetMklTensor(false);
+
         AllocateOutputSetMklShape(context, kOutputIndex_Dst, &output_tensor,
                                   src_tf_shape, output_mkl_shape);
 
@@ -596,55 +614,79 @@ class MklConv2DOp : public OpKernel {
       filter.SetOpMemDesc(filter_dims, memory::format::any);
       output.SetOpMemDesc(output_dims_mkl_order, memory::format::any);
 
-      // If bias is enabled, then do the same steps as above for bias.
+      // MKLDNN dilation starts from 0.
+      dilations[kDilationH] -= 1;
+      dilations[kDilationW] -= 1;
+
       if (biasEnabled) {
-        MklDnnData<T> bias(&cpu_engine);
-        memory::dims bias_size;
-        conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
-        const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
-        bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
-        bias.SetOpMemDesc(bias_size, memory::format::any);
-
-        // Create convolution primitive with Bias.
-        auto conv_desc = convolution_forward::desc(
-            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
-            filter.GetOpMemDesc(), bias.GetOpMemDesc(), output.GetOpMemDesc(),
-            strides, padding_l, padding_r, TFPaddingToMklDnnPadding(padding_));
-
-        auto conv_prim_desc =
-            convolution_forward::primitive_desc(conv_desc, cpu_engine);
-        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
-                             tf_fmt, &output_tensor);
-        // Set data handle for output.
-        output.SetUsrMemDataHandle(output_tensor);
-
-        Tensor* filter_out_tensor = nullptr;
-        AllocateFilterOutputTensor(context, conv_prim_desc,
-                                   TFShapeToMklDnnDims(filter_tf_shape),
-                                   &filter_out_tensor);
-
-        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
-                             filter_out_tensor);
+          // Create convolution primitive with Bias.
+          MklDnnData<T> bias(&cpu_engine);
+          memory::dims bias_size;
+          conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
+          const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
+          bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
+          bias.SetOpMemDesc(bias_size, memory::format::any);
+
+          // Create convolution primitive with Bias.
+          // Use MKLDNN dilated convolution in case of dilated rate (>0).
+          auto conv_desc = (dilations[kDilationH] > 0 ||
+              dilations[kDilationW] > 0) ?
+              convolution_forward::desc(prop_kind::forward,
+                      convolution_direct, src.GetOpMemDesc(),
+                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
+                      output.GetOpMemDesc(), strides, dilations,
+                      padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_)):
+              convolution_forward::desc(prop_kind::forward,
+                      convolution_direct, src.GetOpMemDesc(),
+                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
+                      output.GetOpMemDesc(), strides,
+                      padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_));
+
+          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
+                                                                  cpu_engine);
+          AllocateOutputTensor(context, conv_prim_desc,
+                               output_dims_mkl_order, tf_fmt, &output_tensor);
+          // Set data handle for output.
+          output.SetUsrMemDataHandle(output_tensor);
+
+          Tensor* filter_out_tensor = nullptr;
+          AllocateFilterOutputTensor(context, conv_prim_desc,
+                TFShapeToMklDnnDims(filter_tf_shape),
+                &filter_out_tensor);
+
+          PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
+                               filter_out_tensor);
       } else {
-        // Create convolution primitive without Bias.
-        auto conv_desc = convolution_forward::desc(
-            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
-            filter.GetOpMemDesc(), output.GetOpMemDesc(), strides, padding_l,
-            padding_r, TFPaddingToMklDnnPadding(padding_));
-
-        auto conv_prim_desc =
-            convolution_forward::primitive_desc(conv_desc, cpu_engine);
-        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
-                             tf_fmt, &output_tensor);
-        // Set data handle for output.
-        output.SetUsrMemDataHandle(output_tensor);
-
-        Tensor* filter_out_tensor = nullptr;
-        AllocateFilterOutputTensor(context, conv_prim_desc,
-                                   TFShapeToMklDnnDims(filter_tf_shape),
-                                   &filter_out_tensor);
-        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, nullptr, &output,
-                             filter_out_tensor);
+          // Create convolution primitive without Bias.
+          // Use MKLDNN dilated convolution in case of dilated rate (>0).
+          auto conv_desc = (dilations[kDilationH] > 0 ||
+            dilations[kDilationW] > 0) ?
+            convolution_forward::desc(prop_kind::forward,
+              convolution_direct, src.GetOpMemDesc(),
+              filter.GetOpMemDesc(), output.GetOpMemDesc(),
+              strides, dilations, padding_l, padding_r,
+              TFPaddingToMklDnnPadding(padding_)):
+          convolution_forward::desc(prop_kind::forward,
+              convolution_direct, src.GetOpMemDesc(),
+              filter.GetOpMemDesc(), output.GetOpMemDesc(),
+              strides, padding_l, padding_r,
+              TFPaddingToMklDnnPadding(padding_));
+
+          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
+                                                                  cpu_engine);
+          AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
+                               tf_fmt, &output_tensor);
+          // Set data handle for output.
+          output.SetUsrMemDataHandle(output_tensor);
+
+          Tensor* filter_out_tensor = nullptr;
+          AllocateFilterOutputTensor(context, conv_prim_desc,
+                TFShapeToMklDnnDims(filter_tf_shape),
+                &filter_out_tensor);
+          PrepareAndExecuteNet(conv_prim_desc, &src, &filter,
+                              nullptr, &output, filter_out_tensor);
       }
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -658,10 +700,12 @@ class MklConv2DOp : public OpKernel {
 
  private:
   std::vector<int32> strides_;
+  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
+  const int kDilationH = 0, kDilationW = 1;
 
   // Allocate output tensor.
   void AllocateOutputTensor(
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 9dd88221a8..7ca10db895 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -58,13 +58,16 @@ class MklDnnConvUtil {
  protected:
   OpKernelContext* context_;  // We don't own this.
   std::vector<int32> strides_;
+  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
 
  public:
   MklDnnConvUtil(OpKernelContext* context, const std::vector<int32>& strides,
-                 Padding pad, TensorFormat fm)
-      : context_(context), strides_(strides), padding_(pad), data_format_(fm) {}
+                 Padding pad, TensorFormat fm,
+                 const std::vector<int32>& dilations) :
+    context_(context), strides_(strides), padding_(pad),
+    data_format_(fm), dilations_(dilations) {}
 
   virtual ~MklDnnConvUtil() { context_ = nullptr; }
 
@@ -78,6 +81,16 @@ class MklDnnConvUtil {
     *strides = {stride_rows, stride_cols};
   }
 
+  // Calculate Convolution dilations
+  virtual inline void GetDilationsInMklOrder(memory::dims *dilations) {
+    // For now we take the dilation from the second and third dimensions only
+    // (we do not support dilation on the batch or depth dimension).
+    CHECK_NOTNULL(dilations);
+    int dilations_rows = GetTensorDim(dilations_, data_format_, 'H');
+    int dilations_cols = GetTensorDim(dilations_, data_format_, 'W');
+    *dilations = {dilations_rows, dilations_cols};
+  }
+
   // Calculate Convolution input size in MKL-DNN order. MKL-DNN
   // requires input in NCHW format. Function does not return anything.
   // But errors arising from sanity checks are returned in context's
@@ -213,7 +226,8 @@ class MklDnnConvUtil {
   // TODO(nhasabni): Add similar function for input and filter in MklShape.
   virtual inline void GetOutputAndPadSizeInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
-      const memory::dims& strides, memory::dims* output_dims_tf_order,
+      const memory::dims& strides, const memory::dims& dilations,
+      memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -232,6 +246,8 @@ class MklDnnConvUtil {
     // Stride is vector of 2 elements: {s_r, s_c}
     int stride_rows = strides[0];
     int stride_cols = strides[1];
+    int dilation_rows = dilations[0];
+    int dilation_cols = dilations[1];
 
     // Output batch is same as input batch.
     int out_batch = GetTensorDim(input_shape, data_format_, 'N');
@@ -241,11 +257,13 @@ class MklDnnConvUtil {
     int64 out_rows = 0, out_cols = 0;
     int64 pad_top = 0, pad_bottom = 0, pad_left, pad_right;
 
-    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
-                                 input_rows, filter_rows, stride_rows, padding_,
+    OP_REQUIRES_OK(context_,
+            GetWindowedOutputSizeVerboseV2(input_rows, filter_rows,
+                                 dilation_rows, stride_rows, padding_,
                                  &out_rows, &pad_top, &pad_bottom));
-    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
-                                 input_cols, filter_cols, stride_cols, padding_,
+    OP_REQUIRES_OK(context_,
+            GetWindowedOutputSizeVerboseV2(input_cols, filter_cols,
+                                 dilation_cols, stride_cols, padding_,
                                  &out_cols, &pad_left, &pad_right));
 
     // Tensorflow output is in data_format order. (NHWC or NCHW)
@@ -271,7 +289,8 @@ class MklDnnConvUtil {
   //
   // Function does not return anything, but sets error in context status.
   inline void GetOutputAndPadSizeInMklOrder(
-      size_t src_index, size_t filter_index, const memory::dims& strides,
+      size_t src_index, size_t filter_index,
+      const memory::dims& strides, const memory::dims& dilations,
       memory::dims* output_dims_tf_order, memory::dims* output_dims_mkl_order,
       memory::dims* pad_l, memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -286,9 +305,9 @@ class MklDnnConvUtil {
                 errors::InvalidArgument("input must be 4-dimensional",
                                         input_tf_shape.DebugString()));
 
-    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape, strides,
-                                  output_dims_tf_order, output_dims_mkl_order,
-                                  pad_l, pad_r);
+    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape,
+                                  strides, dilations, output_dims_tf_order,
+                                  output_dims_mkl_order, pad_l, pad_r);
   }
 
   // Wrapper function to calculate input, filter, and output sizes of
@@ -300,12 +319,14 @@ class MklDnnConvUtil {
   inline void GetConvFwdSizesInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
       memory::dims* input_dims, memory::dims* filter_dims,
-      memory::dims* strides, memory::dims* output_dims_tf_order,
+      memory::dims* strides, memory::dims *dilations,
+      memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(input_dims);
     CHECK_NOTNULL(filter_dims);
     CHECK_NOTNULL(strides);
+    CHECK_NOTNULL(dilations);
     CHECK_NOTNULL(output_dims_tf_order);
     CHECK_NOTNULL(output_dims_mkl_order);
     CHECK_NOTNULL(pad_l);
@@ -316,7 +337,9 @@ class MklDnnConvUtil {
     GetFilterSizeInMklOrder(input_shape, filter_shape, filter_dims);
     if (!context_->status().ok()) return;
     GetStridesInMklOrder(strides);
-    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape, *strides,
+    GetDilationsInMklOrder(dilations);
+    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape,
+                                  *strides, *dilations,
                                   output_dims_tf_order, output_dims_mkl_order,
                                   pad_l, pad_r);
     if (!context_->status().ok()) return;
@@ -344,7 +367,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
         context, (stride_n == 1 && stride_c == 1),
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
-
+    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
+    OP_REQUIRES(context, dilations_.size() == 4,
+                errors::InvalidArgument("Sliding window dilations field must "
+                                        "specify 4 dimensions"));
+    int dilation_n = GetTensorDim(dilations_, data_format_, 'N');
+    int dilation_c = GetTensorDim(dilations_, data_format_, 'C');
+    int dilation_h = GetTensorDim(dilations_, data_format_, 'H');
+    int dilation_w = GetTensorDim(dilations_, data_format_, 'W');
+    OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1),
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "dilations in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, dilation_h > 0 && dilation_w > 0,
+        errors::InvalidArgument("Dilated rates should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
   }
 
@@ -406,15 +443,16 @@ class MklConv2DBackpropCommonOp : public OpKernel {
       // By default, all dims are in MKL order. Only dims in TF order
       // are those with prefix tf_order.
       memory::dims outbprop_dims, fwd_input_dims, fwd_filter_dims;
-      memory::dims padding_l, padding_r, strides, fwd_output_dims;
+      memory::dims padding_l, padding_r, dilations, strides, fwd_output_dims;
       memory::dims fwd_output_dims_tf_order;
 
       // Get forward convolution parameters.
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
+                             dilations_);
       conv_utl.GetConvFwdSizesInMklOrder(
           input_tf_shape, filter_tf_shape, &fwd_input_dims, &fwd_filter_dims,
-          &strides, &fwd_output_dims_tf_order, &fwd_output_dims, &padding_l,
-          &padding_r);
+          &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims,
+          &padding_l, &padding_r);
       if (!context->status().ok()) return;
 
       // Create Convolution forward descriptor since Convolution backward
@@ -437,10 +475,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
                                               memory::format::hwio);
       // Tensorflow Output of Conv2D is in data_format order.
       auto fwd_out_md = memory::desc(fwd_output_dims, MklDnnType<T>(), tf_fmt);
-      auto fwd_desc = convolution_forward::desc(
-          prop_kind::forward, convolution_direct, fwd_input_md, fwd_filter_md,
-          fwd_out_md, strides, padding_l, padding_r,
-          TFPaddingToMklDnnPadding(padding_));
+
+      const int kDilationH = 0, kDilationW = 1;
+      dilations[kDilationH] -= 1;
+      dilations[kDilationW] -= 1;
+      auto fwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0)?
+              convolution_forward::desc(prop_kind::forward,
+                     convolution_direct, fwd_input_md,
+                     fwd_filter_md, fwd_out_md,
+                     strides, dilations, padding_l, padding_r,
+                     TFPaddingToMklDnnPadding(padding_)) :
+              convolution_forward::desc(prop_kind::forward,
+                     convolution_direct, fwd_input_md,
+                     fwd_filter_md, fwd_out_md,
+                     strides, padding_l, padding_r,
+                     TFPaddingToMklDnnPadding(padding_));
       auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine);
 
       // Create memory for user data. Describe how the inputs and outputs of
@@ -485,8 +534,9 @@ class MklConv2DBackpropCommonOp : public OpKernel {
 
       // Operator-specific call to create and execute primitive.
       CreatePrimitive(context, cpu_engine, fwd_pd, &input, &filter,
-                      &outbackprop, &output, &output_tensor, strides, padding_l,
-                      padding_r, TFPaddingToMklDnnPadding(padding_),
+                      &outbackprop, &output, &output_tensor,
+                      strides, dilations, padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_),
                       bwd_output_dims, bwd_output_format);
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -535,20 +585,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
   virtual memory::format GetOutputFormat(const memory::format data_format) = 0;
 
   /// Create and execute the primitive storing output in the output_tensor.
-  virtual void CreatePrimitive(
-      OpKernelContext* context, const engine& cpu_engine,
-      const convolution_forward::primitive_desc& conv_fwd_pd,
-      MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
-      MklDnnData<T>* output, Tensor** output_tensor,
-      const memory::dims& strides, const memory::dims& padding_l,
-      const memory::dims& padding_r, padding_kind padding,
-      const memory::dims& bwd_output_dims,
-      memory::format bwd_output_format) = 0;
+  virtual void CreatePrimitive(OpKernelContext* context,
+    const engine& cpu_engine,
+    const convolution_forward::primitive_desc& conv_fwd_pd,
+    MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
+    MklDnnData<T>* output, Tensor** output_tensor, const memory::dims& strides,
+    const memory::dims& dilations, const memory::dims& padding_l,
+    const memory::dims& padding_r, padding_kind padding,
+    const memory::dims& bwd_output_dims,
+    memory::format bwd_output_format) = 0;
 
   // Get the data_format {NCHW, NHWC}
   TensorFormat GetTFDataFormat() { return data_format_; }
 
  private:
+  std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
   TensorFormat data_format_;
diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc
index e9a2376b54..d91f7107c5 100644
--- a/tensorflow/core/kernels/mkl_input_conversion_op.cc
+++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc
@@ -442,12 +442,11 @@ class MklInputConversionOp : public OpKernel {
       auto input_tf_md = mkl_output_mkl_shape.GetTfLayout();
       tf_input.SetUsrMem(input_tf_md, tf_tensor);
 
-      // Create reorder between tensorflow layout and Mkl layout.
+      // Create reorder between tensorflow layout and Mkl layout if necessary
       std::vector<primitive> net;
-      CHECK_EQ(tf_input.CheckReorderToOpMem(
+      tf_input.CheckReorderToOpMem(
                    memory::primitive_desc(output_mkl_md, cpu_engine),
-                   tensor_out, &net),
-               true);
+                   tensor_out, &net);
       stream(stream::kind::eager).submit(net).wait();
 
       // -- The tensor in MKL format passes through --
diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl_relu_op.cc
index 267f4f8d12..0a0f69522f 100644
--- a/tensorflow/core/kernels/mkl_relu_op.cc
+++ b/tensorflow/core/kernels/mkl_relu_op.cc
@@ -392,7 +392,7 @@ class MklReluOpBase : public OpKernel {
 
       Tensor* dst_tensor = nullptr;
       if (src_tensor.dims() == 0) {
-        Compute_Scalar(context);
+        Compute_Scalar(context); // scalar case doesn't use in-place operation
         return;
       }
 
@@ -437,11 +437,15 @@ class MklReluOpBase : public OpKernel {
         dnn_shape_dst.SetMklTensor(false);
         tf_shape_dst = src_tensor.shape();
       }
-      AllocateOutputSetMklShape(context, dst_index, &dst_tensor, tf_shape_dst,
-                                dnn_shape_dst);
+      
+      // Allocate output and MklDnnShape tensors separately for possible
+      // in-place operation
+      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                      {src_index}, dst_index, tf_shape_dst, &dst_tensor));
+      AllocateOutputSetMklShape(context, dst_index, dnn_shape_dst);
 
       // Destination memory descriptor is same as source memory descriptor.
-      auto dst_md = src_md;
+      auto &dst_md = src_md;
       dst.SetUsrMem(dst_md, dst_tensor);
 
       // execute net
@@ -492,7 +496,7 @@ class MklReluGradOpBase : public OpKernel {
 
       int src_dims_size = src_tensor.dims();
       if (src_dims_size == 0) {
-        Compute_Scalar(context);
+        Compute_Scalar(context); // scalar case doesn't use in-place operation
         return;
       }
 
@@ -603,8 +607,13 @@ class MklReluGradOpBase : public OpKernel {
         // so it is ok to get TensorFlow shape.
         tf_shape_diff_src = src_tensor.shape();
       }
-      AllocateOutputSetMklShape(context, diff_src_index, &diff_src_tensor,
-                                tf_shape_diff_src, dnn_shape_diff_src);
+
+      // Allocate diff_src and MklDnnShape tensors separately for possible
+      // in-place operation
+      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                      {diff_dst_index}, diff_src_index, tf_shape_diff_src,
+                                      &diff_src_tensor));
+      AllocateOutputSetMklShape(context, diff_src_index, dnn_shape_diff_src);
 
       // diff_src memory descriptor is same as memory descriptor for both
       // inputs.
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index 4abfbfb1a6..d0703d7576 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -13,8 +13,16 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
-#define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+
+
+// This file requires the following include because it uses CudaAtomicMax:
+// #include "tensorflow/core/util/cuda_kernel_helper.h"
+
+// Unfortunately we can't add the #include, since it breaks compilation for
+// non-GPU targets. This only breaks in clang, because it's more strict for
+// template code and CudaAtomicMax is used in template context.
 
 // This file requires the following include because it uses CudaAtomicMax:
 // #include "tensorflow/core/util/cuda_kernel_helper.h"
@@ -130,4 +138,4 @@ struct Highest {
 }  // namespace functor
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+#endif  // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
diff --git a/tensorflow/core/kernels/spacetodepth_op.cc b/tensorflow/core/kernels/spacetodepth_op.cc
index 23df1c35e5..e59adfc6ac 100644
--- a/tensorflow/core/kernels/spacetodepth_op.cc
+++ b/tensorflow/core/kernels/spacetodepth_op.cc
@@ -187,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     SpaceToDepthOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
+    SpaceToDepthOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     SpaceToDepthOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
index db05ca1ed2..f38459724a 100644
--- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
@@ -234,6 +234,12 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// Instantiate the GPU implementations for Eigen::half.
+template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NCHW>;
+template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NHWC>;
+
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 5bd79778a6..0b006fa2b4 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -55,6 +55,4 @@ StringPiece StringPiece::substr(size_t pos, size_t n) const {
   return StringPiece(data_ + pos, n);
 }
 
-const StringPiece::size_type StringPiece::npos = size_type(-1);
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 910e4d9e2a..2d00f717dc 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -65,7 +65,7 @@ class StringPiece {
   iterator begin() const { return data_; }
   iterator end() const { return data_ + size_; }
 
-  static const size_t npos;
+  static const size_t npos = size_type(-1);
 
   // Return the ith byte in the referenced data.
   // REQUIRES: n < size()
diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc
index 254fdf115d..6de850bb20 100644
--- a/tensorflow/core/lib/io/record_reader.cc
+++ b/tensorflow/core/lib/io/record_reader.cc
@@ -205,7 +205,9 @@ Status RecordReader::SkipNBytes(uint64 offset) {
     if (options_.buffer_size > 0) {
       TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset));
     }
+#if !defined(IS_SLIM_BUILD)
   }
+#endif
   return Status::OK();
 }  // namespace io
 
diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h
index 62dd2efb79..26278e0328 100644
--- a/tensorflow/core/lib/io/record_reader.h
+++ b/tensorflow/core/lib/io/record_reader.h
@@ -16,10 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_RECORD_READER_H_
 #define TENSORFLOW_LIB_IO_RECORD_READER_H_
 
-#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
-#if !defined(IS_SLIM_BUILD)
 #include "tensorflow/core/lib/io/inputstream_interface.h"
+#if !defined(IS_SLIM_BUILD)
 #include "tensorflow/core/lib/io/zlib_compression_options.h"
 #include "tensorflow/core/lib/io/zlib_inputstream.h"
 #endif  // IS_SLIM_BUILD
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index bdbbf6d7c3..9a4b616e5d 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -265,6 +265,16 @@ REGISTER_OP("BatchDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
+// TODO(mrry): move SlideDataset to contrib in the future.
+REGISTER_OP("SlideDataset")
+    .Input("input_dataset: variant")
+    .Input("window_size: int64")
+    .Input("stride: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("PaddedBatchDataset")
     .Input("input_dataset: variant")
     .Input("batch_size: int64")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 910fbaca9e..d6a0f38033 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1498,6 +1498,7 @@ REGISTER_OP("_MklConv2D")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn(shape_inference::Conv2DShape)
     .Doc(R"doc(
 MKL version of Conv2D operator. Uses MKL DNN APIs to perform 2D convolution.
@@ -1516,6 +1517,7 @@ REGISTER_OP("__MklDummyConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 Dummy node that enables fusing Conv2D and BiasAdd operator for MKL. This node
 does not perform anything. It is just created as an intermediate output of
@@ -1541,6 +1543,7 @@ REGISTER_OP("_MklConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2D and BiasAdd operator. Uses MKL DNN APIs to perform
 2D convolution and add Bias to the output of convolution.
@@ -1563,6 +1566,7 @@ REGISTER_OP("_MklConv2DBackpropFilter")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
@@ -1589,6 +1593,7 @@ REGISTER_OP("__MklDummyConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1633,6 +1638,7 @@ REGISTER_OP("_MklConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1668,6 +1674,7 @@ REGISTER_OP("_MklConv2DWithBiasBackpropBias")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2DBackpropBias. Uses MKL DNN APIs to compute the
 gradients of convolution with respect to the bias.
@@ -1690,6 +1697,7 @@ REGISTER_OP("_MklConv2DBackpropInput")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h
index 8f7bff1bb0..eebbeaeba6 100644
--- a/tensorflow/core/platform/tracing.h
+++ b/tensorflow/core/platform/tracing.h
@@ -103,7 +103,7 @@ class Tracing {
   friend class ScopedAnnotation;
   friend class TraceMe;
 
-  static std::atomic<Tracing::Engine*> tracing_engine_;
+  TF_EXPORT static std::atomic<Tracing::Engine*> tracing_engine_;
   static Tracing::Engine* engine() {
     return tracing_engine_.load(std::memory_order_acquire);
   }
diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc
index b6b3722caa..682e46e0fc 100644
--- a/tensorflow/core/platform/windows/windows_file_system.cc
+++ b/tensorflow/core/platform/windows/windows_file_system.cc
@@ -382,7 +382,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
 
 Status WindowsFileSystem::FileExists(const string& fname) {
   constexpr int kOk = 0;
-  if (_access(TranslateName(fname).c_str(), kOk) == 0) {
+  std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
+  if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
     return Status::OK();
   }
   return errors::NotFound(fname, " not found");
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 7405e01e14..22f2c02b78 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX "-rc1"
+#define TF_VERSION_SUFFIX ""
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md
index 9f6fe91b14..6d0458e678 100644
--- a/tensorflow/docs_src/community/welcome.md
+++ b/tensorflow/docs_src/community/welcome.md
@@ -51,6 +51,8 @@ Europe:
 TensorFlow provides multiple communication paths.  To pick the right path,
 please read the following list carefully:
 
+  * For new release announcements and security updates, subscribe to
+    [announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
   * To ask or answer technical questions about TensorFlow, use
     [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).
     For example, ask or search Stack Overflow about a particular error message
@@ -65,5 +67,5 @@ please read the following list carefully:
     on GitHub.  For example, use the issue tracker to request a
     new operation in TensorFlow.
   * To report vulnerabilities, please follow our
-    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/SECURITY.md).
+    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 818798555a..0481c97885 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 4c6dfa8daf..8f89898c92 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 527884863e..0ee9c849e1 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0-rc1</version>
+                 <version>1.6.0</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0-rc1.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0-rc1.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>d
+<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index e3e115d9f6..3e8744bf9d 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -41,7 +41,8 @@ must be installed on your system:
     [NVIDIA's documentation](https://developer.nvidia.com/cudnn).
     Ensure that you create the `CUDA_HOME` environment variable as
     described in the NVIDIA documentation.
-  * GPU card with CUDA Compute Capability 3.0 or higher.  See
+  * GPU card with CUDA Compute Capability 3.0 or higher for building
+    from source and 3.5 or higher for our binaries. See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for
     a list of supported GPU cards.
   * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
@@ -188,7 +189,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -293,7 +294,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -356,24 +357,23 @@ where:
     to 6006.
   * <tt><i>TensorFlowCPUImage</i></tt> is required. It identifies the Docker
     container. Specify one of the following values:
-    * <tt>gcr.io/tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:latest-devel</tt>, which is the latest
+    * <tt>tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
+    * <tt>tensorflow/tensorflow:latest-devel</tt>, which is the latest
       TensorFlow CPU Binary image plus source code.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i></tt>, which is the
+    * <tt>tensorflow/tensorflow:<i>version</i></tt>, which is the
       specified version (for example, 1.1.0rc1) of TensorFlow CPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
+    * <tt>tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
       the specified version (for example, 1.1.0rc1) of the TensorFlow GPU
       binary image plus source code.
 
-    <tt>gcr.io</tt> is the Google Container Registry. Note that some
-    TensorFlow images are also available at
+    TensorFlow images are available at
     [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches the latest TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
+$ <b>docker run -it tensorflow/tensorflow bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow CPU binary image in a
@@ -381,7 +381,7 @@ Docker container. However, in this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b>
+$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -405,14 +405,14 @@ where:
     <tt><i>hostPort</i></tt> and <code><em>containerPort</em></code> to `8888`.
   * <i>TensorFlowGPUImage</i> specifies the Docker container. You must
     specify one of the following values:
-    * <tt>gcr.io/tensorflow/tensorflow:latest-gpu</tt>, which is the latest
+    * <tt>tensorflow/tensorflow:latest-gpu</tt>, which is the latest
       TensorFlow GPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:latest-devel-gpu</tt>, which is
+    * <tt>tensorflow/tensorflow:latest-devel-gpu</tt>, which is
       the latest TensorFlow GPU Binary image plus source code.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
+    * <tt>tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
       specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
+    * <tt>tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
       the specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image plus source code.
 
@@ -421,7 +421,7 @@ following command launches the latest TensorFlow GPU binary image in a
 Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>nvidia-docker run -it gcr.io/tensorflow/tensorflow:latest-gpu bash</b>
+$ <b>nvidia-docker run -it tensorflow/tensorflow:latest-gpu bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow GPU binary image
@@ -429,13 +429,13 @@ in a Docker container. In this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:latest-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:latest-gpu</b>
 </pre>
 
 The following command installs an older TensorFlow version (0.12.1):
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:0.12.1-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:0.12.1-gpu</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -480,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -505,7 +505,7 @@ If you installed through Docker, start a Docker container
 from which you can run bash. For example:
 
 <pre>
-$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
+$ <b>docker run -it tensorflow/tensorflow bash</b>
 </pre>
 
 
@@ -647,14 +647,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -666,14 +666,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -685,14 +685,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -704,14 +704,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 623ca6bb79..205db8e6bd 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -292,24 +292,23 @@ where:
     to 6006.
   * <i>TensorFlowImage</i> is required. It identifies the Docker container.
     You must specify one of the following values:
-    * <code>gcr.io/tensorflow/tensorflow</code>: TensorFlow binary image.
-    * <code>gcr.io/tensorflow/tensorflow:latest-devel</code>: TensorFlow
+    * <code>tensorflow/tensorflow</code>: TensorFlow binary image.
+    * <code>tensorflow/tensorflow:latest-devel</code>: TensorFlow
       Binary image plus source code.
 
-<code>gcr.io</code> is the Google Container Registry. Note that some
-TensorFlow images are also available at
+The TensorFlow images are available at
 [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches a TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
-<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
 
 The following command also launches a TensorFlow CPU binary image in a
 Docker container. However, in this Docker container, you can run
 TensorFlow programs in a Jupyter notebook:
 
-<pre>$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b></pre>
+<pre>$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b></pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
 
@@ -351,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -376,7 +375,7 @@ do the following:
 If you installed through Docker, start a Docker container that runs bash.
 For example:
 
-<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
 
 
 
@@ -524,7 +523,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
 </pre>
 
 
@@ -532,5 +531,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index acf0af0d9d..c09c9c2c0c 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -359,10 +359,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0rc1 on Linux:
+for TensorFlow 1.6.0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0rc1-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index f0a30ee394..2413bc9cfb 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -17,7 +17,7 @@ You must choose one of the following types of TensorFlow to install:
     NVIDIA® GPU, you must install this version. Note that this version of
     TensorFlow is typically much easier to install (typically,
     in 5 or 10 minutes), so even if you have an NVIDIA GPU, we recommend
-    installing this version first.
+    installing this version first. Prebuilt binaries will use AVX instructions. 
   * **TensorFlow with GPU support**. TensorFlow programs typically run
     significantly faster on a GPU than on a CPU. Therefore, if your
     system has a NVIDIA® GPU meeting the prerequisites shown below
@@ -41,7 +41,8 @@ installed on your system:
     Note that cuDNN is typically installed in a different location from the
     other CUDA DLLs. Ensure that you add the directory where you installed
     the cuDNN DLL to your `%PATH%` environment variable.
-  * GPU card with CUDA Compute Capability 3.0 or higher.  See
+  * GPU card with CUDA Compute Capability 3.0 or higher for building
+    from source and 3.5 or higher for our binaries. See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a
     list of supported GPU cards.
 
diff --git a/tensorflow/docs_src/performance/xla/jit.md b/tensorflow/docs_src/performance/xla/jit.md
index d4dc3e57c8..d9a979ccbd 100644
--- a/tensorflow/docs_src/performance/xla/jit.md
+++ b/tensorflow/docs_src/performance/xla/jit.md
@@ -157,7 +157,7 @@ to fuse Ops is visible by starting at `hlo_graph_0.dot` and viewing each diagram
 in succession.
 
 To Render the .dot file into a png, install
-[GraphViz](http://www.graphviz.org/Download..php) and run:
+[GraphViz](https://www.graphviz.org/download/) and run:
 
 ```shell
 dot -Tpng hlo_graph_80.dot -o hlo_graph_80.png
diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 5fb1c2da88..d1399814ee 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -459,7 +459,7 @@ accuracy_score = classifier.evaluate(x=test_set.data,
 
 
 [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py),
-based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to
+based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.2/get_started/tflearn), contains a full example of how to
 use the tfdbg with `Estimator`s. To run this example, do:
 
 ```none
@@ -753,6 +753,7 @@ There are three possible workarounds or solutions:
    # For LocalCLIDebugHook
    hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")]
    ```
+
    Make sure that the directory pointed to by dump_root is empty or nonexistent.
    tfdbg cleans up the dump directories before exiting.
 *  Reduce the batch size used during the runs.
diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md
index 1548d43877..392ac6f7f1 100644
--- a/tensorflow/docs_src/programmers_guide/faq.md
+++ b/tensorflow/docs_src/programmers_guide/faq.md
@@ -159,7 +159,7 @@ available. These operations allow you to build sophisticated
 @{$reading_data$input pipelines}, at the cost of making the
 TensorFlow computation somewhat more complicated. See the how-to documentation
 for
-@{$reading_data#creating-threads-to-prefetch-using-queuerunner-objects$using `QueueRunner` objects to drive queues and readers}
+@{$reading_data#creating_threads_to_prefetch_using_queuerunner_objects$using `QueueRunner` objects to drive queues and readers}
 for more information on how to use them.
 
 ## Variables
@@ -272,7 +272,7 @@ Prefer predefined TensorFlow operations such as @{tf.decode_raw},
 
 If your data is not easily parsable with the built-in TensorFlow operations,
 consider converting it, offline, to a format that is easily parsable, such
-as ${tf.python_io.TFRecordWriter$`TFRecord`} format.
+as @{tf.python_io.TFRecordWriter$`TFRecord`} format.
 
 The more efficient method to customize the parsing behavior is to
 @{$adding_an_op$add a new op written in C++} that parses your
diff --git a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
index 79280d246a..fadfa03e78 100644
--- a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
+++ b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
@@ -83,7 +83,7 @@ data than you need, though. Instead, consider running the merged summary op
 every `n` steps.
 
 The code example below is a modification of the
-@{$layers$simple MNIST tutorial},
+[simple MNIST tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py),
 in which we have added some summary ops, and run them every ten steps. If you
 run this and then launch `tensorboard --logdir=/tmp/tensorflow/mnist`, you'll be able
 to visualize statistics, such as how the weights or accuracy varied during
diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md
index d74d7f3181..a9c2cb3e33 100644
--- a/tensorflow/docs_src/programmers_guide/using_tpu.md
+++ b/tensorflow/docs_src/programmers_guide/using_tpu.md
@@ -129,10 +129,9 @@ my_tpu_estimator = tf.contrib.tpu.TPUEstimator(
 Typically the `FLAGS` would be set by command line arguments. To switch from
 training locally to training on a cloud TPU you would need to:
 
-  1) Set `FLAGS.use_tpu` to `True`
-  1) Set `FLAGS.tpu_name` so the
-     `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
-  1) Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
+* Set `FLAGS.use_tpu` to `True`
+* Set `FLAGS.tpu_name` so the `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
+* Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
 
 
 ## Optimizer
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index ee03f440c9..9b17d0d4d5 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -193,7 +193,7 @@ to calculate loss, configure the training op, and generate predictions. If
 you're already experienced with CNNs and @{$get_started/custom_estimators$TensorFlow `Estimator`s},
 and find the above code intuitive, you may want to skim these sections or just
 skip ahead to ["Training and Evaluating the CNN MNIST
-Classifier"](#training-and-evaluating-the-cnn-mnist-classifier).
+Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
 
 ### Input Layer
 
@@ -446,7 +446,7 @@ tf.nn.softmax(logits, name="softmax_tensor")
 
 > Note: We use the `name` argument to explicitly name this operation
 > `softmax_tensor`, so we can reference it later. (We'll set up logging for the
-> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook).
+> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook)).
 
 We compile our predictions in a dict, and return an `EstimatorSpec` object:
 
@@ -534,9 +534,8 @@ if mode == tf.estimator.ModeKeys.TRAIN:
 ```
 
 > Note: For a more in-depth look at configuring training ops for Estimator model
-> functions, see @{$get_started/custom_estimators#defining-the-training-op-for-the-model$"Defining
-> the training op for the model"} in the @{$get_started/custom_estimators$"Creating Estimations in
-> tf.estimator"} tutorial.
+> functions, see @{$get_started/custom_estimators#defining_the_training_op_for_the_model$"Defining the training op for the model"} 
+> in the @{$get_started/custom_estimators$"Creating Estimators in tf.estimator."} tutorial.
 
 ### Add evaluation metrics
 
@@ -625,7 +624,8 @@ operation earlier when we generated the probabilities in `cnn_model_fn`.
 > Note: If you don't explicitly assign a name to an operation via the `name`
 > argument, TensorFlow will assign a default name. A couple easy ways to
 > discover the names applied to operations are to visualize your graph on
-> @{$graph_viz$TensorBoard}) or to enable the @{$debugger$TensorFlow Debugger (tfdbg)}.
+> @{$graph_viz$TensorBoard}) or to enable the
+> @{$programmers_guide/debugger$TensorFlow Debugger (tfdbg)}.
 
 Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the
 `tensors` argument. We set `every_n_iter=50`, which specifies that probabilities
diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index e22536adb6..7584a76ba5 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -109,7 +109,8 @@ This download will take a while and download a bit more than 23GB of data.
 
 To convert the `ndjson` files to
 @{$python/python_io#tfrecords_format_details$TFRecord} files containing
-${tf.train.Example} protos run the following command.
+[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+protos run the following command.
 
 ```shell
    python create_dataset.py --ndjson_path rnn_tutorial_data \
diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 005dc020f9..27ce75a30d 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -74,8 +74,8 @@ Here's a list of columns available in the Census Income dataset:
 | relationship   | Categorical | Wife, Own-child, Husband,         |
 :                :             : Not-in-family, Other-relative,    :
 :                :             : Unmarried.                        :
-| race           | Categorical | White, Asian-Pac-Islander,        |
-:                :             : Amer-Indian-Eskimo, Other, Black. :
+| race           | Categorical | Amer-Indian-Eskimo, Asian-Pac-    |
+:                :             : Islander, Black, White, Other.    :
 | gender         | Categorical | Female, Male.                     |
 | capital_gain   | Continuous  | Capital gains recorded.           |
 | capital_loss   | Continuous  | Capital Losses recorded.          |
@@ -247,7 +247,7 @@ hours_per_week = tf.feature_column.numeric_column('hours_per_week')
 ### Making Continuous Features Categorical through Bucketization
 
 Sometimes the relationship between a continuous feature and the label is not
-linear. As an hypothetical example, a person's income may grow with age in the
+linear. As a hypothetical example, a person's income may grow with age in the
 early stage of one's career, then the growth may slow at some point, and finally
 the income decreases after retirement. In this scenario, using the raw `age` as
 a real-valued feature column might not be a good choice because the model can
@@ -361,6 +361,16 @@ The first line of the final output should be something like
 `accuracy: 0.83557522`, which means the accuracy is 83.6%. Feel free to try more
 features and transformations and see if you can do even better!
 
+After the model is evaluated, we can use the model to predict whether an individual has an annual income of over
+50,000 dollars given an individual's information input.
+```python
+  pred_iter = model.predict(input_fn=lambda: input_fn(FLAGS.test_data, 1, False, 1))
+  for pred in pred_iter:
+    print(pred['classes'])
+```
+
+The model prediction output would be like `[b'1']` or `[b'0']` which means whether corresponding individual has an annual income of over 50,000 dollars or not.
+
 If you'd like to see a working end-to-end example, you can download our
 [example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py)
 and set the `model_type` flag to `wide`.
diff --git a/tensorflow/examples/android/AndroidManifest.xml b/tensorflow/examples/android/AndroidManifest.xml
index bb75431a1f..5c47ce6b67 100644
--- a/tensorflow/examples/android/AndroidManifest.xml
+++ b/tensorflow/examples/android/AndroidManifest.xml
@@ -40,6 +40,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -49,6 +50,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -58,6 +60,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -67,6 +70,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
     </application>
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
index 8bd4abb154..429138abe5 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
@@ -351,6 +351,10 @@ public abstract class CameraActivity extends Activity
 
   protected void setFragment() {
     String cameraId = chooseCamera();
+    if (cameraId == null) {
+      Toast.makeText(this, "No Camera Detected", Toast.LENGTH_SHORT).show();
+      finish();
+    }
 
     Fragment fragment;
     if (useCamera2API) {
@@ -416,7 +420,8 @@ public abstract class CameraActivity extends Activity
 
   @Override
   public boolean onKeyDown(final int keyCode, final KeyEvent event) {
-    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP) {
+    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP
+            || keyCode == KeyEvent.KEYCODE_BUTTON_L1 || keyCode == KeyEvent.KEYCODE_DPAD_CENTER) {
       debug = !debug;
       requestRender();
       onSetDebug(debug);
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
index 6a66ec3927..33ec65e9f7 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
@@ -16,8 +16,10 @@
 
 package org.tensorflow.demo;
 
+import android.app.UiModeManager;
 import android.content.Context;
 import android.content.res.AssetManager;
+import android.content.res.Configuration;
 import android.graphics.Bitmap;
 import android.graphics.Bitmap.Config;
 import android.graphics.BitmapFactory;
@@ -31,9 +33,11 @@ import android.graphics.Typeface;
 import android.media.ImageReader.OnImageAvailableListener;
 import android.os.Bundle;
 import android.os.SystemClock;
+import android.util.DisplayMetrics;
 import android.util.Size;
 import android.util.TypedValue;
 import android.view.Display;
+import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.View.OnClickListener;
@@ -43,6 +47,7 @@ import android.widget.BaseAdapter;
 import android.widget.Button;
 import android.widget.GridView;
 import android.widget.ImageView;
+import android.widget.RelativeLayout;
 import android.widget.Toast;
 import java.io.IOException;
 import java.io.InputStream;
@@ -381,6 +386,27 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
     grid = (GridView) findViewById(R.id.grid_layout);
     grid.setAdapter(adapter);
     grid.setOnTouchListener(gridTouchAdapter);
+
+    // Change UI on Android TV
+    UiModeManager uiModeManager = (UiModeManager) getSystemService(UI_MODE_SERVICE);
+    if (uiModeManager.getCurrentModeType() == Configuration.UI_MODE_TYPE_TELEVISION) {
+      DisplayMetrics displayMetrics = new DisplayMetrics();
+      getWindowManager().getDefaultDisplay().getMetrics(displayMetrics);
+      int styleSelectorHeight = displayMetrics.heightPixels;
+      int styleSelectorWidth = displayMetrics.widthPixels - styleSelectorHeight;
+      RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(styleSelectorWidth, ViewGroup.LayoutParams.MATCH_PARENT);
+
+      // Calculate number of style in a row, so all the style can show up without scrolling
+      int numOfStylePerRow = 3;
+      while (styleSelectorWidth / numOfStylePerRow * Math.ceil((float) (adapter.getCount() - 2) / numOfStylePerRow) > styleSelectorHeight) {
+        numOfStylePerRow++;
+      }
+      grid.setNumColumns(numOfStylePerRow);
+      layoutParams.addRule(RelativeLayout.ALIGN_PARENT_RIGHT);
+      grid.setLayoutParams(layoutParams);
+      adapter.buttons.clear();
+    }
+
     setStyle(adapter.items[0], 1.0f);
   }
 
@@ -602,4 +628,38 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
 
     borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines);
   }
+
+  @Override
+  public boolean onKeyDown(int keyCode, KeyEvent event) {
+    int moveOffset = 0;
+    switch (keyCode) {
+      case KeyEvent.KEYCODE_DPAD_LEFT:
+        moveOffset = -1;
+        break;
+      case KeyEvent.KEYCODE_DPAD_RIGHT:
+        moveOffset = 1;
+        break;
+      case KeyEvent.KEYCODE_DPAD_UP:
+        moveOffset = -1 * grid.getNumColumns();
+        break;
+      case KeyEvent.KEYCODE_DPAD_DOWN:
+        moveOffset = grid.getNumColumns();
+        break;
+      default:
+        return super.onKeyDown(keyCode, event);
+    }
+
+    // get the highest selected style
+    int currentSelect = 0;
+    float highestValue = 0;
+    for (int i = 0; i < adapter.getCount(); i++) {
+      if (adapter.items[i].value > highestValue) {
+        currentSelect = i;
+        highestValue = adapter.items[i].value;
+      }
+    }
+    setStyle(adapter.items[(currentSelect + moveOffset + adapter.getCount()) % adapter.getCount()], 1);
+
+    return true;
+  }
 }
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a206685af6..3cbeb34c54 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -28,6 +28,7 @@ load("//tensorflow:tensorflow.bzl", "py_tests")
 load("//tensorflow:tensorflow.bzl", "tf_py_build_info_genrule")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_library_additional_deps_impl")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
 load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
@@ -106,20 +107,19 @@ py_library(
         ":training",
         ":util",
         ":weights_broadcast_ops",
-        "//third_party/py/numpy",
+        "//tensorflow/contrib:contrib_py",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras",
-        "//tensorflow/python/ops/losses",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
+        "//tensorflow/python/ops/losses",
         "//tensorflow/python/profiler",
         "//tensorflow/python/saved_model",
-    ] + if_not_windows([
-        "//tensorflow/contrib:contrib_py",
-    ]),
+        "//third_party/py/numpy",
+    ],
 )
 
 tf_py_build_info_genrule()
@@ -947,7 +947,6 @@ py_test(
     srcs = ["framework/contrib_test.py"],
     main = "framework/contrib_test.py",
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
@@ -1312,7 +1311,6 @@ py_test(
     srcs = ["framework/dtypes_test.py"],
     main = "framework/dtypes_test.py",
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":framework_for_generated_wrappers",
         ":framework_test_lib",
@@ -1654,7 +1652,6 @@ py_test(
     size = "small",
     srcs = ["ops/clip_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":clip_ops",
@@ -2721,7 +2718,6 @@ cuda_py_test(
     ],
     data = ["//tensorflow/core:image_testdata"],
     shard_count = 5,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -3305,6 +3301,65 @@ tf_py_wrap_cc(
          tf_additional_gdr_deps()),
 )
 
+# ** Targets for Windows build (start) **
+# We need the following targets to expose symbols from _pywrap_tensorflow.dll
+
+# Build a cc_binary from tf_custom_op_library_additional_deps_impl,
+# it contains all object code from its dependencies.
+cc_binary(
+    name = "tf_custom_op_library_additional_deps.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = tf_custom_op_library_additional_deps_impl(),
+)
+
+# Get a DEF file generated by parsing all object files
+# of tf_custom_op_library_additional_deps.so
+filegroup(
+    name = "pywrap_tensorflow_def_file",
+    srcs = [":tf_custom_op_library_additional_deps.so"],
+    output_group = "def_file",
+)
+
+# Filter the DEF file to reduce the number of symbols to 64K or less.
+# Note that we also write the name of the pyd file into DEF file so that
+# the dynamic libraries of custom ops can find it at runtime.
+genrule(
+    name = "pywrap_tensorflow_filtered_def_file",
+    srcs = [":pywrap_tensorflow_def_file"],
+    outs = ["pywrap_tensorflow_filtered_def_file.def"],
+    cmd = select({
+        "//tensorflow:windows": """
+              $(location @local_config_def_file_filter//:def_file_filter) \\
+              --input $(location :pywrap_tensorflow_def_file) \\
+              --output $@ \\
+              --target _pywrap_tensorflow_internal.pyd
+          """,
+        "//conditions:default": "touch $@",  # Just a placeholder for Unix platforms
+    }),
+    tools = ["@local_config_def_file_filter//:def_file_filter"],
+)
+
+# Get the import library of  _pywrap_tensorflow_internal.dll
+filegroup(
+    name = "pywrap_tensorflow_import_lib_file",
+    srcs = [":_pywrap_tensorflow_internal.so"],
+    output_group = "interface_library",
+)
+
+# Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll
+# so that custom ops' dynamic libraries can link against it.
+cc_import(
+    name = "pywrap_tensorflow_import_lib",
+    interface_library = select({
+        "//tensorflow:windows": ":pywrap_tensorflow_import_lib_file",
+        "//conditions:default": "not_exsiting_on_unix.lib",  # Just a placeholder for Unix platforms
+    }),
+    system_provided = 1,
+)
+
+# ** Targets for Windows build (end) **
+
 py_library(
     name = "lib",
     srcs = [
@@ -3677,7 +3732,6 @@ py_test(
     size = "small",
     srcs = ["lib/core/bfloat16_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":lib",
@@ -3958,7 +4012,11 @@ py_test(
     srcs = ["training/checkpoint_utils_test.py"],
     srcs_version = "PY2AND3",
     tags = [
+        "manual",
+        "no_cuda_on_cpu_tap",
+        "no_oss",
         "no_windows",
+        "notap",
     ],
     deps = [
         ":client",
@@ -3981,7 +4039,6 @@ py_test(
     size = "small",
     srcs = ["training/checkpoint_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":checkpoint_ops_gen",
         ":client",
@@ -4022,10 +4079,7 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",
-        "notsan",  # b/67945581
-    ],
+    tags = ["notsan"],  # b/67945581
     deps = [
         ":array_ops",
         ":client_testlib",
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 512d292ee2..c60f692390 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -913,6 +913,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 9fcbd4ff77..5245a050a1 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -139,8 +139,8 @@ class Estimator(object):
                  to configure Estimators from hyper parameter tuning.
           * `config`: Optional configuration object. Will receive what is passed
                  to Estimator in `config` parameter, or the default `config`.
-                 Allows updating things in your model_fn based on configuration
-                 such as `num_ps_replicas`, or `model_dir`.
+                 Allows updating things in your `model_fn` based on
+                 configuration such as `num_ps_replicas`, or `model_dir`.
 
         * Returns:
           `EstimatorSpec`
@@ -301,11 +301,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
+          * A tuple (features, labels): Where `features` is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and `labels` is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            features and labels are consumed by `model_fn`. They should satisfy
-            the expectation of `model_fn` from inputs.
+            `features` and `labels` are consumed by `model_fn`. They should
+            satisfy the expectation of `model_fn` from inputs.
 
       hooks: List of `SessionRunHook` subclass instances. Used for callbacks
         inside the training loop.
@@ -381,11 +381,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
+          * A tuple (features, labels): Where `features` is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and `labels` is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            features and labels are consumed by `model_fn`. They should satisfy
-            the expectation of `model_fn` from inputs.
+            `features` and `labels` are consumed by `model_fn`. They should
+            satisfy the expectation of `model_fn` from inputs.
 
       steps: Number of steps for which to evaluate model. If `None`, evaluates
         until `input_fn` raises an end-of-input exception.
@@ -457,17 +457,17 @@ class Estimator(object):
       checkpoint_path: Path of a specific checkpoint to predict. If `None`, the
         latest checkpoint in `model_dir` is used.
       yield_single_examples: If False, yield the whole batch as returned by the
-        model_fn instead of decomposing the batch into individual elements. This
-        is useful if model_fn return some tensor with first dimension not
-        equal to the batch size
+        `model_fn` instead of decomposing the batch into individual elements.
+        This is useful if `model_fn` returns some tensors whose first dimension
+        is not equal to the batch size.
 
     Yields:
       Evaluated values of `predictions` tensors.
 
     Raises:
-      ValueError: Could not find a trained model in model_dir.
-      ValueError: if batch length of predictions are not same and
-        yield_single_examples is True.
+      ValueError: Could not find a trained model in `model_dir`.
+      ValueError: If batch length of predictions is not the same and
+        `yield_single_examples` is True.
       ValueError: If there is a conflict between `predict_keys` and
         `predictions`. For example if `predict_keys` is not `None` but
         `EstimatorSpec.predictions` is not a `dict`.
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 2cc3331a15..e38b765da5 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -128,9 +128,16 @@ class TrainSpec(
     """Creates a validated `TrainSpec` instance.
 
     Args:
-      input_fn: Training input function returning a tuple of:
-          features - `Tensor` or dictionary of string feature name to `Tensor`.
-          labels - `Tensor` or dictionary of `Tensor` with labels.
+      input_fn: A function that provides input data for training as minibatches.
+        See @{$get_started/premade_estimators#create_input_functions} for more
+        information. The function should construct and return one of
+        the following:
+          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
+            tuple (features, labels) with same constraints as below.
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
+            `Tensor` or a dictionary of string label name to `Tensor`.
+            
       max_steps: Int. Positive number of total steps for which to train model.
         If `None`, train forever. The training `input_fn` is not expected to
         generate `OutOfRangeError` or `StopIteration` exceptions. See the
@@ -185,9 +192,16 @@ class EvalSpec(
     """Creates a validated `EvalSpec` instance.
 
     Args:
-      input_fn: Evaluation input function returning a tuple of:
-          features - `Tensor` or dictionary of string feature name to `Tensor`.
-          labels - `Tensor` or dictionary of `Tensor` with labels.
+      input_fn: A function that constructs the input data for evaluation.
+        See @{$get_started/premade_estimators#create_input_functions} for more
+        information. The function should construct and return one of
+        the following:
+          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
+            tuple (features, labels) with same constraints as below.
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
+            `Tensor` or a dictionary of string label name to `Tensor`.
+            
       steps: Int. Positive number of steps for which to evaluate model. If
         `None`, evaluates until `input_fn` raises an end-of-input exception.
         See `Estimator.evaluate` for details.
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index eef91e9c5b..f74881f179 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -636,7 +636,10 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/utils/io_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 5b0c38fa5d..f27ca5c205 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -295,7 +295,6 @@ tf_py_test(
         "//tensorflow/python:nn_grad",
     ],
     data = ["//tensorflow/core:image_testdata"],
-    tags = ["no_windows"],
 )
 
 tf_py_test(
@@ -1139,7 +1138,6 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     data = ["//tensorflow/core:lmdb_testdata"],
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2329,7 +2327,6 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 4,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2460,7 +2457,6 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index f4fe01f868..25525cc128 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -970,7 +970,7 @@ class Conv2DTest(test.TestCase):
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 6, 1],
@@ -984,7 +984,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -998,7 +998,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1012,7 +1012,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 4, 3],
@@ -1026,7 +1026,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 3, 1],
@@ -1040,7 +1040,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 6, 1],
@@ -1054,7 +1054,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1068,7 +1068,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[0, 2, 3, 1],
@@ -1082,7 +1082,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputDilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         # The GPU version of this test is not very stable. So adjusting the
         # error threshold to 1e-4.
@@ -1098,7 +1098,7 @@ class Conv2DTest(test.TestCase):
             err=1e-4)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropInputDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 3, 1],
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 96c9718b83..f0beabb4e2 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -35,8 +35,8 @@ from tensorflow.python.platform import tf_logging
 
 class DepthToSpaceTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs):
-    input_nhwc = math_ops.to_float(inputs)
+  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
+    input_nhwc = math_ops.cast(inputs, dtype)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.depth_to_space(input_nhwc, block_size)
@@ -59,6 +59,12 @@ class DepthToSpaceTest(test.TestCase):
     x_out = [[[[1], [2]], [[3], [4]]]]
     self._testOne(x_np, block_size, x_out)
 
+  def testBasicFloat16(self):
+    x_np = [[[[1, 2, 3, 4]]]]
+    block_size = 2
+    x_out = [[[[1], [2]], [[3], [4]]]]
+    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
+
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testBlockSize2(self):
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index b76135764f..cd90d16aac 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -34,8 +34,8 @@ from tensorflow.python.platform import tf_logging
 
 class SpaceToDepthTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs):
-    input_nhwc = math_ops.to_float(inputs)
+  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
+    input_nhwc = math_ops.cast(inputs, dtype)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
@@ -58,6 +58,12 @@ class SpaceToDepthTest(test.TestCase):
     x_out = [[[[1, 2, 3, 4]]]]
     self._testOne(x_np, block_size, x_out)
 
+  def testBasicFloat16(self):
+    x_np = [[[[1], [2]], [[3], [4]]]]
+    block_size = 2
+    x_out = [[[[1, 2, 3, 4]]]]
+    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
+
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testLargerInput2x2(self):
diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py
index a751607aaa..223858edfa 100644
--- a/tensorflow/python/lib/io/file_io_test.py
+++ b/tensorflow/python/lib/io/file_io_test.py
@@ -485,6 +485,11 @@ class FileIoTest(test.TestCase):
     f.flush()
     self.assertEqual(content, f.read(len(content) + 1))
 
+  def testUTF8StringPathExists(self):
+    file_path = os.path.join(self._base_dir, "UTF8测试_file_exist")
+    file_io.write_string_to_file(file_path, "testing")
+    v = file_io.file_exists(file_path)
+    self.assertEqual(v, True)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index fb3fe77b4d..87fe253f18 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -699,7 +699,7 @@ def convolution(
   `padded_input` is obtained by zero padding the input using an effective
   spatial filter shape of `(spatial_filter_shape-1) * dilation_rate + 1` and
   output striding `strides` as described in the
-  @{tf.nn.convolution$comment here}.
+  @{$python/nn#Convolution$comment here}.
 
   In the case that `data_format` does start with `"NC"`, the `input` and output
   (but not the `filter`) are simply transposed as follows:
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index c59eccc174..42af7f8b27 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -867,7 +867,7 @@ def raw_rnn(cell, loop_fn,
 
   ```python
   time = tf.constant(0, dtype=tf.int32)
-  (finished, next_input, initial_state, _, loop_state) = loop_fn(
+  (finished, next_input, initial_state, emit_structure, loop_state) = loop_fn(
       time=time, cell_output=None, cell_state=None, loop_state=None)
   emit_ta = TensorArray(dynamic_size=True, dtype=initial_state.dtype)
   state = initial_state
@@ -878,7 +878,7 @@ def raw_rnn(cell, loop_fn,
         loop_state=loop_state)
     # Emit zeros and copy forward state for minibatch entries that are finished.
     state = tf.where(finished, state, next_state)
-    emit = tf.where(finished, tf.zeros_like(emit), emit)
+    emit = tf.where(finished, tf.zeros_like(emit_structure), emit)
     emit_ta = emit_ta.write(time, emit)
     # If any new minibatch entries are marked as finished, mark these.
     finished = tf.logical_or(finished, next_finished)
@@ -938,10 +938,15 @@ def raw_rnn(cell, loop_fn,
       and `emit_output`: the output to store for this iteration.
 
       Note that `emit_output` should be a `Tensor` or (possibly nested)
-      tuple of tensors with shapes and structure matching `cell.output_size`
-      and `cell_output` above.  The parameter `cell_state` and output
-      `next_cell_state` may be either a single or (possibly nested) tuple
-      of tensors.  The parameter `loop_state` and
+      tuple of tensors which is aggregated in the `emit_ta` inside the
+      `while_loop`. For the first call to `loop_fn`, the `emit_output`
+      corresponds to the `emit_structure` which is then used to determine the
+      size of the `zero_tensor` for the `emit_ta` (defaults to
+      `cell.output_size`). For the subsequent calls to the `loop_fn`, the
+      `emit_output` corresponds to the actual output tensor
+      that is to be aggregated in the `emit_ta`. The parameter `cell_state`
+      and output `next_cell_state` may be either a single or (possibly nested)
+      tuple of tensors.  The parameter `loop_state` and
       output `next_loop_state` may be either a single or (possibly nested) tuple
       of `Tensor` and `TensorArray` objects.  This last parameter
       may be ignored by `loop_fn` and the return value may be `None`.  If it
diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py
index 6d7eaababc..5e2146b79f 100644
--- a/tensorflow/python/ops/special_math_ops.py
+++ b/tensorflow/python/ops/special_math_ops.py
@@ -163,7 +163,7 @@ def einsum(equation, *inputs, **kwargs):
     if '...' in equation:
       raise ValueError('Subscripts with ellipses are not yet supported.')
 
-    match = re.match('([a-z,]+)(->[a-z]*)?', equation)
+    match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
     if not match:
       raise ValueError('Indices have incorrect format: %s' % equation)
 
@@ -402,7 +402,7 @@ def _exponential_space_einsum(equation, *inputs):
   if '...' in equation:
     raise ValueError('Subscripts with ellipses are not yet supported.')
 
-  match = re.match('([a-z,]+)(->[a-z]*)?', equation)
+  match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
   if not match:
     raise ValueError('Indices have incorrect format: %s' % equation)
 
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 2c212f4548..d7c3a7e8dc 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -192,6 +192,9 @@ class EinsumTest(test.TestCase):
       'abc,cba',
       'dba,ead,cad->bce',
       'aef,fbc,dca->bde',
+      'iJ,Jk->ik',
+      'iJ,Ki->JK',
+      'iJk,Jklm->Jk'
   ]
 
   long_cases = [
@@ -208,6 +211,8 @@ class EinsumTest(test.TestCase):
       'ijk ijk',
       'ij.jk->ik',
       'ij...,jk...->ik...',
+      'ij,k ->kji',
+      'ij,k-> kji',
 
       # axis in output that does not exist
       'ij,jk->im',
diff --git a/tensorflow/python/tools/freeze_graph.py b/tensorflow/python/tools/freeze_graph.py
index a52f325ddb..e9f1def48c 100644
--- a/tensorflow/python/tools/freeze_graph.py
+++ b/tensorflow/python/tools/freeze_graph.py
@@ -56,8 +56,6 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.tools import saved_model_utils
 from tensorflow.python.training import saver as saver_lib
 
-FLAGS = None
-
 
 def freeze_graph_with_def_protos(input_graph_def,
                                  input_saver_def,
@@ -256,25 +254,24 @@ def freeze_graph(input_graph,
       checkpoint_version=checkpoint_version)
 
 
-def main(unused_args):
-  if FLAGS.checkpoint_version == 1:
+def main(unused_args, flags):
+  if flags.checkpoint_version == 1:
     checkpoint_version = saver_pb2.SaverDef.V1
-  elif FLAGS.checkpoint_version == 2:
+  elif flags.checkpoint_version == 2:
     checkpoint_version = saver_pb2.SaverDef.V2
   else:
     print("Invalid checkpoint version (must be '1' or '2'): %d" %
-          FLAGS.checkpoint_version)
+          flags.checkpoint_version)
     return -1
-  freeze_graph(FLAGS.input_graph, FLAGS.input_saver, FLAGS.input_binary,
-               FLAGS.input_checkpoint, FLAGS.output_node_names,
-               FLAGS.restore_op_name, FLAGS.filename_tensor_name,
-               FLAGS.output_graph, FLAGS.clear_devices, FLAGS.initializer_nodes,
-               FLAGS.variable_names_whitelist, FLAGS.variable_names_blacklist,
-               FLAGS.input_meta_graph, FLAGS.input_saved_model_dir,
-               FLAGS.saved_model_tags, checkpoint_version)
-
+  freeze_graph(flags.input_graph, flags.input_saver, flags.input_binary,
+               flags.input_checkpoint, flags.output_node_names,
+               flags.restore_op_name, flags.filename_tensor_name,
+               flags.output_graph, flags.clear_devices, flags.initializer_nodes,
+               flags.variable_names_whitelist, flags.variable_names_blacklist,
+               flags.input_meta_graph, flags.input_saved_model_dir,
+               flags.saved_model_tags, checkpoint_version)
 
-if __name__ == "__main__":
+def run_main():
   parser = argparse.ArgumentParser()
   parser.register("type", "bool", lambda v: v.lower() == "true")
   parser.add_argument(
@@ -376,5 +373,10 @@ if __name__ == "__main__":
       separated by \',\'. For tag-set contains multiple tags, all tags \
       must be passed in.\
       """)
-  FLAGS, unparsed = parser.parse_known_args()
-  app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  flags, unparsed = parser.parse_known_args()
+
+  my_main = lambda unused_args: main(unused_args, flags)
+  app.run(main=my_main, argv=[sys.argv[0]] + unparsed)
+
+if __name__ == '__main__':
+  run_main()
diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py
index b0e9e3e5ed..b88be4ae04 100644
--- a/tensorflow/python/tools/saved_model_cli.py
+++ b/tensorflow/python/tools/saved_model_cli.py
@@ -38,11 +38,15 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.framework import types_pb2
 from tensorflow.python.client import session
 from tensorflow.python.debug.wrappers import local_cli_wrapper
+from tensorflow.python.framework import meta_graph as meta_graph_lib
 from tensorflow.python.framework import ops as ops_lib
 from tensorflow.python.platform import app  # pylint: disable=unused-import
 from tensorflow.python.saved_model import loader
 from tensorflow.python.tools import saved_model_utils
 
+# Set of ops to blacklist.
+_OP_BLACKLIST = set(['WriteFile', 'ReadFile'])
+
 
 def _show_tag_sets(saved_model_dir):
   """Prints the tag-sets stored in SavedModel directory.
@@ -242,6 +246,27 @@ def get_signature_def_map(saved_model_dir, tag_set):
   return meta_graph.signature_def
 
 
+def scan_meta_graph_def(meta_graph_def):
+  """Scans meta_graph_def and reports if there are ops on blacklist.
+
+  Print ops if they are on black list, or print success if no blacklisted ops
+  found.
+
+  Args:
+    meta_graph_def: MetaGraphDef protocol buffer.
+  """
+  all_ops_set = set(
+      meta_graph_lib.ops_used_by_graph_def(meta_graph_def.graph_def))
+  blacklisted_ops = _OP_BLACKLIST & all_ops_set
+  if blacklisted_ops:
+    # TODO(yifeif): print more warnings
+    print('MetaGraph with tag set %s contains the following blacklisted ops:' %
+          meta_graph_def.meta_info_def.tags, blacklisted_ops)
+  else:
+    print('MetaGraph with tag set %s does not contain blacklisted ops.' %
+          meta_graph_def.meta_info_def.tags)
+
+
 def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key,
                                    input_tensor_key_feed_dict, outdir,
                                    overwrite_flag, tf_debug=False):
@@ -609,6 +634,21 @@ def run(args):
                                  args.overwrite, tf_debug=args.tf_debug)
 
 
+def scan(args):
+  """Function triggered by scan command.
+
+  Args:
+    args: A namespace parsed from command line.
+  """
+  if args.tag_set:
+    scan_meta_graph_def(
+        saved_model_utils.get_meta_graph_def(args.dir, args.tag_set))
+  else:
+    saved_model = reader.read_saved_model(args.dir)
+    for meta_graph_def in saved_model.meta_graphs:
+      scan_meta_graph_def(meta_graph_def)
+
+
 def create_parser():
   """Creates a parser that parse the command line arguments.
 
@@ -730,6 +770,26 @@ def create_parser():
            'SavedModel.')
   parser_run.set_defaults(func=run)
 
+  # scan command
+  scan_msg = ('Usage example:\n'
+              'To scan for blacklisted ops in SavedModel:\n'
+              '$saved_model_cli scan --dir /tmp/saved_model\n'
+              'To scan a specific MetaGraph, pass in --tag_set\n')
+  parser_scan = subparsers.add_parser(
+      'scan',
+      description=scan_msg,
+      formatter_class=argparse.RawTextHelpFormatter)
+  parser_scan.add_argument(
+      '--dir',
+      type=str,
+      required=True,
+      help='directory containing the SavedModel to execute')
+  parser_scan.add_argument(
+      '--tag_set',
+      type=str,
+      help='tag-set of graph in SavedModel to scan, separated by \',\'')
+  parser_scan.set_defaults(func=scan)
+
   return parser
 
 
diff --git a/tensorflow/python/tools/saved_model_cli_test.py b/tensorflow/python/tools/saved_model_cli_test.py
index f99c844845..eedc893a38 100644
--- a/tensorflow/python/tools/saved_model_cli_test.py
+++ b/tensorflow/python/tools/saved_model_cli_test.py
@@ -525,6 +525,28 @@ signature_def['serving_default']:
     y_expected = np.array([[2.5], [3.0]])
     self.assertAllClose(y_expected, y_actual)
 
+  def testScanCommand(self):
+    self.parser = saved_model_cli.create_parser()
+    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
+    args = self.parser.parse_args(['scan', '--dir', base_path])
+    with captured_output() as (out, _):
+      saved_model_cli.scan(args)
+    output = out.getvalue().strip()
+    self.assertTrue('does not contain blacklisted ops' in output)
+
+  def testScanCommandFoundBlacklistedOp(self):
+    self.parser = saved_model_cli.create_parser()
+    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
+    args = self.parser.parse_args(
+        ['scan', '--dir', base_path, '--tag_set', 'serve'])
+    op_blacklist = saved_model_cli._OP_BLACKLIST
+    saved_model_cli._OP_BLACKLIST = set(['VariableV2'])
+    with captured_output() as (out, _):
+      saved_model_cli.scan(args)
+    saved_model_cli._OP_BLACKLIST = op_blacklist
+    output = out.getvalue().strip()
+    self.assertTrue('\'VariableV2\'' in output)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 0b3b060fe7..03e3e0857f 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -274,7 +274,8 @@ CUDNN_DNN_ROUTINE_EACH_R6(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
 // clang-format off
 #if CUDNN_VERSION >= 7000
 #define CUDNN_DNN_ROUTINE_EACH_R7(__macro)                    \
-  __macro(cudnnSetConvolutionMathType)
+  __macro(cudnnSetConvolutionMathType)                        \
+  __macro(cudnnSetRNNMatrixMathType)
 
 // clang-format on
 CUDNN_DNN_ROUTINE_EACH_R7(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
@@ -586,6 +587,19 @@ static bool TensorOpMathEnabled() {
   return is_enabled;
 }
 
+// A helper function to decide whether to enable the TENSOR_OP_MATH math type
+// for RNNs.
+static bool RnnTensorOpMathEnabled() {
+  static bool is_enabled = [] {
+    bool is_disabled = false;
+    TF_CHECK_OK(
+        tensorflow::ReadBoolFromEnvVar("TF_DISABLE_CUDNN_RNN_TENSOR_OP_MATH",
+                                       /*default_val=*/false, &is_disabled));
+    return !is_disabled;
+  }();
+  return is_enabled;
+}
+
 // A helper function to decide whether to use CUDNN_BATCHNORM_SPATIAL_PERSISTENT
 // in batchnorm. This mode can be faster in some tasks because an optimized path
 // may be selected for CUDNN_DATA_FLOAT and CUDNN_DATA_HALF data types, compute
@@ -1124,6 +1138,9 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       SetFailure(cudnn_params_desc_->Status());
       return;
     }
+    if (data_type == CUDNN_DATA_HALF) {
+      set_use_tensor_op_math(true);
+    }
   }
   ~CudnnRnnDescriptor() override {
     if (rnn_desc_) {
@@ -1132,6 +1149,20 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       CUDNN_RETURN_IF_FAIL(status, "Unable to destroy RNN descriptor");
     }
   }
+  void set_use_tensor_op_math(bool use_tensor_op_math) {
+#if CUDNN_VERSION >= 7000
+    cudnnMathType_t math_type =
+        (use_tensor_op_math ? CUDNN_TENSOR_OP_MATH : CUDNN_DEFAULT_MATH);
+    if (RnnTensorOpMathEnabled()) {
+      cudnnStatus_t status =
+          wrap::cudnnSetRNNMatrixMathType(parent_, rnn_desc_, math_type);
+      if (status != CUDNN_STATUS_SUCCESS) {
+        LOG(FATAL) << "could not set cudnn RNN math type: "
+                   << ToString(status);
+      }
+    }
+#endif
+  }
   cudnnRNNDescriptor_t handle() const {
     if (!ok()) return nullptr;
     return rnn_desc_;
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index ae305a28e0..a0ec1708d5 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1168,6 +1168,20 @@ def tf_custom_op_library_additional_deps():
       "@protobuf_archive//:protobuf_headers",
       clean_dep("//third_party/eigen3"),
       clean_dep("//tensorflow/core:framework_headers_lib"),
+  ] + if_windows(["//tensorflow/python:pywrap_tensorflow_import_lib"])
+
+# A list of targets that contains the implemenation of
+# tf_custom_op_library_additional_deps. It's used to generate a DEF file for
+# exporting symbols from _pywrap_tensorflow.dll on Windows.
+def tf_custom_op_library_additional_deps_impl():
+  return [
+      # for @nsync//:nsync_headers
+      "//third_party/nsync:nsync_cpp",
+      # for //third_party/eigen3
+      clean_dep("//third_party/eigen3"),
+      # for //tensorflow/core:framework_headers_lib
+      clean_dep("//tensorflow/core:framework"),
+      clean_dep("//tensorflow/core:reader_base"),
   ]
 
 # Traverse the dependency graph along the "deps" attribute of the
@@ -1254,6 +1268,7 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]):
       deps=deps + if_cuda(cuda_deps),
       data=[name + "_check_deps"],
       copts=tf_copts(is_external=True),
+      features = ["windows_export_all_symbols"],
       linkopts=linkopts + select({
           "//conditions:default": [
               "-lm",
@@ -1400,7 +1415,8 @@ def tf_py_wrap_cc(name,
       ]) + tf_extension_copts()),
       linkopts=tf_extension_linkopts() + extra_linkopts,
       linkstatic=1,
-      deps=deps + extra_deps)
+      deps=deps + extra_deps,
+      **kwargs)
   native.genrule(
       name="gen_" + cc_library_pyd_name,
       srcs=[":" + cc_library_name],
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 5268bba3cc..baa7a0889d 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -247,6 +247,8 @@ class ApiCompatibilityTest(test.TestCase):
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.do_not_descend_map['tf'].append('contrib')
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
+    # TODO(annarev): Make slide_dataset available in API.
+    public_api_visitor.private_map['tf'] = ['slide_dataset']
     traverse.traverse(api, public_api_visitor)
 
     proto_dict = visitor.GetProtos()
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index ec90c83aac..d5dea4f3e4 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -23,11 +23,12 @@ RUN /install/install_deb_packages.sh
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends python-pip
+RUN pip install --upgrade wheel
 RUN pip install --upgrade astor
 RUN pip install --upgrade gast
 RUN pip install --upgrade numpy
 RUN pip install --upgrade termcolor
 
 # Install golang
-RUN add-apt-repository -y ppa:ubuntu-lxc/lxd-stable
-RUN apt-get install -y golang
+RUN apt-get install -t xenial-backports -y golang-1.9
+ENV PATH=${PATH}:/usr/lib/go-1.9/bin
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 8b8ba31a0d..40189a6d1b 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -65,4 +65,5 @@ bazel test -c opt $BUILD_OPTS -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
-  //${PY_TEST_DIR}/tensorflow/python/...
+  //${PY_TEST_DIR}/tensorflow/python/... \
+  //${PY_TEST_DIR}/tensorflow/contrib/...
diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl
new file mode 100644
index 0000000000..3cb72f4979
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/BUILD.tpl
@@ -0,0 +1,15 @@
+# Description:
+# Tools for filtering DEF file for TensorFlow on Windows
+#
+# On Windows, we use a DEF file generated by Bazel to export
+# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll).
+# The maximum number of symbols that can be exported per DLL is 64K,
+# so we have to filter some useless symbols through this python script.
+
+package(default_visibility = ["//visibility:public"])
+
+py_binary(
+    name = "def_file_filter",
+    srcs = ["def_file_filter.py"],
+    srcs_version = "PY2AND3",
+)
diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
new file mode 100644
index 0000000000..8bdc03eb0f
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
@@ -0,0 +1,168 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""def_file_filter.py - tool to filter a windows def file.
+
+The def file can be used to export symbols from the tensorflow dll to enable
+tf.load_library().
+
+Because the linker allows only 64K symbols to be exported per dll
+we filter the symbols down to the essentials. The regular expressions
+we use for this are specific to tensorflow.
+
+TODO: this works fine but there is an issue with exporting
+'const char * const' and importing it from a user_ops. The problem is
+on the importing end and using __declspec(dllimport) works around it.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import io
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+# External tools we use that come with visual studio sdk
+UNDNAME = "%{undname_bin_path}"
+
+# Exclude if matched
+EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::")
+
+# Include if matched before exclude
+INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|"
+                           r"google::protobuf::internal::ArenaImpl::AllocateAligned|" # for contrib/data/_prefetching_ops
+                           r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops
+                           r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops
+                           r"tensorflow::internal::LogMessage|"
+                           r"tensorflow::internal::LogString|"
+                           r"tensorflow::internal::CheckOpMessageBuilder|"
+                           r"tensorflow::internal::MakeCheckOpValueString|"
+                           r"tensorflow::internal::PickUnusedPortOrDie|"
+                           r"tensorflow::internal::ValidateDevice|"
+                           r"tensorflow::ops::internal::Enter|"
+                           r"tensorflow::strings::internal::AppendPieces|"
+                           r"tensorflow::strings::internal::CatPieces|"
+                           r"tensorflow::io::internal::JoinPathImpl")
+
+# Include if matched after exclude
+INCLUDE_RE = re.compile(r"^(TF_\w*)$|"
+                        r"^(TFE_\w*)$|"
+                        r"nsync::|"
+                        r"tensorflow::|"
+                        r"functor::|"
+                        r"perftools::gputools")
+
+# We want to identify data members explicitly in the DEF file, so that no one
+# can implicitly link against the DLL if they use one of the variables exported
+# from the DLL and the header they use does not decorate the symbol with
+# __declspec(dllimport). It is easier to detect what a data symbol does
+# NOT look like, so doing it with the below regex.
+DATA_EXCLUDE_RE = re.compile(r"[)(]|"
+                             r"vftable|"
+                             r"vbtable|"
+                             r"vcall|"
+                             r"RTTI|"
+                             r"protobuf::internal::ExplicitlyConstructed")
+
+def get_args():
+  """Parse command line."""
+  filename_list = lambda x: x.split(";")
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--input", type=filename_list,
+                      help="paths to input def file",
+                      required=True)
+  parser.add_argument("--output", help="output deffile", required=True)
+  parser.add_argument("--target", help="name of the target", required=True)
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  """main."""
+  args = get_args()
+
+  # Pipe dumpbin to extract all linkable symbols from libs.
+  # Good symbols are collected in candidates and also written to
+  # a temp file.
+  candidates = []
+  tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False)
+  for def_file_path in args.input:
+    def_file = open(def_file_path, 'r')
+    for line in def_file:
+      cols = line.split()
+      sym = cols[0]
+      tmpfile.file.write(sym + "\n")
+      candidates.append(sym)
+  tmpfile.file.close()
+
+  # Run the symbols through undname to get their undecorated name
+  # so we can filter on something readable.
+  with open(args.output, "w") as def_fp:
+    # track dupes
+    taken = set()
+
+    # Header for the def file.
+    def_fp.write("LIBRARY " + args.target + "\n")
+    def_fp.write("EXPORTS\n")
+    def_fp.write("\t ??1OpDef@tensorflow@@UEAA@XZ\n")
+
+    # Each symbols returned by undname matches the same position in candidates.
+    # We compare on undname but use the decorated name from candidates.
+    dupes = 0
+    proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE)
+    for idx, line in enumerate(io.TextIOWrapper(proc.stdout, encoding="utf-8")):
+      decorated = candidates[idx]
+      if decorated in taken:
+        # Symbol is already in output, done.
+        dupes += 1
+        continue
+
+      if not INCLUDEPRE_RE.search(line):
+        if EXCLUDE_RE.search(line):
+          continue
+        if not INCLUDE_RE.search(line):
+          continue
+
+      if "deleting destructor" in line:
+        # Some of the symbols convered by INCLUDEPRE_RE export deleting
+        # destructor symbols, which is a bad idea.
+        # So we filter out such symbols here.
+        continue
+
+      if DATA_EXCLUDE_RE.search(line):
+        def_fp.write("\t" + decorated + "\n")
+      else:
+        def_fp.write("\t" + decorated + " DATA\n")
+      taken.add(decorated)
+    def_fp.close()
+
+  exit_code = proc.wait()
+  if exit_code != 0:
+    print("{} failed, exit={}".format(UNDNAME, exit_code))
+    return exit_code
+
+  os.unlink(tmpfile.name)
+
+  print("symbols={}, taken={}, dupes={}"
+        .format(len(candidates), len(taken), dupes))
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main())
diff --git a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
new file mode 100644
index 0000000000..47539b2423
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
@@ -0,0 +1,56 @@
+"""Repository rule for def file filter autoconfiguration.
+
+This repository reuses Bazel's VC detect mechanism to find undname.exe,
+which is a tool used in def_file_filter.py.
+
+def_file_filter.py is for filtering the DEF file for TensorFlow on Windows.
+On Windows, we use a DEF file generated by Bazel to export symbols from the
+tensorflow dynamic library(_pywrap_tensorflow.dll). The maximum number of
+symbols that can be exported per DLL is 64K, so we have to filter some useless
+symbols through this python script.
+
+`def_file_filter_config` depends on the following environment variables:
+  * `BAZEL_VC`
+  * `BAZEL_VS`
+  * `VS90COMNTOOLS`
+  * `VS100COMNTOOLS`
+  * `VS110COMNTOOLS`
+  * `VS120COMNTOOLS`
+  * `VS140COMNTOOLS`
+"""
+
+load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_vc_path")
+load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_msvc_tool")
+load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "auto_configure_fail")
+
+def _def_file_filter_configure_impl(repository_ctx):
+  if repository_ctx.os.name.lower().find("windows") == -1:
+    repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
+    repository_ctx.file("def_file_filter.py", "")
+    return
+  vc_path = find_vc_path(repository_ctx)
+  if vc_path == "visual-studio-not-found":
+    auto_configure_fail("Visual C++ build tools not found on your machine")
+  undname_bin_path = find_msvc_tool(repository_ctx, vc_path, "undname.exe").replace("\\", "\\\\")
+
+  repository_ctx.template(
+    "def_file_filter.py",
+    Label("//tensorflow/tools/def_file_filter:def_file_filter.py.tpl"),
+    {
+      "%{undname_bin_path}": undname_bin_path,
+    })
+  repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
+
+
+def_file_filter_configure = repository_rule(
+    implementation = _def_file_filter_configure_impl,
+    environ = [
+        "BAZEL_VC",
+        "BAZEL_VS",
+        "VS90COMNTOOLS",
+        "VS100COMNTOOLS",
+        "VS110COMNTOOLS",
+        "VS120COMNTOOLS",
+        "VS140COMNTOOLS"
+    ],
+)
diff --git a/tensorflow/tools/dist_test/README.md b/tensorflow/tools/dist_test/README.md
index c1b1f79bbd..228d5ee35d 100644
--- a/tensorflow/tools/dist_test/README.md
+++ b/tensorflow/tools/dist_test/README.md
@@ -17,6 +17,14 @@ cesnsu model:
 
     ./local_test.sh --model_name CENSUS_WIDENDEEP
 
+You can test specify version of TensorFlow:
+
+```shell
+./local_test.sh ${whl_file_url}
+```
+
+For example, you can find these TensorFlow python package URLs from [here](https://www.tensorflow.org/install/install_linux#the_url_of_the_tensorflow_python_package) for Ubuntu.
+
 **2) Launch a remote k8s cluster on Google Kubernetes Engine (GKE) and run the
 test suite on it**
 
diff --git a/tensorflow/tools/dist_test/local_test.sh b/tensorflow/tools/dist_test/local_test.sh
index 435f9d0dc9..caae7fd530 100755
--- a/tensorflow/tools/dist_test/local_test.sh
+++ b/tensorflow/tools/dist_test/local_test.sh
@@ -16,12 +16,11 @@
 #
 # Tests distributed TensorFlow on a locally running TF GRPC cluster.
 #
-# This script peforms the following steps:
-# 1) Build the docker-in-docker (dind) image capable of running docker and
-#    Kubernetes (k8s) cluster inside.
+# This script performs the following steps:
+# 1) Build the docker image capable of running distributed TensorFlow in docker.
 # 2) Run a container from the aforementioned image and start docker service
 #    in it
-# 3) Call a script to launch a k8s TensorFlow GRPC cluster inside the container
+# 3) Call a script to launch a distributed TensorFlow GRPC cluster inside the container
 #    and run the distributed test suite.
 #
 # Usage: local_test.sh <whl_file_location>
@@ -64,15 +63,9 @@ die() {
 
 # Configurations
 DOCKER_IMG_NAME="tensorflow/tf-dist-test-local-cluster"
-LOCAL_K8S_CACHE=${HOME}/kubernetes
 
-# Helper function
-get_container_id_by_image_name() {
-    # Get the id of a container by image name
-    # Usage: get_docker_container_id_by_image_name <img_name>
-
-    docker ps | grep $1 | awk '{print $1}'
-}
+# Use TensorFlow v1.5.0 for Python 2.7 and CPU only as we set num_gpus to 0 in the below
+DEFAULT_WHL_FILE_LOCATION="https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.5.0-cp27-none-linux_x86_64.whl"
 
 # Parse input arguments
 LEAVE_CONTAINER_RUNNING=0
@@ -84,7 +77,8 @@ SYNC_REPLICAS_FLAG=""
 
 WHL_FILE_LOCATION=${1}
 if [[ -z "${WHL_FILE_LOCATION}" ]]; then
-  die "whl file location is not specified"
+  WHL_FILE_LOCATION=${DEFAULT_WHL_FILE_LOCATION}
+  echo "use default whl file location"
 fi
 
 while true; do
@@ -121,7 +115,7 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Get utility functions
 source ${DIR}/scripts/utils.sh
 
-# Build docker-in-docker image for local k8s cluster.
+# Build docker image for local distributed TensorFlow cluster.
 NO_CACHE_FLAG=""
 if [[ ! -z "${TF_DIST_DOCKER_NO_CACHE}" ]] &&
    [[ "${TF_DIST_DOCKER_NO_CACHE}" != "0" ]]; then
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 3630dbd740..cbcdbf5b80 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -114,6 +114,13 @@ def configure(src_base_path, gen_path, debug=False):
   for target, src in link_map.items():
     if src is None:
       open(os.path.join(gen_path, target), "w").write("")
+    elif not os.path.exists(src):
+      # Git repo is configured in a way we don't support such as having
+      # packed refs. Even though in a git repo, tf.__git_version__ will not
+      # be accurate.
+      # TODO(mikecase): Support grabbing git info when using packed refs.
+      open(os.path.join(gen_path, target), "w").write("")
+      spec["git"] = False
     else:
       try:
         # In python 3.5, symlink function exists even on Windows. But requires
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index b7d7fac315..6e21aa2846 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -178,6 +178,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/util/tensor_bundle",
     ],
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index d89afe85c7..d86f65325b 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -182,6 +182,36 @@ Status FuseBatchNormWithConv(const NodeMatch& match,
   return Status::OK();
 }
 
+Status FuseBatchNormWithBatchToSpace(const NodeMatch& match,
+                             std::vector<NodeDef>* new_nodes) {
+  // Calculate the scale and offset values to apply.
+  std::vector<float> scale_values;
+  std::vector<float> offset_values;
+  TF_RETURN_IF_ERROR(
+      GetScaleAndOffsetValues(match, &scale_values, &offset_values));
+
+  // Fuse conv weights, and set the final output node name as batch_norm_node.
+  const NodeDef& batch_norm_node = match.node;
+  const NodeMatch& batch_to_space_node_match = match.inputs[0];
+  const NodeMatch& conv_node_match = batch_to_space_node_match.inputs[0];
+  const NodeDef& batch_to_space_node = batch_to_space_node_match.node;
+  const NodeDef& conv_node = conv_node_match.node;
+
+  string biasadd_name = conv_node.name() + "/biasadd";
+  TF_RETURN_IF_ERROR(
+      FuseScaleOffsetToConvWeights(scale_values, offset_values, conv_node_match,
+                                   biasadd_name , new_nodes));
+
+  NodeDef new_batch_to_space_node = batch_to_space_node;
+  // reuse batch_norm node name
+  new_batch_to_space_node.set_name(batch_norm_node.name());
+  new_batch_to_space_node.set_input(0, biasadd_name);
+  new_nodes->push_back(batch_to_space_node_match.inputs[1].node);
+  new_nodes->push_back(batch_to_space_node_match.inputs[2].node);
+  new_nodes->push_back(new_batch_to_space_node);
+  return Status::OK();
+}
+
 Status FuseBatchNormWithConvConcat(const NodeMatch& match,
                                    std::vector<NodeDef>* new_nodes) {
   // Calculate the scale and offset values to apply.
@@ -284,6 +314,43 @@ Status FoldOldBatchNorms(const GraphDef& input_graph_def,
     current_graph_def = replaced_graph_def;
   } while (did_graph_change);
 
+  do {
+    did_graph_change = false;
+    GraphDef replaced_graph_def;
+    TF_RETURN_IF_ERROR(ReplaceMatchingOpTypes(
+        current_graph_def,  // clang-format off
+        {"BatchNormWithGlobalNormalization|FusedBatchNorm",    // batch_norm_node
+         {
+             {"BatchToSpaceND",                  // batch_to_space_node
+              {
+                  {"Conv2D",                     // conv_node
+                   {
+                       {"*"},                    // input_node
+                       {"Const"},                // weights_node
+                   }
+                  },
+                  {"Const"},                     // block_shape
+                  {"Const"},                     // crops
+              }
+             },
+             {"Const"},                          // mean_node
+             {"Const"},                          // variance_node
+             {"Const"},                          // beta_node
+             {"Const"},                          // gamma_node
+         }
+        },  // clang-format on
+        [&did_graph_change](const NodeMatch& match,
+                            const std::set<string>& input_nodes,
+                            const std::set<string>& output_nodes,
+                            std::vector<NodeDef>* new_nodes) {
+          TF_RETURN_IF_ERROR(FuseBatchNormWithBatchToSpace(match, new_nodes));
+          did_graph_change = true;
+          return Status::OK();
+        },
+        {}, &replaced_graph_def));
+    current_graph_def = replaced_graph_def;
+  } while (did_graph_change);
+
   do {
     did_graph_change = false;
     GraphDef replaced_graph_def;
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index b30ba9ac8b..272410c693 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
+#include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
@@ -298,6 +299,96 @@ class FoldOldBatchNormsTest : public ::testing::Test {
   }
 };
 
+void TestFoldFusedBatchNormsWithBatchToSpace() {
+  auto root = tensorflow::Scope::NewRootScope();
+  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+  Tensor input_data(DT_FLOAT, TensorShape({2, 1, 3, 2}));
+  test::FillValues<float>(
+      &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
+                    -5.0f, -3.0f, -6.0f});
+  Output input_op =
+      Const(root.WithOpName("input_op"), Input::Initializer(input_data));
+
+  Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
+  test::FillValues<float>(&weights_data,
+                          {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
+  Output weights_op =
+      Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
+
+  Output conv_op = Conv2D(root.WithOpName("conv_op"), input_op, weights_op,
+                          {1, 1, 1, 1}, "VALID");
+
+  Tensor block_shape_data(DT_INT32, TensorShape({2}));
+  test::FillValues<int32>(&block_shape_data, {1, 2});
+  Output block_shape_op =
+      Const(root.WithOpName("block_shape_op"), Input::Initializer(block_shape_data));
+
+  Tensor crops_data(DT_INT32, TensorShape({2, 2}));
+  test::FillValues<int32>(&crops_data, {0, 0, 0, 1});
+  Output crops_op =
+      Const(root.WithOpName("crops_op"), Input::Initializer(crops_data));
+
+  Output batch_to_space_op = BatchToSpaceND(root.WithOpName("batch_to_space_op"),
+                                            conv_op, block_shape_op, crops_data);
+
+  Tensor mean_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&mean_data, {10.0f, 20.0f});
+  Output mean_op =
+      Const(root.WithOpName("mean_op"), Input::Initializer(mean_data));
+
+  Tensor variance_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&variance_data, {0.25f, 0.5f});
+  Output variance_op = Const(root.WithOpName("variance_op"),
+                             Input::Initializer(variance_data));
+
+  Tensor beta_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&beta_data, {0.1f, 0.6f});
+  Output beta_op =
+      Const(root.WithOpName("beta_op"), Input::Initializer(beta_data));
+
+  Tensor gamma_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&gamma_data, {1.0f, 2.0f});
+  Output gamma_op =
+      Const(root.WithOpName("gamma_op"), Input::Initializer(gamma_data));
+
+  GraphDef original_graph_def;
+  TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
+
+  NodeDef batch_norm_node;
+  batch_norm_node.set_op("FusedBatchNorm");
+  batch_norm_node.set_name("output");
+  AddNodeInput("batch_to_space_op", &batch_norm_node);
+  AddNodeInput("gamma_op", &batch_norm_node);
+  AddNodeInput("beta_op", &batch_norm_node);
+  AddNodeInput("mean_op", &batch_norm_node);
+  AddNodeInput("variance_op", &batch_norm_node);
+  SetNodeAttr("T", DT_FLOAT, &batch_norm_node);
+  SetNodeAttr("epsilon", 0.00001f, &batch_norm_node);
+  SetNodeAttr("is_training", false, &batch_norm_node);
+  *(original_graph_def.mutable_node()->Add()) = batch_norm_node;
+
+  std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
+  TF_ASSERT_OK(original_session->Create(original_graph_def));
+  std::vector<Tensor> original_outputs;
+  TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
+
+  GraphDef fused_graph_def;
+  TF_ASSERT_OK(FoldOldBatchNorms(original_graph_def, {{}, {"output"}},
+                                 &fused_graph_def));
+
+  std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
+  TF_ASSERT_OK(fused_session->Create(fused_graph_def));
+  std::vector<Tensor> fused_outputs;
+  TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
+
+  test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+
+  for (const NodeDef& node : fused_graph_def.node()) {
+    EXPECT_NE("FusedBatchNormWithBatchToSpace", node.op());
+  }
+}
+
 TEST_F(FoldOldBatchNormsTest, TestFoldOldBatchNorms) {
   TestFoldOldBatchNorms();
 }
@@ -315,5 +406,9 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) {
   TestFoldFusedBatchNormsWithConcat(/*split=*/false);
 }
 
+TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithBatchToSpace) {
+  TestFoldFusedBatchNormsWithBatchToSpace();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 1833d67d82..2607b9d704 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -48,36 +48,65 @@ py_binary(
     deps = ["//tensorflow:tensorflow_py"],
 )
 
+COMMON_PIP_DEPS = [
+    ":licenses",
+    "MANIFEST.in",
+    "README",
+    "setup.py",
+    ":included_headers",
+    "//tensorflow:tensorflow_py",
+    "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
+    "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
+    "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
+    "//tensorflow/contrib/data/python/ops:contrib_op_loader",
+    "//tensorflow/contrib/eager/python/examples:examples_pip",
+    "//tensorflow/contrib/eager/python:checkpointable_utils",
+    "//tensorflow/contrib/eager/python:evaluator",
+    "//tensorflow/contrib/gan:gan",
+    "//tensorflow/contrib/graph_editor:graph_editor_pip",
+    "//tensorflow/contrib/keras:keras",
+    "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
+    "//tensorflow/contrib/nn:nn_py",
+    "//tensorflow/contrib/predictor:predictor_pip",
+    "//tensorflow/contrib/py2tf:py2tf",
+    "//tensorflow/contrib/py2tf/converters:converters",
+    "//tensorflow/contrib/py2tf/converters:test_lib",
+    "//tensorflow/contrib/py2tf/impl:impl",
+    "//tensorflow/contrib/py2tf/pyct:pyct",
+    "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
+    "//tensorflow/contrib/receptive_field:receptive_field_pip",
+    "//tensorflow/contrib/session_bundle:session_bundle_pip",
+    "//tensorflow/contrib/signal:signal_py",
+    "//tensorflow/contrib/signal:test_util",
+    "//tensorflow/contrib/slim:slim",
+    "//tensorflow/contrib/slim/python/slim/data:data_pip",
+    "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+    "//tensorflow/contrib/specs:specs",
+    "//tensorflow/contrib/summary:summary_test_util",
+    "//tensorflow/contrib/tensor_forest:init_py",
+    "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+    "//tensorflow/contrib/timeseries:timeseries_pip",
+    "//tensorflow/contrib/tpu",
+    "//tensorflow/examples/tutorials/mnist:package",
+    "//tensorflow/python:distributed_framework_test_lib",
+    "//tensorflow/python:meta_graph_testdata",
+    "//tensorflow/python:spectral_ops_test_util",
+    "//tensorflow/python:util_example_parser_configuration",
+    "//tensorflow/python/debug:debug_pip",
+    "//tensorflow/python/eager:eager_pip",
+    "//tensorflow/python/saved_model:saved_model",
+    "//tensorflow/python/tools:tools_pip",
+    "//tensorflow/python:test_ops",
+    "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
+]
+
 # On Windows, python binary is a zip file of runfiles tree.
 # Add everything to its data dependency for generating a runfiles tree
 # for building the pip package on Windows.
 py_binary(
     name = "simple_console_for_windows",
     srcs = ["simple_console_for_windows.py"],
-    data = [
-        "MANIFEST.in",
-        "README",
-        "setup.py",
-        ":included_headers",
-        "//tensorflow/contrib/nn:nn_py",
-        "//tensorflow/contrib/session_bundle:session_bundle_pip",
-        "//tensorflow/contrib/signal:signal_py",
-        "//tensorflow/contrib/slim/python/slim/data:data_pip",
-        "//tensorflow/python:util_example_parser_configuration",
-        "//tensorflow/python/debug:debug_pip",
-        "//tensorflow/python/saved_model",
-        "//tensorflow/python:spectral_ops_test_util",
-        "//tensorflow/python/tools:tools_pip",
-        "//tensorflow/python/eager:eager_pip",
-        "//tensorflow/contrib/summary:summary_test_util",
-        # These targets don't build on Windows yet. Exclude them for now.
-        # "//tensorflow/contrib/slim",
-        # "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-        # "//tensorflow/contrib/specs",
-        # "//tensorflow/contrib/tensor_forest:init_py",
-        # "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-        # "//tensorflow/examples/tutorials/mnist:package",
-    ],
+    data = COMMON_PIP_DEPS,
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
@@ -108,6 +137,7 @@ filegroup(
         "@highwayhash//:LICENSE",
         "@jemalloc//:COPYING",
         "@jpeg//:LICENSE.md",
+        "@kafka//:LICENSE",
         "@libxsmm_archive//:LICENSE",
         "@lmdb//:LICENSE",
         "@local_config_sycl//sycl:LICENSE.text",
@@ -137,61 +167,12 @@ sh_binary(
     data = select({
         "//tensorflow:windows": [":simple_console_for_windows"],
         "//tensorflow:windows_msvc": [":simple_console_for_windows"],
-        "//conditions:default": [
-            ":licenses",
-            "MANIFEST.in",
-            "README",
-            "setup.py",
-            ":included_headers",
+        "//conditions:default": COMMON_PIP_DEPS + [
             ":simple_console",
-            "//tensorflow:tensorflow_py",
-            "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
-            "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
-            "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
-            "//tensorflow/contrib/data/python/ops:contrib_op_loader",
-            "//tensorflow/contrib/eager/python/examples:examples_pip",
-            "//tensorflow/contrib/eager/python:checkpointable_utils",
-            "//tensorflow/contrib/eager/python:evaluator",
-            "//tensorflow/contrib/gan:gan",
-            "//tensorflow/contrib/graph_editor:graph_editor_pip",
-            "//tensorflow/contrib/keras:keras",
-            "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
             "//tensorflow/contrib/lite/python:interpreter_test_data",
             "//tensorflow/contrib/lite/toco:toco",
             "//tensorflow/contrib/lite/toco/python:toco_wrapper",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
-            "//tensorflow/contrib/nn:nn_py",
-            "//tensorflow/contrib/predictor:predictor_pip",
-            "//tensorflow/contrib/py2tf:py2tf",
-            "//tensorflow/contrib/py2tf/converters:converters",
-            "//tensorflow/contrib/py2tf/converters:test_lib",
-            "//tensorflow/contrib/py2tf/impl:impl",
-            "//tensorflow/contrib/py2tf/pyct:pyct",
-            "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
-            "//tensorflow/contrib/receptive_field:receptive_field_pip",
-            "//tensorflow/contrib/session_bundle:session_bundle_pip",
-            "//tensorflow/contrib/signal:signal_py",
-            "//tensorflow/contrib/signal:test_util",
-            "//tensorflow/contrib/slim:slim",
-            "//tensorflow/contrib/slim/python/slim/data:data_pip",
-            "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-            "//tensorflow/contrib/specs:specs",
-            "//tensorflow/contrib/summary:summary_test_util",
-            "//tensorflow/contrib/tensor_forest:init_py",
-            "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-            "//tensorflow/contrib/timeseries:timeseries_pip",
-            "//tensorflow/contrib/tpu",
-            "//tensorflow/examples/tutorials/mnist:package",
-            "//tensorflow/python:distributed_framework_test_lib",
-            "//tensorflow/python:meta_graph_testdata",
-            "//tensorflow/python:spectral_ops_test_util",
-            "//tensorflow/python:util_example_parser_configuration",
-            "//tensorflow/python/debug:debug_pip",
-            "//tensorflow/python/eager:eager_pip",
-            "//tensorflow/python/saved_model:saved_model",
-            "//tensorflow/python/tools:tools_pip",
-            "//tensorflow/python:test_ops",
-            "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
         ],
     }) + if_mkl(["//third_party/mkl:intel_binary_blob"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e1a5f091ba..e0152da4df 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0-rc1'
+_VERSION = '1.6.0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -72,7 +72,7 @@ if sys.version_info < (3, 4):
 
 # pylint: disable=line-too-long
 CONSOLE_SCRIPTS = [
-    'freeze_graph = tensorflow.python.tools.freeze_graph:main',
+    'freeze_graph = tensorflow.python.tools.freeze_graph:run_main',
     'toco_from_protos = tensorflow.contrib.lite.toco.python.toco_from_protos:main',
     'toco = tensorflow.contrib.lite.toco.python.toco_wrapper:main',
     'saved_model_cli = tensorflow.python.tools.saved_model_cli:main',
diff --git a/tensorflow/tools/test/upload_test_benchmarks.py b/tensorflow/tools/test/upload_test_benchmarks.py
index 77cc9f75f7..edd093510e 100644
--- a/tensorflow/tools/test/upload_test_benchmarks.py
+++ b/tensorflow/tools/test/upload_test_benchmarks.py
@@ -88,6 +88,7 @@ import os
 import shutil
 
 from google.cloud import datastore
+from six import text_type
 
 
 def is_real_file(dirpath, fname):
@@ -150,7 +151,7 @@ def upload_benchmark_data(client, data):
   """
   test_result = json.loads(data)
 
-  test_name = unicode(test_result["name"])
+  test_name = text_type(test_result["name"])
   start_time = datetime.datetime.utcfromtimestamp(
       float(test_result["startTime"]))
   batch = []
@@ -162,7 +163,7 @@ def upload_benchmark_data(client, data):
   t_val.update({
       "test": test_name,
       "start": start_time,
-      "info": unicode(data)
+      "info": text_type(data)
   })
   batch.append(t_val)
 
@@ -170,7 +171,7 @@ def upload_benchmark_data(client, data):
   # the attribute to be fetched and displayed.  The full entry information is
   # also stored as a non-indexed JSON blob.
   for ent in test_result["entries"].get("entry", []):
-    ent_name = unicode(ent["name"])
+    ent_name = text_type(ent["name"])
     e_key = client.key("Entry")
     e_val = datastore.Entity(e_key, exclude_from_indexes=["info"])
     e_val.update({
@@ -178,7 +179,7 @@ def upload_benchmark_data(client, data):
         "start": start_time,
         "entry": ent_name,
         "timing": ent["wallTime"],
-        "info": unicode(json.dumps(ent))
+        "info": text_type(json.dumps(ent))
     })
     batch.append(e_val)
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index abc9eb9bc1..a922808a70 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -12,6 +12,8 @@ load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compil
 load("//third_party:repo.bzl", "tf_http_archive")
 load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
+load("//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
+     "def_file_filter_configure")
 
 def _extract_version_number(bazel_version):
   """Extracts the semantic version number from a version string
@@ -67,7 +69,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # We must check the bazel version before trying to parse any other BUILD
   # files, in case the parsing of those build files depends on the bazel
   # version we require here.
-  check_bazel_version_at_least("0.5.4")
+  check_bazel_version_at_least("0.10.0")
   clang6_configure(name="local_config_clang6")
   cuda_configure(name="local_config_cuda")
   tensorrt_configure(name="local_config_tensorrt")
@@ -75,6 +77,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   sycl_configure(name="local_config_sycl")
   python_configure(name="local_config_python")
 
+  # For windows bazel build
+  # TODO: Remove def file filter when TensorFlow can export symbols properly on Windows.
+  def_file_filter_configure(name = "local_config_def_file_filter")
+
   # Point //external/local_config_arm_compiler to //external/arm_compiler
   arm_compiler_configure(
       name="local_config_arm_compiler",
diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD
index 87a23925c4..4418ac32fc 100644
--- a/third_party/jpeg/jpeg.BUILD
+++ b/third_party/jpeg/jpeg.BUILD
@@ -526,12 +526,12 @@ config_setting(
 
 config_setting(
     name = "armeabi-v7a",
-    values = {"android_cpu": "armeabi-v7a"},
+    values = {"cpu": "armeabi-v7a"},
 )
 
 config_setting(
     name = "arm64-v8a",
-    values = {"android_cpu": "arm64-v8a"},
+    values = {"cpu": "arm64-v8a"},
 )
 
 config_setting(
diff --git a/third_party/kafka/BUILD b/third_party/kafka/BUILD
index a61a9e1f6c..a839ca717e 100644
--- a/third_party/kafka/BUILD
+++ b/third_party/kafka/BUILD
@@ -130,12 +130,16 @@ cc_library(
     ],
     hdrs = [
         "config.h",
+        "src-cpp/rdkafkacpp.h",
+        "src-cpp/rdkafkacpp_int.h",
+        "src/lz4.c",
+        "src/snappy_compat.h",
     ],
-    defines = [
+    copts = [
+        "-Iexternal/kafka/src",
+        "-Iexternal/kafka/src-cpp",
     ],
-    includes = [
-        "src",
-        "src-cpp",
+    defines = [
     ],
     linkopts = [
         "-lpthread",
@@ -143,5 +147,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "@boringssl//:ssl",
+        "@zlib_archive//:zlib",
     ],
 )
diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl
index de06ad5f27..1dd8ab433a 100644
--- a/third_party/py/BUILD.tpl
+++ b/third_party/py/BUILD.tpl
@@ -2,20 +2,26 @@ licenses(["restricted"])
 
 package(default_visibility = ["//visibility:public"])
 
+# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+# See https://docs.python.org/3/extending/windows.html
+cc_import(
+    name = "python_lib",
+    interface_library = select({
+        ":windows": ":python_import_lib",
+        # A placeholder for Unix platforms which makes --no_build happy.
+        "//conditions:default": "not-existing.lib",
+    }),
+    system_provided = 1,
+)
+
 cc_library(
     name = "python_headers",
     hdrs = [":python_include"],
-    data = select({
-        ":windows": [":python_import_lib"],
+    deps = select({
+        ":windows": [":python_lib"],
         "//conditions:default": [],
     }),
     includes = ["python_include"],
-    linkopts = select({
-        # TODO(pcloudy): Ideally, this should just go into deps after resolving
-        # https://github.com/bazelbuild/bazel/issues/3237,
-        ":windows": ["$(locations :python_import_lib)"],
-        "//conditions:default": [],
-    }),
 )
 
 cc_library(
-- 
GitLab


From 310c32e4f793b591983ec6cf9635d1d0a86602fa Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 15 Mar 2018 13:01:28 -0700
Subject: [PATCH 072/960] TFLite iOS Makefile: Disable SSE4.1 for x86_64 build.

PiperOrigin-RevId: 189232136
---
 tensorflow/contrib/lite/ios_makefile.inc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/contrib/lite/ios_makefile.inc b/tensorflow/contrib/lite/ios_makefile.inc
index fc6594c3a0..079320586f 100644
--- a/tensorflow/contrib/lite/ios_makefile.inc
+++ b/tensorflow/contrib/lite/ios_makefile.inc
@@ -31,9 +31,6 @@ ifeq ($(TARGET), IOS)
 		${IPHONEOS_SYSROOT} \
 		-arch $(IOS_ARCH) \
 		-O3
-	ifeq ($(IOS_ARCH), x86_64)
-		CXXFLAGS += -msse4.1
-	endif
 	CCFLAGS += -miphoneos-version-min=$(MIN_SDK_VERSION) \
 		-fembed-bitcode \
 		-mno-thumb \
-- 
GitLab


From 19e4a6ab8e400c90ccd0c95c6396a0d2cc925324 Mon Sep 17 00:00:00 2001
From: Surya Bhupatiraju <sbhupatiraju@google.com>
Date: Thu, 15 Mar 2018 13:02:23 -0700
Subject: [PATCH 073/960] Add mean-only FID and diagonal-covariance-only FID
 variants to TFGAN.

PiperOrigin-RevId: 189232299
---
 .../eval/python/classifier_metrics_impl.py    | 190 +++++++++++++++---
 .../eval/python/classifier_metrics_test.py    |  60 ++++++
 2 files changed, 226 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index fdfabd07c1..323cbe6e76 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -44,11 +44,11 @@ from tensorflow.python.ops import functional_ops
 from tensorflow.python.ops import image_ops
 from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn_impl
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import resource_loader
 
-
 __all__ = [
     'get_graph_def_from_disk',
     'get_graph_def_from_resource',
@@ -62,10 +62,11 @@ __all__ = [
     'frechet_inception_distance',
     'frechet_classifier_distance',
     'frechet_classifier_distance_from_activations',
+    'mean_only_frechet_classifier_distance_from_activations',
+    'diagonal_only_frechet_classifier_distance_from_activations',
     'INCEPTION_DEFAULT_IMAGE_SIZE',
 ]
 
-
 INCEPTION_URL = 'http://download.tensorflow.org/models/frozen_inception_v1_2015_12_05.tar.gz'
 INCEPTION_FROZEN_GRAPH = 'inceptionv1_for_inception_score.pb'
 INCEPTION_INPUT = 'Mul:0'
@@ -77,8 +78,7 @@ INCEPTION_DEFAULT_IMAGE_SIZE = 299
 def _validate_images(images, image_size):
   images = ops.convert_to_tensor(images)
   images.shape.with_rank(4)
-  images.shape.assert_is_compatible_with(
-      [None, image_size, image_size, None])
+  images.shape.assert_is_compatible_with([None, image_size, image_size, None])
   return images
 
 
@@ -109,9 +109,10 @@ def _symmetric_matrix_square_root(mat, eps=1e-10):
       math_ops.matmul(u, array_ops.diag(si)), v, transpose_b=True)
 
 
-def preprocess_image(
-    images, height=INCEPTION_DEFAULT_IMAGE_SIZE,
-    width=INCEPTION_DEFAULT_IMAGE_SIZE, scope=None):
+def preprocess_image(images,
+                     height=INCEPTION_DEFAULT_IMAGE_SIZE,
+                     width=INCEPTION_DEFAULT_IMAGE_SIZE,
+                     scope=None):
   """Prepare a batch of images for evaluation.
 
   This is the preprocessing portion of the graph from
@@ -272,8 +273,11 @@ def run_inception(images,
   return activations
 
 
-def run_image_classifier(tensor, graph_def, input_tensor,
-                         output_tensor, scope='RunClassifier'):
+def run_image_classifier(tensor,
+                         graph_def,
+                         input_tensor,
+                         output_tensor,
+                         scope='RunClassifier'):
   """Runs a network from a frozen graph.
 
   Args:
@@ -433,8 +437,8 @@ def trace_sqrt_product(sigma, sigma_v):
   sqrt_sigma = _symmetric_matrix_square_root(sigma)
 
   # This is sqrt(A sigma_v A) above
-  sqrt_a_sigmav_a = math_ops.matmul(
-      sqrt_sigma, math_ops.matmul(sigma_v, sqrt_sigma))
+  sqrt_a_sigmav_a = math_ops.matmul(sqrt_sigma,
+                                    math_ops.matmul(sigma_v, sqrt_sigma))
 
   return math_ops.trace(_symmetric_matrix_square_root(sqrt_a_sigmav_a))
 
@@ -452,7 +456,7 @@ def frechet_classifier_distance(real_images,
   Given two Gaussian distribution with means m and m_w and covariance matrices
   C and C_w, this function calcuates
 
-  |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
+              |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
 
   which captures how different the distributions of real images and generated
   images (or more accurately, their visual features) are. Note that unlike the
@@ -511,10 +515,140 @@ def frechet_classifier_distance(real_images,
   return frechet_classifier_distance_from_activations(real_a, gen_a)
 
 
-def frechet_classifier_distance_from_activations(
+def mean_only_frechet_classifier_distance_from_activations(
     real_activations, generated_activations):
   """Classifier distance for evaluating a generative model from activations.
 
+  Given two Gaussian distribution with means m and m_w and covariance matrices
+  C and C_w, this function calcuates
+
+                                |m - m_w|^2
+
+  which captures how different the distributions of real images and generated
+  images (or more accurately, their visual features) are. Note that unlike the
+  Inception score, this is a true distance and utilizes information about real
+  world images.
+
+  Note that when computed using sample means and sample covariance matrices,
+  Frechet distance is biased. It is more biased for small sample sizes. (e.g.
+  even if the two distributions are the same, for a small sample size, the
+  expected Frechet distance is large). It is important to use the same
+  sample size to compute frechet classifier distance when comparing two
+  generative models.
+
+  In this variant, we only compute the difference between the means of the
+  fitted Gaussians. The computation leads to O(n) vs. O(n^2) memory usage, yet
+  still retains much of the same information as FID.
+
+  Args:
+    real_activations: 2D array of activations of real images of size
+      [num_images, num_dims] to use to compute Frechet Inception distance.
+    generated_activations: 2D array of activations of generated images of size
+      [num_images, num_dims] to use to compute Frechet Inception distance.
+
+  Returns:
+    The mean-only Frechet Inception distance. A floating-point scalar of the
+    same type as the output of the activations.
+  """
+  real_activations.shape.assert_has_rank(2)
+  generated_activations.shape.assert_has_rank(2)
+
+  activations_dtype = real_activations.dtype
+  if activations_dtype != dtypes.float64:
+    real_activations = math_ops.to_double(real_activations)
+    generated_activations = math_ops.to_double(generated_activations)
+
+  # Compute means of activations.
+  m = math_ops.reduce_mean(real_activations, 0)
+  m_w = math_ops.reduce_mean(generated_activations, 0)
+
+  # Next the distance between means.
+  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
+  mofid = mean
+  if activations_dtype != dtypes.float64:
+    mofid = math_ops.cast(mofid, activations_dtype)
+
+  return mofid
+
+
+def diagonal_only_frechet_classifier_distance_from_activations(
+    real_activations, generated_activations):
+  """Classifier distance for evaluating a generative model.
+
+  This is based on the Frechet Inception distance, but for an arbitrary
+  classifier.
+
+  This technique is described in detail in https://arxiv.org/abs/1706.08500.
+  Given two Gaussian distribution with means m and m_w and covariance matrices
+  C and C_w, this function calcuates
+
+          |m - m_w|^2 + (sigma + sigma_w - 2(sigma x sigma_w)^(1/2))
+
+  which captures how different the distributions of real images and generated
+  images (or more accurately, their visual features) are. Note that unlike the
+  Inception score, this is a true distance and utilizes information about real
+  world images. In this variant, we compute diagonal-only covariance matrices.
+  As a result, instead of computing an expensive matrix square root, we can do
+  something much simpler, and has O(n) vs O(n^2) space complexity.
+
+  Note that when computed using sample means and sample covariance matrices,
+  Frechet distance is biased. It is more biased for small sample sizes. (e.g.
+  even if the two distributions are the same, for a small sample size, the
+  expected Frechet distance is large). It is important to use the same
+  sample size to compute frechet classifier distance when comparing two
+  generative models.
+
+  Args:
+    real_activations: Real images to use to compute Frechet Inception distance.
+    generated_activations: Generated images to use to compute Frechet Inception
+      distance.
+
+  Returns:
+    The diagonal-only Frechet Inception distance. A floating-point scalar of
+    the same type as the output of the activations.
+
+  Raises:
+    ValueError: If the shape of the variance and mean vectors are not equal.
+  """
+  real_activations.shape.assert_has_rank(2)
+  generated_activations.shape.assert_has_rank(2)
+
+  activations_dtype = real_activations.dtype
+  if activations_dtype != dtypes.float64:
+    real_activations = math_ops.to_double(real_activations)
+    generated_activations = math_ops.to_double(generated_activations)
+
+  # Compute mean and covariance matrices of activations.
+  m, var = nn_impl.moments(real_activations, axes=[0])
+  m_w, var_w = nn_impl.moments(generated_activations, axes=[0])
+
+  actual_shape = var.get_shape()
+  expected_shape = m.get_shape()
+
+  if actual_shape != expected_shape:
+    raise ValueError('shape: {} must match expected shape: {}'.format(
+        actual_shape, expected_shape))
+
+  # Compute the two components of FID.
+
+  # First the covariance component.
+  # Here, note that trace(A + B) = trace(A) + trace(B)
+  trace = math_ops.reduce_sum(
+      (var + var_w) - 2.0 * math_ops.sqrt(math_ops.multiply(var, var_w)))
+
+  # Next the distance between means.
+  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
+  dofid = trace + mean
+  if activations_dtype != dtypes.float64:
+    dofid = math_ops.cast(dofid, activations_dtype)
+
+  return dofid
+
+
+def frechet_classifier_distance_from_activations(real_activations,
+                                                 generated_activations):
+  """Classifier distance for evaluating a generative model.
+
   This methods computes the Frechet classifier distance from activations of
   real images and generated images. This can be used independently of the
   frechet_classifier_distance() method, especially in the case of using large
@@ -525,13 +659,20 @@ def frechet_classifier_distance_from_activations(
   Given two Gaussian distribution with means m and m_w and covariance matrices
   C and C_w, this function calcuates
 
-  |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
+                |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
 
   which captures how different the distributions of real images and generated
   images (or more accurately, their visual features) are. Note that unlike the
   Inception score, this is a true distance and utilizes information about real
   world images.
 
+  Note that when computed using sample means and sample covariance matrices,
+  Frechet distance is biased. It is more biased for small sample sizes. (e.g.
+  even if the two distributions are the same, for a small sample size, the
+  expected Frechet distance is large). It is important to use the same
+  sample size to compute frechet classifier distance when comparing two
+  generative models.
+
   Args:
     real_activations: 2D Tensor containing activations of real data. Shape is
       [batch_size, activation_size].
@@ -553,36 +694,37 @@ def frechet_classifier_distance_from_activations(
 
   # Compute mean and covariance matrices of activations.
   m = math_ops.reduce_mean(real_activations, 0)
-  m_v = math_ops.reduce_mean(generated_activations, 0)
+  m_w = math_ops.reduce_mean(generated_activations, 0)
   num_examples = math_ops.to_double(array_ops.shape(real_activations)[0])
 
   # sigma = (1 / (n - 1)) * (X - mu) (X - mu)^T
   real_centered = real_activations - m
   sigma = math_ops.matmul(
-      real_centered, real_centered, transpose_a=True) / (num_examples - 1)
+      real_centered, real_centered, transpose_a=True) / (
+          num_examples - 1)
 
-  gen_centered = generated_activations - m_v
-  sigma_v = math_ops.matmul(
-      gen_centered, gen_centered, transpose_a=True) / (num_examples - 1)
+  gen_centered = generated_activations - m_w
+  sigma_w = math_ops.matmul(
+      gen_centered, gen_centered, transpose_a=True) / (
+          num_examples - 1)
 
-  # Find the Tr(sqrt(sigma sigma_v)) component of FID
-  sqrt_trace_component = trace_sqrt_product(sigma, sigma_v)
+  # Find the Tr(sqrt(sigma sigma_w)) component of FID
+  sqrt_trace_component = trace_sqrt_product(sigma, sigma_w)
 
   # Compute the two components of FID.
 
   # First the covariance component.
   # Here, note that trace(A + B) = trace(A) + trace(B)
-  trace = math_ops.trace(sigma + sigma_v) - 2.0 * sqrt_trace_component
+  trace = math_ops.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component
 
   # Next the distance between means.
-  mean = math_ops.square(linalg_ops.norm(m - m_v))  # This uses the L2 norm.
+  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
   fid = trace + mean
   if activations_dtype != dtypes.float64:
     fid = math_ops.cast(fid, activations_dtype)
 
   return fid
 
-
 frechet_inception_distance = functools.partial(
     frechet_classifier_distance,
     classifier_fn=functools.partial(
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
index 61dc8646dd..663e49bdca 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_test.py
@@ -50,6 +50,26 @@ def _expected_inception_score(logits):
   return np.exp(np.mean(per_example_logincscore))
 
 
+def _expected_mean_only_fid(real_imgs, gen_imgs):
+  m = np.mean(real_imgs, axis=0)
+  m_v = np.mean(gen_imgs, axis=0)
+  mean = np.square(m - m_v).sum()
+  mofid = mean
+  return mofid
+
+
+def _expected_diagonal_only_fid(real_imgs, gen_imgs):
+  m = np.mean(real_imgs, axis=0)
+  m_v = np.mean(gen_imgs, axis=0)
+  var = np.var(real_imgs, axis=0)
+  var_v = np.var(gen_imgs, axis=0)
+  sqcc = np.sqrt(var * var_v)
+  mean = (np.square(m - m_v)).sum()
+  trace = (var + var_v - 2 * sqcc).sum()
+  dofid = mean + trace
+  return dofid
+
+
 def _expected_fid(real_imgs, gen_imgs):
   m = np.mean(real_imgs, axis=0)
   m_v = np.mean(gen_imgs, axis=0)
@@ -285,6 +305,46 @@ class ClassifierMetricsTest(test.TestCase):
 
     self.assertAllClose(_expected_inception_score(logits), incscore_np)
 
+  def test_mean_only_frechet_classifier_distance_value(self):
+    """Test that `frechet_classifier_distance` gives the correct value."""
+    np.random.seed(0)
+
+    pool_real_a = np.float32(np.random.randn(256, 2048))
+    pool_gen_a = np.float32(np.random.randn(256, 2048))
+
+    tf_pool_real_a = array_ops.constant(pool_real_a)
+    tf_pool_gen_a = array_ops.constant(pool_gen_a)
+
+    mofid_op = classifier_metrics.mean_only_frechet_classifier_distance_from_activations(  # pylint: disable=line-too-long
+        tf_pool_real_a, tf_pool_gen_a)
+
+    with self.test_session() as sess:
+      actual_mofid = sess.run(mofid_op)
+
+    expected_mofid = _expected_mean_only_fid(pool_real_a, pool_gen_a)
+
+    self.assertAllClose(expected_mofid, actual_mofid, 0.0001)
+
+  def test_diagonal_only_frechet_classifier_distance_value(self):
+    """Test that `frechet_classifier_distance` gives the correct value."""
+    np.random.seed(0)
+
+    pool_real_a = np.float32(np.random.randn(256, 2048))
+    pool_gen_a = np.float32(np.random.randn(256, 2048))
+
+    tf_pool_real_a = array_ops.constant(pool_real_a)
+    tf_pool_gen_a = array_ops.constant(pool_gen_a)
+
+    dofid_op = classifier_metrics.diagonal_only_frechet_classifier_distance_from_activations(  # pylint: disable=line-too-long
+        tf_pool_real_a, tf_pool_gen_a)
+
+    with self.test_session() as sess:
+      actual_dofid = sess.run(dofid_op)
+
+    expected_dofid = _expected_diagonal_only_fid(pool_real_a, pool_gen_a)
+
+    self.assertAllClose(expected_dofid, actual_dofid, 0.0001)
+
   def test_frechet_classifier_distance_value(self):
     """Test that `frechet_classifier_distance` gives the correct value."""
     np.random.seed(0)
-- 
GitLab


From 5f8fae9e8645be3ed76ba2b23a0d8c388e1e51e1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 13:19:16 -0700
Subject: [PATCH 074/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 189234789
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 31 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 31 +++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 85dd1a423a..1834cc998c 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -51522,6 +51522,37 @@ op {
     }
   }
 }
+op {
+  name: "SlideDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "stride"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "Snapshot"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 3faa4eeada..763b104305 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -24319,6 +24319,37 @@ op {
     }
   }
 }
+op {
+  name: "SlideDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "stride"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "Snapshot"
   input_arg {
-- 
GitLab


From c17ba11c799be3ab24b826a0f1bace86de26c055 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 13:46:52 -0700
Subject: [PATCH 075/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 189239161

---
 tensorflow/go/op/wrappers.go | 190 +++++++++++++++++------------------
 1 file changed, 95 insertions(+), 95 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 469d1e9adb..0424c12fd9 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -509,6 +509,101 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
+// Scatter `updates` into a new (initially zero) tensor according to `indices`.
+//
+// Creates a new tensor by applying sparse `updates` to individual
+// values or slices within a zero tensor of the given `shape` according to
+// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
+// extracts values or slices from a given tensor.
+//
+// **WARNING**: The order in which updates are applied is nondeterministic, so the
+// output will be nondeterministic if `indices` contains duplicates.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of scatter is to insert individual elements in a tensor by
+// index. For example, say we want to insert 4 scattered elements in a rank-1
+// tensor with 8 elements.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     shape = tf.constant([8])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [0, 11, 0, 10, 9, 0, 0, 12]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     shape = tf.constant([4, 4, 4])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
+//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
+//
+// Arguments:
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//	shape: 1-D. The shape of the resulting tensor.
+//
+// Returns A new tensor with the given shape and updates applied according
+// to the indices.
+func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ScatterNd",
+		Input: []tf.Input{
+			indices, updates, shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Replaces the contents of the table with the specified keys and values.
 //
 // The tensor `keys` must be of the same type as the keys of the table.
@@ -24972,101 +25067,6 @@ func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Scatter `updates` into a new (initially zero) tensor according to `indices`.
-//
-// Creates a new tensor by applying sparse `updates` to individual
-// values or slices within a zero tensor of the given `shape` according to
-// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
-// extracts values or slices from a given tensor.
-//
-// **WARNING**: The order in which updates are applied is nondeterministic, so the
-// output will be nondeterministic if `indices` contains duplicates.
-//
-// `indices` is an integer tensor containing indices into a new tensor of shape
-// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
-//
-//     indices.shape[-1] <= shape.rank
-//
-// The last dimension of `indices` corresponds to indices into elements
-// (if `indices.shape[-1] = shape.rank`) or slices
-// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
-// `shape`.  `updates` is a tensor with shape
-//
-//     indices.shape[:-1] + shape[indices.shape[-1]:]
-//
-// The simplest form of scatter is to insert individual elements in a tensor by
-// index. For example, say we want to insert 4 scattered elements in a rank-1
-// tensor with 8 elements.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[4], [3], [1], [7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     shape = tf.constant([8])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [0, 11, 0, 10, 9, 0, 0, 12]
-//
-// We can also, insert entire slices of a higher rank tensor all at once. For
-// example, if we wanted to insert two slices in the first dimension of a
-// rank-3 tensor with two matrices of new values.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[0], [2]])
-//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]],
-//                            [[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
-//     shape = tf.constant([4, 4, 4])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
-//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
-//
-// Arguments:
-//	indices: Index tensor.
-//	updates: Updates to scatter into output.
-//	shape: 1-D. The shape of the resulting tensor.
-//
-// Returns A new tensor with the given shape and updates applied according
-// to the indices.
-func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ScatterNd",
-		Input: []tf.Input{
-			indices, updates, shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // SpaceToDepthAttr is an optional argument to SpaceToDepth.
 type SpaceToDepthAttr func(optionalAttr)
 
-- 
GitLab


From 856438f65e5705b373413bce29758a92194ff9b6 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 15 Mar 2018 13:52:16 -0700
Subject: [PATCH 076/960] Fix a bug which caused slot variables to be shared
 when executing eagerly

Threads the uid() name of a ResourceVariable through to the Optimizer. Tests
that slot variables are unique in several ways.

Previously ResourceVariable._shared_name was the Optimizer's slot key for a
variable, which for tfe.Variable() is just the non-uniquified name.

PiperOrigin-RevId: 189240115
---
 .../eager/python/checkpointable_utils_test.py | 30 +++++++++++++++++++
 .../python/ops/resource_variable_ops.py       | 24 +++++++++------
 tensorflow/python/training/adam_test.py       |  9 ++++++
 tensorflow/python/training/optimizer.py       |  2 +-
 4 files changed, 55 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 1ab94b88bd..4e0a9923ff 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -23,6 +23,7 @@ import six
 
 from tensorflow.contrib.eager.python import checkpointable_utils
 from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
 from tensorflow.python.eager import context
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
@@ -523,6 +524,35 @@ class CheckpointingTests(test.TestCase):
     name, = named_variables.keys()
     self.assertEqual(name, "..ATTRIBUTES/a/.ATTRIBUTES/VARIABLE_VALUE")
 
+  def testAnonymousVarsInInit(self):
+
+    class Model(training.Model):
+
+      def __init__(self):
+        super(Model, self).__init__()
+        self.w = resource_variable_ops.ResourceVariable(0.0)
+        self.b = resource_variable_ops.ResourceVariable(0.0)
+        self.vars = [self.w, self.b]
+
+      def call(self, x):
+        return x * self.w + self.b
+
+    with context.eager_mode():
+      model = Model()
+      optimizer = adam.AdamOptimizer(learning_rate=0.05)
+      checkpoint_directory = self.get_temp_dir()
+      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+      checkpoint = checkpointable_utils.Checkpoint(
+          model=model, optimizer=optimizer)
+      for _ in range(2):
+        with backprop.GradientTape() as tape:
+          loss = (constant_op.constant(1.)
+                  - model(constant_op.constant(1.))) ** 2
+        grad = tape.gradient(loss, model.vars)
+        optimizer.apply_gradients(
+            [(g, v) for g, v in zip(grad, model.vars)])
+        checkpoint.save(checkpoint_prefix)
+
   @test_util.run_in_graph_and_eager_modes()
   def testLateDependencyTracking(self):
 
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 54191ee765..affa7ae629 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -46,10 +46,6 @@ def _eager_safe_variable_handle(shape, dtype, shared_name, name, graph_mode):
   container = ops.get_default_graph()._container  # pylint: disable=protected-access
   if container is None:
     container = ""
-  if not graph_mode:
-    # When in eager mode use a uid for the shared_name, to prevent accidental
-    # sharing.
-    shared_name = str(ops.uid())
   handle = gen_resource_variable_ops.var_handle_op(shape=shape, dtype=dtype,
                                                    shared_name=shared_name,
                                                    name=name,
@@ -368,6 +364,12 @@ class ResourceVariable(variables.Variable):
                           if init_from_fn else [initial_value]) as name:
         # pylint: disable=protected-access
         handle_name = ops._name_from_scope_name(name)
+        if self._in_graph_mode:
+          shared_name = handle_name
+        else:
+          # When in eager mode use a uid for the shared_name, to prevent
+          # accidental sharing.
+          shared_name = "%s_%d" % (handle_name, ops.uid())
         if init_from_fn:
           # Use attr_scope and device(None) to simulate the behavior of
           # colocate_with when the variable we want to colocate with doesn't
@@ -383,7 +385,7 @@ class ResourceVariable(variables.Variable):
               self._handle = _eager_safe_variable_handle(
                   shape=initial_value.get_shape(),
                   dtype=initial_value.dtype.base_dtype,
-                  shared_name=handle_name,
+                  shared_name=shared_name,
                   name=name,
                   graph_mode=self._in_graph_mode)
               self._shape = initial_value.get_shape()
@@ -395,7 +397,7 @@ class ResourceVariable(variables.Variable):
             self._handle = _eager_safe_variable_handle(
                 shape=initial_value.get_shape(),
                 dtype=initial_value.dtype.base_dtype,
-                shared_name=handle_name,
+                shared_name=shared_name,
                 name=name,
                 graph_mode=False)
             self._shape = initial_value.get_shape()
@@ -418,11 +420,12 @@ class ResourceVariable(variables.Variable):
           self._handle = _eager_safe_variable_handle(
               shape=initial_value.get_shape(),
               dtype=initial_value.dtype.base_dtype,
-              shared_name=handle_name,
+              shared_name=shared_name,
               name=name,
               graph_mode=self._in_graph_mode)
           self._shape = initial_value.get_shape()
 
+        self._unique_id = shared_name
         self._initial_value = initial_value if self._in_graph_mode else None
         self._handle_name = handle_name + ":0"
         self._dtype = initial_value.dtype.base_dtype
@@ -503,6 +506,7 @@ class ResourceVariable(variables.Variable):
     self._shape = tensor_shape.TensorShape(
         self._handle.op.get_attr("shape"))
     self._handle_name = self._handle.name
+    self._unique_id = self._handle_name
     self._initializer_op = g.as_graph_element(
         ops.prepend_name_scope(
             variable_def.initializer_name, import_scope=import_scope))
@@ -851,7 +855,8 @@ class ResourceVariable(variables.Variable):
       tape.watch_variable(self)
     return _UnreadVariable(
         self._handle, self.dtype, self._shape, self._in_graph_mode,
-        self._handle_deleter if not self._in_graph_mode else None, op)
+        self._handle_deleter if not self._in_graph_mode else None, op,
+        self._unique_id)
 
   def assign(self, value, use_locking=None, name=None, read_value=True):
     """Assigns a new value to this variable.
@@ -966,7 +971,7 @@ class _UnreadVariable(ResourceVariable):
   """
 
   def __init__(self, handle, dtype,  # pylint: disable=super-init-not-called
-               shape, in_graph_mode, deleter, parent_op):
+               shape, in_graph_mode, deleter, parent_op, unique_id):
     # We do not call super init on purpose.
     self._trainable = False
     self._save_slice_info = None
@@ -979,6 +984,7 @@ class _UnreadVariable(ResourceVariable):
       self._handle_name = ""
     else:
       self._handle_name = self._handle.name
+    self._unique_id = unique_id
     self._dtype = dtype
     self._constraint = None
     self._cached_value = None
diff --git a/tensorflow/python/training/adam_test.py b/tensorflow/python/training/adam_test.py
index af87d6f0e5..9be8b6aafe 100644
--- a/tensorflow/python/training/adam_test.py
+++ b/tensorflow/python/training/adam_test.py
@@ -319,6 +319,15 @@ class AdamOptimizerTest(test.TestCase):
         # fails.
         optimizer.apply_gradients([(grads0, var0)])
 
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = resource_variable_ops.ResourceVariable(1.)
+      v2 = resource_variable_ops.ResourceVariable(1.)
+      opt = adam.AdamOptimizer(1.)
+      opt.minimize(lambda: v1 + v2)
+      # There should be two non-slot variables, and two unique slot variables
+      # for v1 and v2 respectively.
+      self.assertEqual(6, len(set(opt.variables())))
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index 7adaedef5b..af9cc3491c 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -61,7 +61,7 @@ def _deduplicate_indexed_slices(values, indices):
 
 def _var_key(var):
   if context.executing_eagerly():
-    return var._shared_name  # pylint: disable=protected-access
+    return var._unique_id  # pylint: disable=protected-access
   return (var.op.graph, var.op.name)
 
 
-- 
GitLab


From 002b488dcd9c1eaac4f4aba2ac1301c32c6beb06 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 14:27:04 -0700
Subject: [PATCH 077/960] Fix the HLO alias analysis and copy insertion to cope
 with the new kConditional instruction.

PiperOrigin-RevId: 189245979
---
 .../compiler/xla/service/copy_insertion.cc    | 44 +++++++++---
 .../xla/service/hlo_alias_analysis.cc         | 70 +++++++++++++++----
 .../xla/service/hlo_dataflow_analysis.cc      | 10 +--
 .../xla/service/hlo_dataflow_analysis_test.cc | 67 +++++++++++-------
 .../compiler/xla/service/hlo_ordering.cc      | 42 +++++++++--
 .../compiler/xla/service/hlo_ordering_test.cc | 61 ++++++++++++++++
 .../compiler/xla/tests/conditional_test.cc    | 51 ++++++++++++++
 7 files changed, 287 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/xla/service/copy_insertion.cc b/tensorflow/compiler/xla/service/copy_insertion.cc
index e9c974a046..40519ecc79 100644
--- a/tensorflow/compiler/xla/service/copy_insertion.cc
+++ b/tensorflow/compiler/xla/service/copy_insertion.cc
@@ -78,8 +78,9 @@ SpecialCaseCopyPolicy GetSpecialCaseCopyPolicy(const CallGraphNode& node,
     policy.copy_root_replicated_buffers = true;
   }
   for (const CallSite& site : node.caller_callsites()) {
-    // The kWhile instruction does not have an handling here, as the
-    // AddCopiesForWhile() API takes care of adding its own copies.
+    // The AddCopiesForConditional() already adds copies, but the copy remover
+    // removes them, so we re-add them by returning the policy here. But really
+    // the copy remover should not be removing them.
     if (site.instruction()->opcode() == HloOpcode::kConditional) {
       policy.copy_parameters_and_constants = true;
       policy.copy_root_replicated_buffers = true;
@@ -321,6 +322,29 @@ Status AddCopiesForWhile(const HloAliasAnalysis& alias_analysis,
   return Status::OK();
 }
 
+// We add copies for all the indices of the true and false computaiton roots,
+// in order to resolve interference. We later rely on the CopyRemover to drop
+// the unnecessary ones.
+Status AddCopiesForConditional(const HloAliasAnalysis& alias_analysis,
+                               HloInstruction* conditional) {
+  VLOG(2) << "Adding copies for kConditional instruction "
+          << conditional->name();
+  TF_RET_CHECK(conditional->opcode() == HloOpcode::kConditional);
+
+  for (HloComputation* computation :
+       {conditional->true_computation(), conditional->false_computation()}) {
+    HloInstruction* root = computation->root_instruction();
+    std::vector<HloInstruction*> users = root->users();
+    TF_ASSIGN_OR_RETURN(HloInstruction * deep_copy,
+                        computation->DeepCopyInstruction(root));
+    for (HloInstruction* user : users) {
+      TF_RETURN_IF_ERROR(root->ReplaceUseWith(user, deep_copy));
+    }
+    computation->set_root_instruction(deep_copy);
+  }
+  return Status::OK();
+}
+
 // Removes any control dependencies to or from the given instruction.
 Status StripControlDependenciesFrom(HloInstruction* instruction) {
   while (!instruction->control_successors().empty()) {
@@ -348,6 +372,9 @@ Status AddCopiesToResolveInterference(HloModule* module) {
     for (HloInstruction* instruction : computation->instructions()) {
       if (instruction->opcode() == HloOpcode::kWhile) {
         TF_RETURN_IF_ERROR(AddCopiesForWhile(*alias_analysis, instruction));
+      } else if (instruction->opcode() == HloOpcode::kConditional) {
+        TF_RETURN_IF_ERROR(
+            AddCopiesForConditional(*alias_analysis, instruction));
       }
     }
   }
@@ -596,6 +623,7 @@ class CopyRemover {
 
       auto is_live_range_before = [this](const ValueNode& a,
                                          const ValueNode& b) {
+        VLOG(3) << "Checking live range of " << *a.value << " WRT " << *b.value;
         if (LiveRangeBefore(a, b)) {
           VLOG(2) << "  Live range of " << a.value->ToShortString()
                   << " is before " << b.value->ToShortString();
@@ -610,7 +638,7 @@ class CopyRemover {
       VLOG(3) << copy->name() << " copies value "
               << src->value->ToShortString();
       VLOG(3) << "Source buffer values: " << ValueListToString(src);
-      VLOG(3) << "Dest buffer values: " << ValueListToString(src);
+      VLOG(3) << "Dest buffer values: " << ValueListToString(dest);
 
       // A kCopy instruction copies an HLO value from a source buffer and
       // defines an HLO value in a destination buffer. Most generally, the
@@ -786,16 +814,16 @@ class CopyRemover {
     // updated as copies are removed.
     bool LiveRangeBefore(const ValueNode& a, const ValueNode& b) {
       if (a.uses.empty()) {
-        VLOG(2) << "Empty uses";
+        VLOG(2) << "Empty uses for " << *a.value;
         return ordering_.IsDefinedBefore(*a.value, *b.value);
       }
       for (const HloUse* use : a.uses) {
-        VLOG(2) << "use: " << *use;
-        VLOG(2) << "is before:" << *b.value;
+        VLOG(2) << "Checking use " << *use << " against " << *b.value;
         if (!ordering_.UseIsBeforeValueDefinition(*use, *b.value, dataflow_)) {
-          VLOG(2) << "Not before";
+          VLOG(2) << "Use " << *use << " is NOT before " << *b.value;
           return false;
         }
+        VLOG(2) << "Use " << *use << " is before " << *b.value;
       }
       return true;
     }
@@ -931,7 +959,6 @@ Status RemoveUnnecessaryCopies(
   CopyRemover copy_remover(*alias_analysis, ordering, module);
   XLA_VLOG_LINES(3, copy_remover.ToString());
 
-  tensorflow::gtl::FlatSet<int> existing_copies;
   for (HloComputation* computation : module->computations()) {
     for (HloInstruction* instruction : computation->instructions()) {
       if (instruction->opcode() == HloOpcode::kCopy &&
@@ -940,7 +967,6 @@ Status RemoveUnnecessaryCopies(
       }
     }
   }
-
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
index 30e32a46d7..a88283ed9a 100644
--- a/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_alias_analysis.cc
@@ -171,24 +171,21 @@ class BufferValueMap {
     return value_to_buffer_number_.at(&value);
   }
 
-  // Compute and return a vector of buffers that the given value must be
-  // contained in due to HLO aliasing rules.
-  std::vector<BufferNumber> ComputeAliasedBuffers(const HloValue& value) {
+  void ComputeWhileAliasedBuffers(const HloValue& value,
+                                  std::vector<BufferNumber>* aliased_buffers) {
+    VLOG(3) << "Compute kWhile aliases";
     // Value is init of a while (use is while).
-    std::vector<BufferNumber> aliased_buffers;
     for (const HloUse& use : value.uses()) {
-      VLOG(2) << "use of value " << value.ToShortString() << ": " << use;
       if (use.instruction->opcode() == HloOpcode::kWhile) {
         // Determine the while value that this shares a buffer with.
         const HloValue& while_value =
             dataflow_.GetUniqueValueAt(use.instruction, use.operand_index);
-        aliased_buffers.push_back(GetBufferForValue(while_value));
+        aliased_buffers->push_back(GetBufferForValue(while_value));
         VLOG(3) << "  value is init value to a while; must share buffer with "
                    "while value "
                 << while_value.ToShortString();
       }
     }
-
     // Value is a parameter of a while body/condition.
     if (value.defining_instruction()->opcode() == HloOpcode::kParameter) {
       const HloComputation* computation =
@@ -205,11 +202,10 @@ class BufferValueMap {
           VLOG(3) << "  value is parameter value of the body or condition of a "
                      "while; must share buffer with while value "
                   << while_value.ToShortString();
-          aliased_buffers.push_back(GetBufferForValue(while_value));
+          aliased_buffers->push_back(GetBufferForValue(while_value));
         }
       }
     }
-
     // Value is the root of a while body.
     for (const HloPosition& position : value.positions()) {
       const HloComputation* computation = position.instruction->parent();
@@ -224,27 +220,71 @@ class BufferValueMap {
 
             const HloValue& while_value = dataflow_.GetUniqueValueAt(
                 callsite.instruction(), position.index);
-            VLOG(3) << "  value is root the body computation of a while; must "
-                       "share buffer with while value "
+            VLOG(3) << "  value @ " << position << " is root of "
+                    << callsite.instruction()->name()
+                    << "; body root and while value root must share buffer "
+                       "among them : "
                     << while_value.ToShortString();
-            aliased_buffers.push_back(GetBufferForValue(while_value));
+            aliased_buffers->push_back(GetBufferForValue(while_value));
           }
         }
       }
     }
-
     // Value is the output of the while instruction itself.
     if (value.defining_instruction()->opcode() == HloOpcode::kWhile) {
       VLOG(3) << "  value is output of a while instruction";
-      aliased_buffers.push_back(GetBufferForValue(value));
+      aliased_buffers->push_back(GetBufferForValue(value));
+    }
+  }
+
+  void ComputeConditionalAliasedBuffers(
+      const HloValue& value, std::vector<BufferNumber>* aliased_buffers) {
+    VLOG(3) << "Compute kConditional aliases";
+    // Aliases the buffers of the true/false computations roots, with the one of
+    // the conditional.
+    for (const HloPosition& position : value.positions()) {
+      const HloComputation* computation = position.instruction->parent();
+      const CallGraphNode& call_graph_node =
+          dataflow_.call_graph().GetNode(computation);
+      if (position.instruction == computation->root_instruction()) {
+        for (const CallSite& callsite : call_graph_node.caller_callsites()) {
+          if (callsite.instruction()->opcode() == HloOpcode::kConditional) {
+            // Call graph must have been flattened.
+            CHECK_EQ(call_graph_node.caller_callsites().size(), 1);
+
+            const HloValue& cond_value = dataflow_.GetUniqueValueAt(
+                callsite.instruction(), position.index);
+            VLOG(3)
+                << "  value @ " << position << " is root of "
+                << callsite.instruction()->name()
+                << "; true/false branch roots must share buffer among them : "
+                << cond_value.ToShortString();
+            aliased_buffers->push_back(GetBufferForValue(cond_value));
+          }
+        }
+      }
+    }
+    // Value is the output of the conditional instruction itself.
+    if (value.defining_instruction()->opcode() == HloOpcode::kConditional) {
+      VLOG(3) << "  value is output of a conditional instruction";
+      aliased_buffers->push_back(GetBufferForValue(value));
     }
+  }
 
+  // Compute and return a vector of buffers that the given value must be
+  // contained in due to HLO aliasing rules.
+  std::vector<BufferNumber> ComputeAliasedBuffers(const HloValue& value) {
+    for (const HloUse& use : value.uses()) {
+      VLOG(2) << "Use of value " << value.ToShortString() << ": " << use;
+    }
+    std::vector<BufferNumber> aliased_buffers;
+    ComputeWhileAliasedBuffers(value, &aliased_buffers);
+    ComputeConditionalAliasedBuffers(value, &aliased_buffers);
     // Uniquify aliased buffers.
     std::sort(aliased_buffers.begin(), aliased_buffers.end());
     aliased_buffers.erase(
         std::unique(aliased_buffers.begin(), aliased_buffers.end()),
         aliased_buffers.end());
-
     return aliased_buffers;
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
index 934e43ba48..0c37a8d75f 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis.cc
@@ -368,11 +368,11 @@ bool HloDataflowAnalysis::UpdateConditionalValueSet(
           conditional->true_computation()->root_instruction()),
       &GetInstructionValueSet(
           conditional->false_computation()->root_instruction())};
-  // A phi-node is not defined for a kConditional instruction even though it
-  // represents a join point. This is because the current approach is to define
-  // a phi-node only for kWhile to account for the dataflow through back-edges
-  // and deal with the ambiguity in other cases.
-  return GetInstructionValueSet(conditional).AssignUnionOf(inputs);
+  if (ssa_form_) {
+    return Phi(conditional, inputs);
+  } else {
+    return GetInstructionValueSet(conditional).AssignUnionOf(inputs);
+  }
 }
 
 bool HloDataflowAnalysis::UpdateCopyValueSet(HloInstruction* copy) {
diff --git a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
index 7bf3a1a060..07f69b8e13 100644
--- a/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_dataflow_analysis_test.cc
@@ -1602,11 +1602,17 @@ TEST_P(HloDataflowAnalysisTest, ConditionalWithIdentity) {
   EXPECT_THAT(analysis.GetValueDefinedAt(constant2).uses(),
               ElementsAre(HloUse{conditional, 2, {}}));
 
-  EXPECT_EQ(analysis.values().size(), 3);
-  EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
-  EXPECT_THAT(HloValuesAt(conditional),
-              UnorderedElementsAre(analysis.GetValueDefinedAt(constant1),
-                                   analysis.GetValueDefinedAt(constant2)));
+  bool ssa_form = GetParam();
+  if (ssa_form) {
+    EXPECT_EQ(analysis.values().size(), 4);
+    EXPECT_TRUE(analysis.ValueIsDefinedAt(conditional));
+  } else {
+    EXPECT_EQ(analysis.values().size(), 3);
+    EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
+    EXPECT_THAT(HloValuesAt(conditional),
+                UnorderedElementsAre(analysis.GetValueDefinedAt(constant1),
+                                     analysis.GetValueDefinedAt(constant2)));
+  }
 }
 
 TEST_P(HloDataflowAnalysisTest, ConditionalTakingTupleOperand) {
@@ -1713,11 +1719,17 @@ TEST_P(HloDataflowAnalysisTest, ConditionalTakingTupleOperand) {
                   HloUse{true_x, 0, {}}, HloUse{true_y, 0, {}},
                   HloUse{false_x, 0, {}}, HloUse{false_y, 0, {}}));
 
-  EXPECT_EQ(analysis.values().size(), 6);
-  EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
-  EXPECT_THAT(HloValuesAt(conditional),
-              UnorderedElementsAre(analysis.GetValueDefinedAt(add),
-                                   analysis.GetValueDefinedAt(sub)));
+  bool ssa_form = GetParam();
+  if (ssa_form) {
+    EXPECT_EQ(analysis.values().size(), 7);
+    EXPECT_TRUE(analysis.ValueIsDefinedAt(conditional));
+  } else {
+    EXPECT_EQ(analysis.values().size(), 6);
+    EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
+    EXPECT_THAT(HloValuesAt(conditional),
+                UnorderedElementsAre(analysis.GetValueDefinedAt(add),
+                                     analysis.GetValueDefinedAt(sub)));
+  }
 }
 
 TEST_P(HloDataflowAnalysisTest, NestedConditionals) {
@@ -1834,20 +1846,27 @@ TEST_P(HloDataflowAnalysisTest, NestedConditionals) {
   EXPECT_EQ(analysis.GetUniqueValueAt(false_operand_cond),
             analysis.GetValueDefinedAt(constant2));
 
-  EXPECT_EQ(analysis.values().size(), 9);
-  EXPECT_FALSE(analysis.ValueIsDefinedAt(inner_conditional));
-  EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
-  EXPECT_THAT(
-      HloValuesAt(inner_conditional),
-      UnorderedElementsAre(
-          analysis.GetValueDefinedAt(computation1->root_instruction()),
-          analysis.GetValueDefinedAt(computation2->root_instruction())));
-  EXPECT_THAT(
-      HloValuesAt(conditional),
-      UnorderedElementsAre(
-          analysis.GetValueDefinedAt(computation1->root_instruction()),
-          analysis.GetValueDefinedAt(computation2->root_instruction()),
-          analysis.GetValueDefinedAt(computation3->root_instruction())));
+  bool ssa_form = GetParam();
+  if (ssa_form) {
+    EXPECT_EQ(analysis.values().size(), 11);
+    EXPECT_TRUE(analysis.ValueIsDefinedAt(inner_conditional));
+    EXPECT_TRUE(analysis.ValueIsDefinedAt(conditional));
+  } else {
+    EXPECT_EQ(analysis.values().size(), 9);
+    EXPECT_FALSE(analysis.ValueIsDefinedAt(inner_conditional));
+    EXPECT_FALSE(analysis.ValueIsDefinedAt(conditional));
+    EXPECT_THAT(
+        HloValuesAt(inner_conditional),
+        UnorderedElementsAre(
+            analysis.GetValueDefinedAt(computation1->root_instruction()),
+            analysis.GetValueDefinedAt(computation2->root_instruction())));
+    EXPECT_THAT(
+        HloValuesAt(conditional),
+        UnorderedElementsAre(
+            analysis.GetValueDefinedAt(computation1->root_instruction()),
+            analysis.GetValueDefinedAt(computation2->root_instruction()),
+            analysis.GetValueDefinedAt(computation3->root_instruction())));
+  }
 }
 
 INSTANTIATE_TEST_CASE_P(HloDataflowAnalysisInstantiation,
diff --git a/tensorflow/compiler/xla/service/hlo_ordering.cc b/tensorflow/compiler/xla/service/hlo_ordering.cc
index 1b24d8da9e..e89d94bede 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering.cc
@@ -66,6 +66,28 @@ bool HloOrdering::ExecutesBefore(const HloInstruction* a,
     }
   }
 
+  // If the common ancestor is a conditional instruction, even though the true
+  // and false computations are not really ordered per-se, we define the true
+  // computation to be ordered before the false one.
+  // This ensures that buffers can still be shared among the two computations
+  // as they will forcibly have disjoint liveness.
+  if (a_ancestor == b_ancestor &&
+      a_ancestor->opcode() == HloOpcode::kConditional) {
+    const HloComputation* true_computation = a_ancestor->true_computation();
+    const HloComputation* false_computation = a_ancestor->false_computation();
+    if (call_graph_->InstructionIsNestedIn(a, true_computation) &&
+        call_graph_->InstructionIsNestedIn(b, false_computation)) {
+      return true;
+    }
+    // If 'b' is the conditional ancestor, and 'a' is within the true or false
+    // computations, 'a' executes before 'b'.
+    if (b == a_ancestor &&
+        (call_graph_->InstructionIsNestedIn(a, true_computation) ||
+         call_graph_->InstructionIsNestedIn(a, false_computation))) {
+      return true;
+    }
+  }
+
   return ExecutesBeforeInSameComputation(a_ancestor, b_ancestor);
 }
 
@@ -118,7 +140,18 @@ bool HloOrdering::IsDefinedBefore(const HloValue& a, const HloValue& b) const {
            b.defining_instruction()->while_condition()))) {
     return true;
   }
-
+  // If 'b' is a conditional phi and 'a' is in the true or false computation,
+  // then 'a' executes before 'b'.
+  if (b.is_phi() &&
+      b.defining_instruction()->opcode() == HloOpcode::kConditional &&
+      (call_graph_->InstructionIsNestedIn(
+           a.defining_instruction(),
+           b.defining_instruction()->true_computation()) ||
+       call_graph_->InstructionIsNestedIn(
+           a.defining_instruction(),
+           b.defining_instruction()->false_computation()))) {
+    return true;
+  }
   return ExecutesBefore(a.defining_instruction(), b.defining_instruction());
 }
 
@@ -212,18 +245,17 @@ bool HloOrdering::LiveRangeStrictlyBefore(
   VLOG(4) << "LiveRangeStrictlyBefore(a = " << a.ToShortString()
           << ", b = " << b.ToShortString() << ")";
   if (!IsDefinedBefore(a, b)) {
-    VLOG(4) << "a not defined before b";
+    VLOG(4) << a << " not defined before " << b;
     return false;
   }
-
   // All uses of 'a' must be before 'b' is defined.
   for (const HloUse& use : a.uses()) {
     if (!UseIsBeforeValueDefinition(use, b, dataflow)) {
-      VLOG(4) << "use of a (" << use << ") not before b is defined";
+      VLOG(4) << "use of " << a << " (" << use << ") not before " << b
+              << " is defined";
       return false;
     }
   }
-
   return true;
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
index a989fce632..441d790f0e 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
@@ -362,5 +362,66 @@ ENTRY while.v11 {
   ordering.ToString();  // Shouldn't crash.
 }
 
+TEST_F(HloOrderingTest, ConditionalInstructionOrdering) {
+  const char* module_str = R"(
+HloModule test_conditional_module
+
+true_branch {
+  param.1 = (s32[], s32[]) parameter(0)
+  get-tuple-element.1 = s32[] get-tuple-element(param.1), index=0
+  get-tuple-element.2 = s32[] get-tuple-element(param.1), index=1
+  add.1 = s32[] add(get-tuple-element.1, get-tuple-element.2)
+  ROOT tuple.1 = (s32[], s32[]) tuple(add.1, get-tuple-element.1)
+}
+
+false_branch {
+  param.2 = (s32[], s32[]) parameter(0)
+  get-tuple-element.3 = s32[] get-tuple-element(param.2), index=0
+  get-tuple-element.4 = s32[] get-tuple-element(param.2), index=1
+  add.2 = s32[] add(get-tuple-element.3, get-tuple-element.4)
+  ROOT tuple.2 = (s32[], s32[]) tuple(add.2, get-tuple-element.4)
+}
+
+ENTRY root {
+  param.3 = (pred[], (s32[], s32[])) parameter(0)
+  pred.1 = pred[] get-tuple-element(param.3), index=0
+  cond_arg.1 = (s32[], s32[]) get-tuple-element(param.3), index=1
+  conditional = (s32[], s32[]) conditional(pred.1, cond_arg.1, cond_arg.1), true_computation=true_branch, false_computation=false_branch
+  cond_res.1 = s32[] get-tuple-element(conditional), index=0
+  cond_res.2 = s32[] get-tuple-element(conditional), index=1
+  add.3 = s32[] add(cond_res.1, cond_res.2)
+  ROOT result = (s32[], s32[], s32[]) tuple(add.3, cond_res.1, cond_res.2)
+})";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          tools::Parse(module_str));
+  TF_ASSERT_OK_AND_ASSIGN(auto dataflow,
+                          HloDataflowAnalysis::Run(*module, /*ssa_form=*/true));
+  DependencyHloOrdering ordering(module.get());
+
+  // Even though the true and false branches has no ordering, since they do not
+  // interfere (as they are mutually exclusive), we define the true computation
+  // to be before the false one.
+  // Similarly, any instruction in the true or false branches are considered
+  // before the conditional instruction. The roots are effectively "at the same
+  // time" WRT the conditional, but they are Phi-ed anyway.
+  HloInstruction* add_1 = FindInstruction(module.get(), "add.1");
+  HloInstruction* add_2 = FindInstruction(module.get(), "add.2");
+  HloInstruction* add_3 = FindInstruction(module.get(), "add.3");
+  HloInstruction* conditional = FindInstruction(module.get(), "conditional");
+  EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(add_1),
+                                       dataflow->GetValueDefinedAt(add_2)));
+  EXPECT_TRUE(
+      ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(add_2),
+                               dataflow->GetValueDefinedAt(conditional)));
+  EXPECT_TRUE(
+      ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(add_1),
+                               dataflow->GetValueDefinedAt(conditional)));
+  EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(add_1),
+                                       dataflow->GetValueDefinedAt(add_3)));
+  EXPECT_TRUE(ordering.IsDefinedBefore(dataflow->GetValueDefinedAt(add_2),
+                                       dataflow->GetValueDefinedAt(add_3)));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/conditional_test.cc b/tensorflow/compiler/xla/tests/conditional_test.cc
index bc82167482..b917dee77b 100644
--- a/tensorflow/compiler/xla/tests/conditional_test.cc
+++ b/tensorflow/compiler/xla/tests/conditional_test.cc
@@ -571,5 +571,56 @@ XLA_TEST_F(ConditionalOpTest, ShapeMismatch) {
                                    "only parameter of true_computation"));
 }
 
+XLA_TEST_F(ConditionalOpTest, SwappedInputsInSequentialConditionals) {
+  Shape tuple_shape = ShapeUtil::MakeTupleShape({r0f32_, r0f32_});
+  Computation swapper;
+  {
+    ComputationBuilder builder(client_, TestName() + ".swapper");
+    auto param0 = builder.Parameter(0, tuple_shape, "sp0");
+    auto x = builder.GetTupleElement(param0, 0);
+    auto y = builder.GetTupleElement(param0, 1);
+    builder.Tuple({y, x});
+    swapper = builder.Build().ConsumeValueOrDie();
+  }
+  Computation forwarder;
+  {
+    ComputationBuilder builder(client_, TestName() + ".forwarder");
+    auto param0 = builder.Parameter(0, tuple_shape, "fp0");
+    auto x = builder.GetTupleElement(param0, 0);
+    auto y = builder.GetTupleElement(param0, 1);
+    builder.Tuple({x, y});
+    forwarder = builder.Build().ConsumeValueOrDie();
+  }
+  Computation main;
+  {
+    ComputationBuilder builder(client_, TestName() + ".main");
+    auto param0 = builder.Parameter(0, tuple_shape, "mp0");
+    auto x = builder.GetTupleElement(param0, 0);
+    auto y = builder.GetTupleElement(param0, 1);
+    auto lt_pred = builder.Lt(x, y);
+    auto res = builder.Conditional(lt_pred, param0, forwarder, param0, swapper);
+    auto ge_pred = builder.Ge(x, y);
+    builder.Conditional(ge_pred, res, swapper, res, forwarder);
+    main = builder.Build().ConsumeValueOrDie();
+  }
+
+  auto test_swap = [&](float a, float b) {
+    ComputationBuilder builder(client_, TestName());
+    auto x = builder.ConstantR0<float>(a);
+    auto y = builder.ConstantR0<float>(b);
+    auto tuple_operand = builder.Tuple({x, y});
+    builder.Call(main, {tuple_operand});
+
+    ComputeAndCompareTuple(
+        &builder,
+        *Literal::MakeTuple({Literal::CreateR0<float>(a).get(),
+                             Literal::CreateR0<float>(b).get()}),
+        {}, error_spec_);
+  };
+
+  test_swap(3.11f, 9.4f);
+  test_swap(11.24f, 5.55f);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From eb9b8aaecedc89f6375167e001a1096281d7191d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 14:36:53 -0700
Subject: [PATCH 078/960] Internal cleanup.

PiperOrigin-RevId: 189247461
---
 .../contrib/lite/java/demo/app/src/main/BUILD |    2 +-
 .../java/demo/app/src/main/assets/labels.txt  | 1001 -----------------
 .../Camera2BasicFragment.java                 |   12 +-
 .../ImageClassifierQuantizedMobileNet.java    |    3 +-
 4 files changed, 8 insertions(+), 1010 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/java/demo/app/src/main/assets/labels.txt

diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/BUILD b/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
index 654fa9d6d2..5eb749aae6 100644
--- a/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
+++ b/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
@@ -6,7 +6,7 @@ android_binary(
     name = "TfLiteCameraDemo",
     srcs = glob(["java/**/*.java"]),
     assets = [
-        "@tflite_mobilenet//:labels.txt",
+        "//tensorflow/contrib/lite/java/demo/app/src/main/assets:labels_mobilenet_quant_v1_224.txt",
         "@tflite_mobilenet//:mobilenet_quant_v1_224.tflite",
     ],
     assets_dir = "",
diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/assets/labels.txt b/tensorflow/contrib/lite/java/demo/app/src/main/assets/labels.txt
deleted file mode 100644
index fe811239d8..0000000000
--- a/tensorflow/contrib/lite/java/demo/app/src/main/assets/labels.txt
+++ /dev/null
@@ -1,1001 +0,0 @@
-background
-tench
-goldfish
-great white shark
-tiger shark
-hammerhead
-electric ray
-stingray
-cock
-hen
-ostrich
-brambling
-goldfinch
-house finch
-junco
-indigo bunting
-robin
-bulbul
-jay
-magpie
-chickadee
-water ouzel
-kite
-bald eagle
-vulture
-great grey owl
-European fire salamander
-common newt
-eft
-spotted salamander
-axolotl
-bullfrog
-tree frog
-tailed frog
-loggerhead
-leatherback turtle
-mud turtle
-terrapin
-box turtle
-banded gecko
-common iguana
-American chameleon
-whiptail
-agama
-frilled lizard
-alligator lizard
-Gila monster
-green lizard
-African chameleon
-Komodo dragon
-African crocodile
-American alligator
-triceratops
-thunder snake
-ringneck snake
-hognose snake
-green snake
-king snake
-garter snake
-water snake
-vine snake
-night snake
-boa constrictor
-rock python
-Indian cobra
-green mamba
-sea snake
-horned viper
-diamondback
-sidewinder
-trilobite
-harvestman
-scorpion
-black and gold garden spider
-barn spider
-garden spider
-black widow
-tarantula
-wolf spider
-tick
-centipede
-black grouse
-ptarmigan
-ruffed grouse
-prairie chicken
-peacock
-quail
-partridge
-African grey
-macaw
-sulphur-crested cockatoo
-lorikeet
-coucal
-bee eater
-hornbill
-hummingbird
-jacamar
-toucan
-drake
-red-breasted merganser
-goose
-black swan
-tusker
-echidna
-platypus
-wallaby
-koala
-wombat
-jellyfish
-sea anemone
-brain coral
-flatworm
-nematode
-conch
-snail
-slug
-sea slug
-chiton
-chambered nautilus
-Dungeness crab
-rock crab
-fiddler crab
-king crab
-American lobster
-spiny lobster
-crayfish
-hermit crab
-isopod
-white stork
-black stork
-spoonbill
-flamingo
-little blue heron
-American egret
-bittern
-crane
-limpkin
-European gallinule
-American coot
-bustard
-ruddy turnstone
-red-backed sandpiper
-redshank
-dowitcher
-oystercatcher
-pelican
-king penguin
-albatross
-grey whale
-killer whale
-dugong
-sea lion
-Chihuahua
-Japanese spaniel
-Maltese dog
-Pekinese
-Shih-Tzu
-Blenheim spaniel
-papillon
-toy terrier
-Rhodesian ridgeback
-Afghan hound
-basset
-beagle
-bloodhound
-bluetick
-black-and-tan coonhound
-Walker hound
-English foxhound
-redbone
-borzoi
-Irish wolfhound
-Italian greyhound
-whippet
-Ibizan hound
-Norwegian elkhound
-otterhound
-Saluki
-Scottish deerhound
-Weimaraner
-Staffordshire bullterrier
-American Staffordshire terrier
-Bedlington terrier
-Border terrier
-Kerry blue terrier
-Irish terrier
-Norfolk terrier
-Norwich terrier
-Yorkshire terrier
-wire-haired fox terrier
-Lakeland terrier
-Sealyham terrier
-Airedale
-cairn
-Australian terrier
-Dandie Dinmont
-Boston bull
-miniature schnauzer
-giant schnauzer
-standard schnauzer
-Scotch terrier
-Tibetan terrier
-silky terrier
-soft-coated wheaten terrier
-West Highland white terrier
-Lhasa
-flat-coated retriever
-curly-coated retriever
-golden retriever
-Labrador retriever
-Chesapeake Bay retriever
-German short-haired pointer
-vizsla
-English setter
-Irish setter
-Gordon setter
-Brittany spaniel
-clumber
-English springer
-Welsh springer spaniel
-cocker spaniel
-Sussex spaniel
-Irish water spaniel
-kuvasz
-schipperke
-groenendael
-malinois
-briard
-kelpie
-komondor
-Old English sheepdog
-Shetland sheepdog
-collie
-Border collie
-Bouvier des Flandres
-Rottweiler
-German shepherd
-Doberman
-miniature pinscher
-Greater Swiss Mountain dog
-Bernese mountain dog
-Appenzeller
-EntleBucher
-boxer
-bull mastiff
-Tibetan mastiff
-French bulldog
-Great Dane
-Saint Bernard
-Eskimo dog
-malamute
-Siberian husky
-dalmatian
-affenpinscher
-basenji
-pug
-Leonberg
-Newfoundland
-Great Pyrenees
-Samoyed
-Pomeranian
-chow
-keeshond
-Brabancon griffon
-Pembroke
-Cardigan
-toy poodle
-miniature poodle
-standard poodle
-Mexican hairless
-timber wolf
-white wolf
-red wolf
-coyote
-dingo
-dhole
-African hunting dog
-hyena
-red fox
-kit fox
-Arctic fox
-grey fox
-tabby
-tiger cat
-Persian cat
-Siamese cat
-Egyptian cat
-cougar
-lynx
-leopard
-snow leopard
-jaguar
-lion
-tiger
-cheetah
-brown bear
-American black bear
-ice bear
-sloth bear
-mongoose
-meerkat
-tiger beetle
-ladybug
-ground beetle
-long-horned beetle
-leaf beetle
-dung beetle
-rhinoceros beetle
-weevil
-fly
-bee
-ant
-grasshopper
-cricket
-walking stick
-cockroach
-mantis
-cicada
-leafhopper
-lacewing
-dragonfly
-damselfly
-admiral
-ringlet
-monarch
-cabbage butterfly
-sulphur butterfly
-lycaenid
-starfish
-sea urchin
-sea cucumber
-wood rabbit
-hare
-Angora
-hamster
-porcupine
-fox squirrel
-marmot
-beaver
-guinea pig
-sorrel
-zebra
-hog
-wild boar
-warthog
-hippopotamus
-ox
-water buffalo
-bison
-ram
-bighorn
-ibex
-hartebeest
-impala
-gazelle
-Arabian camel
-llama
-weasel
-mink
-polecat
-black-footed ferret
-otter
-skunk
-badger
-armadillo
-three-toed sloth
-orangutan
-gorilla
-chimpanzee
-gibbon
-siamang
-guenon
-patas
-baboon
-macaque
-langur
-colobus
-proboscis monkey
-marmoset
-capuchin
-howler monkey
-titi
-spider monkey
-squirrel monkey
-Madagascar cat
-indri
-Indian elephant
-African elephant
-lesser panda
-giant panda
-barracouta
-eel
-coho
-rock beauty
-anemone fish
-sturgeon
-gar
-lionfish
-puffer
-abacus
-abaya
-academic gown
-accordion
-acoustic guitar
-aircraft carrier
-airliner
-airship
-altar
-ambulance
-amphibian
-analog clock
-apiary
-apron
-ashcan
-assault rifle
-backpack
-bakery
-balance beam
-balloon
-ballpoint
-Band Aid
-banjo
-bannister
-barbell
-barber chair
-barbershop
-barn
-barometer
-barrel
-barrow
-baseball
-basketball
-bassinet
-bassoon
-bathing cap
-bath towel
-bathtub
-beach wagon
-beacon
-beaker
-bearskin
-beer bottle
-beer glass
-bell cote
-bib
-bicycle-built-for-two
-bikini
-binder
-binoculars
-birdhouse
-boathouse
-bobsled
-bolo tie
-bonnet
-bookcase
-bookshop
-bottlecap
-bow
-bow tie
-brass
-brassiere
-breakwater
-breastplate
-broom
-bucket
-buckle
-bulletproof vest
-bullet train
-butcher shop
-cab
-caldron
-candle
-cannon
-canoe
-can opener
-cardigan
-car mirror
-carousel
-carpenter's kit
-carton
-car wheel
-cash machine
-cassette
-cassette player
-castle
-catamaran
-CD player
-cello
-cellular telephone
-chain
-chainlink fence
-chain mail
-chain saw
-chest
-chiffonier
-chime
-china cabinet
-Christmas stocking
-church
-cinema
-cleaver
-cliff dwelling
-cloak
-clog
-cocktail shaker
-coffee mug
-coffeepot
-coil
-combination lock
-computer keyboard
-confectionery
-container ship
-convertible
-corkscrew
-cornet
-cowboy boot
-cowboy hat
-cradle
-crane
-crash helmet
-crate
-crib
-Crock Pot
-croquet ball
-crutch
-cuirass
-dam
-desk
-desktop computer
-dial telephone
-diaper
-digital clock
-digital watch
-dining table
-dishrag
-dishwasher
-disk brake
-dock
-dogsled
-dome
-doormat
-drilling platform
-drum
-drumstick
-dumbbell
-Dutch oven
-electric fan
-electric guitar
-electric locomotive
-entertainment center
-envelope
-espresso maker
-face powder
-feather boa
-file
-fireboat
-fire engine
-fire screen
-flagpole
-flute
-folding chair
-football helmet
-forklift
-fountain
-fountain pen
-four-poster
-freight car
-French horn
-frying pan
-fur coat
-garbage truck
-gasmask
-gas pump
-goblet
-go-kart
-golf ball
-golfcart
-gondola
-gong
-gown
-grand piano
-greenhouse
-grille
-grocery store
-guillotine
-hair slide
-hair spray
-half track
-hammer
-hamper
-hand blower
-hand-held computer
-handkerchief
-hard disc
-harmonica
-harp
-harvester
-hatchet
-holster
-home theater
-honeycomb
-hook
-hoopskirt
-horizontal bar
-horse cart
-hourglass
-iPod
-iron
-jack-o'-lantern
-jean
-jeep
-jersey
-jigsaw puzzle
-jinrikisha
-joystick
-kimono
-knee pad
-knot
-lab coat
-ladle
-lampshade
-laptop
-lawn mower
-lens cap
-letter opener
-library
-lifeboat
-lighter
-limousine
-liner
-lipstick
-Loafer
-lotion
-loudspeaker
-loupe
-lumbermill
-magnetic compass
-mailbag
-mailbox
-maillot
-maillot
-manhole cover
-maraca
-marimba
-mask
-matchstick
-maypole
-maze
-measuring cup
-medicine chest
-megalith
-microphone
-microwave
-military uniform
-milk can
-minibus
-miniskirt
-minivan
-missile
-mitten
-mixing bowl
-mobile home
-Model T
-modem
-monastery
-monitor
-moped
-mortar
-mortarboard
-mosque
-mosquito net
-motor scooter
-mountain bike
-mountain tent
-mouse
-mousetrap
-moving van
-muzzle
-nail
-neck brace
-necklace
-nipple
-notebook
-obelisk
-oboe
-ocarina
-odometer
-oil filter
-organ
-oscilloscope
-overskirt
-oxcart
-oxygen mask
-packet
-paddle
-paddlewheel
-padlock
-paintbrush
-pajama
-palace
-panpipe
-paper towel
-parachute
-parallel bars
-park bench
-parking meter
-passenger car
-patio
-pay-phone
-pedestal
-pencil box
-pencil sharpener
-perfume
-Petri dish
-photocopier
-pick
-pickelhaube
-picket fence
-pickup
-pier
-piggy bank
-pill bottle
-pillow
-ping-pong ball
-pinwheel
-pirate
-pitcher
-plane
-planetarium
-plastic bag
-plate rack
-plow
-plunger
-Polaroid camera
-pole
-police van
-poncho
-pool table
-pop bottle
-pot
-potter's wheel
-power drill
-prayer rug
-printer
-prison
-projectile
-projector
-puck
-punching bag
-purse
-quill
-quilt
-racer
-racket
-radiator
-radio
-radio telescope
-rain barrel
-recreational vehicle
-reel
-reflex camera
-refrigerator
-remote control
-restaurant
-revolver
-rifle
-rocking chair
-rotisserie
-rubber eraser
-rugby ball
-rule
-running shoe
-safe
-safety pin
-saltshaker
-sandal
-sarong
-sax
-scabbard
-scale
-school bus
-schooner
-scoreboard
-screen
-screw
-screwdriver
-seat belt
-sewing machine
-shield
-shoe shop
-shoji
-shopping basket
-shopping cart
-shovel
-shower cap
-shower curtain
-ski
-ski mask
-sleeping bag
-slide rule
-sliding door
-slot
-snorkel
-snowmobile
-snowplow
-soap dispenser
-soccer ball
-sock
-solar dish
-sombrero
-soup bowl
-space bar
-space heater
-space shuttle
-spatula
-speedboat
-spider web
-spindle
-sports car
-spotlight
-stage
-steam locomotive
-steel arch bridge
-steel drum
-stethoscope
-stole
-stone wall
-stopwatch
-stove
-strainer
-streetcar
-stretcher
-studio couch
-stupa
-submarine
-suit
-sundial
-sunglass
-sunglasses
-sunscreen
-suspension bridge
-swab
-sweatshirt
-swimming trunks
-swing
-switch
-syringe
-table lamp
-tank
-tape player
-teapot
-teddy
-television
-tennis ball
-thatch
-theater curtain
-thimble
-thresher
-throne
-tile roof
-toaster
-tobacco shop
-toilet seat
-torch
-totem pole
-tow truck
-toyshop
-tractor
-trailer truck
-tray
-trench coat
-tricycle
-trimaran
-tripod
-triumphal arch
-trolleybus
-trombone
-tub
-turnstile
-typewriter keyboard
-umbrella
-unicycle
-upright
-vacuum
-vase
-vault
-velvet
-vending machine
-vestment
-viaduct
-violin
-volleyball
-waffle iron
-wall clock
-wallet
-wardrobe
-warplane
-washbasin
-washer
-water bottle
-water jug
-water tower
-whiskey jug
-whistle
-wig
-window screen
-window shade
-Windsor tie
-wine bottle
-wing
-wok
-wooden spoon
-wool
-worm fence
-wreck
-yawl
-yurt
-web site
-comic book
-crossword puzzle
-street sign
-traffic light
-book jacket
-menu
-plate
-guacamole
-consomme
-hot pot
-trifle
-ice cream
-ice lolly
-French loaf
-bagel
-pretzel
-cheeseburger
-hotdog
-mashed potato
-head cabbage
-broccoli
-cauliflower
-zucchini
-spaghetti squash
-acorn squash
-butternut squash
-cucumber
-artichoke
-bell pepper
-cardoon
-mushroom
-Granny Smith
-strawberry
-orange
-lemon
-fig
-pineapple
-banana
-jackfruit
-custard apple
-pomegranate
-hay
-carbonara
-chocolate sauce
-dough
-meat loaf
-pizza
-potpie
-burrito
-red wine
-espresso
-cup
-eggnog
-alp
-bubble
-cliff
-coral reef
-geyser
-lakeside
-promontory
-sandbar
-seashore
-valley
-volcano
-ballplayer
-groom
-scuba diver
-rapeseed
-daisy
-yellow lady's slipper
-corn
-acorn
-hip
-buckeye
-coral fungus
-agaric
-gyromitra
-stinkhorn
-earthstar
-hen-of-the-woods
-bolete
-ear
-toilet tissue
diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java b/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
index 9b9fdffab5..300786c3ca 100644
--- a/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
+++ b/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java
@@ -299,7 +299,7 @@ public class Camera2BasicFragment extends Fragment
       // create either a new ImageClassifierQuantizedMobileNet or an ImageClassifierFloatInception
       classifier = new ImageClassifierQuantizedMobileNet(getActivity());
     } catch (IOException e) {
-      Log.e(TAG, "Failed to initialize an image classifier.");
+      Log.e(TAG, "Failed to initialize an image classifier.", e);
     }
     startBackgroundThread();
   }
@@ -433,7 +433,7 @@ public class Camera2BasicFragment extends Fragment
         return;
       }
     } catch (CameraAccessException e) {
-      e.printStackTrace();
+      Log.e(TAG, "Failed to access Camera", e);
     } catch (NullPointerException e) {
       // Currently an NPE is thrown when the Camera2API is used but not supported on the
       // device this code runs.
@@ -478,7 +478,7 @@ public class Camera2BasicFragment extends Fragment
       }
       manager.openCamera(cameraId, stateCallback, backgroundHandler);
     } catch (CameraAccessException e) {
-      e.printStackTrace();
+      Log.e(TAG, "Failed to open Camera", e);
     } catch (InterruptedException e) {
       throw new RuntimeException("Interrupted while trying to lock camera opening.", e);
     }
@@ -545,7 +545,7 @@ public class Camera2BasicFragment extends Fragment
         runClassifier = false;
       }
     } catch (InterruptedException e) {
-      e.printStackTrace();
+      Log.e(TAG, "Interrupted when stopping background thread", e);
     }
   }
 
@@ -604,7 +604,7 @@ public class Camera2BasicFragment extends Fragment
                 captureSession.setRepeatingRequest(
                     previewRequest, captureCallback, backgroundHandler);
               } catch (CameraAccessException e) {
-                e.printStackTrace();
+                Log.e(TAG, "Failed to set up config to capture Camera", e);
               }
             }
 
@@ -615,7 +615,7 @@ public class Camera2BasicFragment extends Fragment
           },
           null);
     } catch (CameraAccessException e) {
-      e.printStackTrace();
+      Log.e(TAG, "Failed to preview Camera", e);
     }
   }
 
diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java b/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
index c533de7927..e164ac7554 100644
--- a/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
+++ b/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/ImageClassifierQuantizedMobileNet.java
@@ -16,7 +16,6 @@ limitations under the License.
 package com.example.android.tflitecamerademo;
 
 import android.app.Activity;
-
 import java.io.IOException;
 
 /**
@@ -49,7 +48,7 @@ public class ImageClassifierQuantizedMobileNet extends ImageClassifier {
 
   @Override
   protected String getLabelPath() {
-    return "labels.txt";
+    return "labels_mobilenet_quant_v1_224.txt";
   }
 
   @Override
-- 
GitLab


From bb84d4deb1267f24e8105f4b1b40c264674bc134 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Thu, 15 Mar 2018 14:38:25 -0700
Subject: [PATCH 079/960] [Checkpointable] Make EagerIterator checkpointable.

Use object-based save/restore to make dataset/iterator checkpointable in eager mode, this could potentially be extended to graph mode as well.

PiperOrigin-RevId: 189247720
---
 tensorflow/contrib/eager/python/BUILD         |  1 +
 tensorflow/contrib/eager/python/datasets.py   | 31 +++++++++-
 .../contrib/eager/python/datasets_test.py     | 58 +++++++++++++++++++
 3 files changed, 89 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 384ef7f963..eb810e06dd 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -70,6 +70,7 @@ cuda_py_test(
     srcs = ["datasets_test.py"],
     additional_deps = [
         ":datasets",
+        ":checkpointable_utils",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/python:dtypes",
diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index 332bada57b..a4c3283dac 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -31,6 +31,8 @@ from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_dataset_ops
 from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.training import checkpointable
+from tensorflow.python.training.saver import BaseSaverBuilder
 
 _uid_counter = 0
 _uid_lock = threading.Lock()
@@ -44,7 +46,7 @@ def _generate_shared_name(prefix):
   return "{}{}".format(prefix, uid)
 
 
-class Iterator(iterator_ops.EagerIterator):
+class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
   """An iterator producing tf.Tensor objects from a tf.data.Dataset.
 
   NOTE: Unlike the iterator created by the
@@ -116,3 +118,30 @@ class Iterator(iterator_ops.EagerIterator):
           self._output_shapes, self._output_classes)
     else:
       return super(Iterator, self)._next_internal()
+
+  # TODO(shivaniagrawal): Expose checkpointable stateful objects from dataset
+  # attributes(potential).
+
+  class _Saveable(BaseSaverBuilder.SaveableObject):
+    """SaveableObject for saving/restoring iterator state."""
+
+    def __init__(self, iterator_resource, name):
+      serialized_iterator = gen_dataset_ops.serialize_iterator(
+          iterator_resource)
+      specs = [
+          BaseSaverBuilder.SaveSpec(serialized_iterator, "", name + "_STATE")
+      ]
+      # pylint: disable=protected-access
+      super(Iterator._Saveable, self).__init__(iterator_resource, specs, name)
+
+    def restore(self, restored_tensors, restored_shapes):
+      with ops.colocate_with(self.op):
+        return gen_dataset_ops.deserialize_iterator(self.op,
+                                                    restored_tensors[0])
+
+  def _gather_saveables_for_checkpoint(self):
+
+    def _saveable_factory(name):
+      return self._Saveable(self._resource, name)
+
+    return {"ITERATOR": _saveable_factory}
diff --git a/tensorflow/contrib/eager/python/datasets_test.py b/tensorflow/contrib/eager/python/datasets_test.py
index 4afadd88f5..c658505de4 100644
--- a/tensorflow/contrib/eager/python/datasets_test.py
+++ b/tensorflow/contrib/eager/python/datasets_test.py
@@ -16,6 +16,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+
 import threading
 import time
 
@@ -24,6 +26,7 @@ import numpy as np
 from tensorflow.contrib import lookup
 from tensorflow.contrib.data.python.ops import threadpool
 from tensorflow.contrib.data.python.ops import unique
+from tensorflow.contrib.eager.python import checkpointable_utils
 from tensorflow.contrib.eager.python import datasets
 from tensorflow.python.data import Dataset
 from tensorflow.python.eager import test
@@ -221,6 +224,61 @@ class IteratorTest(test.TestCase):
       # perform work.
       self.assertLessEqual(len(thread_ids), num_threads)
 
+  def testSaveRestore(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
+    dataset = Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
+    dataset = dataset.map(math_ops.square).batch(2)
+    iterator = datasets.Iterator(dataset)
+    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
+    self.assertAllEqual([1, 4], iterator.get_next().numpy())
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertAllEqual([9, 16], iterator.get_next().numpy())
+    self.assertAllEqual([25, 36], iterator.get_next().numpy())
+    checkpoint.restore(save_path)
+    self.assertAllEqual([9, 16], iterator.get_next().numpy())
+    self.assertAllEqual([25, 36], iterator.get_next().numpy())
+
+  def testSaveRestoreMultipleIterator(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
+    dataset = Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11])
+    dataset = dataset.map(math_ops.square).batch(2)
+    iterator_1 = datasets.Iterator(dataset)
+    iterator_2 = datasets.Iterator(dataset)
+    dataset_2 = Dataset.range(10)
+    iterator_3 = datasets.Iterator(dataset_2)
+
+    checkpoint = checkpointable_utils.Checkpoint(
+        iterator_1=iterator_1, iterator_2=iterator_2, iterator_3=iterator_3)
+    self.assertAllEqual([1, 4], iterator_1.get_next().numpy())
+    self.assertEqual(0, iterator_3.get_next().numpy())
+    self.assertEqual(1, iterator_3.get_next().numpy())
+    self.assertEqual(2, iterator_3.get_next().numpy())
+
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertAllEqual([1, 4], iterator_2.get_next().numpy())
+    self.assertAllEqual([9, 16], iterator_2.get_next().numpy())
+    self.assertEqual(3, iterator_3.get_next().numpy())
+    checkpoint.restore(save_path)
+    self.assertAllEqual([9, 16], iterator_1.get_next().numpy())
+    self.assertAllEqual([1, 4], iterator_2.get_next().numpy())
+    self.assertEqual(3, iterator_3.get_next().numpy())
+
+  def testRestoreExhaustedIterator(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
+    dataset = Dataset.range(3)
+    iterator = datasets.Iterator(dataset)
+
+    checkpoint = checkpointable_utils.Checkpoint(iterator=iterator)
+    self.assertEqual(0, iterator.get_next().numpy())
+    self.assertEqual(1, iterator.get_next().numpy())
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.assertEqual(2, iterator.get_next().numpy())
+    checkpoint.restore(save_path)
+    self.assertEqual(2, iterator.get_next().numpy())
+
 
 class DatasetConstructorBenchmark(test.Benchmark):
 
-- 
GitLab


From a325b0ae208b3295dca574f93c214f7c924cea11 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 14:40:12 -0700
Subject: [PATCH 080/960] Adds a GPU kernel registration for
 PlaceholderWithDefault, so we can avoid the issue of using it with a
 registered GPU device without soft placement.

PiperOrigin-RevId: 189248024
---
 tensorflow/core/kernels/identity_op.cc        | 34 ++++++++++++-------
 .../python/kernel_tests/constant_op_test.py   |  9 ++---
 2 files changed, 27 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/kernels/identity_op.cc b/tensorflow/core/kernels/identity_op.cc
index a18a72c66d..b5603fecd8 100644
--- a/tensorflow/core/kernels/identity_op.cc
+++ b/tensorflow/core/kernels/identity_op.cc
@@ -101,6 +101,10 @@ REGISTER_SYCL_HOST_KERNEL(bool);
   REGISTER_KERNEL_BUILDER(Name("DebugGradientIdentity")                     \
                               .Device(DEVICE_GPU)                           \
                               .TypeConstraint<type>("T"),                   \
+                          IdentityOp);                                      \
+  REGISTER_KERNEL_BUILDER(Name("PlaceholderWithDefault")                    \
+                              .Device(DEVICE_GPU)                           \
+                              .TypeConstraint<type>("dtype"),               \
                           IdentityOp)
 
 TF_CALL_NUMBER_TYPES_NO_INT32(REGISTER_GPU_KERNEL);
@@ -112,18 +116,24 @@ REGISTER_GPU_KERNEL(Variant);
 // A special GPU kernel for int32 and bool.
 // TODO(b/25387198): Also enable int32 in device memory. This kernel
 // registration requires all int32 inputs and outputs to be in host memory.
-#define REGISTER_GPU_HOST_KERNEL(type)                    \
-  REGISTER_KERNEL_BUILDER(Name("Identity")                \
-                              .Device(DEVICE_GPU)         \
-                              .HostMemory("input")        \
-                              .HostMemory("output")       \
-                              .TypeConstraint<type>("T"), \
-                          IdentityOp);                    \
-  REGISTER_KERNEL_BUILDER(Name("RefIdentity")             \
-                              .Device(DEVICE_GPU)         \
-                              .HostMemory("input")        \
-                              .HostMemory("output")       \
-                              .TypeConstraint<type>("T"), \
+#define REGISTER_GPU_HOST_KERNEL(type)                        \
+  REGISTER_KERNEL_BUILDER(Name("Identity")                    \
+                              .Device(DEVICE_GPU)             \
+                              .HostMemory("input")            \
+                              .HostMemory("output")           \
+                              .TypeConstraint<type>("T"),     \
+                          IdentityOp);                        \
+  REGISTER_KERNEL_BUILDER(Name("RefIdentity")                 \
+                              .Device(DEVICE_GPU)             \
+                              .HostMemory("input")            \
+                              .HostMemory("output")           \
+                              .TypeConstraint<type>("T"),     \
+                          IdentityOp);                        \
+  REGISTER_KERNEL_BUILDER(Name("PlaceholderWithDefault")      \
+                              .Device(DEVICE_GPU)             \
+                              .HostMemory("input")            \
+                              .HostMemory("output")           \
+                              .TypeConstraint<type>("dtype"), \
                           IdentityOp)
 
 REGISTER_GPU_HOST_KERNEL(int32);
diff --git a/tensorflow/python/kernel_tests/constant_op_test.py b/tensorflow/python/kernel_tests/constant_op_test.py
index ffbdb0e61a..18796f7095 100644
--- a/tensorflow/python/kernel_tests/constant_op_test.py
+++ b/tensorflow/python/kernel_tests/constant_op_test.py
@@ -881,7 +881,7 @@ versions {
 class PlaceholderWithDefaultTest(test.TestCase):
 
   def testFullShape(self):
-    with self.test_session():
+    with self.test_session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([[2, 2], [2, 2]], shape=[2, 2])
       a = array_ops.identity(p)
       self.assertAllEqual([[2, 2], [2, 2]], a.eval())
@@ -892,7 +892,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
         a.eval(feed_dict={p: [[6, 6, 6], [6, 6, 6]]})
 
   def testPartialShape(self):
-    with self.test_session():
+    with self.test_session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([1, 2, 3], shape=[None])
       a = array_ops.identity(p)
       self.assertAllEqual([1, 2, 3], a.eval())
@@ -902,7 +902,7 @@ class PlaceholderWithDefaultTest(test.TestCase):
         a.eval(feed_dict={p: [[2, 2], [2, 2]]})
 
   def testNoShape(self):
-    with self.test_session():
+    with self.test_session(force_gpu=test_util.is_gpu_available()):
       p = array_ops.placeholder_with_default([17], shape=None)
       a = array_ops.identity(p)
       self.assertAllEqual([17], a.eval())
@@ -911,11 +911,12 @@ class PlaceholderWithDefaultTest(test.TestCase):
           [[3, 3], [3, 3]], a.eval(feed_dict={p: [[3, 3], [3, 3]]}))
 
   def testGradient(self):
-    with self.test_session():
+    with self.test_session(force_gpu=test_util.is_gpu_available()):
       x = array_ops.placeholder(dtypes_lib.float32, [5, 7])
       y = array_ops.placeholder_with_default(x, None)
       err = gradient_checker.compute_gradient_error(x, [5, 7], y, [5, 7])
       self.assertLess(err, 1e-3)
 
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From f9dc34df6d56d2bcb67b563ade81a3f12bbcacd2 Mon Sep 17 00:00:00 2001
From: Pete Warden <petewarden@google.com>
Date: Thu, 15 Mar 2018 14:45:34 -0700
Subject: [PATCH 081/960] Check for very large chunk sizes in WAV decoding
 Change how chunk sizes larger than 2GB are handled, since they're stored as
 unsigned int32s, so there are lots of ways for conversions to confuse the
 decoding logic. The new behavior is to fail with an error, since such large
 WAV files are not common, and are unsupported by many readers.

PiperOrigin-RevId: 189248857
---
 tensorflow/core/lib/wav/wav_io.cc      |  70 +++---
 tensorflow/core/lib/wav/wav_io.h       |  33 +++
 tensorflow/core/lib/wav/wav_io_test.cc | 319 +++++++++++++++++++++++++
 3 files changed, 390 insertions(+), 32 deletions(-)

diff --git a/tensorflow/core/lib/wav/wav_io.cc b/tensorflow/core/lib/wav/wav_io.cc
index 77d3c88998..51b9c6cd82 100644
--- a/tensorflow/core/lib/wav/wav_io.cc
+++ b/tensorflow/core/lib/wav/wav_io.cc
@@ -81,13 +81,38 @@ inline float Int16SampleToFloat(int16 data) {
   return data * kMultiplier;
 }
 
+}  // namespace
+
+// Handles moving the data index forward, validating the arguments, and avoiding
+// overflow or underflow.
+Status IncrementOffset(int old_offset, size_t increment, size_t max_size,
+                       int* new_offset) {
+  if (old_offset < 0) {
+    return errors::InvalidArgument("Negative offsets are not allowed: ",
+                                   old_offset);
+  }
+  if (old_offset > max_size) {
+    return errors::InvalidArgument("Initial offset is outside data range: ",
+                                   old_offset);
+  }
+  *new_offset = old_offset + increment;
+  if (*new_offset > max_size) {
+    return errors::InvalidArgument("Data too short when trying to read string");
+  }
+  // See above for the check that the input offset is positive. If it's negative
+  // here then it means that there's been an overflow in the arithmetic.
+  if (*new_offset < 0) {
+    return errors::InvalidArgument("Offset too large, overflowed: ",
+                                   *new_offset);
+  }
+  return Status::OK();
+}
+
 Status ExpectText(const string& data, const string& expected_text,
                   int* offset) {
-  const int new_offset = *offset + expected_text.size();
-  if (new_offset > data.size()) {
-    return errors::InvalidArgument("Data too short when trying to read ",
-                                   expected_text);
-  }
+  int new_offset;
+  TF_RETURN_IF_ERROR(
+      IncrementOffset(*offset, expected_text.size(), data.size(), &new_offset));
   const string found_text(data.begin() + *offset, data.begin() + new_offset);
   if (found_text != expected_text) {
     return errors::InvalidArgument("Header mismatch: Expected ", expected_text,
@@ -97,40 +122,16 @@ Status ExpectText(const string& data, const string& expected_text,
   return Status::OK();
 }
 
-template <class T>
-Status ReadValue(const string& data, T* value, int* offset) {
-  const int new_offset = *offset + sizeof(T);
-  if (new_offset > data.size()) {
-    return errors::InvalidArgument("Data too short when trying to read value");
-  }
-  if (port::kLittleEndian) {
-    memcpy(value, data.data() + *offset, sizeof(T));
-  } else {
-    *value = 0;
-    const uint8* data_buf =
-        reinterpret_cast<const uint8*>(data.data() + *offset);
-    int shift = 0;
-    for (int i = 0; i < sizeof(T); ++i, shift += 8) {
-      *value = *value | (data_buf[i] << shift);
-    }
-  }
-  *offset = new_offset;
-  return Status::OK();
-}
-
 Status ReadString(const string& data, int expected_length, string* value,
                   int* offset) {
-  const int new_offset = *offset + expected_length;
-  if (new_offset > data.size()) {
-    return errors::InvalidArgument("Data too short when trying to read string");
-  }
+  int new_offset;
+  TF_RETURN_IF_ERROR(
+      IncrementOffset(*offset, expected_length, data.size(), &new_offset));
   *value = string(data.begin() + *offset, data.begin() + new_offset);
   *offset = new_offset;
   return Status::OK();
 }
 
-}  // namespace
-
 Status EncodeAudioAsS16LEWav(const float* audio, size_t sample_rate,
                              size_t num_channels, size_t num_frames,
                              string* wav_string) {
@@ -272,6 +273,11 @@ Status DecodeLin16WaveAsFloatVector(const string& wav_string,
     TF_RETURN_IF_ERROR(ReadString(wav_string, 4, &chunk_id, &offset));
     uint32 chunk_size;
     TF_RETURN_IF_ERROR(ReadValue<uint32>(wav_string, &chunk_size, &offset));
+    if (chunk_size > std::numeric_limits<int32>::max()) {
+      return errors::InvalidArgument(
+          "WAV data chunk '", chunk_id, "' is too large: ", chunk_size,
+          " bytes, but the limit is ", std::numeric_limits<int32>::max());
+    }
     if (chunk_id == kDataChunkId) {
       if (was_data_found) {
         return errors::InvalidArgument("More than one data chunk found in WAV");
diff --git a/tensorflow/core/lib/wav/wav_io.h b/tensorflow/core/lib/wav/wav_io.h
index adca0ee303..f004524177 100644
--- a/tensorflow/core/lib/wav/wav_io.h
+++ b/tensorflow/core/lib/wav/wav_io.h
@@ -21,6 +21,9 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/lib/core/casts.h"
+#include "tensorflow/core/lib/core/coding.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -55,6 +58,36 @@ Status DecodeLin16WaveAsFloatVector(const string& wav_string,
                                     uint32* sample_count, uint16* channel_count,
                                     uint32* sample_rate);
 
+// Everything below here is only exposed publicly for testing purposes.
+
+// Handles moving the data index forward, validating the arguments, and avoiding
+// overflow or underflow.
+Status IncrementOffset(int old_offset, size_t increment, size_t max_size,
+                       int* new_offset);
+
+// This function is only exposed in the header for testing purposes, as a
+// template that needs to be instantiated. Reads a typed numeric value from a
+// stream of data.
+template <class T>
+Status ReadValue(const string& data, T* value, int* offset) {
+  int new_offset;
+  TF_RETURN_IF_ERROR(
+      IncrementOffset(*offset, sizeof(T), data.size(), &new_offset));
+  if (port::kLittleEndian) {
+    memcpy(value, data.data() + *offset, sizeof(T));
+  } else {
+    *value = 0;
+    const uint8* data_buf =
+        reinterpret_cast<const uint8*>(data.data() + *offset);
+    int shift = 0;
+    for (int i = 0; i < sizeof(T); ++i, shift += 8) {
+      *value = *value | (data_buf[i] << shift);
+    }
+  }
+  *offset = new_offset;
+  return Status::OK();
+}
+
 }  // namespace wav
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/lib/wav/wav_io_test.cc b/tensorflow/core/lib/wav/wav_io_test.cc
index 40ddd94abe..d8a83fc464 100644
--- a/tensorflow/core/lib/wav/wav_io_test.cc
+++ b/tensorflow/core/lib/wav/wav_io_test.cc
@@ -25,6 +25,12 @@ limitations under the License.
 namespace tensorflow {
 namespace wav {
 
+// These are defined in wav_io.cc, and the signatures are here so we don't have
+// to expose them in the public header.
+Status ExpectText(const string& data, const string& expected_text, int* offset);
+Status ReadString(const string& data, int expected_length, string* value,
+                  int* offset);
+
 TEST(WavIO, BadArguments) {
   float audio[] = {0.0f, 0.1f, 0.2f, 0.3f, 0.4f, 0.5f};
   string result;
@@ -155,5 +161,318 @@ TEST(WavIO, BasicStereo) {
   EXPECT_EQ(expected, result);
 }
 
+// Test how chunk sizes larger than 2GB are handled, since they're stored as
+// unsigned int32s, so there are lots of ways for conversions to confuse the
+// decoding logic. The expected behavior is to fail with an error, since such
+// large WAV files are not common, and are unsupported by many readers.
+// See b/72655902.
+TEST(WavIO, ChunkSizeOverflow) {
+  std::vector<uint8> wav_data = {
+      'R', 'I', 'F', 'F',      // ChunkID
+      60, 0, 0, 0,             // ChunkSize: 36 + SubChunk2Size
+      'W', 'A', 'V', 'E',      // Format
+      'f', 'm', 't', ' ',      // Subchunk1ID
+      16, 0, 0, 0,             // Subchunk1Size
+      1, 0,                    // AudioFormat: 1=PCM
+      1, 0,                    // NumChannels
+      0x44, 0xac, 0, 0,        // SampleRate: 44100
+      0x88, 0x58, 0x1, 0,      // BytesPerSecond: SampleRate * NumChannels *
+                               //                 BitsPerSample/8
+      2, 0,                    // BytesPerSample: NumChannels * BitsPerSample/8
+      16, 0,                   // BitsPerSample
+      'd', 'a', 't', 'a',      // Subchunk2ID
+      8, 0, 0, 0,              // Subchunk2Size: NumSamples * NumChannels *
+                               //                BitsPerSample/8
+      0, 0,                    // Sample 1: 0
+      0xff, 0x7f,              // Sample 2: 32767 (saturated)
+      0, 0,                    // Sample 3: 0
+      0x00, 0x80,              // Sample 4: -32768 (saturated)
+      'f', 'o', 'o', 'o',      // Subchunk2ID
+      0xff, 0xff, 0xff, 0xf8,  // Chunk size that could cause an infinite loop.
+      0, 0,                    // Sample 1: 0
+      0xff, 0x7f,              // Sample 2: 32767 (saturated)
+      0, 0,                    // Sample 3: 0
+      0x00, 0x80,              // Sample 4: -32768 (saturated)
+  };
+  string wav_data_string(wav_data.begin(), wav_data.end());
+  std::vector<float> decoded_audio;
+  uint32 decoded_sample_count;
+  uint16 decoded_channel_count;
+  uint32 decoded_sample_rate;
+  Status decode_status = DecodeLin16WaveAsFloatVector(
+      wav_data_string, &decoded_audio, &decoded_sample_count,
+      &decoded_channel_count, &decoded_sample_rate);
+  EXPECT_FALSE(decode_status.ok());
+  EXPECT_TRUE(StringPiece(decode_status.error_message()).contains("too large"))
+      << decode_status.error_message();
+}
+
+TEST(WavIO, IncrementOffset) {
+  int new_offset = -1;
+  TF_EXPECT_OK(IncrementOffset(0, 10, 20, &new_offset));
+  EXPECT_EQ(10, new_offset);
+
+  new_offset = -1;
+  TF_EXPECT_OK(IncrementOffset(10, 4, 20, &new_offset));
+  EXPECT_EQ(14, new_offset);
+
+  new_offset = -1;
+  TF_EXPECT_OK(IncrementOffset(99, 1, 100, &new_offset));
+  EXPECT_EQ(100, new_offset);
+
+  new_offset = -1;
+  EXPECT_FALSE(IncrementOffset(-1, 1, 100, &new_offset).ok());
+
+  new_offset = -1;
+  EXPECT_FALSE(IncrementOffset(0, -1, 100, &new_offset).ok());
+
+  new_offset = -1;
+  EXPECT_FALSE(IncrementOffset(std::numeric_limits<int>::max(), 1,
+                               std::numeric_limits<int>::max(), &new_offset)
+                   .ok());
+
+  new_offset = -1;
+  EXPECT_FALSE(IncrementOffset(101, 1, 100, &new_offset).ok());
+}
+
+TEST(WavIO, ExpectText) {
+  std::vector<uint8> test_data = {
+      'E', 'x', 'p', 'e', 'c', 't', 'e', 'd',
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  TF_EXPECT_OK(ExpectText(test_string, "Expected", &offset));
+  EXPECT_EQ(8, offset);
+
+  offset = 0;
+  Status expect_status = ExpectText(test_string, "Unexpected", &offset);
+  EXPECT_FALSE(expect_status.ok());
+
+  offset = 0;
+  TF_EXPECT_OK(ExpectText(test_string, "Exp", &offset));
+  EXPECT_EQ(3, offset);
+  TF_EXPECT_OK(ExpectText(test_string, "ected", &offset));
+  EXPECT_EQ(8, offset);
+  expect_status = ExpectText(test_string, "foo", &offset);
+  EXPECT_FALSE(expect_status.ok());
+}
+
+TEST(WavIO, ReadString) {
+  std::vector<uint8> test_data = {
+      'E', 'x', 'p', 'e', 'c', 't', 'e', 'd',
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  string read_value;
+  TF_EXPECT_OK(ReadString(test_string, 2, &read_value, &offset));
+  EXPECT_EQ("Ex", read_value);
+  EXPECT_EQ(2, offset);
+
+  TF_EXPECT_OK(ReadString(test_string, 6, &read_value, &offset));
+  EXPECT_EQ("pected", read_value);
+  EXPECT_EQ(8, offset);
+
+  Status read_status = ReadString(test_string, 3, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueInt8) {
+  std::vector<uint8> test_data = {0x00, 0x05, 0xff, 0x80};
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  int8 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(1, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(5, read_value);
+  EXPECT_EQ(2, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(-1, read_value);
+  EXPECT_EQ(3, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(-128, read_value);
+  EXPECT_EQ(4, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueUInt8) {
+  std::vector<uint8> test_data = {0x00, 0x05, 0xff, 0x80};
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  uint8 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(1, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(5, read_value);
+  EXPECT_EQ(2, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(255, read_value);
+  EXPECT_EQ(3, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(128, read_value);
+  EXPECT_EQ(4, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueInt16) {
+  std::vector<uint8> test_data = {
+      0x00, 0x00,  // 0
+      0xff, 0x00,  // 255
+      0x00, 0x01,  // 256
+      0xff, 0xff,  // -1
+      0x00, 0x80,  // -32768
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  int16 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(2, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(255, read_value);
+  EXPECT_EQ(4, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(256, read_value);
+  EXPECT_EQ(6, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(-1, read_value);
+  EXPECT_EQ(8, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(-32768, read_value);
+  EXPECT_EQ(10, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueUInt16) {
+  std::vector<uint8> test_data = {
+      0x00, 0x00,  // 0
+      0xff, 0x00,  // 255
+      0x00, 0x01,  // 256
+      0xff, 0xff,  // 65535
+      0x00, 0x80,  // 32768
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  uint16 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(2, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(255, read_value);
+  EXPECT_EQ(4, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(256, read_value);
+  EXPECT_EQ(6, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(65535, read_value);
+  EXPECT_EQ(8, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(32768, read_value);
+  EXPECT_EQ(10, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueInt32) {
+  std::vector<uint8> test_data = {
+      0x00, 0x00, 0x00, 0x00,  // 0
+      0xff, 0x00, 0x00, 0x00,  // 255
+      0x00, 0xff, 0x00, 0x00,  // 65280
+      0x00, 0x00, 0xff, 0x00,  // 16,711,680
+      0xff, 0xff, 0xff, 0xff,  // -1
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  int32 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(4, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(255, read_value);
+  EXPECT_EQ(8, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(65280, read_value);
+  EXPECT_EQ(12, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(16711680, read_value);
+  EXPECT_EQ(16, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(-1, read_value);
+  EXPECT_EQ(20, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
+TEST(WavIO, ReadValueUInt32) {
+  std::vector<uint8> test_data = {
+      0x00, 0x00, 0x00, 0x00,  // 0
+      0xff, 0x00, 0x00, 0x00,  // 255
+      0x00, 0xff, 0x00, 0x00,  // 65280
+      0x00, 0x00, 0xff, 0x00,  // 16,711,680
+      0xff, 0xff, 0xff, 0xff,  // 4,294,967,295
+  };
+  string test_string(test_data.begin(), test_data.end());
+
+  int offset = 0;
+  uint32 read_value;
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(0, read_value);
+  EXPECT_EQ(4, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(255, read_value);
+  EXPECT_EQ(8, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(65280, read_value);
+  EXPECT_EQ(12, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(16711680, read_value);
+  EXPECT_EQ(16, offset);
+
+  TF_EXPECT_OK(ReadValue(test_string, &read_value, &offset));
+  EXPECT_EQ(4294967295, read_value);
+  EXPECT_EQ(20, offset);
+
+  Status read_status = ReadValue(test_string, &read_value, &offset);
+  EXPECT_FALSE(read_status.ok());
+}
+
 }  // namespace wav
 }  // namespace tensorflow
-- 
GitLab


From 67fd87c5812f227193145d8436e4bdf9b315dfb7 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Mar 2018 14:53:52 -0700
Subject: [PATCH 082/960] Change ParseAndVerifyModule to take a StringPiece;
 NFC

PiperOrigin-RevId: 189250126
---
 .../cpu/parallel_task_assignment_test.cc      |  4 ++--
 .../xla/service/while_loop_simplifier_test.cc | 22 +++++++++----------
 .../xla/tests/hlo_verified_test_base.cc       |  3 ++-
 .../xla/tests/hlo_verified_test_base.h        |  2 +-
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
index 79b00135c6..90191221eb 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
@@ -44,7 +44,7 @@ TEST_F(ParallelTaskAssignmentTest, DotOperationNotParallelized) {
     }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
                                             max_parallelism_, shape_size_func_)
                                             .Run(&module()));
@@ -73,7 +73,7 @@ TEST_F(ParallelTaskAssignmentTest,
     }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
                                             max_parallelism_, shape_size_func_)
                                             .Run(&module()));
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
index f1fea6d763..619e87caa5 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier_test.cc
@@ -68,7 +68,7 @@ void WhileLoopSimplifierTest::MakeModuleWithSimpleLoop(int num_iters) {
       hlo_string_template, "{{LOOP_BOUND}}",
       tensorflow::strings::StrCat(42 + num_iters),
       /*replace_all=*/true);
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
 }
 
 void WhileLoopSimplifierTest::MakeModuleWithSimpleLoopTupleElementLoopBound(
@@ -107,7 +107,7 @@ void WhileLoopSimplifierTest::MakeModuleWithSimpleLoopTupleElementLoopBound(
       hlo_string_template, "{{LOOP_BOUND}}",
       tensorflow::strings::StrCat(42 + num_iters),
       /*replace_all=*/true);
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
 }
 
 TEST_F(WhileLoopSimplifierTest, LoopWithZeroIterationSimiplified) {
@@ -235,7 +235,7 @@ TEST_F(WhileLoopSimplifierTest, NonTupleShapedLoopNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -267,7 +267,7 @@ TEST_F(WhileLoopSimplifierTest, LoopSwappingTupleElementsNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -296,7 +296,7 @@ TEST_F(WhileLoopSimplifierTest,
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -319,7 +319,7 @@ TEST_F(WhileLoopSimplifierTest, LoopWithEmptyTupleNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -347,7 +347,7 @@ TEST_F(WhileLoopSimplifierTest, LoopWithElemUsedTwiceNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -389,7 +389,7 @@ TEST_F(WhileLoopSimplifierTest, RemoveUnusedLoopOperands) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   HloModule* the_module = &module();
   EXPECT_TRUE(WhileLoopSimplifier().Run(the_module).ValueOrDie());
 
@@ -439,7 +439,7 @@ TEST_F(WhileLoopSimplifierTest, LoopWithNonTupleBodyShapeNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -472,7 +472,7 @@ TEST_F(WhileLoopSimplifierTest,
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
@@ -504,7 +504,7 @@ TEST_F(WhileLoopSimplifierTest, LoopWithArrayConstantNotSimplified) {
   }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   EXPECT_FALSE(WhileLoopSimplifier().Run(&module()).ValueOrDie());
 }
 
diff --git a/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc b/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
index 641907acf2..da4cf4ae0c 100644
--- a/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_verified_test_base.cc
@@ -64,7 +64,8 @@ HloModule& HloVerifiedTestBase::module() {
   return *module_;
 }
 
-void HloVerifiedTestBase::ParseAndVerifyModule(const char* hlo_text) {
+void HloVerifiedTestBase::ParseAndVerifyModule(
+    tensorflow::StringPiece hlo_text) {
   CHECK(!module_) << "Called ParseModule when test already has a module.";
   TF_ASSERT_OK_AND_ASSIGN(module_, tools::Parse(hlo_text));
   VerifyModule();
diff --git a/tensorflow/compiler/xla/tests/hlo_verified_test_base.h b/tensorflow/compiler/xla/tests/hlo_verified_test_base.h
index c0cb12bc93..e5bb14a883 100644
--- a/tensorflow/compiler/xla/tests/hlo_verified_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_verified_test_base.h
@@ -44,7 +44,7 @@ class HloVerifiedTestBase : public HloTestBase {
   // Returns the default HloModule, lazily creating it if necessary via
   // HloTestBase::CreateNewModule().
   HloModule& module();
-  void ParseAndVerifyModule(const char* hlo_text);
+  void ParseAndVerifyModule(tensorflow::StringPiece hlo_text);
 
   // Sets the shape-size function used during hlo verification. If this isn't
   // called, a default ShapeVerifier is used instead.
-- 
GitLab


From 239eb8b652f94b43d51f7c7ffdbbfc02ad094a9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 15:16:38 -0700
Subject: [PATCH 083/960] PIE binaries that depends on static libraries usually
 have text relocations in the final executable, which causes link
 warnings/errors (different linker behaves differently). The optimal way to
 fix this is to link the binary with shared library, however, the libraries
 are NVIDIA-proprietary, not all of them have shared version (for example:
 cuda_9_0/lib64/libculibos.a)

PiperOrigin-RevId: 189254317
---
 tensorflow/tensorflow.bzl | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index a0ec1708d5..bab1e82c86 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -905,6 +905,14 @@ def tf_cuda_library(deps=None, cuda_deps=None, copts=tf_copts(), **kwargs):
   if not cuda_deps:
     cuda_deps = []
 
+  if 'linkstatic' not in kwargs or kwargs['linkstatic'] != 1:
+    enable_text_relocation_linkopt = select({
+          clean_dep("//tensorflow:darwin"): [],
+          "//conditions:default": ['-Wl,-z,notext'],})
+    if 'linkopts' in kwargs:
+      kwargs['linkopts'] += enable_text_relocation_linkopt
+    else:
+      kwargs['linkopts'] = enable_text_relocation_linkopt
   native.cc_library(
       deps=deps + if_cuda(cuda_deps + [
           clean_dep("//tensorflow/core:cuda"),
-- 
GitLab


From 6c62e650252ab32f83637a8de6720e73ffeca226 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 15:29:39 -0700
Subject: [PATCH 084/960] Pass error reporter to file copy allocation, and
 avoid loading model from file twice

PiperOrigin-RevId: 189256489
---
 .../native/nativeinterpreterwrapper_jni.cc    | 27 ++++---
 .../lite/NativeInterpreterWrapperTest.java    | 16 +++-
 tensorflow/contrib/lite/model.cc              | 74 ++++++++++++-------
 tensorflow/contrib/lite/model.h               | 37 +++++-----
 tensorflow/contrib/lite/model_test.cc         | 32 ++++++++
 5 files changed, 131 insertions(+), 55 deletions(-)

diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
index 21bcff40bd..cc448b03c3 100644
--- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
+++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
@@ -334,6 +334,19 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createErrorReporter(
   return reinterpret_cast<jlong>(error_reporter);
 }
 
+// Verifies whether the model is a flatbuffer file.
+class JNIFlatBufferVerifier : public tflite::TfLiteVerifier {
+ public:
+  bool Verify(const char* data, int length,
+              tflite::ErrorReporter* reporter) override {
+    if (!VerifyModel(data, length)) {
+      reporter->Report("The model is not a valid Flatbuffer file");
+      return false;
+    }
+    return true;
+  }
+};
+
 JNIEXPORT jlong JNICALL
 Java_org_tensorflow_lite_NativeInterpreterWrapper_createModel(
     JNIEnv* env, jclass clazz, jstring model_file, jlong error_handle) {
@@ -342,17 +355,11 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createModel(
   if (error_reporter == nullptr) return 0;
   const char* path = env->GetStringUTFChars(model_file, nullptr);
 
-  {
-    tflite::FileCopyAllocation allocation(path, nullptr);
-    if (!VerifyModel(allocation.base(), allocation.bytes())) {
-      throwException(env, kIllegalArgumentException,
-                     "Contents of %s is not a valid flatbuffer model", path);
-      env->ReleaseStringUTFChars(model_file, path);
-      return 0;
-    }
-  }
+  std::unique_ptr<tflite::TfLiteVerifier> verifier;
+  verifier.reset(new JNIFlatBufferVerifier());
 
-  auto model = tflite::FlatBufferModel::BuildFromFile(path, error_reporter);
+  auto model = tflite::FlatBufferModel::VerifyAndBuildFromFile(
+      path, verifier.get(), error_reporter);
   if (!model) {
     throwException(env, kIllegalArgumentException,
                    "Contents of %s does not encode a valid TensorFlowLite "
diff --git a/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java b/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
index d6b4e9f438..dbe45e5a05 100644
--- a/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
+++ b/tensorflow/contrib/lite/java/src/test/java/org/tensorflow/lite/NativeInterpreterWrapperTest.java
@@ -47,6 +47,9 @@ public final class NativeInterpreterWrapperTest {
   private static final String MODEL_WITH_CUSTOM_OP_PATH =
       "tensorflow/contrib/lite/java/src/testdata/with_custom_op.lite";
 
+  private static final String NONEXISTING_MODEL_PATH =
+      "tensorflow/contrib/lite/java/src/testdata/nonexisting_model.bin";
+
   @Test
   public void testConstructor() {
     NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(FLOAT_MODEL_PATH);
@@ -60,7 +63,18 @@ public final class NativeInterpreterWrapperTest {
       NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(INVALID_MODEL_PATH);
       fail();
     } catch (IllegalArgumentException e) {
-      assertThat(e).hasMessageThat().contains("is not a valid flatbuffer model");
+      assertThat(e).hasMessageThat().contains("The model is not a valid Flatbuffer file");
+    }
+  }
+
+  @Test
+  public void testConstructorWithNonexistingModel() {
+    try {
+      NativeInterpreterWrapper wrapper = new NativeInterpreterWrapper(NONEXISTING_MODEL_PATH);
+      fail();
+    } catch (IllegalArgumentException e) {
+      assertThat(e).hasMessageThat().contains("The model is not a valid Flatbuffer file");
+      assertThat(e).hasMessageThat().contains("Could not open");
     }
   }
 
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 3cf6bcbfcd..f28d56af67 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -32,11 +32,46 @@ namespace tflite {
 
 const char* kEmptyTensorName = "";
 
+// Loads a model from `filename`. If `mmap_file` is true then use mmap,
+// otherwise make a copy of the model in a buffer.
+std::unique_ptr<Allocation> GetAllocationFromFile(const char* filename,
+                                                  bool mmap_file,
+                                                  ErrorReporter* error_reporter,
+                                                  bool use_nnapi) {
+  std::unique_ptr<Allocation> allocation;
+  if (mmap_file) {
+    if (use_nnapi && NNAPIExists())
+      allocation.reset(new NNAPIAllocation(filename, error_reporter));
+    else
+      allocation.reset(new MMAPAllocation(filename, error_reporter));
+  } else {
+    allocation.reset(new FileCopyAllocation(filename, error_reporter));
+  }
+  return allocation;
+}
+
 std::unique_ptr<FlatBufferModel> FlatBufferModel::BuildFromFile(
     const char* filename, ErrorReporter* error_reporter) {
   std::unique_ptr<FlatBufferModel> model;
-  model.reset(new FlatBufferModel(filename, /*mmap_file=*/true, error_reporter,
-                                  /*use_nnapi=*/true));
+  auto allocation = GetAllocationFromFile(filename, /*mmap_file=*/true,
+                                          error_reporter, /*use_nnapi=*/true);
+  model.reset(new FlatBufferModel(allocation.release(), error_reporter));
+  if (!model->initialized()) model.reset();
+  return model;
+}
+
+std::unique_ptr<FlatBufferModel> FlatBufferModel::VerifyAndBuildFromFile(
+    const char* filename, TfLiteVerifier* verifier,
+    ErrorReporter* error_reporter) {
+  std::unique_ptr<FlatBufferModel> model;
+  auto allocation = GetAllocationFromFile(filename, /*mmap_file=*/true,
+                                          error_reporter, /*use_nnapi=*/true);
+  if (verifier &&
+      !verifier->Verify(static_cast<const char*>(allocation->base()),
+                        allocation->bytes(), error_reporter)) {
+    return model;
+  }
+  model.reset(new FlatBufferModel(allocation.release(), error_reporter));
   if (!model->initialized()) model.reset();
   return model;
 }
@@ -44,7 +79,9 @@ std::unique_ptr<FlatBufferModel> FlatBufferModel::BuildFromFile(
 std::unique_ptr<FlatBufferModel> FlatBufferModel::BuildFromBuffer(
     const char* buffer, size_t buffer_size, ErrorReporter* error_reporter) {
   std::unique_ptr<FlatBufferModel> model;
-  model.reset(new FlatBufferModel(buffer, buffer_size, error_reporter));
+  Allocation* allocation =
+      new MemoryAllocation(buffer, buffer_size, error_reporter);
+  model.reset(new FlatBufferModel(allocation, error_reporter));
   if (!model->initialized()) model.reset();
   return model;
 }
@@ -57,23 +94,6 @@ std::unique_ptr<FlatBufferModel> FlatBufferModel::BuildFromModel(
   return model;
 }
 
-FlatBufferModel::FlatBufferModel(const char* filename, bool mmap_file,
-                                 ErrorReporter* error_reporter, bool use_nnapi)
-    : error_reporter_(error_reporter ? error_reporter
-                                     : DefaultErrorReporter()) {
-  if (mmap_file) {
-    if (use_nnapi && NNAPIExists())
-      allocation_ = new NNAPIAllocation(filename, error_reporter);
-    else
-      allocation_ = new MMAPAllocation(filename, error_reporter);
-  } else {
-    allocation_ = new FileCopyAllocation(filename, error_reporter);
-  }
-  if (!allocation_->valid() || !CheckModelIdentifier()) return;
-
-  model_ = ::tflite::GetModel(allocation_->base());
-}
-
 bool FlatBufferModel::CheckModelIdentifier() const {
   if (!tflite::ModelBufferHasIdentifier(allocation_->base())) {
     const char* ident = flatbuffers::GetBufferIdentifier(allocation_->base());
@@ -85,21 +105,21 @@ bool FlatBufferModel::CheckModelIdentifier() const {
   return true;
 }
 
-FlatBufferModel::FlatBufferModel(const char* ptr, size_t num_bytes,
+FlatBufferModel::FlatBufferModel(const Model* model,
                                  ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  allocation_ = new MemoryAllocation(ptr, num_bytes, error_reporter);
-  if (!allocation_->valid()) return;
-
-  model_ = ::tflite::GetModel(allocation_->base());
+  model_ = model;
 }
 
-FlatBufferModel::FlatBufferModel(const Model* model,
+FlatBufferModel::FlatBufferModel(Allocation* allocation,
                                  ErrorReporter* error_reporter)
     : error_reporter_(error_reporter ? error_reporter
                                      : DefaultErrorReporter()) {
-  model_ = model;
+  allocation_ = allocation;
+  if (!allocation_->valid() || !CheckModelIdentifier()) return;
+
+  model_ = ::tflite::GetModel(allocation_->base());
 }
 
 FlatBufferModel::~FlatBufferModel() { delete allocation_; }
diff --git a/tensorflow/contrib/lite/model.h b/tensorflow/contrib/lite/model.h
index 8dc1c794dc..38eea0e26b 100644
--- a/tensorflow/contrib/lite/model.h
+++ b/tensorflow/contrib/lite/model.h
@@ -41,6 +41,17 @@ limitations under the License.
 
 namespace tflite {
 
+// Abstract interface that verifies whether a given model is legit.
+// It facilitates the use-case to verify and build a model without loading it
+// twice.
+class TfLiteVerifier {
+ public:
+  // Returns true if the model is legit.
+  virtual bool Verify(const char* data, int length,
+                      ErrorReporter* reporter) = 0;
+  virtual ~TfLiteVerifier() {}
+};
+
 // An RAII object that represents a read-only tflite model, copied from disk,
 // or mmapped. This uses flatbuffers as the serialization format.
 class FlatBufferModel {
@@ -50,6 +61,12 @@ class FlatBufferModel {
       const char* filename,
       ErrorReporter* error_reporter = DefaultErrorReporter());
 
+  // Verifies whether the content of the file is legit, then builds a model
+  // based on the file. Returns a nullptr in case of failure.
+  static std::unique_ptr<FlatBufferModel> VerifyAndBuildFromFile(
+      const char* filename, TfLiteVerifier* verifier = nullptr,
+      ErrorReporter* error_reporter = DefaultErrorReporter());
+
   // Builds a model based on a pre-loaded flatbuffer. The caller retains
   // ownership of the buffer and should keep it alive until the returned object
   // is destroyed. Returns a nullptr in case of failure.
@@ -82,23 +99,9 @@ class FlatBufferModel {
   bool CheckModelIdentifier() const;
 
  private:
-  // Loads a model from `filename`. If `mmap_file` is true then use mmap,
-  // otherwise make a copy of the model in a buffer.
-  //
-  // Note, if `error_reporter` is null, then a DefaultErrorReporter() will be
-  // used.
-  explicit FlatBufferModel(
-      const char* filename, bool mmap_file = true,
-      ErrorReporter* error_reporter = DefaultErrorReporter(),
-      bool use_nnapi = false);
-
-  // Loads a model from `ptr` and `num_bytes` of the model file. The `ptr` has
-  // to remain alive and unchanged until the end of this flatbuffermodel's
-  // lifetime.
-  //
-  // Note, if `error_reporter` is null, then a DefaultErrorReporter() will be
-  // used.
-  FlatBufferModel(const char* ptr, size_t num_bytes,
+  // Loads a model from a given allocation. FlatBufferModel will take over the
+  // ownership of `allocation`, and delete it in desctructor.
+  FlatBufferModel(Allocation* allocation,
                   ErrorReporter* error_reporter = DefaultErrorReporter());
 
   // Loads a model from Model flatbuffer. The `model` has to remain alive and
diff --git a/tensorflow/contrib/lite/model_test.cc b/tensorflow/contrib/lite/model_test.cc
index 66f22fd66a..ae6c1ece18 100644
--- a/tensorflow/contrib/lite/model_test.cc
+++ b/tensorflow/contrib/lite/model_test.cc
@@ -209,6 +209,38 @@ TEST(BasicFlatBufferModel, TestNullModel) {
   ASSERT_EQ(interpreter.get(), nullptr);
 }
 
+// Mocks the verifier by setting the result in ctor.
+class FakeVerifier : public tflite::TfLiteVerifier {
+ public:
+  explicit FakeVerifier(bool result) : result_(result) {}
+  bool Verify(const char* data, int length,
+              tflite::ErrorReporter* reporter) override {
+    return result_;
+  }
+
+ private:
+  bool result_;
+};
+
+TEST(BasicFlatBufferModel, TestWithTrueVerifier) {
+  FakeVerifier verifier(true);
+  ASSERT_TRUE(FlatBufferModel::VerifyAndBuildFromFile(
+      "tensorflow/contrib/lite/testdata/test_model.bin",
+      &verifier));
+}
+
+TEST(BasicFlatBufferModel, TestWithFalseVerifier) {
+  FakeVerifier verifier(false);
+  ASSERT_FALSE(FlatBufferModel::VerifyAndBuildFromFile(
+      "tensorflow/contrib/lite/testdata/test_model.bin",
+      &verifier));
+}
+
+TEST(BasicFlatBufferModel, TestWithNullVerifier) {
+  ASSERT_TRUE(FlatBufferModel::VerifyAndBuildFromFile(
+      "tensorflow/contrib/lite/testdata/test_model.bin", nullptr));
+}
+
 struct TestErrorReporter : public ErrorReporter {
   int Report(const char* format, va_list args) override {
     calls++;
-- 
GitLab


From 1e75c69339da2fbf2c5c5fbeb891243badae7ff8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 15:44:56 -0700
Subject: [PATCH 085/960] Automated g4 rollback of changelist 189231636

PiperOrigin-RevId: 189258641
---
 README.md                                     |    4 -
 SECURITY.md                                   |   14 +-
 configure.py                                  |    7 +-
 .../xla/service/generic_transfer_manager.cc   |    9 +-
 .../compiler/xla/tests/convolution_test.cc    |    2 +-
 tensorflow/contrib/BUILD                      |   10 +-
 tensorflow/contrib/__init__.py                |    6 +-
 .../boosted_trees/lib/utils/batch_features.h  |    6 +-
 tensorflow/contrib/cmake/README.md            |   12 +-
 tensorflow/contrib/cmake/external/grpc.cmake  |    1 -
 .../contrib/cmake/external/protobuf.cmake     |    2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |    4 -
 tensorflow/contrib/data/__init__.py           |    4 -
 .../contrib/data/python/kernel_tests/BUILD    |   17 -
 .../kernel_tests/slide_dataset_op_test.py     |  242 ---
 tensorflow/contrib/data/python/ops/BUILD      |    1 -
 tensorflow/contrib/data/python/ops/sliding.py |  102 --
 tensorflow/contrib/distributions/BUILD        |    2 -
 tensorflow/contrib/eager/python/BUILD         |    5 +-
 .../python/examples/linear_regression/BUILD   |    1 -
 tensorflow/contrib/factorization/BUILD        |    5 +-
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      |    9 +-
 tensorflow/contrib/gan/BUILD                  |    1 -
 tensorflow/contrib/kafka/BUILD                |  108 +-
 .../kafka/kernels/kafka_dataset_ops.cc        |    4 +-
 tensorflow/contrib/kafka/ops/dataset_ops.cc   |   44 -
 .../kafka/python/ops/kafka_dataset_ops.py     |    9 +-
 .../kafka/python/ops/kafka_op_loader.py       |   24 -
 .../contrib/kfac/python/kernel_tests/BUILD    |    1 -
 tensorflow/contrib/labeled_tensor/BUILD       |    1 -
 tensorflow/contrib/layers/BUILD               |    2 -
 .../layers/python/layers/embedding_ops.py     |    2 +-
 tensorflow/contrib/learn/BUILD                |   12 +-
 .../learn/python/learn/ops/embeddings_ops.py  |    2 +-
 tensorflow/contrib/lite/Makefile              |    9 +-
 tensorflow/contrib/lite/arena_planner.h       |    2 +-
 tensorflow/contrib/lite/build_rpi_lib.sh      |   22 -
 tensorflow/contrib/lite/builtin_ops.h         |    2 +-
 tensorflow/contrib/lite/error_reporter.h      |    2 +-
 tensorflow/contrib/lite/g3doc/rpi.md          |   50 -
 tensorflow/contrib/lite/interpreter.h         |    2 +-
 tensorflow/contrib/lite/interpreter_test.cc   |    2 +-
 tensorflow/contrib/lite/kernels/conv.cc       |    2 +-
 .../contrib/lite/kernels/depthwise_conv.cc    |    2 +-
 .../contrib/lite/kernels/fully_connected.cc   |    2 +-
 tensorflow/contrib/lite/kernels/kernel_util.h |    2 +-
 .../contrib/lite/kernels/lsh_projection.cc    |    2 +-
 tensorflow/contrib/lite/kernels/lstm.cc       |    6 +-
 tensorflow/contrib/lite/kernels/reshape.cc    |   12 +-
 .../contrib/lite/kernels/reshape_test.cc      |    2 +-
 tensorflow/contrib/lite/kernels/test_util.cc  |    4 +-
 .../kernels/unidirectional_sequence_lstm.cc   |    2 +-
 tensorflow/contrib/lite/memory_planner.h      |    4 +-
 tensorflow/contrib/lite/model.h               |    2 +-
 .../contrib/lite/nnapi/NeuralNetworksShim.h   |    2 +-
 tensorflow/contrib/lite/rpi_makefile.inc      |   33 -
 .../schema/builtin_ops_header/generator.cc    |    2 +-
 .../contrib/lite/simple_memory_arena.cc       |    6 +-
 tensorflow/contrib/lite/simple_memory_arena.h |    6 +-
 tensorflow/contrib/lookup/BUILD               |    1 -
 tensorflow/contrib/makefile/README.md         |    2 -
 tensorflow/contrib/makefile/build_all_ios.sh  |    3 +-
 tensorflow/contrib/mpi/mpi_utils.h            |    2 -
 .../contrib/predictor/predictor_factories.py  |    4 +-
 tensorflow/contrib/py2tf/converters/BUILD     |    2 -
 .../contrib/py2tf/converters/single_return.py |    2 +-
 tensorflow/contrib/py2tf/utils/BUILD          |    1 -
 .../quantize/python/fold_batch_norms.py       |    4 +-
 .../contrib/quantize/python/quant_ops.py      |    4 +-
 .../contrib/quantize/python/quantize.py       |    2 +-
 .../contrib/quantize/python/quantize_graph.py |    2 +-
 .../python/quantize_parameterized_test.py     |    8 +-
 .../contrib/quantize/python/quantize_test.py  |    2 +-
 .../contrib/remote_fused_graph/pylib/BUILD    |    1 +
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |    6 +-
 tensorflow/contrib/saved_model/BUILD          |    1 -
 .../seq2seq/python/ops/beam_search_decoder.py |    7 +-
 tensorflow/contrib/session_bundle/BUILD       |    1 -
 .../contrib/slim/python/slim/data/BUILD       |    1 -
 tensorflow/contrib/tensor_forest/BUILD        |    1 +
 tensorflow/contrib/tensorboard/BUILD          |    1 -
 tensorflow/contrib/tensorrt/BUILD             |    2 -
 tensorflow/contrib/tensorrt/README.md         |   23 +-
 tensorflow/contrib/tensorrt/__init__.py       |   18 +-
 .../contrib/tensorrt/convert/convert_graph.cc |  256 +--
 .../contrib/tensorrt/convert/convert_graph.h  |    8 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 1469 +++--------------
 .../contrib/tensorrt/convert/convert_nodes.h  |   53 +-
 .../contrib/tensorrt/kernels/trt_calib_op.cc  |   11 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |   39 +-
 tensorflow/contrib/tensorrt/log/trt_logger.cc |    8 +-
 tensorflow/contrib/tensorrt/log/trt_logger.h  |    4 +-
 .../contrib/tensorrt/python/__init__.py       |    1 -
 .../contrib/tensorrt/python/trt_convert.py    |   68 +-
 .../tensorrt/resources/trt_int8_calibrator.cc |   56 +-
 .../tensorrt/resources/trt_int8_calibrator.h  |   15 +-
 .../contrib/tensorrt/test/test_tftrt.py       |   57 +-
 tensorflow/contrib/tensorrt/trt_conversion.i  |   63 +-
 tensorflow/contrib/timeseries/examples/BUILD  |    5 +-
 .../timeseries/python/timeseries/BUILD        |    5 +-
 .../timeseries/state_space_models/BUILD       |    1 -
 tensorflow/contrib/tpu/BUILD                  |    1 -
 tensorflow/contrib/util/loader.py             |    7 +-
 tensorflow/core/BUILD                         |    4 -
 .../base_api/api_def_SlideDataset.pbtxt       |   18 -
 .../core/distributed_runtime/tensor_coding.cc |    4 +-
 tensorflow/core/framework/dataset.h           |    4 +-
 tensorflow/core/graph/mkl_layout_pass.cc      |   31 +-
 .../grappler/optimizers/loop_optimizer.cc     |    8 +-
 tensorflow/core/kernels/BUILD                 |   12 +-
 tensorflow/core/kernels/data/BUILD            |   14 -
 .../core/kernels/data/slide_dataset_op.cc     |  252 ---
 tensorflow/core/kernels/depthtospace_op.cc    |    3 -
 .../core/kernels/depthtospace_op_gpu.cu.cc    |    6 -
 tensorflow/core/kernels/hexagon/BUILD         |    1 -
 .../core/kernels/mkl_conv_grad_filter_ops.cc  |   81 +-
 .../core/kernels/mkl_conv_grad_input_ops.cc   |   18 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  146 +-
 tensorflow/core/kernels/mkl_conv_ops.h        |  117 +-
 .../core/kernels/mkl_input_conversion_op.cc   |    7 +-
 tensorflow/core/kernels/mkl_relu_op.cc        |   23 +-
 .../core/kernels/segment_reduction_ops.h      |   14 +-
 tensorflow/core/kernels/spacetodepth_op.cc    |    3 -
 .../core/kernels/spacetodepth_op_gpu.cu.cc    |    6 -
 tensorflow/core/lib/core/stringpiece.cc       |    2 +
 tensorflow/core/lib/core/stringpiece.h        |    2 +-
 tensorflow/core/lib/io/record_reader.cc       |    2 -
 tensorflow/core/lib/io/record_reader.h        |    4 +-
 tensorflow/core/ops/dataset_ops.cc            |   12 +-
 tensorflow/core/ops/nn_ops.cc                 |    8 -
 tensorflow/core/platform/tracing.h            |    2 +-
 .../platform/windows/windows_file_system.cc   |    3 +-
 tensorflow/core/public/version.h              |    2 +-
 tensorflow/docs_src/community/welcome.md      |    4 +-
 tensorflow/docs_src/install/install_c.md      |    2 +-
 tensorflow/docs_src/install/install_go.md     |    2 +-
 tensorflow/docs_src/install/install_java.md   |   22 +-
 tensorflow/docs_src/install/install_linux.md  |   56 +-
 tensorflow/docs_src/install/install_mac.md    |   23 +-
 .../docs_src/install/install_sources.md       |    4 +-
 .../docs_src/install/install_windows.md       |    5 +-
 tensorflow/docs_src/performance/xla/jit.md    |    2 +-
 .../docs_src/programmers_guide/debugger.md    |    3 +-
 tensorflow/docs_src/programmers_guide/faq.md  |    4 +-
 .../summaries_and_tensorboard.md              |    2 +-
 .../docs_src/programmers_guide/using_tpu.md   |    7 +-
 tensorflow/docs_src/tutorials/layers.md       |   12 +-
 .../docs_src/tutorials/recurrent_quickdraw.md |    3 +-
 tensorflow/docs_src/tutorials/wide.md         |   16 +-
 .../examples/android/AndroidManifest.xml      |    4 -
 .../org/tensorflow/demo/CameraActivity.java   |    7 +-
 .../org/tensorflow/demo/StylizeActivity.java  |   60 -
 tensorflow/python/BUILD                       |   84 +-
 tensorflow/python/debug/BUILD                 |    1 -
 tensorflow/python/estimator/estimator.py      |   32 +-
 tensorflow/python/estimator/training.py       |   26 +-
 tensorflow/python/keras/BUILD                 |    5 +-
 tensorflow/python/kernel_tests/BUILD          |    4 +
 .../python/kernel_tests/conv_ops_test.py      |   20 +-
 .../kernel_tests/depthtospace_op_test.py      |   10 +-
 .../kernel_tests/spacetodepth_op_test.py      |   10 +-
 tensorflow/python/lib/io/file_io_test.py      |    5 -
 tensorflow/python/ops/nn_ops.py               |    2 +-
 tensorflow/python/ops/rnn.py                  |   17 +-
 tensorflow/python/ops/special_math_ops.py     |    4 +-
 .../python/ops/special_math_ops_test.py       |    5 -
 tensorflow/python/tools/freeze_graph.py       |   36 +-
 tensorflow/python/tools/saved_model_cli.py    |   60 -
 .../python/tools/saved_model_cli_test.py      |   22 -
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |   33 +-
 tensorflow/tensorflow.bzl                     |   18 +-
 .../tools/api/tests/api_compatibility_test.py |    2 -
 tensorflow/tools/ci_build/Dockerfile.cmake    |    5 +-
 .../windows/cpu/pip/build_tf_windows.sh       |    3 +-
 tensorflow/tools/def_file_filter/BUILD        |    0
 tensorflow/tools/def_file_filter/BUILD.tpl    |   15 -
 .../def_file_filter/def_file_filter.py.tpl    |  168 --
 .../def_file_filter_configure.bzl             |   56 -
 tensorflow/tools/dist_test/README.md          |    8 -
 tensorflow/tools/dist_test/local_test.sh      |   22 +-
 tensorflow/tools/git/gen_git_source.py        |    7 -
 tensorflow/tools/graph_transforms/BUILD       |    1 -
 .../graph_transforms/fold_old_batch_norms.cc  |   67 -
 .../fold_old_batch_norms_test.cc              |   95 --
 tensorflow/tools/pip_package/BUILD            |  129 +-
 tensorflow/tools/pip_package/setup.py         |    4 +-
 .../tools/test/upload_test_benchmarks.py      |    9 +-
 tensorflow/workspace.bzl                      |    8 +-
 third_party/jpeg/jpeg.BUILD                   |    4 +-
 third_party/kafka/BUILD                       |   13 +-
 third_party/py/BUILD.tpl                      |   22 +-
 191 files changed, 920 insertions(+), 4244 deletions(-)
 delete mode 100644 tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
 delete mode 100644 tensorflow/contrib/data/python/ops/sliding.py
 delete mode 100644 tensorflow/contrib/kafka/ops/dataset_ops.cc
 delete mode 100644 tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
 delete mode 100755 tensorflow/contrib/lite/build_rpi_lib.sh
 delete mode 100644 tensorflow/contrib/lite/g3doc/rpi.md
 delete mode 100644 tensorflow/contrib/lite/rpi_makefile.inc
 delete mode 100644 tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
 delete mode 100644 tensorflow/core/kernels/data/slide_dataset_op.cc
 delete mode 100644 tensorflow/tools/def_file_filter/BUILD
 delete mode 100644 tensorflow/tools/def_file_filter/BUILD.tpl
 delete mode 100644 tensorflow/tools/def_file_filter/def_file_filter.py.tpl
 delete mode 100644 tensorflow/tools/def_file_filter/def_file_filter_configure.bzl

diff --git a/README.md b/README.md
index 3cdb6e478d..ef5bdc66ef 100644
--- a/README.md
+++ b/README.md
@@ -22,10 +22,6 @@ organization for the purposes of conducting machine learning and deep neural
 networks research.  The system is general enough to be applicable in a wide
 variety of other domains, as well.
 
-Keep up to date with release announcements and security updates by
-subscribing to
-[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
-
 ## Installation
 *See [Installing TensorFlow](https://www.tensorflow.org/get_started/os_setup.html) for instructions on how to install our release binaries or how to build from source.*
 
diff --git a/SECURITY.md b/SECURITY.md
index 2aaa9202d5..fea24b2739 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -6,7 +6,7 @@ report vulnerabilities in TensorFlow.
 
 ## TensorFlow models are programs
 
-TensorFlow's runtime system interprets and executes programs. What machine
+TensorFlow's runtime system interprets and executes programs. What machine 
 learning practitioners term
 [**models**](https://developers.google.com/machine-learning/glossary/#model) are
 expressed as programs that TensorFlow executes.  TensorFlow programs are encoded
@@ -28,12 +28,12 @@ data you supply to TensorFlow to train a model, or to use a model to run
 inference on the data.
 
 **TensorFlow models are programs, and need to be treated as such from a security
-perspective.**
+perspective.** 
 
 ## Running untrusted models
 
 As a general rule: **Always** execute untrusted models inside a sandbox (e.g.,
-[nsjail](https://github.com/google/nsjail)).
+[nsjail](https://github.com/google/nsjail)). 
 
 There are several ways in which a model could become untrusted. Obviously, if an
 untrusted party supplies TensorFlow kernels, arbitrary code may be executed.
@@ -109,11 +109,11 @@ graphs known to the `ModelServer`. This means that an attacker may run
 graphs using untrusted inputs as described above, but they would not be able to
 execute arbitrary graphs. It is possible to safely expose a `ModelServer`
 directly to an untrusted network, **but only if the graphs it is configured to
-use have been carefully audited to be safe**.
+use have been carefully audited to be safe**. 
 
 Similar to best practices for other servers, we recommend running any
 `ModelServer` with appropriate privileges (i.e., using a separate user with
-reduced permissions). In the spirit of defense in depth, we recommend
+reduced permisisons). In the spirit of defense in depth, we recommend
 authenticating requests to any TensorFlow server connected to an untrusted
 network, as well as sandboxing the server to minimize the adverse effects of
 any breach.
@@ -133,7 +133,7 @@ which exhibit unexpected or unwanted behaviors. The fact that TensorFlow models
 can perform arbitrary computations means that they may read and write files,
 communicate via the network, produce deadlocks and infinite loops, or run out
 of memory. It is only when these behaviors are outside the specifications of the
-operations involved that such behavior is a vulnerability.
+operations involved that such behavior is a vulnerability. 
 
 A `FileWriter` writing a file is not unexpected behavior and therefore is not a
 vulnerability in TensorFlow. A `MatMul` allowing arbitrary binary code execution
@@ -168,7 +168,7 @@ below).
 
 Please use a descriptive subject line for your report email. After the initial
 reply to your report, the security team will endeavor to keep you informed of
-the progress being made towards a fix and announcement.
+the progress being made towards a fix and announcement. 
 
 If you believe that an existing (public) issue is security-related, please send
 an email to `security@tensorflow.org`. The email should include the issue ID and
diff --git a/configure.py b/configure.py
index d14edef1be..97f46757ee 100644
--- a/configure.py
+++ b/configure.py
@@ -1048,10 +1048,7 @@ def set_tf_tensorrt_install_path(environ_cp):
 
     for lib_file in possible_files:
       if is_compatible(lib_file, cuda_ver, cudnn_ver):
-        matches = nvinfer_pattern.search(lib_file)
-        if len(matches.groups()) == 0:
-          continue
-        ver_str = matches.group(1)
+        ver_str = nvinfer_pattern.search(lib_file).group(1)
         ver = convert_version_to_int(ver_str) if len(ver_str) else 0
         if ver > highest_ver[0]:
           highest_ver = [ver, ver_str, lib_file]
@@ -1380,7 +1377,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.10.0')
+  check_bazel_version('0.5.4')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.cc b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
index a99e2b7794..78dc0ad4fc 100644
--- a/tensorflow/compiler/xla/service/generic_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
@@ -38,7 +38,14 @@ namespace xla {
 
 GenericTransferManager::GenericTransferManager(se::Platform::Id platform_id,
                                                size_t pointer_size)
-    : platform_id_(platform_id), pointer_size_(pointer_size) {}
+    : platform_id_(platform_id), pointer_size_(pointer_size) {
+  // We currently only support kHostPlatformId for CPU, kCudaPlatformId for
+  // GPU and kInterpreterPlatformId for Interpreter. Before supporting other
+  // platforms, we need to test this transfer manager on them.
+  CHECK(platform_id_ == se::host::kHostPlatformId ||
+        platform_id_ == se::interpreter::kInterpreterPlatformId ||
+        platform_id_ == se::cuda::kCudaPlatformId);
+}
 
 se::Platform::Id GenericTransferManager::PlatformId() const {
   return platform_id_;
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 72715398de..99640f5bb5 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -723,7 +723,7 @@ INSTANTIATE_TEST_CASE_P(
 );
 #endif
 
-XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
+TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
   ComputationBuilder builder(client_, TestName());
   Shape input_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
   Shape filter_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 986b61b3ea..bab37e8906 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -8,7 +8,6 @@ package(default_visibility = ["//tensorflow:__subpackages__"])
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
-load("//tensorflow:tensorflow.bzl", "if_not_windows")
 
 py_library(
     name = "contrib_py",
@@ -52,6 +51,7 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
+        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -63,6 +63,7 @@ py_library(
         "//tensorflow/contrib/linalg:linalg_py",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
+        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
@@ -109,10 +110,6 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]) + if_not_windows([
-        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",  # unix dependency, need to fix code
-        "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
-        "//tensorflow/contrib/kafka",  # has some linking issue on opensssl.
     ]),
 )
 
@@ -124,7 +121,6 @@ cc_library(
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -151,7 +147,7 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
+        "//tensorflow/contrib/kafka:kafka_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 669d611b01..4f6f539027 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
@@ -85,8 +83,7 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-if os.name != 'nt':
-  from tensorflow.contrib.lite.python import lite
+from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
@@ -95,7 +92,6 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
-del os
 del LazyLoader
 
 del absolute_import
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
index 7815fa049a..da5e744851 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
@@ -48,9 +48,9 @@ class BatchFeatures {
   Status GetFeatureColumnSizes(int64* const num_dense_float_features,
                                int64* const num_sparse_float_features,
                                int64* const num_sparse_int_features) const {
-    QCHECK_NE(num_dense_float_features, (int64*) nullptr);
-    QCHECK_NE(num_sparse_float_features, (int64*) nullptr);
-    QCHECK_NE(num_sparse_int_features, (int64*) nullptr);
+    QCHECK_NE(num_dense_float_features, nullptr);
+    QCHECK_NE(num_sparse_float_features, nullptr);
+    QCHECK_NE(num_sparse_int_features, nullptr);
     *num_dense_float_features = dense_float_feature_columns_.size();
     *num_sparse_float_features = sparse_float_feature_columns_.size();
     *num_sparse_int_features = sparse_int_feature_columns_.size();
diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index fe83bb3204..8f85a75ee4 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -26,7 +26,7 @@ The CMake files in this directory can build the core TensorFlow runtime, an
 example C++ binary, and a PIP package containing the runtime and Python
 bindings.
 
-### Prerequisites
+### Pre-requisites
 
 * CMake version 3.5 or later.
 
@@ -34,16 +34,14 @@ bindings.
 
 * [SWIG](http://www.swig.org/download.html)
 
-* Additional prerequisites for Microsoft Windows:
+* Additional pre-requisites for Microsoft Windows:
   - Visual Studio 2015
   - Python 3.5
+  - NumPy 1.11.0 or later
 
-* Additional prerequisites for Linux:
+* Additional pre-requisites for Linux:
   - Python 2.7 or later
   - [Docker](https://www.docker.com/) (for automated testing)
-
-* Python dependencies:
-  - wheel
   - NumPy 1.11.0 or later
 
 ### Known-good configurations
@@ -104,7 +102,7 @@ ops or APIs.
 Step-by-step Windows build
 ==========================
 
-1. Install the prerequisites detailed above, and set up your environment.
+1. Install the pre-requisites detailed above, and set up your environment.
 
    * The following commands assume that you are using the Windows Command
      Prompt (`cmd.exe`). You will need to set up your environment to use the
diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index 17f65999fa..a9f43a3ecb 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -35,7 +35,6 @@ else()
   set(grpc_STATIC_LIBRARIES
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
-      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
 endif()
 
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index ab464bc99a..aba8a5244e 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
 set(PROTOBUF_URL https://github.com/google/protobuf.git)
-set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
+set(PROTOBUF_TAG 396336eb961b75f03b25824fe86cf6490fb75e3a)
 
 if(WIN32)
   if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index b3e5b30826..9f96a4b797 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -476,10 +476,6 @@ if (tensorflow_BUILD_CC_TESTS)
     "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*_test.cc"
   )
 
-  list(REMOVE_ITEM tf_test_src_simple
-    ${tf_core_profiler_test_srcs}
-  )
-
   set(tf_test_lib tf_test_lib)
   add_library(${tf_test_lib} STATIC ${tf_src_testlib})
 
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 9212b69700..f09d156832 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -40,7 +40,6 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@rejection_resample
 @@scan
 @@shuffle_and_repeat
-@@sliding_window_batch
 @@sloppy_interleave
 @@unbatch
 
@@ -73,9 +72,6 @@ from tensorflow.contrib.data.python.ops.readers import SqlDataset
 from tensorflow.contrib.data.python.ops.resampling import rejection_resample
 from tensorflow.contrib.data.python.ops.scan_ops import scan
 from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat
-from tensorflow.contrib.data.python.ops.sliding import sliding_window_batch
-from tensorflow.python.data.ops.iterator_ops import Iterator
-from tensorflow.python.ops.parsing_ops import parse_single_example_v2 as parse_single_example
 # pylint: enable=unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 2c4d4adfda..22418b38e3 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -498,23 +498,6 @@ py_test(
     ],
 )
 
-tf_py_test(
-    name = "slide_dataset_op_test",
-    size = "small",
-    srcs = ["slide_dataset_op_test.py"],
-    additional_deps = [
-        "//tensorflow/contrib/data/python/ops:dataset_ops",
-        "//tensorflow/contrib/data/python/ops:transformation_ops",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:dtypes",
-        "//tensorflow/python:errors",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:sparse_tensor",
-        "//third_party/py/numpy",
-    ],
-)
-
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
deleted file mode 100644
index 33c48e20be..0000000000
--- a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
+++ /dev/null
@@ -1,242 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for the experimental input pipeline ops."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.data.python.ops import sliding
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.platform import test
-
-
-class SlideDatasetTest(test.TestCase):
-
-  def testSlideDataset(self):
-    """Test an dataset that maps a TF function across its input elements."""
-    components = (np.arange(7),
-                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
-                  np.array(37.0) * np.arange(7))
-
-    count = array_ops.placeholder(dtypes.int64, shape=[])
-    window_size = array_ops.placeholder(dtypes.int64, shape=[])
-    stride = array_ops.placeholder(dtypes.int64, shape=[])
-
-    def _map_fn(x, y, z):
-      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
-
-    # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
-    # RepeatDataset(count) -> _SlideDataset(window_size, stride).
-    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
-                .map(_map_fn)
-                .repeat(count)
-                .apply(sliding.sliding_window_batch(window_size, stride))
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
-                     [t.shape.as_list() for t in get_next])
-
-    with self.test_session() as sess:
-      # Slide over a finite input, where the window_size divides the
-      # total number of elements.
-      sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 7})
-      # Same formula with convolution layer.
-      num_batches = (20 * 7 - 14) // 7 + 1
-      for i in range(num_batches):
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range(14):
-            self.assertAllEqual(component[(i*7 + j) % 7]**2,
-                                result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Slide over a finite input, where the window_size does not
-      # divide the total number of elements.
-      sess.run(init_op, feed_dict={count: 20, window_size: 17, stride: 9})
-
-      num_batches = (20 * 7 - 17) // 9 + 1
-      for i in range(num_batches):
-        result = sess.run(get_next)
-        for component, result_component in zip(components, result):
-          for j in range(17):
-            self.assertAllEqual(component[(i*9 + j) % 7]**2,
-                                result_component[j])
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Slide over a finite input, which is less than window_size,
-      # should fail straight away.
-      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Slide over an empty input should fail straight away.
-      sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4})
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-      # Empty window_size should be an initialization time error.
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(init_op, feed_dict={count: 14, window_size: 0, stride: 0})
-
-      # Invalid stride should be an initialization time error.
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 0})
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 3})
-      with self.assertRaises(errors.InvalidArgumentError):
-        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 5})
-
-  def assertSparseValuesEqual(self, a, b):
-    self.assertAllEqual(a.indices, b.indices)
-    self.assertAllEqual(a.values, b.values)
-    self.assertAllEqual(a.dense_shape, b.dense_shape)
-
-  def testSlideSparse(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=[[0]], values=(i * [1]), dense_shape=[1])
-
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
-        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      num_batches = (10 - 5) // 3 + 1
-      for i in range(num_batches):
-        actual = sess.run(get_next)
-        expected = sparse_tensor.SparseTensorValue(
-            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
-            values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
-            dense_shape=[5, 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testSlideSparseWithDifferentDenseShapes(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=array_ops.expand_dims(
-              math_ops.range(i, dtype=dtypes.int64), 1),
-          values=array_ops.fill([math_ops.to_int32(i)], i),
-          dense_shape=[i])
-
-    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
-        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      num_batches = (10 - 5) // 3 + 1
-      for i in range(num_batches):
-        actual = sess.run(get_next)
-        expected_indices = []
-        expected_values = []
-        for j in range(5):
-          for k in range(i * 3 + j):
-            expected_indices.append([j, k])
-            expected_values.append(i * 3 + j)
-        expected = sparse_tensor.SparseTensorValue(
-            indices=expected_indices,
-            values=expected_values,
-            dense_shape=[5, i * 3 + 5 - 1])
-        self.assertTrue(sparse_tensor.is_sparse(actual))
-        self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testNestedSlideSparse(self):
-
-    def _sparse(i):
-      return sparse_tensor.SparseTensorValue(
-          indices=[[0]], values=(i * [1]), dense_shape=[1])
-
-    iterator = (dataset_ops.Dataset.range(10)
-                .map(_sparse)
-                .apply(sliding.sliding_window_batch(4, 2))
-                .apply(sliding.sliding_window_batch(3, 1))
-                .make_initializable_iterator())
-    init_op = iterator.initializer
-    get_next = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(init_op)
-      # Slide: 1st batch.
-      actual = sess.run(get_next)
-      expected = sparse_tensor.SparseTensorValue(
-          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
-                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
-                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
-          values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7],
-          dense_shape=[3, 4, 1])
-      self.assertTrue(sparse_tensor.is_sparse(actual))
-      self.assertSparseValuesEqual(actual, expected)
-      # Slide: 2nd batch.
-      actual = sess.run(get_next)
-      expected = sparse_tensor.SparseTensorValue(
-          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
-                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
-                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
-          values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9],
-          dense_shape=[3, 4, 1])
-      self.assertTrue(sparse_tensor.is_sparse(actual))
-      self.assertSparseValuesEqual(actual, expected)
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(get_next)
-
-  def testSlideShapeError(self):
-
-    def generator():
-      yield [1.0, 2.0, 3.0]
-      yield [4.0, 5.0, 6.0]
-      yield [7.0, 8.0, 9.0, 10.0]
-
-    iterator = (dataset_ops.Dataset.from_generator(generator, dtypes.float32,
-                                                   output_shapes=[None])
-                .apply(sliding.sliding_window_batch(3, 1))
-                .make_initializable_iterator())
-    next_element = iterator.get_next()
-
-    with self.test_session() as sess:
-      sess.run(iterator.initializer)
-      with self.assertRaisesRegexp(
-          errors.InvalidArgumentError,
-          r"Cannot batch tensors with different shapes in component 0. "
-          r"First element had shape \[3\] and element 2 had shape \[4\]."):
-        sess.run(next_element)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index c3331e9636..f03430c5c5 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -106,7 +106,6 @@ py_library(
         "interleave_ops.py",
         "resampling.py",
         "scan_ops.py",
-        "sliding.py",
         "stats_ops.py",
         "threadpool.py",
         "unique.py",
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
deleted file mode 100644
index 19cc3cb89f..0000000000
--- a/tensorflow/contrib/data/python/ops/sliding.py
+++ /dev/null
@@ -1,102 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Sliding dataset transformations."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.data.util import nest
-from tensorflow.python.data.util import sparse
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import gen_dataset_ops
-
-
-class _SlideDataset(dataset_ops.Dataset):
-  """A `Dataset` that passes a sliding window over its input."""
-
-  def __init__(self, input_dataset, window_size, stride=1):
-    """See `sliding_window_batch` for details."""
-    super(_SlideDataset, self).__init__()
-    self._input_dataset = input_dataset
-    self._window_size = ops.convert_to_tensor(
-        window_size, dtype=dtypes.int64, name="window_size")
-    self._stride = ops.convert_to_tensor(
-        stride, dtype=dtypes.int64, name="stride")
-
-  def _as_variant_tensor(self):
-    return gen_dataset_ops.slide_dataset(
-        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        window_size=self._window_size,
-        stride=self._stride,
-        output_shapes=nest.flatten(
-            sparse.as_dense_shapes(self.output_shapes, self.output_classes)),
-        output_types=nest.flatten(
-            sparse.as_dense_types(self.output_types, self.output_classes)))
-
-  @property
-  def output_classes(self):
-    return self._input_dataset.output_classes
-
-  @property
-  def output_shapes(self):
-    input_shapes = self._input_dataset.output_shapes
-    return nest.pack_sequence_as(input_shapes, [
-        tensor_shape.vector(None).concatenate(s)
-        for s in nest.flatten(self._input_dataset.output_shapes)
-    ])
-
-  @property
-  def output_types(self):
-    return self._input_dataset.output_types
-
-
-def sliding_window_batch(window_size, stride=1):
-  """A sliding window with size of `window_size` and step of `stride`.
-
-  This transformation passes a sliding window over this dataset. The
-  window size is `window_size` and step size is `stride`. If the left
-  elements cannot fill up the sliding window, this transformation will
-  drop the final smaller element. For example:
-
-  ```python
-  # NOTE: The following examples use `{ ... }` to represent the
-  # contents of a dataset.
-  a = { [1], [2], [3], [4], [5], [6] }
-
-  a.apply(tf.contrib.data.sliding_window_batch(window_size=3, stride=2)) ==
-  {
-      [[1], [2], [3]],
-      [[3], [4], [5]],
-  }
-  ```
-
-  Args:
-    window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
-      elements in the sliding window.
-    stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the
-      steps moving the sliding window forward for one iteration. The default
-      is `1`. It must be in `[1, window_size)`.
-
-  Returns:
-    A `Dataset` transformation function, which can be passed to
-    @{tf.data.Dataset.apply}.
-  """
-  def _apply_fn(dataset):
-    return _SlideDataset(dataset, window_size, stride)
-
-  return _apply_fn
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 1bd73ee704..6bd3f5f09b 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -454,7 +454,6 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
@@ -1144,7 +1143,6 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index eb810e06dd..32aa2c0a4a 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -267,10 +267,7 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",
-    ],
+    tags = ["notsan"],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 2f6cfdf31e..f86331af6f 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -22,7 +22,6 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index ad8568ad44..90f10f1fa8 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -224,10 +224,7 @@ py_test(
     srcs = ["python/ops/kmeans_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
-    tags = [
-        "nomac",  # b/73741358
-        "notsan",  # b/67512932
-    ],
+    tags = ["notsan"],  # b/67512932
     deps = [
         ":factorization_py",
         ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO",
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index 35341406a0..e61221a6b0 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -256,9 +256,6 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
         if (p != std::string::npos) {
           string rgb24 = line.substr(p + 9, line.find(" ", p + 9));
           rgb24 = rgb24.substr(0, rgb24.find(","));
-          // Strip anything after " ", in case the format is
-          // `640x360 [SAR 1:1 DAR 16:9]`
-          rgb24 = rgb24.substr(0, rgb24.find(" "));
           string rgb24_width = rgb24.substr(0, rgb24.find("x"));
           string rgb24_height = rgb24.substr(rgb24_width.length() + 1);
           if (strings::safe_strtou32(rgb24_width, &width_value) &&
@@ -273,10 +270,8 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
       // We only look for the first stream mapping to have the number of the
       // frames.
       // Once processed we will not further process stream mapping section.
-      if (line.find("frame=") == 0) {
-        // The format might be `frame=  166 ` or `frame=12488 `
-        string number = line.substr(6);
-        number = number.substr(number.find_first_not_of(" "));
+      if (line.find("frame=  ") == 0) {
+        string number = line.substr(8, line.find(" ", 8));
         number = number.substr(0, number.find(" "));
         if (strings::safe_strtou32(number, &frames_value)) {
           in_mapping = false;
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index ff6f3b7441..0eb0e3cbe2 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -354,7 +354,6 @@ py_test(
     name = "classifier_metrics_test",
     srcs = ["python/eval/python/classifier_metrics_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":classifier_metrics",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/kafka/BUILD b/tensorflow/contrib/kafka/BUILD
index 1c3974871c..efb403462a 100644
--- a/tensorflow/contrib/kafka/BUILD
+++ b/tensorflow/contrib/kafka/BUILD
@@ -1,93 +1,66 @@
-package(default_visibility = ["//tensorflow:internal"])
+package(
+    default_visibility = ["//visibility:private"],
+)
 
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load(
-    "//tensorflow:tensorflow.bzl",
-    "tf_gen_op_wrapper_py",
-    "tf_kernel_library",
-    "tf_custom_op_library",
-    "tf_custom_op_py_library",
-    "tf_gen_op_libs",
-    "tf_py_test",
-)
-
-py_library(
-    name = "kafka",
-    srcs = ["__init__.py"],
-    srcs_version = "PY2AND3",
-    deps = [
-        ":dataset_ops",
-    ],
-)
-
-tf_custom_op_library(
-    name = "_dataset_ops.so",
-    srcs = ["ops/dataset_ops.cc"],
-    deps = [":dataset_kernels"],
-)
-
-tf_gen_op_libs(
-    op_lib_names = ["dataset_ops"],
-)
+load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
+load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
+load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
 
-cc_library(
-    name = "dataset_kernels",
+tf_kernel_library(
+    name = "kafka_kernels",
     srcs = ["kernels/kafka_dataset_ops.cc"],
+    visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:bounds_check_lib",
+        "//tensorflow/core/kernels:dataset",
         "//third_party/eigen3",
         "@kafka",
-        "@protobuf_archive//:protobuf_headers",
     ],
-    alwayslink = 1,
 )
 
-py_library(
-    name = "dataset_ops",
-    srcs = [
-        "python/ops/kafka_dataset_ops.py",
-    ],
-    srcs_version = "PY2AND3",
+tf_gen_op_libs(
+    op_lib_names = ["kafka_ops"],
     deps = [
-        ":kafka_op_loader",
-        "//tensorflow/python:dataset_ops_gen",
-        "//tensorflow/python:util",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/util:nest",
+        "//tensorflow/core:lib",
     ],
 )
 
 tf_gen_op_wrapper_py(
-    name = "gen_dataset_ops",
-    out = "python/ops/gen_dataset_ops.py",
-    deps = ["//tensorflow/contrib/kafka:dataset_ops_op_lib"],
-)
-
-tf_kernel_library(
-    name = "dataset_ops_kernels",
-    deps = [
-        ":dataset_kernels",
-        "//tensorflow/core:framework",
-    ],
-    alwayslink = 1,
+    name = "gen_kafka_ops",
+    out = "python/ops/gen_kafka_ops.py",
+    require_shape_functions = True,
+    deps = [":kafka_ops_op_lib"],
 )
 
-tf_custom_op_py_library(
-    name = "kafka_op_loader",
-    srcs = ["python/ops/kafka_op_loader.py"],
-    dso = ["//tensorflow/contrib/kafka:_dataset_ops.so"],
-    kernels = [
-        ":dataset_ops_kernels",
-        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
+py_library(
+    name = "kafka",
+    srcs = [
+        "__init__.py",
+        "python/ops/kafka_dataset_ops.py",
     ],
     srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
     deps = [
-        ":gen_dataset_ops",
+        ":gen_kafka_ops",
         "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
+        "//tensorflow/python/data/ops:readers",
     ],
 )
 
@@ -115,7 +88,6 @@ tf_py_test(
     ],
     tags = [
         "manual",
-        "no_windows",
         "notap",
     ],
 )
@@ -123,9 +95,7 @@ tf_py_test(
 filegroup(
     name = "all_files",
     srcs = glob(
-        include = [
-            "**/*",
-        ],
+        ["**/*"],
         exclude = [
             "**/METADATA",
             "**/OWNERS",
diff --git a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
index a4cd4a2cc4..88ef5f3571 100644
--- a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
+++ b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
@@ -13,7 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/framework/dataset.h"
+#include "tensorflow/core/kernels/dataset.h"
+
+#include "tensorflow/core/framework/tensor.h"
 
 #include "src-cpp/rdkafkacpp.h"
 
diff --git a/tensorflow/contrib/kafka/ops/dataset_ops.cc b/tensorflow/contrib/kafka/ops/dataset_ops.cc
deleted file mode 100644
index 8cdf16103b..0000000000
--- a/tensorflow/contrib/kafka/ops/dataset_ops.cc
+++ /dev/null
@@ -1,44 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/framework/common_shape_fns.h"
-#include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/shape_inference.h"
-
-namespace tensorflow {
-
-REGISTER_OP("KafkaDataset")
-    .Input("topics: string")
-    .Input("servers: string")
-    .Input("group: string")
-    .Input("eof: bool")
-    .Input("timeout: int64")
-    .Output("handle: variant")
-    .SetIsStateful()
-    .SetShapeFn(shape_inference::ScalarShape)
-    .Doc(R"doc(
-Creates a dataset that emits the messages of one or more Kafka topics.
-
-topics: A `tf.string` tensor containing one or more subscriptions,
-  in the format of [topic:partition:offset:length],
-  by default length is -1 for unlimited.
-servers: A list of bootstrap servers.
-group: The consumer group id.
-eof: If True, the kafka reader will stop on EOF.
-timeout: The timeout value for the Kafka Consumer to wait
-  (in millisecond).
-)doc");
-
-}  // namespace tensorflow
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
index a1624614d1..8e51d27a34 100644
--- a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
+++ b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
@@ -17,9 +17,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.kafka.python.ops import kafka_op_loader  # pylint: disable=unused-import
-from tensorflow.contrib.kafka.python.ops import gen_dataset_ops
-from tensorflow.python.data.ops.dataset_ops import Dataset
+from tensorflow.contrib.kafka.python.ops import gen_kafka_ops
+from tensorflow.python.data.ops.readers import Dataset
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -59,8 +58,8 @@ class KafkaDataset(Dataset):
         timeout, dtype=dtypes.int64, name="timeout")
 
   def _as_variant_tensor(self):
-    return gen_dataset_ops.kafka_dataset(self._topics, self._servers,
-                                         self._group, self._eof, self._timeout)
+    return gen_kafka_ops.kafka_dataset(self._topics, self._servers, self._group,
+                                       self._eof, self._timeout)
 
   @property
   def output_classes(self):
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py b/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
deleted file mode 100644
index ec2fdea962..0000000000
--- a/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
+++ /dev/null
@@ -1,24 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Python helper for loading kafka ops and kernels."""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.contrib.util import loader
-from tensorflow.python.platform import resource_loader
-
-_dataset_ops = loader.load_op_library(
-    resource_loader.get_path_to_datafile("../../_dataset_ops.so"))
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index d1c449402a..146ae8b7e2 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -114,7 +114,6 @@ py_test(
     name = "utils_test",
     srcs = ["utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         "//tensorflow/contrib/kfac/python/ops:utils",
         "//tensorflow/contrib/tpu",
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 544065dac6..894e6f6946 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -70,7 +70,6 @@ py_test(
         "python/ops/core_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":_typecheck",
         ":core",
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index cc7bbabf21..852d06e1e3 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -188,7 +188,6 @@ py_test(
     size = "small",
     srcs = ["python/layers/normalization_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/contrib/framework:framework_py",
@@ -354,7 +353,6 @@ py_test(
     size = "small",
     srcs = ["python/ops/sparse_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py
index ffa208540d..b62e3050cd 100644
--- a/tensorflow/contrib/layers/python/layers/embedding_ops.py
+++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py
@@ -470,7 +470,7 @@ def embedding_lookup_unique(params, ids, name=None):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops.shape(ids)
     ids_flat = array_ops.reshape(
-        ids, math_ops.reduce_prod(shape, keepdims=True))
+        ids, math_ops.reduce_prod(shape, keep_dims=True))
     unique_ids, idx = array_ops.unique(ids_flat)
     unique_embeddings = embedding_ops.embedding_lookup(params, unique_ids)
     embeds_flat = array_ops.gather(unique_embeddings, idx)
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index b05f5eeaee..f837ca3265 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -5,8 +5,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "py_test")
-
 package(default_visibility = [
     "//engedu/ml/tf_from_scratch:__pkg__",
     "//tensorflow:internal",
@@ -117,7 +115,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/data_feeder_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/python:client_testlib",
@@ -173,7 +170,6 @@ tf_py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
@@ -192,7 +188,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/graph_actions_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
@@ -431,10 +426,7 @@ py_test(
     size = "medium",
     srcs = ["python/learn/estimators/kmeans_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "noasan",  # b/73741358
-        "nomac",
-    ],
+    tags = ["noasan"],
     deps = [
         ":learn",
         "//tensorflow/python:array_ops",
@@ -593,7 +585,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/io_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/learn/python/learn/datasets",
@@ -823,7 +814,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/utils/saved_model_export_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
index 8f9811cf25..b3b067b8e1 100644
--- a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
+++ b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
@@ -61,7 +61,7 @@ def embedding_lookup(params, ids, name='embedding_lookup'):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops_.shape(ids)
     ids_flat = array_ops_.reshape(
-        ids, math_ops.reduce_prod(shape, keepdims=True))
+        ids, math_ops.reduce_prod(shape, keep_dims=True))
     embeds_flat = nn.embedding_lookup(params, ids_flat, name)
     embed_shape = array_ops_.concat([shape, [-1]], 0)
     embeds = array_ops_.reshape(embeds_flat, embed_shape)
diff --git a/tensorflow/contrib/lite/Makefile b/tensorflow/contrib/lite/Makefile
index b4504f246a..7f31629272 100644
--- a/tensorflow/contrib/lite/Makefile
+++ b/tensorflow/contrib/lite/Makefile
@@ -27,10 +27,10 @@ LIBDIR := $(MAKEFILE_DIR)/gen/lib/
 GENDIR := $(MAKEFILE_DIR)/gen/obj/
 
 # Settings for the host compiler.
-CXX := $(CC_PREFIX)gcc
+CXX := $(CC_PREFIX) gcc
 CXXFLAGS := --std=c++11 -O3 -DNDEBUG
-CC := $(CC_PREFIX)gcc
-CFLAGS := -O3 -DNDEBUG
+CC := $(CC_PREFIX) gcc
+CFLAGS :=
 LDOPTS :=
 LDOPTS += -L/usr/local/lib
 ARFLAGS := -r
@@ -57,11 +57,10 @@ LIBS := \
 
 # If we're on Linux, also link in the dl library.
 ifeq ($(HOST_OS),LINUX)
-	LIBS += -ldl
+	LIBS += -ldl -lpthread
 endif
 
 include $(MAKEFILE_DIR)/ios_makefile.inc
-include $(MAKEFILE_DIR)/rpi_makefile.inc
 
 # This library is the main target for this makefile. It will contain a minimal
 # runtime that can be linked in to other programs.
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index f84b3dad95..58bc164619 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -33,7 +33,7 @@ class AllocationInfo;
 // each tensor needs to be allocated and deallocated, and preallocates all the
 // necessary memory (the PlanAllocations phase). It then assigns portions of
 // this memory buffer to each tensor (the ExecuteAllocations phase). Tensors may
-// share some of the buffer if a tensor B is to be allocated after another tensor
+// share some of the bufer if a tensor B is to be allocated after another tensor
 // A has been deallocated.
 //
 // If dynamic tensors are used the planning steps can be repeated during model
diff --git a/tensorflow/contrib/lite/build_rpi_lib.sh b/tensorflow/contrib/lite/build_rpi_lib.sh
deleted file mode 100755
index 3824b16412..0000000000
--- a/tensorflow/contrib/lite/build_rpi_lib.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash -x
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-set -e
-
-SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-cd "$SCRIPT_DIR/../../.."
-
-CC_PREFIX=arm-linux-gnueabihf- make -j 3 -f tensorflow/contrib/lite/Makefile TARGET=RPI TARGET_ARCH=armv7
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index ea3ae3489e..2218ea8eac 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -24,7 +24,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real builtin
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
 // ops.
 typedef enum {
   kTfLiteBuiltinAdd = 0,
diff --git a/tensorflow/contrib/lite/error_reporter.h b/tensorflow/contrib/lite/error_reporter.h
index 3c5f805f12..da193d2586 100644
--- a/tensorflow/contrib/lite/error_reporter.h
+++ b/tensorflow/contrib/lite/error_reporter.h
@@ -30,7 +30,7 @@ namespace tflite {
 //  va_list args;
 //  foo.Report("test %d", args); // where args is va_list
 //
-// Subclass ErrorReporter to provide another reporting destination.
+// Sublclass ErrorReporter to provide another reporting destination.
 // For example, if you have a GUI program, you might redirect to a buffer
 // that drives a GUI error log box.
 class ErrorReporter {
diff --git a/tensorflow/contrib/lite/g3doc/rpi.md b/tensorflow/contrib/lite/g3doc/rpi.md
deleted file mode 100644
index 7a3a231626..0000000000
--- a/tensorflow/contrib/lite/g3doc/rpi.md
+++ /dev/null
@@ -1,50 +0,0 @@
-# TensorFlow Lite for Raspberry Pi
-
-## Cross compiling
-### Installing toolchian
-This has been tested on Ubuntu 16.04.3 64bit and Tensorflow devel docker image [tensorflow/tensorflow:nightly-devel](https://hub.docker.com/r/tensorflow/tensorflow/tags/).
-
-To cross compiling TensorFlow Lite. First you should install the toolchain and libs.
-```bash
-sudo apt-get update
-sudo apt-get install crossbuild-essential-armhf
-```
-> If you are using docker, you may not use `sudo`
-
-### Building
-Clone this Tensorflow repository, Run this script at the root of the repository to download all the dependencies:
-> The Tensorflow repository is in `/tensorflow` if you are using `tensorflow/tensorflow:nightly-devel` docker image, just try it.
-```bash
-./tensorflow/contrib/lite/download_dependencies.sh
-```
-Note than you only need to to this once.
-
-You should then be able to compile:
-```bash
-./tensorflow/contrib/lite/build_rpi_lib.sh
-```
-
-This should compile a static library in:
-`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
-
-## Native compiling
-This has been tested on Raspberry Pi 3b, Raspbian GNU/Linux 9.1 (stretch), gcc version 6.3.0 20170516 (Raspbian 6.3.0-18+rpi1).
-
-Log in to you RPI, install the toolchain.
-```bash
-sudo apt-get instal build-essential
-```
-
-First, clone this TensorFlow repository. Run this at the root of the repository:
-```bash
-./tensorflow/contrib/lite/download_dependencies.sh
-```
-Note than you only need to to this once.
-
-You should then be able to compile:
-```bash
-./tensorflow/contrib/lite/build_rpi_lib.sh
-```
-
-This should compile a static library in:
-`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index 3749869f58..af143370ee 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -481,7 +481,7 @@ class Interpreter {
   // During Invoke(), Interpreter will allocate input tensors first, which are
   // known to be fixed size. Then it will allocate outputs from nodes as many
   // as possible. When there is a node that produces dynamic sized tensor.
-  // Interpreter will stop allocating tensors, set the value of next allocate
+  // Intepreter will stop allocating tensors, set the value of next allocate
   // node id, and execute the node to generate the output tensor before continue
   // to allocate successors. This process repeats until all nodes are executed.
   // NOTE: this relies on the order of nodes that is in topological order.
diff --git a/tensorflow/contrib/lite/interpreter_test.cc b/tensorflow/contrib/lite/interpreter_test.cc
index 72d4acedbe..7a029c7df8 100644
--- a/tensorflow/contrib/lite/interpreter_test.cc
+++ b/tensorflow/contrib/lite/interpreter_test.cc
@@ -40,7 +40,7 @@ TEST(BasicInterpreter, InvokeInvalidModel) {
   ASSERT_EQ(interpreter.Invoke(), kTfLiteOk);
 }
 
-// Test size accessor functions.
+// Test size accesser functions.
 TEST(BasicInterpreter, TestSizeFunctions) {
   Interpreter interpreter;
   int base_index;
diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc
index e0cd12f1b4..b91ba1a03d 100644
--- a/tensorflow/contrib/lite/kernels/conv.cc
+++ b/tensorflow/contrib/lite/kernels/conv.cc
@@ -64,7 +64,7 @@ struct OpData {
 
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multiplier plus a left shift.
+  // be represented as a fixed point multipler plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/depthwise_conv.cc b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
index cad9ce114c..15dbfe08c8 100644
--- a/tensorflow/contrib/lite/kernels/depthwise_conv.cc
+++ b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
@@ -52,7 +52,7 @@ enum KernelType {
 struct OpData {
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multiplier plus a left shift.
+  // be represented as a fixed point multipler plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc
index 888e67966c..a77fe94e49 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected.cc
@@ -48,7 +48,7 @@ enum KernelType {
 
 struct OpData {
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multiplier plus a left shift.
+  // be represented as a fixed point multipler plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/kernel_util.h b/tensorflow/contrib/lite/kernels/kernel_util.h
index 21da1daff7..28f53b9fbb 100644
--- a/tensorflow/contrib/lite/kernels/kernel_util.h
+++ b/tensorflow/contrib/lite/kernels/kernel_util.h
@@ -58,7 +58,7 @@ inline bool IsConstantTensor(TfLiteTensor* tensor) {
 }
 
 // Determines whether tensor is dynamic. Note that a tensor can be non-const and
-// not dynamic. This function specifically checks for a dynamic tensor.
+// not dynamic. This function specificially checks for a dynamic tensor.
 inline bool IsDynamicTensor(TfLiteTensor* tensor) {
   return tensor->allocation_type == kTfLiteDynamic;
 }
diff --git a/tensorflow/contrib/lite/kernels/lsh_projection.cc b/tensorflow/contrib/lite/kernels/lsh_projection.cc
index 0ee35775d5..5f73b56ed9 100644
--- a/tensorflow/contrib/lite/kernels/lsh_projection.cc
+++ b/tensorflow/contrib/lite/kernels/lsh_projection.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// LSH Projection projects an input to a bit vector via locality sensitive
+// LSH Projection projects an input to a bit vector via locality senstive
 // hashing.
 //
 // Options:
diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc
index 8cf1165135..b9255b23a5 100644
--- a/tensorflow/contrib/lite/kernels/lstm.cc
+++ b/tensorflow/contrib/lite/kernels/lstm.cc
@@ -213,9 +213,9 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
   // present.
   // 2) If projection weight is present, then projection bias is optional.
   // TODO(ghodrat): make sure this is correct.
-  const bool projection_tensors_consistent =
+  const bool projecton_tensors_consistent =
       ((projection_weights != nullptr) || (projection_bias == nullptr));
-  TF_LITE_ENSURE(context, projection_tensors_consistent == true);
+  TF_LITE_ENSURE(context, projecton_tensors_consistent == true);
 
   return kTfLiteOk;
 }
@@ -357,7 +357,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existence of only one to get the condition.
+  // check the existense of only one to the get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/kernels/reshape.cc b/tensorflow/contrib/lite/kernels/reshape.cc
index 438f70d311..f3e6ddc9f4 100644
--- a/tensorflow/contrib/lite/kernels/reshape.cc
+++ b/tensorflow/contrib/lite/kernels/reshape.cc
@@ -49,20 +49,20 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   TfLiteIntArray* output_size = TfLiteIntArrayCreate(params->num_dimensions);
   int num_output_elements = 1;
-  int stretch_dim = -1;
+  int strech_dim = -1;
   for (int i = 0; i < params->num_dimensions; ++i) {
     int value = params->shape[i];
     if (value == -1) {
-      TF_LITE_ENSURE_EQ(context, stretch_dim, -1);
-      stretch_dim = i;
+      TF_LITE_ENSURE_EQ(context, strech_dim, -1);
+      strech_dim = i;
     } else {
       num_output_elements *= value;
       output_size->data[i] = value;
     }
   }
-  if (stretch_dim != -1) {
-    output_size->data[stretch_dim] = num_input_elements / num_output_elements;
-    num_output_elements *= output_size->data[stretch_dim];
+  if (strech_dim != -1) {
+    output_size->data[strech_dim] = num_input_elements / num_output_elements;
+    num_output_elements *= output_size->data[strech_dim];
   }
 
   TF_LITE_ENSURE_EQ(context, num_input_elements, num_output_elements);
diff --git a/tensorflow/contrib/lite/kernels/reshape_test.cc b/tensorflow/contrib/lite/kernels/reshape_test.cc
index aecbd0399f..0fbcf6e6aa 100644
--- a/tensorflow/contrib/lite/kernels/reshape_test.cc
+++ b/tensorflow/contrib/lite/kernels/reshape_test.cc
@@ -60,7 +60,7 @@ TEST(ReshapeOpTest, TooManyDimensions) {
 
 TEST(ReshapeOpTest, TooManySpecialDimensions) {
   EXPECT_DEATH(ReshapeOpModel({1, 2, 4, 1}, {-1, -1, 2, 4}),
-               "stretch_dim != -1");
+               "strech_dim != -1");
 }
 
 TEST(ReshapeOpTest, SimpleTest) {
diff --git a/tensorflow/contrib/lite/kernels/test_util.cc b/tensorflow/contrib/lite/kernels/test_util.cc
index 0bb28b50b2..373310bd87 100644
--- a/tensorflow/contrib/lite/kernels/test_util.cc
+++ b/tensorflow/contrib/lite/kernels/test_util.cc
@@ -141,8 +141,8 @@ void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
 
 void SingleOpModel::SetCustomOp(
     const string& name, const std::vector<uint8_t>& custom_option,
-    const std::function<TfLiteRegistration*()>& registration) {
-  custom_registrations_[name] = registration;
+    const std::function<TfLiteRegistration*()>& registeration) {
+  custom_registrations_[name] = registeration;
   opcodes_.push_back(
       CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
   operators_.push_back(CreateOperator(
diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
index 42941a97db..508a570e2e 100644
--- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
+++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
@@ -360,7 +360,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existence of only one to get the condition.
+  // check the existense of only one to the get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/memory_planner.h b/tensorflow/contrib/lite/memory_planner.h
index 0294ec815c..5cd6c20850 100644
--- a/tensorflow/contrib/lite/memory_planner.h
+++ b/tensorflow/contrib/lite/memory_planner.h
@@ -34,8 +34,8 @@ class MemoryPlanner {
   // [first_node, last_node].
   virtual TfLiteStatus ExecuteAllocations(int first_node, int last_node) = 0;
 
-  // Invalidates allocations made earlier. This is called when tensors sizes
-  // have changed. All planned allocations remain, but can't be used until
+  // Invalidates allocations made earliers. This is called when tensors sizes
+  // have change. All planned allocations remain, but can't be used until
   // ExecuteAllocations() is called.
   virtual TfLiteStatus ResetAllocations() = 0;
 };
diff --git a/tensorflow/contrib/lite/model.h b/tensorflow/contrib/lite/model.h
index 38eea0e26b..51a622a28d 100644
--- a/tensorflow/contrib/lite/model.h
+++ b/tensorflow/contrib/lite/model.h
@@ -81,7 +81,7 @@ class FlatBufferModel {
       const tflite::Model* model_spec,
       ErrorReporter* error_reporter = DefaultErrorReporter());
 
-  // Releases memory or unmaps mmaped memory.
+  // Releases memory or unmaps mmaped meory.
   ~FlatBufferModel();
 
   // Copying or assignment is disallowed to simplify ownership semantics.
diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
index bd49d327c9..76032771af 100644
--- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
@@ -569,7 +569,7 @@ enum {
   ANEURALNETWORKS_LOGISTIC = 14,
 
   /**
-   * Projects an input to a bit vector via locality sensitive hashing.
+   * Projects an input to a bit vector via locality senstive hashing.
    *
    * Inputs:
    * * 0: Hash functions. Dim.size == 2, DataType: Float.
diff --git a/tensorflow/contrib/lite/rpi_makefile.inc b/tensorflow/contrib/lite/rpi_makefile.inc
deleted file mode 100644
index 832ef5824b..0000000000
--- a/tensorflow/contrib/lite/rpi_makefile.inc
+++ /dev/null
@@ -1,33 +0,0 @@
-# Settings for Raspberry Pi.
-ifeq ($(TARGET), RPI)
-	ifeq ($(TARGET_ARCH), armv7)
-		CXXFLAGS += \
-			-march=armv7-a \
-			-mfpu=neon-vfpv4 \
-			-funsafe-math-optimizations \
-			-ftree-vectorize
-
-		CCFLAGS += \
-			-march=armv7-a \
-			-mfpu=neon-vfpv4 \
-			-funsafe-math-optimizations \
-			-ftree-vectorize
-
-		LDFLAGS := \
-			-Wl,--no-export-dynamic \
-			-Wl,--exclude-libs,ALL \
-			-Wl,--gc-sections \
-			-Wl,--as-needed
-	endif
-
-	LIBS := \
-	-lstdc++ \
-	-lpthread \
-	-lm \
-	-ldl
-
-	OBJDIR := $(OBJDIR)rpi_$(TARGET_ARCH)/
-	LIBDIR := $(LIBDIR)rpi_$(TARGET_ARCH)/
-	BINDIR := $(BINDIR)rpi_$(TARGET_ARCH)/
-	DEPDIR := $(DEPDIR)rpi_$(TARGET_ARCH)/
-endif
diff --git a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
index 640972de77..08bcfe4516 100644
--- a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
+++ b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
@@ -46,7 +46,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real builtin
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
 // ops.
 typedef enum {
 )";
diff --git a/tensorflow/contrib/lite/simple_memory_arena.cc b/tensorflow/contrib/lite/simple_memory_arena.cc
index 2f2004f56b..4aab244989 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.cc
+++ b/tensorflow/contrib/lite/simple_memory_arena.cc
@@ -113,21 +113,21 @@ TfLiteStatus SimpleMemoryArena::Commit(TfLiteContext* context) {
     underlying_buffer_size_ = required_size;
     underlying_buffer_aligned_ptr_ = new_underlying_buffer_aligned_ptr;
   }
-  committed_ = true;
+  commited_ = true;
   return underlying_buffer_ != nullptr ? kTfLiteOk : kTfLiteError;
 }
 
 TfLiteStatus SimpleMemoryArena::ResolveAlloc(TfLiteContext* context,
                                              const ArenaAlloc& alloc,
                                              char** output_ptr) {
-  TF_LITE_ENSURE(context, committed_);
+  TF_LITE_ENSURE(context, commited_);
   TF_LITE_ENSURE(context, output_ptr != nullptr);
   *output_ptr = underlying_buffer_aligned_ptr_ + alloc.offset;
   return kTfLiteOk;
 }
 
 TfLiteStatus SimpleMemoryArena::Clear() {
-  committed_ = false;
+  commited_ = false;
   high_water_mark_ = 0;
   allocs_.clear();
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/simple_memory_arena.h b/tensorflow/contrib/lite/simple_memory_arena.h
index 5faf78b59e..0535522374 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.h
+++ b/tensorflow/contrib/lite/simple_memory_arena.h
@@ -22,7 +22,7 @@ limitations under the License.
 namespace tflite {
 
 // This little structure holds the offset and the size for a dynamic memory
-// allocation in the memory arena. When the arena is committed and the
+// allocation in the memory arena. When the arena is commited and the
 // underlying buffer is set, the alloc can be resolved into an actual memory
 // pointer.
 struct ArenaAlloc {
@@ -43,7 +43,7 @@ struct ArenaAlloc {
 class SimpleMemoryArena {
  public:
   explicit SimpleMemoryArena(size_t arena_alignment)
-      : committed_(false),
+      : commited_(false),
         arena_alignment_(arena_alignment),
         high_water_mark_(0),
         underlying_buffer_size_(0),
@@ -73,7 +73,7 @@ class SimpleMemoryArena {
   }
 
  private:
-  bool committed_;
+  bool commited_;
   size_t arena_alignment_;
   size_t high_water_mark_;
   std::unique_ptr<char[]> underlying_buffer_;
diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD
index 0a6edc33c5..8ca03f4193 100644
--- a/tensorflow/contrib/lookup/BUILD
+++ b/tensorflow/contrib/lookup/BUILD
@@ -46,7 +46,6 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     grpc_enabled = True,
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 filegroup(
diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md
index 6c3b02e12b..995230dfa8 100644
--- a/tensorflow/contrib/makefile/README.md
+++ b/tensorflow/contrib/makefile/README.md
@@ -194,8 +194,6 @@ with:
 srcs = glob(["libs/arm64-v8a/*.so"]),
 ```
 
-If you are building for Android TV (Shield TV devices), replace "portrait" with "landscape" for android:screenOrientation in all four activities in tensorflow/examples/android/AndroidManifest.xml
-
 Then run:
 ```bash
 # Create dir for native libs
diff --git a/tensorflow/contrib/makefile/build_all_ios.sh b/tensorflow/contrib/makefile/build_all_ios.sh
index 9b148688c4..2d99791839 100755
--- a/tensorflow/contrib/makefile/build_all_ios.sh
+++ b/tensorflow/contrib/makefile/build_all_ios.sh
@@ -80,9 +80,10 @@ if [[ ! -z "${OPTIMIZE_FOR_GRAPH}" ]]; then
         fi
     else
         echo "${PRNT_SLCTV_BIN} found. Using it"
+        ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
+
     fi
 
-    ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
 fi
 
 if [[ "${ONLY_MAKE_TENSORFLOW}" != "true" ]]; then
diff --git a/tensorflow/contrib/mpi/mpi_utils.h b/tensorflow/contrib/mpi/mpi_utils.h
index df055ff567..fa297c28cb 100644
--- a/tensorflow/contrib/mpi/mpi_utils.h
+++ b/tensorflow/contrib/mpi/mpi_utils.h
@@ -24,8 +24,6 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/str_util.h"
 
-// Skip MPI C++ bindings support, this matches the usage in other places
-#define OMPI_SKIP_MPICXX
 #include "third_party/mpi/mpi.h"
 #define MPI_CHECK(cmd)                                                \
   do {                                                                \
diff --git a/tensorflow/contrib/predictor/predictor_factories.py b/tensorflow/contrib/predictor/predictor_factories.py
index 6e77e934fe..04b5d5bdf1 100644
--- a/tensorflow/contrib/predictor/predictor_factories.py
+++ b/tensorflow/contrib/predictor/predictor_factories.py
@@ -53,7 +53,7 @@ def from_contrib_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, core_estimator.Estimator):
-    raise TypeError('Expected estimator to be of type '
+    raise TypeError('Espected estimator to be of type '
                     'tf.contrib.learn.Estimator, but got type '
                     'tf.python.estimator.Estimator. You likely want to call '
                     'from_estimator.')
@@ -88,7 +88,7 @@ def from_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, contrib_estimator.Estimator):
-    raise TypeError('Expected estimator to be of type '
+    raise TypeError('Espected estimator to be of type '
                     'tf.python.estimator.Estimator, but got type '
                     'tf.contrib.learn.Estimator. You likely want to call '
                     'from_contrib_estimator.')
diff --git a/tensorflow/contrib/py2tf/converters/BUILD b/tensorflow/contrib/py2tf/converters/BUILD
index 4bb6f76019..f624c42686 100644
--- a/tensorflow/contrib/py2tf/converters/BUILD
+++ b/tensorflow/contrib/py2tf/converters/BUILD
@@ -81,7 +81,6 @@ py_test(
     name = "builtin_functions_test",
     srcs = ["builtin_functions_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/python:client_testlib",
@@ -92,7 +91,6 @@ py_test(
     name = "call_trees_test",
     srcs = ["call_trees_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/contrib/py2tf/impl",
diff --git a/tensorflow/contrib/py2tf/converters/single_return.py b/tensorflow/contrib/py2tf/converters/single_return.py
index 1194b98f5e..90bc22008f 100644
--- a/tensorflow/contrib/py2tf/converters/single_return.py
+++ b/tensorflow/contrib/py2tf/converters/single_return.py
@@ -212,7 +212,7 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
 
   def __init__(self):
     self.cant_return = False
-    super(DetectReturnInUnsupportedControlFlow, self).__init__()
+    super(gast.NodeVisitor, self).__init__()
 
   def visit_While(self, node):
     self.cant_return = True
diff --git a/tensorflow/contrib/py2tf/utils/BUILD b/tensorflow/contrib/py2tf/utils/BUILD
index 8bc338e801..d029289f5a 100644
--- a/tensorflow/contrib/py2tf/utils/BUILD
+++ b/tensorflow/contrib/py2tf/utils/BUILD
@@ -83,7 +83,6 @@ py_test(
     name = "py_func_test",
     srcs = ["py_func_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":utils",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index 1afcbb8504..b278265639 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -237,7 +237,7 @@ def _FindFusedBatchNorms(graph):
       # The batch variance used during forward and backward prop is biased,
       # i.e it is calculated as: V=sum(x(k)-mu)^2/N. For the moving average
       # calculation, the variance is corrected by the term N/N-1 (Bessel's
-      # correction). The variance tensor read from FuseBatchNorm has Bessel's
+      # correction). The variance tensor read from FuseBatchNorm has bessel's
       # correction applied, so we undo it here.
       scope, sep, _ = bn_op.name.rpartition('/')
       g = ops.get_default_graph()
@@ -306,7 +306,7 @@ def _ComputeBatchNormCorrections(context, match, freeze_batch_norm_delay,
 
   Args:
     context: The scope under which we look for batch norm params
-    match: Object containing required batch norm tensors for correction
+    match: Object containg required batch norm tensors for correction
       computation.
     freeze_batch_norm_delay: Delay in steps at which computation switches
       from regular batch norm to frozen mean and variance.
diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py
index a4f7b1b221..0a8e35080c 100644
--- a/tensorflow/contrib/quantize/python/quant_ops.py
+++ b/tensorflow/contrib/quantize/python/quant_ops.py
@@ -282,8 +282,8 @@ def _FakeQuantWithMinMaxVars(inputs, min_var, max_var, per_channel, num_bits,
   Args:
     inputs: a tensor containing values to be quantized.
     min_var: a variable containing quantization range lower end(s).
-    max_var: a variable containing quantization range upper end(s).
-    per_channel: a boolean specifying whether to use per-channel quantization.
+    max_var: a variable containing quantization range lupper end(s).
+    per_channel: a boolean specifying whether to use per-channel quantizatioh.
     num_bits: Number of bits to use for quantization, must be between 2 and 8.
     narrow_range: Whether to use the narrow quantization range
       [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1].
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index ec721afbc8..0608ab9302 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -267,7 +267,7 @@ def _InsertQuantOp(context,
   """Inserts a quant op between a producer op and (multiple) consumer ops.
 
   Args:
-    context: Context where producer and consumer operations are nested.
+    context: Context w,here producer and consumer operations are nested.
     name: Name for the new quantization op within the context.
     producer: Producer operation of the pairs where quantization will be
       inserted.
diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index 5abdcd2475..5a3a74cec4 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -158,7 +158,7 @@ def experimental_create_training_graph(input_graph=None,
   often fail.
 
   Args:
-    input_graph: The tf.Graph to be transformed, if None then defaults to the
+    input_graph: The tf.Graph to be transformed,if None then defaults to the
       default graph.
     weight_bits: Number of bits to use for quantizing weights.
     activation_bits: Number of bits to use for quantizing activations.
diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
index db745aa562..0624cc878b 100644
--- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
@@ -419,7 +419,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optional) and an activation.
+      # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -470,7 +470,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optional) and an activation.
+      # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -526,7 +526,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optional) and an activation.
+      # Manually add a bypass (optionaly) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -565,7 +565,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initializes with a truncated normal variable.
+      An initialized that initialzes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index b2e5707a6d..ef59475167 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -144,7 +144,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initializes with a truncated normal variable.
+      An initialized that initialzes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
index 54c66271cd..27f0a7f58f 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD
+++ b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
@@ -38,6 +38,7 @@ py_test(
     size = "small",
     srcs = ["python/ops/remote_fused_graph_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":remote_fused_graph_ops_py",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 358b2eb02b..73f2607d84 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2133,7 +2133,7 @@ class Conv1DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_1d_lstm_cell", **kwargs):
     """Construct Conv1DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, name=name, **kwargs)
+    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, **kwargs)
 
 
 class Conv2DLSTMCell(ConvLSTMCell):
@@ -2144,7 +2144,7 @@ class Conv2DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_2d_lstm_cell", **kwargs):
     """Construct Conv2DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, name=name, **kwargs)
+    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, **kwargs)
 
 
 class Conv3DLSTMCell(ConvLSTMCell):
@@ -2155,7 +2155,7 @@ class Conv3DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_3d_lstm_cell", **kwargs):
     """Construct Conv3DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, name=name, **kwargs)
+    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, **kwargs)
 
 
 def _conv(args, filter_size, num_features, bias, bias_start=0.0):
diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index b10757df47..245fe07f2b 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -53,7 +53,6 @@ py_test(
     size = "small",
     srcs = ["python/saved_model/reader_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":saved_model_py",
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 6adbb8be40..03fe31abf7 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -299,13 +299,12 @@ class BeamSearchDecoder(decoder.Decoder):
     """
     finished, start_inputs = self._finished, self._start_inputs
 
-    dtype = nest.flatten(self._initial_cell_state)[0].dtype
     log_probs = array_ops.one_hot(  # shape(batch_sz, beam_sz)
         array_ops.zeros([self._batch_size], dtype=dtypes.int32),
         depth=self._beam_width,
-        on_value=ops.convert_to_tensor(0.0, dtype=dtype),
-        off_value=ops.convert_to_tensor(-np.Inf, dtype=dtype),
-        dtype=dtype)
+        on_value=0.0,
+        off_value=-np.Inf,
+        dtype=nest.flatten(self._initial_cell_state)[0].dtype)
 
     initial_state = BeamSearchDecoderState(
         cell_state=self._initial_cell_state,
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 3ad88a8a22..67011c8fef 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -165,7 +165,6 @@ py_test(
     name = "gc_test",
     srcs = ["gc_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":gc",
diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD
index 7aa1684839..5daabbd62e 100644
--- a/tensorflow/contrib/slim/python/slim/data/BUILD
+++ b/tensorflow/contrib/slim/python/slim/data/BUILD
@@ -61,7 +61,6 @@ py_test(
     name = "dataset_data_provider_test",
     srcs = ["dataset_data_provider_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":dataset",
         ":dataset_data_provider",
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 07b6b1f142..1e4cc3f095 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -553,6 +553,7 @@ py_test(
     srcs = ["client/random_forest_test.py"],
     srcs_version = "PY2AND3",
     tags = [
+        "no_windows",
         "nomac",  # b/63258195
         "notsan",
     ],
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index db2e000ef8..d833744d0c 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -9,7 +9,6 @@ exports_files(["LICENSE"])
 
 # For platform specific build config
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
-load("//tensorflow:tensorflow.bzl", "py_test")
 
 tf_proto_library(
     name = "protos_all",
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 906cc3f034..c832c6f2e0 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -83,7 +83,6 @@ cc_library(
         "kernels/trt_engine_op.h",
     ],
     copts = tf_copts(),
-    visibility = ["//visibility:public"],
     deps = [
         ":trt_logging",
         ":trt_resources",
@@ -155,7 +154,6 @@ py_library(
     deps = [
         ":trt_convert_py",
         ":trt_ops_py",
-        "//tensorflow/python:errors",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index 461e627e99..dfcce0fd00 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -2,8 +2,7 @@ Using TensorRT in TensorFlow
 ============================
 
 This module provides necessary bindings and introduces TRT_engine_op
-operator that wraps a subgraph in TensorRT. This is still a work in progress
-but should be useable with most common graphs.
+operator that wraps a subgraph in TensorRT.
 
 Compilation
 -----------
@@ -16,10 +15,26 @@ configure script should find the necessary components from the system
 automatically. If installed from tar packages, user has to set path to
 location where the library is installed during configuration.
 
-```shell
+
+```
 bazel build --config=cuda --config=opt //tensorflow/tools/pip_package:build_pip_package
 bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use can be found in test/test_tftrt.py directory
+will be available. An example use is shown below.
+
+```python
+import tensorflow as tf
+import tensorflow.contrib.tensorrt as trt
+#... create and train or load model
+gdef = sess.graph.as_graph_def()
+trt_gdef = trt.create_inference_graph(
+    gdef, #original graph_def
+    ["output"], #name of output node(s)
+    max_batch_size, #maximum batch size to run the inference
+    max_workspace_size_bytes) # max memory for TensorRT to use
+tf.reset_default_graph()
+tf.import_graph_def(graph_def=trt_gdef)
+#...... run inference
+```
diff --git a/tensorflow/contrib/tensorrt/__init__.py b/tensorflow/contrib/tensorrt/__init__.py
index 140ad48282..fd551d70b4 100644
--- a/tensorflow/contrib/tensorrt/__init__.py
+++ b/tensorflow/contrib/tensorrt/__init__.py
@@ -18,18 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.python.framework import errors
-
-# pylint: disable=unused-import,wildcard-import,g-import-not-at-top
-try:
-  from tensorflow.contrib.tensorrt.python import *
-except errors.NotFoundError as e:
-  no_trt_message = (
-      '**** Failed to initialize TensorRT. This is either because the TensorRT'
-      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
-      ' it installed. If not installed, please go to'
-      ' https://developer.nvidia.com/tensorrt to download and install'
-      ' TensorRT ****')
-  print(no_trt_message)
-  raise e
-# pylint: enable=unused-import,wildcard-import,g-import-not-at-top
+# pylint: disable=unused-import,wildcard-import
+from tensorflow.contrib.tensorrt.python import *
+# pylint: enable=unused-import,wildcard-import
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index eea8c8efa2..970f810473 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -15,7 +15,6 @@ limitations under the License.
 
 #include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
 
-#include <list>
 #include <map>
 #include <set>
 #include <unordered_map>
@@ -49,29 +48,13 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+static bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
-      "Identity",
-      "Const",
-      "Conv2D",
-      "MaxPool",
-      "BiasAdd",
-      "Relu",
-      "Add",
-      "Mul",
-      "Sub",
-      "Rsqrt",
-      "Pad",
-      "Mean",
-      "AvgPool",
-      "ConcatV2",
-      "DepthwiseConv2dNative",
-      "FusedBatchNorm",
-      "FusedBatchNormV2",
-      // TODO(ben,jie): ...
+      "Identity", "Const", "Conv2D", "MaxPool", "BiasAdd", "Relu",
+      "Add",      "Mul",   "Sub",    "Rsqrt",   "Pad"  // "Placeholder" ,"Mean"
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
   return candidate_ops.count(node_def.op());
@@ -86,8 +69,6 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
       if (!subgraph_node_ids.count(edge->src()->id()) &&
           !edge->src()->IsSource()) {
         incoming_edges->insert(edge);
-      } else {
-        VLOG(2) << edge->src()->name() << " N, ";
       }
     }
   }
@@ -101,10 +82,7 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
           !edge->dst()->IsSink()) {
-        VLOG(2) << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
-      } else {
-        VLOG(2) << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -131,150 +109,74 @@ std::unordered_map<string, std::vector<int>> BuildTensorNameMap(
   }
   return result;
 }
-// TODO(sami): convert references to pointers
-struct ConvertGraphParams {
-  ConvertGraphParams(
-      tensorflow::Graph& inp_graph,
-      const std::vector<string>& output_node_names,
-      const std::set<int>& subgraph_node_id_numbers,
-      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
-      const tensorflow::grappler::GraphProperties& current_graph_properties,
-      std::unordered_map<string, std::pair<int, string>>* output_edges,
-      int engine_precision_mode)
-      : graph(inp_graph),
-        output_names(output_node_names),
-        subgraph_node_ids(subgraph_node_id_numbers),
-        max_batch_size(max_supported_batch_size),
-        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
-        graph_properties(current_graph_properties),
-        output_edge_map(output_edges),
-        precision_mode(engine_precision_mode) {}
-  tensorflow::Graph& graph;
-  const std::vector<string>& output_names;
-  const std::set<int>& subgraph_node_ids;
-  size_t max_batch_size;
-  size_t max_workspace_size_bytes;
-  const tensorflow::grappler::GraphProperties& graph_properties;
-  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
-  int precision_mode;
-  std::vector<std::pair<int, int>> subgraph_inputs;
-  std::vector<std::pair<int, int>> subgraph_outputs;
+
+tensorflow::Status ConvertSubGraphToTensorRT(
+    const std::vector<string>& output_names,
+    const std::set<int>& subgraph_node_ids,
+    size_t max_batch_size,  // Max batch size that engine will be created for
+    // Max amount of memory that engine will be allowed to consume, in bytes
+    size_t max_workspace_size_bytes,
+    const tensorflow::grappler::GraphProperties& graph_properties,
+    tensorflow::Graph* graph) {
   tensorflow::EdgeSet subgraph_incoming_edges;
-  tensorflow::EdgeSet subgraph_outgoing_edges;
-};
+  GetSubGraphIncomingEdges(*graph, subgraph_node_ids, &subgraph_incoming_edges);
+
+  std::vector<std::pair<int, int>> subgraph_inputs;
 
-static tensorflow::Status FillSubGraphEdgeSets(ConvertGraphParams* p) {
-  GetSubGraphIncomingEdges(p->graph, p->subgraph_node_ids,
-                           &p->subgraph_incoming_edges);
-  for (const tensorflow::Edge* edge : p->subgraph_incoming_edges) {
-    p->subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
+  // Collect inputs by looking for incoming edges
+  for (const tensorflow::Edge* edge : subgraph_incoming_edges) {
+    subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
   }
-  auto output_name_to_index_map = BuildTensorNameMap(p->output_names);
   std::set<std::pair<int, int>> subgraph_outputs_set;
   // Collect outputs referenced from output_names
-  for (int node_id : p->subgraph_node_ids) {
-    tensorflow::Node* node = p->graph.FindNodeId(node_id);
+  auto output_name_to_index_map = BuildTensorNameMap(output_names);
+  for (int node_id : subgraph_node_ids) {
+    tensorflow::Node* node = graph->FindNodeId(node_id);
     if (output_name_to_index_map.count(node->name())) {
       for (int index : output_name_to_index_map.at(node->name())) {
         subgraph_outputs_set.insert({node_id, index});
       }
     }
   }
-  GetSubGraphOutgoingEdges(p->graph, p->subgraph_node_ids,
-                           &p->subgraph_outgoing_edges);
-  for (const tensorflow::Edge* edge : p->subgraph_outgoing_edges) {
+  // Collect outputs referenced from outgoing edges
+  tensorflow::EdgeSet subgraph_outgoing_edges;
+  GetSubGraphOutgoingEdges(*graph, subgraph_node_ids, &subgraph_outgoing_edges);
+  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
     subgraph_outputs_set.insert({edge->src()->id(), edge->src_output()});
   }
-  p->subgraph_outputs.reserve(subgraph_outputs_set.size());
-  p->subgraph_outputs.insert(p->subgraph_outputs.begin(),
-                             subgraph_outputs_set.begin(),
-                             subgraph_outputs_set.end());
-  return tensorflow::Status::OK();
-};
-
-tensorflow::Status GetCalibNode(ConvertGraphParams* params) {
-  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
+  // Impose an ordering on the outputs
+  std::vector<std::pair<int, int>> subgraph_outputs(
+      subgraph_outputs_set.begin(), subgraph_outputs_set.end());
+  // Build TensorRT node and add it to the graph
   tensorflow::NodeDef trt_node_def;
-  SubGraphParams s(params->graph, params->subgraph_node_ids,
-                   params->subgraph_inputs, params->subgraph_outputs,
-                   params->max_batch_size, params->max_workspace_size_bytes,
-                   params->graph_properties, params->output_edge_map,
-                   &trt_node_def, params->precision_mode);
-  TF_RETURN_IF_ERROR(InjectCalibrationNode(s));
+  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(
+      *graph, subgraph_node_ids, subgraph_inputs, subgraph_outputs,
+      max_batch_size, max_workspace_size_bytes, graph_properties,
+      &trt_node_def));
   tensorflow::Status status;
-  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
-
-  TF_RETURN_IF_ERROR(status);
-
-  for (auto in_edge :
-       params->subgraph_incoming_edges) {  // loop over incoming edges and
-                                           // attach them to calib node
-    // tensorflow::Node* src_node = in_edge->src();
-    auto src_output = in_edge->src_output();
-    auto dst_node = in_edge->dst();
-    auto dst_input = in_edge->dst_input();
-    VLOG(1) << " update edge " << trt_node->name() << ":" << src_output
-            << " -> " << dst_node->name() << ":" << dst_input;
-    TF_RETURN_IF_ERROR(
-        params->graph.UpdateEdge(trt_node, src_output, dst_node, dst_input));
-  }
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertSubGraphToTensorRT(ConvertGraphParams* params) {
-  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
-  tensorflow::NodeDef trt_node_def;
-
-  SubGraphParams s(params->graph, params->subgraph_node_ids,
-                   params->subgraph_inputs, params->subgraph_outputs,
-                   params->max_batch_size, params->max_workspace_size_bytes,
-                   params->graph_properties, params->output_edge_map,
-                   &trt_node_def, params->precision_mode);
-  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(s));
-  tensorflow::Status status;
-  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
-
-  // AddNode does not wire edges.
-  // Re-map incoming edges to use the new TRT node instead of the orig subgraph
-  std::map<std::pair<int, int>, int> subgraph_edge_to_input_map;
-  for (size_t i = 0; i < params->subgraph_inputs.size(); ++i) {
-    subgraph_edge_to_input_map.insert({params->subgraph_inputs.at(i), i});
-  }
-  for (const tensorflow::Edge* edge : params->subgraph_incoming_edges) {
-    std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
-    int new_src_output = subgraph_edge_to_input_map.at(old_src);
-    params->graph.AddEdge(edge->src(), edge->src_output(), trt_node,
-                          new_src_output);
-    params->graph.RemoveEdge(edge);
-  }
-
-  VLOG(2) << "new wiring edges: " << trt_node->in_edges().size();
-  for (const tensorflow::Edge* edge : trt_node->in_edges()) {
-    VLOG(2) << edge->src()->name() << " port: " << edge->src_output();
-  }
-
+  tensorflow::Node* trt_node = graph->AddNode(trt_node_def, &status);
   TF_RETURN_IF_ERROR(status);
 
   // Re-map outgoing edges to use the new TRT node instead of the orig subgraph
   std::map<std::pair<int, int>, int> subgraph_edge_to_output_map;
-  for (size_t i = 0; i < params->subgraph_outputs.size(); ++i) {
-    subgraph_edge_to_output_map.insert({params->subgraph_outputs.at(i), i});
+  for (size_t i = 0; i < subgraph_outputs.size(); ++i) {
+    subgraph_edge_to_output_map.insert({subgraph_outputs.at(i), i});
   }
   TF_RETURN_IF_ERROR(status);
-  for (const tensorflow::Edge* edge : params->subgraph_outgoing_edges) {
+  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
     std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
     int new_src_output = subgraph_edge_to_output_map.at(old_src);
-    TF_RETURN_IF_ERROR(params->graph.UpdateEdge(
-        trt_node, new_src_output, edge->dst(), edge->dst_input()));
+    TF_RETURN_IF_ERROR(graph->UpdateEdge(trt_node, new_src_output, edge->dst(),
+                                         edge->dst_input()));
   }
   // Remove the original subgraph
-  for (int node_id : params->subgraph_node_ids) {
-    tensorflow::Node* node = params->graph.FindNodeId(node_id);
+  for (int node_id : subgraph_node_ids) {
+    tensorflow::Node* node = graph->FindNodeId(node_id);
     // Don't remove the input placeholders
     if (node->type_string() == "Placeholder") {
       continue;
     }
-    params->graph.RemoveNode(node);
+    graph->RemoveNode(node);
   }
   return tensorflow::Status::OK();
 }
@@ -292,39 +194,12 @@ tensorflow::Status BuildNodeMap(
 }
 
 }  // namespace
-tensorflow::Status ConvertCalibGraphToInferGraph(
-    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph) {
-  VLOG(0) << "Starting Calib Conversion";
-  tensorflow::Graph graph(tensorflow::OpRegistry::Global());
-  TF_RETURN_IF_ERROR(tensorflow::ConvertGraphDefToGraph(
-      tensorflow::GraphConstructorOptions(), graph_def, &graph));
-  //  get calib nodes
-  std::vector<tensorflow::Node*> calib_nodes;
-  for (auto node : graph.op_nodes()) {
-    if (node->type_string() == "TRTCalibOp") {
-      VLOG(1) << "Found Calib Node";
-      calib_nodes.push_back(node);
-    }
-  }
-  VLOG(0) << "Num Calib nodes in graph= " << calib_nodes.size();
-  if (calib_nodes.size() == 0)
-    return tensorflow::errors::FailedPrecondition(
-        "Graph doesn't contain any calibration nodes!."
-        " Please generate calibration graph and run calibration first");
-  for (auto n : calib_nodes) {
-    TF_RETURN_IF_ERROR(
-        tensorrt::convert::ConvertCalibrationNodeToEngineNode(graph, n));
-  }
-  graph.ToGraphDef(infer_graph);
-  return tensorflow::Status::OK();
-}
 
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
-    int precision_mode = FP32MODE, int minimum_segment_size = 3) {
-  // optimization pass
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def) {
+  // Optimization pass
   tensorflow::grappler::GrapplerItem item;
   item.fetch = output_names;
   tensorflow::GraphDef gdef;
@@ -334,23 +209,16 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   tensorflow::grappler::LayoutOptimizer optimizer;
   tensorflow::grappler::Cluster* cluster;
 
-  // virtual cluster
+  // Virtual cluster
   tensorflow::DeviceProperties device_properties;
-
   device_properties.set_type("GPU");
   device_properties.mutable_environment()->insert({"architecture", "6"});
   cluster =
       new tensorflow::grappler::VirtualCluster({{"/GPU:0", device_properties}});
 
-  // single machine
-  int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
-  int num_gpus = tensorflow::grappler::GetNumAvailableGPUs();
-  VLOG(2) << "cpu_cores: " << num_cpu_cores;
-  VLOG(2) << "gpus: " << num_gpus;
-
   TF_RETURN_IF_ERROR(optimizer.Optimize(cluster, item, &gdef));
 
-  // constant folding
+  // Constant folding
   item.graph = gdef;
   tensorflow::grappler::ConstantFolding fold(nullptr);
   TF_RETURN_IF_ERROR(fold.Optimize(nullptr, item, &gdef));
@@ -358,6 +226,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   // AJ refactoring shape inference through grappler/GraphProperties.
   tensorflow::grappler::GraphProperties static_graph_properties(item);
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(false));
+
   // Build full graph
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
                                              gdef.library());
@@ -374,7 +243,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
 
   // TODO(sami): this should be passed as a knob!!!!
-  segment_options.minimum_segment_size = minimum_segment_size;
+  segment_options.minimum_segment_size = 2;
   tensorflow::tensorrt::segment::SegmentNodesVector segments;
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       gdef, IsTensorRTCandidate, segment_options, &segments));
@@ -383,37 +252,14 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
   std::unordered_map<string, tensorflow::Node*> node_map;
   TF_RETURN_IF_ERROR(BuildNodeMap(graph, &node_map));
-  std::unordered_map<string, std::pair<int, string>> output_edge_map;
-  int count = 0;
-  float total_num_nodes_in_segments = 0.;
-  for (auto s : segments) {
-    total_num_nodes_in_segments += s.size();
-  }
   for (const std::set<string>& subgraph_node_names : segments) {
     std::set<int> subgraph_node_ids;
-    size_t max_mem_per_engine =
-        max_workspace_size_bytes *
-        ((float)subgraph_node_names.size() / total_num_nodes_in_segments);
-    std::stringstream oss;
     for (const string& node_name : subgraph_node_names) {
-      oss << " " << node_name;
       subgraph_node_ids.insert(node_map.at(node_name)->id());
     }
-    VLOG(2) << "Subgraph nodes" << oss.str();
-    ConvertGraphParams p(graph, output_names, subgraph_node_ids, max_batch_size,
-                         max_mem_per_engine, static_graph_properties,
-                         &output_edge_map, precision_mode);
-    if (precision_mode == INT8MODE) {
-      TF_RETURN_IF_ERROR(GetCalibNode(&p));
-    } else {
-      tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
-      if (status != tensorflow::Status::OK()) {
-        LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \n"
-                     << status.ToString() << " SKIPPING......";
-      }
-      count++;
-    }
+    TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRT(
+        output_names, subgraph_node_ids, max_batch_size,
+        max_workspace_size_bytes, static_graph_properties, &graph));
   }
   graph.ToGraphDef(new_graph_def);
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index e1596e89e2..154ad3f2e8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -28,11 +28,6 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-// This method converts an already generated calibration graph which was used in
-// calibration runs to an inference graph
-tensorflow::Status ConvertCalibGraphToInferGraph(
-    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* new_graph_def);
-
 // max_batch_size: maximum batch size which can be used for inference for
 //                 optimization targets inference run with max batch size.
 // max_workspace_size_bytes: The upper bound of memory allowence for
@@ -40,8 +35,7 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
-    int precision_mode, int minimum_segment_size);
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def);
 
 }  // namespace convert
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 75a3c3d034..9ee717dd7f 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -24,10 +24,6 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
-#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
-#include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"  // NOLINT
 #include "tensorflow/core/framework/types.h"
@@ -36,7 +32,6 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tensor_coding.h"
@@ -44,6 +39,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
+#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorrt/include/NvInfer.h"
 
 //  Check if the types are equal. Cast to int first so that failure log message
@@ -53,7 +49,6 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
-using ::tensorflow::strings::StrCat;
 
 namespace {
 
@@ -70,8 +65,7 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
       *trt_dtype = nvinfer1::DataType::kHALF;
       break;
     default:
-      return tensorflow::errors::InvalidArgument(
-          "Unsupported data type " + tensorflow::DataTypeString(tf_dtype));
+      return tensorflow::errors::InvalidArgument("Unsupported data type");
   }
   return tensorflow::Status::OK();
 }
@@ -118,18 +112,6 @@ static std::vector<std::pair<int, int>> CreateSamePadding(
   return padding;
 }
 
-string GetCommonNameScope(const string& op_name_a, const string& op_name_b) {
-  size_t last_scope_separator = 0;
-  for (size_t i = 0; i < std::min(op_name_a.size(), op_name_b.size()); ++i) {
-    if (op_name_a[i] != op_name_b[i]) {
-      break;
-    } else if (op_name_a[i] == '/') {
-      last_scope_separator = i + 1;
-    }
-  }
-  return op_name_a.substr(0, last_scope_separator);
-}
-
 class TRT_ShapedWeights {
  public:
   TRT_ShapedWeights(tensorflow::DataType type, const void* values,
@@ -262,11 +244,6 @@ std::vector<int> TFAttrs::get<std::vector<int>>(string key) const {
   return std::vector<int>(attr.begin(), attr.end());
 }
 
-template <>
-std::vector<string> TFAttrs::get<std::vector<string>>(string key) const {
-  auto attr = this->at(key)->list().s();
-  return std::vector<string>(attr.begin(), attr.end());
-}
 template <>
 nvinfer1::Dims TFAttrs::get<nvinfer1::Dims>(string key) const {
   auto values = this->get<std::vector<int>>(key);
@@ -289,17 +266,6 @@ tensorflow::DataType TFAttrs::get<tensorflow::DataType>(string key) const {
   return this->at(key)->type();
 }
 
-template <>
-float TFAttrs::get<float>(string key) const {
-  return this->at(key)->f();
-}
-
-template <>
-bool TFAttrs::get<bool>(string key) const {
-  return this->at(key)->b();
-}
-
-// TODO(jie): reorder4 & reorder2 should be merged?
 template <typename T>
 void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
               nvinfer1::DimsNCHW istrides, T* odata,
@@ -317,87 +283,29 @@ void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
   }
 }
 
-template <typename T>
-void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides,
-              T* odata, nvinfer1::DimsHW ostrides) {
-  for (int h = 0; h < shape.h(); ++h) {
-    for (int w = 0; w < shape.w(); ++w) {
-      odata[h * ostrides.h() + w * ostrides.w()] =
-          idata[h * ostrides.h() + w * ostrides.w()];
-    }
-  }
-}
-
-// TODO(jie): fallback to tensorflow!!
-void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
-                   TRT_ShapedWeights* oweights) {
-  int c = iweights.shape_.d[0];
-  int k = iweights.shape_.d[1];
-  oweights->shape_.d[0] = k;
-  oweights->shape_.d[1] = c;
-  nvinfer1::DimsHW istrides = {1, k};
-  nvinfer1::DimsHW ostrides = {c, 1};
-  switch (iweights.type_) {
-    case tensorflow::DataType::DT_FLOAT: {
-      Reorder2({k, c}, static_cast<float const*>(iweights.GetValues()),
-               istrides,
-               static_cast<float*>(const_cast<void*>(oweights->GetValues())),
-               ostrides);
-      break;
-    }
-    case tensorflow::DataType::DT_HALF: {
-      Reorder2(
-          {k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
-          istrides,
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
-      break;
-    }
-    default:
-      LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
-                 << DataTypeString(iweights.type_);
-  }
-}
-
 void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights,
-                       TRT_ShapedWeights* oweights, int num_groups) {
+                       TRT_ShapedWeights* oweights) {
   CHECK_EQ(iweights.type_, oweights->type_);
   CHECK_EQ(iweights.size_bytes(), oweights->size_bytes());
   int r = iweights.shape_.d[0];
   int s = iweights.shape_.d[1];
-  // TRT requires GKcRS, while TF depthwise has RSCK
-  //   where c=1, C=G
-  VLOG(2) << "num_groups: " << num_groups;
-  int c = iweights.shape_.d[2] / num_groups;
-  VLOG(2) << "c" << iweights.shape_.d[2] << " then " << c;
-  int k = iweights.shape_.d[3] * num_groups;
-  VLOG(2) << "k" << iweights.shape_.d[3] << " then " << k;
-  oweights->shape_.d[0] = k / num_groups;
-  oweights->shape_.d[1] = c * num_groups;
+  int c = iweights.shape_.d[2];
+  int k = iweights.shape_.d[3];
+  oweights->shape_.d[0] = k;
+  oweights->shape_.d[1] = c;
   oweights->shape_.d[2] = r;
   oweights->shape_.d[3] = s;
   nvinfer1::DimsNCHW istrides = {1, k, s * k * c, c * k};
   nvinfer1::DimsNCHW ostrides = {c * r * s, r * s, s, 1};
   switch (iweights.type_) {
-    case tensorflow::DataType::DT_FLOAT: {
+    case tensorflow::DataType::DT_FLOAT:
       Reorder4({k, c, r, s}, static_cast<float const*>(iweights.GetValues()),
                istrides,
                static_cast<float*>(const_cast<void*>(oweights->GetValues())),
                ostrides);
       break;
-    }
-    case tensorflow::DataType::DT_HALF: {
-      Reorder4(
-          {k, c, r, s}, static_cast<Eigen::half const*>(iweights.GetValues()),
-          istrides,
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
-      break;
-    }
-
     default:
-      LOG(FATAL) << "Unsupported type, expected fp32 or fp16 but got "
-                 << DataTypeString(iweights.type_);
+      LOG(FATAL) << "!!!!!!!!!!!!!!!!!!!!!!!!broke!!!!!!!!!!!!";
   }
 }
 
@@ -415,11 +323,12 @@ inline std::shared_ptr<T> infer_object(T* obj) {
   return std::shared_ptr<T>(obj, InferDeleter());
 }
 
+// Logger for GIE info/warning/errors
 class Converter;
 
 using OpConverter =
     std::function<tensorflow::Status(Converter&, const tensorflow::NodeDef&,
-                                     const std::vector<TRT_TensorOrWeights>&,
+                                     std::vector<TRT_TensorOrWeights> const&,
                                      std::vector<TRT_TensorOrWeights>*)>;
 
 class Converter {
@@ -427,57 +336,34 @@ class Converter {
   std::unordered_map<string, OpConverter> op_registry_;
   nvinfer1::INetworkDefinition* trt_network_;
   std::list<std::vector<uint8_t>> temp_bufs_;
-  tensorflow::tensorrt::TRTWeightStore* weight_store_;
-  bool fp16_;
+
   void register_op_converters();
+
   std::vector<TRT_TensorOrWeights> get_inputs(
       const tensorflow::NodeDef& node_def) {
     std::vector<TRT_TensorOrWeights> inputs;
-    for (auto const& input_name : node_def.input()) {
-      /*************************************************************************
-       * TODO(jie) handle case 1) here
-       * Normalizes the inputs and extracts associated metadata:
-       * 1) Inputs can contain a colon followed by a suffix of characters.
-       *    That suffix may be a single number (e.g. inputName:1) or several
-       *    word characters separated from a number by a colon
-       *    (e.g. inputName:foo:1). The
-       *    latter case is used to denote inputs and outputs of functions.
-       * 2) Control dependency inputs contain caret at the beginning and we
-       *    remove this and annotate the edge as a control dependency.
-       ************************************************************************/
-      string name = input_name[0] == '^' ? input_name.substr(1) : input_name;
-      auto first = name.find_first_of(':');
-      if (first != string::npos && first + 2 == name.size() &&
-          name[first + 1] == '0')
-        name.erase(first);
-
-      VLOG(2) << "retrieve input: " << name;
-      if (trt_tensors_.count(name)) {
-        inputs.push_back(trt_tensors_.at(name));
-      } else {
-        LOG(FATAL) << "input: " << name << " not availabled for node at, "
-                   << node_def.name();
-      }
+    for (const auto& input_name : node_def.input()) {
+      VLOG(2) << "Retrieve input: " << input_name;
+      inputs.push_back(trt_tensors_.at(input_name));
     }
     return inputs;
   }
 
  public:
-  explicit Converter(nvinfer1::INetworkDefinition* trt_network,
-                     tensorflow::tensorrt::TRTWeightStore* ws, bool fp16)
-      : trt_network_(trt_network), weight_store_(ws), fp16_(fp16) {
+  explicit Converter(nvinfer1::INetworkDefinition* trt_network)
+      : trt_network_(trt_network) {
     this->register_op_converters();
   }
-  tensorflow::tensorrt::TRTWeightStore* weight_store() { return weight_store_; }
+
   TRT_ShapedWeights get_temp_weights(tensorflow::DataType type,
                                      nvinfer1::Dims shape) {
     TRT_ShapedWeights weights(type, nullptr, shape);
     // TODO(jie): check weights size_bytes. 0 means type error
-    weight_store_->store_.push_back(std::vector<uint8_t>(weights.size_bytes()));
-    weights.SetValues(weight_store_->store_.back().data());
+    temp_bufs_.push_back(std::vector<uint8_t>(weights.size_bytes()));
+    weights.SetValues(temp_bufs_.back().data());
     return weights;
   }
-  bool isFP16() { return fp16_; };
+
   TRT_ShapedWeights get_temp_weights_like(const TRT_ShapedWeights& weights) {
     return this->get_temp_weights(weights.type_, weights.shape_);
   }
@@ -496,7 +382,7 @@ class Converter {
       TRT_TensorOrWeights output = outputs.at(i);
       // TODO(jie): tf protobuf seems to be omitting the :0 suffix
       string output_name = node_def.name();
-      if (i != 0) output_name = StrCat(output_name, ":", i);
+      if (i != 0) output_name = output_name + ":" + std::to_string(i);
       if (output.is_tensor()) {
         output.tensor()->setName(output_name.c_str());
       }
@@ -562,7 +448,7 @@ struct LambdaFactory {
     switch (op) {
       case OP_CATEGORY::RSQRT: {
         VLOG(2) << "RSQRT GETS DONE";
-        return [](T t) -> T { return 1.0 / sqrt(t); };
+        return [](T t) -> T { return 1.0 / std::sqrt(t); };
       }
       case OP_CATEGORY::NEG:
         return [](T t) -> T { return -t; };
@@ -648,22 +534,6 @@ struct LambdaFactory {
   }
 };
 
-template <>
-std::function<Eigen::half(Eigen::half)> LambdaFactory::unary<Eigen::half>() {
-  switch (op) {
-    case OP_CATEGORY::RSQRT: {
-      VLOG(2) << "RSQRT GETS DONE";
-      return [](Eigen::half t) -> Eigen::half {
-        return Eigen::half(1.0 / sqrt(float(t)));
-      };
-    }
-    case OP_CATEGORY::NEG:
-      return [](Eigen::half t) -> Eigen::half { return -t; };
-    default:
-      VLOG(2) << "Not supported op for unary: " << static_cast<int>(op);
-      return nullptr;
-  }
-}
 tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
                                 TRT_ShapedWeights* oweights,
                                 LambdaFactory unary_op) {
@@ -675,14 +545,6 @@ tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
       std::transform(inp, inp + iweights.count(), oup, unary_op.unary<float>());
       break;
     }
-    case tensorflow::DataType::DT_HALF: {
-      auto inp = static_cast<Eigen::half const*>(iweights.GetValues());
-      auto oup =
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
-      std::transform(inp, inp + iweights.count(), oup,
-                     unary_op.unary<Eigen::half>());
-      break;
-    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -726,32 +588,6 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
       }
       break;
     }
-    case tensorflow::DataType::DT_HALF: {
-      auto inp_l = static_cast<const Eigen::half*>(iweights_l.GetValues());
-      auto inp_r = static_cast<const Eigen::half*>(iweights_r.GetValues());
-      auto oup =
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
-
-      if (iweights_l.count() != iweights_r.count()) {
-        // We only supports broadcast of RankZero
-        if (iweights_l.count() == 1) {
-          VLOG(2) << "I bet it is not working!" << (*inp_l);
-          std::transform(inp_r, inp_r + iweights_r.count(), oup,
-                         binary_op.broadcast_l<Eigen::half>(*inp_l));
-        } else if (iweights_r.count() == 1) {
-          VLOG(2) << "I bet it is not working!" << (*inp_r);
-          std::transform(inp_l, inp_l + iweights_l.count(), oup,
-                         binary_op.broadcast_r<Eigen::half>(*inp_r));
-        } else {
-          return tensorflow::errors::Unimplemented(
-              "Binary op with non-rankZero broadcast not supported");
-        }
-      } else {
-        std::transform(inp_l, inp_l + iweights_l.count(), inp_r, oup,
-                       binary_op.binary<Eigen::half>());
-      }
-      break;
-    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -763,7 +599,7 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
 
 tensorflow::Status ConstantFoldUnary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights> const& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input = inputs.at(0).weights();
 
@@ -777,12 +613,13 @@ tensorflow::Status ConstantFoldUnary(
   CHECK_EQ(weights_input.type_,
            TFAttrs(node_def).get<tensorflow::DataType>("T"));
 
+  // Maybe I should do a switch
   LambdaFactory unary_op;
   if (node_def.op() == "Rsqrt") {
     // Compute rsqrt
     unary_op.op = LambdaFactory::OP_CATEGORY::RSQRT;
     auto ret = UnaryCompute(weights_input, &weights_output, unary_op);
-    // Pass the output
+    // PAss the output
     if (ret == tensorflow::Status::OK()) {
       outputs->push_back(TRT_TensorOrWeights(weights_output));
     }
@@ -794,11 +631,11 @@ tensorflow::Status ConstantFoldUnary(
 }
 
 // TODO(jie,ben) broadcast is needed yet not implemented
-// Let's get the simple stuff working first. Maybe we should fall back to TF
+// Let's get the simple stuff working first. Maybe we should fall bakc to TF
 //   approach for constant folding
 tensorflow::Status ConstantFoldBinary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights> const& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input_l = inputs.at(0).weights();
   TRT_ShapedWeights weights_input_r = inputs.at(1).weights();
@@ -811,12 +648,12 @@ tensorflow::Status ConstantFoldBinary(
         "Binary op implicit broadcast not supported: " + node_def.op());
 
   // TODO(jie): constant fold should really fall back to TF.
-  int num_dims = weights_input_l.shape_.nbDims;
+  int nb_dims = weights_input_l.shape_.nbDims;
   nvinfer1::Dims output_shape;
-  output_shape.nbDims = num_dims;
-  VLOG(2) << "nb_dims: " << num_dims
+  output_shape.nbDims = nb_dims;
+  VLOG(2) << "nb_dims: " << nb_dims
           << ", the other: " << weights_input_r.shape_.nbDims;
-  for (int i = 0; i < num_dims; i++) {
+  for (int i = 0; i < nb_dims; i++) {
     if (weights_input_l.shape_.d[i] == weights_input_r.shape_.d[i]) {
       output_shape.d[i] = weights_input_l.shape_.d[i];
     } else if (weights_input_l.shape_.d[i] == 1 ||
@@ -841,6 +678,7 @@ tensorflow::Status ConstantFoldBinary(
   // Allocate output weights
   TRT_ShapedWeights weights_output = ctx.get_temp_weights(dtype, output_shape);
 
+  // Maybe I should do a switch
   LambdaFactory binary_op;
   if (node_def.op() == "Sub") {
     binary_op.op = LambdaFactory::OP_CATEGORY::SUB;
@@ -874,90 +712,48 @@ tensorflow::Status BinaryTensorOpWeight(
   // Maybe this part has to be moved into the block of rsqrt later
 
   // Check type consistency
+  auto dtype = TFAttrs(node_def).get<nvinfer1::DataType>("T");
+  CHECK_EQ_TYPE(tensor->getType(), dtype);  // Cast to int for error messages
   nvinfer1::DataType ttype;
   TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
+  CHECK_EQ_TYPE(ttype, dtype);  // Cast to int for error message
 
   // Check scale mode
   auto dims_w = weights.shape_;
   auto dims_t = tensor->getDimensions();
 
-  // default to element-wise
+  // Default to channel-wise
   auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
 
-  // TODO(jie): maybe use a permuatation instead to support more cases;
-  bool permutation_flag = false;
-
   if (weights.count() == 1) {
     VLOG(2) << "UNIFORM";
     scale_mode = nvinfer1::ScaleMode::kUNIFORM;
   } else {
-    // no broadcasting on Batch dimension;
-    VLOG(2) << "WEIGHTS DIM: " << dims_w.nbDims
-            << " tensor DIM: " << dims_t.nbDims;
-    if (dims_w.nbDims == dims_t.nbDims + 1) {
-      if (dims_w.d[0] == 1) {
-        for (int i = 1; i < dims_w.nbDims; i++) {
-          dims_w.d[i - 1] = dims_w.d[i];
-        }
-        dims_w.nbDims--;
-      } else {
-        return tensorflow::errors::InvalidArgument(
-            "Binary op cannot operate on batch, " + node_def.name());
-      }
-    }
+    // No broadcasting on Batch dimension;
+    assert(dims_w.d[0] == 1);
 
-    if (dims_w.nbDims == dims_t.nbDims && dims_w.d[0] == dims_t.d[0]) {
-      scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
-      // default is element;
-      for (int i = 1; i < dims_w.nbDims; i++) {
-        if (dims_w.d[i] != dims_t.d[i]) {
-          // if dimension does not match, switch back to channel;
-          VLOG(2) << "channel";
-          scale_mode = nvinfer1::ScaleMode::kCHANNEL;
-          break;
-        }
-      }
-      // if channel as candidate, validate it
-      if (scale_mode == nvinfer1::ScaleMode::kCHANNEL) {
-        for (int i = 1; i < dims_w.nbDims; i++) {
-          if (dims_w.d[i] != 1)
-            return tensorflow::errors::InvalidArgument(
-                "Weight shape not compatible at, " + node_def.name());
-        }
-      } else {
-        VLOG(2) << "elementwise";
+    // Broadcasting on Channel dimension only allowed in kUNIFORM
+    assert(dims_w.d[1] == dims_t.d[0]);
+    assert(dims_w.nbDims == dims_t.nbDims);
+
+    // Default is element;
+    for (int i = 2; i < dims_w.nbDims; i++) {
+      if (dims_w.d[i] != dims_t.d[i - 1]) {
+        scale_mode = nvinfer1::ScaleMode::kCHANNEL;
+        break;
       }
-    } else if (dims_w.nbDims == 1 &&
-               dims_w.d[0] == dims_t.d[dims_t.nbDims - 1]) {
-      // channel wise and broadcast required;
-      permutation_flag = true;
-      scale_mode = nvinfer1::ScaleMode::kCHANNEL;
-    } else {
-      return tensorflow::errors::InvalidArgument(
-          "Weight shape not compatible at, " + node_def.name());
     }
-  }
-
-  // transpose last dimension
-  std::vector<int> permutation(dims_t.nbDims + 1);
-  if (permutation_flag) {
-    if (scale_mode == nvinfer1::ScaleMode::kCHANNEL && dims_t.nbDims > 1) {
-      // we swap the last dimension into channel for trt.
-      // because of tensorflow default broadcasting rules.
-      for (int i = 0; i < static_cast<int>(permutation.size()); i++) {
-        permutation[i] = i;
+    if (scale_mode == nvinfer1::ScaleMode::kELEMENTWISE) {
+      scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
+      for (int i = 2; i < dims_w.nbDims; i++) {
+        if (dims_w.d[i] != 1)
+          return tensorflow::errors::InvalidArgument(
+              "Weight shape not compatible at, " + node_def.name());
       }
-      permutation[1] = dims_t.nbDims;
-      permutation[dims_t.nbDims] = 1;
-      tensor = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor),
-                                   permutation);
-    } else {
-      return tensorflow::errors::InvalidArgument(
-          "Transpose cannot be applied, " + node_def.name());
     }
   }
 
-  // prepare weights
+  // Prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
   TRT_ShapedWeights power_weights(weights.type_);
@@ -983,26 +779,88 @@ tensorflow::Status BinaryTensorOpWeight(
       scale_weights, power_weights);
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  // transpose back dimension
-  if (permutation_flag) {
-    output_tensor = ctx.TransposeTensor(output_tensor, permutation);
-  }
 
   // Pass the output
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
-enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
+tensorflow::Status BinaryTensorOpTensor(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
+      {"Add", nvinfer1::ElementWiseOperation::kSUM},
+      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
+      // {"max", nvinfer1::ElementWiseOperation::kMAX},
+      // {"min", nvinfer1::ElementWiseOperation::kMIN},
+      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
+      {"Div", nvinfer1::ElementWiseOperation::kDIV},
+  };
+
+  // FIXME assume type matches input weights
+  // Get trt type & shape
+  TFAttrs attrs(node_def);
+  // Maybe this part has to be moved into the block of rsqrt later
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+
+  // Check type consistency
+  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
+  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
+  auto op_pair = ops.find(node_def.op());
+  if (op_pair == ops.end())
+    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
+                                             " not supported at: " +
+                                             node_def.name());
+
+  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
+      *const_cast<nvinfer1::ITensor*>(tensor_l),
+      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  // Pass the output
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
 
-tensorflow::Status ConvertConv2DHelper(
+tensorflow::Status ConvertPlaceholder(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs,
-    int group  // group ==0 specifies depthwise conv
-) {
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+    std::vector<TRT_TensorOrWeights> const& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  VLOG(2) << "Placeholder should have been replace already";
+  return tensorflow::errors::Unimplemented(", cannot convert Placeholder op");
+  // OK this make sense since we are supposed to replace it with input
+  TFAttrs attrs(node_def);
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
+  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
+
+  dims.nbDims--;
+  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
+
+  nvinfer1::ITensor* output =
+      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
+  if (!output) {
+    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
+  }
+  outputs->push_back(TRT_TensorOrWeights(output));
+  return tensorflow::Status::OK();
+}
 
+tensorflow::Status ConvertConv2D(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  // TODO(jie): handle NHWC/NCHW transpose;
+  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
+  ReorderRSCKToKCRS(weights_rsck, &weights);
+  TRT_ShapedWeights biases(weights.type_);
+  int noutput = weights.shape_.d[0];
+  nvinfer1::DimsHW kernel_size;
+  kernel_size.h() = weights.shape_.d[2];
+  kernel_size.w() = weights.shape_.d[3];
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -1016,31 +874,11 @@ tensorflow::Status ConvertConv2DHelper(
     // TODO(jie): transpose it
   }
 
-  // tensor after transpose (NCHW)
-  auto tensor_dim = tensor->getDimensions();
-
-  int num_groups = group;
-  if (num_groups == 0)  // depthwise convolution
-    num_groups = tensor_dim.d[0];
-  VLOG(2) << "groups count: " << num_groups;
-
-  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
-  ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
-  TRT_ShapedWeights biases(weights.type_);
-  int noutput = weights.shape_.d[0] * num_groups;
-  nvinfer1::DimsHW kernel_size;
-  kernel_size.h() = weights.shape_.d[2];
-  kernel_size.w() = weights.shape_.d[3];
-  VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w();
-
   // TODO(jie): stride. (NHWC/NCHW)
   auto tf_stride = attrs.get<std::vector<int>>("strides");
-  VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index;
-  VLOG(2) << "stride!!!: " << tf_stride[0] << tf_stride[1] << tf_stride[2]
-          << tf_stride[3];
   nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
 
+  auto tensor_dim = tensor->getDimensions();
   std::vector<std::pair<int, int>> padding;
   // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
@@ -1081,11 +919,10 @@ tensorflow::Status ConvertConv2DHelper(
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
-  layer->setNbGroups(num_groups);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
   auto dim_after = output_tensor->getDimensions();
-  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1] << ", "
+  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1]
           << dim_after.d[2] << ", " << dim_after.d[3];
 
   if (data_format == "NHWC") {
@@ -1098,101 +935,11 @@ tensorflow::Status ConvertConv2DHelper(
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertConv2DHelper(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs, ConvolutionType type) {
-  switch (type) {
-    case ConvolutionType::DEFAULT:
-      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 1);
-    case ConvolutionType::DEPTHWISE_CONV:
-      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 0);
-  }
-  return tensorflow::errors::Unimplemented("unsupported convolution type at, " +
-                                           node_def.name());
-}
-
-tensorflow::Status BinaryTensorOpTensor(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
-      {"Add", nvinfer1::ElementWiseOperation::kSUM},
-      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
-      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
-      {"Div", nvinfer1::ElementWiseOperation::kDIV},
-  };
-
-  // FIXME assume type matches input weights
-  // get trt type & shape
-  TFAttrs attrs(node_def);
-  // maybe this part has to be moved into the block of rsqrt later
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
-
-  // check type consistency
-  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
-  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
-  auto op_pair = ops.find(node_def.op());
-  if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
-
-  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
-      *const_cast<nvinfer1::ITensor*>(tensor_l),
-      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
-
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-
-  // pass the output
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertPlaceholder(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  VLOG(2) << "Placeholder should have been replace already";
-  return tensorflow::errors::Unimplemented("cannot convert Placeholder op");
-  // OK this make sense since we are supposed to replace it with input
-  TFAttrs attrs(node_def);
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
-  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
-
-  dims.nbDims--;
-  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
-
-  nvinfer1::ITensor* output =
-      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
-  if (!output) {
-    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
-  }
-  outputs->push_back(TRT_TensorOrWeights(output));
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertConv2D(Converter& ctx,
-                                 const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
-                                 std::vector<TRT_TensorOrWeights>* outputs) {
-  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
-                             ConvolutionType::DEFAULT);
-}
-
-tensorflow::Status ConvertConv2DDepthwise(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
-                             ConvolutionType::DEPTHWISE_CONV);
-}
-
 tensorflow::Status ConvertPool(Converter& ctx,
                                const tensorflow::NodeDef& node_def,
-                               const std::vector<TRT_TensorOrWeights>& inputs,
+                               std::vector<TRT_TensorOrWeights> const& inputs,
                                std::vector<TRT_TensorOrWeights>* outputs) {
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -1210,8 +957,6 @@ tensorflow::Status ConvertPool(Converter& ctx,
   // TODO(jie): support other pooling type
   if (node_def.op() == "MaxPool")
     type = nvinfer1::PoolingType::kMAX;
-  else if (node_def.op() == "AvgPool")
-    type = nvinfer1::PoolingType::kAVERAGE;
   else
     return tensorflow::errors::Unimplemented("Only supports Max pool");
 
@@ -1274,9 +1019,9 @@ tensorflow::Status ConvertPool(Converter& ctx,
 
 tensorflow::Status ConvertActivation(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights> const& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
   nvinfer1::IActivationLayer* layer = ctx.network()->addActivation(
       *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ActivationType::kRELU);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
@@ -1286,14 +1031,14 @@ tensorflow::Status ConvertActivation(
 
 tensorflow::Status ConvertScale(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                const std::vector<TRT_TensorOrWeights>& inputs,
+                                std::vector<TRT_TensorOrWeights> const& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
     return tensorflow::errors::Unimplemented(
         "Only supports tensor op weight for now, at " + node_def.name());
   // Implement tensor binaryOp weight [channel wise] for now;
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
 
   // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
@@ -1310,26 +1055,9 @@ tensorflow::Status ConvertScale(Converter& ctx,
   } else {
     VLOG(2) << "NCHW !!!!";
   }
-
-  auto dims = tensor->getDimensions();
-  VLOG(2) << "tensor dimensions: " << dims.nbDims;
-  for (int i = 0; i < dims.nbDims; i++) {
-    VLOG(2) << "i: " << dims.d[i];
-  }
-  dims = weights.shape_;
-  VLOG(2) << "tensor dimensions: " << dims.nbDims;
-  for (int i = 0; i < dims.nbDims; i++) {
-    VLOG(2) << "i: " << dims.d[i];
-  }
-
-  nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
-  if (weights.shape_.d[0] == 1) {
-    mode = nvinfer1::ScaleMode::kUNIFORM;
-  }
-
-  nvinfer1::IScaleLayer* layer =
-      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
-                              weights, empty_weights, empty_weights);
+  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
+      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
+      weights, empty_weights, empty_weights);
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   if (data_format == "NHWC") {
@@ -1344,7 +1072,7 @@ tensorflow::Status ConvertScale(Converter& ctx,
 
 tensorflow::Status ConvertConst(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                const std::vector<TRT_TensorOrWeights>& inputs,
+                                std::vector<TRT_TensorOrWeights> const& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   const auto& weights_tensor = node_def.attr().at("value").tensor();
 
@@ -1363,144 +1091,20 @@ tensorflow::Status ConvertConst(Converter& ctx,
     VLOG(2) << "SCALAR!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
     if (tensor.dims() > 0) {
-      VLOG(2) << "dimensions: " << tensor.dims();
-      VLOG(2) << "size: " << weights_tensor.float_val_size();
-      scalar_shape = GetTensorShape(tensor);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        VLOG(2) << scalar_shape.d[i];
-      if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size()) {
-        if (weights_tensor.float_val_size() == 1 ||
-            scalar_shape.d[0] == weights_tensor.float_val_size()) {
-          scalar_shape.nbDims = 1;
-          // no dimension provided. flatten it
-          scalar_shape.d[0] = weights_tensor.float_val_size();
-          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
-        } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
-        }
-      }
-    } else {
       VLOG(2) << "Dimensions: " << tensor.dims();
-      scalar_shape.nbDims = 1;
-      // no dimension provided. flatten it
-      scalar_shape.d[0] = weights_tensor.float_val_size();
-      scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
-      for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
-        scalar_shape.d[i] = 0;
-        scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
-      }
-    }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      auto half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      half_tensor.device(defd) =
-          tensor.flat<float>().template cast<Eigen::half>();
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<float> tensor_data(
-          weights_tensor.float_val().begin(),
-          weights_tensor.float_val()
-              .end());  //  make a local copy first to flatten
-      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
-  } else if (!weights_tensor.int_val().empty()) {
-    VLOG(2) << "int!!!" << node_def.name();
-    nvinfer1::Dims scalar_shape;
-    if (tensor.dims() > 0) {
-      VLOG(2) << "dimensions: " << tensor.dims();
-      scalar_shape = GetTensorShape(tensor);
-      if (GetShapeSize(scalar_shape) != weights_tensor.int_val_size()) {
-        if (weights_tensor.int_val_size() == 1 ||
-            scalar_shape.d[0] == weights_tensor.int_val_size()) {
-          scalar_shape.nbDims = 1;
-          // no dimension provided. flatten it
-          scalar_shape.d[0] = weights_tensor.int_val_size();
-          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
-        } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
-        }
-      }
+      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
+                                  GetTensorShape(tensor));
     } else {
-      VLOG(2) << "dimensions: " << tensor.dims();
+      VLOG(2) << "Dimensions: " << tensor.dims();
       scalar_shape.nbDims = 1;
-      // no dimension provided. flatten it
-      scalar_shape.d[0] = weights_tensor.int_val_size();
+      scalar_shape.d[0] = 1;
       scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
       for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
         scalar_shape.d[i] = 0;
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
-    }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      switch (dtype) {
-        case (tensorflow::DT_INT32): {
-          half_tensor.device(defd) =
-              tensor.flat<int32>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT16): {
-          half_tensor.device(defd) =
-              tensor.flat<int16>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT8): {
-          half_tensor.device(defd) =
-              tensor.flat<int8>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_UINT8): {
-          half_tensor.device(defd) =
-              tensor.flat<uint8>().template cast<Eigen::half>();
-          break;
-        }
-        default:
-          return tensorflow::errors::InvalidArgument(
-              "Datatype " + tensorflow::DataTypeString(dtype) +
-              " for FP16 conversion");
-          break;
-      };
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
-      len_data = std::max(len_data, len_tensor);
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<int32> tensor_data(
-          weights_tensor.int_val().begin(),
-          weights_tensor.int_val()
-              .end());  //  make a local copy first to flatten
-                        //  doesn't have to be contigous
-      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
+                                  scalar_shape);
     }
   } else if (!weights_tensor.tensor_content().empty()) {
     VLOG(2) << "TENSOR!!!" << node_def.name();
@@ -1526,7 +1130,7 @@ tensorflow::Status ConvertConst(Converter& ctx,
 
 tensorflow::Status ConvertIdentity(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights> const& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   outputs->push_back(inputs.at(0));
   return tensorflow::Status::OK();
@@ -1534,7 +1138,7 @@ tensorflow::Status ConvertIdentity(
 
 tensorflow::Status ConvertBinary(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights> const& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2)
     return tensorflow::errors::FailedPrecondition(
@@ -1561,7 +1165,7 @@ tensorflow::Status ConvertBinary(Converter& ctx,
 
 tensorflow::Status ConvertUnary(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                const std::vector<TRT_TensorOrWeights>& inputs,
+                                std::vector<TRT_TensorOrWeights> const& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 1)
     return tensorflow::errors::FailedPrecondition(
@@ -1579,7 +1183,7 @@ tensorflow::Status ConvertUnary(Converter& ctx,
 
 tensorflow::Status ConvertReduce(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights> const& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1587,7 +1191,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1625,7 +1229,6 @@ tensorflow::Status ConvertReduce(Converter& ctx,
       return tensorflow::errors::InvalidArgument("TRT cannot reduce at 0, at" +
                                                  node_def.name());
     if (index_list_data[i] == 1) permuted_index = 1;
-
     idx_set.emplace(index_list_data[i]);
   }
 
@@ -1633,7 +1236,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
   nvinfer1::DimsHW pool_kernel;
   if (permuted_index == 1) {
     for (int i = 2; i < nb_dims; i++) {
-      if (idx_set.count(i) == 0) {
+      if (idx_set.count(i)) {
         permuted_index = i;
         break;
       }
@@ -1668,13 +1271,12 @@ tensorflow::Status ConvertReduce(Converter& ctx,
     output_tensor = ctx.TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), permutation_order);
   }
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
 tensorflow::Status ConvertPad(Converter& ctx,
                               const tensorflow::NodeDef& node_def,
-                              const std::vector<TRT_TensorOrWeights>& inputs,
+                              std::vector<TRT_TensorOrWeights> const& inputs,
                               std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1682,7 +1284,7 @@ tensorflow::Status ConvertPad(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1769,287 +1371,19 @@ tensorflow::Status ConvertPad(Converter& ctx,
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status ConvertConcat(Converter& ctx,
-                                 const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
-                                 std::vector<TRT_TensorOrWeights>* outputs) {
-  // not including the last input (axis) here
-  int input_size = static_cast<int>(inputs.size()) - 1;
-
-  if (!inputs.at(0).is_tensor())
-    return tensorflow::errors::InvalidArgument(
-        "Concat in TRT support only Tensor input, at " + node_def.name());
-
-  // We are retrieving the axis
-  TRT_ShapedWeights axis = inputs.at(input_size).weights();
-
-  TFAttrs attrs(node_def);
-  // auto attr_size = attrs.at("N")->i();
-  // auto data_type = attrs.get<nvinfer1::DataType>("T");
-  auto index_type = attrs.get<tensorflow::DataType>("Tidx");
-
-  // TODO(jie): handle data type
-  // Only expect to handle INT32 as index attributes for now
-  if (index_type != tensorflow::DataType::DT_INT32)
-    return tensorflow::errors::Unimplemented(
-        "Tidx supports only DT_INT32, at " + node_def.name());
-
-  int index = *(static_cast<int*>(const_cast<void*>(axis.GetValues())));
-
-  // TODO(jie): early termination with no-op (attr_size==1)
-
-  auto dim = inputs.at(0).tensor()->getDimensions();
-  // dimension check
-  if (index > dim.nbDims + 1)
-    return tensorflow::errors::InvalidArgument(
-        "Concatenate on axis out of dimension range, at " + node_def.name());
-
-  if (index == 0)
-    return tensorflow::errors::InvalidArgument(
-        "Concatenate on batch dimension not supported, at " + node_def.name());
-
-  // incase we need permutation;
-  std::vector<int> permutation_order(dim.nbDims + 1);
-
-  for (int i = 0; i < dim.nbDims + 1; i++) permutation_order[i] = i;
-
-  if (index != 1) {
-    permutation_order[1] = index - 1;
-    permutation_order[index - 1] = 1;
-  }
-
-  std::vector<nvinfer1::ITensor const*> inputs_vec;
-  // Shap chack (all input tensor should have same shape)
-  // starting from 0 since we are probably also doing transpose here;
-  for (int i = 0; i < input_size; i++) {
-    auto tensor_i = inputs.at(i).tensor();
-    auto dim_i = tensor_i->getDimensions();
-    if (dim_i.nbDims != dim.nbDims)
-      return tensorflow::errors::InvalidArgument(
-          "Concatenate receives inputs with inconsistent dimensions, at " +
-          node_def.name());
-
-    for (int j = 0; j < dim.nbDims; j++) {
-      // check dimension consistency on non-concatenate axis
-      if (j != index - 1 && dim_i.d[j] != dim.d[j])
-        return tensorflow::errors::InvalidArgument(
-            "Concatenate receives inputs with inconsistent shape, at" +
-            node_def.name());
-    }
-
-    // TRT does concatenation only on channel!
-    if (index != 1)
-      tensor_i = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor_i),
-                                     permutation_order);
-
-    inputs_vec.push_back(tensor_i);
-  }
-
-  // nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  nvinfer1::IConcatenationLayer* layer = ctx.network()->addConcatenation(
-      const_cast<nvinfer1::ITensor* const*>(inputs_vec.data()),
-      inputs_vec.size());
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-
-  if (index != 1) {
-    output_tensor = ctx.TransposeTensor(output_tensor, permutation_order);
-  }
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertFusedBatchNorm(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  TFAttrs attrs(node_def);
-  float epsilon = attrs.get<float>("epsilon");
-  auto data_format = attrs.get<string>("data_format");
-  if (data_format != "NCHW") {
-    return tensorflow::errors::Unimplemented(
-        "only data_format=NCHW is supported, at " + node_def.name());
-  }
-  bool is_training = attrs.get<bool>("is_training");
-  if (is_training) {
-    return tensorflow::errors::Unimplemented(
-        "only is_training=false is supported, at " + node_def.name());
-  }
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
-  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
-  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
-  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
-  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
-  TRT_ShapedWeights combined_scale_weights =
-      ctx.get_temp_weights_like(scale_weights);
-  TRT_ShapedWeights combined_offset_weights =
-      ctx.get_temp_weights_like(offset_weights);
-  size_t nweight = scale_weights.count();
-  if ((scale_weights.type_ == offset_weights.type_) &&
-      (mean_weights.type_ == variance_weights.type_) &&
-      (scale_weights.type_ == variance_weights.type_)) {
-    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
-        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
-      return tensorflow::errors::Unimplemented(
-          "only float32 or float16 weight data type is supported, for node " +
-          node_def.name() + " got " +
-          tensorflow::DataTypeString(scale_weights.type_));
-    }
-    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
-        float offset =
-            (static_cast<float const*>(offset_weights.GetValues()))[i];
-        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
-        float variance =
-            (static_cast<float const*>(variance_weights.GetValues()))[i];
-        float& combined_scale_ref = const_cast<float*>(
-            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
-        float& combined_offset_ref = const_cast<float*>(
-            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
-        combined_scale_ref = scale / sqrtf(variance + epsilon);
-        combined_offset_ref = offset - mean * combined_scale_ref;
-      }
-    } else {
-      const Eigen::half* scale_vals =
-          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
-      const Eigen::half* off_vals =
-          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
-      const Eigen::half* mean_vals =
-          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
-      const Eigen::half* variance_vals =
-          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
-      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
-      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale(scale_vals[i]);
-        float offset(off_vals[i]);
-        float mean(mean_vals[i]);
-        float variance(variance_vals[i]);
-        float combined_scale_ref = scale / sqrtf(variance + epsilon);
-        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
-        float combined_offset_ref = offset - mean * combined_scale_ref;
-        comb_off_vals[i] = Eigen::half(combined_offset_ref);
-      }
-    }
-  }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      combined_offset_weights.GetWeightsForTRT(),
-      combined_scale_weights.GetWeightsForTRT(),
-      dummy_power_weights.GetWeightsForTRT());
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertMatMul(Converter& ctx,
-                                 const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
-                                 std::vector<TRT_TensorOrWeights>* outputs) {
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-
-  // TODO(jie): transpose!
-  TFAttrs attrs(node_def);
-
-  TRT_ShapedWeights weights_ck = inputs.at(1).weights();
-  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_ck);
-  ReorderCKtoKC(weights_ck, &weights);
-  TRT_ShapedWeights biases(weights.type_);
-
-  int noutput = weights.shape_.d[0];
-
-  nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected(
-      *const_cast<nvinfer1::ITensor*>(tensor), noutput, weights, biases);
-
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status ConvertReshape(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
-      !inputs.at(1).is_weights())
-    return tensorflow::errors::InvalidArgument(
-        "Input expects tensor and weights, at" + node_def.name());
-
-  // implement tensor binaryOp weight [channel wise] for now;
-  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
-  auto dims = tensor->getDimensions();
-  // restore implicit batch dimension
-
-  TRT_ShapedWeights shape = inputs.at(1).weights();
-
-  TFAttrs attrs(node_def);
-
-  auto padding_type = attrs.get<tensorflow::DataType>("Tshape");
-
-  if (shape.shape_.nbDims != 1)
-    return tensorflow::errors::InvalidArgument(
-        "reshape new shape is not 1 dimensional, at " + node_def.name());
-
-  // Only expect to handle INT32 as attributes for now
-  if (padding_type != tensorflow::DataType::DT_INT32)
-    return tensorflow::errors::Unimplemented(
-        "reshape new shape supports only DT_INT32, at " + node_def.name());
-
-  auto shape_data = static_cast<int*>(const_cast<void*>(shape.GetValues()));
-
-  if (shape_data[0] != -1)
-    return tensorflow::errors::InvalidArgument(
-        "reshape new shape first dimension is not -1, at " + node_def.name());
-
-  auto shape_num_dims = shape.shape_.d[0];
-  VLOG(2) << "shape dimensions: " << shape_num_dims;
-  int volume_w = 1;
-  for (int i = 1; i < shape.shape_.d[0]; i++) volume_w *= shape_data[i];
-
-  int volume_t = 1;
-  for (int i = 0; i < dims.nbDims; i++) volume_t *= dims.d[i];
-
-  VLOG(2) << "volume: " << volume_t << " volume weights: " << volume_w;
-  if (volume_w != volume_t)
-    return tensorflow::errors::InvalidArgument(
-        "volume does not agree between tensor and new shape, at " +
-        node_def.name());
-
-  nvinfer1::IShuffleLayer* layer =
-      ctx.network()->addShuffle(*const_cast<nvinfer1::ITensor*>(tensor));
-
-  nvinfer1::Dims reshape_dims;
-  VLOG(2) << "new dimension: " << shape_num_dims - 1;
-  reshape_dims.nbDims = shape_num_dims - 1;
-  for (int32_t i = 0; i < reshape_dims.nbDims; ++i) {
-    reshape_dims.d[i] = shape_data[i + 1];
-  }
-  layer->setReshapeDimensions(reshape_dims);
-  VLOG(2) << "new dimension: " << shape_num_dims - 1;
-
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  auto dims_output = output_tensor->getDimensions();
-  VLOG(2) << "output tensor dimension:" << dims_output.nbDims;
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
-
 void Converter::register_op_converters() {
   // vgg_16 slim implementation
   op_registry_["Placeholder"] = ConvertPlaceholder;
   op_registry_["Conv2D"] = ConvertConv2D;
-  op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
-  op_registry_["AvgPool"] = ConvertPool;
   // This could be really handled as ConvertBinary
   op_registry_["BiasAdd"] = ConvertScale;
   op_registry_["Const"] = ConvertConst;
+  // op_registry_["MatMul"] = ConvertFullyConnected;  // Not used in vgg
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
+  // op_registry_["AvgPool"] = ConvertPool;
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -2059,364 +1393,26 @@ void Converter::register_op_converters() {
   op_registry_["Mean"] = ConvertReduce;
   op_registry_["Pad"] = ConvertPad;
   // TODO(ben,jie): Add more ops
-
-  op_registry_["ConcatV2"] = ConvertConcat;
-  op_registry_["MatMul"] = ConvertMatMul;
-  op_registry_["Reshape"] = ConvertReshape;
-  op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm;
-  op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm;
 }
 
 }  // namespace
-tensorflow::Status GetTensorRTGraph(tensorrt::convert::SubGraphParams& s) {
-  return tensorflow::errors::Unimplemented("Not implemented yet");
-}
-tensorflow::Status ConvertCalibrationNodeToEngineNode(
-    tensorflow::Graph& graph, tensorflow::Node* c_node) {
-  const auto ndef = c_node->def();
-
-  TFAttrs attrs(ndef);
-  std::vector<string> segment_nodes(
-      attrs.get<std::vector<string>>("segment_nodes"));
-  std::vector<string> output_nodes(
-      attrs.get<std::vector<string>>("segment_output_names"));
-  std::vector<string> input_names(
-      attrs.get<std::vector<string>>("input_names"));
-  string res_name = attrs.get<string>("resource_name");
-  VLOG(1) << "Node name " << c_node->name() << " res_name " << res_name;
-  string engine_name = "my_trt_op";
-  {
-    const auto node_id = tensorflow::str_util::Split(res_name, "_");
-    engine_name += node_id.back();
-  }
-  std::map<string, tensorflow::Node*> node_maps;
-
-  for (auto n : graph.op_nodes()) {
-    node_maps.insert({n->name(), n});
-  }
-  VLOG(1) << "Output Nodes:";
-  std::vector<tensorflow::DataType> out_types;
-  std::vector<const tensorflow::Edge*> out_edges;
-  for (auto& i : output_nodes) {
-    auto node_port = tensorflow::str_util::Split(i, ":");
-    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
-    auto out_node_name = node_port.at(0);
-    if (node_port.size() > 1) {
-      VLOG(1) << "Multi port output" << node_port.at(0) << " "
-              << node_port.at(1) << " size=" << node_port.size();
-    }
-    auto node_it = node_maps.find(out_node_name);
-    if (node_it != node_maps.end()) {
-      tensorflow::Node* out_node = node_it->second;
-      int port = 0;
-      if (node_port.size() == 2) {
-        port = std::strtoul(node_port.at(1).c_str(), nullptr, 10);
-        out_types.push_back(out_node->output_type(port));
-      } else {
-        out_types.push_back(out_node->output_type(0));
-      }
-      for (auto out_edge : out_node->out_edges()) {
-        if (out_edge->src_output() == port) {
-          out_edges.push_back(out_edge);
-          break;
-        }
-      }
-    } else {
-      LOG(WARNING) << " couldn't find output node " << out_node_name;
-    }
-  }
-  VLOG(1) << "Input Nodes:";
-  for (auto& i : input_names) {
-    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
-  }
-  auto trt_rm = tensorflow::tensorrt::TRTResourceManager::instance();
-  auto resmgr = trt_rm->getManager("TRTCalibOps");
-  tensorflow::tensorrt::TRTCalibrationResource* calib_res = nullptr;
-  auto status = resmgr->Lookup(res_name, res_name, &calib_res);
-  if (!status.ok() || !calib_res->calibrator_) {
-    return tensorflow::errors::FailedPrecondition(
-        "You must run calibration"
-        " and inference conversion in the same proces");
-  }
-
-  calib_res->calibrator_->setDone();
-  calib_res->thr_->join();
-  delete calib_res->thr_;
-  if (!calib_res->engine_) {
-    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
-                  "calibration graph?";
-  }
-  auto weight_rmgr = trt_rm->getManager("WeightStore");
-  TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
-      res_name, res_name));
-  auto engine_plan = calib_res->engine_->serialize();
-  calib_res->engine_->destroy();
-  calib_res->network_->destroy();
-  calib_res->builder_->destroy();
-  calib_res->thr_ = nullptr;
-  calib_res->engine_ = nullptr;
-  calib_res->builder_ = nullptr;
-  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
-  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
-  for (const auto in_edge : c_node->in_edges()) {
-    auto src = in_edge->src();
-    int dest_port = in_edge->dst_input();
-    income_edges.emplace_back(src->name(), in_edge->src_output(),
-                              c_node->input_type(dest_port));
-  }
-  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
-      income_edges);
-  op_builder.Input(input_list);
-  tensorflow::NodeDef engine_node;
-  const char* engine_plan_data = static_cast<const char*>(engine_plan->data());
-  string engine_plan_string(engine_plan_data,
-                            engine_plan_data + engine_plan->size());
-  status = op_builder.Attr("serialized_engine", engine_plan_string)
-               .Attr("input_nodes", input_names)
-               .Attr("output_nodes", output_nodes)
-               .Attr("OutT", out_types)
-               .Finalize(&engine_node);
-  if (!status.ok()) {
-    LOG(ERROR) << "Engine Node creation failed";
-    return status;
-  }
-  auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_CHECK_OK(status);
-  for (size_t i = 0; i < out_edges.size(); i++) {
-    VLOG(1) << "Connecting trt_engine_node output " << i << " with "
-            << out_edges.at(i)->dst()->name() << " port "
-            << out_edges.at(i)->dst_input();
-    TF_RETURN_IF_ERROR(graph.UpdateEdge(trt_engine_node, i,
-                                        out_edges.at(i)->dst(),
-                                        out_edges.at(i)->dst_input()));
-  }
-  VLOG(1) << "Segment nodes:";
-  for (auto& i : segment_nodes) {
-    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
-    auto it = node_maps.find(i);
-    if (it != node_maps.end()) {
-      graph.RemoveNode(it->second);
-    }
-  }
-  graph.RemoveNode(c_node);
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
-  // Visit nodes in reverse topological order and construct the TRT network.
-
-  // Toposort
-  std::vector<tensorflow::Node*> order_vec;
-  tensorflow::GetPostOrder(s.graph, &order_vec);
-  // Select just the subgraph
-  std::list<tensorflow::Node*> order;
-  for (tensorflow::Node* node : order_vec) {
-    if (s.subgraph_node_ids.count(node->id())) {
-      order.push_front(node);  // we want topological order to contstruct the
-      // network layer by layer
-    }
-  }
-  // topological order is needed to build TRT network
-  static int static_id = 0;
-  string subgraph_name_scope;
-  if (!order.empty()) {
-    subgraph_name_scope = order.front()->name();
-  }
-  for (const tensorflow::Node* node : order) {
-    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
-  }
-  // TODO(sami,ben,jie): proper naming!
-  string calib_op_name =
-      StrCat(subgraph_name_scope, "my_trt_calib_op_", static_id);
-  string engine_name = StrCat(subgraph_name_scope, "my_trt_op", static_id);
-  static_id++;
-  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
-  auto op_rmgr = trt_rmgr->getManager("TRTCalibOps");
-  auto op_res = new tensorflow::tensorrt::TRTCalibrationResource();
-  TF_CHECK_OK(op_rmgr->Create(calib_op_name, calib_op_name, op_res));
-  op_res->logger_ = new tensorflow::tensorrt::Logger();
-  op_res->builder_ = nvinfer1::createInferBuilder(*(op_res->logger_));
-
-  if (!op_res->builder_) {
-    return tensorflow::errors::Internal(
-        "failed to create TensorRT builder object");
-  }
-
-  op_res->network_ = op_res->builder_->createNetwork();
-  if (!op_res->network_) {
-    return tensorflow::errors::Internal(
-        "failed to create TensorRT network object");
-  }
-
-  // Build the network
-  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
-  auto ws = new tensorflow::tensorrt::TRTWeightStore();
-  TF_CHECK_OK(weight_rmgr->Create(calib_op_name, calib_op_name, ws));
-  Converter converter(op_res->network_, ws, s.precision_mode == FP16MODE);
-  std::vector<string> input_names;
-  std::vector<tensorflow::DataType> input_dtypes;
-  for (const std::pair<int, int>& input : s.input_inds) {
-    VLOG(2) << "parsing input. Node id= " << input.first;
-    int node_id = input.first;
-    int output_idx = input.second;
-    tensorflow::Node* node = s.graph.FindNodeId(node_id);
-    auto node_name = node->name();
-    input_names.push_back(node_name);  // insert original node name without port
-    // TODO(jie): alternative :)
-    if (!s.graph_properties.HasOutputProperties(node_name))
-      return tensorflow::errors::Internal("failed to find input node: " +
-                                          node_name);
-
-    auto op_info_vec = s.graph_properties.GetOutputProperties(node_name);
-    if (static_cast<int>(op_info_vec.size()) < output_idx)
-      return tensorflow::errors::Internal(
-          "accessing output index of: ", output_idx, ", at node: ", node_name,
-          "with output entry from shape_map: ", op_info_vec.size());
-
-    auto op_info = op_info_vec.at(output_idx);
-
-    tensorflow::DataType tf_dtype = op_info.dtype();
-    input_dtypes.push_back(tf_dtype);
-
-    nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
-
-    VLOG(2) << "accessing output index of: " << output_idx
-            << ", at node: " << node_name
-            << "with output entry from shape_map: " << op_info_vec.size();
-
-    // TODO(ben,jie): update TRT input format/dimension
-    nvinfer1::DimsCHW input_dim_psuedo_chw;
-    for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
-
-    for (int i = 1; i < op_info.shape().dim_size(); i++) {
-      VLOG(2) << "dimension: " << i
-              << " , size: " << op_info.shape().dim(i).size();
-      input_dim_psuedo_chw.d[i - 1] = op_info.shape().dim(i).size();
-    }
-
-    // TODO(ben,jie): proper way to restore input tensor name?
-    auto input_tensor_name = node_name;
-    if (output_idx != 0) input_tensor_name = StrCat(node_name, ":", output_idx);
-
-    nvinfer1::ITensor* input_tensor = converter.network()->addInput(
-        input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
-
-    if (!input_tensor)
-      return tensorflow::errors::InvalidArgument(
-          "Failed to create Input layer");
-    VLOG(2) << "input tensor name :" << input_tensor_name;
-
-    if (!converter.insert_input_tensor(input_tensor_name, input_tensor))
-      return tensorflow::errors::AlreadyExists(
-          "output tensor already exists for op: " + input_tensor_name);
-  }
-
-  VLOG(2) << "finished sorting";
-
-  for (const tensorflow::Node* node : order) {
-    const tensorflow::NodeDef& node_def = node->def();
-    VLOG(2) << "converting node: " << node_def.name() << " , " << node_def.op();
-    TF_RETURN_IF_ERROR(converter.convert_node(node_def));
-  }
-
-  VLOG(2) << "finished conversion";
-
-  // Gather output metadata
-  std::vector<string> output_names;
-  std::vector<tensorflow::DataType> output_dtypes;
-  int trt_engine_op_output_idx = 0;
-  for (const std::pair<int, int>& output : s.output_inds) {
-    int node_id = output.first;
-    int output_idx = output.second;
-    tensorflow::Node* node = s.graph.FindNodeId(node_id);
-    string op_name = node->name();
-    string tensor_name = op_name;
-
-    s.output_edge_map->insert(
-        {trt_engine_op_output_idx == 0
-             ? engine_name
-             : StrCat(engine_name, ":", trt_engine_op_output_idx),
-         {output_idx, tensor_name}});
-    trt_engine_op_output_idx++;
-    if (output_idx != 0) {
-      tensor_name = StrCat(tensor_name, ":", output_idx);
-    }
-    VLOG(1) << "output tensor name: " << tensor_name;
-    output_names.push_back(tensor_name);
-    auto tensor_or_weights = converter.get_tensor(tensor_name);
-    if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
-    }
-    nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
-    if (!tensor) {
-      return tensorflow::errors::NotFound("Output tensor not found: " +
-                                          tensor_name);
-    }
-    converter.network()->markOutput(*tensor);
-    tensorflow::DataType tf_dtype = node->output_type(output_idx);
-    output_dtypes.push_back(tf_dtype);
-    nvinfer1::DataType trt_dtype = nvinfer1::DataType::kFLOAT;
-    TF_RETURN_IF_ERROR(ConvertDType(tf_dtype, &trt_dtype));
-    tensor->setType(trt_dtype);
-  }
-
-  VLOG(2) << "finished output";
-
-  // Build the engine
-  op_res->builder_->setMaxBatchSize(s.max_batch_size);
-  op_res->builder_->setMaxWorkspaceSize(s.max_workspace_size_bytes);
-
-  // Build the TRT op
-  // TODO(sami,ben,jie): proper naming!
-  tensorflow::NodeDefBuilder op_builder(calib_op_name, "TRTCalibOp");
-  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
-  for (size_t i = 0; i < input_names.size(); ++i) {
-    int output_idx = s.input_inds.at(i).second;
-    // we wired up the input here already, it is redundant to do it again in
-    //  ConvertSubGraphToTensorRT(convert_graph.cc)
-    auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
-        input_names.at(i), output_idx, input_dtypes.at(i));
-    VLOG(1) << calib_op_name << " input " << i << " = " << input_names.at(i)
-            << ":" << output_idx
-            << " dType= " << tensorflow::DataTypeString(input_dtypes.at(i));
-    income_edges.push_back(incoming_edge);
-  }
-  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
-      income_edges);
-  op_builder.Input(input_list);
-  std::vector<string> segment_names;
-  segment_names.reserve(s.subgraph_node_ids.size());
-  for (int i : s.subgraph_node_ids) {
-    auto node = s.graph.FindNodeId(i);
-    segment_names.push_back(node->name());
-  }
-  LOG(INFO) << "finished op preparation";
-
-  auto status = op_builder.Attr("segment_nodes", segment_names)
-                    .Attr("input_names", input_names)
-                    .Attr("segment_output_names", output_names)
-                    .Attr("resource_name", calib_op_name)
-                    .Finalize(s.trt_node);
-
-  LOG(INFO) << status.ToString();
-  LOG(INFO) << "finished op building";
-
-  return tensorflow::Status::OK();
-}
 
 tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
-    tensorrt::convert::SubGraphParams& s) {
+    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
+    const std::vector<std::pair<int, int>>& input_inds,
+    const std::vector<std::pair<int, int>>& output_inds, size_t max_batch_size,
+    size_t max_workspace_size_bytes,
+    const tensorflow::grappler::GraphProperties& graph_properties,
+    tensorflow::NodeDef* trt_node) {
   // Visit nodes in reverse topological order and construct the TRT network.
 
   // Toposort
   std::vector<tensorflow::Node*> order_vec;
-  tensorflow::GetPostOrder(s.graph, &order_vec);
+  tensorflow::GetPostOrder(graph, &order_vec);
   // Select just the subgraph
   std::list<tensorflow::Node*> order;
   for (tensorflow::Node* node : order_vec) {
-    if (s.subgraph_node_ids.count(node->id())) {
+    if (subgraph_node_ids.count(node->id())) {
       // We want topological order to contstruct the
       // network layer by layer
       order.push_front(node);
@@ -2438,86 +1434,46 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
         "Failed to create TensorRT network object");
   }
 
-  string subgraph_name_scope;
-  if (!order.empty()) {
-    subgraph_name_scope = order.front()->name();
-  }
-  for (const tensorflow::Node* node : order) {
-    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
-  }
-  static int static_id = 0;
-  // TODO(sami,ben,jie): proper naming!
-  string engine_name = StrCat(subgraph_name_scope, "my_trt_op");
-  engine_name = StrCat(engine_name, static_id++);
-  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
-  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
-  auto ws = new tensorflow::tensorrt::TRTWeightStore();
-  TF_CHECK_OK(weight_rmgr->Create(engine_name, engine_name, ws));
-
   // Build the network
-  Converter converter(trt_network.get(), ws, s.precision_mode == FP16MODE);
+  Converter converter(trt_network.get());
 
   std::vector<string> input_names;
   std::vector<tensorflow::DataType> input_dtypes;
-  for (const std::pair<int, int>& input : s.input_inds) {
-    VLOG(2) << "parsing input!!!!!";
+  for (std::pair<int, int> const& input : input_inds) {
     int node_id = input.first;
     int output_idx = input.second;
-    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    tensorflow::Node* node = graph.FindNodeId(node_id);
     auto node_name = node->name();
-    // input_names should use the node name in the graph
-    // here it should be the input tensor name -> matching the binding
-    // insert original node name without port
-    auto tensor_name = node_name;
-    if (output_idx != 0) {
-      tensor_name = StrCat(tensor_name, ":", output_idx);
-    }
-
-    VLOG(2) << "input name: " << node_name << " tensor_name: " << tensor_name
-            << " idx: " << output_idx;
-
-    auto shape_inference_node_name = node_name;
-    auto shape_inference_output_idx = output_idx;
-    // rewire the shape inference to original node in the graph
-    if (s.output_edge_map->count(tensor_name)) {
-      shape_inference_node_name = s.output_edge_map->at(tensor_name).second;
-      shape_inference_output_idx = s.output_edge_map->at(tensor_name).first;
-    }
-    if (shape_inference_output_idx < 0) continue;
-    VLOG(2) << "shapeinference name: " << shape_inference_node_name
-            << " idx: " << shape_inference_output_idx;
-
-    if (!s.graph_properties.HasOutputProperties(shape_inference_node_name))
-      return tensorflow::errors::Internal("failed to find input node: " +
-                                          shape_inference_node_name);
+    input_names.push_back(node_name);  // Insert original node name without port
+    // TODO(jie): alternative :)
+    if (!graph_properties.HasOutputProperties(node_name))
+      return tensorflow::errors::Internal("Failed to find input node: " +
+                                          node_name);
 
-    auto op_info_vec =
-        s.graph_properties.GetOutputProperties(shape_inference_node_name);
-    if (static_cast<int>(op_info_vec.size()) <= shape_inference_output_idx)
+    auto op_info_vec = graph_properties.GetOutputProperties(node_name);
+    if (static_cast<int>(op_info_vec.size()) < output_idx)
       return tensorflow::errors::Internal(
-          "accessing output index of: ", shape_inference_output_idx,
-          ", at node: ", shape_inference_node_name,
-          " with output entry from shape_map: ", op_info_vec.size());
+          "Accessing output index of: " + std::to_string(output_idx) +
+          ", at node: " + node_name + " with output entry from shape_map: " +
+          std::to_string(op_info_vec.size()));
+
+    auto op_info = op_info_vec.at(output_idx);
 
-    auto op_info = op_info_vec.at(shape_inference_output_idx);
     tensorflow::DataType tf_dtype = op_info.dtype();
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
-    VLOG(2) << "Accessing output index of: " << output_idx
+    VLOG(2) << "Accessing output index of: " << std::to_string(output_idx)
             << ", at node: " << node_name
-            << " with output entry from shape_map: " << op_info_vec.size();
+            << " with output entry from shape_map: "
+            << std::to_string(op_info_vec.size());
+
     // TODO(ben,jie): update TRT input format/dimension
     nvinfer1::DimsCHW input_dim_psuedo_chw;
     for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
 
-    // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
-    //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4)
-      return tensorflow::errors::Unimplemented("require 4 dimensional input");
-
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
               << " , size: " << op_info.shape().dim(i).size();
@@ -2526,11 +1482,9 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(ben,jie): proper way to restore input tensor name?
     auto input_tensor_name = node_name;
-    if (output_idx != 0) {
-      input_tensor_name = StrCat(node_name, ":", output_idx);
-    }
+    if (output_idx != 0)
+      input_tensor_name = node_name + ":" + std::to_string(output_idx);
 
-    input_names.push_back(input_tensor_name);
     nvinfer1::ITensor* input_tensor = converter.network()->addInput(
         input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
 
@@ -2557,22 +1511,14 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   // Gather output metadata
   std::vector<string> output_names;
   std::vector<tensorflow::DataType> output_dtypes;
-  int trt_engine_op_output_idx = 0;
-  for (const std::pair<int, int>& output : s.output_inds) {
+  for (std::pair<int, int> const& output : output_inds) {
     int node_id = output.first;
     int output_idx = output.second;
-    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    tensorflow::Node* node = graph.FindNodeId(node_id);
     string op_name = node->name();
     string tensor_name = op_name;
-
-    s.output_edge_map->insert(
-        {trt_engine_op_output_idx == 0
-             ? engine_name
-             : StrCat(engine_name, ":", trt_engine_op_output_idx),
-         {output_idx, tensor_name}});
-    trt_engine_op_output_idx++;
     if (output_idx != 0)
-      tensorflow::strings::StrAppend(&tensor_name, ":", output_idx);
+      tensor_name = tensor_name + ":" + std::to_string(output_idx);
     VLOG(2) << "Output tensor name: " << tensor_name;
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
@@ -2594,25 +1540,19 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
 
   VLOG(2) << "Finished output";
+  // TODO(jie): static_id is not thread safe.
+  static int static_id = 0;
 
   // Build the engine
-  trt_builder->setMaxBatchSize(s.max_batch_size);
-  trt_builder->setMaxWorkspaceSize(s.max_workspace_size_bytes);
-  VLOG(0) << "Max batch size= " << s.max_batch_size
-          << " max workspace size= " << s.max_workspace_size_bytes;
-  if (s.precision_mode == FP16MODE) {
-    trt_builder->setHalf2Mode(true);
-    VLOG(0) << "Using FP16 precision mode";
-  }
-  LOG(INFO) << "starting build engine";
+  trt_builder->setMaxBatchSize(max_batch_size);
+  trt_builder->setMaxWorkspaceSize(max_workspace_size_bytes);
+  VLOG(0) << "Starting build engine " << static_id;
+  // TODO(ben,jie): half2 and int8 mode support
   string engine_plan_string;
   {
     auto trt_engine =
         infer_object(trt_builder->buildCudaEngine(*converter.network()));
     VLOG(0) << "Built network";
-    if (trt_engine.get() == nullptr) {
-      return tensorflow::errors::Internal("Engine building failure");
-    }
     auto engine_plan = infer_object(trt_engine->serialize());
     VLOG(0) << "Serialized engine";
     const char* engine_plan_data =
@@ -2620,19 +1560,18 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     engine_plan_string =
         string(engine_plan_data, engine_plan_data + engine_plan->size());
   }
-  TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
-      engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name;
+
+  VLOG(0) << "Finished engine";
 
   // Build the TRT op
-  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
+  // TODO(sami,ben,jie): proper naming!
+  tensorflow::NodeDefBuilder op_builder(
+      tensorflow::strings::StrCat("my_trt_op", static_id++), "TRTEngineOp");
   std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
-  VLOG(2) << "input edge size: " << input_names.size();
   for (size_t i = 0; i < input_names.size(); ++i) {
-    VLOG(2) << "input edges: " << i << " " << input_names.at(i);
-    int output_idx = s.input_inds.at(i).second;
-    // we wired up the input here already, it is redundant to do it again in
-    //  ConvertSubGraphToTensorRT(convert_graph.cc)
+    int output_idx = input_inds.at(i).second;
+    // We wired up the input here already, it is redundant to do it again in
+    // ConvertSubGraphToTensorRT(convert_graph.cc)
     auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
         input_names.at(i), output_idx, input_dtypes.at(i));
     income_edges.push_back(incoming_edge);
@@ -2647,7 +1586,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
                     .Attr("input_nodes", input_names)
                     .Attr("output_nodes", output_names)
                     .Attr("OutT", output_dtypes)
-                    .Finalize(s.trt_node);
+                    .Finalize(trt_node);
 
   VLOG(0) << status.ToString() << " finished op building";
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 954a1e72f8..2e7fd19566 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -17,8 +17,6 @@ limitations under the License.
 #define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
 
 #include <set>
-#include <string>
-#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -34,49 +32,16 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-const int FP32MODE = 0;
-const int FP16MODE = 1;
-const int INT8MODE = 2;
+tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
+    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
+    const std::vector<std::pair<int, int>>&
+        input_inds,  // {node_id, output_idx}
+    const std::vector<std::pair<int, int>>&
+        output_inds,  // {node_id, output_idx}
+    size_t max_batch_size, size_t max_workspace_size_bytes,
+    const tensorflow::grappler::GraphProperties& graph_prop,
+    tensorflow::NodeDef* trt_node);
 
-struct SubGraphParams {
-  SubGraphParams(
-      tensorflow::Graph& inp_graph,
-      const std::set<int>& subgraph_node_id_numbers,
-      const std::vector<std::pair<int, int>>& input_indices,
-      const std::vector<std::pair<int, int>>& output_indices,
-      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
-      const tensorflow::grappler::GraphProperties& current_graph_properties,
-      std::unordered_map<string, std::pair<int, string>>* output_edges,
-      tensorflow::NodeDef* constructed_trt_node,
-      int engine_precision_mode = FP32MODE)
-      : graph(inp_graph),
-        subgraph_node_ids(subgraph_node_id_numbers),
-        input_inds(input_indices),
-        output_inds(output_indices),
-        max_batch_size(max_supported_batch_size),
-        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
-        graph_properties(current_graph_properties),
-        output_edge_map(output_edges),
-        trt_node(constructed_trt_node),
-        precision_mode(engine_precision_mode) {}
-
-  tensorflow::Graph& graph;
-  const std::set<int>& subgraph_node_ids;
-  const std::vector<std::pair<int, int>>& input_inds;   // {node_id, output_idx}
-  const std::vector<std::pair<int, int>>& output_inds;  // {node_id, output_idx}
-  size_t max_batch_size;
-  size_t max_workspace_size_bytes;
-  const tensorflow::grappler::GraphProperties& graph_properties;
-  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
-  tensorflow::NodeDef* trt_node;
-  const int precision_mode;
-};
-
-// TODO(sami): Replace references with const reference or pointers
-tensorflow::Status ConvertSubGraphToTensorRTNodeDef(SubGraphParams& params);
-tensorflow::Status InjectCalibrationNode(SubGraphParams& params);
-tensorflow::Status ConvertCalibrationNodeToEngineNode(tensorflow::Graph& graph,
-                                                      tensorflow::Node* c_node);
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
index aea44fd8a2..1dcb87e768 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
@@ -21,11 +21,10 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
-#include "tensorflow/core/platform/stream_executor.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda/include/cuda_runtime_api.h"
+#include "cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
@@ -114,13 +113,7 @@ void TRTCalibOp::Compute(tensorflow::OpKernelContext* ctx) {
     ctx->set_output(i, t);
   }
   VLOG(2) << "Filled map for sending";
-  // copied from cuda_kernel_helper since it seems only valid in *.cu.cc files
-  const cudaStream_t* stream = CHECK_NOTNULL(
-      reinterpret_cast<const cudaStream_t*>(ctx->op_device_context()
-                                                ->stream()
-                                                ->implementation()
-                                                ->CudaStreamMemberHack()));
-  calib_res->calibrator_->setBatch(input_data, *stream);
+  calib_res->calibrator_->setBatch(input_data);
   VLOG(2) << "Passed calibration data";
   // TODO(aaroey): make sure we wait for the completion of calibration on the
   // last batch in future PR.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index b32371b642..8efdf63ebe 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -24,12 +24,8 @@ limitations under the License.
 #include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
-static ::tensorflow::tensorrt::Logger logger;
-namespace gpu = ::perftools::gputools;
-using IRuntime = nvinfer1::IRuntime;
-using Dims = nvinfer1::Dims;
-
 namespace tensorrt {
+static ::tensorflow::tensorrt::Logger logger;
 
 TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // read serialized_engine
@@ -44,21 +40,10 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // TODO(samikama) runtime should be taken from a resourcemanager as well.
   // Only engine should be in the op and context and runtime should be taken
   // from resourcemanager
-  // TODO(jie): cudaSetDevice make sure trt engine is allocated on the same
-  // gpu where the input/output is also located.
-  int gpu_id = context->device()->tensorflow_gpu_device_info()->gpu_id;
-  cudaSetDevice(gpu_id);
-  int device;
-  cudaGetDevice(&device);
-  if (gpu_id != device) LOG(FATAL) << "set device failed!";
-
-  // TODO(samikama) runtime should be taken from a resourcemanager as well.
-  // Only engine should be in the op and context and runtime should be taken
-  // from resourcemanager
-
-  IRuntime* infer = nvinfer1::createInferRuntime(logger);
+  nvinfer1::IRuntime* infer = nvinfer1::createInferRuntime(logger);
   trt_engine_ptr_.reset(infer->deserializeCudaEngine(
       serialized_engine.c_str(), serialized_engine.size(), nullptr));
+
   trt_execution_context_ptr_.reset(trt_engine_ptr_->createExecutionContext());
   // Runtime is safe to delete after engine creation
   infer->destroy();
@@ -70,6 +55,7 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
 
   size_t binding_index;
   int num_batch = 0;
+  bool valid = true;
   for (int i = 0; i < context->num_inputs(); i++) {
     // Grab the input tensor
     binding_index = trt_engine_ptr_->getBindingIndex(input_nodes_[i].c_str());
@@ -78,12 +64,8 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     const TensorShape& input_shape = input_tensor.shape();
     if (i == 0) {
       num_batch = input_shape.dim_size(0);
-      if (num_batch > trt_engine_ptr_->getMaxBatchSize()) {
-        LOG(FATAL) << "input tensor batch larger than max_batch_size: "
-                   << trt_engine_ptr_->getMaxBatchSize();
-      }
     } else if (num_batch != input_shape.dim_size(0)) {
-      LOG(FATAL) << "input data inconsistent batch size";
+      valid = false;
       break;
     }
     switch (trt_engine_ptr_->getBindingDataType(binding_index)) {
@@ -99,6 +81,9 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     }
   }
 
+  // Might want a different way to inform the user of batch size inconsistency
+  if (!valid) LOG(WARNING) << "input data inconsistent batch size";
+
   for (int i = 0; i < static_cast<int>(output_nodes_.size()); i++) {
     // This is bad that we have to reallocate output buffer every run.
     // Create an output tensor
@@ -141,11 +126,9 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
                                                 ->implementation()
                                                 ->CudaStreamMemberHack()));
 
-  // TODO(jie): trt enqueue does not return error
-  auto ret = trt_execution_context_ptr_->enqueue(num_batch, &buffers[0],
-                                                 *stream, nullptr);
-  VLOG(2) << "enqueue returns: " << ret;
-  // sync should be done by TF.
+  // execution handled by TF since we are getting stream from TF.
+  // it is safe for CPU pointer array (buffers) to go out of scope after enqueue
+  trt_execution_context_ptr_->enqueue(num_batch, &buffers[0], *stream, nullptr);
 }
 
 REGISTER_KERNEL_BUILDER(Name("TRTEngineOp").Device(DEVICE_GPU), TRTEngineOp);
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.cc b/tensorflow/contrib/tensorrt/log/trt_logger.cc
index dda0dc9e71..7add8cb8b3 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.cc
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.cc
@@ -27,19 +27,19 @@ void Logger::log(Severity severity, const char* msg) {
   // Suppress info-level messages
   switch (severity) {
     case Severity::kINFO: {  // Mark TRT info messages as debug!
-      VLOG(2) << name_ << " " << msg;
+      VLOG(2) << msg;
       break;
     }
     case Severity::kWARNING: {
-      LOG(WARNING) << name_ << " " << msg;
+      LOG(WARNING) << msg;
       break;
     }
     case Severity::kERROR: {
-      LOG(ERROR) << name_ << " " << msg;
+      LOG(ERROR) << msg;
       break;
     }
     case Severity::kINTERNAL_ERROR: {
-      LOG(FATAL) << name_ << " " << msg;
+      LOG(FATAL) << msg;
       break;
     }
     // This is useless for now. But would catch it in future if enum changes. It
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/contrib/tensorrt/log/trt_logger.h
index 7f3544f8cf..d71f66b933 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.h
@@ -27,11 +27,9 @@ namespace tensorrt {
 
 // Logger for GIE info/warning/errors
 class Logger : public nvinfer1::ILogger {
- public:
-  Logger(string name = "DefaultLogger") : name_(name){};
+ private:
   void log(nvinfer1::ILogger::Severity severity, const char* msg) override;
 
- private:
   string name_;
 };
 
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 0b2321b5fc..7e050a768c 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -20,6 +20,5 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
-from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 666220d78c..9454862f85 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -20,17 +20,11 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 import six as _six
-from tensorflow.contrib.tensorrt.wrap_conversion import calib_convert
 from tensorflow.contrib.tensorrt.wrap_conversion import trt_convert
 from tensorflow.core.framework import graph_pb2
-from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import errors_impl as _impl
-from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
-from tensorflow.python.grappler import tf_optimizer
-from tensorflow.python.util import compat
-# pylint: enable=unused-import,line-too-long
 
 
 # TODO(skama): get outputs from session when implemented as c++
@@ -38,33 +32,22 @@ from tensorflow.python.util import compat
 def create_inference_graph(input_graph_def,
                            outputs,
                            max_batch_size=1,
-                           max_workspace_size_bytes=2 << 20,
-                           precision_mode="FP32",
-                           minimum_segment_size=3):
+                           max_workspace_size_bytes=2 << 20):
   """Python wrapper for the TRT transormation.
 
+
   Args:
     input_graph_def: GraphDef object containing a model to be transformed.
-    outputs: list of tensors or node names for the model outputs.
+    outputs: List of tensors or node names for the model outputs.
     max_batch_size: max size for the input batch
     max_workspace_size_bytes: parameter to control memory allocation (in Bytes)
-    precision_mode: one of 'FP32', 'FP16' and 'INT8'
-    minimum_segment_size: the minimum number of nodes required for a subgraph to
-      be replaced by TRTEngineOp.
 
   Returns:
     New GraphDef with TRTEngineOps placed in graph replacing subgraphs.
 
   Raises:
-    ValueError: if the provided precision mode is invalid.
     RuntimeError: if the returned status message is malformed.
   """
-  supported_precision_modes = {"FP32": 0, "FP16": 1, "INT8": 2}
-  if precision_mode.upper() not in supported_precision_modes:
-    raise ValueError(("precision mode '{}' is not supported."
-                      "It should be one of {}").format(
-                          precision_mode, "{'FP32', 'FP16', 'INT8'}"))
-  mode = supported_precision_modes[precision_mode.upper()]
 
   def py2bytes(inp):
     return inp
@@ -100,7 +83,7 @@ def create_inference_graph(input_graph_def,
   # pair or strings where first one is encoded status and the second
   # one is the transformed graphs protobuf string.
   out = trt_convert(input_graph_def_str, out_names, max_batch_size,
-                    max_workspace_size_bytes, mode, minimum_segment_size)
+                    max_workspace_size_bytes)
   status = to_string(out[0])
   output_graph_def_string = out[1]
   del input_graph_def_str  # Save some memory
@@ -118,46 +101,3 @@ def create_inference_graph(input_graph_def,
   output_graph_def.ParseFromString(output_graph_def_string)
   del output_graph_def_string  # Save some memory
   return output_graph_def
-
-
-def calib_graph_to_infer_graph(calibration_graph_def):
-  """Convert an existing calibration graph to inference graph.
-
-  Args:
-    calibration_graph_def: the calibration GraphDef object with calibration data
-  Returns:
-    New GraphDef with TRTEngineOps placed in graph replacing calibration nodes.
-  Raises:
-    RuntimeError: if the returned status message is malformed.
-  """
-
-  def py2string(inp):
-    return inp
-
-  def py3string(inp):
-    return inp.decode("utf-8")
-
-  if _six.PY2:
-    to_string = py2string
-  else:
-    to_string = py3string
-
-  graph_str = calibration_graph_def.SerializeToString()
-  out = calib_convert(graph_str)
-  status = to_string(out[0])
-  output_graph_def_string = out[1]
-  del graph_str  # Save some memory
-  if len(status) < 2:
-    raise _impl.UnknownError(None, None, status)
-  if status[:2] != "OK":
-    msg = status.split(";")
-    if len(msg) == 1:
-      raise RuntimeError("Status message is malformed {}".format(status))
-    # pylint: disable=protected-access
-    raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
-                                         int(msg[0]))
-    # pylint: enable=protected-access
-  output_graph_def = graph_pb2.GraphDef()
-  output_graph_def.ParseFromString(output_graph_def_string)
-  del output_graph_def_string  # Save some memory
-  return output_graph_def
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
index 74df75902e..3d5cc76c42 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda/include/cuda_runtime_api.h"
+#include "cuda_runtime_api.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -38,18 +38,22 @@ TRTInt8Calibrator::TRTInt8Calibrator(
       done_(false),
       dev_buffers_(dev_buffers),
       calib_running_(false),
-      batch_is_set_(false),
       engine_name_(engine_name) {}
 
-bool TRTInt8Calibrator::setBatch(const std::unordered_map<string, void*>& data,
-                                 const cudaStream_t stream) {
-  tensorflow::mutex_lock lock(cond_mtx_);
-  while ((calib_running_ || batch_is_set_) &&
-         !done_) {  // wait while calibration is running
-    cond_.wait(lock);
-  }
+bool TRTInt8Calibrator::setBatch(
+    const std::unordered_map<string, void*>& data) {
+  // TODO(aaroey): make sure that in future PR:
+  // 1. the mutex_lock is outside of the loop
+  // 2. wait() is used instead of wait_for()
+  // 3. done_ is to be protected by the mutex
+  // 4. the first batch is not missed
   if (done_) return false;
-  CHECK(!calib_running_ && !batch_is_set_);
+  while (calib_running_.load(
+      std::memory_order_acquire)) {  // wait while calibration is running
+    tensorflow::mutex_lock l(cond_mtx_);
+    cond_.wait_for(l, std::chrono::milliseconds(50));
+    if (done_) return false;
+  }
   VLOG(1) << "Set Batch Waiting finished";
   for (const auto it : data) {
     auto devptr = dev_buffers_.find(it.first);
@@ -61,32 +65,27 @@ bool TRTInt8Calibrator::setBatch(const std::unordered_map<string, void*>& data,
 
     // TODO(aaroey): we should not use sync copy on default stream. Make sure
     // stream->ThenMemcpy() is used in future PRs.
-    // TODO(sami,aaroey): Need to figureout a way to ensure synchronization
-    // between stream, perhaps using a tensor?
-    auto status = cudaMemcpyAsync(d.first, it.second, d.second,
-                                  cudaMemcpyDeviceToDevice, stream);
+    auto status =
+        cudaMemcpy(d.first, it.second, d.second, cudaMemcpyDeviceToDevice);
     if (status != cudaSuccess) {
       LOG(FATAL) << "cudaMemcpy " << engine_name_ << " for '" << it.first
                  << "' failed with " << status;
     }
   }
-
-  // TODO(Sami, aaorey): Find an alternative way!
-  cudaStreamSynchronize(
-      stream);  // we have to wait for the stream before returning!
-  batch_is_set_ = true;
+  calib_running_.store(true, std::memory_order_release);  // release builder
   cond_.notify_all();
   return true;
 }
 
 bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
                                  int num_bindings) {
-  tensorflow::mutex_lock lock(cond_mtx_);
-  calib_running_ = false;
+  calib_running_.store(false, std::memory_order_release);  // wait for new batch
   cond_.notify_all();
-  while ((!batch_is_set_ && !done_)) {  // wait until new batch arrives
-    cond_.wait(lock);
-
+  while (!calib_running_.load(
+      std::memory_order_acquire)) {  // wait until new batch arrives
+    tensorflow::mutex_lock l(cond_mtx_);
+    cond_.wait_for(l, std::chrono::milliseconds(50));
+    if (done_) return false;
   }
   if (done_) {
     return false;
@@ -101,8 +100,6 @@ bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
 
     bindings[i] = it->second.first;
   }
-  batch_is_set_ = false;
-  calib_running_ = true;
   return true;
 }
 
@@ -110,12 +107,6 @@ const void* TRTInt8Calibrator::readCalibrationCache(std::size_t& length) {
   return nullptr;
 }
 
-void TRTInt8Calibrator::setDone() {
-  tensorflow::mutex_lock lock(cond_mtx_);
-  done_ = true;
-  cond_.notify_all();
-}
-
 void TRTInt8Calibrator::writeCalibrationCache(const void* ptr,
                                               std::size_t length) {}
 TRTInt8Calibrator::~TRTInt8Calibrator() {
@@ -124,6 +115,5 @@ TRTInt8Calibrator::~TRTInt8Calibrator() {
 
 }  // namespace tensorrt
 }  // namespace tensorflow
-
 #endif
 #endif
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
index d77aa2c5ab..8830f7efe7 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
@@ -24,10 +24,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-
-#include "cuda/include/cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
-
 namespace tensorflow {
 namespace tensorrt {
 // This class provides a 1 element queue to match TFs push model to
@@ -42,9 +39,8 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   int getBatchSize() const override;
   bool getBatch(void* bindings[], const char* names[],
                 int num_bindings) override;
-  bool setBatch(const std::unordered_map<string, void*>& data,
-                const cudaStream_t stream);
-  void setDone();
+  bool setBatch(const std::unordered_map<string, void*>& data);
+  void setDone() { done_ = true; }
   const void* readCalibrationCache(std::size_t& length) override;
   void writeCalibrationCache(const void* ptr, std::size_t length) override;
   ~TRTInt8Calibrator();
@@ -59,14 +55,11 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   const std::unordered_map<string, std::pair<void*, size_t>>
       dev_buffers_;  // map to keep tensorrt input buffers and sizes keyed with
                      // buffer names
-  bool calib_running_;
-  bool batch_is_set_;
+  std::atomic_bool calib_running_;
   string engine_name_;
 };
-
 }  // namespace tensorrt
 }  // namespace tensorflow
-
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
 #endif
 #endif
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index 0b661bd536..c78f6f2224 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -60,7 +60,6 @@ def get_simple_graph_def():
 
 
 def run_graph(gdef, dumm_inp):
-  """Run given graphdef once."""
   gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
   ops.reset_default_graph()
   g = ops.Graph()
@@ -75,65 +74,15 @@ def run_graph(gdef, dumm_inp):
   return val
 
 
-# Use real data that is representatitive of the inference dataset
-# for calibration. For this test script it is random data.
-def run_calibration(gdef, dumm_inp):
-  """Run given calibration graph multiple times."""
-  gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
-  ops.reset_default_graph()
-  g = ops.Graph()
-  with g.as_default():
-    inp, out = importer.import_graph_def(
-        graph_def=gdef, return_elements=["input", "output"])
-    inp = inp.outputs[0]
-    out = out.outputs[0]
-  with csess.Session(
-      config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess:
-    # run over real calibration data here, we are mimicking a calibration set of
-    # 30 different batches. Use as much calibration data as you want
-    for _ in range(30):
-      val = sess.run(out, {inp: dumm_inp})
-  return val
-
-
 if "__main__" in __name__:
   inp_dims = (100, 24, 24, 2)
   dummy_input = np.random.random_sample(inp_dims)
-  orig_graph = get_simple_graph_def()  # use a frozen graph for inference
+  gdef = get_simple_graph_def()
   # Get optimized graph
-  trt_graph = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="FP32",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2  # minimum number of nodes in an engine
-  )
-  o1 = run_graph(orig_graph, dummy_input)
+  trt_graph = trt.create_inference_graph(gdef, ["output"], inp_dims[0])
+  o1 = run_graph(gdef, dummy_input)
   o2 = run_graph(trt_graph, dummy_input)
   o3 = run_graph(trt_graph, dummy_input)
   assert np.array_equal(o1, o2)
   assert np.array_equal(o3, o2)  # sanity check
-  fp16_graph = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="FP16",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2  # minimum number of nodes in an engine
-  )
-  int8_calib_gdef = trt.create_inference_graph(
-      input_graph_def=orig_graph,
-      outputs=["output"],
-      max_batch_size=inp_dims[0],
-      max_workspace_size_bytes=1 << 25,
-      precision_mode="INT8",  # TRT Engine precision "FP32","FP16" or "INT8"
-      minimum_segment_size=2  # minimum number of nodes in an engine
-  )
-  o4 = run_graph(fp16_graph, dummy_input)
-  _ = run_calibration(int8_calib_gdef, dummy_input)
-  int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
-  o5 = run_graph(int8_graph, dummy_input)
-  assert np.allclose(o1, o4)
-  assert np.allclose(o1, o5)
   print("Pass")
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index 46480e99a1..d679945d56 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -64,17 +64,13 @@ PyObject* pair_helper(std::pair<string, string>* in) {
 %ignoreall
 %unignore tensorflow;
 %unignore trt_convert;
-%unignore calib_convert;
 
 %{
-
 std::pair<string, string> trt_convert(
     string graph_def_string,  // The serialized GraphDef string.
     std::vector<string> output_names,
     size_t max_batch_size,
-    size_t max_workspace_size_bytes,
-    int precision_mode,
-    int minimum_segment_size
+    size_t max_workspace_size_bytes
     // Unfortunately we can't use TF_Status here since it
     // is in c/c_api and brings in a lot of other libraries
     // which in turn declare ops. These ops are included
@@ -94,64 +90,16 @@ std::pair<string, string> trt_convert(
     return std::pair<string, string>{out_status, ""};
   }
 
-  if(precision_mode < 0 || precision_mode > 2){
-    out_status = "InvalidArgument;Invalid precision_mode";
-    return std::pair<string, string>{out_status, ""};
-  }
   if (!output_names.size()) {
     out_status = "InvalidArgument;Size of the output_names vector is 0";
     return std::pair<string, string>{out_status, ""};
+    // return "";
   }
   tensorflow::GraphDef outGraph;
   tensorflow::Status conversion_status =
       tensorflow::tensorrt::convert::ConvertGraphDefToTensorRT(
           graph_def, output_names, max_batch_size, max_workspace_size_bytes,
-          &outGraph, precision_mode, minimum_segment_size);
-  if (!conversion_status.ok()) {
-    auto retCode = (int)conversion_status.code();
-    char buff[2000];
-    snprintf(buff, 2000, "%d;%s", retCode,
-             conversion_status.error_message().c_str());
-    out_status = buff;
-    return std::pair<string, string>{out_status, ""};
-  }
-  string result;
-  if (!outGraph.SerializeToString(&result)) {
-    out_status = "InvalidArgument;Couldn't serialize output as a GraphDef";
-    return std::pair<string, string>{out_status, ""};
-  }
-  out_status = "OK;All good!";
-  return std::pair<string, string>{out_status, result};
-#else
-  // Returns FAILED_PRECONDITION.
-  return std::pair<string, string>{"9;TensorRT is not enabled!", ""};
-#endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
-}
-
-std::pair<string, string> calib_convert(string graph_def_string  //  const tensorflow::GraphDef&
-    // unfortunately we can't use TF_Status here since it
-    // is in c/c_api and brings in a lot of other libraries
-    // which in turn declare ops. These ops are included
-    // statically in our library and cause an abort when
-    // module is loaded due to double registration
-    // until Tensorflow properly exposes these headers
-    // we have to work around this by returning a string
-    // and converting it to exception on python side.
-    //,TF_Status* out_status) {
-) {
-#if GOOGLE_CUDA && GOOGLE_TENSORRT
-  string out_status;
-
-  tensorflow::GraphDef graph_def;
-  if (!graph_def.ParseFromString(graph_def_string)) {
-    out_status = "InvalidArgument;Couldn't interpret input as a GraphDef";
-    return std::pair<string, string>{out_status, ""};
-  }
-
-  tensorflow::GraphDef outGraph;
-  tensorflow::Status conversion_status =
-      tensorflow::tensorrt::convert::ConvertCalibGraphToInferGraph(graph_def,
-                                                                   &outGraph);
+          &outGraph);
   if (!conversion_status.ok()) {
     auto retCode = (int)conversion_status.code();
     char buff[2000];
@@ -174,13 +122,10 @@ std::pair<string, string> calib_convert(string graph_def_string  //  const tenso
 }
 %}
 
-std::pair<string, string> calib_convert(string graph_def_string);
-
 std::pair<string, string> trt_convert(string graph_def_string,
                                       std::vector<string> output_names,
                                       size_t max_batch_size,
-                                      size_t max_workspace_size_bytes,
-                                      int precision_mode, int minimum_segment_size);
+                                      size_t max_workspace_size_bytes);
 
 
 %unignoreall
diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 70bf67c779..bb86ecb220 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -25,10 +25,7 @@ py_test(
     srcs = ["predict_test.py"],
     data = ["data/period_trend.csv"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",  # b/67513579
-    ],
+    tags = ["notsan"],  # b/67513579
     deps = [
         ":predict",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 64f5cd8357..ed3ed4c0e1 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -156,7 +156,9 @@ py_test(
         "head_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_pip_gpu"],  # b/63391119
+    tags = [
+        "no_pip_gpu",  # b/63391119
+    ],
     deps = [
         ":feature_keys",
         ":head",
@@ -425,7 +427,6 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip_gpu",  # b/63391119
-        "no_windows",  # TODO: needs investigation on Windows
     ],
     deps = [
         ":feature_keys",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index 07df7bc9a5..c86d06e923 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -40,7 +40,6 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["state_space_model_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":state_space_model",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index f9d433a45b..ed930e44e8 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -225,7 +225,6 @@ tf_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:layers",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 tf_py_test(
diff --git a/tensorflow/contrib/util/loader.py b/tensorflow/contrib/util/loader.py
index dca01d26f4..f4283cd9ed 100644
--- a/tensorflow/contrib/util/loader.py
+++ b/tensorflow/contrib/util/loader.py
@@ -42,10 +42,9 @@ def load_op_library(path):
     plugin.
   """
   if os.name == 'nt':
-    # To avoid making every user_ops aware of windows, re-write
-    # the file extension from .so to .dll if .so file doesn't exist.
-    if not os.path.exists(path):
-      path = re.sub(r'\.so$', '.dll', path)
+    # To avoid makeing every user_ops aware of windows, re-write
+    # the file extension from .so to .dll.
+    path = re.sub(r'\.so$', '.dll', path)
 
     # Currently we have only some user_ops as dlls on windows - don't try
     # to load them if the dll is not found.
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 5c9fd2f406..352e183104 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3327,10 +3327,6 @@ tf_cc_test(
     size = "small",
     srcs = ["common_runtime/function_test.cc"],
     linkstatic = tf_kernel_tests_linkstatic(),
-    tags = [
-        "manual",
-        "no_oss",
-    ],
     deps = [
         ":core",
         ":core_cpu",
diff --git a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
deleted file mode 100644
index 9fabe7863e..0000000000
--- a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
+++ /dev/null
@@ -1,18 +0,0 @@
-op {
-  graph_op_name: "SlideDataset"
-  in_arg {
-    name: "window_size"
-    description: <<END
-A scalar representing the number of elements in the
-sliding window.
-END
-  }
-  in_arg {
-    name: "stride"
-    description: <<END
-A scalar representing the steps moving the sliding window
-forward in one iteration. It must be in `[1, window_size)`.
-END
-  }
-  summary: "Creates a dataset that passes a sliding window over `input_dataset`."
-}
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.cc b/tensorflow/core/distributed_runtime/tensor_coding.cc
index fe2d1a1293..34a4013547 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.cc
+++ b/tensorflow/core/distributed_runtime/tensor_coding.cc
@@ -81,7 +81,7 @@ void TensorResponse::InitPartial(const RecvTensorResponse& response) {
 Status TensorResponse::ParseFrom(Source* source) {
   if (!on_host_) {
     protobuf::io::CodedInputStream input(source->contents());
-    input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
+    input.SetTotalBytesLimit(INT_MAX);  // Unlimited
 
     // Pre-parse into local storage, then delegate to device.
     if (!meta_.ParseFromCodedStream(&input) || !input.ConsumedEntireMessage()) {
@@ -217,7 +217,7 @@ bool TensorResponse::ParseTensorSubmessage(
 
 bool TensorResponse::ParseFast(Source* source) {
   protobuf::io::CodedInputStream input(source->contents());
-  input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
+  input.SetTotalBytesLimit(INT_MAX);  // Unlimited
   while (true) {
     auto p = input.ReadTagWithCutoff(127);
     int tag = GetTagFieldNumber(p.first);
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index cfe23d1ffe..beaf0adbc5 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -474,11 +474,11 @@ class GraphDatasetBase : public DatasetBase {
   }
 
   // Key for storing the Dataset graph in the serialized format.
-  TF_EXPORT static const char kDatasetGraphKey[];
+  static const char kDatasetGraphKey[];
 
   // Key for storing the output node of the Dataset graph in the serialized
   // format.
-  TF_EXPORT static const char kDatasetGraphOutputNodeKey[];
+  static const char kDatasetGraphOutputNodeKey[];
 
  private:
   Status Serialize(OpKernelContext* ctx, string* serialized_graph_def,
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 1507b6eae2..02038c5d77 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2492,10 +2492,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       mkl_op_registry::GetMklOpName(csinfo_.identity),
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.lrn, mkl_op_registry::GetMklOpName(csinfo_.lrn),
-                      CopyAttrsLRN, LrnRewrite});
+                      CopyAttrsLRN, AlwaysRewrite});
     rinfo_.push_back({csinfo_.lrn_grad,
                       mkl_op_registry::GetMklOpName(csinfo_.lrn_grad),
-                      CopyAttrsLRN, LrnRewrite});
+                      CopyAttrsLRN, AlwaysRewrite});
     rinfo_.push_back({csinfo_.max_pool,
                       mkl_op_registry::GetMklOpName(csinfo_.max_pool),
                       CopyAttrsPooling, NonDepthBatchWisePoolRewrite});
@@ -2865,28 +2865,6 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return false;
   }
 
-  // If the depth_radius of LRN is not 2, then MKL DNN takes unoptimized 
-  // path. The unoptimized path is slow. Thus we dont rewrite the node 
-  // and use default Eigen. But for depth_radius=2, MKL DNN optimized 
-  // path is taken, i.e., eigen node is rewritten by MKl DNN node.
-  static bool LrnRewrite(const Node* n) {
-    CHECK_NOTNULL(n);
-
-    int depth_radius;
-    CHECK_EQ(GetNodeAttr(n->def(), "depth_radius", &depth_radius).ok(), true);
-
-    // if the depth_radius of LRN is not 2, don't rewrite the node by MKL DNN
-    // and use eigen node instead 
-    if (depth_radius == 2) {
-      return true;
-    }
-    VLOG(1) << "LrnRewrite: The model sets depth_radius as not 2 which"
-            << "case is not optimized by Intel MKL, thus using Eigen op"
-            << "for LRN " ; 
-
-    return false;
-  }
-
   static bool AddNRewrite(const Node* n) {
     CHECK_NOTNULL(n);
 
@@ -3550,13 +3528,11 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   string data_format;
   string padding;
   std::vector<int32> strides;
-  std::vector<int32> dilations;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
-  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
   TF_CHECK_OK(
@@ -3565,7 +3541,6 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
-  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
@@ -3803,14 +3778,12 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   DataType T_pred, T_succ;
   string padding;
   std::vector<int32> strides;
-  std::vector<int32> dilations;
   string data_format_pred, data_format_succ;
   bool use_cudnn_on_gnu;
   TF_CHECK_OK(GetNodeAttr(pred->def(), "T", &T_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "T", &T_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "strides", &strides));
-  TF_CHECK_OK(GetNodeAttr(pred->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "data_format", &data_format_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 8f13c4a702..244653504d 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -25,7 +25,6 @@ limitations under the License.
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
-#include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -409,7 +408,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
       frame_children_[frame_ids[0]].insert(frame_ids[1]);
       frame_parent_[frame_ids.back()] = frame_ids[frame_ids.size() - 2];
     }
-    if (frame_ids.size() >= 1) {
+    if (!frame_ids.empty()) {
       frame_children_.insert(std::make_pair(frame_ids.back(), empty_set_));
       if (node->op() == "LoopCond") {
         if (loop_cond_.count(frame_ids.back())) {
@@ -428,7 +427,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
   }
 
   for (auto it = frame_children_.begin(); it != frame_children_.end(); ++it) {
-    if (it->second.size() == 0) {
+    if (it->second.empty()) {
       worklist.push_back(it->first);
     }
   }
@@ -441,7 +440,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
     if (parent_it != frame_parent_.end()) {
       int parent_id = parent_it->second;
       frame_children_[parent_id].erase(frame_id);
-      if (frame_children_[parent_id].size() == 0) {
+      if (frame_children_[parent_id].empty()) {
         worklist.push_back(parent_id);
       }
     }
@@ -465,6 +464,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
 Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                GraphDef* optimized_graph) {
   TF_RETURN_IF_ERROR(RemoveStackOps(item.graph, optimized_graph));
+
   optimized_graph_ = optimized_graph;
 
   // Set up helper data structures.
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 2e39f25fc1..48d5955ad1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5157,6 +5157,7 @@ tf_kernel_library(
     ],
     hdrs = [
         "meta_support.h",
+        "quantization_utils.h",
         "reference_gemm.h",
     ],
     deps = [
@@ -5231,7 +5232,6 @@ tf_cc_test(
     name = "quantization_utils_test",
     srcs = ["quantization_utils_test.cc"],
     deps = [
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5294,7 +5294,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5356,7 +5355,6 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5379,7 +5377,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5444,7 +5441,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5465,7 +5461,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5505,7 +5500,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5562,7 +5556,6 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5585,7 +5578,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5622,7 +5614,6 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5644,7 +5635,6 @@ tf_cc_test(
     deps = [
         ":batch_norm_op",
         ":ops_testutil",
-        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 01754ec21a..484d4f88d6 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -113,19 +113,6 @@ tf_kernel_library(
     ],
 )
 
-tf_kernel_library(
-    name = "slide_dataset_op",
-    srcs = ["slide_dataset_op.cc"],
-    deps = [
-        ":dataset",
-        "//tensorflow/core:dataset_ops_op_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:batch_util",
-    ],
-)
-
 tf_kernel_library(
     name = "padded_batch_dataset_op",
     srcs = ["padded_batch_dataset_op.cc"],
@@ -551,7 +538,6 @@ tf_kernel_library(
         ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
-        ":slide_dataset_op",
         ":sparse_tensor_slice_dataset_op",
         ":sql_dataset_ops",
         ":stats_aggregator_ops",
diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/slide_dataset_op.cc
deleted file mode 100644
index 4f3537b691..0000000000
--- a/tensorflow/core/kernels/data/slide_dataset_op.cc
+++ /dev/null
@@ -1,252 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include "tensorflow/core/framework/partial_tensor_shape.h"
-#include "tensorflow/core/framework/tensor.h"
-#include "tensorflow/core/kernels/batch_util.h"
-#include "tensorflow/core/kernels/data/dataset.h"
-
-namespace tensorflow {
-
-namespace {
-
-// See documentation in ../ops/dataset_ops.cc for a high-level
-// description of the following op.
-
-class SlideDatasetOp : public UnaryDatasetOpKernel {
- public:
-  explicit SlideDatasetOp(OpKernelConstruction* ctx)
-      : UnaryDatasetOpKernel(ctx) {}
-
-  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
-                   DatasetBase** output) override {
-    int64 window_size = 0;
-    int64 stride = 1;
-    OP_REQUIRES_OK(ctx,
-                   ParseScalarArgument<int64>(ctx, "window_size", &window_size));
-    OP_REQUIRES_OK(ctx,
-                   ParseScalarArgument<int64>(ctx, "stride", &stride));
-    OP_REQUIRES(
-        ctx, window_size > 0,
-        errors::InvalidArgument("Window size must be greater than zero."));
-    OP_REQUIRES(
-        ctx, stride > 0 && stride < window_size,
-        errors::InvalidArgument("Stride must be in [1, window_size)."));
-
-    *output = new Dataset(ctx, window_size, stride, input);
-  }
-
- private:
-  class Dataset : public GraphDatasetBase {
-   public:
-    Dataset(OpKernelContext* ctx, int64 window_size, int64 stride, const DatasetBase* input)
-        : GraphDatasetBase(ctx), window_size_(window_size), stride_(stride), input_(input) {
-      input_->Ref();
-
-      const auto& input_shapes = input_->output_shapes();
-      output_shapes_.reserve(input_shapes.size());
-      for (const auto& input_shape : input_shapes) {
-        output_shapes_.emplace_back(
-            PartialTensorShape({-1}).Concatenate(input_shape));
-      }
-    }
-
-    ~Dataset() override { input_->Unref(); }
-
-    std::unique_ptr<IteratorBase> MakeIterator(
-        const string& prefix) const override {
-      return std::unique_ptr<IteratorBase>(new Iterator(
-          Iterator::Params{this, strings::StrCat(prefix, "::Slide")}));
-    }
-
-    const DataTypeVector& output_dtypes() const override {
-      return input_->output_dtypes();
-    }
-
-    const std::vector<PartialTensorShape>& output_shapes() const override {
-      return output_shapes_;
-    }
-
-    string DebugString() override {
-      return strings::StrCat("SlideDatasetOp(", window_size_, ", ", stride_, ")::Dataset");
-    }
-
-   protected:
-    Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b,
-                              Node** output) const override {
-      Node* input_graph_node = nullptr;
-      TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node));
-      Node* window_size = nullptr;
-      Node* stride = nullptr;
-      TF_RETURN_IF_ERROR(b->AddScalar(window_size_, &window_size));
-      TF_RETURN_IF_ERROR(b->AddScalar(stride_, &stride));
-      TF_RETURN_IF_ERROR(
-          b->AddDataset(this, {input_graph_node, window_size, stride}, output));
-      return Status::OK();
-    }
-
-   private:
-
-    class Iterator : public DatasetIterator<Dataset> {
-     public:
-      explicit Iterator(const Params& params)
-          : DatasetIterator<Dataset>(params),
-            input_impl_(params.dataset->input_->MakeIterator(params.prefix)) {}
-
-      Status GetNextInternal(IteratorContext* ctx,
-                             std::vector<Tensor>* out_tensors,
-                             bool* end_of_sequence) override {
-        const int64 window_size = dataset()->window_size_;
-        const int64 stride = dataset()->stride_;
-        std::vector<std::vector<Tensor>> batch_elements;
-        {
-          mutex_lock l(mu_);
-          if (!input_impl_) {
-            *end_of_sequence = true;
-            return Status::OK();
-          }
-          batch_elements.reserve(window_size);
-          const bool first_call = cache_.empty();
-          if (first_call) {
-            cache_.reserve(window_size);
-          } else {
-            // Reuse cache in the previous iteration.
-            cache_.swap(batch_elements);
-          }
-          // Fill up with new elements.
-          *end_of_sequence = false;
-          for (size_t i = batch_elements.size(); i < window_size && !*end_of_sequence;
-              ++i) {
-            std::vector<Tensor> batch_element_tuple;
-            TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &batch_element_tuple,
-                                                    end_of_sequence));
-            if (!*end_of_sequence) {
-              batch_elements.push_back(std::move(batch_element_tuple));
-            } else {
-              input_impl_.reset();
-            }
-          }
-          // Drop the final smaller blocks.
-          if (batch_elements.size() < window_size) {
-            DCHECK(*end_of_sequence);
-            return Status::OK();
-          }
-          // Cache the data used for the next iteration.
-          for (size_t i = stride; i < window_size; ++i) {
-            cache_.emplace_back(batch_elements[i]);
-          }
-        }
-
-        // Construct output tensors.
-        // Those codes below are copied from batch_dataset_op.cc.
-        const size_t num_tuple_components = batch_elements[0].size();
-        const int64 num_batch_elements = batch_elements.size();
-        for (size_t component_index = 0; component_index < num_tuple_components;
-             ++component_index) {
-          const Tensor& first_element = batch_elements[0][component_index];
-          TensorShape batch_component_shape({num_batch_elements});
-          batch_component_shape.AppendShape(first_element.shape());
-          Tensor batch_component(cpu_allocator(), first_element.dtype(),
-                                 batch_component_shape);
-          // Build the output tuple component by copying one slice
-          // from each input element in the batch.
-          for (size_t i = 0; i < num_batch_elements; ++i) {
-            if (batch_elements[i][component_index].shape() !=
-                first_element.shape()) {
-              return errors::InvalidArgument(
-                  "Cannot batch tensors with different shapes in component ",
-                  component_index, ". First element had shape ",
-                  first_element.shape().DebugString(), " and element ", i,
-                  " had shape ",
-                  batch_elements[i][component_index].shape().DebugString(),
-                  ".");
-            }
-            TF_RETURN_IF_ERROR(batch_util::CopyElementToSlice(
-                std::move(batch_elements[i][component_index]), &batch_component,
-                i));
-          }
-          out_tensors->emplace_back(std::move(batch_component));
-        }
-        *end_of_sequence = false;
-        return Status::OK();
-      }
-
-     protected:
-      Status SaveInternal(IteratorStateWriter* writer) override {
-        mutex_lock l(mu_);
-        if (!input_impl_) {
-          TF_RETURN_IF_ERROR(
-              writer->WriteScalar(full_name("input_impl_empty"), ""));
-        } else {
-          TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_));
-        }
-        // Save cache.
-        TF_RETURN_IF_ERROR(
-            writer->WriteScalar(strings::StrCat("cache_size"), cache_.size()));
-        for (int64 i = 0; i < cache_.size(); i++) {
-          TF_RETURN_IF_ERROR(writer->WriteScalar(
-              strings::StrCat("cache[", i, "]_size"), cache_[i].size()));
-          for (int64 j = 0; j < cache_[i].size(); j++) {
-            TF_RETURN_IF_ERROR(writer->WriteTensor(
-                strings::StrCat("cache[", i, "][", j, "]"), cache_[i][j]));
-          }
-        }
-        return Status::OK();
-      }
-
-      Status RestoreInternal(IteratorContext* ctx,
-                             IteratorStateReader* reader) override {
-        mutex_lock l(mu_);
-        if (!reader->Contains(full_name("input_impl_empty"))) {
-          TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_));
-        } else {
-          input_impl_.reset();
-        }
-        // Restore cache.
-        int64 cache_size;
-        TF_RETURN_IF_ERROR(
-            reader->ReadScalar(strings::StrCat("cache_size"), &cache_size));
-        cache_.resize(cache_size);
-        for (int64 i = 0; i < cache_size; i++) {
-          int64 vector_size;
-          TF_RETURN_IF_ERROR(reader->ReadScalar(
-              strings::StrCat("cache[", i, "]_size"), &vector_size));
-          cache_[i].resize(vector_size);
-          for (int64 j = 0; j < vector_size; j++) {
-            TF_RETURN_IF_ERROR(reader->ReadTensor(
-                strings::StrCat("cache[", i, "][", j, "]"), &cache_[i][j]));
-          }
-        }
-        return Status::OK();
-      }
-
-     private:
-      mutex mu_;
-      std::vector<std::vector<Tensor>> cache_ GUARDED_BY(mu_);
-      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
-    };
-
-    const int64 window_size_;
-    const int64 stride_;
-    const DatasetBase* const input_;
-    std::vector<PartialTensorShape> output_shapes_;
-  };
-};
-
-REGISTER_KERNEL_BUILDER(Name("SlideDataset").Device(DEVICE_CPU),
-                        SlideDatasetOp);
-
-}  // namespace
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/depthtospace_op.cc b/tensorflow/core/kernels/depthtospace_op.cc
index b74a09e2cb..39aa3e9eb0 100644
--- a/tensorflow/core/kernels/depthtospace_op.cc
+++ b/tensorflow/core/kernels/depthtospace_op.cc
@@ -187,9 +187,6 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     DepthToSpaceOp<GPUDevice, float>);
-REGISTER_KERNEL_BUILDER(
-    Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
-    DepthToSpaceOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     DepthToSpaceOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
index 0656081177..184c703599 100644
--- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
@@ -238,12 +238,6 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
-// Instantiate the GPU implementations for Eigen::half.
-template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
-                                               FORMAT_NCHW>;
-template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
-                                               FORMAT_NHWC>;
-
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 7688305019..108d59db2c 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -45,7 +45,6 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/kernels:cwise_op",
-        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/kernels:reduction_ops",
         "//tensorflow/core/kernels:remote_fused_graph_execute_utils",
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index e0706568b1..1401bc65a4 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -444,7 +444,6 @@ class MklConv2DCustomBackpropFilterOp
   ~MklConv2DCustomBackpropFilterOp() {}
 
  private:
-  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -493,9 +492,7 @@ class MklConv2DCustomBackpropFilterOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor,
-                       const memory::dims& strides,
-                       const memory::dims& dilations,
+                       Tensor** output_tensor, const memory::dims& strides,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -521,32 +518,31 @@ class MklConv2DCustomBackpropFilterOp
       bias_grad->SetOpMemDesc(bias_grad_dims, memory::format::x);
     }
 
-    if (biasEnabled && (bias_grad != nullptr)) {
-      // Create convolution backward weights with bias primitive.
-      // Use dilated convolution in case dilate rates are greater than zero.
-      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
-        convolution_backward_weights::desc(convolution_direct,
-                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
-                                  bias_grad->GetOpMemDesc(),
-                                  outbackprop->GetOpMemDesc(), strides,
-                                  dilations, padding_l, padding_r, padding) :
-        convolution_backward_weights::desc(convolution_direct,
-                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
-                                  bias_grad->GetOpMemDesc(),
-                                  outbackprop->GetOpMemDesc(),
-                                  strides, padding_l, padding_r, padding);
-      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
-                                                            cpu_engine,
-                                                            conv_fwd_pd);
-
-      // Allocate output tensor.
-      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
-                           bwd_output_format, output_tensor);
-
-      CHECK_NOTNULL(*output_tensor);
-      // Set buffer handle using allocated output tensor.
-      output->SetUsrMemDataHandle(*output_tensor);
+    // Create convolution backward weights primitive.
+    auto bwd_desc =
+        (biasEnabled && (bias_grad != nullptr))
+            ? convolution_backward_weights::desc(
+                  convolution_direct, input->GetOpMemDesc(),
+                  output->GetOpMemDesc(), bias_grad->GetOpMemDesc(),
+                  outbackprop->GetOpMemDesc(), strides, padding_l, padding_r,
+                  padding)
+            : convolution_backward_weights::desc(
+                  convolution_direct, input->GetOpMemDesc(),
+                  output->GetOpMemDesc(), outbackprop->GetOpMemDesc(), strides,
+                  padding_l, padding_r, padding);
+
+    auto bwd_pd = convolution_backward_weights::primitive_desc(
+        bwd_desc, cpu_engine, conv_fwd_pd);
+
+    // Allocate output tensor.
+    AllocateOutputTensor(context, bwd_pd, bwd_output_dims, bwd_output_format,
+                         output_tensor);
+
+    CHECK_NOTNULL(*output_tensor);
+    // Set buffer handle using allocated output tensor.
+    output->SetUsrMemDataHandle(*output_tensor);
 
+    if (biasEnabled && (bias_grad != nullptr)) {
       // Allocate bias_grad tensor
       TensorShape bias_grad_shape({depth});
       Tensor* bias_grad_tensor = nullptr;
@@ -557,32 +553,11 @@ class MklConv2DCustomBackpropFilterOp
           memory::desc({bias_grad_dims}, MklDnnType<T>(), memory::format::x);
       bias_grad->SetUsrMem(bias_grad_md, bias_grad_tensor);
       bias_grad->SetUsrMemDataHandle(bias_grad_tensor);
+    }
 
-      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output,
-                                  bias_grad);
+    if (biasEnabled && (bias_grad != nullptr)) {
+      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output, bias_grad);
     } else {
-      // Create convolution backward weights primitive.
-      // Use dilated convolution in case dilate rates are greater than zero.
-      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
-        convolution_backward_weights::desc(convolution_direct,
-                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
-                                  outbackprop->GetOpMemDesc(), strides,
-                                  dilations, padding_l, padding_r, padding) :
-        convolution_backward_weights::desc(convolution_direct,
-                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
-                                  outbackprop->GetOpMemDesc(),
-                                  strides, padding_l, padding_r, padding);
-      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
-                                                            cpu_engine,
-                                                            conv_fwd_pd);
-
-      // Allocate output tensor.
-      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
-                           bwd_output_format, output_tensor);
-
-      CHECK_NOTNULL(*output_tensor);
-      // Set buffer handle using allocated output tensor.
-      output->SetUsrMemDataHandle(*output_tensor);
       PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output);
     }
   }
diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
index d203c04934..eeed009531 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
@@ -369,7 +369,6 @@ class MklConv2DCustomBackpropInputOp
  private:
   const int kInputIndex_Filter = 1, kInputIndex_InputSizes = 0,
             kInputIndex_OutBackProp = 2;
-  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -420,9 +419,7 @@ class MklConv2DCustomBackpropInputOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor,
-                       const memory::dims& strides,
-                       const memory::dims& dilations,
+                       Tensor** output_tensor, const memory::dims& strides,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -435,16 +432,9 @@ class MklConv2DCustomBackpropInputOp
     CHECK_NOTNULL(output_tensor);
 
     // Create convolution backward data primitive.
-    // Use dilated convolution in case dilate rates are greater than zero.
-    auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
-        convolution_backward_data::desc(convolution_direct,
-                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
-                      outbackprop->GetOpMemDesc(), strides,
-                      dilations, padding_l, padding_r, padding):
-        convolution_backward_data::desc(convolution_direct,
-                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
-                      outbackprop->GetOpMemDesc(),
-                      strides, padding_l, padding_r, padding);
+    auto bwd_desc = convolution_backward_data::desc(
+        convolution_direct, output->GetOpMemDesc(), filter->GetOpMemDesc(),
+        outbackprop->GetOpMemDesc(), strides, padding_l, padding_r, padding);
 
     auto bwd_pd = convolution_backward_data::primitive_desc(
         bwd_desc, cpu_engine, conv_fwd_pd);
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index f0818eb96d..1440da8f82 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -493,7 +493,6 @@ class MklConv2DOp : public OpKernel {
   ~MklConv2DOp() {}
 
   explicit MklConv2DOp(OpKernelConstruction* context) : OpKernel(context) {
-    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
     string data_format;
     OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
@@ -510,20 +509,6 @@ class MklConv2DOp : public OpKernel {
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
-    OP_REQUIRES(context, dilations_.size() == 4,
-                errors::InvalidArgument("Sliding window dilations field must "
-                                        "specify 4 dimensions"));
-    const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N');
-    const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C');
-    const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H');
-    const int64 dilation_w = GetTensorDim(dilations_, data_format_, 'W');
-    OP_REQUIRES(context, dilation_n == 1 && dilation_c == 1,
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "dilations in the batch and depth dimensions."));
-    OP_REQUIRES(
-        context, dilation_h > 0 && dilation_w > 0,
-        errors::InvalidArgument("Dilated rates should be larger than 0."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -545,19 +530,17 @@ class MklConv2DOp : public OpKernel {
       MklDnnData<T> filter(&cpu_engine);
       MklDnnData<T> output(&cpu_engine);
 
-      memory::dims src_dims, filter_dims, padding_l, padding_r,
-                   dilations, strides;
+      memory::dims src_dims, filter_dims, padding_l, padding_r, strides;
       memory::dims output_dims_tf_order, output_dims_mkl_order;
 
       // Get shapes of input tensors in MKL-DNN order
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
-                             dilations_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
       auto src_tf_shape = GetTfShape(context, kInputIndex_Src);
       auto filter_tf_shape = GetTfShape(context, kInputIndex_Filter);
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides,
-          &dilations, &output_dims_tf_order, &output_dims_mkl_order,
-          &padding_l, &padding_r);
+          &output_dims_tf_order, &output_dims_mkl_order, &padding_l,
+          &padding_r);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -570,7 +553,6 @@ class MklConv2DOp : public OpKernel {
         //               Need semantics for Null MKL tensor
         MklDnnShape output_mkl_shape;
         output_mkl_shape.SetMklTensor(false);
-
         AllocateOutputSetMklShape(context, kOutputIndex_Dst, &output_tensor,
                                   src_tf_shape, output_mkl_shape);
 
@@ -614,79 +596,55 @@ class MklConv2DOp : public OpKernel {
       filter.SetOpMemDesc(filter_dims, memory::format::any);
       output.SetOpMemDesc(output_dims_mkl_order, memory::format::any);
 
-      // MKLDNN dilation starts from 0.
-      dilations[kDilationH] -= 1;
-      dilations[kDilationW] -= 1;
-
+      // If bias is enabled, then do the same steps as above for bias.
       if (biasEnabled) {
-          // Create convolution primitive with Bias.
-          MklDnnData<T> bias(&cpu_engine);
-          memory::dims bias_size;
-          conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
-          const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
-          bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
-          bias.SetOpMemDesc(bias_size, memory::format::any);
-
-          // Create convolution primitive with Bias.
-          // Use MKLDNN dilated convolution in case of dilated rate (>0).
-          auto conv_desc = (dilations[kDilationH] > 0 ||
-              dilations[kDilationW] > 0) ?
-              convolution_forward::desc(prop_kind::forward,
-                      convolution_direct, src.GetOpMemDesc(),
-                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
-                      output.GetOpMemDesc(), strides, dilations,
-                      padding_l, padding_r,
-                      TFPaddingToMklDnnPadding(padding_)):
-              convolution_forward::desc(prop_kind::forward,
-                      convolution_direct, src.GetOpMemDesc(),
-                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
-                      output.GetOpMemDesc(), strides,
-                      padding_l, padding_r,
-                      TFPaddingToMklDnnPadding(padding_));
-
-          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
-                                                                  cpu_engine);
-          AllocateOutputTensor(context, conv_prim_desc,
-                               output_dims_mkl_order, tf_fmt, &output_tensor);
-          // Set data handle for output.
-          output.SetUsrMemDataHandle(output_tensor);
-
-          Tensor* filter_out_tensor = nullptr;
-          AllocateFilterOutputTensor(context, conv_prim_desc,
-                TFShapeToMklDnnDims(filter_tf_shape),
-                &filter_out_tensor);
-
-          PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
-                               filter_out_tensor);
+        MklDnnData<T> bias(&cpu_engine);
+        memory::dims bias_size;
+        conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
+        const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
+        bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
+        bias.SetOpMemDesc(bias_size, memory::format::any);
+
+        // Create convolution primitive with Bias.
+        auto conv_desc = convolution_forward::desc(
+            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
+            filter.GetOpMemDesc(), bias.GetOpMemDesc(), output.GetOpMemDesc(),
+            strides, padding_l, padding_r, TFPaddingToMklDnnPadding(padding_));
+
+        auto conv_prim_desc =
+            convolution_forward::primitive_desc(conv_desc, cpu_engine);
+        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
+                             tf_fmt, &output_tensor);
+        // Set data handle for output.
+        output.SetUsrMemDataHandle(output_tensor);
+
+        Tensor* filter_out_tensor = nullptr;
+        AllocateFilterOutputTensor(context, conv_prim_desc,
+                                   TFShapeToMklDnnDims(filter_tf_shape),
+                                   &filter_out_tensor);
+
+        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
+                             filter_out_tensor);
       } else {
-          // Create convolution primitive without Bias.
-          // Use MKLDNN dilated convolution in case of dilated rate (>0).
-          auto conv_desc = (dilations[kDilationH] > 0 ||
-            dilations[kDilationW] > 0) ?
-            convolution_forward::desc(prop_kind::forward,
-              convolution_direct, src.GetOpMemDesc(),
-              filter.GetOpMemDesc(), output.GetOpMemDesc(),
-              strides, dilations, padding_l, padding_r,
-              TFPaddingToMklDnnPadding(padding_)):
-          convolution_forward::desc(prop_kind::forward,
-              convolution_direct, src.GetOpMemDesc(),
-              filter.GetOpMemDesc(), output.GetOpMemDesc(),
-              strides, padding_l, padding_r,
-              TFPaddingToMklDnnPadding(padding_));
-
-          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
-                                                                  cpu_engine);
-          AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
-                               tf_fmt, &output_tensor);
-          // Set data handle for output.
-          output.SetUsrMemDataHandle(output_tensor);
-
-          Tensor* filter_out_tensor = nullptr;
-          AllocateFilterOutputTensor(context, conv_prim_desc,
-                TFShapeToMklDnnDims(filter_tf_shape),
-                &filter_out_tensor);
-          PrepareAndExecuteNet(conv_prim_desc, &src, &filter,
-                              nullptr, &output, filter_out_tensor);
+        // Create convolution primitive without Bias.
+        auto conv_desc = convolution_forward::desc(
+            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
+            filter.GetOpMemDesc(), output.GetOpMemDesc(), strides, padding_l,
+            padding_r, TFPaddingToMklDnnPadding(padding_));
+
+        auto conv_prim_desc =
+            convolution_forward::primitive_desc(conv_desc, cpu_engine);
+        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
+                             tf_fmt, &output_tensor);
+        // Set data handle for output.
+        output.SetUsrMemDataHandle(output_tensor);
+
+        Tensor* filter_out_tensor = nullptr;
+        AllocateFilterOutputTensor(context, conv_prim_desc,
+                                   TFShapeToMklDnnDims(filter_tf_shape),
+                                   &filter_out_tensor);
+        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, nullptr, &output,
+                             filter_out_tensor);
       }
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -700,12 +658,10 @@ class MklConv2DOp : public OpKernel {
 
  private:
   std::vector<int32> strides_;
-  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
-  const int kDilationH = 0, kDilationW = 1;
 
   // Allocate output tensor.
   void AllocateOutputTensor(
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 7ca10db895..9dd88221a8 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -58,16 +58,13 @@ class MklDnnConvUtil {
  protected:
   OpKernelContext* context_;  // We don't own this.
   std::vector<int32> strides_;
-  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
 
  public:
   MklDnnConvUtil(OpKernelContext* context, const std::vector<int32>& strides,
-                 Padding pad, TensorFormat fm,
-                 const std::vector<int32>& dilations) :
-    context_(context), strides_(strides), padding_(pad),
-    data_format_(fm), dilations_(dilations) {}
+                 Padding pad, TensorFormat fm)
+      : context_(context), strides_(strides), padding_(pad), data_format_(fm) {}
 
   virtual ~MklDnnConvUtil() { context_ = nullptr; }
 
@@ -81,16 +78,6 @@ class MklDnnConvUtil {
     *strides = {stride_rows, stride_cols};
   }
 
-  // Calculate Convolution dilations
-  virtual inline void GetDilationsInMklOrder(memory::dims *dilations) {
-    // For now we take the dilation from the second and third dimensions only
-    // (we do not support dilation on the batch or depth dimension).
-    CHECK_NOTNULL(dilations);
-    int dilations_rows = GetTensorDim(dilations_, data_format_, 'H');
-    int dilations_cols = GetTensorDim(dilations_, data_format_, 'W');
-    *dilations = {dilations_rows, dilations_cols};
-  }
-
   // Calculate Convolution input size in MKL-DNN order. MKL-DNN
   // requires input in NCHW format. Function does not return anything.
   // But errors arising from sanity checks are returned in context's
@@ -226,8 +213,7 @@ class MklDnnConvUtil {
   // TODO(nhasabni): Add similar function for input and filter in MklShape.
   virtual inline void GetOutputAndPadSizeInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
-      const memory::dims& strides, const memory::dims& dilations,
-      memory::dims* output_dims_tf_order,
+      const memory::dims& strides, memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -246,8 +232,6 @@ class MklDnnConvUtil {
     // Stride is vector of 2 elements: {s_r, s_c}
     int stride_rows = strides[0];
     int stride_cols = strides[1];
-    int dilation_rows = dilations[0];
-    int dilation_cols = dilations[1];
 
     // Output batch is same as input batch.
     int out_batch = GetTensorDim(input_shape, data_format_, 'N');
@@ -257,13 +241,11 @@ class MklDnnConvUtil {
     int64 out_rows = 0, out_cols = 0;
     int64 pad_top = 0, pad_bottom = 0, pad_left, pad_right;
 
-    OP_REQUIRES_OK(context_,
-            GetWindowedOutputSizeVerboseV2(input_rows, filter_rows,
-                                 dilation_rows, stride_rows, padding_,
+    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
+                                 input_rows, filter_rows, stride_rows, padding_,
                                  &out_rows, &pad_top, &pad_bottom));
-    OP_REQUIRES_OK(context_,
-            GetWindowedOutputSizeVerboseV2(input_cols, filter_cols,
-                                 dilation_cols, stride_cols, padding_,
+    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
+                                 input_cols, filter_cols, stride_cols, padding_,
                                  &out_cols, &pad_left, &pad_right));
 
     // Tensorflow output is in data_format order. (NHWC or NCHW)
@@ -289,8 +271,7 @@ class MklDnnConvUtil {
   //
   // Function does not return anything, but sets error in context status.
   inline void GetOutputAndPadSizeInMklOrder(
-      size_t src_index, size_t filter_index,
-      const memory::dims& strides, const memory::dims& dilations,
+      size_t src_index, size_t filter_index, const memory::dims& strides,
       memory::dims* output_dims_tf_order, memory::dims* output_dims_mkl_order,
       memory::dims* pad_l, memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -305,9 +286,9 @@ class MklDnnConvUtil {
                 errors::InvalidArgument("input must be 4-dimensional",
                                         input_tf_shape.DebugString()));
 
-    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape,
-                                  strides, dilations, output_dims_tf_order,
-                                  output_dims_mkl_order, pad_l, pad_r);
+    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape, strides,
+                                  output_dims_tf_order, output_dims_mkl_order,
+                                  pad_l, pad_r);
   }
 
   // Wrapper function to calculate input, filter, and output sizes of
@@ -319,14 +300,12 @@ class MklDnnConvUtil {
   inline void GetConvFwdSizesInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
       memory::dims* input_dims, memory::dims* filter_dims,
-      memory::dims* strides, memory::dims *dilations,
-      memory::dims* output_dims_tf_order,
+      memory::dims* strides, memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(input_dims);
     CHECK_NOTNULL(filter_dims);
     CHECK_NOTNULL(strides);
-    CHECK_NOTNULL(dilations);
     CHECK_NOTNULL(output_dims_tf_order);
     CHECK_NOTNULL(output_dims_mkl_order);
     CHECK_NOTNULL(pad_l);
@@ -337,9 +316,7 @@ class MklDnnConvUtil {
     GetFilterSizeInMklOrder(input_shape, filter_shape, filter_dims);
     if (!context_->status().ok()) return;
     GetStridesInMklOrder(strides);
-    GetDilationsInMklOrder(dilations);
-    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape,
-                                  *strides, *dilations,
+    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape, *strides,
                                   output_dims_tf_order, output_dims_mkl_order,
                                   pad_l, pad_r);
     if (!context_->status().ok()) return;
@@ -367,21 +344,7 @@ class MklConv2DBackpropCommonOp : public OpKernel {
         context, (stride_n == 1 && stride_c == 1),
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
-    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
-    OP_REQUIRES(context, dilations_.size() == 4,
-                errors::InvalidArgument("Sliding window dilations field must "
-                                        "specify 4 dimensions"));
-    int dilation_n = GetTensorDim(dilations_, data_format_, 'N');
-    int dilation_c = GetTensorDim(dilations_, data_format_, 'C');
-    int dilation_h = GetTensorDim(dilations_, data_format_, 'H');
-    int dilation_w = GetTensorDim(dilations_, data_format_, 'W');
-    OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1),
-                errors::InvalidArgument(
-                    "Current implementation does not yet support "
-                    "dilations in the batch and depth dimensions."));
-    OP_REQUIRES(
-        context, dilation_h > 0 && dilation_w > 0,
-        errors::InvalidArgument("Dilated rates should be larger than 0."));
+
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
   }
 
@@ -443,16 +406,15 @@ class MklConv2DBackpropCommonOp : public OpKernel {
       // By default, all dims are in MKL order. Only dims in TF order
       // are those with prefix tf_order.
       memory::dims outbprop_dims, fwd_input_dims, fwd_filter_dims;
-      memory::dims padding_l, padding_r, dilations, strides, fwd_output_dims;
+      memory::dims padding_l, padding_r, strides, fwd_output_dims;
       memory::dims fwd_output_dims_tf_order;
 
       // Get forward convolution parameters.
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
-                             dilations_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
       conv_utl.GetConvFwdSizesInMklOrder(
           input_tf_shape, filter_tf_shape, &fwd_input_dims, &fwd_filter_dims,
-          &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims,
-          &padding_l, &padding_r);
+          &strides, &fwd_output_dims_tf_order, &fwd_output_dims, &padding_l,
+          &padding_r);
       if (!context->status().ok()) return;
 
       // Create Convolution forward descriptor since Convolution backward
@@ -475,21 +437,10 @@ class MklConv2DBackpropCommonOp : public OpKernel {
                                               memory::format::hwio);
       // Tensorflow Output of Conv2D is in data_format order.
       auto fwd_out_md = memory::desc(fwd_output_dims, MklDnnType<T>(), tf_fmt);
-
-      const int kDilationH = 0, kDilationW = 1;
-      dilations[kDilationH] -= 1;
-      dilations[kDilationW] -= 1;
-      auto fwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0)?
-              convolution_forward::desc(prop_kind::forward,
-                     convolution_direct, fwd_input_md,
-                     fwd_filter_md, fwd_out_md,
-                     strides, dilations, padding_l, padding_r,
-                     TFPaddingToMklDnnPadding(padding_)) :
-              convolution_forward::desc(prop_kind::forward,
-                     convolution_direct, fwd_input_md,
-                     fwd_filter_md, fwd_out_md,
-                     strides, padding_l, padding_r,
-                     TFPaddingToMklDnnPadding(padding_));
+      auto fwd_desc = convolution_forward::desc(
+          prop_kind::forward, convolution_direct, fwd_input_md, fwd_filter_md,
+          fwd_out_md, strides, padding_l, padding_r,
+          TFPaddingToMklDnnPadding(padding_));
       auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine);
 
       // Create memory for user data. Describe how the inputs and outputs of
@@ -534,9 +485,8 @@ class MklConv2DBackpropCommonOp : public OpKernel {
 
       // Operator-specific call to create and execute primitive.
       CreatePrimitive(context, cpu_engine, fwd_pd, &input, &filter,
-                      &outbackprop, &output, &output_tensor,
-                      strides, dilations, padding_l, padding_r,
-                      TFPaddingToMklDnnPadding(padding_),
+                      &outbackprop, &output, &output_tensor, strides, padding_l,
+                      padding_r, TFPaddingToMklDnnPadding(padding_),
                       bwd_output_dims, bwd_output_format);
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -585,21 +535,20 @@ class MklConv2DBackpropCommonOp : public OpKernel {
   virtual memory::format GetOutputFormat(const memory::format data_format) = 0;
 
   /// Create and execute the primitive storing output in the output_tensor.
-  virtual void CreatePrimitive(OpKernelContext* context,
-    const engine& cpu_engine,
-    const convolution_forward::primitive_desc& conv_fwd_pd,
-    MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
-    MklDnnData<T>* output, Tensor** output_tensor, const memory::dims& strides,
-    const memory::dims& dilations, const memory::dims& padding_l,
-    const memory::dims& padding_r, padding_kind padding,
-    const memory::dims& bwd_output_dims,
-    memory::format bwd_output_format) = 0;
+  virtual void CreatePrimitive(
+      OpKernelContext* context, const engine& cpu_engine,
+      const convolution_forward::primitive_desc& conv_fwd_pd,
+      MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
+      MklDnnData<T>* output, Tensor** output_tensor,
+      const memory::dims& strides, const memory::dims& padding_l,
+      const memory::dims& padding_r, padding_kind padding,
+      const memory::dims& bwd_output_dims,
+      memory::format bwd_output_format) = 0;
 
   // Get the data_format {NCHW, NHWC}
   TensorFormat GetTFDataFormat() { return data_format_; }
 
  private:
-  std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
   TensorFormat data_format_;
diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc
index d91f7107c5..e9a2376b54 100644
--- a/tensorflow/core/kernels/mkl_input_conversion_op.cc
+++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc
@@ -442,11 +442,12 @@ class MklInputConversionOp : public OpKernel {
       auto input_tf_md = mkl_output_mkl_shape.GetTfLayout();
       tf_input.SetUsrMem(input_tf_md, tf_tensor);
 
-      // Create reorder between tensorflow layout and Mkl layout if necessary
+      // Create reorder between tensorflow layout and Mkl layout.
       std::vector<primitive> net;
-      tf_input.CheckReorderToOpMem(
+      CHECK_EQ(tf_input.CheckReorderToOpMem(
                    memory::primitive_desc(output_mkl_md, cpu_engine),
-                   tensor_out, &net);
+                   tensor_out, &net),
+               true);
       stream(stream::kind::eager).submit(net).wait();
 
       // -- The tensor in MKL format passes through --
diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl_relu_op.cc
index 0a0f69522f..267f4f8d12 100644
--- a/tensorflow/core/kernels/mkl_relu_op.cc
+++ b/tensorflow/core/kernels/mkl_relu_op.cc
@@ -392,7 +392,7 @@ class MklReluOpBase : public OpKernel {
 
       Tensor* dst_tensor = nullptr;
       if (src_tensor.dims() == 0) {
-        Compute_Scalar(context); // scalar case doesn't use in-place operation
+        Compute_Scalar(context);
         return;
       }
 
@@ -437,15 +437,11 @@ class MklReluOpBase : public OpKernel {
         dnn_shape_dst.SetMklTensor(false);
         tf_shape_dst = src_tensor.shape();
       }
-      
-      // Allocate output and MklDnnShape tensors separately for possible
-      // in-place operation
-      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                      {src_index}, dst_index, tf_shape_dst, &dst_tensor));
-      AllocateOutputSetMklShape(context, dst_index, dnn_shape_dst);
+      AllocateOutputSetMklShape(context, dst_index, &dst_tensor, tf_shape_dst,
+                                dnn_shape_dst);
 
       // Destination memory descriptor is same as source memory descriptor.
-      auto &dst_md = src_md;
+      auto dst_md = src_md;
       dst.SetUsrMem(dst_md, dst_tensor);
 
       // execute net
@@ -496,7 +492,7 @@ class MklReluGradOpBase : public OpKernel {
 
       int src_dims_size = src_tensor.dims();
       if (src_dims_size == 0) {
-        Compute_Scalar(context); // scalar case doesn't use in-place operation
+        Compute_Scalar(context);
         return;
       }
 
@@ -607,13 +603,8 @@ class MklReluGradOpBase : public OpKernel {
         // so it is ok to get TensorFlow shape.
         tf_shape_diff_src = src_tensor.shape();
       }
-
-      // Allocate diff_src and MklDnnShape tensors separately for possible
-      // in-place operation
-      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                      {diff_dst_index}, diff_src_index, tf_shape_diff_src,
-                                      &diff_src_tensor));
-      AllocateOutputSetMklShape(context, diff_src_index, dnn_shape_diff_src);
+      AllocateOutputSetMklShape(context, diff_src_index, &diff_src_tensor,
+                                tf_shape_diff_src, dnn_shape_diff_src);
 
       // diff_src memory descriptor is same as memory descriptor for both
       // inputs.
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index d0703d7576..4abfbfb1a6 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -13,16 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
-#define THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
-
-
-// This file requires the following include because it uses CudaAtomicMax:
-// #include "tensorflow/core/util/cuda_kernel_helper.h"
-
-// Unfortunately we can't add the #include, since it breaks compilation for
-// non-GPU targets. This only breaks in clang, because it's more strict for
-// template code and CudaAtomicMax is used in template context.
+#ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+#define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
 
 // This file requires the following include because it uses CudaAtomicMax:
 // #include "tensorflow/core/util/cuda_kernel_helper.h"
@@ -138,4 +130,4 @@ struct Highest {
 }  // namespace functor
 }  // namespace tensorflow
 
-#endif  // THIRD_PARTY_TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
+#endif  // TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
diff --git a/tensorflow/core/kernels/spacetodepth_op.cc b/tensorflow/core/kernels/spacetodepth_op.cc
index e59adfc6ac..23df1c35e5 100644
--- a/tensorflow/core/kernels/spacetodepth_op.cc
+++ b/tensorflow/core/kernels/spacetodepth_op.cc
@@ -187,9 +187,6 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     SpaceToDepthOp<GPUDevice, float>);
-REGISTER_KERNEL_BUILDER(
-    Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
-    SpaceToDepthOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     SpaceToDepthOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
index f38459724a..db05ca1ed2 100644
--- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
@@ -234,12 +234,6 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
-// Instantiate the GPU implementations for Eigen::half.
-template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
-                                               FORMAT_NCHW>;
-template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
-                                               FORMAT_NHWC>;
-
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 0b006fa2b4..5bd79778a6 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -55,4 +55,6 @@ StringPiece StringPiece::substr(size_t pos, size_t n) const {
   return StringPiece(data_ + pos, n);
 }
 
+const StringPiece::size_type StringPiece::npos = size_type(-1);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 2d00f717dc..910e4d9e2a 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -65,7 +65,7 @@ class StringPiece {
   iterator begin() const { return data_; }
   iterator end() const { return data_ + size_; }
 
-  static const size_t npos = size_type(-1);
+  static const size_t npos;
 
   // Return the ith byte in the referenced data.
   // REQUIRES: n < size()
diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc
index 6de850bb20..254fdf115d 100644
--- a/tensorflow/core/lib/io/record_reader.cc
+++ b/tensorflow/core/lib/io/record_reader.cc
@@ -205,9 +205,7 @@ Status RecordReader::SkipNBytes(uint64 offset) {
     if (options_.buffer_size > 0) {
       TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset));
     }
-#if !defined(IS_SLIM_BUILD)
   }
-#endif
   return Status::OK();
 }  // namespace io
 
diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h
index 26278e0328..62dd2efb79 100644
--- a/tensorflow/core/lib/io/record_reader.h
+++ b/tensorflow/core/lib/io/record_reader.h
@@ -16,10 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_RECORD_READER_H_
 #define TENSORFLOW_LIB_IO_RECORD_READER_H_
 
-#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/lib/io/inputstream_interface.h"
 #if !defined(IS_SLIM_BUILD)
+#include "tensorflow/core/lib/io/inputstream_interface.h"
 #include "tensorflow/core/lib/io/zlib_compression_options.h"
 #include "tensorflow/core/lib/io/zlib_inputstream.h"
 #endif  // IS_SLIM_BUILD
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 9a4b616e5d..bdbbf6d7c3 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -1,4 +1,4 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -265,16 +265,6 @@ REGISTER_OP("BatchDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
-// TODO(mrry): move SlideDataset to contrib in the future.
-REGISTER_OP("SlideDataset")
-    .Input("input_dataset: variant")
-    .Input("window_size: int64")
-    .Input("stride: int64")
-    .Output("handle: variant")
-    .Attr("output_types: list(type) >= 1")
-    .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
-
 REGISTER_OP("PaddedBatchDataset")
     .Input("input_dataset: variant")
     .Input("batch_size: int64")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index d6a0f38033..910fbaca9e 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1498,7 +1498,6 @@ REGISTER_OP("_MklConv2D")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn(shape_inference::Conv2DShape)
     .Doc(R"doc(
 MKL version of Conv2D operator. Uses MKL DNN APIs to perform 2D convolution.
@@ -1517,7 +1516,6 @@ REGISTER_OP("__MklDummyConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 Dummy node that enables fusing Conv2D and BiasAdd operator for MKL. This node
 does not perform anything. It is just created as an intermediate output of
@@ -1543,7 +1541,6 @@ REGISTER_OP("_MklConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2D and BiasAdd operator. Uses MKL DNN APIs to perform
 2D convolution and add Bias to the output of convolution.
@@ -1566,7 +1563,6 @@ REGISTER_OP("_MklConv2DBackpropFilter")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
@@ -1593,7 +1589,6 @@ REGISTER_OP("__MklDummyConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1638,7 +1633,6 @@ REGISTER_OP("_MklConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1674,7 +1668,6 @@ REGISTER_OP("_MklConv2DWithBiasBackpropBias")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2DBackpropBias. Uses MKL DNN APIs to compute the
 gradients of convolution with respect to the bias.
@@ -1697,7 +1690,6 @@ REGISTER_OP("_MklConv2DBackpropInput")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
-    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h
index eebbeaeba6..8f7bff1bb0 100644
--- a/tensorflow/core/platform/tracing.h
+++ b/tensorflow/core/platform/tracing.h
@@ -103,7 +103,7 @@ class Tracing {
   friend class ScopedAnnotation;
   friend class TraceMe;
 
-  TF_EXPORT static std::atomic<Tracing::Engine*> tracing_engine_;
+  static std::atomic<Tracing::Engine*> tracing_engine_;
   static Tracing::Engine* engine() {
     return tracing_engine_.load(std::memory_order_acquire);
   }
diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc
index 682e46e0fc..b6b3722caa 100644
--- a/tensorflow/core/platform/windows/windows_file_system.cc
+++ b/tensorflow/core/platform/windows/windows_file_system.cc
@@ -382,8 +382,7 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
 
 Status WindowsFileSystem::FileExists(const string& fname) {
   constexpr int kOk = 0;
-  std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
-  if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
+  if (_access(TranslateName(fname).c_str(), kOk) == 0) {
     return Status::OK();
   }
   return errors::NotFound(fname, " not found");
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 22f2c02b78..7405e01e14 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc1"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md
index 6d0458e678..9f6fe91b14 100644
--- a/tensorflow/docs_src/community/welcome.md
+++ b/tensorflow/docs_src/community/welcome.md
@@ -51,8 +51,6 @@ Europe:
 TensorFlow provides multiple communication paths.  To pick the right path,
 please read the following list carefully:
 
-  * For new release announcements and security updates, subscribe to
-    [announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
   * To ask or answer technical questions about TensorFlow, use
     [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).
     For example, ask or search Stack Overflow about a particular error message
@@ -67,5 +65,5 @@ please read the following list carefully:
     on GitHub.  For example, use the issue tracker to request a
     new operation in TensorFlow.
   * To report vulnerabilities, please follow our
-    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).
+    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/SECURITY.md).
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 0481c97885..818798555a 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 8f89898c92..4c6dfa8daf 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0-rc1.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 0ee9c849e1..527884863e 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.6.0-rc1</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0</version>
+                 <version>1.6.0-rc1</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.6.0-rc1</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0</version>
+  <version>1.6.0-rc1</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0-rc1.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.6.0-rc1.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>d
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 3e8744bf9d..e3e115d9f6 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -41,8 +41,7 @@ must be installed on your system:
     [NVIDIA's documentation](https://developer.nvidia.com/cudnn).
     Ensure that you create the `CUDA_HOME` environment variable as
     described in the NVIDIA documentation.
-  * GPU card with CUDA Compute Capability 3.0 or higher for building
-    from source and 3.5 or higher for our binaries. See
+  * GPU card with CUDA Compute Capability 3.0 or higher.  See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for
     a list of supported GPU cards.
   * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
@@ -189,7 +188,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -294,7 +293,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -357,23 +356,24 @@ where:
     to 6006.
   * <tt><i>TensorFlowCPUImage</i></tt> is required. It identifies the Docker
     container. Specify one of the following values:
-    * <tt>tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
-    * <tt>tensorflow/tensorflow:latest-devel</tt>, which is the latest
+    * <tt>gcr.io/tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
+    * <tt>gcr.io/tensorflow/tensorflow:latest-devel</tt>, which is the latest
       TensorFlow CPU Binary image plus source code.
-    * <tt>tensorflow/tensorflow:<i>version</i></tt>, which is the
+    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i></tt>, which is the
       specified version (for example, 1.1.0rc1) of TensorFlow CPU binary image.
-    * <tt>tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
+    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
       the specified version (for example, 1.1.0rc1) of the TensorFlow GPU
       binary image plus source code.
 
-    TensorFlow images are available at
+    <tt>gcr.io</tt> is the Google Container Registry. Note that some
+    TensorFlow images are also available at
     [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches the latest TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>docker run -it tensorflow/tensorflow bash</b>
+$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow CPU binary image in a
@@ -381,7 +381,7 @@ Docker container. However, in this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b>
+$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -405,14 +405,14 @@ where:
     <tt><i>hostPort</i></tt> and <code><em>containerPort</em></code> to `8888`.
   * <i>TensorFlowGPUImage</i> specifies the Docker container. You must
     specify one of the following values:
-    * <tt>tensorflow/tensorflow:latest-gpu</tt>, which is the latest
+    * <tt>gcr.io/tensorflow/tensorflow:latest-gpu</tt>, which is the latest
       TensorFlow GPU binary image.
-    * <tt>tensorflow/tensorflow:latest-devel-gpu</tt>, which is
+    * <tt>gcr.io/tensorflow/tensorflow:latest-devel-gpu</tt>, which is
       the latest TensorFlow GPU Binary image plus source code.
-    * <tt>tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
+    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
       specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image.
-    * <tt>tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
+    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
       the specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image plus source code.
 
@@ -421,7 +421,7 @@ following command launches the latest TensorFlow GPU binary image in a
 Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>nvidia-docker run -it tensorflow/tensorflow:latest-gpu bash</b>
+$ <b>nvidia-docker run -it gcr.io/tensorflow/tensorflow:latest-gpu bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow GPU binary image
@@ -429,13 +429,13 @@ in a Docker container. In this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:latest-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:latest-gpu</b>
 </pre>
 
 The following command installs an older TensorFlow version (0.12.1):
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:0.12.1-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:0.12.1-gpu</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -480,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -505,7 +505,7 @@ If you installed through Docker, start a Docker container
 from which you can run bash. For example:
 
 <pre>
-$ <b>docker run -it tensorflow/tensorflow bash</b>
+$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
 </pre>
 
 
@@ -647,14 +647,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -666,14 +666,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -685,14 +685,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -704,14 +704,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 205db8e6bd..623ca6bb79 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -292,23 +292,24 @@ where:
     to 6006.
   * <i>TensorFlowImage</i> is required. It identifies the Docker container.
     You must specify one of the following values:
-    * <code>tensorflow/tensorflow</code>: TensorFlow binary image.
-    * <code>tensorflow/tensorflow:latest-devel</code>: TensorFlow
+    * <code>gcr.io/tensorflow/tensorflow</code>: TensorFlow binary image.
+    * <code>gcr.io/tensorflow/tensorflow:latest-devel</code>: TensorFlow
       Binary image plus source code.
 
-The TensorFlow images are available at
+<code>gcr.io</code> is the Google Container Registry. Note that some
+TensorFlow images are also available at
 [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches a TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
-<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
 
 The following command also launches a TensorFlow CPU binary image in a
 Docker container. However, in this Docker container, you can run
 TensorFlow programs in a Jupyter notebook:
 
-<pre>$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b></pre>
+<pre>$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b></pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
 
@@ -350,7 +351,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -375,7 +376,7 @@ do the following:
 If you installed through Docker, start a Docker container that runs bash.
 For example:
 
-<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
 
 
 
@@ -523,7 +524,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl
 </pre>
 
 
@@ -531,5 +532,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index c09c9c2c0c..acf0af0d9d 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -359,10 +359,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0 on Linux:
+for TensorFlow 1.6.0rc1 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0rc1-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index 2413bc9cfb..f0a30ee394 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -17,7 +17,7 @@ You must choose one of the following types of TensorFlow to install:
     NVIDIA® GPU, you must install this version. Note that this version of
     TensorFlow is typically much easier to install (typically,
     in 5 or 10 minutes), so even if you have an NVIDIA GPU, we recommend
-    installing this version first. Prebuilt binaries will use AVX instructions. 
+    installing this version first.
   * **TensorFlow with GPU support**. TensorFlow programs typically run
     significantly faster on a GPU than on a CPU. Therefore, if your
     system has a NVIDIA® GPU meeting the prerequisites shown below
@@ -41,8 +41,7 @@ installed on your system:
     Note that cuDNN is typically installed in a different location from the
     other CUDA DLLs. Ensure that you add the directory where you installed
     the cuDNN DLL to your `%PATH%` environment variable.
-  * GPU card with CUDA Compute Capability 3.0 or higher for building
-    from source and 3.5 or higher for our binaries. See
+  * GPU card with CUDA Compute Capability 3.0 or higher.  See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a
     list of supported GPU cards.
 
diff --git a/tensorflow/docs_src/performance/xla/jit.md b/tensorflow/docs_src/performance/xla/jit.md
index d9a979ccbd..d4dc3e57c8 100644
--- a/tensorflow/docs_src/performance/xla/jit.md
+++ b/tensorflow/docs_src/performance/xla/jit.md
@@ -157,7 +157,7 @@ to fuse Ops is visible by starting at `hlo_graph_0.dot` and viewing each diagram
 in succession.
 
 To Render the .dot file into a png, install
-[GraphViz](https://www.graphviz.org/download/) and run:
+[GraphViz](http://www.graphviz.org/Download..php) and run:
 
 ```shell
 dot -Tpng hlo_graph_80.dot -o hlo_graph_80.png
diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index d1399814ee..5fb1c2da88 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -459,7 +459,7 @@ accuracy_score = classifier.evaluate(x=test_set.data,
 
 
 [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py),
-based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.2/get_started/tflearn), contains a full example of how to
+based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to
 use the tfdbg with `Estimator`s. To run this example, do:
 
 ```none
@@ -753,7 +753,6 @@ There are three possible workarounds or solutions:
    # For LocalCLIDebugHook
    hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")]
    ```
-
    Make sure that the directory pointed to by dump_root is empty or nonexistent.
    tfdbg cleans up the dump directories before exiting.
 *  Reduce the batch size used during the runs.
diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md
index 392ac6f7f1..1548d43877 100644
--- a/tensorflow/docs_src/programmers_guide/faq.md
+++ b/tensorflow/docs_src/programmers_guide/faq.md
@@ -159,7 +159,7 @@ available. These operations allow you to build sophisticated
 @{$reading_data$input pipelines}, at the cost of making the
 TensorFlow computation somewhat more complicated. See the how-to documentation
 for
-@{$reading_data#creating_threads_to_prefetch_using_queuerunner_objects$using `QueueRunner` objects to drive queues and readers}
+@{$reading_data#creating-threads-to-prefetch-using-queuerunner-objects$using `QueueRunner` objects to drive queues and readers}
 for more information on how to use them.
 
 ## Variables
@@ -272,7 +272,7 @@ Prefer predefined TensorFlow operations such as @{tf.decode_raw},
 
 If your data is not easily parsable with the built-in TensorFlow operations,
 consider converting it, offline, to a format that is easily parsable, such
-as @{tf.python_io.TFRecordWriter$`TFRecord`} format.
+as ${tf.python_io.TFRecordWriter$`TFRecord`} format.
 
 The more efficient method to customize the parsing behavior is to
 @{$adding_an_op$add a new op written in C++} that parses your
diff --git a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
index fadfa03e78..79280d246a 100644
--- a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
+++ b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
@@ -83,7 +83,7 @@ data than you need, though. Instead, consider running the merged summary op
 every `n` steps.
 
 The code example below is a modification of the
-[simple MNIST tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py),
+@{$layers$simple MNIST tutorial},
 in which we have added some summary ops, and run them every ten steps. If you
 run this and then launch `tensorboard --logdir=/tmp/tensorflow/mnist`, you'll be able
 to visualize statistics, such as how the weights or accuracy varied during
diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md
index a9c2cb3e33..d74d7f3181 100644
--- a/tensorflow/docs_src/programmers_guide/using_tpu.md
+++ b/tensorflow/docs_src/programmers_guide/using_tpu.md
@@ -129,9 +129,10 @@ my_tpu_estimator = tf.contrib.tpu.TPUEstimator(
 Typically the `FLAGS` would be set by command line arguments. To switch from
 training locally to training on a cloud TPU you would need to:
 
-* Set `FLAGS.use_tpu` to `True`
-* Set `FLAGS.tpu_name` so the `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
-* Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
+  1) Set `FLAGS.use_tpu` to `True`
+  1) Set `FLAGS.tpu_name` so the
+     `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
+  1) Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
 
 
 ## Optimizer
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index 9b17d0d4d5..ee03f440c9 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -193,7 +193,7 @@ to calculate loss, configure the training op, and generate predictions. If
 you're already experienced with CNNs and @{$get_started/custom_estimators$TensorFlow `Estimator`s},
 and find the above code intuitive, you may want to skim these sections or just
 skip ahead to ["Training and Evaluating the CNN MNIST
-Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
+Classifier"](#training-and-evaluating-the-cnn-mnist-classifier).
 
 ### Input Layer
 
@@ -446,7 +446,7 @@ tf.nn.softmax(logits, name="softmax_tensor")
 
 > Note: We use the `name` argument to explicitly name this operation
 > `softmax_tensor`, so we can reference it later. (We'll set up logging for the
-> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook)).
+> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook).
 
 We compile our predictions in a dict, and return an `EstimatorSpec` object:
 
@@ -534,8 +534,9 @@ if mode == tf.estimator.ModeKeys.TRAIN:
 ```
 
 > Note: For a more in-depth look at configuring training ops for Estimator model
-> functions, see @{$get_started/custom_estimators#defining_the_training_op_for_the_model$"Defining the training op for the model"} 
-> in the @{$get_started/custom_estimators$"Creating Estimators in tf.estimator."} tutorial.
+> functions, see @{$get_started/custom_estimators#defining-the-training-op-for-the-model$"Defining
+> the training op for the model"} in the @{$get_started/custom_estimators$"Creating Estimations in
+> tf.estimator"} tutorial.
 
 ### Add evaluation metrics
 
@@ -624,8 +625,7 @@ operation earlier when we generated the probabilities in `cnn_model_fn`.
 > Note: If you don't explicitly assign a name to an operation via the `name`
 > argument, TensorFlow will assign a default name. A couple easy ways to
 > discover the names applied to operations are to visualize your graph on
-> @{$graph_viz$TensorBoard}) or to enable the
-> @{$programmers_guide/debugger$TensorFlow Debugger (tfdbg)}.
+> @{$graph_viz$TensorBoard}) or to enable the @{$debugger$TensorFlow Debugger (tfdbg)}.
 
 Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the
 `tensors` argument. We set `every_n_iter=50`, which specifies that probabilities
diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index 7584a76ba5..e22536adb6 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -109,8 +109,7 @@ This download will take a while and download a bit more than 23GB of data.
 
 To convert the `ndjson` files to
 @{$python/python_io#tfrecords_format_details$TFRecord} files containing
-[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
-protos run the following command.
+${tf.train.Example} protos run the following command.
 
 ```shell
    python create_dataset.py --ndjson_path rnn_tutorial_data \
diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 27ce75a30d..005dc020f9 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -74,8 +74,8 @@ Here's a list of columns available in the Census Income dataset:
 | relationship   | Categorical | Wife, Own-child, Husband,         |
 :                :             : Not-in-family, Other-relative,    :
 :                :             : Unmarried.                        :
-| race           | Categorical | Amer-Indian-Eskimo, Asian-Pac-    |
-:                :             : Islander, Black, White, Other.    :
+| race           | Categorical | White, Asian-Pac-Islander,        |
+:                :             : Amer-Indian-Eskimo, Other, Black. :
 | gender         | Categorical | Female, Male.                     |
 | capital_gain   | Continuous  | Capital gains recorded.           |
 | capital_loss   | Continuous  | Capital Losses recorded.          |
@@ -247,7 +247,7 @@ hours_per_week = tf.feature_column.numeric_column('hours_per_week')
 ### Making Continuous Features Categorical through Bucketization
 
 Sometimes the relationship between a continuous feature and the label is not
-linear. As a hypothetical example, a person's income may grow with age in the
+linear. As an hypothetical example, a person's income may grow with age in the
 early stage of one's career, then the growth may slow at some point, and finally
 the income decreases after retirement. In this scenario, using the raw `age` as
 a real-valued feature column might not be a good choice because the model can
@@ -361,16 +361,6 @@ The first line of the final output should be something like
 `accuracy: 0.83557522`, which means the accuracy is 83.6%. Feel free to try more
 features and transformations and see if you can do even better!
 
-After the model is evaluated, we can use the model to predict whether an individual has an annual income of over
-50,000 dollars given an individual's information input.
-```python
-  pred_iter = model.predict(input_fn=lambda: input_fn(FLAGS.test_data, 1, False, 1))
-  for pred in pred_iter:
-    print(pred['classes'])
-```
-
-The model prediction output would be like `[b'1']` or `[b'0']` which means whether corresponding individual has an annual income of over 50,000 dollars or not.
-
 If you'd like to see a working end-to-end example, you can download our
 [example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py)
 and set the `model_type` flag to `wide`.
diff --git a/tensorflow/examples/android/AndroidManifest.xml b/tensorflow/examples/android/AndroidManifest.xml
index 5c47ce6b67..bb75431a1f 100644
--- a/tensorflow/examples/android/AndroidManifest.xml
+++ b/tensorflow/examples/android/AndroidManifest.xml
@@ -40,7 +40,6 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
-                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -50,7 +49,6 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
-                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -60,7 +58,6 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
-                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -70,7 +67,6 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
-                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
     </application>
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
index 429138abe5..8bd4abb154 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
@@ -351,10 +351,6 @@ public abstract class CameraActivity extends Activity
 
   protected void setFragment() {
     String cameraId = chooseCamera();
-    if (cameraId == null) {
-      Toast.makeText(this, "No Camera Detected", Toast.LENGTH_SHORT).show();
-      finish();
-    }
 
     Fragment fragment;
     if (useCamera2API) {
@@ -420,8 +416,7 @@ public abstract class CameraActivity extends Activity
 
   @Override
   public boolean onKeyDown(final int keyCode, final KeyEvent event) {
-    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP
-            || keyCode == KeyEvent.KEYCODE_BUTTON_L1 || keyCode == KeyEvent.KEYCODE_DPAD_CENTER) {
+    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP) {
       debug = !debug;
       requestRender();
       onSetDebug(debug);
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
index 33ec65e9f7..6a66ec3927 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
@@ -16,10 +16,8 @@
 
 package org.tensorflow.demo;
 
-import android.app.UiModeManager;
 import android.content.Context;
 import android.content.res.AssetManager;
-import android.content.res.Configuration;
 import android.graphics.Bitmap;
 import android.graphics.Bitmap.Config;
 import android.graphics.BitmapFactory;
@@ -33,11 +31,9 @@ import android.graphics.Typeface;
 import android.media.ImageReader.OnImageAvailableListener;
 import android.os.Bundle;
 import android.os.SystemClock;
-import android.util.DisplayMetrics;
 import android.util.Size;
 import android.util.TypedValue;
 import android.view.Display;
-import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.View.OnClickListener;
@@ -47,7 +43,6 @@ import android.widget.BaseAdapter;
 import android.widget.Button;
 import android.widget.GridView;
 import android.widget.ImageView;
-import android.widget.RelativeLayout;
 import android.widget.Toast;
 import java.io.IOException;
 import java.io.InputStream;
@@ -386,27 +381,6 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
     grid = (GridView) findViewById(R.id.grid_layout);
     grid.setAdapter(adapter);
     grid.setOnTouchListener(gridTouchAdapter);
-
-    // Change UI on Android TV
-    UiModeManager uiModeManager = (UiModeManager) getSystemService(UI_MODE_SERVICE);
-    if (uiModeManager.getCurrentModeType() == Configuration.UI_MODE_TYPE_TELEVISION) {
-      DisplayMetrics displayMetrics = new DisplayMetrics();
-      getWindowManager().getDefaultDisplay().getMetrics(displayMetrics);
-      int styleSelectorHeight = displayMetrics.heightPixels;
-      int styleSelectorWidth = displayMetrics.widthPixels - styleSelectorHeight;
-      RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(styleSelectorWidth, ViewGroup.LayoutParams.MATCH_PARENT);
-
-      // Calculate number of style in a row, so all the style can show up without scrolling
-      int numOfStylePerRow = 3;
-      while (styleSelectorWidth / numOfStylePerRow * Math.ceil((float) (adapter.getCount() - 2) / numOfStylePerRow) > styleSelectorHeight) {
-        numOfStylePerRow++;
-      }
-      grid.setNumColumns(numOfStylePerRow);
-      layoutParams.addRule(RelativeLayout.ALIGN_PARENT_RIGHT);
-      grid.setLayoutParams(layoutParams);
-      adapter.buttons.clear();
-    }
-
     setStyle(adapter.items[0], 1.0f);
   }
 
@@ -628,38 +602,4 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
 
     borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines);
   }
-
-  @Override
-  public boolean onKeyDown(int keyCode, KeyEvent event) {
-    int moveOffset = 0;
-    switch (keyCode) {
-      case KeyEvent.KEYCODE_DPAD_LEFT:
-        moveOffset = -1;
-        break;
-      case KeyEvent.KEYCODE_DPAD_RIGHT:
-        moveOffset = 1;
-        break;
-      case KeyEvent.KEYCODE_DPAD_UP:
-        moveOffset = -1 * grid.getNumColumns();
-        break;
-      case KeyEvent.KEYCODE_DPAD_DOWN:
-        moveOffset = grid.getNumColumns();
-        break;
-      default:
-        return super.onKeyDown(keyCode, event);
-    }
-
-    // get the highest selected style
-    int currentSelect = 0;
-    float highestValue = 0;
-    for (int i = 0; i < adapter.getCount(); i++) {
-      if (adapter.items[i].value > highestValue) {
-        currentSelect = i;
-        highestValue = adapter.items[i].value;
-      }
-    }
-    setStyle(adapter.items[(currentSelect + moveOffset + adapter.getCount()) % adapter.getCount()], 1);
-
-    return true;
-  }
 }
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 3cbeb34c54..a206685af6 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -28,7 +28,6 @@ load("//tensorflow:tensorflow.bzl", "py_tests")
 load("//tensorflow:tensorflow.bzl", "tf_py_build_info_genrule")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_library_additional_deps_impl")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
 load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
@@ -107,19 +106,20 @@ py_library(
         ":training",
         ":util",
         ":weights_broadcast_ops",
-        "//tensorflow/contrib:contrib_py",
+        "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras",
+        "//tensorflow/python/ops/losses",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
-        "//tensorflow/python/ops/losses",
         "//tensorflow/python/profiler",
         "//tensorflow/python/saved_model",
-        "//third_party/py/numpy",
-    ],
+    ] + if_not_windows([
+        "//tensorflow/contrib:contrib_py",
+    ]),
 )
 
 tf_py_build_info_genrule()
@@ -947,6 +947,7 @@ py_test(
     srcs = ["framework/contrib_test.py"],
     main = "framework/contrib_test.py",
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
@@ -1311,6 +1312,7 @@ py_test(
     srcs = ["framework/dtypes_test.py"],
     main = "framework/dtypes_test.py",
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":framework_for_generated_wrappers",
         ":framework_test_lib",
@@ -1652,6 +1654,7 @@ py_test(
     size = "small",
     srcs = ["ops/clip_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":clip_ops",
@@ -2718,6 +2721,7 @@ cuda_py_test(
     ],
     data = ["//tensorflow/core:image_testdata"],
     shard_count = 5,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -3301,65 +3305,6 @@ tf_py_wrap_cc(
          tf_additional_gdr_deps()),
 )
 
-# ** Targets for Windows build (start) **
-# We need the following targets to expose symbols from _pywrap_tensorflow.dll
-
-# Build a cc_binary from tf_custom_op_library_additional_deps_impl,
-# it contains all object code from its dependencies.
-cc_binary(
-    name = "tf_custom_op_library_additional_deps.so",
-    linkshared = 1,
-    linkstatic = 1,
-    deps = tf_custom_op_library_additional_deps_impl(),
-)
-
-# Get a DEF file generated by parsing all object files
-# of tf_custom_op_library_additional_deps.so
-filegroup(
-    name = "pywrap_tensorflow_def_file",
-    srcs = [":tf_custom_op_library_additional_deps.so"],
-    output_group = "def_file",
-)
-
-# Filter the DEF file to reduce the number of symbols to 64K or less.
-# Note that we also write the name of the pyd file into DEF file so that
-# the dynamic libraries of custom ops can find it at runtime.
-genrule(
-    name = "pywrap_tensorflow_filtered_def_file",
-    srcs = [":pywrap_tensorflow_def_file"],
-    outs = ["pywrap_tensorflow_filtered_def_file.def"],
-    cmd = select({
-        "//tensorflow:windows": """
-              $(location @local_config_def_file_filter//:def_file_filter) \\
-              --input $(location :pywrap_tensorflow_def_file) \\
-              --output $@ \\
-              --target _pywrap_tensorflow_internal.pyd
-          """,
-        "//conditions:default": "touch $@",  # Just a placeholder for Unix platforms
-    }),
-    tools = ["@local_config_def_file_filter//:def_file_filter"],
-)
-
-# Get the import library of  _pywrap_tensorflow_internal.dll
-filegroup(
-    name = "pywrap_tensorflow_import_lib_file",
-    srcs = [":_pywrap_tensorflow_internal.so"],
-    output_group = "interface_library",
-)
-
-# Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll
-# so that custom ops' dynamic libraries can link against it.
-cc_import(
-    name = "pywrap_tensorflow_import_lib",
-    interface_library = select({
-        "//tensorflow:windows": ":pywrap_tensorflow_import_lib_file",
-        "//conditions:default": "not_exsiting_on_unix.lib",  # Just a placeholder for Unix platforms
-    }),
-    system_provided = 1,
-)
-
-# ** Targets for Windows build (end) **
-
 py_library(
     name = "lib",
     srcs = [
@@ -3732,6 +3677,7 @@ py_test(
     size = "small",
     srcs = ["lib/core/bfloat16_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":lib",
@@ -4012,11 +3958,7 @@ py_test(
     srcs = ["training/checkpoint_utils_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-        "manual",
-        "no_cuda_on_cpu_tap",
-        "no_oss",
         "no_windows",
-        "notap",
     ],
     deps = [
         ":client",
@@ -4039,6 +3981,7 @@ py_test(
     size = "small",
     srcs = ["training/checkpoint_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":checkpoint_ops_gen",
         ":client",
@@ -4079,7 +4022,10 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67945581
+    tags = [
+        "no_windows",
+        "notsan",  # b/67945581
+    ],
     deps = [
         ":array_ops",
         ":client_testlib",
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index c60f692390..512d292ee2 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -913,7 +913,6 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 5245a050a1..9fcbd4ff77 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -139,8 +139,8 @@ class Estimator(object):
                  to configure Estimators from hyper parameter tuning.
           * `config`: Optional configuration object. Will receive what is passed
                  to Estimator in `config` parameter, or the default `config`.
-                 Allows updating things in your `model_fn` based on
-                 configuration such as `num_ps_replicas`, or `model_dir`.
+                 Allows updating things in your model_fn based on configuration
+                 such as `num_ps_replicas`, or `model_dir`.
 
         * Returns:
           `EstimatorSpec`
@@ -301,11 +301,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where `features` is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and `labels` is a
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            `features` and `labels` are consumed by `model_fn`. They should
-            satisfy the expectation of `model_fn` from inputs.
+            features and labels are consumed by `model_fn`. They should satisfy
+            the expectation of `model_fn` from inputs.
 
       hooks: List of `SessionRunHook` subclass instances. Used for callbacks
         inside the training loop.
@@ -381,11 +381,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where `features` is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and `labels` is a
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            `features` and `labels` are consumed by `model_fn`. They should
-            satisfy the expectation of `model_fn` from inputs.
+            features and labels are consumed by `model_fn`. They should satisfy
+            the expectation of `model_fn` from inputs.
 
       steps: Number of steps for which to evaluate model. If `None`, evaluates
         until `input_fn` raises an end-of-input exception.
@@ -457,17 +457,17 @@ class Estimator(object):
       checkpoint_path: Path of a specific checkpoint to predict. If `None`, the
         latest checkpoint in `model_dir` is used.
       yield_single_examples: If False, yield the whole batch as returned by the
-        `model_fn` instead of decomposing the batch into individual elements.
-        This is useful if `model_fn` returns some tensors whose first dimension
-        is not equal to the batch size.
+        model_fn instead of decomposing the batch into individual elements. This
+        is useful if model_fn return some tensor with first dimension not
+        equal to the batch size
 
     Yields:
       Evaluated values of `predictions` tensors.
 
     Raises:
-      ValueError: Could not find a trained model in `model_dir`.
-      ValueError: If batch length of predictions is not the same and
-        `yield_single_examples` is True.
+      ValueError: Could not find a trained model in model_dir.
+      ValueError: if batch length of predictions are not same and
+        yield_single_examples is True.
       ValueError: If there is a conflict between `predict_keys` and
         `predictions`. For example if `predict_keys` is not `None` but
         `EstimatorSpec.predictions` is not a `dict`.
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index e38b765da5..2cc3331a15 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -128,16 +128,9 @@ class TrainSpec(
     """Creates a validated `TrainSpec` instance.
 
     Args:
-      input_fn: A function that provides input data for training as minibatches.
-        See @{$get_started/premade_estimators#create_input_functions} for more
-        information. The function should construct and return one of
-        the following:
-          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
-            tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
-            `Tensor` or a dictionary of string label name to `Tensor`.
-            
+      input_fn: Training input function returning a tuple of:
+          features - `Tensor` or dictionary of string feature name to `Tensor`.
+          labels - `Tensor` or dictionary of `Tensor` with labels.
       max_steps: Int. Positive number of total steps for which to train model.
         If `None`, train forever. The training `input_fn` is not expected to
         generate `OutOfRangeError` or `StopIteration` exceptions. See the
@@ -192,16 +185,9 @@ class EvalSpec(
     """Creates a validated `EvalSpec` instance.
 
     Args:
-      input_fn: A function that constructs the input data for evaluation.
-        See @{$get_started/premade_estimators#create_input_functions} for more
-        information. The function should construct and return one of
-        the following:
-          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
-            tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
-            `Tensor` or a dictionary of string label name to `Tensor`.
-            
+      input_fn: Evaluation input function returning a tuple of:
+          features - `Tensor` or dictionary of string feature name to `Tensor`.
+          labels - `Tensor` or dictionary of `Tensor` with labels.
       steps: Int. Positive number of steps for which to evaluate model. If
         `None`, evaluates until `input_fn` raises an end-of-input exception.
         See `Estimator.evaluate` for details.
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index f74881f179..eef91e9c5b 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -636,10 +636,7 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/utils/io_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",
-    ],
+    tags = ["notsan"],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index f27ca5c205..5b0c38fa5d 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -295,6 +295,7 @@ tf_py_test(
         "//tensorflow/python:nn_grad",
     ],
     data = ["//tensorflow/core:image_testdata"],
+    tags = ["no_windows"],
 )
 
 tf_py_test(
@@ -1138,6 +1139,7 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     data = ["//tensorflow/core:lmdb_testdata"],
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2327,6 +2329,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 4,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2457,6 +2460,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 25525cc128..f4fe01f868 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -970,7 +970,7 @@ class Conv2DTest(test.TestCase):
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 6, 1],
@@ -984,7 +984,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -998,7 +998,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1012,7 +1012,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 4, 3],
@@ -1026,7 +1026,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 3, 1],
@@ -1040,7 +1040,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 6, 1],
@@ -1054,7 +1054,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1068,7 +1068,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[0, 2, 3, 1],
@@ -1082,7 +1082,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputDilation2x1(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         # The GPU version of this test is not very stable. So adjusting the
         # error threshold to 1e-4.
@@ -1098,7 +1098,7 @@ class Conv2DTest(test.TestCase):
             err=1e-4)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropInputDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
+    if test.is_gpu_available(cuda_only=True):
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 3, 1],
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index f0beabb4e2..96c9718b83 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -35,8 +35,8 @@ from tensorflow.python.platform import tf_logging
 
 class DepthToSpaceTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
-    input_nhwc = math_ops.cast(inputs, dtype)
+  def _testOne(self, inputs, block_size, outputs):
+    input_nhwc = math_ops.to_float(inputs)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.depth_to_space(input_nhwc, block_size)
@@ -59,12 +59,6 @@ class DepthToSpaceTest(test.TestCase):
     x_out = [[[[1], [2]], [[3], [4]]]]
     self._testOne(x_np, block_size, x_out)
 
-  def testBasicFloat16(self):
-    x_np = [[[[1, 2, 3, 4]]]]
-    block_size = 2
-    x_out = [[[[1], [2]], [[3], [4]]]]
-    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
-
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testBlockSize2(self):
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index cd90d16aac..b76135764f 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -34,8 +34,8 @@ from tensorflow.python.platform import tf_logging
 
 class SpaceToDepthTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
-    input_nhwc = math_ops.cast(inputs, dtype)
+  def _testOne(self, inputs, block_size, outputs):
+    input_nhwc = math_ops.to_float(inputs)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
@@ -58,12 +58,6 @@ class SpaceToDepthTest(test.TestCase):
     x_out = [[[[1, 2, 3, 4]]]]
     self._testOne(x_np, block_size, x_out)
 
-  def testBasicFloat16(self):
-    x_np = [[[[1], [2]], [[3], [4]]]]
-    block_size = 2
-    x_out = [[[[1, 2, 3, 4]]]]
-    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
-
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testLargerInput2x2(self):
diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py
index 223858edfa..a751607aaa 100644
--- a/tensorflow/python/lib/io/file_io_test.py
+++ b/tensorflow/python/lib/io/file_io_test.py
@@ -485,11 +485,6 @@ class FileIoTest(test.TestCase):
     f.flush()
     self.assertEqual(content, f.read(len(content) + 1))
 
-  def testUTF8StringPathExists(self):
-    file_path = os.path.join(self._base_dir, "UTF8测试_file_exist")
-    file_io.write_string_to_file(file_path, "testing")
-    v = file_io.file_exists(file_path)
-    self.assertEqual(v, True)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 87fe253f18..fb3fe77b4d 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -699,7 +699,7 @@ def convolution(
   `padded_input` is obtained by zero padding the input using an effective
   spatial filter shape of `(spatial_filter_shape-1) * dilation_rate + 1` and
   output striding `strides` as described in the
-  @{$python/nn#Convolution$comment here}.
+  @{tf.nn.convolution$comment here}.
 
   In the case that `data_format` does start with `"NC"`, the `input` and output
   (but not the `filter`) are simply transposed as follows:
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index 42af7f8b27..c59eccc174 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -867,7 +867,7 @@ def raw_rnn(cell, loop_fn,
 
   ```python
   time = tf.constant(0, dtype=tf.int32)
-  (finished, next_input, initial_state, emit_structure, loop_state) = loop_fn(
+  (finished, next_input, initial_state, _, loop_state) = loop_fn(
       time=time, cell_output=None, cell_state=None, loop_state=None)
   emit_ta = TensorArray(dynamic_size=True, dtype=initial_state.dtype)
   state = initial_state
@@ -878,7 +878,7 @@ def raw_rnn(cell, loop_fn,
         loop_state=loop_state)
     # Emit zeros and copy forward state for minibatch entries that are finished.
     state = tf.where(finished, state, next_state)
-    emit = tf.where(finished, tf.zeros_like(emit_structure), emit)
+    emit = tf.where(finished, tf.zeros_like(emit), emit)
     emit_ta = emit_ta.write(time, emit)
     # If any new minibatch entries are marked as finished, mark these.
     finished = tf.logical_or(finished, next_finished)
@@ -938,15 +938,10 @@ def raw_rnn(cell, loop_fn,
       and `emit_output`: the output to store for this iteration.
 
       Note that `emit_output` should be a `Tensor` or (possibly nested)
-      tuple of tensors which is aggregated in the `emit_ta` inside the
-      `while_loop`. For the first call to `loop_fn`, the `emit_output`
-      corresponds to the `emit_structure` which is then used to determine the
-      size of the `zero_tensor` for the `emit_ta` (defaults to
-      `cell.output_size`). For the subsequent calls to the `loop_fn`, the
-      `emit_output` corresponds to the actual output tensor
-      that is to be aggregated in the `emit_ta`. The parameter `cell_state`
-      and output `next_cell_state` may be either a single or (possibly nested)
-      tuple of tensors.  The parameter `loop_state` and
+      tuple of tensors with shapes and structure matching `cell.output_size`
+      and `cell_output` above.  The parameter `cell_state` and output
+      `next_cell_state` may be either a single or (possibly nested) tuple
+      of tensors.  The parameter `loop_state` and
       output `next_loop_state` may be either a single or (possibly nested) tuple
       of `Tensor` and `TensorArray` objects.  This last parameter
       may be ignored by `loop_fn` and the return value may be `None`.  If it
diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py
index 5e2146b79f..6d7eaababc 100644
--- a/tensorflow/python/ops/special_math_ops.py
+++ b/tensorflow/python/ops/special_math_ops.py
@@ -163,7 +163,7 @@ def einsum(equation, *inputs, **kwargs):
     if '...' in equation:
       raise ValueError('Subscripts with ellipses are not yet supported.')
 
-    match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
+    match = re.match('([a-z,]+)(->[a-z]*)?', equation)
     if not match:
       raise ValueError('Indices have incorrect format: %s' % equation)
 
@@ -402,7 +402,7 @@ def _exponential_space_einsum(equation, *inputs):
   if '...' in equation:
     raise ValueError('Subscripts with ellipses are not yet supported.')
 
-  match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
+  match = re.match('([a-z,]+)(->[a-z]*)?', equation)
   if not match:
     raise ValueError('Indices have incorrect format: %s' % equation)
 
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index d7c3a7e8dc..2c212f4548 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -192,9 +192,6 @@ class EinsumTest(test.TestCase):
       'abc,cba',
       'dba,ead,cad->bce',
       'aef,fbc,dca->bde',
-      'iJ,Jk->ik',
-      'iJ,Ki->JK',
-      'iJk,Jklm->Jk'
   ]
 
   long_cases = [
@@ -211,8 +208,6 @@ class EinsumTest(test.TestCase):
       'ijk ijk',
       'ij.jk->ik',
       'ij...,jk...->ik...',
-      'ij,k ->kji',
-      'ij,k-> kji',
 
       # axis in output that does not exist
       'ij,jk->im',
diff --git a/tensorflow/python/tools/freeze_graph.py b/tensorflow/python/tools/freeze_graph.py
index e9f1def48c..a52f325ddb 100644
--- a/tensorflow/python/tools/freeze_graph.py
+++ b/tensorflow/python/tools/freeze_graph.py
@@ -56,6 +56,8 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.tools import saved_model_utils
 from tensorflow.python.training import saver as saver_lib
 
+FLAGS = None
+
 
 def freeze_graph_with_def_protos(input_graph_def,
                                  input_saver_def,
@@ -254,24 +256,25 @@ def freeze_graph(input_graph,
       checkpoint_version=checkpoint_version)
 
 
-def main(unused_args, flags):
-  if flags.checkpoint_version == 1:
+def main(unused_args):
+  if FLAGS.checkpoint_version == 1:
     checkpoint_version = saver_pb2.SaverDef.V1
-  elif flags.checkpoint_version == 2:
+  elif FLAGS.checkpoint_version == 2:
     checkpoint_version = saver_pb2.SaverDef.V2
   else:
     print("Invalid checkpoint version (must be '1' or '2'): %d" %
-          flags.checkpoint_version)
+          FLAGS.checkpoint_version)
     return -1
-  freeze_graph(flags.input_graph, flags.input_saver, flags.input_binary,
-               flags.input_checkpoint, flags.output_node_names,
-               flags.restore_op_name, flags.filename_tensor_name,
-               flags.output_graph, flags.clear_devices, flags.initializer_nodes,
-               flags.variable_names_whitelist, flags.variable_names_blacklist,
-               flags.input_meta_graph, flags.input_saved_model_dir,
-               flags.saved_model_tags, checkpoint_version)
+  freeze_graph(FLAGS.input_graph, FLAGS.input_saver, FLAGS.input_binary,
+               FLAGS.input_checkpoint, FLAGS.output_node_names,
+               FLAGS.restore_op_name, FLAGS.filename_tensor_name,
+               FLAGS.output_graph, FLAGS.clear_devices, FLAGS.initializer_nodes,
+               FLAGS.variable_names_whitelist, FLAGS.variable_names_blacklist,
+               FLAGS.input_meta_graph, FLAGS.input_saved_model_dir,
+               FLAGS.saved_model_tags, checkpoint_version)
+
 
-def run_main():
+if __name__ == "__main__":
   parser = argparse.ArgumentParser()
   parser.register("type", "bool", lambda v: v.lower() == "true")
   parser.add_argument(
@@ -373,10 +376,5 @@ def run_main():
       separated by \',\'. For tag-set contains multiple tags, all tags \
       must be passed in.\
       """)
-  flags, unparsed = parser.parse_known_args()
-
-  my_main = lambda unused_args: main(unused_args, flags)
-  app.run(main=my_main, argv=[sys.argv[0]] + unparsed)
-
-if __name__ == '__main__':
-  run_main()
+  FLAGS, unparsed = parser.parse_known_args()
+  app.run(main=main, argv=[sys.argv[0]] + unparsed)
diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py
index b88be4ae04..b0e9e3e5ed 100644
--- a/tensorflow/python/tools/saved_model_cli.py
+++ b/tensorflow/python/tools/saved_model_cli.py
@@ -38,15 +38,11 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.framework import types_pb2
 from tensorflow.python.client import session
 from tensorflow.python.debug.wrappers import local_cli_wrapper
-from tensorflow.python.framework import meta_graph as meta_graph_lib
 from tensorflow.python.framework import ops as ops_lib
 from tensorflow.python.platform import app  # pylint: disable=unused-import
 from tensorflow.python.saved_model import loader
 from tensorflow.python.tools import saved_model_utils
 
-# Set of ops to blacklist.
-_OP_BLACKLIST = set(['WriteFile', 'ReadFile'])
-
 
 def _show_tag_sets(saved_model_dir):
   """Prints the tag-sets stored in SavedModel directory.
@@ -246,27 +242,6 @@ def get_signature_def_map(saved_model_dir, tag_set):
   return meta_graph.signature_def
 
 
-def scan_meta_graph_def(meta_graph_def):
-  """Scans meta_graph_def and reports if there are ops on blacklist.
-
-  Print ops if they are on black list, or print success if no blacklisted ops
-  found.
-
-  Args:
-    meta_graph_def: MetaGraphDef protocol buffer.
-  """
-  all_ops_set = set(
-      meta_graph_lib.ops_used_by_graph_def(meta_graph_def.graph_def))
-  blacklisted_ops = _OP_BLACKLIST & all_ops_set
-  if blacklisted_ops:
-    # TODO(yifeif): print more warnings
-    print('MetaGraph with tag set %s contains the following blacklisted ops:' %
-          meta_graph_def.meta_info_def.tags, blacklisted_ops)
-  else:
-    print('MetaGraph with tag set %s does not contain blacklisted ops.' %
-          meta_graph_def.meta_info_def.tags)
-
-
 def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key,
                                    input_tensor_key_feed_dict, outdir,
                                    overwrite_flag, tf_debug=False):
@@ -634,21 +609,6 @@ def run(args):
                                  args.overwrite, tf_debug=args.tf_debug)
 
 
-def scan(args):
-  """Function triggered by scan command.
-
-  Args:
-    args: A namespace parsed from command line.
-  """
-  if args.tag_set:
-    scan_meta_graph_def(
-        saved_model_utils.get_meta_graph_def(args.dir, args.tag_set))
-  else:
-    saved_model = reader.read_saved_model(args.dir)
-    for meta_graph_def in saved_model.meta_graphs:
-      scan_meta_graph_def(meta_graph_def)
-
-
 def create_parser():
   """Creates a parser that parse the command line arguments.
 
@@ -770,26 +730,6 @@ def create_parser():
            'SavedModel.')
   parser_run.set_defaults(func=run)
 
-  # scan command
-  scan_msg = ('Usage example:\n'
-              'To scan for blacklisted ops in SavedModel:\n'
-              '$saved_model_cli scan --dir /tmp/saved_model\n'
-              'To scan a specific MetaGraph, pass in --tag_set\n')
-  parser_scan = subparsers.add_parser(
-      'scan',
-      description=scan_msg,
-      formatter_class=argparse.RawTextHelpFormatter)
-  parser_scan.add_argument(
-      '--dir',
-      type=str,
-      required=True,
-      help='directory containing the SavedModel to execute')
-  parser_scan.add_argument(
-      '--tag_set',
-      type=str,
-      help='tag-set of graph in SavedModel to scan, separated by \',\'')
-  parser_scan.set_defaults(func=scan)
-
   return parser
 
 
diff --git a/tensorflow/python/tools/saved_model_cli_test.py b/tensorflow/python/tools/saved_model_cli_test.py
index eedc893a38..f99c844845 100644
--- a/tensorflow/python/tools/saved_model_cli_test.py
+++ b/tensorflow/python/tools/saved_model_cli_test.py
@@ -525,28 +525,6 @@ signature_def['serving_default']:
     y_expected = np.array([[2.5], [3.0]])
     self.assertAllClose(y_expected, y_actual)
 
-  def testScanCommand(self):
-    self.parser = saved_model_cli.create_parser()
-    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
-    args = self.parser.parse_args(['scan', '--dir', base_path])
-    with captured_output() as (out, _):
-      saved_model_cli.scan(args)
-    output = out.getvalue().strip()
-    self.assertTrue('does not contain blacklisted ops' in output)
-
-  def testScanCommandFoundBlacklistedOp(self):
-    self.parser = saved_model_cli.create_parser()
-    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
-    args = self.parser.parse_args(
-        ['scan', '--dir', base_path, '--tag_set', 'serve'])
-    op_blacklist = saved_model_cli._OP_BLACKLIST
-    saved_model_cli._OP_BLACKLIST = set(['VariableV2'])
-    with captured_output() as (out, _):
-      saved_model_cli.scan(args)
-    saved_model_cli._OP_BLACKLIST = op_blacklist
-    output = out.getvalue().strip()
-    self.assertTrue('\'VariableV2\'' in output)
-
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 03e3e0857f..0b3b060fe7 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -274,8 +274,7 @@ CUDNN_DNN_ROUTINE_EACH_R6(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
 // clang-format off
 #if CUDNN_VERSION >= 7000
 #define CUDNN_DNN_ROUTINE_EACH_R7(__macro)                    \
-  __macro(cudnnSetConvolutionMathType)                        \
-  __macro(cudnnSetRNNMatrixMathType)
+  __macro(cudnnSetConvolutionMathType)
 
 // clang-format on
 CUDNN_DNN_ROUTINE_EACH_R7(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
@@ -587,19 +586,6 @@ static bool TensorOpMathEnabled() {
   return is_enabled;
 }
 
-// A helper function to decide whether to enable the TENSOR_OP_MATH math type
-// for RNNs.
-static bool RnnTensorOpMathEnabled() {
-  static bool is_enabled = [] {
-    bool is_disabled = false;
-    TF_CHECK_OK(
-        tensorflow::ReadBoolFromEnvVar("TF_DISABLE_CUDNN_RNN_TENSOR_OP_MATH",
-                                       /*default_val=*/false, &is_disabled));
-    return !is_disabled;
-  }();
-  return is_enabled;
-}
-
 // A helper function to decide whether to use CUDNN_BATCHNORM_SPATIAL_PERSISTENT
 // in batchnorm. This mode can be faster in some tasks because an optimized path
 // may be selected for CUDNN_DATA_FLOAT and CUDNN_DATA_HALF data types, compute
@@ -1138,9 +1124,6 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       SetFailure(cudnn_params_desc_->Status());
       return;
     }
-    if (data_type == CUDNN_DATA_HALF) {
-      set_use_tensor_op_math(true);
-    }
   }
   ~CudnnRnnDescriptor() override {
     if (rnn_desc_) {
@@ -1149,20 +1132,6 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       CUDNN_RETURN_IF_FAIL(status, "Unable to destroy RNN descriptor");
     }
   }
-  void set_use_tensor_op_math(bool use_tensor_op_math) {
-#if CUDNN_VERSION >= 7000
-    cudnnMathType_t math_type =
-        (use_tensor_op_math ? CUDNN_TENSOR_OP_MATH : CUDNN_DEFAULT_MATH);
-    if (RnnTensorOpMathEnabled()) {
-      cudnnStatus_t status =
-          wrap::cudnnSetRNNMatrixMathType(parent_, rnn_desc_, math_type);
-      if (status != CUDNN_STATUS_SUCCESS) {
-        LOG(FATAL) << "could not set cudnn RNN math type: "
-                   << ToString(status);
-      }
-    }
-#endif
-  }
   cudnnRNNDescriptor_t handle() const {
     if (!ok()) return nullptr;
     return rnn_desc_;
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index bab1e82c86..9b0db8a112 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1176,20 +1176,6 @@ def tf_custom_op_library_additional_deps():
       "@protobuf_archive//:protobuf_headers",
       clean_dep("//third_party/eigen3"),
       clean_dep("//tensorflow/core:framework_headers_lib"),
-  ] + if_windows(["//tensorflow/python:pywrap_tensorflow_import_lib"])
-
-# A list of targets that contains the implemenation of
-# tf_custom_op_library_additional_deps. It's used to generate a DEF file for
-# exporting symbols from _pywrap_tensorflow.dll on Windows.
-def tf_custom_op_library_additional_deps_impl():
-  return [
-      # for @nsync//:nsync_headers
-      "//third_party/nsync:nsync_cpp",
-      # for //third_party/eigen3
-      clean_dep("//third_party/eigen3"),
-      # for //tensorflow/core:framework_headers_lib
-      clean_dep("//tensorflow/core:framework"),
-      clean_dep("//tensorflow/core:reader_base"),
   ]
 
 # Traverse the dependency graph along the "deps" attribute of the
@@ -1276,7 +1262,6 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]):
       deps=deps + if_cuda(cuda_deps),
       data=[name + "_check_deps"],
       copts=tf_copts(is_external=True),
-      features = ["windows_export_all_symbols"],
       linkopts=linkopts + select({
           "//conditions:default": [
               "-lm",
@@ -1423,8 +1408,7 @@ def tf_py_wrap_cc(name,
       ]) + tf_extension_copts()),
       linkopts=tf_extension_linkopts() + extra_linkopts,
       linkstatic=1,
-      deps=deps + extra_deps,
-      **kwargs)
+      deps=deps + extra_deps)
   native.genrule(
       name="gen_" + cc_library_pyd_name,
       srcs=[":" + cc_library_name],
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index baa7a0889d..5268bba3cc 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -247,8 +247,6 @@ class ApiCompatibilityTest(test.TestCase):
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.do_not_descend_map['tf'].append('contrib')
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
-    # TODO(annarev): Make slide_dataset available in API.
-    public_api_visitor.private_map['tf'] = ['slide_dataset']
     traverse.traverse(api, public_api_visitor)
 
     proto_dict = visitor.GetProtos()
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index d5dea4f3e4..ec90c83aac 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -23,12 +23,11 @@ RUN /install/install_deb_packages.sh
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends python-pip
-RUN pip install --upgrade wheel
 RUN pip install --upgrade astor
 RUN pip install --upgrade gast
 RUN pip install --upgrade numpy
 RUN pip install --upgrade termcolor
 
 # Install golang
-RUN apt-get install -t xenial-backports -y golang-1.9
-ENV PATH=${PATH}:/usr/lib/go-1.9/bin
+RUN add-apt-repository -y ppa:ubuntu-lxc/lxd-stable
+RUN apt-get install -y golang
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 40189a6d1b..8b8ba31a0d 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -65,5 +65,4 @@ bazel test -c opt $BUILD_OPTS -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
-  //${PY_TEST_DIR}/tensorflow/python/... \
-  //${PY_TEST_DIR}/tensorflow/contrib/...
+  //${PY_TEST_DIR}/tensorflow/python/...
diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl
deleted file mode 100644
index 3cb72f4979..0000000000
--- a/tensorflow/tools/def_file_filter/BUILD.tpl
+++ /dev/null
@@ -1,15 +0,0 @@
-# Description:
-# Tools for filtering DEF file for TensorFlow on Windows
-#
-# On Windows, we use a DEF file generated by Bazel to export
-# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll).
-# The maximum number of symbols that can be exported per DLL is 64K,
-# so we have to filter some useless symbols through this python script.
-
-package(default_visibility = ["//visibility:public"])
-
-py_binary(
-    name = "def_file_filter",
-    srcs = ["def_file_filter.py"],
-    srcs_version = "PY2AND3",
-)
diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
deleted file mode 100644
index 8bdc03eb0f..0000000000
--- a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+++ /dev/null
@@ -1,168 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""def_file_filter.py - tool to filter a windows def file.
-
-The def file can be used to export symbols from the tensorflow dll to enable
-tf.load_library().
-
-Because the linker allows only 64K symbols to be exported per dll
-we filter the symbols down to the essentials. The regular expressions
-we use for this are specific to tensorflow.
-
-TODO: this works fine but there is an issue with exporting
-'const char * const' and importing it from a user_ops. The problem is
-on the importing end and using __declspec(dllimport) works around it.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import io
-import os
-import re
-import subprocess
-import sys
-import tempfile
-
-# External tools we use that come with visual studio sdk
-UNDNAME = "%{undname_bin_path}"
-
-# Exclude if matched
-EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::")
-
-# Include if matched before exclude
-INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|"
-                           r"google::protobuf::internal::ArenaImpl::AllocateAligned|" # for contrib/data/_prefetching_ops
-                           r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops
-                           r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops
-                           r"tensorflow::internal::LogMessage|"
-                           r"tensorflow::internal::LogString|"
-                           r"tensorflow::internal::CheckOpMessageBuilder|"
-                           r"tensorflow::internal::MakeCheckOpValueString|"
-                           r"tensorflow::internal::PickUnusedPortOrDie|"
-                           r"tensorflow::internal::ValidateDevice|"
-                           r"tensorflow::ops::internal::Enter|"
-                           r"tensorflow::strings::internal::AppendPieces|"
-                           r"tensorflow::strings::internal::CatPieces|"
-                           r"tensorflow::io::internal::JoinPathImpl")
-
-# Include if matched after exclude
-INCLUDE_RE = re.compile(r"^(TF_\w*)$|"
-                        r"^(TFE_\w*)$|"
-                        r"nsync::|"
-                        r"tensorflow::|"
-                        r"functor::|"
-                        r"perftools::gputools")
-
-# We want to identify data members explicitly in the DEF file, so that no one
-# can implicitly link against the DLL if they use one of the variables exported
-# from the DLL and the header they use does not decorate the symbol with
-# __declspec(dllimport). It is easier to detect what a data symbol does
-# NOT look like, so doing it with the below regex.
-DATA_EXCLUDE_RE = re.compile(r"[)(]|"
-                             r"vftable|"
-                             r"vbtable|"
-                             r"vcall|"
-                             r"RTTI|"
-                             r"protobuf::internal::ExplicitlyConstructed")
-
-def get_args():
-  """Parse command line."""
-  filename_list = lambda x: x.split(";")
-  parser = argparse.ArgumentParser()
-  parser.add_argument("--input", type=filename_list,
-                      help="paths to input def file",
-                      required=True)
-  parser.add_argument("--output", help="output deffile", required=True)
-  parser.add_argument("--target", help="name of the target", required=True)
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  """main."""
-  args = get_args()
-
-  # Pipe dumpbin to extract all linkable symbols from libs.
-  # Good symbols are collected in candidates and also written to
-  # a temp file.
-  candidates = []
-  tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False)
-  for def_file_path in args.input:
-    def_file = open(def_file_path, 'r')
-    for line in def_file:
-      cols = line.split()
-      sym = cols[0]
-      tmpfile.file.write(sym + "\n")
-      candidates.append(sym)
-  tmpfile.file.close()
-
-  # Run the symbols through undname to get their undecorated name
-  # so we can filter on something readable.
-  with open(args.output, "w") as def_fp:
-    # track dupes
-    taken = set()
-
-    # Header for the def file.
-    def_fp.write("LIBRARY " + args.target + "\n")
-    def_fp.write("EXPORTS\n")
-    def_fp.write("\t ??1OpDef@tensorflow@@UEAA@XZ\n")
-
-    # Each symbols returned by undname matches the same position in candidates.
-    # We compare on undname but use the decorated name from candidates.
-    dupes = 0
-    proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE)
-    for idx, line in enumerate(io.TextIOWrapper(proc.stdout, encoding="utf-8")):
-      decorated = candidates[idx]
-      if decorated in taken:
-        # Symbol is already in output, done.
-        dupes += 1
-        continue
-
-      if not INCLUDEPRE_RE.search(line):
-        if EXCLUDE_RE.search(line):
-          continue
-        if not INCLUDE_RE.search(line):
-          continue
-
-      if "deleting destructor" in line:
-        # Some of the symbols convered by INCLUDEPRE_RE export deleting
-        # destructor symbols, which is a bad idea.
-        # So we filter out such symbols here.
-        continue
-
-      if DATA_EXCLUDE_RE.search(line):
-        def_fp.write("\t" + decorated + "\n")
-      else:
-        def_fp.write("\t" + decorated + " DATA\n")
-      taken.add(decorated)
-    def_fp.close()
-
-  exit_code = proc.wait()
-  if exit_code != 0:
-    print("{} failed, exit={}".format(UNDNAME, exit_code))
-    return exit_code
-
-  os.unlink(tmpfile.name)
-
-  print("symbols={}, taken={}, dupes={}"
-        .format(len(candidates), len(taken), dupes))
-  return 0
-
-
-if __name__ == "__main__":
-  sys.exit(main())
diff --git a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
deleted file mode 100644
index 47539b2423..0000000000
--- a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+++ /dev/null
@@ -1,56 +0,0 @@
-"""Repository rule for def file filter autoconfiguration.
-
-This repository reuses Bazel's VC detect mechanism to find undname.exe,
-which is a tool used in def_file_filter.py.
-
-def_file_filter.py is for filtering the DEF file for TensorFlow on Windows.
-On Windows, we use a DEF file generated by Bazel to export symbols from the
-tensorflow dynamic library(_pywrap_tensorflow.dll). The maximum number of
-symbols that can be exported per DLL is 64K, so we have to filter some useless
-symbols through this python script.
-
-`def_file_filter_config` depends on the following environment variables:
-  * `BAZEL_VC`
-  * `BAZEL_VS`
-  * `VS90COMNTOOLS`
-  * `VS100COMNTOOLS`
-  * `VS110COMNTOOLS`
-  * `VS120COMNTOOLS`
-  * `VS140COMNTOOLS`
-"""
-
-load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_vc_path")
-load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_msvc_tool")
-load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "auto_configure_fail")
-
-def _def_file_filter_configure_impl(repository_ctx):
-  if repository_ctx.os.name.lower().find("windows") == -1:
-    repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
-    repository_ctx.file("def_file_filter.py", "")
-    return
-  vc_path = find_vc_path(repository_ctx)
-  if vc_path == "visual-studio-not-found":
-    auto_configure_fail("Visual C++ build tools not found on your machine")
-  undname_bin_path = find_msvc_tool(repository_ctx, vc_path, "undname.exe").replace("\\", "\\\\")
-
-  repository_ctx.template(
-    "def_file_filter.py",
-    Label("//tensorflow/tools/def_file_filter:def_file_filter.py.tpl"),
-    {
-      "%{undname_bin_path}": undname_bin_path,
-    })
-  repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
-
-
-def_file_filter_configure = repository_rule(
-    implementation = _def_file_filter_configure_impl,
-    environ = [
-        "BAZEL_VC",
-        "BAZEL_VS",
-        "VS90COMNTOOLS",
-        "VS100COMNTOOLS",
-        "VS110COMNTOOLS",
-        "VS120COMNTOOLS",
-        "VS140COMNTOOLS"
-    ],
-)
diff --git a/tensorflow/tools/dist_test/README.md b/tensorflow/tools/dist_test/README.md
index 228d5ee35d..c1b1f79bbd 100644
--- a/tensorflow/tools/dist_test/README.md
+++ b/tensorflow/tools/dist_test/README.md
@@ -17,14 +17,6 @@ cesnsu model:
 
     ./local_test.sh --model_name CENSUS_WIDENDEEP
 
-You can test specify version of TensorFlow:
-
-```shell
-./local_test.sh ${whl_file_url}
-```
-
-For example, you can find these TensorFlow python package URLs from [here](https://www.tensorflow.org/install/install_linux#the_url_of_the_tensorflow_python_package) for Ubuntu.
-
 **2) Launch a remote k8s cluster on Google Kubernetes Engine (GKE) and run the
 test suite on it**
 
diff --git a/tensorflow/tools/dist_test/local_test.sh b/tensorflow/tools/dist_test/local_test.sh
index caae7fd530..435f9d0dc9 100755
--- a/tensorflow/tools/dist_test/local_test.sh
+++ b/tensorflow/tools/dist_test/local_test.sh
@@ -16,11 +16,12 @@
 #
 # Tests distributed TensorFlow on a locally running TF GRPC cluster.
 #
-# This script performs the following steps:
-# 1) Build the docker image capable of running distributed TensorFlow in docker.
+# This script peforms the following steps:
+# 1) Build the docker-in-docker (dind) image capable of running docker and
+#    Kubernetes (k8s) cluster inside.
 # 2) Run a container from the aforementioned image and start docker service
 #    in it
-# 3) Call a script to launch a distributed TensorFlow GRPC cluster inside the container
+# 3) Call a script to launch a k8s TensorFlow GRPC cluster inside the container
 #    and run the distributed test suite.
 #
 # Usage: local_test.sh <whl_file_location>
@@ -63,9 +64,15 @@ die() {
 
 # Configurations
 DOCKER_IMG_NAME="tensorflow/tf-dist-test-local-cluster"
+LOCAL_K8S_CACHE=${HOME}/kubernetes
 
-# Use TensorFlow v1.5.0 for Python 2.7 and CPU only as we set num_gpus to 0 in the below
-DEFAULT_WHL_FILE_LOCATION="https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.5.0-cp27-none-linux_x86_64.whl"
+# Helper function
+get_container_id_by_image_name() {
+    # Get the id of a container by image name
+    # Usage: get_docker_container_id_by_image_name <img_name>
+
+    docker ps | grep $1 | awk '{print $1}'
+}
 
 # Parse input arguments
 LEAVE_CONTAINER_RUNNING=0
@@ -77,8 +84,7 @@ SYNC_REPLICAS_FLAG=""
 
 WHL_FILE_LOCATION=${1}
 if [[ -z "${WHL_FILE_LOCATION}" ]]; then
-  WHL_FILE_LOCATION=${DEFAULT_WHL_FILE_LOCATION}
-  echo "use default whl file location"
+  die "whl file location is not specified"
 fi
 
 while true; do
@@ -115,7 +121,7 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Get utility functions
 source ${DIR}/scripts/utils.sh
 
-# Build docker image for local distributed TensorFlow cluster.
+# Build docker-in-docker image for local k8s cluster.
 NO_CACHE_FLAG=""
 if [[ ! -z "${TF_DIST_DOCKER_NO_CACHE}" ]] &&
    [[ "${TF_DIST_DOCKER_NO_CACHE}" != "0" ]]; then
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index cbcdbf5b80..3630dbd740 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -114,13 +114,6 @@ def configure(src_base_path, gen_path, debug=False):
   for target, src in link_map.items():
     if src is None:
       open(os.path.join(gen_path, target), "w").write("")
-    elif not os.path.exists(src):
-      # Git repo is configured in a way we don't support such as having
-      # packed refs. Even though in a git repo, tf.__git_version__ will not
-      # be accurate.
-      # TODO(mikecase): Support grabbing git info when using packed refs.
-      open(os.path.join(gen_path, target), "w").write("")
-      spec["git"] = False
     else:
       try:
         # In python 3.5, symlink function exists even on Windows. But requires
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index 6e21aa2846..b7d7fac315 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -178,7 +178,6 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
-        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/util/tensor_bundle",
     ],
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index d86f65325b..d89afe85c7 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -182,36 +182,6 @@ Status FuseBatchNormWithConv(const NodeMatch& match,
   return Status::OK();
 }
 
-Status FuseBatchNormWithBatchToSpace(const NodeMatch& match,
-                             std::vector<NodeDef>* new_nodes) {
-  // Calculate the scale and offset values to apply.
-  std::vector<float> scale_values;
-  std::vector<float> offset_values;
-  TF_RETURN_IF_ERROR(
-      GetScaleAndOffsetValues(match, &scale_values, &offset_values));
-
-  // Fuse conv weights, and set the final output node name as batch_norm_node.
-  const NodeDef& batch_norm_node = match.node;
-  const NodeMatch& batch_to_space_node_match = match.inputs[0];
-  const NodeMatch& conv_node_match = batch_to_space_node_match.inputs[0];
-  const NodeDef& batch_to_space_node = batch_to_space_node_match.node;
-  const NodeDef& conv_node = conv_node_match.node;
-
-  string biasadd_name = conv_node.name() + "/biasadd";
-  TF_RETURN_IF_ERROR(
-      FuseScaleOffsetToConvWeights(scale_values, offset_values, conv_node_match,
-                                   biasadd_name , new_nodes));
-
-  NodeDef new_batch_to_space_node = batch_to_space_node;
-  // reuse batch_norm node name
-  new_batch_to_space_node.set_name(batch_norm_node.name());
-  new_batch_to_space_node.set_input(0, biasadd_name);
-  new_nodes->push_back(batch_to_space_node_match.inputs[1].node);
-  new_nodes->push_back(batch_to_space_node_match.inputs[2].node);
-  new_nodes->push_back(new_batch_to_space_node);
-  return Status::OK();
-}
-
 Status FuseBatchNormWithConvConcat(const NodeMatch& match,
                                    std::vector<NodeDef>* new_nodes) {
   // Calculate the scale and offset values to apply.
@@ -314,43 +284,6 @@ Status FoldOldBatchNorms(const GraphDef& input_graph_def,
     current_graph_def = replaced_graph_def;
   } while (did_graph_change);
 
-  do {
-    did_graph_change = false;
-    GraphDef replaced_graph_def;
-    TF_RETURN_IF_ERROR(ReplaceMatchingOpTypes(
-        current_graph_def,  // clang-format off
-        {"BatchNormWithGlobalNormalization|FusedBatchNorm",    // batch_norm_node
-         {
-             {"BatchToSpaceND",                  // batch_to_space_node
-              {
-                  {"Conv2D",                     // conv_node
-                   {
-                       {"*"},                    // input_node
-                       {"Const"},                // weights_node
-                   }
-                  },
-                  {"Const"},                     // block_shape
-                  {"Const"},                     // crops
-              }
-             },
-             {"Const"},                          // mean_node
-             {"Const"},                          // variance_node
-             {"Const"},                          // beta_node
-             {"Const"},                          // gamma_node
-         }
-        },  // clang-format on
-        [&did_graph_change](const NodeMatch& match,
-                            const std::set<string>& input_nodes,
-                            const std::set<string>& output_nodes,
-                            std::vector<NodeDef>* new_nodes) {
-          TF_RETURN_IF_ERROR(FuseBatchNormWithBatchToSpace(match, new_nodes));
-          did_graph_change = true;
-          return Status::OK();
-        },
-        {}, &replaced_graph_def));
-    current_graph_def = replaced_graph_def;
-  } while (did_graph_change);
-
   do {
     did_graph_change = false;
     GraphDef replaced_graph_def;
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 272410c693..b30ba9ac8b 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
-#include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
@@ -299,96 +298,6 @@ class FoldOldBatchNormsTest : public ::testing::Test {
   }
 };
 
-void TestFoldFusedBatchNormsWithBatchToSpace() {
-  auto root = tensorflow::Scope::NewRootScope();
-  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
-
-  Tensor input_data(DT_FLOAT, TensorShape({2, 1, 3, 2}));
-  test::FillValues<float>(
-      &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
-                    -5.0f, -3.0f, -6.0f});
-  Output input_op =
-      Const(root.WithOpName("input_op"), Input::Initializer(input_data));
-
-  Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
-  test::FillValues<float>(&weights_data,
-                          {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
-  Output weights_op =
-      Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
-
-  Output conv_op = Conv2D(root.WithOpName("conv_op"), input_op, weights_op,
-                          {1, 1, 1, 1}, "VALID");
-
-  Tensor block_shape_data(DT_INT32, TensorShape({2}));
-  test::FillValues<int32>(&block_shape_data, {1, 2});
-  Output block_shape_op =
-      Const(root.WithOpName("block_shape_op"), Input::Initializer(block_shape_data));
-
-  Tensor crops_data(DT_INT32, TensorShape({2, 2}));
-  test::FillValues<int32>(&crops_data, {0, 0, 0, 1});
-  Output crops_op =
-      Const(root.WithOpName("crops_op"), Input::Initializer(crops_data));
-
-  Output batch_to_space_op = BatchToSpaceND(root.WithOpName("batch_to_space_op"),
-                                            conv_op, block_shape_op, crops_data);
-
-  Tensor mean_data(DT_FLOAT, TensorShape({2}));
-  test::FillValues<float>(&mean_data, {10.0f, 20.0f});
-  Output mean_op =
-      Const(root.WithOpName("mean_op"), Input::Initializer(mean_data));
-
-  Tensor variance_data(DT_FLOAT, TensorShape({2}));
-  test::FillValues<float>(&variance_data, {0.25f, 0.5f});
-  Output variance_op = Const(root.WithOpName("variance_op"),
-                             Input::Initializer(variance_data));
-
-  Tensor beta_data(DT_FLOAT, TensorShape({2}));
-  test::FillValues<float>(&beta_data, {0.1f, 0.6f});
-  Output beta_op =
-      Const(root.WithOpName("beta_op"), Input::Initializer(beta_data));
-
-  Tensor gamma_data(DT_FLOAT, TensorShape({2}));
-  test::FillValues<float>(&gamma_data, {1.0f, 2.0f});
-  Output gamma_op =
-      Const(root.WithOpName("gamma_op"), Input::Initializer(gamma_data));
-
-  GraphDef original_graph_def;
-  TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
-
-  NodeDef batch_norm_node;
-  batch_norm_node.set_op("FusedBatchNorm");
-  batch_norm_node.set_name("output");
-  AddNodeInput("batch_to_space_op", &batch_norm_node);
-  AddNodeInput("gamma_op", &batch_norm_node);
-  AddNodeInput("beta_op", &batch_norm_node);
-  AddNodeInput("mean_op", &batch_norm_node);
-  AddNodeInput("variance_op", &batch_norm_node);
-  SetNodeAttr("T", DT_FLOAT, &batch_norm_node);
-  SetNodeAttr("epsilon", 0.00001f, &batch_norm_node);
-  SetNodeAttr("is_training", false, &batch_norm_node);
-  *(original_graph_def.mutable_node()->Add()) = batch_norm_node;
-
-  std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
-  TF_ASSERT_OK(original_session->Create(original_graph_def));
-  std::vector<Tensor> original_outputs;
-  TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
-
-  GraphDef fused_graph_def;
-  TF_ASSERT_OK(FoldOldBatchNorms(original_graph_def, {{}, {"output"}},
-                                 &fused_graph_def));
-
-  std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
-  TF_ASSERT_OK(fused_session->Create(fused_graph_def));
-  std::vector<Tensor> fused_outputs;
-  TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
-
-  test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
-
-  for (const NodeDef& node : fused_graph_def.node()) {
-    EXPECT_NE("FusedBatchNormWithBatchToSpace", node.op());
-  }
-}
-
 TEST_F(FoldOldBatchNormsTest, TestFoldOldBatchNorms) {
   TestFoldOldBatchNorms();
 }
@@ -406,9 +315,5 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) {
   TestFoldFusedBatchNormsWithConcat(/*split=*/false);
 }
 
-TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithBatchToSpace) {
-  TestFoldFusedBatchNormsWithBatchToSpace();
-}
-
 }  // namespace graph_transforms
 }  // namespace tensorflow
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 2607b9d704..1833d67d82 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -48,65 +48,36 @@ py_binary(
     deps = ["//tensorflow:tensorflow_py"],
 )
 
-COMMON_PIP_DEPS = [
-    ":licenses",
-    "MANIFEST.in",
-    "README",
-    "setup.py",
-    ":included_headers",
-    "//tensorflow:tensorflow_py",
-    "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
-    "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
-    "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
-    "//tensorflow/contrib/data/python/ops:contrib_op_loader",
-    "//tensorflow/contrib/eager/python/examples:examples_pip",
-    "//tensorflow/contrib/eager/python:checkpointable_utils",
-    "//tensorflow/contrib/eager/python:evaluator",
-    "//tensorflow/contrib/gan:gan",
-    "//tensorflow/contrib/graph_editor:graph_editor_pip",
-    "//tensorflow/contrib/keras:keras",
-    "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
-    "//tensorflow/contrib/nn:nn_py",
-    "//tensorflow/contrib/predictor:predictor_pip",
-    "//tensorflow/contrib/py2tf:py2tf",
-    "//tensorflow/contrib/py2tf/converters:converters",
-    "//tensorflow/contrib/py2tf/converters:test_lib",
-    "//tensorflow/contrib/py2tf/impl:impl",
-    "//tensorflow/contrib/py2tf/pyct:pyct",
-    "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
-    "//tensorflow/contrib/receptive_field:receptive_field_pip",
-    "//tensorflow/contrib/session_bundle:session_bundle_pip",
-    "//tensorflow/contrib/signal:signal_py",
-    "//tensorflow/contrib/signal:test_util",
-    "//tensorflow/contrib/slim:slim",
-    "//tensorflow/contrib/slim/python/slim/data:data_pip",
-    "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-    "//tensorflow/contrib/specs:specs",
-    "//tensorflow/contrib/summary:summary_test_util",
-    "//tensorflow/contrib/tensor_forest:init_py",
-    "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-    "//tensorflow/contrib/timeseries:timeseries_pip",
-    "//tensorflow/contrib/tpu",
-    "//tensorflow/examples/tutorials/mnist:package",
-    "//tensorflow/python:distributed_framework_test_lib",
-    "//tensorflow/python:meta_graph_testdata",
-    "//tensorflow/python:spectral_ops_test_util",
-    "//tensorflow/python:util_example_parser_configuration",
-    "//tensorflow/python/debug:debug_pip",
-    "//tensorflow/python/eager:eager_pip",
-    "//tensorflow/python/saved_model:saved_model",
-    "//tensorflow/python/tools:tools_pip",
-    "//tensorflow/python:test_ops",
-    "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
-]
-
 # On Windows, python binary is a zip file of runfiles tree.
 # Add everything to its data dependency for generating a runfiles tree
 # for building the pip package on Windows.
 py_binary(
     name = "simple_console_for_windows",
     srcs = ["simple_console_for_windows.py"],
-    data = COMMON_PIP_DEPS,
+    data = [
+        "MANIFEST.in",
+        "README",
+        "setup.py",
+        ":included_headers",
+        "//tensorflow/contrib/nn:nn_py",
+        "//tensorflow/contrib/session_bundle:session_bundle_pip",
+        "//tensorflow/contrib/signal:signal_py",
+        "//tensorflow/contrib/slim/python/slim/data:data_pip",
+        "//tensorflow/python:util_example_parser_configuration",
+        "//tensorflow/python/debug:debug_pip",
+        "//tensorflow/python/saved_model",
+        "//tensorflow/python:spectral_ops_test_util",
+        "//tensorflow/python/tools:tools_pip",
+        "//tensorflow/python/eager:eager_pip",
+        "//tensorflow/contrib/summary:summary_test_util",
+        # These targets don't build on Windows yet. Exclude them for now.
+        # "//tensorflow/contrib/slim",
+        # "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+        # "//tensorflow/contrib/specs",
+        # "//tensorflow/contrib/tensor_forest:init_py",
+        # "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+        # "//tensorflow/examples/tutorials/mnist:package",
+    ],
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
@@ -137,7 +108,6 @@ filegroup(
         "@highwayhash//:LICENSE",
         "@jemalloc//:COPYING",
         "@jpeg//:LICENSE.md",
-        "@kafka//:LICENSE",
         "@libxsmm_archive//:LICENSE",
         "@lmdb//:LICENSE",
         "@local_config_sycl//sycl:LICENSE.text",
@@ -167,12 +137,61 @@ sh_binary(
     data = select({
         "//tensorflow:windows": [":simple_console_for_windows"],
         "//tensorflow:windows_msvc": [":simple_console_for_windows"],
-        "//conditions:default": COMMON_PIP_DEPS + [
+        "//conditions:default": [
+            ":licenses",
+            "MANIFEST.in",
+            "README",
+            "setup.py",
+            ":included_headers",
             ":simple_console",
+            "//tensorflow:tensorflow_py",
+            "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
+            "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
+            "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
+            "//tensorflow/contrib/data/python/ops:contrib_op_loader",
+            "//tensorflow/contrib/eager/python/examples:examples_pip",
+            "//tensorflow/contrib/eager/python:checkpointable_utils",
+            "//tensorflow/contrib/eager/python:evaluator",
+            "//tensorflow/contrib/gan:gan",
+            "//tensorflow/contrib/graph_editor:graph_editor_pip",
+            "//tensorflow/contrib/keras:keras",
+            "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
             "//tensorflow/contrib/lite/python:interpreter_test_data",
             "//tensorflow/contrib/lite/toco:toco",
             "//tensorflow/contrib/lite/toco/python:toco_wrapper",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
+            "//tensorflow/contrib/nn:nn_py",
+            "//tensorflow/contrib/predictor:predictor_pip",
+            "//tensorflow/contrib/py2tf:py2tf",
+            "//tensorflow/contrib/py2tf/converters:converters",
+            "//tensorflow/contrib/py2tf/converters:test_lib",
+            "//tensorflow/contrib/py2tf/impl:impl",
+            "//tensorflow/contrib/py2tf/pyct:pyct",
+            "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
+            "//tensorflow/contrib/receptive_field:receptive_field_pip",
+            "//tensorflow/contrib/session_bundle:session_bundle_pip",
+            "//tensorflow/contrib/signal:signal_py",
+            "//tensorflow/contrib/signal:test_util",
+            "//tensorflow/contrib/slim:slim",
+            "//tensorflow/contrib/slim/python/slim/data:data_pip",
+            "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+            "//tensorflow/contrib/specs:specs",
+            "//tensorflow/contrib/summary:summary_test_util",
+            "//tensorflow/contrib/tensor_forest:init_py",
+            "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+            "//tensorflow/contrib/timeseries:timeseries_pip",
+            "//tensorflow/contrib/tpu",
+            "//tensorflow/examples/tutorials/mnist:package",
+            "//tensorflow/python:distributed_framework_test_lib",
+            "//tensorflow/python:meta_graph_testdata",
+            "//tensorflow/python:spectral_ops_test_util",
+            "//tensorflow/python:util_example_parser_configuration",
+            "//tensorflow/python/debug:debug_pip",
+            "//tensorflow/python/eager:eager_pip",
+            "//tensorflow/python/saved_model:saved_model",
+            "//tensorflow/python/tools:tools_pip",
+            "//tensorflow/python:test_ops",
+            "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
         ],
     }) + if_mkl(["//third_party/mkl:intel_binary_blob"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e0152da4df..e1a5f091ba 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0'
+_VERSION = '1.6.0-rc1'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -72,7 +72,7 @@ if sys.version_info < (3, 4):
 
 # pylint: disable=line-too-long
 CONSOLE_SCRIPTS = [
-    'freeze_graph = tensorflow.python.tools.freeze_graph:run_main',
+    'freeze_graph = tensorflow.python.tools.freeze_graph:main',
     'toco_from_protos = tensorflow.contrib.lite.toco.python.toco_from_protos:main',
     'toco = tensorflow.contrib.lite.toco.python.toco_wrapper:main',
     'saved_model_cli = tensorflow.python.tools.saved_model_cli:main',
diff --git a/tensorflow/tools/test/upload_test_benchmarks.py b/tensorflow/tools/test/upload_test_benchmarks.py
index edd093510e..77cc9f75f7 100644
--- a/tensorflow/tools/test/upload_test_benchmarks.py
+++ b/tensorflow/tools/test/upload_test_benchmarks.py
@@ -88,7 +88,6 @@ import os
 import shutil
 
 from google.cloud import datastore
-from six import text_type
 
 
 def is_real_file(dirpath, fname):
@@ -151,7 +150,7 @@ def upload_benchmark_data(client, data):
   """
   test_result = json.loads(data)
 
-  test_name = text_type(test_result["name"])
+  test_name = unicode(test_result["name"])
   start_time = datetime.datetime.utcfromtimestamp(
       float(test_result["startTime"]))
   batch = []
@@ -163,7 +162,7 @@ def upload_benchmark_data(client, data):
   t_val.update({
       "test": test_name,
       "start": start_time,
-      "info": text_type(data)
+      "info": unicode(data)
   })
   batch.append(t_val)
 
@@ -171,7 +170,7 @@ def upload_benchmark_data(client, data):
   # the attribute to be fetched and displayed.  The full entry information is
   # also stored as a non-indexed JSON blob.
   for ent in test_result["entries"].get("entry", []):
-    ent_name = text_type(ent["name"])
+    ent_name = unicode(ent["name"])
     e_key = client.key("Entry")
     e_val = datastore.Entity(e_key, exclude_from_indexes=["info"])
     e_val.update({
@@ -179,7 +178,7 @@ def upload_benchmark_data(client, data):
         "start": start_time,
         "entry": ent_name,
         "timing": ent["wallTime"],
-        "info": text_type(json.dumps(ent))
+        "info": unicode(json.dumps(ent))
     })
     batch.append(e_val)
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index a922808a70..abc9eb9bc1 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -12,8 +12,6 @@ load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compil
 load("//third_party:repo.bzl", "tf_http_archive")
 load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
-load("//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
-     "def_file_filter_configure")
 
 def _extract_version_number(bazel_version):
   """Extracts the semantic version number from a version string
@@ -69,7 +67,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # We must check the bazel version before trying to parse any other BUILD
   # files, in case the parsing of those build files depends on the bazel
   # version we require here.
-  check_bazel_version_at_least("0.10.0")
+  check_bazel_version_at_least("0.5.4")
   clang6_configure(name="local_config_clang6")
   cuda_configure(name="local_config_cuda")
   tensorrt_configure(name="local_config_tensorrt")
@@ -77,10 +75,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   sycl_configure(name="local_config_sycl")
   python_configure(name="local_config_python")
 
-  # For windows bazel build
-  # TODO: Remove def file filter when TensorFlow can export symbols properly on Windows.
-  def_file_filter_configure(name = "local_config_def_file_filter")
-
   # Point //external/local_config_arm_compiler to //external/arm_compiler
   arm_compiler_configure(
       name="local_config_arm_compiler",
diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD
index 4418ac32fc..87a23925c4 100644
--- a/third_party/jpeg/jpeg.BUILD
+++ b/third_party/jpeg/jpeg.BUILD
@@ -526,12 +526,12 @@ config_setting(
 
 config_setting(
     name = "armeabi-v7a",
-    values = {"cpu": "armeabi-v7a"},
+    values = {"android_cpu": "armeabi-v7a"},
 )
 
 config_setting(
     name = "arm64-v8a",
-    values = {"cpu": "arm64-v8a"},
+    values = {"android_cpu": "arm64-v8a"},
 )
 
 config_setting(
diff --git a/third_party/kafka/BUILD b/third_party/kafka/BUILD
index a839ca717e..a61a9e1f6c 100644
--- a/third_party/kafka/BUILD
+++ b/third_party/kafka/BUILD
@@ -130,23 +130,18 @@ cc_library(
     ],
     hdrs = [
         "config.h",
-        "src-cpp/rdkafkacpp.h",
-        "src-cpp/rdkafkacpp_int.h",
-        "src/lz4.c",
-        "src/snappy_compat.h",
-    ],
-    copts = [
-        "-Iexternal/kafka/src",
-        "-Iexternal/kafka/src-cpp",
     ],
     defines = [
     ],
+    includes = [
+        "src",
+        "src-cpp",
+    ],
     linkopts = [
         "-lpthread",
     ],
     visibility = ["//visibility:public"],
     deps = [
         "@boringssl//:ssl",
-        "@zlib_archive//:zlib",
     ],
 )
diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl
index 1dd8ab433a..de06ad5f27 100644
--- a/third_party/py/BUILD.tpl
+++ b/third_party/py/BUILD.tpl
@@ -2,26 +2,20 @@ licenses(["restricted"])
 
 package(default_visibility = ["//visibility:public"])
 
-# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
-# See https://docs.python.org/3/extending/windows.html
-cc_import(
-    name = "python_lib",
-    interface_library = select({
-        ":windows": ":python_import_lib",
-        # A placeholder for Unix platforms which makes --no_build happy.
-        "//conditions:default": "not-existing.lib",
-    }),
-    system_provided = 1,
-)
-
 cc_library(
     name = "python_headers",
     hdrs = [":python_include"],
-    deps = select({
-        ":windows": [":python_lib"],
+    data = select({
+        ":windows": [":python_import_lib"],
         "//conditions:default": [],
     }),
     includes = ["python_include"],
+    linkopts = select({
+        # TODO(pcloudy): Ideally, this should just go into deps after resolving
+        # https://github.com/bazelbuild/bazel/issues/3237,
+        ":windows": ["$(locations :python_import_lib)"],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
-- 
GitLab


From 829f2aff4e663c13b71253707c8a867ca929bb1e Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Mar 2018 15:47:09 -0700
Subject: [PATCH 086/960] Rename CreateXyzHlo utilities to MakeXyzHlo as
 discussed on cr/188968478; NFC

The rationale here is that MakeXyzHlo is less likely to be confused with
HloInstruction::CreateXyz and we already have a convention of using a "Make"
prefix for ergonomic factory functions.
PiperOrigin-RevId: 189259036
---
 .../xla/service/algebraic_simplifier.cc       | 33 +++++-----
 .../compiler/xla/service/gather_expander.cc   | 28 ++++----
 .../compiler/xla/service/gpu/pad_insertion.cc | 10 +--
 .../xla/service/hlo_creation_utils.cc         | 64 +++++++++----------
 .../compiler/xla/service/hlo_creation_utils.h | 34 +++++-----
 tensorflow/compiler/xla/service/while_util.cc | 12 ++--
 6 files changed, 89 insertions(+), 92 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index be7aa307d2..971c2935c8 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -385,7 +385,7 @@ Status AlgebraicSimplifierVisitor::HandleAdd(HloInstruction* add) {
     auto* c2 = rhs;
 
     TF_ASSIGN_OR_RETURN(auto* sum_of_constants,
-                        CreateBinaryHlo(HloOpcode::kAdd, c1, c2));
+                        MakeBinaryHlo(HloOpcode::kAdd, c1, c2));
     return ReplaceWithNewInstruction(
         add, HloInstruction::CreateBinary(add->shape(), HloOpcode::kAdd,
                                           lhs->mutable_operand(0),
@@ -636,16 +636,14 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
   // (A / B) / (C / D)  =>  (A / B)*(D / C) => (A * D) / (B * C)
   if (lhs->opcode() == HloOpcode::kDivide &&
       rhs->opcode() == HloOpcode::kDivide) {
-    TF_ASSIGN_OR_RETURN(
-        auto a_times_d,
-        CreateBinaryHlo(HloOpcode::kMultiply, lhs->mutable_operand(0),
-                        rhs->mutable_operand(1)));
-    TF_ASSIGN_OR_RETURN(
-        auto b_times_c,
-        CreateBinaryHlo(HloOpcode::kMultiply, lhs->mutable_operand(1),
-                        rhs->mutable_operand(0)));
-    TF_ASSIGN_OR_RETURN(auto new_divide, CreateBinaryHlo(HloOpcode::kDivide,
-                                                         a_times_d, b_times_c));
+    TF_ASSIGN_OR_RETURN(auto a_times_d, MakeBinaryHlo(HloOpcode::kMultiply,
+                                                      lhs->mutable_operand(0),
+                                                      rhs->mutable_operand(1)));
+    TF_ASSIGN_OR_RETURN(auto b_times_c, MakeBinaryHlo(HloOpcode::kMultiply,
+                                                      lhs->mutable_operand(1),
+                                                      rhs->mutable_operand(0)));
+    TF_ASSIGN_OR_RETURN(auto new_divide, MakeBinaryHlo(HloOpcode::kDivide,
+                                                       a_times_d, b_times_c));
 
     return ReplaceInstruction(divide, new_divide);
   }
@@ -654,7 +652,7 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
   if (lhs->opcode() == HloOpcode::kDivide) {
     TF_ASSIGN_OR_RETURN(
         auto b_times_c,
-        CreateBinaryHlo(HloOpcode::kMultiply, lhs->mutable_operand(1), rhs));
+        MakeBinaryHlo(HloOpcode::kMultiply, lhs->mutable_operand(1), rhs));
     return ReplaceWithNewInstruction(
         divide,
         HloInstruction::CreateBinary(divide->shape(), HloOpcode::kDivide,
@@ -663,9 +661,8 @@ Status AlgebraicSimplifierVisitor::HandleDivide(HloInstruction* divide) {
 
   // A / (B / C) => (A*C) / B
   if (rhs->opcode() == HloOpcode::kDivide) {
-    TF_ASSIGN_OR_RETURN(
-        auto a_times_c,
-        CreateBinaryHlo(HloOpcode::kMultiply, lhs, rhs->mutable_operand(1)));
+    TF_ASSIGN_OR_RETURN(auto a_times_c, MakeBinaryHlo(HloOpcode::kMultiply, lhs,
+                                                      rhs->mutable_operand(1)));
     return ReplaceWithNewInstruction(
         divide,
         HloInstruction::CreateBinary(divide->shape(), HloOpcode::kDivide,
@@ -1300,8 +1297,8 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) {
     }
 
     TF_ASSIGN_OR_RETURN(HloInstruction * nonzero_pad,
-                        CreatePadHlo(pad->mutable_operand(0),
-                                     pad->mutable_operand(1), nonzero_padding));
+                        MakePadHlo(pad->mutable_operand(0),
+                                   pad->mutable_operand(1), nonzero_padding));
     // Copy the layout from the original pad instructions. The new pad and the
     // slice instruction should all have the same layout.
     TF_RETURN_IF_ERROR(LayoutUtil::CopyLayoutBetweenShapes(
@@ -1329,7 +1326,7 @@ Status AlgebraicSimplifierVisitor::HandlePad(HloInstruction* pad) {
 
     TF_ASSIGN_OR_RETURN(
         HloInstruction * slice,
-        CreateSliceHlo(nonzero_pad, start_indices, end_indices, strides));
+        MakeSliceHlo(nonzero_pad, start_indices, end_indices, strides));
 
     // Verify that the slice shape matches the pad shape.
     TF_RET_CHECK(ShapeUtil::Compatible(slice->shape(), pad->shape()));
diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index a133d81067..58c62d8ce9 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -39,7 +39,7 @@ static StatusOr<HloInstruction*> TransposeIndexVectorDimToLast(
     }
   }
   permutation.push_back(index_vector_dim);
-  return CreateTransposeHlo(gather_indices, permutation);
+  return MakeTransposeHlo(gather_indices, permutation);
 }
 
 // If the gather_indices holds scalar indices (i.e. gather_indices has rank N
@@ -133,16 +133,16 @@ static StatusOr<HloInstruction*> ExpandIndexVectorIntoOperandSpace(
         dim_numbers.gather_dims_to_operand_dims_size()) {
       TF_ASSIGN_OR_RETURN(
           HloInstruction * component_to_concat,
-          CreateSliceHlo(
-              index_vector, /*start_indices=*/{index_vector_dim_index},
-              /*limit_indices=*/{index_vector_dim_index + 1}, /*strides=*/{1}));
+          MakeSliceHlo(index_vector, /*start_indices=*/{index_vector_dim_index},
+                       /*limit_indices=*/{index_vector_dim_index + 1},
+                       /*strides=*/{1}));
       expanded_index_components.push_back(component_to_concat);
     } else {
       expanded_index_components.push_back(zero);
     }
   }
 
-  return CreateConcatHlo(expanded_index_components, /*dimension=*/0);
+  return MakeConcatHlo(expanded_index_components, /*dimension=*/0);
 }
 
 // This generates the body of the while that implements the main data movement
@@ -159,8 +159,8 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * induction_var_as_vector,
-      CreateBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
-                         /*result_shape_bounds=*/{1}));
+      MakeBroadcastHlo(induction_var, /*broadcast_dimensions=*/{},
+                       /*result_shape_bounds=*/{1}));
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * index_into_gather_indices,
@@ -169,8 +169,8 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * index_vector_2d,
-      CreateDynamicSliceHlo(gather_indices, index_into_gather_indices,
-                            {1, index_vector_size}));
+      MakeDynamicSliceHlo(gather_indices, index_into_gather_indices,
+                          {1, index_vector_size}));
 
   TF_ASSIGN_OR_RETURN(HloInstruction * index_vector,
                       ElideDegenerateDims(index_vector_2d, {0}));
@@ -181,8 +181,8 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
                           operand->shape().dimensions_size()));
 
   TF_ASSIGN_OR_RETURN(HloInstruction * gathered_slice,
-                      CreateDynamicSliceHlo(operand, gathered_slice_start,
-                                            gather.gather_window_bounds()));
+                      MakeDynamicSliceHlo(operand, gathered_slice_start,
+                                          gather.gather_window_bounds()));
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * gathered_slice_for_update,
@@ -197,8 +197,8 @@ static StatusOr<std::vector<HloInstruction*>> GatherLoopBody(
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * updated_accumulator,
-      CreateDynamicUpdateSliceHlo(output_accumulator, gathered_slice_for_update,
-                                  index_vector_into_accumulator));
+      MakeDynamicUpdateSliceHlo(output_accumulator, gathered_slice_for_update,
+                                index_vector_into_accumulator));
 
   // New loop state -- only the accumulator has changed.  The
   // WhileUtil::MakeCountedLoop functions takes care of the induction variable
@@ -250,7 +250,7 @@ static StatusOr<HloInstruction*> PermuteGatherAndWindowDims(
     }
   }
 
-  return CreateTransposeHlo(accumulator, permutation);
+  return MakeTransposeHlo(accumulator, permutation);
 }
 
 // High Level Algorithm
diff --git a/tensorflow/compiler/xla/service/gpu/pad_insertion.cc b/tensorflow/compiler/xla/service/gpu/pad_insertion.cc
index fa405b9329..7bda4e2fcd 100644
--- a/tensorflow/compiler/xla/service/gpu/pad_insertion.cc
+++ b/tensorflow/compiler/xla/service/gpu/pad_insertion.cc
@@ -69,7 +69,7 @@ HloInstruction* MaybePaddedAndSlicedInput(
     HloInstruction* padding =
         computation->AddInstruction(HloInstruction::CreateConstant(
             MakeUnique<Literal>(Literal::Zero(element_type))));
-    input = CreatePadHlo(input, padding, padding_config).ValueOrDie();
+    input = MakePadHlo(input, padding, padding_config).ValueOrDie();
   }
 
   if (window_util::HasNegativePadding(conv_window)) {
@@ -92,8 +92,8 @@ HloInstruction* MaybePaddedAndSlicedInput(
           std::max<int64>(0LL, -conv_window.dimensions(i).padding_high());
     }
 
-    input = CreateSliceHlo(input, start_indices, limit_indices, strides)
-                .ValueOrDie();
+    input =
+        MakeSliceHlo(input, start_indices, limit_indices, strides).ValueOrDie();
   }
 
   return input;
@@ -126,7 +126,7 @@ HloInstruction* MaybePaddedKernel(const Window& conv_window,
   HloInstruction* padding =
       computation->AddInstruction(HloInstruction::CreateConstant(
           MakeUnique<Literal>(Literal::Zero(element_type))));
-  return CreatePadHlo(kernel, padding, padding_config).ValueOrDie();
+  return MakePadHlo(kernel, padding, padding_config).ValueOrDie();
 }
 }  // namespace
 
@@ -238,7 +238,7 @@ bool PadInsertion::CanonicalizeBackwardFilterConvolution(
       computation->AddInstruction(HloInstruction::CreateConstant(
           MakeUnique<Literal>(Literal::Zero(input->shape().element_type()))));
   HloInstruction* padded_input =
-      CreatePadHlo(input, padding, input_padding_config).ValueOrDie();
+      MakePadHlo(input, padding, input_padding_config).ValueOrDie();
 
   // The shape of the backward_conv CustomCall is a tuple (conv_result,
   // scratch_buffer).  Extract out the shape of conv_result.
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index 4585bffa42..fbe71f8d5b 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -23,8 +23,8 @@ namespace xla {
 using tensorflow::gtl::ArraySlice;
 using tensorflow::strings::StrCat;
 
-StatusOr<HloInstruction*> CreateBinaryHlo(HloOpcode opcode, HloInstruction* lhs,
-                                          HloInstruction* rhs) {
+StatusOr<HloInstruction*> MakeBinaryHlo(HloOpcode opcode, HloInstruction* lhs,
+                                        HloInstruction* rhs) {
   HloComputation* computation = lhs->parent();
   CHECK_EQ(computation, rhs->parent());
   TF_ASSIGN_OR_RETURN(Shape binary_op_shape,
@@ -33,9 +33,9 @@ StatusOr<HloInstruction*> CreateBinaryHlo(HloOpcode opcode, HloInstruction* lhs,
       HloInstruction::CreateBinary(binary_op_shape, opcode, lhs, rhs));
 }
 
-StatusOr<HloInstruction*> CreatePadHlo(HloInstruction* operand,
-                                       HloInstruction* padding_value,
-                                       const PaddingConfig& padding_config) {
+StatusOr<HloInstruction*> MakePadHlo(HloInstruction* operand,
+                                     HloInstruction* padding_value,
+                                     const PaddingConfig& padding_config) {
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, padding_value->parent());
   TF_ASSIGN_OR_RETURN(
@@ -46,10 +46,10 @@ StatusOr<HloInstruction*> CreatePadHlo(HloInstruction* operand,
       pad_shape, operand, padding_value, padding_config));
 }
 
-StatusOr<HloInstruction*> CreateSliceHlo(HloInstruction* operand,
-                                         ArraySlice<int64> start_indices,
-                                         ArraySlice<int64> limit_indices,
-                                         ArraySlice<int64> strides) {
+StatusOr<HloInstruction*> MakeSliceHlo(HloInstruction* operand,
+                                       ArraySlice<int64> start_indices,
+                                       ArraySlice<int64> limit_indices,
+                                       ArraySlice<int64> strides) {
   HloComputation* computation = operand->parent();
   TF_ASSIGN_OR_RETURN(Shape slice_shape, ShapeInference::InferSliceShape(
                                              operand->shape(), start_indices,
@@ -58,7 +58,7 @@ StatusOr<HloInstruction*> CreateSliceHlo(HloInstruction* operand,
       slice_shape, operand, start_indices, limit_indices, strides));
 }
 
-StatusOr<HloInstruction*> CreateConvolveHlo(
+StatusOr<HloInstruction*> MakeConvolveHlo(
     HloInstruction* lhs, HloInstruction* rhs, const Window& window,
     const ConvolutionDimensionNumbers& dimension_numbers) {
   HloComputation* computation = lhs->parent();
@@ -70,8 +70,8 @@ StatusOr<HloInstruction*> CreateConvolveHlo(
       convolve_shape, lhs, rhs, window, dimension_numbers));
 }
 
-StatusOr<HloInstruction*> CreateTransposeHlo(HloInstruction* operand,
-                                             ArraySlice<int64> dimensions) {
+StatusOr<HloInstruction*> MakeTransposeHlo(HloInstruction* operand,
+                                           ArraySlice<int64> dimensions) {
   HloComputation* computation = operand->parent();
   TF_ASSIGN_OR_RETURN(
       Shape transpose_shape,
@@ -80,23 +80,23 @@ StatusOr<HloInstruction*> CreateTransposeHlo(HloInstruction* operand,
       HloInstruction::CreateTranspose(transpose_shape, operand, dimensions));
 }
 
-StatusOr<HloInstruction*> CreateReshapeHlo(const Shape& result_shape,
-                                           HloInstruction* operand) {
+StatusOr<HloInstruction*> MakeReshapeHlo(const Shape& result_shape,
+                                         HloInstruction* operand) {
   HloComputation* computation = operand->parent();
   return computation->AddInstruction(
       HloInstruction::CreateReshape(result_shape, operand));
 }
 
-StatusOr<HloInstruction*> CreateReshapeHlo(
+StatusOr<HloInstruction*> MakeReshapeHlo(
     ArraySlice<int64> result_shape_dim_bounds, HloInstruction* operand) {
   Shape new_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
                                          result_shape_dim_bounds);
-  return CreateReshapeHlo(new_shape, operand);
+  return MakeReshapeHlo(new_shape, operand);
 }
 
-StatusOr<HloInstruction*> CreateDynamicSliceHlo(HloInstruction* operand,
-                                                HloInstruction* start_indices,
-                                                ArraySlice<int64> slice_sizes) {
+StatusOr<HloInstruction*> MakeDynamicSliceHlo(HloInstruction* operand,
+                                              HloInstruction* start_indices,
+                                              ArraySlice<int64> slice_sizes) {
   HloComputation* computation = operand->parent();
   CHECK_EQ(computation, start_indices->parent());
   TF_ASSIGN_OR_RETURN(
@@ -107,7 +107,7 @@ StatusOr<HloInstruction*> CreateDynamicSliceHlo(HloInstruction* operand,
       dynamic_slice_shape, operand, start_indices, slice_sizes));
 }
 
-StatusOr<HloInstruction*> CreateDynamicUpdateSliceHlo(
+StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
     HloInstruction* operand, HloInstruction* update,
     HloInstruction* start_indices) {
   HloComputation* computation = operand->parent();
@@ -121,7 +121,7 @@ StatusOr<HloInstruction*> CreateDynamicUpdateSliceHlo(
       dynamic_update_slice_shape, operand, update, start_indices));
 }
 
-StatusOr<HloInstruction*> CreateBroadcastHlo(
+StatusOr<HloInstruction*> MakeBroadcastHlo(
     HloInstruction* operand, ArraySlice<int64> broadcast_dimensions,
     ArraySlice<int64> result_shape_bounds) {
   HloComputation* computation = operand->parent();
@@ -132,8 +132,8 @@ StatusOr<HloInstruction*> CreateBroadcastHlo(
       broadcast_shape, operand, broadcast_dimensions));
 }
 
-StatusOr<HloInstruction*> CreateGetTupleElementHlo(HloInstruction* operand,
-                                                   int64 index) {
+StatusOr<HloInstruction*> MakeGetTupleElementHlo(HloInstruction* operand,
+                                                 int64 index) {
   HloComputation* computation = operand->parent();
 
   TF_ASSIGN_OR_RETURN(
@@ -143,8 +143,8 @@ StatusOr<HloInstruction*> CreateGetTupleElementHlo(HloInstruction* operand,
       HloInstruction::CreateGetTupleElement(gte_shape, operand, index));
 }
 
-StatusOr<HloInstruction*> CreateConcatHlo(ArraySlice<HloInstruction*> operands,
-                                          int64 dimension) {
+StatusOr<HloInstruction*> MakeConcatHlo(ArraySlice<HloInstruction*> operands,
+                                        int64 dimension) {
   CHECK_GT(operands.size(), 0);
 
   HloComputation* computation = operands[0]->parent();
@@ -181,7 +181,7 @@ StatusOr<HloInstruction*> CollapseFirstNDims(HloInstruction* operand, int64 n) {
   Shape output_shape =
       ShapeUtil::MakeShape(operand_shape.element_type(), new_shape_dims);
 
-  return CreateReshapeHlo(output_shape, operand);
+  return MakeReshapeHlo(output_shape, operand);
 }
 
 StatusOr<HloInstruction*> ExpandFirstDimIntoNDims(
@@ -198,7 +198,7 @@ StatusOr<HloInstruction*> ExpandFirstDimIntoNDims(
             std::back_inserter(expanded_shape_dim_bounds));
   Shape new_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
                                          expanded_shape_dim_bounds);
-  return CreateReshapeHlo(new_shape, operand);
+  return MakeReshapeHlo(new_shape, operand);
 }
 
 StatusOr<HloInstruction*> ExpandLastDimIntoNDims(
@@ -216,7 +216,7 @@ StatusOr<HloInstruction*> ExpandLastDimIntoNDims(
   c_copy(expanded_dims, std::back_inserter(expanded_shape_dim_bounds));
   Shape new_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
                                          expanded_shape_dim_bounds);
-  return CreateReshapeHlo(new_shape, operand);
+  return MakeReshapeHlo(new_shape, operand);
 }
 
 StatusOr<HloInstruction*> ElideDegenerateDims(HloInstruction* operand,
@@ -241,7 +241,7 @@ StatusOr<HloInstruction*> ElideDegenerateDims(HloInstruction* operand,
   c_reverse(new_shape_dim_bounds);
   Shape output_shape =
       ShapeUtil::MakeShape(input_shape.element_type(), new_shape_dim_bounds);
-  return CreateReshapeHlo(output_shape, operand);
+  return MakeReshapeHlo(output_shape, operand);
 }
 
 StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
@@ -258,7 +258,7 @@ StatusOr<HloInstruction*> PadVectorWithZeros(HloInstruction* operand,
   HloInstruction* zero =
       computation->AddInstruction(HloInstruction::CreateConstant(
           MakeUnique<Literal>(Literal::Zero(operand->shape().element_type()))));
-  return CreatePadHlo(operand, zero, padding_config);
+  return MakePadHlo(operand, zero, padding_config);
 }
 
 StatusOr<HloInstruction*> BroadcastZeros(
@@ -267,8 +267,8 @@ StatusOr<HloInstruction*> BroadcastZeros(
   HloInstruction* zero =
       computation->AddInstruction(HloInstruction::CreateConstant(
           MakeUnique<Literal>(Literal::Zero(element_type))));
-  return CreateBroadcastHlo(zero, /*broadcast_dimensions=*/{},
-                            /*result_shape_bounds=*/broadcast_dimensions);
+  return MakeBroadcastHlo(zero, /*broadcast_dimensions=*/{},
+                          /*result_shape_bounds=*/broadcast_dimensions);
 }
 
 StatusOr<std::unique_ptr<HloComputation>> CreateComputationWithSignature(
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 2b03a849cf..6032ebab74 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -28,73 +28,73 @@ namespace xla {
 
 // Creates a binary HLO instruction and adds it to the computation containing
 // `lhs` and `rhs` (`lhs` and `rhs` must be in the same computation).
-StatusOr<HloInstruction*> CreateBinaryHlo(HloOpcode opcode, HloInstruction* lhs,
-                                          HloInstruction* rhs);
+StatusOr<HloInstruction*> MakeBinaryHlo(HloOpcode opcode, HloInstruction* lhs,
+                                        HloInstruction* rhs);
 
 // Creates a pad HLO instruction and adds it to the computation containing
 // `operand` and `padding_value` (`operand` and `padding_value` must be in the
 // same computation).
-StatusOr<HloInstruction*> CreatePadHlo(HloInstruction* operand,
-                                       HloInstruction* padding_value,
-                                       const PaddingConfig& padding_config);
+StatusOr<HloInstruction*> MakePadHlo(HloInstruction* operand,
+                                     HloInstruction* padding_value,
+                                     const PaddingConfig& padding_config);
 
 // Creates a slice HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> CreateSliceHlo(
+StatusOr<HloInstruction*> MakeSliceHlo(
     HloInstruction* operand, tensorflow::gtl::ArraySlice<int64> start_indices,
     tensorflow::gtl::ArraySlice<int64> limit_indices,
     tensorflow::gtl::ArraySlice<int64> strides);
 
 // Creates a convolution HLO instruction and adds it to the computation
 // containing `lhs` and `rhs` (`lhs` and `rhs` must be in the same computation).
-StatusOr<HloInstruction*> CreateConvolveHlo(
+StatusOr<HloInstruction*> MakeConvolveHlo(
     HloInstruction* lhs, HloInstruction* rhs, const Window& window,
     const ConvolutionDimensionNumbers& dimension_numbers);
 
 // Creates a transpose HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> CreateTransposeHlo(
+StatusOr<HloInstruction*> MakeTransposeHlo(
     HloInstruction* operand, tensorflow::gtl::ArraySlice<int64> dimensions);
 
 // Creates a reshape HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> CreateReshapeHlo(const Shape& result_shape,
-                                           HloInstruction* operand);
+StatusOr<HloInstruction*> MakeReshapeHlo(const Shape& result_shape,
+                                         HloInstruction* operand);
 
-StatusOr<HloInstruction*> CreateReshapeHlo(
+StatusOr<HloInstruction*> MakeReshapeHlo(
     tensorflow::gtl::ArraySlice<int64> result_shape_dim_bounds,
     HloInstruction* operand);
 
 // Creates a dynamic-slice HLO instruction and adds it to the computation
 // containing `operand` and `start_indices` (`operand` and `start_indices` must
 // be in the same computation).
-StatusOr<HloInstruction*> CreateDynamicSliceHlo(
+StatusOr<HloInstruction*> MakeDynamicSliceHlo(
     HloInstruction* operand, HloInstruction* start_indices,
     tensorflow::gtl::ArraySlice<int64> slice_sizes);
 
 // Creates a dynamic-update-slice HLO instruction and adds it to the computation
 // containing `operand`, `update` and `start_indices` (`operand`, `update` and
 // `start_indices` must be in the same computation).
-StatusOr<HloInstruction*> CreateDynamicUpdateSliceHlo(
+StatusOr<HloInstruction*> MakeDynamicUpdateSliceHlo(
     HloInstruction* operand, HloInstruction* update,
     HloInstruction* start_indices);
 
 // Creates a broadcast HLO instruction and adds it to the computation containing
 // `operand`.
-StatusOr<HloInstruction*> CreateBroadcastHlo(
+StatusOr<HloInstruction*> MakeBroadcastHlo(
     HloInstruction* operand,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions,
     tensorflow::gtl::ArraySlice<int64> result_shape_bounds);
 
 // Creates a GetTupleElement HLO instruction and adds it to the computation
 // containing `operand`.
-StatusOr<HloInstruction*> CreateGetTupleElementHlo(HloInstruction* operand,
-                                                   int64 index);
+StatusOr<HloInstruction*> MakeGetTupleElementHlo(HloInstruction* operand,
+                                                 int64 index);
 
 // Creates a Concatenate HLO instruction and adds it to the computation
 // containing `operands` (`operands` must be non-empty and every element must be
 // contained in the same computation).
-StatusOr<HloInstruction*> CreateConcatHlo(
+StatusOr<HloInstruction*> MakeConcatHlo(
     tensorflow::gtl::ArraySlice<HloInstruction*> operands, int64 dimension);
 
 // -----------------------------------------------------------------------------
diff --git a/tensorflow/compiler/xla/service/while_util.cc b/tensorflow/compiler/xla/service/while_util.cc
index 7441a7ad39..8cd5882f32 100644
--- a/tensorflow/compiler/xla/service/while_util.cc
+++ b/tensorflow/compiler/xla/service/while_util.cc
@@ -155,10 +155,10 @@ MakeCountedLoopConditionComputation(const Shape& loop_state_shape,
 
   HloInstruction* param = cond_computation->parameter_instruction(0);
   TF_ASSIGN_OR_RETURN(HloInstruction * counter,
-                      CreateGetTupleElementHlo(param, 0));
+                      MakeGetTupleElementHlo(param, 0));
   TF_ASSIGN_OR_RETURN(
       HloInstruction * compare,
-      CreateBinaryHlo(HloOpcode::kLt, counter, trip_count_constant));
+      MakeBinaryHlo(HloOpcode::kLt, counter, trip_count_constant));
   cond_computation->set_root_instruction(compare);
   return std::move(cond_computation);
 }
@@ -175,14 +175,14 @@ static StatusOr<std::unique_ptr<HloComputation>> MakeCountedLoopBodyComputation(
 
   HloInstruction* param = body_computation->parameter_instruction(0);
   TF_ASSIGN_OR_RETURN(HloInstruction * indvar,
-                      CreateGetTupleElementHlo(param, 0));
+                      MakeGetTupleElementHlo(param, 0));
   TF_ASSIGN_OR_RETURN(HloInstruction * next_indvar,
-                      CreateBinaryHlo(HloOpcode::kAdd, indvar, one));
+                      MakeBinaryHlo(HloOpcode::kAdd, indvar, one));
 
   std::vector<HloInstruction*> loop_body_generator_args;
   for (int64 i = 1, e = loop_state_shape.tuple_shapes_size(); i < e; i++) {
     TF_ASSIGN_OR_RETURN(HloInstruction * tuple_element,
-                        CreateGetTupleElementHlo(param, i));
+                        MakeGetTupleElementHlo(param, i));
     loop_body_generator_args.push_back(tuple_element);
   }
   TF_ASSIGN_OR_RETURN(std::vector<HloInstruction*> next_state,
@@ -238,7 +238,7 @@ static Shape MakeLoopStateShape(const WhileUtil::LoopStateTy& init_values) {
   std::vector<HloInstruction*> result;
   for (int64 i = 0, e = init_values.size(); i < e; i++) {
     TF_ASSIGN_OR_RETURN(HloInstruction * user_state,
-                        CreateGetTupleElementHlo(while_instr, i + 1));
+                        MakeGetTupleElementHlo(while_instr, i + 1));
     result.push_back(user_state);
   }
   return result;
-- 
GitLab


From 72a0b0efbc74a03fb60a547a474c0fdb2115d898 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 15 Mar 2018 15:58:44 -0700
Subject: [PATCH 087/960] Clarifying when is it possible to use a tape while it
 is still active.

PiperOrigin-RevId: 189260773
---
 tensorflow/c/eager/tape.h                 |  2 ++
 tensorflow/python/eager/backprop_test.py  | 11 +++++++++++
 tensorflow/python/eager/pywrap_tfe_src.cc | 11 ++++++++++-
 3 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h
index bdb0815d6b..c7bd3bdafd 100644
--- a/tensorflow/c/eager/tape.h
+++ b/tensorflow/c/eager/tape.h
@@ -152,6 +152,8 @@ class GradientTape {
                          gtl::ArraySlice<Gradient*> output_gradients,
                          std::vector<Gradient*>* result);
 
+  bool IsPersistent() const { return persistent_; }
+
  private:
   TensorTape tensor_tape_;
   OpTape<BackwardFunction> op_tape_;
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 07a2155d24..5934293dfc 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -195,6 +195,17 @@ class BackpropTest(test.TestCase):
     g, = backprop.gradients_function(loss, [0])(logits, labels)
     self.assertAllEqual(g.numpy(), [[-0.5, 0.5]])
 
+  def testGradientWithinTapeBlock(self):
+    v1 = resource_variable_ops.ResourceVariable(1.)
+    with backprop.GradientTape() as t:
+      loss = 2 * v1
+      with self.assertRaises(RuntimeError):
+        t.gradient(loss, [v1])
+    with backprop.GradientTape(persistent=True) as t:
+      loss = 2 * v1
+      grad = t.gradient(loss, [v1])
+    self.assertAllEqual(grad[0], 2.0)
+
   @test_util.assert_no_new_tensors
   def testSecondGrad(self):
 
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index fe9785dc66..701f68b8f7 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1323,6 +1323,16 @@ std::vector<PyObject*> MakeTensorList(PyObject* tensors) {
 PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* vspace,
                               PyObject* target, PyObject* sources,
                               PyObject* output_gradients, TF_Status* status) {
+  TFE_Py_Tape* tape_obj = reinterpret_cast<TFE_Py_Tape*>(tape);
+  if (!tape_obj->tape->IsPersistent()) {
+    auto* tape_set = GetTapeSet();
+    if (tape_set->find(tape_obj) != tape_set->end()) {
+      PyErr_SetString(PyExc_RuntimeError,
+                      "Trying to call tape.gradient on a non-persistent tape "
+                      "while it is still active.");
+      return nullptr;
+    }
+  }
   PyVSpace c_vspace(vspace);
   if (!c_vspace.Initialize().ok()) {
     return nullptr;
@@ -1348,7 +1358,6 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* vspace,
       Py_INCREF(tensor);
     }
   }
-  TFE_Py_Tape* tape_obj = reinterpret_cast<TFE_Py_Tape*>(tape);
   std::vector<PyObject*> result;
   status->status = tape_obj->tape->ComputeGradient(
       c_vspace, target_vec, sources_vec, outgrad_vec, &result);
-- 
GitLab


From cc13b530198d2454c1c422a42aad4edf07257e9d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 16:29:14 -0700
Subject: [PATCH 088/960] Add initial host transfer ops for XLA support for
 computation on the host during a compiled computation.

PiperOrigin-RevId: 189265297
---
 .../jit/encapsulate_subgraphs_pass.cc         |  49 +++---
 .../jit/encapsulate_subgraphs_pass_test.cc    | 142 ++++++++++--------
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  12 +-
 tensorflow/compiler/tf2xla/xla_compiler.h     |   8 +-
 4 files changed, 124 insertions(+), 87 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 2d175c40f9..0685036c9d 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -53,6 +53,8 @@ namespace tensorflow {
 const char* const kXlaCompiledKernelAttr = "_XlaCompiledKernel";
 const char* const kXlaNumConstantArgsAttr = "_XlaNumConstantArgs";
 const char* const kXlaNumResourceArgsAttr = "_XlaNumResourceArgs";
+const char* const kXlaHostTransferSequencerAttr =
+    "_xla_host_transfer_sequencer";
 
 namespace {
 
@@ -143,7 +145,7 @@ struct NodeSlot {
 // everything to use it.
 static const char* const kArgOp = "_Arg";
 static const char* const kRetValOp = "_Retval";
-static const char* const kHostComputeOp = "_XlaHostCompute";
+static const char* const kHostComputeOp = "XlaHostCompute";
 static const char* const kSendFromHostOp = "_XlaSendFromHost";
 static const char* const kRecvAtHostOp = "_XlaRecvAtHost";
 
@@ -328,8 +330,8 @@ class Encapsulator {
     Status MakeSequencingNode(const string& subgraph_name, Graph* graph_out);
 
     // If there is a sequencer node, adds a control edge from the sequencer to
-    // all the downstream nodes of call_node_outputs.
-    void ConnectSequencerToOutputs(Graph* graph_out);
+    // the call node.
+    void ConnectSequencerToCallNode(Graph* graph_out);
 
     Status AddShapeInferenceInfo(
         const string& outside_compilation_subgraph_name,
@@ -425,6 +427,10 @@ class Encapsulator {
     // NodeDef for the function call node.
     NodeDef call_node_def_;
 
+    // Name that is used for the call node. This may not be
+    // call_node_def_.name() if the client supplies a rewrite lambda.
+    string function_def_name_;
+
     // Placeholder node simulating the host compute key in the output graph.
     // Not owned.
     Node* host_compute_key_placeholder_ = nullptr;
@@ -863,25 +869,21 @@ Status Encapsulator::Subgraph::MakeSequencingNode(const string& subgraph_name,
     NodeDef seq_def;
     NodeDefBuilder builder(strings::StrCat(subgraph_name, "_sequencer"),
                            "NoOp");
+    builder.Attr(kXlaHostTransferSequencerAttr, subgraph_name);
+    builder.Device(device_);
     Status s = builder.Finalize(&seq_def);
     if (!s.ok()) return s;
 
     sequencer_ = graph_out->AddNode(seq_def, &s);
     if (!s.ok()) return s;
-    sequencer_->set_assigned_device_name(device_);
   }
   return Status::OK();
 }
 
-void Encapsulator::Subgraph::ConnectSequencerToOutputs(Graph* graph_out) {
+void Encapsulator::Subgraph::ConnectSequencerToCallNode(Graph* graph_out) {
   if (sequencer_ != nullptr) {
-    std::unordered_set<Node*> output_dependencies;
-    for (Node* node : call_node_outputs_->out_nodes()) {
-      output_dependencies.insert(node);
-    }
-    for (Node* node : output_dependencies) {
-      graph_out->AddControlEdge(sequencer_, node);
-    }
+    VLOG(2) << "ConnectSequencerToCallNode";
+    graph_out->AddControlEdge(sequencer_, call_node_inputs_);
   }
 }
 
@@ -927,6 +929,8 @@ Status Encapsulator::Subgraph::BuildFunctionDef(
     name = call_node_def_.op();
   }
 
+  function_def_name_ = name;
+
   FunctionDef fdef;
   TF_RETURN_IF_ERROR(GraphToFunctionDef(*graph_, name, &fdef));
 
@@ -982,7 +986,7 @@ Status Encapsulator::Subgraph::AddShapeInferenceInfo(
 
 Status Encapsulator::Subgraph::ReplaceFunctionDef(
     FunctionLibraryDefinition* library) {
-  const string& name = call_node_def_.name();
+  const string& name = function_def_name_;
 
   FunctionDef fdef;
   TF_RETURN_IF_ERROR(GraphToFunctionDef(*graph_, name, &fdef));
@@ -1126,6 +1130,8 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
   NodeDefBuilder builder(strings::StrCat("outside_compilation_", subgraph_name,
                                          "_", oc_subgraph_name, "_recv"),
                          kRecvAtHostOp);
+  // TODO(misard) When we add replication the device placement will have to be
+  // redone.
   builder.Device(device_);
   builder.Attr("Toutputs", dtypes);
   // TODO(misard) For now we only support TPU device 0.
@@ -1138,7 +1144,6 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
 
   oc_subgraph->recv_at_host = graph_out->AddNode(recv_def, &s);
   if (!s.ok()) return s;
-  oc_subgraph->recv_at_host->set_assigned_device_name(device_);
   graph_out->AddEdge(host_compute_key_placeholder_, 0,
                      oc_subgraph->recv_at_host, 0);
 
@@ -1178,6 +1183,8 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
   NodeDefBuilder builder(strings::StrCat("outside_compilation_", subgraph_name,
                                          "_", oc_subgraph_name, "_send"),
                          kSendFromHostOp);
+  // TODO(misard) When we add replication the device placement will have to be
+  // redone.
   builder.Device(device_);
   builder.Attr("Tinputs", dtypes);
   builder.Attr("key", strings::StrCat("host_compute_channel_", subgraph_name,
@@ -1191,7 +1198,6 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
 
   oc_subgraph->send_from_host = graph_out->AddNode(send_def, &s);
   if (!s.ok()) return s;
-  oc_subgraph->send_from_host->set_assigned_device_name(device_);
   graph_out->AddEdge(host_compute_key_placeholder_, 0,
                      oc_subgraph->send_from_host, inputs.size());
 
@@ -1675,7 +1681,7 @@ Status Encapsulator::AddEdgesToOutputGraph(
 
   for (auto& subgraph_entry : subgraphs_) {
     Subgraph& subgraph = subgraph_entry.second;
-    subgraph.ConnectSequencerToOutputs(graph_out);
+    subgraph.ConnectSequencerToCallNode(graph_out);
   }
 
   return Status::OK();
@@ -1827,8 +1833,12 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
               (*static_shape_out)[in_edge->dst_input()] = proto;
             }
           } else {
+            has_parent_with_unknown_shape = true;
             if (!visited[src_node->id()]) {
-              has_parent_with_unknown_shape = true;
+              if (VLOG_IS_ON(2)) {
+                TensorShapeProto proto;
+                context->ShapeHandleToProto(shape, &proto);
+              }
               stack.push_back({src_node, false});
             }
           }
@@ -1981,6 +1991,11 @@ Status Encapsulator::GetShapeInfoForOutsideCompilationSends(
   TF_RETURN_IF_ERROR(MakeGraphForOutsideCompilationSends(
       *graph_out, &pruned_graph, &shape_refiner, &node_images, library));
 
+  if (VLOG_IS_ON(1)) {
+    dump_graph::DumpGraphToFile("pruned_graph_for_shape_inference",
+                                *pruned_graph, library);
+  }
+
   for (auto& subgraph_entry : subgraphs_) {
     Subgraph& subgraph = subgraph_entry.second;
     // Find all the recv_at_host nodes in this subgraph.
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index d7bea56a72..711b1424c7 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -29,6 +29,9 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
+const char* const kXlaHostTransferSequencerAttr =
+    "_xla_host_transfer_sequencer";
+
 template <class Tkey, class Tvalue>
 bool EqualProtoMap(const ::tensorflow::protobuf::Map<Tkey, Tvalue>& a,
                    const ::tensorflow::protobuf::Map<Tkey, Tvalue>& b,
@@ -248,7 +251,7 @@ bool EqualFunctionDefLibrary(const FunctionDefLibrary& expected,
 
 // These dummy Op registrations are here because the real Op registrations live
 // in contrib and there can't be a dependence from this test to contrib.
-REGISTER_OP("_XlaHostCompute")
+REGISTER_OP("XlaHostCompute")
     .Input("inputs: Tinputs")
     .Output("outputs: Toutputs")
     .Attr("Tinputs: list(type) >= 0")
@@ -321,8 +324,13 @@ REGISTER_OP("AddNLikeTest")
     .SetIsCommutative()
     .SetIsAggregate();
 
-Node* NoOp(const GraphDefBuilder::Options& opts) {
-  return ops::SourceOp("NoOp", opts);
+Node* Sequencer(const GraphDefBuilder::Options& opts,
+                const string& call_node_name) {
+  if (opts.HaveError()) return nullptr;
+  NodeBuilder node_builder(opts.GetNameForOp("NoOp"), "NoOp",
+                           opts.op_registry());
+  return opts.WithAttr(kXlaHostTransferSequencerAttr, call_node_name)
+      .FinalizeBuilder(&node_builder);
 }
 
 Node* Input(const GraphDefBuilder::Options& opts) {
@@ -870,7 +878,7 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
            {},
            {"outside_compilation_O1_host_compute"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"C:o:0", "c:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -888,10 +896,6 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
     Node* a = Input(b2.opts().WithName("A"));
     Node* b = Input(b2.opts().WithName("B"));
 
-    NodeBuilder node_builder("F1", "F1", lib_def.get());
-    node_builder.Input(a).Input(b);
-    Node* call = b2.opts().FinalizeBuilder(&node_builder);
-
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
     Node* recv =
@@ -906,10 +910,16 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
                                   .WithName("outside_compilation_F1_O1_send")
                                   .WithControlInput(e));
 
-    Node* s = NoOp(
-        b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}));
+    Node* s = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}),
+        "F1");
+
+    NodeBuilder node_builder("F1", "F1", lib_def.get());
+    node_builder.Input(a).Input(b);
+    Node* call =
+        b2.opts().WithControlInputs({s}).FinalizeBuilder(&node_builder);
 
-    Binary(a, call, b2.opts().WithName("G").WithControlInputs({s, e}));
+    Binary(a, call, b2.opts().WithName("G").WithControlInputs({e}));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1014,7 +1024,7 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
            {},
            {"outside_compilation_O1_host_compute"}},
           {{"outside_compilation_O2_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"D:o:0", "F:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1023,7 +1033,7 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"F"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"C:o:0", "D:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1041,10 +1051,6 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
     Node* a = Input(b2.opts().WithName("A"));
     Node* b = Input(b2.opts().WithName("B"));
 
-    NodeBuilder node_builder("F1", "F1", lib_def.get());
-    node_builder.Input(a).Input(b);
-    Node* call = b2.opts().FinalizeBuilder(&node_builder);
-
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
     Node* recv1 =
@@ -1070,11 +1076,16 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
         ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2", {h},
         b2.opts().WithName("outside_compilation_F1_O2_send"));
 
-    Node* s = NoOp(b2.opts()
-                       .WithName("F1_sequencer")
-                       .WithControlInputs({recv1, send1, recv2, send2}));
+    Node* s = Sequencer(b2.opts()
+                            .WithName("F1_sequencer")
+                            .WithControlInputs({recv1, send1, recv2, send2}),
+                        "F1");
 
-    Binary(g, call, b2.opts().WithName("J").WithControlInput(s));
+    NodeBuilder node_builder("F1", "F1", lib_def.get());
+    node_builder.Input(a).Input(b);
+    Node* call = b2.opts().WithControlInput(s).FinalizeBuilder(&node_builder);
+
+    Binary(g, call, b2.opts().WithName("J"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1156,7 +1167,7 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
            {},
            {"outside_compilation_O1_host_compute"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"C:o:0", "D:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1176,7 +1187,7 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
            "BinaryTest",
            {"f_0_arg", "outside_compilation_O1_host_compute:outputs:0"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"G:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1207,32 +1218,34 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
                                b2.opts()
                                    .WithName("outside_compilation_F1_O1_send")
                                    .WithControlInput(e));
+    Node* s1 = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
+        "F1");
+
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
-    Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
-    Node* s1 = NoOp(
-        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}));
+    Node* call1 =
+        b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1);
 
     Node* key_constant2 =
         KeyPlaceholder("F2", b2.opts().WithName("F2_key_placeholder"));
     Node* recv2 = RecvAtHost(
         ops::NodeOut(key_constant2, 0), "host_compute_channel_F2_O1",
         {DT_FLOAT}, b2.opts().WithName("outside_compilation_F2_O1_recv"));
-    Node* h = Binary(ops::NodeOut(call1, 1), recv2,
-                     b2.opts().WithName("H").WithControlInput(s1));
+    Node* h = Binary(ops::NodeOut(call1, 1), recv2, b2.opts().WithName("H"));
     Node* send2 = SendFromHost(
         ops::NodeOut(key_constant2, 0), "host_compute_channel_F2_O1", {h},
         b2.opts().WithName("outside_compilation_F2_O1_send"));
 
+    Node* s2 = Sequencer(
+        b2.opts().WithName("F2_sequencer").WithControlInputs({recv2, send2}),
+        "F2");
     NodeBuilder node_builder2("F2", "F2", lib_def.get());
     node_builder2.Input(e).Input(call1);
     Node* call2 = b2.opts()
-                      .WithControlInputs({s1, e, call1})
+                      .WithControlInputs({s2, e, call1})
                       .FinalizeBuilder(&node_builder2);
-    Node* s2 = NoOp(
-        b2.opts().WithName("F2_sequencer").WithControlInputs({recv2, send2}));
-    Binary(call2, ops::NodeOut(call2, 1),
-           b2.opts().WithName("J").WithControlInput(s2));
+    Binary(call2, ops::NodeOut(call2, 1), b2.opts().WithName("J"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1280,7 +1293,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
            "BinaryTest",
            {"D:o:0", "outside_compilation_O1_host_compute:outputs:0"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {},
            {{"Tinputs", gtl::ArraySlice<DataType>({})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1304,12 +1317,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
     Node* send1 = SendFromHost(
         ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {e},
         b2.opts().WithName("outside_compilation_F1_O1_send"));
+    Node* s1 = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInput(send1), "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
-    Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
-    Node* s1 = NoOp(b2.opts().WithName("F1_sequencer").WithControlInput(send1));
+    Node* call1 =
+        b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1);
 
-    Unary(call1, b2.opts().WithName("G").WithControlInput(s1));
+    Unary(call1, b2.opts().WithName("G"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1358,7 +1373,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
            "BinaryTest",
            {"D:o:0", "outside_compilation_O1_host_compute:outputs:0"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {},
            {{"Tinputs", gtl::ArraySlice<DataType>({})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1386,13 +1401,15 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
     Node* send1 = SendFromHost(
         ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {e},
         b2.opts().WithName("outside_compilation_F1_O1_send"));
+    Node* s1 = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
+        "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
-    Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
-    Node* s1 = NoOp(
-        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}));
+    Node* call1 =
+        b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1);
 
-    Unary(call1, b2.opts().WithName("G").WithControlInput(s1));
+    Unary(call1, b2.opts().WithName("G"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1434,7 +1451,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
           {{"D"}, "BinaryTest", {"b_0_arg", "C:o:0"}},
           {{"F"}, "UnaryTest", {"D:o:0"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"D:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({})},
@@ -1457,12 +1474,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
         ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {DT_FLOAT},
         b2.opts().WithName("outside_compilation_F1_O1_recv"));
     Node* e = Unary(recv1, b2.opts().WithName("E"));
+    Node* s1 = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInput(recv1), "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
-    Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
-    Node* s1 = NoOp(b2.opts().WithName("F1_sequencer").WithControlInput(recv1));
+    Node* call1 =
+        b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1);
 
-    Binary(e, call1, b2.opts().WithName("G").WithControlInput(s1));
+    Binary(e, call1, b2.opts().WithName("G"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1509,7 +1528,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
            {},
            {"outside_compilation_O1_host_compute"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"D:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({})},
@@ -1537,13 +1556,15 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
                                b2.opts()
                                    .WithName("outside_compilation_F1_O1_send")
                                    .WithControlInput(e));
+    Node* s1 = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
+        "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
-    Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
-    Node* s1 = NoOp(
-        b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}));
+    Node* call1 =
+        b2.opts().WithControlInput(s1).FinalizeBuilder(&node_builder1);
 
-    Binary(e, call1, b2.opts().WithName("G").WithControlInput(s1));
+    Binary(e, call1, b2.opts().WithName("G"));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
@@ -1668,7 +1689,7 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
            {},
            {"outside_compilation_O1_host_compute"}},
           {{"outside_compilation_O1_host_compute"},
-           "_XlaHostCompute",
+           "XlaHostCompute",
            {"c:o:0"},
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
@@ -1687,11 +1708,6 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
     Node* b = Input(b2.opts().WithName("B"));
     Node* c = Unary(a, b2.opts().WithName("C"));
 
-    NodeBuilder node_builder("F1", "F1", lib_def.get());
-    node_builder.Input(b).Input(c);
-    Node* call =
-        b2.opts().WithControlInputs({c}).FinalizeBuilder(&node_builder);
-
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
     Node* recv = RecvAtHost(
@@ -1706,10 +1722,16 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
                                   .WithName("outside_compilation_F1_O1_send")
                                   .WithControlInput(e));
 
-    Node* s = NoOp(
-        b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}));
+    Node* s = Sequencer(
+        b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}),
+        "F1");
+
+    NodeBuilder node_builder("F1", "F1", lib_def.get());
+    node_builder.Input(b).Input(c);
+    Node* call =
+        b2.opts().WithControlInputs({s, c}).FinalizeBuilder(&node_builder);
 
-    Binary(a, call, b2.opts().WithName("G").WithControlInputs({s, e}));
+    Binary(a, call, b2.opts().WithName("G").WithControlInputs({e}));
     TF_EXPECT_OK(b2.ToGraphDef(&graphdef_expected));
   }
 
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 0dc5118c9c..7cdf4d1b3e 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -718,8 +718,8 @@ Status XlaCompiler::GetChannelHandle(const string& key,
 
 namespace {
 
-void SetTransfer(const string& key, const std::vector<DataType>& types,
-                 const std::vector<TensorShape>& shapes,
+void SetTransfer(const string& key, gtl::ArraySlice<DataType> types,
+                 gtl::ArraySlice<TensorShape> shapes,
                  tf2xla::HostTransferMetadata* transfer) {
   transfer->set_key(key);
   CHECK(types.size() == shapes.size());
@@ -733,8 +733,8 @@ void SetTransfer(const string& key, const std::vector<DataType>& types,
 }  // namespace
 
 Status XlaCompiler::SetDeviceToHostMetadata(
-    const string& key, const std::vector<DataType>& types,
-    const std::vector<TensorShape>& shapes) {
+    const string& key, gtl::ArraySlice<DataType> types,
+    gtl::ArraySlice<TensorShape> shapes) {
   if (host_compute_sends_.find(key) != host_compute_sends_.end()) {
     return errors::InvalidArgument(
         "Duplicate calls to SetDeviceToHostMetadata with key ", key);
@@ -760,8 +760,8 @@ Status XlaCompiler::GetDeviceToHostShapes(
 }
 
 Status XlaCompiler::SetHostToDeviceMetadata(
-    const string& key, const std::vector<DataType>& types,
-    const std::vector<TensorShape>& shapes) {
+    const string& key, gtl::ArraySlice<DataType> types,
+    gtl::ArraySlice<TensorShape> shapes) {
   if (host_compute_recvs_.find(key) != host_compute_sends_.end()) {
     return errors::InvalidArgument(
         "Duplicate calls to SetHostToDeviceMetadata with key ", key);
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index a70d2637e0..5f1c631976 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -304,8 +304,8 @@ class XlaCompiler {
   // Sets the shapes and types for the device to host transfer associated with
   // 'key'.
   Status SetDeviceToHostMetadata(const string& key,
-                                 const std::vector<DataType>& types,
-                                 const std::vector<TensorShape>& shapes);
+                                 gtl::ArraySlice<DataType> types,
+                                 gtl::ArraySlice<TensorShape> shapes);
 
   // Gets the shapes the device to host transfer associated with 'key'.
   Status GetDeviceToHostShapes(const string& key,
@@ -314,8 +314,8 @@ class XlaCompiler {
   // Sets the shapes and types for the host to device transfer associated with
   // 'key'.
   Status SetHostToDeviceMetadata(const string& key,
-                                 const std::vector<DataType>& types,
-                                 const std::vector<TensorShape>& shapes);
+                                 gtl::ArraySlice<DataType> types,
+                                 gtl::ArraySlice<TensorShape> shapes);
 
   const Options& options() const { return options_; }
   xla::Client* client() const { return options_.client; }
-- 
GitLab


From f4719f33c0507e0e216f8370898bf56397654d1e Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 15 Mar 2018 16:52:23 -0700
Subject: [PATCH 089/960] Symbolic gradient optimization

PiperOrigin-RevId: 189268327
---
 tensorflow/core/BUILD                         |   6 +-
 tensorflow/core/grappler/optimizers/BUILD     |   3 +
 .../grappler/optimizers/function_optimizer.cc | 128 ++++++++++++++++++
 .../grappler/optimizers/function_optimizer.h  |   6 +-
 .../optimizers/function_optimizer_test.cc     |  55 +++++++-
 .../grappler/optimizers/meta_optimizer.cc     |   6 +-
 6 files changed, 192 insertions(+), 12 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 352e183104..14769c3770 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -2065,10 +2065,13 @@ tf_cuda_library(
 
 CORE_CPU_BASE_HDRS = GRAPH_HDRS + [
     "common_runtime/device.h",
+    "common_runtime/device_mgr.h",
     "common_runtime/eval_const_tensor.h",
     "common_runtime/graph_runner.h",
     "common_runtime/shape_refiner.h",
     "framework/versions.h",
+    "common_runtime/process_function_library_runtime.h",
+    "common_runtime/function.h",
 ]
 
 tf_cuda_library(
@@ -2115,19 +2118,16 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/costmodel_manager.h",
     "common_runtime/debugger_state_interface.h",
     "common_runtime/device_factory.h",
-    "common_runtime/device_mgr.h",
     "common_runtime/device_set.h",
     "common_runtime/dma_helper.h",
     "common_runtime/eigen_thread_pool.h",
     "common_runtime/executor.h",
-    "common_runtime/function.h",
     "common_runtime/graph_optimizer.h",
     "common_runtime/local_device.h",
     "common_runtime/memory_types.h",
     "common_runtime/mkl_cpu_allocator.h",
     "common_runtime/optimization_registry.h",
     "common_runtime/pending_counts.h",
-    "common_runtime/process_function_library_runtime.h",
     "common_runtime/process_util.h",
     "common_runtime/profile_handler.h",
     "common_runtime/renamed_device.h",
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 0df5307d9c..3499879dee 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -141,6 +141,8 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         ":graph_optimizer",
+        "//tensorflow/core:core_cpu_base",
+        "//tensorflow/core:framework",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:op_types",
@@ -156,6 +158,7 @@ tf_cc_test(
         ":function_optimizer",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:cc_ops_internal",
+        "//tensorflow/cc:functional_ops",
         "//tensorflow/core:all_kernels",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index b17715e742..d47887bfc8 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -15,10 +15,16 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 #include <unordered_map>
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/process_function_library_runtime.h"
+#include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/function.pb.h"
+#include "tensorflow/core/framework/graph_def_util.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op_def.pb.h"
 #include "tensorflow/core/framework/versions.pb.h"
+#include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
@@ -128,6 +134,121 @@ Status InlineFunction(const NodeDef& node, const FunctionDef& func,
   return Status::OK();
 }
 
+class FakeCPUDevice : public Device {
+ public:
+  FakeCPUDevice(Env* env, const DeviceAttributes& attr) : Device(env, attr) {}
+  Status Sync() override { return Status::OK(); }
+};
+
+Status InlineSymbolicGradient(const NodeDef& node,
+                              const FunctionDefLibrary& library,
+                              GraphDef* inlined_graph) {
+  Env* env = Env::Default();
+  DeviceAttributes attr;
+  attr.set_name("/device:CPU:0");
+  attr.set_device_type("CPU");
+  FakeCPUDevice* dev = new FakeCPUDevice(env, attr);
+  std::vector<Device*> devices;
+  devices.push_back(dev);
+  DeviceMgr dvc_mgr(devices);
+  FunctionLibraryDefinition function_library(OpRegistry::Global(), library);
+
+  GraphDef graph_def;
+
+  // Create a node to anchor the gradient inputs
+  NodeDef* inlined_input = graph_def.add_node();
+  inlined_input->set_name("FunctionInputs");
+  inlined_input->set_op("IdentityN");
+  AttrValue::ListValue* type_list =
+      (*inlined_input->mutable_attr())["T"].mutable_list();
+  for (const auto& type : node.attr().at("Tin").list().type()) {
+    type_list->add_type(static_cast<DataType>(type));
+  }
+
+  // Add the gradient node
+  NodeDef* inlined = graph_def.add_node();
+  *inlined = node;
+  inlined->clear_input();
+  for (int i = 0; i < node.attr().at("Tin").list().type_size(); ++i) {
+    inlined->add_input(strings::StrCat(inlined_input->name(), ":", i));
+  }
+
+  // Create a node to anchor the gradient outputs
+  NodeDef* inlined_output = graph_def.add_node();
+  inlined_output->set_name("FunctionOutputs");
+  inlined_output->set_op("IdentityN");
+  type_list = (*inlined_output->mutable_attr())["T"].mutable_list();
+  for (const auto& type : node.attr().at("Tout").list().type()) {
+    type_list->add_type(static_cast<DataType>(type));
+  }
+  for (int i = 0; i < node.attr().at("Tout").list().type_size(); ++i) {
+    inlined_output->add_input(strings::StrCat(inlined->name(), ":", i));
+  }
+
+  // Convert the graphdef to a graph
+  OptimizerOptions optimizer_opts;
+  optimizer_opts.set_do_function_inlining(true);
+  ProcessFunctionLibraryRuntime pflr(&dvc_mgr, env,
+                                     graph_def.versions().producer(),
+                                     &function_library, optimizer_opts);
+  FunctionLibraryRuntime* flr = pflr.GetFLR(dev->name());
+  CHECK(flr);
+  GraphConstructorOptions graph_ctor_opts;
+  graph_ctor_opts.allow_internal_ops = true;
+  graph_ctor_opts.expect_device_spec = false;
+  Graph graph(function_library);
+  TF_RETURN_IF_ERROR(
+      ConvertGraphDefToGraph(graph_ctor_opts, graph_def, &graph));
+
+  // Recursively inline the functions until there is nothing more to inline. We
+  // should at least expand one function.
+  int counter = 0;
+  while (counter < 50 && ExpandInlineFunctions(flr, &graph)) {
+    ++counter;
+  }
+  if (counter == 0) {
+    // Nothing was inlined
+    return errors::InvalidArgument(
+        strings::StrCat("Failed to inline node ", node.name()));
+  }
+
+  GraphDef inlined_graph_def;
+  graph.ToGraphDef(&inlined_graph_def);
+
+  // Add the default values of attributes to the nodes that have been inlined.
+  TF_RETURN_IF_ERROR(AddDefaultAttrsToGraphDef(&inlined_graph_def,
+                                               *graph.op_registry(), 0, true));
+
+  // Add the inlined nodes to the graph
+  for (NodeDef& inlined_node : *inlined_graph_def.mutable_node()) {
+    if (inlined_node.name() == "FunctionOutputs") {
+      inlined_node.set_name(node.name());
+      for (int i = 0; i < inlined_node.input_size(); ++i) {
+        inlined_node.set_input(
+            i, strings::StrCat(node.name(), "/", inlined_node.input(i)));
+      }
+    } else if (inlined_node.name() == "FunctionInputs") {
+      inlined_node.set_name(
+          strings::StrCat(node.name(), "/", inlined_node.name()));
+      inlined_node.clear_input();
+      for (int i = 0; i < node.input_size(); ++i) {
+        inlined_node.add_input(node.input(i));
+      }
+    } else {
+      inlined_node.set_name(
+          strings::StrCat(node.name(), "/", inlined_node.name()));
+      for (int i = 0; i < inlined_node.input_size(); ++i) {
+        inlined_node.set_input(
+            i, strings::StrCat(node.name(), "/", inlined_node.input(i)));
+      }
+    }
+    inlined_node.set_device(node.device());
+    inlined_graph->add_node()->Swap(&inlined_node);
+  }
+
+  return Status::OK();
+}
+
 Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                    GraphDef* optimized_graph) {
   std::unordered_map<string, const FunctionDef*> functions;
@@ -159,6 +280,13 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   // Inline functions when possible.
   for (const NodeDef& node : item.graph.node()) {
+    if (opt_level_ == RewriterConfig::AGGRESSIVE) {
+      if (node.op() == "SymbolicGradient") {
+        TF_RETURN_IF_ERROR(InlineSymbolicGradient(node, item.graph.library(),
+                                                  optimized_graph));
+        continue;
+      }
+    }
     auto it = functions.find(node.op());
     if (it == functions.end()) {
       *optimized_graph->add_node() = node;
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.h b/tensorflow/core/grappler/optimizers/function_optimizer.h
index 5c80226e9d..b124efe01d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_GRAPPLER_OPTIMIZERS_FUNCTION_OPTIMIZER_H_
 
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -25,7 +26,7 @@ namespace grappler {
 // operations to make the overall graph more efficient.
 class FunctionOptimizer : public GraphOptimizer {
  public:
-  FunctionOptimizer() {}
+  FunctionOptimizer(RewriterConfig::Toggle opt_level) : opt_level_(opt_level) {}
   ~FunctionOptimizer() override {}
 
   string name() const override { return "function_optimizer"; };
@@ -35,6 +36,9 @@ class FunctionOptimizer : public GraphOptimizer {
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimized_graph, double result) override;
+
+ private:
+  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index bafcdf4923..b1c55d838d 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
+#include "tensorflow/cc/ops/functional_ops.h"
+#include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -40,7 +42,7 @@ TEST_F(FunctionOptimizerTest, SimpleFunction) {
           test::function::XTimesTwo(),
       });
 
-  FunctionOptimizer optimizer;
+  FunctionOptimizer optimizer(RewriterConfig::DEFAULT);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -133,7 +135,7 @@ TEST_F(FunctionOptimizerTest, FixedTypeFunction) {
           x_times_two,
       });
 
-  FunctionOptimizer optimizer;
+  FunctionOptimizer optimizer(RewriterConfig::DEFAULT);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -219,7 +221,7 @@ TEST_F(FunctionOptimizerTest, FunctionWithOutputMapping) {
           func,
       });
 
-  FunctionOptimizer optimizer;
+  FunctionOptimizer optimizer(RewriterConfig::DEFAULT);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -317,7 +319,7 @@ TEST_F(FunctionOptimizerTest, FunctionWithInputForwarding) {
           func,
       });
 
-  FunctionOptimizer optimizer;
+  FunctionOptimizer optimizer(RewriterConfig::DEFAULT);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -368,7 +370,7 @@ TEST_F(FunctionOptimizerTest, FunctionWithoutInput) {
           func,
       });
 
-  FunctionOptimizer optimizer;
+  FunctionOptimizer optimizer(RewriterConfig::DEFAULT);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -377,6 +379,49 @@ TEST_F(FunctionOptimizerTest, FunctionWithoutInput) {
   EXPECT_EQ(item.graph.DebugString(), output.DebugString());
 }
 
+TEST_F(FunctionOptimizerTest, SymbolicGradients) {
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+  //  auto T = DT_FLOAT;
+  FunctionDef func = FunctionDefHelper::Define(
+      "TestFunc", {"x:float", "y:float"}, {"l:float"}, {},
+      {
+          {{"z"}, "Add", {"x", "y"}, {{"T", DT_FLOAT}}},
+          FunctionDefHelper::Const("zero", 0),
+          FunctionDefHelper::Const("one", 1),
+          {{"r"}, "Rank", {"z"}, {{"T", DT_FLOAT}}},
+          {{"indices"}, "Range", {"zero", "r", "one"}},
+          {{"l"}, "Sum", {"z", "indices"}, {{"T", DT_FLOAT}}},
+      });
+
+  auto dummy_variable = ops::Variable(scope, {2, 2}, DT_FLOAT);
+  auto x = ops::Const(scope, 1.0f);
+  auto y = ops::Const(scope, 2.0f);
+  auto dl = ops::Const(scope, 3.0f);
+
+  NameAttrList fn;
+  fn.set_name("TestFunc");
+  (*fn.mutable_attr())["T"].set_type(DT_FLOAT);
+  auto g0 = ops::SymbolicGradient(scope, std::initializer_list<Input>{x, y, dl},
+                                  {DT_FLOAT, DT_FLOAT}, fn);
+  auto out1 = ops::Identity(scope.WithOpName("out1"), g0.output[0]);
+  auto out2 = ops::Identity(scope.WithOpName("out2"), g0.output[1]);
+
+  GrapplerItem item;
+  TF_EXPECT_OK(scope.ToGraphDef(&item.graph));
+  *item.graph.mutable_library()->add_function() = func;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  std::vector<Tensor> expected = EvaluateNodes(item.graph, {"out1", "out2"});
+  std::vector<Tensor> optimized = EvaluateNodes(output, {"out1", "out2"});
+  test::ExpectTensorEqual<float>(expected[0], optimized[0]);
+  test::ExpectTensorEqual<float>(expected[1], optimized[1]);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 3a764937fd..7b2e7a1fe0 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -58,7 +58,7 @@ std::unique_ptr<GraphOptimizer> MetaOptimizer::NewOptimizer(
     graph_optimizer.reset(new ModelPruner());
   }
   if (optimizer == "function") {
-    graph_optimizer.reset(new FunctionOptimizer());
+    graph_optimizer.reset(new FunctionOptimizer(cfg_.function_optimization()));
   }
   if (optimizer == "constfold") {
     graph_optimizer.reset(new ConstantFolding(cpu_device_));
@@ -95,8 +95,8 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(new ModelPruner()));
     }
     if (cfg_.function_optimization() != RewriterConfig::OFF) {
-      optimizers.push_back(
-          std::unique_ptr<GraphOptimizer>(new FunctionOptimizer()));
+      optimizers.push_back(std::unique_ptr<GraphOptimizer>(
+          new FunctionOptimizer(cfg_.function_optimization())));
     }
     if (cfg_.constant_folding() != RewriterConfig::OFF) {
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
-- 
GitLab


From 2dd9a59d292c1226d03a1fa5bd551e5ccc5e6f9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 17:46:39 -0700
Subject: [PATCH 090/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 189274859

---
 tensorflow/go/op/wrappers.go | 190 +++++++++++++++++------------------
 1 file changed, 95 insertions(+), 95 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 0424c12fd9..469d1e9adb 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -509,101 +509,6 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
-// Scatter `updates` into a new (initially zero) tensor according to `indices`.
-//
-// Creates a new tensor by applying sparse `updates` to individual
-// values or slices within a zero tensor of the given `shape` according to
-// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
-// extracts values or slices from a given tensor.
-//
-// **WARNING**: The order in which updates are applied is nondeterministic, so the
-// output will be nondeterministic if `indices` contains duplicates.
-//
-// `indices` is an integer tensor containing indices into a new tensor of shape
-// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
-//
-//     indices.shape[-1] <= shape.rank
-//
-// The last dimension of `indices` corresponds to indices into elements
-// (if `indices.shape[-1] = shape.rank`) or slices
-// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
-// `shape`.  `updates` is a tensor with shape
-//
-//     indices.shape[:-1] + shape[indices.shape[-1]:]
-//
-// The simplest form of scatter is to insert individual elements in a tensor by
-// index. For example, say we want to insert 4 scattered elements in a rank-1
-// tensor with 8 elements.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[4], [3], [1], [7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     shape = tf.constant([8])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [0, 11, 0, 10, 9, 0, 0, 12]
-//
-// We can also, insert entire slices of a higher rank tensor all at once. For
-// example, if we wanted to insert two slices in the first dimension of a
-// rank-3 tensor with two matrices of new values.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[0], [2]])
-//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]],
-//                            [[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
-//     shape = tf.constant([4, 4, 4])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
-//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
-//
-// Arguments:
-//	indices: Index tensor.
-//	updates: Updates to scatter into output.
-//	shape: 1-D. The shape of the resulting tensor.
-//
-// Returns A new tensor with the given shape and updates applied according
-// to the indices.
-func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ScatterNd",
-		Input: []tf.Input{
-			indices, updates, shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Replaces the contents of the table with the specified keys and values.
 //
 // The tensor `keys` must be of the same type as the keys of the table.
@@ -25067,6 +24972,101 @@ func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
+// Scatter `updates` into a new (initially zero) tensor according to `indices`.
+//
+// Creates a new tensor by applying sparse `updates` to individual
+// values or slices within a zero tensor of the given `shape` according to
+// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
+// extracts values or slices from a given tensor.
+//
+// **WARNING**: The order in which updates are applied is nondeterministic, so the
+// output will be nondeterministic if `indices` contains duplicates.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of scatter is to insert individual elements in a tensor by
+// index. For example, say we want to insert 4 scattered elements in a rank-1
+// tensor with 8 elements.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     shape = tf.constant([8])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [0, 11, 0, 10, 9, 0, 0, 12]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     shape = tf.constant([4, 4, 4])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
+//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
+//
+// Arguments:
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//	shape: 1-D. The shape of the resulting tensor.
+//
+// Returns A new tensor with the given shape and updates applied according
+// to the indices.
+func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ScatterNd",
+		Input: []tf.Input{
+			indices, updates, shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SpaceToDepthAttr is an optional argument to SpaceToDepth.
 type SpaceToDepthAttr func(optionalAttr)
 
-- 
GitLab


From 6b9eebb0c024ee3500fef5ec8019c7b5ee4cb6c3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 18:22:09 -0700
Subject: [PATCH 091/960] Internal cleanup.

PiperOrigin-RevId: 189278596
---
 tensorflow/c/eager/c_api.cc | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 0811bd363f..455bc19be8 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -1213,8 +1213,11 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
     // Note that `h` may not be currently ready. However execution order will
     // make sure that `h` is ready before the copy is actually done.
     CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
+    TFE_TensorHandle* output = node->dst();
+    // Note that calling Add makes `node` accessible by the TFE_Executor thread.
+    // So further accesses need to be thread-safe.
     ctx->executor.Add(node);
-    return node->dst();
+    return output;
   } else {
     TFE_TensorHandle* output = nullptr;
     status->status = TensorHandleCopyToDevice(h, ctx, dstd, &output);
-- 
GitLab


From 4c1081cfddbba33dbd12a28635265bc718697e23 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Thu, 15 Mar 2018 21:33:44 -0700
Subject: [PATCH 092/960] Implement out of bounds behavior for gather in the
 HLO evaluator

This makes the OOB behavior of gather in the HLO evaluator consistent with
DynamicSlice while we figure out the semantics we want long term.

The HLO->HLO gather expander inherits the wrapping behavior of dynamic-slice
because it lowers the gather ops to loops of dynamic slices.

PiperOrigin-RevId: 189293175
---
 .../compiler/xla/service/hlo_evaluator.cc     | 11 ++-
 .../xla/tests/gather_operation_test.cc        | 80 ++++++++++++++++++-
 2 files changed, 88 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 91341b5d35..693004d364 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -2771,6 +2771,8 @@ Status HloEvaluator::HandleGather(HloInstruction* gather) {
       gather->gather_dimension_numbers(), /*input_shape=*/operand.shape(),
       /*output_shape=*/shape);
 
+  const Shape& operand_shape = operand.shape();
+
   auto gather_inner_loop_body =
       [&](ArraySlice<int64> output_window_index,
           ArraySlice<int64> input_gather_index,
@@ -2780,9 +2782,16 @@ Status HloEvaluator::HandleGather(HloInstruction* gather) {
         output_window_index_to_input_index(output_window_index));
     for (int i = 0, e = output_index.size(); i < e; i++) {
       output_index[i] = output_gather_index[i] + output_window_index[i];
+      DCHECK_LT(output_index[i], shape.dimensions(i));
     }
     for (int i = 0, e = input_index.size(); i < e; i++) {
-      input_index[i] = input_gather_index[i] + input_window_index[i];
+      // TODO(b/74360564): We should implement whatever out of bounds behavior
+      // we decide for dynamic-slice here as well.
+      input_index[i] = (input_gather_index[i] + input_window_index[i]) %
+                       operand_shape.dimensions(i);
+      if (input_index[i] < 0) {
+        input_index[i] += operand_shape.dimensions(i);
+      }
     }
     TF_RETURN_IF_ERROR(
         result->CopyElementFrom(operand, input_index, output_index));
diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc
index 4e2f19ade1..0830e9c8f0 100644
--- a/tensorflow/compiler/xla/tests/gather_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc
@@ -31,12 +31,16 @@ class GatherOperationTest : public HloTestBase {
  protected:
   void RunTest(const string& hlo_text, Literal* operand,
                Literal* gather_indices) {
+    RunTest(hlo_text, {operand, gather_indices});
+  }
+
+  void RunTest(const string& hlo_text,
+               tensorflow::gtl::ArraySlice<Literal*> args) {
     HloModuleConfig config;
     config.set_debug_options(GetDebugOptionsForTest());
     TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
                             tools::Parse(hlo_text, config));
-    EXPECT_TRUE(
-        RunAndCompare(std::move(module), {operand, gather_indices}, nullopt));
+    EXPECT_TRUE(RunAndCompare(std::move(module), args, nullopt));
   }
 };
 
@@ -259,5 +263,77 @@ ENTRY main {
   RunTest(hlo_text, operand.get(), gather_indices.get());
 }
 
+XLA_TEST_F(GatherOperationTest, OutOfBoundsIndex) {
+  // Out of bounds indices must not crash, and the indices in range should
+  // produce the same values across all backends.
+  //
+  // TODO(b/74360564): Once we have a well defined semantics for OOB accesses,
+  // we should get rid of the mask and check that backends produce the same
+  // value for OOB indices too.
+
+  const string hlo_text = R"(
+HloModule BatchDynamicSlice
+
+ENTRY main {
+  operand = s32[3,3]{1,0} parameter(0)
+  indices = s32[6,2]{1,0} parameter(1)
+  gather = s32[6,1,1]{2,1,0} gather(operand, indices),
+      output_window_dims={1,2},
+      elided_window_dims={},
+      gather_dims_to_operand_dims={0,1},
+      index_vector_dim=1,
+      window_bounds={1,1}
+  gather_reshaped = s32[6]{0} reshape(gather)
+  in_bounds_mask = s32[6]{0} parameter(2)
+  ROOT result = s32[6]{0} multiply(gather_reshaped, in_bounds_mask)
+}
+)";
+  std::unique_ptr<Literal> operand =
+      Literal::CreateR2<int32>({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}});
+  std::unique_ptr<Literal> gather_indices = Literal::CreateR2<int32>(
+      {{2, 7}, {2, 1}, {1, 1}, {5, 1}, {2147483647, 1}, {1, 2}});
+  std::unique_ptr<Literal> in_bounds_mask =
+      Literal::CreateR1<int32>({0, 1, 1, 0, 0, 1});
+
+  RunTest(hlo_text,
+          {operand.get(), gather_indices.get(), in_bounds_mask.get()});
+}
+
+XLA_TEST_F(GatherOperationTest, NegativeIndex) {
+  // Negative indices must not crash, and the indices in range should produce
+  // the same values across all backends.
+  //
+  // TODO(b/74360564): Once we have a well defined semantics for negative
+  // accesses, we should get rid of the mask and check that backends produce the
+  // same value for negative indices too.
+
+  const string hlo_text = R"(
+HloModule BatchDynamicSlice
+
+ENTRY main {
+  operand = s32[3,3]{1,0} parameter(0)
+  indices = s32[6,2]{1,0} parameter(1)
+  gather = s32[6,1,1]{2,1,0} gather(operand, indices),
+      output_window_dims={1,2},
+      elided_window_dims={},
+      gather_dims_to_operand_dims={0,1},
+      index_vector_dim=1,
+      window_bounds={1,1}
+  gather_reshaped = s32[6]{0} reshape(gather)
+  in_bounds_mask = s32[6]{0} parameter(2)
+  ROOT result = s32[6]{0} multiply(gather_reshaped, in_bounds_mask)
+}
+)";
+  std::unique_ptr<Literal> operand =
+      Literal::CreateR2<int32>({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}});
+  std::unique_ptr<Literal> gather_indices = Literal::CreateR2<int32>(
+      {{2, -1}, {2, 1}, {1, 1}, {-500, 1}, {-2147483648, 1}, {1, 2}});
+  std::unique_ptr<Literal> in_bounds_mask =
+      Literal::CreateR1<int32>({0, 1, 1, 0, 0, 1});
+
+  RunTest(hlo_text,
+          {operand.get(), gather_indices.get(), in_bounds_mask.get()});
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 048a149cfec52db38c457fc7937eee0f4cc50888 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 15 Mar 2018 21:39:15 -0700
Subject: [PATCH 093/960] Automated g4 rollback of changelist 189234789

PiperOrigin-RevId: 189293458
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 31 -------------------
 tensorflow/core/ops/ops.pbtxt                 | 31 -------------------
 2 files changed, 62 deletions(-)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 1834cc998c..85dd1a423a 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -51522,37 +51522,6 @@ op {
     }
   }
 }
-op {
-  name: "SlideDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "window_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "stride"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "Snapshot"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 763b104305..3faa4eeada 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -24319,37 +24319,6 @@ op {
     }
   }
 }
-op {
-  name: "SlideDataset"
-  input_arg {
-    name: "input_dataset"
-    type: DT_VARIANT
-  }
-  input_arg {
-    name: "window_size"
-    type: DT_INT64
-  }
-  input_arg {
-    name: "stride"
-    type: DT_INT64
-  }
-  output_arg {
-    name: "handle"
-    type: DT_VARIANT
-  }
-  attr {
-    name: "output_types"
-    type: "list(type)"
-    has_minimum: true
-    minimum: 1
-  }
-  attr {
-    name: "output_shapes"
-    type: "list(shape)"
-    has_minimum: true
-    minimum: 1
-  }
-}
 op {
   name: "Snapshot"
   input_arg {
-- 
GitLab


From 6485bb7029c6d856c7ffa744168a8864ef6c986c Mon Sep 17 00:00:00 2001
From: Tatiana Shpeisman <shpeisman@google.com>
Date: Thu, 15 Mar 2018 22:04:33 -0700
Subject: [PATCH 094/960] MKL DNN: fix the TF1.6 speed issue by fixing MKL DNN
 LRN taking the optimum path (#17605) (#17751)

* MKL DNN: fix the TF1.6 speed issue by fixing MKL DNN LRN

* fixed typos in the doc for LrnRewrite
---
 tensorflow/core/graph/mkl_layout_pass.cc | 26 ++++++++++++++++++++++--
 1 file changed, 24 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 02038c5d77..568fc87e65 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2492,10 +2492,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       mkl_op_registry::GetMklOpName(csinfo_.identity),
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.lrn, mkl_op_registry::GetMklOpName(csinfo_.lrn),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.lrn_grad,
                       mkl_op_registry::GetMklOpName(csinfo_.lrn_grad),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.max_pool,
                       mkl_op_registry::GetMklOpName(csinfo_.max_pool),
                       CopyAttrsPooling, NonDepthBatchWisePoolRewrite});
@@ -2865,6 +2865,28 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return false;
   }
 
+  // If the depth_radius of LRN is not 2, then MKL DNN takes unoptimized 
+  // path. The unoptimized path is slow. Thus we dont rewrite the node 
+  // and use default Eigen. But for depth_radius=2, MKL DNN optimized 
+  // path is taken, i.e., eigen node is rewritten by MKl DNN node.
+  static bool LrnRewrite(const Node* n) {
+    CHECK_NOTNULL(n);
+
+    int depth_radius;
+    CHECK_EQ(GetNodeAttr(n->def(), "depth_radius", &depth_radius).ok(), true);
+
+    // if the depth_radius of LRN is not 2, don't rewrite the node by MKL DNN
+    // and use eigen node instead 
+    if (depth_radius == 2) {
+      return true;
+    }
+    VLOG(1) << "LrnRewrite: The model sets depth_radius as not 2 which"
+            << "case is not optimized by Intel MKL, thus using Eigen op"
+            << "for LRN " ; 
+
+    return false;
+  }
+
   static bool AddNRewrite(const Node* n) {
     CHECK_NOTNULL(n);
 
-- 
GitLab


From 17024c0afd19e3713ab5016602372c5244e11183 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 15 Mar 2018 22:28:26 -0700
Subject: [PATCH 095/960] Propagate min-max when resolving constant Reshape op.

PiperOrigin-RevId: 189296593
---
 .../resolve_constant_unary.cc                 | 58 ++++++++++++++-----
 1 file changed, 42 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
index 6d5636d744..d4db6f1c00 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_unary.cc
@@ -28,24 +28,45 @@ limitations under the License.
 
 namespace toco {
 
+bool CopyMinMaxFromFirstInput(const Operator& op, Model* model) {
+  auto& output_array = model->GetArray(op.outputs[0]);
+  if (output_array.minmax) {
+    return false;
+  }
+  const auto& input_array = model->GetArray(op.inputs[0]);
+  if (!input_array.minmax) {
+    return false;
+  }
+  const auto& input_minmax = input_array.GetMinMax();
+  CHECK(!output_array.minmax);
+  auto& output_minmax = output_array.GetOrCreateMinMax();
+  output_minmax.min = input_minmax.min;
+  output_minmax.max = input_minmax.max;
+  return true;
+}
+
 bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
   const auto unary_it = model->operators.begin() + op_index;
   const auto* unary_op = unary_it->get();
-  // Test for unary ops of types that we know how to resolve
-  if (unary_op->type != OperatorType::kCast &&
-      unary_op->type != OperatorType::kNeg &&
-      unary_op->type != OperatorType::kTensorFlowRsqrt &&
-      unary_op->type != OperatorType::kTensorFlowSqrt &&
-      unary_op->type != OperatorType::kTensorFlowSquare &&
-      unary_op->type != OperatorType::kTensorFlowSum &&
-      unary_op->type != OperatorType::kTensorFlowMin &&
-      unary_op->type != OperatorType::kTensorFlowMax &&
-      unary_op->type != OperatorType::kTensorFlowReshape &&
-      unary_op->type != OperatorType::kRelu6 &&
-      unary_op->type != OperatorType::kRelu1 &&
-      unary_op->type != OperatorType::kRelu) {
-    return false;
+  // Test for unary ops of types that we know how to resolve.
+  switch (unary_op->type) {
+    case OperatorType::kCast:
+    case OperatorType::kNeg:
+    case OperatorType::kTensorFlowRsqrt:
+    case OperatorType::kTensorFlowSqrt:
+    case OperatorType::kTensorFlowSquare:
+    case OperatorType::kTensorFlowSum:
+    case OperatorType::kTensorFlowMin:
+    case OperatorType::kTensorFlowMax:
+    case OperatorType::kTensorFlowReshape:
+    case OperatorType::kRelu6:
+    case OperatorType::kRelu1:
+    case OperatorType::kRelu:
+      break;
+    default:
+      return false;
   }
+
   // Check if the input is a constant parameter.
   if (!IsConstantParameterArray(*model, unary_op->inputs[0])) {
     return false;
@@ -79,6 +100,12 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
     return false;
   }
 
+  // The min-max is only copied for ops that copy data without arithmetic.
+  // In future trivial transpose, etc, can be handled here.
+  if (unary_op->type == OperatorType::kTensorFlowReshape) {
+    CopyMinMaxFromFirstInput(*unary_op, model);
+  }
+
   const auto& input_array = model->GetArray(unary_op->inputs[0]);
   // We have already tested above for existence of buffers (synonymous to being
   // a constant param).
@@ -138,8 +165,7 @@ bool ResolveConstantUnaryOperator::Run(Model* model, std::size_t op_index) {
     }
   } else if (unary_op->type == OperatorType::kTensorFlowReshape) {
     CHECK(input_buffer_size == output_buffer_size);
-    memcpy(output_float_data.data(), (*input_float_data).data(),
-           output_buffer_size * sizeof(output_float_data[0]));
+    output_float_data = *input_float_data;
   } else if (unary_op->type == OperatorType::kTensorFlowSum) {
     CHECK_EQ(unary_op->inputs.size(), 2) << "Sum needs 2 inputs";
     if (!IsConstantParameterArray(*model, unary_op->inputs[1])) {
-- 
GitLab


From 47407ccb99a61fd5115130020ff8ef5ef9272433 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Fri, 16 Mar 2018 14:49:16 +0800
Subject: [PATCH 096/960] Fix minor typo in saved_model.md (#17513)

* Fix minor typo in saved_model.md

* Fix broken link and revert wrong typo fix
---
 tensorflow/docs_src/programmers_guide/saved_model.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md
index d01d187e86..55ee42dd64 100644
--- a/tensorflow/docs_src/programmers_guide/saved_model.md
+++ b/tensorflow/docs_src/programmers_guide/saved_model.md
@@ -3,7 +3,7 @@
 The @{tf.train.Saver} class provides methods to save and restore models. The
 @{tf.saved_model.simple_save} function is an easy way to build a
 @{tf.saved_model$saved model} suitable for serving.
-[Estimators](/programmers_guide/estimators) automatically save and restore
+[Estimators](@{$programmers_guide/estimators}) automatically save and restore
 variables in the `model_dir`.
 
 ## Save and restore variables
@@ -400,7 +400,7 @@ defined in:
 
 After training an `Estimator` model, you may want to create a service
 from that model that takes requests and returns a result.  You can run such a
-service locally on your machine or deploy it scalably in the cloud.
+service locally on your machine or deploy it in the cloud.
 
 To prepare a trained Estimator for serving, you must export it in the standard
 SavedModel format. This section explains how to:
-- 
GitLab


From bd1dac4cba7831b327d0c9410e494ee7cf2b93a5 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Thu, 15 Mar 2018 23:56:10 -0700
Subject: [PATCH 097/960] Don't put quantization variables in EMA collection by
 default.

PiperOrigin-RevId: 189302082
---
 tensorflow/contrib/quantize/python/quantize.py       | 4 ++--
 tensorflow/contrib/quantize/python/quantize_graph.py | 8 ++++++++
 2 files changed, 10 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 0608ab9302..6cc097b20e 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -44,7 +44,7 @@ def Quantize(graph,
              activation_bits=8,
              ema_decay=0.999,
              quant_delay=None,
-             vars_collection=ops.GraphKeys.MOVING_AVERAGE_VARIABLES):
+             vars_collection=ops.GraphKeys.GLOBAL_VARIABLES):
   """Updates graph with quantization operations.
 
   Args:
@@ -262,7 +262,7 @@ def _InsertQuantOp(context,
                    bits=8,
                    ema_decay=0.999,
                    quant_delay=None,
-                   vars_collection=ops.GraphKeys.MOVING_AVERAGE_VARIABLES,
+                   vars_collection=ops.GraphKeys.GLOBAL_VARIABLES,
                    narrow_range=False):
   """Inserts a quant op between a producer op and (multiple) consumer ops.
 
diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index 5a3a74cec4..be4fc39651 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -72,6 +72,8 @@ def _create_graph(input_graph=None,
 def create_training_graph(input_graph=None, quant_delay=0):
   """Rewrites a training input_graph in place for simulated quantization.
 
+  Variables added by the rewrite get added to the global variables collection.
+
   The graph has fake quantization ops inserted to simulate the error
   introduced by quantization. Since the graph is transformed in place,
   the expected behavior of previously held references to nodes and tensors may
@@ -118,6 +120,8 @@ def create_training_graph(input_graph=None, quant_delay=0):
 def create_eval_graph(input_graph=None):
   """Rewrites an eval input_graph in place for simulated quantization.
 
+  Variables added by the rewrite get added to the global variables collection.
+
   The graph has fake quantization ops inserted to simulate the error
   introduced by quantization. Since the graph is transformed in place,
   the expected behavior of previously held references to nodes and tensors may
@@ -141,6 +145,8 @@ def experimental_create_training_graph(input_graph=None,
                                        freeze_bn_delay=int(2e5)):
   """Rewrites a training input_graph in place for simulated quantization.
 
+  Variables added by the rewrite get added to the global variables collection.
+
   This function has additional experimental options not (yet) available to
   create_training_graph. The resulting behavior may be undefined.
 
@@ -188,6 +194,8 @@ def experimental_create_eval_graph(input_graph=None,
                                    activation_bits=8):
   """Rewrites an eval input_graph in place for simulated quantization.
 
+  Variables added by the rewrite get added to the global variables collection.
+
   This function has additional experimental options not (yet) available to
   create_eval_graph. The resulting behavior may be undefined.
 
-- 
GitLab


From 9c4a5b962fb84661ebbf128d83345157e9e3d224 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 03:45:32 -0700
Subject: [PATCH 098/960] [tf2xla] Introduce XlaTensorInfo

XlaTensorInfo is side-band data for Tensors. It can be used to store
information about Tensors that is not possible to store in the Tensor
itself.  The XlaTensorInfos are managed by XlaTensorInfoManager, which
is an Allocator, which allows it to release the TensorInfos when the
underlying Tensor is released.  Looking up an XlaTensorInfo for a
Tensor requires a hash table lookup. This implementation keeps this
off the fast path and only looks the tensorinfos up when they are
required.

PiperOrigin-RevId: 189319553
---
 tensorflow/compiler/jit/BUILD                 |  17 ++
 .../compiler/jit/kernels/xla_launch_op.cc     |  17 +-
 tensorflow/compiler/jit/xla_cpu_device.cc     |   7 +-
 tensorflow/compiler/jit/xla_device.cc         |  38 ++--
 tensorflow/compiler/jit/xla_device.h          |  23 ++-
 tensorflow/compiler/jit/xla_device_context.cc |  53 ++++--
 tensorflow/compiler/jit/xla_device_context.h  |  13 +-
 tensorflow/compiler/jit/xla_gpu_device.cc     |   7 +-
 tensorflow/compiler/jit/xla_launch_util.cc    | 163 ++++++++++--------
 tensorflow/compiler/jit/xla_launch_util.h     |  67 +++++--
 tensorflow/compiler/jit/xla_tensor_info.cc    |  56 ++++++
 tensorflow/compiler/jit/xla_tensor_info.h     |  85 +++++++++
 tensorflow/core/framework/tensor.h            |   2 +
 13 files changed, 414 insertions(+), 134 deletions(-)
 create mode 100644 tensorflow/compiler/jit/xla_tensor_info.cc
 create mode 100644 tensorflow/compiler/jit/xla_tensor_info.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index c4a2d4ab03..39eb390f38 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -118,6 +118,21 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "xla_tensor_info",
+    srcs = ["xla_tensor_info.cc"],
+    hdrs = ["xla_tensor_info.h"],
+    deps = [
+        ":common",
+        "//tensorflow/compiler/xla/service:shaped_buffer",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 cc_library(
     name = "xla_device",
     srcs = [
@@ -136,6 +151,7 @@ cc_library(
         ":common",
         ":jit_compilation_passes",
         ":xla_launch_util",
+        ":xla_tensor_info",
         "//tensorflow/compiler/jit/ops:xla_ops",
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:dump_graph",
@@ -182,6 +198,7 @@ cc_library(
     deps = [
         ":common",
         ":xla_compilation_cache",
+        ":xla_tensor_info",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
diff --git a/tensorflow/compiler/jit/kernels/xla_launch_op.cc b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
index cd7f8dd779..e24a9a0751 100644
--- a/tensorflow/compiler/jit/kernels/xla_launch_op.cc
+++ b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
@@ -114,10 +114,16 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
   // this is more obviously correct.)
   core::ScopedUnref cache_ref(cache);
 
+  const XlaDevice::Metadata* metadata;
+  Status s = XlaDevice::GetMetadata(ctx, &metadata);
+
+  XlaTensorInfoManager* tensor_info_manager = nullptr;
+  if (s.ok()) {
+    tensor_info_manager = &metadata->tensor_info_manager();
+  }
+
   // Get the platform_id_ for XLA_* devices.
   if (platform_id_ == nullptr) {
-    const XlaDevice::Metadata* metadata;
-    Status s = XlaDevice::GetMetadata(ctx, &metadata);
     if (s.ok()) {
       platform_id_ = metadata->platform()->id();
     }
@@ -148,8 +154,8 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
 
   VLOG(1) << "Executing XLA Computation...";
 
-  XlaComputationLaunchContext launch_context(num_resource_args_, client,
-                                             &xla_allocator);
+  XlaComputationLaunchContext launch_context(
+      num_resource_args_, client, &xla_allocator, tensor_info_manager);
   launch_context.PopulateInputs(ctx, kernel, variables);
 
   // Execute the computation.
@@ -166,8 +172,7 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
   auto elapsed = env->NowMicros() - start_time;
   VLOG(2) << "Elapsed time: " << elapsed << "us";
 
-  launch_context.PopulateOutputs(ctx, kernel,
-                                 run_result.ConsumeValueOrDie()->release());
+  launch_context.PopulateOutputs(ctx, kernel, run_result.ConsumeValueOrDie());
   VLOG(1) << "Done";
 }
 
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index e238252751..db3bf3ea33 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -39,9 +39,10 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& options,
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  TF_RETURN_IF_ERROR(XlaDevice::Create(
-      "Host", DEVICE_XLA_CPU, 0, DEVICE_CPU_XLA_JIT, options, name_prefix,
-      /*register_device_for_compilation=*/true, &device));
+  TF_RETURN_IF_ERROR(XlaDevice::Create("Host", DEVICE_XLA_CPU, 0,
+                                       DEVICE_CPU_XLA_JIT, options, name_prefix,
+                                       /*register_device_for_compilation=*/true,
+                                       /*transfer_as_literal=*/false, &device));
   devices->push_back(device.release());
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index d4d8fe1c1d..e4e11d4ce2 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -109,7 +109,7 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
     const string& platform_name, const string& device_name, int device_ordinal,
     const string& jit_device_name, const SessionOptions& options,
     const string& name_prefix, bool register_device_for_compilation,
-    std::unique_ptr<XlaDevice>* device) {
+    bool transfer_as_literal, std::unique_ptr<XlaDevice>* device) {
   VLOG(1) << "XlaDevice::Create " << platform_name << " " << device_name << ":"
           << device_ordinal;
 
@@ -137,15 +137,17 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
 
   device->reset(new XlaDevice(options, attrs, device_ordinal,
                               DeviceType(jit_device_name),
-                              platform.ValueOrDie()));
+                              platform.ValueOrDie(), transfer_as_literal));
   return Status::OK();
 }
 
-XlaDevice::Metadata::Metadata(int device_ordinal, se::Platform* platform,
-                              const DeviceType& device_type)
+XlaDevice::Metadata::Metadata(
+    int device_ordinal, se::Platform* platform, const DeviceType& device_type,
+    std::unique_ptr<XlaTensorInfoManager>* tensor_info_manager)
     : device_ordinal_(device_ordinal),
       device_type_(device_type),
-      platform_(platform) {}
+      platform_(platform),
+      tensor_info_manager_(*tensor_info_manager) {}
 
 int XlaDevice::Metadata::device_ordinal() const { return device_ordinal_; }
 
@@ -160,6 +162,10 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const {
   return device_type_;
 }
 
+XlaTensorInfoManager& XlaDevice::Metadata::tensor_info_manager() const {
+  return *tensor_info_manager_;
+}
+
 /* static */ Status XlaDevice::GetMetadata(OpKernelContext* ctx,
                                            const Metadata** metadata) {
   XlaDevice* xla_device =
@@ -177,13 +183,19 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const {
 
 XlaDevice::XlaDevice(const SessionOptions& options,
                      const DeviceAttributes& attrs, int device_ordinal,
-                     const DeviceType& jit_device_name, se::Platform* platform)
+                     const DeviceType& jit_device_name, se::Platform* platform,
+                     bool transfer_as_literal)
     : LocalDevice(options, attrs),
-      xla_metadata_(device_ordinal, platform, jit_device_name),
+      xla_metadata_(
+          device_ordinal, platform, jit_device_name,
+          // Pass tensor_info_manager_ by reference as it is initialized lazily.
+          &tensor_info_manager_),
       device_ordinal_(device_ordinal),
       jit_device_name_(jit_device_name),
       xla_allocator_(nullptr),
-      platform_(platform) {}
+      platform_(platform),
+      tensor_info_manager_(nullptr),
+      transfer_as_literal_(transfer_as_literal) {}
 
 XlaDevice::~XlaDevice() {}
 
@@ -208,6 +220,7 @@ Allocator* XlaDevice::GetAllocator(AllocatorAttributes attr) {
     xla::Backend* backend = client()->mutable_backend();
     xla_allocator_ = XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
         backend, device_ordinal_);
+    tensor_info_manager_.reset(new XlaTensorInfoManager(xla_allocator_));
   }
   return xla_allocator_;
 }
@@ -225,7 +238,11 @@ Status XlaDevice::FillContextMap(const Graph* graph,
   VLOG(1) << "XlaDevice::FillContextMap";
   device_context_map->resize(graph->num_node_ids());
   TF_ASSIGN_OR_RETURN(se::Stream * stream, GetStream());
-  auto ctx = new XlaDeviceContext(stream);
+  // Call GetAllocator for the side-effect of ensuring the allocator and
+  // XlaTensorInfoManager is created.
+  (void)GetAllocator({});
+  auto ctx = new XlaDeviceContext(stream, tensor_info_manager_.get(),
+                                  transfer_as_literal_);
   for (Node* n : graph->nodes()) {
     VLOG(2) << n->id() << " : " << n->type_string() << " : " << n->name();
     ctx->Ref();
@@ -273,7 +290,8 @@ Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
     Tensor copy(GetAllocator(alloc_attrs), parsed.dtype(), parsed.shape());
     Notification n;
     TF_ASSIGN_OR_RETURN(se::Stream * stream, GetStream());
-    XlaTransferManager manager(stream);
+    XlaTransferManager manager(stream, tensor_info_manager_.get(),
+                               transfer_as_literal_);
     manager.CopyCPUTensorToDevice(&parsed, this, &copy,
                                   [&n, &status](const Status& s) {
                                     status = s;
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index d2ec38293c..0f4476296b 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -26,6 +26,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_
 #define TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_
 
+#include "tensorflow/compiler/jit/xla_tensor_info.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/local_device.h"
@@ -48,7 +49,8 @@ class XlaDevice : public LocalDevice {
   class Metadata {
    public:
     Metadata(int device_ordinal, perftools::gputools::Platform* platform,
-             const DeviceType& device_type);
+             const DeviceType& device_type,
+             std::unique_ptr<XlaTensorInfoManager>* tensor_info_manager);
 
     // The index of the device on this host.
     int device_ordinal() const;
@@ -56,11 +58,13 @@ class XlaDevice : public LocalDevice {
     perftools::gputools::Platform* platform() const;
     xla::LocalClient* client() const;
     const DeviceType& jit_device_type() const;
+    XlaTensorInfoManager& tensor_info_manager() const;
 
    private:
     const int device_ordinal_;
     const DeviceType device_type_;
     perftools::gputools::Platform* platform_;  // Not owned.
+    std::unique_ptr<XlaTensorInfoManager>& tensor_info_manager_;
 
     TF_DISALLOW_COPY_AND_ASSIGN(Metadata);
   };
@@ -71,15 +75,20 @@ class XlaDevice : public LocalDevice {
   // Factory function. 'platform_name' is the name of the XLA platform.
   // 'device_name' is the name of the Tensorflow device to create.
   // 'jit_device_name' is the name of the corresponding JIT device.
+  // 'transfer_as_literal' is true if device<->host transfers must be done using
+  // XLA's TransferLiteral{To,From}Device interface. If false, we can use
+  // ThenMemcpy instead.
   static Status Create(const string& platform_name, const string& device_name,
                        int device_ordinal, const string& jit_device_name,
                        const SessionOptions& options, const string& name_prefix,
                        bool register_device_for_compilation,
+                       bool transfer_as_literal,
                        std::unique_ptr<XlaDevice>* device);
 
   XlaDevice(const SessionOptions& options, const DeviceAttributes& attrs,
             int device_ordinal, const DeviceType& jit_device_name,
-            ::perftools::gputools::Platform* platform);
+            ::perftools::gputools::Platform* platform,
+            bool transfer_as_literal);
   ~XlaDevice() override;
 
   Allocator* GetAllocator(AllocatorAttributes attr) override;
@@ -113,6 +122,16 @@ class XlaDevice : public LocalDevice {
   // copying back and forth between CPU and the device, and
   // computations enqueued by XLA.
   xla::Backend::StreamPtr stream_;
+  // Manages sideband data about tensors, in particular the on-device shape tree
+  // if the tensor requires multiple device buffers to represent (for example,
+  // tuple shapes).
+  // This is a unique_ptr because XlaTensorInfoManager is non-copy-constructible
+  // and we need to initialize this lazily (as we also lazily initialize the
+  // underlying allocator).
+  std::unique_ptr<XlaTensorInfoManager> tensor_info_manager_;
+  // Must we use XLA's transfer manager for correct host<->device transfers? if
+  // false, we can use ThenMemcpy() instead.
+  bool transfer_as_literal_;
 };
 
 // Builds dummy OpKernel registrations on 'device' for the JIT operators
diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index c936222f32..b57f82f98e 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/jit/xla_device_context.h"
 
+#include "tensorflow/compiler/jit/xla_launch_util.h"
 #include "tensorflow/compiler/tf2xla/literal_util.h"
 #include "tensorflow/compiler/tf2xla/shape_util.h"
 #include "tensorflow/compiler/xla/util.h"
@@ -52,7 +53,12 @@ void XlaDeviceAllocator::DeallocateRaw(void* ptr) {
 
 void XlaDeviceAllocator::GetStats(AllocatorStats* stats) { stats->Clear(); }
 
-XlaTransferManager::XlaTransferManager(se::Stream* stream) : stream_(stream) {}
+XlaTransferManager::XlaTransferManager(
+    se::Stream* stream, XlaTensorInfoManager* tensor_info_manager,
+    bool transfer_as_literal)
+    : stream_(stream),
+      tensor_info_manager_(tensor_info_manager),
+      transfer_as_literal_(transfer_as_literal) {}
 
 void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
                                                Device* device,
@@ -72,13 +78,19 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
     se::DeviceMemoryBase dev_dst_ptr(dst_ptr, total_bytes);
 
     Status status;
-    stream_->ThenMemcpy(&dev_dst_ptr, src_ptr, total_bytes);
-    // TODO(hpucha): Make this asynchronous.
-    Status block_status = stream_->BlockHostUntilDone();
-    if (!block_status.ok()) {
-      status = xla::InternalError(
-          "Failed to complete data transfer on stream %p: %s", stream_,
-          block_status.error_message().c_str());
+    if (transfer_as_literal_) {
+      status = xla::Unimplemented(
+          "XlaTransferManager::CopyCPUTensorToDevice not implemented for "
+          "literals");
+    } else {
+      stream_->ThenMemcpy(&dev_dst_ptr, src_ptr, total_bytes);
+      // TODO(hpucha): Make this asynchronous.
+      Status block_status = stream_->BlockHostUntilDone();
+      if (!block_status.ok()) {
+        status = xla::InternalError(
+            "Failed to complete data transfer on stream %p: %s", stream_,
+            block_status.error_message().c_str());
+      }
     }
 
     done(status);
@@ -108,13 +120,19 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor,
     void* dst_ptr = DMAHelper::base(cpu_tensor);
 
     Status status;
-    stream_->ThenMemcpy(dst_ptr, dev_src_ptr, total_bytes);
-    // TODO(hpucha): Make this asynchronous.
-    Status block_status = stream_->BlockHostUntilDone();
-    if (!block_status.ok()) {
-      status = xla::InternalError(
-          "Failed to complete data transfer on stream %p: %s", stream_,
-          block_status.error_message().c_str());
+    if (transfer_as_literal_) {
+      status = xla::Unimplemented(
+          "XlaTransferManager::CopyDeviceTensorToCPU not implemented for "
+          "literals");
+    } else {
+      stream_->ThenMemcpy(dst_ptr, dev_src_ptr, total_bytes);
+      // TODO(hpucha): Make this asynchronous.
+      Status block_status = stream_->BlockHostUntilDone();
+      if (!block_status.ok()) {
+        status = xla::InternalError(
+            "Failed to complete data transfer on stream %p: %s", stream_,
+            block_status.error_message().c_str());
+      }
     }
 
     done(status);
@@ -125,7 +143,10 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor,
   done(Status::OK());
 }
 
-XlaDeviceContext::XlaDeviceContext(se::Stream* stream) : manager_(stream) {}
+XlaDeviceContext::XlaDeviceContext(se::Stream* stream,
+                                   XlaTensorInfoManager* tensor_info_manager,
+                                   bool transfer_as_literal)
+    : manager_(stream, tensor_info_manager, transfer_as_literal) {}
 
 void XlaDeviceContext::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
                                              Device* device,
diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h
index c4edcd474e..df02f4eac4 100644
--- a/tensorflow/compiler/jit/xla_device_context.h
+++ b/tensorflow/compiler/jit/xla_device_context.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <memory>
 
+#include "tensorflow/compiler/jit/xla_tensor_info.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/framework/allocator.h"
@@ -49,7 +50,9 @@ class XlaDeviceAllocator : public Allocator {
 // Helper class for managing data transfers between host and XLA devices.
 class XlaTransferManager {
  public:
-  explicit XlaTransferManager(perftools::gputools::Stream* stream);
+  explicit XlaTransferManager(perftools::gputools::Stream* stream,
+                              XlaTensorInfoManager* tensor_info_manager,
+                              bool transfer_as_literal);
 
   void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device,
                              Tensor* device_tensor, StatusCallback done) const;
@@ -62,6 +65,10 @@ class XlaTransferManager {
   // Stream obtained from a Device, used to transfer tensors between
   // CPU and device.
   perftools::gputools::Stream* stream_;
+  // The tensor info manager, for access to sideband information about tensors.
+  XlaTensorInfoManager* tensor_info_manager_;
+  // True if we must use XLA's TransferManager for correct device transfers.
+  bool transfer_as_literal_;
 };
 
 // DeviceContext for operators assigned to XlaDevice devices. The
@@ -69,7 +76,9 @@ class XlaTransferManager {
 // wraps the methods in XlaTransferManager.
 class XlaDeviceContext : public DeviceContext {
  public:
-  explicit XlaDeviceContext(perftools::gputools::Stream* stream);
+  explicit XlaDeviceContext(perftools::gputools::Stream* stream,
+                            XlaTensorInfoManager* tensor_info_manager,
+                            bool transfer_as_literal);
 
   void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device,
                              Tensor* device_tensor,
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 2326070358..383ed879ef 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -39,9 +39,10 @@ Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& options,
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  Status status = XlaDevice::Create(
-      "CUDA", DEVICE_XLA_GPU, 0, DEVICE_GPU_XLA_JIT, options, name_prefix,
-      /*register_device_for_compilation=*/true, &device);
+  Status status = XlaDevice::Create("CUDA", DEVICE_XLA_GPU, 0,
+                                    DEVICE_GPU_XLA_JIT, options, name_prefix,
+                                    /*register_device_for_compilation=*/true,
+                                    /*transfer_as_literal=*/false, &device);
   if (!status.ok()) {
     // Treat failures as non-fatal; there might not be a GPU in the machine.
     VLOG(1) << "Failed to create XLA_GPU device: " << status;
diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 8322dd2e82..689fa3299c 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -56,74 +56,56 @@ XlaAllocator::XlaAllocator(const gpu::Platform* platform,
                            OpKernelContext* op_context)
     : xla::DeviceMemoryAllocator(platform), op_context_(op_context) {}
 
-XlaAllocator::~XlaAllocator() = default;
+XlaAllocator::~XlaAllocator() { CHECK(allocated_.empty()); }
 
 xla::StatusOr<gpu::DeviceMemoryBase> XlaAllocator::Allocate(
     int device_ordinal, uint64 size, bool retry_on_failure) {
-  AllocatorAttributes allocator_attrs;
-  allocator_attrs.set_on_host(false);
-
-  AllocationAttributes allocation_attrs;
-  allocation_attrs.no_retry_on_failure = !retry_on_failure;
-
-  Tensor t;
-  Status status = op_context_->allocate_temp(
-      DT_UINT8, TensorShape({static_cast<int64>(size)}), &t, allocator_attrs,
-      allocation_attrs);
-  if (!status.ok()) {
-    VLOG(2) << "Allocation failed " << size;
-    return status;
-  }
-  void* data =
-      reinterpret_cast<void*>(const_cast<char*>(t.tensor_data().data()));
-  tensors_[data] = t;
+  void* data = op_context_->device()->GetAllocator({})->AllocateRaw(
+      Allocator::kAllocatorAlignment, size);
+  allocated_.insert(data);
   return gpu::DeviceMemoryBase(data, size);
 }
 
-Status XlaAllocator::RegisterArgument(const Tensor* t) {
-  void* data =
-      reinterpret_cast<void*>(const_cast<char*>(t->tensor_data().data()));
-  tensors_[data] = *t;
-  return Status::OK();
-}
+void XlaAllocator::Release(void* ptr) { allocated_.erase(ptr); }
 
 Status XlaAllocator::Deallocate(int device_ordinal,
                                 gpu::DeviceMemoryBase* mem) {
-  if (mem->opaque() != nullptr) {
-    if (tensors_.erase(mem->opaque()) == 0) {
-      return tensorflow::errors::InvalidArgument("Unknown tensor address");
-    }
+  if (allocated_.count(mem->opaque())) {
+    op_context_->device()->GetAllocator({})->DeallocateRaw(mem->opaque());
+    allocated_.erase(mem->opaque());
   }
   return Status::OK();
 }
 
-Status XlaAllocator::MakeTensorFromBuffer(gpu::DeviceMemoryBase buffer,
-                                          DataType dtype,
-                                          const TensorShape& shape,
-                                          Tensor* out_tensor) const {
-  void* ptr = const_cast<void*>(buffer.opaque());
-  auto it = tensors_.find(ptr);
-  if (it == tensors_.end()) {
-    return errors::InvalidArgument("Unknown tensor address");
-  }
-  const Tensor& tensor = it->second;
-
-  int64 output_size = DataTypeSize(dtype) * shape.num_elements();
-  if (tensor.TotalBytes() == output_size) {
-    out_tensor->UnsafeCopyFromInternal(tensor, dtype, shape);
-  } else {
-    Tensor slice = tensor.Slice(0, output_size);
-    out_tensor->UnsafeCopyFromInternal(slice, dtype, shape);
-  }
-  return Status::OK();
+namespace {
+// Return the 'index''th subtree of the given ShapedBuffer as a ShapedBuffer.
+xla::ShapedBuffer ExtractSubShapedBuffer(const xla::ShapedBuffer& shaped_buffer,
+                                         int index) {
+  xla::Shape on_host_shape = xla::ShapeUtil::GetTupleElementShape(
+      shaped_buffer.on_host_shape(), index);
+  xla::Shape on_device_shape = xla::ShapeUtil::GetTupleElementShape(
+      shaped_buffer.on_device_shape(), index);
+
+  xla::ShapedBuffer sub_shaped_buffer(on_host_shape, on_device_shape,
+                                      shaped_buffer.platform(),
+                                      shaped_buffer.device_ordinal());
+
+  auto& shape_tree = shaped_buffer.buffers();
+  auto& sub_shape_tree = sub_shaped_buffer.buffers();
+  sub_shape_tree.CopySubtreeFrom(shape_tree,
+                                 /*source_base_index=*/{index},
+                                 /*target_base_index=*/{});
+  return sub_shaped_buffer;
 }
+}  // namespace
 
 XlaComputationLaunchContext::XlaComputationLaunchContext(
     int64 num_resource_args, xla::LocalClient* client,
-    XlaAllocator* xla_allocator)
+    XlaAllocator* xla_allocator, XlaTensorInfoManager* tensor_info_manager)
     : num_resource_args_(num_resource_args),
       client_(client),
-      xla_allocator_(xla_allocator) {}
+      xla_allocator_(xla_allocator),
+      tensor_info_manager_(tensor_info_manager) {}
 
 void XlaComputationLaunchContext::PopulateInputs(
     OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel,
@@ -145,29 +127,35 @@ void XlaComputationLaunchContext::PopulateInputs(
       t = &(ctx->input(arg_num));
     }
 
-    gpu::DeviceMemoryBase dmem = gpu::DeviceMemoryBase(
-        const_cast<char*>(t->tensor_data().data()), t->tensor_data().size());
-
     const xla::Shape on_device_shape =
         client_->backend().transfer_manager()->HostShapeToDeviceShape(shape);
-    CHECK(xla::ShapeUtil::Equal(shape, on_device_shape))
-        << "On-device shape "
-        << xla::ShapeUtil::HumanStringWithLayout(on_device_shape)
-        << " not the same as on-host shape "
-        << xla::ShapeUtil::HumanStringWithLayout(shape);
-    arg_buffers_[i] = xla::MakeUnique<xla::ShapedBuffer>(
-        /*on_host_shape=*/shape, /*on_device_shape=*/shape, client_->platform(),
-        client_->default_device_ordinal());
-    arg_buffers_[i]->set_buffer(dmem, /*index=*/{});
-    arg_ptrs_[i] = arg_buffers_[i].get();
-
-    OP_REQUIRES_OK(ctx, xla_allocator_->RegisterArgument(t));
+    if (xla::ShapeUtil::IsTuple(on_device_shape)) {
+      CHECK(tensor_info_manager_);
+      const XlaTensorInfo* tensor_info =
+          tensor_info_manager_->GetTensorInfo(*t);
+      CHECK(tensor_info && tensor_info->has_shaped_buffer());
+      arg_ptrs_[i] =
+          const_cast<xla::ShapedBuffer*>(&tensor_info->shaped_buffer());
+    } else {
+      CHECK(xla::ShapeUtil::Equal(shape, on_device_shape))
+          << "On-device shape "
+          << xla::ShapeUtil::HumanStringWithLayout(on_device_shape)
+          << " not the same as on-host shape "
+          << xla::ShapeUtil::HumanStringWithLayout(shape);
+      gpu::DeviceMemoryBase dmem = gpu::DeviceMemoryBase(
+          const_cast<char*>(t->tensor_data().data()), t->tensor_data().size());
+      arg_buffers_[i] = xla::MakeUnique<xla::ShapedBuffer>(
+          /*on_host_shape=*/shape, /*on_device_shape=*/shape,
+          client_->platform(), client_->default_device_ordinal());
+      arg_buffers_[i]->set_buffer(dmem, /*index=*/{});
+      arg_ptrs_[i] = arg_buffers_[i].get();
+    }
   }
 }
 
 void XlaComputationLaunchContext::PopulateOutputs(
     OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel,
-    std::unique_ptr<xla::ShapedBuffer> output) {
+    std::unique_ptr<xla::ScopedShapedBuffer> output) {
   gpu::Stream* stream =
       ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr;
 
@@ -180,6 +168,11 @@ void XlaComputationLaunchContext::PopulateOutputs(
   // Copy XLA results to the OpOutputList.
   int output_num = 0;
   for (int i = 0; i < ctx->num_outputs(); ++i) {
+    AllocatorAttributes alloc_attrs = ctx->output_alloc_attr(i);
+    Allocator* allocator = ctx->device()->GetAllocator(alloc_attrs);
+    if (tensor_info_manager_ && !alloc_attrs.on_host()) {
+      allocator = tensor_info_manager_;
+    }
     if (kernel->outputs[i].is_constant) {
       // Output is a constant.
       const Tensor& const_tensor = kernel->outputs[i].constant_value;
@@ -204,11 +197,19 @@ void XlaComputationLaunchContext::PopulateOutputs(
       VLOG(2) << "Retval " << i << " shape " << shape.DebugString();
 
       gpu::DeviceMemoryBase buffer = output->buffer({output_num});
-      Tensor output_tensor;
-      // Looks up the owning Tensor by buffer address.
-      OP_REQUIRES_OK(ctx, xla_allocator_->MakeTensorFromBuffer(
-                              buffer, ctx->expected_output_dtype(i), shape,
-                              &output_tensor));
+      Tensor output_tensor = XlaTensorBuffer::MakeTensor(
+          ctx->expected_output_dtype(i), shape, buffer, allocator);
+      xla_allocator_->Release(buffer.opaque());
+
+      xla::Shape output_shape = xla::ShapeUtil::GetTupleElementShape(
+          output->on_device_shape(), output_num);
+      if (xla::ShapeUtil::IsTuple(output_shape)) {
+        CHECK(tensor_info_manager_);
+        XlaTensorInfo* tensor_info =
+            tensor_info_manager_->GetOrCreateTensorInfo(output_tensor);
+        tensor_info->set_shaped_buffer(
+            ExtractSubShapedBuffer(*output, output_num));
+      }
       ctx->set_output(i, output_tensor);
       ++output_num;
     }
@@ -221,6 +222,10 @@ void XlaComputationLaunchContext::PopulateOutputs(
   // Apply variable updates, if any.
   VLOG(2) << "Applying variable updates";
   for (int i = 0; i < kernel->resource_updates.size(); ++i) {
+    Allocator* allocator = ctx->device()->GetAllocator({});
+    if (tensor_info_manager_) {
+      allocator = tensor_info_manager_;
+    }
     const XlaCompiler::ResourceUpdate& write = kernel->resource_updates[i];
     OP_REQUIRES(ctx,
                 write.input_index >= 0 && write.input_index < ctx->num_inputs(),
@@ -243,11 +248,19 @@ void XlaComputationLaunchContext::PopulateOutputs(
     mutex_lock ml(*variable->mu());
     OP_REQUIRES(ctx, variable->tensor()->dtype() == write.type,
                 errors::Internal("Mismatched type in variable write"));
-
-    // Looks up the owning Tensor by buffer address.
-    OP_REQUIRES_OK(ctx,
-                   xla_allocator_->MakeTensorFromBuffer(
-                       buffer, write.type, write.shape, variable->tensor()));
+    *variable->tensor() =
+        XlaTensorBuffer::MakeTensor(write.type, write.shape, buffer, allocator);
+    xla_allocator_->Release(buffer.opaque());
+
+    xla::Shape output_shape = xla::ShapeUtil::GetTupleElementShape(
+        output->on_device_shape(), output_num);
+    if (xla::ShapeUtil::IsTuple(output_shape)) {
+      CHECK(tensor_info_manager_);
+      XlaTensorInfo* tensor_info =
+          tensor_info_manager_->GetOrCreateTensorInfo(*variable->tensor());
+      tensor_info->set_shaped_buffer(
+          ExtractSubShapedBuffer(*output, output_num));
+    }
     ++output_num;
   }
 }
diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 9fd356fce5..8694f6ce58 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -19,8 +19,10 @@ limitations under the License.
 #define TENSORFLOW_COMPILER_JIT_XLA_LAUNCH_UTIL_H_
 
 #include "tensorflow/compiler/jit/xla_compilation_cache.h"
+#include "tensorflow/compiler/jit/xla_tensor_info.h"
 #include "tensorflow/compiler/tf2xla/xla_compiler.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/core/framework/allocation_description.pb.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/variable_ops.h"
@@ -52,16 +54,8 @@ class XlaAllocator : public xla::DeviceMemoryAllocator {
   Status Deallocate(int device_ordinal,
                     perftools::gputools::DeviceMemoryBase* mem) override;
 
-  // Register an Tensor (input or resource variable) with the allocator. If
-  // the operation returns an alias to one of its inputs, then the allocator
-  // needs to be able to handle it.
-  Status RegisterArgument(const Tensor* t);
-
-  // Makes 'tensor' a wrapper around the data buffer at 'ptr'. The buffer is
-  // interpreted as having data type 'dtype' and shape 'shape'.
-  Status MakeTensorFromBuffer(perftools::gputools::DeviceMemoryBase buffer,
-                              DataType dtype, const TensorShape& shape,
-                              Tensor* out_tensor) const;
+  // Un-track 'ptr' - do not delete it on destruction.
+  void Release(void* ptr);
 
   // The Tensorflow BFC allocator used on GPU allows host-side deallocation
   // before GPU execution takes place. Tensorflow uses the ordering of the main
@@ -74,11 +68,7 @@ class XlaAllocator : public xla::DeviceMemoryAllocator {
 
  private:
   OpKernelContext* const op_context_;
-
-  // Map from pointer address to the owning Tensor; used by
-  // MakeTensorFromBuffer. Also used to automatically release Tensors when the
-  // allocator is freed.
-  std::unordered_map<void*, Tensor> tensors_;
+  std::unordered_set<void*> allocated_;
 };
 
 // Helper class to perform the marshalling of TensorFlow inputs and outputs to
@@ -86,7 +76,8 @@ class XlaAllocator : public xla::DeviceMemoryAllocator {
 class XlaComputationLaunchContext {
  public:
   XlaComputationLaunchContext(int64 num_resource_args, xla::LocalClient* client,
-                              XlaAllocator* xla_allocator);
+                              XlaAllocator* xla_allocator,
+                              XlaTensorInfoManager* tensor_info_manager);
 
   // Add all inputs within `ctx` as XLA arguments (returned by arguments()).
   // `variables` is a map from TensorFlow argument number to resource variable.
@@ -97,7 +88,7 @@ class XlaComputationLaunchContext {
   // Given the XLA output in `output`, populate all outputs of `ctx`.
   void PopulateOutputs(OpKernelContext* ctx,
                        const XlaCompiler::CompilationResult* kernel,
-                       std::unique_ptr<xla::ShapedBuffer> output);
+                       std::unique_ptr<xla::ScopedShapedBuffer> output);
 
   // Return the argument list. Only valid after PopulateInputs() has been
   // called.
@@ -107,10 +98,52 @@ class XlaComputationLaunchContext {
   int64 num_resource_args_;
   xla::LocalClient* client_;
   XlaAllocator* xla_allocator_;
+  XlaTensorInfoManager* tensor_info_manager_;
   std::vector<std::unique_ptr<xla::ShapedBuffer>> arg_buffers_;
   std::vector<xla::ShapedBuffer*> arg_ptrs_;
 };
 
+// A simple TensorBuffer implementation that allows us to create Tensors that
+// take ownership of pre-allocated memory.
+class XlaTensorBuffer : public TensorBuffer {
+ public:
+  XlaTensorBuffer(const void* ptr, size_t expected_size, size_t actual_size,
+                  Allocator* allocator)
+      : expected_size_(expected_size),
+        actual_size_(actual_size),
+        allocator_(allocator) {
+    data_ = const_cast<void*>(ptr);
+  }
+
+  ~XlaTensorBuffer() override { allocator_->DeallocateRaw(data_); }
+
+  void* data() const override { return data_; }
+  size_t size() const override { return expected_size_; }
+
+  TensorBuffer* root_buffer() override { return this; }
+
+  void FillAllocationDescription(AllocationDescription* proto) const override {
+    proto->set_allocated_bytes(actual_size_);
+  }
+
+  static Tensor MakeTensor(DataType dtype, const TensorShape& shape,
+                           perftools::gputools::DeviceMemoryBase buffer,
+                           Allocator* allocator) {
+    size_t expected_size = shape.num_elements() * DataTypeSize(dtype);
+    auto* tensor_buffer = new XlaTensorBuffer(buffer.opaque(), expected_size,
+                                              buffer.size(), allocator);
+    Tensor t(dtype, shape, tensor_buffer);
+    tensor_buffer->Unref();
+    return t;
+  }
+
+ private:
+  void* data_;
+  size_t expected_size_;
+  size_t actual_size_;
+  Allocator* allocator_;
+};
+
 }  // namespace tensorflow
 
 #endif
diff --git a/tensorflow/compiler/jit/xla_tensor_info.cc b/tensorflow/compiler/jit/xla_tensor_info.cc
new file mode 100644
index 0000000000..0ce18c27cb
--- /dev/null
+++ b/tensorflow/compiler/jit/xla_tensor_info.cc
@@ -0,0 +1,56 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/jit/xla_tensor_info.h"
+
+namespace tensorflow {
+
+const XlaTensorInfo* XlaTensorInfoManager::GetTensorInfo(
+    const void* device_ptr) const {
+  mutex_lock lock(lock_);
+  auto iterator = tensor_infos_.find(device_ptr);
+  return (iterator == tensor_infos_.end()) ? nullptr
+                                           : tensor_infos_.at(device_ptr).get();
+}
+
+XlaTensorInfo* XlaTensorInfoManager::GetOrCreateTensorInfo(
+    const void* device_ptr) {
+  mutex_lock lock(lock_);
+  auto iterator = tensor_infos_.find(device_ptr);
+  if (iterator != tensor_infos_.end()) {
+    return iterator->second.get();
+  }
+  auto iterator_and_inserted =
+      tensor_infos_.emplace(device_ptr, MakeUnique<XlaTensorInfo>());
+  CHECK(iterator_and_inserted.second);
+  return iterator_and_inserted.first->second.get();
+}
+
+const XlaTensorInfo* XlaTensorInfoManager::GetTensorInfo(const Tensor& tensor) {
+  return GetTensorInfo(tensor.tensor_data().data());
+}
+
+XlaTensorInfo* XlaTensorInfoManager::GetOrCreateTensorInfo(
+    const Tensor& tensor) {
+  return GetOrCreateTensorInfo(tensor.tensor_data().data());
+}
+
+void XlaTensorInfoManager::DeallocateRaw(void* ptr) {
+  wrapped()->DeallocateRaw(ptr);
+  mutex_lock lock(lock_);
+  tensor_infos_.erase(ptr);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/xla_tensor_info.h b/tensorflow/compiler/jit/xla_tensor_info.h
new file mode 100644
index 0000000000..0b0736bf01
--- /dev/null
+++ b/tensorflow/compiler/jit/xla_tensor_info.h
@@ -0,0 +1,85 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_JIT_XLA_TENSOR_INFO_H_
+#define TENSORFLOW_COMPILER_JIT_XLA_TENSOR_INFO_H_
+
+#include "tensorflow/compiler/xla/service/shaped_buffer.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/device_base.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+
+// Information about a tensor. The XlaTensorInfoManager can maintain one of
+// these per device Tensor.
+class XlaTensorInfo {
+ public:
+  XlaTensorInfo() {}
+
+  // Some Tensors can have complex on-device shapes, including tuple shapes. To
+  // manage the memory for these tensors a ShapedBuffer may be required.
+
+  // Return true if this TensorInfo contains a ShapedBuffer.
+  bool has_shaped_buffer() const { return shaped_buffer_ != nullptr; }
+  // Return the contained ShapedBuffer.
+  // REQUIRES: has_shaped_buffer()
+  const xla::ShapedBuffer& shaped_buffer() const { return *shaped_buffer_; }
+  // Mutates the TensorInfo to set the ShapedBuffer.
+  void set_shaped_buffer(xla::ShapedBuffer shaped_buffer) {
+    shaped_buffer_.reset(new xla::ShapedBuffer(std::move(shaped_buffer)));
+  }
+
+ private:
+  // The optional contained ShapedBuffer.
+  std::unique_ptr<xla::ShapedBuffer> shaped_buffer_;
+};
+
+// Manages XlaTensorInfo objects. This class is also an Allocator, so that
+// XlaTensorInfo objects can be deleted when their Tensor is deallocated.
+class XlaTensorInfoManager : public AllocatorWrapper {
+ public:
+  // Creates a new XlaTensorInfoManager, delegating all DeallocateRaw calls to
+  // allocator.
+  XlaTensorInfoManager(Allocator* allocator) : AllocatorWrapper(allocator) {}
+
+  // Returns the XlaTensorInfo for the given device memory pointer or nullptr if
+  // none exists.
+  const XlaTensorInfo* GetTensorInfo(const void* device_ptr) const;
+  // Returns the XlaTensorInfo for the device memory pointer extracted from
+  // tensor or nullptr if none exists.
+  const XlaTensorInfo* GetTensorInfo(const Tensor& tensor);
+
+  // Returns the XlaTensorInfo for the given device memory pointer, creating one
+  // if necessary.
+  XlaTensorInfo* GetOrCreateTensorInfo(const Tensor& tensor);
+  // Returns the XlaTensorInfo for the device memory pointer extracted from
+  // tensor, creating one if necessary.
+  XlaTensorInfo* GetOrCreateTensorInfo(const void* device_ptr);
+
+  // Allocator interface
+  void DeallocateRaw(void* ptr) override;
+
+ private:
+  mutable mutex lock_;
+  // The managed tensor infos. The mapped value is a unique_ptr so that returned
+  // references are stable over rehashes.
+  std::unordered_map<const void*, std::unique_ptr<XlaTensorInfo>> tensor_infos_
+      GUARDED_BY(lock_);
+};
+}  // namespace tensorflow
+
+#endif
diff --git a/tensorflow/core/framework/tensor.h b/tensorflow/core/framework/tensor.h
index 9ae4bb5a2c..4d10f7efb5 100644
--- a/tensorflow/core/framework/tensor.h
+++ b/tensorflow/core/framework/tensor.h
@@ -483,6 +483,8 @@ class Tensor {
   friend class TensorTestHelper;      // For access to set_shape
   friend class OpKernelContext;       // For access to RefCountIsOne().
   friend class ScopedAllocator;       // For access to buf_.
+  friend class XlaTensorBuffer;  // For access to the private constructor taking
+                                 // the buffer
   template <typename Device, typename T>
   friend class AssignVariableOp;  // For access to RefCountIsOne().
   template <typename Device, typename T>
-- 
GitLab


From 3abdbfd431241f05c383eaaf0f8b499b063aa1cb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 05:53:38 -0700
Subject: [PATCH 099/960] Clean up and clarify the 'install from source' page.

Remove reference to CUDA and cuDNN versions for 'install from source' from the 'install Linux' documentation. The 'install from source' should be the authoritative page for this.

PiperOrigin-RevId: 189328669
---
 tensorflow/docs_src/install/install_linux.md  | 51 +++++++------------
 .../docs_src/install/install_sources.md       | 45 +++++++---------
 2 files changed, 35 insertions(+), 61 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index e3e115d9f6..88ceca3cda 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -31,49 +31,32 @@ If you are installing TensorFlow with GPU support using one of the
 mechanisms described in this guide, then the following NVIDIA software
 must be installed on your system:
 
-  * CUDA® Toolkit 9.0. For details, see
-    [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/#axzz4VZnqTJ2A).
-    Ensure that you append the relevant Cuda pathnames to the
+  * [CUDA Toolkit 9.0](http://nvidia.com/cuda). For details, see
+    [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/).
+    Ensure that you append the relevant CUDA pathnames to the
     `LD_LIBRARY_PATH` environment variable as described in the
     NVIDIA documentation.
-  * The NVIDIA drivers associated with CUDA Toolkit 9.0.
-  * cuDNN v7.0. For details, see
-    [NVIDIA's documentation](https://developer.nvidia.com/cudnn).
+  * [cuDNN SDK v7](http://developer.nvidia.com/cudnn). For details, see
+    [NVIDIA's documentation](http://docs.nvidia.com/deeplearning/sdk/cudnn-install/).
     Ensure that you create the `CUDA_HOME` environment variable as
-    described in the NVIDIA documentation.
-  * GPU card with CUDA Compute Capability 3.0 or higher.  See
-    [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for
-    a list of supported GPU cards.
-  * The libcupti-dev library, which is the NVIDIA CUDA Profile Tools Interface.
-    This library provides advanced profiling support. To install this library,
-    issue the following command for CUDA Toolkit >= 8.0:
-
-    <pre>
-    $ <b>sudo apt-get install cuda-command-line-tools</b>
-    </pre>
+    described in NVIDIA's documentation.
+  * [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but
+    you also need to append its path to the `LD_LIBRARY_PATH` environment
+    variable:
 
-    and add its path to your `LD_LIBRARY_PATH` environment variable:
+    <pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
 
-    <pre>
-    $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b>
-    </pre>
+In order to run TensorFlow computations on the GPU, you also need:
 
-    For CUDA Toolkit <= 7.5 do:
-
-    <pre>
-    $ <b>sudo apt-get install libcupti-dev</b>
-    </pre>
+  * A GPU card with CUDA Compute Capability 3.0 or higher.  See
+    [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for
+    a list of supported GPU cards.
+  * [GPU drivers](http://nvidia.com/driver) supporting your version of the CUDA
+    Toolkit.
 
 If you have an earlier version of the preceding packages, please upgrade to
 the specified versions. If upgrading is not possible, then you may still run
-TensorFlow with GPU support, but only if you do the following:
-
-  * Install TensorFlow from sources as documented in
-    @{$install_sources$Installing TensorFlow from Sources}.
-  * Install or upgrade to at least the following NVIDIA versions:
-    * CUDA toolkit 7.0 or greater
-    * cuDNN v3 or greater
-    * GPU card with CUDA Compute Capability 3.0 or higher.
+TensorFlow with GPU support, if you @{$install_sources$install TensorFlow from Sources}.
 
 
 ## Determine how to install TensorFlow
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index acf0af0d9d..4e7b07d78b 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -133,30 +133,21 @@ The following NVIDIA <i>hardware</i> must be installed on your system:
 
 The following NVIDIA <i>software</i> must be installed on your system:
 
-  * NVIDIA's Cuda Toolkit (>= 7.0). We recommend version 9.0.
+  * [CUDA Toolkit](http://nvidia.com/cuda) (>= 7.0). We recommend version 9.0.
     For details, see
-    [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/#axzz4VZnqTJ2A).
-    Ensure that you append the relevant Cuda pathnames to the
+    [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/).
+    Ensure that you append the relevant CUDA pathnames to the
     `LD_LIBRARY_PATH` environment variable as described in the
     NVIDIA documentation.
-  * The NVIDIA drivers associated with NVIDIA's Cuda Toolkit.
-  * cuDNN (>= v3). We recommend version 6.0. For details, see
-    [NVIDIA's documentation](https://developer.nvidia.com/cudnn),
-    particularly the description of appending the appropriate pathname
-    to your `LD_LIBRARY_PATH` environment variable.
-
-Finally, you must also install `libcupti` which for Cuda Toolkit >= 8.0 you do via
-
-<pre> $ <b>sudo apt-get install cuda-command-line-tools</b> </pre>
-
-and add its path to your `LD_LIBRARY_PATH` environment variable:
-
-<pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
-
-For Cuda Toolkit <= 7.5, you install `libcupti-dev` by invoking the following command:
-
-<pre> $ <b>sudo apt-get install libcupti-dev</b> </pre>
+  * [GPU drivers](http://nvidia.com/driver) supporting your version of the CUDA
+    Toolkit.
+  * [cuDNN SDK](http://developer.nvidia.com/cudnn) (>= v3). We recommend version 7.0. For details, see
+    [NVIDIA's documentation](http://docs.nvidia.com/deeplearning/sdk/cudnn-install/).
+  * [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but
+    you also need to append its path to the `LD_LIBRARY_PATH` environment
+    variable:
 
+    <pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
 
 ### Next
 
@@ -240,8 +231,8 @@ such as compiler flags. You must run this script *prior* to
 creating the pip package and installing TensorFlow.
 
 If you wish to build TensorFlow with GPU, `configure` will ask
-you to specify the version numbers of Cuda and cuDNN. If several
-versions of Cuda or cuDNN are installed on your system, explicitly select
+you to specify the version numbers of CUDA and cuDNN. If several
+versions of CUDA or cuDNN are installed on your system, explicitly select
 the desired version instead of relying on the default.
 
 One of the questions that `configure` will ask is as follows:
@@ -289,12 +280,12 @@ Do you wish to build TensorFlow with CUDA support? [y/N] <b>Y</b>
 CUDA support will be enabled for TensorFlow
 Do you want to use clang as CUDA compiler? [y/N]
 nvcc will be used as CUDA compiler
-Please specify the Cuda SDK version you want to use, e.g. 7.0. [Leave empty to default to CUDA 9.0]: <b>9.0</b>
+Please specify the CUDA SDK version you want to use, e.g. 7.0. [Leave empty to default to CUDA 9.0]: <b>9.0</b>
 Please specify the location where CUDA 9.0 toolkit is installed. Refer to README.md for more details. [Default is /usr/local/cuda]:
 Please specify which gcc should be used by nvcc as the host compiler. [Default is /usr/bin/gcc]:
 Please specify the cuDNN version you want to use. [Leave empty to default to cuDNN 7.0]: <b>7</b>
 Please specify the location where cuDNN 7 library is installed. Refer to README.md for more details. [Default is /usr/local/cuda]:
-Please specify a list of comma-separated Cuda compute capabilities you want to build with.
+Please specify a list of comma-separated CUDA compute capabilities you want to build with.
 You can find the compute capability of your device at: https://developer.nvidia.com/cuda-gpus.
 Please note that each additional compute capability significantly increases your build time and binary size.
 [Default is: "3.5,5.2"]: <b>3.0</b>
@@ -304,14 +295,14 @@ Configuration finished
 </pre>
 
 If you told `configure` to build for GPU support, then `configure`
-will create a canonical set of symbolic links to the Cuda libraries
-on your system.  Therefore, every time you change the Cuda library paths,
+will create a canonical set of symbolic links to the CUDA libraries
+on your system.  Therefore, every time you change the CUDA library paths,
 you must rerun the `configure` script before re-invoking
 the <code>bazel build</code> command.
 
 Note the following:
 
-  * Although it is possible to build both Cuda and non-Cuda configs
+  * Although it is possible to build both CUDA and non-CUDA configs
     under the same source tree, we recommend running `bazel clean` when
     switching between these two configurations in the same source tree.
   * If you don't run the `configure` script *before* running the
-- 
GitLab


From 9e62c648a84f664fe338e1dec2db0f5e89ec3147 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Fri, 16 Mar 2018 07:40:49 -0700
Subject: [PATCH 100/960] [XLA:python] Fix a bug where returning an status
 would not incref Py_None.

PiperOrigin-RevId: 189337748
---
 tensorflow/compiler/xla/python/local_computation_builder.i | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index b5354131c9..b2681d5e8b 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -216,6 +216,7 @@ tensorflow::ImportNumpy();
         PyExc_RuntimeError, $1.ToString().c_str());
     return NULL;
   }
+  Py_INCREF(Py_None);
   $result = Py_None;
 }
 
-- 
GitLab


From f4c6bd6b422c6383ac814c50aa2243442e1049cc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 07:54:08 -0700
Subject: [PATCH 101/960] - Adds support for shared embedding layers (e.g. in
 RNNs), and shared Conv2D layers. - Some minor refactoring of internal
 structure in fisher_blocks and layer_collection

PiperOrigin-RevId: 189338874
---
 .../kernel_tests/layer_collection_test.py     |  18 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  | 330 ++++++++++++++----
 .../kfac/python/ops/layer_collection.py       | 260 ++++++++++----
 tensorflow/contrib/kfac/python/ops/utils.py   |   8 +
 4 files changed, 483 insertions(+), 133 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index bae6bd7a3b..ba22099340 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -135,8 +135,22 @@ class LayerCollectionTest(test.TestCase):
           array_ops.constant(6),
           16,
           approx=layer_collection.APPROX_DIAGONAL_NAME)
-
-      self.assertEqual(9, len(lc.get_blocks()))
+      lc.register_fully_connected_multi(
+          array_ops.constant(1),
+          (array_ops.constant(2), array_ops.constant(3)),
+          (array_ops.constant(4), array_ops.constant(5)))
+      lc.register_conv2d_multi(
+          params=array_ops.ones((2, 3, 4, 5)),
+          strides=[1, 1, 1, 1],
+          padding='SAME',
+          inputs=(array_ops.ones((1, 2, 3, 4)), array_ops.ones((5, 6, 7, 8))),
+          outputs=(array_ops.ones((1, 1, 1, 5)), array_ops.ones((2, 2, 2, 10))))
+      lc.register_embedding_multi(
+          array_ops.constant((1,)),
+          (array_ops.constant(2), array_ops.constant(3)),
+          (array_ops.constant(4), array_ops.constant(5)))
+
+      self.assertEqual(12, len(lc.get_blocks()))
 
   def testRegisterBlocksMultipleRegistrations(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 31f4689fbf..79d0424dca 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -48,6 +48,7 @@ from tensorflow.contrib.kfac.python.ops import utils
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.util import nest
 
 # For blocks corresponding to convolutional layers, or any type of block where
 # the parameters can be thought of as being replicated in time or space,
@@ -74,6 +75,86 @@ def set_global_constants(normalize_damping_power=None, pi_type=None):
     PI_TYPE = pi_type
 
 
+def _make_partitionedtensors_inputs(inputs):
+  """Constructs PartitionedTensor for inputs.
+
+  The purpose of this method is to package up the towers/minibatch dimension
+  of these arrays into PartitionedTensor objects.
+
+  Args:
+    inputs: a 1-D list of Tensors. Index is tower/mini-batch.
+
+  Returns:
+    A PartitionedTensor.
+  """
+  return utils.PartitionedTensor(inputs)
+
+
+def _make_partitionedtensors_grads(grads_list):
+  """Constructs PartitionedTensor for grads_list.
+
+  The purpose of this method is to package up the towers/minibatch dimension
+  of these arrays into PartitionedTensor objects.
+
+  Args:
+    grads_list: 2-D list of Tensors. First index is for source, second
+      index for tower.
+
+  Returns:
+    Tuple of PartitionedTensors, one per source.
+  """
+  return tuple(utils.PartitionedTensor(grads) for grads in grads_list)
+
+
+def _make_partitionedtensors_multi_inputs(inputs):
+  """Constructs PartitionedTensors for inputs.
+
+  The purpose of this method is to package up the towers/minibatch dimension
+  of these arrays into PartitionedTensor objects.
+
+  This version of this function is for use with FisherBlocks that deal with
+  multiple uses or time-steps. One PartitionedTensor is created for each
+  use/time-step.  The FisherBlock will be responsible for concatenating
+  (or doing whatever else it wants) with the resulting lists.
+
+  Args:
+    inputs: a 2-D list of Tensors. First index is tower/mini-batch, second is
+      use/time-step.
+
+  Returns:
+    A tuple of PartitionedTensor's, one per use/time-step.
+  """
+  num_uses = len(inputs[0])
+  assert all(len(input_) == num_uses for input_ in inputs)
+
+  return tuple(utils.PartitionedTensor(input_) for input_ in zip(*inputs))
+
+
+def _make_partitionedtensors_multi_grads(grads_list):
+  """Constructs PartitionedTensors for grads_list.
+
+  The purpose of this method is to package up the towers/minibatch dimension
+  of these arrays into PartitionedTensor objects.
+
+  This version of this function is for use with FisherBlocks that deal with
+  multiple uses or time-steps. One PartitionedTensor is created for each
+  use/time-step.  The FisherBlock will be responsible for concatenating
+  (or doing whatever else it wants) with the resulting lists.
+
+  Args:
+    grads_list: 3-D list of Tensors. First index is for source, second is for
+      tower, third is for use/time-step.
+
+  Returns:
+    2-D tuple of PartitionedTensors. First index is for source, second is for
+    use/time-step.
+  """
+  num_uses = len(grads_list[0][0])
+  assert all(len(grad) == num_uses for grads in grads_list for grad in grads)
+  return tuple(tuple(utils.PartitionedTensor(grad)
+                     for grad in zip(*grads)) for grads in grads_list)
+
+
 def normalize_damping(damping, num_replications):
   """Normalize damping after adjusting scale by NORMALIZE_DAMPING_POWER."""
   if NORMALIZE_DAMPING_POWER:
@@ -396,57 +477,6 @@ class InputOutputMultiMinibatch(object):
   def _outputs(self):
     return self.__outputs
 
-  def _package_minibatches(self, grads_list):
-    """Constructs PartitionedTensor for inputs, grads_list.
-
-    The purpose of this method is to package up the towers/minibatch dimension
-    of these arrays into PartitionedTensor objects.
-
-    Args:
-      grads_list: 2-D list of Tensors. First index is for source, second
-        index for tower.
-
-    Returns:
-      inputs: PartitionedTensor.
-      grads_list: Tuple of PartitionedTensors, one per source.
-    """
-    inputs = utils.PartitionedTensor(self._inputs)
-    grads_list = tuple(utils.PartitionedTensor(grads) for grads in grads_list)
-
-    return inputs, grads_list
-
-  def _package_minibatches_multi(self, grads_list):
-    """Constructs PartitionedTensors for inputs, grads_list.
-
-    The purpose of this method is to package up the towers/minibatch dimension
-    of these arrays into PartitionedTensor objects.
-
-    This version of this function is for use with FisherBlocks that deal with
-    multiple uses or time-steps. One PartitionedTensor is created for each
-    use/time-step.
-
-    Args:
-      grads_list: 3-D tuple of Tensors. First index is for source, second
-        index is for tower, third is for use/time-step.
-
-    Returns:
-      inputs: A tuple of PartitionedTensor's, one per use/time-step.
-      grads_list: 2-D tuple of PartitionedTensors. First index is for source,
-        second is for use/time-step.
-    """
-    # self._inputs is a 2-D tuple.  First index is tower/mini-batch, second is
-    # use/time-step.
-    inputs = self._inputs
-    num_uses = len(inputs[0])
-    assert all(len(input_) == num_uses for input_ in inputs)
-    assert all(len(grad) == num_uses for grads in grads_list for grad in grads)
-
-    inputs = tuple(utils.PartitionedTensor(input_) for input_ in zip(*inputs))
-    grads_list = tuple(tuple(utils.PartitionedTensor(grad)
-                             for grad in zip(*grads)) for grads in grads_list)
-
-    return inputs, grads_list
-
 
 class FullyConnectedDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
   """FisherBlock for fully-connected (dense) layers using a diagonal approx.
@@ -485,7 +515,8 @@ class FullyConnectedDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
     super(FullyConnectedDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    inputs, grads_list = self._package_minibatches(grads_list)
+    inputs = _make_partitionedtensors_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
 
     self._factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedDiagonalFactor,
@@ -598,7 +629,8 @@ class ConvDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
     super(ConvDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    inputs, grads_list = self._package_minibatches(grads_list)
+    inputs = _make_partitionedtensors_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
 
     # Infer number of locations upon which convolution is applied.
     self._num_locations = num_conv_locations(inputs.shape.as_list(),
@@ -711,7 +743,7 @@ class KroneckerProductFB(FisherBlock):
 class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
   """K-FAC FisherBlock for embedding layers.
 
-  This FisherBlock is similar to EmbeddingKFACFB, except that its
+  This FisherBlock is similar to FullyConnectedKFACBasicFB, except that its
   input factor is approximated by a diagonal matrix. In the case that each
   example references exactly one embedding, this approximation is exact.
 
@@ -740,17 +772,78 @@ class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
       damping: 0-D Tensor or float. 'damping' * identity is approximately added
         to this FisherBlock's Fisher approximation.
     """
-    inputs, grads_list = self._package_minibatches(grads_list)
+    inputs = _make_partitionedtensors_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
 
-    self._input_factor = self._layer_collection.make_or_get_factor(  #
-        fisher_factors.EmbeddingInputKroneckerFactor,  #
+    self._input_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.EmbeddingInputKroneckerFactor,
         (inputs, self._vocab_size))
-    self._output_factor = self._layer_collection.make_or_get_factor(  #
-        fisher_factors.FullyConnectedKroneckerFactor,  #
-        (grads_list,))
+    self._output_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.FullyConnectedKroneckerFactor, (grads_list,))
     self._setup_damping(damping)
 
 
+class EmbeddingKFACMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
+  """K-FAC FisherBlock for embedding layers used multiple times in the graph.
+
+  Similar to EmbeddingKFACFB except that this version supports multiple uses
+  of the parameter within a single model. These uses could correspond to
+  "time-steps", but they don't have to.
+
+  Does not support bias parameters.
+  """
+
+  def __init__(self, layer_collection, vocab_size):
+    """Creates a EmbeddingKFACMultiIndepFB block.
+
+    Args:
+      layer_collection: The collection of all layers in the K-FAC approximate
+          Fisher information matrix to which this FisherBlock belongs.
+      vocab_size: int. Size of vocabulary for this embedding layer.
+    """
+    self._vocab_size = vocab_size
+
+    super(EmbeddingKFACMultiIndepFB, self).__init__(layer_collection)
+
+  def instantiate_factors(self, grads_list, damping):
+    """Instantiate Kronecker Factors for this FisherBlock.
+
+    Args:
+      grads_list: List of list of list of Tensors. grads_list[i][j][k] is the
+        gradient of the loss with respect to 'outputs' from source 'i',
+        tower/mini-batch 'j', and use/time-step 'k'. Each Tensor has shape
+        [tower_minibatch_size, output_size].
+      damping: 0-D Tensor or float. 'damping' * identity is approximately added
+        to this FisherBlock's Fisher approximation.
+    """
+    inputs = self._inputs
+    self._num_uses = num_uses = len(inputs[0])
+
+    # Check that all mini-batches/towers have the same number of uses
+    assert all(len(input_) == num_uses for input_ in inputs)
+    # Do the same for grads_list
+    assert all(len(grad) == num_uses for grad in grads for grads in grads_list)
+    # Merge uses and towers/minibatches dimensions together so we can handle
+    # it using a non-multi factor.
+    inputs = nest.flatten(inputs)
+
+    # Note that we call the multi version of make_partitionedtensors only for
+    # grads_list here.
+    inputs = _make_partitionedtensors_inputs(inputs)
+    grads_list = _make_partitionedtensors_multi_grads(grads_list)
+
+    self._input_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.EmbeddingInputKroneckerFactor,
+        (inputs, self._vocab_size))
+    self._output_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.FullyConnectedMultiKF, (grads_list,))
+    self._setup_damping(damping, normalization=num_uses)
+
+  @property
+  def _renorm_coeff(self):
+    return self._num_uses
+
+
 class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
   """K-FAC FisherBlock for fully-connected (dense) layers.
 
@@ -781,13 +874,14 @@ class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
       damping: 0-D Tensor or float. 'damping' * identity is approximately added
         to this FisherBlock's Fisher approximation.
     """
-    inputs, grads_list = self._package_minibatches(grads_list)
+    inputs = _make_partitionedtensors_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
 
-    self._input_factor = self._layer_collection.make_or_get_factor(  #
-        fisher_factors.FullyConnectedKroneckerFactor,  #
+    self._input_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.FullyConnectedKroneckerFactor,
         ((inputs,), self._has_bias))
-    self._output_factor = self._layer_collection.make_or_get_factor(  #
-        fisher_factors.FullyConnectedKroneckerFactor,  #
+    self._output_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.FullyConnectedKroneckerFactor,
         (grads_list,))
     self._setup_damping(damping)
 
@@ -858,12 +952,13 @@ class ConvKFCBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
     super(ConvKFCBasicFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    inputs, grads_list = self._package_minibatches(grads_list)
-
     # Infer number of locations upon which convolution is applied.
     self._num_locations = num_conv_locations(self._inputs[0].shape.as_list(),
                                              self._strides)
 
+    inputs = _make_partitionedtensors_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
+
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvInputKroneckerFactor,
         (inputs, self._filter_shape, self._padding, self._strides,
@@ -1139,6 +1234,10 @@ def num_conv_locations(input_shape, strides):
 
 class FullyConnectedMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
   """FisherBlock for fully-connected layers that share parameters.
+
+  This class implements the "independence across time" approximation from the
+  following paper:
+    https://openreview.net/pdf?id=HyMTkQZAb
   """
 
   def __init__(self, layer_collection, has_bias=False):
@@ -1156,7 +1255,8 @@ class FullyConnectedMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
   def instantiate_factors(self, grads_list, damping):
 
     self._num_uses = float(len(self._inputs[0]))
-    inputs, grads_list = self._package_minibatches_multi(grads_list)
+    inputs = _make_partitionedtensors_multi_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_multi_grads(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedMultiKF,
@@ -1175,6 +1275,92 @@ class FullyConnectedMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
     return self._outputs
 
 
+class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
+  """FisherBlock for 2D convolutional layers using the basic KFC approx.
+
+  Similar to ConvKFCBasicFB except that this version supports multiple
+  uses/time-steps via a standard independence approximation.  Similar to the
+  "independence across time" used in FullyConnectedMultiIndepFB but generalized
+  in the obvious way to conv layers.
+  """
+
+  def __init__(self,
+               layer_collection,
+               params,
+               padding,
+               strides=None,
+               dilation_rate=None,
+               data_format=None,
+               extract_patches_fn=None):
+    """Creates a ConvKFCBasicMultiIndepFB block.
+
+    Args:
+      layer_collection: The collection of all layers in the K-FAC approximate
+          Fisher information matrix to which this FisherBlock belongs.
+      params: The parameters (Tensor or tuple of Tensors) of this layer. If
+        kernel alone, a Tensor of shape [..spatial_filter_shape..,
+        in_channels, out_channels]. If kernel and bias, a tuple of 2 elements
+        containing the previous and a Tensor of shape [out_channels].
+      padding: str. Padding method.
+      strides: List of ints or None. Contains [..spatial_filter_strides..] if
+        'extract_patches_fn' is compatible with tf.nn.convolution(), else
+        [1, ..spatial_filter_strides, 1].
+      dilation_rate: List of ints or None. Rate for dilation along each spatial
+        dimension if 'extract_patches_fn' is compatible with
+        tf.nn.convolution(), else [1, ..spatial_dilation_rates.., 1].
+      data_format: str or None. Format of input data.
+      extract_patches_fn: str or None. Name of function that extracts image
+        patches. One of "extract_convolution_patches", "extract_image_patches",
+        "extract_pointwise_conv2d_patches".
+    """
+    self._padding = padding
+    self._strides = maybe_tuple(strides)
+    self._dilation_rate = maybe_tuple(dilation_rate)
+    self._data_format = data_format
+    self._extract_patches_fn = extract_patches_fn
+    self._has_bias = isinstance(params, (tuple, list))
+
+    fltr = params[0] if self._has_bias else params
+    self._filter_shape = tuple(fltr.shape.as_list())
+
+    super(ConvKFCBasicMultiIndepFB, self).__init__(layer_collection)
+
+  def instantiate_factors(self, grads_list, damping):
+    # Infer number of locations upon which convolution is applied.
+    self._num_locations = num_locations = num_conv_locations(
+        self._inputs[0][0].shape.as_list(), self._strides)
+
+    # The first index is tower/minibatch, the second is use/time-step
+    inputs = self._inputs
+    self._num_uses = num_uses = len(inputs[0])
+
+    # Check that all mini-batches/towers have the same number of uses
+    assert all(len(input_) == num_uses for input_ in inputs)
+    assert all(len(grad) == num_uses for grads in grads_list for grad in grads)
+
+    # Fold uses/time-step and towers/minibatches dimensions together
+    inputs = nest.flatten(inputs)
+    # And do the same for grads_list
+    grads_list = tuple(nest.flatten(grads) for grads in grads_list)
+
+    inputs = _make_partitionedtensors_inputs(inputs)
+    grads_list = _make_partitionedtensors_grads(grads_list)
+
+    self._input_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.ConvInputKroneckerFactor,
+        (inputs, self._filter_shape, self._padding, self._strides,
+         self._dilation_rate, self._data_format, self._extract_patches_fn,
+         self._has_bias))
+    self._output_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.ConvOutputKroneckerFactor, (grads_list,))
+
+    self._setup_damping(damping, normalization=(num_locations * num_uses))
+
+  @property
+  def _renorm_coeff(self):
+    return self._num_locations * self._num_uses
+
+
 class SeriesFBApproximation(enum.IntEnum):
   """See FullyConnectedSeriesFB.__init__ for description and usage."""
   option1 = 1
@@ -1184,7 +1370,8 @@ class SeriesFBApproximation(enum.IntEnum):
 class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
   """FisherBlock for fully-connected layers that share parameters across time.
 
-  See the following preprint for details:
+  This class implements the "Option 1" and "Option 2" approximation from the
+  following paper:
     https://openreview.net/pdf?id=HyMTkQZAb
 
   See the end of the appendix of the paper for a pseudo-code of the
@@ -1218,7 +1405,10 @@ class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
   def instantiate_factors(self, grads_list, damping):
 
     self._num_timesteps = len(self._inputs[0])
-    inputs, grads_list = self._package_minibatches_multi(grads_list)
+    assert len(grads_list[0][0]) == self._num_timesteps
+
+    inputs = _make_partitionedtensors_multi_inputs(self._inputs)
+    grads_list = _make_partitionedtensors_multi_grads(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedMultiKF, ((inputs,), self._has_bias))
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 4eb5e4c092..00eae8b399 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -60,6 +60,10 @@ _CONV2D_APPROX_TO_BLOCK_TYPES = {
     APPROX_DIAGONAL_NAME: fb.ConvDiagonalFB,
 }
 
+_EMBEDDING_APPROX_TO_BLOCK_TYPES = {
+    APPROX_KRONECKER_NAME: fb.EmbeddingKFACFB
+}
+
 APPROX_KRONECKER_INDEP_NAME = "kron_indep"
 APPROX_KRONECKER_SERIES_1_NAME = "kron_series_1"
 APPROX_KRONECKER_SERIES_2_NAME = "kron_series_2"
@@ -72,6 +76,14 @@ _FULLY_CONNECTED_MULTI_APPROX_TO_BLOCK_TYPES = {
                                             option=2)
 }
 
+_CONV2D_MULTI_APPROX_TO_BLOCK_TYPES = {
+    APPROX_KRONECKER_INDEP_NAME: fb.ConvKFCBasicMultiIndepFB
+}
+
+_EMBEDDING_MULTI_APPROX_TO_BLOCK_TYPES = {
+    APPROX_KRONECKER_INDEP_NAME: fb.EmbeddingKFACMultiIndepFB
+}
+
 # Possible value for 'reuse' keyword argument. Sets 'reuse' to
 # tf.get_variable_scope().reuse.
 VARIABLE_SCOPE = "VARIABLE_SCOPE"
@@ -169,9 +181,12 @@ class LayerCollection(object):
     self._default_generic_approximation = APPROX_FULL_NAME
     self._default_embedding_approximation = APPROX_KRONECKER_NAME
     self._default_fully_connected_approximation = APPROX_KRONECKER_NAME
-    self._default_convolution_2d_approximation = APPROX_KRONECKER_NAME
+    self._default_conv2d_approximation = APPROX_KRONECKER_NAME
     self._default_fully_connected_multi_approximation = (
-        APPROX_KRONECKER_SERIES_2_NAME)
+        APPROX_KRONECKER_INDEP_NAME)
+    self._default_conv2d_multi_approximation = (
+        APPROX_KRONECKER_INDEP_NAME)
+    self._default_embedding_multi_approximation = APPROX_KRONECKER_INDEP_NAME
     self.loss_colocation_ops = {}
     self._vars_to_uses = defaultdict(lambda: 0)
 
@@ -245,14 +260,14 @@ class LayerCollection(object):
 
   @property
   def default_conv2d_approximation(self):
-    return self._default_convolution_2d_approximation
+    return self._default_conv2d_approximation
 
   def set_default_conv2d_approximation(self, value):
     if value not in _CONV2D_APPROX_TO_BLOCK_TYPES:
       raise ValueError(
           "{} is not a valid approximation for 2d convolutional layers.".format(
               value))
-    self._default_convolution_2d_approximation = value
+    self._default_conv2d_approximation = value
 
   @property
   def default_fully_connected_multi_approximation(self):
@@ -264,6 +279,14 @@ class LayerCollection(object):
                        "multi layer.".format(value))
     self._default_fully_connected_multi_approximation = value
 
+  @property
+  def default_conv2d_multi_approximation(self):
+    return self._default_conv2d_multi_approximation
+
+  @property
+  def default_embedding_multi_approximation(self):
+    return self._default_embedding_multi_approximation
+
   def register_block(self, layer_key, fisher_block, reuse=VARIABLE_SCOPE):
     """Validates and registers the layer_key associated with the fisher_block.
 
@@ -526,13 +549,24 @@ class LayerCollection(object):
     else:
       return None
 
+  def _get_block_type(self, params, approx, default, approx_to_type):
+    if approx is None:
+      approx = self._get_linked_approx(params)
+      if approx is None:
+        approx = default
+
+    if approx not in approx_to_type:
+      raise ValueError("Bad value {} for approx.".format(approx))
+
+    return approx_to_type[approx], approx
+
   def register_embedding(self,
                          params,
                          inputs,
                          outputs,
                          approx=None,
                          reuse=VARIABLE_SCOPE):
-    """Registers a fully connnected layer.
+    """Registers an embedding layer.
 
     Args:
       params: Embedding matrix of shape [vocab_size, embedding_size].
@@ -540,7 +574,8 @@ class LayerCollection(object):
         into embedding matrix.
       outputs: Tensor of shape [batch_size, output_size]. Outputs
         produced by layer.
-      approx: str. Must be "kron".
+      approx: str or None. If not None must be "kron".  The Fisher
+        approximation to use. If None the default value is used. (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -550,20 +585,15 @@ class LayerCollection(object):
       KeyError: If reuse == True but no FisherBlock found for 'params'.
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
-    if approx is None:
-      approx = self._get_linked_approx(params)
-      if approx is None:
-        approx = self.default_embedding_approximation
-
-    if approx != APPROX_KRONECKER_NAME:
-      raise ValueError("Bad value {} for approx.".format(approx))
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_embedding_approximation,
+        _EMBEDDING_APPROX_TO_BLOCK_TYPES)
 
     if isinstance(params, (tuple, list)):
       raise ValueError("Bias not supported.")
-
     vocab_size = int(params.shape[0])
     block = self.register_block(
-        params, fb.EmbeddingKFACFB(self, vocab_size), reuse=reuse)
+        params, block_type(self, vocab_size), reuse=reuse)
     block.register_additional_minibatch(inputs, outputs)
 
     self._add_uses(params, 1)
@@ -583,7 +613,9 @@ class LayerCollection(object):
       inputs: Tensor of shape [batch_size, input_size]. Inputs to layer.
       outputs: Tensor of shape [batch_size, output_size]. Outputs
         produced by layer.
-      approx: str. One of "kron" or "diagonal".
+      approx: str or None. If not None must be one of "kron" or "diagonal".
+        The Fisher approximation to use. If None the default value is used.
+        (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -593,17 +625,12 @@ class LayerCollection(object):
       KeyError: If reuse == True but no FisherBlock found for 'params'.
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
-    if approx is None:
-      approx = self._get_linked_approx(params)
-      if approx is None:
-        approx = self.default_fully_connected_approximation
 
-    if approx not in _FULLY_CONNECTED_APPROX_TO_BLOCK_TYPES:
-      raise ValueError("Bad value {} for approx.".format(approx))
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_fully_connected_approximation,
+        _FULLY_CONNECTED_APPROX_TO_BLOCK_TYPES)
 
-    block_type = _FULLY_CONNECTED_APPROX_TO_BLOCK_TYPES[approx]
     has_bias = isinstance(params, (tuple, list))
-
     block = self.register_block(params, block_type(self, has_bias=has_bias),
                                 reuse=reuse)
     block.register_additional_minibatch(inputs, outputs)
@@ -635,7 +662,9 @@ class LayerCollection(object):
         Output produced by layer.
       data_format: str or None. Format of data.
       dilations: List of 4 ints. Dilations along each dimension.
-      approx: str. One of "kron" or "diagonal".
+      approx: str or None. If not None must be one of "kron" or "diagonal".
+        The Fisher approximation to use. If None the default value is used.
+        (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -646,15 +675,14 @@ class LayerCollection(object):
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
 
-    if approx is None:
-      approx = self._get_linked_approx(params)
-      if approx is None:
-        approx = self.default_conv2d_approximation
-
-    if approx not in _CONV2D_APPROX_TO_BLOCK_TYPES:
-      raise ValueError("Bad value {} for approx.".format(approx))
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_conv2d_approximation,
+        _CONV2D_APPROX_TO_BLOCK_TYPES)
 
-    block_type = _CONV2D_APPROX_TO_BLOCK_TYPES[approx]
+    # It feels bad to pass in configuration that has to do with the internal
+    # implementation.  And then we can't use the same constructor for both
+    # anymore and are thus forced to use this ugly if-statement.
+    # TODO(b/74793309): Clean this up?
     if approx == APPROX_KRONECKER_NAME:
       block = self.register_block(
           params,
@@ -680,7 +708,7 @@ class LayerCollection(object):
               data_format=data_format),
           reuse=reuse)
     else:
-      raise NotImplementedError
+      raise NotImplementedError(approx)
 
     block.register_additional_minibatch(inputs, outputs)
 
@@ -712,7 +740,9 @@ class LayerCollection(object):
       dilation_rate: List of ints of length len(..input_spatial_size..).
         Dilations along spatial dimension.
       data_format: str or None. Format of data.
-      approx: str. One of "kron" or "diagonal".
+      approx: str or None. If not None must be one of "kron" or "diagonal".
+        The Fisher approximation to use. If None the default value is used.
+        (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -722,6 +752,8 @@ class LayerCollection(object):
       KeyError: If reuse == True but no FisherBlock found for 'params'.
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
+    # TODO(b/74793309): Have this use _get_block_type like the other
+    # registration functions?
     assert approx is None or approx == APPROX_KRONECKER_NAME
 
     block = self.register_block(
@@ -762,7 +794,8 @@ class LayerCollection(object):
       rate: None or List of ints of length 2. Dilation rates in spatial
         dimensions.
       data_format: str or None. Format of data.
-      approx: None or str. Must be "diagonal" if non-None.
+      approx: str or None. If not None must "diagonal".  The Fisher
+        approximation to use. If None the default value is used. (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -772,6 +805,8 @@ class LayerCollection(object):
       KeyError: If reuse == True but no FisherBlock found for 'params'.
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
+    # TODO(b/74793309): Have this use _get_block_type like the other
+    # registration functions?
     assert approx is None or approx == APPROX_DIAGONAL_NAME
     assert data_format in [None, "NHWC"]
 
@@ -803,7 +838,7 @@ class LayerCollection(object):
                                 reuse=VARIABLE_SCOPE):
     """Register a call to tf.nn.separable_conv2d().
 
-    Note: This requires access to intermediate outputs betwee depthwise and
+    Note: This requires access to intermediate outputs between depthwise and
     pointwise convolutions.
 
     Args:
@@ -824,7 +859,9 @@ class LayerCollection(object):
       rate: None or List of ints of length 2. Dilation rate of depthwise conv2d
         kernel in spatial dimensions.
       data_format: str or None. Format of data.
-      approx: None or str. Must be "kron" if non-None.
+      approx: str or None. If not None must be one of "kron" or "diagonal".
+        The Fisher approximation to use. If None the default value is used.
+        (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -865,7 +902,9 @@ class LayerCollection(object):
     Args:
       params: Tensor or tuple of Tensors corresponding to the parameters.
       batch_size: 0-D Tensor. Size of the minibatch.
-      approx: str. One of "full" or "diagonal".
+      approx: str or None. It not None, must be one of "full" or "diagonal".
+        The Fisher approximation to use. If None the default value is used.
+        (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -875,16 +914,10 @@ class LayerCollection(object):
       KeyError: If reuse == True but no FisherBlock found for 'params'.
       ValueError: If reuse == True and FisherBlock found but of the wrong type.
     """
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_generic_approximation,
+        _GENERIC_APPROX_TO_BLOCK_TYPES)
 
-    if approx is None:
-      approx = self._get_linked_approx(params)
-      if approx is None:
-        approx = self.default_generic_approximation
-
-    if approx not in _GENERIC_APPROX_TO_BLOCK_TYPES:
-      raise ValueError("Bad value {} for approx.".format(approx))
-
-    block_type = _GENERIC_APPROX_TO_BLOCK_TYPES[approx]
     block = self.register_block(params, block_type(self, params), reuse=reuse)
     block.register_additional_minibatch(batch_size)
 
@@ -903,11 +936,15 @@ class LayerCollection(object):
         this layer. Weight matrix should have shape [input_size, output_size].
         Bias should have shape [output_size].
       inputs: A list of tensors, each of shape [batch_size, input_size]. Inputs
-        to layer. In the case of RNNs, one Tensor per time step.
+        to layer. The list indexes each use in the graph (which might
+        correspond to a "time-step" in an RNN).
       outputs: A list of tensors, the same length as 'inputs', each of shape
-        [batch_size, output_size]. Outputs produced by layer. In the case of
-        RNNs, one Tensor per time step.
-      approx: str. One of "kron_indep", "kron_series_1", or "kron_series_2".
+        [batch_size, output_size]. Outputs produced by layer. The list indexes
+        each use in the graph (which might correspond to a "time-step" in an
+        RNN). Needs to correspond with the order used in 'inputs'.
+      approx: str or None. If not None, must be of "kron_indep", "kron_series_1"
+        or "kron_series_2". The Fisher approximation to use. If None the default
+        value is used. (Default: None)
       reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
         create a new FisherBlock.  If "VARIABLE_SCOPE", use
         tf.get_variable_scope().reuse.
@@ -915,28 +952,129 @@ class LayerCollection(object):
     Raises:
       ValueError: For improper value to 'approx'.
     """
-    if approx is None:
-      approx = self._get_linked_approx(params)
-      if approx is None:
-        approx = self.default_fully_connected_multi_approximation
-    has_bias = isinstance(params, (tuple, list))
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_fully_connected_multi_approximation,
+        _FULLY_CONNECTED_MULTI_APPROX_TO_BLOCK_TYPES)
 
     # TODO(b/70283649): something along the lines of find_canonical_output
     # should be added back in here (and for the other block types, arguably).
 
-    if approx not in _FULLY_CONNECTED_MULTI_APPROX_TO_BLOCK_TYPES:
-      raise ValueError("Bad value {} for approx.".format(approx))
-    block_type = _FULLY_CONNECTED_MULTI_APPROX_TO_BLOCK_TYPES[approx]
-
+    has_bias = isinstance(params, (tuple, list))
     block = self.register_block(params, block_type(self, has_bias=has_bias),
                                 reuse=reuse)
     block.register_additional_minibatch(inputs, outputs)
+
+    assert len(inputs) == len(outputs)
+    self._add_uses(params, len(inputs))
+
+  def register_conv2d_multi(self,
+                            params,
+                            strides,
+                            padding,
+                            inputs,
+                            outputs,
+                            data_format=None,
+                            dilations=None,
+                            approx=None,
+                            reuse=VARIABLE_SCOPE):
+    """Registers convolutional layers with shared parameters.
+
+    Args:
+      params: Tensor or 2-tuple of Tensors corresponding to weight and bias of
+        this layer. Weight matrix should have shape [kernel_height,
+        kernel_width, in_channels, out_channels].  Bias should have shape
+        [out_channels].
+      strides: 1-D Tensor of length 4. Strides for convolution kernel.
+      padding: string. see tf.nn.conv2d for valid values.
+      inputs: A list of Tensors, each of shape [batch_size, height, width,
+        in_channels]. Inputs to layer. The list indexes each use in the graph
+        (which might correspond to a "time-step" in an RNN).
+      outputs: A list of Tensors, each of shape [batch_size, height, width,
+        out_channels]. Output produced by layer. The list indexes each use
+        in the graph (which might correspond to a "time-step" in an RNN).
+        Needs to correspond with the order used in 'inputs'.
+      data_format: str or None. Format of data.
+      dilations: List of 4 ints. Dilations along each dimension.
+      approx: str or None. If not None must by "kron_indep". The Fisher
+        approximation to use. If None the default value is used.
+        (Default: None)
+      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
+        create a new FisherBlock.  If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse.
+
+    Raises:
+      ValueError: For improper value to 'approx'.
+      KeyError: If reuse == True but no FisherBlock found for 'params'.
+      ValueError: If reuse == True and FisherBlock found but of the wrong type.
+    """
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_conv2d_multi_approximation,
+        _CONV2D_MULTI_APPROX_TO_BLOCK_TYPES)
+
+    block = self.register_block(
+        params,
+        block_type(
+            layer_collection=self,
+            params=params,
+            padding=padding,
+            strides=strides,
+            data_format=data_format,
+            dilation_rate=dilations,
+            extract_patches_fn="extract_image_patches"),
+        reuse=reuse)
+
+    block.register_additional_minibatch(inputs, outputs)
+
+    assert len(inputs) == len(outputs)
     self._add_uses(params, len(inputs))
 
   # TODO(b/74108452): change the loss registration functions names to refer
   # to "loss functions" instead of distributions.  Following naming convention
   # of the loss function classes themselves.
 
+  def register_embedding_multi(self,
+                               params,
+                               inputs,
+                               outputs,
+                               approx=None,
+                               reuse=VARIABLE_SCOPE):
+    """Registers embedding layers with shared parameters.
+
+    Args:
+      params: Embedding matrix of shape [vocab_size, embedding_size].
+      inputs: A list of Tensors, each of shape [batch_size, input_size] and
+        dtype int32. Indices into embedding matrix. The list indexes each use
+        in the graph (which might correspond to a "time-step" in an RNN).
+      outputs: A list of Tensors, each of shape [batch_size, output_size].
+        Outputs produced by layer. The list indexes each use in the graph
+        (which might correspond to a "time-step" in an RNN). Needs to
+        correspond with the order used in 'inputs'.
+      approx: str or None. If not None must by "kron_indep". The Fisher
+        approximation to use. If None the default value is used.
+        (Default: None)
+      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
+        create a new FisherBlock.  If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse.
+
+    Raises:
+      ValueError: For improper value to 'approx'.
+      KeyError: If reuse == True but no FisherBlock found for 'params'.
+      ValueError: If reuse == True and FisherBlock found but of the wrong type.
+    """
+    block_type, approx = self._get_block_type(
+        params, approx, self.default_embedding_multi_approximation,
+        _EMBEDDING_MULTI_APPROX_TO_BLOCK_TYPES)
+
+    if isinstance(params, (tuple, list)):
+      raise ValueError("Bias not supported.")
+    vocab_size = int(params.shape[0])
+
+    block = self.register_block(
+        params, block_type(self, vocab_size), reuse=reuse)
+    block.register_additional_minibatch(inputs, outputs)
+
+    self._add_uses(params, len(inputs))
+
   def register_categorical_predictive_distribution(self,
                                                    logits,
                                                    seed=None,
diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py
index af26f5e56b..c589b18193 100644
--- a/tensorflow/contrib/kfac/python/ops/utils.py
+++ b/tensorflow/contrib/kfac/python/ops/utils.py
@@ -659,6 +659,14 @@ class PartitionedTensor(object):
   def __hash__(self):
     return hash(tuple(self.tensors))
 
+  def __eq__(self, other):
+    if not isinstance(other, PartitionedTensor):
+      return False
+    return self.tensors == other.tensors
+
+  def __ne__(self, other):
+    return not self == other  # pylint: disable=g-comparison-negation
+
   def as_tensor(self, dtype=None, name=None, as_ref=False):
     with ops.name_scope(name, "PartitionedTensor.as_tensor", self.tensors):
       assert not as_ref
-- 
GitLab


From 09d3d2dcc1b1b9ee7282b37bc4e0f212c577f6a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 08:59:02 -0700
Subject: [PATCH 102/960] Fix a typo in docstring for index_table_from_tensor.

PiperOrigin-RevId: 189345585
---
 tensorflow/contrib/lookup/lookup_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index 62f1c810fc..a57a1e5421 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -105,7 +105,7 @@ def index_table_from_tensor(mapping,
   ...
   tf.tables_initializer().run()
 
-  ids.eval()  ==> [0, 1, 4, 2]
+  ids.eval()  ==> [0, 1, 3, 2]
   ```
 
   Args:
-- 
GitLab


From 4b5cb6c49934b6c62fcfc4bf710a30dcce2568d3 Mon Sep 17 00:00:00 2001
From: Frank Perbet <fkp@google.com>
Date: Fri, 16 Mar 2018 09:02:48 -0700
Subject: [PATCH 103/960] Make the graph_editor C-API friendly: always
 construct ops with their inputs.

PiperOrigin-RevId: 189346024
---
 .../graph_editor/tests/transform_test.py      | 32 +++++++--
 tensorflow/contrib/graph_editor/transform.py  | 66 +++++++++++++------
 2 files changed, 74 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py
index ca00394388..2a1b78042d 100644
--- a/tensorflow/contrib/graph_editor/tests/transform_test.py
+++ b/tensorflow/contrib/graph_editor/tests/transform_test.py
@@ -23,6 +23,7 @@ from tensorflow.contrib import graph_editor as ge
 from tensorflow.contrib.graph_editor.tests import match
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -84,9 +85,9 @@ class TransformTest(test.TestCase):
   def test_transform(self):
     transformer = ge.Transformer()
 
-    def my_transform_op_handler(info, op):
+    def my_transform_op_handler(info, op, new_inputs):
       add_noise = op.name.startswith("Add")
-      op_, op_outputs_ = ge.transform.copy_op_handler(info, op)
+      op_, op_outputs_ = ge.transform.copy_op_handler(info, op, new_inputs)
       if not add_noise:
         return op_, op_outputs_
       # add some noise to op
@@ -201,15 +202,36 @@ class TransformTest(test.TestCase):
                         get_operation_by_name("res/grad/mul1_grad/Mul_1"))
 
     # Make sure _original_ops are as expected.
-    self.assertEquals(original_mul1_grad._original_op.name, u"mul1")
-    self.assertEquals(result_mul1_grad._original_op.name, u"res/mul1")
-    self.assertNotEquals(res.name, g.name)
+    self.assertEqual(original_mul1_grad._original_op.name, u"mul1")
+    self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1")
+    self.assertNotEqual(res.name, g.name)
     with session.Session() as sess:
       sess.run(variables.global_variables_initializer())
       g_val, res_val = sess.run([g, res])
     self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
     self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
 
+  def test_graph_while_loop(self):
+    graph = ops.Graph()
+    with graph.as_default():
+      max_index = array_ops.placeholder(dtype=dtypes.int32, shape=tuple())
+      index_start = constant_op.constant(1)
+      sum_start = constant_op.constant(0)
+      _, result = control_flow_ops.while_loop(
+          cond=lambda i, unused_s: i <= max_index,
+          body=lambda i, s: (i + 1, s + i),
+          loop_vars=[index_start, sum_start])
+    copied_graph = ops.Graph()
+    _, copy_info = ge.copy(
+        graph, dst_graph=copied_graph, dst_scope="imported")
+    copied_result = copy_info.transformed(result)
+    copied_max_index = copy_info.transformed(max_index)
+    with copied_graph.as_default():
+      with session.Session() as sess:
+        n = 10
+        sum_val = sess.run(copied_result, feed_dict={copied_max_index: n})
+        self.assertEqual(sum_val, 55)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py
index 14ac529665..03c9afe813 100644
--- a/tensorflow/contrib/graph_editor/transform.py
+++ b/tensorflow/contrib/graph_editor/transform.py
@@ -30,6 +30,7 @@ from tensorflow.contrib.graph_editor import select
 from tensorflow.contrib.graph_editor import subgraph
 from tensorflow.contrib.graph_editor import util
 from tensorflow.python.framework import ops as tf_ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
 
 
@@ -129,20 +130,26 @@ def transform_op_if_inside_handler(info, op, keep_if_possible=True):
       return None
 
 
-def copy_op_handler(info, op, copy_shape=True):
+def copy_op_handler(info, op, new_inputs, copy_shape=True):
   """Copy a `tf.Operation`.
 
   Args:
     info: Transform._TmpInfo instance.
     op: the `tf.Operation` to be copied.
+    new_inputs: The new inputs for this op.
     copy_shape: also copy the shape of the tensor
   Returns:
     A `(op, op_outputs)` tuple containing the transformed op and its outputs.
   """
+  # The `new_inputs` was added to this function. For compatibility reason,
+  # let's raise an error if `new_inputs` is a boolean.
+  if isinstance(new_inputs, bool):
+    raise TypeError("the `new_inputs` argument must be an iterable.")
+
   # pylint: disable=protected-access
 
   # Clone the node def:
-  node_def_ = deepcopy(op._node_def)
+  node_def_ = deepcopy(op.node_def)
 
   # Transform name:
   name_ = info.new_name(op.name)
@@ -155,10 +162,10 @@ def copy_op_handler(info, op, copy_shape=True):
 
   # Make a copy of the op_def too.
   # Its unique to every _type_ of Operation.
-  op_def_ = deepcopy(op._op_def)
+  op_def_ = deepcopy(op.op_def)
 
   # Initialize a new Operation instance
-  op_ = tf_ops.Operation(node_def_, info.graph_, [], output_types_,
+  op_ = tf_ops.Operation(node_def_, info.graph_, new_inputs, output_types_,
                          [], input_types_, None, op_def_)
 
   # copy the shape over
@@ -170,6 +177,7 @@ def copy_op_handler(info, op, copy_shape=True):
   # attribute to exist, we will create a dummy original_op first and then
   # later finalise it with the actual original_op when all the ops have
   # been copied.
+  # TODO(fkp): Stop worrying about _original_op and remove this code?
   if op._original_op:
     op_._original_op = op._original_op
 
@@ -328,6 +336,14 @@ class _TmpInfo(object):
                             for key in self.graph.get_all_collection_keys())
     self.cyclic_ops = []
     self.transform_original_op_handler = transform_op_if_inside_handler
+    # The graph is transformed op by op, in the same order the original ops
+    # where created. However, this is sometimes not possible due to cycles
+    # (e.g. while loops). So when the transformer creates a new op whose
+    # inputs do not exist yet, temporary placeholders are created and stored
+    # in this `tmp_cyclic_ts` container. During a second pass,
+    # those temporary tensors are replaced by the proper transformed tensors
+    # (see the function `_finalize_cycle`).
+    self.tmp_cyclic_ts = []
 
   def new_name(self, name):
     """Compute a destination name from a source name.
@@ -428,10 +444,10 @@ class Transformer(object):
 
     # Create temporary info used during this transform call
     info = _TmpInfo(sgv, dst_graph, dst_scope, src_scope)
-    info.transform_original_op_handler = self.transform_original_op_handler
 
     self._copy_ops(info)
-    self._connect_ops(info)
+    self._finalize_cycle(info)
+    self._connect_control_inputs(info)
 
     # Compute information about the transformation
     res_info = TransformerInfo(info)
@@ -440,10 +456,12 @@ class Transformer(object):
 
   def _copy_ops(self, info):
     """Copy ops without connecting them."""
-    for op in info.sgv.ops:
+    sorted_ops = sorted(info.sgv.ops, key=lambda op: op._id)  # pylint: disable=protected-access
+    for op in sorted_ops:
       logging.debug("Copying op: %s", op.name)
+      new_inputs = [self._transformed_t(info, t) for t in op.inputs]
       # TODO(fkp): return a subgraph?
-      op_, op_outputs_ = self.transform_op_handler(info, op)
+      op_, op_outputs_ = self.transform_op_handler(info, op, new_inputs)
       if op is op_:
         raise ValueError("In-place transformation not allowed.")
 
@@ -456,27 +474,31 @@ class Transformer(object):
         info.transformed_ts[op_output] = op_output_
         self.assign_collections_handler(info, op_output, op_output_)
 
-  def _connect_ops(self, info):
+  def _finalize_cycle(self, info):
+    for t, tmp_t_ in info.tmp_cyclic_ts:
+      if t not in info.transformed_ts:
+        raise ValueError("The tensor {} should be transformed by now.".format(
+            t.name))
+      op_ = tmp_t_.consumers()[0]
+      t_ = info.transformed_ts[t]
+      op_._update_input(list(op_.inputs).index(tmp_t_), t_)  # pylint: disable=protected-access
+
+  def _connect_control_inputs(self, info):
     """Connect the previously copied ops."""
     for op in info.sgv.ops:
-      logging.debug("Finalizing op: %s", op.name)
+      logging.debug("Connecting control inputs of op: %s", op.name)
       op_ = info.transformed_ops[op]
 
-      # pylint: disable=protected-access
-      if op_.inputs:
-        raise ValueError("The newly transformed op should not have "
-                         "any inputs yet: {}".format(op_.name))
-      inputs_ = [self._transformed_t(info, t) for t in op.inputs]
-      for t in inputs_:
-        op_._add_input(t)
-
       # Finalize original op.
+      # TODO(fkp): Stop worrying about _original_op and remove this code?
+      # pylint: disable=protected-access
       if op._original_op:
-        original_op = info.transform_original_op_handler(info, op._original_op)
+        original_op = self.transform_original_op_handler(info, op._original_op)
         if original_op is None:
           logging.debug("Could not find original op for: %s", op_.name)
         else:
           op_._original_op = original_op
+      # pylint: enable=protected-access
 
       # Finalize control inputs:
       control_inputs_ = [self.transform_control_input_handler(info, ci)
@@ -528,6 +550,12 @@ class Transformer(object):
   def _transformed_t(self, info, t):
     """Return tre transformed tensor of `t`."""
     if t not in info.transformed_ts:
+      if t.op in info.ops:
+        with info.graph_.as_default():
+          tmp_t_ = array_ops.placeholder(
+              shape=t.shape, dtype=t.dtype, name="ge_tmp")
+        info.tmp_cyclic_ts.append((t, tmp_t_))
+        return tmp_t_
       # If op is not in the subgraph.
       if t in info.sgv_inputs_set:
         # t is an input of the subgraph.
-- 
GitLab


From aef5751d17cf5d53de45b7833c26d11cd9473cde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 09:32:43 -0700
Subject: [PATCH 104/960] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 189349737
---
 tensorflow/contrib/cmake/external/grpc.cmake | 2 +-
 tensorflow/workspace.bzl                     | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index a9f43a3ecb..95106dba1f 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 730b778632e79cc3c96ad237f282d687ee325ce7)
+set(GRPC_TAG 575bda39755b98d1f7099406bb57a6e3b2074874)
 
 if(WIN32)
   if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index abc9eb9bc1..8d739158c5 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -452,11 +452,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "grpc",
       urls = [
-          "https://mirror.bazel.build/github.com/grpc/grpc/archive/730b778632e79cc3c96ad237f282d687ee325ce7.tar.gz",
-          "https://github.com/grpc/grpc/archive/730b778632e79cc3c96ad237f282d687ee325ce7.tar.gz",
+          "https://mirror.bazel.build/github.com/grpc/grpc/archive/575bda39755b98d1f7099406bb57a6e3b2074874.tar.gz",
+          "https://github.com/grpc/grpc/archive/575bda39755b98d1f7099406bb57a6e3b2074874.tar.gz",
       ],
-      sha256 = "8c91a8d12e1e868cf51f7340b75507a8aa017a7e1b56f46ed6816aeb803dc9bd",
-      strip_prefix = "grpc-730b778632e79cc3c96ad237f282d687ee325ce7",
+      sha256 = "f08a5c8e265191b39cc74915b1bc1fd380d86cd0176c92b7cce30b6ac50514ad",
+      strip_prefix = "grpc-575bda39755b98d1f7099406bb57a6e3b2074874",
   )
 
   tf_http_archive(
-- 
GitLab


From 14009941a5dbe9f58c0866d9152a4ddd5ebc7d54 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 09:42:45 -0700
Subject: [PATCH 105/960] Doc grammar and style fixes for macOS installation.

PiperOrigin-RevId: 189351224
---
 tensorflow/docs_src/install/install_mac.md | 9 ++-------
 1 file changed, 2 insertions(+), 7 deletions(-)

diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 623ca6bb79..99745fcc6d 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -238,7 +238,7 @@ take the following steps:
      operating system and Python version. Find the appropriate
      value for <i>tfBinaryURL</i>
      [here](#the_url_of_the_tensorflow_python_package).  For example, if
-     you are installing TensorFlow for Mac OS and Python 2.7
+     you are installing TensorFlow for macOS and Python 2.7
      issue the following command:
 
      <pre> $ <b>sudo pip install --upgrade \
@@ -513,12 +513,7 @@ RuntimeError: Broken toolchain: cannot link a simple C program</pre>
 ## The URL of the TensorFlow Python package
 
 A few installation mechanisms require the URL of the TensorFlow Python package.
-The value you specify depends on three factors:
-
-  * operating system
-  * Python version
-
-This section documents the relevant values for Mac OS installations.
+The value you specify depends on your Python version.
 
 ### Python 2.7
 
-- 
GitLab


From 6dd0b7e350133e62ea74dc1fb6a502044184b071 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 09:53:20 -0700
Subject: [PATCH 106/960] Eliminate use of grpc::CoreCodegenInterface, which is
 marked as an internal interface

PiperOrigin-RevId: 189352860
---
 tensorflow/core/distributed_runtime/rpc/BUILD |  1 +
 .../rpc/grpc_serialization_traits.h           | 57 +++++++------------
 .../rpc/grpc_worker_service_impl.h            |  4 +-
 3 files changed, 23 insertions(+), 39 deletions(-)

diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index e9d5390c63..9dae1b9859 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -259,6 +259,7 @@ cc_library(
     hdrs = ["grpc_serialization_traits.h"],
     deps = [
         "@grpc//:grpc++_unsecure",
+        "@grpc//:grpc_unsecure",
     ],
 )
 
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h b/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h
index 730124c25e..e7f5fb0c6a 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_serialization_traits.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "grpc++/impl/codegen/proto_utils.h"
 #include "grpc++/support/slice.h"
+#include "grpc/grpc.h"
 
 namespace grpc {
 
@@ -30,13 +31,13 @@ class GrpcBufferWriter final
  public:
   explicit GrpcBufferWriter(grpc_byte_buffer** bp, int block_size)
       : block_size_(block_size), byte_count_(0), have_backup_(false) {
-    *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(NULL, 0);
+    *bp = grpc_raw_byte_buffer_create(NULL, 0);
     slice_buffer_ = &(*bp)->data.raw.slice_buffer;
   }
 
   ~GrpcBufferWriter() override {
     if (have_backup_) {
-      g_core_codegen_interface->grpc_slice_unref(backup_slice_);
+      grpc_slice_unref(backup_slice_);
     }
   }
 
@@ -45,24 +46,24 @@ class GrpcBufferWriter final
       slice_ = backup_slice_;
       have_backup_ = false;
     } else {
-      slice_ = g_core_codegen_interface->grpc_slice_malloc(block_size_);
+      slice_ = grpc_slice_malloc(block_size_);
     }
     *data = GRPC_SLICE_START_PTR(slice_);
     // On win x64, int is only 32bit
     GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX);
     byte_count_ += * size = (int)GRPC_SLICE_LENGTH(slice_);
-    g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_);
+    grpc_slice_buffer_add(slice_buffer_, slice_);
     return true;
   }
 
   void BackUp(int count) override {
-    g_core_codegen_interface->grpc_slice_buffer_pop(slice_buffer_);
+    grpc_slice_buffer_pop(slice_buffer_);
     if (count == block_size_) {
       backup_slice_ = slice_;
     } else {
-      backup_slice_ = g_core_codegen_interface->grpc_slice_split_tail(
-          &slice_, GRPC_SLICE_LENGTH(slice_) - count);
-      g_core_codegen_interface->grpc_slice_buffer_add(slice_buffer_, slice_);
+      backup_slice_ =
+          grpc_slice_split_tail(&slice_, GRPC_SLICE_LENGTH(slice_) - count);
+      grpc_slice_buffer_add(slice_buffer_, slice_);
     }
     // It's dangerous to keep an inlined grpc_slice as the backup slice, since
     // on a following Next() call, a reference will be returned to this slice
@@ -85,29 +86,12 @@ class GrpcBufferWriter final
 
 class GrpcBufferReader final
     : public ::grpc::protobuf::io::ZeroCopyInputStream {
-  typedef void (CoreCodegenInterface::*OldReaderInitAPI)(
-      grpc_byte_buffer_reader* reader, grpc_byte_buffer* buffer);
-  typedef int (CoreCodegenInterface::*NewReaderInitAPI)(
-      grpc_byte_buffer_reader* reader, grpc_byte_buffer* buffer);
-  void ReaderInit(OldReaderInitAPI ptr, grpc_byte_buffer_reader* reader,
-                  grpc_byte_buffer* buffer) {
-    (g_core_codegen_interface->*ptr)(reader, buffer);
-  }
-  void ReaderInit(NewReaderInitAPI ptr, grpc_byte_buffer_reader* reader,
-                  grpc_byte_buffer* buffer) {
-    int result = (g_core_codegen_interface->*ptr)(reader, buffer);
-    (void)result;
-  }
-
  public:
   explicit GrpcBufferReader(grpc_byte_buffer* buffer)
       : byte_count_(0), backup_count_(0) {
-    ReaderInit(&CoreCodegenInterface::grpc_byte_buffer_reader_init, &reader_,
-               buffer);
-  }
-  ~GrpcBufferReader() override {
-    g_core_codegen_interface->grpc_byte_buffer_reader_destroy(&reader_);
+    (void)grpc_byte_buffer_reader_init(&reader_, buffer);
   }
+  ~GrpcBufferReader() override { grpc_byte_buffer_reader_destroy(&reader_); }
 
   bool Next(const void** data, int* size) override {
     if (backup_count_ > 0) {
@@ -118,11 +102,10 @@ class GrpcBufferReader final
       backup_count_ = 0;
       return true;
     }
-    if (!g_core_codegen_interface->grpc_byte_buffer_reader_next(&reader_,
-                                                                &slice_)) {
+    if (!grpc_byte_buffer_reader_next(&reader_, &slice_)) {
       return false;
     }
-    g_core_codegen_interface->grpc_slice_unref(slice_);
+    grpc_slice_unref(slice_);
     *data = GRPC_SLICE_START_PTR(slice_);
     // On win x64, int is only 32bit
     GPR_CODEGEN_ASSERT(GRPC_SLICE_LENGTH(slice_) <= INT_MAX);
@@ -176,18 +159,18 @@ class UnlimitedSizeProtoSerializationTraits {
       return Status(StatusCode::INTERNAL, "Message length was negative");
     } else if (byte_size <=
                tensorflow_helper::kGrpcBufferWriterMaxBufferLength) {
-      grpc_slice slice = g_core_codegen_interface->grpc_slice_malloc(byte_size);
+      grpc_slice slice = grpc_slice_malloc(byte_size);
       GPR_CODEGEN_ASSERT(
           GRPC_SLICE_END_PTR(slice) ==
           msg.SerializeWithCachedSizesToArray(GRPC_SLICE_START_PTR(slice)));
-      *bp = g_core_codegen_interface->grpc_raw_byte_buffer_create(&slice, 1);
-      g_core_codegen_interface->grpc_slice_unref(slice);
-      return g_core_codegen_interface->ok();
+      *bp = grpc_raw_byte_buffer_create(&slice, 1);
+      grpc_slice_unref(slice);
+      return Status::OK;
     } else {
       tensorflow_helper::GrpcBufferWriter writer(
           bp, tensorflow_helper::kGrpcBufferWriterMaxBufferLength);
       return msg.SerializeToZeroCopyStream(&writer)
-                 ? g_core_codegen_interface->ok()
+                 ? Status::OK
                  : Status(StatusCode::INTERNAL, "Failed to serialize message");
     }
   }
@@ -197,7 +180,7 @@ class UnlimitedSizeProtoSerializationTraits {
     if (buffer == nullptr) {
       return Status(StatusCode::INTERNAL, "No payload");
     }
-    Status result = g_core_codegen_interface->ok();
+    Status result = Status::OK;
     {
       tensorflow_helper::GrpcBufferReader reader(buffer);
       ::grpc::protobuf::io::CodedInputStream decoder(&reader);
@@ -214,7 +197,7 @@ class UnlimitedSizeProtoSerializationTraits {
         result = Status(StatusCode::INTERNAL, "Did not read entire message");
       }
     }
-    g_core_codegen_interface->grpc_byte_buffer_destroy(buffer);
+    grpc_byte_buffer_destroy(buffer);
     return result;
   }
 };
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
index 1a5e2edfb2..2a2f7e3ffb 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_worker_service_impl.h
@@ -88,7 +88,7 @@ class SerializationTraits<tensorflow::TensorResponse>
     if (buffer == nullptr) {
       return Status(StatusCode::INTERNAL, "No payload");
     }
-    Status result = g_core_codegen_interface->ok();
+    Status result = Status::OK;
     if (result.ok()) {
       ::tensorflow::GrpcByteSource source(buffer);
       auto s = msg->ParseFrom(&source);
@@ -98,7 +98,7 @@ class SerializationTraits<tensorflow::TensorResponse>
                             "TensorResponse parse error", s.ToString()));
       }
     }
-    g_core_codegen_interface->grpc_byte_buffer_destroy(buffer);
+    grpc_byte_buffer_destroy(buffer);
     return result;
   }
 };
-- 
GitLab


From 6271467df2f0bf7c776f888bed3c6722502efc51 Mon Sep 17 00:00:00 2001
From: Jakub Kolodziejczyk <jakub.kolodziejczyk@mail.com>
Date: Sat, 17 Mar 2018 01:58:09 +0900
Subject: [PATCH 107/960] Updated tf.Session.run(...) documentation

Added information that order in which `fetches` are evaluated inside the call is undefined.
Discussion: https://github.com/tensorflow/tensorflow/issues/13133
---
 tensorflow/python/client/session.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 29f06c8f22..9b7e853e8c 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -889,6 +889,8 @@ class BaseSession(SessionInterface):
       Either a single value if `fetches` is a single graph element, or
       a list of values if `fetches` is a list, or a dictionary with the
       same keys as `fetches` if that is a dictionary (described above).
+      Order in which `fetches` operations are evaluated inside the call
+      is undefined.
 
     Raises:
       RuntimeError: If this `Session` is in an invalid state (e.g. has been
-- 
GitLab


From a0bd058ad1406585634330772bfda76fd27d87d7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 09:54:27 -0700
Subject: [PATCH 108/960] Move CreateSubProcess from test.h to subprocess.h

PiperOrigin-RevId: 189353033
---
 .../xla/legacy_flags/parse_flags_from_env_test.cc      |  1 +
 .../core/distributed_runtime/rpc/grpc_testlib.cc       |  2 +-
 tensorflow/core/distributed_runtime/rpc/grpc_testlib.h |  1 +
 tensorflow/core/platform/posix/subprocess.cc           | 10 ++++++++++
 tensorflow/core/platform/posix/test.cc                 |  9 ---------
 tensorflow/core/platform/subprocess.h                  |  9 +++++++++
 tensorflow/core/platform/test.h                        |  7 -------
 tensorflow/core/platform/windows/subprocess.h          | 10 ++++++++++
 tensorflow/core/platform/windows/test.cc               |  5 -----
 9 files changed, 32 insertions(+), 22 deletions(-)

diff --git a/tensorflow/compiler/xla/legacy_flags/parse_flags_from_env_test.cc b/tensorflow/compiler/xla/legacy_flags/parse_flags_from_env_test.cc
index a3b4286f4c..7b6ae311c1 100644
--- a/tensorflow/compiler/xla/legacy_flags/parse_flags_from_env_test.cc
+++ b/tensorflow/compiler/xla/legacy_flags/parse_flags_from_env_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/subprocess.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/util/command_line_flags.h"
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_testlib.cc b/tensorflow/core/distributed_runtime/rpc/grpc_testlib.cc
index c237f2dce4..89f83f9f24 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_testlib.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_testlib.cc
@@ -57,7 +57,7 @@ Status TestCluster::MakeTestCluster(const SessionOptions& options, int n,
          tf_jobs, "--tf_job=localhost", strings::StrCat("--tf_task=", i),
          strings::StrCat("--num_cpus=", num_cpus),
          strings::StrCat("--num_gpus=", num_gpus)});
-    ret->subprocesses_.emplace_back(testing::CreateSubProcess(argv));
+    ret->subprocesses_.emplace_back(CreateSubProcess(argv));
     bool success = ret->subprocesses_[i]->Start();
     if (!success) {
       return errors::Internal("Could not start subprocess");
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_testlib.h b/tensorflow/core/distributed_runtime/rpc/grpc_testlib.h
index 4b3a03b1d7..d5baaae353 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_testlib.h
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_testlib.h
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/framework/device_attributes.pb.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/macros.h"
+#include "tensorflow/core/platform/subprocess.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/session_options.h"
diff --git a/tensorflow/core/platform/posix/subprocess.cc b/tensorflow/core/platform/posix/subprocess.cc
index cefc66831a..a661c34ef0 100644
--- a/tensorflow/core/platform/posix/subprocess.cc
+++ b/tensorflow/core/platform/posix/subprocess.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include <string.h>
 #include <sys/types.h>
 #include <sys/wait.h>
+#include <memory>
+#include <vector>
 
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/subprocess.h"
@@ -461,4 +463,12 @@ int SubProcess::Communicate(const string* stdin_input, string* stdout_output,
   return WaitInternal(&status) ? status : -1;
 }
 
+std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
+  std::unique_ptr<SubProcess> proc(new SubProcess());
+  proc->SetProgram(argv[0], argv);
+  proc->SetChannelAction(CHAN_STDERR, ACTION_DUPPARENT);
+  proc->SetChannelAction(CHAN_STDOUT, ACTION_DUPPARENT);
+  return proc;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/platform/posix/test.cc b/tensorflow/core/platform/posix/test.cc
index a69127b3e8..28f7478a6d 100644
--- a/tensorflow/core/platform/posix/test.cc
+++ b/tensorflow/core/platform/posix/test.cc
@@ -20,19 +20,10 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/subprocess.h"
 
 namespace tensorflow {
 namespace testing {
 
-std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
-  std::unique_ptr<SubProcess> proc(new SubProcess());
-  proc->SetProgram(argv[0], argv);
-  proc->SetChannelAction(CHAN_STDERR, ACTION_DUPPARENT);
-  proc->SetChannelAction(CHAN_STDOUT, ACTION_DUPPARENT);
-  return proc;
-}
-
 int PickUnusedPortOrDie() { return internal::PickUnusedPortOrDie(); }
 
 string TensorFlowSrcRoot() {
diff --git a/tensorflow/core/platform/subprocess.h b/tensorflow/core/platform/subprocess.h
index dfdcf82173..dcc0c1a4ee 100644
--- a/tensorflow/core/platform/subprocess.h
+++ b/tensorflow/core/platform/subprocess.h
@@ -16,6 +16,9 @@ limitations under the License.
 #ifndef TENSORFLOW_PLATFORM_SUBPROCESS_H_
 #define TENSORFLOW_PLATFORM_SUBPROCESS_H_
 
+#include <memory>
+#include <vector>
+
 namespace tensorflow {
 
 // Channel identifiers.
@@ -43,6 +46,12 @@ enum ChannelAction {
 // Supports spawning and killing child processes.
 class SubProcess;
 
+// Returns an object that represents a child process that will be
+// launched with the given command-line arguments `argv`. The process
+// must be explicitly started by calling the Start() method on the
+// returned object.
+std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv);
+
 }  // namespace tensorflow
 
 #include "tensorflow/core/platform/platform.h"
diff --git a/tensorflow/core/platform/test.h b/tensorflow/core/platform/test.h
index 295957c3d8..99bae63edf 100644
--- a/tensorflow/core/platform/test.h
+++ b/tensorflow/core/platform/test.h
@@ -21,7 +21,6 @@ limitations under the License.
 
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/platform.h"
-#include "tensorflow/core/platform/subprocess.h"
 #include "tensorflow/core/platform/types.h"
 
 // As of September 2016, we continue to attempt to avoid the use of gmock aka
@@ -49,12 +48,6 @@ string TensorFlowSrcRoot();
 // Returns the same value for the lifetime of the process.
 int RandomSeed();
 
-// Returns an object that represents a child process that will be
-// launched with the given command-line arguments `argv`. The process
-// must be explicitly started by calling the Start() method on the
-// returned object.
-std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv);
-
 // Returns an unused port number, for use in multi-process testing.
 // NOTE: This function is not thread-safe.
 int PickUnusedPortOrDie();
diff --git a/tensorflow/core/platform/windows/subprocess.h b/tensorflow/core/platform/windows/subprocess.h
index 66ec44885d..f00471d484 100644
--- a/tensorflow/core/platform/windows/subprocess.h
+++ b/tensorflow/core/platform/windows/subprocess.h
@@ -16,11 +16,21 @@ limitations under the License.
 #ifndef TENSORFLOW_PLATFORM_WINDOWS_SUBPROCESS_H_
 #define TENSORFLOW_PLATFORM_WINDOWS_SUBPROCESS_H_
 
+#include <memory>
+#include <vector>
+
+#include "tensorflow/core/platform/logging.h"
+
 namespace tensorflow {
 
 // SubProcess is not yet implemented for Windows.
 class SubProcess {};
 
+std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
+  LOG(FATAL) << "CreateSubProcess NOT IMPLEMENTED for Windows yet ! ";
+  return nullptr;
+}
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_PLATFORM_WINDOWS_SUBPROCESS_H_
diff --git a/tensorflow/core/platform/windows/test.cc b/tensorflow/core/platform/windows/test.cc
index 584acad91b..ad2b7bc6ff 100644
--- a/tensorflow/core/platform/windows/test.cc
+++ b/tensorflow/core/platform/windows/test.cc
@@ -22,11 +22,6 @@ limitations under the License.
 namespace tensorflow {
 namespace testing {
 
-std::unique_ptr<SubProcess> CreateSubProcess(const std::vector<string>& argv) {
-  LOG(FATAL) << "CreateSubProcess NOT IMPLEMENTED for Windows yet ! ";
-  return nullptr;
-}
-
 int PickUnusedPortOrDie() { return internal::PickUnusedPortOrDie(); }
 
 string TensorFlowSrcRoot() {
-- 
GitLab


From 386ba370d41e8872a9db0d45239d7b00c14ef309 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 10:10:16 -0700
Subject: [PATCH 109/960] Added StrContains, StartsWith, and EndsWith functions
 to str_util.h. Marked contains, starts_with, ends_with, and consume
 StringPiece methods as deprecated.

This will allow tensorflow::StringPiece to be more easily replaced with absl::string_view (once the deprecated methods are removed) as absl::string_view does not contain those methods.

PiperOrigin-RevId: 189355316
---
 tensorflow/core/lib/core/stringpiece.h       |  4 ++
 tensorflow/core/lib/strings/str_util.cc      | 21 +++++++-
 tensorflow/core/lib/strings/str_util.h       | 15 ++++++
 tensorflow/core/lib/strings/str_util_test.cc | 52 ++++++++++++++++++++
 4 files changed, 90 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 910e4d9e2a..79409cce4b 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -88,11 +88,13 @@ class StringPiece {
 
   size_t find(char c, size_t pos = 0) const;
   size_t rfind(char c, size_t pos = npos) const;
+  // DEPRECATED: Use tensorflow::str_util::StrContains instead.
   bool contains(StringPiece s) const;
 
   // Checks whether StringPiece starts with x and if so advances the beginning
   // of it to past the match.  It's basically a shortcut for starts_with
   // followed by remove_prefix.
+  // DEPRECATED: Use tensorflow::str_util::ConsumePrefix instead.
   bool Consume(StringPiece x) {
     if (starts_with(x)) {
       remove_prefix(x.size_);
@@ -113,10 +115,12 @@ class StringPiece {
   int compare(StringPiece b) const;
 
   // Return true iff "x" is a prefix of "*this"
+  // DEPRECATED: Use tensorflow::str_util::StartsWith instead.
   bool starts_with(StringPiece x) const {
     return ((size_ >= x.size_) && (memcmp(data_, x.data_, x.size_) == 0));
   }
   // Return true iff "x" is a suffix of "*this"
+  // DEPRECATED: Use tensorflow::str_util::EndsWith instead.
   bool ends_with(StringPiece x) const {
     return ((size_ >= x.size_) &&
             (memcmp(data_ + (size_ - x.size_), x.data_, x.size_) == 0));
diff --git a/tensorflow/core/lib/strings/str_util.cc b/tensorflow/core/lib/strings/str_util.cc
index d28857803d..9dbb74f6b8 100644
--- a/tensorflow/core/lib/strings/str_util.cc
+++ b/tensorflow/core/lib/strings/str_util.cc
@@ -373,7 +373,7 @@ size_t RemoveWhitespaceContext(StringPiece* text) {
 }
 
 bool ConsumePrefix(StringPiece* s, StringPiece expected) {
-  if (s->starts_with(expected)) {
+  if (StartsWith(*s, expected)) {
     s->remove_prefix(expected.size());
     return true;
   }
@@ -381,7 +381,7 @@ bool ConsumePrefix(StringPiece* s, StringPiece expected) {
 }
 
 bool ConsumeSuffix(StringPiece* s, StringPiece expected) {
-  if (s->ends_with(expected)) {
+  if (EndsWith(*s, expected)) {
     s->remove_suffix(expected.size());
     return true;
   }
@@ -452,5 +452,22 @@ bool SplitAndParseAsFloats(StringPiece text, char delim,
                                     result);
 }
 
+bool StrContains(StringPiece haystack, StringPiece needle) {
+  return std::search(haystack.begin(), haystack.end(), needle.begin(),
+                     needle.end()) != haystack.end();
+}
+
+bool StartsWith(StringPiece text, StringPiece prefix) {
+  return prefix.empty() ||
+         (text.size() >= prefix.size() &&
+          memcmp(text.data(), prefix.data(), prefix.size()) == 0);
+}
+
+bool EndsWith(StringPiece text, StringPiece suffix) {
+  return suffix.empty() || (text.size() >= suffix.size() &&
+                            memcmp(text.data() + (text.size() - suffix.size()),
+                                   suffix.data(), suffix.size()) == 0);
+}
+
 }  // namespace str_util
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/strings/str_util.h b/tensorflow/core/lib/strings/str_util.h
index 44c52850fa..f062eddef8 100644
--- a/tensorflow/core/lib/strings/str_util.h
+++ b/tensorflow/core/lib/strings/str_util.h
@@ -141,6 +141,21 @@ bool SplitAndParseAsInts(StringPiece text, char delim,
 bool SplitAndParseAsFloats(StringPiece text, char delim,
                            std::vector<float>* result);
 
+// StartsWith()
+//
+// Returns whether a given string `text` begins with `prefix`.
+bool StartsWith(StringPiece text, StringPiece prefix);
+
+// EndsWith()
+//
+// Returns whether a given string `text` ends with `suffix`.
+bool EndsWith(StringPiece text, StringPiece suffix);
+
+// StrContains()
+//
+// Returns whether a given string `haystack` contains the substring `needle`.
+bool StrContains(StringPiece haystack, StringPiece needle);
+
 // ------------------------------------------------------------------
 // Implementation details below
 template <typename T>
diff --git a/tensorflow/core/lib/strings/str_util_test.cc b/tensorflow/core/lib/strings/str_util_test.cc
index 6d461241f7..63643c3e8e 100644
--- a/tensorflow/core/lib/strings/str_util_test.cc
+++ b/tensorflow/core/lib/strings/str_util_test.cc
@@ -430,4 +430,56 @@ TEST(StringReplace, EmptyStringReplaceAll) {
   EXPECT_EQ("", str_util::StringReplace("", "a", "X", /*replace_all=*/true));
 }
 
+TEST(StartsWith, Basic) {
+  const string s1(
+      "123"
+      "\0"
+      "456",
+      7);
+  const StringPiece a("foobar");
+  const StringPiece b(s1);
+  const StringPiece e;
+  EXPECT_TRUE(str_util::StartsWith(a, a));
+  EXPECT_TRUE(str_util::StartsWith(a, "foo"));
+  EXPECT_TRUE(str_util::StartsWith(a, e));
+  EXPECT_TRUE(str_util::StartsWith(b, s1));
+  EXPECT_TRUE(str_util::StartsWith(b, b));
+  EXPECT_TRUE(str_util::StartsWith(b, e));
+  EXPECT_TRUE(str_util::StartsWith(e, ""));
+  EXPECT_FALSE(str_util::StartsWith(a, b));
+  EXPECT_FALSE(str_util::StartsWith(b, a));
+  EXPECT_FALSE(str_util::StartsWith(e, a));
+}
+
+TEST(EndsWith, Basic) {
+  const string s1(
+      "123"
+      "\0"
+      "456",
+      7);
+  const StringPiece a("foobar");
+  const StringPiece b(s1);
+  const StringPiece e;
+  EXPECT_TRUE(str_util::EndsWith(a, a));
+  EXPECT_TRUE(str_util::EndsWith(a, "bar"));
+  EXPECT_TRUE(str_util::EndsWith(a, e));
+  EXPECT_TRUE(str_util::EndsWith(b, s1));
+  EXPECT_TRUE(str_util::EndsWith(b, b));
+  EXPECT_TRUE(str_util::EndsWith(b, e));
+  EXPECT_TRUE(str_util::EndsWith(e, ""));
+  EXPECT_FALSE(str_util::EndsWith(a, b));
+  EXPECT_FALSE(str_util::EndsWith(b, a));
+  EXPECT_FALSE(str_util::EndsWith(e, a));
+}
+
+TEST(StrContains, Basic) {
+  StringPiece a("abcdefg");
+  StringPiece b("abcd");
+  StringPiece c("efg");
+  StringPiece d("gh");
+  EXPECT_TRUE(str_util::StrContains(a, b));
+  EXPECT_TRUE(str_util::StrContains(a, c));
+  EXPECT_TRUE(!str_util::StrContains(a, d));
+}
+
 }  // namespace tensorflow
-- 
GitLab


From 86398ed80a09030255226678eee5d4be583a61c4 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Fri, 16 Mar 2018 10:23:41 -0700
Subject: [PATCH 110/960] Fix sed invocation in copy_binary.py script for Mac.

Script was explicitly calling /bin/sed which was not being found on MacOS Kokoro builds.
Removing calling "sed" in script.

PiperOrigin-RevId: 189357296
---
 tensorflow/tools/ci_build/copy_binary.py | 55 +++++++++++++-----------
 1 file changed, 30 insertions(+), 25 deletions(-)

diff --git a/tensorflow/tools/ci_build/copy_binary.py b/tensorflow/tools/ci_build/copy_binary.py
index ff26b052f3..b5a282b64a 100755
--- a/tensorflow/tools/ci_build/copy_binary.py
+++ b/tensorflow/tools/ci_build/copy_binary.py
@@ -29,13 +29,9 @@ import argparse
 import os
 import re
 import shutil
-import subprocess
+import tempfile
 import zipfile
 
-UNZIP_CMD = "/usr/bin/unzip"
-ZIP_CMD = "/usr/bin/zip"
-SED_CMD = "/bin/sed"
-
 TF_NIGHTLY_REGEX = r"(.+)tf_nightly(|_gpu)-(\d\.\d\.\d.dev[\d]{0,8})-(.+)\.whl"
 BINARY_STRING_TEMPLATE = "%s-%s-%s.whl"
 
@@ -65,26 +61,35 @@ def copy_binary(directory, origin_tag, new_tag, version, gpu=False):
   origin_binary = BINARY_STRING_TEMPLATE % (package, version, origin_tag)
   new_binary = BINARY_STRING_TEMPLATE % (package, version, new_tag)
   zip_ref = zipfile.ZipFile(directory + origin_binary, "r")
-  zip_ref.extractall()
-  zip_ref.close()
-  old_py_ver = re.search(r"(cp\d\d-cp\d\d)", origin_tag).group(1)
-  new_py_ver = re.search(r"(cp\d\d-cp\d\d)", new_tag).group(1)
-  subprocess.check_call(
-      "%s -i s/%s/%s/g %s-%s.dist-info/WHEEL" % (SED_CMD, old_py_ver,
-                                                 new_py_ver, package, version),
-      shell=True)
-  zout = zipfile.ZipFile(directory + new_binary, "w", zipfile.ZIP_DEFLATED)
-  zip_these_files = [
-      "%s-%s.dist-info" % (package, version),
-      "%s-%s.data" % (package, version)
-  ]
-  for dirname in zip_these_files:
-    for root, _, files in os.walk(dirname):
-      for filename in files:
-        zout.write(os.path.join(root, filename))
-  zout.close()
-  for dirname in zip_these_files:
-    shutil.rmtree(dirname)
+
+  try:
+    tmpdir = tempfile.mkdtemp()
+    os.chdir(tmpdir)
+
+    zip_ref.extractall()
+    zip_ref.close()
+    old_py_ver = re.search(r"(cp\d\d-cp\d\d)", origin_tag).group(1)
+    new_py_ver = re.search(r"(cp\d\d-cp\d\d)", new_tag).group(1)
+
+    wheel_file = os.path.join(
+        tmpdir, "%s-%s.dist-info" % (package, version), "WHEEL")
+    with open(wheel_file, "r") as f:
+      content = f.read()
+    with open(wheel_file, "w") as f:
+      f.write(content.replace(old_py_ver, new_py_ver))
+
+    zout = zipfile.ZipFile(directory + new_binary, "w", zipfile.ZIP_DEFLATED)
+    zip_these_files = [
+        "%s-%s.dist-info" % (package, version),
+        "%s-%s.data" % (package, version),
+    ]
+    for dirname in zip_these_files:
+      for root, _, files in os.walk(dirname):
+        for filename in files:
+          zout.write(os.path.join(root, filename))
+    zout.close()
+  finally:
+    shutil.rmtree(tmpdir)
 
 
 def main():
-- 
GitLab


From 4091e498ba8dedc8f4ad5952dfe1262e735e7f42 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 10:29:11 -0700
Subject: [PATCH 111/960] Fix naming BatchNorm_Fold//batch_norm_correction ->
 BatchNorm_Fold/batch_norm_correction.

PiperOrigin-RevId: 189358090
---
 .../contrib/quantize/python/fold_batch_norms.py      |  3 ++-
 .../contrib/quantize/python/fold_batch_norms_test.py | 12 ++++++++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index b278265639..e8a0d41425 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -317,7 +317,8 @@ def _ComputeBatchNormCorrections(context, match, freeze_batch_norm_delay,
   """
 
   g = ops.get_default_graph()
-  with g.name_scope(context + '/batch_norm_correction'):
+  prefix = '' if not context else context + '/'
+  with g.name_scope(prefix + 'batch_norm_correction'):
     recip_sigma_mv = math_ops.rsqrt(
         match.moving_variance_tensor + match.batch_epsilon)
     recip_sigma = math_ops.rsqrt(match.variance_tensor + match.batch_epsilon)
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
index c90a18ab03..af31467476 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms_test.py
@@ -128,6 +128,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
+    for op in g.get_operations():
+      self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)
+
   def testFoldConv2d(self):
     self._RunTestOverParameters(self._TestFoldConv2d)
 
@@ -196,6 +199,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
+    for op in g.get_operations():
+      self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)
+
   def testFoldConv2dUnknownShape(self):
     self._RunTestOverParameters(self._TestFoldConv2dUnknownShape)
 
@@ -260,6 +266,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
+    for op in g.get_operations():
+      self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)
+
   def testFoldFullyConnectedLayer(self):
     self._RunTestOverParameters(self._TestFoldFullyConnectedLayer)
 
@@ -337,6 +346,9 @@ class FoldBatchNormsTest(test_util.TensorFlowTestCase):
     output_op_names = ['test/Add' if with_bypass else 'test/' + relu_op_name]
     self._AssertOutputGoesToOps(folded_add, g, output_op_names)
 
+    for op in g.get_operations():
+      self.assertFalse('//' in op.name, 'Double slash in op %s' % op.name)
+
   def testFoldDepthwiseConv2d(self):
     self._RunTestOverParameters(self._TestFoldDepthwiseConv2d)
 
-- 
GitLab


From 7bc1f803ea53f06677bc1f96cba59d1c751fc09a Mon Sep 17 00:00:00 2001
From: Frank Perbet <fkp@google.com>
Date: Fri, 16 Mar 2018 10:48:32 -0700
Subject: [PATCH 112/960] Automated g4 rollback of changelist 189346024

PiperOrigin-RevId: 189361083
---
 .../graph_editor/tests/transform_test.py      | 32 ++-------
 tensorflow/contrib/graph_editor/transform.py  | 66 ++++++-------------
 2 files changed, 24 insertions(+), 74 deletions(-)

diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py
index 2a1b78042d..ca00394388 100644
--- a/tensorflow/contrib/graph_editor/tests/transform_test.py
+++ b/tensorflow/contrib/graph_editor/tests/transform_test.py
@@ -23,7 +23,6 @@ from tensorflow.contrib import graph_editor as ge
 from tensorflow.contrib.graph_editor.tests import match
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -85,9 +84,9 @@ class TransformTest(test.TestCase):
   def test_transform(self):
     transformer = ge.Transformer()
 
-    def my_transform_op_handler(info, op, new_inputs):
+    def my_transform_op_handler(info, op):
       add_noise = op.name.startswith("Add")
-      op_, op_outputs_ = ge.transform.copy_op_handler(info, op, new_inputs)
+      op_, op_outputs_ = ge.transform.copy_op_handler(info, op)
       if not add_noise:
         return op_, op_outputs_
       # add some noise to op
@@ -202,36 +201,15 @@ class TransformTest(test.TestCase):
                         get_operation_by_name("res/grad/mul1_grad/Mul_1"))
 
     # Make sure _original_ops are as expected.
-    self.assertEqual(original_mul1_grad._original_op.name, u"mul1")
-    self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1")
-    self.assertNotEqual(res.name, g.name)
+    self.assertEquals(original_mul1_grad._original_op.name, u"mul1")
+    self.assertEquals(result_mul1_grad._original_op.name, u"res/mul1")
+    self.assertNotEquals(res.name, g.name)
     with session.Session() as sess:
       sess.run(variables.global_variables_initializer())
       g_val, res_val = sess.run([g, res])
     self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
     self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
 
-  def test_graph_while_loop(self):
-    graph = ops.Graph()
-    with graph.as_default():
-      max_index = array_ops.placeholder(dtype=dtypes.int32, shape=tuple())
-      index_start = constant_op.constant(1)
-      sum_start = constant_op.constant(0)
-      _, result = control_flow_ops.while_loop(
-          cond=lambda i, unused_s: i <= max_index,
-          body=lambda i, s: (i + 1, s + i),
-          loop_vars=[index_start, sum_start])
-    copied_graph = ops.Graph()
-    _, copy_info = ge.copy(
-        graph, dst_graph=copied_graph, dst_scope="imported")
-    copied_result = copy_info.transformed(result)
-    copied_max_index = copy_info.transformed(max_index)
-    with copied_graph.as_default():
-      with session.Session() as sess:
-        n = 10
-        sum_val = sess.run(copied_result, feed_dict={copied_max_index: n})
-        self.assertEqual(sum_val, 55)
-
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py
index 03c9afe813..14ac529665 100644
--- a/tensorflow/contrib/graph_editor/transform.py
+++ b/tensorflow/contrib/graph_editor/transform.py
@@ -30,7 +30,6 @@ from tensorflow.contrib.graph_editor import select
 from tensorflow.contrib.graph_editor import subgraph
 from tensorflow.contrib.graph_editor import util
 from tensorflow.python.framework import ops as tf_ops
-from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
 
 
@@ -130,26 +129,20 @@ def transform_op_if_inside_handler(info, op, keep_if_possible=True):
       return None
 
 
-def copy_op_handler(info, op, new_inputs, copy_shape=True):
+def copy_op_handler(info, op, copy_shape=True):
   """Copy a `tf.Operation`.
 
   Args:
     info: Transform._TmpInfo instance.
     op: the `tf.Operation` to be copied.
-    new_inputs: The new inputs for this op.
     copy_shape: also copy the shape of the tensor
   Returns:
     A `(op, op_outputs)` tuple containing the transformed op and its outputs.
   """
-  # The `new_inputs` was added to this function. For compatibility reason,
-  # let's raise an error if `new_inputs` is a boolean.
-  if isinstance(new_inputs, bool):
-    raise TypeError("the `new_inputs` argument must be an iterable.")
-
   # pylint: disable=protected-access
 
   # Clone the node def:
-  node_def_ = deepcopy(op.node_def)
+  node_def_ = deepcopy(op._node_def)
 
   # Transform name:
   name_ = info.new_name(op.name)
@@ -162,10 +155,10 @@ def copy_op_handler(info, op, new_inputs, copy_shape=True):
 
   # Make a copy of the op_def too.
   # Its unique to every _type_ of Operation.
-  op_def_ = deepcopy(op.op_def)
+  op_def_ = deepcopy(op._op_def)
 
   # Initialize a new Operation instance
-  op_ = tf_ops.Operation(node_def_, info.graph_, new_inputs, output_types_,
+  op_ = tf_ops.Operation(node_def_, info.graph_, [], output_types_,
                          [], input_types_, None, op_def_)
 
   # copy the shape over
@@ -177,7 +170,6 @@ def copy_op_handler(info, op, new_inputs, copy_shape=True):
   # attribute to exist, we will create a dummy original_op first and then
   # later finalise it with the actual original_op when all the ops have
   # been copied.
-  # TODO(fkp): Stop worrying about _original_op and remove this code?
   if op._original_op:
     op_._original_op = op._original_op
 
@@ -336,14 +328,6 @@ class _TmpInfo(object):
                             for key in self.graph.get_all_collection_keys())
     self.cyclic_ops = []
     self.transform_original_op_handler = transform_op_if_inside_handler
-    # The graph is transformed op by op, in the same order the original ops
-    # where created. However, this is sometimes not possible due to cycles
-    # (e.g. while loops). So when the transformer creates a new op whose
-    # inputs do not exist yet, temporary placeholders are created and stored
-    # in this `tmp_cyclic_ts` container. During a second pass,
-    # those temporary tensors are replaced by the proper transformed tensors
-    # (see the function `_finalize_cycle`).
-    self.tmp_cyclic_ts = []
 
   def new_name(self, name):
     """Compute a destination name from a source name.
@@ -444,10 +428,10 @@ class Transformer(object):
 
     # Create temporary info used during this transform call
     info = _TmpInfo(sgv, dst_graph, dst_scope, src_scope)
+    info.transform_original_op_handler = self.transform_original_op_handler
 
     self._copy_ops(info)
-    self._finalize_cycle(info)
-    self._connect_control_inputs(info)
+    self._connect_ops(info)
 
     # Compute information about the transformation
     res_info = TransformerInfo(info)
@@ -456,12 +440,10 @@ class Transformer(object):
 
   def _copy_ops(self, info):
     """Copy ops without connecting them."""
-    sorted_ops = sorted(info.sgv.ops, key=lambda op: op._id)  # pylint: disable=protected-access
-    for op in sorted_ops:
+    for op in info.sgv.ops:
       logging.debug("Copying op: %s", op.name)
-      new_inputs = [self._transformed_t(info, t) for t in op.inputs]
       # TODO(fkp): return a subgraph?
-      op_, op_outputs_ = self.transform_op_handler(info, op, new_inputs)
+      op_, op_outputs_ = self.transform_op_handler(info, op)
       if op is op_:
         raise ValueError("In-place transformation not allowed.")
 
@@ -474,31 +456,27 @@ class Transformer(object):
         info.transformed_ts[op_output] = op_output_
         self.assign_collections_handler(info, op_output, op_output_)
 
-  def _finalize_cycle(self, info):
-    for t, tmp_t_ in info.tmp_cyclic_ts:
-      if t not in info.transformed_ts:
-        raise ValueError("The tensor {} should be transformed by now.".format(
-            t.name))
-      op_ = tmp_t_.consumers()[0]
-      t_ = info.transformed_ts[t]
-      op_._update_input(list(op_.inputs).index(tmp_t_), t_)  # pylint: disable=protected-access
-
-  def _connect_control_inputs(self, info):
+  def _connect_ops(self, info):
     """Connect the previously copied ops."""
     for op in info.sgv.ops:
-      logging.debug("Connecting control inputs of op: %s", op.name)
+      logging.debug("Finalizing op: %s", op.name)
       op_ = info.transformed_ops[op]
 
-      # Finalize original op.
-      # TODO(fkp): Stop worrying about _original_op and remove this code?
       # pylint: disable=protected-access
+      if op_.inputs:
+        raise ValueError("The newly transformed op should not have "
+                         "any inputs yet: {}".format(op_.name))
+      inputs_ = [self._transformed_t(info, t) for t in op.inputs]
+      for t in inputs_:
+        op_._add_input(t)
+
+      # Finalize original op.
       if op._original_op:
-        original_op = self.transform_original_op_handler(info, op._original_op)
+        original_op = info.transform_original_op_handler(info, op._original_op)
         if original_op is None:
           logging.debug("Could not find original op for: %s", op_.name)
         else:
           op_._original_op = original_op
-      # pylint: enable=protected-access
 
       # Finalize control inputs:
       control_inputs_ = [self.transform_control_input_handler(info, ci)
@@ -550,12 +528,6 @@ class Transformer(object):
   def _transformed_t(self, info, t):
     """Return tre transformed tensor of `t`."""
     if t not in info.transformed_ts:
-      if t.op in info.ops:
-        with info.graph_.as_default():
-          tmp_t_ = array_ops.placeholder(
-              shape=t.shape, dtype=t.dtype, name="ge_tmp")
-        info.tmp_cyclic_ts.append((t, tmp_t_))
-        return tmp_t_
       # If op is not in the subgraph.
       if t in info.sgv_inputs_set:
         # t is an input of the subgraph.
-- 
GitLab


From 53cbb5a50e94a772b595dcadd63af3b79bb884c7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 11:29:00 -0700
Subject: [PATCH 113/960] Fixing constant output arrays by inserting synthetic
 reshapes.

PiperOrigin-RevId: 189368237
---
 tensorflow/contrib/lite/toco/toco_tooling.cc |   4 +
 tensorflow/contrib/lite/toco/tooling_util.cc | 111 +++++++++++++++++++
 tensorflow/contrib/lite/toco/tooling_util.h  |  12 ++
 3 files changed, 127 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 024335b5e4..ca66110ba3 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -289,6 +289,10 @@ void Transform(const TocoFlags& toco_flags, Model* model) {
     EncodeConstantArraysMinMaxByWrappingThemInFakeQuantNodes(model);
   }
 
+  // Fix any issues with IO edges. This must happen after any transform that
+  // may modify the structure of the edges.
+  FixEdgeArrays(model);
+
   LogDump(kLogLevelModelChanged, "AFTER TRANSFORMATIONS", *model);
 
   if (output_format != GRAPHVIZ_DOT && output_format != TFLITE) {
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index e70291ad0e..2362206a14 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -1047,6 +1047,117 @@ void CheckModelCounts(const Model& model) {
   }
 }
 
+void FixEdgeArrays(Model* model) {
+  for (const string& output_array_name : model->flags.output_arrays()) {
+    if (!GetOpWithOutput(*model, output_array_name)) {
+      // Output has no operator producing it. Change that by inserting a copy.
+      LOG(WARNING) << "Fixing constant output array " << output_array_name
+                   << " by inserting a copy. This is not optimal.";
+      string intermediate_array_name =
+          AvailableArrayName(*model, output_array_name + "_copy");
+      CloneArray(model, output_array_name, intermediate_array_name);
+      InsertCopyOperator(model, intermediate_array_name, output_array_name);
+    }
+  }
+}
+
+void InsertCopyOperator(Model* model, const string& source_array_name,
+                        const string& target_array_name) {
+  // Drop constant data from the target array as the copy will be done at
+  // runtime.
+  Array& target_array = model->GetOrCreateArray(target_array_name);
+  target_array.buffer.reset();
+
+  // Reshape to the same size. This should be a no-op.
+  const Array& source_array = model->GetArray(source_array_name);
+  std::vector<int> shape = source_array.shape().dims();
+
+  // Insert copy operator.
+  auto* copy_op = new TensorFlowReshapeOperator;
+  copy_op->inputs = {
+      source_array_name,
+      CreateInt32Array(model, target_array_name + "_copy_shape", shape)};
+  copy_op->outputs = {target_array_name};
+  model->operators.emplace_back(copy_op);
+}
+
+namespace {
+template <ArrayDataType A>
+void CopyArrayBuffer(const Array& source_array, Array* target_array) {
+  if (source_array.buffer) {
+    const auto& source_buffer = source_array.GetBuffer<A>();
+    auto& target_buffer = target_array->GetMutableBuffer<A>();
+    target_buffer.data = source_buffer.data;
+  }
+}
+}  // namespace
+
+void CloneArray(Model* model, const string& source_array_name,
+                const string& target_array_name) {
+  CHECK(!model->HasArray(target_array_name));
+  const Array& source_array = model->GetArray(source_array_name);
+  Array& target_array = model->GetOrCreateArray(target_array_name);
+
+  switch (source_array.data_type) {
+    case ArrayDataType::kBool:
+      CopyArrayBuffer<ArrayDataType::kBool>(source_array, &target_array);
+      break;
+    case ArrayDataType::kFloat:
+      CopyArrayBuffer<ArrayDataType::kFloat>(source_array, &target_array);
+      break;
+    case ArrayDataType::kInt8:
+      CopyArrayBuffer<ArrayDataType::kInt8>(source_array, &target_array);
+      break;
+    case ArrayDataType::kUint8:
+      CopyArrayBuffer<ArrayDataType::kUint8>(source_array, &target_array);
+      break;
+    case ArrayDataType::kInt16:
+      CopyArrayBuffer<ArrayDataType::kInt16>(source_array, &target_array);
+      break;
+    case ArrayDataType::kUint16:
+      CopyArrayBuffer<ArrayDataType::kUint16>(source_array, &target_array);
+      break;
+    case ArrayDataType::kInt32:
+      CopyArrayBuffer<ArrayDataType::kInt32>(source_array, &target_array);
+      break;
+    case ArrayDataType::kUint32:
+      CopyArrayBuffer<ArrayDataType::kUint32>(source_array, &target_array);
+      break;
+    case ArrayDataType::kInt64:
+      CopyArrayBuffer<ArrayDataType::kInt64>(source_array, &target_array);
+      break;
+    case ArrayDataType::kUint64:
+      CopyArrayBuffer<ArrayDataType::kUint64>(source_array, &target_array);
+      break;
+    case ArrayDataType::kString:
+      CopyArrayBuffer<ArrayDataType::kString>(source_array, &target_array);
+      break;
+    default:
+      LOG(FATAL) << "Unsupported data type: "
+                 << ArrayDataTypeName(source_array.data_type);
+      return;
+  }
+
+  if (source_array.minmax) {
+    const auto& smm = source_array.GetMinMax();
+    auto& tmm = target_array.GetOrCreateMinMax();
+    tmm.min = smm.min;
+    tmm.max = smm.max;
+  }
+
+  if (source_array.quantization_params) {
+    const auto& sqp = source_array.GetQuantizationParams();
+    auto& tqp = target_array.GetOrCreateQuantizationParams();
+    tqp.zero_point = sqp.zero_point;
+    tqp.scale = sqp.scale;
+  }
+
+  target_array.data_type = source_array.data_type;
+  target_array.final_data_type = source_array.final_data_type;
+
+  target_array.copy_shape(source_array.shape());
+}
+
 void MakeArrayDims(int num_dims, int batch, int height, int width, int depth,
                    std::vector<int>* out_dims) {
   CHECK(out_dims->empty());
diff --git a/tensorflow/contrib/lite/toco/tooling_util.h b/tensorflow/contrib/lite/toco/tooling_util.h
index 05360e3b0a..d3b7224fe3 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.h
+++ b/tensorflow/contrib/lite/toco/tooling_util.h
@@ -144,6 +144,18 @@ void FixOperatorOrdering(Model* model);
 void FixNoMissingArray(Model* model);
 void FixNoOrphanedArray(Model* model);
 
+// Fixes input/output arrays that may have issues during export or inference.
+void FixEdgeArrays(Model* model);
+
+// Inserts a no-op reshape operator between the source array and the target
+// array. This effectively just copies the data.
+void InsertCopyOperator(Model* model, const string& source_array_name,
+                        const string& target_array_name);
+
+// Clones an array with all data and parameters.
+void CloneArray(Model* model, const string& source_array_name,
+                const string& target_array_name);
+
 void ResolveModelFlags(const ModelFlags& model_flags, Model* model);
 
 template <ArrayDataType A>
-- 
GitLab


From fff901507e932188932fe21ae56c55e4aba5ae52 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 11:45:42 -0700
Subject: [PATCH 114/960] Set number of threads at Java interpreter constructor
 so that Conv Kernels can be selected properly. Remove setNumThreads in the
 Java API as its behavior is ambiguous.

PiperOrigin-RevId: 189370770
---
 .../java/org/tensorflow/lite/Interpreter.java | 11 ++++++++
 .../lite/NativeInterpreterWrapper.java        | 25 ++++++++++++-------
 .../native/nativeinterpreterwrapper_jni.cc    | 18 +++----------
 .../native/nativeinterpreterwrapper_jni.h     | 16 +++---------
 .../java/org/tensorflow/lite/TestHelper.java  | 14 -----------
 tensorflow/contrib/lite/model.cc              |  8 +++++-
 tensorflow/contrib/lite/model.h               |  2 ++
 7 files changed, 43 insertions(+), 51 deletions(-)

diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
index cc17b491f2..14f461f5f9 100644
--- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
+++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
@@ -77,6 +77,17 @@ public final class Interpreter implements AutoCloseable {
     wrapper = new NativeInterpreterWrapper(mappedByteBuffer);
   }
 
+  /**
+   * Initializes a {@code Interpreter} with a {@code MappedByteBuffer} to the model file and
+   * specifies the number of threads used for inference.
+   *
+   * <p>The {@code MappedByteBuffer} should remain unchanged after the construction of a {@code
+   * Interpreter}.
+   */
+  public Interpreter(@NonNull MappedByteBuffer mappedByteBuffer, int numThreads) {
+    wrapper = new NativeInterpreterWrapper(mappedByteBuffer, numThreads);
+  }
+
   /**
    * Runs model inference if the model takes only one input, and provides only one output.
    *
diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
index 518e8b3a96..dbf8f8f7cc 100644
--- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
+++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
@@ -34,7 +34,7 @@ final class NativeInterpreterWrapper implements AutoCloseable {
   NativeInterpreterWrapper(String modelPath) {
     errorHandle = createErrorReporter(ERROR_BUFFER_SIZE);
     modelHandle = createModel(modelPath, errorHandle);
-    interpreterHandle = createInterpreter(modelHandle, errorHandle);
+    interpreterHandle = createInterpreter(modelHandle, errorHandle, /* numThreads= */ -1);
     isMemoryAllocated = true;
   }
 
@@ -47,7 +47,20 @@ final class NativeInterpreterWrapper implements AutoCloseable {
     modelByteBuffer = mappedByteBuffer;
     errorHandle = createErrorReporter(ERROR_BUFFER_SIZE);
     modelHandle = createModelWithBuffer(modelByteBuffer, errorHandle);
-    interpreterHandle = createInterpreter(modelHandle, errorHandle);
+    interpreterHandle = createInterpreter(modelHandle, errorHandle, /* numThreads= */ -1);
+    isMemoryAllocated = true;
+  }
+
+  /**
+   * Initializes a {@code NativeInterpreterWrapper} with a {@code MappedByteBuffer} and specifies
+   * the number of inference threads. The MappedByteBuffer should not be modified after the
+   * construction of a {@code NativeInterpreterWrapper}.
+   */
+  NativeInterpreterWrapper(MappedByteBuffer mappedByteBuffer, int numThreads) {
+    modelByteBuffer = mappedByteBuffer;
+    errorHandle = createErrorReporter(ERROR_BUFFER_SIZE);
+    modelHandle = createModelWithBuffer(modelByteBuffer, errorHandle);
+    interpreterHandle = createInterpreter(modelHandle, errorHandle, numThreads);
     isMemoryAllocated = true;
   }
 
@@ -140,10 +153,6 @@ final class NativeInterpreterWrapper implements AutoCloseable {
     useNNAPI(interpreterHandle, useNNAPI);
   }
 
-  void setNumThreads(int numRecommendedThreads) {
-    numThreads(interpreterHandle, numRecommendedThreads);
-  }
-
   /** Gets index of an input given its name. */
   int getInputIndex(String name) {
     if (inputsIndexes == null) {
@@ -312,15 +321,13 @@ final class NativeInterpreterWrapper implements AutoCloseable {
 
   private static native void useNNAPI(long interpreterHandle, boolean state);
 
-  private static native void numThreads(long interpreterHandle, int numRecommendedThreads);
-
   private static native long createErrorReporter(int size);
 
   private static native long createModel(String modelPathOrBuffer, long errorHandle);
 
   private static native long createModelWithBuffer(MappedByteBuffer modelBuffer, long errorHandle);
 
-  private static native long createInterpreter(long modelHandle, long errorHandle);
+  private static native long createInterpreter(long modelHandle, long errorHandle, int numThreads);
 
   private static native void delete(long errorHandle, long modelHandle, long interpreterHandle);
 
diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
index cc448b03c3..844226203b 100644
--- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
+++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.cc
@@ -14,7 +14,6 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h"
-
 namespace {
 
 const int kByteBufferValue = 999;
@@ -316,16 +315,6 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_useNNAPI(JNIEnv* env,
   interpreter->UseNNAPI(static_cast<bool>(state));
 }
 
-JNIEXPORT void JNICALL
-Java_org_tensorflow_lite_NativeInterpreterWrapper_numThreads(JNIEnv* env,
-                                                             jclass clazz,
-                                                             jlong handle,
-                                                             jint num_threads) {
-  tflite::Interpreter* interpreter = convertLongToInterpreter(env, handle);
-  if (interpreter == nullptr) return;
-  interpreter->SetNumThreads(static_cast<int>(num_threads));
-}
-
 JNIEXPORT jlong JNICALL
 Java_org_tensorflow_lite_NativeInterpreterWrapper_createErrorReporter(
     JNIEnv* env, jclass clazz, jint size) {
@@ -401,7 +390,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createModelWithBuffer(
 
 JNIEXPORT jlong JNICALL
 Java_org_tensorflow_lite_NativeInterpreterWrapper_createInterpreter(
-    JNIEnv* env, jclass clazz, jlong model_handle, jlong error_handle) {
+    JNIEnv* env, jclass clazz, jlong model_handle, jlong error_handle,
+    jint num_threads) {
   tflite::FlatBufferModel* model = convertLongToModel(env, model_handle);
   if (model == nullptr) return 0;
   BufferErrorReporter* error_reporter =
@@ -409,8 +399,8 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createInterpreter(
   if (error_reporter == nullptr) return 0;
   auto resolver = ::tflite::CreateOpResolver();
   std::unique_ptr<tflite::Interpreter> interpreter;
-  TfLiteStatus status =
-      tflite::InterpreterBuilder(*model, *(resolver.get()))(&interpreter);
+  TfLiteStatus status = tflite::InterpreterBuilder(*model, *(resolver.get()))(
+      &interpreter, static_cast<int>(num_threads));
   if (status != kTfLiteOk) {
     throwException(env, kIllegalArgumentException,
                    "Cannot create interpreter: %s",
diff --git a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
index fb76125471..0e28a77fee 100644
--- a/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
+++ b/tensorflow/contrib/lite/java/src/main/native/nativeinterpreterwrapper_jni.h
@@ -69,17 +69,6 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_useNNAPI(JNIEnv* env,
                                                            jlong handle,
                                                            jboolean state);
 
-/*
- *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
- *  Method:
- *  Signature: (JI)
- */
-JNIEXPORT void JNICALL
-Java_org_tensorflow_lite_NativeInterpreterWrapper_numThreads(JNIEnv* env,
-                                                             jclass clazz,
-                                                             jlong handle,
-                                                             jint num_threads);
-
 /*
  *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
  *  Method:
@@ -110,11 +99,12 @@ Java_org_tensorflow_lite_NativeInterpreterWrapper_createModelWithBuffer(
 /*
  *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
  *  Method:
- *  Signature: (JJ)J
+ *  Signature: (JJI)J
  */
 JNIEXPORT jlong JNICALL
 Java_org_tensorflow_lite_NativeInterpreterWrapper_createInterpreter(
-    JNIEnv* env, jclass clazz, jlong model_handle, jlong error_handle);
+    JNIEnv* env, jclass clazz, jlong model_handle, jlong error_handle,
+    jint num_threads);
 
 /*
  *  Class:     org_tensorflow_lite_NativeInterpreterWrapper
diff --git a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
index 3722e51b3b..3aef0c3bb6 100644
--- a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
+++ b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/TestHelper.java
@@ -33,20 +33,6 @@ public class TestHelper {
     }
   }
 
-  /**
-   * Sets the number of threads for an {@code Interpreter}.
-   *
-   * @param interpreter an instance of {@code Interpreter}. If it is not initialized, an {@code
-   *     IllegalArgumentException} will be thrown.
-   * @param numRecommendedThreads an integer value indicating the number of recommended threads.
-   */
-  public static void setNumThreads(Interpreter interpreter, int numRecommendedThreads) {
-    if (interpreter != null && interpreter.wrapper != null) {
-      interpreter.wrapper.setNumThreads(numRecommendedThreads);
-    } else {
-      throw new IllegalArgumentException("Interpreter has not initialized; Failed to setUseNNAPI.");
-    }
-  }
   /**
    * Gets the last inference duration in nanoseconds. It returns null if there is no previous
    * inference run or the last inference run failed.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index f28d56af67..f7daa6fc9d 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -759,6 +759,11 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
 
 TfLiteStatus InterpreterBuilder::operator()(
     std::unique_ptr<Interpreter>* interpreter) {
+  return operator()(interpreter, /*num_threads=*/-1);
+}
+
+TfLiteStatus InterpreterBuilder::operator()(
+    std::unique_ptr<Interpreter>* interpreter, int num_threads) {
   if (!interpreter) {
     error_reporter_->Report(
         "Null output pointer passed to InterpreterBuilder.");
@@ -813,7 +818,8 @@ TfLiteStatus InterpreterBuilder::operator()(
   if ((**interpreter).AddTensors(tensors->Length()) != kTfLiteOk) {
     return cleanup_and_error();
   }
-
+  // Set num threads
+  (**interpreter).SetNumThreads(num_threads);
   // Parse inputs/outputs
   (**interpreter).SetInputs(FlatBufferIntArrayToVector(subgraph->inputs()));
   (**interpreter).SetOutputs(FlatBufferIntArrayToVector(subgraph->outputs()));
diff --git a/tensorflow/contrib/lite/model.h b/tensorflow/contrib/lite/model.h
index 51a622a28d..0c777760cb 100644
--- a/tensorflow/contrib/lite/model.h
+++ b/tensorflow/contrib/lite/model.h
@@ -154,6 +154,8 @@ class InterpreterBuilder {
   InterpreterBuilder(const InterpreterBuilder&) = delete;
   InterpreterBuilder& operator=(const InterpreterBuilder&) = delete;
   TfLiteStatus operator()(std::unique_ptr<Interpreter>* interpreter);
+  TfLiteStatus operator()(std::unique_ptr<Interpreter>* interpreter,
+                          int num_threads);
 
  private:
   TfLiteStatus BuildLocalIndexToRegistrationMapping();
-- 
GitLab


From 0b247ec25de24813a6185a0dc4fbf17edd1c177a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 16 Mar 2018 11:47:26 -0700
Subject: [PATCH 115/960] Allow variable lists to change when saving repeatedly
 using tfe.Checkpoint

For example allows saving a checkpoint before slot variables have been created

When graph building, restore() is still bound to a frozen set of variables.

PiperOrigin-RevId: 189371256
---
 .../eager/python/checkpointable_utils.py      | 28 ++++++-----
 .../eager/python/checkpointable_utils_test.py | 48 ++++++++++++++++++-
 2 files changed, 64 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index 389d4a03c8..0a34f3b3f6 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -523,6 +523,18 @@ class _SessionWithFeedDictAdditions(session_lib.SessionInterface):
         fetches=fetches, feed_dict=feed_dict, **kwargs)
 
 
+def _copy_saver_with_new_var_list(old_saver, new_var_list):
+  """Copy a `tf.train.Saver`'s state to a new Saver with different variables."""
+  new_saver = saver_lib.Saver(var_list=new_var_list)
+  # TODO(allenl): Move to copying functionality to Saver?
+  # pylint: disable=protected-access
+  new_saver._last_checkpoints = old_saver._last_checkpoints
+  new_saver._checkpoints_to_be_deleted = old_saver._checkpoints_to_be_deleted
+  new_saver._next_checkpoint_time = old_saver._next_checkpoint_time
+  # pylint: enable=protected-access
+  return new_saver
+
+
 class CheckpointableSaver(object):
   """Saves and restores a `Checkpointable` object and its dependencies.
 
@@ -623,19 +635,13 @@ class CheckpointableSaver(object):
         name=_OBJECT_GRAPH_PROTO_KEY)
     if self._last_save_object_graph != graph_proto:
       if self._last_save_object_graph is not None:
-        raise NotImplementedError(
-            "Using a single Saver to save a mutated object graph is not "
-            "currently supported when graph building. Use a different Saver "
-            "when the object graph changes (save ops will be duplicated when "
-            "graph building), or file a feature request if this limitation "
-            "bothers you.")
-      saver = saver_lib.Saver(var_list=named_variables)
-      self._last_save_saver = saver
+        self._last_save_saver = _copy_saver_with_new_var_list(
+            old_saver=self._last_save_saver, new_var_list=named_variables)
+      else:
+        self._last_save_saver = saver_lib.Saver(var_list=named_variables)
       self._last_save_object_graph = graph_proto
-    else:
-      saver = self._last_save_saver
     with ops.device("/cpu:0"):
-      save_path = saver.save(
+      save_path = self._last_save_saver.save(
           sess=_SessionWithFeedDictAdditions(
               session=session, feed_additions=feed_additions),
           save_path=file_prefix,
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 4e0a9923ff..690f3ee67a 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -545,13 +545,13 @@ class CheckpointingTests(test.TestCase):
       checkpoint = checkpointable_utils.Checkpoint(
           model=model, optimizer=optimizer)
       for _ in range(2):
+        checkpoint.save(checkpoint_prefix)
         with backprop.GradientTape() as tape:
           loss = (constant_op.constant(1.)
                   - model(constant_op.constant(1.))) ** 2
         grad = tape.gradient(loss, model.vars)
         optimizer.apply_gradients(
             [(g, v) for g, v in zip(grad, model.vars)])
-        checkpoint.save(checkpoint_prefix)
 
   @test_util.run_in_graph_and_eager_modes()
   def testLateDependencyTracking(self):
@@ -899,6 +899,52 @@ class CheckpointingTests(test.TestCase):
         expected_filenames,
         os.listdir(checkpoint_directory))
 
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testCheckpointCleanupChangingVarList(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    obj = checkpointable.Checkpointable()
+    obj.var = variable_scope.get_variable(name="v", initializer=0.)
+    self.evaluate(checkpointable_utils.gather_initializers(obj))
+    checkpoint = checkpointable_utils.Checkpoint(obj=obj)
+    looped_variables = []
+    for iteration in range(10):
+      new_variable = resource_variable_ops.ResourceVariable(iteration)
+      self.evaluate(new_variable.initializer)
+      setattr(checkpoint, "var_%d" % iteration, new_variable)
+      checkpoint.save(checkpoint_prefix)
+      looped_variables.append(new_variable)
+    expected_filenames = ["checkpoint"]
+    # We've copied the saver each time, but checkpoint management should still
+    # be consistent.
+    for checkpoint_number in range(6, 11):
+      expected_filenames.append("ckpt-%d.index" % (checkpoint_number,))
+      expected_filenames.append(
+          "ckpt-%d.data-00000-of-00001" % (checkpoint_number,))
+    six.assertCountEqual(
+        self,
+        expected_filenames,
+        os.listdir(checkpoint_directory))
+    for v in looped_variables:
+      self.evaluate(v.assign(314))
+    checkpoint.restore(checkpoint_prefix + "-6").run_restore_ops()
+    self.assertEqual(314, self.evaluate(checkpoint.var_9))
+    self.assertEqual(314, self.evaluate(checkpoint.var_8))
+    self.assertEqual(314, self.evaluate(checkpoint.var_6))
+    self.assertEqual(5, self.evaluate(checkpoint.var_5))
+    self.assertEqual(1, self.evaluate(checkpoint.var_1))
+    self.assertEqual(0, self.evaluate(checkpoint.var_0))
+    if context.executing_eagerly():
+      checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops()
+      self.assertEqual(9, self.evaluate(checkpoint.var_9))
+      self.assertEqual(8, self.evaluate(checkpoint.var_8))
+      self.assertEqual(1, self.evaluate(checkpoint.var_1))
+      self.assertEqual(0, self.evaluate(checkpoint.var_0))
+    else:
+      # Restoring into modified graphs is an error while graph building.
+      with self.assertRaises(NotImplementedError):
+        checkpoint.restore(checkpoint_prefix + "-10").run_restore_ops()
+
   def testManyRestoresGraph(self):
     """Restores after the first should not modify the graph."""
     with context.graph_mode():
-- 
GitLab


From 13354ed1adb221af60f159cd2b8034dc6e5ef43a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Fri, 16 Mar 2018 11:53:14 -0700
Subject: [PATCH 116/960] [TF:XLA] Bump open source llvm revision to r327616

PiperOrigin-RevId: 189372065
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 8d739158c5..db70e4515b 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -475,11 +475,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/738ee045416377e8c2094f7f61508ac1c178ff37.tar.gz",
-          "https://github.com/llvm-mirror/llvm/archive/738ee045416377e8c2094f7f61508ac1c178ff37.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cfb3cd346a75b17856c4e2ba6365e15d9ab0c763.tar.gz",
+          "https://github.com/llvm-mirror/llvm/archive/cfb3cd346a75b17856c4e2ba6365e15d9ab0c763.tar.gz",
       ],
-      sha256 = "4442ed6a05c13752338036b1b9f16b09264de24b6c0bf62325fb9ff75a09340f",
-      strip_prefix = "llvm-738ee045416377e8c2094f7f61508ac1c178ff37",
+      sha256 = "2cf79b1891926b7af6173c1031d040fc07b2682ff66039c5822e074566c48956",
+      strip_prefix = "llvm-cfb3cd346a75b17856c4e2ba6365e15d9ab0c763",
       build_file = str(Label("//third_party/llvm:llvm.BUILD")),
   )
 
-- 
GitLab


From 6655570f12dba22fe752796471635e109e682056 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 14 Mar 2018 18:39:25 -0700
Subject: [PATCH 117/960] [tf.data] Fix Python shape inference for
 `tf.contrib.data.map_and_batch()`.

Previously, it would incorrectly report that all batches have the same size, not accounting for the possibility of the last batch being partial.

Fixes #17720.

PiperOrigin-RevId: 189121488
---
 .../python/kernel_tests/batch_dataset_op_test.py   | 14 ++++++++++++++
 tensorflow/contrib/data/python/ops/batching.py     |  3 +--
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 71dc1c1172..a2da953c7b 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -387,6 +387,20 @@ class BatchDatasetTest(test.TestCase):
   def testBatchAndMapDatasetWithParallelBatching(self):
     return self._testBatchAndMapDatasetHelper(num_parallel_batches=10)
 
+  def testMapAndBatchYieldsPartialBatch(self):
+    iterator = (dataset_ops.Dataset.range(10)
+                .apply(batching.map_and_batch(
+                    lambda x: array_ops.reshape(x * x, [1]), 4))
+                .make_one_shot_iterator())
+    self.assertEqual([None, 1], iterator.output_shapes.as_list())
+    next_element = iterator.get_next()
+    with self.test_session() as sess:
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index 6eb512dec6..6463d75750 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -374,8 +374,7 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
   @property
   def output_shapes(self):
     return nest.pack_sequence_as(self._output_shapes, [
-        tensor_shape.vector(tensor_util.constant_value(
-            self._batch_size)).concatenate(s)
+        tensor_shape.vector(None).concatenate(s)
         for s in nest.flatten(self._output_shapes)
     ])
 
-- 
GitLab


From 7432b190a6485dffc0d548c5be0a69ffd36a7828 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 16 Mar 2018 12:11:18 -0700
Subject: [PATCH 118/960] [tf.data] Fix typo in `Dataset.prefetch()` docstring.

PiperOrigin-RevId: 189374898
---
 tensorflow/python/data/ops/dataset_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index 390ce852b1..a0c5a43a45 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -563,7 +563,7 @@ class Dataset(object):
 
     Args:
       buffer_size: A `tf.int64` scalar `tf.Tensor`, representing the
-        maximum number elements that will be buffered when prefetching.
+        maximum number of elements that will be buffered when prefetching.
 
     Returns:
       Dataset: A `Dataset`.
-- 
GitLab


From 7858441854f2101fff7c450e3a72348c3d739e9d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 12:12:52 -0700
Subject: [PATCH 119/960] Remove empty buckets in latency_stats as it makes the
 report unreadable. This is also consistent with how SummaryHistoOp in
 summary_op.cc works.

PiperOrigin-RevId: 189375113
---
 tensorflow/core/kernels/data/stats_aggregator_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/stats_aggregator_ops.cc b/tensorflow/core/kernels/data/stats_aggregator_ops.cc
index 5a2dd9c43d..17103627e0 100644
--- a/tensorflow/core/kernels/data/stats_aggregator_ops.cc
+++ b/tensorflow/core/kernels/data/stats_aggregator_ops.cc
@@ -47,7 +47,7 @@ class StatsAggregatorImpl : public StatsAggregator {
       Summary::Value* value = out_summary->add_value();
       value->set_tag(name);
       histogram.EncodeToProto(value->mutable_histo(),
-                              true /* preserve_zero_buckets */);
+                              false /* doesn't preserve zero buckets */);
     }
   }
 
-- 
GitLab


From 504d103a405654f029e8902d97d4dd8f3aa07513 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Fri, 16 Mar 2018 12:34:34 -0700
Subject: [PATCH 120/960] [XLA:python] Plumb hlo_profile flag.

PiperOrigin-RevId: 189377860
---
 .../compiler/xla/client/executable_build_options.cc |  7 +++++++
 .../compiler/xla/client/executable_build_options.h  |  6 ++++++
 .../compiler/xla/python/local_computation_builder.i | 13 +++++++++++++
 tensorflow/compiler/xla/python/xla_client.py        |  1 +
 tensorflow/compiler/xla/service/local_service.cc    |  2 ++
 5 files changed, 29 insertions(+)

diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc
index 804e34f5e7..d84f2018e1 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.cc
+++ b/tensorflow/compiler/xla/client/executable_build_options.cc
@@ -76,4 +76,11 @@ ExecutableBuildOptions::generate_hlo_graph() const {
   return generate_hlo_graph_;
 }
 
+ExecutableBuildOptions& ExecutableBuildOptions::set_hlo_profile(bool enabled) {
+  hlo_profile_ = enabled;
+  return *this;
+}
+
+bool ExecutableBuildOptions::hlo_profile() const { return hlo_profile_; }
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index 3a52dbac9a..3e18e5de64 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -57,11 +57,17 @@ class ExecutableBuildOptions {
   ExecutableBuildOptions& set_generate_hlo_graph(string regex);
   const tensorflow::gtl::optional<string>& generate_hlo_graph() const;
 
+  // If set, specifies that we should record an HLO profile during execution and
+  // log it after execution (as in DebugOptions).
+  ExecutableBuildOptions& set_hlo_profile(bool enabled);
+  bool hlo_profile() const;
+
   // Returns a string representation of the build options, suitable for
   // debugging.
   string ToString() const;
 
  private:
+  bool hlo_profile_ = false;
   int device_ordinal_ = -1;
   Shape result_layout_;
   bool result_layout_set_ = false;
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index b2681d5e8b..ca91cf0d50 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -833,6 +833,19 @@ tensorflow::ImportNumpy();
     }
     Py_DECREF(o);
 
+    o = PyObject_GetAttrString($input, "hlo_profile");
+    if (o == NULL) {
+      return NULL;
+    }
+    if (o != Py_None) {
+      if (!PyBool_Check(o)) {
+        PyErr_SetString(PyExc_TypeError, "ExecutableBuildOptions.hlo_profile must be a bool or None.");
+        return NULL;
+      }
+      build_options.set_hlo_profile(o == Py_True);
+    }
+    Py_DECREF(o);
+
     o = PyObject_GetAttrString($input, "result_shape");
     if (o == nullptr) {
       return nullptr;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index 90cda42f32..d747a0b65c 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -320,6 +320,7 @@ class CompileOptions(object):
 
   def __init__(self):
     self.generate_hlo_graph = None
+    self.hlo_profile = False
 
 
 def transfer_to_infeed(value, replica_number=None):
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 07f989d4fa..74aa6eaa17 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -119,6 +119,8 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
   }
 
   ExecutionOptions execution_options = CreateDefaultExecutionOptions();
+  execution_options.mutable_debug_options()->set_xla_hlo_profile(
+      build_options.hlo_profile());
   if (build_options.generate_hlo_graph().has_value()) {
     execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
         build_options.generate_hlo_graph().value());
-- 
GitLab


From 784d7736bdfc9c9d06098db56d4dc883bc12f0ea Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 12:55:18 -0700
Subject: [PATCH 121/960] BREAKING_CHANGE: Remove SigmoidCentered bijector.  -
 SoftmaxCentered solely works on vector events, and supports broadcasting.  -
 Sigmoid exists for event_ndims=0 cases.

PiperOrigin-RevId: 189380445
---
 tensorflow/contrib/distributions/BUILD        |  19 ---
 .../kernel_tests/bijectors/chain_test.py      |   8 +-
 .../kernel_tests/bijectors/invert_test.py     |   7 +-
 .../bijectors/sigmoid_centered_test.py        |  57 ---------
 .../bijectors/softmax_centered_test.py        |  56 +++------
 .../transformed_distribution_test.py          |  12 +-
 .../python/ops/bijectors/__init__.py          |   2 -
 .../python/ops/bijectors/sigmoid_centered.py  |  39 ------
 .../python/ops/bijectors/softmax_centered.py  | 114 +++++-------------
 .../python/ops/vector_diffeomixture.py        |   2 +-
 .../python/contrib.distributions.bijectors.md |   1 -
 11 files changed, 59 insertions(+), 258 deletions(-)
 delete mode 100644 tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_centered_test.py
 delete mode 100644 tensorflow/contrib/distributions/python/ops/bijectors/sigmoid_centered.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 6bd3f5f09b..e9c827a618 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -1105,25 +1105,6 @@ cuda_py_test(
     ],
 )
 
-cuda_py_test(
-    name = "sigmoid_centered_test",
-    size = "small",
-    srcs = ["python/kernel_tests/bijectors/sigmoid_centered_test.py"],
-    additional_deps = [
-        ":bijectors_py",
-        ":distributions_py",
-        "//third_party/py/numpy",
-        "@six_archive//:six",
-        "//tensorflow/contrib/linalg:linalg_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
 # Tests for SinhArcSinh bijector.  The file name has the extra "_bijector" to
 # avoid BUILD rule name conflicts with the distribution by the same name.
 cuda_py_test(
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py
index 20e7543084..a748acd667 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/chain_test.py
@@ -66,12 +66,10 @@ class ChainBijectorTest(test.TestCase):
   def testShapeGetters(self):
     with self.test_session():
       bijector = Chain([
-          SoftmaxCentered(
-              event_ndims=1, validate_args=True),
-          SoftmaxCentered(
-              event_ndims=0, validate_args=True)
+          SoftmaxCentered(validate_args=True),
+          SoftmaxCentered(validate_args=True),
       ])
-      x = tensor_shape.TensorShape([])
+      x = tensor_shape.TensorShape([1])
       y = tensor_shape.TensorShape([2 + 1])
       self.assertAllEqual(y, bijector.forward_event_shape(x))
       self.assertAllEqual(
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py
index 28e3e31354..58ba9cedb1 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/invert_test.py
@@ -37,8 +37,7 @@ class InvertBijectorTest(test.TestCase):
           bijectors.Exp(event_ndims=1),
           bijectors.Affine(shift=[0., 1.], scale_diag=[2., 3.]),
           bijectors.Softplus(event_ndims=1),
-          bijectors.SoftmaxCentered(event_ndims=1),
-          bijectors.SigmoidCentered(),
+          bijectors.SoftmaxCentered(),
       ]:
         rev = bijectors.Invert(fwd)
         self.assertEqual("_".join(["invert", fwd.name]), rev.name)
@@ -61,9 +60,9 @@ class InvertBijectorTest(test.TestCase):
 
   def testShapeGetters(self):
     with self.test_session():
-      bijector = bijectors.Invert(bijectors.SigmoidCentered(validate_args=True))
+      bijector = bijectors.Invert(bijectors.SoftmaxCentered(validate_args=True))
       x = tensor_shape.TensorShape([2])
-      y = tensor_shape.TensorShape([])
+      y = tensor_shape.TensorShape([1])
       self.assertAllEqual(y, bijector.forward_event_shape(x))
       self.assertAllEqual(
           y.as_list(),
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_centered_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_centered_test.py
deleted file mode 100644
index 4ff3f334cc..0000000000
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/sigmoid_centered_test.py
+++ /dev/null
@@ -1,57 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for Bijector."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import SigmoidCentered
-from tensorflow.python.platform import test
-
-
-class SigmoidCenteredBijectorTest(test.TestCase):
-  """Tests correctness of the Y = g(X) = (1 + exp(-X))^-1 transformation."""
-
-  def testBijector(self):
-    with self.test_session():
-      sigmoid = SigmoidCentered()
-      self.assertEqual("sigmoid_centered", sigmoid.name)
-      x = np.log([[2., 3, 4],
-                  [4., 8, 12]])
-      y = [[[2. / 3, 1. / 3],
-            [3. / 4, 1. / 4],
-            [4. / 5, 1. / 5]],
-           [[4. / 5, 1. / 5],
-            [8. / 9, 1. / 9],
-            [12. / 13, 1. / 13]]]
-      self.assertAllClose(y, sigmoid.forward(x).eval())
-      self.assertAllClose(x, sigmoid.inverse(y).eval())
-      self.assertAllClose(
-          -np.sum(np.log(y), axis=2),
-          sigmoid.inverse_log_det_jacobian(y).eval(),
-          atol=0.,
-          rtol=1e-7)
-      self.assertAllClose(
-          -sigmoid.inverse_log_det_jacobian(y).eval(),
-          sigmoid.forward_log_det_jacobian(x).eval(),
-          atol=0.,
-          rtol=1e-7)
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py
index 4a7679daad..cad4dd1ac8 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/softmax_centered_test.py
@@ -34,34 +34,9 @@ rng = np.random.RandomState(42)
 class SoftmaxCenteredBijectorTest(test.TestCase):
   """Tests correctness of the Y = g(X) = exp(X) / sum(exp(X)) transformation."""
 
-  def testBijectorScalar(self):
-    with self.test_session():
-      softmax = SoftmaxCentered()  # scalar by default
-      self.assertEqual("softmax_centered", softmax.name)
-      x = np.log([[2., 3, 4],
-                  [4., 8, 12]])
-      y = [[[2. / 3, 1. / 3],
-            [3. / 4, 1. / 4],
-            [4. / 5, 1. / 5]],
-           [[4. / 5, 1. / 5],
-            [8. / 9, 1. / 9],
-            [12. / 13, 1. / 13]]]
-      self.assertAllClose(y, softmax.forward(x).eval())
-      self.assertAllClose(x, softmax.inverse(y).eval())
-      self.assertAllClose(
-          -np.sum(np.log(y), axis=2),
-          softmax.inverse_log_det_jacobian(y).eval(),
-          atol=0.,
-          rtol=1e-7)
-      self.assertAllClose(
-          -softmax.inverse_log_det_jacobian(y).eval(),
-          softmax.forward_log_det_jacobian(x).eval(),
-          atol=0.,
-          rtol=1e-7)
-
   def testBijectorVector(self):
     with self.test_session():
-      softmax = SoftmaxCentered(event_ndims=1)
+      softmax = SoftmaxCentered()
       self.assertEqual("softmax_centered", softmax.name)
       x = np.log([[2., 3, 4], [4., 8, 12]])
       y = [[0.2, 0.3, 0.4, 0.1], [0.16, 0.32, 0.48, 0.04]]
@@ -80,7 +55,7 @@ class SoftmaxCenteredBijectorTest(test.TestCase):
 
   def testBijectorUnknownShape(self):
     with self.test_session():
-      softmax = SoftmaxCentered(event_ndims=1)
+      softmax = SoftmaxCentered()
       self.assertEqual("softmax_centered", softmax.name)
       x = array_ops.placeholder(shape=[2, None], dtype=dtypes.float32)
       real_x = np.log([[2., 3, 4], [4., 8, 12]])
@@ -106,24 +81,21 @@ class SoftmaxCenteredBijectorTest(test.TestCase):
 
   def testShapeGetters(self):
     with self.test_session():
-      for x, y, b in ((tensor_shape.TensorShape([]),
-                       tensor_shape.TensorShape([2]),
-                       SoftmaxCentered(
-                           event_ndims=0, validate_args=True)),
-                      (tensor_shape.TensorShape([4]),
-                       tensor_shape.TensorShape([5]),
-                       SoftmaxCentered(
-                           event_ndims=1, validate_args=True))):
-        self.assertAllEqual(y, b.forward_event_shape(x))
-        self.assertAllEqual(y.as_list(),
-                            b.forward_event_shape_tensor(x.as_list()).eval())
-        self.assertAllEqual(x, b.inverse_event_shape(y))
-        self.assertAllEqual(x.as_list(),
-                            b.inverse_event_shape_tensor(y.as_list()).eval())
+      x = tensor_shape.TensorShape([4])
+      y = tensor_shape.TensorShape([5])
+      bijector = SoftmaxCentered(validate_args=True)
+      self.assertAllEqual(y, bijector.forward_event_shape(x))
+      self.assertAllEqual(y.as_list(),
+                          bijector.forward_event_shape_tensor(
+                              x.as_list()).eval())
+      self.assertAllEqual(x, bijector.inverse_event_shape(y))
+      self.assertAllEqual(x.as_list(),
+                          bijector.inverse_event_shape_tensor(
+                              y.as_list()).eval())
 
   def testBijectiveAndFinite(self):
     with self.test_session():
-      softmax = SoftmaxCentered(event_ndims=1)
+      softmax = SoftmaxCentered()
       x = np.linspace(-50, 50, num=10).reshape(5, 2).astype(np.float32)
       # Make y values on the simplex with a wide range.
       y_0 = np.ones(5).astype(np.float32)
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
index af13553c32..f0ba1ec3eb 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/transformed_distribution_test.py
@@ -186,12 +186,14 @@ class TransformedDistributionTest(test.TestCase):
       standard_normal = ds.Normal(loc=0., scale=1.)
       multi_logit_normal = self._cls()(
           distribution=standard_normal,
-          bijector=softmax)
-      x = [[-np.log(3.), 0.],
-           [np.log(3), np.log(5)]]
+          bijector=softmax,
+          event_shape=[1])
+      x = [[[-np.log(3.)], [0.]],
+           [[np.log(3)], [np.log(5)]]]
       y = softmax.forward(x).eval()
-      expected_log_pdf = (stats.norm(loc=0., scale=1.).logpdf(x) -
-                          np.sum(np.log(y), axis=-1))
+      expected_log_pdf = (
+          np.squeeze(stats.norm(loc=0., scale=1.).logpdf(x)) -
+          np.sum(np.log(y), axis=-1))
       self.assertAllClose(expected_log_pdf,
                           multi_logit_normal.log_prob(y).eval())
       self.assertAllClose(
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
index 452f1caa30..bc6b02542e 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/__init__.py
@@ -35,7 +35,6 @@
 @@RealNVP
 @@Reshape
 @@Sigmoid
-@@SigmoidCentered
 @@SinhArcsinh
 @@SoftmaxCentered
 @@Softplus
@@ -72,7 +71,6 @@ from tensorflow.contrib.distributions.python.ops.bijectors.power_transform impor
 from tensorflow.contrib.distributions.python.ops.bijectors.real_nvp import *
 from tensorflow.contrib.distributions.python.ops.bijectors.reshape import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid import *
-from tensorflow.contrib.distributions.python.ops.bijectors.sigmoid_centered import *
 from tensorflow.contrib.distributions.python.ops.bijectors.sinh_arcsinh import *
 from tensorflow.contrib.distributions.python.ops.bijectors.softmax_centered import *
 from tensorflow.contrib.distributions.python.ops.bijectors.softplus import *
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid_centered.py
deleted file mode 100644
index 223bc9d042..0000000000
--- a/tensorflow/contrib/distributions/python/ops/bijectors/sigmoid_centered.py
+++ /dev/null
@@ -1,39 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""SigmoidCentered bijector."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.contrib.distributions.python.ops.bijectors import softmax_centered
-
-
-__all__ = [
-    "SigmoidCentered",
-]
-
-
-class SigmoidCentered(softmax_centered.SoftmaxCentered):
-  """Bijector which computes Y = g(X) = exp([X 0]) / (1 + exp(-X)).
-
-  Equivalent to: `bijector.SoftmaxCentered(event_ndims=0)`.
-
-  See `bijector.SoftmaxCentered` for more details.
-  """
-
-  def __init__(self, validate_args=False, name="sigmoid_centered"):
-    super(SigmoidCentered, self).__init__(
-        event_ndims=0, validate_args=validate_args, name=name)
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
index 24add40445..dc94fd0a38 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/softmax_centered.py
@@ -19,10 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.distributions.python.ops import distribution_util
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
@@ -45,17 +42,14 @@ class SoftmaxCentered(bijector.Bijector):
   e.g., `softmax(x) = exp(x-c) / sum(exp(x-c))` where `c` is the implicit last
   coordinate.
 
-  Because we append a coordinate, this bijector only supports `event_ndim in [0,
-  1]`, i.e., scalars and vectors.
-
   Example Use:
 
   ```python
-  bijector.SoftmaxCentered(event_ndims=1).forward(tf.log([2, 3, 4]))
+  bijector.SoftmaxCentered().forward(tf.log([2, 3, 4]))
   # Result: [0.2, 0.3, 0.4, 0.1]
   # Extra result: 0.1
 
-  bijector.SoftmaxCentered(event_ndims=1).inverse([0.2, 0.3, 0.4, 0.1])
+  bijector.SoftmaxCentered().inverse([0.2, 0.3, 0.4, 0.1])
   # Result: tf.log([2, 3, 4])
   # Extra coordinate removed.
   ```
@@ -67,82 +61,47 @@ class SoftmaxCentered(bijector.Bijector):
   """
 
   def __init__(self,
-               event_ndims=0,
                validate_args=False,
                name="softmax_centered"):
     self._graph_parents = []
     self._name = name
-    with self._name_scope("init", values=[event_ndims]):
-      event_ndims = ops.convert_to_tensor(event_ndims, name="event_ndims")
-      event_ndims = tensor_util.constant_value(event_ndims)
-      if event_ndims is None or event_ndims not in [0, 1]:
-        raise ValueError("`event_ndims` must be a TF constant which is 0 or 1")
-    self._static_event_ndims = event_ndims
     super(SoftmaxCentered, self).__init__(
-        event_ndims=event_ndims,
+        event_ndims=1,
         validate_args=validate_args,
         name=name)
 
   def _forward_event_shape(self, input_shape):
-    if input_shape.ndims is None:
+    if input_shape.ndims is None or input_shape[-1] is None:
       return input_shape
-    if input_shape.ndims != self._static_event_ndims:
-      raise ValueError("input_shape.dims = %d != %d" %
-                       (input_shape.ndims, self._static_event_ndims))
-    if input_shape.ndims == 0:
-      return tensor_shape.TensorShape([2])
-    if input_shape.ndims == 1:
-      return tensor_shape.TensorShape(input_shape[0] + 1)
-    # Unreachable code:
-    raise ValueError("event_ndims = %d must be 0 or 1" % input_shape.ndims)
+    return tensor_shape.TensorShape([input_shape[-1] + 1])
 
   def _forward_event_shape_tensor(self, input_shape):
-    ndims = array_ops.shape(input_shape)
-    if self.validate_args:
-      # It is not possible for a negative shape so we need only check <= 1.
-      is_zero_or_one = check_ops.assert_equal(
-          ndims, 0 if self._static_event_ndims == 0 else 1,
-          message="event_ndims must be 0 or 1")
-      ndims = control_flow_ops.with_dependencies([is_zero_or_one], ndims)
-    if self._static_event_ndims == 0:
-      return ops.convert_to_tensor(
-          [2], dtype=dtypes.int32, name="output_shape")
-    return input_shape + 1
+    return (input_shape[-1] + 1)[..., array_ops.newaxis]
 
   def _inverse_event_shape(self, output_shape):
-    if output_shape.ndims is None:
+    if output_shape.ndims is None or output_shape[-1] is None:
       return output_shape
-    if output_shape.ndims != 1:
-      raise ValueError("output_shape.ndims = %d != 1" % output_shape.ndims)
-    if self._static_event_ndims == 0:
-      return tensor_shape.TensorShape([])
-    return tensor_shape.TensorShape(output_shape[0] - 1)
+    if output_shape[-1] <= 1:
+      raise ValueError("output_shape[-1] = %d <= 1" % output_shape[-1])
+    return tensor_shape.TensorShape([output_shape[-1] - 1])
 
   def _inverse_event_shape_tensor(self, output_shape):
-    ndims = array_ops.shape(output_shape)[0]
     if self.validate_args:
       # It is not possible for a negative shape so we need only check <= 1.
-      is_one = check_ops.assert_equal(
-          ndims, 1, message="event_ndims must be 1")
-      ndims = control_flow_ops.with_dependencies([is_one], ndims)
-    if self._static_event_ndims == 0:
-      return ops.convert_to_tensor([], dtype=dtypes.int32, name="output_shape")
-    return array_ops.expand_dims(output_shape[0] - 1, dim=0)
+      is_greater_one = check_ops.assert_greater(
+          output_shape[-1], 1, message="Need last dimension greater than 1.")
+      output_shape = control_flow_ops.with_dependencies(
+          [is_greater_one], output_shape)
+    return (output_shape[-1] - 1)[..., array_ops.newaxis]
 
   def _forward(self, x):
     # Pad the last dim with a zeros vector. We need this because it lets us
     # infer the scale in the inverse function.
-    y = array_ops.expand_dims(x, dim=-1) if self._static_event_ndims == 0 else x
-    y = distribution_util.pad(y, axis=-1, back=True)
+    y = distribution_util.pad(x, axis=-1, back=True)
 
     # Set shape hints.
     if x.shape.ndims is not None:
-      shape = x.shape.as_list()
-      if self._static_event_ndims == 0:
-        shape += [2]
-      elif shape[-1] is not None:
-        shape[-1] += 1
-      shape = tensor_shape.TensorShape(shape)
+      shape = x.shape[:-1].concatenate(x.shape[-1] + 1)
       y.shape.assert_is_compatible_with(shape)
       y.set_shape(shape)
 
@@ -167,17 +126,9 @@ class SoftmaxCentered(bijector.Bijector):
     log_normalization = (-x[..., -1])[..., array_ops.newaxis]
     x = x[..., :-1] + log_normalization
 
-    if self._static_event_ndims == 0:
-      x = array_ops.squeeze(x, squeeze_dims=-1)
-
     # Set shape hints.
     if y.shape.ndims is not None:
-      shape = y.shape.as_list()
-      if self._static_event_ndims == 0:
-        shape = shape[:-1]
-      elif shape[-1] is not None:
-        shape[-1] -= 1
-      shape = tensor_shape.TensorShape(shape)
+      shape = y.shape[:-1].concatenate(y.shape[-1] - 1)
       x.shape.assert_is_compatible_with(shape)
       x.set_shape(shape)
 
@@ -203,19 +154,16 @@ class SoftmaxCentered(bijector.Bijector):
     return -math_ops.reduce_sum(math_ops.log(y), axis=-1)
 
   def _forward_log_det_jacobian(self, x):
-    if self._static_event_ndims == 0:
-      return x - 2. * nn_ops.softplus(x)
-    else:
-      # This code is similar to nn_ops.log_softmax but different because we have
-      # an implicit zero column to handle. I.e., instead of:
-      #   reduce_sum(logits - reduce_sum(exp(logits), dim))
-      # we must do:
-      #   log_normalization = 1 + reduce_sum(exp(logits))
-      #   -log_normalization + reduce_sum(logits - log_normalization)
-      log_normalization = nn_ops.softplus(
-          math_ops.reduce_logsumexp(x, axis=-1, keep_dims=True))
-      fldj = (-log_normalization +
-              math_ops.reduce_sum(x - log_normalization,
-                                  axis=-1,
-                                  keep_dims=True))
-      return array_ops.squeeze(fldj, squeeze_dims=-1)
+    # This code is similar to nn_ops.log_softmax but different because we have
+    # an implicit zero column to handle. I.e., instead of:
+    #   reduce_sum(logits - reduce_sum(exp(logits), dim))
+    # we must do:
+    #   log_normalization = 1 + reduce_sum(exp(logits))
+    #   -log_normalization + reduce_sum(logits - log_normalization)
+    log_normalization = nn_ops.softplus(
+        math_ops.reduce_logsumexp(x, axis=-1, keep_dims=True))
+    fldj = (-log_normalization +
+            math_ops.reduce_sum(x - log_normalization,
+                                axis=-1,
+                                keep_dims=True))
+    return array_ops.squeeze(fldj, squeeze_dims=-1)
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 0c747f8e68..3208ecdf64 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -181,7 +181,7 @@ def quadrature_scheme_softmaxnormal_quantiles(
       edges = array_ops.reshape(edges, shape=array_ops.concat([
           [-1], array_ops.ones([batch_ndims], dtype=dtypes.int32)], axis=0))
       quantiles = dist.quantile(edges)
-      quantiles = SoftmaxCentered(event_ndims=1).forward(quantiles)
+      quantiles = SoftmaxCentered().forward(quantiles)
       # Cyclically permute left by one.
       perm = array_ops.concat([
           math_ops.range(1, 1 + batch_ndims), [0]], axis=0)
diff --git a/tensorflow/docs_src/api_guides/python/contrib.distributions.bijectors.md b/tensorflow/docs_src/api_guides/python/contrib.distributions.bijectors.md
index 0ce187b329..e169897f31 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.distributions.bijectors.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.distributions.bijectors.md
@@ -28,6 +28,5 @@ To apply a `Bijector`, use `distributions.TransformedDistribution`.
 *   @{tf.contrib.distributions.bijectors.Inline}
 *   @{tf.contrib.distributions.bijectors.Invert}
 *   @{tf.contrib.distributions.bijectors.PowerTransform}
-*   @{tf.contrib.distributions.bijectors.SigmoidCentered}
 *   @{tf.contrib.distributions.bijectors.SoftmaxCentered}
 *   @{tf.contrib.distributions.bijectors.Softplus}
-- 
GitLab


From dcbac5e690598f7fe80dbcdc264f1e5a851c8499 Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Fri, 16 Mar 2018 12:56:26 -0700
Subject: [PATCH 122/960] [XLA] BF16 conversion folding for CRS; remove no-op
 conversions in propagation.

If CRS has tuple output, it needs special handling in conversion folding.
BF16 propagation could result in BF16->BF16 conversions, which can be removed.

PiperOrigin-RevId: 189380578
---
 .../service/bfloat16_conversion_folding.cc    | 87 +++++++++++++++----
 .../bfloat16_conversion_folding_test.cc       | 51 ++++++++++-
 .../xla/service/bfloat16_propagation.cc       | 25 ++++++
 .../xla/service/bfloat16_propagation.h        |  5 ++
 .../xla/service/bfloat16_propagation_test.cc  | 40 +++++++++
 5 files changed, 189 insertions(+), 19 deletions(-)

diff --git a/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc b/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc
index 432448e9bb..08d0152e3c 100644
--- a/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_conversion_folding.cc
@@ -34,6 +34,9 @@ class BFloat16ConversionFoldingVisitor : public DfsHloVisitorWithDefault {
 
   Status DefaultAction(HloInstruction* hlo) override;
 
+  // Special handling for cross-replica-sum which can have a tuple output.
+  Status HandleCrossReplicaSum(HloInstruction* crs) override;
+
   static bool Run(HloComputation* computation,
                   const BFloat16Support* bfloat16_support) {
     BFloat16ConversionFoldingVisitor visitor(computation, bfloat16_support);
@@ -84,6 +87,25 @@ Status BFloat16ConversionFoldingVisitor::FoldOperandConversion(
   return Status::OK();
 }
 
+namespace {
+
+// Returns whether hlo has users and all users are conversions from F32 to BF16.
+bool AllUsersAreF32ToBF16Converts(const HloInstruction* hlo) {
+  if (hlo->user_count() == 0 || hlo->shape().element_type() != F32) {
+    return false;
+  }
+  for (const auto user : hlo->users()) {
+    if (user->opcode() == HloOpcode::kConvert &&
+        user->shape().element_type() == BF16) {
+      continue;
+    }
+    return false;
+  }
+  return true;
+}
+
+}  // namespace
+
 Status BFloat16ConversionFoldingVisitor::TryFoldBF16Conversions(
     HloInstruction* hlo) {
   std::vector<int64> bf16_to_f32_operands;
@@ -104,22 +126,9 @@ Status BFloat16ConversionFoldingVisitor::TryFoldBF16Conversions(
     }
   }
 
-  bool fold_output_conversion = hlo->user_count() > 0 &&
-                                hlo->shape().element_type() == F32 &&
-                                bfloat16_support_->SupportsBF16Output(*hlo) &&
-                                hlo != computation_->root_instruction();
-  if (fold_output_conversion) {
-    for (auto user : hlo->users()) {
-      if (user->opcode() == HloOpcode::kConvert &&
-          user->shape().element_type() == BF16) {
-        continue;
-      }
-      // We should not change the output type if any user is not a conversion
-      // from F32 to BF16.
-      fold_output_conversion = false;
-      break;
-    }
-  }
+  const bool fold_output_conversion =
+      AllUsersAreF32ToBF16Converts(hlo) &&
+      bfloat16_support_->SupportsBF16Output(*hlo);
 
   if (!bfloat16_support_->SupportsMixedPrecisions(*hlo)) {
     if (has_other_f32_operands ||
@@ -171,6 +180,52 @@ Status BFloat16ConversionFoldingVisitor::DefaultAction(HloInstruction* hlo) {
   return TryFoldBF16Conversions(hlo);
 }
 
+Status BFloat16ConversionFoldingVisitor::HandleCrossReplicaSum(
+    HloInstruction* crs) {
+  if (!ShapeUtil::IsTuple(crs->shape()) ||
+      !bfloat16_support_->SupportsMixedPrecisions(*crs)) {
+    return DefaultAction(crs);
+  }
+
+  // First use DefaultAction() to handle the operands. It can't handle
+  // tuple-shaped output.
+  TF_RETURN_IF_ERROR(DefaultAction(crs));
+
+  // Then do per-tuple-element handling on the output.
+  std::vector<std::vector<HloInstruction*>> per_tuple_element_gtes(
+      crs->operand_count());
+  for (auto user : crs->users()) {
+    if (user->opcode() != HloOpcode::kGetTupleElement) {
+      return Status::OK();
+    }
+    per_tuple_element_gtes[user->tuple_index()].push_back(user);
+  }
+
+  for (int64 i = 0; i < crs->operand_count(); ++i) {
+    // Fold conversions only when all the get-tuple-elements' users are
+    // conversions from F32 to BF16.
+    auto all_gte_users_are_bf16_convert = [&per_tuple_element_gtes, i]() {
+      for (auto gte : per_tuple_element_gtes[i]) {
+        if (!AllUsersAreF32ToBF16Converts(gte)) {
+          return false;
+        }
+      }
+      return true;
+    };
+    if (!all_gte_users_are_bf16_convert()) {
+      continue;
+    }
+
+    ShapeUtil::GetMutableSubshape(crs->mutable_shape(), {i})
+        ->set_element_type(BF16);
+    for (auto gte : per_tuple_element_gtes[i]) {
+      TF_RETURN_IF_ERROR(FoldOutputConversions(gte));
+    }
+  }
+
+  return Status::OK();
+}
+
 StatusOr<bool> BFloat16ConversionFolding::Run(HloModule* module) {
   XLA_VLOG_LINES(
       2, "BFloat16ConversionFolding::Run(), before:\n" + module->ToString());
diff --git a/tensorflow/compiler/xla/service/bfloat16_conversion_folding_test.cc b/tensorflow/compiler/xla/service/bfloat16_conversion_folding_test.cc
index cb37759439..28e71c2054 100644
--- a/tensorflow/compiler/xla/service/bfloat16_conversion_folding_test.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_conversion_folding_test.cc
@@ -37,7 +37,8 @@ class TestBFloat16Support : public BFloat16Support {
     if (hlo.opcode() == HloOpcode::kAdd ||
         hlo.opcode() == HloOpcode::kSubtract ||
         hlo.opcode() == HloOpcode::kTuple ||
-        hlo.opcode() == HloOpcode::kGetTupleElement) {
+        hlo.opcode() == HloOpcode::kGetTupleElement ||
+        hlo.opcode() == HloOpcode::kCrossReplicaSum) {
       return true;
     }
     return false;
@@ -47,7 +48,8 @@ class TestBFloat16Support : public BFloat16Support {
     if (hlo.opcode() == HloOpcode::kAdd ||
         hlo.opcode() == HloOpcode::kSubtract ||
         hlo.opcode() == HloOpcode::kTuple ||
-        hlo.opcode() == HloOpcode::kGetTupleElement) {
+        hlo.opcode() == HloOpcode::kGetTupleElement ||
+        hlo.opcode() == HloOpcode::kCrossReplicaSum) {
       return true;
     }
     return false;
@@ -55,7 +57,8 @@ class TestBFloat16Support : public BFloat16Support {
 
   bool SupportsMixedPrecisions(const HloInstruction& hlo) const override {
     if (hlo.opcode() == HloOpcode::kAdd || hlo.opcode() == HloOpcode::kTuple ||
-        hlo.opcode() == HloOpcode::kGetTupleElement) {
+        hlo.opcode() == HloOpcode::kGetTupleElement ||
+        hlo.opcode() == HloOpcode::kCrossReplicaSum) {
       return true;
     }
     return false;
@@ -206,4 +209,46 @@ TEST_F(BFloat16ConversionFoldingTest, DoNotFoldTuple) {
   EXPECT_EQ(tuple->operand(1), convert0);
 }
 
+TEST_F(BFloat16ConversionFoldingTest, FoldCrossReplicaSumTupleOutput) {
+  auto builder = HloComputation::Builder(TestName());
+  Shape f32_shape = ShapeUtil::MakeShape(F32, {2, 4});
+  Shape bf16_shape = ShapeUtil::MakeShape(BF16, {2, 4});
+
+  HloInstruction* a = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, bf16_shape, "a"));
+  HloInstruction* convert_a =
+      builder.AddInstruction(HloInstruction::CreateConvert(f32_shape, a));
+  HloInstruction* b = builder.AddInstruction(
+      HloInstruction::CreateParameter(1, f32_shape, "b"));
+
+  HloInstruction* crs =
+      builder.AddInstruction(HloInstruction::CreateCrossReplicaSum(
+          ShapeUtil::MakeTupleShape({f32_shape, f32_shape}), {convert_a, b}));
+  HloInstruction* gte_a = builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(f32_shape, crs, 0));
+  HloInstruction* gte_b = builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(f32_shape, crs, 1));
+  HloInstruction* convert_gte_b =
+      builder.AddInstruction(HloInstruction::CreateConvert(bf16_shape, gte_b));
+  HloInstruction* tuple = builder.AddInstruction(
+      HloInstruction::CreateTuple({gte_a, convert_gte_b}));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+
+  EXPECT_TRUE(FoldConversions(module.get()));
+
+  EXPECT_EQ(computation->root_instruction(), tuple);
+  EXPECT_EQ(tuple->operand(0), gte_a);
+  EXPECT_EQ(tuple->operand(1), gte_b);
+  EXPECT_EQ(gte_a->shape().element_type(), F32);
+  EXPECT_EQ(gte_b->shape().element_type(), BF16);
+  EXPECT_EQ(crs->operand(0), a);
+  EXPECT_EQ(crs->operand(1), b);
+  EXPECT_EQ(a->shape().element_type(), BF16);
+  EXPECT_EQ(b->shape().element_type(), F32);
+  EXPECT_EQ(ShapeUtil::GetSubshape(crs->shape(), {0}).element_type(), F32);
+  EXPECT_EQ(ShapeUtil::GetSubshape(crs->shape(), {1}).element_type(), BF16);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/bfloat16_propagation.cc b/tensorflow/compiler/xla/service/bfloat16_propagation.cc
index 531f36e8c5..7195c31d9c 100644
--- a/tensorflow/compiler/xla/service/bfloat16_propagation.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_propagation.cc
@@ -627,6 +627,27 @@ Status BFloat16Propagation::ResolveInconsistencyOfAliasingBuffers(
   return Status::OK();
 }
 
+Status BFloat16Propagation::RemoveNoopConversions(HloModule* module) {
+  for (auto computation : module->computations()) {
+    for (auto hlo : computation->MakeInstructionPostOrder()) {
+      if (hlo->opcode() != HloOpcode::kConvert) {
+        continue;
+      }
+      auto source = hlo->mutable_operand(0);
+      if (!ShapeUtil::Equal(source->shape(), hlo->shape())) {
+        continue;
+      }
+      const bool is_root = hlo == computation->root_instruction();
+      TF_RETURN_IF_ERROR(hlo->ReplaceAllUsesWith(source));
+      if (is_root) {
+        computation->set_root_instruction(source);
+      }
+      TF_RETURN_IF_ERROR(computation->RemoveInstructionAndUnusedOperands(hlo));
+    }
+  }
+  return Status::OK();
+}
+
 // The algorithm first does a forward pass (parameters to root) to determine a
 // set of instructions to consider using bfloat16, then does a backward pass to
 // determine the precisions of those instructions according to the need of
@@ -677,6 +698,10 @@ StatusOr<bool> BFloat16Propagation::Run(HloModule* module) {
   // defining instruction's shape has changed. So we need to adjust the output
   // shapes of instructions according to the HLO values they refer to.
   TF_RETURN_IF_ERROR(ResolveInconsistencyOfAliasingBuffers(module));
+
+  // This pass could have turned an F32 -> BF16 conversion to a no-op (BF16 ->
+  // BF16), so we remove them now.
+  TF_RETURN_IF_ERROR(RemoveNoopConversions(module));
   return true;
 }
 
diff --git a/tensorflow/compiler/xla/service/bfloat16_propagation.h b/tensorflow/compiler/xla/service/bfloat16_propagation.h
index 89a5ac5db1..1744e9db90 100644
--- a/tensorflow/compiler/xla/service/bfloat16_propagation.h
+++ b/tensorflow/compiler/xla/service/bfloat16_propagation.h
@@ -133,6 +133,11 @@ class BFloat16Propagation : public HloPassInterface {
   // by the given HLO.
   void AdjustCalledComputationRoot(HloInstruction* hlo);
 
+  // ***************************
+  // Removes no-op conversions (same source and target shapes) that can be
+  // produced this pass.
+  Status RemoveNoopConversions(HloModule* module);
+
   // ***************************
   // Functions called and state used by two or more passes.
 
diff --git a/tensorflow/compiler/xla/service/bfloat16_propagation_test.cc b/tensorflow/compiler/xla/service/bfloat16_propagation_test.cc
index 5950b004b3..88f8301416 100644
--- a/tensorflow/compiler/xla/service/bfloat16_propagation_test.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_propagation_test.cc
@@ -617,4 +617,44 @@ TEST_F(BFloat16PropagationTest, DoNotPropagateWhilesCallingSameComputation) {
   EXPECT_EQ(computation->root_instruction(), dot);
 }
 
+// Tests that if this pass turns an F32 -> BF16 conversion into a no-op (BF16 ->
+// BF16 conversion), then it will remove that conversion.
+TEST_F(BFloat16PropagationTest, NoopConversionRemoved) {
+  auto builder = HloComputation::Builder(TestName());
+  Shape f32_shape = ShapeUtil::MakeShape(F32, {4, 4});
+  Shape bf16_shape = ShapeUtil::MakeShape(BF16, {4, 4});
+
+  HloInstruction* param = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, f32_shape, "param"));
+  HloInstruction* add0 = builder.AddInstruction(
+      HloInstruction::CreateBinary(f32_shape, HloOpcode::kAdd, param, param));
+  HloInstruction* add1 = builder.AddInstruction(
+      HloInstruction::CreateBinary(f32_shape, HloOpcode::kAdd, param, param));
+  HloInstruction* tuple =
+      builder.AddInstruction(HloInstruction::CreateTuple({add0, add1}));
+  HloInstruction* gte0 = builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(f32_shape, tuple, 0));
+  HloInstruction* gte1 = builder.AddInstruction(
+      HloInstruction::CreateGetTupleElement(f32_shape, tuple, 1));
+  HloInstruction* convert0 =
+      builder.AddInstruction(HloInstruction::CreateConvert(bf16_shape, gte0));
+  HloInstruction* convert1 =
+      builder.AddInstruction(HloInstruction::CreateConvert(bf16_shape, gte1));
+  HloInstruction* add2 = builder.AddInstruction(HloInstruction::CreateBinary(
+      bf16_shape, HloOpcode::kAdd, convert0, convert1));
+
+  auto module = CreateNewModule();
+  auto computation = module->AddEntryComputation(builder.Build());
+
+  EXPECT_TRUE(PropagatePrecision(module.get()));
+
+  EXPECT_EQ(computation->root_instruction(), add2);
+  EXPECT_EQ(add2->operand(0), gte0);
+  EXPECT_EQ(add2->operand(1), gte1);
+  EXPECT_EQ(gte0->shape().element_type(), BF16);
+  EXPECT_EQ(gte1->shape().element_type(), BF16);
+  EXPECT_EQ(add0->shape().element_type(), BF16);
+  EXPECT_EQ(add1->shape().element_type(), BF16);
+}
+
 }  // namespace xla
-- 
GitLab


From a5dd6369841a2ea9ecd6e2a56416859afb0c7716 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 13:00:31 -0700
Subject: [PATCH 123/960] Move if_op kernel to
 //third_party/tensorflow/compiler/tf2xla/kernels

PiperOrigin-RevId: 189381067
---
 tensorflow/compiler/tf2xla/kernels/BUILD    |  17 ++
 tensorflow/compiler/tf2xla/kernels/if_op.cc | 226 ++++++++++++++++++++
 tensorflow/compiler/tf2xla/kernels/if_op.h  |  59 +++++
 3 files changed, 302 insertions(+)
 create mode 100644 tensorflow/compiler/tf2xla/kernels/if_op.cc
 create mode 100644 tensorflow/compiler/tf2xla/kernels/if_op.h

diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index d2fa933cf9..0bbfe86de3 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -93,6 +93,7 @@ tf_kernel_library(
         "shape_util.h",
     ],
     deps = [
+        ":if_op",
         ":while_op",
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:xla_compiler",
@@ -154,6 +155,22 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "if_op",
+    srcs = ["if_op.cc"],
+    hdrs = ["if_op.h"],
+    deps = [
+        "//tensorflow/compiler/tf2xla:common",
+        "//tensorflow/compiler/tf2xla:xla_compiler",
+        "//tensorflow/compiler/tf2xla/ops:functional_ops",
+        "//tensorflow/compiler/xla:literal_util",
+        "//tensorflow/compiler/xla/client:computation_builder",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
 # Kernels that only work on CPU, because they use XLA custom calls.
 # Only link this when using the CPU backend for XLA.
 tf_kernel_library(
diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.cc b/tensorflow/compiler/tf2xla/kernels/if_op.cc
new file mode 100644
index 0000000000..eefbe55c81
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/kernels/if_op.cc
@@ -0,0 +1,226 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/tf2xla/kernels/if_op.h"
+
+#include "tensorflow/compiler/tf2xla/shape_util.h"
+#include "tensorflow/compiler/tf2xla/xla_context.h"
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+
+namespace tensorflow {
+
+XlaIfOp::XlaIfOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+  const NameAttrList* name_attr;
+  OP_REQUIRES_OK(ctx, ctx->GetAttr("then_branch", &name_attr));
+  then_branch_ = *name_attr;
+  OP_REQUIRES_OK(ctx, ctx->GetAttr("else_branch", &name_attr));
+  else_branch_ = *name_attr;
+
+  OP_REQUIRES_OK(ctx, ctx->GetAttr("Tcond", &cond_type_));
+  OP_REQUIRES_OK(ctx, ctx->GetAttr("Tin", &input_types_));
+  OP_REQUIRES_OK(ctx, ctx->GetAttr("Tout", &output_types_));
+}
+
+// TODO(b/35949885): There is duplication here with the handling of the
+// while_op. Refactor the common code out/rework.
+void XlaIfOp::Compile(XlaOpKernelContext* ctx) {
+  xla::ComputationBuilder* b = ctx->builder();
+
+  OP_REQUIRES(ctx, cond_type_ == DT_BOOL,
+              errors::InvalidArgument(
+                  "Condition argument must be a boolean for XLA compilation"));
+  OP_REQUIRES(ctx, TensorShapeUtils::IsScalar(ctx->InputShape(0)),
+              errors::InvalidArgument(
+                  "Condition argument must be a scalar for XLA compilation"));
+
+  VLOG(1) << "Building If: " << input_types_.size() << " inputs";
+
+  std::vector<xla::ComputationDataHandle> inputs(input_types_.size());
+  std::vector<XlaCompiler::Argument> arguments(input_types_.size());
+  for (int i = 0; i < input_types_.size(); ++i) {
+    XlaCompiler::Argument& arg = arguments[i];
+    DataType type = ctx->input_type(i + 1);
+    if (type == DT_RESOURCE) {
+      XlaResource* resource;
+      OP_REQUIRES_OK(ctx, ctx->GetResourceInput(i + 1, &resource));
+
+      arg.initialized = resource->initialized();
+      arg.kind = XlaCompiler::Argument::kResource;
+      arg.resource_kind = resource->kind();
+      OP_REQUIRES_OK(ctx, resource->Pack(&inputs[i], b));
+
+      arg.type = resource->type();
+      arg.shape = resource->shape();
+      OP_REQUIRES(ctx, arg.initialized,
+                  errors::Unimplemented("Uninitialized arguments: ", arg.name));
+      arg.tensor_array_size = resource->tensor_array_size();
+      for (const auto& gradient : resource->tensor_array_gradients()) {
+        arg.tensor_array_gradients.insert(gradient.first);
+      }
+      arg.name = resource->name();
+      VLOG(2) << "Resource " << resource->name()
+              << " type: " << DataTypeString(arg.type)
+              << " shape: " << arg.shape.DebugString()
+              << " initialized: " << arg.initialized;
+    } else {
+      arg.kind = XlaCompiler::Argument::kParameter;
+      arg.type = input_types_[i];
+      arg.shape = ctx->InputShape(i + 1);
+      inputs[i] = ctx->Input(i + 1);
+      VLOG(2) << "Arg type: " << DataTypeString(arg.type)
+              << " shape: " << arg.shape.DebugString();
+    }
+  }
+
+  // Compile both branches of the conditional.
+  XlaCompiler::CompileOptions options;
+  options.use_tuple_arg = true;
+  options.resolve_compile_time_constants = false;
+  options.return_updated_values_for_all_resources = true;
+  options.is_entry_computation = false;
+  XlaCompiler* compiler = ctx->compiler();
+
+  XlaCompiler::CompilationResult then_result;
+  OP_REQUIRES_OK(ctx, compiler->CompileFunction(options, then_branch_,
+                                                arguments, &then_result));
+  XlaCompiler::CompilationResult else_result;
+  OP_REQUIRES_OK(ctx, compiler->CompileFunction(options, else_branch_,
+                                                arguments, &else_result));
+
+  for (XlaCompiler::CompilationResult* result : {&then_result, &else_result}) {
+    for (const XlaCompiler::ResourceUpdate& update : result->resource_updates) {
+      XlaResource* resource;
+      OP_REQUIRES_OK(ctx,
+                     ctx->GetResourceInput(update.input_index + 1, &resource));
+      XlaCompiler::Argument& arg = arguments[update.input_index];
+
+      // Add any TensorArray gradients touched by the then/else computation to
+      // the enclosing graph.
+      for (const string& grad_source : update.tensor_array_gradients_accessed) {
+        VLOG(5) << "TensorArray " << resource->name() << " accessed gradient "
+                << grad_source;
+        XlaResource* gradient;
+        OP_REQUIRES_OK(ctx, resource->GetOrCreateTensorArrayGradient(
+                                grad_source, b, &gradient));
+      }
+      // Add all of the TensorArray gradients to the argument. For simplicity,
+      // we always pass all known gradients.
+      for (const auto& gradient : resource->tensor_array_gradients()) {
+        arg.tensor_array_gradients.insert(gradient.first);
+      }
+    }
+  }
+
+  // Check that both branches have identical input shapes.
+  OP_REQUIRES(ctx, then_result.xla_input_shapes.size() == 1,
+              errors::FailedPrecondition("Expected one input shape"));
+  xla::Shape then_input_shape = then_result.xla_input_shapes[0];
+  OP_REQUIRES(ctx, xla::ShapeUtil::IsTuple(then_input_shape),
+              errors::FailedPrecondition("Expected tuple shape"));
+  OP_REQUIRES(ctx, else_result.xla_input_shapes.size() == 1,
+              errors::FailedPrecondition("Expected one input shape"));
+  xla::Shape else_input_shape = else_result.xla_input_shapes[0];
+  OP_REQUIRES(ctx, xla::ShapeUtil::IsTuple(else_input_shape),
+              errors::FailedPrecondition("Expected tuple shape"));
+  OP_REQUIRES(ctx,
+              xla::ShapeUtil::Compatible(then_input_shape, else_input_shape),
+              errors::InvalidArgument(
+                  "Input shapes of then and else branches do not match: ",
+                  xla::ShapeUtil::HumanString(then_input_shape), " vs. ",
+                  xla::ShapeUtil::HumanString(else_input_shape)));
+
+  // Check that both branches have identical output shapes.
+  OP_REQUIRES(
+      ctx,
+      xla::ShapeUtil::Compatible(then_result.xla_output_shape,
+                                 else_result.xla_output_shape),
+      errors::InvalidArgument(
+          "Output shapes of then and else branches do not match: ",
+          xla::ShapeUtil::HumanString(then_result.xla_output_shape), " vs. ",
+          xla::ShapeUtil::HumanString(else_result.xla_output_shape)));
+
+  VLOG(2) << "Input shape: " << xla::ShapeUtil::HumanString(then_input_shape);
+  VLOG(2) << "Output shape: "
+          << xla::ShapeUtil::HumanString(then_result.xla_output_shape);
+
+  // We set return_updated_values_for_all_resources=true and we pass the same
+  // arguments to both computations, so the resource update count must match.
+  OP_REQUIRES(ctx,
+              then_result.resource_updates.size() ==
+                  else_result.resource_updates.size(),
+              errors::FailedPrecondition(
+                  "Different number of resources in then and else branch"));
+  for (int i = 0; i < then_result.resource_updates.size(); ++i) {
+    const auto& lhs = then_result.resource_updates[i];
+    const auto& rhs = else_result.resource_updates[i];
+    bool equal = lhs.input_index == rhs.input_index && lhs.shape == rhs.shape &&
+                 lhs.tensor_array_gradients_accessed ==
+                     rhs.tensor_array_gradients_accessed;
+    OP_REQUIRES(
+        ctx, equal,
+        errors::FailedPrecondition(
+            "Mismatch in resource of then and else branch for resource ", i));
+  }
+
+  xla::ComputationDataHandle outputs =
+      b->Conditional(ctx->Input(0), b->Tuple(inputs), *then_result.computation,
+                     b->Tuple(inputs), *else_result.computation);
+  // Sets non-variable outputs.
+  for (int i = 0; i < output_types_.size(); ++i) {
+    if (ctx->input_type(i) != DT_RESOURCE) {
+      xla::ComputationDataHandle output_handle = b->GetTupleElement(outputs, i);
+      if (VLOG_IS_ON(2)) {
+        LOG(INFO) << "Setting output " << i;
+        auto shape_or = b->GetShape(output_handle);
+        if (shape_or.ok()) {
+          LOG(INFO) << "Shape for output " << i << ": "
+                    << xla::ShapeUtil::HumanString(*shape_or.ValueOrDie());
+        } else {
+          LOG(INFO) << "Shape unknown for output " << i;
+        }
+      }
+      ctx->SetOutput(i, output_handle);
+    }
+  }
+
+  // Updates the values of any resource variables modified by the conditional
+  // bodies.
+  for (XlaCompiler::CompilationResult* result : {&then_result, &else_result}) {
+    for (int i = 0; i < result->resource_updates.size(); ++i) {
+      const XlaCompiler::ResourceUpdate& update = result->resource_updates[i];
+      XlaResource* resource;
+      OP_REQUIRES_OK(ctx,
+                     ctx->GetResourceInput(update.input_index + 1, &resource));
+      if (update.modified) {
+        int pos = result->outputs.size() + i;
+        OP_REQUIRES_OK(ctx,
+                       resource->SetFromPack(
+                           arguments[update.input_index].tensor_array_gradients,
+                           b->GetTupleElement(outputs, pos), b));
+      }
+      VLOG(2) << "If variable: pos: " << update.input_index
+              << " name: " << resource->name()
+              << " modified: " << update.modified
+              << " type: " << DataTypeString(update.type)
+              << " shape: " << update.shape.DebugString();
+    }
+  }
+  VLOG(1) << "Done building If";
+}
+
+REGISTER_XLA_OP(Name("XlaIf").AllowResourceTypes(), XlaIfOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/if_op.h b/tensorflow/compiler/tf2xla/kernels/if_op.h
new file mode 100644
index 0000000000..f9bc98a198
--- /dev/null
+++ b/tensorflow/compiler/tf2xla/kernels/if_op.h
@@ -0,0 +1,59 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_TF2XLA_KERNELS_IF_OP_H_
+#define TENSORFLOW_COMPILER_TF2XLA_KERNELS_IF_OP_H_
+
+#include "tensorflow/compiler/tf2xla/xla_op_kernel.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
+
+namespace tensorflow {
+
+// This TensorFlow op provides a functional conditional primitive.
+//
+// The outputs of the then/else branches must agree on the number, types, and
+// shapes of the Tensors carried around the two bodies.
+//
+// Computations in then/else bodies may read from and write to resource
+// variables.
+// Resource variables may be passed as arguments to the then/else function's
+// bodies. The XlaCompiler converts resource variable arguments
+// into parameters to the XLA computation and moves them to the end of the
+// parameter list, and by using the `return_updated_values_for_all_variables`
+// we ensure that all variables that appear in the input also appear at the
+// end of the then/else bodies output. This ensures the then/else bodies output
+// signatures match.
+//
+// It is the user's responsibility to ensure that each non-variable _Arg matches
+// the corresponding _Retval.
+class XlaIfOp : public XlaOpKernel {
+ public:
+  explicit XlaIfOp(OpKernelConstruction* ctx);
+
+  void Compile(XlaOpKernelContext* ctx) override;
+
+ private:
+  TF_DISALLOW_COPY_AND_ASSIGN(XlaIfOp);
+
+  NameAttrList then_branch_;
+  NameAttrList else_branch_;
+  DataType cond_type_;
+  DataTypeVector input_types_;
+  DataTypeVector output_types_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_TF2XLA_KERNELS_IF_OP_H_
-- 
GitLab


From a243f6e0299c733935d9e11a5f5eeafd2fe929ce Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Fri, 16 Mar 2018 13:11:36 -0700
Subject: [PATCH 124/960] Estimate prediction size and error out if it is
 larger than protobuf limit.

PiperOrigin-RevId: 189382429
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 31 ++++++++++++++-----
 1 file changed, 24 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index b3a7a4bd8d..32f15e60cd 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -25,6 +25,7 @@ import threading
 import time
 import traceback
 
+import numpy as np
 import six
 from six.moves import queue as Queue  # pylint: disable=redefined-builtin
 from six.moves import xrange  # pylint: disable=redefined-builtin
@@ -69,6 +70,7 @@ _TPU_ESTIMATOR = 'tpu_estimator'
 _ITERATIONS_PER_LOOP_VAR = 'iterations_per_loop'
 _BATCH_SIZE_KEY = 'batch_size'
 _CROSS_REPLICA_SUM_OP = 'CrossReplicaSum'
+_ONE_GIGABYTE = 1024 * 1024 * 1024
 
 _RESERVED_PARAMS_KEYS = [_BATCH_SIZE_KEY]
 
@@ -2083,6 +2085,10 @@ class TPUEstimator(estimator_lib.Estimator):
           host_ops = host_call_ret['host_call']
 
         predictions = host_call_ret['predictions']
+        _verify_cross_hosts_transfer_size(
+            predictions, message=(
+                'The estimated size for TPUEstimatorSpec.predictions is too '
+                'large.'))
         stopping_signals = host_call_ret['signals']
 
         with ops.control_dependencies(host_ops):
@@ -2096,13 +2102,6 @@ class TPUEstimator(estimator_lib.Estimator):
                                                      host_ops),
         ] + input_hooks
 
-        # TODO(b/73813593): Delete this logging once the bug is resolved.
-        logging.info(
-            'If the Tensors in TPUEstimatorSpec.predictions dict are large, '
-            'you might observe the TPU program getting stuck (b/73813593). '
-            'Consider using small Tensors in the predictions dict to verify '
-            'the issue and report on the bug.')
-
         return model_fn_lib.EstimatorSpec(
             mode,
             prediction_hooks=hooks,
@@ -2515,3 +2514,21 @@ class _SignalsHelper(object):
   @staticmethod
   def as_tensor_list(signals):
     return [signals[key] for key in sorted(signals.iterkeys())]
+
+
+def _verify_cross_hosts_transfer_size(tensor_dict, message):
+  total_size = 0
+  tensor_structure = {}
+  for key, tensor in tensor_dict.items():
+    shape = tensor.shape
+    size = np.product(shape) * tensor.dtype.size
+    tensor_structure[key] = shape
+    total_size += size
+  if total_size >= _ONE_GIGABYTE:
+    raise ValueError(
+        '{} The transfer size is larger than the protobuf limit. Please '
+        'consider to use Tensors with smaller shapes or reduce batch '
+        'size. Given:\n'
+        '{}'.format(message, '\n'.join([
+            ' -- Key: {}, Shape: {}'.format(k, v)
+            for k, v in tensor_structure.items()])))
-- 
GitLab


From 77c17f79399c4444ad49ac10f02e141c266c740e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 13:43:12 -0700
Subject: [PATCH 125/960] Fixed a typo.

PiperOrigin-RevId: 189386932
---
 tensorflow/contrib/lite/builtin_ops.h                          | 3 +--
 tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc | 3 +--
 2 files changed, 2 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 2218ea8eac..e4652a3e70 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -24,8 +24,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
-// ops.
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real built-in ops.
 typedef enum {
   kTfLiteBuiltinAdd = 0,
   kTfLiteBuiltinAveragePool2d = 1,
diff --git a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
index 08bcfe4516..ac408d2f94 100644
--- a/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
+++ b/tensorflow/contrib/lite/schema/builtin_ops_header/generator.cc
@@ -46,8 +46,7 @@ extern "C" {
 #endif  // __cplusplus
 
 // The enum for builtin operators.
-// Note: CUSTOM and DELEGATE are 2 special ops which are not real biultin
-// ops.
+// Note: CUSTOM and DELEGATE are 2 special ops which are not real built-in ops.
 typedef enum {
 )";
 
-- 
GitLab


From af1c668e2db5bcfcd3b156800c4b982c9423e858 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 16 Mar 2018 14:39:08 -0700
Subject: [PATCH 126/960] Downgrade run-after-mutation error to a log warning.

This is to ease the transition to the C API. Some tests currently mutate the
graph after running it but currently pass. This error was meant to guard
against existing behavior, so it's not a regression to make it a warning
instead for now.

PiperOrigin-RevId: 189395472
---
 tensorflow/c/c_api.cc                    |  19 +--
 tensorflow/c/c_api_internal.h            |  17 +--
 tensorflow/python/client/session_test.py | 140 -----------------------
 3 files changed, 19 insertions(+), 157 deletions(-)

diff --git a/tensorflow/c/c_api.cc b/tensorflow/c/c_api.cc
index 778cb667e2..18eeb28168 100644
--- a/tensorflow/c/c_api.cc
+++ b/tensorflow/c/c_api.cc
@@ -647,11 +647,11 @@ void RecordMutation(TF_Graph* graph, const TF_Operation& op,
   for (auto it : graph->sessions) {
     mutex_lock session_lock(it.first->mu);
     if (it.first->last_num_graph_nodes > op.node.id()) {
-      it.second = FailedPrecondition(
+      it.second = strings::StrCat(
           "Operation '", op.node.DebugString(), "' was changed by ",
           mutation_type,
-          " after it was run by a session. Nodes can be mutated "
-          "only before they are executed by a session. Either don't modify "
+          " after it was run by a session. This mutation will have no effect, "
+          "and will trigger an error in the future. Either don't modify "
           "nodes after running them or create a new session.");
     }
   }
@@ -722,10 +722,11 @@ bool ExtendSessionGraphHelper(TF_Session* session, TF_Status* status) {
     mutex_lock session_lock(session->mu);
     const Graph& graph = session->graph->graph;
 
-    status->status = session->graph->sessions[session];
-    if (!status->status.ok()) {
-      session->graph->mu.unlock();
-      return false;
+    const string& mutation_warning = session->graph->sessions[session];
+    if (!mutation_warning.empty()) {
+      // TODO(b/74949947): turn this back into an error status
+      LOG(WARNING) << mutation_warning;
+      session->graph->sessions[session].clear();
     }
 
     const auto num_nodes = graph.num_node_ids();
@@ -2475,7 +2476,7 @@ TF_Session* TF_NewSession(TF_Graph* graph, const TF_SessionOptions* opt,
     TF_Session* new_session = new TF_Session(session, graph);
     if (graph != nullptr) {
       mutex_lock l(graph->mu);
-      graph->sessions[new_session] = Status::OK();
+      graph->sessions[new_session] = "";
     }
     return new_session;
   } else {
@@ -2541,7 +2542,7 @@ TF_Session* TF_LoadSessionFromSavedModel(
 
   TF_Session* session = new TF_Session(bundle.session.release(), graph);
 
-  graph->sessions[session] = Status::OK();
+  graph->sessions[session] = "";
   session->last_num_graph_nodes = graph->graph.num_node_ids();
   return session;
 #endif  // __ANDROID__
diff --git a/tensorflow/c/c_api_internal.h b/tensorflow/c/c_api_internal.h
index e885a69927..95652a1137 100644
--- a/tensorflow/c/c_api_internal.h
+++ b/tensorflow/c/c_api_internal.h
@@ -84,19 +84,20 @@ struct TF_Graph {
   std::unordered_map<tensorflow::string, tensorflow::Node*> name_map
       GUARDED_BY(mu);
 
-  // The keys of this map are all the active sessions using this graph.
-  // Each value is the current "runnability" status of the corresponding
-  // session. Under normal conditions all statuses are Status::OK(), but
-  // if some operation is mutated after it was run by a session (this
-  // is detected in RecordMutation function), that session is no longer
-  // safe to run. Its status will contain the error that will be returned
-  // to the user, should she try running this session.
+  // The keys of this map are all the active sessions using this graph. Each
+  // value records whether the graph has been mutated since the corresponding
+  // session has been run (this is detected in RecordMutation function). If the
+  // string is empty, no mutation has occurred. Otherwise the string is a
+  // description of the mutation suitable for returning to the user.
   //
   // Sessions are added to this map in TF_NewSession, and removed in
   // TF_DeleteSession.
   // TF_Graph may only / must be deleted when
   //   sessions.size() == 0 && delete_requested == true
-  tensorflow::gtl::FlatMap<TF_Session*, tensorflow::Status> sessions
+  //
+  // TODO(b/74949947): mutations currently trigger a warning instead of a bad
+  // status, this should be reverted when possible.
+  tensorflow::gtl::FlatMap<TF_Session*, tensorflow::string> sessions
       GUARDED_BY(mu);
   bool delete_requested GUARDED_BY(mu);  // set true by TF_DeleteGraph
 
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 6c7339f3d8..3b12e06f43 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -29,7 +29,6 @@ import six
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.core.framework import attr_value_pb2
-from tensorflow.core.framework import types_pb2
 from tensorflow.core.lib.core import error_codes_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
@@ -1884,144 +1883,5 @@ class SessionTest(test_util.TensorFlowTestCase):
         sess.run(a, feed_dict={a: 1})
 
 
-class GraphMutationTest(test_util.TensorFlowTestCase):
-
-  def setUp(self):
-    self._original_use_c_api_value = ops._USE_C_API
-    ops._USE_C_API = True
-    super(GraphMutationTest, self).setUp()
-
-  def tearDown(self):
-    ops._USE_C_API = self._original_use_c_api_value
-    super(GraphMutationTest, self).tearDown()
-
-  def testUpdateInputAfterRunning(self):
-    with ops.Graph().as_default() as g:
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-    with session.Session(graph=g) as sess:
-      self.assertAllEqual(3.0, sess.run(c))
-      c.op._update_input(1, a)  # pylint: disable=protected-access
-      with self.assertRaisesRegexp(
-          errors.FailedPreconditionError,
-          'add.*was changed by updating input tensor after it was run'):
-        sess.run(c)
-
-      # Check that running the graph with a new session is fine
-      with session.Session(graph=g) as sess2:
-        self.assertAllEqual(2.0, sess2.run(c))
-
-  def testSetDeviceAfterRunning(self):
-    with ops.Graph().as_default() as g:
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-    with session.Session(graph=g) as sess:
-      self.assertAllEqual(3.0, sess.run(c))
-      c.op._set_device('/cpu:0')  # pylint: disable=protected-access
-      with self.assertRaisesRegexp(
-          errors.FailedPreconditionError,
-          'add.*was changed by setting device after it was run'):
-        sess.run(c)
-
-  def testSetAttrAfterRunning(self):
-    with ops.Graph().as_default() as g:
-      a = constant_op.constant(1.0, dtype=dtypes.float32)
-      b = math_ops.cast(a, dtypes.float64)
-
-    with session.Session(graph=g) as sess:
-      self.assertAllEqual(1.0, sess.run(b))
-      b.op._set_attr('DstT', attr_value_pb2.AttrValue(type=types_pb2.DT_FLOAT))
-      with self.assertRaisesRegexp(
-          errors.FailedPreconditionError,
-          'Cast.*was changed by setting attribute after it was run'):
-        sess.run(b)
-
-  def testRunModifyRun(self):
-    with ops.Graph().as_default() as g:
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-      with session.Session(graph=g) as sess:
-        self.assertAllEqual(3.0, sess.run(c))
-
-        d = b + c
-        d.op._update_input(0, a)  # pylint: disable=protected-access
-        self.assertAllEqual(3.0, sess.run(c))
-        self.assertAllEqual(4.0, sess.run(d))
-
-  def testRunModifyRunTwoSessions(self):
-    with ops.Graph().as_default() as g:
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-      with session.Session(graph=g) as sess1:
-        with session.Session(graph=g) as sess2:
-          self.assertAllEqual(3.0, sess1.run(c))
-          self.assertAllEqual(3.0, sess2.run(c))
-
-          d = b + c
-          d.op._update_input(0, a)  # pylint: disable=protected-access
-          self.assertAllEqual(3.0, sess2.run(c))
-          self.assertAllEqual(4.0, sess2.run(d))
-
-          d.op._update_input(0, b)  # pylint: disable=protected-access
-          self.assertAllEqual(3.0, sess1.run(c))
-          self.assertAllEqual(5.0, sess1.run(d))
-
-          with self.assertRaisesRegexp(
-              errors.FailedPreconditionError,
-              'add.*was changed by updating input tensor after it was run'):
-            sess2.run(c)
-
-  def testTwoSessionsOneRunBeforeModification(self):
-    with ops.Graph().as_default() as g, ops.device('/cpu:0'):
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-    with session.Session(graph=g) as sess1:
-      with session.Session(graph=g) as sess2:
-        sess1.run(c)
-
-        c.op._set_device('/cpu:0')  # pylint: disable=protected-access
-
-        with self.assertRaisesRegexp(
-            errors.FailedPreconditionError,
-            'add.*was changed by setting device after it was run'):
-          sess1.run(c)
-
-        # sess2 was not run before modification
-        self.assertAllEqual(3.0, sess2.run(c))
-
-  def testTwoSessionsBothRunBeforeModification(self):
-    with ops.Graph().as_default() as g, ops.device('/cpu:0'):
-      a = constant_op.constant(1.0)
-      b = constant_op.constant(2.0)
-      c = a + b
-
-    with session.Session(graph=g) as sess1:
-      with session.Session(graph=g) as sess2:
-        sess1.run(c)
-        sess2.run(c)
-
-        c.op._set_device('/cpu:0')  # pylint: disable=protected-access
-
-        with self.assertRaisesRegexp(
-            errors.FailedPreconditionError,
-            'add.*was changed by setting device after it was run'):
-          sess1.run(c)
-
-        with self.assertRaisesRegexp(
-            errors.FailedPreconditionError,
-            'add.*was changed by setting device after it was run'):
-          sess2.run(c)
-
-
 if __name__ == '__main__':
   googletest.main()
-- 
GitLab


From 7b33deb9c8a7c62f194e21b6364400d26e511567 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Fri, 16 Mar 2018 15:10:39 -0700
Subject: [PATCH 127/960] Don't inline functions in the grappler item builder
 since this part of the code doesn't support custom ops. Instead we will rely
 on the function optimizer.

PiperOrigin-RevId: 189400462
---
 .../core/grappler/grappler_item_builder.cc    |  5 +-
 .../core/grappler/grappler_item_builder.h     |  2 -
 .../grappler/grappler_item_builder_test.cc    | 92 -------------------
 tensorflow/python/grappler/item.i             |  1 -
 tensorflow/python/grappler/tf_optimizer.i     |  1 -
 5 files changed, 2 insertions(+), 99 deletions(-)

diff --git a/tensorflow/core/grappler/grappler_item_builder.cc b/tensorflow/core/grappler/grappler_item_builder.cc
index d7b300321a..288587ce9b 100644
--- a/tensorflow/core/grappler/grappler_item_builder.cc
+++ b/tensorflow/core/grappler/grappler_item_builder.cc
@@ -78,7 +78,7 @@ void InitializeTensor(DataType type, Tensor* tensor) {
 // correct optimizations.
 Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
                      const ItemConfig& cfg) {
-  if (!cfg.apply_optimizations && !cfg.inline_functions) {
+  if (!cfg.apply_optimizations && !cfg.erase_noinline_attributes) {
     return Status::OK();
   }
 
@@ -88,7 +88,7 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   // Make a local copy of graph def, because we need to change some things.
   GraphDef graph_def(graph_def_arg);
 
-  if (cfg.inline_functions && cfg.erase_noinline_attributes) {
+  if (cfg.erase_noinline_attributes) {
     // TF optimizer doesn't inline functions with "_noinline" attribute,
     // so let's go over the function library and erase it.
     for (auto& func : *graph_def.mutable_library()->mutable_function()) {
@@ -113,7 +113,6 @@ Status OptimizeGraph(const GraphDef& graph_def_arg, GraphDef* output_graph_def,
   } else {
     optimizer_opts->set_opt_level(::tensorflow::OptimizerOptions_Level_L0);
   }
-  optimizer_opts->set_do_function_inlining(cfg.inline_functions);
 
   // Create the function library runtime.
   std::unique_ptr<ProcessFunctionLibraryRuntime> pflr(
diff --git a/tensorflow/core/grappler/grappler_item_builder.h b/tensorflow/core/grappler/grappler_item_builder.h
index c877d91163..6d181e49e6 100644
--- a/tensorflow/core/grappler/grappler_item_builder.h
+++ b/tensorflow/core/grappler/grappler_item_builder.h
@@ -40,8 +40,6 @@ struct ItemConfig {
   int placeholder_unknown_output_shape_dim = -1;
   // If true, does L1 optimizations.
   bool apply_optimizations = false;
-  // If true, does inlining.
-  bool inline_functions = false;
   // If true, erases all "_noinline" attributes from user-defined functions.
   // Has no effect if "inline_functions" is disabled.
   bool erase_noinline_attributes = false;
diff --git a/tensorflow/core/grappler/grappler_item_builder_test.cc b/tensorflow/core/grappler/grappler_item_builder_test.cc
index 29488e4b7e..4b90bf3038 100644
--- a/tensorflow/core/grappler/grappler_item_builder_test.cc
+++ b/tensorflow/core/grappler/grappler_item_builder_test.cc
@@ -35,96 +35,6 @@ namespace {
 
 class GrapplerItemBuilderTest : public ::testing::Test {};
 
-// Create a sample graph with a symbolic gradient for sum.
-void SampleSumSymbolicGradientGraphdef(
-    GraphDef *def, CollectionDef *fetches,
-    std::vector<string> *names_of_ops_of_inline) {
-  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
-
-  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
-
-  auto dummy_variable = Variable(scope, {2, 2}, DT_FLOAT);
-  auto x = Const(scope, 1.0f);
-  auto y = Const(scope, 2);
-  auto z = Const(scope, 3.0f);
-  TF_ASSERT_OK(scope.status());
-
-  NameAttrList fn;
-  fn.set_name("Sum");
-  (*fn.mutable_attr())["T"].set_type(DT_FLOAT);
-  auto g0 = SymbolicGradient(scope, std::initializer_list<Input>{x, y, z},
-                             {DT_FLOAT, DT_INT32}, fn);
-
-  // TODO(bsteiner): we should rewrite the feed/fetch nodes to reflect the
-  // inlining that's done in the item builder
-  // fetches->mutable_node_list()->add_value(g0[0].name());
-  fetches->mutable_node_list()->add_value("SymbolicGradient/dx");
-  fetches->mutable_node_list()->add_value("SymbolicGradient/dy_reshaped");
-
-  TF_CHECK_OK(scope.ToGraphDef(def));
-
-  // Add names of the ops that replace the Mul symbolic gradient during
-  // inlining. This is for validation.
-  *names_of_ops_of_inline = {
-      "SymbolicGradient/dx",          "SymbolicGradient/tile_scaling",
-      "SymbolicGradient/dy_reshaped", "SymbolicGradient/y_shape",
-      "SymbolicGradient/x_shape",     "SymbolicGradient/stitch_idx0",
-      "SymbolicGradient/x_rank",      "SymbolicGradient/stitch_val1",
-      "SymbolicGradient/i_shape",     "SymbolicGradient/di",
-      "SymbolicGradient/zero",        "SymbolicGradient/one"};
-}
-
-std::unique_ptr<GrapplerItem> CreateGrapplerItem(const GraphDef &def,
-                                                 const CollectionDef &fetches) {
-  MetaGraphDef meta_def;
-  ItemConfig cfg;
-  cfg.inline_functions = true;
-  *meta_def.mutable_graph_def() = def;
-  (*meta_def.mutable_collection_def())["train_op"] = fetches;
-  return GrapplerItemFromMetaGraphDef("0", meta_def, cfg);
-}
-
-int CountSymbolicGradientOps(const std::unique_ptr<GrapplerItem> &item) {
-  int n_symb_grads = 0;
-  for (const auto &node : item->graph.node()) {
-    if (node.op() == FunctionLibraryDefinition::kGradientOp) {
-      n_symb_grads++;
-    }
-  }
-  return n_symb_grads;
-}
-
-int CountOpsWithNames(const std::unique_ptr<GrapplerItem> &item,
-                      const std::vector<string> &names) {
-  std::set<string> names_set(names.begin(), names.end());
-  int n_with_names = 0;
-  for (const auto &node : item->graph.node()) {
-    if (names_set.find(node.name()) != names_set.end()) {
-      n_with_names++;
-    }
-  }
-  return n_with_names;
-}
-
-TEST_F(GrapplerItemBuilderTest, SymbolicGradientInlining) {
-  // Create sample sum symbolic gradient graph.
-  GraphDef def;
-  CollectionDef fetches;
-  std::vector<string> ops_of_inline;
-  SampleSumSymbolicGradientGraphdef(&def, &fetches, &ops_of_inline);
-
-  // Create the inlined graph.
-  std::unique_ptr<GrapplerItem> with_inline = CreateGrapplerItem(def, fetches);
-
-  // For the inlined graph, there should be 0 symbolic gradient ops.
-  EXPECT_EQ(0, CountSymbolicGradientOps(with_inline));
-
-  // For the inlined graph, make sure all the required expanded op’s are in the
-  // graph.
-  EXPECT_EQ(ops_of_inline.size(),
-            CountOpsWithNames(with_inline, ops_of_inline));
-}
-
 TEST_F(GrapplerItemBuilderTest, AssetFilepathOverrideTest) {
   MetaGraphDef meta_graph;
 
@@ -273,7 +183,6 @@ TEST_F(GrapplerItemBuilderTest, GraphWithFunctions) {
   (*meta_graph.mutable_collection_def())["train_op"] = train_op;
 
   ItemConfig cfg;
-  cfg.inline_functions = false;
 
   std::unique_ptr<GrapplerItem> item =
       GrapplerItemFromMetaGraphDef("0", meta_graph, cfg);
@@ -294,7 +203,6 @@ TEST_F(GrapplerItemBuilderTest, GraphWithCustomOps) {
   (*meta_graph.mutable_collection_def())["train_op"] = train_op;
 
   ItemConfig cfg;
-  cfg.inline_functions = false;
 
   std::unique_ptr<GrapplerItem> item =
       GrapplerItemFromMetaGraphDef("0", meta_graph, cfg);
diff --git a/tensorflow/python/grappler/item.i b/tensorflow/python/grappler/item.i
index 9a84c60b04..593d38206d 100644
--- a/tensorflow/python/grappler/item.i
+++ b/tensorflow/python/grappler/item.i
@@ -83,7 +83,6 @@ static GItem TF_NewItem(
   tensorflow::grappler::ItemConfig cfg;
   cfg.ignore_user_placement = ignore_user_placement;
   cfg.ignore_colocation = ignore_colocation;
-  cfg.inline_functions = true;
   std::unique_ptr<tensorflow::grappler::GrapplerItem> item =
       tensorflow::grappler::GrapplerItemFromMetaGraphDef("item", meta_graph, cfg);
   if (!item) {
diff --git a/tensorflow/python/grappler/tf_optimizer.i b/tensorflow/python/grappler/tf_optimizer.i
index de9326ccfc..39ca71e99a 100644
--- a/tensorflow/python/grappler/tf_optimizer.i
+++ b/tensorflow/python/grappler/tf_optimizer.i
@@ -98,7 +98,6 @@ PyObject* TF_OptimizeGraph(
       const tensorflow::MetaGraphDef& metagraph,
       bool verbose, const string& graph_id, TF_Status* out_status) {
     tensorflow::grappler::ItemConfig item_config;
-    item_config.inline_functions = false;
     item_config.apply_optimizations = false;
     item_config.ignore_user_placement = false;
     std::unique_ptr<tensorflow::grappler::GrapplerItem> grappler_item =
-- 
GitLab


From 42e319741cbf50a2414aee330d8a1e385f083d20 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Fri, 16 Mar 2018 15:13:25 -0700
Subject: [PATCH 128/960] [XLA] Fix forward for HLO profiling test, explicitly
 set profiling preference.

PiperOrigin-RevId: 189400869
---
 tensorflow/compiler/xla/service/hlo_execution_profile.cc | 4 ++--
 tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_execution_profile.cc b/tensorflow/compiler/xla/service/hlo_execution_profile.cc
index f0df93b61d..c3ccbf0f0c 100644
--- a/tensorflow/compiler/xla/service/hlo_execution_profile.cc
+++ b/tensorflow/compiler/xla/service/hlo_execution_profile.cc
@@ -111,8 +111,8 @@ HloExecutionProfile::HloExecutionProfile(
     : hlo_profile_printer_data_(*hlo_profile_printer_data),
       hlo_profile_index_map_(*hlo_profile_index_map),
       profile_counters_(
-          /*count*/ hlo_profile_index_map_.total_count(),
-          /*value*/ 0) {}
+          /*count=*/hlo_profile_index_map_.total_count(),
+          /*value=*/0) {}
 
 void HloExecutionProfile::SetCyclesTakenBy(const HloInstruction* hlo,
                                            uint64 cycles_taken) {
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index 9ad2a19853..24b9f37a80 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -144,7 +144,7 @@ void ExecuteAndFetchProfile(string* profile_output, LocalClient* client,
   TF_ASSERT_OK_AND_ASSIGN(
       std::unique_ptr<LocalExecutable> local_executable,
       client->Compile(computation, {&lhs_arg_shape, &rhs_arg_shape},
-                      ExecutableBuildOptions()));
+                      ExecutableBuildOptions().set_hlo_profile(true)));
 
   Executable* executable = local_executable->executable();
   HloExecutionProfile hlo_execution_profile(
-- 
GitLab


From 6f0dd0425c51360fe2be5a938a8f3fb39e420fa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Jiongyan=20Zhang=20=28=E5=BC=A0=E7=82=AF=E8=A1=8D=29?=
 <qmick@live.cn>
Date: Sat, 17 Mar 2018 06:33:45 +0800
Subject: [PATCH 129/960] Add doc on the order of eigenvalues returned by
 tf.self_adjoint_eig (#16909)

* Supplement docs for selfAdjointEig

* Replace increasing with non-decreasing
---
 tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt  | 3 ++-
 .../core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt       | 3 ++-
 tensorflow/python/ops/linalg_ops.py                            | 2 +-
 3 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
index 51d63eeb56..7be9a958ab 100644
--- a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
@@ -19,6 +19,7 @@ form square matrices, with the same constraints as the single matrix
 SelfAdjointEig.
 
 The result is a [..., M+1, M] matrix with [..., 0,:] containing the
-eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
+eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. The eigenvalues
+are sorted in non-decreasing order.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
index 4a5e125258..fae9e84fc8 100644
--- a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
@@ -31,7 +31,8 @@ END
   summary: "Computes the eigen decomposition of one or more square self-adjoint matrices."
   description: <<END
 Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-`input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+`input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. The eigenvalues
+are sorted in non-decreasing order.
 
 ```python
 # a is a tensor.
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 37470e00d7..5b4fb4f7c8 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -341,7 +341,7 @@ def self_adjoint_eig(tensor, name=None):
     name: string, optional name of the operation.
 
   Returns:
-    e: Eigenvalues. Shape is `[..., N]`.
+    e: Eigenvalues. Shape is `[..., N]`. Sorted in non-decreasing order.
     v: Eigenvectors. Shape is `[..., N, N]`. The columns of the inner most
       matrices contain eigenvectors of the corresponding matrices in `tensor`
   """
-- 
GitLab


From 8e974f13f62cb284346f32744dc42c411a520f00 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Fri, 16 Mar 2018 15:31:10 -0700
Subject: [PATCH 130/960] Don't fail when optimizing the gradients of noinline
 functions

PiperOrigin-RevId: 189403170
---
 .../grappler/optimizers/function_optimizer.cc |  10 +-
 .../optimizers/function_optimizer_test.cc     | 103 +++++++++++++++++-
 2 files changed, 103 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index d47887bfc8..3f2afdeef1 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -189,7 +189,7 @@ Status InlineSymbolicGradient(const NodeDef& node,
   OptimizerOptions optimizer_opts;
   optimizer_opts.set_do_function_inlining(true);
   ProcessFunctionLibraryRuntime pflr(&dvc_mgr, env,
-                                     graph_def.versions().producer(),
+                                     inlined_graph->versions().producer(),
                                      &function_library, optimizer_opts);
   FunctionLibraryRuntime* flr = pflr.GetFLR(dev->name());
   CHECK(flr);
@@ -206,11 +206,6 @@ Status InlineSymbolicGradient(const NodeDef& node,
   while (counter < 50 && ExpandInlineFunctions(flr, &graph)) {
     ++counter;
   }
-  if (counter == 0) {
-    // Nothing was inlined
-    return errors::InvalidArgument(
-        strings::StrCat("Failed to inline node ", node.name()));
-  }
 
   GraphDef inlined_graph_def;
   graph.ToGraphDef(&inlined_graph_def);
@@ -278,7 +273,7 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     return Status::OK();
   }
 
-  // Inline functions when possible.
+  *optimized_graph->mutable_versions() = item.graph.versions();
   for (const NodeDef& node : item.graph.node()) {
     if (opt_level_ == RewriterConfig::AGGRESSIVE) {
       if (node.op() == "SymbolicGradient") {
@@ -301,7 +296,6 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   // TODO(bsteiner): trim the library to remove unused function definitions
   *optimized_graph->mutable_library() = item.graph.library();
-  *optimized_graph->mutable_versions() = item.graph.versions();
 
   return Status::OK();
 }
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index b1c55d838d..52a1118080 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -382,7 +382,6 @@ TEST_F(FunctionOptimizerTest, FunctionWithoutInput) {
 TEST_F(FunctionOptimizerTest, SymbolicGradients) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
 
-  //  auto T = DT_FLOAT;
   FunctionDef func = FunctionDefHelper::Define(
       "TestFunc", {"x:float", "y:float"}, {"l:float"}, {},
       {
@@ -394,7 +393,6 @@ TEST_F(FunctionOptimizerTest, SymbolicGradients) {
           {{"l"}, "Sum", {"z", "indices"}, {{"T", DT_FLOAT}}},
       });
 
-  auto dummy_variable = ops::Variable(scope, {2, 2}, DT_FLOAT);
   auto x = ops::Const(scope, 1.0f);
   auto y = ops::Const(scope, 2.0f);
   auto dl = ops::Const(scope, 3.0f);
@@ -422,6 +420,107 @@ TEST_F(FunctionOptimizerTest, SymbolicGradients) {
   test::ExpectTensorEqual<float>(expected[1], optimized[1]);
 }
 
+TEST_F(FunctionOptimizerTest, SymbolicGradientsIdentity) {
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+  FunctionDef func = FunctionDefHelper::Create(
+      // Name
+      "Identity_func",
+      // Args
+      {"in: float"},
+      // Return values
+      {"out: float"},
+      // Attr def
+      {},
+      // Nodes
+      {{{"Identity"}, "Identity", {"in"}, {{"T", DT_FLOAT}}}},
+      // Mapping
+      {{"out", "Identity:output:0"}});
+
+  auto x = ops::Const(scope, 1.0f, {3, 5, 7});
+  auto z = ops::Const(scope, 3.0f, {3, 5, 7});
+
+  NameAttrList fn;
+  fn.set_name("Identity_func");
+  auto g0 = ops::SymbolicGradient(scope, std::initializer_list<Input>{x, z},
+                                  {DT_FLOAT}, fn);
+  auto out = ops::Identity(scope.WithOpName("out"), g0.output[0]);
+
+  GrapplerItem item;
+  TF_EXPECT_OK(scope.ToGraphDef(&item.graph));
+  *item.graph.mutable_library()->add_function() = func;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(13, output.node_size());
+  EXPECT_EQ("Const", output.node(0).name());
+  EXPECT_EQ("Const_1", output.node(1).name());
+  EXPECT_EQ("SymbolicGradient/FunctionInputs", output.node(2).name());
+  EXPECT_EQ("SymbolicGradient", output.node(3).name());
+  EXPECT_EQ("SymbolicGradient/SymbolicGradient/Identity",
+            output.node(4).name());
+  EXPECT_EQ("SymbolicGradient/Func/_0", output.node(5).name());
+  EXPECT_EQ("SymbolicGradient/Func/_1", output.node(6).name());
+  EXPECT_EQ("SymbolicGradient/Func/_2", output.node(7).name());
+  EXPECT_EQ("SymbolicGradient/SymbolicGradient/Func/_1/dx",
+            output.node(8).name());
+  EXPECT_EQ("SymbolicGradient/Func/_3", output.node(9).name());
+  EXPECT_EQ("SymbolicGradient/Func/_4", output.node(10).name());
+  EXPECT_EQ("SymbolicGradient/Func/_5", output.node(11).name());
+  EXPECT_EQ("out", output.node(12).name());
+  for (int i = 2; i < 4; ++i) {
+    EXPECT_EQ("IdentityN", output.node(i).op());
+  }
+  for (int i = 4; i < 11; ++i) {
+    EXPECT_EQ("Identity", output.node(i).op());
+  }
+
+  std::vector<Tensor> expected = EvaluateNodes(item.graph, {"out"});
+  std::vector<Tensor> optimized = EvaluateNodes(output, {"out"});
+  test::ExpectTensorEqual<float>(expected[0], optimized[0]);
+}
+
+TEST_F(FunctionOptimizerTest, SymbolicGradientsNoInlineFunc) {
+  FunctionDef func = FunctionDefHelper::Define(
+      "TestFunc", {"x:float", "y:float"}, {"l:float"}, {},
+      {
+          {{"z"}, "Add", {"x", "y"}, {{"T", DT_FLOAT}}},
+          FunctionDefHelper::Const("zero", 0),
+          FunctionDefHelper::Const("one", 1),
+          {{"r"}, "Rank", {"z"}, {{"T", DT_FLOAT}}},
+          {{"indices"}, "Range", {"zero", "r", "one"}},
+          {{"l"}, "Sum", {"z", "indices"}, {{"T", DT_FLOAT}}},
+      });
+  (*func.mutable_attr())["_noinline"].set_b(true);
+
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+  auto x = ops::Const(scope, 1.0f);
+  auto y = ops::Const(scope, 2.0f);
+  auto dl = ops::Const(scope, 3.0f);
+
+  NameAttrList fn;
+  fn.set_name("TestFunc");
+  (*fn.mutable_attr())["T"].set_type(DT_FLOAT);
+  auto g0 = ops::SymbolicGradient(scope, std::initializer_list<Input>{x, y, dl},
+                                  {DT_FLOAT, DT_FLOAT}, fn);
+  auto out1 = ops::Identity(scope.WithOpName("out1"), g0.output[0]);
+  auto out2 = ops::Identity(scope.WithOpName("out2"), g0.output[1]);
+
+  GrapplerItem item;
+  TF_EXPECT_OK(scope.ToGraphDef(&item.graph));
+  *item.graph.mutable_library()->add_function() = func;
+
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  // The optimizer should succeed but the graphs should be the same.
+  TF_EXPECT_OK(status);
+  CompareGraphs(item.graph, output);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 8cabc19c6b986d86c3e2b2d8d40f49a9400f926b Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Fri, 16 Mar 2018 15:36:15 -0700
Subject: [PATCH 131/960] Consolidate all moving_average updates in batchnorm
 into one implementation.

PiperOrigin-RevId: 189404070
---
 tensorflow/python/layers/normalization.py | 50 +++++++++--------------
 1 file changed, 20 insertions(+), 30 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index c23d755a8e..8b79a92cc4 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -319,7 +319,6 @@ class BatchNormalization(base.Layer):
           initializer=self.moving_variance_initializer,
           trainable=False)
 
-      self._one_minus_decay = 1.0 - self.momentum
       if self.renorm:
         # Create variables to maintain the moving mean and standard deviation.
         # These are used in training and thus are different from the moving
@@ -360,20 +359,14 @@ class BatchNormalization(base.Layer):
         self._scope.set_partitioner(partitioner)
     self.built = True
 
-  def _assign_moving_average(self, variable, value, one_minus_decay):
+  def _assign_moving_average(self, variable, value, momentum):
     with ops.name_scope(None, 'AssignMovingAvg',
-                        [variable, value, one_minus_decay]) as scope:
+                        [variable, value, momentum]) as scope:
       with ops.colocate_with(variable):
+        decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
         update_delta = math_ops.multiply(
-            math_ops.subtract(variable.read_value(), value),
-            one_minus_decay)
-        if isinstance(variable, resource_variable_ops.ResourceVariable):
-          # state_ops.assign_sub does an extra read_variable_op after the
-          # assign. We avoid that here.
-          return gen_resource_variable_ops.assign_sub_variable_op(
-              variable.handle, update_delta, name=scope)
-        else:
-          return state_ops.assign_sub(variable, update_delta, name=scope)
+            math_ops.subtract(variable.read_value(), value), decay)
+        return state_ops.assign_sub(variable, update_delta, name=scope)
 
   def _fused_batch_norm(self, inputs, training):
     """Returns the output of fused batch norm."""
@@ -412,22 +405,16 @@ class BatchNormalization(base.Layer):
 
     training_value = utils.constant_value(training)
     if training_value is None:
-      one_minus_decay = utils.smart_cond(training,
-                                         lambda: self._one_minus_decay,
-                                         lambda: 0.)
+      momentum = utils.smart_cond(training, lambda: self.momentum, lambda: 1.0)
     else:
-      one_minus_decay = ops.convert_to_tensor(self._one_minus_decay)
+      momentum = ops.convert_to_tensor(self.momentum)
     if training_value or training_value is None:
       mean_update = self._assign_moving_average(self.moving_mean, mean,
-                                                one_minus_decay)
+                                                momentum)
       variance_update = self._assign_moving_average(self.moving_variance,
-                                                    variance, one_minus_decay)
-      if not context.executing_eagerly():
-        # Note that in Eager mode, the updates are already executed when running
-        # assign_moving_averages. So we do not need to put them into
-        # collections.
-        self.add_update(mean_update, inputs=inputs)
-        self.add_update(variance_update, inputs=inputs)
+                                                    variance, momentum)
+      self.add_update(mean_update, inputs=inputs)
+      self.add_update(variance_update, inputs=inputs)
 
     return output
 
@@ -464,6 +451,7 @@ class BatchNormalization(base.Layer):
       """Updates a moving average and weight, returns the unbiased value."""
       value = array_ops.identity(value)
       def _do_update():
+        """Updates the var and weight, returns their updated ratio."""
         # Update the variables without zero debiasing. The debiasing will be
         # accomplished by dividing the exponential moving average by the weight.
         # For example, after a single update, the moving average would be
@@ -472,11 +460,14 @@ class BatchNormalization(base.Layer):
         # Make sure the weight is not updated until before r and d computation.
         with ops.control_dependencies([value]):
           weight_value = array_ops.constant(1., dtype=weight.dtype)
-        new_var = moving_averages.assign_moving_average(
-            var, value, self.renorm_momentum, zero_debias=False)
-        new_weight = moving_averages.assign_moving_average(
-            weight, weight_value, self.renorm_momentum, zero_debias=False)
+        new_var = self._assign_moving_average(var, value, self.renorm_momentum)
+        new_weight = self._assign_moving_average(weight, weight_value,
+                                                 self.renorm_momentum)
+        # TODO(yuefengz): the updates to var and weighted can not be batched
+        # together if we fetch their updated values here. Consider calculating
+        # new values and delaying the updates.
         return new_var / new_weight
+
       def _fake_update():
         return array_ops.identity(var)
       return utils.smart_cond(training, _do_update, _fake_update)
@@ -601,8 +592,7 @@ class BatchNormalization(base.Layer):
         if in_eager_mode and not self.trainable:
           return
 
-        return moving_averages.assign_moving_average(
-            var, value, self.momentum, zero_debias=False)
+        return self._assign_moving_average(var, value, self.momentum)
 
       mean_update = utils.smart_cond(
           training,
-- 
GitLab


From d1a173677ea98c34f5ce872897796923b14c0d6a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 15:55:14 -0700
Subject: [PATCH 132/960] Remove identity transpose nodes.

PiperOrigin-RevId: 189406518
---
 .../optimizers/arithmetic_optimizer.cc        | 75 ++++++++++++-------
 .../optimizers/arithmetic_optimizer.h         |  2 +-
 .../optimizers/arithmetic_optimizer_test.cc   | 27 ++++---
 3 files changed, 64 insertions(+), 40 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 675cd8f072..3a67c4b056 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -933,55 +933,67 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
 };
 
 // Removes inverse transpose nodes
-class RemoveInverseTranspose : public ArithmeticOptimizerStage {
+class RemoveIdentityTranspose : public ArithmeticOptimizerStage {
  public:
-  explicit RemoveInverseTranspose(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("RemoveInverseTranspose", ctx) {}
-  ~RemoveInverseTranspose() override = default;
+  explicit RemoveIdentityTranspose(const ArithmeticOptimizerContext& ctx)
+      : ArithmeticOptimizerStage("RemoveIdentityTranspose", ctx) {}
+  ~RemoveIdentityTranspose() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
     return IsTranspose(*node) || IsConjugateTranspose(*node);
   }
 
+  // TODO(rmlarsen): Forward control dependencies on the bypassed
+  // transpose nodes.
   Status TrySimplify(const NodeDef* node,
                      string* simplified_node_name) override {
     CHECK(IsSupported(node));
 
     NodeDef* input;
     TF_RETURN_IF_ERROR(GetInputNode(node->input(0), &input));
+    NodeDef* node_perm;
+    TF_RETURN_IF_ERROR(GetInputNode(node->input(1), &node_perm));
+    std::vector<int64> node_perm_values;
+    TF_RETURN_IF_ERROR(GetPermutation(*node_perm, &node_perm_values));
 
     if (input->op() == node->op()) {
-      NodeDef* node_perm;
+      // Remove pairs of transposes that cancel each other.
       NodeDef* input_perm;
-
-      TF_RETURN_IF_ERROR(GetInputNode(node->input(1), &node_perm));
       TF_RETURN_IF_ERROR(GetInputNode(input->input(1), &input_perm));
-
-      // Try 32-bit indices.
-      std::vector<int> node_perm_values;
-      std::vector<int> input_perm_values;
-      if (ValuesFromConstNode(*node_perm, &node_perm_values) &&
-          ValuesFromConstNode(*input_perm, &input_perm_values) &&
-          AreInversePermutations(node_perm_values, input_perm_values)) {
+      std::vector<int64> input_perm_values;
+      TF_RETURN_IF_ERROR(GetPermutation(*input_perm, &input_perm_values));
+      if (AreInversePermutations(node_perm_values, input_perm_values)) {
         *simplified_node_name = input->input(0);
       }
-      // Try 64-bit indices.
-      std::vector<int64> node_perm_values64;
-      std::vector<int64> input_perm_values64;
-      if (ValuesFromConstNode(*node_perm, &node_perm_values64) &&
-          ValuesFromConstNode(*input_perm, &input_perm_values64) &&
-          AreInversePermutations(node_perm_values64, input_perm_values64)) {
-        *simplified_node_name = input->input(0);
+    } else {
+      // Remove simple identity transposes.
+      if (IsIdentityPermutation(node_perm_values)) {
+        *simplified_node_name = node->input(0);
       }
     }
-
     return Status::OK();
   }
 
  private:
-  template <typename T>
-  bool AreInversePermutations(const std::vector<T>& a,
-                              const std::vector<T>& b) {
+  Status GetPermutation(const NodeDef& node_perm,
+                        std::vector<int64>* perm64) const {
+    std::vector<int> perm32;
+    if (ValuesFromConstNode(node_perm, &perm32)) {
+      perm64->reserve(perm32.size());
+      for (int val : perm32) {
+        perm64->push_back(static_cast<int64>(val));
+      }
+      return Status::OK();
+    }
+    if (ValuesFromConstNode(node_perm, perm64)) {
+      return Status::OK();
+    }
+    return errors::InvalidArgument("Couldn't extract permutation from ",
+                                   node_perm.name());
+  }
+
+  bool AreInversePermutations(const std::vector<int64>& a,
+                              const std::vector<int64>& b) {
     if (a.size() != b.size()) {
       return false;
     }
@@ -992,6 +1004,15 @@ class RemoveInverseTranspose : public ArithmeticOptimizerStage {
     }
     return true;
   }
+
+  bool IsIdentityPermutation(const std::vector<int64>& perm) {
+    for (int64 i = 0; i < perm.size(); ++i) {
+      if (i != perm[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
 };
 
 // Remove redundant Bitcasts.
@@ -1663,9 +1684,9 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new HoistCommonFactorOutOfAggregation(ctx)));
   }
-  if (options_.remove_inverse_transpose) {
+  if (options_.remove_identity_transpose) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveInverseTranspose(ctx)));
+        new RemoveIdentityTranspose(ctx)));
   }
   if (options_.remove_redundant_bitcast) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index bd2f42ee8c..95c1e14258 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -60,7 +60,7 @@ class ArithmeticOptimizer : public GraphOptimizer {
     bool enable_try_simplify_and_replace = true;
     bool combine_add_to_addn = false;
     bool hoist_common_factor_out_of_aggregation = true;
-    bool remove_inverse_transpose = true;
+    bool remove_identity_transpose = true;
     bool remove_redundant_bitcast = true;
     bool remove_redundant_cast = true;
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index d677aee589..6f7a95c2ed 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -92,7 +92,7 @@ class ArithmeticOptimizerTest : public GrapplerTest {
     options.enable_try_simplify_and_replace = false;
     options.combine_add_to_addn = false;
     options.hoist_common_factor_out_of_aggregation = false;
-    options.remove_inverse_transpose = false;
+    options.remove_identity_transpose = false;
     options.remove_redundant_bitcast = false;
     options.remove_redundant_cast = false;
     optimizer->options_ = options;
@@ -112,9 +112,9 @@ class ArithmeticOptimizerTest : public GrapplerTest {
     optimizer->options_.hoist_common_factor_out_of_aggregation = true;
   }
 
-  void EnableOnlyRemoveInverseTranspose(ArithmeticOptimizer* optimizer) {
+  void EnableOnlyRemoveIdentityTranspose(ArithmeticOptimizer* optimizer) {
     DisableAllStages(optimizer);
-    optimizer->options_.remove_inverse_transpose = true;
+    optimizer->options_.remove_identity_transpose = true;
   }
 
   void EnableOnlyRemoveRedundantBitcast(ArithmeticOptimizer* optimizer) {
@@ -876,7 +876,7 @@ TEST_F(ArithmeticOptimizerTest, NoReorderTransposeCast) {
   EXPECT_EQ(1, num_transposes);
 }
 
-TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposes) {
+TEST_F(ArithmeticOptimizerTest, RemoveIdentityTransposes) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output inputs_shape =
       ops::Const(s.WithOpName("inputs_shape"), {8, 3, 28, 28}, {4});
@@ -884,18 +884,21 @@ TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposes) {
       ops::RandomUniform(s.WithOpName("inputs"), inputs_shape, DT_FLOAT);
   Output perm1 = ops::Const(s.WithOpName("perm1"), {0, 2, 3, 1}, {4});
   Output perm2 = ops::Const(s.WithOpName("perm2"), {0, 3, 1, 2}, {4});
+  Output perm3 = ops::Const(s.WithOpName("perm2"), {0, 1, 2, 3}, {4});
   Output transpose1 = ops::Transpose(s.WithOpName("transpose1"), inputs, perm1);
   Output transpose2 =
       ops::Transpose(s.WithOpName("transpose2"), transpose1, perm2);
-  Output outputs = ops::Identity(s.WithOpName("outputs"), transpose2);
+  Output transpose3 = ops::Transpose(s.WithOpName("transpose3"), inputs, perm3);
+  Output id1 = ops::Identity(s.WithOpName("id1"), transpose2);
+  Output id2 = ops::Identity(s.WithOpName("id2"), transpose3);
 
   GrapplerItem item;
-  item.fetch = {"outputs"};
+  item.fetch = {"id1", "id2"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
   GraphDef output;
   ArithmeticOptimizer optimizer;
-  EnableOnlyRemoveInverseTranspose(&optimizer);
+  EnableOnlyRemoveIdentityTranspose(&optimizer);
   OptimizeAndPrune(&optimizer, &item, &output);
 
   std::set<string> nodes_after_optimization;
@@ -903,10 +906,10 @@ TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposes) {
     nodes_after_optimization.insert(node.name());
   }
   EXPECT_EQ(nodes_after_optimization,
-            std::set<string>({"inputs_shape", "inputs", "outputs"}));
+            std::set<string>({"id1", "id2", "inputs_shape", "inputs"}));
 }
 
-TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposesMultipleOutputs) {
+TEST_F(ArithmeticOptimizerTest, RemoveIdentityTransposesMultipleOutputs) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output inputs_shape =
       ops::Const(s.WithOpName("inputs_shape"), {8, 9, 28, 28}, {4});
@@ -927,7 +930,7 @@ TEST_F(ArithmeticOptimizerTest, RemoveInverseTransposesMultipleOutputs) {
 
   GraphDef output;
   ArithmeticOptimizer optimizer;
-  EnableOnlyRemoveInverseTranspose(&optimizer);
+  EnableOnlyRemoveIdentityTranspose(&optimizer);
   OptimizeAndPrune(&optimizer, &item, &output);
 
   for (const NodeDef& node : output.node()) {
@@ -955,7 +958,7 @@ TEST_F(ArithmeticOptimizerTest, RemoveTransposesWithControlDependency) {
 
   GraphDef output;
   ArithmeticOptimizer optimizer;
-  EnableOnlyRemoveInverseTranspose(&optimizer);
+  EnableOnlyRemoveIdentityTranspose(&optimizer);
   OptimizeAndPrune(&optimizer, &item, &output);
 
   NodeMap node_map(&output);
@@ -983,7 +986,7 @@ TEST_F(ArithmeticOptimizerTest, NotRemoveTransposes) {
 
   GraphDef output;
   ArithmeticOptimizer optimizer;
-  EnableOnlyRemoveInverseTranspose(&optimizer);
+  EnableOnlyRemoveIdentityTranspose(&optimizer);
   OptimizeAndPrune(&optimizer, &item, &output);
 
   EXPECT_EQ(6, output.node_size());
-- 
GitLab


From dfe8e75a693d3d7613ce1c7e349747a070df9858 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 16 Mar 2018 16:00:11 -0700
Subject: [PATCH 133/960] Increase kMaxEagerTensorParentSize to 64.

PiperOrigin-RevId: 189407226
---
 tensorflow/python/eager/pywrap_tensor.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/pywrap_tensor.cc b/tensorflow/python/eager/pywrap_tensor.cc
index 105c09e81f..519814b979 100644
--- a/tensorflow/python/eager/pywrap_tensor.cc
+++ b/tensorflow/python/eager/pywrap_tensor.cc
@@ -163,7 +163,7 @@ PyObject* PyIntFromDataType(TF_DataType l) {
 
 extern "C" {
 
-static const int kMaxEagerTensorParentSize = 32;
+static const int kMaxEagerTensorParentSize = 64;
 
 // TODO(agarwal): store context handle in EagerTensor.
 typedef struct EagerTensor {
-- 
GitLab


From f06cce76e1f134a8787e28e7adc7182e24cdf1a5 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Fri, 16 Mar 2018 16:07:06 -0700
Subject: [PATCH 134/960] Deleted dead code

PiperOrigin-RevId: 189408200
---
 tensorflow/core/grappler/costs/graph_properties.h | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h
index 5aa4962072..93a722f038 100644
--- a/tensorflow/core/grappler/costs/graph_properties.h
+++ b/tensorflow/core/grappler/costs/graph_properties.h
@@ -67,12 +67,6 @@ class GraphProperties {
   void ClearInputProperties(const string& node_name);
   void ClearOutputProperties(const string& node_name);
 
-  static void FillTensorPropertiesFromContext(
-      const shape_inference::ShapeHandle&, const DataType&,
-      shape_inference::InferenceContext*,
-      std::unordered_map<const shape_inference::Dimension*, int>* dim_ids,
-      OpInfo::TensorProperties*);
-
  private:
   // Merges shapes <shapes_and_types>, determined from an EnqueueV2 node, into
   // <*queue_shapes_and_types>.
-- 
GitLab


From 139c6ce00610b43e05855135dab987997bcfe11f Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 16 Mar 2018 16:10:46 -0700
Subject: [PATCH 135/960] Always sets self._built in tfe.metrics when build()
 is called.

PiperOrigin-RevId: 189408745
---
 tensorflow/contrib/eager/python/metrics_impl.py | 12 ++++++++++++
 tensorflow/contrib/eager/python/metrics_test.py |  9 +++++++++
 2 files changed, 21 insertions(+)

diff --git a/tensorflow/contrib/eager/python/metrics_impl.py b/tensorflow/contrib/eager/python/metrics_impl.py
index 1490c2ccac..2f2347736a 100644
--- a/tensorflow/contrib/eager/python/metrics_impl.py
+++ b/tensorflow/contrib/eager/python/metrics_impl.py
@@ -109,6 +109,18 @@ class Metric(checkpointable.CheckpointableBase):
       pos = scope.name.rfind(scope_name)
       self._name = name + scope.name[pos + len(scope_name):]
       self._scope = scope
+
+    # Ensures that if the user calls build directly we still set self._built to
+    # True to prevent variables from being recreated.
+    self._build = self.build
+
+    def actual_build(*args, **kwargs):
+      self._build(*args, **kwargs)
+      self._built = True
+    self.build = actual_build
+    self.build.__doc__ = self._build.__doc__
+
+    # Captures construction scope for proper initialization.
     if context.executing_eagerly():
       self._construction_scope = context.eager_mode
     else:
diff --git a/tensorflow/contrib/eager/python/metrics_test.py b/tensorflow/contrib/eager/python/metrics_test.py
index 6b5450ba89..15ac889191 100644
--- a/tensorflow/contrib/eager/python/metrics_test.py
+++ b/tensorflow/contrib/eager/python/metrics_test.py
@@ -195,6 +195,15 @@ class MetricsTest(test.TestCase):
         m2 = metrics.Mean()
         m2(2)
 
+  def testBuildMean(self):
+    # Verify that calling build() on Mean and then calling it won't recreate
+    # variables.
+    m = metrics.Mean()
+    m.build()
+    old_numer = m.numer
+    m(0.0)
+    self.assertTrue(old_numer is m.numer)
+
   def testMetricsChain(self):
     with context.graph_mode(), self.test_session():
       m1 = metrics.Mean()
-- 
GitLab


From 71728ef0eb2a823cf77ac22ec45b76d4c10196de Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 16 Mar 2018 16:27:02 -0700
Subject: [PATCH 136/960] Relax "_output_shapes" error checking in C++ graph
 importer.

This is to make the behavior inline with what Python's
import_graph_def does. It'd probably be better to raise an error, but
some people are already depending on the import_graph_def behavior in
order to import modified ops.

PiperOrigin-RevId: 189411025
---
 tensorflow/core/graph/graph_constructor.cc | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc
index 627309078a..76ee88e684 100644
--- a/tensorflow/core/graph/graph_constructor.cc
+++ b/tensorflow/core/graph/graph_constructor.cc
@@ -568,13 +568,22 @@ Status GraphConstructor::ValidateShape(Node* node) {
   auto* ic = refiner_->GetContext(node);
   DCHECK(ic != nullptr)
       << "ShapeRefiner::AddNode() should have created the InferenceContext";
-  if (shape_attrs.size() != node->num_outputs()) {
+  if (shape_attrs.size() < node->num_outputs()) {
     return errors::InvalidArgument(
         "Node '", node->name(), "' has ", node->num_outputs(),
         " outputs but the ", kAttrName, " attribute specifies shapes for ",
         shape_attrs.size(), " outputs");
   }
-  for (int i = 0; i < shape_attrs.size(); ++i) {
+  // NOTE(skyewm): we don't raise an error here because some users depend on
+  // this behavior, even though it's unsafe.
+  // TODO(b/74619486): raise an error.
+  if (shape_attrs.size() > node->num_outputs()) {
+    LOG(WARNING) << "Node '" << node->name() << "' has " << node->num_outputs()
+                 << " outputs but the " << kAttrName
+                 << " attribute specifies shapes for " << shape_attrs.size()
+                 << " outputs. Output shapes may be inaccurate.";
+  }
+  for (int i = 0; i < node->num_outputs(); ++i) {
     const TensorShapeProto& p = shape_attrs[i];
     shape_inference::ShapeHandle h;
     Status s = ic->MakeShapeFromShapeProto(p, &h);
-- 
GitLab


From bd33984641fda2f892b77bb2a1ac8c33c7a2211a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 16 Mar 2018 17:00:17 -0700
Subject: [PATCH 137/960] TFTS: Allow cold-starting from SavedModels

This means the model starts from its default start state and is fed a series
(filtering) to warm up its state. This warmed up state can then be used to
make predictions.

Some shape fiddling with the receiver_fn to make feeding state optional, and a
new signature for cold-starting which uses the model's default start state.

Some other shape fiddling to make feeding strings to SavedModels work more
smoothly in the cold-start part of the LSTM example. I was squeezing out the
last dimension of "scalar" exogenous features, now I'm leaving them, which
matches the placeholder generation logic.

PiperOrigin-RevId: 189414869
---
 .../timeseries/examples/known_anomaly.py      | 17 ++---
 .../contrib/timeseries/examples/lstm.py       | 22 ++++++-
 .../python/timeseries/estimators.py           | 40 ++++++++----
 .../python/timeseries/feature_keys.py         |  1 +
 .../timeseries/python/timeseries/head.py      | 11 +++-
 .../python/timeseries/input_pipeline.py       |  3 +-
 .../python/timeseries/saved_model_utils.py    | 62 +++++++++++++++++--
 7 files changed, 126 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/timeseries/examples/known_anomaly.py b/tensorflow/contrib/timeseries/examples/known_anomaly.py
index c08c0b0acb..e77628ddd3 100644
--- a/tensorflow/contrib/timeseries/examples/known_anomaly.py
+++ b/tensorflow/contrib/timeseries/examples/known_anomaly.py
@@ -53,6 +53,15 @@ def train_and_evaluate_exogenous(csv_file_name=_DATA_FILE, train_steps=300):
   one_hot_feature = tf.feature_column.indicator_column(
       categorical_column=string_feature)
 
+  def _exogenous_update_condition(times, features):
+    del times  # unused
+    # Make exogenous updates sparse by setting an update condition. This in
+    # effect allows missing exogenous features: if the condition evaluates to
+    # False, no update is performed. Otherwise we sometimes end up with "leaky"
+    # updates which add unnecessary uncertainty to the model even when there is
+    # no changepoint.
+    return tf.equal(tf.squeeze(features["is_changepoint"], axis=-1), "yes")
+
   estimator = tf.contrib.timeseries.StructuralEnsembleRegressor(
       periodicities=12,
       # Extract a smooth period by constraining the number of latent values
@@ -60,13 +69,7 @@ def train_and_evaluate_exogenous(csv_file_name=_DATA_FILE, train_steps=300):
       cycle_num_latent_values=3,
       num_features=1,
       exogenous_feature_columns=[one_hot_feature],
-      # Make exogenous updates sparse by setting an update condition. This in
-      # effect allows missing exogenous features: if the condition evaluates to
-      # False, no update is performed. Otherwise we sometimes end up with
-      # "leaky" updates which add unnecessary uncertainty to the model even when
-      # there is no changepoint.
-      exogenous_update_condition=
-      lambda times, features: tf.equal(features["is_changepoint"], "yes"))
+      exogenous_update_condition=_exogenous_update_condition)
   reader = tf.contrib.timeseries.CSVReader(
       csv_file_name,
       # Indicate the format of our CSV file. First we have two standard columns,
diff --git a/tensorflow/contrib/timeseries/examples/lstm.py b/tensorflow/contrib/timeseries/examples/lstm.py
index 2eee878196..b1c7475442 100644
--- a/tensorflow/contrib/timeseries/examples/lstm.py
+++ b/tensorflow/contrib/timeseries/examples/lstm.py
@@ -236,20 +236,36 @@ def train_and_predict(
       [evaluation["mean"][0], predictions["mean"]], axis=0))
   all_times = numpy.concatenate([times, predictions["times"]], axis=0)
 
-  # Export the model in SavedModel format.
+  # Export the model in SavedModel format. We include a bit of extra boilerplate
+  # for "cold starting" as if we didn't have any state from the Estimator, which
+  # is the case when serving from a SavedModel. If Estimator output is
+  # available, the result of "Estimator.evaluate" can be passed directly to
+  # `tf.contrib.timeseries.saved_model_utils.predict_continuation` as the
+  # `continue_from` argument.
+  with tf.Graph().as_default():
+    filter_feature_tensors, _ = evaluation_input_fn()
+    with tf.train.MonitoredSession() as session:
+      # Fetch the series to "warm up" our state, which will allow us to make
+      # predictions for its future values. This is just a dictionary of times,
+      # values, and exogenous features mapping to numpy arrays. The use of an
+      # input_fn is just a convenience for the example; they can also be
+      # specified manually.
+      filter_features = session.run(filter_feature_tensors)
   if export_directory is None:
     export_directory = tempfile.mkdtemp()
   input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
   export_location = estimator.export_savedmodel(
       export_directory, input_receiver_fn)
-  # Predict using the SavedModel
+  # Warm up and predict using the SavedModel
   with tf.Graph().as_default():
     with tf.Session() as session:
       signatures = tf.saved_model.loader.load(
           session, [tf.saved_model.tag_constants.SERVING], export_location)
+      state = tf.contrib.timeseries.saved_model_utils.cold_start_filter(
+          signatures=signatures, session=session, features=filter_features)
       saved_model_output = (
           tf.contrib.timeseries.saved_model_utils.predict_continuation(
-              continue_from=evaluation, signatures=signatures,
+              continue_from=state, signatures=signatures,
               session=session, steps=100,
               exogenous_features=predict_exogenous_features))
       # The exported model gives the same results as the Estimator.predict()
diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
index 8d13343e82..469cea4fd2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
@@ -33,9 +33,11 @@ from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.training import training as train
+from tensorflow.python.util import nest
 
 
 class TimeSeriesRegressor(estimator_lib.Estimator):
@@ -98,11 +100,11 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
     def _serving_input_receiver_fn():
       """A receiver function to be passed to export_savedmodel."""
       placeholders = {}
-      placeholders[feature_keys.TrainEvalFeatures.TIMES] = (
-          array_ops.placeholder(
-              name=feature_keys.TrainEvalFeatures.TIMES,
-              dtype=dtypes.int64,
-              shape=[default_batch_size, default_series_length]))
+      time_placeholder = array_ops.placeholder(
+          name=feature_keys.TrainEvalFeatures.TIMES,
+          dtype=dtypes.int64,
+          shape=[default_batch_size, default_series_length])
+      placeholders[feature_keys.TrainEvalFeatures.TIMES] = time_placeholder
       # Values are only necessary when filtering. For prediction the default
       # value will be ignored.
       placeholders[feature_keys.TrainEvalFeatures.VALUES] = (
@@ -145,15 +147,29 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
       # use only static metadata from the returned Tensors.
       with ops.Graph().as_default():
         self._model.initialize_graph()
-        model_start_state = self._model.get_start_state()
-      for prefixed_state_name, state_tensor in ts_head_lib.state_to_dictionary(
-          model_start_state).items():
+        # Evaluate the initial state as same-dtype "zero" values. These zero
+        # constants aren't used, but are necessary for feeding to
+        # placeholder_with_default for the "cold start" case where state is not
+        # fed to the model.
+        def _zeros_like_constant(tensor):
+          return tensor_util.constant_value(array_ops.zeros_like(tensor))
+        start_state = nest.map_structure(
+            _zeros_like_constant, self._model.get_start_state())
+      batch_size_tensor = array_ops.shape(time_placeholder)[0]
+      for prefixed_state_name, state in ts_head_lib.state_to_dictionary(
+          start_state).items():
         state_shape_with_batch = tensor_shape.TensorShape(
-            (default_batch_size,)).concatenate(state_tensor.get_shape())
-        placeholders[prefixed_state_name] = array_ops.placeholder(
+            (default_batch_size,)).concatenate(state.shape)
+        default_state_broadcast = array_ops.tile(
+            state[None, ...],
+            multiples=array_ops.concat(
+                [batch_size_tensor[None],
+                 array_ops.ones(len(state.shape), dtype=dtypes.int32)],
+                axis=0))
+        placeholders[prefixed_state_name] = array_ops.placeholder_with_default(
+            input=default_state_broadcast,
             name=prefixed_state_name,
-            shape=state_shape_with_batch,
-            dtype=state_tensor.dtype)
+            shape=state_shape_with_batch)
       return export_lib.ServingInputReceiver(placeholders, placeholders)
 
     return _serving_input_receiver_fn
diff --git a/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py b/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py
index 970b9aa8ac..56566ee2e3 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/feature_keys.py
@@ -72,3 +72,4 @@ class SavedModelLabels(object):
   """Names of signatures exported with export_savedmodel."""
   PREDICT = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
   FILTER = "filter"
+  COLD_START_FILTER = "cold_start_filter"
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py
index f4d9351432..3d7e615290 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head.py
@@ -150,6 +150,12 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
     with variable_scope.variable_scope("model", reuse=True):
       filtering_outputs = self.create_loss(
           features, estimator_lib.ModeKeys.EVAL)
+    with variable_scope.variable_scope("model", reuse=True):
+      no_state_features = {
+          k: v for k, v in features.items()
+          if not k.startswith(feature_keys.State.STATE_PREFIX)}
+      cold_filtering_outputs = self.create_loss(
+          no_state_features, estimator_lib.ModeKeys.EVAL)
     return estimator_lib.EstimatorSpec(
         mode=estimator_lib.ModeKeys.PREDICT,
         export_outputs={
@@ -157,7 +163,10 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
                 export_lib.PredictOutput(prediction_outputs),
             feature_keys.SavedModelLabels.FILTER:
                 export_lib.PredictOutput(
-                    state_to_dictionary(filtering_outputs.end_state))
+                    state_to_dictionary(filtering_outputs.end_state)),
+            feature_keys.SavedModelLabels.COLD_START_FILTER:
+                export_lib.PredictOutput(
+                    state_to_dictionary(cold_filtering_outputs.end_state))
         },
         # Likely unused, but it is necessary to return `predictions` to satisfy
         # the Estimator's error checking.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py b/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py
index 04225333b9..403c6e2cb4 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/input_pipeline.py
@@ -492,8 +492,7 @@ class CSVReader(ReaderBaseTimeSeriesParser):
       features_lists.setdefault(column_name, []).append(value)
     features = {}
     for column_name, values in features_lists.items():
-      if (len(values) == 1 and
-          column_name != feature_keys.TrainEvalFeatures.VALUES):
+      if column_name == feature_keys.TrainEvalFeatures.TIMES:
         features[column_name] = values[0]
       else:
         features[column_name] = array_ops.stack(values, axis=1)
diff --git a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py b/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
index 97f6d36a87..0461abdc19 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/saved_model_utils.py
@@ -15,6 +15,7 @@
 """Convenience functions for working with time series saved_models.
 
 @@predict_continuation
+@@cold_start_filter
 @@filter_continuation
 """
 
@@ -30,10 +31,12 @@ from tensorflow.contrib.timeseries.python.timeseries import model_utils as _mode
 from tensorflow.python.util.all_util import remove_undocumented
 
 
-def _colate_features_to_feeds_and_fetches(continue_from, signature, features,
-                                          graph):
+def _colate_features_to_feeds_and_fetches(signature, features, graph,
+                                          continue_from=None):
   """Uses a saved model signature to construct feed and fetch dictionaries."""
-  if _feature_keys.FilteringResults.STATE_TUPLE in continue_from:
+  if continue_from is None:
+    state_values = {}
+  elif _feature_keys.FilteringResults.STATE_TUPLE in continue_from:
     # We're continuing from an evaluation, so we need to unpack/flatten state.
     state_values = _head.state_to_dictionary(
         continue_from[_feature_keys.FilteringResults.STATE_TUPLE])
@@ -115,6 +118,55 @@ def predict_continuation(continue_from,
   return output
 
 
+def cold_start_filter(signatures, session, features):
+  """Perform filtering using an exported saved model.
+
+  Filtering refers to updating model state based on new observations.
+  Predictions based on the returned model state will be conditioned on these
+  observations.
+
+  Starts from the model's default/uninformed state.
+
+  Args:
+    signatures: The `MetaGraphDef` protocol buffer returned from
+      `tf.saved_model.loader.load`. Used to determine the names of Tensors to
+      feed and fetch. Must be from the same model as `continue_from`.
+    session: The session to use. The session's graph must be the one into which
+      `tf.saved_model.loader.load` loaded the model.
+    features: A dictionary mapping keys to Numpy arrays, with several possible
+      shapes (requires keys `FilteringFeatures.TIMES` and
+      `FilteringFeatures.VALUES`):
+        Single example; `TIMES` is a scalar and `VALUES` is either a scalar or a
+          vector of length [number of features].
+        Sequence; `TIMES` is a vector of shape [series length], `VALUES` either
+          has shape [series length] (univariate) or [series length x number of
+          features] (multivariate).
+        Batch of sequences; `TIMES` is a vector of shape [batch size x series
+          length], `VALUES` has shape [batch size x series length] or [batch
+          size x series length x number of features].
+      In any case, `VALUES` and any exogenous features must have their shapes
+      prefixed by the shape of the value corresponding to the `TIMES` key.
+  Returns:
+    A dictionary containing model state updated to account for the observations
+    in `features`.
+  """
+  filter_signature = signatures.signature_def[
+      _feature_keys.SavedModelLabels.COLD_START_FILTER]
+  features = _input_pipeline._canonicalize_numpy_data(  # pylint: disable=protected-access
+      data=features,
+      require_single_batch=False)
+  output_tensors_by_name, feed_dict = _colate_features_to_feeds_and_fetches(
+      signature=filter_signature,
+      features=features,
+      graph=session.graph)
+  output = session.run(output_tensors_by_name, feed_dict=feed_dict)
+  # Make it easier to chain filter -> predict by keeping track of the current
+  # time.
+  output[_feature_keys.FilteringResults.TIMES] = features[
+      _feature_keys.FilteringFeatures.TIMES]
+  return output
+
+
 def filter_continuation(continue_from, signatures, session, features):
   """Perform filtering using an exported saved model.
 
@@ -124,8 +176,8 @@ def filter_continuation(continue_from, signatures, session, features):
 
   Args:
     continue_from: A dictionary containing the results of either an Estimator's
-      evaluate method or a previous filter_continuation. Used to determine the
-      model state to start filtering from.
+      evaluate method or a previous filter step (cold start or
+      continuation). Used to determine the model state to start filtering from.
     signatures: The `MetaGraphDef` protocol buffer returned from
       `tf.saved_model.loader.load`. Used to determine the names of Tensors to
       feed and fetch. Must be from the same model as `continue_from`.
-- 
GitLab


From e3930fc11f042416a34ed5526bc506e1e0e32660 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 16 Mar 2018 17:05:51 -0700
Subject: [PATCH 138/960] Add user_ops.my_fact to the new TensorFlow API.

PiperOrigin-RevId: 189415577
---
 tensorflow/python/BUILD                              |  1 +
 tensorflow/python/user_ops/user_ops.py               |  2 ++
 tensorflow/tools/api/generator/BUILD                 |  1 +
 tensorflow/tools/api/tests/api_compatibility_test.py | 11 -----------
 4 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a206685af6..9a29986c3b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2585,6 +2585,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":user_ops_gen",
+        ":util",
         "@six_archive//:six",
     ],
 )
diff --git a/tensorflow/python/user_ops/user_ops.py b/tensorflow/python/user_ops/user_ops.py
index 6f9b5d92bb..20ea3b0f62 100644
--- a/tensorflow/python/user_ops/user_ops.py
+++ b/tensorflow/python/user_ops/user_ops.py
@@ -23,8 +23,10 @@ from tensorflow.python.ops import gen_user_ops as _gen_user_ops
 
 # go/tf-wildcard-import
 from tensorflow.python.ops.gen_user_ops import *  # pylint: disable=wildcard-import
+from tensorflow.python.util.tf_export import tf_export
 
 
+@tf_export('user_ops.my_fact')
 def my_fact():
   """Example of overriding the generated code for an Op."""
   return _gen_user_ops.fact()
diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD
index 14ce8dbeb3..d9b0260c9f 100644
--- a/tensorflow/tools/api/generator/BUILD
+++ b/tensorflow/tools/api/generator/BUILD
@@ -127,6 +127,7 @@ genrule(
         "api/test/__init__.py",
         "api/train/__init__.py",
         "api/train/queue_runner/__init__.py",
+        "api/user_ops/__init__.py",
     ],
     cmd = "$(location create_python_api) $(OUTS)",
     tools = ["create_python_api"],
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 5268bba3cc..99c47fd601 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -268,17 +268,6 @@ class ApiCompatibilityTest(test.TestCase):
         for filename in golden_file_list
     }
 
-    # user_ops is an empty module. It is currently available in TensorFlow API
-    # but we don't keep empty modules in the new API.
-    # We delete user_ops from golden_proto_dict to make sure assert passes
-    # when diffing new API against goldens.
-    # TODO(annarev): remove user_ops from goldens once we switch to new API.
-    tf_module = golden_proto_dict['tensorflow'].tf_module
-    for i in range(len(tf_module.member)):
-      if tf_module.member[i].name == 'user_ops':
-        del tf_module.member[i]
-        break
-
     # Diff them. Do not fail if called with update.
     # If the test is run to update goldens, only report diffs but do not fail.
     self._AssertProtoDictEquals(
-- 
GitLab


From 85087076d1e0a50a96870643e1413720ba77403f Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 16 Mar 2018 17:10:16 -0700
Subject: [PATCH 139/960] Automated g4 rollback of changelist 189228094

PiperOrigin-RevId: 189416074
---
 tensorflow/python/client/session.py      | 19 ---------------
 tensorflow/python/client/session_test.py | 31 ------------------------
 2 files changed, 50 deletions(-)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 29f06c8f22..924d62992a 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import functools
 import re
 import threading
-import warnings
 
 import numpy as np
 
@@ -1625,9 +1624,6 @@ class InteractiveSession(BaseSession):
   ```
   """
 
-  _count_lock = threading.Lock()
-  _active_session_count = 0  # GUARDED_BY(_count_lock)
-
   def __init__(self, target='', graph=None, config=None):
     """Creates a new interactive TensorFlow session.
 
@@ -1656,15 +1652,6 @@ class InteractiveSession(BaseSession):
     config.graph_options.place_pruned_graph = True
 
     super(InteractiveSession, self).__init__(target, graph, config)
-    with InteractiveSession._count_lock:
-      if InteractiveSession._active_session_count > 0:
-        warnings.warn('An interactive session is already active. This can '
-                      'cause out-of-memory errors in some cases. You must '
-                      'explicitly call `InteractiveSession.close()` to release '
-                      'resources held by the other session(s).')
-      InteractiveSession._active_session_count += 1
-    self._closed = False
-
     self._default_session = self.as_default()
     self._default_session.enforce_nesting = False
     self._default_session.__enter__()
@@ -1677,12 +1664,6 @@ class InteractiveSession(BaseSession):
   def close(self):
     """Closes an `InteractiveSession`."""
     super(InteractiveSession, self).close()
-    with InteractiveSession._count_lock:
-      if not self._closed:
-        InteractiveSession._active_session_count -= 1
-        self._closed = True
-      else:
-        return
     if self._explicit_graph is not None:
       self._default_graph.__exit__(None, None, None)
     self._default_session.__exit__(None, None, None)
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 3b12e06f43..3bf2a9e4dd 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -22,7 +22,6 @@ import os
 import sys
 import threading
 import time
-import warnings
 
 import numpy as np
 import six
@@ -66,10 +65,6 @@ ops.RegisterShape('ConstructionFails')(common_shapes.unknown_shape)
 # @test_util.with_c_api
 class SessionTest(test_util.TensorFlowTestCase):
 
-  def setUp(self):
-    super(SessionTest, self).setUp()
-    warnings.simplefilter('always')
-
   def testUseExistingGraph(self):
     with ops.Graph().as_default() as g, ops.device('/cpu:0'):
       a = constant_op.constant(6.0, shape=[1, 1])
@@ -1195,32 +1190,6 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([[24.0]], e.eval())
       sess.close()
 
-  def testMultipleInteractiveSessionsWarning(self):
-    # Reinitialize the global state to ensure that the expected warnings will
-    # be emitted.
-    session.InteractiveSession._active_session_count = 0  # pylint: disable=protected-access
-
-    sess = session.InteractiveSession()
-    sess.close()
-    # Opening and closing interactive sessions serially should not warn.
-    with warnings.catch_warnings(record=True) as w:
-      sess = session.InteractiveSession()
-      sess.close()
-    self.assertEqual(0, len(w))
-
-    with warnings.catch_warnings(record=True) as w:
-      sess = session.InteractiveSession()
-    self.assertEqual(0, len(w))
-    with warnings.catch_warnings(record=True) as w:
-      sess2 = session.InteractiveSession()
-    self.assertEqual(1, len(w))
-    self.assertTrue('An interactive session is already active. This can cause '
-                    'out-of-memory errors in some cases. You must explicitly '
-                    'call `InteractiveSession.close()` to release resources '
-                    'held by the other session(s).' in str(w[0].message))
-    sess2.close()
-    sess.close()
-
   def testInteractivePlacePrunedGraph(self):
     sess = session.InteractiveSession()
 
-- 
GitLab


From dc492520e408a9f3f771ec960d3c4db96be42265 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Sat, 17 Mar 2018 10:14:20 +0800
Subject: [PATCH 140/960] Fix incorrect link of checkpoint files in CNN
 tutorials (#17776)

---
 tensorflow/docs_src/tutorials/deep_cnn.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/deep_cnn.md b/tensorflow/docs_src/tutorials/deep_cnn.md
index 6797540204..6a4c9a9b07 100644
--- a/tensorflow/docs_src/tutorials/deep_cnn.md
+++ b/tensorflow/docs_src/tutorials/deep_cnn.md
@@ -268,7 +268,7 @@ in `cifar10_input.py`.
 
 `cifar10_train.py` periodically @{tf.train.Saver$saves}
 all model parameters in
-@{$variables#saving-and-restoring$checkpoint files}
+@{$programmers_guide/saved_model$checkpoint files}
 but it does *not* evaluate the model. The checkpoint file
 will be used by `cifar10_eval.py` to measure the predictive
 performance (see [Evaluating a Model](#evaluating-a-model) below).
-- 
GitLab


From d8315b74750683cec1758149afffa1ec9213120f Mon Sep 17 00:00:00 2001
From: Raghuraman Krishnamoorthi <raghuramank@google.com>
Date: Fri, 16 Mar 2018 19:26:56 -0700
Subject: [PATCH 141/960] Update docs for fake quant to reflect support for
 bitwidths from 2 to 16 inclusive.

PiperOrigin-RevId: 189426173
---
 .../core/api_def/base_api/api_def_FakeQuantWithMinMaxArgs.pbtxt | 2 +-
 .../core/api_def/base_api/api_def_FakeQuantWithMinMaxVars.pbtxt | 2 +-
 .../base_api/api_def_FakeQuantWithMinMaxVarsPerChannel.pbtxt    | 2 +-
 .../api_def_FakeQuantWithMinMaxVarsPerChannelGradient.pbtxt     | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxArgs.pbtxt b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxArgs.pbtxt
index 561c86ddf6..599bbce65f 100644
--- a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxArgs.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxArgs.pbtxt
@@ -6,7 +6,7 @@ Attributes `[min; max]` define the clamping range for the `inputs` data.
 `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
 when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
 then de-quantized and output as floats in `[min; max]` interval.
-`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+`num_bits` is the bitwidth of the quantization; between 2 and 16, inclusive.
 
 Quantization is called fake since the output is still in floating point.
 END
diff --git a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVars.pbtxt b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVars.pbtxt
index 2713c01b27..1976ffb8aa 100644
--- a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVars.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVars.pbtxt
@@ -8,7 +8,7 @@ and `max` to 'outputs' tensor of same shape as `inputs`.
 `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
 when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
 then de-quantized and output as floats in `[min; max]` interval.
-`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+`num_bits` is the bitwidth of the quantization; between 2 and 16, inclusive.
 
 This operation has a gradient and thus allows for training `min` and `max`
 values.
diff --git a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannel.pbtxt b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannel.pbtxt
index e293d4d084..c0fac6a445 100644
--- a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannel.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannel.pbtxt
@@ -9,7 +9,7 @@ to 'outputs' tensor of same shape as `inputs`.
 `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
 when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
 then de-quantized and output as floats in `[min; max]` interval.
-`num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+`num_bits` is the bitwidth of the quantization; between 2 and 16, inclusive.
 
 This operation has a gradient and thus allows for training `min` and `max`
 values.
diff --git a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannelGradient.pbtxt b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannelGradient.pbtxt
index 8a4ab368b5..2051903f6d 100644
--- a/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannelGradient.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_FakeQuantWithMinMaxVarsPerChannelGradient.pbtxt
@@ -40,7 +40,7 @@ END
   attr {
     name: "num_bits"
     description: <<END
-The bitwidth of the quantization; between 2 and 8, inclusive.
+The bitwidth of the quantization; between 2 and 16, inclusive.
 END
   }
   attr {
-- 
GitLab


From a7733cc85e6ce90d513668301144c8b4d4fe88cb Mon Sep 17 00:00:00 2001
From: ManHyuk <manhyuk@kw.ac.kr>
Date: Sat, 17 Mar 2018 11:44:41 +0900
Subject: [PATCH 142/960] Fix typo (#17759)

* fix typo
---
 tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index 272410c693..7651a03fe5 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -398,7 +398,7 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNorms) {
 }
 
 TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) {
-  // Test axis is not 3, so all weigths and offsets are fused to each of inputs
+  // Test axis is not 3, so all weights and offsets are fused to each of inputs
   // of conv2d.
   TestFoldFusedBatchNormsWithConcat(/*split=*/true);
   // Test axis = 3, BatchNorm weights and offsets will be split before fused
-- 
GitLab


From c941c087a9dfd5b27eff00ead928c9ee208e9a35 Mon Sep 17 00:00:00 2001
From: Giovanni Terlingen <me@giovanniterlingen.com>
Date: Sat, 17 Mar 2018 04:33:26 +0100
Subject: [PATCH 143/960] Add missing linking libs for iOS (#17764)

* Add missing linking libs for iOS
---
 tensorflow/examples/ios/README.md | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/ios/README.md b/tensorflow/examples/ios/README.md
index 5bdaeb43ce..5d7bd36837 100644
--- a/tensorflow/examples/ios/README.md
+++ b/tensorflow/examples/ios/README.md
@@ -119,11 +119,13 @@ rundown:
    `tensorflow/contrib/makefile/gen/lib` to the Library Search Paths setting.
 
  - You'll also need to add `libprotobuf.a` and `libprotobuf-lite.a` from
-   `tensorflow/contrib/makefile/gen/protobuf_ios/lib` to your _Build Stages_ and
-   _Library Search Paths_.
+   `tensorflow/contrib/makefile/gen/protobuf_ios/lib`
+   and `nsync.a` from `tensorflow/contrib/makefile/downloads/nsync/builds/lipo.ios.c++11` 
+   to your _Build Stages_ and _Library Search Paths_.
 
  - The _Header Search_ paths needs to contain:
    - the root folder of tensorflow,
+   - `tensorflow/contrib/makefile/downloads/nsync/public`
    - `tensorflow/contrib/makefile/downloads/protobuf/src`
    - `tensorflow/contrib/makefile/downloads`,
    - `tensorflow/contrib/makefile/downloads/eigen`, and
-- 
GitLab


From a14114177ffec4f0c8665db60bec8819aade0c81 Mon Sep 17 00:00:00 2001
From: Yifei Feng <1192265+yifeif@users.noreply.github.com>
Date: Fri, 16 Mar 2018 22:58:50 -0700
Subject: [PATCH 144/960] Update version strings for 1.7.0-rc1 (#17786)

---
 tensorflow/core/public/version.h              |  2 +-
 tensorflow/docs_src/install/install_c.md      |  2 +-
 tensorflow/docs_src/install/install_go.md     |  2 +-
 tensorflow/docs_src/install/install_java.md   | 22 +++++++++----------
 tensorflow/docs_src/install/install_linux.md  | 22 +++++++++----------
 tensorflow/docs_src/install/install_mac.md    | 10 ++++-----
 .../docs_src/install/install_sources.md       | 14 ++++++------
 tensorflow/tools/pip_package/setup.py         |  2 +-
 8 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 15082bb337..40eebd1db0 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX "-rc0"
+#define TF_VERSION_SUFFIX "-rc1"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 733c7a6625..9059b3f3b6 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 421215f367..2e47a6d212 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 7758520c50..eff066d200 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.7.0-rc0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.7.0-rc0</version>
+                 <version>1.7.0-rc1</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.7.0-rc0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.7.0-rc0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.7.0-rc0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index f4d4e65548..378946b459 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -189,7 +189,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -294,7 +294,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -480,7 +480,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -647,14 +647,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -666,14 +666,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -685,14 +685,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -704,14 +704,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 055a463718..fa6951a8f1 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -523,7 +523,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
 </pre>
 
 
@@ -531,5 +531,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 10840295f9..0454c172f8 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -359,10 +359,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.7.0rc0 on Linux:
+for TensorFlow 1.7.0rc1 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc0-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -459,8 +459,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -480,7 +480,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -495,8 +495,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 69825a0d7c..7a3184d64d 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.7.0-rc0'
+_VERSION = '1.7.0-rc1'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
-- 
GitLab


From dd2d558714ccbf24e180f075746bdc4bbb745b97 Mon Sep 17 00:00:00 2001
From: Aghasy <agasy18@gmail.com>
Date: Sat, 17 Mar 2018 17:27:24 +0400
Subject: [PATCH 145/960] fix nested scope issue

---
 tensorflow/core/platform/default/logging.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/platform/default/logging.h b/tensorflow/core/platform/default/logging.h
index f0efa31d55..2c134f1be9 100644
--- a/tensorflow/core/platform/default/logging.h
+++ b/tensorflow/core/platform/default/logging.h
@@ -64,11 +64,11 @@ class LogMessageFatal : public LogMessage {
 };
 
 #define _TF_LOG_INFO \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::INFO)
 #define _TF_LOG_WARNING \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::WARNING)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::WARNING)
 #define _TF_LOG_ERROR \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::ERROR)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::ERROR)
 #define _TF_LOG_FATAL \
   ::tensorflow::internal::LogMessageFatal(__FILE__, __LINE__)
 
-- 
GitLab


From 6a0b4e177620626596c610f129a66233ffb6f5af Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Sat, 17 Mar 2018 09:57:03 -0700
Subject: [PATCH 146/960] [XLA] Fix points-to set calculation in HLO
 ListScheduler.

Previously the list scheduler considered that an instruction used only the buffers defined by its operands. This is inaccurate in the presence of aliasing?an instruction may potentially use anything in the points-to set of the operand, including buffers defined by an ancestor of an operand. Change to use the full points-to set instead.

PiperOrigin-RevId: 189460681
---
 tensorflow/compiler/xla/service/BUILD         |   1 +
 .../compiler/xla/service/hlo_ordering_test.cc |  47 --------
 .../compiler/xla/service/hlo_scheduling.cc    |   9 +-
 .../xla/service/hlo_scheduling_test.cc        | 102 ++++++++++++++++++
 4 files changed, 108 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index fba20c94ca..43c56484ea 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1129,6 +1129,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/compiler/xla/tools/parser:hlo_parser",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/hlo_ordering_test.cc b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
index 441d790f0e..37a7fbad97 100644
--- a/tensorflow/compiler/xla/service/hlo_ordering_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_ordering_test.cc
@@ -34,53 +34,6 @@ namespace {
 
 class HloOrderingTest : public HloTestBase {};
 
-TEST_F(HloOrderingTest, LastUseScheduledFirst) {
-  // Tests scheduling of the following HLO code:
-  //
-  //   %ab = abs(%param)
-  //   %exp = exp(%param)
-  //   %add = add(%ab, %exp)
-  //   %negate = negate(%exp)
-  //   %sub = subtract(%add, %negate)
-  //
-  // %add should be scheduled before %negate because %add is the last (and only)
-  // use of %ab. Scheduling %add first then frees up %ab's buffer.
-  const Shape vec = ShapeUtil::MakeShape(xla::F32, {42});
-  auto builder = HloComputation::Builder(TestName());
-  auto param =
-      builder.AddInstruction(HloInstruction::CreateParameter(0, vec, "param"));
-  auto ab = builder.AddInstruction(
-      HloInstruction::CreateUnary(vec, HloOpcode::kAbs, param));
-  auto exp = builder.AddInstruction(
-      HloInstruction::CreateUnary(vec, HloOpcode::kExp, param));
-
-  auto add = builder.AddInstruction(
-      HloInstruction::CreateBinary(vec, HloOpcode::kAdd, ab, exp));
-  auto negate = builder.AddInstruction(
-      HloInstruction::CreateUnary(vec, HloOpcode::kNegate, exp));
-  auto sub = builder.AddInstruction(
-      HloInstruction::CreateBinary(vec, HloOpcode::kSubtract, add, negate));
-
-  auto module = CreateNewModule();
-  module->AddEntryComputation(builder.Build());
-
-  TF_ASSERT_OK_AND_ASSIGN(
-      SequentialHloOrdering::HloModuleSequence sequence,
-      CreateMemoryMinimizingSequence(*module, [](const LogicalBuffer& buffer) {
-        return ShapeUtil::ByteSizeOf(buffer.shape());
-      }));
-  // Verify that all instructions are in the sequence.
-  EXPECT_EQ(module->entry_computation()->instruction_count(),
-            sequence.at(module->entry_computation()).size());
-
-  // The first instruction should be the parameter and the last the root "sub".
-  EXPECT_EQ(param, sequence.at(module->entry_computation()).front());
-  EXPECT_EQ(sub, sequence.at(module->entry_computation()).back());
-
-  SequentialHloOrdering ordering(module.get(), sequence);
-  EXPECT_TRUE(ordering.ExecutesBefore(add, negate));
-}
-
 TEST_F(HloOrderingTest, InstructionsInDifferentComputations) {
   // Tests the ordering of instructions in different computations using the
   // following HLO code:
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling.cc b/tensorflow/compiler/xla/service/hlo_scheduling.cc
index da448ed71a..099dd8dd8e 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling.cc
+++ b/tensorflow/compiler/xla/service/hlo_scheduling.cc
@@ -103,10 +103,11 @@ class ListScheduler {
     for (auto* instruction : computation.instructions()) {
       tensorflow::gtl::FlatSet<const LogicalBuffer*> instr_uses;
       for (auto* operand : instruction->operands()) {
-        for (const LogicalBuffer* buffer :
-             points_to_analysis.GetBuffersDefinedByInstruction(operand)) {
-          instr_uses.insert(buffer);
-        }
+        points_to_analysis.GetPointsToSet(operand).ForEachElement(
+            [&](const ShapeIndex& /*index*/,
+                const PointsToSet::BufferList& buffers) {
+              instr_uses.insert(buffers.begin(), buffers.end());
+            });
       }
       buffer_uses_[instruction] = std::vector<const LogicalBuffer*>(
           instr_uses.begin(), instr_uses.end());
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling_test.cc b/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
index 7fb338e704..2dd6e43851 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_ordering.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
@@ -89,5 +90,106 @@ TEST_F(MinimumMemoryForSequenceTest, MultiComputation) {
             MinimumMemoryForSequence(module_sequence, size_fn).ValueOrDie());
 }
 
+class HloSchedulingTest : public HloTestBase {};
+
+TEST_F(HloSchedulingTest, LastUseScheduledFirst) {
+  // Tests scheduling of the following HLO code:
+  //
+  //   %ab = abs(%param)
+  //   %exp = exp(%param)
+  //   %add = add(%ab, %exp)
+  //   %negate = negate(%exp)
+  //   %sub = subtract(%add, %negate)
+  //
+  // %add should be scheduled before %negate because %add is the last (and only)
+  // use of %ab. Scheduling %add first then frees up %ab's buffer.
+  const Shape vec = ShapeUtil::MakeShape(xla::F32, {42});
+  auto builder = HloComputation::Builder(TestName());
+  auto param =
+      builder.AddInstruction(HloInstruction::CreateParameter(0, vec, "param"));
+  auto ab = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec, HloOpcode::kAbs, param));
+  auto exp = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec, HloOpcode::kExp, param));
+
+  auto add = builder.AddInstruction(
+      HloInstruction::CreateBinary(vec, HloOpcode::kAdd, ab, exp));
+  auto negate = builder.AddInstruction(
+      HloInstruction::CreateUnary(vec, HloOpcode::kNegate, exp));
+  auto sub = builder.AddInstruction(
+      HloInstruction::CreateBinary(vec, HloOpcode::kSubtract, add, negate));
+
+  auto module = CreateNewModule();
+  module->AddEntryComputation(builder.Build());
+
+  TF_ASSERT_OK_AND_ASSIGN(
+      SequentialHloOrdering::HloModuleSequence sequence,
+      CreateMemoryMinimizingSequence(*module, [](const LogicalBuffer& buffer) {
+        return ShapeUtil::ByteSizeOf(buffer.shape());
+      }));
+  // Verify that all instructions are in the sequence.
+  EXPECT_EQ(module->entry_computation()->instruction_count(),
+            sequence.at(module->entry_computation()).size());
+
+  // The first instruction should be the parameter and the last the root "sub".
+  EXPECT_EQ(param, sequence.at(module->entry_computation()).front());
+  EXPECT_EQ(sub, sequence.at(module->entry_computation()).back());
+
+  SequentialHloOrdering ordering(module.get(), sequence);
+  EXPECT_TRUE(ordering.ExecutesBefore(add, negate));
+}
+
+TEST_F(HloSchedulingTest, ListSchedulerHandlesAliasing) {
+  const char* module_str = R"(
+HloModule test_aliasing_module
+
+ENTRY root {
+  param = s32[1000] parameter(0)
+  p0 = s32[1000] copy(param)
+  p1 = s32[1000] copy(param)
+  t = (s32[1000], s32[1000]) tuple(p0, p1)
+  a = s32[1000] get-tuple-element(t), index=0
+  b = s32[1000] get-tuple-element(t), index=1
+  c = s32[1000] add(a, b)
+  d = s32[1000] add(c, b)
+  e = s32[1000] add(c, c)
+  f = s32[1000] add(e, e)
+  ROOT result = (s32[1000], s32[1000], s32[1000]) tuple(d, e, f)
+})";
+
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          tools::Parse(module_str));
+
+  auto size_fn = [](const LogicalBuffer& buffer) {
+    return ShapeUtil::ByteSizeOf(buffer.shape(), /*pointer_size=*/8);
+  };
+  TF_ASSERT_OK_AND_ASSIGN(
+      SequentialHloOrdering::HloModuleSequence sequence,
+      CreateMemoryMinimizingSequence(*module, size_fn,
+                                     SchedulerAlgorithm::kListSchedule));
+  // Verify that all instructions are in the sequence.
+  EXPECT_EQ(module->entry_computation()->instruction_count(),
+            sequence.at(module->entry_computation()).size());
+
+  std::unordered_map<string, const HloInstruction*> instructions_by_name;
+  for (const HloInstruction* instruction :
+       sequence.at(module->entry_computation())) {
+    instructions_by_name[instruction->name()] = instruction;
+  }
+
+  // The first instruction should be the parameter and the last the root.
+  EXPECT_EQ(instructions_by_name.at("param"),
+            sequence.at(module->entry_computation()).front());
+  EXPECT_EQ(instructions_by_name.at("result"),
+            sequence.at(module->entry_computation()).back());
+
+  // Instructions "d" and "e" will both be schedulable at the same time, but
+  // instruction "d" allows us to free the buffer of "p1", so the list scheduler
+  // should prefer it.
+  SequentialHloOrdering ordering(module.get(), sequence);
+  EXPECT_TRUE(ordering.ExecutesBefore(instructions_by_name.at("d"),
+                                      instructions_by_name.at("e")));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 6e20f3bdbdaf9bae2a67ee9cc9728963bc8b563f Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Sat, 17 Mar 2018 19:16:21 +0100
Subject: [PATCH 147/960] Added training parameter to batch_normalization
 (#16134)

* Added training parameter to batch_normalization

According to the docs the batch_normalization layer does not work properly if the parameter is not set correctly.

* Clean up multiple tests for training mode
---
 tensorflow/examples/learn/resnet.py | 12 +++++++-----
 1 file changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py
index 9542e55250..c00de932a8 100755
--- a/tensorflow/examples/learn/resnet.py
+++ b/tensorflow/examples/learn/resnet.py
@@ -53,6 +53,8 @@ def res_net_model(features, labels, mode):
     ndim = int(sqrt(input_shape[1]))
     x = tf.reshape(x, [-1, ndim, ndim, 1])
 
+  training = (mode == tf.estimator.ModeKeys.TRAIN)
+  
   # First convolution expands to 64 channels
   with tf.variable_scope('conv_layer1'):
     net = tf.layers.conv2d(
@@ -60,7 +62,7 @@ def res_net_model(features, labels, mode):
         filters=64,
         kernel_size=7,
         activation=tf.nn.relu)
-    net = tf.layers.batch_normalization(net)
+    net = tf.layers.batch_normalization(net, training=training)
 
   # Max pool
   net = tf.layers.max_pooling2d(
@@ -88,7 +90,7 @@ def res_net_model(features, labels, mode):
             kernel_size=1,
             padding='valid',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       with tf.variable_scope(name + '/conv_bottleneck'):
         conv = tf.layers.conv2d(
@@ -97,7 +99,7 @@ def res_net_model(features, labels, mode):
             kernel_size=3,
             padding='same',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       # 1x1 convolution responsible for restoring dimension
       with tf.variable_scope(name + '/conv_out'):
@@ -108,7 +110,7 @@ def res_net_model(features, labels, mode):
             kernel_size=1,
             padding='valid',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       # shortcut connections that turn the network into its counterpart
       # residual function (identity shortcut)
@@ -154,7 +156,7 @@ def res_net_model(features, labels, mode):
   loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
 
   # Create training op.
-  if mode == tf.estimator.ModeKeys.TRAIN:
+  if training:
     optimizer = tf.train.AdagradOptimizer(learning_rate=0.01)
     train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
     return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
-- 
GitLab


From 1fb724f28486d1eec7d9368b6c3b8600664cf8a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sat, 17 Mar 2018 11:21:02 -0700
Subject: [PATCH 148/960] Normally tf2xla (autoclustering, jit_scope and
 rewrite) rely on graph optimization passes to outline subgraphs. The XLA
 device itself only sees Compute() calls for _XlaLaunch ops. All other ops are
 registered with a dummy op factory that just prints an error.

This patch adds an alternative, selected at registration time, that disables
default graph optimization and instead registers a non-dummy op implementation.

This op implementation compiles the op "on demand"; it generates a fake graph containing
_Arg and _Retval nodes and calls into the XlaCompiler code as usual.

This allows the device to be used as a "normal" TensorFlow device, as well as from
Eager mode, at the expense of performance.

Later additions will add the ability to create traces to amortize kernel launch overhead,
and the ability to combine op-by-op/tracing and autoclustering with jit_scope annotations.

PiperOrigin-RevId: 189463593
---
 tensorflow/compiler/jit/BUILD                 |   3 +
 .../compiler/jit/kernels/xla_launch_op.cc     |   6 +-
 tensorflow/compiler/jit/legacy_flags/BUILD    |  12 ++
 .../jit/legacy_flags/xla_device_flags.cc      |  56 +++++
 .../jit/legacy_flags/xla_device_flags.h       |  47 ++++
 .../compiler/jit/xla_compilation_cache.cc     | 202 +++++++++---------
 .../compiler/jit/xla_compilation_cache.h      |  30 ++-
 .../compiler/jit/xla_compile_on_demand_op.cc  | 178 +++++++++++++++
 .../compiler/jit/xla_compile_on_demand_op.h   |  56 +++++
 tensorflow/compiler/jit/xla_cpu_device.cc     |  13 +-
 tensorflow/compiler/jit/xla_device.cc         |  31 ++-
 tensorflow/compiler/jit/xla_device.h          |   7 +-
 tensorflow/compiler/jit/xla_device_context.cc |   4 +
 tensorflow/compiler/jit/xla_gpu_device.cc     |  14 +-
 tensorflow/compiler/jit/xla_launch_util.cc    |  14 +-
 tensorflow/compiler/jit/xla_tensor_info.h     |  16 ++
 tensorflow/compiler/tests/BUILD               |  11 +-
 tensorflow/compiler/tests/xla_test.py         |   8 +
 .../tf2xla/kernels/batchtospace_op.cc         |   4 +-
 .../tf2xla/kernels/segment_reduction_ops.cc   |   4 +-
 .../tf2xla/kernels/stateless_random_ops.cc    |   2 +
 tensorflow/compiler/tf2xla/xla_compiler.cc    |  42 ++++
 tensorflow/compiler/tf2xla/xla_compiler.h     |   8 +
 23 files changed, 637 insertions(+), 131 deletions(-)
 create mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
 create mode 100644 tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
 create mode 100644 tensorflow/compiler/jit/xla_compile_on_demand_op.cc
 create mode 100644 tensorflow/compiler/jit/xla_compile_on_demand_op.h

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 39eb390f38..0475cd9ff2 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -76,6 +76,7 @@ cc_library(
         ":jit_compilation_passes",
         ":xla_device",
         "//tensorflow/compiler/jit/kernels:xla_launch_op",
+        "//tensorflow/compiler/jit/legacy_flags:xla_device_flags",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/tf2xla/kernels:xla_ops",
         "//tensorflow/compiler/xla/service:cpu_plugin",  # buildcleaner: keep
@@ -136,11 +137,13 @@ cc_library(
 cc_library(
     name = "xla_device",
     srcs = [
+        "xla_compile_on_demand_op.cc",
         "xla_device.cc",
         "xla_device_context.cc",
         "xla_device_ops.cc",
     ],
     hdrs = [
+        "xla_compile_on_demand_op.h",
         "xla_device.h",
         "xla_device_context.h",
         "xla_device_ops.h",
diff --git a/tensorflow/compiler/jit/kernels/xla_launch_op.cc b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
index e24a9a0751..8a8e8bb8df 100644
--- a/tensorflow/compiler/jit/kernels/xla_launch_op.cc
+++ b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
@@ -148,7 +148,11 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
   const XlaCompiler::CompilationResult* kernel;
   xla::LocalExecutable* executable;
 
-  OP_REQUIRES_OK(ctx, cache->Compile(options, function_, num_constant_args_,
+  std::map<int, Tensor> constant_args;
+  for (int i = 0; i < num_constant_args_; ++i) {
+    constant_args.insert({i, ctx->input(i)});
+  }
+  OP_REQUIRES_OK(ctx, cache->Compile(options, function_, constant_args,
                                      variables, ctx, &kernel, &executable,
                                      /*compile_options=*/nullptr));
 
diff --git a/tensorflow/compiler/jit/legacy_flags/BUILD b/tensorflow/compiler/jit/legacy_flags/BUILD
index 4491dd6ac8..9cd66fc13c 100644
--- a/tensorflow/compiler/jit/legacy_flags/BUILD
+++ b/tensorflow/compiler/jit/legacy_flags/BUILD
@@ -52,6 +52,18 @@ cc_library(
         ],
 )
 
+cc_library(
+    name = "xla_device_flags",
+    srcs = ["xla_device_flags.cc"],
+    hdrs = ["xla_device_flags.h"],
+    deps =
+        [
+            "//tensorflow/compiler/xla/legacy_flags:parse_flags_from_env",
+            "//tensorflow/core:framework_internal",
+            "//tensorflow/core:lib",
+        ],
+)
+
 # -----------------------------------------------------------------------------
 
 filegroup(
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
new file mode 100644
index 0000000000..1bb2fce2db
--- /dev/null
+++ b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.cc
@@ -0,0 +1,56 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Legacy flags for the XLA bridge's xla_device module.
+
+#include <mutex>
+#include <vector>
+
+#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h"
+#include "tensorflow/compiler/xla/legacy_flags/parse_flags_from_env.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+namespace legacy_flags {
+
+// Pointers to the parsed value of the flags and flag descriptors, initialized
+// via flags_init.
+static XlaDeviceFlags* flags;
+static std::vector<Flag>* flag_list;
+static std::once_flag flags_init;
+
+// Allocate *flags.  Called via call_once(&flags_init,...).
+static void AllocateFlags() {
+  flags = new XlaDeviceFlags;
+  flags->tf_xla_compile_on_demand = false;
+  flag_list = new std::vector<Flag>({
+      Flag("tf_xla_compile_on_demand", &flags->tf_xla_compile_on_demand,
+           "Switch a device into 'on-demand' mode, where instead of "
+           "autoclustering ops are compiled one by one just-in-time."),
+  });
+  xla::legacy_flags::ParseFlagsFromEnv(*flag_list);
+}
+
+// Return a pointer to the XlaDeviceFlags struct;
+// repeated calls return the same pointer.
+// This should be called only after Flags::Parse() has returned.
+XlaDeviceFlags* GetXlaDeviceFlags() {
+  std::call_once(flags_init, &AllocateFlags);
+  return flags;
+}
+
+}  // namespace legacy_flags
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
new file mode 100644
index 0000000000..27b22121ac
--- /dev/null
+++ b/tensorflow/compiler/jit/legacy_flags/xla_device_flags.h
@@ -0,0 +1,47 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
+#define TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
+
+// Legacy flags for the XLA bridge's xla_device module.
+
+#include <vector>
+
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/command_line_flags.h"
+
+namespace tensorflow {
+namespace legacy_flags {
+
+// The values of flags associated with the XLA bridge's
+// xla_device module.
+typedef struct {
+  // Switch the CPU device into "on-demand" mode, where instead of
+  // autoclustering ops are compiled one by one just-in-time.
+  // Enabling this mode by a legacy flag is a temporary mechanism. When this
+  // feature is battle-tested, we will switch this to be a session option.
+  bool tf_xla_compile_on_demand;
+} XlaDeviceFlags;
+
+// Return a pointer to the XlaDeviceFlags struct;
+// repeated calls return the same pointer.
+// This should be called only after Flags::Parse() has returned.
+XlaDeviceFlags* GetXlaDeviceFlags();
+
+}  // namespace legacy_flags
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_JIT_LEGACY_FLAGS_XLA_DEVICE_FLAGS_H_
diff --git a/tensorflow/compiler/jit/xla_compilation_cache.cc b/tensorflow/compiler/jit/xla_compilation_cache.cc
index 8cc79a9bd0..6430975335 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.cc
+++ b/tensorflow/compiler/jit/xla_compilation_cache.cc
@@ -92,39 +92,30 @@ uint64 XlaCompilationCache::Signature::Hash::operator()(
 }
 
 Status XlaCompilationCache::BuildSignature(
-    const NameAttrList& function, int num_constant_args,
+    const NameAttrList& function, const std::map<int, Tensor>& constant_args,
     const std::map<int, OptionalTensor>& variable_args, OpKernelContext* ctx,
     Signature* signature) {
   signature->name = Canonicalize(function.name(), AttrSlice(&function.attr()));
-  signature->arg_values.resize(num_constant_args);
-
-  signature->arg_types.reserve(ctx->num_inputs() - num_constant_args);
-
-  // Inputs are in the order: constants, non-constants, resource variables.
-  int input_num = 0;
-  // Use the values of compile time constants in the signature->
-  while (input_num < num_constant_args) {
-    signature->arg_values[input_num] = ctx->input(input_num);
-    ++input_num;
-  }
-  // Add the types and shapes of the remaining arguments.
-  while (input_num < ctx->num_inputs() - variable_args.size()) {
-    signature->arg_types.emplace_back(ctx->input_dtype(input_num),
-                                      ctx->input(input_num).shape());
-    ++input_num;
-  }
-  // For variable signatures, use the type and shape of the variable's
-  // current value.
-  for (auto& iterator : variable_args) {
-    const OptionalTensor& variable = iterator.second;
-    TF_RET_CHECK(input_num < ctx->num_inputs());
-    if (variable.present) {
-      signature->arg_types.emplace_back(variable.value.dtype(),
-                                        variable.value.shape());
+  signature->arg_values.reserve(constant_args.size());
+
+  signature->arg_types.reserve(ctx->num_inputs() - constant_args.size());
+
+  for (int i = 0; i < ctx->num_inputs(); ++i) {
+    if (constant_args.count(i) > 0) {
+      // Use the values of compile time constants in the signature.
+      signature->arg_values.push_back(constant_args.at(i));
+    } else if (variable_args.count(i) > 0) {
+      const OptionalTensor& variable = variable_args.at(i);
+      if (variable.present) {
+        signature->arg_types.emplace_back(variable.value.dtype(),
+                                          variable.value.shape());
+      } else {
+        signature->arg_types.emplace_back(DT_INVALID, TensorShape());
+      }
     } else {
-      signature->arg_types.emplace_back(DT_INVALID, TensorShape());
+      signature->arg_types.emplace_back(ctx->input_dtype(i),
+                                        ctx->input(i).shape());
     }
-    ++input_num;
   }
   return Status::OK();
 }
@@ -132,74 +123,58 @@ Status XlaCompilationCache::BuildSignature(
 namespace {
 
 // Builds a XlaCompiler::Argument vector from the arguments to the _XlaLaunch
-// op. The first `num_constant_args` arguments must be host-memory Tensors.
-Status BuildArguments(int num_constant_args,
+// op.
+Status BuildArguments(const std::map<int, Tensor>& constant_args,
                       const std::map<int, OptionalTensor>& variable_args,
                       OpKernelContext* ctx,
                       std::vector<XlaCompiler::Argument>* args) {
   args->resize(ctx->num_inputs());
 
-  int input_num = 0;
-
-  // Handles compile-time constants.
-  TF_RET_CHECK(num_constant_args <= ctx->num_inputs());
-  while (input_num < num_constant_args) {
-    const Tensor& input = ctx->input(input_num);
-    TF_RET_CHECK(input.dtype() != DT_RESOURCE);
-    XlaCompiler::Argument& arg = (*args)[input_num];
-    arg.kind = XlaCompiler::Argument::kConstant;
-    arg.type = input.dtype();
-    arg.shape = input.shape();
-    arg.constant_value = input;
-    ++input_num;
-  }
-
-  // Handles the non-constant arguments.
-  int num_variable_args = variable_args.size();
-  int num_nonconst_args =
-      ctx->num_inputs() - num_variable_args - num_constant_args;
-  TF_RET_CHECK(num_nonconst_args >= 0);
-  while (input_num < num_constant_args + num_nonconst_args) {
-    const Tensor& input = ctx->input(input_num);
-    TF_RET_CHECK(input.dtype() != DT_RESOURCE);
+  for (int64 input_num = 0; input_num < ctx->num_inputs(); ++input_num) {
     XlaCompiler::Argument& arg = (*args)[input_num];
-    if (input.NumElements() > 0) {
-      arg.kind = XlaCompiler::Argument::kParameter;
-    } else {
+    if (constant_args.count(input_num) > 0) {
+      // Handles compile-time constants.
+      const Tensor& input = constant_args.at(input_num);
+      TF_RET_CHECK(input.dtype() != DT_RESOURCE);
       arg.kind = XlaCompiler::Argument::kConstant;
+      arg.type = input.dtype();
+      arg.shape = input.shape();
       arg.constant_value = input;
-    }
-    arg.type = input.dtype();
-    arg.shape = input.shape();
-    ++input_num;
-  }
-
-  // Handles resource variables.
-  TF_RET_CHECK(input_num + num_variable_args == ctx->num_inputs());
-  for (auto& iterator : variable_args) {
-    const Tensor& input = ctx->input(input_num);
-    TF_RET_CHECK(input.dtype() == DT_RESOURCE);
-
-    XlaCompiler::Argument& arg = (*args)[input_num];
-
-    arg.name = iterator.second.name;
-    arg.kind = XlaCompiler::Argument::kResource;
-    arg.resource_kind = XlaResource::kVariable;
-    if (iterator.second.present) {
-      const Tensor& value = iterator.second.value;
-      arg.type = value.dtype();
-      arg.shape = value.shape();
-      arg.initialized = true;
+    } else if (variable_args.count(input_num) == 0) {
+      // Handles the non-constant arguments.
+      const Tensor& input = ctx->input(input_num);
+      TF_RET_CHECK(input.dtype() != DT_RESOURCE);
+      if (input.NumElements() > 0) {
+        arg.kind = XlaCompiler::Argument::kParameter;
+      } else {
+        arg.kind = XlaCompiler::Argument::kConstant;
+        arg.constant_value = input;
+      }
+      arg.type = input.dtype();
+      arg.shape = input.shape();
     } else {
-      // The values of uninitialized variables are not passed as inputs, since
-      // they are meaningless. However, it is legal to assign to a resource
-      // variable for the first time inside the XLA computation, so we do permit
-      // uninitialized variables.
-      arg.initialized = false;
-      arg.type = DT_INVALID;
-      arg.shape = TensorShape();
+      // Handles resource variables.
+      const Tensor& input = ctx->input(input_num);
+      TF_RET_CHECK(input.dtype() == DT_RESOURCE);
+      const OptionalTensor& variable = variable_args.at(input_num);
+      arg.name = variable.name;
+      arg.kind = XlaCompiler::Argument::kResource;
+      arg.resource_kind = XlaResource::kVariable;
+      if (variable.present) {
+        const Tensor& value = variable.value;
+        arg.type = value.dtype();
+        arg.shape = value.shape();
+        arg.initialized = true;
+      } else {
+        // The values of uninitialized variables are not passed as inputs, since
+        // they are meaningless. However, it is legal to assign to a resource
+        // variable for the first time inside the XLA computation, so we do
+        // permit uninitialized variables.
+        arg.initialized = false;
+        arg.type = DT_INVALID;
+        arg.shape = TensorShape();
+      }
     }
-    ++input_num;
   }
 
   return Status::OK();
@@ -234,16 +209,43 @@ Status XlaCompilationCache::BuildExecutable(
 
 Status XlaCompilationCache::Compile(
     const XlaCompiler::Options& options, const NameAttrList& function,
-    int num_constant_args, const std::map<int, OptionalTensor>& variable_args,
-    OpKernelContext* ctx,
+    const std::map<int, Tensor>& constant_args,
+    const std::map<int, OptionalTensor>& variable_args, OpKernelContext* ctx,
     const XlaCompiler::CompilationResult** compilation_result,
     xla::LocalExecutable** executable,
     const XlaCompiler::CompileOptions* compile_options) {
+  return CompileImpl(options, function, constant_args, variable_args, ctx,
+                     compilation_result, executable, compile_options, false);
+}
+
+Status XlaCompilationCache::CompileSingleOp(
+    const XlaCompiler::Options& options,
+    const std::map<int, Tensor>& constant_args,
+    const std::map<int, OptionalTensor>& variable_args, OpKernelContext* ctx,
+    const XlaCompiler::CompilationResult** compilation_result,
+    xla::LocalExecutable** executable,
+    const XlaCompiler::CompileOptions* compile_options) {
+  const NodeDef& def = ctx->op_kernel().def();
+  NameAttrList name;
+  name.set_name(def.op());
+  *name.mutable_attr() = def.attr();
+  return CompileImpl(options, name, constant_args, variable_args, ctx,
+                     compilation_result, executable, compile_options, true);
+}
+
+Status XlaCompilationCache::CompileImpl(
+    const XlaCompiler::Options& options, const NameAttrList& function,
+    const std::map<int, Tensor>& constant_args,
+    const std::map<int, OptionalTensor>& variable_args, OpKernelContext* ctx,
+    const XlaCompiler::CompilationResult** compilation_result,
+    xla::LocalExecutable** executable,
+    const XlaCompiler::CompileOptions* compile_options,
+    bool compile_single_op) {
   VLOG(1) << "XlaCompilationCache::Compile " << DebugString();
 
   if (VLOG_IS_ON(2)) {
     VLOG(2) << "num_inputs=" << ctx->num_inputs()
-            << " num_constant_args=" << num_constant_args
+            << " num_constant_args=" << constant_args.size()
             << " num_variable_args=" << variable_args.size();
     for (int i = 0; i < ctx->num_inputs(); i++) {
       TensorShape shape = ctx->input(i).shape();
@@ -264,11 +266,12 @@ Status XlaCompilationCache::Compile(
     }
   }
 
-  TF_RET_CHECK(num_constant_args + variable_args.size() <= ctx->num_inputs());
+  TF_RET_CHECK(constant_args.size() + variable_args.size() <=
+               ctx->num_inputs());
 
   Signature signature;
-  TF_RETURN_IF_ERROR(BuildSignature(function, num_constant_args, variable_args,
-                                    ctx, &signature));
+  TF_RETURN_IF_ERROR(
+      BuildSignature(function, constant_args, variable_args, ctx, &signature));
 
   VLOG(2) << "Signature: " << SignatureDebugString(signature);
   // The outer lock protects the existence of the cache entry. It does not
@@ -295,13 +298,20 @@ Status XlaCompilationCache::Compile(
     // a long time.)
     std::vector<XlaCompiler::Argument> args;
     TF_RETURN_IF_ERROR(
-        BuildArguments(num_constant_args, variable_args, ctx, &args));
+        BuildArguments(constant_args, variable_args, ctx, &args));
 
     XlaCompiler compiler(options);
     entry->compiled = true;
-    entry->compilation_status = compiler.CompileFunction(
-        compile_options ? *compile_options : XlaCompiler::CompileOptions(),
-        function, args, &entry->compilation_result);
+
+    if (compile_single_op) {
+      entry->compilation_status = compiler.CompileSingleOp(
+          compile_options ? *compile_options : XlaCompiler::CompileOptions(),
+          signature.name, ctx, args, &entry->compilation_result);
+    } else {
+      entry->compilation_status = compiler.CompileFunction(
+          compile_options ? *compile_options : XlaCompiler::CompileOptions(),
+          function, args, &entry->compilation_result);
+    }
   }
   *compilation_result = &entry->compilation_result;
   if (entry->compilation_status.ok() && executable) {
diff --git a/tensorflow/compiler/jit/xla_compilation_cache.h b/tensorflow/compiler/jit/xla_compilation_cache.h
index d506378314..5c0c79b880 100644
--- a/tensorflow/compiler/jit/xla_compilation_cache.h
+++ b/tensorflow/compiler/jit/xla_compilation_cache.h
@@ -52,8 +52,8 @@ class XlaCompilationCache : public ResourceBase {
   // Compiles a function into a XlaCompiler::CompilationResult that can be used
   // to execute an XLA Computation. Compilation results are cached.
   // `function` is the name of a Tensorflow function to compile.
-  // `num_constant_args` is the number of compile-time constant arguments to
-  // `function`. `variable_args` is a snapshot of the current values of the
+  // `constant_args` is a maps of tensorflow argument number to constant value.
+  // `variable_args` is a snapshot of the current values of the
   // resource variable arguments to `function`; uninitialized variables are
   // represented by an absent OptionalTensor.
   // The result of compilation is written to `*compilation_result`, which must
@@ -62,19 +62,40 @@ class XlaCompilationCache : public ResourceBase {
   // executable pointer may be null if the computation has no non-constant
   // outputs.
   Status Compile(const XlaCompiler::Options& options,
-                 const NameAttrList& function, int num_constant_args,
+                 const NameAttrList& function,
+                 const std::map<int, Tensor>& constant_args,
                  const std::map<int, OptionalTensor>& variable_args,
                  OpKernelContext* ctx,
                  const XlaCompiler::CompilationResult** compilation_result,
                  xla::LocalExecutable** executable,
                  const XlaCompiler::CompileOptions* compile_options);
 
+  // As above, but calls XlaCompiler::CompileSingleOp instead of
+  // XlaCompiler::CompileFunction.
+  Status CompileSingleOp(
+      const XlaCompiler::Options& options,
+      const std::map<int, Tensor>& constant_args,
+      const std::map<int, OptionalTensor>& variable_args, OpKernelContext* ctx,
+      const XlaCompiler::CompilationResult** compilation_result,
+      xla::LocalExecutable** executable,
+      const XlaCompiler::CompileOptions* compile_options);
+
   xla::LocalClient* client() const { return client_; }
   const DeviceType& device_type() const { return device_type_; }
 
   string DebugString() override;
 
  private:
+  // Common implementation of Compile and CompileSingleOp.
+  Status CompileImpl(const XlaCompiler::Options& options,
+                     const NameAttrList& function,
+                     const std::map<int, Tensor>& constant_args,
+                     const std::map<int, OptionalTensor>& variable_args,
+                     OpKernelContext* ctx,
+                     const XlaCompiler::CompilationResult** compilation_result,
+                     xla::LocalExecutable** executable,
+                     const XlaCompiler::CompileOptions* compile_options,
+                     bool compile_single_op);
   // Takes `result` which has been compiled from a Tensorflow subgraph to a
   // XLA computation already, and generates an XLA LocalExecutable `executable`.
   Status BuildExecutable(const XlaCompiler::Options& options,
@@ -104,7 +125,8 @@ class XlaCompilationCache : public ResourceBase {
   static string SignatureDebugString(const Signature& sig);
 
   // Builds the signature for a compilation.
-  Status BuildSignature(const NameAttrList& function, int num_constant_args,
+  Status BuildSignature(const NameAttrList& function,
+                        const std::map<int, Tensor>& constant_args,
                         const std::map<int, OptionalTensor>& variable_args,
                         OpKernelContext* ctx, Signature* signature);
 
diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
new file mode 100644
index 0000000000..915b9ce84a
--- /dev/null
+++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
@@ -0,0 +1,178 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// Defines the XlaCompileOnDemandOp.
+
+#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
+#include "tensorflow/compiler/jit/xla_device.h"
+#include "tensorflow/compiler/jit/xla_launch_util.h"
+#include "tensorflow/compiler/tf2xla/xla_compiler.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
+
+namespace tensorflow {
+
+namespace {
+std::map<int, OptionalTensor> GetVariables(OpKernelContext* ctx) {
+  std::map<int, OptionalTensor> variables;
+  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
+    if (ctx->input(i).dtype() == DT_RESOURCE) {
+      Var* variable = nullptr;
+      ResourceHandle handle = HandleFromInput(ctx, i);
+      OptionalTensor& optional = variables[i];
+      optional.name = handle.name();
+      if (LookupResource(ctx, handle, &variable).ok()) {
+        tf_shared_lock lock(*variable->mu());
+        optional.present = true;
+        optional.value = *variable->tensor();
+      }
+    }
+  }
+  return variables;
+}
+}  // namespace
+
+Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx,
+                                 const XlaDevice::Metadata& metadata,
+                                 const XlaCompiler::CompilationResult* result,
+                                 xla::LocalExecutable* executable) {
+  std::map<int, OptionalTensor> variables = GetVariables(ctx);
+  int64 num_resource_args = variables.size();
+
+  xla::LocalClient* client = metadata.client();
+  XlaTensorInfoManager* tensor_info_manager = &metadata.tensor_info_manager();
+
+  // Builds an XLA allocator for the device.
+  XlaAllocator xla_allocator(client->platform(), ctx);
+  XlaComputationLaunchContext launch_context(
+      num_resource_args, client, &xla_allocator, tensor_info_manager);
+
+  launch_context.PopulateInputs(ctx, result, variables);
+
+  perftools::gputools::Stream* stream =
+      ctx->op_device_context() ? ctx->op_device_context()->stream() : nullptr;
+  TF_RET_CHECK(stream);
+
+  VLOG(2) << "Executing computation.";
+  xla::ExecutableRunOptions run_options;
+  run_options.set_stream(stream);
+  run_options.set_allocator(&xla_allocator);
+  run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device());
+
+  auto run_result = executable->Run(launch_context.arguments(), run_options);
+  TF_RETURN_IF_ERROR(run_result.status());
+
+  launch_context.PopulateOutputs(ctx, result, run_result.ConsumeValueOrDie());
+  return Status::OK();
+}
+
+bool XlaCompileOnDemandOp::MustArgumentBeConstant(const OpKernel* op_kernel,
+                                                  int64 argument_idx) {
+  // TODO(jmolloy): This could be expensive, so memoize.
+  auto* constant_inputs = tensorflow::XlaOpRegistry::CompileTimeConstantInputs(
+      op_kernel->def().op());
+  CHECK(constant_inputs);
+  std::set<int64> constant_input_indices;
+  for (const auto& name : *constant_inputs) {
+    int start, stop;
+    TF_CHECK_OK(op_kernel->InputRange(name, &start, &stop));
+    for (int i = start; i < stop; ++i) {
+      constant_input_indices.insert(i);
+    }
+  }
+  return constant_input_indices.count(argument_idx) > 0;
+}
+
+bool XlaCompileOnDemandOp::ShouldArgumentBeConstant(const OpKernel* op_kernel,
+                                                    int64 argument_idx) {
+  // Right now we only create kConstant arguments when absolutely required, but
+  // there may be benefit in eagerly constant-folding a larger subset of
+  // arguments in the future.
+  return MustArgumentBeConstant(op_kernel, argument_idx);
+}
+
+Status XlaCompileOnDemandOp::Compile(
+    OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
+    const XlaCompiler::CompilationResult** result,
+    xla::LocalExecutable** executable) {
+  XlaTensorInfoManager* tensor_info_manager = &metadata.tensor_info_manager();
+
+  std::map<int, Tensor> constant_arguments;
+  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
+    const Tensor& device_tensor = ctx->input(i);
+    if (const XlaTensorInfo* tensor_info =
+            tensor_info_manager->GetTensorInfo(device_tensor)) {
+      if (tensor_info->has_host_tensor() &&
+          ShouldArgumentBeConstant(&ctx->op_kernel(), i)) {
+        constant_arguments[i] = tensor_info->host_tensor();
+      }
+    }
+    if (constant_arguments.count(i) == 0 &&
+        MustArgumentBeConstant(&ctx->op_kernel(), i)) {
+      // Slow path; the argument is not available as a host constant so we must
+      // fetch it synchronously.
+      Tensor host_tensor;
+      TF_RETURN_IF_ERROR(ctx->allocate_temp(
+          device_tensor.dtype(), device_tensor.shape(), &host_tensor));
+      Notification n;
+      ctx->op_device_context()->CopyDeviceTensorToCPU(
+          &device_tensor, "ConstantArgument",
+          reinterpret_cast<Device*>(ctx->device()), &host_tensor,
+          [&](Status status) { n.Notify(); });
+      n.WaitForNotification();
+      constant_arguments[i] = host_tensor;
+    }
+  }
+
+  // We store information about the JIT-compiled XLA computation
+  // in the ResourceMgr.
+  ResourceMgr* rm = ctx->resource_manager();
+  CHECK(rm);
+
+  XlaCompilationCache* cache;
+  TF_RETURN_IF_ERROR(rm->LookupOrCreate<XlaCompilationCache>(
+      rm->default_container(), "xla_cache", &cache,
+      [&](XlaCompilationCache** cache) {
+        *cache = new XlaCompilationCache(metadata.client(),
+                                         metadata.jit_device_type());
+        return Status::OK();
+      }));
+  // Hold the reference to the JIT during evaluation. (We could probably
+  // free it sooner because the ResourceMgr will retain a reference, but
+  // this is more obviously correct.)
+  core::ScopedUnref cache_ref(cache);
+
+  XlaCompiler::Options options;
+  DeviceType device_type = metadata.jit_device_type();
+  options.device_type = &device_type;
+  options.client = metadata.client();
+  options.flib_def =
+      new FunctionLibraryDefinition(OpRegistry::Global(), FunctionDefLibrary{});
+
+  std::map<int, OptionalTensor> variable_args = GetVariables(ctx);
+  return cache->CompileSingleOp(options, constant_arguments, variable_args, ctx,
+                                result, executable,
+                                /*compile_options=*/nullptr);
+}
+
+void XlaCompileOnDemandOp::Compute(OpKernelContext* ctx) {
+  const XlaCompiler::CompilationResult* result;
+  xla::LocalExecutable* executable;
+  const XlaDevice::Metadata* metadata;
+  OP_REQUIRES_OK(ctx, XlaDevice::GetMetadata(ctx, &metadata));
+  OP_REQUIRES_OK(ctx, Compile(ctx, *metadata, &result, &executable));
+  OP_REQUIRES_OK(ctx, Run(ctx, *metadata, result, executable));
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.h b/tensorflow/compiler/jit/xla_compile_on_demand_op.h
new file mode 100644
index 0000000000..23c6f3903f
--- /dev/null
+++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.h
@@ -0,0 +1,56 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+// The XlaCompileOnDemandOp is an OpKernel that, when its Compute method is
+// called, will generate an xla::Computation and run it asynchronously.
+
+#ifndef TENSORFLOW_COMPILER_JIT_XLA_COMPILE_ON_DEMAND_OP_H_
+#define TENSORFLOW_COMPILER_JIT_XLA_COMPILE_ON_DEMAND_OP_H_
+
+#include "tensorflow/compiler/jit/xla_device.h"
+#include "tensorflow/compiler/tf2xla/xla_compiler.h"
+#include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+// An OpKernel that compiles an op to an XLA computation and runs it. Unlike
+// _XlaLaunch this doesn't rely on any rewrites of the graphdef - it will run a
+// vanilla TensorFlow op as long as the bridge supports it.
+//
+// Importantly _XlaLaunch assumes all input and output tensors are on the host,
+// whereas XlacompileOnDemandOp works with tensors in device memory.
+class XlaCompileOnDemandOp : public OpKernel {
+ public:
+  explicit XlaCompileOnDemandOp(OpKernelConstruction* ctx) : OpKernel(ctx) {}
+  void Compute(OpKernelContext* ctx) override;
+
+ private:
+  XlaCompiler::Argument CreateCompilerArgument(OpKernelContext* ctx, int64 i);
+  bool ShouldArgumentBeConstant(const OpKernel* op_kernel, int64 argument_idx);
+  bool MustArgumentBeConstant(const OpKernel* op_kernel, int64 argument_idx);
+  Status Compile(OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
+                 const XlaCompiler::CompilationResult** result,
+                 xla::LocalExecutable** executable);
+  Status Run(OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
+             const XlaCompiler::CompilationResult* result,
+             xla::LocalExecutable* executable);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMPILER_JIT_XLA_COMPILE_ON_DEMAND_OP_H_
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index db3bf3ea33..d2dfdeea68 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -17,6 +17,8 @@ limitations under the License.
 // operators using XLA via the XLA "Host" (CPU) backend.
 
 #include "tensorflow/compiler/jit/kernels/xla_launch_op.h"
+#include "tensorflow/compiler/jit/legacy_flags/xla_device_flags.h"
+#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
 #include "tensorflow/compiler/jit/xla_device.h"
 #include "tensorflow/compiler/jit/xla_device_ops.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
@@ -34,6 +36,15 @@ class XlaCpuDeviceFactory : public DeviceFactory {
 Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& options,
                                           const string& name_prefix,
                                           std::vector<Device*>* devices) {
+  legacy_flags::XlaDeviceFlags* flags = legacy_flags::GetXlaDeviceFlags();
+  bool compile_on_demand = flags->tf_xla_compile_on_demand;
+
+  XlaOpRegistry::DeviceRegistration registration;
+  registration.compilation_device_name = DEVICE_CPU_XLA_JIT;
+  registration.requires_compilation = !compile_on_demand;
+  registration.enable_jit_by_default = false;
+  registration.compile_resource_ops = true;
+
   static XlaDeviceOpRegistrations* registrations =
       RegisterXlaDeviceKernels(DEVICE_XLA_CPU, DEVICE_CPU_XLA_JIT);
   (void)registrations;
@@ -41,7 +52,7 @@ Status XlaCpuDeviceFactory::CreateDevices(const SessionOptions& options,
   std::unique_ptr<XlaDevice> device;
   TF_RETURN_IF_ERROR(XlaDevice::Create("Host", DEVICE_XLA_CPU, 0,
                                        DEVICE_CPU_XLA_JIT, options, name_prefix,
-                                       /*register_device_for_compilation=*/true,
+                                       registration,
                                        /*transfer_as_literal=*/false, &device));
   devices->push_back(device.release());
   return Status::OK();
diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index e4e11d4ce2..82048f5d78 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include <unordered_set>
 
 #include "tensorflow/compiler/jit/defs.h"
+#include "tensorflow/compiler/jit/xla_compile_on_demand_op.h"
 #include "tensorflow/compiler/jit/xla_device_context.h"
 #include "tensorflow/compiler/jit/xla_device_ops.h"
 #include "tensorflow/compiler/tf2xla/dump_graph.h"
@@ -108,21 +109,15 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
 /* static */ Status XlaDevice::Create(
     const string& platform_name, const string& device_name, int device_ordinal,
     const string& jit_device_name, const SessionOptions& options,
-    const string& name_prefix, bool register_device_for_compilation,
+    const string& name_prefix,
+    const XlaOpRegistry::DeviceRegistration& registration,
     bool transfer_as_literal, std::unique_ptr<XlaDevice>* device) {
   VLOG(1) << "XlaDevice::Create " << platform_name << " " << device_name << ":"
           << device_ordinal;
 
-  if (register_device_for_compilation) {
-    // These are no-ops if they have already been done previously for
-    // this device_name/compilation_device_name pair.
-    XlaOpRegistry::DeviceRegistration registration;
-    registration.compilation_device_name = jit_device_name;
-    registration.requires_compilation = true;
-    registration.enable_jit_by_default = false;
-    registration.compile_resource_ops = true;
-    XlaOpRegistry::RegisterCompilationDevice(device_name, registration);
-  }
+  // These are no-ops if they have already been done previously for
+  // this device_name/compilation_device_name pair.
+  XlaOpRegistry::RegisterCompilationDevice(device_name, registration);
 
   auto platform = se::MultiPlatformManager::PlatformWithName(platform_name);
   if (!platform.ok()) {
@@ -306,19 +301,23 @@ Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
 
 XlaDeviceOpRegistrations* RegisterXlaDeviceKernels(const char* device,
                                                    const char* jit_device) {
+  // Any op assigned to the device that isn't rewritten by the graph rewriter
+  // gets executed by a n XlaCompileOnDemandOp, which compiles it and executes
+  // it just-in-time.
+  kernel_factory::OpKernelRegistrar::Factory factory =
+      [](OpKernelConstruction* context) -> OpKernel* {
+    return new XlaCompileOnDemandOp(context);
+  };
   XlaOpRegistry::RegisterCompilationKernels();
   XlaDeviceOpRegistrations* registrations = new XlaDeviceOpRegistrations;
-  auto dummy_factory = [](OpKernelConstruction* context) -> OpKernel* {
-    return new XlaDeviceDummyOp(context);
-  };
   for (const KernelDef* jit_def : XlaOpRegistry::DeviceKernels(
            jit_device,
            /*include_compilation_only_kernels=*/false)) {
     KernelDef* def = new KernelDef(*jit_def);
     def->set_device_type(device);
     registrations->op_kernel_registrars.emplace_back(
-        new kernel_factory::OpKernelRegistrar(def, "XlaDeviceDummyOp",
-                                              dummy_factory));
+        new kernel_factory::OpKernelRegistrar(def, "XlaCompileOnDemandOp",
+                                              factory));
   }
   return registrations;
 }
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 0f4476296b..9cd9167e52 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -27,6 +27,7 @@ limitations under the License.
 #define TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_
 
 #include "tensorflow/compiler/jit/xla_tensor_info.h"
+#include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/local_device.h"
@@ -81,7 +82,7 @@ class XlaDevice : public LocalDevice {
   static Status Create(const string& platform_name, const string& device_name,
                        int device_ordinal, const string& jit_device_name,
                        const SessionOptions& options, const string& name_prefix,
-                       bool register_device_for_compilation,
+                       const XlaOpRegistry::DeviceRegistration& registration,
                        bool transfer_as_literal,
                        std::unique_ptr<XlaDevice>* device);
 
@@ -113,7 +114,7 @@ class XlaDevice : public LocalDevice {
   // Which hardware device in the client's platform this XlaDevice controls.
   const int device_ordinal_;
   // The name of the device that is used to compile Ops for this XlaDevice.
-  const DeviceType& jit_device_name_;
+  DeviceType jit_device_name_;
   // Memory allocator associated with this device.
   Allocator* xla_allocator_;                   // Not owned.
   ::perftools::gputools::Platform* platform_;  // Not owned.
@@ -134,7 +135,7 @@ class XlaDevice : public LocalDevice {
   bool transfer_as_literal_;
 };
 
-// Builds dummy OpKernel registrations on 'device' for the JIT operators
+// Builds OpKernel registrations on 'device' for the JIT operators
 // registered on 'jit_device'. Returns ownership of a XlaDeviceOpRegistrations
 // object that encapsulates the kernel registrations.
 struct XlaDeviceOpRegistrations {
diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index b57f82f98e..88f7c15f0b 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -93,6 +93,10 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
       }
     }
 
+    XlaTensorInfo* tensor_info =
+        tensor_info_manager_->GetOrCreateTensorInfo(*device_tensor);
+    tensor_info->set_host_tensor(*cpu_tensor);
+
     done(status);
     return;
   }
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 383ed879ef..5a1db81774 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -34,15 +34,21 @@ class XlaGpuDeviceFactory : public DeviceFactory {
 Status XlaGpuDeviceFactory::CreateDevices(const SessionOptions& options,
                                           const string& name_prefix,
                                           std::vector<Device*>* devices) {
+  XlaOpRegistry::DeviceRegistration registration;
+  registration.compilation_device_name = DEVICE_GPU_XLA_JIT;
+  registration.requires_compilation = true;
+  registration.enable_jit_by_default = false;
+  registration.compile_resource_ops = true;
+
   static XlaDeviceOpRegistrations* registrations =
       RegisterXlaDeviceKernels(DEVICE_XLA_GPU, DEVICE_GPU_XLA_JIT);
   (void)registrations;
 
   std::unique_ptr<XlaDevice> device;
-  Status status = XlaDevice::Create("CUDA", DEVICE_XLA_GPU, 0,
-                                    DEVICE_GPU_XLA_JIT, options, name_prefix,
-                                    /*register_device_for_compilation=*/true,
-                                    /*transfer_as_literal=*/false, &device);
+  Status status =
+      XlaDevice::Create("CUDA", DEVICE_XLA_GPU, 0, DEVICE_GPU_XLA_JIT, options,
+                        name_prefix, registration,
+                        /*transfer_as_literal=*/false, &device);
   if (!status.ok()) {
     // Treat failures as non-fatal; there might not be a GPU in the machine.
     VLOG(1) << "Failed to create XLA_GPU device: " << status;
diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 689fa3299c..076cbd2084 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -176,21 +176,33 @@ void XlaComputationLaunchContext::PopulateOutputs(
     if (kernel->outputs[i].is_constant) {
       // Output is a constant.
       const Tensor& const_tensor = kernel->outputs[i].constant_value;
+      Tensor* output_tensor;
       const size_t total_bytes = const_tensor.TotalBytes();
       if (stream && total_bytes > 0) {
         // Copy host -> device. (Empty tensors don't have backing buffers.)
         VLOG(1) << "Constant output tensor on device";
-        Tensor* output_tensor;
+
         TF_CHECK_OK(
             ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
 
         const void* src_ptr = DMAHelper::base(&const_tensor);
         void* dst_ptr = DMAHelper::base(output_tensor);
         gpu::DeviceMemoryBase gpu_dst_ptr(dst_ptr, total_bytes);
+        // Memcpying asynchronously is safe for the GPU, but the CPU uses a
+        // shared allocator so hold a reference to the copied-to buffer until
+        // complete.
+        TensorReference ref(*output_tensor);
         stream->ThenMemcpy(&gpu_dst_ptr, src_ptr, total_bytes);
+        stream->ThenDoHostCallback([ref] { ref.Unref(); });
       } else {
         // No copy required.
         ctx->set_output(i, const_tensor);
+        output_tensor = ctx->mutable_output(i);
+      }
+      if (tensor_info_manager_) {
+        XlaTensorInfo* tensor_info =
+            tensor_info_manager_->GetOrCreateTensorInfo(*output_tensor);
+        tensor_info->set_host_tensor(const_tensor);
       }
     } else {
       const TensorShape& shape = kernel->outputs[i].shape;
diff --git a/tensorflow/compiler/jit/xla_tensor_info.h b/tensorflow/compiler/jit/xla_tensor_info.h
index 0b0736bf01..fbd6ad770f 100644
--- a/tensorflow/compiler/jit/xla_tensor_info.h
+++ b/tensorflow/compiler/jit/xla_tensor_info.h
@@ -43,9 +43,25 @@ class XlaTensorInfo {
     shaped_buffer_.reset(new xla::ShapedBuffer(std::move(shaped_buffer)));
   }
 
+  // Some tensors on the device may have known values on the host. We use these
+  // in on-demand mode to avoid re-copying values from the device if we know the
+  // host value already.
+
+  // Return true if this TensorInfo contains a host tensor.
+  bool has_host_tensor() const { return host_tensor_ != nullptr; }
+  // Return the contained host tensor.
+  // REQUIRES: has_host_tensor()
+  const Tensor& host_tensor() const { return *host_tensor_; }
+  // Sets the contained host tensor.
+  void set_host_tensor(const Tensor& tensor) {
+    host_tensor_.reset(new Tensor(tensor));
+  }
+
  private:
   // The optional contained ShapedBuffer.
   std::unique_ptr<xla::ShapedBuffer> shaped_buffer_;
+  // An optional host tensor value.
+  std::unique_ptr<Tensor> host_tensor_;
 };
 
 // Manages XlaTensorInfo objects. This class is also an Allocator, so that
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 85a2adab28..bbb6089ea8 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -86,7 +86,10 @@ tf_xla_py_test(
     # ArgMax needs CustomCall on CPU, which is not available in normal
     # (not precompiled) TensorFlow. The flag below excludes the CPU
     # backend.
-    disabled_backends = "cpu",
+    disabled_backends = [
+        "cpu",
+        "cpu_ondemand",
+    ],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
@@ -315,6 +318,8 @@ tf_xla_py_test(
     name = "function_test",
     size = "small",
     srcs = ["function_test.py"],
+    # Functions are not implemented in the on-demand compilation model yet.
+    disabled_backends = "cpu_ondemand",
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
@@ -551,6 +556,8 @@ tf_xla_py_test(
     name = "stack_ops_test",
     size = "small",
     srcs = ["stack_ops_test.py"],
+    # Stack ops are not implemented in the on-demand compilation model yet.
+    disabled_backends = "cpu_ondemand",
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
@@ -577,6 +584,8 @@ tf_xla_py_test(
     name = "tensor_array_ops_test",
     size = "small",
     srcs = ["tensor_array_ops_test.py"],
+    # TensorArray ops are not implemented in the on-demand compilation model yet.
+    disabled_backends = "cpu_ondemand",
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/compiler/tests/xla_test.py b/tensorflow/compiler/tests/xla_test.py
index cc778f1c3c..e924fe1e61 100644
--- a/tensorflow/compiler/tests/xla_test.py
+++ b/tensorflow/compiler/tests/xla_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import contextlib
+import os
 import random
 import re
 
@@ -44,6 +45,8 @@ flags.DEFINE_string('test_device', None,
 flags.DEFINE_string('types', None, 'Types to test. Comma-separated list.')
 flags.DEFINE_string('disabled_manifest', None,
                     'Path to a file with a list of tests that should not run.')
+flags.DEFINE_string('tf_xla_flags', None,
+                    'Value to set the TF_XLA_FLAGS environment variable to')
 
 
 class XLATestCase(test.TestCase):
@@ -97,6 +100,8 @@ class XLATestCase(test.TestCase):
       disabled_tests = []
       disabled_method_types = []
       for l in manifest_file.read().splitlines():
+        if not l:
+          continue
         entry = comments_re.sub('', l).strip().split(' ')
         if len(entry) == 1:
           disabled_tests.append(entry[0])
@@ -113,6 +118,9 @@ class XLATestCase(test.TestCase):
             for name in types])
       manifest_file.close()
 
+    if FLAGS.tf_xla_flags is not None:
+      os.environ['TF_XLA_FLAGS'] = FLAGS.tf_xla_flags
+
   @property
   def all_tf_types(self):
     name = '{}.{}'.format(type(self).__name__, self._testMethodName)
diff --git a/tensorflow/compiler/tf2xla/kernels/batchtospace_op.cc b/tensorflow/compiler/tf2xla/kernels/batchtospace_op.cc
index cbade79e85..569950c2df 100644
--- a/tensorflow/compiler/tf2xla/kernels/batchtospace_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batchtospace_op.cc
@@ -184,9 +184,7 @@ class BatchToSpaceOp : public XlaOpKernel {
  private:
   int block_size_;
 };
-REGISTER_XLA_OP(Name("BatchToSpace")
-                    .CompileTimeConstInput("crops")
-                    .CompileTimeConstInput("block_shape"),
+REGISTER_XLA_OP(Name("BatchToSpace").CompileTimeConstInput("crops"),
                 BatchToSpaceOp);
 
 }  // namespace
diff --git a/tensorflow/compiler/tf2xla/kernels/segment_reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/segment_reduction_ops.cc
index 80d6df6c48..498342a988 100644
--- a/tensorflow/compiler/tf2xla/kernels/segment_reduction_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/segment_reduction_ops.cc
@@ -83,7 +83,9 @@ class UnsortedSegmentSum : public XlaOpKernel {
   DataType dtype_;
 };
 
-REGISTER_XLA_OP(Name("UnsortedSegmentSum"), UnsortedSegmentSum);
+REGISTER_XLA_OP(
+    Name("UnsortedSegmentSum").CompileTimeConstInput("num_segments"),
+    UnsortedSegmentSum);
 
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
index b10880de77..5bb773d97f 100644
--- a/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/stateless_random_ops.cc
@@ -239,6 +239,7 @@ class StatelessRandomUniformOp : public XlaOpKernel {
 
 // TODO(phawkins): generalize to non-float, non-int32 seed types.
 REGISTER_XLA_OP(Name("StatelessRandomUniform")
+                    .CompileTimeConstInput("shape")
                     .TypeConstraint("dtype", DT_FLOAT)
                     .TypeConstraint("Tseed", DT_INT32),
                 StatelessRandomUniformOp);
@@ -272,6 +273,7 @@ class StatelessRandomNormalOp : public XlaOpKernel {
 
 // TODO(phawkins): generalize to non-float, non-int32 seed types.
 REGISTER_XLA_OP(Name("StatelessRandomNormal")
+                    .CompileTimeConstInput("shape")
                     .TypeConstraint("dtype", DT_FLOAT)
                     .TypeConstraint("Tseed", DT_INT32),
                 StatelessRandomNormalOp);
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.cc b/tensorflow/compiler/tf2xla/xla_compiler.cc
index 7cdf4d1b3e..86263d847a 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler.cc
@@ -600,6 +600,48 @@ Status XlaCompiler::BuildArguments(
   return Status::OK();
 }
 
+Status XlaCompiler::CompileSingleOp(
+    const XlaCompiler::CompileOptions& options, string const& name,
+    OpKernelContext* ctx, const std::vector<XlaCompiler::Argument>& args,
+    CompilationResult* result) {
+  // TODO(b/74182462): We implement this by creating a new dummy Graph including
+  // _Arg nodes, and let CompileGraph walk it. This could be optimized.
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+
+  Status status;
+  // First create the actual node we care about computing.
+  Node* main_node = graph->AddNode(ctx->op_kernel().def(), &status);
+  TF_RETURN_IF_ERROR(status);
+
+  // Create dummy _Arg nodes. Link these to `node` and also via a control
+  // dependency edge to the _SOURCE node.
+  for (int64 i = 0; i < ctx->num_inputs(); ++i) {
+    Node* node;
+    string name = strings::StrCat(ctx->op_kernel().name(), "_", i, "_arg");
+    Status status = NodeBuilder(name, "_Arg")
+                        .ControlInput(graph->source_node())
+                        .Attr("T", ctx->input_dtype(i))
+                        .Attr("index", i)
+                        .Finalize(graph.get(), &node);
+    TF_RETURN_IF_ERROR(status);
+    graph->AddEdge(node, 0, main_node, i);
+  }
+
+  // Similarly with return values, create dummy _Retval nodes fed by `node`.
+  for (int64 i = 0; i < ctx->num_outputs(); ++i) {
+    Node* node;
+    string name = strings::StrCat(ctx->op_kernel().name(), "_", i, "_retval");
+    Status status = NodeBuilder(name, "_Retval")
+                        .Input(main_node, i)
+                        .Attr("T", ctx->expected_output_dtype(i))
+                        .Attr("index", i)
+                        .Finalize(graph.get(), &node);
+    TF_RETURN_IF_ERROR(status);
+  }
+
+  return CompileGraph(options, name, std::move(graph), args, result);
+}
+
 Status XlaCompiler::CompileGraph(const XlaCompiler::CompileOptions& options,
                                  string const& name,
                                  std::unique_ptr<Graph> graph,
diff --git a/tensorflow/compiler/tf2xla/xla_compiler.h b/tensorflow/compiler/tf2xla/xla_compiler.h
index 5f1c631976..a6747bbe72 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler.h
+++ b/tensorflow/compiler/tf2xla/xla_compiler.h
@@ -289,6 +289,14 @@ class XlaCompiler {
                       const std::vector<Argument>& args,
                       CompilationResult* result);
 
+  // Compiles a single Op, given by an OpKernelContext, into an
+  // xla::Computation. Similar to CompileFunction but takes a single Op as
+  // input.
+  Status CompileSingleOp(const CompileOptions& options, string const& name,
+                         OpKernelContext* ctx,
+                         const std::vector<Argument>& args,
+                         CompilationResult* result);
+
   // Returns the shape of the XLA parameter for an argument 'arg'.
   // See the class comment for more details about the argument passing
   // convention.
-- 
GitLab


From 91cbf1e83d85930b59c071553109506b076cee01 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Sat, 17 Mar 2018 21:12:18 +0200
Subject: [PATCH 149/960] Fix typo in `dataset_ops.py`: `datset` -> `dataset`.

Tested:
This is a noop.
---
 tensorflow/python/data/ops/dataset_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index c1ba67e474..9c62d5700c 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -769,7 +769,7 @@ class Dataset(object):
     return PaddedBatchDataset(self, batch_size, padded_shapes, padding_values)
 
   def map(self, map_func, num_parallel_calls=None):
-    """Maps `map_func` across this datset.
+    """Maps `map_func` across this dataset.
 
     Args:
       map_func: A function mapping a nested structure of tensors (having
-- 
GitLab


From 705afa34fc4540593b6aa6dc6dd22ae02d41abea Mon Sep 17 00:00:00 2001
From: brett koonce <koonce@hello.com>
Date: Sat, 17 Mar 2018 12:22:23 -0700
Subject: [PATCH 150/960] contrib: minor spelling tweaks (#17788)

packages:
  model_pruning
  rnn
  solvers
  tensorrt
---
 tensorflow/contrib/model_pruning/python/layers/layers.py  | 2 +-
 tensorflow/contrib/model_pruning/python/pruning.py        | 2 +-
 tensorflow/contrib/rnn/ops/gru_ops.cc                     | 2 +-
 .../contrib/rnn/python/kernel_tests/lstm_ops_test.py      | 2 +-
 tensorflow/contrib/rnn/python/ops/rnn_cell.py             | 4 ++--
 tensorflow/contrib/solvers/python/ops/least_squares.py    | 2 +-
 tensorflow/contrib/solvers/python/ops/linear_equations.py | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_graph.h       | 2 +-
 tensorflow/contrib/tensorrt/convert/convert_nodes.cc      | 8 ++++----
 tensorflow/contrib/tensorrt/python/trt_convert.py         | 2 +-
 .../contrib/tensorrt/resources/trt_int8_calibrator.cc     | 2 +-
 tensorflow/contrib/tensorrt/test/test_tftrt.py            | 2 +-
 12 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/model_pruning/python/layers/layers.py b/tensorflow/contrib/model_pruning/python/layers/layers.py
index 988748ad75..466daf204a 100644
--- a/tensorflow/contrib/model_pruning/python/layers/layers.py
+++ b/tensorflow/contrib/model_pruning/python/layers/layers.py
@@ -214,7 +214,7 @@ def masked_convolution(inputs,
     elif data_format == 'NCHW':
       df = 'channels_first'
     else:
-      raise ValueError('Unsupported data fromat', data_format)
+      raise ValueError('Unsupported data format', data_format)
 
     layer = layer_class(
         filters=num_outputs,
diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py
index 86963be4b8..5146a4a2de 100644
--- a/tensorflow/contrib/model_pruning/python/pruning.py
+++ b/tensorflow/contrib/model_pruning/python/pruning.py
@@ -216,7 +216,7 @@ def _partitioned_variable_assign(partitioned_var, new_value):
   """Assign op for partitioned variables.
 
   Args:
-    partitioned_var: A partitioned tensotflow variable
+    partitioned_var: A partitioned tensorflow variable
     new_value: Value to be assigned to the variable var
 
   Returns:
diff --git a/tensorflow/contrib/rnn/ops/gru_ops.cc b/tensorflow/contrib/rnn/ops/gru_ops.cc
index e91d1e8a80..9c8e40851a 100644
--- a/tensorflow/contrib/rnn/ops/gru_ops.cc
+++ b/tensorflow/contrib/rnn/ops/gru_ops.cc
@@ -69,7 +69,7 @@ Element-wise dot product of a and b is represented by ab
 Element-wise dot product is represented by \circ
 Matrix multiplication is represented by *
 
-Baises are initialized with :
+Biases are initialized with :
 `b_ru` - constant_initializer(1.0)
 `b_c` - constant_initializer(0.0)
 
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 7957edf68c..ffd2421894 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -54,7 +54,7 @@ def blocks_match(sess, use_peephole):
   initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=19890212)
 
   with variable_scope.variable_scope("test", initializer=initializer):
-    # magic naming so that the cells pick up these variables and resuse them
+    # magic naming so that the cells pick up these variables and reuse them
     if use_peephole:
       wci = variable_scope.get_variable(
           "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 358b2eb02b..2f6ae9f367 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -534,7 +534,7 @@ class GridLSTMCell(rnn_cell_impl.RNNCell):
       initializer: (optional) The initializer to use for the weight and
         projection matrices, default None.
       num_unit_shards: (optional) int, default 1, How to split the weight
-        matrix. If > 1,the weight matrix is stored across num_unit_shards.
+        matrix. If > 1, the weight matrix is stored across num_unit_shards.
       forget_bias: (optional) float, default 1.0, The initial bias of the
         forget gates, used to reduce the scale of forgetting at the beginning
         of the training.
@@ -993,7 +993,7 @@ class BidirectionalGridLSTMCell(GridLSTMCell):
       initializer: (optional) The initializer to use for the weight and
         projection matrices, default None.
       num_unit_shards: (optional) int, default 1, How to split the weight
-        matrix. If > 1,the weight matrix is stored across num_unit_shards.
+        matrix. If > 1, the weight matrix is stored across num_unit_shards.
       forget_bias: (optional) float, default 1.0, The initial bias of the
         forget gates, used to reduce the scale of forgetting at the beginning
         of the training.
diff --git a/tensorflow/contrib/solvers/python/ops/least_squares.py b/tensorflow/contrib/solvers/python/ops/least_squares.py
index fb7c0eb649..6e164f5342 100644
--- a/tensorflow/contrib/solvers/python/ops/least_squares.py
+++ b/tensorflow/contrib/solvers/python/ops/least_squares.py
@@ -33,7 +33,7 @@ def cgls(operator, rhs, tol=1e-6, max_iter=20, name="cgls"):
   r"""Conjugate gradient least squares solver.
 
   Solves a linear least squares problem \\(||A x - rhs||_2\\) for a single
-  righ-hand side, using an iterative, matrix-free algorithm where the action of
+  right-hand side, using an iterative, matrix-free algorithm where the action of
   the matrix A is represented by `operator`. The CGLS algorithm implicitly
   applies the symmetric conjugate gradient algorithm to the normal equations
   \\(A^* A x = A^* rhs\\). The iteration terminates when either
diff --git a/tensorflow/contrib/solvers/python/ops/linear_equations.py b/tensorflow/contrib/solvers/python/ops/linear_equations.py
index d791d46763..9305c6a11c 100644
--- a/tensorflow/contrib/solvers/python/ops/linear_equations.py
+++ b/tensorflow/contrib/solvers/python/ops/linear_equations.py
@@ -41,7 +41,7 @@ def conjugate_gradient(operator,
   r"""Conjugate gradient solver.
 
   Solves a linear system of equations `A*x = rhs` for selfadjoint, positive
-  definite matrix `A` and righ-hand side vector `rhs`, using an iterative,
+  definite matrix `A` and right-hand side vector `rhs`, using an iterative,
   matrix-free algorithm where the action of the matrix A is represented by
   `operator`. The iteration terminates when either the number of iterations
   exceeds `max_iter` or when the residual norm has been reduced to `tol`
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index e1596e89e2..e01e4a5328 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -35,7 +35,7 @@ tensorflow::Status ConvertCalibGraphToInferGraph(
 
 // max_batch_size: maximum batch size which can be used for inference for
 //                 optimization targets inference run with max batch size.
-// max_workspace_size_bytes: The upper bound of memory allowence for
+// max_workspace_size_bytes: The upper bound of memory allowance for
 //                 engine building.
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 75a3c3d034..92a692baa7 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -455,7 +455,7 @@ class Converter {
       if (trt_tensors_.count(name)) {
         inputs.push_back(trt_tensors_.at(name));
       } else {
-        LOG(FATAL) << "input: " << name << " not availabled for node at, "
+        LOG(FATAL) << "input: " << name << " not available for node at, "
                    << node_def.name();
       }
     }
@@ -884,7 +884,7 @@ tensorflow::Status BinaryTensorOpWeight(
   // default to element-wise
   auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
 
-  // TODO(jie): maybe use a permuatation instead to support more cases;
+  // TODO(jie): maybe use a permutation instead to support more cases;
   bool permutation_flag = false;
 
   if (weights.count() == 1) {
@@ -1498,7 +1498,7 @@ tensorflow::Status ConvertConst(Converter& ctx,
           weights_tensor.int_val().begin(),
           weights_tensor.int_val()
               .end());  //  make a local copy first to flatten
-                        //  doesn't have to be contigous
+                        //  doesn't have to be contiguous
       memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
       weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
     }
@@ -2212,7 +2212,7 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
   std::list<tensorflow::Node*> order;
   for (tensorflow::Node* node : order_vec) {
     if (s.subgraph_node_ids.count(node->id())) {
-      order.push_front(node);  // we want topological order to contstruct the
+      order.push_front(node);  // we want topological order to construct the
       // network layer by layer
     }
   }
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 666220d78c..338475d90e 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -41,7 +41,7 @@ def create_inference_graph(input_graph_def,
                            max_workspace_size_bytes=2 << 20,
                            precision_mode="FP32",
                            minimum_segment_size=3):
-  """Python wrapper for the TRT transormation.
+  """Python wrapper for the TRT transformation.
 
   Args:
     input_graph_def: GraphDef object containing a model to be transformed.
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
index 74df75902e..dc7c93f869 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
@@ -61,7 +61,7 @@ bool TRTInt8Calibrator::setBatch(const std::unordered_map<string, void*>& data,
 
     // TODO(aaroey): we should not use sync copy on default stream. Make sure
     // stream->ThenMemcpy() is used in future PRs.
-    // TODO(sami,aaroey): Need to figureout a way to ensure synchronization
+    // TODO(sami,aaroey): Need to figure out a way to ensure synchronization
     // between stream, perhaps using a tensor?
     auto status = cudaMemcpyAsync(d.first, it.second, d.second,
                                   cudaMemcpyDeviceToDevice, stream);
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index 0b661bd536..ad01bedd8f 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -75,7 +75,7 @@ def run_graph(gdef, dumm_inp):
   return val
 
 
-# Use real data that is representatitive of the inference dataset
+# Use real data that is representative of the inference dataset
 # for calibration. For this test script it is random data.
 def run_calibration(gdef, dumm_inp):
   """Run given calibration graph multiple times."""
-- 
GitLab


From 10cddb4268b174f879956a2d1124b8ae1044c425 Mon Sep 17 00:00:00 2001
From: "Xiaoming (Jason) Cui" <xiaoming.cui@intel.com>
Date: Sat, 17 Mar 2018 13:17:13 -0700
Subject: [PATCH 151/960] Fixed issue #92, timeline_test unit test fails,
 changed the test so that it can take cpu name changed with MKLDNN naming
 conversion (#17775)

---
 tensorflow/python/client/timeline_test.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index 9641b8b7f2..5e6b5acdb0 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -155,9 +155,12 @@ class TimelineTest(test.TestCase):
     ctf = step_analysis.chrome_trace.format_to_string()
     self._validateTrace(ctf)
     maximums = step_analysis.allocator_maximums
-    self.assertTrue('cpu' in maximums)
+    cpuname = 'cpu'
+    if 'mklcpu' in maximums:
+      cpuname = 'mkl' + cpuname
+    self.assertTrue(cpuname in maximums)
     cpu_max = maximums[
-        'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums['cpu']
+        'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums[cpuname]
     # At least num1 + num2, both float32s (4 bytes each)
     self.assertGreater(cpu_max.num_bytes, 8)
     self.assertGreater(cpu_max.timestamp, 0)
-- 
GitLab


From 2fa81dc522f984666bf5ba8f2392cbacb464a852 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Sun, 18 Mar 2018 17:23:05 +0800
Subject: [PATCH 152/960] Fix broken link of internal anchor in rnn quickdraw

---
 tensorflow/docs_src/tutorials/recurrent_quickdraw.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index 7584a76ba5..fd1a56c1b1 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -38,8 +38,8 @@ To try the code for this tutorial:
 1.  [Download the data](#download-the-data) in `TFRecord` format from
     [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) and unzip it. More details about [how to
     obtain the original Quick, Draw!
-    data](#optional-download-the-full-quick-draw-data) and [how to convert that
-    to `TFRecord` files](#optional-converting-the-data) is available below.
+    data](#optional_download_the_full_quick_draw_data) and [how to convert that
+    to `TFRecord` files](#optional_converting_the_data) is available below.
 
 1.  Execute the tutorial code with the following command to train the RNN-based
     model described in this tutorial. Make sure to adjust the paths to point to
-- 
GitLab


From a52a22912fd6a5a5d8434d08753a11ab8de4bdd3 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Sun, 18 Mar 2018 14:39:42 +0200
Subject: [PATCH 153/960] Simplify `rejection_resample` test to remove
 unnecessary iterator initialization.

Tested:
- bazel test :resample_test
---
 tensorflow/contrib/data/python/kernel_tests/resample_test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 0ac8d7359f..c16207fa48 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -45,12 +45,10 @@ class ResampleTest(test.TestCase):
                 target_dist=target_dist,
                 initial_dist=initial_dist,
                 class_func=lambda c, _: c,
-                seed=27)).make_initializable_iterator())
-    init_op = iterator.initializer
+                seed=27)).make_one_shot_iterator())
     get_next = iterator.get_next()
 
     with self.test_session() as sess:
-      sess.run(init_op)
       returned = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
-- 
GitLab


From 168559c27a2070d8f069e1f11ffd641ca579afc6 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Sun, 18 Mar 2018 22:10:10 +0800
Subject: [PATCH 154/960] Fix two more case-sensitive anchor link

---
 tensorflow/docs_src/tutorials/recurrent_quickdraw.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index fd1a56c1b1..5d83fbe2a3 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -108,7 +108,7 @@ This download will take a while and download a bit more than 23GB of data.
 ### Optional: Converting the data
 
 To convert the `ndjson` files to
-@{$python/python_io#tfrecords_format_details$TFRecord} files containing
+@{$python/python_io#TFRecords_Format_Details$TFRecord} files containing
 [`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
 protos run the following command.
 
@@ -118,7 +118,7 @@ protos run the following command.
 ```
 
 This will store the data in 10 shards of
-@{$python/python_io#tfrecords_format_details$TFRecord} files with 10000 items
+@{$python/python_io#TFRecords_Format_Details$TFRecord} files with 10000 items
 per class for the training data and 1000 items per class as eval data.
 
 This conversion process is described in more detail in the following.
-- 
GitLab


From 485bbb94dc35bf619e59f2be26a54f97b443c451 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Sun, 18 Mar 2018 22:17:45 +0800
Subject: [PATCH 155/960] Fix several broken links in kernel method tutorials

---
 tensorflow/docs_src/tutorials/kernel_methods.md | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md
index b1f06ce0a3..2b35f0a157 100644
--- a/tensorflow/docs_src/tutorials/kernel_methods.md
+++ b/tensorflow/docs_src/tutorials/kernel_methods.md
@@ -1,7 +1,11 @@
 # Improving Linear Models Using Explicit Kernel Methods
 
 Note: This document uses a deprecated version of @{tf.estimator},
+<<<<<<< HEAD
 which has a different interface (see `tf.contrib.learn Estimator`).
+=======
+which has a @{tf.contrib.learn.estimator$different interface}.
+>>>>>>> Fix several broken links in kernel method tutorials
 It also uses other `contrib` methods whose
 @{$version_compat#not_covered$API may not be stable}.
 
@@ -53,7 +57,7 @@ In order to feed data to a `tf.contrib.learn Estimator`, it is helpful to conver
 it to Tensors. For this, we will use an `input function` which adds Ops to the
 TensorFlow graph that, when executed, create mini-batches of Tensors to be used
 downstream. For more background on input functions, check
-@{$get_started/premade_estimators#input_fn$this section on input functions}.
+@{$get_started/premade_estimators#create_input_functions$this section on input functions}.
 In this example, we will use the `tf.train.shuffle_batch` Op which, besides
 converting numpy arrays to Tensors, allows us to specify the batch_size and
 whether to randomize the input every time the input_fn Ops are executed
-- 
GitLab


From 1aa0acf2ffa471b3fbd24481113d2ba8adb14b95 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Sun, 18 Mar 2018 22:25:05 +0800
Subject: [PATCH 156/960] Forgot to save when rebase master

---
 tensorflow/docs_src/tutorials/kernel_methods.md | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md
index 2b35f0a157..e322ccf7c5 100644
--- a/tensorflow/docs_src/tutorials/kernel_methods.md
+++ b/tensorflow/docs_src/tutorials/kernel_methods.md
@@ -1,11 +1,7 @@
 # Improving Linear Models Using Explicit Kernel Methods
 
 Note: This document uses a deprecated version of @{tf.estimator},
-<<<<<<< HEAD
 which has a different interface (see `tf.contrib.learn Estimator`).
-=======
-which has a @{tf.contrib.learn.estimator$different interface}.
->>>>>>> Fix several broken links in kernel method tutorials
 It also uses other `contrib` methods whose
 @{$version_compat#not_covered$API may not be stable}.
 
-- 
GitLab


From 70bb4240b9ccd1099d378548ffed87d88d160441 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Sun, 18 Mar 2018 22:27:56 +0800
Subject: [PATCH 157/960] Fix different interface link in kernel method

---
 tensorflow/docs_src/tutorials/kernel_methods.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md
index e322ccf7c5..73e5c51057 100644
--- a/tensorflow/docs_src/tutorials/kernel_methods.md
+++ b/tensorflow/docs_src/tutorials/kernel_methods.md
@@ -1,7 +1,7 @@
 # Improving Linear Models Using Explicit Kernel Methods
 
 Note: This document uses a deprecated version of @{tf.estimator},
-which has a different interface (see `tf.contrib.learn Estimator`).
+which has a @{tf.contrib.learn.Estimator$different interface}.
 It also uses other `contrib` methods whose
 @{$version_compat#not_covered$API may not be stable}.
 
-- 
GitLab


From 8851c6fdedfd226f5f9c7da09cecaf6cdea06477 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 18 Mar 2018 07:48:30 -0700
Subject: [PATCH 158/960] Fix build

PiperOrigin-RevId: 189506945
---
 tensorflow/compiler/jit/xla_interpreter_device.cc | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_interpreter_device.cc b/tensorflow/compiler/jit/xla_interpreter_device.cc
index a329451b14..9e098c46f4 100644
--- a/tensorflow/compiler/jit/xla_interpreter_device.cc
+++ b/tensorflow/compiler/jit/xla_interpreter_device.cc
@@ -41,10 +41,17 @@ Status XlaInterpreterDeviceFactory::CreateDevices(
       DEVICE_XLA_INTERPRETER, DEVICE_INTERPRETER_XLA_JIT);
   (void)registrations;
 
+  XlaOpRegistry::DeviceRegistration registration;
+  registration.compilation_device_name = DEVICE_INTERPRETER_XLA_JIT;
+  registration.requires_compilation = true;
+  registration.enable_jit_by_default = false;
+  registration.compile_resource_ops = true;
+
   std::unique_ptr<XlaDevice> device;
-  TF_RETURN_IF_ERROR(XlaDevice::Create(
-      "Interpreter", DEVICE_XLA_INTERPRETER, 0, DEVICE_INTERPRETER_XLA_JIT,
-      options, name_prefix, /*register_device_for_compilation=*/true, &device));
+  TF_RETURN_IF_ERROR(XlaDevice::Create("Interpreter", DEVICE_XLA_INTERPRETER, 0,
+                                       DEVICE_INTERPRETER_XLA_JIT, options,
+                                       name_prefix, registration,
+                                       /*transfer_as_literal=*/false, &device));
   devices->push_back(device.release());
   return Status::OK();
 }
-- 
GitLab


From 2b1b9ea110bcf26f047689564298de43ab83db18 Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Mon, 19 Mar 2018 00:34:25 +0800
Subject: [PATCH 159/960] Fix the broken link ofr build the op library in
 extend tutorials

---
 tensorflow/docs_src/extend/add_filesys.md      | 2 +-
 tensorflow/docs_src/extend/new_data_formats.md | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/extend/add_filesys.md b/tensorflow/docs_src/extend/add_filesys.md
index 06f11de4eb..bc0f662f0c 100644
--- a/tensorflow/docs_src/extend/add_filesys.md
+++ b/tensorflow/docs_src/extend/add_filesys.md
@@ -225,7 +225,7 @@ it will use the `FooBarFileSystem` implementation.
 Next, you must build a shared object containing this implementation. An example
 of doing so using bazel's `cc_binary` rule can be found
 [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD#L244),
-but you may use any build system to do so. See the section on @{$adding_an_op#build-the-op-library$building the op library} for similar
+but you may use any build system to do so. See the section on @{$adding_an_op#build_the_op_library$building the op library} for similar
 instructions.
 
 The result of building this target is a `.so` shared object file.
diff --git a/tensorflow/docs_src/extend/new_data_formats.md b/tensorflow/docs_src/extend/new_data_formats.md
index b3cc968047..10e717c280 100644
--- a/tensorflow/docs_src/extend/new_data_formats.md
+++ b/tensorflow/docs_src/extend/new_data_formats.md
@@ -167,7 +167,7 @@ REGISTER_KERNEL_BUILDER(Name("TextLineReader").Device(DEVICE_CPU),
 ```
 
 The last step is to add the Python wrapper.  You can either do this by
-@{$adding_an_op#building_the_op_library$compiling a dynamic library}
+@{$adding_an_op#build_the_op_library$compiling a dynamic library}
 or, if you are building TensorFlow from source, adding to `user_ops.py`.
 For the latter, you will import `tensorflow.python.ops.io_ops` in
 [`tensorflow/python/user_ops/user_ops.py`](https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py)
-- 
GitLab


From 70cd9ed2d2ea37a6da6f813a99b32c03e90736a4 Mon Sep 17 00:00:00 2001
From: Brent Yi <yibrenth@gmail.com>
Date: Sun, 18 Mar 2018 14:35:24 -0700
Subject: [PATCH 160/960] Fix random_uniform documentation formatting (#17805)

---
 tensorflow/python/ops/random_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index db8159579a..6a2dd3f1cd 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -209,7 +209,7 @@ def random_uniform(shape,
     maxval: A 0-D Tensor or Python value of type `dtype`. The upper bound on
       the range of random values to generate.  Defaults to 1 if `dtype` is
       floating point.
-    dtype: The type of the output: 'float16`, `float32`, `float64`, `int32`,
+    dtype: The type of the output: `float16`, `float32`, `float64`, `int32`,
       or `int64`.
     seed: A Python integer. Used to create a random seed for the distribution.
       See @{tf.set_random_seed}
-- 
GitLab


From 838a8f54f92452a15e3bb62a23ad5cd67e86933f Mon Sep 17 00:00:00 2001
From: Guillaume Klein <guillaumekln@users.noreply.github.com>
Date: Sun, 18 Mar 2018 18:18:47 -0400
Subject: [PATCH 161/960] Support TensorArray in BeamSearchDecoder state.
 (#13312)

* Support TensorArray in BeamSearchDecoder state.

* Use gather_nd for reordering and test more shapes.

* Add a flag to disable TensorArrays reordering.

* Add shape checks before reordering a TensorArray.

* Directly use float32 member of dtypes

* Directly access dimension value if defined

* Add more TensorArrays reordering constraints

* Do not unstack reordered TensorArrays

* Improve warning for ignored TensorArrays

* Consistent static and runtime dimensions check

* Use comparison operators

* Fix dynamic checks and add tests

* Make static checks error a warning

* Fix pylint errors
---
 .../kernel_tests/attention_wrapper_test.py    |   3 +
 .../kernel_tests/beam_search_decoder_test.py  | 104 ++++++++++-
 .../seq2seq/python/ops/attention_wrapper.py   |  19 +-
 .../seq2seq/python/ops/beam_search_decoder.py | 169 +++++++++++++++++-
 4 files changed, 280 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index b427dff88b..c4139dde49 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -222,6 +222,9 @@ class AttentionWrapperTest(test.TestCase):
           self.assertEqual(
               (None, batch_size, None),
               tuple(state_alignment_history.get_shape().as_list()))
+        nest.assert_same_structure(
+            cell.state_size,
+            cell.zero_state(batch_size, dtypes.float32))
         # Remove the history from final_state for purposes of the
         # remainder of the tests.
         final_state = final_state._replace(alignment_history=())  # pylint: disable=protected-access
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
index 9265540317..178328619f 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.seq2seq.python.ops import beam_search_ops
 from tensorflow.contrib.seq2seq.python.ops import decoder
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -70,6 +71,98 @@ class TestGatherTree(test.TestCase):
 
     self.assertAllEqual(expected_result, res_)
 
+  def _test_gather_tree_from_array(self,
+                                   depth_ndims=0,
+                                   merged_batch_beam=False):
+    array = np.array(
+        [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [0, 0, 0]],
+         [[2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 0]]]).transpose([1, 0, 2])
+    parent_ids = np.array(
+        [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]],
+         [[0, 0, 0], [1, 1, 0], [2, 0, 1], [0, 1, 0]]]).transpose([1, 0, 2])
+    expected_array = np.array(
+        [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [0, 0, 0]],
+         [[2, 3, 2], [7, 5, 7], [8, 9, 8], [11, 12, 0]]]).transpose([1, 0, 2])
+    sequence_length = [[3, 3, 3], [4, 4, 3]]
+
+    array = ops.convert_to_tensor(
+        array, dtype=dtypes.float32)
+    parent_ids = ops.convert_to_tensor(
+        parent_ids, dtype=dtypes.int32)
+    expected_array = ops.convert_to_tensor(
+        expected_array, dtype=dtypes.float32)
+
+    max_time = array_ops.shape(array)[0]
+    batch_size = array_ops.shape(array)[1]
+    beam_width = array_ops.shape(array)[2]
+
+    def _tile_in_depth(tensor):
+      # Generate higher rank tensors by concatenating tensor and tensor + 1.
+      for _ in range(depth_ndims):
+        tensor = array_ops.stack([tensor, tensor + 1], -1)
+      return tensor
+
+    if merged_batch_beam:
+      array = array_ops.reshape(
+          array, [max_time, batch_size * beam_width])
+      expected_array = array_ops.reshape(
+          expected_array, [max_time, batch_size * beam_width])
+
+    if depth_ndims > 0:
+      array = _tile_in_depth(array)
+      expected_array = _tile_in_depth(expected_array)
+
+    sorted_array = beam_search_decoder.gather_tree_from_array(
+        array, parent_ids, sequence_length)
+
+    with self.test_session() as sess:
+      sorted_array = sess.run(sorted_array)
+      expected_array = sess.run(expected_array)
+      self.assertAllEqual(expected_array, sorted_array)
+
+  def test_gather_tree_from_array_scalar(self):
+    self._test_gather_tree_from_array()
+
+  def test_gather_tree_from_array_1d(self):
+    self._test_gather_tree_from_array(depth_ndims=1)
+
+  def test_gather_tree_from_array_1d_with_merged_batch_beam(self):
+    self._test_gather_tree_from_array(depth_ndims=1, merged_batch_beam=True)
+
+  def test_gather_tree_from_array_2d(self):
+    self._test_gather_tree_from_array(depth_ndims=2)
+
+
+class TestArrayShapeChecks(test.TestCase):
+
+  def _test_array_shape_dynamic_checks(self, static_shape, dynamic_shape,
+                                       batch_size, beam_width, is_valid=True):
+    t = array_ops.placeholder_with_default(
+        np.random.randn(*static_shape).astype(np.float32),
+        shape=dynamic_shape)
+
+    batch_size = array_ops.constant(batch_size)
+    check_op = beam_search_decoder._check_batch_beam(t, batch_size, beam_width)  # pylint: disable=protected-access
+
+    with self.test_session() as sess:
+      if is_valid:
+        sess.run(check_op)
+      else:
+        with self.assertRaises(errors.InvalidArgumentError):
+          sess.run(check_op)
+
+  def test_array_shape_dynamic_checks(self):
+    self._test_array_shape_dynamic_checks(
+        (8, 4, 5, 10), (None, None, 5, 10), 4, 5, is_valid=True)
+    self._test_array_shape_dynamic_checks(
+        (8, 20, 10), (None, None, 10), 4, 5, is_valid=True)
+    self._test_array_shape_dynamic_checks(
+        (8, 21, 10), (None, None, 10), 4, 5, is_valid=False)
+    self._test_array_shape_dynamic_checks(
+        (8, 4, 6, 10), (None, None, None, 10), 4, 5, is_valid=False)
+    self._test_array_shape_dynamic_checks(
+        (8, 4), (None, None), 4, 5, is_valid=False)
+
 
 class TestEosMasking(test.TestCase):
   """Tests EOS masking used in beam search."""
@@ -319,7 +412,8 @@ class TestLargeBeamStep(test.TestCase):
 
 class BeamSearchDecoderTest(test.TestCase):
 
-  def _testDynamicDecodeRNN(self, time_major, has_attention):
+  def _testDynamicDecodeRNN(self, time_major, has_attention,
+                            with_alignment_history=False):
     encoder_sequence_length = np.array([3, 2, 3, 1, 1])
     decoder_sequence_length = np.array([2, 0, 1, 2, 3])
     batch_size = 5
@@ -359,7 +453,7 @@ class BeamSearchDecoderTest(test.TestCase):
             cell=cell,
             attention_mechanism=attention_mechanism,
             attention_layer_size=attention_depth,
-            alignment_history=False)
+            alignment_history=with_alignment_history)
       cell_state = cell.zero_state(
           dtype=dtypes.float32, batch_size=batch_size_tensor * beam_width)
       if has_attention:
@@ -420,6 +514,12 @@ class BeamSearchDecoderTest(test.TestCase):
   def testDynamicDecodeRNNBatchMajorYesAttention(self):
     self._testDynamicDecodeRNN(time_major=False, has_attention=True)
 
+  def testDynamicDecodeRNNBatchMajorYesAttentionWithAlignmentHistory(self):
+    self._testDynamicDecodeRNN(
+        time_major=False,
+        has_attention=True,
+        with_alignment_history=True)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index f8da5a3e17..9ff8a343f1 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -1278,7 +1278,8 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
         attention_state=self._item_or_tuple(
             a.state_size for a in self._attention_mechanisms),
         alignment_history=self._item_or_tuple(
-            () for _ in self._attention_mechanisms))  # sometimes a TensorArray
+            a.alignments_size if self._alignment_history else ()
+            for a in self._attention_mechanisms))  # sometimes a TensorArray
 
   def zero_state(self, batch_size, dtype):
     """Return an initial (zero) state tuple for this `AttentionWrapper`.
@@ -1318,22 +1319,26 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
         cell_state = nest.map_structure(
             lambda s: array_ops.identity(s, name="checked_cell_state"),
             cell_state)
+      initial_alignments = [
+          attention_mechanism.initial_alignments(batch_size, dtype)
+          for attention_mechanism in self._attention_mechanisms]
       return AttentionWrapperState(
           cell_state=cell_state,
           time=array_ops.zeros([], dtype=dtypes.int32),
           attention=_zero_state_tensors(self._attention_layer_size, batch_size,
                                         dtype),
-          alignments=self._item_or_tuple(
-              attention_mechanism.initial_alignments(batch_size, dtype)
-              for attention_mechanism in self._attention_mechanisms),
+          alignments=self._item_or_tuple(initial_alignments),
           attention_state=self._item_or_tuple(
               attention_mechanism.initial_state(batch_size, dtype)
               for attention_mechanism in self._attention_mechanisms),
           alignment_history=self._item_or_tuple(
-              tensor_array_ops.TensorArray(dtype=dtype, size=0,
-                                           dynamic_size=True)
+              tensor_array_ops.TensorArray(
+                  dtype,
+                  size=0,
+                  dynamic_size=True,
+                  element_shape=alignment.shape)
               if self._alignment_history else ()
-              for _ in self._attention_mechanisms))
+              for alignment in initial_alignments))
 
   def call(self, inputs, state):
     """Perform a step of attention-wrapped RNN.
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 6adbb8be40..a26107b0d7 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -35,6 +35,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.platform import tf_logging
 from tensorflow.python.util import nest
 
 __all__ = [
@@ -121,14 +122,114 @@ def tile_batch(t, multiplier, name=None):
     return nest.map_structure(lambda t_: _tile_batch(t_, multiplier), t)
 
 
+def gather_tree_from_array(t, parent_ids, sequence_length):
+  """Calculates the full beams for `TensorArray`s.
+
+  Args:
+    t: A stacked `TensorArray` of size `max_time` that contains `Tensor`s of
+      shape `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]`
+      where `s` is the depth shape.
+    parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`.
+    sequence_length: The sequence length of shape `[batch_size, beam_width]`.
+
+  Returns:
+    A `Tensor` which is a stacked `TensorArray` of the same size and type as
+    `t` and where beams are sorted in each `Tensor` according to `parent_ids`.
+  """
+  max_time = parent_ids.shape[0].value or array_ops.shape(parent_ids)[0]
+  batch_size = parent_ids.shape[1].value or array_ops.shape(parent_ids)[1]
+  beam_width = parent_ids.shape[2].value or array_ops.shape(parent_ids)[2]
+
+  # Generate beam ids that will be reordered by gather_tree.
+  beam_ids = array_ops.expand_dims(
+      array_ops.expand_dims(math_ops.range(beam_width), 0), 0)
+  beam_ids = array_ops.tile(beam_ids, [max_time, batch_size, 1])
+
+  mask = array_ops.sequence_mask(
+      sequence_length, maxlen=max_time, dtype=dtypes.int32)
+  mask = array_ops.transpose(mask, perm=[2, 0, 1])
+
+  # Use beam_width + 1 to mark the end of beam.
+  masked_beam_ids = (beam_ids * mask) + (1 - mask) * (beam_width + 1)
+
+  max_sequence_lengths = math_ops.to_int32(
+      math_ops.reduce_max(sequence_length, axis=1))
+  sorted_beam_ids = beam_search_ops.gather_tree(
+      step_ids=masked_beam_ids,
+      parent_ids=parent_ids,
+      max_sequence_lengths=max_sequence_lengths,
+      end_token=beam_width + 1)
+
+  # For out of range steps, simply copy the same beam.
+  sorted_beam_ids = array_ops.where(
+      math_ops.cast(mask, dtypes.bool), x=sorted_beam_ids, y=beam_ids)
+
+  # Generate indices for gather_nd.
+  time_ind = array_ops.tile(array_ops.reshape(
+      math_ops.range(max_time), [-1, 1, 1]), [1, batch_size, beam_width])
+  batch_ind = array_ops.tile(array_ops.reshape(
+      math_ops.range(batch_size), [-1, 1, 1]), [1, max_time, beam_width])
+  batch_ind = array_ops.transpose(batch_ind, perm=[1, 0, 2])
+  indices = array_ops.stack([time_ind, batch_ind, sorted_beam_ids], -1)
+
+  # Gather from a tensor with collapsed additional dimensions.
+  gather_from = t
+  final_shape = array_ops.shape(gather_from)
+  gather_from = array_ops.reshape(
+      gather_from, [max_time, batch_size, beam_width, -1])
+  ordered = array_ops.gather_nd(gather_from, indices)
+  ordered = array_ops.reshape(ordered, final_shape)
+
+  return ordered
+
+
 def _check_maybe(t):
-  if isinstance(t, tensor_array_ops.TensorArray):
-    raise TypeError(
-        "TensorArray state is not supported by BeamSearchDecoder: %s" % t.name)
   if t.shape.ndims is None:
     raise ValueError(
         "Expected tensor (%s) to have known rank, but ndims == None." % t)
 
+def _check_static_batch_beam_maybe(shape, batch_size, beam_width):
+  """Raises an exception if dimensions are known statically and can not be
+  reshaped to [batch_size, beam_size, -1].
+  """
+  reshaped_shape = tensor_shape.TensorShape([batch_size, beam_width, None])
+  if (batch_size is not None and shape[0].value is not None
+      and (shape[0] != batch_size * beam_width
+           or (shape.ndims >= 2 and shape[1].value is not None
+               and (shape[0] != batch_size or shape[1] != beam_width)))):
+    tf_logging.warn("TensorArray reordering expects elements to be "
+                    "reshapable to %s which is incompatible with the "
+                    "current shape %s. Consider setting "
+                    "reorder_tensor_arrays to False to disable TensorArray "
+                    "reordering during the beam search."
+                    % (reshaped_shape, shape))
+    return False
+  return True
+
+def _check_batch_beam(t, batch_size, beam_width):
+  """Returns an Assert operation checking that the elements of the stacked
+  TensorArray can be reshaped to [batch_size, beam_size, -1]. At this point,
+  the TensorArray elements have a known rank of at least 1.
+  """
+  error_message = ("TensorArray reordering expects elements to be "
+                   "reshapable to [batch_size, beam_size, -1] which is "
+                   "incompatible with the dynamic shape of %s elements. "
+                   "Consider setting reorder_tensor_arrays to False to disable "
+                   "TensorArray reordering during the beam search."
+                   % (t.name))
+  rank = t.shape.ndims
+  shape = array_ops.shape(t)
+  if rank == 2:
+    condition = math_ops.equal(shape[1], batch_size * beam_width)
+  else:
+    condition = math_ops.logical_or(
+        math_ops.equal(shape[1], batch_size * beam_width),
+        math_ops.logical_and(
+            math_ops.equal(shape[1], batch_size),
+            math_ops.equal(shape[2], beam_width)))
+  return control_flow_ops.Assert(condition, [error_message])
+
+
 
 class BeamSearchDecoder(decoder.Decoder):
   """BeamSearch sampling decoder.
@@ -173,7 +274,8 @@ class BeamSearchDecoder(decoder.Decoder):
                initial_state,
                beam_width,
                output_layer=None,
-               length_penalty_weight=0.0):
+               length_penalty_weight=0.0,
+               reorder_tensor_arrays=True):
     """Initialize the BeamSearchDecoder.
 
     Args:
@@ -188,6 +290,12 @@ class BeamSearchDecoder(decoder.Decoder):
         `tf.layers.Dense`.  Optional layer to apply to the RNN output prior
         to storing the result or sampling.
       length_penalty_weight: Float weight to penalize length. Disabled with 0.0.
+      reorder_tensor_arrays: If `True`, `TensorArray`s' elements within the cell
+        state will be reordered according to the beam search path. If the
+        `TensorArray` can be reordered, the stacked form will be returned.
+        Otherwise, the `TensorArray` will be returned as is. Set this flag to
+        `False` if the cell state contains `TensorArray`s that are not amenable
+        to reordering.
 
     Raises:
       TypeError: if `cell` is not an instance of `RNNCell`,
@@ -202,6 +310,7 @@ class BeamSearchDecoder(decoder.Decoder):
           "output_layer must be a Layer, received: %s" % type(output_layer))
     self._cell = cell
     self._output_layer = output_layer
+    self._reorder_tensor_arrays = reorder_tensor_arrays
 
     if callable(embedding):
       self._embedding_fn = embedding
@@ -342,6 +451,11 @@ class BeamSearchDecoder(decoder.Decoder):
         outputs.parent_ids,
         max_sequence_lengths=max_sequence_lengths,
         end_token=self._end_token)
+    if self._reorder_tensor_arrays:
+      final_state = final_state._replace(cell_state=nest.map_structure(
+          lambda t: self._maybe_sort_array_beams(
+              t, outputs.parent_ids, final_state.lengths),
+          final_state.cell_state))
     outputs = FinalBeamSearchDecoderOutput(
         beam_search_decoder_output=outputs, predicted_ids=predicted_ids)
     return outputs, final_state
@@ -432,9 +546,10 @@ class BeamSearchDecoder(decoder.Decoder):
       returned unchanged.
 
     Raises:
-      TypeError: If `t` is an instance of `TensorArray`.
       ValueError: If the rank of `t` is not statically known.
     """
+    if isinstance(t, tensor_array_ops.TensorArray):
+      return t
     _check_maybe(t)
     if t.shape.ndims >= 1:
       return self._split_batch_beams(t, s)
@@ -455,15 +570,55 @@ class BeamSearchDecoder(decoder.Decoder):
       A reshaped version of t with shape `[batch_size, beam_width] + s`.
 
     Raises:
-      TypeError: If `t` is an instance of `TensorArray`.
       ValueError:  If the rank of `t` is not statically known.
     """
+    if isinstance(t, tensor_array_ops.TensorArray):
+      return t
     _check_maybe(t)
     if t.shape.ndims >= 2:
       return self._merge_batch_beams(t, s)
     else:
       return t
 
+  def _maybe_sort_array_beams(self, t, parent_ids, sequence_length):
+    """Maybe sorts beams within a `TensorArray`.
+
+    Args:
+      t: A `TensorArray` of size `max_time` that contains `Tensor`s of shape
+        `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]` where
+        `s` is the depth shape.
+      parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`.
+      sequence_length: The sequence length of shape `[batch_size, beam_width]`.
+
+    Returns:
+      A `TensorArray` where beams are sorted in each `Tensor` or `t` itself if
+      it is not a `TensorArray` or does not meet shape requirements.
+    """
+    if not isinstance(t, tensor_array_ops.TensorArray):
+      return t
+    # pylint: disable=protected-access
+    if (not t._infer_shape or not t._element_shape
+        or t._element_shape[0].ndims is None
+        or t._element_shape[0].ndims < 1):
+      shape = (
+          t._element_shape[0] if t._infer_shape and t._element_shape
+          else tensor_shape.TensorShape(None))
+      tf_logging.warn("The TensorArray %s in the cell state is not amenable to "
+                      "sorting based on the beam search result. For a "
+                      "TensorArray to be sorted, its elements shape must be "
+                      "defined and have at least a rank of 1, but saw shape: %s"
+                      % (t.handle.name, shape))
+      return t
+    shape = t._element_shape[0]
+    # pylint: enable=protected-access
+    if not _check_static_batch_beam_maybe(
+        shape, tensor_util.constant_value(self._batch_size), self._beam_width):
+      return t
+    t = t.stack()
+    with ops.control_dependencies(
+        [_check_batch_beam(t, self._batch_size, self._beam_width)]):
+      return gather_tree_from_array(t, parent_ids, sequence_length)
+
   def step(self, time, inputs, state, name=None):
     """Perform a decoding step.
 
@@ -758,6 +913,8 @@ def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size,
     output: Gathered tensor of shape tf.shape(gather_from)[:1+len(gather_shape)]
       or the original tensor if its dimensions are too small.
   """
+  if isinstance(gather_from, tensor_array_ops.TensorArray):
+    return gather_from
   _check_maybe(gather_from)
   if gather_from.shape.ndims >= len(gather_shape):
     return _tensor_gather_helper(
-- 
GitLab


From d28d4f4366b24876862b39351f67eed78c87f5eb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 18 Mar 2018 15:18:06 -0700
Subject: [PATCH 162/960] Add precision and recall metrics to
 _BinaryLogisticHeadWithSigmoidCrossEntropyLoss.

This change makes most of the binary classifiers in the canned estimators provide precision and recall metrics during evaluation. This matches the behavior of the canned estimators defined in the deprecated tf.contrib.learn.estimator.

PiperOrigin-RevId: 189522420
---
 .../python/estimator/canned/baseline_test.py       |  7 ++++++-
 .../python/estimator/canned/dnn_testing_utils.py   |  3 +++
 tensorflow/python/estimator/canned/head.py         | 12 ++++++++++++
 tensorflow/python/estimator/canned/head_test.py    | 14 +++++++++++++-
 .../estimator/canned/linear_testing_utils.py       |  6 ++++++
 tensorflow/python/estimator/canned/metric_keys.py  |  2 ++
 6 files changed, 42 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/estimator/canned/baseline_test.py b/tensorflow/python/estimator/canned/baseline_test.py
index 96639e88ea..7833df2052 100644
--- a/tensorflow/python/estimator/canned/baseline_test.py
+++ b/tensorflow/python/estimator/canned/baseline_test.py
@@ -1071,6 +1071,8 @@ class BaselineClassifierEvaluationTest(test.TestCase):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: 1.3133,
           metric_keys.MetricKeys.ACCURACY: 0.,
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: 0.2689,
           metric_keys.MetricKeys.LABEL_MEAN: 1.,
           metric_keys.MetricKeys.ACCURACY_BASELINE: 1,
@@ -1132,6 +1134,8 @@ class BaselineClassifierEvaluationTest(test.TestCase):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2,
           metric_keys.MetricKeys.ACCURACY: 0.5,
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: 0.2689,
           metric_keys.MetricKeys.LABEL_MEAN: 0.5,
           metric_keys.MetricKeys.ACCURACY_BASELINE: 0.5,
@@ -1207,6 +1211,8 @@ class BaselineClassifierEvaluationTest(test.TestCase):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: loss_mean,
           metric_keys.MetricKeys.ACCURACY: 2. / (1. + 2.),
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: predictions_mean,
           metric_keys.MetricKeys.LABEL_MEAN: label_mean,
           metric_keys.MetricKeys.ACCURACY_BASELINE: (
@@ -1542,4 +1548,3 @@ class BaselineLogitFnTest(test.TestCase):
 
 if __name__ == '__main__':
   test.main()
-
diff --git a/tensorflow/python/estimator/canned/dnn_testing_utils.py b/tensorflow/python/estimator/canned/dnn_testing_utils.py
index 9a7d088778..85b058caf3 100644
--- a/tensorflow/python/estimator/canned/dnn_testing_utils.py
+++ b/tensorflow/python/estimator/canned/dnn_testing_utils.py
@@ -1035,6 +1035,8 @@ class BaseDNNClassifierEvaluateTest(object):
         metric_keys.MetricKeys.LOSS: expected_loss,
         metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2.,
         metric_keys.MetricKeys.ACCURACY: 0.5,
+        metric_keys.MetricKeys.PRECISION: 0.0,
+        metric_keys.MetricKeys.RECALL: 0.0,
         metric_keys.MetricKeys.PREDICTION_MEAN: 0.11105597,
         metric_keys.MetricKeys.LABEL_MEAN: 0.5,
         metric_keys.MetricKeys.ACCURACY_BASELINE: 0.5,
@@ -1042,6 +1044,7 @@ class BaseDNNClassifierEvaluateTest(object):
         # that is what the algorithm returns.
         metric_keys.MetricKeys.AUC: 0.5,
         metric_keys.MetricKeys.AUC_PR: 0.75,
+
         ops.GraphKeys.GLOBAL_STEP: global_step
     }, dnn_classifier.evaluate(input_fn=_input_fn, steps=1))
 
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index 8d742a2c61..f68204a35e 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -940,6 +940,18 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
                   predictions=class_ids,
                   weights=weights,
                   name=keys.ACCURACY),
+          _summary_key(self._name, keys.PRECISION):
+              metrics_lib.precision(
+                  labels=labels,
+                  predictions=class_ids,
+                  weights=weights,
+                  name=keys.PRECISION),
+          _summary_key(self._name, keys.RECALL):
+              metrics_lib.recall(
+                  labels=labels,
+                  predictions=class_ids,
+                  weights=weights,
+                  name=keys.RECALL),
           _summary_key(self._name, keys.PREDICTION_MEAN):
               _predictions_mean(
                   predictions=logistic,
diff --git a/tensorflow/python/estimator/canned/head_test.py b/tensorflow/python/estimator/canned/head_test.py
index b40758f8fe..b5d35c9b45 100644
--- a/tensorflow/python/estimator/canned/head_test.py
+++ b/tensorflow/python/estimator/canned/head_test.py
@@ -1559,6 +1559,8 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
         # loss_mean = loss/2 = 41./2 = 20.5
         keys.LOSS_MEAN: 20.5,
         keys.ACCURACY: 1./2,
+        keys.PRECISION: 1.,
+        keys.RECALL: 1./2,
         keys.PREDICTION_MEAN: 1./2,
         keys.LABEL_MEAN: 2./2,
         keys.ACCURACY_BASELINE: 2./2,
@@ -1602,11 +1604,13 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
     expected_metric_keys = [
         '{}/some_binary_head'.format(metric_keys.MetricKeys.LOSS_MEAN),
         '{}/some_binary_head'.format(metric_keys.MetricKeys.ACCURACY),
+        '{}/some_binary_head'.format(metric_keys.MetricKeys.PRECISION),
+        '{}/some_binary_head'.format(metric_keys.MetricKeys.RECALL),
         '{}/some_binary_head'.format(metric_keys.MetricKeys.PREDICTION_MEAN),
         '{}/some_binary_head'.format(metric_keys.MetricKeys.LABEL_MEAN),
         '{}/some_binary_head'.format(metric_keys.MetricKeys.ACCURACY_BASELINE),
         '{}/some_binary_head'.format(metric_keys.MetricKeys.AUC),
-        '{}/some_binary_head'.format(metric_keys.MetricKeys.AUC_PR)
+        '{}/some_binary_head'.format(metric_keys.MetricKeys.AUC_PR),
     ]
     self.assertItemsEqual(expected_metric_keys, spec.eval_metric_ops.keys())
 
@@ -1637,6 +1641,8 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
         keys.LOSS_MEAN: expected_unregularized_loss,
         keys.LOSS_REGULARIZATION: expected_regularization_loss,
         keys.ACCURACY: 1./2,
+        keys.PRECISION: 1.,
+        keys.RECALL: 1./2,
         keys.PREDICTION_MEAN: 1./2,
         keys.LABEL_MEAN: 2./2,
         keys.ACCURACY_BASELINE: 2./2,
@@ -1742,6 +1748,8 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
     expected_metrics = {
         keys.LOSS_MEAN: 1.62652338 / 2.,
         keys.ACCURACY: 1./2,
+        keys.PRECISION: 1.,
+        keys.RECALL: .5,
         keys.PREDICTION_MEAN: 1./2,
         keys.LABEL_MEAN: 2./2,
         keys.ACCURACY_BASELINE: 2./2,
@@ -2187,6 +2195,8 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
         keys.LOSS_MEAN: 26.9615384615,
         # accuracy = (1*1 + .1*0 + 1.5*0)/(1 + .1 + 1.5) = 1/2.6 = .38461538461
         keys.ACCURACY: .38461538461,
+        keys.PRECISION: 1./2.5,
+        keys.RECALL: 1./1.1,
         # prediction_mean = (1*1 + .1*0 + 1.5*1)/(1 + .1 + 1.5) = 2.5/2.6
         #                 = .96153846153
         keys.PREDICTION_MEAN: .96153846153,
@@ -2486,6 +2496,8 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
     expected_metrics = {
         keys.LOSS_MEAN: expected_loss / np.sum(weights),
         keys.ACCURACY: (1.*0. + 1.5*1. + 2.*1. + 2.5*0.) / np.sum(weights),
+        keys.PRECISION: 2.0/3.0,
+        keys.RECALL: 2.0/4.5,
         keys.PREDICTION_MEAN: (1.*1 + 1.5*0 + 2.*1 + 2.5*0) / np.sum(weights),
         keys.LABEL_MEAN: (1.*0 + 1.5*0 + 2.*1 + 2.5*1) / np.sum(weights),
         keys.ACCURACY_BASELINE: (1.*0 + 1.5*0 + 2.*1 + 2.5*1) / np.sum(weights),
diff --git a/tensorflow/python/estimator/canned/linear_testing_utils.py b/tensorflow/python/estimator/canned/linear_testing_utils.py
index 8e506a7631..da3ce86999 100644
--- a/tensorflow/python/estimator/canned/linear_testing_utils.py
+++ b/tensorflow/python/estimator/canned/linear_testing_utils.py
@@ -1337,6 +1337,8 @@ class BaseLinearClassifierEvaluationTest(object):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: 41.,
           metric_keys.MetricKeys.ACCURACY: 0.,
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: 0.,
           metric_keys.MetricKeys.LABEL_MEAN: 1.,
           metric_keys.MetricKeys.ACCURACY_BASELINE: 1,
@@ -1406,6 +1408,8 @@ class BaseLinearClassifierEvaluationTest(object):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2,
           metric_keys.MetricKeys.ACCURACY: 0.,
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: 0.5,
           metric_keys.MetricKeys.LABEL_MEAN: 0.5,
           metric_keys.MetricKeys.ACCURACY_BASELINE: 0.5,
@@ -1487,6 +1491,8 @@ class BaseLinearClassifierEvaluationTest(object):
           ops.GraphKeys.GLOBAL_STEP: 100,
           metric_keys.MetricKeys.LOSS_MEAN: loss_mean,
           metric_keys.MetricKeys.ACCURACY: 0.,
+          metric_keys.MetricKeys.PRECISION: 0.,
+          metric_keys.MetricKeys.RECALL: 0.,
           metric_keys.MetricKeys.PREDICTION_MEAN: predictions_mean,
           metric_keys.MetricKeys.LABEL_MEAN: label_mean,
           metric_keys.MetricKeys.ACCURACY_BASELINE: (
diff --git a/tensorflow/python/estimator/canned/metric_keys.py b/tensorflow/python/estimator/canned/metric_keys.py
index 44eb680939..f374d31549 100644
--- a/tensorflow/python/estimator/canned/metric_keys.py
+++ b/tensorflow/python/estimator/canned/metric_keys.py
@@ -28,6 +28,8 @@ class MetricKeys(object):
   LOSS_REGULARIZATION = 'regularization_loss'
 
   ACCURACY = 'accuracy'
+  PRECISION = 'precision'
+  RECALL = 'recall'
   # This is the best the model could do by always predicting one class.
   # Should be < ACCURACY in a trained model.
   ACCURACY_BASELINE = 'accuracy_baseline'
-- 
GitLab


From d99731f28ab7566762da9b22cdc24486e3308a60 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 4 Nov 2017 22:22:37 +0000
Subject: [PATCH 163/960] Add int64 support of `axis` (`Tidx`) for ConcatV2

In `array_ops.cc`, it was specified that ConcatV2 support
both int32 and int64 data types of `axis` (`Tidx`):
```
    .Attr("Tidx: {int32, int64} = DT_INT32")
```

However, in actual kernel implementations only int32 is supported
as there is an unnecessary `.TypeConstraint<int32>("Tidx")` specified.

This fix tries to address the discrepancy between the ops declaration
and kernel registration by adding the int64 axis (`Tidx`) support
for `ConcatV2`.

This fix removes the TypeConstraint and adds additional processing
so that differnt types (int32 or int64) of `axis` could be processed
correctly.

Additional test cases have been added to cover the changes as well.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/concat_op.cc | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index 7011550f7e..c4850150e1 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -62,8 +62,19 @@ class ConcatBaseOp : public OpKernel {
                     axis_attribute_name,
                     " tensor should be a scalar integer, but got shape ",
                     concat_dim_tensor->shape().DebugString()));
-    const int32 concat_dim =
-        internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
+    int64 concat_dim;
+    // In case of ConcatV2, "axis" could be int32 or int64
+    if (AxisArgName == NAME_IS_AXIS) {
+      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32 || concat_dim_tensor->dtype() == DT_INT64), errors::InvalidArgument(axis_attribute_name, " tensor should be int32 or int64, but got ", concat_dim_tensor->dtype()));
+    } else {
+      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32), errors::InvalidArgument(axis_attribute_name, " tensor should be int32, but got ", concat_dim_tensor->dtype()));
+    }
+    if (concat_dim_tensor->dtype() == DT_INT32) {
+      concat_dim = internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
+    } else {
+      concat_dim = internal::SubtleMustCopy(concat_dim_tensor->scalar<int64>()());
+    }
+
     OpInputList values;
     OP_REQUIRES_OK(c, c->input_list("values", &values));
     const int N = values.size();
@@ -163,7 +174,6 @@ using ConcatV2Op = ConcatBaseOp<Device, T, NAME_IS_AXIS>;
   REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
                               .Device(DEVICE_CPU)            \
                               .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
                               .HostMemory("axis"),           \
                           ConcatV2Op<CPUDevice, type>)
 
@@ -187,7 +197,6 @@ REGISTER_CONCAT(qint32);
   REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
                               .Device(DEVICE_GPU)            \
                               .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
                               .HostMemory("axis"),           \
                           ConcatV2Op<GPUDevice, type>)
 
@@ -212,7 +221,6 @@ REGISTER_KERNEL_BUILDER(Name("Concat")
 REGISTER_KERNEL_BUILDER(Name("ConcatV2")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tidx")
                             .HostMemory("values")
                             .HostMemory("axis")
                             .HostMemory("output"),
@@ -230,7 +238,6 @@ REGISTER_KERNEL_BUILDER(Name("ConcatV2")
   REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
                               .Device(DEVICE_SYCL)           \
                               .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
                               .HostMemory("axis"),           \
                           ConcatV2Op<SYCLDevice, type>)
 
@@ -246,7 +253,6 @@ REGISTER_KERNEL_BUILDER(Name("Concat")
 REGISTER_KERNEL_BUILDER(Name("ConcatV2")
                             .Device(DEVICE_SYCL)
                             .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tidx")
                             .HostMemory("values")
                             .HostMemory("axis")
                             .HostMemory("output"),
-- 
GitLab


From beff710f6230bc3c27ffe53a3d788bd6503359ac Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 4 Nov 2017 22:27:43 +0000
Subject: [PATCH 164/960] Add test cases for int64 support of `axis` (`Tidx`)
 for ConcatV2

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/concat_op_test.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 81c6a4aa6e..073611628c 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -606,6 +606,17 @@ class ConcatOpTest(test.TestCase):
           inp_tensors_placeholders, -2, output_shape=[2, 3],
           gather_indexes=[2, 0], feed_dict=feed_dict)
 
+  def testConcatAxisType(self):
+    for dtype in [dtypes.int32, dtypes.int64]:
+      with self.test_session(use_gpu=True):
+        t1 = [[1, 2, 3], [4, 5, 6]]
+        t2 = [[7, 8, 9], [10, 11, 12]]
+
+        c = gen_array_ops._concat_v2([t1, t2],
+                                     constant_op.constant(1, dtype=dtype))
+        self.assertEqual([2, 6], c.get_shape().as_list())
+        output = c.eval()
+        self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
 class ConcatOffsetTest(test.TestCase):
 
-- 
GitLab


From a508dcb2b732e8423794635630437b6c73deecba Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 4 Nov 2017 22:28:36 +0000
Subject: [PATCH 165/960] Sanitize concat_op.cc with clang-format -i
 --style=Google

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/kernels/concat_op.cc | 86 ++++++++++++++++------------
 1 file changed, 48 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index c4850150e1..f16766315f 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <limits>
 #include <vector>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -28,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -53,9 +53,9 @@ class ConcatBaseOp : public OpKernel {
   void Compute(OpKernelContext* c) override {
     const Tensor* concat_dim_tensor;
     const char* axis_attribute_name =
-        AxisArgName == NAME_IS_AXIS
-            ? "axis"
-            : AxisArgName == NAME_IS_CONCAT_DIM ? "concat_dim" : "<invalid>";
+        AxisArgName == NAME_IS_AXIS ? "axis" : AxisArgName == NAME_IS_CONCAT_DIM
+                                                   ? "concat_dim"
+                                                   : "<invalid>";
     OP_REQUIRES_OK(c, c->input(axis_attribute_name, &concat_dim_tensor));
     OP_REQUIRES(c, IsLegacyScalar(concat_dim_tensor->shape()),
                 errors::InvalidArgument(
@@ -65,14 +65,24 @@ class ConcatBaseOp : public OpKernel {
     int64 concat_dim;
     // In case of ConcatV2, "axis" could be int32 or int64
     if (AxisArgName == NAME_IS_AXIS) {
-      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32 || concat_dim_tensor->dtype() == DT_INT64), errors::InvalidArgument(axis_attribute_name, " tensor should be int32 or int64, but got ", concat_dim_tensor->dtype()));
+      OP_REQUIRES(
+          c, (concat_dim_tensor->dtype() == DT_INT32 ||
+              concat_dim_tensor->dtype() == DT_INT64),
+          errors::InvalidArgument(axis_attribute_name,
+                                  " tensor should be int32 or int64, but got ",
+                                  concat_dim_tensor->dtype()));
     } else {
-      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32), errors::InvalidArgument(axis_attribute_name, " tensor should be int32, but got ", concat_dim_tensor->dtype()));
+      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32),
+                  errors::InvalidArgument(axis_attribute_name,
+                                          " tensor should be int32, but got ",
+                                          concat_dim_tensor->dtype()));
     }
     if (concat_dim_tensor->dtype() == DT_INT32) {
-      concat_dim = internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
+      concat_dim =
+          internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
     } else {
-      concat_dim = internal::SubtleMustCopy(concat_dim_tensor->scalar<int64>()());
+      concat_dim =
+          internal::SubtleMustCopy(concat_dim_tensor->scalar<int64>()());
     }
 
     OpInputList values;
@@ -165,16 +175,16 @@ using ConcatOp = ConcatBaseOp<Device, T, NAME_IS_CONCAT_DIM>;
 template <typename Device, typename T>
 using ConcatV2Op = ConcatBaseOp<Device, T, NAME_IS_AXIS>;
 
-#define REGISTER_CONCAT(type)                                \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_CPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<CPUDevice, type>)         \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_CPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("axis"),           \
+#define REGISTER_CONCAT(type)                            \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<CPUDevice, type>)     \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<CPUDevice, type>)
 
 TF_CALL_POD_STRING_TYPES(REGISTER_CONCAT);
@@ -188,16 +198,16 @@ REGISTER_CONCAT(qint32);
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU(type)                                   \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_GPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<GPUDevice, type>)         \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_GPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("axis"),           \
+#define REGISTER_GPU(type)                               \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_GPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<GPUDevice, type>)     \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_GPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<GPUDevice, type>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU);
@@ -229,16 +239,16 @@ REGISTER_KERNEL_BUILDER(Name("ConcatV2")
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL(type)                                  \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_SYCL)           \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<SYCLDevice, type>)        \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_SYCL)           \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("axis"),           \
+#define REGISTER_SYCL(type)                              \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_SYCL)       \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<SYCLDevice, type>)    \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_SYCL)       \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<SYCLDevice, type>)
 
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL);
-- 
GitLab


From 343d30aec78b9f8f58a132988ae237e4fd9ce917 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 19 Mar 2018 00:06:38 +0000
Subject: [PATCH 166/960] Change `gen_array_ops._concat_v2` to
 gen_array_ops.concat_v2`

as `_` is not needed any more.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/concat_op_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 073611628c..c22934ce47 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -612,8 +612,8 @@ class ConcatOpTest(test.TestCase):
         t1 = [[1, 2, 3], [4, 5, 6]]
         t2 = [[7, 8, 9], [10, 11, 12]]
 
-        c = gen_array_ops._concat_v2([t1, t2],
-                                     constant_op.constant(1, dtype=dtype))
+        c = gen_array_ops.concat_v2([t1, t2],
+                                    constant_op.constant(1, dtype=dtype))
         self.assertEqual([2, 6], c.get_shape().as_list())
         output = c.eval()
         self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
-- 
GitLab


From 4b1b779b48aca2059319b9af20295e04d60fa1f1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 05:01:05 -0700
Subject: [PATCH 167/960] Add new helpers to HLO sharding.

PiperOrigin-RevId: 189569053
---
 tensorflow/compiler/xla/service/hlo_sharding.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h
index e715dff9a0..38273236f9 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.h
+++ b/tensorflow/compiler/xla/service/hlo_sharding.h
@@ -173,7 +173,7 @@ class HloSharding {
 
   bool operator==(const HloSharding& other) const {
     return replicated_ == other.replicated_ && maximal_ == other.maximal_ &&
-           protobuf_util::ProtobufEquals(tile_shape_, other.tile_shape_) &&
+           ShapeUtil::Compatible(tile_shape_, other.tile_shape_) &&
            tile_assignment_ == other.tile_assignment_ &&
            tuple_elements_ == other.tuple_elements_;
   }
@@ -207,6 +207,13 @@ class HloSharding {
   // REQUIRES: !IsReplicated() && !IsTuple()
   const Array<int64>& tile_assignment() const { return tile_assignment_; }
 
+  // Returns the flattened list of all the leaf shardings in a tuple shape, by
+  // pre-order walk (ShapeTree iterator order).
+  // REQUIRES: IsTuple().
+  const std::vector<HloSharding>& tuple_elements() const {
+    return tuple_elements_;
+  }
+
   // Return a new sharding that can apply to the given new shape.
   // If this sharding is tile-maximal, the returned sharding will be the same as
   // this sharding. If this sharding is not tile-maximal, the returned
-- 
GitLab


From 8bd5da29ca4e502591fb38dfd27ecd86c9cef7ab Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 06:27:00 -0700
Subject: [PATCH 168/960] Adding non-linear image warping ops to
 tf.contrib.image

New ops are:
tf.contrib.image.sparse_image_warp, tf.contrib.image.dense_image_warp, and tf.contrib.image.interpolate_spline.

PiperOrigin-RevId: 189574951
---
 tensorflow/contrib/cmake/tf_tests.cmake       |   2 +
 tensorflow/contrib/image/BUILD                | 113 +++++++
 tensorflow/contrib/image/__init__.py          |   7 +
 .../kernel_tests/dense_image_warp_test.py     | 267 ++++++++++++++++
 .../kernel_tests/interpolate_spline_test.py   | 264 ++++++++++++++++
 .../kernel_tests/sparse_image_warp_test.py    | 254 +++++++++++++++
 .../test_data/Yellow_Smiley_Face.png          | Bin 0 -> 14060 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-0.png | Bin 0 -> 18537 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-1.png | Bin 0 -> 19086 bytes
 ...llow_Smiley_Face_Warp-interp-1-clamp-4.png | Bin 0 -> 18884 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-0.png | Bin 0 -> 18109 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-1.png | Bin 0 -> 19251 bytes
 ...llow_Smiley_Face_Warp-interp-2-clamp-4.png | Bin 0 -> 19132 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-0.png | Bin 0 -> 17500 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-1.png | Bin 0 -> 18058 bytes
 ...llow_Smiley_Face_Warp-interp-3-clamp-4.png | Bin 0 -> 19313 bytes
 .../image/python/ops/dense_image_warp.py      | 201 ++++++++++++
 .../image/python/ops/interpolate_spline.py    | 291 ++++++++++++++++++
 .../image/python/ops/sparse_image_warp.py     | 201 ++++++++++++
 .../tools/pip_package/pip_smoke_test.py       |   1 +
 20 files changed, 1601 insertions(+)
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
 create mode 100644 tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
 create mode 100644 tensorflow/contrib/image/python/ops/dense_image_warp.py
 create mode 100644 tensorflow/contrib/image/python/ops/interpolate_spline.py
 create mode 100644 tensorflow/contrib/image/python/ops/sparse_image_warp.py

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 9f96a4b797..cdf48b3584 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -195,9 +195,11 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/python/profiler/model_analyzer_test.py"
     # Fails because uses data dependencies with bazel
     "${tensorflow_source_dir}/tensorflow/python/saved_model/saved_model_test.py"
+    "${tensorflow_source_dir}/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py"
     # requires scipy
     "${tensorflow_source_dir}/tensorflow/contrib/keras/python/keras/preprocessing/*_test.py"
     "${tensorflow_source_dir}/tensorflow/contrib/tfprof/python/tools/tfprof/pprof_profiler_test.py"
+    "${tensorflow_source_dir}/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py"
     # Takes very long to run without sharding (defined in bazel build file).
     "${tensorflow_source_dir}/tensorflow/python/kernel_tests/cwise_ops_test.py"
     # Loading resources in contrib doesn't seem to work on Windows
diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 3ff02e085e..2924aef815 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -78,7 +78,10 @@ tf_custom_op_py_library(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        ":dense_image_warp_py",
         ":image_ops",
+        ":interpolate_spline_py",
+        ":sparse_image_warp_py",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:common_shapes",
@@ -194,6 +197,116 @@ cuda_py_test(
     ],
 )
 
+py_library(
+    name = "dense_image_warp_py",
+    srcs = [
+        "python/ops/dense_image_warp.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+        "//third_party/py/numpy",
+    ],
+)
+
+py_library(
+    name = "interpolate_spline_py",
+    srcs = [
+        "python/ops/interpolate_spline.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+    ],
+)
+
+py_library(
+    name = "sparse_image_warp_py",
+    srcs = [
+        "python/ops/sparse_image_warp.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dense_image_warp_py",
+        ":interpolate_spline_py",
+        "//tensorflow/contrib/util:util_py",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:util",
+    ],
+)
+
+cuda_py_test(
+    name = "sparse_image_warp_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/sparse_image_warp_test.py"],
+    additional_deps = [
+        ":sparse_image_warp_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+    ],
+    data = [":sparse_image_warp_test_data"],
+)
+
+filegroup(
+    name = "sparse_image_warp_test_data",
+    srcs = glob(["python/kernel_tests/test_data/*.png"]),
+)
+
+cuda_py_test(
+    name = "dense_image_warp_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/dense_image_warp_test.py"],
+    additional_deps = [
+        ":dense_image_warp_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:random_ops",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
+cuda_py_test(
+    name = "interpolate_spline_test",
+    size = "medium",
+    srcs = ["python/kernel_tests/interpolate_spline_test.py"],
+    additional_deps = [
+        ":interpolate_spline_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:io_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:image_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/core:protos_all_py",
+    ],
+)
+
 tf_py_test(
     name = "segmentation_test",
     size = "medium",
diff --git a/tensorflow/contrib/image/__init__.py b/tensorflow/contrib/image/__init__.py
index cc8ed117ba..e982030bc8 100755
--- a/tensorflow/contrib/image/__init__.py
+++ b/tensorflow/contrib/image/__init__.py
@@ -30,6 +30,9 @@ projective transforms (including rotation) are supported.
 @@transform
 @@translate
 @@translations_to_projective_transforms
+@@dense_image_warp
+@@interpolate_spline
+@@sparse_image_warp
 
 ## Image Segmentation `Ops`
 
@@ -47,6 +50,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.image.python.ops.dense_image_warp import dense_image_warp
+
 from tensorflow.contrib.image.python.ops.distort_image_ops import adjust_hsv_in_yiq
 from tensorflow.contrib.image.python.ops.distort_image_ops import random_hsv_in_yiq
 
@@ -57,7 +62,9 @@ from tensorflow.contrib.image.python.ops.image_ops import rotate
 from tensorflow.contrib.image.python.ops.image_ops import transform
 from tensorflow.contrib.image.python.ops.image_ops import translate
 from tensorflow.contrib.image.python.ops.image_ops import translations_to_projective_transforms
+from tensorflow.contrib.image.python.ops.interpolate_spline import interpolate_spline
 from tensorflow.contrib.image.python.ops.single_image_random_dot_stereograms import single_image_random_dot_stereograms
+from tensorflow.contrib.image.python.ops.sparse_image_warp import sparse_image_warp
 
 from tensorflow.python.util.all_util import remove_undocumented
 
diff --git a/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
new file mode 100644
index 0000000000..a58b6a247e
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/dense_image_warp_test.py
@@ -0,0 +1,267 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for dense_image_warp."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import math
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import dense_image_warp
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+from tensorflow.python.training import adam
+
+
+class DenseImageWarpTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    np.random.seed(0)
+
+  def test_interpolate_small_grid_ij(self):
+    grid = constant_op.constant(
+        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
+    query_points = constant_op.constant(
+        [[0., 0.], [1., 0.], [2., 0.5], [1.5, 1.5]], shape=[1, 4, 2])
+    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def test_interpolate_small_grid_xy(self):
+    grid = constant_op.constant(
+        [[0., 1., 2.], [3., 4., 5.], [6., 7., 8.]], shape=[1, 3, 3, 1])
+    query_points = constant_op.constant(
+        [[0., 0.], [0., 1.], [0.5, 2.0], [1.5, 1.5]], shape=[1, 4, 2])
+    expected_results = np.reshape(np.array([0., 3., 6.5, 6.]), [1, 4, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(
+        grid, query_points, indexing='xy')
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def test_interpolate_small_grid_batched(self):
+    grid = constant_op.constant(
+        [[[0., 1.], [3., 4.]], [[5., 6.], [7., 8.]]], shape=[2, 2, 2, 1])
+    query_points = constant_op.constant([[[0., 0.], [1., 0.], [0.5, 0.5]],
+                                         [[0.5, 0.], [1., 0.], [1., 1.]]])
+    expected_results = np.reshape(
+        np.array([[0., 3., 2.], [6., 7., 8.]]), [2, 3, 1])
+
+    interp = dense_image_warp._interpolate_bilinear(grid, query_points)
+
+    with self.test_session() as sess:
+      predicted = sess.run(interp)
+      self.assertAllClose(expected_results, predicted)
+
+  def get_image_and_flow_placeholders(self, shape, image_type, flow_type):
+    batch_size, height, width, numchannels = shape
+    image_shape = [batch_size, height, width, numchannels]
+    flow_shape = [batch_size, height, width, 2]
+
+    tf_type = {
+        'float16': dtypes.half,
+        'float32': dtypes.float32,
+        'float64': dtypes.float64
+    }
+
+    image = array_ops.placeholder(dtype=tf_type[image_type], shape=image_shape)
+
+    flows = array_ops.placeholder(dtype=tf_type[flow_type], shape=flow_shape)
+    return image, flows
+
+  def get_random_image_and_flows(self, shape, image_type, flow_type):
+    batch_size, height, width, numchannels = shape
+    image_shape = [batch_size, height, width, numchannels]
+    image = np.random.normal(size=image_shape)
+    flow_shape = [batch_size, height, width, 2]
+    flows = np.random.normal(size=flow_shape) * 3
+    return image.astype(image_type), flows.astype(flow_type)
+
+  def assert_correct_interpolation_value(self,
+                                         image,
+                                         flows,
+                                         pred_interpolation,
+                                         batch_index,
+                                         y_index,
+                                         x_index,
+                                         low_precision=False):
+    """Assert that the tf interpolation matches hand-computed value."""
+
+    height = image.shape[1]
+    width = image.shape[2]
+    displacement = flows[batch_index, y_index, x_index, :]
+    float_y = y_index - displacement[0]
+    float_x = x_index - displacement[1]
+    floor_y = max(min(height - 2, math.floor(float_y)), 0)
+    floor_x = max(min(width - 2, math.floor(float_x)), 0)
+    ceil_y = floor_y + 1
+    ceil_x = floor_x + 1
+
+    alpha_y = min(max(0.0, float_y - floor_y), 1.0)
+    alpha_x = min(max(0.0, float_x - floor_x), 1.0)
+
+    floor_y = int(floor_y)
+    floor_x = int(floor_x)
+    ceil_y = int(ceil_y)
+    ceil_x = int(ceil_x)
+
+    top_left = image[batch_index, floor_y, floor_x, :]
+    top_right = image[batch_index, floor_y, ceil_x, :]
+    bottom_left = image[batch_index, ceil_y, floor_x, :]
+    bottom_right = image[batch_index, ceil_y, ceil_x, :]
+
+    interp_top = alpha_x * (top_right - top_left) + top_left
+    interp_bottom = alpha_x * (bottom_right - bottom_left) + bottom_left
+    interp = alpha_y * (interp_bottom - interp_top) + interp_top
+    atol = 1e-6
+    rtol = 1e-6
+    if low_precision:
+      atol = 1e-2
+      rtol = 1e-3
+    self.assertAllClose(
+        interp,
+        pred_interpolation[batch_index, y_index, x_index, :],
+        atol=atol,
+        rtol=rtol)
+
+  def check_zero_flow_correctness(self, shape, image_type, flow_type):
+    """Assert using zero flows doesn't change the input image."""
+
+    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
+                                                        flow_type)
+    interp = dense_image_warp.dense_image_warp(image, flows)
+
+    with self.test_session() as sess:
+      rand_image, rand_flows = self.get_random_image_and_flows(
+          shape, image_type, flow_type)
+      rand_flows *= 0
+
+      predicted_interpolation = sess.run(
+          interp, feed_dict={
+              image: rand_image,
+              flows: rand_flows
+          })
+      self.assertAllClose(rand_image, predicted_interpolation)
+
+  def test_zero_flows(self):
+    """Apply check_zero_flow_correctness() for a few sizes and types."""
+
+    shapes_to_try = [[3, 4, 5, 6], [1, 2, 2, 1]]
+    for shape in shapes_to_try:
+      self.check_zero_flow_correctness(
+          shape, image_type='float32', flow_type='float32')
+
+  def check_interpolation_correctness(self,
+                                      shape,
+                                      image_type,
+                                      flow_type,
+                                      num_probes=5):
+    """Interpolate, and then assert correctness for a few query locations."""
+
+    image, flows = self.get_image_and_flow_placeholders(shape, image_type,
+                                                        flow_type)
+    interp = dense_image_warp.dense_image_warp(image, flows)
+    low_precision = image_type == 'float16' or flow_type == 'float16'
+    with self.test_session() as sess:
+      rand_image, rand_flows = self.get_random_image_and_flows(
+          shape, image_type, flow_type)
+
+      pred_interpolation = sess.run(
+          interp, feed_dict={
+              image: rand_image,
+              flows: rand_flows
+          })
+
+      for _ in range(num_probes):
+        batch_index = np.random.randint(0, shape[0])
+        y_index = np.random.randint(0, shape[1])
+        x_index = np.random.randint(0, shape[2])
+
+        self.assert_correct_interpolation_value(
+            rand_image,
+            rand_flows,
+            pred_interpolation,
+            batch_index,
+            y_index,
+            x_index,
+            low_precision=low_precision)
+
+  def test_interpolation(self):
+    """Apply check_interpolation_correctness() for a few sizes and types."""
+
+    shapes_to_try = [[3, 4, 5, 6], [1, 5, 5, 3], [1, 2, 2, 1]]
+    for im_type in ['float32', 'float64', 'float16']:
+      for flow_type in ['float32', 'float64', 'float16']:
+        for shape in shapes_to_try:
+          self.check_interpolation_correctness(shape, im_type, flow_type)
+
+  def test_gradients_exist(self):
+    """Check that backprop can run.
+
+    The correctness of the gradients is assumed, since the forward propagation
+    is tested to be correct and we only use built-in tf ops.
+    However, we perform a simple test to make sure that backprop can actually
+    run. We treat the flows as a tf.Variable and optimize them to minimize
+    the difference between the interpolated image and the input image.
+    """
+
+    batch_size, height, width, numchannels = [4, 5, 6, 7]
+    image_shape = [batch_size, height, width, numchannels]
+    image = random_ops.random_normal(image_shape)
+    flow_shape = [batch_size, height, width, 2]
+    init_flows = np.float32(np.random.normal(size=flow_shape) * 0.25)
+    flows = variables.Variable(init_flows)
+
+    interp = dense_image_warp.dense_image_warp(image, flows)
+    loss = math_ops.reduce_mean(math_ops.square(interp - image))
+
+    optimizer = adam.AdamOptimizer(1.0)
+    grad = gradients.gradients(loss, [flows])
+    opt_func = optimizer.apply_gradients(zip(grad, [flows]))
+    init_op = variables.global_variables_initializer()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(10):
+        sess.run(opt_func)
+
+  def test_size_exception(self):
+    """Make sure it throws an exception for images that are too small."""
+
+    shape = [1, 2, 1, 1]
+    msg = 'Should have raised an exception for invalid image size'
+    with self.assertRaises(ValueError, msg=msg):
+      self.check_interpolation_correctness(shape, 'float32', 'float32')
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
new file mode 100644
index 0000000000..1939caaa2d
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/interpolate_spline_test.py
@@ -0,0 +1,264 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for interpolate_spline."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from scipy import interpolate as sc_interpolate
+
+from tensorflow.contrib.image.python.ops import interpolate_spline
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+
+from tensorflow.python.training import momentum
+
+
+class _InterpolationProblem(object):
+  """Abstract class for interpolation problem descriptions."""
+
+  def get_problem(self, optimizable=False, extrapolate=True, dtype='float32'):
+    """Make data for an interpolation problem where all x vectors are n-d.
+
+    Args:
+      optimizable: If True, then make train_points a tf.Variable.
+      extrapolate: If False, then clamp the query_points values to be within
+      the max and min of train_points.
+      dtype: The data type to use.
+
+    Returns:
+      query_points, query_values, train_points, train_values: training and
+      test tensors for interpolation problem
+    """
+
+    # The values generated here depend on a seed of 0.
+    np.random.seed(0)
+
+    batch_size = 1
+    num_training_points = 10
+    num_query_points = 4
+
+    init_points = np.random.uniform(
+        size=[batch_size, num_training_points, self.DATA_DIM])
+
+    init_points = init_points.astype(dtype)
+    train_points = (
+        variables.Variable(init_points)
+        if optimizable else constant_op.constant(init_points))
+    train_values = self.tf_function(train_points)
+
+    query_points_np = np.random.uniform(
+        size=[batch_size, num_query_points, self.DATA_DIM])
+    query_points_np = query_points_np.astype(dtype)
+    if not extrapolate:
+      query_points_np = np.clip(query_points_np, np.min(init_points),
+                                np.max(init_points))
+
+    query_points = constant_op.constant(query_points_np)
+    query_values = self.np_function(query_points_np)
+
+    return query_points, query_values, train_points, train_values
+
+
+class _QuadraticPlusSinProblem1D(_InterpolationProblem):
+  """1D interpolation problem used for regression testing."""
+  DATA_DIM = 1
+  HARDCODED_QUERY_VALUES = {
+      (1.0, 0.0): [6.2647187603, -7.84362604077, -5.63690142322, 1.42928896387],
+      (1.0,
+       0.01): [6.77688289946, -8.02163669853, -5.79491157027, 1.4063285693],
+      (2.0,
+       0.0): [8.67110264937, -8.41281390883, -5.80190044693, 1.50155606059],
+      (2.0,
+       0.01): [6.70797816797, -7.49709587663, -5.28965776238, 1.52284731741],
+      (3.0,
+       0.0): [9.37691802935, -8.50390141515, -5.80786417426, 1.63467762122],
+      (3.0,
+       0.01): [4.47106304758, -5.71266128361, -3.92529303296, 1.86755293857],
+      (4.0,
+       0.0): [9.58172461111, -8.51432104771, -5.80967675388, 1.63361164256],
+      (4.0, 0.01): [
+          -3.87902711352, -0.0253462273846, 1.79857618022, -0.769339675725
+      ]
+  }
+
+  def np_function(self, x):
+    """Takes np array, evaluates the test function, and returns np array."""
+    return np.sum(
+        np.power((x - 0.5), 3) - 0.25 * x + 10 * np.sin(x * 10),
+        axis=2,
+        keepdims=True)
+
+  def tf_function(self, x):
+    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
+    return math_ops.reduce_mean(
+        math_ops.pow((x - 0.5), 3) - 0.25 * x + 10 * math_ops.sin(x * 10),
+        2,
+        keepdims=True)
+
+
+class _QuadraticPlusSinProblemND(_InterpolationProblem):
+  """3D interpolation problem used for regression testing."""
+
+  DATA_DIM = 3
+  HARDCODED_QUERY_VALUES = {
+      (1.0, 0.0): [1.06609663962, 1.28894849357, 1.10882405595, 1.63966936885],
+      (1.0, 0.01): [1.03123780748, 1.2952930985, 1.10366822954, 1.65265118569],
+      (2.0, 0.0): [0.627787735064, 1.43802857251, 1.00194632358, 1.91667538215],
+      (2.0, 0.01): [0.730159985046, 1.41702471595, 1.0065827217, 1.85758519312],
+      (3.0, 0.0): [0.350460417862, 1.67223539464, 1.00475331246, 2.31580322491],
+      (3.0,
+       0.01): [0.624557250556, 1.63138876667, 0.976588193162, 2.12511237866],
+      (4.0,
+       0.0): [0.898129669986, 1.24434133638, -0.938056116931, 1.59910338833],
+      (4.0,
+       0.01): [0.0930360338179, -3.38791305538, -1.00969032567, 0.745535080382],
+  }
+
+  def np_function(self, x):
+    """Takes np array, evaluates the test function, and returns np array."""
+    return np.sum(
+        np.square(x - 0.5) + 0.25 * x + 1 * np.sin(x * 15),
+        axis=2,
+        keepdims=True)
+
+  def tf_function(self, x):
+    """Takes tf tensor, evaluates the test function,  and returns tf tensor."""
+    return math_ops.reduce_sum(
+        math_ops.square(x - 0.5) + 0.25 * x + 1 * math_ops.sin(x * 15),
+        2,
+        keepdims=True)
+
+
+class InterpolateSplineTest(test_util.TensorFlowTestCase):
+
+  def test_1d_linear_interpolation(self):
+    """For 1d linear interpolation, we can compare directly to scipy."""
+
+    tp = _QuadraticPlusSinProblem1D()
+    (query_points, _, train_points, train_values) = tp.get_problem(
+        extrapolate=False, dtype='float64')
+    interpolation_order = 1
+
+    with ops.name_scope('interpolator'):
+      interpolator = interpolate_spline.interpolate_spline(
+          train_points, train_values, query_points, interpolation_order)
+      with self.test_session() as sess:
+        fetches = [query_points, train_points, train_values, interpolator]
+        query_points_, train_points_, train_values_, interp_ = sess.run(fetches)
+
+        # Just look at the first element of the minibatch.
+        # Also, trim the final singleton dimension.
+        interp_ = interp_[0, :, 0]
+        query_points_ = query_points_[0, :, 0]
+        train_points_ = train_points_[0, :, 0]
+        train_values_ = train_values_[0, :, 0]
+
+        # Compute scipy interpolation.
+        scipy_interp_function = sc_interpolate.interp1d(
+            train_points_, train_values_, kind='linear')
+
+        scipy_interpolation = scipy_interp_function(query_points_)
+        scipy_interpolation_on_train = scipy_interp_function(train_points_)
+
+        # Even with float64 precision, the interpolants disagree with scipy a
+        # bit due to the fact that we add the EPSILON to prevent sqrt(0), etc.
+        tol = 1e-3
+
+        self.assertAllClose(
+            train_values_, scipy_interpolation_on_train, atol=tol, rtol=tol)
+        self.assertAllClose(interp_, scipy_interpolation, atol=tol, rtol=tol)
+
+  def test_1d_interpolation(self):
+    """Regression test for interpolation with 1-D points."""
+
+    tp = _QuadraticPlusSinProblem1D()
+    (query_points, _, train_points,
+     train_values) = tp.get_problem(dtype='float64')
+
+    for order in (1, 2, 3):
+      for reg_weight in (0, 0.01):
+        interpolator = interpolate_spline.interpolate_spline(
+            train_points, train_values, query_points, order, reg_weight)
+
+        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
+        target_interpolation = np.array(target_interpolation)
+        with self.test_session() as sess:
+          interp_val = sess.run(interpolator)
+          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
+
+  def test_nd_linear_interpolation(self):
+    """Regression test for interpolation with N-D points."""
+
+    tp = _QuadraticPlusSinProblemND()
+    (query_points, _, train_points,
+     train_values) = tp.get_problem(dtype='float64')
+
+    for order in (1, 2, 3):
+      for reg_weight in (0, 0.01):
+        interpolator = interpolate_spline.interpolate_spline(
+            train_points, train_values, query_points, order, reg_weight)
+
+        target_interpolation = tp.HARDCODED_QUERY_VALUES[(order, reg_weight)]
+        target_interpolation = np.array(target_interpolation)
+        with self.test_session() as sess:
+          interp_val = sess.run(interpolator)
+          self.assertAllClose(interp_val[0, :, 0], target_interpolation)
+
+  def test_interpolation_gradient(self):
+    """Make sure that backprop can run. Correctness of gradients is assumed.
+
+    Here, we create a use a small 'training' set and a more densely-sampled
+    set of query points, for which we know the true value in advance. The goal
+    is to choose x locations for the training data such that interpolating using
+    this training data yields the best reconstruction for the function
+    values at the query points. The training data locations are optimized
+    iteratively using gradient descent.
+    """
+    tp = _QuadraticPlusSinProblemND()
+    (query_points, query_values, train_points,
+     train_values) = tp.get_problem(optimizable=True)
+
+    regularization = 0.001
+    for interpolation_order in (1, 2, 3, 4):
+      interpolator = interpolate_spline.interpolate_spline(
+          train_points, train_values, query_points, interpolation_order,
+          regularization)
+
+      loss = math_ops.reduce_mean(math_ops.square(query_values - interpolator))
+
+      optimizer = momentum.MomentumOptimizer(0.001, 0.9)
+      grad = gradients.gradients(loss, [train_points])
+      grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
+      opt_func = optimizer.apply_gradients(zip(grad, [train_points]))
+      init_op = variables.global_variables_initializer()
+
+      with self.test_session() as sess:
+        sess.run(init_op)
+        for _ in range(100):
+          sess.run([loss, opt_func])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
new file mode 100644
index 0000000000..0135c66e29
--- /dev/null
+++ b/tensorflow/contrib/image/python/kernel_tests/sparse_image_warp_test.py
@@ -0,0 +1,254 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for sparse_image_warp."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import sparse_image_warp
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import image_ops
+from tensorflow.python.ops import io_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.platform import test
+
+from tensorflow.python.training import momentum
+
+
+class SparseImageWarpTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    np.random.seed(0)
+
+  def testGetBoundaryLocations(self):
+    image_height = 11
+    image_width = 11
+    num_points_per_edge = 4
+    locs = sparse_image_warp._get_boundary_locations(image_height, image_width,
+                                                     num_points_per_edge)
+    num_points = locs.shape[0]
+    self.assertEqual(num_points, 4 + 4 * num_points_per_edge)
+    locs = [(locs[i, 0], locs[i, 1]) for i in range(num_points)]
+    for i in (0, image_height - 1):
+      for j in (0, image_width - 1):
+        self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+      for i in (2, 4, 6, 8):
+        for j in (0, image_width - 1):
+          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+      for i in (0, image_height - 1):
+        for j in (2, 4, 6, 8):
+          self.assertIn((i, j), locs, '{},{} not in the locations'.format(i, j))
+
+  def testGetGridLocations(self):
+    image_height = 5
+    image_width = 3
+    grid = sparse_image_warp._get_grid_locations(image_height, image_width)
+    for i in range(image_height):
+      for j in range(image_width):
+        self.assertEqual(grid[i, j, 0], i)
+        self.assertEqual(grid[i, j, 1], j)
+
+  def testZeroShift(self):
+    """Run assertZeroShift for various hyperparameters."""
+    for order in (1, 2):
+      for regularization in (0, 0.01):
+        for num_boundary_points in (0, 1):
+          self.assertZeroShift(order, regularization, num_boundary_points)
+
+  def assertZeroShift(self, order, regularization, num_boundary_points):
+    """Check that warping with zero displacements doesn't change the image."""
+    batch_size = 1
+    image_height = 4
+    image_width = 4
+    channels = 3
+
+    image = np.random.uniform(
+        size=[batch_size, image_height, image_width, channels])
+
+    input_image_op = constant_op.constant(np.float32(image))
+
+    control_point_locations = [[1., 1.], [2., 2.], [2., 1.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)))
+
+    control_point_displacements = np.zeros(
+        control_point_locations.shape.as_list())
+    control_point_displacements = constant_op.constant(
+        np.float32(control_point_displacements))
+
+    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
+        input_image_op,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        interpolation_order=order,
+        regularization_weight=regularization,
+        num_boundary_points=num_boundary_points)
+
+    with self.test_session() as sess:
+      warped_image, input_image, _ = sess.run(
+          [warped_image_op, input_image_op, flow_field])
+
+      self.assertAllClose(warped_image, input_image)
+
+  def testMoveSinglePixel(self):
+    """Run assertMoveSinglePixel for various hyperparameters and data types."""
+    for order in (1, 2):
+      for num_boundary_points in (1, 2):
+        for type_to_use in (dtypes.float32, dtypes.float64):
+          self.assertMoveSinglePixel(order, num_boundary_points, type_to_use)
+
+  def assertMoveSinglePixel(self, order, num_boundary_points, type_to_use):
+    """Move a single block in a small grid using warping."""
+    batch_size = 1
+    image_height = 7
+    image_width = 7
+    channels = 3
+
+    image = np.zeros([batch_size, image_height, image_width, channels])
+    image[:, 3, 3, :] = 1.0
+    input_image_op = constant_op.constant(image, dtype=type_to_use)
+
+    # Place a control point at the one white pixel.
+    control_point_locations = [[3., 3.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)),
+        dtype=type_to_use)
+    # Shift it one pixel to the right.
+    control_point_displacements = [[0., 1.0]]
+    control_point_displacements = constant_op.constant(
+        np.float32(np.expand_dims(control_point_displacements, 0)),
+        dtype=type_to_use)
+
+    (warped_image_op, flow_field) = sparse_image_warp.sparse_image_warp(
+        input_image_op,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        interpolation_order=order,
+        num_boundary_points=num_boundary_points)
+
+    with self.test_session() as sess:
+      warped_image, input_image, flow = sess.run(
+          [warped_image_op, input_image_op, flow_field])
+      # Check that it moved the pixel correctly.
+      self.assertAllClose(
+          warped_image[0, 4, 5, :],
+          input_image[0, 4, 4, :],
+          atol=1e-5,
+          rtol=1e-5)
+
+      # Test that there is no flow at the corners.
+      for i in (0, image_height - 1):
+        for j in (0, image_width - 1):
+          self.assertAllClose(
+              flow[0, i, j, :], np.zeros([2]), atol=1e-5, rtol=1e-5)
+
+  def load_image(self, image_file, sess):
+    image_op = image_ops.decode_png(
+        io_ops.read_file(image_file), dtype=dtypes.uint8, channels=4)[:, :, 0:3]
+    return sess.run(image_op)
+
+  def testSmileyFace(self):
+    """Check warping accuracy by comparing to hardcoded warped images."""
+
+    test_data_dir = test.test_src_dir_path('contrib/image/python/'
+                                           'kernel_tests/test_data/')
+    input_file = test_data_dir + 'Yellow_Smiley_Face.png'
+    with self.test_session() as sess:
+      input_image = self.load_image(input_file, sess)
+    control_points = np.asarray([[64, 59], [180 - 64, 59], [39, 111],
+                                 [180 - 39, 111], [90, 143], [58, 134],
+                                 [180 - 58, 134]])  # pyformat: disable
+    control_point_displacements = np.asarray(
+        [[-10.5, 10.5], [10.5, 10.5], [0, 0], [0, 0], [0, -10], [-20, 10.25],
+         [10, 10.75]])
+    control_points_op = constant_op.constant(
+        np.expand_dims(np.float32(control_points[:, [1, 0]]), 0))
+    control_point_displacements_op = constant_op.constant(
+        np.expand_dims(np.float32(control_point_displacements[:, [1, 0]]), 0))
+    float_image = np.expand_dims(np.float32(input_image) / 255, 0)
+    input_image_op = constant_op.constant(float_image)
+
+    for interpolation_order in (1, 2, 3):
+      for num_boundary_points in (0, 1, 4):
+        warp_op, _ = sparse_image_warp.sparse_image_warp(
+            input_image_op,
+            control_points_op,
+            control_points_op + control_point_displacements_op,
+            interpolation_order=interpolation_order,
+            num_boundary_points=num_boundary_points)
+        with self.test_session() as sess:
+          warped_image = sess.run(warp_op)
+          out_image = np.uint8(warped_image[0, :, :, :] * 255)
+          target_file = (
+              test_data_dir +
+              'Yellow_Smiley_Face_Warp-interp' + '-{}-clamp-{}.png'.format(
+                  interpolation_order, num_boundary_points))
+
+          target_image = self.load_image(target_file, sess)
+
+          # Check that the target_image and out_image difference is no
+          # bigger than 2 (on a scale of 0-255). Due to differences in
+          # floating point computation on different devices, the float
+          # output in warped_image may get rounded to a different int
+          # than that in the saved png file loaded into target_image.
+          self.assertAllClose(target_image, out_image, atol=2, rtol=1e-3)
+
+  def testThatBackpropRuns(self):
+    """Run optimization to ensure that gradients can be computed."""
+
+    batch_size = 1
+    image_height = 9
+    image_width = 12
+    image = variables.Variable(
+        np.float32(
+            np.random.uniform(size=[batch_size, image_height, image_width, 3])))
+    control_point_locations = [[3., 3.]]
+    control_point_locations = constant_op.constant(
+        np.float32(np.expand_dims(control_point_locations, 0)))
+    control_point_displacements = [[0.25, -0.5]]
+    control_point_displacements = constant_op.constant(
+        np.float32(np.expand_dims(control_point_displacements, 0)))
+    warped_image, _ = sparse_image_warp.sparse_image_warp(
+        image,
+        control_point_locations,
+        control_point_locations + control_point_displacements,
+        num_boundary_points=3)
+
+    loss = math_ops.reduce_mean(math_ops.abs(warped_image - image))
+    optimizer = momentum.MomentumOptimizer(0.001, 0.9)
+    grad = gradients.gradients(loss, [image])
+    grad, _ = clip_ops.clip_by_global_norm(grad, 1.0)
+    opt_func = optimizer.apply_gradients(zip(grad, [image]))
+    init_op = variables.global_variables_initializer()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(5):
+        sess.run([loss, opt_func])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face.png
new file mode 100644
index 0000000000000000000000000000000000000000..7e303881e213a82e412d18de9d9d86f368726f06
GIT binary patch
literal 14060
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}9Bd2>47O+4j2IXg*pj^6T^Rm@;DWu&Co?cG
zu$OrHy0YJ7<rh|!=$WbelYv2j!PCVtq+-t7xs??nuK$kRpLvq=NrQ@tq6}xagTWmM
zmYxQMu0<;zG<K|Jy0`n)sa3nReidK+8Wnpz_I#|qO@XMO0DnkGKp>yv!2(t%1$ISo
z&l83--`$s=x@`68)xTz)^St-`<?`LDS5@z-w);GH`>NHe8~I9CILp2YXt%nukZ-GS
zrPOjh(~VYon`(bDd|{e!Yq|e&MNZDSZY^zX-}w3YU&=^I3f)^7>=EJ7p|7OG6y(f&
zC^hn%i+gp7=cK^6dk;QVC{*v|-Fo%vr0x6mE!?(k+ro9r)B?hzU$39G<iNa1LjSk@
zsrm26z|Qc5kz>n~liN&78FJTnFf4B8duwHDXIB-oJui0Bq)EGuELrB(vV2)vnWx{k
z7IhVmj_%2VLQ0O)R2hsIIOY0z6c{4aUaQy^D9xV|x;-;vLO_IuqJ6wmUfzPV_xIXP
z&5^11E!?<&(V|1~5wWrVo&Ed{M={K1xU%KM)ePoW3{k6R9654A^5sj;>1wK_Z|Bdt
zwe|K4!@!MyfB#kr3QB2d)pXFB;Kg=Yh;_4FJmW0Z7*FNbPuK&awG-dRao*m>x@zCg
zWAU@*zt>;6BICy7>CgYihl<)U2VBevUomwxbB03ln;T15Yk#Tu+dZA~?qm8KLFwu9
z-hC{6sdVu1Ld7SIuX)m}j4lVxKDT)BUa>OHNh%AMxE$iQXL%L2jz{L?$xMw^tGH@5
zmaqS!)(}7YPuJF@xoaHwXIgChKA(;6jp@HL#?{CA<zxl#E@t1z$>njxXohQ|;FBtk
zpA%kResN;0OXgcE-aWhK&DBl5x~gsQ{Tg8n9ldGS-rsIdXMB@$Y4r`RQX|H@xo2iP
zu{JjP6>pRoH2=uQ$I6QyJ<8y?nyPAadXr0Iu1Mikk5#>|ju~VxeCnQT6*T$98U58d
z-jiDH#Bk>CRTZuO+#6HyMdO}*g}{H7fLkfy+eBC1Wu7}l#4z*Djz<1IX>)_Wdp-+2
zs{S^`mg_*3x8o&|A6II&HZAYHQx$#SnRFz#ScF)b;Dayy&eKlTRbGsqrkZn;>E+Vt
z6TaWhzY!YH(0=;#$5P?AcVgkytX3&6F6>}UK0fc$@0*+3zepLYtqHEM;9hrvee=X9
zZh^nb3EE|kYU-{m`|f){q%t?Ra;wg6{!Q|0b0;z~goYl7&R_WG@Ar-GPYBvn|1h}c
zV4WkBlkd>pD}O)x<jdvhXXYBGI}}yzG7n+A{+RJg-1Oc{w>r*bUsMo`;aU0WaHT=^
z;w{@=_T8Ot_UY5<*Vk1&rGK83$#^2w$nfE$`ijbru5ZqtKYr;>j1A+0du#O<EWH}n
zz<2hH&(F(KG?T^7%{E`)?$veN-62g`PkL%9n}gtkG`U%RkE6Ng**=Wq*>^ll*<!!Y
z%CA@7{1cs#Dmm+^e0|IRjmI}Uzo7Uh{ePVe^MZrD`ioVsK5LY-Hmkh&<Ye;uov&6M
z)ZANLU2AgW+M-Q`yEkQ7Z+V>)`M2MRW%ItPP9`508_T}^`SzyPpQ;5ncj_>|zP|X~
zN%f89_VWGm77f>%{slz^<chtxa^=P4v@;fOr><TvXPodMAzp$(!acR2HK01&Uu5Rj
z*CpB<>wAtL-Y)jX<Lviyi#+mD%#WO{`qcUQm|Lb*?o%(B>UWlZW=(FsA0Co0`SAK>
zu2;nx`{vqKKZ*YQ+<#WVF-gNfyZKjc{9@2}kQJ@oqtkAoB)wf*I+}Z5zggFTHRWlx
z8)h9hd%W@|<5{z+3!x2DQ|F1UT+cqQM#$anzfAn!U)o{|d+)a91UvLgPv4aB?ak(Q
z`#+s(JUQ2T^?|IltarN>em|2XV#+qV_n3*yqHSq!w>>(c-FIitiBsD$GU`LFDQywU
z&`x;%oVomNYwYf_hYSDy{<v&!{;H=}!W#VcJey@4cBkg^l!q^0CO*_UV$%>gq0-!I
z%UTJgcpavo-Y<V<%l!HKlF@vVP}t=o4x6WXM7~%$HC67;>FJ9fwTgc@d~NLv_60Y-
znO{HiY3mop8JUM#rmTLq^LZD4-tOF(s-z^3j^mFTk3VL8(0Kgu#WPbo5BNPhWyCNc
zmyv1XBz@!g8o{?G)}+SzhS!N?*(97^G5O3-o+|lQ`+Mr7BftO5nq?RN`*GUoW-sIU
z+EO+Wg3sr+|7+)ed}x=I$h7I(-`#$>R`SJ(oyEuX;x|1x6@A`HThq)Wgj<h4O)BAh
zXMS7jP3PK5oe6Sh#ZUb+c%eV{pW};#m%G_5_>*)O*c_-lzKC_^z3o+Uzy3;{efhM5
z<H4TK0@qgxtI6B-1oz7vl-hDhZC_ZXWl->9w%FpcJ)zxVbFb-Zonl@a9=6Y<Gb&xT
zZN;bKtG{&C7{~q;e)@WQ_`g>_?QX9)dG5rdj@d6>*enhc3_N-v=UmjpW_HE9WzyTy
z(jRYnl=Q~*k5aV5T0N)q)TRI4ov)ww|J<HR=U^qJJq%MaR=6aZ_V(_(#qoOKsgH})
z-{;50yX~%@=eqFptGAr`pH>OZn!kLjaGGHBHJQzJTG@+t@9Wu{AuG&$eof?yiPPI9
zuU^&A(LKs{kTp|qtJ(tF=`lusma@yusK4IAd3W*NIDY0$S;{Bout^8}&&g=Gn!5J>
z)hKQ&U;9dvfQ;|?W*`4-|5R!7<7Lq?hiu)pZ|nztWh__te1MH%KK~3W(Ys2@g=vnA
zLfu^-B%?Z`xGyAji~0U~^RRvW4g*tDBc2aM4q@t6N}CxjE4;p3S?M*4X^os$ZRJek
zjkizCs$KQ_!PXNQ2UE5`-1;PgF?{;Z1ncz@T?^P6dU#6gzJ~Ptcx%q1uW_JV!EU+h
zRjGz|Rp0M!-uM4>eA+xQ;m4mpMH!xWV%jjf_tGB?pOie^7bcU|_3zsASL(0Ek&Dy*
zD?faieelWFvKa@nV+@}xm1KPTk-^iG<LoJ?U+ecjI<v<{hdto<ZmY9P#aONM^o*WP
zbbpq==jk-#OUlZ7<!-5O7Mzv)Fk!JS%UW@UY}GCIdF;JDN9p!7ENy&J`}0@FnrTnn
zE7v}~P@VNlVvED)oX%qT+-YnK7cYL8SIigm{msFfKY#u>@!nmunst`9E?dd|x{H71
zD;|`;UcW0V<^6>K{VjqK@(Z{Ai`*(@vLPk%*^=g4Vccu9k~?~JTi4v|x9AsLzedcf
z?ueLc>b$cBY+L4<FmUj*r=4;5^zQDAu&_4&RjUK@3Lg0`6JxYeRxW&D`#t9C(>a#T
zK{a-)ai!~ePk3!vmKe5p71O3k%btC_mGV^h_R7jXD^9s3?45Hsi*L8aqHcw2a?9_f
zMX=xUoTW6M-{QH9%DY>wD^pW{EZQWb+9aG~WNLk(*I@nL4~HB*<JL26<kz~Q;wwEf
zb4O)hRBOoI6TdjtKFdyg;OhGM#c6MDHLcq>*`@cMRW`l-DKRx~+A%%mhU?sOYR!a-
ziVChxRQ|DP6_-1&|B+Nv?F)Z@AOEuZ_q)pYv$IS!PQDIh2z|@e7}<Nl=KdP?ncIZa
zgETc>7QHgkUEBNMq*O{?Ll%F9_O-)S2mBnK{jILFIq>9U$Njz1lfAq)>{_+zXVn{1
z=?iyvxqge;l=APFx4!HwC#B6?8^zMzcczM_SSfwZTh#G}O?K(mdo#~IW=>BLpQUAF
z{W~yY?bN><Z&i9@!=8L)3SO?1_`A&K<mtnoo~_cTn*G?B;mDox`%|U$_S-C8o@Jad
zW9!AJua`{j+<Lpk%q8Fd%Yn;htKUq2kf=8C%qhv-Z1*?MXUK|gHhu1XdiIo>ILpl@
z+Vh2lm~4LQ_{}w7oX^lvSYw&9!m@$y_vv`OxcaYG7gznR={mnuc?)}kKl6G!uk7b*
zmaAPll(R*zZQ}HFd5w*08eU!NcjD2T`&{hAqT04r#{YlR+fvh#EE^22`}r+c#&)at
ztf_GL|9{_u+gjSS3pBT{VKh6m>6`QGjeFHkWhie53|XeUQE<gUrz?9F+?pCVrA#>G
zV*1e|4FCU`pOY|M^-^9WBZ$BDr0U`SpI<C4*6--#{P@yZ;^kt)lj^#1Q9CZ~kmX*f
zwJYej{^LIXt=BJIlv4X8rFq>m_W4=Y({D_-K3=(a(F0i<iN9rU9DEWL?XFKRaQ9{V
z^5o>o@cZ@u|JW~G=GK<|{@2XIYks9aJ@q15m*JOFcJ`lh3Axz|qHN7;_lZtednn74
zpJCEuL8a3k=T9DNymV<<WxxW@20!0ksgmol;d^ZhlahLv^j^>YTF#O=-)Fmf_F^vX
zb#iH8*Y>~IW^QRK|9Afk*$HzU)_z%j?u5hgH#t2z8V6pz%9?U8)wH#}Z_l?|pTp)>
z=e@6J`2XFxJ3cX3r~O3Si|>y2U2QC%oZGo8kTK<R&ox`Q2M^mX=3h|IVHS9J$}^xy
z_Qh#^`OoJLwI=`mS#wOTx8tw&lIcGWuluOn5cni(ebIZHyDJ!Gt-Tx^A@Si#u*AHH
zjpDJ^oo+j`EdA_lpT1J{es=HH>hKDU1-)~AP2<T8xf;f>b>ECNm;G~?-|!!}rQ*n4
z>mA*8Eg^Ku>$CGWpDz2<b15n5!N;E$iofSQ+41wi+S!6uA78CLUtjX#g5N|IvpPk&
zmaNE?g|WZa%==X{Pf09IebX!V+l%J>O^)0s=cm)A@wQ7_z`Vvnj%`B1XO`9?-WS>H
z&;I%_$@`b;C9#I<^PAH3w>&<S$sQJXJ#|%2;FXPXGrPL`mCd&^ZQbg|Z?K*BLw0tm
zjD^C(7lHRS-mvyt{a{ukU!c_7e=inG7$mXW+fb4q_Jr%g<6}#8cD-D7Qu&0$nO`r~
zFs$H9X1!q6@T7a*vb03A&AeN_#3v*sioV<PIrz(~uOhq3>zP)rJg=nniZdlu_voh7
zo3oZLdVBH>`xCn#n|Sv8&eK)%({T<7;k<O|)Wit^jraDh-cV7vmmx1Z>(mP$|JdD3
z+~P{cy1e05b`igJ#NKefzI@)S{mB;>IiCCyYrE8Y#lO7y`_g4!ul1L?)7jzoENaG4
zW$DgkrrDbckBYuuv}aFPg;DVGZwEJ}8Vg8EbG`k*<aSsnV&<A@rc!<3cikf+nV+4}
zJa}vCBlUSn$+i{0-wOE5=X)lzrrUhhq^YH=Gbgz``}{oly88SFCM=fI;w}jOy5#M?
z*S7rKfs}7fneqFaZ?ly3@Q4N6yCL!BwW3Qx_u?nr`gupr%*>ZK#KSPZeqJ}H@V%o8
zo%fteOuKhwEk{m`US%bt&zy)H4r7KX(@$Icud$D-yZLkT>bwY^K&iRkUo7^Jw-V8}
zu)E;K%36|@^(FK3GheH{-r{eYXRqp2$gKIVEw`t2t6Y`#p(dV;h0E;2kBP^>IoNAn
z*V4w89sj0JR&~dRLsM6*Pv5&$^mfA0{nqUJ_r)Ds6M6K;#>4q5Yn+)2{(fbw`Pgdx
z?Qq83JgL3+7PA}VN}cQJJ<4&My-T5So?Y#i)57X*?kfeSf4!iSlB05!Ig7_w`sB;M
z4V(EyL}U)%+4){UV1xKlvwXWI>vs|Sb~RxOOeb%yy;JBeX_mk3J6op%!wl<iCr8H*
z0lNj~y1v|gx9!LllRu|FIx2OPz1WyMdHw$H_dc)p@%i%Wg<0y~27_CPY+v5a`O>`B
zOD;P&BB1hHGyBdK`#%qr*TmN!xM=zB%Vois0!OJylav;+_lF;R`jp)`^ZUlqY}s|Y
z&)pREy?WKbmDQkl+2XVQCo*?$4&0?QpTDEe?acc9j|wU+w^>@+Ef=&Z`SfICIlt5s
z?qGK5=e#XSpU&h+-S!dB@ZooenVFP$?LYUMd$l6#_wPIUP?KpB&z<u7{SOY8Et=xF
z@8$A{d*AL=pKi)%WsIv8y&-)hqvW8;dC8@he;TJNX<15kGqzY*y16@~r!D)Fd2i1_
z%~Opl=SAkLiun6JpSXo#i>STrouq2VZP(omb6u*ROw`x?oi;IS-I=4`-Yzv@Kc{hD
z@5`@0YooI_R(&muK7MZ1LdJ?O7Z;Qz?yfwiGxy%|N#)07U(BD?6+K60zVL#>%Y7s_
zZ}692@6WHj{;AV!zkFpWl{*qSfky*&$AmvW&Mv>>^R3t0@7+4QWX8cv_L%J3d%sSz
z?tilPzg>X%lN)Xa-O}Ua_Sz&e9jg9%h{>Yj&{4AskB@)fRex^}qwN;)yC?n&UYN2l
z{;2X@X<p~mxy@Ib*Y2OaBhA;WPqsKcWL5e7vzaoS_cIoys-L&nbb3?I_j{|$p2nOy
z-RN)6TW5bTis#<H7e)1dlz%N>_NZ+4@l^2_6AB+2JzKV<!f%gVZ{LYq8f#jk*1qVB
zHtU)F{EX6;Z=b`~{W<$?tH7%Rjb1tT<Cu&;ta-ZmrP9jVIk&^Q@5FpRCjS40^7S%S
zg&!PeyD!b0$?^84aQLBR)3>pHnV@*L&OH5GL&{aNjNBHfqg!4^WiNViQSj!;BfA$b
zdsqMY+2Jz3vmvw1wVN3_l~&I!kxKcTvA1&b?#J?Z_uibVd-E$r`M^}8lAGD<9QVuF
z8k|38c$4YZm&^WpYcD9|YcL8W>}UC>ees4zj?B`tie~-Swr;I^bEELcxs@|tJ)b|7
zW37FDvctiX&5rZzvhwe~(z{??#`1o#=*^EO-G1)Ts6A)d;o)<{x2*5D(S^=V!QiyB
zQg_<73Gcam^OntD-JIMMbEfInQ;z>$s38}1_y4)Mu8!ODHr(3QoBJ~1;j{#vm60K0
z`QJaEf48se_Ivg+vxCC5VKtlPYP|i-I6bE6-^Ps=mtPr1u>JaSnO#rP+Ds=%M6~k#
z=fw|&e?LjcVVD21*!`~jlI1&gh*fdil<z%ZXVLp4MN9Lc{a)c7uNWCO!-$67MPHsy
zS5~u>ERVJoTkTM8{r5{F^WnocrPaP{GRu<QKPRYhMsD}E$McU?K7DE$cK-gRQ>R`f
z-u*Adn4>;Pc<S`szw?<HQrFqfdnJ>w_;2Yu9v-QdMuuHcmTcY*{gRWnR2=JRVG(uP
z`}V@VP2sQSZoPeTtE5{w_s{=gqN4AZW8?goeVsL%Zd9>|y>IvNWh~iwda*e>_l6he
z79MBTiD<Z2`|ClP_VGl{7d<_U;$PorR)2|KvCD^z{Yho=)}C)u)6*Jm&#Wlto$d5x
z-_~hfIj(D53Ing?R~K<@Si!^4BgL8@Bcr$R)6cL5IjhW|`6{*tY!ZG7&dhzb`BPH)
zYr#nSqephww@AJGkk6(wSFP#pzUh6vrS)4nZ$w-#moz!Br&jZ+x}S?tgJat5nQ<#S
zg{$@D?Cq4iIuET$zxFG!R!+WP)46>SllJTGJ-1ozjUlUmv~*rs)V8Zzvn94X_}CEI
z=Q&YUTd8foX2ZLzlM}5bpRl{3>z`kKGu7k8efjOPYPY?Nozci#-P_+CtYoy{AKQ{+
zsh*SF<Tg)o*wv#|T*k!k_QST4U2#Un;nRLjP*K@oSH~$Nbm(CF<gJ!&7jLc;RqX06
zmVfc1;(9q}gG0s@<q7QDr+%va&17geBjel8wD7DKOFV5Kzts1zwquJHoYGR}^!04n
zyu+Jc&EJ~Y%wF#5(Xm^AM_SB#uDW#Z>{DG6HXUwn-&LizaN&Um2iLs|(%2o=e77&o
z{xOdVqeRk{xevF`n>lk!#+Mfz8E?fkjL&mlI`C-?+kK}Mf9Kw9oN=$-rA6Gi?TBQ&
z;cO{GC9xWXv)7upD=EA0;{I}B;ghwYzN)(!q$5SDel~Fn%%7E2qkTu<Mcm#==Y0I0
zxvCm&F}dNjKi&35n%&H~J$BRYif4Y5u-&!$;>w%XRs<Sea6aA~7+HS4YX|R_6BF0H
zJGZIg!L5}Fd{QhWFPl!frS>-TSw5X|<lP>=U%&L*+0}V18E?PHTe$V)3-_gG?f?G?
z+59>sGBa~-;j5Jw=VT{{EM&|+;Iu)$RLV?;W#!6uy|>B^vaWmUrJ??y>)YZpd*c*z
zdG9io2u>Aenma?HGA}GF$NXyC3h@}}*0TwZm+pSL*mbR(&@%t`y)QD#XT8_kyG8wo
zea^S}?c3j(SWPzRcR%*#=DU!R<mAT8%P}{nt~Tnr_oYAa7>|mpn^Ogw(}K&Fh4o}D
zE(kAT@Y&TS<(_(v!{b?h=teVV7QMOoVk=c!+N%prs@9%ezogSf?!l+i!3lh7IqPn3
zRtk@K$ai6Ly7}6Av)~QelK)h+e1Fr~?fR5QspL9i?5+<{CO6EIi+9}FDE&?OTWmQm
z|E{gc&(5Tzq-c~K)_66`HAQaAwY4WcJhYorUsv$oLQ(O+{C^@>^m0z>-gbPyzPfxt
zu=+<{v5VX9YU-}dmZ@cKF}Z&6Vj%yfZSAtsoogHNm-cWZ{oTgQ=JI&e>R-2arb$)W
z9AM_>X)@fCcCDGs#6tVigXW2jjvsg5XZ-ui^8P;6BiUa)yqsiMnwNGr$k~?Vg!EkV
z%FwraJ9lfX#S4u*wiQ$R+~3sQ$c>n=<#+vyg@GPEJzL7YO582-w&#%+<JB$MbAHKX
zM#BXMBPTvQT)X3?T6&$LCTGP5NB7xgJ6+6NuGCDQ+M37U_W9dU)~Xj5)?NPf{kv<%
zew&$}F6vCllHB!p;txY@^@%qkUHtv}4qRF)yl{{5#Z||8UUnUqw{QJysbtOQXZd~Y
ziOK$MNnbLmb>-~jKU|Auc{OcH@c%PePdr?{zB>B;%XwBSJG)<B<ZOHPZoU;eg<Vws
z&$BP*zijlKIjKcF&VpA_@!Gn3E0?<^|NOL(gPVEH!5lWDy*!gkleLA!)E*w@S2j%P
z*t*LxL*v6G?*mIbr*-Yt+x}<sCVd7Ee|9Bh!@lsg>d;-KuP4cfEn{`LeJWAcs;m0y
z&Fqa2{zt62C2AB~ool})s#Gm0xmP>v^0Lz7+wb?edwFU6n4f*r{j(0c{;bc}&aU<F
z>gwT@K4e(B%J9m@jR#hR9<kmoUTNd7+|R*<Wm4tOJ=%}FeclM2ZCd@M{Mpv93lFlY
zFLWyHu;}Vuo}j#~W%GHzBlB#ZCw+Y6_~P#F%bmh}ykC{HwdLOLHD{Y&GpTcig#4F=
zPYY5PZwj1}I=vTUp%2f8m&+GMY&+AwMu*>N<?PPA)y(C0S>N2eY;}9iTMv<k$w>m-
zeEav<H(fa{cly?rg2d+*sSjMAw$6Oh(DRD>1>@JL$KQBAam;nEUlRD<%J?K(@!}V1
zfyI)ls--JdeNr|w<Pj8%>sYvn$yonxM&!oAu<O0ej=5`F7AY<87FRfTH&4g0TlUMH
zoPd&xOL+S9P9K<CzUaZ$>jq*66n|~5xa%Gq9PGS%0{7B}l{GiN?7CyYAMkU*r!DUv
zUn>_a<<6MtY5s5bvW0!!+^#Wm`n{gJx(RMk`rw-SCW^m0Oht+5#pUB~c5UtH`LlWM
zLG_pGR=#}sf=Pjw=fbWDcR}+$Cr{2RxEHVV`_6*0yDvhy(ykroT3e81|KhRnW*Zyn
zge|?1BIQZ8H{QO9oH*I_?=M9yJGotc8+LKbogiRor0CdWq1Eh_tll}9|J){?x)ign
zYkTHzIy3Y0B8lDSnigzTy0&<;<e5bszRPFV>dii{UH{3W^HS|EhK(B~%BD!_MYES=
zWJK)WVexd)Cmr^==Vm8t;5#RCW!J^&$?O@%2hOf(&^W7i#_hB~%{*c4qE9Q``57AC
z?OV?GqNC%&IYmXTEw2`bZd$YUjpTafERki7rk~D*?J@cN^y7il>#lKEI5JYL@9mPE
zaaL*HRQZORzfP+6x0h_-;&c?5y`0&~=#seGue=q3Gh4%>HJ2PT?K-ok)PcvsKgVU+
zWmkvK3s{es9p-S@Y%}rb2C0p28)`l=t<})+v9H_0_u|8c2?bvtX`U@AO5Qy84#%z?
zsm*^F4*X$Y*jJvmc>BzVjF+kRc5!4m@C(gN6Ir{~C3CgiEuOfD2>Z$-b`8=}tKKhI
z!fd5;Mta@XlU^!ei?&|5A)_An_WYz}dv|_(%sVY{lg=c;3#abn%w*pbDI#+4dE4@`
zsCBicPhEca@S&JeNyZBG5ZQaDZ|_Jhbe+J!=R1qZ{I{9TtAd%9yLBdUBw3y1$S|I8
z@XXqOe`hnk+w$nfzqM^EdT)9C-g15Q^XbLOuG3DgR^HRQefv>S$HLQWv4$*@**-LG
z(>7l4vecMcY1W;NYhhY_X$H-k?|n-t_u<=I`?IF1!NQ@s=1FL7Sfbe09EPeJ2R+NR
zv;}8Pf0nW-OK?p?*0mOPN6TF~w<q6ZF!574{Y>zd`#kf<kvz4h+vJKkroHqyJKNey
z_FzopHZGHd{YMXHm543<m>|A&%7zQ&9)T<?g?>3iX)+WjZTXh#&k~np-xF<iVp~r1
zmboY1ERmh%XUrVMVyck$alz*|F0T``yWbdY&0T$QPEsS^m1|xLW<TXv;&o(MV%W5^
zi#Cfcc&U-6vfDx0HS<7ZcYE~3TV1vbKU*DHwM~pcbS`6F3!ljf@t`NuRsEtj?^E)q
z<7tp967#XX)_gZC#WrP`4u3?T6|eA&osvh=3_ibUk35q9yXdw(bHj1wySv|&IQ;(0
z*l&2oWW^rq)&{P%6VtcdPFe2pM7{Sw*fX2m2ffXkZ}&Yl=sFOlJ>{+QqJJ0szKe3a
z(C1^%IO)HuRV(+Do33=#ZN<|i)m!dVT*z5s7G-8DIyrUswZrd@G@XquPjbFd@-61p
z@*b;yvppy2xV*S^tIf*P>2tK%k=zRpLbzVm{PMWV78RMkc$V-2#?NB>+j{PP%UsOi
zUw`bRruKyER~^6avzEP3TFSdM>$yr%bL%>uNxM7TrrHZ$*f1f@HEfZQ^>fEF>ozNB
zpMG;kC+hQpZA=yHXVQ`ue)}DGdfG+C*kIum;rgrgr8-AThFnTzVc!yblkpZu!GWl5
z_E<lWnUMjFd^2Xdef|1jbEeGchwHO4CJ0EK`jsAdn(Zgk6|)Rhu3bFmvrO18q+M$k
zpX>TF?7*_ibE{tR&Gejha`jwkse+!fmu-VJJR56WUA?XZsDIp1nl|}zBF7w`gLQpV
zizgqfuM0}E+t2)L*_DTf85(MHcJaNKB=rCM&!15fdK38y*rse`YWv$Bw|bS-hfk;7
z142U5v#pa~WG;BqbUyF&y3hCK-!Mzi(_x9LlU%VT=cV#Rua6gZ?sUl1tV{b`>hS#e
z?{YZ{ffbB%_V`~oCmsLxfd8^?y;BF9SSu^u?M&+a9dN0+^G@1v^{#7gw&k*?G5olF
zzk!vzp<O=jrSd~<b@r)Cm;UJWecn5#B;ozNHZN17&GBN^E)xH{)o!y(JwLcVCwP)d
zO8R84-23|!)t<g((9!b#FHvFe$wAaL`&!?H+gts3rQM$8?_ZlAyNf4{lYv333A7eq
zgZ5^|ls9kOFRqJS!Bo(odzjfO|GZrN+E1>Z-nQ^aRj@`>2_1f{6IJ&+SM}4UKOG$(
z9xUo!#PWN~tH$0%`ur(kF&8CN(k{-r&UDGD^rGt$^XNRi4T;W|>!!utNR_;?U*44A
z(@h(>3yqh8>StP6Bs;LLuG87QZ2gzNkL6Vv8o2d)oJve$?wG%qKcV#{Y_X~S`ciEx
zsTr1&b?pC^oR)3pcz4QcO0%%&j>F+$AwRlby*d+Zc4VsKa^>z%%k%fGJjN~`qGWQV
zS=IYdPLA}MLYB*q4*oNUcyYUJnvUZ7jTQg@F6+M(rh59o$Hxw(rd$<2V-~Dl{ks0f
zX7@iwweH>Ei}<!(@$$xG-W?y0IWZhq<Z9^QFa9%K-T8}IX*c`sJKIbu3^rs4M#gR7
zSg~r;etvT+rHj9r8e(@h%#!+ZK>IMOmHP6B-MP1!LicE_X%C9`Oj+A$c;b;_NvdVT
z&(GW&lih8FglcyD|0i+x`OW!JtBx+tef976$?AjUZy27P=jZb=*J@+fkjky~`O^nc
zeSedrZ~ph|Uh9YM6Y&g+*5+97r)z=E>%_uayA)3g<?Vi}VfSZ)l%k^Cp5J*)j~?6h
zeN(NQzp#+IPtHqSS1$K>(uXM%81{VTt6Y1OU7n%*&O^T~si|yMwqM?b$1fB)kan-m
z#o_vsr$4<_J9AHNs1j~vJFu^R?Rx%%dwY^>KJ)C{{PW4=jWbL%&pi4T7Pmad)OLng
z|2FBVk-jsTZ2nn9{`-C3bIH2vUyHM)J}OlN3VJcr9#=W=z_xYjRNmcQUT0=K=ur%;
z=UKxm=sbOHZ2!s0-5W|WP4~n-$gl4{xORKlvzl!eZ*OqPH2QH|Un_t4(ZzpVQs?FE
z=B;=lc;GSPy^3asec97Ln&sW#E~{^Cb;#8%>pI2Jc<%i7{r?~L2kqrr_A`E0?f*4?
zQ>QZOL<Th`9`4$Z$o%uCx6p=?mxX?ni!=03aWC`loZFT3`)YLlN{4mv{(1XsJ~aHj
z{n$?-{8Y_%*{si>J~XbF6F7bQjBfqDKTm%?uiZOk;*Y&?TiNAottRdAInVgCQ`{_8
zvS7)UkYnHP|K3<tDtoVHGuzaue~h*D-u3;c__qDHwsvrF<b-q2PB)dBozKa-ucfD!
zU-(qhN@~X^uLpOE8w(#d<lWse&wK{wfzI$i521pyrYB<eN;xwY+>WVOza$`JV|~5z
zM4jp?O@Yb$6H1-y1vkdJUVYA$DtTkCb;I&`s~i&-ww&^kY;6_#_J(tgMMgjS3@cNf
z?f0ZQ7b!_zk6r$a=gb+MgZK8n>2|I-aOrhegIUuI-Q&zVYTZ+!7SFsM^Y6L+l0!ev
z7#|RiXZZh1e5Q?D!UG2dL&1(NCHp4(e}WPQ2WH+)sFz^pUnIaZgE4tQV4&a+D{JeN
zFAol~_<Xh1{H4C<DtpGkDT)(+?*G5`p!WI~wSOKQw3|~~CXu&8k%^gomHw1H@2s|-
z*;mW>=TEV18~<|t>pU58l44<He$84<Q77#yx0GrZd{|)STN!$%^0}b?-#?!Z9G7R_
zT|T+=R{Zg!E9cH>+9+*Z#&GrOk4r1oMC$)uSbk6P*VRWy?^PU)tJu5hD#L+|aw*%_
zT-DH;v>|7vyj(x~#fuI>lf;vcw<#$$)^q6Zk7;M+US*nDS8v`jC*hXX1)CL>uaz?V
z<0dd3c<=t0+aq-EH>rZFp?^LdJ$j(ue$uIESJsW3(_`I~w#BYqc!Njv?fQDxkH+T}
zxG#L)6rA$whDX}6@6$!KYn(gl-ITA?yytnb*TnO2b<W$_8@EhKJ@8uh`V!j@4lM6>
z`%4PlI1zN=Wmv<uBy&GM-AE^`x-BZ6YuP`hu!iiOtiouV_F%g2><Nh_dzQVwuB$ig
z+Vf|xqV=w<Te!z&b6mMs<VS8ZTRG0x*GjL+oc{a$deZMQ`!`#|PBfLS{%uj2cyCuI
z|L5)ZqndB(++X!H;MAMj>{F&alHQzb##(jX&gS<y>q~*6PEsCkW1S2S7%ksp=@S_#
z*CVym_DH92|Kl5m<@a=MGj56y6+O_xd27oVpW0a-ehZ$u7P)#)f8^oEH_!BUjCQ`m
z{hfJp=lJhmb1&=5R;CR%>!#^lmD-VfJVhrWKwZP%Jy|{Q&y%)@i8{>JW1OqDUab0C
z`uT0or#T@kQ$O9WZD(Ro+g`MWvG-Dt&i!4YlP*qH&$-W+e1W;CXpQZJ3G?{wl+X7s
zwmChG^+SLCmBVw{3;gr?&s@IjKe<kNi(Ey6$=;bKwRVK9X`l4%vi}4x?~P|pZH$)n
zUhsNN<c&{<xILEU_hqOTF74mDa?yUq#oPKmgv9gDDe98{{Vnr+)ln|_v~wxjABGCJ
zPCdrA@z)FG2Tj~tGGD$tpueAC;lf96*M$8#^Qm&`!B*~xyOUVkm^H-Z92V=iW-z4Q
zym=!wH1*5s%eMrLXD1%_Zg2IhcrI)EW^+3GhQefbkuq;-b0+clHU9Ctu1KUsrCq$4
z$FhDE)3-MV^EWUUJGj|zU(&`V)75uuP2l1y;h7(^cJFy{)~s>j^}`Db{g%Hztrgra
zXJjMCe{F5By|w^D*qRB22U(3y+|IXNP`WC1cf-oyuXBIM*G$-=EbYm%LbRe+$&%sE
z_x&32o7t{kzpi}#ROJn+eHlkMa(?|t?)midLPfJH%La?{I`=&q8D8ptxS4+I%M_6Z
zr}a<jUgx_RUS?)cX%uk4=88$J)rKn5pX+*`T#1u->Mith-*W%s({gVcoO-o0Vyg^8
z!y+Y{?P<I(+p|gn($sU+`=^~}-}6)L!s_W}e?KICYJGDr_j!&LOGo0nyR%D#w{ma3
zY7=O;_s>c76_!;Z=WkfbM>{-!{<!!5Kk?r$n~NV!b*T(n;OD_`ubThG70=mqKMV{q
zg*Mdg{`%SS=GIj1ooPB}=2}h6mXXijv-f7XwpRWTp4fLkpKtvCA#qRU+nkOZM&1Pv
zA2{y$`MCeX?7yWS_^!QU6?nPj@(BhFyJc(>=KD+iv;TYLp6A@2Uq6x+w7b9TT=4Ys
z#BA%bHD7Bjf99QC^tt^0!ha_O_izQR-mY_cs*|gz=#7dO0?*FOY-UmRKjSqeeM;Vv
zFC{{2!`EM1cK!O~^aoX67~J}{l;^2;&#RZ4Vcy^K;N+{F&pqy^8Q*NZ;dNl~>A(NJ
z-`HAfz0NI|nO8wTsOiJO=EAi1_p-hRD+j!F@L4|7(Kd}|rA^J@I}fL;Bs}%{6mC_j
zwL-2+eS21R<LmY8|K9)q5pcI$zTo-XXTJp_U01){W3%hbuF@61@Bjbc;mW#idv%{*
zgIlCzomGWEvEFW;yNq>H(o#i}%+B(>@|^5(v^DQs@RmyB&IiHECfMy%-(2=yZLL`L
zwGW@=>m+u3kNf<$qapRQ#m4Dg_vHUIypgpokj-&RKcIMlVabJnkka>ko7s4sEY>%O
z?pV9zV)4XHn;K4dB)pBPjZ0Mga7ug0qT*Yb&wgj+Pg41`<l7v}0}Gvh)%3GhJmO6F
znDn{!<16oyI0G@4lm8zuhpdZg-Q6d9*z9$>=iL6uy&}C5jqmM$am;-9@X4VIxtANh
znq_EbhJ882ec-G)Gw-f)Wx-x)_B~&O3{pBi7w383+SqLK^D}dKk7CjH_w)a=i0N(F
zX0Of57<TDK$x`9^T-}0$tT#54YU_M^dHHl^3me0gD+$@Pwt9067-Q%BN?B~OxOARd
zzgnzJ_<FmK(*k?>WSQb>1U0mjF3!sRm91}8HKpPqEBBXvKh0Oycq2A4&9J?F%dS^J
zbgJOe&Trp;Jl6R2r#R->_r)h}1-?w(e$#Jp7gx^DFTpia=f8i)IeV>i<}H&4yWbzN
zJuD@4;_w`1HU@t?&MQ|UZs$i|`1tsU9gl>=xxg>}Q|I|D`Zh)L+PCa)cddS|yQyg}
z|LNA&i0Ihf1@BZ?L>DqXI3c*kwt!)?-X4LE4fpge6e(LpcV7AUF{NbwotOvL<A0`R
zWuMgjGCf{xdxWu3LqT5dg0HV{7(Q6Jyy3JyyWO7&|KBfpaZ1~Brkw4Xx3}{)Y!qJl
za*uT0fpzkA2U;8c-EA%tF}c9-iivSb`Rg{#<#QI5#RNT>c60-GX@$j0%WK>9E6>E4
zl!TU=uJqaWfhi!4Z^_b$`yX#xQ2V>#xSaBYb#cDuT$jAa2&#|RbmSNRb-$1@silk$
z_I?l8m?bJUefsC)t2ZlOfAvS%E#a*7`-H0N>$Em_*)yn%I$X5co#R^lohdwq@z0mZ
zm67YG{}I)XaA5nBmm9cy+q5r{buU#vId4rmdaO@%vE0|!>3@sY9LS!M9;O+xY?D#K
zx15?ywbti0)|DjfyxB3e;G%29|M&ID3b!96ir-q+uxnxC9oF67<6b+5ySg(!pX03b
zcjt4axz@AuFN>^Nr?!37{rdT5-pc%M=4Ud$oAUnIBx!@quS=wCO%5Jvy|Fim)sH<O
z=`Y98qOGB3y1d1Cdqw}xvKBA&oBifR=|!X4cDd|t4!55?(ENUH`ujUdN*>%^VPZ3l
zj6H9c2d`Mh_Tyi55qrs9t4km6oe=+QlFnB-Ma%N$FEh6bUDM_s(mj1(_4-9mTE#ac
z+_%%&US5{8vu?JcGV}WX&+TX2-fh0eW<_)FF}KrYU*2v%F#o^LGd~e2uV3-}vR-Vu
zZ|=|ka{Ap(VGplM{O1m>+Bfgcmy@TJ=dSwpM$wkF{_oT~S3kdw`<3k*8F?W7KUa@@
ze!jK3!R6Cs4%1Q<)Y%(Zxj!7!-~XWe+_@?G*BjXcCQfbr>ODOvBtk;|#Wf~VhpK0H
ztJj;>=eo!47P>7ey!hW;bCVT|Ij2tD`MWatI1|6E%dd4eLW>W_#Pi2gX`1EUXy9b9
zh+Q&gwPu6c_jfPMYpv{R{}g^%tld#|pp<D-Y`pL5+4&9W^HqJL7Ja*YZP^5YPrtTC
z=WVo$W78IMh?^xlqq1^K`n1#sebyWDuWHQ+jayfLK4C-A;+uZUUu8`R4PUIF!!x;!
z*I9kBnz?$VO-Apwt&8kxJKpd4m1UImY@zcz_rUE}_S>&~vG*fe!P~7*UJC8r#Jo}C
zewN(dnd|p;JrdO}$aZA>ztsNVDo>>?iyj{Sc;hht<7pQc3nr#nyPS3lys@H5!e>j^
zmO|y1(R*twcY0o)^y)JE4AbsAHXAShoTo0hVUNvroemx5*=7sBz1#iq%(mPe+B=K|
zR(m$GH3Tl+@aF0C!ay&s8oz~R(PmzYpC8Qlo>#=2_BP6>s<yE3!*7!yCnkeJrGjgb
zDVvN=bkAJlbjG`wYn##2XG#T+dw<;bw?Ftq^3#JWPaK#zPM<lVp1-Hj)6nplsEhS!
zw@kNU^O7G1A3mQ?xwWm=<-;DEOCLQq-RQplT2)tRZ}nvM`gUICHh%SBudeFjSu94@
z<=i<{t5z^PJ9{|X{!hb71?O1PnX{NP7&<I}=hXZ?r7fYB^^@Ta>rIX{^Mh$mzfXR)
zL)raAQRviLj9NxQFD|v3{49O%m$1#Mp<DPS<IK72(Sb{%P1fy`vR@n9y^=vI#@^HI
zEUSrCx!a{JnLpm_d|r^{(P6ye?J~);H=lk}*`}0cR{m(Y>(4JwzB8NO?|D1FzHen{
zsKujqk)N)rtd+6(`FF*t6$g&*E_?W9W$?rU((e!Q7-+AR4!bIMP;9Htjk&u8=SoX8
z+|ECKj#s)ME>rW<U2E@TQ|Su~0ims3lZ0w^*4r=hnb}mKJmHPm<(&(+ahUPPRqeFQ
zW#|1ITdw$f_4<NBN5*e#_0z6hUE{#)5GvXyCZ7B@e}CeqEt$VOWO+{Hg|S6j-*Nj8
z<y)L&U6Hxw?2f->Zz8tyiHnLTSnt@pWqlgE6oZy_V9^(iYkxj*tIe-96S~QmyrwBf
z{6>OOtv9n@^T~DW%$qXu)O}~Zdesn}cW@i8bU~dX<F}jkrypEZa$t5SH9f^IcclIH
zw!}|QPI`!awa(Mqofz(0%j3qd`L!?O79;bf8FBjB%<J~c&Az_&@Xn=6Epp!K+zsz$
z<ekv<=-%Fn^v`KM()MzD7cD+%-1lbs@)MnRH|&<*ezb1di#rjl+$MW<W@(=;JD7Hz
zU&i4<oAigTyGm!g`@Z<fElIQ9D3KR#SD!vjd^t^5qUe;ShqqVLx>PfX-4ENA7A#h6
zohG`XGWCsvkG^&9wKOs3X^p<K1B#!|jo4SHoW{7}X7=sct!xVz9ZsD;UVVG};?>8S
z8zm(L_zWJX#Vo2d-9C3ilpD)BjrE#2?6!NqNp=1ISH5RQskYd`gBtF>3$HPmFf3TX
zG5_fkh4SlqJvN_xN|Lp^-@4{8oLy&bdf`h|tn7vg!*6S2I-|?qnbcp6DqeL}F*vGA
z_UF$ZKVGjlv@;g=G&KCMAwI>R)-1}ob<@s<Wz*j}=`mTUvp2eSZ%F-HHK+bZ!Q~Q0
zUbnS;84MP6GE3R{AARkWp7`qX!sjcOXX$oa?F!9qn7(`(%dF{XqA#w;yPtICf4J9o
zc0pk7T5JBTyVizrmMoU)>e^8AJ?}}*trkIHH3dDTLsn<D++n+Y>ff1DGv>D6cDQ`H
zEY~ekvf#AtjlF5Sd1ucZyMN(`?X#x9Y&I!|fVjT7H*Z#?A7)D_dDqk8*nH(+&I|q1
zdRxMN7ny`kDHmG$cFk_lnbUQhkJ^5JaC-gzV?R?<ZzR=S(On(JU$VGQT<p<VS?huq
zxAP63Y`H8TFS=BBL5<+SLpNj0mYRm}OihhTId{Ij=5g<hty!Y)&YU}D%P>P+b-Ulz
zpy`<_X7upb*r#%--7jR{rG1#SW7k}**I(|uGRlvglD6F^`@_XOCt`w`ZT^~c+5HvC
zE`N7~mqCXw`?A#)u|_tABS#WS?(eI3wrsY+mp`8cbFa;hirl{WOtkpYo4S{!qO2lT
zOxJgR`uY6h>33pc;=|hF&-8Ejy7Da3S`A&^a8>o>k8igx{(q#Cqx{pAQzto0_U_4(
zeKoI6s`}HX58IvF-|Tf;yLIJR=DRB<TnV{zXSKN4zmNSd=FjX5Sl*w!lW|L^{JS!5
z_T73+5i{$*%f6al!BqYE^T$8`{{Ap;Ff&r>jCjkM;hMN|<qlg*$<JbXF1NnC><D>h
z7PXi2a$JMhS!TwBr(Rd~eBvtVlR3Eh_xHz=41DZz*|k~u4%(lVE=h>{{%*(9N8K0S
zcWDb;-nsd&b&qeVxya63>z-$!p$BH?FM4yBe`0V%$9{2<2f14&{S(hywU?>ZY2(Hd
zUVg`Ho7oZ{G_k(0%@UDx5!!HFC~3;GB|H!KE~K3;w*Sil+RL#$oxe_7<G|t6)B+2e
z)Y}2O*=Ge^-(qs-&Q@)`nFfp}r^olT?5ln2eD{(=bYtaxyM&iPGB<Y!{&aP%u-E55
zzeuu8AcNOxX2UYm9rg8hgrvJW)8{lUnQ1I&Us3R3s_WvCi7o-RKc3p;)AlfSr_j;Y
z*B5WtBNPArc6&Y3jnwxUxAyskEy(13d0YJRu9AaGwwS2QE9RSYtlz!j`@46QS=O7f
z47s=)d8Gq(7pZdb%efS$s$P?`=`d$}Q}Vsz)-<bzYpc9dRF-{v(|L2Qb#g+G(4+Gf
z%y-@TTtcdL{faxhjNvxNlgGy!V|O?B&JOtc^}2?p-m#0Tv{Ifwe_VgOKUue7)dRb|
ze^)SPB)c+6++8hxuBXQ(Ek$Kga<bs$X>BLIl$5-@EgrbHdUD~53jUwBDp^*q>*IMi
zdpi4*CksApzaKDlcK57LMIkcweNDHbYUWRGU(fi5V>&x$zW*zRTl@AfR55tGy~ew|
z<aOIahF`yGI=Hx<cNn#{IDGpl@b3FzX=(1rX^mZ66;95ItV}ENGZ<dI4qWcj$e_je
zL_|E<+m&?_qYOiVL__PU<aVnodp<2tt|(c7*b659*Z$F-e%F>eDjOLX7#KWV{an^L
HB{Ts5wR(V`

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..7fd9e4e6d69f3120428d1d778846d495cea1a989
GIT binary patch
literal 18537
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3Fz|G745^rtlpw*nn1P*v
zfq~)we<{|*4C+IOdE|U2Z!?1cv!lKG#DB9{InVitat0sNyU+JyzGR)99iR84jLX+}
zR6RFvE@<tQWU8H~rr?++zT0=x?Yr;PGH1U&Rrzh-?{9bfYZyVkzjMivtHpq)_2H2v
zx=Yt_-TU9rbgflmYvs&V_ZdeI*!&Z=*=K)b;i=CuL6;;<-OfgOPYhZXRT}BEcGl-9
zmqeKMcW%(?oUmJXCr9f87n3%|b8pU6q%wTlB*DD*vCYQ!+tzK%&EP%W`1(-o;__;_
zEo__P>p$ML1Bd&SOSW7s1_BNb3(wZKDMZ|?_#fR_sueh?bHYsBZ4nXQn9BOY>-&!<
zp3!EJZ7TFTp0UhRzdJVE{JdD;(G`NBURp~T1^2MOzZoUzF3HB2z`*#U?ZkY}1NQ$P
zFE&+7QqpYoei-$;wm7?Ho^ATQZTsHODc-NA_UQ48t%o-6*aP;v)TGYl{{G(S)0^Ad
zoBR8n9UFrk8@-*Hf*l&>&YvIa(-Z2^F|)tFarSR};mh`J$Nozg_D<#PtoyU+|Aio)
zO~?NKU-;<%zrVl#{{H{{{r`QnzrX)9-lFsK%gg(3Z`bR`|11ChZ~yoA|Ns8^`Tpzc
z|NHBBODu#yi4m0Oj5?d=&yNk5;O)>59W%$<yF1*k&pl*HuuDsD&?H}vj)~o`?YX%P
zI~gYTs_rXPnO?ubm3x*^1yhr_)BguY&%D3?e_#Fo?@v$vm$&~{`}^Dbo16bH_y51Q
z>g)Z<>i_L)e*AxQ)PY@(51Ov8bT-eP9U3tsI%0-@&?I;F_V~z|;Z9A_UR}{HEuGWF
zFBvr`MQ}EnvI=MzZ|FL(Sf>6_!*Yf#EliJ-Jgxr!|NsB*@B97o|Lgz$`u_6r{@>sK
zzrXwY{VCD5@bwb+_ND&#khp+ZmJ@1MNhkBAIa4MGOqd{W;hNZ`Yhn{7KAbw`iI{Nm
z!>6eUiH_&LTWIkeI(qE?{~1&Hik7)wZVHo2Sew0}TY>YR>)Pgje}A*TWc>U4|NsAw
zkMp?q8N9iF{=*Bw!?VqK<n4|A{5tyKzy0C=|Cz<aA@-e_wvo@lL4dWr&FxW>z}Kmd
z{%iOCzt9zQe%&LXox1~q-{*ZzQ-AMX5E6Q?^w_N`nc0aNx3|6ceP@QrX_1N|XMx-;
z50ol<{{MTnVX~W4$GSzwwB%It^#mP4ZLTnV-!XBOO6Tp62R1j$mvvUAMcy#GCMwwX
znAuAYoP?!*965OYyq>zchQ5AYqT<ShiVEt-wG|H@JbwJdnKKQ$Cme|LGcZ~d%Kl$c
za^sT<3ueog39pwYT+{p?(0gcBl|U)iwnr?9xAxZ;TwZXpqmgkU)4#vTJL=~@Ji^(Q
zd|V=KU&@Zs?uVDv+v4|2{QI*o{tCp)QX7sOJaR-s{rK`lO1XJ@$w^9T+NZS@51u@H
z_{5nWeNzJ?ql9ej2kXN=Y%w-i&A%q=!fi2&NqjbQ4|a>Ov3=RnI;m%Zv=R@`{|B1P
zakKyYKmXw|`{BR8d3Ki@-`RP%;I$ZtV_$Dn^WU%E0+e9?|CeewqNsCb;S!Z)i<RF$
zQd+;}jJoE@lgE!AI&_FZGqx~9?8a)L8B9sDXGVw~@DeX6?AdW-y2XUliCTOWzYhB(
z{(or77bMBdt5k8Sq4@v*^on0o9v)(C;})05yPITD-uLixdfR+^$$x(rC;BjggLYB;
zqlA<c0nw*du7;Kr1Z>?fAt>aDm`F-`a<hq=M1Zn%lSILChBX)Y`!hW>{x`eZtl8z*
z9U@}7!<(a`vLWnJg1^j(T)sQ8(iMS?cYd^TI58Fli_8oA_dor|hsKBBa@)GaCHrKP
zLE-zexJ^GkLo^i<xQl`x2?#&lx;3=4__DdF=%gu6o;*oON^Crlpeem!s+fi$m$x=g
z#A*q*WBXSG@l+^@x-&NK&uLigt|?@2>3;l`i9%1Fv`Sbx>lQ6|%5Y7I#j^9yzq18j
zwGQ9i&C||jILFR+wVeUPi<OT80tL5hxl&Mib=%gdkDjL|rndT|DvI;)+z?`kn%lH@
zPJ)Ap*8hb?f)_g%@9(mm^-)FSk8JpoHiI-}=67;QUVZ<0cp_Gbrm*b^`1!H<;YH;(
z?Qn@c`E-kl2@gLeYw@3FhFFyN=)seeuiwr4It+|WL<B`r(vp%AT5h$4*%%}o(JoRs
znEFtYGp^`?md%CL{2sB*fsO|zE3A9cWWc*~*TGvBM_;fl4%fcXcKV-UdC5c(VLurj
z56>sB!`rT}m(07H6tSh_;k($j=jSE&)h_nqfh3qkb&nF#(xyy*zI|ih=PxGZrBhSW
z(-V@Kz8FoI%=B#=tIi|I3y<0u_ul--@5?3krR5Z-j#`45hoRN#8HN38<aA{hh)PFf
zE&c!h|3bcwg-a~|WHQa{zVrU?*Y1aRt=npUOKi(e2PK@d(poO65MQP}N=Qt6^5{`P
zK>#<8$=1!m6DK6NdJAvpaG$~(QLV8fm|4lIfHzMm++@X7uA-QvX#zW<R3fB>Up(b<
z%WPTNaPY;}MJpX&KQ=HBm}v1J#^TrC*$<z3w{6XqG|Nw`_|)?7Fnd?qIcO^J16d{{
z^yt!+DIdO?e*bV`!qlgUsja)_G&V9?`YNc3yJ&Vzsl8aFzRY6Up6)3<8(eJKbc((m
zopoh{it?5~zdc|1u03hGx2>TxY2y^12}c}jP9-?#x`NW@=k|v`3fr!(meh+&*-_4K
z#pBNgwbbd+gNKRNFHI^ay!i2h!L`f6X{o6#OV}c;8g|WO+^Xf+RhrVWl%Zkr4q?W$
z&Zhk~4#r|j7dEV9%T5*!KXSPD(2Lm$rxbHSIipr7X%#-m>;`51H<E|{{pGPPGniw2
z{KG$6%iK_?WhR9lo}3oe*}r~BL`G<IcWrXyZ50#cS@D&F*-gmNQ$S*~dO@MiPr2hK
zL`^nbl;YjdwsV4)xX!Z#ZQTo31big_C+2dmS)B4eK$zXAnX9w8vGL*W^0wXOl3{Ba
z&OQJ8$1~~)B&jIb7jD?NF(b>%-DSq^-LEw?IX7-JG&DHyTlvJvCPuNjJQ<lxuVXm3
z{bKe1+Njx~%2jBd_A%S#<hvWIM9(Z@)xCH?m~q_{#;tr?r#YHfUr>^h`!2wytbDFf
zV8<`}dI`Ilgoyp~5?_b4tqzy8t5r%25Cx|fCH2A$8w@->K14@l9XS#)Yu2gHA2%8r
z8X6cW^*F5yNYHF(Y-D_-@kW*9-Z_qcuO7%xaywJR6wc*vxk<A%Ec?%K)zb|Z_cTlw
z^C;|(3o%%*-8jIDZ;P2`)UN|teeHk#&wqHFzwK~4DDha-%>kt(dHas83`poEDH<jk
zeE6{8)vHsLRZ>5HNv&AbrJ&Be<jY5S%~}Ux_AKF+#>PfYF%zE+I})2z6lPfV7`!cb
zSMTQLHiN_HWQR%9p$08(L;h|tm9H5LzN&Xh)^th!`|o~EJVR|kW=hNR(v;Ny|NkF1
zV6=9)#;0i+5Zk23#4aPl<LAedl*DuC61cG6fBwTy<HXkxCn@O`dU$-eaXb6>&zTXC
zn%&)>4UCKo3>2=beF$Ol7vtH`@`-<%)Pq*WF0m^a4olc(SQJG#d%C%?g$TAt^*lM+
zAa-9uWnIXj2Cde2Y%1rK4*dVmv%#h3s*lP313Uj09@SIvJ+SnHI(J2B^Rn)lGadc=
z9UU4RothF4vn^UB0d<2};f9Tdr%!xfXTSa9$BXQ&uFoGf7#bNV9%pg;<|utN!D(&4
zj0qC|6AI5aq$ot4OLoz!yO1n$HEqquvmMr~3fc({svlPz&tfY3{{R2^j#&*GmNW0*
zC`x9U&aq6-ry<nKukYaP*^VJo5-%!&%F26t4{vv1fg}g5!h*t$85tffZdSj3zV!6?
zuxXQ#u@NW;4jOArYZAS9#lcTlpkbzhfop(~Sk_5Ke^m}vP;LnN+cZ^0^V-B*N8|Pd
zVr*<zUi|ql@9o*Vg!Q3&#U=+2;n?L5G}9y`C0kl0-@K7<agk8-O|mF%dw7_A&Qnk)
z<o|!AnCpcF8)nQ}rJ`a{QJwwy(<Wmhr%No#%F4o5tlZpkME}>bO_AeT_mB13yc6~^
z$?+$G{wJ*D_5RM-9_Lg2KlyUg-m9IyALAe8IEuTxeX(e3?!&jiE{_&WICAR8&&dy8
z2De=cYjbXwu&X;F_Z1q{b1e4k^_?{Nt*q?Sn>Qm9GMVPtJ?NC($;xo@=KqOFe6N38
zwU0<qJ|wu7xydA{aS>0s&w-mfM~(`6da_^En4-sTc3i;6uwL?l^{yA4%`+4O*LX7a
zE3o|idHBOiVdqfM!v~vr^6w{gRza<sV6kXX&ZNn?zP?W*BW5Ioata4+N@$a43fXUP
zfQkFufj8Ys{mF+O%h^Ug;LBj+(NuaYt$XUDyR@OF!2ZA^YJ3*rrCXZ0Tv^v0VR5hj
zZ?vcK@P+&5H&mM+-kkno@d}6~hi*Q5@Gy~|)9~!Mve0nx<kZ&d${*Ooh38FT4%`2y
z`KaoW{;r!2jixdI+O9baYekvZ|8HD%jB&#3?FGl`d5ljo{5SL{Vwfl!RkGXAO0B@r
z$4n|AElof?-NQ5S!GnYwJ10MU=se|{COE+ynq)C!hD2&A>z}{8#l^Nqj_!<%keGO<
z_s*e&maVD}b5i90HGeJoe>jM(DEYI!kE)Zv%B!9BH5-pHUYqu^o`*G~O<+qaAH&VW
zj)DfBx&$|yYfBi|S*rI0WCu*1*u~X6UHtI(_dLJ98{1SidB{UN=;SYR=1kA3RZ4Yr
zZk(LwgoI+woas<v%>2NxR<LVYjKWbJVdlf}OYB&S9y!RRrXP^z{r@nkKt(F4N}lDc
zdInP{gYu;Ee~Y=2P73|tW;PN0b641B>I}8nvy5tfAN%lD`f$HI&xf}&;g)^+<aF=e
zvBbnMF0Pmxt9V*!4cKCMg?s+;hZ!vRA!#hDkeTjqsas~&haJbIDpf?E9Z?aSw4{Um
z#L1=>t)3Z-JWnEyuVM{m{Ir5|lUQ_gn|-}R-Jb-D0)|`lkXD|Pr_7>7Hg$E&7A{ni
zl4AOC_jtpV;?@rjRMr{Vg{+O>uC&o?kZ}C-?7{U8`3q~i4)T4tqR%7Y%DRkWo*BDQ
zLR#8}O-<9})%}fXejR~0I;qjqfQzfh#U-MpRWc*XDb45KVWr>pRbB}R4yVpe_{Y)m
zmw#(T!*t0E)?@V{Z+TiIB6hMcuJmjv(`?|HDjxOAp)Y?;b8<%WDwg(m#fI+ggLmeD
zS{&l~5_<7z5IdUG_~hjH?Cl?Sa0r{43WkNTRcz*)X7u19XX;`1j)}k4W!U{c-XI~J
z>X6l#q-*dmxnr@af(e`4-V>X6ZY=r0d@Iy&$+W%~4B^%y;nht3P6tmlFc=!LRadw9
z&zHEjHyL6}Qer}afWU)ITS7cNHUtI>KY7x!gp)1emqVFMn##fIRrP`1;uVEW4lBI(
z2ZTK0Tl}Ag=YReGf4_f!muG9M&&lyQbLPZTfmzl6l8xsxKJ)X5^h)lw`SAVy|M>m?
z>i_?n-}}~{o&8CKAY0B)M(zWb6;cm`{hna;nTfHeh{w!~=lpzQs5=A&61Hy(^YYtv
z^{Qw}a&nV8caPFT`J8P$JIhb8<U1`|_q-vyaXHVLkL|`~_5c5Ue|Pt~b$Oh9&5wsi
zyU)+J|NrCncX?~;%^M8bySf^7?P}=Z;de`T$QAY`lds$-Vd_zxT^nZg$U3yG){p&F
z`|Qm6+uQ%|sr>x?dp=uO7+d1Q|EEs%xViPTwe`frs7Q2gt!up98o4lGn%E4jhYd4l
zI(l~>yf_h(oRThHnez4H#qAp>CM72~xT+NW7q~aiL9oU0qkPV}9>%RR8nvC*#awXu
zBR?r=O30^APp2mzmwR)w`1zHU>f7`7{{8gy`u_U=k`fX(Zq96Nu~@RS($usyCwHl`
z(iIEKnGda>7aZ5P=Agdm-oa$4Fw=+w-{1fDpZ{-X#>MT$&+GQq{Ct17z5YY3YEBN%
zp+h~NKJ_eH*5m2vbL$q*hSr-cj0e9oo=H8BQTafqk%5noEjSq3tV_zt=jY(PI(f>I
z2Hsxf^Q#S&dZ!yNERCG_YaNe7b3(_AW7{5yUspN0e}Db|xII5gPEPt9z3tA9^7nV|
z@4w+Zr@PxVYS!@!0c(~V@dyrXm_567;zxa*Q~?>a|NT~pZKrcjXmx&MvikY|KPY+Z
zt^1pPtVe%!c>Uh;_vg>g*Vt-ZWb*lgK}Lqbv**T@RYn|~f0b9p9k$xqrrB3o#rk@N
z)f{ecd6#5UxhcfEdwwTpu2pa6#ECN-8yPiI4HoiEOZJq^cqx9{De1=<j%f=2zP)|^
z^Rxe|)#dZ-{yw?A{r(T<mWdM^XU`4|oVb43u_I@F!yFn!qvs^LnawM42)gswkjty?
z@2~eSFW(mxecvbZ^XIRx_J8grCL|;zB?*Y8zIc)F<y)G7sH$S&fya|7*yeP5pWR>2
z^Z!3AeFla;O-yP!Kg-BaX_?)>L(-BP)-cRpmVUrYsZXdl!<02^gHpG1@yUMq{r+<*
zBEq-7esQtgh3)us{r&p!b=IY?o|@<HJ9=zYVWFY1v2nr`S(evQY&_c+zC4s?-xjD5
zR`d7Q{~H_SC#%=n)cknf&fgNu#>{+pe?5<WyirK#-{%h!K731i@Kn`JSR|p$Uqa$S
zw%D06Hs)=!SZ_K&b5NtXX+mAI!9T^XQIZ!XF>Tve%yuQKl~Fm&U@`~S<2N^t2QQbi
zFMK4(>3HbCfddB$UR*G)s|zzU<SQ?ijEfV?+1UM(?aY)@4XY+Gddkdfw125!GSB8`
z^4C}EXPZy1Id<yvbDrX71|3~S78V8u#*>$c7f<PFWaJiq?ZZ3Ei$fX`eTfb{Q3eWA
z<pokFsPwPMo)E;DG)r!$pQG%rBU0KCmyW-_9&eVvZ(r_h-k4sVKM#&{PCs*|=hiKz
zO`BqDY|dCH@t-y1S-}&d@-K*G>4)Zq`2GL(SADI1du#n{vxz(BR{pPJi;iY9HD!y8
z6e>w{)1A|;*?lcNG2uYa90LPThnM01|E6_+(o&A>QWeN8cqGqU@{m>P*Xa{K#J8C|
z)IS!)wx(NJpy&Vp|M@qIa&K>wwy&E|dFSfu>-Kqfer%}xd|2IopOX`V+oP451uGK|
z1RQ8kalP^6`}_a7H$HrQa#CJDep02(q5btd(c27ScDbBeY1`AIp}ahUXH8Rqt^nI6
z3rHw9=_dStctg5F^slS8p24wrm6lj;qhDWM{=T@FKkrV3YmtXS<MZ?Tbz>@izr3tJ
zA^1kY5t)OFni!pJnfdn4cwo<b7@UhLKBiw>q+k9npk&?cr>A+!-%qNP<1Ug(?yc})
z3lo-*fVO9wObiUWYrjlsELp&yBKzm)RF?@Qe~(5_aM-EtZ?~uJ@74GBm!u1;eE9fy
z{q61caeIILJJ@`_!kdFlCfA>B$)Pv$W}1IE`MT!W{H%R+<a@Vxecd0AiRE?wQY`A{
zJ$xDLbw(keXu--OvtpnL?NG#o@A8Z58pEwlq%s#}eiV<6ZQi=%;3T_q^XtDoIqBZj
z#K^WT{(r%LyOho6>T@pkOv@+;)tIsA#3P;^rG|I*{{DSu=ll@MD}O$vemb9UQ1oVn
zm*5&Th~Z5p9+C;VzGu|bPig5d@#7NqQ7#H+UB}~Z_>bT1UwXoi|Ig3wZ|A?izs|RH
zamK&Dzv~|#JI>B;XKc`s+#P!+PH2|(#BN@;B^}3Sosihjb!X4d-={=h%_*;cEygS?
z49ZYGJ}sijOlxYRw$$m&O4`A+!wq6{lB3_ho}Ld=7}w`LxU>I6CTp37bd3Oy_<^WO
zWl*}Ee}Dh~e}AnT74?%2?5_uni2izXv^;04%C#uA?5_tKqPem&8z((`cmMz2ABLrV
z6aW61`S371Xi&|<;>5+ioAQm!XFl@BoMeS05l20(2?8t!es8J2=p}fAE%1wL(u_VA
z)r*IJU0(kF|B8|+&;I}Ck+=I(_5YvxyG@N1E&<$|L<^!N=Gp%K{8G4d&*76NKm506
z<~~_Hb0Nd)9x3K+ElpcrbAzfVP##d{Qws_HXJoWUNAkD?OY(tfMiXvkgqo(gexBd>
z@cw-JdWG-lfvh#ZzWlzsd%ncS_G1dm`V@@5E|B=~^z{AX{r_wJc1`3x$>ik3V_!F6
zzJ-KDff(lu=F}ZsVvx{k3fG=+vG7NNICD_vp@wM|6TVG7%=-WEj)I0~dT(0Ug@u`|
zts|#unH@NB;=||X@rxEuV|4EpUE6YCvBb^U|L^br@13a5D#^h)(Z;EZJ3)-6LQASc
z#|7m2|NoO3eRuzF&o@m-`O_VE^niNggwy<NJzT8*CmVf~oTg2m9vU*GyV0v*_UxIR
zoe!U%-~Z-KkHk^7YgLTzY;2x#ZEBV*koeu-_z*N!{qOJZ^QTVzc+M}%#jmN!CMf8|
z+c-t|$7CjE<}y=6>E8I0S7=rb56^!^SB@a=W<kxJ3X?d3Ubfts?zmB;Gv&aE-2aji
z|F}6etXjo2;c1~xLw9%U)Tu9TZ@>TMO+me(Kv(seT>mX$+<I>{PTAG}JAUev$J#gR
z)}2#Obd+7(vBt~b_6BIU>0E8>sZZ`(kSza9DJg_G*F^Sp#!Gv)w)#7F&iwkdC@1F%
zXCNbEn55*n`2GJD9J_DNHfQ25dF6Ni6TO9hOy*0f@d%wdbI&|=H3Oq00ii`rmKm8%
zjJGY=uRSzkV}sW9PV#o{vUi)7NUsU}C!W0FkNmL}me*Jsjdq+n@gv^fKO#PU3fI8{
z2M(M*y=d`a_T4L56RIECXEizW=O|6$^Llyc^yvlb&VBl<F8WHu&#!HP0-K!NDc>fh
zCs!JHE}Y`u5?KxLOH!k`>&2+K9xv>fw{$YDO!{~D&4D$(8@5?qkjOY{%FNCloUq{2
z;dc4>y^CH538tmFm6fR}E7#@arCsrGO<o)-z$3vW@nO{w6Jake4}()D41(_&K7XFx
z(WR7Y%U;)TC--2?md5OT|NhrQ{gvdTt#yHI=l}ozyQS8gXgwHzMW;1PD`A>aePPDW
z=g-vAQ&l|^1w1?pGqOH?`e;~OV6f=eeftpaLQ&}*E*FkS7O2eq-_+D2B-C^4*cuTt
z>z*D7CudC_o)U)jb{}|FCxb?c82<lHa_ZJS!FuMy6Q=bs3ZFk2=%sbDgwHx~Rmri_
zI5zg+9vj=TvM#1juP$XBpIOb#g4?uM{pXAM&t~rDC=zm<{P*~Q0|!q0i1+Qid9<~6
z(rRuC3yDjYBy@B&IXNdWuA90R5_(Nv0tGj=n*HvQS!|$rCX96+OJih0Lgu1P8x5^2
zJeDrc=H%Eg@wsheq@S`fqq1_Hot;>ZhKn};zeE?~b)T5SE1O0CrY8JYzwB5_%8{I$
zGd=lkjOGdH=^b5399&ApC+37S#LP($)8*mefmoR26!olm=H~y8G}JFh>|psAVpwoY
zzT)$?ZPzr+!g_l?HFtJ8dUtPM;W2aWTtQ{o2Tl=EygM#)%}Zk2x$Ecg10U|6>atX8
zo7=M@DLtKoFYU#Pgal>dW&wtbi~VdHwn0bKlbraBmQ=CqU6GLThyC@tN9#>LoJd;6
z|3u@Dw$h=ns8!J+OAa4&bo!N$o?cWUa`sHflxb<tA0!+IG)M^3h!Ezw_KA73f}@(`
zg`l1t7K@viz5V#C)zvu-kDog?Vcxlb`11;0bL*tV8Y0Udyj<n51M0IsAC(z1i$o7K
zoZo!l*k!@hE)&_$U1V4{>5F}kHD9E_OqT`n_W$(aYL+j4d}(iW%X_`Y4->a-7WVX=
z`u(F|^V=zZ*t?C-%s6&`&jHnrt*wpI>;hasRm;am^DT?(bYp-0zO4Sn)XGReBqbv!
z#ls`v!NY?$&6dbKxYf~+&J1n1{`sgpud$KQ-O9m_uV=$8j$FG|#>Lv*k`fWZ>}kIm
zjQ;)qU;pyb?##=1I<Zy%Z*SkfgDE_0nz4q5k!h%3>|E(-M*oDpgexv_G55JNsMkL5
z(yssi=jZ<?C-3)4|M!{w?fJdE|971G_2faqhfhfwT1gwW9b4DX#qi%xeGRN){_|0p
zEpsXxvruPbl){lf#)DV-Ol1`|Z(K6<{{H{>c6==S_~`fZb9;B!{yx1r+`jJ54^T=y
zb7sY|V~<~Iipl6?W@>i4&Ei}0l3{78L&>j(1^aK9JThcI_4n)R|7T~{o9F!4oqBrz
z>+6#HY8uZS;^r{W&@jlzHa>FPc*cx?Vh?6}1vN*7&Co*p&qw81jH<B-YuzVw^QJV|
zM<$&8#3FpWKIPqf`~P-zf4)9EeBQqP-`}5~`&(NbgC@<LJzMf;UXoKPv!0n@D$l`F
z9R9b(4*vcB|Niam|K;sce*Bx;ICEy><jIZ^GaO^)gys6jG|V{Ccm`UZ{`siPWcKA`
z!{Qqa0sjrtk{%Y9zQ{Z9*d|0MT+8wCq0c7F+dPg=wfL}vzoOD1rPA3aD?!2OAOmv~
zCzG!Lq)Ptz|G#{YF_ZYo1{2lO4T^kQrKV5bp!uU;vCp*D!NtIuwV+VRo6qQ!P?D5P
zMwY~ckV88^yvljN_>9TXMDNoR8D?h4Oh{7W{432{-t+EEOKh3X(0TF30Ut3I@!ttX
z1xtC>ys4K6Vv&}~sGOc4^H|gPg9`T_H(s%Z{qMUpp+5Qf|G$3L$;Ley%&Yh^pFgfw
z`lS}}=Fl=B<4A?#iQe3g1P@BF*Sk7z@NSxUYeIir!hvNu1&mFcOp=hy(DdHs*cum!
z0B@!k?WzwuS!Q!wQ)21u4YN^LF43gP94vWIOygR_@x}8B>{+h{H;X^;*M?|JYHVfw
z*xAgumE&iB<c$X&VkaK`H$1X&n*5S!GCz{%F3|bdE7bHd`NJ}vN8J1nElw(Z|Nl!q
z_$#=id#S|4QlVQN2RZ~bB6daybEVB}3>DA~k~DBBaJ6C9?B$6OJkOB);Qc$kvW7Vv
zAC`cI#~J?rKNMADV3gp`b;R6ggVtdm;T<j#29G+IG_^$Sl~~hNC@S5&S4+{NtA8b^
z5Cb(0<_L9`M~bggy!c-tM<m)nV#g8#VU|rIb3XDrGtT>=tkErYkjLdqlELX2^()kP
z?oDB2JR%}-LN*HG;H1X;N7~HTx*~G~%+<fhn~P^_IeN3qXo@(^!`4%gs-Pr0LEWjQ
zx@-gU8g386>&p(`gy?Gu*XEg#8?@x<qKwR^_I=aX*DYx{c5GII!DaTehmE3!0uyy4
zIyCq<6mfd-9{Z;i){xE5{mhTm0@5owG}Xdj_B0-z7jp%-*l{znO=mnj;lD-7|77Kl
z;?gXF*B-I>PFT{?q<xns8e*MO-h@MoWh52&TMZLD79Q18c4IuyFuT`uVZt(gwkt0j
zA3ySxJaSqn{xr|tRV=~}vMLicL2_MF{*r(E;;y+d0+%lwka9bsBDlu&M7>PRnIj+h
zmFIof?)1ULw?kgu*{uEqgVJs-*gS#bSM_Nd^>b3$c3s`!x<O6QLOfisDY9<CQQLJ&
zjf~1xotl0pG(6hm4$P@i<$yXdyIA4%fi>~}lsl7s1$Icu%xGm#YjSw}NK5yFI^Q1-
zf29P4UExMg7@t)+rf8;^fwP!XcJTuL)rZoHBqbjnUKVkC)ee^pYOFqAXDnr9&PqvG
z#?!R)L_N>c9j|8nKf=3>@8^`pty>Q4su5s<+9|Er>S27S%A)7PY2OLAlp>lrbsv2c
zk>i*OSv_EHY^;1~<CPskEX;i?4o&7OP!(YdIm#k@<bX<d{eOvne-eNEpWoQqdnxpI
z#U&O!M-E7Sb($TWn4n;D>#O|AUkBp*A9yRhzxUU+;KAvm-S_wG{dwlhj)2mIsvK-R
z6^DddfBe?`arvNvmhQVdi8pplb#!PrczJSGy5R$+9YUa0LI3|Ztzk)auv*j3n9a}X
zDW>b2P?o_7nvVYQ;bD33<<H*J>o#mQ&dA*O>4Sly=I{Pha=N|-3P)8pDI82`tXB}-
z{6Ny9;EjZHxOnp<q2`&A%{wfvZqpISX;*^g(?daP1vchL7#A*B%FM<iDDn>E`Tze<
zpFDZu)F}m}Lq`sM;&<j<`IOz}(ZwSlr*rKPI?K@f|Gy*;ujIYG$v^(jKeJfPkXI#0
z09s%;tqd#BXK^tUR%@N9Fk$Ea!U;S)JpX6Tkno%H<Ja5U_N=UCdU|Ypeg8K+{U0IB
z!ko25(fGvd-ouO2HZ<JdFUi9LT0(GRbGu`3al##FMBPsPFMnI}>i-S0A--qqs$!y7
z|F^CE_4?f0`PbL|oiZt@tni`B)z+1QuO(mAPdM`b@SK+Ve*0^G_nw$1rPDBRqGRC1
zgC{y3{w!|0y<JkyPNi6cySUKOh}l;a+D%MKa(J`mf4|JNi1TH?{>$rL6llJ1uKw5e
z&(H6#4&Q%%_V<act%h1%?oLgD&bGG#9cOHs5%$1;&5z4p6uiU=6m-rcq#svMI<z5A
z_HZjV&;5N9XWA(+gx8DKJ3;bM(joaH|NlD+ANl`3gO$_sC37OE`qq#Ab?@@$e{XN^
zkBghb$CvZ}-@>JAe>8bsCmKae=3d9r62`(8_u%#Eg#7a#p79<&(#aFP&fw2q@4#OT
z%v+RL`k_PKNrx;ZJhl(o^rJf@VZs#mi&y?Ps&xPV|NrgX-&^<o{kyZ+J~T8kDQU-(
zCo^Wu_>gg==ly?=O((8$&D-3<R;}8^y<xNQk|hQbl7@5a&tJHIzTk_-;rI7dmhew9
zQ%bn|9x~C?<iF?ORl`H!^#}5G+|Af8r|{@iMSlXP6yrPl{#KuzwO-o1Zhzt9?Hde&
zqolUambSLGzBWx^7DrLcKlevMIYmitr}9W1ICUx^=gfpjhb~+@wxLw}@b-M3$VeVD
zGZh&X_YFA*;-PKXq{dLmj>IC9sXQ;Nm;PI<rFg;i=l}oz`{n<Cd;i}s|K4?3>pBO&
z`hS0ZzMpBl-^h5U!IorGuI&1j#wjgEM<%h#eE4j<r~dqhN34ghuji@$Ze&w?YQn^W
z51t=a(DI5BpT6f8sMPuY|InPNM>BcWtp6<^s(S9o$M5g|>&O3_YyCa@VAJ|`zP~je
zAHDCF4_LZp4Nq2<&;Gi<smX_T&1STGchO~o(vBn(v$UBeM-IHd&m(VV;Be^eZS%I`
z=MtMXfoAzVyplF-Nf8iIN-W?B4;Jy?1x*Ku4!iuN%8sxq|JmTEa;*MJ;OGAplUSJ#
zgSxQycYiPc`YJ#Bdfon_r{=d9MVXj-^yB^{Cm&AbU|DcgZAR<ydX>~2L1!+oy}xI8
zXWv-{%f`Ou=EnB+hi9eR-rkms+n2(@o$TS4_Tb5pA{VQV+-E;tw}$4R$T!TMEdT#I
zm!C8;FgTDc(~)UlHqWT};pgY|*Vp~Mx98{aHIec6_qE2&IM`9u^nboR&#PBH@%#Th
zewKD1O@ukiD&c_G>mL^l7-eL5s=qnhw68N@VF5MRZ0gPw{4j8ilsvr1l}AvJhmVhE
z)hZq)rWaM5$82XMndriM#ZD$36Wu)iD_@tIkuJaZnm*5mmaosx|1W?4Z*A1q+QV({
zpP&2t@596W9UP0Nh<u)H{(qjWb4XlI56`b(XC_VRnKLItli{w<p0&IcOO0(NvobRu
z{{Nq+{JoKfxAC4@Zqdr$(-v-;^ze5%TX1k&>S>8}@#!;6Iv*b4WD5^xla^*<Wo6UT
zoAdnO#EWMkk#Q&5mb=-3r}jos(}Kfq^>*#Jt7O5k?q8Cn>j!a*)Y8%~>>9Vc#ko!2
zUf7v`n*F5Er+X9b%#__{qr@>M`iW{(>ipD70omI9k-LRt3*TJb>NRQhwYb}PF5<Ql
zEHl)^+isLU|NY|u-^0S|D-wh!ehyDKW|1_{KR8>1HBE%I$1!x_>jd34M&D)~=Xr~D
z5_v9(1gfmw8$bWQg|<JWa*KJ)njh-$NbX*7<5km)%txR|0ZoH^dh-5gH)xGs>5mWV
zkN5xg@-S#^-^|Xw`Sb~ckDrb8qP{%7vGKl>Q_uT*ieKvfo)Qo^aOlJj`Ni_xGk<el
zQ{y?jW{U&c{r$#&{+<2sPIk)gce2eJ4Ywpbc#vRVk`fS{{NZbA#O`T{*Y(<@%|Y`=
z5xb{3#>{baYjX^p>gd|)=-PVl#tg>@iKjvGXB-V;I3exKJ9F1^H77{4&hN}HsBsnK
znYQ)rfdd+B`UhY1N~xa^5H!*0x?!sMFE??AyjjHkm-@@+d}x_o{H|J?|LwNfuTQ@g
z-;>OKVO#dBbIUK6*!jB}nx9RQ2ohMaZ@o<arc}>OJ|e!g`>pz9T=Ok<-S=MAm1@Xy
z`r5Hyo)R@IP5CE(r1vVPb_$fpHY^Ibe&d@kgGujU#>+=aJ%t|4(0N?6aMg+87KiJc
z&NlqF|2_OV)iMtp!gppFay2^$xbAl|%Vgo+C~)M`A?-JsHm$wWuejMXwDvxD*L60E
zBdjyIt3$=BsKLkfu8rNEhxuB&zwawEtW-~)cYUAwQZdC9xzl&mg<Os0*cJD%GEF3Q
z)0=53XTJL$`><=ywIesOZo386OgJd9WQoBMlZmOaQJv344vJg+tM5=yJyxj9>>!}f
z*Pq?Sc%Ak61DQ;&r4ycKNR`O8F-9%cd1xUmlV1DxFH1EOc-8AHTdw8+ftKUT9R(aW
zh>IMVuN)Ck5!auzoLwNgQq13_^T=J3PEVdmcG0}iXQur&JhS*s@s1tEFD!~~=cQKW
z&RYB4Q!}h<S?6@tzQs>>28vAAF}@VCJ=gd8%a`YtopdUQy+6rikMZ$cN`0=^110MA
z_Zt^_SpTb^;mD#H7`pzc&EfBkC9`rP#F`k^btfJ$aPZP#J?|`1Yw`Y(df6TD;?yUx
zyE$6|1m@hWZTu+Q!{fqZVjys2-d4ViY}^|d3<cjmS=)I+R!i)`6$73#>}loN&v`<o
zGQ8cDAFJ>CIkD}1ut34ASvr$f8M!JJ)|lUC+I()aws+?3tDg&w#4F8E>s69soBq7U
zm32?BR-Ge@;FrJmlPe{XWaQ<ldK$NG5c6bWSXCYSJz~v+E<H#~xoO^;&+;$#7~EKO
z5H!O2_~Fm|j1wxlXZ2K$sHh#&&^xE0cTPq7^pW%DIo3NWgegQHKXKy2+uQYJ-~Oxn
z|9kkrA+z`2-_r`3Cl8$XF@NSy`<zC8=01~y227J#Wk3brcfC`QQ_h^>xpnJ=GM@!w
zk&Ln7L^I$1IWCY=w<+crv%b~?F=HcxKhGZ}UAUGuWlGA!2MJG}rapO`oRHMiUy!g(
zCQxiL7q{`9-QUAqS{~js7TNB=^YGcf<7du%c*?EJ&%e&?SO?qwI>SA6XFt3WJ-j+R
z05o|U7|52F2O15&v2pJ1e#xg5Ts%p~j{p0g4z*cMf<3eSkh`nmQVwnH<_3nv3lx}{
znO&wa>T#Ph-ZV)$_WJt&B}*me+5Od5Zho=S*uY`>?f-vHe)#>p-ptHq?v06jar@IO
z3Yi2e-@M_`(&G95KiOZB$HMHQvB3g$rWmdbUXVhx>D;M9(vlw*`v3T+`h$c0m4Mcv
zf{xo0j@JMGcmC|z4?jNIGctmf+|8IN`QZEIn%~C`9Q*eFv^ZN?8E9fXBJzK-aq*I7
z^L&XuxzrzDxdjt#Y(TSQD^@+a(zLURdF8|gr6&7Y1#Qrf-T(iGV%{{{^tbM8`XK*i
zO`*wAjZ+bk|MypYoj!f~!*6f@uUW%$!d$$t&@g8Au>)t%e)#nCzkU6L_lsp%13;@q
z54ZF5%Ng3#^$7g=+<x#t!^8XbOrng-I5sJKV&Ryj6P66|lGFZU-1DRqcp`!t?(c0{
z_Q1Z#t7-Yv`v3n<o-nYf|L5=Y=)d@tZo|4i2@|HLSA2bC&wu{^|NoM8|JAhixbgk?
z-v98rew+V%3BNfB4*R?V{Q4XN1mf+*_zZ%X4=&SKw~8HH_#`z=6*%#fMQPUm|Nq&&
zB{Mo7$X|M@He>1O|K|DsI=lYWf9YtPFMm{X%^IHl^~N=SPks2NdsWPpPyN7=BOhK0
zAMTUoxxY_w+UJ5IqZ6l%KYVzwC!X<ohlQaj2Xj_uQx2$$^Z&n7TrJ<;WiS8#|L?K#
zi9Pc#-?;x1UrFg4Wn(_P-@g9e#}9w1|Nh#)!y>7L`BBIIddaxGNeL+_760Zuyspot
zrp9*a{Vf^42any`Hm6IzyPI6`TTIYMMuw-R=E%FagI9kXh-!Y|>5Qo0cOHBFzkbRl
zMom+P)1psO8Y9mfxVdJ{|9g9Wo<Dy4!$;*`Q}~62na#})uZ`x3iQ!ovXIS&M>qSR@
zle|4>Np{7Tj)z||+veK~KKlRPSVF>J!=_1M0WM<FCtO*hq9Boz)L8EOKl#kj|L$}4
zA7Hp&$p3z`Lhi2Pk2owX+vM#fT3Rp8{qbJ@@N$2iH*a{pzcbX(HU=%fe#|~s&-n=d
zy}ijh%6cEZ4rjBp%#c$}+tJze;C(!sn_HWCzQngTi9ddcu6&f96cCUwVUkk8N%47g
zI?UTXH8V!Pa=Z*3mpdrp6s^o5-2N?p;;ZHVEAISyE8RRp;_&b97tBQN$kzWe`1AKP
zD6w^mgH|JSbSMb1+bTak&i}&U+{?>6udf@wxpVTvGv1P}IdkVWwzeL;KVQ)jw65&o
z`*^k+MW(ly($d<j%O&_^5-NU)yt3g)NJ^S8DJdW%B_Z|bIzi*e2U7nIO|`qgCB*UT
z$3#e&Y?-r=tHD9QCH{h%M%lp{Pl5f%<AwjQtlpnsD3NohQNpL(af^gsz?H)(_cy%Z
zvEFR3=5vYap1>zx{g?9k8JAyI<$1sE@(Q2J3M_{m1SY6t)TSu@f7Tx4BYgUhMbPyS
z(V$JaYhG)9c(zB2>(nG2<@27wHS;b0Z8UW7`ZaIc($_oO@@)API~Mit*vXp~{qgy%
zndxQVB)(?uMy_TDo;G_Mc6kRYtsRz~oBbbT-+!Rck=Hc2`9alo*?C^=UeT+AG|q}R
zG1>p&<ZKl9?r}`h`M20+&2SehzGDw|-N~!jlRhuKN0H-&)uN6<7mi6R$pTF0tJmh$
zO;bNnni^Xuvu5ck3AQPVay7Y56&<cL<?|PAnBw6%Ytr*0hKU~2as%rv>*`NzS|Ab?
zeBFfi+Pi?CcAiboW3N37fBf5~u*CYT`r4P(@4yaTGRK;)$w8paeiA3I+X@eh#v{fR
zJb5P+-@I92zG35*s1-U(9GUYHH{axNOn!9#+~OJERVGht-KGEaaq#g-*GUtWbxcSt
zx!aSXTYKJQp{bIKpzq^PMKU+@HcnG;ICArZ|C;Xj-E#Lrzgs7Ea7^dkE<Ky~SU~8U
z^F7MfwWExAj1R^y5b@f0d*l7!X)=5wt(WDG{}0=p`RM19j}J-~t;>uSoBpfv-M1T|
zd$zxcf7bcF@sVWD!-^-YptixB+55Sg5=4&Hch&t=cj2%zkZze4w_y2;n^g;zJ!31(
zC_3fKImKm*Lq}WFD{bjrmuJ2!*eP>v=bUT}8G|*JY%iZ=$FA==?z8;LRw-NdQ<EMG
zo>%s~P<m*Bje3_r)2c(?cg6@XU0?cogHFqW%ni9b&C47w7s&Xj-<qbdK*K`Dz}HRr
z(F~o%?=`(Awdfo!w3t7C;>`WpzMjELQZ^^<P1~BZ@jz*$-o#1UQjID(*4*FpvGhu8
zduC8*&h|Ioj~Jdh8{qY<IBm1}<>e-S_6pBG_C!=$?J3{8{{p*Z&A|~fXO=DZAqSo|
z`R+##BCjVs?BdhoitYL@A2{LBwUx~m4mz*3SfN{WXl3)G?M-btCnxYvGFQ}$`x^K9
z)AF)UF4J0bm08w#%h=D>X36ZjR$}$#_m$Tce1R^TJ5wh<uaLW~``b2XQ^baw3aXzi
zSBv%?E;*IrwDnY;yUgYoB^ED3sn`Y`&pf%c*WVxhB|rE45-&m59eQiUjw|2Qb8Ij)
zk=k;5iDu9u({|>mr+&7y7)m&=Y`He&bc$r8+uCCZ4M8o6hZQUTH5_VX+SdENzUEW)
z?61`?Ur+j!8n*XSh0prqImb9bQMhK7A$L=NNUOb^63e#@2UHcA7B19Osi@x_|3G5O
zYHw#3g^ZUI_9!eo*d`aKcJabrX7P8HHFubgSA~9GY4tYY(y^2JI-*S<CC+EhW<C@9
zYL-Tlo%;EmF*E!k*9tT}obXui`qJx<3l`irzZmY7GyARB=|hDljLxMmc~a$?9jt!i
ztySLl@0A`imMzijJDk=mpd)6&yI3&s#M`#_#<L5p1X<e3CUgF_o##F;_&N`p&h!jl
znb~3Sk^6EprKUKo%)jrJ9`W|bNp&Zy-XIYb&ex`)M`c02S~A0tyU9VIb^q5*?9Hi;
zI-I*$oZRcT?O(K7ou5ti7kgEPq2R%omk-oL0-X-Wv?b~t{%~*hc8}s3ou907i5?;w
z;})A9+q0!*L1squ6s7YQrOq$bxbgOdRi1fM!>%-m54Al9KTo)ncp&A#XO^~`jen14
z-p#pi>*T{-W%_3h&FFkk;o{Yz;A<)KqB>0b(Tvwq|H>&!xNnPI#kEw!d-+*?@ik9t
zp4HVqzi*-Dd8f$k*mwS?yTfXwXI_f(Q{QxE?n`EnALhi^a<>@p)H*w}7rshzEV%4>
z!Tw!;K}msD-l82GO&tq59uypIRhoC=ZmAY;Yt6?=)18h!Kbt;b$A->y4o@K$!A)=C
zjy^tM(#TPJUN+<Mhu?MA4K*Xe%Vy4%xV7OgJCBpWZZ#(vWuAw>1GxGQ&$r5xZeuLv
zO%4cENc!~k>?SX%!?QQF1f3|}q1SR?lLE(uZ5O0|_f@t{k#Ohxe^W{GVWmfK#^24o
z+a(L8#RVLf+vi;zSkV0q+-FESc<X_}GP%a>DIZn%PMk6;E^eEwe(|x;p}*hxr55~8
zG~V*TOFT{}U}@QdhmB@G_SdHy7=XKh;pqWEP48UO($W%=nv^=O*8elGsXkWll<V-v
z$2`x^J3s#V)VpX++nexa75%g{(6ZziGbP{MbLHVTGEUZ9@$X-1LHY5@<F@md7_`M#
zGnpK`zhAOXF4dw+Bycyh)|$1JtJOikjlY9StygTer>@%KgW(@Nd(|##vCVhlsXpAJ
z^pEwSpH#$+#;4KGY7XCV_|R>%&P40u68`(@z9)^^9-etv&s(_v*n`R=hK>!Xf^90C
zmFv0%4|T5Ve!PM)?edFVn{-?RQ$1rZn`~M8>tW@x_gjy&yQy8eZ{J|^AX9|phFPd<
z;GWp8ERx4-;`w*ynjcqAo}sp@ukeZ_{|9iyuYowRb-$@*Q<BJ$G?DL&EcX-Suc{kx
zOV=>1K5KDsnfaO+$8Y~w{AY7cw&OEy-&xjY-}qp2cSY#5ExOn4HScHtdd!65L~$Rp
z;47)Mnp}q_=qz4!VU~s6#(-1v-Ly+LJyQBytKMb6mOMl4nWW&f*1{uElUOFUuUA}d
zvh;Pvto!d5-52LuUwKDuN73yU)oqQXdw=PJi^Mgvb@^Hxcv|->GKz3=GWqXtG@a)B
zV5fSLNQR17;}iGIO`MEvb5ka-&6@T6)|T3{s~2v1rDR*~a_7PIUmeM(gw{&7?H6?y
zkVu=cY}rHkD+*l!T{_M!HtDx5;$ELSezErW=e$XI+hffg1zjZEr?u`{mV0qlW>V}+
zt7nq;&8xT9UjKRW(1b+`R(VWXFJb+9+kdO1E3u{($7Z+ftB-5813NLP<~yHZwUa1Y
z*oSAl&R$%HkM&xJF4)2OjM2<2Lr`6bNtU@mXVc%0&5q&I9i5sUKKExcG-P9DW@BY-
z(~n=URYF}Iw6?dW$7If%)-^7PiHV>UL{Gih($d-{t4sRLO-e{jt@tzJ;W=4QXXyEP
z$$x*76Ox^0)~IvLm?5!dt>mj$5^vsYP)|B@i)C}s!Nm-|fBx6Mu&IodJ-jZqqjo1~
zqK@JJ|21>GA?=joUg26I{R>`*+Blwn<Z9zw&E>yHditf*!iNp-j!fi>J>#&jtK{OF
zPYMD;|5$_`N&K^T8vSr%mh}18M;{;jd+zNqn|i;#vjqw)>$-!=EGA6SNsbg~GTHh=
z{<w?a>#XosY%P;o47Tvq|DW=;e3qJ@`jkbP1+l;8?Q+vxvLttB%nK`q<B1X<YL89)
zJTrBpHq+XVHe1v<P84@Fy)NDVS~mRHx`U0^VwPn2Mh1o2%)j68oefkv?U`-J)#AY8
zI)84f5=RuLlE#hb3cvLa3~gG&_#$p}%#Gq%Y!H#NG5F=XO*tD^2B|Fh@o@5df$Ye*
z#dYQTIv@JhGdw@J>r!cms8`{<9_7T2TxHe3miL?AFVS>NP>G+mF_(ogGL6AQsJQ3H
zdC9-*mv7}*{E|EEl-+XRvp`eju|f$p<@lvXN@uD)l2{6I>n4-)S>?MK{>Zf37az8L
z?IeBN@p8cRExJK*+K^;%WzI&f1_K_K`#~>M_#Oqlc>XK9u5od9^zIvF1p!=qXHx=8
zzMPn1di?1FrxqC-zq$Q?9>h5{%$cbcwCMfJ^pjy;B}ux93rvq~SeL}pUB70&^k1gq
zjX^ommfWJV*q*!SBp->|v;1{N^u!5LTc6fMXyngY`fYA%qJ)I|&OJLHuz<oWspfNA
zfuX}BCKDm?AHT(|9`wq5uc)#;%rEdhvPqOhjhSD%FK$o542y}0r^Q|*<@-Q-)k#j*
z&Nr+TIi%X>%_!B$ZD8!&+}6q(sKt|W>cIPiRm`uJ@uVF6p)Jnw`>A)^*VmHY-X>Rk
z=c_)IkdOd1z>!<9V)+;Qtn=(Q)pv;eIWCbb+oH<2tp4MPB&`L;2SOJbBrM^1qQn?E
zjWIf|_V+Z<rmEcA5^?*E%rt^8&^u(gok={=MV8aGuXN2z=3v#s3pkTZ1lKSNorrU|
z>}?)@pqVe5E8)OgqX)MNL$!H$c;fdP#q0);xSyS^(H&|CDS?w3dz6d?rt=kaE^ajn
zi?nLdDROf8Y&=hiG5ke>8h1=v!1eO~f!4}AEtgor*qHmKeEh6_`0Q++=<P-__I?$N
zB9L5O`EQP6l+-PzAOGc5MLIm?GZ<DYF!#Oc6l!=Tu_ow;H*3-*mawA{n_1E-o%c+?
z$Rw=$OY(%T(hSy<^X+-c-x=iGI<g^G`jmksXn!hby^LjfUt@Q-qX);W@&i%954?D1
z=<=j!+PIp}Qp~WFGqx65!`jiGY13txdXbAKMem=@fd=#5q+1<p*v{72cs4SYpK`dy
zt=}bQpBAxyUgBM=HgSCp3r<jv7-afPlg`HG=7XoWcJWU>@ugAy-Nz+tXIPFj=n8B!
zdQkQ8xcR>{mQ5T7LrynL4A>wR_~Wdi(c}mIGCWVz1XI4e5<R>sRAmVd8*FV<($1Qh
ziMLGH^z_(*gI}butbEDr%b0dyLI~qSCKh9*e=II3Txv#<kGIRNo_Rz)eoKVAu!>!$
z!eV}T2{GNI8ygy1rW`zQ0Gc{dQc@l~N!hR=;l$Zg1H(qWlEX5$4$0hgntVvaQP@2#
z;i(Ykg;xrHPJMWx8YEulF7$=z+MKpb!L&yS2c~(Q&^FswpFYE&G4Z)y8#}+mzIycs
z2`NyMe?oS4X@^UsolQzeNH8#M^<xtXmV2ac^KbIKHuZfLiV+hYOjHfxz7}yLXTxL0
z5AQT)Jv=f+hNnW9o%!O?WB2#-+`S9he`QnSaqEu(d~HHvLIP;S<7+l(h~U||)Pw`m
z>?Z8(+PdsOaN&dZ?~<4%y^>1W@<5;G!kiX&<*4i_3Hy0gTka5ImrHe-6cK#v_jjJN
zXD{64Pnk0rI(y2=Z)nI?Tidoog-u#I<L{w{-9-mQ4vM%7yWHqn>~b*l+oSa^99oAb
zGkuuR_RPS<G~?dES8Qu?l4Lq1|MJ}E;_?ydgKS7%xPAJ=AB8VY2^m2G!AVlF_G{O}
zH<6%$f_?Sr1!ZXu9yr|Z>fkx?ZucYhh}UX1l6=$knzBAfwDNEqQ%$?ucva)ckHf*c
z1cLV``0AbLnQt$7ZeHpPv)+eCIJ?3OK`HA0f2a6_KksA@uMFmi-)B(s=fs1jM-QCm
zzx#)eea2NT!(O4JB-U_^WHy<DX0mGH{R(=iJ0`Ktd&l;%lYcsA#VdtBo-BI*J}Mtx
z7t6yf=I}iw7-TAFCBvp)f1dldeSR)!SC>-pcb1|`i=$IhV=L=kg;oZo-|LPn=V1%7
zQeauD5aIZsT=ZJX!3HVMBNo!DHy(My=yP;OmY7A=myX24Y@mWD@9v^QU9pg0I26w8
z?QL99ZQRmkv|^=WKSP2`)ThG@+s(VDcI$M!<GXS&EQY~%T0(X+qcZP9_KHUm^_!b3
z8dl$IoM};DSa4PC_jlu*n}-Ts2pqn={KvDXPSAK3!~g$E59|X2*@A=F^7Gl!)7cmq
zUkF&QGvtqnF)=e@)-99t;3-*it25B8XJ&yzoYH}0yUI@HaNAGs<q!Y<&ciKcAY*^7
z;HS~y<NYDl&7k!l|Nkq!$+xp(Q&nZ-<_4vP!om#gsf@=PB$)eJnq;2-5N2P=U~DKD
zf9SE1faHYd1~#UeX^#$QH9ZrG>Fv>vmpC^!IbzR@hi4?)%=10uP96Xm3UcSE*WkSl
zplK>eNuF1)cor?{XinSkEnm_gh+*9(mYpx#@7AV#t^KgwtRvHS(iAn0vZn<%n?hTD
zOnUst6*STI?S1-=I@!NA3adat3NllXCn8c(L_{JcMnXnb(!^BK#YMpT?+2b70Uwzk
z!EDTey#8w%-Y_xj{~00MaFfB^to_6>qluYJhkktNOnev9_V~DD-aSw>Tj}yZD;uS?
z7g9i*$x|mxOZ)I8Wy7|#3DeUBL{%?vHD(_<rJ!7BATaT(V#AVU8ey?4cbh{6_A77P
zX1QUTy}@pk39n~w%T2zqq4D9T)VBBcIp?(bDT18(|G(1aRpO$l1_lWmwxnFR-u}+9
z)!b#mV)@OF4&)RBs1^zY9~b5<nAyC1dP_XZ^y&*j>{06_b{=TR@v8pbm-s!e?d<G@
z>U%GM>JN~&_bZ+|dQ?H<#DWC}9=tlWV9Ak$gaZm{ZJXC}ZSDEO#?B%jEXm_CUE>Gi
z#J!?1yoqb{j`*<ithm^p#uo$KiXFEn;l{@1hgUS)Zf{>%utV}A#F>k>Oh0z&l!DF~
z1?|%dmLE^ZIFXRTVtbS=%jz8?|C&zb?5AfN1m8utN_<e``}9iLhI4vCV@n#x$6g+e
z$0vnB18DE=B<`q~2s)JD`+LcKbxXfn!i@G4pEq?XXyKP*qhs7WN5{s4cjr1rOILLX
zJ-NdjbCUI_ro>x|KMdJl9=u=vAl$9=gZz6FF6OMPAHSzJ&XlZDx&HAnPyRiFn4O0{
zypaS=FyG&I@$l9eP=`PJ!KbawY;4RdF5bMwgjrs`dA4+OC+C+mqZ5(&KW?uo(3`lH
z#pYCo`1c*FoKlWI$jn_(`2SC0#b?mg%C`IaLFwSee)%e{dykLv)c<$7J)0RcS<UeO
z|Dsn9<nIKywf+AO+LXz{Dw&cZaptUKN~&Z_i^P(pk|LrDSF+R#9Tah2x9a1S*GKr%
zCa`p5K1$G4dGM<KU&@bfy`UC$US8Y!c#tJKY9x0}dUj5>t@^vfz8c4z2w9cuJrGZv
znQP6};K1V|f2b~irTV=>_5bdOn>V^!?M0Z)8n*@<bTTOAXyj>;Jn=-hV9sHQuU$Ic
zThl(z6Ih|OsVDWI<)=9&dk;9i`Q#S+U}DUkL&u*kab2_{h3DEa6Txl1XWiyrUj09F
z(R)cY7Y(j8OLH@&j!jfaoVPOjuE=Yz>l@!kJ((}R8gvH8|Nm$1T7ynoIrecsf0G3B
z!LS^b_w}0Do;CJd+h;g_W8$`E=g@59(R}v(?;gv0r;6uVdpgAii&(f{zVN-`{O1JS
zT<`0Fp*@R}*4;~9`^w$0DAsF9y>^^ok<6}{xje25PqGJHKC)>;?i=gpD=%_<TEad3
zgV~~n!Xp3VF9Mu@Bf^~&So)exmg*GpB~S5^W83=oD9ePM3bVgP9<wRFKTA}xzIXc4
z-=BYhTS0r~Zsa<kz~geBOVm-IMepzb_rgakUMR@wdHiPlYNrsXC#m;g(|MWs>(+1V
z6Fhe?YR>YyxvFa)?3yqsPh7C4@pqWolQ}CymudtZs%}|OX2`bJnPsaJ%UVs&O4oBo
zYK5jb73-W)7H(}dS=h(CS9@;JLI#bn(~;4_o+oD6O#kfnGq`4_BdCD;|DTyb`c>Br
TwZ<3qpvd%e^>bP0l+XkKzc=NG

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..86d225e5d2158804f88dca881f69ed3ab287d866
GIT binary patch
literal 19086
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s33iWhx45^rtlpw*nn1P*v
zfq~)we<{|*4C=IH9_imGctC;YNKY@PEbs4r59i6&x8IB0WGs<#(Av&?&M;m2apu~t
z4XaiZt<?!~(P-+}RDNp7ch(7KUiQTs&p2qy@aAo{>$A5rm21E*zH`ZutHptb<$y}g
zh1S5pgWdoC&(=`l<7k@v&i0LdOO?}JWB1#?Y?d$kxh$kS*0gVa;)cvk`?gOx5*u>V
zC*(heL(4m-PnAtev^*FXnph45DW$)$o~C+D^tRY@$?KxuWBV3A`Ey9$9&Fc@kd0i;
z4kFd%&%dq<3K0+w{P6$(iK9o)dOFs2zh8Q|G~>HdX4`}{iYM!*AC~Y^Ubiaxx~OmP
zm9LB6zd!uo_O+=?&l)}6+k5eu<<SlSfhH^V<S863`fPSNLJj*eJ2W;My{>#@(Gq!S
zZqDp;oBI~~OkVSP>-$YQhyJ|I*w+Y(mH+>*Ots}|P7rV@zrLVGLdBJ->=EOU-|+{e
zE0_+({c*f7cO#eKy#sk|;pdun&X-vw<oe9dc;##VdHcH`g@&HrRFg9E*v8g`%vFjW
zlOAXvEd3G8cCx-?V)OG=T%Xz*)@U!|eB4p_t@gOVg515C7ryV)d0~|^dj_ZGcI(B=
zpww|CM4PX{Kt%hdyUq^D3&A_?{^uy*6>7TMG$Z2R1cwvC7EZ3N(N|6Wp0${g)f32Y
z>)Np!?H7)|Xg^rRzjkVIZ?VC%N=~8f#{$NW7SDNfc2|;$cj7spPL=DW^2e=wOXuy>
zxnahj;8+*)%v%N=-coNmn`h3PICpOA)Txm%bG)6MgWTP(9dQW=ojP;&?1{a-oz2aS
zQ$?jJZ*XyL`NW_tzW0oSuXcf6QPIJ3ZR&zMES3cP{Q3F*a{vG5=KU>yaNvLScZqv@
zlPi8tdwAcz&Ay(kT8j~4(Vouc$&+U`H^+udarX7~2@k(_E6T~axwreZy|J;f^zHhz
zv<i`b?p~7r|NZ}e{6xoAxr3KZIB2Ckc+?@p<ji9E$nf>t`UzpWL6J=te*gZyU*7)T
z-+zDqUta!S-QVcVos%EFXdYf2&Xa%N=+EENA6{Q~XgB49X0x77XNQKlvuAtz_QnSX
zA3NuF^U9Rz^Z(l`Kdu)OV{<$9zo%zLU|&b+44on_hxNV>CUQ-0>NGGg_;WOPv%*rI
zH!)APw1)gXaC&O}|9^k~f9(c&t^B=&S#FX=)s%-{v)gu;OaA+lDAAG|0kQGSwAF$K
z6a-w#jb_Y95D1(p!6TcXIg@WiquwLq6&nOO^yV{ND9^k9ZqLRC25Hx-KGv1y@h5H$
z({ydPtit(2w0mAgv0LNm)lY5rZ!;|7c@lc)<DZ|6iz9U_Ys*{rELk>zB|zKbO4k$t
z<vwQ1#UgW_JKU=RryeO0P;3VJ^hC$RoH*ok>O#QDqpndtJrgGi?!5S2e(ffQM~l??
zY<zj2gsJT5Ip`r%KhgBiX1kzWE`9<hHbrR5o_WThQt9wpf5N>Vz0y3duNmIif4<<O
z(&2V~p7?zZ$DXoDKs+e*;K;!<XEc-!tzDy&m#LYZvLqo_S5xT_2WPpPn}=opG`kPi
z)m}V`;O1G=$&g*u<YFIno5@|ZHFEg^5&ILJJo79R7|!i2Zcn_f*Oq-<LN6|*;#<$d
z+uIrQHiC-J|No^<965OW_`=oNxk*atxocCBlr+_jad<L*Xk@=~iAk?xui3%SwF%lz
z1utEfNLK_lu5~)NChUUr<{7RWHnj!@e-8VI73isS%2*vKc*Jt}>S`YU`FttSERdu)
z17!BHH9C1oi(Wigl9ZyNs(O@5Q$2x|f!StWL4VTU!xLKF_?J9!_e#|0WZtRcxJubT
zdf|yq=4#32sK*XBSNO_2xx}=#Co$oGhgiYH{q>UTVv{Wjn;!lsY~vP}xVKMsh6>ah
z4M!BUwV%9Lk(a+dDN9RR|2%7Ek&%Ic!gK)-Q;us^tm4e;p6M4Ya?IVQP@=e8dX2AN
zkJVHAv?<CrJ0g?*CMW#h&R(sg-dE2fag~eNP3Y&x=7+CB+YYx&`prwRDDQjtx4P=M
z2-sJPvL6YEKfiQ6yrksHhV9{Dp`s5SCAA#=_5c6>i?ilfL`$4#7XC0vCi=fvw$!1}
z%?X}XD>~Vh`Z>NT|8vyGaE2+j+tyF=_dX>Y2;6x<Qr^xWX2*eo9|ni#+47v9XIS&g
zW&aO|mlnl8x^QL6hmVFEwufK7EG#aRl$`SC_=yt@S7tcmr7Te4u8?L@`z`Lw%f)Q^
zb-~x{4Os`L2V7`7y5XC9fG|tcBZmS-Q;|LU4$rdKFsc5((VmKfAZr?#dEEO9_Efnp
z*I<Tt$NiCjNXqsN7dUuKHf#!-H1T0tYHEwC1an^(^V*#awz3JY7bZMEbW802MJ4Hz
zEN?PdlQ+nC|7@ISVX*X{lBo2F*qDqT5Bz0#HYhzg&flgTF5x#PA!1|G!=J@%=jUh2
z6+#m7qRK}B;o{%EnSK9o;nL;F51ytrs&l&?`_IF}<6$Xn+_^wR?nA=SP@w}-V*d@I
z)GJ;|P5IJj6~yN$@Pc`n#8eI^nVm1YX07_CwAJ#$DG43fs1zY1uBM)Q`%*0mn-Xst
zfh>;Ovmo~lq!e6~`)I?Kke=@14I2ZeO@E%8nCRdq<{^0gz=03_jg5?!Nd{@Q8@4M3
zOkJp|_T(J*yp|^0ix0AL6|#>@DE@7C=Bk|`dNPHnlUc9z4%g4m?GGQhw!OVAxh*Gg
zM|pqSMNkXl|Nkws_VTtG@U-mjR$^PQAdy*xE8^i*okJEp$E_A_$Wd@w?{LVq<X@+-
zruy;L&*dwpr$$}BZ}BbnwgqRC%(Xt%bhk-d9Fv5PIxyW)@=Qn!Z(!6cDD?i=t9s~6
zby>4Q;rYs{ckHoOPig#DmDdYf8OE>GsT8>OVb?}Ah4g1L*Ck8bF#GoRuk_#P;1X$3
z=pzrmvQuYD0>Z@7($iBC5*m)k1kL;?ztXJHRa=QgQEOo!hqo}#hk&${BclE%Rx;X#
zO%N6fYD#eF(%Q1(Z-16fm*xNe|J{qsX2o{2eEM&1^Kg;mp(6~k9BMK%CZsej3ksXo
zIDful%$$W=r#?I*d8lmyD5x3!|6f$J_0W+c2hN|LIQ_Z2WTuMl+4E<5J_zl3(qL{m
z@z(7Bk_Bb%hOY&ul(B7N`mcZNj`Th+mSC~1Z#!f4KgmdE$$k;f-6xm1hG%n;;;J;q
z)$tRuXU?~me0Mkb#ft=sl8%SJ%eDBoLlbM-qlCo7ppYkL&+~u&d|62BSyEEdQyq=J
z^CwPZ43jvJ%(u4H(PuJKw?)(<$J{l4jvBB`?2*{edwlWPqrx|p8ePA|&(P@#{IE^e
z?KiVv!wFt)8>JIXANunA`42C2<~ctvu=p6LGW-93QPiV^<YY6WN#{;|DlZLv`uusr
zls_9D&2417b|oPp!NE)Thd0NUl*U;{Cr)NHnKe;K&Ob>hKt({yNkHsis-$G#FH;M(
z%>PQuwU#uoq(3~^FiXV1Gv&)W*~3px^6V}*wy8Mq;kS9|BWRi8H@msH`RE}hLw)@V
z*TUw_o;`Kq#7@SjnZ}A&r+k?5vA?lV(Cp<Ymd8d01_p|zMh_m%D7D;k_TZ5NAH>tt
zyqp>1!Va1&+M4o^VXbO6(}ry_fx1pYsVPTFY*w9?ytwDXfBRX<7jloBa?M_M$R}Aq
zC`m&v?ZM;Z8(TUbzKd;}Z*LguX9UToYoZ+a4hRUyEPD8W|E>O=KMVI4d@()$an@VM
za82K;0+qumn<tyjj!58JJGHpFXZiPbL+S55VdWdFRO>w(x|(Eo-YdFxXlZX`zBlWf
zP}0%h!(SEJrTX<&{=O1#@pcUpL!h{ErRI;XET79<M86q#eOh^*arp)TN3V!(fdwK#
zPUpDm<l`Xal1br)O-9lZC&b0rZr|44uzB+#FK(A#4HYaa`I++Tj1B~GGTS&hoN@7E
zToKFZqRck$?SxALUb7rT-HlnYHJQR2G`nNi5_}6Xx(qMtEowh>;J}an_R2-(qV<i9
zJR4TEGpnhw`T4c=@UW$&wLLv8`R?A*Eq_5t`v3n$8igA+7-VLhnlhEu%1Ud~rcDY<
zdDJRSHHgYxu;I-%H&8S+eXvSkDr-@=u&LLBxz1;rq~7VpWHP<uaZ_PSD>^83IV0-d
z;+?#jx<&^c$*p>7uXI*<4%bK4O|cQ}<^@ZcV?ud^g_)(LovW;vnVAnavGVlG2Vbd&
z*sf#`wmtLYl&O{0*4mpl86_{_QFA%u5Temh#-{Qk;iwLCkZo_wJY{t$w<Rf!Icj&f
zVpJKUraaa(kl3Kg#Ktz|O2cuklyJo$HY4%F2X~xFak$idPEF(Vp+gg9ocQos{P3rz
zJpFPL=i5QzT*(-0VU|}~npIWx>rERKuPc9;a<Iii>cXy{_CdFq<i1-J+uO`f3^5nj
zJ)xGT-^QfD^i<(gu87B+*|!d{ZV(CIAjZZ#hf(<WJcR{v$7Db5*pcxlCWfc8ljr|G
z!!=PyKfDl}`IQ?SuSrVI{M_7=XU&Qz+7#>2!D(&HTy$Ib+9U@(n;8l`5mOJg$Q=J5
z(88dicgM_ai4e!Ep9z(M>lm~ZRxm_P6P&Fer`uy8!6Ol~;&)=Ndy1LWArTj`#DoLu
zc&6`YU~qO}X<o12JVk_A*z)5UsP7_$g@qTdS>q5fW9R(&oaW{#*A{MM^=J3cY?hp6
zG($pi=7eAdB?ZAyek1W?&6$m?+SjJBuq2$l$ugNwu0ir+&yLiIk`cLyF5Z#_u_-Ft
zJ2+?Tn!vhrspOwO5;?h&Ev<!SMIS*{G5r6pWLe0;QLt;ri(kKIdw6V67H9S-<xti+
zsxV!w;}i4N7KbIGChrwn71Gu>GP=K+6cY8okVSWyq+y8AWR{J~gm(A~s&f7N)XW%d
z)i7VaF?1TEra@_`<e$HiJUkK>79UzU!ax@N|F5+6?7@>Kv+~vn2_`OFb*kTVy>h5t
z&ju!DrN8{H^8{^oJd8?M?EE2+<;MlqDAm96?9#jfZ`LHa8SlHsuu973Xv9(p!`n(9
z%FMI0rLVqhd6sYJI$^c~&xREnHz)u2G4bKs=(f|-J*90}!BM89Sh!)cv9!dABgbY*
zOM5##^EFt_z~<7(*q@+wMbKbNKym|roq>_j0f~sk?8a&n+V}iRK4MiT%(0B&5o1x)
zfh`<&_ZzS&zp^-6AJlIC;d}f9Mq}gVZtlaUr}M<`Gq|(+_=o5G8|5HHypm#}hZh$M
ztK{ult2S*kJh144{hqn3<%-*4`d+^H?-8tfg6je2Yx%HZg9G(FOB~|X2ruT|%%HHZ
zC-K0vvcOA*%EuJg#ln-MxgCT1O;taBFxauf094q1QayaU-{r^D#>U1*Xg=F^_Tcg3
znfdEqzFMWK(&FJiVNaink~PQ1Su$Hy98@%CC`M#S1~H^1HY{g4^dyKoVVj>spx~?|
zoqwwP)SEQ>)-*6`Dm`fO^5WUIkLTDio;PofRB=KAOslY<Xj4RlMoYW2mgdJpTs{9i
zmMSq{+F=nhr;*W$t;_6l=QN`WrbY$^4?65kSdWNki7aDoI_ndYbzzA?Fb`w$M-}ch
zFPj;ovmB<icFdk(z`|m%Vzu!Gf$&6V0?;b-@Z|jU>y@OGmyz+}YKEO>J9dTqkY6*w
zVP<i=xz2=JV*fiG3L2*RZuoFO*+WiYhKqr;(vB&D(OnO;6bqJ$`W!iSEFt@JLh8{2
zXMLxrLqlYa1q+LXtjv|%+^Ii)%$RUnlIO+Gd5p?#{}tbw|LNZKj@wdIk@?ILhpb-*
zxIUZ^x}2vVXP`8(;Pd~(QPLS&oHKfsuW-<o_|UO<rn2hM2XD_NWO;==g*0cCCRp6K
zG2_dZls|vBii%bmc$H3xW6QCT4vbQmAJ&ljpD``8Y4w(bhU_B~!#;#~PEp}bac~oD
z-hbx6E8m<lfvm&_nr0iau3t|tC{xS20jV~X0)BRMe6X}${rY9+<_!n3Sza9C-g1oH
zr;9Tz^1$-Ai&_d>mM*TiV9&<J_G4GO|NOez-`^(Bojb9;ed^Syj9b?vHQel2&DtD!
zXd?TDZvh<Nk|ZQ0{{8=V_QTK5|L5EPFD^Fz@X^q~z~I0&Ho>WOAHD}9EK^(}AyIIu
zsKCI&(r}Lb`45j+8~@&f#L1!Q7CUxE>gqg^l@)DmRXn8-$h}cm?)!vS&73PYHLQ8i
zbbh1ZfkWJuh7Y?u*erhkuW#F5|KG0WN8QCm#q4}{>c79U78Yh!SMT0qv%j@<Nptg>
z#zu}aHIux!n_NW>^7!TmHM;!YS-k(-yT6r>j;wd@|5pd<!q>AUCbGG^Kl@RkR$0lW
zrnX4UuESG4L+Jxc<IJ8XPbF<^B;MUm-%-GLe*>sf`Tzf+=@u#~EU__Hu3TkJSgx07
zCcfcG>5@atsi)>OGVXlyvp-kyx%~{z|NsB~{{H9Z@ulA9&(AD=_V#xEeEa`@{z_Z%
zR=>S9n_E0i#`f2ZyRnK|UUMf;?(FPjEKgBB#M19yxZ?Qywhc@!Dt~?-|M2$qe^%~&
z@9zA3d~2)y`nbP;Ke1{ZJ$0(!>#O~u+7iM-Gm?^LE?KJSB|jsGheyM+MKJR`w@X)N
z=fP854{sQ@WncfG3Tlrr{QrMwrbR{NO&#5*XU;6yxl>Y7z&Y-JqFBN*p(m;~JqAK@
zDH*ks>u+%JYyA5D{{G9${pvmzo3gLJ{r-OcfA`*w^YiO#|NVJ=w0pgNTutr&f7L~s
z%<S#C*W|b5uonGn;o%VDIl|zkQf%OP?Ee1$`|JPb|N5dIy}fRK-QV-m^)<}?{X6^N
zpY7px{{Pig|KfIhSihj*<e{dCot<ZXv7S4&CUU~gjMSb90?qC0pg!<&|Hm@Mkh1g8
zObZoNR#}-TA|g93aPi!DSD~g-SkcJ1-u*%G7ak8yX5GSs_3{7y|1W=e>GP~vvF`nL
zfBwm)ulo1p<@KG#>+Amg`TYO?{xv$Xd3j<H50s?4*&k^Axo~9y^S{5zKR!LZe`RI7
zS<a8T*Vo?5+ZXJ3V1M}ccb@2Nf9k)!%8!>{yil>~Pa^AyPKM372OHKt0rfd0Bn@A@
zFo@aHVGQk=9g3Ut@#Dtm=+iT2coY;Q^aVd$6?4H{(!l6nYiZJdrK?giG?^`H4ffRi
zHPz35ywLgk$0a-*KmPqazcu^)ww#^&OJ6tp&)=|ao0`Pmg47)h>r7hQxgYN?|8JJ}
zXJ78^@3*)Af8ZtQu`j`*YKo%vYkN<2?F7E!hc6!`@pU*Zsi=^ssg?ZlRdPc6dT@W@
z|NletERG!cv3Bj#)Ktqt6^<`wl|S@p*PP(K*MGEZ%O(4sDIfW_J${^@{CM5t<MMxh
zxn8-nH2?m;nEn6qUS685A6N7D-(UW$GYdlvc`Q6kQjfUUE43u@w9Jg(|F8PbkMoOM
z<MZzS`~25BF!lU=d!F?3MkXc(H9rr1n9m`b@pxXw-Y~Wvl|u~^y&i1(^^51#E1orL
zZs<ZvutSqT`P;`wg_q~Z%^L=nS<*5aSr2w|Jz2;nlgX6$>HYos?{9W)E_vB%U+)p|
z<;S<T=cBjV<=iOvf4E)VLTNUm`<=-lT@SqCUOYHEdw=$|n!VNEzaQ^c`SIwye4D7Y
zMA>__GJD2#R~oc^9~iYvo_z4~WXD+9>TYPRRZ`PAlb)%mt$)5Rrtko}`h~K7$6xCN
z9JefK$ZeKNi)vif^Z)<<`@6rFU0R|nZ!e_PVR+#4^ZVO!tABiY`h9(Tz2Z{KH4M6u
z0+C%0m|kp}Z}+$Q%Zu;V*51FjN3k;SeLFwT<}|~Yy<Q;?4>qh#J2)ljKwz4J_J%Df
z0U<~1ZbO2=$zNv88lR}BMQ(1BWMp_Gc)6!U3SGJKL0;4M!7AT>hrS6Vd8*d`|7ThG
z>EXk}{2p8Z96$d2{C;}+{eJoSkMCrQPB$k6ItD2nP}!JK^YzvHjmh@+_Wk|+R=Uuz
zhs$qHf<?uIIY$r72<+Q`#^KtDg9jhHJEx%7=qd)TRh|50oSb6B#DY$plF69ZySC_{
zj^YJw{zn(;+1l#)<@ep+`T5#%e-&|Qfe#N4pU=H5SO5PXd)Xr~ITpqUX(b+s!v2$P
zVLd$m{{KJ!;pyr9y3zX<md13l%i9~hxp}zYIUi?KfvLUW)dyN05)u+NHWE{&Y*?ZK
zDcYU<Wfm=}v9c1fvFVXW?7Ftd#d-txQXxg9&IQ*k#Pw`!%in2ERMk?e`2X+v-QDx!
z_Wk+x@v;Amhdetz|FX}y*fX>3U$RERfoVnyqVDbg|M|_$_w4-tYX12Y&1EuuVSo7P
zX`bVK6PIz<&lXwB*%WFmF5Wy%Y}!f-sO6qAH8nY6Vvnq>YI=J98)%8V$z<IrbkIY%
z{$rSGhsvk_|NnQ3&$q93`zM&OzyAO4zrWwVy*=Nq?vJa>%!3VQgHHrSTus(;@YHyJ
z@9*yF@89q57s++1`P%jHiD%Mm*%J(Fa}KC>pK-k=Bz8<ezkeO1o$ll<b7$8sp`@e_
zokCB<*cNR$U?I=A)YGwKW1~X-zdyC_?p!`UU-Opazvt)wXJ5Zp|C1^I-D6Ls8DF`h
zekC}Bx*5DQFfcHv`Ss=fudneUa`QiZRzLjlvC8{KH|e(xn+tiX6JA}tk}_eUqd7Eo
zJ!KXxiiwI+($V>Em?T(zwyAT1ctL`J!O{iWY^uL~ytueO<+`T8hqt%qYlq+a|KlUO
zdsf1Mi7o}A6L<_vXRz`xPtswk{r&Czr>FPr>w}*0vbDAIOTN3W?ymfAnZ~?hnl5`6
zD>I9WcY#9n|9>YHzPh?HX=zSV)3PTAoZglsh#$N*=|}fOsSiu0*2~-1DKBR4W~=-2
z<M+3>{!1>;Ggv&QF`)Nw!$R)JIgFu(yJr6X|G)n4uk-KkPf_dhpD%H5pK7Fd)NJO+
zIS(2nSSqR)i_1Y{PlYcpk1sV<u<qag4Q%XD)t~iWHQDR*^5o?5HHDtBXKSmUZ~y<t
z$L1HTpDhlokN<C1_lI@2H+OcHgEqHh!4jc0rhM0o*dh%N|NqZ3-~Rvq&+V6v96aD~
z(eU|?_A`ZBJtZY2mn^v;3>lM3YVzjhE`Ifb!_ZK1yKzCN_W!`kEG9b-_?an1^e^bH
z`Ty^?{89J0e}8>_e|h=+`u|B*65((D9iGj%;gu`fi%p<m_dh@7L;GTs-j^iVC@=i+
zTEt?*cK66Q(Dd8?|4nLqd3j;Q#e%7+W>*S+woCQ>pD=sQjI-P_%}+C~@=jaT-rL(b
zbLN`0`y?DcyqBN%rf)0T&vtX(g)5KiOp}-})zH}ZQ779Y#&sw8CNUiR7<`7$ASGNE
zG(*Dh|9_Jkr?vI$HER|uP>5RjVLs#YGm>7ROxA`HYdsE}`q!Ox3zW~9nAX_XOlf16
zmKHWQ=f0#Mafgd7CzUPj=_h_=--IwFo{A77qkqpIC#R$?b$aE{rx?U&nEYx-NoS`a
z2d5$nq@CNO#wRBi#>&blv%KxUyEm6X;g?fPrCIZ4oMO$IvOr<?1_Sk<4-XyyfkTH*
zoIl@n>siAQnOX5G`X0ROcl7Asy#B@fV?%$xqf3j=UKV$Y53iFF6E9qIJM#e2he=9G
zNJvOc4GMa6@#@n<9F97IoQj>&YR64EzKQpW2tN)8d!}w|lhDxH>lrz-y|-5|+OmDW
zs-szwgW0sXa~)k<TYGz*>J2=+lO>B3I3Hw9ND6JR;W~8kWI`HS?Ef<d4nUF?*qY?z
zut32HlagH4HcvdhZ^N>w|0~+vH!kCU)5@-{zIo|V6?S%4_9=#jhFRI4KYrRIFr7of
zAWJ@GC40Zf9St_Nw*P<r@a)>P=FAz9Q0+5kc&1I``6Kc~#OAMH)8@iAZ+Jd^;<2#_
z&;u91VEYmi#e^R}eei>Q_a$*%*#%1_*7SDn-w`dH@wR8b#Ib*=DSy)Q^cF5T(ti1>
zOT&Eo|EpFMbarkO3iNoeGF75uh06x+t`37$D-DA!&M9i0dho>MnWN1V29K`9q@)KA
z97>=QJ55t$d^T;`^TADZTWIb7gz_1Gox3jXUm?m8mHCl<w!%4)#Y<Hmyj|<C?f#9M
zGmjj#Oi8tLb-7WX!nEZ{-|9;TL;Viy<k=zA)xoiNk<#WF62c#?ry4fyv~2F>{dwzQ
z!wJo56I~u2o;!DVii!dRVWF9lA|R4-`N9K-b&emV$Y+>eQ_%f(FlzY&FZmbBFKlei
zR905S#H3u)iHSM$@S)M7ML8?l7YMKMRZ#kIS|TRqOUjWGKjJ+)47IwN`}@z#YMnb*
zQM1!AY?{qo4hDbWh;)^L-U&_&_V&%Qq_%v3*_oIa6#8`Xv^1A*T6<a?qJ_Icmj5|^
z@zS5}B@F(5o92WqXfiUI^YiDflonQLX=Qo&?yXxF%#mT=7R9`|=tz7>f|`DlCg&>G
zs96^;PW1KY=-iW*86fcWwa6dO#zw|kZ*F^g&}hh!zcBlfl0?LwiHW#O5&ocjXJRm?
z%t5s@#l?vgTHG<BT-QD^pVH91T3#|m$+Tq0jv2<rqF1k0?$|lA;D~KnZj*MWz?%3C
zV$wVUdg9{EU0lg-Y`Z52oNR4xVR5dvUu5VZ>aZc|pib0{o01k53-(4L*(vfQDe+*a
z{QtuboP-3FHf_AB`9t_@e9qDH@oa7NO-(v`_oi*zCiLO81qV}HoLF^rpt-rBr6nH^
zd$O+U2NjMOPex0Y4cAOLl2uhCY;7ZRb0Sz+Jw4=$ig-RhGwkRVUf;~cQ2AJRPe7xL
zE4$<(NZUIpDJe-r_^}J0mz&!bGZwF45j&o)Id%!U0YTn^I-7p|ViH!L_4~&Ofy*-<
zJxK`*ew>+i?ZV~9iD}1tuFc4lHNN}Dwd?)<9l`?4>sGONzLIA?T%Y!=e$}d;yLacb
zwfVS+F;`Xb1TQz*v1^jl&M8bWqP*tjTh2m?n53k%v?)_l4|6!WI5lqA_}2Mnd*sW6
zCCn);p<=qcf76qGtzUoMzI<J3;YPK8zgqkIkDl|ZxM#y08mgC;cIFaaHf!U4i-Wh$
z9f&%ekmWeR$LIgjCI8OPKK}5@ku#^e&vX0u@H95^e12|RQ7Kq`i%+#-#zH<eHa11Y
zj7QL5fA~0g;*^&8)_jS7{{NSM+YrdnEI5-<IQ<v@einx3E9dffY!dh&Kechre+dbR
zf88A$%F3nL*iVaz&fQ)2_wjW7`Uk-e*R1JbVp^l6b!XnZM<-8idh{sg%o&~?%F>o)
zk_IfC%Qzp2E|_uV*thtwZ~rH$u9>WEoOkEP_h)DS$M0L9BcGCzA|RGF!>IM(`ToYq
zmu51*Ver2#{C}5LVj`%^8WzR|b?L-Oj|4<h6B2&3mzdaoI@`nZ|NsBPHWOwsR(YwU
z$m>MSa=2&Td-SY8@I()`D~qJh{moBQ+>t;3a4YxwKDpZB@9(DT$JcG%Xz1y&Av@!P
zg!E@^&BBcv4Ijk4?ot=qc+5nu#XdDbH{_q8;os*E6IZNA;NtjE|L4bkZt;Kib!R^O
zwPv=qZeFC++|1m(T6^m3`5Ft)bl9@~`QNXf>QKVTZOp-8U|?(vb?n2($>KtXd?ZhN
z?BzC0Ft`xJ92Ir&QAmy}2Ul}*^T9(-7Y;c+xDW0_{{8-5URqjNSeW_P*Q-a4*dDyp
z`a`6_T`bVWF)BYQiYF?nr~d!H_B->A9CPcJV$oJ8Qd~Ac094Qa|7QrbE-^j*>9c?Q
z-U2;5JSWNp3>9M?T4t(UaFH;$#GV#)utfISA?8C4>o_E4%=ou?v+*IGgQDwMxY?el
zsTgKHm0wl;LF0fjcjk&k>n|)ZR;W{!)z;p^2AQ2rN=#2rOlg@enK6}vxk$;uOM{1p
z=YQc8c^(g*HL1tyb9(vT9A)ckV$|i6@X%M`n8xBF<*}o%Yl;jz&x7?|8=CglOaA+x
zj&P@2ZovV;WgP3QSSO!tFt7dNTx@giQ}c`?jvML@PFCw!6Yt?=m7&HXp`aY;aNz!a
z(9keUbF#BtAA8XpgFj-0Z%#68U6<G(`jw@dcLzsNCZqVz{*Y9I1#Grb(`S@4sGsT#
zbO^e1P)B`&Br|h|k64DS)Q)NO|0VwYO@^4(q|#UKv6aoMv9ZdXk2&jiGh?}C)3F1O
zDz2z;q&;jj?fi3C;D5U_@4p^DHzp;umps#^G;%+fAK}Qhzux%I|ML)2k`j}Coj>v8
z{r~zCKkIo`aI#igixjRGVrCA~X4!nR(Y`PtRpGd;(PMd`X2x=nBrYQXFV!O=LPwVH
zpZRwD0CNm0vt#Up<^?dr5*o~%Bn_nIu~=*JNaV2X$!uZ_Wh`n+I1pB6kl-T0oRwnm
zpu;9+<7o%S#zt+`#AW@Cj0L=<0xT1fnqlLWNr|bcEmxKQPvomo-Zc4;L)aq+wiz#(
zXC9el*Wq~NOHzksg@+WM<c~g1nF-U46m+M~w_kV~md6^;hC2owvzIx_mKHpPQSFEP
zv9trzbP80b{J*hdM;9+!k3^Ef75{_6A2S5cG(<m*S6SWp<`-XR!<~k|KZUC-G$9Ru
zpP!%0uX%D{zR`y54in$`raJg?v7F{#^OE_T+6=o1i>EF5;+V2ue6?GOG<)^u$sP?i
z*Z=%Cm(gejq)U<1cy>*azrc;EwE_(L=kzeDo$HWtP*LSuqsl&~TSk?qV)9XYm9Iy>
z2S{u<`AC2A1Fi?R*u@)MER7B=Wf1=Jhf`J&TCO*){e5V@k;H{l{5i%Oc~<bS^?Yf3
zDNvDOyr{^~Go?kdNquU9FFOl2XA%p0eq-ZIBZW=||0qYc`}>`%wV_e+^E>}09p+o^
zsv!#v4xRgz-f}mcVSk<^Uq}9gMN@eT1lPE7{JV0%!R2>z<C-N08rHHoZY$4VXiDXj
z`SXL())o}5|Nkfb{4E|=<k04|aJr>a7VjB1wl`V+8BGmS6(WRLKALi0GihvhG?;22
zaJT>HERF(I*)0#JE#xg&I`#eig`Lp0;?M8z-~azV!P@ZO-~asYuQ@$C^3l|_bG282
z6_2!dcGN*1F&>LW0`(gm1G*Yt#GMJyGU#Cc@a5xQ>n+<Mo${og-}(0y2(3NXX!DD6
zzw|Ul_eVeamDdVL_U#a2|CiLvXsYz_h%Iwp85?sO$H7BsGn(uFOYEy_-DU|7iudpL
z*Z&X5=Sz24rK9jdo|*ZR6+hda+VCgWRMi@~oTp9x^W$TGGxH{6*|l2|RFn!6L{)B_
zR`4_|5_HOEm}#D#rk49ys`bHf{<iY>3-*da%>|9bE%*PQF!x72Pe<zE{}WjY6=G%w
ze0X2}>-d2W{S!al-<UjKTYLT-tFMylt74A*J09>&{A+>4nsrAHA6+GvC$;zB-`_m8
zwmi%I3<GMO2y#C7U*G0GUt*pelH33MeBaLhpM_O&Vw>Iu6VnO1w|YEfXZGPWdThT+
zfq7NZ4(osUcYf6U`%^9>!^6gwGiQ#E!PFnx;x?&l%$>R1O|LfYo@1Br*sbl9SKAB;
zHgE4;w|>8qJ$$&G=l(uJ0|Nu7Un)O8d;k0Ue-Y7#-6uGCI+Y9s4lmxo##MABX>E+a
zo$dMm|NZ{{|N8p>lO~<{|Np=9BqwiSo)sTkv}+T#DRyK!Jn#}=zQ5<f_4I_))QXQy
z4{sW?Wo8PQB*{KeNSnvH=-E~6w&&*;UWN3sl79aB`v3R$`iaN1{w1sNT{)(?N9QO1
z>udjx2Wl=7HPGOz`Ty_x&(HUd9-Vpg$P5ch!^h0o&CD~qZ~7g{nJIMnpF?&iPtB%;
zgoGde=Q}zyIEGGrcviYi+T6oKrb3AEp9pjM{+dpU7ebIc{p;J?|I7VPSl@Wy_^rS3
z;s5{t1wMkjaMM7rNz`nX6wB}b|M&m@_jhl}%ijl^?=M*zIcKh=;kP@B-j@IBS8P39
zW^;SRhF!cr3k07$N^f9PR&HLR(!5gha4R>@^>v0df4vTgXnRiHn6dK7|N6Fm`Guui
z%*@cF|Lf=H`}6JVANmGLG%aKEn2_kp>k_lP$#`nazCVlCzn8VPPWIvHZ($K$^+vvj
zrE#X(2D5}tPxcH={>_#~5e+*mn)li^uh4Lgmp{BU+a*)6=)92Jhv)o<pP%nIycOCG
z`}Oy?J@euF`|AsjnOtg=`CxJQlIDTEra}{~|NsC0`Tzg>*5!3RbASE2wKZN(PiEo5
zn3iAGJ1()z>z$S%_3&c9_s0a5o*o`YN1nDe9{qTuFJA<zJ^4TC@~o-4zdya=uT<eo
z4QM!ie*S*C{{MZozuTi0{sFBhTJ1PtGHY7akKh0Q$LDA=*P83N++6Byujcz}N9JX7
zNy%qlO5_|J+4fk<6&vfyO|7^jb}cHcKvZ-_Lx2CmWo?epvlH*zwRujK(2G}F810=k
zTSV#i55vQEcXwp3g8HuN|G)Q-kN@9S?|%N*irFs&Uox+}$*9`7r{Vea_4Vrhb@`b;
zH{{$5XKVW(8F}a1x12R=W(0OedI%`{^}LzjDORvx{rL$K4<0ywenX9=bGG*3iOL;p
zjCBp^`YP+5eEM&Hcx^OKzntQ?FR+s4@9c-?=I-BL_ct{mq3NsFgzs+7JPvYi6C8vZ
zrTQP=x%b~7`oDeszw)1-@;9f|R{r>)pQ3p3^Yi-3%8l2)MC{i6Z+PQ>fa`}6w+UDI
z4G)|?ec<%z58reTPuJ&}ZEn1$>ga?C2S8(fYHf840yjfBi?k~L&3SlPy+sEc##iR<
z=4~+GagpCIxQY4jMuFWI3>cIzED+hE$zgIs#DhhME9y>U*~@2h{dWn?c)nMICBWAH
z)1=htua0uAHvg$DWZ<;AaLwATgtzxfqi_F8<cL%Wb&Y+#>Qh;W=D}qevz>BGrlvff
zuG&<r!+l`arSCq%k3B9Yoax?aQ<Pt$AG+uGPWP&xjG)z{Nr?#wKYo4HSJ9d|L!epm
z#N&lMAn<?Ux3vzv&1+sIG-NrHzrMf!zkbXQ+w^nWWv%P%s=j>xU(Y5c_M(T6o&E9V
z^!mi`$Yfc*gZ6t4Ht0@GaCMgc^SgaVUYu*IqgU6%m%(l8^x7<oB_>UloHJK)%NB{2
zmV$dyJ}I-AmA*dV1eH%GwyQw%QDQ<u#rJpT54ZnUQF-9KUt+?tM8TtMH8c5r^lSF=
zcct&E{r&&J!TXcd>(|HsPrkozzKl%Iu3b9Ywyk-8@9*txyth1)l9DD&PyhA#xjZ+w
zvYS;x!^T2U2_9v|NQVRS?Rid5bG*4`4UeiSPyIgwpLu6MOa1Qb<Z0(OmXI)zkTP1a
z%BZ8$kb}eE!^el#`F#HsZcNqYDSy9c3T$L&ip+oiIX|-V&b4eY`5>>Nsc?SZA+{3>
zIa!VRxmQ>HtN-!g|JT>`<?sL5R)0HxbMyZ7aeqI*xEL=g`fP(inY+80n_Jr%pCh({
zMn*nKNjw``-}CGcW{+XnDf)AMKy&hs2624}7Z(Wy^O^T|9!|(Uec;%!1&a?Ce9<_(
zEte<#oWUH6g9i>BQcyjrpmj<?<;a3XhbGKCsi5!w#zn74sPfN@hi{{&h#<o9$M5M6
zKR({S;n=hKB~uUgtY~H5Q^=+Gzy9~P|DT@jpKboX>dTAwo74Z#v-|t|$4AiYVqu}0
zt?jfXCO&ufFjdtwKEAe{#q|oefBpJuucyb>^TH)LA-CN~VXL}?LDtMf)_;GKK?_=s
za%-5|%z60Ko2|5zEi{x(Q<E(>x9#t5Nj~|s9Yw7VZyLAV-YyvtDaj|7^5A*;iPOms
z9xASt|NW5p@NDxJ<)@)#Kw?5dLQ2YypP%0!@AqM1>gmxi*7<)?Y=Tp7(>5~+y_g@y
zrLX#%+3V|nAN%n6`TzU-B`Yc=b#)?QVk2#BBBoBAX=y2`kl8VL@<aPc%%-L@>;E>%
zTO4E9Jdsg(?E^2>!`oIku+{%Js`+!`!#mk2r;oK+Y??QF_Q5Mt4qlnEa7D+%qug!E
z?h<l#DHVTbJ-lts799NMDBHitsn+EZdH3CV3LKzSC@9bTo&E6X>Hqxl|AYk`^2!Vr
zE$jckLiB9Cn;U4_jg!N`!fMl}&&D1e1}|T1kdQRg)-pOM!`^#Q-u|C`^|$Z8N?p<?
z&7SQTGw0#`{q;URGdh{i9rcl!vGrsDn|-~}96R3B;zC0MqokeYvu4h844QQCaP!00
z;cfZ%CF=gA2#BUWc%X16)X`gYd!FH)-Nzfs&O+vqpUk!8YH{Fc-QTk=f=8JnEY8uU
z|9ktyOjbb_)95n~_)nYNey4r8wdKjg=@Vpbm>v7bk@&t^xb@oRgA7YzKYy;!RPEP_
zRge6p#kFzTs!7WXQ#F<P4OS<u^_#qHZN>ba(z7=5=InNsFU{Fb*)L&T;Kgyh{8eeH
zM3Bz=R!5fP0>0;U6JIJ%V|_0#+}B|aF6o-)`F(wP`G5ZXe}8`-K5(jQ&#QLEpbCYx
zQh)eoA52e3Nf8iA+OjdRX{(Q<gh2bU#XJ6;o&A5!n*R@{S8U#BcxTsF|Hzq)Mmv}|
zxg<KbwjVNZ>f%cI@LnD?Ir;sa<IbQU9{YNuKfl{Q+Ze7{*_aU3*kNCDqTsjLw2k4A
z*lPM#ZC9UG@#n|?@9$4Mk6e6a#(`yT{wJT=_UE`n-v53Zwy*#H*DEgkzufO{qj=>1
ze});^#`ku&+h}$zE6?b!3}!VjG`zF(@P|*TQ?xZt9yr>nnwr4Z_*m1pKvZ{ozH!Wc
zzPn7FJW#vT`t;)^&dvLKzM&*qy5<{8=`UVYvB2;O1$99e8Dpb=yK8^1U!kFFllbYA
z(Vw4(51jiaUq0(c{}G8uRyG;X!q#6VB^Q{2$}=((!#xY6Soweapa1Z8d0YB<fwOTb
z5KErSHRNMZ<Y2iz?bP`n|I=?|CU4nh>bNe@(_;4O{46i0vYRYhM1z?&=j1N55;<`C
zvfdoq<Mm}pQHObyGJUURy#EmA@=PMHUVMkt<o`zMMQ&@o6}uEz1a3Aw`jD9*;pN!#
z-R3>GZgApRUi;@n!84vCI$JLNvv1=lD*Y_J@`J<c#DB>NKaQXI^Zw=K`ZH&EK18dr
zuu9t1{`&v?e0|up`W31yM%6COprwy@5_eRKpL+T@dBW7AGmRuBJk#sQY+1B)Dm%Z#
zxp_zDfo3om{{Me7SC{XA0#ECHmfnZ|o;Nx_;N+Xou}nBQ=jMq?KW)PUzhAok<-3Zz
zW{ZIu8{hfH=-|3Ex$vElwGyELf~9>DrD73TOUru#1#2%nVt2@{@b@~rMeO;#bF$|r
ze8@bUdVR{A>$8}0z^O6GDU54ALkqK;+l!wUJx;R=p5?xwajkyIC05;i{|${m6RY~4
zY>)Hs@Vt5R|Nh?J|9^ka*mQuY?r+kL>gf*;v9_G}{+=f+EJDrqL%IKkRW88{k`9O8
z9C3ogj8nvq@A`-T{^t4qZlOePqOcd2L#!i*ZpejOllIsB{r&&H{im59=FH*gmpA6%
z{-2&E=PMj>;Qf6b^L!(lx*mZ>pOp{tD;`a3y6!$h^V!$zw&Ld(+|NFQnyu1jUoY|Q
zPGZIXc{8fGs}&D6yxkYc*Y|(E-QWNJWG8<3|0nUmv$TpY9S^VT3$bOn@m2hv_wYmF
zk$((!c5JMyY<_-Bp=^e^K_(NyNi_AxAIaTfY;2(JAH)CuPMWLs)h6$#lfCGa=AhfR
zA*xfYp|kGKkMoBP9XQdk>EFla{%z&&CF=e!d#!gtk#S#LDrhbFpUNX1X2!<NJv`GI
zi(?qpJ~|*=cz>T^&W(o62M@vq_nkatKr@p!8229PWYny6IHkH}S6kzbXZ%Yhe|pI5
z^yGU!XvuR$g+$#S2l+Gd4<EdbZ_|&LI5#)B;tR*b={Igj*w}8IukU?^g^RiG$*1@7
zhrhn=_{0hA{yOD9v8g{-@R;qK7_ZI32dfSn%>DhG|M2#Fo~9;__Zv4Dcz8Si&TL*4
z1*#oynI?sQxWwepW`1st<E^kGM-uY<u1K)WIo)6$@{mhHQgWJ(V@bybP#X~xCi4BY
zADa?yo3Ziox_nu@H1dfOOC;~QI16p<<~2Hp7rS%Bf@+h0XFmM1Jv`f-M@i|&uJ<Oa
z_5TbE3=M039obN%dU%>H&vZS*n%~Dh{5EH{wq`arcMcMAE;D0(BBZw~wx3B-bZ>>A
z;iR)CPbw%kr&=1SIe0rnu<x6&^ZI(@nEic4l_F}8_}?>kFK252Ppf?ilOyZK0%`RE
zmhg=Qk_Q=|oJ-U;J0!~D`myQ9toHwV<XQYhgXRbnd2p~4ovEIs!Szb&bke+wTkiT;
zb|<XvIW5(%9Q3I2$xl6(`@x|z)cQ7mvT0bh@b!h#O+6}(98;WDelL98q@mQu$#h8I
z`cZ?tztjJ-=6Xjeu>`F<mw4d&#<%trb~$>{*1r3<@IGJ#wIQASW$OPKfL8y%muv3e
zxT5f<u>Tp8kEx8z1?z<m8YWJ33=r5=`Ru;^i?d-SyY|YyTf#5DpmrlWzeJx*;*CwM
z5ARwZX-fBDh)my|zzAAG`tKi4T^)~?7tg6vAv^S>RxAJipZ??5)Q3knzl6@Yc8%xJ
zBcA&Ii}V%UvK<m~8y~GqlCw*(C}yk8<c9{<nz^=Itp+>|_A((K81$U@A6yaUxVOXn
z!c9jJ**SMuN(;m@Wku%ys<3ZXt@ir0)bm!`y@v*Ei3>6(6s=#|f1@z0*x^I}@#t$$
zb9}RpO{`qDj+4oTKlxjHkK7&mWt_K<6?zDLnzL{DYmF}N`!Cgv@|-oNDD_PgQtVJV
zRIuB+CSl&}(CMu@i*;763fFqIi=**<eRa&a(x2zod}uqT7BzX<mbdR)^1)%ZWcFUJ
zW&@E{`xx%71rJ_5y0Fl$r`i8SHjB8s^-d3Uc54pND+lFWtTnD~^YX4<R-jPs!S{0q
z=gk9|kxDG<ezQ#3t8X%S|Gg{a?@k!CIZo0^yI#1rXt$tihE$i+yv3^+YVD2d=APbr
za7*7Lmr!5*nkAZrKC!`}EsBl~w=bEfoZLTmq5#Lij~&kwZ(H!~I`^<4)sQEC?i&57
z3A;3Q#8}M#SgVrB>3+M&Dsk3r;rh!K=e943{I>I)EGXXR%-qj)$bsir{jEiJ8e4Z+
z+}zhRCCU2NqJ7;89_>xt5nju^mM_-Y)~(QSnDd{tBTwX})|gLEPnWkn+Y$3_`L<^#
z=eM7{ef#qf*B+K{Kd<}Fo+8t)BUT}ITJ^D^qeJ8)3!ZivK8{9#ExJw#x99kAIIeFG
z`tqBH?TO{IqZuxf7Kj9Gy1g+j_<H%u?HU_&cHLe7>3;pTyO&;0NImoXOq!3{4!uMV
zna%ef|7N)U+L`ZwiLTX^r>nGPwV9-DeN<D&xcTK24Ws2B+1TdyW$w?o;UQ!;d%oLK
zHn7L$?B{MO5IAa|@UwaX3u}8pN5-|C3;%EZzhI@~FJZ1likV#s(k+JsesSHJkTTKv
z>yDSp%<tT-E;znb`lCsznKH+{DNm=~WS;-?+PlKo9BI$2J%#g%_deQn=6Q?aa^b==
z8Aq~O7d`m>=6i-z-{CUv?cJ#xZwH31&?@wql{8C+Z_RZHw(Zihd0iV53+&9l><j6Q
z4-EZkq}0`+#KycV*rxxZMC|hW=|&GLj(iRXZCP-AvBrxM4eu6*KmjICrPCflEeRV}
z{4cQKKg$@IX0c}VTl-0SUY}Oh6QAZi>B;Jk#q3}|&h_VN3J^F}KSj4l@spB);6>NB
zR*m0ZyCrH{H{3gLqE)X+?0nP?gQ~4B93{jq<+LZ79X|8@to$;2i(Au7mTE+-y?3r^
zB^Ps4;%3Qt*AsmgPdwK4-MX;nal-AK?Hr9k9-WT^u1`_zW(^7wsgMgQvM`umyt~jU
zQKIMgT<*31C$*$L{LOK&q$_BaX~@-p&(~+A&)a-oxNU~oY~H{lj~icaiYwkL#PTcq
zR?*3i9RgESeHZuhpG%+Aa(ZcSu$u6q7uf}W+UDDDi9D9~-7@a4nm@RDpEGMecT<4C
zar><Eo0c40rodH{B`EXffAkXDqRDL>iz<9=1$=6FIj7l6?RvRG`dyoT`Th5Wdxh+L
zZ{|g=7ktDm`F7(+$(J&6QJW(k?9vEzJtcJcNtNpPLYYY|FGY?&EK0OVGjZ~2Ik4%*
zTM3>EyZ$Y+nXC2u=bv^r4#&U+rk^bx52PI~I+=4UCFt!w9cQmgQ~vH-q_gerN2lu>
z%)i|CjS_2Tp4XncSS0bp+qU<{GLur*gv~i$As6(iVu8qU<u=E_D<=$ini@1-X<QLl
zx@EzI>8$U{AN_b8_4hVpE5VvshFmQM0<QC$GByaV$$5LkY_Iu;UDvp{<qkOCkaTou
z+7zwA5*EARhNVN-$>(S1pZ_CreA~tES4<<z?A_PhsHj(0TGz5b=lr(AcTV{Udp-Q!
zv-s<_J{CUadP82%r1y%8I5a0tTBLBA!`WLQIiTV31B(?}-{vMh__`=JQmlKmx`~-n
z;mMNgf-Gh3y~$hDd;_@@FU%@+?I_lHUFqRDZ+`r~f8W=J?U2?`i_4OF_A~P`XkGsQ
z|4ZgrbF~_XxcDzj&|LYZsdd|}o!s8rpNG9+vDUuZ)Vq{Jtn%nohd%-j=d!klm><3)
zdE#|}414W)oB31EADvb=op+&*=cMf8%JUrF@%kBmSD$oOvN`C9rI#Q}ilnP!=t9%m
zZH6t1flp?*cmMf6cbN(AWtZ!QJUZO3Pgzto9kmX7=yLvO^o{fa{x@5ft?PW;zWCm;
zL)vn`{cc-XFP<(BPC19njsASoJ$z*)&$n+J*&F&Q4GbQvE>UoBX;D;Ycz8}$>$~iP
z+;s3_hzXNGiz0sfnR)Qiq=P3q9R2zldwU&2rZ_q?-%@X}kh_$7j9KBXYDZOUESs%u
z+jM<Nzj-Zu^6~r5Y{->n78c$j?i+jH)TxBLa|>2?`|}q4c)Tj(!lIWi0^2sHd!BBD
zRQGe{_;NV}h_vpXGCzndi0xR<ncTxS%ybU+ow;|%NX0rpP37bR&3P}5UYDKAdgt8U
zxqZic+7v@w=Os&cO^e8TFk7S4w5aBiiAd|D#Gk#X8*|ra2exDy^{H@Xc0EtLeZ(-b
z?Y*(@j(6MdZ=P1U?0uwI8>3=hL&f!r+bsGX_0<crT>0v3zV=&9t#st4z-b%{&Hcm6
zz~QuJrXp8sfPgE%8mn*Ayn|{pSZ-@Ko!)pmw$IyuJFHMtX-A``ln#I3p9kMRr!zAj
zo_lz%HNXAR)P<QEU8h&oCj0F?=$w6O(yqF+PlaEs`XsjpbtFU{*_b(TS&QQBwxASA
z#|Dml{vro8J%3l<H1kZ-nEU*U@#7AT%P!YvPDz#cU~}!O_}U$Mp<fSn-LpNSUARTJ
z_P?Lx0X|TXwr7qZSF?kFt2~EPAb*0e;E_YtH?w92FZrNy&-Kc^LrytIlw1$!s!S5u
zEfC49=)3IJ&$~tjf4+-<t6r5pD`x$YmA@9&?w9VH{dh%o=rj@5$0s&z$nBC<(LA%+
zOM1TlApyk?HjgAuA2R4rs+s?L?&p~*fo5To?(KS&_x-#o%ju@a7q;;*w>e(EP&z}c
zwD+|V%jK5b;@x{!`nElq6zr!i$KTXoqkh`X=2Pb3d)iUEuRmS*erL>zRrB9}WdgbW
z%*>5^&H*BA_RZEEYDzqMYWfLARSuKOKUk@`O%7FA*UFg5e#xwNS?2P8>kIbHvA=IQ
z|KSCbNi8<&U2?(o;o8d!_9!e833a`HdD|N8>D;mHuZ7ya*akha=uzI+pJ{adhFROe
zO$tXC59S^EKkfD9$^IP6I+rPOXn0Ray1gP>r+Zzp1YiF5J&)I|dVRz&vuld_g8z5a
zez%=A>UzZOx$`-=VpPhqYnVP=QM-HLf`-Oc*6K|Mrg{}jmH$xC(j#hQ>|9+kA;)F=
zSK$(d3*HYSCG6@zgFTRq-E47jZRYtBEG!cD_NNO7H1xf25DQLXvbo4Vv30t@iI|S7
z|Nk4mxp(%$%}%5F?lzE2prj>dSD$ucL*v1lGY=kM*t?aHpXpDeWbq+ZrUyG&jI>WX
z9G`0Xg>ix$FEjI&WddStU8heN7#KXb-NnQh5zHLL|IPm0xwqZopdr~idpw^RK*pJr
zww3Iwot=1yrS1IujbD;CRxxj8G_7D*rd)WEPv<}hXWFI%m%aY}o16I9txeY2Bd`4s
zRG(SlpWnwf<jWsk6M3Qg+(TE%c~=@jZ#popVeB<ckt{fVkikkKyy5Yu>-YEbsQVk8
znc;XS!4ldeRazMP{W1IDGc$SE`2?N@__VC)-o<d!A?D<i#(>7gOtpVX6WmU+%7vXc
z*t*)j$u!d8!eV!xzkhlB=LbEJfCk6fw*Ma*7jB>K7$;|VuYpbBbliml|NmbQ)81r0
zVd_Kqs^ccS#hsi{a}K_BDprva-~6#<B~S4^2IJa<xq(VT3o6+y6_bn#RP(mT{CFvR
zc#0;Exc;O!5++d7ce+=6R64xG^Fo;Uze$p7Lt2!5j<OW9{WD_NfAvSPM4j`8*=~DQ
zf6!=P=Tp7s;#U9PNGIyZhID?>Z_uSqN;~sGa~A7jd5nyBSXp^wWCFq$rRs0_^w|Dh
z&_nxed%RT+EESKG>Ho>H_FsQgOG0S>)+tFYvFrP<TKV*zh%ph;jWl@RA@l#AVb0xC
z8)__1?NZzX4f^GefBdrr?e==~h^MfS=ggT4bvo-r8Z`gOAK#_;=~N0^&m{TjMr#<R
z&o-PYI60s|)$57J(Wz<|Bq~lBu-(yo{G+ff^Rk4T&4P0x=HP-%sW{}}&tj0^kDro%
z{svxRH~P_k@ruLbqcZkORXQhx@Hj9S7WS;<IdoJ<VuI*`qr6dTl)E2m+J7)M2tGDd
zn`iFa3!;-lK+Xegm#SB05EgEpBjfDN-Mrbjq=dh&LG;!QOEu=+lmpu)_VGAmnL74(
z`J|p;6%BK~@TX~c#o<#&eYkj5a5gb2*Z%o^yx_Z@bD`nkqup2ToP+GWS(GuILs_|b
ziiop2d-G;vW^L_hip6In#hzR&S!1mK!9is=N8{|~<*rU;znZiE^shbK+TOeR-Qm#R
z6GGPB*?l}9<Ve9Uld0y*Aj83n4v8rl8n#^vW8>#%D=Rx<!_B-+iA9^Uls&6!p+TX7
zyP_n6|Lw^Vx-(q4=cRN_ZJg@jE%QO|-r8uMr%!p@`-PsGPJ$$qMd$t*1qQZlGGeQ*
zXJcf1^Rl_|#gWFT-kb+S4PC;tT~-~B(*5_pe_p@@Va7CfHvi4dj9Z^Hl-KIVrDWu!
zSd{gg$!P#px(uMbsVW>?MlLP}OO_a%IQ3{vyu=1m(+d(D(=S@oPMBiBVbwNc60>ok
z!2>@ro*R=mqE6k|)|>b$q^<aQM(KT9a6B%G<1sM!@WEilY~v|Y9M<1TOK`|sWR#%7
z8pFxj>+LDb$mh|U^T#)Q%PA)Q=`)tGG547?dLFts-=634a^p2oM+<(Ntqo#>P2?>~
zdz6rnkdT}_VM2n2ZfZecqJWs%oi_(gIau^HePdTs*teP~yx&{$+LZ$iE4CbXC3l8R
zlX)9gD`QxKfq}uEy0aJVo!d}r-B?_y0b0fi@?z1W1_nk(w)ptA6CP~L%o&1C8=fBQ
zFtl|{IdwqzR|MlM-4up1*Yy4!IoPoMi9$rng0s4BZXPzMwK{xq@)rw3$by4Ko*;KU
zc#t3<p6=n9_~BF1goy{^_CI)e%IJblp#bxT62l3y%`?wkaQPr2x1(wUkHkynxUN0b
z#|<)t4xgKQA$Lg$DDpt=34PSSpr^-{oy}HW-nKyD#VYkPQEadH?o8#@bKD_vZ651C
z5$BYxP4UW=oy^(xt&Gt#8T}>W_Zz*rcXmSp<KdacCALZYkf6^6IVvG3DIq<*pdg{3
zJe`BPc^%V{t1hVuwuK3-KitJTOeQ4cPkVEu@v7JvHue<%pKou2!aZikfe(KyeSfX4
z0J-Y_|3#IL8kpna*iuv5dU)Cz7}(f)lvvKsU=I_tVc6{JFfWtE+Tw>W`@b>`De=t*
z8s@CuQl@+O;^L0Ik4xB~Cd5B#U{F+K<KzUTN^5Jj$VfIvN0)BVUKa~N(=QuOJ$ltS
z!SGNckHPFYM^>mC6)q4}|1jm_1IEKIFN4bJH}}qdc*tB@c5xys-pU^}F!=eg2@0|?
zGqZ7Xvx$nbm6oz`a=I)QeQ2@gDCZ`Nd3jm~Z!z*KEY5D8Q^UE<d7|*C@9%lU^}(y~
zf180!j^EFhvZ(>&M27$WrDhyqWIS_*XU`s<KYw_vtavV6;!#uMId_gHCg#H7&kvR_
zd8p~}Kq;g1vHh_mL7Q7Ex&J+mJ=$^R-}nB9$NAgp|4YoXO}??M7j$Su`gsXC+oT_V
zXNRQ=O$1H0gUmm1gpqy1-k=HEd`p)~>gY<Al!7KmcI;dj%6~0S_unfvU&%^e%b2tz
zGn>M%{RXW6|0Y-boAdCVZQJ|%lI!A9E$YA=^L(~XPf}t)uKxdDN~DvS*P&1E=@Zbj
zLqIra-d{i{>B6O?3zwSaO+9?<7-;hI`0)eB+B$ykuNSoL*uXT&&4uHd?+zXpZA-hl
zlpmj1Uu~FeF6lQv4K%Ov*c~+7XIGm98qs+jF4b2b%mK0L%pF6nRs#Xoi>p_-Fm+Bi
zr1jumdCJn>*47&=E+z?v%>Tvp4s35VKNC{C&qC(Eq1*3uT22ereVixpvij+bA~m<R
zMu7~ezUC=k47TcCdn9u&`K|v3k@XQ1^i~F(nmqr=_3%)^JLf0Q(d6nmXnAVVYbDjt
ziSK8xJ1lW+YGF#G$ica4&t}%9-!8H3koo=>)B^ec|IA%Ot_B01)`#^Ty=(9NpTH*@
z<Knc^xk<^Q>&LkXs*N@8-yi-cv48HIzb1Q?581HaeG=z?Zbh2`+w{b{w{qD(`q!67
zZPTsW>gv%sB|tUZYvGgt?O2PoQ%h4NW~dz#G|ozmyzza4_?bn%$5?muzZcGb3eH$k
zlRBF#4jf~9tF<_HrBj-*0`C@wE&+}tmszGQQ?%{t3~K)K$u4dYF`kgD`R||Xk9n-R
fkPyUvQWP^oPUPjdAN#Ie1F7<K^>bP0l+XkKdlJbi

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-1-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..37e8ffae114625d0cc6a07ab2b8dbbb7413a3829
GIT binary patch
literal 18884
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3?DlkV45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpNBR#6GAQ!2#4P?6zh3S1-gT_08Vp~o9>@jF-t4Zi$UURypSEXE
zwD*B~HyX|>UY@yb(J8Y~UEzw_qHcbW1xL;s3NZ-sI9!xSlk{9({(o|&XbaQHdrjp{
z|L?wTsP2e4HzTOyqjO0|--)VxX9fmSwhjlEXltcaRWr`dsqVS^JMmueyQpx@?#Q_H
zF`t-0M&7w%&DCHaz<NNOSCFYmUO=?ye=;ZIL8l2KW@($xABw(lj`7*{;-2KqI=R<>
zCVu$*#<@W8M(NsRC%@RvNshf7#gxV4$tt4Yuz-Q(i?iU_$ocN)H^13^OyhP~(wXCm
znkQ?Yzn(PZ-^`QTV9PFT<Z5;hU<sJ?x$n&}FV@}KfB(<lC~AMeN$|XLetIwqYqjEp
zm2AtT@BM#}aPHNw*lCraPwy39-m_=BYhuT|Gi@&%mz-O<)xeFPD_%Ij_IIk)#T81b
zInqfk91&uij@LvFE5uH}74|D`ndh^}V}if`z1Gh=#SXIh$|Y;A0}3FU4>cT#_;FDB
z&0l`y3YN7Z(yL>GSyh*K3mSd(G2V0kaY0{pt;A+#4u-t9YO2O<np?RpO1?gAWc|XQ
zscq8E%9A?`rG%BTdW)7#F;HD2n(Xr4kVUICYH`rMy63TekW{d$)7h&lJYq(;SC@PI
z{P?(e;W2Zf9UFt4nmT9ywigyYSJDtJVC;Kf=M7=@po^dQgSOQC`SJeV-umr%e`~+L
zdw+TP|8w*IfB*AyfBt<5yE>T{AC$PEsVb?{Ib@2rM~A;p&*?)>E`ELiv9TVWouLsw
z?1hC*_DM=kTr6Y!>*Es9Gg@qW|2U-nZ+-puzx=$61xoe*{#5_@ale(jKL75o-=CiT
zH%|Yzzv}3R&+3P#>odzXEdkj8O2bl}&66ibM$8EJ=(v5z$tN__$=^T3ySsP(fBP&Y
zhrHNFD}4nn_RN~}Z=tAAh469vC1yG*g$qoQA9nr!`}_N+r~7Mv|BK!G>;KQ<Hg0hV
zyV|55ADbUuU*E_VtOT;+|9`0~oy`*`Hcp=&8!#c-*Z05)k1&7#TZfwBL%!JC$SSDv
zO?i=?((=vv-~a#o-WnZ`Smcg>IMO&v=K`0|8&7uKg$uNtRrsDbJ^hy3wl-Qa?{3nL
z>gf+3I=8*QFIo35MdHk2HK;vHI-6(CoY>wT>(LP%AAjab$jy_Ue!jgkdwJ8d9K;;8
zjQ{=r|DS&=hogydg61>(ET8}B2LdY;#6@phI$mG#uyNLse@>U!l^@ri_;B<4dmeT^
zgEu!07JSz`ytkTXzP<6EU#1?DIUrG90=BBTIW%U@@jEg1j=9~v8R-->Nl<8dPY=(D
ztL%MWK8h=`_Ip=6ayb3#Llb-UdX}~3-Alxc1ez8Jntom2t++k$|NrzK|K~rvZ{PO+
zzocGlQpDD-hmTy_+W95z>Qa7uPQRck2=$mtXLD!g#MV|%pPt*7f^MGnK7P#2#i60I
zw^wlS#!d};11Yf&iaKYesZCR2+$`j{I>sPbHoCLv+N~e$cVb*~SN}^+Xjs<p|G#A2
z-=rTuCqMj{+_p7al20a~;#14R$L?$%uOwCARylih9Y5rB^>Xm-OF=F!EmJ2??(BSN
zzoz9tQ0juO-XFI)eC$X_UtAh#D3G?XaqF%|^}hWEg)Wa8ChV^_uK9K3!w1E~*Vgh>
ze>bwJK4wti<_<Mjq|-TI!rdDYCy%?IKjq~VHEVKv`^=e=6V(E;1-2~HNL%A9av+)e
zl<Qhefv&*Yt2)}4bg~4ea$M|utRLm%n7wI%Dj!>f=BIDEhv!=J+}>{d=HA&4pHvU`
z%QH(V3qbAoaU?N8;ql|-7cVrHuQ{WyaNy9PLk&^Q3gPk*ml?$z&#Yp}WMW;)cj&6p
z!4;YTDTcq)Ui1cEDQQ+aeV}2Yg~F5b^LftCGmP1HW<#Cr;hn`i=jZcBEOCGuej+kb
zRh7}(JGZ``H#XLglantjj4eWuk<q88=YK*@hEiHeN=xCxp8o=qCQkhLzkY{;@R>RP
zSDbqA!v2hj8e3bvM^gruf?@j_9?1iu&LKud1_nwy1%f3X|Nr09(_?X4<jotNt}dQO
zk9drXc*OOMbfS-a_@a4uwz&$s&Q_=!HcUEv%FEQ`%85HMVZOcdCr{>_rY|{xtMh+^
z^g@ZB5`kt9^5Pb}JgSpiFx57~?bro3qYbN$ygA`ub=AQ1hBqGvr=z^R(HxtT1wRZ9
ze|yW*FVF8I!w5;y4M!HP(n`o%CoZ!lHAO{R>Clk_2O6%ta>;8<>dmRx%v_z#yj8W)
zQmAX`y94vXF1R@vzUDjQ(yUzg$Mq5`>xq+1jGCzjZf?&roMU--Lx$kt$H$oiS|HBe
zGj$_xg8>iA{SHO}W)~5U4USUJPHYH?RLlM#a!hT>B8MGCap$aqmh9iZQRwlMpgY?t
zW83#FPtBPuY;e(ew=zHT*|p5`E}4qFb6YY~uwmB!s}1qrcZCF9R94}Y<bM~kE$FB7
z_f`JIUnC{Aec`<)ahf3^Qh@20pk?y?z*FY?KWu^&(~IIC35Y)3vgJx&m(ivT0b-)5
zi76=!OF7w=Y-43!Q`T<q?=aiGfGFt=8`{*FGMXDEezafn$UWOu!JWq|s`2RedXHuX
zPt~HJ<c6h+J`C5p*RWYmU7#-8({kbfgSX_0HV0P=OGz6WiIh}H5mCvoHQ>y3S^bjp
zd>u$IEOLL8kdkuYib!#>@u#m>jg2NHrKdMc74F&a<zP!lzkz{~(kz7&t?X+&IT8&F
z4Gatn{y66@Qkul7!nkmm=$1WwF>?$K-D#e1s3Xec>xq@G>{ofEL`x*R4LGxi)z;L3
z!9`%_pM->j3l|e7OidLKOS`e9^Wi1cHgWwIhGA^rNM2MqyScUX<dqORl_Q6)hPr!p
zPVekw46AiGP#>YqxO48u`HaUGh~4U7jp}Au_Eg`j@XIvKoE)ypN>45prgTp@9(Pgw
z-UL}@dD(*!PaK{tGBNE?1QlYxr$4-)*miff<hS?f6(1TNzK_3kOcr8c?#YAa&L!oq
z7nIJ<Okb|4b?O*HZuKE<9*%@2b5jXP$pqHJ^-H=MI2X74;Lqk#;L5rZBpY`oA+3DM
zHr6w?Cm0f0UwIYlTxHyo*A(jcKXI!xvzs*Q-aiV;HZx~R%E(9@IV!25B4L)3u%oc~
z;dyzZ9z`)|U?)C$@GPyo__DXpmTlX@9zJOi|N8%-VuzGN#$u@nEC(BIWjN$}Ch&w9
zC`s~}bNMQ7iu<wQNc|Oy<Rf8`3ru1Xdj3mrJ@Dmqxzw~o&Z2XU#fED-1r1wu7cD)i
zpmgZK`STl+SPwrvZ4x5|Y9s#tzsM65xdOtEi_6VB`nQ`IPE1a1HNP%7;g!gieXs4W
zur&H8x!V_6H!y0-E||`{AyBiz>VOWj#KU#2LhWkYA%!WCPoABqzhcIs9K)6Az%%dM
zh8Cf)22;lcjE0773l!Mo<l2^cOX|g?{P@(W<_j(H7S%mUNK2bCUHsEm)9$`)S1t)9
zr@4Jpa?@%wx0yLXh?lu&X@gYdhi_^{HOk@I9U)D3_*_j{j%qBq#B{TuVlJz>@KN~{
ztpd!<LFMApL?)c(EZMH8?r6lt&&I|U7snPK&lVTgrs^$mZcbvw_r8bs?Pc61K$|9u
ziXJ5-CI$rybMu(+b8ij^6iiBLx~Aq)*L+-Q&qPK}oeFDaDftZvu9{{6t9jV2xHPDA
z#cf?YRXgJ>-;^S`vjsUp8CyB`96og5zySt-ffY+x!;UAMe8UpVX_SzhTu_$w;6cKJ
zXK5B?JrA#HYncZ^im63uAX_h8n#9AsnUBLDEJ(;@-Ia&?oV4U76x(}!Is944&9+Hs
zM~{Z%S9wnsUDgAqMW6iYV6@EIIF*;j!&Q)Zl@}Y^lohUWPYxt2d^2ejdwL=!MwM%m
zg|TsSFE6vUHnX(!;f2mT`f&lqwa}Cl1q$PkU}0|l?R<P@lP5oJHWihepd`)oAeT=q
zVs`V#|Mfb51#?Q-nX^td2DF8+8Z0>ZNJD5t*u@ik2YgsEv;?^(RBPY3WgehubCo?!
zC^6x{bgv1nGiFHW=tx9FNJvP6tKy%NvR*Pna*ZD-Elrpz%E!H#i^pW*<j2V=DNR>6
zHeAf!aee~(zJJC>|9=0U&uMn4A*gvlU%OIzA={dnLYqqz*KJB@xtbAib-{9{C5L#z
z-=(*d2PNnxu~mF(HBarx?|P&eexYHbVaUOEcX{&f2Q9A!b$R~(U*z{lK<pVe_hv3m
z!{7jch9x`^zZ$+vC~_^F%8~Pf|6Z-@xw0OQm^o%jrD6gbJEFEJUs2*ra<gR4y7O35
z{omqOP8lxm92Ae*I*H$!ps`1vIdl4h^>!D6c=V3?`0yM%#`FK5Va@+@8`Akp%}zi=
z)#=fL=g+w~4g0$`1%!w+sj^m_N;srp`R}lr@4v&RIN5r7_)gB{@!(|5?m9TD!65ip
zJ)4du^K`v4M$FSi$|7YrMY`iZta+=j?7#t$l%ymB)6^fI+aKOGZ*y*6v0)kyBp^)+
z3yL@Y{3WHL60v#(hp{23R8?HcllEjvW6Z(>&wT^98Qf;L{Wzk@?4v)$Ex7H8fLqSQ
zcD3&1@m-Nx1_>FCuZ0tJTLN}?eC1GHXU4uKAtB+wrPQNQM;~Z{;_u8EiGB6yKfZQ9
z{8898-+m)2+eN7FGzvFt-n?UXwz}%g%*;+>W8*_ok~}L?*^0I@v1d%>`1FIFxu-j9
zt8&}sk0ItWwh06^toA!N`J4pDQl2MMy6#<Zn0wIuqW6Zajy}H<Qd2i<Yg^WDoUS2d
z&kFXJl6~QZ4F<PvX&pU&TUzquh7AS=3ZX0>b0gFuPBr{sh|4ihI=jhe5#JnTABWUr
z*{v<Fo7fGng{c%hNz_&Oz_3PZVlV@<7IXI0mdGCux5$P+bgcRH;7Q7ct*I5ir#<|f
z-j;p+#QQxEuPE6Uu2|9G;&Nl=Z0!vj4HL`sCY<6?TcX5x$*@uXgG=^qN5)OzhCzqx
zdoCQG_1MsP{*PzYQB(dO-@p;Rv2nSD<BT@R%Rj1D&0q-s<#0_^;lP78XFfdPJpB7R
zkN^BhH!C68I!Vc0n3<WKJv(E?*2$Bl<mI`8W+gbh+Pz4W;X_m#=c*>b%_mPX2&XtC
zD>F_>VPXDXc-;1y#Gz9MSPfD)u1Y_$mZ4j;hoj+x<{TF>#m>evQzs^Ja3vX-rza#I
zer<mT>J#Ha78Z$?7KyIz;sbYCm^%*oKACdx1V{dgB(V(w4G~Wobt4mc>tZsS5}rI{
zbkTAS?y*l@urV;p$1P&A>^!xCrJ`%N46ZulCZ&Mt@e2tZy{QJKsR>Hj^B}%Za-TeP
zs*h9C-bG6GZRQ56*($UWorKz#b5^|h=l)HU?LfR*(llO`Om#QLy3Z4rN)#4PWWD%F
z_*=yBP?ZE<`5Dg)jEoEn4J9NESy&BMteW)5RRZdA^+FdnD{0Bj%^Uu-n+h2g8z^0y
zxL98E(Zi~Xd)igsbTD@ab(HLV^72qrPQp}I=ZKI;o?<S`_+yUpu(LO>)^1*^%4}?W
z(vAg^dUOj73=NMRjr{#%MNzTw!C0QJO3KW8vKl%M-~7N@!%^t-RzTQ5NNirl;Wy0-
zHhwhdU`bI4-l1o+lCe5UF{0c+V4|JDSwZ1s53eKxbM>3jP)F((&Y0n`Z0XJJjtvLB
zxt}cLxDskpF!BG2Zu5m4^BN>1FMeC~fbqZKBFXSy3LGc26CN;ZzTuGfFiBOiV2P-V
zn85Wb4U>5!JXvmPh>Eto$Y67IRnlz-)vgTx|0^jL78Gn?VYkl8{%l~lkk?YdOh93)
z_C#}qgO}M;x&;?aIB@ZhgEz~LS2`BrlmE&qe+`%+rN9t=MJVUR15dGtQ~Vap_1se$
z8RvGd<Zg<HkhpV4qNLPwbqv()W`!Fz8U6h6!qs)<=8XpzPh(W(dk}NPtMJOy`x@z{
zilP!c9=9$EFt2%YaI%Ysmi-BDRhMrK7raDb${uDm85lfR>v$ohb4{d#q~wpEl3H34
zTef(-*a<Z|Y~h?a4lXTQ8yGhI`hTLZ$7hjOqOaut3n4;=0ga)Co@x^mxfy$o6n-=h
zoOqP;`obNT*w?u<OzmD47&!6Z*<MEnhW@P@P_wiO3ko+{SiQ2a;yUaf+&Ga(BK1K1
zl3ERm?+cPvxx4<~z;=Io{(iffA0PkzeqaCJpquN-BJO&rMPdg+rx_%z`uTf0sEFj3
z|5yLtZ<pzkF98#_CR}qobB4##l4sSbBYfS^l$`Y=COBBh=n})eVBVEGKlD$WD7luy
z(M$5gaWR9WB}?i(rm|k+dTP&n`2YU;|9dMxKYepky<e{O$CsC&{<ESYpQoo@Wo4YC
zq@11Ik8NITOLiQ;<&&78ptbgar$f(;B@dcrZ0h;{|9}1efBzS|*UQ=d`uyqXeSZ0W
zhK7HiKmM(+e?BASh=Rt66K8xvl3HDK*JWBR;T6zOir5(w!=tImqpQoqFE6xB9Fm~t
zSX5NpShLo&q-2L7lenFCQ|E)(^A?(F?@?~P8FO^v-T%f0fBygYczkEEf7$zewJ$C-
zYlp|x{rNGuv9WRH%)sE_8;6}wp7wSO5040#&^vP`<JH^~4B20ljX#9D&B*$oF4%J7
z{{H{*dw-RGd$V5Fy6)Z0&!4}&-9O>#LnbDkU%!0%`h5KSdVc-lsj0De<8I6s{VT!2
zi*bh1j&JvW{W4k;bMnI@)`y(X_Jq=%%|>R^-oE)_WGtMTl$4f~l#-B;mXeZ`kkF)i
zy}@9{18qx%O*}jr#+nAx6`$vPIop%*;^yZ5-R1YAHavK9aWQ}W|6`k8-8<s4XVIZ6
zHzTL@_p_F*xMSaw(w=%<g-v{VQqq*fwYLBN|NZ^`?QQ$l*Xw6neLem3bVS+GjT;Sb
z+%Qm8HMFue_VD;~d|QYh_W_&5jN0O>IgD;?ZeO^l>ET6Xrfbkd5K#2t<HqR7l`B?$
zG%$2j51DAbRq3kc{{+qk&SS3czW+~e=$+*Frz0?J!F2ij`*pGV{v3L8^0=JcpN7-9
z)Ajb|Wm^9H{XJe^ztYZ5OlB+FwW3G%Y||M%RXrcrx0M|O#r}^E`A56n2QRPNSuOs%
z?r8jeiEnRzmw(qg{QA02VIhx`lS`!AhDQo%PXGVcvvG5?<>t19ujg1J25BcLT~G)O
z68iXM%Jav`t^JXP3${sbkX8P}|FUG>{t3EAC8B2<t`==kw6FVfZ^Of^*t>Z)n$3-k
z55K>!H_zKM+xq*dPT}+aoUZF6EaPYEi8wwr#vn9F^dL)yax72%f1{k+-)e7eTF)={
z=ivTnj~+U=ZQ4{Hz3q>B=1H-6a$MJrX)Y_%(`#Fy!6qokBnAnwL(?sG?1<pueL7`o
zWn%ho$M&5C=HeIn^OI9rlycHPCF(!_|L<@4n;XrWHs#*mC%9wVlmGQ?@%#VHv-(<e
zeVzMsy}buLBzYvx{q9#@DBz<S;VrwSVPa$B!v_cDtH1r(Q}oo_aP@|||7jXJf9gLd
zZ1n!c@hxCN=p!c|A09R~o>{Ya-n{wo22^7H|9@z@#gZi~GBQhATW=;Dj?@V3Hfw9_
z+%R`x!)meg27`Zpe^);~cKX}5zwhr0Uw!`jd;i1R+waddzrVlq_2J*&?F(J(|9Q3a
zd|_nNn5s6X=HsLHKR@sP_Wu9Bui6{GUitl<=k8se>H5OE=L^<P7d4)mVCv}5aPV|@
z<9zw~Mv$WF&`gVnNXe~RRJLwqP3+{}Y38W!I>FtvfT!l6lSb=>-{0?FTer97$A{WS
z;d+06e*XXO@A>2X`zt;?5Z?BQhi}6)Rh}Cnpqi#&n!LE)ADiN5&mSLG(K-A3|NMt9
z0^6o)OWxbB9>)F8h&fnh^M{1wWCPRGfS{y=gLmIRYPmx*Em~T3=<2Gfs9G92%d9=f
znD1|3WOQIve|^O9bB2>xy*fYr|G)om`}?(##Xq0>UlOx5GBkX5<Kv5$m-Xk{{XNuC
zeMaJ_YWK<+jQ&aow0=K3dp~-6oxHW%r)3TA?@OBHr`_1t-14=if^DC{YjK|gM~*1y
zoqO=&#DTNDmqD#+hX4N$%>>24*Do#=6%q+g8PmiC*H(U%FQ0Wl$L2wmVD7~D{dM_w
zD&Ahxoy33k_s`GoH>cmvyZ`UYtE=X0Y-~YNG8HM3212~c+)7sNF0a48_xJ22Nz8oj
z?j}d<ob=>y#0PcOKU4Yi^_x36n>QHv9fl^sc@{f%Mt=S3`t+%yZN!s_+GjMwToyU*
z-ouo?v2khC&Vq+G8=1e?{Zo0qMoLWVn6mr4`X3*J%M!S*O=+|*Jisa$@K|oOVvmHI
z;h+EKKm7Q3pIf|c;&!$7-}BpSt0m6OQFP=!Ske%x8q?C*=@>QZ;DLt5X6D^Lp>h6>
zpOG;yIXSSfP;7}nqnpZ`D(NS?*i>Z`=BV+#U!|~kzl7(L`{(AG%iAdIy5`Dyf8Srr
z@^@#iuiwAF_IK}<6RvDWrU?I-t-OQ5M6&kJkLPc1@1L$Waev^G-{oyzUwdA5+VHb;
zKew{t!Gy%d&1rBCySVNQ4Q;(~GjI}X7I(8lKC5QupQ9eyCs^5>Qm(JNy@ZuV=*Q2`
z_ut?DKhOUEw};G2+&i>2A82tFnEDzlHQ;URvakR5{p)LcdAmQ0p9oBKZkO0stL)3Y
zPEFmgH%w2DO;wf6(D02Kq|iAu$zsQjh_7E;MMZaBh!S0KHfT-nsd|sOaW<aM?cIca
z{{Q^^{p#@eiDFtS8m3Ntd9Ydk{=UBlW0yZzsUuMku2In7EyHzh-{0SVe#VD|dD$E+
zcrA8#ZL~_h@xl9rN0+iL^7iXHcws{0WZ`ZpXrRurxUprctBZ?7;#%H6cLQUB>K`6n
z>|$d5Z?4_nCy(8`%KrWP`}_U3xBdI;{~w*m^zUI`_#aU&y~r&gM?bv0yq}%FZu5o-
za)BNX4zsrfFPAv6dz*y+f7bqvB`VErY~L;#K&_c%QBisG>Xnv=$cwA^J<c}W;c$ID
z`_J*)e6D^PGc=$5KRbK9rcKbJ-{1H9&#Q6u<@#T6<b{3D6Q`QiLk)>iB7c8<{eN=u
zevw~~{@1rjo1eI;(0_A-!{np4)KU`?64KKrOg(D16B2|@{xWOU`0U!%q@^`SLXvND
zl;g_02boNDhJqhoyuWXMeqODLrA@%W_3{7r)&D<z=upQ*sS84kQD2g$X=KQyAGmwP
zf$hJfq@-QlpWpKjT{(E*!yCz&XZhF`O=e{d@@{QxbPS#9=*@kaO9A3DCwaN#w6vfw
zF(L8vlmv&oFvlKu!Ne4o2?l~smQF1ABH!84@P9>5ef_^bmA|JQ`EY<?!;ymxYtJaG
z<UG{iEaNNSeem<g$MvWBW2(OQJvqY2*wgc3^Hv@njkDr0kBp3Xii&u2bY4V%f~3bK
zF@{_R6nOUNFkWxk{Qv&~zJ;0Wt5t4U-4I}6Q*2)DV*8F&MpO38tFI@lEgu_2*)$|q
z9=u?&=6mMhws(P{rSB^b&+6!!6KZ!`f8n;M_;fZkfi;XQ^DEn2G@f1$T^e=Hde;$`
z)n?Ofeg8X0-HD?^@QkuXKDZU+v_9;}QOlCj%8jA&C#G^ddm0%Ly{vv?;YJgS88iMB
z7fjfv$SrDMY`kfsqQ1FeM9~8w_sK~I7S3b5TEHeI#>U4t=XveKM}jq<Wq&ZP3*?RI
zddS1e%N7_|0_t-z{QvLdFQcX9<mIK~QsOq@w&Z_9mkyVJULBqPtt|yTmjrol+?-ia
z5fSj1|E8Ep;hIdQ&C3!RIE@5!l4s16WMQ3ovM=1?!NY_1HmY!N^kpe{D{7}DBwV<X
z5)j~U|0<-;S`ur>b-;jQPeQwZmZI~WPyfT+j6_tHIC{ORDC(AZ^zwlg-^4kaXLa7a
zDRB7hv1{}Fs}yt07v?+PUOt7%tL56e$yGjr%7*c);;WjQ4~O<&&+G{B`jMH+c%^H~
zxtIIYbf!hRu$<A(hJ=>CjFA!3nKK{!*rV1mo{ihl#eSoA_Kry$`{H_*E>&q|H8MBv
zo+BfEYlq#c$AMgyIV)N8jy4GXopG|2(etJ~TigGb7@k$DbVT@?3=Ir2G7~%1l^C!6
z)B1O?lp`TAQ9!^UoD*V^r;LvekDJ>CHufAr@u(Sx^v}1N+ieIFYYAqKda`o$=hLS@
zPn1;Q0f9Sr&RALf$;s)F&}86u3O?r062}&!#KhEd>(-i#^u3e0SXd-{d`{T-G4ow`
z#kYnx^hu-@$A#<e+ac{>r-v(7a;{impzJP}^|NuB?8QkZ;+H&a_-_!zc5N2x&aR*N
z>FU0o9R|k6uU>xq^x?rx-Nvb>8v++OFf21VQ_`HYY=y?+Ra&3l-@mb_DPzIPlM7aQ
zb~658xOP>pV8Pn63aYJ2ka%+Plwo4J^XO5AVrxvQ!}=GhI~)X}zx;R3t^echtM^}0
z;$Lb?#^krTa&lsVf@YPKesOVPIak@2aa@y{oWRJ;cw=g3hk=%cL3fwY!}uAU@-A*h
z9$u5y%W5CE{~)13RPNNN7mpL5dDm0s&z~3{pAL(|Wv#|Vfgk1F*E&9#!ewP-EWCN!
zG==m3gMwu2?1avo;gOhr_pv+A5m95Fif&$w=7}HoPY_6UVPTh-pJ~c%Z|_`XRT;IB
z`O)mco*tf<m>0&-QF$j%87C*1|Nj<NNZ+ntt2&ZwV$HGco<sIThn;86{D1x0S4Gt`
zZ{_BVhMirTDl0lZe%_qmqMLTJAxtA_8QYpvb({5xzBK`+moD}A`K{Trr{_s&Mt1f|
z`#Zdn0x?e-mkIG*xF!uP_B>@gR#*Q!s3qva_2d74{a3DK^^y~+G?w%fEclxL|M1Rl
z--IF~qwd^U!n$^kO}a}<&khS|c6Mbq#rCk;gk@}No-$5k<uBUx^ZbDi^V{1UBWG$V
zHpWKG*m9`3v-9BLW<}jz72g|-A2XSX#oYY(Tp{fsClx+3Gqu1#HMe8`H~etRk1#Mg
z6h7&ntJLQo{mQ2<z7n?y-c<ka<iy@yMdM&)MaQXAr{4IstAk_q%$W%Z3s~gkjSF3w
z=BYRtq%P1*IIzVpXp*5q!_^BDW}fifz{A7i<HK|98c%HOlkW+Ij9U)!{Fz)(|IgsW
z>BT(fpo!&3;{t`*9URKS!d?}MU&9nSpV=R)`5WJ1(YNY}zMHDkf@6Y0Nx3;DCr$(?
ztFeFhv}wjn&TU&Vx;i&1`tDRzKJ)0IeBX>?k&yzEQXRz7lYg&Yb?VBcLn{{__IVO9
zZL#v<7EYd9Pt1=OGX1$LZ1Um1ee)KRQ^)e4$w!5ck58<wjx8<iOis%Kr(Zf5f!-^e
zes;I~OVADvSS8_htp3oUHNxt%wr!VwWWTthzyIjPz?5S=wUUx$VPR}NERFk%4u(#B
zpjFi285E$&*Z1GbipMy;XvR##$B{qIA2@K}$PoeY;|A$`%?lNmCQD~9gl8SFw^UbZ
zy7U54A0#<VN={1tef|)Wrhx%B^BQkQ!_%Gh7F}F6SGmL@-Hz2ieLCxS|L)ndxHfKF
z$S%#zy||03JZ##dNvds^F0Ikg`Cn+wuN(R2xLQZ#u?IoTGgoa;mn!@Af7Y!3=6NaI
zor%Y`I!*5HPrR(w#>y=rB61?~c{}@q`3sM-&X_4_VG&>giG?J`Ny$y6p$p6#55H?l
zNN|w(#lO>@t<GLt_|+8~Vdlf}TW;1|3H>u8k)iywy@YsR!0cIEGFDGY9vzwe_V)bv
zou58FJ|2JOOwXiAGHPl}=gujyvDNVW;N$$oZY;J|=U}MK1IHOl1)0s911_-}f3A<a
z_y6)T!{f(|b8dfo|Kj3*|9J*~ejPn<_UwYSX9Yx#Jb3H-*Nv^N!TzX$k<p)@&69kf
zVdXgO_xFCrXT>L$mi0&!_H1)`ET8r0NB=^GnJ(o#KBwz%WGPf^YUnmDC@`?F;IOvf
z`1|+w`{Vul|Nr~@_(_UWZk$0vw@u8%F7<sMH&0}hyl{zeT@<I$|Afqw37IE<JU{<`
zeY~WEgv5>=5*qXC5AvUg>-qLs-Rmu6cFf6BrvCq-&8(o{aC1vrqFpO!o^XJRzaq17
zPTquWE)z4q{R@mHC^H;PO-pN-)8p-O?pTkONg>;wPUe2jAD4Nrg>i3rk(8*oA$#Vv
zh8ZHw+k_y)&`v6S|Nl?)?)&et@}YdnTJcx09RDY3$+G3V-giNWi@ER1NBOCL<)xb?
zTvAem52rb->XGQU)~BB;&?hFvye9jA!O{ytpcO~||2wJl)&KwZ_vh#S#)tp^|NpOG
z{#0SAz>HMpTS9?bmNe`LWbI6GNLp<9&Hjvx)-wJrE0PXOy!&v|k>zr0ns#taYV?=n
zxnZRr!98idePCZBERXc<umAu5C$pzy$C8G{PZ{^iFz+mK4ErFj9Qj~X*o8|h%%(!@
zMYS#|#VtQBu?BP5mj^I!QTg;){nS>_%qpm9urBQ1;VSJ2>4pC%{!;r;_QL*#(wF}$
zx+ZoDSUC8}|2OOsNMTDmmEh1T*TJz)s`12?hLx=KcU#nVJ_xdzz;Hf^FNPsE(HK%+
zg3OXy^5LEKmPoEn?<GMkf-7SjwT(BFrAth3*}(MYC>wK@i&26Lk4^ui5V?dbm({f&
zLd+v3GtGMAz;q&j{oj;E&EL=Yy|%(r{*V8!#ZxpmFFi;|O8LW`J<aIAF@ptPPyGLX
zq55vZTcHly8J;W;qhw+#lnw;%eBdv}qAX_|C&?kI6){<4lUqz1Pgfb6p(eBTkAJdD
zKR^R5bHcxWyk8Y-PH>9}oLO;IJ<9rv{kul}Kb|an2A5fww|!#vKXo8}&*Z2%38Lu^
zz11-%lNy5ASOk0|kC=EsQoGa2u)6>My06SjXt*ld16o8-%2hP0AzW`l8hcdhp@xI0
z4x-0PWQ&9j`l$X7Jo@ptwq$3UilEKXa|f6|1s(hUpJ#vFLYP_JG7~Z*ez;!r@mL(!
zvw=^`h`BhYnbWkPU@5;&Uh~8mvQcML4mDilI(YTjfyq@H%seCmxS84Zq@GE5Gh;!h
zhDxa3{`$pJWT3wDl(DZ@nl5v~dC>>!Cdmu0M3z{JdpvB6`CuThD(s)KB*$~31w|Xx
z_=3Cz*Cg}r*z&-im&c;3_aN(sImNv@6NTqB?5lU52x>U~|L^22GeNWT*#G~6$8R~i
zIsN~ie!xnE&0F$>^(sjNFA<-Hn9U1X-_+-*tYc|ZH#HEL9@8P1av*RHPtE$4kcQU_
zN;rOeWL7PNx=i14LYw=-r|imG!hZHo(<$I-H#l&EN22M+ot{$;&xC^OCQM<vaEjk%
zuY}D}{!Pm5{2<M1;hO*b{eAw(7k<lMf$ioG*7C9~2@GV*>E&k&VGGk^&Jsvo#+hQ4
zR&r=3&x)Tj_9QhYOf;6bJx_(v=57D~|C0ayB>wm(>&gpBr!y;le|-FYd;b55O2OkA
z2WBlgu~vZX#+2LfIeR177D+EW!F7R)Cq-}zQ!``v9wP&Rj^rdMnH@(AcW-s(5y+Xj
zum1dp&+1*!(GsVb(Ut%Iy+7XnzoK%Y^z0K$kJk5Cim+Yz)Xcb;L-Fk26+9-BW*l+!
zIWM+fD#=Hps$s^$OV7_s-rM7tSq$}@O5gqc|LyDl{l2?<zmbv81U7b^)C1O<Jv<#V
zt17q-#QZwy;F1fPZT;~5{r~Ih|8sC}-mu9?iPg%8d2(+g!-N(NGfT+=?Zw%f8hWe@
zjExP9gLidIIC1aIWOW|>_(dAfLei(_@Bi~3K0f|`ef>qVT?-XdG}m}L{#QPk^e)iB
z-Mz0SFRyN2@pJZ^==6^n?s*bI{r{BSHXeADlVbko_=Kh9jfKzooNH|lw{nXt<w;_@
z_>k+}{d9-j^Fb5Z|NlGPwEOb*_I~sHe+C8{{`^1BGSjm0q=C{@)r`l>c}~Re%zyCZ
z=KudZ=J|E;`~Lj>|NsA+Hx@sCC}^L(=*68PxQ553H`qk4vX!ey;ezOzh=T^1LWlR)
z^Bg;-vV?!jN1ik1wB&mE<rkel1t}VwK*{{y|4&c(FZpm4$$p&AXyt#Jfm{E{m4@sG
z+{?G`tNZ)?>FN3F<Ntek85tWZ_n%(i%DL`H<9=U<6x*hiZH!xQNlAcO9Mi-O-`~%p
zA8)j$uJ?0GVoa5xFw4B(_Z~br%+9vu@;hh*dCJuP`}6<f<M;RX|Br}}NO<``e$9ad
z2idm?2RsBKF0-UfXW^PL^Z)xBA0s_FBHh|rySo_|zi{)GS+a~jCY71FaOsHy%!Y<+
zm6dJQ<r4F3lLbUo%ff{!y}K3P&OSf?;#O!g^QO?J=jZqL%m1(W+jXa^;mxF@RTs|N
zFA-xf{A%m;<o*5n+uLgIZGJwxg{8ZlUHx8h?#G-JNA5IuUXY17%eP}Wo9n5Bl$3yg
zgbUZxE$ZexeB`<)uRy8#sHMx}&FMVz?G{SIL+}2+ztulJyl>~9(HoG|7xIJK+j&M|
z^RhSZmwMk%KmYH*$&)jt9MRT1*)oyi!}|uO;Lq~TGOy#TV%|xXXfiV`St2oKj>L~2
z5?0od=jN$yyX#Q)u<*0m;m5~ctb`^9P>pP4Q}yNj+vqPDZKBc|2e|@2|Nmd#;U!Z$
z$3COB_y7O@clZ7-KG@X1-0yEe!Gn&TCx<5LbZB~SQ0u5)u~h2dmaq8%OF1nP($YG5
zQZ8IiulUlDcuJ%#`?^O*>7QI~lc@rq-bA*YpMNnGS}5Po{PFGW{q^zxlaBnYmk{p@
z&=RrP8=vy!{r&r^!|Q!!eJMQJr9WxXnVB<fnwofAp5&}4>|ZYN!})~jiMJOHa<i~V
zNJ&eEgi4mZNtj{M`S6)<Qa;BKt`Ey+Ecf3yrw6on;{Sgqd(Owdzt@8@>+k6kXG%`W
zn{fPu;9apRnW1b;981_#*fue>`Op6+Yxk%4VAJ~D<^L1X(oURwxNUp7^Q+|ves_!o
zvbvOtC%$g2w^+g!SUuSzA?M75i3cxSJ-VSp^YH2EJgKQS*6&~NWY5?C|0V6}k6n^V
zO@Q^N{AJ?z|EvD@=l%QpJ!j7FL_B{Hc)i%*fXJWr;{VR;9omZj|Nrsv|GByL(cAyU
zY<zTkZS;OcW!?Ap{yu$rR5bL&qZ4oHB{G{BmETDgsB|b!(@|)cKHV{9&cnaeZR_Jf
z-mdsFLs75q;Q4+>Kfb>XkxH)xO&9{pp7VX11WBNWd}RLX$NUf%_|VUIt335U#P`;F
z{jSkx8cjYinil;1_5Snodh3!G;(Mge9y+vO*|9(W|J%=SJL9J#85_xVCo}1VDmTZE
z@BI%SySM4bOGrpa)cs4bsGs-nFne3}b@0SqiG+%ZL`6lxyt@u5Q9u9Be|Sc+O<aG&
z3TPDj%hdlfh}r*d`b<eXzGGkh+qX2@IGtippU7taPg(lYySx8SPya7t@ge(Y*Zc49
z|37`28WNQB?|c8lcX#*m@bL8X@bt_5ZCtDVt6#Y=K};@Uu}pT5gI;o*K|)H(jzZ=t
z^Zt#7&8xHyH?#9(Uo&`f@9cvYClazxC*+(_&^&qI+_?j%yb{FN{AWKt$o%HJ38Zwm
z5@QHjDLtoKXf2DiF3Z9FKSUfq$lEz*OWQw`f8%m7$dBc*>Gn9TW{;$<DKoQH-%UzS
zoDuu&(eG;}xlTFDcFH%%9zNS}JF~g|1UFkkWWe<++qSIBns?P8$JaU5z1BZSiRXP_
z3Rj~^?Hg5|W=lThpjl;k56!nWYRf%n=9=I1_`Ra)rZ=L|Vv|@NcWiPAU8phP`Gryz
z#u)K!yyu-|4u7vdv0lW2{rUTQFZw`B-v9qUG|A%Lo}c9(AKA0ByItXoU~@aR-{ZB7
zgo0-ac>UwC&FS_2^Zq>k_jmvOeSiP|`T76$_WzX?5>uzlSiI<ltV{$iPee=WOg@<(
zr9X|XaOULjY*1ddpwC4~GBB8dxsT;w!$h^VGTtUpoBDGK3I{%XQe|dlKHSa^>TmM!
z@WjROgs(R`GwbMvVs+30-R*gXcXl5?aOB8=GiMYuP6Pxtrs^t$F$nIjoBQyd?HloS
zNaTZDz4v$hrzi2%)oM5LR9{Tui)d=_UwOb>_0Rs&*Wd5%uK)Kx{mH{b5%Kg*8xt>G
zNii}?+PE=MRIrh6pMCAG@YuO8pPjAe;o*tj_oqGQR-1)TXUc(#VF%(nc@C!>JaFK{
zfBVB17xS!7xUR&;W>oW=&Gw#i^z4NjCoWvj@bG1D+v(|&ar;t!{F(XizCD||dRzT}
zi5W9E&a5>sP!NgPf9}H@$!~t3M#2C8cjkhVgzNottCmP?h`QjFySL^0{<hY+t*v6G
z-Yw%;(>X&%VgKxv@`u|S|K~ptNR&BT7$9QEqMT%Bo8}YI;bL&lm?!M(#MCKEGo@~q
zzK_fljF`6KR5#CpRXtl*UVm<`H2vf3+drHu&)dvr{l04X(i6pV4qM)uwl6*QX1UXX
zb?a+=x2G%XZtu<9RrhN0JaDV$P~D^FyUYK_?M<38tt~J7=l}oudmOp+btM+u7L=GV
zQ!+GE^6Hg{1orEOhqUTg9z0IoQSq_AgX8gR^Z$|(ptTn}cS`>I^Yi$jCYirHD^@l(
z1{|K8xI>Mb1GI?dil#=iS65>@`{}C}KZH0HToM&_SoZS$or5ziAerRQCEj1(`ybx3
zZL9ts&}R4l|Ns7S6%nb#72?9qOF@gfe}1+vEnUVhzo75$@97VJmrq%JE>UG-hK0C9
z!G~9(hp(;<_;!Ck&$ewW=Qvc<Oc#V|m%lT(v#a&<HP|3{Tv5%R6CXY*Z>i*XdFl|C
z_LHx6AHFFwugPSRXKZHnSUk0{_38hK%JnyHEO@u0{QdkL7JH%+4_q`%ba+)|&@+LB
zk-h%E(HzUeAHHZ#(N;R7pwqLC%~JEg%^h95K1c2AL2IHvd{p)`hx*5^;nQpJ!=Il|
z5Yzco!lqPwxNq07dG9AKS@0<T{=dgh|MI`|F*Y*F$TG66`qJO|(ted&)0>Htj1(T6
zlWjB4Uyvg#+&16d(~Qyo@2CIvpmP8E`p3IFg&<Ki&%&-g?Z#H_suy?v|K~Z;$zUhi
z{x*qs$|+XOyMNNt|NsB+u>J(&ivRzOYkq%=5BOl;=62@iHo;js5)u;k_9R-A@mT#i
zaP+8x#t8+5hIMS`btNkpY}VS;o+@}JvyZzEnx86j-d$eKbAO-Wb^j9|+4sKSza((5
zC%k8><Ntg6{~tfvy5e?Ai^RSC|G$6LPE%|Q^Xu7R!pOcoU-{LP`27+kB@z_|c?X^K
zgdU_93EUL<`*-%kui0(e^FL}r>x!8cAg9#H9JR2R-1t!bQd9r`6%)DiEdvD1zyH?`
z|L1IVxu=Kc`}_aTpQI$DACKzn`2x;`Z>0}^ea+)P&td1GLp-~7{W!Ksf#dTrS)0Gp
z&(D|SlR0qxK?1BsJT%i{UrpkU3c;fmJzqLByaR5_N6b>_m-{O$`j_9_-J#*(-QEA~
z>;4?SV9EumZtUvQEb3$o|14eB_H<d~nzI~_w5QA4OZ3UR+eZt5ItQSszjCSHo6}#c
zZZ_BzG(n9c?Kkt%8wn|Y^mYEczh~Q`SpVOs=HH+H_xIN?KFK8f=jY*q_i{-s&7GZ%
z{r!$nQm4Oa967TiNmeU0A;F@w>)~Z}COv3y&$RgWFLg&D^Q|ea?_`p1vA=$*q|ULW
zu&=)S{lE74_kaJlKin<OvuDr$|NoX|WL_>{wXZjd*>T{*FViW%-^({o6l~sMAvW~{
z``JZD1&t2KSp5G#Kk=zoi|7t$ikxZj?|*v5uPF~-haXZEHDlG_Y@PVgUR-=p=$Ru&
z6jY8B{QsA6>Hy!peW@0elO7&s*9ecA<rpn}`iR6jJ}tS(9ZkE-1=q&fKurr@yQltq
z!5fJyE?c6dKJ?#6?%H3`_+r(c-^V|E)jqu3pQrv`z|Tn&6B+i^r|+l|y(pECk|H3Q
zI$?rCezCy)Z3zd$igNDzg*<{9CvJIvpP`I3TW`pgl{P-=O8vb_T92Qn|M(@U_~|h_
zv#{{t)6*^7zq+e|_Er2YZ%aNd;Cu1^f60gl2_D{!^Zk8&zBaCWeb>6JTfCF25@M**
zgyQ@AQY*gmWogWs{b9?ri#4C#%a?F!vlti|ZP;XFQ*q$HIlm~r1*ShfFkY;d5%}<&
z|M1(}JmPu^Zn}{X5;e6OV-6ndD7kI&?_klHj@>mgAD)wav%2~M%yf%&@##N4vD%s@
zlr8(%yuV&@U%k7@N`;?~pZmAD_e=cyC%fUx$LIW@4upNZkxtYR0ii<+793cx?AU_U
zrxg?rE?9f^!u8`HUW<qD-aT-8J7{OX9IMtBiC&(G`;T|__;Il~MD6yUFC_W~V&j&%
zhI|eVA};gWqLiu`*bC&@3%GiPbJt(6XgVT##Lac<&#tQ)Zzf%s!Tl%x?ELvBQ{ud=
zTqWE;*~}@4{cd_}rb^1p<-M*Ord6gzp4+A*ZdF#F7<<!fQO7>_>|K0<-|hz291FN^
z!kg+Dx=<wZSM};iTP2)tSIMWz1dH(d)i$_W?VsFL>?3^m!`E5z;BsioEL*Nt2c8!D
zn|>cy%DSF!*e<Z=o?J@)f~n#H(t_TKU1yub#O%KP)eKw4tzs}C(WvFl1ZS4y6Pu>@
z?G*fH-J|^Xr-@}!tl+`#_h(F5l)G_SZt^_NrsZ>1GAlb>Ht73eo7!pUBkUE+)x0&L
z`&x!?;DqNcf`vX3c1)K(M~JQJTK9WV=B!C7>R-ci9^~AuFXLq{tu6jtv=O`;Zq3|<
zTx|{lE&GccdD^no470X$=kMWvpd8WSy!)ktfxOZoc@bB>kPr9wB_@`Yl}X-s{(ax!
z0~SKZr+a;<`g<dzhADt+sosbA>4zrlQ}?|n^_-*W?wxNF?ae03B<MPb1YS4cy=LSq
zak#L%dAqVRi{PrRMGBr8-k;`_{yqQwh3>T0b%!NxeY^8vtJ>*zoANF^=ng($=~5>6
zQ>_jhc1z~ka<v5TwAfE=kBIJwd$92N$Buc<^Vl!A<=8ad>R6!k<FHtl-syiF(m||F
z9L6n~hyJL!$6nsH<DGQWo#REZ`mFoR-vw>g_+TT`UU+2NMU!VgFT|^#c|K+Hrgg`?
z&zmT+?DD&8z+=Ie>N)T7wlB7oev=efP8StCogcupbiyPJB@PZ}8~)QyUfbTTD}P-)
zr<#{J)OD87afxHDe3y@WuQZhZTPI=HdD(V%pU&=Tvj)g0<(#>O+)V)j$LseTacoJ?
z=RLUdpkhngG`s9Cfhrt@LM#UzrS2&#bN1uAdLi1#^VP9t>30_I>g(V2&GzqGrg%DH
z{qoM&9Pw(*Un(}lY>EmDbyH@Eb)O`_a>Q_-zeJFTM(dO-ra?)!x9EB)vP3<u)Der(
zv*4SS`fYm5bQ|;C*RN}5dfiWs%<#SZ=W{_ySQnGRc}tlad7VmAk~SyqU%u|F;;p}S
zhFm|VgfGmEZfpEJ^K$3yx4gk+){8O@6}W`$H(9>;QK3!HG<%ERETFA~TjttxHzf!h
ztLKthnzYNr<Ng7|jP3as+P?^!n3{HMVadO*;1wq$CB(uRzw)5M;(C*k$GXRDZOU!R
z&F_3yS=Dv4>8H%)nhV0QbtQ9cJVSgBZ<70v)$M#eQg5c;L?J~E=WE%s+7uMd-|nwZ
zk$i64%oynwdFlP;`<rh+{4K)j^q`U<XwmEKt9`|1*+*JWTdtXTs=`O+v4xA^I&T4k
z*@ZSnQi->(n9getnxW>~93;V3>bw5<OP9a}BCADj>(1s4y64Q(?xQw8<iqaJL*B<?
z^kn!{PV(n6?_dIXZqDr8T+InQt@cYEd|RyG)!yW{%hLK+g5Bh3c6V{}<4t->k5Yxa
z9N8Ur{n7B4xUI7J$&;cYk9{W!UR2)_JASj1RYvmI&kjMSe_wLHRRvzZVdmK+Ai-9s
zGf|IAroD2R&d#`*YHf^)5n{&zpS=i6ls@^_zb8D@rvEl?tb6EEmCL0za=&kW-)YL*
zuxP^b8EUiKL<HMn^diLeoi{mIS`u3ryNc`bl1vr9$YTZt`#;w#(%3CN%T1u>UNGOK
zqvmImV?Bdcy_$aEfHcT2Q|9<`H#rEj%`Xa^`0bkE3Fl`glOOPZt1o$b?O^hZdqtXJ
zLaj=Z_@rwiTK0ZVWP6snZ^q2HCnt(eZMt2sKg`NLsq<iomqgIC{70p}^B!B|m_>*M
zEi!ddU~z2}h*_UeoBptwrM_@K^ZCQucE)Vbc{D>qiQ~iH=XZB){8N**Th%vmZN&Qb
zhj-nNt~$2TYoW+tMbD<!Mp7H^Z;mLiF`B8M>U%inSl;o9GY=2EbP1i9y5;Td;9mzD
zEYj@WXT4OJ;$Oonr&g&u{ru9y^L8+T{Ig_^Emw<yK&$<h2}`1MkKQ^Ib};P2ih0fg
zYpz&VaMTN^J`!uYTN)U7qrCB?&pn?>mCkia?)N9B9{G6tUid_dwW{+Y!sYs%({iNS
z4qCk0wkyc$*Xc#u|1UlIr{3y@*}V2CGt?X%B!padto!%k@+K4I?_b~B^50kYotge>
zmd)~2U#fr9m1G@%_t+%U$(+~W_979d01*qm*mePpqXr639ithRsCb^68lm@2=6i{z
z_pE0{r~YvHgUc-?twI9>gP471K0IJdGU4$*+hqE+qr*T#+BoM{i;~CP$jBX6=gi^x
z^M~jDKEn+g3?d>8GqQ|0I1LL54K*|jUc3a6e}1#e&OCZwyp^YK<tif;6@xhz2SFRx
zetzbO-*0qh@9BcSRwvIjyYQ&&bg;_%keRh*%E1#S7A!yh;Av|fQxuoNva^hi>*EY%
zEF3DnvOzbADw!49R2?b!XS3zz!G>FZj@(i>>Z5e;9*?c<6AMA{g!FbXv!Ws%`+B3A
z|K}32PAyo$5i7J{`SFA_HeZ%Qhj^5fIC_12TzYqI+-P_p+$B2FFe1{>rshP!SFOXl
z%X$3g8OYe2DEMQ+tgnCg_V$o@b}8xUAHKO28b}?O%CTY9pYQz-f0ed1vwvJ`bR0B1
z&hY>LoH>S|p1<q;B*lmoQ#Y*W-P#*|QM*F*e3D{BWF-rKg8GdYVoI}}4A~}aIJy4c
z8I%1IKWZe7nat*PoEID#acw4dOT{E>`xBvY0bPq0tn-~3EBC}=jrP298F3}m#1oqY
zSe9t6>%OMK8F_5mLes*Y_lr$`=ZR?6AKhi-d&BIo!)>F!Gsdyy%QA%T|LoIP^nT)}
zr9U$j-_6gs?>BXkPUfuoDWKNe|NmQN8}hjZ2)Ochdff|tA+ksy$#7b#GFNBoiWj0K
zTvJ&&=Ed6x+J98-syEV}vGb%)-m`rMc{cr<RQmQG`@f)nv)`*(CbmY8!?&0x+}1d{
zNq{9sy07`Yv2TlF;E{dGs-8s^_X>Li+&4u0pRx9J{E`pccWW=F$Vjp+S+XiaYRaPb
znb9o^UT@IhX|Al>^lJSMy~?`%!ShNBR!c0^;M(-&tL7@uBDDYi*UYu$Y7P)+vFEOA
zQfkt)X%G`LyLX8BiGth#Wsh^JH=;V4Cb&s=PL298`I)0ii$hbeZgh_E;Y06q-ta#X
zXnH->WNPluZiRc3+^?8!m#*}?og{I?tkz#3VHQW@>qizj((~GvZCv+b-<DQ=@t<G6
z99jCcGeYc}Z0J&*K#^mCa*g-ihln}_h!i=@P8VricX*QoTglwx%%_uPaX3$E(Wug|
zOOcJcBmcPnSBBK189Ki&N7sSN;VZKt+wzxtDrn8ni?!PLVOP_Gt2r*0mhrP4JS6gn
zf6M2&39BEz)%orBGfcGB|FTWuV<D~II^3Uo1bl>p)EXCx1TCA*Z5{mYa?a;=hn^>v
zhaILZRX-$A*taP~#*pp(#aW9EM7{c@VfJp@Bx4nei>fTQrmg#Z?{gR=!>(1>SKgoa
zyX-}PmGZW=|4!tb=wVjrn|PVs*tjHyF;|s=nR!v1#3jvU#%FbWd>JXLR~s7`7@Rm|
zWN<*O?ZHa<An8X^ObG{O8%>a2m?1k8w4*sD;l{41i{v29QYBrXfB#Y=_RM&AQCY|D
z;L|*t+?S24Kb&Wl@wNF#9tyTsv41dYvXb=P>f;9aa)*1RMJz=jF0*X6t3MYI&As<M
zr*Tc8Xsv)tdV|=(9hdZG>4bg#!aik5&;9+9)AUkr?4A82P#-#+rnLOWz1`Cj?^<Ql
z2(RIrcvT^Lq27U64w42+68;-|PCNJ*Kjg|_JmJBWQT6|CvPDr#;=7nHCv`vrFQDB9
z%RhhrZGL!)CQn-%kCarv@rScy+D)t<Jg_f(ZGYj}w+%)Y7&htr%TTrp`gpq|M`T@L
z;v>c=Pqz&QZkM}z-rY%j@%E@uoC9da1Z3LghoB|RPfqg4+fSN&>7gputx%@|HkUb#
zj7#_XwYf$xi!*Vw*)8*m>5?q?t~tTh^}??IH%>(<*|J{$`i$4P-v02`>?@JtpswZr
z|4PL-Ufj0_?Y0aFmHhN6;MTnv5B}Fr`C4Bf^!kdd)NgyYX-7p4<}=mB?l`h>npj1b
zfX%l>qG>;BTV`I7C|K^t)}!+2ul3<AnLNM01uk>~PXdD+S?}}lFngP8H)#KAQ&Ye>
zvz8@nDGgye4@_OiY94jh#X!w~QF7%g`)#xMEUg~qITvevJa>4#(Al#t)66rJ{{J)l
z^YgGlGTY%@r7Ajts-SK14FCTx$~f@zWpLX(IW|Q_Hg<M4Hnt-Z*#2nvJP~?iFZg+T
z#NsX+hoF_s@z&RtI4sF5NS)F4KvU<VIv-oll#k!_56`jW@tz)(xDpmGi%hg;%#aWf
zmHhQf;>=mW;v$|6UCi?=F5C#%9k}yE(~OsGAq`hq)=XvfR0}xTI%BCCv!*J$(f9lI
zZS(CV#dH#8nDPF8_-zU(MS>jXWS5$fGGSWUi<gNTwzuoeRGeXCaGAYm=0T5BOfEJI
z;Zqym6a_JET7AN}kx{vKkt+8Ng@bOOdDr-T24`j-HmJ2a+|D23C&|MDn&$_ZcuUV%
zP_XS+6<d8h8z0{kP4hLH%({gMn;t5KtjU<bcyPthtE(2M6#j9Kb?eOGtJLieUoVk%
zE+t|E!)e}_5KtE%WJ=nB&mRq!EHOB8+?hY}SX7-sf{1j4<coDGi&U1|NKTAmpO~n8
zFpb@?llfBi-inEd_iWmHXP<~X>I~h)zo=;4lLrYKHYR%brf%5o9`nC(tKY$?0TKl+
z_C;ZB%SBIdi#Fx*C51d<kKpD>(eBW<QFw7)zU}I2$*{Fa7PYe$J-x;Pa?Ah!i|Qtx
zJ#$7u_w0lB=O;`!prG9R;?M(6C64P4GRhJ{oqZlD+<1N@pDC(Hh5rDzjm=$QHW^iR
zX1CxkH;vmmg~5{nKPNrA9105Y|Nj>`)}A{60_V;xSanK3_UwbFtt(O`3QR0MglJb-
zaUXG!Sf|r{ye7rL^N~VC!hx#~{KX26^~>|TzGkQscc$Q<&D=CYsKY`ZH86yQvDMbH
zm6x|YNqHf;`p0hFA2&}lxol(qsLt1PYh_UJLw}hJt_{;9L8JY9PF%QpwBV=FOzwXt
zKtoHQaL#?yz|6|p_8<YY(Y3OYO;2x1R2!q_(LewDRWBwiV-6B#X9-x%>!@-dezrpK
z#}L)~`wU}t9odj0dHCJkD>A=SgrNQkew2`qkdT@xAea~soc!TiT0nTZfvIc!^QP6^
z-0VlV9|bx5QD8s#t3%;HaKeH@!^ubY*Mr&$F?&vINMt@d)0ij!Uc&yg?rTu<;vY3I
z^YO7UGO|fZvUz*6>Fcwxva<2<WfUH2FjRL>(H9p#pw+$q%E90p6Id=tiSbCRWb9@1
zml4-D%DI28;ETrLx3^6sZz(~`Ua5v7jXW-{k`)yaKE9Gurb?QaO7ieZ9yy|syHQBl
zbqUj-UePtF%wa44uXy})hU?LOd7k)v1~T^N42snbPgW0^r=$qo<0bU~y!@3XCkM3f
zYtkehCMF&^Ii6p?c$}Pgf`WLOnlv7(o`~A8$<X&$zJ8rU)}sSU#lr&`EgAm*DsAf)
zm*kUckvnBj0r97lNGCJzhK+_FJ{WLt7_3-faN>kPM~A_VorW2ihJSvs`nsJkFI8SH
zedZji;bYYwjMqdO_V;&Pvv!@<RO0ymPvVZU-o)#Apb_<V_fji9Ha&cPzA-cO0jLuS
zI@qMAlbP?q>?l@N$(9ySsn*jY5fTDg`_s}Y`Q+(Bb=O2zl~9KYABBCVj$gf(l92G@
zQ!8kM-haMC+@1uBiV2|8IK=fO?(LQC;bfA8y7@{cv+#xu4*rpXf{pLG9zRODa3#gS
zAYsC^ws*@~=Bzq!u(7yYrS<UKil{aJ|2uzw?bK(wyS)G5o5;4`-z7ogj74Be&d-;$
ztC28j-v>&J|Nl!RbuvpHcr6|l_M$IRPmj&ekgc?o&DE7nP%z`;xl^YOoIQKsG`Fjb
zjmB0X=8KuEn_h^<&2XH=`eel!^ZW(s8#in)sQGuM;IGx;mzR0g$AJpt51&*`c0Smt
zn|%Q^UkeJ5L!He00zyr4Cz&$$Ib^8@2C^k4zUV!7<_wRH4$qo38Y=AUCAK;)j4X~z
zA|fUK{Y|d;KkwmP>$dds67y^lEB?z>ZTP>xo+tml)A^VdnSH-sUQSrA-vL@?2r_+M
zC$q4Dp<zLh)AmFoBd289rU%?N6mFO7uWx_n|L6bwhwtNGXy$(?eJF2d@aIqGJ`b@X
nfo~r`-UsO>bdnO%QA^AWsab}<kG%G~0*Vq(S3j3^P6<r_-pxi^

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..e49b5816120d43a669264915f1b6747606e080e0
GIT binary patch
literal 18109
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjYw>h(45^rtlpw*nn1P*v
zfq~)we<{|*4C<rI|3~^a3LY>JIHd9|{-!_I@()a%p#=`C_vAjvcg<QL$Hwa8!4P#T
zc9HeB3)daeBUT%m``4ag1_j`qN!ENV20Sjw?8o&cX6*R?bAkZxgdf{o`!*c)vYz!!
z^s>#_|8tlH1r~;>Fe!<w{C)2BwQE~DLnfZ;IdenG=<z0nK*d>?Sv1!A`dl^OZeW_Q
zv?X%a@imb%TC;i{?tNbRv+`HrkIzqPpZ`84kvT0*QgH9`SN^><bNRuscV*H>zGed+
z*2w~@UMo^2PVD{vU$<q(3=S2KZ?7kWmAnbNy6fSai@WT*owKiUZMbG}z4e0c<z$&;
zuf`L?mP*qn$+9`8{E9o;A@IO<P3yG)t^?lpI6myW_g(G4TV6qf#bGN&mp-sjea<#r
zY3Yf=6Wcy+`e1W<*QtHpuhr6b>K*;_TD{KJAMB7bQ>?if3<O+EXL&Sfx;S#Y_}iZ{
z(Kte7!N!7XZhsWAR-T@ATI93b?{dpaQwryCO!PR}J27|lY&CuF(9limjFVjgk3_k9
z2_{<v@4r3wizD}*pM`rZZ*F!KT$``V(xY72FYqj}p*ZF3oze|rwf&h&&ma7~{@3mt
z69Xs-_jERQcTb%>xp(^XSeKSycXtQBKL4Oe?oLhKZf&tHEuHP{jZ?qcD=S}naYXoX
z#FYi2iV~V_4qm_h%g28E|G%DxN8m`qk_YlCNB{i%{Qvv=`u+9)|9^gde{=f(^>Kg8
zpPjj%ef?kUj}QOPO1JULvzcAl2+A-Zn{_&y+uNu1_j?CSi1z9_ej~!eu`xbsmV4B!
zV3(Gevwz!jbDQvSOy&J=FvZbDV!~sg>{$oq>J&_qUtxOAe#;Z4hl|t})Mse@|NsB{
z|NsBZ^Z&)}`0)S7$NKQ~MtkbdfB2$#_;^20{ePoBzmI?T|KGt4l)eA|mpXCe;Hgt5
z&Y#!RJEyLxnUSWpXrZEl_GxXsbBZcQ&K&$Wzmrq$1y96cnPaI52@OXWnYX1hsw!O!
zl8EX_y8Mq{_vM1DpY5yIbayIf|6^a@kf8NnQS-<5_xJy+`%9dgo4liL?!%M9ZM(}Q
z<?Pdb{F?gkx_+B|J)1%k$lL$_OEnxhc<7LZ(jiS<-Bs)LGjnuSEj_BPbxKv|%%PK<
zFMX24`2SD1%osCgqTt>Y?$<h2FFbI9L)Rl<_M#6eKlnSlCoxTk5{;0SpTQfkV_N-*
zrW08YT)FS~c<BFpAK&IbU*g@J#2w}R55MKMb&E^p-Aw|y;IjIzV<O;`wkZ3Nfbiq6
zFtMu_CzcjX3J!lRB>MEhqokzNzvoY!Xqd9xu$$}59A5Fn16TcyXm?B#dGKw~RE>;O
zwr`sl&(=KYXpd-#nVsOR@jqbepZ0Lo35iqv=S#%xO{(}a<Kd^&Hs9G2_x2`N{GRsk
zIDgd<afp@fj}p?-LV|_OOhO7vL$_@Z5EM^OO-*f3=cout>{T>r@#Q_Yq&aHBpTh?1
z*(}Bjj>%th;@Ir>vws%T1uv#2ZCcZ`UjF~Te}f+X72XM78mD>H{ZISxZ|=iKu5DLW
zOX|fYRs5Xv@UHbOHyMb1xsL?Io?W><ecQ&1U%!V>nx6jnL4pI9I9pFsV`C#@xH-#i
z1?9*8|NsAg@RFq0tOH?d6|}gw9BFhmoEF65%oxSnw70c!>kWssF&B<VN(4;ih;rIl
zIXUrtTpKexD0C}6w>^CBKg&fPVw>lqgruY?Q=e|$5Kvfpb=%gdQ>UdRB_uSsax!nb
z)WjIRMq#<A(k0$H){0f?Ov@6p8<Tt`BZPUc$?+CF7Fox$uA!09)KOuwz_I=HJpS{I
zWNc3rd{jDoZ!gdL`^GhYy`Gk7L+nX=l#r70;8BvO$dk`suND<vG%^qn7EW$d;aD=~
zRKr?pu9IhZPP}!zvZP_JmZM&^!o%)a|2AyVo_NzDhG~8L1Yfs}tkH^%p-<%76s8?w
zUg_t!jn8`~WBP%m|Nf=^_|W+9j%C|qbxA(iq#d=hAHEJ><*EU(H|o)YM@hlq;^n2G
z+c#gna9vzbAfffEtE5E0uDKtK8W@ca8ALDe=;isN%A&+nplX-H$?YmrI)~YK{o_sI
z!Mw~u-}SexYGIrfFyZ#X3{UpzM+ct#`+xq!dwEdi?3XiK6L+@Yi^k!}>OAuHL5hBi
z5L=xdJ$RlT93ZfHTiEx{moHtN`s8tP(^n742}~0ix6VpGAlxnC$zr?vaI=Ka!3}O~
zHrFO{^VFQ=aOU!MI+Bt5z-Q%+DpieFhOhcS@tm5Fa3E~$ANP`|fXS@ajxaL*|7ZB;
z_wj;HDu-`v<@x>1(5BWa^9-n}|NnoHN#TlBT|0NbuBdqN;#C(n_h#ouoq}0fMas;Z
z4>+7Y5|Pa3{k1XV?t@j?AGT;4a!ar+S;gWVmEs`6JmXtZ?vy`=C&oOuv!6$!Ir;wv
z!v%TD5jhHzKeT*$DSWt*nJ4?2LCn50A6|(v-<*6HQXVK77aABEa&Q*jxbdQ`T{=4I
z6gTHaV`Jw}hRQ`v4!SlEJXz8O4nGpznV~o#yRfHK;PSa2?Ye<U$4ttK4R$GNT;)o;
zEv&mXfiHi7ne~P%Iu$`|%*;xu2efD1-Y!}8Hp!x9#=}3AM%j{3>(mPi3O2lW?cLID
zU0u2I^oauJYbOp|V!X04Lz6k&B;lHJgtVkbC!1}e;}vCdafL3<(%cW`$`dwQ>mAMb
z%DT)gg>^z@)tcDIHLC0n)7Y=2Fg|y>A#Z08v-i}7a{a@*N_o!DSKcDP3AN3#kb}Fp
zy83lW%8!hU4}}E=i`BSYTA7qD2C-FT9SC0dAggQ9RjD7+8aEzIV$9BUU9suFI}Z!-
zQWZJXWgC}@rbN1|Y@TNE!QA@8gcCjX^`N50qGm?oEt9s{=8_&!+u6X$MM<|XBg;!w
z?bgvF5uZOA8X6iN3SwhcJ|yA6koYb3Vzb=hmknxta=%VIlJMj*=1t6<B@q4fL0Yn@
z@Y=wpFoOk4jVm0P{3Sa?n$I-wHa_r?i3n!C*7E7U{o(cTJmz@@TecX?u{`|YtM*1$
z4v2+jg&#f?+_;^srh3!bdbKm#eo0A5#an^eB5V<}zu5Z-gxfZ4Z8;F?k)Z2gFqu0|
z#ZW-}<_u5v{Y-0CvaDTnFuZ7ismlXR=6jzGSlwz|u61y}g~Hki&(BM$`K4G?OnCS*
zxGjD^hr1QTvN;wJkuya^UH|^|b#^|xWy_J7Gb9q9vM2{N9&89%_{V*&xuc)9hGJic
z#Dc9t3O)O8cyVmm(yE%&(d`{{bB9Dkww{irXt5!?nPbK5DFr7Rw`v`nl{ewyY;zv%
zaHE`iJhFO+<6EE+Ho>Bz;)aiph_jQ<-Mghrmj0BGl$4N|@!%zsYN^5{7Ms)uJ+4bW
zaz3dz8a3^YyRX#`B?jgD$!G8Iw1^0suutFnW9NreEeVetJRasxc=UclLg>%;@`soE
z^HhH~(uq1!@Ym|_@%|rCiV)jsqIU{581PKC;9Aj;%w)Epg>{poTC%~kg=}2Mk2x&(
zIQOY~|J3y!KTk?6di_e^n8fl*xm(+mGS6%<KKfhQ_?>ycj!Rx_i485>o3cN}URGPl
z|9G{m`sQWk?_^hm{o9r5Yj06f!vD2m=TBGF%QLjp?<;t4sfYLq?O0pHtb0Lurk+u&
zIO{An#Rcy->O_R++<yMA5Zoj>G|ytj3<*}&o6XI-%1S!CJWDJsA|xeeCd^B8$;w?Y
zMTTd^XSEGX4V-R93ZgtG)D)7sCr<qMzrKURCelD@<_CWFCI@e^o)v}7GM+svPDy+?
zW*%miR<rnK)A6!546~g6E#7`_0%O#j!^+bA9AUiz2M!-jNIZDp{P_zP4ir4&Irk0J
zyZHbA(4-qPW=Q<}VId?mXVK!brY1)sB4$WR%t)}5o@XR6ag!Z0^Q#qXZmpmA*H$)a
z9y-wxwm-p4`QJj(b343?VnkUCLz(qbGtz`E>-}s$+f{!<<j>KZSq~)Hm2Vt6pv5QN
zve>>JRG(>RCG99+{9U34PHahsL>@hR=HdVS)QJt(FHK5FOl(`qtfs;ytgI|NhtZLT
z$A^dKjK`(`-4ic4)ebS_Z*-`#?TO%WaZ_nA^-WfaImM=aAkOK4*+aua<&PEl#SVC?
zsj*1=a?NZE73+(LkPs1-ymd<=Cr9GO&5ah2-0QSH%*CZ5I5>9FWL*`NA2Sy4@;tFf
zbJB7Z*c-WIQnTR75C4y!;K+A#m~^;)Mxe%{9opUhB{p_1*IU3Mp!X*;Q(3uf`v1eX
zB{L-1>$#bIWp?gmT3<dvbi-!j6|0P%JT=^~!9d3P*oVK?UUyl*esuDbnKbE-p<&p{
zmDiGzY|fm~C}q#eW?ov?c#~Z}Q1anisU;!}8O{f8NoL%XvTTs(SiiwD&};*P<=V&P
z22Na+r}c_fGlg?A^E-HJ6dX8vcEQ@S0&?d*{4_p%w42AiUdadQFBLvj)rTD%-Cs)7
z6&3kz%(1>Ug)!g5=#ZD7+mdapUWp9yf{wmQJsu02r!^Ls@*8%G`AqHEakSH9+Oh_v
z-50F%XBhVGKXX7N#X-x7g;jFRT1mV5w2J@p9$wRJf##s5ZI&USu1?Or!a_2vEH`G%
zT)0@mV<{7}56dApKNFt~O)Qq03fYat7t`Vl7(X6#4{}KUrvHNTUvj}i4&Q_IA*nAI
zvL7?A6>Zx3lrcRax6)xk2S;-^x3dpV^A?jsZjfx6l$4M#VdBH`vTMBj+ow%PNKQ?4
zD&pF-`~P7s<|qF+mdO3=wvzfNAC{^V#Azfx{rSWr4-?rVzcZM!O_%C<;8w_KW_l!J
zq3n<I2M&Ph{rC6(*Z)iT@wuHbnN`Y3QmUavCCbVnd(I+NZnmCl8k-b0?Py_X-fG&s
z+q!v%L<pqSo+K#nV9VyqU7Z_4LZ2q5rMVb=w_nrKaD7U_a_6X7{}Y+}PTc#~KP6zo
zWr1}W4ub#x&wqG*y}i7BUHzvgk3T=3zpv)!$pate&z<WTGsoMnFWjq(`)2qKrwIZI
zZl~+-bS%h>IQt*uK2T@k{r}i~e}3QD8GnBM|G)qK%BOgEMa^;yof;TC*)imcJvTRV
z-fjVgow`k{(;RQ~E;i+w-YUIx@!<y#4itRVI=niZhlhs;8ghk&7khe4E?*XI)%lr{
z(&BEjXF@^)f4ZU3BA#@|yPT_a44&V=IIZu|$G7wC|HtkBSM=k<-mkCM&$amY<L2h`
z`|JLuCL|;zB|UkX`uKVJ^eIowOhxr|OoW6VKX{UomX>zFK(ar!$0LYGBJ0DB-kvEf
zc@OUI=h<IpXjAni{qL{*)6?sBSAFFd7B11&VcD?R*u%phE6b>@-FWA2V=phEMD-x;
z7RG39W?fBz^#=|%KD??8>O$n*OMyB+W{1Je<L=GP%`+zo-g0++y=l=l+nEz)P4nSd
zp(z-arr<yE&v7%QmbQQOe}8@c^0NK%vbi^Qe*X6SynW%;!!u_}9zC+-&)=zg_B^q*
zeL7=?#EcmRXWji5R7mc)al~iqf|E(<fByb{e`@Od?CW)RcYOSPcX$21x}(!??bsoq
zq$II<^UR(e1Mk!~4Cil%t4UvPkdXTRF1C%8Tf!_4=D?b7Yv#_M&wJ@-i%dJq>otca
zvnuDjk+DoY;JM|<NBOO14w!~@cu3yg_t&QGk7LxWUG4n$|Nr&2SiWhqv9{*NpTD)+
zS~Xw4=2m{Y{g`P^mwl(e{D2P60*-f}KJ4G$+jDMCH_hJn=FZQ*-}4t)&F|^qNlfH%
zb?rHJ?ElP}ot|<(l4e~gVZM3&|9|_#|Nio9PBVlC+{5STZK0C?B_$;O|8M6@KbS4x
z=Ek<=q0W+jT8qAStSve?^{0U9hxzvP^KE`g$nSsk^Yi)t{{(~EjQ{-k`Tf_|@7LGw
zumAcgI%3A8&6`cu%u937y&*ZlWrLf2702RL=?4PZu5$k`ocZC+&G~b!_n({d^W~RJ
zrD=W#dVieXkjTtz|G!>Nj!joLsWdS+%isVPr(4fW>p!O4%*@SOO%MP43`?DkS+Pb3
z7Rle**08kyzsKjLGdAaac=xVeg>9Zrw}<xs|Nnn~eeLgiwvS(apYTkh2?t+YEziIA
z=fKI6Yu4&2=$z@&J#<0hN3v8f<18<e5AvE9E{NLH{z^aIxBltV_xoxTSCu)i+1DGb
zi9DQ;bVxz{_<_UDO>S!f&oxNBE)d)yw&qo2q~xN-(1z)psNGyG1_E<#{C@wF%X{{f
zf6fmMT=H8t(Zp_+vSjy*jk~z+wdR(YEs<(dG5%CGvw){`lBRU*ZtZE>@7w}Ka#<!B
z25z~(GAuU1`1XsK*MF<${pUR2|M>XtgSWyje^GIH^vUn1qS=EJoLXicF{{t1{x^E;
zcxmU-Rm%h}O+GL+PI1Ayi>3DU&9A<{;@rvX-!8&>N(j>NN@^4p`Y`E%{hmJ#TE-F&
zlw&u1V$R}YKDB}8+M3SD@6N)f-ruk1m)-UK$4B-{Obnbge?cK|{LLGmW5=8xJ$hqf
zbLK|Xb)JgH626S<R9dxPE~x(Y`ucwT_<uJyJ$?Q0vB=vzyNU@9FRQoR-!FONMn_Qp
z>KhHa`11oBLzTtFnZXrS6MwjZnAxGzoHwf0?q9)r%>IdHT16(~<$vPL3#VFjC_VZ0
zb^81J_5VLMFFL#b{{H{A<?qh^`pPeF^P@>5HzL}S$D)hn%q{lmaW>DK4_;ngZ<hP3
z>hZDl^7bBQzOHDPI5F{kTv9OGCcQI>dTc#HtgH2w`0%AdQgYLF%Ll$(Kl&=zI!@HT
zkU3p{rA&i4=x1Y&#x<eYjM20I-+zC9|NFbu-m~6sY*hE(_wL3=mzX)R?(Nf$ur%(s
zX#CA_`6G|TnLqWvzrBy$y>D-o>&wPTK0Z9<?<cJk55GN`QS;r#Q%*MuOd-}KIYpi5
zoVNd>S5y6pT<#VT<;U?oZk)3k>kXFv`~QFW0)>L7r@W1r88~ZxfBXLIZ2$9fdk>05
zpV-QLP3fV1mT!wrN95)N-#87^i+ij0=iU3$TW`}W@!_jBv$i&~WO`tdtd9G%rp+78
zd?#FmTG&)Ax=r?{e%7XjaKns8^@*ZfX;O#Q#7sE+OMi{sum6vZs+*b$PO9za-BbJf
z^P8LH8)iL~cl)!@U9r#NU;B|(2Af^~7O!eIdUyZ-=Vxc*Po3ZM@Be&9mzFt44mK=y
za`fiPC|j7>4(dxY{QsZibna14&xymc3MSU`bf|u=pV-A)v8ZveJr7U4!7Q~Itz1qq
z&dxJr?k6|QxBnlz=f}Tez5Nms+gI;%xRPhFWx>nT1Ir|g?;6<D{P_L5+^d_HLqI(J
z!6T)3>uZyk>oXgVK^h)OKmW_`<6yqDr-481Pr8H84&z1dTpi@4j;>fTQ8MEucOwhS
z4N=jbC%@YYB_t(1c#=}};{$h}#bF(W3#+8hv@|_S@QF`Qn^K>D@6X>)PvxzxAJ4a+
zvW!12jx8~fjggV7*HQk!fm3a(U9x9E0<ozib)xtuhagVoOPBm~7RMPVKgcmzy&>!>
zPlltW?%B0#b(M6^{QUN|yy#Ca%fCNA&!0a1>(kTnJ(kkpcapd18gvUc_5A1I;mN=E
z=jY$w_HXvob~Q{BYhI-E^OodBUD3n$YmdHp2kD<DIek)@wu~*SnR)5fM#Cqqc4}?A
zbGs*8y(YG8)1<ToYytuaPah`!{P2)HtG6N2{m<cL5A0V>Hw<PA(>=j@-kz<ket+HH
z@W8~x#Df=8Rahe;0;X6fFx1~>T3wZ*3-wKoh7-5=4vteE8f(`65)Wmpif3Eg`jOpP
zMsOAv*B(YjttoAc%*@Qphv(b>-?Fvx|G&TOQj2#;as4-xY2Iq&Wc*N`k2OF~UcPyo
z8TZpzQ8l%;2?A`KoLxMQueS*p7j=PR36vU|Ow1COuU+V(oAX1)Tk^wydzG~bWu6>j
ziW-mp{8_Vc<E*f-pbw%MPZ;_5mdV@y<K*7V#kq0OF`1}6|6E0nhcwJwvSFLQq3@65
z2R_7yO$+qv>uhJQTF<|;F7d<1#0}fh7I=Ma5f!ks2Nn0A#JASs-@!>7NnWhST8#p#
zH_X`hk3V{rgSJxumrz((+oL39XXik#n$u^_NJwd&J$keykn53SglK}V<cq89>q-uv
z`4b;C>*V$6&6C!x*=XqO#&+(a&nCu+t{g^?fxe`~gdgnhCmmpOemGgD!0gb{|Njp!
zi}<kfpS{*kd1nF33;!#tPJR4nSY4g}?`!uQht$Nxkg#Xhu09P7e|{{?^g*J5f~x`Z
zwutlt9(pW2SFfgS+mv*3@9dW!lZ*ZwI(jr9^yq^pE^T2t3_Bya;<q&NeCUS6SCUhn
zn?ky^M<i2z*P(8)E1#HOAGj{xv+a3(i0KE$V2wvlQc8*@y?eX$(ggtpWsZOU|I3T1
z<)*72Idi5VOSb4^o2lN$NjsJpc#G?l3X5N_FE5{VBSJ{sRRRQDTqLBVCHL&v@ZMgT
zKS505h&Zg}eP*qJk-?vKQBDolqyPVF2yZAj_WwVRhxFWs5A6Ba(j1#&>I@7F{-h`U
znm=n*VuHeysii+YJ^g;9dR~9Os-B;of`e1c9A4$Dk5|+pyv1W&Io2^TU!G9U!}C8i
zMdI(Di1{`@x7YmCf8_ruI5bs5C&j}@<=$%-#x<$ThOQjU%+Q)4T>HbS7D2PGOp?66
z+owoPSoUh;70v&J1^*XslAO>a#I)e!!E@&ll9hKQw^#d~z3Dq!Zf~u3#i{6!DY|NI
z+cpGvhE0>rk5gZ^?7;k_4<WV@m)d7aDqCA?vmgIIRl8oec+&ipnuo8f<Y8ko$!Sqy
zUb9#s<H7%WSPK60&z3t=-FVU2##<W_j?6jls%1Rk@jO9imd!UD9xqy)fB#?1-d}mg
zdS-Kr-#c0yJ8zp=e%`vIm^nP`?8^Txb8)JQ|Iut-AQiLe|GwJa=Jo&Pdrtm$XIHDO
zmV~6_m8+6QMiNO$5-lr@CpNG?i1hBM-w0}7{r{iz=btUJrmRALi_rqLS^o>fn3<V#
z!j%s(&hT?eJ>bk6%(BzMQE-3#|Ig><p6`>dFaG>&`R(od_f&uHZftb)>N<SP?Z)Nc
zGw1v~0w+#pe7>+@hT(As4Yy7mdyZxNETGAR^7sGl?)iCtlInlGxRy9xO@#vs79Co!
z?%adNhabE-qu>?j-fM7RzSM^$pqV6w|NpndIC3#4a>yL)zF__R|MrCkZaBY5JLolC
z)W6K2V1~oomOUx$N50jZ`21VPso<pCG3~;0lRmh|_wSQg%x<j0v1I$RRR)t6KC{&*
zJI8K&N1gG4Wm4IvispB7SvZZSG(|16I-2!Aa$n)R!%x{jlQwIj9Qh6?aA+5c3;q9p
z=V52Wfeqb9X0$1X1v$lsi!L;*V0jaD#AE5BE5;U?%Wo?^OR-bBeQurBDt^(w-;bNS
zWX72!e)+U0VC|)+%sD(~cJdw;e?HMp<rh<A){?beu@^xF;s5_hjg_4DY!eewT9z}j
zT`6hE(+|3m<aZ`X*M36k|HWFr+f$_ebsRtQH$#bM$7AX2SqjTGD6U|u7d8ISkigFx
z3>nF7Qs-lfXl`Z{yF5YcxP8u9KDM4K45so^8>e=={X3qpc&YTC+@}8)pS1reG3s(j
z7KpITP)pd#k^RUa%pigxO<l|$Qn)m!@!eqJ<49-;m4DGH#JpvK!*&sA@$gdzMBKf3
zBZ4`vNgR@|-=V~x9i<o~c~FjPj$s5B58IyN%`<AFQlLW{i3vZtP0lqYsks=oiLATR
zpy}uHtAYEUbI@mgXRi99gYpxPX#Z<ZR94NsbdYI7U)4fW#|`T^q4mzs|MEXCusoT@
z&OGTs{SJj7!6v0{<<tkB@)a&0)LGfszGNt9E))p%7d(9Q@(GEah9z9ALZ=}GR8yFC
z#@o;>D^1<pwlwmd$@wThTV&mrMxB5B&RoV*1#~=4XsC4fXs3i|H;W6*w^PW0YTIrp
z@xYgNO|S0_w(KoVt&HxQ8j1})*>W!S>vU|reaInflfvR0g{>zXv!C)<IRrH}s<DEb
zi%E^KYg>6Pv>muu`cU*>%HoMTSPVrMTvg!Vv3SfgU1a6NUU$YQrG{em{7ntaTg=!G
zGBNW(tz2VwT!+6Ps>S(<{kCK5>y|Vu6AZU>6nL0l|G-N!qS~ZllHfAk8J%ey3`UnA
zqf|+bpXK)a?z&cUG^2e|NYn$R!or6+YeIR7S^`%uIyl+F;IdehmE4|&A5Ki51~tR~
z|4yDVH_rU(k6fgn`|QBvuoHgrA5soB2-?ng$!7e?U};BxDcho}JTsY^8Kccbi#24m
zrNO>x>Mu@U{cpI_A}#bn(8>B7r5zqIo-CVn9KBgScpNenaJqDY{a;Gs%T<aD5|S6C
zWq4LF$6LH&Ujd3gP;J}zQ~vi3^-s5qaxz)78y$l*ol=ApIVaBLkmv|7Q(-*l#3XEB
zWCRI^Bu6bR$%OCclHaaPaI;i<Y_I5;bcubI#kw^1s3k)4Uv`SS2+Joi)SFM0fVPvJ
z-ko|B)T+d1AILXbC+c0g!_!Be!U7F9&m|m4NIrY&$QHegS?bHS91u2YmYr~m$u$^M
zaQy%8q-j}@dGOfLS!x$<@T_>)ee!@xp{QKOrVPQ84d#vyy!3YPq&S#XW-$LaC2P^C
z(|6JhTJ$=-KjkF)|M;a3N3MA$9`KNyp|bDFhxhmYPuKr{XV=$Q_jc!q8Iqn{Yt0<R
z9A6tf$g+Ll#khk-lFeTgG!^mnHP4<sJXu*hIu`36D)T1HtvkGm2bu<wnq>AgZb^Qs
ze`)RVj$F+X(aMS9TK^AkyUQCXbNyW;%g<L=*MIt?zHVLla{v0judm0mebFt9aNW@L
z-}$au^UHbrkL*aFAR;l_%&?{1_{$fAJu2THDBHBMd~$O50}b6nVi5r?yS57GMkZWu
zN-owZd-wl;{m~;c*2Vlhci=<3Q`67y@9lR?7MYj3z2I0q&)W!LmUSuS8k1OAR;)7G
zQ+aqpv3m0iiNKq4c^Y;mi3meXJ+yVF$ho7l6gEr~tGL8orl<UP|L5oX|NZ?P@6_ZP
zGNrS-n=xAK_|)r#OP&5?GELvi?!74?A>qgW`HpUFj$U1jGiN?L!ucs&LHPcm1EG&0
zW+ojHZ2C~dWVu#BT#IEg>$<boG(PSveZ5><|6fF8p_Igig|j&Chjqrt2~1AfaI0JW
zD9@r4M#i2Vo;PoJ=FQ`Y-)HdVe*fQlZcz+pf*@m$Nl6a*)BiS4{dwcFWLj*q;I9+E
zlRvsX|M~eopZvdL$Bt#BA6L}x*PfZQR_)(o&w?9L@)5$97kG;mD5xAs$U1f4@L|xj
z)!ErguIi|i#urXi5`dWN^d(F}Qc_^LlA@c@1uL}zwsn1+dZ{mZ^6&rCjr;TQ%}x2T
zG8sX^7#^PbiC_LX2lXtQuF&-QZ~H}O$Fc(p*nE6={QP*LqIiz?8^5`Ec)@}L2hRF7
zm94qTBAE*+fI->zNqA4&(fSI@#wullb-QHzJDbk^`2YX?mdy9z>;LI!pPn@Tyt>Mf
zhQ&Mw;!PJ^Rm*rAyhioDfQO`jg5tpmGfo^hbjYAs?eJ3XCoZ~0)ssEGW<bKYsVvoS
z_M8o33{j=cckcggS7}xEUBKbg$1=<O{o^+`=g*iSaq05Tw{IdgUggLTOME-)(H2%^
zN#2<zjO;!>JooPLG&b^V&oeA3G1yb(TDQIR$3AT-h{=b7E+6jybg=os_AKGpia)JO
zdcME^?=$bu%4N?E?=Js;>GICSi!5f&EL2rnqslmO>6HuGS_X{#{LX=b&Aq&bpPl8g
zwB&hzU-)+0fmJ;{Um*2aQlq|Y`+6bQT$2Uc?E{W`e%xYtA({s?%lP&6{WCM?tNZ`^
z^yz85zCK@VZ5SgX8(Ukw$F6^g60J8s{b+CQURiO1xwMomGqdggf5~fW5)+b>SFBGD
z2vRag^D+<=m~RJhxs$Y=d**~g(^~$x3m?C><8Y6k$hUs^|LfxZ-o5eT<B5s$-{1eA
zo{+Gk;NkWC^>gg&|FyoAI(p?qgYq@bZEo%SKmN>ocz{u(`g0nanOR#C6I*TVqA!p9
zS&m$Rb+enyEhHrW8!ouFyV)e=-+acWe=KZdqkO*n|NVXc?d|pF&)4rSea-K2Dj^}E
z;{U(<6P4}b_xxy1Zt0z&#k_f?^_>HIDtYAX9B$6HH`a+e6A*Q3L%luo$MQb{K1w_m
zRpy8R(1T&K4HB*hzIZg@5sRntx1`7K@BbIq`!mn%?de6X_YK&K{pS6dJMrMTh83$e
zh)HC`JgH(oB-fV7#1ywb?ZyViiF`V`l6Ex-H+D}44UzQlv^6k%IlR8GH2&nf4bZTP
ze#0EzvMqzz+nPC9heg%u|KH#LKR(_+*Sh}9oS$ERe2hPHhG*HbHU9Jdw9eo7{jD^!
zwe{g^Yxf%&-vq@ytI5QA4<+Z_8$AU;bHZOV4zG{z5Da2wW)l~0Q+AiQwkGk!$wZ5?
z9z~-`2M;tjIxy5$%F7%GXL+QrIDv6f6~qxudfy!53c0L>ni(@^t1t)6Yz%17={xh|
z^Yj0!!|QL~-oGa9@7#$K6>a7mzA@wJnVI&-`xnd1-<EwnKR$li@&5lOP8l7##Ln#Y
z=tsY<;~)8Zn$3-k58p<&c~6(<lWi_iRgw7jH@ToR^}@x(2TxOH*vw44Yt=SgUs6d)
zV$U9lipq=O%|dUsIWSy1eDFaPBc#53H?>v&!GUe3UfZX95Sf;AL1e-59(MkJI+34h
ze|=eRU-xJG#zbG=ry_!hH6I?lKRJ2-lP5FeY@Mz!&6?%2J@0R7Qc_ydp*JjvYJK&}
zioOm9?(gTBZ*Oc<*CX)oQIdgi@`I<T51yuG<fIf7B+M{uPP}i|wzpdH&Rt0*Wl0;G
z3+{o1OZnHV?lTlMFff3GlGD4jf7(wfh8zB0$jjrhxB1!?hbub#a(`~_{r!APW_<qr
zf6BUNWyFsQN}gP{;J}}^xA)in{x@UB2A$}(OMSa)f6s3|{NH=}zm9I>#4nK>)cOQj
z_SL1{*v@}&Z+X9C=+uX&#b4ZXm^O`viHXO_iRboq<D6SZHq_fU@3n27E!{j-w0WIg
z2zSxkru{7kKrJOup>|LE_k6+Hg}>^TSa&Qu#B-rEsl(?FD2?j>n`isG`tdRO>hJ%m
zetlWL-8?!=t9`PtHaj~r^Wp#6;r}=|3}kG6%x-Rec$~jQw5F!Vz5n0WkBPs2ezv!^
zW_~xtP|1ti;s<CVLci_l>A+{nfB#7At8<;gD<i|x*2c4RDbKE5Jl@j`=2#poc))PD
zSDNSbHNzV>3@j`RX3S9Vo93jab>W-pgD4jcP=tb-Fb89Pvg<!g{9Lf3N|opTKf{{8
zzuuplYd_oU@10FgpD*>^-`g8GYgWag#Wxe#cm1vZ^+jG^|M0`Z^&%oO=Gpx{edGv(
zz4)Kc&;LJq)Z;(zPxaql_VV(~s~##FMjkl$_BK!ceIuJ{wx^#y7<~Agyfe|+vr|#K
z`{4Qh#IsUuywVbW^V2H6^?=qk6+f5g>XP{KwR3I81W|?c0-(VB|3AsGuc&9~;{O+S
z2(T^D`T6(v|L5obKR^3>U*6q*Vf8v`34`t1i;o}K(A{;&pEdQz*Vpq~SPtLYTYu%s
zjBjs$pFMl_!&mJ?noUhS_xJz*|NHy>`}=z~FcmLR=VoJTv#*!fSC_h@kU6mE&;Rou
zewmhpN<6zIpPAV<MT9LoyUn&*!f#&6j+&W?r^VO;1KIri*qE6`re?UZF0+t=v>%_u
zT5~l!2-Lo|QqhRMaU=Rv!o%L2J+tQ>a^~R9e#jVhFw618agl5X6-D+FSB$UP+_Q=M
zcx?VuS+RWIs(WV@AEde;v#^_$mU}z&b@epX(%QAFxQiEv?3wv&d52l@yug-qyIU7p
z*eOjEx~6pA@civf3ciIw2KCYB5+w|L6`K5ym+_@?W=~X+|JCXs(=6$=P$Pk7OWTwe
z)dtNkv_t<anE<M0|NnQ&D>hIt)eHzx`kvD-_jhaR>GMmy>usvP`3Fq6cyFGm#f-Cu
zn?wEj1k3bnnVFfFZ2a{9>+Apab$@Q}{A_M)e0X#E|B0`s?5~%ssF2uK^Yi~VT~Kj1
z-~NC7-(UW(WOpCt|MT<khi|%v-`?h_|7UREyzlhs=|BF<SD$KN_VZ&iHDz;kW#i>#
zV`gsq{$6s+R>^&}$q$~puTZ>k%R$=-l8vv#D)NDb>dmg3gnVG+`M~tA;fY|4Nb|vo
z4?H<InRwWEGBc~&96Vq7-Cc2Z#@SgnB{$wYugjL_R@(G5)Ms*@RAlI;bx&=imrhu=
zMW-@wgU)V$F1gDN9DRp_vTl8Qa;I!Ar*qEjY&j=`z7xef&7Der{^fnXpQyX#*si3=
z3*W^(HJ7vq2prlU7Guz+;wY1Rf1yazfy+#GXFq(32N!*3=Gt<#I`Fi(vser6X?{_;
z#&%uD+=`~PI+t=E^RJtILuZ}F`$Sd0nOb*Kmi>5?-xu8f?(!RpqU*W7o3CvRJSDX8
z+N9@+p{}WtqOH~OwZ$yvfv3)G{$i`n+4gYP1rx>74l37nO}a5-`iT$sLj)2c9a^>>
zo%m+P!5)*=Yx4?!d%LT2K6bBtAwO#YM;X{Xcjj*7YEY19@#o-dZ<{7}Y<kA4z}il;
z<qJXuTG$+&mTuD3nJ{bj$NPC!`yRLNzWQ8p`GK7#eSsYsPnTWE^Z8^~828mymhD#Q
zk_J)n6DryKYx^^AB_G*jBcIC2xb%bEYc2+_MnjXc-81X8SuENs-C~tfW(($WXXi5p
z2CaPXt8e|!k1U{JgC}$Mg0eu1y}#?hxDDGrSgv7-zuf*I)rGG+_|Hob6`5+y=1CL2
z+*FsCk;<MX@y%EEQk41mOWXRFS#F({ImsrxRCY>gL}=oys=_m-47>B$c04XPcEX~|
z@XOxY->sSMt8+wzPhwfC$+hNal*_*_6&(viZXerpp;W(ZN|MA0qeU;e*Hm2J?_+;X
zom;d5ywt)e@;sxY+Wx}F=XaIve|GNg!6Qc&tUM_rcP=aI)S5+ymaI9WqtH-O$r^Q@
zDRAMRzo$QZe7t{s{QuZ}fBqkAu77_oaid(&gpO|GJvAp6tUdeTpY0YuQDNn4;fHw)
z3=M1koB%BXeR-KDCx_?5vETdac}|_Wz*wrNSfeH|$0YH<><ojYkR%N%99jYdP8VNo
zpIXHrE$IJv=HAAs3AxG1${q44-}gqmShc%D?trGTkEM@?M3UTH(@B;8JTI5Wu2$Lj
z=!dAQn?#p_fsKF7%xC*Ww)y5)T)$X2BQ-}#jCIM8AG5W54sBa#x{TAyF*`)GMt<FH
zts|$UY*)20=5}vl>D+h!<<2PrNxqRgY72H*Xo#Q4>^{N+imfwq4MEG3UE^oD@N~5H
zGUgpP_Hiepw0S}j1E-;)hm+L>rKy)LZIt_;5P400-+rCwx7~402EK9G@gm#bvZhVz
zR1rL{ykxg}*+HJ?9M196UT<0x`t<RPZMOQZ*=>zB%QHQKclBu;l4xSk+$gqm-PfP}
zg(s4E&MB$}{bS{x>#TWf;=bkU;#rQ{B)?7W`+KRk+4`Sn${ywAvr3%bfs^%?xxQS7
z3<O&DOEn91i9}@|T)SMLs=nYgqfW~-r71!I8dD_1*E+u9&#O7j8F^;5;>p>KXMW6n
z;Cy!1**2A#Pb(kj_Md8hfA-Tmt%shUb}TB0Ew`4rBk(Vr!%$8lv+G)lWN2&99KSc;
z=bT>?=Fur|z{Go>u!#<DBfFmX8duFFOLSId@9I-X*tz7rVcW8WxjW+u`+mwS?MU7h
zU1ohPn@v*n$bIz$4#&tvrpJ{NI}CpahIX?Cwrq{~eX;QTnfLy@{m)}3yX*w7Ltis@
zV;7SnPn&&NdO=Ld*6UHyHxGTUXOK_u7If7SiD4^y>|<;key5;(g-p2267D4@<?BAF
ztGwz~3-UT<Ah%zytFAvPuFrm{dJ4m}WBdGbXSX%3JAJZZ(P06DHUWV|BS)5OOPL!v
zH{M39-x(93BjBi+W-Q0Q<=oSyBIg#GdMP@EKD0d-c>Rm*cInTSsh+o!bW7(heQ~)n
z@qmQOjyTKtmU18Gty-j^;k{*<#-`Q-87`MEnB;6<6L#c8$*H1~`%jn5Z+K_ra^wBS
zdB5k*&u30PvXf_GeAA!1U&XlcXHSQWHLRJno9mE+fNTFVM<=bDBAP90FW+GKZ&RO;
zD%#Q8t7tc&fm2iGh{^`BbwXj<g{NjFc2&l{Gx+wq`Pp9kV}%lb8)~=q{eE2fBPey{
zthURY+m3GY6<RHFS@Uqi`{JDns=kZ0x7;<Fzd24>HL`6=%=$U!ITqUXyZ@5YJXvhR
ze|nL|hgz9-o9mu;-rsy{VlY*8Hm}ozU3c=XY@NAMeWF>4(ZNkCv@VGxY*g^<NKP|e
z)bVtZ!IIbe=FF%QSi0l>M;ku&SF^5mB^sO-W8qlH{xV*H``M8?EyyC=DRXT><(k`k
zr=}L)gqv@3*F0j}<FGDk?ky{Ej%;129G4C;Gd7ms9Rl?p3g!{dr1zO`esf0em}2-X
z>-qhr*S4obFYt-nbico!-PbwN$W-(Gvxeo5Y&bN$Yvhk_oOWTGhv&3HyU8jZnTMB~
zZl8Vian;k3Me8)VmTZ|5*0t}r1RuLi|MSV$PpAFaAFX$+urMdtpzZNQ0~gIlGc+C*
zB}xc1DQsx_AO6-zeZlLZ{lTG2G@~Xz=WdM=SA1brwmUxJ=QDxm>KXfg{q|jC4$fI~
zW^Dv{$Yp+lfRCU^a`LR-e5?Ot{yTClPPgTV(~+Pl++u;}L<Ey3aX04dkJdT-?#&yW
zPdnWdYws%CPh|A*y1325Gt#K9YVwxE1tNzJ<hfsdP&q}_X~J_B#@W2rwVfs`v*3$u
z&rFJp(W{Wl^^Se~d(L?t=54-#3q)?~Mw*GoN4x!tK74D!B#UDvH{W+E`KkB0^Rdgc
z8yqR!d6%O}{qDu0#hFicRCheB^$%54^>n%)Eb^<2({TTVEk7!D8%;Zs-*n^onG2V$
zT7$i_X0|PNvjb0?{2z(DmkVV0WaqZ7yZh(#U%{iUf(yI5O5P+~<Xh@txr0w`?Uq}5
zXD28Ib13FdGB0$fpVFe~`J2`Eo=Mc@(jRuAscI7=ce%U{&(P`Kcl^7jkeT!;p@glA
z)=ixBS59;1eM`B>^`1fuiXOo&26rFr;IG*B{l8bMYGBu8&fCj!HMtgbC?srG?&Eac
z)bmzsX+f+_f1*L%t!<n~JC^qye%mH6=X{J_%=#ymzK0(sY&#abQRnbfp2L5vzxrPa
zJs<l^C-Tda!`~P|Me`9zf@`%;acy>ST;rsr&9C_6PE#-Yh6zWVjwpz2THDdu>p9W2
zIl*aF%!5$=XMfKaL_7WRYuY~P+iR1X3#PYT?$mMSxFx1&@$2}XiiF6X#n&GC9{%bT
z`Ka=Yu~*`(E2fSP^Rl;UEM=>?Ip5Ucr+4i3C{E3lS~c@k9+o{WQuOUimTCVyQ)AI<
z4(B$;(}zrEs^vt_{`Ol|afQ~ed5au06VCG}a2W8!eE)Fa-(kNOd3}GTR5Czzo6MPO
z2ufV8_jegKMQ|uJ&E4B;{^5$*yaP@w{R<Stx)eHgv@VJCoTy>=pW(1bzx|)ge~(YB
zmTL0n<I6bbRen$Vo%Q7wfga`a&ow0*5;x@DF#Fsy<%U^pf9l0;7JObW)vInN&AKJp
zvO}uEJN^FQ|4T$_`=>mybZmG%DRqgalYr|H!&J{n7fN^8m&Jaxc_np!xkiN8s;<NZ
znfs3at5oADjLo&&|Kic6JC7c-f&zQVTu^4<Y1^OWe^5G><)G`Gn)LZcTx;0n9p}o(
ze6g{$nxm_f#C>*-Zv7;E$w}%n&PzSM8zAR?Yr}k}^>+Dl&a#OHa4Ej?yCl*4+Q`>}
zFY5ivr01d9;<t_Znts^Nacp?LG53sdlyIAiU}DD<j*ZC^R3B~IJ0ttu*=v*5Eq*W9
zv`FFnO07#%ESBk<G&(j>#Zj{-F;ar<+SFx<^Hyfp{Fl0TNkr4|*>0mev3UDU*H6BM
zxO$c$SE~U}+y1q;mMT0-apiyTaBpj|`&=Chjiw9SKia-^uK2>MAG}a<)+fP9e|Dzx
zy)R~LeQlYvPis#49A}O-Pq+E~`SRRa@=Go&i^91jfB6NTv#WAODW?jQoSL+2*{T!8
zQ=C>F*f@V?Rcy)a8EVG_lOv`n{Muu-*WbENpw&?0sKEsj4#)e8O=I0}C+WH#+{Z3<
zeX7aPozp9(I(co<+4fR6@zL|($Kj9?6colS1_CbglNPs@bvZG8tK0Kmu|r;A+4hME
zQo$|SldSnZTJ+z}+U=WQbo0!FCvQ&eQ+}{qxm2dzJ0~t9OKQq5yG_%X%d@)waqd62
zX6d?@R;#*H6weDP)d=aa{61I6a<n!5@DtbQ`4QcQXN(`8Flc#hxb5w%pY~_{vKMhI
z^$4Dm9I3(eOHOm9+NnvUypQG?UY|Yp+7{iQNvY-C^FCQ$*!e{Cc?F~}nPbS+V8G)l
zzi1uTwWE$K>Mt4hc=PM@%{zFN`Gvw!MORl>>lvEKliKD#F<05M;85)UBR#P#$<tU>
z&gXxR&-(waY~JM5Gsa#Qx0&!}zG^$aYuPG>HL0<lLQM~DdK~OeJs7p+n8dEWed!W2
z)XMF??Q8t-Zo!uGhjzG3oc-+BL<>8H$A%(N!P7)q-wRg+E!DrPz5L8Qu*0^@-pkeE
zz|&%{<MQX`-LMT?H{ZDCcK=@3{6p;q(J_LKk?)UK?^qekY9yI9siJ@G=kEMSm#s}(
zE!sKG$;v<a?>tYKDYW%_g!imVrC)4I&u$A8G2xBeCaxWNC;EZbgwh~q1y2dKV-quz
z=0%3@Nss)N5X&DR=QMkrmY2%5lywGd$r;h7J}rLmBe#D~>coy+E*0RS>QJ6~nvmkM
zKR<p?cl7Lh_%0T-h=Y~2?e}-d3E?8af(|{R!pRRFB>ec)dhpgPNB{oD>C+uuS`J>A
z;OO1`@Hl_l{~KXXCRu&s=@jyq;Al2EcJHYVuSA=Bc@J+)zHqSgL=J~!LqgA{O-3(X
z8fs`7dU!uJVT{}I_+{IJ_3j%s{r{8r;6VbUjlE_zWS5(K9hYKQSlO*?g{dpQ+_`t?
zp4wK8ldLS=9T6)cQuH?HaOananeCkZ@UGXtAKo&LWkQeID6^FI8a5U$PInMdRL%4X
zyzt%Q<i6l(&(#;ShdIQ?KDUh8tP^qC;6iDS^6g{0jC|ECoHb8pNKILk8}Ix^nPH!K
zzy0<Q`;24@NJPznIAp%OK+cv;92%J}NA$!`nA)slKf$=CEhAYvCuy>mqtXkR#96hw
zjdQBzD~Rt^>fe-de`WTPl*k)qYyKYFr1G&a{JJ4eiezs0H4)Zx7fs|8=KS898WC#8
zChb1uki@mA`_gB1DD^3qtgj51r_2(ySVZ90b4`I)&VI6mXKaeWy`MF+?71BRM2^<G
z&TE}9_24(PpPRMk-~9eTFd{zfyb2qe*UppqdoQoO{I={|;nV41EBY<eJT<(JO{}zQ
zZay!Q7Bx%AHAmXF`I<)SH4)a+P0#PHIUMhFI$WmRASL!>@ur@A$D>wttvejGtTWMJ
zWpui`pTL9PTXain_4M3-OtAA?|H+6g`35+ZY=Op2yx<1Iq{r(|2nOBcXi}=k%YVQc
zH0#QwYv+{0nliYabDTc3AwTWt+3oX+=Dv46U;JuTZTh^8(>AGiE}AJcAyu4-(_~ks
zg!>dHjxB1wmVMLh=kFI~GSie~TjQF2Yg+2T>Yl_{O|CiJ2G=f_t<qZ7xo`Pvy%oQf
z=zq`Bk>l0glv{D06I9~wnQI8jNv`vGEHYBn;~a91o;t-`;h3M4`8?#CnVf@wH@nCj
zT{p3<5o^t^<_k~ybAZ2d@z>U{eX|t5v*(L{IJ{<#S)YdM{mUji%%=~T*v|jBZPk}u
zLdPYXB_^f*uq*TlOyHU4E+T99XSwGd^R<FamCIfi$V^$3E7-K|aB072;BCKi#k+Vv
z?GtA4@SHT^`4Y|4&MD_xiu_uOzwT!@&*8c478@K|15Qk5W<%Y2ZCaQ41__aFO`Ems
z8#js`5st{rRGg;F_ee}yL^9&qIgyo2nR1GM{&2`xMC|za|LozYz{Rhl-kA9|8=l#0
z6EuHcmVoz+tu2bb{bWvCSI(Q9`oqp^qKbs_qa(8)%N&Sr-0}KF^|h(nQg6>tD|F^#
z4m`3+fJLcPqxJbh)6A}GB1cPq{lD1#yr$s!0bOgpV-r^`()kH?;FdWM2i{*K=C`<M
zLAk&|%MIT@2;Dl_(8@gTfRjk7+jhxqh1}hDL;f>(a@gxv*8Z7UJ$c9L=SQ=2k8OCb
zanxdwNN%?wACpj1Wtqh3U6WGtw#J4&v2;>c`g?!mv<kW7AFJ)&-`|+qq8PRN^pSOk
zze}<$>r`>fzBDCOkz)>Hl!%L9kkIk|v{&t(1>*6h1@mIWuU-0n#nkt(G$^vS%(3Na
zFc4|pzw=*=gT~&5+2&uW&NW@w*er7J+}j1t++4D2I`YrROcFS>`-I~&HOq6SkBI-D
zd-UYA;xopD5xdrH$<0z;qQdfO);@QRS#60PoA1q6I(m1Z_lN%Hwu%9vI^0ePK0*@N
zfnB+aUk5Gg^UdhAnik>KXFutg#f|c1orNyhe&BFw3e!Fz8}&zpxigVH(&_t8<HNVN
zUucbff6ws4C!;+zCl4Gtq@a5A!#mmL2?EVMJS8`zjf|P(FE3F#kW=%}OJ;{i0AJmo
zhI8&t(<JKtaJ>EMQ1!s&+O3G+>t+O=b6CN4@XC+x`iD<X=P^z*SQCBx!!usU3LYm-
zO9_svYAGUe|0k;5P<WgtZ*Nrd|D1x>sShuO55K?9lazE}qkzY|g;n0d+08Q3MI)B7
zg`O5)n~<Qe*X*Dct7H8?gFO`o17gn>JYqTA&d&oWY@J@7O6r^P-~H{Qf9k0YJWLO?
z6nAq;NGuS3%EQwU#k_OQf+U{>s@!KnoC3t}T<2+4=uG&^S7h0^hHdSK16h36CKY{7
zYg6}^*jJza<Ntin05oWT_j2=0jty!YtDfh-Vp{XC)tk|=n9W(p@v~r&VT1RC87Yl1
zD+~+_?(95V@K);Z=jRtB9U-nsN<0u2Vvx{XcP6MMQQ(O>8(WF`LDq{`92n}@7r#98
zMRSg?|NF(RTFcB0MJLSC`S(Bl$KTnY4W85W1-UOl%y61t(Y$%%HRTygPaF_a%m{4N
z@lcuW@p<DW$rrPo3q*M$qUC-ZHSC`A|G(tDy~#VudLMqvJ+iP4qB-f1=$ait+Ut&Z
zr#DP{wUH~SsMAC&`@_PGhO!PzM2|3U*>Su?ed4AWk7fk%@bIwn8N}>3@Zp_o(jv&V
z*+X$_6LbwGyp_1|iTQFe%Vdi=-I0ufTUuvKVvcOla%_y$GB8?HV0hJFft&rT%!toI
zGBLZ46}%QZyg6N?Jr(MNgFcKqF0r#LSSrM}Ys>Ww#~8y*f9&k3cz)o*k_nUTVpuNA
zm>3&geenK%BAfNH>JKh9^QJVeZgEZq4R}B0K74jIXe(OHpN=F*EtaHsW7~ny90P*|
z%e73*LK(vD3xzaydk4s@mf8~1B-t8a%#(J-=z&Mv46TXWoF<byHH{5ci^eGZ|8M-~
z_wf%e1P_0G%@e+Uk^X$pl7j#Lo%Ag?9MfAD#di5{1xu%oNrhLP%~3YyZ3RbOvHl5D
zW?rMp9+v9TB@qAVp%$~j&WOtbdYUP$1*!4-jbe5k*^thEcy;&-El5gn+G4q38IR3P
zCjYuaJghO9tj3xXLdp}k9_hPw<QEtuEa6PqZ}8x&_&TSi1$&P?VOKiMzvks*d$*%s
z{@a7rr?c}L?WyTZb%ahV9DFRlM$YXs^EEEk@QQ~YbvG4XRFcS+7T`^gYC5K>7C9lX
z=s}TOn()*aLDLq*|4V+Yq}csfEB{||$w7st^7lqDI}Q|lR{Qnypa3K)oL;Vd;3W{j
z&9h~LLvr#jne5r~6L=3b2);YA;>cpTehazO9gl?0mh-f{(qH3x;{X2}r4Re_*{*aP
zH<`ewoN8dCbmRYj<C;Gw3Ldi^-dW6ZeqQ4JPSB3W|NotMnhX;}BxWd8%z8cJsur^w
zo6~0%DQ08tq-U{bA`T{7L{$hfe-M|8@a9|7cQDxCK@oqNjB$cD`;3Jx+G`WkWS>m=
z_?Z3h-fEuo^PpgDs1(xxP4I(ff1V24?3G~aIpDN9%uso%M1$$kgj0)HS?X9%^esH%
zS&^mM#2XZ|X$P0YAG@PH)h4MCsT05cYxg}n-|FawV)aL1;yw`XA2Q-N*0A`1!m_vn
zPZQ7BaW9LDVaSd;xQb!5)T3bLWnO>I^8NYAa#rRH3uC`WlW({Qf6`gk8JSF*|1@mg
zU!QJKIVtflTbsK7hqqRcGVIX4Uw`b~OdWl^xi07b*j;d<=?AxTlUCtU&MisI(|?}W
zG-IzZn@p-=T0??p%k?i2(V`KPxo3&an3g0aQ88uV(w_bGl6>-MH@0*>{Fck~+Yp+F
zW^Ot@-%di}!eqgJe?Xzbnjv>MbhmNB5xuM3yj!01sdcPga^#lR(f37bnkr;?_<fRQ
z$?~kwWY+urUH|aLWS-q+hB5nmD(nTp<$KaW(ZvzcGb%puuJhws78vnaEbOo$YoVyY
zjvXqd76#nXXIh%BDy6J{=%vRa@v&>U+M)QmKk8TB99Z@2e|=m1f6(6k8{7LIUeR1s
zCIoR|(!mn*G>rr?iHb=~>kcTX9Tz@RQ*mQP%o?Q~4D0^%ZvAi|bb>(w%Zd6&S)omO
z$7eAWbUyZyyuQxx&CSCFKMW3ke(o}NAuNsaaBtah{FcA->69HxqL(N0?C?(W`CP~<
z`L>JKr*{Vz{~AlVbv*uD#~w5MS=er0`@mCg!j!`Ia)-aY<@x>H_|AU+H$RnN@w=Ug
zea2d?KYzK`^z!UD8DZOLDk~tliDl}<z?_7qkJg{=cv=5&RZ>Byd<3`jmI7yCM{eFH
zQy5?FSQi1Fiagw|^5!$NRCs?%DZ2m7&;L7SEAKePA9GYQtsvoQANK=);RtSqq#(|9
zER9!vY>vt;GpfjE^|dWpcff=_e3xTzgSX6~wl~&)zV#-)3TX>oE+J=|^y5=&i*PW+
zkxnzC4?Z<qb2mKZQN<~Ki|HqX=FL!X-D5O~;ol+V%kNKY+94(LV{VuI!sJaGr)ivc
z<`8u3@o}EZ%Z)*M`Cf@0zP^5mY&$d>5)XK=S8(yLvFYTn<w!DVA6Vngu`TL@il$9W
z%uhM~15x4+CaS(LSYye4nd_15(T0f%20KqY0u@|44s5701?|j!f6wsGFV~_xh>3@G
z&STE5Y>141;HA35X<EWE_s?d_ypp<jm9ka;Fw{Rh*zmC2<me7A{vC=ZuJSJ1`y)E}
zf1!!>gD8dNGH>pkE%;$@@;lQ3s98qZ-7}YWo;m9>P5uOfzo>J{)#jHfO1=u0tZaD2
z*Q{g-e=at?cSqN>{}Q=AKF9w5e-Xp+_)jG$$ok~dEox`YnH=~45@yT{nzKFPPMUgU
Og7}`UelF{r5}E*j-wS^L

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..df3cf2004312ed0ed0ebf1f0340cbfec7fd9ac46
GIT binary patch
literal 19251
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3`sL~37*a7ODM5mDF#|gT
z0|UeV|5B`r8Po?o^It<}XJcpP0$D>7MzFJw^gD932JkS?zW(ZXYsYu{r3xDR%zkp)
zHa(D0n0)X$`@NLNwnCl-o^G4Eu7`ep^ZM89WsC*i#Tq-<7!1l&qo&>wO<Zzr@+3y1
z<oD-2maUo*+0eIm+uCn)w_m&)#hfsy>i@QXwg132UAc6Ux5Yq!<$z@JHJND@5}kiO
zZ`NVq*Yp2w*U==gYTtC_mS<PHE}s8v;ahwCuT#Nch0~Y5?ai0Eziq#SvsbQ{o?Fl2
zQrSK2lJnWqm;@9&jjwTq7WS#EKkf448G{0o;EJHBt1SHASzi}jx<KSv&h56xE0);$
zpOxIYN|N!OdH>_ju~2tR+Q`-HAmCD-Z(5{uf>ZJ3yZ`@BI%yv~<g~?Nv9YF*lgCuO
zlrGJ<pB4WF-6NkVRu~ue8HSb0uDh;v?DEq`d%{DX*Vft1pXKJ95O~0n;f~n^k%d}k
zGTcHt0$P+RY=1J||2cVr%0!{yLvO!n@T@vvlqSi@$GnR#xBARyKc%I~G8eyZ(z#*g
zCwwjRj7#XZzt_$7-;)KUpfi(fxj_yK7cV-*>9+C1;*FpG|CfyB-nvOxXV1~d+1&!3
zIg{5uGYx*Tr+!MaWq_)~X`^+kOit$n+Ptzl_jyU~d|zQH<#Q7a7j6}CJ=Jr1%l&V4
z?(5HWuCJN+Y_tEW^lvBLSkFDYDPfjDAA`c8!wu`4?<e0bndjNU5EXvwd;I@j-}t~0
zb>`Y$zGed+w~P92T!LC2IxL->oxR`XE5&ErXckC0kj8N9OZtmzRo@>~v;7zE)43e=
z+GOdjZN9<V-mcYnWaGc;RK)6@iMh!R9dawDRIRGc|K@qP`tgLL+bx{LuK)VjG)JS^
zl`SRmhULe3i#wjr-0k-(?waV~hQgNO8EO^$N(TRb{L|me3=WW~hFlE>Jg)T@0yE9U
z9ba6?{_-z*VMY6+3$6u5g)5V~&&|?wHrjjqfBnS65^J|^+a`Vdprx^|lZ)5Z+I}O6
ztPmH;V_R)beq83E=q{;}zx43b`lpP>>GAn%r^LHd++Ug*(UN?`COL9~&>nY=i0~yW
zf;qRZ8=SKQhl13JBL|Ni`}6(1y{#>0X{n{KaH6KBA~SQKq-5Lu{r@*^_>-QLl#uX)
z`~2CSp8o;HRz*CHlQ_QJVl-Cb+-ehW#?e^J`0v6_hgA|Dlf~xQJ?Lb1IuvvI!+ZI|
z%l&!Q#~J?lb+q88(cyP@dF<<rV)mZ;@LB!v?d{AG#sSbmWX6$$$B!R7e}3&k#hk2F
ziFtZTYRA?tIB@pxVFqu;55JhQB^$O%HcoOk{Icjtg-N6guZYf3&TCfOYlRw@GkXZR
znEm-*U-7G9_Q?<b?U`Ht{G8l4ckaVyzHQ9x68H8bTGY*X_|&`Y`FY8If2B7p;{^M8
zQTd~U#KcS2rd_%sVs0K@STI3c;K%vXr;i<Mw9QM1{3E~T(Sd0$|D1}niq0GiP0BVu
zFk5rRQ3hr=VHUxOC+3RBsIt%D)n7DWGT)MyU8|-yGHTX39QgmAM?cQs&fe1>UI-qZ
zXUlVay>ZQt10P-rGyk<d2J!2n_(ut8X;&^!zH&W$)7Gixh7-eslKy{xuWxM3T(riy
zD@MkZ_1ltW-E#%k>{fI#R9|wu+!`2_aLxJzCsSaBf~Ip=XV^{$W_IJ+KmX5vcqx2%
zdp^%}ePbDm0|j5T4mY#&yuWAo=a1)hiDamK?vEZkOZ)qOzM;~gJ6A(}V`D=+JEu<-
zjWbNv{J(K4=fA{mr=(TVZ%%T@RX*&fI5UYMJhye8MS!ncAdkP;iq1)mjHZzalmGuW
z{`2?rhu7kVR|fN>pEH<af4<-u&*64{o}QjBFK$8Ys(dtIqTq&2lRkghVqz*PD*X7-
zv$T{1hbv4gUNYM*Gn87Gak2SMk3*+KMQuxt&b&hIEk_z}1vrHF<XA-6HYwj+IHh1k
z=VN(hw^I(u2ZT@k{?4;K-*}GY;SI^`haVs3neb6)7N}}s`2T-V;v)f(l<@HBpFdyz
z_}yGoIQiL=l%^i%#i5KFZZYZxUHoKwO^eO{WaFLrF5Mn6t-@)i9M**euw1-llEKSu
z(*E~9zvt8Xhfmpc79Q{x$>=zEnNeAJno>(+<HP&*ZT#{Qa<)k~c1?Zwqp;1oT*9t?
zS)vf65?K`b=)who!Xl%R^6T3+PJI3#AtAxRPwa(ebK^t%HP^dN%sQdiH!-7QW$Feu
z%OcK})d2^i?j?xjd|2Tse%wsp6Mr0+lBxH@muylQ6+-MW2mk-)ssCs2=ik|aM=XcG
zz2#Z%Z){U>;KNsKUnN0ELahTOm`&TldO8a?YzPPndX$>f)ULy9W3GH%?Sv_l^5b~6
zDH`tn!XA&=BaTc2`D2OSv01Y(u1PpBc|!s38b#jl1xs@7bTaRC<eJ{ZE*ll(Al9fN
z{v_(zefzfK{gQn0X*c%IfA}J>jbDDneJ+SM*UU8JJ7mD)dVhOD;>xJbox#(J`tB`j
zf4#7(i;uIp)3aM*!ij{VeSd{-$A6k|y8hv=cV_4NQ)dS6`u<n+16P<C$EH8~=kTye
zvV`r<o@%J&y3UK~YJqy>ij9g(R2(*0U5sx#rhe*wXMvM7>nTR&_#zYQWuM!$mL7RK
zFJ$V&HP3A49cYwayVuq<Sw@m=S?839@NK=d|M#Zfzb*_aEf+<dJb37kimvXP=jn6i
zzE#mYdG-_+Cm&md-_aFe{}NvxTr}mzjZ^$jcClSN<(%87q@8>qs^URLvBGri7gGhd
zDTe4Q6+Gs_)-`)#+M5Zfik%WtlXm>tenRO)kA1zwy}ijdHn%4}^=exkE@@YPY~HpL
zpyUlobbgN>JW08BaUwV8#-2ViagmhtloqvOHa51LlT3S+4jgpI&PrG&aVCvL$tsZj
z%Em3y6J#2~#2>6t_;5x4$7_)*EL~>~%(U3BjMF6b1n<@R`z8BilOwivJv_qM_Wb-x
z;}xKm*#G~FoVFf1d^jy}p^#YG)5nX|6%L%^@D?|a3aU6XnS<Hr<cS~e|JS=!tn3ci
za9EIe&9a1)hHd{O6oL%8Cv>N}Es=U|e@AgeCBx3PB$rDk8Uha~himw3&~U%Tu`=?3
z=#hFEl_LT!vRsJ{t0dN#avL?6TTJ*Q-NZdHDJdx-IeEk8WC5Wh0rB)3Te}|~WY+qU
z1oHg<|4U{b<ZV#kajlo;uQE{NNqp2O*z@jS&zU0{Ujq&{RZU`6yl}xuzDDTZAw%E8
z`_>!g9p7kS{wnM3R>lp2)3wg{EP30fsFXBO>sVIV<{%@*xAW|Ne+=VZH+lJs37kSl
z*EF+T^R|vk->p<pk@I6kgmCGfm#@8Y-tF&lU2>R}!Px15&Ee`LEpz_YeVE4pP5~N)
z8#Wt9M{268Sw%%_fBvkzoG0VN;aSQ56ICsRbrwvN&il~euI(i7lT+!_e|wvhi(C>O
z_8fAdKa@D*?j4X}D`#f*;oYOko$sAA(bahV#E<rEdna7eP+r5>t#Kg1kwwzp{i9CW
zl*5cMT>e6dDJcN~2^~Et4;~~$Z0~#cDYa|X3P`vt(kRTxKAn<Qb@PVC$B%^uMn(&d
z=^a^nVOMh_2yli~90+D+&N|@O)wSQFuxCYpL_#8${xm~juAoGQje?u|8q`JDX6*ex
zG38;TOx%_Qk&QV_ChsnCsYgt4KKuW_eD-~=-h&NqwAl7|vIr-H86G;sQ&`B8lLIO>
zV)l5fv77;Mhmw7vhIaS#>Fm}P8pVYh4<72^j4L{Dhf81g;Z?3BTn!8!=Av>pKg4hd
zwrp7`@buLG|2`YqRAdfxGfq)(^D5@?n0Tr|N}DGmlhxGM!08m9O6rSOigv9l6I%)e
zpPsFM;Z)|PQrOZT;H@?%myazclQr9!g>S|Ti5oX17cG{&aYJI6Z}N^x;r^RHpzbjS
zxo7J1-<1_BHg0m_;!$&RQ(Mx*C!f>McUplr*+9WmUqP|^j-In#vgE``Qp!PG4U#82
z#9nI@c{H7PYR^3BVL|GLZ(N5MGS)VA_^rw87S!D7(9N@?O?Q@*&teW0ZVyJ6tY{X#
zhzJQLX32<1Nj0B@9i`nH6k!3PUbtdK$CBmNtt}FpHW?kdCHa9P&8f?b?ZykSTM~lT
zp0O4QH8MQmdTPH$u`}l4f+LbeO%1Eu`1Q{i$?U!1pr>dcrTSte`^ubzhRF6gc@yUL
zX4EWLmDr;0Sg~0o+b7w~!hoCKr&mUXXWl%XGiP}I{^couH(}2QS#bJPvMfAt>XWNW
z#PK5$MFj_L&->t!nv~#>S7~q{OV~wOWb1?m^LH0&TIc8_eCp_84rW=a^q*h4G1VcI
z;d)=#Y=Z@>jx6Q5$--9gMLC-Hu~L@ml{W5K!59CWX83M&OMyM*4u{dI#;wk$8>WO^
zV46I8_Q9)DA0FjyJK8NNXQ#5Zo*Uwb+{Fs~{Hr%_t_zr8Xm8KGN3ccasRxgmM^xh!
zRf$w)uZNAx=1vq06I&Q=&QtJJqLWj2fvD4rR}()QsvR=n-5(esaAdB)dxt|@o-C(Q
zlbn8tN*=tD{P5AlBp0c=mg33^2_Yc~85xO|){U*gdC+)Y_#`1I>B^-^{9Hz1Ay3?S
znL2_6o-zha{%o&P|G)if*o3VH3nZIVe#mFrIh;K{BT_)`oI^}tBPWN5=!!=!QXKkc
zBqSvyCM*kASjzqFm|9)KR*r*9KmM<0<Kt_4lhf8OFIo03<;Ry!j&NfG0|Ntytvxdw
zxfm2VT=(ZSg>B~9)3&|yU45_E!w9!4dOX^1e+$H)bYz*hJ}h+Vl!{+>H%4gvRzLnC
zC~<E`veKc1Z59$4l84XhDY7TV6mq88zIExItoH48x$e7FMco_CME=f|e<p2Nc<bwT
z!5M*EQO#OKbsj<t3&72GC9}c}n~bU}Gk5OF`uuUDk%56hg8aqgBO3MstjudlT)o9I
zQrQ+wY*b!WI92QfkBy~t_$x)V(1%4l|4Q1HKX6pxmXMA=BHSq;cGzXc3<(KI$%qJv
zEn6h^?D1F;1M!-YS>cJ3oh&R7Cr%U?8X6u5VvFj3VE^oZK~bFJRHtv8%0&#0Jxz)+
zH>D&KbiE}yv_BV0cC^jdV96Yw*nHJC!drHWPm_kFM2Cjl1kXRYf;+tI?AX-R+2Z5b
z?CcgXf!b65|0`(~c61e~s@{x<@;V^aZG515#s&)q$u~{sgN%$8i5v9v@MJWxybcd(
z*dVw_x?Re|!AC6pfPmr*CC*hx*2EiZU0~h3#EEI=9n~3j5|WZjmPp*WE9v7aIb&vF
zH6I()BU*(wZmeKsf4zB=(sBk1huIU`mrML;v1XnmQY2t-XqBQ*!}5Y-s~E!5j>bO;
zV4V<Hp`h*Ht<<qetbS#~@>h*g?as=^#?2E2n_F3%CkQ0&hqP3b=2%Rb>iXwzZ%Rsp
z!bU!S`4gE;$2UE!KfyIYTXsg^|BX+J)-bF#@;UgwBDIrc$0cn&wmQ8cD}`k{oF*NZ
z_?wd}DfO^tSy|f-3%2ZRwz9G{*OZ{{onWzJN5q@Arfb%!CNljG6bw69-_ui{m>uzN
z(YZDSb^VxS51Ck$=e@b&pr{aGktkL)<IxV&!;eHStvhmg$$>RV8&)NzqzH(leE6O|
zVS++}DI`x$uvoIBMMg%XwdKZzQ*29iF-QK9Uvt1gVT-Pc%M`JSN!)sZNnAXiCh>W0
z6-vuxN>AQ4!NXu?0(WyGqw3uSOBLDJN{;gI@Z{z3<mItwi9_6aXu5@kMMO@nC=ajd
zjhm8*s}E0%7CkXL{GllOr&oLTd#rq7udMv5je(7gt>+N)(%z=UQ=F0?u!rdQNf<7Y
z4;O6?ZDBmA{A5eZhwUbXQ&o2eiD_#)7Z^0=>4F`7XuidbGiRqRS!x+DTYSROi5a|n
zUznJK(iBRg|2_0$<hgN)otgRYe|h`5`d?o@AMdyS|8L<`UY6`djjNIr7_ys1xc>c5
z|MU0v|LOYw_t&Q@9G<&m8UGGFww@<T*PMCq`m};V!(~0FPt0rb^7L}EwND>A)(~Rg
z@L05{LF!J{5>1m6hwPqM20gjbu<e?-x6J=vU%#K8E<f8W*1GJ?#p~<sD=Tg+Sz3AJ
zDyyk!>yxLIKYv67x(9#Q66KL~;lSLk`MoP8e8qwdU0Oc=|Nq~<?oa*SU-z5Y>+|ma
z`}^VHeSZ0Wh6aDagOa{{PM$JZ`JsnT#hS-a(rkgv=}~-#o<2)6FipL&W8%Zh>Tjy!
zpbnjBQBiT@&D*c3DG`YiPjRRxh8<FttGs!qeu<*;3aKS8MYl}7^Ww$R)92gy?`P-T
ztN;IR_x=6<|Ni;O{)jg*(N9^q(BEH`pTAO6)b7N|>=nFxNw+rt5K?V33Szuwa^%1P
zhMNsv{y#lk-_G~<&YqvgCo1osXY-T2lc%hVEibQao!+q?o@EOZ+*DPCj%~X*iS?F^
z$TJn)XEF9EXRchF`0!_OTmJnG5s*A`Xr{%NFH4%5p3I#2KlzI0iOeR!m46!a+0q&&
zf2h;%ak(AWAQXCje*HYt*GIm*RFB{Hr%7x_{=J&}TVEf1dHFoQOvQ}3Yx6TT4<0?r
zBDwnLMWux@0+VNOs3{+-|Nrm*zq18DKfRCKyf5eGr^i2wr^x*Jtp3EK^?z?~z5l$F
zGw0H_Y)*dgQ1Ok0+nv*P5)v1VPUPm<QPv^F$;nn)%BHCJrjr?x%I1MG91qWv))t8k
ztO7;3pZK@g$m=yl)N@ClY6#oBKvi(X#`t}I=2(Bf(j)molUZ_4{r}?QeeC>lH98sw
zU%pKf7fd{SS6gja$bqA3LIz4x#WKD*_5S)l|KaKB|9^k~Kf~nZ<(0wv_thNCJi5Q0
z=l1sh|G&Nc|NlRao?eT)g2aSN?AJo=GqfZPR>|!U<K}Mjoh>oX=3o*l%ymbO{?ySu
z8WE{@@qELUvNpl3azFW3yGu-v_`t-x@7f$e<!2YZbL8Hws=c|1TR&dthTxyi&+lhn
zzjyVz{Qu9-``g>s^zamE_DDKs@R&@ExLI(ZBTGsyHX};l!P(jSkN5x2$op6E_SXCN
z_c>Me{r@l7CzmQ9kRTwOY+!gWNGpNq%$I`=%9;iXrpXk%<>BF3wTfrgE}r!BNt!m$
z6nTjyDJ^Zvbn&2|N6Cpl`)_6cJ>O7O!T8Fs|3aYt4UQZ2SJvrBIeBWFpI;xh``f3x
zyYv75R~KmF;o;d||G)av6YuorJ8i4w_Sf@UK4;TiwLwg=LW}i`N^9hwgw*6^mMlMC
zUfzFsdHvPv^$KdK2lgD>P;Gv=PnPHXec{D(8X4W~TN#fxNQ9aC`0&)#@hB-p@PNjP
z|NlQU$s!_h=GLvPOPB7HxbS_m@-c%6?7KH?3Uohg5X^b!%>V!Y|KH#L|J=>Z{`G&A
z%DC_EujeU$R}-`G(YJSZ`vcp3E_kp#QkqxMkjZrJOX0US=TlGbi`n<5`@|GR#`yh4
zId@JJJY+igJN9Ewzu|&qiYBH!JUmubJexK}oMC}P-6V^D|I;sAnIa&P(lS*tqw=qQ
zQBfm*gOSl8;e+u@9u}`jWs7-Q&(>By-@bl-+1sa!-Q_*FdDz<aR?pA7@!`R_x&Hg>
z{<;<^hBc^1Ca5W|;h4TLLxsh_uIkJ02M6E3zE-Q6eE!u`ZJz0RhBXIoemTUqfv+<u
zb#Z8fz`Et?(=LJr))~OzFlFk`-X513Gaf`wo~S*geuebyhoPKl4;!c16^P3HkKg;N
z=I^i7_VpV|w|s1}um2an=SSMhOSk3i{<J(bR$43;xcFgc^Gdl$naw6i2?-5)%=fmx
zKYnqsy@!_Zg9p$3+a{|^#_d&d<i6&`GB4(V_mqjVl}{+?=<rBMMO0mcCXz`OCZ<nA
zLn0<diC*Yvk#}k`Yix>|s^Gs<adExJllqin@9)>=-K}yC`f%{z-s<_`Yj>5uzqfq8
z{r?2*4~DZvcqBIQ%n<BRY&|W&7V@s1t?hsP|9`(fJ$-+1vAy41rT=&KpPufiXDRVv
zoACy%W@a@tHak1E$jC3QauEBS{AGN6dVc+SBqQ_RaI=Mtpn|+M!(*paLVuRBrq%sD
zec;3VsZ(DbWL_klu;Rh-{{7|e>#T~NJU!eVUtq#<;etI+Bpc7v{|09chBQY`b8y{P
z^RxWbmG9Nx>uP_w_-$ll{Qlnf&kx6_&CIVSSgMH|FLZDb5dpQRA_A<}!(GtT)%5Gv
zn+czv8_t@Rq4eJ{n4!;#b*GM_w_g4I{r|r`<^H+5w*KFr@>f@0OPkvz95G3gN%l4T
zZxF=5+@^P?h$+&V>&Ks;_dh<ikK4E8MNLoseTjMY>XG8lY`E3L*ZMVnFZ%gY^2g7M
zypZ(g<S+B*&z+hY9f|PC4?2ssOlgdqnotyY>G*Vgdk;Rgkb;+&b{9Xt_b4O%fbxtb
zM$L@QV&R2NX+Qq+%m0(J{`UE*wimzu&%d)*EaVGfTsNtC;zUP({@MaDaN>6IlsR*z
zCn@PoQ<DsjhP#fElF-)`vQNrh#9t8>`M6v~Waa+)|6Cje0goP<=hxla_xI)N>-Q_P
z1=*CcB-vCx)^7-UXy10Fo~N+oVf_Ao)n8tG-=6>Ex~71`w~L$^y}oQRqFibQhKT|q
zE`pHC#K~L6M@!3Q(V{aOcDQqEx?DMAwVK0h62~(Ii8Loi5yhAook^4Kq_nXxGcz+E
zetv%c`q<w;zPvQ=(K*+kEBoj0ghR8u6d2YX_|YH!$DWz_@N)nE^X&fq{;a;Vvv~&V
z0T#xn)+U{fn@OUguJQMvR(i_pulp-1@Zj;oMlt>g)6|%~f<<iCCP=?I^k!kg+JFTf
zF>|~F0~;nO{P_I*{@dI0C)`@X<B=(~c}s$WuM&@epv_aRFlUK?s}9{W{pbIy`~R>1
zkn&C!SIHwsPhRX;UU49&Mj<;fCFRGrUPpiarM{4H7$;Aem>3-|FD8$JT``@1k{u#t
zxcv3%4JUET^I}QOm1t2^JGOASx|g4a!k)7y4j=yX=coCd)Wi0B0{h~{vMpP-*8Mzw
z;KTpp{r~su@ezp&)6zbzpz2zdAt3S9NzDD-V*^9O6|0Rud{VjsO_(2BriiScC&!*P
z>p<AfKhCdq|2ryhFe)Trn(BWEiGLqIZL+XTl-{E%_2=*J`O~L={rXz|h$sWkil<!H
zrZmQ^dtfeXYxwuqSAU0w&O?XgN=rD_|2xWm!ENt_B^#K^%GkujL?WRUdCD9*WOL`v
zen|;~-CFg9Q+Pi}J=ZVNX6gLX9CiB7;RpYc6Mj5@pm6y3aW~Hh4$h56hK37{Nfw>@
z*RR#5{YY|xM8>WE4?P45iWfA!dBYPFbY_Z-+L?m~A3Sq=v&^K4VeR`aOLJ~^A0M9j
ze+#yh!Y#G2c~k%IPpZS5c|XO&$`zWVFU7kCG4#It|KHHv>Ejl&f6Cq)HyNd*yx6#T
z@_n}C<m747pa1&tQT*L9wtrl#;Z_O}*B6}j@-#eoI{3<rD_2uDY+M*+FLqWYMw5Aw
zk6T+K^UB8@usreS&xFO{3(j&pnR?hGeW&wpwGFdZ8AuC9bhq!9*rah%Rn6`05tkmD
z|C=`${QL3I-MP7a?p#54mc<(ly;*Ixa?WdMiZuTl9G)&Fn3%Hgfx^BcE^L*RpSBu5
zc(*%gno@>CoLp)fq+#Hs!l$PvR#=#(r}v{s?#a(z`y~W>H0%{TSt`TC+&6w~Z+ys~
zwXxB_H#&O5k|PuL9CCJO2o0YO8gjC>R^Dd9sQBM#feOcy5GS8q2PZ!eX!`&E|MV#-
zAz^9PuBjQ9c61x7s83!lDw`lC#Nw@-nJE$r4Nw(6H#fJuJh8mI75Njh#qLaAH^X^;
z#BnnY^N^V{{;{&0SiU@ak|3A7{lBtuaZ%x9XQ>&A5y}r%CVdFE+Hp&M!hxUXPn>x0
z<jAt+?tYI1gp+63%xvskbWNO@nfcL;>_3O&BcxZYS;GMuhGzKx-$|wK{{H`l1{*eP
zR9wWnQp-`WrMD_^u4kjUQ{htPlrQYMs;1_%ckS|;EG6dY`AJLb&Y?p(8KI)@zOfXQ
zHbmAbi02$p5iGfEVk#*s6LIE@hqpWv6AvpZPhH)TUvkz5td=zL7>G;%|9}3&KU=SG
zXyK;P7r+1CrcFf&B0?F8Kc+M8{mbY&VH<~8%htb*r6OPT<>fE$sodu1_(@0S$hPA-
zIW4iVu3x|YG)NNHJ``;5z$+p`a)z6t$e;iJ?KL{C-ks~}*B23AP-G+`YM7EPymiYA
z1qVH?4_h1zxu?q82UK}NgEn(QSJ$RQ6OISG{uP@X5*I96V$bv9D*HM!&8ntmWl71t
zo153JT2;c?r)+3w_~PY9RrS+9e|K-*w8&CAdoq)+KsICZ)`|nc2@W@SPI$IY{5ZdX
z!MV_|IznRh4vU9h4jwsTP^#q|DS36*`htczMjbN~6H`(iJaoLG1GQhIR!67j&><ck
zp8pfJsTev){QqxkWN2vQaDA$=p~0WjloTi1FY_6XHEfuuEH=IPpR(bE!_yr-I<_u!
zynicd$;y|PPEOAM|L5SOAFVABbLY4`dXzGEu4_cZhwai56BbQoj4o1K7T7ek+x>&Q
z{r~-|w89b}{`a2tXJ74a`)i#hX=!Y=wQcj{*xcP?KDr7X&~f?@Za8NSPfkupZ#mS3
zDtv)~WqI%H|Nr}&+Sb?nCpjUZA-?#_e|ur&$MH87ZtgMYURW(?Fp1+1XGAdDr!s!F
zGe-^{_%MIs$NjTs@0dCB@5jgMudl1!Q~!VU@&5gP|NWic+36TG>GCnR6Zhi}9dwL{
znKO0bL`L_Y{r0~)JFAmK{u~x?>EPh{r>~W?ZNrcKxwr2tyZ^hphjZ`Rc{67^#>_c*
zWs0J~goCHM9j7ch$(+QvXKI%wAA3y=Py9ZGbDJPpb<12^&@5}~{G{yTJd45$i{JfS
zugEpm#pJx>k)+T91Lw9iKPyg7m@mxtyk{%7XOV5*Ud4M^Dv#O4d;YAgh}qzw+4<9<
ziG#6A<evGx>kP{?c@yt7UrUSsRK8l$zv*M=?C*Upj8cnZr*F-bV>%Z2TW*zrW2hs`
zxxYqRCf*Vjo4zY}nnc^@iRalsL)b}<L8pGazaOu*tllH^u>F=#5B}Fbxx9Mf_ale8
z1=yC9G?-3Zuv?)1SA#Mi9~&QESWM*<saD31r#?&-%Xlks!a<4sxIJIr|D+_ISFe63
zPgHA2Xk=vX>EW4gue<{qNq#as_xCwUmH7Ps|35h;#ldRsOwLZ;l}Qo--M#-;oa9K9
zWou^CG(E8M#L^S>A%a^X5*sG&Ffu$Sk$T`Ri(_7_&}6j_D%>f0o356y9tlxB@!-0?
zh$hr*mA?A_|LT8#;}=Vrz%4&xL#X5jr547QPx9Gvc-Z!=4Rn)Wp6TK(#v@_q5_Hp{
zMrLw~#k!EzYabocE=xb~664u0rP1>Fr~mecSBGn)8^R6UU;qE_kAn>UG8@9wmEXJ)
zR@?IB;DLr3MK<P5dJ@vTE`>ZooZ>(GPq{hs+(>F&5~04#LQy|7QH?KU(mjU%O^j<k
z{}9l-CHewxAf(SNuUTkdXz-``?wq4XPW<4%X6ra7li!nfgB<UljAFJKBFyh5H8U=r
zaa2c3B0#yJ@r8!^zJSC7%NV9xtbEBF=gDhxz&QAP!<2w~dm3dp89}KDR3v)J*w_6@
zPWbU%zUs3Gv&}n6i3g!PCNB^83%0Iy*%vEr+WA2K`H%jCM^8C)8=tXITfa#mRK~x3
z8snxVLZMj-jGsPO&eLLxeDyYZO6ysui~VHk_t*YzzasLt{|aC0{q#1o4F^v!czbx5
zt3J>&zTjdonRQNgWKKfY1gW!6mh!MYa<ecv-NVS%v*P;fWY!r$@9!_3A_Fx`)AHY+
z#D+?KZ)=gT7K24<d~8dWsd~<hXi*OI^wDeSRlnBRnWMCOlgi6OSJh+_4xBjg;k|rg
zN@M7^tJ-by9=kiB=BV`9*Z=$b?=Qb*>VbT-lp`KWAGT?IQebCZ=%(SfJZS<?#c3WJ
z!IKU#+04FN!Qvk~d9wX08W{hvX)e==dGgeI$`NQg?%gS)n!jBBGCu$R8!w3c(H$10
zuvDlimF-^|`<&j@XB_&R*m^V)9lE5qlyP3$$h)aqzG>^31q>-75(PKS^8-Yn?ujh^
z@R+?sfM@3)2R-8pzcdBdZrz<}5SjG1>C~~Qb_HAAI{TxZ%<-+UY>bzTl9JV7xOb~z
zIoqF;Oicg&I@|w*tN=JP%i>>hPga7j<cz5to=4fPD6imbVw~C?X7k|Xh6nwAY>%c)
zoco4N-@#92hHr?)@Bj5}+w&#&Es{G48qoaz-^o+PzW!h9UFCioo35>_U3adhHk`d<
zw1BU}aKWZRR;9E@Kl*#S=XiDWcn4iIm@uz_P18Z!?89j%iTry8HUIk-@j%)t+Lr$g
z+?H9P8(gtoG;XTFg3{E7lYe_WGc+>zqrU4&!T}9q34=w9{tp{hA2?wDr*(gEcl_A|
z7M9)XBd$6#q&<QLiN6dukGJ&oD!JM(`rkA~;X;<!8XgPr%X=eS4Nmj0^=LSD+cybz
z?qAuc{_=rQfd#`BlP~Y(51*d?BKkKpde(;}C^l_9c(7q{QKG`4t&QqFotl;s9o^wJ
zPwhW;Nft;O`7oqi{Pa~DG#j7s8#)5(<SDbi{{PS4-{o@xWHxJdNxV6f5od6ytNp)$
zAe+k9h&k-#PnY}G|Noo(;Bj(O?vdZ^OBgJ@{w2D|t>Z{ru%@(grUKi7*}tCiAAWwG
z=g*%b^QQT$9Ef`P=l}T+?_>|pwSI9G-X8byskg8Bp{~T#pQCnTh2=Fi%{xMFds5kU
zc~7u>fA8;1<L7sOf3Kf3xiuv8=7cK?RQV?D+7#h#EzyxTe^I0f!-N0zZMxACda+3z
z-N^v~3;G&=+*Sgw*yxU13k^umY4`X4ul>wsH08_t|NsAQkj?*}_}KgZg>7mIy*hsn
z@AYOC|1QsbI9_ep|IO+9Z|?f~`1Ex9jW1nq*L0WOwp(_ho=5oKLaEkFyEz}_ug+;?
z)Q>k>6LWGy3g_Wwc9rdV8<qa-l8}&icmH_N4pm43@RWIf|9|b@U-Ha{>vPiW5_{Py
zyqfZN{%e=ow?j!tO1@wIf5n#<;>O0!`|PYQ`7PhSLR08+){|WQ8J!I0)3nxjid}AH
z=CQDl`0+zR&i+{4Oh!kCtW!V!&wu!@b_#2g0L)`D_5Y3bRDKqJtKw<WqS8G_K|%Fw
zz0AUX<ApkLfBxOKXS24}i;Fv>(kl6JMx^}ejTMua>P#Y{=GN5kJbJ|A<HHlZU1&Fx
zq<@yeGMCDqLaBCQpveP<|NoPmjGsO}ZqF0ZSN@CNoBh9}#6RbAZOvD#e*_EtQ+odE
zt9<mfnuiY)we-XeM4Kh3iL5E~KVzolbD3=i+d-}wGb9cjlBoNaa$|G*!)v;?Zhif4
ze|UX7PyYQy<)IQ15>V%R%k=#J@7!H-W^zWND$6t%XSY*7K0ZD_Mf1I!{lB?WkLv56
z?^%1`N4uBO>3{BC<}BWVYtjzgwfAV;X676sa(Ht(&+4#+I(H9qb>vS^xccLZ=Hb`Z
zBl13hyzu|Olef(On>YXe{mstlSi#wt;i>Sp$z?UGz)F_#$fl0v|Ns9#H}`kp+gtit
zT5B>h@7!3#8?0{=nc&MiLzVs76$6$@tSo>2pBIonf8f}$4K<dBPfljJmTn+$QT`zF
zy7=^p4-DS0eCjO&>dXEA#oC|qgzug`+YQqO33mra<v0K9*Sr<|67&22|NW(}?fhqb
zX`DE5;Zj$JuxXspiXX*GpX+wHR%nZ_QRUtAu-P`!fk9ZfxslPi#N_Z?Yo6L#9{qTs
z)e{r)St~d3I?c1`TxxpizyWB`Nk0Ga>+Ah?{{R2~{blzw>FN2u@!UmY0|TYliyb!K
zzscX`VBd84-~XSV>pwnP-`gv1U-RSojT7%*ojP`=$7UA;`-zPmHerr`0@$BLToYhE
zbcn~$5VTGu=gx@(M~)ce%l))o`|<pP`}_Ns9y|!Q)KljEzQ6UKp2)umsbO0(m%T&N
zWQnOAk3?{GgfKg^O<vsIU)H&|rb`;{+f(&bU$@uVc5e4vS<6oof|=8T5*)mo|4(rF
zx0owwks{BTGZLFNN#xy0h}b>t;c@=9mBAb5aJZH=GcWpnTD+}aexo!r7UgHxetRP?
zEL>#$*nUkiQ)+_9Ck{b3H?|vdbu@Ud{{R2K_V>5%7Z%DdT`I5U_vg>2r~Ci^mpnKB
z|ItHD`)#{#wjEgGXrL{Xa@5&6)xf~u&;RoUk622!Bp>Hvv$t;xT^$hi>EKjB`)!SD
zx*HoGo|A10U%w$6T6Ah!Mnp(NME?Kx?{9xEuTj}Q{;(p4O*(t#IB4qrbIzaN%(ue1
z&(106{D;TK|35vw-+$((uaA!27jQQ)GCFa}Xm8C=_b91(Ru9iAo=IwEbf4I;+DX#x
z!Eye!^7jiC->Ux)S~qv$!hsF-_Ra0=&HL<{m#CaGUew{E{r|u59IK-r*p8Q)L!%os
zAjA_B^WU!iAHVgAX2!)U8S`@u4rNKQxw(1FW?q{6@xA=v>+5;=<^EU}J-Pnq=>79^
ze(GwRSh4>6rza=xo9F-g|Nnf!&rkFH=l}EaJjh;oA&c?MD#o3epX8@@$L!?ME_lRp
z_~qq*GA}M>dHKW7&hm7N8^`QFmymQQA>+h?B}WdNKY!rt*#oCfA2{6n{)c|d($imG
zM|3HH`sR>UQ~iIVn&034UtV7C<kYeD%L;YjMOzIN8YWH@oK)~2mF@6s^Z#Y<{%o!N
zeO}qU?(d(U_8J|w1{41N`@4VY)ObmW1-t(~?3KRqsFAU=_u2!8u(q3TB=R2Iw{O#r
z7ck!!6>U6YmXU_G@tYe5LCX$LPUhKPZ>*tZ)X{Cs!DW<@Wwc~@^8Vxe;-C`pws{-B
ze1RBfnhWZ*e+D+Srw$zY#J_UR5B7L_9(|{P@ETBM{Qq<N!>6a~zrXui`|ZvB!|nC@
z@&Ae+9eK~r|BsWyKuW@ZoqhA}of|nh4VNwdpOD;~_w?Dbw3=^k);BR7k$Ixj$e5p9
zVo~DU!SesV@td2>vR9`~OMCDz@xryV3)j*<d{2+q){}Tvs?9iE;@#ckiZ30Ft*sBQ
zXtHT)wyn{*Bqj6xo#CG9_P<KiY;2(R2gtSdAK$&cUVrJ*|B2sjDSb~XEc1DH_xI)N
z-yeT_YybY<-}>L*{?9i5Ur`|;D;v3Z(G6y%8!j#qk`fYmcYhr|+&n|1?&~Xg>D%=;
za-YxWY+zIt<XN4^Vff&xcAM|)1v!C%F7~cLlN6Pk6-}l*JjmSk`MIQ=eOkr;c@MAa
zvxSCEnWq)?wQ=jm!mnC~7rWp1y^)6p)Sv;m%U>oZ2Q;Rqul!lQxRHO&y9RNU3ClDd
zzr9^=ZM}be#K+g)^Vt{~-8ebjyuHJurNfGg#cFHQ^z_&~YU=;}d4El}MZ2VQ=8T#D
z|Ns2l-^%(f<;<t#7By}*wzmKOCG+mP#jGuTFV{RvYRcIoM?lH!^l1gvqYGA@`tXeR
z@N|8i{dI;bRyxjVc+u{Yq$arI$K~ZOw9idsgw(eFpf-E`rzh)sc!byVp5(aA!X5ja
z`Mv>fbl{)=pP$zY3H5MtzEM)@;o|bywrx#Umyd_L+P~z4gde}Z|DUX0AHT0<8rPgT
zAWI60|EDJ%nx!D&R<x<1VSl~ky*-Y1zK97l6xHakY}jPv;c4jMHECzh<Hzo8`tb!~
z(#S!yzs~SLS5-y*6Zt(889gN@gmDO8u4iLA_W%F?(<clnDmD}p8GZU}%*AQQ#c8-{
z)1t4k|BMX{Ykq$E|JYsQ|L<}(F)=pA#{V1K^q2*Y9y`HQ|KCW)o-elH@WF!%R&><<
z>V5N5gpI9D-Ctr|+|lqNNZIftW;fRX1)e<tcjlIS`2YXd$@+r8m=e*;3!{QsF71_M
zzS;NW(=J(=HKyyY?%5-k>gmy0k!oPXc1>>4;+yFYPt0_Fk?gZPsDN*2kLu-}oBjTX
zi80KWD70t(`A;q@zAn_QnHh5TN5`by3m7tZ4fnaug@m}g;@=+=AO8IO|HDW3C8E+7
zRyav%wJMzmkvXRO?f=BBHNU>R|MYafyq!bt!^g=F9{kYP;@Zyfdc|uFUr7N_jK05r
zVaD%&a}OS2Iqj9aMU0J^*(>TG*MpZYzvpLcZ34|PffAj*<-h;w6`!8Sui5g0UH)a$
zRGA3}d0tFlYhir-T4}e?Ba@7kp#IH|kN2g`{~eA!cKGm$b>|M8`}h97eaHNb-(ubM
z5>iqsev4gvFkxESgJ)?!{z`jqX5OjN&|+UxB)|4~6WhOksXNMgA6{0kIR^>NH**d7
z8VopG<DGn7KVq%=zq{o9h3JwM-Q5S5PGaj-|B%nZ)+YWV@!(-ar=Dk4hH7r-+nSe7
zm7a3_^Bkt%>z2+@%F<ot!XeY1C(UnEchP$5rCS`=MFs!s-FmXD^|op8O`nsSCVZa$
zkO4F`{{O$zc7aLn65by9zkxI2$O<-<)B~5;xBO{{?w@7%&zYtE-=F_KKJGu>zi^R^
z3}`-Y`h*|+yT7NJL@sFf|6j81Z&JlCk&FF5KD9df^{qI%<j58qg{*zG$rXQRHF!b1
zbLi4Di$bQO8aH&bmwaW|>&aZ#dHMhUj-}6MtmS2E``^>^KW_iO@*f}Se=Iy{UuWRq
zVbIlSSWuwwzD|NQ02H=gwGYp=?qId}^oeK98WG>K$xX?}Ij(3TQoTN?lH9RF;@=;~
z)%X4#K7Me*v)d995+CG`{GZ?0`0#yvTetZCzdt_8&*PslQ&L1k;@{uj|DT_)_gc4G
zYTbkD`k>{idG}m8Hf#z?yf1z0!}Yn=EyAD#@c+M4zl@q%+kAV;ef7sLeM@XOd(PNE
z;g$>!56_9mFFYQL1-SqDaqz?2+xxSx3vi{RO5V8f&)X}ga`F8S3=@7-e`HF{>FVuu
zbZFRO%Qb^xr|zeh!iR5f?_mB=2r4!}>C0bcf4%XY{pSzx?zWRzE7>Ua{e)kWwqx;y
z>+ApX$^GenbH~13LPb?FB64Lx{M8T46Re(IRA#FX?wvZ-F;Fo6l!G_diCQre7JH9(
zkKJ`<%Nv3W{{P?U{GR{+jrUZu=dMaHF_`$<BZHTFjq_blBhj|z$N#(5F5a_e&TQ=N
zcJ%9e_&)xSW;sifeZ7QTU5Z7$+|uZS2OHN<X+1wl>3i(Pu<!pDp57!;_djh%Y4?Xd
zP`H3*9fO|#|9$+!FVp577Kg%2vJG<n_CJxDlJe)h{NelidHCfNrcK_k$>_vs<3B$R
z9ysc{{I-h0!TI)}^@STY7#J8juD%$)USeNetBy*7fUxtOwOudEts2DjCGPEae<E{2
zRL05a#i<KwS0AwVA4=UArY9h8TkV$cgB#Q&WBC7n&s<wRhX8>Vd#**nd!1GmvrRSL
z-4>*z?=_XF*RAC$N9Dgo0`~>YCZBz$GI^(f`}Mb5uPU{r$*XOZ+{?bjOlX&qkMO?p
zHquXz+h5pb!h5)Z!_{%#V$)R5(1*KL2k3C$R^#YUdNf1DQS+GK`Q2M3_bq+DGJ8`F
zzq%D);*NbB)4S(Pb~z`r?xIQU)Y1rWakOQ&Enl+%PpiEulbGIt1LqVYA}lqgpA_0?
zXuLr{lUaDd1;v0Xr&jIy!1qA@W#qKKIf{=h7{jl=edEm}rm6I8LGIz5{RLB%d^No%
zwPf4=5?NIH{$=$t&i9gRYmQ1xV_kY-mOxAW!onjGyZQt&q;5^yrFL4QE84l9>G!!^
zNpC%`RWwh2p1$_q&gTLj)y@_goVRGdw*SkG4$wlL|NpnlT+7w!Akb=WIEz(K$F8C5
z;2Y`udykkongS1Zom_NK;Sg{2M|O9e@AC{251o^|VQYPO;?%`DCqH|ioI7)cfKvYK
z7_l;M)`HkW0?8dp2R(mrUYlBZ?DLB3Ez2@zsO{=owW}xLTa?XQhWp!Bi+Bkh7h#Qb
z6PU2<$G`YGnd1v{CE1RBe6TH7rE`bPz6W{tZ)fn-{=e~CQXU*|Yvyj`Y%maUk#CAT
zESRGcqq2Et`_xwD9SyEJtg0MhW}1sy7iEYjS3JEs|M&Nu&Gp7IP6@ilWb$?NFP@ux
zx$tM<l3ac}4i4unZXM<3UpQBL_z0Kw8fv%(281$1*>79@_o7LPWToFMAy*a7Ag0?!
zzqk)OTo0VK{8+w>@%cEd*HuB^Cv2Ryubwlx95hT0o=>m)n`BYWr@C^{RkecXi&Z;k
zylm^`P0irnq%ya&^Wpb=HcrmA{q+m4+NeIbZJxoovt0l1-Q7I;aRzfN4i<dTU{+T@
ze13jQb+$_5a(~Hd>ry97OS33yN&H*YcEW?r+4;pMW-dk*EBl%Ri_)&fR#w5JGiP{g
zY<QTMEP~mJHZyPb?aE|jW<LD&^$|Jf7|NR2kRj&%H}Cp>V7PbiBy&Y<Vc`cMB`uAk
zt_6BaUj)3)XBRY9=bv=oG`C{jRMS2l=EMqdndg6>t~8jJdrfMoo=T!$_h+Bm_dedA
zz0gJTrIk(prX?CPmgSa7I6VLSTwjgF>4Hg$WTw}o2fvSeR1>jVviZjDn$U~iuYA22
zzP3}T>40pLYV+qS1&`kIc};wt-@TcMf#LuEDKl;PTmuAJ?WgV#?42fhqlq_SdFl!c
z7sZR&d5%tt1V3JmIjQYYQ8}+**GZG$nwgf9i+PV5C_I(>7W1hv@}HNc39lkwi0|Z8
zi{3|yE#o}B%XnvcZgpww%ygb+lcnokS}p3><f6I6W?7ZC;)SoaeN{Vh%>2dMR5<tf
z%Lubn$Tj5dZ;rbx;XbR)M*Z}@bB(VbS=1l?HCz2D&yk(cB3s^GK4=F{W<4`Eay1)>
zxbo|<9XWV%;o|7O4A1MD@4r%87aPl>*`dc*re&ql(a4vX>zZBrSUT^)#0kI8{jId*
zS$O-7<G$^8{%ZcP>)Tv#w6;HJit2RkprlB7soi#Y|1zdJajfE6yYs$h@UFgHYNrL5
zH2Xhoxw+`J*>hvZ120>i?=>o18`yQcdE3)6<@5hnNUmJAFq6Z1Qj3LJS3&Hej>0Fh
z*6*i2oO=1lSFb<=9`4l{4G)){vzisZzw7<JMvm0Vj)ECgkdU5Z$k*f`;5vWP5iULl
zS7%3^90iq_!+p$^I#-;8rKO!5-}LsHusAYhJ#2fhs`DJH+Rd4jyFVLS&!4<F^G05$
zlBO$9cl8M&SJsERyQXtbTlz}!xN_>rDqn#I79y<6Jg*n-S8lPHb+Kcc>t&0%0!=pR
zo;&W|cpI@+qW`4P$z$;xnk_yDmnrgmw22YFeas|&*OBMD*XKSh$e*;~zVPum=U@D4
zU+dxQ9n8~ww8<dFDs^+B-SSlhGQ0Xtw$wdY>b_uE;PnvE>)NN6{#m|c%GUSBvsdn$
zGU5BRc*FY|LDyyY>fcCKgJY^??q04#1|qHY>Ki>2SA>3GcQ=oX;j(iqzkhyu#=>ar
zA52`YUOZaV5!IEs@X>{UtzVz>D5mWGAehJZeZu*IB^%GY7FGPT_SoxF*5<BZCcSm@
z<ND7&tCVQlq4#xHQERkrpZhG|NiDzUW=;Jo7yG;>ZM!o^&HT4wnvptJzPH{KITN2L
z)#kX=X>ntw(M$t5ZGqtET`}kG-w14(QnH(e&1bpJ^c{MAhu<#ylz3l+Wr|mY+-lLe
zeA{|cA6E9bt%^AOplD&(;}WOqmNFg5OOAYxTtD;dr;SI-W<UJ!xq1JM;y)W)Hv38#
z&HumfPbfdwzf<NKayKOiw8<+9IQ@EcdaXjiiC7MvmIZ5f?!S5K(mRDm0V*Dooi8XY
z<rU+-G{Ggp?QYfKg9o|a|K=&*dLw^-`tGBJSr?yM%lv+^qI&PE(9YAXWwWn#-R^xX
zsJT(6%(`Umq6Y;FZmV{A2z`<{|F}fyeCHygXMZBriXB#*%h$}f?QBX@;uMSh7k{v&
zhl+-}a&$Glp89gi&7Plgo(f%8J>Bs*>Z9$q4;<&us_i>paVDwpv4U#QrT4{l$+LFo
zIc>N(VcDANEr~BH<}KCWIGEA0EOfbszW%O#pD%csbNss?TEGr2CFa<2Hx&pRtM}B+
z>=sxU{osniLlMm@cf{XHTd+KQB<8Es!F%>%(xLl~N2I32>Au|dc*pdI59jTcey45G
z)ueEsQ10hE&v%=1YgSE-5Zf%d`|fF{YO6qzHP?NXzp$EjyuxRa0?S{!M342ypB}2~
zUsYsxecxK~y{nue)9j8LTqyMszP(P}Wx?&gkGb?bO>a(D?Pfi#x~}_qp^eaS7RT=D
zX%}B^xN9Wy=y!zJb?wE1&5TY9GF=3_4n1rTSn|@^-b-k;=wwglC;J%+&7WG%_C37$
zqI?l}q4uHcOU%sEBlgZ}oGfhklA+LK$-5Tg3pX-4nIA2iBJ$;6jhh>roLq)=j9=fw
zlfrC*f}oye+@6F7kCQ8YPI~yieo<S6B7;%)x7Xs$^W~eTiZ=J~G|!P~Ze;w*o*2I`
z#iDA;!+*7H-QtpFc_}|Wv8I1>W&04qJyR($HTB1znT{?kj7IT)|61?j)Y0L2_Kauh
zRGzrF8_$CkbdA@1{v^<%_VvB|;kD5`l9D{(>n1#igtW-l%vR)Uao}-{_iA!-x)AYZ
zhGD`74wmE2#gak>9C?lgEQh-!%WiI%AQ*jh$tvI9zh`grFE!!$VjE?>v3GObx8vUf
zyLK(h+;=YW``lE=meN$2k2cS4_Sxr`l%C`fT-1@Pd|Jc%(i8;+&$<O4<5l`rYAxej
zdZPH&wYupx1%iD)B)<Ee7T<X1vz2AHK${3_)ctiEcO36uee#L>_1becWuLM#fZCVS
zE&lyUw5XhP@DNiL<F$)?XQr_)t9Nmcu(6fYi%qg9;kbKug3O2i_FMG5&NSE?1q6dO
z7)_W48uBP8Pyg|my()rrZR0Gd<~2IaU0ltxr4Ju!<+;7h@Xw!<1^;X~)gFA*6c%pY
zXy{yIG%d4g4Z~U|$GpZvto!TJElRr%L>2o$!s^g;i+}%8Eh;7`noez;EnRy;X6MJo
z<u@7_!ot|p)n6o@eSSW8*Rl4gQ$fuFUS783WHv)Xw&dit0}dH`+qO7Pf7UR0vZGrY
zXn^rm7Mrzon{&H_kFVsnH;EPBc&v<F%?irXSFBH;FyX*G&W7a-Ho81)R}MY?U*Be)
zFVQD+u+oDY)K6ph|6j?V=lJQ<2GwTGlZ6-EY`l5M;c-%KC1^nL&&drnmWTiR<avI6
zvgI~il_Lj^98u6Y^WgE}goFbMDn}A>&M4^lB`EY=V@Uel&d(#RXHfI&$c9Ye5|wA)
za)oaB@bK{H=zw<roIJ@RD9E#H*^h7f2WJZw=&d`(?(*k-eA_a$w#>^O-=x3=)|QzY
zIa?e=T;%n1BqTYw#5VYHh$SQ_vXvz(iYcgN<*gB&F=K~XQ%u3EHPz)3cU~vh9X2@g
zzxImGVx9B8hYFel19=zUxoP%l)~ZFXXDlnu|Izn)(|Z>|FGUtlr&&s>`}`&B!hYvJ
zlRUmK_ht38)@6>_Thd|$4++hDe`o3>g#({8xE@I?eZ&9D|L)~ioa=T~$@C~!o-?qS
z_8DAFE}3<n%Y{Lp-QLo&gV}tiWk<KQ|AU)0Oz#!_6)G8?K2cL$I%U!~huybQ|8FUt
zV|i|Uyz%`r>4g$(g+0%0>z3Y=I5ycLPIc3|E2gQQ^CmB|R?3N1;dD%x^(}u^;gM~E
zt|F}WkDCOBHYKzd)%%IMyUu&Zw0x<4vh(~}|Im%=QY3v38w7~FunKL}QRX;fJj?C+
zLyHKpb>0$-Uaxqi+PqW#M&I6c@8>rropzce;JRP1PZzYq<^TUBvvs)|90XeTCkdA&
z&OW-y{DaHAgUS`UyU!+Vh~$~ZdRXd+t1g@1|621I4?pg0oVhc<;kn@PB`*ut%t<}P
zYTuA}Kw^#R`An%}6BjvnwH%OWYA`rrxkat(MfDV?Q`fh?E%9FI{FhTF`(y2@u62hk
zPYEfm$X>>|^xF3qm2R)UCVkIYx!aVHUtrT$uiKMUQf^KU%x45;!zHkGr{jxv3VKVE
z`ClxoYQ1-yo1^!_tAOQQKf({@pPS%tsG8;8@{3!KS*}@IZ_!uud7r$<Rx!txed#it
zg2^+|uT5RoT+~r-5xIHp@?-O|>Tb=N63~^oC^Mi-AR~I((s}21SY3GkXp-?gW1g>1
z9_vbjQ<>7W<B5qpwY43p+%r5|8X13QNc{WK$hqRiE|+&#0-09KaPeS0q1nv1RqJ5R
z%tgIDj%o`ZJ(BR7t8}>xl5>^HdT&2F%aeY7(U-XwE@|pH#@On`G<>>ne2eVm+!?Pn
z&T4qcI_Zy-qQmBi2?~o>{`@^1v^jHg`bp?o6eY95n*Zl6Ts-*UIsdt5{B=hh3cnp?
zzTni$7+tbxsuIg&(`6RRes1|Jvh49mhw^s@YhpZ|;-!S4mM50J*;&kEoSx(<`Qz^a
zhtms=$md4PZNI;D$0Sxg?Fl<WzK9s+9or%m`Qf^LTj*+uclVZ_08QM1GP+{npFbxz
zR2Uv^<&L=H!s8TO<`CDA<R&^xTB2YHzmJ}rc7ZET*x{9v7(Yn#slR>n>G%KtJkQS?
z=iGF9`6~<RfZW)>&Fnmr)eLKXwaDG9*weDdj5FKmpzyLEfBkFjDQTJ-FIZ+{qse@^
zX3edO5*hmrKdt+-?ScFLdgCu&4Q;C3(!5$h>*qo4(JlP*`*^_zg~NL)BecYoq?Kl5
zvg*$0;8b|zEX>qVD3NA#=+w~?v%1fmRg8LlcTOMR@o(DaV77ek+=q88g~DncLObnB
zmCP@u>+^7n8@+gCWMOG&uwdQ4q$Atdq$-{I9?W2uYnaKoA+7trMUmDVZ=ZaQNPXp3
zLh2mSmzU2_%3$bcN+~?!a`-@lDr>?ro{CAgwsa<5R%81yas3pKk3sHCFaNMPooDOT
z3wMv5ko*0;zkucO%-6^H|1DmpoS1MxD4pj9lhkFINep4p9~j)0wCcM=^}eh)qtdim
zrfGHik{<1FiD&22E539{%ok(>nfd>}QgKLQXQyNQ{KR`UY?+xg$}f0$UgVfeV*LKC
z#)d&x;9?R}_{_!}&3PHeq}Km9esIEHrwb8lo@$4&%}Z)#T+H$Fdp~IZ+^?#(x3@j#
z%{mTU;jt*=#L3g$3)lBMM$BkzWULlSOgJ#fXv4Xafio;5K1`E5amr&W({yV|!>}ul
zwln!ZN^tPy<*}H=%AB>a(d_)Er`*o*@`wNYRLSvHgwB2}(kL`AHa>F1z{JF$p!l(j
z#c``Xzm#7GLMI8wS5D^LAeJ`cDf=cPC1Lp})s8uvI48|G%BIr!=jY)K35?F^>Wx>s
zmOvwE(F+%$6AW+n*Ylh`%M%mBBDH)*P==znn`&>oO!ixcnnefK^mA-0Du`@fz4eGg
z$)*GcEwc&F-zhfDHZ$~@;qY|I!2{3<!bOK{gv7<0=gJ=5lF4IZ)A7`vhbN*-fGtPW
z;95|N!hx%PN9NmI$T@gRMsWrA>fnzY|BhedIsPi`%qiCJM+psAb^iZ1)`>e~P^k2%
z!BPw6ymKnb?Ci~be23>)@=W0V%VB0MQLselQS`DMLhQ`j*7$UrvCq1iqTcs^MXYp;
zE{}z{CCHlwnL>v@J>}t-pY*`=5oCyL(KDVc`|O;HjhlB`PWzg0M4gZAQIeo;Scj+l
z5;eZ0TQU_s$0jQjT=jC<W~uPvvU=OfU`a7OmGn;WiIBvx=+h?_R@SyZHEkz6rql*h
zOyXePw!|UmVWSNDj7>9+3FQ<&2x?o+;pi<h!&4?ZwCU>H9jEx&dfuJ9yBm}TbRrzK
z#%YH^&H8ss)X%SNgF)M?tQQM+pK#@1-nLHXW^L4;PG(=0=^HD&L~L#qW;*(@L<IB8
zxhq}QugIEgUoYV|KP_TIL*jG4N#7Sv;DVZU@{pRD8Cz&*+cLEmb~$T2S!NzNQ{%#9
z(p%9q?+Tkfvzv_YW1BUd%&TY4xXN|y*Hinf`(N)`x7GfZNIR3TqjEA=usRDgj65GT
zFeoarxx2Tuv9SdPzL}j_sLHWPn4S6EqS<?nIJ%umP*5sNc={+m>(9@_1&`Sd&obqS
z-ezD^=d+_|3Cul-j}j6R5)u<9Oi1wXO5)&66%ccqQ`>FYnQ-k6OY)QxM;vt<?%VDV
zV*hxROG^BncICf04^IfTrJk11i(mFIGJOWrU%8JO806&GJU!c<q_9<2vt?$o@$ntH
zE5^3z*L;Bn?OK7$3P**zBfJIIycIfgsxP5gWGzP{_rB`mpfe?w`SQ5;8`;$ReTqwh
zo<^|99b%xNA!uC2*qBXJlub}@PH%A2V|Ers!$9vziWRTK&$#qx7aA-O<2j+pwpp(7
zd*8!1k!{TE65rmZSA1%b)8&D-6c@!mYGBaQdvVV+IG9aVmQ7an$i#aYoUE_xiex=J
z|2p}+Zj>;c!F9ol!_93<5tEZ${kei428Ul>=4t0Q-c!^0^b6lEP^g1W1So$5ZeZ%^
zu~k;G#m2H(T7q^F%F4dz^AI_nc*P<@S}9{;|MDyaW`Ue^f#AP?kDKwFYIt>Mf4!vN
z{InZ;XFt4W+je>R!sB{;pg}Fr;Yu@(Fm9bKxNB>SPmiNlm!ntL!DDR)kFZqTX5;5i
z+QhbB(d1Obj$P6fQt~|(X}2!;G&IET2Q8oY@xA}y-|DtrX~{mB#EO4&9zHzmSR%{|
zuA`64-OJhLAmB1z@6p`l5e*ZiZq>_&C9wZ%T$J>Qb$d-?T9XB9j>$>BGqd&27Eb4I
zZQL3n>gzmDxGhC8)id_7%;hC(bhAB!djgMLx5@6jB(2w1*r#zx>uKSJbve?%&sDy<
z*X3^4mEm~jFl(=(=ej%R{~JrP_+MLEvqN)H`LB8Fk|QPDZ-uSgc=$urqOT72U^kt)
zyOFEKfalo9{T;1gQ2~Nq|K2}&_NcEoyLa@2L%e4X+BO_c<()Zc&HnfM`yOxKBKCdZ
znzcG>Eb1R?lnX0=_vO%N-73)>^r(5M$~Q-jB`Sv6HTAnTG^x5SU9}`SZL3@MEwSvv
zzIBV=OR`N_w2re;;^fR_2Q8n?%oGy4SY32XVp{9-hc=dcd&FlH+>Z?JbiQG>&U?;|
h^3reRufaV=W`?;7RzxxUV7w1X)Sj+>F6*2UngG!74J-fv

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-2-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..e1799a87c8542d7e515b6185d7e8f6f75fe73f3e
GIT binary patch
literal 19132
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3YW8$-45^rtlpw*nn1P*v
zfq~)we<{|*4C?rpM<#3(WKiH~dGLL{Wq#sgmWfTX4&P^7t--ZIMJwyj-w6z7wy;_`
zd*428|M<Hl*vKcB0=XCzIanQ3wGY03SAW>9$>H$!w~n)n&p2D2IVF*GEo<sM*R3nW
zCSH<aJmQm9qQ7>k%MqUM`by^;t(Alh+?!#1H{S9SGsty!E;;hGD)6u#Nb>&AZ|HVY
z;(fjO-4}->d|2eZ%AJ#Zw%Kx-XZu6>L+js{%9y&%OW&^}lq~MF{FTdz>f=6V($db&
z>=2ARXJEW2Ogk%g>4GWKjMs7h?khdI?V&}%ydCeq*?4ej)^0w{4Yn)Pk*mc(V7A!l
z5?v#us*kV!|9`TPqxF@e$a3TA7j}t>iOzbQmaPu|(q%f#Siaobx>h-Bn(CR)tKZvB
zly^7xJ!>GL^w!E%B>QzGM}w5|DyD$crA#58g@rGEo-#*O_2xDq<>g0?AN+mJ@YAko
zbH46-TKQ$ymMQV&A654-g6x<2)Y&|D?#$NKnNz2F22BbM4fP0_67SaLA3ohZaAIW8
zq{hjg^_3sjFX?bkR$e6U$)eXcFCgK~e;%It|NsAg|Nj2}`T76v?*0A#)YSdY&;PIc
z`|Er=fBih0pP&EP9=^Z-|Np<qKR%}~aBeq(M)$4G=I(A+cXyw#un=G06GvLYJv%$w
z+Z$V3C(fNaaq3rj?$h;dZckVmwFMVUms}B4f01kI|NnxjXYcR-U-$pt_s`Gw%iI6^
z`|t1fzrX9BpZj}v`}_Nc+w1q${r&#o;r{ynY*sxQAge&BY*MFlzyxPs-w;Pf7k76D
z@9yA`DZTyup#c+Sf^1~9ZQ3X*eIuCjS`*9KV~xd$FWF|rC<NKp|EvD~?tZuU|9v$-
ze?L0<f3o`jxP5=Ve|`O5Ki=riFVhPP-K;=X{QocI1-8M>Eg&q+!@WJ;x!K>V%iXOl
zIAqGiW@fqL9~Lo%KT=TWzWKvJ>|of%1uA?h2kYhS{_L;+|Gt_1f83rQ<sTpI&%Q2k
zZ(r(<9}^!wKi|mbzK9W=prx2VG2<CBB|Kz`zrTNkx3`P0uS@J)@0d9gr;1Kgn=otI
ze@TgdhDJ%tm~}cCG*uO+OU~$Ip3EE0ANuG0{rD{|%!Y6O|F55K|9^Mg-}|lH|K;pb
zc2rJ&cuuzM{CvrM_3})*LQrp9>1=LqpE`ZIci_bH2OPZI-H%-gity{}oGL1%Izfb?
zQ}Ig1v8zn31^@rIt1GP0<yj-joA$`TKiWf+c_P=tr>Bnh|M#DlVo@>S;X~)P-`^$U
z_9XoHnjH{s4mE!XDAK3Sog3=ze&c|{(c|uSPjrZ0+Le;jq{6y}mAPtW<JyXb<;&|&
zJd%(V(^2Irn%ksYc)<0ak-?w#m%qBF*!}ze^YDlF@`q=e^JHH)dUN~uhgYJ9A0OxW
z|IhHxKidn6eY{YcN;;co&YalXeDsjhy+cliuZDX0^|j8PJ+=9<y|6H|P2Pm6<q=*S
zZf;ZBn1UFag?4{PX^AOZFjew{+42AX{~K_NZc*dN+SznZpmFBs`HhVa-^aJ{%S-I5
zOSPz)l6cdo?fH30Is3GP1c{QvpP^R9fYQL>3jycO`JFxH785ngJ7PxfWZ^g!g9UDi
z*H*FU{xf(ja%K}_Fk?|?)6A*~r`zf+rYu}CyZ-+LEh#oOwi9c3LH;^F-}uhHvm4TQ
z58vI*<3FEooe~Qq9q4pAhfE3f_P%z^?c|*pr^rYLr>42RZ|!gW5OvFKS}omVdhKF2
zOUzV`O-WB2gVu0ztPF7oZd)4AIFW~EgA<d}zWVb8uf-03ea%z-%^+s~xevcg5BJM6
zONtjjJ@Dhm!Q;o5t<uWOU!RnvwRX9>mg>=Sr%o~St6W*9dG1K#&0Q)Xeh;+F0#5O;
z<@oYmnbJ7NMImbfx9E<{`iWhVY;0GKFe={Po4li__2D7bw&ddyda+4AJ|sTaZ49;H
z!I6clw6gNn<z%nUNK{nUKY#4dA%^bE3}?p8Z#`Gc=9;hFxcayb&%eWyM819b^51?>
zeAm`94l7+Alqqg?3%s>TVzI*ihq>HaR6hN;Km7YU&vHLQ8QW6@pHvS2{mmn9&lhsW
zz`(!&YT1S(ikc@EFHm^;e0gI2`jv|hpFPqN!(gCr)MVW$ChsGUSIJ(OwZdaleW8i>
ze?`^4{Sn$~SM+$Qwl)QRQ23sj!0k3+5=WF)<>$7<*I{j&(<S?45<z+M^mN8fB{rxX
z4Mz^1II&=%;>$N{UcFh9p00lKAR~Vjn@uLu)&mD7c^R~An4m5DW^Sm=-bfMC6{0LQ
zYaNb<Z{Y3z>pbh(-|m>l3;OyE6Q;>$cs_X<+*bKnVqILSMg6>ohuMF*i!g(ueNpzK
zgrp=9@#ld-lX&>ezkRwiX-Z1lQqD6hALXN0Hmpwk$e;X5G~MXXGKPK*FVmJVg9N|R
ziPG8bFHb3TE>K}j(UVHPZ`Z~vEx{*~@Z(eK!`I<eZsJh0;~yoYrw0ZKmX==q^3~MP
zOe`_&U%$yBR)2wvtL##pF;gG(RP11iX->G|#q>s#|Mmg#Wjbo&dz}&+`sORCy^@gp
zpS;aea>G>l`u|2Xe@__{s~zqX=9#X4c`FB`@LJ^lC?PHF$>YaYE=}s|+VtVmB{89-
zw8X@e2EN!o?p<I1drWLmP}F7X`Ny%xveQ&`p{Q0yVOuEIAvZbAgH{`x!VD7pwPysE
zoLHP<V4k39c7a9l&Hj4Hy8mewWj%>sv)Yv1CGPECe))|oB(N$UB_t<LoAzw`)~h|e
zTg?mv9zRG(O8V75M<>9Wui`-Bx%G#VMXI?Rx}@2j@bq3${n;;;Yr!Sj!nvZ8Ig+tR
zv+16U<K$oe|I07=-E5e_=F7U{m3Ydh|MhM2?IrK+PmkEpkoZ2Xjh$b@u6EJS4lRhK
zxsM(^Nx6Du3I~q~KbO&zNsp3}lUvleC77+gbehcWTKeQbVDo|{5?6jktTE--Wf2uF
zw)tY$t>XvpeLAql{Q{TJ7Dc0`vq3(3dpLf6Zhv^&yiL|x!Yn6YM@i?yui0#~^wdDD
zsQ>>LB|duaFwxv}s=rT3SwVo9Ktgh2V#`v_j`jjmqYLSm3QaivJq-BCzw>P4#0?jG
z4o@!GaQ?w#KP!z)*0o)Za*D3Lwk_FWY**S&?62p!zu!1!_puE{s)zsm<?)|ybRq8~
z#5Y@JgBl_{uJ`%#-W*G{ZJK@W;L|(DCRb|sPl_yAwej&aAuk2{hk;D3(|Gi+&Rw=;
z-RW($UMC-0%qZN}nsT={n%(ojt&7*Eg&IeN$5sUdTm4xUxT?NQ#a7j+qK&JQOO%60
z&Lb(r@pHo$_v)F|{LlKXyytzt?Tklf(21FqY4<Jrw%4WXgM@3{$%ALkXs8@{{W3Ek
z;>EfJ2hN{6$I@NxVR?`tKk>k4$!*VAmE4uCikv8AoY%#6?(gXX2O3HxL{=&@YaY8G
z-gU_6dwq!J998zL#q8()aYsLCTv2Xt>}*Hwv_m%qSZ3b3yJu$NL#MXg<(WpZko>S_
z_D0?Y1p(Ll*N+_YlaglNSjZWBaP6Gau5WC)TNIL>hdyztvncxSSZqIeaptPTg3mVV
zUSIj18>{_o-*-a+y(ukXbv$d0tQ{A&SNsa?JhV1_nzYrPV7o;Xl3Fhxd2%?Uc}O0s
zc-yndLu)R(T}0BfuqtNd?xaZuyRMh&aG%?l`^NhC$Is$Vm_QA(MQM*7JV*!(7Ut*Q
z-rK!dK<H6QQd1T8GutOoEu5O228IR(2X1t{a8%KrA|!1YHo-fs;)$aY=h|b9s~x@B
z|8G3$)UJ>al)=e;s=-xy(WBr0<zLmQun99CwqN3MnB%j8($_!#`#0A)vPEP|s;RN%
z<*}KYv&qS|?JSl&H$T1N_q16rA%U_e3Y5GrUY^|Bv4NZ4Tu9(S%M`8^^BTjoKO`|r
zt>NM6;o<S&;i;eKB(U>?g9~R+%fY1^R=95yk3M(!%&8NPF1-2o|9`!YX2^F1Yvy?x
ziBmP!Bs44)-BG<DlU1$bozRvkjc1PfDEavCym`a3Z5xRE{M>ks&B+h1M6X@~brk;p
zU*rcWsIFa|+S^^s!)5g3d3t(EO3PB7430nl|I3TDet7YxJ!biV&n7_+8-;jJs2aEk
zggr}E<96%W;ga^p{c4lY!4+<fNfQ}^d4p0}%vpBwG;CHZWuIon?aQ6k*f^8#LdwLY
z5fKtPIua3)k`WOSeR4~qenHCkMNXh1?8=oXeSO<_c}yOF>~7iOFjGPx^l*JfT1tz>
zOitaDiAf@=ZvwazQyh1cZDC?t#HO2IU~KCoWBM#0Wumh5SzaEGr!12!=J0+LD!Sx;
z&3nNL1&!9NuM(4z49wcN-rYa{;hn5#+X=Xp0YQ^Gx;F8083_qINJvRZN=RsO(aKuP
zcxgwn(<@&CLnEc_VuFe7Yd@bn(6F@WKuG^Jivz2;^VHVmsYgj3YY62Hn%MA6ZpFzD
zZQQfGB*f!ZI4te(RGPt)a^T33grq~DHq`m~JiosMCQjpn<TjJS6Q{V`T_f9DBsjT?
z4`vCwMVw^RWt&jFp{M8ngjWeu4l!))Y>!b@R##x_F=-6jm5^4R(qQ{u>EyrTZb=he
zb>9lC4b0gkVE604{4CW4S9P)s8P<ClWD6+$HuCY|NlW9Y|7TG1|6IWjgQdaHI(m^t
zVMo`ew$_>L?Xx?&KPPxgo(N^LIro5-jXCIO<C$Y`nh!8b*}dpvDwwKqVkw`G%;vw1
zq7Sr$3Nl@?BmTH+iz;LV%w7>;d!j+7dwJ@q2J4FkhK4g{8`u0l_u;kp;f={6YfT}o
zCnfvB51)%!*rnawW}Q6AX<&Hpvz1D!f#UPJh@L~*B2T2+8Fd>9MAv52`t_`0iTox%
zNu;Rc-|>ZVavuwX{7ZY)?<z=0m@0?G`uzX@|FDO11jD*z8tc9sJasA|`S6Fw?1#_K
z=UMJI@%|4;3@RBHZrE&GU6JAL7PV?+r<0$zQ&#$c(>zbiG;8cy1-JfiH3%>kwh2}2
zHc$!;IuUfjmv_xy{~fVMnAdc&S;{7eDQ4(778(RaHf>&^5Hw-NmxCuyCZr$#@QnBH
z@qQljya|u;Rp2(gc<I^NdNU*2d(#F7|7%Wuk{h;J3GeZ1G<B7nAhj?<vw=}lBd}`r
zhdf3f)oasO!U}B|bniY`pW0!_r7ix>*X6bJ%B_v6I~TZ`>9HBUY@R>=;N7_o4>GrH
z&zFqbuXa2cVo{QkGrzfcwNFoTYpaa7xbd162a^j4n#>bj5;$2cbrvwJj68neL`P7_
z0fr@#zvWAPWO);=iJUm~FL^?RNHUYnQ+A`yXQn1*lL|Zbuzb}oTH?fVamfVMiz$Y@
zj$%`$q;zyA8yF=eq#b)}e+Octl4W5*(WWEEqksL%;^x?}u(F3SY^Q?Zj08{iz!eU+
z6?;B>X_=!VVAo<CxLDF(NKt4(=8>o<f%!=nCb2FPa|<!6@;e~)G2$}gGQBmoE{bSt
z8BLjH^x}oVhE0#SMb^OWxN%EMP0gyKtH{9Uk<<i-)$sy%qbd$Igg7Zo7vs=B$PguZ
zxIyq&#gY@uCocAMUI<BgmFf`MxmCeYqK~UUjI*MPm+gwc;VVbaoLR8)<b)X~6x7{|
zBA}-07It)hR#mB3vGU`AF3pVlLR%ADRu|5E@a`Ck$Jvhm5>MIxi5;kS>iNH-tMDpQ
zMxmR)gH@3QD^m_k)48yt<H*4Vb5nx^Z}SLNMUG89ZBwTzTFf|jXHMhn*&P2BK`ktX
z|NoT~3pu!ojvR@otU9H*_N9}B)XlXUwyAG<f9k}EhFeE}@E6<eNpN_5=Mm$7Lm|c}
znS`^CmWXMkh&W|zd#rDBAxo)3=wNu7LBbOLrdHl7Ms93nWo@&h*pidk{QQLWKy!~`
z;fz^YYAO*2zL^^@;5AcNFP!GEgy)BoT*}k{Hk%9suAculx_Gx7V`qM+;UazMKory8
z<EM{qk(#b>?GSVJDTirCLqiii*?A-c`<L~cIOXBeqG&V6(T8WfdmzXg|NkqQfr_7B
zzooS`Hy(T>r`O!U=^@&z@<YD3QS4HH6?2d|OLmq6b4FIjO5Y1xf;l!EGg-zIlgjq(
zO2b`+J1?32gPDC9XNfQ~H_w)Co-51D&u{3LkdTl7jR&p5g2Ig!*4dR+T1E<!Sob72
zoL>BYV$fmxC#qFFJ^ur=W!WN<4z1Drw^%8qVdV_Q*-}1^YqSbAo0*$>17da=WDCDp
z%Nz5uErIvYfd;*fQ1MNTH8nhmi9D-T@#q}k&4+qItMJ6>?y9OR4(`oL?&?9_Oo#rp
z&q<PyoEUs8J|*&s6$jrxWl@6+2IGYfG|fC#>7UT^D=_uuyr$2yWSTsWMBupw-P#39
zg?uzUeuwz>9=tj8#xzZE>8v!z;>XXMu`ySAdR=GElr&f*78iWL{rJUH!)t$&mhsrA
zvdru0TYcjImxq^O1f!>K$QsU8p5g*CSIG^k+-yBY%)WCP8yy`Q4jyeicw<ImZ|{b;
z(2$v9v1Cb$tn5@5mm3pii+H$lFbBCTntPNv-mY(3nasq1f-A}u0?b*P99CbEn6Tu6
zi^0w(jB8Sv=P9*Jv{?O7Rmx_qADav(Q~2eU&Q3?y)`O?I8k?IZbW2D;&7WYgV`t>t
zxuT_|t`QLu8$=~;TxQvJrC}#)BV%+bqw9yxBlh+GYJYwC{P?(h{l7oXvtm`2xi;x|
za69-&HpH#{^ZPicbUohBbASKhl`3yM{-m-oyBRWzr#Scq)zt8qnei-J)^hv`Gy*Od
z35q@2ye&*bG_|>sVIJRvofE8D1v3|lGe5gx^G=!JV)C#1`~Tb5|Jzsa@aDh2<?(xd
zoH=z$Md{GWB}dk;(o)brudQ*S&FX5?w%e_Y*;&ea?UGtF9nu_b^sK0w$of#F>)8MQ
z|Ns9u_~F^v{q6k!@9h5m{N&{Q@%#TJr~O;M=+J}Lr&Saj-e~a8GCCtFCtw<}{OJVW
zfbjGS*V8S^c#L}&fdYl$|NleNEg~Xk-nk>v+j}%Y{fzOVzBq$JNv)h=(;U|P=Z`<n
zc;0Y@tnisxtn&{2`~U0f^NWkijnn7Z6g;@MIeq^9JwLy_z8>%9#^&hA$IZR+LjiAn
zeQI1>+7c^1@dH;o4n{dAhB^xvJPK=@Wc2S(;*P4X=VzJTH_!c5`1x7<iC6J)acq{B
z$3j<22n)}&v60xhBOpxp(UwU&N>7}4G$WH^)53>GINOqsf4IpCPJ4%@TU1n9`uI*g
zawH-!yRqKD{(xkLV9%Xm=X1g{I+=U_{(pYn{&>Iri&s_^kB&U%m*1!S{!7*OcgMHo
ze)pSKQ}*G(;(d13a&lr7%rgUO9y75lmN&M2Xe6`wgF{FC|9|zDm&xC~JOBJ#=bKx9
z{4_p%b@l)Jdw=R*U;FPpT_QMGQdPBcmR;(R84KLCB`zGf=*8C|<?G8<TH1Db`N!O)
zkn-u!^=AgAQ@3oIl#+H#s9R#T!2gA%Vc<crf30slUGArKoYiQvum3mC`umb)Y1jAF
zmfzq1|KERmG4;LG-;+N-+dXq;^?dvPzkZv4>e6R(66{<vNy0Iu;ij@{A4|>3&wqda
z_nuz&{>`77mqOn>etd3!`0efg%gg@Gw)uJg^71D+ng?#BJ5F;7W1jx>Ny>+hi4&$D
zPX~>2{{Meyrp1jLGuEuVnwl~rFgzq-8PA!u?>fvhgHDTV_V1pTBxh3*9=`qcmzV1H
z^^fkD|NC!$`1W>tKG~{ES4ti|J}$rF`pgHK_7WR3orC$;vKojyHs|F}`1ASs|7&aS
zi|hO-dV4GXPqyX#J&7~y|Np<U^Z)(5%I7jb*-@P}*qWKSd8T9|qai5Kft~W?>Cd@y
zL=>*iOH|?hQp4{rzrp0=bjHOQj!%`icseZa@Bcr``umkm;r_b6N(GA!?63bHzwb}k
zhX<FJdgt$}`zveuuRv6BN85FY2_=b}Cc1Knc}L8eQvLnkon2pl|NOka)AH}Xv(jy$
zt0lJOtLL7!Sr^E=W>ZE^Ss9zXJ)5KBoHj^@;?PWsoSZABrjZl8Ikq%%dDydVzR@sU
z>_!)FjO&U2-qYvlMwdN2ExzcTz>m+*|1Wl*FRr_*@c+N#`tfy%N5qO|IWYeZl>Be_
zXdYwvm&Z&elqESnot$jX&R27P^Yh=|-@8bD{cnHx?QI_IFaw(!k1a<f{zj}(WnboH
z@$%(K1>N4uHPG;wXVKDPk&`QW^TviWyDdx%dv6?&d39i#PJq_3{q_I#;%eUg{Oo_E
zjf?qkzkI#C{l85)H{UMz-~Z_B<r$W8^;er$i#U2S&bYR4x=i)A8vDXW&u5#fm?-`G
zmugWxE%CD2r6{?Kmko^3iw>@uG;w0%<jDum^fY#J?>-5)e#=(Xw{L%5blYXX&D?XP
zVJ6=OF$n>oz^^vd-wMyqyY0d2v*<uO|Nd+1_t*dacKrMMdLgaPk{_;dADN<5QDncX
z$-Z7f&hAg`rzicJ(>?EneEDU1cwH<{{(Yg<a@RLVq;<1|#|jG`R8VTV3mxG*G|%G8
z*QYM76NOo}++^JSlWnbNQ{2=CD*`T9>31-_ys?q}{QP<~Gn<45-`|6Vr>#m}+-zo_
zpv1oO)B&B(7Z?oB3Z7ZT6)fg$dE~^2f*&9Ht;_4`|M}G1WnumQ&+yKUg9XocIHNXf
zGygb6EW=mW*LRLKbR6i=B#R?QcdlJ?#Nd$ly$1)PoD`JgG*kbyF05Z68}+5>$(NVv
z_xDdOF1olrzJ7a7<(CHs({J=fu?Vs(II7L8XnG{v6IAY=KXRnt(UJb_>nCiwU%riQ
z3tsMVxvA7aHaxR&*~FPM4_=$rI9<GZBQ$^~SxlMY^7YFNNdf1_{{x=8{7XKjH_0fb
zlVS5pM%Vg(e|DF?{(67EO1cpnTU-48e|L9%{rK!GzyG|NgMzyY%noL;t~T;%xXRV?
z>;3)zvHSmhKG<xpA2*?L!tL+xdBWFS_#k1y*0Uuk`S5}B{jQRbl5&zoNT@3d%Zz~j
z_bu1;PxSJf`NX{T%z>y~3)H7<&)>hN^7GsK_C-SM%*>!Uk@t6g);#CilcqR>uV5<6
z4llu^E@74*A0Pj3<+iu4QTXq6bAP?0oUPKrn~aPVv!r(H(k)<p^ytQK9f${={ACs`
zn&ali^ybZl!on?wZxw9!{J*&9;syB}Q2~?MnxCH@+h4mFu;}33-ShMA|7+Q<bRbGO
z!Ix!5)&{S{qJ^*O|NW`|`DuOmdyP{+K!M+5x^5dQb6*zgxu;K44Ga%nN{3XcPV#cX
zLXQGNpC%+GCM5h|e?8C8aaEApvHA_no)QVmO&l+YDKnP{vF@8FQ+bWIa?vD4<BtD-
zex5$&cCAfEqT<Jg({FFf2P*%T@9Q`|RiQ&CXT!F*K$CyJzP>-!8=rqaC`$XmeEp-$
z4<_eE?BwytWU_NTe)#Z$wPz0;Jow?QbXOqMxB8YYE)^{;Hx#zdPtaD}ASS)zmEa-)
z;mh@Wd|g7vkN5ANXZQEp-{10g1kX2^%O<Q1o^aYbr6Ka}f`a=0|NcKa8?VH^uYgfa
z&1k*I(Wr$9=XNb*3=3mROw{4-hx*e~CMU;7OY063)0!KfxUQ*ulHX=0ve~!m+M$N;
zF%KT{g&sJ3_|*CHetsSY4qRVfFK7F!%kUN>Bg<u$c~$Jb)<sS~+npc(e}Dh~-m0(f
zkMnm4v8$^yTfcN{GJPajP#!8VVN&-BjpimMX5p+zsFe>te%P>bWg$0*!6I3;{#&OV
z_DQfV-I9>v@_I+o+av#6jVH{XAL`iH|73y#6DYoZeYri`+&<tj=UMI<OF7b57IP+r
zHcb08VQoT#N*}-czn~k}1q2V?t5x9m(37=*bJ5mlRaMC=S1$&ufD@=wX7Ps2o4Yy+
z3JMh3`Gnt0+5UgS6c$dYifX1h9jQMO7A8j;aszpk`ilw;U%q%Cu}GQEz}UF%|G(#F
zXUp^O)VPGG{x?vXG2=?ZX$O<{fBt_2&88?D8@oRLdGh4MnHCBRJ3|{aTep5afBw9J
zN=sZ3H151*bpHH#<KuJUUej{U9~(Dm_y$Jsb7h}u*v0<d=0`mn8(Vbroyqn`7#SJO
zu4J_)Fn-`KuSs^Z`RCG>^sit3|Bs)N_B9EA{?7h0%hkWXWvx34W6{ILnJJ1VTU%Qj
zXG_=ql7skfb=Zt0ODrDL9kZAubMeiAYugT-a?M_sl)wGd$wF<dN%5Qi{QBx2GUe~{
z^ZRdHVvN!{7@V_!^Ph8QzxqMN*@r*;|6lLv*>&r>jSUYU-;sCLiv$$TU(nK6@E(>G
zRQS@;+(JX;3=P#P794ngw6R^}*wQ;Q&UXKoaDFx;IX>gf{xfHIK7NdP^X5tW$`%%O
zdwXp+t^@JD3!c_(xx&Eca?OIn`Nt0l6O)L#ze@XW7?`C6gsVrJGcyb723R<)OA-;4
zOi2xdj{Q4nTC%WOva()$^j@Di>Ek(jw`2c%dOnDJ;NSVZ@#4gv-3Q&Y?E|8g9XxpI
z*RLnrZ6YEir%v7Z@6S(f7ZIiDtt?5i0zNjY-u3V@I(fn%_|gLpFQs^W=6817jtaWH
z`({F#cTOsO|Nl$w*lBs=hJf+Q71v)L;^z1G)v%mx&0N7Vr&ui=7d*XE&%^V7%NCZ)
zm$ybnp0sT9^68n|-|rebS32{gVbqmHiaw4pp`2+_Ne2|UO~c~mc}C6hjG86WZ(v}s
zVue9V`{d=KvJS^~82na?U{qFS78kE-7J<aqhkt*6pFe-TbI)#;=-JFqccm_IZOrIo
z2>*3pRt#v?FFEO#xY)8MPgULBG(5bEEv&oi{{4}@-CB2IBL}a^qHeJ>Q$F$^_9;p2
z>G>ZTG9xCY<3?@=V@s>#s#PZ>A0FVk5X79O&N{E^!K;gY@1cp&Q)|+sH#dqzKkY3m
zI2Lcwxp=|>AF&UoUhtdg{;#N*k$z@_sGwrfd^sTjk(8-ZpPHD9KX{gQsKX}4lxbbj
zQ~NfBgSVc@>)cd)V`PwEW|A^-y87A=32A8^{mcC}Hb#0Ryxo`NxAnw{1szcPDjMd_
z_4Mju-7N6$zx-NT#{H8d?>+d@E%JIt+nx>6<o{m`@$%T9AN@>0-Tlx<mosO2e*Qeg
z#H6!9S0p2_ySsO5!hxwA8?>CS_1y{lb^P$*2QN=PdhC3NGuOw5M^KQbwbdg0%|w<0
zRo0kLo;j<Rv|RFrOcyw*@OgUj85*)}DQJl;;@tW9ufFnQ`yacKdd%3<HXXSC??BuW
z3ro*4^YgP46kH5AX3UUqak+8oR7hyZjvqf}7#4lDzoMbCEa5=#oIi&Xb-uYQUUuxs
z+q178A9hOVH__=?xLf*uR6G~s#I+%kl2Z;sr+A%I_~hiu;_lf?EM(y2zEZ?w99KA1
zOyCRq*4B^mi)T1~iJ39;zqM7@+Q?*aF}42V-aS1$j~>Y^U8=Qk;iOHQ-t0)`Gv1l-
zUE!S*lW}6ARJZG?v%Qwe&1X+{v;Hp-Gd6b4)I7YWl4sJSC*>v^+W6<WX<VBa%*MtR
z7&wKG8<M6}a@5q)%F3ohRxEifHB(|j-ts2t4wDKldA0N-GZwp-Dco{1G%-`@kd%=8
zudQ9YJ|a<8e*J_gC7+&p>y+P?l;*y2b>+@oT?GXONiO{JR;aptlV@gTez%Inzsu26
z{>+*GckW1tibg&=pB}OE)A`58BOW<>b|zj_V$;_4`5(ygfLS8qwRlW-`oI6_51zP8
zgyk~6w6taO?f?J#&%b~3d-<ZQ2AwNE{?F%pTg4#3=^wO%?VRCcjyt0N13fjGCw|;N
zefp8T)%V!>_T1U~`{>%}_<c1$4<9?WV#SHneEpnsbsgoyebw7V|FyB!ef{vIaZbR7
zu7s5V6MU0X|E^zp^v2brGJ*#;q@BGlYyHo@?#zJ`ClpkUB;=hFkUaU|fy1qjbt;TO
z!EDOPhu6nzw1TG0|NnQ=vy|9h|Nk^YZlb}#X>S)y<@)fYvF)F-DyxmK_6g&H4=POl
zi6>q%pL+Hv>B}df@1KNze|~=d{r!4L$^V--8X6fIKIoJCwBv{1L{0u|;pg(okL{Bm
z|37xD=j_@4imNks?8vaQ@(ViE(ER>>pb%tKf6d&DTn7|*T;@->`Tu`V;U?LZf{Rz2
zGTAs8UEj?+;(1uEdVW_m`^EkhU7r|Lm7G#qb8K4Xd$;r`a;9Cc`5#iEZu~f*K1*v8
z(<Gg_O%|^^IoH4WJ|od>ZNAqZ!TTo|qPPn{1Bw6tJE`>5|NnPTn4S4>JsVq^o7<Dy
z4l@EybE@5N?U=#K$0jkIu~de2?ZL*dodyf0N=mSD7Hu&U=&JKE{XfA+IpXC5q2uvs
z3A#2Dg6}mj#^?r^tbBhza0fJZtMow@r1UpFy#N3Igw+QBl&&iHcyO{l4O2S7yY7Uz
z-a(HiJKVJ`H|V|p(SIcEfSBP7&-NL!nod}xDV%n&d3yYTm*kGNqZfqEZqL^!hE6&-
zsr1da|6lv_)A#@NCm3xrba|dkYG!=7OONfw)PpCk-!y!%B6vfX-igdd_I=0LV_3rN
znZoxNB^(i3rpM;Q)!`|><JFW#e@UJl;y)6&>i%{Wy%mRMj=uQ)|LT6TeODI_I>?Zj
z_{aU#?n5_wcrIL?_uzQ_LoKI(7;fVh#*-#bcBCAb>r${(=+mNU3q%a3@?E>cWZB5a
zaA2y!1BTUazQ5O)y%HJ(dY1ocezWBl8yFZktTXz<pKN?+5{FIgpX1jQQXNEH3JtWR
zEqxbEmFMwLiE45<J#oh^d6lIMMpu&6nAR*k(ZFaF#Kz2gczeEvDkNPbIZpcV`JZjd
zl1VTB|F8G(d?;_T@Be@IEoKw61fLWduQBD8au9A=&g^!@;HZv)N0*I_W?SP9GtIi*
zAKd*1xi|QocRKL<yGFDU)O{*_@9+QLSO1^=nc<c@{dYngr)UeXtvQx>z(X=ZGJ|35
zhXYY92}*21x8y616sn3(3X;3_>}>+e|Np`ofp)b{nxKVc4FCT-y<7XI`a7R-p#m5G
zl7svnKL1^{{1o2uu{qgBm<XCof8;62vqJE&LlRfQfn^M9O;0uGu1)Z5IwAG{(2WyS
z>_QK+s)b~p|M)-u;bZqH&dZ^WlVqv>|Ia)}k^hsyi~18Q*mP3=xXPNP9=P_&;oi2!
z`bg$IR}MxMCHRV+C_HwcLG@avr%45u+$Qe+gG>sZjFN(!7tCHj3w%$R`F4MQ{rRb{
zuP+>wFLo=vu{84G$Htg6hd!tF_^!^d%?m7f!<(5`b6or4$fGpl@kbFGma7V<{~a#)
zpTIc7lSR9!x&FVzzB*S!X!F!lX1@LZy6^A!_4Qx>|NsBr|MZsk!A-GmPjz%jOz;x_
zp}Uf2iHXI6eGal>zxxlW+0{Nc|CGnY#q!#gqYbOLckbs^XXjSfx}~ANt@hWE56q?i
zq#>!$Qzm}jpSthw`uErv?@4A_>GjX~G)qM_Q}%b!nWr00Da^2t2xykD^fg%UER8K^
zV#}F?1F<;*KD;x!Bn`eSS}7CLuxGQ5!Y;cy7k?ROIC{$5-~Yez@2~Uw>(`t)qfr}v
z=5$kFesJ1>i=K|sHx9fsb7Gdu_}HT3?V~?sfzg?VAxRes7(N(1yKmnXzW&1YZ;-JG
zCr_FEb$@IB{P^B4pL6F<$3E>fiH!2M9p0TgAlbvC5z0Kzk*~(@s9RF%0#}=57KTfi
zZf008Y+3dEIDcF9_Y3EbLreSBVgK&!{QUd#^ZGk?{wJ>9AF%A%!@XfWQY@GB5)%><
z9C9lYa_fu|CMP-gGR{!i@u>6154B~2R*xpJx;*)u-WI)G!fvtLOAbhAtq%Klf6vd?
zXJ^+xdURnUZ_eu~34<H|KZa?{h++Bh<Kz2xcjxciIa5Za!ot$<_m&x*&5Zd^5_3z9
z0;W|x=v%10qp<nmDN#0YaW+QAR}vlj7yf!D3mPA>4?OP+b#DBGFQ1;yU+(`uE7LGR
zg`16y&Er4o+GM8G1J7C?s@orAR9<)B@BjG^pPr7NI`#aMClODd&fL1?L3Z@TU+o#2
z3zvvVv&@#|*&%%SKtuX@iBqQ~=h-Aqn4WH6s95&UaAItjPb?1)kAA$7O>LJR)JlJu
z_xJzTetKe_*2>hMd?oeRgk5cojg5^H1$Tb!V%rd|*VDsufB*m5e}A^OvCW<!@acl&
zwWMao@;8&B<Pzo{mX0yyFgFKzV}JeRZ7Q<<n-0tmh})Y~@n0@h01|BeGWK<Us(*hw
z|Np<vh7)co6aW90l$207?fqX_oo~i#!`*8#N_oZH*T($(_dTDjwA3sxP;H426Vrps
z;WKv5VN`x6S)kr}Fs?a)$<UBZP_Qk2zr?q9s%5!W^Byr~D>GXz1dUBUWUjKG*4PNm
zFY>qleSQ7^_V)UJe;Uu+{h!<*ySrhFppc@g$)P%(<7=bs!`A*fa_pFd`0>+cdR9J_
zzZ51O^XNnF*BM&>6IcA{j=q^NQ?ijkOpJ}6zwQ2h$!)pH4~1Ai9r*vB=l;ILm&c7@
zYnz<>W$OR^ss8xr`|0Ts!Uy9+4*vJp_*Fhq)sXYW@&5gj)%RC_dg31=vwoePc2-H!
z-S=O_XDckTSQpZ2dh~#;(Tj$?w#~DpK?}@gn+xx2Y`qc~n9|F`!*hO~;ho*>OP@gV
zl)udWdgD90zCORG{3YS<|Nr_qetdOd?6a6WUj6J=PF;|4=l}ozdNDr|uddQ>YVz^*
zW!g~t>&If@nIGjZ`z+a8a%4&8!Asi?uv%D3PMRdKuRc9uOUJ`kp-GSDEm@`^b7$}A
zg6DkGR3NGRkWuQdhllrDm;W<RnESZ7@(91+(ZlsuG#oGPD11Eq-#`BQ`~J>0_m7B}
z(b&P#7#Wh_%UEx4CBe6>*(%b3L0P$Zs;F~-z~Pz3JagyrxWv706@B1kSn%nd?VIJ9
zki30p)Anb_`~S<^|7*?hYb=ijm1c9fjyD)SV9cNI^etWK{8^&~u7~H_|DUe+_tu3E
zAD^Dy-`mSHZ{CF5bv!G&F3xJt*gLh};G2DjqYbY=TS#;pLqbZ*gb4{AJ}DN(ZHY&@
z+UoyHEb~be5K?;e%Oi<lj|*r$&)@1Vf_xx9g2w%WivIll{r|T43A5J!JUssupR%V+
zx6PW^py8dsUh3ew!PNJkp~0W(e}BqDul`=N=sUmszq2PEXzD&acCq9@ytd+WZI;^$
z3nngPpD|N%&05K<Y)KXtiLmu)9v%rhDuqwya;2=l*YKkDbKAqG-f!5|pdODavZ?v;
z{qFAiIdd-5X(=oJIZ^C(<r0%o<oO1Z<>wC^Xt-mVo|2I8<NN#n)Aj#v&;4zfdg}V;
z=k@i!zk$|1URrwp|G&S^R_Po!{b$66vF%yP6vb)S(Wb4zvStm>^K*{hrE+f{D|pJ~
zTx;v>!qS**<018LSFQBv8ycYDd62&+S%4-Y3LoE}ZT^4BQppF0B|Nw*A27%Fcs!Q*
zANYFue;yvt8cwHkRr|``o<BF&zWmLP!sqAWZ*)jDuhP1Fe*XUn+XNS@@}1e(UT+|j
z9d&R`SR3<?pOYURW_Q_E_o-#!sxDCYA8=sP)@BP0UGy#Sael#_eP=(sk^E)?a>W1t
zhxX0?{@A^umrwHdFLr6+<c1{-H%?ryujool`E&jRhvLH7YWMd3-d+Fy{oC91d3S$3
zejm@Kt1A{8JMHrFdi%OR@Bi1gou2-G!-hZ3UOpMg0U3oo*Y-Q)l_kiYmQs6g&$f+Q
ze1YNgNrx14&u*x*J-pDFr(fPU=gtWQ_2UZ`98gd=aNy`s1-)|#Szd2;9{(`q<8%IL
zsWOOx*}8v!zW+2{vh`p~h=IbhNC`<vg_jI*2{nIz{lBzyzq<dw%J1*)-`@WJ-TnXN
z&(GaA&;M8Z^HY4-G)<E!ckj>tdvEXl*Vq20BqSuHr3nZn{rT~6eFw*<i8udmROO2)
z<tmza(BRJ`R+c}%juyO?nv$Dv;K8d?3l<+Pc)|f%vA5jM@Xf8G3Tnp=96x^G<Vgj+
za{&=29=!31n*I2xcU$`Tf_oPrMZ%$Z7IFLk)xW$HFD?Bf*j(a4D%+bZ(GHO>5Be@I
z|L-&B$KPjXL4o!E*Vq5Y`~S1DN}fF%$;5m!Cido&r!#NdkeH_T_uSdDAHHfIzQ3QR
zrl#l38y=6qZqG%kef29IWo$iT!1DjUam;?c*z1})XAT@YwqO|uCFGyqP_BRYU^9<-
zp242l)1WEA=jVCm+Z*56aqz(#o~s@k&N2$te`t94DfLTs*aL`T@60jeYH{Fko&Va5
zi`gvrcBEzn_j&{NFD^;R+8j|Fw>UOQ=}uw1d6PTmyCO@FjmLxfM{UXXSFX}_)ADD3
zx8bJz>w|H+zMo%O1#+$3tMTdakEzS*=I!5nCj3<M;`gdNw$h2Q;gMoHz4UZ`G#x&9
z<o+baj(};~`Isg>xfYeobIjoU71P|)CL41_Se5$o{mW`C`1ulKB-`dQ&jD9Zhbr4n
zOxORvuQvI~v$Q||<tryJ2evrOS}|eSvGeo)_x8-#x%1}D8xk$8l2@-pELl49hJbtF
zzx0HJAOHT|KiJ&iXyNOtmYBHa;`xSCuG%(jiHDiQD?Tz6O8lIx&U5S53$BVv$(|mG
zEn62xKen`xNZ89RlA7^5?cvMEufyBQ-xru!NWdm(56!cXkd(Bm{q_IRQF~R@3RR~3
zb|-IcwPpW5{H!V~b5mA!V`dg}b7NCeV+#v&yV2;w!G3q&U-7z$4J|B-S7<!$m#?p^
zkO(}@@S}kx?U96(^6U_no*teLlaJ)&O4`+>2#C4u_iJr#Zk#JyeY3$PvR`yh?dgI)
z7W+1aL!#=?JPU<n<@t7wO3e4tQVw}2O_1YaR`K1i;lOE=CljSs<Xc!stc(B8{#{i1
zcKwag7d#eOVrPyRHi4?fetF}Vy{8TwIB?*|5l|97+xvG0@22Os>;9+hsFj`%nur4x
zLBVScHf(fU%w=}qEC0(iegAoQd?sYECNEW(CI8RZz@X;enGIE6_4)YP^y4KYBqZ!=
zf90nf;V_raS5E%1U3Wv%{(4EhxE49tZ;<8ZO-&I~9GiaKx1ZGR{{L@s#ot*EAG_CN
zLA|px{NwNPir}a6%wIyHT-Nd@D*K)|7~P#wTokbK`T75LRbRv>8O@j>@#f8pgmRC?
zyc-uAx&?4E^BNc%*Zl43u*uDxqI)ib;pdgWHD9^YW;aiq`0#`v)As9-mj0oc7IrlW
z7DX(H*Z%*XaD|~a@JB0`$HpK0(=sOH?{Z%tYrWtrleu~G4vWPN4F6aRA4N%IJUGtZ
zmVJGJvF4gJJUKZx-m^M;b7in!UbK|+iOA>Y{D*IDX3^7tw9O99w3sndG9n^iS>?a~
z?)#d5*efefTm9DXR1atza9{oZ?U|S56>l_Ny}#c$BlGX{DS!HP9Tl?WFEVU0;Vx=E
z^!`4NjLeJa3lo^OgfzKJGBY1OJ)KA1P9S0dXoJxI|A%fqOGrtn_}=&My!?^QnHv@c
zas0N=Y1NFGAUJ)tUCodGFD}|wtd0ns>KG%lcCt~?YW|o9_wC!Z=P&%6IA;z|P>_YP
z&}&ZrAIJAeNJyNUlUVV&tpU`r0LAi5a79zv-6;C<$^ZWy&M*J{caORHQNGeac3B&z
z<+7VM|L5QN!EfiLt#n93<A=OvXs;S~3ByZWr(Ie+JUnO4@a(T!cz;Usni&$z6Qkuo
z(^~sMYx5ZQLSweP;@E6+3t{m!^P5(PE1VU)(Zao`>Bsl?_xoh)A9`Pih?Kl><KOC)
zh6P1^FTSbOefX??_;7nibmhL9#D?%w1r5t{4#+%o&@R2VC$Zuy*Xc_FCQyT?TPWCS
ztjUd4ajcl-@N(XPi{Ib>m$&`(|E2IDA6r{D$Ak4A*BAFNl^i&6;=^<PBmpOf2FDPQ
zZYj2$$1)j@GE&)E{{FK)+{$gSJOo<)9hz=oSD%)U>?}FabycJRyYU3;EgE;a#s43^
zd+_-21<OE{d%-5zlMim2w|#ytnEbq^#zNiiV$}v758Kd~2mk9qTOQ)}D!t)>I<egD
z_xbr3vN+!{$!Id)TzlfIj~iRt|Lu8(cXl#QO-)LgFgY1CN0h$&R@8}y><SaVKjl_(
z{PXv;f>zhPXAY~a70w3zaeh{C@bmMF{uPHni2)R4;YBrnP89qy>DoEtauv@CPS&fw
z|Ns9_|M7*>@Y1LE@}P?1(xsAJf#Uo>zVlnHvdB1iu({)iqQ(gYy}r%A5)OpzRAAmB
z*jN8wf=|Zben7t{)V1Q?Yj)^sHZKX@@%w{F())XgQCo9zc$%6b^6ZvLSNxv#@U%Fa
zn;R&*>%}I4_ADIZ1+VO`Yw;^HHZ<($GSbjCe(~Da!oom8!eO<&S&?3~iZEye<3F2{
zE>WRvO-&JQ2NXW`{$G;x#EF&f-JQfAe`bCVJkG-d4X`b<thpKt1YG1h)~;od<`=P9
zDZWuqw@LHJ1mPQz9UZL)`MU&<tlcIvE%BSV|C1SNTgnvH&3cw=c)%hkba$mGhez;p
z@6Yo}_8n)t6xFlv{l;m#mT6?n`da-)<%tDDscfp}JmEHu#wjV1P6Dn+KE9vPSeYh~
zaa&k%(vyAX?#+EVOQY3reMm8Qf7hYu7XSXISA1%D_}rgOPfzEp%o9=79YS2f!iVSE
zYdHS@tlr$faQ5`)_xB}d%#@Unl-!n^Y*9V!;Zg23b$<!HSSQQW2lwsYc(42auhr_u
z?{d)Ap{i=hs%puY7>PZ5B(`joeDYLtulF2<2A`M%uSL|<*pibMnIx_K@{l1Yq~`ar
z4=)7IJ=2wf#EnwG&;S3<et68ztgilRfh}|2Ef(JozsuXo-wRkaOq|#_ckaXQ`LpiC
zvTpqKG5N)TM}4w9)@24Ye@}gQ$jtogZ`P67oQ&1q4Bp&4oRD@bA@k&dRi_?2V2Dj>
z&Q9pc`S2Fpv(b-NT6WXI;==U82jyDNYPgt#Hb4Gd-qtP7S)QZ_amJdt)?5t^JTCHD
zC#K0AxpnDu#Ua~cT^wfLZogz*^q{G=DM`@Yg55(jj)~LQ{OrTOjlHKQeLP~W)M6`{
zwR(e&MDvOB$*J=?m43}zmMD=Cy+*Z<)3Lhvm%{UcKAYoFrzWjZ@;tGrTuf@$r749=
z*74uz3J`fAIN7T6yv0nlU4GUd=4}Np^;Vhy%BL2EO$X2MCGy_-<?<&th>?-a&(G!L
z!9#~W{ILu%Y`w|1=<iA4w*31N_x2>-*wFa!S83bz^^$&b7gf$J{b_Xg>};O!^+r0e
zrzXrgm5_2oLHF!}H9iXx{Zw{TO?~*lvCV(JKx10cAqAbDJc~~$I<x8vJDrzYTbEk#
zcUHu4NN6mXtH{;hAmA!5y-jb%5#b1PX`43ok7Z?xPo@ZLTe0BalS5qx7c6oVxc1Wc
z!$adU|BVggpA@{a(fr{2kLBdobB86ixN)4Hbxle2SU~6*<5bT;1=Yh3lGkXuN-Vmv
zQ#f91S?9k0C!)f4^%bg|_Y9uYVxe~WhS~qViXRtqy<_iN1#>LwQ261!yIJGfC6o7)
zuVudhx00sJRpf3C5NWkHYd4ED(u@i}^639fdy@#C!+ISvOms7a&%UbW_AkgWxSzMD
z;Jj{oG5?$b{u7EUxw{wd*T1|`VcXun8$&;>lX++IvF1@_#56to5DS~WFSgH(J)I(r
zc%J-RHo<C(fiF*U<+CEizMXeDm|~{Co_e{n)RkjeYvGdD8PQ!$tA4SrU|M{<d65E-
zXYjjn8TRvsx0ULz*V5d+eQu=N<u|p89ZEI-EB-|5+TCfJV!~Vcw|x_24@1vvL#{&(
z0<QCqummlcDB@(bTg1^LG4ZjjO`P5W%SN3gzJeT2xS~2bbXLqR)#V9XEL6Ys=(NN8
z{?@1c>VACC{&wB2SLc3T<Mg+l<#X@A{<}&p#(KXqbZVce>XtJsogj4EjpJMOs;;YC
z$v56+{5IVy)&J3kjrq}xpL1^ZC@fgFQAeiz;f9T7+oin<_ZICeuq(Ei^gPFG&Gj96
zrGGz9uD6<)`ok{sRFS8IkgJZE$H|PAYqo_O3^UF?beI$%lE0hj^2^CD*ME;cAOG=V
zX7rY2I{&@SM&AV&C_Qs@`I`y^j@8$jNwoShNvAE5kP-T-EU==E$$r}X0}tM?G#f_>
zZ1!#9l-b32m|^k*?wDN#yScx=xw-f4F88UPhyI>RxS;e{-0KJ1#E%lU^U6IUk9|Fr
z*8E=CbA{F{x5%K-x4D6H*<2NQP8Od_FSEWhMdMMCW5>QD8lDz>hZWlzUmMLgo@@3+
z!R2_&Y7LKIo$jMePbYnyrJ!)W^YQ%pvOM#agsn*?hfHqfeY8oLd9&xaaBTjM_pv!w
zy{0W()uPzW{M&YU@aurU89p1{>TmoPE<OL)fyW0w-q`NH@+m852gaN^w%lMJ##m*v
zIP$&a<>mEl+}-Tx-7J65|5<p(v;&$_O)*l!tVho-+?w&h-}B;Y<GXhY3J%{bz9aj6
z=ZBkSvwK%fR_f4rSXT0H&9;flSQrCKwnvDqJDqZ<sB_)wR`1y2odyyUR5}7gR5+Il
z-_|{@{M<P8=hcAqCj(>C?b5AMEmBO9%!2RR{C`n2tG-TY&Gj{5Z@zbx+;e8Rd}Ldp
zj1KpC=gSiPFG~!3f7QO471wXG{C4X0#{~*~ok~aZczK!)j(nG&yM5-O+NeLL?wtrd
zn&dd`ufT6nknfhv-pJLIAkbhhvS^{etMUyOWA8f^lxM#Tlv0hVk($3SPGN3|P@#ah
zU{qh5impn}zLz`R_T^i?%L`k*sN-*)LGq<Z{S{|^p85ErY~2cxb={epZYSQ}qU&|h
z<cZ}jzga=kIwq+6b+Kx{+VxdRu!-TmyXLmr553>`$Auj=oX#D2t=_o)dFJNS!0Q!q
zN)~*lRof1xIfrhHDA;|mG-kaB>-XOY`<E436j=G{yiT_=Jl7%6=5jGV)E-<WY?-r>
zvjtRAt?t+`QL1C(PTq)yjkXaDwi$-k5(K9yXdId~Q$ljZ!Mq~w<GDq`eg-`A&mZ_>
z-y>`}t8L%$?{7Ij<`zXfnUVha_ZcR=(*jMORn7-?y;kD6S>94G^@GW;K$k!T)l|>e
z$;*!Q{|J_0zuXekqkOpG`QNbMlKHnP`;+IjD>!&fYgI|S-O#_)Z}p;B_t2Y5V>l9@
zzMoyXH23V$9o2il4pq`BjEFE;6M6W+xxPsv1xxukq$U6SJ-wmQ*f~gOT5`ZcQ&wT&
zZ&$DGV33yHqAMwIV8QzH3aUpFGEXXKd++$_uKVGW>fz_-d8X?dpP6^|!xzoNmzT3l
z`?;90qkd!9l%5`*OP6?-F6Hs`e32agL5<Z@_4xjJp4DN7TecbPsc@Vn4;hwJ%6j+b
z7;jtkcL@`d4dxybrH_3GWj)c#ZfxA#%<P<}cX)~Ci_;tr9G!hmHAtL0dUV0clLt<m
zc<}P1fWUzR=gukUol{V1+R$z15HrUyW{zXjEJcF}i64{NK0lYtyPI@l*VM+jb60#b
zI<h}vcEF)SJb{5cA3s{SyBbz2W%ve~99!<s!_F@h=hwptsb`e7RZg3KJ|L7eH|xPF
zuMZ+}|Nk5BsW`ZyK+(D0K69@oM?*>0j2R0y$4{EXqo>DHQ*+_utb+{m>{53Ww?BLt
z+_uykG`w8#Z_dN7*^|1R#l+Y$GuzJiwEd~kY0ix3=BoI3pkb{=qlk1npJ7V+WMNP}
z0$L|;RwyB9m~*RTbK1eFE*B0Q@8>z*Z|pPYM8OLIP#bf9eKNnb2j{`_{CCeJryf0U
z{(M3PC=fuCr%Hzooc7*iZ^&45?yyynGuw|ZosCncCSKQTn{6)n?r!ppEuD?CXFohA
zTk}_^LquA-d5=x=OiAY|tLC-3e(aCESat|;rJOOho}GAJE@^flbeqo_(D;ObfNOke
zvf@h?R_TLIEPES!kAG5-bJ^RLAT7POfy3#|Hjf*J-gGqu`x#H*IsE)@;yb%$0TbTT
z%!-Sixd-069aNIP)g{e#Yg(n7*ODXOZ<bF{>WT<|G(+WJ;fdm|rpF4W<2EG!$Pc)A
z;MlqQ`U^dK(jsfmZ!_%E>Au-7*p#Wn6UcR5nWG?fRhP-$UAMpd{TeJBlM!TVwySTK
z*{tuGptX|!|0^B*e()gA-@iODF*mdYpWI?u`{97p5zuy?zo$SQoay>J=jS_%Ti5<R
zwqWt$4<D78rKOuU7?iA-UGzadc>@PufvbJgPsg}OCPUEdM*V+@x<3gOKPEJ`w?90_
z%ciN>R{mb1qGG}Gu2ZLtH*9n)meo7{SX1kR$`tc_iLNe*vUe(L%bG!hnV@()c=^yF
zp3~C|B_st`gl$l&V%jpzC`MqqkxxSy_kpQ_?<W4YP?P_#B3)sMQ9-D7{XYYp$U_(I
z_#CofhP6Hy&Zzr6E%9a0ivV-U4nBrFK`G{J0b_xv<o44hUJOw|3#v{zTr2vlc6g5E
zljS^YY|!1lN^AM<@0j@ToNSA-!SM{vT{Ctugda)v6+RKw;eYCj|CfxT4F4LAcK!RC
z{Nvx;h3or27}(cAle}(W&7Tti5ul#i6;>XXcGFugl~UR@(}EHXgxMsBs%W^TJUTY3
zYQkK1PKEQK8J_>|+qYR3OMH8?@QN+eh@D^SKPoxb+beC7oZ!SBZZEyI;ovOM3C(Pi
z8`Z4}Q*~^1Jz$t_Y;aY0<_rmgNqo$Lnr%Ns8r1sse0^qr$>h|#yFA?Dlgt>qMWN<Z
zmS0>S4+_VKD5D!U9Og2;T=?gB#Gmi-rM`*>r@gGdaX@?CmL|<UZ*IBR6AWiPGj175
zZ!3sZNj>%7<M8o}M+J^P(;s^(@mO4Xy}zDET+iUkcj34*ZqTle(&oRpNrw!o&6>BE
zcx`WOWV|>v)xox#V>07bxoBoy{~eDf9g<o(<JH9-Dqrn5tY+IxHazX&BNQ%psKJDX
zU(@W}z0?)!+-}}<fE60Xg$9O(6%__Pb59*O>$^j^yYvk6%#ZT<{e1p>$7b<8xGCZ*
z6DiJo@5kXSeE(;#>k0n(mvkU7Dq&K^D{;0ZQ$BuBbdHrh+{nzrSF8kDECxzR`=6gX
zc1%FvfI%V?v$i(pNv6pz7OjHSTORt$@U)zgws|U0WZ9q2(Dfig;@XkMXJ$5E#hJ6R
z5*&QRcz7a$nA1*Gd}vs>Z{94;X4njnQoZ}<_3=EfuN(jQef+}h(+5s=<|jypvmKgz
zbmK9Nbw?WAxg9l|%qHF1b0AA$-7(>?$TgbGlN;}x+fg@n;jUQ^pLiyjFA;$Hc~Qxk
z{)rP0-kh2E$mNJ6<A-nT|B{*+qfa(wTKwy9JolYtu|W3jNh%HLE~3FL65A8iq++ge
zsWI<k`28ZV?NydnyU7O7coD<@|BKF@*w)DCoUY!yP?4FruT%6)n&y?NM_FPi4L8pz
z+~{$=)}14+*KpM##_wpp2y@>N#|=>jC)+)Eaaz1>Yqn%n)d$%;f0(OoZE0^|adu;K
zPEsmK*4=aXP{TyE2P!tRI#+J)OcCbXaIK_b7V8c!%O+3B>2D);9@*~s|8Me+>gkD(
zTwbiv?}a44MbG9)XlpmOvN~sKHqVqy>J!X9b>NATj7v<Lo3}v968@%Zh7bCxpB!n_
z>0Vyi`mAD&ZUCForQ@Kfu>HnsA`Wh-Fg(0A`pU~qk6?cM*RpbgK=WQ(=Tg(=9-c-m
zwpUD-vc=ePRx)n>a_qi<qT24qT{$tG%>9->Tt6)5w0Qj3y{-7UMB14Ii(09wE*oD+
zZMmrKQ3HdaA!rJ2f&iPc@{yIrGjjRZx`H{ERhdjKT*@Exn#rg4h<Sg(Rb?KFQ#@=v
ztDZdfZ?mnIFw0M~C}>#3-QfvyQs^Vl9;BqC2al64TuAuvEv=w1F(61O;HbhKmld2&
zHU}o#dFakGFio=J^lj-W(@^{Q-~RBiUY_FTMsw_ZpUslp1a(v5BL}t#Z;wu#$YW*I
zvG+8O#Vm!2NlczkEz`C<oTTb6o|fO5z%03W;=u-G-<rRt3Ldk8hV`|>jclsj4u`2j
z6a1p!M-9x3jBJL6Y?YO4?(S^a*=&rAY;tmJa&mK0IS(qb*f0c(g|o9vw|t_?XX2h@
z%I>fH_+M??+Gt5WIq;z&Z=<#PvnPUPR6xn9{E-7&Qxi{44v&@=kCPKmRTa;*YdkeI
zJbU)=JbI+@cx}eUj+nFwvn)J98|qgyH+EF9thRV&aP0PWp6&UdQQHlb#-O7c*2nXA
zIoCk^Ds|#WBU?p<goTB~kt3kd@0J#cHEShHN+kaMNt`e#NkH6vOV&dM|3!|>KdNPZ
zF#Zeb`r*bV#az_#@cw?D+uMzE?wtXh3bHkuXMg?Vf=yhYqX8KH|Ce&<Waizl$*7>n
zsG!KHKO;3c`NE~74V#+g9aK^~rl7_qTabL<8jE5k3+sNV7+wV%*~}+P8f;d5{Aql6
zZ8T5$dn21#@KF<IXY<J01tf;@L6fr7o=#@%2M-e;JWG4<EKNW(b;5*%gruYg&(jTz
zLDBx;LBfx}(pAS8_4S#}%}cUX4!St=9EdMW*ygPG<oWpvvtR75OSPz*^YBMuo4CHj
zJp1$?UpgP2mv6hj|D@#B#jx{{k~*0s9r&Dne(!IbI<;~B{Ko0xRa%Sp*fg)vX>MRB
zd9IdH-LXKSd4U46y}i;!2Zo79IoCz}`7K^0bm9O13rcT)7Po1KORS4&T-N#ey5u}t
zP$l*7>+6H_L<}GSe+9Z6OMaV>$g4;H>*KeJiLkS8Y~H@Dx_`~hzLctRd$Z&1ea!D~
zmhVw_3sOq|c<r=vn%1$X<TgpgjMF|U&zOET8j3_$#NOGyt7>lY?H6-DUs;~f8>Q(Q
zo_OxM#De%2;GUz@yiR6e10$!)jLKi4QrN`A*v!l{n3b3;C;t6i{-QJBMEdJP`Fno<
zk=zyd{<?mfef<L0>(lj}tpy)Ivz3%)XEV=%j*Y8&8yTNQEKB5v6e$D`lVWDr?eejJ
SmEWitq`}kG&t;ucLK6V<ok-6B

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-0.png
new file mode 100644
index 0000000000000000000000000000000000000000..2c346e0ce5487e21d41aa4e6306fd83a7b4ffdb4
GIT binary patch
literal 17500
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qjQTB9k45^rtlpw*nn1P*v
zfq~)we<{|*4C<pyI)}rN{)4;+9C!{X>`eLJDEwk^kok&p`cK54y=As{TBNmmvDU87
zGV_FIPCThq=B}osk$BzAgMm$_^4RWsS9{X8yq2sDI<$M~?TPiDz;SqG(nhWp0|D0Z
z-il)aEgox*{Qtj6M0$f@(~8~Itw$TTDtxavdgbt+a+zZf3`{=1S$4ATvHCH!7OR$B
z@3r6eexF)=qclTzi>{JZ!wM~i7(s?RKgzF4A7)jG>Uws@#p#G<p_jVg^E0Q5)@e^y
zx~=wG@A$$q>t|M`#DguDV(M&eZjN+madv13_UMR?nd2Tf(K~FKbJVO*pPtBo2@_|}
zZft&RudU5&{JH*5;J^R&yA+g_^`_2<`j;5Y>6`Wc|9|IeKc1ieZ=V0}-rnEezrEdm
zef|HP#n0bgUS4ls`|J1L-}~p=|NsBb*5IHA8#JkwbT-eO8|l#C95Tf{Xi~hV=ZO;@
z4zY86otir5|F^fcW){9|pEL1PgVa-=86SI>u|>-4oN!<%_cAui&OgqVIT(3({s#!(
zv}bGk|NsC0-~a#rU+(|^-Mzo%&(6%hy!`*Y9UtGny<LBQ|Nq|~n;)K+-_>qZ2(|df
zk%K2r95{ViP3h3;1qvw%3h&<NtXp+TN#n$UQ(T@b6_2EY=P+98DQ0ryact9K-o4S{
z%3<FB?Ppii?{NM1w|SfIKgG=d-~a#r|NQ)adFyZG_xJ4&U-zf__qYGc{dw%`cpO9?
zfO0G-XKpxh@Z?D)y>rW#sbytrzka2)Y{7v;Cp$IXDt`R+-`=Lyz`)>7dP<Af;~5`&
zcZ(dH&op7_h7cJEUH*zk4#AysVkQK0Hu3rkZ)kcVzv)-~|9`bVKD>W)^ndiWKi0Lg
z9^SQX`~F_C?(d@fDG4CIG5r6(sQl4`=g$KI1PslmfBibu)bQccXKqIX(oz|{RW&9a
zIdP&vM)E{dfw~minnx!z`E{)%CuGT7dG%P{Z1F$E%*Jr@{{g2ZrX)2p?!EMr{qm3Z
z|NsB@pKl~%b7Die{^6&mdDh1p*Zg*S9?=B#;i6!$ePLq71p!?A;US@^>4}LAuEte8
zNzY2x1omllERtKf=iodJ18-KFjtNFnZm}4fE-;JwuXx?`geQw;>YwJRCOrTD8P@zb
zQ1DUd@V~!2%l(aG_W8Wu!~#y6i*g@5c#tq{vT$K(C^u)|)$7mGQd1pH{rzuv^Twlj
zjg1fGwZ!FAC;WOUzi<;{B=<U%ul%zWq8>SzGitt07Ig7r@pv0qQS`GtrbuCB%W{4;
zHc)z5_~Nv9+w1F+d~zu_c1?YF&(_M+7Zia1|1a`<l#r4lApH2^#fklW*NO`#3J5gx
z7%GU17O*L;3fTF7!!qdz-R67gi775Bf@k*dZJzsN6Qg_eWJX_s#im`;q$XqwgA(MT
zY1$m7A3ia!+|yXW*;@EP;qarQJj?wq?ui8j-2eZJiXJ5-CQh38aQnuKr_Py$g+5J4
zPj`50WMGhBqVD-fi8Cp)?7$?BD9MJ+XAb!EoyaoC=DNeCB*fx&m{oUQ;3*ys>9}17
zR?9_PJanL8mfeJ#=J`fC5eGIDs~=t+&SN5F%?Js-s7DFO$zef4Wo6g=y)RFnoSdAR
zns8vQ+5=Z6W#vUr`vsr=mz11ny;P&;|Ns9F_4zj_O%i#$GT;Ks#8sRrkw-Y#-);HC
zzw%yVxW<X2oXpJ3UYi`A{rh{m;IrD{W4%1DuNm4@IINxF0t!4(>T`OOkeWJas_2(*
zW?o)KVF3xLsi`eTZJAYM7fk#Ag4gYi>rwljIcFPYHUyL|oGx=>nc#e#2d_GFdW{y8
zId4#vI#Z{!=iyKOP#N8+4$P9_KMx)_@Zr1u;lu4budf^D+&WV5n2lL<c{Rwo|Nj@6
z6moDE|NikpRsD8WcJb!T#!Bj}ZYyq{n0UNln$!g)e&zJV%lUI6<yX7uaI(5f9a=9K
z`P*J5m2;k%W>mIF+}8!_Ok0{*R{lTKFvo5~m`TCM>+2=ed=hSK>wWmt`&G{_NK`4=
z7k>D(DKbh+Mcvxk!sF8i17oF`Ew`Se9PnU&ku2&_c<jJ|5Bw|F{bbkHl9(XGSY#}a
zXEBkLSuS;ktQgZ)PNC_Ha>rL_vhA$?Bp=CJv?(Fs2Y=?$q%WC@yb*6Qne><v|NLeb
zwXd3zcuJ)0@o~w0_3GAn3n5-qGA``s_#h&_yS?pIXZL1fV`pB+)xHlBdshn`T;Vuj
z@@e}J%_$uR!y*NQUmsD9iQ2G?$3*Yimqyi{3aTCdCDsZ&Imd1COOiuX#NyYVnTZFP
z+EPzT$k`+;NJ*0bClDn|kmp$0Ua6{DN=bBV+Gu!i605R~?1H5{U%oVkZAdOKynMu-
z?G5W;dzruv`+pw30!Km|B7`-TDR^-<P7L_B@S5g{Qr>TTy8l8xeb+y{)SJgR-6-e2
z-#-Rzhy}WZ9$s7_VO4A_FS<K6D64X_EtztVp}VN!=_h`*4&(cWmjq}DK51!EHtOgU
zdNi47QPY9d65`V7O$QV%&QhN7_QL)X$!}-8WSGbFVY>W;g`1zAmQ?dis`xo+&Jjo&
zS28Qyu*s;h;>EM4J0qh`85tg&e6&8|k;A&+3Dp}uXt?`{{WqA!myybrrgVTOHRfXT
zwUPtzjuQgc2?TFmV7NhT;{O#ZpU5*;{T7MaxnMb`4_n%)hT{G8={L4^FWfov;X~(1
z!k~uM|Nm>kt@#=hcxp}cUZmaF(KUPV@`A@Y_j<e|+m}bQItt%+n6)ol;+JRU{f@SU
z-Gx!p3|r=XRhg82>}%oH>NZW*gT*-$ST`8XIsEyKGiyfIn^TpK;x6>QtO%MJsvKsM
zsK175RaN$%raOMq+qY?jgg!Z8$g8!r;=WZe$3vNClJ}BhCtlhvn!*lh@hQa=X=ry(
zomSP_B2ipuXl!6$WMt%&G-cvMM#~oyqojF0%xSq5*1#YA_wb9VKh9^r>^POUMpC4H
zYX546154YMxUm`L#Kax`qP*u|W7uf{Ju{AL`v3nM+ti&2h(2BL!{G4q^GDS9ph<9o
zMNf~2w)WpUcT6H8AD(Oy42v=_Qc$hjax-KJ8@o-U!gaL*vui=({UY;<q>ZyU)dZ6+
z3AA=T*5{M{pZG(NTRMBIQ;n9R^L%?_5iz4T_x*EzK}<drxpu=Qqno$0r%(ME5vgfl
zXz0XXQ|NG+St->a<I7DhH@6B_#>D^02?<Flf7n;^NlesYTA|}+`cb|)#o>|B0iTA=
zk^x(d7jzjU1h;V(T$NpNP4oZ9BF^mgDHlFyw6_>X9c^6q+`dSaHI!e{?CHbA6>C!y
z($a41p8oK%`k`yg;PiQDo<&7PL`>{gZtmKi-la2VEXZt{IFVCG_{Jsns?0{4@N*|P
zUVc=%#Lj#;e#sO@e}NA#(p|cWnm#i6i_I`(bf2LVAuPO1%!j#?kwx|o|KY?RUzC<}
zvh{>~dM|G!E^J_E_~-ZW14oW1=${XWI|nMv&duequNV3NY6kuP-!jvdtHFS$HJ+)p
zse3Yef!O+sCwEt{=|3-&>GtSt4hiBd&Gq)W|9jTIBMpmlYdId<_#eJtD11PYJ1nK3
zbNRa7iAL5d-FwgR&SG}%tNHm$vdlZ&S9%%a(U7ne%TKGkkzRQ2>}G!jrQdp*o|hk3
zs2rQ@v8TQGe)F0bmAx|^!a8?0JbUJ_d+}|?FSgTIZ?~2H{h7KS94=1z#T*<36%{Y0
zPGw!SlGD)G_;4_rN?D<Z+=rCL6_X}L&E26DzT-svjw|=~*O$NFSN-P3V(sua+lmj3
z)2I9T`-cQgijJIl{<L@F+_@8H&TO1Hld=4@=;l-E`z)KhTYj{0>mA!4zaP|~`F?J0
zytG-({oUW6?=JtpulD!%hllG=Wbp_JvibW@yAY5jF7C$1_e9|Eyxa-FY$;b{^!TP9
z)#7vi|Ig5->WF~!>4L{>`=Diklc$W0%^E+ysL)Wys;WsnJw7}<J|3A&lUxK;mK^T1
z`pMT@U6DLF;8a7`|9^j<A86bytS)C=_vhp9@Ao&BUH$R%=GLvPXU?jgI>lpQIrHa_
z2*ap<|2GyID$CWkvt|b!xZI#R$7NmI-<_G4uUCHFXH)Uv`F;C2o0pwA!y_ftW1P0(
z#tnt+<9r{??F*N3rhNIV4r<Ww^753wf8-+rN%c-Dd|_c~-rifMO$$g)j(l>9Wt!20
zn6!wkOqw<mj<5N*urSb=aoUuR@9*zt=f7uS{yuv1vs2UcUDjp^iKUenPd<D8dT{7d
zVWFg>(~cT0miKLG+;NC)x|q#L?xh~PtG{3W_}E@b>Yjbom-Z81Efy`}k(BgFP35t%
zu@DmEdEn2!<H$tr{`~tAY3EWRHZ(l^S^Opp?k-*3ne*mpL`SEJiMjO{F>8xon(E3T
zsT29f_0d1)SEmy-jq~o;#m4MOJw5I9j$8L47>$jU&CR`oCb<=D+SSFy&(5y=?n=XI
zl><E?Y)V|rO{{FpEg!$X-+y{~{h8U{Gq0^VzrXIU`pNP+b9f>nd3bqwY;0IU7OAp&
z%xszIwIR$<;NboJl19doW$%yKwQ_@#Vbi%enp#~~uKkLR*50(iz`$Yl@tzA8dE8vA
zIh3#cKNQ8I5qR!I$4(WAb8Zda-|yFtC`h}$E?T8kmYMnR{r&&#Yk%!cICyr(jK9&_
z?S6e`ue4-kp5&UbZShg1S-<L6bh24G8YIm)+>j+Whuc^3{QUnb*Vk{a`g(o7y{ES%
z4-e1%{l;rzPj4txewb=KVMXI{K4D=e7T+$xy}6vCDQRgd)}&0Bcrecn?9`;BgoFuG
zMOUo&P*{4kVJ%;KwWzW6A{B18WA+|7Tf`37_dG~*;QMG_|Ie=GN8YC=$@cY=ELZ+{
zczAuc_<g@Q6{UyU+WqI(9gg*T;KZ{c%AmG)<~)fBE*JK6dK84O{};38$K`pp_v7~{
zT+Zf~GswB+awUf&O-TWiH9NYEj~qAd=uS2SC9?njlZ1pHdwCa^6<>b(sOkKE;{|Ce
z|MR?<%XenVM{(1s3EP}6dMAj99pjZ=S5P2ey0e*mj?K@iUteDDF0*aqOz>riILfIe
zo6<Xjb1vibhsXQ=-q`l`_V#?0X%Z_EKBuv<vCUDAJ;9Kz=$O)#n2_+`Vd94`DF#M|
zf_6btQqrgI=HI_u5f@HQc93OipTZ(x$m_}eKVj?1c#|V1e(-Ny=crvOaOe8^{TntJ
z?XCMeb&^0zLc)))ukW{VzYkkqSMmOydXA`Mlx*W_&ZgA|ezc$U+0xRKcW$P!ecrt#
zC8rx27!J4bMkw)!JFK%dko-}4@{|z^i-Csb!qt$98svbG@aO3*>;ASZV_VW{b|R3w
z|F^wCi-wtm<in@O?XO&E4twHobI~H1@9+0Nex9ywx2K?yU%u|`{r^eF`<6!pEB3iG
zNPf)V<vSy$9VOa4EnorHJiEWO|Nflkm*?0j&Qaaf*w}dE>Y<cy?GINTyf~qt-X0eK
zX^tdq+Bh*a?ce!^h_)~D8yPn?GOgU=pr<wAG>?mxqwD_wVXi2xpZ}kqx7XF(=s%x>
z_n{EuzrV=|X=ye!Kd#QVuQz-&i7`K(*;sRdD)*mOmT5&#UtYF9Kkx6-8qOcz-}5Y5
zr1B=`aQw~$Q5Qd@J$raouH@n3(ny0eRg;oZ|MIVP|F@VoPVuYPg-1X5b#^BBiq&si
zs<<WS$@~5F^*%l-M~>{!<aK2OfwZ(^(cA6jTYO}5&-}>W?3gmK!Iby!;e-FqANcV8
z>}>m*njRmYDZe}-W;jO7XzcE0(cNIlt|4w5`Cw9NLPA1Xnt@T0fqD9aCoU2kkUZdY
zSE^;2TEHWTc?u8C`d5TD1#VfODy!po^4PHh$B(~ywaUfsL}J7F`TtLxD%xB7oB6QP
znxAr;-40x7-TL8xPO`*>NqowW>qSMyPHkWC@`b^NPm`9hW*%YO`@3tuC{v)0%8>_8
zjwmQL3P2;;>D<{9);berOJz*_``?~ReCO(>iL3r4N;GM>Ub)&TEBn;N)$^q6j|26t
zuIuFO|0Rg9ReZAgC&dyTbU?&?hK~C2C2P+3>`c$em7Frw^IGd1JBbh1Oe3P@!oql*
zoH#B+)3cNB*9j_3L3Igb{t}Y^ldk9#eB$30(xmyLzwx2Gy87a&Q%hW0)L0WaSe=}5
z)~xCI^GD~+n;waaZF~P5fADB!+k;K@6K_Oht^WW2e`cnkx3_U;hrr#l2iSJ(l>GBo
zQbkphUqXi={326kWlNh;M~4F|G$kcD=v@BAzB(*m7T^E>&dK5{4;-*M)OhWm!!xDn
z@*1r>XGzsZ&g@*a%;00K*_sE(`|TSWC)wEeY)HEt-&4}CT<d4IV$<@9112mB<u2tF
z7bb4rp00dbzov#~<w_nlwv;UOP1Q|z3>&$N3Jo>1m4#FxQJCZqwdF*EQ44Et_!T9V
zs0j;H6&49b{rxn(^X2{?p7Q$rdzY%Z-c-@@3Uh9DkC?$4z3g~Y%Y&@sB;O-DPANW_
zCBE?3jTtA8wuX9lpE-5(bhn~vYh&xCi;jP0@@B^}GnSR{bah>5Gk{o@nDB!=*8bmo
z#(fK9@;LHN|1(~|H|rql#2qJ84bt8-Omx3{H?_4TA}g!!{q65h&d>kPqrzluov5#0
zsH|*vWljIo#ux*Ggk)vy<c6c>Vf+6aPW)8Q)>hxyxyFCqpPwHd-rt<AQqewhreoyH
z#9t+BjEq5t8Dt+wMY-x(D&*|&;`n~soQ<8mjhS6SLITtyV)*|*$!V3Z!Drs6WZ~ZK
zoBe;=PyJ|LyXC-^r}oT;<AsGMzP$XltLw|m86GPVA|fQ-zG>m+n)3JW(S&5<dc!mB
zryp^68?F-CBK1UnmCm(A9Mz16kA2&}u+dhnt@p<<c4=v6501mLOhY^*D_)5_IlBDA
zI{l*F!$*%USiq2522E~%{_SNn`^gu0sw0PSPVbQ){L@=ymoaSRc2hDgm@0Y0_{)6z
z`to-*duo2p{rXzI?zdaQ;vF3vv**uu@$TN$!Xo`HIl|zx`8pxiU}ffuS{pZjhR0rC
z|IaI3_x;<yeFcpNZf<F6;?dRRadgzM_4Hymc+{@qTBS|pp$oT8ZAj&Yw2PDe{M*~t
z*vL4~PD1kGoBtb@F!}NdG-x&LnIJLK#chVtk40RYs(;+tTFxzg@8;g$H?OXi|Nl4n
z$9MifFC!zPj?RtN*4@jNpT2#ALs6Gs!ASN&S9sH=3X#Qz6DFSkjRl^a_c#6Ir26o6
z3v!}UQ&JL=lLbUmJv<XVJQ5~MU+%YQflc9`|K|e&4{j*aMYxDf?DB<B?hC#g9ZMQa
zeH~afP7|B4k}+5&+hK9~wfFb`+f;x1dUyBy^mBV^|NYs$!(#U`wf<>h{SySVwY8a_
z&GVAE*twkNfc#$<%Sk&nJ&|WV{Qvv=|8MV~FZgM6c(FUroH;z-zVUQ*fzY8tSA4`y
zykxG^WBzwFDdSY#pM)FR`xh>0fo9G>|Mqfe{+c-XXuZbSAM(BerQXUm{>i?QJRXcK
zC)?P~4@x+7r~L_RTE5aFrDxLI9)rosMx7mIT)W>LVfd5E{4b>Fm58$=Bh+t=KUr84
z#3Xn+Oc=LHxh+!Vdt%b4_CsFt!Gy!ymexiI*@{L627ivKwa(nbVa8g(X(X_G?hKKh
zmEyK7FDDp1V9*sz*ebiPZ-U^qmyONMj&5yGS2ng<%P4HCKH&3{e>>Z)=YP*L%wFmo
z)-cV;fYp$BO5+rrZfzzF9h(p83R_%!YO<QE1-$PqR(J5@I?>9qOzO#%#mkN@Sa1NE
zFde<vS6Bt?(0Iu(&E=nRWRJ`&9uJkUBMocg9a4@R6@C4~l|xO0?TS!GV?fkF<<>9-
zUaN*_A_l4d|0XxQ;fGXFP19xmC*GbcA-N!xd6R+>Gt&fVnHeXW8X0Y0Y+S~}w#TvG
z{vGEjp-A@!UMw@h3<T0yH@$CDNk~cg@wxrrrAZKPHT6tzV!6o0%%-BJB$&`)u)xiR
z+3i76*8<PPYkw;iebzUN-66#IF_)DsXH$VQ$1bk3hR&i2+bwtOl!ST(RPK1PoY>eG
z6YmfuArbTBQtOW?2ODN>oN$@t<6PDzQ;us(oNHz`&v?=Pn{~6RQ43?y=EtwYL7g^+
z|Nos#JOYIonVF}})0)HR>2N~uOoPiKPs#tC4o6&sil!tSvr5ZS-Y4ypW#p8`@8(eO
z*GhTY;wA|Ri78W|4sd#R>R<b6wJB;Bf|!|uUL^SP&Umy#t3v5<Go$+^h1tF-n+_~Z
z`@rOOMfc|nhTKBM7#Gh@h~JYAevtc8;_UWfUcJjt#_T4A=R#*%KFP0F*u-&7PibQ1
zlo=_F_RBoihJ8?(zL2qCs(?U)=GS+!&~%oV@T1$LmwW$LrX(2^Uxn#nCno1+EOf3p
zbs(oqU~;fz!0rASf89)Cc`Q;+Y8(7ZHt}OG-Q*zsLn(;c=*Q2=Feh~Qt?=rZr!~df
zE4Z1_UE}d|r5RoZt{pwqMyJ?fI8#qB{PAS5d&ZdP(B`F>%42R0vE3;|J41QihR5=P
ziH*C=XEC~)7(Yl5o^RygrL^Rlsf$+HfwhGmOfjA;O6N{=T>8|IvrAAU1rlLSdBqDR
zXI#!P5Ylb>=ppDM%yUDDVYPys;G|n(N0tb+Dc;?dkZ|Dow<A+s3>dYwq4v#-{@H)!
zNlTZ4yM0tPbEx4#5r-ays2sxu*Eo12xC|aoO2|CepzQh3U(%U>!j*3>!Y$$|5c`_S
zQ~xji>Z{O{@sfERPqd8j0;7M48V9D96dYBWcysQXhdO^aneH8yuVVQBhSP;BSj_hc
zB6X<n{W!w4Eh7DZVxQ?F$qk{QGddWR)jqh0u8a8JzxalN8t+3-?Hz|pPcWVF<^1v6
zXn(yi#8xM5%YO^cE&tQ*nfP*FQinjKU&1l2kPr61YB&5iZZF}H*=zK`*SUG!mIW!M
z0(yIAyte!xE_db(4@7rTWBjhpI}Hu8C;1d7a#zNba_Q+YZ{u(X{Qv&`f9vwPnjatB
zotj+z`dB@=HXn4{kmC^6W^<x#`N^g?4-PWr-A&q2+<x%F1V^W)#va?7JHGu$)_Z(l
zvYmw`G|eVC-MYx)Sby<ce8c3Pp8pBYiw`WlP`+Vd){(92Ib9E4U48%4XZhtzY^<uk
zar<y>TFG~oG2+XjodVmUAIV?RTWo)0%YlmtKA>*Z@jgQrcjGg&Tqi0XDfclbmohLi
z+OPrCBmmXFhorWy=w#S*`n`PCg2hYaVix)<bpHAO|NX&c`SkO3zrVd*zf#kBl}^~=
z3qi~(O@^1x+V?zZz3Ta&E%8)BLc)ekNk4u~O}wtxr6nHIyrg2<&jpc?)}qrE%L}iV
zdc6Nkh|a2r68gb<@#6RQ|L^Vn{rB73cwSyJMMbwOQj9k>)|_>lBqcT@jW6!Np+gIn
z9Qp8C{P5r3JYFI)L5@#2tY%4Bh=Ut1PI{LA4xi#Y<9xqB^ku|nrFllodpoNB|NDMw
zs=S1R1Q!=eWQ0MnW!l+K;$c;Pj~4J8RAPU3_8^~&tK^)ylJjhnEvlz+ElX4rRC*o*
zjX@_aql(rhVa3=Pzu*6_Ke4ac<|Ie9iSnXX*VlvQm~QO*yLzc=dnc!~^zC|?uOc>%
z2af)KU?%x`-5JHnItmT_{Ra;<Jv=YpHd(!M*Ka01582xX_+dudO7p2Me9%?#<oA|B
z5l)M4t{qMP|NpnE{k3}6FHv^(<V3~&OH`Em=5E>Ha^Vu=GW`i#d1j_Cvd^2xla<Az
z9cExt<59Ha5#tf*qbE*4%xsdo$7CJBILl~)SI7Ppjy;n$CqCcxpi^kh(i6-5>(kEt
zDR_71{DTBNc6PI{us<ugHs?8M2O6C2SvX5a!NH|PQLF3W-|Dt+Wo`5AJv&00=8CN0
z4P684L^|;%xs*6r>3q2Caq-)R$Lni9h%sD9O1``6eDSlIkbs1hYoE58x^hl4uUK>7
zign0jRu%&zqn1{~7q7ux_G0zGC=cBPj#ZGjZ<_z6d22+QxXOu5KTa|C`1DMB`QOOk
z&)@(5?>{>mUsYvSP-NscM`OXNQ!LguAKI_ttnlJqQ+qvelFkDMmzIUQW<9*F-?q)H
zO<cd@lG7_G`9E6FqOeJw`-yVe!|hgw^3JH4nm_h2`Tzg_-F<(v9S`pIpa0L)?9<O*
zn~V(uGoBgvh!s3|dp02QWWh5Y=R!kfPjwNY`7Q?6Cme#9b*QT2TK$z@toLMovU|Jz
z4{$X&Ep(=F(|#VF|Ns8}E`M|5`=m+V`Q`ouhNS%a_V)h!`xR0PD}Kq&=q%l#%FXfP
zYj@)Ry0-fN5@G97GIEcWf3*0wjP2h+lNe1-aB^)bN&Ua*s?Lenu$~icE<8`R<;rYq
zjeGY0=jZ&RUHTIz=C6<c|M$QB;jP*AGiLmY-}#9<e_})F3nt%!3N>yHi~4yFujzuO
z&+7k!29Iv+p1z{mYy#uIrO#(*O@w;j<}(MYB!Q<DH9s_u|NsBr-HJ^<Qa|eZLwV-I
z`_Iq+zb@|Y)<usVKR#ZsD7L@;fBvs8>&^53)&2g)ALOL6@82)$J(F3PnGfII&ZDZ@
zA-pbrPXY&DTEsq?=WF|p^iO}R<uPIPjQ{`PrIEIQ!p81GJ+3s#zA(v$fuG;s-d?|U
z?fj&qdk2Hrn3<W4jSqK=|6dpTyZHaV`Rn63zt#O;w5&66o{ofs#J_*7aSK@dWp?f2
z*<Ww`<-2jrj)u?I3fFA(D|pMV7Yj+>PBIehB?fkTCOAx(5z1Ml&0_wm{^O(dD^_g3
zdw2f-zsU}Ex4wVRZ<DwGw?6Xo=VQJ0@%#V%{wZ9!N7VnPp;8bVGxOnIX`bk926uL~
zu5hoZk(f1G(yk_fgEKK=uav7{)MBCBv+{bWERfJ^n&-?V`O(F0tytK?1^zLKTignl
z_1M?{lePY~c+H!~({%0Myp244YDe84htBKz@e&r6lKpaji{IaqzoN!BZ;Iicga7~Y
z%(pX)*?sK8Bi2W8g`%Qtb#-kY3fkV}fa2{}Rhwn8MBM&k=^T26YubwwSRuYlYFz)c
zajoh9_9Jd3aS7h;{}nqI=<+*z7#jTf|Mm6%UtjNMXWRe%`Fa1=skd*;NIgAm{+l;E
zYu0q=Nh&KJ-dS8<St<GL?eFtv&ZMLrYsum`+@YnVz#vqqA7>D==S0CT6Q`K4X^yU~
zj-gW(jVCvDb}n4hl=v!y&CINAfdX4z-WOw)i##rD8Cn;w{a#QA4GRAG&R(of_}<!A
zEM&^ARyz0g_y7O$$CmIc>iPNOV}9Vp_Li3KLP9>l%RacdY$z@^HZV3mal$~c{M!Hj
z=?Q6RJ4#-@Kid6&p55Q__xI-K-*?N%WMrS<Dtdjr@t3cP%kJ#h2`YXsTu)C(NO<rd
z;l#<rh&?kN9_Md6<kYs$uFbxFgAVu6%Rd}~J3*xfs7`Xqs#6M=`7bFc7_OIOVmD#y
zg2lGgbvJJAjokNV_h#ehaPjTa#k+fW4sXx@|Kq1*pZx#kvW(aF_w&fe@SLCj-@5+a
z``6e1@2mg+{j++}E1Av*msHzitpna&{_(k8QLW9<h2_@2gtWAR($o`YQw2m*GqRF8
zx|4TQ2;LRga#F;n;ilJ7AAd-EIawe4VPAB!VG7@cs?8dv95H|X{@yRFUgzoe@57gu
z_m`;j-P~NMqccNN;-7uppXvSY|1Wm0zk4U5?w?Dcrn2(k&(C@M=l|PZ`}_Yy<^OT}
z{(OIYygp)c`H{x@{}S`;({~iLK0L_W7Vk0rh^MFFi4z8Yel~a5*VVOkaI{?sVN+CW
zD}60->9XWBy`!6^a5EW5uX5QI1c|n!#!zh@9*ypO3wk-PtggCn^TdlsN8>lA)kg3B
z#?E~BzvtwNMT>4IsMY`c<R3Th;h&%TSFQ5-{r!KE^Il(uN1OQNCAQ`M)>k@I@%h<*
z`+A8dPbGKk)U=hXJJI<6zhvFtq>67n56{WARexVlzQ?hlhv)G5`8;#xXas7g90`ay
zqo8ysAoOU#Un}Q){ljyuc`jb;n40_{LOm_kFbtm5x2i}OB+E`q`j@zbhwX{ji|zUU
z`DA`rrk~q>cX$2G+nY<ui?g$goR|Fj`}_Ravjy+(#pmU<DZBsc=r+E$yZvaayuFc!
zr{SF)ALqBWI=Zwxylc%?R@Sz^Uh>Au0_z787<+nn?(Z|)Q+aqpt#wI=2-A)o694`r
zUbve2;8@w?$QqmGIWo@4%7;6JFKn4Q=}<xni|K?JpTzxzMZraM(xEk`6D~2<BrvZ%
z<G|x)Tm3ElXxIC*XXDLse?`We*;u5yewmu^9>dcoe!PEsd;gj>|Mg;iDC?g6@aLyJ
zBcl#zPXF+1a~^ShBb(Y&1s@a+UtJAavGM1J<GDvm7joOxrBr-qNc>&a_Wk|E?aC`w
z8y6HjkdfHBEp3LyL`8*$hu?DB_Et-(`8e!cD`9!fuz!lnYTpZHcOk9mGjnwLnjJ)1
z>^%f$^Ex^>Sum`7{y?naT&rWE$A>wLosMgnd=+(fi!R~a61w)~oChm^o;-c#<>a6Z
z-b&S}iT)C=vv$3U5}9r@f9VpPyDEh(4<t*>gE*Ojudkl+Wv^IM;)8Ey75jg#IlEM!
zJF(;1obx`yfi2f=eDk-RZ{Kcs#`v+v?L{J9vT_<u4|cJ0e_}eW*mu&XX8x*0nJcs|
ziJbraOh_(!^0cF}p!v}|bFI0W0|Z>pPu38W7Cd<DnCOk0H+NfEiznpjN9<~24|{hY
zNbZMYe3yiQ{iWseiu8*4izT0Z@|iwo=9G4?dwdrIvZfd5a)pKlPm8R4*RpVfPHkMH
zPIsA%?$d4WcfB`HkO-JRkB9r9jd9-^?S1~2C#XoU1+qM-%sBh?*Zf~Qz9-KL=vucp
zH-2&DWf#q+2dhtVHRj5PJ=%4}G}TjP@-p|Y>cw-^Pt6R;1IPNExrSWL4k9kc@2=>4
zac!HKlH7sdn-x6yj}+hN94U)d+}pZlkyzKeWgZ>|8GI8g4;$G0`!MtLKhDDPmsY2H
zwoOg7T>kRNPpRYERd()L=l1fPq?hL50N>5Mi}&2$zRx{&btaP{``$;Fmj^j{74V&B
zPL^QfV7%@s(IjBNcKVRbVUzgCPx6;cN@QCc0vD`ucv+smN#{dtKv(Hs)|Y+F8cpjC
zZ=3l1gb~ZZ^tHV1HC1kF@@>J+zB1F0ugO8cb-tob22bnOEAtw!SyyO8^*xN_U}eAH
zlBD6I&~PB{k)D{;;zb*`&CPmt{><qo=X+&rVi!E)Hs)FIb;9%MZ^PUI)maiPZs#0H
zGW=-~<eKQGJ~{PSh0M3#3o`|p_WfTqF({+gaL)N7hN+&@1X*PG)`Z<KJFR+Iv-W&V
z-xP~;v0kTlP5%`ivr{$Zx(To6p*;87L81C<H|5sIYX-jhZns0vvElK9$`aYkQ&oTe
zWyt<sn7sP2;7%q`DmgRPmb<Aypnd+8gW@eEg}I{Bm#F^As_jbK-z0xDkp2DvO^xP`
z1rCSmo4drW%G@ctQ?&ni-Qk0Uk8@wSJeb&XFg!}YS9#T|jkoVMJ=hkfeS7V`n<ti?
zy!~iT_vxj!^KTzsc(!=s{q9X7t4_WAa??`c%;MXfOEtS$16y7TwN=O!p2>2Z^uXe8
z{^_Ta7A4pn-?lQFhxzi8D%Z;sp30fJO746)<%fKg72~_|G~-i~9#6Dbrjapg`ah4L
z{?^1lP03b&_w}!voEmYh`c?EvbM8v8kG9O&%hePha=czFD%@xmQ(<->M~y`MQuP;e
zXEkSJW(7paHZ2I~RNAHRB=>_*t!MD<hXodgAD;hd_x$7UKYQa$d1}k!jhBWL#`rCt
zesocfQnry`;*D1&KN?(@PT3WA^<%_bztBl57Oi*`antO!?y{Q}*>&5(d6g9-^gh<6
z88<bYWjnw4cB;XF(wXNadrjsYOXg!g6*T?FZ`-|cjt+qnQYF|9E3WIF@?__^pT5Gk
zj}_LLg#3Hbo3%CQQr_mc&9^s3O<Z<A+)#)6xTBYB&}<Pq;eGuAck<?(|MFXZQvAkC
zPn_O?Q{I%hySokr2(;}#a&(4-$PsO&j@G5;r@DWPWYLz^mRD$9+S=PXK~2DPhD86f
zIl7Y;w53kZt!HC<_UXKVu|d46%%1ZH?JS%%4+X3XJ6R*g+E^lUxv=M?i~g)0{#zcw
zUn?~3PY=?%K6QF`QS8#!KkPEQg6idsCgy+se5lYOA#%-ipXG1LN_Jm)Ez^G{EypZm
zH4mGPSccSN!_%tQwGS({ISLf)znphDLxJVFu~We6i!Yx<t`O}%yQ#;-*GKsFvYSh;
zF1o(>{_~$IeV#%kvbXE@e)<=FVM>c}pUvC}lm5z`Pu_j>^Nzb6k2lq7iRyG~ICb1#
zW;g5GrAa?4Oq6(vAL(>X75Kq;W<owREH-u>au8_Se|pxF4k;DEB$wH%Yn1OhZ20D?
zQ!C&dot>?%E0iPTa#GbP;(J%DQ^4%oS!Jsl=coRwPn-60(x>Gw*S~U=d>f*)^wj6)
zX~FBR?=Q^J6}$g=r?J$>dF~EZw=zv#=6^o;_S!Q>TW_zld0wP=x}m=3#H$}BIbIJd
zZRFfuTl4MnpJy!bVAm4O$A;T{Cq4Mh!uZ|#y7u9P3)MWEQU%+-RFzm=F?BuoHtop5
z*HZ5;y-m6uA?9@9y)cW<@*{?mIRjhXOWR4bok`<Job@uw^Q9@X)DN|1WoI3Jzm`(e
z@Ltum>QT)yjT`SRWh~}e$k{(oVF^5vb+oAS@t&r;cD>g!q=RO$e|VJ7E&TP{(_K6N
zF@o}T&n!c(LjgQ(`!6j|&}CJB8DbS89?qVwuKz%HPmB6TrM$gvnj&8EyEHVWKKg9=
zC2+~othNb?x8*14D^3eaysxoRD^?_H|9)A%r&+3HyDz3G+pJ%26z4P{HOt^ZMM&kk
z#}V_-dt}~wwLj<8{|H`f-ZSQn0?!IQc^d0(`pNWU>$GX99!IyG^<6r%Fv@tgVA~J7
zb*Dd{OmPx1oF6<*!&N|l#p7hbi4w296MK)}dA~z1^yuTp#|1kdRL=2h+p$!|_oUII
z8Ly|R7@pmf>a-zd)7?#Rn<M^Iyw6^Fn|Ifz?@zD%;{|zg%8Y|tO%4KW_8MZW2ff8F
zPj+Nqf48db-cs`qAD#FG%g%T|S!p$8!YtO^vR$rTt<Ow&j5nPAbLP)y{>(^Kt}B1_
zCO?sE+kW-c#)yummu8%2Z#yV{_=16uQ|in$TB65`_J`kIYc=0;u7%vg`7<x3zE8_O
zwMbTXrRl{FrgN%O!sgy|ZY$f}Kkx0X8)iX8I|W#Lmd{WNdi2{z?TO{Iqd9KV6j|oE
z&oYt<3>9JRa~64MvF195<D`ITIc8T(+nFyk@f<KYS-dX%*yD50FGgsuwSQ|FG3)96
ziIB0CJ2UojH6@6&$=5I1>&cZV(2_5(ha)_Hq4W!@T}t}O8AX#FJRF4`+Yc=~cYT7(
z1efCXRo4!Gp7U8s>Gk%r0vDo+_j*m@*?o6u=DR|h`+3hD6QacCyB|BPmaFZ$@O03(
zYD>A3Mz^NzdR8Vae`Wv4VCTaP99|#S)J{EA`m$Pv{ruwB5jqiiE49M3&!y)`-+oqd
z$mC>kgNbW4U;9Mu$%Z_KCY){4IP`ty*}A6VODtw29sYii85ART=GbyII|#JyH%Vg_
z(h=WysXMZAR?LCj9;<AfSx(<*IikxFn0VD!+0b5%{X}`gO1aMo&ptk5d)#hx?4sG(
zDa+nIyzuV3%1OPxxdwMOq^TyfAN=+=rAN1G<$Lv|Igv$Hf!9MskDXb*w*Jx2;}f6f
zZ09)mvE#X-s+S;($H}KoB8n`LZSRZilvSgWd2&yG^}1fT`=P~^uStFRJM^3ue7&Um
zr}zF$J-ctmU$MlwmVuX&-I)z)d<eM8b0~6TYhL@-RQ{1i?x=8ul)r+QSafcB_QDmu
zVPYrqx^yy=6qcNcNtCO(SMx0N=5Js9(|cFtm9G74qj5SrFy_&WV4XFk=XU09PJD6e
znp}IXlzE@W$$gvG{rmg<{D}(rWP@Yn?Vo!DJWd)Yu+;ibTWIPf=vpG1*%hS1dD%ta
z!5sV2Ig=PO-k$yQ^0xhoujW2AT(dx(umAs_%!0UZ{-!1cv98|b8^0LD9mtN1z8M{P
z^JYZG=1n;<K}T|=1UcFGC#<Zq<T?D|fAW!=b$yRl>&S<GJ!7<9<ByGa;)(v|k79@X
zCif;sU!NVcE%S}d<?q(djMK_r)(0K0*?O#WqR`S0HoN*%9J6cY=Pu6eRBCHfa5yFX
zf!TsD^VF_QIueJko1aN#p0oMIUB8$`8=ULG#cxvM6;1oPKh~wM!fXFd|NQ)aMdik)
z&x#_Vjgxx{|2CT&MMVC$um0BG+M4+L+y3+O{@Pc5`hR*l&-eGnd#aCr_^MsfQ&X^v
zr)jo@2K$F?#ww5Q?@3HZYg;C&rq;GSUos*>!mcg_H2d+j`@kx0J-rvNSi{0<c=^~C
z-CmM$SaF6YS0rSebkfiN|L<?l|8Ji6=i;qjU*F!|uc(-F^QO$NUlz0HPv|?jIUO|A
zSO56f{@LdL<M;iMH@f)e5$oak_Df{d9IYE{nvb41ap3sz1ILbicrSnW{QL`g8n%on
zdmetz&p6_sC3xn=L6-S;h8h|M8#Wl!{5}R66McW5XU&?HwTJC5wQdzUxC*jhXV2`7
zT+IPIuJ4=G-xdqZiT!JE@cu{sKdSYP?Bb%bES#){8=Dv2*tgJWVL5+_MazWh<ms-b
zJ_U+Ak=gWS-t+AyrpLTn_nt_&_dD)ZnLOJ{txTyarpq{E*#+-9{9pe$QMpfX5sOOa
zth;AsaT%~Bc4#yeb1+443$9u1<lItu%wpePf0@1){kxi))*qHvWHR3NmRnN3`JtE0
zh5VN>Us*tTG$}LpuT|!ya8c1`M_l?kIKnhF=g3aJzyE*z&rk2)-~T^d|NrlAZ}0#6
z`+vS&>W;$ZhnLmc%=6h6oaXw@mZErfeLT;mO*~9YJofcQd#a8uSbu)OnllP2EV5Hl
z5*zt?xj76zd@|aw(J^^$W8=f;{%!L13l>kauQRCmdFaDC*_4~cC2Bq0+*Vt-?kBmJ
z^E*9DR@4-=^YY%7nNiBMjLAe3(#hWwV-4C5Qp+N~g~LtIW!vVj4ELOD+R8urY>PYM
z)zLbkL*>}bLsL#a+O**^-(_35J^aF7_O5%q;Ps(pl`5R~4jr;oQqnwcu%yst`&${d
zT}?-7kMH^yJtHRXS)l`mi{KWv=PR`CGd+~a?cUTPU~o)*Z)BT|d!Rz!LXjz--YqZQ
zAa<;xfH{FDDtVrw>avs9YkNQ5JGQCj`@y%<6`Qtt1RFfN`u=m}_iuLHH4%Hk8DY&V
zTabfW<tM3j+qd>EHEW8fZkcoZabz`bcS7RA)-tYxN*9xyx_EczJWQT7(^F7_<6-UL
zmWr)!qdCM}e|$=qp>V+FwTbKXm$mzU{fi3s=zP54{(1ZINO{4F`bz_*J&?KGw#+eB
zxi67t6HjRB-rPmsTnY~hF!VHDn|4m6bFZ0bY(SLA0~xPG8AF~4LYG@UyUa_DT$A`p
z|Cx=RlWR|;%J1g26YeO~IRAY-Bfezi>w92l@0qn1RA9FpKV%&sl2hBcSy-A|K1N5U
zhCy1|Rfl!!1mTZ$K@}{DYAc0q<gIGuFS{YRTH}vJ-QVunH_Kf)cKO{h`xMi+SkK$m
zSAtEad*|hPg~v0?Sr}Uy1QK|rG!;9yJwC8WpshnmIN6Kyy6MyY*PnhRZ+!ek{&?aU
zWyJ*|pL=$>U0(2hV{SzF7Tr}`UX20;|Fm8FL?oQ2Df&)S`LIX7_fNU}rRbMtaXY<0
zL;3&z@0n%K-IO44q&{oCl`2PLvjxXa(Q|oqKeqj1XzYH>&864I^=pfW=3nWiydC}p
zcRt#buh!W-<&meDu*dbz#~y(`x!T7chfFK7@(f*QT6)&vQtAJIw$!`-njH@qJox>l
z%z|&5@2pMtm79WG1U-wqGtO5VJ~8iM-`|}1Xx9$CJHHh!r)*X{-<f<aeao`<H}g21
z`PlWuYvk9)n=9Kpv7Ardd(0r9Ygy!(e|8VDXPxm)s(ZR=%Vfx6@GY|qxmp}}T<^c&
z5)*4u=s4O?!ML~nt*DGNx8p%S>E)7hb>}#BvnH1RXLWj@{_wxbdT!&qRiW9t-<L`B
zWp46%o14A6uP`NYM!L=U+{piDdlU{x)XcYW&t6h;rXzV(*RHf!PUo&ey`qX8i{`gJ
zo3voc63yE8XWnd5Ih`TZqv#pXmAiXyXMN{od)xCheu8yxo92I55+-kD>jYYk`~Ux*
z*_PbR0Rk=dDitM7WpZ=HxXk|jzjiKYgNR6U_wq-K`$Q^PngxP9mfcI;om!UmtuWMR
z$&}Mdtrl&L*S=mXb&ou?ZE5C=^snFc=l)la+;whW@~j_rmt7`JNR1HdbDk%B{M1{X
zrv7Qwh4)ORxz4)uox@S$Q6;GLw=3zk2<xo2M1wwao~c2RRe>#;t5U8{H@LwaF7@$o
zYPHz2`>CMeKTzx&a<v$Uu(HQ^2y(jyOMKy5X|sRn1Hm^sqPaPXSdu5V-erx{j{m>w
z=lt+9uiJT^H=Rmd`+irlO#im8mtF2ZmN_jX_K!J*<KRBw<5N^$cNt1FPtkCV4Hn6f
z_Nu*m+1P8>wS%IkiZ4ykNtRflwPZ==1R=#4&%L9+`TdC&yfjNiu}gp{JMfkN$H@)9
z<sWH*_8tEJ@8l^n!EoL2{{MzX|L*VqfB*LO|9vt)|GyMI{Pi`Dj}MQ$z0>lDm(z@X
z{8(`Mu8*%|Pme@JrKF3iq>ip63yVbE|1<%ywr`Fv_<#IdDE;ZmRmmAMCH3OdethY4
zjG5Ee+xzfM<cnoG9<}ae(JY(%*)N@V@GR}e@98(#*!3E&Z9Cv&HDe`c*W~~IM`qh{
zwIuMk%vbthx+tP%X=??mV1HirW7RDmnf@tcC~xH4wfk(7!ZVJ;2AusTQ~Pp-1z+yE
zTexCXt<2JOsj->SE;E+tzxe%P&N&aEXEPTmoaba(vs5Q^rA6J(%eN#%RZbVg{+efU
zb=M~Yfi@G~ef`hx-ZVOvp<flbVVXt|QyFv9-_&^f<1f>{LDpe5srBXG|7U0j8Zi8R
zef|IHZ-1))|NDQi`Tzevi4|WscHI%bv~{t+_2cXMY;kdHmX>Yj=S%A7NSrw%@#BZY
zks}go)+`8J-@xE2);gIhB4WY9l}lJ+Ltni)bKuOGf`?3p*T?hd=zxa%K74j&zZntP
z8KuD)&&w$}SyVK2!nCx6#6(a){S<zBfiI?ENyef3N|1rtBXc16Q2ud~(bU$p993+#
z4eHmmBPyFWZYm5An3-dk*sYs#Gs#@t<D<g-`SWcY_V6D&VbYpAIg#VF)?*DN9q!`?
zOII@eXOs+l8|@wW$Trve`o!m6Zqo!0RbTtJ|7+5gRRw&>FMj{f2wdJ!Y{L85XV$96
zmC18mZ>>qazcQOwL5<V(U}atL@2bn6)w8EVQc=>+KR@rUkN=;4@6Y#VXa65=|G%&9
z@BcHBpel6o`~Uv}6VG*eFhuemJ$Ft){rHEk+RYOMn`cTkFHmS+q0u~5lv!N-@csQE
zua{k7>gWJ9QWO*pB%~cH_^MUH^Y%sHi$zb{`6Z7Wk(g)Kx~!r8|AK$5DUm;}*?O$z
z+cZORlfS!1hoetVV}F0+RMDmWjobof4p)dQm?{sckk8D4R224ob6PEJK1exk*eYoK
z@vn9KfiM9U-M^0O0}NZ8f?7-(Csej1&i+|eo_Dw)<mx>Wk)!uY-6M^BuYcSpuzz;X
zWm&}zrD?4us;58H_9&j;dH?3N@2sN#l{NLZ%vV{X{kY?r40~y>;Thv~-BY&c&U3&2
z*dqL$_4H$2iMmnAx``cASQxzok6Z8^OL+Ua@L6-5@XP5p9)o+lXXY4kwL0*)$o~*b
zR^HUw+dB0H<F|k28>X}7+BDpXc=<2sa%YmVg*ykcq?++h%k9?BcfXsVX3ukrb=LFK
zA&;dwy02BQ>dIZ5yHPCpMDdzX-EW0yEJrr-t<98h-}|>~#j_XHT~2$A1h0uEn?&x=
zaS@Dc<4E<K^rCw4E;Zk+8$P=fu-tJmx7fMJ{LN2rU+Bv0)m#k@0<Qi7iG~s(26}dN
z4hBE}-&Ha*)a|g}pta)QL6xix6`xx=i)R$i-F0tP+S^@cCE6ShJP~ewn0iihYuYRi
zp;MDI4t>|;YI0aNVgHi_oJ%{B*DPH%BmG$57NP9Gu5GG3FRL>z>T*XV^F*rYvRqg7
z{b=(^M&RRvkFU#~Tza~becI{D1aLR!%<Q#X4F)2u_KrE+w*_RlvQKO8`)@qsn(O9J
zo^wT$S0o6YH<mfa|NLH&#J4G*HW{23+G#H#?SEqCy5!p#(ZA*`bJR>aVrjYjhqHMl
zck+>0$ttQ_ragIR8+vrtBGbz)-+Pp=EfTrrWzNa?d`0%p3hgfOO$V<g9kSJxTTu*d
zXk3|N$km`A(Q3cJp@1Vtcw$@F@m>Ep_cXU(5L?J5+q7cag7T*Nio@q0*RJPIJU>%n
z;-;y|`%B#Rt_fQC_{63J9<PVCm#@6I_TR5C#OYjA(K3;1qMFi&V=I>@E_i=&))a%b
zDFuDGhpSr><~2_1nm2iM`rdz%;rH*ngj;?<L4c)wLIaEE;#M|o?UT%Z{^#?s1aYox
z5xK|hGqv_$+39;a=GV27Uqso**05js8>#(QH{B^A){-wssPDt;`AxF(7!8BCqAp8)
z@7=0#bXy|N^;W|<>c7u@elVvbd~KxIp7dEPdw7p~U3b{A2W)FnN<u@~nwL8h*!fsk
z7Id$Ym6bFxkziqw`0`b9&05L2KM4uR&3<$F`9D16-jZDOn2lLo9W*;7Z)Z?YU|>^o
zLP7QDhc}YU($dWh3?+A8&$kDyM6j<jIC08Je4%3>s{-4CtI90XOCAV9dWuOcbEX<A
zXg1HD-8gyj!}IcN($Z|XxonJ#Y_YLy;o)uX?=MVlDq%RWZ&~SSwoQj09A<Bmx0h&X
zkysa#_~3DJ#s7H^zvsW0q*Pnm_WQf!k)x9T{-+Cww5*G5)oE1Qly`KF_Ce9V5lb)J
zfD{Q*8>Y!wTbE2ba-zf0zu(cJ!O^?>;9XfOC6}Pz-4YF@KRCku``s9VQjHA_Ykn}?
zmH+*iy?Kes;kUPW{`}#YZ*Sbu>3Dsi<0=V{9|zc9cCzg7;xpnBf|@<!n4GY1$&ujL
zxsJZPx7Kdo)WjxV8qD@@oz2r%o$Md(@JH^`Qag6w^l8xEBF&QrjvZUD?%aVBCloX|
zzS^byw_9@3b?O(dGfoyJkuVdF96ZgvHpbL+VRq%2vyz~hssHH$0t;s8p1A0fve9vg
z&P|3%b_%&1Y;0_Hb{V`(mb;`bWw!fh3*A3)fcb$dpJ66bFfTY}r5cVLJbn7Zd->)D
zhUPv#W^wW67MA8t&Jtc0(dB*``92r62(rDba`+ZuVBp~X^TtiV_LBwyd&)Q~WlgVL
z<ZC(zSqr$R{8596ab%?Am#+&C+qt>%Xle1(|1&5k3d)@*zpkRyeCq^eZhu>+b~h`P
zsSeYw8yf^HlymeJII?B+hE*vk(4}FEvL7{w%(JgIl8`jy;4=E~(eTAfLkUU8*{?%h
z&PzCEvM0S?(kCOgdd)+R?S3**Ef-hB_T=cy6BO*w0efaq{G*0B<=_5VgFGPO5k33h
z9hqHvZCbv(F_*cH9RGD-8NU{j(Vd=|GLXu1QShULw6uiO)E^(49mA)CmP@s^I>yRY
z9k5hZeptHs2t(FU?YC9hUl^BI$(<L|hXntk;71KQ+@T>U6DB4mBqRicr5PAE%k6V}
zxGR!F!PJFgm3wf5<qi`bhHKN<H$ffd{^-GjhPd_x3l4xHR{wlL)+q(;(;uGmH%|}<
zWcPc(Zql`O;S9SEC3g!okKJ-<2H6DKg<JWkA*SfS!GjAH9XfF8l!Dr^gq$-A*7C;2
z`#n#sk6`coxcc}}*)1=bnF@N_l};~WJo5!IhOj92QG-tPpTE+(uFq_5KX{_!;0@5M
zI9p!ct}u;^mkgWd8=X*|EzwXvW$u-C=l}nQt~*<l2-?dG8KiJ@Z*QDFy>aGD$N2dN
zZ%9;CO8*NE?_@c0L?i9UWRq2fTe$d*(z-vKfF^*@M+r$ydE3GQ69t5le*B&7=-z(t
zhJ@8+4NXs0?T*ZlNAkB7F8L@;esG}a{|&R<Nug})5tCVqR)gkq|NmbU3bOde*KS8I
zF00Ec79LE<IisM-Ik#X@%|f=7cX^dcCuV(WSk7soU{Tqk9r@rR)K<?&4R78(dY(RE
zQqqs_{f=SN4&IpI7%RK0SFngfO0}U(lX<$=mYtmkEyLVisW7Cya+LlI3flkw7u7v#
zc$1lsmNsEJXqmx-$H@j}X%{Xxm-(-_bmaj53ZdeN%f}!12xl<tuP)eXAdzUEuK=1n
z1w~5HqlWp1m@PNF{$J0QoXp0|-1h%JC=)(;x-x-PvX{-rbjo%82<_>NJ{m{F`6EP=
zuYxvU{{O!y?NNh|;s4+4cPD&)%H6!t@XIN~vNASFNv$<vYWk@OvNsfx=Oh;}-LeD~
c9smC`Gq{>WCzfxqH3hlZ)78&qol`;+09#ZQZ~y=R

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-1.png
new file mode 100644
index 0000000000000000000000000000000000000000..6f8b65451cc08a463e4305ddc4be0dbe2879fae9
GIT binary patch
literal 18058
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4qj3-okx45^rtlpw*nn1P*v
zfq~)we<{|*4C<pylyG2bm^g9635^A;kZ`$j!JDtyfXDTsRbf{Xm!#6h|8ET$I8T24
zy!^DoUTcf*56>>$({aM~`q#Tr)_v9Y?N1wZD=^(Q%StVGiFhw8xVyb0B}#MZOl6J&
z!M*F=SDunsTdt;+mO0IVVV%?}z9XA2)xTi}Ma7v*o_S1$0<0HSZM&Vx%)LP*rBn91
zz0`_KHvZJzb!I!AmG&y{*?&^7>3_|WDqcmIZ@)E<y_ZOT@5`a+w!$oVpPu;pn48<!
zZhXEu&u_=wPrFuaeOB<`@kEzbfA7ihi$2VJl=|>)U)B1p+os&w_Vs!2{d`|e&A@X4
zj+#-cyLcRW9kaC-S@z#!Z17<G(r3uv@Ku~4G__63>*Dq7DNOGx=cy;(kVsQg5>QN!
z4KMAR-nv#bc}3V1Cc#hhp3SUw_i}MLcTvKGP4J@0#DAxE*@F|;kx2*h4jBlz953Wh
z>=NGbBtlQ&<A3=hf{iUqLO&9$=e<<Sy!y|<I;z)gck%7-<pp2<-rAYATbIAR|ME)N
zwucpKR<FA8er~ZPOX=K|TFYFrLq+DqvkE9Ikl)vG|N2q|PbG!qeb%RIa%UJjC3y7&
zYHF(b2^Y5=naS*V?1V(5c6R8rh_xoZ9E|r~f4%hE_r0XDXPD@0x7_YU0X@$?UU)k#
zcv0~itBDfsbI-5J{<ha6R&P(-*4vsaci&66=a^O42`cYtXJNRfzvIl8Khy0PL2<Na
z%0{jh1D>}3Nvy0NSuTDp*-<WV|G%M3$OT^+ku9%G%|pCjzjbPR;_syJ^Mr80lgc^E
z93$&Cy$Kb`zi<EO2Dh^n-^rhE|D5{yDnR(ajBh6&99BB);L;w}b>*G@Q?0{mFS)!r
ze@?8rr76g8t>{X*Nw4QE5-7TT;`5Tn7i=zDc%9SiT9J97)I(^C?kg#m6<R#Z*F<ak
zGY{=^=Fs7msQz;N4>;VnOtIx^b`WUYuY1Tz;b_RpoS>y*U;Zt>z^w6x(Pg87vF4xX
zSe?0_KBiTCuzTia;W97s+o{j_fqPDWQf@kNxH^o(&u2;3!S@_Te)_xAEV1g`v2|Tc
z`TnXMn_MS$Jht4fomRDH!{Zlsn3mq;`Mak{@?4_Z{l~Vk?zb0t#R@a|I)@tav>dQl
zbjsvz_+6uVu!|OXK6>yl(cEZ~kwHLV`Sq~Chl0Y7Q<9sXRlB*l9Q^SAzrEYB`VT@(
z?9!?`4UCjr9<U{?jW;;Ozs00+|AZgM8M=+m9BEm_n$6U-%wUH51GYn3{~Xr1y+D-9
z=*NF~tsj4_53i5s$-ifCXV1wGKMW85{LFKHo?*=HV;|njAKsoHVG_g*$;*q<9wnrt
zJb03F@!GV_8v{RmHl8@`*`z5?8m4Ud;KeAnd%}zMrTiIYPC*S}`xU(SwrpydC}=60
zpvL{iQerDl!U55<Gd$T(mT5FiXIT+upzxC^`(Rs4*Mbny4=3FEN;!VNk8k_`Uovig
zTE(Z9hmTy_R)<UK#U|~jo!vNb;=|wNTE|pG;0~EEUA(C5T2cA+&6|UtJxfbTYh#m9
z`S)OB_r!n;lbDP;BftJR?jiqwq7d)2jE0+iCw91)Xl_v9TOx9R!TZh&wPZ2574Lqw
zXGujxr6(jfgy<DCObb(ZvcI0Er-z4M-e^z#`3;52ho7G2`TpK`PxWzzmt29cQ1E+{
zkeX^{G>M<v`0GbQL9w)lkCNPYMcA$sG0hX%u+8$p(s%Y1GkaooDoi!W@MPJokhW&=
zHs^<3k{3i4FXM5sY&`5!F{|Nv3C{|-Bai%gLi$t-4HC4yADEuoU(fUZpJB}2Qw862
z4xgONbG*-RPp$XXXB?1Z-~>tr#>T;?&z5f782Ip6n*_&O^Fxmw=&vbk2rG2Bc|w=d
z&8?!FC9oo)By@sw$&00|Zo;D3D;t(eHm){Mh&ZsolKG(9s;YkrRRx#aVh`Mzpldnd
zX2wdEro4M87KKfThuPYs%_Z#WmN@NagIK9ixM70<C)cK*KO>JHiFozeyQsjx*y!K?
z|MMA*JQ(u(4rYnzsIss*t@btW7N63?;cvL%%4hk|SN1vwA9`x@NG#=J&MG=M)o#Ny
z!;UYFAvYdqz54g}e}BxS1tEGF4B49!9JHJyBqaX*Oa1Y={oxtOw%pqie6mRuzotC=
zobI(~IwQ;(D^_%{u|>JKN3UANWo&eaol8<;!B-BOJr}fCx43)|*Zlg&)l2NZVTe=v
z@(DX7rgB8R_@90tOk6<tyTnsQW;1OA?s@emwiy^X1xX*;nh_gWq2;J&GeKMYPbg2(
zqlOLt|0e(V(wX?!txeWiLN7My$A`v;-}8+gi-YY~GA=YQGV16qK6=dR=<(YpPk-LL
z!N9SnX*qvIm|IrLfyZ*{PPvN~dH!|J-K#WRiD!i?ca)0Z0#V}~Tnu+Qn;F}sDy)(Z
z_!!=p^wQqVjV&U`!Skj^T(g_8(Sgv-3px+GDtJEmRoZ4*EYT;ESn+4Z87)ZgD;XDZ
zaDH@gzrA$XtyL>Hj13G751ryYQKP9-|KGjq`hK1ZhyLs5tYBWNb#Q9lzsF*Z$qGrV
zrm}yM9Wt36yBY8J3ZMDXs4l5E-MJ&BWy%ymzo!2Ci$$e*I?{9&r6nGirncc*%!M5v
zLO#8hKYX~IXLY!-PVA`<e=IFG@__w4XXa_%W&@tK{r9JE?qXoqo4ZhGaY2FwTQdJ6
zxgOpx*W3b^oLGN<qw@2QKioN%z1*tvzOcrNt!v|oiIRo?7@qdsbPQs*max!BoxQv8
z-%<G`S2VwS?0ezJtJ&zOD|&2K`@BQRf@UYzDNdR4!SBKJh-u1qB-eZ^S6V5g@WJ5i
zJI)^mCX|Myh8umK=EvH+?qr~@?XP_&j%>-AaI-h-n1R9hFSf5wZTtHt<{P+nnKEOq
zV1oirYy5{B2dv-Pwl63VlbT*+RAwmlvDHIe)?TsY`@}mRrTgYA58C$G=3DJHVWwm4
z0dMAI>UMu}^Qn96;5zBJ+}Y0VRccwu&z#~D6yLAeFsJmEkK#VRg~~OGZVOI{KL{1G
zlP-3<b1*$WuXOXOvQIx8|IG^vW$@K3QjlBu`JuxRGoEAXij8}8xU-J#dbI1pt^NBq
zftx2&W*G7{DDb$}zdrZs)eF%(bN6z5{}goZ(A^gSb)Od-UTkd?$vFL5|Dn0e;~Uql
z{4ZZ9?P#0yCVlPZ<tG0p{p#ottuflk*tl{ILtfdEB8F#jH4Q4QP8piBd(LrZ8=J6g
zUhn%OB&ngg(QajuLS|ah;w_bX_P!_x=IC0gHRq&Se%^-%{CA#2l}~9(J-9D9c4Mx6
zoueK&il>C%<~*RlW5cTXWxa7p(?84R2mhQSUa&2bpC!KZ!mpVQ9iH#>ewxeN-gc$x
z$)4pYAI-Itnwj<`AAMaX=%gN0!!6HqH}MO*f)L}2w#hlNs!ToCA|_4H3`{ZJ?ri1D
zvR>r!i_Lv|Hd#CljyGXwm2g}mA7FMVkKgR+*}GEe>*6LAI)DCh%iBHXbyrY9YzFW1
z&i?mopti!4u#LP26nKggUNYyzTwvDv$Xc!c);;zYQ;EaE!o-Z33PCKMJ#Xb7e25E*
zD!%6w@k_h1sQ0t9UDvnO4CgM{Wlu1kyY)%q^lLp25*#9KZJ5p1YSp{BMr7^kbNh69
z<1cGp5ZSzHbLG0^*UDM$Ete0nPEy<&wpiYt>CiD(0p)M9&pRI<DDA!KdyN<Dn3-V5
zu*B<3*doF+L2I`G!{W^@R<o8USg@~iYG-_@YyaQvkAizo^ttDSZ_-vz`x*0w|8~X&
z6Q3hZjJnYcM<+Na8K*tmy7GWZrjmPkUUz`gmJN){ue`U_Ik}%{rEqt>pxnnh8h@DP
zHKl)Z*R5Rr`hQH#1n;_NFD=)tuXbIT<$byroQzt+9r+F@aAcnj-*$KJ?7tcR_rH6f
zaN>*`hav9`kIulq%XiFue(krOZA!CLX2|3lO1&bPFLe(UUjD(pyg|85vr4?=&z6j3
z?n}9L7<0B={U9oL=4pkZO6a6o?g>)_WM>PCeX!gn@w52ht;Ss*uddWGY-xLasPMp>
z_>a<{U{EqE)X?r`Ws5p-sz{;VY>`-AM?~XN&WbMz+dTwaHFSamMR|?H<!U@qg}&`@
z>CN<En^nceqZifiAduhPM}kY>RKR2%8v%pK;yUsI6FDvfF*7so;t11=F}yOh2UPJ!
zMe$s__Ci!R3{uT2nHAo+p&=o`VQge{U=piMr-1@*pmPiZzx9+4PN&YExGKOLb;`l(
zN5o~uS@SOlF{Uvk9SG>pR9JVVEtD}V^+b#h8}qWy7bbBtzdO;EUfW^O;u}=u<Qx>l
zV`s+`6~(h=&5p;j8yg!NA+?N>Rv`yx;q4nQIJk=!@EREz7#JA+YxdcEFzd_z4S|6!
zlHD>98zwa{nx-yj)m(StFi-6tPsW?GD~jy@nY4uRUOUpIE|~nvHBou0#tB!oho6`u
zdFON+$S%IoV9?VeQBxzq!cwq%B9eV`EFvN#a&tod{F!p|rldrHPDVqoyzzp*4rMEj
z4^1MS-Rj~-55!UqOl&SpjcE!qNJ@UZQT%e{jxO^Uy(1=!t=-z14?GPcSU;{vOilgp
zJ>9@KIU)V{^F(Oq&9V6R?{9vN&f!C!#KmrYU^{LfVwM!go4JX1(=mBXh6$G}w(U@i
zka(bfd!yE2o%Rh{#~!n-c`6ph{83&1kKfS=3fjLWZdIO~(bmyn@Zz;`LD8dPNpP8=
zG{?fkwDs(nCpT|OPIz`Ikc<CEm9EFEgp@zr{u&<^25qV1)jZguz4k#kr})7v3&WSp
zt3Mm`s>ZQI%J^%#DojmK6K7+qS;?-~%~nw<IcKiqlBI<^nveo!f<;8c3=@+nJiM(F
z#Ig+)Omzy*&id#2^q=#p{t18i<!>C|QD`!3Ide45E_w1HttQb2>r+omJ2YRX;MGwd
z_K4d{B1DZ7yv=!dc&w~=`ucuE$w7l8rYIt!W8>!k?y;$##aEtf49YPu_~W`Z=0Xr#
zQL<R&n>InUPx7HMwjCUtkF<<COt@AX#kf86oH*mrk%;#D?|FD6M0uH+#muf%u}oL=
z^W&+hvG6W|nmxfnMYS>{<Va-1gQZ97BOW%EzGY34=L&k-bjiw*i{puaqh{d0zrVl#
z{(isyzkxHCo>Nw3gLZ8|MAw9@m-T0OKK!?5b~`a)noGgPjjP=KJb8F{goJo@?Xp<@
z37S3)YSt}MO3l*Z@s{k_u(N2zPYI<DsdgO~T6*gh$_xG}b}Ae=Kfm6t_Se3oqerX1
z-zzNKc;eJ2DXFf^EUu3qH!is5@`F+NSPd65+jK$Yxq>EQmmO@HcBBaA{r>s+{^R5S
z<!pX@{`U6%{`&tbR~fx{VX$HI|MVoKW2y#^B_t&z0+xjsEDwLsRG?d+r>xw($EJCX
z470fSH(Opv034cbF=K{=sR@gv#lM5g{2xwU@&DoEr7OI-9t5lOY*=!*A*A!bYXvqH
zHj~af`~T;io|b)i`TRYVpLvy)58vN^fo;vEO%40^XT{8!Hf@>$H#hT{vNJ*)kzC<l
zThx_0{c9>1)|GG?|FS>)`uhKEIX~^Hzisap|L-s@Y{N#wf+C|o|Ia^ocu++0WJcaO
z1x?R6sXY;C4<3jqY9}Wgu*>}Z-=0}pySbhH@G{>U!tCHwd}zLfL`o{_jG2-XCb7<G
z+;@p(5%a0!l!miMoaP=paG+sV9>3zh|NsA&zu%`9SyXqdNBI1_g*wy!{h$Bv++6;>
zdzVkw+bboybmEjHYuBASf9Ozy;NJ=^PPUwrtl3-&G4mc9%+dJp@$vto-TQrJe){^%
zcgk0h{dJDd4}`7%SN@F0IYuTqORKqsg;`ke@Z=LOna>~5Va{r7ToxNR(b1*l;ZbfK
z>tB#~KlGfXuux7|xH2s4$t#9pUG+uDpZ@=kS9uqobK2>@^285~0&U61<-UEBbMuxw
z@!)88{$%y|j^4Y5_5Zf+um7)jcB|0?4K0psY#lkd3A$Y$%+&%e{Qu9hzs~T@y}!z8
z&tgNbs&ARnm~^DRt@^t}Rn<TJh!6j-YCno;zt!oQwm{S2*`lpBGiFNm^lp^qg9Kf8
z5eMhTQz!nkuKk<hkd<O!bf9&?wLk7cKhGa%uqo=hBiuMe^2F5B`>(Ix?=z?3$K&Jj
zKb#+ae_#Ln{C+(ho08w(e$SDSwY1Dz;?uKrZa14jGM}%^>Z&yjEvueCJ-t7Ad)?cY
zKP#S}lTVtKQ2X<cLA6=)O3mgD4rcDc*P0TBV$+NiroNugJDFqofA?@Tl_LiZH~&=-
zfF{H6A`XrX2fie{lW@@YmXQ2^z^i-zhIL$Nf=v3B3wr+l|1WP>^Zw%Eeiv3|=EK+5
z*YDbWzPx<?{>snHL4Oj(R8GXO&5&C2lvheYqi@<O1ICy?|GUNO&2xUtn)U13G2TT%
zY%Dev2MlVhGLLV2@T$ikVT#O*hM6-R<L5iZ$aI4`sQ>>Tnr?CANQ;Y0L_tngs|wSU
zciL;%yCoD|edRNh{!2*w`}_Cz_v8Kd`Q`ViE2>CHi0SQ#jI4Zgb@luDe+yJCEFAhg
z*09=19Z=<C&rmkp!KEni`v3p+@&9$>{_I@-TzO^oK1m6QvNs7kO1o2Hc`ROuH%;x^
zXZOuh3>pG6K^ak0gk|T>i_g3SW-m-k<y3k!F^ID$>%b(n2ZpB(9V&QwsylwahxOeT
zpP%24-fowbX<7bOs%VvyAMXsMBg;AeXfjOa)X3Ub|9^ks<LkQ7`|j@bKDR7QO|5Nh
zwCA<ftv3#+cAWY9bFyQMjJG5+BuzxF-MG=v+uQix@9*<xN^ZJ2mGRn@hILb0O{YHm
z?W?eO#RpM6o)yi_{{uZ;wtjkZQ#~!s&Ex*miVv@^?{DSKFDx|F)_z@FXqecuY=Nsp
zk$Cfoj^k5ZE}T+H`N|{U|M&Oz|7U008>iJ+mA_N>oaZ#-t9;cOe#2TZjsv&k59YHf
z=VWTh%Qvsk5PJ&^h0LG*^7S)jL`X^+^7|eLGWhfVzj*ZAPvV}2QQoY(n7<wGkAL*&
z&J8wa&SwAlb@_LG6ddnU&%ggqWy;R~i7r|!hKpAHJl?RhEy>|h)70*Hd2;suwq{?y
z{`z`-?O(6H{Sp!q`|8{CDpffjtPCj7Y%_3i-S|Eonqy~Lu&~?^6=9iZwm#{Y$*mf$
zs7pVaSN+ppWt7pfaiihM)1M~?YC9iT8!bOwZ*TQOrWC`k^;b$7EYr?6NKW)<s<`sN
zi~YpmDvpBZ=ib-;-dF$cPk&<!sDdyzKPeq^(QO{bCdmZPmd;McP*Lyakl}$tGcA7n
zv^;ubfnbl-@2;>Gg{Z~<|NnQ5y}>yBZ}Td-iIZl`{O{@gd4d?5$AkO!ZI_qde{=6|
z&x;o>-t7MaE(?U2X1rv`-r00XZ(s5A?>me4*G-CC>cA*3&nzyU8TCMwy^eK8<j4E=
zY^kYie0(K^&`=IvTl4c1x7h0subB5XcE(IKF#6{z#eP`FZ-X%l%Z;Z`et2-RseJhL
zwLf<Eyu17VODg*w_-s<-_P@XPbi<N2@>y92Zt)+QJnx^e!Jqnnf6l+YK1EJJOQl71
z8wb-9$DlQ`!ak85)8y?p%(()!aHd7w-(N@1oaw5P<1JY?b)w+J--;($w{C6Rd!0+z
zo$q@+56}N8Q-AvUcFy)Yz*hI~Px;$h$CcgV61%_FugHmU<zDB~uw3ZitP=}*($D`3
z2wHAnV6b6>fri$k#e)01S$lcePi!jCJ#&WV{e9yEu~N9LDylzIQX&kJzT3abkVvyS
zkdyf5s9LK=duqy#yLV4=aY{URkYH8*PWRp?hH#(cWfC!uKA1FWo^Cky<KJI-Y3YfF
zKi_EFWi_*td9!U}_I6#VfRH0W-SFfhAvtsA%=*P;h6@6_V@@kfRXWjn%08sU@?q1Y
zl9-q$5fL6Y;#%1vB4)h1`}@<Ur{y_@y{1-$SGji_nV71`u-T`n>(8&R{f&){lP51a
zl9+l_LEU{>+@HysJUlOyUkE6uwJjBcW>ZPk`}_Y_{rn{UWQS<ZO2)Y^zN!UZ+8ZD8
zX9+gljQOjse}3}Bh3;j)e*F8peu2X6#qRM1+a(NEv#oi=v*R_lO{nOmb@lQ4|4p8d
z@aVC!eIW;@Aq(roJ5sI8O|GmqI~DwI)cx&RdKT(2N%{Bp>yy*c+UDN)E`B@HXM&hU
z%>j|k>$@Wp6u6ZZP3U}CU$tx9ym=gz2S5J(9bZ=`=I57|bFsg>SDp7Dt7L&Xx6R78
z^LJWmtE#rI)!k@)X#-P?E9<&X%t~Kh2u?GDR^&4+R8)4but)?p^H1WKpgp~whv$PF
zR}|NQsDuYu6aQSkD43IBpsMe}!W#Mi&(Gy+&o);T7|sh~37_?&dm&FY*N^tIH7ly(
z;@FIh!}Rpr{zN+!Fp7z_wXw0q#T9)~Vm`BoDN-QU*^HU_@N)l%ZDG)my!<TT;lrQJ
zA_DdQ|GU4Nx?rl<i(dYwm5h@gK2?>MU3=_^OA(WxczSt(z|Su)&5u3fj4|afY~`Al
z(s<I<AR^*lals#ct|W0m#hueq($YM<k}h0U4wbi3hzLnIRaBfTxeDq%^K&16d@Nt`
z@0C5<mCouXe>G364Pk4m-;l(dJ-5m9>z~AgAE(d!S-<_dk&@HQC7+rnPmc8LjCOCI
zK7YRC;yKRep1yEZSZKshfAh2aVT~3eN5^9m1j<rV)AI6!Yz`eb@Ziaj3)kEZ)o_J1
zT$|m>t;Bq$kH^}28m}KT>0f^K<KHvhqT^rn)3!EEvHPdA`QtL?Cs&&9{Sy#Qo^5bY
z$t<jO=FA%`i&mVNIzvNUyL(%|xUu7dRTdkx4*R%8WJ|vJ%BTI>UQ@H}oSz>fqnnbj
zm9TL0YHeqK{^kt^6aO9KoG`uPkYvR#9UdMY`#OU^e>ziHIU#k*WQ#fnHRD8Iu8apx
zQVRtfuY6;l7s$J&^?3Xhj^<NcU7OC$$qbEO@9(ogOQYcFGc8t@j?W(ql8%_nvry|O
zP?c>`bN19*_dhDiCwO_$$`uBJn-^$nbR^y~VT+ADq}9nXBm3|LmWhnU#)lst=dm}q
z_6VA8XIiwh-rTW6V&Wz#rKZ5^xqqB>1OF*r)y!DRDkW~YP(fVfzomslcJ|kAZ;MS$
zMUxT_&U34%jEsqOy?x6hDQU@sb=AMRrHV6J|HtpS%CgRiBbduywngCK<KORJp4?Jp
z+P+FldEVrp88ank&6X6?PYVcC+~3~BX>i=lP<mkj8#njix3?pBppMx-@A3Ql?|FD)
zcrFxma$Y;rxaM|3)7lR|`gh4Po=a$0x~Kj`o?DqN^Wl0PGo@8BGA`fWmD*K(x!T6-
z-zQU9`~Keb_xJxB85wbKZj6lJXlwDPtnBRU*x)d2QiArMkLG5Dn#l@&f9?0=Xq-Iu
zE#AF7@q6AkRqwj|+uz<lJNtiqyz}!YOHfl;(#J<))oRHtTMJ#8Tp6;n9MY7z>wYvG
zWq~A=DY1L`S`0Y;Jt&&Dc$UQ%*&06f`4^@aoAdRqJ1Z))M=CeyvNDeY2Tu(%v%Tk<
zYhTl4cJEt1V`j(U!o}AMyPq+*2XH2&wUn~#`cOQLL&ngKVb1*OJ=aT1&(+R;7AsP6
zf6;0Edz%f9D^At8IdMhT_f5H*bd1ht>7EkG<xWoUc_;EQehJsa`wISjIl66(mqm`>
z1dj?HdbuV4{y&@AU)-kmjSUU{xLSSZJ(SvF>MM~@eOKax8s8JwNCAfJOuDsyk{i~g
z{y%=;z=1P={3B*amK{*`{5j*ti~sD!{856R_)jMO-=KMHmdgSa<;V77)9Mc$`rq36
zKRfGDZ>SbyQz{$Ff$*lgdrp4%sx4Lw36VoHE$aUL`TgZ(KHH7RRU0<XRY>PPEIGln
z=|k32`)x_hjJNG3{+Gz9s4)=e3S`?;%0BTb*FT}w#vMxR+P~$^gz|LGGEUTFmb|m`
z@Q0_|V(=<=#on5q{M)LgG&1fw{nlPt*_ipujPe(P6aW8LJ~y#-?i3H7txePHCY<Jp
z;bg6kJAO4oVQZA)ZJ9T=;#JFdQW{KC4U*=BpFD7&;J4YgBhW-P-NLT6sZ9Rg|NrNY
zuw<qf{QdvGed0t(P1!%`4l*(Q7V<*J8W`5pDm`IpW_16hpt<b=muS+WX+MNmSBuVI
zWxJMQctEq6aiw|po-YFHo;`Ga(*_NL=@$Fy{<7Y`&lWxZhkRL;d5oyA52LW#>}JN&
zmzHAs3r;01<F~mfWxV-OXKaO{OO|#^LSUo))IZMO4PzPBNwI|Me*R}$_7!Sce9@np
z@9(xx5YT3RGa>H8GM0$BpTu|0J;<@wjVZge>E^SBT>_@H3CnocdL}h9Mo*r=sJwjA
zETa>z_aun<{Wo|c%g|7Ig?louV*K7imA6bGrOb4TeKkM-zPcK(sCc4qGtY{*JU`}2
zbUf}z{2NiB>Qu9tdGncs1A?AMj%fVrP}wAMFiVPAP<AQfYOf0nhPs#k|1aF;CMh8S
zb>V!Ay1mukzyGgKt7vTkrK*I6E1EnWj}v%qM9FFB9Y1z7v?#%ijd`0**H&fcASNZ#
zm4*r025mv75)u-26th3KZG@Pl6i~FM_P4m$i=GXCbN^3B`fGp5f_;x)qC&{*gpb@=
zSqTTEqcbKRY0$kMG(+Ko_A%2ph7prkKWa8NHa>h8J4Y2-d)!y}%j_xh|G)DoHV?%{
zi(-juEQw8XKQ)(qVv0J|P(0CDpto0M--(00(>NqPq-AV*#FCw)q`dRN#*6?JZnmBm
z>S-*0pZU(=jexl7&~#Ay>-)dI@=Mx8Yt&fl+9icA9ni3;HAo22s5r&{B*9hNSi+-U
z-A0M0MMzBUvn0n%`-mVmsU~hGw^J9dI55oH%?L?zN(P($KRhfSvHFqypK10>q@LJc
z%CJ(Nq{F<;F-W98VZ%gW7OCk!Bo0L-2wWCUJdnuBY<9F+FEWAY$v=);KGH>&AOBkK
z>+XaaHpk-M|NqAsUUwz<%vE4JC?RnC?}6jTB?7iS;Br5ca<CyPY;sglfQGA$)DnjC
zD=cSSSsNWOt?&R;_qHbveUJbD4_w#Qu!Ltx8wYdHX%RkyY_Tnen6n!h(-&Xm;JGnN
zydp_9`*(A~Ny9IbSeY+^L;8Du8@KoeRZzc};s1Z72^Mzs|Co0c|NdY9Lv@AE2I)T+
zq$GP*Jmr!}D9iZ&IN?CDtX_v{*aa&N9*avX|CThE-ZfY-t26ZP0=6T|d;b5Iw5v_B
zsF%B+x(E{7N)tfYhu2$pf?&ysHPXzVI3$>vnccJ+8Ej56iHmRMI4Gl^eqf@`hn`uF
z{q>Ga5pK4PG)Qo<ubcY+zww@mg9Sg0_8mSBH>>Xdzt4Yv+snySnEZFn@;7igX7lfV
z`Vq6$hZ@SOSX+O8er~U(_T#AIjEzm)4<%n8QEk=i%RCyOrS+g8!LpI_-96QUUQ2MM
zQQE-s?|A?I_xJxd^D-UUuJ)(B`@zljrXxp195=-8{dM#1-#-T${db7ZTx59Rjnayz
z@|TiAeRS&$1@4qH&os)rlW=1L<7qvUB!+`6OoBV><{o(H2zTYZ{r^Az`@8@D|B2r<
z7A)b3d358DtoNxO|Nr0b5`AA)6(?)+gVS06+L8wC+6R%3`=(WWSQ3@8pGW1#&&dmS
z&Rn>%^WibxMOxfj40+htzP!`kBxUjap|j5K<cZM84p{r=*Vq4t**%uFT)Wp&JW-t6
zXu&nj6L-VidV+r0ANdo$zV82zkN0oPczl0<y^l{sca@3WziZ6%s#+M=A9G}>`<Egh
zmbRl>{BK934MX;?16B%biU0qd{qVz((HLsQ1dACnCHL1f-q$<FkeQ;;+kR*+-_t`k
zd2Z;7x%$ia&;Mst`)hU+)9NlRZsE)EGc2yL9Bfd2wDH(?dzmP<>P-y{d3kJwg>CWs
zCC<%FHZW1SFB9|i^#A`n>*EA73JeSkpkcM+Yv0HJ^-r=S0-GJKwI<)L?cbW)7`M>C
z;Lq>R&)=V%eBW>GFFAn&i`Sg#Stt>n+qB1@FPQh77Eg4nu|zN%GxHalB0s;j(A6Fm
z!4rPm<6>q$yfwQ8IzFZ}0W^$IQL*6R@Bj9H)bum%I0Uiz%rW`$|Ns8w{`F^Pe|L0h
zI(ojpyStk)f6}3=OAag*GIr!yq{eD^;L;>V-`<C>Lfa;*bINFUn1wg#^m*GdGaufV
z%pxVj!vmUL1Z9_qqCd}{y5{XY+|4ybP3w}JMa9ZK^`n3O{{Mgf`}_K5XMPsEy3)VF
zATK#NOy(%_IyU9BN`~kE+NWwnnBV4YOkrgF^M_~VOrH66hHIjafB0wnFo<#IsRKUR
zGmN*0>j{)W8tO_DEdKrZ`Te!Ha?uoLKQ-4^m;brj{$LOk_IRYR@!$XdpaIXkJ3mt2
z-SNM6?M-Lroi}ePl;px)o3EK3z4g-d-CRD~2XD?KWSrPgs(pB&v&+ikK8!Q|igWyY
z&VR0S@d;>DOt84O@9+0VtV)k2o?FwZCh{ct;>T%XCypNY@7%s`>$ZK*pQr0+Kb|@{
z`OrkE4X=*&RIg#>EZUv2g2~XZZHGnM`}>k={>QezV$wdaN@AW}+@6FVKPGNagd2Y2
zhQ$4Se@`D^xS9Q*zwGmu`Xy$XTNHl&m#@<|Ra(1XTgrc9gFnB&zrX+R@cp8qd;6-t
zZ{IcR{H<BNot)Xh%zgh7CM&t6aHw;0BqS#n6eoZ9kYG{NvT)a|hc}INobS%q{NYGL
zx47iJJ&J2UaRV;NcI=R7ZTWXlN^*i3d&@`ut(;9uMGkss{NGpqe|Oc_=}(?)H_!W1
zSp1lS|9L{{-}Y5AX1Fxmtai#?!1FVOk$ux9p1L|7CMF(fGlP&2gM_^YInQ%QgdP3=
z-&n@7>FES@AE@69&K*2<OhcJ@rzztmhQIQcOuzhpm~El3dv<{O>;K9tfBye(f4Ez`
ze!BjDyV6(Fe}Dh~@BjIN|8^xyIe7jwzdU4muF>Ye(S|8}Y+pNATR|fmv&|*<)g=D-
z-k*3?t8II}WK@*Im#>l^=DcjMRO!0uqWWnTEc<{H#_=;}7`i3gmP}#1+ug9m^^twe
zd#*ck3qBogub(@2{&Jt6T$&P+l9D2#Gv)06fpU__O`aP<>`_ezO#hhL9Q^*CN8a8k
z{c8Q+B#V*`MfLWBw`U(b+Un@u{-JP6#BUvW`=tA3y`cUKD8WwvwFv+J`DtHU>%sX^
zK91MW$Z2|?)4NB>DS!G)w5$)g+SeJ}+4nd7c;9^Ew3_NKFV;^I`uzU>e|ei9-~Zbm
zKG?iJD*9rZkz|0fv@z=f)#(rC=qNn+nB4aD^};tjJZ<OaOMZKkIK#aE;hRV{QPCGu
zH@tbn<L1V5>J&>))B8!ae`Y*9%DpHD(l$_<V6kK8Obg3}N7-un<T7Iw{(O@KburuW
z@Bg!}`}6(A#{1Ug|Ey|$h)YX5`|?hoCFLD=FTKN=l|^EI%};LW26u1C6Rqs?ju`$q
zxZJ-(a>9oKw!AzxOUpK9cF-!08ygxOqh>YEpWirpwxY!hMN`qJ2?w+V-FjYED%@G_
zFL`gT@~JXtD!B11;lib+o)m{Szx($rl9*>8At9k~^3lV?_0ik@RQ>qyey(+Wzx@Bd
z|Nhns3c7iFUwe^JCM|vK^z{EVzgSFPotJOBzW)Ep6$XditxY{J&Ei2T`zEF*(@qtz
z)&Dm-Glw(R^3-YLj!wgjOv4v14J|ATbmGnwJYqTg{5;S4I70(N!<ZchK0IXpC91pQ
zmdt#6;oU+|cTKR6ko;f&pD(lh(09-O&R$|X6GE6gm-oy6pKbB+_r1OQ?Q4Ji|M(a*
zJp1~!vASxZi`yn{Zet^ZKk>W2^=~kE{Q3ERNePL#y}!PHeQlqXrk2C&zs5JPZDMeO
z!+~aYp5^||H>091ELOOALqbMY^2`~DdwUXh6f{0ODcqKSU&6$6;c<baX<``<&dY<!
zh>kt0KrZ|L|H+(<yv+^*EYH_G7VLPStmLF5)zC5N<cb!y7mSK5k1E+uaOfyof4^q1
z{O*IY{RbAFGT3o_-nm58e^=(&I<3gRy)*Nf#WbDCyRNOewCT#C8_zAaXMRdN;m0=N
zIse4m$T^00j1Sr`IoDs*w{QQP>RC)30xS{UlM-|vpV%ejD)IUCbNB0a!|vSs!UCEX
zelmA0Z-WDmTl~W#je9<`oMg~B;J|ZI$wOuui-*P|y+2%R4~k7qYdwE2_597~J}q$C
z^+P!gSuQS*j2#%KxvuuQb<IlMb>fw+AOC&W!WFSPcKb8S%Pky-H=HewIQ3ys<L?E(
z^mUqCB;0@5wfTPjqL$h9-J<=Te(k}{{Vc5)W*9$T4oXD-|38^)$aO%0$F=^*LC23=
zUDH48i1WX5Raf>_m+Qj2vxQSQ{>-0#pkA`CanFpGJ6FZbeCYI`Z$aC4k)|uNdK8x$
z%w|qIH*>D=@+Ko^uT5nRp*q_h_gfi-zxUwW%G6-kQTX}mX4_*)Ler;KZfOhJ_pJcZ
z44Pn(k|I&}@6Y+epXW~$EZlSX|9>7Ho)5|r3wT4#%-mR6J))o3t9(tc_<sM|+Wm@K
zWrc;C7bqN_ZT>%@*>XpegtEw=|K|&yaxFT1^7!!&zs;F1UD9G>%h6<ze2`URQ|omo
z!Wv@H4TV$zkw5JDSqJ)}e)L~i+p5BPBthdyxrfS@qMZc~`EzP55AUz%(b3_VH0l4$
zng10v<un(IvmCs?pJ#o%aui$rKLZbs1j`b!>~zuWxlG|dA26O<9AyA)uhei@zp=Yt
zzv6S&1heDy7ShVE|2seZ_xwr9jyl<>IR%A=9-e>ye|`P``udDugT<Q6Tyi!E6+eYe
zO`Dh)5Oin}+h(2vp<fROdM<wlsdEp77a15Ezj$Hr=jY)Ar+)F9^)hwFKKaz#$hhsL
zT-d?|*8Po*4?jQuU;XV*<9QAPBcnfmPain@SH8^Ubw*<&(>$9*i)t~e6KBtyS+J%j
z|Fxjefv7nL7?qU|pP%pIyA#?}JT%?n-{0gPzos^}vRY~MZJGANKIl}#t*$WHMLU1o
z-Cdu5?@xRF=|8`wI{Nf9wm-MGwr0+fHQ@rSHmmqK>ETiCjHZSo=JRHnSsb{<;^?h?
zeci*`|2sfSuNc5{IRE~qKX~kHCN)p(L8|bU0}WfJN<5UBxct6^q~wg5|KH#E$Uo1*
z#YLi`LZa^Pum7K)*MG`0kvhgONlh=VrRe|PWC4*B0U@O|g0CePXFPd)yk&adAxLl^
z3TG`WGFq`J>7E%!tRYus`T;+)wr6!63PHk8<e3luumAr~-lii+XVXSQ1LObV!oT|?
z-TRvI?@N4p=jt<O(?-J;tA$sDaZHn(Q1I{9)W(@JTP|ONnEWJWCoh8{k1?-M@P_|c
zE>~6-tm`Oo@hHFWm@{a}2lfB9?3SFLrFZS>W8L#4ap8l<MQKl$q%HFj>|QrVE6dgE
z#Lq7myd4&qcAWXKSfpp8;hz0Q+fs`sxxAmPSAB#Tl-HFmRERdl-r(?`UGRIuw*Uh@
z;~(cw{CIzTeZ9EepZ4zyA|fT9JeAzBQ?l-_(!vgD#Ruo*+1%XPgw+LHS&faGcUYWT
zDm~+4lTK^c@(-Vk?(Fw>dITwH4}~Auu-W*-XXl$9r%e~|+)&Y-(yi0_#)*xsEqePu
z`?@m+&h&gR;O9Slem+l3%#}TLZ5|(<avz>=&*S66BO}9e=8TK!mlvFMe-bPzCOrI{
z?y^l#AVEMl`NwzuYW*n_1*LBOcrC*5`=qc?5F0ZyB$e)&yPK=QfX9`8QS;>P$sG^g
z{r}(Zyx?uZ+^**Jju%&Hlo)0ku@njK;Pl*g;NhLuyRIIbvG?Yim0xt1b3VWD>(Sn8
zB1f;e@P9B3`BRe>y(U$nLurZ=i(t>f*WVULcsk9Rl={Rn)id_8Nr>p@GZsR}>kni^
zzNlW~`lRQ@H;rGq-Sh5TU9W0>L_^dev_TXUxLfAfa<v!;wA%AVczAu}k#o>~pP#F4
zwyl_f`#;A`rRLXK8ISFjc}PA}?f>ch?!?A0<!0f!<;?PV^&*<Wzi0Xibv2oU>i%rM
zJoj*Qk7LU=*Oo$;mc#|`Us`SBP2MtDQs}5f7<cyb6Go>dty;A1<Y$8gEnDBeFFU@=
z$k&AT^rxq#^W^89jx&9{LiKpo%J1d#ZDK!y2hrBdu;prX5NNjFawjc1np@jH;bo$2
z``)Puug^24T*wqDc__-AFL>lQbMSHrkG2+@nTIW}eRcbn{rN{?U+wZ-=LHqk(uX${
z$h<#(s#2-RkcYX{_xi$9HkZHmJZ^n?=d$!s!&N@Qp{|nxrirjdwryRad2L_KwstMw
zW{(>y7j@XUt4y))U;kwL<m<W17;b-yKlvL{BpPrWJJzQ9YvN88@4oZ%CF}k;xE%if
zpXbdRp7-|*KYVg}U3ZGnW=7uqv>$(FK0GbnW?L=6CzJ5wQ|rUm;cdtJJ+-|RAH0uu
zX{!4*CGlfY+tzH!H*Y0-dL^Gckq8Nul##hm%`&Cgf4)TA9*6ls>1k;X9v&28jd&g)
zv0<z8!zV|(Z@e!$4Bh6SbU~rjXX@Yo{hM13OwLnya9#gMf;>yj?_&xYCq6u8Z=NC|
zx=PgJ^S_tDZQj!*?CR5Y6f`E@HfsxCuW)$Nkv=P7=6$uvH}=hac!ZNpS((k#vn~0!
z#Gbv9CMFVf|56@2UVNp%<22i*FR!m^w`~6ZKmEeRgLT!G2VLVo{9--SwuUh|Svm2C
z3^-Wk%+lp*0IgqF+P0Q$V|9mZ+ji0Bz|9Ajs7>A=<`{6nplN@{YwH6BJo0Cnzu$Xc
z(6;XO-siSP`|^t~ZZhCm&eSL}^T<*&UdIpZ{5Ml(Nw~*Mx46gj&PM%ofy}d?*}~_-
zj!jVcVdv^NOUO0D_qI^m|8%d=<*PKjU;Sd)@_gZy&xx+?E^+_5EjH}ie&Tf_I9<<~
zwUM{MfQKcYVMarsKygZt8^hAYOFcX%a=B(|csM@vU}<uAn7-kD{kHpEi`V&2QC)m9
z`cuV}n5s+sW{M*3mDlD<hkmu49mIJh^7OvV>%Q22?%CRtT+^$0D(1h+lFd3i%w88u
zC7f>`6VrDxcxl_HwpM=|qlNqB4;6_gwl7*4I^pbeff>s-odJ!Y{{O#bt~FPK1CNV*
zhunc%+8kB9{x4O($TW5IUTr$j)jD?(hnV)i{SQP|P8RSm;yL;EZE?$i(&>9Mwrw_l
z&0#6+c&kjmWqtOnwnYaF%*=PnzjjXbz1*SH)~KS$F=3L8`ssVEzvX|;Tjg~9W9=o8
z<BEMB_q>!|kr`(s>^?;`)?GsB$iKzYa+4$1S3HsCZP$|0v^NoH6$It}DRXVPS{!&<
z?OR+^1vVCXN%ueS+_PGI!v#OVY)1j^?v)`a8*V10u-=R{3BB}$J??>^$0vT7evZfM
z73c2EN{v)s(YoOM&GJ=UDhCVsrd56XVA#7!=Z4v3m)OOou8z|ZBY6(r{kyiiLN2l6
z{o`HkRZ+(0HA=Hj-ss3YvMEKTQ)yP4iL2KVmA;#1EkEL>H}+L3trw~EyMA%op1Aa4
zYraGe0UN!yheCail-AbF{s37RIoFoEDM8?P{f`d{8XhesCYj9_glZ?+ZPRmV>Q0v8
zeW+k6;vaVFLi5Gh7wc`_S?qWm_phM9-s-_|{cHM}yC0{%C~-P|G4+b0W{X1K$J#EY
z%PA&{*2!O9-hKD0*JTCO*}Tn+flI!>HBj+1=s9lbW8ZQ?TqXB(ij%;D%AVsq?Ga)@
zS^dj}k1Ow%y{)@iH1gY5<9pWEvR8H4sQX6h6vU<(KPx^ZbpLt5fo(fu1e&%jT_@P|
zS59;5$s$jQB0Kx7wM%FC)n9&Ar*->f#I^bm)*ayDZpth}?xp~N<MnO%1zd|3D)zJ;
zxq0&JzVzbY`3IJ+Y%(h>(CAPQmbudLz*NIjLnJlK{O&>D<B!Y#?=Fu&5x6@y_rZT_
z`%iziri!s1SKf85=w!i(yyIWJ)*pWv^0NOnui^=#5K&D|=UchUf@|h~o_Vy%V%gPS
zoDxnUK3ZHd?57WzTr9PcZP8f2Se}2ck2ZU|O5$ckj)?Ui_i*K(&3abk=q1v6lI3>V
z)+C!Wlce28H(AI?@OT7^u=*zNUw->&>BLDldo))4^fBl0nOwJj*@^S#)K`8KI`_PO
zj@a}v@G{aVbL_dB0z{71pYT2+<8*SVp^Q*gm)icO+IaPY3s$x{aL5?VD&kOz`r~Z!
zMMKzg*XPH2?+e~pKDW(dFIMs7cDyPWkS>@Z>AYBQn(O}mrPKS_fA77cvms)_^AyR;
znwh5(6uvG?6+E6I8R{xhXi>0#bKGK$qNvhonO-qfa`TSY=o$Zx<M(wA4GeAh@#6TZ
zMLL>XpJZ+yEBsJfW*w$|Xu{`6wH*7J3*Mr!>sc5d8wMUJ>Rh+^uF>p`Hw|VM*d$5Z
z$(!RBqi6KhYf@UYq6A-ItmHlOReas@>$Tn|ujhAh`jh@>9=I%=GS`r+IY7WA{-g4Z
zl^F`T0++Qa80#l?KjULlX`LFrLEwtfjTIU*)D&CiI|wzWtZIEb?dIIn#2cbTCcN8k
z`#*eA+g0-+?O0m#^V)d}-<2Pm_*qg@l8ui&X1Y!K{_y`Va%QV6%$=!rDI}M<bnTWj
z(YD6ag_%F>0$Zl!Y|mPiJ!?~+?iDA6mQ!|erzF)>k1Kl~dg?SS@pjC5?YPT3=XNp0
zJ}*(?@Cc3(Kd#t!^X-akpUIn+ynku6ic3-KiXu<p9M-fyyBccOORZ0Mx4r%&3%Hb+
zqsZM{AaJz)&vMu0hi8^L8g^uCp4IdFg0LXV=8ixewZ+F;clKJguaQ0e@l@e^+qfNf
zkN-IFi7oL%v+KT@&eJyP*z_;^`Qp6su5)kQB#in}B=`A8DzS7q-PSX73oNkOdtcy<
z(6NUHo;D>W+*aYt?0T;Iywv93!}YBWp${cD#jbY|O!Sy{`TG~!+}->}z8sFBiL<Vl
z_HEYKy6j@<obx<~lZ3*Xlss7&3wvJY9mq=)U0=E;P41{kU}$~1eK|N*=FC~k)e<1k
zYVRnRv#r~aRsKN2g_IAmHDQwzHslEG$qL~~D{OxBuBbJ({Lm6k;d66-CZ_FcNS2uK
z{JNrCzE7#;hk%)fH*q*OF=*QVNWR_IwMM7X=7Y^DE=2>LS5n6lBPHBR{<1x3Jy5za
zcT$UvvM;~ETh^ELMaQ<hpZuJK(Y0|3$H&X^+wV4JI$h3?@|j$^^t~a^u6MUq*UnYw
z>s-`5XVIqCRiU%D?Y-DE_Y*kKl_ppyyfq38Tp@cRl=IUWrhkE4mX>U~x^0`&H=ZeU
zNOQNZPuo#7wXwPR;n(ao`+A8TI~VR%Ik7;0qC@<A$GCZpetidzwSk6(A|&pwu=ixR
z$)c{#mYLbsExyqAXs@(n-5-Tjyeyo073o4jAql$75|W^*KPQ*-y5@|>2@$%f2?-Uy
zrp%cMooiK^U=a~1nRh>J!?t6Vd~Mehj^29n;89XXXQD;joQGetb^Kbo4@9r|z}o84
z_qD6bXvQp~6Q_+=tTw)J%Sb|Up+(OthuFEGWro*u*?fK3Dl6NjYD>oLP5SYj-^$M0
zG9WN<!<LkSqC@*uNNi3xpxQd8WL>=ab0%olNU7pJ7ifFZ#*Htwnigp~ZMb>x&>;n#
zGXiqw3jW(2zPwz;TF~CWFwwv;F(4%6!j+T_8y$HhFKVPZG;A<v?&LhY*qz7Ai>IrL
z$9uZbo*GVBg@r5z#>OX37)VGOrlc=EfAKNnhai^gHmAS8?}#-}0d;LagJ(9p1%Is$
zvGPc`vZk>e6-i7<nJ_JF!`9Rt<^72_jgs8gePB2dAf&F|+|LhMFWAG=yiBcmp4`%7
zlUO1mC8tc4ythA{gEuuGAmPRaMynsSKaEZ`c|2ldXJ@vzZ*FCE4iaka;P`gZW6L>?
zs96VciX9~K?;F`vIBcqxfKBfy&9R8v3)+(A=*;|(?@XP}9n*?Wt9My7PZmDByPU_m
zEMSV=jvW&J{w80zoc!Ti+J(!>0-~uGuBc>1v+Fi2Pyp?;J9myJD2Ruh-^ix+l!Er@
z4e9*M;^O<XEiW<%3p4ZcGh17OR!2lgG;cHeq9PW<;QzHT^v_*uqiz#OPf2Nxg<V~W
zMU{x5DzCqMhM%LJ-vrkO4b9C5FAJZttDCc8or08rNXmmpNk4v1SJdb@czLp;PmiMu
z%Wt)yDy9=F_HQ$5o*{8~sW*?97>|?`kF>dwhL({{70cU+iE}?Dwdi&|ew_T^aWV&Q
zs)4ENO38~1>*REp-A+05E%^Um^4c23;$NWQ%>Vzl%vI!WP7r95H|gM7)bUbV?5%u~
zsE*qIR{Luy7j-n!3};-~v|vYXt_9ofs)rWuo<4XOw<EhSM>^Cs@J@GaQ^Y*|#`OzV
zrATh;eXPJzde$ON=b>%(tQN&tZSN;PXJ5SQUA^qw`^WFCc|K8vhdD?|HTC4C3npJ|
z+Z?aUK3iFNY*R#diL7FP$f6DtRgR9w+}Wp<m(SU3;rMc`%d*aGe?Q9J0Jl-*%mLK_
zJZ<)}HBqcfA6z(il(|Fw!rkr)r<bxaRvI3Hx3}(or1HsX#Rs;HN9POtGi7l)H2=x^
zZMqy9TzT6=djkF@X@%#S+xR~I`>gKy&nF)jtTNzvHgli5mrrr*`meRiIujK<Ih=E5
zYbc%GkZY1#Z~4lEkLw<1kFwJRlRa@StWrI%Cq{13bt>5T;io`b%z6vH%&zBx$9KHD
z*rK|9wTju=*}8wZ!{`6@czWGGcJlMzOt1WF!0ix_%UcZuT;dajIb1eu40rjUb??~k
z3RZui3)h0y<n;6|{V)EkzDYuK`JJqpvfF0Je$x~#^^Lq$vrW4rIP#6_yN-8OTaHO=
zNt2i$lq}&siDl^wwM}QDPB;aKELr*0HBQ~X_Ra4<ZP$PNS@+~~PtX?K(@r86zk75F
ztkJIYlMrG#@-Z~uLi^xW^M8+zukXoSoVjAva{mK-KbS!EkdhgLjEuxI-R5&Ci<a^S
zar38q_^e*y$R;npMMg}#T}+~@i$_a~XTH5N-)Dv6`2`JI_x8+G*x<(hl=bBM`#fvb
z@Z8^L2paV{dGf<UW@c{g<_3nt-``uLS6+xY+F+I66uEPOD&LL=&(HIG{K(VJKlx=5
z1Gx53y7pQ|hR3?xC?hk`Q*y^7R+~Zrp8NLo5_Nx@-u<Zl-q$#DX5;+%5AWNz-QT})
z{RCw>0n47Yl|q}%{X`v(U65JFa<CyVAwf(!{hYy(qZ5q__+aJBwuXNnn-edqP2o)P
zNlz@>*wG^`CUzt)KvD6<p-?rot{^tUZONr(vqU~F6J*x#lsuutBe7C)bu+)bT4f`@
z5!87Lzunu^`tY>)n@)!5+8<l&-ZISdI6Ae@Ftu=|n}5UBGY3Qte&Kd`a$dfzS6VZw
zl^^0M#X_5^BO8)f#nc}a=|!zt?tJ2rm)H@NfFvU&=C54>O$^453RO8`Hf|_V{iS9f
z1ak8K|4NF54q{sz!;&6(ip`KaJU@OVQ^y*n0@b85Px|hs%vvV;sF^Q&C0k<6j{_U3
z%`Kz<PJrrL8(i^O?eG*$jo+LS1r=<Qdza0NYFKb}MvP6{gP?wOsR@!IOlKY(I(1<F
zcZC;+Slhg(Ym`DAs#{p||J;Tw(JwZvHc#c4-F7W>op(4(WS+;-68?1x$3pt=yRA_Y
z`Nf(obcDymC*8on;LqRF8wwOpp6)b++Foh#@?C73WwFG+KZ;sYHfkj(Xm>u)RJ<T!
z@Sr2)k1G4J*a|PcKbF$Lw##%Y{xqq3T%6M$dUJn0&ulZpId;CE#MFGCVQpFX=lAgq
zC7Oq?uD)S?(?w~`!zM|Sh{LRw4IgGSFXNf%ZQz@{>bOa7!kb5l2X5}L@i3VB!14V(
z!!_~#tx3)BjX&uMzurc-T?=cA-`~*{7~?57JzO%NS!i|U4j;7)?yv^UR{MzyE&262
zPAaJRCYPGcGjYG5yt~YBj%9N!<4UMoil;p~A=tJxyVL7Lg#dG3#}QRO8)-fRWr@|s
z2DvW6F3*xoc~}3OVRtA?{3E051^xALhG%9t9(wug2Q*kd`>eaaU((82QbbfzN9V$0
z=1rE<DjL+~9|$>et0nm;b1FzTZS!jteD7(M(JlGG`kDQ<*4he>Ce4?(&D-APy*OpN
z3gi?}P5D1%Zu)tNTel?Q_8gdWOtQdSx#Nts%QKZTdpMpiVl+B)B-Kdo@Wx+K{s9T4
zqRg6_%s0Q){|8UmZ0S64Lm6}s2E+gVi(cGVI$2%v)-8#Bb*?+6_JnD%2xxkwI3#DO
zb4<9JaJA!j;*Vf9W@bC~WTOwuV`E;;w@I8~IeF1c5or7@(kT4%AH0$J(o&6T^M8+L
zII3AoKQv_27VmAmn$lF3`0D5rp~*%cRK&dmW^um1Z|pPUK*2X1)rp<Zag{}1WG=kF
zFIiPB`QxXglk>vpITN>UPhkD;IcM836Lwz~`{@fZS*9kMSxXoQ$uF~z>z)wIIV))k
z%aI1<P(uaPzWe(n#dH#G?3??`T?!J+iw-Gne1Ffdq|E5V34@48$LPfo!N=;C6z6vK
zD5mVq*!5w##F^BOM<N>83f8VYo~5Se|1k6XBd^O9lbB}x`2U}${=d<iyQeoKvHt8f
z`v48^MYpcWb8@!*t7B7F*U7Gzn9x_|7$?`ZQ-tIHuN0pK(@uf?0uwy;&Ukk)CahPd
zb=r&zJjW*=`6w=(agr&h^ZuTb6J~VOosroJGVcHXMbGvbu(Gywak16czcBLOa-?ym
z0gpIyw#vg(TbGDAMKH=adqi1uyz%(M?X<B=AaeNu;o~cVdA@$-d4BF;34bfhl$($F
z<qWP|H~#bQtbkNk{!2?11#ux2SIaO>6GyJqqA}jSANwl>*8KR;k*vmT!j_g>^iAjR
z5>KAz=O2F;bA_7nuP;bavTct|+l2tOygV1><|Ut)&#9^%n!tEat8-5hoBu<{Ak~g#
ztj4MjGs_O#{QuAJ&+p>}U$va8tq)IB-eMar3@+amoh(asbZpyV(sn22$jci?wpebs
z_B!HqKF0#D)~5RoMq-N}>-$M?nA#Ow)j84m_>^edx3V_p_7w*2yX2uRxf)keR|jfy
z|EqgZ=6XV@l#OkYFgx?hPz%FU9?6Coj~J)V<_t4DqSE*OzhvG2G>e)U3-``__{_Iz
z&QWfNBc3|4^6;!$#q;tdPhujEj7&!g<NZl3`_mNjby_51cZfXha$1x2Xyw-r5nZce
zoqQfueD4G8iD%`$z<*(pEYuyjj}j6R1OyU3d{6J_Nii^S(X`=E<CrAGzLml7V{fF9
z;7ldXYic|<gpTj82c;I9niCt+cn|L?<>8l`P<C5L6=DuELlTbzt8nneZ6Lm<tDnm{
Hr-UW|I2X+)

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png b/tensorflow/contrib/image/python/kernel_tests/test_data/Yellow_Smiley_Face_Warp-interp-3-clamp-4.png
new file mode 100644
index 0000000000000000000000000000000000000000..8e78146d955ae8f02230121e6314f3285e87611e
GIT binary patch
literal 19313
zcmeAS@N?(olHy`uVBq!ia0y~yVAuk}983%h44c+ZOl4s3w(xXu45^rtlpw*nn1P*v
zfq~)we<{|*4C+KOkMtkpJ)ppI<ekvLf4`lQ>_XicPBY)AlP%~sv`J5MYpKt@$#nBn
zh<WIw2`hvy{N;ReHmYR5-Oi(oAd8Ni-w4vo8pzXkP*n7{zDK*jrVso#mG*GH`%!kF
zKC3n=I8^ZAoM25)6+`!D3@sA<T8!@{6u4aWUAuYvYyz8c`SCd~YS)+Mt^f0c8D!v{
zD~?<Z3IePbB#-O#FfDW~`M;Z0w<StqQT?*&lJ?`jpUa&4Sa_szUJ}n~qh(KmC)Iwx
zt@iqq;)d@!PD*cN3}??2x@a(C)m5%EWv6VejQ1~Mqn$NRM4sOqFJ5%*_qCt^?R(<q
zKl|&+gM;bLR70*70}&VDB(vT16Fpeo?)|?~XA4K>qSguXe~Kk`?`;aUnep#nb%)Hc
zj75Dfp3YHyb?l<vY4aBU`@0h>{3p*%7jF6{cdvVE0M}YsFURbTEx+9b|7b82xp@gC
z-~GNT*VOIY%=4R<9C;np@9ZU*wsW1>GW9oXAb(!DwveyIK!D}K+q|QkCt8bDc)1S$
z|Ia4XuXNCCMOm3&!NVV{cYajPJhQoQ2J2M?p;>Cnt}OfV^7nU(ZI{1DwOuIfnW$tU
z96D>K*4cBv^x5?eN%z(7=vI`qXm43sp~=VapfDk}HqzjH>HX|;rMI)r?H4}W&VNcA
zWVO_-PUpah{y{+=US09Qlifq7#zxGT+TA^~zrS_<{Lc3F#<_pxm6aE@IA}{u@Zxy0
zDMOPjXoAA#|NsC0|NVVGzx=<w)!*MA?cRTX|Np;#e!ib;U9TVW<NK47``5?+-&_0p
z{qg=rK4nmv1*OEQ&gRa}z_@ww0TcYAW<|%(_jhWF4w>Ta*%=%(Y2xhP^1{MZZyOG}
z`e}cd(zx8>XLs0j1$C*EZ~N>2@2ma&{l&%or>Fn7tNZi*aQpu6@Bi;Ee}BK7|Ns7)
z#2^1^7clpgKvTI<XLE0F=luDTXU}$aYVwYc4+#tmiI4Y)of{e<p?j^+&FtNlPyCTG
zD-{j~Jw9-Xe@YX}J%>gHwzhg7pa1vw|1W=dXuq=izkB=se!sf<Kfk<D&Hr;Bp7S4G
zAI~g1#f2Ga%Nvkq8mCU3+1cqCG$}ed`qqVjb7y=UJUeH0cX!U6J8|yZ#+g6m*Tnvm
zcVClSRH&-paq57NOhi}L|NntpQ~&?}?-V2&^LM`e|9v$-zrVZ;3NhQdKl#V|>YtxC
z-cx`6!xv3vVd2C7|2r>m1?BPo|E1n^Iy*FkIy(nBJ0H6g<PteEIATVyOG|4z`$V+~
zR<aj5j(DiPaAj>XTBG^cKI_qe&?p5V-k@EMf&4FD-r4#2{n^?7`{j+_+&*6L-|q13
z?L71C40%3%G=chPPp7kAUwB;Hr87QG{{8_$K@PEVy&WFdtGFee^@%yk(CEY@)ZNoF
z<F4=@QQo3Q4*u*LzA#?@J-xA!@t$OxaooNXi}JpQ7nR#?Z<o}IOR4xb=i%ey2cs`S
zjW6kJo;x=(EX={r@5l)cA8+qNCp>)o`esg@I^)U*`JMSp-i$LenNPXuS#IbnULdNT
z@s<19HO@)1_IpVESDLE*Us6J%?tj{j%E=EuB(_PLOXS^8tN76H@U(cF<Bj!<;7ly#
z(%C$Dva3r=xU1`pBQDpjgxtFl5)d)t;s1IMb{0$42ieXBMvFukCLC!L{C-1QDyKKB
zLq}zk;xfYoUzruWd?Jw^ryA7z{{R1PY+&%`=iv|E^$#!i=h<Cm_~!2E506+6zrD?~
zzm9jEA1}l@5s-DQ?d`##p%<=%Ts`jY8#{OM<jE6zdl~QPyZQ=bm~z`(P&|C^|HIDa
ziGn-dxp$UUNH*W<IWWuULJ<F!1PAT*3sN#3nIb!*8nY7<5)yv=pa1Z_eOtf0<i6VE
z9d&aPkMXpXzn6G-FZIXw{)f-cH}(o>K!fwak%OmB9XWPv)k4LLymgP?uV1v_fTG5U
zb0<$8IPjr=;zUNvTN}0-Z<xmZ!;9n2t_@P3_?1%^m?TUTQu6wu+^t~hE}+Ra$!3DJ
zj=Y_LP5rro-)4uqM0viyH@2xf^x^mS1Lon-Fg<bP;Gsh*sz={GNqO*m`N}nCR5eeY
zJ=(g$F`!!bzu_WD#nxGQ1)|I+*7{XkJ029Gu=;pM(UTzdoeO`7pG=w4$?PdJQRe|?
zYh&ZXyVh;&{1WTp({F6+efY=~<g<VOWKYQHLqc=Ik%K2szIwX!^|NKE2?|Q;$4{R=
zec-@>2CG}W)A$6gvfTUB@S~+^^B)J<GY(<ROOH9~CM(?8KZ8xl_pxJ$(_u!-LWfh6
zw&zRc-A%Hno%QelW83O*NxK?}85~Io2@DWB8jc)1dp0Xe>-~cTx#{Yw*XXF|oaqTE
zG*DpI-nPX_A#39Dh(PPr8XNn3589sz<A|zfiGFn8)Pbm@3r-ptESM&rvGU1N@3zm+
zCC|-Cys^3c;Z5VVe)*U3fjkiN7nMJnFj4T+=gZ|KAr~(TPoMH6A+60tnz=|PDYtIX
zQQ<UsQEl;Mg2uc5r>7i=+Pq-9!Gt7{R~{Xo1ELpbJY}DEr193jqd_|ry8rw={o#w|
z;f2mT>i$MDHYYwjVx9ZA3lbxX;vWeJJ~S~7E-bsYX>)K;*t691^n`>4y)BKV`VV%>
ztLhm_$!kPP>Z<<z|6ly7#++n{6U-|mk|q8dK9cbM+jOP2^}5;&ss9I$*#x+qRM6$;
zI56Mk-``}5>S+(JXtwc6OU$!LtoS`G&+r)3SHX`00tLT*H!ms(FgKg_<Z*IZ%Afuz
z3jd5B%#l3U<2NPZ&;R)+lvuKNw<IiPu}oc{;jqem!es6$Y1ZN$vMVEh$!9-2*f7yT
z;m!JZ$+9;I7G*t&_wCx2`@a;<gausXqX$n@O-)02ISdL4E(C-;NlQy>5MQ<9wV=yg
zM$Wu5DUGJu%3qZy?7R`omh+5f9oy=Y9Zsy7HoSS<(~fkR@*cc-V)bbWiGPL$3(QRw
z7_y_E{I7587MJv!pBAx4!s?x0HpDMmq71nj40!gujwvV>`OUC!$1jG1$G;iYPAGh3
z`s$8q%afv(33sM!<Ue-WXzug6*xQ9`4BN~mUMN#y2nuJqHf`z>nWa{3AuQ|eOr9We
z`)$zOOoqAbdvBiETzlf`Y{~z&`zNv_xJ|m}W%k4Ry#IyzDNlQ*7@XH&-O4w80z0U}
zU6goo;bLV;$;`)(7Oh>Yd*sLwhVu(HT-42&*;uV5sw#Z=*Txx3S<eOfZTZOG^N`CX
zlxG?n`?WmXX9sMqJ>$+%U|D;XQQz_7RQU{EK@aDzS2Ww2*(J^L({|L(E)zTf@#3P;
zlZmNICrx{nk-b_^^W;H>+zKNFQRW^V9s#{2e;Qhj*GEiC`on+n_KTxz%52ja&$Xt;
zOnJmOllOGrhezx?B^r2a7QA%QoF5?DA7i7yWfQTo;lZIfEFblkENRJ3Z{aT&(6N`8
z&?S(T#>UOfR#?cUs@i5*ETI>pAR_q$R8ukh|G%i{Q9?>eLRwl;;l<OZiZ*Qudh{r%
z<)~rJHd!H=4z0F{f<YJmIP2}5sLfZgM!`uau;H+<pv^u-S6ih*#bdJtqBcw_FqSl!
z#Iec3>s8Cn-bO}S*26ao3YY!=|9{~zb0@|ly&v9&ewflY$K}GL?ny$;69k$&I1XP~
z$>ZLCDdnpGIJYlKdz6rt_T>5VFQ1M5y*8N{Oh`=p*+1uNV&D(?opT!}%1->O8Y6a-
z<%vY9LzgsLj~Z)Q<G}{0-31MEWCS<;|9{w9+$8oyDl_vW5#==vE?bNrylXE`JS*ZB
zVdu!O##g}R5fc;7v}rs}PCUWOjbtnie0VE;`2GDGi`#6VF%dPpdFoV8uP%Fw8(Mnj
zyj)#dW=dX_%9z~5c!}}g3-<+Eh5jw_TDaBH;J3~UUv`~bzKE~=m*NfvG+di|@SbiH
z_o8#(le!%KHr>4CkgYCw`B?+^qSC~Fe-bM`wIrSrdEp=_nqC4e6#PyeJauZxA|)}Q
zBw68PJ#B4<lNU}LRaSm%AHkR0v{6z*pp{Wr=g7jD5^>E5PYm2Lz8%S^U!Zw7u+~6f
zs@sHV0%_kIR<wx9oe1g>;s}yT2$AWqVtwYvwnVF+aixW$iS-6Co(zV#uL&%6&3kN|
zD=nGz^$*|M%fl}hkb4&z-cF$K4ha?Q?%LGfw@pC!aa)C4l&VAUv3fQ(H8wW3h}J_a
z+=X9mwFt_Eb$pWHO8DzG$FcLuQ|G*i4^$Mx6qZI=aGE~+|NsArP$8?%H!&ga4Ga!k
zVpREfP$FY)fy;I=<41*C%{GJ?`uOlDDe<VP^3>Gu%r-N;v;X{ux6;!TltiK7Zc=Dq
zWOU+mcS~E;(IXKZT}8%*h6jV$bS^x2$mh>ELpPw}Kw@8_fl{kZ(b|5Q;{Sn?JRX|@
z^ftL&31ZPZ$KWqq(7kbMpg@zyQ3mF1M~-s@H(Z*~q^iDU9{02~O=h>C1CzaG%#hfz
zL!ziyG9ps)+S=qDh0PDo%g0P~(}6poqvJzsYvj=*5nY{y#zsa5_wcR`7pVMbuj9MG
z)bjt~MJ$mM1a+?jBnx^6^J(r>Na6?;U$@LgIqFiQdG$o8*&1?=FH%~}JtQaW<q>w7
z(H6pS@aq5LQEUlJKB+q<@d*nvD=RyvsWD4SA8umhDSszmCSwn^Tcfa`cyoJu^s!@6
zp5C7iFY4voDv{8z<&A`-q=IJi#E<v?|L@tbk+))7^D*;z2b^R-J8&pxAHULa^O?gW
zC+!;-Oq8ax^Q?$5Nq)#RAzCzJpR?b)7pa`Hm^)q`ciJ(*h|RZa=1fJ+&c@Ep#9Jn9
z>iz|7>V|NuTUutQsYGaNa2OaE7#JBXoF>x|sJY}X3y+5=yYiYhfBVI1Rzyz-v}(!V
zYdESG)^JI%HB{^bD}!>@oyT5rC&CnV%Xk-yW@gM0^tD}csv%tOe_|y2JHa^(OTMu)
zGc$8@JC~Ro-e1pidAYGoz2CngXNb*nV!+)`V}nA6{DilhwvCO_72V%08g*qgPE5Jf
zcva9TVp?)#La@v+jrZpz^q=#s6<shpG@~Jz{e!_F%jWXqANHtqm!AzSXZiR;QGA=2
zV8FAt!E0O@c%lS9iJBBxE!gq=g;V17hti>`39)r2{BQkmv9jo2<I17Is;QX%<}X(v
zxYL@X<jk+FJ-N3x$E|I5UmuIH@t$?ADTNV2iB00_6GWK(#XJI8-FZIn8Spr;K2(nF
z={fP2X@$rUlO-1o+<cq^-X3DUAo5tqQS-?JnT7&ppE-gjC!ToM@+tATpW<p4M~{0h
z;x1e=Z?sxlB>w!7xN}#srL{BbP6D*pRZ=h1(Bw={VUd*b+O)~&pc0R6>H-zPD<Tat
zM|f6j?33BVG5@eFf4Z{w!cv8eqC7n}cFZW`Pfzf5j+oP{c#eZTBA7o)si9iNSo6T%
z9&N`5UV0ghD%z(7<jy4|95`^Y^P!>=B)F9<3p=_#xwu>J+;O6~&`_D3G3!)<lb>Wy
zk57PTkB^6A=ZRk{_Bp-tOsbODJjXr6)+4Cm6lYAXf6S|mY6?E)35tQiHaa{L7^J4p
zIK(`&d;Zx22M(M$vtaF60r~R_)}3oGOY?=c4V2cNJ$U~7ySHoS&&%WUl#~cO-Ekzr
zN$5Ab^p9WMf{wEmx?b!!<(#dTz~t65r7@KE+CL7*10B|p3naU{U5wdP#RYF}Q#he!
zA$$1Pv4q^SA6^K88ujJxCyB8@{i0j=;?=36$D%%eGBQ+n7Ou!}^Z$#PZH<kcjSu;4
zVkR7VVP7Q8lqIms$frT(%YVaNLKzJHy3H}d3$2;i6H*S`b2&Q8=s}E$ARAl6j|0+6
zdM>y++>mXk_Z9ebL6Ujfo+C$(K6rAZ;4$0b#qLYao`DqDN(-YVP7~wdF)=d{as2eK
z{>nzRCxPB>71_-bKiWs#WLO_I;jq+7)sOO~*Cv)K&TiIzBxxw};B|=r<A)IQ6S`U^
zEo&wUYD=$;ZM}KvQ2a%O)sDuzzPp>4oE;dN7b<4nV*^)#N@j%xg&V6XGAgT9ZQ5+C
zsLqr!-@w4&z;D41H;zvGJ@M*+y^aRYHcS)aS#i;!=5FJ1sYA^Y1xL(h85@KeC2SCE
zI`NjDt;eKM$7zzz9PY#!vy58Wj82>okagnWfd-ygp@%1DW%X+h&qC#;Y%VzmkC{(g
z{r|s5_kV$gi68B=n2!61Rwzkqwm7(od$VYh<l>CLZqbC^kUyt3-0Da(H4<-3c-h?8
z7;vz`bm{{y`2s;RGd6Q`Hd|YvsAW(qv_PI{YnPUi_TI3`=s*{5MOFiUkfJ;1uUi~F
zK`fJ8IyCe-7;FrCP1BP%eatwREX!=hHcKaL>WOU$VN-Z`ZUnI%JjEJmC?ziG>0uxu
zI`Kj=G!nE5B_uqmD>Aq_Hy&hp@bZB@oA|~fY?+4-)-UPR3Rs?OqO)O}@dU;&%`;9-
zKNhu0I2cqebk)6fq)T7(!4`?da}o}CnC}Ql%B^&`F>4l&o*qw4jmu7`*>fy@{J2q5
z%WGm%F>^*hnbN_V9)kT5s;yfJUh7}lrJvHvbE2sw#?ePfLI2lE)(v8@JQAABn{OPr
zwW(3EaWO~YfsWD`Q@%}sJv}^Y*6<uX$}?*gkBm&qUQnw2|6gg2#hNuDfB$Z^v{d}I
zz+6ATe4V5DA!YrS&fexLG@1E2^qmTpa{3r=)KX(h(B=7{aX7R>AfGjEX~l6LPBT*j
zL3Va$7nbG)3W=+s?ww%a;<|Is-rm&InFh1OBLrSe3*Qp*Y5qjPTz?&wgk{WY+7@d%
zBpW;kXmD?_Y1kikz(;k8*-C*l!HISf1ydw>Bsh<r>~xHuU+@rappUQU-n~x~-W`~y
zbKzW$OIC5hy05Bij*L0xno+;o7>}t+Pgp1(9@(`2gTv#bTi#KZ9R1{ZEN-nm(e+V9
z#L<y0KmU*}KO_+xnr^XVNkvWV*CR(BRQp^IV)?Yp@XOvL364lE<9&ik_5c6X|NnP<
zbNc-K_5c6Ay1IRP{{H_zn{Dp*OzZh@MRN*63@=BM*2jz%BZb#T@9$0i@!_F;it7FI
z^N;B5Ja9SsplY}1#!W^Nl7=&87&x4>I=~9G)x2iiQq|`V6^}AVe{xzYB4N1g<l(Ca
zgk_ZPTwF1+YuXHjiNE?AAO7E-zh6J5qUz0!!?xA$=2(4gnLXRrug~A#|H{3%YnOt8
z{QBA(87(a~h*|zG^bQm>NRsGQYr2>C^Z)<<`|El1<Nxih`T70l=l$zqe}De+5;PX1
zscFc@X7u`n!HLtx1%-m?=4TTAaPxkAQ<#*JGGSukhK-39Wjy!8K-rw(|NlcXEm~SF
z=gghz>hj<yo3fkgArAG4b2#*!Bqb#zHo88se{!Ya4$IFUACI^3hELPoUGU(*-}wFi
zj0_C+)c=2ca<c#P^ZPAoer)XSuE|)jYUWG}Zf={c57~FjGCC2nWA^`K4P}YHJB~;i
z`u_Vr|KauZ|F`G;jokX`_O-S4`FDOWTLw+)?0opdv#s>?zp5&u)^=knYvYbCp-*B*
zVwUVJ<k|3~X{Vhe|AQwf1_lWiE+$&k$vh8IfQIl)3lY(yZ{HqG6lGV=n&sdv@8hB5
zWAK&7gWL6CV{6QQ0|oaR{qp;_=j^<9vGBvQv)3~`|NZ`MAHIIy&iem{Te<a5pYnP0
zX335nGbWU$33ITma6OSTY0CqxMhU|s8<)@f|6g){-QV(OXV%Y}<)423pVPe%rj@>R
z|59%3_;`P=HBbIM12;D#4o<<oC^oh^Sr>$OZ44v>-`6Lmqy&VfUbvoaQ7e`H2<p~*
z&mKHU*|ufM<mt~-9D3Rh8Qb1Uw0hAoQ82e$>RE>7Grf)fr|a3~XWgoP%vN=(p*jEl
zKN;(9Yj*s2FirQnUG1;dUCiI?*$xS17!*#L)p#V_Gh((Q&;S3%H9tP&AMca*^nC9>
z?@w#??u-Bb^MtQ6@R|9k{&3rWmwCb6TW>aM_ATFi`uOpLv||d|-It+*s)y!TsHn2$
z<R0C*V@AMZq2e!}<&~BHOt6o!7Bk%Wf1=Q-dbYOu`}_ar<lirU&etg1TlfE8`R{M-
zpFS<F{$}&*1LMwBN8(!^_`98$*cB=KkiErvb^N|R_5}~zU5~c6^ZysR<5Be6>~It7
zk4;9qzTS|U$K%n;6;&)clh;vAjc3*@i?g74hX4N$O|qCVLn1Y`b?Vfg5fK+I9jV_^
zaa2R`-$M_L9aj1QB_AH_Jv)2-|KIKwv;Y15{r}?P^Rlw({4y1v-b$zBG9BEJt{`&u
zhqhGA$&P}h-#$P8&&*!uGvmY0cX#cNd~yHxH`$_YPU7#fHuHQA?KXxrk0odD#55}%
zI>eKe#gmi669Fl44ow0TxpU{Ls;JxuY<?hrY2TCj6?gt;gmNBp)UN-yWRiXTKfl>!
zf8NJG;x1_0oF2bErt<lfmD4S_inmxCzWU%mPwbkFePLz~j3n;~xi2_9y*~HuueTQz
zg|@Bx_xJRMT<OET(o5zpZ;*WLl9ia0^x;!dK+vIepezUu02Y=VrY0^sc1lVF_`P<z
zUD~&GS;G>ZHJ*&CAAn+^{N0|NH8Y<Ci<G~s*;Dj%Zut6r|Ns8xk2Y~;n75}d)Nol`
z%uAknrDV_`;NRc-t;=d|@0t0;ceQ!GQOquvLuuh>1cTU|SlNnRUA%H+=Lu$T1@Gi9
z<Kxp)RJ6y->yO9urGMT1R5OyFY%qGP?-u$=e44_)BsGpF(~jTV?5`jjKIuR^|NcIi
z%8iAOo6pa$PhP^nwugr;=VRCEl0*m5<01#Q%xY|GY@9gp<BN;($NT?ReCtVB|M%C?
zf;SR}m-{byc__3f!OZl+4jmnyZQE|NM?i9glfO((&Y5Y`gf3k=^I@NMkntn?ZFl;M
z`dPA?esr%r;9#l#<^TWs_3`yTe@>obyTAN>-7K4*uMW4*|NlSzK%kStamk{<Zf{;G
zhn1m)(`{cIJqem!`2T-@^6~A(&rj@0dwxpPs6X-Ge362cUp^*Im~tdZ3g&egnKh?Q
z9h$)W%254Mfr;7wLs^seSKMA$Z|M8`^Yi!n>;K!=Dm_0i@%{b(ar^${-QTy|e_qYO
zSObHE?pU@QPNtn6heJIRQxCf?cyW7sefYXRI|?3N?-oyqv#)v1XIXvaLx|Fkokm7H
zmo7!{Kx$nle;FgAJ5f<gH@G+}G@I`1amZ^`aQ_o8)5>1u`04w5c{Q~*`}&TIQ;!*r
z_wC(R@=}(WnfYE4bGB{M&AbCk+4`^8hp_VS@cg&0{Z;<sL%(_ciTyKv9p|6aKhfp&
zkw<~eCsR{WHf&6M@XRe7RPp`)@8mDDYuBPrpL8UaY7{9Qh%`9R#k(ey^V+xL_I!M5
zadBannixf6_WgPH;^K7sItMMaH7A)?bI1tH=aUH7njFQLXuPNT`}YeA-@m^XSd#to
z`g)b@gI1T0HU#cppxMR{9nDr(S7Hi}$WNb?YHBR1l_X5Ly{y0duYXd+l-<m1S?kdI
z^wFY><^MlFzuzxc`}M^|@tljD`wb49`8z{u;?~D2O9H;^tN;J})z$d;eF<T<Dh#Xc
zJZ~-iBa$8M5+vjeEk2z*Wo&G8qN18sBsN|<;oz^aVI5ag{!{&BUOaCvsBy^4uWn!v
z%)VC8us;5O%<gX=pPlVjQR8?rrIDu~RMJJv3>0qn_y7O?>gxUY{U`iooSZD!mvyj2
zbaOp<8DpXu7RDAD`la<a)H9wkMMZzAs^&;UvMjZ2y!OVyPwD^v<ZB8UynH%MMMZmF
zy>d!vKXiXTC@N}SUaD5&VqWuzXUAn;=50)m<@uNeALZZs^Y_C;`O4*T4h=0V%{we+
zR&D$izHybKp6iYs5<I*YtwA%e4FCT-dCFY6)HG=l&kDt+&AObDZ=Bv8e<b(sP}ETe
z=8&jaw~x3qotcmT2C1ogik}<5(>NX_p5XQOu-M1{9U8_LT<z-r9Y5xFk;ze2)kv8A
zV9LRU#TQQ<JgA`Dyzd*NfOYbe5fb|I=FR^JX+Z*8nQLw^mxer+SAHD7#H2CO?}3zs
zy9X!d#_DSANy0(~2M!(j^5y0B<^KC0^`)%|?33BLW0vfRsDFvxyb+f!_sjqP@#BQ$
zZ6>~+UP%|16IZ-9tT<q0aQ@7h0|y)TtrCIcu8+;Dw8USXxmkbZ^{ENck{vVJuBdHL
zX|m+_Ccf3lF?-{`)Px_;UumhUxYTWa{Qmy`HEVqO<!X=4=KG)`^<>K-hb>7pQ$^(E
zf2y+U>o+e~Z)C0c!FX#I?;nPFGk*M(Y-tT#0*xpYK2A=*)YNPB|Nk96#koa^Me6AU
zZiWf$`zA~}Vc+wmvAa=*u|6_FqN}T?qT<Ax%XL3Lxx2Lqe+^=JvzAdQ=zRR1lT155
z9GF>>k~&jG#lxPLneEIOiJqR0SvH5c9;BaqsHJ{EN~VA{DT(LPC!RGP6XEv8#qk*$
zx;?qZ5XIZn_F&TlBj&9|KfBjXI54YZ!PDT7DYG^kze`oGU<8e|Jg$$7<Xgh}dd5!4
z`o7~m)i#+g^xHT)9DaX(K7XAaw_$W!LE|2qDH2Me4?IJE2+!%@XkMVeEF2UI>YD%m
z?_^+UapT5~e~Zi97xw)B|Nr2-YPXq*ic7crIDX&*`}d5O^{lM(u7wqUI%f0a=}uOb
zh^0#{4IT+Cb#n3T%V=epx1>4J&@jQ<JYdqypZm9&ZSLWjJVV0rEMImuo4&r$mxnAY
z(b5H}ckb}`_-JH7J?JN+^Z&oGk%C{I`*%wn%|ETV*}uEjR)1oTb~-pKBS1>2r|18c
zt*p0hpFR1(Uifl-eSK&C{kr6y;}6ycbC@X?T-C1+6kUAd#EBpOkN5vqRFrx2NJX8m
zw6txT8JpOxd-GcP9vtNq7WUeB@W276l^-lDC1=i5^iEk=o}D~5;qZz`gKE_sv*!Qb
z$hm07%xTL4BQqT?HCU;uPu{-0WrKnCCDX(V-oE-9J7#Qd$k?^v+vW#fMXo8aXGy6)
zug}g-<K`9<6cm!;6B6Q?I+f?zwIi<&aBc{b|Iy9Mp{dfc2-F4r|39h8d-H}rj~^r)
z3H!5%XM*!8iJZG3F>iTdjvlk`x$*1t!Gi(8i|^maaH;v5l&1FNrRKT?2RbJ{-ni)h
z4JC=SXAa~9CCus!TldHL>ev3>xAmT!yO*gMPdxsku-JIZHX|3;iN$NA)eiWuXRHm0
zaaOo=Nke@Zr0#7};|mK@b97Ym^JCk=ADDmQFZ*h*4Q#e)+#;))o~Sgf{`j%KyL;n?
zfa23)>WPU=b1l=-+?197Z85177f&=ZW4j`8$cHyVm@8~~lCN-t>x7#;C%A6h_@}L5
z;5YBg%2lUaHk6(`eR@NUrLz;0>)U`o491!Xx*jPh0acqJ;hof^{q#Y?qeo8N8}>_x
zUt74<_``JhdP#|Y2ci_~CrbREkg~BX?w_Sq*L;hIfiY_owNCY!_HuJ?Zf)_{wl$ND
zt?RIuQ?jn&g-4(4%~CT|86AraN)tWHa&rEgno5f4KKk$}>Dkkxa_MW<@Z7t{!_IH?
z<JY3=-$j^gjxsZEo5C2VsoQH*2(72yMDXxUn{O98bE;@nj>el00cvh;YHDgHzL~1s
zcp2*BaXiBCK%Y%|gYH%LxDEZ!thPLjIKlJe57*OwKR>@e+Wr3acKhgUZyw&<{NBF)
z-{BJ{6x5Erd!QgBto-<y+Non~;XPuPcifs?!Saxm!SL0_Ywj~%zWKku9#paKuK$1k
z?d|_^w!i*AbZ)EvFCif*siG<w6C?5Hk;Iv^k`)yloq3lI{QuAM|DWQzQYW{HBk_9@
zK7327_#=_U0V(Z{OxwQw&?Hr^4Jij(=5RbX$)wg-cTB5=(OAH^FhMQh$1{U9Dp98!
z8XF(p-(UaVuKwTO*Vp6g>cqmr+<J0<yST~KIVPoWs&P*_(ik)C&)?tgKR^E;zu)M?
z=gTUN3e0OX8NxR;oW8gJynxiH4=)6-fCkJ#6~~s}|NbQY`2S!2k5a;ojWZ@J7Wx1G
z|9O_~K(z>~1Sc={6Yj!~JtaSU{e6OAC&xh_`G6xwzRaILUvk@Bq54w}-CArxT>NY*
zY<00Gvi>D{Bzo$dIrE>DMS_LpL|1MQ!?kZL|Mv0;-1MIxkOdvLb5iMpbd{&ip3SHk
z^pNKXH^0L7MUHw52@R?f_pf-xIZfvOM3##}EP7^9cMJqh9bxHaj43?t;K8_VOVi1}
z+{NlPGx->P_<vcuXukd80!YK#Nzd}%zWV>|-QA2uBBpDKezNNu1k4usFPR|j=<rJ9
ziuGCpPm2ROHvbk+ijb^On!)Ivr=}*GU^-bPwnB(~)|CSclg!*}f3*|^L8C*Z@BRJ%
zd+Ps(N6qSNYy{2UA8hat-VogJuzcl$slpjb5uPRK$`gA$RyH!ahZqWUm9F7rHU76@
zR`XRKpQ$^}xOmI_H@IWQz~JA=xYF7n)xOR}@;M~WIH~mM$Jgzz{QUfXy;wvoqrSw9
zfM90ktX~IwRC!v0*rFabPE(TzI4lyL*{EGCd{ahIEmh!3-_17%eB_xKE=*!gYZtp@
zbd*Kje$kuhQ17est&gwYU;A79+O~#ik`vN2*O;E-*sVR8^PYdBtjXz)kmUvmCe{VL
zO3LDLCt}u|Y+_V?{O_=uN7*Tbh|>Z-mH#)W%2qM2xo)7ad1K#vrvuCVH98?>jFX;a
z-QN2D{f!UxnVE&N!dY)^Xy`OPBqXrpTVexGk;2_`XB)Pg9ek;&a?D=EY^7kE_N<R4
z_8+e(pQw`DAlbUQ?%*`TmrNVZ?K|{)^^b$g{dx9JFnbAib-#SQZT&y?XSPRjcV_ek
zM07QNFE!k4#v{?ha<D!`XlA#V;s3yHE;gki<9QwG`?efdFB_Z{z@T)QMP^S`Ymtit
zB=e~B#qa-TTg@hUzU}_0Z~U663seQ!Dzp@<DwyZ%Z<v^o$rOIZFri!bk*kEq`s@PJ
zDz;7GO>7=s!UZ4R$$I63##;XWcbfU<&xeP<kN3~{^M_}`Gu9x>gC`nFig|*%g=X<3
z1o23?axk+=-xXhx^CYKzrufEV0{^-iPJGj3ewSxp_%dyS3b#j-f^y`a|K~rv6#nI>
z1&wV_nfLeqSN;3LZ*R}Mq_WZQ(7^^1p;JdroM5n?!EJT!V1v=&CY}$|W&Q_BX1r`*
z)cn-ZuluLz#Kq2d$&X^ZVhxsheQF({s?5xw1v44$)1YDJEn`35=I7dV=Z-X7y5*qf
zq;QjaXHlcL77q{4hlK`gGeo4Acb5kUGcsotB)h3hEIi+kJf&$#i9=6sv+M#@4mP$o
zQ_sD={z4i$edpvUGvEGy-Jc)g%*@)%hyVZoFL+q`!^D7?WN8V_<`XH5r3PBe*(V)S
zwiqUKJH=!&MGE-pGiP-jNj{Xo{=-}FNyw*Prd21Qj`Ng>-}k5T#|QJ8U%X+P53FCV
zoEajb&Aj_};S|Y?Uf(SdKl^XR9GHKab6d2hYqRWxTmC=)&wu#T`^yoy0r&U+-&_9v
z`2GDh6LyN`_y)2?h<=vOR~MVyUBD(Mo+Kj2BT>r5eD8*8s7&@s<B-X$*V<SoO4a{M
zsrbh8*Fq5<`1Sw()PH(%eYW|36_pEYo{|Zj43`xC9zVb!|Fy}2WoP3d2I0oZKV*N)
zPwky4GC?w<S*GHVTy7TgeyKlhf?p0zU|%0+Xj9QJ&jM;<{PEvkUar5r{eMTdv9W<e
z+o_4+`G49)B{EVunU^p%^`2Ffuk_m?{QLFw|HA6~v@|!KJh353g@Y~VlgYn_Tx>yE
z!faPcobx|?*FXICH_!inh8o(z=_d{47&O!5>{52rO8=cu1G8$!k6&NsZ%(hT{r``D
z(*cQXKR?>%JYs12XJqi_@Bjb$Ifhm-ALRQ?U(7h|C#WQCzOTQpE+F{xo6F1Z&)Al*
zVWU)QX|D0CEAc$V23r@eZIat?>Cvmuwl_I#|LTPJ=BV%=&|WaxTymbR^ZI^>jfXa^
z`g&^W{P*|&|NhQDOYOmJ!Gx1f<+GX`ShRIi74tg|^%#gu{Qv)d-S2PT54Z1s^yo~*
zVtM~xzuN^5Mg*%bvy8ZFcHkrv6KH<;V^iYiG@&rzN0I;ASMmhEU+NvfHw)4dJv7PU
z-TnW6KRwN7)7dTi(6CEx(+8tx|LbF{b2g^G`~JRuzV-LsnKK>z`+M8l8Ql#VRwo>o
za!5yX!!ov*r)Lu~BQmm#Hf%E5Q+KxDkA<kuVIPedcdEZj81$z>Ix2@IS=`(I|L-&2
znrtSo8wXGEoba5Tv6SQ5h2ygV{vG7qU;qF4w%qx>y_P+_H)qT!nAxyZ;y@Hbf^WqC
zgsV#HrkHn3Vr3~PGP1BX-muB&&7DpoUoW8p!P6EjEPpRyS9c@{?u&hOe}8{^I)8ut
zL~qt9E)2I>JJla?F4_-Tn)v7E`9rPp_xAmL{r>#gB}bZ+6BE}4Z<xmD5@yo8am4|a
z!a|-sdw7ob8Q$6Lz9ZXfgV{l5(Au)v*$+>P>wuQlGJpqk_SOHd{{HSgzx;!P?UPPk
zI3?6`r*lrjzs1M&x-9Ge|GT^4;pb0J=jY^D{P}Bn^Tq}-`yE|MHe31sUHL42a%;1c
zhqv*M9|m)*k8j8p|8+w|&*5Ip?d|zI>F161)b+ZGfy>TAQAL0D*Ze$xdism6tZBSW
z+b&B5EpnK(PJ)v|(EtDc|9^jff4|WAepJ-G&JKg;Pd=PFVX#n*<IE+d@LdO^<~-1>
zW2>Ikz#JFH79Y=MXV*5<SW+)mY1M}Wh1DFD|K~kC!ue*ZIK-lT?C(!buJ@n!C$XV4
zRADXenH>jjeQU`%f4tDcF64Tr@O^FV`*r{RtY6V_{o1tNnUd94nU()1XJ;MQSzmBP
zKjZ0<37%pC2hN;H$UXbu{rQ6bcFx(_hu_}*kvHMUe5L|bagN82-P_vvH)cadN)GLt
z`S<hl{qg($w7&6AOi2MXaxNVZyEQ?B`Iv#mrwy9StMuCH|Nnb``}^Vz9}aI$ulMr)
zdFYUZrR7Dp`3txbcXh~IPij!rOgX4HP2obr1_S39nZvEzJki?>3JMKNN(@e%NU+@7
zyyq6nBJQG2&2<+=pqVJH=+CdO|4&cfzrXHaw(uFf`}H|FE~-tXD;^(iw+~)c<L6OP
z{q<G8zW%u_CVF*sY&ofH-;~%x|Je8ZX%DI6IVr`$Z(x+v(VP0<Sz5&AwuM`!JbV{>
z;+V^uu9GuDfBu>I@K<Tu_xBr5L(^egQO)09@88|6kKg|<J^laj0}Y#2NSNCGZJ#bb
z;peaa{}q*ZdU_`K@aWpq{r^}0?alkh&HM6heR=%$c0CUd&-ZtKA3jZW=yEj?D$ZiA
zY?#B?*=t!~R|9IShOamJ^Y1LEq}o-=Q~u7t#ME%d?#I^mCDwjyy#HZ)zNFvWLzU;O
zAhA2oqV8|fj)I5x`{nB=hL;|=V9)mD=Z*gxrpo-6lvKPu|H19FfB*lS{P5`L{l)I}
zXXpNY``}>wgs#aGCpv~t|9N+}{f5mGF9<O*yZt&4x;a5@aUyHozmy;UWN$_N`QHEV
zRcPBJp|-EDCHL%+ICDlqMKv%>^2e^?nA3YNgq0c?7}WebQ}EyJoC!4QD;v(;-u~ah
zQt|n5!$aa_3Txjb{^6gyu2p@?{}r1z@z~cJ*;ITezrQcOTda0}$;<64G$t=qef{@$
z{e0Ws-~ZbmetCI+qAELcWM}e?X<{cHoRe)6R$nktT>kul<Htdxux-3N*Vh|sXd0e4
zZQRjmSWpN;GiDo~IOW6_mXe^bsPoi;10Q~yAO8LQ#qZ>akV@^)OpAT>|G&SyT+hh(
zgHN)8_Yg~DHP7tz?o*WF_y5~n_xJtgbWp?o|EsI}rOp4{+4c4Ny}kSO<Ns~mnyRb+
zyz1|-`j|WFY;0_4X>5^^%lzm6D=Kn|+NGk(cV^j(4XpM5jqdDXO^uE;jEFMQ&@_~>
zIZ^P_=<v~Q9>`)98ylWGcX;&Uja=NF`M)YHd&(&Ja7WeDhlkncWI<EzOpAH8zsnyU
zinp{>dvdeV&5h0N*#3;OH~wGHp73AZ`djs*Bk%L?|NH;v=l|E&K~B|IJ(`kw^wHzP
zX~~DR)Q*M3{+&8iG^%;i27`ivKgXvTeOTU~cu821%^^>ywM6yDPe}_)$&ykKVq%gI
z(@(P~Xn6S0xoy6^<h}jr760cwJj4o`dR2UsC(*EE-h6w>b8`;bnQ?)_4^$w|w3snd
zQqJ}ld$x~5Uv6T<7W0X(q}YBiT`GS5@7tFe>#8sO#>ULj(#rb!i+5UfZ#K>j5SYEk
z##s3B|M&O*|Nrw-KGX8#e}4IYoE!#^Jh*ryIGY%!+Wt9sem+n1Hiw%|P94@Vsi~4(
zT@pn_67y`6cT`M#cviYi-d^I$)xx+>9o!CKy$@iHnQkE=@$c`?!v{{C`tjfX^8f!I
z^0-r8>l8IJKl=4QEbNS$n$Dj;JVixkmMzmUGV<BLyzHN$fq_lUkMGBN_ZNKC((*}4
z+Hs1Lt>;n`W2a%zf!XFf9}aW8zn>lu+PZ1xOvxQP!Dxp>ORHqv-=rVk`K#90ddA*d
z?*H+sCJ$`(`^g;8azX(Xb@q({N7Ov*9{Ja-?f=NwSQ55qUaL@`iQNI+^|z8Nb|l{n
zw^-I$cqKJ3TQo?+;3$i}y<zC9S?e|{O<7v#x9U{K9KX=cLfJNEzW2S~&wSoD^M{f$
z?|;>an;Tc=%I{+9Zd%58A!q63b+_j-eX&)1!jl8Z^)oF%)mHIyd+FQu5%N#$-(2AP
z@hPYxdrITPiGdy+fe|w}uSK=p(dlH8v-|V=m1xL>nKK(_&TO13>pwer#`i$ZruzR9
z@9re-C~Srd{5!g`F85M;o-<L@+2!5c<b<?i=30>c{h@NPH?z%oHrVF=f1oUrA^ldd
zVd6x_vR&MV>%S;7NJvUZ+}r!R{?`}zu&|8je-F3+U$(60&6~w1nbbi`m%rs^WPbc_
zfB5zF8}C`$7>wL5%(qSc@wNLv;7@4cFaMz`FQ1Yix^#9!NsoNWEZ(ZWZ~p)1F<?C*
ze#ywd;16h0;Q9Ige)IltOLiuwr4^K=rKSGm-<>H_d8anztNRCrw6wPT`vTboDk>5M
zvT0sg4NI0tn@h;qs}*botxo#?|Il=cd3LSpC;xw#(pa{Z^Nkqy7H_8ii`5p?hp+!<
zSt}J)_wkd_i4z7NKK^x|)ziAXbHB$MLEe3}$vbML3%_nypm0)rhBEV$TP>hb0DI8#
z<$L?p1D-%VFx{f=Ps6gX8G`>O=$Sa4IhX7!aH46x-CygPANn<G-^{n?*|m#DNa%l{
zH+PEeVuq?R={?&&y%u+^7MZZR|Nnn*BKtM<!=*B~d7y^eKbfN$Pqdg<zH`v@PI$!g
zVbRmKx9hj(|8Jf?Vb`u6BO{(SZ~iB)K00A*KNHK34~$avIyyW_NfCKH$5=1Snm^-B
z{C<gbajJ8m&F4eYEhHo*>;4{+>1$0ZuV`($a^=VI6C9C@M;bze|JHwdbN^uTg=M<J
z!cV?@{r~^p!O0ghHXUHPw>P=shrle|$Vkb5e-E2a>13O*JbZ)S`J!UuIkw&vFO49H
z@p|;nZ@rFAOo@`rUj9$)iwc`=6(_pLt^M_Tdg9|_^&f&AfBckmagkWERPx4+2bmcn
z3!3-WOWxb>erNWVzt)Gpzvuap|NMZC?uVGOC7;!%^>WOBhD3PLpFbxHp7ZUh_2~Kk
zq11N6YO@R0i~j%r4|4Ei_5Tl4Z%o(c`SpvZrsn_58E&)1Ee`(w&!eNmv)oV7MMQ)n
zLop)ifNxqt!j3ZD`zbmyP-CZC{QK7`_woFJhV_x%SAID7%lGvB-(UN?{GH4pW-BY6
zUAzA8+~GEv)$YS<@xxoQc_vK?(36)xyxjkVbmT(@%dC!_Zjax`>pb6*0P4Vk3dr!a
z8#XBT6&_Djz0_hihr3+o@&5Y%{BnQlzw1pof9%+Tb>}|3m){p?Y1L5wU*g}t*0{{J
zdXbC;3@Z2cS291X`^Pia^B|YPRrY5uES4F0Y^eX;R(DTB58~WCbFH}=3<O-{Jvv%@
zJp^(d|MgFN%c_&3!=mcyw5!AHAJ;^YM{^P+<W3(D`^?r}Tq0X4JN3cAH+OGk>DKzs
zdY9_u)v`dxUT$9J6_bZDr<<lo95yiMJM;OSFUO{yMGID4kXrkY<5zvdp5?jgqZWu%
z`f=pnZctY$x3vqK)UvE|my9$xmbc7R<ZCeCag|p-Eh58kRP}=Kop?PTpNAaY0w*|G
zT6zS<SVF?Lob|u_&n%Zu<~NIq@T?syJ2yUK2yNXuUAolQIre#&!v0B@=M^5Y+>;(z
zz?W^<mgwL$#p=oumyW~tH@R@!nzqh4``0`HjjsFlOXdhMT|TmFS?>$s^McEJmvG$O
z?AkP=^4Rb2X^;|d%Z!B}=eWerICxIaG4Y4c6}y7@CXLcU%v?wLdr~?CIe0jj&TV^i
zDNcW<U8nNB@?Bp|_OuivvMCzySZv?_vt@x!{?sp`d*XjabZ`A)d)ejs!#j@4?%j`C
zEzqOv+iWPp8Wnt9qjieLSMj&Jk#6%YZ~J}c+00c|+hdnax?*|h)ogzEBTS%$-G`=I
z?5j)NQQUs;Ebm<-3x$Xxo9eb5ljnAJItER8cwL{(%&e{ce@2no$@}|x`sIxcjE(2m
zo!yYiefaBZp8NZaZECw#?CCUr{yzSN!GT_B9`ihdGcyi8czpQ5n==bmoLI2xl!Cf@
z?eW8)AsqL9qne*hE5ep6-Kfz#ook1bWQMPR#rKEKN$oz6#IgpKSR9YoP2cd9FKpfw
zVUN6o_q;(Jlk_HRx#$$wBQG+i=w_Bg+g#nU`9~%$bIIR+Ui^ChL$kbdn@W#!wKRNJ
z;k@_%&sq(ZmIbe0RBu_9nGxM`;MQ%<|JrjFuM6zjv_#`X@v5w=fA@>+*kHob^Ehsf
z+LAqKKlIkS&D(g;vZr$E6NWT!ia9jh0#uNFVA%W2Sx;qx=bBsdZ5&1J`uKpny<ziW
zbqg<sluJK9Cb#|lEh%T0Vo^UY@wVBE8yD~IZ`nGT%dRFNVsqQWGm>n&x^2$w5>C#N
zKE9Iw{;3++xC$jCB~^TEdiWyn$OqLW|NCdo{m`Fcuwa@@0cbtSA=aeTqLA<ax#6$#
zj8F1q*}Ba4J~1E75My(5V{>$TQE~eJe^6hLUk=oWRnYNiO*`H#4q8{dI?V9R{qr9_
zsUCiLndkjI$GN`2A099sZf55>b&6+qx$%+X#w~3|28M<wP6r?Vu-sqr+}z|H#q5F4
z3JVNYtP=98-@&%;fo%i_Xdl!)+ay`Yc;1}Z8#x;scwFZ<ttjZ-yR_S)S$k(sb;Ql+
zD7mDC*E<sm#k!U&3H@2Z*H|-Y{)C!|hd<9)Cavhl#PwN<W!?SgY6Iuj-{<>teq;D0
zw^ovERoAvunY)&E&E=+=b$JLqlDMB9J2Sm!;<P*KoHb8P+O<r>Blwfd5BsIjr#B_=
z_!iqvQ{Xviq<;6wJ-^+0PGzRbN5DG}=ghI?b14uxQs3gIDYQsntwkHN#bWIX){LE9
zVIhkqPWa)Kv+;kD_|4er#_MNNp6xyPq_!#Jex}s3NR#mMn?6>pjk)A{d4tZxgUnq@
zQeR@**Dcog{v|nfV(O(S%M{OdDmm_pmw7T%L6^I&@%`gnGt}A|mvyeXV`A9%#WvSF
zHh5ad)#)=2H>e067h!dDm?qeE!)(>B^Sk!uo-p!snzr}hbBl_58y%`X{PI3%(Jph^
z>Gr(}(J$b%)iY-!SF;08n?0YOIr~F4_T_r9bD~~7{*h+aa>4V(rG?I{)}`-NSlROf
z9=hCG7<BO3Ez=f<N%kr~IU`rx-zBG&xoO`y)l#K$nJH2S6M0-6XKjjGY`SiDPtWxY
zD@}8~BPXdG&YAd}W8<6UQ#4w0lW)H%Yf*G;c+Y>Ob&B$i-}BZTmgrgdddK~pF(-`p
z*k`dR?sNVi`sH@7McVGC6)Ig3Yd_db`Q@7C-(w?}aklRK%OhW{o?YE^BWIudgmWJk
z&j0&#ZDi=2^Z)lXG0uZ{XYPLPrT~HC_6q#$(M1L&9O8;CYVA@|f5ne%tl#*Hokjd$
z%$-?L(sQR&B$)kJ5Fd57;&gSv{O@~@7vA~X@VIJe<_sS>(`n{^E5!KiT&`{_jO}xG
zjksR8v%qTOZ9&%%(YD5Q+DsKn>+Kh<Gvw)F(mYwr$Np;8rjTBi`@S4D{FjduMy$P&
z!@+bb?5<14#iJi<>t>$+So<jA?6sTkUs^4Z`FrYa-V@7du9G(0-4wAQW>ef{6QAW<
zbZ43US+vB3<6Qct9u-C3_$Kq&f^CaiKRX)jtN3Ns@c%6zsD$j9wV$ggK;&4x*fCia
zLB*VmH>`QhhhuA)r?=f(<6cnwic`s{<%6VCL4lE|Y>d>dfG_9I{m^Y^wzs(d(?Yr6
zis|(3fA#fw-wIFf@wc1*__xbN=apK1>MaX0Z{+QaE0odcUgykF`{G=h2<v3dO21h_
z*Jb)sBtPUQJ^Z~UOh?S4Ge~DTf0_H?$je8zZOnap+_v8!WwT<Rr_ilwHvhZQB8}#5
zx~-_Xt~-c@v8{2+7u$8G4H9O}3Ou}El|Ylp)0$@m3H$3yFMDl_tAF|<?)<k$?My12
z-`><;;{|zc%IuBYO$h==>*pt3eDdUph^47!<AspfCg19}nYwf<=!vTBu&BHE=z@}7
zSo4Jm8lHiBKOQ{zv7(^g`dzo>sY$>0o;CF^Ih5J{G9tw))#zbG&P*G14h^m)M@kKS
zf7Ry9wr~r*XmUwJ(L;!jy(MA0bI1F3rO>6T3S!fY-IUv$W!Mi_y!@%J;kDOjy_Oia
zP5;%dvuv6ZC+Sqjf1P6&*s^Z({mplc<Tl3{&oADa6gfq8*0F>GKYiA_OBC&V@OzHm
z*SzhSH#SA+-Hu<z*83^{@-g*iWvf&F%R-v|N-}AZl03e?9hN?BZY<JzO~>5W*2f$B
z%szJEW@jC@K+3<S-d;zqF2}fe53g#oadNih-v{+71VmDHR0vw_@jbJN*`wS!Y?`9Y
zoP+238vFYl11CC0%xDQvkh7^cu%TG}@YU5k;p>gw+;)$!6&5a8_wlqiTXuHa_I$}J
zS0(LglM)gWLA{7esu^hx-Dz%aprNiSA#Ja+<|xYuuxMufOU}u1dggTF`8mUs6a$+I
zho*E;Kk@(nIdcs8Tml5z>`yAIIEnl!=S&aC3eo#;?its*h8LbDi>`LD+)Rq!IPJ}?
zzrSaGn*GPP{@Lz@GHlaWk6W&>vP*cJyQxjl(<wGsq`mg(zRTY^9B;qseqoXrX*H=n
zcv<HZ3ARkHX%8!R=s6Z_|HJRu85ANK>n<|!dC%f=3^7^vBgNisnJ?dYTIhCK?9B9*
z9cAa71deO8zK`Etd-T{$`Sgk$Y2VM%W(|;mQ>8f;3Ubcx^o$QgEt_FG{r`=$2PUSH
zixx}9?N75PX0z2%xHI2=;mfuq7xO%f<6`F?yg%PDdiKNr^;vy;AF->c@n~uBSXuGp
z<?(Qf8N9i7_QQYs!>6aWm^%Gh@j*){`&2?|YC&P*iZv+{CLa!xe&oef5hcgN!xO*X
zXpV*B6%$)fGoIo9e<iI#o0<~~*7Z#ixe%pz;wW3%>^XDiDk?NQJkH-%`B|b*UOj6j
zr`~~+oO5q|{hAsOmUiJvih;TMeG{*S28QMahQr7EdD7B&q|J?TZXWvZ%k=O<XOW+M
z6DKy#oT;eRws6A)$5>hK+me>73q<8+XidE7Ki_DMmFuTO==L?GRdd4v4+ey^+<SB2
z=z&|G8XBD2C7zv0ys=k0HLO#iVT(y?tK^iaps|dmCJ7r`NePLL*jATphL`R3m4^zx
zXdJ%0oF{s_QOu452M#zy?Qdvo1f@8amW9jO4jyiH^x-*eAeF|+m|y(hmMgE||Ih7;
z*0U4uT20yuS;(iPRr2<YMA-T^HNK{kOy`br&9hIxv8nap1;w_~*Ag>kNbIXUY$wVz
z(?(b*>A|zK2TxK8ijxBZ6DLef{qQaA$M5NlQ>UJ}SkB>aI^D%jRFGNDHYp(`<;L#m
ziT~@`&d-<RlS}#WU;ftP<fKE7f<3l}1xz@2Ws0M7Gbmu3n9|dwv%e(!@@Cu#UM{g|
zQ^(vy2avNE{{NpdSCOyTLBMtXfd+?$+srzgERDwstll-=C@x?Uk`ZTMVLINF7<|#!
z+4-{5ei_T3KWFawlcKlQkz<OIXO7{UjHcoQnX?}YPaiYr%jgXPw{>rJ#EC8Hs6GDu
zq*08xrY6_6vrXsre-)nfu`W_xefyU+DcQcaA6Z=4T3dA4q($*Cg8<7J<HDZjimH`;
z0!v<+zo<JZnZ8)!@UEa~Ay;_~eqCY)6@^NtzU%0K_GYoNUg!*qv5oKu7Fp}xZC`VL
zPvVbXQybgcL1Rw!|3L#Q_x2`Fn3D3~S=x`kvmISnr1YkQ$Y*TTH#(52v#j@tCBvpo
zFE6XNP1l!nah24IO?vP+`NzMx51;$9nVPoo%S*VpNE|uBp~H7HYW4xvk0IIu2Ol4o
z%*v8j7o(_Ey#Q1KfYS4|%g2uKoStUL!6_*5@PG>s^G1j1jQO(;%)cdJI4v<Lb@D^G
zPg@>)i5<D3GvCgzq}*61!od%*rbVf&>F2A^w#UaeTJxQsX>{V>Len4pw_1}O-kwh6
zjY(mfWm&<a$UkACk-=dxJwJmL|Np1|_&xpM5ze;K*C%`=;X_4A2T!hA#Z&#wprBAt
z&rIPfciKb8#Y!yMZyu#jekHLZ$%jps;UJ5bPxIrO#%(tu+D=bDSq_T&|NoT|Gf%y}
z&10PAFx&D+u;IZ?Elv&>3Sx4asy4bYFmIdG=H{wgcU&OY!+FZzzo#$U^b9<e0Ne4b
zlzaD56DyB--h>Gi4>dVws+g{wFe~#zyZ)?=5l1BhvIUy>4y3Y9WT=;T<i8^#_5a`G
z8Rq?ox6LNSHi0Y$g`aMrP5rqG_s@TL%+9%T#sV+4oT=I?+XQ-aV@`cIA|MxgWcL&M
zd!N|OHb-n0m?)Q?;E-{D=j6n9F_Y4R3_x-4|G(1CXHTA{YG@>!IO!-UaQLd{!JOg;
zS$~pcJCDm$ID|FFvjhwDdMD0i4xT2MIaBE41N%$A)Xq$4QR~odHB?yIBdjj5YPICL
z*u{sOpxbzrDi?hGoDM3Eo;;PTsg;y4m{#}qxY@_HWetp(J2-WH7ic<*9hn?f!1+k<
zxN7T5xtM86>Z~T^Li`tkn8O^;m>g`-t#t@GwlbKfx0mPo`rvd<J8%<BY5Cc+Cr&I_
zapJ<&qY2sFOaCM!I6OBpSTJ3l=SElW{|a}e6W%fvOE{k8`+Q%pQ*!Mbg=6A7Vp^w3
zewZ%Fb3^F({rx=d{YECHhB14+-Y``)LHZv`o1KrHIH90^{KD<i3CYZND|<{M6W*SB
zBpK$!9e-v~t6*uM>AEI%-Us?B9V`*S#?1GQG#*zzk#~4an1sign7Y47H@5XIT-*Eb
zqH@-|XONkPMJu-JsU5p;<;a8Arw<%!+kEz5gL0~Z@rGw+F;C587F-ROA!R74y!s%s
zf$xE-9fr0|%NTrB_1YIs1-X-bozdgZ>7Z@9%X}9*af^TsG+_Aue^H4>dwaX0@nl8Q
zsf<O&+}Bi^+Ff=uPcf2YDp2KXiZaM`FJ_ySWKwwM5RYEp)}>B9Nk`+?8XVa6QBhs6
z<HX)-p0{s#qPGXF)pCTUo<)Z&<h8Y%Hyb;bnKjRpWL8$L5<Tn?)69I-w<khro`hK?
zL%7qysgpW9_wDeqHkmB>AxxX6rK|U7w`AH`r#si2OCce+=$68UEhf$D^+D$BuxOmc
zI=Ne?<MP54KPM&FI`JIHSB^U9utYD$Ez$Q#L^l`PotMmUT`{|lUAT0p;GN95ZrPQf
zPy>1K6HntD8RrTM=P0R?RGojnSYI*SQD)!4aC2p2h4zZo-Al4q{bzoB%noXkXorLA
zuz$9`enPNvc+tr(6MJ}^0|cB444S*Rl1?i|X*EXmFOKYr+&n>Cdk2H#<$|U5XWTgD
z7ITP6@K~JUVe4Ufd|kgS`MAWjHHj8gQy%^-wwk*GQt2=9d<0q-nwIw9adJR-`iCzm
z1!ZXgp+~R(JK!T$A=NV@(8=wRVuaB_-(!mfS9^cRR$nRm>%aZs`};wscf7fKdP9X_
zQn=<_Xefm~a$uXlyZYTb9yd2od!VT3!q<X`-CH&Kl&1tpeAveGA;k8`moN!~M?$N0
zuJQ3HAKRe0jDxvPiRI9bpOX`x`?XD0m)w@us<&VXY%{6*qXuR(Gd4XvHbq4?Z*MkX
zVYbXnHdR%&$jB)!4O=Y|6-54~gjgNfV$$g;!?WU~h_-{C^o3t)JRT<>J@<ccvn+a>
z!JFI1KYUXC`MP8UEEcjs4n2ENMuta8iAP9?M@@}q&Kw>oDIPbs3wu3lYF@C%F4}0w
zE&6CO$3&@*>U?Z_UNUF<OK(2OnRsXC;SHI>ho@@uOxHKwQ+xWu-`@=LA9I6>8qg^-
z8;&%xJ^1PA(9k${uA@(nW5kTc=4MCF&VzU6G`6=lPMqi%C%21j?nJ?=6^xt8)+|f(
zO5}Q!ANc4}t7f;izS`sa_Mlo?FFtKY{d~|V6^Gj;?dsKjo^S@uw1Fx(DV<Jc{s#{d
z64KKXl9CeA(*=Z*5>ityTu8WZxp`gLv!|&OCMQprl=R@4TPCNFrCp<XYQkEb4_6c_
zcB$Oo4|efe>BF<ldD{7nWvq{Z&UyI#{X(wwl=9P=pb0dF|NpPdHRNJY<Y8g=(aHVu
zzxq$3!oP!FTefygm1%!^w?^@Fp7~El3%SC)>z!JP0?MmyuT2Y6zwE`RRIxu-S1G@?
z_^{ya$LrGcx7Ocw&|KoOw#V_@mSc0i{OdfqBasp8g;kx*{0S*30>a4&$;k=n=?@+y
z35cdnn3xE1`sCyZ)6xV)Qy)BPnpeSTFT$Fg%oO?IzI~g0{eosc9i0n1+D#Pa+oxB2
zYI*oExlP`Ffw<6}U97fJ??0!%@^8B+3`vZ4<{EMxP!MsJ=W1F}@n86q{J#eef?Tab
ztV&;o&C+D&(2xFfxWMLARFINe&rO}3dUKLhSmr3F2i>~;{NCd`&(7xdHAe-ywgd$%
zHNBSP=klUf+~s}G$GgiamI{OYpxMdH-=M-FCiddv51tLeArH@Tb0npt{P;Qf;cfG(
jeivQ^h&u?LfW^#kCBJ6FvfZ0pK^i<={an^LB{Ts5;0bTK

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/image/python/ops/dense_image_warp.py b/tensorflow/contrib/image/python/ops/dense_image_warp.py
new file mode 100644
index 0000000000..f9b219ada4
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/dense_image_warp.py
@@ -0,0 +1,201 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Image warping using per-pixel flow vectors."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def _interpolate_bilinear(grid,
+                          query_points,
+                          name='interpolate_bilinear',
+                          indexing='ij'):
+  """Similar to Matlab's interp2 function.
+
+  Finds values for query points on a grid using bilinear interpolation.
+
+  Args:
+    grid: a 4-D float `Tensor` of shape `[batch, height, width, channels]`.
+    query_points: a 3-D float `Tensor` of N points with shape `[batch, N, 2]`.
+    name: a name for the operation (optional).
+    indexing: whether the query points are specified as row and column (ij),
+      or Cartesian coordinates (xy).
+
+  Returns:
+    values: a 3-D `Tensor` with shape `[batch, N, channels]`
+
+  Raises:
+    ValueError: if the indexing mode is invalid, or if the shape of the inputs
+      invalid.
+  """
+  if indexing != 'ij' and indexing != 'xy':
+    raise ValueError('Indexing mode must be \'ij\' or \'xy\'')
+
+  with ops.name_scope(name):
+    grid = ops.convert_to_tensor(grid)
+    query_points = ops.convert_to_tensor(query_points)
+    shape = grid.get_shape().as_list()
+    if len(shape) != 4:
+      msg = 'Grid must be 4 dimensional. Received size: '
+      raise ValueError(msg + str(grid.get_shape()))
+
+    batch_size, height, width, channels = shape
+    query_type = query_points.dtype
+    grid_type = grid.dtype
+
+    if (len(query_points.get_shape()) != 3 or
+        query_points.get_shape()[2].value != 2):
+      msg = ('Query points must be 3 dimensional and size 2 in dim 2. Received '
+             'size: ')
+      raise ValueError(msg + str(query_points.get_shape()))
+
+    _, num_queries, _ = query_points.get_shape().as_list()
+
+    if height < 2 or width < 2:
+      msg = 'Grid must be at least batch_size x 2 x 2 in size. Received size: '
+      raise ValueError(msg + str(grid.get_shape()))
+
+    alphas = []
+    floors = []
+    ceils = []
+
+    index_order = [0, 1] if indexing == 'ij' else [1, 0]
+    unstacked_query_points = array_ops.unstack(query_points, axis=2)
+
+    for dim in index_order:
+      with ops.name_scope('dim-' + str(dim)):
+        queries = unstacked_query_points[dim]
+
+        size_in_indexing_dimension = shape[dim + 1]
+
+        # max_floor is size_in_indexing_dimension - 2 so that max_floor + 1
+        # is still a valid index into the grid.
+        max_floor = math_ops.cast(size_in_indexing_dimension - 2, query_type)
+        min_floor = constant_op.constant(0.0, dtype=query_type)
+        floor = math_ops.minimum(
+            math_ops.maximum(min_floor, math_ops.floor(queries)), max_floor)
+        int_floor = math_ops.cast(floor, dtypes.int32)
+        floors.append(int_floor)
+        ceil = int_floor + 1
+        ceils.append(ceil)
+
+        # alpha has the same type as the grid, as we will directly use alpha
+        # when taking linear combinations of pixel values from the image.
+        alpha = math_ops.cast(queries - floor, grid_type)
+        min_alpha = constant_op.constant(0.0, dtype=grid_type)
+        max_alpha = constant_op.constant(1.0, dtype=grid_type)
+        alpha = math_ops.minimum(math_ops.maximum(min_alpha, alpha), max_alpha)
+
+        # Expand alpha to [b, n, 1] so we can use broadcasting
+        # (since the alpha values don't depend on the channel).
+        alpha = array_ops.expand_dims(alpha, 2)
+        alphas.append(alpha)
+
+    if batch_size * height * width > np.iinfo(np.int32).max / 8:
+      error_msg = """The image size or batch size is sufficiently large
+                     that the linearized addresses used by array_ops.gather
+                     may exceed the int32 limit."""
+      raise ValueError(error_msg)
+
+    flattened_grid = array_ops.reshape(grid,
+                                       [batch_size * height * width, channels])
+    batch_offsets = array_ops.reshape(
+        math_ops.range(batch_size) * height * width, [batch_size, 1])
+
+    # This wraps array_ops.gather. We reshape the image data such that the
+    # batch, y, and x coordinates are pulled into the first dimension.
+    # Then we gather. Finally, we reshape the output back. It's possible this
+    # code would be made simpler by using array_ops.gather_nd.
+    def gather(y_coords, x_coords, name):
+      with ops.name_scope('gather-' + name):
+        linear_coordinates = batch_offsets + y_coords * width + x_coords
+        gathered_values = array_ops.gather(flattened_grid, linear_coordinates)
+        return array_ops.reshape(gathered_values,
+                                 [batch_size, num_queries, channels])
+
+    # grab the pixel values in the 4 corners around each query point
+    top_left = gather(floors[0], floors[1], 'top_left')
+    top_right = gather(floors[0], ceils[1], 'top_right')
+    bottom_left = gather(ceils[0], floors[1], 'bottom_left')
+    bottom_right = gather(ceils[0], ceils[1], 'bottom_right')
+
+    # now, do the actual interpolation
+    with ops.name_scope('interpolate'):
+      interp_top = alphas[1] * (top_right - top_left) + top_left
+      interp_bottom = alphas[1] * (bottom_right - bottom_left) + bottom_left
+      interp = alphas[0] * (interp_bottom - interp_top) + interp_top
+
+    return interp
+
+
+def dense_image_warp(image, flow, name='dense_image_warp'):
+  """Image warping using per-pixel flow vectors.
+
+  Apply a non-linear warp to the image, where the warp is specified by a dense
+  flow field of offset vectors that define the correspondences of pixel values
+  in the output image back to locations in the  source image. Specifically, the
+  pixel value at output[b, j, i, c] is
+  images[b, j - flow[b, j, i, 0], i - flow[b, j, i, 1], c].
+
+  The locations specified by this formula do not necessarily map to an int
+  index. Therefore, the pixel value is obtained by bilinear
+  interpolation of the 4 nearest pixels around
+  (b, j - flow[b, j, i, 0], i - flow[b, j, i, 1]). For locations outside
+  of the image, we use the nearest pixel values at the image boundary.
+
+
+  Args:
+    image: 4-D float `Tensor` with shape `[batch, height, width, channels]`.
+    flow: A 4-D float `Tensor` with shape `[batch, height, width, 2]`.
+    name: A name for the operation (optional).
+
+    Note that image and flow can be of type tf.half, tf.float32, or tf.float64,
+    and do not necessarily have to be the same type.
+
+  Returns:
+    A 4-D float `Tensor` with shape`[batch, height, width, channels]`
+      and same type as input image.
+
+  Raises:
+    ValueError: if height < 2 or width < 2 or the inputs have the wrong number
+                of dimensions.
+  """
+  with ops.name_scope(name):
+    batch_size, height, width, channels = image.get_shape().as_list()
+    # The flow is defined on the image grid. Turn the flow into a list of query
+    # points in the grid space.
+    grid_x, grid_y = array_ops.meshgrid(
+        math_ops.range(width), math_ops.range(height))
+    stacked_grid = math_ops.cast(
+        array_ops.stack([grid_y, grid_x], axis=2), flow.dtype)
+    batched_grid = array_ops.expand_dims(stacked_grid, axis=0)
+    query_points_on_grid = batched_grid - flow
+    query_points_flattened = array_ops.reshape(query_points_on_grid,
+                                               [batch_size, height * width, 2])
+    # Compute values at the query points, then reshape the result back to the
+    # image grid.
+    interpolated = _interpolate_bilinear(image, query_points_flattened)
+    interpolated = array_ops.reshape(interpolated,
+                                     [batch_size, height, width, channels])
+    return interpolated
diff --git a/tensorflow/contrib/image/python/ops/interpolate_spline.py b/tensorflow/contrib/image/python/ops/interpolate_spline.py
new file mode 100644
index 0000000000..daf8c56456
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/interpolate_spline.py
@@ -0,0 +1,291 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Polyharmonic spline interpolation."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import linalg_ops
+from tensorflow.python.ops import math_ops
+
+EPSILON = 0.0000000001
+
+
+def _cross_squared_distance_matrix(x, y):
+  """Pairwise squared distance between two (batch) matrices' rows (2nd dim).
+
+  Computes the pairwise distances between rows of x and rows of y
+  Args:
+    x: [batch_size, n, d] float `Tensor`
+    y: [batch_size, m, d] float `Tensor`
+
+  Returns:
+    squared_dists: [batch_size, n, m] float `Tensor`, where
+    squared_dists[b,i,j] = ||x[b,i,:] - y[b,j,:]||^2
+  """
+  x_norm_squared = math_ops.reduce_sum(math_ops.square(x), 2)
+  y_norm_squared = math_ops.reduce_sum(math_ops.square(y), 2)
+
+  # Expand so that we can broadcast.
+  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
+  y_norm_squared_tile = array_ops.expand_dims(y_norm_squared, 1)
+
+  x_y_transpose = math_ops.matmul(x, y, adjoint_b=True)
+
+  # squared_dists[b,i,j] = ||x_bi - y_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
+  squared_dists = x_norm_squared_tile - 2 * x_y_transpose + y_norm_squared_tile
+
+  return squared_dists
+
+
+def _pairwise_squared_distance_matrix(x):
+  """Pairwise squared distance among a (batch) matrix's rows (2nd dim).
+
+  This saves a bit of computation vs. using _cross_squared_distance_matrix(x,x)
+
+  Args:
+    x: `[batch_size, n, d]` float `Tensor`
+
+  Returns:
+    squared_dists: `[batch_size, n, n]` float `Tensor`, where
+    squared_dists[b,i,j] = ||x[b,i,:] - x[b,j,:]||^2
+  """
+
+  x_x_transpose = math_ops.matmul(x, x, adjoint_b=True)
+  x_norm_squared = array_ops.matrix_diag_part(x_x_transpose)
+  x_norm_squared_tile = array_ops.expand_dims(x_norm_squared, 2)
+
+  # squared_dists[b,i,j] = ||x_bi - x_bj||^2 = x_bi'x_bi- 2x_bi'x_bj + x_bj'x_bj
+  squared_dists = x_norm_squared_tile - 2 * x_x_transpose + array_ops.transpose(
+      x_norm_squared_tile, [0, 2, 1])
+
+  return squared_dists
+
+
+def _solve_interpolation(train_points, train_values, order,
+                         regularization_weight):
+  """Solve for interpolation coefficients.
+
+  Computes the coefficients of the polyharmonic interpolant for the 'training'
+  data defined by (train_points, train_values) using the kernel phi.
+
+  Args:
+    train_points: `[b, n, d]` interpolation centers
+    train_values: `[b, n, k]` function values
+    order: order of the interpolation
+    regularization_weight: weight to place on smoothness regularization term
+
+  Returns:
+    w: `[b, n, k]` weights on each interpolation center
+    v: `[b, d, k]` weights on each input dimension
+  """
+
+  b, n, d = train_points.get_shape().as_list()
+  _, _, k = train_values.get_shape().as_list()
+
+  # First, rename variables so that the notation (c, f, w, v, A, B, etc.)
+  # follows https://en.wikipedia.org/wiki/Polyharmonic_spline.
+  # To account for python style guidelines we use
+  # matrix_a for A and matrix_b for B.
+
+  c = train_points
+  f = train_values
+
+  # Next, construct the linear system.
+  with ops.name_scope('construct_linear_system'):
+
+    matrix_a = _phi(_pairwise_squared_distance_matrix(c), order)  # [b, n, n]
+    if regularization_weight > 0:
+      batch_identity_matrix = np.expand_dims(np.eye(n), 0)
+      batch_identity_matrix = constant_op.constant(
+          batch_identity_matrix, dtype=train_points.dtype)
+
+      matrix_a += regularization_weight * batch_identity_matrix
+
+    # Append ones to the feature values for the bias term in the linear model.
+    ones = array_ops.ones([b, n, 1], train_points.dtype)
+    matrix_b = array_ops.concat([c, ones], 2)  # [b, n, d + 1]
+
+    # [b, n + d + 1, n]
+    left_block = array_ops.concat(
+        [matrix_a, array_ops.transpose(matrix_b, [0, 2, 1])], 1)
+
+    num_b_cols = matrix_b.get_shape()[2]  # d + 1
+    lhs_zeros = array_ops.zeros([b, num_b_cols, num_b_cols], train_points.dtype)
+    right_block = array_ops.concat([matrix_b, lhs_zeros],
+                                   1)  # [b, n + d + 1, d + 1]
+    lhs = array_ops.concat([left_block, right_block],
+                           2)  # [b, n + d + 1, n + d + 1]
+
+    rhs_zeros = array_ops.zeros([b, d + 1, k], train_points.dtype)
+    rhs = array_ops.concat([f, rhs_zeros], 1)  # [b, n + d + 1, k]
+
+  # Then, solve the linear system and unpack the results.
+  with ops.name_scope('solve_linear_system'):
+    w_v = linalg_ops.matrix_solve(lhs, rhs)
+    w = w_v[:, :n, :]
+    v = w_v[:, n:, :]
+
+  return w, v
+
+
+def _apply_interpolation(query_points, train_points, w, v, order):
+  """Apply polyharmonic interpolation model to data.
+
+  Given coefficients w and v for the interpolation model, we evaluate
+  interpolated function values at query_points.
+
+  Args:
+    query_points: `[b, m, d]` x values to evaluate the interpolation at
+    train_points: `[b, n, d]` x values that act as the interpolation centers
+                    ( the c variables in the wikipedia article)
+    w: `[b, n, k]` weights on each interpolation center
+    v: `[b, d, k]` weights on each input dimension
+    order: order of the interpolation
+
+  Returns:
+    Polyharmonic interpolation evaluated at points defined in query_points.
+  """
+
+  batch_size = train_points.get_shape()[0].value
+  num_query_points = query_points.get_shape()[1].value
+
+  # First, compute the contribution from the rbf term.
+  pairwise_dists = _cross_squared_distance_matrix(query_points, train_points)
+  phi_pairwise_dists = _phi(pairwise_dists, order)
+
+  rbf_term = math_ops.matmul(phi_pairwise_dists, w)
+
+  # Then, compute the contribution from the linear term.
+  # Pad query_points with ones, for the bias term in the linear model.
+  query_points_pad = array_ops.concat([
+      query_points,
+      array_ops.ones([batch_size, num_query_points, 1], train_points.dtype)
+  ], 2)
+  linear_term = math_ops.matmul(query_points_pad, v)
+
+  return rbf_term + linear_term
+
+
+def _phi(r, order):
+  """Coordinate-wise nonlinearity used to define the order of the interpolation.
+
+  See https://en.wikipedia.org/wiki/Polyharmonic_spline for the definition.
+
+  Args:
+    r: input op
+    order: interpolation order
+
+  Returns:
+    phi_k evaluated coordinate-wise on r, for k = r
+  """
+
+  # using EPSILON prevents log(0), sqrt0), etc.
+  # sqrt(0) is well-defined, but its gradient is not
+  with ops.name_scope('phi'):
+    if order == 1:
+      r = math_ops.maximum(r, EPSILON)
+      r = math_ops.sqrt(r)
+      return r
+    elif order == 2:
+      return 0.5 * r * math_ops.log(math_ops.maximum(r, EPSILON))
+    elif order == 4:
+      return 0.5 * math_ops.square(r) * math_ops.log(
+          math_ops.maximum(r, EPSILON))
+    elif order % 2 == 0:
+      r = math_ops.maximum(r, EPSILON)
+      return 0.5 * math_ops.pow(r, 0.5 * order) * math_ops.log(r)
+    else:
+      r = math_ops.maximum(r, EPSILON)
+      return math_ops.pow(r, 0.5 * order)
+
+
+def interpolate_spline(train_points,
+                       train_values,
+                       query_points,
+                       order,
+                       regularization_weight=0.0,
+                       name='interpolate_spline'):
+  r"""Interpolate signal using polyharmonic interpolation.
+
+  The interpolant has the form
+  $$f(x) = \sum_{i = 1}^n w_i \phi(||x - c_i||) + v^T x + b.$$
+
+  This is a sum of two terms: (1) a weighted sum of radial basis function (RBF)
+  terms, with the centers \\(c_1, ... c_n\\), and (2) a linear term with a bias.
+  The \\(c_i\\) vectors are 'training' points. In the code, b is absorbed into v
+  by appending 1 as a final dimension to x. The coefficients w and v are
+  estimated such that the interpolant exactly fits the value of the function at
+  the \\(c_i\\) points, the vector w is orthogonal to each \\(c_i\\), and the
+  vector w sums to 0. With these constraints, the coefficients can be obtained
+  by solving a linear system.
+
+  \\(\phi\\) is an RBF, parametrized by an interpolation
+  order. Using order=2 produces the well-known thin-plate spline.
+
+  We also provide the option to perform regularized interpolation. Here, the
+  interpolant is selected to trade off between the squared loss on the training
+  data and a certain measure of its curvature
+  ([details](https://en.wikipedia.org/wiki/Polyharmonic_spline)).
+  Using a regularization weight greater than zero has the effect that the
+  interpolant will no longer exactly fit the training data. However, it may be
+  less vulnerable to overfitting, particularly for high-order interpolation.
+
+  Note the interpolation procedure is differentiable with respect to all inputs
+  besides the order parameter.
+
+  Args:
+    train_points: `[batch_size, n, d]` float `Tensor` of n d-dimensional
+      locations. These do not need to be regularly-spaced.
+    train_values: `[batch_size, n, k]` float `Tensor` of n c-dimensional values
+      evaluated at train_points.
+    query_points: `[batch_size, m, d]` `Tensor` of m d-dimensional locations
+      where we will output the interpolant's values.
+    order: order of the interpolation. Common values are 1 for
+      \\(\phi(r) = r\\), 2 for \\(\phi(r) = r^2 * log(r)\\) (thin-plate spline),
+       or 3 for \\(\phi(r) = r^3\\).
+    regularization_weight: weight placed on the regularization term.
+      This will depend substantially on the problem, and it should always be
+      tuned. For many problems, it is reasonable to use no regularization.
+      If using a non-zero value, we recommend a small value like 0.001.
+    name: name prefix for ops created by this function
+
+  Returns:
+    `[b, m, k]` float `Tensor` of query values. We use train_points and
+    train_values to perform polyharmonic interpolation. The query values are
+    the values of the interpolant evaluated at the locations specified in
+    query_points.
+  """
+  with ops.name_scope(name):
+    train_points = ops.convert_to_tensor(train_points)
+    train_values = ops.convert_to_tensor(train_values)
+    query_points = ops.convert_to_tensor(query_points)
+
+    # First, fit the spline to the observed data.
+    with ops.name_scope('solve'):
+      w, v = _solve_interpolation(train_points, train_values, order,
+                                  regularization_weight)
+
+    # Then, evaluate the spline at the query locations.
+    with ops.name_scope('predict'):
+      query_values = _apply_interpolation(query_points, train_points, w, v,
+                                          order)
+
+  return query_values
diff --git a/tensorflow/contrib/image/python/ops/sparse_image_warp.py b/tensorflow/contrib/image/python/ops/sparse_image_warp.py
new file mode 100644
index 0000000000..54a215d6db
--- /dev/null
+++ b/tensorflow/contrib/image/python/ops/sparse_image_warp.py
@@ -0,0 +1,201 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Image warping using sparse flow defined at control points."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.image.python.ops import dense_image_warp
+from tensorflow.contrib.image.python.ops import interpolate_spline
+
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+
+
+def _get_grid_locations(image_height, image_width):
+  """Wrapper for np.meshgrid."""
+
+  y_range = np.linspace(0, image_height - 1, image_height)
+  x_range = np.linspace(0, image_width - 1, image_width)
+  y_grid, x_grid = np.meshgrid(y_range, x_range, indexing='ij')
+  return np.stack((y_grid, x_grid), -1)
+
+
+def _expand_to_minibatch(np_array, batch_size):
+  """Tile arbitrarily-sized np_array to include new batch dimension."""
+  tiles = [batch_size] + [1] * np_array.ndim
+  return np.tile(np.expand_dims(np_array, 0), tiles)
+
+
+def _get_boundary_locations(image_height, image_width, num_points_per_edge):
+  """Compute evenly-spaced indices along edge of image."""
+  y_range = np.linspace(0, image_height - 1, num_points_per_edge + 2)
+  x_range = np.linspace(0, image_width - 1, num_points_per_edge + 2)
+  ys, xs = np.meshgrid(y_range, x_range, indexing='ij')
+  is_boundary = np.logical_or(
+      np.logical_or(xs == 0, xs == image_width - 1),
+      np.logical_or(ys == 0, ys == image_height - 1))
+  return np.stack([ys[is_boundary], xs[is_boundary]], axis=-1)
+
+
+def _add_zero_flow_controls_at_boundary(control_point_locations,
+                                        control_point_flows, image_height,
+                                        image_width, boundary_points_per_edge):
+  """Add control points for zero-flow boundary conditions.
+
+   Augment the set of control points with extra points on the
+   boundary of the image that have zero flow.
+
+  Args:
+    control_point_locations: input control points
+    control_point_flows: their flows
+    image_height: image height
+    image_width: image width
+    boundary_points_per_edge: number of points to add in the middle of each
+                           edge (not including the corners).
+                           The total number of points added is
+                           4 + 4*(boundary_points_per_edge).
+
+  Returns:
+    merged_control_point_locations: augmented set of control point locations
+    merged_control_point_flows: augmented set of control point flows
+  """
+
+  batch_size = control_point_locations.get_shape()[0].value
+
+  boundary_point_locations = _get_boundary_locations(image_height, image_width,
+                                                     boundary_points_per_edge)
+
+  boundary_point_flows = np.zeros([boundary_point_locations.shape[0], 2])
+
+  type_to_use = control_point_locations.dtype
+  boundary_point_locations = constant_op.constant(
+      _expand_to_minibatch(boundary_point_locations, batch_size),
+      dtype=type_to_use)
+
+  boundary_point_flows = constant_op.constant(
+      _expand_to_minibatch(boundary_point_flows, batch_size), dtype=type_to_use)
+
+  merged_control_point_locations = array_ops.concat(
+      [control_point_locations, boundary_point_locations], 1)
+
+  merged_control_point_flows = array_ops.concat(
+      [control_point_flows, boundary_point_flows], 1)
+
+  return merged_control_point_locations, merged_control_point_flows
+
+
+def sparse_image_warp(image,
+                      source_control_point_locations,
+                      dest_control_point_locations,
+                      interpolation_order=2,
+                      regularization_weight=0.0,
+                      num_boundary_points=0,
+                      name='sparse_image_warp'):
+  """Image warping using correspondences between sparse control points.
+
+  Apply a non-linear warp to the image, where the warp is specified by
+  the source and destination locations of a (potentially small) number of
+  control points. First, we use a polyharmonic spline
+  (@{tf.contrib.image.interpolate_spline}) to interpolate the displacements
+  between the corresponding control points to a dense flow field.
+  Then, we warp the image using this dense flow field
+  (@{tf.contrib.image.dense_image_warp}).
+
+  Let t index our control points. For regularization_weight=0, we have:
+  warped_image[b, dest_control_point_locations[b, t, 0],
+                  dest_control_point_locations[b, t, 1], :] =
+  image[b, source_control_point_locations[b, t, 0],
+           source_control_point_locations[b, t, 1], :].
+
+  For regularization_weight > 0, this condition is met approximately, since
+  regularized interpolation trades off smoothness of the interpolant vs.
+  reconstruction of the interpolant at the control points.
+  See @{tf.contrib.image.interpolate_spline} for further documentation of the
+  interpolation_order and regularization_weight arguments.
+
+
+  Args:
+    image: `[batch, height, width, channels]` float `Tensor`
+    source_control_point_locations: `[batch, num_control_points, 2]` float
+      `Tensor`
+    dest_control_point_locations: `[batch, num_control_points, 2]` float
+      `Tensor`
+    interpolation_order: polynomial order used by the spline interpolation
+    regularization_weight: weight on smoothness regularizer in interpolation
+    num_boundary_points: How many zero-flow boundary points to include at
+      each image edge.Usage:
+        num_boundary_points=0: don't add zero-flow points
+        num_boundary_points=1: 4 corners of the image
+        num_boundary_points=2: 4 corners and one in the middle of each edge
+          (8 points total)
+        num_boundary_points=n: 4 corners and n-1 along each edge
+    name: A name for the operation (optional).
+
+    Note that image and offsets can be of type tf.half, tf.float32, or
+    tf.float64, and do not necessarily have to be the same type.
+
+  Returns:
+    warped_image: `[batch, height, width, channels]` float `Tensor` with same
+      type as input image.
+    flow_field: `[batch, height, width, 2]` float `Tensor` containing the dense
+      flow field produced by the interpolation.
+  """
+
+  image = ops.convert_to_tensor(image)
+  source_control_point_locations = ops.convert_to_tensor(
+      source_control_point_locations)
+  dest_control_point_locations = ops.convert_to_tensor(
+      dest_control_point_locations)
+
+  control_point_flows = (
+      dest_control_point_locations - source_control_point_locations)
+
+  clamp_boundaries = num_boundary_points > 0
+  boundary_points_per_edge = num_boundary_points - 1
+
+  with ops.name_scope(name):
+
+    batch_size, image_height, image_width, _ = image.get_shape().as_list()
+
+    # This generates the dense locations where the interpolant
+    # will be evaluated.
+    grid_locations = _get_grid_locations(image_height, image_width)
+
+    flattened_grid_locations = np.reshape(grid_locations,
+                                          [image_height * image_width, 2])
+
+    flattened_grid_locations = constant_op.constant(
+        _expand_to_minibatch(flattened_grid_locations, batch_size), image.dtype)
+
+    if clamp_boundaries:
+      (dest_control_point_locations,
+       control_point_flows) = _add_zero_flow_controls_at_boundary(
+           dest_control_point_locations, control_point_flows, image_height,
+           image_width, boundary_points_per_edge)
+
+    flattened_flows = interpolate_spline.interpolate_spline(
+        dest_control_point_locations, control_point_flows,
+        flattened_grid_locations, interpolation_order, regularization_weight)
+
+    dense_flows = array_ops.reshape(flattened_flows,
+                                    [batch_size, image_height, image_width, 2])
+
+    warped_image = dense_image_warp.dense_image_warp(image, dense_flows)
+
+    return warped_image, dense_flows
diff --git a/tensorflow/tools/pip_package/pip_smoke_test.py b/tensorflow/tools/pip_package/pip_smoke_test.py
index b66c45ec13..e2518f6cbf 100644
--- a/tensorflow/tools/pip_package/pip_smoke_test.py
+++ b/tensorflow/tools/pip_package/pip_smoke_test.py
@@ -75,6 +75,7 @@ BLACKLIST = [
     "//tensorflow/contrib/timeseries/examples:data/period_trend.csv",  # pylint:disable=line-too-long
     "//tensorflow/contrib/timeseries/python/timeseries:test_utils",
     "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:test_utils",  # pylint:disable=line-too-long
+    "//tensorflow/contrib/image:sparse_image_warp_test_data",
 ]
 
 
-- 
GitLab


From b7cbb2c9b155f3528edd8d26f7595dde8de578a8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 07:29:45 -0700
Subject: [PATCH 169/960] Adds missing protobuf dep to tf.contrib.data ops.

PiperOrigin-RevId: 189580464
---
 tensorflow/contrib/data/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 0458199ff7..5ba2297e7f 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -29,7 +29,10 @@ py_library(
 tf_custom_op_library(
     name = "_dataset_ops.so",
     srcs = ["ops/dataset_ops.cc"],
-    deps = ["//tensorflow/contrib/data/kernels:dataset_kernels"],
+    deps = [
+        "//tensorflow/contrib/data/kernels:dataset_kernels",
+        "//tensorflow/core:lib_proto_parsing",
+    ],
 )
 
 tf_gen_op_libs(
-- 
GitLab


From c07b18684c3b20dd91911a31bbd6169ad9cc1617 Mon Sep 17 00:00:00 2001
From: Alan Lee <secsilm@outlook.com>
Date: Mon, 19 Mar 2018 23:24:53 +0800
Subject: [PATCH 170/960] Fix set_difference doc

---
 tensorflow/python/ops/sets_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/sets_impl.py b/tensorflow/python/ops/sets_impl.py
index b0eecd8a1e..21e08d03d2 100644
--- a/tensorflow/python/ops/sets_impl.py
+++ b/tensorflow/python/ops/sets_impl.py
@@ -247,7 +247,7 @@ def set_difference(a, b, aminusb=True, validate_indices=True):
     #
     # collections.OrderedDict([
     #     ((0, 0, 0), 2),
-    #     ((0, 0, 1), 3),
+    #     ((0, 1, 0), 3),
     # ])
   ```
 
-- 
GitLab


From 31dc58502a4a2a594424fc76a3b4a2a508f28200 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Mon, 19 Mar 2018 17:38:38 +0200
Subject: [PATCH 171/960] Fix typos in `resampling.py`.

- Correct `initial_dist` -> `target_dist`
- `variabes` -> `variables`
---
 tensorflow/contrib/data/python/ops/resampling.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index 56f526a330..f4015f19fb 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -54,7 +54,7 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
   def _apply_fn(dataset):
     """Function from `Dataset` to `Dataset` that applies the transformation."""
     dist_estimation_batch_size = 32
-    target_dist_t = ops.convert_to_tensor(target_dist, name="initial_dist")
+    target_dist_t = ops.convert_to_tensor(target_dist, name="target_dist")
     class_values_ds = dataset.map(class_func)
     if initial_dist is not None:
       initial_dist_t = ops.convert_to_tensor(initial_dist, name="initial_dist")
@@ -151,7 +151,7 @@ def _calculate_acceptance_probs(initial_probs, target_probs):
   ```
 
 
-  A solution for a_i in terms of the other variabes is the following:
+  A solution for a_i in terms of the other variables is the following:
     ```a_i = (t_i / p_i) / max_i[t_i / p_i]```
   """
   # Add tiny to initial_probs to avoid divide by zero.
-- 
GitLab


From 98f522d3e982daafa9ccf136894cc83f496f5a11 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 09:28:58 -0700
Subject: [PATCH 172/960] Remove a few unused #includes

PiperOrigin-RevId: 189593522
---
 tensorflow/compiler/xla/array.h                      | 1 +
 tensorflow/compiler/xla/tests/test_macros.cc         | 1 +
 tensorflow/contrib/tensor_forest/kernels/data_spec.h | 1 +
 tensorflow/core/BUILD                                | 1 +
 tensorflow/core/lib/bfloat16/bfloat16.h              | 4 ++--
 tensorflow/core/lib/io/path.cc                       | 4 ++--
 tensorflow/core/lib/io/path.h                        | 1 -
 tensorflow/core/lib/strings/str_util.cc              | 2 ++
 tensorflow/core/lib/strings/str_util.h               | 1 -
 tensorflow/core/platform/env.cc                      | 1 -
 tensorflow/core/platform/file_system.cc              | 3 ---
 tensorflow/core/platform/file_system.h               | 1 -
 tensorflow/core/platform/types.h                     | 4 ----
 13 files changed, 10 insertions(+), 15 deletions(-)

diff --git a/tensorflow/compiler/xla/array.h b/tensorflow/compiler/xla/array.h
index 24b58bec11..ea75ad32d5 100644
--- a/tensorflow/compiler/xla/array.h
+++ b/tensorflow/compiler/xla/array.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/status.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/lib/core/bits.h"
+#include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
diff --git a/tensorflow/compiler/xla/tests/test_macros.cc b/tensorflow/compiler/xla/tests/test_macros.cc
index 978a669bca..be35ec6c6e 100644
--- a/tensorflow/compiler/xla/tests/test_macros.cc
+++ b/tensorflow/compiler/xla/tests/test_macros.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include <unordered_map>
 
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/regexp.h"
 
 namespace xla {
diff --git a/tensorflow/contrib/tensor_forest/kernels/data_spec.h b/tensorflow/contrib/tensor_forest/kernels/data_spec.h
index 0a3abe56df..bb33400214 100644
--- a/tensorflow/contrib/tensor_forest/kernels/data_spec.h
+++ b/tensorflow/contrib/tensor_forest/kernels/data_spec.h
@@ -21,6 +21,7 @@
 
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
 namespace tensorforest {
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 14769c3770..df44857185 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -594,6 +594,7 @@ cc_library(
         "platform/prefetch.h",
         "platform/thread_annotations.h",
         "platform/types.h",
+        "platform/cpu_info.h",
     ] + if_windows(["platform/windows/integral_types.h"]),
     visibility = ["//visibility:public"],
     deps =
diff --git a/tensorflow/core/lib/bfloat16/bfloat16.h b/tensorflow/core/lib/bfloat16/bfloat16.h
index 6a1cc0994f..075a8d1430 100644
--- a/tensorflow/core/lib/bfloat16/bfloat16.h
+++ b/tensorflow/core/lib/bfloat16/bfloat16.h
@@ -19,8 +19,8 @@ limitations under the License.
 #include <cmath>
 #include <complex>
 
-// We need types.h here in order to pick up __BYTE_ORDER__ from cpu_info.h
-#include "tensorflow/core/platform/types.h"
+// We need cpu_info.h here in order to pick up __BYTE_ORDER__.
+#include "tensorflow/core/platform/cpu_info.h"
 
 #ifdef __CUDACC__
 // All functions callable from CUDA code must be qualified with __device__
diff --git a/tensorflow/core/lib/io/path.cc b/tensorflow/core/lib/io/path.cc
index 83f15e134d..996fbf62e5 100644
--- a/tensorflow/core/lib/io/path.cc
+++ b/tensorflow/core/lib/io/path.cc
@@ -27,9 +27,9 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/lib/strings/scanner.h"
-#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
-#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/mutex.h"
 
 namespace tensorflow {
 namespace io {
diff --git a/tensorflow/core/lib/io/path.h b/tensorflow/core/lib/io/path.h
index 47bb2b998d..818ba99888 100644
--- a/tensorflow/core/lib/io/path.h
+++ b/tensorflow/core/lib/io/path.h
@@ -16,7 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_PATH_H_
 #define TENSORFLOW_LIB_IO_PATH_H_
 
-#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/lib/strings/str_util.cc b/tensorflow/core/lib/strings/str_util.cc
index 9dbb74f6b8..2c9e98357a 100644
--- a/tensorflow/core/lib/strings/str_util.cc
+++ b/tensorflow/core/lib/strings/str_util.cc
@@ -16,9 +16,11 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/str_util.h"
 
 #include <ctype.h>
+#include <algorithm>
 #include <vector>
 #include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
+#include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
 namespace str_util {
diff --git a/tensorflow/core/lib/strings/str_util.h b/tensorflow/core/lib/strings/str_util.h
index f062eddef8..065871c1b4 100644
--- a/tensorflow/core/lib/strings/str_util.h
+++ b/tensorflow/core/lib/strings/str_util.h
@@ -20,7 +20,6 @@ limitations under the License.
 #include <string>
 #include <vector>
 #include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/types.h"
 
diff --git a/tensorflow/core/platform/env.cc b/tensorflow/core/platform/env.cc
index 12509c250e..b9a9ef85eb 100644
--- a/tensorflow/core/platform/env.cc
+++ b/tensorflow/core/platform/env.cc
@@ -33,7 +33,6 @@ limitations under the License.
 #endif
 
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
diff --git a/tensorflow/core/platform/file_system.cc b/tensorflow/core/platform/file_system.cc
index 271d73f5f1..5bc8606e28 100644
--- a/tensorflow/core/platform/file_system.cc
+++ b/tensorflow/core/platform/file_system.cc
@@ -19,15 +19,12 @@ limitations under the License.
 
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
-#include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/file_system.h"
 #include "tensorflow/core/platform/platform.h"
-#include "tensorflow/core/platform/protobuf.h"
 
 namespace tensorflow {
 
diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h
index 3085b6958f..03c0c5ab51 100644
--- a/tensorflow/core/platform/file_system.h
+++ b/tensorflow/core/platform/file_system.h
@@ -27,7 +27,6 @@ limitations under the License.
 #include "tensorflow/core/platform/file_statistics.h"
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/platform.h"
-#include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/types.h"
 
 #ifdef PLATFORM_WINDOWS
diff --git a/tensorflow/core/platform/types.h b/tensorflow/core/platform/types.h
index 38d75dbb32..6308e58847 100644
--- a/tensorflow/core/platform/types.h
+++ b/tensorflow/core/platform/types.h
@@ -31,10 +31,6 @@ limitations under the License.
 #error Define the appropriate PLATFORM_<foo> macro for this platform
 #endif
 
-#if defined(PLATFORM_WINDOWS)
-#include "tensorflow/core/platform/windows/cpu_info.h"
-#endif
-
 namespace tensorflow {
 
 // Define tensorflow::string to refer to appropriate platform specific type.
-- 
GitLab


From 8d172e1a89feb06f906de43d75f0d5e65a2e1a04 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 09:42:14 -0700
Subject: [PATCH 173/960] Refactor code to improve TensorDataSet construction
 speed.

PiperOrigin-RevId: 189595482
---
 tensorflow/contrib/tensor_forest/kernels/v4/input_data.h | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
index c544a8c75e..b991e6339f 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
@@ -44,12 +44,15 @@ class TensorDataSet {
     int column_count = 0;
     for (int i = 0; i < input_spec_.dense_size(); ++i) {
       for (int j = 0; j < input_spec_.dense(i).size(); ++j) {
-        decision_trees::FeatureId id;
-        id.mutable_id()->set_value(strings::StrCat(column_count));
-        available_features_.push_back(id);
         ++column_count;
       }
     }
+    available_features_.reserve(column_count);
+    decision_trees::FeatureId id;
+    for (int i = 0; i < column_count; i++) {
+      id.mutable_id()->set_value(strings::StrCat(i));
+      available_features_.emplace_back(id);
+    }
 
     // Set up the random number generator.
     if (split_sampling_random_seed_ == 0) {
-- 
GitLab


From 9f3a01ee045ea19baebf6e07a2a966564bfa2f3f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 09:43:59 -0700
Subject: [PATCH 174/960] Simple rewrite to remove negation nodes.

PiperOrigin-RevId: 189595735
---
 tensorflow/core/grappler/op_types.cc          |  2 +
 tensorflow/core/grappler/op_types.h           |  1 +
 .../optimizers/arithmetic_optimizer.cc        | 75 ++++++++++++++----
 .../optimizers/arithmetic_optimizer.h         |  1 +
 .../optimizers/arithmetic_optimizer_test.cc   | 77 +++++++++++++++++++
 5 files changed, 143 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 9c9600db5e..259168bb33 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -212,6 +212,8 @@ bool IsMod(const NodeDef& node) { return node.op() == "Mod"; }
 
 bool IsMul(const NodeDef& node) { return node.op() == "Mul"; }
 
+bool IsNeg(const NodeDef& node) { return node.op() == "Neg"; }
+
 bool IsNoOp(const NodeDef& node) { return node.op() == "NoOp"; }
 
 bool IsNotEqual(const NodeDef& node) { return node.op() == "NotEqual"; }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 41ba8bb01e..49e01f68e3 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -89,6 +89,7 @@ bool IsNextIteration(const NodeDef& node);
 bool IsPack(const NodeDef& node);
 bool IsPad(const NodeDef& node);
 bool IsPack(const NodeDef& node);
+bool IsNeg(const NodeDef& node);
 bool IsNoOp(const NodeDef& node);
 bool IsNotEqual(const NodeDef& node);
 bool IsPlaceholder(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 3a67c4b056..c25836ceef 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -344,8 +344,7 @@ class ArithmeticOptimizerStage {
   // will be automatically added to the optimization queue. If a simplified node
   // has the same name as original node it has to be explicitly added to the
   // optimization queue for second pass.
-  virtual Status TrySimplify(const NodeDef* node,
-                             string* simplified_node_name) = 0;
+  virtual Status TrySimplify(NodeDef* node, string* simplified_node_name) = 0;
 
  protected:
   struct ScopedNodeName {
@@ -557,8 +556,7 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
            HasAllInputsOfSymbolicallyEqualShape(*node, properties);
   }
 
-  Status TrySimplify(const NodeDef* node,
-                     string* simplified_node_name) override {
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     CHECK(IsSupported(node));
     AddOpsGroup group;
     TF_RETURN_IF_ERROR(CreateAddOpsGroup(node, &group));
@@ -794,8 +792,7 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
            !IsRewritten(node);
   }
 
-  Status TrySimplify(const NodeDef* node,
-                     string* simplified_node_name) override {
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     CHECK(IsSupported(node));
 
     std::set<string> common_factors;
@@ -945,8 +942,7 @@ class RemoveIdentityTranspose : public ArithmeticOptimizerStage {
 
   // TODO(rmlarsen): Forward control dependencies on the bypassed
   // transpose nodes.
-  Status TrySimplify(const NodeDef* node,
-                     string* simplified_node_name) override {
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     CHECK(IsSupported(node));
 
     NodeDef* input;
@@ -1028,8 +1024,7 @@ class RemoveRedundantBitcastStage : public ArithmeticOptimizerStage {
     return IsBitcast(*node);
   }
 
-  Status TrySimplify(const NodeDef* node,
-                     string* simplified_node_name) override {
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     CHECK(IsSupported(node));
 
     // Bypass Bitcast whose source type and destination type are equal.
@@ -1066,8 +1061,7 @@ class RemoveRedundantCastStage : public ArithmeticOptimizerStage {
 
   bool IsSupported(const NodeDef* node) const override { return IsCast(*node); }
 
-  Status TrySimplify(const NodeDef* node,
-                     string* simplified_node_name) override {
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
     CHECK(IsSupported(node));
     // Bypass Cast whose source type and destination type are equal.
     if (GetSourceDataType(*node) == GetDestinationDataType(*node)) {
@@ -1077,6 +1071,57 @@ class RemoveRedundantCastStage : public ArithmeticOptimizerStage {
   }
 };
 
+class RemoveNegationStage : public ArithmeticOptimizerStage {
+ public:
+  explicit RemoveNegationStage(const ArithmeticOptimizerContext& ctx)
+      : ArithmeticOptimizerStage("RemoveNegation", ctx) {}
+  ~RemoveNegationStage() override = default;
+
+  bool IsSupported(const NodeDef* node) const override {
+    return IsAdd(*node) || IsSub(*node);
+  }
+
+  Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
+    const string node_name = node->name();
+    NodeDef* x;
+    NodeDef* y;
+    TF_RETURN_IF_ERROR(GetInputNode(node->input(0), &x));
+    TF_RETURN_IF_ERROR(GetInputNode(node->input(1), &y));
+    bool updated = false;
+    if (IsAdd(*node)) {
+      if (IsNeg(*x)) {
+        // (-a) + b = b - a
+        node->set_op("Sub");
+        node->mutable_input()->SwapElements(0, 1);
+        node->set_input(1, x->input(0));
+        node->add_input(AsControlDependency(x->name()));
+        ctx_.node_map->AddOutput(NodeName(x->input(0)), node_name);
+        updated = true;
+      } else if (IsNeg(*y)) {
+        // a + (-b) = a - b
+        node->set_op("Sub");
+        node->set_input(1, y->input(0));
+        node->add_input(AsControlDependency(y->name()));
+        ctx_.node_map->AddOutput(NodeName(y->input(0)), node_name);
+        updated = true;
+      }
+    } else if (IsSub(*node)) {
+      if (IsNeg(*y)) {
+        // a - (-b) = a + b
+        node->set_op("Add");
+        node->set_input(1, y->input(0));
+        node->add_input(AsControlDependency(y->name()));
+        ctx_.node_map->AddOutput(NodeName(y->input(0)), node_name);
+        updated = true;
+      }
+    }
+    if (updated) {
+      AddToOptimizationQueue(node);
+    }
+    return Status::OK();
+  }
+};
+
 }  // namespace
 
 class UniqueNodes {
@@ -1696,12 +1741,16 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new RemoveRedundantCastStage(ctx)));
   }
+  if (options_.remove_negation) {
+    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
+        new RemoveNegationStage(ctx)));
+  }
 
   VLOG(1) << "Simplify arithmetic ops using " << stages.size()
           << " arithmetic optimization stages";
 
   while (!nodes_to_simplify.Empty()) {
-    const NodeDef* node = nodes_to_simplify.PopBack();
+    NodeDef* node = nodes_to_simplify.PopBack();
 
     // TODO(ezhulenev): move all rewrites into separate stages
     string simplified_tensor = "";
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 95c1e14258..965f0e9ea2 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -63,6 +63,7 @@ class ArithmeticOptimizer : public GraphOptimizer {
     bool remove_identity_transpose = true;
     bool remove_redundant_bitcast = true;
     bool remove_redundant_cast = true;
+    bool remove_negation = true;
 
     // Choose which arithmetic optimizer stages will be enabled for a given
     // optimization level by default.
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 6f7a95c2ed..3876486d80 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -126,6 +126,11 @@ class ArithmeticOptimizerTest : public GrapplerTest {
     DisableAllStages(optimizer);
     optimizer->options_.remove_redundant_cast = true;
   }
+
+  void EnableOnlyRemoveNegation(ArithmeticOptimizer* optimizer) {
+    DisableAllStages(optimizer);
+    optimizer->options_.remove_negation = true;
+  }
 };
 
 TEST_F(ArithmeticOptimizerTest, NoOp) {
@@ -1498,5 +1503,77 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfSymbolicallyEqualShape) {
   EXPECT_EQ(collapsed_add->name(), updated_outputs->input(0));
 }
 
+TEST_F(ArithmeticOptimizerTest, RemoveNegation) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto x = ops::Variable(s.WithOpName("x"), {2, 2}, DT_FLOAT);
+  auto y = ops::Variable(s.WithOpName("y"), {2, 2}, DT_FLOAT);
+  Output neg_x = ops::Neg(s.WithOpName("Neg_x"), x);
+  Output neg_y = ops::Neg(s.WithOpName("Neg_y"), y);
+  Output add_x_y = ops::Add(s.WithOpName("Add_x_y"), x, y);
+  Output add_negx_y = ops::Add(s.WithOpName("Add_negx_y"), neg_x, y);
+  Output add_x_negy = ops::Add(s.WithOpName("Add_x_negy"), x, neg_y);
+  Output add_negx_negy = ops::Add(s.WithOpName("Add_negx_negy"), neg_x, neg_y);
+  Output sub_x_y = ops::Sub(s.WithOpName("Sub_x_y"), x, y);
+  Output sub_negx_y = ops::Sub(s.WithOpName("Sub_negx_y"), neg_x, y);
+  Output sub_x_negy = ops::Sub(s.WithOpName("Sub_x_negy"), x, neg_y);
+  Output sub_negx_negy = ops::Sub(s.WithOpName("Sub_negx_negy"), neg_x, neg_y);
+  auto add_all = ops::AddN(s.WithOpName("add_all"),
+                           {add_x_y, add_negx_y, add_x_negy, add_negx_negy,
+                            sub_x_y, sub_negx_y, sub_x_negy, sub_negx_negy});
+
+  GrapplerItem item;
+  item.fetch = {"add_all"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyRemoveNegation(&optimizer);
+  OptimizeAndPrune(&optimizer, &item, &output);
+
+  EXPECT_EQ(item.graph.node_size(), output.node_size());
+  int found = 0;
+  for (int i = 0; i < output.node_size(); ++i) {
+    const NodeDef& node = output.node(i);
+    if (node.name() == "Add_negx_y") {
+      ++found;
+      EXPECT_EQ("Sub", node.op());
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ("x", node.input(1));
+      EXPECT_EQ("^Neg_x", node.input(2));
+    } else if (node.name() == "Add_x_negy") {
+      ++found;
+      EXPECT_EQ("Sub", node.op());
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+      EXPECT_EQ("^Neg_y", node.input(2));
+    } else if (node.name() == "Add_negx_negy") {
+      ++found;
+      EXPECT_EQ("Sub", node.op());
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("Neg_y", node.input(0));
+      EXPECT_EQ("x", node.input(1));
+      EXPECT_EQ("^Neg_x", node.input(2));
+    } else if (node.name() == "Sub_x_negy") {
+      ++found;
+      EXPECT_EQ("Add", node.op());
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+      EXPECT_EQ("^Neg_y", node.input(2));
+    } else if (node.name() == "Sub_negx_negy") {
+      ++found;
+      EXPECT_EQ("Sub", node.op());
+      EXPECT_EQ(4, node.input_size());
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ("x", node.input(1));
+      EXPECT_EQ("^Neg_y", node.input(2));
+      EXPECT_EQ("^Neg_x", node.input(3));
+    }
+  }
+  EXPECT_EQ(5, found);
+}
+
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From f90616bc78ddfd1a9fb37ae30ac8851a9a275800 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 09:50:00 -0700
Subject: [PATCH 175/960] Fix misc typos in tensorflow/compiler/xla.

PiperOrigin-RevId: 189596520
---
 tensorflow/compiler/xla/service/algebraic_simplifier.h        | 2 +-
 tensorflow/compiler/xla/service/cpu/cpu_compiler.cc           | 2 +-
 .../compiler/xla/service/cpu/parallel_task_assignment.cc      | 4 ++--
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index 43315f5cdc..f0590943be 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -23,7 +23,7 @@ limitations under the License.
 
 namespace xla {
 
-// A pass which performs AlgebraicSimplications.
+// A pass which performs algebraic simplifications.
 class AlgebraicSimplifier : public HloPassInterface {
  public:
   // Given shapes 'from_shape' and 'to_shape', determines if it is valid to
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
index 0a966fd5a7..e43777c5e5 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_compiler.cc
@@ -318,7 +318,7 @@ Status CpuCompiler::RunHloPasses(HloModule* module, bool is_aot_compile) {
     // Note this is not run for AOT because it would bring in thread pool
     // and thread synchronization dependencies which would likely increase
     // binary size (and most AOT applications are single-threaded).
-    // TODO(29630486) Support multi-threaded AOT.
+    // TODO(b/29630486) Support multi-threaded AOT.
     pipeline.AddPass<ParallelTaskAssigner>(max_parallelism,
                                            ShapeSizeBytesFunction());
   }
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 38f1668159..86e8be8461 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -71,7 +71,7 @@ class DefaultCostModel : public ParallelCostModel {
     if (flops_to_bytes_ratio <= 1.0) {
       // Limit max parallelism for I/O bound instructions by assuming a
       // sub-linear scaling function (fit based on empirical benchmark results).
-      // TODO(29630486) Develop system bandwidth model.
+      // TODO(b/29630486) Develop system bandwidth model.
       max_parallelism =
           std::ceil(std::sqrt(tensorflow::port::NumSchedulableCPUs()));
       // Use shape size instruction cost and L2 cache size min per-thread cost.
@@ -81,7 +81,7 @@ class DefaultCostModel : public ParallelCostModel {
       // Use max parallelism for compute bound instructions.
       max_parallelism = max_parallelism_;
       // Calculate the instruction cost in cycles.
-      // TODO(29630486) Improve on this linear cost model.
+      // TODO(b/29630486) Improve on this linear cost model.
       // Consider making 'min_cost_per_thread' be a function of the target
       // bandwidth limit for instructions with low arithmetic complexity.
       instruction_cost =
-- 
GitLab


From 8dc7a69b3bfc04872fde56fda595a7614ac643fe Mon Sep 17 00:00:00 2001
From: imsheridan <xiaoyudong0512@gmail.com>
Date: Tue, 20 Mar 2018 00:57:43 +0800
Subject: [PATCH 176/960] Fix the comments of
 tf.contrib.lookup.MutableHashTable insert operation

---
 tensorflow/contrib/lookup/lookup_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index 62f1c810fc..cc77cd5431 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -298,7 +298,7 @@ class MutableHashTable(LookupInterface):
   table = tf.contrib.lookup.MutableHashTable(key_dtype=tf.string,
                                              value_dtype=tf.int64,
                                              default_value=-1)
-  table.insert(keys, values)
+  sess.run(table.insert(keys, values))
   out = table.lookup(query_keys)
   print(out.eval())
   ```
@@ -494,7 +494,7 @@ class MutableDenseHashTable(LookupInterface):
                                                   value_dtype=tf.int64,
                                                   default_value=-1,
                                                   empty_key=0)
-  table.insert(keys, values)
+  sess.run(table.insert(keys, values))
   out = table.lookup(query_keys)
   print(out.eval())
   ```
-- 
GitLab


From b1cb65ab5218c13eb9d0f55b7f169cd676e032f3 Mon Sep 17 00:00:00 2001
From: Fanjin Zeng <Fnjn@users.noreply.github.com>
Date: Mon, 19 Mar 2018 10:29:45 -0700
Subject: [PATCH 177/960] Fix related doc clarification request on
 tf.contrib.lookup.MutableHashTable insert operation #17835

Make the doc example executable, and explicitly suggests that MutableDenseHashTable.insert is an operation rather than in-place computation.
---
 tensorflow/contrib/lookup/lookup_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index 62f1c810fc..c7a61fcac3 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -494,7 +494,7 @@ class MutableDenseHashTable(LookupInterface):
                                                   value_dtype=tf.int64,
                                                   default_value=-1,
                                                   empty_key=0)
-  table.insert(keys, values)
+  sess.run(table.insert(keys, values))
   out = table.lookup(query_keys)
   print(out.eval())
   ```
-- 
GitLab


From 6f2f21894fae9384fe52ad77ec751c3c42276aa3 Mon Sep 17 00:00:00 2001
From: Tarang Chugh <tchugh@users.noreply.github.com>
Date: Mon, 19 Mar 2018 18:30:05 +0100
Subject: [PATCH 178/960] Update README.md (#16301)

Correct MobilenetV1 variable
---
 tensorflow/contrib/lite/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index 5194f015b5..2680d515eb 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -165,7 +165,7 @@ bazel-bin/tensorflow/python/tools/freeze_graph\
     --input_graph=/tmp/mobilenet_v1_224.pb \
     --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
     --input_binary=true --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
-    --output_node_names=MobileNet/Predictions/Reshape_1
+    --output_node_names=MobilenetV1/Predictions/Reshape_1
 ```
 
 The user has to first build the freeze_graph script using bazel and then run the script.  The input_binary flag has to be enabled to ensure that the protobuf is read and written in binary format.  The user has to input the .pb and the .ckpt files to freeze the graph The output_node_names may not be obvious outside of the code that built the model. The easiest way to find them is to visualize the graph, either with
-- 
GitLab


From 7f9ab7f8c5e161562656604d9b22939b1f97c791 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 19 Mar 2018 10:37:00 -0700
Subject: [PATCH 179/960] Documentation tweaks and tests for GradientTape with
 graph execution.

PiperOrigin-RevId: 189604536
---
 tensorflow/python/eager/backprop.py           | 76 ++++++++-----------
 tensorflow/python/eager/backprop_test.py      | 25 +++---
 tensorflow/python/framework/ops.py            | 35 ++++++---
 .../python/ops/resource_variable_ops.py       | 32 ++++----
 tensorflow/python/ops/variables.py            |  4 +-
 5 files changed, 88 insertions(+), 84 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 88de1a951f..9b997fed30 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -638,63 +638,53 @@ _default_vspace = imperative_grad.VSpace(
 
 
 class GradientTape(object):
-  """Records operations to use to compute gradients.
+  """Record operations for automatic differentiation.
 
-  Operations are recorded if:
-    - they happen in code marked by this context manager
-    - at least one of their inputs is being watched
+  Operations are recorded if they are executed within this context manager and
+  at least one of their inputs is being "watched".
 
-  Outputs of recorded operations are watched. Variables are automatically
-  watched and tensors can be manually watched by calling the watch method on the
-  context manager.
+  Variables (created by @{tf.contrib.eager.Variable} or @{tf.get_variable})
+  are automatically watched. Tensors can be manually watched by invoking the
+  `watch`
+  method on this context manager.
 
-  Example usage:
+  For example, consider the function `y = x * x`. The gradient at `x = 3.0` can
+  be computed as:
 
   ```python
+  x = tf.constant(3.)
   with tfe.GradientTape() as g:
-    x = tf.constant(3.0)
     g.watch(x)
     y = x * x
-  grad = g.gradient(y, [x])[0]
-  assert grad.numpy() == 6.0
+  grad = g.gradient(y, [x])[0] # Will compute to 6.0
   ```
 
-  It is possible to use GradientTapes to compute higher-order derivatives as
-  follows:
+  GradientTapes can be nested to compute higher-order derivatives. For example,
 
   ```python
+  x = tf.constant(3.0)
   with tfe.GradientTape() as g:
-    x = tf.constant(3.0)
-    g.watch(x)
-    y = x * x
     with tfe.GradientTape() as gg:
-      gg.watch(y)
-      z = 2 * y
-    inner_grad = gg.gradient(z, [y])[0]
-    assert inner_grad.numpy() == 2
-    y = y + inner_grad
-  grad = g.gradient(y, [x])[0]
-  assert grad.numpy() == 6.0
+      gg.watch(x)
+      y = x * x
+    dy_dx = gg.gradient(y, [x])[0]     # Will compute to 6.0
+  d2y_dx2 = g.gradient(dy_dx, [x])[0]  # Will compute to 2.0
   ```
 
   By default, the resources held by a GradientTape are released as soon as
-  GradientTape.gradient() method is called. However, if one need to compute
-  multiple gradients over the same computation, she can create a persistent
-  GradientTape. Persistent tapes allow multiple calls to the gradient() method
-  and release resources when the tape object is destructed.
-
-  Example usage:
+  GradientTape.gradient() method is called. To compute multiple gradients over
+  the same computation, create a persistent gradient tape. This allows multiple
+  calls to the gradient() method as resources are released when the tape object
+  is garbage collected. For example:
 
   ```python
+  x = tf.constant(3.0)
   with tfe.GradientTape(persistent=True) as g:
-    x = tf.constant(3.0)
     g.watch(x)
     y = x * x
     z = y * y
-  dz_dx = g.gradient(z, [x])[0]
-  assert dz_dx.numpy() == 108.0   # 4*x^3 at x = 3
-  dy_dx = g.gradient(y, [x])[0]
-  assert dy_dx.numpy() == 6.0
+  dy_dx = g.gradient(z, [x])[0]  # 6.0
+  dz_dx = g.gradient(y, [x])[0]  # 108.0 (4*x^3 at x = 3)
   del g  # Drop the reference to the tape
   """
 
@@ -703,8 +693,8 @@ class GradientTape(object):
 
     Args:
       persistent: Boolean controlling whether a persistent gradient tape
-        is created. Must be True or False.
-
+        is created. False by default, which means at most one call can
+        be made to the gradient() method on this object.
     """
     self._tape = None
     self._persistent = persistent
@@ -720,7 +710,7 @@ class GradientTape(object):
     """Ensures that `tensor` is being traced by this tape.
 
     Args:
-      tensor: a Tensor or Variable a list of Tensors or Variables.
+      tensor: a Tensor or list of Tensors.
     """
     for t in nest.flatten(tensor):
       if isinstance(t, resource_variable_ops.ResourceVariable):
@@ -735,14 +725,14 @@ class GradientTape(object):
                        key=lambda v: v.handle._id))  # pylint: disable=protected-access
 
   def gradient(self, target, sources, output_gradients=None):
-    """Computes the gradient using information traced by the tape.
+    """Computes the gradient using operations recorded in context of this tape.
 
     Args:
-      target: the tensor to be differentiated.
-      sources: a list of Tensors or Variables, the target will be
-       differentiated with respect to the sources.
+      target: Tensor to be differentiated.
+      sources: a list of Tensors or Variables. `target` will be differentiated
+        against elements in `sources`.
       output_gradients: a list of gradients, one for each element of
-       target. Defaults to None.
+        target. Defaults to None.
 
     Returns:
       a list of Tensors (or IndexedSlices, or None), one for each element in
@@ -750,7 +740,7 @@ class GradientTape(object):
 
     Raises:
       RuntimeError: if called inside the context of the tape, or if called more
-       than once.
+       than once on a non-persistent tape.
     """
     if self._tape is None:
       raise RuntimeError("GradientTape.gradient can only be called once "
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index 5934293dfc..bca2928708 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -195,8 +195,10 @@ class BackpropTest(test.TestCase):
     g, = backprop.gradients_function(loss, [0])(logits, labels)
     self.assertAllEqual(g.numpy(), [[-0.5, 0.5]])
 
+  @test_util.run_in_graph_and_eager_modes()
   def testGradientWithinTapeBlock(self):
     v1 = resource_variable_ops.ResourceVariable(1.)
+    self.evaluate(v1.initializer)
     with backprop.GradientTape() as t:
       loss = 2 * v1
       with self.assertRaises(RuntimeError):
@@ -204,7 +206,7 @@ class BackpropTest(test.TestCase):
     with backprop.GradientTape(persistent=True) as t:
       loss = 2 * v1
       grad = t.gradient(loss, [v1])
-    self.assertAllEqual(grad[0], 2.0)
+    self.assertAllEqual(self.evaluate(grad[0]), 2.0)
 
   @test_util.assert_no_new_tensors
   def testSecondGrad(self):
@@ -367,6 +369,7 @@ class BackpropTest(test.TestCase):
     self.assertEqual(backprop.implicit_grad(f)()[0][0], None)
 
   @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
   def testGradientTape(self):
     with backprop.GradientTape() as g:
       x = constant_op.constant(3.0)
@@ -376,10 +379,10 @@ class BackpropTest(test.TestCase):
         gg.watch(y)
         z = 2 * y
       inner_grad = gg.gradient(z, [y])[0]
-      self.assertEqual(inner_grad.numpy(), 2.0)
+      self.assertEqual(self.evaluate(inner_grad), 2.0)
       y += inner_grad
     grad = g.gradient(y, [x])[0]
-    self.assertEqual(grad.numpy(), 6.0)
+    self.assertEqual(self.evaluate(grad), 6.0)
 
   @test_util.assert_no_new_tensors
   def testGradientTapeGradientCalledMultipleTimes(self):
@@ -394,6 +397,7 @@ class BackpropTest(test.TestCase):
       g.gradient(y, [x])
 
   @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
   def testPersistentTape(self):
     with backprop.GradientTape(persistent=True) as g:
       x = constant_op.constant(3.0)
@@ -401,12 +405,13 @@ class BackpropTest(test.TestCase):
       y = x * x
       z = y * y
     dz_dx = g.gradient(z, [x])[0]
-    self.assertEqual(dz_dx.numpy(), 4*3*3*3)
+    self.assertEqual(self.evaluate(dz_dx), 4 * 3 * 3 * 3)
     dy_dx = g.gradient(y, [x])[0]
-    self.assertEqual(dy_dx.numpy(), 2*3)
+    self.assertEqual(self.evaluate(dy_dx), 2 * 3)
     del g
 
   @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
   def testPersistentNestedTape(self):
     with backprop.GradientTape(persistent=True) as g:
       x = constant_op.constant(3.0)
@@ -417,22 +422,24 @@ class BackpropTest(test.TestCase):
         z = 2 * y
       for _ in range(2):
         inner_grad = gg.gradient(z, [y])[0]
-        self.assertEqual(inner_grad.numpy(), 2.0)
+        self.assertEqual(self.evaluate(inner_grad), 2.0)
       y += inner_grad
       del gg
     grad = g.gradient(y, [x])[0]
-    self.assertEqual(grad.numpy(), 6.0)
+    self.assertEqual(self.evaluate(grad), 6.0)
     grad = g.gradient(z, [x])[0]
-    self.assertEqual(grad.numpy(), 12.0)
+    self.assertEqual(self.evaluate(grad), 12.0)
     del g
 
   @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
   def testGradientTapeVariable(self):
     v = resource_variable_ops.ResourceVariable(1.0, name='v')
+    self.evaluate(v.initializer)
     with backprop.GradientTape() as g:
       y = v * v
     grad = g.gradient(y, [v])[0]
-    self.assertAllEqual(grad, 2.0)
+    self.assertAllEqual(self.evaluate(grad), 2.0)
 
   @test_util.assert_no_new_tensors
   def testEmptyParamsForValueAndGradFunction(self):
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 01a0e03be2..f1cd341d66 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -838,41 +838,51 @@ class _EagerTensorBase(Tensor):
   def set_shape(self, shape):
     if not self.shape.is_compatible_with(shape):
       raise ValueError(
-          "EagerTensor's shape %s is not compatible with supplied shape %s" %
+          "Tensor's shape %s is not compatible with supplied shape %s" %
           (self.shape, shape))
 
   # Methods not supported / implemented for Eager Tensors.
   @property
   def op(self):
-    raise AttributeError("op not supported for Eager Tensors.")
+    raise AttributeError(
+        "Tensor.op is meaningless when eager execution is enabled.")
 
   @property
   def graph(self):
-    raise AttributeError("graph not supported for Eager Tensors.")
+    raise AttributeError(
+        "Tensor.graph is meaningless when eager execution is enabled.")
 
   @property
   def name(self):
-    raise AttributeError("name not supported for Eager Tensors.")
+    raise AttributeError(
+        "Tensor.name is meaningless when eager execution is enabled.")
 
   @property
   def value_index(self):
-    raise AttributeError("value_index not supported for Eager Tensors.")
+    raise AttributeError(
+        "Tensor.value_index is meaningless when eager execution is enabled.")
 
   def consumers(self):
-    raise NotImplementedError("consumers not supported for Eager Tensors.")
+    raise NotImplementedError(
+        "Tensor.consumers is meaningless when eager execution is enabled.")
 
   def _add_consumer(self, consumer):
-    raise NotImplementedError("_add_consumer not supported for Eager Tensors.")
+    raise NotImplementedError(
+        "_add_consumer not supported when eager execution is enabled.")
 
   def _as_node_def_input(self):
     raise NotImplementedError(
-        "_as_node_def_input not supported for Eager Tensors.")
+        "_as_node_def_input not supported when eager execution is enabled.")
 
   def _as_tf_output(self):
-    raise NotImplementedError("_as_tf_output not supported for Eager Tensors.")
+    raise NotImplementedError(
+        "_as_tf_output not supported when eager execution is enabled.")
 
   def eval(self, feed_dict=None, session=None):
-    raise NotImplementedError("eval not supported for Eager Tensors.")
+    raise NotImplementedError(
+        "eval is not supported when eager execution is enabled, "
+        "is .numpy() what you're looking for?"
+    )
 
 
 # This call creates an EagerTensor class, as a subclass of _EagerTensorBase, and
@@ -5937,8 +5947,9 @@ def get_from_proto_function(collection_name):
 def _assert_collection_is_ok(collection_name):
   if context.executing_eagerly():
     if collection_name in GraphKeys._VARIABLE_COLLECTIONS:  # pylint: disable=protected-access
-      raise ValueError("When Eager Execution is enabled, variable "
-                       "collections are not supported.")
+      raise ValueError(
+          "variable collections are not supported when eager execution is enabled."
+      )
 
 
 def _operation_conversion_error(op, dtype=None, name=None, as_ref=False):
diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index affa7ae629..df873da98e 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -149,7 +149,7 @@ def shape_safe_assign_variable_handle(handle, shape, value, name=None):
 class ResourceVariable(variables.Variable):
   """Variable based on resource handles.
 
-  See the @{$python/state_ops$`Variables`} documentation for more details.
+  See the @{$variables$Variables How To} for a high level overview.
 
   A `ResourceVariable` allows you to maintain state across subsequent calls to
   session.run.
@@ -179,24 +179,20 @@ class ResourceVariable(variables.Variable):
   by edges in the graph. Consider the following example, in which two writes
   can cause tf.Variable and tf.ResourceVariable to behave differently:
 
-   ```python
-    a = tf.ResourceVariable(1.0)
-    a.initializer.run()
-
-    assign = a.assign(2.0)
-    with tf.control_dependencies([assign]):
-      b = a.read_value()
-    with tf.control_dependencies([b]):
-      other_assign = a.assign(3.0)
-    with tf.control_dependencies([other_assign]):
-      # Will print 2.0 because the value was read before other_assign ran. If
-      # `a` was a tf.Variable instead, 2.0 or 3.0 could be printed.
-      tf.Print(b, [b]).eval()
+  ```python
+  a = tf.ResourceVariable(1.0)
+  a.initializer.run()
+
+  assign = a.assign(2.0)
+  with tf.control_dependencies([assign]):
+    b = a.read_value()
+  with tf.control_dependencies([b]):
+    other_assign = a.assign(3.0)
+  with tf.control_dependencies([other_assign]):
+    # Will print 2.0 because the value was read before other_assign ran. If
+    # `a` was a tf.Variable instead, 2.0 or 3.0 could be printed.
+    tf.Print(b, [b]).eval()
   ```
-
-  To enforce these consistency properties tf.ResourceVariable might make more
-  copies than an equivalent tf.Variable under the hood, so tf.Variable is still
-  not deprecated.
   """
 
   def __init__(self,
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index 5b9947f441..c37cdd9e27 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -125,8 +125,8 @@ class Variable(checkpointable.CheckpointableBase):
 
   @compatibility(eager)
   `tf.Variable` is not compatible with eager execution.  Use
-  `tfe.Variable` instead which is compatible with both eager execution
-  and graph construction.  See [the TensorFlow Eager Execution
+  `tf.contrib.eager.Variable` instead which is compatible with both eager
+  execution and graph construction.  See [the TensorFlow Eager Execution
   guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/g3doc/guide.md#variables-and-optimizers)
   for details on how variables work in eager execution.
   @end_compatibility
-- 
GitLab


From 20edf09163f2757b5c26e4de4a28dc87efa065c0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 11:11:28 -0700
Subject: [PATCH 180/960] Optimizations to DepthwiseConv

PiperOrigin-RevId: 189610985
---
 .../internal/optimized/depthwiseconv_uint8.h  |  3 +-
 .../depthwiseconv_uint8_3x3_filter.h          | 75 ++++++++++++++++---
 2 files changed, 66 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index 08674a6c59..c71b070680 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -1697,7 +1697,8 @@ inline void DepthwiseConv(const uint8* input_data, const Dims<4>& input_dims,
   // Call kernel optimized for depthwise convolutions using 3x3 filters,
   // stride = 1, no padding, depth_multiplier = 1 and depth a multiple of 16.
   if (filter_width == 3 && filter_height == 3 && depth_multiplier == 1 &&
-      stride_width == 1 && stride_height == 1 && pad_width == 0 &&
+      (stride_width == 1 || stride_width == 2) &&
+      (stride_height == 1 || stride_height == 2) && pad_width == 0 &&
       pad_height == 0 && (input_depth % 16) == 0) {
     DepthwiseConv3by3FilterDepth16(
         input_data, input_dims, input_offset, filter_data, filter_dims,
diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
index e0335b2c74..9dc76e7608 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
@@ -466,8 +466,8 @@ inline void DepthwiseConv3by3FilterDepth16(
   TFLITE_DCHECK(filter_width == 3);
   TFLITE_DCHECK(pad_height == 0);
   TFLITE_DCHECK(pad_width == 0);
-  TFLITE_DCHECK(stride_width == 1);
-  TFLITE_DCHECK(stride_height == 1);
+  TFLITE_DCHECK(stride_width == 1 || stride_width == 2);
+  TFLITE_DCHECK(stride_height == 1 || stride_height == 2);
 
   // The number of outputs to process in the main loop.
   const int num_x_outputs = 1;
@@ -513,6 +513,16 @@ inline void DepthwiseConv3by3FilterDepth16(
     }
   }
 
+  using dot_product_func_t =
+      decltype(&ConvKernel3x3FilterDepth16<1, 2, 1>::Run);
+  dot_product_func_t dot_product_func = nullptr;
+
+  if (stride_width == 1 && stride_height == 1) {
+    dot_product_func = ConvKernel3x3FilterDepth16<1, 2, 1>::Run;
+  } else {
+    dot_product_func = ConvKernel3x3FilterDepth16<1, 2, 2>::Run;
+  }
+
   // Offsets for preloading inputs.
   const int i0 = 0;
   const int i1 = input_depth;
@@ -526,6 +536,9 @@ inline void DepthwiseConv3by3FilterDepth16(
   const int i9 = 3 * input_row_width;
   const int i10 = 3 * input_row_width + input_depth;
   const int i11 = 3 * input_row_width + 2 * input_depth;
+  const int i12 = 4 * input_row_width;
+  const int i13 = 4 * input_row_width + input_depth;
+  const int i14 = 4 * input_row_width + 2 * input_depth;
 
   for (int b = 0; b < batches; ++b) {
     const int32* bias_ptr = bias_data;
@@ -551,10 +564,6 @@ inline void DepthwiseConv3by3FilterDepth16(
         const uint8* input_ptr =
             input_data + depth + in_x_offset + in_y_offset + in_batch_offset;
 
-        uint8* output_ptr = output_data + depth + (out_x * output_depth) +
-                            (output_depth * output_width * out_y) +
-                            out_batch_offset;
-
         // Preload inputs. If input depth is large, preload every value of the
         // input for this depth range. Otherwise, preload only the first values
         // of each row.
@@ -571,19 +580,33 @@ inline void DepthwiseConv3by3FilterDepth16(
           preload_l1_keep(input_ptr + i9);
           preload_l1_keep(input_ptr + i10);
           preload_l1_keep(input_ptr + i11);
+
+          if (stride_height == 2) {
+            preload_l1_keep(input_ptr + i12);
+            preload_l1_keep(input_ptr + i13);
+            preload_l1_keep(input_ptr + i14);
+          }
         } else {
           preload_l1_keep(input_ptr + i0);
           preload_l1_keep(input_ptr + i3);
           preload_l1_keep(input_ptr + i6);
           preload_l1_keep(input_ptr + i9);
+
+          if (stride_height == 2) {
+            preload_l1_keep(input_ptr + i12);
+          }
         }
 
+        uint8* output_ptr = output_data + depth + (out_x * output_depth) +
+                            (output_depth * output_width * out_y) +
+                            out_batch_offset;
+
         for (; out_x < out_x_end; out_x += num_x_outputs) {
-          ConvKernel3x3FilterDepth16<1, 2, 1>::Run(
-              filter, input_ptr, input_depth, input_offset, input_row_width,
-              bias_ptr, output_offset, output_multiplier, output_shift,
-              output_activation_min, output_activation_max, output_ptr,
-              output_depth, output_width);
+          dot_product_func(filter, input_ptr, input_depth, input_offset,
+                           input_row_width, bias_ptr, output_offset,
+                           output_multiplier, output_shift,
+                           output_activation_min, output_activation_max,
+                           output_ptr, output_depth, output_width);
 
           input_ptr += input_ptr_x_increment * num_x_outputs;
           output_ptr += output_depth * num_x_outputs;
@@ -603,6 +626,8 @@ inline void DepthwiseConv3by3FilterDepth16(
             preload_l1_keep(input_ptr + i8);
             preload_l1_keep(input_ptr + i10);
             preload_l1_keep(input_ptr + i11);
+            preload_l1_keep(input_ptr + i13);
+            preload_l1_keep(input_ptr + i14);
           }
         }
 
@@ -624,6 +649,21 @@ inline void DepthwiseConv3by3FilterDepth16(
         const uint8* input_ptr =
             input_data + depth + in_x_offset + in_y_offset + in_batch_offset;
 
+        if (input_depth >= 32) {
+          preload_l1_keep(input_ptr + i0);
+          preload_l1_keep(input_ptr + i1);
+          preload_l1_keep(input_ptr + i2);
+          preload_l1_keep(input_ptr + i3);
+          preload_l1_keep(input_ptr + i4);
+          preload_l1_keep(input_ptr + i5);
+          preload_l1_keep(input_ptr + i6);
+          preload_l1_keep(input_ptr + i7);
+        } else {
+          preload_l1_keep(input_ptr + i0);
+          preload_l1_keep(input_ptr + i3);
+          preload_l1_keep(input_ptr + i6);
+        }
+
         uint8* output_ptr = output_data + depth + (out_x * output_depth) +
                             (output_depth * output_width * out_y) +
                             out_batch_offset;
@@ -637,6 +677,19 @@ inline void DepthwiseConv3by3FilterDepth16(
 
           input_ptr += input_ptr_x_increment;
           output_ptr += output_depth;
+
+          if (stride_width == 1) {
+            preload_l1_keep(input_ptr + i2);
+            preload_l1_keep(input_ptr + i5);
+            preload_l1_keep(input_ptr + i8);
+          } else if (stride_width == 2) {
+            preload_l1_keep(input_ptr + i1);
+            preload_l1_keep(input_ptr + i2);
+            preload_l1_keep(input_ptr + i4);
+            preload_l1_keep(input_ptr + i5);
+            preload_l1_keep(input_ptr + i7);
+            preload_l1_keep(input_ptr + i8);
+          }
         }
       }
       filter_ptr += 16;
-- 
GitLab


From cb7a530e6f9648377d92b32db6347d5f0777cbb3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 11:27:45 -0700
Subject: [PATCH 181/960] Internal change

PiperOrigin-RevId: 189613870
---
 tensorflow/contrib/lite/schema/BUILD    | 8 +++-----
 tensorflow/contrib/session_bundle/BUILD | 4 +---
 2 files changed, 4 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/lite/schema/BUILD b/tensorflow/contrib/lite/schema/BUILD
index a758c5e7e1..da65ec659c 100644
--- a/tensorflow/contrib/lite/schema/BUILD
+++ b/tensorflow/contrib/lite/schema/BUILD
@@ -1,8 +1,6 @@
-package(
-    default_visibility = [
-        "//visibility:public",
-    ],
-)
+package(default_visibility = [
+    "//visibility:public",
+])
 
 licenses(["notice"])  # Apache 2.0
 
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 67011c8fef..75a753ed89 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -1,9 +1,7 @@
 # Description:
 #   TensorFlow Serving session bundle.
 
-package(
-    default_visibility = ["//visibility:public"],
-)
+package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
 
-- 
GitLab


From af35a55a5db07160901ea244c7619e1db5a13e1d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 11:29:44 -0700
Subject: [PATCH 182/960] Do not use SparseMatmul to for bfloat16 as Matmul is
 already supported.

PiperOrigin-RevId: 189614197
---
 tensorflow/python/ops/math_ops.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index e18d0e9501..c893bf9b90 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -2093,8 +2093,9 @@ def matmul(a,
       sparse_matmul_types = [dtypes.bfloat16, dtypes.float32]
       use_sparse_matmul = (
           a.dtype in sparse_matmul_types and b.dtype in sparse_matmul_types)
-    if a.dtype == dtypes.bfloat16 or b.dtype == dtypes.bfloat16:
-      # matmul currently doesn't handle bfloat16 inputs.
+    if (a.dtype == dtypes.bfloat16 or b.dtype == dtypes.bfloat16 and
+        a.dtype != b.dtype):
+      # matmul currently doesn't handle mixed-precision inputs.
       use_sparse_matmul = True
     if use_sparse_matmul:
       ret = sparse_matmul(
-- 
GitLab


From 57d117db96ef84e4fe12b74c9115421767db4531 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 12:15:59 -0700
Subject: [PATCH 183/960]   Add a map from TPU core id to name to TfOpStats.

PiperOrigin-RevId: 189620850
---
 tensorflow/contrib/tpu/profiler/tf_op_stats.proto | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
index e5c798aa2f..20ed7419fd 100644
--- a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
+++ b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
@@ -223,4 +223,6 @@ message TfOpStats {
   optional RunEnvironmentResult run_environment = 7;
   // The result for the host operations.
   optional HostOpsResult host_ops = 8;
+  // A map from core ID to name.
+  map<uint32, string> core_id_to_name_map = 9;
 }
-- 
GitLab


From 12ead8d98e2ff05998b8b502eb0a584ddeb275f4 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 19 Mar 2018 13:01:58 -0700
Subject: [PATCH 184/960] Checkpointable: Small cleanup making better use of
 NewCheckpointReader.

PiperOrigin-RevId: 189627956
---
 tensorflow/contrib/eager/python/BUILD         |  5 ++-
 .../eager/python/checkpointable_utils.py      | 32 +++----------------
 2 files changed, 8 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 32aa2c0a4a..4fba014d6f 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -233,12 +233,15 @@ py_library(
         "//tensorflow/python:constant_op",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:init_ops",
-        "//tensorflow/python:io_ops",
+        "//tensorflow/python:pywrap_tensorflow",
         "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:session",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:training",
+        "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/eager:context",
     ],
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index 0a34f3b3f6..adbb92e43b 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -32,7 +32,6 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import io_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.training import checkpointable as core_checkpointable
@@ -577,7 +576,6 @@ class CheckpointableSaver(object):
     self._last_save_saver = None
 
     # Op caching for restore
-    self._object_graph_restore_tensor = None
     self._last_restore_object_graph = None
     self._last_restore_checkpoint = None
 
@@ -660,7 +658,7 @@ class CheckpointableSaver(object):
             attribute_proto.checkpoint_key]
     return saver_names
 
-  def restore(self, save_path, session=None):
+  def restore(self, save_path):
     """Restore a training checkpoint.
 
     Restores `root_checkpointable` and any objects that it tracks
@@ -670,8 +668,7 @@ class CheckpointableSaver(object):
     constructor after this call will be matched if they have a corresponding
     object in the checkpoint.
 
-    When building a graph, restorations are added to the graph but not run. A
-    session is required to retrieve checkpoint metadata.
+    When building a graph, restorations are added to the graph but not run.
 
     To disallow deferred loading, assert immediately that all checkpointed
     variables have been matched to variable objects:
@@ -709,9 +706,6 @@ class CheckpointableSaver(object):
         object which may run initializers for objects in the dependency
         graph. If the checkpoint was written by the name-based `tf.train.Saver`,
         names are used to match variables.
-      session: The session to retrieve metadata with. Ignored when executing
-        eagerly. If not provided when graph building, the default session is
-        used.
 
     Returns:
       A load status object, which can be used to make assertions about the
@@ -726,32 +720,15 @@ class CheckpointableSaver(object):
       return InitializationOnlyStatus(self._root_checkpointable)
     in_graph_mode = not context.executing_eagerly()
     if in_graph_mode:
-      if session is None:
-        session = ops.get_default_session()
       file_prefix_tensor = self._file_prefix_placeholder
       file_prefix_feed_dict = {self._file_prefix_placeholder: save_path}
     else:
-      session = None
       with ops.device("/cpu:0"):
         file_prefix_tensor = constant_op.constant(save_path)
       file_prefix_feed_dict = None
+    reader = pywrap_tensorflow.NewCheckpointReader(save_path)
     try:
-      if not in_graph_mode or self._object_graph_restore_tensor is None:
-        with ops.device("/cpu:0"):
-          object_graph_string, = io_ops.restore_v2(
-              prefix=file_prefix_tensor,
-              tensor_names=[_OBJECT_GRAPH_PROTO_KEY],
-              shape_and_slices=[""],
-              dtypes=[dtypes.string],
-              name="object_graph_proto_read")
-        if in_graph_mode:
-          self._object_graph_restore_tensor = object_graph_string
-      if in_graph_mode:
-        object_graph_string = session.run(
-            self._object_graph_restore_tensor,
-            feed_dict=file_prefix_feed_dict)
-      else:
-        object_graph_string = object_graph_string.numpy()
+      object_graph_string = reader.get_tensor(_OBJECT_GRAPH_PROTO_KEY)
     except errors_impl.NotFoundError:
       # The object graph proto does not exist in this checkpoint. Try again with
       # name-based saving.
@@ -766,7 +743,6 @@ class CheckpointableSaver(object):
       if in_graph_mode:
         dtype_map = None
       else:
-        reader = pywrap_tensorflow.NewCheckpointReader(save_path)
         dtype_map = reader.get_variable_to_dtype_map()
       checkpoint = core_checkpointable_utils._Checkpoint(  # pylint: disable=protected-access
           object_graph_proto=object_graph_proto,
-- 
GitLab


From 774095829ec262f2e1cb4e73938410e0248bc57c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 13:03:22 -0700
Subject: [PATCH 185/960] Extract GraphOptimizer{Stage,Context}, and use it as
 a base class in ArithmeticOptimizer.

PiperOrigin-RevId: 189628227
---
 tensorflow/core/grappler/optimizers/BUILD     |  32 +++
 .../optimizers/arithmetic_optimizer.cc        | 267 ++++--------------
 .../optimizers/graph_optimizer_stage.cc       | 120 ++++++++
 .../optimizers/graph_optimizer_stage.h        | 185 ++++++++++++
 .../optimizers/graph_optimizer_stage_test.cc  | 168 +++++++++++
 5 files changed, 553 insertions(+), 219 deletions(-)
 create mode 100644 tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
 create mode 100644 tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
 create mode 100644 tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 3499879dee..96ea8f7a83 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -202,6 +202,37 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "graph_optimizer_stage",
+    srcs = ["graph_optimizer_stage.cc"],
+    hdrs = ["graph_optimizer_stage.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:utils",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/utils:frame",
+    ],
+)
+
+tf_cc_test(
+    name = "graph_optimizer_stage_test",
+    size = "small",
+    srcs = ["graph_optimizer_stage_test.cc"],
+    deps = [
+        ":graph_optimizer_stage",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:utils",
+        "//tensorflow/core/grappler/costs:graph_properties",
+        "//tensorflow/core/grappler/utils:grappler_test",
+    ],
+)
+
 cc_library(
     name = "custom_graph_optimizer",
     hdrs = [
@@ -224,6 +255,7 @@ cc_library(
     deps = [
         ":constant_folding",
         ":graph_optimizer",
+        ":graph_optimizer_stage",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index c25836ceef..942724a6ce 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
+#include "tensorflow/core/grappler/optimizers/graph_optimizer_stage.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/lib/core/errors.h"
@@ -288,170 +289,29 @@ NodeDef* GetTailOfValuePreservingChain(
                         is_value_preserving_non_branching);
 }
 
-// Context passed to each arithmetic optimizer stage. Optimizer stage is
-// responsible for updating the node map for all added or deleted nodes, to keep
-// it consistent with optimized graph.
+// Graph optimizer context extension specific to ArithmeticOptimizer
 struct ArithmeticOptimizerContext {
-  ArithmeticOptimizerContext(
-      const std::unordered_set<string>* nodes_to_preserve,
-      GraphDef* optimized_graph, GraphProperties* graph_properties,
-      NodeMap* node_map, FrameMap* frame_map,
-      SetVector<NodeDef*>* nodes_to_simplify)
-      : nodes_to_preserve(nodes_to_preserve),
-        optimized_graph(optimized_graph),
-        graph_properties(graph_properties),
-        node_map(node_map),
-        frame_map(frame_map),
-        nodes_to_simplify(nodes_to_simplify) {}
-
-  const std::unordered_set<string>* nodes_to_preserve;
-  GraphDef* optimized_graph;
-  GraphProperties* graph_properties;
-  NodeMap* node_map;
-  FrameMap* frame_map;
+  explicit ArithmeticOptimizerContext(SetVector<NodeDef*>* nodes_to_simplify)
+      : nodes_to_simplify(nodes_to_simplify) {}
   SetVector<NodeDef*>* nodes_to_simplify;
 };
 
 // Base class for single arithmetic optimization: e.g. Bitcast optimization,
 // AddOps optimization, etc...
-// TODO(ezhulenev): extract this class to be reused by other multi-stage
-// graph optimizers (const_folding, dependency_optimizer, etc...)
-class ArithmeticOptimizerStage {
+class ArithmeticOptimizerStage : public GraphOptimizerStage<string> {
  public:
   explicit ArithmeticOptimizerStage(const string& name,
-                                    const ArithmeticOptimizerContext& ctx)
-      : name_(name), ctx_(ctx) {}
+                                    const GraphOptimizerContext& ctx,
+                                    const ArithmeticOptimizerContext ctx_ext)
+      : GraphOptimizerStage("ArithmeticOptimizer", name, ctx),
+        ctx_ext_(ctx_ext) {}
   virtual ~ArithmeticOptimizerStage() = default;
 
-  // Check if we should try to simplify node. Returning true doesn't
-  // guarantee that node will be simplified.
-  //
-  // Should implement just a basic sanity check, without any expensive graph
-  // traversals.
-  virtual bool IsSupported(const NodeDef* node) const = 0;
-
-  // Try to simplify the given node. If successfully simplified a given node,
-  // return a name of a new simplified version using output parameter.
-  //
-  // Consumers of an old node's outputs will be automatically re-wired to
-  // consume outputs of a new simplified node.
-  //
-  // Return error status only if some precondition is failed, or got an
-  // incorrect graph. In every other case return Status:OK(), even if didn't
-  // simplify anything.
-  //
-  // A simplified node will be always considered for further optimization and
-  // will be automatically added to the optimization queue. If a simplified node
-  // has the same name as original node it has to be explicitly added to the
-  // optimization queue for second pass.
-  virtual Status TrySimplify(NodeDef* node, string* simplified_node_name) = 0;
-
- protected:
-  struct ScopedNodeName {
-    string scope;
-    string name;
-  };
-
-  const ScopedNodeName ParseScopedNodeName(const string& name) const {
-    auto pos = name.find_last_of("/");
-    if (pos == string::npos) {
-      return {"", name};
-    } else {
-      return {name.substr(0, pos), name.substr(pos + 1)};
-    }
-  }
-
-  // Prefix optimized node name with stage name and rewrite_rule
-  const string OptimizedNodeName(const string& rewrite_rule,
-                                 const ScopedNodeName& scoped_node_name) const {
-    return MakeOptimizedNodeName(strings::StrCat(name_, "_", rewrite_rule),
-                                 scoped_node_name);
-  }
-
-  // Prefix optimized node name with stage name and rewrite_rule
-  const string OptimizedNodeName(const string& rewrite_rule,
-                                 const ScopedNodeName& scoped_node_name,
-                                 const std::vector<string>& node_names) const {
-    return MakeOptimizedNodeName(strings::StrCat(name_, "_", rewrite_rule),
-                                 scoped_node_name, node_names);
-  }
-
-  // Prefix optimized node name with stage name
-  const string OptimizedNodeName(const ScopedNodeName& scoped_node_name) const {
-    return MakeOptimizedNodeName(name_, scoped_node_name);
-  }
-
-  // Prefix optimized node name with stage name
-  const string OptimizedNodeName(const ScopedNodeName& scoped_node_name,
-                                 const std::vector<string>& node_names) const {
-    return MakeOptimizedNodeName(name_, scoped_node_name, node_names);
-  }
-
   // Simplification graph rewrite can create additional nodes that are inputs
   // to final simplified node, they can be also added to the arithmetic
   // optimizer queue for further optimization.
   void AddToOptimizationQueue(NodeDef* node) {
-    ctx_.nodes_to_simplify->PushBack(node);
-  }
-
-  // Get a node by input name from a node map. Return an error if node was not
-  // found.
-  Status GetInputNode(const string& input, NodeDef** node) const {
-    string node_name = NodeName(input);
-    NodeDef* node_by_name = ctx_.node_map->GetNode(node_name);
-    if (node_by_name == nullptr) {
-      return errors::FailedPrecondition("Node ", node_name,
-                                        " doesn't exists in a node map");
-    }
-    *node = node_by_name;
-    return Status::OK();
-  }
-
-  // Lookup tensor properties by name. Tensor name might have non-zero port
-  // number. Return an error if tensor node doesn't exists in a graph, or it
-  // doesn't have properties defined for requested port.
-  Status GetTensorProperties(const string& tensor,
-                             OpInfo::TensorProperties* properties) const {
-    int port;
-    string tensor_node_name = ParseNodeName(tensor, &port);
-    if (port < 0) {
-      return errors::InvalidArgument(
-          "Can't get tensor properties of control dependency ", tensor);
-    }
-
-    const auto& output_properties =
-        ctx_.graph_properties->GetOutputProperties(tensor_node_name);
-    auto num_outputs = output_properties.size();
-
-    if (num_outputs == 0 || port > num_outputs - 1) {
-      return errors::InvalidArgument(
-          "Node ", tensor_node_name,
-          " is missing output properties at position :", port,
-          " (num_outputs=", num_outputs, ")");
-    }
-
-    properties->CopyFrom(output_properties[port]);
-    return Status::OK();
-  }
-
-  NodeDef* AddCopyNode(const string& name, const NodeDef* node_to_copy) {
-    CHECK(node_to_copy != nullptr);
-    CHECK(!ctx_.node_map->NodeExists(name))
-        << "Node " << name << " already exists in a graph";
-    NodeDef* new_node = ctx_.optimized_graph->add_node();
-    *new_node = *node_to_copy;
-    new_node->set_name(name);
-    ctx_.node_map->AddNode(name, new_node);
-    return new_node;
-  }
-
-  NodeDef* AddEmptyNode(const string& name) {
-    CHECK(!ctx_.node_map->NodeExists(name))
-        << "Node " << name << " already exists in a graph";
-    NodeDef* new_node = ctx_.optimized_graph->add_node();
-    new_node->set_name(name);
-    ctx_.node_map->AddNode(name, new_node);
-    return new_node;
+    ctx_ext_.nodes_to_simplify->PushBack(node);
   }
 
   // TODO(ezhulenev): remove this method from ArithmeticOptimizer when all
@@ -476,49 +336,9 @@ class ArithmeticOptimizerStage {
     }
   }
 
-  const string name_;
-  const ArithmeticOptimizerContext ctx_;
-
  private:
-  // Get a name for a new node obtained by optimizing a single node of the
-  // original graph. The optimized node is placed under the original node scope.
-  //
-  // Node name uniqueness is guaranteed by unique name of an original node in
-  // a same scope.
-  //
-  // Example: MakeOptimizedNodeName("AwesomeRewrite", "a/b/c/Add_1")
-  // Optimized name: "a/b/c/ArithmeticOptimizer/AwesomeRewrite_Add_1"
-  const string MakeOptimizedNodeName(
-      const string& prefix, const ScopedNodeName& scoped_node_name) const {
-    string node_name;
-    strings::StrAppend(&node_name, scoped_node_name.scope);
-    if (!node_name.empty()) strings::StrAppend(&node_name, "/");
-    strings::StrAppend(&node_name, kArithmeticOptimizer, "/", prefix, "_",
-                       scoped_node_name.name);
-    return node_name;
-  }
-
-  // Get a name for a new node obtained by optimizing multiple nodes of the
-  // original graph, starting from "root". The optimized node is placed under
-  // the original scope of a "root" node.
-  //
-  // Node name uniqueness is guaranteed by unique name of a "root" node in
-  // a same scope.
-  //
-  // Example:
-  //   MakeOptimizedNodeName("AwesomeRewrite", "a/b/Add_AB", ["x/y/Add_XY"])
-  // Optimized name:
-  //   "a/b/ArithmeticOptimizer/AwesomeRewrite_Add_AB_Add_XY"
-  const string MakeOptimizedNodeName(
-      const string& prefix, const ScopedNodeName& scoped_node_name,
-      const std::vector<string>& node_names) const {
-    string node_name = MakeOptimizedNodeName(prefix, scoped_node_name);
-    for (const string& optimized : node_names) {
-      auto scoped_node = ParseScopedNodeName(optimized);
-      strings::StrAppend(&node_name, "_", scoped_node.name);
-    }
-    return node_name;
-  }
+  // extened context required for ArithmeticOptimizer
+  const ArithmeticOptimizerContext ctx_ext_;
 };
 
 // Rewrite a tree of Add/AddN with a single AddN operation, consuming all the
@@ -537,8 +357,10 @@ class ArithmeticOptimizerStage {
 //                         q   e
 class AddOpsRewriteStage : public ArithmeticOptimizerStage {
  public:
-  explicit AddOpsRewriteStage(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("AddOpsRewrite", ctx), rewritten_nodes_() {}
+  explicit AddOpsRewriteStage(const GraphOptimizerContext& ctx,
+                              const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("AddOpsRewrite", ctx, ctx_ext),
+        rewritten_nodes_() {}
 
   ~AddOpsRewriteStage() override = default;
 
@@ -718,7 +540,7 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
   string AddOpsGroupName(const AddOpsGroup& group) const {
     CHECK_NOTNULL(group.root_node);
 
-    auto root = ParseScopedNodeName(group.root_node->name());
+    auto root = ParseNodeScopeAndName(group.root_node->name());
 
     std::vector<string> absorbed_node_names(group.absorbed_nodes.size());
     std::transform(group.absorbed_nodes.begin(), group.absorbed_nodes.end(),
@@ -783,8 +605,9 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
 class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
  public:
   explicit HoistCommonFactorOutOfAggregation(
-      const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("HoistCommonFactor", ctx) {}
+      const GraphOptimizerContext& ctx,
+      const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("HoistCommonFactor", ctx, ctx_ext) {}
   ~HoistCommonFactorOutOfAggregation() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
@@ -845,14 +668,14 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
  private:
   // Get a name for new outer Mul node
   string OuterMulNodeName(const NodeDef* node) const {
-    auto scoped_node = ParseScopedNodeName(node->name());
-    return OptimizedNodeName("Mul", scoped_node);
+    auto scope_and_name = ParseNodeScopeAndName(node->name());
+    return OptimizedNodeName(scope_and_name, "Mul");
   }
 
   // Get a name new inner Add node
   string InnerAddNodeName(const NodeDef* node) const {
-    auto scoped_node = ParseScopedNodeName(node->name());
-    return OptimizedNodeName("Add", scoped_node);
+    auto scope_and_name = ParseNodeScopeAndName(node->name());
+    return OptimizedNodeName(scope_and_name, "Add");
   }
 
   // Determine the set of common factors if the input nodes are all Mul nodes.
@@ -932,8 +755,9 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
 // Removes inverse transpose nodes
 class RemoveIdentityTranspose : public ArithmeticOptimizerStage {
  public:
-  explicit RemoveIdentityTranspose(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("RemoveIdentityTranspose", ctx) {}
+  explicit RemoveIdentityTranspose(const GraphOptimizerContext& ctx,
+                                   const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("RemoveIdentityTranspose", ctx, ctx_ext) {}
   ~RemoveIdentityTranspose() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
@@ -1016,8 +840,10 @@ class RemoveIdentityTranspose : public ArithmeticOptimizerStage {
 // 2) Rewrite Bitcast(Bitcast(x, type1), type2) => Bitcast(x, type2)
 class RemoveRedundantBitcastStage : public ArithmeticOptimizerStage {
  public:
-  explicit RemoveRedundantBitcastStage(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("RemoveRedundantBitcast", ctx) {}
+  explicit RemoveRedundantBitcastStage(
+      const GraphOptimizerContext& ctx,
+      const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("RemoveRedundantBitcast", ctx, ctx_ext) {}
   ~RemoveRedundantBitcastStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
@@ -1055,8 +881,9 @@ class RemoveRedundantBitcastStage : public ArithmeticOptimizerStage {
 // Remove Casts whose source type and destination type are equal.
 class RemoveRedundantCastStage : public ArithmeticOptimizerStage {
  public:
-  explicit RemoveRedundantCastStage(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("RemoveRedundantCast", ctx) {}
+  explicit RemoveRedundantCastStage(const GraphOptimizerContext& ctx,
+                                    const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("RemoveRedundantCast", ctx, ctx_ext) {}
   ~RemoveRedundantCastStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override { return IsCast(*node); }
@@ -1073,8 +900,9 @@ class RemoveRedundantCastStage : public ArithmeticOptimizerStage {
 
 class RemoveNegationStage : public ArithmeticOptimizerStage {
  public:
-  explicit RemoveNegationStage(const ArithmeticOptimizerContext& ctx)
-      : ArithmeticOptimizerStage("RemoveNegation", ctx) {}
+  explicit RemoveNegationStage(const GraphOptimizerContext& ctx,
+                               const ArithmeticOptimizerContext& ctx_ext)
+      : ArithmeticOptimizerStage("RemoveNegation", ctx, ctx_ext) {}
   ~RemoveNegationStage() override = default;
 
   bool IsSupported(const NodeDef* node) const override {
@@ -1715,35 +1543,36 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     nodes_to_simplify.PushBack(optimized_graph_->mutable_node(i));
   }
 
-  const ArithmeticOptimizerContext ctx(&nodes_to_preserve_, optimized_graph_,
-                                       graph_properties_.get(), node_map_.get(),
-                                       &frame_map_, &nodes_to_simplify);
+  const GraphOptimizerContext ctx(&nodes_to_preserve_, optimized_graph_,
+                                  graph_properties_.get(), node_map_.get(),
+                                  &frame_map_);
+  const ArithmeticOptimizerContext ctx_ext(&nodes_to_simplify);
 
   std::vector<std::unique_ptr<ArithmeticOptimizerStage>> stages;
 
   if (options_.combine_add_to_addn) {
-    stages.push_back(
-        std::unique_ptr<ArithmeticOptimizerStage>(new AddOpsRewriteStage(ctx)));
+    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
+        new AddOpsRewriteStage(ctx, ctx_ext)));
   }
   if (options_.hoist_common_factor_out_of_aggregation) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new HoistCommonFactorOutOfAggregation(ctx)));
+        new HoistCommonFactorOutOfAggregation(ctx, ctx_ext)));
   }
   if (options_.remove_identity_transpose) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveIdentityTranspose(ctx)));
+        new RemoveIdentityTranspose(ctx, ctx_ext)));
   }
   if (options_.remove_redundant_bitcast) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveRedundantBitcastStage(ctx)));
+        new RemoveRedundantBitcastStage(ctx, ctx_ext)));
   }
   if (options_.remove_redundant_cast) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveRedundantCastStage(ctx)));
+        new RemoveRedundantCastStage(ctx, ctx_ext)));
   }
   if (options_.remove_negation) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveNegationStage(ctx)));
+        new RemoveNegationStage(ctx, ctx_ext)));
   }
 
   VLOG(1) << "Simplify arithmetic ops using " << stages.size()
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
new file mode 100644
index 0000000000..7044705ade
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
@@ -0,0 +1,120 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/graph_optimizer_stage.h"
+
+namespace tensorflow {
+namespace grappler {
+
+const NodeScopeAndName ParseNodeScopeAndName(const string& node_name) {
+  auto pos = node_name.find_last_of("/");
+  if (pos == string::npos) {
+    return {"", node_name};
+  } else {
+    return {node_name.substr(0, pos), node_name.substr(pos + 1)};
+  }
+};
+
+Status GetInputNode(const GraphOptimizerContext& ctx, const string& input,
+                    NodeDef** node) {
+  string node_name = NodeName(input);
+  NodeDef* node_by_name = ctx.node_map->GetNode(node_name);
+  if (node_by_name == nullptr) {
+    return errors::FailedPrecondition("Node ", node_name,
+                                      " doesn't exists in a node map");
+  }
+  *node = node_by_name;
+  return Status::OK();
+}
+
+Status GetTensorProperties(const GraphOptimizerContext& ctx,
+                           const string& tensor,
+                           OpInfo::TensorProperties* properties) {
+  int port;
+  string tensor_node_name = ParseNodeName(tensor, &port);
+  if (port < 0) {
+    return errors::InvalidArgument(
+        "Can't get tensor properties of control dependency ", tensor);
+  }
+
+  const auto& output_properties =
+      ctx.graph_properties->GetOutputProperties(tensor_node_name);
+  auto num_outputs = output_properties.size();
+
+  if (num_outputs == 0 || port > num_outputs - 1) {
+    return errors::InvalidArgument(
+        "Node ", tensor_node_name,
+        " is missing output properties at position :", port,
+        " (num_outputs=", num_outputs, ")");
+  }
+
+  properties->CopyFrom(output_properties[port]);
+  return Status::OK();
+}
+
+NodeDef* AddCopyNode(const GraphOptimizerContext& ctx, const string& name,
+                     const NodeDef* node_to_copy) {
+  CHECK(node_to_copy != nullptr);
+  CHECK(!ctx.node_map->NodeExists(name))
+      << "Node " << name << " already exists in a graph";
+  NodeDef* new_node = ctx.optimized_graph->add_node();
+  *new_node = *node_to_copy;
+  new_node->set_name(name);
+  ctx.node_map->AddNode(name, new_node);
+  return new_node;
+}
+
+NodeDef* AddEmptyNode(const GraphOptimizerContext& ctx, const string& name) {
+  CHECK(!ctx.node_map->NodeExists(name))
+      << "Node " << name << " already exists in a graph";
+  NodeDef* new_node = ctx.optimized_graph->add_node();
+  new_node->set_name(name);
+  ctx.node_map->AddNode(name, new_node);
+  return new_node;
+}
+
+const string MakeOptimizedNodeName(const NodeScopeAndName& node,
+                                   const string& sub_scope,
+                                   const string& prefix) {
+  CHECK(!sub_scope.empty() || !prefix.empty())
+      << "Either optimized node name prefix or sub-scope must be non-empty";
+  string optimized_node_name;
+  if (!node.scope.empty()) {
+    strings::StrAppend(&optimized_node_name, node.scope, "/");
+  }
+  if (!sub_scope.empty()) {
+    strings::StrAppend(&optimized_node_name, sub_scope, "/");
+  }
+  if (!prefix.empty()) {
+    strings::StrAppend(&optimized_node_name, prefix, "_");
+  }
+  strings::StrAppend(&optimized_node_name, node.name);
+  return optimized_node_name;
+}
+
+const string MakeOptimizedNodeName(const NodeScopeAndName& root,
+                                   const std::vector<string> node_names,
+                                   const string& sub_scope,
+                                   const string& prefix) {
+  string optimized_node_name = MakeOptimizedNodeName(root, sub_scope, prefix);
+  for (const string& node_name : node_names) {
+    auto name_and_scope = ParseNodeScopeAndName(node_name);
+    strings::StrAppend(&optimized_node_name, "_", name_and_scope.name);
+  }
+  return optimized_node_name;
+}
+
+}  // end namespace grappler
+}  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
new file mode 100644
index 0000000000..be95c00d2d
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -0,0 +1,185 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_GRAPPLER_OPTIMIZERS_OPTIMIZER_STAGE_H_
+#define TENSORFLOW_GRAPPLER_OPTIMIZERS_OPTIMIZER_STAGE_H_
+
+#include <unordered_map>
+#include <unordered_set>
+#include "tensorflow/core/grappler/costs/graph_properties.h"
+#include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/grappler/utils/frame.h"
+
+namespace tensorflow {
+namespace grappler {
+
+struct NodeScopeAndName {
+  string scope;
+  string name;
+};
+
+// Parse scope and name: "a/b/c/Add_1" -> {"a/b/c", "Add_1"}
+const NodeScopeAndName ParseNodeScopeAndName(const string& node_name);
+
+// Context owned by GraphOptimizer, and passed to every stage at construction
+// time. Each optimizer stage is responsible for updating it according to the
+// changes it made to the graph.
+//
+// If an optimizer needs access to some helper class that is not present in this
+// context, consider creating an extension context, specific to that
+// optimizer (see example of ArithmeticOptimizerContext). GraphOptimizerContext
+// should only have members that are useful to almost all optimizers.
+struct GraphOptimizerContext {
+  GraphOptimizerContext(const std::unordered_set<string>* nodes_to_preserve,
+                        GraphDef* optimized_graph,
+                        GraphProperties* graph_properties, NodeMap* node_map,
+                        FrameMap* frame_map)
+      : nodes_to_preserve(nodes_to_preserve),
+        optimized_graph(optimized_graph),
+        graph_properties(graph_properties),
+        node_map(node_map),
+        frame_map(frame_map) {}
+
+  const std::unordered_set<string>* nodes_to_preserve;
+  GraphDef* optimized_graph;
+  GraphProperties* graph_properties;
+  NodeMap* node_map;
+  // TODO(ezhulenev): it seems that frame_map is only relevant for loop
+  // optimizer? Move it to loop-optimizer specific context extension.
+  FrameMap* frame_map;
+};
+
+Status GetInputNode(const GraphOptimizerContext& ctx, const string& input,
+                    NodeDef** node);
+Status GetTensorProperties(const GraphOptimizerContext& ctx,
+                           const string& tensor,
+                           OpInfo::TensorProperties* properties);
+
+NodeDef* AddCopyNode(const GraphOptimizerContext& ctx, const string& name,
+                     const NodeDef* node_to_copy);
+NodeDef* AddEmptyNode(const GraphOptimizerContext& ctx, const string& name);
+
+// WARNING:
+// Optimizer stage must try to re-use original nodes of a graph and
+// make all updates in place. This helps to make robust node placement
+// decisions. Create new nodes only if there is a reason for that.
+
+// Make a name for a new node obtained by optimizing a single node of the
+// original graph. The optimized node is placed under the original node scope.
+//
+// Node name uniqueness is guaranteed by unique name of an original node in
+// a same scope.
+//
+// Empty sub_scope or prefix ignored. At least one of them must be non-empty.
+//
+// Example: a/b/c/Add -> a/b/c/${sub_scope}/${prefix}_Add.
+const string MakeOptimizedNodeName(const NodeScopeAndName& node,
+                                   const string& sub_scope,
+                                   const string& prefix);
+// Make a name for a new node obtained by optimizing multiple nodes of the
+// original graph, starting from "root". The optimized node is placed under
+// the original scope of a "root" node.
+//
+// Example: [a/b/c/Add, x/y/z/Mul] -> a/b/c/${sub_scope}/${prefix}_Add_Mul
+const string MakeOptimizedNodeName(const NodeScopeAndName& root,
+                                   const std::vector<string> node_names,
+                                   const string& sub_scope,
+                                   const string& prefix);
+
+// Base class for multi-stage GraphOptimizers (ArithmeticOptimizer, etc...).
+//
+// If a graph optimizer consists of large number of small independent
+// rewrites, each of them should be implemented as a separate stage.
+//
+// * Result:
+// Each graph optimizer choose what result is reported by each stage
+// (e.g. each stage can fill in the name of optimized nodes, or have more
+// complex result).
+template <typename Result>
+class GraphOptimizerStage {
+ public:
+  explicit GraphOptimizerStage(const string& optimizer_name,
+                               const string& stage_name,
+                               const GraphOptimizerContext& ctx)
+      : optimizer_name_(optimizer_name), stage_name_(stage_name), ctx_(ctx) {}
+  virtual ~GraphOptimizerStage() = default;
+
+  // Check if we should try to simplify node. Returning true doesn't
+  // guarantee that node will be simplified.
+  //
+  // Should implement just a basic sanity check, without any expensive graph
+  // traversals.
+  virtual bool IsSupported(const NodeDef* node) const = 0;
+
+  // Try to simplify the given node.
+  //
+  // Return error status only if some precondition is failed, or got an
+  // incorrect graph. In every other case return Status:OK(), even if didn't
+  // simplify anything.
+  //
+  // Report result using output argument. Each GraphOptimizer can choose it's
+  // own Result type.
+  // TODO(ezhulenev): if it will appear that Result output parameter is not
+  // sufficiently useful (used with a reason by most optimizers), get rid of it,
+  // and remove template parameter.
+  virtual Status TrySimplify(NodeDef* node, Result* result) = 0;
+
+  // Get a name for a new node, created by this stage, based on one or multiple
+  // nodes of an original graph.
+  const string OptimizedNodeName(const NodeScopeAndName& node) const {
+    return MakeOptimizedNodeName(node, optimizer_name_, stage_name_);
+  }
+  const string OptimizedNodeName(const NodeScopeAndName& root,
+                                 const std::vector<string>& nodes) const {
+    return MakeOptimizedNodeName(root, nodes, optimizer_name_, stage_name_);
+  }
+  const string OptimizedNodeName(const NodeScopeAndName& node,
+                                 const string& rewrite_rule) const {
+    const string prefix = strings::StrCat(stage_name_, "_", rewrite_rule);
+    return MakeOptimizedNodeName(node, optimizer_name_, prefix);
+  }
+
+  // Get a node by input name from a node map. Return an error if node was not
+  // found.
+  Status GetInputNode(const string& input, NodeDef** node) const {
+    return ::tensorflow::grappler::GetInputNode(ctx_, input, node);
+  }
+  // Lookup tensor properties by name. Tensor name might have non-zero port
+  // number. Return an error if tensor node doesn't exists in a graph, or it
+  // doesn't have properties defined for requested port.
+  Status GetTensorProperties(const string& tensor,
+                             OpInfo::TensorProperties* properties) const {
+    return ::tensorflow::grappler::GetTensorProperties(ctx_, tensor,
+                                                       properties);
+  }
+
+  NodeDef* AddCopyNode(const string& name, const NodeDef* node_to_copy) {
+    return ::tensorflow::grappler::AddCopyNode(ctx_, name, node_to_copy);
+  }
+  NodeDef* AddEmptyNode(const string& name) {
+    return ::tensorflow::grappler::AddEmptyNode(ctx_, name);
+  }
+
+ protected:  // Data members
+  const string optimizer_name_;
+  const string stage_name_;
+  const GraphOptimizerContext ctx_;
+};
+
+}  // end namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_GRAPPLER_OPTIMIZERS_OPTIMIZER_STAGE_H_
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc b/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc
new file mode 100644
index 0000000000..416327e622
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc
@@ -0,0 +1,168 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/graph_optimizer_stage.h"
+
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/grappler/costs/graph_properties.h"
+#include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace grappler {
+namespace {
+
+class GraphOptimizerStageTest : public ::testing::Test {};
+
+struct FakeResult {};
+
+// NoOp optimizer stage that supports all the node types and does nothing
+class FakeOptimizerStage : public GraphOptimizerStage<FakeResult> {
+ public:
+  explicit FakeOptimizerStage(const string& optimizer_name,
+                              const string& stage_name,
+                              const GraphOptimizerContext& ctx)
+      : GraphOptimizerStage(optimizer_name, stage_name, ctx) {}
+  ~FakeOptimizerStage() override = default;
+
+  bool IsSupported(const NodeDef* node) const override { return true; }
+  Status TrySimplify(NodeDef* node, FakeResult* result) override {
+    return Status::OK();
+  }
+};
+
+TEST_F(GraphOptimizerStageTest, ParseNodeNameAndScope_InRoot) {
+  const auto scope_and_name = ParseNodeScopeAndName("Add");
+  EXPECT_EQ("", scope_and_name.scope);
+  EXPECT_EQ("Add", scope_and_name.name);
+}
+
+TEST_F(GraphOptimizerStageTest, ParseNodeNameAndScope_InScope) {
+  const auto scope_and_name = ParseNodeScopeAndName("a/b/c/Add");
+  EXPECT_EQ("a/b/c", scope_and_name.scope);
+  EXPECT_EQ("Add", scope_and_name.name);
+}
+
+TEST_F(GraphOptimizerStageTest, OptimizedNodeName) {
+  GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
+                            /*optimized_graph*/ nullptr,
+                            /*graph_properties*/ nullptr, /*node_name*/ nullptr,
+                            /*frame_map*/ nullptr);
+  FakeOptimizerStage stage("my_opt", "my_stg", ctx);
+
+  const auto node = ParseNodeScopeAndName("a/b/c/Add");
+
+  // Without rewrite rule
+  EXPECT_EQ("a/b/c/my_opt/my_stg_Add", stage.OptimizedNodeName(node));
+  EXPECT_EQ(
+      "a/b/c/my_opt/my_stg_Add_Mul_Sqrt",
+      stage.OptimizedNodeName(node, std::vector<string>({"Mul", "Sqrt"})));
+
+  // With rewrite rule
+  const string rewrite = "my_rewrite";
+  EXPECT_EQ("a/b/c/my_opt/my_stg_my_rewrite_Add",
+            stage.OptimizedNodeName(node, rewrite));
+}
+
+TEST_F(GraphOptimizerStageTest, GetInputNodeAndProperties) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto a = ops::Variable(s.WithOpName("a"), {2, 2}, DT_FLOAT);
+  auto b = ops::Variable(s.WithOpName("b"), {2, 2}, DT_FLOAT);
+  auto add = ops::Add(s.WithOpName("Add"), a, b);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphProperties properties(item);
+  TF_CHECK_OK(properties.InferStatically(/*assume_valid_feeds*/ false));
+
+  NodeMap node_map(&item.graph);
+
+  GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
+                            /*optimized_graph*/ &item.graph,
+                            /*graph_properties*/ &properties,
+                            /*node_name*/ &node_map,
+                            /*frame_map*/ nullptr);
+  FakeOptimizerStage stage("my_opt", "my_stg", ctx);
+
+  NodeDef* add_node;
+  TF_CHECK_OK(stage.GetInputNode("Add", &add_node));
+  EXPECT_EQ("a", add_node->input(0));
+  EXPECT_EQ("b", add_node->input(1));
+
+  OpInfo::TensorProperties add_properties;
+  TF_CHECK_OK(stage.GetTensorProperties("Add", &add_properties));
+  EXPECT_EQ(DT_FLOAT, add_properties.dtype());
+
+  OpInfo::TensorProperties a_properties;
+  TF_CHECK_OK(stage.GetTensorProperties("a:0", &a_properties));
+  EXPECT_EQ(DT_FLOAT_REF, a_properties.dtype());
+
+  OpInfo::TensorProperties b_properties;
+  TF_CHECK_OK(stage.GetTensorProperties("b:0", &b_properties));
+  EXPECT_EQ(DT_FLOAT_REF, b_properties.dtype());
+}
+
+TEST_F(GraphOptimizerStageTest, AddNodes) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto a = ops::Variable(s.WithOpName("a"), {2, 2}, DT_FLOAT);
+  auto b = ops::Variable(s.WithOpName("b"), {2, 2}, DT_FLOAT);
+  auto add = ops::Add(s.WithOpName("Add"), a, b);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphProperties properties(item);
+  TF_CHECK_OK(properties.InferStatically(/*assume_valid_feeds*/ false));
+
+  NodeMap node_map(&item.graph);
+
+  GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
+                            /*optimized_graph*/ &item.graph,
+                            /*graph_properties*/ &properties,
+                            /*node_name*/ &node_map,
+                            /*frame_map*/ nullptr);
+  FakeOptimizerStage stage("my_opt", "my_stg", ctx);
+
+  NodeDef* add_node;
+  TF_CHECK_OK(stage.GetInputNode("Add", &add_node));
+
+  // Add a new copy node
+  NodeDef* add_node_copy = stage.AddCopyNode("Add_1", add_node);
+  EXPECT_EQ("Add_1", add_node_copy->name());
+  EXPECT_EQ("Add", add_node_copy->op());
+  EXPECT_EQ("a", add_node_copy->input(0));
+  EXPECT_EQ("b", add_node_copy->input(1));
+
+  // It must be available for by-name lookup
+  NodeDef* add_node_copy_by_name;
+  TF_CHECK_OK(stage.GetInputNode("Add_1", &add_node_copy_by_name));
+  EXPECT_EQ(add_node_copy, add_node_copy_by_name);
+
+  // Add new empty node
+  NodeDef* empty_node = stage.AddEmptyNode("Add_2");
+  EXPECT_EQ("Add_2", empty_node->name());
+
+  // It must be available for by-name lookup
+  NodeDef* empty_node_by_name;
+  TF_CHECK_OK(stage.GetInputNode("Add_2", &empty_node_by_name));
+  EXPECT_EQ(empty_node, empty_node_by_name);
+}
+
+}  // namespace
+}  // end namespace grappler
+}  // end namespace tensorflow
\ No newline at end of file
-- 
GitLab


From 36ec749ec79c2313924666a1c5324620e493d0c4 Mon Sep 17 00:00:00 2001
From: Terry Koo <terrykoo@google.com>
Date: Mon, 19 Mar 2018 13:20:12 -0700
Subject: [PATCH 186/960] Adds missing protobuf dep to tf.contrib.data ops.
 (#17840)

* Adds missing protobuf dep to tf.contrib.data ops.

I think this will help resolve the following:
https://github.com/tensorflow/serving/issues/421
https://github.com/tensorflow/serving/issues/684
https://github.com/tensorflow/tensorflow/issues/17619

Or at least I was experiencing a similar issue and this change resolved it for me in my local repo.

* s/third_party//
---
 tensorflow/contrib/data/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 0458199ff7..5ba2297e7f 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -29,7 +29,10 @@ py_library(
 tf_custom_op_library(
     name = "_dataset_ops.so",
     srcs = ["ops/dataset_ops.cc"],
-    deps = ["//tensorflow/contrib/data/kernels:dataset_kernels"],
+    deps = [
+        "//tensorflow/contrib/data/kernels:dataset_kernels",
+        "//tensorflow/core:lib_proto_parsing",
+    ],
 )
 
 tf_gen_op_libs(
-- 
GitLab


From eb03b44049328404eb5578efda0729ca1a4f0a11 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 13:26:19 -0700
Subject: [PATCH 187/960] Add bfloat16 support for CPU ops.

PiperOrigin-RevId: 189631659
---
 tensorflow/core/kernels/cwise_op_div.cc        | 8 ++++----
 tensorflow/core/kernels/cwise_op_less.cc       | 4 ++--
 tensorflow/core/kernels/cwise_op_less_equal.cc | 4 ++--
 tensorflow/core/kernels/cwise_op_minimum.cc    | 4 ++--
 tensorflow/core/kernels/cwise_op_sqrt.cc       | 8 ++++----
 5 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_op_div.cc b/tensorflow/core/kernels/cwise_op_div.cc
index c71c756e44..b12652f7fb 100644
--- a/tensorflow/core/kernels/cwise_op_div.cc
+++ b/tensorflow/core/kernels/cwise_op_div.cc
@@ -16,14 +16,14 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER5(BinaryOp, CPU, "Div", functor::div, float, Eigen::half, double,
-          complex64, complex128);
+REGISTER6(BinaryOp, CPU, "Div", functor::div, float, Eigen::half, double,
+          bfloat16, complex64, complex128);
 REGISTER5(BinaryOp, CPU, "Div", functor::safe_div, uint8, uint16, int16, int32,
           int64);
 REGISTER5(BinaryOp, CPU, "TruncateDiv", functor::safe_div, uint8, uint16, int16,
           int32, int64);
-REGISTER5(BinaryOp, CPU, "RealDiv", functor::div, float, Eigen::half, double,
-          complex64, complex128);
+REGISTER6(BinaryOp, CPU, "RealDiv", functor::div, float, Eigen::half, double,
+          bfloat16, complex64, complex128);
 #if GOOGLE_CUDA
 REGISTER9(BinaryOp, GPU, "Div", functor::div, float, Eigen::half, double, uint8,
           uint16, int16, int64, complex64, complex128);
diff --git a/tensorflow/core/kernels/cwise_op_less.cc b/tensorflow/core/kernels/cwise_op_less.cc
index 00cdecdbd1..575968126f 100644
--- a/tensorflow/core/kernels/cwise_op_less.cc
+++ b/tensorflow/core/kernels/cwise_op_less.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER8(BinaryOp, CPU, "Less", functor::less, float, Eigen::half, double,
-          int32, int64, uint8, int8, int16);
+REGISTER9(BinaryOp, CPU, "Less", functor::less, float, Eigen::half, double,
+          bfloat16, int32, int64, uint8, int8, int16);
 #if GOOGLE_CUDA
 REGISTER7(BinaryOp, GPU, "Less", functor::less, float, Eigen::half, double,
           int64, uint8, int8, int16);
diff --git a/tensorflow/core/kernels/cwise_op_less_equal.cc b/tensorflow/core/kernels/cwise_op_less_equal.cc
index 11806c5fc7..499200d054 100644
--- a/tensorflow/core/kernels/cwise_op_less_equal.cc
+++ b/tensorflow/core/kernels/cwise_op_less_equal.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER8(BinaryOp, CPU, "LessEqual", functor::less_equal, float, Eigen::half,
-          double, int32, int64, uint8, int8, int16);
+REGISTER9(BinaryOp, CPU, "LessEqual", functor::less_equal, float, Eigen::half,
+          bfloat16, double, int32, int64, uint8, int8, int16);
 #if GOOGLE_CUDA
 REGISTER7(BinaryOp, GPU, "LessEqual", functor::less_equal, float, Eigen::half,
           double, int64, uint8, int8, int16);
diff --git a/tensorflow/core/kernels/cwise_op_minimum.cc b/tensorflow/core/kernels/cwise_op_minimum.cc
index dff83df828..9bc3700387 100644
--- a/tensorflow/core/kernels/cwise_op_minimum.cc
+++ b/tensorflow/core/kernels/cwise_op_minimum.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER5(BinaryOp, CPU, "Minimum", functor::minimum, float, Eigen::half,
-          double, int32, int64);
+REGISTER6(BinaryOp, CPU, "Minimum", functor::minimum, float, Eigen::half,
+          bfloat16, double, int32, int64);
 #if GOOGLE_CUDA
 REGISTER4(BinaryOp, GPU, "Minimum", functor::minimum, float, Eigen::half,
           double, int64);
diff --git a/tensorflow/core/kernels/cwise_op_sqrt.cc b/tensorflow/core/kernels/cwise_op_sqrt.cc
index 497756133d..205070761f 100644
--- a/tensorflow/core/kernels/cwise_op_sqrt.cc
+++ b/tensorflow/core/kernels/cwise_op_sqrt.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER5(UnaryOp, CPU, "Sqrt", functor::sqrt, float, Eigen::half, double,
-          complex64, complex128);
+REGISTER6(UnaryOp, CPU, "Sqrt", functor::sqrt, float, Eigen::half, double,
+          bfloat16, complex64, complex128);
 
 #if GOOGLE_CUDA
 REGISTER3(UnaryOp, GPU, "Sqrt", functor::sqrt, float, Eigen::half, double);
@@ -27,8 +27,8 @@ REGISTER3(UnaryOp, GPU, "Sqrt", functor::sqrt, float, Eigen::half, double);
 REGISTER2(UnaryOp, SYCL, "Sqrt", functor::sqrt, float, double);
 #endif  // TENSORFLOW_USE_SYCL
 
-REGISTER5(SimpleBinaryOp, CPU, "SqrtGrad", functor::sqrt_grad, float,
-          Eigen::half, double, complex64, complex128);
+REGISTER6(SimpleBinaryOp, CPU, "SqrtGrad", functor::sqrt_grad, float,
+          Eigen::half, bfloat16, double, complex64, complex128);
 #if GOOGLE_CUDA
 REGISTER3(SimpleBinaryOp, GPU, "SqrtGrad", functor::sqrt_grad, float,
           Eigen::half, double);
-- 
GitLab


From a78c5033e005f76b83df4fd97d0074fcc990f603 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Mon, 19 Mar 2018 13:38:23 -0700
Subject: [PATCH 188/960] TFE: Fix bug encountered when using
 `optimizer.apply_gradients` in a defun.

Prior to this change, `Optimizer` assumed that `not
context.executing_eagerly()` implied that every variable that it was to update
was constructed in a graph. That assumption is incorrect --- TensorFlow
functions can mutate variables captured from or lifted into the eager context. As such, this change removes that assumption.

Fixes #17792

PiperOrigin-RevId: 189633630
---
 tensorflow/python/eager/function_test.py | 32 ++++++++++++++++++++++++
 tensorflow/python/ops/variables.py       |  6 +++++
 tensorflow/python/training/optimizer.py  | 11 +++++++-
 3 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index b9cde16867..fd1d2c25ff 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -37,6 +37,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
+from tensorflow.python.training import gradient_descent
 
 
 class FunctionTest(test.TestCase):
@@ -762,6 +763,37 @@ class AutomaticControlDependenciesTest(test.TestCase):
 
       self.assertAllEqual(f().eval(), 4.0)
 
+  def testOptimizerInDefun(self):
+    def loss(v):
+      return v**2
+
+    optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0)
+
+    @function.defun
+    def train():
+      v = resource_variable_ops.ResourceVariable(1.0)
+      grad = backprop.implicit_grad(loss)(v)
+      optimizer.apply_gradients(grad)
+      return v.read_value()
+
+    value = train()
+    self.assertEqual(value.numpy(), -1.0)
+
+  def testOptimizerInDefunWithCapturedVariable(self):
+    v = resource_variable_ops.ResourceVariable(1.0)
+    def loss():
+      return v**2
+
+    optimizer = gradient_descent.GradientDescentOptimizer(learning_rate=1.0)
+
+    @function.defun
+    def train():
+      grad = backprop.implicit_grad(loss)()
+      optimizer.apply_gradients(grad)
+
+    train()
+    self.assertEqual(v.numpy(), -1.0)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/ops/variables.py b/tensorflow/python/ops/variables.py
index c37cdd9e27..c646f79589 100644
--- a/tensorflow/python/ops/variables.py
+++ b/tensorflow/python/ops/variables.py
@@ -293,6 +293,7 @@ class Variable(checkpointable.CheckpointableBase):
     Raises:
       ValueError: If the initial value is not specified, or does not have a
         shape and `validate_shape` is `True`.
+      RuntimeError: If lifted into the eager context.
     """
     _ = expected_shape
     if initial_value is None:
@@ -319,6 +320,11 @@ class Variable(checkpointable.CheckpointableBase):
     if trainable and ops.GraphKeys.TRAINABLE_VARIABLES not in collections:
       collections = list(collections) + [ops.GraphKeys.TRAINABLE_VARIABLES]
     with ops.init_scope():
+      # Ensure that we weren't lifted into the eager context.
+      if context.executing_eagerly():
+        raise RuntimeError(
+            "tf.Variable not supported when eager execution is enabled. "
+            "Please use tf.contrib.eager.Variable instead")
       with ops.name_scope(name, "Variable", [] if init_from_fn else
                           [initial_value]) as name:
 
diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index af9cc3491c..bf79714f96 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -191,6 +191,10 @@ def _get_processor(v):
       return _TensorProcessor(v)
     else:
       return _DenseResourceVariableProcessor(v)
+  if isinstance(
+      v, resource_variable_ops.ResourceVariable) and not v._in_graph_mode:  # pylint: disable=protected-access
+    # True if and only if `v` was initialized eagerly.
+    return _DenseResourceVariableProcessor(v)
   if v.op.type == "VarHandleOp":
     return _DenseResourceVariableProcessor(v)
   if isinstance(v, variables.Variable):
@@ -546,7 +550,12 @@ class Optimizer(
         # We colocate all ops created in _apply_dense or _apply_sparse
         # on the same device as the variable.
         # TODO(apassos): figure out how to get the variable name here.
-        scope_name = "" if context.executing_eagerly() else var.op.name
+        if context.executing_eagerly() or isinstance(
+            var,
+            resource_variable_ops.ResourceVariable) and not var._in_graph_mode:  # pylint: disable=protected-access
+          scope_name = ""
+        else:
+          scope_name = var.op.name
         with ops.name_scope("update_" + scope_name), ops.colocate_with(var):
           update_ops.append(processor.update_op(self, grad))
       if global_step is None:
-- 
GitLab


From 60a37c43c1504f5a1957f2f319bcd1e907be4c18 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 19 Mar 2018 13:43:50 -0700
Subject: [PATCH 189/960] Moves TFE_Executor to tensorflow::EagerExecutor in
 tensorflow/core/common_runtime/eager

PiperOrigin-RevId: 189634404
---
 tensorflow/c/eager/BUILD                      |   2 +
 tensorflow/c/eager/c_api.cc                   | 154 ++----------------
 tensorflow/c/eager/c_api_internal.h           | 104 +-----------
 tensorflow/core/BUILD                         |   3 +
 .../common_runtime/eager/eager_executor.cc    | 152 +++++++++++++++++
 .../common_runtime/eager/eager_executor.h     | 138 ++++++++++++++++
 6 files changed, 312 insertions(+), 241 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/eager_executor.cc
 create mode 100644 tensorflow/core/common_runtime/eager/eager_executor.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 3046d9064a..73a3450e0e 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -27,6 +27,7 @@ tf_cuda_library(
             ":runtime",
             "//tensorflow/c:c_api",
             "//tensorflow/c:c_api_internal",
+            "//tensorflow/core:core_cpu",
             "//tensorflow/core:core_cpu_internal",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
@@ -54,6 +55,7 @@ tf_cuda_library(
         ":runtime",
         "//tensorflow/c:c_api",
         "//tensorflow/c:c_api_internal",
+        "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_lib",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 455bc19be8..4e5703ffe0 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -165,7 +165,7 @@ void TFE_ContextSetThreadLocalDevicePlacementPolicy(
 
 // Note: this function looks up a thread local policy. So it should be called in
 // the appropriate client thread. In particular, in async mode, it may not be
-// safe to call this function from the async TFE_Executor threads.
+// safe to call this function from the async EagerExecutor threads.
 extern TFE_ContextDevicePlacementPolicy TFE_ContextGetDevicePlacementPolicy(
     TFE_Context* ctx) {
   tensorflow::mutex_lock ml(ctx->policy_map_mu);
@@ -731,15 +731,15 @@ tensorflow::Status Execute(
   return tensorflow::Status::OK();
 }
 
-// TODO(agarwal): move TFE_Executor and TFE_Node related code to a separate
+// TODO(agarwal): move EagerExecutor and EagerNode related code to a separate
 // file.
-class ExecuteNode : public TFE_Node {
+class ExecuteNode : public tensorflow::EagerNode {
  public:
   ExecuteNode(TFE_Op* op, tensorflow::KernelAndDevice* kernel,
               tensorflow::NodeExecStats* maybe_stats,
               const tensorflow::DataTypeVector& output_dtypes,
               TFE_TensorHandle** retvals, int num_retvals)
-      : TFE_Node(op->ctx->executor.NextId()),
+      : tensorflow::EagerNode(op->ctx->executor.NextId()),
         ctx_(op->ctx),
         op_device_(op->device),
         inputs_(op->inputs),
@@ -791,11 +791,11 @@ class ExecuteNode : public TFE_Node {
   tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 2> retvals_;
 };
 
-class CopyToDeviceNode : public TFE_Node {
+class CopyToDeviceNode : public tensorflow::EagerNode {
  public:
   CopyToDeviceNode(TFE_TensorHandle* src, tensorflow::Device* dstd,
                    TFE_Context* ctx)
-      : TFE_Node(ctx->executor.NextId()),
+      : tensorflow::EagerNode(ctx->executor.NextId()),
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
@@ -1182,8 +1182,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
     // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
-    TFE_Node* node = new ExecuteNode(op, kernel, maybe_stats.release(),
-                                     output_dtypes, retvals, *num_retvals);
+    tensorflow::EagerNode* node =
+        new ExecuteNode(op, kernel, maybe_stats.release(), output_dtypes,
+                        retvals, *num_retvals);
     ctx->executor.Add(node);
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
@@ -1214,8 +1215,8 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
     // make sure that `h` is ready before the copy is actually done.
     CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
     TFE_TensorHandle* output = node->dst();
-    // Note that calling Add makes `node` accessible by the TFE_Executor thread.
-    // So further accesses need to be thread-safe.
+    // Note that calling Add makes `node` accessible by the EagerExecutor
+    // thread. So further accesses need to be thread-safe.
     ctx->executor.Add(node);
     return output;
   } else {
@@ -1356,137 +1357,6 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
 }
 }  // namespace tensorflow
 
-TFE_Node::TFE_Node(tensorflow::uint64 id) : id(id) {}
-
-TFE_Executor::~TFE_Executor() {
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  thread_done_ = true;
-  nodes_pending_.notify_all();
-}
-
-tensorflow::uint64 TFE_Executor::NextId() {
-  tensorflow::mutex_lock l(next_id_mutex_);
-  return next_id_++;
-}
-
-void TFE_Executor::EnableAsync() {
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  if (thread_ == nullptr) {
-    thread_.reset(tensorflow::Env::Default()->StartThread(
-        tensorflow::ThreadOptions(), "eager_async_executor",
-        std::bind(&TFE_Executor::Run, this)));
-  }
-}
-
-void TFE_Executor::Add(TFE_Node* node) {
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  DCHECK(thread_) << "EnableAsync should have been called before Add";
-  if (!status_.ok()) {
-    delete node;
-    return;
-  }
-  int qlen = node_queue_.size();
-  if (qlen > 0) {
-    if (node_queue_.back()->id >= node->id) {
-      status_ = tensorflow::errors::InvalidArgument(
-          "Inserting TFE_Node with non-increasing ids:", node_queue_.back()->id,
-          " vs ", node->id);
-      delete node;
-      return;
-    }
-    node_queue_.push(node);
-  } else {
-    node_queue_.push(node);
-    nodes_pending_.notify_all();
-  }
-}
-
-tensorflow::Status TFE_Executor::WaitFor(tensorflow::uint64 node_id) {
-  return WaitImpl(false, node_id);
-}
-
-tensorflow::Status TFE_Executor::WaitForAllPendingNodes() {
-  return WaitImpl(true, 0);
-}
-
-tensorflow::Status TFE_Executor::WaitImpl(bool wait_all,
-                                          tensorflow::uint64 node_id) {
-  tensorflow::condition_variable cond;
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  // Don't wait if an error is already set.
-  if (!status_.ok()) return status_;
-  if (node_queue_.empty()) return tensorflow::Status::OK();
-  if (wait_all) {
-    node_id = node_queue_.back()->id;
-  } else if (node_id < node_queue_.front()->id) {
-    // Note that we are relying on the ops being dispatched sequentially from
-    // the queue.
-    return tensorflow::Status::OK();
-  }
-  node_done_notifications_.insert(std::make_pair(node_id, &cond));
-  cond.wait(l);
-  // Note that we could be woken up if an error occurs, even though the node has
-  // not actually executed.
-  return status_;
-}
-
-void TFE_Executor::ClearError() {
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  if (status_.ok()) return;
-  // If an error was set, node_done_notifications_ and node_queue_ should have
-  // been cleared, and no new entries should have been added since.
-  DCHECK(node_done_notifications_.empty());
-  DCHECK(node_queue_.empty());
-  status_ = tensorflow::Status::OK();
-  nodes_pending_.notify_all();
-}
-
-tensorflow::Status TFE_Executor::status() {
-  tensorflow::mutex_lock l(node_queue_mutex_);
-  return status_;
-}
-
-void TFE_Executor::Run() {
-  while (true) {
-    std::unique_ptr<TFE_Node> curr_node;
-    {
-      tensorflow::mutex_lock l(node_queue_mutex_);
-      while (node_queue_.empty() || !status_.ok()) {
-        if (thread_done_) return;
-        nodes_pending_.wait(l);
-      }
-      curr_node.reset(node_queue_.front());
-    }
-    tensorflow::Status status = curr_node->Run();
-    const bool ok = status.ok();
-    tensorflow::mutex_lock l(node_queue_mutex_);
-    node_queue_.pop();
-    if (!ok) {
-      status_ = status;
-      // TODO(agarwal): mark all affected handles as corrupted before clearing
-      // this queue.
-      // We remove any pending ops so that we don't try to execute them if
-      // ClearError is called.
-      for (int i = 0; i < node_queue_.size(); ++i) {
-        delete node_queue_.front();
-        node_queue_.pop();
-      }
-    }
-    if (!node_done_notifications_.empty()) {
-      tensorflow::uint64 node_id = curr_node->id;
-      // Note that we notify all waiting threads in case an error has occurred.
-      // These calling threads are responsible for checking status_ before
-      // proceeding.
-      const auto range = ok ? node_done_notifications_.equal_range(node_id)
-                            : make_pair(node_done_notifications_.begin(),
-                                        node_done_notifications_.end());
-      for (auto it = range.first; it != range.second; ++it) {
-        it->second->notify_all();
-      }
-      node_done_notifications_.erase(range.first, range.second);
-    }
-  }
-}
 
 bool TFE_Context::Async() const {
   tensorflow::mutex_lock l(async_map_mu);
@@ -1502,7 +1372,7 @@ bool TFE_TensorHandle::IsReady() {
 
 tensorflow::Status TFE_TensorHandle::WaitReady() {
   if (node_id == 0) return tensorflow::Status::OK();
-  TFE_Executor* executor = nullptr;
+  tensorflow::EagerExecutor* executor = nullptr;
   {
     tensorflow::mutex_lock l(ctx_mutex_);
     if (ctx_ == nullptr) return tensorflow::Status::OK();
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 8dba12f47b..1edbe81992 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
@@ -40,101 +41,6 @@ limitations under the License.
 #include "tensorflow/core/platform/thread_annotations.h"
 #include "tensorflow/core/public/version.h"
 
-// A unit of execution for the TFE_Executor class below. Example subclasses
-// encapsulate execution of a TFE_Op, or copying a TFE_TensorHandle from one
-// device to another.
-class TFE_Node {
- public:
-  explicit TFE_Node(tensorflow::uint64 id);
-
-  virtual ~TFE_Node() {}
-
-  // Runs the computation corresponding to this node and blocks till the
-  // execution is done.
-  virtual tensorflow::Status Run() = 0;
-
-  // An id unique to the TFE_Context under which this node is created. Allocated
-  // monotonically.
-  const tensorflow::uint64 id;
-};
-
-// A class for handling async execution (see TFE_ContextSetAsync).
-// Note that this class is thread-safe.
-// TODO(agarwal): TFE_OpAddInput may currently block if it tries to access the
-// device of the input handle. Fix that.
-// TODO(agarwal): On error, mark all affected handles as corrupted.
-// TODO(agarwal): Implement support for control dependencies.
-// TODO(agarwal): Support out-of-order execution and dispatching multiple
-// TFE_Node in parallel.
-// TODO(agarwal): Implement optimizations over TFE_Node traces.
-class TFE_Executor {
- public:
-  ~TFE_Executor();
-
-  // This is called whenever async mode is enabled. Note that it may be called
-  // multiple times as different calling threads may switch async mode on or off
-  // independently.
-  void EnableAsync();
-
-  // Helper function to create monotonically increasing ids unique to this
-  // object.
-  tensorflow::uint64 NextId();
-
-  // Schedules `node` for execution.
-  // Note that Add must be called in monotonically increasing order of node->id.
-  void Add(TFE_Node* node);
-
-  // Causes the caller to block till node with id `node_id` has finished
-  // execution.
-  tensorflow::Status WaitFor(tensorflow::uint64 node_id);
-
-  // Blocks till all currently pending ops are done.
-  tensorflow::Status WaitForAllPendingNodes();
-
-  // Clears all currently set errors which re-enables async execution.
-  void ClearError();
-
-  // Returns Status based on any errors that occurred during async execution.
-  tensorflow::Status status();
-
- private:
-  // Starts execution of pending TFE_Nodes. This function loops till
-  // thread_done_ is set to true. If any errors are encontered, these are set
-  // inside `status_`. The loop blocks anytime there are no pending nodes, or if
-  // `status_` is not ok.
-  void Run();
-
-  tensorflow::Status WaitImpl(bool wait_all, tensorflow::uint64 node_id);
-
-  tensorflow::mutex node_queue_mutex_;
-
-  // Used to signal that some TFE_Nodes are pending execution.
-  tensorflow::condition_variable nodes_pending_ GUARDED_BY(node_queue_mutex_);
-
-  // Queue of pending TFE_Nodes.
-  std::queue<TFE_Node*> node_queue_ GUARDED_BY(node_queue_mutex_);
-
-  // `status_` is set based on any errors raised during execution of a TFE_Node.
-  // It remains set until ClearError is called.
-  tensorflow::Status status_ GUARDED_BY(node_queue_mutex_);
-
-  // Map from id of a TFE_Node to condition_variables (not owned by the map).
-  // These condition_variables are notified and removed when that TFE_Node is
-  // done executing, or if an error is found in execution of any TFE_Node.
-  std::multimap<tensorflow::uint64, tensorflow::condition_variable*>
-      node_done_notifications_ GUARDED_BY(node_queue_mutex_);
-
-  // Thread object that calls the `Run` method. Currently we use only one thread
-  // for executing the TFE_Nodes one-by-one.
-  std::unique_ptr<tensorflow::Thread> thread_ GUARDED_BY(node_queue_mutex_);
-
-  // Indicates that `thread_` should stop as soon as it is done executing the
-  // current TFE_Node.
-  bool thread_done_ GUARDED_BY(node_queue_mutex_) = false;
-
-  tensorflow::mutex next_id_mutex_;
-  tensorflow::uint64 next_id_ GUARDED_BY(next_id_mutex_) = 1;
-};
 
 struct TFE_ContextOptions {
   TF_SessionOptions session_options;
@@ -203,8 +109,8 @@ struct TFE_Context {
   tensorflow::mutex metadata_mu;
   tensorflow::RunMetadata run_metadata GUARDED_BY(metadata_mu);
   const bool log_device_placement;
-  // TFE_Executor for async execution.
-  TFE_Executor executor;
+  // EagerExecutor for async execution.
+  tensorflow::EagerExecutor executor;
 
   // True if running in asynchronous mode.
   bool Async() const;
@@ -263,13 +169,13 @@ struct TFE_TensorHandle : public tensorflow::core::RefCounted {
 
  private:
   // If the contents of the Tensor pointed to by this handle is yet to be
-  // computed by a TFE_Node, this function will block till that compuatation is
+  // computed by a EagerNode, this function will block till that compuatation is
   // done and the handle is "ready".
   tensorflow::Status WaitReady();
 
   bool IsReady();
 
-  // Id for the TFE_Node that will compute the value pointed to by this handle.
+  // Id for the EagerNode that will compute the value pointed to by this handle.
   // If the value is 0, the handle is already ready, but not vice-versa.
   const tensorflow::uint64 node_id;
 
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index df44857185..cf29444065 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -793,6 +793,7 @@ tf_cuda_library(
     hdrs = [
         "common_runtime/device.h",
         "common_runtime/device_factory.h",
+        "common_runtime/eager/eager_executor.h",
         "common_runtime/optimization_registry.h",
         "common_runtime/shape_refiner.h",
         "graph/algorithm.h",
@@ -2141,6 +2142,7 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/stats_publisher_interface.h",
     "common_runtime/step_stats_collector.h",
     "common_runtime/threadpool_device.h",
+    "common_runtime/eager/eager_executor.h",
     "graph/gradients.h",
     "graph/quantize_training.h",
 ] + if_mkl(["graph/mkl_graph_util.h"])
@@ -2160,6 +2162,7 @@ tf_cuda_library(
         "common_runtime/device_factory.cc",
         "common_runtime/device_mgr.cc",
         "common_runtime/device_set.cc",
+        "common_runtime/eager/eager_executor.cc",
         "common_runtime/executor.cc",
         "common_runtime/function.cc",
         "common_runtime/graph_optimizer.cc",
diff --git a/tensorflow/core/common_runtime/eager/eager_executor.cc b/tensorflow/core/common_runtime/eager/eager_executor.cc
new file mode 100644
index 0000000000..b699036e96
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/eager_executor.cc
@@ -0,0 +1,152 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+
+namespace tensorflow {
+
+EagerNode::EagerNode(tensorflow::uint64 id) : id(id) {}
+
+EagerExecutor::~EagerExecutor() {
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  thread_done_ = true;
+  nodes_pending_.notify_all();
+}
+
+tensorflow::uint64 EagerExecutor::NextId() {
+  tensorflow::mutex_lock l(next_id_mutex_);
+  return next_id_++;
+}
+
+void EagerExecutor::EnableAsync() {
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  if (thread_ == nullptr) {
+    thread_.reset(tensorflow::Env::Default()->StartThread(
+        tensorflow::ThreadOptions(), "eager_async_executor",
+        std::bind(&EagerExecutor::Run, this)));
+  }
+}
+
+void EagerExecutor::Add(EagerNode* node) {
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  DCHECK(thread_) << "EnableAsync should have been called before Add";
+  if (!status_.ok()) {
+    delete node;
+    return;
+  }
+  int64 qlen = node_queue_.size();
+  if (qlen > 0) {
+    if (node_queue_.back()->id >= node->id) {
+      status_ = tensorflow::errors::InvalidArgument(
+          "Inserting EagerNode with non-increasing ids:",
+          node_queue_.back()->id, " vs ", node->id);
+      delete node;
+      return;
+    }
+    node_queue_.push(node);
+  } else {
+    node_queue_.push(node);
+    nodes_pending_.notify_all();
+  }
+}
+
+tensorflow::Status EagerExecutor::WaitFor(tensorflow::uint64 node_id) {
+  return WaitImpl(false, node_id);
+}
+
+tensorflow::Status EagerExecutor::WaitForAllPendingNodes() {
+  return WaitImpl(true, 0);
+}
+
+tensorflow::Status EagerExecutor::WaitImpl(bool wait_all,
+                                           tensorflow::uint64 node_id) {
+  tensorflow::condition_variable cond;
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  // Don't wait if an error is already set.
+  if (!status_.ok()) return status_;
+  if (node_queue_.empty()) return tensorflow::Status::OK();
+  if (wait_all) {
+    node_id = node_queue_.back()->id;
+  } else if (node_id < node_queue_.front()->id) {
+    // Note that we are relying on the ops being dispatched sequentially from
+    // the queue.
+    return tensorflow::Status::OK();
+  }
+  node_done_notifications_.insert(std::make_pair(node_id, &cond));
+  cond.wait(l);
+  // Note that we could be woken up if an error occurs, even though the node has
+  // not actually executed.
+  return status_;
+}
+
+void EagerExecutor::ClearError() {
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  if (status_.ok()) return;
+  // If an error was set, node_done_notifications_ and node_queue_ should have
+  // been cleared, and no new entries should have been added since.
+  DCHECK(node_done_notifications_.empty());
+  DCHECK(node_queue_.empty());
+  status_ = tensorflow::Status::OK();
+  nodes_pending_.notify_all();
+}
+
+tensorflow::Status EagerExecutor::status() {
+  tensorflow::mutex_lock l(node_queue_mutex_);
+  return status_;
+}
+
+void EagerExecutor::Run() {
+  while (true) {
+    std::unique_ptr<EagerNode> curr_node;
+    {
+      tensorflow::mutex_lock l(node_queue_mutex_);
+      while (node_queue_.empty() || !status_.ok()) {
+        if (thread_done_) return;
+        nodes_pending_.wait(l);
+      }
+      curr_node.reset(node_queue_.front());
+    }
+    tensorflow::Status status = curr_node->Run();
+    const bool ok = status.ok();
+    tensorflow::mutex_lock l(node_queue_mutex_);
+    node_queue_.pop();
+    if (!ok) {
+      status_ = status;
+      // TODO(agarwal): mark all affected handles as corrupted before clearing
+      // this queue.
+      // We remove any pending ops so that we don't try to execute them if
+      // ClearError is called.
+      for (int i = 0; i < node_queue_.size(); ++i) {
+        delete node_queue_.front();
+        node_queue_.pop();
+      }
+    }
+    if (!node_done_notifications_.empty()) {
+      tensorflow::uint64 node_id = curr_node->id;
+      // Note that we notify all waiting threads in case an error has occurred.
+      // These calling threads are responsible for checking status_ before
+      // proceeding.
+      const auto range = ok ? node_done_notifications_.equal_range(node_id)
+                            : make_pair(node_done_notifications_.begin(),
+                                        node_done_notifications_.end());
+      for (auto it = range.first; it != range.second; ++it) {
+        it->second->notify_all();
+      }
+      node_done_notifications_.erase(range.first, range.second);
+    }
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/eager_executor.h b/tensorflow/core/common_runtime/eager/eager_executor.h
new file mode 100644
index 0000000000..021daeb21d
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/eager_executor.h
@@ -0,0 +1,138 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_EXECUTOR_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_EXECUTOR_H_
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <queue>
+#include <string>
+#include <thread>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+// A unit of execution for the EagerExecutor class below. Example subclasses
+// encapsulate execution of a TFE_Op, or copying a TFE_TensorHandle from one
+// device to another.
+class EagerNode {
+ public:
+  explicit EagerNode(uint64 id);
+
+  virtual ~EagerNode() {}
+
+  // Runs the computation corresponding to this node and blocks till the
+  // execution is done.
+  virtual Status Run() = 0;
+
+  // An id unique to the TFE_Context under which this node is created. Allocated
+  // monotonically.
+  const uint64 id;
+};
+
+// A class for handling async execution (see TFE_ContextSetAsync).
+// Note that this class is thread-safe.
+// TODO(agarwal): TFE_OpAddInput may currently block if it tries to access the
+// device of the input handle. Fix that.
+// TODO(agarwal): On error, mark all affected handles as corrupted.
+// TODO(agarwal): Implement support for control dependencies.
+// TODO(agarwal): Support out-of-order execution and dispatching multiple
+// EagerNode in parallel.
+// TODO(agarwal): Implement optimizations over EagerNode traces.
+class EagerExecutor {
+ public:
+  ~EagerExecutor();
+
+  // This is called whenever async mode is enabled. Note that it may be called
+  // multiple times as different calling threads may switch async mode on or off
+  // independently.
+  void EnableAsync();
+
+  // Helper function to create monotonically increasing ids unique to this
+  // object.
+  uint64 NextId();
+
+  // Schedules `node` for execution.
+  // Note that Add must be called in monotonically increasing order of node->id.
+  void Add(EagerNode* node);
+
+  // Causes the caller to block till node with id `node_id` has finished
+  // execution.
+  Status WaitFor(uint64 node_id);
+
+  // Blocks till all currently pending ops are done.
+  Status WaitForAllPendingNodes();
+
+  // Clears all currently set errors which re-enables async execution.
+  void ClearError();
+
+  // Returns Status based on any errors that occurred during async execution.
+  Status status();
+
+ private:
+  // Starts execution of pending EagerNodes. This function loops till
+  // thread_done_ is set to true. If any errors are encontered, these are set
+  // inside `status_`. The loop blocks anytime there are no pending nodes, or if
+  // `status_` is not ok.
+  void Run();
+
+  Status WaitImpl(bool wait_all, uint64 node_id);
+
+  mutex node_queue_mutex_;
+
+  // Used to signal that some EagerNodes are pending execution.
+  condition_variable nodes_pending_ GUARDED_BY(node_queue_mutex_);
+
+  // Queue of pending EagerNodes.
+  std::queue<EagerNode*> node_queue_ GUARDED_BY(node_queue_mutex_);
+
+  // `status_` is set based on any errors raised during execution of a
+  // EagerNode.  It remains set until ClearError is called.
+  Status status_ GUARDED_BY(node_queue_mutex_);
+
+  // Map from id of a EagerNode to condition_variables (not owned by the map).
+  // These condition_variables are notified and removed when that EagerNode is
+  // done executing, or if an error is found in execution of any EagerNode.
+  std::multimap<uint64, condition_variable*> node_done_notifications_
+      GUARDED_BY(node_queue_mutex_);
+
+  // Thread object that calls the `Run` method. Currently we use only one thread
+  // for executing the EagerNodes one-by-one.
+  std::unique_ptr<Thread> thread_ GUARDED_BY(node_queue_mutex_);
+
+  // Indicates that `thread_` should stop as soon as it is done executing the
+  // current EagerNode.
+  bool thread_done_ GUARDED_BY(node_queue_mutex_) = false;
+
+  mutex next_id_mutex_;
+  uint64 next_id_ GUARDED_BY(next_id_mutex_) = 1;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EAGER_EXECUTOR_H_
-- 
GitLab


From a80fb2b1cad1bb9c868222b8c25f162d69a509e6 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 19 Mar 2018 13:44:23 -0700
Subject: [PATCH 190/960] Automated g4 rollback of changelist 189416074

PiperOrigin-RevId: 189634491
---
 tensorflow/python/client/session.py      | 25 ++++++++++++++++++
 tensorflow/python/client/session_test.py | 32 ++++++++++++++++++++++++
 2 files changed, 57 insertions(+)

diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 924d62992a..6e9ce9b080 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import functools
 import re
 import threading
+import warnings
 
 import numpy as np
 
@@ -1624,6 +1625,9 @@ class InteractiveSession(BaseSession):
   ```
   """
 
+  _count_lock = threading.Lock()
+  _active_session_count = 0  # GUARDED_BY(_count_lock)
+
   def __init__(self, target='', graph=None, config=None):
     """Creates a new interactive TensorFlow session.
 
@@ -1652,6 +1656,19 @@ class InteractiveSession(BaseSession):
     config.graph_options.place_pruned_graph = True
 
     super(InteractiveSession, self).__init__(target, graph, config)
+    with InteractiveSession._count_lock:
+      if InteractiveSession._active_session_count > 0:
+        warnings.warn('An interactive session is already active. This can '
+                      'cause out-of-memory errors in some cases. You must '
+                      'explicitly call `InteractiveSession.close()` to release '
+                      'resources held by the other session(s).')
+      InteractiveSession._active_session_count += 1
+    # NOTE(mrry): We do not use `Session._closed` here because it has unhelpful
+    # semantics (in particular, it is not set to true if `Session.close()` is
+    # called on a session that has not been "opened" by running a step) and we
+    # cannot change those semantics without breaking existing code.
+    self._explicitly_closed = False
+
     self._default_session = self.as_default()
     self._default_session.enforce_nesting = False
     self._default_session.__enter__()
@@ -1664,6 +1681,14 @@ class InteractiveSession(BaseSession):
   def close(self):
     """Closes an `InteractiveSession`."""
     super(InteractiveSession, self).close()
+    with InteractiveSession._count_lock:
+      if not self._explicitly_closed:
+        InteractiveSession._active_session_count -= 1
+        self._explicitly_closed = True
+      else:
+        return
     if self._explicit_graph is not None:
       self._default_graph.__exit__(None, None, None)
+      self._default_graph = None
     self._default_session.__exit__(None, None, None)
+    self._default_session = None
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 3bf2a9e4dd..44ff440cc5 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -22,6 +22,7 @@ import os
 import sys
 import threading
 import time
+import warnings
 
 import numpy as np
 import six
@@ -65,6 +66,10 @@ ops.RegisterShape('ConstructionFails')(common_shapes.unknown_shape)
 # @test_util.with_c_api
 class SessionTest(test_util.TensorFlowTestCase):
 
+  def setUp(self):
+    super(SessionTest, self).setUp()
+    warnings.simplefilter('always')
+
   def testUseExistingGraph(self):
     with ops.Graph().as_default() as g, ops.device('/cpu:0'):
       a = constant_op.constant(6.0, shape=[1, 1])
@@ -1190,6 +1195,33 @@ class SessionTest(test_util.TensorFlowTestCase):
       self.assertAllEqual([[24.0]], e.eval())
       sess.close()
 
+  def testMultipleInteractiveSessionsWarning(self):
+    # Reinitialize the global state to ensure that the expected warnings will
+    # be emitted.
+    session.InteractiveSession._active_session_count = 0  # pylint: disable=protected-access
+
+    sess = session.InteractiveSession()
+    sess.run(constant_op.constant(4.0))  # Run so that the session is "opened".
+    sess.close()
+    # Opening and closing interactive sessions serially should not warn.
+    with warnings.catch_warnings(record=True) as w:
+      sess = session.InteractiveSession()
+      sess.close()
+    self.assertEqual(0, len(w))
+
+    with warnings.catch_warnings(record=True) as w:
+      sess = session.InteractiveSession()
+    self.assertEqual(0, len(w))
+    with warnings.catch_warnings(record=True) as w:
+      sess2 = session.InteractiveSession()
+    self.assertEqual(1, len(w))
+    self.assertTrue('An interactive session is already active. This can cause '
+                    'out-of-memory errors in some cases. You must explicitly '
+                    'call `InteractiveSession.close()` to release resources '
+                    'held by the other session(s).' in str(w[0].message))
+    sess2.close()
+    sess.close()
+
   def testInteractivePlacePrunedGraph(self):
     sess = session.InteractiveSession()
 
-- 
GitLab


From e6affeb79ee0cfda24b76368e3e788a7ec23df32 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 14:09:52 -0700
Subject: [PATCH 191/960] Add a helper that allows constructing simple
 expression ASTs from string. Useful to simplify the representation of
 composite symbols, e.g. 'py2tf.foo'.

PiperOrigin-RevId: 189638901
---
 tensorflow/contrib/py2tf/pyct/parser.py      | 22 ++++++++++++++++++--
 tensorflow/contrib/py2tf/pyct/parser_test.py | 17 +++++++++------
 2 files changed, 31 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/parser.py b/tensorflow/contrib/py2tf/pyct/parser.py
index dc7df883b3..c961efa892 100644
--- a/tensorflow/contrib/py2tf/pyct/parser.py
+++ b/tensorflow/contrib/py2tf/pyct/parser.py
@@ -29,12 +29,30 @@ from tensorflow.python.util import tf_inspect
 
 
 def parse_entity(entity):
-  """Return the AST of given entity."""
+  """Returns the AST of given entity."""
   source = tf_inspect.getsource(entity)
   source = textwrap.dedent(source)
   return parse_str(source), source
 
 
 def parse_str(src):
-  """Return the AST of given piece of code."""
+  """Returns the AST of given piece of code."""
   return gast.parse(src)
+
+
+def parse_expression(src):
+  """Returns the AST of given identifier.
+
+  Args:
+    src: A piece of code that represents a single Python expression
+  Returns:
+    A gast.AST object.
+  Raises:
+    ValueError: if src does not consist of a single Expression.
+  """
+  node = parse_str(src)
+  assert isinstance(node, gast.Module)
+  if len(node.body) != 1 and not isinstance(node.body[0], gast.Expr):
+    raise ValueError(
+        'Expected a single expression, found instead %s' % node.body)
+  return node.body[0].value
diff --git a/tensorflow/contrib/py2tf/pyct/parser_test.py b/tensorflow/contrib/py2tf/pyct/parser_test.py
index f35dfa04c7..c58ffc7e0c 100644
--- a/tensorflow/contrib/py2tf/pyct/parser_test.py
+++ b/tensorflow/contrib/py2tf/pyct/parser_test.py
@@ -24,24 +24,29 @@ from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.python.platform import test
 
 
-def f(x):
-  return x + 1
-
-
 class ParserTest(test.TestCase):
 
   def test_parse_entity(self):
+
+    def f(x):
+      return x + 1
+
     mod, _ = parser.parse_entity(f)
     self.assertEqual('f', mod.body[0].name)
 
   def test_parse_str(self):
     mod = parser.parse_str(
         textwrap.dedent("""
-        def f(x):
-          return x + 1
+            def f(x):
+              return x + 1
     """))
     self.assertEqual('f', mod.body[0].name)
 
+  def test_parse_expression(self):
+    node = parser.parse_expression('a.b')
+    self.assertEqual('a', node.value.id)
+    self.assertEqual('b', node.attr)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From eaa61ab7514c56b9ce219bb5f5f38a5b3ad78657 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 19 Mar 2018 14:24:00 -0700
Subject: [PATCH 192/960] Turned on gradient optimization by default

PiperOrigin-RevId: 189641300
---
 .../grappler/optimizers/function_optimizer.cc | 24 +++++++++++--------
 tensorflow/python/BUILD                       |  1 +
 2 files changed, 15 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 3f2afdeef1..97effae8c8 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -220,21 +220,27 @@ Status InlineSymbolicGradient(const NodeDef& node,
       inlined_node.set_name(node.name());
       for (int i = 0; i < inlined_node.input_size(); ++i) {
         inlined_node.set_input(
-            i, strings::StrCat(node.name(), "/", inlined_node.input(i)));
+            i, AddPrefixToNodeName(inlined_node.input(i), node.name()));
       }
     } else if (inlined_node.name() == "FunctionInputs") {
       inlined_node.set_name(
-          strings::StrCat(node.name(), "/", inlined_node.name()));
+          AddPrefixToNodeName(inlined_node.name(), node.name()));
       inlined_node.clear_input();
       for (int i = 0; i < node.input_size(); ++i) {
         inlined_node.add_input(node.input(i));
       }
     } else {
       inlined_node.set_name(
-          strings::StrCat(node.name(), "/", inlined_node.name()));
+          AddPrefixToNodeName(inlined_node.name(), node.name()));
       for (int i = 0; i < inlined_node.input_size(); ++i) {
         inlined_node.set_input(
-            i, strings::StrCat(node.name(), "/", inlined_node.input(i)));
+            i, AddPrefixToNodeName(inlined_node.input(i), node.name()));
+      }
+      // If the node has no input, hook it up to the function input node to make
+      // sure it runs in the same frame as the other nodes of the function body.
+      if (inlined_node.input_size() == 0) {
+        *inlined_node.add_input() = AsControlDependency(
+            AddPrefixToNodeName("FunctionInputs", node.name()));
       }
     }
     inlined_node.set_device(node.device());
@@ -275,12 +281,10 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   *optimized_graph->mutable_versions() = item.graph.versions();
   for (const NodeDef& node : item.graph.node()) {
-    if (opt_level_ == RewriterConfig::AGGRESSIVE) {
-      if (node.op() == "SymbolicGradient") {
-        TF_RETURN_IF_ERROR(InlineSymbolicGradient(node, item.graph.library(),
-                                                  optimized_graph));
-        continue;
-      }
+    if (node.op() == "SymbolicGradient") {
+      TF_RETURN_IF_ERROR(
+          InlineSymbolicGradient(node, item.graph.library(), optimized_graph));
+      continue;
     }
     auto it = functions.find(node.op());
     if (it == functions.end()) {
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9a29986c3b..a029ecd4d0 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1033,6 +1033,7 @@ cuda_py_tests(
         "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
     ],
+    shard_count = 10,
 )
 
 py_test(
-- 
GitLab


From e613e0844a95814457f3530eedb9baf812cf1e87 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 14:27:06 -0700
Subject: [PATCH 193/960] Enable stack push removal optimization by default.

PiperOrigin-RevId: 189641729
---
 .../grappler/optimizers/loop_optimizer.cc     | 36 +++++++++++--------
 .../optimizers/loop_optimizer_test.cc         | 24 ++++++++-----
 .../grappler/optimizers/meta_optimizer.cc     |  4 +--
 .../core/protobuf/rewriter_config.proto       |  2 +-
 tensorflow/python/kernel_tests/BUILD          |  2 ++
 5 files changed, 42 insertions(+), 26 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index 244653504d..f78036d78c 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -45,8 +45,9 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-std::vector<int> GetStackPushNodesToConvert(const SimpleGraphView& graph_view,
-                                            int stack_node_idx) {
+std::vector<int> GetStackPushNodesToConvert(
+    const SimpleGraphView& graph_view,
+    const std::unordered_set<string>& nodes_to_preserve, int stack_node_idx) {
   VLOG(1) << "Stack node: " << graph_view.graph()->node(stack_node_idx).name();
   const std::unordered_set<string> op_types_to_traverse(
       {"Stack", "StackV2", "Enter", "RefEnter", "Switch", "RefSwitch",
@@ -64,7 +65,9 @@ std::vector<int> GetStackPushNodesToConvert(const SimpleGraphView& graph_view,
                    op_types_to_traverse.end()) {
       continue;
     } else if (!IsStackPopOp(fanout_node) ||
-               !graph_view.outputs(fanout_idx).empty()) {
+               (!graph_view.outputs(fanout_idx).empty() ||
+                nodes_to_preserve.find(fanout_node.name()) !=
+                    nodes_to_preserve.end())) {
       // The node is either a stack pop with consumers or something unexpected
       // so we leave the graph alone.
       nodes_to_convert.clear();
@@ -74,15 +77,17 @@ std::vector<int> GetStackPushNodesToConvert(const SimpleGraphView& graph_view,
   return nodes_to_convert;
 }
 
-Status RemoveStackOps(const GraphDef& graph, GraphDef* optimized_graph) {
+Status RemoveStackOps(const GrapplerItem& item, GraphDef* optimized_graph) {
+  const std::unordered_set<string> nodes_to_preserve = item.NodesToPreserve();
+  const GraphDef& graph = item.graph;
   *optimized_graph = graph;
   NodeMap node_map(optimized_graph);
   SimpleGraphView graph_view;
   TF_RETURN_IF_ERROR(graph_view.Initialize(graph));
   for (int node_idx = 0; node_idx < graph.node_size(); ++node_idx) {
     if (IsStackOp(graph.node(node_idx))) {
-      for (int push_node_idx :
-           GetStackPushNodesToConvert(graph_view, node_idx)) {
+      for (int push_node_idx : GetStackPushNodesToConvert(
+               graph_view, nodes_to_preserve, node_idx)) {
         // We found push nodes without corresponding pops. Convert them to
         // Identity passing the data through and add a control dependency from
         // the op supplying the stack handle.
@@ -463,17 +468,18 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
 
 Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                GraphDef* optimized_graph) {
-  TF_RETURN_IF_ERROR(RemoveStackOps(item.graph, optimized_graph));
+  TF_RETURN_IF_ERROR(RemoveStackOps(item, optimized_graph));
 
-  optimized_graph_ = optimized_graph;
-
-  // Set up helper data structures.
-  node_map_.reset(new NodeMap(optimized_graph_));
-  int num_frames;
-  TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
-                                               &frame_map_, &num_frames));
+  if (opt_level_ == RewriterConfig::AGGRESSIVE) {
+    optimized_graph_ = optimized_graph;
+    // Set up helper data structures.
+    node_map_.reset(new NodeMap(optimized_graph_));
+    int num_frames;
+    TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
+                                                 &frame_map_, &num_frames));
+    TF_RETURN_IF_ERROR(LoopInvariantNodeMotion());
+  }
 
-  TF_RETURN_IF_ERROR(LoopInvariantNodeMotion());
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
index 0d45ba9b56..a0bd335197 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer_test.cc
@@ -81,7 +81,7 @@ TEST_F(LoopOptimizerTest, Basic) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -128,7 +128,7 @@ TEST_F(LoopOptimizerTest, Const) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -175,7 +175,7 @@ TEST_F(LoopOptimizerTest, ControlOutput) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -235,7 +235,7 @@ TEST_F(LoopOptimizerTest, NestedLoop1) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -302,7 +302,7 @@ TEST_F(LoopOptimizerTest, NestedLoop2) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -365,7 +365,7 @@ TEST_F(LoopOptimizerTest, NestedLoopConst1) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -429,7 +429,7 @@ TEST_F(LoopOptimizerTest, NestedLoopConst2) {
   GrapplerItem item;
   item.graph = graph;
 
-  LoopOptimizer optimizer;
+  LoopOptimizer optimizer(RewriterConfig::AGGRESSIVE);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -502,6 +502,7 @@ TEST_F(LoopOptimizerTest, RemovePush_NoOp) {
   AddSimpleNode("stack3", "StackV2", {}, &graph);
   AddSimpleNode("push3", "StackPushV2", {"stack3", "c"}, &graph);
   AddSimpleNode("stop", "StopGradient", {"stack3"}, &graph);
+
   LoopOptimizer optimizer;
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -525,12 +526,19 @@ TEST_F(LoopOptimizerTest, RemovePushWithoutMatchingPop) {
   AddSimpleNode("stack3", "StackV2", {}, &graph);
   AddSimpleNode("push3", "StackPushV2", {"stack3", "c"}, &graph);
   AddSimpleNode("pop3", "StackPopV2", {"stack3"}, &graph);
+  // Push for a Pop without consumer that is fetched should not be removed.
+  AddSimpleNode("stack4", "StackV2", {}, &graph);
+  AddSimpleNode("push4", "StackPushV2", {"stack4", "c"}, &graph);
+  AddSimpleNode("pop4", "StackPopV2", {"stack4"}, &graph);
+
+  item.fetch.push_back("pop4");
 
   LoopOptimizer optimizer;
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-  EXPECT_EQ(10, output.node_size());
+
+  EXPECT_EQ(13, output.node_size());
   for (int i = 0; i < output.node_size(); ++i) {
     const NodeDef& node = output.node(i);
     if (node.name() == "push1") {
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 7b2e7a1fe0..6eb2bbc547 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -106,7 +106,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new ArithmeticOptimizer(cfg_.arithmetic_optimization())));
     }
-    if (cfg_.loop_optimization() == RewriterConfig::ON) {
+    if (cfg_.loop_optimization() != RewriterConfig::OFF) {
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new LoopOptimizer(cfg_.loop_optimization())));
     }
@@ -234,7 +234,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) {
          cfg.function_optimization() != RewriterConfig::OFF ||
          cfg.constant_folding() != RewriterConfig::OFF ||
          cfg.arithmetic_optimization() != RewriterConfig::OFF ||
-         cfg.loop_optimization() == RewriterConfig::ON ||
+         cfg.loop_optimization() != RewriterConfig::OFF ||
          cfg.dependency_optimization() != RewriterConfig::OFF ||
          cfg.auto_parallel().enable() ||
          cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT ||
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index b1fceaacf4..fdf16aa1da 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -42,7 +42,7 @@ message RewriterConfig {
   // Control dependency optimizations (default is ON).
   // Remove redundant control dependencies, which may enable other optimization.
   Toggle dependency_optimization = 8;
-  // Loop optimizations (default is OFF).
+  // Loop optimizations (default is ON).
   Toggle loop_optimization = 9;
   // Function optimizations (default is ON).
   Toggle function_optimization = 10;
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 5b0c38fa5d..d9571fa2be 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -393,6 +393,7 @@ tf_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:nn_ops_gen",
     ],
+    shard_count = 5,
 )
 
 tf_py_test(
@@ -408,6 +409,7 @@ tf_py_test(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:nn_ops_gen",
     ],
+    shard_count = 5,
 )
 
 tf_py_test(
-- 
GitLab


From ff43dff34ab525dd333128c73ebfb0f9723c34c0 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Mon, 19 Mar 2018 14:27:52 -0700
Subject: [PATCH 194/960] TFLite Delegate: Add an `allow_dynamic_tensors`
 parameter.

PiperOrigin-RevId: 189641833
---
 tensorflow/contrib/lite/BUILD               |   1 +
 tensorflow/contrib/lite/interpreter.cc      |  80 +++++++++++--
 tensorflow/contrib/lite/interpreter.h       |  22 +++-
 tensorflow/contrib/lite/interpreter_test.cc | 118 +++++++++++++++++---
 4 files changed, 191 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/lite/BUILD b/tensorflow/contrib/lite/BUILD
index 5cfbb544b7..dafe6f136e 100644
--- a/tensorflow/contrib/lite/BUILD
+++ b/tensorflow/contrib/lite/BUILD
@@ -170,6 +170,7 @@ cc_test(
     deps = [
         ":framework",
         ":string_util",
+        "//tensorflow/contrib/lite/kernels:kernel_util",
         "//tensorflow/contrib/lite/kernels/internal:tensor_utils",
         "//tensorflow/contrib/lite/schema:schema_fbs",
         "//tensorflow/contrib/lite/testing:util",
diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc
index cee57bba5e..937c185b0a 100644
--- a/tensorflow/contrib/lite/interpreter.cc
+++ b/tensorflow/contrib/lite/interpreter.cc
@@ -356,7 +356,11 @@ TfLiteStatus Interpreter::AllocateTensors() {
   }
 
   TF_LITE_ENSURE_STATUS(PrepareOpsAndTensors());
-  invokable_ = true;
+  if (state_ == kStateUninvokable) {
+    state_ = kStateInvokable;
+  }
+  TF_LITE_ENSURE(&context_, state_ == kStateInvokable ||
+                                state_ == kStateInvokableAndImmutable);
   return kTfLiteOk;
 }
 
@@ -364,7 +368,12 @@ TfLiteStatus Interpreter::AddNodeWithParameters(
     const std::vector<int>& inputs, const std::vector<int>& outputs,
     const char* init_data, size_t init_data_size, void* builtin_data,
     const TfLiteRegistration* registration, int* node_index) {
-  invokable_ = false;
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(&context_,
+                "AddNodeWithParameters is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  state_ = kStateUninvokable;
 
   std::unique_ptr<void, decltype(free)*> builtin_data_deleter(builtin_data,
                                                               free);
@@ -420,12 +429,17 @@ TfLiteStatus Interpreter::AddNodeWithParameters(
 
 TfLiteStatus Interpreter::ResizeInputTensor(int tensor_index,
                                             const std::vector<int>& dims) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(&context_,
+                "ResizeInputTensor is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+  state_ = kStateUninvokable;
+
   // TODO(aselle): All bounds checks can be implemented as one-sided bounds
   // checks by casting to unsigned for efficiency. Profile before doing this.
-
   TF_LITE_ENSURE(&context_,
                  tensor_index < context_.tensors_size && tensor_index >= 0);
-  invokable_ = false;
   TfLiteIntArray* dims_lite = ConvertVectorToTfLiteIntArray(dims);
   return ResizeTensorImpl(&context_.tensors[tensor_index], dims_lite);
 }
@@ -490,7 +504,7 @@ TfLiteStatus Interpreter::Invoke() {
     ReportError(&context_, "Invoke called on model that is not consistent.");
     return kTfLiteError;
   }
-  if (!invokable_) {
+  if (state_ == kStateUninvokable) {
     ReportError(&context_, "Invoke called on model that is not ready.");
     return kTfLiteError;
   }
@@ -622,6 +636,13 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     int tensor_index, TfLiteType type, const char* name, const int rank,
     const int* dims, TfLiteQuantizationParams quantization, const char* buffer,
     size_t bytes, const Allocation* allocation) {
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        &context_,
+        "SetTensorParametersReadOnly is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
+
   TF_LITE_ENSURE(&context_,
                  tensor_index < context_.tensors_size && tensor_index >= 0);
   // For most tensors we know exactly how much memory is necessary so we can
@@ -645,7 +666,7 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
     tensor.allocation_type = kTfLiteMmapRo;
     tensor.allocation = allocation;
   } else {
-    invokable_ = false;
+    state_ = kStateUninvokable;
     TfLiteTensorReset(type, name, ConvertArrayToTfLiteIntArray(rank, dims),
                       quantization, const_cast<char*>(buffer), bytes,
                       kTfLiteMmapRo, allocation, &tensor);
@@ -660,7 +681,12 @@ TfLiteStatus Interpreter::SetTensorParametersReadOnly(
 TfLiteStatus Interpreter::SetTensorParametersReadWrite(
     int tensor_index, TfLiteType type, const char* name, const int rank,
     const int* dims, TfLiteQuantizationParams quantization) {
-  invokable_ = false;
+  if (state_ == kStateInvokableAndImmutable) {
+    ReportError(
+        &context_,
+        "SetTensorParametersReadWrite is disallowed when graph is immutable.");
+    return kTfLiteError;
+  }
   TF_LITE_ENSURE(&context_,
                  tensor_index < context_.tensors_size && tensor_index >= 0);
   size_t required_bytes = 0;
@@ -738,19 +764,55 @@ void Interpreter::SetNumThreads(int num_threads) {
   context_.recommended_num_threads = num_threads;
 }
 
-TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate) {
+TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate,
+                                                  bool allow_dynamic_tensors) {
+  if (!allow_dynamic_tensors) {
+    int last_execution_plan_index_prepared;
+    TF_LITE_ENSURE_OK(&context_, PrepareOpsStartingAt(
+                                     0, &last_execution_plan_index_prepared));
+
+    bool has_dynamic_tensors = true;
+    // Dynamic tensors exist if not all nodes can be prepared.
+    if (last_execution_plan_index_prepared + 1 == execution_plan_.size()) {
+      // If all the nodes can be prepared, check if the last node has dynamic
+      // tensors.
+      int node_index = execution_plan_[last_execution_plan_index_prepared];
+      TfLiteNode& node = nodes_and_registration_[node_index].first;
+      if (!HasDynamicTensor(context_, node.outputs)) {
+        has_dynamic_tensors = false;
+      }
+    }
+    if (has_dynamic_tensors) {
+      ReportError(&context_, "Attempting to resize a fixed-size tensor.");
+      return kTfLiteError;
+    }
+  }
+
   // TODO(aselle): Consider if it is worth storing pointers to delegates.
-  // Setup additional context interface
+  // Setup additional context interface.
   context_.GetNodeAndRegistration = GetNodeAndRegistration;
   context_.ReplaceSubgraphsWithDelegateKernels =
       ReplaceSubgraphsWithDelegateKernels;
   context_.GetExecutionPlan = GetExecutionPlan;
 
   TfLiteStatus status = delegate->Prepare(&context_, delegate);
+
   // Remove additional context info.
   SetForbiddenContextFunction(&context_.GetNodeAndRegistration);
   SetForbiddenContextFunction(&context_.ReplaceSubgraphsWithDelegateKernels);
   SetForbiddenContextFunction(&context_.GetExecutionPlan);
+
+  TF_LITE_ENSURE_OK(&context_, status);
+
+  if (!allow_dynamic_tensors) {
+    TF_LITE_ENSURE_OK(&context_, AllocateTensors());
+    TF_LITE_ENSURE(&context_, state_ == kStateInvokable ||
+                                  state_ == kStateInvokableAndImmutable);
+    // After using a delegate which doesn't support dynamic tensors, make the
+    // entire graph immutable.
+    state_ = kStateInvokableAndImmutable;
+  }
+
   return status;
 }
 
diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index af143370ee..788546fd60 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -272,7 +272,9 @@ class Interpreter {
   // Allow a delegate to look at the graph and modify the graph to handle
   // parts of the graph themselves. After this is called, the graph may
   // contain new nodes that replace 1 more nodes.
-  TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate);
+  // WARNING: This is an experimental API and subject to change.
+  TfLiteStatus ModifyGraphWithDelegate(TfLiteDelegate* delegate,
+                                       bool allow_dynamic_tensors = false);
 
   // Ensure the data in `tensor.data` is readable. In case delegate is used,
   // it might require to copy the data from delegate buffer to raw memory.
@@ -447,6 +449,20 @@ class Interpreter {
     }
   }
 
+  // The state of the Interpreter.
+  enum State {
+    // The interpreter isn't ready to be invoked.
+    // `AllocateTensor` need to be called to enter an invokable state.
+    kStateUninvokable = 0,
+    // The interpreter is ready to be invoked.
+    kStateInvokable,
+    // The interpreter is ready to be invoked, and graph can't be further
+    // modified. The interpreter will enter this state when calling
+    // `ModifyGraphWithDelegate` with `allow_dynamic_tensors=false`.
+    kStateInvokableAndImmutable,
+  };
+  State state_ = kStateUninvokable;
+
   // A pure C data structure used to communicate with the pure C plugin
   // interface. To avoid copying tensor metadata, this is also the definitive
   // structure to store tensors.
@@ -462,10 +478,6 @@ class Interpreter {
   // the tensor array.
   bool consistent_ = true;
 
-  // Whether the model is safe to invoke (if any errors occurred this
-  // will be false).
-  bool invokable_ = false;
-
   // Array of indices representing the tensors that are inputs to the
   // interpreter.
   std::vector<int> inputs_;
diff --git a/tensorflow/contrib/lite/interpreter_test.cc b/tensorflow/contrib/lite/interpreter_test.cc
index 7a029c7df8..efb29d5c9d 100644
--- a/tensorflow/contrib/lite/interpreter_test.cc
+++ b/tensorflow/contrib/lite/interpreter_test.cc
@@ -17,9 +17,11 @@ limitations under the License.
 #include <gtest/gtest.h>
 #include "tensorflow/contrib/lite/error_reporter.h"
 #include "tensorflow/contrib/lite/kernels/internal/compatibility.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
 #include "tensorflow/contrib/lite/schema/schema_generated.h"
 #include "tensorflow/contrib/lite/string_util.h"
 #include "tensorflow/contrib/lite/testing/util.h"
+
 namespace tflite {
 namespace {
 
@@ -439,12 +441,12 @@ TEST(BasicInterpreter, ThreeStepAllocate) {
   // String-in String-out node.
   TfLiteRegistration reg_copy = {nullptr, nullptr, nullptr, nullptr};
   reg_copy.invoke = [](TfLiteContext* context, TfLiteNode* node) {
-    TfLiteTensor* a0 = &context->tensors[node->inputs->data[0]];
-    TfLiteTensor* a1 = &context->tensors[node->outputs->data[0]];
+    TfLiteTensor* input = &context->tensors[node->inputs->data[0]];
+    TfLiteTensor* output = &context->tensors[node->outputs->data[0]];
     DynamicBuffer buf;
-    StringRef str_ref = GetString(a0, 0);
+    StringRef str_ref = GetString(input, 0);
     buf.AddString(str_ref);
-    buf.WriteToTensor(a1);
+    buf.WriteToTensor(output);
     return kTfLiteOk;
   };
 
@@ -778,13 +780,17 @@ TfLiteRegistration AddOpRegistration() {
 
   reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
     // Set output size to input size
-    TfLiteTensor* tensor0 = &context->tensors[node->inputs->data[0]];
-    TfLiteTensor* tensor1 = &context->tensors[node->inputs->data[1]];
-    TfLiteTensor* tensor2 = &context->tensors[node->outputs->data[0]];
-    TfLiteIntArray* newSize = TfLiteIntArrayCopy(tensor0->dims);
-    TfLiteIntArray* newSizeOther = TfLiteIntArrayCopy(tensor1->dims);
-    TF_LITE_ENSURE_EQ(context, newSize->size, newSizeOther->size);
-    TF_LITE_ENSURE_STATUS(context->ResizeTensor(context, tensor2, newSize));
+    TfLiteTensor* input1 = &context->tensors[node->inputs->data[0]];
+    TfLiteTensor* input2 = &context->tensors[node->inputs->data[1]];
+    TfLiteTensor* output = &context->tensors[node->outputs->data[0]];
+
+    TF_LITE_ENSURE_EQ(context, input1->dims->size, input2->dims->size);
+    for (int i = 0; i < input1->dims->size; ++i) {
+      TF_LITE_ENSURE_EQ(context, input1->dims->data[i], input2->dims->data[i]);
+    }
+
+    TF_LITE_ENSURE_STATUS(context->ResizeTensor(
+        context, output, TfLiteIntArrayCopy(input1->dims)));
     return kTfLiteOk;
   };
 
@@ -818,6 +824,8 @@ class TestDelegate : public ::testing::Test {
                                                quant);
     interpreter_->SetTensorParametersReadWrite(3, kTfLiteFloat32, "", {3},
                                                quant);
+    interpreter_->SetTensorParametersReadWrite(4, kTfLiteFloat32, "", {3},
+                                               quant);
     TfLiteRegistration reg = AddOpRegistration();
     interpreter_->AddNodeWithParameters({0, 0}, {2}, nullptr, 0, nullptr, &reg);
     interpreter_->AddNodeWithParameters({1, 1}, {3}, nullptr, 0, nullptr, &reg);
@@ -916,7 +924,6 @@ class TestDelegate : public ::testing::Test {
 };
 
 TEST_F(TestDelegate, BasicDelegate) {
-  interpreter_->Invoke();
   delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({0, 1, 2}));
   interpreter_->ModifyGraphWithDelegate(delegate_->get_tf_lite_delegate());
 
@@ -944,7 +951,6 @@ TEST_F(TestDelegate, BasicDelegate) {
 }
 
 TEST_F(TestDelegate, ComplexDeligate) {
-  interpreter_->Invoke();
   delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({1, 2}));
   interpreter_->ModifyGraphWithDelegate(delegate_->get_tf_lite_delegate());
 
@@ -959,7 +965,6 @@ TEST_F(TestDelegate, ComplexDeligate) {
 }
 
 TEST_F(TestDelegate, SetBufferHandleToInput) {
-  interpreter_->Invoke();
   delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({0, 1, 2}));
   TfLiteDelegate* delegate = delegate_->get_tf_lite_delegate();
   interpreter_->ModifyGraphWithDelegate(delegate);
@@ -978,7 +983,6 @@ TEST_F(TestDelegate, SetBufferHandleToInput) {
 }
 
 TEST_F(TestDelegate, SetBufferHandleToOutput) {
-  interpreter_->Invoke();
   delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({0, 1, 2}));
   TfLiteDelegate* delegate = delegate_->get_tf_lite_delegate();
   interpreter_->ModifyGraphWithDelegate(delegate);
@@ -1002,7 +1006,7 @@ TEST_F(TestDelegate, SetInvalidHandleToTensor) {
   interpreter_->Invoke();
   delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({0, 1, 2}));
   TfLiteDelegate* delegate = delegate_->get_tf_lite_delegate();
-  interpreter_->ModifyGraphWithDelegate(delegate);
+  interpreter_->ModifyGraphWithDelegate(delegate, true);
 
   SimpleDelegate another_simple_delegate({0, 1, 2});
 
@@ -1023,6 +1027,88 @@ TEST_F(TestDelegate, SetInvalidHandleToTensor) {
   EXPECT_EQ(tensor->buffer_handle, kTfLiteNullBufferHandle);
 }
 
+TEST_F(TestDelegate, ResizeInputWithNonDynamicDelegateShouldFail) {
+  delegate_ = std::unique_ptr<SimpleDelegate>(new SimpleDelegate({0, 1, 2}));
+  ASSERT_EQ(interpreter_->ResizeInputTensor(0, {1, 2}), kTfLiteOk);
+  ASSERT_EQ(interpreter_->ResizeInputTensor(1, {1, 2}), kTfLiteOk);
+  ASSERT_EQ(
+      interpreter_->ModifyGraphWithDelegate(delegate_->get_tf_lite_delegate()),
+      kTfLiteOk);
+  ASSERT_EQ(interpreter_->ResizeInputTensor(0, {1, 2}), kTfLiteError);
+}
+
+class TestDelegateWithDynamicTensors : public ::testing::Test {
+ protected:
+  void SetUp() override {
+    interpreter_.reset(new Interpreter);
+
+    interpreter_->AddTensors(2);
+    interpreter_->SetInputs({0});
+    interpreter_->SetOutputs({1});
+    TfLiteQuantizationParams quant;
+    interpreter_->SetTensorParametersReadWrite(0, kTfLiteFloat32, "", {3},
+                                               quant);
+    interpreter_->SetTensorParametersReadWrite(1, kTfLiteFloat32, "", {3},
+                                               quant);
+    TfLiteRegistration reg = DynamicCopyOpRegistration();
+    interpreter_->AddNodeWithParameters({0}, {1}, nullptr, 0, nullptr, &reg);
+
+    delegate_.Prepare = [](TfLiteContext* context,
+                           TfLiteDelegate* delegate) -> TfLiteStatus {
+      // In this test, the delegate replaces all the nodes if this function is
+      // called.
+      TfLiteIntArray* execution_plan;
+      TF_LITE_ENSURE_STATUS(
+          context->GetExecutionPlan(context, &execution_plan));
+      context->ReplaceSubgraphsWithDelegateKernels(
+          context, DelegateRegistration(), execution_plan, delegate);
+      return kTfLiteOk;
+    };
+  }
+
+  static TfLiteRegistration DynamicCopyOpRegistration() {
+    TfLiteRegistration reg = {nullptr, nullptr, nullptr, nullptr};
+
+    reg.prepare = [](TfLiteContext* context, TfLiteNode* node) {
+      TfLiteTensor* output = &context->tensors[node->outputs->data[0]];
+      SetTensorToDynamic(output);
+      return kTfLiteOk;
+    };
+
+    reg.invoke = [](TfLiteContext* context, TfLiteNode* node) {
+      // Not implemented since this isn't required in testing.
+      return kTfLiteOk;
+    };
+    return reg;
+  }
+
+  static TfLiteRegistration DelegateRegistration() {
+    TfLiteRegistration reg = {nullptr, nullptr, nullptr, nullptr};
+    return reg;
+  }
+
+  std::unique_ptr<Interpreter> interpreter_;
+  TfLiteDelegate delegate_;
+};
+
+TEST_F(TestDelegateWithDynamicTensors, DisallowDynamicTensors) {
+  interpreter_->ModifyGraphWithDelegate(&delegate_, false);
+
+  ASSERT_EQ(interpreter_->execution_plan().size(), 1);
+  // The interpreter should not call delegate's `Prepare` when dynamic tensors
+  // exist. So the node ID isn't changed.
+  ASSERT_EQ(interpreter_->execution_plan()[0], 0);
+}
+
+TEST_F(TestDelegateWithDynamicTensors, AllowDynamicTensors) {
+  interpreter_->ModifyGraphWithDelegate(&delegate_, true);
+
+  ASSERT_EQ(interpreter_->execution_plan().size(), 1);
+  // The node should be replaced because dynamic tensors are allowed. Therefore
+  // only node ID in the execution plan is changed from 0 to 1.
+  ASSERT_EQ(interpreter_->execution_plan()[0], 1);
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From e69a3e1ac1cd0c31a8e8078982212cca1fbf988e Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Mon, 19 Mar 2018 14:48:23 -0700
Subject: [PATCH 195/960] Fix build breakage with downloadable clang and
 -fopenmp.

By disabling openmp when building with clang.
If we want to enable openmp with clang, we'll probably have to have libomp as
an explicit dependency.

This fixes a breakage found by OS CI:
https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu-clang/215/

PiperOrigin-RevId: 189644968
---
 third_party/mkl_dnn/mkldnn.BUILD | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 58bb7a6a5d..752a0d8498 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -1,5 +1,13 @@
 exports_files(["LICENSE"])
 
+config_setting(
+    name = "clang_linux_x86_64",
+    values = {
+        "cpu": "k8",
+        "define": "using_cuda_clang=true",
+    },
+)
+
 cc_library(
     name = "mkl_dnn",
     srcs = glob([
@@ -9,8 +17,11 @@ cc_library(
     hdrs = glob(["include/*"]),
     copts = ["-fexceptions"] + select({
         "@org_tensorflow//tensorflow:linux_x86_64": [
-            "-fopenmp",
+            "-fopenmp",  # only works with gcc
         ],
+        # TODO(ibiryukov): enable openmp with clang by including libomp as a
+        # dependency.
+        ":clang_linux_x86_64": [],
         "//conditions:default": [],
     }),
     includes = [
-- 
GitLab


From 8dec85d39480ce19130bac56ebb54c00b53085ce Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 19 Mar 2018 14:57:09 -0700
Subject: [PATCH 196/960] Maintain an updateable map of devices in the eager
 context.

PiperOrigin-RevId: 189646358
---
 tensorflow/c/eager/c_api.cc         |  9 +++++++--
 tensorflow/c/eager/c_api_internal.h | 11 ++++++++++-
 2 files changed, 17 insertions(+), 3 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 4e5703ffe0..a23015c99e 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -346,8 +346,13 @@ void TFE_DeleteOp(TFE_Op* op) { delete op; }
 void TFE_OpSetDevice(TFE_Op* op, const char* device_name, TF_Status* status) {
   tensorflow::Device* d = nullptr;
   if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status = op->ctx->device_manager->LookupDevice(device_name, &d);
-    if (!status->status.ok()) return;
+    auto it = op->ctx->devices_map.find(device_name);
+    if (it == op->ctx->devices_map.end()) {
+      status->status =
+          tensorflow::errors::InvalidArgument(device_name, " unknown device.");
+      return;
+    }
+    d = it->second;
   }
   op->device = d;
 }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 1edbe81992..cc5ed48b48 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
@@ -70,6 +71,10 @@ struct TFE_Context {
             opts.session_options.options.config.log_device_placement()),
         async_default(opts.async) {
     if (async_default) executor.EnableAsync();
+
+    for (auto* device : devices) {
+      devices_map[tensorflow::StringPiece(device->name())] = device;
+    }
   }
 
   const bool soft_placement;
@@ -83,7 +88,11 @@ struct TFE_Context {
 
   std::unique_ptr<tensorflow::DeviceMgr> device_manager;
   // Devices owned by device_manager
-  const std::vector<tensorflow::Device*> devices;
+  std::vector<tensorflow::Device*> devices;
+  // All devices are not owned.
+  tensorflow::gtl::FlatMap<tensorflow::StringPiece, tensorflow::Device*,
+                           tensorflow::StringPieceHasher>
+      devices_map;
   tensorflow::Rendezvous* const rendezvous;
 
   tensorflow::mutex functions_mu;
-- 
GitLab


From dee1bc350ac0826822161f211f7fa8a1e1ae62f0 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 19 Mar 2018 15:06:40 -0700
Subject: [PATCH 197/960] Allowing the FunctionBufferingResource to be passed
 in thread_pool_size=0 in which case we wouldn't pass in a runner to the
 FLR::Run call and rely on the underlying device threadpool instead.

PiperOrigin-RevId: 189648051
---
 .../data/kernels/prefetching_kernels.cc       | 24 ++++++++++++-------
 .../data/python/ops/prefetching_ops.py        |  2 +-
 2 files changed, 16 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 1baac3ea52..2f986f2bb1 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -49,16 +49,18 @@ class FunctionBufferingResource : public ResourceBase {
         source_device_(source_device),
         target_device_(target_device),
         func_args_(func_args),
-        thread_pool_(new thread::ThreadPool(Env::Default(), ThreadOptions(),
-                                            "buffer_resource", thread_pool_size,
-                                            false /* low_latency_hint */)),
         handle_(kInvalidHandle),
         is_buffering_(false),
         end_of_sequence_(false),
         cancelled_(false) {
-    runner_ = [this](std::function<void()> c) {
-      thread_pool_->Schedule(std::move(c));
-    };
+    if (thread_pool_size > 0) {
+      thread_pool_ = new thread::ThreadPool(Env::Default(), ThreadOptions(),
+                                            "buffer_resource", thread_pool_size,
+                                            false /* low_latency_hint */);
+      runner_ = [this](std::function<void()> c) {
+        thread_pool_->Schedule(std::move(c));
+      };
+    }
   }
 
   ~FunctionBufferingResource() override {
@@ -69,7 +71,9 @@ class FunctionBufferingResource : public ResourceBase {
         cond_var_.wait(l);
       }
     }
-    delete thread_pool_;
+    if (thread_pool_ != nullptr) {
+      delete thread_pool_;
+    }
   }
 
   string DebugString() override {
@@ -175,7 +179,9 @@ class FunctionBufferingResource : public ResourceBase {
     FunctionLibraryRuntime::Options opts;
     // Copied from CapturedFunction::generate_step_id();
     opts.step_id = -std::abs(static_cast<int64>(random::New64()));
-    opts.runner = &runner_;
+    if (runner_ != nullptr) {
+      opts.runner = &runner_;
+    }
     opts.source_device = source_device_;
     AllocatorAttributes arg_alloc_attr;
     arg_alloc_attr.set_on_host(true);
@@ -231,7 +237,7 @@ class FunctionBufferingResource : public ResourceBase {
   const string source_device_;
   const string target_device_;
   const std::vector<Tensor> func_args_;
-  thread::ThreadPool* thread_pool_;
+  thread::ThreadPool* thread_pool_ = nullptr;
   FunctionLibraryRuntime::Handle handle_ GUARDED_BY(mu_);
   std::deque<BufferElement> buffer_ GUARDED_BY(mu_);
   std::deque<FunctionBufferCallback> requests_ GUARDED_BY(mu_);
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 7059b358f3..b16f12c4ee 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -27,7 +27,7 @@ def function_buffering_resource(string_arg,
                                 target_device,
                                 f,
                                 buffer_size,
-                                thread_pool_size=1,
+                                thread_pool_size=0,
                                 container="",
                                 shared_name=None,
                                 name=None):
-- 
GitLab


From 12baea6c9a2ccb15f24ca79f18bcdd639b149592 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 15:09:23 -0700
Subject: [PATCH 198/960] Use fully-qualified function names and avoid the need
 to replace attributes.

PiperOrigin-RevId: 189648496
---
 .../py2tf/converters/logical_expressions.py   | 52 ++++++++-----------
 1 file changed, 23 insertions(+), 29 deletions(-)

diff --git a/tensorflow/contrib/py2tf/converters/logical_expressions.py b/tensorflow/contrib/py2tf/converters/logical_expressions.py
index 10192e6a03..e0abf74ebc 100644
--- a/tensorflow/contrib/py2tf/converters/logical_expressions.py
+++ b/tensorflow/contrib/py2tf/converters/logical_expressions.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 import gast
 
 from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.contrib.py2tf.pyct import templates
 from tensorflow.contrib.py2tf.pyct import transformer
 
@@ -44,17 +45,18 @@ class LogicalExpressionTransformer(transformer.Base):
   def __init__(self, context):
     super(LogicalExpressionTransformer, self).__init__(context)
     # TODO(mdan): Look into replacing with bitwise operators instead.
+    # TODO(mdan): Skip replacing if the function is trivial.
     self.op_mapping = {
-        gast.And: 'logical_and',
-        gast.Eq: 'equal',
-        gast.Gt: 'greater',
-        gast.GtE: 'greater_equal',
-        gast.Lt: 'less',
-        gast.LtE: 'less_equal',
-        gast.Not: 'logical_not',
-        gast.NotEq: 'not_equal',
-        gast.Or: 'logical_or',
-        gast.USub: 'negative',
+        gast.And: 'tf.logical_and',
+        gast.Eq: 'tf.equal',
+        gast.Gt: 'tf.greater',
+        gast.GtE: 'tf.greater_equal',
+        gast.Lt: 'tf.less',
+        gast.LtE: 'tf.less_equal',
+        gast.Not: 'tf.logical_not',
+        gast.NotEq: 'tf.not_equal',
+        gast.Or: 'tf.logical_or',
+        gast.USub: 'tf.negative',
         gast.Is: 'py2tf_utils.dynamic_is',
         gast.IsNot: 'py2tf_utils.dynamic_is_not'
     }
@@ -70,27 +72,19 @@ class LogicalExpressionTransformer(transformer.Base):
         '"a.x or b"; for a workaround, assign the expression to a local '
         'variable and use that instead, for example "tmp = a.x", "tmp or b"')
 
-  def _matching_tf_op(self, operator):
+  def _matching_func(self, operator):
     op_type = type(operator)
     mapped_op = self.op_mapping.get(op_type)
     if not mapped_op:
       raise NotImplementedError('operator %s is not yet supported' % op_type)
     return mapped_op
 
-  def _inline_tf_op(self, op_name, args):
-    if 'py2tf_utils' in op_name:
-      # TODO(alexbw): explicitly spelling out the attribute function name
-      # until fix for issue highlighted in cl/188931581 lands.
-      template = """
-      py2tf_utils.op_name(args)
+  def _as_function(self, func_name, args):
+    template = """
+      func_name(args)
     """
-      op_name = op_name.replace('py2tf_utils.', '')
-    else:
-      template = """
-        tf.op_name(args)
-      """
     replacement = templates.replace_as_expression(
-        template, op_name=op_name, args=args)
+        template, func_name=parser.parse_expression(func_name), args=args)
     anno.setanno(replacement, SAFE_BOOLEAN_OPERAND, True)
     return replacement
 
@@ -104,14 +98,14 @@ class LogicalExpressionTransformer(transformer.Base):
     #   a < b < c   ->   a < b and b < c
     while ops_and_comps:
       op, right = ops_and_comps.pop(0)
-      binary_comparison = self._inline_tf_op(self._matching_tf_op(op),
-                                             (left, right))
+      binary_comparison = self._as_function(
+          self._matching_func(op), (left, right))
       if isinstance(left, gast.Name) and isinstance(right, gast.Name):
         anno.setanno(binary_comparison, SAFE_BOOLEAN_OPERAND, True)
       if op_tree:
         self._expect_simple_symbol(right)
-        op_tree = self._inline_tf_op('logical_and',
-                                     (binary_comparison, op_tree))
+        op_tree = self._as_function('tf.logical_and',
+                                    (binary_comparison, op_tree))
       else:
         op_tree = binary_comparison
       left = right
@@ -120,7 +114,7 @@ class LogicalExpressionTransformer(transformer.Base):
 
   def visit_UnaryOp(self, node):
     node = self.generic_visit(node)
-    return self._inline_tf_op(self._matching_tf_op(node.op), node.operand)
+    return self._as_function(self._matching_func(node.op), node.operand)
 
   def visit_BoolOp(self, node):
     node = self.generic_visit(node)
@@ -130,7 +124,7 @@ class LogicalExpressionTransformer(transformer.Base):
     while node_values:
       left = node_values.pop()
       self._expect_simple_symbol(left)
-      right = self._inline_tf_op(self._matching_tf_op(node.op), (left, right))
+      right = self._as_function(self._matching_func(node.op), (left, right))
     return right
 
 
-- 
GitLab


From 2d6176e66b48956946b34d595c1dacedd2703fff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 15:13:53 -0700
Subject: [PATCH 199/960] Run flatbuffer verifier before reading a TFLITE file
 into toco.

PiperOrigin-RevId: 189649236
---
 tensorflow/contrib/lite/toco/tflite/import.cc | 11 ++++
 .../contrib/lite/toco/tflite/import_test.cc   | 62 ++++++++++++++-----
 2 files changed, 57 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/tflite/import.cc b/tensorflow/contrib/lite/toco/tflite/import.cc
index e16784fd21..867395e881 100644
--- a/tensorflow/contrib/lite/toco/tflite/import.cc
+++ b/tensorflow/contrib/lite/toco/tflite/import.cc
@@ -162,8 +162,19 @@ void ImportIOTensors(const ::tflite::Model& input_model,
   }
 }
 
+namespace {
+bool Verify(const void* buf, size_t len) {
+  ::flatbuffers::Verifier verifier(static_cast<const uint8_t*>(buf), len);
+  return ::tflite::VerifyModelBuffer(verifier);
+}
+}  // namespace
+
 std::unique_ptr<Model> Import(const ModelFlags& model_flags,
                               const string& input_file_contents) {
+  if (!Verify(input_file_contents.data(), input_file_contents.size())) {
+    LOG(FATAL) << "Invalid flatbuffer.";
+  }
+
   const ::tflite::Model* input_model =
       ::tflite::GetModel(input_file_contents.data());
 
diff --git a/tensorflow/contrib/lite/toco/tflite/import_test.cc b/tensorflow/contrib/lite/toco/tflite/import_test.cc
index f25b170876..937a291cf7 100644
--- a/tensorflow/contrib/lite/toco/tflite/import_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/import_test.cc
@@ -66,15 +66,43 @@ class ImportTest : public ::testing::Test {
   }
 
   Offset<Vector<Offset<::tflite::OperatorCode>>> BuildOpCodes() {
-    auto c1 =
-        ::tflite::CreateOperatorCode(builder_, ::tflite::BuiltinOperator_CUSTOM,
-                                     builder_.CreateString("custom_op_one"));
+    auto c1 = ::tflite::CreateOperatorCode(
+        builder_, ::tflite::BuiltinOperator_MAX_POOL_2D, 0);
     auto c2 = ::tflite::CreateOperatorCode(
         builder_, ::tflite::BuiltinOperator_CONV_2D, 0);
     return builder_.CreateVector(
         std::vector<Offset<::tflite::OperatorCode>>({c1, c2}));
   }
 
+  Offset<Vector<Offset<::tflite::Operator>>> BuildOperators() {
+    auto is = builder_.CreateVector<int>({0});
+    auto os = builder_.CreateVector<int>({1});
+    auto op = ::tflite::CreateOperator(
+        builder_, 0, is, os, ::tflite::BuiltinOptions_Conv2DOptions,
+        ::tflite::CreateConv2DOptions(builder_, ::tflite::Padding_VALID, 1, 1,
+                                      ::tflite::ActivationFunctionType_NONE)
+            .Union(),
+        /*custom_options=*/0, ::tflite::CustomOptionsFormat_FLEXBUFFERS);
+
+    return builder_.CreateVector(std::vector<Offset<::tflite::Operator>>({op}));
+  }
+
+  Offset<Vector<Offset<::tflite::SubGraph>>> BuildSubGraphs(
+      Offset<Vector<Offset<::tflite::Tensor>>> tensors,
+      Offset<Vector<Offset<::tflite::Operator>>> operators,
+      int num_sub_graphs = 1) {
+    std::vector<int32_t> inputs = {0};
+    std::vector<int32_t> outputs = {1};
+    std::vector<Offset<::tflite::SubGraph>> v;
+    for (int i = 0; i < num_sub_graphs; ++i) {
+      v.push_back(::tflite::CreateSubGraph(
+          builder_, tensors, builder_.CreateVector(inputs),
+          builder_.CreateVector(outputs), operators,
+          builder_.CreateString("subgraph")));
+    }
+    return builder_.CreateVector(v);
+  }
+
   // This is a very simplistic model. We are not interested in testing all the
   // details here, since tf.mini's testing framework will be exercising all the
   // conversions multiple times, and the conversion of operators is tested by
@@ -83,14 +111,13 @@ class ImportTest : public ::testing::Test {
     auto buffers = BuildBuffers();
     auto tensors = BuildTensors();
     auto opcodes = BuildOpCodes();
-
-    auto subgraph = ::tflite::CreateSubGraph(builder_, tensors, 0, 0, 0);
-    std::vector<flatbuffers::Offset<::tflite::SubGraph>> subgraph_vector(
-        {subgraph});
-    auto subgraphs = builder_.CreateVector(subgraph_vector);
+    auto operators = BuildOperators();
+    auto subgraphs = BuildSubGraphs(tensors, operators);
     auto s = builder_.CreateString("");
-    builder_.Finish(::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION,
-                                          opcodes, subgraphs, s, buffers));
+
+    ::tflite::FinishModelBuffer(
+        builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION,
+                                        opcodes, subgraphs, s, buffers));
 
     input_model_ = ::tflite::GetModel(builder_.GetBufferPointer());
   }
@@ -99,7 +126,6 @@ class ImportTest : public ::testing::Test {
                   builder_.GetSize());
   }
   flatbuffers::FlatBufferBuilder builder_;
-  // const uint8_t* buffer_ = nullptr;
   const ::tflite::Model* input_model_ = nullptr;
 };
 
@@ -116,7 +142,7 @@ TEST_F(ImportTest, LoadOperatorsTable) {
 
   details::OperatorsTable operators;
   details::LoadOperatorsTable(*input_model_, &operators);
-  EXPECT_THAT(operators, ElementsAre("custom_op_one", "CONV_2D"));
+  EXPECT_THAT(operators, ElementsAre("MAX_POOL_2D", "CONV_2D"));
 }
 
 TEST_F(ImportTest, Tensors) {
@@ -143,13 +169,17 @@ TEST_F(ImportTest, Tensors) {
   EXPECT_EQ(100, q->zero_point);
 }
 
-TEST_F(ImportTest, NoSubGraphs) {
+TEST_F(ImportTest, MultipleSubGraphs) {
   auto buffers = BuildBuffers();
+  auto tensors = BuildTensors();
   auto opcodes = BuildOpCodes();
-  auto subgraphs = 0;  // no subgraphs in this model
+  auto operators = BuildOperators();
+  auto subgraphs = BuildSubGraphs(tensors, operators, 2);
   auto comment = builder_.CreateString("");
-  builder_.Finish(::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION,
-                                        opcodes, subgraphs, comment, buffers));
+  ::tflite::FinishModelBuffer(
+      builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
+                                      subgraphs, comment, buffers));
+
   input_model_ = ::tflite::GetModel(builder_.GetBufferPointer());
 
   EXPECT_DEATH(Import(ModelFlags(), InputModelAsString()),
-- 
GitLab


From 7cabd979a46febafdb90a83865cd743233d4449f Mon Sep 17 00:00:00 2001
From: Alan Du <alan.h.du@gmail.com>
Date: Mon, 19 Mar 2018 18:42:08 -0400
Subject: [PATCH 200/960] Don't use NCHW or NHCW in tf.layers.conv1d (#17455)

* Don't use NCHW or NHCW for conv1d

Fixes deprecation warning when using tf.layers.conv1d
---
 tensorflow/python/ops/nn_ops.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 9b2aaa4c1c..a74de39eab 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -150,14 +150,12 @@ class _NonAtrousConvolution(object):
                                                               conv_dims))
     if conv_dims == 1:
       # conv1d uses the 2-d data format names
-      if data_format is None or data_format == "NWC":
-        data_format_2d = "NHWC"
-      elif data_format == "NCW":
-        data_format_2d = "NCHW"
-      else:
+      if data_format is None:
+        data_format = "NWC"
+      elif data_format not in {"NCW", "NWC", "NCHW", "NHWC"}:
         raise ValueError("data_format must be \"NWC\" or \"NCW\".")
       self.strides = strides[0]
-      self.data_format = data_format_2d
+      self.data_format = data_format
       self.conv_op = self._conv1d
     elif conv_dims == 2:
       if data_format is None or data_format == "NHWC":
-- 
GitLab


From 41781e61d04763dbc1ebca77292410b8136c2adb Mon Sep 17 00:00:00 2001
From: Xiaoqiang Zheng <zhengxq@google.com>
Date: Mon, 19 Mar 2018 15:40:37 -0700
Subject: [PATCH 201/960] A few changes to improve the real data performance: *
 Turn off the force_gpu_compatible by default. * Move the cast operator within
 the processing operator. * Have the map_and_batch operator produce
 gpu_compatible output. * Add an option to produce fp16 tensors for network
 transfer by default.

On DGX-1 V100, with resnet50, I got 5050 images/sec on real data, 5395 images/sec on synthetic data.
With trivial model, I got 13000+ images/sec on real data.

PiperOrigin-RevId: 189653575
---
 tensorflow/core/kernels/data/map_and_batch_dataset_op.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
index 9ce263732f..e22200f758 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
@@ -183,7 +183,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
               TensorShape component_shape(
                   batch_results_[current_batch_index_].output[i].shape());
               component_shape.set_dim(0, num_elements);
-              Tensor component(ctx->allocator({}), output[i].dtype(),
+              AllocatorAttributes attr;
+              attr.set_gpu_compatible(true);
+              Tensor component(ctx->allocator(attr), output[i].dtype(),
                                component_shape);
               TF_RETURN_IF_ERROR(
                   CopyPartialBatch(&component, output[i], num_elements));
@@ -255,7 +257,9 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
         for (size_t i = 0; i < num_components; ++i) {
           TensorShape component_shape({dataset()->batch_size_});
           component_shape.AppendShape(return_values[i].shape());
-          Tensor component(ctx->allocator({}), return_values[i].dtype(),
+          AllocatorAttributes attr;
+          attr.set_gpu_compatible(true);
+          Tensor component(ctx->allocator(attr), return_values[i].dtype(),
                            component_shape);
           batch_result->output.emplace_back(std::move(component));
         }
-- 
GitLab


From 448d65c673980d167fbd97206334bec641d118e6 Mon Sep 17 00:00:00 2001
From: Zhixian Yan <zhixianyan@google.com>
Date: Mon, 19 Mar 2018 15:50:02 -0700
Subject: [PATCH 202/960] Disable lstm test in generated_example due to state
 non-definitive init.

PiperOrigin-RevId: 189654943
---
 tensorflow/contrib/lite/testing/BUILD                         | 1 -
 .../contrib/lite/testing/generated_examples_zip_test.cc       | 4 ----
 2 files changed, 5 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 631601656d..f1b18ad30f 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -35,7 +35,6 @@ gen_zipped_test_files(
         "l2norm.zip",
         "local_response_norm.zip",
         "log_softmax.zip",
-        "lstm.zip",
         "max_pool.zip",
         "mean.zip",
         "mul.zip",
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 88c5aaa099..5e76e7c510 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -88,9 +88,6 @@ std::map<string, string> kBrokenTests = {
 
     // Transpose only supports 1D-4D input tensors.
     {R"(^\/transpose.*input_shape=\[.,.,.,.,.\])", "71545879"},
-
-    // Lstm kernel gets different results on tsan, asan, msan.
-    {R"(^\/lstmdtype=tf.float32.*)", "73830845"},
 };
 
 // Allows test data to be unzipped into a temporary directory and makes
@@ -250,7 +247,6 @@ INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
-INSTANTIATE_TESTS(lstm)
 INSTANTIATE_TESTS(max_pool)
 INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(mul)
-- 
GitLab


From 8fded7872fe0921e0f90fac1891cda0c46a26855 Mon Sep 17 00:00:00 2001
From: jjsjann123 <alex.jann2012@gmail.com>
Date: Mon, 19 Mar 2018 16:35:29 -0700
Subject: [PATCH 203/960] [update TensorRT converter] (#17772)

* [update TensorRT converter]
  fixed FusedBatchNorm to support broadcast;
  remove fp16 conversion for type int const
  add Snapshot in conversion (treated as identity)

* [TensorRT converter batchnorm code cleaning]

* TRT batchnorm code cleaning
---
 .../contrib/tensorrt/convert/convert_graph.cc |   1 +
 .../contrib/tensorrt/convert/convert_nodes.cc | 245 +++++++++---------
 2 files changed, 118 insertions(+), 128 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index eea8c8efa2..90447ee666 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -55,6 +55,7 @@ bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
       "Identity",
+      "Snapshot",
       "Const",
       "Conv2D",
       "MaxPool",
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 4c00630cfe..7f4b57f9f4 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -346,11 +346,10 @@ void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
       break;
     }
     case tensorflow::DataType::DT_HALF: {
-      Reorder2(
-          {k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
-          istrides,
-          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
-          ostrides);
+      Reorder2({k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
+               istrides, static_cast<Eigen::half*>(
+                             const_cast<void*>(oweights->GetValues())),
+               ostrides);
       break;
     }
     default:
@@ -998,9 +997,7 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
     const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs,
-    int group  // group ==0 specifies depthwise conv
-) {
+    std::vector<TRT_TensorOrWeights>* outputs, int group) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   TFAttrs attrs(node_def);
@@ -1134,9 +1131,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented(
-        "binary op: " + node_def.op() +
-        " not supported at: " + node_def.name());
+    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
+                                             " not supported at: " +
+                                             node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1447,62 +1444,23 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      switch (dtype) {
-        case (tensorflow::DT_INT32): {
-          half_tensor.device(defd) =
-              tensor.flat<int32>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT16): {
-          half_tensor.device(defd) =
-              tensor.flat<int16>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT8): {
-          half_tensor.device(defd) =
-              tensor.flat<int8>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_UINT8): {
-          half_tensor.device(defd) =
-              tensor.flat<uint8>().template cast<Eigen::half>();
-          break;
-        }
-        default:
-          return tensorflow::errors::InvalidArgument(
-              "Datatype " + tensorflow::DataTypeString(dtype) +
-              " for FP16 conversion");
-          break;
-      };
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
-      len_data = std::max(len_data, len_tensor);
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<int32> tensor_data(
-          weights_tensor.int_val().begin(),
-          weights_tensor.int_val()
-              .end());  //  make a local copy first to flatten
-                        //  doesn't have to be contigous
-      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    //  we should not have converted //if (ctx.isFP16()) {
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+    len_data = std::max(len_data, len_tensor);
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<int32> tensor_data(
+        weights_tensor.int_val().begin(),
+        weights_tensor.int_val().end());  //  make a local copy first to flatten
+                                          //  doesn't have to be contigous
+    memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.tensor_content().empty()) {
+    //  obsolete method.
+    //  After optimization path, we do not see weights in this format.
+    //  fp16 conversion technically should be needed here.
     VLOG(2) << "TENSOR!!!" << node_def.name();
     const auto& content = weights_tensor.tensor_content();
 
@@ -1784,8 +1742,6 @@ tensorflow::Status ConvertConcat(Converter& ctx,
   TRT_ShapedWeights axis = inputs.at(input_size).weights();
 
   TFAttrs attrs(node_def);
-  // auto attr_size = attrs.at("N")->i();
-  // auto data_type = attrs.get<nvinfer1::DataType>("T");
   auto index_type = attrs.get<tensorflow::DataType>("Tidx");
 
   // TODO(jie): handle data type
@@ -1875,71 +1831,103 @@ tensorflow::Status ConvertFusedBatchNorm(
         "only is_training=false is supported, at " + node_def.name());
   }
   nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
-  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
-  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
-  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
-  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
-  TRT_ShapedWeights combined_scale_weights =
-      ctx.get_temp_weights_like(scale_weights);
-  TRT_ShapedWeights combined_offset_weights =
-      ctx.get_temp_weights_like(offset_weights);
-  size_t nweight = scale_weights.count();
-  if ((scale_weights.type_ == offset_weights.type_) &&
-      (mean_weights.type_ == variance_weights.type_) &&
-      (scale_weights.type_ == variance_weights.type_)) {
-    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
-        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+
+  //  Check parameter types
+  auto parameter_type = inputs.at(1).weights().type_;
+  if ((parameter_type != tensorflow::DataType::DT_FLOAT) &&
+      (parameter_type != tensorflow::DataType::DT_HALF)) {
+    return tensorflow::errors::Unimplemented(
+        "only float32 or float16 weight data type is supported, for node " +
+        node_def.name() + " got " + tensorflow::DataTypeString(parameter_type));
+  }
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().type_ != parameter_type) {
       return tensorflow::errors::Unimplemented(
-          "only float32 or float16 weight data type is supported, for node " +
-          node_def.name() + " got " +
-          tensorflow::DataTypeString(scale_weights.type_));
+          "Inconsistent parameter type for batchnormis not supported, at: " +
+          node_def.name());
     }
-    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
-        float offset =
-            (static_cast<float const*>(offset_weights.GetValues()))[i];
-        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
-        float variance =
-            (static_cast<float const*>(variance_weights.GetValues()))[i];
-        float& combined_scale_ref = const_cast<float*>(
-            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
-        float& combined_offset_ref = const_cast<float*>(
-            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
-        combined_scale_ref = scale / sqrtf(variance + epsilon);
-        combined_offset_ref = offset - mean * combined_scale_ref;
-      }
-    } else {
-      const Eigen::half* scale_vals =
-          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
-      const Eigen::half* off_vals =
-          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
-      const Eigen::half* mean_vals =
-          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
-      const Eigen::half* variance_vals =
-          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
-      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
-      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale(scale_vals[i]);
-        float offset(off_vals[i]);
-        float mean(mean_vals[i]);
-        float variance(variance_vals[i]);
-        float combined_scale_ref = scale / sqrtf(variance + epsilon);
-        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
-        float combined_offset_ref = offset - mean * combined_scale_ref;
-        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+  }
+
+  TRT_ShapedWeights dummy_power_weights(parameter_type);
+  size_t nweight = 0;
+  for (int i = 1; i < 5; i++) {
+    nweight = std::max(nweight, (size_t)inputs.at(i).weights().count());
+  }
+  TRT_ShapedWeights* ptr_shape_weights = nullptr;
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().count() == nweight) {
+      ptr_shape_weights =
+          const_cast<TRT_ShapedWeights*>(&(inputs.at(i).weights()));
+    } else if (inputs.at(i).weights().count() != 1) {
+      return tensorflow::errors::InvalidArgument(
+          "Inconsistent batchnorm parameter count, at: " + node_def.name());
+    }
+  }
+  //  We could technically have two weights with different shape.
+  //  that requires two addScale op, arguably less performant
+  TRT_ShapedWeights combined_scale_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+  TRT_ShapedWeights combined_offset_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+
+  const Eigen::half* cast_vals_array[4];
+  const float* vals_array[4];
+  for (int j = 0; j < 4; j++) {
+    cast_vals_array[j] =
+        static_cast<Eigen::half const*>(inputs.at(j + 1).weights().GetValues());
+    vals_array[j] =
+        static_cast<float const*>(inputs.at(j + 1).weights().GetValues());
+  }
+  Eigen::half* cast_combined_scale_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+  Eigen::half* cast_combined_offset_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+  float* combined_scale_vals = const_cast<float*>(
+      static_cast<float const*>(combined_scale_weights.GetValues()));
+  float* combined_offset_vals = const_cast<float*>(
+      static_cast<float const*>(combined_offset_weights.GetValues()));
+
+  for (size_t i = 0; i < nweight; ++i) {
+    float batchnorm_data[4];
+    for (int j = 0; j < 4; j++) {
+      if (inputs.at(j + 1).weights().count() != 1) {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][i];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][i]);
+        }
+      } else {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][0];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][0]);
+        }
       }
     }
+    float scale = batchnorm_data[0];
+    float offset = batchnorm_data[1];
+    float mean = batchnorm_data[2];
+    float variance = batchnorm_data[3];
+    float combined_scale_val = scale / sqrtf(variance + epsilon);
+    float combined_offset_val = offset - mean * combined_scale_val;
+    if (parameter_type == tensorflow::DT_FLOAT) {
+      combined_scale_vals[i] = combined_scale_val;
+      combined_offset_vals[i] = combined_offset_val;
+    } else if (parameter_type == tensorflow::DT_HALF) {
+      cast_combined_scale_vals[i] = Eigen::half(combined_scale_val);
+      cast_combined_offset_vals[i] = Eigen::half(combined_offset_val);
+    }
   }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      combined_offset_weights.GetWeightsForTRT(),
-      combined_scale_weights.GetWeightsForTRT(),
-      dummy_power_weights.GetWeightsForTRT());
+
+  nvinfer1::ScaleMode mode = nweight == 1 ? nvinfer1::ScaleMode::kUNIFORM
+                                          : nvinfer1::ScaleMode::kCHANNEL;
+  nvinfer1::IScaleLayer* layer =
+      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
+                              combined_offset_weights.GetWeightsForTRT(),
+                              combined_scale_weights.GetWeightsForTRT(),
+                              dummy_power_weights.GetWeightsForTRT());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -2050,6 +2038,7 @@ void Converter::register_op_converters() {
   op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
+  op_registry_["Snapshot"] = ConvertIdentity;  // Snapshot should be removed
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
-- 
GitLab


From b1208ba0197547e75c3860b385d036e3909f8ea9 Mon Sep 17 00:00:00 2001
From: Lukasz Kaiser <lukaszkaiser@google.com>
Date: Mon, 19 Mar 2018 16:56:44 -0700
Subject: [PATCH 204/960] Automated g4 rollback of changelist 188440916

PiperOrigin-RevId: 189664854
---
 tensorflow/core/kernels/pad_op.cc             | 124 ++----------------
 tensorflow/python/kernel_tests/pad_op_test.py |  25 ----
 2 files changed, 11 insertions(+), 138 deletions(-)

diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index 04c71e384b..a7238ef67b 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -104,144 +104,42 @@ class PadOp : public OpKernel {
       return;
     }
 
-    TensorShape collapsed_input_shape;
-    TensorShape collapsed_output_shape;
-    Tensor collapsed_paddings;
-    if (fixed_dims > 1 &&
-        CollapseAdjacentNonPaddedDimensions(
-            in0.shape(), in1, output_shape, &collapsed_input_shape,
-            &collapsed_paddings, &collapsed_output_shape)) {
-      Tensor collapsed_input;
-      CHECK(collapsed_input.CopyFrom(in0, collapsed_input_shape));
-      Tensor collapsed_output;
-      OP_REQUIRES_OK(context, context->allocate_temp(collapsed_input.dtype(),
-                                                     collapsed_output_shape,
-                                                     &collapsed_output));
-      const Tensor& collapsed_paddings_ref = collapsed_paddings;
-      typename TTypes<Tpadding>::ConstMatrix collapsed_paddings_matrix =
-          collapsed_paddings_ref.matrix<Tpadding>();
+    Tensor* output = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output));
 
-      OperateWithVariableRank(context, collapsed_input_shape.dims(),
-                              collapsed_input, collapsed_paddings_matrix,
-                              pad_value, &collapsed_output);
-
-      Tensor output;
-      CHECK(output.CopyFrom(collapsed_output, output_shape));
-      context->set_output(0, output);
-    } else {
-      Tensor* output = nullptr;
-      OP_REQUIRES_OK(context,
-                     context->allocate_output(0, output_shape, &output));
-      OperateWithVariableRank(context, fixed_dims, in0, paddings, pad_value,
-                              output);
-    }
-  }
-
- private:
-  // Collapses adjacent dimensions that are not padded to one dimension for
-  // speed. Returns true if any two dimensions are collapsed. For example,
-  //
-  //   Pad(input_shape=[8, 28, 28, 3],
-  //       paddings=[[0, 0], [0, 0], [0, 0], [0, 1]]
-  // is equivalent to
-  //   Pad(input_shape=[6272, 3],
-  //       paddings=[[0, 0], [0, 1]])
-  //
-  // input_shape: the original input shape.
-  // paddings_as_tensor: the original paddings.
-  // output_shape: the original output shape.
-  // collapsed_input_shape: the input shape after collapsing.
-  // collapsed_paddings_as_tensor: the paddings after collapsing.
-  // collapsed_output_shape: the output shape after collapsing.
-  static bool CollapseAdjacentNonPaddedDimensions(
-      const TensorShape& input_shape, const Tensor& paddings_as_tensor,
-      const TensorShape& output_shape, TensorShape* collapsed_input_shape,
-      Tensor* collapsed_paddings_as_tensor,
-      TensorShape* collapsed_output_shape) {
-    bool collapsed = false;
-    typename TTypes<Tpadding>::ConstMatrix paddings =
-        paddings_as_tensor.matrix<Tpadding>();
-    std::vector<std::pair<int, int>> collapsed_paddings;
-    int i = 0;
-    while (i < paddings.dimension(0)) {
-      if (paddings(i, 0) != 0 || paddings(i, 1) != 0) {
-        // If padded, copy the original dimension over.
-        collapsed_input_shape->InsertDim(collapsed_input_shape->dims(),
-                                         input_shape.dim_size(i));
-        collapsed_output_shape->InsertDim(collapsed_output_shape->dims(),
-                                          output_shape.dim_size(i));
-        collapsed_paddings.push_back({paddings(i, 0), paddings(i, 1)});
-        ++i;
-      } else {
-        // If not padded, find the next dimension that is padded and collapse
-        // all dimensions in between to one dimension.
-        int64 collapsed_input_dim_size = input_shape.dim_size(i);
-        int64 collapsed_output_dim_size = output_shape.dim_size(i);
-        ++i;
-        while (i < paddings.dimension(0) && paddings(i, 0) == 0 &&
-               paddings(i, 1) == 0) {
-          collapsed = true;
-          collapsed_input_dim_size *= input_shape.dim_size(i);
-          collapsed_output_dim_size *= output_shape.dim_size(i);
-          ++i;
-        }
-        collapsed_input_shape->InsertDim(collapsed_input_shape->dims(),
-                                         collapsed_input_dim_size);
-        collapsed_output_shape->InsertDim(collapsed_output_shape->dims(),
-                                          collapsed_output_dim_size);
-        collapsed_paddings.push_back({0, 0});
-      }
-    }
-
-    // Copy collapsed_paddings to collapsed_paddings_as_tensor.
-    *collapsed_paddings_as_tensor =
-        Tensor(paddings_as_tensor.dtype(),
-               TensorShape({static_cast<int64>(collapsed_paddings.size()), 2}));
-    auto collapsed_paddings_as_matrix =
-        collapsed_paddings_as_tensor->matrix<Tpadding>();
-    for (size_t i = 0; i < collapsed_paddings.size(); ++i) {
-      collapsed_paddings_as_matrix(i, 0) = collapsed_paddings[i].first;
-      collapsed_paddings_as_matrix(i, 1) = collapsed_paddings[i].second;
-    }
-    return collapsed;
-  }
-
-  void OperateWithVariableRank(OpKernelContext* context, int fixed_dims,
-                               const Tensor& input,
-                               typename TTypes<Tpadding>::ConstMatrix paddings,
-                               T pad_value, Tensor* output) {
     // Invoke the dims-specific implementation.
     switch (fixed_dims) {
       case 0:
-        Operate<0>(context, input.tensor<T, 0>(), paddings, pad_value, output);
+        Operate<0>(context, in0.tensor<T, 0>(), paddings, pad_value, output);
         break;
       case 1:
         // TODO(irving): Once Pad doesn't need a scalar special case,
         // change flat to tensor.  That is, once !allow_legacy_scalars().
-        Operate<1>(context, input.flat<T>(), paddings, pad_value, output);
+        Operate<1>(context, in0.flat<T>(), paddings, pad_value, output);
         break;
       case 2:
-        Operate<2>(context, input.tensor<T, 2>(), paddings, pad_value, output);
+        Operate<2>(context, in0.tensor<T, 2>(), paddings, pad_value, output);
         break;
       case 3:
-        Operate<3>(context, input.tensor<T, 3>(), paddings, pad_value, output);
+        Operate<3>(context, in0.tensor<T, 3>(), paddings, pad_value, output);
         break;
       case 4:
-        Operate<4>(context, input.tensor<T, 4>(), paddings, pad_value, output);
+        Operate<4>(context, in0.tensor<T, 4>(), paddings, pad_value, output);
         break;
       case 5:
-        Operate<5>(context, input.tensor<T, 5>(), paddings, pad_value, output);
+        Operate<5>(context, in0.tensor<T, 5>(), paddings, pad_value, output);
         break;
       case 6:
-        Operate<6>(context, input.tensor<T, 6>(), paddings, pad_value, output);
+        Operate<6>(context, in0.tensor<T, 6>(), paddings, pad_value, output);
         break;
       default:
         OP_REQUIRES(context, false,
                     errors::InvalidArgument("Only ranks up to 6 supported: ",
-                                            input.shape().DebugString()));
+                                            in0.shape().DebugString()));
     }
   }
 
+ private:
   template <int Dims>
   void Operate(OpKernelContext* context,
                typename TTypes<T, Dims>::ConstTensor input,
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index 14632ec29a..9ed5947aae 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -336,30 +336,5 @@ class PadOpTest(test.TestCase):
       self.assertAllEqual(inp, out)
       self.assertShapeEqual(inp, tf_val)
 
-  def testCollapseAdjacentNonPaddedDimensions(self):
-    # pyformat: disable
-    for paddings_value in [[[0, 0], [0, 0], [0, 0], [0, 1]],
-                           [[0, 0], [2, 3], [0, 0], [0, 0]],
-                           [[0, 0], [0, 0], [0, 0], [0, 0]]]:
-      # pyformat: enable
-      inp = constant_op.constant(1.0, shape=[8, 28, 28, 3])
-      paddings = constant_op.constant(paddings_value, dtype=dtypes.int32)
-      padded = array_ops.pad(inp, paddings)
-      middle = array_ops.slice(padded, [row[0] for row in paddings_value],
-                               [dim.value for dim in inp.shape.dims])
-      left = array_ops.slice(padded, [0, 0, 0, 0],
-                             [row[0] for row in paddings_value])
-      right = array_ops.slice(
-          padded,
-          [paddings_value[i][0] + inp.shape.dims[i].value for i in range(4)],
-          [-1, -1, -1, -1])
-      with self.test_session(use_gpu=True):
-        self.assertAllEqual(inp.eval(), middle.eval())
-        self.assertAllEqual(
-            np.zeros([row[0] for row in paddings_value]), left.eval())
-        self.assertAllEqual(
-            np.zeros([row[1] for row in paddings_value]), right.eval())
-
-
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 4f5b7b42e2f8cb6b6e6730b6ada0edbee67dbfe3 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Mon, 19 Mar 2018 17:04:28 -0700
Subject: [PATCH 205/960] Fix test failure

PiperOrigin-RevId: 189666053
---
 tensorflow/python/layers/normalization.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 8b79a92cc4..11daf01670 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -364,8 +364,9 @@ class BatchNormalization(base.Layer):
                         [variable, value, momentum]) as scope:
       with ops.colocate_with(variable):
         decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
-        update_delta = math_ops.multiply(
-            math_ops.subtract(variable.read_value(), value), decay)
+        if decay.dtype != variable.dtype.base_dtype:
+          decay = math_ops.cast(decay, variable.dtype.base_dtype)
+        update_delta = (variable - value) * decay
         return state_ops.assign_sub(variable, update_delta, name=scope)
 
   def _fused_batch_norm(self, inputs, training):
-- 
GitLab


From 2bd7f5e190db4ad2d111f824163855c3dfcb9566 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Mon, 19 Mar 2018 17:13:11 -0700
Subject: [PATCH 206/960] [tf.data] Combine implementations of FlatMapDataset,
 InterleaveDataset and ParallelInterleaveDataset.

PiperOrigin-RevId: 189667086
---
 tensorflow/python/data/ops/dataset_ops.py | 54 ++------------------
 tensorflow/python/data/ops/readers.py     | 60 +++--------------------
 2 files changed, 10 insertions(+), 104 deletions(-)

diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index a0c5a43a45..c0a6283be4 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -1950,47 +1950,13 @@ class FlatMapDataset(Dataset):
     return self._output_types
 
 
-class InterleaveDataset(Dataset):
+class InterleaveDataset(FlatMapDataset):
   """A `Dataset` that maps a function over its input and interleaves the result.
   """
 
   def __init__(self, input_dataset, map_func, cycle_length, block_length):
     """See `Dataset.interleave()` for details."""
-    super(InterleaveDataset, self).__init__()
-    self._input_dataset = input_dataset
-
-    @function.Defun(*nest.flatten(
-        sparse.as_dense_types(input_dataset.output_types,
-                              input_dataset.output_classes)))
-    def tf_map_func(*args):
-      """A wrapper for Defun that facilitates shape inference."""
-      # Pass in shape information from the input_dataset.
-      dense_shapes = sparse.as_dense_shapes(input_dataset.output_shapes,
-                                            input_dataset.output_classes)
-      for arg, shape in zip(args, nest.flatten(dense_shapes)):
-        arg.set_shape(shape)
-
-      nested_args = nest.pack_sequence_as(input_dataset.output_types, args)
-      nested_args = sparse.deserialize_sparse_tensors(
-          nested_args, input_dataset.output_types, input_dataset.output_shapes,
-          input_dataset.output_classes)
-      if _should_unpack_args(nested_args):
-        dataset = map_func(*nested_args)
-      else:
-        dataset = map_func(nested_args)
-
-      if not isinstance(dataset, Dataset):
-        raise TypeError("`map_func` must return a `Dataset` object.")
-
-      self._output_classes = dataset.output_classes
-      self._output_types = dataset.output_types
-      self._output_shapes = dataset.output_shapes
-
-      return dataset._as_variant_tensor()  # pylint: disable=protected-access
-
-    self._map_func = tf_map_func
-    self._map_func.add_to_graph(ops.get_default_graph())
-
+    super(InterleaveDataset, self).__init__(input_dataset, map_func)
     self._cycle_length = ops.convert_to_tensor(
         cycle_length, dtype=dtypes.int64, name="cycle_length")
     self._block_length = ops.convert_to_tensor(
@@ -1999,27 +1965,15 @@ class InterleaveDataset(Dataset):
   def _as_variant_tensor(self):
     return gen_dataset_ops.interleave_dataset(
         self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
-        self._map_func.captured_inputs,
+        self._map_func.captured_inputs,  # pylint: disable=protected-access
         self._cycle_length,
         self._block_length,
-        f=self._map_func,
+        f=self._map_func,  # pylint: disable=protected-access
         output_types=nest.flatten(
             sparse.as_dense_types(self.output_types, self.output_classes)),
         output_shapes=nest.flatten(
             sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
 
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
-
 
 class FilterDataset(Dataset):
   """A `Dataset` that filters its input according to a predicate function."""
diff --git a/tensorflow/python/data/ops/readers.py b/tensorflow/python/data/ops/readers.py
index 6c493d8163..fe033f5546 100644
--- a/tensorflow/python/data/ops/readers.py
+++ b/tensorflow/python/data/ops/readers.py
@@ -22,7 +22,6 @@ from tensorflow.python.data.util import convert
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
@@ -121,51 +120,14 @@ class _TFRecordDataset(dataset_ops.Dataset):
     return dtypes.string
 
 
-class ParallelInterleaveDataset(dataset_ops.Dataset):
+class ParallelInterleaveDataset(dataset_ops.InterleaveDataset):
   """A `Dataset` that maps a function over its input and flattens the result."""
 
   def __init__(self, input_dataset, map_func, cycle_length, block_length,
                sloppy, buffer_output_elements, prefetch_input_elements):
     """See `tf.contrib.data.parallel_interleave()` for details."""
-    super(ParallelInterleaveDataset, self).__init__()
-    self._input_dataset = input_dataset
-
-    @function.Defun(*nest.flatten(
-        sparse.as_dense_types(input_dataset.output_types,
-                              input_dataset.output_classes)))
-    def tf_map_func(*args):
-      """A wrapper for Defun that facilitates shape inference."""
-      # Pass in shape information from the input_dataset.
-      dense_shapes = sparse.as_dense_shapes(input_dataset.output_shapes,
-                                            input_dataset.output_classes)
-      for arg, shape in zip(args, nest.flatten(dense_shapes)):
-        arg.set_shape(shape)
-
-      nested_args = nest.pack_sequence_as(input_dataset.output_types, args)
-      nested_args = sparse.deserialize_sparse_tensors(
-          nested_args, input_dataset.output_types, input_dataset.output_shapes,
-          input_dataset.output_classes)
-      if dataset_ops._should_unpack_args(nested_args):  # pylint: disable=protected-access
-        dataset = map_func(*nested_args)
-      else:
-        dataset = map_func(nested_args)
-
-      if not isinstance(dataset, dataset_ops.Dataset):
-        raise TypeError("`map_func` must return a `Dataset` object.")
-
-      self._output_classes = dataset.output_classes
-      self._output_types = dataset.output_types
-      self._output_shapes = dataset.output_shapes
-
-      return dataset._as_variant_tensor()  # pylint: disable=protected-access
-
-    self._map_func = tf_map_func
-    self._map_func.add_to_graph(ops.get_default_graph())
-
-    self._cycle_length = ops.convert_to_tensor(
-        cycle_length, dtype=dtypes.int64, name="cycle_length")
-    self._block_length = ops.convert_to_tensor(
-        block_length, dtype=dtypes.int64, name="block_length")
+    super(ParallelInterleaveDataset, self).__init__(input_dataset, map_func,
+                                                    cycle_length, block_length)
     self._sloppy = ops.convert_to_tensor(
         sloppy, dtype=dtypes.bool, name="sloppy")
     self._buffer_output_elements = convert.optional_param_to_tensor(
@@ -178,8 +140,9 @@ class ParallelInterleaveDataset(dataset_ops.Dataset):
         argument_default=2 * cycle_length)
 
   def _as_variant_tensor(self):
+    # pylint: disable=protected-access
     return gen_dataset_ops.parallel_interleave_dataset(
-        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        self._input_dataset._as_variant_tensor(),
         self._map_func.captured_inputs,
         self._cycle_length,
         self._block_length,
@@ -191,18 +154,7 @@ class ParallelInterleaveDataset(dataset_ops.Dataset):
             sparse.as_dense_types(self.output_types, self.output_classes)),
         output_shapes=nest.flatten(
             sparse.as_dense_shapes(self.output_shapes, self.output_classes)))
-
-  @property
-  def output_classes(self):
-    return self._output_classes
-
-  @property
-  def output_shapes(self):
-    return self._output_shapes
-
-  @property
-  def output_types(self):
-    return self._output_types
+    # pylint: enable=protected-access
 
 
 @tf_export("data.TFRecordDataset")
-- 
GitLab


From 41335abb46f80ca644b5738550daef6136ba5476 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 17:23:20 -0700
Subject: [PATCH 207/960] Improve flatbuffer verification.

PiperOrigin-RevId: 189668634
---
 tensorflow/contrib/lite/toco/tflite/BUILD     |   2 +
 tensorflow/contrib/lite/toco/tflite/import.cc |   7 +-
 .../contrib/lite/toco/tflite/import_test.cc   | 106 +++++++++++++++---
 tensorflow/contrib/lite/tools/verifier.cc     |  71 ++++++++++--
 tensorflow/contrib/lite/tools/verifier.h      |  15 +++
 .../contrib/lite/tools/verifier_test.cc       |   4 +-
 6 files changed, 175 insertions(+), 30 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/tflite/BUILD b/tensorflow/contrib/lite/toco/tflite/BUILD
index a2b8145a67..9d3e1daf12 100644
--- a/tensorflow/contrib/lite/toco/tflite/BUILD
+++ b/tensorflow/contrib/lite/toco/tflite/BUILD
@@ -115,9 +115,11 @@ cc_library(
     deps = [
         ":operator",
         ":types",
+        "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite/schema:schema_fbs",
         "//tensorflow/contrib/lite/toco:model",
         "//tensorflow/contrib/lite/toco:tooling_util",
+        "//tensorflow/contrib/lite/tools:verifier",
         "@flatbuffers",
     ],
 )
diff --git a/tensorflow/contrib/lite/toco/tflite/import.cc b/tensorflow/contrib/lite/toco/tflite/import.cc
index 867395e881..c0e7ab2ef5 100644
--- a/tensorflow/contrib/lite/toco/tflite/import.cc
+++ b/tensorflow/contrib/lite/toco/tflite/import.cc
@@ -15,10 +15,12 @@ limitations under the License.
 #include "tensorflow/contrib/lite/toco/tflite/import.h"
 
 #include "flatbuffers/flexbuffers.h"
+#include "tensorflow/contrib/lite/model.h"
 #include "tensorflow/contrib/lite/schema/schema_generated.h"
 #include "tensorflow/contrib/lite/toco/tflite/operator.h"
 #include "tensorflow/contrib/lite/toco/tflite/types.h"
 #include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/contrib/lite/tools/verifier.h"
 
 namespace toco {
 
@@ -171,10 +173,11 @@ bool Verify(const void* buf, size_t len) {
 
 std::unique_ptr<Model> Import(const ModelFlags& model_flags,
                               const string& input_file_contents) {
-  if (!Verify(input_file_contents.data(), input_file_contents.size())) {
+  ::tflite::AlwaysTrueResolver r;
+  if (!::tflite::Verify(input_file_contents.data(), input_file_contents.size(),
+                        r, ::tflite::DefaultErrorReporter())) {
     LOG(FATAL) << "Invalid flatbuffer.";
   }
-
   const ::tflite::Model* input_model =
       ::tflite::GetModel(input_file_contents.data());
 
diff --git a/tensorflow/contrib/lite/toco/tflite/import_test.cc b/tensorflow/contrib/lite/toco/tflite/import_test.cc
index 937a291cf7..edd22f783f 100644
--- a/tensorflow/contrib/lite/toco/tflite/import_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/import_test.cc
@@ -36,12 +36,13 @@ class ImportTest : public ::testing::Test {
     return builder_.CreateVector(reinterpret_cast<const uint8_t*>(data.data()),
                                  sizeof(T) * data.size());
   }
+
   Offset<Vector<Offset<::tflite::Buffer>>> BuildBuffers() {
     auto buf0 = ::tflite::CreateBuffer(builder_, CreateDataVector<float>({}));
-    auto buf1 =
-        ::tflite::CreateBuffer(builder_, CreateDataVector<float>({1.0f, 2.0f}));
+    auto buf1 = ::tflite::CreateBuffer(
+        builder_, CreateDataVector<float>({1.0f, 2.0f, 3.0f, 4.0f}));
     auto buf2 =
-        ::tflite::CreateBuffer(builder_, CreateDataVector<float>({3.0f}));
+        ::tflite::CreateBuffer(builder_, CreateDataVector<float>({3.0f, 4.0f}));
     return builder_.CreateVector(
         std::vector<Offset<::tflite::Buffer>>({buf0, buf1, buf2}));
   }
@@ -53,10 +54,10 @@ class ImportTest : public ::testing::Test {
         /*max=*/builder_.CreateVector<float>({0.2f}),
         /*scale=*/builder_.CreateVector<float>({0.3f}),
         /*zero_point=*/builder_.CreateVector<int64_t>({100ll}));
-    auto t1 = ::tflite::CreateTensor(builder_,
-                                     builder_.CreateVector<int>({1, 2, 3, 4}),
-                                     ::tflite::TensorType_FLOAT32, 1,
-                                     builder_.CreateString("tensor_one"), q);
+    auto t1 =
+        ::tflite::CreateTensor(builder_, builder_.CreateVector<int>({1, 2, 2}),
+                               ::tflite::TensorType_FLOAT32, 1,
+                               builder_.CreateString("tensor_one"), q);
     auto t2 =
         ::tflite::CreateTensor(builder_, builder_.CreateVector<int>({2, 1}),
                                ::tflite::TensorType_FLOAT32, 2,
@@ -65,18 +66,26 @@ class ImportTest : public ::testing::Test {
         std::vector<Offset<::tflite::Tensor>>({t1, t2}));
   }
 
+  Offset<Vector<Offset<::tflite::OperatorCode>>> BuildOpCodes(
+      std::initializer_list<::tflite::BuiltinOperator> op_codes) {
+    std::vector<Offset<::tflite::OperatorCode>> op_codes_vector;
+    for (auto op : op_codes) {
+      op_codes_vector.push_back(::tflite::CreateOperatorCode(builder_, op, 0));
+    }
+    return builder_.CreateVector(op_codes_vector);
+  }
+
   Offset<Vector<Offset<::tflite::OperatorCode>>> BuildOpCodes() {
-    auto c1 = ::tflite::CreateOperatorCode(
-        builder_, ::tflite::BuiltinOperator_MAX_POOL_2D, 0);
-    auto c2 = ::tflite::CreateOperatorCode(
-        builder_, ::tflite::BuiltinOperator_CONV_2D, 0);
-    return builder_.CreateVector(
-        std::vector<Offset<::tflite::OperatorCode>>({c1, c2}));
+    return BuildOpCodes({::tflite::BuiltinOperator_MAX_POOL_2D,
+                         ::tflite::BuiltinOperator_CONV_2D});
   }
 
-  Offset<Vector<Offset<::tflite::Operator>>> BuildOperators() {
-    auto is = builder_.CreateVector<int>({0});
-    auto os = builder_.CreateVector<int>({1});
+  Offset<Vector<Offset<::tflite::Operator>>> BuildOperators(
+      std::initializer_list<int> inputs, std::initializer_list<int> outputs) {
+    auto is = builder_.CreateVector<int>(inputs);
+    if (inputs.size() == 0) is = 0;
+    auto os = builder_.CreateVector<int>(outputs);
+    if (outputs.size() == 0) os = 0;
     auto op = ::tflite::CreateOperator(
         builder_, 0, is, os, ::tflite::BuiltinOptions_Conv2DOptions,
         ::tflite::CreateConv2DOptions(builder_, ::tflite::Padding_VALID, 1, 1,
@@ -87,6 +96,10 @@ class ImportTest : public ::testing::Test {
     return builder_.CreateVector(std::vector<Offset<::tflite::Operator>>({op}));
   }
 
+  Offset<Vector<Offset<::tflite::Operator>>> BuildOperators() {
+    return BuildOperators({0}, {1});
+  }
+
   Offset<Vector<Offset<::tflite::SubGraph>>> BuildSubGraphs(
       Offset<Vector<Offset<::tflite::Tensor>>> tensors,
       Offset<Vector<Offset<::tflite::Operator>>> operators,
@@ -154,9 +167,9 @@ TEST_F(ImportTest, Tensors) {
   Array& a1 = model->GetArray("tensor_one");
   EXPECT_EQ(ArrayDataType::kFloat, a1.data_type);
   EXPECT_THAT(a1.GetBuffer<ArrayDataType::kFloat>().data,
-              ElementsAre(1.0f, 2.0f));
+              ElementsAre(1.0f, 2.0f, 3.0f, 4.0f));
   ASSERT_TRUE(a1.has_shape());
-  EXPECT_THAT(a1.shape().dims(), ElementsAre(1, 2, 3, 4));
+  EXPECT_THAT(a1.shape().dims(), ElementsAre(1, 2, 2));
 
   const auto& mm = a1.minmax;
   ASSERT_TRUE(mm.get());
@@ -169,6 +182,63 @@ TEST_F(ImportTest, Tensors) {
   EXPECT_EQ(100, q->zero_point);
 }
 
+TEST_F(ImportTest, NoBuffers) {
+  auto buffers = 0;
+  auto tensors = BuildTensors();
+  auto opcodes = BuildOpCodes();
+  auto operators = BuildOperators();
+  auto subgraphs = BuildSubGraphs(tensors, operators);
+  auto comment = builder_.CreateString("");
+  ::tflite::FinishModelBuffer(
+      builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
+                                      subgraphs, comment, buffers));
+  EXPECT_DEATH(Import(ModelFlags(), InputModelAsString()),
+               "Missing 'buffers' section.");
+}
+
+TEST_F(ImportTest, NoInputs) {
+  auto buffers = BuildBuffers();
+  auto tensors = BuildTensors();
+  auto opcodes = BuildOpCodes();
+  auto operators = BuildOperators({}, {1});
+  auto subgraphs = BuildSubGraphs(tensors, operators);
+  auto comment = builder_.CreateString("");
+  ::tflite::FinishModelBuffer(
+      builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
+                                      subgraphs, comment, buffers));
+  EXPECT_DEATH(Import(ModelFlags(), InputModelAsString()),
+               "Missing 'inputs' for operator.");
+}
+
+TEST_F(ImportTest, NoOutputs) {
+  auto buffers = BuildBuffers();
+  auto tensors = BuildTensors();
+  auto opcodes = BuildOpCodes();
+  auto operators = BuildOperators({0}, {});
+  auto subgraphs = BuildSubGraphs(tensors, operators);
+  auto comment = builder_.CreateString("");
+  ::tflite::FinishModelBuffer(
+      builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
+                                      subgraphs, comment, buffers));
+  EXPECT_DEATH(Import(ModelFlags(), InputModelAsString()),
+               "Missing 'outputs' for operator.");
+}
+
+TEST_F(ImportTest, InvalidOpCode) {
+  auto buffers = BuildBuffers();
+  auto tensors = BuildTensors();
+  auto opcodes = BuildOpCodes({static_cast<::tflite::BuiltinOperator>(-1),
+                               ::tflite::BuiltinOperator_CONV_2D});
+  auto operators = BuildOperators();
+  auto subgraphs = BuildSubGraphs(tensors, operators);
+  auto comment = builder_.CreateString("");
+  ::tflite::FinishModelBuffer(
+      builder_, ::tflite::CreateModel(builder_, TFLITE_SCHEMA_VERSION, opcodes,
+                                      subgraphs, comment, buffers));
+  EXPECT_DEATH(Import(ModelFlags(), InputModelAsString()),
+               "Operator id '-1' is out of range.");
+}
+
 TEST_F(ImportTest, MultipleSubGraphs) {
   auto buffers = BuildBuffers();
   auto tensors = BuildTensors();
diff --git a/tensorflow/contrib/lite/tools/verifier.cc b/tensorflow/contrib/lite/tools/verifier.cc
index 59c74205f0..8818a7dc85 100644
--- a/tensorflow/contrib/lite/tools/verifier.cc
+++ b/tensorflow/contrib/lite/tools/verifier.cc
@@ -148,11 +148,52 @@ bool VerifyNumericTensorBuffer(const Tensor& tensor, const Buffer& buffer,
   // TODO(yichengfan): verify quantized tensors.
 }
 
+using flatbuffers::Offset;
+using flatbuffers::Vector;
+
+bool VerifyOperators(const Vector<Offset<Operator>>& operators,
+                     ErrorReporter* error_reporter) {
+  for (const auto& op : operators) {
+    if (!op->inputs()) {
+      ReportError(error_reporter, "Missing 'inputs' for operator.");
+      return false;
+    }
+    if (!op->outputs()) {
+      ReportError(error_reporter, "Missing 'outputs' for operator.");
+      return false;
+    }
+  }
+  return true;
+}
+
+bool VerifySubGraphs(const Model& model, ErrorReporter* error_reporter) {
+  if (!model.subgraphs()) {
+    ReportError(error_reporter, "Missing 'subgraphs' section.");
+    return false;
+  }
+  for (const auto& subgraph : *model.subgraphs()) {
+    if (!subgraph->operators()) {
+      ReportError(error_reporter, "Missing 'operators' section in subgraph.");
+      return false;
+    }
+
+    if (!VerifyOperators(*subgraph->operators(), error_reporter)) {
+      return false;
+    }
+  }
+  return true;
+}
+
 // Verifies tensors have valid properties and legit buffer if set.
 bool VerifyTensors(const Model& model, ErrorReporter* error_reporter) {
   if (!model.subgraphs()) {
     return true;
   }
+  if (!model.buffers()) {
+    ReportError(error_reporter, "Missing 'buffers' section.");
+    return false;
+  }
+
   for (const auto& subgraph : *model.subgraphs()) {
     if (!subgraph->tensors()) {
       continue;
@@ -167,19 +208,23 @@ bool VerifyTensors(const Model& model, ErrorReporter* error_reporter) {
         return false;
       }
       auto* buffer = model.buffers()->Get(tensor->buffer());
-      if (!buffer || !buffer->data()) {
+      if (!buffer) {
         ReportError(error_reporter, "Tensor buffer %d not set",
                     tensor->buffer());
         return false;
       }
 
-      if (tensor->type() == TensorType_STRING) {
-        if (!VerifyStringTensorBuffer(*buffer, error_reporter)) {
-          return false;
-        }
-      } else {
-        if (!VerifyNumericTensorBuffer(*tensor, *buffer, error_reporter)) {
-          return false;
+      // Many transient tensors don't have data in the flatbuffer. Their
+      // buffers will be allocated by the interpreter at run-time.
+      if (buffer->data()) {
+        if (tensor->type() == TensorType_STRING) {
+          if (!VerifyStringTensorBuffer(*buffer, error_reporter)) {
+            return false;
+          }
+        } else {
+          if (!VerifyNumericTensorBuffer(*tensor, *buffer, error_reporter)) {
+            return false;
+          }
         }
       }
     }
@@ -193,6 +238,13 @@ bool VerifyOps(const Model& model, const OpResolver& resolver,
     return true;
   }
   for (const auto& opcode : *model.operator_codes()) {
+    if (opcode->builtin_code() < BuiltinOperator_MIN ||
+        opcode->builtin_code() > BuiltinOperator_MAX) {
+      ReportError(error_reporter, "Operator id '%d' is out of range.",
+                  opcode->builtin_code());
+      return false;
+    }
+
     if (opcode->builtin_code() == BuiltinOperator_CUSTOM) {
       if (!resolver.FindOp(opcode->custom_code()->c_str())) {
         ReportError(error_reporter, "Unsupported custom op: %s",
@@ -223,6 +275,9 @@ bool Verify(const void* buf, size_t len, const OpResolver& resolver,
     ReportError(error_reporter, "Invalid model version %d", model->version());
     return false;
   }
+  if (!VerifySubGraphs(*model, error_reporter)) {
+    return false;
+  }
   if (!VerifyTensors(*model, error_reporter)) {
     return false;
   }
diff --git a/tensorflow/contrib/lite/tools/verifier.h b/tensorflow/contrib/lite/tools/verifier.h
index c2ee11215c..b7ce4e8305 100644
--- a/tensorflow/contrib/lite/tools/verifier.h
+++ b/tensorflow/contrib/lite/tools/verifier.h
@@ -23,6 +23,21 @@ limitations under the License.
 
 namespace tflite {
 
+class AlwaysTrueResolver : public OpResolver {
+ public:
+  AlwaysTrueResolver() {}
+  TfLiteRegistration* FindOp(tflite::BuiltinOperator op) const override {
+    static TfLiteRegistration null_registration = {nullptr, nullptr, nullptr,
+                                                   nullptr};
+    return &null_registration;
+  }
+  TfLiteRegistration* FindOp(const char* op) const override {
+    static TfLiteRegistration null_registration = {nullptr, nullptr, nullptr,
+                                                   nullptr};
+    return &null_registration;
+  }
+};
+
 // Verifies the integrity of a Tensorflow Lite flatbuffer model file.
 // Currently, it verifies:
 // * The file is following a legit flatbuffer schema.
diff --git a/tensorflow/contrib/lite/tools/verifier_test.cc b/tensorflow/contrib/lite/tools/verifier_test.cc
index b3e611f999..03b93afe3e 100644
--- a/tensorflow/contrib/lite/tools/verifier_test.cc
+++ b/tensorflow/contrib/lite/tools/verifier_test.cc
@@ -113,8 +113,8 @@ TEST(VerifyModel, TestEmptyModel) {
                            /*description=*/0, /*buffers=*/0);
   ::tflite::FinishModelBuffer(builder, model);
 
-  ASSERT_TRUE(Verify(builder.GetBufferPointer(), builder.GetSize(),
-                     MutableOpResolver{}, DefaultErrorReporter()));
+  ASSERT_FALSE(Verify(builder.GetBufferPointer(), builder.GetSize(),
+                      MutableOpResolver{}, DefaultErrorReporter()));
 }
 
 TEST(VerifyModel, TestSimpleModel) {
-- 
GitLab


From b6b4ec642a632af9abaf3ca7a2b1348ab2e94bef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 17:29:19 -0700
Subject: [PATCH 208/960] Add a clif build rule for saved_model.

PiperOrigin-RevId: 189669509
---
 tensorflow/core/BUILD | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index cf29444065..1d283e240d 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -1405,6 +1405,13 @@ tf_pyclif_proto_library(
     visibility = ["//visibility:public"],
 )
 
+tf_pyclif_proto_library(
+    name = "protobuf/device_properties_pyclif",
+    proto_lib = ":protos_all_cc",
+    proto_srcfile = "protobuf/device_properties.proto",
+    visibility = ["//visibility:public"],
+)
+
 tf_pyclif_proto_library(
     name = "protobuf/meta_graph_pyclif",
     proto_lib = ":protos_all_cc",
@@ -1413,9 +1420,9 @@ tf_pyclif_proto_library(
 )
 
 tf_pyclif_proto_library(
-    name = "protobuf/device_properties_pyclif",
+    name = "protobuf/saved_model_pyclif",
     proto_lib = ":protos_all_cc",
-    proto_srcfile = "protobuf/device_properties.proto",
+    proto_srcfile = "protobuf/saved_model.proto",
     visibility = ["//visibility:public"],
 )
 
-- 
GitLab


From 2714c07c93c2fd84480f816e0da44030a0a2bd45 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 19 Mar 2018 17:34:47 -0700
Subject: [PATCH 209/960] Make _USE_C_API = True and_USE_C_SHAPES = False work
 with import_graph_def.

Without this change, shapes wouldn't be correctly computed for
operations created via import_graph_def.

PiperOrigin-RevId: 189670312
---
 tensorflow/python/client/session_test.py      |  3 +-
 tensorflow/python/framework/importer.py       | 34 ++++++++++---------
 tensorflow/python/framework/importer_test.py  |  4 +--
 .../python/framework/meta_graph_test.py       |  3 +-
 tensorflow/python/framework/ops.py            | 29 +++++++++++-----
 tensorflow/python/training/saver_test.py      |  3 +-
 6 files changed, 43 insertions(+), 33 deletions(-)

diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 44ff440cc5..6e2640efd1 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -62,8 +62,7 @@ from tensorflow.python.util import compat
 ops.RegisterShape('ConstructionFails')(common_shapes.unknown_shape)
 
 
-# TODO(skyewm): reenable when this works with _USE_C_SHAPES=False
-# @test_util.with_c_api
+@test_util.with_c_api
 class SessionTest(test_util.TensorFlowTestCase):
 
   def setUp(self):
diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index 783e9259ad..a9e399f59b 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -489,23 +489,25 @@ def import_graph_def(graph_def,
           # Convert to ValueError for backwards compatibility.
           raise ValueError(str(e))
 
-      _ProcessNewOps(graph)
+      # Create _DefinedFunctions for any imported functions.
+      #
+      # We do this by creating _DefinedFunctions directly from `graph_def`, and
+      # adding them to `graph`. Adding an existing function to a TF_Graph is a
+      # no-op, so this only has the effect of updating the Python state (usually
+      # _DefinedFunction.add_to_graph also adds the function to the TF_Graph).
+      #
+      # TODO(skyewm): fetch the TF_Functions directly from the TF_Graph
+      # TODO(skyewm): avoid sending serialized FunctionDefs back to the TF_Graph
+      # TODO(b/74620627): move this after _ProcessNewOps outside the lock once
+      # _USE_C_SHAPES is removed.
+      if graph_def.library and graph_def.library.function:
+        # pylint: disable=protected-access
+        functions = function._from_library(graph_def.library)
+        for f in functions:
+          f.add_to_graph(graph)
+        # pylint: enable=protected-access
 
-    # Create _DefinedFunctions for any imported functions.
-    #
-    # We do this by creating _DefinedFunctions directly from `graph_def`, and
-    # adding them to `graph`. Adding an existing function to a TF_Graph is a
-    # no-op, so this only has the effect of updating the Python state (usually
-    # _DefinedFunction.add_to_graph also adds the function to the TF_Graph).
-    #
-    # TODO(skyewm): fetch the TF_Functions directly from the TF_Graph
-    # TODO(skyewm): avoid sending serialized FunctionDefs back to the TF_Graph
-    if graph_def.library and graph_def.library.function:
-      # pylint: disable=protected-access
-      functions = function._from_library(graph_def.library)
-      for f in functions:
-        f.add_to_graph(graph)
-      # pylint: enable=protected-access
+      _ProcessNewOps(graph)
 
     # Treat input mappings that don't appear in the graph as an error, because
     # they are likely to be due to a typo.
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index c39191e6d9..bf5d9fe093 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import function
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_ops  # pylint: disable=unused-import
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import versions
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -43,8 +44,7 @@ import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
 
-# TODO(skyewm): reenable when this works with _USE_C_SHAPES=False
-# @test_util.with_c_api
+@test_util.with_c_api
 class ImportGraphDefTest(test.TestCase):
 
   def _MakeGraphDef(self,
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 06cec504e4..21963d0bee 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -285,8 +285,7 @@ class SimpleMetaGraphTest(test.TestCase):
       self.assertIs(global_vars[0], trainable_vars[0])
 
 
-# TODO(skyewm): reenable when this works with _USE_C_SHAPES=False
-# @test_util.with_c_api
+@test_util.with_c_api
 class ScopedMetaGraphTest(test.TestCase):
 
   def _testScopedExport(self, test_dir, exported_filenames):
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index f1cd341d66..4be2e2c15d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3303,6 +3303,20 @@ class Graph(object):
           input_types=input_types,
           original_op=self._default_original_op,
           op_def=op_def)
+
+      # TODO(vrv): Instead of eagerly filling in shape property for every op,
+      # only populate the shape when requested.
+      #
+      # TODO(skyewm): unlike in the original Python implementation, the C API
+      # always computes shape information (even for function calls, which the
+      # original Python shape inference code doesn't handle). Deprecate the
+      # compute_shapes argument.
+      #
+      # TODO(b/74620627): move this back to _create_op_helper once _USE_C_SHAPES
+      # is removed
+      if (ret._c_op and _USE_C_SHAPES) or compute_shapes:  # pylint: disable=protected-access
+        set_shapes_for_outputs(ret)
+
       self._create_op_helper(ret, compute_shapes=compute_shapes,
                              compute_device=compute_device)
     return ret
@@ -3336,15 +3350,6 @@ class Graph(object):
 
   def _create_op_helper(self, op, compute_shapes=True, compute_device=True):
     """Common logic for creating an op in this graph."""
-    # TODO(vrv): Instead of eagerly filling in shape property for every op, only
-    # populate the shape when requested.
-    #
-    # TODO(skyewm): unlike in the original Python implementation, the C API
-    # always computes shape information (even for function calls, which the
-    # original Python shape inference code doesn't handle). Deprecate the
-    # compute_shapes argument.
-    if (op._c_op and _USE_C_SHAPES) or compute_shapes:  # pylint: disable=protected-access
-      set_shapes_for_outputs(op)
     # TODO(b/XXXX): move to Operation.__init__ once _USE_C_API flag is removed.
     self._add_op(op)
 
@@ -3449,6 +3454,12 @@ class Graph(object):
     ]
 
     for op in new_ops:
+      # The Python shape inference code does not support imported functions. It
+      # also needs access to op.inputs, which is why we call it here.
+      # TODO(b/74620627): move this back to _create_op_helper once _USE_C_SHAPES
+      # is removed.
+      if not self._is_function(op.type) or _USE_C_SHAPES:
+        set_shapes_for_outputs(op)
       new_control_inputs = self._control_dependencies_for_inputs(op.inputs)
       # pylint: disable=protected-access
       op._add_control_inputs(new_control_inputs)
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 787582ae70..7de778f298 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -1739,8 +1739,7 @@ class CheckpointStateTest(test.TestCase):
                      os.path.join(save_dir, "./model.ckpt-687529"))
 
 
-# TODO(skyewm): reenable when this works with _USE_C_SHAPES=False
-# @test_util.with_c_api
+@test_util.with_c_api
 class MetaGraphTest(test.TestCase):
 
   def _get_test_dir(self, dirname):
-- 
GitLab


From e2c90615a4ab9033a36111299f6a0d4485f4f16a Mon Sep 17 00:00:00 2001
From: Dustin Tran <trandustin@google.com>
Date: Mon, 19 Mar 2018 17:41:25 -0700
Subject: [PATCH 210/960] Standardize bib references and Examples subsection in
 docstrings.

Recipe:

+ Write a #### Examples subsection below Args/Returns/Raises to illustrate examples. If the docstring's last line is a ``` closing a code snippet, add an empty line before closing the docstring with """. This properly displays the code snippet.

+ Write a #### References subsection at the bottom of any docstring with citations. Enumerate all references in alphabetical order. Individual bibentries use ICLR?s bibliography style, which borrows from icml2010.bst and which itself borrows from plainnl.bst. Add a link to the paper if the publication is open source (ideally, arXiv).

PiperOrigin-RevId: 189670932
---
 .../python/ops/autoregressive.py              |  24 +--
 .../python/ops/bijectors/affine.py            |   2 +-
 .../ops/bijectors/batch_normalization.py      |  31 ++--
 .../ops/bijectors/cholesky_outer_product.py   |   2 +-
 .../ops/bijectors/masked_autoregressive.py    |  99 +++++-----
 .../python/ops/bijectors/real_nvp.py          |  82 +++++----
 .../python/ops/bijectors/square.py            |   2 +-
 .../distributions/python/ops/kumaraswamy.py   |  10 +-
 .../distributions/python/ops/moving_stats.py  |  20 ++-
 .../contrib/distributions/python/ops/shape.py | 169 +++++++++---------
 .../python/ops/vector_diffeomixture.py        |  13 +-
 11 files changed, 251 insertions(+), 203 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/ops/autoregressive.py b/tensorflow/contrib/distributions/python/ops/autoregressive.py
index 852298bf33..69f3d57ff0 100644
--- a/tensorflow/contrib/distributions/python/ops/autoregressive.py
+++ b/tensorflow/contrib/distributions/python/ops/autoregressive.py
@@ -36,7 +36,8 @@ class Autoregressive(distribution_lib.Distribution):
     "Autoregressive models decompose the joint density as a product of
     conditionals, and model each conditional in turn. Normalizing flows
     transform a base density (e.g. a standard Gaussian) into the target density
-    by an invertible transformation with tractable Jacobian." [1]
+    by an invertible transformation with tractable Jacobian." [(Papamakarios et
+    al., 2016)][1]
 
   In other words, the "autoregressive property" is equivalent to the
   decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided
@@ -45,17 +46,18 @@ class Autoregressive(distribution_lib.Distribution):
 
   Practically speaking the autoregressive property means that there exists a
   permutation of the event coordinates such that each coordinate is a
-  diffeomorphic function of only preceding coordinates. [2]
+  diffeomorphic function of only preceding coordinates
+  [(van den Oord et al., 2016)][2].
 
   #### Mathematical Details
 
-  The probability function is,
+  The probability function is
 
   ```none
   prob(x; fn, n) = fn(x).prob(x)
   ```
 
-  And a sample is generated by,
+  And a sample is generated by
 
   ```none
   x = fn(...fn(fn(x0).sample()).sample()).sample()
@@ -93,13 +95,15 @@ class Autoregressive(distribution_lib.Distribution):
 
   ```
 
-  [1]: "Masked Autoregressive Flow for Density Estimation."
-       George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017.
-       https://arxiv.org/abs/1705.07057
+  #### References
 
-  [2]: "Conditional Image Generation with PixelCNN Decoders."
-       Aaron van den Oord, Nal Kalchbrenner, Oriol Vinyals, Lasse Espeholt, Alex
-       Graves, Koray Kavukcuoglu. Arxiv, 2016.
+  [1]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked
+       Autoregressive Flow for Density Estimation. In _Neural Information
+       Processing Systems_, 2017. https://arxiv.org/abs/1705.07057
+
+  [2]: Aaron van den Oord, Nal Kalchbrenner, Oriol Vinyals, Lasse Espeholt,
+       Alex Graves, and Koray Kavukcuoglu. Conditional Image Generation with
+       PixelCNN Decoders. In _Neural Information Processing Systems_, 2016.
        https://arxiv.org/abs/1606.05328
   """
 
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py b/tensorflow/contrib/distributions/python/ops/bijectors/affine.py
index 7fe73ada44..bef7bbb49b 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/affine.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/affine.py
@@ -62,7 +62,7 @@ class Affine(bijector.Bijector):
   matrices, i.e., the matmul is [matrix-free](
   https://en.wikipedia.org/wiki/Matrix-free_methods) when possible.
 
-  Examples:
+  #### Examples
 
   ```python
   # Y = X
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py b/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py
index be72ff3081..33fdd32d7a 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/batch_normalization.py
@@ -76,15 +76,16 @@ def _undo_batch_normalization(x,
 class BatchNormalization(bijector.Bijector):
   """Compute `Y = g(X) s.t. X = g^-1(Y) = (Y - mean(Y)) / std(Y)`.
 
-  Applies Batch Normalization [1] to samples from a data distribution. This can
-  be used to stabilize training of normalizing flows [2, 3].
+  Applies Batch Normalization [(Ioffe and Szegedy, 2015)][1] to samples from a
+  data distribution. This can be used to stabilize training of normalizing
+  flows ([Papamakarios et al., 2016][3]; [Dinh et al., 2017][2])
 
   When training Deep Neural Networks (DNNs), it is common practice to
   normalize or whiten features by shifting them to have zero mean and
   scaling them to have unit variance.
 
-  The `inverse()` method of the BatchNorm bijector, which is used in the
-  log-likelihood computation of data samples, implements the normalization
+  The `inverse()` method of the `BatchNormalization` bijector, which is used in
+  the log-likelihood computation of data samples, implements the normalization
   procedure (shift-and-scale) using the mean and standard deviation of the
   current minibatch.
 
@@ -92,7 +93,6 @@ class BatchNormalization(bijector.Bijector):
   `X*std(Y) + mean(Y)` with the running-average mean and standard deviation
   computed at training-time. De-normalization is useful for sampling.
 
-
   ```python
 
   dist = tfd.TransformedDistribution(
@@ -112,19 +112,20 @@ class BatchNormalization(bijector.Bijector):
   `BatchNorm.forward(BatchNorm.inverse(...))` will be identical when
   `training=False` but may be different when `training=True`.
 
-  [1]: "Batch Normalization: Accelerating Deep Network Training by Reducing
-       Internal Covariate Shift."
-       Sergey Ioffe, Christian Szegedy. Arxiv. 2015.
-       https://arxiv.org/abs/1502.03167
+  #### References
 
-  [2]: "Density Estimation using Real NVP."
-     Laurent Dinh, Jascha Sohl-Dickstein, Samy Bengio. ICLR. 2017.
-     https://arxiv.org/abs/1605.08803
+  [1]: Sergey Ioffe and Christian Szegedy. Batch Normalization: Accelerating
+       Deep Network Training by Reducing Internal Covariate Shift. In
+       _International Conference on Machine Learning_, 2015.
+       https://arxiv.org/abs/1502.03167
 
-  [3]: "Masked Autoregressive Flow for Density Estimation."
-       George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017.
-       https://arxiv.org/abs/1705.07057
+  [2]: Laurent Dinh, Jascha Sohl-Dickstein, and Samy Bengio. Density Estimation
+       using Real NVP. In _International Conference on Learning
+       Representations_, 2017. https://arxiv.org/abs/1605.08803
 
+  [3]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked
+       Autoregressive Flow for Density Estimation. In _Neural Information
+       Processing Systems_, 2017. https://arxiv.org/abs/1705.07057
   """
 
   def __init__(self,
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
index 43208ff088..8f09e16058 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/cholesky_outer_product.py
@@ -57,7 +57,7 @@ class CholeskyOuterProduct(bijector.Bijector):
   that, if `I = L_3 @ L_3.T`, with L_3 being lower-triangular with positive-
   diagonal, then `L_3 = I`. Thus, `L_1 = L_2`, proving injectivity of g.
 
-  Examples:
+  #### Examples
 
   ```python
   bijector.CholeskyOuterProduct().forward(x=[[1., 0], [2, 1]])
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
index 5251dbcb57..84b2340c75 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/masked_autoregressive.py
@@ -45,14 +45,15 @@ __all__ = [
 class MaskedAutoregressiveFlow(bijector_lib.Bijector):
   """Affine MaskedAutoregressiveFlow bijector for vector-valued events.
 
-  The affine autoregressive flow [1] provides a relatively simple framework for
-  user-specified (deep) architectures to learn a distribution over vector-valued
-  events. Regarding terminology,
+  The affine autoregressive flow [(Papamakarios et al., 2016)][3] provides a
+  relatively simple framework for user-specified (deep) architectures to learn
+  a distribution over vector-valued events. Regarding terminology,
 
     "Autoregressive models decompose the joint density as a product of
     conditionals, and model each conditional in turn. Normalizing flows
     transform a base density (e.g. a standard Gaussian) into the target density
-    by an invertible transformation with tractable Jacobian." [1]
+    by an invertible transformation with tractable Jacobian."
+    [(Papamakarios et al., 2016)][3]
 
   In other words, the "autoregressive property" is equivalent to the
   decomposition, `p(x) = prod{ p(x[i] | x[0:i]) : i=0, ..., d }`. The provided
@@ -75,26 +76,26 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector):
 
   Given a `shift_and_log_scale_fn`, the forward and inverse transformations are
   (a sequence of) affine transformations. A "valid" `shift_and_log_scale_fn`
-  must compute each `shift` (aka `loc` or "mu" [2]) and `log(scale)` (aka
-  "alpha" [2]) such that each are broadcastable with the arguments to `forward`
-  and `inverse`, i.e., such that the calculations in `forward`, `inverse`
-  [below] are possible.
+  must compute each `shift` (aka `loc` or "mu" in [Germain et al. (2015)][1])
+  and `log(scale)` (aka "alpha" in [Germain et al. (2015)][1]) such that each
+  are broadcastable with the arguments to `forward` and `inverse`, i.e., such
+  that the calculations in `forward`, `inverse` [below] are possible.
 
   For convenience, `masked_autoregressive_default_template` is offered as a
   possible `shift_and_log_scale_fn` function. It implements the MADE
-  architecture [2]. MADE is a feed-forward network that computes a `shift` and
-  `log(scale)` using `masked_dense` layers in a deep neural network. Weights are
-  masked to ensure the autoregressive property. It is possible that this
-  architecture is suboptimal for your task. To build alternative networks,
-  either change the arguments to `masked_autoregressive_default_template`, use
-  the `masked_dense` function to roll-out your own, or use some other
-  architecture, e.g., using `tf.layers`.
+  architecture [(Germain et al., 2015)][1]. MADE is a feed-forward network that
+  computes a `shift` and `log(scale)` using `masked_dense` layers in a deep
+  neural network. Weights are masked to ensure the autoregressive property. It
+  is possible that this architecture is suboptimal for your task. To build
+  alternative networks, either change the arguments to
+  `masked_autoregressive_default_template`, use the `masked_dense` function to
+  roll-out your own, or use some other architecture, e.g., using `tf.layers`.
 
   Warning: no attempt is made to validate that the `shift_and_log_scale_fn`
   enforces the "autoregressive property".
 
   Assuming `shift_and_log_scale_fn` has valid shape and autoregressive
-  semantics, the forward transformation is,
+  semantics, the forward transformation is
 
   ```python
   def forward(x):
@@ -106,7 +107,7 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector):
     return y
   ```
 
-  and the inverse transformation is,
+  and the inverse transformation is
 
   ```python
   def inverse(y):
@@ -121,7 +122,7 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector):
   the "last" `y` used to compute `shift`, `log_scale`. (Roughly speaking, this
   also proves the transform is bijective.)
 
-  #### Example Use
+  #### Examples
 
   ```python
   tfd = tf.contrib.distributions
@@ -142,7 +143,8 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector):
   maf.log_prob(x)   # Almost free; uses Bijector caching.
   maf.log_prob(0.)  # Cheap; no `tf.while_loop` despite no Bijector caching.
 
-  # [1] also describes an "Inverse Autoregressive Flow", e.g.,
+  # [Papamakarios et al. (2016)][3] also describe an Inverse Autoregressive
+  # Flow [(Kingma et al., 2016)][2]:
   iaf = tfd.TransformedDistribution(
       distribution=tfd.Normal(loc=0., scale=1.),
       bijector=tfb.Invert(tfb.MaskedAutoregressiveFlow(
@@ -168,14 +170,20 @@ class MaskedAutoregressiveFlow(bijector_lib.Bijector):
       event_shape=[dims])
   ```
 
-  [1]: "Masked Autoregressive Flow for Density Estimation."
-       George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017.
-       https://arxiv.org/abs/1705.07057
+  #### References
 
-  [2]: "MADE: Masked Autoencoder for Distribution Estimation."
-       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
-       https://arxiv.org/abs/1502.03509
+  [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE:
+       Masked Autoencoder for Distribution Estimation. In _International
+       Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509
 
+  [2]: Diederik P. Kingma, Tim Salimans, Rafal Jozefowicz, Xi Chen, Ilya
+       Sutskever, and Max Welling. Improving Variational Inference with Inverse
+       Autoregressive Flow. In _Neural Information Processing Systems_, 2016.
+       https://arxiv.org/abs/1606.04934
+
+  [3]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked
+       Autoregressive Flow for Density Estimation. In _Neural Information
+       Processing Systems_, 2017. https://arxiv.org/abs/1705.07057
   """
 
   def __init__(self,
@@ -329,11 +337,7 @@ def masked_dense(inputs,
                  **kwargs):
   """A autoregressively masked dense layer. Analogous to `tf.layers.dense`.
 
-  See [1] for detailed explanation.
-
-  [1]: "MADE: Masked Autoencoder for Distribution Estimation."
-       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
-       https://arxiv.org/abs/1502.03509
+  See [Germain et al. (2015)][1] for detailed explanation.
 
   Arguments:
     inputs: Tensor input.
@@ -358,6 +362,12 @@ def masked_dense(inputs,
   Raises:
     NotImplementedError: if rightmost dimension of `inputs` is unknown prior to
       graph execution.
+
+  #### References
+
+  [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE:
+       Masked Autoencoder for Distribution Estimation. In _International
+       Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509
   """
   # TODO(b/67594795): Better support of dynamic shape.
   input_depth = inputs.shape.with_rank_at_least(1)[-1].value
@@ -398,23 +408,24 @@ def masked_autoregressive_default_template(
     name=None,
     *args,
     **kwargs):
-  """Build the MADE Model [1].
+  """Build the Masked Autoregressive Density Estimator (Germain et al., 2015).
 
   This will be wrapped in a make_template to ensure the variables are only
-  created once. It takes the input and returns the `loc` ("mu" [1]) and
-  `log_scale` ("alpha" [1]) from the MADE network.
+  created once. It takes the input and returns the `loc` ("mu" in [Germain et
+  al. (2015)][1]) and `log_scale` ("alpha" in [Germain et al. (2015)][1]) from
+  the MADE network.
 
   Warning: This function uses `masked_dense` to create randomly initialized
   `tf.Variables`. It is presumed that these will be fit, just as you would any
   other neural architecture which uses `tf.layers.dense`.
 
-  #### About Hidden Layers:
+  #### About Hidden Layers
 
   Each element of `hidden_layers` should be greater than the `input_depth`
   (i.e., `input_depth = tf.shape(input)[-1]` where `input` is the input to the
   neural network). This is necessary to ensure the autoregressivity property.
 
-  #### About Clipping:
+  #### About Clipping
 
   This function also optionally clips the `log_scale` (but possibly not its
   gradient). This is useful because if `log_scale` is too small/large it might
@@ -427,11 +438,7 @@ def masked_autoregressive_default_template(
   `grad[exp(clip(x))] = grad[x] exp(clip(x))` rather than the usual
   `grad[clip(x)] exp(clip(x))`.
 
-  [1]: "MADE: Masked Autoencoder for Distribution Estimation."
-       Mathieu Germain, Karol Gregor, Iain Murray, Hugo Larochelle. ICML. 2015.
-       https://arxiv.org/abs/1502.03509
-
-  Arguments:
+  Args:
     hidden_layers: Python `list`-like of non-negative integer, scalars
       indicating the number of units in each hidden layer. Default: `[512, 512].
     shift_only: Python `bool` indicating if only the `shift` term shall be
@@ -450,12 +457,20 @@ def masked_autoregressive_default_template(
     **kwargs: `tf.layers.dense` keyword arguments.
 
   Returns:
-    shift: `Float`-like `Tensor` of shift terms (the "mu" in [2]).
-    log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in [2]).
+    shift: `Float`-like `Tensor` of shift terms (the "mu" in
+      [Germain et al.  (2015)][1]).
+    log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in
+      [Germain et al. (2015)][1]).
 
   Raises:
     NotImplementedError: if rightmost dimension of `inputs` is unknown prior to
       graph execution.
+
+  #### References
+
+  [1]: Mathieu Germain, Karol Gregor, Iain Murray, and Hugo Larochelle. MADE:
+       Masked Autoencoder for Distribution Estimation. In _International
+       Conference on Machine Learning_, 2015. https://arxiv.org/abs/1502.03509
   """
 
   with ops.name_scope(name, "masked_autoregressive_default_template",
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py b/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py
index 2840f52e74..71ab369d01 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/real_nvp.py
@@ -38,7 +38,7 @@ class RealNVP(bijector_lib.Bijector):
   """RealNVP "affine coupling layer" for vector-valued events.
 
   Real NVP models a normalizing flow on a `D`-dimensional distribution via a
-  single `D-d`-dimensional conditional distribution [1]:
+  single `D-d`-dimensional conditional distribution [(Dinh et al., 2017)][1]:
 
   `y[d:D] = y[d:D] * math_ops.exp(log_scale_fn(y[d:D])) + shift_fn(y[d:D])`
   `y[0:d] = x[0:d]`
@@ -51,31 +51,34 @@ class RealNVP(bijector_lib.Bijector):
 
   Masking is currently only supported for base distributions with
   `event_ndims=1`. For more sophisticated masking schemes like checkerboard or
-  channel-wise masking [2], use the `tfb.Permute` bijector to re-order desired
-  masked units into the first `d` units. For base distributions with
-  `event_ndims > 1`, use the `tfb.Reshape` bijector to flatten the event shape.
-
-  Recall that the MAF bijector [2] implements a normalizing flow via an
-  autoregressive transformation. MAF and IAF have opposite computational
-  tradeoffs - MAF can train all units in parallel but must sample units
-  sequentially, while IAF must train units sequentially but can sample in
-  parallel. In contrast, Real NVP can compute both forward and inverse
-  computations in parallel. However, the lack of an autoregressive
+  channel-wise masking [(Papamakarios et al., 2016)[4], use the `tfb.Permute`
+  bijector to re-order desired masked units into the first `d` units. For base
+  distributions with `event_ndims > 1`, use the `tfb.Reshape` bijector to
+  flatten the event shape.
+
+  Recall that the MAF bijector [(Papamakarios et al., 2016)][4] implements a
+  normalizing flow via an autoregressive transformation. MAF and IAF have
+  opposite computational tradeoffs - MAF can train all units in parallel but
+  must sample units sequentially, while IAF must train units sequentially but
+  can sample in parallel. In contrast, Real NVP can compute both forward and
+  inverse computations in parallel. However, the lack of an autoregressive
   transformations makes it less expressive on a per-bijector basis.
 
   A "valid" `shift_and_log_scale_fn` must compute each `shift` (aka `loc` or
-  "mu" [2]) and `log(scale)` (aka "alpha" [2]) such that each are broadcastable
-  with the arguments to `forward` and `inverse`, i.e., such that the
-  calculations in `forward`, `inverse` [below] are possible. For convenience,
+  "mu" in [Papamakarios et al. (2016)][4]) and `log(scale)` (aka "alpha" in
+  [Papamakarios et al. (2016)][4]) such that each are broadcastable with the
+  arguments to `forward` and `inverse`, i.e., such that the calculations in
+  `forward`, `inverse` [below] are possible. For convenience,
   `real_nvp_default_nvp` is offered as a possible `shift_and_log_scale_fn`
   function.
 
-  NICE [3] is a special case of the Real NVP bijector which discards the scale
-  transformation, resulting in a constant-time inverse-log-determinant-Jacobian.
-  To use a NICE bijector instead of Real NVP, `shift_and_log_scale_fn` should
-  return `(shift, None)`, and `is_constant_jacobian` should be set to `True` in
-  the `RealNVP` constructor. Calling `real_nvp_default_template` with
-  `shift_only=True` returns one such NICE-compatible `shift_and_log_scale_fn`.
+  NICE [(Dinh et al., 2014)][2] is a special case of the Real NVP bijector
+  which discards the scale transformation, resulting in a constant-time
+  inverse-log-determinant-Jacobian. To use a NICE bijector instead of Real
+  NVP, `shift_and_log_scale_fn` should return `(shift, None)`, and
+  `is_constant_jacobian` should be set to `True` in the `RealNVP` constructor.
+  Calling `real_nvp_default_template` with `shift_only=True` returns one such
+  NICE-compatible `shift_and_log_scale_fn`.
 
   Caching: the scalar input depth `D` of the base distribution is not known at
   construction time. The first call to any of `forward(x)`, `inverse(x)`,
@@ -103,23 +106,24 @@ class RealNVP(bijector_lib.Bijector):
   nvp.log_prob(0.)
   ```
 
-  For more examples, see [4].
+  For more examples, see [Jang (2018)][3].
 
-  [1]: "Density Estimation using Real NVP."
-       Laurent Dinh, Jascha Sohl-Dickstein, Samy Bengio. ICLR. 2017.
-       https://arxiv.org/abs/1605.08803
+  #### References
 
-  [2]: "Masked Autoregressive Flow for Density Estimation."
-       George Papamakarios, Theo Pavlakou, Iain Murray. Arxiv. 2017.
-       https://arxiv.org/abs/1705.07057
+  [1]: Laurent Dinh, Jascha Sohl-Dickstein, and Samy Bengio. Density Estimation
+       using Real NVP. In _International Conference on Learning
+       Representations_, 2017. https://arxiv.org/abs/1605.08803
 
-  [3]: "NICE: Non-linear Independent Components Estimation."
-       Laurent Dinh, David Krueger, Yoshua Bengio. ICLR. 2015.
-       https://arxiv.org/abs/1410.8516
+  [2]: Laurent Dinh, David Krueger, and Yoshua Bengio. NICE: Non-linear
+       Independent Components Estimation. _arXiv preprint arXiv:1410.8516_,
+       2014. https://arxiv.org/abs/1410.8516
 
-  [4]: "Normalizing Flows Tutorial, Part 2: Modern Normalizing Flows."
-       Eric Jang. Blog post. January 2018.
-       http://blog.evjang.com/2018/01/nf2.html
+  [3]: Eric Jang. Normalizing Flows Tutorial, Part 2: Modern Normalizing Flows.
+       _Technical Report_, 2018. http://blog.evjang.com/2018/01/nf2.html
+
+  [4]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked
+       Autoregressive Flow for Density Estimation. In _Neural Information
+       Processing Systems_, 2017. https://arxiv.org/abs/1705.07057
   """
 
   def __init__(self,
@@ -250,12 +254,20 @@ def real_nvp_default_template(
     **kwargs: `tf.layers.dense` keyword arguments.
 
   Returns:
-    shift: `Float`-like `Tensor` of shift terms (the "mu" in [2]).
-    log_scale: `Float`-like `Tensor` of log(scale) terms (the "alpha" in [2]).
+    shift: `Float`-like `Tensor` of shift terms ("mu" in
+      [Papamakarios et al.  (2016)][1]).
+    log_scale: `Float`-like `Tensor` of log(scale) terms ("alpha" in
+      [Papamakarios et al. (2016)][1]).
 
   Raises:
     NotImplementedError: if rightmost dimension of `inputs` is unknown prior to
       graph execution.
+
+  #### References
+
+  [1]: George Papamakarios, Theo Pavlakou, and Iain Murray. Masked
+       Autoregressive Flow for Density Estimation. In _Neural Information
+       Processing Systems_, 2017. https://arxiv.org/abs/1705.07057
   """
 
   with ops.name_scope(name, "real_nvp_default_template"):
diff --git a/tensorflow/contrib/distributions/python/ops/bijectors/square.py b/tensorflow/contrib/distributions/python/ops/bijectors/square.py
index 2831a92df8..1e9dbf3509 100644
--- a/tensorflow/contrib/distributions/python/ops/bijectors/square.py
+++ b/tensorflow/contrib/distributions/python/ops/bijectors/square.py
@@ -37,7 +37,7 @@ class Square(bijector.Bijector):
   g is a bijection between the non-negative real numbers (R_+) and the
   non-negative real numbers.
 
-  Examples:
+  #### Examples
 
   ```python
   bijector.Square().forward(x=[[1., 0], [2, 1]])
diff --git a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
index 120b38db3c..192dede6ff 100644
--- a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
+++ b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
@@ -44,18 +44,16 @@ _kumaraswamy_sample_note = """Note: `x` must have dtype `self.dtype` and be in
 def _harmonic_number(x):
   """Compute the harmonic number from its analytic continuation.
 
-  Derivation from [1] and Euler's constant [2].
-  [1] -
-  https://en.wikipedia.org/wiki/Digamma_function#Relation_to_harmonic_numbers
-  [2] - https://en.wikipedia.org/wiki/Euler%E2%80%93Mascheroni_constant
-
+  Derivation from [here](
+  https://en.wikipedia.org/wiki/Digamma_function#Relation_to_harmonic_numbers)
+  and [Euler's constant](
+  https://en.wikipedia.org/wiki/Euler%E2%80%93Mascheroni_constant).
 
   Args:
     x: input float.
 
   Returns:
     z: The analytic continuation of the harmonic number for the input.
-
   """
   one = array_ops.ones([], dtype=x.dtype)
   return math_ops.digamma(x + one) - math_ops.digamma(one)
diff --git a/tensorflow/contrib/distributions/python/ops/moving_stats.py b/tensorflow/contrib/distributions/python/ops/moving_stats.py
index 20f85643b9..87d40805a3 100644
--- a/tensorflow/contrib/distributions/python/ops/moving_stats.py
+++ b/tensorflow/contrib/distributions/python/ops/moving_stats.py
@@ -47,9 +47,7 @@ def assign_moving_mean_variance(
   Note: `mean_var` is updated *after* `variance_var`, i.e., `variance_var` uses
   the lag-1 mean.
 
-  For derivation justification, see equation 143 of:
-    T. Finch, Feb 2009. "Incremental calculation of weighted mean and variance".
-    http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf
+  For derivation justification, see [Finch (2009; Eq. 143)][1].
 
   Args:
     mean_var: `float`-like `Variable` representing the exponentially weighted
@@ -72,6 +70,12 @@ def assign_moving_mean_variance(
     TypeError: if `mean_var` does not have float type `dtype`.
     TypeError: if `mean_var`, `variance_var`, `value`, `decay` have different
       `base_dtype`.
+
+  #### References
+
+  [1]: Tony Finch. Incremental calculation of weighted mean and variance.
+       _Technical Report_, 2009.
+       http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf
   """
   with ops.name_scope(name, "assign_moving_mean_variance",
                       [variance_var, mean_var, value, decay]):
@@ -183,9 +187,7 @@ def moving_mean_variance(value, decay, collections=None, name=None):
   Note: `mean_var` is updated *after* `variance_var`, i.e., `variance_var` uses
   the lag-`1` mean.
 
-  For derivation justification, see equation 143 of:
-    T. Finch, Feb 2009. "Incremental calculation of weighted mean and variance".
-    http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf
+  For derivation justification, see [Finch (2009; Eq. 143)][1].
 
   Unlike `assign_moving_mean_variance`, this function handles
   variable creation.
@@ -208,6 +210,12 @@ def moving_mean_variance(value, decay, collections=None, name=None):
   Raises:
     TypeError: if `value_var` does not have float type `dtype`.
     TypeError: if `value`, `decay` have different `base_dtype`.
+
+  #### References
+
+  [1]: Tony Finch. Incremental calculation of weighted mean and variance.
+       _Technical Report_, 2009.
+       http://people.ds.cam.ac.uk/fanf2/hermes/doc/antiforgery/stats.pdf
   """
   if collections is None:
     collections = [ops.GraphKeys.GLOBAL_VARIABLES]
diff --git a/tensorflow/contrib/distributions/python/ops/shape.py b/tensorflow/contrib/distributions/python/ops/shape.py
index 5fb6f0c7ea..bac0b79d59 100644
--- a/tensorflow/contrib/distributions/python/ops/shape.py
+++ b/tensorflow/contrib/distributions/python/ops/shape.py
@@ -32,45 +32,50 @@ from tensorflow.python.ops.distributions import util as distribution_util
 class _DistributionShape(object):
   """Manage and manipulate `Distribution` shape.
 
-  Terminology:
-    Recall that a `Tensor` has:
-      - `shape`: size of `Tensor` dimensions,
-      - `ndims`: size of `shape`; number of `Tensor` dimensions,
-      - `dims`: indexes into `shape`; useful for transpose, reduce.
-
-    `Tensor`s sampled from a `Distribution` can be partitioned by `sample_dims`,
-    `batch_dims`, and `event_dims`. To understand the semantics of these
-    dimensions, consider when two of the three are fixed and the remaining
-    is varied:
-      - `sample_dims`: indexes independent draws from identical
-                       parameterizations of the `Distribution`.
-      - `batch_dims`:  indexes independent draws from non-identical
-                       parameterizations of the `Distribution`.
-      - `event_dims`:  indexes event coordinates from one sample.
-
-    The `sample`, `batch`, and `event` dimensions constitute the entirety of a
-    `Distribution` `Tensor`'s shape.
-
-    The dimensions are always in `sample`, `batch`, `event` order.
-
-  Purpose:
-    This class partitions `Tensor` notions of `shape`, `ndims`, and `dims` into
-    `Distribution` notions of `sample,` `batch,` and `event` dimensions. That
-    is, it computes any of:
+  #### Terminology
 
-    ```
-    sample_shape     batch_shape     event_shape
-    sample_dims      batch_dims      event_dims
-    sample_ndims     batch_ndims     event_ndims
-    ```
+  Recall that a `Tensor` has:
+    - `shape`: size of `Tensor` dimensions,
+    - `ndims`: size of `shape`; number of `Tensor` dimensions,
+    - `dims`: indexes into `shape`; useful for transpose, reduce.
+
+  `Tensor`s sampled from a `Distribution` can be partitioned by `sample_dims`,
+  `batch_dims`, and `event_dims`. To understand the semantics of these
+  dimensions, consider when two of the three are fixed and the remaining
+  is varied:
+    - `sample_dims`: indexes independent draws from identical
+                     parameterizations of the `Distribution`.
+    - `batch_dims`:  indexes independent draws from non-identical
+                     parameterizations of the `Distribution`.
+    - `event_dims`:  indexes event coordinates from one sample.
+
+  The `sample`, `batch`, and `event` dimensions constitute the entirety of a
+  `Distribution` `Tensor`'s shape.
+
+  The dimensions are always in `sample`, `batch`, `event` order.
+
+  #### Purpose
+
+  This class partitions `Tensor` notions of `shape`, `ndims`, and `dims` into
+  `Distribution` notions of `sample,` `batch,` and `event` dimensions. That
+  is, it computes any of:
+
+  ```
+  sample_shape     batch_shape     event_shape
+  sample_dims      batch_dims      event_dims
+  sample_ndims     batch_ndims     event_ndims
+  ```
 
-    for a given `Tensor`, e.g., the result of
-    `Distribution.sample(sample_shape=...)`.
+  for a given `Tensor`, e.g., the result of
+  `Distribution.sample(sample_shape=...)`.
 
-    For a given `Tensor`, this class computes the above table using minimal
-    information: `batch_ndims` and `event_ndims`.
+  For a given `Tensor`, this class computes the above table using minimal
+  information: `batch_ndims` and `event_ndims`.
+
+  #### Examples
+
+  We show examples of distribution shape semantics.
 
-  Examples of `Distribution` `shape` semantics:
     - Sample dimensions:
       Computing summary statistics, i.e., the average is a reduction over sample
       dimensions.
@@ -111,52 +116,54 @@ class _DistributionShape(object):
       tf.div(1., tf.reduce_prod(x, event_dims))
       ```
 
-  Examples using this class:
-    Write `S, B, E` for `sample_shape`, `batch_shape`, and `event_shape`.
-
-    ```python
-    # 150 iid samples from one multivariate Normal with two degrees of freedom.
-    mu = [0., 0]
-    sigma = [[1., 0],
-             [0,  1]]
-    mvn = MultivariateNormal(mu, sigma)
-    rand_mvn = mvn.sample(sample_shape=[3, 50])
-    shaper = DistributionShape(batch_ndims=0, event_ndims=1)
-    S, B, E = shaper.get_shape(rand_mvn)
-    # S = [3, 50]
-    # B = []
-    # E = [2]
-
-    # 12 iid samples from one Wishart with 2x2 events.
-    sigma = [[1., 0],
-             [2,  1]]
-    wishart = Wishart(df=5, scale=sigma)
-    rand_wishart = wishart.sample(sample_shape=[3, 4])
-    shaper = DistributionShape(batch_ndims=0, event_ndims=2)
-    S, B, E = shaper.get_shape(rand_wishart)
-    # S = [3, 4]
-    # B = []
-    # E = [2, 2]
-
-    # 100 iid samples from two, non-identical trivariate Normal distributions.
-    mu    = ...  # shape(2, 3)
-    sigma = ...  # shape(2, 3, 3)
-    X = MultivariateNormal(mu, sigma).sample(shape=[4, 25])
-    # S = [4, 25]
-    # B = [2]
-    # E = [3]
-    ```
-
-  Argument Validation:
-    When `validate_args=False`, checks that cannot be done during
-    graph construction are performed at graph execution. This may result in a
-    performance degradation because data must be switched from GPU to CPU.
-
-    For example, when `validate_args=False` and `event_ndims` is a
-    non-constant `Tensor`, it is checked to be a non-negative integer at graph
-    execution. (Same for `batch_ndims`). Constant `Tensor`s and non-`Tensor`
-    arguments are always checked for correctness since this can be done for
-    "free," i.e., during graph construction.
+  We show examples using this class.
+
+  Write `S, B, E` for `sample_shape`, `batch_shape`, and `event_shape`.
+
+  ```python
+  # 150 iid samples from one multivariate Normal with two degrees of freedom.
+  mu = [0., 0]
+  sigma = [[1., 0],
+           [0,  1]]
+  mvn = MultivariateNormal(mu, sigma)
+  rand_mvn = mvn.sample(sample_shape=[3, 50])
+  shaper = DistributionShape(batch_ndims=0, event_ndims=1)
+  S, B, E = shaper.get_shape(rand_mvn)
+  # S = [3, 50]
+  # B = []
+  # E = [2]
+
+  # 12 iid samples from one Wishart with 2x2 events.
+  sigma = [[1., 0],
+           [2,  1]]
+  wishart = Wishart(df=5, scale=sigma)
+  rand_wishart = wishart.sample(sample_shape=[3, 4])
+  shaper = DistributionShape(batch_ndims=0, event_ndims=2)
+  S, B, E = shaper.get_shape(rand_wishart)
+  # S = [3, 4]
+  # B = []
+  # E = [2, 2]
+
+  # 100 iid samples from two, non-identical trivariate Normal distributions.
+  mu    = ...  # shape(2, 3)
+  sigma = ...  # shape(2, 3, 3)
+  X = MultivariateNormal(mu, sigma).sample(shape=[4, 25])
+  # S = [4, 25]
+  # B = [2]
+  # E = [3]
+  ```
+
+  #### Argument Validation
+
+  When `validate_args=False`, checks that cannot be done during
+  graph construction are performed at graph execution. This may result in a
+  performance degradation because data must be switched from GPU to CPU.
+
+  For example, when `validate_args=False` and `event_ndims` is a
+  non-constant `Tensor`, it is checked to be a non-negative integer at graph
+  execution. (Same for `batch_ndims`). Constant `Tensor`s and non-`Tensor`
+  arguments are always checked for correctness since this can be done for
+  "free," i.e., during graph construction.
   """
 
   def __init__(self,
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 3208ecdf64..971d65c4a6 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -248,11 +248,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
   The default quadrature scheme chooses `z_{N, n}` as `N` midpoints of
   the quantiles of `p(z)` (generalized quantiles if `K > 2`).
 
-  See [1] for more details.
-
-  [1]. "Quadrature Compound: An approximating family of distributions"
-       Joshua Dillon, Ian Langmore, arXiv preprints
-       https://arxiv.org/abs/1801.03080
+  See [Dillon and Langmore (2018)][1] for more details.
 
   #### About `Vector` distributions in TensorFlow.
 
@@ -313,6 +309,13 @@ class VectorDiffeomixture(distribution_lib.Distribution):
             is_positive_definite=True),
       ],
       validate_args=True)
+  ```
+
+  #### References
+
+  [1]: Joshua Dillon and Ian Langmore. Quadrature Compound: An approximating
+       family of distributions. _arXiv preprint arXiv:1801.03080_, 2018.
+       https://arxiv.org/abs/1801.03080
   """
 
   def __init__(self,
-- 
GitLab


From 4e330dcdeaafc92944a713a950355700f906ecfc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 17:45:10 -0700
Subject: [PATCH 211/960]   add option to save trace table to model directory's
 profile plugin subdirectory.

PiperOrigin-RevId: 189671290
---
 .../tpu/profiler/capture_tpu_profile.cc       | 19 +++++++++++++++----
 .../contrib/tpu/profiler/tpu_profiler.proto   |  9 ++++++++-
 2 files changed, 23 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
index b1ef9fde37..f86aff47e1 100644
--- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
+++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
@@ -29,6 +29,9 @@ limitations under the License.
 #include "tensorflow/contrib/tpu/profiler/version.h"
 #include "tensorflow/core/distributed_runtime/rpc/grpc_util.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/numbers.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -62,10 +65,13 @@ Status ValidateHostPortPair(const string& host_port) {
 }
 
 ProfileResponse Profile(const string& service_addr, int duration_ms,
+                        const string& repository_root, const string& session_id,
                         const ProfileOptions& opts) {
   ProfileRequest request;
   request.set_duration_ms(duration_ms);
   request.set_max_events(kMaxEvents);
+  request.set_repository_root(repository_root);
+  request.set_session_id(session_id);
   request.add_tools("input_pipeline");
   request.add_tools("overview_page");
   *request.mutable_opts() = opts;
@@ -137,10 +143,17 @@ int main(int argc, char** argv) {
   opts.set_include_dataset_ops(FLAGS_include_dataset_ops);
   tensorflow::ProfileResponse response;
 
+  // Use the current timestamp as the run name.
+  tensorflow::string session_id =
+      tensorflow::tpu::GetCurrentTimeStampAsString();
+  constexpr char kProfilePluginDirectory[] = "plugins/profile/";
+  string repository_root =
+      ::tensorflow::io::JoinPath(FLAGS_logdir, kProfilePluginDirectory);
   while (true) {
     std::cout << "Starting to profile TPU traces for " << duration_ms << " ms. "
               << "Remaining attempt(s): " << remaining_attempts-- << std::endl;
-    response = tensorflow::tpu::Profile(FLAGS_service_addr, duration_ms, opts);
+    response = tensorflow::tpu::Profile(FLAGS_service_addr, duration_ms,
+                                        repository_root, session_id, opts);
     if (remaining_attempts <= 0 || !response.encoded_trace().empty()) break;
     std::cout << "No trace event is collected. Automatically retrying."
               << std::endl
@@ -158,10 +171,8 @@ int main(int argc, char** argv) {
     return 0;
   }
 
-  // Use the current timestamp as the run name.
-  tensorflow::string run = tensorflow::tpu::GetCurrentTimeStampAsString();
   TF_CHECK_OK(tensorflow::tpu::WriteTensorboardTPUProfile(
-      FLAGS_logdir, run, response, &std::cout));
+      FLAGS_logdir, session_id, response, &std::cout));
   // Print this at the end so that it's not buried in irrelevant LOG messages.
   std::cout
       << "NOTE: using the trace duration " << duration_ms << "ms." << std::endl
diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler.proto b/tensorflow/contrib/tpu/profiler/tpu_profiler.proto
index f3f3302ceb..cddc3cd1b4 100644
--- a/tensorflow/contrib/tpu/profiler/tpu_profiler.proto
+++ b/tensorflow/contrib/tpu/profiler/tpu_profiler.proto
@@ -36,10 +36,17 @@ message ProfileRequest {
   // Optional profiling options that control how a TF session will be profiled.
   ProfileOptions opts = 4;
 
+  // The place where we will dump profile data. We will normally use
+  // MODEL_DIR/plugin/profile/ as our repository root.
+  string repository_root = 5;
+
+  // The user provided profile session identifier.
+  string session_id = 6;
+
   // In future, the caller will indicate which TF session is being profiled, and
   // only data relating to that program will be returned. For now, we assume
   // all activity during the profiling period is relevant.
-  // next-field: 5
+  // next-field: 7
 }
 
 message ProfileToolData {
-- 
GitLab


From c8d0b125f62a3b8785494f53d013809f8e7c8c29 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Mon, 19 Mar 2018 17:50:05 -0700
Subject: [PATCH 212/960] Register gradient for argmin (cf. #15278).

PiperOrigin-RevId: 189671974
---
 tensorflow/python/ops/math_grad.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/python/ops/math_grad.py b/tensorflow/python/ops/math_grad.py
index eb33687cb5..02e07dc7b1 100644
--- a/tensorflow/python/ops/math_grad.py
+++ b/tensorflow/python/ops/math_grad.py
@@ -41,6 +41,12 @@ def _ArgMaxGrad(op, grad):
   return [None, None]
 
 
+@ops.RegisterGradient("ArgMin")
+def _ArgMinGrad(op, grad):
+  del op, grad
+  return [None, None]
+
+
 @ops.RegisterGradient("Sum")
 def _SumGrad(op, grad):
   """Gradient for Sum."""
-- 
GitLab


From d548cb4e811fc8a04dd10370c576441fc56b03f2 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Mon, 19 Mar 2018 17:56:02 -0700
Subject: [PATCH 213/960] Add docstring pointing to tf.contrib.quantize.

PiperOrigin-RevId: 189672549
---
 tensorflow/python/training/quantize_training.i | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/training/quantize_training.i b/tensorflow/python/training/quantize_training.i
index 17ffcd6e07..fb5e47efa0 100644
--- a/tensorflow/python/training/quantize_training.i
+++ b/tensorflow/python/training/quantize_training.i
@@ -56,6 +56,11 @@ PyObject* DoQuantizeTrainingOnGraphDefHelper(
 
 %insert("python") %{
 def do_quantize_training_on_graphdef(input_graph, num_bits):
+  """A general quantization scheme is being developed in @{tf.contrib.quantize}.
+
+  Consider using that instead, though since it is in the tf.contrib namespace,
+  it is not subject to backward compatibility guarantees.
+  """
   from tensorflow.core.framework.graph_pb2 import GraphDef
   from tensorflow.python.framework import errors
   with errors.raise_exception_on_not_ok_status() as status:
-- 
GitLab


From 331bbe2886712fffc96ed9a7fb33fc9f09600240 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 18:20:12 -0700
Subject: [PATCH 214/960] Support general permutation.

PiperOrigin-RevId: 189675019
---
 tensorflow/core/kernels/data_format_ops.cc |  64 +++++-----
 tensorflow/core/kernels/data_format_ops.h  | 131 ++-------------------
 2 files changed, 42 insertions(+), 153 deletions(-)

diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc
index bea3af98eb..39ef8ee3ac 100644
--- a/tensorflow/core/kernels/data_format_ops.cc
+++ b/tensorflow/core/kernels/data_format_ops.cc
@@ -28,15 +28,6 @@ namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef Eigen::GpuDevice GPUDevice;
 
-namespace {
-inline functor::DataFormat FormatNameToEnum(const string& name) {
-  if (name == "NHWC") return functor::DataFormat::NHWC;
-  if (name == "NCHW") return functor::DataFormat::NCHW;
-  if (name == "HWNC") return functor::DataFormat::HWNC;
-  return functor::DataFormat::UNKNOWN;
-}
-}  // namespace
-
 template <typename Device, typename T>
 class DataFormatDimMapOp : public OpKernel {
  public:
@@ -76,17 +67,8 @@ class DataFormatVecPermuteOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format));
     string dst_format;
     OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format));
-    OP_REQUIRES(context,
-                (src_format == "NHWC" && dst_format == "NCHW") ||
-                    (src_format == "NCHW" && dst_format == "NHWC") ||
-                    (src_format == "NHWC" && dst_format == "HWNC") ||
-                    (src_format == "HWNC" && dst_format == "NHWC"),
-                errors::InvalidArgument(strings::StrCat(
-                    "Current implementation only supports NHWC<->NCHW and "
-                    "NHWC<->HWNC conversion; got source format ",
-                    src_format, " and destination format ", dst_format)));
-    src_format_ = FormatNameToEnum(src_format);
-    dst_format_ = FormatNameToEnum(dst_format);
+    src_format_ = src_format;
+    dst_format_ = dst_format;
   }
 
   void Compute(OpKernelContext* context) override {
@@ -116,14 +98,34 @@ class DataFormatVecPermuteOp : public OpKernel {
     Tensor* output = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input.shape(), &output));
-    functor::DataFormatVecPermute<Device, T>()(
-        context->eigen_device<Device>(), input.flat<T>(), output->flat<T>(),
-        src_format_, dst_format_);
+    // Support 1D and 2D cases.
+    Eigen::DSizes<Eigen::DenseIndex, 8> dst_idx;
+    ComputeDstIndex(input.dims(), &dst_idx);
+
+    functor::DataFormatVecPermute<Device, T>()(context->eigen_device<Device>(),
+                                               input.flat<T>(),
+                                               output->flat<T>(), dst_idx);
   }
 
  private:
-  functor::DataFormat src_format_;
-  functor::DataFormat dst_format_;
+  // Finds out the destination index. Support 1D and 2D cases.
+  // Example: HWNC --> NHWC
+  // 1D: dst = [1, 2, 0, 3],
+  // 2D: dst = [2, 3, 4, 5, 0, 1, 6, 7]
+  void ComputeDstIndex(int num_dim, Eigen::DSizes<Eigen::DenseIndex, 8>* dst) {
+    for (int i = 0; i < src_format_.size(); ++i) {
+      for (int j = 0; j < dst_format_.size(); ++j) {
+        if (dst_format_[j] != src_format_[i]) continue;
+        // Found the dst index. Set output based on the number of dims.
+        for (int k = 0; k < num_dim; ++k) {
+          (*dst)[i * num_dim + k] = j * num_dim + k;
+        }
+      }
+    }
+  }
+
+  string src_format_;
+  string dst_format_;
 };
 
 #define REGISTER_KERNEL(T)                                                \
@@ -156,12 +158,12 @@ TF_CALL_int32(DECLARE_GPU_SPECS);
 TF_CALL_int64(DECLARE_GPU_SPECS);
 #undef DECLARE_GPU_SPEC
 
-#define DECLARE_GPU_SPEC(T)                                   \
-  template <>                                                 \
-  void DataFormatVecPermute<GPUDevice, T>::operator()(        \
-      const GPUDevice& d, typename TTypes<T>::ConstFlat x,    \
-      typename TTypes<T>::Vec y, const DataFormat src_format, \
-      const DataFormat dst_format);                           \
+#define DECLARE_GPU_SPEC(T)                                \
+  template <>                                              \
+  void DataFormatVecPermute<GPUDevice, T>::operator()(     \
+      const GPUDevice& d, typename TTypes<T>::ConstFlat x, \
+      typename TTypes<T>::Vec y,                           \
+      const Eigen::DSizes<Eigen::DenseIndex, 8>& dst_idx); \
   extern template struct DataFormatVecPermute<GPUDevice, T>;
 #define DECLARE_GPU_SPECS(T) DECLARE_GPU_SPEC(T);
 TF_CALL_int32(DECLARE_GPU_SPECS);
diff --git a/tensorflow/core/kernels/data_format_ops.h b/tensorflow/core/kernels/data_format_ops.h
index d27415ed91..2ccc919586 100644
--- a/tensorflow/core/kernels/data_format_ops.h
+++ b/tensorflow/core/kernels/data_format_ops.h
@@ -23,13 +23,6 @@ limitations under the License.
 namespace tensorflow {
 namespace functor {
 
-enum class DataFormat {
-  UNKNOWN = 0,
-  NHWC,
-  NCHW,
-  HWNC,
-};
-
 // Functor used by DataFormatDimMapOP to do the computations.
 template <typename Device, typename T>
 struct DataFormatDimMap {
@@ -47,65 +40,8 @@ struct DataFormatDimMap {
 };
 
 template <typename T>
-struct VecPermuteNHWCToNCHW {
-  Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
-      typename TTypes<T>::ConstFlat input) const {
-    Eigen::DSizes<Eigen::DenseIndex, 1> result;
-    result[0] = input.dimension(0);
-    return result;
-  }
-  template <typename Output, typename Device>
-  void eval(typename TTypes<T>::ConstFlat input, Output& output,
-            const Device& d) const {
-    if (input.size() == 8) {
-      output.template chip<0>(0).device(d) = input.template chip<0>(0);
-      output.template chip<0>(1).device(d) = input.template chip<0>(1);
-      output.template chip<0>(2).device(d) = input.template chip<0>(6);
-      output.template chip<0>(3).device(d) = input.template chip<0>(7);
-      output.template chip<0>(4).device(d) = input.template chip<0>(2);
-      output.template chip<0>(5).device(d) = input.template chip<0>(3);
-      output.template chip<0>(6).device(d) = input.template chip<0>(4);
-      output.template chip<0>(7).device(d) = input.template chip<0>(5);
-    } else {
-      output.template chip<0>(0).device(d) = input.template chip<0>(0);
-      output.template chip<0>(1).device(d) = input.template chip<0>(3);
-      output.template chip<0>(2).device(d) = input.template chip<0>(1);
-      output.template chip<0>(3).device(d) = input.template chip<0>(2);
-    }
-  }
-};
-
-template <typename T>
-struct VecPermuteNCHWToNHWC {
-  Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
-      typename TTypes<T>::ConstFlat input) const {
-    Eigen::DSizes<Eigen::DenseIndex, 1> result;
-    result[0] = input.dimension(0);
-    return result;
-  }
-  template <typename Output, typename Device>
-  void eval(typename TTypes<T>::ConstFlat input, Output& output,
-            const Device& d) const {
-    if (input.size() == 8) {
-      output.template chip<0>(0).device(d) = input.template chip<0>(0);
-      output.template chip<0>(1).device(d) = input.template chip<0>(1);
-      output.template chip<0>(2).device(d) = input.template chip<0>(4);
-      output.template chip<0>(3).device(d) = input.template chip<0>(5);
-      output.template chip<0>(4).device(d) = input.template chip<0>(6);
-      output.template chip<0>(5).device(d) = input.template chip<0>(7);
-      output.template chip<0>(6).device(d) = input.template chip<0>(2);
-      output.template chip<0>(7).device(d) = input.template chip<0>(3);
-    } else {
-      output.template chip<0>(0).device(d) = input.template chip<0>(0);
-      output.template chip<0>(1).device(d) = input.template chip<0>(2);
-      output.template chip<0>(2).device(d) = input.template chip<0>(3);
-      output.template chip<0>(3).device(d) = input.template chip<0>(1);
-    }
-  }
-};
-
-template <typename T>
-struct VecPermuteNHWCToHWNC {
+struct VecPermute {
+  VecPermute(const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) : dst_(dst) {}
   Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
       typename TTypes<T>::ConstFlat input) const {
     Eigen::DSizes<Eigen::DenseIndex, 1> result;
@@ -115,71 +51,22 @@ struct VecPermuteNHWCToHWNC {
   template <typename Output, typename Device>
   void eval(typename TTypes<T>::ConstFlat input, Output& output,
             const Device& d) const {
-    if (input.size() == 8) {
-      output.template chip<0>(0).device(d) = input.template chip<0>(2);
-      output.template chip<0>(1).device(d) = input.template chip<0>(3);
-      output.template chip<0>(2).device(d) = input.template chip<0>(4);
-      output.template chip<0>(3).device(d) = input.template chip<0>(5);
-      output.template chip<0>(4).device(d) = input.template chip<0>(0);
-      output.template chip<0>(5).device(d) = input.template chip<0>(1);
-      output.template chip<0>(6).device(d) = input.template chip<0>(6);
-      output.template chip<0>(7).device(d) = input.template chip<0>(7);
-    } else {
-      output.template chip<0>(0).device(d) = input.template chip<0>(1);
-      output.template chip<0>(1).device(d) = input.template chip<0>(2);
-      output.template chip<0>(2).device(d) = input.template chip<0>(0);
-      output.template chip<0>(3).device(d) = input.template chip<0>(3);
+    for (int i = 0; i < input.size(); ++i) {
+      output.template chip<0>(dst_[i]).device(d) = input.template chip<0>(i);
     }
   }
-};
 
-template <typename T>
-struct VecPermuteHWNCToNHWC {
-  Eigen::DSizes<Eigen::DenseIndex, 1> dimensions(
-      typename TTypes<T>::ConstFlat input) const {
-    Eigen::DSizes<Eigen::DenseIndex, 1> result;
-    result[0] = input.dimension(0);
-    return result;
-  }
-  template <typename Output, typename Device>
-  void eval(typename TTypes<T>::ConstFlat input, Output& output,
-            const Device& d) const {
-    if (input.size() == 8) {
-      output.template chip<0>(0).device(d) = input.template chip<0>(4);
-      output.template chip<0>(1).device(d) = input.template chip<0>(5);
-      output.template chip<0>(2).device(d) = input.template chip<0>(0);
-      output.template chip<0>(3).device(d) = input.template chip<0>(1);
-      output.template chip<0>(4).device(d) = input.template chip<0>(2);
-      output.template chip<0>(5).device(d) = input.template chip<0>(3);
-      output.template chip<0>(6).device(d) = input.template chip<0>(6);
-      output.template chip<0>(7).device(d) = input.template chip<0>(7);
-    } else {
-      output.template chip<0>(0).device(d) = input.template chip<0>(2);
-      output.template chip<0>(1).device(d) = input.template chip<0>(0);
-      output.template chip<0>(2).device(d) = input.template chip<0>(1);
-      output.template chip<0>(3).device(d) = input.template chip<0>(3);
-    }
-  }
+ private:
+  Eigen::DSizes<Eigen::DenseIndex, 8> dst_;
 };
 
 // Functor used by DataFormatVecPermuteOp to do the computations.
 template <typename Device, typename T>
 struct DataFormatVecPermute {
   void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
-                  typename TTypes<T>::Flat y, const DataFormat src_format,
-                  const DataFormat dst_format) {
-    if (src_format == DataFormat::NHWC && dst_format == DataFormat::NCHW) {
-      y.device(d) = x.customOp(VecPermuteNHWCToNCHW<T>());
-    } else if (src_format == DataFormat::NCHW &&
-               dst_format == DataFormat::NHWC) {
-      y.device(d) = x.customOp(VecPermuteNCHWToNHWC<T>());
-    } else if (src_format == DataFormat::NHWC &&
-               dst_format == DataFormat::HWNC) {
-      y.device(d) = x.customOp(VecPermuteNHWCToHWNC<T>());
-    } else if (src_format == DataFormat::HWNC &&
-               dst_format == DataFormat::NHWC) {
-      y.device(d) = x.customOp(VecPermuteHWNCToNHWC<T>());
-    }
+                  typename TTypes<T>::Flat y,
+                  const Eigen::DSizes<Eigen::DenseIndex, 8>& dst) {
+    y.device(d) = x.customOp(VecPermute<T>(dst));
   }
 };
 
-- 
GitLab


From 28a6a8b235dafd6610e95dc05676d5b64fa5a404 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 19 Mar 2018 18:32:13 -0700
Subject: [PATCH 215/960] Export tf.GradientTape

tf.GradientTape can be used both for eager execution and graph construction
to compute gradients (unlike tf.gradients, which works only for graph
construction).

PiperOrigin-RevId: 189676004
---
 tensorflow/python/BUILD                       |  1 +
 tensorflow/python/eager/backprop.py           |  4 +++-
 tensorflow/python/ops/gradients.py            |  2 ++
 tensorflow/python/ops/standard_ops.py         |  1 +
 tensorflow/python/training/training.py        |  1 +
 .../golden/tensorflow.-gradient-tape.pbtxt    | 21 +++++++++++++++++++
 tensorflow/tools/api/golden/tensorflow.pbtxt  |  4 ++++
 7 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.-gradient-tape.pbtxt

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a029ecd4d0..ec67f43190 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1804,6 +1804,7 @@ py_library(
         ":platform",
         ":spectral_grad",
         ":util",
+        "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
         "//tensorflow/python/eager:tape",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 9b997fed30..06e11f6ef9 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -40,6 +40,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util import tf_inspect
+from tensorflow.python.util.tf_export import tf_export
 
 
 _op_attr_type_cache = {}
@@ -637,13 +638,14 @@ _default_vspace = imperative_grad.VSpace(
     ones=_ones)
 
 
+@tf_export("GradientTape")
 class GradientTape(object):
   """Record operations for automatic differentiation.
 
   Operations are recorded if they are executed within this context manager and
   at least one of their inputs is being "watched".
 
-  Variables (created by @{tf.contrib.eager.Variable} or @{tf.get_variable})
+  Variables (created by `tf.contrib.eager.Variable` or @{tf.get_variable})
   are automatically watched. Tensors can be manually watched by invoking the
   `watch`
   method on this context manager.
diff --git a/tensorflow/python/ops/gradients.py b/tensorflow/python/ops/gradients.py
index 63d9a23222..2668e8f60c 100644
--- a/tensorflow/python/ops/gradients.py
+++ b/tensorflow/python/ops/gradients.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import
+from tensorflow.python.eager.backprop import GradientTape
 from tensorflow.python.ops.custom_gradient import custom_gradient
 from tensorflow.python.ops.gradients_impl import AggregationMethod
 from tensorflow.python.ops.gradients_impl import gradients
@@ -29,6 +30,7 @@ from tensorflow.python.util.all_util import remove_undocumented
 _allowed_symbols = [
     # TODO(drpng): find a good place to reference this.
     "AggregationMethod",
+    "GradientTape",
     "custom_gradient",
     "gradients",  # tf.gradients.gradients.
     "hessians",  # tf.gradients.hessians
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index 60a98aca7f..230b7ef937 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -218,6 +218,7 @@ _allowed_symbols_gradients = [
     # Documented in training.py:
     # Not importing training.py to avoid complex graph dependencies.
     "AggregationMethod",
+    "GradientTape",
     "custom_gradient",
     "gradients",  # tf.gradients = gradients.gradients
     "hessians",
diff --git a/tensorflow/python/training/training.py b/tensorflow/python/training/training.py
index 6880cfc4db..b759b156d7 100644
--- a/tensorflow/python/training/training.py
+++ b/tensorflow/python/training/training.py
@@ -31,6 +31,7 @@ See the @{$python/train} guide.
 @@custom_gradient
 @@gradients
 @@AggregationMethod
+@@GradientTape
 @@stop_gradient
 @@hessians
 @@clip_by_value
diff --git a/tensorflow/tools/api/golden/tensorflow.-gradient-tape.pbtxt b/tensorflow/tools/api/golden/tensorflow.-gradient-tape.pbtxt
new file mode 100644
index 0000000000..7405202b89
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.-gradient-tape.pbtxt
@@ -0,0 +1,21 @@
+path: "tensorflow.GradientTape"
+tf_class {
+  is_instance: "<class \'tensorflow.python.eager.backprop.GradientTape\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'persistent\'], varargs=None, keywords=None, defaults=[\'False\'], "
+  }
+  member_method {
+    name: "gradient"
+    argspec: "args=[\'self\', \'target\', \'sources\', \'output_gradients\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+  member_method {
+    name: "watch"
+    argspec: "args=[\'self\', \'tensor\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "watched_variables"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 99e09c3759..55b82dd765 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -84,6 +84,10 @@ tf_module {
     name: "GRAPH_DEF_VERSION_MIN_PRODUCER"
     mtype: "<type \'int\'>"
   }
+  member {
+    name: "GradientTape"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "Graph"
     mtype: "<type \'type\'>"
-- 
GitLab


From 48adc7ba73177f2a9331918b160bc3d0775985b8 Mon Sep 17 00:00:00 2001
From: Surya Bhupatiraju <sbhupatiraju@google.com>
Date: Mon, 19 Mar 2018 18:51:06 -0700
Subject: [PATCH 216/960] Make L2 norm computation more stable.

Avoids the potentially numerically instable square root in the linalg_ops.norm() function because we 'undo' that operation with a math_ops.square() operation anyway.

PiperOrigin-RevId: 189677716
---
 .../gan/python/eval/python/classifier_metrics_impl.py    | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 323cbe6e76..7e86d10b64 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -563,7 +563,8 @@ def mean_only_frechet_classifier_distance_from_activations(
   m_w = math_ops.reduce_mean(generated_activations, 0)
 
   # Next the distance between means.
-  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
+  mean = math_ops.reduce_sum(
+      math_ops.squared_difference(m, m_w))  # Equivalent to L2 but more stable.
   mofid = mean
   if activations_dtype != dtypes.float64:
     mofid = math_ops.cast(mofid, activations_dtype)
@@ -637,7 +638,8 @@ def diagonal_only_frechet_classifier_distance_from_activations(
       (var + var_w) - 2.0 * math_ops.sqrt(math_ops.multiply(var, var_w)))
 
   # Next the distance between means.
-  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
+  mean = math_ops.reduce_sum(
+      math_ops.squared_difference(m, m_w))  # Equivalent to L2 but more stable.
   dofid = trace + mean
   if activations_dtype != dtypes.float64:
     dofid = math_ops.cast(dofid, activations_dtype)
@@ -718,7 +720,8 @@ def frechet_classifier_distance_from_activations(real_activations,
   trace = math_ops.trace(sigma + sigma_w) - 2.0 * sqrt_trace_component
 
   # Next the distance between means.
-  mean = math_ops.square(linalg_ops.norm(m - m_w))  # This uses the L2 norm.
+  mean = math_ops.reduce_sum(
+      math_ops.squared_difference(m, m_w))  # Equivalent to L2 but more stable.
   fid = trace + mean
   if activations_dtype != dtypes.float64:
     fid = math_ops.cast(fid, activations_dtype)
-- 
GitLab


From df9fdc7a74ab5ce786a91c7c62f6ad0d36b24f42 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 19:24:24 -0700
Subject: [PATCH 217/960] Update GraphProperties comments

PiperOrigin-RevId: 189680477
---
 tensorflow/core/grappler/costs/graph_properties.h | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.h b/tensorflow/core/grappler/costs/graph_properties.h
index 93a722f038..8ff572fe4f 100644
--- a/tensorflow/core/grappler/costs/graph_properties.h
+++ b/tensorflow/core/grappler/costs/graph_properties.h
@@ -29,9 +29,12 @@ namespace grappler {
 class SymbolicShapeRefiner;
 class TopoQueue;
 
-// A TensorFlow model to optimize.
-// Models are represented by the combination of a graph, one of more fetch
-// nodes, and potentially a set of nodes to feed.
+// Infer OpInfo::TensorProperties for graph nodes inputs/outputs.
+//
+// Typical use case, is to infer tensor properties from a graph, before doing
+// optimization pass. Nodes modified during optimization pass have to be
+// invalidated, to prevent further incorrect optimizations based on wrong shape
+// and data type properties.
 class GraphProperties {
  public:
   explicit GraphProperties(const GrapplerItem& item) : item_(item) {}
@@ -64,6 +67,9 @@ class GraphProperties {
       const string& node_name) const;
   const std::vector<OpInfo::TensorProperties>& GetOutputProperties(
       const string& node_name) const;
+  // Invalidate input/output properties for nodes modified during graph
+  // optimization pass, to prevent potential optimizations, based on incorrect
+  // shape information.
   void ClearInputProperties(const string& node_name);
   void ClearOutputProperties(const string& node_name);
 
-- 
GitLab


From 886df46bdfe89af739ee4f4a81b4c88bbe572c64 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Mon, 19 Mar 2018 19:24:26 -0700
Subject: [PATCH 218/960] Disable freeze_bn_delay by default.

PiperOrigin-RevId: 189680481
---
 .../contrib/quantize/python/quantize_graph.py       | 13 ++-----------
 1 file changed, 2 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index be4fc39651..d0fb55da74 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -99,16 +99,7 @@ def create_training_graph(input_graph=None, quant_delay=0):
   # TODO(raghuramank) Need to have freeze_bn_delay be a function of batch size
   # Currently the values below are hardcoded for mobilenetV1 on imagenet
   # Please use the experimental API if you need to tune these values.
-  if quant_delay == 0:
-    # Corresponds to case of restoring from a floating point checkpoint
-    # In this case, we can freeze the moving mean and variance early on and
-    # switch to using them during training. Therefore, freeze_bn_delay is set to
-    # 2e5.
-    freeze_bn_delay = int(2e5)
-  else:
-    # If training from scratch, set freeze_bn_delay to 100 epochs after quant
-    # delay. With a batch size of 64, this corresponds to 20000*100=2M steps.
-    freeze_bn_delay = quant_delay + int(2e6)
+  freeze_bn_delay = None
 
   _create_graph(
       input_graph=input_graph,
@@ -142,7 +133,7 @@ def experimental_create_training_graph(input_graph=None,
                                        weight_bits=8,
                                        activation_bits=8,
                                        quant_delay=0,
-                                       freeze_bn_delay=int(2e5)):
+                                       freeze_bn_delay=None):
   """Rewrites a training input_graph in place for simulated quantization.
 
   Variables added by the rewrite get added to the global variables collection.
-- 
GitLab


From fea994f9dd2bcb15eba0515c8c051aac9aed0399 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 19:30:23 -0700
Subject: [PATCH 219/960] Avoid attaching fqn annotations to live values that
 don't have a `__name__`.

PiperOrigin-RevId: 189680937
---
 .../py2tf/pyct/static_analysis/live_values.py   |  6 ++++--
 .../pyct/static_analysis/live_values_test.py    | 17 +++++++++++++++--
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py b/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py
index 0388be5d25..ac5697900a 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py
+++ b/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py
@@ -55,11 +55,13 @@ class LiveValueResolver(transformer.Base):
       if not symbol_is_local and not symbol_is_param:
         if node.id in self.literals:
           anno.setanno(node, 'live_val', self.literals[node.id])
-          # TODO(mdan): Could live values have FQNs? i.e. 'a'.join()
         elif node.id in self.context.namespace:
           obj = self.context.namespace[node.id]
           anno.setanno(node, 'live_val', obj)
-          anno.setanno(node, 'fqn', (obj.__name__,))
+          if hasattr(obj, '__name__'):
+            # If the symbol value is for example a primitive, then it will not
+            # have a name.
+            anno.setanno(node, 'fqn', (obj.__name__,))
         else:
           pass
           # TODO(mdan): Should we raise an error here?
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py b/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py
index c133a455b3..a56dff824e 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py
+++ b/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py
@@ -57,13 +57,26 @@ class LiveValuesResolverTest(test.TestCase):
 
   def test_literals(self):
 
+    a = None
+
     def test_fn():
-      return Foo  # pylint: disable=undefined-variable
+      return a
 
-    node = self._parse_and_analyze(test_fn, {}, {'Foo': 'bar'})
+    node = self._parse_and_analyze(test_fn, {}, literals={'a': 'bar'})
     retval_node = node.body[0].body[0].value
     self.assertEquals('bar', anno.getanno(retval_node, 'live_val'))
 
+  def test_primitive_values(self):
+
+    a = None
+
+    def test_fn():
+      return a
+
+    node = self._parse_and_analyze(test_fn, {'a': True})
+    retval_node = node.body[0].body[0].value
+    self.assertFalse(anno.hasanno(retval_node, 'fqn'))
+
   def test_namespace(self):
 
     def foo():
-- 
GitLab


From a2e0f8c24776f63b04a29fad9c66bf3d66e94f4d Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 19 Mar 2018 19:52:06 -0700
Subject: [PATCH 220/960] Handle non-broadcastables shapes in eager
 assert_equal

Before this change assert_equal would fail when producing an error
message for non-equal shapes because array_ops.boolean_mask only
works for equal shapes.

This part of the error message is fairly confusing in presence
of non-equal shapes. This change removes it.

PiperOrigin-RevId: 189682518
---
 .../python/kernel_tests/check_ops_test.py     |  6 +++
 tensorflow/python/ops/check_ops.py            | 39 ++++++++++---------
 2 files changed, 27 insertions(+), 18 deletions(-)

diff --git a/tensorflow/python/kernel_tests/check_ops_test.py b/tensorflow/python/kernel_tests/check_ops_test.py
index 26d3df9e63..5a83ec8d30 100644
--- a/tensorflow/python/kernel_tests/check_ops_test.py
+++ b/tensorflow/python/kernel_tests/check_ops_test.py
@@ -212,6 +212,12 @@ First 2 elements of y:
         out = array_ops.identity(small)
       self.evaluate(out)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def test_raises_when_not_equal_and_broadcastable_shapes(self):
+    cond = constant_op.constant([True, False], name="small")
+    with self.assertRaisesRegexp(errors.InvalidArgumentError, "fail"):
+      check_ops.assert_equal(cond, False, message="fail")
+
   @test_util.run_in_graph_and_eager_modes()
   def test_doesnt_raise_when_both_empty(self):
     larry = constant_op.constant([])
diff --git a/tensorflow/python/ops/check_ops.py b/tensorflow/python/ops/check_ops.py
index d6d75e4ef9..9cea3e91f7 100644
--- a/tensorflow/python/ops/check_ops.py
+++ b/tensorflow/python/ops/check_ops.py
@@ -363,27 +363,30 @@ def assert_equal(x, y, data=None, summarize=None, message=None, name=None):
                          (x_sum, x_np[:x_sum],
                           y_sum, y_np[:y_sum]))
 
-        # Get the values that actually differed and their indices.
-        mask = math_ops.logical_not(eq)
-        indices = array_ops.where(mask)
-        indices_np = indices.numpy()
-        x_vals = array_ops.boolean_mask(x, mask)
-        y_vals = array_ops.boolean_mask(y, mask)
-        summarize = min(summarize, indices_np.shape[0])
+        index_and_values_str = ''
+        if x.shape == y.shape:
+          # If the shapes of x and y are the same,
+          # Get the values that actually differed and their indices.
+          # If shapes are different this information is more confusing
+          # than useful.
+          mask = math_ops.logical_not(eq)
+          indices = array_ops.where(mask)
+          indices_np = indices.numpy()
+          x_vals = array_ops.boolean_mask(x, mask)
+          y_vals = array_ops.boolean_mask(y, mask)
+          summarize = min(summarize, indices_np.shape[0])
+          index_and_values_str = (
+              'Indices of first %s different values:\n%s\n'
+              'Corresponding x values:\n%s\n'
+              'Corresponding y values:\n%s\n' %
+              (summarize, indices_np[:summarize],
+               x_vals.numpy().reshape((-1,))[:summarize],
+               y_vals.numpy().reshape((-1,))[:summarize]))
 
         raise errors.InvalidArgumentError(
             node_def=None, op=None,
-            message=('%s\nCondition x == y did not hold.\n'
-                     'Indices of first %s different values:\n%s\n'
-                     'Corresponding x values:\n%s\n'
-                     'Corresponding y values:\n%s\n'
-                     '%s'
-                     %
-                     (message or '',
-                      summarize, indices_np[:summarize],
-                      x_vals.numpy().reshape((-1,))[:summarize],
-                      y_vals.numpy().reshape((-1,))[:summarize],
-                      summary_msg)))
+            message=('%s\nCondition x == y did not hold.\n%s%s' %
+                     (message or '', index_and_values_str, summary_msg)))
       return
 
     if data is None:
-- 
GitLab


From 79d06a6261a523866ace67f7b831d7f617d550e6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 19:58:03 -0700
Subject: [PATCH 221/960] Apply output_min/output_max to the result in the NEON
 implementation of Add operator.

Both non-NEON and reference implementation have this, but it's missing from NEON version.

PiperOrigin-RevId: 189682984
---
 .../lite/kernels/internal/optimized/optimized_ops.h        | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index edd65c9170..004433498d 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -1583,6 +1583,8 @@ inline void Add(int left_shift, const uint8* input1_data,
   TFLITE_DCHECK_LT(input1_offset, 256);
   TFLITE_DCHECK_LT(input2_offset, 256);
 #ifdef USE_NEON
+  const auto output_activation_min_vector = vdup_n_u8(output_activation_min);
+  const auto output_activation_max_vector = vdup_n_u8(output_activation_max);
   for (; i <= size - 8; i += 8) {
     const auto input1_val_original = vld1_u8(input1_data + i);
     const auto input2_val_original = vld1_u8(input2_data + i);
@@ -1628,7 +1630,10 @@ inline void Add(int left_shift, const uint8* input1_data,
     const auto s2_narrowed = vmovn_s32(s2);
     const auto s = vaddq_s16(vcombine_s16(s1_narrowed, s2_narrowed),
                              vdupq_n_s16(output_offset));
-    vst1_u8(output_data + i, vqmovun_s16(s));
+    const auto clamped =
+        vmax_u8(output_activation_min_vector,
+                vmin_u8(output_activation_max_vector, vqmovun_s16(s)));
+    vst1_u8(output_data + i, clamped);
   }
 #endif  // NEON
 
-- 
GitLab


From 56555d0604c029e8b92fcd354de3bf32b63b62d8 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Mon, 19 Mar 2018 20:06:26 -0700
Subject: [PATCH 222/960] Adds final partial batch support for
 TPUEstimator.predict.

PiperOrigin-RevId: 189683528
---
 tensorflow/contrib/tpu/BUILD                  |  11 +
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 212 +++++++++----
 .../python/tpu/tpu_estimator_signals_test.py  | 291 ++++++++++++++++++
 3 files changed, 458 insertions(+), 56 deletions(-)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index ed930e44e8..eea19e9465 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -271,6 +271,17 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "tpu_estimator_signals_test",
+    size = "small",
+    srcs = ["python/tpu/tpu_estimator_signals_test.py"],
+    additional_deps = [
+        ":tpu_estimator",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 32f15e60cd..5a8fa04e7c 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -49,6 +49,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
@@ -62,6 +63,7 @@ from tensorflow.python.training import evaluation
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training
 from tensorflow.python.training import training_util
+from tensorflow.python.util import nest
 from tensorflow.python.util import tf_inspect
 
 _INITIAL_LOSS = 1e7
@@ -678,8 +680,11 @@ def generate_per_host_enqueue_ops_fn_for_host(
         raise TypeError(
             'For mode PREDICT, `input_fn` must return `Dataset` instead of '
             '`features` and `labels`.')
+      if batch_axis is not None:
+        raise TypeError('For mode PREDICT, batch_axis is not supported yet.')
       inputs = _InputsWithStoppingSignals(
-          dataset=inputs.dataset, batch_size=ctx.batch_size_for_input_fn)
+          dataset=inputs.dataset, batch_size=ctx.batch_size_for_input_fn,
+          add_padding=True)
 
     if is_dataset:
       hooks.append(inputs.dataset_initializer_hook())
@@ -1620,11 +1625,6 @@ class TPUEstimator(estimator_lib.Estimator):
   2. `input_fn` must return a `Dataset` instance rather than `features`. In
   fact, .train() and .evaluate() also support Dataset as return value.
 
-  3. Each batch returned by `Dataset`'s iterator must have the *same static*
-     shape. This means two things:
-     - batch_size cannot be `None`
-     - the final batch must be padded by user to a full batch.
-
   Example (MNIST):
   ----------------
   ```
@@ -1639,41 +1639,9 @@ class TPUEstimator(estimator_lib.Estimator):
         [total_examples, height, width, 3], minval=-1, maxval=1)
 
     dataset = tf.data.Dataset.from_tensor_slices(images)
-    dataset = dataset.batch(batch_size)
     dataset = dataset.map(lambda images: {'image': images})
 
-    def pad(tensor, missing_count):
-        # Pads out the batch dimension to the complete batch_size.
-        rank = len(tensor.shape)
-        assert rank > 0
-        padding = tf.stack([[0, missing_count]] + [[0, 0]] * (rank - 1))
-        padded_shape = (batch_size,) + tuple(tensor.shape[1:])
-        padded_tensor = tf.pad(tensor, padding)
-        padded_tensor.set_shape(padded_shape)
-        return padded_tensor
-
-    def pad_batch_if_incomplete(batch_features):
-      # Pads out the batch dimension for all features.
-      real_batch_size = tf.shape(batch_features["image"])[0]
-
-      missing_count = tf.constant(batch_size, tf.int32) - real_batch_size
-
-      padded_features = {
-          key: pad(tensor, missing_count)
-          for key, tensor in batch_features.iteritems()
-      }
-      padding_mask = tf.concat(
-          [
-              tf.zeros((real_batch_size, 1), dtype=tf.int32),
-              tf.ones((missing_count, 1), dtype=tf.int32)
-          ],
-          axis=0)
-      padding_mask.set_shape((batch_size, 1))
-      padded_features["is_padding"] = padding_mask
-      return padded_features
-
-    dataset = dataset.map(pad_batch_if_incomplete)
-
+    dataset = dataset.batch(batch_size)
     return dataset
 
   def model_fn(features, labels, params, mode):
@@ -2089,12 +2057,14 @@ class TPUEstimator(estimator_lib.Estimator):
             predictions, message=(
                 'The estimated size for TPUEstimatorSpec.predictions is too '
                 'large.'))
-        stopping_signals = host_call_ret['signals']
+        signals = host_call_ret['signals']
 
         with ops.control_dependencies(host_ops):
           host_ops = []  # Empty, we do do not need it anymore.
           scalar_stopping_signal = _StopSignals.as_scalar_stopping_signal(
-              stopping_signals)
+              signals)
+          predictions = _PaddingSignals.slice_tensor_or_dict(
+              predictions, signals)
 
         hooks = [
             _StoppingPredictHook(scalar_stopping_signal),
@@ -2389,20 +2359,19 @@ class _Inputs(object):
     return self._dataset
 
 
-# TODO(xiejw): Extend this to support final partial batch.
 class _InputsWithStoppingSignals(_Inputs):
   """Inputs with `_StopSignals` inserted into the dataset."""
 
-  def __init__(self, dataset, batch_size):
+  def __init__(self, dataset, batch_size, add_padding=False):
 
     assert dataset is not None
 
     user_provided_dataset = dataset.map(
         _InputsWithStoppingSignals.insert_stopping_signal(
-            stop=False, batch_size=batch_size))
+            stop=False, batch_size=batch_size, add_padding=add_padding))
     final_batch_dataset = dataset.take(1).map(
         _InputsWithStoppingSignals.insert_stopping_signal(
-            stop=True, batch_size=batch_size))
+            stop=True, batch_size=batch_size, add_padding=add_padding))
     dataset = user_provided_dataset.concatenate(final_batch_dataset).prefetch(2)
 
     super(_InputsWithStoppingSignals, self).__init__(dataset=dataset)
@@ -2432,7 +2401,7 @@ class _InputsWithStoppingSignals(_Inputs):
     return signals
 
   @staticmethod
-  def insert_stopping_signal(stop, batch_size):
+  def insert_stopping_signal(stop, batch_size, add_padding=False):
     """Inserts stopping_signal into dataset via _map_fn.
 
     Here we change the data structure in the dataset, such that the return value
@@ -2443,6 +2412,7 @@ class _InputsWithStoppingSignals(_Inputs):
     Args:
       stop: bool, state of current stopping signals.
       batch_size: int, batch size.
+      add_padding: bool, whether to pad the tensor to full batch size.
 
     Returns:
       A map_fn passed to dataset.map API.
@@ -2456,11 +2426,25 @@ class _InputsWithStoppingSignals(_Inputs):
         args = args[0]
       features, labels = _Inputs._parse_inputs(args)
       new_input_dict = {}
-      new_input_dict['features'] = features
-      if labels is not None:
-        new_input_dict['labels'] = labels
+
+      if add_padding:
+        padding_mask, features, labels = (
+            _PaddingSignals.pad_features_and_labels(
+                features, labels, batch_size))
+
+        new_input_dict['features'] = features
+        if labels is not None:
+          new_input_dict['labels'] = labels
+
+      else:
+        new_input_dict['features'] = features
+        if labels is not None:
+          new_input_dict['labels'] = labels
+        padding_mask = None
+
       new_input_dict['signals'] = _StopSignals(
-          stop=stop, batch_size=batch_size).as_dict()
+          stop=stop, batch_size=batch_size, padding_mask=padding_mask).as_dict()
+
       return new_input_dict
 
     return _map_fn
@@ -2469,23 +2453,28 @@ class _InputsWithStoppingSignals(_Inputs):
 class _StopSignals(object):
   """Signals class holding all logic to handle TPU stopping condition."""
 
-  NON_STOPPING_SIGNAL = 0.0
-  STOPPING_SIGNAL = 1.0
+  NON_STOPPING_SIGNAL = False
+  STOPPING_SIGNAL = True
 
-  def __init__(self, stop, batch_size):
+  def __init__(self, stop, batch_size, padding_mask=None):
     self._stop = stop
     self._batch_size = batch_size
+    self._padding_mask = padding_mask
 
   def as_dict(self):
+    """Returns the signals as Python dict."""
     shape = [self._batch_size, 1]
-    dtype = dtypes.float32
+    dtype = dtypes.bool
 
     if self._stop:
       stopping = array_ops.ones(shape=shape, dtype=dtype)
     else:
       stopping = array_ops.zeros(shape=shape, dtype=dtype)
 
-    return {'stopping': stopping}
+    signals = {'stopping': stopping}
+    if self._padding_mask is not None:
+      signals['padding_mask'] = self._padding_mask
+    return signals
 
   @staticmethod
   def as_scalar_stopping_signal(signals):
@@ -2493,7 +2482,118 @@ class _StopSignals(object):
 
   @staticmethod
   def should_stop(scalar_stopping_signal):
-    return scalar_stopping_signal >= _StopSignals.STOPPING_SIGNAL
+    if isinstance(scalar_stopping_signal, ops.Tensor):
+      # STOPPING_SIGNAL is a constant True. Here, the logical_and is just the TF
+      # way to express the bool check whether scalar_stopping_signal is True.
+      return math_ops.logical_and(
+          scalar_stopping_signal, _StopSignals.STOPPING_SIGNAL)
+    else:
+      # For non Tensor case, it is used in SessionRunHook. So, we cannot modify
+      # the graph anymore. Here, we use pure Python.
+      return bool(scalar_stopping_signal)
+
+
+class _PaddingSignals(object):
+  """Signals class holding all logic to handle padding."""
+
+  @staticmethod
+  def pad_features_and_labels(features, labels, batch_size):
+    """Pads out the batch dimension of features and labels."""
+    real_batch_size = array_ops.shape(
+        _PaddingSignals._find_any_tensor(features))[0]
+
+    batch_size_tensor = constant_op.constant(batch_size, dtypes.int32)
+
+    check_greater = check_ops.assert_greater_equal(
+        batch_size_tensor, real_batch_size,
+        data=(batch_size_tensor, real_batch_size),
+        message='The real batch size should not be greater than batch_size.')
+
+    with ops.control_dependencies([check_greater]):
+      missing_count = batch_size_tensor - real_batch_size
+
+    def pad_single_tensor(tensor):
+      """Pads out the batch dimension of a tensor to the complete batch_size."""
+      rank = len(tensor.shape)
+      assert rank > 0
+      padding = array_ops.stack([[0, missing_count]] + [[0, 0]] * (rank - 1))
+      padded_shape = (batch_size,) + tuple(tensor.shape[1:])
+      padded_tensor = array_ops.pad(tensor, padding)
+      padded_tensor.set_shape(padded_shape)
+      return padded_tensor
+
+    def nest_pad(tensor_or_dict):
+      return nest.map_structure(pad_single_tensor, tensor_or_dict)
+
+    features = nest_pad(features)
+    if labels is not None:
+      labels = nest_pad(labels)
+
+    padding_mask = _PaddingSignals._padding_mask(
+        real_batch_size, missing_count, batch_size)
+
+    return padding_mask, features, labels
+
+  @staticmethod
+  def slice_tensor_or_dict(tensor_or_dict, signals):
+    """Slice the real Tensors according to padding mask in signals."""
+
+    padding_mask = signals['padding_mask']
+    batch_size = array_ops.shape(padding_mask)[0]
+
+    def verify_batch_size(tensor):
+      check_batch_size = math_ops.equal(batch_size, tensor.shape[0])
+      with ops.control_dependencies([check_batch_size]):
+        return array_ops.identity(tensor)
+
+    def slice_single_tensor(tensor):
+      rank = len(tensor.shape)
+      assert rank > 0
+      real_batch_size = batch_size - math_ops.reduce_sum(padding_mask)
+      return verify_batch_size(tensor)[0:real_batch_size]
+
+    # As we split the Tensors to all TPU cores and concat them back, it is
+    # important to ensure the real data is placed before padded ones, i.e.,
+    # order is preserved. By that, the sliced padding mask should have all 0's.
+    # If this assertion failed, # the slice logic here would not hold.
+    sliced_padding_mask = slice_single_tensor(padding_mask)
+    assert_padding_mask = math_ops.equal(
+        math_ops.reduce_sum(sliced_padding_mask), 0)
+
+    with ops.control_dependencies([assert_padding_mask]):
+      should_stop = _StopSignals.should_stop(
+          _StopSignals.as_scalar_stopping_signal(signals))
+
+    is_full_batch = math_ops.equal(math_ops.reduce_sum(padding_mask), 0)
+
+    def slice_fn(tensor):
+      # If the current batch is full batch or part of stopping signals, we do
+      # not need to slice to save performance.
+      return control_flow_ops.cond(
+          math_ops.logical_or(should_stop, is_full_batch),
+          (lambda: verify_batch_size(tensor)),
+          (lambda: slice_single_tensor(tensor)))
+
+    return nest.map_structure(slice_fn, tensor_or_dict)
+
+  @staticmethod
+  def _find_any_tensor(batch_features):
+    tensors = [x for x in nest.flatten(batch_features)
+               if isinstance(x, ops.Tensor)]
+    if not tensors:
+      raise ValueError('Cannot find any Tensor in features dict.')
+    return tensors[0]
+
+  @staticmethod
+  def _padding_mask(real_batch_size, missing_count, batch_size):
+    padding_mask = array_ops.concat(
+        [
+            array_ops.zeros((real_batch_size,), dtype=dtypes.int32),
+            array_ops.ones((missing_count,), dtype=dtypes.int32)
+        ],
+        axis=0)
+    padding_mask.set_shape((batch_size,))
+    return padding_mask
 
 
 class _SignalsHelper(object):
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
new file mode 100644
index 0000000000..3e90957e6d
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator_signals_test.py
@@ -0,0 +1,291 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TPU Estimator Signalling Tests."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.tpu.python.tpu import tpu_estimator
+from tensorflow.python import data as dataset_lib
+from tensorflow.python.client import session
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import test
+
+
+def make_input_fn(num_samples):
+  a = np.linspace(0, 100.0, num=num_samples)
+  b = np.reshape(np.array(a, dtype=np.float32), (len(a), 1))
+
+  def input_fn(params):
+    batch_size = params['batch_size']
+    da1 = dataset_lib.Dataset.from_tensor_slices(a)
+    da2 = dataset_lib.Dataset.from_tensor_slices(b)
+
+    dataset = dataset_lib.Dataset.zip((da1, da2))
+    dataset = dataset.map(lambda fa, fb: {'a': fa, 'b': fb})
+    dataset = dataset.batch(batch_size)
+    return dataset
+  return input_fn, (a, b)
+
+
+def make_input_fn_with_labels(num_samples):
+  a = np.linspace(0, 100.0, num=num_samples)
+  b = np.reshape(np.array(a, dtype=np.float32), (len(a), 1))
+
+  def input_fn(params):
+    batch_size = params['batch_size']
+    da1 = dataset_lib.Dataset.from_tensor_slices(a)
+    da2 = dataset_lib.Dataset.from_tensor_slices(b)
+
+    dataset = dataset_lib.Dataset.zip((da1, da2))
+    dataset = dataset.map(lambda fa, fb: ({'a': fa}, fb))
+    dataset = dataset.batch(batch_size)
+    return dataset
+  return input_fn, (a, b)
+
+
+class TPUEstimatorStoppingSignalsTest(test.TestCase):
+
+  def test_normal_output_without_signals(self):
+    num_samples = 4
+    batch_size = 2
+
+    params = {'batch_size': batch_size}
+    input_fn, (a, b) = make_input_fn(num_samples=num_samples)
+
+    with ops.Graph().as_default():
+      dataset = input_fn(params)
+      features = dataset.make_one_shot_iterator().get_next()
+
+      # With tf.data.Dataset.batch, the batch is None, i.e., dynamic shape.
+      self.assertIsNone(features['a'].shape.as_list()[0])
+
+      with session.Session() as sess:
+        result = sess.run(features)
+        self.assertAllEqual(a[:batch_size], result['a'])
+        self.assertAllEqual(b[:batch_size], result['b'])
+
+        # This run should work as num_samples / batch_size = 2.
+        result = sess.run(features)
+        self.assertAllEqual(a[batch_size:num_samples], result['a'])
+        self.assertAllEqual(b[batch_size:num_samples], result['b'])
+
+        with self.assertRaises(errors.OutOfRangeError):
+          # Given num_samples and batch_size, this run should fail.
+          sess.run(features)
+
+  def test_output_with_stopping_signals(self):
+    num_samples = 4
+    batch_size = 2
+
+    params = {'batch_size': batch_size}
+    input_fn, (a, b) = make_input_fn(num_samples=num_samples)
+
+    with ops.Graph().as_default():
+      dataset = input_fn(params)
+      inputs = tpu_estimator._InputsWithStoppingSignals(dataset, batch_size)
+      hook = inputs.dataset_initializer_hook()
+      features, _ = inputs.features_and_labels()
+      signals = inputs.signals()
+
+      # With tf.data.Dataset.batch, the batch is None, i.e., dynamic shape.
+      self.assertIsNone(features['a'].shape.as_list()[0])
+
+      with session.Session() as sess:
+        hook.begin()
+        hook.after_create_session(sess, coord=None)
+
+        result, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual(a[:batch_size], result['a'])
+        self.assertAllEqual(b[:batch_size], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+
+        # This run should work as num_samples / batch_size = 2.
+        result, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual(a[batch_size:num_samples], result['a'])
+        self.assertAllEqual(b[batch_size:num_samples], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+
+        # This run should work, *but* see STOP ('1') as signals
+        _, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual([[1.]] * batch_size, evaluated_signals['stopping'])
+
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(features)
+
+
+class TPUEstimatorStoppingSignalsWithPaddingTest(test.TestCase):
+
+  def test_num_samples_divisible_by_batch_size(self):
+    num_samples = 4
+    batch_size = 2
+
+    params = {'batch_size': batch_size}
+    input_fn, (a, b) = make_input_fn(num_samples=num_samples)
+
+    with ops.Graph().as_default():
+      dataset = input_fn(params)
+      inputs = tpu_estimator._InputsWithStoppingSignals(dataset, batch_size,
+                                                        add_padding=True)
+      hook = inputs.dataset_initializer_hook()
+      features, _ = inputs.features_and_labels()
+      signals = inputs.signals()
+
+      # With padding, all shapes are static now.
+      self.assertEqual(batch_size, features['a'].shape.as_list()[0])
+
+      with session.Session() as sess:
+        hook.begin()
+        hook.after_create_session(sess, coord=None)
+
+        result, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual(a[:batch_size], result['a'])
+        self.assertAllEqual(b[:batch_size], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+        self.assertAllEqual([0.] * batch_size,
+                            evaluated_signals['padding_mask'])
+
+        # This run should work as num_samples / batch_size = 2.
+        result, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual(a[batch_size:num_samples], result['a'])
+        self.assertAllEqual(b[batch_size:num_samples], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+        self.assertAllEqual([0.] * batch_size,
+                            evaluated_signals['padding_mask'])
+
+        # This run should work, *but* see STOP ('1') as signals
+        _, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual([[1.]] * batch_size, evaluated_signals['stopping'])
+
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(features)
+
+  def test_num_samples_not_divisible_by_batch_size(self):
+    num_samples = 5
+    batch_size = 2
+
+    params = {'batch_size': batch_size}
+    input_fn, (a, b) = make_input_fn_with_labels(num_samples=num_samples)
+
+    with ops.Graph().as_default():
+      dataset = input_fn(params)
+      inputs = tpu_estimator._InputsWithStoppingSignals(dataset, batch_size,
+                                                        add_padding=True)
+      hook = inputs.dataset_initializer_hook()
+      features, labels = inputs.features_and_labels()
+      signals = inputs.signals()
+
+      # With padding, all shapes are static.
+      self.assertEqual(batch_size, features['a'].shape.as_list()[0])
+
+      with session.Session() as sess:
+        hook.begin()
+        hook.after_create_session(sess, coord=None)
+
+        evaluated_features, evaluated_labels, evaluated_signals = (
+            sess.run([features, labels, signals]))
+        self.assertAllEqual(a[:batch_size], evaluated_features['a'])
+        self.assertAllEqual(b[:batch_size], evaluated_labels)
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+        self.assertAllEqual([0.] * batch_size,
+                            evaluated_signals['padding_mask'])
+
+        # This run should work as num_samples / batch_size >= 2.
+        evaluated_features, evaluated_labels, evaluated_signals = (
+            sess.run([features, labels, signals]))
+        self.assertAllEqual(a[batch_size:2*batch_size], evaluated_features['a'])
+        self.assertAllEqual(b[batch_size:2*batch_size], evaluated_labels)
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+        self.assertAllEqual([0.] * batch_size,
+                            evaluated_signals['padding_mask'])
+
+        # This is the final partial batch.
+        evaluated_features, evaluated_labels, evaluated_signals = (
+            sess.run([features, labels, signals]))
+        real_batch_size = num_samples % batch_size
+
+        # Assert the real part.
+        self.assertAllEqual(a[2*batch_size:num_samples],
+                            evaluated_features['a'][:real_batch_size])
+        self.assertAllEqual(b[2*batch_size:num_samples],
+                            evaluated_labels[:real_batch_size])
+        # Assert the padded part.
+        self.assertAllEqual([0.0] * (batch_size - real_batch_size),
+                            evaluated_features['a'][real_batch_size:])
+        self.assertAllEqual([[0.0]] * (batch_size - real_batch_size),
+                            evaluated_labels[real_batch_size:])
+
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+
+        padding = ([.0] * real_batch_size
+                   + [1.] * (batch_size - real_batch_size))
+        self.assertAllEqual(padding, evaluated_signals['padding_mask'])
+
+        # This run should work, *but* see STOP ('1') as signals
+        _, evaluated_signals = sess.run([features, signals])
+        self.assertAllEqual([[1.]] * batch_size, evaluated_signals['stopping'])
+
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(features)
+
+  def test_slice(self):
+    num_samples = 3
+    batch_size = 2
+
+    params = {'batch_size': batch_size}
+    input_fn, (a, b) = make_input_fn(num_samples=num_samples)
+
+    with ops.Graph().as_default():
+      dataset = input_fn(params)
+      inputs = tpu_estimator._InputsWithStoppingSignals(dataset, batch_size,
+                                                        add_padding=True)
+      hook = inputs.dataset_initializer_hook()
+      features, _ = inputs.features_and_labels()
+      signals = inputs.signals()
+
+      sliced_features = (
+          tpu_estimator._PaddingSignals.slice_tensor_or_dict(
+              features, signals))
+
+      with session.Session() as sess:
+        hook.begin()
+        hook.after_create_session(sess, coord=None)
+
+        result, evaluated_signals = sess.run([sliced_features, signals])
+        self.assertAllEqual(a[:batch_size], result['a'])
+        self.assertAllEqual(b[:batch_size], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+
+        # This is the final partial batch.
+        result, evaluated_signals = sess.run([sliced_features, signals])
+        self.assertEqual(1, len(result['a']))
+        self.assertAllEqual(a[batch_size:num_samples], result['a'])
+        self.assertAllEqual(b[batch_size:num_samples], result['b'])
+        self.assertAllEqual([[0.]] * batch_size, evaluated_signals['stopping'])
+
+        # This run should work, *but* see STOP ('1') as signals
+        _, evaluated_signals = sess.run([sliced_features, signals])
+        self.assertAllEqual([[1.]] * batch_size, evaluated_signals['stopping'])
+
+        with self.assertRaises(errors.OutOfRangeError):
+          sess.run(sliced_features)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From 88334807a5beb8b61a967d21e534ed238e7916c0 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 19 Mar 2018 20:21:45 -0700
Subject: [PATCH 223/960] Always imports the contrib summary ops when importing
 tensorflow.

Fixes #17802

PiperOrigin-RevId: 189684619
---
 tensorflow/contrib/cmake/tf_python.cmake      |   3 +-
 tensorflow/contrib/summary/BUILD              |   8 +-
 tensorflow/contrib/summary/summary_ops.py     |   2 +-
 tensorflow/core/BUILD                         |   1 +
 .../base_api/api_def_CloseSummaryWriter.pbtxt |   4 +
 .../api_def_CreateSummaryDbWriter.pbtxt       |   4 +
 .../api_def_CreateSummaryFileWriter.pbtxt     |   4 +
 .../base_api/api_def_FlushSummaryWriter.pbtxt |   4 +
 .../base_api/api_def_ImportEvent.pbtxt        |   4 +
 .../base_api/api_def_SummaryWriter.pbtxt      |   4 +
 .../base_api/api_def_WriteAudioSummary.pbtxt  |   4 +
 .../base_api/api_def_WriteGraphSummary.pbtxt  |   4 +
 .../api_def_WriteHistogramSummary.pbtxt       |   4 +
 .../base_api/api_def_WriteImageSummary.pbtxt  |   4 +
 .../base_api/api_def_WriteScalarSummary.pbtxt |   4 +
 .../base_api/api_def_WriteSummary.pbtxt       |   4 +
 .../api_def_CloseSummaryWriter.pbtxt          |   4 +
 .../api_def_CreateSummaryDbWriter.pbtxt       |   4 +
 .../api_def_CreateSummaryFileWriter.pbtxt     |   4 +
 .../api_def_FlushSummaryWriter.pbtxt          |   4 +
 .../python_api/api_def_ImportEvent.pbtxt      |   4 +
 .../python_api/api_def_SummaryWriter.pbtxt    |   4 +
 .../api_def_WriteAudioSummary.pbtxt           |   4 +
 .../api_def_WriteGraphSummary.pbtxt           |   4 +
 .../api_def_WriteHistogramSummary.pbtxt       |   4 +
 .../api_def_WriteImageSummary.pbtxt           |   4 +
 .../api_def_WriteScalarSummary.pbtxt          |   4 +
 .../python_api/api_def_WriteSummary.pbtxt     |   4 +
 tensorflow/core/ops/summary_ops.cc            | 191 ++----------------
 tensorflow/python/BUILD                       |   7 +
 tensorflow/python/summary/summary.py          |   2 +
 31 files changed, 121 insertions(+), 189 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CloseSummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CreateSummaryDbWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CreateSummaryFileWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_FlushSummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ImportEvent.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_SummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteGraphSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteHistogramSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteImageSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteScalarSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_WriteSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CloseSummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CreateSummaryDbWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CreateSummaryFileWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_FlushSummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ImportEvent.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_SummaryWriter.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteAudioSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteGraphSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteHistogramSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteImageSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteScalarSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_WriteSummary.pbtxt

diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index b730ebd3ba..1e354bf212 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -348,6 +348,7 @@ GENERATE_PYTHON_OP_LIB("state_ops")
 GENERATE_PYTHON_OP_LIB("sparse_ops")
 GENERATE_PYTHON_OP_LIB("spectral_ops")
 GENERATE_PYTHON_OP_LIB("string_ops")
+GENERATE_PYTHON_OP_LIB("summary_ops")
 GENERATE_PYTHON_OP_LIB("user_ops")
 GENERATE_PYTHON_OP_LIB("training_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/training/gen_training_ops.py)
@@ -419,8 +420,6 @@ GENERATE_PYTHON_OP_LIB("stateless_random_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/stateless/gen_stateless_random_ops.py)
 GENERATE_PYTHON_OP_LIB("debug_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/debug/ops/gen_debug_ops.py)
-GENERATE_PYTHON_OP_LIB("summary_ops"
-  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/summary/gen_summary_ops.py)
 
 add_custom_target(tf_python_ops SOURCES ${tf_python_ops_generated_files} ${PYTHON_PROTO_GENFILES})
 add_dependencies(tf_python_ops tf_python_op_gen_main)
diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD
index b58c83fdaf..80563c5e15 100644
--- a/tensorflow/contrib/summary/BUILD
+++ b/tensorflow/contrib/summary/BUILD
@@ -10,12 +10,6 @@ load(
     "tf_gen_op_wrapper_py",
 )
 
-tf_gen_op_wrapper_py(
-    name = "gen_summary_ops",
-    out = "gen_summary_ops.py",
-    deps = ["//tensorflow/core:summary_ops_op_lib"],
-)
-
 py_test(
     name = "summary_ops_test",
     srcs = ["summary_ops_test.py"],
@@ -61,7 +55,6 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:internal"],
     deps = [
-        ":gen_summary_ops",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:constant_op",
@@ -72,6 +65,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:summary_op_util",
+        "//tensorflow/python:summary_ops_gen",
         "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python/eager:context",
diff --git a/tensorflow/contrib/summary/summary_ops.py b/tensorflow/contrib/summary/summary_ops.py
index c1724c6e43..bc763fe655 100644
--- a/tensorflow/contrib/summary/summary_ops.py
+++ b/tensorflow/contrib/summary/summary_ops.py
@@ -26,7 +26,6 @@ import time
 
 import six
 
-from tensorflow.contrib.summary import gen_summary_ops
 from tensorflow.core.framework import graph_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
@@ -35,6 +34,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_summary_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import summary_op_util
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1d283e240d..8124280914 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -722,6 +722,7 @@ cc_library(
         ":sendrecv_ops_op_lib",
         ":set_ops_op_lib",
         ":sparse_ops_op_lib",
+        ":summary_ops_op_lib",
         ":spectral_ops_op_lib",
         ":state_ops_op_lib",
         ":stateless_random_ops_op_lib",
diff --git a/tensorflow/core/api_def/base_api/api_def_CloseSummaryWriter.pbtxt b/tensorflow/core/api_def/base_api/api_def_CloseSummaryWriter.pbtxt
new file mode 100644
index 0000000000..f6fd7d9316
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CloseSummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CloseSummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CreateSummaryDbWriter.pbtxt b/tensorflow/core/api_def/base_api/api_def_CreateSummaryDbWriter.pbtxt
new file mode 100644
index 0000000000..28da46a0f8
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CreateSummaryDbWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CreateSummaryDbWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CreateSummaryFileWriter.pbtxt b/tensorflow/core/api_def/base_api/api_def_CreateSummaryFileWriter.pbtxt
new file mode 100644
index 0000000000..2ce2c4d37e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CreateSummaryFileWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CreateSummaryFileWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_FlushSummaryWriter.pbtxt b/tensorflow/core/api_def/base_api/api_def_FlushSummaryWriter.pbtxt
new file mode 100644
index 0000000000..3ada43c9b8
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_FlushSummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "FlushSummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ImportEvent.pbtxt b/tensorflow/core/api_def/base_api/api_def_ImportEvent.pbtxt
new file mode 100644
index 0000000000..d8813b58f3
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ImportEvent.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ImportEvent"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_SummaryWriter.pbtxt b/tensorflow/core/api_def/base_api/api_def_SummaryWriter.pbtxt
new file mode 100644
index 0000000000..1fe57ecf19
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_SummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "SummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt
new file mode 100644
index 0000000000..520952cd41
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteAudioSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteAudioSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteGraphSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteGraphSummary.pbtxt
new file mode 100644
index 0000000000..3653477b20
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteGraphSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteGraphSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteHistogramSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteHistogramSummary.pbtxt
new file mode 100644
index 0000000000..26e1482630
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteHistogramSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteHistogramSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteImageSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteImageSummary.pbtxt
new file mode 100644
index 0000000000..78db8700f0
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteImageSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteImageSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteScalarSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteScalarSummary.pbtxt
new file mode 100644
index 0000000000..7bae8638d2
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteScalarSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteScalarSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_WriteSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_WriteSummary.pbtxt
new file mode 100644
index 0000000000..db86883e21
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_WriteSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CloseSummaryWriter.pbtxt b/tensorflow/core/api_def/python_api/api_def_CloseSummaryWriter.pbtxt
new file mode 100644
index 0000000000..f6fd7d9316
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CloseSummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CloseSummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CreateSummaryDbWriter.pbtxt b/tensorflow/core/api_def/python_api/api_def_CreateSummaryDbWriter.pbtxt
new file mode 100644
index 0000000000..28da46a0f8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CreateSummaryDbWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CreateSummaryDbWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CreateSummaryFileWriter.pbtxt b/tensorflow/core/api_def/python_api/api_def_CreateSummaryFileWriter.pbtxt
new file mode 100644
index 0000000000..2ce2c4d37e
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CreateSummaryFileWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CreateSummaryFileWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_FlushSummaryWriter.pbtxt b/tensorflow/core/api_def/python_api/api_def_FlushSummaryWriter.pbtxt
new file mode 100644
index 0000000000..3ada43c9b8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_FlushSummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "FlushSummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ImportEvent.pbtxt b/tensorflow/core/api_def/python_api/api_def_ImportEvent.pbtxt
new file mode 100644
index 0000000000..d8813b58f3
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ImportEvent.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ImportEvent"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_SummaryWriter.pbtxt b/tensorflow/core/api_def/python_api/api_def_SummaryWriter.pbtxt
new file mode 100644
index 0000000000..1fe57ecf19
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_SummaryWriter.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "SummaryWriter"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteAudioSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteAudioSummary.pbtxt
new file mode 100644
index 0000000000..520952cd41
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteAudioSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteAudioSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteGraphSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteGraphSummary.pbtxt
new file mode 100644
index 0000000000..3653477b20
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteGraphSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteGraphSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteHistogramSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteHistogramSummary.pbtxt
new file mode 100644
index 0000000000..26e1482630
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteHistogramSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteHistogramSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteImageSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteImageSummary.pbtxt
new file mode 100644
index 0000000000..78db8700f0
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteImageSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteImageSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteScalarSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteScalarSummary.pbtxt
new file mode 100644
index 0000000000..7bae8638d2
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteScalarSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteScalarSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_WriteSummary.pbtxt b/tensorflow/core/api_def/python_api/api_def_WriteSummary.pbtxt
new file mode 100644
index 0000000000..db86883e21
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_WriteSummary.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "WriteSummary"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/ops/summary_ops.cc b/tensorflow/core/ops/summary_ops.cc
index aa7458f903..742a221adc 100644
--- a/tensorflow/core/ops/summary_ops.cc
+++ b/tensorflow/core/ops/summary_ops.cc
@@ -22,15 +22,7 @@ REGISTER_OP("SummaryWriter")
     .Output("writer: resource")
     .Attr("shared_name: string = ''")
     .Attr("container: string = ''")
-    .SetShapeFn(shape_inference::ScalarShape)
-    .Doc(R"doc(
-Returns a handle to be used to access a summary writer.
-
-The summary writer is an in-graph resource which can be used by ops to write
-summaries to event files.
-
-writer: the summary writer resource. Scalar handle.
-)doc");
+    .SetShapeFn(shape_inference::ScalarShape);
 
 REGISTER_OP("CreateSummaryFileWriter")
     .Input("writer: resource")
@@ -38,17 +30,7 @@ REGISTER_OP("CreateSummaryFileWriter")
     .Input("max_queue: int32")
     .Input("flush_millis: int32")
     .Input("filename_suffix: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Creates a summary file writer accessible by the given resource handle.
-
-writer: A handle to the summary writer resource
-logdir: Directory where the event file will be written.
-max_queue: Size of the queue of pending events and summaries.
-flush_millis: How often, in milliseconds, to flush the pending events and
-  summaries to disk.
-filename_suffix: Every event file's name is suffixed with this suffix.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("CreateSummaryDbWriter")
     .Input("writer: resource")
@@ -56,47 +38,15 @@ REGISTER_OP("CreateSummaryDbWriter")
     .Input("experiment_name: string")
     .Input("run_name: string")
     .Input("user_name: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Creates summary database writer accessible by given resource handle.
-
-This can be used to write tensors from the execution graph directly
-to a database. Only SQLite is supported right now. This function
-will create the schema if it doesn't exist. Entries in the Users,
-Experiments, and Runs tables will be created automatically if they
-don't already exist.
-
-writer: Handle to SummaryWriter resource to overwrite.
-db_uri: For example "file:/tmp/foo.sqlite".
-experiment_name: Can't contain ASCII control characters or <>. Case
-  sensitive. If empty, then the Run will not be associated with any
-  Experiment.
-run_name: Can't contain ASCII control characters or <>. Case sensitive.
-  If empty, then each Tag will not be associated with any Run.
-user_name: Must be valid as both a DNS label and Linux username. If
-  empty, then the Experiment will not be associated with any User.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("FlushSummaryWriter")
     .Input("writer: resource")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"(
-Flushes the writer's unwritten events.
-
-writer: A handle to the summary writer resource.
-)");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("CloseSummaryWriter")
     .Input("writer: resource")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"(
-Flushes and closes the summary writer.
-
-Also removes it from the resource manager. To reopen, use another
-CreateSummaryFileWriter op.
-
-writer: A handle to the summary writer resource.
-)");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteSummary")
     .Input("writer: resource")
@@ -105,31 +55,12 @@ REGISTER_OP("WriteSummary")
     .Input("tag: string")
     .Input("summary_metadata: string")
     .Attr("T: type")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Outputs a `Summary` protocol buffer with a tensor.
-
-writer: A handle to a summary writer.
-step: The step to write the summary for.
-tensor: A tensor to serialize.
-tag: The summary's tag.
-summary_metadata: Serialized SummaryMetadata protocol buffer containing
- plugin-related metadata for this summary.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("ImportEvent")
     .Input("writer: resource")
     .Input("event: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Outputs a `tf.Event` protocol buffer.
-
-When CreateSummaryDbWriter is being used, this op can be useful for
-importing data from event logs.
-
-writer: A handle to a summary writer.
-event: A string containing a binary-encoded tf.Event proto.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteScalarSummary")
     .Input("writer: resource")
@@ -137,17 +68,7 @@ REGISTER_OP("WriteScalarSummary")
     .Input("tag: string")
     .Input("value: T")
     .Attr("T: realnumbertype")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Writes a `Summary` protocol buffer with scalar values.
-
-The input `tag` and `value` must have the scalars.
-
-writer: A handle to a summary writer.
-step: The step to write the summary for.
-tag: Tag for the summary.
-value: Value for the summary.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteHistogramSummary")
     .Input("writer: resource")
@@ -155,21 +76,7 @@ REGISTER_OP("WriteHistogramSummary")
     .Input("tag: string")
     .Input("values: T")
     .Attr("T: realnumbertype = DT_FLOAT")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Writes a `Summary` protocol buffer with a histogram.
-
-The generated
-[`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-has one summary value containing a histogram for `values`.
-
-This op reports an `InvalidArgument` error if any value is not finite.
-
-writer: A handle to a summary writer.
-step: The step to write the summary for.
-tag: Scalar.  Tag to use for the `Summary.Value`.
-values: Any shape. Values to use to build the histogram.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteImageSummary")
     .Input("writer: resource")
@@ -179,52 +86,7 @@ REGISTER_OP("WriteImageSummary")
     .Input("bad_color: uint8")
     .Attr("max_images: int >= 1 = 3")
     .Attr("T: {uint8, float, half} = DT_FLOAT")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Writes a `Summary` protocol buffer with images.
-
-The summary has up to `max_images` summary values containing images. The
-images are built from `tensor` which must be 4-D with shape `[batch_size,
-height, width, channels]` and where `channels` can be:
-
-*  1: `tensor` is interpreted as Grayscale.
-*  3: `tensor` is interpreted as RGB.
-*  4: `tensor` is interpreted as RGBA.
-
-The images have the same number of channels as the input tensor. For float
-input, the values are normalized one image at a time to fit in the range
-`[0, 255]`.  `uint8` values are unchanged.  The op uses two different
-normalization algorithms:
-
-*  If the input values are all positive, they are rescaled so the largest one
-   is 255.
-
-*  If any input value is negative, the values are shifted so input value 0.0
-   is at 127.  They are then rescaled so that either the smallest value is 0,
-   or the largest one is 255.
-
-The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-build the `tag` of the summary values:
-
-*  If `max_images` is 1, the summary value tag is '*tag*/image'.
-*  If `max_images` is greater than 1, the summary value tags are
-   generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
-
-The `bad_color` argument is the color to use in the generated images for
-non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
-Each element must be in the range `[0, 255]` (It represents the value of a
-pixel in the output image).  Non-finite values in the input tensor are
-replaced by this tensor in the output image.  The default value is the color
-red.
-
-writer: A handle to a summary writer.
-step: The step to write the summary for.
-tag: Scalar. Used to build the `tag` attribute of the summary values.
-tensor: 4-D of shape `[batch_size, height, width, channels]` where
-  `channels` is 1, 3, or 4.
-max_images: Max number of batch elements to generate images for.
-bad_color: Color to use for pixels with non-finite values.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteAudioSummary")
     .Input("writer: resource")
@@ -233,41 +95,12 @@ REGISTER_OP("WriteAudioSummary")
     .Input("tensor: float")
     .Input("sample_rate: float")
     .Attr("max_outputs: int >= 1 = 3")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Writes a `Summary` protocol buffer with audio.
-
-The summary has up to `max_outputs` summary values containing audio. The
-audio is built from `tensor` which must be 3-D with shape `[batch_size,
-frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
-
-The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-build the `tag` of the summary values:
-
-*  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-*  If `max_outputs` is greater than 1, the summary value tags are
-   generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
-
-writer: A handle to a summary writer.
-step: The step to write the summary for.
-tag: Scalar. Used to build the `tag` attribute of the summary values.
-tensor: 2-D of shape `[batch_size, frames]`.
-sample_rate: The sample rate of the signal in hertz.
-max_outputs: Max number of batch elements to generate audio for.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 REGISTER_OP("WriteGraphSummary")
     .Input("writer: resource")
     .Input("step: int64")
     .Input("tensor: string")
-    .SetShapeFn(shape_inference::NoOutputs)
-    .Doc(R"doc(
-Writes a `GraphDef` protocol buffer to a `SummaryWriter`.
-
-writer: Handle of `SummaryWriter`.
-step: The step to write the summary for.
-tensor: A scalar string of the serialized tf.GraphDef proto.
-)doc");
+    .SetShapeFn(shape_inference::NoOutputs);
 
 }  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index ec67f43190..7ece482ea7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1358,6 +1358,12 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "summary_ops_gen",
+    visibility = ["//tensorflow:__subpackages__"],
+    deps = ["//tensorflow/core:summary_ops_op_lib"],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "audio_ops_gen",
     require_shape_functions = True,
@@ -4110,6 +4116,7 @@ py_library(
         ":pywrap_tensorflow",
         ":summary_op_util",
         ":summary_ops",
+        ":summary_ops_gen",
         ":util",
         "//tensorflow/python/eager:context",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index f1b2be0a1a..97f2ddfdfc 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -48,10 +48,12 @@ from tensorflow.core.util.event_pb2 import SessionLog
 from tensorflow.core.util.event_pb2 import TaggedRunMetadata
 # pylint: enable=unused-import
 
+
 from tensorflow.python.eager import context as _context
 from tensorflow.python.framework import dtypes as _dtypes
 from tensorflow.python.framework import ops as _ops
 from tensorflow.python.ops import gen_logging_ops as _gen_logging_ops
+from tensorflow.python.ops import gen_summary_ops as _gen_summary_ops  # pylint: disable=unused-import
 from tensorflow.python.ops import summary_op_util as _summary_op_util
 
 # exports tensor-related summaries
-- 
GitLab


From 1f4ee9d3d705a9c64af69e51e9fb5c738e145802 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Mon, 19 Mar 2018 20:42:00 -0700
Subject: [PATCH 224/960] Quantize bypasses after activations.

PiperOrigin-RevId: 189686219
---
 .../contrib/quantize/python/quantize.py       | 84 +++++++++++++++++--
 .../contrib/quantize/python/quantize_test.py  | 29 +++++++
 2 files changed, 108 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 6cc097b20e..9780e6dbcc 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -123,10 +123,47 @@ def Quantize(graph,
           vars_collection=vars_collection,
           bits=activation_bits)
 
+    if layer_match.post_activation_bypass_op is not None:
+      _InsertQuantOp(
+          add_context,
+          'post_activation_bypass_quant',
+          layer_match.post_activation_bypass_op,
+          input_to_ops_map.ConsumerOperations(
+              layer_match.post_activation_bypass_op),
+          is_training,
+          moving_avg=True,
+          ema_decay=ema_decay,
+          quant_delay=quant_delay,
+          vars_collection=vars_collection,
+          bits=activation_bits)
+
 
 def _FindLayersToQuantize(graph):
   """Matches layers in graph to quantize.
 
+  The following patterns get matched. Nodes surrounded by [] will be
+  optionally matched:
+
+          weight|folded_weight
+                /
+         conv|fc
+            |
+    [post_conv_correction]
+            |
+     biasadd|folded_bias
+            |
+         [bypass]
+            |
+        activation
+            |
+   [post_activation_bypass]
+
+  Match replacements:
+    If weight_folded_weight is found, FakeQuant is added afterwards.
+    If bypass is found, FakeQuant is added before and after.
+    If activation is found, FakeQuant is added afterwards.
+    If post_activation_bypass is found, FakeQuant is added afterwards.
+
   Args:
     graph: Graph to perform match on.
 
@@ -179,7 +216,7 @@ def _FindLayersToQuantize(graph):
               [bias_add_pattern, folded_bias_add_pattern])
       ])
 
-  # The input to the activation can come from bias add, fold bias add or the
+  # The input to the activation can come from bias add, fold bias add, the
   # bypasses.
   activation_pattern = graph_matcher.OpTypePattern(
       '|'.join(_ACTIVATION_TYPES),
@@ -190,7 +227,16 @@ def _FindLayersToQuantize(graph):
           ])
       ])
 
-  layer_matcher = graph_matcher.GraphMatcher(activation_pattern)
+  post_activation_bypass_pattern_a = graph_matcher.OpTypePattern(
+      'Add', inputs=['*', activation_pattern])
+  post_activation_bypass_pattern_b = graph_matcher.OpTypePattern(
+      'Add', inputs=[activation_pattern, '*'])
+
+  layer_matcher = graph_matcher.GraphMatcher(
+      graph_matcher.OneofPattern([
+          post_activation_bypass_pattern_a, post_activation_bypass_pattern_b,
+          activation_pattern
+      ]))
   for match_result in layer_matcher.match_graph(graph):
     layer_op = match_result.get_op(layer_pattern)
     weight_tensor = match_result.get_tensor(weight_pattern)
@@ -203,8 +249,19 @@ def _FindLayersToQuantize(graph):
     bypass_op = match_result.get_op(bypass_pattern_a)
     if bypass_op is None:
       bypass_op = match_result.get_op(bypass_pattern_b)
+    post_activation_bypass_op = match_result.get_op(
+        post_activation_bypass_pattern_a)
+    if post_activation_bypass_op is None:
+      post_activation_bypass_op = match_result.get_op(
+          post_activation_bypass_pattern_b)
+    # If we don't find a post_activation_bypass_op but activation_op has a
+    # bypass following it, then we need to skip this match, since there will be
+    # another match that includes post_activation_bypass_op.
+    if post_activation_bypass_op is None and _HasPostActivationBypass(
+        activation_op):
+      continue
     yield _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op,
-                      bias_add_op)
+                      post_activation_bypass_op, bias_add_op)
 
   # Match the final layer, where there will not be an activation and instead
   # the output of the final BiasAdd must be quantized, so we treat it as the
@@ -215,19 +272,32 @@ def _FindLayersToQuantize(graph):
   for match_result in final_layer_matcher.match_graph(graph):
     layer_op = match_result.get_op(layer_pattern)
     weight_tensor = match_result.get_tensor(weight_pattern)
+    if weight_tensor is None:
+      weight_tensor = match_result.get_tensor(folded_weight_pattern)
     activation_op = match_result.get_op(bias_add_pattern)
-    yield _LayerMatch(layer_op, weight_tensor, activation_op, None, None)
+    if activation_op is None:
+      activation_op = match_result.get_op(folded_bias_add_pattern)
+    yield _LayerMatch(layer_op, weight_tensor, activation_op, None, None, None)
+
+
+def _HasPostActivationBypass(activation_op):
+  for activation_tensor in activation_op.outputs:
+    for output_op in activation_tensor.consumers():
+      if output_op.type == 'Add':
+        return True
+  return False
 
 
 class _LayerMatch(object):
   """Contains all information related to a matched Layer."""
 
   def __init__(self, layer_op, weight_tensor, activation_op, bypass_op,
-               bias_add_op):
+               post_activation_bypass_op, bias_add_op):
     self._layer_op = layer_op
     self._weight_tensor = weight_tensor
     self._activation_op = activation_op
     self._bypass_op = bypass_op
+    self._post_activation_bypass_op = post_activation_bypass_op
     self._bias_add_op = bias_add_op
 
   @property
@@ -246,6 +316,10 @@ class _LayerMatch(object):
   def bypass_op(self):
     return self._bypass_op
 
+  @property
+  def post_activation_bypass_op(self):
+    return self._post_activation_bypass_op
+
   @property
   def bias_add_op(self):
     return self._bias_add_op
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index ef59475167..8e60f4b661 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -135,6 +135,35 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       self.assertTrue('FakeQuantWithMinMaxVars' in
                       [op.type for op in bias_add_op.outputs[0].consumers()])
 
+  def testPostActivationBypassQuantized(self):
+    self._RunTestOverParameters(self._TestPostActivationBypassQuantized)
+
+  def _TestPostActivationBypassQuantized(self, is_training):
+    graph = ops.Graph()
+    with graph.as_default():
+      batch_size, height, width, depth = 5, 128, 128, 3
+      input1 = array_ops.zeros((batch_size, height, width, depth))
+      input2 = array_ops.zeros((batch_size, height / 2, width / 2, 32))
+      conv = conv2d(
+          input1,
+          32, [5, 5],
+          stride=2,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=array_ops.identity,
+          scope='test/test')
+      bypass_tensor = math_ops.add(conv, input2, name='test/add')
+      _ = array_ops.identity(bypass_tensor, name='test/output')
+
+      quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
+
+      # Ensure that the bypass node is preceded and followed by
+      # FakeQuantWithMinMaxVars operations.
+      self.assertTrue('FakeQuantWithMinMaxVars' in
+                      [c.type for c in bypass_tensor.consumers()])
+      self.assertTrue('FakeQuantWithMinMaxVars' in
+                      [i.op.type for i in bypass_tensor.op.inputs])
+
   def _WeightInit(self, stddev):
     """Returns truncated normal variable initializer.
 
-- 
GitLab


From 63fee8ee24dc86d4a008ae153505ff838fb38849 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 20:55:51 -0700
Subject: [PATCH 225/960] Add `ostream<<` to `tensorflow::TensorShapeBase`.

Reason: Allow `LOG(ERROR) << shape` (currently disallowed).
PiperOrigin-RevId: 189687162
---
 tensorflow/core/framework/tensor_shape.h       | 8 +++++++-
 tensorflow/core/framework/tensor_shape_test.cc | 7 +++++++
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/framework/tensor_shape.h b/tensorflow/core/framework/tensor_shape.h
index fe2ba375aa..be7e740c33 100644
--- a/tensorflow/core/framework/tensor_shape.h
+++ b/tensorflow/core/framework/tensor_shape.h
@@ -25,7 +25,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
-#include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
@@ -271,6 +271,12 @@ class TensorShapeBase : public TensorShapeRep {
   friend Status MakeShapeHelper(const T*, int64, S*);
 };
 
+/// Outputs `TensorShapeBase` to `std::ostream`.
+template <typename Shape>
+std::ostream& operator<<(std::ostream& os, const TensorShapeBase<Shape>& tsb) {
+  return os << tsb.DebugString();
+}
+
 /// Represents the shape of a Tensor.
 ///
 /// A tensor's shape is denoted by its number of dimensions and a size for each
diff --git a/tensorflow/core/framework/tensor_shape_test.cc b/tensorflow/core/framework/tensor_shape_test.cc
index d7517bb311..6329aa6d8e 100644
--- a/tensorflow/core/framework/tensor_shape_test.cc
+++ b/tensorflow/core/framework/tensor_shape_test.cc
@@ -198,6 +198,13 @@ TEST(TensorShapeTest, DataType) {
   EXPECT_EQ(TensorShapeTestHelper::data_type(&s2), DT_INVALID);
 }
 
+TEST(TensorShapeTest, ostream) {
+  TensorShape s({10, 5, 4});
+  std::stringstream ss;
+  ss << s;
+  EXPECT_EQ(ss.str(), "[10,5,4]");
+}
+
 // -----------------------------------------------------------------------
 // An old implementation of TensorShape using a different representation,
 // preserved here in the unittest to allow us to have a randomized unittest
-- 
GitLab


From ea1718feb535d4dfc47c136b5cb59cf18b77259b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 21:18:06 -0700
Subject: [PATCH 226/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 189688675
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 305 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 305 ++++++++++++++++++
 2 files changed, 610 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 85dd1a423a..992e943966 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -10867,6 +10867,14 @@ op {
     }
   }
 }
+op {
+  name: "CloseSummaryWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  is_stateful: true
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
@@ -12822,6 +12830,54 @@ op {
     }
   }
 }
+op {
+  name: "CreateSummaryDbWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "db_uri"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "experiment_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "run_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "user_name"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "CreateSummaryFileWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "logdir"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "max_queue"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "flush_millis"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "filename_suffix"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
 op {
   name: "CropAndResize"
   input_arg {
@@ -19468,6 +19524,14 @@ op {
     }
   }
 }
+op {
+  name: "FlushSummaryWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  is_stateful: true
+}
 op {
   name: "FractionalAvgPool"
   input_arg {
@@ -21770,6 +21834,18 @@ op {
     type: "string"
   }
 }
+op {
+  name: "ImportEvent"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "event"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
 op {
   name: "InTopK"
   input_arg {
@@ -62152,6 +62228,28 @@ op {
     }
   }
 }
+op {
+  name: "SummaryWriter"
+  output_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "Svd"
   input_arg {
@@ -66477,6 +66575,39 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "WriteAudioSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "tensor"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "sample_rate"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "max_outputs"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
 op {
   name: "WriteFile"
   input_arg {
@@ -66488,6 +66619,180 @@ op {
     type: DT_STRING
   }
 }
+op {
+  name: "WriteGraphSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tensor"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteHistogramSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_INT64
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteImageSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "bad_color"
+    type: DT_UINT8
+  }
+  attr {
+    name: "max_images"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_FLOAT
+        type: DT_HALF
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteScalarSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "value"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_INT64
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "summary_metadata"
+    type: DT_STRING
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  is_stateful: true
+}
 op {
   name: "ZerosLike"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 3faa4eeada..3beebdc6d4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4384,6 +4384,14 @@ op {
     }
   }
 }
+op {
+  name: "CloseSummaryWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  is_stateful: true
+}
 op {
   name: "CompareAndBitpack"
   input_arg {
@@ -5473,6 +5481,54 @@ op {
     }
   }
 }
+op {
+  name: "CreateSummaryDbWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "db_uri"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "experiment_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "run_name"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "user_name"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "CreateSummaryFileWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "logdir"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "max_queue"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "flush_millis"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "filename_suffix"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
 op {
   name: "CropAndResize"
   input_arg {
@@ -8800,6 +8856,14 @@ op {
     }
   }
 }
+op {
+  name: "FlushSummaryWriter"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  is_stateful: true
+}
 op {
   name: "FractionalAvgPool"
   input_arg {
@@ -10367,6 +10431,18 @@ op {
     type: "string"
   }
 }
+op {
+  name: "ImportEvent"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "event"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
 op {
   name: "InTopK"
   input_arg {
@@ -28659,6 +28735,28 @@ op {
     }
   }
 }
+op {
+  name: "SummaryWriter"
+  output_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "Svd"
   input_arg {
@@ -31562,6 +31660,39 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "WriteAudioSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "tensor"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "sample_rate"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "max_outputs"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
 op {
   name: "WriteFile"
   input_arg {
@@ -31573,6 +31704,180 @@ op {
     type: DT_STRING
   }
 }
+op {
+  name: "WriteGraphSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tensor"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteHistogramSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "values"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_INT64
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteImageSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "bad_color"
+    type: DT_UINT8
+  }
+  attr {
+    name: "max_images"
+    type: "int"
+    default_value {
+      i: 3
+    }
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "T"
+    type: "type"
+    default_value {
+      type: DT_FLOAT
+    }
+    allowed_values {
+      list {
+        type: DT_UINT8
+        type: DT_FLOAT
+        type: DT_HALF
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteScalarSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "value"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_INT64
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "WriteSummary"
+  input_arg {
+    name: "writer"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "step"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tensor"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "tag"
+    type: DT_STRING
+  }
+  input_arg {
+    name: "summary_metadata"
+    type: DT_STRING
+  }
+  attr {
+    name: "T"
+    type: "type"
+  }
+  is_stateful: true
+}
 op {
   name: "ZerosLike"
   input_arg {
-- 
GitLab


From 407ddd1c0539cfc5d33ab2629230eab5a958b7d4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 21:45:45 -0700
Subject: [PATCH 227/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 189690096

---
 tensorflow/go/op/wrappers.go | 12047 ++++++++++++++++-----------------
 1 file changed, 5827 insertions(+), 6220 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 469d1e9adb..e5256af1e8 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -38,188 +38,6 @@ func makeOutputList(op *tf.Operation, start int, output string) ([]tf.Output, in
 	return list, start + size, nil
 }
 
-// WriteImageSummaryAttr is an optional argument to WriteImageSummary.
-type WriteImageSummaryAttr func(optionalAttr)
-
-// WriteImageSummaryMaxImages sets the optional max_images attribute to value.
-//
-// value: Max number of batch elements to generate images for.
-// If not specified, defaults to 3
-//
-// REQUIRES: value >= 1
-func WriteImageSummaryMaxImages(value int64) WriteImageSummaryAttr {
-	return func(m optionalAttr) {
-		m["max_images"] = value
-	}
-}
-
-// Writes a `Summary` protocol buffer with images.
-//
-// The summary has up to `max_images` summary values containing images. The
-// images are built from `tensor` which must be 4-D with shape `[batch_size,
-// height, width, channels]` and where `channels` can be:
-//
-// *  1: `tensor` is interpreted as Grayscale.
-// *  3: `tensor` is interpreted as RGB.
-// *  4: `tensor` is interpreted as RGBA.
-//
-// The images have the same number of channels as the input tensor. For float
-// input, the values are normalized one image at a time to fit in the range
-// `[0, 255]`.  `uint8` values are unchanged.  The op uses two different
-// normalization algorithms:
-//
-// *  If the input values are all positive, they are rescaled so the largest one
-//    is 255.
-//
-// *  If any input value is negative, the values are shifted so input value 0.0
-//    is at 127.  They are then rescaled so that either the smallest value is 0,
-//    or the largest one is 255.
-//
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
-//
-// *  If `max_images` is 1, the summary value tag is '*tag*/image'.
-// *  If `max_images` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
-//
-// The `bad_color` argument is the color to use in the generated images for
-// non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
-// Each element must be in the range `[0, 255]` (It represents the value of a
-// pixel in the output image).  Non-finite values in the input tensor are
-// replaced by this tensor in the output image.  The default value is the color
-// red.
-//
-// Arguments:
-//	writer: A handle to a summary writer.
-//	step: The step to write the summary for.
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 4-D of shape `[batch_size, height, width, channels]` where
-// `channels` is 1, 3, or 4.
-//	bad_color: Color to use for pixels with non-finite values.
-//
-// Returns the created operation.
-func WriteImageSummary(scope *Scope, writer tf.Output, step tf.Output, tag tf.Output, tensor tf.Output, bad_color tf.Output, optional ...WriteImageSummaryAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteImageSummary",
-		Input: []tf.Input{
-			writer, step, tag, tensor, bad_color,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Outputs a `tf.Event` protocol buffer.
-//
-// When CreateSummaryDbWriter is being used, this op can be useful for
-// importing data from event logs.
-//
-// Arguments:
-//	writer: A handle to a summary writer.
-//	event: A string containing a binary-encoded tf.Event proto.
-//
-// Returns the created operation.
-func ImportEvent(scope *Scope, writer tf.Output, event tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ImportEvent",
-		Input: []tf.Input{
-			writer, event,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Outputs a `Summary` protocol buffer with a tensor.
-//
-// Arguments:
-//	writer: A handle to a summary writer.
-//	step: The step to write the summary for.
-//	tensor: A tensor to serialize.
-//	tag: The summary's tag.
-//	summary_metadata: Serialized SummaryMetadata protocol buffer containing
-// plugin-related metadata for this summary.
-//
-// Returns the created operation.
-func WriteSummary(scope *Scope, writer tf.Output, step tf.Output, tensor tf.Output, tag tf.Output, summary_metadata tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteSummary",
-		Input: []tf.Input{
-			writer, step, tensor, tag, summary_metadata,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Creates summary database writer accessible by given resource handle.
-//
-// This can be used to write tensors from the execution graph directly
-// to a database. Only SQLite is supported right now. This function
-// will create the schema if it doesn't exist. Entries in the Users,
-// Experiments, and Runs tables will be created automatically if they
-// don't already exist.
-//
-// Arguments:
-//	writer: Handle to SummaryWriter resource to overwrite.
-//	db_uri: For example "file:/tmp/foo.sqlite".
-//	experiment_name: Can't contain ASCII control characters or <>. Case
-// sensitive. If empty, then the Run will not be associated with any
-// Experiment.
-//	run_name: Can't contain ASCII control characters or <>. Case sensitive.
-// If empty, then each Tag will not be associated with any Run.
-//	user_name: Must be valid as both a DNS label and Linux username. If
-// empty, then the Experiment will not be associated with any User.
-//
-// Returns the created operation.
-func CreateSummaryDbWriter(scope *Scope, writer tf.Output, db_uri tf.Output, experiment_name tf.Output, run_name tf.Output, user_name tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "CreateSummaryDbWriter",
-		Input: []tf.Input{
-			writer, db_uri, experiment_name, run_name, user_name,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Creates a summary file writer accessible by the given resource handle.
-//
-// Arguments:
-//	writer: A handle to the summary writer resource
-//	logdir: Directory where the event file will be written.
-//	max_queue: Size of the queue of pending events and summaries.
-//	flush_millis: How often, in milliseconds, to flush the pending events and
-// summaries to disk.
-//	filename_suffix: Every event file's name is suffixed with this suffix.
-//
-// Returns the created operation.
-func CreateSummaryFileWriter(scope *Scope, writer tf.Output, logdir tf.Output, max_queue tf.Output, flush_millis tf.Output, filename_suffix tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "CreateSummaryFileWriter",
-		Input: []tf.Input{
-			writer, logdir, max_queue, flush_millis, filename_suffix,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // FakeQuantWithMinMaxVarsPerChannelGradientAttr is an optional argument to FakeQuantWithMinMaxVarsPerChannelGradient.
 type FakeQuantWithMinMaxVarsPerChannelGradientAttr func(optionalAttr)
 
@@ -509,63 +327,317 @@ func FakeQuantWithMinMaxArgs(scope *Scope, inputs tf.Output, optional ...FakeQua
 	return op.Output(0)
 }
 
-// Replaces the contents of the table with the specified keys and values.
+// Scatter `updates` into a new (initially zero) tensor according to `indices`.
 //
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
+// Creates a new tensor by applying sparse `updates` to individual
+// values or slices within a zero tensor of the given `shape` according to
+// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
+// extracts values or slices from a given tensor.
+//
+// **WARNING**: The order in which updates are applied is nondeterministic, so the
+// output will be nondeterministic if `indices` contains duplicates.
+//
+// `indices` is an integer tensor containing indices into a new tensor of shape
+// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
+//
+//     indices.shape[-1] <= shape.rank
+//
+// The last dimension of `indices` corresponds to indices into elements
+// (if `indices.shape[-1] = shape.rank`) or slices
+// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
+// `shape`.  `updates` is a tensor with shape
+//
+//     indices.shape[:-1] + shape[indices.shape[-1]:]
+//
+// The simplest form of scatter is to insert individual elements in a tensor by
+// index. For example, say we want to insert 4 scattered elements in a rank-1
+// tensor with 8 elements.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[4], [3], [1], [7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     shape = tf.constant([8])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [0, 11, 0, 10, 9, 0, 0, 12]
+//
+// We can also, insert entire slices of a higher rank tensor all at once. For
+// example, if we wanted to insert two slices in the first dimension of a
+// rank-3 tensor with two matrices of new values.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
+// </div>
+//
+// In Python, this scatter operation would look like this:
+//
+// ```python
+//     indices = tf.constant([[0], [2]])
+//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]],
+//                            [[5, 5, 5, 5], [6, 6, 6, 6],
+//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
+//     shape = tf.constant([4, 4, 4])
+//     scatter = tf.scatter_nd(indices, updates, shape)
+//     with tf.Session() as sess:
+//       print(sess.run(scatter))
+// ```
+//
+// The resulting tensor would look like this:
+//
+//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
+//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
+//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
 //
 // Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
+//	indices: Index tensor.
+//	updates: Updates to scatter into output.
+//	shape: 1-D. The shape of the resulting tensor.
 //
-// Returns the created operation.
-func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+// Returns A new tensor with the given shape and updates applied according
+// to the indices.
+func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableImportV2",
+		Type: "ScatterNd",
 		Input: []tf.Input{
-			table_handle, keys, values,
+			indices, updates, shape,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
+// QuantizeAndDequantizeV2Attr is an optional argument to QuantizeAndDequantizeV2.
+type QuantizeAndDequantizeV2Attr func(optionalAttr)
 
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// QuantizeAndDequantizeV2SignedInput sets the optional signed_input attribute to value.
 //
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
+// value: If the quantization is signed or unsigned.
+// If not specified, defaults to true
+func QuantizeAndDequantizeV2SignedInput(value bool) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["signed_input"] = value
 	}
 }
 
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// QuantizeAndDequantizeV2NumBits sets the optional num_bits attribute to value.
 //
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
+// value: The bitwidth of the quantization.
+// If not specified, defaults to 8
+func QuantizeAndDequantizeV2NumBits(value int64) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
-		m["memory_limit"] = value
+		m["num_bits"] = value
 	}
 }
 
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
+// QuantizeAndDequantizeV2RangeGiven sets the optional range_given attribute to value.
+//
+// value: If the range is given or should be computed from the tensor.
+// If not specified, defaults to false
+func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["range_given"] = value
 	}
 }
 
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
+// Quantizes then dequantizes a tensor.
+//
+// This op simulates the precision loss from the quantized forward pass by:
+// 1. Quantizing the tensor to fixed point numbers, which should match the target
+//    quantization method when it is used in inference.
+// 2. Dequantizing it back to floating point numbers for the following ops, most
+//    likely matmul.
+//
+// There are different ways to quantize. This version does not use the full range
+// of the output type, choosing to elide the lowest possible value for symmetry
+// (e.g., output range is -127 to 127, not -128 to 127 for signed 8 bit
+// quantization), so that 0.0 maps to 0.
+//
+// To perform this op, we first find the range of values in our tensor. The range
+// we use is always centered on 0, so we find m such that
+//
+// 1. m = max(abs(input_min), abs(input_max)) if range_given is true,
+// 2. m = max(abs(min_elem(input)), abs(max_elem(input))) otherwise.
+//
+// Our input tensor range is then [-m, m].
+//
+// Next, we choose our fixed-point quantization buckets, [min_fixed, max_fixed].
+// If signed_input is true, this is
+//
+//   [min_fixed, max_fixed ] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1].
+//
+// Otherwise, if signed_input is false, the fixed-point range is
+//
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1].
+//
+// From this we compute our scaling factor, s:
+//
+//   s = (max_fixed - min_fixed) / (2 * m).
+//
+// Now we can quantize and dequantize the elements of our tensor.  An element e
+// is transformed into e':
+//
+//   e' = (e * s).round_to_nearest() / s.
+//
+// Note that we have a different number of buckets in the signed vs. unsigned
+// cases.  For example, if num_bits == 8, we get 254 buckets in the signed case
+// vs. 255 in the unsigned case.
+//
+// For example, suppose num_bits = 8 and m = 1.  Then
+//
+//   [min_fixed, max_fixed] = [-127, 127], and
+//   s = (127 + 127) / 2 = 127.
+//
+// Given the vector {-1, -0.5, 0, 0.3}, this is quantized to
+// {-127, -63, 0, 38}, and dequantized to {-1, -63.0/127, 0, 38.0/127}.
+//
+// Arguments:
+//	input: Tensor to quantize and then dequantize.
+//	input_min: If range_given, this is the min of the range, otherwise this input
+// will be ignored.
+//	input_max: If range_given, this is the max of the range, otherwise this input
+// will be ignored.
+func QuantizeAndDequantizeV2(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, optional ...QuantizeAndDequantizeV2Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizeAndDequantizeV2",
+		Input: []tf.Input{
+			input, input_min, input_max,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Bitcasts a tensor from one type to another without copying data.
+//
+// Given a tensor `input`, this operation returns a tensor that has the same buffer
+// data as `input` with datatype `type`.
+//
+// If the input datatype `T` is larger than the output datatype `type` then the
+// shape changes from [...] to [..., sizeof(`T`)/sizeof(`type`)].
+//
+// If `T` is smaller than `type`, the operator requires that the rightmost
+// dimension be equal to sizeof(`type`)/sizeof(`T`). The shape then goes from
+// [..., sizeof(`type`)/sizeof(`T`)] to [...].
+//
+// *NOTE*: Bitcast is implemented as a low-level cast, so machines with different
+// endian orderings will give different results.
+func Bitcast(scope *Scope, input tf.Output, type_ tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"type": type_}
+	opspec := tf.OpSpec{
+		Type: "Bitcast",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Extract `patches` from `images` and put them in the "depth" output dimension.
+//
+// Arguments:
+//	images: 4-D Tensor with shape `[batch, in_rows, in_cols, depth]`.
+//	ksizes: The size of the sliding window for each dimension of `images`.
+//	strides: 1-D of length 4. How far the centers of two consecutive patches are in
+// the images. Must be: `[1, stride_rows, stride_cols, 1]`.
+//	rates: 1-D of length 4. Must be: `[1, rate_rows, rate_cols, 1]`. This is the
+// input stride, specifying how far two consecutive patch samples are in the
+// input. Equivalent to extracting patches with
+// `patch_sizes_eff = patch_sizes + (patch_sizes - 1) * (rates - 1)`, followed by
+// subsampling them spatially by a factor of `rates`. This is equivalent to
+// `rate` in dilated (a.k.a. Atrous) convolutions.
+//	padding: The type of padding algorithm to use.
+//
+// We specify the size-related attributes as:
+//
+// ```python
+//       ksizes = [1, ksize_rows, ksize_cols, 1]
+//       strides = [1, strides_rows, strides_cols, 1]
+//       rates = [1, rates_rows, rates_cols, 1]
+// ```
+//
+// Returns 4-D Tensor with shape `[batch, out_rows, out_cols, ksize_rows *
+// ksize_cols * depth]` containing image patches with size
+// `ksize_rows x ksize_cols x depth` vectorized in the "depth" dimension. Note
+// `out_rows` and `out_cols` are the dimensions of the output patches.
+func ExtractImagePatches(scope *Scope, images tf.Output, ksizes []int64, strides []int64, rates []int64, padding string) (patches tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksizes": ksizes, "strides": strides, "rates": rates, "padding": padding}
+	opspec := tf.OpSpec{
+		Type: "ExtractImagePatches",
+		Input: []tf.Input{
+			images,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
+
+// MapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekCapacity(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapPeekContainer(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
 func MapPeekSharedName(value string) MapPeekAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
@@ -1645,6 +1717,54 @@ func Mul(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// BiasAddAttr is an optional argument to BiasAdd.
+type BiasAddAttr func(optionalAttr)
+
+// BiasAddDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the bias tensor will be added to the last dimension
+// of the value tensor.
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// The tensor will be added to "in_channels", the third-to-the-last
+//     dimension.
+// If not specified, defaults to "NHWC"
+func BiasAddDataFormat(value string) BiasAddAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Adds `bias` to `value`.
+//
+// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
+// Broadcasting is supported, so `value` may have any number of dimensions.
+//
+// Arguments:
+//	value: Any number of dimensions.
+//	bias: 1-D with size the last dimension of `value`.
+//
+// Returns Broadcasted sum of `value` and `bias`.
+func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "BiasAdd",
+		Input: []tf.Input{
+			value, bias,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseReduceSumSparseAttr is an optional argument to SparseReduceSumSparse.
 type SparseReduceSumSparseAttr func(optionalAttr)
 
@@ -1698,109 +1818,13 @@ func SparseReduceSumSparse(scope *Scope, input_indices tf.Output, input_values t
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// BiasAddAttr is an optional argument to BiasAdd.
-type BiasAddAttr func(optionalAttr)
-
-// BiasAddDataFormat sets the optional data_format attribute to value.
+// Returns x + y element-wise.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the bias tensor will be added to the last dimension
-// of the value tensor.
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// The tensor will be added to "in_channels", the third-to-the-last
-//     dimension.
-// If not specified, defaults to "NHWC"
-func BiasAddDataFormat(value string) BiasAddAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Adds `bias` to `value`.
-//
-// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
-// Broadcasting is supported, so `value` may have any number of dimensions.
-//
-// Arguments:
-//	value: Any number of dimensions.
-//	bias: 1-D with size the last dimension of `value`.
-//
-// Returns Broadcasted sum of `value` and `bias`.
-func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "BiasAdd",
-		Input: []tf.Input{
-			value, bias,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// BiasAddGradAttr is an optional argument to BiasAddGrad.
-type BiasAddGradAttr func(optionalAttr)
-
-// BiasAddGradDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the bias tensor will be added to the last dimension
-// of the value tensor.
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// The tensor will be added to "in_channels", the third-to-the-last
-//     dimension.
-// If not specified, defaults to "NHWC"
-func BiasAddGradDataFormat(value string) BiasAddGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// The backward operation for "BiasAdd" on the "bias" tensor.
-//
-// It accumulates all the values from out_backprop into the feature dimension.
-// For NHWC data format, the feature dimension is the last. For NCHW data format,
-// the feature dimension is the third-to-last.
-//
-// Arguments:
-//	out_backprop: Any number of dimensions.
-//
-// Returns 1-D with size the feature dimension of `out_backprop`.
-func BiasAddGrad(scope *Scope, out_backprop tf.Output, optional ...BiasAddGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "BiasAddGrad",
-		Input: []tf.Input{
-			out_backprop,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns x + y element-wise.
-//
-// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
 		Type: "AddV2",
@@ -2278,125 +2302,6 @@ func CholeskyGrad(scope *Scope, l tf.Output, grad tf.Output) (output tf.Output)
 	return op.Output(0)
 }
 
-// Computes inverse hyperbolic cosine of x element-wise.
-func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Acosh",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SerializeManySparseAttr is an optional argument to SerializeManySparse.
-type SerializeManySparseAttr func(optionalAttr)
-
-// SerializeManySparseOutType sets the optional out_type attribute to value.
-//
-// value: The `dtype` to use for serialization; the supported types are `string`
-// (default) and `variant`.
-// If not specified, defaults to DT_STRING
-func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
-//
-// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
-// is treated as the minibatch dimension.  Elements of the `SparseTensor`
-// must be sorted in increasing order of this first dimension.  The serialized
-// `SparseTensor` objects going into each row of `serialized_sparse` will have
-// rank `R-1`.
-//
-// The minibatch size `N` is extracted from `sparse_shape[0]`.
-//
-// Arguments:
-//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
-func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SerializeManySparse",
-		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// TensorArrayV2Attr is an optional argument to TensorArrayV2.
-type TensorArrayV2Attr func(optionalAttr)
-
-// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
-	return func(m optionalAttr) {
-		m["element_shape"] = value
-	}
-}
-
-// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
-// If not specified, defaults to false
-func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
-	return func(m optionalAttr) {
-		m["dynamic_size"] = value
-	}
-}
-
-// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
-// If not specified, defaults to true
-func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
-	return func(m optionalAttr) {
-		m["clear_after_read"] = value
-	}
-}
-
-// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
-// If not specified, defaults to ""
-func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
-	return func(m optionalAttr) {
-		m["tensor_array_name"] = value
-	}
-}
-
-// Deprecated. Use TensorArrayV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArrayV3
-func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayV2",
-		Input: []tf.Input{
-			size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes the mean along sparse segments of a tensor.
 //
 // Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
@@ -3197,30 +3102,6 @@ func QuantizedAvgPool(scope *Scope, input tf.Output, min_input tf.Output, max_in
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Updates the table to associates keys with values.
-//
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
-//
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
-//
-// Returns the created operation.
-func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableInsertV2",
-		Input: []tf.Input{
-			table_handle, keys, values,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // FractionalAvgPoolAttr is an optional argument to FractionalAvgPool.
 type FractionalAvgPoolAttr func(optionalAttr)
 
@@ -3802,152 +3683,17 @@ func IsNan(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// FractionalAvgPoolGradAttr is an optional argument to FractionalAvgPoolGrad.
-type FractionalAvgPoolGradAttr func(optionalAttr)
-
-// FractionalAvgPoolGradOverlapping sets the optional overlapping attribute to value.
-//
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
-//
-// `index  0  1  2  3  4`
+// Computes rectified linear gradients for a Relu operation.
 //
-// `value  20 5  16 3  7`
+// Arguments:
+//	gradients: The backpropagated gradients to the corresponding Relu operation.
+//	features: The features passed as input to the corresponding Relu operation, OR
+// the outputs of that operation (both work equivalently).
 //
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [41/3, 26/3] for fractional avg pooling.
-// If not specified, defaults to false
-func FractionalAvgPoolGradOverlapping(value bool) FractionalAvgPoolGradAttr {
-	return func(m optionalAttr) {
-		m["overlapping"] = value
-	}
-}
-
-// Computes gradient of the FractionalAvgPool function.
-//
-// Unlike FractionalMaxPoolGrad, we don't need to find arg_max for
-// FractionalAvgPoolGrad, we just need to evenly back-propagate each element of
-// out_backprop to those indices that form the same pooling cell. Therefore, we
-// just need to know the shape of original input tensor, instead of the whole
-// tensor.
-//
-// Arguments:
-//	orig_input_tensor_shape: Original input tensor shape for `fractional_avg_pool`
-//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
-// w.r.t. the output of `fractional_avg_pool`.
-//	row_pooling_sequence: row pooling sequence, form pooling region with
-// col_pooling_sequence.
-//	col_pooling_sequence: column pooling sequence, form pooling region with
-// row_pooling sequence.
-//
-// Returns 4-D.  Gradients w.r.t. the input of `fractional_avg_pool`.
-func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalAvgPoolGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "FractionalAvgPoolGrad",
-		Input: []tf.Input{
-			orig_input_tensor_shape, out_backprop, row_pooling_sequence, col_pooling_sequence,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for the exponential linear (Elu) operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Elu operation.
-//	outputs: The outputs of the corresponding Elu operation.
-//
-// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
-// `gradients` otherwise.
-func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "EluGrad",
-		Input: []tf.Input{
-			gradients, outputs,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process.
-//
-// Note that the hash function may change from time to time.
-// This functionality will be deprecated and it's recommended to use
-// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
-//
-// Arguments:
-//
-//	num_buckets: The number of buckets.
-//
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
-	opspec := tf.OpSpec{
-		Type: "StringToHashBucket",
-		Input: []tf.Input{
-			string_tensor,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that contains `count` elements from the `input_dataset`.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be taken. A value of `-1` indicates that all of `input_dataset`
-// is taken.
-//
-//
-func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "TakeDataset",
-		Input: []tf.Input{
-			input_dataset, count,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes rectified linear gradients for a Relu operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Relu operation.
-//	features: The features passed as input to the corresponding Relu operation, OR
-// the outputs of that operation (both work equivalently).
-//
-// Returns `gradients * (features > 0)`.
-func ReluGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
+// Returns `gradients * (features > 0)`.
+func ReluGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
 		Type: "ReluGrad",
@@ -4273,44 +4019,6 @@ func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padd
 	return op.Output(0)
 }
 
-// Bucketizes 'input' based on 'boundaries'.
-//
-// For example, if the inputs are
-//     boundaries = [0, 10, 100]
-//     input = [[-5, 10000]
-//              [150,   10]
-//              [5,    100]]
-//
-// then the output will be
-//     output = [[0, 3]
-//               [3, 2]
-//               [1, 3]]
-//
-// Arguments:
-//	input: Any shape of Tensor contains with int or float type.
-//	boundaries: A sorted list of floats gives the boundary of the buckets.
-//
-// Returns Same shape with 'input', each value of input replaced with bucket index.
-//
-// @compatibility(numpy)
-// Equivalent to np.digitize.
-// @end_compatibility
-func Bucketize(scope *Scope, input tf.Output, boundaries []float32) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"boundaries": boundaries}
-	opspec := tf.OpSpec{
-		Type: "Bucketize",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes gradients of the maxpooling function.
 //
 // Arguments:
@@ -4717,45 +4425,47 @@ func ReaderRestoreStateV2(scope *Scope, reader_handle tf.Output, state tf.Output
 	return scope.AddOperation(opspec)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
+// MaxPoolGradAttr is an optional argument to MaxPoolGrad.
+type MaxPoolGradAttr func(optionalAttr)
 
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+// MaxPoolGradDataFormat sets the optional data_format attribute to value.
 //
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradDataFormat(value string) MaxPoolGradAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["data_format"] = value
 	}
 }
 
-// Gather specific elements from the TensorArray into output `value`.
-//
-// All elements selected by `indices` must have the same shape.
+// Computes gradients of the maxpooling function.
 //
 // Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+// Returns Gradients w.r.t. the input to `max_pool`.
+func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "MaxPoolGrad",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			orig_input, orig_output, grad,
 		},
 		Attrs: attrs,
 	}
@@ -4763,333 +4473,310 @@ func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow
 	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
-//
-// Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
+// CropAndResizeAttr is an optional argument to CropAndResize.
+type CropAndResizeAttr func(optionalAttr)
+
+// CropAndResizeMethod sets the optional method attribute to value.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
-	opspec := tf.OpSpec{
-		Type: "StringToHashBucketFast",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
+// value: A string specifying the interpolation method. Only 'bilinear' is
+// supported for now.
+// If not specified, defaults to "bilinear"
+func CropAndResizeMethod(value string) CropAndResizeAttr {
+	return func(m optionalAttr) {
+		m["method"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
+// CropAndResizeExtrapolationValue sets the optional extrapolation_value attribute to value.
 //
-// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: Value used for extrapolation, when applicable.
+// If not specified, defaults to 0
+func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
+	return func(m optionalAttr) {
+		m["extrapolation_value"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Maximum",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Outputs all keys and values in the table.
+// Extracts crops from the input image tensor and bilinearly resizes them (possibly
 //
-// Arguments:
-//	table_handle: Handle to the table.
+// with aspect ratio change) to a common output size specified by `crop_size`. This
+// is more general than the `crop_to_bounding_box` op which extracts a fixed size
+// slice from the input image and does not allow resizing or aspect ratio change.
 //
+// Returns a tensor with `crops` from the input `image` at positions defined at the
+// bounding box locations in `boxes`. The cropped boxes are all resized (with
+// bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
+// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
+// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
+// method will give identical results to using `tf.image.resize_bilinear()`
+// with `align_corners=True`.
 //
+// Arguments:
+//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+// Both `image_height` and `image_width` need to be positive.
+//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+// specifies the coordinates of a box in the `box_ind[i]` image and is specified
+// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
+// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
+// `[0, 1]` interval of normalized image height is mapped to
+// `[0, image_height - 1]` in image height coordinates. We do allow `y1` > `y2`, in
+// which case the sampled crop is an up-down flipped version of the original
+// image. The width dimension is treated similarly. Normalized coordinates
+// outside the `[0, 1]` range are allowed, in which case we use
+// `extrapolation_value` to extrapolate the input image values.
+//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
+// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
+//	crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`. All
+// cropped image patches are resized to this size. The aspect ratio of the image
+// content is not preserved. Both `crop_height` and `crop_width` need to be
+// positive.
 //
-// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
-func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+// Returns A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+func CropAndResize(scope *Scope, image tf.Output, boxes tf.Output, box_ind tf.Output, crop_size tf.Output, optional ...CropAndResizeAttr) (crops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableExportV2",
+		Type: "CropAndResize",
 		Input: []tf.Input{
-			table_handle,
+			image, boxes, box_ind, crop_size,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Real-valued fast Fourier transform.
+// Fills empty rows in the input 2-D `SparseTensor` with a default value.
 //
-// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most dimension of `input`.
+// The input `SparseTensor` is represented via the tuple of inputs
+// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
+// same `dense_shape` but with indices `output_indices` and values
+// `output_values`.
 //
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
-// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
-// followed by the `fft_length / 2` positive-frequency terms.
+// This op inserts a single entry for every row that doesn't have any values.
+// The index is created as `[row, 0, ..., 0]` and the inserted value
+// is `default_value`.
 //
-// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// For example, suppose `sp_input` has shape `[5, 6]` and non-empty values:
 //
-// Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//     [0, 1]: a
+//     [0, 3]: b
+//     [2, 0]: c
+//     [3, 1]: d
 //
-// Returns A complex64 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
-//   frequency components of its 1D Fourier transform.
+// Rows 1 and 4 are empty, so the output will be of shape `[5, 6]` with values:
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft
-// @end_compatibility
-func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RFFT",
-		Input: []tf.Input{
-			input, fft_length,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ComplexAttr is an optional argument to Complex.
-type ComplexAttr func(optionalAttr)
-
-// ComplexTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_COMPLEX64
-func ComplexTout(value tf.DataType) ComplexAttr {
-	return func(m optionalAttr) {
-		m["Tout"] = value
-	}
-}
-
-// Converts two real numbers to a complex number.
+//     [0, 1]: a
+//     [0, 3]: b
+//     [1, 0]: default_value
+//     [2, 0]: c
+//     [3, 1]: d
+//     [4, 0]: default_value
 //
-// Given a tensor `real` representing the real part of a complex number, and a
-// tensor `imag` representing the imaginary part of a complex number, this
-// operation returns complex numbers elementwise of the form \\(a + bj\\), where
-// *a* represents the `real` part and *b* represents the `imag` part.
+// The output `SparseTensor` will be in row-major order and will have the
+// same shape as the input.
 //
-// The input tensors `real` and `imag` must have the same shape.
+// This op also returns an indicator vector shaped `[dense_shape[0]]` such that
 //
-// For example:
+//     empty_row_indicator[i] = True iff row i was an empty row.
 //
-// ```
-// # tensor 'real' is [2.25, 3.25]
-// # tensor `imag` is [4.75, 5.75]
-// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
-// ```
-func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
+// And a reverse index map vector shaped `[indices.shape[0]]` that is used during
+// backpropagation,
+//
+//     reverse_index_map[j] = out_j s.t. indices[j, :] == output_indices[out_j, :]
+//
+// Arguments:
+//	indices: 2-D. the indices of the sparse tensor.
+//	values: 1-D. the values of the sparse tensor.
+//	dense_shape: 1-D. the shape of the sparse tensor.
+//	default_value: 0-D. default value to insert into location `[row, 0, ..., 0]`
+//   for rows missing from the input sparse tensor.
+// output indices: 2-D. the indices of the filled sparse tensor.
+//
+// Returns 1-D. the values of the filled sparse tensor.1-D. whether the dense row was missing in the
+// input sparse tensor.1-D. a map from the input indices to the output indices.
+func SparseFillEmptyRows(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output, default_value tf.Output) (output_indices tf.Output, output_values tf.Output, empty_row_indicator tf.Output, reverse_index_map tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Complex",
+		Type: "SparseFillEmptyRows",
 		Input: []tf.Input{
-			real, imag,
+			indices, values, dense_shape, default_value,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ImagAttr is an optional argument to Imag.
-type ImagAttr func(optionalAttr)
-
-// ImagTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ImagTout(value tf.DataType) ImagAttr {
-	return func(m optionalAttr) {
-		m["Tout"] = value
-	}
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
 }
 
-// Returns the imaginary part of a complex number.
+// Reverses specific dimensions of a tensor.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the imaginary part of each element in `input`. All
-// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part returned by this operation.
+// Given a `tensor`, and a `bool` tensor `dims` representing the dimensions
+// of `tensor`, this operation reverses each dimension i of `tensor` where
+// `dims[i]` is `True`.
+//
+// `tensor` can have up to 8 dimensions. The number of dimensions
+// of `tensor` must equal the number of elements in `dims`. In other words:
+//
+// `rank(tensor) = size(dims)`
 //
 // For example:
 //
 // ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.imag(input) ==> [4.75, 5.75]
+// # tensor 't' is [[[[ 0,  1,  2,  3],
+// #                  [ 4,  5,  6,  7],
+// #                  [ 8,  9, 10, 11]],
+// #                 [[12, 13, 14, 15],
+// #                  [16, 17, 18, 19],
+// #                  [20, 21, 22, 23]]]]
+// # tensor 't' shape is [1, 2, 3, 4]
+//
+// # 'dims' is [False, False, False, True]
+// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
+//                         [ 7,  6,  5,  4],
+//                         [ 11, 10, 9, 8]],
+//                        [[15, 14, 13, 12],
+//                         [19, 18, 17, 16],
+//                         [23, 22, 21, 20]]]]
+//
+// # 'dims' is [False, True, False, False]
+// reverse(t, dims) ==> [[[[12, 13, 14, 15],
+//                         [16, 17, 18, 19],
+//                         [20, 21, 22, 23]
+//                        [[ 0,  1,  2,  3],
+//                         [ 4,  5,  6,  7],
+//                         [ 8,  9, 10, 11]]]]
+//
+// # 'dims' is [False, False, True, False]
+// reverse(t, dims) ==> [[[[8, 9, 10, 11],
+//                         [4, 5, 6, 7],
+//                         [0, 1, 2, 3]]
+//                        [[20, 21, 22, 23],
+//                         [16, 17, 18, 19],
+//                         [12, 13, 14, 15]]]]
 // ```
-func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
+//
+// Arguments:
+//	tensor: Up to 8-D.
+//	dims: 1-D. The dimensions to reverse.
+//
+// Returns The same shape as `tensor`.
+func Reverse(scope *Scope, tensor tf.Output, dims tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Imag",
+		Type: "Reverse",
 		Input: []tf.Input{
-			input,
+			tensor, dims,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
+// Computes log softmax activations.
 //
-// The Hurwitz zeta function is defined as:
+// For each batch `i` and class `j` we have
 //
+//     logsoftmax[i, j] = logits[i, j] - log(sum(exp(logits[i])))
 //
-// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
-func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+// Arguments:
+//	logits: 2-D with shape `[batch_size, num_classes]`.
+//
+// Returns Same shape as `logits`.
+func LogSoftmax(scope *Scope, logits tf.Output) (logsoftmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Zeta",
+		Type: "LogSoftmax",
 		Input: []tf.Input{
-			x, q,
+			logits,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// LRNGradAttr is an optional argument to LRNGrad.
-type LRNGradAttr func(optionalAttr)
-
-// LRNGradDepthRadius sets the optional depth_radius attribute to value.
+// Computes the inverse permutation of a tensor.
 //
-// value: A depth radius.
-// If not specified, defaults to 5
-func LRNGradDepthRadius(value int64) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["depth_radius"] = value
-	}
-}
-
-// LRNGradBias sets the optional bias attribute to value.
+// This operation computes the inverse of an index permutation. It takes a 1-D
+// integer tensor `x`, which represents the indices of a zero-based array, and
+// swaps each value with its index position. In other words, for an output tensor
+// `y` and an input tensor `x`, this operation computes the following:
 //
-// value: An offset (usually > 0 to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNGradBias(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["bias"] = value
-	}
-}
-
-// LRNGradAlpha sets the optional alpha attribute to value.
+// `y[x[i]] = i for i in [0, 1, ..., len(x) - 1]`
 //
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNGradAlpha(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["alpha"] = value
-	}
-}
-
-// LRNGradBeta sets the optional beta attribute to value.
+// The values must include 0. There can be no duplicate values or negative values.
 //
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNGradBeta(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["beta"] = value
-	}
-}
-
-// Gradients for Local Response Normalization.
+// For example:
+//
+// ```
+// # tensor `x` is [3, 4, 0, 2, 1]
+// invert_permutation(x) ==> [2, 4, 3, 0, 1]
+// ```
 //
 // Arguments:
-//	input_grads: 4-D with shape `[batch, height, width, channels]`.
-//	input_image: 4-D with shape `[batch, height, width, channels]`.
-//	output_image: 4-D with shape `[batch, height, width, channels]`.
+//	x: 1-D.
 //
-// Returns The gradients for LRN.
-func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
+// Returns 1-D.
+func InvertPermutation(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "LRNGrad",
+		Type: "InvertPermutation",
 		Input: []tf.Input{
-			input_grads, input_image, output_image,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AnyAttr is an optional argument to Any.
-type AnyAttr func(optionalAttr)
-
-// AnyKeepDims sets the optional keep_dims attribute to value.
+// Gradient op for `MirrorPad` op. This op folds a mirror-padded tensor.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func AnyKeepDims(value bool) AnyAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the "logical or" of elements across dimensions of a tensor.
+// This operation folds the padded areas of `input` by `MirrorPad` according to the
+// `paddings` you specify. `paddings` must be the same as `paddings` argument
+// given to the corresponding `MirrorPad` op.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// The folded size of each dimension D of the output is:
+//
+// `input.dim_size(D) - paddings(D, 0) - paddings(D, 1)`
+//
+// For example:
+//
+// ```
+// # 't' is [[1, 2, 3], [4, 5, 6], [7, 8, 9]].
+// # 'paddings' is [[0, 1]], [0, 1]].
+// # 'mode' is SYMMETRIC.
+// # rank of 't' is 2.
+// pad(t, paddings) ==> [[ 1,  5]
+//                       [11, 28]]
+// ```
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	input: The input tensor to be folded.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	mode: The mode used in the `MirrorPad` op.
 //
-// Returns The reduced tensor.
-func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
+// Returns The folded tensor.
+func MirrorPadGrad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"mode": mode}
 	opspec := tf.OpSpec{
-		Type: "Any",
+		Type: "MirrorPadGrad",
 		Input: []tf.Input{
-			input, axis,
+			input, paddings,
 		},
 		Attrs: attrs,
 	}
@@ -5097,41 +4784,36 @@ func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (ou
 	return op.Output(0)
 }
 
-// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
-type ResourceApplyFtrlAttr func(optionalAttr)
+// BiasAddGradAttr is an optional argument to BiasAddGrad.
+type BiasAddGradAttr func(optionalAttr)
 
-// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
+// BiasAddGradDataFormat sets the optional data_format attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the bias tensor will be added to the last dimension
+// of the value tensor.
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// The tensor will be added to "in_channels", the third-to-the-last
+//     dimension.
+// If not specified, defaults to "NHWC"
+func BiasAddGradDataFormat(value string) BiasAddGradAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["data_format"] = value
 	}
 }
 
-// Update '*var' according to the Ftrl-proximal scheme.
+// The backward operation for "BiasAdd" on the "bias" tensor.
 //
-// accum_new = accum + grad * grad
-// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// It accumulates all the values from out_backprop into the feature dimension.
+// For NHWC data format, the feature dimension is the last. For NCHW data format,
+// the feature dimension is the third-to-last.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 regulariation. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
+//	out_backprop: Any number of dimensions.
 //
-// Returns the created operation.
-func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
+// Returns 1-D with size the feature dimension of `out_backprop`.
+func BiasAddGrad(scope *Scope, out_backprop tf.Output, optional ...BiasAddGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5140,237 +4822,272 @@ func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrl",
+		Type: "BiasAddGrad",
 		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, lr_power,
+			out_backprop,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// RandomUniformAttr is an optional argument to RandomUniform.
-type RandomUniformAttr func(optionalAttr)
+// FusedBatchNormV2Attr is an optional argument to FusedBatchNormV2.
+type FusedBatchNormV2Attr func(optionalAttr)
 
-// RandomUniformSeed sets the optional seed attribute to value.
+// FusedBatchNormV2Epsilon sets the optional epsilon attribute to value.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformSeed(value int64) RandomUniformAttr {
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormV2Epsilon(value float32) FusedBatchNormV2Attr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["epsilon"] = value
 	}
 }
 
-// RandomUniformSeed2 sets the optional seed2 attribute to value.
+// FusedBatchNormV2DataFormat sets the optional data_format attribute to value.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformSeed2(value int64) RandomUniformAttr {
+// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormV2DataFormat(value string) FusedBatchNormV2Attr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["data_format"] = value
 	}
 }
 
-// Outputs random values from a uniform distribution.
+// FusedBatchNormV2IsTraining sets the optional is_training attribute to value.
 //
-// The generated values follow a uniform distribution in the range `[0, 1)`. The
-// lower bound 0 is included in the range, while the upper bound 1 is excluded.
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormV2IsTraining(value bool) FusedBatchNormV2Attr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// Batch normalization.
+//
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	offset: A 1D Tensor for offset, to shift to the normalized x.
+//	mean: A 1D Tensor for population mean. Used for inference only;
+// must be empty for training.
+//	variance: A 1D Tensor for population variance. Used for inference only;
+// must be empty for training.
 //
-// Returns A tensor of the specified shape filled with uniform random values.
-func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
+// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
+// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
+// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
+// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
+// in the cuDNN case), to be reused in the gradient computation.
+func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormV2Attr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniform",
+		Type: "FusedBatchNormV2",
 		Input: []tf.Input{
-			shape,
+			x, scale, offset, mean, variance,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// AssertAttr is an optional argument to Assert.
-type AssertAttr func(optionalAttr)
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
 
-// AssertSummarize sets the optional summarize attribute to value.
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
 //
-// value: Print this many entries of each tensor.
-// If not specified, defaults to 3
-func AssertSummarize(value int64) AssertAttr {
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
 	return func(m optionalAttr) {
-		m["summarize"] = value
+		m["element_shape"] = value
 	}
 }
 
-// Asserts that the given condition is true.
+// Gather specific elements from the TensorArray into output `value`.
 //
-// If `condition` evaluates to false, print the list of tensors in `data`.
-// `summarize` determines how many entries of the tensors to print.
+// All elements selected by `indices` must have the same shape.
 //
 // Arguments:
-//	condition: The condition to evaluate.
-//	data: The tensors to print out when condition is false.
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// Returns the created operation.
-func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Assert",
+		Type: "TensorArrayGatherV3",
 		Input: []tf.Input{
-			condition, tf.OutputList(data),
+			handle, indices, flow_in,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// For each entry in `x`, calculates the number of `1` (on) bits in the binary
-// representation of that entry.
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
 //
-// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
-// `int32` or `int64` and perform the bitcount on the result, than to feed in
-// 8- or 16-bit inputs and then aggregate the resulting counts.
-func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
+// Arguments:
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "PopulationCount",
+		Type: "StringToHashBucketFast",
 		Input: []tf.Input{
-			x,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Split a `SparseTensor` into `num_split` tensors along one dimension.
-//
-// If the `shape[split_dim]` is not an integer multiple of `num_split`. Slices
-// `[0 : shape[split_dim] % num_split]` gets one extra dimension.
-// For example, if `split_dim = 1` and `num_split = 2` and the input is
-//
-//     input_tensor = shape = [2, 7]
-//     [    a   d e  ]
-//     [b c          ]
-//
-// Graphically the output tensors are:
-//
-//     output_tensor[0] = shape = [2, 4]
-//     [    a  ]
-//     [b c    ]
-//
-//     output_tensor[1] = shape = [2, 3]
-//     [ d e  ]
-//     [      ]
-//
-// Arguments:
-//	split_dim: 0-D.  The dimension along which to split.  Must be in the range
-// `[0, rank(shape))`.
-//	indices: 2-D tensor represents the indices of the sparse tensor.
-//	values: 1-D tensor represents the values of the sparse tensor.
-//	shape: 1-D. tensor represents the shape of the sparse tensor.
-// output indices: A list of 1-D tensors represents the indices of the output
-// sparse tensors.
-//	num_split: The number of ways to split.
+// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
 //
-// Returns A list of 1-D tensors represents the values of the output sparse
-// tensors.A list of 1-D tensors represents the shape of the output sparse
-// tensors.
-func SparseSplit(scope *Scope, split_dim tf.Output, indices tf.Output, values tf.Output, shape tf.Output, num_split int64) (output_indices []tf.Output, output_values []tf.Output, output_shape []tf.Output) {
+// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "SparseSplit",
+		Type: "Maximum",
 		Input: []tf.Input{
-			split_dim, indices, values, shape,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output_indices, idx, err = makeOutputList(op, idx, "output_indices"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	if output_values, idx, err = makeOutputList(op, idx, "output_values"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	if output_shape, idx, err = makeOutputList(op, idx, "output_shape"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	return output_indices, output_values, output_shape
+	return op.Output(0)
 }
 
-// Returns the truth value of (x < y) element-wise.
+// Real-valued fast Fourier transform.
 //
-// *NOTE*: `Less` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most dimension of `input`.
+//
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+// followed by the `fft_length / 2` positive-frequency terms.
+//
+// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
+//
+// Returns A complex64 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
+//   frequency components of its 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft
+// @end_compatibility
+func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Less",
+		Type: "RFFT",
 		Input: []tf.Input{
-			x, y,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedReluXAttr is an optional argument to QuantizedReluX.
-type QuantizedReluXAttr func(optionalAttr)
+// LRNGradAttr is an optional argument to LRNGrad.
+type LRNGradAttr func(optionalAttr)
 
-// QuantizedReluXOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
+// LRNGradDepthRadius sets the optional depth_radius attribute to value.
+//
+// value: A depth radius.
+// If not specified, defaults to 5
+func LRNGradDepthRadius(value int64) LRNGradAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
+// LRNGradBias sets the optional bias attribute to value.
 //
-// Arguments:
+// value: An offset (usually > 0 to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNGradBias(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["bias"] = value
+	}
+}
+
+// LRNGradAlpha sets the optional alpha attribute to value.
 //
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNGradAlpha(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNGradBeta sets the optional beta attribute to value.
 //
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNGradBeta(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Gradients for Local Response Normalization.
 //
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+// Arguments:
+//	input_grads: 4-D with shape `[batch, height, width, channels]`.
+//	input_image: 4-D with shape `[batch, height, width, channels]`.
+//	output_image: 4-D with shape `[batch, height, width, channels]`.
+//
+// Returns The gradients for LRN.
+func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5379,39 +5096,43 @@ func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_f
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedReluX",
+		Type: "LRNGrad",
 		Input: []tf.Input{
-			features, max_value, min_features, max_features,
+			input_grads, input_image, output_image,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// RandomPoissonAttr is an optional argument to RandomPoisson.
-type RandomPoissonAttr func(optionalAttr)
-
-// RandomPoissonSeed sets the optional seed attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed(value int64) RandomPoissonAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+// AnyAttr is an optional argument to Any.
+type AnyAttr func(optionalAttr)
 
-// RandomPoissonSeed2 sets the optional seed2 attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed2(value int64) RandomPoissonAttr {
+// AnyKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AnyKeepDims(value bool) AnyAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Use RandomPoissonV2 instead.
+// Computes the "logical or" of elements across dimensions of a tensor.
 //
-// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
-func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
+//
+// Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5420,9 +5141,9 @@ func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...Ra
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomPoisson",
+		Type: "Any",
 		Input: []tf.Input{
-			shape, rate,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -5430,28 +5151,25 @@ func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...Ra
 	return op.Output(0)
 }
 
-// ResourceSparseApplyFtrlV2Attr is an optional argument to ResourceSparseApplyFtrlV2.
-type ResourceSparseApplyFtrlV2Attr func(optionalAttr)
+// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
+type ResourceApplyFtrlAttr func(optionalAttr)
 
-// ResourceSparseApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
+// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
 //
 // value: If `True`, updating of the var and accum tensors will be protected
 // by a lock; otherwise the behavior is undefined, but may exhibit less
 // contention.
 // If not specified, defaults to false
-func ResourceSparseApplyFtrlV2UseLocking(value bool) ResourceSparseApplyFtrlV2Attr {
+func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
+// Update '*var' according to the Ftrl-proximal scheme.
 //
-// That is for rows we have grad for, we update var, accum and linear as follows:
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// accum_new = accum + grad * grad
+// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
 // quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
 // var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
 // accum = accum_new
@@ -5461,15 +5179,13 @@ func ResourceSparseApplyFtrlV2UseLocking(value bool) ResourceSparseApplyFtrlV2At
 //	accum: Should be from a Variable().
 //	linear: Should be from a Variable().
 //	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
 //	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
-//
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 regulariation. Must be a scalar.
 //	lr_power: Scaling factor. Must be a scalar.
 //
 // Returns the created operation.
-func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlV2Attr) (o *tf.Operation) {
+func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5478,92 +5194,93 @@ func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, li
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyFtrlV2",
+		Type: "ResourceApplyFtrl",
 		Input: []tf.Input{
-			var_, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power,
+			var_, accum, linear, grad, lr, l1, l2, lr_power,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Associates the given iterator with the given statistics aggregator.
+// RandomUniformAttr is an optional argument to RandomUniform.
+type RandomUniformAttr func(optionalAttr)
+
+// RandomUniformSeed sets the optional seed attribute to value.
 //
-// Returns the created operation.
-func IteratorSetStatsAggregator(scope *Scope, iterator_handle tf.Output, stats_aggregator_handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IteratorSetStatsAggregator",
-		Input: []tf.Input{
-			iterator_handle, stats_aggregator_handle,
-		},
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformSeed(value int64) RandomUniformAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Returns element-wise smallest integer in not less than x.
-func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Ceil",
-		Input: []tf.Input{
-			x,
-		},
+// RandomUniformSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformSeed2(value int64) RandomUniformAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the number of elements in the given table.
+// Outputs random values from a uniform distribution.
+//
+// The generated values follow a uniform distribution in the range `[0, 1)`. The
+// lower bound 0 is included in the range, while the upper bound 1 is excluded.
 //
 // Arguments:
-//	table_handle: Handle to the table.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns Scalar that contains number of elements in the table.
-func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
+// Returns A tensor of the specified shape filled with uniform random values.
+func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableSizeV2",
+		Type: "RandomUniform",
 		Input: []tf.Input{
-			table_handle,
+			shape,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
-type ResizeBilinearGradAttr func(optionalAttr)
+// AssertAttr is an optional argument to Assert.
+type AssertAttr func(optionalAttr)
 
-// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
+// AssertSummarize sets the optional summarize attribute to value.
 //
-// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of grads and original_image. If false, rescale by
-// orig_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
+// value: Print this many entries of each tensor.
+// If not specified, defaults to 3
+func AssertSummarize(value int64) AssertAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["summarize"] = value
 	}
 }
 
-// Computes the gradient of bilinear interpolation.
+// Asserts that the given condition is true.
+//
+// If `condition` evaluates to false, print the list of tensors in `data`.
+// `summarize` determines how many entries of the tensors to print.
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
+//	condition: The condition to evaluate.
+//	data: The tensors to print out when condition is false.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
+// Returns the created operation.
+func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5572,71 +5289,126 @@ func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBilinearGrad",
+		Type: "Assert",
 		Input: []tf.Input{
-			grads, original_image,
+			condition, tf.OutputList(data),
 		},
 		Attrs: attrs,
 	}
+	return scope.AddOperation(opspec)
+}
+
+// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
+//
+// For each entry in `x`, calculates the number of `1` (on) bits in the binary
+// representation of that entry.
+//
+// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
+// `int32` or `int64` and perform the bitcount on the result, than to feed in
+// 8- or 16-bit inputs and then aggregate the resulting counts.
+func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "PopulationCount",
+		Input: []tf.Input{
+			x,
+		},
+	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
+// Split a `SparseTensor` into `num_split` tensors along one dimension.
 //
-// N is the size of the segment being reduced.
+// If the `shape[split_dim]` is not an integer multiple of `num_split`. Slices
+// `[0 : shape[split_dim] % num_split]` gets one extra dimension.
+// For example, if `split_dim = 1` and `num_split = 2` and the input is
 //
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+//     input_tensor = shape = [2, 7]
+//     [    a   d e  ]
+//     [b c          ]
 //
-// Arguments:
+// Graphically the output tensors are:
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//     output_tensor[0] = shape = [2, 4]
+//     [    a  ]
+//     [b c    ]
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+//     output_tensor[1] = shape = [2, 3]
+//     [ d e  ]
+//     [      ]
+//
+// Arguments:
+//	split_dim: 0-D.  The dimension along which to split.  Must be in the range
+// `[0, rank(shape))`.
+//	indices: 2-D tensor represents the indices of the sparse tensor.
+//	values: 1-D tensor represents the values of the sparse tensor.
+//	shape: 1-D. tensor represents the shape of the sparse tensor.
+// output indices: A list of 1-D tensors represents the indices of the output
+// sparse tensors.
+//	num_split: The number of ways to split.
+//
+// Returns A list of 1-D tensors represents the values of the output sparse
+// tensors.A list of 1-D tensors represents the shape of the output sparse
+// tensors.
+func SparseSplit(scope *Scope, split_dim tf.Output, indices tf.Output, values tf.Output, shape tf.Output, num_split int64) (output_indices []tf.Output, output_values []tf.Output, output_shape []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtN",
+		Type: "SparseSplit",
 		Input: []tf.Input{
-			data, indices, segment_ids,
+			split_dim, indices, values, shape,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output_indices, idx, err = makeOutputList(op, idx, "output_indices"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
+	}
+	if output_values, idx, err = makeOutputList(op, idx, "output_values"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
+	}
+	if output_shape, idx, err = makeOutputList(op, idx, "output_shape"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
+	}
+	return output_indices, output_values, output_shape
 }
 
-// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
-type StatelessTruncatedNormalAttr func(optionalAttr)
+// RandomPoissonAttr is an optional argument to RandomPoisson.
+type RandomPoissonAttr func(optionalAttr)
 
-// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
-//
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
+// RandomPoissonSeed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func RandomPoissonSeed(value int64) RandomPoissonAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["seed"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom values from a truncated normal distribution.
-//
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
-//
-// The outputs are a deterministic function of `shape` and `seed`.
-//
-// Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+// RandomPoissonSeed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func RandomPoissonSeed2(value int64) RandomPoissonAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Use RandomPoissonV2 instead.
 //
-// Returns Random values with specified shape.
-func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
+// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
+func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5645,9 +5417,9 @@ func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, opt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessTruncatedNormal",
+		Type: "RandomPoisson",
 		Input: []tf.Input{
-			shape, seed,
+			shape, rate,
 		},
 		Attrs: attrs,
 	}
@@ -5655,94 +5427,111 @@ func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, opt
 	return op.Output(0)
 }
 
-// RestoreSliceAttr is an optional argument to RestoreSlice.
-type RestoreSliceAttr func(optionalAttr)
+// ResourceSparseApplyFtrlV2Attr is an optional argument to ResourceSparseApplyFtrlV2.
+type ResourceSparseApplyFtrlV2Attr func(optionalAttr)
 
-// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
+// ResourceSparseApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
 //
-// value: Index of file to open first if multiple files match
-// `file_pattern`. See the documentation for `Restore`.
-// If not specified, defaults to -1
-func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyFtrlV2UseLocking(value bool) ResourceSparseApplyFtrlV2Attr {
 	return func(m optionalAttr) {
-		m["preferred_shard"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Restores a tensor from checkpoint files.
-//
-// This is like `Restore` except that restored tensor can be listed as filling
-// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
-// larger tensor and the slice that the restored tensor covers.
+// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
 //
-// The `shape_and_slice` input has the same format as the
-// elements of the `shapes_and_slices` input of the `SaveSlices` op.
+// That is for rows we have grad for, we update var, accum and linear as follows:
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
-//	file_pattern: Must have a single element. The pattern of the files from
-// which we read the tensor.
-//	tensor_name: Must have a single element. The name of the tensor to be
-// restored.
-//	shape_and_slice: Scalar. The shapes and slice specifications to use when
-// restoring a tensors.
-//	dt: The type of the tensor to be restored.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
 //
-// Returns The restored tensor.
-func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dt": dt}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RestoreSlice",
+		Type: "ResourceSparseApplyFtrlV2",
 		Input: []tf.Input{
-			file_pattern, tensor_name, shape_and_slice,
+			var_, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
-type UniqueWithCountsAttr func(optionalAttr)
+// Associates the given iterator with the given statistics aggregator.
+//
+// Returns the created operation.
+func IteratorSetStatsAggregator(scope *Scope, iterator_handle tf.Output, stats_aggregator_handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IteratorSetStatsAggregator",
+		Input: []tf.Input{
+			iterator_handle, stats_aggregator_handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
 
-// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
+// DataFormatVecPermuteAttr is an optional argument to DataFormatVecPermute.
+type DataFormatVecPermuteAttr func(optionalAttr)
+
+// DataFormatVecPermuteSrcFormat sets the optional src_format attribute to value.
+//
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatVecPermuteSrcFormat(value string) DataFormatVecPermuteAttr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["src_format"] = value
 	}
 }
 
-// Finds unique elements in a 1-D tensor.
-//
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. Finally, it returns a third tensor `count` that
-// contains the count of each element of `y` in `x`. In other words:
-//
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+// DataFormatVecPermuteDstFormat sets the optional dst_format attribute to value.
 //
-// For example:
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatVecPermuteDstFormat(value string) DataFormatVecPermuteAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the permuted vector/tensor in the destination data format given the
 //
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx, count = unique_with_counts(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// count ==> [2, 1, 3, 1, 2]
-// ```
+// one in the source data format.
 //
 // Arguments:
-//	x: 1-D.
+//	x: Vector of size 4 or Tensor of shape (4, 2) in source data format.
 //
-// Returns 1-D.1-D.1-D.
-func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
+// Returns Vector of size 4 or Tensor of shape (4, 2) in destination data format.
+func DataFormatVecPermute(scope *Scope, x tf.Output, optional ...DataFormatVecPermuteAttr) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5751,32 +5540,77 @@ func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UniqueWithCounts",
+		Type: "DataFormatVecPermute",
 		Input: []tf.Input{
 			x,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// StatelessRandomNormalAttr is an optional argument to StatelessRandomNormal.
-type StatelessRandomNormalAttr func(optionalAttr)
+// Computes tan of x element-wise.
+func Tan(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Tan",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// StatelessRandomNormalDtype sets the optional dtype attribute to value.
+// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
+//
+// N is the size of the segment being reduced.
+//
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
+//
+// Arguments:
+//
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseSegmentSqrtN",
+		Input: []tf.Input{
+			data, indices, segment_ids,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
+type StatelessTruncatedNormalAttr func(optionalAttr)
+
+// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
 //
 // value: The type of the output.
 // If not specified, defaults to DT_FLOAT
-func StatelessRandomNormalDtype(value tf.DataType) StatelessRandomNormalAttr {
+func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
 	return func(m optionalAttr) {
 		m["dtype"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom values from a normal distribution.
+// Outputs deterministic pseudorandom values from a truncated normal distribution.
 //
-// The generated values will have mean 0 and standard deviation 1.
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
 // The outputs are a deterministic function of `shape` and `seed`.
 //
@@ -5785,7 +5619,7 @@ func StatelessRandomNormalDtype(value tf.DataType) StatelessRandomNormalAttr {
 //	seed: 2 seeds (shape [2]).
 //
 // Returns Random values with specified shape.
-func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomNormalAttr) (output tf.Output) {
+func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5794,7 +5628,7 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessRandomNormal",
+		Type: "StatelessTruncatedNormal",
 		Input: []tf.Input{
 			shape, seed,
 		},
@@ -5804,49 +5638,287 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 	return op.Output(0)
 }
 
-// Reshapes a quantized tensor as per the Reshape op.
+// RestoreSliceAttr is an optional argument to RestoreSlice.
+type RestoreSliceAttr func(optionalAttr)
+
+// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
 //
-// ```
+// value: Index of file to open first if multiple files match
+// `file_pattern`. See the documentation for `Restore`.
+// If not specified, defaults to -1
+func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
+	return func(m optionalAttr) {
+		m["preferred_shard"] = value
+	}
+}
+
+// Restores a tensor from checkpoint files.
 //
-// Arguments:
+// This is like `Restore` except that restored tensor can be listed as filling
+// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
+// larger tensor and the slice that the restored tensor covers.
 //
-//	shape: Defines the shape of the output tensor.
-//	input_min: The minimum value of the input.
-//	input_max: The maximum value of the input.
+// The `shape_and_slice` input has the same format as the
+// elements of the `shapes_and_slices` input of the `SaveSlices` op.
 //
-// Returns This value is copied from input_min.This value is copied from input_max.
-func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Arguments:
+//	file_pattern: Must have a single element. The pattern of the files from
+// which we read the tensor.
+//	tensor_name: Must have a single element. The name of the tensor to be
+// restored.
+//	shape_and_slice: Scalar. The shapes and slice specifications to use when
+// restoring a tensors.
+//	dt: The type of the tensor to be restored.
+//
+// Returns The restored tensor.
+func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dt": dt}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedReshape",
+		Type: "RestoreSlice",
 		Input: []tf.Input{
-			tensor, shape, input_min, input_max,
+			file_pattern, tensor_name, shape_and_slice,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// GatherAttr is an optional argument to Gather.
-type GatherAttr func(optionalAttr)
+// ImagAttr is an optional argument to Imag.
+type ImagAttr func(optionalAttr)
 
-// GatherValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func GatherValidateIndices(value bool) GatherAttr {
+// ImagTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ImagTout(value tf.DataType) ImagAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["Tout"] = value
 	}
 }
 
-// Gather slices from `params` according to `indices`.
+// Returns the imaginary part of a complex number.
 //
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the imaginary part of each element in `input`. All
+// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part returned by this operation.
 //
-// ```python
-//     # Scalar indices
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.imag(input) ==> [4.75, 5.75]
+// ```
+func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Imag",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ComplexAttr is an optional argument to Complex.
+type ComplexAttr func(optionalAttr)
+
+// ComplexTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_COMPLEX64
+func ComplexTout(value tf.DataType) ComplexAttr {
+	return func(m optionalAttr) {
+		m["Tout"] = value
+	}
+}
+
+// Converts two real numbers to a complex number.
+//
+// Given a tensor `real` representing the real part of a complex number, and a
+// tensor `imag` representing the imaginary part of a complex number, this
+// operation returns complex numbers elementwise of the form \\(a + bj\\), where
+// *a* represents the `real` part and *b* represents the `imag` part.
+//
+// The input tensors `real` and `imag` must have the same shape.
+//
+// For example:
+//
+// ```
+// # tensor 'real' is [2.25, 3.25]
+// # tensor `imag` is [4.75, 5.75]
+// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
+// ```
+func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Complex",
+		Input: []tf.Input{
+			real, imag,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
+type UniqueWithCountsAttr func(optionalAttr)
+
+// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Finds unique elements in a 1-D tensor.
+//
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. Finally, it returns a third tensor `count` that
+// contains the count of each element of `y` in `x`. In other words:
+//
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+//
+// For example:
+//
+// ```
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx, count = unique_with_counts(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+// count ==> [2, 1, 3, 1, 2]
+// ```
+//
+// Arguments:
+//	x: 1-D.
+//
+// Returns 1-D.1-D.1-D.
+func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UniqueWithCounts",
+		Input: []tf.Input{
+			x,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// StatelessRandomNormalAttr is an optional argument to StatelessRandomNormal.
+type StatelessRandomNormalAttr func(optionalAttr)
+
+// StatelessRandomNormalDtype sets the optional dtype attribute to value.
+//
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessRandomNormalDtype(value tf.DataType) StatelessRandomNormalAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs deterministic pseudorandom values from a normal distribution.
+//
+// The generated values will have mean 0 and standard deviation 1.
+//
+// The outputs are a deterministic function of `shape` and `seed`.
+//
+// Arguments:
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
+//
+// Returns Random values with specified shape.
+func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomNormalAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StatelessRandomNormal",
+		Input: []tf.Input{
+			shape, seed,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Reshapes a quantized tensor as per the Reshape op.
+//
+// ```
+//
+// Arguments:
+//
+//	shape: Defines the shape of the output tensor.
+//	input_min: The minimum value of the input.
+//	input_max: The maximum value of the input.
+//
+// Returns This value is copied from input_min.This value is copied from input_max.
+func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizedReshape",
+		Input: []tf.Input{
+			tensor, shape, input_min, input_max,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// GatherAttr is an optional argument to Gather.
+type GatherAttr func(optionalAttr)
+
+// GatherValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func GatherValidateIndices(value bool) GatherAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Gather slices from `params` according to `indices`.
+//
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
+//
+// ```python
+//     # Scalar indices
 //     output[:, ..., :] = params[indices, :, ... :]
 //
 //     # Vector indices
@@ -6008,44 +6080,33 @@ func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ..
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// WriteAudioSummaryAttr is an optional argument to WriteAudioSummary.
-type WriteAudioSummaryAttr func(optionalAttr)
+// ResizeBilinearAttr is an optional argument to ResizeBilinear.
+type ResizeBilinearAttr func(optionalAttr)
 
-// WriteAudioSummaryMaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
+// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
 //
-// REQUIRES: value >= 1
-func WriteAudioSummaryMaxOutputs(value int64) WriteAudioSummaryAttr {
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
 	return func(m optionalAttr) {
-		m["max_outputs"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Writes a `Summary` protocol buffer with audio.
-//
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
-//
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// Resize `images` to `size` using bilinear interpolation.
 //
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+// Input images can be of different types but output images are always float.
 //
 // Arguments:
-//	writer: A handle to a summary writer.
-//	step: The step to write the summary for.
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns the created operation.
-func WriteAudioSummary(scope *Scope, writer tf.Output, step tf.Output, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...WriteAudioSummaryAttr) (o *tf.Operation) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6054,103 +6115,14 @@ func WriteAudioSummary(scope *Scope, writer tf.Output, step tf.Output, tag tf.Ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "WriteAudioSummary",
+		Type: "ResizeBilinear",
 		Input: []tf.Input{
-			writer, step, tag, tensor, sample_rate,
+			images, size,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// ProdAttr is an optional argument to Prod.
-type ProdAttr func(optionalAttr)
-
-// ProdKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func ProdKeepDims(value bool) ProdAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the product of elements across dimensions of a tensor.
-//
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
-//
-// Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
-//
-// Returns The reduced tensor.
-func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Prod",
-		Input: []tf.Input{
-			input, axis,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResizeBilinearAttr is an optional argument to ResizeBilinear.
-type ResizeBilinearAttr func(optionalAttr)
-
-// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
-//
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
-	}
-}
-
-// Resize `images` to `size` using bilinear interpolation.
-//
-// Input images can be of different types but output images are always float.
-//
-// Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
-//
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResizeBilinear",
-		Input: []tf.Input{
-			images, size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
 // Computes softsign: `features / (abs(features) + 1)`.
@@ -6482,6 +6454,83 @@ func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...Fix
 	return op.Output(0)
 }
 
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
+//
+// The hash function is deterministic on the content of the string within the
+// process.
+//
+// Note that the hash function may change from time to time.
+// This functionality will be deprecated and it's recommended to use
+// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
+//
+// Arguments:
+//
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
+	opspec := tf.OpSpec{
+		Type: "StringToHashBucket",
+		Input: []tf.Input{
+			string_tensor,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes gradients for the exponential linear (Elu) operation.
+//
+// Arguments:
+//	gradients: The backpropagated gradients to the corresponding Elu operation.
+//	outputs: The outputs of the corresponding Elu operation.
+//
+// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
+// `gradients` otherwise.
+func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "EluGrad",
+		Input: []tf.Input{
+			gradients, outputs,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that contains `count` elements from the `input_dataset`.
+//
+// Arguments:
+//
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be taken. A value of `-1` indicates that all of `input_dataset`
+// is taken.
+//
+//
+func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "TakeDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // The gradient operator for the SparseAdd op.
 //
 // The SparseAdd op calculates A + B, where A, B, and the sum are all represented
@@ -6667,72 +6716,6 @@ func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyReso
 	return scope.AddOperation(opspec)
 }
 
-// SummaryWriterAttr is an optional argument to SummaryWriter.
-type SummaryWriterAttr func(optionalAttr)
-
-// SummaryWriterSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func SummaryWriterSharedName(value string) SummaryWriterAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// SummaryWriterContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func SummaryWriterContainer(value string) SummaryWriterAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// Returns a handle to be used to access a summary writer.
-//
-// The summary writer is an in-graph resource which can be used by ops to write
-// summaries to event files.
-//
-// Returns the summary writer resource. Scalar handle.
-func SummaryWriter(scope *Scope, optional ...SummaryWriterAttr) (writer tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SummaryWriter",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for SparseSegmentMean.
-//
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
-//
-// Arguments:
-//	grad: gradient propagated to the SparseSegmentMean op.
-//	indices: indices passed to the corresponding SparseSegmentMean op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
-func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanGrad",
-		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Applies softmax to a batched N-D `SparseTensor`.
 //
 // The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
@@ -6886,33 +6869,82 @@ func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.O
 	return scope.AddOperation(opspec)
 }
 
-// CumprodAttr is an optional argument to Cumprod.
-type CumprodAttr func(optionalAttr)
+// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
+type ResourceApplyPowerSignAttr func(optionalAttr)
 
-// CumprodExclusive sets the optional exclusive attribute to value.
+// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
 //
-// value: If `True`, perform exclusive cumprod.
+// value: If `True`, updating of the var and m tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
 // If not specified, defaults to false
-func CumprodExclusive(value bool) CumprodAttr {
+func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
 	return func(m optionalAttr) {
-		m["exclusive"] = value
+		m["use_locking"] = value
 	}
 }
 
-// CumprodReverse sets the optional reverse attribute to value.
+// Update '*var' according to the AddSign update.
 //
-// value: A `bool` (default: False).
-// If not specified, defaults to false
-func CumprodReverse(value bool) CumprodAttr {
-	return func(m optionalAttr) {
-		m["reverse"] = value
-	}
-}
-
-// Compute the cumulative product of the tensor `x` along `axis`.
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
+// variable <- variable - lr_t * update
 //
-// By default, this op performs an inclusive cumprod, which means that the first
-// element of the input is identical to the first element of the output:
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	logbase: Must be a scalar.
+//	sign_decay: Must be a scalar.
+//	beta: Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyPowerSign",
+		Input: []tf.Input{
+			var_, m, lr, logbase, sign_decay, beta, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// CumprodAttr is an optional argument to Cumprod.
+type CumprodAttr func(optionalAttr)
+
+// CumprodExclusive sets the optional exclusive attribute to value.
+//
+// value: If `True`, perform exclusive cumprod.
+// If not specified, defaults to false
+func CumprodExclusive(value bool) CumprodAttr {
+	return func(m optionalAttr) {
+		m["exclusive"] = value
+	}
+}
+
+// CumprodReverse sets the optional reverse attribute to value.
+//
+// value: A `bool` (default: False).
+// If not specified, defaults to false
+func CumprodReverse(value bool) CumprodAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Compute the cumulative product of the tensor `x` along `axis`.
+//
+// By default, this op performs an inclusive cumprod, which means that the first
+// element of the input is identical to the first element of the output:
 //
 // ```python
 // tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
@@ -7214,27 +7246,6 @@ func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.Data
 	return op.Output(0)
 }
 
-// Writes a `GraphDef` protocol buffer to a `SummaryWriter`.
-//
-// Arguments:
-//	writer: Handle of `SummaryWriter`.
-//	step: The step to write the summary for.
-//	tensor: A scalar string of the serialized tf.GraphDef proto.
-//
-// Returns the created operation.
-func WriteGraphSummary(scope *Scope, writer tf.Output, step tf.Output, tensor tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteGraphSummary",
-		Input: []tf.Input{
-			writer, step, tensor,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
 type ResourceSparseApplyAdagradAttr func(optionalAttr)
 
@@ -8379,6 +8390,136 @@ func Erf(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// OneHotAttr is an optional argument to OneHot.
+type OneHotAttr func(optionalAttr)
+
+// OneHotAxis sets the optional axis attribute to value.
+//
+// value: The axis to fill (default: -1, a new inner-most axis).
+// If not specified, defaults to -1
+func OneHotAxis(value int64) OneHotAttr {
+	return func(m optionalAttr) {
+		m["axis"] = value
+	}
+}
+
+// Returns a one-hot tensor.
+//
+// The locations represented by indices in `indices` take value `on_value`,
+// while all other locations take value `off_value`.
+//
+// If the input `indices` is rank `N`, the output will have rank `N+1`,
+// The new axis is created at dimension `axis` (default: the new axis is
+// appended at the end).
+//
+// If `indices` is a scalar the output shape will be a vector of length `depth`.
+//
+// If `indices` is a vector of length `features`, the output shape will be:
+// ```
+//   features x depth if axis == -1
+//   depth x features if axis == 0
+// ```
+//
+// If `indices` is a matrix (batch) with shape `[batch, features]`,
+// the output shape will be:
+// ```
+//   batch x features x depth if axis == -1
+//   batch x depth x features if axis == 1
+//   depth x batch x features if axis == 0
+// ```
+//
+//
+// Examples
+// =========
+//
+// Suppose that
+//
+// ```
+//   indices = [0, 2, -1, 1]
+//   depth = 3
+//   on_value = 5.0
+//   off_value = 0.0
+//   axis = -1
+// ```
+//
+// Then output is `[4 x 3]`:
+//
+//     ```output =
+//       [5.0 0.0 0.0]  // one_hot(0)
+//       [0.0 0.0 5.0]  // one_hot(2)
+//       [0.0 0.0 0.0]  // one_hot(-1)
+//       [0.0 5.0 0.0]  // one_hot(1)
+//     ```
+//
+// Suppose that
+//
+// ```
+//   indices = [0, 2, -1, 1]
+//   depth = 3
+//   on_value = 0.0
+//   off_value = 3.0
+//   axis = 0
+// ```
+//
+// Then output is `[3 x 4]`:
+//
+//     ```output =
+//       [0.0 3.0 3.0 3.0]
+//       [3.0 3.0 3.0 0.0]
+//       [3.0 3.0 3.0 3.0]
+//       [3.0 0.0 3.0 3.0]
+//     //  ^                one_hot(0)
+//     //      ^            one_hot(2)
+//     //          ^        one_hot(-1)
+//     //              ^    one_hot(1)
+//     ```
+// Suppose that
+//
+// ```
+//   indices = [[0, 2], [1, -1]]
+//   depth = 3
+//   on_value = 1.0
+//   off_value = 0.0
+//   axis = -1
+// ```
+//
+// Then output is `[2 x 2 x 3]`:
+//
+//     ```output =
+//       [
+//         [1.0, 0.0, 0.0]  // one_hot(0)
+//         [0.0, 0.0, 1.0]  // one_hot(2)
+//       ][
+//         [0.0, 1.0, 0.0]  // one_hot(1)
+//         [0.0, 0.0, 0.0]  // one_hot(-1)
+//       ]```
+//
+// Arguments:
+//	indices: A tensor of indices.
+//	depth: A scalar defining the depth of the one hot dimension.
+//	on_value: A scalar defining the value to fill in output when `indices[j] = i`.
+//	off_value: A scalar defining the value to fill in output when `indices[j] != i`.
+//
+// Returns The one-hot tensor.
+func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output, off_value tf.Output, optional ...OneHotAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "OneHot",
+		Input: []tf.Input{
+			indices, depth, on_value, off_value,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Reads the value of a variable.
 //
 // The tensor returned by this operation is immutable.
@@ -8691,269 +8832,36 @@ func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr)
 	return op.Output(0)
 }
 
-// Fills empty rows in the input 2-D `SparseTensor` with a default value.
-//
-// The input `SparseTensor` is represented via the tuple of inputs
-// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
-// same `dense_shape` but with indices `output_indices` and values
-// `output_values`.
-//
-// This op inserts a single entry for every row that doesn't have any values.
-// The index is created as `[row, 0, ..., 0]` and the inserted value
-// is `default_value`.
-//
-// For example, suppose `sp_input` has shape `[5, 6]` and non-empty values:
-//
-//     [0, 1]: a
-//     [0, 3]: b
-//     [2, 0]: c
-//     [3, 1]: d
-//
-// Rows 1 and 4 are empty, so the output will be of shape `[5, 6]` with values:
-//
-//     [0, 1]: a
-//     [0, 3]: b
-//     [1, 0]: default_value
-//     [2, 0]: c
-//     [3, 1]: d
-//     [4, 0]: default_value
-//
-// The output `SparseTensor` will be in row-major order and will have the
-// same shape as the input.
-//
-// This op also returns an indicator vector shaped `[dense_shape[0]]` such that
-//
-//     empty_row_indicator[i] = True iff row i was an empty row.
+// Computes softmax cross entropy cost and gradients to backpropagate.
 //
-// And a reverse index map vector shaped `[indices.shape[0]]` that is used during
-// backpropagation,
+// Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
+// a matrix of label probabilities, but rather a single label per row
+// of features.  This label is considered to have probability 1.0 for the
+// given row.
 //
-//     reverse_index_map[j] = out_j s.t. indices[j, :] == output_indices[out_j, :]
+// Inputs are the logits, not probabilities.
 //
 // Arguments:
-//	indices: 2-D. the indices of the sparse tensor.
-//	values: 1-D. the values of the sparse tensor.
-//	dense_shape: 1-D. the shape of the sparse tensor.
-//	default_value: 0-D. default value to insert into location `[row, 0, ..., 0]`
-//   for rows missing from the input sparse tensor.
-// output indices: 2-D. the indices of the filled sparse tensor.
+//	features: batch_size x num_classes matrix
+//	labels: batch_size vector with values in [0, num_classes).
+// This is the label for the given minibatch entry.
 //
-// Returns 1-D. the values of the filled sparse tensor.1-D. whether the dense row was missing in the
-// input sparse tensor.1-D. a map from the input indices to the output indices.
-func SparseFillEmptyRows(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output, default_value tf.Output) (output_indices tf.Output, output_values tf.Output, empty_row_indicator tf.Output, reverse_index_map tf.Output) {
+// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
+func SparseSoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseFillEmptyRows",
+		Type: "SparseSoftmaxCrossEntropyWithLogits",
 		Input: []tf.Input{
-			indices, values, dense_shape, default_value,
+			features, labels,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+	return op.Output(0), op.Output(1)
 }
 
-// Reverses specific dimensions of a tensor.
-//
-// Given a `tensor`, and a `bool` tensor `dims` representing the dimensions
-// of `tensor`, this operation reverses each dimension i of `tensor` where
-// `dims[i]` is `True`.
-//
-// `tensor` can have up to 8 dimensions. The number of dimensions
-// of `tensor` must equal the number of elements in `dims`. In other words:
-//
-// `rank(tensor) = size(dims)`
-//
-// For example:
-//
-// ```
-// # tensor 't' is [[[[ 0,  1,  2,  3],
-// #                  [ 4,  5,  6,  7],
-// #                  [ 8,  9, 10, 11]],
-// #                 [[12, 13, 14, 15],
-// #                  [16, 17, 18, 19],
-// #                  [20, 21, 22, 23]]]]
-// # tensor 't' shape is [1, 2, 3, 4]
-//
-// # 'dims' is [False, False, False, True]
-// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
-//                         [ 7,  6,  5,  4],
-//                         [ 11, 10, 9, 8]],
-//                        [[15, 14, 13, 12],
-//                         [19, 18, 17, 16],
-//                         [23, 22, 21, 20]]]]
-//
-// # 'dims' is [False, True, False, False]
-// reverse(t, dims) ==> [[[[12, 13, 14, 15],
-//                         [16, 17, 18, 19],
-//                         [20, 21, 22, 23]
-//                        [[ 0,  1,  2,  3],
-//                         [ 4,  5,  6,  7],
-//                         [ 8,  9, 10, 11]]]]
-//
-// # 'dims' is [False, False, True, False]
-// reverse(t, dims) ==> [[[[8, 9, 10, 11],
-//                         [4, 5, 6, 7],
-//                         [0, 1, 2, 3]]
-//                        [[20, 21, 22, 23],
-//                         [16, 17, 18, 19],
-//                         [12, 13, 14, 15]]]]
-// ```
-//
-// Arguments:
-//	tensor: Up to 8-D.
-//	dims: 1-D. The dimensions to reverse.
-//
-// Returns The same shape as `tensor`.
-func Reverse(scope *Scope, tensor tf.Output, dims tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Reverse",
-		Input: []tf.Input{
-			tensor, dims,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes log softmax activations.
-//
-// For each batch `i` and class `j` we have
-//
-//     logsoftmax[i, j] = logits[i, j] - log(sum(exp(logits[i])))
-//
-// Arguments:
-//	logits: 2-D with shape `[batch_size, num_classes]`.
-//
-// Returns Same shape as `logits`.
-func LogSoftmax(scope *Scope, logits tf.Output) (logsoftmax tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LogSoftmax",
-		Input: []tf.Input{
-			logits,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the inverse permutation of a tensor.
-//
-// This operation computes the inverse of an index permutation. It takes a 1-D
-// integer tensor `x`, which represents the indices of a zero-based array, and
-// swaps each value with its index position. In other words, for an output tensor
-// `y` and an input tensor `x`, this operation computes the following:
-//
-// `y[x[i]] = i for i in [0, 1, ..., len(x) - 1]`
-//
-// The values must include 0. There can be no duplicate values or negative values.
-//
-// For example:
-//
-// ```
-// # tensor `x` is [3, 4, 0, 2, 1]
-// invert_permutation(x) ==> [2, 4, 3, 0, 1]
-// ```
-//
-// Arguments:
-//	x: 1-D.
-//
-// Returns 1-D.
-func InvertPermutation(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "InvertPermutation",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Gradient op for `MirrorPad` op. This op folds a mirror-padded tensor.
-//
-// This operation folds the padded areas of `input` by `MirrorPad` according to the
-// `paddings` you specify. `paddings` must be the same as `paddings` argument
-// given to the corresponding `MirrorPad` op.
-//
-// The folded size of each dimension D of the output is:
-//
-// `input.dim_size(D) - paddings(D, 0) - paddings(D, 1)`
-//
-// For example:
-//
-// ```
-// # 't' is [[1, 2, 3], [4, 5, 6], [7, 8, 9]].
-// # 'paddings' is [[0, 1]], [0, 1]].
-// # 'mode' is SYMMETRIC.
-// # rank of 't' is 2.
-// pad(t, paddings) ==> [[ 1,  5]
-//                       [11, 28]]
-// ```
-//
-// Arguments:
-//	input: The input tensor to be folded.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	mode: The mode used in the `MirrorPad` op.
-//
-// Returns The folded tensor.
-func MirrorPadGrad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"mode": mode}
-	opspec := tf.OpSpec{
-		Type: "MirrorPadGrad",
-		Input: []tf.Input{
-			input, paddings,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes softmax cross entropy cost and gradients to backpropagate.
-//
-// Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
-// a matrix of label probabilities, but rather a single label per row
-// of features.  This label is considered to have probability 1.0 for the
-// given row.
-//
-// Inputs are the logits, not probabilities.
-//
-// Arguments:
-//	features: batch_size x num_classes matrix
-//	labels: batch_size vector with values in [0, num_classes).
-// This is the label for the given minibatch entry.
-//
-// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
-func SparseSoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSoftmaxCrossEntropyWithLogits",
-		Input: []tf.Input{
-			features, labels,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Fast Fourier transform.
+// Fast Fourier transform.
 //
 // Computes the 1-dimensional discrete Fourier transform over the inner-most
 // dimension of `input`.
@@ -9367,173 +9275,37 @@ func MutableHashTableOfTensorsV2(scope *Scope, key_dtype tf.DataType, value_dtyp
 	return op.Output(0)
 }
 
-// HashTableV2Attr is an optional argument to HashTableV2.
-type HashTableV2Attr func(optionalAttr)
-
-// HashTableV2Container sets the optional container attribute to value.
+// Inverse 2D fast Fourier transform.
 //
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func HashTableV2Container(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// HashTableV2SharedName sets the optional shared_name attribute to value.
+// Computes the inverse 2-dimensional discrete Fourier transform over the
+// inner-most 2 dimensions of `input`.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func HashTableV2SharedName(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// Arguments:
+//	input: A complex64 tensor.
 //
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifft2
+// @end_compatibility
+func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IFFT2D",
+		Input: []tf.Input{
+			input,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Creates a non-initialized hash table.
+// Creates a tensor filled with a scalar value.
 //
-// This op creates a hash table, specifying the type of its keys and values.
-// Before using the table you will have to initialize it.  After initialization the
-// table will be immutable.
-//
-// Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
-//
-// Returns Handle to a table.
-func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "HashTableV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
-type MapUnstageNoKeyAttr func(optionalAttr)
-
-// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageNoKeyContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns a random (key, value)
-//
-// from the underlying container.   If the underlying container
-// does not contain elements, the op will block until it does.
-func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapUnstageNoKey",
-		Input: []tf.Input{
-			indices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	key = op.Output(idx)
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstageNoKey", err)
-		return
-	}
-	return key, values
-}
-
-// Inverse 2D fast Fourier transform.
-//
-// Computes the inverse 2-dimensional discrete Fourier transform over the
-// inner-most 2 dimensions of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.ifft2
-// @end_compatibility
-func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IFFT2D",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a tensor filled with a scalar value.
-//
-// This operation creates a tensor of shape `dims` and fills it with `value`.
+// This operation creates a tensor of shape `dims` and fills it with `value`.
 //
 // For example:
 //
@@ -9940,55 +9712,53 @@ func TopK(scope *Scope, input tf.Output, k int64, optional ...TopKAttr) (values
 	return op.Output(0), op.Output(1)
 }
 
-// Transforms a Tensor into a serialized TensorProto proto.
+// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
 //
-// Arguments:
-//	tensor: A Tensor of type `T`.
+// The Hurwitz zeta function is defined as:
 //
-// Returns A serialized TensorProto proto of the input tensor.
-func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
+//
+// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
+func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SerializeTensor",
+		Type: "Zeta",
 		Input: []tf.Input{
-			tensor,
+			x, q,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixSolveAttr is an optional argument to MatrixSolve.
-type MatrixSolveAttr func(optionalAttr)
+// ProdAttr is an optional argument to Prod.
+type ProdAttr func(optionalAttr)
 
-// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+// ProdKeepDims sets the optional keep_dims attribute to value.
 //
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-// adjoint.
+// value: If true, retain reduced dimensions with length 1.
 // If not specified, defaults to false
-func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
+func ProdKeepDims(value bool) ProdAttr {
 	return func(m optionalAttr) {
-		m["adjoint"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Solves systems of linear equations.
+// Computes the product of elements across dimensions of a tensor.
 //
-// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
-// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
-// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `True` then each output matrix satisfies
-// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
+// Returns The reduced tensor.
+func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9997,9 +9767,9 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixSolve",
+		Type: "Prod",
 		Input: []tf.Input{
-			matrix, rhs,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -10007,65 +9777,162 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 	return op.Output(0)
 }
 
-// Looks up keys in a table, outputs the corresponding values.
+// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
+type FusedResizeAndPadConv2DAttr func(optionalAttr)
+
+// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
 //
-// The tensor `keys` must of the same type as the keys of the table.
-// The output `values` is of the type of the table values.
+// value: If true, rescale input by (new_height - 1) / (height - 1),
+// which exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
+	return func(m optionalAttr) {
+		m["resize_align_corners"] = value
+	}
+}
+
+// Performs a resize and padding as a preprocess during a convolution.
 //
-// The scalar `default_value` is the value output for keys not present in the
-// table. It must also be of the same type as the table values.
+// It's often possible to do spatial transformations more efficiently as part of
+// the packing stage of a convolution, so this op allows for an optimized
+// implementation where these stages are fused together. This prevents the need to
+// write out the intermediate results as whole tensors, reducing memory pressure,
+// and we can get some latency gains by merging the transformation calculations.
+// The data_format attribute for Conv2D isn't supported by this op, and defaults to
+// 'NHWC' order.
+// Internally this op uses a single per-graph scratch buffer, which means that it
+// will block if multiple versions are being run in parallel. This is because this
+// operator is primarily an optimization to minimize memory usage.
 //
 // Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//
+//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
+//	size: A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.
 //
-// Returns Same shape as `keys`.  Values found in the table, or `default_values`
-// for missing keys.
-func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
+//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
+// of `input`. Must be in the same order as the dimension specified with format.
+//	padding: The type of padding algorithm to use.
+func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string, optional ...FusedResizeAndPadConv2DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableFindV2",
+		Type: "FusedResizeAndPadConv2D",
 		Input: []tf.Input{
-			table_handle, keys, default_value,
+			input, size, paddings, filter,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse 3D fast Fourier transform.
-//
-// Computes the inverse 3-dimensional discrete Fourier transform over the
-// inner-most 3 dimensions of `input`.
+// Transforms a Tensor into a serialized TensorProto proto.
 //
 // Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
-//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+//	tensor: A Tensor of type `T`.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.ifftn with 3 dimensions.
-// @end_compatibility
-func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns A serialized TensorProto proto of the input tensor.
+func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IFFT3D",
+		Type: "SerializeTensor",
 		Input: []tf.Input{
-			input,
+			tensor,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Adds `bias` to `value`.
-//
+// MatrixSolveAttr is an optional argument to MatrixSolve.
+type MatrixSolveAttr func(optionalAttr)
+
+// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+//
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+// adjoint.
+// If not specified, defaults to false
+func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
+	return func(m optionalAttr) {
+		m["adjoint"] = value
+	}
+}
+
+// Solves systems of linear equations.
+//
+// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `True` then each output matrix satisfies
+// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
+//
+// Returns Shape is `[..., M, K]`.
+func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixSolve",
+		Input: []tf.Input{
+			matrix, rhs,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Inverse 3D fast Fourier transform.
+//
+// Computes the inverse 3-dimensional discrete Fourier transform over the
+// inner-most 3 dimensions of `input`.
+//
+// Arguments:
+//	input: A complex64 tensor.
+//
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
+//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifftn with 3 dimensions.
+// @end_compatibility
+func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IFFT3D",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Adds `bias` to `value`.
+//
 // This is a deprecated version of BiasAdd and will be soon removed.
 //
 // This is a special case of `tf.add` where `bias` is restricted to be 1-D.
@@ -11123,174 +10990,157 @@ func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_s
 	return op.Output(0)
 }
 
-// Flushes the writer's unwritten events.
-//
-// Arguments:
-//	writer: A handle to the summary writer resource.
+// Computes sigmoid of `x` element-wise.
 //
-// Returns the created operation.
-func FlushSummaryWriter(scope *Scope, writer tf.Output) (o *tf.Operation) {
+// Specifically, `y = 1 / (1 + exp(-x))`.
+func Sigmoid(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "FlushSummaryWriter",
+		Type: "Sigmoid",
 		Input: []tf.Input{
-			writer,
+			x,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// QuantizeV2Attr is an optional argument to QuantizeV2.
-type QuantizeV2Attr func(optionalAttr)
+// FusedBatchNormAttr is an optional argument to FusedBatchNorm.
+type FusedBatchNormAttr func(optionalAttr)
 
-// QuantizeV2Mode sets the optional mode attribute to value.
-// If not specified, defaults to "MIN_COMBINED"
-func QuantizeV2Mode(value string) QuantizeV2Attr {
+// FusedBatchNormEpsilon sets the optional epsilon attribute to value.
+//
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormEpsilon(value float32) FusedBatchNormAttr {
 	return func(m optionalAttr) {
-		m["mode"] = value
+		m["epsilon"] = value
 	}
 }
 
-// QuantizeV2RoundMode sets the optional round_mode attribute to value.
-// If not specified, defaults to "HALF_AWAY_FROM_ZERO"
-func QuantizeV2RoundMode(value string) QuantizeV2Attr {
+// FusedBatchNormDataFormat sets the optional data_format attribute to value.
+//
+// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormDataFormat(value string) FusedBatchNormAttr {
 	return func(m optionalAttr) {
-		m["round_mode"] = value
+		m["data_format"] = value
 	}
 }
 
-// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
-//
-// [min_range, max_range] are scalar floats that specify the range for
-// the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.  The
-// 'round_mode' attribute controls which rounding tie-breaking algorithm is used
-// when rounding float values to their quantized equivalents.
-//
-// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
-//
-// ```
-// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
-// if T == qint8, out[i] -= (range(T) + 1) / 2.0
-// ```
-// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
-//
-// *MIN_COMBINED Mode Example*
-//
-// Assume the input is type float and has a possible range of [0.0, 6.0] and the
-// output type is quint8 ([0, 255]). The min_range and max_range values should be
-// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each
-// value of the input by 255/6 and cast to quint8.
-//
-// If the output type was qint8 ([-128, 127]), the operation will additionally
-// subtract each value by 128 prior to casting, so that the range of values aligns
-// with the range of qint8.
-//
-// If the mode is 'MIN_FIRST', then this approach is used:
-//
-// ```
-// num_discrete_values = 1 << (# of bits in T)
-// range_adjust = num_discrete_values / (num_discrete_values - 1)
-// range = (range_max - range_min) * range_adjust
-// range_scale = num_discrete_values / range
-// quantized = round(input * range_scale) - round(range_min * range_scale) +
-//   numeric_limits<T>::min()
-// quantized = max(quantized, numeric_limits<T>::min())
-// quantized = min(quantized, numeric_limits<T>::max())
-// ```
-//
-// The biggest difference between this and MIN_COMBINED is that the minimum range
-// is rounded first, before it's subtracted from the rounded value. With
-// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing
-// and dequantizing will introduce a larger and larger error.
-//
-// *SCALED mode Example*
-//
-// `SCALED` mode matches the quantization approach used in
-// `QuantizeAndDequantize{V2|V3}`.
-//
-// If the mode is `SCALED`, we do not use the full range of the output type,
-// choosing to elide the lowest possible value for symmetry (e.g., output range is
-// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
-// 0.
-//
-// We first find the range of values in our tensor. The
-// range we use is always centered on 0, so we find m such that
-// ```c++
-//   m = max(abs(input_min), abs(input_max))
-// ```
-//
-// Our input tensor range is then `[-m, m]`.
-//
-// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
-// If T is signed, this is
-// ```
-//   num_bits = sizeof(T) * 8
-//   [min_fixed, max_fixed] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
-// ```
-//
-// Otherwise, if T is unsigned, the fixed-point range is
-// ```
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
-// ```
-//
-// From this we compute our scaling factor, s:
-// ```c++
-//   s = (max_fixed - min_fixed) / (2 * m)
-// ```
+// FusedBatchNormIsTraining sets the optional is_training attribute to value.
 //
-// Now we can quantize the elements of our tensor:
-// ```c++
-// result = round(input * s)
-// ```
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormIsTraining(value bool) FusedBatchNormAttr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// Batch normalization.
 //
-// One thing to watch out for is that the operator may choose to adjust the
-// requested minimum and maximum values slightly during the quantization process,
-// so you should always use the output ports as the range for further calculations.
-// For example, if the requested minimum and maximum values are close to equal,
-// they will be separated by a small epsilon value to prevent ill-formed quantized
-// buffers from being created. Otherwise, you can end up with buffers where all the
-// quantized values map to the same float value, which causes problems for
-// operations that have to perform further calculations on them.
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
 //
 // Arguments:
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	offset: A 1D Tensor for offset, to shift to the normalized x.
+//	mean: A 1D Tensor for population mean. Used for inference only;
+// must be empty for training.
+//	variance: A 1D Tensor for population variance. Used for inference only;
+// must be empty for training.
 //
-//	min_range: The minimum scalar value possibly produced for the input.
-//	max_range: The maximum scalar value possibly produced for the input.
-//
-//
-// Returns The quantized data produced from the float input.The actual minimum scalar value used for the output.The actual maximum scalar value used for the output.
-func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
+// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
+// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
+// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
+// in the cuDNN case), to be reused in the gradient computation.
+func FusedBatchNorm(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormAttr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"T": T}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeV2",
+		Type: "FusedBatchNorm",
 		Input: []tf.Input{
-			input, min_range, max_range,
+			x, scale, offset, mean, variance,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// Component-wise divides a SparseTensor by a dense Tensor.
-//
-// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
-// the other direction.
+// RandomStandardNormalAttr is an optional argument to RandomStandardNormal.
+type RandomStandardNormalAttr func(optionalAttr)
+
+// RandomStandardNormalSeed sets the optional seed attribute to value.
 //
-// Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomStandardNormalSeed(value int64) RandomStandardNormalAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomStandardNormalSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomStandardNormalSeed2(value int64) RandomStandardNormalAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from a normal distribution.
+//
+// The generated values will have mean 0 and standard deviation 1.
+//
+// Arguments:
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
+//
+// Returns A tensor of the specified shape filled with random normal values.
+func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomStandardNormalAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "RandomStandardNormal",
+		Input: []tf.Input{
+			shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Component-wise divides a SparseTensor by a dense Tensor.
+//
+// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+// the other direction.
+//
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
 //	sp_shape: 1-D.  Shape of the input SparseTensor.
 //	dense: `R`-D.  The dense Tensor operand.
 //
@@ -11309,6 +11159,89 @@ func SparseDenseCwiseDiv(scope *Scope, sp_indices tf.Output, sp_values tf.Output
 	return op.Output(0)
 }
 
+// FractionalAvgPoolGradAttr is an optional argument to FractionalAvgPoolGrad.
+type FractionalAvgPoolGradAttr func(optionalAttr)
+
+// FractionalAvgPoolGradOverlapping sets the optional overlapping attribute to value.
+//
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
+//
+// `index  0  1  2  3  4`
+//
+// `value  20 5  16 3  7`
+//
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [41/3, 26/3] for fractional avg pooling.
+// If not specified, defaults to false
+func FractionalAvgPoolGradOverlapping(value bool) FractionalAvgPoolGradAttr {
+	return func(m optionalAttr) {
+		m["overlapping"] = value
+	}
+}
+
+// Computes gradient of the FractionalAvgPool function.
+//
+// Unlike FractionalMaxPoolGrad, we don't need to find arg_max for
+// FractionalAvgPoolGrad, we just need to evenly back-propagate each element of
+// out_backprop to those indices that form the same pooling cell. Therefore, we
+// just need to know the shape of original input tensor, instead of the whole
+// tensor.
+//
+// Arguments:
+//	orig_input_tensor_shape: Original input tensor shape for `fractional_avg_pool`
+//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
+// w.r.t. the output of `fractional_avg_pool`.
+//	row_pooling_sequence: row pooling sequence, form pooling region with
+// col_pooling_sequence.
+//	col_pooling_sequence: column pooling sequence, form pooling region with
+// row_pooling sequence.
+//
+// Returns 4-D.  Gradients w.r.t. the input of `fractional_avg_pool`.
+func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalAvgPoolGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "FractionalAvgPoolGrad",
+		Input: []tf.Input{
+			orig_input_tensor_shape, out_backprop, row_pooling_sequence, col_pooling_sequence,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Concatenates tensors along one dimension.
+//
+// Arguments:
+//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [0, rank(values)).
+//	values: The `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.
+func Concat(scope *Scope, concat_dim tf.Output, values []tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Concat",
+		Input: []tf.Input{
+			concat_dim, tf.OutputList(values),
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // ResourceApplyMomentumAttr is an optional argument to ResourceApplyMomentum.
 type ResourceApplyMomentumAttr func(optionalAttr)
 
@@ -11417,327 +11350,86 @@ func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output,
 	return op.Output(0)
 }
 
-// Returns the truth value of (x >= y) element-wise.
+// Returns element-wise integer closest to x.
 //
-// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// If the result is midway between two representable values,
+// the even representable is chosen.
+// For example:
+//
+// ```
+// rint(-1.5) ==> -2.0
+// rint(0.5000001) ==> 1.0
+// rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.]
+// ```
+func Rint(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "GreaterEqual",
+		Type: "Rint",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv3DAttr is an optional argument to Conv3D.
-type Conv3DAttr func(optionalAttr)
+// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey.
+type OrderedMapUnstageNoKeyAttr func(optionalAttr)
 
-// Conv3DDataFormat sets the optional data_format attribute to value.
+// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func Conv3DDataFormat(value string) Conv3DAttr {
+// REQUIRES: value >= 0
+func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["capacity"] = value
 	}
 }
 
-// Conv3DDilations sets the optional dilations attribute to value.
+// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// value: 1-D tensor of length 5.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DDilations(value []int64) Conv3DAttr {
+// REQUIRES: value >= 0
+func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["memory_limit"] = value
 	}
 }
 
-// Computes a 3-D convolution given 5-D `input` and `filter` tensors.
-//
-// In signal processing, cross-correlation is a measure of similarity of
-// two waveforms as a function of a time-lag applied to one of them. This
-// is also known as a sliding dot product or sliding inner-product.
-//
-// Our Conv3D implements a form of cross-correlation.
+// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns the (key, value) element with the smallest
 //
-// Arguments:
-//	input: Shape `[batch, in_depth, in_height, in_width, in_channels]`.
-//	filter: Shape `[filter_depth, filter_height, filter_width, in_channels,
-// out_channels]`. `in_channels` must match between `input` and `filter`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv3DAttr) (output tf.Output) {
+// key from the underlying container.   If the underlying container
+// does not contain elements, the op will block until it does.
+func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv3D",
+		Type: "OrderedMapUnstageNoKey",
 		Input: []tf.Input{
-			input, filter,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Adds up a SparseTensor and a dense Tensor, using these special rules:
-//
-// (1) Broadcasts the dense side to have the same shape as the sparse side, if
-//     eligible;
-// (2) Then, only the dense values pointed to by the indices of the SparseTensor
-//     participate in the cwise addition.
-//
-// By these rules, the result is a logical SparseTensor with exactly the same
-// indices and shape, but possibly with different non-zero values.  The output of
-// this Op is the resultant non-zero values.
-//
-// Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//	dense: `R`-D.  The dense Tensor operand.
-//
-// Returns 1-D.  The `N` values that are operated on.
-func SparseDenseCwiseAdd(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseDenseCwiseAdd",
-		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape, dense,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Read an element from the TensorArray into output `value`.
-//
-// Arguments:
-//	handle: The handle to a TensorArray.
-//
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
-//
-// Returns The tensor that is read from the TensorArray.
-func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV3",
-		Input: []tf.Input{
-			handle, index, flow_in,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// EncodePngAttr is an optional argument to EncodePng.
-type EncodePngAttr func(optionalAttr)
-
-// EncodePngCompression sets the optional compression attribute to value.
-//
-// value: Compression level.
-// If not specified, defaults to -1
-func EncodePngCompression(value int64) EncodePngAttr {
-	return func(m optionalAttr) {
-		m["compression"] = value
-	}
-}
-
-// PNG-encode an image.
-//
-// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
-// where `channels` is:
-//
-// *   1: for grayscale.
-// *   2: for grayscale + alpha.
-// *   3: for RGB.
-// *   4: for RGBA.
-//
-// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
-// default or a value from 0 to 9.  9 is the highest compression level, generating
-// the smallest output, but is slower.
-//
-// Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
-//
-// Returns 0-D. PNG-encoded image.
-func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "EncodePng",
-		Input: []tf.Input{
-			image,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// DataFormatVecPermuteAttr is an optional argument to DataFormatVecPermute.
-type DataFormatVecPermuteAttr func(optionalAttr)
-
-// DataFormatVecPermuteSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatVecPermuteSrcFormat(value string) DataFormatVecPermuteAttr {
-	return func(m optionalAttr) {
-		m["src_format"] = value
-	}
-}
-
-// DataFormatVecPermuteDstFormat sets the optional dst_format attribute to value.
-//
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatVecPermuteDstFormat(value string) DataFormatVecPermuteAttr {
-	return func(m optionalAttr) {
-		m["dst_format"] = value
-	}
-}
-
-// Returns the permuted vector/tensor in the destination data format given the
-//
-// one in the source data format.
-//
-// Arguments:
-//	x: Vector of size 4 or Tensor of shape (4, 2) in source data format.
-//
-// Returns Vector of size 4 or Tensor of shape (4, 2) in destination data format.
-func DataFormatVecPermute(scope *Scope, x tf.Output, optional ...DataFormatVecPermuteAttr) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DataFormatVecPermute",
-		Input: []tf.Input{
-			x,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns element-wise integer closest to x.
-//
-// If the result is midway between two representable values,
-// the even representable is chosen.
-// For example:
-//
-// ```
-// rint(-1.5) ==> -2.0
-// rint(0.5000001) ==> 1.0
-// rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.]
-// ```
-func Rint(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Rint",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey.
-type OrderedMapUnstageNoKeyAttr func(optionalAttr)
-
-// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the (key, value) element with the smallest
-//
-// key from the underlying container.   If the underlying container
-// does not contain elements, the op will block until it does.
-func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OrderedMapUnstageNoKey",
-		Input: []tf.Input{
-			indices,
+			indices,
 		},
 		Attrs: attrs,
 	}
@@ -11909,35 +11601,154 @@ func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.Dat
 	return outputs
 }
 
-// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
-type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+// SerializeManySparseAttr is an optional argument to SerializeManySparse.
+type SerializeManySparseAttr func(optionalAttr)
 
-// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+// SerializeManySparseOutType sets the optional out_type attribute to value.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+// value: The `dtype` to use for serialization; the supported types are `string`
+// (default) and `variant`.
+// If not specified, defaults to DT_STRING
+func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["out_type"] = value
 	}
 }
 
-// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Generates labels for candidate sampling with a learned unigram distribution.
+// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
+// is treated as the minibatch dimension.  Elements of the `SparseTensor`
+// must be sorted in increasing order of this first dimension.  The serialized
+// `SparseTensor` objects going into each row of `serialized_sparse` will have
+// rank `R-1`.
 //
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
+// The minibatch size `N` is extracted from `sparse_shape[0]`.
+//
+// Arguments:
+//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
+func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SerializeManySparse",
+		Input: []tf.Input{
+			sparse_indices, sparse_values, sparse_shape,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes inverse hyperbolic cosine of x element-wise.
+func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Acosh",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// TensorArrayV2Attr is an optional argument to TensorArrayV2.
+type TensorArrayV2Attr func(optionalAttr)
+
+// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
+// If not specified, defaults to false
+func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["dynamic_size"] = value
+	}
+}
+
+// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
+// If not specified, defaults to true
+func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["clear_after_read"] = value
+	}
+}
+
+// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
+// If not specified, defaults to ""
+func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["tensor_array_name"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayV3
+//
+// DEPRECATED at GraphDef version 26: Use TensorArrayV3
+func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayV2",
+		Input: []tf.Input{
+			size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
+type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+
+// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Generates labels for candidate sampling with a learned unigram distribution.
+//
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
 //
 // For each batch, this op picks a single set of sampled candidate labels.
 //
@@ -12455,60 +12266,106 @@ func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
-type SparseTensorDenseMatMulAttr func(optionalAttr)
+// Computes gradients for SparseSegmentMean.
+//
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
+//
+// Arguments:
+//	grad: gradient propagated to the SparseSegmentMean op.
+//	indices: indices passed to the corresponding SparseSegmentMean op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
+func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseSegmentMeanGrad",
+		Input: []tf.Input{
+			grad, indices, segment_ids, output_dim0,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
+// Returns the truth value of (x >= y) element-wise.
 //
-// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
-// is transpose(conj(A)).  Otherwise it's transpose(A).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
+// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GreaterEqual",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Conv3DAttr is an optional argument to Conv3D.
+type Conv3DAttr func(optionalAttr)
+
+// Conv3DDataFormat sets the optional data_format attribute to value.
+//
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func Conv3DDataFormat(value string) Conv3DAttr {
 	return func(m optionalAttr) {
-		m["adjoint_a"] = value
+		m["data_format"] = value
 	}
 }
 
-// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
+// Conv3DDilations sets the optional dilations attribute to value.
 //
-// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
-// is transpose(conj(B)).  Otherwise it's transpose(B).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
+// value: 1-D tensor of length 5.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
+func Conv3DDilations(value []int64) Conv3DAttr {
 	return func(m optionalAttr) {
-		m["adjoint_b"] = value
+		m["dilations"] = value
 	}
 }
 
-// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
+// Computes a 3-D convolution given 5-D `input` and `filter` tensors.
 //
-// No validity checking is performed on the indices of A.  However, the following
-// input format is recommended for optimal behavior:
+// In signal processing, cross-correlation is a measure of similarity of
+// two waveforms as a function of a time-lag applied to one of them. This
+// is also known as a sliding dot product or sliding inner-product.
 //
-// if adjoint_a == false:
-//   A should be sorted in lexicographically increasing order.  Use SparseReorder
-//   if you're not sure.
-// if adjoint_a == true:
-//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
-//   order instead of "row major" order).
+// Our Conv3D implements a form of cross-correlation.
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
-//	b: 2-D.  A dense Matrix.
-func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
+//	input: Shape `[batch, in_depth, in_height, in_width, in_channels]`.
+//	filter: Shape `[filter_depth, filter_height, filter_width, in_channels,
+// out_channels]`. `in_channels` must match between `input` and `filter`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv3DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseMatMul",
+		Type: "Conv3D",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
+			input, filter,
 		},
 		Attrs: attrs,
 	}
@@ -12516,235 +12373,254 @@ func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Outp
 	return op.Output(0)
 }
 
-// Deserialize and concatenate `SparseTensors` from a serialized minibatch.
+// Adds up a SparseTensor and a dense Tensor, using these special rules:
 //
-// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
-// `N` is the minibatch size and the rows correspond to packed outputs of
-// `SerializeSparse`.  The ranks of the original `SparseTensor` objects
-// must all match.  When the final `SparseTensor` is created, it has rank one
-// higher than the ranks of the incoming `SparseTensor` objects
-// (they have been concatenated along a new row dimension).
+// (1) Broadcasts the dense side to have the same shape as the sparse side, if
+//     eligible;
+// (2) Then, only the dense values pointed to by the indices of the SparseTensor
+//     participate in the cwise addition.
 //
-// The output `SparseTensor` object's shape values for all dimensions but the
-// first are the max across the input `SparseTensor` objects' shape values
-// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
-// size.
+// By these rules, the result is a logical SparseTensor with exactly the same
+// indices and shape, but possibly with different non-zero values.  The output of
+// this Op is the resultant non-zero values.
 //
-// The input `SparseTensor` objects' indices are assumed ordered in
-// standard lexicographic order.  If this is not the case, after this
-// step run `SparseReorder` to restore index ordering.
-//
-// For example, if the serialized input is a `[2 x 3]` matrix representing two
-// original `SparseTensor` objects:
-//
-//     index = [ 0]
-//             [10]
-//             [20]
-//     values = [1, 2, 3]
-//     shape = [50]
-//
-// and
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//	dense: `R`-D.  The dense Tensor operand.
 //
-//     index = [ 2]
-//             [10]
-//     values = [4, 5]
-//     shape = [30]
+// Returns 1-D.  The `N` values that are operated on.
+func SparseDenseCwiseAdd(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseDenseCwiseAdd",
+		Input: []tf.Input{
+			sp_indices, sp_values, sp_shape, dense,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Read an element from the TensorArray into output `value`.
 //
-// then the final deserialized `SparseTensor` will be:
+// Arguments:
+//	handle: The handle to a TensorArray.
 //
-//     index = [0  0]
-//             [0 10]
-//             [0 20]
-//             [1  2]
-//             [1 10]
-//     values = [1, 2, 3, 4, 5]
-//     shape = [2 50]
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// Arguments:
-//	serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects.
-// Must have 3 columns.
-//	dtype: The `dtype` of the serialized `SparseTensor` objects.
-func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+// Returns The tensor that is read from the TensorArray.
+func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "DeserializeManySparse",
+		Type: "TensorArrayReadV3",
 		Input: []tf.Input{
-			serialized_sparse,
+			handle, index, flow_in,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// StringJoinAttr is an optional argument to StringJoin.
-type StringJoinAttr func(optionalAttr)
+// QuantizeV2Attr is an optional argument to QuantizeV2.
+type QuantizeV2Attr func(optionalAttr)
 
-// StringJoinSeparator sets the optional separator attribute to value.
-//
-// value: string, an optional join separator.
-// If not specified, defaults to ""
-func StringJoinSeparator(value string) StringJoinAttr {
+// QuantizeV2Mode sets the optional mode attribute to value.
+// If not specified, defaults to "MIN_COMBINED"
+func QuantizeV2Mode(value string) QuantizeV2Attr {
 	return func(m optionalAttr) {
-		m["separator"] = value
+		m["mode"] = value
 	}
 }
 
-// Joins the strings in the given list of string tensors into one tensor;
+// QuantizeV2RoundMode sets the optional round_mode attribute to value.
+// If not specified, defaults to "HALF_AWAY_FROM_ZERO"
+func QuantizeV2RoundMode(value string) QuantizeV2Attr {
+	return func(m optionalAttr) {
+		m["round_mode"] = value
+	}
+}
+
+// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
 //
-// with the given separator (default is an empty separator).
+// [min_range, max_range] are scalar floats that specify the range for
+// the 'input' data. The 'mode' attribute controls exactly which calculations are
+// used to convert the float values to their quantized equivalents.  The
+// 'round_mode' attribute controls which rounding tie-breaking algorithm is used
+// when rounding float values to their quantized equivalents.
+//
+// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+//
+// ```
+// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
+// if T == qint8, out[i] -= (range(T) + 1) / 2.0
+// ```
+// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+//
+// *MIN_COMBINED Mode Example*
+//
+// Assume the input is type float and has a possible range of [0.0, 6.0] and the
+// output type is quint8 ([0, 255]). The min_range and max_range values should be
+// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each
+// value of the input by 255/6 and cast to quint8.
+//
+// If the output type was qint8 ([-128, 127]), the operation will additionally
+// subtract each value by 128 prior to casting, so that the range of values aligns
+// with the range of qint8.
+//
+// If the mode is 'MIN_FIRST', then this approach is used:
+//
+// ```
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
+// range = (range_max - range_min) * range_adjust
+// range_scale = num_discrete_values / range
+// quantized = round(input * range_scale) - round(range_min * range_scale) +
+//   numeric_limits<T>::min()
+// quantized = max(quantized, numeric_limits<T>::min())
+// quantized = min(quantized, numeric_limits<T>::max())
+// ```
+//
+// The biggest difference between this and MIN_COMBINED is that the minimum range
+// is rounded first, before it's subtracted from the rounded value. With
+// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing
+// and dequantizing will introduce a larger and larger error.
+//
+// *SCALED mode Example*
+//
+// `SCALED` mode matches the quantization approach used in
+// `QuantizeAndDequantize{V2|V3}`.
+//
+// If the mode is `SCALED`, we do not use the full range of the output type,
+// choosing to elide the lowest possible value for symmetry (e.g., output range is
+// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+// 0.
+//
+// We first find the range of values in our tensor. The
+// range we use is always centered on 0, so we find m such that
+// ```c++
+//   m = max(abs(input_min), abs(input_max))
+// ```
+//
+// Our input tensor range is then `[-m, m]`.
+//
+// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+// If T is signed, this is
+// ```
+//   num_bits = sizeof(T) * 8
+//   [min_fixed, max_fixed] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+// ```
+//
+// Otherwise, if T is unsigned, the fixed-point range is
+// ```
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+// ```
+//
+// From this we compute our scaling factor, s:
+// ```c++
+//   s = (max_fixed - min_fixed) / (2 * m)
+// ```
+//
+// Now we can quantize the elements of our tensor:
+// ```c++
+// result = round(input * s)
+// ```
+//
+// One thing to watch out for is that the operator may choose to adjust the
+// requested minimum and maximum values slightly during the quantization process,
+// so you should always use the output ports as the range for further calculations.
+// For example, if the requested minimum and maximum values are close to equal,
+// they will be separated by a small epsilon value to prevent ill-formed quantized
+// buffers from being created. Otherwise, you can end up with buffers where all the
+// quantized values map to the same float value, which causes problems for
+// operations that have to perform further calculations on them.
 //
 // Arguments:
-//	inputs: A list of string tensors.  The tensors must all have the same shape,
-// or be scalars.  Scalars may be mixed in; these will be broadcast to the shape
-// of non-scalar inputs.
-func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) {
+//
+//	min_range: The minimum scalar value possibly produced for the input.
+//	max_range: The maximum scalar value possibly produced for the input.
+//
+//
+// Returns The quantized data produced from the float input.The actual minimum scalar value used for the output.The actual maximum scalar value used for the output.
+func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"T": T}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringJoin",
+		Type: "QuantizeV2",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			input, min_range, max_range,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns immutable tensor from memory region.
-//
-// The current implementation memmaps the tensor from a file.
+// Returns the truth value of (x < y) element-wise.
 //
-// Arguments:
-//	dtype: Type of the returned tensor.
-//	shape: Shape of the returned tensor.
-//	memory_region_name: Name of readonly memory region used by the tensor, see
-// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
-func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
+// *NOTE*: `Less` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
 	opspec := tf.OpSpec{
-		Type: "ImmutableConst",
-
-		Attrs: attrs,
+		Type: "Less",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse real-valued fast Fourier transform.
-//
-// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most dimension of `input`.
-//
-// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
-// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
-// `fft_length` is not provided, it is computed from the size of the inner-most
-// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
-// compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
-//
-// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
-// than the corresponding dimension of `input`, the dimension is cropped. If it is
-// larger, the dimension is padded with zeros.
+// QuantizedReluXAttr is an optional argument to QuantizedReluX.
+type QuantizedReluXAttr func(optionalAttr)
+
+// QuantizedReluXOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
 //
 // Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-// Returns A float32 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length` samples of its inverse
-//   1D Fourier transform.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.irfft
-// @end_compatibility
-func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IRFFT",
-		Input: []tf.Input{
-			input, fft_length,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Concatenates a list of `SparseTensor` along the specified dimension.
-//
-// Concatenation is with respect to the dense versions of these sparse tensors.
-// It is assumed that each input is a `SparseTensor` whose elements are ordered
-// along increasing dimension number.
-//
-// All inputs' shapes must match, except for the concat dimension.  The
-// `indices`, `values`, and `shapes` lists must have the same length.
-//
-// The output shape is identical to the inputs', except along the concat
-// dimension, where it is the sum of the inputs' sizes along that dimension.
-//
-// The output elements will be resorted to preserve the sort order along
-// increasing dimension number.
-//
-// This op runs in `O(M log M)` time, where `M` is the total number of non-empty
-// values across all inputs. This is due to the need for an internal sort in
-// order to concatenate efficiently across an arbitrary dimension.
-//
-// For example, if `concat_dim = 1` and the inputs are
-//
-//     sp_inputs[0]: shape = [2, 3]
-//     [0, 2]: "a"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
-//
-//     sp_inputs[1]: shape = [2, 4]
-//     [0, 1]: "d"
-//     [0, 2]: "e"
-//
-// then the output will be
-//
-//     shape = [2, 7]
-//     [0, 2]: "a"
-//     [0, 4]: "d"
-//     [0, 5]: "e"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
-//
-// Graphically this is equivalent to doing
-//
-//     [    a] concat [  d e  ] = [    a   d e  ]
-//     [b c  ]        [       ]   [b c          ]
-//
-// Arguments:
-//	indices: 2-D.  Indices of each input `SparseTensor`.
-//	values: 1-D.  Non-empty values of each `SparseTensor`.
-//	shapes: 1-D.  Shapes of each `SparseTensor`.
-//	concat_dim: Dimension to concatenate along. Must be in range [-rank, rank),
-// where rank is the number of dimensions in each input `SparseTensor`.
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
 //
-// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
-func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"concat_dim": concat_dim}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseConcat",
+		Type: "QuantizedReluX",
 		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes),
+			features, max_value, min_features, max_features,
 		},
 		Attrs: attrs,
 	}
@@ -12752,118 +12628,38 @@ func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Generates sparse cross from a list of sparse and dense tensors.
-//
-// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
-// representing features of one feature column. It outputs a 2D `SparseTensor` with
-// the batchwise crosses of these features.
-//
-// For example, if the inputs are
-//
-//     inputs[0]: SparseTensor with shape = [2, 2]
-//     [0, 0]: "a"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
-//
-//     inputs[1]: SparseTensor with shape = [2, 1]
-//     [0, 0]: "d"
-//     [1, 0]: "e"
-//
-//     inputs[2]: Tensor [["f"], ["g"]]
-//
-// then the output will be
-//
-//     shape = [2, 2]
-//     [0, 0]: "a_X_d_X_f"
-//     [1, 0]: "b_X_e_X_g"
-//     [1, 1]: "c_X_e_X_g"
-//
-// if hashed_output=true then the output will be
-//
-//     shape = [2, 2]
-//     [0, 0]: FingerprintCat64(
-//                 Fingerprint64("f"), FingerprintCat64(
-//                     Fingerprint64("d"), Fingerprint64("a")))
-//     [1, 0]: FingerprintCat64(
-//                 Fingerprint64("g"), FingerprintCat64(
-//                     Fingerprint64("e"), Fingerprint64("b")))
-//     [1, 1]: FingerprintCat64(
-//                 Fingerprint64("g"), FingerprintCat64(
-//                     Fingerprint64("e"), Fingerprint64("c")))
-//
-// Arguments:
-//	indices: 2-D.  Indices of each input `SparseTensor`.
-//	values: 1-D.   values of each `SparseTensor`.
-//	shapes: 1-D.   Shapes of each `SparseTensor`.
-//	dense_inputs: 2-D.    Columns represented by dense `Tensor`.
-//	hashed_output: If true, returns the hash of the cross instead of the string.
-// This will allow us avoiding string manipulations.
-//	num_buckets: It is used if hashed_output is true.
-// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value.
-//	hash_key: Specify the hash_key that will be used by the `FingerprintCat64`
-// function to combine the crosses fingerprints.
-//
-//
+// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
+type WholeFileReaderV2Attr func(optionalAttr)
+
+// WholeFileReaderV2Container sets the optional container attribute to value.
 //
-// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated or hashed
-// `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
-func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type}
-	opspec := tf.OpSpec{
-		Type: "SparseCross",
-		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs),
-		},
-		Attrs: attrs,
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ListDiffAttr is an optional argument to ListDiff.
-type ListDiffAttr func(optionalAttr)
-
-// ListDiffOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func ListDiffOutIdx(value tf.DataType) ListDiffAttr {
+// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Computes the difference between two lists of numbers or strings.
-//
-// Given a list `x` and a list `y`, this operation returns a list `out` that
-// represents all values that are in `x` but not in `y`. The returned list `out`
-// is sorted in the same order that the numbers appear in `x` (duplicates are
-// preserved). This operation also returns a list `idx` that represents the
-// position of each `out` element in `x`. In other words:
-//
-// `out[i] = x[idx[i]] for i in [0, 1, ..., len(out) - 1]`
-//
-// For example, given this input:
-//
-// ```
-// x = [1, 2, 3, 4, 5, 6]
-// y = [1, 3, 5]
-// ```
-//
-// This operation would return:
-//
-// ```
-// out ==> [2, 4, 6]
-// idx ==> [1, 3, 5]
-// ```
+// A Reader that outputs the entire contents of a file as a value.
 //
-// Arguments:
-//	x: 1-D. Values to keep.
-//	y: 1-D. Values to remove.
+// To use, enqueue filenames in a Queue.  The output of ReaderRead will
+// be a filename (key) and the contents of that file (value).
 //
-// Returns 1-D. Values present in `x` but not in `y`.1-D. Positions of `x` values preserved in `out`.
-func ListDiff(scope *Scope, x tf.Output, y tf.Output, optional ...ListDiffAttr) (out tf.Output, idx tf.Output) {
+// Returns The handle to reference the Reader.
+func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12872,108 +12668,140 @@ func ListDiff(scope *Scope, x tf.Output, y tf.Output, optional ...ListDiffAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ListDiff",
-		Input: []tf.Input{
-			x, y,
-		},
+		Type: "WholeFileReaderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
-//
-// This Op does not require `a_indices` be sorted in standard lexicographic order.
+// Transforms a tf.Example proto (as a string) into typed tensors.
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
-//	b: `ndims`-D Tensor.  With shape `a_shape`.
-func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	dense_defaults: A list of Tensors (some may be empty), whose length matches
+// the length of `dense_keys`. dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	num_sparse: The number of sparse features to be parsed from the example. This
+// must match the lengths of `sparse_keys` and `sparse_types`.
+//	sparse_keys: A list of `num_sparse` strings.
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: The keys expected in the Examples' features associated with dense
+// values.
+//	sparse_types: A list of `num_sparse` types; the data types of data in each
+// Feature given in sparse_keys.
+// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: The shapes of data in each Feature given in dense_keys.
+// The length of this list must match the length of `dense_keys`.  The
+// number of elements in the Feature corresponding to dense_key[j] must
+// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
+// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
+// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
+// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
+// D1, .., DN), where M is the number of blocks of elements of length
+// D1 * .... * DN, in the input.
+func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseAdd",
+		Type: "ParseSingleExample",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
+			serialized, tf.OutputList(dense_defaults),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
+		return
+	}
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
 }
 
-// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
-type SparseToSparseSetOperationAttr func(optionalAttr)
+// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
+type QuantizedConv2DAttr func(optionalAttr)
 
-// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
+// QuantizedConv2DOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["out_type"] = value
 	}
 }
 
-// Applies set operation along last dimension of 2 `SparseTensor` inputs.
-//
-// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
-//
-// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
-// order and range of `set1` and `set2` indices.
-//
-// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
-// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
-//
-// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
-// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
+// QuantizedConv2DDilations sets the optional dilations attribute to value.
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set1`
-// and `set2` indices.
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func QuantizedConv2DDilations(value []int64) QuantizedConv2DAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes a 2D convolution given quantized 4D input and filter tensors.
 //
-// Output `result` is a `SparseTensor` represented by `result_indices`,
-// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
-// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
-// dimension contains the result of `set_operation` applied to the corresponding
-// `[0...n-1]` dimension of `set`.
+// The inputs are quantized tensors where the lowest value represents the real
+// number of the associated minimum, and the highest represents the maximum.
+// This means that you can only interpret the quantized output in the same way, by
+// taking the returned minimum and maximum values into account.
 //
 // Arguments:
-//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
-// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
-// max set size across `0...n-1` dimensions.
-//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
-// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
-// max set size across `0...n-1` dimensions.
 //
+//	filter: filter's input_depth dimension must match input's depth dimensions.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	min_filter: The float value that the lowest quantized filter value represents.
+//	max_filter: The float value that the highest quantized filter value represents.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
-// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
-// is the max result set size across all `0...n-1` dimensions.
-func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"set_operation": set_operation}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseToSparseSetOperation",
+		Type: "QuantizedConv2D",
 		Input: []tf.Input{
-			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
+			input, filter, min_input, max_input, min_filter, max_filter,
 		},
 		Attrs: attrs,
 	}
@@ -12981,66 +12809,44 @@ func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_value
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes numerical negative value element-wise.
-//
-// I.e., \\(y = -x\\).
-func Neg(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Neg",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
-type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
-
-// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
-	return func(m optionalAttr) {
-		m["num_bits"] = value
-	}
-}
+// ResourceGatherAttr is an optional argument to ResourceGather.
+type ResourceGatherAttr func(optionalAttr)
 
-// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
-// If not specified, defaults to false
-func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
+// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
 	return func(m optionalAttr) {
-		m["narrow_range"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
+// Gather slices from the variable pointed to by `resource` according to `indices`.
 //
-// and `max` to 'outputs' tensor of same shape as `inputs`.
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
 //
-// `[min; max]` define the clamping range for the `inputs` data.
-// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
-// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
-// then de-quantized and output as floats in `[min; max]` interval.
-// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+// ```python
+//     # Scalar indices
+//     output[:, ..., :] = params[indices, :, ... :]
 //
-// This operation has a gradient and thus allows for training `min` and `max`
-// values.
-func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
+//     # Vector indices
+//     output[i, :, ..., :] = params[indices[i], :, ... :]
+//
+//     # Higher rank indices
+//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
+// ```
+func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FakeQuantWithMinMaxVars",
+		Type: "ResourceGather",
 		Input: []tf.Input{
-			inputs, min, max,
+			resource, indices,
 		},
 		Attrs: attrs,
 	}
@@ -13048,29 +12854,23 @@ func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max
 	return op.Output(0)
 }
 
-// Writes a `Summary` protocol buffer with a histogram.
-//
-// The generated
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// has one summary value containing a histogram for `values`.
+// Delete the TensorArray from its resource container.
 //
-// This op reports an `InvalidArgument` error if any value is not finite.
+// This enables the user to close and release the resource in the middle
+// of a step/run.
 //
 // Arguments:
-//	writer: A handle to a summary writer.
-//	step: The step to write the summary for.
-//	tag: Scalar.  Tag to use for the `Summary.Value`.
-//	values: Any shape. Values to use to build the histogram.
+//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
 //
 // Returns the created operation.
-func WriteHistogramSummary(scope *Scope, writer tf.Output, step tf.Output, tag tf.Output, values tf.Output) (o *tf.Operation) {
+func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "WriteHistogramSummary",
+		Type: "TensorArrayCloseV3",
 		Input: []tf.Input{
-			writer, step, tag, values,
+			handle,
 		},
 	}
 	return scope.AddOperation(opspec)
@@ -13375,59 +13175,6 @@ func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, n
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns the element-wise min of two SparseTensors.
-//
-// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
-//
-// Arguments:
-//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, in the canonical lexicographic ordering.
-//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
-//	a_shape: 1-D.  Shape of the input SparseTensor.
-//	b_indices: counterpart to `a_indices` for the other operand.
-//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
-//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
-//
-// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
-func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSparseMinimum",
-		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Constructs a tensor by tiling a given tensor.
-//
-// This operation creates a new tensor by replicating `input` `multiples` times.
-// The output tensor's i'th dimension has `input.dims(i) * multiples[i]` elements,
-// and the values of `input` are replicated `multiples[i]` times along the 'i'th
-// dimension. For example, tiling `[a b c d]` by `[2]` produces
-// `[a b c d a b c d]`.
-//
-// Arguments:
-//	input: 1-D or higher.
-//	multiples: 1-D. Length must be the same as the number of dimensions in `input`
-func Tile(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Tile",
-		Input: []tf.Input{
-			input, multiples,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Saves the input tensors to disk.
 //
 // The size of `tensor_names` must match the number of tensors in `data`. `data[i]`
@@ -13476,40 +13223,75 @@ func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// TakeManySparseFromTensorsMapAttr is an optional argument to TakeManySparseFromTensorsMap.
-type TakeManySparseFromTensorsMapAttr func(optionalAttr)
+// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
+type SparseTensorDenseMatMulAttr func(optionalAttr)
 
-// TakeManySparseFromTensorsMapContainer sets the optional container attribute to value.
+// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
 //
-// value: The container name for the `SparseTensorsMap` read by this op.
-// If not specified, defaults to ""
-func TakeManySparseFromTensorsMapContainer(value string) TakeManySparseFromTensorsMapAttr {
+// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
+// is transpose(conj(A)).  Otherwise it's transpose(A).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["adjoint_a"] = value
 	}
 }
 
-// TakeManySparseFromTensorsMapSharedName sets the optional shared_name attribute to value.
+// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
 //
-// value: The shared name for the `SparseTensorsMap` read by this op.
-// It should not be blank; rather the `shared_name` or unique Operation name
-// of the Op that created the original `SparseTensorsMap` should be used.
-// If not specified, defaults to ""
-func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTensorsMapAttr {
+// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
+// is transpose(conj(B)).  Otherwise it's transpose(B).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["adjoint_b"] = value
 	}
 }
 
-// Read `SparseTensors` from a `SparseTensorsMap` and concatenate them.
+// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
 //
-// The input `sparse_handles` must be an `int64` matrix of shape `[N, 1]` where
-// `N` is the minibatch size and the rows correspond to the output handles of
-// `AddSparseToTensorsMap` or `AddManySparseToTensorsMap`.  The ranks of the
-// original `SparseTensor` objects that went into the given input ops must all
-// match.  When the final `SparseTensor` is created, it has rank one
+// No validity checking is performed on the indices of A.  However, the following
+// input format is recommended for optimal behavior:
+//
+// if adjoint_a == false:
+//   A should be sorted in lexicographically increasing order.  Use SparseReorder
+//   if you're not sure.
+// if adjoint_a == true:
+//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
+//   order instead of "row major" order).
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
+//	b: 2-D.  A dense Matrix.
+func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseTensorDenseMatMul",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Deserialize and concatenate `SparseTensors` from a serialized minibatch.
+//
+// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
+// `N` is the minibatch size and the rows correspond to packed outputs of
+// `SerializeSparse`.  The ranks of the original `SparseTensor` objects
+// must all match.  When the final `SparseTensor` is created, it has rank one
 // higher than the ranks of the incoming `SparseTensor` objects
-// (they have been concatenated along a new row dimension on the left).
+// (they have been concatenated along a new row dimension).
 //
 // The output `SparseTensor` object's shape values for all dimensions but the
 // first are the max across the input `SparseTensor` objects' shape values
@@ -13520,29 +13302,24 @@ func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTens
 // standard lexicographic order.  If this is not the case, after this
 // step run `SparseReorder` to restore index ordering.
 //
-// For example, if the handles represent an input, which is a `[2, 3]` matrix
-// representing two original `SparseTensor` objects:
+// For example, if the serialized input is a `[2 x 3]` matrix representing two
+// original `SparseTensor` objects:
 //
-// ```
 //     index = [ 0]
 //             [10]
 //             [20]
 //     values = [1, 2, 3]
 //     shape = [50]
-// ```
 //
 // and
 //
-// ```
 //     index = [ 2]
 //             [10]
 //     values = [4, 5]
 //     shape = [30]
-// ```
 //
-// then the final `SparseTensor` will be:
+// then the final deserialized `SparseTensor` will be:
 //
-// ```
 //     index = [0  0]
 //             [0 10]
 //             [0 20]
@@ -13550,27 +13327,20 @@ func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTens
 //             [1 10]
 //     values = [1, 2, 3, 4, 5]
 //     shape = [2 50]
-// ```
 //
 // Arguments:
-//	sparse_handles: 1-D, The `N` serialized `SparseTensor` objects.
-// Shape: `[N]`.
-//	dtype: The `dtype` of the `SparseTensor` objects stored in the
-// `SparseTensorsMap`.
-//
-// Returns 2-D.  The `indices` of the minibatch `SparseTensor`.1-D.  The `values` of the minibatch `SparseTensor`.1-D.  The `shape` of the minibatch `SparseTensor`.
-func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype tf.DataType, optional ...TakeManySparseFromTensorsMapAttr) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+//	serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects.
+// Must have 3 columns.
+//	dtype: The `dtype` of the serialized `SparseTensor` objects.
+func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TakeManySparseFromTensorsMap",
+		Type: "DeserializeManySparse",
 		Input: []tf.Input{
-			sparse_handles,
+			serialized_sparse,
 		},
 		Attrs: attrs,
 	}
@@ -13578,240 +13348,290 @@ func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Says whether the targets are in the top `K` predictions.
-//
-// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
-// prediction for the target class is among the top `k` predictions among
-// all predictions for example `i`. Note that the behavior of `InTopK` differs
-// from the `TopK` op in its handling of ties; if multiple classes have the
-// same prediction value and straddle the top-`k` boundary, all of those
-// classes are considered to be in the top `k`.
-//
-// More formally, let
+// StringJoinAttr is an optional argument to StringJoin.
+type StringJoinAttr func(optionalAttr)
+
+// StringJoinSeparator sets the optional separator attribute to value.
 //
-//   \\(predictions_i\\) be the predictions for all classes for example `i`,
-//   \\(targets_i\\) be the target class for example `i`,
-//   \\(out_i\\) be the output for example `i`,
+// value: string, an optional join separator.
+// If not specified, defaults to ""
+func StringJoinSeparator(value string) StringJoinAttr {
+	return func(m optionalAttr) {
+		m["separator"] = value
+	}
+}
+
+// Joins the strings in the given list of string tensors into one tensor;
 //
-// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+// with the given separator (default is an empty separator).
 //
 // Arguments:
-//	predictions: A `batch_size` x `classes` tensor.
-//	targets: A `batch_size` vector of class ids.
-//	k: Number of top elements to look at for computing precision.
-//
-// Returns Computed precision at `k` as a `bool Tensor`.
-func InTopKV2(scope *Scope, predictions tf.Output, targets tf.Output, k tf.Output) (precision tf.Output) {
+//	inputs: A list of string tensors.  The tensors must all have the same shape,
+// or be scalars.  Scalars may be mixed in; these will be broadcast to the shape
+// of non-scalar inputs.
+func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "InTopKV2",
+		Type: "StringJoin",
 		Input: []tf.Input{
-			predictions, targets, k,
+			tf.OutputList(inputs),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Assigns a new value to a variable.
+// Returns immutable tensor from memory region.
 //
-// Any ReadVariableOp with a control dependency on this op is guaranteed to return
-// this value or a subsequent newer value of the variable.
+// The current implementation memmaps the tensor from a file.
 //
 // Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value to set the new tensor to use.
-//
-// Returns the created operation.
-func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+//	dtype: Type of the returned tensor.
+//	shape: Shape of the returned tensor.
+//	memory_region_name: Name of readonly memory region used by the tensor, see
+// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
+func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
 	opspec := tf.OpSpec{
-		Type: "AssignVariableOp",
-		Input: []tf.Input{
-			resource, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
+		Type: "ImmutableConst",
 
-// Returns a tensor of ones with the same shape and type as x.
-//
-// Arguments:
-//	x: a tensor of type T.
-//
-// Returns a tensor of the same shape and type as x but filled with ones.
-func OnesLike(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "OnesLike",
-		Input: []tf.Input{
-			x,
-		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// The gradient of SparseFillEmptyRows.
+// Inverse real-valued fast Fourier transform.
 //
-// Takes vectors reverse_index_map, shaped `[N]`, and grad_values,
-// shaped `[N_full]`, where `N_full >= N` and copies data into either
-// `d_values` or `d_default_value`.  Here `d_values` is shaped `[N]` and
-// `d_default_value` is a scalar.
+// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most dimension of `input`.
 //
-//   d_values[j] = grad_values[reverse_index_map[j]]
-//   d_default_value = sum_{k : 0 .. N_full - 1} (
-//      grad_values[k] * 1{k not in reverse_index_map})
+// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
+// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
+// `fft_length` is not provided, it is computed from the size of the inner-most
+// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
+// compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
 //
-// Arguments:
-//	reverse_index_map: 1-D.  The reverse index map from SparseFillEmptyRows.
-//	grad_values: 1-D.  The gradients from backprop.
+// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
+// than the corresponding dimension of `input`, the dimension is cropped. If it is
+// larger, the dimension is padded with zeros.
 //
-// Returns 1-D.  The backprop into values.0-D.  The backprop into default_value.
-func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_values tf.Output) (d_values tf.Output, d_default_value tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseFillEmptyRowsGrad",
-		Input: []tf.Input{
-			reverse_index_map, grad_values,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)`
+// Arguments:
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-// if < 0, `scale * features` otherwise.
+// Returns A float32 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length` samples of its inverse
+//   1D Fourier transform.
 //
-// See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
-func Selu(scope *Scope, features tf.Output) (activations tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.fft.irfft
+// @end_compatibility
+func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Selu",
+		Type: "IRFFT",
 		Input: []tf.Input{
-			features,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SetSizeAttr is an optional argument to SetSize.
-type SetSizeAttr func(optionalAttr)
-
-// SetSizeValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func SetSizeValidateIndices(value bool) SetSizeAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
-}
-
-// Number of unique elements along last dimension of input `set`.
+// Concatenates a list of `SparseTensor` along the specified dimension.
 //
-// Input `set` is a `SparseTensor` represented by `set_indices`, `set_values`,
-// and `set_shape`. The last dimension contains values in a set, duplicates are
-// allowed but ignored.
+// Concatenation is with respect to the dense versions of these sparse tensors.
+// It is assumed that each input is a `SparseTensor` whose elements are ordered
+// along increasing dimension number.
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set`
-// indices.
+// All inputs' shapes must match, except for the concat dimension.  The
+// `indices`, `values`, and `shapes` lists must have the same length.
+//
+// The output shape is identical to the inputs', except along the concat
+// dimension, where it is the sum of the inputs' sizes along that dimension.
+//
+// The output elements will be resorted to preserve the sort order along
+// increasing dimension number.
+//
+// This op runs in `O(M log M)` time, where `M` is the total number of non-empty
+// values across all inputs. This is due to the need for an internal sort in
+// order to concatenate efficiently across an arbitrary dimension.
+//
+// For example, if `concat_dim = 1` and the inputs are
+//
+//     sp_inputs[0]: shape = [2, 3]
+//     [0, 2]: "a"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+//     sp_inputs[1]: shape = [2, 4]
+//     [0, 1]: "d"
+//     [0, 2]: "e"
+//
+// then the output will be
+//
+//     shape = [2, 7]
+//     [0, 2]: "a"
+//     [0, 4]: "d"
+//     [0, 5]: "e"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+// Graphically this is equivalent to doing
+//
+//     [    a] concat [  d e  ] = [    a   d e  ]
+//     [b c  ]        [       ]   [b c          ]
 //
 // Arguments:
-//	set_indices: 2D `Tensor`, indices of a `SparseTensor`.
-//	set_values: 1D `Tensor`, values of a `SparseTensor`.
-//	set_shape: 1D `Tensor`, shape of a `SparseTensor`.
+//	indices: 2-D.  Indices of each input `SparseTensor`.
+//	values: 1-D.  Non-empty values of each `SparseTensor`.
+//	shapes: 1-D.  Shapes of each `SparseTensor`.
+//	concat_dim: Dimension to concatenate along. Must be in range [-rank, rank),
+// where rank is the number of dimensions in each input `SparseTensor`.
 //
-// Returns For `set` ranked `n`, this is a `Tensor` with rank `n-1`, and the same 1st
-// `n-1` dimensions as `set`. Each value is the number of unique elements in
-// the corresponding `[0...n-1]` dimension of `set`.
-func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shape tf.Output, optional ...SetSizeAttr) (size tf.Output) {
+// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
+func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"concat_dim": concat_dim}
 	opspec := tf.OpSpec{
-		Type: "SetSize",
+		Type: "SparseConcat",
 		Input: []tf.Input{
-			set_indices, set_values, set_shape,
+			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the sign and the log of the absolute value of the determinant of
+// Generates sparse cross from a list of sparse and dense tensors.
 //
-// one or more square matrices.
+// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
+// representing features of one feature column. It outputs a 2D `SparseTensor` with
+// the batchwise crosses of these features.
 //
-// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
-// form square matrices. The outputs are two tensors containing the signs and
-// absolute values of the log determinants for all N input submatrices
-// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
-// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
-// is the LU decomposition of the input and P is the corresponding
-// permutation matrix.
+// For example, if the inputs are
+//
+//     inputs[0]: SparseTensor with shape = [2, 2]
+//     [0, 0]: "a"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+//     inputs[1]: SparseTensor with shape = [2, 1]
+//     [0, 0]: "d"
+//     [1, 0]: "e"
+//
+//     inputs[2]: Tensor [["f"], ["g"]]
+//
+// then the output will be
+//
+//     shape = [2, 2]
+//     [0, 0]: "a_X_d_X_f"
+//     [1, 0]: "b_X_e_X_g"
+//     [1, 1]: "c_X_e_X_g"
+//
+// if hashed_output=true then the output will be
+//
+//     shape = [2, 2]
+//     [0, 0]: FingerprintCat64(
+//                 Fingerprint64("f"), FingerprintCat64(
+//                     Fingerprint64("d"), Fingerprint64("a")))
+//     [1, 0]: FingerprintCat64(
+//                 Fingerprint64("g"), FingerprintCat64(
+//                     Fingerprint64("e"), Fingerprint64("b")))
+//     [1, 1]: FingerprintCat64(
+//                 Fingerprint64("g"), FingerprintCat64(
+//                     Fingerprint64("e"), Fingerprint64("c")))
 //
 // Arguments:
-//	input: Shape is `[N, M, M]`.
+//	indices: 2-D.  Indices of each input `SparseTensor`.
+//	values: 1-D.   values of each `SparseTensor`.
+//	shapes: 1-D.   Shapes of each `SparseTensor`.
+//	dense_inputs: 2-D.    Columns represented by dense `Tensor`.
+//	hashed_output: If true, returns the hash of the cross instead of the string.
+// This will allow us avoiding string manipulations.
+//	num_buckets: It is used if hashed_output is true.
+// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value.
+//	hash_key: Specify the hash_key that will be used by the `FingerprintCat64`
+// function to combine the crosses fingerprints.
 //
-// Returns The signs of the log determinants of the inputs. Shape is `[N]`.The logs of the absolute values of the determinants
-// of the N input matrices.  Shape is `[N]`.
-func LogMatrixDeterminant(scope *Scope, input tf.Output) (sign tf.Output, log_abs_determinant tf.Output) {
+//
+//
+// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated or hashed
+// `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
+func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type}
 	opspec := tf.OpSpec{
-		Type: "LogMatrixDeterminant",
+		Type: "SparseCross",
 		Input: []tf.Input{
-			input,
+			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// SumAttr is an optional argument to Sum.
-type SumAttr func(optionalAttr)
+// ListDiffAttr is an optional argument to ListDiff.
+type ListDiffAttr func(optionalAttr)
 
-// SumKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SumKeepDims(value bool) SumAttr {
+// ListDiffOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func ListDiffOutIdx(value tf.DataType) ListDiffAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["out_idx"] = value
 	}
 }
 
-// Computes the sum of elements across dimensions of a tensor.
+// Computes the difference between two lists of numbers or strings.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Given a list `x` and a list `y`, this operation returns a list `out` that
+// represents all values that are in `x` but not in `y`. The returned list `out`
+// is sorted in the same order that the numbers appear in `x` (duplicates are
+// preserved). This operation also returns a list `idx` that represents the
+// position of each `out` element in `x`. In other words:
+//
+// `out[i] = x[idx[i]] for i in [0, 1, ..., len(out) - 1]`
+//
+// For example, given this input:
+//
+// ```
+// x = [1, 2, 3, 4, 5, 6]
+// y = [1, 3, 5]
+// ```
+//
+// This operation would return:
+//
+// ```
+// out ==> [2, 4, 6]
+// idx ==> [1, 3, 5]
+// ```
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	x: 1-D. Values to keep.
+//	y: 1-D. Values to remove.
 //
-// Returns The reduced tensor.
-func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
+// Returns 1-D. Values present in `x` but not in `y`.1-D. Positions of `x` values preserved in `out`.
+func ListDiff(scope *Scope, x tf.Output, y tf.Output, optional ...ListDiffAttr) (out tf.Output, idx tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -13820,81 +13640,163 @@ func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Sum",
+		Type: "ListDiff",
 		Input: []tf.Input{
-			input, axis,
+			x, y,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Delete the tensor specified by its handle in the session.
+// Concatenates quantized tensors along one dimension.
 //
 // Arguments:
-//	handle: The handle for a tensor stored in the session state.
+//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [0, rank(values)).
+//	values: The `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//	input_mins: The minimum scalar values for each of the input tensors.
+//	input_maxes: The maximum scalar values for each of the input tensors.
 //
-// Returns the created operation.
-func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
+func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DeleteSessionTensor",
+		Type: "QuantizedConcat",
 		Input: []tf.Input{
-			handle,
+			concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes),
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// L2 Loss.
+// Slice a `SparseTensor` based on the `start` and `size`.
 //
-// Computes half the L2 norm of a tensor without the `sqrt`:
+// For example, if the input is
 //
-//     output = sum(t ** 2) / 2
+//     input_tensor = shape = [2, 7]
+//     [    a   d e  ]
+//     [b c          ]
+//
+// Graphically the output tensors are:
+//
+//     sparse_slice([0, 0], [2, 4]) = shape = [2, 4]
+//     [    a  ]
+//     [b c    ]
+//
+//     sparse_slice([0, 4], [2, 3]) = shape = [2, 3]
+//     [ d e  ]
+//     [      ]
 //
 // Arguments:
-//	t: Typically 2-D, but may have any dimensions.
+//	indices: 2-D tensor represents the indices of the sparse tensor.
+//	values: 1-D tensor represents the values of the sparse tensor.
+//	shape: 1-D. tensor represents the shape of the sparse tensor.
+//	start: 1-D. tensor represents the start of the slice.
+//	size: 1-D. tensor represents the size of the slice.
+// output indices: A list of 1-D tensors represents the indices of the output
+// sparse tensors.
 //
-// Returns 0-D.
-func L2Loss(scope *Scope, t tf.Output) (output tf.Output) {
+// Returns A list of 1-D tensors represents the values of the output sparse
+// tensors.A list of 1-D tensors represents the shape of the output sparse
+// tensors.
+func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "L2Loss",
+		Type: "SparseSlice",
 		Input: []tf.Input{
-			t,
+			indices, values, shape, start, size,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DenseToSparseSetOperationAttr is an optional argument to DenseToSparseSetOperation.
-type DenseToSparseSetOperationAttr func(optionalAttr)
-
-// DenseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperationAttr {
+// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
+//
+// This Op does not require `a_indices` be sorted in standard lexicographic order.
+//
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
+//	b: `ndims`-D Tensor.  With shape `a_shape`.
+func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseTensorDenseAdd",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the set of files matching one or more glob patterns.
+//
+// Note that this routine only supports wildcard characters in the
+// basename portion of the pattern, not in the directory portion.
+//
+// Arguments:
+//	pattern: Shell wildcard pattern(s). Scalar or vector of type string.
+//
+// Returns A vector of matching filenames.
+func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "MatchingFiles",
+		Input: []tf.Input{
+			pattern,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
+type SparseToSparseSetOperationAttr func(optionalAttr)
+
+// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
 	return func(m optionalAttr) {
 		m["validate_indices"] = value
 	}
 }
 
-// Applies set operation along last dimension of `Tensor` and `SparseTensor`.
+// Applies set operation along last dimension of 2 `SparseTensor` inputs.
 //
 // See SetOperationOp::SetOperationFromContext for values of `set_operation`.
 //
+// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
+// order and range of `set1` and `set2` indices.
+//
+// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
+// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
+//
 // Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
 // and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
 // as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
 // ignored.
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set2`
-// indices.
+// If `validate_indices` is `True`, this op validates the order and range of `set1`
+// and `set2` indices.
 //
 // Output `result` is a `SparseTensor` represented by `result_indices`,
 // `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
@@ -13903,21 +13805,26 @@ func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperat
 // `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
+// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
+// max set size across `0...n-1` dimensions.
 //	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
 // order.
 //	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
 // order.
 //	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
-// be the same as the 1st `n-1` dimensions of `set1`, `result_shape[n]` is the
-// max set size across `n-1` dimensions.
+// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
+// max set size across `0...n-1` dimensions.
 //
 //
 // Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
 // the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
 // is the max result set size across all `0...n-1` dimensions.
-func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...DenseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -13926,9 +13833,9 @@ func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Out
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DenseToSparseSetOperation",
+		Type: "SparseToSparseSetOperation",
 		Input: []tf.Input{
-			set1, set2_indices, set2_values, set2_shape,
+			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
 		},
 		Attrs: attrs,
 	}
@@ -13936,58 +13843,66 @@ func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Out
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
-type FusedResizeAndPadConv2DAttr func(optionalAttr)
-
-// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
+// Computes numerical negative value element-wise.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1),
-// which exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
+// I.e., \\(y = -x\\).
+func Neg(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Neg",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
+type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
+
+// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
+	return func(m optionalAttr) {
+		m["num_bits"] = value
+	}
+}
+
+// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
 // If not specified, defaults to false
-func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
+func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
 	return func(m optionalAttr) {
-		m["resize_align_corners"] = value
+		m["narrow_range"] = value
 	}
 }
 
-// Performs a resize and padding as a preprocess during a convolution.
+// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
 //
-// It's often possible to do spatial transformations more efficiently as part of
-// the packing stage of a convolution, so this op allows for an optimized
-// implementation where these stages are fused together. This prevents the need to
-// write out the intermediate results as whole tensors, reducing memory pressure,
-// and we can get some latency gains by merging the transformation calculations.
-// The data_format attribute for Conv2D isn't supported by this op, and defaults to
-// 'NHWC' order.
-// Internally this op uses a single per-graph scratch buffer, which means that it
-// will block if multiple versions are being run in parallel. This is because this
-// operator is primarily an optimization to minimize memory usage.
+// and `max` to 'outputs' tensor of same shape as `inputs`.
 //
-// Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
-//	size: A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.
+// `[min; max]` define the clamping range for the `inputs` data.
+// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+// then de-quantized and output as floats in `[min; max]` interval.
+// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
 //
-//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
-// of `input`. Must be in the same order as the dimension specified with format.
-//	padding: The type of padding algorithm to use.
-func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string, optional ...FusedResizeAndPadConv2DAttr) (output tf.Output) {
+// This operation has a gradient and thus allows for training `min` and `max`
+// values.
+func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedResizeAndPadConv2D",
+		Type: "FakeQuantWithMinMaxVars",
 		Input: []tf.Input{
-			input, size, paddings, filter,
+			inputs, min, max,
 		},
 		Attrs: attrs,
 	}
@@ -13995,132 +13910,154 @@ func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, padd
 	return op.Output(0)
 }
 
-// Subtracts a value from the current value of a variable.
-//
-// Any ReadVariableOp which depends directly or indirectly on this assign is
-// guaranteed to see the incremented value or a subsequent newer one.
+// Returns the element-wise min of two SparseTensors.
 //
-// Outputs the incremented value, which can be used to totally order the
-// increments to this variable.
+// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
 //
 // Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value by which the variable will be incremented.
+//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, in the canonical lexicographic ordering.
+//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
+//	a_shape: 1-D.  Shape of the input SparseTensor.
+//	b_indices: counterpart to `a_indices` for the other operand.
+//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
+//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
 //
-// Returns the created operation.
-func AssignSubVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
+func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AssignSubVariableOp",
+		Type: "SparseSparseMinimum",
 		Input: []tf.Input{
-			resource, value,
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
 		},
 	}
-	return scope.AddOperation(opspec)
-}
-
-// RestoreAttr is an optional argument to Restore.
-type RestoreAttr func(optionalAttr)
-
-// RestorePreferredShard sets the optional preferred_shard attribute to value.
-//
-// value: Index of file to open first if multiple files match
-// `file_pattern`.
-// If not specified, defaults to -1
-func RestorePreferredShard(value int64) RestoreAttr {
-	return func(m optionalAttr) {
-		m["preferred_shard"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Restores a tensor from checkpoint files.
-//
-// Reads a tensor stored in one or several files. If there are several files (for
-// instance because a tensor was saved as slices), `file_pattern` may contain
-// wildcard symbols (`*` and `?`) in the filename portion only, not in the
-// directory portion.
-//
-// If a `file_pattern` matches several files, `preferred_shard` can be used to hint
-// in which file the requested tensor is likely to be found. This op will first
-// open the file at index `preferred_shard` in the list of matching files and try
-// to restore tensors from that file.  Only if some tensors or tensor slices are
-// not found in that first file, then the Op opens all the files. Setting
-// `preferred_shard` to match the value passed as the `shard` input
-// of a matching `Save` Op may speed up Restore.  This attribute only affects
-// performance, not correctness.  The default value -1 means files are processed in
-// order.
+// Constructs a tensor by tiling a given tensor.
 //
-// See also `RestoreSlice`.
+// This operation creates a new tensor by replicating `input` `multiples` times.
+// The output tensor's i'th dimension has `input.dims(i) * multiples[i]` elements,
+// and the values of `input` are replicated `multiples[i]` times along the 'i'th
+// dimension. For example, tiling `[a b c d]` by `[2]` produces
+// `[a b c d a b c d]`.
 //
 // Arguments:
-//	file_pattern: Must have a single element. The pattern of the files from
-// which we read the tensor.
-//	tensor_name: Must have a single element. The name of the tensor to be
-// restored.
-//	dt: The type of the tensor to be restored.
-//
-// Returns The restored tensor.
-func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf.DataType, optional ...RestoreAttr) (tensor tf.Output) {
+//	input: 1-D or higher.
+//	multiples: 1-D. Length must be the same as the number of dimensions in `input`
+func Tile(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dt": dt}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Restore",
+		Type: "Tile",
 		Input: []tf.Input{
-			file_pattern, tensor_name,
+			input, multiples,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedResizeBilinearAttr is an optional argument to QuantizedResizeBilinear.
-type QuantizedResizeBilinearAttr func(optionalAttr)
+// TakeManySparseFromTensorsMapAttr is an optional argument to TakeManySparseFromTensorsMap.
+type TakeManySparseFromTensorsMapAttr func(optionalAttr)
 
-// QuantizedResizeBilinearAlignCorners sets the optional align_corners attribute to value.
+// TakeManySparseFromTensorsMapContainer sets the optional container attribute to value.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func QuantizedResizeBilinearAlignCorners(value bool) QuantizedResizeBilinearAttr {
+// value: The container name for the `SparseTensorsMap` read by this op.
+// If not specified, defaults to ""
+func TakeManySparseFromTensorsMapContainer(value string) TakeManySparseFromTensorsMapAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["container"] = value
 	}
 }
 
-// Resize quantized `images` to `size` using quantized bilinear interpolation.
+// TakeManySparseFromTensorsMapSharedName sets the optional shared_name attribute to value.
 //
-// Input images and output images must be quantized types.
+// value: The shared name for the `SparseTensorsMap` read by this op.
+// It should not be blank; rather the `shared_name` or unique Operation name
+// of the Op that created the original `SparseTensorsMap` should be used.
+// If not specified, defaults to ""
+func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTensorsMapAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Read `SparseTensors` from a `SparseTensorsMap` and concatenate them.
 //
-// Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+// The input `sparse_handles` must be an `int64` matrix of shape `[N, 1]` where
+// `N` is the minibatch size and the rows correspond to the output handles of
+// `AddSparseToTensorsMap` or `AddManySparseToTensorsMap`.  The ranks of the
+// original `SparseTensor` objects that went into the given input ops must all
+// match.  When the final `SparseTensor` is created, it has rank one
+// higher than the ranks of the incoming `SparseTensor` objects
+// (they have been concatenated along a new row dimension on the left).
+//
+// The output `SparseTensor` object's shape values for all dimensions but the
+// first are the max across the input `SparseTensor` objects' shape values
+// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
+// size.
 //
+// The input `SparseTensor` objects' indices are assumed ordered in
+// standard lexicographic order.  If this is not the case, after this
+// step run `SparseReorder` to restore index ordering.
 //
+// For example, if the handles represent an input, which is a `[2, 3]` matrix
+// representing two original `SparseTensor` objects:
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min tf.Output, max tf.Output, optional ...QuantizedResizeBilinearAttr) (resized_images tf.Output, out_min tf.Output, out_max tf.Output) {
+// ```
+//     index = [ 0]
+//             [10]
+//             [20]
+//     values = [1, 2, 3]
+//     shape = [50]
+// ```
+//
+// and
+//
+// ```
+//     index = [ 2]
+//             [10]
+//     values = [4, 5]
+//     shape = [30]
+// ```
+//
+// then the final `SparseTensor` will be:
+//
+// ```
+//     index = [0  0]
+//             [0 10]
+//             [0 20]
+//             [1  2]
+//             [1 10]
+//     values = [1, 2, 3, 4, 5]
+//     shape = [2 50]
+// ```
+//
+// Arguments:
+//	sparse_handles: 1-D, The `N` serialized `SparseTensor` objects.
+// Shape: `[N]`.
+//	dtype: The `dtype` of the `SparseTensor` objects stored in the
+// `SparseTensorsMap`.
+//
+// Returns 2-D.  The `indices` of the minibatch `SparseTensor`.1-D.  The `values` of the minibatch `SparseTensor`.1-D.  The `shape` of the minibatch `SparseTensor`.
+func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype tf.DataType, optional ...TakeManySparseFromTensorsMapAttr) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedResizeBilinear",
+		Type: "TakeManySparseFromTensorsMap",
 		Input: []tf.Input{
-			images, size, min, max,
+			sparse_handles,
 		},
 		Attrs: attrs,
 	}
@@ -14128,182 +14065,164 @@ func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the minimum along segments of a tensor.
+// Says whether the targets are in the top `K` predictions.
 //
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+// prediction for the target class is among the top `k` predictions among
+// all predictions for example `i`. Note that the behavior of `InTopK` differs
+// from the `TopK` op in its handling of ties; if multiple classes have the
+// same prediction value and straddle the top-`k` boundary, all of those
+// classes are considered to be in the top `k`.
 //
-// Computes a tensor such that
-// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
-// that `segment_ids[j] == i`.
+// More formally, let
 //
-// If the min is empty for a given segment ID `i`, `output[i] = 0`.
+//   \\(predictions_i\\) be the predictions for all classes for example `i`,
+//   \\(targets_i\\) be the target class for example `i`,
+//   \\(out_i\\) be the output for example `i`,
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
-// </div>
+// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
 //
 // Arguments:
+//	predictions: A `batch_size` x `classes` tensor.
+//	targets: A `batch_size` vector of class ids.
+//	k: Number of top elements to look at for computing precision.
 //
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns Computed precision at `k` as a `bool Tensor`.
+func InTopKV2(scope *Scope, predictions tf.Output, targets tf.Output, k tf.Output) (precision tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SegmentMin",
+		Type: "InTopKV2",
 		Input: []tf.Input{
-			data, segment_ids,
+			predictions, targets, k,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
-type SdcaOptimizerAttr func(optionalAttr)
-
-// SdcaOptimizerAdaptative sets the optional adaptative attribute to value.
+// Assigns a new value to a variable.
 //
-// value: Whether to use Adapative SDCA for the inner loop.
-// If not specified, defaults to false
-func SdcaOptimizerAdaptative(value bool) SdcaOptimizerAttr {
-	return func(m optionalAttr) {
-		m["adaptative"] = value
+// Any ReadVariableOp with a control dependency on this op is guaranteed to return
+// this value or a subsequent newer value of the variable.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value to set the new tensor to use.
+//
+// Returns the created operation.
+func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AssignVariableOp",
+		Input: []tf.Input{
+			resource, value,
+		},
 	}
+	return scope.AddOperation(opspec)
 }
 
-// Distributed version of Stochastic Dual Coordinate Ascent (SDCA) optimizer for
-//
-// linear models with L1 + L2 regularization. As global optimization objective is
-// strongly-convex, the optimizer optimizes the dual objective at each step. The
-// optimizer applies each update one example at a time. Examples are sampled
-// uniformly, and the optimizer is learning rate free and enjoys linear convergence
-// rate.
+// Returns a tensor of ones with the same shape and type as x.
 //
-// [Proximal Stochastic Dual Coordinate Ascent](http://arxiv.org/pdf/1211.2717v1.pdf).<br>
-// Shai Shalev-Shwartz, Tong Zhang. 2012
+// Arguments:
+//	x: a tensor of type T.
 //
-// $$Loss Objective = \sum f_{i} (wx_{i}) + (l2 / 2) * |w|^2 + l1 * |w|$$
+// Returns a tensor of the same shape and type as x but filled with ones.
+func OnesLike(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "OnesLike",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// The gradient of SparseFillEmptyRows.
 //
-// [Adding vs. Averaging in Distributed Primal-Dual Optimization](http://arxiv.org/abs/1502.03508).<br>
-// Chenxin Ma, Virginia Smith, Martin Jaggi, Michael I. Jordan,
-// Peter Richtarik, Martin Takac. 2015
+// Takes vectors reverse_index_map, shaped `[N]`, and grad_values,
+// shaped `[N_full]`, where `N_full >= N` and copies data into either
+// `d_values` or `d_default_value`.  Here `d_values` is shaped `[N]` and
+// `d_default_value` is a scalar.
 //
-// [Stochastic Dual Coordinate Ascent with Adaptive Probabilities](https://arxiv.org/abs/1502.08053).<br>
-// Dominik Csiba, Zheng Qu, Peter Richtarik. 2015
+//   d_values[j] = grad_values[reverse_index_map[j]]
+//   d_default_value = sum_{k : 0 .. N_full - 1} (
+//      grad_values[k] * 1{k not in reverse_index_map})
 //
 // Arguments:
-//	sparse_example_indices: a list of vectors which contain example indices.
-//	sparse_feature_indices: a list of vectors which contain feature indices.
-//	sparse_feature_values: a list of vectors which contains feature value
-// associated with each feature group.
-//	dense_features: a list of matrices which contains the dense feature values.
-//	example_weights: a vector which contains the weight associated with each
-// example.
-//	example_labels: a vector which contains the label/target associated with each
-// example.
-//	sparse_indices: a list of vectors where each value is the indices which has
-// corresponding weights in sparse_weights. This field maybe omitted for the
-// dense approach.
-//	sparse_weights: a list of vectors where each value is the weight associated with
-// a sparse feature group.
-//	dense_weights: a list of vectors where the values are the weights associated
-// with a dense feature group.
-//	example_state_data: a list of vectors containing the example state data.
-//	loss_type: Type of the primal loss. Currently SdcaSolver supports logistic,
-// squared and hinge losses.
-//	l1: Symmetric l1 regularization strength.
-//	l2: Symmetric l2 regularization strength.
-//	num_loss_partitions: Number of partitions of the global loss function.
-//	num_inner_iterations: Number of iterations per mini-batch.
+//	reverse_index_map: 1-D.  The reverse index map from SparseFillEmptyRows.
+//	grad_values: 1-D.  The gradients from backprop.
 //
-// Returns a list of vectors containing the updated example state
-// data.a list of vectors where each value is the delta
-// weights associated with a sparse feature group.a list of vectors where the values are the delta
-// weights associated with a dense feature group.
-func SdcaOptimizer(scope *Scope, sparse_example_indices []tf.Output, sparse_feature_indices []tf.Output, sparse_feature_values []tf.Output, dense_features []tf.Output, example_weights tf.Output, example_labels tf.Output, sparse_indices []tf.Output, sparse_weights []tf.Output, dense_weights []tf.Output, example_state_data tf.Output, loss_type string, l1 float32, l2 float32, num_loss_partitions int64, num_inner_iterations int64, optional ...SdcaOptimizerAttr) (out_example_state_data tf.Output, out_delta_sparse_weights []tf.Output, out_delta_dense_weights []tf.Output) {
+// Returns 1-D.  The backprop into values.0-D.  The backprop into default_value.
+func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_values tf.Output) (d_values tf.Output, d_default_value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"loss_type": loss_type, "l1": l1, "l2": l2, "num_loss_partitions": num_loss_partitions, "num_inner_iterations": num_inner_iterations}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SdcaOptimizer",
+		Type: "SparseFillEmptyRowsGrad",
 		Input: []tf.Input{
-			tf.OutputList(sparse_example_indices), tf.OutputList(sparse_feature_indices), tf.OutputList(sparse_feature_values), tf.OutputList(dense_features), example_weights, example_labels, tf.OutputList(sparse_indices), tf.OutputList(sparse_weights), tf.OutputList(dense_weights), example_state_data,
+			reverse_index_map, grad_values,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)`
+//
+// if < 0, `scale * features` otherwise.
+//
+// See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
+func Selu(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	out_example_state_data = op.Output(idx)
-	if out_delta_sparse_weights, idx, err = makeOutputList(op, idx, "out_delta_sparse_weights"); err != nil {
-		scope.UpdateErr("SdcaOptimizer", err)
-		return
-	}
-	if out_delta_dense_weights, idx, err = makeOutputList(op, idx, "out_delta_dense_weights"); err != nil {
-		scope.UpdateErr("SdcaOptimizer", err)
-		return
-	}
-	return out_example_state_data, out_delta_sparse_weights, out_delta_dense_weights
-}
-
-// SparseMatMulAttr is an optional argument to SparseMatMul.
-type SparseMatMulAttr func(optionalAttr)
-
-// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
-	}
-}
-
-// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_b"] = value
+	opspec := tf.OpSpec{
+		Type: "Selu",
+		Input: []tf.Input{
+			features,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["a_is_sparse"] = value
-	}
-}
+// SetSizeAttr is an optional argument to SetSize.
+type SetSizeAttr func(optionalAttr)
 
-// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
+// SetSizeValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func SetSizeValidateIndices(value bool) SetSizeAttr {
 	return func(m optionalAttr) {
-		m["b_is_sparse"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Multiply matrix "a" by matrix "b".
+// Number of unique elements along last dimension of input `set`.
 //
-// The inputs must be two-dimensional matrices and the inner dimension of "a" must
-// match the outer dimension of "b". This op is optimized for the case where at
-// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
-// matrix multiply on one platform was 30% zero values in the sparse matrix.
+// Input `set` is a `SparseTensor` represented by `set_indices`, `set_values`,
+// and `set_shape`. The last dimension contains values in a set, duplicates are
+// allowed but ignored.
 //
-// The gradient computation of this operation will only take advantage of sparsity
-// in the input gradient when that gradient comes from a Relu.
-func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
+// If `validate_indices` is `True`, this op validates the order and range of `set`
+// indices.
+//
+// Arguments:
+//	set_indices: 2D `Tensor`, indices of a `SparseTensor`.
+//	set_values: 1D `Tensor`, values of a `SparseTensor`.
+//	set_shape: 1D `Tensor`, shape of a `SparseTensor`.
+//
+// Returns For `set` ranked `n`, this is a `Tensor` with rank `n-1`, and the same 1st
+// `n-1` dimensions as `set`. Each value is the number of unique elements in
+// the corresponding `[0...n-1]` dimension of `set`.
+func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shape tf.Output, optional ...SetSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14312,9 +14231,9 @@ func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatM
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseMatMul",
+		Type: "SetSize",
 		Input: []tf.Input{
-			a, b,
+			set_indices, set_values, set_shape,
 		},
 		Attrs: attrs,
 	}
@@ -14322,52 +14241,64 @@ func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatM
 	return op.Output(0)
 }
 
-// Computes the power of one value to another.
+// Computes the sign and the log of the absolute value of the determinant of
 //
-// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
-// corresponding elements in `x` and `y`. For example:
+// one or more square matrices.
 //
-// ```
-// # tensor 'x' is [[2, 2]], [3, 3]]
-// # tensor 'y' is [[8, 16], [2, 3]]
-// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
-// ```
-func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
+// form square matrices. The outputs are two tensors containing the signs and
+// absolute values of the log determinants for all N input submatrices
+// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
+// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
+// is the LU decomposition of the input and P is the corresponding
+// permutation matrix.
+//
+// Arguments:
+//	input: Shape is `[N, M, M]`.
+//
+// Returns The signs of the log determinants of the inputs. Shape is `[N]`.The logs of the absolute values of the determinants
+// of the N input matrices.  Shape is `[N]`.
+func LogMatrixDeterminant(scope *Scope, input tf.Output) (sign tf.Output, log_abs_determinant tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Pow",
+		Type: "LogMatrixDeterminant",
 		Input: []tf.Input{
-			x, y,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// ShapeAttr is an optional argument to Shape.
-type ShapeAttr func(optionalAttr)
+// SumAttr is an optional argument to Sum.
+type SumAttr func(optionalAttr)
 
-// ShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func ShapeOutType(value tf.DataType) ShapeAttr {
+// SumKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SumKeepDims(value bool) SumAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Returns the shape of a tensor.
+// Computes the sum of elements across dimensions of a tensor.
 //
-// This operation returns a 1-D integer tensor representing the shape of `input`.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
-// For example:
+// Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
+// Returns The reduced tensor.
+func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14376,9 +14307,9 @@ func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Shape",
+		Type: "Sum",
 		Input: []tf.Input{
-			input,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -14386,203 +14317,191 @@ func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Outp
 	return op.Output(0)
 }
 
-// Computes fingerprints of the input strings.
+// Delete the tensor specified by its handle in the session.
 //
 // Arguments:
-//	input: vector of strings to compute fingerprints on.
+//	handle: The handle for a tensor stored in the session state.
 //
-// Returns a (N,2) shaped matrix where N is the number of elements in the input
-// vector. Each row contains the low and high parts of the fingerprint.
-func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns the created operation.
+func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SdcaFprint",
+		Type: "DeleteSessionTensor",
 		Input: []tf.Input{
-			input,
+			handle,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
-type RandomPoissonV2Attr func(optionalAttr)
-
-// RandomPoissonV2Seed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// RandomPoissonV2Dtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
-	}
+	return scope.AddOperation(opspec)
 }
 
-// Outputs random values from the Poisson distribution(s) described by rate.
+// L2 Loss.
 //
-// This op uses two algorithms, depending on rate. If rate >= 10, then
-// the algorithm by Hormann is used to acquire samples via
-// transformation-rejection.
-// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
+// Computes half the L2 norm of a tensor without the `sqrt`:
 //
-// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
-// random variables.
-// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
-// Programming, Volume 2. Addison Wesley
+//     output = sum(t ** 2) / 2
 //
 // Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in rate.
-//	rate: A tensor in which each scalar is a "rate" parameter describing the
-// associated poisson distribution.
+//	t: Typically 2-D, but may have any dimensions.
 //
-// Returns A tensor with shape `shape + shape(rate)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `rate[i0, i1, ...iN]`.
-func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
+// Returns 0-D.
+func L2Loss(scope *Scope, t tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomPoissonV2",
+		Type: "L2Loss",
 		Input: []tf.Input{
-			shape, rate,
+			t,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
-type MatrixTriangularSolveAttr func(optionalAttr)
+// DenseToSparseSetOperationAttr is an optional argument to DenseToSparseSetOperation.
+type DenseToSparseSetOperationAttr func(optionalAttr)
 
-// MatrixTriangularSolveLower sets the optional lower attribute to value.
-//
-// value: Boolean indicating whether the innermost matrices in `matrix` are
-// lower or upper triangular.
+// DenseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
 // If not specified, defaults to true
-func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
+func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperationAttr {
 	return func(m optionalAttr) {
-		m["lower"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
-//
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-//          adjoint.
+// Applies set operation along last dimension of `Tensor` and `SparseTensor`.
 //
-// @compatibility(numpy)
-// Equivalent to np.linalg.triangular_solve
-// @end_compatibility
-// If not specified, defaults to false
-func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
-	return func(m optionalAttr) {
-		m["adjoint"] = value
-	}
-}
-
-// Solves systems of linear equations with upper or lower triangular matrices by
+// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
 //
-// backsubstitution.
+// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
+// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
 //
-// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
-// square matrices. If `lower` is `True` then the strictly upper triangular part
-// of each inner-most matrix is assumed to be zero and not accessed.
-// If `lower` is False then the strictly lower triangular part of each inner-most
-// matrix is assumed to be zero and not accessed.
-// `rhs` is a tensor of shape `[..., M, K]`.
+// If `validate_indices` is `True`, this op validates the order and range of `set2`
+// indices.
 //
-// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
-// `True` then the innermost matrices in `output` satisfy matrix equations
-// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `False` then the strictly then the  innermost matrices in
-// `output` satisfy matrix equations
-// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+// Output `result` is a `SparseTensor` represented by `result_indices`,
+// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+// dimension contains the result of `set_operation` applied to the corresponding
+// `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
+// be the same as the 1st `n-1` dimensions of `set1`, `result_shape[n]` is the
+// max set size across `n-1` dimensions.
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
+//
+// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
+// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
+// is the max result set size across all `0...n-1` dimensions.
+func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...DenseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"set_operation": set_operation}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixTriangularSolve",
+		Type: "DenseToSparseSetOperation",
 		Input: []tf.Input{
-			matrix, rhs,
+			set1, set2_indices, set2_values, set2_shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes inverse hyperbolic sine of x element-wise.
-func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
+// Subtracts a value from the current value of a variable.
+//
+// Any ReadVariableOp which depends directly or indirectly on this assign is
+// guaranteed to see the incremented value or a subsequent newer one.
+//
+// Outputs the incremented value, which can be used to totally order the
+// increments to this variable.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value by which the variable will be incremented.
+//
+// Returns the created operation.
+func AssignSubVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Asinh",
+		Type: "AssignSubVariableOp",
 		Input: []tf.Input{
-			x,
+			resource, value,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Creates a dataset with a range of values. Corresponds to python's xrange.
+// RestoreAttr is an optional argument to Restore.
+type RestoreAttr func(optionalAttr)
+
+// RestorePreferredShard sets the optional preferred_shard attribute to value.
 //
-// Arguments:
-//	start: corresponds to start in python's xrange().
-//	stop: corresponds to stop in python's xrange().
-//	step: corresponds to step in python's xrange().
+// value: Index of file to open first if multiple files match
+// `file_pattern`.
+// If not specified, defaults to -1
+func RestorePreferredShard(value int64) RestoreAttr {
+	return func(m optionalAttr) {
+		m["preferred_shard"] = value
+	}
+}
+
+// Restores a tensor from checkpoint files.
 //
+// Reads a tensor stored in one or several files. If there are several files (for
+// instance because a tensor was saved as slices), `file_pattern` may contain
+// wildcard symbols (`*` and `?`) in the filename portion only, not in the
+// directory portion.
 //
-func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// If a `file_pattern` matches several files, `preferred_shard` can be used to hint
+// in which file the requested tensor is likely to be found. This op will first
+// open the file at index `preferred_shard` in the list of matching files and try
+// to restore tensors from that file.  Only if some tensors or tensor slices are
+// not found in that first file, then the Op opens all the files. Setting
+// `preferred_shard` to match the value passed as the `shard` input
+// of a matching `Save` Op may speed up Restore.  This attribute only affects
+// performance, not correctness.  The default value -1 means files are processed in
+// order.
+//
+// See also `RestoreSlice`.
+//
+// Arguments:
+//	file_pattern: Must have a single element. The pattern of the files from
+// which we read the tensor.
+//	tensor_name: Must have a single element. The name of the tensor to be
+// restored.
+//	dt: The type of the tensor to be restored.
+//
+// Returns The restored tensor.
+func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf.DataType, optional ...RestoreAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"dt": dt}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "RangeDataset",
+		Type: "Restore",
 		Input: []tf.Input{
-			start, stop, step,
+			file_pattern, tensor_name,
 		},
 		Attrs: attrs,
 	}
@@ -14590,212 +14509,229 @@ func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output,
 	return op.Output(0)
 }
 
-// DepthwiseConv2dNativeBackpropInputAttr is an optional argument to DepthwiseConv2dNativeBackpropInput.
-type DepthwiseConv2dNativeBackpropInputAttr func(optionalAttr)
+// QuantizedResizeBilinearAttr is an optional argument to QuantizedResizeBilinear.
+type QuantizedResizeBilinearAttr func(optionalAttr)
 
-// DepthwiseConv2dNativeBackpropInputDataFormat sets the optional data_format attribute to value.
+// QuantizedResizeBilinearAlignCorners sets the optional align_corners attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
-// If not specified, defaults to "NHWC"
-func DepthwiseConv2dNativeBackpropInputDataFormat(value string) DepthwiseConv2dNativeBackpropInputAttr {
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func QuantizedResizeBilinearAlignCorners(value bool) QuantizedResizeBilinearAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["align_corners"] = value
 	}
 }
 
-// DepthwiseConv2dNativeBackpropInputDilations sets the optional dilations attribute to value.
+// Resize quantized `images` to `size` using quantized bilinear interpolation.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func DepthwiseConv2dNativeBackpropInputDilations(value []int64) DepthwiseConv2dNativeBackpropInputAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of depthwise convolution with respect to the input.
+// Input images and output images must be quantized types.
 //
 // Arguments:
-//	input_sizes: An integer vector representing the shape of `input`, based
-// on `data_format`.  For example, if `data_format` is 'NHWC' then
-//  `input` is a 4-D `[batch, height, width, channels]` tensor.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, depthwise_multiplier]`.
-//	out_backprop: 4-D with shape  based on `data_format`.
-// For example, if `data_format` is 'NHWC' then
-// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution.
-//	padding: The type of padding algorithm to use.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns 4-D with shape according to `data_format`.  For example, if
-// `data_format` is 'NHWC', output shape is `[batch, in_height,
-// in_width, in_channels]`.  Gradient w.r.t. the input of the
-// convolution.
-func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropInputAttr) (output tf.Output) {
+//
+//
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min tf.Output, max tf.Output, optional ...QuantizedResizeBilinearAttr) (resized_images tf.Output, out_min tf.Output, out_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DepthwiseConv2dNativeBackpropInput",
+		Type: "QuantizedResizeBilinear",
 		Input: []tf.Input{
-			input_sizes, filter, out_backprop,
+			images, size, min, max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Adds sparse updates to the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] += updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] += updates[i, ...]
+// Computes the minimum along segments of a tensor.
 //
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions add.
+// Computes a tensor such that
+// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
+// that `segment_ids[j] == i`.
 //
-// Requires `updates.shape = indices.shape + ref.shape[1:]`.
+// If the min is empty for a given segment ID `i`, `output[i] = 0`.
 //
 // <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
 // </div>
 //
 // Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
 //
-// Returns the created operation.
-func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceScatterAdd",
-		Input: []tf.Input{
-			resource, indices, updates,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Computes the gradient for the inverse of `x` wrt its input.
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
 //
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReciprocalGrad",
+		Type: "SegmentMin",
 		Input: []tf.Input{
-			y, dy,
+			data, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+// SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
+type SdcaOptimizerAttr func(optionalAttr)
+
+// SdcaOptimizerAdaptative sets the optional adaptative attribute to value.
 //
-// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Minimum",
+// value: Whether to use Adapative SDCA for the inner loop.
+// If not specified, defaults to false
+func SdcaOptimizerAdaptative(value bool) SdcaOptimizerAttr {
+	return func(m optionalAttr) {
+		m["adaptative"] = value
+	}
+}
+
+// Distributed version of Stochastic Dual Coordinate Ascent (SDCA) optimizer for
+//
+// linear models with L1 + L2 regularization. As global optimization objective is
+// strongly-convex, the optimizer optimizes the dual objective at each step. The
+// optimizer applies each update one example at a time. Examples are sampled
+// uniformly, and the optimizer is learning rate free and enjoys linear convergence
+// rate.
+//
+// [Proximal Stochastic Dual Coordinate Ascent](http://arxiv.org/pdf/1211.2717v1.pdf).<br>
+// Shai Shalev-Shwartz, Tong Zhang. 2012
+//
+// $$Loss Objective = \sum f_{i} (wx_{i}) + (l2 / 2) * |w|^2 + l1 * |w|$$
+//
+// [Adding vs. Averaging in Distributed Primal-Dual Optimization](http://arxiv.org/abs/1502.03508).<br>
+// Chenxin Ma, Virginia Smith, Martin Jaggi, Michael I. Jordan,
+// Peter Richtarik, Martin Takac. 2015
+//
+// [Stochastic Dual Coordinate Ascent with Adaptive Probabilities](https://arxiv.org/abs/1502.08053).<br>
+// Dominik Csiba, Zheng Qu, Peter Richtarik. 2015
+//
+// Arguments:
+//	sparse_example_indices: a list of vectors which contain example indices.
+//	sparse_feature_indices: a list of vectors which contain feature indices.
+//	sparse_feature_values: a list of vectors which contains feature value
+// associated with each feature group.
+//	dense_features: a list of matrices which contains the dense feature values.
+//	example_weights: a vector which contains the weight associated with each
+// example.
+//	example_labels: a vector which contains the label/target associated with each
+// example.
+//	sparse_indices: a list of vectors where each value is the indices which has
+// corresponding weights in sparse_weights. This field maybe omitted for the
+// dense approach.
+//	sparse_weights: a list of vectors where each value is the weight associated with
+// a sparse feature group.
+//	dense_weights: a list of vectors where the values are the weights associated
+// with a dense feature group.
+//	example_state_data: a list of vectors containing the example state data.
+//	loss_type: Type of the primal loss. Currently SdcaSolver supports logistic,
+// squared and hinge losses.
+//	l1: Symmetric l1 regularization strength.
+//	l2: Symmetric l2 regularization strength.
+//	num_loss_partitions: Number of partitions of the global loss function.
+//	num_inner_iterations: Number of iterations per mini-batch.
+//
+// Returns a list of vectors containing the updated example state
+// data.a list of vectors where each value is the delta
+// weights associated with a sparse feature group.a list of vectors where the values are the delta
+// weights associated with a dense feature group.
+func SdcaOptimizer(scope *Scope, sparse_example_indices []tf.Output, sparse_feature_indices []tf.Output, sparse_feature_values []tf.Output, dense_features []tf.Output, example_weights tf.Output, example_labels tf.Output, sparse_indices []tf.Output, sparse_weights []tf.Output, dense_weights []tf.Output, example_state_data tf.Output, loss_type string, l1 float32, l2 float32, num_loss_partitions int64, num_inner_iterations int64, optional ...SdcaOptimizerAttr) (out_example_state_data tf.Output, out_delta_sparse_weights []tf.Output, out_delta_dense_weights []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"loss_type": loss_type, "l1": l1, "l2": l2, "num_loss_partitions": num_loss_partitions, "num_inner_iterations": num_inner_iterations}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "SdcaOptimizer",
 		Input: []tf.Input{
-			x, y,
+			tf.OutputList(sparse_example_indices), tf.OutputList(sparse_feature_indices), tf.OutputList(sparse_feature_values), tf.OutputList(dense_features), example_weights, example_labels, tf.OutputList(sparse_indices), tf.OutputList(sparse_weights), tf.OutputList(dense_weights), example_state_data,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	out_example_state_data = op.Output(idx)
+	if out_delta_sparse_weights, idx, err = makeOutputList(op, idx, "out_delta_sparse_weights"); err != nil {
+		scope.UpdateErr("SdcaOptimizer", err)
+		return
+	}
+	if out_delta_dense_weights, idx, err = makeOutputList(op, idx, "out_delta_dense_weights"); err != nil {
+		scope.UpdateErr("SdcaOptimizer", err)
+		return
+	}
+	return out_example_state_data, out_delta_sparse_weights, out_delta_dense_weights
 }
 
-// MfccAttr is an optional argument to Mfcc.
-type MfccAttr func(optionalAttr)
+// SparseMatMulAttr is an optional argument to SparseMatMul.
+type SparseMatMulAttr func(optionalAttr)
 
-// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
-//
-// value: The highest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 4000
-func MfccUpperFrequencyLimit(value float32) MfccAttr {
+// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
+// If not specified, defaults to false
+func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["upper_frequency_limit"] = value
+		m["transpose_a"] = value
 	}
 }
 
-// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
-//
-// value: The lowest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 20
-func MfccLowerFrequencyLimit(value float32) MfccAttr {
+// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
+// If not specified, defaults to false
+func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["lower_frequency_limit"] = value
+		m["transpose_b"] = value
 	}
 }
 
-// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
-//
-// value: Resolution of the Mel bank used internally.
-// If not specified, defaults to 40
-func MfccFilterbankChannelCount(value int64) MfccAttr {
+// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["filterbank_channel_count"] = value
+		m["a_is_sparse"] = value
 	}
 }
 
-// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
-//
-// value: How many output channels to produce per time slice.
-// If not specified, defaults to 13
-func MfccDctCoefficientCount(value int64) MfccAttr {
+// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["dct_coefficient_count"] = value
+		m["b_is_sparse"] = value
 	}
 }
 
-// Transforms a spectrogram into a form that's useful for speech recognition.
+// Multiply matrix "a" by matrix "b".
 //
-// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
-// been effective as an input feature for machine learning. They are created by
-// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
-// higher frequencies that are less significant to the human ear. They have a long
-// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
-// is a good resource to learn more.
+// The inputs must be two-dimensional matrices and the inner dimension of "a" must
+// match the outer dimension of "b". This op is optimized for the case where at
+// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
+// matrix multiply on one platform was 30% zero values in the sparse matrix.
 //
-// Arguments:
-//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
-// set to true.
-//	sample_rate: How many samples per second the source audio used.
-func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
+// The gradient computation of this operation will only take advantage of sparsity
+// in the input gradient when that gradient comes from a Relu.
+func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14804,9 +14740,9 @@ func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional .
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Mfcc",
+		Type: "SparseMatMul",
 		Input: []tf.Input{
-			spectrogram, sample_rate,
+			a, b,
 		},
 		Attrs: attrs,
 	}
@@ -14814,244 +14750,259 @@ func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional .
 	return op.Output(0)
 }
 
-// Returns the element-wise sum of a list of tensors.
-//
-// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
-// wait for all of its inputs to be ready before beginning to sum. This can
-// save memory if inputs are ready at different times, since minimum temporary
-// storage is proportional to the output size rather than the inputs size.
-//
-// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+// MultinomialAttr is an optional argument to Multinomial.
+type MultinomialAttr func(optionalAttr)
+
+// MultinomialSeed sets the optional seed attribute to value.
 //
-// Returns a `Tensor` of same shape and type as the elements of `inputs`.
+// value: If either seed or seed2 is set to be non-zero, the internal random number
+// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// If not specified, defaults to 0
+func MultinomialSeed(value int64) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// MultinomialSeed2 sets the optional seed2 attribute to value.
 //
-// Arguments:
-//	inputs: A list of `Tensor` objects, each with same shape and type.
-//	shape: Shape of elements of `inputs`.
-func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func MultinomialSeed2(value int64) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	attrs := map[string]interface{}{"shape": shape}
-	opspec := tf.OpSpec{
-		Type: "AccumulateNV2",
-		Input: []tf.Input{
-			tf.OutputList(inputs),
-		},
-		Attrs: attrs,
+}
+
+// MultinomialOutputDtype sets the optional output_dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["output_dtype"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Convert the quantized 'input' tensor into a lower-precision 'output', using the
-//
-// actual distribution of the values to maximize the usage of the lower bit depth
-// and adjusting the output min and max ranges accordingly.
-//
-// [input_min, input_max] are scalar floats that specify the range for the float
-// interpretation of the 'input' data. For example, if input_min is -1.0f and
-// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
-// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
-//
-// This operator tries to squeeze as much precision as possible into an output with
-// a lower bit depth by calculating the actual min and max values found in the
-// data. For example, maybe that quint16 input has no values lower than 16,384 and
-// none higher than 49,152. That means only half the range is actually needed, all
-// the float interpretations are between -0.5f and 0.5f, so if we want to compress
-// the data into a quint8 output, we can use that range rather than the theoretical
-// -1.0f to 1.0f that is suggested by the input min and max.
-//
-// In practice, this is most useful for taking output from operations like
-// QuantizedMatMul that can produce higher bit-depth outputs than their inputs and
-// may have large potential output ranges, but in practice have a distribution of
-// input values that only uses a small fraction of the possible range. By feeding
-// that output into this operator, we can reduce it from 32 bits down to 8 with
-// minimal loss of accuracy.
+// Draws samples from a multinomial distribution.
 //
 // Arguments:
+//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
+// represents the unnormalized log probabilities for all classes.
+//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
 //
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//	out_type: The type of the output. Should be a lower bit depth than Tinput.
-//
-// Returns The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
-func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
+// contains the drawn class labels with range `[0, num_classes)`.
+func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeDownAndShrinkRange",
+		Type: "Multinomial",
 		Input: []tf.Input{
-			input, input_min, input_max,
+			logits, num_samples,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// RandomGammaAttr is an optional argument to RandomGamma.
-type RandomGammaAttr func(optionalAttr)
+// EncodeJpegAttr is an optional argument to EncodeJpeg.
+type EncodeJpegAttr func(optionalAttr)
 
-// RandomGammaSeed sets the optional seed attribute to value.
+// EncodeJpegFormat sets the optional format attribute to value.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomGammaSeed(value int64) RandomGammaAttr {
+// value: Per pixel image format.
+// If not specified, defaults to ""
+func EncodeJpegFormat(value string) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["format"] = value
 	}
 }
 
-// RandomGammaSeed2 sets the optional seed2 attribute to value.
+// EncodeJpegQuality sets the optional quality attribute to value.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomGammaSeed2(value int64) RandomGammaAttr {
+// value: Quality of the compression from 0 to 100 (higher is better and slower).
+// If not specified, defaults to 95
+func EncodeJpegQuality(value int64) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["quality"] = value
 	}
 }
 
-// Outputs random values from the Gamma distribution(s) described by alpha.
-//
-// This op uses the algorithm by Marsaglia et al. to acquire samples via
-// transformation-rejection from pairs of uniform and normal random variables.
-// See http://dl.acm.org/citation.cfm?id=358414
+// EncodeJpegProgressive sets the optional progressive attribute to value.
 //
-// Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in alpha.
-//	alpha: A tensor in which each scalar is a "shape" parameter describing the
-// associated gamma distribution.
+// value: If True, create a JPEG that loads progressively (coarse to fine).
+// If not specified, defaults to false
+func EncodeJpegProgressive(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["progressive"] = value
+	}
+}
+
+// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
 //
-// Returns A tensor with shape `shape + shape(alpha)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
-func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: If True, spend CPU/RAM to reduce size with no quality change.
+// If not specified, defaults to false
+func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["optimize_size"] = value
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+}
+
+// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
+//
+// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
+// If not specified, defaults to true
+func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["chroma_downsampling"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "RandomGamma",
-		Input: []tf.Input{
-			shape, alpha,
-		},
-		Attrs: attrs,
+}
+
+// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+//
+// value: Unit used to specify `x_density` and `y_density`:
+// pixels per inch (`'in'`) or centimeter (`'cm'`).
+// If not specified, defaults to "in"
+func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["density_unit"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
-type QuantizedConv2DAttr func(optionalAttr)
+// EncodeJpegXDensity sets the optional x_density attribute to value.
+//
+// value: Horizontal pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegXDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["x_density"] = value
+	}
+}
 
-// QuantizedConv2DOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
+// EncodeJpegYDensity sets the optional y_density attribute to value.
+//
+// value: Vertical pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegYDensity(value int64) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["y_density"] = value
 	}
 }
 
-// QuantizedConv2DDilations sets the optional dilations attribute to value.
+// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func QuantizedConv2DDilations(value []int64) QuantizedConv2DAttr {
+// value: If not empty, embed this XMP metadata in the image header.
+// If not specified, defaults to ""
+func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["xmp_metadata"] = value
 	}
 }
 
-// Computes a 2D convolution given quantized 4D input and filter tensors.
+// JPEG-encode an image.
 //
-// The inputs are quantized tensors where the lowest value represents the real
-// number of the associated minimum, and the highest represents the maximum.
-// This means that you can only interpret the quantized output in the same way, by
-// taking the returned minimum and maximum values into account.
+// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
 //
-// Arguments:
+// The attr `format` can be used to override the color format of the encoded
+// output.  Values can be:
 //
-//	filter: filter's input_depth dimension must match input's depth dimensions.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	min_filter: The float value that the lowest quantized filter value represents.
-//	max_filter: The float value that the highest quantized filter value represents.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor.
-//	padding: The type of padding algorithm to use.
+// *   `''`: Use a default format based on the number of channels in the image.
+// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
+//     of `image` must be 1.
+// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
+//     of `image` must be 3.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
+// If `format` is not specified or is the empty string, a default format is picked
+// in function of the number of channels in `image`:
+//
+// *   1: Output a grayscale image.
+// *   3: Output an RGB image.
+//
+// Arguments:
+//	image: 3-D with shape `[height, width, channels]`.
+//
+// Returns 0-D. JPEG-encoded image.
+func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedConv2D",
+		Type: "EncodeJpeg",
 		Input: []tf.Input{
-			input, filter, min_input, max_input, min_filter, max_filter,
+			image,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// ResourceGatherAttr is an optional argument to ResourceGather.
-type ResourceGatherAttr func(optionalAttr)
+// Computes the power of one value to another.
+//
+// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
+// corresponding elements in `x` and `y`. For example:
+//
+// ```
+// # tensor 'x' is [[2, 2]], [3, 3]]
+// # tensor 'y' is [[8, 16], [2, 3]]
+// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
+// ```
+func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Pow",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
+// ShapeAttr is an optional argument to Shape.
+type ShapeAttr func(optionalAttr)
+
+// ShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func ShapeOutType(value tf.DataType) ShapeAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["out_type"] = value
 	}
 }
 
-// Gather slices from the variable pointed to by `resource` according to `indices`.
-//
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
+// Returns the shape of a tensor.
 //
-// ```python
-//     # Scalar indices
-//     output[:, ..., :] = params[indices, :, ... :]
+// This operation returns a 1-D integer tensor representing the shape of `input`.
 //
-//     # Vector indices
-//     output[i, :, ..., :] = params[indices[i], :, ... :]
+// For example:
 //
-//     # Higher rank indices
-//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
 // ```
-func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceGather",
+		Type: "Shape",
 		Input: []tf.Input{
-			resource, indices,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -15059,70 +15010,82 @@ func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype t
 	return op.Output(0)
 }
 
-// Delete the TensorArray from its resource container.
-//
-// This enables the user to close and release the resource in the middle
-// of a step/run.
+// Computes fingerprints of the input strings.
 //
 // Arguments:
-//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
+//	input: vector of strings to compute fingerprints on.
 //
-// Returns the created operation.
-func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
+// Returns a (N,2) shaped matrix where N is the number of elements in the input
+// vector. Each row contains the low and high parts of the fingerprint.
+func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayCloseV3",
+		Type: "SdcaFprint",
 		Input: []tf.Input{
-			handle,
+			input,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// RandomUniformIntAttr is an optional argument to RandomUniformInt.
-type RandomUniformIntAttr func(optionalAttr)
+// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
+type RandomPoissonV2Attr func(optionalAttr)
 
-// RandomUniformIntSeed sets the optional seed attribute to value.
+// RandomPoissonV2Seed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
+func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Outputs random integers from a uniform distribution.
+// RandomPoissonV2Dtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from the Poisson distribution(s) described by rate.
 //
-// The generated values are uniform integers in the range `[minval, maxval)`.
-// The lower bound `minval` is included in the range, while the upper bound
-// `maxval` is excluded.
+// This op uses two algorithms, depending on rate. If rate >= 10, then
+// the algorithm by Hormann is used to acquire samples via
+// transformation-rejection.
+// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
 //
-// The random integers are slightly biased unless `maxval - minval` is an exact
-// power of two.  The bias is small for values of `maxval - minval` significantly
-// smaller than the range of the output (either `2^32` or `2^64`).
+// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+// random variables.
+// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+// Programming, Volume 2. Addison Wesley
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	minval: 0-D.  Inclusive lower bound on the generated integers.
-//	maxval: 0-D.  Exclusive upper bound on the generated integers.
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in rate.
+//	rate: A tensor in which each scalar is a "rate" parameter describing the
+// associated poisson distribution.
 //
-// Returns A tensor of the specified shape filled with uniform random integers.
-func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
+// Returns A tensor with shape `shape + shape(rate)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `rate[i0, i1, ...iN]`.
+func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15131,9 +15094,9 @@ func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniformInt",
+		Type: "RandomPoissonV2",
 		Input: []tf.Input{
-			shape, minval, maxval,
+			shape, rate,
 		},
 		Attrs: attrs,
 	}
@@ -15141,98 +15104,109 @@ func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf
 	return op.Output(0)
 }
 
-// SkipgramAttr is an optional argument to Skipgram.
-type SkipgramAttr func(optionalAttr)
+// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
+type MatrixTriangularSolveAttr func(optionalAttr)
 
-// SkipgramWindowSize sets the optional window_size attribute to value.
+// MatrixTriangularSolveLower sets the optional lower attribute to value.
 //
-// value: The number of words to predict to the left and right of the target.
-// If not specified, defaults to 5
-func SkipgramWindowSize(value int64) SkipgramAttr {
+// value: Boolean indicating whether the innermost matrices in `matrix` are
+// lower or upper triangular.
+// If not specified, defaults to true
+func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
 	return func(m optionalAttr) {
-		m["window_size"] = value
+		m["lower"] = value
 	}
 }
 
-// SkipgramMinCount sets the optional min_count attribute to value.
+// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
 //
-// value: The minimum number of word occurrences for it to be included in the
-// vocabulary.
-// If not specified, defaults to 5
-func SkipgramMinCount(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["min_count"] = value
-	}
-}
-
-// SkipgramSubsample sets the optional subsample attribute to value.
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+//          adjoint.
 //
-// value: Threshold for word occurrence. Words that appear with higher
-// frequency will be randomly down-sampled. Set to 0 to disable.
-// If not specified, defaults to 0.001
-func SkipgramSubsample(value float32) SkipgramAttr {
+// @compatibility(numpy)
+// Equivalent to np.linalg.triangular_solve
+// @end_compatibility
+// If not specified, defaults to false
+func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
 	return func(m optionalAttr) {
-		m["subsample"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Parses a text file and creates a batch of examples.
+// Solves systems of linear equations with upper or lower triangular matrices by
 //
-// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+// backsubstitution.
+//
+// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
+// square matrices. If `lower` is `True` then the strictly upper triangular part
+// of each inner-most matrix is assumed to be zero and not accessed.
+// If `lower` is False then the strictly lower triangular part of each inner-most
+// matrix is assumed to be zero and not accessed.
+// `rhs` is a tensor of shape `[..., M, K]`.
+//
+// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
+// `True` then the innermost matrices in `output` satisfy matrix equations
+// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `False` then the strictly then the  innermost matrices in
+// `output` satisfy matrix equations
+// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
 //
 // Arguments:
-//	filename: The corpus's text file name.
-//	batch_size: The size of produced batch.
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
 //
-// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
-func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
+// Returns Shape is `[..., M, K]`.
+func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Skipgram",
-
+		Type: "MatrixTriangularSolve",
+		Input: []tf.Input{
+			matrix, rhs,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
+	return op.Output(0)
 }
 
-// StringToNumberAttr is an optional argument to StringToNumber.
-type StringToNumberAttr func(optionalAttr)
-
-// StringToNumberOutType sets the optional out_type attribute to value.
-//
-// value: The numeric type to interpret each string in `string_tensor` as.
-// If not specified, defaults to DT_FLOAT
-func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
+// Computes inverse hyperbolic sine of x element-wise.
+func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Asinh",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to the specified numeric type.
+// Creates a dataset with a range of values. Corresponds to python's xrange.
 //
-// (Note that int32 overflow results in an error while float overflow
-// results in a rounded value.)
+// Arguments:
+//	start: corresponds to start in python's xrange().
+//	stop: corresponds to stop in python's xrange().
+//	step: corresponds to step in python's xrange().
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
+//
+func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "StringToNumber",
+		Type: "RangeDataset",
 		Input: []tf.Input{
-			string_tensor,
+			start, stop, step,
 		},
 		Attrs: attrs,
 	}
@@ -15240,271 +15214,253 @@ func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToN
 	return op.Output(0)
 }
 
-// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
-type ResourceApplyFtrlV2Attr func(optionalAttr)
+// DepthwiseConv2dNativeBackpropInputAttr is an optional argument to DepthwiseConv2dNativeBackpropInput.
+type DepthwiseConv2dNativeBackpropInputAttr func(optionalAttr)
 
-// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
+// DepthwiseConv2dNativeBackpropInputDataFormat sets the optional data_format attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func DepthwiseConv2dNativeBackpropInputDataFormat(value string) DepthwiseConv2dNativeBackpropInputAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["data_format"] = value
 	}
 }
 
-// Update '*var' according to the Ftrl-proximal scheme.
+// DepthwiseConv2dNativeBackpropInputDilations sets the optional dilations attribute to value.
 //
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func DepthwiseConv2dNativeBackpropInputDilations(value []int64) DepthwiseConv2dNativeBackpropInputAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of depthwise convolution with respect to the input.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
-//
-//	lr_power: Scaling factor. Must be a scalar.
+//	input_sizes: An integer vector representing the shape of `input`, based
+// on `data_format`.  For example, if `data_format` is 'NHWC' then
+//  `input` is a 4-D `[batch, height, width, channels]` tensor.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, depthwise_multiplier]`.
+//	out_backprop: 4-D with shape  based on `data_format`.
+// For example, if `data_format` is 'NHWC' then
+// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution.
+//	padding: The type of padding algorithm to use.
 //
-// Returns the created operation.
-func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
+// Returns 4-D with shape according to `data_format`.  For example, if
+// `data_format` is 'NHWC', output shape is `[batch, in_height,
+// in_width, in_channels]`.  Gradient w.r.t. the input of the
+// convolution.
+func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropInputAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrlV2",
+		Type: "DepthwiseConv2dNativeBackpropInput",
 		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
+			input_sizes, filter, out_backprop,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// TruncatedNormalAttr is an optional argument to TruncatedNormal.
-type TruncatedNormalAttr func(optionalAttr)
-
-// TruncatedNormalSeed sets the optional seed attribute to value.
+// Adds sparse updates to the variable referenced by `resource`.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
+// This operation computes
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from a truncated normal distribution.
+//     # Scalar indices
+//     ref[indices, ...] += updates[...]
 //
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] += updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions add.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
 //
-// Returns A tensor of the specified shape filled with random truncated normal
-// values.
-func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
+// Returns the created operation.
+func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TruncatedNormal",
+		Type: "ResourceScatterAdd",
 		Input: []tf.Input{
-			shape,
+			resource, indices, updates,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// RandomShuffleAttr is an optional argument to RandomShuffle.
-type RandomShuffleAttr func(optionalAttr)
-
-// RandomShuffleSeed sets the optional seed attribute to value.
+// Computes the gradient for the inverse of `x` wrt its input.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomShuffleSeed(value int64) RandomShuffleAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// RandomShuffleSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomShuffleSeed2(value int64) RandomShuffleAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+	opspec := tf.OpSpec{
+		Type: "ReciprocalGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Randomly shuffles a tensor along its first dimension.
-//
-//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
-//   to one and only one `output[i]`. For example, a mapping that might occur for a
-//   3x2 tensor is:
-//
-// ```
-// [[1, 2],       [[5, 6],
-//  [3, 4],  ==>   [1, 2],
-//  [5, 6]]        [3, 4]]
-// ```
-//
-// Arguments:
-//	value: The tensor to be shuffled.
+// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
 //
-// Returns A tensor of same shape and type as `value`, shuffled along its first
-// dimension.
-func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
+// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomShuffle",
+		Type: "Minimum",
 		Input: []tf.Input{
-			value,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
-type OrderedMapIncompleteSizeAttr func(optionalAttr)
+// MfccAttr is an optional argument to Mfcc.
+type MfccAttr func(optionalAttr)
 
-// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
+// value: The highest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 4000
+func MfccUpperFrequencyLimit(value float32) MfccAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["upper_frequency_limit"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
+// value: The lowest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 20
+func MfccLowerFrequencyLimit(value float32) MfccAttr {
 	return func(m optionalAttr) {
-		m["memory_limit"] = value
+		m["lower_frequency_limit"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
+// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
+//
+// value: Resolution of the Mel bank used internally.
+// If not specified, defaults to 40
+func MfccFilterbankChannelCount(value int64) MfccAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["filterbank_channel_count"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
+// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
+//
+// value: How many output channels to produce per time slice.
+// If not specified, defaults to 13
+func MfccDctCoefficientCount(value int64) MfccAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["dct_coefficient_count"] = value
 	}
 }
 
-// Op returns the number of incomplete elements in the underlying container.
-func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
+// Transforms a spectrogram into a form that's useful for speech recognition.
+//
+// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
+// been effective as an input feature for machine learning. They are created by
+// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
+// higher frequencies that are less significant to the human ear. They have a long
+// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+// is a good resource to learn more.
+//
+// Arguments:
+//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
+// set to true.
+//	sample_rate: How many samples per second the source audio used.
+func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapIncompleteSize",
-
+		Type: "Mfcc",
+		Input: []tf.Input{
+			spectrogram, sample_rate,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DecodeRawAttr is an optional argument to DecodeRaw.
-type DecodeRawAttr func(optionalAttr)
-
-// DecodeRawLittleEndian sets the optional little_endian attribute to value.
+// Returns the element-wise sum of a list of tensors.
 //
-// value: Whether the input `bytes` are in little-endian order.
-// Ignored for `out_type` values that are stored in a single byte like
-// `uint8`.
-// If not specified, defaults to true
-func DecodeRawLittleEndian(value bool) DecodeRawAttr {
-	return func(m optionalAttr) {
-		m["little_endian"] = value
-	}
-}
-
-// Reinterpret the bytes of a string as a vector of numbers.
+// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+// wait for all of its inputs to be ready before beginning to sum. This can
+// save memory if inputs are ready at different times, since minimum temporary
+// storage is proportional to the output size rather than the inputs size.
 //
-// Arguments:
-//	bytes: All the elements must have the same length.
+// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
 //
+// Returns a `Tensor` of same shape and type as the elements of `inputs`.
 //
-// Returns A Tensor with one more dimension than the input `bytes`.  The
-// added dimension will have size equal to the length of the elements
-// of `bytes` divided by the number of bytes to represent `out_type`.
-func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...DecodeRawAttr) (output tf.Output) {
+// Arguments:
+//	inputs: A list of `Tensor` objects, each with same shape and type.
+//	shape: Shape of elements of `inputs`.
+func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "DecodeRaw",
+		Type: "AccumulateNV2",
 		Input: []tf.Input{
-			bytes,
+			tf.OutputList(inputs),
 		},
 		Attrs: attrs,
 	}
@@ -15512,99 +15468,95 @@ func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...
 	return op.Output(0)
 }
 
-// Copy a tensor setting everything outside a central band in each innermost matrix
+// Convert the quantized 'input' tensor into a lower-precision 'output', using the
 //
-// to zero.
+// actual distribution of the values to maximize the usage of the lower bit depth
+// and adjusting the output min and max ranges accordingly.
 //
-// The `band` part is computed as follows:
-// Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
-// tensor with the same shape where
+// [input_min, input_max] are scalar floats that specify the range for the float
+// interpretation of the 'input' data. For example, if input_min is -1.0f and
+// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
 //
-// `band[i, j, k, ..., m, n] = in_band(m, n) * input[i, j, k, ..., m, n]`.
+// This operator tries to squeeze as much precision as possible into an output with
+// a lower bit depth by calculating the actual min and max values found in the
+// data. For example, maybe that quint16 input has no values lower than 16,384 and
+// none higher than 49,152. That means only half the range is actually needed, all
+// the float interpretations are between -0.5f and 0.5f, so if we want to compress
+// the data into a quint8 output, we can use that range rather than the theoretical
+// -1.0f to 1.0f that is suggested by the input min and max.
 //
-// The indicator function
+// In practice, this is most useful for taking output from operations like
+// QuantizedMatMul that can produce higher bit-depth outputs than their inputs and
+// may have large potential output ranges, but in practice have a distribution of
+// input values that only uses a small fraction of the possible range. By feeding
+// that output into this operator, we can reduce it from 32 bits down to 8 with
+// minimal loss of accuracy.
 //
-// `in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) &&
-//                  (num_upper < 0 || (n-m) <= num_upper)`.
+// Arguments:
 //
-// For example:
-//
-// ```
-// # if 'input' is [[ 0,  1,  2, 3]
-//                  [-1,  0,  1, 2]
-//                  [-2, -1,  0, 1]
-//                  [-3, -2, -1, 0]],
-//
-// tf.matrix_band_part(input, 1, -1) ==> [[ 0,  1,  2, 3]
-//                                        [-1,  0,  1, 2]
-//                                        [ 0, -1,  0, 1]
-//                                        [ 0,  0, -1, 0]],
-//
-// tf.matrix_band_part(input, 2, 1) ==> [[ 0,  1,  0, 0]
-//                                       [-1,  0,  1, 0]
-//                                       [-2, -1,  0, 1]
-//                                       [ 0, -2, -1, 0]]
-// ```
-//
-// Useful special cases:
-//
-// ```
-//  tf.matrix_band_part(input, 0, -1) ==> Upper triangular part.
-//  tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.
-//  tf.matrix_band_part(input, 0, 0) ==> Diagonal.
-// ```
-//
-// Arguments:
-//	input: Rank `k` tensor.
-//	num_lower: 0-D tensor. Number of subdiagonals to keep. If negative, keep entire
-// lower triangle.
-//	num_upper: 0-D tensor. Number of superdiagonals to keep. If negative, keep
-// entire upper triangle.
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//	out_type: The type of the output. Should be a lower bit depth than Tinput.
 //
-// Returns Rank `k` tensor of the same shape as input. The extracted banded tensor.
-func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_upper tf.Output) (band tf.Output) {
+// Returns The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
+func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "MatrixBandPart",
+		Type: "QuantizeDownAndShrinkRange",
 		Input: []tf.Input{
-			input, num_lower, num_upper,
+			input, input_min, input_max,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DecodeCompressedAttr is an optional argument to DecodeCompressed.
-type DecodeCompressedAttr func(optionalAttr)
+// RandomGammaAttr is an optional argument to RandomGamma.
+type RandomGammaAttr func(optionalAttr)
 
-// DecodeCompressedCompressionType sets the optional compression_type attribute to value.
+// RandomGammaSeed sets the optional seed attribute to value.
 //
-// value: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-// If not specified, defaults to ""
-func DecodeCompressedCompressionType(value string) DecodeCompressedAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomGammaSeed(value int64) RandomGammaAttr {
 	return func(m optionalAttr) {
-		m["compression_type"] = value
+		m["seed"] = value
 	}
 }
 
-// Decompress strings.
+// RandomGammaSeed2 sets the optional seed2 attribute to value.
 //
-// This op decompresses each element of the `bytes` input `Tensor`, which
-// is assumed to be compressed using the given `compression_type`.
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomGammaSeed2(value int64) RandomGammaAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from the Gamma distribution(s) described by alpha.
 //
-// The `output` is a string `Tensor` of the same shape as `bytes`,
-// each element containing the decompressed data from the corresponding
-// element in `bytes`.
+// This op uses the algorithm by Marsaglia et al. to acquire samples via
+// transformation-rejection from pairs of uniform and normal random variables.
+// See http://dl.acm.org/citation.cfm?id=358414
 //
 // Arguments:
-//	bytes: A Tensor of string which is compressed.
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in alpha.
+//	alpha: A tensor in which each scalar is a "shape" parameter describing the
+// associated gamma distribution.
 //
-// Returns A Tensor with the same shape as input `bytes`, uncompressed
-// from bytes.
-func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompressedAttr) (output tf.Output) {
+// Returns A tensor with shape `shape + shape(alpha)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
+func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15613,9 +15565,9 @@ func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompresse
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeCompressed",
+		Type: "RandomGamma",
 		Input: []tf.Input{
-			bytes,
+			shape, alpha,
 		},
 		Attrs: attrs,
 	}
@@ -15623,38 +15575,48 @@ func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompresse
 	return op.Output(0)
 }
 
-// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
-type WholeFileReaderV2Attr func(optionalAttr)
+// RandomUniformIntAttr is an optional argument to RandomUniformInt.
+type RandomUniformIntAttr func(optionalAttr)
 
-// WholeFileReaderV2Container sets the optional container attribute to value.
+// RandomUniformIntSeed sets the optional seed attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["seed"] = value
 	}
 }
 
-// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
+// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["seed2"] = value
 	}
 }
 
-// A Reader that outputs the entire contents of a file as a value.
+// Outputs random integers from a uniform distribution.
 //
-// To use, enqueue filenames in a Queue.  The output of ReaderRead will
-// be a filename (key) and the contents of that file (value).
+// The generated values are uniform integers in the range `[minval, maxval)`.
+// The lower bound `minval` is included in the range, while the upper bound
+// `maxval` is excluded.
 //
-// Returns The handle to reference the Reader.
-func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
+// The random integers are slightly biased unless `maxval - minval` is an exact
+// power of two.  The bias is small for values of `maxval - minval` significantly
+// smaller than the range of the output (either `2^32` or `2^64`).
+//
+// Arguments:
+//	shape: The shape of the output tensor.
+//	minval: 0-D.  Inclusive lower bound on the generated integers.
+//	maxval: 0-D.  Exclusive upper bound on the generated integers.
+//
+// Returns A tensor of the specified shape filled with uniform random integers.
+func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15663,165 +15625,280 @@ func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "WholeFileReaderV2",
-
+		Type: "RandomUniformInt",
+		Input: []tf.Input{
+			shape, minval, maxval,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Transforms a tf.Example proto (as a string) into typed tensors.
+// SkipgramAttr is an optional argument to Skipgram.
+type SkipgramAttr func(optionalAttr)
+
+// SkipgramWindowSize sets the optional window_size attribute to value.
+//
+// value: The number of words to predict to the left and right of the target.
+// If not specified, defaults to 5
+func SkipgramWindowSize(value int64) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["window_size"] = value
+	}
+}
+
+// SkipgramMinCount sets the optional min_count attribute to value.
+//
+// value: The minimum number of word occurrences for it to be included in the
+// vocabulary.
+// If not specified, defaults to 5
+func SkipgramMinCount(value int64) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["min_count"] = value
+	}
+}
+
+// SkipgramSubsample sets the optional subsample attribute to value.
+//
+// value: Threshold for word occurrence. Words that appear with higher
+// frequency will be randomly down-sampled. Set to 0 to disable.
+// If not specified, defaults to 0.001
+func SkipgramSubsample(value float32) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["subsample"] = value
+	}
+}
+
+// Parses a text file and creates a batch of examples.
+//
+// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
 //
 // Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	dense_defaults: A list of Tensors (some may be empty), whose length matches
-// the length of `dense_keys`. dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	num_sparse: The number of sparse features to be parsed from the example. This
-// must match the lengths of `sparse_keys` and `sparse_types`.
-//	sparse_keys: A list of `num_sparse` strings.
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: The keys expected in the Examples' features associated with dense
-// values.
-//	sparse_types: A list of `num_sparse` types; the data types of data in each
-// Feature given in sparse_keys.
-// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: The shapes of data in each Feature given in dense_keys.
-// The length of this list must match the length of `dense_keys`.  The
-// number of elements in the Feature corresponding to dense_key[j] must
-// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
-// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
-// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
-// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
-// D1, .., DN), where M is the number of blocks of elements of length
-// D1 * .... * DN, in the input.
-func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+//	filename: The corpus's text file name.
+//	batch_size: The size of produced batch.
+//
+// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
+func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ParseSingleExample",
-		Input: []tf.Input{
-			serialized, tf.OutputList(dense_defaults),
-		},
+		Type: "Skipgram",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
+}
+
+// StringToNumberAttr is an optional argument to StringToNumber.
+type StringToNumberAttr func(optionalAttr)
+
+// StringToNumberOutType sets the optional out_type attribute to value.
+//
+// value: The numeric type to interpret each string in `string_tensor` as.
+// If not specified, defaults to DT_FLOAT
+func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
 	}
-	var idx int
-	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
+}
+
+// Converts each string in the input Tensor to the specified numeric type.
+//
+// (Note that int32 overflow results in an error while float overflow
+// results in a rounded value.)
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
+	if scope.Err() != nil {
 		return
 	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "StringToNumber",
+		Input: []tf.Input{
+			string_tensor,
+		},
+		Attrs: attrs,
 	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
+type ResourceApplyFtrlV2Attr func(optionalAttr)
+
+// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
 	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
 }
 
-// Computes acos of x element-wise.
-func Acos(scope *Scope, x tf.Output) (y tf.Output) {
+// Update '*var' according to the Ftrl-proximal scheme.
+//
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
+//
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Acos",
+		Type: "ResourceApplyFtrlV2",
 		Input: []tf.Input{
-			x,
+			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
-type MaxPoolWithArgmaxAttr func(optionalAttr)
+// TruncatedNormalAttr is an optional argument to TruncatedNormal.
+type TruncatedNormalAttr func(optionalAttr)
 
-// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
-// If not specified, defaults to DT_INT64
-func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
+// TruncatedNormalSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
 	return func(m optionalAttr) {
-		m["Targmax"] = value
+		m["seed"] = value
 	}
 }
 
-// Performs max pooling on the input and outputs both max values and indices.
+// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
 //
-// The indices in `argmax` are flattened, so that a maximum value at position
-// `[b, y, x, c]` becomes flattened index
-// `((b * height + y) * width + x) * channels + c`.
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from a truncated normal distribution.
 //
-// The indices returned are always in `[0, height) x [0, width)` before flattening,
-// even if padding is involved and the mathematically correct answer is outside
-// (either negative or too large).  This is a bug, but fixing it is difficult to do
-// in a safe backwards compatible way, especially due to flattening.
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
-func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
+// Returns A tensor of the specified shape filled with random truncated normal
+// values.
+func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolWithArgmax",
+		Type: "TruncatedNormal",
 		Input: []tf.Input{
-			input,
+			shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
+// RandomShuffleAttr is an optional argument to RandomShuffle.
+type RandomShuffleAttr func(optionalAttr)
+
+// RandomShuffleSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomShuffleSeed(value int64) RandomShuffleAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomShuffleSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomShuffleSeed2(value int64) RandomShuffleAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Randomly shuffles a tensor along its first dimension.
+//
+//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
+//   to one and only one `output[i]`. For example, a mapping that might occur for a
+//   3x2 tensor is:
+//
+// ```
+// [[1, 2],       [[5, 6],
+//  [3, 4],  ==>   [1, 2],
+//  [5, 6]]        [3, 4]]
+// ```
 //
 // Arguments:
-//	serialized: A scalar string containing a serialized TensorProto proto.
-//	out_type: The type of the serialized tensor.  The provided type must match the
-// type of the serialized tensor and no implicit conversion will take place.
+//	value: The tensor to be shuffled.
 //
-// Returns A Tensor of type `out_type`.
-func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
+// Returns A tensor of same shape and type as `value`, shuffled along its first
+// dimension.
+func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ParseTensor",
+		Type: "RandomShuffle",
 		Input: []tf.Input{
-			serialized,
+			value,
 		},
 		Attrs: attrs,
 	}
@@ -15829,49 +15906,47 @@ func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (outp
 	return op.Output(0)
 }
 
-// MapClearAttr is an optional argument to MapClear.
-type MapClearAttr func(optionalAttr)
+// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
+type OrderedMapIncompleteSizeAttr func(optionalAttr)
 
-// MapClearCapacity sets the optional capacity attribute to value.
+// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapClearCapacity(value int64) MapClearAttr {
+func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// MapClearMemoryLimit sets the optional memory_limit attribute to value.
+// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapClearMemoryLimit(value int64) MapClearAttr {
+func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// MapClearContainer sets the optional container attribute to value.
+// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func MapClearContainer(value string) MapClearAttr {
+func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// MapClearSharedName sets the optional shared_name attribute to value.
+// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func MapClearSharedName(value string) MapClearAttr {
+func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op removes all elements in the underlying container.
-//
-// Returns the created operation.
-func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
+// Op returns the number of incomplete elements in the underlying container.
+func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15880,167 +15955,161 @@ func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapClear",
+		Type: "OrderedMapIncompleteSize",
 
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DecodeCSVAttr is an optional argument to DecodeCSV.
-type DecodeCSVAttr func(optionalAttr)
-
-// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
-//
-// value: char delimiter to separate fields in a record.
-// If not specified, defaults to ","
-func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["field_delim"] = value
-	}
-}
+// DecodeRawAttr is an optional argument to DecodeRaw.
+type DecodeRawAttr func(optionalAttr)
 
-// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
+// DecodeRawLittleEndian sets the optional little_endian attribute to value.
 //
-// value: If false, treats double quotation marks as regular
-// characters inside of the string fields (ignoring RFC 4180, Section 2,
-// Bullet 5).
+// value: Whether the input `bytes` are in little-endian order.
+// Ignored for `out_type` values that are stored in a single byte like
+// `uint8`.
 // If not specified, defaults to true
-func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
+func DecodeRawLittleEndian(value bool) DecodeRawAttr {
 	return func(m optionalAttr) {
-		m["use_quote_delim"] = value
+		m["little_endian"] = value
 	}
 }
 
-// DecodeCSVNaValue sets the optional na_value attribute to value.
-//
-// value: Additional string to recognize as NA/NaN.
-// If not specified, defaults to ""
-func DecodeCSVNaValue(value string) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["na_value"] = value
-	}
-}
-
-// Convert CSV records to tensors. Each column maps to one tensor.
-//
-// RFC 4180 format is expected for the CSV records.
-// (https://tools.ietf.org/html/rfc4180)
-// Note that we allow leading and trailing spaces with int or float field.
+// Reinterpret the bytes of a string as a vector of numbers.
 //
 // Arguments:
-//	records: Each string is a record/row in the csv and all records should have
-// the same format.
-//	record_defaults: One tensor per column of the input record, with either a
-// scalar default value for that column or empty if the column is required.
+//	bytes: All the elements must have the same length.
 //
-// Returns Each tensor will have the same shape as records.
-func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
+//
+// Returns A Tensor with one more dimension than the input `bytes`.  The
+// added dimension will have size equal to the length of the elements
+// of `bytes` divided by the number of bytes to represent `out_type`.
+func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...DecodeRawAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"out_type": out_type}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeCSV",
+		Type: "DecodeRaw",
 		Input: []tf.Input{
-			records, tf.OutputList(record_defaults),
+			bytes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("DecodeCSV", err)
-		return
-	}
-	return output
+	return op.Output(0)
 }
 
-// Returns the rank of a tensor.
+// Copy a tensor setting everything outside a central band in each innermost matrix
 //
-// This operation returns an integer representing the rank of `input`.
+// to zero.
+//
+// The `band` part is computed as follows:
+// Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
+// tensor with the same shape where
+//
+// `band[i, j, k, ..., m, n] = in_band(m, n) * input[i, j, k, ..., m, n]`.
+//
+// The indicator function
+//
+// `in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) &&
+//                  (num_upper < 0 || (n-m) <= num_upper)`.
 //
 // For example:
 //
 // ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// # shape of tensor 't' is [2, 2, 3]
-// rank(t) ==> 3
+// # if 'input' is [[ 0,  1,  2, 3]
+//                  [-1,  0,  1, 2]
+//                  [-2, -1,  0, 1]
+//                  [-3, -2, -1, 0]],
+//
+// tf.matrix_band_part(input, 1, -1) ==> [[ 0,  1,  2, 3]
+//                                        [-1,  0,  1, 2]
+//                                        [ 0, -1,  0, 1]
+//                                        [ 0,  0, -1, 0]],
+//
+// tf.matrix_band_part(input, 2, 1) ==> [[ 0,  1,  0, 0]
+//                                       [-1,  0,  1, 0]
+//                                       [-2, -1,  0, 1]
+//                                       [ 0, -2, -1, 0]]
 // ```
 //
-// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
-// of a tensor is the number of indices required to uniquely select each element
-// of the tensor. Rank is also known as "order", "degree", or "ndims."
-func Rank(scope *Scope, input tf.Output) (output tf.Output) {
+// Useful special cases:
+//
+// ```
+//  tf.matrix_band_part(input, 0, -1) ==> Upper triangular part.
+//  tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.
+//  tf.matrix_band_part(input, 0, 0) ==> Diagonal.
+// ```
+//
+// Arguments:
+//	input: Rank `k` tensor.
+//	num_lower: 0-D tensor. Number of subdiagonals to keep. If negative, keep entire
+// lower triangle.
+//	num_upper: 0-D tensor. Number of superdiagonals to keep. If negative, keep
+// entire upper triangle.
+//
+// Returns Rank `k` tensor of the same shape as input. The extracted banded tensor.
+func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_upper tf.Output) (band tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Rank",
+		Type: "MatrixBandPart",
 		Input: []tf.Input{
-			input,
+			input, num_lower, num_upper,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Output a fact about factorials.
-func Fact(scope *Scope) (fact tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Fact",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
+// DecodeCompressedAttr is an optional argument to DecodeCompressed.
+type DecodeCompressedAttr func(optionalAttr)
 
-// Makes its input available to the next iteration.
-//
-// Arguments:
-//	data: The tensor to be made available to the next iteration.
+// DecodeCompressedCompressionType sets the optional compression_type attribute to value.
 //
-// Returns The same tensor as `data`.
-func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "NextIteration",
-		Input: []tf.Input{
-			data,
-		},
+// value: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+// If not specified, defaults to ""
+func DecodeCompressedCompressionType(value string) DecodeCompressedAttr {
+	return func(m optionalAttr) {
+		m["compression_type"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Creates a dataset that skips `count` elements from the `input_dataset`.
+// Decompress strings.
 //
-// Arguments:
+// This op decompresses each element of the `bytes` input `Tensor`, which
+// is assumed to be compressed using the given `compression_type`.
 //
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
+// The `output` is a string `Tensor` of the same shape as `bytes`,
+// each element containing the decompressed data from the corresponding
+// element in `bytes`.
 //
+// Arguments:
+//	bytes: A Tensor of string which is compressed.
 //
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns A Tensor with the same shape as input `bytes`, uncompressed
+// from bytes.
+func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompressedAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SkipDataset",
+		Type: "DecodeCompressed",
 		Input: []tf.Input{
-			input_dataset, count,
+			bytes,
 		},
 		Attrs: attrs,
 	}
@@ -16048,13 +16117,13 @@ func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_
 	return op.Output(0)
 }
 
-// Computes hyperbolic tangent of `x` element-wise.
-func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes acos of x element-wise.
+func Acos(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Tanh",
+		Type: "Acos",
 		Input: []tf.Input{
 			x,
 		},
@@ -16063,71 +16132,37 @@ func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes the maximum along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the max is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentMax",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
-type AvgPoolGradAttr func(optionalAttr)
+// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
+type MaxPoolWithArgmaxAttr func(optionalAttr)
 
-// AvgPoolGradDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
+// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
+// If not specified, defaults to DT_INT64
+func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["Targmax"] = value
 	}
 }
 
-// Computes gradients of the average pooling function.
+// Performs max pooling on the input and outputs both max values and indices.
+//
+// The indices in `argmax` are flattened, so that a maximum value at position
+// `[b, y, x, c]` becomes flattened index
+// `((b * height + y) * width + x) * channels + c`.
+//
+// The indices returned are always in `[0, height) x [0, width)` before flattening,
+// even if padding is involved and the mathematically correct answer is outside
+// (either negative or too large).  This is a bug, but fixing it is difficult to do
+// in a safe backwards compatible way, especially due to flattening.
 //
 // Arguments:
-//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
-// the output of `avg_pool`.
-//	ksize: The size of the sliding window for each dimension of the input.
-//	strides: The stride of the sliding window for each dimension of the input.
+//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
 //	padding: The type of padding algorithm to use.
 //
-// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
-func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
+func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16136,50 +16171,74 @@ func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AvgPoolGrad",
+		Type: "MaxPoolWithArgmax",
 		Input: []tf.Input{
-			orig_input_shape, grad,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
-
-// StageClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
+//
+// Arguments:
+//	serialized: A scalar string containing a serialized TensorProto proto.
+//	out_type: The type of the serialized tensor.  The provided type must match the
+// type of the serialized tensor and no implicit conversion will take place.
+//
+// Returns A Tensor of type `out_type`.
+func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"out_type": out_type}
+	opspec := tf.OpSpec{
+		Type: "ParseTensor",
+		Input: []tf.Input{
+			serialized,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MapClearAttr is an optional argument to MapClear.
+type MapClearAttr func(optionalAttr)
+
+// MapClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
+func MapClearCapacity(value int64) MapClearAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// MapClearMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
+func MapClearMemoryLimit(value int64) MapClearAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// StageClearContainer sets the optional container attribute to value.
+// MapClearContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
+func MapClearContainer(value string) MapClearAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// StageClearSharedName sets the optional shared_name attribute to value.
+// MapClearSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
+func MapClearSharedName(value string) MapClearAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
@@ -16188,7 +16247,7 @@ func StageClearSharedName(value string) StageClearAttr {
 // Op removes all elements in the underlying container.
 //
 // Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16197,135 +16256,167 @@ func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StageClear",
+		Type: "MapClear",
 
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
-type ComputeAccidentalHitsAttr func(optionalAttr)
+// DecodeCSVAttr is an optional argument to DecodeCSV.
+type DecodeCSVAttr func(optionalAttr)
 
-// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
+// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
+// value: char delimiter to separate fields in a record.
+// If not specified, defaults to ","
+func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["field_delim"] = value
 	}
 }
 
-// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
+// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+// value: If false, treats double quotation marks as regular
+// characters inside of the string fields (ignoring RFC 4180, Section 2,
+// Bullet 5).
+// If not specified, defaults to true
+func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["use_quote_delim"] = value
 	}
 }
 
-// Computes the ids of the positions in sampled_candidates that match true_labels.
+// DecodeCSVNaValue sets the optional na_value attribute to value.
 //
-// When doing log-odds NCE, the result of this op should be passed through a
-// SparseToDense op, then added to the logits of the sampled candidates. This has
-// the effect of 'removing' the sampled labels that match the true labels by
-// making the classifier sure that they are sampled labels.
+// value: Additional string to recognize as NA/NaN.
+// If not specified, defaults to ""
+func DecodeCSVNaValue(value string) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["na_value"] = value
+	}
+}
+
+// Convert CSV records to tensors. Each column maps to one tensor.
+//
+// RFC 4180 format is expected for the CSV records.
+// (https://tools.ietf.org/html/rfc4180)
+// Note that we allow leading and trailing spaces with int or float field.
 //
 // Arguments:
-//	true_classes: The true_classes output of UnpackSparseLabels.
-//	sampled_candidates: The sampled_candidates output of CandidateSampler.
-//	num_true: Number of true labels per context.
+//	records: Each string is a record/row in the csv and all records should have
+// the same format.
+//	record_defaults: One tensor per column of the input record, with either a
+// scalar default value for that column or empty if the column is required.
 //
-// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
-// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
-// is -FLOAT_MAX.
-func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+// Returns Each tensor will have the same shape as records.
+func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ComputeAccidentalHits",
+		Type: "DecodeCSV",
 		Input: []tf.Input{
-			true_classes, sampled_candidates,
+			records, tf.OutputList(record_defaults),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("DecodeCSV", err)
+		return
+	}
+	return output
 }
 
-// Computes sigmoid of `x` element-wise.
+// Returns the rank of a tensor.
 //
-// Specifically, `y = 1 / (1 + exp(-x))`.
-func Sigmoid(scope *Scope, x tf.Output) (y tf.Output) {
+// This operation returns an integer representing the rank of `input`.
+//
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// # shape of tensor 't' is [2, 2, 3]
+// rank(t) ==> 3
+// ```
+//
+// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
+// of a tensor is the number of indices required to uniquely select each element
+// of the tensor. Rank is also known as "order", "degree", or "ndims."
+func Rank(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Sigmoid",
+		Type: "Rank",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RandomStandardNormalAttr is an optional argument to RandomStandardNormal.
-type RandomStandardNormalAttr func(optionalAttr)
-
-// RandomStandardNormalSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomStandardNormalSeed(value int64) RandomStandardNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// Output a fact about factorials.
+func Fact(scope *Scope) (fact tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Fact",
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// RandomStandardNormalSeed2 sets the optional seed2 attribute to value.
+// Makes its input available to the next iteration.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomStandardNormalSeed2(value int64) RandomStandardNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+// Arguments:
+//	data: The tensor to be made available to the next iteration.
+//
+// Returns The same tensor as `data`.
+func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NextIteration",
+		Input: []tf.Input{
+			data,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Outputs random values from a normal distribution.
-//
-// The generated values will have mean 0 and standard deviation 1.
+// Creates a dataset that skips `count` elements from the `input_dataset`.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
 //
-// Returns A tensor of the specified shape filled with random normal values.
-func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomStandardNormalAttr) (output tf.Output) {
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
+//
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "RandomStandardNormal",
+		Type: "SkipDataset",
 		Input: []tf.Input{
-			shape,
+			input_dataset, count,
 		},
 		Attrs: attrs,
 	}
@@ -16333,343 +16424,320 @@ func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, opti
 	return op.Output(0)
 }
 
-// FusedBatchNormAttr is an optional argument to FusedBatchNorm.
-type FusedBatchNormAttr func(optionalAttr)
-
-// FusedBatchNormEpsilon sets the optional epsilon attribute to value.
-//
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormEpsilon(value float32) FusedBatchNormAttr {
-	return func(m optionalAttr) {
-		m["epsilon"] = value
+// Computes hyperbolic tangent of `x` element-wise.
+func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// FusedBatchNormDataFormat sets the optional data_format attribute to value.
-//
-// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormDataFormat(value string) FusedBatchNormAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
+	opspec := tf.OpSpec{
+		Type: "Tanh",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FusedBatchNormIsTraining sets the optional is_training attribute to value.
+// Computes the maximum along segments of a tensor.
 //
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormIsTraining(value bool) FusedBatchNormAttr {
-	return func(m optionalAttr) {
-		m["is_training"] = value
-	}
-}
-
-// Batch normalization.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// Computes a tensor such that
+// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the max is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
+// </div>
 //
 // Arguments:
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	offset: A 1D Tensor for offset, to shift to the normalized x.
-//	mean: A 1D Tensor for population mean. Used for inference only;
-// must be empty for training.
-//	variance: A 1D Tensor for population variance. Used for inference only;
-// must be empty for training.
 //
-// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
-// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
-// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
-// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
-// in the cuDNN case), to be reused in the gradient computation.
-func FusedBatchNorm(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormAttr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNorm",
+		Type: "SegmentMax",
 		Input: []tf.Input{
-			x, scale, offset, mean, variance,
+			data, segment_ids,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0)
 }
 
-// Computes tan of x element-wise.
-func Tan(scope *Scope, x tf.Output) (y tf.Output) {
+// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
+type AvgPoolGradAttr func(optionalAttr)
+
+// AvgPoolGradDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes gradients of the average pooling function.
+//
+// Arguments:
+//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
+// the output of `avg_pool`.
+//	ksize: The size of the sliding window for each dimension of the input.
+//	strides: The stride of the sliding window for each dimension of the input.
+//	padding: The type of padding algorithm to use.
+//
+// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
+func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Tan",
+		Type: "AvgPoolGrad",
 		Input: []tf.Input{
-			x,
+			orig_input_shape, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FusedBatchNormV2Attr is an optional argument to FusedBatchNormV2.
-type FusedBatchNormV2Attr func(optionalAttr)
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
 
-// FusedBatchNormV2Epsilon sets the optional epsilon attribute to value.
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormV2Epsilon(value float32) FusedBatchNormV2Attr {
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
 	return func(m optionalAttr) {
-		m["epsilon"] = value
+		m["capacity"] = value
 	}
 }
 
-// FusedBatchNormV2DataFormat sets the optional data_format attribute to value.
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormV2DataFormat(value string) FusedBatchNormV2Attr {
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["memory_limit"] = value
 	}
 }
 
-// FusedBatchNormV2IsTraining sets the optional is_training attribute to value.
-//
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormV2IsTraining(value bool) FusedBatchNormV2Attr {
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
 	return func(m optionalAttr) {
-		m["is_training"] = value
+		m["container"] = value
 	}
 }
 
-// Batch normalization.
-//
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
-//
-// Arguments:
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	offset: A 1D Tensor for offset, to shift to the normalized x.
-//	mean: A 1D Tensor for population mean. Used for inference only;
-// must be empty for training.
-//	variance: A 1D Tensor for population variance. Used for inference only;
-// must be empty for training.
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
 //
-// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
-// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
-// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
-// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
-// in the cuDNN case), to be reused in the gradient computation.
-func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormV2Attr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNormV2",
-		Input: []tf.Input{
-			x, scale, offset, mean, variance,
-		},
+		Type: "StageClear",
+
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return scope.AddOperation(opspec)
 }
 
-// MultinomialAttr is an optional argument to Multinomial.
-type MultinomialAttr func(optionalAttr)
+// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
+type ComputeAccidentalHitsAttr func(optionalAttr)
 
-// MultinomialSeed sets the optional seed attribute to value.
+// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
 //
-// value: If either seed or seed2 is set to be non-zero, the internal random number
-// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
 // If not specified, defaults to 0
-func MultinomialSeed(value int64) MultinomialAttr {
+func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// MultinomialSeed2 sets the optional seed2 attribute to value.
+// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
 //
-// value: A second seed to avoid seed collision.
+// value: An second seed to avoid seed collision.
 // If not specified, defaults to 0
-func MultinomialSeed2(value int64) MultinomialAttr {
+func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// MultinomialOutputDtype sets the optional output_dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["output_dtype"] = value
-	}
-}
-
-// Draws samples from a multinomial distribution.
+// Computes the ids of the positions in sampled_candidates that match true_labels.
+//
+// When doing log-odds NCE, the result of this op should be passed through a
+// SparseToDense op, then added to the logits of the sampled candidates. This has
+// the effect of 'removing' the sampled labels that match the true labels by
+// making the classifier sure that they are sampled labels.
 //
 // Arguments:
-//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
-// represents the unnormalized log probabilities for all classes.
-//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
+//	true_classes: The true_classes output of UnpackSparseLabels.
+//	sampled_candidates: The sampled_candidates output of CandidateSampler.
+//	num_true: Number of true labels per context.
 //
-// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
-// contains the drawn class labels with range `[0, num_classes)`.
-func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
+// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
+// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
+// is -FLOAT_MAX.
+func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_true": num_true}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Multinomial",
+		Type: "ComputeAccidentalHits",
 		Input: []tf.Input{
-			logits, num_samples,
+			true_classes, sampled_candidates,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// EncodeJpegAttr is an optional argument to EncodeJpeg.
-type EncodeJpegAttr func(optionalAttr)
-
-// EncodeJpegFormat sets the optional format attribute to value.
+// Looks up keys in a table, outputs the corresponding values.
 //
-// value: Per pixel image format.
-// If not specified, defaults to ""
-func EncodeJpegFormat(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["format"] = value
-	}
-}
-
-// EncodeJpegQuality sets the optional quality attribute to value.
+// The tensor `keys` must of the same type as the keys of the table.
+// The output `values` is of the type of the table values.
 //
-// value: Quality of the compression from 0 to 100 (higher is better and slower).
-// If not specified, defaults to 95
-func EncodeJpegQuality(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["quality"] = value
+// The scalar `default_value` is the value output for keys not present in the
+// table. It must also be of the same type as the table values.
+//
+// Arguments:
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//
+//
+// Returns Same shape as `keys`.  Values found in the table, or `default_values`
+// for missing keys.
+func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LookupTableFindV2",
+		Input: []tf.Input{
+			table_handle, keys, default_value,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegProgressive sets the optional progressive attribute to value.
+// Bucketizes 'input' based on 'boundaries'.
 //
-// value: If True, create a JPEG that loads progressively (coarse to fine).
-// If not specified, defaults to false
-func EncodeJpegProgressive(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["progressive"] = value
-	}
-}
-
-// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
+// For example, if the inputs are
+//     boundaries = [0, 10, 100]
+//     input = [[-5, 10000]
+//              [150,   10]
+//              [5,    100]]
 //
-// value: If True, spend CPU/RAM to reduce size with no quality change.
-// If not specified, defaults to false
-func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["optimize_size"] = value
-	}
-}
-
-// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
+// then the output will be
+//     output = [[0, 3]
+//               [3, 2]
+//               [1, 3]]
 //
-// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
-// If not specified, defaults to true
-func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["chroma_downsampling"] = value
-	}
-}
-
-// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+// Arguments:
+//	input: Any shape of Tensor contains with int or float type.
+//	boundaries: A sorted list of floats gives the boundary of the buckets.
 //
-// value: Unit used to specify `x_density` and `y_density`:
-// pixels per inch (`'in'`) or centimeter (`'cm'`).
-// If not specified, defaults to "in"
-func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["density_unit"] = value
-	}
-}
-
-// EncodeJpegXDensity sets the optional x_density attribute to value.
+// Returns Same shape with 'input', each value of input replaced with bucket index.
 //
-// value: Horizontal pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegXDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["x_density"] = value
+// @compatibility(numpy)
+// Equivalent to np.digitize.
+// @end_compatibility
+func Bucketize(scope *Scope, input tf.Output, boundaries []float32) (output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// EncodeJpegYDensity sets the optional y_density attribute to value.
-//
-// value: Vertical pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegYDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["y_density"] = value
+	attrs := map[string]interface{}{"boundaries": boundaries}
+	opspec := tf.OpSpec{
+		Type: "Bucketize",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
+// EncodePngAttr is an optional argument to EncodePng.
+type EncodePngAttr func(optionalAttr)
+
+// EncodePngCompression sets the optional compression attribute to value.
 //
-// value: If not empty, embed this XMP metadata in the image header.
-// If not specified, defaults to ""
-func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
+// value: Compression level.
+// If not specified, defaults to -1
+func EncodePngCompression(value int64) EncodePngAttr {
 	return func(m optionalAttr) {
-		m["xmp_metadata"] = value
+		m["compression"] = value
 	}
 }
 
-// JPEG-encode an image.
-//
-// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
-//
-// The attr `format` can be used to override the color format of the encoded
-// output.  Values can be:
+// PNG-encode an image.
 //
-// *   `''`: Use a default format based on the number of channels in the image.
-// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
-//     of `image` must be 1.
-// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
-//     of `image` must be 3.
+// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
+// where `channels` is:
 //
-// If `format` is not specified or is the empty string, a default format is picked
-// in function of the number of channels in `image`:
+// *   1: for grayscale.
+// *   2: for grayscale + alpha.
+// *   3: for RGB.
+// *   4: for RGBA.
 //
-// *   1: Output a grayscale image.
-// *   3: Output an RGB image.
+// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
+// default or a value from 0 to 9.  9 is the highest compression level, generating
+// the smallest output, but is slower.
 //
 // Arguments:
 //	image: 3-D with shape `[height, width, channels]`.
 //
-// Returns 0-D. JPEG-encoded image.
-func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
+// Returns 0-D. PNG-encoded image.
+func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16678,7 +16746,7 @@ func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (cont
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeJpeg",
+		Type: "EncodePng",
 		Input: []tf.Input{
 			image,
 		},
@@ -16688,164 +16756,91 @@ func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (cont
 	return op.Output(0)
 }
 
-// MaxPoolGradAttr is an optional argument to MaxPoolGrad.
-type MaxPoolGradAttr func(optionalAttr)
-
-// MaxPoolGradDataFormat sets the optional data_format attribute to value.
+// Updates the table to associates keys with values.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradDataFormat(value string) MaxPoolGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Computes gradients of the maxpooling function.
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
 //
-// Returns Gradients w.r.t. the input to `max_pool`.
-func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradAttr) (output tf.Output) {
+// Returns the created operation.
+func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGrad",
+		Type: "LookupTableInsertV2",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			table_handle, keys, values,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// CropAndResizeAttr is an optional argument to CropAndResize.
-type CropAndResizeAttr func(optionalAttr)
-
-// CropAndResizeMethod sets the optional method attribute to value.
-//
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
-// If not specified, defaults to "bilinear"
-func CropAndResizeMethod(value string) CropAndResizeAttr {
-	return func(m optionalAttr) {
-		m["method"] = value
+// Returns element-wise smallest integer in not less than x.
+func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// CropAndResizeExtrapolationValue sets the optional extrapolation_value attribute to value.
-//
-// value: Value used for extrapolation, when applicable.
-// If not specified, defaults to 0
-func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
-	return func(m optionalAttr) {
-		m["extrapolation_value"] = value
+	opspec := tf.OpSpec{
+		Type: "Ceil",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Extracts crops from the input image tensor and bilinearly resizes them (possibly
-//
-// with aspect ratio change) to a common output size specified by `crop_size`. This
-// is more general than the `crop_to_bounding_box` op which extracts a fixed size
-// slice from the input image and does not allow resizing or aspect ratio change.
-//
-// Returns a tensor with `crops` from the input `image` at positions defined at the
-// bounding box locations in `boxes`. The cropped boxes are all resized (with
-// bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
-// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
-// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
-// method will give identical results to using `tf.image.resize_bilinear()`
-// with `align_corners=True`.
+// Computes the number of elements in the given table.
 //
 // Arguments:
-//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-// Both `image_height` and `image_width` need to be positive.
-//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
-// specifies the coordinates of a box in the `box_ind[i]` image and is specified
-// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
-// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
-// `[0, 1]` interval of normalized image height is mapped to
-// `[0, image_height - 1]` in image height coordinates. We do allow `y1` > `y2`, in
-// which case the sampled crop is an up-down flipped version of the original
-// image. The width dimension is treated similarly. Normalized coordinates
-// outside the `[0, 1]` range are allowed, in which case we use
-// `extrapolation_value` to extrapolate the input image values.
-//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
-// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
-//	crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`. All
-// cropped image patches are resized to this size. The aspect ratio of the image
-// content is not preserved. Both `crop_height` and `crop_width` need to be
-// positive.
+//	table_handle: Handle to the table.
 //
-// Returns A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
-func CropAndResize(scope *Scope, image tf.Output, boxes tf.Output, box_ind tf.Output, crop_size tf.Output, optional ...CropAndResizeAttr) (crops tf.Output) {
+// Returns Scalar that contains number of elements in the table.
+func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "CropAndResize",
+		Type: "LookupTableSizeV2",
 		Input: []tf.Input{
-			image, boxes, box_ind, crop_size,
+			table_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
-type ResourceApplyPowerSignAttr func(optionalAttr)
+// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
+type ResizeBilinearGradAttr func(optionalAttr)
 
-// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
+// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
 //
-// value: If `True`, updating of the var and m tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
+// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of grads and original_image. If false, rescale by
+// orig_height / height. Treat similarly the width dimension.
 // If not specified, defaults to false
-func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
+func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Update '*var' according to the AddSign update.
-//
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
-// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
-// variable <- variable - lr_t * update
+// Computes the gradient of bilinear interpolation.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	logbase: Must be a scalar.
-//	sign_decay: Must be a scalar.
-//	beta: Must be a scalar.
-//	grad: The gradient.
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
 //
-// Returns the created operation.
-func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16854,110 +16849,57 @@ func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Out
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyPowerSign",
+		Type: "ResizeBilinearGrad",
 		Input: []tf.Input{
-			var_, m, lr, logbase, sign_decay, beta, grad,
+			grads, original_image,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
-type MutableHashTableV2Attr func(optionalAttr)
-
-// MutableHashTableV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
-//
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Creates an empty hash table.
-//
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a scalar. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
+// Outputs all keys and values in the table.
 //
 // Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
+//	table_handle: Handle to the table.
 //
-// Returns Handle to a table.
-func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MutableHashTableV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Deprecated. Disallowed in GraphDef version >= 2.
 //
-// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
-func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
+//
+// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
+func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
 	opspec := tf.OpSpec{
-		Type: "AdjustContrast",
+		Type: "LookupTableExportV2",
 		Input: []tf.Input{
-			images, contrast_factor, min_value, max_value,
+			table_handle,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Table initializer that takes two tensors for keys and values respectively.
+// Replaces the contents of the table with the specified keys and values.
+//
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
 // Arguments:
-//	table_handle: Handle to a table which will be initialized.
-//	keys: Keys of type Tkey.
-//	values: Values of type Tval.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
 //
 // Returns the created operation.
-func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "InitializeTableV2",
+		Type: "LookupTableImportV2",
 		Input: []tf.Input{
 			table_handle, keys, values,
 		},
@@ -16965,263 +16907,629 @@ func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, val
 	return scope.AddOperation(opspec)
 }
 
-// PrintAttr is an optional argument to Print.
-type PrintAttr func(optionalAttr)
+// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
+type MapUnstageNoKeyAttr func(optionalAttr)
 
-// PrintMessage sets the optional message attribute to value.
+// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: A string, prefix of the error message.
-// If not specified, defaults to ""
-func PrintMessage(value string) PrintAttr {
+// REQUIRES: value >= 0
+func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["message"] = value
+		m["capacity"] = value
 	}
 }
 
-// PrintFirstN sets the optional first_n attribute to value.
+// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// value: Only log `first_n` number of times. -1 disables logging.
-// If not specified, defaults to -1
-func PrintFirstN(value int64) PrintAttr {
+// REQUIRES: value >= 0
+func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["first_n"] = value
+		m["memory_limit"] = value
 	}
 }
 
-// PrintSummarize sets the optional summarize attribute to value.
-//
-// value: Only print this many entries of each tensor.
-// If not specified, defaults to 3
-func PrintSummarize(value int64) PrintAttr {
+// MapUnstageNoKeyContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["summarize"] = value
+		m["container"] = value
 	}
 }
 
-// Prints a list of tensors.
-//
-// Passes `input` through to `output` and prints `data` when evaluating.
-//
-// Arguments:
-//	input: The tensor passed to `output`
-//	data: A list of tensors to print out when op is evaluated.
+// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns a random (key, value)
 //
-// Returns = The unmodified `input` tensor
-func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
+// from the underlying container.   If the underlying container
+// does not contain elements, the op will block until it does.
+func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Print",
+		Type: "MapUnstageNoKey",
 		Input: []tf.Input{
-			input, tf.OutputList(data),
+			indices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
-//
-// Arguments:
-//	tag: A string attached to this summary. Used for organization in TensorBoard.
-//	tensor: A tensor to serialize.
-//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
-// data.
-func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "TensorSummaryV2",
-		Input: []tf.Input{
-			tag, tensor, serialized_summary_metadata,
-		},
+	var idx int
+	var err error
+	key = op.Output(idx)
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstageNoKey", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return key, values
 }
 
-// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
-//
-// Arguments:
-//
-//	buffer_size: The maximum number of elements to buffer in an iterator over
-// this dataset.
-//
+// HashTableV2Attr is an optional argument to HashTableV2.
+type HashTableV2Attr func(optionalAttr)
+
+// HashTableV2Container sets the optional container attribute to value.
 //
-func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "PrefetchDataset",
-		Input: []tf.Input{
-			input_dataset, buffer_size,
-		},
-		Attrs: attrs,
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func HashTableV2Container(value string) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// TensorSummaryAttr is an optional argument to TensorSummary.
-type TensorSummaryAttr func(optionalAttr)
-
-// TensorSummaryDescription sets the optional description attribute to value.
+// HashTableV2SharedName sets the optional shared_name attribute to value.
 //
-// value: A json-encoded SummaryDescription proto.
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
 // If not specified, defaults to ""
-func TensorSummaryDescription(value string) TensorSummaryAttr {
+func HashTableV2SharedName(value string) HashTableV2Attr {
 	return func(m optionalAttr) {
-		m["description"] = value
+		m["shared_name"] = value
 	}
 }
 
-// TensorSummaryLabels sets the optional labels attribute to value.
+// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 //
-// value: An unused list of strings.
-// If not specified, defaults to <>
-func TensorSummaryLabels(value []string) TensorSummaryAttr {
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
+// If not specified, defaults to false
+func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
 	return func(m optionalAttr) {
-		m["labels"] = value
+		m["use_node_name_sharing"] = value
 	}
 }
 
-// TensorSummaryDisplayName sets the optional display_name attribute to value.
+// Creates a non-initialized hash table.
 //
-// value: An unused string.
+// This op creates a hash table, specifying the type of its keys and values.
+// Before using the table you will have to initialize it.  After initialization the
+// table will be immutable.
+//
+// Arguments:
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
+//
+// Returns Handle to a table.
+func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "HashTableV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
+type MutableHashTableV2Attr func(optionalAttr)
+
+// MutableHashTableV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
 // If not specified, defaults to ""
-func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
 	return func(m optionalAttr) {
-		m["display_name"] = value
+		m["container"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with a tensor.
+// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
 //
-// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
-// a tag as well as a serialized SummaryMetadata proto string that contains
-// plugin-specific data. We will keep this op to maintain backwards compatibility.
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+//
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
+// If not specified, defaults to false
+func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["use_node_name_sharing"] = value
+	}
+}
+
+// Creates an empty hash table.
+//
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a scalar. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
 //
 // Arguments:
-//	tensor: A tensor to serialize.
-func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
+//
+// Returns Handle to a table.
+func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorSummary",
-		Input: []tf.Input{
-			tensor,
-		},
+		Type: "MutableHashTableV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the gradient for the tanh of `x` wrt its input.
+// DequantizeAttr is an optional argument to Dequantize.
+type DequantizeAttr func(optionalAttr)
+
+// DequantizeMode sets the optional mode attribute to value.
+// If not specified, defaults to "MIN_COMBINED"
+func DequantizeMode(value string) DequantizeAttr {
+	return func(m optionalAttr) {
+		m["mode"] = value
+	}
+}
+
+// Dequantize the 'input' tensor into a float Tensor.
 //
-// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
-// is the corresponding input gradient.
-func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// [min_range, max_range] are scalar floats that specify the range for
+// the 'input' data. The 'mode' attribute controls exactly which calculations are
+// used to convert the float values to their quantized equivalents.
+//
+// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+//
+// ```
+// if T == qint8, in[i] += (range(T) + 1)/ 2.0
+// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
+// ```
+// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+//
+// *MIN_COMBINED Mode Example*
+//
+// If the input comes from a QuantizedRelu6, the output type is
+// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
+// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
+// Dequantize on quint8 will take each value, cast to float, and multiply
+// by 6 / 255.
+// Note that if quantizedtype is qint8, the operation will additionally add
+// each value by 128 prior to casting.
+//
+// If the mode is 'MIN_FIRST', then this approach is used:
+//
+// ```c++
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
+// range = (range_max - range_min) * range_adjust
+// range_scale = range / num_discrete_values
+// const double offset_input = static_cast<double>(input) - lowest_quantized;
+// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
+// ```
+//
+// *SCALED mode Example*
+//
+// `SCALED` mode matches the quantization approach used in
+// `QuantizeAndDequantize{V2|V3}`.
+//
+// If the mode is `SCALED`, we do not use the full range of the output type,
+// choosing to elide the lowest possible value for symmetry (e.g., output range is
+// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+// 0.
+//
+// We first find the range of values in our tensor. The
+// range we use is always centered on 0, so we find m such that
+// ```c++
+//   m = max(abs(input_min), abs(input_max))
+// ```
+//
+// Our input tensor range is then `[-m, m]`.
+//
+// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+// If T is signed, this is
+// ```
+//   num_bits = sizeof(T) * 8
+//   [min_fixed, max_fixed] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+// ```
+//
+// Otherwise, if T is unsigned, the fixed-point range is
+// ```
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+// ```
+//
+// From this we compute our scaling factor, s:
+// ```c++
+//   s = (2 * m) / (max_fixed - min_fixed)
+// ```
+//
+// Now we can dequantize the elements of our tensor:
+// ```c++
+// result = input * s
+// ```
+//
+// Arguments:
+//
+//	min_range: The minimum scalar value possibly produced for the input.
+//	max_range: The maximum scalar value possibly produced for the input.
+func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TanhGrad",
+		Type: "Dequantize",
 		Input: []tf.Input{
-			y, dy,
+			input, min_range, max_range,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Outputs a `Summary` protocol buffer with scalar values.
-//
-// The input `tags` and `values` must have the same shape.  The generated summary
-// has a summary value for each tag-value pair in `tags` and `values`.
-//
-// Arguments:
-//	tags: Tags for the summary.
-//	values: Same shape as `tags.  Values for the summary.
+// Flips all bits elementwise.
 //
-// Returns Scalar.  Serialized `Summary` protocol buffer.
-func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
+// The result will have exactly those bits set, that are not set in `x`. The
+// computation is performed on the underlying representation of x.
+func Invert(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ScalarSummary",
+		Type: "Invert",
 		Input: []tf.Input{
-			tags, values,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Outputs a `Summary` protocol buffer with a histogram.
-//
-// The generated
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// has one summary value containing a histogram for `values`.
-//
-// This op reports an `InvalidArgument` error if any value is not finite.
-//
-// Arguments:
-//	tag: Scalar.  Tag to use for the `Summary.Value`.
-//	values: Any shape. Values to use to build the histogram.
+// Deprecated. Disallowed in GraphDef version >= 2.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
+// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
+func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "HistogramSummary",
+		Type: "AdjustContrast",
 		Input: []tf.Input{
-			tag, values,
+			images, contrast_factor, min_value, max_value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the number of elements in the given queue.
+// Table initializer that takes two tensors for keys and values respectively.
 //
 // Arguments:
-//	handle: The handle to a queue.
+//	table_handle: Handle to a table which will be initialized.
+//	keys: Keys of type Tkey.
+//	values: Values of type Tval.
 //
-// Returns The number of elements in the given queue.
-func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
+// Returns the created operation.
+func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueSizeV2",
+		Type: "InitializeTableV2",
 		Input: []tf.Input{
-			handle,
+			table_handle, keys, values,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ImageSummaryAttr is an optional argument to ImageSummary.
-type ImageSummaryAttr func(optionalAttr)
+// PrintAttr is an optional argument to Print.
+type PrintAttr func(optionalAttr)
+
+// PrintMessage sets the optional message attribute to value.
+//
+// value: A string, prefix of the error message.
+// If not specified, defaults to ""
+func PrintMessage(value string) PrintAttr {
+	return func(m optionalAttr) {
+		m["message"] = value
+	}
+}
+
+// PrintFirstN sets the optional first_n attribute to value.
+//
+// value: Only log `first_n` number of times. -1 disables logging.
+// If not specified, defaults to -1
+func PrintFirstN(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["first_n"] = value
+	}
+}
+
+// PrintSummarize sets the optional summarize attribute to value.
+//
+// value: Only print this many entries of each tensor.
+// If not specified, defaults to 3
+func PrintSummarize(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["summarize"] = value
+	}
+}
+
+// Prints a list of tensors.
+//
+// Passes `input` through to `output` and prints `data` when evaluating.
+//
+// Arguments:
+//	input: The tensor passed to `output`
+//	data: A list of tensors to print out when op is evaluated.
+//
+// Returns = The unmodified `input` tensor
+func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Print",
+		Input: []tf.Input{
+			input, tf.OutputList(data),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
+//
+// Arguments:
+//	tag: A string attached to this summary. Used for organization in TensorBoard.
+//	tensor: A tensor to serialize.
+//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
+// data.
+func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorSummaryV2",
+		Input: []tf.Input{
+			tag, tensor, serialized_summary_metadata,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
+//
+// Arguments:
+//
+//	buffer_size: The maximum number of elements to buffer in an iterator over
+// this dataset.
+//
+//
+func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "PrefetchDataset",
+		Input: []tf.Input{
+			input_dataset, buffer_size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// TensorSummaryAttr is an optional argument to TensorSummary.
+type TensorSummaryAttr func(optionalAttr)
+
+// TensorSummaryDescription sets the optional description attribute to value.
+//
+// value: A json-encoded SummaryDescription proto.
+// If not specified, defaults to ""
+func TensorSummaryDescription(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["description"] = value
+	}
+}
+
+// TensorSummaryLabels sets the optional labels attribute to value.
+//
+// value: An unused list of strings.
+// If not specified, defaults to <>
+func TensorSummaryLabels(value []string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["labels"] = value
+	}
+}
+
+// TensorSummaryDisplayName sets the optional display_name attribute to value.
+//
+// value: An unused string.
+// If not specified, defaults to ""
+func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["display_name"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with a tensor.
+//
+// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
+// a tag as well as a serialized SummaryMetadata proto string that contains
+// plugin-specific data. We will keep this op to maintain backwards compatibility.
+//
+// Arguments:
+//	tensor: A tensor to serialize.
+func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorSummary",
+		Input: []tf.Input{
+			tensor,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the gradient for the tanh of `x` wrt its input.
+//
+// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+// is the corresponding input gradient.
+func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TanhGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Outputs a `Summary` protocol buffer with scalar values.
+//
+// The input `tags` and `values` must have the same shape.  The generated summary
+// has a summary value for each tag-value pair in `tags` and `values`.
+//
+// Arguments:
+//	tags: Tags for the summary.
+//	values: Same shape as `tags.  Values for the summary.
+//
+// Returns Scalar.  Serialized `Summary` protocol buffer.
+func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ScalarSummary",
+		Input: []tf.Input{
+			tags, values,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Outputs a `Summary` protocol buffer with a histogram.
+//
+// The generated
+// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+// has one summary value containing a histogram for `values`.
+//
+// This op reports an `InvalidArgument` error if any value is not finite.
+//
+// Arguments:
+//	tag: Scalar.  Tag to use for the `Summary.Value`.
+//	values: Any shape. Values to use to build the histogram.
+//
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "HistogramSummary",
+		Input: []tf.Input{
+			tag, values,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the number of elements in the given queue.
+//
+// Arguments:
+//	handle: The handle to a queue.
+//
+// Returns The number of elements in the given queue.
+func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "QueueSizeV2",
+		Input: []tf.Input{
+			handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ImageSummaryAttr is an optional argument to ImageSummary.
+type ImageSummaryAttr func(optionalAttr)
 
 // ImageSummaryMaxImages sets the optional max_images attribute to value.
 //
@@ -17533,31 +17841,7 @@ func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values
 	return op.Output(0)
 }
 
-// Writes a `Summary` protocol buffer with scalar values.
-//
-// The input `tag` and `value` must have the scalars.
-//
-// Arguments:
-//	writer: A handle to a summary writer.
-//	step: The step to write the summary for.
-//	tag: Tag for the summary.
-//	value: Value for the summary.
-//
-// Returns the created operation.
-func WriteScalarSummary(scope *Scope, writer tf.Output, step tf.Output, tag tf.Output, value tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteScalarSummary",
-		Input: []tf.Input{
-			writer, step, tag, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Computes the matrix exponential of one or more square matrices:
+// Computes the matrix exponential of one or more square matrices:
 //
 // exp(A) = \sum_{n=0}^\infty A^n/n!
 //
@@ -18841,29 +19125,6 @@ func GetSessionHandleV2(scope *Scope, value tf.Output) (handle tf.Output) {
 	return op.Output(0)
 }
 
-// Returns the set of files matching one or more glob patterns.
-//
-// Note that this routine only supports wildcard characters in the
-// basename portion of the pattern, not in the directory portion.
-//
-// Arguments:
-//	pattern: Shell wildcard pattern(s). Scalar or vector of type string.
-//
-// Returns A vector of matching filenames.
-func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "MatchingFiles",
-		Input: []tf.Input{
-			pattern,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
 type ResizeBicubicGradAttr func(optionalAttr)
 
@@ -20681,77 +20942,6 @@ func FixedLengthRecordDataset(scope *Scope, filenames tf.Output, header_bytes tf
 	return op.Output(0)
 }
 
-// Slice a `SparseTensor` based on the `start` and `size`.
-//
-// For example, if the input is
-//
-//     input_tensor = shape = [2, 7]
-//     [    a   d e  ]
-//     [b c          ]
-//
-// Graphically the output tensors are:
-//
-//     sparse_slice([0, 0], [2, 4]) = shape = [2, 4]
-//     [    a  ]
-//     [b c    ]
-//
-//     sparse_slice([0, 4], [2, 3]) = shape = [2, 3]
-//     [ d e  ]
-//     [      ]
-//
-// Arguments:
-//	indices: 2-D tensor represents the indices of the sparse tensor.
-//	values: 1-D tensor represents the values of the sparse tensor.
-//	shape: 1-D. tensor represents the shape of the sparse tensor.
-//	start: 1-D. tensor represents the start of the slice.
-//	size: 1-D. tensor represents the size of the slice.
-// output indices: A list of 1-D tensors represents the indices of the output
-// sparse tensors.
-//
-// Returns A list of 1-D tensors represents the values of the output sparse
-// tensors.A list of 1-D tensors represents the shape of the output sparse
-// tensors.
-func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSlice",
-		Input: []tf.Input{
-			indices, values, shape, start, size,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// Concatenates quantized tensors along one dimension.
-//
-// Arguments:
-//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [0, rank(values)).
-//	values: The `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//	input_mins: The minimum scalar values for each of the input tensors.
-//	input_maxes: The maximum scalar values for each of the input tensors.
-//
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
-func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "QuantizedConcat",
-		Input: []tf.Input{
-			concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes),
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
 // Gradients for batch normalization.
 //
 // DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
@@ -22596,28 +22786,6 @@ func Abs(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Flushes and closes the summary writer.
-//
-// Also removes it from the resource manager. To reopen, use another
-// CreateSummaryFileWriter op.
-//
-// Arguments:
-//	writer: A handle to the summary writer resource.
-//
-// Returns the created operation.
-func CloseSummaryWriter(scope *Scope, writer tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "CloseSummaryWriter",
-		Input: []tf.Input{
-			writer,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
 // StackV2Attr is an optional argument to StackV2.
 type StackV2Attr func(optionalAttr)
 
@@ -24972,101 +25140,6 @@ func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Scatter `updates` into a new (initially zero) tensor according to `indices`.
-//
-// Creates a new tensor by applying sparse `updates` to individual
-// values or slices within a zero tensor of the given `shape` according to
-// indices.  This operator is the inverse of the @{tf.gather_nd} operator which
-// extracts values or slices from a given tensor.
-//
-// **WARNING**: The order in which updates are applied is nondeterministic, so the
-// output will be nondeterministic if `indices` contains duplicates.
-//
-// `indices` is an integer tensor containing indices into a new tensor of shape
-// `shape`.  The last dimension of `indices` can be at most the rank of `shape`:
-//
-//     indices.shape[-1] <= shape.rank
-//
-// The last dimension of `indices` corresponds to indices into elements
-// (if `indices.shape[-1] = shape.rank`) or slices
-// (if `indices.shape[-1] < shape.rank`) along dimension `indices.shape[-1]` of
-// `shape`.  `updates` is a tensor with shape
-//
-//     indices.shape[:-1] + shape[indices.shape[-1]:]
-//
-// The simplest form of scatter is to insert individual elements in a tensor by
-// index. For example, say we want to insert 4 scattered elements in a rank-1
-// tensor with 8 elements.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd1.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[4], [3], [1], [7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     shape = tf.constant([8])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [0, 11, 0, 10, 9, 0, 0, 12]
-//
-// We can also, insert entire slices of a higher rank tensor all at once. For
-// example, if we wanted to insert two slices in the first dimension of a
-// rank-3 tensor with two matrices of new values.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/ScatterNd2.png" alt>
-// </div>
-//
-// In Python, this scatter operation would look like this:
-//
-// ```python
-//     indices = tf.constant([[0], [2]])
-//     updates = tf.constant([[[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]],
-//                            [[5, 5, 5, 5], [6, 6, 6, 6],
-//                             [7, 7, 7, 7], [8, 8, 8, 8]]])
-//     shape = tf.constant([4, 4, 4])
-//     scatter = tf.scatter_nd(indices, updates, shape)
-//     with tf.Session() as sess:
-//       print(sess.run(scatter))
-// ```
-//
-// The resulting tensor would look like this:
-//
-//     [[[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]],
-//      [[5, 5, 5, 5], [6, 6, 6, 6], [7, 7, 7, 7], [8, 8, 8, 8]],
-//      [[0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0], [0, 0, 0, 0]]]
-//
-// Arguments:
-//	indices: Index tensor.
-//	updates: Updates to scatter into output.
-//	shape: 1-D. The shape of the resulting tensor.
-//
-// Returns A new tensor with the given shape and updates applied according
-// to the indices.
-func ScatterNd(scope *Scope, indices tf.Output, updates tf.Output, shape tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ScatterNd",
-		Input: []tf.Input{
-			indices, updates, shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // SpaceToDepthAttr is an optional argument to SpaceToDepth.
 type SpaceToDepthAttr func(optionalAttr)
 
@@ -25917,31 +25990,6 @@ func ParallelConcat(scope *Scope, values []tf.Output, shape tf.Shape) (output tf
 	return op.Output(0)
 }
 
-// Concatenates tensors along one dimension.
-//
-// Arguments:
-//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [0, rank(values)).
-//	values: The `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.
-func Concat(scope *Scope, concat_dim tf.Output, values []tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Concat",
-		Input: []tf.Input{
-			concat_dim, tf.OutputList(values),
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Compute the lower regularized incomplete Gamma function `Q(a, x)`.
 //
 // The lower regularized incomplete Gamma function is defined as:
@@ -26131,129 +26179,122 @@ func ZerosLike(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Flips all bits elementwise.
+// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
+type QuantizedInstanceNormAttr func(optionalAttr)
+
+// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
 //
-// The result will have exactly those bits set, that are not set in `x`. The
-// computation is performed on the underlying representation of x.
-func Invert(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Invert",
-		Input: []tf.Input{
-			x,
-		},
+// value: If True, `given_y_min` and `given_y_min`
+// and `given_y_max` are used as the output range. Otherwise,
+// the implementation computes the output range.
+// If not specified, defaults to false
+func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["output_range_given"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// DequantizeAttr is an optional argument to Dequantize.
-type DequantizeAttr func(optionalAttr)
-
-// DequantizeMode sets the optional mode attribute to value.
-// If not specified, defaults to "MIN_COMBINED"
-func DequantizeMode(value string) DequantizeAttr {
+// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
+//
+// value: Output in `y_min` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
 	return func(m optionalAttr) {
-		m["mode"] = value
+		m["given_y_min"] = value
 	}
 }
 
-// Dequantize the 'input' tensor into a float Tensor.
-//
-// [min_range, max_range] are scalar floats that specify the range for
-// the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.
-//
-// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
-//
-// ```
-// if T == qint8, in[i] += (range(T) + 1)/ 2.0
-// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
-// ```
-// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
-//
-// *MIN_COMBINED Mode Example*
-//
-// If the input comes from a QuantizedRelu6, the output type is
-// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
-// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
-// Dequantize on quint8 will take each value, cast to float, and multiply
-// by 6 / 255.
-// Note that if quantizedtype is qint8, the operation will additionally add
-// each value by 128 prior to casting.
+// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
 //
-// If the mode is 'MIN_FIRST', then this approach is used:
+// value: Output in `y_max` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["given_y_max"] = value
+	}
+}
+
+// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
 //
-// ```c++
-// num_discrete_values = 1 << (# of bits in T)
-// range_adjust = num_discrete_values / (num_discrete_values - 1)
-// range = (range_max - range_min) * range_adjust
-// range_scale = range / num_discrete_values
-// const double offset_input = static_cast<double>(input) - lowest_quantized;
-// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
-// ```
+// value: A small float number to avoid dividing by 0.
+// If not specified, defaults to 1e-05
+func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["variance_epsilon"] = value
+	}
+}
+
+// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
 //
-// *SCALED mode Example*
+// value: Minimum value of `y_max - y_min`
+// If not specified, defaults to 0.001
+func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["min_separation"] = value
+	}
+}
+
+// Quantized Instance normalization.
 //
-// `SCALED` mode matches the quantization approach used in
-// `QuantizeAndDequantize{V2|V3}`.
+// Arguments:
+//	x: A 4D input Tensor.
+//	x_min: The value represented by the lowest quantized input.
+//	x_max: The value represented by the highest quantized input.
 //
-// If the mode is `SCALED`, we do not use the full range of the output type,
-// choosing to elide the lowest possible value for symmetry (e.g., output range is
-// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
-// 0.
+// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
+func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizedInstanceNorm",
+		Input: []tf.Input{
+			x, x_min, x_max,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// Returns the diagonal part of the tensor.
 //
-// We first find the range of values in our tensor. The
-// range we use is always centered on 0, so we find m such that
-// ```c++
-//   m = max(abs(input_min), abs(input_max))
-// ```
+// This operation returns a tensor with the `diagonal` part
+// of the `input`. The `diagonal` part is computed as follows:
 //
-// Our input tensor range is then `[-m, m]`.
+// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
+// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
 //
-// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
-// If T is signed, this is
-// ```
-//   num_bits = sizeof(T) * 8
-//   [min_fixed, max_fixed] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
-// ```
+// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
 //
-// Otherwise, if T is unsigned, the fixed-point range is
-// ```
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
-// ```
+// For example:
 //
-// From this we compute our scaling factor, s:
-// ```c++
-//   s = (2 * m) / (max_fixed - min_fixed)
 // ```
+// # 'input' is [[1, 0, 0, 0]
+//               [0, 2, 0, 0]
+//               [0, 0, 3, 0]
+//               [0, 0, 0, 4]]
 //
-// Now we can dequantize the elements of our tensor:
-// ```c++
-// result = input * s
+// tf.diag_part(input) ==> [1, 2, 3, 4]
 // ```
 //
 // Arguments:
+//	input: Rank k tensor where k is even and not zero.
 //
-//	min_range: The minimum scalar value possibly produced for the input.
-//	max_range: The maximum scalar value possibly produced for the input.
-func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
+// Returns The extracted diagonal.
+func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Dequantize",
+		Type: "DiagPart",
 		Input: []tf.Input{
-			input, min_range, max_range,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -26876,39 +26917,53 @@ func TileGrad(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Out
 	return op.Output(0)
 }
 
-// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
-type DataFormatDimMapAttr func(optionalAttr)
+// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
+type QuantizeAndDequantizeAttr func(optionalAttr)
 
-// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["src_format"] = value
+		m["signed_input"] = value
 	}
 }
 
-// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
-//
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["dst_format"] = value
+		m["num_bits"] = value
 	}
 }
 
-// Returns the dimension index in the destination data format given the one in
-//
-// the source data format.
-//
-// Arguments:
-//	x: A Tensor with each element as a dimension index in source data format.
-// Must be in the range [-4, 4).
+// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
+// If not specified, defaults to false
+func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["range_given"] = value
+	}
+}
+
+// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["input_min"] = value
+	}
+}
+
+// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["input_max"] = value
+	}
+}
+
+// Use QuantizeAndDequantizeV2 instead.
 //
-// Returns A Tensor with each element as a dimension index in destination data format.
-func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
+func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26917,9 +26972,9 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DataFormatDimMap",
+		Type: "QuantizeAndDequantize",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -26927,29 +26982,212 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt
 	return op.Output(0)
 }
 
-// Return the shape of s0 op s1 with broadcast.
-//
-// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
-// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
-func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BroadcastArgs",
-		Input: []tf.Input{
-			s0, s1,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
+// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
+type QueueDequeueV2Attr func(optionalAttr)
 
-// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
+// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// This is typically used by gradient computations for a broadcasting operation.
-func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
-	if scope.Err() != nil {
+// value: If the queue is empty, this operation will block for up to
+// timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
+	return func(m optionalAttr) {
+		m["timeout_ms"] = value
+	}
+}
+
+// Dequeues a tuple of one or more tensors from the given queue.
+//
+// This operation has k outputs, where k is the number of components
+// in the tuples stored in the given queue, and output i is the ith
+// component of the dequeued tuple.
+//
+// N.B. If the queue is empty, this operation will block until an element
+// has been dequeued (or 'timeout_ms' elapses, if specified).
+//
+// Arguments:
+//	handle: The handle to a queue.
+//	component_types: The type of each component in a tuple.
+//
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"component_types": component_types}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QueueDequeueV2",
+		Input: []tf.Input{
+			handle,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueV2", err)
+		return
+	}
+	return components
+}
+
+// Returns locations of nonzero / true values in a tensor.
+//
+// This operation returns the coordinates of true elements in `condition`. The
+// coordinates are returned in a 2-D tensor where the first dimension (rows)
+// represents the number of true elements, and the second dimension (columns)
+// represents the coordinates of the true elements. Keep in mind, the shape of
+// the output tensor can vary depending on how many true values there are in
+// `condition`. Indices are output in row-major order.
+//
+// For example:
+//
+// ```
+// # 'input' tensor is [[True, False]
+// #                    [True, False]]
+// # 'input' has two true values, so output has two coordinates.
+// # 'input' has rank of 2, so coordinates have two indices.
+// where(input) ==> [[0, 0],
+//                   [1, 0]]
+//
+// # `condition` tensor is [[[True, False]
+// #                     [True, False]]
+// #                    [[False, True]
+// #                     [False, True]]
+// #                    [[False, False]
+// #                     [False, True]]]
+// # 'input' has 5 true values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
+//
+// # `condition` tensor is [[[1.5,  0.0]
+// #                     [-0.5, 0.0]]
+// #                    [[0.0,  0.25]
+// #                     [0.0,  0.75]]
+// #                    [[0.0,  0.0]
+// #                     [0.0,  0.01]]]
+// # 'input' has 5 nonzero values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
+//
+// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
+// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
+// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
+// ```
+func Where(scope *Scope, condition tf.Output) (index tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Where",
+		Input: []tf.Input{
+			condition,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
+type DataFormatDimMapAttr func(optionalAttr)
+
+// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
+//
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["src_format"] = value
+	}
+}
+
+// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+//
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the dimension index in the destination data format given the one in
+//
+// the source data format.
+//
+// Arguments:
+//	x: A Tensor with each element as a dimension index in source data format.
+// Must be in the range [-4, 4).
+//
+// Returns A Tensor with each element as a dimension index in destination data format.
+func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DataFormatDimMap",
+		Input: []tf.Input{
+			x,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Return the shape of s0 op s1 with broadcast.
+//
+// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BroadcastArgs",
+		Input: []tf.Input{
+			s0, s1,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
+//
+// This is typically used by gradient computations for a broadcasting operation.
+func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
+	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
@@ -27293,134 +27531,24 @@ func SpaceToBatchND(scope *Scope, input tf.Output, block_shape tf.Output, paddin
 	return op.Output(0)
 }
 
-// QuantizeAndDequantizeV2Attr is an optional argument to QuantizeAndDequantizeV2.
-type QuantizeAndDequantizeV2Attr func(optionalAttr)
-
-// QuantizeAndDequantizeV2SignedInput sets the optional signed_input attribute to value.
+// SpaceToBatch for 4-D tensors of type T.
 //
-// value: If the quantization is signed or unsigned.
-// If not specified, defaults to true
-func QuantizeAndDequantizeV2SignedInput(value bool) QuantizeAndDequantizeV2Attr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
-
-// QuantizeAndDequantizeV2NumBits sets the optional num_bits attribute to value.
+// This is a legacy version of the more general SpaceToBatchND.
 //
-// value: The bitwidth of the quantization.
-// If not specified, defaults to 8
-func QuantizeAndDequantizeV2NumBits(value int64) QuantizeAndDequantizeV2Attr {
-	return func(m optionalAttr) {
-		m["num_bits"] = value
-	}
-}
-
-// QuantizeAndDequantizeV2RangeGiven sets the optional range_given attribute to value.
+// Zero-pads and then rearranges (permutes) blocks of spatial data into batch.
+// More specifically, this op outputs a copy of the input tensor where values from
+// the `height` and `width` dimensions are moved to the `batch` dimension. After
+// the zero-padding, both `height` and `width` of the input must be divisible by the
+// block size.
 //
-// value: If the range is given or should be computed from the tensor.
-// If not specified, defaults to false
-func QuantizeAndDequantizeV2RangeGiven(value bool) QuantizeAndDequantizeV2Attr {
-	return func(m optionalAttr) {
-		m["range_given"] = value
-	}
-}
-
-// Quantizes then dequantizes a tensor.
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, depth]`.
+//	paddings: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
+//   the padding of the input with zeros across the spatial dimensions as follows:
 //
-// This op simulates the precision loss from the quantized forward pass by:
-// 1. Quantizing the tensor to fixed point numbers, which should match the target
-//    quantization method when it is used in inference.
-// 2. Dequantizing it back to floating point numbers for the following ops, most
-//    likely matmul.
+//       paddings = [[pad_top, pad_bottom], [pad_left, pad_right]]
 //
-// There are different ways to quantize. This version does not use the full range
-// of the output type, choosing to elide the lowest possible value for symmetry
-// (e.g., output range is -127 to 127, not -128 to 127 for signed 8 bit
-// quantization), so that 0.0 maps to 0.
-//
-// To perform this op, we first find the range of values in our tensor. The range
-// we use is always centered on 0, so we find m such that
-//
-// 1. m = max(abs(input_min), abs(input_max)) if range_given is true,
-// 2. m = max(abs(min_elem(input)), abs(max_elem(input))) otherwise.
-//
-// Our input tensor range is then [-m, m].
-//
-// Next, we choose our fixed-point quantization buckets, [min_fixed, max_fixed].
-// If signed_input is true, this is
-//
-//   [min_fixed, max_fixed ] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1].
-//
-// Otherwise, if signed_input is false, the fixed-point range is
-//
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1].
-//
-// From this we compute our scaling factor, s:
-//
-//   s = (max_fixed - min_fixed) / (2 * m).
-//
-// Now we can quantize and dequantize the elements of our tensor.  An element e
-// is transformed into e':
-//
-//   e' = (e * s).round_to_nearest() / s.
-//
-// Note that we have a different number of buckets in the signed vs. unsigned
-// cases.  For example, if num_bits == 8, we get 254 buckets in the signed case
-// vs. 255 in the unsigned case.
-//
-// For example, suppose num_bits = 8 and m = 1.  Then
-//
-//   [min_fixed, max_fixed] = [-127, 127], and
-//   s = (127 + 127) / 2 = 127.
-//
-// Given the vector {-1, -0.5, 0, 0.3}, this is quantized to
-// {-127, -63, 0, 38}, and dequantized to {-1, -63.0/127, 0, 38.0/127}.
-//
-// Arguments:
-//	input: Tensor to quantize and then dequantize.
-//	input_min: If range_given, this is the min of the range, otherwise this input
-// will be ignored.
-//	input_max: If range_given, this is the max of the range, otherwise this input
-// will be ignored.
-func QuantizeAndDequantizeV2(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, optional ...QuantizeAndDequantizeV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantizeV2",
-		Input: []tf.Input{
-			input, input_min, input_max,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SpaceToBatch for 4-D tensors of type T.
-//
-// This is a legacy version of the more general SpaceToBatchND.
-//
-// Zero-pads and then rearranges (permutes) blocks of spatial data into batch.
-// More specifically, this op outputs a copy of the input tensor where values from
-// the `height` and `width` dimensions are moved to the `batch` dimension. After
-// the zero-padding, both `height` and `width` of the input must be divisible by the
-// block size.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, height, width, depth]`.
-//	paddings: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
-//   the padding of the input with zeros across the spatial dimensions as follows:
-//
-//       paddings = [[pad_top, pad_bottom], [pad_left, pad_right]]
-//
-//   The effective spatial dimensions of the zero-padded input tensor will be:
+//   The effective spatial dimensions of the zero-padded input tensor will be:
 //
 //       height_pad = pad_top + height + pad_bottom
 //       width_pad = pad_left + width + pad_right
@@ -27549,734 +27677,213 @@ func UnpackAxis(value int64) UnpackAttr {
 // Arguments:
 //	value: 1-D or higher, with `axis` dimension size equal to `num`.
 //
-//
-// Returns The list of tensors unpacked from `value`.
-func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num": num}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Unpack",
-		Input: []tf.Input{
-			value,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Unpack", err)
-		return
-	}
-	return output
-}
-
-// Increments variable pointed to by 'resource' until it reaches 'limit'.
-//
-// Arguments:
-//	resource: Should be from a scalar `Variable` node.
-//	limit: If incrementing ref would bring it above limit, instead generates an
-// 'OutOfRange' error.
-//
-//
-// Returns A copy of the input before increment. If nothing else modifies the
-// input, the values produced will all be distinct.
-func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"limit": limit, "T": T}
-	opspec := tf.OpSpec{
-		Type: "ResourceCountUpTo",
-		Input: []tf.Input{
-			resource,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Delete the stack from its resource container.
-//
-// Arguments:
-//	handle: The handle to a stack.
-//
-// Returns the created operation.
-func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "StackCloseV2",
-		Input: []tf.Input{
-			handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// BatchToSpace for N-D tensors of type T.
-//
-// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
-// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
-// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
-// the input.  The spatial dimensions of this intermediate result are then
-// optionally cropped according to `crops` to produce the output.  This is the
-// reverse of SpaceToBatch.  See below for a precise description.
-//
-// Arguments:
-//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
-// where spatial_shape has M dimensions.
-//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
-//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
-//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
-//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
-//   required that
-//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
-//
-// This operation is equivalent to the following steps:
-//
-// 1. Reshape `input` to `reshaped` of shape:
-//      [block_shape[0], ..., block_shape[M-1],
-//       batch / prod(block_shape),
-//       input_shape[1], ..., input_shape[N-1]]
-//
-// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1], block_shape[0],
-//       ...,
-//       input_shape[M], block_shape[M-1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0],
-//       ...,
-//       input_shape[M] * block_shape[M-1],
-//
-//       input_shape[M+1],
-//       ...,
-//       input_shape[N-1]]
-//
-// 4. Crop the start and end of dimensions `[1, ..., M]` of
-//    `reshaped_permuted` according to `crops` to produce the output of shape:
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
-//       ...,
-//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// Some examples:
-//
-// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 1]` and value:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 3]` and value:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-//
-// The output tensor has shape `[1, 4, 4, 1]` and value:
-//
-// ```
-// x = [[[1],   [2],  [3],  [4]],
-//      [[5],   [6],  [7],  [8]],
-//      [[9],  [10], [11],  [12]],
-//      [[13], [14], [15],  [16]]]
-// ```
-//
-// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [2, 0]]`:
-//
-// ```
-// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
-//      [[[0], [2], [4]]], [[[0], [10], [12]]],
-//      [[[0], [5], [7]]], [[[0], [13], [15]]],
-//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
-// ```
-//
-// The output tensor has shape `[2, 2, 4, 1]` and value:
-//
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
-func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BatchToSpaceND",
-		Input: []tf.Input{
-			input, block_shape, crops,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Extract `patches` from `images` and put them in the "depth" output dimension.
-//
-// Arguments:
-//	images: 4-D Tensor with shape `[batch, in_rows, in_cols, depth]`.
-//	ksizes: The size of the sliding window for each dimension of `images`.
-//	strides: 1-D of length 4. How far the centers of two consecutive patches are in
-// the images. Must be: `[1, stride_rows, stride_cols, 1]`.
-//	rates: 1-D of length 4. Must be: `[1, rate_rows, rate_cols, 1]`. This is the
-// input stride, specifying how far two consecutive patch samples are in the
-// input. Equivalent to extracting patches with
-// `patch_sizes_eff = patch_sizes + (patch_sizes - 1) * (rates - 1)`, followed by
-// subsampling them spatially by a factor of `rates`. This is equivalent to
-// `rate` in dilated (a.k.a. Atrous) convolutions.
-//	padding: The type of padding algorithm to use.
-//
-// We specify the size-related attributes as:
-//
-// ```python
-//       ksizes = [1, ksize_rows, ksize_cols, 1]
-//       strides = [1, strides_rows, strides_cols, 1]
-//       rates = [1, rates_rows, rates_cols, 1]
-// ```
-//
-// Returns 4-D Tensor with shape `[batch, out_rows, out_cols, ksize_rows *
-// ksize_cols * depth]` containing image patches with size
-// `ksize_rows x ksize_cols x depth` vectorized in the "depth" dimension. Note
-// `out_rows` and `out_cols` are the dimensions of the output patches.
-func ExtractImagePatches(scope *Scope, images tf.Output, ksizes []int64, strides []int64, rates []int64, padding string) (patches tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksizes": ksizes, "strides": strides, "rates": rates, "padding": padding}
-	opspec := tf.OpSpec{
-		Type: "ExtractImagePatches",
-		Input: []tf.Input{
-			images,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Bitcasts a tensor from one type to another without copying data.
-//
-// Given a tensor `input`, this operation returns a tensor that has the same buffer
-// data as `input` with datatype `type`.
-//
-// If the input datatype `T` is larger than the output datatype `type` then the
-// shape changes from [...] to [..., sizeof(`T`)/sizeof(`type`)].
-//
-// If `T` is smaller than `type`, the operator requires that the rightmost
-// dimension be equal to sizeof(`type`)/sizeof(`T`). The shape then goes from
-// [..., sizeof(`type`)/sizeof(`T`)] to [...].
-//
-// *NOTE*: Bitcast is implemented as a low-level cast, so machines with different
-// endian orderings will give different results.
-func Bitcast(scope *Scope, input tf.Output, type_ tf.DataType) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"type": type_}
-	opspec := tf.OpSpec{
-		Type: "Bitcast",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// OneHotAttr is an optional argument to OneHot.
-type OneHotAttr func(optionalAttr)
-
-// OneHotAxis sets the optional axis attribute to value.
-//
-// value: The axis to fill (default: -1, a new inner-most axis).
-// If not specified, defaults to -1
-func OneHotAxis(value int64) OneHotAttr {
-	return func(m optionalAttr) {
-		m["axis"] = value
-	}
-}
-
-// Returns a one-hot tensor.
-//
-// The locations represented by indices in `indices` take value `on_value`,
-// while all other locations take value `off_value`.
-//
-// If the input `indices` is rank `N`, the output will have rank `N+1`,
-// The new axis is created at dimension `axis` (default: the new axis is
-// appended at the end).
-//
-// If `indices` is a scalar the output shape will be a vector of length `depth`.
-//
-// If `indices` is a vector of length `features`, the output shape will be:
-// ```
-//   features x depth if axis == -1
-//   depth x features if axis == 0
-// ```
-//
-// If `indices` is a matrix (batch) with shape `[batch, features]`,
-// the output shape will be:
-// ```
-//   batch x features x depth if axis == -1
-//   batch x depth x features if axis == 1
-//   depth x batch x features if axis == 0
-// ```
-//
-//
-// Examples
-// =========
-//
-// Suppose that
-//
-// ```
-//   indices = [0, 2, -1, 1]
-//   depth = 3
-//   on_value = 5.0
-//   off_value = 0.0
-//   axis = -1
-// ```
-//
-// Then output is `[4 x 3]`:
-//
-//     ```output =
-//       [5.0 0.0 0.0]  // one_hot(0)
-//       [0.0 0.0 5.0]  // one_hot(2)
-//       [0.0 0.0 0.0]  // one_hot(-1)
-//       [0.0 5.0 0.0]  // one_hot(1)
-//     ```
-//
-// Suppose that
-//
-// ```
-//   indices = [0, 2, -1, 1]
-//   depth = 3
-//   on_value = 0.0
-//   off_value = 3.0
-//   axis = 0
-// ```
-//
-// Then output is `[3 x 4]`:
-//
-//     ```output =
-//       [0.0 3.0 3.0 3.0]
-//       [3.0 3.0 3.0 0.0]
-//       [3.0 3.0 3.0 3.0]
-//       [3.0 0.0 3.0 3.0]
-//     //  ^                one_hot(0)
-//     //      ^            one_hot(2)
-//     //          ^        one_hot(-1)
-//     //              ^    one_hot(1)
-//     ```
-// Suppose that
-//
-// ```
-//   indices = [[0, 2], [1, -1]]
-//   depth = 3
-//   on_value = 1.0
-//   off_value = 0.0
-//   axis = -1
-// ```
-//
-// Then output is `[2 x 2 x 3]`:
-//
-//     ```output =
-//       [
-//         [1.0, 0.0, 0.0]  // one_hot(0)
-//         [0.0, 0.0, 1.0]  // one_hot(2)
-//       ][
-//         [0.0, 1.0, 0.0]  // one_hot(1)
-//         [0.0, 0.0, 0.0]  // one_hot(-1)
-//       ]```
-//
-// Arguments:
-//	indices: A tensor of indices.
-//	depth: A scalar defining the depth of the one hot dimension.
-//	on_value: A scalar defining the value to fill in output when `indices[j] = i`.
-//	off_value: A scalar defining the value to fill in output when `indices[j] != i`.
-//
-// Returns The one-hot tensor.
-func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output, off_value tf.Output, optional ...OneHotAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OneHot",
-		Input: []tf.Input{
-			indices, depth, on_value, off_value,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
-type QueueDequeueV2Attr func(optionalAttr)
-
-// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
-//
-// value: If the queue is empty, this operation will block for up to
-// timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
-	}
-}
-
-// Dequeues a tuple of one or more tensors from the given queue.
-//
-// This operation has k outputs, where k is the number of components
-// in the tuples stored in the given queue, and output i is the ith
-// component of the dequeued tuple.
-//
-// N.B. If the queue is empty, this operation will block until an element
-// has been dequeued (or 'timeout_ms' elapses, if specified).
-//
-// Arguments:
-//	handle: The handle to a queue.
-//	component_types: The type of each component in a tuple.
-//
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QueueDequeueV2",
-		Input: []tf.Input{
-			handle,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueV2", err)
-		return
-	}
-	return components
-}
-
-// Returns locations of nonzero / true values in a tensor.
-//
-// This operation returns the coordinates of true elements in `condition`. The
-// coordinates are returned in a 2-D tensor where the first dimension (rows)
-// represents the number of true elements, and the second dimension (columns)
-// represents the coordinates of the true elements. Keep in mind, the shape of
-// the output tensor can vary depending on how many true values there are in
-// `condition`. Indices are output in row-major order.
-//
-// For example:
-//
-// ```
-// # 'input' tensor is [[True, False]
-// #                    [True, False]]
-// # 'input' has two true values, so output has two coordinates.
-// # 'input' has rank of 2, so coordinates have two indices.
-// where(input) ==> [[0, 0],
-//                   [1, 0]]
-//
-// # `condition` tensor is [[[True, False]
-// #                     [True, False]]
-// #                    [[False, True]
-// #                     [False, True]]
-// #                    [[False, False]
-// #                     [False, True]]]
-// # 'input' has 5 true values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-//
-// # `condition` tensor is [[[1.5,  0.0]
-// #                     [-0.5, 0.0]]
-// #                    [[0.0,  0.25]
-// #                     [0.0,  0.75]]
-// #                    [[0.0,  0.0]
-// #                     [0.0,  0.01]]]
-// # 'input' has 5 nonzero values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-//
-// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
-// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
-// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-// ```
-func Where(scope *Scope, condition tf.Output) (index tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Where",
-		Input: []tf.Input{
-			condition,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
-type QuantizeAndDequantizeAttr func(optionalAttr)
-
-// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
-
-// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["num_bits"] = value
-	}
-}
-
-// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
-// If not specified, defaults to false
-func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["range_given"] = value
-	}
-}
-
-// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["input_min"] = value
-	}
-}
-
-// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["input_max"] = value
-	}
-}
-
-// Use QuantizeAndDequantizeV2 instead.
-//
-// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
-func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
+//
+// Returns The list of tensors unpacked from `value`.
+func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num": num}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantize",
+		Type: "Unpack",
 		Input: []tf.Input{
-			input,
+			value,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Unpack", err)
+		return
+	}
+	return output
 }
 
-// Returns the diagonal part of the tensor.
-//
-// This operation returns a tensor with the `diagonal` part
-// of the `input`. The `diagonal` part is computed as follows:
-//
-// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
-// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
-//
-// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
-//
-// For example:
-//
-// ```
-// # 'input' is [[1, 0, 0, 0]
-//               [0, 2, 0, 0]
-//               [0, 0, 3, 0]
-//               [0, 0, 0, 4]]
-//
-// tf.diag_part(input) ==> [1, 2, 3, 4]
-// ```
+// Increments variable pointed to by 'resource' until it reaches 'limit'.
 //
 // Arguments:
-//	input: Rank k tensor where k is even and not zero.
+//	resource: Should be from a scalar `Variable` node.
+//	limit: If incrementing ref would bring it above limit, instead generates an
+// 'OutOfRange' error.
 //
-// Returns The extracted diagonal.
-func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
+//
+// Returns A copy of the input before increment. If nothing else modifies the
+// input, the values produced will all be distinct.
+func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"limit": limit, "T": T}
 	opspec := tf.OpSpec{
-		Type: "DiagPart",
+		Type: "ResourceCountUpTo",
 		Input: []tf.Input{
-			input,
+			resource,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
-type QuantizedInstanceNormAttr func(optionalAttr)
-
-// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
-//
-// value: If True, `given_y_min` and `given_y_min`
-// and `given_y_max` are used as the output range. Otherwise,
-// the implementation computes the output range.
-// If not specified, defaults to false
-func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["output_range_given"] = value
-	}
-}
-
-// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
+// Delete the stack from its resource container.
 //
-// value: Output in `y_min` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["given_y_min"] = value
-	}
-}
-
-// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
+// Arguments:
+//	handle: The handle to a stack.
 //
-// value: Output in `y_max` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["given_y_max"] = value
+// Returns the created operation.
+func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
-//
-// value: A small float number to avoid dividing by 0.
-// If not specified, defaults to 1e-05
-func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["variance_epsilon"] = value
+	opspec := tf.OpSpec{
+		Type: "StackCloseV2",
+		Input: []tf.Input{
+			handle,
+		},
 	}
+	return scope.AddOperation(opspec)
 }
 
-// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
+// BatchToSpace for N-D tensors of type T.
 //
-// value: Minimum value of `y_max - y_min`
-// If not specified, defaults to 0.001
-func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["min_separation"] = value
-	}
-}
-
-// Quantized Instance normalization.
+// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
+// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
+// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
+// the input.  The spatial dimensions of this intermediate result are then
+// optionally cropped according to `crops` to produce the output.  This is the
+// reverse of SpaceToBatch.  See below for a precise description.
 //
 // Arguments:
-//	x: A 4D input Tensor.
-//	x_min: The value represented by the lowest quantized input.
-//	x_max: The value represented by the highest quantized input.
+//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+// where spatial_shape has M dimensions.
+//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
+//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
+//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
+//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
+//   required that
+//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
 //
-// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
-func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
+// This operation is equivalent to the following steps:
+//
+// 1. Reshape `input` to `reshaped` of shape:
+//      [block_shape[0], ..., block_shape[M-1],
+//       batch / prod(block_shape),
+//       input_shape[1], ..., input_shape[N-1]]
+//
+// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1], block_shape[0],
+//       ...,
+//       input_shape[M], block_shape[M-1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0],
+//       ...,
+//       input_shape[M] * block_shape[M-1],
+//
+//       input_shape[M+1],
+//       ...,
+//       input_shape[N-1]]
+//
+// 4. Crop the start and end of dimensions `[1, ..., M]` of
+//    `reshaped_permuted` according to `crops` to produce the output of shape:
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
+//       ...,
+//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// Some examples:
+//
+// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 3]` and value:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// The output tensor has shape `[1, 4, 4, 1]` and value:
+//
+// ```
+// x = [[[1],   [2],  [3],  [4]],
+//      [[5],   [6],  [7],  [8]],
+//      [[9],  [10], [11],  [12]],
+//      [[13], [14], [15],  [16]]]
+// ```
+//
+// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [2, 0]]`:
+//
+// ```
+// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+//      [[[0], [2], [4]]], [[[0], [10], [12]]],
+//      [[[0], [5], [7]]], [[[0], [13], [15]]],
+//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
+// ```
+//
+// The output tensor has shape `[2, 2, 4, 1]` and value:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedInstanceNorm",
+		Type: "BatchToSpaceND",
 		Input: []tf.Input{
-			x, x_min, x_max,
+			input, block_shape, crops,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
-- 
GitLab


From 1a8258e0593270a8e2370517dff8faafce40a687 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Mon, 19 Mar 2018 22:29:33 -0700
Subject: [PATCH 228/960] Added infeed support for experimental C APIs
 associated with TPU graph rewrite.

This initial design of the C API is different from (and mostly higher level
than) the python API counterparts for infeed, in that the python API has
explicit graph construction APIs for generating infeed enqueue/dequeue
ops (e.g. split_inputs_and_generate_enqueue_ops() and generate_dequeue_op()),
while the C API takes an input graph and redirects all input nodes to feed the
infeed enqueue.

One requirement/restriction is that the input nodes in the TF
graph (e.g. Placeholder) must specify their tensor shapes, for infeed enqueue
and dequeue nodes to properly compile with XLA. The API for more general shape
support will be designed and implemented later.

PiperOrigin-RevId: 189693028
---
 tensorflow/c/c_api_experimental.cc | 204 +++++++++++++++++++++--------
 tensorflow/c/c_api_experimental.h  |  13 +-
 tensorflow/c/c_test_util.cc        |  10 +-
 tensorflow/c/c_test_util.h         |   3 +-
 4 files changed, 172 insertions(+), 58 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index f6d8949bb0..eb17e16d3e 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -26,6 +26,7 @@ using tensorflow::Node;
 using tensorflow::NodeBuilder;
 using tensorflow::NodeDef;
 using tensorflow::Status;
+using tensorflow::string;
 
 namespace {
 
@@ -38,12 +39,28 @@ TF_Operation* ToTF_Operation(Node* node) {
 
 // Graph rewrite algorithm (modeled after the python TPU graph rewrite path):
 //
-// 1. For each input node I, feed it to a new TPUReplicatedInput node, which in
-// turn feeds a new Identity node N, and store the mapping I->N.
+// 1. For each input node I, with C being the consumer node of I's output:
 //
-// 2. Rewrite all existing graph nodes by adding a attribute on TPU cluster. For
-// each node reading some input node I, rewire it to read from N instead based
-// on the I->N mapping in step #1.
+// a) When infeed is not specified, feed I to a new TPUReplicatedInput node
+// (both running on CPU), which in turn feeds a new Identity node N, and N feeds
+// C (both running on TPU).
+//
+// b) Otherwise, feed I to a new InfeedEnqueueTuple node IE, both running on
+// CPU. Also set an InfeedDequeueTuple node ID to feed C, both running on
+// TPU.
+//
+// In case b), if we have multiple input nodes, they all feed into the same
+// InfeedEnqueueTuple node, so that the graph has a single pair of infeed
+// enqueue and dequeue nodes. The list of output tensors from the dequeue node
+// can go to different consumer nodes. For example, say the original graph has
+// input nodes I1 and I2 respectively feeding nodes C1 and C2. After the rewrite
+// with infeed ops, we will have: I1 and I2 feed a single infeed enqueue node
+// IE, and a corresponding infeed dequeue node ID produces a list of two
+// tensors, respectively feeding C1 and C2.
+//
+// 2. Rewrite all existing graph nodes by adding an attribute on TPU
+// cluster. For each node C reading some input node I, rewire it to read from a
+// new input node generated in step #1 above.
 //
 // 3. For each output node O, feed it to a new Identity node, which in turn
 // feeds a new TPUReplicatedOutput node, which in turn feeds a new Identity node
@@ -66,7 +83,8 @@ class GraphRewriter {
     for (int i = 0; i < num_input_nodes; ++i) {
       //  Will fill in the value part later when we create the associated new
       //  input node.
-      input_node_map_[input_nodes[i].oper->node.name()] = nullptr;
+      input_node_map_[input_nodes[i].oper->node.name()] =
+          NodeBuilder::NodeOut(nullptr, -1);
     }
 
     // Grab all existing nodes for the upcoming rewrite, before mutating the
@@ -84,19 +102,24 @@ class GraphRewriter {
   // On success, sets `config_op` and `shutdown_op` to the corresponding
   // "ConfigureDistributedTPU" and "ShutdownDistributedTPU" nodes added to the
   // graph.
-  tensorflow::Status Rewrite(TF_Output* new_output_nodes, TF_Output* config_op,
-                             TF_Output* shutdown_op)
+  tensorflow::Status Rewrite(TF_Output* new_output_nodes,
+                             TF_Operation** infeed_enqueue_node,
+                             TF_Output* config_op, TF_Output* shutdown_op)
       EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
-    TF_RETURN_IF_ERROR(ProcessInputNodes());
+    TF_RETURN_IF_ERROR(ProcessInputNodes(infeed_enqueue_node));
 
     return RewriteGraphAndAddOutputNodes(new_output_nodes, config_op,
                                          shutdown_op);
   }
 
  private:
-  // Synthensizes new nodes for the input nodes, and creates a replicated
-  // metadata node.
-  tensorflow::Status ProcessInputNodes() EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
+  // Synthesizes new graph nodes (infeed enqueue or TPU replicated input
+  // nodes) for the input nodes, and creates a replicated metadata node.
+  //
+  // When `infeed_enqueue_node` is non-NULL and there are some input nodes,
+  // also adds the infeed dequeue node.
+  tensorflow::Status ProcessInputNodes(TF_Operation** infeed_enqueue_node)
+      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
     Node* metadata_node;
     TF_RETURN_IF_ERROR(
         NodeBuilder(metadata_node_name_.c_str(), "TPUReplicateMetadata")
@@ -104,34 +127,85 @@ class GraphRewriter {
             .Attr("_tpu_replicate", cluster_name_.c_str())
             .Finalize(&graph_->graph, &metadata_node));
 
-    for (int i = 0; i < input_node_map_.size(); ++i) {
-      VLOG(1) << "Handling input node " << input_nodes_[i].oper->node.name();
-      Node* replicated_input_node;
-      {
-        std::string replicated_input_name("TPUReplicate/input" +
-                                          std::to_string(i));
-        NodeBuilder::NodeOut input(&input_nodes_[i].oper->node,
-                                   input_nodes_[i].index);
-        std::vector<NodeBuilder::NodeOut> input_list;
-        input_list.push_back(input);
+    Node* dequeue_node = nullptr;
+    // Be deterministic in the corner case where `use_infeed` below is false.
+    if (infeed_enqueue_node) *infeed_enqueue_node = nullptr;
+    const bool use_infeed =
+        infeed_enqueue_node != nullptr && !input_node_map_.empty();
+    if (use_infeed) {
+      std::vector<NodeBuilder::NodeOut> new_input_list;
+      new_input_list.reserve(input_node_map_.size());
+      std::vector<tensorflow::DataType> input_dtypes;
+      input_dtypes.reserve(input_node_map_.size());
+      std::vector<tensorflow::TensorShape> input_shapes;
+      input_shapes.reserve(input_node_map_.size());
+      for (int i = 0; i < input_node_map_.size(); ++i) {
+        Node& input_node = input_nodes_[i].oper->node;
+        new_input_list.push_back(
+            NodeBuilder::NodeOut(&input_node, input_nodes_[i].index));
+        input_dtypes.push_back(input_node.output_type(input_nodes_[i].index));
+        tensorflow::TensorShapeProto shape;
         TF_RETURN_IF_ERROR(
-            NodeBuilder(replicated_input_name.c_str(), "TPUReplicatedInput")
-                // This op requires an input list.
-                .Input(input_list)
-                .Finalize(&graph_->graph, &replicated_input_node));
+            tensorflow::GetNodeAttr(input_node.attrs(), "shape", &shape));
+        VLOG(1) << "Input node " << i << " has shape " << shape.DebugString();
+        input_shapes.push_back(shape);
       }
+      // Enqueue always runs on CPU.
+      Node* enqueue_node;
+      TF_RETURN_IF_ERROR(NodeBuilder("InfeedEnqueueTuple", "InfeedEnqueueTuple")
+                             .Input(new_input_list)
+                             .Device("/device:CPU:0")
+                             .Attr("device_ordinal", 0)
+                             .Attr("dtypes", input_dtypes)
+                             .Attr("shapes", input_shapes)
+                             .Finalize(&graph_->graph, &enqueue_node));
+      *infeed_enqueue_node = ToTF_Operation(enqueue_node);
+      // The dequeue node should be put onto the "_tpu_replicate" cluster.
+      TF_RETURN_IF_ERROR(
+          NodeBuilder("TPUReplicate/InfeedDequeueTuple", "InfeedDequeueTuple")
+              .ControlInput(metadata_node)
+              .Attr("_tpu_replicate", cluster_name_.c_str())
+              .Attr("dtypes", input_dtypes)
+              .Attr("shapes", input_shapes)
+              .Finalize(&graph_->graph, &dequeue_node));
+    }
 
-      {
-        Node* new_input_node;
-        const std::string new_input_name("TPUReplicate/replicated_input_" +
-                                         std::to_string(i));
-        TF_RETURN_IF_ERROR(NodeBuilder(new_input_name.c_str(), "Identity")
-                               .Input(replicated_input_node, 0)
-                               .ControlInput(metadata_node)
-                               .Attr("_tpu_replicate", cluster_name_.c_str())
-                               .Finalize(&graph_->graph, &new_input_node));
-        DCHECK_GT(input_node_map_.count(input_nodes_[i].oper->node.name()), 0);
-        input_node_map_[input_nodes_[i].oper->node.name()] = new_input_node;
+    for (int i = 0; i < input_node_map_.size(); ++i) {
+      VLOG(1) << "Handling input node " << input_nodes_[i].oper->node.name();
+      if (use_infeed) {
+        DCHECK(dequeue_node);
+        input_node_map_[input_nodes_[i].oper->node.name()] =
+            NodeBuilder::NodeOut(dequeue_node, i);
+      } else {
+        Node* replicated_input_node;
+        {
+          std::string replicated_input_name("TPUReplicate/input" +
+                                            std::to_string(i));
+          NodeBuilder::NodeOut input(&input_nodes_[i].oper->node,
+                                     input_nodes_[i].index);
+          std::vector<NodeBuilder::NodeOut> input_list;
+          input_list.push_back(input);
+          TF_RETURN_IF_ERROR(
+              NodeBuilder(replicated_input_name.c_str(), "TPUReplicatedInput")
+                  // This op requires an input list.
+                  .Input(input_list)
+                  .Finalize(&graph_->graph, &replicated_input_node));
+        }
+
+        {
+          Node* new_input_node;
+          const std::string new_input_name("TPUReplicate/replicated_input_" +
+                                           std::to_string(i));
+          TF_RETURN_IF_ERROR(NodeBuilder(new_input_name.c_str(), "Identity")
+                                 .Input(replicated_input_node, 0)
+                                 .ControlInput(metadata_node)
+                                 .Attr("_tpu_replicate", cluster_name_.c_str())
+                                 .Finalize(&graph_->graph, &new_input_node));
+          DCHECK_GT(input_node_map_.count(input_nodes_[i].oper->node.name()),
+                    0);
+          input_node_map_[input_nodes_[i].oper->node.name()] =
+              NodeBuilder::NodeOut(new_input_node, 0);
+        }
       }
     }
     return Status::OK();
@@ -163,7 +237,9 @@ class GraphRewriter {
       }
 
       const NodeDef& old_def = n->def();
-      Node* new_node;
+      // Let node C be the consumer of `n`'s output in the original graph.
+      // This new node will feed into C in the rewritten graph.
+      NodeBuilder::NodeOut new_node;
       if (input_node_map_.count(n->name())) {
         new_node = input_node_map_[n->name()];
       } else {
@@ -173,10 +249,19 @@ class GraphRewriter {
         new_def.set_name(new_node_name);
         new_def.clear_input();
         for (int i = 0; i < old_def.input_size(); ++i) {
-          const std::string& old_input_name = old_def.input(i);
-          const std::string new_input_name =
+          const string old_input_name = old_def.input(i);
+          // When there are multiple input nodes that get mapped to the same
+          // infeed dequeue node, use different output ports of the dequeue
+          // node. e.g. Say in the original graph, input I1 feeds C1, and I2
+          // feeds C2. After the rewrite, I1 and I2 both feed a new infeed
+          // enqueue node, and the corresponding dequeue node has its output
+          // port 0 feeding C1, and output port 1 feeding C2. Note C1 and C2
+          // could be the same node (e.g. an Add that takes 2 inputs).
+          const string new_input_name =
               input_node_map_.count(old_input_name) > 0
-                  ? std::string(input_node_map_[old_input_name]->name())
+                  ? tensorflow::strings::StrCat(
+                        input_node_map_[old_input_name].node->name(), ":",
+                        input_node_map_[old_input_name].index)
                   : "TPUReplicate/" + old_input_name;
           new_def.add_input(new_input_name);
         }
@@ -192,11 +277,12 @@ class GraphRewriter {
         }
         tensorflow::AddNodeAttr("_tpu_replicate", cluster_name_.c_str(),
                                 &new_def);
-        new_node = graph_->graph.AddNode(new_def, &s);
+        new_node = NodeBuilder::NodeOut(graph_->graph.AddNode(new_def, &s), 0);
         if (!s.ok()) {
           return s;
         }
-        VLOG(1) << "The rewritten node node is " << new_node->DebugString();
+        VLOG(1) << "The rewritten node node is "
+                << new_node.node->DebugString();
       }
 
       if (output_node_map_.count(n->name()) > 0) {
@@ -206,7 +292,17 @@ class GraphRewriter {
           const PortIndexPair& pair = it->second;
           Node* out_identity_node;
           {
-            VLOG(1) << "Handling its output port " << pair.port
+            // If this output node is also an input, use the input_node_map_'s
+            // stored port, which would also work for an infeed dequeue op.
+            // Otherwise use pair.port.
+            // An example of the former: Say the graph has input nodes I1 and
+            // I2, and the output nodes are also I1 and I2. In the rewritten
+            // graph with infeed, the 2 output nodes will both come from a
+            // single infeed dequeue node ID, with output ports respectively
+            // set to 0 and 1.
+            const int output_port =
+                input_node_map_.count(n->name()) ? new_node.index : pair.port;
+            VLOG(1) << "Handling its output port " << output_port
                     << " at output index " << pair.index;
             std::string output_node_name = "TPUReplicate/Identity";
             if (pair.index > 0) {
@@ -214,7 +310,7 @@ class GraphRewriter {
             }
             TF_RETURN_IF_ERROR(
                 NodeBuilder(output_node_name.c_str(), "Identity")
-                    .Input(new_node, pair.port)
+                    .Input(new_node.node, output_port)
                     .Device(!old_def.device().empty()
                                 ? old_def.device()
                                 : tensorflow::strings::StrCat(
@@ -289,16 +385,18 @@ class GraphRewriter {
   // Keep mappings from the current input nodes to newly created input nodes,
   // which we will use to rewrite existing nodes that read these
   // inputs. e.g. A node that reads input node PlaceHolder could be rewired to
-  // read the created TPUReplicate/replicated_input_0 node.
-  std::unordered_map<std::string, Node*> input_node_map_;
+  // read the created TPUReplicate/replicated_input_0 node or some output port
+  // of the created TPUReplicate/InfeedDequeueTuple node. Because of the latter
+  // case, we the map entries store NodeBuilder::NodeOut, and not just Node*.
+  std::unordered_map<std::string, NodeBuilder::NodeOut> input_node_map_;
 
   std::vector<Node*> nodes_to_rewrite_;
 
   // Map from name to set{(output port, output tensor idx)}.
-  // e.g. Say ther are 3 output tensors, respectively produced by (node 0,
+  // e.g. Say there are 3 output tensors, respectively produced by (node 0,
   // port 0), (node 0, port 1), (node 1, port 0). Then the mapping entries
   // are: node 0 -> {(port 0, idx 0), (port 1, idx 1)} node 1 -> {(port 0, idx
-  // 2)} Based on these mappings, we will generated 3 new output nodes.
+  // 2)} Based on these mappings, we will generate 3 new output nodes.
   struct PortIndexPair {
     int port;
     int index;
@@ -331,7 +429,9 @@ TF_Output TF_SetupTPUExecution(TF_Session* session, int num_input_nodes,
                                const TF_Output* input_nodes,
                                int num_output_nodes,
                                const TF_Output* output_nodes,
-                               TF_Output* new_output_nodes, TF_Status* status) {
+                               TF_Output* new_output_nodes,
+                               TF_Operation** infeed_enqueue_node,
+                               TF_Status* status) {
   TF_Output config_op, shutdown_op;
   {
     auto graph = session->graph;
@@ -341,8 +441,8 @@ TF_Output TF_SetupTPUExecution(TF_Session* session, int num_input_nodes,
             << graph->graph.ToGraphDefDebug().DebugString();
     GraphRewriter rewriter(graph, num_input_nodes, input_nodes,
                            num_output_nodes, output_nodes);
-    status->status =
-        rewriter.Rewrite(new_output_nodes, &config_op, &shutdown_op);
+    status->status = rewriter.Rewrite(new_output_nodes, infeed_enqueue_node,
+                                      &config_op, &shutdown_op);
     if (!status->status.ok()) {
       return shutdown_op;
     }
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index af65123131..2bad278d63 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -63,7 +63,15 @@ TF_CAPI_EXPORT extern void TF_EnableXLACompilation(TF_SessionOptions* options,
 // Sets up TPU execution, by rewriting the graph accordingly, and initializing
 // TPU system.
 //
-// On success, returns a shutdown node to be used in a subsequent
+// When `infeed_enqueue_node` is non-NULL and there are input tensors, rewrites
+// the graph by adding the relevant infeed enqueue/dequeue ops, and returns the
+// enqueue op in `infeed_enqueue_node` on success, so that user can run that
+// node and feed input tensors. When there are no input tensors,
+// `infeed_enqueue_node` is ignored, and user should not run that node later.
+// TODO(hongm): In this case, we currently only support input tensors of dim 0
+// shape. Lift that constraint.
+//
+// On success, also returns a shutdown node to be used in a subsequent
 // TF_ShutdownTPUExecution(), and sets the new output nodes in
 // `new_output_nodes` for caller to fetch from. Must be called exactly once
 // before TF_SessionRun().
@@ -76,7 +84,8 @@ TF_CAPI_EXPORT extern void TF_EnableXLACompilation(TF_SessionOptions* options,
 TF_CAPI_EXPORT extern TF_Output TF_SetupTPUExecution(
     TF_Session* session, int num_input_nodes, const TF_Output* input_nodes,
     int num_output_nodes, const TF_Output* output_nodes,
-    TF_Output* new_output_nodes, TF_Status* status);
+    TF_Output* new_output_nodes, TF_Operation** infeed_enqueue_node,
+    TF_Status* status);
 
 // Shuts down TPU system. For any `session` where TF_SetupTPUExecution() has
 // been successfully called, this call must be made exactly once before the
diff --git a/tensorflow/c/c_test_util.cc b/tensorflow/c/c_test_util.cc
index 22f77e7b87..f3b28c1708 100644
--- a/tensorflow/c/c_test_util.cc
+++ b/tensorflow/c/c_test_util.cc
@@ -94,18 +94,22 @@ TF_Tensor* FloatTensor(float v) {
 // one cannot call ASSERT_* methods in non-void-returning functions (when
 // exceptions are disabled during compilation)
 void PlaceholderHelper(TF_Graph* graph, TF_Status* s, const char* name,
-                       TF_DataType dtype, TF_Operation** op) {
+                       TF_DataType dtype, const std::vector<int64_t>& dims,
+                       TF_Operation** op) {
   TF_OperationDescription* desc = TF_NewOperation(graph, "Placeholder", name);
   TF_SetAttrType(desc, "dtype", dtype);
+  if (!dims.empty()) {
+    TF_SetAttrShape(desc, "shape", dims.data(), dims.size());
+  }
   *op = TF_FinishOperation(desc, s);
   ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
   ASSERT_NE(*op, nullptr);
 }
 
 TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s, const char* name,
-                          TF_DataType dtype) {
+                          TF_DataType dtype, const std::vector<int64_t>& dims) {
   TF_Operation* op;
-  PlaceholderHelper(graph, s, name, dtype, &op);
+  PlaceholderHelper(graph, s, name, dtype, dims, &op);
   return op;
 }
 
diff --git a/tensorflow/c/c_test_util.h b/tensorflow/c/c_test_util.h
index d87c57fd51..cd19cf8d62 100644
--- a/tensorflow/c/c_test_util.h
+++ b/tensorflow/c/c_test_util.h
@@ -48,7 +48,8 @@ TF_Tensor* FloatTensor(float v);
 
 TF_Operation* Placeholder(TF_Graph* graph, TF_Status* s,
                           const char* name = "feed",
-                          TF_DataType dtype = TF_INT32);
+                          TF_DataType dtype = TF_INT32,
+                          const std::vector<int64_t>& dims = {});
 
 TF_Operation* Const(TF_Tensor* t, TF_Graph* graph, TF_Status* s,
                     const char* name = "const");
-- 
GitLab


From 2311e9ced599d08f705afd631ee45cf027d05618 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 19 Mar 2018 22:56:01 -0700
Subject: [PATCH 229/960] Predictions have to be updated for exported output
 signatures

PiperOrigin-RevId: 189694707
---
 tensorflow/contrib/estimator/BUILD            |  1 +
 .../estimator/python/estimator/extenders.py   | 13 +++++-
 .../python/estimator/extenders_test.py        | 45 +++++++++++++++++++
 3 files changed, 58 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 26d6bc5ae6..2f7ed7cd73 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -142,6 +142,7 @@ py_test(
     deps = [
         ":extenders",
         "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/predictor",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:constant_op",
         "//tensorflow/python:framework_ops",
diff --git a/tensorflow/contrib/estimator/python/estimator/extenders.py b/tensorflow/contrib/estimator/python/estimator/extenders.py
index 2b6881b814..266ae93305 100644
--- a/tensorflow/contrib/estimator/python/estimator/extenders.py
+++ b/tensorflow/contrib/estimator/python/estimator/extenders.py
@@ -23,6 +23,7 @@ import six
 from tensorflow.python.estimator import estimator as estimator_lib
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.estimator import util as estimator_util
+from tensorflow.python.estimator.export.export_output import PredictOutput
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 from tensorflow.python.ops import clip_ops
@@ -233,7 +234,17 @@ def forward_features(estimator, keys=None):
             'argument of forward_features to filter unwanted features. Type of '
             'features[{}] is {}.'.format(key, key, type(feature)))
       predictions[key] = feature
-    return spec._replace(predictions=predictions)
+    spec = spec._replace(predictions=predictions)
+    if spec.export_outputs:
+      for ekey in ['predict', 'serving_default']:
+        if (ekey in spec.export_outputs and
+            isinstance(spec.export_outputs[ekey],
+                       PredictOutput)):
+          export_outputs = spec.export_outputs[ekey].outputs
+          for key in get_keys(features):
+            export_outputs[key] = predictions[key]
+
+    return spec
 
   return estimator_lib.Estimator(
       model_fn=new_model_fn,
diff --git a/tensorflow/contrib/estimator/python/estimator/extenders_test.py b/tensorflow/contrib/estimator/python/estimator/extenders_test.py
index ad1a8ef152..407af2deaf 100644
--- a/tensorflow/contrib/estimator/python/estimator/extenders_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/extenders_test.py
@@ -18,20 +18,27 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+import tempfile
 import numpy as np
 
 from tensorflow.contrib.estimator.python.estimator import extenders
+from tensorflow.contrib.predictor import from_saved_model
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.estimator import estimator_lib
 from tensorflow.python.estimator.canned import linear
 from tensorflow.python.feature_column import feature_column as fc
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.ops import variables
+from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.training import training
+from tensorflow.python.util import compat
 
 
 def get_input_fn(x, y):
@@ -177,6 +184,44 @@ class ForwardFeaturesTest(test.TestCase):
     self.assertIn('id', predictions)
     self.assertEqual(101, predictions['id'])
 
+  def test_forward_in_exported(self):
+
+    def serving_input_fn():
+      features_ph = {
+          'x': array_ops.placeholder(dtypes.float32, [None]),
+          'id': array_ops.placeholder(dtypes.int32, [None])
+      }
+      features = {
+          key: array_ops.expand_dims(tensor, -1)
+          for key, tensor in features_ph.items()
+      }
+      return estimator_lib.export.ServingInputReceiver(features, features_ph)
+    def input_fn():
+      return {'x': [[3.], [5.]], 'id': [[101], [102]]}, [[1.], [2.]]
+    # create estimator
+    feature_columns = [fc.numeric_column('x')]
+    estimator = linear.LinearRegressor(feature_columns)
+    estimator.train(input_fn=input_fn, steps=1)
+    estimator = extenders.forward_features(estimator, 'id')
+
+    # export saved model
+    tmpdir = tempfile.mkdtemp()
+    export_dir_base = os.path.join(
+        compat.as_bytes(tmpdir), compat.as_bytes('export'))
+    export_dir = estimator.export_savedmodel(export_dir_base, serving_input_fn)
+    self.assertTrue(gfile.Exists(export_dir))
+
+    # restore model
+    predict_fn = from_saved_model(export_dir, signature_def_key='predict')
+    predictions = predict_fn({'x': [3], 'id': [101]})
+
+    # verify that 'id' exists in predictions
+    self.assertIn('id', predictions)
+    self.assertEqual(101, predictions['id'])
+
+    # Clean up.
+    gfile.DeleteRecursively(tmpdir)
+
   def test_forward_list(self):
 
     def input_fn():
-- 
GitLab


From e2e67c528316be8ea4f624af8757e80d7f00b5b6 Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Mon, 19 Mar 2018 23:15:42 -0700
Subject: [PATCH 230/960] Fix some edge cases around scalar indices in the
 gather expander

I discovered these when changing the tf2xla bridge to directly emit gather
operations.

 - DeScalarizeGatherIndices was assuming that gather_indices must be of at least
   rank 1.  Fix this to be more general.

 - We were passing in the wrong version of gather indices to
   ExpandFirstDimIntoNDims.  We don't strictly need to pass in
   transposed_gather_indices (since if transposed_gather_indices is rank 1 then
   the transpose has to be an identity transpose), passing in
   descalarized_gather_indices would also have been fine, but
   transposed_gather_indices seems more uniform.

 - ExpandGatherDimsInAccumulator was assuming that gather_indices must be of at
   least rank 1 (by calling CollapseFirstNDims).  Fix this to be more general.

 - We were trying to go through with emitting zero sized gather operations.  I
   don't think it is worth dealing with all of the edge cases this would expose
   so now we just punt to ZeroSizedHloElimination.

PiperOrigin-RevId: 189696444
---
 .../compiler/xla/service/gather_expander.cc   | 44 +++++++++----
 .../xla/service/hlo_creation_utils.cc         | 18 ------
 .../compiler/xla/service/hlo_creation_utils.h | 10 ---
 .../xla/tests/gather_operation_test.cc        | 62 +++++++++++++++++++
 4 files changed, 93 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index 58c62d8ce9..488bed35fe 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -53,9 +53,14 @@ static StatusOr<HloInstruction*> DeScalarizeGatherIndices(
     return gather_indices;
   }
 
-  int64 last_index = gather_indices_shape.dimensions(
-      gather_indices_shape.dimensions_size() - 1);
-  return ExpandLastDimIntoNDims(gather_indices, {last_index, 1});
+  DCHECK_EQ(index_vector_dim, gather_indices_shape.dimensions_size());
+
+  std::vector<int64> result_shape_dims;
+  c_copy(gather_indices_shape.dimensions(),
+         std::back_inserter(result_shape_dims));
+  result_shape_dims.push_back(1);
+
+  return MakeReshapeHlo(result_shape_dims, gather_indices);
 }
 
 // Canonicalizes the gather_indices tensors so that we only have deal with some
@@ -81,16 +86,17 @@ static StatusOr<HloInstruction*> CanonicalizeGatherIndices(
   // all of the non-index-vector dimensions.
   const Shape& shape = transposed_gather_indices->shape();
   if (shape.dimensions_size() == 1) {
-    return ExpandFirstDimIntoNDims(gather_indices, {1, shape.dimensions(0)});
+    return ExpandFirstDimIntoNDims(transposed_gather_indices,
+                                   {1, shape.dimensions(0)});
   } else {
     return CollapseFirstNDims(transposed_gather_indices,
                               shape.dimensions_size() - 1);
   }
 }
 
-// Expands out the gather dimensions in the accumulator produced by the while
-// loop.
-static StatusOr<HloInstruction*> ExpandGatherDimsInAccumulator(
+// Expands out or contracts away the gather dimensions in the accumulator
+// produced by the while loop.
+static StatusOr<HloInstruction*> AdjustGatherDimsInAccumulator(
     const Shape& gather_indices_shape, HloInstruction* accumulator,
     int64 index_vector_dim) {
   std::vector<int64> output_gather_dim_bounds;
@@ -103,9 +109,14 @@ static StatusOr<HloInstruction*> ExpandGatherDimsInAccumulator(
 
   if (output_gather_dim_bounds.empty()) {
     // If output_gather_dim_bounds is empty we must be lowering a (effectively)
-    // dynamic-slice.
+    // dynamic-slice.  In that case, there is a leading degenerate gather
+    // dimension that we added to make this special case play well with the
+    // general while loop which we need to remove now.
     CHECK_EQ(accumulator->shape().dimensions(0), 1);
-    return CollapseFirstNDims(accumulator, 2);
+    ArraySlice<int64> reshaped_dim_sizes =
+        AsInt64Slice(accumulator->shape().dimensions());
+    reshaped_dim_sizes.remove_prefix(1);
+    return MakeReshapeHlo(reshaped_dim_sizes, accumulator);
   }
 
   return ExpandFirstDimIntoNDims(accumulator, output_gather_dim_bounds);
@@ -290,6 +301,8 @@ static StatusOr<HloInstruction*> PermuteGatherAndWindowDims(
 
 StatusOr<HloInstruction*> GatherExpander::ExpandGather(
     HloInstruction* gather_instr) {
+  CHECK(!ShapeUtil::HasZeroElements(gather_instr->shape()));
+
   HloComputation* computation = gather_instr->parent();
   HloInstruction* operand = gather_instr->mutable_operand(0);
   HloInstruction* gather_indices = gather_instr->mutable_operand(1);
@@ -331,7 +344,7 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * accumulator_with_output_gather_dims_decanonicalized,
-      ExpandGatherDimsInAccumulator(gather_indices->shape(),
+      AdjustGatherDimsInAccumulator(gather_indices->shape(),
                                     accumulator_with_window_dims_elided,
                                     dim_numbers.index_vector_dim()));
 
@@ -341,12 +354,17 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
 }
 
 StatusOr<bool> GatherExpander::Run(HloModule* module) {
+  auto is_nontrivial_gather = [](HloInstruction* inst) {
+    return inst->opcode() == HloOpcode::kGather &&
+           // Avoid expanding gather ops that produce zero sized tensors,
+           // instead punt these to ZeroSizedHloElimination.
+           !ShapeUtil::HasZeroElements(inst->shape());
+  };
+
   std::vector<HloInstruction*> gather_instrs;
   for (HloComputation* computation : module->MakeNonfusionComputations()) {
     c_copy_if(computation->instructions(), std::back_inserter(gather_instrs),
-              [](HloInstruction* inst) {
-                return inst->opcode() == HloOpcode::kGather;
-              });
+              is_nontrivial_gather);
   }
 
   for (HloInstruction* inst : gather_instrs) {
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.cc b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
index fbe71f8d5b..b186767ce7 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.cc
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.cc
@@ -201,24 +201,6 @@ StatusOr<HloInstruction*> ExpandFirstDimIntoNDims(
   return MakeReshapeHlo(new_shape, operand);
 }
 
-StatusOr<HloInstruction*> ExpandLastDimIntoNDims(
-    HloInstruction* operand, ArraySlice<int64> expanded_dims) {
-  CHECK_GT(operand->shape().dimensions_size(), 0);
-  CHECK_EQ(operand->shape().dimensions(operand->shape().dimensions_size() - 1),
-           Product(expanded_dims));
-
-  std::vector<int64> expanded_shape_dim_bounds;
-  expanded_shape_dim_bounds.reserve(expanded_dims.size() +
-                                    operand->shape().dimensions_size() - 1);
-  std::copy(operand->shape().dimensions().begin(),
-            operand->shape().dimensions().end() - 1,
-            std::back_inserter(expanded_shape_dim_bounds));
-  c_copy(expanded_dims, std::back_inserter(expanded_shape_dim_bounds));
-  Shape new_shape = ShapeUtil::MakeShape(operand->shape().element_type(),
-                                         expanded_shape_dim_bounds);
-  return MakeReshapeHlo(new_shape, operand);
-}
-
 StatusOr<HloInstruction*> ElideDegenerateDims(HloInstruction* operand,
                                               ArraySlice<int64> dims_to_elide) {
   CHECK(c_is_sorted(dims_to_elide));
diff --git a/tensorflow/compiler/xla/service/hlo_creation_utils.h b/tensorflow/compiler/xla/service/hlo_creation_utils.h
index 6032ebab74..d99e32a737 100644
--- a/tensorflow/compiler/xla/service/hlo_creation_utils.h
+++ b/tensorflow/compiler/xla/service/hlo_creation_utils.h
@@ -119,16 +119,6 @@ StatusOr<HloInstruction*> CollapseFirstNDims(HloInstruction* operand, int64 n);
 StatusOr<HloInstruction*> ExpandFirstDimIntoNDims(
     HloInstruction* operand, tensorflow::gtl::ArraySlice<int64> expanded_dims);
 
-// Expands (via reshape) the last (logical) dimension of `operand` into a
-// sequence of `expanded_dims` dimensions.  `operand` must at least be of rank 1
-// and the number of elements in its last dimension must be equal to the
-// product of `expanded_dims`.
-//
-// For instance if `operand` has shape f32[9,7,200] and expanded_dims is
-// {2,5,20} the result is `operand` reshaped to [9,7,2,5,20].
-StatusOr<HloInstruction*> ExpandLastDimIntoNDims(
-    HloInstruction* operand, tensorflow::gtl::ArraySlice<int64> expanded_dims);
-
 // Elides (via reshape) a set of degenerate dimensions (dimensions containing
 // exactly one element), `dims_to_elide` from `operand`.  Every dimension in
 // `dims_to_elide` must be a degenerate dimension.  `dims_to_elide` must be
diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc
index 0830e9c8f0..8ba91946c0 100644
--- a/tensorflow/compiler/xla/tests/gather_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc
@@ -335,5 +335,67 @@ ENTRY main {
           {operand.get(), gather_indices.get(), in_bounds_mask.get()});
 }
 
+XLA_TEST_F(GatherOperationTest, OneScalarIndex) {
+  const char* hlo_text = R"(
+HloModule OneScalarIndex
+
+ENTRY main {
+  operand = s32[2,3,2]{2,1,0} parameter(0)
+  index = s32[] parameter(1)
+  ROOT gather = s32[1,3,2]{2,1,0} gather(operand, index),
+      output_window_dims={0,1,2},
+      elided_window_dims={},
+      gather_dims_to_operand_dims={0},
+      index_vector_dim=0,
+      window_bounds={1,3,2}
+}
+)";
+  std::unique_ptr<Literal> operand = Literal::CreateR3<int32>(
+      {{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}});
+  std::unique_ptr<Literal> gather_indices = Literal::CreateR0<int32>(1);
+  RunTest(hlo_text, operand.get(), gather_indices.get());
+}
+
+XLA_TEST_F(GatherOperationTest, ScalarResult) {
+  const char* hlo_text = R"(
+HloModule ScalarResult
+
+ENTRY main {
+  operand = s32[4]{0} parameter(0)
+  index = s32[] parameter(1)
+  ROOT gather = s32[] gather(operand, index),
+      output_window_dims={},
+      elided_window_dims={0},
+      gather_dims_to_operand_dims={0},
+      index_vector_dim=0,
+      window_bounds={1}
+}
+)";
+  std::unique_ptr<Literal> operand = Literal::CreateR1<int32>({1, 2, 3, 4});
+  std::unique_ptr<Literal> gather_indices = Literal::CreateR0<int32>(1);
+  RunTest(hlo_text, operand.get(), gather_indices.get());
+}
+
+XLA_TEST_F(GatherOperationTest, ZeroSizedResult) {
+  const string hlo_text = R"(
+HloModule ZeroSizedResult
+
+ENTRY main {
+  operand = s32[3,3] parameter(0)
+  indices = s32[0] parameter(1)
+  ROOT gather = s32[0,3] gather(operand, indices),
+      output_window_dims={1},
+      elided_window_dims={0},
+      gather_dims_to_operand_dims={0},
+      index_vector_dim=1,
+      window_bounds={1, 3}
+}
+)";
+  std::unique_ptr<Literal> operand =
+      Literal::CreateR2<int32>({{1, 2, 3}, {4, 5, 6}, {7, 8, 9}});
+  std::unique_ptr<Literal> gather_indices = Literal::CreateR1<int32>({});
+  RunTest(hlo_text, operand.get(), gather_indices.get());
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 28db3a7eae4986e3e662de16188cf7a03be33768 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 02:26:31 -0700
Subject: [PATCH 231/960] Fix bug

PiperOrigin-RevId: 189712233
---
 tensorflow/compiler/jit/xla_launch_util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 076cbd2084..bb7316c60c 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -169,7 +169,7 @@ void XlaComputationLaunchContext::PopulateOutputs(
   int output_num = 0;
   for (int i = 0; i < ctx->num_outputs(); ++i) {
     AllocatorAttributes alloc_attrs = ctx->output_alloc_attr(i);
-    Allocator* allocator = ctx->device()->GetAllocator(alloc_attrs);
+    Allocator* allocator = ctx->device()->GetAllocator({});
     if (tensor_info_manager_ && !alloc_attrs.on_host()) {
       allocator = tensor_info_manager_;
     }
-- 
GitLab


From 163bb675579bbc3a115c0caac9b42891f629bfd4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 03:11:32 -0700
Subject: [PATCH 232/960] - Added support for data to be specified in RNN
 classes as large tensors with time folded into the batch dimension instead of
 lists of tensors - Significant refactoring of RNN classes - Fixed a bunch of
 issues in the LayerCollection docstrings, especially around the 'reuse'
 argument.

PiperOrigin-RevId: 189716331
---
 .../kernel_tests/fisher_factors_test.py       |  12 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  | 344 +++++++++---------
 .../contrib/kfac/python/ops/fisher_factors.py |  65 ++--
 .../kfac/python/ops/layer_collection.py       | 163 ++++++---
 tensorflow/contrib/kfac/python/ops/utils.py   |   3 +
 5 files changed, 314 insertions(+), 273 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
index 16f02f1199..e007f70939 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
@@ -862,8 +862,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default():
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3), name='a/b/c')
-      tensor_list = [tensor]
-      factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=False)
+      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=False)
       factor.instantiate_cov_variables()
       self.assertEqual([3, 3], factor.get_cov().get_shape().as_list())
 
@@ -872,8 +871,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
       dtype = dtypes.float64_ref
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c')
-      tensor_list = [tensor]
-      factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=False)
+      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=False)
       factor.instantiate_cov_variables()
       cov = factor.get_cov()
       self.assertEqual(cov.dtype, dtype)
@@ -883,8 +881,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      tensor_list = [tensor]
-      factor = ff.FullyConnectedMultiKF((tensor_list,), has_bias=True)
+      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=True)
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -895,8 +892,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      tensor_list = [tensor]
-      factor = ff.FullyConnectedMultiKF((tensor_list,))
+      factor = ff.FullyConnectedMultiKF((tensor,))
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index 79d0424dca..f517e3148f 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -106,55 +106,6 @@ def _make_partitionedtensors_grads(grads_list):
   return tuple(utils.PartitionedTensor(grads) for grads in grads_list)
 
 
-def _make_partitionedtensors_multi_inputs(inputs):
-  """Constructs PartitionedTensors for inputs.
-
-  The purpose of this method is to package up the towers/minibatch dimension
-  of these arrays into PartitionedTensor objects.
-
-  This version of this function is for use with FisherBlocks that deal with
-  multiple uses or time-steps. One PartitionedTensor is created for each
-  use/time-step.  The FisherBlock will be responsible for concatenating
-  (or doing whatever else it wants) with the resulting lists.
-
-  Args:
-    inputs: a 2-D list of Tensors. First index is tower/mini-batch, second is
-      use/time-step.
-
-  Returns:
-    A tuple of PartitionedTensor's, one per use/time-step.
-  """
-  num_uses = len(inputs[0])
-  assert all(len(input_) == num_uses for input_ in inputs)
-
-  return tuple(utils.PartitionedTensor(input_) for input_ in zip(*inputs))
-
-
-def _make_partitionedtensors_multi_grads(grads_list):
-  """Constructs PartitionedTensors for grads_list.
-
-  The purpose of this method is to package up the towers/minibatch dimension
-  of these arrays into PartitionedTensor objects.
-
-  This version of this function is for use with FisherBlocks that deal with
-  multiple uses or time-steps. One PartitionedTensor is created for each
-  use/time-step.  The FisherBlock will be responsible for concatenating
-  (or doing whatever else it wants) with the resulting lists.
-
-  Args:
-    grads_list: 3-D list of Tensors. First index is for source, second is for
-      tower, third is for use/time-step.
-
-  Returns:
-    2-D tuple of PartitionedTensors. First index is for source, second is for
-    use/time-step.
-  """
-  num_uses = len(grads_list[0][0])
-  assert all(len(grad) == num_uses for grads in grads_list for grad in grads)
-  return tuple(tuple(utils.PartitionedTensor(grad)
-                     for grad in zip(*grads)) for grads in grads_list)
-
-
 def normalize_damping(damping, num_replications):
   """Normalize damping after adjusting scale by NORMALIZE_DAMPING_POWER."""
   if NORMALIZE_DAMPING_POWER:
@@ -662,7 +613,7 @@ class ConvDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
 
 
 class KroneckerProductFB(FisherBlock):
-  """A base class for FisherBlocks with separate input and output factors.
+  """A base class for blocks with separate input and output Kronecker factors.
 
   The Fisher block is approximated as a Kronecker product of the input and
   output factors.
@@ -783,67 +734,6 @@ class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
     self._setup_damping(damping)
 
 
-class EmbeddingKFACMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
-  """K-FAC FisherBlock for embedding layers used multiple times in the graph.
-
-  Similar to EmbeddingKFACFB except that this version supports multiple uses
-  of the parameter within a single model. These uses could correspond to
-  "time-steps", but they don't have to.
-
-  Does not support bias parameters.
-  """
-
-  def __init__(self, layer_collection, vocab_size):
-    """Creates a EmbeddingKFACMultiIndepFB block.
-
-    Args:
-      layer_collection: The collection of all layers in the K-FAC approximate
-          Fisher information matrix to which this FisherBlock belongs.
-      vocab_size: int. Size of vocabulary for this embedding layer.
-    """
-    self._vocab_size = vocab_size
-
-    super(EmbeddingKFACMultiIndepFB, self).__init__(layer_collection)
-
-  def instantiate_factors(self, grads_list, damping):
-    """Instantiate Kronecker Factors for this FisherBlock.
-
-    Args:
-      grads_list: List of list of list of Tensors. grads_list[i][j][k] is the
-        gradient of the loss with respect to 'outputs' from source 'i',
-        tower/mini-batch 'j', and use/time-step 'k'. Each Tensor has shape
-        [tower_minibatch_size, output_size].
-      damping: 0-D Tensor or float. 'damping' * identity is approximately added
-        to this FisherBlock's Fisher approximation.
-    """
-    inputs = self._inputs
-    self._num_uses = num_uses = len(inputs[0])
-
-    # Check that all mini-batches/towers have the same number of uses
-    assert all(len(input_) == num_uses for input_ in inputs)
-    # Do the same for grads_list
-    assert all(len(grad) == num_uses for grad in grads for grads in grads_list)
-    # Merge uses and towers/minibatches dimensions together so we can handle
-    # it using a non-multi factor.
-    inputs = nest.flatten(inputs)
-
-    # Note that we call the multi version of make_partitionedtensors only for
-    # grads_list here.
-    inputs = _make_partitionedtensors_inputs(inputs)
-    grads_list = _make_partitionedtensors_multi_grads(grads_list)
-
-    self._input_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.EmbeddingInputKroneckerFactor,
-        (inputs, self._vocab_size))
-    self._output_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedMultiKF, (grads_list,))
-    self._setup_damping(damping, normalization=num_uses)
-
-  @property
-  def _renorm_coeff(self):
-    return self._num_uses
-
-
 class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
   """K-FAC FisherBlock for fully-connected (dense) layers.
 
@@ -1232,7 +1122,70 @@ def num_conv_locations(input_shape, strides):
   return spatial_input_locations // spatial_strides_divisor
 
 
-class FullyConnectedMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
+class InputOutputMultiMinibatchMultiUse(InputOutputMultiMinibatch):
+  """Adds methods for multi-use/time-step case to InputOutputMultiMinibatch."""
+
+  def __init__(self, num_uses=None, *args, **kwargs):
+    self._num_uses = num_uses
+    super(InputOutputMultiMinibatchMultiUse, self).__init__(*args, **kwargs)
+
+  def _process_data(self, grads_list):
+    """Process temporal/multi-use data into a standard format."""
+
+    inputs = self._inputs
+
+    # The first possible data format is where inputs is a list of tensors,
+    # one for each use/time-step.
+    if isinstance(inputs[0], (list, tuple)):
+      # The first index is tower/minibatch, the second is use/time-step
+      num_uses = len(inputs[0])
+      if self._num_uses is not None and self._num_uses != num_uses:
+        raise ValueError("num_uses argument doesn't match length of inputs.")
+      else:
+        self._num_uses = num_uses
+
+      # Check that all mini-batches/towers have the same number of uses
+      if not all(len(input_) == num_uses for input_ in inputs):
+        raise ValueError("Length of inputs argument is inconsistent across "
+                         "mini-batches/towers.")
+      # Fold uses/time-step and towers/minibatches dimensions together
+      inputs = nest.flatten(inputs)
+
+      inputs = _make_partitionedtensors_inputs(inputs)
+    # If inputs is not a tuple then we assume that inputs is a tensor
+    # with 'uses' folded into the batch dimension. (And grads_list is a list
+    # across sources of such Tensors.)  This is the native format that the
+    # factor will take as arguments.
+
+    # Now we perform the analogous processing for grads_list
+    if isinstance(grads_list[0][0], (list, tuple)):
+      num_uses = len(grads_list[0][0])
+      if self._num_uses is not None and self._num_uses != num_uses:
+        raise ValueError("num_uses argument doesn't match length of outputs, "
+                         "or length of outputs is inconsistent with length of "
+                         "inputs.")
+      else:
+        self._num_uses = num_uses
+
+      if not all(len(grad) == num_uses for grads in grads_list
+                 for grad in grads):
+        raise ValueError("Length of outputs argument is inconsistent across "
+                         "mini-batches/towers.")
+
+      grads_list = tuple(nest.flatten(grads) for grads in grads_list)
+      grads_list = _make_partitionedtensors_grads(grads_list)
+
+    if self._num_uses is None:
+      raise ValueError("You must supply a value for the num_uses argument if "
+                       "the number of uses cannot be inferred from inputs or "
+                       "outputs arguments (e.g. if they are both given in the "
+                       "single Tensor format, instead of as lists of Tensors.")
+
+    return inputs, grads_list
+
+
+class FullyConnectedMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+                                 KroneckerProductFB):
   """FisherBlock for fully-connected layers that share parameters.
 
   This class implements the "independence across time" approximation from the
@@ -1240,42 +1193,43 @@ class FullyConnectedMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
     https://openreview.net/pdf?id=HyMTkQZAb
   """
 
-  def __init__(self, layer_collection, has_bias=False):
+  def __init__(self, layer_collection, has_bias=False, num_uses=None):
     """Creates a FullyConnectedMultiIndepFB block.
 
     Args:
       layer_collection: LayerCollection instance.
       has_bias: bool. If True, estimates Fisher with respect to a bias
         parameter as well as the layer's parameters.
+      num_uses: int or None. Number of uses of the layer in the model's graph.
+        Only required if the data is formatted with uses/time folded into the
+        batch dimension (instead of uses/time being a list dimension).
+        (Default: None)
     """
     self._has_bias = has_bias
 
-    super(FullyConnectedMultiIndepFB, self).__init__(layer_collection)
+    super(FullyConnectedMultiIndepFB, self).__init__(
+        layer_collection=layer_collection,
+        num_uses=num_uses)
 
   def instantiate_factors(self, grads_list, damping):
-
-    self._num_uses = float(len(self._inputs[0]))
-    inputs = _make_partitionedtensors_multi_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_multi_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedMultiKF,
-        ((inputs,), self._has_bias))
+        ((inputs,), self._num_uses, self._has_bias))
 
     self._output_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedMultiKF, (grads_list,))
+        fisher_factors.FullyConnectedMultiKF, (grads_list, self._num_uses))
 
     self._setup_damping(damping, normalization=self._num_uses)
 
   @property
   def _renorm_coeff(self):
-    return self._num_uses
-
-  def tensors_to_compute_grads(self):
-    return self._outputs
+    return float(self._num_uses)
 
 
-class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
+class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+                               KroneckerProductFB):
   """FisherBlock for 2D convolutional layers using the basic KFC approx.
 
   Similar to ConvKFCBasicFB except that this version supports multiple
@@ -1291,7 +1245,8 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
                strides=None,
                dilation_rate=None,
                data_format=None,
-               extract_patches_fn=None):
+               extract_patches_fn=None,
+               num_uses=None):
     """Creates a ConvKFCBasicMultiIndepFB block.
 
     Args:
@@ -1312,6 +1267,10 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
       extract_patches_fn: str or None. Name of function that extracts image
         patches. One of "extract_convolution_patches", "extract_image_patches",
         "extract_pointwise_conv2d_patches".
+      num_uses: int or None. Number of uses of the layer in the model's graph.
+        Only required if the data is formatted with uses/time folded into the
+        batch dimension (instead of uses/time being a list dimension).
+        (Default: None)
     """
     self._padding = padding
     self._strides = maybe_tuple(strides)
@@ -1323,28 +1282,16 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
     fltr = params[0] if self._has_bias else params
     self._filter_shape = tuple(fltr.shape.as_list())
 
-    super(ConvKFCBasicMultiIndepFB, self).__init__(layer_collection)
+    super(ConvKFCBasicMultiIndepFB, self).__init__(
+        layer_collection=layer_collection,
+        num_uses=num_uses)
 
   def instantiate_factors(self, grads_list, damping):
-    # Infer number of locations upon which convolution is applied.
-    self._num_locations = num_locations = num_conv_locations(
-        self._inputs[0][0].shape.as_list(), self._strides)
-
-    # The first index is tower/minibatch, the second is use/time-step
-    inputs = self._inputs
-    self._num_uses = num_uses = len(inputs[0])
-
-    # Check that all mini-batches/towers have the same number of uses
-    assert all(len(input_) == num_uses for input_ in inputs)
-    assert all(len(grad) == num_uses for grads in grads_list for grad in grads)
-
-    # Fold uses/time-step and towers/minibatches dimensions together
-    inputs = nest.flatten(inputs)
-    # And do the same for grads_list
-    grads_list = tuple(nest.flatten(grads) for grads in grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
-    inputs = _make_partitionedtensors_inputs(inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    # Infer number of locations upon which convolution is applied.
+    self._num_locations = num_conv_locations(inputs.shape.as_list(),
+                                             self._strides)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvInputKroneckerFactor,
@@ -1354,20 +1301,75 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatch, KroneckerProductFB):
     self._output_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvOutputKroneckerFactor, (grads_list,))
 
-    self._setup_damping(damping, normalization=(num_locations * num_uses))
+    self._setup_damping(damping, normalization=
+                        (self._num_locations * self._num_uses))
 
   @property
   def _renorm_coeff(self):
     return self._num_locations * self._num_uses
 
 
+class EmbeddingKFACMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+                                KroneckerProductFB):
+  """K-FAC FisherBlock for embedding layers used multiple times in the graph.
+
+  Similar to EmbeddingKFACFB except that this version supports multiple uses
+  of the parameter within a single model. These uses could correspond to time
+  steps in an RNN architecture, but they don't have to.
+
+  Does not support bias parameters.
+  """
+
+  def __init__(self, layer_collection, vocab_size, num_uses):
+    """Creates a EmbeddingKFACMultiIndepFB block.
+
+    Args:
+      layer_collection: The collection of all layers in the K-FAC approximate
+          Fisher information matrix to which this FisherBlock belongs.
+      vocab_size: int. Size of vocabulary for this embedding layer.
+      num_uses: int or None. Number of uses of the layer in the model's graph.
+        Only required if the data is formatted with time folded into the batch
+        dimension (instead of time being a list dimension). (Default: None)
+    """
+    self._vocab_size = vocab_size
+
+    super(EmbeddingKFACMultiIndepFB, self).__init__(
+        layer_collection=layer_collection,
+        num_uses=num_uses)
+
+  def instantiate_factors(self, grads_list, damping):
+    """Instantiate Kronecker Factors for this FisherBlock.
+
+    Args:
+      grads_list: List of list of list of Tensors. grads_list[i][j][k] is the
+        gradient of the loss with respect to 'outputs' from source 'i',
+        tower/mini-batch 'j', and use/time-step 'k'. Each Tensor has shape
+        [tower_minibatch_size, output_size].
+      damping: 0-D Tensor or float. 'damping' * identity is approximately added
+        to this FisherBlock's Fisher approximation.
+    """
+    inputs, grads_list = self._process_data(grads_list)
+
+    self._input_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.EmbeddingInputKroneckerFactor,
+        (inputs, self._vocab_size))
+    self._output_factor = self._layer_collection.make_or_get_factor(
+        fisher_factors.FullyConnectedMultiKF, (grads_list, self._num_uses))
+    self._setup_damping(damping, normalization=self._num_uses)
+
+  @property
+  def _renorm_coeff(self):
+    return float(self._num_uses)
+
+
 class SeriesFBApproximation(enum.IntEnum):
   """See FullyConnectedSeriesFB.__init__ for description and usage."""
   option1 = 1
   option2 = 2
 
 
-class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
+class FullyConnectedSeriesFB(InputOutputMultiMinibatchMultiUse,
+                             KroneckerProductFB):
   """FisherBlock for fully-connected layers that share parameters across time.
 
   This class implements the "Option 1" and "Option 2" approximation from the
@@ -1383,6 +1385,7 @@ class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
   def __init__(self,
                layer_collection,
                has_bias=False,
+               num_uses=None,
                option=SeriesFBApproximation.option2):
     """Constructs a new `FullyConnectedSeriesFB`.
 
@@ -1390,6 +1393,10 @@ class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
       layer_collection: The collection of all layers in the K-FAC approximate
         Fisher information matrix to which this FisherBlock belongs.
       has_bias: Whether the layer includes a bias parameter.
+      num_uses: int or None. Number of time-steps over which the layer
+        is used. Only required if the data is formatted with time folded into
+        the batch dimension (instead of time being a list dimension).
+        (Default: None)
       option: A `SeriesFBApproximation` specifying the simplifying assumption
         to be used in this block. `option1` approximates the cross-covariance
         over time as a symmetric matrix, while `option2` makes
@@ -1400,39 +1407,33 @@ class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
     self._has_bias = has_bias
     self._option = option
 
-    super(FullyConnectedSeriesFB, self).__init__(layer_collection)
+    super(FullyConnectedSeriesFB, self).__init__(
+        layer_collection=layer_collection,
+        num_uses=num_uses)
 
-  def instantiate_factors(self, grads_list, damping):
+  @property
+  def _num_timesteps(self):
+    return self._num_uses
 
-    self._num_timesteps = len(self._inputs[0])
-    assert len(grads_list[0][0]) == self._num_timesteps
+  @property
+  def _renorm_coeff(self):
+    # This should no longer be used since the multiply_X functions from the base
+    # class have been overridden
+    assert False
 
-    inputs = _make_partitionedtensors_multi_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_multi_grads(grads_list)
+  def instantiate_factors(self, grads_list, damping):
+    inputs, grads_list = self._process_data(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedMultiKF, ((inputs,), self._has_bias))
+        fisher_factors.FullyConnectedMultiKF,
+        ((inputs,), self._num_uses, self._has_bias))
     self._input_factor.register_cov_dt1()
 
     self._output_factor = self._layer_collection.make_or_get_factor(
-        fisher_factors.FullyConnectedMultiKF, (grads_list,))
+        fisher_factors.FullyConnectedMultiKF, (grads_list, self._num_uses))
     self._output_factor.register_cov_dt1()
 
-    def compute_damping():
-      normalized_damping = normalize_damping(damping, self._num_timesteps)
-      return compute_pi_adjusted_damping(self._input_factor.get_cov(),
-                                         self._output_factor.get_cov(),
-                                         normalized_damping**0.5)
-
-    damping_id = ("compute_pi_adjusted_damping",
-                  "cov", self._input_factor.name,
-                  "cov", self._output_factor.name,
-                  "normalize_damping",
-                  damping, self._num_timesteps, "power", 0.5)
-    self._input_damping_func = _package_func(lambda: compute_damping()[0],
-                                             damping_id + ("ref", 0))
-    self._output_damping_func = _package_func(lambda: compute_damping()[1],
-                                              damping_id + ("ref", 1))
+    self._setup_damping(damping, normalization=self._num_uses)
 
   def register_matpower(self, exp):
     if exp != -1:
@@ -1562,6 +1563,3 @@ class FullyConnectedSeriesFB(InputOutputMultiMinibatch, FisherBlock):
     return utils.mat2d_to_layer_params(vector, Z)
 
     # pylint: enable=invalid-name
-
-  def tensors_to_compute_grads(self):
-    return self._outputs
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index 6fc163e232..f521363536 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -35,7 +35,6 @@ from tensorflow.python.ops import special_math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import moving_averages
-from tensorflow.python.util import nest
 
 # Whether to initialize covariance estimators at a zero matrix (or the identity
 # matrix).
@@ -1227,27 +1226,24 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor):
     return compute_cov(reshaped_tensor)
 
 
-class FullyConnectedMultiKF(InverseProvidingFactor):
+class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
   """Kronecker factor for a fully connected layer used multiple times."""
 
   def __init__(self,
-               tensor_lists,
+               tensors,
+               num_uses=None,
                has_bias=False):
     """Constructs a new `FullyConnectedMultiKF`.
 
     Args:
-      tensor_lists: 2D array (list of lists) of Tensors of shape
-        [batch_size, n]. Each of these tensors is usually a layer's inputs or
-        its output's gradients. The first dimension of the array is the source,
-        and the second is the use in the graph (which is sometimes a
-        "time-step").
+      tensors: List of Tensors of shape, each of shape [batch_size, n]. Each of
+        these tensors is usually a layer's inputs or its output's gradients.
+        The list is over sources.
+      num_uses: int. The number of time-steps / uses.
       has_bias: bool. If True, '1' is appended to each row.
     """
 
-    self._tensor_lists = tensor_lists
-    self._has_bias = has_bias
-    self._num_timesteps = len(tensor_lists[0])
-    self._tensors = [None] * len(tensor_lists)
+    self._num_uses = num_uses
 
     self._cov_dt1 = None
     self._make_cov_dt1 = False
@@ -1256,20 +1252,17 @@ class FullyConnectedMultiKF(InverseProvidingFactor):
     self._option1quants_registrations = set()
     self._option2quants_registrations = set()
 
-    super(FullyConnectedMultiKF, self).__init__()
-
-  @property
-  def _var_scope(self):
-    return "ff_fc_multi_" + scope_string_from_params(
-        tuple(nest.flatten(self._tensor_lists)) + (self._has_bias,))
+    super(FullyConnectedMultiKF, self).__init__(tensors=tensors,
+                                                has_bias=has_bias)
 
   @property
-  def _num_sources(self):
-    return len(self._tensor_lists)
+  def _num_timesteps(self):
+    return self._num_uses
 
   @property
-  def _dtype(self):
-    return self._tensor_lists[0][0].dtype
+  def _var_scope(self):
+    return "ff_fc_multi_" + scope_string_from_params(
+        tuple(self._tensors) + (self._num_timesteps, self._has_bias,))
 
   def make_covariance_update_op(self, ema_decay):
 
@@ -1291,36 +1284,28 @@ class FullyConnectedMultiKF(InverseProvidingFactor):
 
     return op
 
-  def _compute_new_cov(self, idx=0):
-    # Concatenate across time/replications
-    tensor = array_ops.concat(self._tensor_lists[idx], 0)
+  def _compute_new_cov_dt1(self, idx=0):  # pylint: disable=missing-docstring
+    tensor = self._tensors[idx]
     if self._has_bias:
+      # This appending is technically done twice (the other time is for
+      # _compute_new_cov())
       tensor = append_homog(tensor)
-    # We save these so they can be used by _compute_new_cov_dt1
-    self._tensors[idx] = tensor
-    return compute_cov(tensor)
 
-  def _compute_new_cov_dt1(self, idx=0):  # pylint: disable=missing-docstring
-    tensor = self._tensors[idx]
-    batch_size = array_ops.shape(self._tensor_lists[idx][0])[0]
-    # Is there a more elegant way to do this computation?
+    total_len = array_ops.shape(tensor)[0]
+    batch_size = total_len // self._num_timesteps
+
     tensor_present = tensor[:-batch_size, :]
     tensor_future = tensor[batch_size:, :]
+
     # We specify a normalizer for this computation to ensure a PSD Fisher
     # block estimate.  This is equivalent to padding with zeros, as was done
     # in Section B.2 of the appendix.
-    normalizer = self._num_timesteps * batch_size
     return compute_cov(
-        tensor_future, tensor_right=tensor_present, normalizer=normalizer)
-
-  @property
-  def _cov_shape(self):
-    size = self._tensor_lists[0][0].shape[1] + self._has_bias
-    return [size, size]
+        tensor_future, tensor_right=tensor_present, normalizer=total_len)
 
   @property
   def _vec_shape(self):
-    size = self._tensor_lists[0][0].shape[1] + self._has_bias
+    size = self._tensors[0].shape[1] + self._has_bias
     return [size]
 
   def get_option1quants(self, damping_func):
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 00eae8b399..7727c607db 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -572,13 +572,15 @@ class LayerCollection(object):
       params: Embedding matrix of shape [vocab_size, embedding_size].
       inputs: Tensor of shape [batch_size, input_size] and dtype int32. Indices
         into embedding matrix.
-      outputs: Tensor of shape [batch_size, output_size]. Outputs
+      outputs: Tensor of shape [batch_size, embedding_size]. Outputs
         produced by layer.
       approx: str or None. If not None must be "kron".  The Fisher
         approximation to use. If None the default value is used. (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -616,9 +618,11 @@ class LayerCollection(object):
       approx: str or None. If not None must be one of "kron" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -665,9 +669,11 @@ class LayerCollection(object):
       approx: str or None. If not None must be one of "kron" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -743,9 +749,11 @@ class LayerCollection(object):
       approx: str or None. If not None must be one of "kron" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -796,9 +804,11 @@ class LayerCollection(object):
       data_format: str or None. Format of data.
       approx: str or None. If not None must "diagonal".  The Fisher
         approximation to use. If None the default value is used. (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -862,9 +872,11 @@ class LayerCollection(object):
       approx: str or None. If not None must be one of "kron" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'inputs' and 'outputs' as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -905,9 +917,10 @@ class LayerCollection(object):
       approx: str or None. It not None, must be one of "full" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str. If True, this adds 'batch_size' to the total
+        mini-batch size use when estimating the Fisher block for this layer
+        (which must have already been registered). If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse. (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -924,7 +937,8 @@ class LayerCollection(object):
     self._add_uses(params, float("inf"))
 
   def register_fully_connected_multi(self, params, inputs, outputs,
-                                     approx=None, reuse=VARIABLE_SCOPE):
+                                     num_uses=None, approx=None,
+                                     reuse=VARIABLE_SCOPE):
     """Register fully connected layers with shared parameters.
 
     This can handle general fully-connected layers with shared parameters, but
@@ -935,19 +949,31 @@ class LayerCollection(object):
       params: Tensor or 2-tuple of Tensors corresponding to weight and bias of
         this layer. Weight matrix should have shape [input_size, output_size].
         Bias should have shape [output_size].
-      inputs: A list of tensors, each of shape [batch_size, input_size]. Inputs
+      inputs: A list of Tensors, each of shape [batch_size, input_size]. Inputs
         to layer. The list indexes each use in the graph (which might
-        correspond to a "time-step" in an RNN).
-      outputs: A list of tensors, the same length as 'inputs', each of shape
+        correspond to a "time-step" in an RNN). OR, can be single Tensor, of
+        shape [batch_size * num_uses, input_size], which is a reshaped version
+        of a Tensor of shape [batch_size, num_uses, input_size].
+      outputs: A list of Tensors, the same length as 'inputs', each of shape
         [batch_size, output_size]. Outputs produced by layer. The list indexes
         each use in the graph (which might correspond to a "time-step" in an
-        RNN). Needs to correspond with the order used in 'inputs'.
+        RNN). Needs to correspond with the order used in 'inputs'.  OR, can be
+        a single Tensor of shape [batch_size * num_uses, output_size], which is
+        a reshaped version of a Tensor of shape [batch_size, num_uses,
+        output_size].
+      num_uses: int or None. The number uses/time-steps in the graph where the
+        layer appears. Only needed if both inputs and outputs are given in the
+        single Tensor format. (Default: None)
       approx: str or None. If not None, must be of "kron_indep", "kron_series_1"
         or "kron_series_2". The Fisher approximation to use. If None the default
         value is used. (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds inputs and outputs as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.  (Note that the
+        word 'use' here has a completely different meaning to "use in the graph"
+        as it perturns to the 'inputs', 'outputs', and 'num_uses' arguments.)
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -960,7 +986,8 @@ class LayerCollection(object):
     # should be added back in here (and for the other block types, arguably).
 
     has_bias = isinstance(params, (tuple, list))
-    block = self.register_block(params, block_type(self, has_bias=has_bias),
+    block = self.register_block(params, block_type(self, has_bias=has_bias,
+                                                   num_uses=num_uses),
                                 reuse=reuse)
     block.register_additional_minibatch(inputs, outputs)
 
@@ -973,6 +1000,7 @@ class LayerCollection(object):
                             padding,
                             inputs,
                             outputs,
+                            num_uses=None,
                             data_format=None,
                             dilations=None,
                             approx=None,
@@ -988,19 +1016,32 @@ class LayerCollection(object):
       padding: string. see tf.nn.conv2d for valid values.
       inputs: A list of Tensors, each of shape [batch_size, height, width,
         in_channels]. Inputs to layer. The list indexes each use in the graph
-        (which might correspond to a "time-step" in an RNN).
+        (which might correspond to a "time-step" in an RNN). OR, can be single
+        Tensor, of shape [batch_size * num_uses, height, width, in_channels],
+        which is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        height, width, in_channels].
       outputs: A list of Tensors, each of shape [batch_size, height, width,
         out_channels]. Output produced by layer. The list indexes each use
         in the graph (which might correspond to a "time-step" in an RNN).
-        Needs to correspond with the order used in 'inputs'.
+        Needs to correspond with the order used in 'inputs'.  OR, can be a
+        single Tensor, of shape [batch_size*num_uses, height, width,
+        out_channels], which is a reshaped version of a Tensor of shape
+        [batch_size, num_uses, height, width, out_channels].
+      num_uses: int or None. The number uses/time-steps in the graph where the
+        layer appears. Only needed if both inputs and outputs are given in the
+        single Tensor format. (Default: None)
       data_format: str or None. Format of data.
       dilations: List of 4 ints. Dilations along each dimension.
       approx: str or None. If not None must by "kron_indep". The Fisher
         approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds inputs and outputs as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.  (Note that the
+        word 'use' here has a completely different meaning to "use in the graph"
+        as it perturns to the 'inputs', 'outputs', and 'num_uses' arguments.)
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -1020,7 +1061,8 @@ class LayerCollection(object):
             strides=strides,
             data_format=data_format,
             dilation_rate=dilations,
-            extract_patches_fn="extract_image_patches"),
+            extract_patches_fn="extract_image_patches",
+            num_uses=num_uses),
         reuse=reuse)
 
     block.register_additional_minibatch(inputs, outputs)
@@ -1036,6 +1078,7 @@ class LayerCollection(object):
                                params,
                                inputs,
                                outputs,
+                               num_uses=None,
                                approx=None,
                                reuse=VARIABLE_SCOPE):
     """Registers embedding layers with shared parameters.
@@ -1045,16 +1088,29 @@ class LayerCollection(object):
       inputs: A list of Tensors, each of shape [batch_size, input_size] and
         dtype int32. Indices into embedding matrix. The list indexes each use
         in the graph (which might correspond to a "time-step" in an RNN).
-      outputs: A list of Tensors, each of shape [batch_size, output_size].
+        OR, can be single Tensor, of shape [batch_size * num_uses, input_size],
+        which is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        input_size].
+      outputs: A list of Tensors, each of shape [batch_size, embedding_size].
         Outputs produced by layer. The list indexes each use in the graph
         (which might correspond to a "time-step" in an RNN). Needs to
-        correspond with the order used in 'inputs'.
+        correspond with the order used in 'inputs'. OR, can be a
+        single Tensor, of shape [batch_size*num_uses, embedding_size], which
+        is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        embedding_size].
+      num_uses: int or None. The number uses/time-steps in the graph where the
+        layer appears. Only needed if both inputs and outputs are given in the
+        single Tensor format. (Default: None)
       approx: str or None. If not None must by "kron_indep". The Fisher
         approximation to use. If None the default value is used.
         (Default: None)
-      reuse: bool or str.  If True, reuse an existing FisherBlock. If False,
-        create a new FisherBlock.  If "VARIABLE_SCOPE", use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds inputs and outputs as an
+        additional mini-batch/tower of data to use when estimating the Fisher
+        block for this layer (which must have already been registered). If
+        "VARIABLE_SCOPE", use tf.get_variable_scope().reuse.  (Note that the
+        word 'use' here has a completely different meaning to "use in the graph"
+        as it perturns to the 'inputs', 'outputs', and 'num_uses' arguments.)
+        (Default: "VARIABLE_SCOPE")
 
     Raises:
       ValueError: For improper value to 'approx'.
@@ -1070,7 +1126,7 @@ class LayerCollection(object):
     vocab_size = int(params.shape[0])
 
     block = self.register_block(
-        params, block_type(self, vocab_size), reuse=reuse)
+        params, block_type(self, vocab_size, num_uses=num_uses), reuse=reuse)
     block.register_additional_minibatch(inputs, outputs)
 
     self._add_uses(params, len(inputs))
@@ -1093,9 +1149,10 @@ class LayerCollection(object):
         (Default: None)
       name: (OPTIONAL) str or None. Unique name for this loss function. If None,
         a new name is generated. (Default: None)
-      reuse: (OPTIONAL) bool or str.  If True, reuse an existing FisherBlock.
-        If False, create a new FisherBlock.  If VARIABLE_SCOPE, use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'logits' as an additional
+        mini-batch/tower of inputs to the loss-function/predictive distribution
+        (which must have already been registered). If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse. (Default: "VARIABLE_SCOPE")
     """
     loss = lf.CategoricalLogitsNegativeLogProbLoss(logits, targets=targets,
                                                    seed=seed)
@@ -1126,9 +1183,10 @@ class LayerCollection(object):
         (Default: None)
       name: (OPTIONAL) str or None. Unique name for this loss function. If None,
         a new name is generated. (Default: None)
-      reuse: (OPTIONAL) bool or str.  If True, reuse an existing FisherBlock.
-        If False, create a new FisherBlock.  If VARIABLE_SCOPE, use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'mean' and 'var' as an additional
+        mini-batch/tower of inputs to the loss-function/predictive distribution
+        (which must have already been registered). If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse. (Default: "VARIABLE_SCOPE")
     """
     loss = lf.NormalMeanNegativeLogProbLoss(mean, var, targets=targets,
                                             seed=seed)
@@ -1154,9 +1212,10 @@ class LayerCollection(object):
         (Default: None)
       name: (OPTIONAL) str or None. Unique name for this loss function. If None,
         a new name is generated. (Default: None)
-      reuse: (OPTIONAL) bool or str.  If True, reuse an existing FisherBlock.
-        If False, create a new FisherBlock.  If VARIABLE_SCOPE, use
-        tf.get_variable_scope().reuse.
+      reuse: bool or str.  If True, this adds 'logits' as an additional
+        mini-batch/tower of inputs to the loss-function/predictive distribution
+        (which must have already been registered). If "VARIABLE_SCOPE", use
+        tf.get_variable_scope().reuse. (Default: "VARIABLE_SCOPE")
     """
     loss = lf.MultiBernoulliNegativeLogProbLoss(logits, targets=targets,
                                                 seed=seed)
diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py
index c589b18193..c9de0c7270 100644
--- a/tensorflow/contrib/kfac/python/ops/utils.py
+++ b/tensorflow/contrib/kfac/python/ops/utils.py
@@ -667,6 +667,9 @@ class PartitionedTensor(object):
   def __ne__(self, other):
     return not self == other  # pylint: disable=g-comparison-negation
 
+  def __getitem__(self, key):
+    return self.as_tensor()[key]
+
   def as_tensor(self, dtype=None, name=None, as_ref=False):
     with ops.name_scope(name, "PartitionedTensor.as_tensor", self.tensors):
       assert not as_ref
-- 
GitLab


From 9183c33884c4492589f2d8648178d00645c30691 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 20 Mar 2018 03:48:38 -0700
Subject: [PATCH 233/960] Don't spin in a loop when we're not waiting on any
 GPU events.

PiperOrigin-RevId: 189719711
---
 .../core/common_runtime/gpu/gpu_event_mgr.cc  | 53 ++++++++++---------
 .../core/common_runtime/gpu/gpu_event_mgr.h   |  3 +-
 tensorflow/core/protobuf/config.proto         |  4 +-
 3 files changed, 31 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
index 2452efc779..af6a59a85d 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.cc
@@ -30,10 +30,6 @@ EventMgr::EventMgr(gpu::StreamExecutor* se, const GPUOptions& gpu_options)
       polling_active_delay_usecs_(gpu_options.polling_active_delay_usecs()
                                       ? gpu_options.polling_active_delay_usecs()
                                       : 10),
-      polling_inactive_delay_msecs_(
-          gpu_options.polling_inactive_delay_msecs()
-              ? gpu_options.polling_inactive_delay_msecs()
-              : 1),
       accumulated_stream_(nullptr),
       accumulated_tensors_(new TensorReferenceVector),
       accumulated_tensor_bytes_(0),
@@ -78,16 +74,22 @@ EventMgr::~EventMgr() {
 
 void EventMgr::StartPollingLoop() {
   CHECK(polling_stopped_ == nullptr);
-  stop_polling_.reset(new Notification);
+  {
+    mutex_lock l(mu_);
+    stop_polling_ = false;
+  }
   polling_stopped_.reset(new Notification);
   threadpool_.Schedule([this]() { PollLoop(); });
 }
 
 void EventMgr::StopPollingLoop() {
-  if (stop_polling_) {
-    stop_polling_->Notify();
+  if (polling_stopped_) {
+    {
+      mutex_lock l(mu_);
+      stop_polling_ = true;
+      events_pending_.notify_all();
+    }
     polling_stopped_->WaitForNotification();
-    stop_polling_.reset(nullptr);
     polling_stopped_.reset(nullptr);
   }
 }
@@ -121,28 +123,31 @@ void EventMgr::FlushAccumulatedTensors() {
   accumulated_stream_ = nullptr;
 }
 
-// A polling loop to detect completion of GPU events.  There's a
-// tradeoff between achieving low latency detection, which argues for
-// little delay between calls, and minimizing CPU use and lock
-// contention, which argue for longer delay.  The current strategy is
-// to poll frequently when the queue is non-empty, and infrequently
-// otherwise.
+// A polling loop to detect completion of GPU events.
+//
+// While one or more events is outstanding, poll for completed events.  When no
+// events are outstanding, we sleep until one is enqueued.
 void EventMgr::PollLoop() {
-  bool queue_empty = false;
-  while (!stop_polling_->HasBeenNotified()) {
-    if (queue_empty) {
-      mutex_lock l(mu_);
-      WaitForMilliseconds(&l, &events_pending_, polling_inactive_delay_msecs_);
-    } else {
-      Env::Default()->SleepForMicroseconds(polling_active_delay_usecs_);
-    }
-    ToFreeVector to_free;
+  ToFreeVector to_free;
+  while (true) {
+    bool events_still_pending;
     {
       mutex_lock l(mu_);
+      if (stop_polling_) {
+        break;
+      }
+      if (used_events_.empty()) {
+        events_pending_.wait(l);
+      }
       PollEvents(true, &to_free);
-      queue_empty = used_events_.empty();
+      events_still_pending = !used_events_.empty();
     }
     FreeMemory(to_free);
+    to_free.clear();
+
+    if (events_still_pending) {
+      Env::Default()->SleepForMicroseconds(polling_active_delay_usecs_);
+    }
   }
   polling_stopped_->Notify();
 }
diff --git a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
index 9692b24084..d23898e1f2 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_event_mgr.h
@@ -94,7 +94,6 @@ class EventMgr {
   perftools::gputools::StreamExecutor* const exec_;
   const int64 deferred_bytes_threshold_;
   const int32 polling_active_delay_usecs_;
-  const int32 polling_inactive_delay_msecs_;
   mutex mu_;
   condition_variable events_pending_ GUARDED_BY(mu_);
 
@@ -180,7 +179,7 @@ class EventMgr {
   // A FIFO queue of InUse events and associated tensors.
   std::deque<InUse> used_events_ GUARDED_BY(mu_);
 
-  std::unique_ptr<Notification> stop_polling_;
+  bool stop_polling_ GUARDED_BY(mu_);
   std::unique_ptr<Notification> polling_stopped_;
 
   // The main PollLoop for the event manager runs in this threadpool.
diff --git a/tensorflow/core/protobuf/config.proto b/tensorflow/core/protobuf/config.proto
index abbbe392aa..a3557e4721 100644
--- a/tensorflow/core/protobuf/config.proto
+++ b/tensorflow/core/protobuf/config.proto
@@ -67,9 +67,7 @@ message GPUOptions {
   // set or set to 0, gets set to a non-zero default.
   int32 polling_active_delay_usecs = 6;
 
-  // In the event polling loop sleep this many millisconds between
-  // PollEvents calls, when the queue is empty.  If value is not
-  // set or set to 0, gets set to a non-zero default.
+  // This field is deprecated and ignored.
   int32 polling_inactive_delay_msecs = 7;
 
   // Force all tensors to be gpu_compatible. On a GPU-enabled TensorFlow,
-- 
GitLab


From 198bf16225b3224e9af6bafd9f1b4c1433557281 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Tue, 20 Mar 2018 13:18:13 +0200
Subject: [PATCH 234/960] Fix dataset resampling bug introduced by a bug in
 datasets itself.

Fixes github issue #16606.

The core issue is that in the case of certain random Tensors, the
following two lines aren't the same:

```
rand_0s_and_1s_ds = ...
gather_ds = rand_0s_and_1s_ds.map(lambda i: tf.gather([0, 1], i))
tup_ds = tf.data.Dataset.zip(gather_ds, rand_0s_and_1s_ds)

```

```
rand_0s_and_1s_ds = ...
tup_ds = rand_0s_and_1s_ds.map(lambda i: (tf.gather([0, 1], i), i))
```

Note that this does NOT fix the underlying issue of drawing multiple
sampes from the underlying distribution.

Tested:
With the new test, `bazel test :resample_test` fails before and succeeds
after.
---
 .../data/python/kernel_tests/resample_test.py | 36 +++++++++++++++++++
 .../contrib/data/python/ops/resampling.py     | 10 +++---
 2 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index c16207fa48..a76c6b1e39 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -21,8 +21,11 @@ import numpy as np
 
 from tensorflow.contrib.data.python.ops import dataset_ops
 from tensorflow.contrib.data.python.ops import resampling
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -68,6 +71,39 @@ class ResampleTest(test.TestCase):
     returned_dist = class_counts / total_returned
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
+  def testRandomClasses(self):
+      init_dist = [0.25, 0.25, 0.25, 0.25]
+      target_dist = [0.0, 0.0, 0.0, 1.0]
+      num_classes = len(init_dist)
+      num_samples = 100   # We don't need many samples to test a dirac-delta target distribution
+      data_np = np.random.choice(num_classes, num_samples, p=init_dist)
+
+      dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
+
+      # Apply a random mapping that preserves the data distribution.
+      def _remap_fn(_):
+          return math_ops.cast(random_ops.random_uniform([1]) * num_classes, dtypes.int32)[0]
+      dataset = dataset.map(_remap_fn)
+
+      # Reshape distribution.
+      dataset = dataset.apply(
+          resampling.rejection_resample(
+              class_func=lambda x: x,
+              target_dist=target_dist,
+              initial_dist=init_dist))
+
+      get_next = dataset.make_one_shot_iterator().get_next()
+
+      with self.test_session() as sess:
+          returned = []
+          with self.assertRaises(errors.OutOfRangeError):
+              while True:
+                  returned.append(sess.run(get_next))
+
+      classes, _ = zip(*returned)
+      bincount = np.bincount(np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
+
+      self.assertAllClose(target_dist, bincount, atol=1e-2)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index 56f526a330..c99cdd3ff2 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -101,11 +101,11 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
                                                    initial_dist_ds))
                           .map(maybe_warn_on_large_rejection))
 
-    current_probabilities_ds = dataset_ops.Dataset.zip(
-        (acceptance_dist_ds, class_values_ds)).map(array_ops.gather)
-    filtered_ds = (
-        dataset_ops.Dataset.zip((class_values_ds, current_probabilities_ds,
-                                 dataset))
+    def _gather_and_copy(class_val, acceptance_prob, data):
+        return (class_val, array_ops.gather(acceptance_prob, class_val), data)
+    current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
+        (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
+    filtered_ds = (current_probabilities_and_class_and_data_ds\
         .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
-- 
GitLab


From 87b715325a74d34f1331d14d8df640308ec10d12 Mon Sep 17 00:00:00 2001
From: Brian Patton <bjp@google.com>
Date: Tue, 20 Mar 2018 06:25:19 -0700
Subject: [PATCH 235/960] Adds float64 support for avg pool and its gradient.

Eigen NumTraits is modified to directly use std::numeric_limits, which resolves a broken test caused by inconsistency between the host and devices values of Eigen::NumTraits<double>::highest(). This returns +inf on device, due to third_party/eigen3/Eigen/src/Core/util/Meta.h, and __DBL_MAX__ (1.7976931348623157e+308) on host, making the behavior for doubles (on device) inconsistent with both the behavior of floats Eigen::NumTraits<float>::highest() and the behavior of std::numeric_limits<double>::max()

PiperOrigin-RevId: 189731521
---
 tensorflow/core/kernels/avgpooling_op.cc      | 20 +++++++++++++++++++
 .../core/kernels/avgpooling_op_gpu.cu.cc      |  7 +++++++
 tensorflow/core/kernels/eigen_pooling.h       | 11 +++++-----
 .../python/kernel_tests/pooling_ops_test.py   |  7 +++++--
 4 files changed, 37 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/avgpooling_op.cc b/tensorflow/core/kernels/avgpooling_op.cc
index ec9cbc2a9b..a763f1321f 100644
--- a/tensorflow/core/kernels/avgpooling_op.cc
+++ b/tensorflow/core/kernels/avgpooling_op.cc
@@ -102,6 +102,9 @@ class AvgPoolingOp : public UnaryOp<T> {
   TensorFormat data_format_;
 };
 
+REGISTER_KERNEL_BUILDER(
+    Name("AvgPool").Device(DEVICE_CPU).TypeConstraint<double>("T"),
+    AvgPoolingOp<CPUDevice, double>);
 REGISTER_KERNEL_BUILDER(
     Name("AvgPool").Device(DEVICE_CPU).TypeConstraint<float>("T"),
     AvgPoolingOp<CPUDevice, float>);
@@ -153,11 +156,13 @@ class AvgPoolingOp<GPUDevice, T> : public UnaryOp<T> {
     TensorShape output_shape = params.forward_output_shape();
 
     if (data_format_ == FORMAT_NCHW) {
+      LOG(INFO) << "DnnPoolingOp";
       DnnPoolingOp<T>::Compute(
           context, perftools::gputools::dnn::PoolingMode::kAverage, ksize_,
           stride_, padding_, data_format_, tensor_in, output_shape,
           /*propagate_nans=*/false);
     } else {
+      LOG(INFO) << "SpatialAvgPooling";
       Tensor* output = nullptr;
       OP_REQUIRES_OK(context,
                      context->allocate_output(0, output_shape, &output));
@@ -189,6 +194,7 @@ namespace functor {
 
 DECLARE_GPU_SPEC(Eigen::half);
 DECLARE_GPU_SPEC(float);
+DECLARE_GPU_SPEC(double);
 #undef DECLARE_GPU_SPEC
 }  // namespace functor
 
@@ -198,6 +204,9 @@ REGISTER_KERNEL_BUILDER(
 REGISTER_KERNEL_BUILDER(
     Name("AvgPool").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     AvgPoolingOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("AvgPool").Device(DEVICE_GPU).TypeConstraint<double>("T"),
+    AvgPoolingOp<GPUDevice, double>);
 #endif  // GOOGLE_CUDA
 
 // The operation to compute AvgPool gradients.
@@ -423,6 +432,12 @@ class AvgPoolingGradOp<GPUDevice, T> : public OpKernel {
   TensorFormat data_format_;
 };
 
+REGISTER_KERNEL_BUILDER(Name("AvgPoolGrad")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<double>("T")
+                            .HostMemory("orig_input_shape")
+                            .Label("cudnn"),
+                        AvgPoolingGradOp<GPUDevice, double>);
 REGISTER_KERNEL_BUILDER(Name("AvgPoolGrad")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<float>("T")
@@ -553,6 +568,11 @@ REGISTER_KERNEL_BUILDER(Name("AvgPoolGrad")
                             .TypeConstraint<float>("T")
                             .HostMemory("orig_input_shape"),
                         AvgPoolingGradOpCustomGPUKernel<float>);
+REGISTER_KERNEL_BUILDER(Name("AvgPoolGrad")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<double>("T")
+                            .HostMemory("orig_input_shape"),
+                        AvgPoolingGradOpCustomGPUKernel<double>);
 REGISTER_KERNEL_BUILDER(Name("AvgPoolGrad")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<Eigen::half>("T")
diff --git a/tensorflow/core/kernels/avgpooling_op_gpu.cu.cc b/tensorflow/core/kernels/avgpooling_op_gpu.cu.cc
index 6537b42f1e..35511d5c31 100644
--- a/tensorflow/core/kernels/avgpooling_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/avgpooling_op_gpu.cu.cc
@@ -35,6 +35,7 @@ typedef Eigen::GpuDevice GPUDevice;
 
 DEFINE_GPU_KERNELS(Eigen::half)
 DEFINE_GPU_KERNELS(float)
+DEFINE_GPU_KERNELS(double)
 
 #undef DEFINE_GPU_KERNELS
 
@@ -99,6 +100,12 @@ bool RunAvePoolBackwardNHWC(const T* const top_diff, const int num,
   return d.ok();
 }
 
+template bool RunAvePoolBackwardNHWC(
+    const double* const top_diff, const int num, const int height,
+    const int width, const int channels, const int pooled_height,
+    const int pooled_width, const int kernel_h, const int kernel_w,
+    const int stride_h, const int stride_w, const int pad_t, const int pad_l,
+    double* const bottom_diff, const GPUDevice& d);
 template bool RunAvePoolBackwardNHWC(
     const float* const top_diff, const int num, const int height,
     const int width, const int channels, const int pooled_height,
diff --git a/tensorflow/core/kernels/eigen_pooling.h b/tensorflow/core/kernels/eigen_pooling.h
index 896c995761..2f83780525 100644
--- a/tensorflow/core/kernels/eigen_pooling.h
+++ b/tensorflow/core/kernels/eigen_pooling.h
@@ -334,7 +334,8 @@ struct AvgPoolMeanReducer {
   }
 
   template <typename Packet>
-  void reducePacketWithType(T, const Packet& p, Packet* accum) {
+  EIGEN_DEVICE_FUNC EIGEN_STRONG_INLINE void reducePacketWithType(
+      T, const Packet& p, Packet* accum) {
     Packet skip_mask =
         pequal(p, pset1<Packet>(-Eigen::NumTraits<T>::highest()));
     (*accum) = padd<Packet>(*accum, psel(p, pset1<Packet>(0), skip_mask));
@@ -480,11 +481,9 @@ SpatialAvgPooling(const Input& input, DenseIndex patchRows,
                              Eigen::type2index<3> > >::type reduction_dims;
 #endif
   return input
-      .extract_image_patches(
-          patchRows, patchCols, strideRows, strideCols, in_strideRows,
-          in_strideCols, padding_type,
-          -Eigen::NumTraits<typename internal::remove_const<
-              typename internal::traits<Input>::Scalar>::type>::highest())
+      .extract_image_patches(patchRows, patchCols, strideRows, strideCols,
+                             in_strideRows, in_strideCols, padding_type,
+                             -Eigen::NumTraits<CoeffReturnType>::highest())
       .reduce(reduction_dims, mean_with_nan)
       .reshape(post_reduce_dims);
 }
diff --git a/tensorflow/python/kernel_tests/pooling_ops_test.py b/tensorflow/python/kernel_tests/pooling_ops_test.py
index 2f3bea5825..ed44a1a4d1 100644
--- a/tensorflow/python/kernel_tests/pooling_ops_test.py
+++ b/tensorflow/python/kernel_tests/pooling_ops_test.py
@@ -123,8 +123,9 @@ class PoolingTest(test.TestCase):
       if input_sizes[-1] % 4 != 0:
         tf_logging.info("Skipping test for depth %d", input_sizes[-1])
         return
-    tf_logging.info("Running %s test. %r %r %d %r %r %r", data_format, v2,
-                    input_sizes, total_size, pool_func, ksize, strides)
+    tf_logging.info("Running %s test. %r %r %d %r %r %r %s", data_format, v2,
+                    input_sizes, total_size, pool_func, ksize, strides,
+                    data_type)
     # Initializes the input tensor with array containing incrementing
     # numbers from 1, wrapping round to -127 after 127 to support int8.
     x = [((f + 128) % 255) - 127 for f in range(total_size)]
@@ -193,6 +194,8 @@ class PoolingTest(test.TestCase):
 
     self._VerifyOneType(pool_func, input_sizes, ksize, strides, padding,
                         data_format, dtypes.float32, expected, use_gpu, v2)
+    self._VerifyOneType(pool_func, input_sizes, ksize, strides, padding,
+                        data_format, dtypes.float64, expected, use_gpu, v2)
 
     if not use_gpu or test_util.CudaSupportsHalfMatMulAndConv():
       self._VerifyOneType(pool_func, input_sizes, ksize, strides, padding,
-- 
GitLab


From e28a79eae228be8e65b5dff8bb8aa5ee2f41f70a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 06:28:09 -0700
Subject: [PATCH 236/960] Make TensorSpec and BoundedTensorSpec serializable.

PiperOrigin-RevId: 189731737
---
 tensorflow/python/framework/tensor_spec.py      |  6 ++++++
 tensorflow/python/framework/tensor_spec_test.py | 10 ++++++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/python/framework/tensor_spec.py b/tensorflow/python/framework/tensor_spec.py
index 546c48adba..6676cfcaa3 100644
--- a/tensorflow/python/framework/tensor_spec.py
+++ b/tensorflow/python/framework/tensor_spec.py
@@ -110,6 +110,9 @@ class TensorSpec(object):
   def __ne__(self, other):
     return not self == other
 
+  def __reduce__(self):
+    return TensorSpec, (self._shape, self._dtype, self._name)
+
 
 class BoundedTensorSpec(TensorSpec):
   """A `TensorSpec` that specifies minimum and maximum values.
@@ -210,4 +213,7 @@ class BoundedTensorSpec(TensorSpec):
     return (tensor_spec_eq and np.allclose(self.minimum, other.minimum) and
             np.allclose(self.maximum, other.maximum))
 
+  def __reduce__(self):
+    return BoundedTensorSpec, (self._shape, self._dtype, self._minimum,
+                               self._maximum, self._name)
 
diff --git a/tensorflow/python/framework/tensor_spec_test.py b/tensorflow/python/framework/tensor_spec_test.py
index b33d769d86..2e9e43e122 100644
--- a/tensorflow/python/framework/tensor_spec_test.py
+++ b/tensorflow/python/framework/tensor_spec_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import pickle
+
 import numpy as np
 
 from tensorflow.python.framework import constant_op
@@ -143,6 +145,10 @@ class TensorSpecTest(test_util.TensorFlowTestCase):
     unbounded_spec = tensor_spec.TensorSpec((1, 2), dtypes.int32)
     self.assertFalse(unbounded_spec.is_bounded())
 
+  def testSerialization(self):
+    desc = tensor_spec.TensorSpec([1, 5], dtypes.float32, "test")
+    self.assertEqual(pickle.loads(pickle.dumps(desc)), desc)
+
 
 class BoundedTensorSpecTest(test_util.TensorFlowTestCase):
 
@@ -243,6 +249,10 @@ class BoundedTensorSpecTest(test_util.TensorFlowTestCase):
     self.assertEqual(spec.dtype.max, bounded_spec.maximum)
     self.assertEqual(spec.name, bounded_spec.name)
 
+  def testSerialization(self):
+    desc = tensor_spec.BoundedTensorSpec([1, 5], dtypes.float32, -1, 1, "test")
+    self.assertEqual(pickle.loads(pickle.dumps(desc)), desc)
+
 
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From a40c8024f9beec346c2c1d98e9238c5d48ea0dca Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 07:27:16 -0700
Subject: [PATCH 237/960] Drop name_scope from operation names during
 quantization to avoid doubling it up.

PiperOrigin-RevId: 189737746
---
 tensorflow/contrib/quantize/python/common.py  |  8 +++++++
 .../contrib/quantize/python/quantize.py       |  6 +++++
 .../contrib/quantize/python/quantize_test.py  | 24 +++++++++++++++++++
 3 files changed, 38 insertions(+)

diff --git a/tensorflow/contrib/quantize/python/common.py b/tensorflow/contrib/quantize/python/common.py
index 3138149468..bf648e158e 100644
--- a/tensorflow/contrib/quantize/python/common.py
+++ b/tensorflow/contrib/quantize/python/common.py
@@ -123,3 +123,11 @@ def CreateOrGetQuantizationStep():
         # normal variables to return a tensor of the same name.
         return array_ops.identity(
             state_ops.assign_add(quantization_step_tensor, 1))
+
+
+def DropStringPrefix(s, prefix):
+  """If the string starts with this prefix, drops it."""
+  if s.startswith(prefix):
+    return s[len(prefix):]
+  else:
+    return s
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 9780e6dbcc..2b5b877e8e 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -367,6 +367,12 @@ def _InsertQuantOp(context,
       consumer operation.
   """
   name_prefix = _AddContextToName(context, name)
+  # This is needed on TPU where name_scope == 'TPUReplicate/loop', and
+  # name_prefix starts with 'TPUReplicate/loop/'; without dropping it
+  # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which
+  # breaks things later.
+  name_prefix = common.DropStringPrefix(name_prefix, ops.get_name_scope() + '/')
+
   inputs = producer.outputs[0]
   if moving_avg:
     quant = (
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index 8e60f4b661..216310abe4 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -164,6 +164,30 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       self.assertTrue('FakeQuantWithMinMaxVars' in
                       [i.op.type for i in bypass_tensor.op.inputs])
 
+  def testWithNameScope(self):
+    self._RunTestOverParameters(self._TestWithNameScope)
+
+  def _TestWithNameScope(self, is_training):
+    graph = ops.Graph()
+    with graph.as_default():
+      with graph.name_scope('name_scope'):
+        batch_size, height, width, depth = 5, 128, 128, 3
+        input1 = array_ops.zeros((batch_size, height, width, depth))
+        _ = conv2d(
+            input1,
+            32, [5, 5],
+            stride=2,
+            padding='SAME',
+            weights_initializer=self._WeightInit(0.09),
+            activation_fn=None,
+            scope='test')
+
+        quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
+
+    for op in graph.get_operations():
+      self.assertTrue(not op.name.startswith('name_scope/name_scope/'),
+                      'Broken op: %s' % op.name)
+
   def _WeightInit(self, stddev):
     """Returns truncated normal variable initializer.
 
-- 
GitLab


From 222c2b858fb1722f9aaf45e780b08e505e845665 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Tue, 20 Mar 2018 08:08:48 -0700
Subject: [PATCH 238/960] tfdbg: Curses CLI: Fill line-end whitespace with
 default color pair

to prevent spurious color pairs from appearing in certain text
terminal environments.

RELNOTES: Bug fix: tfdbg curses CLI: fix spurious background colors in some text terminals.
PiperOrigin-RevId: 189742433
---
 tensorflow/python/debug/cli/curses_ui.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/tensorflow/python/debug/cli/curses_ui.py b/tensorflow/python/debug/cli/curses_ui.py
index bb52f90512..f66cefb427 100644
--- a/tensorflow/python/debug/cli/curses_ui.py
+++ b/tensorflow/python/debug/cli/curses_ui.py
@@ -1185,6 +1185,22 @@ class CursesUI(base_ui.BaseUI):
       self._main_menu = None
       self._main_menu_pad = None
 
+  def _pad_line_end_with_whitespace(self, pad, row, line_end_x):
+    """Pad the whitespace at the end of a line with the default color pair.
+
+    Prevents spurious color pairs from appearing at the end of the lines in
+    certain text terimnals.
+
+    Args:
+      pad: The curses pad object to operate on.
+      row: (`int`) row index.
+      line_end_x: (`int`) column index of the end of the line (beginning of
+        the whitespace).
+    """
+    if line_end_x < self._max_x - 2:
+      pad.addstr(row, line_end_x, " " * (self._max_x - 3 - line_end_x),
+                 self._default_color_pair)
+
   def _screen_add_line_to_output_pad(self, pad, row, txt, color_segments=None):
     """Render a line in a text pad.
 
@@ -1208,6 +1224,7 @@ class CursesUI(base_ui.BaseUI):
 
     if not color_segments:
       pad.addstr(row, 0, txt, self._default_color_pair)
+      self._pad_line_end_with_whitespace(pad, row, len(txt))
       return
 
     if not isinstance(color_segments, list):
@@ -1248,6 +1265,8 @@ class CursesUI(base_ui.BaseUI):
     for segment, color_pair in zip(all_segments, all_color_pairs):
       if segment[1] < self._max_x:
         pad.addstr(row, segment[0], txt[segment[0]:segment[1]], color_pair)
+    if all_segments:
+      self._pad_line_end_with_whitespace(pad, row, all_segments[-1][1])
 
   def _screen_scroll_output_pad(self, pad, viewport_top, viewport_left,
                                 screen_location_top, screen_location_left,
-- 
GitLab


From 4a4c13788634e73f3c1bd01abd142a607c2fd253 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Tue, 20 Mar 2018 08:32:30 -0700
Subject: [PATCH 239/960] Fixed the bug that the export code triggers the TPU
 validation.

PiperOrigin-RevId: 189745966
---
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 38 ++++++++++++++-----
 1 file changed, 28 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 5a8fa04e7c..f61f6bb52e 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -1044,8 +1044,8 @@ class _ModelFnWrapper(object):
     self._params = params
     self._ctx = ctx
 
-  def call_without_tpu(self, features, labels):
-    return self._call_model_fn(features, labels)
+  def call_without_tpu(self, features, labels, is_export_mode):
+    return self._call_model_fn(features, labels, is_export_mode=is_export_mode)
 
   def convert_to_single_tpu_train_step(self, dequeue_fn):
     """Converts user provided model_fn` as a single train step on TPU.
@@ -1204,7 +1204,7 @@ class _ModelFnWrapper(object):
 
     return predict_step, host_calls, captured_scaffold_fn
 
-  def _call_model_fn(self, features, labels, is_export_mode=True):
+  def _call_model_fn(self, features, labels, is_export_mode=False):
     """Calls the model_fn with required parameters."""
     model_fn_args = util.fn_args(self._model_fn)
     kwargs = {}
@@ -1230,7 +1230,11 @@ class _ModelFnWrapper(object):
                        'required by TPUEstimator to pass batch size as '
                        'params[\'batch_size\']'.format(self._model_fn))
 
-    batch_size_for_model_fn = self._ctx.batch_size_for_model_fn
+    if is_export_mode:
+      batch_size_for_model_fn = None
+    else:
+      batch_size_for_model_fn = self._ctx.batch_size_for_model_fn
+
     if batch_size_for_model_fn is not None:
       params[_BATCH_SIZE_KEY] = batch_size_for_model_fn
 
@@ -1778,6 +1782,8 @@ class TPUEstimator(estimator_lib.Estimator):
         eval_batch_size, predict_batch_size,
         use_tpu)
 
+    self._is_input_fn_invoked = None
+
   def _create_global_step(self, graph):
     """Creates a global step suitable for TPUs.
 
@@ -1860,6 +1866,9 @@ class TPUEstimator(estimator_lib.Estimator):
     if 'mode' in input_fn_args:
       kwargs['mode'] = mode
 
+    # Records the fact input_fn has been invoked.
+    self._is_input_fn_invoked = True
+
     with self._ctx.with_mode(mode) as ctx:
       # Setting the batch size in params first. This helps user to have same
       # input_fn for use_tpu=True/False.
@@ -1907,15 +1916,24 @@ class TPUEstimator(estimator_lib.Estimator):
       with self._ctx.with_mode(mode) as ctx:
         model_fn_wrapper = _ModelFnWrapper(model_fn, config, params, ctx)
 
-        # For export_savedmodel, input_fn is never passed to Estimator. So,
-        # if features is callable, it means it is the input_fn passed by
-        # TPUEstimator._call_input_fn. Then we can know if the mode == PREDICT,
-        # it implies, it is the .predict API, not export_savedmodel API.
-        is_export_mode = not callable(features)
+        if mode != model_fn_lib.ModeKeys.PREDICT:
+          is_export_mode = False
+        else:
+          # For export_savedmodel, input_fn is never passed to Estimator. So, by
+          # checking the self._is_input_fn_invoked bit, we can know, given the
+          # mode == PREDICT, it is the .predict API, not export_savedmodel API.
+          if self._is_input_fn_invoked:
+            is_export_mode = False
+          else:
+            is_export_mode = True
+
+        # Clear the bit.
+        self._is_input_fn_invoked = None
 
         if ctx.is_running_on_cpu(is_export_mode=is_export_mode):
           logging.info('Running %s on CPU', mode)
-          return model_fn_wrapper.call_without_tpu(features, labels)
+          return model_fn_wrapper.call_without_tpu(
+              features, labels, is_export_mode=is_export_mode)
 
         assert labels is None, '`labels` passed to `model_fn` must be `None`.'
         # TPUEstimator._call_input_fn passes `input_fn` as features to here.
-- 
GitLab


From 6a16e22421626ceffecb025b1cd80722c36aea0d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 09:27:24 -0700
Subject: [PATCH 240/960] Revise the main API for more consistent notation and
 add a supplemental shortcut to mark functions as "run in py_func". This is an
 intermediate step to simplifying the execution of plotting code.

PiperOrigin-RevId: 189753509
---
 tensorflow/contrib/py2tf/__init__.py      |   7 +-
 tensorflow/contrib/py2tf/impl/BUILD       |   1 +
 tensorflow/contrib/py2tf/impl/api.py      | 107 +++++++++++-----------
 tensorflow/contrib/py2tf/impl/api_test.py |  38 +++++---
 tensorflow/contrib/py2tf/utils/py_func.py |  40 +++++++-
 5 files changed, 122 insertions(+), 71 deletions(-)

diff --git a/tensorflow/contrib/py2tf/__init__.py b/tensorflow/contrib/py2tf/__init__.py
index 6531183cb5..a4b62a0976 100644
--- a/tensorflow/contrib/py2tf/__init__.py
+++ b/tensorflow/contrib/py2tf/__init__.py
@@ -24,15 +24,16 @@ from __future__ import print_function
 from tensorflow.contrib.py2tf import utils
 from tensorflow.contrib.py2tf.impl.api import convert
 from tensorflow.contrib.py2tf.impl.api import converted_call
-from tensorflow.contrib.py2tf.impl.api import graph_ready
+from tensorflow.contrib.py2tf.impl.api import do_not_convert
+from tensorflow.contrib.py2tf.impl.api import RunMode
 from tensorflow.contrib.py2tf.impl.api import to_code
 from tensorflow.contrib.py2tf.impl.api import to_graph
 from tensorflow.contrib.py2tf.pyct.transformer import PyFlowParseError
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
-    'to_graph', 'to_code', 'convert', 'graph_ready', 'converted_call', 'utils',
-    'PyFlowParseError'
+    'utils', 'convert', 'converted_call', 'do_not_convert', 'RunMode',
+    'to_code', 'to_graph', 'PyFlowParseError'
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/py2tf/impl/BUILD b/tensorflow/contrib/py2tf/impl/BUILD
index 90ffabbc9b..cc49d71b78 100644
--- a/tensorflow/contrib/py2tf/impl/BUILD
+++ b/tensorflow/contrib/py2tf/impl/BUILD
@@ -42,6 +42,7 @@ py_test(
         ":impl",
         "//tensorflow/contrib/py2tf/utils",
         "//tensorflow/python:client_testlib",
+        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/contrib/py2tf/impl/api.py b/tensorflow/contrib/py2tf/impl/api.py
index 883b304089..3a40729e5a 100644
--- a/tensorflow/contrib/py2tf/impl/api.py
+++ b/tensorflow/contrib/py2tf/impl/api.py
@@ -20,8 +20,12 @@ from __future__ import print_function
 
 from functools import wraps
 
+from enum import Enum
+
+# pylint:disable=g-bad-import-order
 import gast
 import six
+# pylint:enable=g-bad-import-order
 
 from tensorflow.contrib.py2tf.impl import config
 from tensorflow.contrib.py2tf.impl import conversion
@@ -29,6 +33,7 @@ from tensorflow.contrib.py2tf.pyct import compiler
 from tensorflow.contrib.py2tf.pyct import inspect_utils
 from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.contrib.py2tf.utils import builtins
+from tensorflow.contrib.py2tf.utils import py_func
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_inspect
 
@@ -37,55 +42,6 @@ from tensorflow.python.util import tf_inspect
 # (currently we require (module + class name, type))
 
 
-def graph_ready(f):
-  """No-op decorator that explicitly marks a function as graph-ready.
-
-  Graph-ready functions are assumed to not need any conversion.
-
-  Args:
-    f: Any callable.
-  Returns:
-    f itself.
-  """
-  setattr(f, '__pyct_is_compile_decorator', True)
-  return f
-
-
-def convert_inline(f, *args, **kwargs):
-  """Shorthand to convert and call a function.
-
-  For example, the following two statements are equivalent:
-
-      @convert()
-      def foo():
-        ...
-      foo(bar)
-
-      def foo():
-        ...
-      convert_inline(foo, bar)
-
-  Args:
-    f: Function to convert. Only this call will be converted.
-    *args: Passed through to f.
-    **kwargs: Passed through to f, with the following exceptions:
-        * arg_value_hints: A dict mapping parameter names to objects that can
-            hint at the type of those parameters.
-
-  Returns:
-    The result of the converted f applied to args and kwargs.
-  """
-  if 'arg_value_hints' in kwargs:
-    arg_value_hints = kwargs['arg_value_hints']
-    del kwargs['arg_value_hints']
-  else:
-    arg_value_hints = None
-  if tf_inspect.ismethod(f):
-    # When converting methods, the result is still an unbound function.
-    args = (f.__self__,) + args
-  return convert(arg_value_hints)(f)(*args, **kwargs)
-
-
 def convert(recursive=False, verbose=False, arg_types=None):
   """Decorator that compiles a function to graph mode.
 
@@ -122,6 +78,55 @@ def convert(recursive=False, verbose=False, arg_types=None):
   return decorator
 
 
+class RunMode(Enum):
+  GRAPH = 1
+  PY_FUNC = 2
+
+
+def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
+  """Decorator that suppresses compilation of a function.
+
+  Args:
+    run_as: RunMode value. Whether to run the function as-is, or wrap it into
+        a py_func.
+    return_dtypes: See py2tf.utils.py_func.wrap_py_func. Setting to None or
+        empty list or tuple will create a dummy return value that can be used
+        to set control dependencies.
+
+  Returns:
+    A decorator that wraps the original function.
+  """
+  def decorator(f):
+    """Decorator implementation."""
+
+    @wraps(f)
+    def graph_wrapper(*args, **kwargs):
+      return f(*args, **kwargs)
+
+    @wraps(f)
+    def py_func_wrapper(*args, **kwargs):
+      if kwargs:
+        raise NotImplementedError(
+            'RunMode.PY_FUNC does not yet support kwargs')
+      # TODO(mdan): Add support for kwargs.
+      return py_func.wrap_py_func(
+          f, return_dtypes, args, use_dummy_return=not return_dtypes)
+
+    if run_as == RunMode.GRAPH:
+      wrapper = graph_wrapper
+    elif run_as == RunMode.PY_FUNC:
+      wrapper = py_func_wrapper
+    else:
+      raise ValueError('unknown value for run_as: %s' % run_as)
+
+    # Sometimes the decorator is just desugared, making it impossible to detect.
+    # This attribute makes detection easier.
+    setattr(wrapper, '__pyct_is_compile_decorator', True)
+    return wrapper
+
+  return decorator
+
+
 def converted_call(f, recursive, verbose, arg_types, *args, **kwargs):
   """Compiles a function call inline."""
   # TODO(mdan): This needs cleanup.
@@ -227,7 +232,7 @@ def to_graph(e,
   """
   conversion_map = conversion.ConversionMap(
       recursive=recursive,
-      nocompile_decorators=(convert, graph_ready, convert_inline),
+      nocompile_decorators=(convert, do_not_convert, converted_call),
       partial_types=partial_types,
       api_module=tf_inspect.getmodule(to_graph))
   _, name = conversion.entity_to_graph(e, conversion_map, arg_values, arg_types)
@@ -274,7 +279,7 @@ def to_code(e,
   """
   conversion_map = conversion.ConversionMap(
       recursive=recursive,
-      nocompile_decorators=(convert, graph_ready, convert_inline),
+      nocompile_decorators=(convert, do_not_convert, converted_call),
       partial_types=partial_types,
       api_module=tf_inspect.getmodule(to_graph))
   conversion.entity_to_graph(e, conversion_map, arg_values, arg_types)
diff --git a/tensorflow/contrib/py2tf/impl/api_test.py b/tensorflow/contrib/py2tf/impl/api_test.py
index 13f8e66018..a7b1aba852 100644
--- a/tensorflow/contrib/py2tf/impl/api_test.py
+++ b/tensorflow/contrib/py2tf/impl/api_test.py
@@ -18,10 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib.py2tf import utils
 from tensorflow.contrib.py2tf.impl import api
 from tensorflow.contrib.py2tf.impl import config
 from tensorflow.contrib.py2tf.pyct import parser
+from tensorflow.contrib.py2tf.utils import py_func
 from tensorflow.python.framework import constant_op
 from tensorflow.python.platform import test
 
@@ -81,11 +84,11 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], sess.run(x).tolist())
 
-  def test_decorator_calls_converted(self):
+  def test_decorator_calls_unconverted_graph(self):
 
     class TestClass(object):
 
-      @api.graph_ready
+      @api.do_not_convert(api.RunMode.GRAPH)
       def called_member(self, a):
         return tf.negative(a)
 
@@ -102,20 +105,23 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], sess.run(x).tolist())
 
-  def test_decorator_calls_decorated(self):
+  def test_decorator_calls_unconverted_py_func(self):
 
     class TestClass(object):
 
-      @api.convert()
+      @api.do_not_convert(
+          api.RunMode.PY_FUNC, return_dtypes=py_func.MatchDType(1))
       def called_member(self, a):
-        if a < 0:
-          a = -a
-        return a
+        return np.negative(a)
 
       @api.convert(recursive=True)
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
-          x //= self.called_member(a)
+          y = self.called_member(a)
+          # set_shape works around while_loop's limitations.
+          # TODO(mdan): Allow specifying shapes (or ShapeLike) instead.
+          y.set_shape(a.shape)
+          x //= y
         return x
 
     tc = TestClass()
@@ -125,10 +131,11 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], sess.run(x).tolist())
 
-  def test_convert_call_site_decorator(self):
+  def test_decorator_calls_decorated(self):
 
     class TestClass(object):
 
+      @api.convert()
       def called_member(self, a):
         if a < 0:
           a = -a
@@ -137,7 +144,7 @@ class ApiTest(test.TestCase):
       @api.convert(recursive=True)
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
-          x //= api.convert_inline(self.called_member, a)
+          x //= self.called_member(a)
         return x
 
     tc = TestClass()
@@ -147,17 +154,20 @@ class ApiTest(test.TestCase):
           constant_op.constant(-2))
       self.assertListEqual([0, 1], sess.run(x).tolist())
 
-  def test_graph_ready_call_site_decorator(self):
+  def test_convert_call_site_decorator(self):
 
     class TestClass(object):
 
       def called_member(self, a):
-        return tf.negative(a)
+        if a < 0:
+          a = -a
+        return a
 
       @api.convert(recursive=True)
       def test_method(self, x, s, a):
         while tf.reduce_sum(x) > s:
-          x //= api.graph_ready(self.called_member(a))
+          x //= api.converted_call(self.called_member, False, False, {}, self,
+                                   a)
         return x
 
     tc = TestClass()
@@ -168,6 +178,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([0, 1], sess.run(x).tolist())
 
   def test_to_graph_basic(self):
+
     def test_fn(x, s):
       while tf.reduce_sum(x) > s:
         x //= 2
@@ -180,6 +191,7 @@ class ApiTest(test.TestCase):
       self.assertListEqual([1, 2], sess.run(x).tolist())
 
   def test_to_code_basic(self):
+
     def test_fn(x, s):
       while tf.reduce_sum(x) > s:
         x /= 2
diff --git a/tensorflow/contrib/py2tf/utils/py_func.py b/tensorflow/contrib/py2tf/utils/py_func.py
index 838872d092..79920cd841 100644
--- a/tensorflow/contrib/py2tf/utils/py_func.py
+++ b/tensorflow/contrib/py2tf/utils/py_func.py
@@ -18,11 +18,23 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from collections import namedtuple
+
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import script_ops
 
 
+class MatchDType(namedtuple('MatchDType', ('arg_number',))):
+  """Allows matching the dtype of an argument.
+
+  Used in conjunction with function calls. For example, MatchDType(0) will
+  match the DType of the first argument.
+  """
+
+  pass
+
+
 def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
   """Helper that wraps a callable to py_func.
 
@@ -34,10 +46,12 @@ def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
 
   Args:
     f: Callable
-    return_dtypes: DType, tuple, list or None, the data type for each of f's
-        return value. None if f has no return values or use_dummy_return is
-        True.
-    arguments: Arguments for f
+    return_dtypes: None, individual of tuple/list of DType or MatchDType, the
+        data type for each of f's return value(s). Set to None if f has no
+        return values or use_dummy_return is True. Use MatchDType to define a
+        dtype identical to that of `i`th argument (argument 0 is the first);
+        an argument must of Tensor type if it is to be used with MatchDType.
+    arguments: Arguments for f, as list or tuple.
     use_dummy_return: If True, the function will return a dummy value of 1
         and discard its actual return value.
   Returns:
@@ -58,6 +72,24 @@ def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
     if arg_is_tensor[j]:
       i += 1
 
+  def match_argument(arg_number):
+    arg = arguments[arg_number]
+    if not arg_is_tensor[arg_number]:
+      raise ValueError(
+          'argument %d was used with MatchDType and must be a tf.Tensor, but '
+          'was %s instead' % (arg_number, type(arg)))
+    return arg.dtype
+
+  if return_dtypes:
+    if isinstance(return_dtypes, MatchDType):
+      return_dtypes = match_argument(return_dtypes.arg_number)
+    elif isinstance(return_dtypes, (list, tuple)):
+      return_dtypes = tuple(
+          match_argument(a.arg_number) if isinstance(a, MatchDType) else a
+          for a in return_dtypes)
+    else:
+      assert isinstance(return_dtypes, dtypes.DType)
+
   def f_wrapper(*tensor_args):
     f_args = tuple(tensor_args[index_in_tensor_list[i]]
                    if arg_is_tensor[i] else arguments[i] for i in range(n))
-- 
GitLab


From 1a28fe156080d2c4a1986026c5bbb776a99b4ec9 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 20 Mar 2018 10:39:51 -0700
Subject: [PATCH 241/960] Mark sparse wrap test as no-pip

---
 tensorflow/contrib/image/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 2924aef815..8ba51a1d83 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -243,6 +243,7 @@ cuda_py_test(
     name = "sparse_image_warp_test",
     size = "medium",
     srcs = ["python/kernel_tests/sparse_image_warp_test.py"],
+    tags = ["no_pip"],
     additional_deps = [
         ":sparse_image_warp_py",
         "//third_party/py/numpy",
-- 
GitLab


From e543ae60a73bb8137227b417ed4f9a80f10f63a1 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Tue, 20 Mar 2018 11:03:51 -0700
Subject: [PATCH 242/960] Fix BUILD file formatting

---
 tensorflow/contrib/image/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 8ba51a1d83..79eb3762ed 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -243,7 +243,6 @@ cuda_py_test(
     name = "sparse_image_warp_test",
     size = "medium",
     srcs = ["python/kernel_tests/sparse_image_warp_test.py"],
-    tags = ["no_pip"],
     additional_deps = [
         ":sparse_image_warp_py",
         "//third_party/py/numpy",
@@ -260,6 +259,7 @@ cuda_py_test(
         "//tensorflow/core:protos_all_py",
     ],
     data = [":sparse_image_warp_test_data"],
+    tags = ["no_pip"],
 )
 
 filegroup(
-- 
GitLab


From 27a4b79704f03569fc6edd5b3a30ff6cf599310d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 11:04:44 -0700
Subject: [PATCH 243/960] Add support for automatically wrapping NumPy
 functions based on a whitelist.

PiperOrigin-RevId: 189771575
---
 .../contrib/py2tf/converters/call_trees.py    | 31 ++++++++++++++++++-
 .../py2tf/converters/call_trees_test.py       | 18 +++++++++++
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/py2tf/converters/call_trees.py b/tensorflow/contrib/py2tf/converters/call_trees.py
index ca8726f916..74fbf80677 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees.py
+++ b/tensorflow/contrib/py2tf/converters/call_trees.py
@@ -22,6 +22,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from collections import namedtuple
 import types
 
 import gast
@@ -34,6 +35,16 @@ from tensorflow.contrib.py2tf.pyct import transformer
 from tensorflow.python.util import tf_inspect
 
 
+class FunctionInfo(namedtuple('FunctionInfo', ('dtype',))):
+  pass
+
+
+# TODO(mdan): Move this to config.py.
+KNOWN_NUMPY_FUNCTIONS = {
+    ('numpy', 'random', 'binomial'): FunctionInfo(dtype='tf.int64'),
+}
+
+
 class FunctionNamer(object):
   """Describes the interface for CallTreeTransformer's namer."""
 
@@ -185,6 +196,18 @@ class CallTreeTransformer(transformer.Base):
     """
     return templates.replace(template, func=node.func, original_args=node.args)
 
+  def _wrap_to_py_func_single_return(self, node, fqn):
+    # TODO(mdan): Properly handle varargs, kwargs, etc.
+    template = """
+      py2tf_utils.wrap_py_func(func, dtype, (original_args,), False)
+    """
+    dtype = KNOWN_NUMPY_FUNCTIONS[fqn].dtype
+    return templates.replace_as_expression(
+        template,
+        func=node.func,
+        dtype=parser.parse_expression(dtype),
+        original_args=node.args)
+
   def _insert_dynamic_conversion(self, node):
     """Inlines a dynamic conversion for a dynamic function."""
     # TODO(mdan): Pass information on the statically compiled functions.
@@ -248,10 +271,16 @@ class CallTreeTransformer(transformer.Base):
     self.generic_visit(node)
     if anno.hasanno(node.func, 'live_val'):
       target_entity = anno.getanno(node.func, 'live_val')
+      if anno.hasanno(node.func, 'fqn'):
+        target_fqn = anno.getanno(node.func, 'fqn')
       if self._function_is_compilable(target_entity):
         node = self._rename_compilable_function(node)
+      elif target_fqn in KNOWN_NUMPY_FUNCTIONS:
+        # TODO(mdan): Should we replace these with equivalent TF ops instead?
+        node = self._wrap_to_py_func_single_return(node, target_fqn)
       else:
-        raise NotImplementedError('py_func with return values')
+        raise NotImplementedError(
+            'py_func with return values (unknown function)')
     else:
       if self.context.recursive:
         node = self._insert_dynamic_conversion(node)
diff --git a/tensorflow/contrib/py2tf/converters/call_trees_test.py b/tensorflow/contrib/py2tf/converters/call_trees_test.py
index d482a9ef78..1106432da6 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees_test.py
+++ b/tensorflow/contrib/py2tf/converters/call_trees_test.py
@@ -18,9 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib.py2tf.converters import call_trees
 from tensorflow.contrib.py2tf.converters import converter_test_base
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
@@ -105,6 +109,20 @@ class CallTreesTest(converter_test_base.TestCase):
         sess.run(sess.graph.get_operations()[0])
         self.assertEquals('bar', a.foo)
 
+  def test_py_func_wrap_known_function(self):
+
+    def test_fn():
+      return np.random.binomial(2, 0.5)
+
+    node = self.parse_and_analyze(test_fn, {'np': np})
+    node = call_trees.transform(node, self.ctx, (), ())
+
+    with self.compiled(node, dtypes.int64) as result:
+      result.np = np
+      with self.test_session() as sess:
+        self.assertTrue(isinstance(result.test_fn(), ops.Tensor))
+        self.assertIn(sess.run(result.test_fn()), (0, 1, 2))
+
   def test_uncompiled_modules(self):
 
     def test_fn(a):
-- 
GitLab


From 3bca4298aacd9f89de2ac532bb7fedcdec1a5bb6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 11:12:47 -0700
Subject: [PATCH 244/960] Replace std::clock with random::New64() for setting
 random seed

PiperOrigin-RevId: 189773399
---
 tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc | 4 ++--
 tensorflow/contrib/tensor_forest/kernels/v4/input_data.h  | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc b/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc
index da600d34ea..63d4d9ba50 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/grow_stats.cc
@@ -19,6 +19,7 @@
 #include "tensorflow/contrib/tensor_forest/kernels/tree_utils.h"
 #include "tensorflow/contrib/tensor_forest/kernels/v4/stat_utils.h"
 #include "tensorflow/core/lib/random/distribution_sampler.h"
+#include "tensorflow/core/lib/random/random.h"
 
 namespace tensorflow {
 namespace tensorforest {
@@ -122,9 +123,8 @@ ClassificationStats::ClassificationStats(const TensorForestParams& params,
     right_gini_.reset(new RunningGiniScores());
   }
 
-  uint64 time_seed = static_cast<uint64>(std::clock());
   single_rand_ = std::unique_ptr<random::PhiloxRandom>(
-      new random::PhiloxRandom(time_seed));
+      new random::PhiloxRandom(random::New64()));
   rng_ = std::unique_ptr<random::SimplePhilox>(
       new random::SimplePhilox(single_rand_.get()));
 }
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
index b991e6339f..95f75b4d7e 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/input_data.h
@@ -23,6 +23,7 @@
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/lib/random/philox_random.h"
+#include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
 
 namespace tensorflow {
@@ -56,9 +57,8 @@ class TensorDataSet {
 
     // Set up the random number generator.
     if (split_sampling_random_seed_ == 0) {
-      uint64 time_seed = static_cast<uint64>(std::clock());
       single_rand_ = std::unique_ptr<random::PhiloxRandom>(
-          new random::PhiloxRandom(time_seed));
+          new random::PhiloxRandom(random::New64()));
     } else {
       single_rand_ = std::unique_ptr<random::PhiloxRandom>(
           new random::PhiloxRandom(split_sampling_random_seed_));
-- 
GitLab


From 1c4e42b39fd9ae2da14d7eb323bedc144a6e659b Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Mar 2018 11:13:48 -0700
Subject: [PATCH 245/960] Use 32 bit induction variable in gather expander

Right now this is unconditional (and we fail with Unimplemented() if a 32 bit
induction variable is not large enough), but eventually we may want to be
smarter about this.

PiperOrigin-RevId: 189773581
---
 tensorflow/compiler/xla/service/BUILD         | 12 ++++
 .../compiler/xla/service/gather_expander.cc   | 19 +++++-
 .../xla/service/gather_expander_test.cc       | 51 ++++++++++++++++
 tensorflow/compiler/xla/service/while_util.cc | 21 +++----
 tensorflow/compiler/xla/service/while_util.h  |  2 +-
 tensorflow/compiler/xla/tests/BUILD           |  2 +
 .../xla/tests/gather_operation_test.cc        | 60 +++++++++++++++++++
 tensorflow/compiler/xla/util.h                |  9 +++
 8 files changed, 163 insertions(+), 13 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gather_expander_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index 43c56484ea..d4d67872cf 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1276,6 +1276,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "gather_expander_test",
+    srcs = ["gather_expander_test.cc"],
+    deps = [
+        ":gather_expander",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla/tests:test_macros_header",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",  # fixdeps: keep
+        "//tensorflow/compiler/xla/tools/parser:hlo_parser",
+    ],
+)
+
 cc_library(
     name = "conditional_simplifier",
     srcs = ["conditional_simplifier.cc"],
diff --git a/tensorflow/compiler/xla/service/gather_expander.cc b/tensorflow/compiler/xla/service/gather_expander.cc
index 488bed35fe..221ff7900f 100644
--- a/tensorflow/compiler/xla/service/gather_expander.cc
+++ b/tensorflow/compiler/xla/service/gather_expander.cc
@@ -306,18 +306,33 @@ StatusOr<HloInstruction*> GatherExpander::ExpandGather(
   HloComputation* computation = gather_instr->parent();
   HloInstruction* operand = gather_instr->mutable_operand(0);
   HloInstruction* gather_indices = gather_instr->mutable_operand(1);
+  const Shape& gather_indices_shape = gather_indices->shape();
   const Shape& output_shape = gather_instr->shape();
   int64 output_rank = output_shape.dimensions_size();
 
   const GatherDimensionNumbers& dim_numbers =
       gather_instr->gather_dimension_numbers();
 
+  int64 gather_loop_trip_count = 1;
+  for (int64 i = 0, e = gather_indices_shape.dimensions_size(); i < e; i++) {
+    if (i != dim_numbers.index_vector_dim()) {
+      gather_loop_trip_count *= gather_indices_shape.dimensions(i);
+    }
+  }
+
+  if (!IsInt32(gather_loop_trip_count)) {
+    return Unimplemented(
+        "Gather operations with more than 2147483647 gather indices are not "
+        "supported. This error occurred for %s.",
+        gather_instr->ToString().c_str());
+  }
+
   TF_ASSIGN_OR_RETURN(HloInstruction * canonical_gather_indices,
                       CanonicalizeGatherIndices(
                           gather_indices, dim_numbers.index_vector_dim()));
 
-  const int64 gather_loop_trip_count =
-      canonical_gather_indices->shape().dimensions(0);
+  CHECK_EQ(gather_loop_trip_count,
+           canonical_gather_indices->shape().dimensions(0));
 
   TF_ASSIGN_OR_RETURN(
       HloInstruction * accumulator_init,
diff --git a/tensorflow/compiler/xla/service/gather_expander_test.cc b/tensorflow/compiler/xla/service/gather_expander_test.cc
new file mode 100644
index 0000000000..ba41ee8428
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gather_expander_test.cc
@@ -0,0 +1,51 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gather_expander.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/test_macros.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+namespace xla {
+namespace {
+TEST(GatherExpanderTest, ErrorStatusOnTooManyIndices) {
+  const string hlo_text = R"(
+HloModule TensorFlowGatherMultipleBatchDims
+
+ENTRY main {
+  operand = s32[3,3] parameter(0)
+  indices = s32[2147483647,5] parameter(1)
+  ROOT gather = s32[2147483647,3,5] gather(operand, indices),
+      output_window_dims={1},
+      elided_window_dims={1},
+      gather_dims_to_operand_dims={1},
+      index_vector_dim=2,
+      window_bounds={3, 1}
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          tools::Parse(hlo_text));
+
+  Status status = GatherExpander{}.Run(module.get()).status();
+  EXPECT_EQ(status.code(), tensorflow::error::UNIMPLEMENTED);
+
+  ASSERT_THAT(
+      status.error_message(),
+      ::testing::HasSubstr("Gather operations with more than 2147483647 gather "
+                           "indices are not supported."));
+}
+
+}  // namespace
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/while_util.cc b/tensorflow/compiler/xla/service/while_util.cc
index 8cd5882f32..bd07941843 100644
--- a/tensorflow/compiler/xla/service/while_util.cc
+++ b/tensorflow/compiler/xla/service/while_util.cc
@@ -142,23 +142,23 @@ WhileUtil::MakeInstructionsLiveIn(
 
 static StatusOr<std::unique_ptr<HloComputation>>
 MakeCountedLoopConditionComputation(const Shape& loop_state_shape,
-                                    int64 trip_count) {
+                                    int32 trip_count) {
   Shape scalar_pred = ShapeUtil::MakeShape(PRED, {});
-  Shape scalar_s64 = ShapeUtil::MakeShape(S64, {});
 
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> cond_computation,
                       CreateComputationWithSignature(
                           {&loop_state_shape}, scalar_pred, "while_cond"));
 
   HloInstruction* trip_count_constant = cond_computation->AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<int64>(trip_count)));
+      HloInstruction::CreateConstant(Literal::CreateR0<int32>(trip_count)));
 
   HloInstruction* param = cond_computation->parameter_instruction(0);
-  TF_ASSIGN_OR_RETURN(HloInstruction * counter,
+  TF_ASSIGN_OR_RETURN(HloInstruction * indvar,
                       MakeGetTupleElementHlo(param, 0));
+
   TF_ASSIGN_OR_RETURN(
       HloInstruction * compare,
-      MakeBinaryHlo(HloOpcode::kLt, counter, trip_count_constant));
+      MakeBinaryHlo(HloOpcode::kLt, indvar, trip_count_constant));
   cond_computation->set_root_instruction(compare);
   return std::move(cond_computation);
 }
@@ -171,8 +171,7 @@ static StatusOr<std::unique_ptr<HloComputation>> MakeCountedLoopBodyComputation(
                       CreateComputationWithSignature(
                           {&loop_state_shape}, loop_state_shape, "while_body"));
   HloInstruction* one = body_computation->AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<int64>(1)));
-
+      HloInstruction::CreateConstant(Literal::CreateR0<int32>(1)));
   HloInstruction* param = body_computation->parameter_instruction(0);
   TF_ASSIGN_OR_RETURN(HloInstruction * indvar,
                       MakeGetTupleElementHlo(param, 0));
@@ -200,7 +199,7 @@ static StatusOr<HloInstruction*> MakeInitTupleFromInitValues(
   std::vector<HloInstruction*> init_values_with_indvar;
   init_values_with_indvar.reserve(init_values.size() + 1);
   HloInstruction* zero = computation->AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR0<int64>(0)));
+      HloInstruction::CreateConstant(Literal::CreateR0<int32>(0)));
   init_values_with_indvar.push_back(zero);
   c_copy(init_values, std::back_inserter(init_values_with_indvar));
   return computation->AddInstruction(
@@ -210,16 +209,18 @@ static StatusOr<HloInstruction*> MakeInitTupleFromInitValues(
 static Shape MakeLoopStateShape(const WhileUtil::LoopStateTy& init_values) {
   std::vector<Shape> loop_state_shape_components;
   loop_state_shape_components.reserve(init_values.size() + 1);
-  loop_state_shape_components.push_back(ShapeUtil::MakeShape(S64, {}));
+  loop_state_shape_components.push_back(ShapeUtil::MakeShape(S32, {}));
   c_transform(init_values, std::back_inserter(loop_state_shape_components),
               [](HloInstruction* instr) { return instr->shape(); });
   return ShapeUtil::MakeTupleShape(loop_state_shape_components);
 }
 
 /*static*/ StatusOr<WhileUtil::LoopStateTy> WhileUtil::MakeCountedLoop(
-    HloComputation* computation, int64 trip_count,
+    HloComputation* computation, int32 trip_count,
     const WhileUtil::LoopStateTy& init_values,
     const WhileUtil::LoopBodyGeneratorTy& loop_body_generator) {
+  CHECK_GE(trip_count, 0);
+
   Shape loop_state_shape = MakeLoopStateShape(init_values);
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<HloComputation> cond,
diff --git a/tensorflow/compiler/xla/service/while_util.h b/tensorflow/compiler/xla/service/while_util.h
index 80f7e16e64..1688d46742 100644
--- a/tensorflow/compiler/xla/service/while_util.h
+++ b/tensorflow/compiler/xla/service/while_util.h
@@ -71,7 +71,7 @@ class WhileUtil {
   //    return loop_state;
   //  }
   static StatusOr<LoopStateTy> MakeCountedLoop(
-      HloComputation* computation, int64 trip_count,
+      HloComputation* computation, int32 trip_count,
       const LoopStateTy& init_values,
       const LoopBodyGeneratorTy& loop_body_generator);
 };
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 025ac129d7..04a9c1ef79 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -676,7 +676,9 @@ xla_test(
     name = "gather_operation_test",
     srcs = ["gather_operation_test.cc"],
     deps = [
+        ":client_library_test_base",
         ":hlo_test_base",
+        "//tensorflow/compiler/xla:execution_options_util",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/gather_operation_test.cc b/tensorflow/compiler/xla/tests/gather_operation_test.cc
index 8ba91946c0..9db68ff7a6 100644
--- a/tensorflow/compiler/xla/tests/gather_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/gather_operation_test.cc
@@ -13,8 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/compiler/xla/execution_options_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
@@ -397,5 +399,63 @@ ENTRY main {
   RunTest(hlo_text, operand.get(), gather_indices.get());
 }
 
+class GatherClientLibraryTest : public ClientLibraryTestBase {};
+
+// TODO(b/30671675): Asynchronous execution on stream is not yet supported on
+// GPU and CPU_PARALLEL.
+XLA_TEST_F(GatherClientLibraryTest,
+           DISABLED_ON_CPU_PARALLEL(DISABLED_ON_GPU(Basic))) {
+  // We create this HLO, but using the ComputationBuilder API.
+  //
+  // ENTRY main {
+  //   operand = s32[3,3] parameter(0)
+  //   indices = s32[2] parameter(1)
+  //   ROOT gather = s32[2,3] gather(operand, indices),
+  //       output_window_dims={1},
+  //       elided_window_dims={0},
+  //       gather_dims_to_operand_dims={0},
+  //       index_vector_dim=1,
+  //       window_bounds={1, 3}
+  // }
+
+  ComputationBuilder builder(client_, "gather_basic");
+
+  Shape operand_shape = ShapeUtil::MakeShape(S32, {3, 3});
+  Shape indices_shape = ShapeUtil::MakeShape(S32, {2});
+
+  auto operand = builder.Parameter(0, operand_shape, "operand");
+  auto indices = builder.Parameter(1, indices_shape, "indices");
+  GatherDimensionNumbers dim_numbers;
+  dim_numbers.add_output_window_dims(1);
+  dim_numbers.add_elided_window_dims(0);
+  dim_numbers.add_gather_dims_to_operand_dims(0);
+  dim_numbers.set_index_vector_dim(1);
+  builder.Gather(operand, indices, dim_numbers, {1, 3});
+
+  std::vector<int32> expected = {};
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<GlobalData> operand_arg,
+                          client_->TransferToServer(*Literal::CreateR2<int32>(
+                              {{1, 2, 3}, {4, 5, 6}, {7, 8, 9}})));
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::unique_ptr<GlobalData> indices_arg,
+      client_->TransferToServer(*Literal::CreateR1<int32>({0, 2})));
+  TF_ASSERT_OK_AND_ASSIGN(std::vector<xla::DeviceHandle> devices,
+                          client_->GetDeviceHandles(1));
+  xla::ExecutionOptions execution_options = CreateDefaultExecutionOptions();
+  *execution_options.add_device_handles() = devices[0];
+  TF_ASSERT_OK_AND_ASSIGN(Computation computation, builder.Build());
+  std::vector<xla::Client::ComputationInstance> computation_instances = {
+      {computation,
+       {operand_arg.get(), indices_arg.get()},
+       execution_options,
+       /*execution_profile=*/nullptr}};
+  TF_ASSERT_OK_AND_ASSIGN(
+      std::vector<std::unique_ptr<xla::GlobalData>> result_data,
+      client_->ExecuteParallel(computation_instances));
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<Literal> result_literal,
+                          client_->Transfer(*(result_data[0])));
+  LiteralTestUtil::ExpectEqual(
+      *result_literal, *Literal::CreateR2<int32>({{1, 2, 3}, {7, 8, 9}}));
+}
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/util.h b/tensorflow/compiler/xla/util.h
index ff99d3728d..2da9f9ed6f 100644
--- a/tensorflow/compiler/xla/util.h
+++ b/tensorflow/compiler/xla/util.h
@@ -519,6 +519,15 @@ int64 FindIndex(const C& c, Value&& value) {
   auto it = c_find(c, std::forward<Value>(value));
   return std::distance(c.begin(), it);
 }
+
+// Returns true if `x` fits in 32-bits.
+template <typename T>
+bool IsInt32(T x) {
+  // Following conversion rules: "the value is unchanged if it can be
+  // represented in the destination type (and bit-field width); otherwise, the
+  // value is implementation-defined."
+  return static_cast<int32>(x) == x;
+}
 }  // namespace xla
 
 #define XLA_LOG_LINES(SEV, STRING) \
-- 
GitLab


From beaf17d4b2b2e79e97b08b0382b302771ae6081e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 11:21:40 -0700
Subject: [PATCH 246/960] Update version of nsync used by TensorFlow.

The primary change is that on Linux, the C++11 build of nsync will now use
underlying system primitives to implement a semaphore instead of the C++11
primitives, which are currently surprisingly slow on Linux.

PiperOrigin-RevId: 189775201
---
 tensorflow/contrib/cmake/external/nsync.cmake         | 2 +-
 tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt | 8 +++++++-
 tensorflow/contrib/makefile/compile_nsync.sh          | 5 +++--
 tensorflow/workspace.bzl                              | 8 ++++----
 4 files changed, 15 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/nsync.cmake b/tensorflow/contrib/cmake/external/nsync.cmake
index f3a37ff508..b9d1dd88d4 100644
--- a/tensorflow/contrib/cmake/external/nsync.cmake
+++ b/tensorflow/contrib/cmake/external/nsync.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(nsync_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/nsync/public)
 set(nsync_URL https://github.com/google/nsync)
-set(nsync_TAG 8502189abfa44c249c01c2cad64e6ed660a9a668)
+set(nsync_TAG 0559ce013feac8db639ee1bf776aca0325d28777)
 set(nsync_BUILD ${CMAKE_CURRENT_BINARY_DIR}/nsync/src/nsync)
 set(nsync_INSTALL ${CMAKE_CURRENT_BINARY_DIR}/nsync/install)
 
diff --git a/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt b/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt
index aaae18a313..6f059c7225 100644
--- a/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt
+++ b/tensorflow/contrib/cmake/patches/nsync/CMakeLists.txt
@@ -42,7 +42,6 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
   include_directories ("${PROJECT_SOURCE_DIR}/platform/c++11")
   add_definitions ("-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11")
   set (NSYNC_OS_CPP_SRC
-    "platform/c++11/src/nsync_semaphore_mutex.cc"
     "platform/c++11/src/per_thread_waiter.cc"
     "platform/c++11/src/yield.cc"
     "platform/c++11/src/time_rep_timespec.cc"
@@ -52,6 +51,7 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
     include_directories ("${PROJECT_SOURCE_DIR}/platform/win32")
     add_compile_options ("/TP")
     set (NSYNC_OS_SRC
+      "platform/c++11/src/nsync_semaphore_mutex.cc"
       "platform/win32/src/clock_gettime.c"
       "platform/win32/src/pthread_key_win32.cc"
       ${NSYNC_OS_CPP_SRC}
@@ -68,6 +68,7 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
     add_compile_options ("-std=c++11")
     set (NSYNC_OS_SRC
       ${NSYNC_OS_CPP_SRC}
+      "platform/c++11/src/nsync_semaphore_mutex.cc"
       "platform/posix/src/clock_gettime.c"
       "platform/posix/src/nsync_semaphore_mutex.c"
     )
@@ -75,9 +76,11 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
       "platform/posix/src/start_thread.c"
     )
   elseif ("${CMAKE_SYSTEM_NAME}X" STREQUAL "LinuxX")
+    include_directories (BEFORE "${PROJECT_SOURCE_DIR}/platform/c++11.futex")
     include_directories ("${PROJECT_SOURCE_DIR}/platform/posix")
     add_compile_options ("-std=c++11")
     set (NSYNC_OS_SRC
+      "platform/linux/src/nsync_semaphore_futex.c"
       ${NSYNC_OS_CPP_SRC}
     )
     set (NSYNC_TEST_OS_SRC
@@ -87,6 +90,7 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
     include_directories ("${PROJECT_SOURCE_DIR}/platform/posix")
     add_compile_options ("-std=c++11")
     set (NSYNC_OS_SRC
+      "platform/c++11/src/nsync_semaphore_mutex.cc"
       ${NSYNC_OS_CPP_SRC}
     )
     set (NSYNC_TEST_OS_SRC
@@ -96,6 +100,7 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
     include_directories ("${PROJECT_SOURCE_DIR}/platform/posix")
     add_compile_options ("-std=c++11")
     set (NSYNC_OS_SRC
+      "platform/c++11/src/nsync_semaphore_mutex.cc"
       ${NSYNC_OS_CPP_SRC}
     )
     set (NSYNC_TEST_OS_SRC
@@ -105,6 +110,7 @@ if ("${NSYNC_LANGUAGE}X" STREQUAL "c++11X")
     include_directories ("${PROJECT_SOURCE_DIR}/platform/posix")
     add_compile_options ("-std=c++11")
     set (NSYNC_OS_SRC
+      "platform/c++11/src/nsync_semaphore_mutex.cc"
       ${NSYNC_OS_CPP_SRC}
     )
     set (NSYNC_TEST_OS_SRC
diff --git a/tensorflow/contrib/makefile/compile_nsync.sh b/tensorflow/contrib/makefile/compile_nsync.sh
index 7927997678..e8c6edd7ba 100755
--- a/tensorflow/contrib/makefile/compile_nsync.sh
+++ b/tensorflow/contrib/makefile/compile_nsync.sh
@@ -109,17 +109,18 @@ for arch in $archs; do
         linux)  makefile='
                         CC=${CC_PREFIX} g++
                         PLATFORM_CPPFLAGS=-DNSYNC_USE_CPP11_TIMEPOINT -DNSYNC_ATOMIC_CPP11 \
+                                          -I../../platform/c++11.futex \
                                           -I../../platform/c++11 -I../../platform/gcc \
                                           -I../../platform/posix -pthread
                         PLATFORM_CFLAGS=-std=c++11 -Werror -Wall -Wextra -pedantic
                         PLATFORM_LDFLAGS=-pthread
                         MKDEP=${CC} -M -std=c++11
-                        PLATFORM_C=../../platform/c++11/src/nsync_semaphore_mutex.cc \
+                        PLATFORM_C=../../platform/linux/src/nsync_semaphore_futex.c \
                                    ../../platform/c++11/src/per_thread_waiter.cc \
                                    ../../platform/c++11/src/yield.cc \
                                    ../../platform/c++11/src/time_rep_timespec.cc \
                                    ../../platform/c++11/src/nsync_panic.cc
-                        PLATFORM_OBJS=nsync_semaphore_mutex.o per_thread_waiter.o yield.o \
+                        PLATFORM_OBJS=nsync_semaphore_futex.o per_thread_waiter.o yield.o \
                                       time_rep_timespec.o nsync_panic.o
                         TEST_PLATFORM_C=../../platform/c++11/src/start_thread.cc
                         TEST_PLATFORM_OBJS=start_thread.o
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index db70e4515b..cf1611a883 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -388,11 +388,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "nsync",
       urls = [
-          "https://mirror.bazel.build/github.com/google/nsync/archive/8502189abfa44c249c01c2cad64e6ed660a9a668.tar.gz",
-          "https://github.com/google/nsync/archive/8502189abfa44c249c01c2cad64e6ed660a9a668.tar.gz",
+          "https://mirror.bazel.build/github.com/google/nsync/archive/0559ce013feac8db639ee1bf776aca0325d28777.tar.gz",
+          "https://github.com/google/nsync/archive/0559ce013feac8db639ee1bf776aca0325d28777.tar.gz",
       ],
-      sha256 = "51f81ff4202bbb820cdbedc061bd2eb6765f2b5c06489e7a8694bedac329e8f8",
-      strip_prefix = "nsync-8502189abfa44c249c01c2cad64e6ed660a9a668",
+      sha256 = "6284454c5cd8b1dae2eeb8cf5eb63004de930b5427ed5f6b1aa793513df6b361",
+      strip_prefix = "nsync-0559ce013feac8db639ee1bf776aca0325d28777",
   )
 
   tf_http_archive(
-- 
GitLab


From 4e5900eb874668e569cfa1b75c463a9f0b15738f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 11:27:54 -0700
Subject: [PATCH 247/960] The Quantized BroadcastSub portion of #17123

PiperOrigin-RevId: 189776376
---
 .../internal/optimized/optimized_ops.h        |  59 ++++++++++
 .../internal/reference/reference_ops.h        |  59 ++++++++++
 tensorflow/contrib/lite/kernels/sub.cc        |  56 ++++++++++
 tensorflow/contrib/lite/kernels/sub_test.cc   | 101 ++++++++++++++++++
 4 files changed, 275 insertions(+)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index 004433498d..f7840258ec 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -2280,6 +2280,65 @@ void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+inline void BroadcastSub(int left_shift, const uint8* input1_data,
+                         const Dims<4>& input1_dims, int32 input1_offset,
+                         int32 input1_multiplier, int input1_shift,
+                         const uint8* input2_data, const Dims<4>& input2_dims,
+                         int32 input2_offset, int32 input2_multiplier,
+                         int input2_shift, int32 output_offset,
+                         int32 output_multiplier, int output_shift,
+                         int32 output_activation_min,
+                         int32 output_activation_max, uint8* output_data,
+                         const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          const int32 input1_val =
+              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
+          const int32 input2_val =
+              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
+          const int32 shifted_input1_val = input1_val * (1 << left_shift);
+          const int32 shifted_input2_val = input2_val * (1 << left_shift);
+          const int32 scaled_input1_val =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  shifted_input1_val, input1_multiplier, input1_shift);
+          const int32 scaled_input2_val =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  shifted_input2_val, input2_multiplier, input2_shift);
+          const int32 raw_sub = scaled_input1_val - scaled_input2_val;
+          const int32 raw_output =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  raw_sub, output_multiplier, output_shift) +
+              output_offset;
+          const int32 clamped_output =
+              std::min(output_activation_max,
+                       std::max(output_activation_min, raw_output));
+          output_data[Offset(output_dims, c, x, y, b)] =
+              static_cast<uint8>(clamped_output);
+        }
+      }
+    }
+  }
+}
+
 template <FusedActivationFunctionType Ac, typename Scalar>
 void Concatenation(int concat_dim, const Scalar* const* input_data,
                    const Dims<4>* const* input_dims, int inputs_count,
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 527276f7bd..472ddc60df 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -1461,6 +1461,65 @@ void BroadcastSub(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+inline void BroadcastSub(int left_shift, const uint8* input1_data,
+                         const Dims<4>& input1_dims, int32 input1_offset,
+                         int32 input1_multiplier, int input1_shift,
+                         const uint8* input2_data, const Dims<4>& input2_dims,
+                         int32 input2_offset, int32 input2_multiplier,
+                         int input2_shift, int32 output_offset,
+                         int32 output_multiplier, int output_shift,
+                         int32 output_activation_min,
+                         int32 output_activation_max, uint8* output_data,
+                         const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("BroadcastSub/8bit");
+
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  // In Tensorflow, the dimensions are canonically named (batch_number, row,
+  // col, channel), with extents (batches, height, width, depth), with the
+  // trailing dimension changing most rapidly (channels has the smallest stride,
+  // typically 1 element).
+  //
+  // In generated C code, we store arrays with the dimensions reversed. The
+  // first dimension has smallest stride.
+  //
+  // We name our variables by their Tensorflow convention, but generate C code
+  // nesting loops such that the innermost loop has the smallest stride for the
+  // best cache behavior.
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          const int32 input1_val =
+              input1_offset + input1_data[SubscriptToIndex(desc1, c, x, y, b)];
+          const int32 input2_val =
+              input2_offset + input2_data[SubscriptToIndex(desc2, c, x, y, b)];
+          const int32 shifted_input1_val = input1_val * (1 << left_shift);
+          const int32 shifted_input2_val = input2_val * (1 << left_shift);
+          const int32 scaled_input1_val =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  shifted_input1_val, input1_multiplier, input1_shift);
+          const int32 scaled_input2_val =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  shifted_input2_val, input2_multiplier, input2_shift);
+          const int32 raw_sub = scaled_input1_val - scaled_input2_val;
+          const int32 raw_output =
+              MultiplyByQuantizedMultiplierSmallerThanOne(
+                  raw_sub, output_multiplier, output_shift) +
+              output_offset;
+          const int32 clamped_output =
+              std::min(output_activation_max,
+                       std::max(output_activation_min, raw_output));
+          output_data[Offset(output_dims, c, x, y, b)] =
+              static_cast<uint8>(clamped_output);
+        }
+      }
+    }
+  }
+}
+
 template <FusedActivationFunctionType Ac, typename Scalar>
 void Concatenation(int concat_dim, const Scalar* const* input_data,
                    const Dims<4>* const* input_dims, int inputs_count,
diff --git a/tensorflow/contrib/lite/kernels/sub.cc b/tensorflow/contrib/lite/kernels/sub.cc
index c15a7a50a4..66b06aeaec 100644
--- a/tensorflow/contrib/lite/kernels/sub.cc
+++ b/tensorflow/contrib/lite/kernels/sub.cc
@@ -106,6 +106,59 @@ void EvalFloat(TfLiteContext* context, TfLiteNode* node,
 #undef TF_LITE_SUB
 }
 
+template <KernelType kernel_type>
+void EvalQuantized(TfLiteContext* context, TfLiteNode* node,
+                   TfLiteSubParams* params, const OpData* data,
+                   TfLiteTensor* input1, TfLiteTensor* input2,
+                   TfLiteTensor* output) {
+  auto input1_offset = -input1->params.zero_point;
+  auto input2_offset = -input2->params.zero_point;
+  auto output_offset = output->params.zero_point;
+  const int left_shift = 20;
+  const double twice_max_input_scale =
+      2 * std::max(input1->params.scale, input2->params.scale);
+  const double real_input1_multiplier =
+      input1->params.scale / twice_max_input_scale;
+  const double real_input2_multiplier =
+      input2->params.scale / twice_max_input_scale;
+  const double real_output_multiplier =
+      twice_max_input_scale / ((1 << left_shift) * output->params.scale);
+
+  int32 input1_multiplier;
+  int input1_shift;
+  QuantizeMultiplierSmallerThanOne(real_input1_multiplier, &input1_multiplier,
+                                   &input1_shift);
+  int32 input2_multiplier;
+  int input2_shift;
+  QuantizeMultiplierSmallerThanOne(real_input2_multiplier, &input2_multiplier,
+                                   &input2_shift);
+  int32 output_multiplier;
+  int output_shift;
+  QuantizeMultiplierSmallerThanOne(real_output_multiplier, &output_multiplier,
+                                   &output_shift);
+
+  int32 output_activation_min, output_activation_max;
+  CalculateActivationRangeUint8(params->activation, output,
+                                &output_activation_min, &output_activation_max);
+
+#define TF_LITE_SUB(type, opname)                                            \
+  type::opname(left_shift, GetTensorData<uint8_t>(input1),                   \
+               GetTensorDims(input1), input1_offset, input1_multiplier,      \
+               input1_shift, GetTensorData<uint8_t>(input2),                 \
+               GetTensorDims(input2), input2_offset, input2_multiplier,      \
+               input2_shift, output_offset, output_multiplier, output_shift, \
+               output_activation_min, output_activation_max,                 \
+               GetTensorData<uint8_t>(output), GetTensorDims(output));
+  // The quantized version of Sub doesn't support activations, so we
+  // always use BroadcastSub.
+  if (kernel_type == kReference) {
+    TF_LITE_SUB(reference_ops, BroadcastSub);
+  } else {
+    TF_LITE_SUB(optimized_ops, BroadcastSub);
+  }
+#undef TF_LITE_SUB
+}
+
 template <KernelType kernel_type>
 TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   auto* params = reinterpret_cast<TfLiteSubParams*>(node->builtin_data);
@@ -117,6 +170,9 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 
   if (output->type == kTfLiteFloat32) {
     EvalFloat<kernel_type>(context, node, params, data, input1, input2, output);
+  } else if (output->type == kTfLiteUInt8) {
+    EvalQuantized<kernel_type>(context, node, params, data, input1, input2,
+                               output);
   } else {
     context->ReportError(context, "Inputs and outputs not all float types.");
     return kTfLiteError;
diff --git a/tensorflow/contrib/lite/kernels/sub_test.cc b/tensorflow/contrib/lite/kernels/sub_test.cc
index fdbb4243bb..ff07aeec49 100644
--- a/tensorflow/contrib/lite/kernels/sub_test.cc
+++ b/tensorflow/contrib/lite/kernels/sub_test.cc
@@ -52,6 +52,23 @@ class FloatSubOpModel : public BaseSubOpModel {
   std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
 };
 
+class QuantizedSubOpModel : public BaseSubOpModel {
+ public:
+  using BaseSubOpModel::BaseSubOpModel;
+
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
+                               GetScale(output_), GetZeroPoint(output_));
+  }
+};
+
+// for quantized Sub, the error shouldn't exceed 2*step
+float GetTolerance(int min, int max) {
+  float kQuantizedStep = (max - min) / 255.0;
+  float kQuantizedTolerance = 2.0 * kQuantizedStep;
+  return kQuantizedTolerance;
+}
+
 TEST(FloatSubOpModel, NoActivation) {
   FloatSubOpModel m({TensorType_FLOAT32, {1, 2, 2, 1}},
                     {TensorType_FLOAT32, {1, 2, 2, 1}},
@@ -108,6 +125,90 @@ TEST(FloatSubOpModel, WithBroadcast) {
   }
 }
 
+TEST(QuantizedSubOpModel, QuantizedTestsNoActivation) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<std::initializer_list<float>> inputs1 = {
+      {0.1, 0.2, 0.3, 0.4}, {-0.2, 0.2, 0.4, 0.7}, {-0.01, 0.2, 0.7, 0.3}};
+  std::vector<std::initializer_list<float>> inputs2 = {
+      {0.6, 0.4, 0.3, 0.1}, {0.6, 0.4, 0.5, -0.2}, {0.6, 0.4, -0.18, 0.5}};
+  std::vector<std::initializer_list<float>> results = {
+      {-0.5, -0.2, 0.0, 0.3},
+      {-0.8, -0.2, -0.1, 0.9},
+      {-0.61, -0.2, 0.88, -0.2}};
+  for (int i = 0; i < inputs1.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0},
+                          {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0},
+                          {TensorType_UINT8, {}, -1.0, 1.0},
+                          ActivationFunctionType_NONE);
+    m.QuantizeAndPopulate<uint8_t>(m.input1(), inputs1[i]);
+    m.QuantizeAndPopulate<uint8_t>(m.input2(), inputs2[i]);
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear(
+                                              results[i], kQuantizedTolerance)))
+        << "With test number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedTestsActivationRELU_N1_TO_1) {
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::vector<std::initializer_list<float>> inputs1 = {{-0.8, 0.2, 0.9, 0.7},
+                                                       {-0.8, 0.2, 0.7, 0.5}};
+  std::vector<std::initializer_list<float>> inputs2 = {{0.6, 0.4, 0.9, -0.8},
+                                                       {0.6, 0.4, -0.8, 0.3}};
+  std::vector<std::initializer_list<float>> results = {{-1.0, -0.2, 0.0, 1.0},
+                                                       {-1.0, -0.2, 1.0, 0.2}};
+  for (int i = 0; i < inputs1.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0},
+                          {TensorType_UINT8, {1, 2, 2, 1}, -1.0, 1.0},
+                          {TensorType_UINT8, {}, -1.0, 1.0},
+                          ActivationFunctionType_RELU_N1_TO_1);
+    m.QuantizeAndPopulate<uint8_t>(m.input1(), inputs1[i]);
+    m.QuantizeAndPopulate<uint8_t>(m.input2(), inputs2[i]);
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear(
+                                              results[i], kQuantizedTolerance)))
+        << "With test number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedVariousInputShapes) {
+  float kQuantizedTolerance = GetTolerance(-3.0, 3.0);
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_UINT8, test_shapes[i], -3.0, 3.0},
+                          {TensorType_UINT8, test_shapes[i], -3.0, 3.0},
+                          {TensorType_UINT8, {}, -3.0, 3.0},
+                          ActivationFunctionType_NONE);
+    m.QuantizeAndPopulate<uint8_t>(m.input1(), {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0});
+    m.QuantizeAndPopulate<uint8_t>(m.input2(), {0.1, 0.3, 0.3, 0.5, 1.1, 0.1});
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutput(),
+                ElementsAreArray(ArrayFloatNear(
+                    {-2.1, -0.1, 0.4, 0.3, 0.0, 1.9}, kQuantizedTolerance)))
+        << "With shape number " << i;
+  }
+}
+
+TEST(QuantizedSubOpModel, QuantizedWithBroadcast) {
+  float kQuantizedTolerance = GetTolerance(-3.0, 3.0);
+  std::vector<std::initializer_list<int>> test_shapes = {
+      {6}, {2, 3}, {2, 1, 3}, {1, 3, 1, 2}};
+  for (int i = 0; i < test_shapes.size(); ++i) {
+    QuantizedSubOpModel m({TensorType_UINT8, test_shapes[i], -3.0, 3.0},
+                          {TensorType_UINT8, {}, -3.0, 3.0},
+                          {TensorType_UINT8, {}, -3.0, 3.0},
+                          ActivationFunctionType_NONE);
+    m.QuantizeAndPopulate<uint8_t>(m.input1(), {-2.0, 0.2, 0.7, 0.8, 1.1, 2.0});
+    m.QuantizeAndPopulate<uint8_t>(m.input2(), {0.7});
+    m.Invoke();
+    EXPECT_THAT(m.GetDequantizedOutput(),
+                ElementsAreArray(ArrayFloatNear(
+                    {-2.7, -0.5, 0.0, 0.1, 0.4, 1.3}, kQuantizedTolerance)))
+        << "With shape number " << i;
+  }
+}
+
 }  // namespace
 }  // namespace tflite
 int main(int argc, char** argv) {
-- 
GitLab


From f57f7d09eeb7402f2455564fafbcebf7ac4b8fe3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 11:36:07 -0700
Subject: [PATCH 248/960] Don't run tensorflow/python:function_test under asan.

It gets flaky timeouts.

PiperOrigin-RevId: 189777986
---
 tensorflow/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 7ece482ea7..11195b3565 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1034,6 +1034,7 @@ cuda_py_tests(
         "//tensorflow/core:protos_all_py",
     ],
     shard_count = 10,
+    tags = ["noasan"],
 )
 
 py_test(
-- 
GitLab


From 15d6e8310e1f2ffaa901110903ce7403717b4d2b Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 20 Mar 2018 11:45:23 -0700
Subject: [PATCH 249/960] Improved accuracy of op_level_cost_estimator
 (QuantizeV2, Dequantize, Gather).

PiperOrigin-RevId: 189779691
---
 .../grappler/costs/op_level_cost_estimator.cc | 81 ++++++++++++++-----
 .../grappler/costs/op_level_cost_estimator.h  | 16 ++--
 .../costs/op_level_cost_estimator_test.cc     | 35 +++++++-
 3 files changed, 103 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 29ef317e46..84ad8a3e84 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/clusters/utils.h"
 
 namespace tensorflow {
@@ -46,6 +47,7 @@ constexpr char kShape[] = "Shape";
 constexpr char kSize[] = "Size";
 constexpr char kStopGradient[] = "StopGradient";
 constexpr char kPreventGradient[] = "PreventGradient";
+constexpr char kGather[] = "Gather";
 
 static const Costs::Duration kMinComputeTime(1);
 
@@ -167,6 +169,8 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
 
+      {kGather, wrap(&OpLevelCostEstimator::PredictGather)},
+
       {kPlaceholder, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kIdentity, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kRefIdentity, wrap(&OpLevelCostEstimator::PredictIdentity)},
@@ -184,6 +188,17 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       {kShape, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kSize, wrap(&OpLevelCostEstimator::PredictMetadata)}};
 
+  // Quantize = apply min and max bounds, multiply by scale factor and round.
+  const int quantize_v2_cost =
+      Eigen::internal::functor_traits<
+          Eigen::internal::scalar_product_op<float>>::Cost +
+      Eigen::internal::functor_traits<
+          Eigen::internal::scalar_max_op<float>>::Cost +
+      Eigen::internal::functor_traits<
+          Eigen::internal::scalar_min_op<float>>::Cost +
+      Eigen::internal::functor_traits<
+          Eigen::internal::scalar_round_op<float>>::Cost;
+
   elementwise_ops_ = {
       // Unary ops alphabetically sorted
       {"Acos", Eigen::internal::functor_traits<
@@ -200,6 +215,8 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
                    Eigen::internal::scalar_ceil_op<float>>::Cost},
       {"Cos", Eigen::internal::functor_traits<
                   Eigen::internal::scalar_cos_op<float>>::Cost},
+      {"Dequantize", Eigen::internal::functor_traits<
+                         Eigen::internal::scalar_product_op<float>>::Cost},
       {"Erf", 1},
       {"Erfc", 1},
       {"Exp", Eigen::internal::functor_traits<
@@ -218,6 +235,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
                     Eigen::internal::scalar_log1p_op<float>>::Cost},
       {"Neg", Eigen::internal::functor_traits<
                   Eigen::internal::scalar_opposite_op<float>>::Cost},
+      {"QuantizeV2", quantize_v2_cost},
       {"Reciprocal", Eigen::internal::functor_traits<
                          Eigen::internal::scalar_inverse_op<float>>::Cost},
       {"Rint", 1},
@@ -411,28 +429,33 @@ Costs OpLevelCostEstimator::PredictCostOfAnUnknownOp(
 }
 
 Costs OpLevelCostEstimator::PredictOpCountBasedCost(
-    double operations, const OpInfo& op_features) const {
-  DeviceInfo device_perf = GetDeviceInfo(op_features.device());
-  if (device_perf.gigaops <= 0 || device_perf.gb_per_sec <= 0) {
-    VLOG(1) << "BAD DEVICE. Op:" << op_features.op()
-            << " device type:" << op_features.device().type()
-            << " device model:" << op_features.device().model();
-  }
+    double operations, const OpInfo& op_info) const {
+  bool unknown_shapes = false;
+  const double input_size = CalculateInputSize(op_info, &unknown_shapes);
+  const double output_size = CalculateOutputSize(op_info, &unknown_shapes);
+  const double total_io_bytes = input_size + output_size;
+  Costs costs = PredictOpCountBasedCost(operations, total_io_bytes, op_info);
+  costs.inaccurate = unknown_shapes;
+  costs.max_memory = output_size;
+  return costs;
+}
 
-  Costs::NanoSeconds compute_cost(std::ceil(operations / device_perf.gigaops));
-  VLOG(1) << "Op:" << op_features.op() << " GOps:" << operations / 1e9
-          << " Execution Time (ns):" << compute_cost.count();
+Costs OpLevelCostEstimator::PredictOpCountBasedCost(
+    double operations, double total_io_bytes, const OpInfo& op_info) const {
+  const DeviceInfo device_info = GetDeviceInfo(op_info.device());
+  if (device_info.gigaops <= 0 || device_info.gb_per_sec <= 0) {
+    VLOG(1) << "BAD DEVICE. Op:" << op_info.op()
+            << " device type:" << op_info.device().type()
+            << " device model:" << op_info.device().model();
+  }
 
-  bool found_unknown_shapes = false;
-  const double total_input_size =
-      CalculateInputSize(op_features, &found_unknown_shapes);
-  const double total_output_size =
-      CalculateOutputSize(op_features, &found_unknown_shapes);
-  const double total_io_size = total_input_size + total_output_size;
+  Costs::NanoSeconds compute_cost(std::ceil(operations / device_info.gigaops));
+  VLOG(1) << "Op:" << op_info.op() << " GOps:" << operations / 1e9
+          << " Compute Time (ns):" << compute_cost.count();
 
   Costs::NanoSeconds memory_cost(
-      std::ceil(total_io_size / device_perf.gb_per_sec));
-  VLOG(1) << "Op:" << op_features.op() << " Size (KB):" << (total_io_size) / 1e3
+      std::ceil(total_io_bytes / device_info.gb_per_sec));
+  VLOG(1) << "Op:" << op_info.op() << " Size (KB):" << (total_io_bytes) / 1e3
           << " Memory Time (ns):" << memory_cost.count();
 
   Costs costs;
@@ -443,8 +466,6 @@ Costs OpLevelCostEstimator::PredictOpCountBasedCost(
   } else {
     costs.execution_time = compute_cost + memory_cost;
   }
-  costs.inaccurate = found_unknown_shapes;
-  costs.max_memory = total_output_size;
   return costs;
 }
 
@@ -867,7 +888,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
 
 int64 OpLevelCostEstimator::CalculateTensorElementCount(
     const OpInfo::TensorProperties& tensor, bool* found_unknown_shapes) const {
-  VLOG(2) << "   with " << tensor.dtype() << " tensor of shape "
+  VLOG(2) << "   with " << DataTypeString(tensor.dtype()) << " tensor of shape "
           << tensor.shape().DebugString();
   int64 tensor_size = 1;
   int num_dims = std::max(1, tensor.shape().dim_size());
@@ -1028,5 +1049,23 @@ Costs OpLevelCostEstimator::PredictMetadata(const OpContext& op_context) const {
   return costs;
 }
 
+Costs OpLevelCostEstimator::PredictGather(const OpContext& op_context) const {
+  // Gather op can have a very large input, but only the size of the output
+  // matters, because indices may select only a very small subset of input.
+
+  const auto& op_info = op_context.op_info;
+
+  bool unknown_shapes = false;
+  const int64 op_count =
+      CalculateTensorElementCount(op_info.outputs(0), &unknown_shapes);
+  const double output_size = CalculateOutputSize(op_info, &unknown_shapes);
+  const double total_io = 2 * output_size;
+  Costs costs = PredictOpCountBasedCost(op_count, total_io, op_info);
+  costs.inaccurate = unknown_shapes;
+  costs.max_memory = output_size;
+
+  return costs;
+}
+
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index 7bb530fe31..e5dd31a7a2 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -51,10 +51,15 @@ class OpLevelCostEstimator {
   // Predict cost of an op for which no accurate estimator is defined.
   Costs PredictCostOfAnUnknownOp(const OpContext& op_context) const;
 
-  // Naive cost estimate based on operations divided by device ops/sec,
-  // and input/output tensor sizes.
-  Costs PredictOpCountBasedCost(double operations,
-                                const OpInfo& op_features) const;
+  // Naive cost estimate based on the given operations count and total
+  // input/output tensor sizes of the given op_info combined.
+  Costs PredictOpCountBasedCost(double operations, const OpInfo& op_info) const;
+
+  // Naive cost estimate based on the given operations count and the given total
+  // io size in bytes. Sizes of op_info inputs and outputs are not taken into
+  // consideration.
+  Costs PredictOpCountBasedCost(double operations, double total_io_bytes,
+                                const OpInfo& op_info) const;
 
   // This family of routines counts the number of operations to perform the
   // specified TensorFlow Op.
@@ -125,7 +130,7 @@ class OpLevelCostEstimator {
   // implementation just divides the operations to
   // perform the op (from the "Count" routines,
   // above) by the device peak operations per
-  // second. Override to supply a better estimate.
+  // second.
   // Implementation of costs other than
   // execution_time is optional, depending on the
   // device.
@@ -139,6 +144,7 @@ class OpLevelCostEstimator {
   Costs PredictVariable(const OpContext& op_context) const;
   Costs PredictBatchMatMul(const OpContext& op_context) const;
   Costs PredictMetadata(const OpContext& op_context) const;
+  Costs PredictGather(const OpContext& op_context) const;
 
   // Utility function for safe division. Returns 0
   // if rhs is 0 or negative.
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 4790b9bab2..d5360cba24 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -75,8 +75,8 @@ OpContext DescribeMatMulUnknownShape() {
 // Wrangles the minimum number of proto fields to set up an input of
 // arbitrary rank and type.
 void DescribeArbitraryRankInput(const std::vector<int>& dims, DataType dtype,
-                                OpInfo* op_features) {
-  auto input = op_features->add_inputs();
+                                OpInfo* op_info) {
+  auto input = op_info->add_inputs();
   input->set_dtype(dtype);
   auto shape = input->mutable_shape();
   for (auto d : dims) {
@@ -84,6 +84,18 @@ void DescribeArbitraryRankInput(const std::vector<int>& dims, DataType dtype,
   }
 }
 
+// Wrangles the minimum number of proto fields to set up an output of
+// arbitrary rank and type.
+void DescribeArbitraryRankOutput(const std::vector<int>& dims, DataType dtype,
+                                 OpInfo* op_info) {
+  auto output = op_info->add_outputs();
+  output->set_dtype(dtype);
+  auto shape = output->mutable_shape();
+  for (auto d : dims) {
+    shape->add_dim()->set_size(d);
+  }
+}
+
 // Returns an OpInfo for a BatchMatMul
 OpContext DescribeBatchMatMul(const std::vector<int>& dims_a,
                               const std::vector<int>& dims_b) {
@@ -200,6 +212,23 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
   OpLevelCostEstimator estimator_;
 };
 
+TEST_F(OpLevelCostEstimatorTest, TestGatherCosts) {
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  op_context.op_info.set_op("Gather");
+
+  // Huge first input shouldn't affect Gather execution and memory costs.
+  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+  DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
+
+  auto cost = estimator_.PredictCosts(op_context);
+  EXPECT_EQ(Costs::Duration(128), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(16), cost.compute_time);
+  EXPECT_EQ(Costs::Duration(144), cost.execution_time);
+  EXPECT_FALSE(cost.inaccurate);
+}
+
 TEST_F(OpLevelCostEstimatorTest, BiasAddExecutionTime) {
   auto cost = PredictCosts(DescribeBiasAdd(1000, 10));
   EXPECT_EQ(Costs::Duration(8400), cost.memory_time);
@@ -354,7 +383,7 @@ TEST_F(OpLevelCostEstimatorTest, GetTensorShapeProtoFromTensorProto) {
   TensorProto tensor_proto;
   TensorShapeProto tensor_shape_proto;
 
-  // Dimention larger than max value; should fail while converting to Tensor
+  // Dimension larger than max value; should fail while converting to Tensor
   // class.
   tensor_proto.mutable_tensor_shape()->add_dim()->set_size(255);
   EXPECT_FALSE(
-- 
GitLab


From 98c955ee73e95591b00793f8fe9de5b1d588a0ea Mon Sep 17 00:00:00 2001
From: jjsjann123 <alex.jann2012@gmail.com>
Date: Tue, 20 Mar 2018 12:45:54 -0700
Subject: [PATCH 250/960] improve fp16 tftrt prediction (#17857)

delay fp32 to fp16 conversion to reduce accumulated rounding error
---
 .../contrib/tensorrt/convert/convert_nodes.cc | 63 ++++++++++---------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 7f4b57f9f4..979b5648c2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -547,6 +547,19 @@ class Converter {
   }
 };
 
+TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx,
+                                    const TRT_ShapedWeights& weights_src) {
+  auto dtype_new = tensorflow::DataType::DT_HALF;
+  TRT_ShapedWeights weights =
+      ctx.get_temp_weights(dtype_new, weights_src.shape_);
+  const float* src = static_cast<const float*>(weights_src.GetValues());
+  Eigen::half* dst = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(weights.GetValues()));
+  for (int64_t i = 0; i < weights_src.count(); i++) {
+    dst[i] = Eigen::half_impl::float_to_half_rtne(src[i]);
+  }
+  return weights;
+}
 // ****************************************************************************
 // Constant folding functions
 // TODO(jie): once optimizer kicks in, we should have done constant folding
@@ -956,6 +969,10 @@ tensorflow::Status BinaryTensorOpWeight(
     }
   }
 
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, weights);
+  }
+
   // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
@@ -1022,6 +1039,10 @@ tensorflow::Status ConvertConv2DHelper(
   VLOG(2) << "groups count: " << num_groups;
 
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights_rsck = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
   TRT_ShapedWeights biases(weights.type_);
@@ -1292,8 +1313,11 @@ tensorflow::Status ConvertScale(Converter& ctx,
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
-  // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights empty_weights(weights.type_);
 
   TFAttrs attrs(node_def);
@@ -1388,33 +1412,16 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      auto half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      half_tensor.device(defd) =
-          tensor.flat<float>().template cast<Eigen::half>();
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<float> tensor_data(
-          weights_tensor.float_val().begin(),
-          weights_tensor.float_val()
-              .end());  //  make a local copy first to flatten
-      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<float> tensor_data(
+        weights_tensor.float_val().begin(),
+        weights_tensor.float_val()
+            .end());  //  make a local copy first to flatten
+    memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.int_val().empty()) {
     VLOG(2) << "int!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
-- 
GitLab


From be7adf828e5a23cdd883b0d43756b7f123c4088a Mon Sep 17 00:00:00 2001
From: Sanjoy Das <sanjoy@google.com>
Date: Tue, 20 Mar 2018 12:51:50 -0700
Subject: [PATCH 251/960] [TF:XLA] Bump open source llvm revision to r327958

PiperOrigin-RevId: 189792132
---
 tensorflow/workspace.bzl | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index cf1611a883..675acbe5f6 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -475,11 +475,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/cfb3cd346a75b17856c4e2ba6365e15d9ab0c763.tar.gz",
-          "https://github.com/llvm-mirror/llvm/archive/cfb3cd346a75b17856c4e2ba6365e15d9ab0c763.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/1c3cdea2f181d8e14ee184466c5fb237f1b4cda8.tar.gz",
+          "https://github.com/llvm-mirror/llvm/archive/1c3cdea2f181d8e14ee184466c5fb237f1b4cda8.tar.gz",
       ],
-      sha256 = "2cf79b1891926b7af6173c1031d040fc07b2682ff66039c5822e074566c48956",
-      strip_prefix = "llvm-cfb3cd346a75b17856c4e2ba6365e15d9ab0c763",
+      sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
+      strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
       build_file = str(Label("//third_party/llvm:llvm.BUILD")),
   )
 
-- 
GitLab


From 13b993095f155bd4dd7fc3b057a7b5043ef0a06c Mon Sep 17 00:00:00 2001
From: Mingxing Tan <tanmingxing@google.com>
Date: Tue, 20 Mar 2018 12:54:01 -0700
Subject: [PATCH 252/960] Add broadcasting support for fused add or sub.

PiperOrigin-RevId: 189792542
---
 .../fuse_binary_into_preceding_affine.cc         | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/fuse_binary_into_preceding_affine.cc b/tensorflow/contrib/lite/toco/graph_transformations/fuse_binary_into_preceding_affine.cc
index 5b57178b18..76c6be00d4 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/fuse_binary_into_preceding_affine.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/fuse_binary_into_preceding_affine.cc
@@ -50,7 +50,17 @@ void FuseAddOrSubParamsIntoPrecedingAffine(Model* model, Operator* preceding_op,
 
   // TODO(b/62904716): Bias array should become 1-D when padding removed.
   const int depth = bias_shape.dims(bias_shape.dimensions_count() - 1);
-  CHECK_EQ(depth, operand_shape.dims(operand_shape.dimensions_count() - 1));
+  int operand_channel_increment = 0;
+  if (operand_shape.dimensions_count() >= 1 &&
+      operand_shape.dims(operand_shape.dimensions_count() - 1) ==
+          bias_shape.dims(bias_shape.dimensions_count() - 1)) {
+    operand_channel_increment = 1;
+  } else if (operand_shape.dimensions_count() == 0 ||
+             operand_shape.dims(operand_shape.dimensions_count() - 1) == 1) {
+    operand_channel_increment = 0;
+  } else {
+    LOG(FATAL) << "Operand shape mismatch.";
+  }
 
   enum class OpType { BiasPlusOperand, BiasMinusOperand, OperandMinusBias };
 
@@ -60,9 +70,10 @@ void FuseAddOrSubParamsIntoPrecedingAffine(Model* model, Operator* preceding_op,
                                   ? OpType::BiasMinusOperand
                                   : OpType::OperandMinusBias;
 
+  int operand_channel = 0;
   for (int i = 0; i < depth; i++) {
     float& bias_val = bias_data[i];
-    const float operand_val = operand_data[i];
+    const float operand_val = operand_data[operand_channel];
     if (optype == OpType::BiasPlusOperand) {
       bias_val += operand_val;
     } else if (optype == OpType::BiasMinusOperand) {
@@ -72,6 +83,7 @@ void FuseAddOrSubParamsIntoPrecedingAffine(Model* model, Operator* preceding_op,
     } else {
       LOG(FATAL) << "Should not get here.";
     }
+    operand_channel += operand_channel_increment;
   }
 }
 
-- 
GitLab


From e4313551d184932c9a135d4edacf42711e5b3483 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 12:56:51 -0700
Subject: [PATCH 253/960] TFBT: Pass label_keys to the head class constructor.

PiperOrigin-RevId: 189793004
---
 tensorflow/contrib/boosted_trees/estimator_batch/estimator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
index 01752416b3..70454aa6db 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/estimator.py
@@ -81,7 +81,8 @@ class GradientBoostedDecisionTreeClassifier(estimator.Estimator):
         n_classes=n_classes,
         weight_column_name=weight_column_name,
         enable_centered_bias=False,
-        loss_fn=loss_fn)
+        loss_fn=loss_fn,
+        label_keys=label_keys)
     if learner_config.num_classes == 0:
       learner_config.num_classes = n_classes
     elif learner_config.num_classes != n_classes:
-- 
GitLab


From 278ead7d06e427df09f910031cb9195c8a4da559 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 13:02:36 -0700
Subject: [PATCH 254/960] In allocate_transient_arrays.cc, was not handling the
 case where the same array occurs more than once in the list of inputs or
 outputs of a node.

PiperOrigin-RevId: 189794090
---
 .../lite/toco/allocate_transient_arrays.cc    | 32 +++++++++++++++----
 1 file changed, 26 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/allocate_transient_arrays.cc b/tensorflow/contrib/lite/toco/allocate_transient_arrays.cc
index 49cc1fc2aa..621fbcb98d 100644
--- a/tensorflow/contrib/lite/toco/allocate_transient_arrays.cc
+++ b/tensorflow/contrib/lite/toco/allocate_transient_arrays.cc
@@ -248,29 +248,49 @@ void AllocateTransientArrays(Model* model,
        op_index++) {
     const auto& op = model->operators[op_index];
     // Allocate those arrays whose lifespan starts exactly here.
+    std::vector<string> arrays_to_allocate;
     for (const auto& input : op->inputs) {
       if (StartsAt(array_lifespans[input], op_index)) {
-        AllocateTransientArray(*model, input, &allocator,
-                               transient_data_alignment);
+        if (std::find(arrays_to_allocate.begin(), arrays_to_allocate.end(),
+                      input) == arrays_to_allocate.end()) {
+          arrays_to_allocate.push_back(input);
+        }
       }
     }
     for (const auto& output : op->outputs) {
       if (StartsAt(array_lifespans[output], op_index)) {
-        AllocateTransientArray(*model, output, &allocator,
-                               transient_data_alignment);
+        if (std::find(arrays_to_allocate.begin(), arrays_to_allocate.end(),
+                      output) == arrays_to_allocate.end()) {
+          arrays_to_allocate.push_back(output);
+        }
       }
     }
+    for (const string& array : arrays_to_allocate) {
+      AllocateTransientArray(*model, array, &allocator,
+                             transient_data_alignment);
+    }
+
     // Deallocate those arrays whose lifespan ends exactly here.
+    std::vector<string> arrays_to_deallocate;
     for (const auto& input : op->inputs) {
       if (EndsAt(array_lifespans[input], op_index)) {
-        DeallocateTransientArray(*model, input, &allocator);
+        if (std::find(arrays_to_deallocate.begin(), arrays_to_deallocate.end(),
+                      input) == arrays_to_deallocate.end()) {
+          arrays_to_deallocate.push_back(input);
+        }
       }
     }
     for (const auto& output : op->outputs) {
       if (EndsAt(array_lifespans[output], op_index)) {
-        DeallocateTransientArray(*model, output, &allocator);
+        if (std::find(arrays_to_deallocate.begin(), arrays_to_deallocate.end(),
+                      output) == arrays_to_deallocate.end()) {
+          arrays_to_deallocate.push_back(output);
+        }
       }
     }
+    for (const string& array : arrays_to_deallocate) {
+      DeallocateTransientArray(*model, array, &allocator);
+    }
   }
 
   // Just out of curiosity (not used in the actual allocation process)
-- 
GitLab


From d3e3b78c631a975df498ed8cee65d505ddbe9aac Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Tue, 20 Mar 2018 22:06:21 +0200
Subject: [PATCH 255/960] Adjust indentations to conform to pylint.

---
 .../data/python/kernel_tests/resample_test.py | 64 +++++++++----------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index a76c6b1e39..897815656a 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -72,38 +72,38 @@ class ResampleTest(test.TestCase):
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
   def testRandomClasses(self):
-      init_dist = [0.25, 0.25, 0.25, 0.25]
-      target_dist = [0.0, 0.0, 0.0, 1.0]
-      num_classes = len(init_dist)
-      num_samples = 100   # We don't need many samples to test a dirac-delta target distribution
-      data_np = np.random.choice(num_classes, num_samples, p=init_dist)
-
-      dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
-
-      # Apply a random mapping that preserves the data distribution.
-      def _remap_fn(_):
-          return math_ops.cast(random_ops.random_uniform([1]) * num_classes, dtypes.int32)[0]
-      dataset = dataset.map(_remap_fn)
-
-      # Reshape distribution.
-      dataset = dataset.apply(
-          resampling.rejection_resample(
-              class_func=lambda x: x,
-              target_dist=target_dist,
-              initial_dist=init_dist))
-
-      get_next = dataset.make_one_shot_iterator().get_next()
-
-      with self.test_session() as sess:
-          returned = []
-          with self.assertRaises(errors.OutOfRangeError):
-              while True:
-                  returned.append(sess.run(get_next))
-
-      classes, _ = zip(*returned)
-      bincount = np.bincount(np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
-
-      self.assertAllClose(target_dist, bincount, atol=1e-2)
+    init_dist = [0.25, 0.25, 0.25, 0.25]
+    target_dist = [0.0, 0.0, 0.0, 1.0]
+    num_classes = len(init_dist)
+    num_samples = 100   # We don't need many samples to test a dirac-delta target distribution
+    data_np = np.random.choice(num_classes, num_samples, p=init_dist)
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
+
+    # Apply a random mapping that preserves the data distribution.
+    def _remap_fn(_):
+      return math_ops.cast(random_ops.random_uniform([1]) * num_classes, dtypes.int32)[0]
+    dataset = dataset.map(_remap_fn)
+
+    # Reshape distribution.
+    dataset = dataset.apply(
+      resampling.rejection_resample(
+        class_func=lambda x: x,
+        target_dist=target_dist,
+        initial_dist=init_dist))
+
+    get_next = dataset.make_one_shot_iterator().get_next()
+
+    with self.test_session() as sess:
+      returned = []
+      with self.assertRaises(errors.OutOfRangeError):
+        while True:
+          returned.append(sess.run(get_next))
+
+    classes, _ = zip(*returned)
+    bincount = np.bincount(np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
+
+    self.assertAllClose(target_dist, bincount, atol=1e-2)
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 5483b5894fb06ffeae49af74e573c114b2e3b787 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Tue, 20 Mar 2018 22:12:01 +0200
Subject: [PATCH 256/960] Fix indent mistake.

---
 tensorflow/contrib/data/python/ops/resampling.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index 44de42e94d..e440d4a35f 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -102,10 +102,11 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
                           .map(maybe_warn_on_large_rejection))
 
     def _gather_and_copy(class_val, acceptance_prob, data):
-        return (class_val, array_ops.gather(acceptance_prob, class_val), data)
+      return (class_val, array_ops.gather(acceptance_prob, class_val), data)
     current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
-        (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
-    filtered_ds = (current_probabilities_and_class_and_data_ds\
+      (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
+    filtered_ds = (
+      current_probabilities_and_class_and_data_ds
         .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
-- 
GitLab


From 13ae129449cdeb7afbad98bc8a00ad5c82a0ca31 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 20 Mar 2018 13:34:02 -0700
Subject: [PATCH 257/960] Improved the performance of the function optimizer.

PiperOrigin-RevId: 189799697
---
 .../grappler/optimizers/function_optimizer.cc | 78 +++++++++++++------
 1 file changed, 54 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 97effae8c8..2a6b8a325f 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -140,19 +140,53 @@ class FakeCPUDevice : public Device {
   Status Sync() override { return Status::OK(); }
 };
 
-Status InlineSymbolicGradient(const NodeDef& node,
-                              const FunctionDefLibrary& library,
-                              GraphDef* inlined_graph) {
-  Env* env = Env::Default();
-  DeviceAttributes attr;
-  attr.set_name("/device:CPU:0");
-  attr.set_device_type("CPU");
-  FakeCPUDevice* dev = new FakeCPUDevice(env, attr);
-  std::vector<Device*> devices;
-  devices.push_back(dev);
-  DeviceMgr dvc_mgr(devices);
-  FunctionLibraryDefinition function_library(OpRegistry::Global(), library);
+class SymbolicGradientEnv {
+ public:
+  SymbolicGradientEnv(int graph_version, const FunctionDefLibrary& library)
+      : graph_version_(graph_version), library_(library) {}
+
+  FunctionLibraryDefinition* function_library() {
+    InitializeIfNeeded();
+    return fld_.get();
+  }
+  FunctionLibraryRuntime* function_library_runtime() {
+    InitializeIfNeeded();
+    return flr_;
+  }
+
+ private:
+  // This initialization is expensive. Do it lazily to avoid paying for it
+  // unless it's needed.
+  void InitializeIfNeeded() {
+    if (flr_) {
+      return;
+    }
+    Env* env = Env::Default();
+    DeviceAttributes attr;
+    attr.set_name("/device:CPU:0");
+    attr.set_device_type("CPU");
+    FakeCPUDevice* dev = new FakeCPUDevice(env, attr);
+    std::vector<Device*> devices;
+    devices.push_back(dev);
+    dvc_mgr_.reset(new DeviceMgr(devices));
+    fld_.reset(new FunctionLibraryDefinition(OpRegistry::Global(), library_));
+    OptimizerOptions optimizer_opts;
+    optimizer_opts.set_do_function_inlining(true);
+    pflr_.reset(new ProcessFunctionLibraryRuntime(
+        dvc_mgr_.get(), env, graph_version_, fld_.get(), optimizer_opts));
+    flr_ = pflr_->GetFLR(dev->name());
+  }
+
+  const int graph_version_;
+  const FunctionDefLibrary& library_;
+  std::unique_ptr<DeviceMgr> dvc_mgr_;
+  std::unique_ptr<FunctionLibraryDefinition> fld_;
+  std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+  FunctionLibraryRuntime* flr_ = nullptr;
+};
 
+Status InlineSymbolicGradient(const NodeDef& node, SymbolicGradientEnv* env,
+                              GraphDef* inlined_graph) {
   GraphDef graph_def;
 
   // Create a node to anchor the gradient inputs
@@ -186,24 +220,18 @@ Status InlineSymbolicGradient(const NodeDef& node,
   }
 
   // Convert the graphdef to a graph
-  OptimizerOptions optimizer_opts;
-  optimizer_opts.set_do_function_inlining(true);
-  ProcessFunctionLibraryRuntime pflr(&dvc_mgr, env,
-                                     inlined_graph->versions().producer(),
-                                     &function_library, optimizer_opts);
-  FunctionLibraryRuntime* flr = pflr.GetFLR(dev->name());
-  CHECK(flr);
   GraphConstructorOptions graph_ctor_opts;
   graph_ctor_opts.allow_internal_ops = true;
   graph_ctor_opts.expect_device_spec = false;
-  Graph graph(function_library);
+  Graph graph(env->function_library());
   TF_RETURN_IF_ERROR(
       ConvertGraphDefToGraph(graph_ctor_opts, graph_def, &graph));
 
   // Recursively inline the functions until there is nothing more to inline. We
   // should at least expand one function.
   int counter = 0;
-  while (counter < 50 && ExpandInlineFunctions(flr, &graph)) {
+  while (counter < 50 &&
+         ExpandInlineFunctions(env->function_library_runtime(), &graph)) {
     ++counter;
   }
 
@@ -279,11 +307,12 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     return Status::OK();
   }
 
-  *optimized_graph->mutable_versions() = item.graph.versions();
+  SymbolicGradientEnv env(item.graph.versions().producer(),
+                          item.graph.library());
+
   for (const NodeDef& node : item.graph.node()) {
     if (node.op() == "SymbolicGradient") {
-      TF_RETURN_IF_ERROR(
-          InlineSymbolicGradient(node, item.graph.library(), optimized_graph));
+      TF_RETURN_IF_ERROR(InlineSymbolicGradient(node, &env, optimized_graph));
       continue;
     }
     auto it = functions.find(node.op());
@@ -299,6 +328,7 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
   // inlined based on the context in which they're instantiated.
 
   // TODO(bsteiner): trim the library to remove unused function definitions
+  *optimized_graph->mutable_versions() = item.graph.versions();
   *optimized_graph->mutable_library() = item.graph.library();
 
   return Status::OK();
-- 
GitLab


From 4a6ab2cb8c2f33ffb6b64d61bd09f006e75982c8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 13:38:09 -0700
Subject: [PATCH 258/960] Build tflite interpreter from buffer in python
 interface

PiperOrigin-RevId: 189800400
---
 tensorflow/contrib/lite/python/interpreter.py | 26 ++++++---
 .../contrib/lite/python/interpreter_test.py   | 53 ++++++++++---------
 .../interpreter_wrapper.cc                    |  9 +++-
 .../interpreter_wrapper/interpreter_wrapper.h |  6 ++-
 4 files changed, 62 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/lite/python/interpreter.py b/tensorflow/contrib/lite/python/interpreter.py
index 5b5a7c3199..accdd04671 100644
--- a/tensorflow/contrib/lite/python/interpreter.py
+++ b/tensorflow/contrib/lite/python/interpreter.py
@@ -23,19 +23,33 @@ from tensorflow.contrib.lite.python.interpreter_wrapper import tensorflow_wrap_i
 class Interpreter(object):
   """Interpreter inferace for TF-Lite Models."""
 
-  def __init__(self, model_path):
+  def __init__(self, model_path=None, model_content=None):
     """Constructor.
 
     Args:
       model_path: Path to TF-Lite Flatbuffer file.
+      model_content: Content of model.
 
     Raises:
-      ValueError: If the interpreter was unable to open the model.
+      ValueError: If the interpreter was unable to create.
     """
-    self._interpreter = (
-        interpreter_wrapper.InterpreterWrapper_CreateWrapperCPP(model_path))
-    if not self._interpreter:
-      raise ValueError('Failed to open {}'.format(model_path))
+    if model_path and not model_content:
+      self._interpreter = (
+          interpreter_wrapper.InterpreterWrapper_CreateWrapperCPPFromFile(
+              model_path))
+      if not self._interpreter:
+        raise ValueError('Failed to open {}'.format(model_path))
+    elif model_content and not model_path:
+      self._interpreter = (
+          interpreter_wrapper.InterpreterWrapper_CreateWrapperCPPFromBuffer(
+              model_content, len(model_content)))
+      if not self._interpreter:
+        raise ValueError(
+            'Failed to create model from {} bytes'.format(len(model_content)))
+    elif not model_path and not model_path:
+      raise ValueError('`model_path` or `model_content` must be specified.')
+    else:
+      raise ValueError('Can\'t both provide `model_path` and `model_content`')
 
   def allocate_tensors(self):
     if not self._interpreter.AllocateTensors():
diff --git a/tensorflow/contrib/lite/python/interpreter_test.py b/tensorflow/contrib/lite/python/interpreter_test.py
index e0215b721c..e85390c56c 100644
--- a/tensorflow/contrib/lite/python/interpreter_test.py
+++ b/tensorflow/contrib/lite/python/interpreter_test.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import io
 import numpy as np
 
 from tensorflow.contrib.lite.python import interpreter as interpreter_wrapper
@@ -29,7 +30,8 @@ class InterpreterTest(test_util.TensorFlowTestCase):
 
   def testFloat(self):
     interpreter = interpreter_wrapper.Interpreter(
-        resource_loader.get_path_to_datafile('testdata/permute_float.tflite'))
+        model_path=resource_loader.get_path_to_datafile(
+            'testdata/permute_float.tflite'))
     interpreter.allocate_tensors()
 
     input_details = interpreter.get_input_details()
@@ -53,29 +55,32 @@ class InterpreterTest(test_util.TensorFlowTestCase):
     self.assertTrue((expected_output == output_data).all())
 
   def testUint8(self):
-    interpreter = interpreter_wrapper.Interpreter(
-        resource_loader.get_path_to_datafile('testdata/permute_uint8.tflite'))
-    interpreter.allocate_tensors()
-
-    input_details = interpreter.get_input_details()
-    self.assertEqual(1, len(input_details))
-    self.assertEqual('input', input_details[0]['name'])
-    self.assertEqual(np.uint8, input_details[0]['dtype'])
-    self.assertTrue(([1, 4] == input_details[0]['shape']).all())
-
-    output_details = interpreter.get_output_details()
-    self.assertEqual(1, len(output_details))
-    self.assertEqual('output', output_details[0]['name'])
-    self.assertEqual(np.uint8, output_details[0]['dtype'])
-    self.assertTrue(([1, 4] == output_details[0]['shape']).all())
-
-    test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
-    expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
-    interpreter.set_tensor(input_details[0]['index'], test_input)
-    interpreter.invoke()
-
-    output_data = interpreter.get_tensor(output_details[0]['index'])
-    self.assertTrue((expected_output == output_data).all())
+    model_path = resource_loader.get_path_to_datafile(
+        'testdata/permute_uint8.tflite')
+    with io.open(model_path, 'rb') as model_file:
+      data = model_file.read()
+      interpreter = interpreter_wrapper.Interpreter(model_content=data)
+      interpreter.allocate_tensors()
+
+      input_details = interpreter.get_input_details()
+      self.assertEqual(1, len(input_details))
+      self.assertEqual('input', input_details[0]['name'])
+      self.assertEqual(np.uint8, input_details[0]['dtype'])
+      self.assertTrue(([1, 4] == input_details[0]['shape']).all())
+
+      output_details = interpreter.get_output_details()
+      self.assertEqual(1, len(output_details))
+      self.assertEqual('output', output_details[0]['name'])
+      self.assertEqual(np.uint8, output_details[0]['dtype'])
+      self.assertTrue(([1, 4] == output_details[0]['shape']).all())
+
+      test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
+      expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
+      interpreter.set_tensor(input_details[0]['index'], test_input)
+      interpreter.invoke()
+
+      output_data = interpreter.get_tensor(output_details[0]['index'])
+      self.assertTrue((expected_output == output_data).all())
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index f30067de94..14e1190c80 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -302,12 +302,19 @@ PyObject* InterpreterWrapper::GetTensor(int i) const {
   return PyArray_Return(reinterpret_cast<PyArrayObject*>(np_array));
 }
 
-InterpreterWrapper* InterpreterWrapper::CreateWrapperCPP(
+InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromFile(
     const char* model_path) {
   std::unique_ptr<tflite::FlatBufferModel> model =
       tflite::FlatBufferModel::BuildFromFile(model_path);
   return model ? new InterpreterWrapper(std::move(model)) : nullptr;
 }
 
+InterpreterWrapper* InterpreterWrapper::CreateWrapperCPPFromBuffer(
+    const char* data, size_t len) {
+  std::unique_ptr<tflite::FlatBufferModel> model =
+      tflite::FlatBufferModel::BuildFromBuffer(data, len);
+  return model ? new InterpreterWrapper(std::move(model)) : nullptr;
+}
+
 }  // namespace interpreter_wrapper
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
index dea71ca879..63bdb30f79 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
@@ -37,7 +37,11 @@ namespace interpreter_wrapper {
 class InterpreterWrapper {
  public:
   // SWIG caller takes ownership of pointer.
-  static InterpreterWrapper* CreateWrapperCPP(const char* model_path);
+  static InterpreterWrapper* CreateWrapperCPPFromFile(const char* model_path);
+
+  // SWIG caller takes ownership of pointer.
+  static InterpreterWrapper* CreateWrapperCPPFromBuffer(const char* data,
+                                                        size_t len);
 
   ~InterpreterWrapper();
   bool AllocateTensors();
-- 
GitLab


From b697da432aada697f2485734827b7bed5dbf2599 Mon Sep 17 00:00:00 2001
From: Terry Koo <terrykoo@google.com>
Date: Tue, 20 Mar 2018 13:39:42 -0700
Subject: [PATCH 259/960] Makes protobuf dep in tf.contrib.data conditional
 using if_static().

In non-monolithic builds, adding it unconditionally would duplicate of protobuf symbols among tf.contrib op libraries.  Guarding it with if_static() restricts the dep to monolithic builds, which should be able to dedupe the symbols at link time.

PiperOrigin-RevId: 189800612
---
 tensorflow/contrib/data/BUILD | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 5ba2297e7f..d787ed8a1a 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -9,6 +9,10 @@ load(
     "tf_custom_op_library",
     "tf_gen_op_libs",
 )
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "if_static",
+)
 
 py_library(
     name = "data",
@@ -29,10 +33,11 @@ py_library(
 tf_custom_op_library(
     name = "_dataset_ops.so",
     srcs = ["ops/dataset_ops.cc"],
-    deps = [
-        "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/core:lib_proto_parsing",
-    ],
+    deps = ["//tensorflow/contrib/data/kernels:dataset_kernels"] +
+           if_static(
+               extra_deps = ["//tensorflow/core:lib_proto_parsing"],
+               otherwise = [],
+           ),
 )
 
 tf_gen_op_libs(
-- 
GitLab


From 46321876c6ece27677f6c51400b799a9a8540324 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Tue, 20 Mar 2018 22:54:22 +0200
Subject: [PATCH 260/960] Fix more lint errors.

---
 tensorflow/contrib/data/python/kernel_tests/resample_test.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 7f9a16430c..99e56e9a31 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -82,7 +82,8 @@ class ResampleTest(test.TestCase):
 
     # Apply a random mapping that preserves the data distribution.
     def _remap_fn(_):
-      return math_ops.cast(random_ops.random_uniform([1]) * num_classes, dtypes.int32)[0]
+      return math_ops.cast(random_ops.random_uniform([1]) * num_classes,
+                           dtypes.int32)[0]
     dataset = dataset.map(_remap_fn)
 
     # Reshape distribution.
-- 
GitLab


From 9a24e8acfcd8c9046e1abaac9dbf5e146186f4c2 Mon Sep 17 00:00:00 2001
From: Piotr Czapla <github@piotrczapla.com>
Date: Tue, 20 Mar 2018 22:20:20 +0100
Subject: [PATCH 261/960] Add training parameter to dropout to make it work
 (#16133)

* Add training parameter to dropout to make it work

I think that without this parameter set dropout is disabled all the time. At least this is what I read in the documentation, besides adding this improves training.

* Removing redundant if statement around dropout

* Fix linter error: line longer than 80.
---
 tensorflow/examples/learn/mnist.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py
index 98819b20bf..3ead8614b6 100644
--- a/tensorflow/examples/learn/mnist.py
+++ b/tensorflow/examples/learn/mnist.py
@@ -61,8 +61,10 @@ def conv_model(features, labels, mode):
 
   # Densely connected layer with 1024 neurons.
   h_fc1 = tf.layers.dense(h_pool2_flat, 1024, activation=tf.nn.relu)
-  if mode == tf.estimator.ModeKeys.TRAIN:
-    h_fc1 = tf.layers.dropout(h_fc1, rate=0.5)
+  h_fc1 = tf.layers.dropout(
+      h_fc1, 
+      rate=0.5, 
+      training=(mode == tf.estimator.ModeKeys.TRAIN))
 
   # Compute logits (1 per class) and compute loss.
   logits = tf.layers.dense(h_fc1, N_DIGITS, activation=None)
-- 
GitLab


From 4d7d62d5101f069017c8714c53299be022b4ff74 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Tue, 20 Mar 2018 14:23:39 -0700
Subject: [PATCH 262/960] Internal Change.

PiperOrigin-RevId: 189809845
---
 tensorflow/python/framework/test_util.py |  6 +++---
 tensorflow/python/keras/BUILD            | 14 ++++++++------
 2 files changed, 11 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index e9e86e452b..d8f8569939 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -902,9 +902,9 @@ class TensorFlowTestCase(googletest.TestCase):
 
     Use the `use_gpu` and `force_gpu` options to control where ops are run. If
     `force_gpu` is True, all ops are pinned to `/device:GPU:0`. Otherwise, if
-    `use_gpu`
-    is True, TensorFlow tries to run as many ops on the GPU as possible. If both
-    `force_gpu and `use_gpu` are False, all ops are pinned to the CPU.
+    `use_gpu` is True, TensorFlow tries to run as many ops on the GPU as
+    possible. If both `force_gpu and `use_gpu` are False, all ops are pinned to
+    the CPU.
 
     Example:
     ```python
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index eef91e9c5b..3180b9f410 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -8,6 +8,7 @@ exports_files(["LICENSE"])
 package(default_visibility = ["//visibility:public"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
 config_setting(
     name = "empty_condition",
@@ -656,16 +657,17 @@ py_test(
     ],
 )
 
-py_test(
+cuda_py_test(
     name = "multi_gpu_utils_test",
-    size = "medium",
     srcs = ["_impl/keras/utils/multi_gpu_utils_test.py"],
-    srcs_version = "PY2AND3",
-    tags = ["multi_gpu"],
-    deps = [
+    additional_deps = [
         ":keras",
-        "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+    ],
+    tags = [
+        "guitar",
+        "multi_gpu",
     ],
 )
 
-- 
GitLab


From a0e07f998b388f0ecc7b7cf2256522f28482b285 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 20 Mar 2018 14:30:36 -0700
Subject: [PATCH 263/960] [tf.data] Improve docstring for
 `tf.contrib.data.Counter`.

PiperOrigin-RevId: 189811108
---
 tensorflow/contrib/data/python/ops/counter.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/counter.py b/tensorflow/contrib/data/python/ops/counter.py
index 63226fe781..6ef65f9624 100644
--- a/tensorflow/contrib/data/python/ops/counter.py
+++ b/tensorflow/contrib/data/python/ops/counter.py
@@ -25,7 +25,7 @@ from tensorflow.python.framework import ops
 
 
 def Counter(start=0, step=1, dtype=dtypes.int64):
-  """Creates a `Dataset` of a `step`-separated count startin from `start`.
+  """Creates a `Dataset` that counts from `start` in steps of size `step`.
 
   For example:
 
@@ -38,12 +38,13 @@ def Counter(start=0, step=1, dtype=dtypes.int64):
   ```
 
   Args:
-    start: starting value for count.
-    step: step size.
-    dtype: counter data type.
+    start: (Optional.) The starting value for the counter. Defaults to 0.
+    step: (Optional.) The step size for the counter. Defaults to 1.
+    dtype: (Optional.) The data type for counter elements. Defaults to
+      `tf.int64`.
 
   Returns:
-    A `Dataset` of scalar elements.
+    A `Dataset` of scalar `dtype` elements.
   """
   with ops.name_scope("counter"):
     start = ops.convert_to_tensor(start, dtype=dtype, name="start")
-- 
GitLab


From 546d1d467372a176f337f2614165c6d754a386da Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 14:33:19 -0700
Subject: [PATCH 264/960] [XLA] Simplify the HLO proto: don't nest the fusion
 computation in an fusion HloInstructionProto.

PiperOrigin-RevId: 189811729
---
 tensorflow/compiler/xla/service/hlo.proto     |  3 +-
 .../compiler/xla/service/hlo_computation.cc   | 18 ++++++------
 .../compiler/xla/service/hlo_computation.h    | 10 +------
 .../compiler/xla/service/hlo_instruction.cc   | 28 ++++++++++---------
 .../compiler/xla/service/hlo_instruction.h    |  7 +----
 tensorflow/compiler/xla/service/hlo_module.cc | 22 ++-------------
 6 files changed, 30 insertions(+), 58 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index bf903d6a39..b86fbd821b 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -38,6 +38,8 @@ option cc_enable_arenas = true;
 message HloInstructionProto {
   reserved 10;
   reserved "parameter_name";
+  reserved 12;
+  reserved "fused_instructions_computation";
 
   string name = 1;
   string opcode = 2;
@@ -58,7 +60,6 @@ message HloInstructionProto {
 
   // Fusion state, only present for kFusion.
   string fusion_kind = 11;
-  HloComputationProto fused_instructions_computation = 12;
 
   // Index for kGetTupleElement.
   int64 tuple_index = 13;
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index f99c7cf5e4..4e852190a8 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -406,18 +406,15 @@ HloComputationProto HloComputation::ToProto() const {
 /* static */ StatusOr<std::unique_ptr<HloComputation>>
 HloComputation::CreateFromProto(
     HloModule* module, const HloComputationProto& proto,
-    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map,
-    const std::function<void(std::unique_ptr<HloComputation>)>&
-        add_fused_computation,
-    HloInstruction* fusion_instruction) {
+    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map) {
   std::vector<std::unique_ptr<HloInstruction>> instructions;
   tensorflow::gtl::FlatMap<string, HloInstruction*> instruction_map;
   int64 parameter_count = 0;
   for (const HloInstructionProto& instruction_proto : proto.instructions()) {
-    TF_ASSIGN_OR_RETURN(std::unique_ptr<HloInstruction> instruction,
-                        HloInstruction::CreateFromProto(
-                            module, instruction_proto, instruction_map,
-                            computation_map, add_fused_computation));
+    TF_ASSIGN_OR_RETURN(
+        std::unique_ptr<HloInstruction> instruction,
+        HloInstruction::CreateFromProto(module, instruction_proto,
+                                        instruction_map, computation_map));
     if (instruction->opcode() == HloOpcode::kParameter) {
       parameter_count++;
     }
@@ -429,8 +426,9 @@ HloComputation::CreateFromProto(
   TF_RET_CHECK(!proto.root_name().empty());
   TF_RET_CHECK(ContainsKey(instruction_map, proto.root_name()));
   HloInstruction* root = instruction_map.at(proto.root_name());
-  return WrapUnique(new HloComputation(
-      proto.name(), parameter_count, &instructions, root, fusion_instruction));
+  return WrapUnique(new HloComputation(proto.name(), parameter_count,
+                                       &instructions, root,
+                                       /*fusion_instruction=*/nullptr));
 }
 
 void HloComputation::FuseInstructionsInto(
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index dd9d346999..630d3675de 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -163,17 +163,9 @@ class HloComputation {
   //   computation_map: a map from computation name to HloComputation*. This map
   //     must contain all computations which the newly constructed computation
   //     calls.
-  //   add_fused_computation: A function to call to add a fused
-  //     computation. Used only when the instruction is a fusion instruction.
-  //   fusion_instruction: if non-null then the newly created computation will
-  //     be constructed as a fused computation with this instruction as its
-  //     fusion parent.
   static StatusOr<std::unique_ptr<HloComputation>> CreateFromProto(
       HloModule* module, const HloComputationProto& proto,
-      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map,
-      const std::function<void(std::unique_ptr<HloComputation>)>&
-          add_fused_computation,
-      HloInstruction* fusion_instruction = nullptr);
+      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map);
 
   // Gets the instructions in this computation.
   //
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index d33add23d0..83fcc5da6d 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
@@ -52,9 +53,7 @@ using ::tensorflow::strings::StrCat;
 StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     HloModule* module, const HloInstructionProto& proto,
     const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
-    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map,
-    const std::function<void(std::unique_ptr<HloComputation>)>&
-        add_fused_computation) {
+    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map) {
   TF_RET_CHECK(!proto.opcode().empty());
   TF_ASSIGN_OR_RETURN(HloOpcode opcode, StringToHloOpcode(proto.opcode()));
   TF_RET_CHECK(proto.has_shape());
@@ -76,17 +75,20 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
   // HloInstructionProto and do not appear as an HloComputationProto within the
   // HloModuleProto.
   if (instruction->opcode() == HloOpcode::kFusion) {
-    TF_RET_CHECK(proto.has_fused_instructions_computation());
     TF_RET_CHECK(!proto.fusion_kind().empty());
     TF_ASSIGN_OR_RETURN(instruction->fusion_kind_,
                         StringToFusionKind(proto.fusion_kind()));
-    TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> fused_computation,
-                        HloComputation::CreateFromProto(
-                            module, proto.fused_instructions_computation(),
-                            computation_map, add_fused_computation,
-                            /*fusion_instruction=*/instruction.get()));
-    instruction->called_computations_.push_back(fused_computation.get());
-    add_fused_computation(std::move(fused_computation));
+
+    // Find the fused computation and set its fusion instruction.
+    TF_RET_CHECK(proto.called_computation_names_size() == 1)
+        << "Expect 1 called computation for fusion instruction, but sees "
+        << proto.called_computation_names_size();
+    const string& fusion_name = proto.called_computation_names(0);
+    auto* fused_computation = FindPtrOrNull(computation_map, fusion_name);
+    TF_RET_CHECK(fused_computation != nullptr)
+        << "No fusion computation named " << fusion_name;
+    fused_computation->SetFusionInstruction(instruction.get());
+    instruction->called_computations_.push_back(fused_computation);
   } else {
     for (const string& computation_name : proto.called_computation_names()) {
       TF_RET_CHECK(ContainsKey(computation_map, computation_name))
@@ -2330,8 +2332,8 @@ HloInstructionProto HloInstruction::ToProto() const {
   proto.set_parameter_number(parameter_number_);
   if (opcode() == HloOpcode::kFusion) {
     proto.set_fusion_kind(xla::ToString(fusion_kind()));
-    *proto.mutable_fused_instructions_computation() =
-        fused_instructions_computation()->ToProto();
+    *proto.add_called_computation_names() =
+        fused_instructions_computation()->name();
   } else {
     for (const HloComputation* computation : called_computations_) {
       *proto.add_called_computation_names() = computation->name();
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index e4c86214c2..a111e1e4a6 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -184,15 +184,10 @@ class HloInstruction {
   //   computation_map: a map from computation name to HloComputation*. This map
   //     must contain all computations which the newly constructed instruction
   //     calls.
-  //   add_fused_computation: A function to call to add a fused
-  //     computation. Used (clearly) when the instruction is a fusion
-  //     instruction.
   static StatusOr<std::unique_ptr<HloInstruction>> CreateFromProto(
       HloModule* module, const HloInstructionProto& proto,
       const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
-      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map,
-      const std::function<void(std::unique_ptr<HloComputation>)>&
-          add_fused_computation);
+      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map);
 
   // Creates a parameter-retrieving instruction.
   static std::unique_ptr<HloInstruction> CreateParameter(int64 parameter_number,
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index cdea3d5978..4091ebbfd3 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -207,11 +207,6 @@ HloModuleProto HloModule::ToProto() const {
   proto.set_name(name_);
   proto.set_entry_computation_name(entry_computation_->name());
   for (const HloComputation* computation : MakeComputationPostOrder()) {
-    // Fusion computations are added when the fusion instructions are created by
-    // HloInstruction::CreateFromProto.
-    if (computation->IsFusionComputation()) {
-      continue;
-    }
     HloComputationProto computation_proto = computation->ToProto();
     if (computation->name() == entry_computation_->name()) {
       *proto.mutable_program_shape() = computation_proto.program_shape();
@@ -256,16 +251,9 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
 
   tensorflow::gtl::FlatMap<string, HloComputation*> computation_map;
   for (const HloComputationProto& computation_proto : proto.computations()) {
-    TF_ASSIGN_OR_RETURN(
-        std::unique_ptr<HloComputation> computation,
-        HloComputation::CreateFromProto(
-            module.get(), computation_proto, computation_map,
-            /*add_fused_computation=*/
-            [&module](std::unique_ptr<HloComputation> fused_computation) {
-              module->AddComputationInternal(std::move(fused_computation),
-                                             /*is_entry=*/false,
-                                             /*uniquify_names=*/false);
-            }));
+    TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> computation,
+                        HloComputation::CreateFromProto(
+                            module.get(), computation_proto, computation_map));
     CHECK_NE(computation.get(), nullptr);
     TF_RET_CHECK(!ContainsKey(computation_map, computation->name()));
     string computation_name = computation->name();
@@ -283,10 +271,6 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   tensorflow::gtl::FlatSet<string> computation_names;
   tensorflow::gtl::FlatSet<string> instruction_names;
   for (HloComputation* computation : module->computations()) {
-    if (computation->IsFusionComputation()) {
-      continue;
-    }
-
     TF_RET_CHECK(!ContainsKey(computation_names, computation->name()))
         << "Computation name is not unique: " << computation->name();
     computation_names.insert(computation->name());
-- 
GitLab


From 38a5c2dba2806951ae4defba0f1392469ae422de Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Tue, 20 Mar 2018 23:46:39 +0200
Subject: [PATCH 265/960] Hopefully final indent fix.

---
 .../contrib/data/python/kernel_tests/resample_test.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 99e56e9a31..0e3131b725 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -88,10 +88,10 @@ class ResampleTest(test.TestCase):
 
     # Reshape distribution.
     dataset = dataset.apply(
-      resampling.rejection_resample(
-        class_func=lambda x: x,
-        target_dist=target_dist,
-        initial_dist=init_dist))
+        resampling.rejection_resample(
+            class_func=lambda x: x,
+            target_dist=target_dist,
+            initial_dist=init_dist))
 
     get_next = dataset.make_one_shot_iterator().get_next()
 
@@ -102,7 +102,8 @@ class ResampleTest(test.TestCase):
           returned.append(sess.run(get_next))
 
     classes, _ = zip(*returned)
-    bincount = np.bincount(np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
+    bincount = np.bincount(
+      np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
 
     self.assertAllClose(target_dist, bincount, atol=1e-2)
 
-- 
GitLab


From c3ed1c402ff5d21f2e46b931e95f87991f2c3099 Mon Sep 17 00:00:00 2001
From: Terry Koo <terrykoo@google.com>
Date: Tue, 20 Mar 2018 14:52:58 -0700
Subject: [PATCH 266/960] Revert "Adds missing protobuf dep to tf.contrib.data
 ops. (#17840)" (#17864)

* Revert "Adds missing protobuf dep to tf.contrib.data ops. (#17840)"

This reverts commit 36ec749ec79c2313924666a1c5324620e493d0c4.

* Protect lib_proto_parsing dep with if_static().
---
 tensorflow/contrib/data/BUILD | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 5ba2297e7f..d787ed8a1a 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -9,6 +9,10 @@ load(
     "tf_custom_op_library",
     "tf_gen_op_libs",
 )
+load(
+    "//tensorflow/core:platform/default/build_config_root.bzl",
+    "if_static",
+)
 
 py_library(
     name = "data",
@@ -29,10 +33,11 @@ py_library(
 tf_custom_op_library(
     name = "_dataset_ops.so",
     srcs = ["ops/dataset_ops.cc"],
-    deps = [
-        "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/core:lib_proto_parsing",
-    ],
+    deps = ["//tensorflow/contrib/data/kernels:dataset_kernels"] +
+           if_static(
+               extra_deps = ["//tensorflow/core:lib_proto_parsing"],
+               otherwise = [],
+           ),
 )
 
 tf_gen_op_libs(
-- 
GitLab


From b7f42a4ed5e363660562e9f020875d79ce3c1300 Mon Sep 17 00:00:00 2001
From: joel-shor <joelshor@google.com>
Date: Wed, 21 Mar 2018 00:04:13 +0200
Subject: [PATCH 267/960] Really the last indendation bug fix.

---
 .../contrib/data/python/kernel_tests/resample_test.py       | 3 ++-
 tensorflow/contrib/data/python/ops/resampling.py            | 6 +++---
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 0e3131b725..38efcd3cba 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -103,7 +103,8 @@ class ResampleTest(test.TestCase):
 
     classes, _ = zip(*returned)
     bincount = np.bincount(
-      np.array(classes), minlength=num_classes).astype(np.float32) / len(classes)
+        np.array(classes),
+        minlength=num_classes).astype(np.float32) / len(classes)
 
     self.assertAllClose(target_dist, bincount, atol=1e-2)
 
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index e440d4a35f..0e127f72cd 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -104,10 +104,10 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
     def _gather_and_copy(class_val, acceptance_prob, data):
       return (class_val, array_ops.gather(acceptance_prob, class_val), data)
     current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
-      (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
+          (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
     filtered_ds = (
-      current_probabilities_and_class_and_data_ds
-        .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
+          current_probabilities_and_class_and_data_ds
+              .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
   return _apply_fn
-- 
GitLab


From 8030be47e2eee7a43a55a349ca034e0c80abcc0b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 15:10:07 -0700
Subject: [PATCH 268/960] Tweak statistical testing test to avoid making a
 zillion TF session.run calls.

PiperOrigin-RevId: 189819449
---
 tensorflow/contrib/distributions/BUILD        |  6 +--
 .../kernel_tests/statistical_testing_test.py  | 40 ++++++++++---------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index e9c827a618..4ddec73ec8 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -486,11 +486,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    tags = [
-        "manual",
-        "noasan",
-        "noguitar",
-    ],
+    tags = ["noasan"],  # Was found to time out in asan
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
index 3548ac1807..fc071c273d 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
@@ -31,30 +31,34 @@ class StatisticalTestingTest(test.TestCase):
   def test_dkwm_design_mean_one_sample_soundness(self):
     numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
+    def check_soundness(ff, fp):
+      sufficient_n = st.min_num_samples_for_dkwm_mean_test(
+          numbers, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
+      detectable_d = st.min_discrepancy_of_true_means_detectable_by_dkwm(
+          sufficient_n, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
+      return check_ops.assert_less_equal(detectable_d, numbers)
     with self.test_session() as sess:
-      for ff in rates:
-        for fp in rates:
-          sufficient_n = st.min_num_samples_for_dkwm_mean_test(
-              numbers, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-          detectable_d = st.min_discrepancy_of_true_means_detectable_by_dkwm(
-              sufficient_n, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
+      sess.run([check_soundness(ff, fp)
+                for ff in rates
+                for fp in rates])
 
   def test_dkwm_design_mean_two_sample_soundness(self):
     numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
+    def check_soundness(ff, fp):
+      (sufficient_n1,
+       sufficient_n2) = st.min_num_samples_for_dkwm_mean_two_sample_test(
+           numbers, 0., 1., 0., 1.,
+           false_fail_rate=ff, false_pass_rate=fp)
+      d_fn = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample
+      detectable_d = d_fn(
+          sufficient_n1, 0., 1., sufficient_n2, 0., 1.,
+          false_fail_rate=ff, false_pass_rate=fp)
+      return check_ops.assert_less_equal(detectable_d, numbers)
     with self.test_session() as sess:
-      for ff in rates:
-        for fp in rates:
-          (sufficient_n1,
-           sufficient_n2) = st.min_num_samples_for_dkwm_mean_two_sample_test(
-               numbers, 0., 1., 0., 1.,
-               false_fail_rate=ff, false_pass_rate=fp)
-          d_fn = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample
-          detectable_d = d_fn(
-              sufficient_n1, 0., 1., sufficient_n2, 0., 1.,
-              false_fail_rate=ff, false_pass_rate=fp)
-          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
+      sess.run([check_soundness(ff, fp)
+                for ff in rates
+                for fp in rates])
 
   def test_true_mean_confidence_interval_by_dkwm_one_sample(self):
     rng = np.random.RandomState(seed=0)
-- 
GitLab


From 44c55c0fbf531043368866683fa01bba42b9a1d0 Mon Sep 17 00:00:00 2001
From: Ankit Gupta <ankitgupta@alumni.harvard.edu>
Date: Tue, 20 Mar 2018 15:41:45 -0700
Subject: [PATCH 269/960] LoggingTensorHook to read from runconfig in Estimator
 (#17157)

* got loggingtensor to read from runconfig

* updated pydoc
---
 tensorflow/python/estimator/estimator.py  | 2 +-
 tensorflow/python/estimator/run_config.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 5245a050a1..6a4132bca2 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -849,7 +849,7 @@ class Estimator(object):
                   'loss': estimator_spec.loss,
                   'step': global_step_tensor
               },
-              every_n_iter=100)
+              every_n_iter=self._config.log_step_count_steps)
       ])
       worker_hooks.extend(estimator_spec.training_hooks)
 
diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 62f035bce5..820fda7765 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -423,7 +423,7 @@ class RunConfig(object):
         to be saved. The default value of 10,000 hours effectively disables
         the feature.
       log_step_count_steps: The frequency, in number of global steps, that the
-        global step/sec will be logged during training.
+        global step/sec and the loss will be logged during training.
 
 
     Raises:
-- 
GitLab


From 0bd851b38810540034069d92a2f76a026429bced Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 20 Mar 2018 16:11:23 -0700
Subject: [PATCH 270/960] [XLA] Make HLO memory schedulers pluggable. Introduce
 a typedef MemorySchedulerAlgorithm  which is a function instead of an enum to
 allow experimentation with non-standard schedulers. Refactoring only; no
 functional changes to the scheduling itself.

PiperOrigin-RevId: 189830685
---
 .../xla/service/hlo_rematerialization.cc      |  2 +-
 .../xla/service/hlo_rematerialization.h       |  6 +-
 .../xla/service/hlo_rematerialization_test.cc | 16 ++---
 .../compiler/xla/service/hlo_scheduling.cc    | 61 +++++++++++--------
 .../compiler/xla/service/hlo_scheduling.h     | 43 +++++++++----
 .../xla/service/hlo_scheduling_test.cc        |  3 +-
 6 files changed, 80 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.cc b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
index 98b8d34be1..b063244893 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.cc
@@ -1320,7 +1320,7 @@ StatusOr<bool> HloRematerialization::Run(
 /* static */ StatusOr<bool> HloRematerialization::RematerializeAndSchedule(
     const HloRematerialization::ShapeSizeFunction& size_function,
     int64 memory_limit_bytes, HloModule* hlo_module,
-    SchedulerAlgorithm scheduler_algorithm,
+    MemorySchedulerAlgorithm scheduler_algorithm,
     SequentialHloOrdering::HloModuleSequence* sequence,
     RematerializationSizes* sizes) {
   HloRematerialization remat(scheduler_algorithm, size_function);
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization.h b/tensorflow/compiler/xla/service/hlo_rematerialization.h
index 5255343903..2ee2dd0571 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization.h
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization.h
@@ -66,12 +66,12 @@ class HloRematerialization {
   // code generation.
   static StatusOr<bool> RematerializeAndSchedule(
       const ShapeSizeFunction& size_function, int64 memory_limit_bytes,
-      HloModule* hlo_module, SchedulerAlgorithm scheduler_algorithm,
+      HloModule* hlo_module, MemorySchedulerAlgorithm scheduler_algorithm,
       SequentialHloOrdering::HloModuleSequence* sequence,
       RematerializationSizes* sizes = nullptr);
 
  protected:
-  HloRematerialization(SchedulerAlgorithm scheduler_algorithm,
+  HloRematerialization(MemorySchedulerAlgorithm scheduler_algorithm,
                        const ShapeSizeFunction& size_function)
       : scheduler_algorithm_(scheduler_algorithm),
         size_function_(size_function) {}
@@ -108,7 +108,7 @@ class HloRematerialization {
       const HloInstruction* instruction) const;
 
   // Selects an algorithm to use for HLO scheduling.
-  SchedulerAlgorithm scheduler_algorithm_;
+  MemorySchedulerAlgorithm scheduler_algorithm_;
 
   // Function which computes the size of the top-level buffer of a shape.
   const ShapeSizeFunction size_function_;
diff --git a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
index 1b7d26dde5..83de54f3fa 100644
--- a/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_rematerialization_test.cc
@@ -162,7 +162,7 @@ TEST_F(HloRematerializationTest, SingleComputation) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/14 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
 
   // Root should not have changed.
@@ -195,7 +195,7 @@ TEST_F(HloRematerializationTest, SingleComputationNoRematerialization) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/20 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
 
   // No instructions should have been materialized.
   EXPECT_FALSE(changed);
@@ -236,7 +236,7 @@ TEST_F(HloRematerializationTest, RematerializeAroundWhile) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/17 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
 
   // Only the entry computation should have a rematerialized instruction added.
@@ -272,7 +272,7 @@ TEST_F(HloRematerializationTest, RematerializeEntryAndWhileBody) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/15 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
 
   // Both computations should have a rematerialized instruction added.
@@ -314,7 +314,7 @@ TEST_F(HloRematerializationTest, RematerializeNestedComputations) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/13 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
 
   // All computations should have a rematerialized instruction added.
@@ -385,7 +385,7 @@ TEST_F(HloRematerializationTest, RngNotRematerialized) {
       bool changed, HloRematerialization::RematerializeAndSchedule(
                         ByteSizeOf,
                         /*memory_limit_bytes=*/4 * ByteSizeOf(vec1024_shape_),
-                        module.get(), SchedulerAlgorithm::kAuto, &sequence));
+                        module.get(), DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
   // The rng should not have been rematerialized.
   EXPECT_EQ(count_rngs(entry_computation), 1);
@@ -480,7 +480,7 @@ TEST_F(HloRematerializationTest, InstructionRematerializedMultipleTimes) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/22 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   EXPECT_TRUE(changed);
 
   // The broadcast should have been rematerialized 3 times.
@@ -577,7 +577,7 @@ TEST_P(IndirectUseTest, IndirectUseNotRematerialized) {
                           HloRematerialization::RematerializeAndSchedule(
                               ByteSizeOf,
                               /*memory_limit_bytes=*/22 * 1024, module.get(),
-                              SchedulerAlgorithm::kAuto, &sequence));
+                              DefaultMemoryScheduler, &sequence));
   // Rematerialization should only occur if the rematerializable instruction has
   // no indirect uses.
   if (indirectly_used) {
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling.cc b/tensorflow/compiler/xla/service/hlo_scheduling.cc
index 099dd8dd8e..1a767628f6 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling.cc
+++ b/tensorflow/compiler/xla/service/hlo_scheduling.cc
@@ -340,7 +340,33 @@ int64 SumLogicalBufferSizes(
   return size;
 }
 
-StatusOr<std::vector<const HloInstruction*>> RunDFSMemoryScheduler(
+StatusOr<int64> MinimumMemoryForComputation(
+    const HloComputation& computation,
+    const std::vector<const HloInstruction*>& sequence,
+    const TuplePointsToAnalysis& points_to_analysis,
+    const LogicalBuffer::SizeFunction& size_function) {
+  TF_ASSIGN_OR_RETURN(
+      HeapSimulator::Result result,
+      HeapSimulator::Run(MakeUnique<NoFragmentationStatsHeap>(), computation,
+                         sequence, points_to_analysis, size_function));
+  return result.heap_size;
+}
+
+StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
+    const HloComputation& computation,
+    const TuplePointsToAnalysis& points_to_analysis,
+    const LogicalBuffer::SizeFunction& size_function,
+    const MemorySchedulerAlgorithm& algorithm) {
+  VLOG(2) << "Computation: " << computation.name();
+  if (algorithm) {
+    return algorithm(computation, points_to_analysis, size_function);
+  }
+  return DefaultMemoryScheduler(computation, points_to_analysis, size_function);
+}
+
+}  // namespace
+
+StatusOr<std::vector<const HloInstruction*>> DFSMemoryScheduler(
     const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function) {
@@ -397,32 +423,17 @@ StatusOr<std::vector<const HloInstruction*>> RunDFSMemoryScheduler(
   return sequence;
 }
 
-StatusOr<int64> MinimumMemoryForComputation(
+StatusOr<std::vector<const HloInstruction*>> ListMemoryScheduler(
     const HloComputation& computation,
-    const std::vector<const HloInstruction*>& sequence,
     const TuplePointsToAnalysis& points_to_analysis,
     const LogicalBuffer::SizeFunction& size_function) {
-  TF_ASSIGN_OR_RETURN(
-      HeapSimulator::Result result,
-      HeapSimulator::Run(MakeUnique<NoFragmentationStatsHeap>(), computation,
-                         sequence, points_to_analysis, size_function));
-  return result.heap_size;
+  return ListScheduler::Run(computation, points_to_analysis, size_function);
 }
 
-StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
+StatusOr<std::vector<const HloInstruction*>> DefaultMemoryScheduler(
     const HloComputation& computation,
     const TuplePointsToAnalysis& points_to_analysis,
-    const LogicalBuffer::SizeFunction& size_function,
-    SchedulerAlgorithm algorithm) {
-  VLOG(2) << "Computation: " << computation.name();
-  if (algorithm == SchedulerAlgorithm::kListSchedule) {
-    return ListScheduler::Run(computation, points_to_analysis, size_function);
-  }
-  if (algorithm == SchedulerAlgorithm::kDfsSchedule) {
-    return RunDFSMemoryScheduler(computation, points_to_analysis,
-                                 size_function);
-  }
-
+    const LogicalBuffer::SizeFunction& size_function) {
   // We try both a list-scheduler based ordering and a DFS based ordering, and
   // choose whichever returns a lower min-memory, not accounting for
   // fragmentation.
@@ -432,7 +443,7 @@ StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
   // within the caller's context. But it's good enough for now.
   TF_ASSIGN_OR_RETURN(
       std::vector<const HloInstruction*> list_sequence,
-      ListScheduler::Run(computation, points_to_analysis, size_function));
+      ListMemoryScheduler(computation, points_to_analysis, size_function));
   TF_ASSIGN_OR_RETURN(
       const int64 list_memory,
       MinimumMemoryForComputation(computation, list_sequence,
@@ -441,7 +452,7 @@ StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
 
   TF_ASSIGN_OR_RETURN(
       std::vector<const HloInstruction*> dfs_sequence,
-      RunDFSMemoryScheduler(computation, points_to_analysis, size_function));
+      DFSMemoryScheduler(computation, points_to_analysis, size_function));
   TF_ASSIGN_OR_RETURN(
       const int64 dfs_memory,
       MinimumMemoryForComputation(computation, dfs_sequence, points_to_analysis,
@@ -459,12 +470,10 @@ StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
   }
 }
 
-}  // namespace
-
 StatusOr<SequentialHloOrdering::HloModuleSequence>
 CreateMemoryMinimizingSequence(const HloModule& module,
                                const LogicalBuffer::SizeFunction& size_function,
-                               SchedulerAlgorithm algorithm) {
+                               const MemorySchedulerAlgorithm& algorithm) {
   SequentialHloOrdering::HloModuleSequence sequence;
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
                       TuplePointsToAnalysis::Run(&module));
@@ -480,7 +489,7 @@ CreateMemoryMinimizingSequence(const HloModule& module,
 StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
     const HloComputation& computation,
     const LogicalBuffer::SizeFunction& size_function,
-    SchedulerAlgorithm algorithm) {
+    const MemorySchedulerAlgorithm& algorithm) {
   CHECK(!computation.IsFusionComputation());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<TuplePointsToAnalysis> points_to_analysis,
                       TuplePointsToAnalysis::Run(computation.parent()));
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling.h b/tensorflow/compiler/xla/service/hlo_scheduling.h
index 1d1eb1e064..068e68383d 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling.h
+++ b/tensorflow/compiler/xla/service/hlo_scheduling.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_ordering.h"
 #include "tensorflow/compiler/xla/service/logical_buffer.h"
+#include "tensorflow/compiler/xla/service/tuple_points_to_analysis.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/types.h"
 
@@ -33,28 +34,48 @@ StatusOr<int64> MinimumMemoryForSequence(
     const SequentialHloOrdering::HloModuleSequence& module_sequence,
     const LogicalBuffer::SizeFunction& size_function);
 
-enum class SchedulerAlgorithm {
-  kListSchedule,
-  kDfsSchedule,
+// A memory scheduler computes an execution sequence for the HLO instructions in
+// 'computation' that minimizes peak memory, given a points-to analysis result
+// that describes buffer aliasing, together with a target-specific size function
+// that maps a tensor's logical size to its padded size.
+typedef std::function<StatusOr<std::vector<const HloInstruction*>>(
+    const HloComputation&, const TuplePointsToAnalysis&,
+    const LogicalBuffer::SizeFunction&)>
+    MemorySchedulerAlgorithm;
 
-  // Selects the available scheduler algorithm that had the minimum memory in
-  // the resulting sequence (a la MinimumMemoryForSequence).
-  kAuto,
-};
+// List scheduler
+StatusOr<std::vector<const HloInstruction*>> ListMemoryScheduler(
+    const HloComputation& computation,
+    const TuplePointsToAnalysis& points_to_analysis,
+    const LogicalBuffer::SizeFunction& size_function);
+
+// DFS-order scheduler
+StatusOr<std::vector<const HloInstruction*>> DFSMemoryScheduler(
+    const HloComputation& computation,
+    const TuplePointsToAnalysis& points_to_analysis,
+    const LogicalBuffer::SizeFunction& size_function);
+
+// The default scheduling algorithm. Runs both the list scheduler
+// and the DFS scheduler, and chooses whichever returns a lower min-memory,
+// not accounting for fragmentation.
+StatusOr<std::vector<const HloInstruction*>> DefaultMemoryScheduler(
+    const HloComputation& computation,
+    const TuplePointsToAnalysis& points_to_analysis,
+    const LogicalBuffer::SizeFunction& size_function);
 
 // Returns an HloModuleSequence which seeks to minimize the memory required for
 // the computation. size_function is the function returning the number of bytes
 // required for a LogicalBuffer.
 StatusOr<SequentialHloOrdering::HloModuleSequence>
-CreateMemoryMinimizingSequence(
-    const HloModule& module, const LogicalBuffer::SizeFunction& size_function,
-    SchedulerAlgorithm algorithm = SchedulerAlgorithm::kAuto);
+CreateMemoryMinimizingSequence(const HloModule& module,
+                               const LogicalBuffer::SizeFunction& size_function,
+                               const MemorySchedulerAlgorithm& algorithm = {});
 
 // Overload of above that computes the sequence for a single computation.
 StatusOr<std::vector<const HloInstruction*>> CreateMemoryMinimizingSequence(
     const HloComputation& computation,
     const LogicalBuffer::SizeFunction& size_function,
-    SchedulerAlgorithm algorithm = SchedulerAlgorithm::kAuto);
+    const MemorySchedulerAlgorithm& algorithm = {});
 
 }  // namespace xla
 
diff --git a/tensorflow/compiler/xla/service/hlo_scheduling_test.cc b/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
index 2dd6e43851..74544c4a67 100644
--- a/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_scheduling_test.cc
@@ -165,8 +165,7 @@ ENTRY root {
   };
   TF_ASSERT_OK_AND_ASSIGN(
       SequentialHloOrdering::HloModuleSequence sequence,
-      CreateMemoryMinimizingSequence(*module, size_fn,
-                                     SchedulerAlgorithm::kListSchedule));
+      CreateMemoryMinimizingSequence(*module, size_fn, ListMemoryScheduler));
   // Verify that all instructions are in the sequence.
   EXPECT_EQ(module->entry_computation()->instruction_count(),
             sequence.at(module->entry_computation()).size());
-- 
GitLab


From 49ee96a60bea1b595cff3cb550cfc8d2ade5ed8b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 16:13:58 -0700
Subject: [PATCH 271/960] [XLA] Use IDs instead of names to represent the edges
 of HLO graph in hlo.proto.

PiperOrigin-RevId: 189831057
---
 .../xla/client/xla_client/xla_builder.cc      | 17 +++---
 tensorflow/compiler/xla/service/hlo.proto     | 34 +++++++-----
 .../compiler/xla/service/hlo_computation.cc   | 27 +++++-----
 .../compiler/xla/service/hlo_computation.h    | 18 ++++---
 .../compiler/xla/service/hlo_instruction.cc   | 54 ++++++++++---------
 .../compiler/xla/service/hlo_instruction.h    |  8 +--
 tensorflow/compiler/xla/service/hlo_module.cc | 18 +++++--
 7 files changed, 104 insertions(+), 72 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 6328a4f350..8829fc6cca 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -99,16 +99,17 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
 
   // Not all instructions can be roots. Walk backwards from the last added
   // instruction until a valid root is found.
+  entry.set_root_id(-1);
   for (int64 i = instructions_.size() - 1; i >= 0; i--) {
     TF_ASSIGN_OR_RETURN(HloOpcode opcode,
                         StringToHloOpcode(instructions_[i].opcode()));
     if (CanBeRoot(opcode)) {
-      entry.set_root_name(instructions_[i].name());
+      entry.set_root_id(instructions_[i].id());
       *program_shape->mutable_result() = instructions_[i].shape();
       break;
     }
   }
-  if (entry.root_name().empty()) {
+  if (entry.root_id() == -1) {
     return FailedPrecondition("no root instruction was found");
   }
 
@@ -141,7 +142,9 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
   XlaComputation computation(id);
   HloModuleProto* module = computation.mutable_proto();
   module->set_name(entry.name());
+  module->set_id(entry.id());
   module->set_entry_computation_name(entry.name());
+  module->set_entry_computation_id(entry.id());
   *module->mutable_program_shape() = entry.program_shape();
   for (auto& e : embedded_) {
     module->add_computations()->Swap(&e.second);
@@ -162,8 +165,8 @@ XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
                         ShapeInference::InferBinaryOpShape(
                             HloOpcode::kAdd, lhs_instr->shape(),
                             rhs_instr->shape(), broadcast_dimensions));
-    instr.add_operand_names(lhs_instr->name());
-    instr.add_operand_names(rhs_instr->name());
+    instr.add_operand_ids(lhs_instr->id());
+    instr.add_operand_ids(rhs_instr->id());
     return AddInstruction(std::move(instr));
   };
   return NoteErrorOrReturn(op());
@@ -195,11 +198,12 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
     // Add input operands.
     for (const auto& operand : operands) {
       TF_ASSIGN_OR_RETURN(auto operand_instr, LookUpInstruction(operand));
-      instr.add_operand_names(operand_instr->name());
+      instr.add_operand_ids(operand_instr->id());
     }
 
     // Add called computation.
-    *instr.add_called_computation_names() = computation.proto().name();
+    instr.add_called_computation_ids(
+        computation.proto().entry_computation_id());
     for (const HloComputationProto& e : computation.proto().computations()) {
       embedded_.insert({e.id(), e});
     }
@@ -229,6 +233,7 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
 
 XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr) {
   const int64 handle = instructions_.size();
+  instr.set_id(handle);
   if (instr.name().empty()) {
     instr.set_name(StrCat(instr.opcode(), ".", handle));
   } else {
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index b86fbd821b..406feadfd4 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -13,13 +13,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// DO NOT USE THESE PROTO MESSAGES FOR ANYTHING OTHER THAN DEBUGGING.
-//
-// Don't use these protos in the real compilation or execution codepaths. The
-// data format is meant for debugging only, and may change without notice.
+// This proto file defines messages which represent the HLO module. This is a
+// full fidelity serialization of the c++ HLO constructs.
 //
 // Many of the protos below are simple 1-to-1 serializations of the
-// corresponding C++ classes.
+// corresponding C++ classes, e.g., HloModule, HloComputation, and
+// HloInstruction.
 //
 // FIELD NAMES ARE IMPORTANT
 //
@@ -40,16 +39,17 @@ message HloInstructionProto {
   reserved "parameter_name";
   reserved 12;
   reserved "fused_instructions_computation";
+  reserved 4;
+  reserved "operand_names";
+  reserved 5;
+  reserved "control_predecessor_names";
+  reserved 6;
+  reserved "called_computation_names";
 
   string name = 1;
   string opcode = 2;
   xla.Shape shape = 3;
 
-  // TODO(b/67782397): Replace instruction names with HloInstruction ids.
-  repeated string operand_names = 4;
-  repeated string control_predecessor_names = 5;
-  repeated string called_computation_names = 6;
-
   xla.OpMetadata metadata = 7;
 
   // Literal, only present for kConstant.
@@ -137,30 +137,38 @@ message HloInstructionProto {
 
   // The id of this instruction.
   int64 id = 35;
+
+  repeated int64 operand_ids = 36;
+  repeated int64 control_predecessor_ids = 37;
+  repeated int64 called_computation_ids = 38;
 }
 
 // Serialization of HloComputation.
 message HloComputationProto {
+  reserved 3;
+  reserved "root_name";
+
   string name = 1;
 
   // The array of instructions is always in a valid dependency order, where
   // operands appear before their users.
   repeated HloInstructionProto instructions = 2;
 
-  // The name of the root of the computation.
-  string root_name = 3;
-
   // The program shape (with layout) of this computation.
   xla.ProgramShape program_shape = 4;
 
   // The id of this computation.
   int64 id = 5;
+
+  // The id of the root of the computation.
+  int64 root_id = 6;
 }
 
 // Serialization of HloModule.
 message HloModuleProto {
   string name = 1;
   string entry_computation_name = 2;
+  int64 entry_computation_id = 6;
 
   // The array of computations is always in a valid dependency order, where
   // callees appear before their callers.
diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 4e852190a8..6f983d0b95 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -65,6 +65,7 @@ HloComputation::HloComputation(
     std::vector<std::unique_ptr<HloInstruction>>* instructions,
     HloInstruction* root_instruction, HloInstruction* fusion_instruction)
     : name_(name),
+      unique_id_(-1),
       root_instruction_(root_instruction),
       fusion_instruction_(fusion_instruction) {
   param_instructions_.resize(parameter_count, nullptr);
@@ -101,7 +102,7 @@ HloInstruction* HloComputation::AddInstructionInternal(
     instruction->UniquifyName(&parent()->instruction_name_uniquer());
     instruction->SetUniqueId(parent()->NewUniqueInstructionId());
   }
-  Reparent(instruction.get());
+  instruction->set_parent(this);
   HloInstruction* pinst = instruction.get();
   instruction_iterators_[pinst] =
       instructions_.insert(instructions_.end(), std::move(instruction));
@@ -158,10 +159,6 @@ Status HloComputation::RemoveParameter(int64 param_no) {
   return Status::OK();
 }
 
-void HloComputation::Reparent(HloInstruction* instruction) {
-  instruction->set_parent(this);
-}
-
 bool HloComputation::IsRemovable(const HloInstruction* instruction) {
   // If the instruction has control predecessors or successors then we cannot
   // remove the instruction without violating ordering constraints (added, for
@@ -393,12 +390,16 @@ string HloComputation::ToString(const HloPrintOptions& options) const {
 
 HloComputationProto HloComputation::ToProto() const {
   HloComputationProto proto;
+  CHECK(unique_id_ != -1)
+      << "This computation does not have a valid id. Please make sure the "
+         "computation is inside a module before dumping it.";
+  proto.set_id(unique_id_);
   proto.set_name(name_);
   for (const HloInstruction* instruction : MakeInstructionPostOrder()) {
     HloInstructionProto instruction_proto = instruction->ToProto();
     proto.add_instructions()->Swap(&instruction_proto);
   }
-  proto.set_root_name(root_instruction()->name());
+  proto.set_root_id(root_instruction()->unique_id());
   *proto.mutable_program_shape() = ComputeProgramShape();
   return proto;
 }
@@ -406,9 +407,9 @@ HloComputationProto HloComputation::ToProto() const {
 /* static */ StatusOr<std::unique_ptr<HloComputation>>
 HloComputation::CreateFromProto(
     HloModule* module, const HloComputationProto& proto,
-    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map) {
+    const tensorflow::gtl::FlatMap<int64, HloComputation*>& computation_map) {
   std::vector<std::unique_ptr<HloInstruction>> instructions;
-  tensorflow::gtl::FlatMap<string, HloInstruction*> instruction_map;
+  tensorflow::gtl::FlatMap<int64, HloInstruction*> instruction_map;
   int64 parameter_count = 0;
   for (const HloInstructionProto& instruction_proto : proto.instructions()) {
     TF_ASSIGN_OR_RETURN(
@@ -418,14 +419,14 @@ HloComputation::CreateFromProto(
     if (instruction->opcode() == HloOpcode::kParameter) {
       parameter_count++;
     }
-    TF_RET_CHECK(!ContainsKey(instruction_map, instruction->name()));
-    instruction_map[instruction->name()] = instruction.get();
+    TF_RET_CHECK(!ContainsKey(instruction_map, instruction_proto.id()));
+    instruction_map[instruction_proto.id()] = instruction.get();
     instructions.push_back(std::move(instruction));
   }
 
-  TF_RET_CHECK(!proto.root_name().empty());
-  TF_RET_CHECK(ContainsKey(instruction_map, proto.root_name()));
-  HloInstruction* root = instruction_map.at(proto.root_name());
+  TF_RET_CHECK(proto.root_id() != -1);
+  TF_RET_CHECK(ContainsKey(instruction_map, proto.root_id()));
+  HloInstruction* root = instruction_map.at(proto.root_id());
   return WrapUnique(new HloComputation(proto.name(), parameter_count,
                                        &instructions, root,
                                        /*fusion_instruction=*/nullptr));
diff --git a/tensorflow/compiler/xla/service/hlo_computation.h b/tensorflow/compiler/xla/service/hlo_computation.h
index 630d3675de..9d3f6e9a2c 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.h
+++ b/tensorflow/compiler/xla/service/hlo_computation.h
@@ -160,12 +160,12 @@ class HloComputation {
   //   module: the module which will contain the computation. The newly created
   //     computation is *not* added to the module, however.
   //   proto: the proto to convert from.
-  //   computation_map: a map from computation name to HloComputation*. This map
+  //   computation_map: a map from computation id to HloComputation*. This map
   //     must contain all computations which the newly constructed computation
   //     calls.
   static StatusOr<std::unique_ptr<HloComputation>> CreateFromProto(
       HloModule* module, const HloComputationProto& proto,
-      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map);
+      const tensorflow::gtl::FlatMap<int64, HloComputation*>& computation_map);
 
   // Gets the instructions in this computation.
   //
@@ -334,6 +334,15 @@ class HloComputation {
     fusion_instruction_ = fusion_instruction;
   }
 
+  // The id of this computation should be unique within the module.
+  void SetUniqueId(int64 id) {
+    CHECK_EQ(unique_id_, -1);
+    CHECK_GE(id, 0);
+    unique_id_ = id;
+  }
+
+  int64 unique_id() const { return unique_id_; }
+
  private:
   explicit HloComputation(
       const string& name, int parameter_count,
@@ -344,10 +353,6 @@ class HloComputation {
   HloInstruction* AddInstructionInternal(
       std::unique_ptr<HloInstruction> instruction);
 
-  // Helper for setting the parent of instructions that are added to this
-  // computation.
-  void Reparent(HloInstruction* instruction);
-
   // Fuses HLOs in instructions_to_fuse into fusion_instruction.
   //
   // Pre-condition: fusion_instruction's opcode is kFusion.
@@ -365,6 +370,7 @@ class HloComputation {
   std::vector<HloInstruction*> CollectUnreachableRoots() const;
 
   string name_;
+  int64 unique_id_;
   HloInstruction* root_instruction_;
 
   // If this computation is a fusion computation, this field points to the
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index 83fcc5da6d..a2a2c1e615 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -52,22 +52,22 @@ using ::tensorflow::strings::StrCat;
 /* static */
 StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     HloModule* module, const HloInstructionProto& proto,
-    const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
-    const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map) {
+    const tensorflow::gtl::FlatMap<int64, HloInstruction*>& instruction_map,
+    const tensorflow::gtl::FlatMap<int64, HloComputation*>& computation_map) {
   TF_RET_CHECK(!proto.opcode().empty());
   TF_ASSIGN_OR_RETURN(HloOpcode opcode, StringToHloOpcode(proto.opcode()));
   TF_RET_CHECK(proto.has_shape());
 
   auto instruction = WrapUnique(new HloInstruction(opcode, proto.shape()));
-  for (const string& operand_name : proto.operand_names()) {
-    TF_RET_CHECK(ContainsKey(instruction_map, operand_name))
-        << "No instruction named " << operand_name;
-    instruction->AppendOperand(instruction_map.at(operand_name));
-  }
-  for (const string& predecessor_name : proto.control_predecessor_names()) {
-    TF_RET_CHECK(ContainsKey(instruction_map, predecessor_name))
-        << "No instruction named " << predecessor_name;
-    TF_RETURN_IF_ERROR(instruction_map.at(predecessor_name)
+  for (const int64 operand_id : proto.operand_ids()) {
+    TF_RET_CHECK(ContainsKey(instruction_map, operand_id))
+        << "No instruction with id " << operand_id;
+    instruction->AppendOperand(instruction_map.at(operand_id));
+  }
+  for (const int64 predecessor_id : proto.control_predecessor_ids()) {
+    TF_RET_CHECK(ContainsKey(instruction_map, predecessor_id))
+        << "No instruction with id " << predecessor_id;
+    TF_RETURN_IF_ERROR(instruction_map.at(predecessor_id)
                            ->AddControlDependencyTo(instruction.get()));
   }
 
@@ -80,21 +80,21 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
                         StringToFusionKind(proto.fusion_kind()));
 
     // Find the fused computation and set its fusion instruction.
-    TF_RET_CHECK(proto.called_computation_names_size() == 1)
+    TF_RET_CHECK(proto.called_computation_ids_size() == 1)
         << "Expect 1 called computation for fusion instruction, but sees "
-        << proto.called_computation_names_size();
-    const string& fusion_name = proto.called_computation_names(0);
-    auto* fused_computation = FindPtrOrNull(computation_map, fusion_name);
+        << proto.called_computation_ids_size();
+    const int64 fusion_id = proto.called_computation_ids(0);
+    auto* fused_computation = FindPtrOrNull(computation_map, fusion_id);
     TF_RET_CHECK(fused_computation != nullptr)
-        << "No fusion computation named " << fusion_name;
+        << "No fusion computation with id " << fusion_id;
     fused_computation->SetFusionInstruction(instruction.get());
     instruction->called_computations_.push_back(fused_computation);
   } else {
-    for (const string& computation_name : proto.called_computation_names()) {
-      TF_RET_CHECK(ContainsKey(computation_map, computation_name))
-          << "No computation named " << computation_name;
+    for (const int64 computation_id : proto.called_computation_ids()) {
+      TF_RET_CHECK(ContainsKey(computation_map, computation_id))
+          << "No computation with id " << computation_id;
       instruction->called_computations_.push_back(
-          computation_map.at(computation_name));
+          computation_map.at(computation_id));
     }
   }
 
@@ -2315,14 +2315,18 @@ string HloInstruction::ToShortString() const {
 
 HloInstructionProto HloInstruction::ToProto() const {
   HloInstructionProto proto;
+  CHECK(unique_id_ != -1)
+      << "This instruction does not have a valid id. Please make sure the "
+         "instruction is inside a module before dumping it.";
+  proto.set_id(unique_id_);
   proto.set_name(name_);
   proto.set_opcode(HloOpcodeString(opcode_));
   *proto.mutable_shape() = shape_;
   for (const HloInstruction* operand : operands_) {
-    *proto.add_operand_names() = operand->name();
+    proto.add_operand_ids(operand->unique_id());
   }
   for (const HloInstruction* control : control_predecessors_) {
-    *proto.add_control_predecessor_names() = control->name();
+    proto.add_control_predecessor_ids(control->unique_id());
   }
 
   *proto.mutable_metadata() = metadata_;
@@ -2332,11 +2336,11 @@ HloInstructionProto HloInstruction::ToProto() const {
   proto.set_parameter_number(parameter_number_);
   if (opcode() == HloOpcode::kFusion) {
     proto.set_fusion_kind(xla::ToString(fusion_kind()));
-    *proto.add_called_computation_names() =
-        fused_instructions_computation()->name();
+    proto.add_called_computation_ids(
+        fused_instructions_computation()->unique_id());
   } else {
     for (const HloComputation* computation : called_computations_) {
-      *proto.add_called_computation_names() = computation->name();
+      proto.add_called_computation_ids(computation->unique_id());
     }
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index a111e1e4a6..a94ba145df 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -179,15 +179,15 @@ class HloInstruction {
   //   module: the module which will contain the instruction. The newly created
   //     instruction is *not* added to the module or any computation, however.
   //   proto: the proto to convert from.
-  //   instruction_map: a map from instruction name to HloInstruction*. This map
+  //   instruction_map: a map from instruction id to HloInstruction*. This map
   //     must contain all operands of the newly constructed instruction.
-  //   computation_map: a map from computation name to HloComputation*. This map
+  //   computation_map: a map from computation id to HloComputation*. This map
   //     must contain all computations which the newly constructed instruction
   //     calls.
   static StatusOr<std::unique_ptr<HloInstruction>> CreateFromProto(
       HloModule* module, const HloInstructionProto& proto,
-      const tensorflow::gtl::FlatMap<string, HloInstruction*>& instruction_map,
-      const tensorflow::gtl::FlatMap<string, HloComputation*>& computation_map);
+      const tensorflow::gtl::FlatMap<int64, HloInstruction*>& instruction_map,
+      const tensorflow::gtl::FlatMap<int64, HloComputation*>& computation_map);
 
   // Creates a parameter-retrieving instruction.
   static std::unique_ptr<HloInstruction> CreateParameter(int64 parameter_number,
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 4091ebbfd3..2037764dae 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -83,6 +83,11 @@ HloComputation* HloModule::AddComputationInternal(
   for (auto* instruction : computation->instructions()) {
     instruction->SetUniqueId(NewUniqueInstructionId());
   }
+  // Set unique id to this computation.
+  CHECK_NE(computation->root_instruction()->unique_id(), -1)
+      << "Root has no valid id: " << computation->ToString();
+  computation->SetUniqueId(computation->root_instruction()->unique_id());
+
   computation->set_parent(this);
   computations_.push_back(std::move(computation));
   return computations_.back().get();
@@ -204,8 +209,10 @@ string HloModule::ToString(const HloPrintOptions& options) const {
 
 HloModuleProto HloModule::ToProto() const {
   HloModuleProto proto;
+  proto.set_id(unique_id_);
   proto.set_name(name_);
   proto.set_entry_computation_name(entry_computation_->name());
+  proto.set_entry_computation_id(entry_computation_->unique_id());
   for (const HloComputation* computation : MakeComputationPostOrder()) {
     HloComputationProto computation_proto = computation->ToProto();
     if (computation->name() == entry_computation_->name()) {
@@ -249,19 +256,20 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   auto module = MakeUnique<HloModule>(proto.name(), entry_computation_handle,
                                       module_config);
 
-  tensorflow::gtl::FlatMap<string, HloComputation*> computation_map;
+  tensorflow::gtl::FlatMap<int64, HloComputation*> computation_map;
   for (const HloComputationProto& computation_proto : proto.computations()) {
     TF_ASSIGN_OR_RETURN(std::unique_ptr<HloComputation> computation,
                         HloComputation::CreateFromProto(
                             module.get(), computation_proto, computation_map));
     CHECK_NE(computation.get(), nullptr);
-    TF_RET_CHECK(!ContainsKey(computation_map, computation->name()));
-    string computation_name = computation->name();
+    int64 computation_id = computation_proto.id();
+    TF_RET_CHECK(computation_id != -1);
+    TF_RET_CHECK(!ContainsKey(computation_map, computation_id));
     // Don't uniquify names because we want names to be stable across
     // serialization and deserialization.
-    computation_map[computation_name] = module->AddComputationInternal(
+    computation_map[computation_id] = module->AddComputationInternal(
         std::move(computation),
-        /*is_entry=*/proto.entry_computation_name() == computation_name,
+        /*is_entry=*/proto.entry_computation_id() == computation_id,
         /*uniquify_names=*/false);
   }
   TF_RET_CHECK(module->entry_computation_ != nullptr);
-- 
GitLab


From 9d1c63a6516290a79b70c54aea1b8fd917be17f3 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Tue, 20 Mar 2018 16:32:22 -0700
Subject: [PATCH 272/960] Use softmax_crossentropy_with_logits_v2 in tf.keras
 since softmax_crossentropy_with_logits is deprecated.

PiperOrigin-RevId: 189833677
---
 tensorflow/python/keras/_impl/keras/backend.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py
index 04866fbe0f..7baf27642a 100644
--- a/tensorflow/python/keras/_impl/keras/backend.py
+++ b/tensorflow/python/keras/_impl/keras/backend.py
@@ -3373,7 +3373,7 @@ def categorical_crossentropy(target, output, from_logits=False):
         target * math_ops.log(output),
         axis=len(output.get_shape()) - 1)
   else:
-    return nn.softmax_cross_entropy_with_logits(labels=target, logits=output)
+    return nn.softmax_cross_entropy_with_logits_v2(labels=target, logits=output)
 
 
 @tf_export('keras.backend.sparse_categorical_crossentropy')
-- 
GitLab


From 4321469f1db7a6ff220c2415c63f433df6e7161d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 17:00:33 -0700
Subject: [PATCH 273/960]  Fixing bug in MultitaskOptimizerWrapper where types
 of tensors were mismatching.

PiperOrigin-RevId: 189837743
---
 .../opt/python/training/multitask_optimizer_wrapper.py     | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py b/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py
index cb6c77a86f..9076cc9d12 100644
--- a/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py
+++ b/tensorflow/contrib/opt/python/training/multitask_optimizer_wrapper.py
@@ -22,6 +22,7 @@ import types
 import six
 
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
@@ -40,8 +41,10 @@ def _get_wrapper(fn, opt):
 
   def wrapper(self, grad, *args, **kwargs):  # pylint: disable=unused-argument
     all_zeros = _is_all_zeros(grad)
-    return control_flow_ops.cond(all_zeros, control_flow_ops.no_op,
-                                 lambda: fn(grad, *args, **kwargs))
+    def call_fn():
+      with ops.control_dependencies([fn(grad, *args, **kwargs)]):
+        return control_flow_ops.no_op()
+    return control_flow_ops.cond(all_zeros, control_flow_ops.no_op, call_fn)
 
   wrapper = types.MethodType(wrapper, opt)
   return wrapper
-- 
GitLab


From 8dbcacd6a0ab68f2ebe90bda93bb915699313946 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 18:36:33 -0700
Subject: [PATCH 274/960] [XLA] Simpify XlaBuilder: extract common add
 instruction logic.

PiperOrigin-RevId: 189848174
---
 .../xla/client/xla_client/xla_builder.cc      | 61 ++++++++-----------
 .../xla/client/xla_client/xla_builder.h       |  6 +-
 2 files changed, 30 insertions(+), 37 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 8829fc6cca..82b61d4d51 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -51,21 +51,16 @@ bool CanBeRoot(HloOpcode opcode) {
   }
 }
 
-void SetOpcode(HloInstructionProto* instr, HloOpcode opcode) {
-  instr->set_opcode(HloOpcodeString(opcode));
-}
-
 }  // namespace
 
-StatusOr<std::unique_ptr<Shape>> XlaBuilder::GetShape(const XlaOp& op) const {
+StatusOr<Shape> XlaBuilder::GetShape(const XlaOp& op) const {
   TF_ASSIGN_OR_RETURN(auto instr, LookUpInstruction(op));
-  return MakeUnique<Shape>(instr->shape());
+  return instr->shape();
 }
 
 StatusOr<Shape> XlaOp::GetShape() const {
   TF_RET_CHECK(builder_ != nullptr);
-  TF_ASSIGN_OR_RETURN(auto shape, builder_->GetShape(*this));
-  return *shape;
+  return builder_->GetShape(*this);
 }
 
 XlaBuilder::XlaBuilder(const string& computation_name)
@@ -158,49 +153,41 @@ XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
   auto op = [&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    SetOpcode(&instr, HloOpcode::kAdd);
-    TF_ASSIGN_OR_RETURN(const auto* lhs_instr, LookUpInstruction(lhs));
-    TF_ASSIGN_OR_RETURN(const auto* rhs_instr, LookUpInstruction(rhs));
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
-                        ShapeInference::InferBinaryOpShape(
-                            HloOpcode::kAdd, lhs_instr->shape(),
-                            rhs_instr->shape(), broadcast_dimensions));
-    instr.add_operand_ids(lhs_instr->id());
-    instr.add_operand_ids(rhs_instr->id());
-    return AddInstruction(std::move(instr));
+    TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, lhs.GetShape());
+    TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, rhs.GetShape());
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferBinaryOpShape(HloOpcode::kAdd, lhs_shape,
+                                           rhs_shape, broadcast_dimensions));
+    return AddInstruction(std::move(instr), HloOpcode::kAdd, {lhs, rhs});
   };
   return NoteErrorOrReturn(op());
 }
 
 XlaOp XlaBuilder::ConstantLiteral(const Literal& literal) {
   HloInstructionProto instr;
-  SetOpcode(&instr, HloOpcode::kConstant);
   *instr.mutable_shape() = literal.shape();
   *instr.mutable_literal() = literal.ToProto();
-  return AddInstruction(std::move(instr));
+  return AddInstruction(std::move(instr), HloOpcode::kConstant);
 }
 
 XlaOp XlaBuilder::Call(const XlaComputation& computation,
                        tensorflow::gtl::ArraySlice<XlaOp> operands) {
   auto op = [&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    SetOpcode(&instr, HloOpcode::kCall);
-    std::vector<const Shape*> operand_shapes;
+    std::vector<const Shape*> operand_shape_ptrs;
+    std::vector<Shape> operand_shapes;
     for (const auto& operand : operands) {
-      TF_ASSIGN_OR_RETURN(const auto* input, LookUpInstruction(operand));
-      operand_shapes.push_back(&input->shape());
+      TF_ASSIGN_OR_RETURN(const Shape& shape, operand.GetShape());
+      operand_shapes.push_back(shape);
     }
+    c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
+                [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
                         ShapeInference::InferCallShape(
-                            operand_shapes,
+                            operand_shape_ptrs,
                             /*to_apply=*/computation.GetProgramShape()));
 
-    // Add input operands.
-    for (const auto& operand : operands) {
-      TF_ASSIGN_OR_RETURN(auto operand_instr, LookUpInstruction(operand));
-      instr.add_operand_ids(operand_instr->id());
-    }
-
     // Add called computation.
     instr.add_called_computation_ids(
         computation.proto().entry_computation_id());
@@ -208,7 +195,7 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
       embedded_.insert({e.id(), e});
     }
 
-    return AddInstruction(std::move(instr));
+    return AddInstruction(std::move(instr), HloOpcode::kCall, operands);
   };
   return NoteErrorOrReturn(op());
 }
@@ -217,7 +204,6 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
                             const string& name) {
   auto op = [&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
-    SetOpcode(&instr, HloOpcode::kParameter);
     if (parameter_numbers_.find(parameter_number) != parameter_numbers_.end()) {
       return InvalidArgument("parameter %lld already registered",
                              parameter_number);
@@ -226,20 +212,25 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
     instr.set_parameter_number(parameter_number);
     instr.set_name(name);
     *instr.mutable_shape() = shape;
-    return AddInstruction(std::move(instr));
+    return AddInstruction(std::move(instr), HloOpcode::kParameter);
   };
   return NoteErrorOrReturn(op());
 }
 
-XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr) {
+XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
+                                 tensorflow::gtl::ArraySlice<XlaOp> operands) {
   const int64 handle = instructions_.size();
   instr.set_id(handle);
+  instr.set_opcode(HloOpcodeString(opcode));
   if (instr.name().empty()) {
     instr.set_name(StrCat(instr.opcode(), ".", handle));
   } else {
     // Append the handle to make sure the name is unique.
     instr.set_name(StrCat(instr.name(), ".", handle));
   }
+  for (const auto& operand : operands) {
+    instr.add_operand_ids(operand.handle());
+  }
   instructions_.push_back(instr);
 
   XlaOp op(handle, this);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 7632bd289d..f1d10ecdb9 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -26,6 +26,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -157,14 +158,15 @@ class XlaBuilder {
   XlaOp ConstantR0(NativeT value);
 
   // Returns the shape of the given op.
-  StatusOr<std::unique_ptr<Shape>> GetShape(const XlaOp& op) const;
+  StatusOr<Shape> GetShape(const XlaOp& op) const;
 
   // Builds the computation with the requested operations, or returns a non-ok
   // status.
   StatusOr<XlaComputation> Build();
 
  private:
-  XlaOp AddInstruction(HloInstructionProto&& instr);
+  XlaOp AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
+                       tensorflow::gtl::ArraySlice<XlaOp> operands = {});
 
   // Notes that the error occurred by:
   // * storing it internally and capturing a backtrace if it's the first error
-- 
GitLab


From 8e4ee96bcce043bdb221e8fe116581ae90bae00d Mon Sep 17 00:00:00 2001
From: "Ziyue(Louis) Lu" <luziyuel@gmail.com>
Date: Tue, 20 Mar 2018 18:43:20 -0700
Subject: [PATCH 275/960] Updated README.md

Edited the word 'lets you' to 'enables you to', which I think is better for this sentence. The word 'enable' here gives the readers a feeling that the architecture makes some awesome things possible, while 'let' is more like to give some permissions to the users.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 3cdb6e478d..0a309ebe2d 100644
--- a/README.md
+++ b/README.md
@@ -12,7 +12,7 @@
 **TensorFlow** is an open source software library for numerical computation using
 data flow graphs.  The graph nodes represent mathematical operations, while
 the graph edges represent the multidimensional data arrays (tensors) that flow
-between them.  This flexible architecture lets you deploy computation to one
+between them.  This flexible architecture enables you to deploy computation to one
 or more CPUs or GPUs in a desktop, server, or mobile device without rewriting
 code.  TensorFlow also includes TensorBoard, a data visualization toolkit.
 
-- 
GitLab


From 03b742c7edd04d18ef0b1bdd5539fd543fe34c4a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 18:45:17 -0700
Subject: [PATCH 276/960] Minor documentation fix

PiperOrigin-RevId: 189848838
---
 tensorflow/python/training/saver.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 5f68eec6ce..5ef8bd9e9c 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1969,7 +1969,7 @@ def export_meta_graph(filename=None,
     saver_def: `SaverDef` protocol buffer.
     collection_list: List of string keys to collect.
     as_text: If `True`, writes the `MetaGraphDef` as an ASCII proto.
-    graph: The `Graph` to import into. If `None`, use the default graph.
+    graph: The `Graph` to export. If `None`, use the default graph.
     export_scope: Optional `string`. Name scope under which to extract
       the subgraph. The scope name will be striped from the node definitions
       for easy import later into new name scopes. If `None`, the whole graph
-- 
GitLab


From 239769f55cfd703270e14257cedb3abdf51a6423 Mon Sep 17 00:00:00 2001
From: Chris Leary <leary@google.com>
Date: Tue, 20 Mar 2018 19:14:26 -0700
Subject: [PATCH 277/960] [XLA] Plumb hlo dump options via local client.

PiperOrigin-RevId: 189851211
---
 .../xla/client/executable_build_options.cc    | 22 ++++++++
 .../xla/client/executable_build_options.h     | 15 ++++++
 .../xla/python/local_computation_builder.i    | 52 +++++++++++++++----
 .../compiler/xla/python/numpy_bridge.cc       |  3 +-
 tensorflow/compiler/xla/python/numpy_bridge.h |  3 ++
 tensorflow/compiler/xla/python/xla_client.py  |  2 +
 .../compiler/xla/service/local_service.cc     | 10 ++++
 7 files changed, 94 insertions(+), 13 deletions(-)

diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc
index d84f2018e1..4ff4da6215 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.cc
+++ b/tensorflow/compiler/xla/client/executable_build_options.cc
@@ -76,6 +76,28 @@ ExecutableBuildOptions::generate_hlo_graph() const {
   return generate_hlo_graph_;
 }
 
+ExecutableBuildOptions& ExecutableBuildOptions::set_dump_optimized_hlo_proto_to(
+    tensorflow::StringPiece dirpath) {
+  dump_optimized_hlo_proto_to_ = dirpath.ToString();
+  return *this;
+}
+
+const tensorflow::gtl::optional<string>&
+ExecutableBuildOptions::dump_optimized_hlo_proto_to() const {
+  return dump_optimized_hlo_proto_to_;
+}
+
+ExecutableBuildOptions& ExecutableBuildOptions::set_dump_per_pass_hlo_proto_to(
+    tensorflow::StringPiece dirpath) {
+  dump_per_pass_hlo_proto_to_ = dirpath.ToString();
+  return *this;
+}
+
+const tensorflow::gtl::optional<string>&
+ExecutableBuildOptions::dump_per_pass_hlo_proto_to() const {
+  return dump_per_pass_hlo_proto_to_;
+}
+
 ExecutableBuildOptions& ExecutableBuildOptions::set_hlo_profile(bool enabled) {
   hlo_profile_ = enabled;
   return *this;
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index 3e18e5de64..85b2cd96cb 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/optional.h"
 
 namespace xla {
@@ -57,6 +58,18 @@ class ExecutableBuildOptions {
   ExecutableBuildOptions& set_generate_hlo_graph(string regex);
   const tensorflow::gtl::optional<string>& generate_hlo_graph() const;
 
+  // If set, specifies a dirpath to dump the end-of-optimization-pipeline HLO
+  // protobuf to (as in DebugOptions).
+  ExecutableBuildOptions& set_dump_optimized_hlo_proto_to(
+      tensorflow::StringPiece dirpath);
+  const tensorflow::gtl::optional<string>& dump_optimized_hlo_proto_to() const;
+
+  // If set, specifies a dirpath to dump the per-pass-in-pipeline HLO protobufs
+  // to (as in DebugOptions).
+  ExecutableBuildOptions& set_dump_per_pass_hlo_proto_to(
+      tensorflow::StringPiece dirpath);
+  const tensorflow::gtl::optional<string>& dump_per_pass_hlo_proto_to() const;
+
   // If set, specifies that we should record an HLO profile during execution and
   // log it after execution (as in DebugOptions).
   ExecutableBuildOptions& set_hlo_profile(bool enabled);
@@ -72,6 +85,8 @@ class ExecutableBuildOptions {
   Shape result_layout_;
   bool result_layout_set_ = false;
   tensorflow::gtl::optional<string> generate_hlo_graph_;
+  tensorflow::gtl::optional<string> dump_optimized_hlo_proto_to_;
+  tensorflow::gtl::optional<string> dump_per_pass_hlo_proto_to_;
   DeviceMemoryAllocator* device_allocator_ = nullptr;
 };
 
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index ca91cf0d50..8f231d1a12 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -141,6 +141,33 @@ bool GetIntAttr(PyObject* o, const char* field, int64* result) {
   return true;
 }
 
+// Returns "ok"; true if there is no error, false if there was an error.
+bool HandleStringAttribute(PyObject* o,
+                           const char* attr_name,
+                           std::function<void(string s)> f) {
+  if (!PyObject_HasAttrString(o, attr_name)) {
+    return true;  // It's ok for the object to not have the attribute.
+  }
+  PyObject* attr = PyObject_GetAttrString(o, attr_name);
+  if (attr == nullptr) {
+    return false;  // An error occurred getting the attribute.
+  }
+  if (attr == Py_None) {
+    Py_DECREF(attr);
+    return true;  // The attribute is None, which we consider ok.
+  }
+  if (!PyString_Check(attr)) {
+    string message = tensorflow::strings::Printf("%s must be a string or none; got %s",
+        attr_name, numpy::PyObjectCppRepr(attr).c_str());
+    PyErr_SetString(PyExc_TypeError, message.c_str());
+    Py_DECREF(attr);
+    return false;  // Type error, not ok.
+  }
+  f(PyString_AsString(attr));
+  Py_DECREF(attr);
+  return true;  // Handled string attribute, ok!
+}
+
 }
 }
 %}
@@ -820,20 +847,23 @@ tensorflow::ImportNumpy();
   if ($input == Py_None) {
     $1 = NULL;
   } else {
-    PyObject* o = PyObject_GetAttrString($input, "generate_hlo_graph");
-    if (!o) {
-      return NULL;
+    if (!HandleStringAttribute($input, "generate_hlo_graph", [&](string s) {
+      build_options.set_generate_hlo_graph(std::move(s));
+    })) {
+      return nullptr;
     }
-    if (o != Py_None) {
-      if (!PyString_Check(o)) {
-        PyErr_SetString(PyExc_TypeError, "ExecutableBuildOptions.generate_hlo_graph must be a string or None.");
-        return NULL;
-      }
-      build_options.set_generate_hlo_graph(PyString_AsString(o));
+    if (!HandleStringAttribute($input, "dump_optimized_hlo_proto_to", [&](string s) {
+      build_options.set_dump_optimized_hlo_proto_to(std::move(s));
+    })) {
+      return nullptr;
+    }
+    if (!HandleStringAttribute($input, "dump_per_pass_hlo_proto_to", [&](string s) {
+      build_options.set_dump_per_pass_hlo_proto_to(std::move(s));
+    })) {
+      return nullptr;
     }
-    Py_DECREF(o);
 
-    o = PyObject_GetAttrString($input, "hlo_profile");
+    PyObject* o = PyObject_GetAttrString($input, "hlo_profile");
     if (o == NULL) {
       return NULL;
     }
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.cc b/tensorflow/compiler/xla/python/numpy_bridge.cc
index 3d87480728..eec48479c9 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.cc
+++ b/tensorflow/compiler/xla/python/numpy_bridge.cc
@@ -170,8 +170,7 @@ static string PyObjectCppStr(PyObject* o) {
   return ExtractStringAndDecref(s);
 }
 
-// Safely returns a repr of the given Python object o as a C++ string.
-static string PyObjectCppRepr(PyObject* o) {
+string PyObjectCppRepr(PyObject* o) {
   PyObject* r = PyObject_Repr(o);
   return ExtractStringAndDecref(r);
 }
diff --git a/tensorflow/compiler/xla/python/numpy_bridge.h b/tensorflow/compiler/xla/python/numpy_bridge.h
index adfcc3b858..9656cb1c31 100644
--- a/tensorflow/compiler/xla/python/numpy_bridge.h
+++ b/tensorflow/compiler/xla/python/numpy_bridge.h
@@ -107,6 +107,9 @@ void CopyLiteralToNumpyArray(const Literal& literal, PyArrayObject* py_array) {
   std::copy(source.begin(), source.end(), dest);
 }
 
+// Safely returns a repr of the given Python object o as a C++ string.
+string PyObjectCppRepr(PyObject* o);
+
 // Workarounds for Python 2 and 3 interop
 
 PyObject* LongToPyIntOrPyLong(long x);  // NOLINT
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index d747a0b65c..e548d420f4 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -320,6 +320,8 @@ class CompileOptions(object):
 
   def __init__(self):
     self.generate_hlo_graph = None
+    self.dump_optimized_hlo_proto_to = None
+    self.dump_per_pass_hlo_proto_to = None
     self.hlo_profile = False
 
 
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 74aa6eaa17..7fd1ccd1a8 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -125,6 +125,16 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
     execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
         build_options.generate_hlo_graph().value());
   }
+  if (build_options.dump_optimized_hlo_proto_to().has_value()) {
+    execution_options.mutable_debug_options()
+        ->set_xla_dump_optimized_hlo_proto_to(
+            build_options.dump_optimized_hlo_proto_to().value());
+  }
+  if (build_options.dump_per_pass_hlo_proto_to().has_value()) {
+    execution_options.mutable_debug_options()
+        ->set_xla_dump_per_pass_hlo_proto_to(
+            build_options.dump_per_pass_hlo_proto_to().value());
+  }
   if (build_options.result_layout() != nullptr) {
     *execution_options.mutable_shape_with_output_layout() =
         *build_options.result_layout();
-- 
GitLab


From 5e8f1530b4f355d66f02929e948ed02c096bcaaa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 20 Mar 2018 19:48:18 -0700
Subject: [PATCH 278/960] Add reduce window tests for the cases when the input
 shape has 1 element.

PiperOrigin-RevId: 189853631
---
 .../compiler/xla/tests/reduce_window_test.cc  | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 8b736f62f0..f66fb5cacc 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -1351,5 +1351,41 @@ ENTRY R2Window {
   EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{0.001}));
 }
 
+TEST_F(ReduceWindowTextTest, R2EffectiveScalar) {
+  const string& hlo_string = R"(
+HloModule R2Window
+mul {
+  lhs = f32[] parameter(0)
+  rhs = f32[] parameter(1)
+  ROOT mul = f32[] multiply(lhs, rhs)
+}
+ENTRY R2Window {
+  operand = f32[1,1]{1,0} parameter(0)
+  negate = f32[1,1]{1,0} negate(operand)
+  constant = f32[] constant(1)
+  ROOT reduce-window = f32[1,1]{1,0} reduce-window(negate, constant), window={size=1x1 pad=0_0x0_0}, to_apply=mul
+}
+)";
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{0.001}));
+}
+
+TEST_F(ReduceWindowTextTest, R3EffectiveScalar) {
+  const string& hlo_string = R"(
+HloModule R3Window
+mul {
+  lhs = f32[] parameter(0)
+  rhs = f32[] parameter(1)
+  ROOT mul = f32[] multiply(lhs, rhs)
+}
+ENTRY R3Window {
+  operand = f32[1,1,1]{2,1,0} parameter(0)
+  negate = f32[1,1,1]{2,1,0} negate(operand)
+  constant = f32[] constant(1)
+  ROOT reduce-window = f32[1,1,1]{2,1,0} reduce-window(negate, constant), window={size=1x1x1 pad=0_0x0_0x0_0}, to_apply=mul
+}
+)";
+  EXPECT_TRUE(RunAndCompare(hlo_string, ErrorSpec{0.001}));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From aab6d07a0df3326ce416c331ff951a77fe802a8e Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Tue, 20 Mar 2018 20:25:03 -0700
Subject: [PATCH 279/960] Remove recently introduced LOG INFO statements from
 AvgPoolingOp

PiperOrigin-RevId: 189856039
---
 tensorflow/core/kernels/avgpooling_op.cc | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/core/kernels/avgpooling_op.cc b/tensorflow/core/kernels/avgpooling_op.cc
index a763f1321f..c581d1451f 100644
--- a/tensorflow/core/kernels/avgpooling_op.cc
+++ b/tensorflow/core/kernels/avgpooling_op.cc
@@ -156,13 +156,11 @@ class AvgPoolingOp<GPUDevice, T> : public UnaryOp<T> {
     TensorShape output_shape = params.forward_output_shape();
 
     if (data_format_ == FORMAT_NCHW) {
-      LOG(INFO) << "DnnPoolingOp";
       DnnPoolingOp<T>::Compute(
           context, perftools::gputools::dnn::PoolingMode::kAverage, ksize_,
           stride_, padding_, data_format_, tensor_in, output_shape,
           /*propagate_nans=*/false);
     } else {
-      LOG(INFO) << "SpatialAvgPooling";
       Tensor* output = nullptr;
       OP_REQUIRES_OK(context,
                      context->allocate_output(0, output_shape, &output));
-- 
GitLab


From 9ef37abc6f3da1d3d1699293f1afb52494125161 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 20 Mar 2018 20:28:36 -0700
Subject: [PATCH 280/960] Revert "Fix dataset resampling bug introduced by a
 bug in datasets itself. fixes #16606 " (#17874)

---
 .../data/python/kernel_tests/resample_test.py | 39 -------------------
 .../contrib/data/python/ops/resampling.py     | 11 +++---
 2 files changed, 5 insertions(+), 45 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 38efcd3cba..913ab9b9f8 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -21,11 +21,8 @@ import numpy as np
 
 from tensorflow.contrib.data.python.ops import resampling
 from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -71,42 +68,6 @@ class ResampleTest(test.TestCase):
     returned_dist = class_counts / total_returned
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
-  def testRandomClasses(self):
-    init_dist = [0.25, 0.25, 0.25, 0.25]
-    target_dist = [0.0, 0.0, 0.0, 1.0]
-    num_classes = len(init_dist)
-    num_samples = 100   # We don't need many samples to test a dirac-delta target distribution
-    data_np = np.random.choice(num_classes, num_samples, p=init_dist)
-
-    dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
-
-    # Apply a random mapping that preserves the data distribution.
-    def _remap_fn(_):
-      return math_ops.cast(random_ops.random_uniform([1]) * num_classes,
-                           dtypes.int32)[0]
-    dataset = dataset.map(_remap_fn)
-
-    # Reshape distribution.
-    dataset = dataset.apply(
-        resampling.rejection_resample(
-            class_func=lambda x: x,
-            target_dist=target_dist,
-            initial_dist=init_dist))
-
-    get_next = dataset.make_one_shot_iterator().get_next()
-
-    with self.test_session() as sess:
-      returned = []
-      with self.assertRaises(errors.OutOfRangeError):
-        while True:
-          returned.append(sess.run(get_next))
-
-    classes, _ = zip(*returned)
-    bincount = np.bincount(
-        np.array(classes),
-        minlength=num_classes).astype(np.float32) / len(classes)
-
-    self.assertAllClose(target_dist, bincount, atol=1e-2)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index 0e127f72cd..f4015f19fb 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -101,13 +101,12 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
                                                    initial_dist_ds))
                           .map(maybe_warn_on_large_rejection))
 
-    def _gather_and_copy(class_val, acceptance_prob, data):
-      return (class_val, array_ops.gather(acceptance_prob, class_val), data)
-    current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
-          (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
+    current_probabilities_ds = dataset_ops.Dataset.zip(
+        (acceptance_dist_ds, class_values_ds)).map(array_ops.gather)
     filtered_ds = (
-          current_probabilities_and_class_and_data_ds
-              .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
+        dataset_ops.Dataset.zip((class_values_ds, current_probabilities_ds,
+                                 dataset))
+        .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
   return _apply_fn
-- 
GitLab


From e79eb0b8de130bf905a101608681e9c18561356c Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Tue, 20 Mar 2018 20:28:38 -0700
Subject: [PATCH 281/960] Fix windows GPU build scripts. (#17870)

PiperOrigin-RevId: 188629017
---
 tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat b/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat
index b87e4a9bec..4656afe025 100644
--- a/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat
+++ b/tensorflow/tools/ci_build/windows/gpu/cmake/run_build.bat
@@ -37,7 +37,7 @@ SET CMAKE_DIR=%REPO_ROOT%\tensorflow\contrib\cmake
 SET MSBUILD_EXE="C:\Program Files (x86)\MSBuild\14.0\Bin\msbuild.exe"
 
 :: Run cmake to create Visual Studio Project files.
-%CMAKE_EXE% %CMAKE_DIR% -A x64 -DSWIG_EXECUTABLE=%SWIG_EXE% -DPYTHON_EXECUTABLE=%PY_EXE% -DCMAKE_BUILD_TYPE=Release -DPYTHON_LIBRARIES=%PY_LIB% -Dtensorflow_BUILD_PYTHON_TESTS=%BUILD_PYTHON_TESTS% -Dtensorflow_BUILD_CC_TESTS=%BUILD_CC_TESTS% -Dtensorflow_ENABLE_GPU=ON -DCUDNN_HOME=%CUDNN_HOME% -Dtensorflow_TF_NIGHTLY=%TF_NIGHTLY% -Dtensorflow_DISABLE_EIGEN_FORCEINLINE=%DISABLE_FORCEINLINE% -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX
+%CMAKE_EXE% %CMAKE_DIR% -A x64 -DSWIG_EXECUTABLE=%SWIG_EXE% -DPYTHON_EXECUTABLE=%PY_EXE% -DCMAKE_BUILD_TYPE=Release -DPYTHON_LIBRARIES=%PY_LIB% -Dtensorflow_BUILD_PYTHON_TESTS=%BUILD_PYTHON_TESTS% -Dtensorflow_BUILD_CC_TESTS=%BUILD_CC_TESTS% -Dtensorflow_ENABLE_GPU=ON -DCUDNN_HOME=%CUDNN_HOME% -Dtensorflow_TF_NIGHTLY=%TF_NIGHTLY% -Dtensorflow_DISABLE_EIGEN_FORCEINLINE=%DISABLE_FORCEINLINE% -Dtensorflow_WIN_CPU_SIMD_OPTIONS=/arch:AVX -G"Visual Studio 14"
 
 :: Run msbuild in the resulting VS project files to build a pip package.
 %MSBUILD_EXE% /p:Configuration=Release /maxcpucount:32 tf_python_build_pip_package.vcxproj
-- 
GitLab


From 4e108ef30d7cd7ae5e1c550ec5ae27e79b8c6e39 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 20 Mar 2018 20:28:58 -0700
Subject: [PATCH 282/960] Revert "Windows: Enable tensorflow/contrib in Bazel
 build (#16659)" (#17774)

* Revert "Windows: Enable tensorflow/contrib in Bazel build (#16659)"

This reverts commit c6a12c77a50778e28de3590f4618bc2b62f3ecab.

* Add kafka back to contrib_py rule.

* Update __init__.py
---
 configure.py                                  |   2 +-
 tensorflow/contrib/BUILD                      |   8 +-
 tensorflow/contrib/__init__.py                |   6 +-
 .../boosted_trees/lib/utils/batch_features.h  |   6 +-
 tensorflow/contrib/distributions/BUILD        |   2 -
 tensorflow/contrib/eager/python/BUILD         |   5 +-
 .../python/examples/linear_regression/BUILD   |   1 -
 tensorflow/contrib/gan/BUILD                  |   1 -
 .../contrib/kfac/python/kernel_tests/BUILD    |   1 -
 tensorflow/contrib/labeled_tensor/BUILD       |   1 -
 tensorflow/contrib/layers/BUILD               |   2 -
 tensorflow/contrib/learn/BUILD                |   5 -
 tensorflow/contrib/lookup/BUILD               |   1 -
 tensorflow/contrib/py2tf/converters/BUILD     |   2 -
 tensorflow/contrib/py2tf/utils/BUILD          |   1 -
 .../contrib/remote_fused_graph/pylib/BUILD    |   1 +
 tensorflow/contrib/saved_model/BUILD          |   1 -
 tensorflow/contrib/session_bundle/BUILD       |   1 -
 .../contrib/slim/python/slim/data/BUILD       |   1 -
 tensorflow/contrib/tensor_forest/BUILD        |   1 +
 tensorflow/contrib/tensorboard/BUILD          |   1 -
 tensorflow/contrib/timeseries/examples/BUILD  |   5 +-
 .../timeseries/python/timeseries/BUILD        |   5 +-
 .../timeseries/state_space_models/BUILD       |   1 -
 tensorflow/contrib/tpu/BUILD                  |   1 -
 tensorflow/contrib/util/loader.py             |   7 +-
 tensorflow/core/framework/dataset.h           |   4 +-
 tensorflow/core/lib/core/stringpiece.cc       |   2 +
 tensorflow/core/lib/core/stringpiece.h        |   2 +-
 tensorflow/core/platform/tracing.h            |   2 +-
 tensorflow/python/BUILD                       |  92 ++--------
 tensorflow/python/debug/BUILD                 |   1 -
 tensorflow/python/keras/BUILD                 |   5 +-
 tensorflow/python/kernel_tests/BUILD          |   4 +
 tensorflow/tensorflow.bzl                     |  20 +--
 .../windows/cpu/pip/build_tf_windows.sh       |   3 +-
 tensorflow/tools/def_file_filter/BUILD        |   0
 tensorflow/tools/def_file_filter/BUILD.tpl    |  15 --
 .../def_file_filter/def_file_filter.py.tpl    | 168 ------------------
 .../def_file_filter_configure.bzl             |  56 ------
 tensorflow/tools/pip_package/BUILD            | 128 +++++++------
 tensorflow/workspace.bzl                      |   8 +-
 42 files changed, 125 insertions(+), 454 deletions(-)
 delete mode 100644 tensorflow/tools/def_file_filter/BUILD
 delete mode 100644 tensorflow/tools/def_file_filter/BUILD.tpl
 delete mode 100644 tensorflow/tools/def_file_filter/def_file_filter.py.tpl
 delete mode 100644 tensorflow/tools/def_file_filter/def_file_filter_configure.bzl

diff --git a/configure.py b/configure.py
index d14edef1be..7d61c2e5e3 100644
--- a/configure.py
+++ b/configure.py
@@ -1380,7 +1380,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.10.0')
+  check_bazel_version('0.5.4')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index c2663c5e83..d103da79e3 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -8,7 +8,6 @@ package(default_visibility = ["//tensorflow:__subpackages__"])
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
-load("//tensorflow:tensorflow.bzl", "if_not_windows")
 
 py_library(
     name = "contrib_py",
@@ -40,6 +39,7 @@ py_library(
         "//tensorflow/contrib/estimator:estimator_py",
         "//tensorflow/contrib/factorization:factorization_py",
         "//tensorflow/contrib/feature_column:feature_column_py",
+        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/fused_conv:fused_conv_py",
         "//tensorflow/contrib/gan",
@@ -51,6 +51,7 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
+        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -62,6 +63,7 @@ py_library(
         "//tensorflow/contrib/linalg:linalg_py",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
+        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
@@ -108,10 +110,6 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]) + if_not_windows([
-        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",  # unix dependency, need to fix code
-        "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
-        "//tensorflow/contrib/kafka",  # has some linking issue on opensssl.
     ]),
 )
 
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 669d611b01..4f6f539027 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -18,8 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
-
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
@@ -85,8 +83,7 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-if os.name != 'nt':
-  from tensorflow.contrib.lite.python import lite
+from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
@@ -95,7 +92,6 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
-del os
 del LazyLoader
 
 del absolute_import
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
index 7815fa049a..da5e744851 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
@@ -48,9 +48,9 @@ class BatchFeatures {
   Status GetFeatureColumnSizes(int64* const num_dense_float_features,
                                int64* const num_sparse_float_features,
                                int64* const num_sparse_int_features) const {
-    QCHECK_NE(num_dense_float_features, (int64*) nullptr);
-    QCHECK_NE(num_sparse_float_features, (int64*) nullptr);
-    QCHECK_NE(num_sparse_int_features, (int64*) nullptr);
+    QCHECK_NE(num_dense_float_features, nullptr);
+    QCHECK_NE(num_sparse_float_features, nullptr);
+    QCHECK_NE(num_sparse_int_features, nullptr);
     *num_dense_float_features = dense_float_feature_columns_.size();
     *num_sparse_float_features = sparse_float_feature_columns_.size();
     *num_sparse_int_features = sparse_int_feature_columns_.size();
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 80dd1ccd04..4ddec73ec8 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -454,7 +454,6 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
@@ -1121,7 +1120,6 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 1c5ebbc6ca..4fba014d6f 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -270,10 +270,7 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",
-    ],
+    tags = ["notsan"],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index 2f6cfdf31e..f86331af6f 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -22,7 +22,6 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index ff6f3b7441..0eb0e3cbe2 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -354,7 +354,6 @@ py_test(
     name = "classifier_metrics_test",
     srcs = ["python/eval/python/classifier_metrics_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":classifier_metrics",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index d1c449402a..146ae8b7e2 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -114,7 +114,6 @@ py_test(
     name = "utils_test",
     srcs = ["utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         "//tensorflow/contrib/kfac/python/ops:utils",
         "//tensorflow/contrib/tpu",
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 544065dac6..894e6f6946 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -70,7 +70,6 @@ py_test(
         "python/ops/core_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":_typecheck",
         ":core",
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index cc7bbabf21..852d06e1e3 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -188,7 +188,6 @@ py_test(
     size = "small",
     srcs = ["python/layers/normalization_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/contrib/framework:framework_py",
@@ -354,7 +353,6 @@ py_test(
     size = "small",
     srcs = ["python/ops/sparse_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index b05f5eeaee..9c59150580 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -117,7 +117,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/data_feeder_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/python:client_testlib",
@@ -173,7 +172,6 @@ tf_py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
@@ -192,7 +190,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/graph_actions_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
@@ -593,7 +590,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/io_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/learn/python/learn/datasets",
@@ -823,7 +819,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/utils/saved_model_export_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD
index 0a6edc33c5..8ca03f4193 100644
--- a/tensorflow/contrib/lookup/BUILD
+++ b/tensorflow/contrib/lookup/BUILD
@@ -46,7 +46,6 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     grpc_enabled = True,
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 filegroup(
diff --git a/tensorflow/contrib/py2tf/converters/BUILD b/tensorflow/contrib/py2tf/converters/BUILD
index 4bb6f76019..f624c42686 100644
--- a/tensorflow/contrib/py2tf/converters/BUILD
+++ b/tensorflow/contrib/py2tf/converters/BUILD
@@ -81,7 +81,6 @@ py_test(
     name = "builtin_functions_test",
     srcs = ["builtin_functions_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/python:client_testlib",
@@ -92,7 +91,6 @@ py_test(
     name = "call_trees_test",
     srcs = ["call_trees_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":test_lib",
         "//tensorflow/contrib/py2tf/impl",
diff --git a/tensorflow/contrib/py2tf/utils/BUILD b/tensorflow/contrib/py2tf/utils/BUILD
index 8bc338e801..d029289f5a 100644
--- a/tensorflow/contrib/py2tf/utils/BUILD
+++ b/tensorflow/contrib/py2tf/utils/BUILD
@@ -83,7 +83,6 @@ py_test(
     name = "py_func_test",
     srcs = ["py_func_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":utils",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
index 54c66271cd..27f0a7f58f 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD
+++ b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
@@ -38,6 +38,7 @@ py_test(
     size = "small",
     srcs = ["python/ops/remote_fused_graph_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":remote_fused_graph_ops_py",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index b10757df47..245fe07f2b 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -53,7 +53,6 @@ py_test(
     size = "small",
     srcs = ["python/saved_model/reader_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":saved_model_py",
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 6b5d8b323d..75a753ed89 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -163,7 +163,6 @@ py_test(
     name = "gc_test",
     srcs = ["gc_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":gc",
diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD
index 7aa1684839..5daabbd62e 100644
--- a/tensorflow/contrib/slim/python/slim/data/BUILD
+++ b/tensorflow/contrib/slim/python/slim/data/BUILD
@@ -61,7 +61,6 @@ py_test(
     name = "dataset_data_provider_test",
     srcs = ["dataset_data_provider_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":dataset",
         ":dataset_data_provider",
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 07b6b1f142..1e4cc3f095 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -553,6 +553,7 @@ py_test(
     srcs = ["client/random_forest_test.py"],
     srcs_version = "PY2AND3",
     tags = [
+        "no_windows",
         "nomac",  # b/63258195
         "notsan",
     ],
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index db2e000ef8..d833744d0c 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -9,7 +9,6 @@ exports_files(["LICENSE"])
 
 # For platform specific build config
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
-load("//tensorflow:tensorflow.bzl", "py_test")
 
 tf_proto_library(
     name = "protos_all",
diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 70bf67c779..bb86ecb220 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -25,10 +25,7 @@ py_test(
     srcs = ["predict_test.py"],
     data = ["data/period_trend.csv"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",  # b/67513579
-    ],
+    tags = ["notsan"],  # b/67513579
     deps = [
         ":predict",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 64f5cd8357..ed3ed4c0e1 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -156,7 +156,9 @@ py_test(
         "head_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = ["no_pip_gpu"],  # b/63391119
+    tags = [
+        "no_pip_gpu",  # b/63391119
+    ],
     deps = [
         ":feature_keys",
         ":head",
@@ -425,7 +427,6 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip_gpu",  # b/63391119
-        "no_windows",  # TODO: needs investigation on Windows
     ],
     deps = [
         ":feature_keys",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index 07df7bc9a5..c86d06e923 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -40,7 +40,6 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["state_space_model_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":state_space_model",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 371d1b6672..eea19e9465 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -225,7 +225,6 @@ tf_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:layers",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 tf_py_test(
diff --git a/tensorflow/contrib/util/loader.py b/tensorflow/contrib/util/loader.py
index dca01d26f4..f4283cd9ed 100644
--- a/tensorflow/contrib/util/loader.py
+++ b/tensorflow/contrib/util/loader.py
@@ -42,10 +42,9 @@ def load_op_library(path):
     plugin.
   """
   if os.name == 'nt':
-    # To avoid making every user_ops aware of windows, re-write
-    # the file extension from .so to .dll if .so file doesn't exist.
-    if not os.path.exists(path):
-      path = re.sub(r'\.so$', '.dll', path)
+    # To avoid makeing every user_ops aware of windows, re-write
+    # the file extension from .so to .dll.
+    path = re.sub(r'\.so$', '.dll', path)
 
     # Currently we have only some user_ops as dlls on windows - don't try
     # to load them if the dll is not found.
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index cfe23d1ffe..beaf0adbc5 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -474,11 +474,11 @@ class GraphDatasetBase : public DatasetBase {
   }
 
   // Key for storing the Dataset graph in the serialized format.
-  TF_EXPORT static const char kDatasetGraphKey[];
+  static const char kDatasetGraphKey[];
 
   // Key for storing the output node of the Dataset graph in the serialized
   // format.
-  TF_EXPORT static const char kDatasetGraphOutputNodeKey[];
+  static const char kDatasetGraphOutputNodeKey[];
 
  private:
   Status Serialize(OpKernelContext* ctx, string* serialized_graph_def,
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 0b006fa2b4..5bd79778a6 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -55,4 +55,6 @@ StringPiece StringPiece::substr(size_t pos, size_t n) const {
   return StringPiece(data_ + pos, n);
 }
 
+const StringPiece::size_type StringPiece::npos = size_type(-1);
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 835b938cbf..79409cce4b 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -65,7 +65,7 @@ class StringPiece {
   iterator begin() const { return data_; }
   iterator end() const { return data_ + size_; }
 
-  static const size_t npos = size_type(-1);
+  static const size_t npos;
 
   // Return the ith byte in the referenced data.
   // REQUIRES: n < size()
diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h
index eebbeaeba6..8f7bff1bb0 100644
--- a/tensorflow/core/platform/tracing.h
+++ b/tensorflow/core/platform/tracing.h
@@ -103,7 +103,7 @@ class Tracing {
   friend class ScopedAnnotation;
   friend class TraceMe;
 
-  TF_EXPORT static std::atomic<Tracing::Engine*> tracing_engine_;
+  static std::atomic<Tracing::Engine*> tracing_engine_;
   static Tracing::Engine* engine() {
     return tracing_engine_.load(std::memory_order_acquire);
   }
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 86548f4346..3a8ba2db04 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -28,7 +28,6 @@ load("//tensorflow:tensorflow.bzl", "py_tests")
 load("//tensorflow:tensorflow.bzl", "tf_py_build_info_genrule")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_library_additional_deps_impl")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
 load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
@@ -87,7 +86,6 @@ py_library(
         ":ops",
         ":platform",
         ":pywrap_tensorflow",
-        ":saver_test_utils",
         ":script_ops",
         ":session_ops",
         ":sets",
@@ -97,29 +95,31 @@ py_library(
         ":standard_ops",
         ":state_ops",
         ":string_ops",
-        ":subscribe",
         ":summary",
         ":tensor_array_ops",
+        ":training",
+        ":saver_test_utils",
+        ":subscribe",
         ":test_ops",  # TODO: Break testing code out into separate rule.
-        ":tf_cluster",
         ":tf_item",
+        ":tf_cluster",
         ":tf_optimizer",
-        ":training",
         ":util",
         ":weights_broadcast_ops",
-        "//tensorflow/contrib:contrib_py",
+        "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras",
+        "//tensorflow/python/ops/losses",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
-        "//tensorflow/python/ops/losses",
         "//tensorflow/python/profiler",
         "//tensorflow/python/saved_model",
-        "//third_party/py/numpy",
-    ],
+    ] + if_not_windows([
+        "//tensorflow/contrib:contrib_py",
+    ]),
 )
 
 tf_py_build_info_genrule()
@@ -947,6 +947,7 @@ py_test(
     srcs = ["framework/contrib_test.py"],
     main = "framework/contrib_test.py",
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
@@ -1313,6 +1314,7 @@ py_test(
     srcs = ["framework/dtypes_test.py"],
     main = "framework/dtypes_test.py",
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":framework_for_generated_wrappers",
         ":framework_test_lib",
@@ -1660,6 +1662,7 @@ py_test(
     size = "small",
     srcs = ["ops/clip_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":clip_ops",
@@ -2728,6 +2731,7 @@ cuda_py_test(
     ],
     data = ["//tensorflow/core:image_testdata"],
     shard_count = 5,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -3264,10 +3268,6 @@ tf_py_wrap_cc(
         "util/transform_graph.i",
         "util/util.i",
     ],
-    win_def_file = select({
-        "//tensorflow:windows": ":pywrap_tensorflow_filtered_def_file",
-        "//conditions:default": None,
-    }),
     deps = [
         ":bfloat16_lib",
         ":cost_analyzer_lib",
@@ -3311,65 +3311,6 @@ tf_py_wrap_cc(
          tf_additional_gdr_deps()),
 )
 
-# ** Targets for Windows build (start) **
-# We need the following targets to expose symbols from _pywrap_tensorflow.dll
-
-# Build a cc_binary from tf_custom_op_library_additional_deps_impl,
-# it contains all object code from its dependencies.
-cc_binary(
-    name = "tf_custom_op_library_additional_deps.so",
-    linkshared = 1,
-    linkstatic = 1,
-    deps = tf_custom_op_library_additional_deps_impl(),
-)
-
-# Get a DEF file generated by parsing all object files
-# of tf_custom_op_library_additional_deps.so
-filegroup(
-    name = "pywrap_tensorflow_def_file",
-    srcs = [":tf_custom_op_library_additional_deps.so"],
-    output_group = "def_file",
-)
-
-# Filter the DEF file to reduce the number of symbols to 64K or less.
-# Note that we also write the name of the pyd file into DEF file so that
-# the dynamic libraries of custom ops can find it at runtime.
-genrule(
-    name = "pywrap_tensorflow_filtered_def_file",
-    srcs = [":pywrap_tensorflow_def_file"],
-    outs = ["pywrap_tensorflow_filtered_def_file.def"],
-    cmd = select({
-        "//tensorflow:windows": """
-              $(location @local_config_def_file_filter//:def_file_filter) \\
-              --input $(location :pywrap_tensorflow_def_file) \\
-              --output $@ \\
-              --target _pywrap_tensorflow_internal.pyd
-          """,
-        "//conditions:default": "touch $@",  # Just a placeholder for Unix platforms
-    }),
-    tools = ["@local_config_def_file_filter//:def_file_filter"],
-)
-
-# Get the import library of  _pywrap_tensorflow_internal.dll
-filegroup(
-    name = "pywrap_tensorflow_import_lib_file",
-    srcs = [":_pywrap_tensorflow_internal.so"],
-    output_group = "interface_library",
-)
-
-# Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll
-# so that custom ops' dynamic libraries can link against it.
-cc_import(
-    name = "pywrap_tensorflow_import_lib",
-    interface_library = select({
-        "//tensorflow:windows": ":pywrap_tensorflow_import_lib_file",
-        "//conditions:default": "not_exsiting_on_unix.lib",  # Just a placeholder for Unix platforms
-    }),
-    system_provided = 1,
-)
-
-# ** Targets for Windows build (end) **
-
 py_library(
     name = "lib",
     srcs = [
@@ -3742,6 +3683,7 @@ py_test(
     size = "small",
     srcs = ["lib/core/bfloat16_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":lib",
@@ -4049,6 +3991,7 @@ py_test(
     size = "small",
     srcs = ["training/checkpoint_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":checkpoint_ops_gen",
         ":client",
@@ -4089,7 +4032,10 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67945581
+    tags = [
+        "no_windows",
+        "notsan",  # b/67945581
+    ],
     deps = [
         ":array_ops",
         ":client_testlib",
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index c60f692390..512d292ee2 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -913,7 +913,6 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
-    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index c9b68594cf..3180b9f410 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -637,10 +637,7 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/utils/io_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",
-    ],
+    tags = ["notsan"],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 3499086d0a..d9571fa2be 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -295,6 +295,7 @@ tf_py_test(
         "//tensorflow/python:nn_grad",
     ],
     data = ["//tensorflow/core:image_testdata"],
+    tags = ["no_windows"],
 )
 
 tf_py_test(
@@ -1140,6 +1141,7 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     data = ["//tensorflow/core:lmdb_testdata"],
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2329,6 +2331,7 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 4,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2459,6 +2462,7 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index c8d175dcf2..9b0db8a112 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -1176,22 +1176,6 @@ def tf_custom_op_library_additional_deps():
       "@protobuf_archive//:protobuf_headers",
       clean_dep("//third_party/eigen3"),
       clean_dep("//tensorflow/core:framework_headers_lib"),
-  ] + if_windows(["//tensorflow/python:pywrap_tensorflow_import_lib"])
-
-# A list of targets that contains the implemenation of
-# tf_custom_op_library_additional_deps. It's used to generate a DEF file for
-# exporting symbols from _pywrap_tensorflow.dll on Windows.
-def tf_custom_op_library_additional_deps_impl():
-  return [
-      # for @protobuf_archive//:protobuf_headers
-      "@protobuf_archive//:protobuf",
-      # for @nsync//:nsync_headers
-      "@nsync//:nsync_cpp",
-      # for //third_party/eigen3
-      clean_dep("//third_party/eigen3"),
-      # for //tensorflow/core:framework_headers_lib
-      clean_dep("//tensorflow/core:framework"),
-      clean_dep("//tensorflow/core:reader_base"),
   ]
 
 # Traverse the dependency graph along the "deps" attribute of the
@@ -1278,7 +1262,6 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]):
       deps=deps + if_cuda(cuda_deps),
       data=[name + "_check_deps"],
       copts=tf_copts(is_external=True),
-      features = ["windows_export_all_symbols"],
       linkopts=linkopts + select({
           "//conditions:default": [
               "-lm",
@@ -1425,8 +1408,7 @@ def tf_py_wrap_cc(name,
       ]) + tf_extension_copts()),
       linkopts=tf_extension_linkopts() + extra_linkopts,
       linkstatic=1,
-      deps=deps + extra_deps,
-      **kwargs)
+      deps=deps + extra_deps)
   native.genrule(
       name="gen_" + cc_library_pyd_name,
       srcs=[":" + cc_library_name],
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 40189a6d1b..8b8ba31a0d 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -65,5 +65,4 @@ bazel test -c opt $BUILD_OPTS -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
-  //${PY_TEST_DIR}/tensorflow/python/... \
-  //${PY_TEST_DIR}/tensorflow/contrib/...
+  //${PY_TEST_DIR}/tensorflow/python/...
diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD
deleted file mode 100644
index e69de29bb2..0000000000
diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl
deleted file mode 100644
index 3cb72f4979..0000000000
--- a/tensorflow/tools/def_file_filter/BUILD.tpl
+++ /dev/null
@@ -1,15 +0,0 @@
-# Description:
-# Tools for filtering DEF file for TensorFlow on Windows
-#
-# On Windows, we use a DEF file generated by Bazel to export
-# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll).
-# The maximum number of symbols that can be exported per DLL is 64K,
-# so we have to filter some useless symbols through this python script.
-
-package(default_visibility = ["//visibility:public"])
-
-py_binary(
-    name = "def_file_filter",
-    srcs = ["def_file_filter.py"],
-    srcs_version = "PY2AND3",
-)
diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
deleted file mode 100644
index 8bdc03eb0f..0000000000
--- a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
+++ /dev/null
@@ -1,168 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-"""def_file_filter.py - tool to filter a windows def file.
-
-The def file can be used to export symbols from the tensorflow dll to enable
-tf.load_library().
-
-Because the linker allows only 64K symbols to be exported per dll
-we filter the symbols down to the essentials. The regular expressions
-we use for this are specific to tensorflow.
-
-TODO: this works fine but there is an issue with exporting
-'const char * const' and importing it from a user_ops. The problem is
-on the importing end and using __declspec(dllimport) works around it.
-"""
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import argparse
-import io
-import os
-import re
-import subprocess
-import sys
-import tempfile
-
-# External tools we use that come with visual studio sdk
-UNDNAME = "%{undname_bin_path}"
-
-# Exclude if matched
-EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::")
-
-# Include if matched before exclude
-INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|"
-                           r"google::protobuf::internal::ArenaImpl::AllocateAligned|" # for contrib/data/_prefetching_ops
-                           r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops
-                           r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops
-                           r"tensorflow::internal::LogMessage|"
-                           r"tensorflow::internal::LogString|"
-                           r"tensorflow::internal::CheckOpMessageBuilder|"
-                           r"tensorflow::internal::MakeCheckOpValueString|"
-                           r"tensorflow::internal::PickUnusedPortOrDie|"
-                           r"tensorflow::internal::ValidateDevice|"
-                           r"tensorflow::ops::internal::Enter|"
-                           r"tensorflow::strings::internal::AppendPieces|"
-                           r"tensorflow::strings::internal::CatPieces|"
-                           r"tensorflow::io::internal::JoinPathImpl")
-
-# Include if matched after exclude
-INCLUDE_RE = re.compile(r"^(TF_\w*)$|"
-                        r"^(TFE_\w*)$|"
-                        r"nsync::|"
-                        r"tensorflow::|"
-                        r"functor::|"
-                        r"perftools::gputools")
-
-# We want to identify data members explicitly in the DEF file, so that no one
-# can implicitly link against the DLL if they use one of the variables exported
-# from the DLL and the header they use does not decorate the symbol with
-# __declspec(dllimport). It is easier to detect what a data symbol does
-# NOT look like, so doing it with the below regex.
-DATA_EXCLUDE_RE = re.compile(r"[)(]|"
-                             r"vftable|"
-                             r"vbtable|"
-                             r"vcall|"
-                             r"RTTI|"
-                             r"protobuf::internal::ExplicitlyConstructed")
-
-def get_args():
-  """Parse command line."""
-  filename_list = lambda x: x.split(";")
-  parser = argparse.ArgumentParser()
-  parser.add_argument("--input", type=filename_list,
-                      help="paths to input def file",
-                      required=True)
-  parser.add_argument("--output", help="output deffile", required=True)
-  parser.add_argument("--target", help="name of the target", required=True)
-  args = parser.parse_args()
-  return args
-
-
-def main():
-  """main."""
-  args = get_args()
-
-  # Pipe dumpbin to extract all linkable symbols from libs.
-  # Good symbols are collected in candidates and also written to
-  # a temp file.
-  candidates = []
-  tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False)
-  for def_file_path in args.input:
-    def_file = open(def_file_path, 'r')
-    for line in def_file:
-      cols = line.split()
-      sym = cols[0]
-      tmpfile.file.write(sym + "\n")
-      candidates.append(sym)
-  tmpfile.file.close()
-
-  # Run the symbols through undname to get their undecorated name
-  # so we can filter on something readable.
-  with open(args.output, "w") as def_fp:
-    # track dupes
-    taken = set()
-
-    # Header for the def file.
-    def_fp.write("LIBRARY " + args.target + "\n")
-    def_fp.write("EXPORTS\n")
-    def_fp.write("\t ??1OpDef@tensorflow@@UEAA@XZ\n")
-
-    # Each symbols returned by undname matches the same position in candidates.
-    # We compare on undname but use the decorated name from candidates.
-    dupes = 0
-    proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE)
-    for idx, line in enumerate(io.TextIOWrapper(proc.stdout, encoding="utf-8")):
-      decorated = candidates[idx]
-      if decorated in taken:
-        # Symbol is already in output, done.
-        dupes += 1
-        continue
-
-      if not INCLUDEPRE_RE.search(line):
-        if EXCLUDE_RE.search(line):
-          continue
-        if not INCLUDE_RE.search(line):
-          continue
-
-      if "deleting destructor" in line:
-        # Some of the symbols convered by INCLUDEPRE_RE export deleting
-        # destructor symbols, which is a bad idea.
-        # So we filter out such symbols here.
-        continue
-
-      if DATA_EXCLUDE_RE.search(line):
-        def_fp.write("\t" + decorated + "\n")
-      else:
-        def_fp.write("\t" + decorated + " DATA\n")
-      taken.add(decorated)
-    def_fp.close()
-
-  exit_code = proc.wait()
-  if exit_code != 0:
-    print("{} failed, exit={}".format(UNDNAME, exit_code))
-    return exit_code
-
-  os.unlink(tmpfile.name)
-
-  print("symbols={}, taken={}, dupes={}"
-        .format(len(candidates), len(taken), dupes))
-  return 0
-
-
-if __name__ == "__main__":
-  sys.exit(main())
diff --git a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
deleted file mode 100644
index 47539b2423..0000000000
--- a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
+++ /dev/null
@@ -1,56 +0,0 @@
-"""Repository rule for def file filter autoconfiguration.
-
-This repository reuses Bazel's VC detect mechanism to find undname.exe,
-which is a tool used in def_file_filter.py.
-
-def_file_filter.py is for filtering the DEF file for TensorFlow on Windows.
-On Windows, we use a DEF file generated by Bazel to export symbols from the
-tensorflow dynamic library(_pywrap_tensorflow.dll). The maximum number of
-symbols that can be exported per DLL is 64K, so we have to filter some useless
-symbols through this python script.
-
-`def_file_filter_config` depends on the following environment variables:
-  * `BAZEL_VC`
-  * `BAZEL_VS`
-  * `VS90COMNTOOLS`
-  * `VS100COMNTOOLS`
-  * `VS110COMNTOOLS`
-  * `VS120COMNTOOLS`
-  * `VS140COMNTOOLS`
-"""
-
-load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_vc_path")
-load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_msvc_tool")
-load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "auto_configure_fail")
-
-def _def_file_filter_configure_impl(repository_ctx):
-  if repository_ctx.os.name.lower().find("windows") == -1:
-    repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
-    repository_ctx.file("def_file_filter.py", "")
-    return
-  vc_path = find_vc_path(repository_ctx)
-  if vc_path == "visual-studio-not-found":
-    auto_configure_fail("Visual C++ build tools not found on your machine")
-  undname_bin_path = find_msvc_tool(repository_ctx, vc_path, "undname.exe").replace("\\", "\\\\")
-
-  repository_ctx.template(
-    "def_file_filter.py",
-    Label("//tensorflow/tools/def_file_filter:def_file_filter.py.tpl"),
-    {
-      "%{undname_bin_path}": undname_bin_path,
-    })
-  repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
-
-
-def_file_filter_configure = repository_rule(
-    implementation = _def_file_filter_configure_impl,
-    environ = [
-        "BAZEL_VC",
-        "BAZEL_VS",
-        "VS90COMNTOOLS",
-        "VS100COMNTOOLS",
-        "VS110COMNTOOLS",
-        "VS120COMNTOOLS",
-        "VS140COMNTOOLS"
-    ],
-)
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 2607b9d704..d55a883df5 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -48,65 +48,36 @@ py_binary(
     deps = ["//tensorflow:tensorflow_py"],
 )
 
-COMMON_PIP_DEPS = [
-    ":licenses",
-    "MANIFEST.in",
-    "README",
-    "setup.py",
-    ":included_headers",
-    "//tensorflow:tensorflow_py",
-    "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
-    "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
-    "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
-    "//tensorflow/contrib/data/python/ops:contrib_op_loader",
-    "//tensorflow/contrib/eager/python/examples:examples_pip",
-    "//tensorflow/contrib/eager/python:checkpointable_utils",
-    "//tensorflow/contrib/eager/python:evaluator",
-    "//tensorflow/contrib/gan:gan",
-    "//tensorflow/contrib/graph_editor:graph_editor_pip",
-    "//tensorflow/contrib/keras:keras",
-    "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
-    "//tensorflow/contrib/nn:nn_py",
-    "//tensorflow/contrib/predictor:predictor_pip",
-    "//tensorflow/contrib/py2tf:py2tf",
-    "//tensorflow/contrib/py2tf/converters:converters",
-    "//tensorflow/contrib/py2tf/converters:test_lib",
-    "//tensorflow/contrib/py2tf/impl:impl",
-    "//tensorflow/contrib/py2tf/pyct:pyct",
-    "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
-    "//tensorflow/contrib/receptive_field:receptive_field_pip",
-    "//tensorflow/contrib/session_bundle:session_bundle_pip",
-    "//tensorflow/contrib/signal:signal_py",
-    "//tensorflow/contrib/signal:test_util",
-    "//tensorflow/contrib/slim:slim",
-    "//tensorflow/contrib/slim/python/slim/data:data_pip",
-    "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-    "//tensorflow/contrib/specs:specs",
-    "//tensorflow/contrib/summary:summary_test_util",
-    "//tensorflow/contrib/tensor_forest:init_py",
-    "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-    "//tensorflow/contrib/timeseries:timeseries_pip",
-    "//tensorflow/contrib/tpu",
-    "//tensorflow/examples/tutorials/mnist:package",
-    "//tensorflow/python:distributed_framework_test_lib",
-    "//tensorflow/python:meta_graph_testdata",
-    "//tensorflow/python:spectral_ops_test_util",
-    "//tensorflow/python:util_example_parser_configuration",
-    "//tensorflow/python/debug:debug_pip",
-    "//tensorflow/python/eager:eager_pip",
-    "//tensorflow/python/saved_model:saved_model",
-    "//tensorflow/python/tools:tools_pip",
-    "//tensorflow/python:test_ops",
-    "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
-]
-
 # On Windows, python binary is a zip file of runfiles tree.
 # Add everything to its data dependency for generating a runfiles tree
 # for building the pip package on Windows.
 py_binary(
     name = "simple_console_for_windows",
     srcs = ["simple_console_for_windows.py"],
-    data = COMMON_PIP_DEPS,
+    data = [
+        "MANIFEST.in",
+        "README",
+        "setup.py",
+        ":included_headers",
+        "//tensorflow/contrib/nn:nn_py",
+        "//tensorflow/contrib/session_bundle:session_bundle_pip",
+        "//tensorflow/contrib/signal:signal_py",
+        "//tensorflow/contrib/slim/python/slim/data:data_pip",
+        "//tensorflow/python:util_example_parser_configuration",
+        "//tensorflow/python/debug:debug_pip",
+        "//tensorflow/python/saved_model",
+        "//tensorflow/python:spectral_ops_test_util",
+        "//tensorflow/python/tools:tools_pip",
+        "//tensorflow/python/eager:eager_pip",
+        "//tensorflow/contrib/summary:summary_test_util",
+        # These targets don't build on Windows yet. Exclude them for now.
+        # "//tensorflow/contrib/slim",
+        # "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+        # "//tensorflow/contrib/specs",
+        # "//tensorflow/contrib/tensor_forest:init_py",
+        # "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+        # "//tensorflow/examples/tutorials/mnist:package",
+    ],
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
@@ -167,12 +138,61 @@ sh_binary(
     data = select({
         "//tensorflow:windows": [":simple_console_for_windows"],
         "//tensorflow:windows_msvc": [":simple_console_for_windows"],
-        "//conditions:default": COMMON_PIP_DEPS + [
+        "//conditions:default": [
+            ":licenses",
+            "MANIFEST.in",
+            "README",
+            "setup.py",
+            ":included_headers",
             ":simple_console",
+            "//tensorflow:tensorflow_py",
+            "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
+            "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
+            "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
+            "//tensorflow/contrib/data/python/ops:contrib_op_loader",
+            "//tensorflow/contrib/eager/python/examples:examples_pip",
+            "//tensorflow/contrib/eager/python:checkpointable_utils",
+            "//tensorflow/contrib/eager/python:evaluator",
+            "//tensorflow/contrib/gan:gan",
+            "//tensorflow/contrib/graph_editor:graph_editor_pip",
+            "//tensorflow/contrib/keras:keras",
+            "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
             "//tensorflow/contrib/lite/python:interpreter_test_data",
             "//tensorflow/contrib/lite/toco:toco",
             "//tensorflow/contrib/lite/toco/python:toco_wrapper",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
+            "//tensorflow/contrib/nn:nn_py",
+            "//tensorflow/contrib/predictor:predictor_pip",
+            "//tensorflow/contrib/py2tf:py2tf",
+            "//tensorflow/contrib/py2tf/converters:converters",
+            "//tensorflow/contrib/py2tf/converters:test_lib",
+            "//tensorflow/contrib/py2tf/impl:impl",
+            "//tensorflow/contrib/py2tf/pyct:pyct",
+            "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
+            "//tensorflow/contrib/receptive_field:receptive_field_pip",
+            "//tensorflow/contrib/session_bundle:session_bundle_pip",
+            "//tensorflow/contrib/signal:signal_py",
+            "//tensorflow/contrib/signal:test_util",
+            "//tensorflow/contrib/slim:slim",
+            "//tensorflow/contrib/slim/python/slim/data:data_pip",
+            "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+            "//tensorflow/contrib/specs:specs",
+            "//tensorflow/contrib/summary:summary_test_util",
+            "//tensorflow/contrib/tensor_forest:init_py",
+            "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+            "//tensorflow/contrib/timeseries:timeseries_pip",
+            "//tensorflow/contrib/tpu",
+            "//tensorflow/examples/tutorials/mnist:package",
+            "//tensorflow/python:distributed_framework_test_lib",
+            "//tensorflow/python:meta_graph_testdata",
+            "//tensorflow/python:spectral_ops_test_util",
+            "//tensorflow/python:util_example_parser_configuration",
+            "//tensorflow/python/debug:debug_pip",
+            "//tensorflow/python/eager:eager_pip",
+            "//tensorflow/python/saved_model:saved_model",
+            "//tensorflow/python/tools:tools_pip",
+            "//tensorflow/python:test_ops",
+            "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
         ],
     }) + if_mkl(["//third_party/mkl:intel_binary_blob"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0e910b774d..675acbe5f6 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -12,8 +12,6 @@ load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compil
 load("//third_party:repo.bzl", "tf_http_archive")
 load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
-load("//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
-     "def_file_filter_configure")
 
 def _extract_version_number(bazel_version):
   """Extracts the semantic version number from a version string
@@ -69,7 +67,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # We must check the bazel version before trying to parse any other BUILD
   # files, in case the parsing of those build files depends on the bazel
   # version we require here.
-  check_bazel_version_at_least("0.10.0")
+  check_bazel_version_at_least("0.5.4")
   clang6_configure(name="local_config_clang6")
   cuda_configure(name="local_config_cuda")
   tensorrt_configure(name="local_config_tensorrt")
@@ -77,10 +75,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   sycl_configure(name="local_config_sycl")
   python_configure(name="local_config_python")
 
-  # For windows bazel build
-  # TODO: Remove def file filter when TensorFlow can export symbols properly on Windows.
-  def_file_filter_configure(name = "local_config_def_file_filter")
-
   # Point //external/local_config_arm_compiler to //external/arm_compiler
   arm_compiler_configure(
       name="local_config_arm_compiler",
-- 
GitLab


From bf741007d1f6f440a2671b9fa8894af3df10ed44 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 20 Mar 2018 21:30:02 -0700
Subject: [PATCH 283/960] C API: fix device + colocation edge case in
 import_graph_def

This change makes the C API consistent with the Python API, by making
sure that all nodes in a colocation group have the device of the op
named in the "_class" attr (all other ops' devices are ignored). This
is currently done by preserving the current Python logic for
colocation and devices, which only works if all ops start with no
device set. Without this change, imported nodes would have the device
specified in the GraphDef. This change unsets any device before
running the Python logic.

PiperOrigin-RevId: 189859688
---
 tensorflow/python/framework/importer.py      | 11 +++--
 tensorflow/python/framework/importer_test.py | 43 ++++++++++++++++++++
 2 files changed, 50 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index a9e399f59b..4ea34d7bb2 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -301,14 +301,17 @@ def _ProcessNewOps(graph):
   colocation_pairs = {}
 
   for new_op in graph._add_new_tf_operations(compute_devices=False):  # pylint: disable=protected-access
+    original_device = new_op.device
+    new_op._set_device('')  # pylint: disable=protected-access
     colocation_names = _GetColocationNames(new_op)
     if colocation_names:
       colocation_pairs[new_op] = colocation_names
-      # Don't apply this op's device function, since colocation constraints
-      # override device functions. Note that this op's device may still be set
-      # by the loop below.
+      # Don't set a device for this op, since colocation constraints override
+      # device functions and the original device. Note that this op's device may
+      # still be set by the loop below.
+      # TODO(skyewm): why does it override the original device?
     else:
-      with _MaybeDevice(new_op.device):
+      with _MaybeDevice(original_device):
         graph._apply_device_functions(new_op)  # pylint: disable=protected-access
 
   # The following loop populates the device field of ops that are colocated
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index bf5d9fe093..6593b17184 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -680,6 +680,49 @@ class ImportGraphDefTest(test.TestCase):
           "list { s: 'loc:@imported_graph/A' }",
           b.node_def.attr["_class"])
 
+  def testColocationAndDevice(self):
+    # A and B are colocated, device set on A.
+    original_graph_def = self._MakeGraphDef("""
+          node { name: 'A' op: 'None' device: '/device:CPU:0' attr {
+            key: '_class'
+            value { list { s: 'loc:@A' } }
+          } }
+          node { name: 'B' op: 'None'  attr {
+            key: '_class'
+            value { list { s: 'loc:@A' } }
+          } }""")
+
+    with ops.Graph().as_default():
+      a, b = importer.import_graph_def(original_graph_def,
+                                       return_elements=["A", "B"],
+                                       name="")
+      self.assertEqual(a.device, "/device:CPU:0")
+      self.assertEqual(b.device, "/device:CPU:0")
+      self.assertEqual(a.colocation_groups(), [b"loc:@A"])
+      self.assertEqual(b.colocation_groups(), [b"loc:@A"])
+
+    # A and B are colocated, device set on B.
+    original_graph_def = self._MakeGraphDef("""
+          node { name: 'A' op: 'None' attr {
+            key: '_class'
+            value { list { s: 'loc:@A' } }
+          } }
+          node { name: 'B' op: 'None' device: '/device:CPU:0' attr {
+            key: '_class'
+            value { list { s: 'loc:@A' } }
+          } }""")
+
+    with ops.Graph().as_default():
+      a, b = importer.import_graph_def(original_graph_def,
+                                       return_elements=["A", "B"],
+                                       name="")
+      # TODO(skyewm): this behavior seems inconsistent with the above. Why is
+      # B's device ignored?
+      self.assertEqual(a.device, "")
+      self.assertEqual(b.device, "")
+      self.assertEqual(a.colocation_groups(), [b"loc:@A"])
+      self.assertEqual(b.colocation_groups(), [b"loc:@A"])
+
   def testColocationWithDeviceFn(self):
     original_graph_def = self._MakeGraphDef("""
           node { name: 'A' op: 'None' attr {
-- 
GitLab


From 73d17507de23db1c843de587441a958342c2f1e7 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Tue, 20 Mar 2018 21:39:16 -0700
Subject: [PATCH 284/960] Make variable scope and scope counts local to current
 thread so that they work correctly in multi-threaded environments.

PiperOrigin-RevId: 189860229
---
 tensorflow/contrib/eager/python/network.py    |   2 +-
 .../kernel_tests/variable_scope_test.py       |  87 +++++++++++++
 tensorflow/python/ops/template.py             |   2 +-
 tensorflow/python/ops/variable_scope.py       | 121 ++++++++++++------
 4 files changed, 171 insertions(+), 41 deletions(-)

diff --git a/tensorflow/contrib/eager/python/network.py b/tensorflow/contrib/eager/python/network.py
index 4c937716e8..e55a9276ab 100644
--- a/tensorflow/contrib/eager/python/network.py
+++ b/tensorflow/contrib/eager/python/network.py
@@ -149,7 +149,7 @@ class Network(base.Layer):
     # check we might have name collisions if the parent scope on init gets
     # closed before build is called.
     self._variable_scope_counts_on_init = (
-        variable_scope._get_default_variable_store().variable_scopes_count)
+        variable_scope.get_variable_scope_store().variable_scopes_count)
 
   def _name_scope_name(self, current_variable_scope):
     """Overrides Layer op naming to match variable naming."""
diff --git a/tensorflow/python/kernel_tests/variable_scope_test.py b/tensorflow/python/kernel_tests/variable_scope_test.py
index 531d0cdf90..86ab9fbb70 100644
--- a/tensorflow/python/kernel_tests/variable_scope_test.py
+++ b/tensorflow/python/kernel_tests/variable_scope_test.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import gc
+import threading
 
 import numpy
 
@@ -1349,5 +1350,91 @@ class PartitionInfoTest(test.TestCase):
     self.assertEqual(0, partition_info.single_slice_dim([2, 3]))
 
 
+class VariableScopeMultithreadedTest(test.TestCase):
+
+  def testTwoThreadsDisjointScopeEntry(self):
+
+    def thread_fn(i, graph):
+      with graph.as_default():
+        with variable_scope.variable_scope("foo"):
+          if i == 0:
+            v = variable_scope.get_variable("v", [])
+            self.assertEquals("foo/v:0", v.name)
+          else:
+            # Any thread after the first one should fail to create variable
+            # with the same name.
+            with self.assertRaises(ValueError):
+              variable_scope.get_variable("v", [])
+
+    graph = ops.get_default_graph()
+    threads = [
+        threading.Thread(target=thread_fn, args=(i, graph,)) for i in range(2)]
+
+    threads[0].start()
+    # Allow thread 0 to finish before starting thread 1.
+    threads[0].join()
+    threads[1].start()
+    threads[1].join()
+
+  def testTwoThreadsNestedScopeEntry(self):
+
+    def thread_fn(i, graph, run_event, pause_event):
+      with graph.as_default():
+        with variable_scope.variable_scope("foo"):
+          if i == 0:
+            v = variable_scope.get_variable("v", [])
+            self.assertEquals("foo/v:0", v.name)
+          else:
+            # Any thread after the first one should fail to create variable
+            # with the same name.
+            with self.assertRaises(ValueError):
+              variable_scope.get_variable("v", [])
+          pause_event.set()
+          run_event.wait()
+
+    graph = ops.get_default_graph()
+    run_events = [threading.Event() for _ in range(2)]
+    pause_events = [threading.Event() for _ in range(2)]
+    threads = [
+        threading.Thread(
+            target=thread_fn, args=(i, graph, run_events[i], pause_events[i]))
+        for i in range(2)
+    ]
+
+    # Start first thread.
+    threads[0].start()
+    pause_events[0].wait()
+    # Start next thread once the first thread has paused.
+    threads[1].start()
+    pause_events[1].wait()
+    # Resume both threads.
+    run_events[0].set()
+    run_events[1].set()
+    threads[0].join()
+    threads[1].join()
+
+  def testReenterMainScope(self):
+
+    def thread_fn(graph, main_thread_scope):
+      with graph.as_default():
+        # Variable created with main scope will have prefix "main".
+        with variable_scope.variable_scope(main_thread_scope):
+          with variable_scope.variable_scope("foo"):
+            v = variable_scope.get_variable("v", [])
+            self.assertEquals("main/foo/v:0", v.name)
+
+        # Variable created outside main scope will not have prefix "main".
+        with variable_scope.variable_scope("bar"):
+          v = variable_scope.get_variable("v", [])
+          self.assertEquals("bar/v:0", v.name)
+
+    graph = ops.get_default_graph()
+    with variable_scope.variable_scope("main") as main_thread_scope:
+      thread = threading.Thread(
+          target=thread_fn, args=(graph, main_thread_scope))
+      thread.start()
+      thread.join()
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/template.py b/tensorflow/python/ops/template.py
index 0a391d896a..0294ecee54 100644
--- a/tensorflow/python/ops/template.py
+++ b/tensorflow/python/ops/template.py
@@ -583,7 +583,7 @@ class _EagerTemplateVariableStore(object):
       if self._variable_scope_name is None:
         raise RuntimeError("A variable scope must be set before an "
                            "_EagerTemplateVariableStore object exits.")
-      self._eager_variable_store._store.close_variable_subscopes(  # pylint: disable=protected-access
+      variable_scope.get_variable_scope_store().close_variable_subscopes(
           self._variable_scope_name)
 
   def _variables_in_scope(self, variable_list):
diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index c1af8ff8d3..c35735ca65 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -24,6 +24,7 @@ import copy
 import enum  # pylint: disable=g-bad-import-order
 import functools
 import sys
+import threading
 import traceback
 
 import six
@@ -211,23 +212,8 @@ class _VariableStore(object):
     """Create a variable store."""
     self._vars = {}  # A dictionary of the stored TensorFlow variables.
     self._partitioned_vars = {}  # A dict of the stored PartitionedVariables.
-    self.variable_scopes_count = {}  # Count re-used variable scopes.
     self._store_eager_variables = False
 
-  def open_variable_scope(self, scope_name):
-    if scope_name in self.variable_scopes_count:
-      self.variable_scopes_count[scope_name] += 1
-    else:
-      self.variable_scopes_count[scope_name] = 1
-
-  def close_variable_subscopes(self, scope_name):
-    for k in self.variable_scopes_count:
-      if not scope_name or k.startswith(scope_name + "/"):
-        self.variable_scopes_count[k] = 0
-
-  def variable_scope_count(self, scope_name):
-    return self.variable_scopes_count.get(scope_name, 0)
-
   def get_variable(self, name, shape=None, dtype=dtypes.float32,
                    initializer=None, regularizer=None, reuse=None,
                    trainable=True, collections=None, caching_device=None,
@@ -1160,18 +1146,49 @@ class VariableScope(object):
 
 
 _VARSTORE_KEY = ("__variable_store",)
-_VARSCOPE_KEY = ("__varscope",)
+_VARSCOPESTORE_KEY = ("__varscope",)
+
+
+class _VariableScopeStore(threading.local):
+  """A thread local store for the current variable scope and scope counts."""
+
+  def __init__(self):
+    super(_VariableScopeStore, self).__init__()
+    self.current_scope = VariableScope(False)
+    self.variable_scopes_count = {}
+
+  def open_variable_scope(self, scope_name):
+    if scope_name in self.variable_scopes_count:
+      self.variable_scopes_count[scope_name] += 1
+    else:
+      self.variable_scopes_count[scope_name] = 1
+
+  def close_variable_subscopes(self, scope_name):
+    for k in self.variable_scopes_count:
+      if not scope_name or k.startswith(scope_name + "/"):
+        self.variable_scopes_count[k] = 0
+
+  def variable_scope_count(self, scope_name):
+    return self.variable_scopes_count.get(scope_name, 0)
+
+
+def get_variable_scope_store():
+  """Returns the variable scope store for current thread."""
+  scope_store = ops.get_collection(_VARSCOPESTORE_KEY)
+
+  if not scope_store:
+    scope_store = _VariableScopeStore()
+    ops.add_to_collection(_VARSCOPESTORE_KEY, scope_store)
+  else:
+    scope_store = scope_store[0]
+
+  return scope_store
 
 
 @tf_export("get_variable_scope")
 def get_variable_scope():
   """Returns the current variable scope."""
-  scope = ops.get_collection(_VARSCOPE_KEY)
-  if scope:  # This collection has at most 1 element, the default scope at [0].
-    return scope[0]
-  scope = VariableScope(False)
-  ops.add_to_collection(_VARSCOPE_KEY, scope)
-  return scope
+  return get_variable_scope_store().current_scope
 
 
 def _get_default_variable_store():
@@ -1575,10 +1592,8 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
     self._dtype = dtype
     self._use_resource = use_resource
     self._constraint = constraint
-    get_variable_scope()  # Ensure that a default exists, then get a pointer.
-    # Get the reference to the collection as we want to modify it in place.
-    self._default_varscope = ops.get_collection_ref(_VARSCOPE_KEY)
     self._var_store = _get_default_variable_store()
+    self._var_scope_store = get_variable_scope_store()
     if isinstance(self._name_or_scope, VariableScope):
       self._new_name = self._name_or_scope.name
       name_scope = self._name_or_scope._name_scope  # pylint: disable=protected-access
@@ -1626,10 +1641,11 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
         a reuse scope, or if reuse is not `None` or `True`.
       TypeError: when the types of some arguments are not appropriate.
     """
-    self._old = self._default_varscope[0]
+    self._old = self._var_scope_store.current_scope
     if isinstance(self._name_or_scope, VariableScope):
-      self._var_store.open_variable_scope(self._new_name)
-      self._old_subscopes = copy.copy(self._var_store.variable_scopes_count)
+      self._var_scope_store.open_variable_scope(self._new_name)
+      self._old_subscopes = copy.copy(
+          self._var_scope_store.variable_scopes_count)
       variable_scope_object = self._cached_variable_scope_object
     else:
       # Handler for the case when we just prolong current variable scope.
@@ -1672,17 +1688,17 @@ class _pure_variable_scope(object):  # pylint: disable=invalid-name
         variable_scope_object.set_dtype(self._dtype)
       if self._use_resource is not None:
         variable_scope_object.set_use_resource(self._use_resource)
-      self._var_store.open_variable_scope(self._new_name)
-    self._default_varscope[0] = variable_scope_object
+      self._var_scope_store.open_variable_scope(self._new_name)
+    self._var_scope_store.current_scope = variable_scope_object
     return variable_scope_object
 
   def __exit__(self, type_arg, value_arg, traceback_arg):
     # If jumping out from a non-prolonged scope, restore counts.
     if isinstance(self._name_or_scope, VariableScope):
-      self._var_store.variable_scopes_count = self._old_subscopes
+      self._var_scope_store.variable_scopes_count = self._old_subscopes
     else:
-      self._var_store.close_variable_subscopes(self._new_name)
-    self._default_varscope[0] = self._old
+      self._var_scope_store.close_variable_subscopes(self._new_name)
+    self._var_scope_store.current_scope = self._old
 
 
 def _maybe_wrap_custom_getter(custom_getter, old_getter):
@@ -1707,13 +1723,13 @@ def _maybe_wrap_custom_getter(custom_getter, old_getter):
 
 def _get_unique_variable_scope(prefix):
   """Get a name with the given prefix unique in the current variable scope."""
-  var_store = _get_default_variable_store()
+  var_scope_store = get_variable_scope_store()
   current_scope = get_variable_scope()
   name = current_scope.name + "/" + prefix if current_scope.name else prefix
-  if var_store.variable_scope_count(name) == 0:
+  if var_scope_store.variable_scope_count(name) == 0:
     return prefix
   idx = 1
-  while var_store.variable_scope_count(name + ("_%d" % idx)) > 0:
+  while var_scope_store.variable_scope_count(name + ("_%d" % idx)) > 0:
     idx += 1
   return prefix + ("_%d" % idx)
 
@@ -1729,9 +1745,10 @@ class variable_scope(object):
   graph, ensures that graph is the default graph, and pushes a name scope and a
   variable scope.
 
-  If `name_or_scope` is not None, it is used as is. If `scope` is None, then
-  `default_name` is used.  In that case, if the same name has been previously
-  used in the same scope, it will be made unique by appending `_N` to it.
+  If `name_or_scope` is not None, it is used as is. If `name_or_scope` is None,
+  then `default_name` is used.  In that case, if the same name has been
+  previously used in the same scope, it will be made unique by appending `_N`
+  to it.
 
   Variable scope allows you to create new variables and to share already created
   ones while providing checks to not create or share by accident. For details,
@@ -1810,6 +1827,32 @@ class variable_scope(object):
   discouraged) to pass False to the reuse argument, yielding undocumented
   behaviour slightly different from None. Starting at 1.1.0 passing None and
   False as reuse has exactly the same effect.
+
+  A note about using variable scopes in multi-threaded environment: Variable
+  scopes are thread local, so one thread will not see another thread's current
+  scope. Also, when using `default_name`, unique scopes names are also generated
+  only on a per thread basis. If the same name was used within a different
+  thread, that doesn't prevent a new thread from creating the same scope.
+  However, the underlying variable store is shared across threads (within the
+  same graph). As such, if another thread tries to create a new variable with
+  the same name as a variable created by a previous thread, it will fail unless
+  reuse is True.
+
+  Further, each thread starts with an empty variable scope. So if you wish to
+  preserve name prefixes from a scope from the main thread, you should capture
+  the main thread's scope and re-enter it in each thread. For e.g.
+
+  ```
+  main_thread_scope = variable_scope.get_variable_scope()
+
+  # Thread's target function:
+  def thread_target_fn(captured_scope):
+    with variable_scope.variable_scope(captured_scope):
+      # .... regular code for this thread
+
+
+  thread = threading.Thread(target=thread_target_fn, args=(main_thread_scope,))
+  ```
   """
 
   def __init__(self,
-- 
GitLab


From d5d74b0aaaa221ad64aa1d86cb8428df2b885cf7 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Tue, 20 Mar 2018 23:07:37 -0700
Subject: [PATCH 285/960] Make graph's name scope thread local so that two
 threads opening the same scope don't get nested under each other.

PiperOrigin-RevId: 189865854
---
 tensorflow/python/framework/ops.py      | 13 +++++++++--
 tensorflow/python/framework/ops_test.py | 29 +++++++++++++++++++++++++
 2 files changed, 40 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 4be2e2c15d..50a1d3fe04 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2727,8 +2727,6 @@ class Graph(object):
     self._next_id_counter = 0  # GUARDED_BY(self._lock)
     self._nodes_by_name = dict()  # GUARDED_BY(self._lock)
     self._version = 0  # GUARDED_BY(self._lock)
-    # Current name stack: uniquified names
-    self._name_stack = ""
     # Maps a name used in the graph to the next id to use for that name.
     self._names_in_use = {}
     self._stack_state_is_thread_local = False
@@ -3907,6 +3905,17 @@ class Graph(object):
     finally:
       self._default_original_op = old_original_op
 
+  @property
+  def _name_stack(self):
+    # This may be called from a thread where name_stack doesn't yet exist.
+    if not hasattr(self._thread_local, "_name_stack"):
+      self._thread_local._name_stack = ""
+    return self._thread_local._name_stack
+
+  @_name_stack.setter
+  def _name_stack(self, name_stack):
+    self._thread_local._name_stack = name_stack
+
   # pylint: disable=g-doc-return-or-yield,line-too-long
   @tf_contextlib.contextmanager
   def name_scope(self, name):
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index d96e0708f8..aa51391871 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -1556,6 +1556,35 @@ class MultithreadedGraphStateTest(test_util.TensorFlowTestCase):
              input: "^ColocateWithMe_2" }
     """, gd)
 
+  def testNameStack(self):
+
+    class NameSettingThread(self.TestThread):
+
+      def run(self):
+        with g.name_scope("foo"):
+          op1 = g.create_op("FloatOutput", [], [dtypes.float32])
+          self.has_mutated_graph.set()
+          self.should_continue.wait()
+          self.should_continue.clear()
+          op2 = g.create_op("FloatOutput", [], [dtypes.float32])
+          self.result = (op1, op2)
+
+    g = ops.Graph()
+    threads = [NameSettingThread(g, i) for i in range(3)]
+    for t in threads:
+      t.start()
+      t.has_mutated_graph.wait()
+      t.has_mutated_graph.clear()
+
+    for t in threads:
+      t.should_continue.set()
+      t.join()
+
+    suffixes = ["", "_1", "_2"]
+    for t, s in zip(threads, suffixes):
+      self.assertEquals("foo" + s + "/FloatOutput", t.result[0].name)
+      self.assertEquals("foo" + s + "/FloatOutput_1", t.result[1].name)
+
 
 @test_util.with_c_api
 class ObjectWithName(object):
-- 
GitLab


From e69000c347ddf023a3b1926d812881fd8c5a055b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 04:09:32 -0700
Subject: [PATCH 286/960] We were ValueOrDie()ing in one place, and
 TF_CHECK_OK()ing in another. Both should gracefully return an error
 condition.

Add some tests to check this.

PiperOrigin-RevId: 189888700
---
 tensorflow/compiler/jit/xla_device_context.cc | 12 ++--
 tensorflow/compiler/jit/xla_launch_util.cc    | 14 +++-
 tensorflow/compiler/tests/BUILD               | 20 ++++++
 tensorflow/compiler/tests/oom_test.py         | 72 +++++++++++++++++++
 4 files changed, 111 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/compiler/tests/oom_test.py

diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index 88f7c15f0b..93e0dbb9b9 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -36,10 +36,14 @@ XlaDeviceAllocator::~XlaDeviceAllocator() = default;
 string XlaDeviceAllocator::Name() { return "xla"; }
 
 void* XlaDeviceAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
-  se::DeviceMemoryBase dmem =
-      backend_->memory_allocator()
-          ->Allocate(device_ordinal_, num_bytes, /*retry_on_failure=*/false)
-          .ValueOrDie();
+  auto status_or_dmem = backend_->memory_allocator()->Allocate(
+      device_ordinal_, num_bytes, /*retry_on_failure=*/false);
+  if (!status_or_dmem.status().ok()) {
+    LOG(ERROR) << "Failed to allocate memory: "
+               << status_or_dmem.status().ToString();
+    return nullptr;
+  }
+  se::DeviceMemoryBase dmem = status_or_dmem.ValueOrDie();
   VLOG(2) << "Allocated XLA device tensor " << dmem.opaque() << "(" << num_bytes
           << ")";
   return dmem.opaque();
diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index bb7316c60c..21f58c8310 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -56,12 +56,20 @@ XlaAllocator::XlaAllocator(const gpu::Platform* platform,
                            OpKernelContext* op_context)
     : xla::DeviceMemoryAllocator(platform), op_context_(op_context) {}
 
-XlaAllocator::~XlaAllocator() { CHECK(allocated_.empty()); }
+XlaAllocator::~XlaAllocator() {
+  for (void* ptr : allocated_) {
+    op_context_->device()->GetAllocator({})->DeallocateRaw(ptr);
+  }
+}
 
 xla::StatusOr<gpu::DeviceMemoryBase> XlaAllocator::Allocate(
     int device_ordinal, uint64 size, bool retry_on_failure) {
   void* data = op_context_->device()->GetAllocator({})->AllocateRaw(
       Allocator::kAllocatorAlignment, size);
+  if (!data) {
+    return errors::ResourceExhausted(
+        "OOM when allocating temporary tensor with size ", size);
+  }
   allocated_.insert(data);
   return gpu::DeviceMemoryBase(data, size);
 }
@@ -182,8 +190,8 @@ void XlaComputationLaunchContext::PopulateOutputs(
         // Copy host -> device. (Empty tensors don't have backing buffers.)
         VLOG(1) << "Constant output tensor on device";
 
-        TF_CHECK_OK(
-            ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
+        OP_REQUIRES_OK(
+            ctx, ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
 
         const void* src_ptr = DMAHelper::base(&const_tensor);
         void* dst_ptr = DMAHelper::base(output_tensor);
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index bbb6089ea8..26d4ca0c13 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -191,6 +191,26 @@ tf_xla_py_test(
     ],
 )
 
+tf_xla_py_test(
+    name = "oom_test",
+    size = "medium",
+    srcs = ["oom_test.py"],
+    disabled_backends = [
+        "cpu",
+        "cpu_ondemand",
+    ],
+    deps = [
+        ":xla_test",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:array_ops_gen",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:gradient_checker",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 tf_xla_py_test(
     name = "conv2d_test",
     size = "medium",
diff --git a/tensorflow/compiler/tests/oom_test.py b/tensorflow/compiler/tests/oom_test.py
new file mode 100644
index 0000000000..66be0d61d0
--- /dev/null
+++ b/tensorflow/compiler/tests/oom_test.py
@@ -0,0 +1,72 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional tests for out-of-memory conditions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.compiler.tests import xla_test
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.platform import googletest
+
+
+class OutOfMemoryTest(xla_test.XLATestCase):
+
+  def testOutputOutOfMemory(self):
+    """Allocates tensors until out of memory.
+
+    Generates a large rank-1 tensor. The tensor is an output of an XLA
+    computation, not constant.
+
+    Check that a ResourceExhaustedError is raised and can be caught.
+    """
+    size = 5e8
+    with self.test_session():
+      # Force the compiled code to not be constant by feeding in an addend.
+      p = array_ops.placeholder(dtypes.float32, shape=[])
+      with self.test_scope():
+        # Create a large R1 tensor.
+        c = array_ops.zeros([size]) + p
+
+        self.assertRaises(
+            errors.ResourceExhaustedError, lambda: c.eval(feed_dict={p: 1.0}))
+
+  def testConstantOutOfMemory(self):
+    """Allocates constant tensors until out of memory.
+
+    Generates a large rank-1 tensor and a small rank-1 tensor. The tensors are
+    constant outputs of an XLA computation, not variable.
+
+    Multiple constant outputs are created, one small, one large. The small
+    tensor will have already been allocated when the large tensor fails.
+
+    Check that a ResourceExhaustedError is raised and can be caught.
+    """
+    size = 5e8
+    with self.test_session() as sess:
+      with self.test_scope():
+        # Create two R1 tensors, size 5 and size n.
+        b = array_ops.zeros([5])
+        c = array_ops.zeros([size])
+        e = control_flow_ops.tuple([b, c])
+        self.assertRaises(errors.ResourceExhaustedError, lambda: sess.run(e))
+
+
+if __name__ == "__main__":
+  googletest.main()
-- 
GitLab


From 73a5fb686c10f044f245b27f246ff56f690ada1f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 05:55:32 -0700
Subject: [PATCH 287/960] Minor cosmetic improvement to error message.

PiperOrigin-RevId: 189895415
---
 tensorflow/contrib/py2tf/converters/lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/py2tf/converters/lists.py b/tensorflow/contrib/py2tf/converters/lists.py
index 06e1dad8f4..12ebd00062 100644
--- a/tensorflow/contrib/py2tf/converters/lists.py
+++ b/tensorflow/contrib/py2tf/converters/lists.py
@@ -61,7 +61,7 @@ class ListTransformer(transformer.Base):
     return templates.replace_as_expression(template, dtype_name=dtype_name)
 
   def _pre_populated_list(self, node):
-    raise NotImplementedError()
+    raise NotImplementedError('pre-populated lists')
 
   def visit_Expr(self, node):
     node = self.generic_visit(node)
-- 
GitLab


From abd5b15ababbb5601f02691620d4d8e094cff64e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 06:57:06 -0700
Subject: [PATCH 288/960] Tensorflow/GCS: Check whether we are running under
 GCE before trying to obtain auth token from GCE. Before this change, if a
 process is not running under GCE, the token request to http://metadata server
 would time out after 3+ minutes of retry. After this change, the check is
 bypassed, and we return an empty token to the caller. At that point, the
 caller's request to read/write a file in GCS would either succeed or fail
 depending on whether the bucket is publicly accessible.

PiperOrigin-RevId: 189900977
---
 tensorflow/core/platform/cloud/BUILD          |  41 +++++
 tensorflow/core/platform/cloud/fake_env.cc    |  62 +++++++
 tensorflow/core/platform/cloud/fake_env.h     |  60 +++++++
 .../core/platform/cloud/gce_env_utils.cc      | 159 ++++++++++++++++++
 .../core/platform/cloud/gce_env_utils.h       |  29 ++++
 .../core/platform/cloud/gcp_env_utils_test.cc |  53 ++++++
 .../platform/cloud/google_auth_provider.cc    |  11 ++
 .../platform/cloud/google_auth_provider.h     |   7 +-
 .../cloud/google_auth_provider_test.cc        |  39 +++--
 tensorflow/core/platform/env.h                |   4 +-
 10 files changed, 449 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/core/platform/cloud/fake_env.cc
 create mode 100644 tensorflow/core/platform/cloud/fake_env.h
 create mode 100644 tensorflow/core/platform/cloud/gce_env_utils.cc
 create mode 100644 tensorflow/core/platform/cloud/gce_env_utils.h
 create mode 100644 tensorflow/core/platform/cloud/gcp_env_utils_test.cc

diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index 21636641e7..e43639e9c7 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -79,6 +79,18 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "gce_env_utils",
+    srcs = ["gce_env_utils.cc"],
+    hdrs = ["gce_env_utils.h"],
+    copts = tf_copts(),
+    visibility = ["//visibility:private"],
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 cc_library(
     name = "gcs_file_system",
     srcs = ["gcs_file_system.cc"],
@@ -158,6 +170,7 @@ cc_library(
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
         ":curl_http_request",
+        ":gce_env_utils",
         ":oauth_client",
         ":retrying_utils",
         "//tensorflow/core:lib",
@@ -243,6 +256,21 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "fake_env",
+    srcs = [
+        "fake_env.cc",
+    ],
+    hdrs = [
+        "fake_env.h",
+    ],
+    copts = tf_copts(),
+    deps = [
+        "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
 tf_cc_test(
     name = "expiring_lru_cache_test",
     size = "small",
@@ -348,6 +376,7 @@ tf_cc_test(
         "testdata/service_account_credentials.json",
     ],
     deps = [
+        ":fake_env",
         ":google_auth_provider",
         ":http_request_fake",
         ":oauth_client",
@@ -394,3 +423,15 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
+
+tf_cc_test(
+    name = "gce_env_utils_test",
+    size = "small",
+    srcs = ["gcp_env_utils_test.cc"],
+    deps = [
+        ":fake_env",
+        ":gce_env_utils",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
diff --git a/tensorflow/core/platform/cloud/fake_env.cc b/tensorflow/core/platform/cloud/fake_env.cc
new file mode 100644
index 0000000000..221166839e
--- /dev/null
+++ b/tensorflow/core/platform/cloud/fake_env.cc
@@ -0,0 +1,62 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/cloud/fake_env.h"
+
+namespace tensorflow {
+namespace test {
+
+Status FakeEnv::FakeRandomAccessFile::Read(uint64 offset, size_t n,
+                                           StringPiece* result,
+                                           char* scratch) const {
+  CHECK_EQ(offset, 0);
+  CHECK_EQ(n, 256);
+  Status s;
+  string platform;
+  switch (env_type_) {
+    case kGoogle: {
+      platform = "Google\n  ";
+      s = errors::OutOfRange("");
+      break;
+    }
+    case kGce: {
+      platform = "  Google Compute Engine\n  ";
+      s = errors::OutOfRange("");
+      break;
+    }
+    case kLocal: {
+      platform = "HP Linux Workstation";
+      s = Status::OK();
+      break;
+    }
+    case kBad: {
+      platform = "";
+      s = errors::Internal("Expected");
+      break;
+    }
+  }
+  strncpy(scratch, platform.data(), strlen(platform.data()));
+  *result = StringPiece(scratch, platform.length());
+  return s;
+}
+
+Status FakeEnv::NewRandomAccessFile(const string& fname,
+                                    std::unique_ptr<RandomAccessFile>* result) {
+  result->reset(new FakeRandomAccessFile(env_type_));
+  return Status::OK();
+}
+
+}  // namespace test
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/fake_env.h b/tensorflow/core/platform/cloud/fake_env.h
new file mode 100644
index 0000000000..7c162d9d66
--- /dev/null
+++ b/tensorflow/core/platform/cloud/fake_env.h
@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
+#define TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
+
+#include "tensorflow/core/platform/env.h"
+
+namespace tensorflow {
+namespace test {
+
+/// Env implementation that stubs out the calls to read a file and time.
+class FakeEnv : public EnvWrapper {
+ public:
+  enum EnvType {
+    kGoogle,
+    kGce,
+    kLocal,
+    kBad,
+  };
+
+  FakeEnv(EnvType env_type) : EnvWrapper(Env::Default()), env_type_(env_type) {}
+
+  class FakeRandomAccessFile : public RandomAccessFile {
+   public:
+    FakeRandomAccessFile(EnvType env_type) : env_type_(env_type) {}
+
+    Status Read(uint64 offset, size_t n, StringPiece* result,
+                char* scratch) const override;
+
+   private:
+    EnvType env_type_;
+  };
+
+  Status NewRandomAccessFile(
+      const string& fname, std::unique_ptr<RandomAccessFile>* result) override;
+
+  uint64 NowSeconds() override { return now; }
+  uint64 now = 10000;
+
+ private:
+  EnvType env_type_;
+};
+
+}  // namespace test
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
diff --git a/tensorflow/core/platform/cloud/gce_env_utils.cc b/tensorflow/core/platform/cloud/gce_env_utils.cc
new file mode 100644
index 0000000000..d78374c4b8
--- /dev/null
+++ b/tensorflow/core/platform/cloud/gce_env_utils.cc
@@ -0,0 +1,159 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/cloud/gce_env_utils.h"
+
+#if defined(PLATFORM_WINDOWS)
+#include <algorithm>
+#include <cctype>
+#include <iostream>
+#include <string>
+
+// The order if these includes is important, windows.h has to come first.
+// clang-format off
+#include <windows.h>   // NOLINT
+#include <tchar.h>     // NOLINT
+#include <shellapi.h>  // NOLINT
+// clang-format on
+#else
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#endif
+
+namespace tensorflow {
+
+constexpr char kExpectedGoogleProductName[] = "Google";
+constexpr char kExpectedGceProductName[] = "Google Compute Engine";
+
+constexpr char kWinCheckCommand[] = "powershell.exe";
+constexpr char kWinCheckCommandArgs[] =
+    "(Get-WmiObject -Class Win32_BIOS).Manufacturer";
+
+constexpr char kLinuxProductNameFile[] = "/sys/class/dmi/id/product_name";
+
+const size_t kBiosDataBufferSize = 256;
+
+namespace {
+
+#if defined(PLATFORM_WINDOWS)
+
+Status IsRunningOnWinGce(bool* is_running_under_gce) {
+  *is_running_under_gce = FALSE;
+  SECURITY_ATTRIBUTES sa;
+  sa.nLength = sizeof(sa);
+  sa.lpSecurityDescriptor = NULL;
+  sa.bInheritHandle = TRUE;
+
+  // Handles to input and output of the pipe connecting us
+  // to the child process running powershell(). The output of this
+  // child process will be written to 'process_output_in' and read from
+  // 'process_output_in'.
+  HANDLE process_output_out = NULL;
+  HANDLE process_output_in = NULL;
+
+  // Create the actually pipe connecting us to the child process.
+  if (!CreatePipe(&process_output_out, &process_output_in, &sa, 0)) {
+    return errors::Internal("CreatePipe() failed");
+  }
+  if (!SetHandleInformation(process_output_out, HANDLE_FLAG_INHERIT, 0)) {
+    return errors::Internal("SetHandleInformation() failed");
+  }
+
+  PROCESS_INFORMATION pi;
+  STARTUPINFO si;
+  DWORD flags = CREATE_NO_WINDOW;
+  ZeroMemory(&pi, sizeof(pi));
+  ZeroMemory(&si, sizeof(si));
+  si.cb = sizeof(si);
+  si.dwFlags |= STARTF_USESTDHANDLES;
+  si.hStdInput = NULL;
+
+  // Connect the process to pipe's input.
+  si.hStdError = process_output_in;
+  si.hStdOutput = process_output_in;
+  // Execute (and wait for) powershell command to read the product information
+  // out of the registry.
+  TCHAR cmd[kBiosDataBufferSize];
+  snprintf(cmd, kBiosDataBufferSize, "%s %s", _T(kWinCheckCommand),
+           _T(kWinCheckCommandArgs));
+
+  if (!CreateProcess(NULL, cmd, NULL, NULL, TRUE, flags, NULL, NULL, &si,
+                     &pi)) {
+    return errors::Internal("CreateProcess() failed");
+  }
+
+  WaitForSingleObject(pi.hProcess, INFINITE);
+  CloseHandle(pi.hProcess);
+  CloseHandle(pi.hThread);
+
+  // Read data from the pipe. Note that we are reading only kBiosDataBufferSize
+  // chars. There might be technically more data than that but we are looking
+  // for Google product identifiers that are much shorter than
+  // kBiosDataBufferSize.
+  DWORD dwread = 0;
+  CHAR buffer[kBiosDataBufferSize];
+  if (!ReadFile(process_output_out, buffer, kBiosDataBufferSize, &dwread,
+                NULL)) {
+    return errors::Internal("Failed reading from the pipe.");
+  }
+  std::string output(buffer, 0, dwread);
+  // Trim whitespaces
+  output.erase(output.begin(),
+               std::find_if(output.begin(), output.end(),
+                            [](int ch) { return !std::isspace(ch); }));
+  output.erase(std::find_if(output.rbegin(), output.rend(),
+                            [](int ch) { return !std::isspace(ch); })
+                   .base(),
+               output.end());
+  *is_running_under_gce =
+      output == kExpectedGceProductName || output == kExpectedGoogleProductName;
+  return Status::OK();
+}
+
+#else
+
+Status IsRunningOnLinuxGce(Env* env, bool* is_running_under_gce) {
+  std::unique_ptr<RandomAccessFile> file;
+  TF_RETURN_IF_ERROR(env->NewRandomAccessFile(kLinuxProductNameFile, &file));
+  char buf[kBiosDataBufferSize + 1];
+  std::fill(buf, buf + kBiosDataBufferSize + 1, '\0');
+  StringPiece product_name;
+  const Status s = file->Read(0, kBiosDataBufferSize, &product_name, buf);
+  if (!s.ok() && !errors::IsOutOfRange(s)) {
+    // We expect OutOfRange error because bios file doesn't correspond to its
+    // state size,
+    return s;
+  }
+  str_util::RemoveLeadingWhitespace(&product_name);
+  str_util::RemoveTrailingWhitespace(&product_name);
+  *is_running_under_gce = (product_name == kExpectedGceProductName ||
+                           product_name == kExpectedGoogleProductName);
+  return Status::OK();
+}
+
+#endif
+
+}  // namespace
+
+Status IsRunningOnGce(Env* env, bool* is_running_under_gce) {
+  *is_running_under_gce = false;
+#if defined(PLATFORM_WINDOWS)
+  return IsRunningOnWinGce(is_running_under_gce);
+#else
+  return IsRunningOnLinuxGce(env, is_running_under_gce);
+#endif
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/gce_env_utils.h b/tensorflow/core/platform/cloud/gce_env_utils.h
new file mode 100644
index 0000000000..25aaeb7db3
--- /dev/null
+++ b/tensorflow/core/platform/cloud/gce_env_utils.h
@@ -0,0 +1,29 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
+#define TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/platform/env.h"
+
+namespace tensorflow {
+
+// Check whether the current process is running under GCE.
+Status IsRunningOnGce(Env* env, bool* is_running_under_gce);
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
diff --git a/tensorflow/core/platform/cloud/gcp_env_utils_test.cc b/tensorflow/core/platform/cloud/gcp_env_utils_test.cc
new file mode 100644
index 0000000000..910397b52b
--- /dev/null
+++ b/tensorflow/core/platform/cloud/gcp_env_utils_test.cc
@@ -0,0 +1,53 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/cloud/gce_env_utils.h"
+
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/cloud/fake_env.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+
+namespace {
+
+TEST(GcpEnvUtils, IsRunningOnGce) {
+  {
+    test::FakeEnv env(test::FakeEnv::kGoogle);
+    bool is_running_on_gcp = false;
+    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
+    EXPECT_TRUE(is_running_on_gcp);
+  }
+  {
+    test::FakeEnv env(test::FakeEnv::kGce);
+    bool is_running_on_gcp = false;
+    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
+    EXPECT_TRUE(is_running_on_gcp);
+  }
+  {
+    test::FakeEnv env(test::FakeEnv::kLocal);
+    bool is_running_on_gcp = false;
+    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
+    EXPECT_FALSE(is_running_on_gcp);
+  }
+  {
+    test::FakeEnv env(test::FakeEnv::kBad);
+    bool is_running_on_gcp = false;
+    EXPECT_TRUE(errors::IsInternal(IsRunningOnGce(&env, &is_running_on_gcp)));
+  }
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.cc b/tensorflow/core/platform/cloud/google_auth_provider.cc
index 7e39b63e3e..0e8a620464 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/base64.h"
 #include "tensorflow/core/platform/cloud/curl_http_request.h"
+#include "tensorflow/core/platform/cloud/gce_env_utils.h"
 #include "tensorflow/core/platform/cloud/retrying_utils.h"
 #include "tensorflow/core/platform/env.h"
 
@@ -207,6 +208,16 @@ Status GoogleAuthProvider::GetTokenFromFiles() {
 }
 
 Status GoogleAuthProvider::GetTokenFromGce() {
+  if (!is_running_on_gce_.has_value()) {
+    bool is_running_on_gce = false;
+    TF_RETURN_IF_ERROR(IsRunningOnGce(env_, &is_running_on_gce));
+    is_running_on_gce_ = is_running_on_gce;
+  }
+  if (!is_running_on_gce_.value()) {
+    // Assume bucket is world-accessible. If not, the access will be rejected.
+    current_token_ = "";
+    return Status::OK();
+  }
   const auto get_token_from_gce = [this]() {
     std::unique_ptr<HttpRequest> request(http_request_factory_->Create());
     std::vector<char> response_buffer;
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.h b/tensorflow/core/platform/cloud/google_auth_provider.h
index 00da25a959..79a57ff2a0 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.h
+++ b/tensorflow/core/platform/cloud/google_auth_provider.h
@@ -17,6 +17,7 @@ limitations under the License.
 #define TENSORFLOW_CORE_PLATFORM_GOOGLE_AUTH_PROVIDER_H_
 
 #include <memory>
+#include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/cloud/auth_provider.h"
 #include "tensorflow/core/platform/cloud/oauth_client.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -46,7 +47,10 @@ class GoogleAuthProvider : public AuthProvider {
   /// standard gcloud tool's location.
   Status GetTokenFromFiles() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
-  /// Gets the bearer token from Google Compute Engine environment.
+  /// Gets the bearer token from Google Compute Engine environment. May return
+  /// an empty token if the current process is not running under GCE. If that
+  /// happens the caller will try to use the empty token and either succeed
+  /// if the resource is publicly accessible or fail with a permissions error.
   Status GetTokenFromGce() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   /// Gets the bearer token from the systen env variable, for testing purposes.
@@ -57,6 +61,7 @@ class GoogleAuthProvider : public AuthProvider {
   Env* env_;
   mutex mu_;
   string current_token_ GUARDED_BY(mu_);
+  tensorflow::gtl::optional<bool> is_running_on_gce_ GUARDED_BY(mu_);
   uint64 expiration_timestamp_sec_ GUARDED_BY(mu_) = 0;
   // The initial delay for exponential backoffs when retrying failed calls.
   const int64 initial_retry_delay_usec_;
diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
index 4281c6c737..55829f84d9 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include <stdlib.h>
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/cloud/fake_env.h"
 #include "tensorflow/core/platform/cloud/http_request_fake.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -26,14 +27,6 @@ namespace {
 
 constexpr char kTestData[] = "core/platform/cloud/testdata/";
 
-class FakeEnv : public EnvWrapper {
- public:
-  FakeEnv() : EnvWrapper(Env::Default()) {}
-
-  uint64 NowSeconds() override { return now; }
-  uint64 now = 10000;
-};
-
 class FakeOAuthClient : public OAuthClient {
  public:
   Status GetTokenFromServiceAccountJson(
@@ -89,7 +82,7 @@ TEST_F(GoogleAuthProviderTest, EnvironmentVariable_Caching) {
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> requests;
 
-  FakeEnv env;
+  test::FakeEnv env(test::FakeEnv::kGoogle);
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -123,7 +116,7 @@ TEST_F(GoogleAuthProviderTest, GCloudRefreshToken) {
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> requests;
 
-  FakeEnv env;
+  test::FakeEnv env(test::FakeEnv::kGoogle);
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -169,7 +162,7 @@ TEST_F(GoogleAuthProviderTest, RunningOnGCE) {
                 "token_type":"Bearer"
               })")});
 
-  FakeEnv env;
+  test::FakeEnv env(test::FakeEnv::kGoogle);
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -195,7 +188,7 @@ TEST_F(GoogleAuthProviderTest, OverrideForTesting) {
 
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> empty_requests;
-  FakeEnv env;
+  test::FakeEnv env(test::FakeEnv::kGoogle);
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&empty_requests)),
@@ -215,7 +208,25 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
       "Header Metadata-Flavor: Google\n",
       "", errors::NotFound("404"), 404)});
 
-  FakeEnv env;
+  test::FakeEnv env(test::FakeEnv::kGoogle);
+  GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
+                              std::unique_ptr<HttpRequest::Factory>(
+                                  new FakeHttpRequestFactory(&requests)),
+                              &env, 0);
+
+  string token;
+  TF_EXPECT_OK(provider.GetToken(&token));
+  EXPECT_EQ("", token);
+}
+
+TEST_F(GoogleAuthProviderTest, AccessingPublicBucket) {
+  setenv("CLOUDSDK_CONFIG",
+         io::JoinPath(testing::TensorFlowSrcRoot(), kTestData).c_str(), 1);
+
+  auto oauth_client = new FakeOAuthClient;
+  std::vector<HttpRequest*> requests;
+
+  test::FakeEnv env(test::FakeEnv::kLocal);
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -223,6 +234,8 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
 
   string token;
   TF_EXPECT_OK(provider.GetToken(&token));
+  // We are assuming we are accessing a public bucket (and we are not running
+  // on GCE) so we an empty token is returned.
   EXPECT_EQ("", token);
 }
 
diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index 4ce4e0b4e0..2a114d47a8 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -88,8 +88,8 @@ class Env {
   /// The ownership of the returned RandomAccessFile is passed to the caller
   /// and the object should be deleted when is not used. The file object
   /// shouldn't live longer than the Env object.
-  Status NewRandomAccessFile(const string& fname,
-                             std::unique_ptr<RandomAccessFile>* result);
+  virtual Status NewRandomAccessFile(const string& fname,
+                                     std::unique_ptr<RandomAccessFile>* result);
 
   /// \brief Creates an object that writes to a new file with the specified
   /// name.
-- 
GitLab


From 39dd4ee6a3727a0eb30a8d5b8f39390383a1e761 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 21 Mar 2018 07:33:03 -0700
Subject: [PATCH 289/960] [XLA] Initialize arrays using cudaMemset when
 possible.

Previously we were using our own hand-rolled initializer thunk.  This
worked OK for reduces, because the amount of data we were initializing
is usually small.  But for e.g. select-and-scatter, it's quite slow.

This patch lets us use cudaMemset instead.

PiperOrigin-RevId: 189904720
---
 tensorflow/compiler/xla/service/gpu/BUILD     |   4 +
 .../xla/service/gpu/ir_emitter_unnested.cc    | 129 ++++++++++++------
 .../xla/service/gpu/ir_emitter_unnested.h     |  10 +-
 .../compiler/xla/service/gpu/memset_thunk.cc  |  39 ++++++
 .../compiler/xla/service/gpu/memset_thunk.h   |  65 +++++++++
 tensorflow/compiler/xla/service/gpu/thunk.h   |   2 +
 tensorflow/compiler/xla/tests/reduce_test.cc  |  42 ++++++
 7 files changed, 247 insertions(+), 44 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/gpu/memset_thunk.cc
 create mode 100644 tensorflow/compiler/xla/service/gpu/memset_thunk.h

diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index a3b7e10ae8..93b2f2a474 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -241,6 +241,7 @@ cc_library(
         "gpu_executable.cc",
         "infeed_thunk.cc",
         "kernel_thunk.cc",
+        "memset_thunk.cc",
         "sequential_thunk.cc",
         "thunk_schedule.cc",
         "tuple_thunk.cc",
@@ -257,6 +258,7 @@ cc_library(
         "gpu_executable.h",
         "infeed_thunk.h",
         "kernel_thunk.h",
+        "memset_thunk.h",
         "sequential_thunk.h",
         "thunk.h",
         "thunk_schedule.h",
@@ -273,6 +275,7 @@ cc_library(
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:shape_tree",
         "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:status",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:types",
@@ -293,6 +296,7 @@ cc_library(
         "//tensorflow/core/platform/default/build_config:cudnn_plugin",
         "//tensorflow/core/platform/default/build_config:cufft_plugin",
         "//tensorflow/core/platform/default/build_config:stream_executor_cuda",  # build_cleaner: keep
+        "//tensorflow/stream_executor",
     ],
 )
 
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 2381d7a7d5..135a607ab9 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <algorithm>
+#include <cstring>
 #include <memory>
 #include <string>
 #include <vector>
@@ -44,6 +46,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/gpu/ir_emission_utils.h"
 #include "tensorflow/compiler/xla/service/gpu/ir_emitter_context.h"
 #include "tensorflow/compiler/xla/service/gpu/kernel_thunk.h"
+#include "tensorflow/compiler/xla/service/gpu/memset_thunk.h"
 #include "tensorflow/compiler/xla/service/gpu/parallel_loop_emitter.h"
 #include "tensorflow/compiler/xla/service/gpu/partition_assignment.h"
 #include "tensorflow/compiler/xla/service/gpu/sequential_thunk.h"
@@ -498,12 +501,11 @@ Status IrEmitterUnnested::HandleFusion(HloInstruction* fusion) {
     switch (root->opcode()) {
       case HloOpcode::kReduce: {
         VLOG(3) << "Emitting fused reduction to vector: " << fusion->ToString();
+        TF_ASSIGN_OR_RETURN(std::unique_ptr<Thunk> initializer_thunk,
+                            BuildInitializerThunk(fusion));
         std::vector<std::unique_ptr<Thunk>> thunks;
-        thunks.emplace_back(BuildKernelThunk(fusion));
-        TF_RETURN_IF_ERROR(EmitInitializer(
-            fusion, static_cast<KernelThunk*>(thunks.back().get())));
-        bindings_.UnbindAllLocalIrValues();
-        thunks.emplace_back(BuildKernelThunk(fusion));
+        thunks.push_back(std::move(initializer_thunk));
+        thunks.push_back(BuildKernelThunk(fusion));
         thunk_sequence_->emplace_back(
             MakeUnique<SequentialThunk>(std::move(thunks), fusion));
         std::vector<llvm_ir::IrArray> parameter_arrays;
@@ -1635,14 +1637,14 @@ Status IrEmitterUnnested::HandleReduce(HloInstruction* reduce) {
   if (IsReductionToVector(*reduce) &&
       // NVPTX backend can't do atomic cmpxchg any narrower than 32 bits
       32 <= primitive_util::BitWidth(reduce->shape().element_type())) {
+    TF_ASSIGN_OR_RETURN(std::unique_ptr<Thunk> initializer_thunk,
+                        BuildInitializerThunk(reduce));
     std::vector<std::unique_ptr<Thunk>> thunks;
-    thunks.emplace_back(BuildKernelThunk(reduce));
-    TF_RETURN_IF_ERROR(EmitInitializer(
-        reduce, static_cast<KernelThunk*>(thunks.back().get())));
-    bindings_.UnbindAllLocalIrValues();
-    thunks.emplace_back(BuildKernelThunk(reduce));
+    thunks.push_back(std::move(initializer_thunk));
+    thunks.push_back(BuildKernelThunk(reduce));
     thunk_sequence_->emplace_back(
         MakeUnique<SequentialThunk>(std::move(thunks), reduce));
+
     return EmitReductionToVector(
         reduce, input->shape(),
         [&](const llvm_ir::IrArray::Index& index) {
@@ -1706,16 +1708,13 @@ Status IrEmitterUnnested::HandleSelectAndScatter(
   CHECK_EQ(rank, ShapeUtil::Rank(source->shape()));
   CHECK_EQ(rank, window.dimensions_size());
 
-  {
-    std::vector<std::unique_ptr<Thunk>> thunks;
-    thunks.emplace_back(BuildKernelThunk(select_and_scatter));
-    TF_RETURN_IF_ERROR(EmitInitializer(
-        select_and_scatter, static_cast<KernelThunk*>(thunks.back().get())));
-    bindings_.UnbindAllLocalIrValues();
-    thunks.emplace_back(BuildKernelThunk(select_and_scatter));
-    thunk_sequence_->emplace_back(
-        MakeUnique<SequentialThunk>(std::move(thunks), select_and_scatter));
-  }
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<Thunk> initializer_thunk,
+                      BuildInitializerThunk(select_and_scatter));
+  std::vector<std::unique_ptr<Thunk>> thunks;
+  thunks.push_back(std::move(initializer_thunk));
+  thunks.push_back(BuildKernelThunk(select_and_scatter));
+  thunk_sequence_->emplace_back(
+      MakeUnique<SequentialThunk>(std::move(thunks), select_and_scatter));
 
   // TODO(b/31410564): Implement dilation rate for select-and-scatter.
   if (window_util::HasDilation(window)) {
@@ -2036,7 +2035,7 @@ Status IrEmitterUnnested::HandleGather(HloInstruction* gather) {
   return Unimplemented("Gather is not implemented on GPUs.");
 }
 
-std::unique_ptr<Thunk> IrEmitterUnnested::BuildKernelThunk(
+std::unique_ptr<KernelThunk> IrEmitterUnnested::BuildKernelThunk(
     const HloInstruction* inst) {
   const BufferAssignment& buffer_assn =
       ir_emitter_context_->buffer_assignment();
@@ -2260,37 +2259,87 @@ std::unique_ptr<Thunk> IrEmitterUnnested::BuildFftThunk(
                               /*output_shape=*/inst->shape(), inst);
 }
 
-Status IrEmitterUnnested::EmitInitializer(const HloInstruction* hlo,
-                                          KernelThunk* thunk) {
+StatusOr<std::unique_ptr<Thunk>> IrEmitterUnnested::BuildInitializerThunk(
+    const HloInstruction* hlo) {
   bool fused = HloOpcode::kFusion == hlo->opcode();
-
   const HloInstruction* inst = fused ? hlo->fused_expression_root() : hlo;
-  CHECK(inst->opcode() == HloOpcode::kSelectAndScatter ||
-        inst->opcode() == HloOpcode::kReduce);
-  const HloInstruction* init_value = nullptr;
-  switch (inst->opcode()) {
-    case HloOpcode::kSelectAndScatter:
-      init_value = inst->operand(2);
-      break;
-    case HloOpcode::kReduce:
-      init_value = inst->operand(1);
-      break;
-    default:
-      LOG(FATAL) << "Opcode " << inst->opcode()
-                 << " should not need an initializer.";
-  }
+  const HloInstruction* init_value = [&] {
+    switch (inst->opcode()) {
+      case HloOpcode::kSelectAndScatter:
+        return inst->operand(2);
+      case HloOpcode::kReduce:
+        return inst->operand(1);
+      default:
+        LOG(FATAL) << "Opcode " << inst->opcode()
+                   << " should not need an initializer.";
+    }
+  }();
 
   if (fused && init_value->opcode() == HloOpcode::kParameter) {
     init_value = hlo->operand(init_value->parameter_number());
   }
 
-  return EmitTargetElementLoopInThunk(
+  // In the common case, the initializer is a constant.  In this case, emit a
+  // device-memset call if we can.  Currently StreamExecutor only supports
+  // zeroing and 32-bit memsets.
+  if (init_value->IsConstant()) {
+    CHECK(ShapeUtil::IsScalar(init_value->shape()));
+    int64 num_bytes = ShapeUtil::ByteSizeOfElements(init_value->shape());
+    const auto& literal = init_value->literal();
+
+    // Are all the bytes of this scalar equal to 0?  If so, we can create a
+    // MemzeroThunk.
+    ArraySlice<uint8> literal_bytes(
+        reinterpret_cast<const uint8*>(literal.untyped_data()), num_bytes);
+    if (c_all_of(literal_bytes, [](uint8 byte) { return byte == 0; })) {
+      return {MakeUnique<MemzeroThunk>(GetAllocationSlice(*hlo), hlo)};
+    }
+
+    // If the literal is 8 or 16 bits wide, we can emit a 32-bit memset by
+    // repeating the literal 4 or 2 times, so long as the destination buffer is
+    // an even multiple of 32 bits long.
+    if ((num_bytes == 1 || num_bytes == 2) &&
+        ShapeUtil::ByteSizeOf(hlo->shape()) % 4 == 0) {
+      uint16 pattern16;
+      if (num_bytes == 1) {
+        uint8 b = literal_bytes.front();
+        pattern16 = uint16{b} | (uint16{b} << 8);
+      } else {
+        pattern16 = literal_bytes.front();
+      }
+      uint32 pattern32 = uint32{pattern16} | (uint32{pattern16} << 16);
+      return {MakeUnique<Memset32BitValueThunk>(pattern32,
+                                                GetAllocationSlice(*hlo), hlo)};
+    }
+
+    // If the literal is an even multiple of 32 bits wide, we can emit a 32-bit
+    // memset so long as all 32-bit words of the scalar are equal to each other.
+    if (num_bytes >= 4 && num_bytes % 4 == 0 &&
+        memcmp(literal_bytes.data(), literal_bytes.data() + 4,
+               literal_bytes.size() - 4) == 0) {
+      uint32 word;
+      memcpy(&word, literal_bytes.data(), sizeof(word));
+      return {MakeUnique<Memset32BitValueThunk>(word, GetAllocationSlice(*hlo),
+                                                hlo)};
+    }
+  }
+
+  // Otherwise fall back to our slow initializer code.
+  std::unique_ptr<KernelThunk> kernel_thunk = BuildKernelThunk(hlo);
+  TF_RETURN_IF_ERROR(EmitTargetElementLoopInThunk(
       *hlo,
       [=](const llvm_ir::IrArray::Index& index) {
         return GetIrArray(*init_value, *hlo)
             .EmitReadArrayElement(index, &ir_builder_);
       },
-      thunk);
+      kernel_thunk.get()));
+
+  // Clean up state left behind by emitting the loop above.  (This is normally
+  // done in IrEmitterUnnested::Postprocess().)
+  bindings_.UnbindAllLocalIrValues();
+
+  // Convert unique_ptr<KernelThunk> to StatusOr<unique_ptr<Thunk>>.
+  return {std::move(kernel_thunk)};
 }
 
 namespace {
diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
index b83a2337e2..66c62e2d2d 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.h
@@ -148,13 +148,10 @@ class IrEmitterUnnested : public IrEmitter {
       tensorflow::gtl::ArraySlice<int64> dimensions_to_reduce,
       HloComputation* reducer);
 
-  // Emits code to initialize buffer of `inst` in given `thunk`.
-  Status EmitInitializer(const HloInstruction* inst, KernelThunk* thunk);
-
   // Returns a KernelThunk that invokes the kernel emitted for `inst`. The
   // caller needs to make sure `inst` outlives the lifetime of the returned
   // Thunk object.
-  std::unique_ptr<Thunk> BuildKernelThunk(const HloInstruction* inst);
+  std::unique_ptr<KernelThunk> BuildKernelThunk(const HloInstruction* inst);
 
   // Returns a FftThunk that calls cuFFT to implement `inst`.
   std::unique_ptr<Thunk> BuildFftThunk(const HloInstruction* inst);
@@ -163,6 +160,11 @@ class IrEmitterUnnested : public IrEmitter {
   // to make sure `inst` outlives the lifetime of the returned Thunk object.
   std::unique_ptr<Thunk> BuildGemmThunk(const HloInstruction* inst);
 
+  // Returns a thunk that, given a reduce or select-and-scatter op, initializes
+  // its memory to the appropriate initial value.
+  StatusOr<std::unique_ptr<Thunk>> BuildInitializerThunk(
+      const HloInstruction* hlo);
+
   // Returns a thunk that calls host-to-device cuMemcpy to implement `inst`.
   std::unique_ptr<Thunk> BuildHostToDeviceCopyThunk(const HloInstruction* inst);
 
diff --git a/tensorflow/compiler/xla/service/gpu/memset_thunk.cc b/tensorflow/compiler/xla/service/gpu/memset_thunk.cc
new file mode 100644
index 0000000000..18e673542c
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/memset_thunk.cc
@@ -0,0 +1,39 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/gpu/memset_thunk.h"
+#include "tensorflow/stream_executor/stream_executor.h"
+
+namespace xla {
+namespace gpu {
+
+namespace se = ::perftools::gputools;
+
+Status MemzeroThunk::ExecuteOnStream(
+    const BufferAllocations& buffer_allocations, se::Stream* stream) {
+  se::DeviceMemoryBase dest_data = buffer_allocations.GetDeviceAddress(dest_);
+  stream->ThenMemZero(&dest_data, dest_data.size());
+  return Status::OK();
+}
+
+Status Memset32BitValueThunk::ExecuteOnStream(
+    const BufferAllocations& buffer_allocations, se::Stream* stream) {
+  se::DeviceMemoryBase dest_data = buffer_allocations.GetDeviceAddress(dest_);
+  stream->ThenMemset32(&dest_data, value_, dest_data.size());
+  return Status::OK();
+}
+
+}  // namespace gpu
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/service/gpu/memset_thunk.h b/tensorflow/compiler/xla/service/gpu/memset_thunk.h
new file mode 100644
index 0000000000..b4bb74d1dd
--- /dev/null
+++ b/tensorflow/compiler/xla/service/gpu/memset_thunk.h
@@ -0,0 +1,65 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_GPU_MEMSET_THUNK_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_GPU_MEMSET_THUNK_H_
+
+#include "tensorflow/compiler/xla/service/buffer_assignment.h"
+#include "tensorflow/compiler/xla/service/gpu/thunk.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/status.h"
+#include "tensorflow/stream_executor/stream_executor.h"
+
+// This file contains thunks that set a buffer's elements to a particular value.
+// This can be faster than emitting a kernel to set the elements.
+
+namespace xla {
+namespace gpu {
+
+// Thunk that zeroes out a given chunk of memory.
+class MemzeroThunk : public Thunk {
+ public:
+  explicit MemzeroThunk(const BufferAllocation::Slice& dest,
+                        const HloInstruction* hlo)
+      : Thunk(Kind::kMemzero, hlo), dest_(dest) {}
+
+  Status ExecuteOnStream(const BufferAllocations& buffer_allocations,
+                         perftools::gputools::Stream* stream) override;
+
+ private:
+  const BufferAllocation::Slice dest_;
+};
+
+// Thunk that sets a given chunk of memory to a particular 32-bit value.  The
+// destination chunk must have size divisible by 32 bits.
+class Memset32BitValueThunk : public Thunk {
+ public:
+  explicit Memset32BitValueThunk(uint32 value,
+                                 const BufferAllocation::Slice& dest,
+                                 const HloInstruction* hlo)
+      : Thunk(Kind::kMemset32BitValue, hlo), value_(value), dest_(dest) {}
+
+  Status ExecuteOnStream(const BufferAllocations& buffer_allocations,
+                         perftools::gputools::Stream* stream) override;
+
+ private:
+  uint32 value_;
+  const BufferAllocation::Slice dest_;
+};
+
+}  // namespace gpu
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_GPU_MEMSET_THUNK_H_
diff --git a/tensorflow/compiler/xla/service/gpu/thunk.h b/tensorflow/compiler/xla/service/gpu/thunk.h
index 2c3032d79b..9eea958d12 100644
--- a/tensorflow/compiler/xla/service/gpu/thunk.h
+++ b/tensorflow/compiler/xla/service/gpu/thunk.h
@@ -51,6 +51,8 @@ class Thunk {
     kGemm,
     kInfeed,
     kKernel,
+    kMemset32BitValue,
+    kMemzero,
     kSequential,
     kTuple,
     kWhile,
diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index 50d7b5074d..3a097a01ab 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -884,5 +884,47 @@ XLA_TEST_F(ReduceTest, ReduceOrPredR2_64x32_To_R1) {
   RunR2ToR1PredTest</*cols=32*/ 32>(/*and_reduce=false*/ false, /*rows=64*/ 64);
 }
 
+// Tests reductions with different initial values.  There's no test macro that
+// combines TYPED_TEST and TYPED_P, so we have to do it manually.
+class ReduceInitializerTest : public ReduceTest {
+ protected:
+  template <typename T>
+  void DoTest(T initializer, int num_elems) {
+    ComputationBuilder builder(client_, TestName());
+    Computation max_fn = CreateScalarMaxComputation(
+        primitive_util::NativeToPrimitiveType<T>(), &builder);
+
+    auto init = builder.ConstantR0<T>(initializer);
+    std::vector<T> input_arr(num_elems, std::numeric_limits<T>::lowest());
+    auto input_literal = Literal::CreateR1<T>(input_arr);
+    auto input_data =
+        client_->TransferToServer(*input_literal).ConsumeValueOrDie();
+    builder.Reduce(builder.Parameter(0, input_literal->shape(), "input"), init,
+                   max_fn, {0});
+
+    ComputeAndCompareR0<T>(&builder, initializer, {input_data.get()});
+  }
+};
+
+XLA_TEST_F(ReduceInitializerTest, U8Small) { DoTest<uint8>(42, 2); }
+
+XLA_TEST_F(ReduceInitializerTest, U8BigPowerOf2) { DoTest<uint8>(42, 4096); }
+
+XLA_TEST_F(ReduceInitializerTest, U8InitializerBigNonPowerOf2) {
+  DoTest<uint8>(42, 4095);
+}
+
+XLA_TEST_F(ReduceInitializerTest, U64InitializerZero) {
+  DoTest<uint64>(0, 1024);
+}
+
+XLA_TEST_F(ReduceInitializerTest, U64InitializerOne) {
+  DoTest<uint64>(1, 1024);
+}
+
+XLA_TEST_F(ReduceInitializerTest, U64InitializerBigValue) {
+  DoTest<uint64>(1234556789123, 1024);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 5a1fddfdf20bd978963050c24ac71d7937071ca5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 07:34:28 -0700
Subject: [PATCH 290/960] Install documentation: adds note for virtual env with
 fish shell

PiperOrigin-RevId: 189904848
---
 tensorflow/docs_src/install/install_linux.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 88ceca3cda..2741b61bb2 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -131,7 +131,8 @@ Take the following steps to install TensorFlow with Virtualenv:
      commands:
 
      <pre>$ <b>source ~/tensorflow/bin/activate</b> # bash, sh, ksh, or zsh
-    $ <b>source ~/tensorflow/bin/activate.csh</b>  # csh or tcsh</pre>
+    $ <b>source ~/tensorflow/bin/activate.csh</b>  # csh or tcsh
+    $ <b>. ~/tensorflow/bin/activate.fish</b>  # fish</pre>
 
      The preceding <tt>source</tt> command should change your prompt
      to the following:
-- 
GitLab


From 2a9387d771f4ba99ba09b197ede82a6ea9671af0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 08:21:28 -0700
Subject: [PATCH 291/960] Automated g4 rollback of changelist 189888700

PiperOrigin-RevId: 189910239
---
 tensorflow/compiler/jit/xla_device_context.cc | 12 ++--
 tensorflow/compiler/jit/xla_launch_util.cc    | 14 +---
 tensorflow/compiler/tests/BUILD               | 20 ------
 tensorflow/compiler/tests/oom_test.py         | 72 -------------------
 4 files changed, 7 insertions(+), 111 deletions(-)
 delete mode 100644 tensorflow/compiler/tests/oom_test.py

diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index 93e0dbb9b9..88f7c15f0b 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -36,14 +36,10 @@ XlaDeviceAllocator::~XlaDeviceAllocator() = default;
 string XlaDeviceAllocator::Name() { return "xla"; }
 
 void* XlaDeviceAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
-  auto status_or_dmem = backend_->memory_allocator()->Allocate(
-      device_ordinal_, num_bytes, /*retry_on_failure=*/false);
-  if (!status_or_dmem.status().ok()) {
-    LOG(ERROR) << "Failed to allocate memory: "
-               << status_or_dmem.status().ToString();
-    return nullptr;
-  }
-  se::DeviceMemoryBase dmem = status_or_dmem.ValueOrDie();
+  se::DeviceMemoryBase dmem =
+      backend_->memory_allocator()
+          ->Allocate(device_ordinal_, num_bytes, /*retry_on_failure=*/false)
+          .ValueOrDie();
   VLOG(2) << "Allocated XLA device tensor " << dmem.opaque() << "(" << num_bytes
           << ")";
   return dmem.opaque();
diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index 21f58c8310..bb7316c60c 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -56,20 +56,12 @@ XlaAllocator::XlaAllocator(const gpu::Platform* platform,
                            OpKernelContext* op_context)
     : xla::DeviceMemoryAllocator(platform), op_context_(op_context) {}
 
-XlaAllocator::~XlaAllocator() {
-  for (void* ptr : allocated_) {
-    op_context_->device()->GetAllocator({})->DeallocateRaw(ptr);
-  }
-}
+XlaAllocator::~XlaAllocator() { CHECK(allocated_.empty()); }
 
 xla::StatusOr<gpu::DeviceMemoryBase> XlaAllocator::Allocate(
     int device_ordinal, uint64 size, bool retry_on_failure) {
   void* data = op_context_->device()->GetAllocator({})->AllocateRaw(
       Allocator::kAllocatorAlignment, size);
-  if (!data) {
-    return errors::ResourceExhausted(
-        "OOM when allocating temporary tensor with size ", size);
-  }
   allocated_.insert(data);
   return gpu::DeviceMemoryBase(data, size);
 }
@@ -190,8 +182,8 @@ void XlaComputationLaunchContext::PopulateOutputs(
         // Copy host -> device. (Empty tensors don't have backing buffers.)
         VLOG(1) << "Constant output tensor on device";
 
-        OP_REQUIRES_OK(
-            ctx, ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
+        TF_CHECK_OK(
+            ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
 
         const void* src_ptr = DMAHelper::base(&const_tensor);
         void* dst_ptr = DMAHelper::base(output_tensor);
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 26d4ca0c13..bbb6089ea8 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -191,26 +191,6 @@ tf_xla_py_test(
     ],
 )
 
-tf_xla_py_test(
-    name = "oom_test",
-    size = "medium",
-    srcs = ["oom_test.py"],
-    disabled_backends = [
-        "cpu",
-        "cpu_ondemand",
-    ],
-    deps = [
-        ":xla_test",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:array_ops_gen",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:gradient_checker",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-    ],
-)
-
 tf_xla_py_test(
     name = "conv2d_test",
     size = "medium",
diff --git a/tensorflow/compiler/tests/oom_test.py b/tensorflow/compiler/tests/oom_test.py
deleted file mode 100644
index 66be0d61d0..0000000000
--- a/tensorflow/compiler/tests/oom_test.py
+++ /dev/null
@@ -1,72 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Functional tests for out-of-memory conditions."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.compiler.tests import xla_test
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.platform import googletest
-
-
-class OutOfMemoryTest(xla_test.XLATestCase):
-
-  def testOutputOutOfMemory(self):
-    """Allocates tensors until out of memory.
-
-    Generates a large rank-1 tensor. The tensor is an output of an XLA
-    computation, not constant.
-
-    Check that a ResourceExhaustedError is raised and can be caught.
-    """
-    size = 5e8
-    with self.test_session():
-      # Force the compiled code to not be constant by feeding in an addend.
-      p = array_ops.placeholder(dtypes.float32, shape=[])
-      with self.test_scope():
-        # Create a large R1 tensor.
-        c = array_ops.zeros([size]) + p
-
-        self.assertRaises(
-            errors.ResourceExhaustedError, lambda: c.eval(feed_dict={p: 1.0}))
-
-  def testConstantOutOfMemory(self):
-    """Allocates constant tensors until out of memory.
-
-    Generates a large rank-1 tensor and a small rank-1 tensor. The tensors are
-    constant outputs of an XLA computation, not variable.
-
-    Multiple constant outputs are created, one small, one large. The small
-    tensor will have already been allocated when the large tensor fails.
-
-    Check that a ResourceExhaustedError is raised and can be caught.
-    """
-    size = 5e8
-    with self.test_session() as sess:
-      with self.test_scope():
-        # Create two R1 tensors, size 5 and size n.
-        b = array_ops.zeros([5])
-        c = array_ops.zeros([size])
-        e = control_flow_ops.tuple([b, c])
-        self.assertRaises(errors.ResourceExhaustedError, lambda: sess.run(e))
-
-
-if __name__ == "__main__":
-  googletest.main()
-- 
GitLab


From 56054e42a474a527f12f4d8d0b1f37eb1efd189d Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 21 Mar 2018 08:25:34 -0700
Subject: [PATCH 292/960] [tf.contrib CriticalSection] Avoid deadlocks using
 additional control dependencies on the lock op.

PiperOrigin-RevId: 189910726
---
 .../python/ops/critical_section_ops.py        | 203 ++++++++++++------
 .../python/ops/critical_section_test.py       | 143 +++++++++++-
 2 files changed, 277 insertions(+), 69 deletions(-)

diff --git a/tensorflow/contrib/framework/python/ops/critical_section_ops.py b/tensorflow/contrib/framework/python/ops/critical_section_ops.py
index cc19372acf..1893d7b466 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_ops.py
+++ b/tensorflow/contrib/framework/python/ops/critical_section_ops.py
@@ -24,10 +24,8 @@ import collections
 # from tensorflow.core.protobuf import critical_section_pb2
 
 from tensorflow.python.eager import context
-from tensorflow.python.eager import function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import gen_resource_variable_ops
@@ -48,6 +46,26 @@ class _ExecutionSignature(
   pass
 
 
+def _identity(x):
+  """Identity op that recognizes `TensorArray`, `Operation`, and `Tensor`."""
+  if isinstance(x, tensor_array_ops.TensorArray):
+    return x.identity()
+  elif isinstance(x, ops.Operation):
+    return control_flow_ops.group(x)
+  elif context.executing_eagerly() and x is None:
+    return None
+  else:
+    return array_ops.identity(x)
+
+
+def _get_colocation(op):
+  """Get colocation symbol from op, if any."""
+  try:
+    return op.get_attr("_class")
+  except ValueError:
+    return None
+
+
 class CriticalSection(object):
   """Critical section.
 
@@ -180,8 +198,8 @@ class CriticalSection(object):
       The tensors returned from `fn(*args, **kwargs)`.
 
     Raises:
-      ValueError: If `fn` attempts to use this `CriticalSection` in any nested
-        way.
+      ValueError: If `fn` attempts to lock this `CriticalSection` in any nested
+        or lazy way that may cause a deadlock.
       ValueError: If `exclusive_resource_access` is not provided (is `True`) and
         another `CriticalSection` has an execution requesting the same
         resources as in `*args`, `**kwargs`, and any additionaly captured
@@ -193,69 +211,52 @@ class CriticalSection(object):
     exclusive_resource_access = kwargs.pop("exclusive_resource_access", True)
 
     with ops.name_scope(name, "critical_section_execute", []):
-      lock = gen_resource_variable_ops.mutex_lock(self._handle)
-
-      with ops.control_dependencies([lock]):
-        c_known_ops = set()
-        c_captured_tensors = set()
 
-        def add_op_internal(op):
-          c_known_ops.add(op)
-          for i in op.inputs:
-            if i.op not in c_known_ops:
-              c_captured_tensors.add(i)
+      # Ensure that mutex locking only happens *after* all args and
+      # kwargs have been executed.  This avoids certain types of deadlocks.
+      lock = gen_resource_variable_ops.mutex_lock(self._handle)
 
-        c = function.HelperContext(add_op_internal)
-        with c:
+      if not context.executing_eagerly():
+        # NOTE(ebrevdo): This is to ensure we don't pick up spurious
+        # Operations created by other threads.
+        with ops.get_default_graph()._lock:  # pylint: disable=protected-access
+          existing_ops = ops.get_default_graph().get_operations()
+          with ops.control_dependencies([lock]):
+            r = fn(*args, **kwargs)
+          # TODO(ebrevdo): If creating critical sections in a python loop, this
+          # makes graph creation time quadratic.  Revisit if this
+          # becomes a problem.
+          created_ops = (set(ops.get_default_graph().get_operations())
+                         .difference(existing_ops))
+      else:
+        with ops.control_dependencies([lock]):
           r = fn(*args, **kwargs)
 
-        resource_inputs = set([
-            x for x in
-            list(nest.flatten(args)) + nest.flatten(kwargs.values()) +
-            list(c_captured_tensors)
-            if tensor_util.is_tensor(x) and x.dtype == dtypes.resource])
-
-      if self._handle in resource_inputs:
-        raise ValueError("The function fn attempts to access the "
-                         "CriticalSection in which it would be running.  "
-                         "This is illegal and would cause deadlocks.  "
-                         "CriticalSection: %s." % self._handle)
-
       if not context.executing_eagerly():
-        # Collections and op introspection does not work in eager
-        # mode.  This is generally ok; since eager mode (as of
-        # writing) executes sequentially anyway.
-        for sg in ops.get_collection(CRITICAL_SECTION_EXECUTIONS):
-          sg_handle_name = ops.convert_to_tensor(sg.handle).name
-          self_handle_name = ops.convert_to_tensor(self._handle).name
-          if sg_handle_name == self_handle_name:
-            # Other executions in the same critical section are allowed.
-            continue
-          if not (exclusive_resource_access or sg.exclusive_resource_access):
-            # Neither execution requested exclusive access.
-            continue
-          resource_intersection = resource_inputs.intersection(sg.resources)
-          if resource_intersection:
-            raise ValueError(
-                "This execution would access resources: %s.  Either this "
-                "lock (CriticalSection: %s) or lock '%s' "
-                "(CriticalSection: %s) requested exclusive resource access "
-                "of this resource.  Did you mean to call execute with keyword "
-                "argument exclusive_resource_access=False?" %
-                (list(resource_intersection), self._handle.name,
-                 sg.op.name, sg.handle.name))
-
-      def identity(x):  # pylint: disable=invalid-name
-        if isinstance(x, tensor_array_ops.TensorArray):
-          return x.identity()
-        elif isinstance(x, ops.Operation):
-          return control_flow_ops.group(x)
-        elif context.executing_eagerly() and x is None:
-          return None
-        else:
-          return array_ops.identity(x)
-
-      r_flat = [identity(x) for x in nest.flatten(r)]
+        self._add_control_dependencies_to_lock(created_ops, lock.op)
+
+        # captured_resources is a list of resources that are directly
+        # accessed only by ops created during fn(), not by any
+        # ancestors of those ops in the graph.
+        captured_resources = set([
+            input_ for op in created_ops
+            for input_ in op.inputs
+            if input_.dtype == dtypes.resource
+        ])
+
+        # NOTE(ebrevdo): The only time self._is_self_handle() is True
+        # in this call is if one of the recently created ops, within
+        # the execute(), themselves attempt to access the
+        # CriticalSection.  This will cause a deadlock.
+        if any(self._is_self_handle(x) for x in captured_resources):
+          raise ValueError("The function fn attempts to directly access the "
+                           "CriticalSection in which it would be running.  "
+                           "This is illegal and would cause deadlocks.")
+
+        self._check_multiple_access_to_resources(
+            captured_resources, exclusive_resource_access)
+
+      r_flat = [_identity(x) for x in nest.flatten(r)]
 
       with ops.control_dependencies(r_flat):
         # The identity must run on the same machine as self._handle
@@ -268,23 +269,93 @@ class CriticalSection(object):
 
         # Make sure that if any element of r is accessed, all of
         # them are executed together.
-        r = nest.pack_sequence_as(
-            r, control_flow_ops.tuple(nest.flatten(r)))
+        r = nest.pack_sequence_as(r, control_flow_ops.tuple(nest.flatten(r)))
 
       with ops.control_dependencies([ensure_lock_exists]):
-        outputs = nest.map_structure(identity, r)
+        outputs = nest.map_structure(_identity, r)
 
       if not context.executing_eagerly():
         signature = _ExecutionSignature(
             op=lock.op,
             handle=self._handle,
-            resources=list(resource_inputs),
+            resources=list(captured_resources),
             exclusive_resource_access=exclusive_resource_access)
         ops.add_to_collections(
             CRITICAL_SECTION_EXECUTIONS, signature)
 
       return outputs
 
+  def _add_control_dependencies_to_lock(self, created_ops, lock_op):
+    """To avoid deadlocks, all args must be executed before lock_op."""
+    # Get all arguments (explicit and captured) of all ops created by fn().
+    all_args = set([input_.op for op in created_ops for input_ in op.inputs])
+    all_args.update(
+        input_op for op in created_ops for input_op in op.control_inputs)
+    # Unfortunately, we can't use sets throughout because TF seems to
+    # create new Operation objects for the same op sometimes; and we
+    # can't rely on id(op).
+
+    # pylint: disable=protected-access
+    all_args_dict = dict((op._id, op) for op in all_args)
+
+    # Remove ops created within fn, or that lock_op already has a
+    # control dependency on.  Also remove a possible self-loop.
+    for op in created_ops:
+      all_args_dict.pop(op._id, None)
+    for op in lock_op.control_inputs:
+      all_args_dict.pop(op._id, None)
+    for input_ in lock_op.inputs:
+      all_args_dict.pop(input_.op._id, None)
+    all_args_dict.pop(lock_op._id, None)
+
+    lock_op._add_control_inputs(all_args_dict.values())
+    # pylint: enable=protected-access
+
+  def _is_self_handle(self, x):
+    """Check if the tensor `x` is the same Mutex as `self._handle`."""
+    return (x.op.type == "MutexV2"
+            # blank shared_name means the op will create a unique one.
+            and x.op.get_attr("shared_name")
+            and (x.op.get_attr("shared_name") ==
+                 self._handle.op.get_attr("shared_name"))
+            and (x.op.device == self._handle.op.device
+                 or _get_colocation(x.op) == _get_colocation(self._handle.op)))
+
+  def _check_multiple_access_to_resources(
+      self, captured_resources, exclusive_resource_access):
+    """Raise if captured_resources are accessed by another CriticalSection.
+
+    Args:
+      captured_resources: Set of tensors of type resource.
+      exclusive_resource_access: Whether this execution requires exclusive
+        resource access.
+
+    Raises:
+      ValueError: If any tensors in `captured_resources` are also accessed
+        by another `CriticalSection`, and at least one of them requires
+        exclusive resource access.
+    """
+    # Collections and op introspection does not work in eager
+    # mode.  This is generally ok; since eager mode (as of
+    # writing) executes sequentially anyway.
+    for sg in ops.get_collection(CRITICAL_SECTION_EXECUTIONS):
+      if self._is_self_handle(sg.handle):
+        # Other executions in the same critical section are allowed.
+        continue
+      if not (exclusive_resource_access or sg.exclusive_resource_access):
+        # Neither execution requested exclusive access.
+        continue
+      resource_intersection = captured_resources.intersection(sg.resources)
+      if resource_intersection:
+        raise ValueError(
+            "This execution would access resources: %s.  Either this "
+            "lock (CriticalSection: %s) or lock '%s' "
+            "(CriticalSection: %s) requested exclusive resource access "
+            "of this resource.  Did you mean to call execute with keyword "
+            "argument exclusive_resource_access=False?" %
+            (list(resource_intersection), self._handle.name,
+             sg.op.name, sg.handle.name))
+
   # TODO(ebrevdo): Re-enable once CriticalSection is in core.
 
   # def to_proto(self, export_scope=None):
diff --git a/tensorflow/contrib/framework/python/ops/critical_section_test.py b/tensorflow/contrib/framework/python/ops/critical_section_test.py
index c916592ce1..e24140bd72 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_test.py
+++ b/tensorflow/contrib/framework/python/ops/critical_section_test.py
@@ -25,6 +25,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
 # TODO(ebrevdo): Re-enable once CriticalSection is in core.
 # from tensorflow.python.training import saver as saver_lib
 
@@ -37,7 +38,7 @@ class CriticalSectionTest(test.TestCase):
     v = resource_variable_ops.ResourceVariable(0.0, name="v")
 
     def fn(a, b):
-      c = v.read_value()
+      c = v.value()
       with ops.control_dependencies([c]):
         nv = v.assign_add(a * b)
         with ops.control_dependencies([nv]):
@@ -140,15 +141,151 @@ class CriticalSectionTest(test.TestCase):
          ops.get_collection(critical_section_ops.CRITICAL_SECTION_EXECUTIONS)])
 
   def testRecursiveCriticalSectionAccessIsIllegal(self):
+    # This does not work properly in eager mode.  Eager users will
+    # just hit a deadlock if they do this.  But at least it'll be easier
+    # to debug.
+    cs = critical_section_ops.CriticalSection()
+    def fn(x):
+      return cs.execute(lambda y: y + 1, x)
+    with self.assertRaisesRegexp(
+        ValueError,
+        r"attempts to directly access the CriticalSection in which it "
+        r"would be running"):
+      cs.execute(fn, 1.0)
+
+  def testRecursiveCriticalSectionAccessViaCapturedTensorIsProtected(self):
+    # This one is subtle; and we're being overly cautious here.  The
+    # deadlock we are ensuring we catch is:
+    #
+    # to_capture = CS[lambda x: x + 1](1.0)
+    # deadlocked = CS[lambda x: x + to_capture](1.0)
+    #
+    # This would have caused a deadlock because executing `deadlocked` will
+    # lock the mutex on CS; but then due to dependencies, will attempt
+    # to compute `to_capture`.  This computation requires locking CS,
+    # but that is not possible now because CS is already locked by
+    # `deadlocked`.
+    #
+    # We check that CriticalSection.execute properly inserts new
+    # control dependencies to its lock to ensure all captured
+    # operations are finished before anything runs within the critical section.
+    cs = critical_section_ops.CriticalSection(shared_name="cs")
+    fn = array_ops.identity
+    to_capture = cs.execute(fn, 1.0)
+    fn_captures = lambda x: x + to_capture
+    to_capture_too = array_ops.identity(to_capture)
+
+    ex_0 = cs.execute(fn_captures, 1.0)
+
+    with ops.control_dependencies([to_capture]):
+      # This is OK because to_capture will execute before this next call
+      ex_1 = cs.execute(fn_captures, 1.0)
+
+    dependency = array_ops.identity(to_capture)
+
+    fn_captures_dependency = lambda x: x + dependency
+
+    ex_2 = cs.execute(fn_captures_dependency, 1.0)
+
+    with ops.control_dependencies([to_capture_too]):
+      ex_3 = cs.execute(fn_captures_dependency, 1.0)
+
+    # Ensure there's no actual deadlock on to_execute.
+    self.assertEquals(2.0, self.evaluate(ex_0))
+    self.assertEquals(2.0, self.evaluate(ex_1))
+    self.assertEquals(2.0, self.evaluate(ex_2))
+    self.assertEquals(2.0, self.evaluate(ex_3))
+
+  def testRecursiveCriticalSectionAccessWithinLoopIsProtected(self):
+    cs = critical_section_ops.CriticalSection(shared_name="cs")
+
+    def body_implicit_capture(i, j):
+      # This would have caused a deadlock if not for logic in execute
+      # that inserts additional control dependencies onto the lock op:
+      #   * Loop body argument j is captured by fn()
+      #   * i is running in parallel to move forward the execution
+      #   * j is not being checked by the predicate function
+      #   * output of cs.execute() is returned as next j.
+      fn = lambda: j + 1
+      return (i + 1, cs.execute(fn))
+
+    (i_n, j_n) = control_flow_ops.while_loop(
+        lambda i, _: i < 1000,
+        body_implicit_capture,
+        [0, 0],
+        parallel_iterations=25)
+    logging.warn(
+        "\n==============\nRunning "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_implicit_capture'\n"
+        "==============\n")
+    self.assertEquals((1000, 1000), self.evaluate((i_n, j_n)))
+    logging.warn(
+        "\n==============\nSuccessfully finished running "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_implicit_capture'\n"
+        "==============\n")
+
+    def body_implicit_capture_protected(i, j):
+      # This version is ok because we manually add a control
+      # dependency on j, which is an argument to the while_loop body
+      # and captured by fn.
+      fn = lambda: j + 1
+      with ops.control_dependencies([j]):
+        return (i + 1, cs.execute(fn))
+
+    (i_n, j_n) = control_flow_ops.while_loop(
+        lambda i, _: i < 1000,
+        body_implicit_capture_protected,
+        [0, 0],
+        parallel_iterations=25)
+    logging.warn(
+        "\n==============\nRunning "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_implicit_capture_protected'\n"
+        "==============\n")
+    self.assertEquals((1000, 1000), self.evaluate((i_n, j_n)))
+    logging.warn(
+        "\n==============\nSuccessfully finished running "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_implicit_capture_protected'\n"
+        "==============\n")
+
+    def body_args_capture(i, j):
+      # This version is ok because j is an argument to fn and we can
+      # ensure there's a control dependency on j.
+      fn = lambda x: x + 1
+      return (i + 1, cs.execute(fn, j))
+
+    (i_n, j_n) = control_flow_ops.while_loop(
+        lambda i, _: i < 1000,
+        body_args_capture,
+        [0, 0],
+        parallel_iterations=25)
+    logging.warn(
+        "\n==============\nRunning "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_args_capture'\n"
+        "==============\n")
+    self.assertEquals((1000, 1000), self.evaluate((i_n, j_n)))
+    logging.warn(
+        "\n==============\nSuccessfully finished running "
+        "'testRecursiveCriticalSectionAccessWithinLoopDoesNotDeadlock "
+        "body_args_capture'\n"
+        "==============\n")
+
+  def testRecursiveCriticalSectionAccessIsIllegalSameSharedName(self):
     # This does not work properly in eager mode.  Eager users will
     # just hit a deadlock if they do this.  But at least it'll be easier
     # to debug.
     cs = critical_section_ops.CriticalSection(shared_name="cs")
+    cs_same = critical_section_ops.CriticalSection(shared_name="cs")
     def fn(x):
-      return cs.execute(lambda x: x+1, x)
+      return cs_same.execute(lambda x: x+1, x)
     with self.assertRaisesRegexp(
         ValueError,
-        r"attempts to access the CriticalSection in which it would be running"):
+        r"attempts to directly access the CriticalSection in which it "
+        r"would be running"):
       cs.execute(fn, 1.0)
 
   def testMultipleCSExecutionsRequestSameResource(self):
-- 
GitLab


From 73cea1b095c0211b532663ea5edf0dc50ff5a448 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 21 Mar 2018 08:40:35 -0700
Subject: [PATCH 293/960] More accurate shape inference for TensorArrayGatherV3
 and TensorArrayScatterV3

PiperOrigin-RevId: 189912762
---
 tensorflow/core/ops/data_flow_ops.cc | 37 +++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/ops/data_flow_ops.cc b/tensorflow/core/ops/data_flow_ops.cc
index 4f946fb3ca..3112f35da4 100644
--- a/tensorflow/core/ops/data_flow_ops.cc
+++ b/tensorflow/core/ops/data_flow_ops.cc
@@ -668,13 +668,31 @@ REGISTER_OP("TensorArrayGatherV3")
     .Attr("dtype: type")
     .Attr("element_shape: shape = { unknown_rank: true }")
     .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle indices;
       ShapeHandle unused;
       DimensionHandle unused_dim;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &indices));
       TF_RETURN_IF_ERROR(c->WithValue(c->Dim(c->input(0), 0), 2, &unused_dim));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused));
-      return shape_inference::UnknownShape(c);
+      auto shapes = c->input_handle_shapes_and_types(0);
+      if (shapes != nullptr && !shapes->empty()) {
+        ShapeHandle tensor_shape = shapes->at(0).shape;
+        ShapeHandle output_shape;
+        TF_RETURN_IF_ERROR(
+            c->Concatenate(indices, tensor_shape, &output_shape));
+        c->set_output(0, output_shape);
+        return Status::OK();
+      } else {
+        PartialTensorShape p;
+        TF_RETURN_IF_ERROR(c->GetAttr("element_shape", &p));
+        ShapeHandle s;
+        TF_RETURN_IF_ERROR(c->MakeShapeFromPartialTensorShape(p, &s));
+        ShapeHandle output_shape;
+        TF_RETURN_IF_ERROR(c->Concatenate(indices, s, &output_shape));
+        c->set_output(0, output_shape);
+        return Status::OK();
+      }
     });
 
 REGISTER_OP("TensorArrayScatterV3")
@@ -685,12 +703,25 @@ REGISTER_OP("TensorArrayScatterV3")
     .Output("flow_out: float")
     .Attr("T: type")
     .SetShapeFn([](InferenceContext* c) {
+      ShapeHandle indices;
       ShapeHandle unused;
       DimensionHandle unused_dim;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &unused));
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &unused));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &indices));
       TF_RETURN_IF_ERROR(c->WithValue(c->Dim(c->input(0), 0), 2, &unused_dim));
       TF_RETURN_IF_ERROR(c->WithRank(c->input(3), 0, &unused));
+      ShapeHandle value_shape;
+      // Assert that the length of the indices tensor is equal to the first
+      // dimension of the value tensor.
+      TF_RETURN_IF_ERROR(
+          c->MergePrefix(c->input(2), indices, &value_shape, &indices));
+      auto shapes = c->input_handle_shapes_and_types(0);
+      if (shapes != nullptr && !shapes->empty()) {
+        ShapeHandle tensor_shape = shapes->at(0).shape;
+        ShapeHandle fed_shape;
+        TF_RETURN_IF_ERROR(c->Subshape(value_shape, 1, &fed_shape));
+        TF_RETURN_IF_ERROR(c->Merge(tensor_shape, fed_shape, &fed_shape));
+      }
       return shape_inference::ScalarShape(c);
     });
 
-- 
GitLab


From 8337e1778a485102494f99d1924dda546daef4a9 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 21 Mar 2018 08:45:19 -0700
Subject: [PATCH 294/960] Refactor pruning code to support custom node rewrites
 for feeds and fetches.

PiperOrigin-RevId: 189913309
---
 .../common_runtime/graph_execution_state.cc   |  49 ++-
 .../common_runtime/graph_execution_state.h    |   5 +
 tensorflow/core/graph/subgraph.cc             | 354 ++++++++++--------
 tensorflow/core/graph/subgraph.h              | 108 +++++-
 4 files changed, 333 insertions(+), 183 deletions(-)

diff --git a/tensorflow/core/common_runtime/graph_execution_state.cc b/tensorflow/core/common_runtime/graph_execution_state.cc
index f5e3d78242..2f17af273f 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.cc
+++ b/tensorflow/core/common_runtime/graph_execution_state.cc
@@ -237,6 +237,42 @@ void GraphExecutionState::RestoreStatefulNodes(Graph* graph) {
   }
 }
 
+Status GraphExecutionState::PruneGraph(
+    const BuildGraphOptions& options, Graph* graph,
+    subgraph::RewriteGraphMetadata* out_rewrite_metadata) {
+  std::vector<std::unique_ptr<subgraph::PruneRewrite>> feed_rewrites;
+  feed_rewrites.reserve(options.callable_options.feed_size());
+  std::vector<std::unique_ptr<subgraph::PruneRewrite>> fetch_rewrites;
+  fetch_rewrites.reserve(options.callable_options.fetch_size());
+  const DeviceAttributes* device_info =
+      &device_set_->client_device()->attributes();
+  if (options.use_function_convention) {
+    for (int i = 0; i < options.callable_options.feed_size(); ++i) {
+      feed_rewrites.emplace_back(new subgraph::ArgFeedRewrite(
+          &options.callable_options.feed(i), device_info, i));
+    }
+    for (int i = 0; i < options.callable_options.fetch_size(); ++i) {
+      fetch_rewrites.emplace_back(new subgraph::RetvalFetchRewrite(
+          &options.callable_options.fetch(i), device_info, i));
+    }
+  } else {
+    for (const string& feed : options.callable_options.feed()) {
+      feed_rewrites.emplace_back(
+          new subgraph::RecvFeedRewrite(&feed, device_info));
+    }
+    for (const string& fetch : options.callable_options.fetch()) {
+      fetch_rewrites.emplace_back(
+          new subgraph::SendFetchRewrite(&fetch, device_info));
+    }
+  }
+  std::vector<string> target_node_names(
+      options.callable_options.target().begin(),
+      options.callable_options.target().end());
+  return subgraph::RewriteGraphForExecution(graph, feed_rewrites,
+                                            fetch_rewrites, target_node_names,
+                                            out_rewrite_metadata);
+}
+
 Status GraphExecutionState::InitBaseGraph(const BuildGraphOptions& options) {
   const GraphDef* graph_def = &original_graph_def_;
 
@@ -251,10 +287,8 @@ Status GraphExecutionState::InitBaseGraph(const BuildGraphOptions& options) {
       session_options_->config.graph_options().place_pruned_graph()) {
     // Rewrite the graph before placement.
     rewrite_metadata_.reset(new subgraph::RewriteGraphMetadata);
-    TF_RETURN_IF_ERROR(subgraph::RewriteGraphForExecution(
-        new_graph.get(), options.callable_options,
-        device_set_->client_device()->attributes(),
-        options.use_function_convention, rewrite_metadata_.get()));
+    TF_RETURN_IF_ERROR(
+        PruneGraph(options, new_graph.get(), rewrite_metadata_.get()));
   }
 
   // Save stateful placements before placing.
@@ -404,12 +438,7 @@ Status GraphExecutionState::BuildGraph(const BuildGraphOptions& options,
   subgraph::RewriteGraphMetadata rewrite_metadata;
   if (session_options_ == nullptr ||
       !session_options_->config.graph_options().place_pruned_graph()) {
-    // Extract the subset of the graph that needs to be run, adding feed/fetch
-    // ops as needed.
-    TF_RETURN_IF_ERROR(subgraph::RewriteGraphForExecution(
-        ng.get(), options.callable_options,
-        device_set_->client_device()->attributes(),
-        options.use_function_convention, &rewrite_metadata));
+    TF_RETURN_IF_ERROR(PruneGraph(options, ng.get(), &rewrite_metadata));
   } else {
     // This GraphExecutionState represents a graph that was
     // pruned when this was constructed, so we copy the metadata from
diff --git a/tensorflow/core/common_runtime/graph_execution_state.h b/tensorflow/core/common_runtime/graph_execution_state.h
index 2312e1a89f..2154ef5bd3 100644
--- a/tensorflow/core/common_runtime/graph_execution_state.h
+++ b/tensorflow/core/common_runtime/graph_execution_state.h
@@ -177,6 +177,11 @@ class GraphExecutionState {
   void SaveStatefulNodes(Graph* graph);
   void RestoreStatefulNodes(Graph* graph);
 
+  // Extract the subset of the graph that needs to be run, adding feed/fetch
+  // ops as needed.
+  Status PruneGraph(const BuildGraphOptions& options, Graph* graph,
+                    subgraph::RewriteGraphMetadata* out_rewrite_metadata);
+
   Status OptimizeGraph(const BuildGraphOptions& options,
                        std::unique_ptr<Graph>* optimized_graph);
 
diff --git a/tensorflow/core/graph/subgraph.cc b/tensorflow/core/graph/subgraph.cc
index ca93d049d0..193cf88aed 100644
--- a/tensorflow/core/graph/subgraph.cc
+++ b/tensorflow/core/graph/subgraph.cc
@@ -28,13 +28,13 @@ limitations under the License.
 #include "tensorflow/core/graph/algorithm.h"
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
-#include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
+namespace subgraph {
 
 // ----------------------------------------------------------------------------
 // Subgraph construction-related routines
@@ -44,6 +44,8 @@ namespace tensorflow {
 
 namespace {
 
+typedef std::unordered_map<StringPiece, Node*, StringPieceHasher> NameIndex;
+
 // Rewrite graph by replacing the output tensors specified in
 // "fed_outputs" with special feed nodes for each specified output
 // tensor, and removing any nodes that are now disconnected from the
@@ -53,59 +55,33 @@ namespace {
 // Return true on success.  On error, return false and sets *error to
 // an appropriate error message (and *g is left in an indeterminate
 // state).
-static Status FeedInputs(Graph* g, const DeviceAttributes& device_info,
-                         const gtl::ArraySlice<string>& fed_outputs,
-                         bool use_function_convention,
-                         subgraph::NameIndex* name_index,
-                         DataTypeVector* out_feed_types) {
+Status FeedInputs(
+    Graph* g, const std::vector<std::unique_ptr<PruneRewrite>>& feed_rewrites,
+    NameIndex* name_index, DataTypeVector* out_feed_types) {
   out_feed_types->clear();
-  out_feed_types->reserve(fed_outputs.size());
-  for (size_t i = 0; i < fed_outputs.size(); ++i) {
-    const string& t = fed_outputs[i];
+  out_feed_types->reserve(feed_rewrites.size());
+  for (size_t i = 0; i < feed_rewrites.size(); ++i) {
+    const string& t = feed_rewrites[i]->endpoint_name();
     TensorId id(ParseTensorName(t));
 
     auto iter = name_index->find(id.first);
     if (iter == name_index->end()) {
       return errors::NotFound("FeedInputs: unable to find feed output ", t);
     }
-    const Node* n = iter->second;
+    Node* n = iter->second;
     DCHECK_EQ(n->name(), id.first);
     if (id.second >= n->num_outputs()) {
       return errors::InvalidArgument(
           "FeedInputs: ", t, " should have output index < ", n->num_outputs());
     }
 
-    Node* recv_node;
-
-    if (!use_function_convention) {
-      TF_RETURN_IF_ERROR(
-          NodeBuilder(strings::StrCat("_recv_", id.first, "_", id.second),
-                      "_Recv")
-              .Attr("tensor_type", BaseType(n->output_type(id.second)))
-              .Attr("tensor_name", t)
-              .Attr("send_device", device_info.name())
-              .Attr("recv_device", device_info.name())
-              .Attr("send_device_incarnation",
-                    static_cast<int64>(device_info.incarnation()))
-              .Attr("client_terminated", true)
-              .Finalize(g, &recv_node));
-    } else {
-      // NOTE(mrry): We must include the index as part of the node
-      // name, because _Arg is a "stateful" kernel and therefore
-      // its name must uniquely identify a kernel instance across all
-      // graphs in the same session.
-      TF_RETURN_IF_ERROR(NodeBuilder(strings::StrCat("_arg_", id.first, "_",
-                                                     id.second, "_", i),
-                                     "_Arg")
-                             .Attr("T", BaseType(n->output_type(id.second)))
-                             .Attr("index", static_cast<int32>(i))
-                             .Finalize(g, &recv_node));
-    }
-    recv_node->set_assigned_device_name(device_info.name());
+    Node* feed_node;
+    TF_RETURN_IF_ERROR(
+        feed_rewrites[i]->AddNode(g, {n, id.second}, &feed_node));
 
     // Update name_index
-    (*name_index)[recv_node->name()] = recv_node;
-    g->AddControlEdge(g->source_node(), recv_node);
+    (*name_index)[feed_node->name()] = feed_node;
+    g->AddControlEdge(g->source_node(), feed_node);
 
     // Look through edges coming out of "n" for edges whose src_output() index
     // matches "output_index".  If found, replace the edges with a connection
@@ -119,7 +95,7 @@ static Status FeedInputs(Graph* g, const DeviceAttributes& device_info,
                   n->type_string() == "PlaceholderV2")) {
         // When feeding a Placeholder node, any outgoing control edges
         // will be replaced with a control edge from the replacement
-        // recv_node.
+        // feed_node.
         // TODO(josh11b,mrry): Come up with a more elegant way of addressing
         // the general version of this problem.
         to_remove.emplace_back(e);
@@ -128,10 +104,10 @@ static Status FeedInputs(Graph* g, const DeviceAttributes& device_info,
 
     for (const Edge* e : to_remove) {
       if (e->src_output() == id.second) {
-        g->AddEdge(recv_node, 0, e->dst(), e->dst_input());
+        g->AddEdge(feed_node, 0, e->dst(), e->dst_input());
       } else {
         CHECK_EQ(Graph::kControlSlot, e->src_output());
-        g->AddControlEdge(recv_node, e->dst());
+        g->AddControlEdge(feed_node, e->dst());
       }
       g->RemoveEdge(e);
     }
@@ -140,9 +116,61 @@ static Status FeedInputs(Graph* g, const DeviceAttributes& device_info,
   return Status::OK();
 }
 
-static bool AddNodeToTargets(const string& node_or_tensor_name,
-                             const subgraph::NameIndex& name_index,
-                             std::unordered_set<const Node*>* targets) {
+Status FetchOutputs(
+    Graph* g, const std::vector<std::unique_ptr<PruneRewrite>>& fetch_rewrites,
+    NameIndex* name_index, std::vector<Node*>* out_fetch_nodes,
+    DataTypeVector* out_fetch_types) {
+  out_fetch_nodes->clear();
+  out_fetch_nodes->reserve(fetch_rewrites.size());
+  for (size_t i = 0; i < fetch_rewrites.size(); ++i) {
+    const string& t = fetch_rewrites[i]->endpoint_name();
+
+    // Parse t into node_name and output_index.
+    TensorId id(ParseTensorName(t));
+
+    // Find node in graph with that name.
+    auto iter = name_index->find(id.first);
+    if (iter == name_index->end()) {
+      return errors::NotFound("FetchOutputs node ", t, ": not found");
+    }
+    Node* n = iter->second;
+    DCHECK_EQ(n->name(), id.first);
+    VLOG(2) << "Found fetch node for " << t;
+
+    // Validate output_index
+    if (n->num_outputs() == 0) {
+      return errors::InvalidArgument(
+          "Tried to fetch data for '", t,
+          "', which produces no output.  To run to a node but not fetch any "
+          "data, pass '",
+          t,
+          "' as an argument to the 'target_node_names' argument of the "
+          "Session::Run API.");
+    } else if (id.second >= n->num_outputs()) {
+      return errors::InvalidArgument("FetchOutputs ", t,
+                                     ": output index too large, must be < ",
+                                     n->num_outputs());
+    }
+
+    // Create the fetch Node and connect it up
+    Node* fetch_node;
+    TF_RETURN_IF_ERROR(
+        fetch_rewrites[i]->AddNode(g, {n, id.second}, &fetch_node));
+
+    // Update the index.
+    (*name_index)[fetch_node->name()] = fetch_node;
+
+    g->AddControlEdge(fetch_node, g->sink_node());
+    out_fetch_nodes->push_back(fetch_node);
+    out_fetch_types->push_back(BaseType(n->output_type(id.second)));
+  }
+
+  return Status::OK();
+}
+
+bool AddNodeToTargets(const string& node_or_tensor_name,
+                      const NameIndex& name_index,
+                      std::unordered_set<const Node*>* targets) {
   TensorId id = ParseTensorName(node_or_tensor_name);
   auto iter = name_index.find(id.first);
   if (iter == name_index.end()) {
@@ -154,9 +182,9 @@ static bool AddNodeToTargets(const string& node_or_tensor_name,
   return true;
 }
 
-static Status PruneForTargets(Graph* g, const subgraph::NameIndex& name_index,
-                              const std::vector<Node*>& fetch_nodes,
-                              const gtl::ArraySlice<string>& target_nodes) {
+Status PruneForTargets(Graph* g, const NameIndex& name_index,
+                       const std::vector<Node*>& fetch_nodes,
+                       const gtl::ArraySlice<string>& target_nodes) {
   string not_found;
   std::unordered_set<const Node*> targets;
   for (Node* n : fetch_nodes) {
@@ -183,108 +211,149 @@ static Status PruneForTargets(Graph* g, const subgraph::NameIndex& name_index,
 
 }  // namespace
 
-namespace subgraph {
+Status ArgFeedRewrite::AddNode(Graph* g, NodeBuilder::NodeOut feed_tensor,
+                               Node** out_node) {
+  // NOTE(mrry): We must include the index as part of the node
+  // name, because _Arg is a "stateful" kernel and therefore
+  // its name must uniquely identify a kernel instance across all
+  // graphs in the same session.
+  TF_RETURN_IF_ERROR(
+      NodeBuilder(strings::StrCat("_arg_", feed_tensor.node->name(), "_",
+                                  feed_tensor.index, "_", arg_index_),
+                  "_Arg")
+          .Attr("T", BaseType(feed_tensor.node->output_type(feed_tensor.index)))
+          .Attr("index", arg_index_)
+          .Finalize(g, out_node));
+  (*out_node)->set_assigned_device_name(device_info().name());
+  return Status::OK();
+}
 
-Status FetchOutputs(Graph* g, const DeviceAttributes& device_info,
-                    const gtl::ArraySlice<string>& fetch_outputs,
-                    bool use_function_convention, NameIndex* name_index,
-                    std::vector<Node*>* out_fetch_nodes,
-                    DataTypeVector* out_fetch_types) {
-  out_fetch_nodes->clear();
-  out_fetch_nodes->reserve(fetch_outputs.size());
-  for (size_t i = 0; i < fetch_outputs.size(); ++i) {
-    const string& t = fetch_outputs[i];
+Status RecvFeedRewrite::AddNode(Graph* g, NodeBuilder::NodeOut feed_tensor,
+                                Node** out_node) {
+  TF_RETURN_IF_ERROR(
+      NodeBuilder(strings::StrCat("_recv_", feed_tensor.node->name(), "_",
+                                  feed_tensor.index),
+                  "_Recv")
+          .Attr("tensor_type",
+                BaseType(feed_tensor.node->output_type(feed_tensor.index)))
+          .Attr("tensor_name", endpoint_name())
+          .Attr("send_device", device_info().name())
+          .Attr("recv_device", device_info().name())
+          .Attr("send_device_incarnation",
+                static_cast<int64>(device_info().incarnation()))
+          .Attr("client_terminated", true)
+          .Finalize(g, out_node));
+
+  (*out_node)->set_assigned_device_name(device_info().name());
+  return Status::OK();
+}
 
-    // Parse t into node_name and output_index.
-    TensorId id(ParseTensorName(t));
+Status RetvalFetchRewrite::AddNode(Graph* g, NodeBuilder::NodeOut fetch_tensor,
+                                   Node** out_node) {
+  // NOTE(mrry): We must include the index as part of the node
+  // name, because _Retval is a "stateful" kernel and therefore
+  // its name must uniquely identify a kernel instance across all
+  // graphs in the same session.
+  TF_RETURN_IF_ERROR(
+      NodeBuilder(strings::StrCat("_retval_", fetch_tensor.node->name(), "_",
+                                  fetch_tensor.index, "_", retval_index_),
+                  "_Retval")
+          .Input(fetch_tensor.node, fetch_tensor.index)
+          .Attr("T",
+                BaseType(fetch_tensor.node->output_type(fetch_tensor.index)))
+          .Attr("index", retval_index_)
+          .Finalize(g, out_node));
+  (*out_node)->set_assigned_device_name(device_info().name());
+  return Status::OK();
+}
 
-    // Find node in graph with that name.
-    auto iter = name_index->find(id.first);
-    if (iter == name_index->end()) {
-      return errors::NotFound("FetchOutputs node ", t, ": not found");
-    }
-    Node* n = iter->second;
-    DCHECK_EQ(n->name(), id.first);
-    VLOG(2) << "Found fetch node for " << t;
+Status SendFetchRewrite::AddNode(Graph* g, NodeBuilder::NodeOut fetch_tensor,
+                                 Node** out_node) {
+  TF_RETURN_IF_ERROR(
+      NodeBuilder(strings::StrCat("_send_", fetch_tensor.node->name(), "_",
+                                  fetch_tensor.index),
+                  "_Send")
+          .Input(fetch_tensor.node, fetch_tensor.index)
+          .Attr("tensor_name", endpoint_name())
+          .Attr("send_device", device_info().name())
+          .Attr("recv_device", device_info().name())
+          .Attr("send_device_incarnation",
+                static_cast<int64>(device_info().incarnation()))
+          .Attr("client_terminated", true)
+          .Finalize(g, out_node));
+  (*out_node)->set_assigned_device_name(device_info().name());
+  return Status::OK();
+}
 
-    // Validate output_index
-    if (n->num_outputs() == 0) {
-      return errors::InvalidArgument(
-          "Tried to fetch data for '", t,
-          "', which produces no output.  To run to a node but not fetch any "
-          "data, pass '",
-          t,
-          "' as an argument to the 'target_node_names' argument of the "
-          "Session::Run API.");
-    } else if (id.second >= n->num_outputs()) {
-      return errors::InvalidArgument("FetchOutputs ", t,
-                                     ": output index too large, must be < ",
-                                     n->num_outputs());
+Status RewriteGraphForExecution(
+    Graph* g, const gtl::ArraySlice<string>& fed_outputs,
+    const gtl::ArraySlice<string>& fetch_outputs,
+    const gtl::ArraySlice<string>& target_node_names,
+    const DeviceAttributes& device_info, bool use_function_convention,
+    RewriteGraphMetadata* out_metadata) {
+  std::vector<std::unique_ptr<PruneRewrite>> feed_rewrites;
+  feed_rewrites.reserve(fed_outputs.size());
+  if (use_function_convention) {
+    for (size_t i = 0; i < fed_outputs.size(); ++i) {
+      feed_rewrites.emplace_back(new ArgFeedRewrite(
+          &fed_outputs[i], &device_info, static_cast<int32>(i)));
     }
-
-    // Create the fetch Node and connect it up
-    Node* send_node;
-    if (!use_function_convention) {
-      TF_RETURN_IF_ERROR(
-          NodeBuilder(strings::StrCat("_send_", id.first, "_", id.second),
-                      "_Send")
-              .Input(n, id.second)
-              .Attr("tensor_name", t)
-              .Attr("send_device", device_info.name())
-              .Attr("recv_device", device_info.name())
-              .Attr("send_device_incarnation",
-                    static_cast<int64>(device_info.incarnation()))
-              .Attr("client_terminated", true)
-              .Finalize(g, &send_node));
-    } else {
-      // NOTE(mrry): We must include the index as part of the node
-      // name, because _Retval is a "stateful" kernel and therefore
-      // its name must uniquely identify a kernel instance across all
-      // graphs in the same session.
-      TF_RETURN_IF_ERROR(NodeBuilder(strings::StrCat("_retval_", id.first, "_",
-                                                     id.second, "_", i),
-                                     "_Retval")
-                             .Input(n, id.second)
-                             .Attr("T", BaseType(n->output_type(id.second)))
-                             .Attr("index", static_cast<int32>(i))
-                             .Finalize(g, &send_node));
+  } else {
+    for (const string& fed_output : fed_outputs) {
+      feed_rewrites.emplace_back(
+          new RecvFeedRewrite(&fed_output, &device_info));
     }
-    send_node->set_assigned_device_name(device_info.name());
-
-    // Update the index.
-    (*name_index)[send_node->name()] = send_node;
+  }
 
-    g->AddControlEdge(send_node, g->sink_node());
-    out_fetch_nodes->push_back(send_node);
-    out_fetch_types->push_back(BaseType(n->output_type(id.second)));
+  std::vector<std::unique_ptr<PruneRewrite>> fetch_rewrites;
+  fetch_rewrites.reserve(fetch_outputs.size());
+  if (use_function_convention) {
+    for (size_t i = 0; i < fetch_outputs.size(); ++i) {
+      fetch_rewrites.emplace_back(new RetvalFetchRewrite(
+          &fetch_outputs[i], &device_info, static_cast<int32>(i)));
+    }
+  } else {
+    for (const string& fetch_output : fetch_outputs) {
+      fetch_rewrites.emplace_back(
+          new SendFetchRewrite(&fetch_output, &device_info));
+    }
   }
 
-  return Status::OK();
+  return RewriteGraphForExecution(g, feed_rewrites, fetch_rewrites,
+                                  target_node_names, out_metadata);
+}
+
+namespace {
+template <typename StringContainer>
+std::vector<string> ConvertToVector(StringContainer field) {
+  return std::vector<string>(field.begin(), field.end());
 }
+}  // namespace
 
 Status RewriteGraphForExecution(
-    Graph* g, const gtl::ArraySlice<string>& fed_outputs,
-    const gtl::ArraySlice<string>& fetch_outputs,
+    Graph* g, const std::vector<std::unique_ptr<PruneRewrite>>& feed_rewrites,
+    const std::vector<std::unique_ptr<PruneRewrite>>& fetch_rewrites,
     const gtl::ArraySlice<string>& target_node_names,
-    const DeviceAttributes& device_info, bool use_function_convention,
     RewriteGraphMetadata* out_metadata) {
-  if (fetch_outputs.empty() && target_node_names.empty()) {
+  if (fetch_rewrites.empty() && target_node_names.empty()) {
     return errors::InvalidArgument(
         "Must specify at least one target to fetch or execute.");
   }
 
   std::unordered_set<string> endpoints;
-  for (const string& endpoint_name : fed_outputs) {
-    auto result = endpoints.insert(endpoint_name);
+  for (const auto& feed_rewrite : feed_rewrites) {
+    auto result = endpoints.insert(feed_rewrite->endpoint_name());
     if (!result.second) {
-      return errors::InvalidArgument("Endpoint \"", endpoint_name,
+      return errors::InvalidArgument("Endpoint \"",
+                                     feed_rewrite->endpoint_name(),
                                      "\" fed more than once.");
     }
   }
 
-  for (const auto& fetch : fetch_outputs) {
-    if (endpoints.count(fetch) > 0) {
-      return errors::InvalidArgument(fetch, " is both fed and fetched.");
+  for (const auto& fetch_rewrite : fetch_rewrites) {
+    if (endpoints.count(fetch_rewrite->endpoint_name()) > 0) {
+      return errors::InvalidArgument(fetch_rewrite->endpoint_name(),
+                                     " is both fed and fetched.");
     }
   }
 
@@ -297,19 +366,17 @@ Status RewriteGraphForExecution(
   }
 
   // Add the feeds.  This may replace nodes in the graph, including the nodes
-  // currently listed in "fetch_nodes".  We pass "name_index" so the index is
+  // currently listed in "fetch_rewrites".  We pass "name_index" so the index is
   // kept up to date.
-  if (!fed_outputs.empty()) {
-    TF_RETURN_IF_ERROR(FeedInputs(g, device_info, fed_outputs,
-                                  use_function_convention, &name_index,
-                                  &out_metadata->feed_types));
+  if (!feed_rewrites.empty()) {
+    TF_RETURN_IF_ERROR(
+        FeedInputs(g, feed_rewrites, &name_index, &out_metadata->feed_types));
   }
 
   // Add the fetch nodes, also updating "name_index".
   std::vector<Node*> fetch_nodes;
-  if (!fetch_outputs.empty()) {
-    TF_RETURN_IF_ERROR(FetchOutputs(g, device_info, fetch_outputs,
-                                    use_function_convention, &name_index,
+  if (!fetch_rewrites.empty()) {
+    TF_RETURN_IF_ERROR(FetchOutputs(g, fetch_rewrites, &name_index,
                                     &fetch_nodes, &out_metadata->fetch_types));
   }
 
@@ -323,25 +390,6 @@ Status RewriteGraphForExecution(
   return Status::OK();
 }
 
-namespace {
-template <typename StringContainer>
-std::vector<string> ConvertToVector(StringContainer field) {
-  return std::vector<string>(field.begin(), field.end());
-}
-}  // namespace
-
-Status RewriteGraphForExecution(Graph* g,
-                                const CallableOptions& callable_options,
-                                const DeviceAttributes& device_info,
-                                bool use_function_convention,
-                                RewriteGraphMetadata* out_metadata) {
-  return RewriteGraphForExecution(g, ConvertToVector(callable_options.feed()),
-                                  ConvertToVector(callable_options.fetch()),
-                                  ConvertToVector(callable_options.target()),
-                                  device_info, use_function_convention,
-                                  out_metadata);
-}
-
 }  // namespace subgraph
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/graph/subgraph.h b/tensorflow/core/graph/subgraph.h
index 0dc59582f4..ba35846d93 100644
--- a/tensorflow/core/graph/subgraph.h
+++ b/tensorflow/core/graph/subgraph.h
@@ -20,6 +20,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/device_attributes.pb.h"
 #include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 #include "tensorflow/core/protobuf/config.pb.h"
@@ -39,6 +40,37 @@ struct RewriteGraphMetadata {
   DataTypeVector fetch_types;
 };
 
+// Describes the action to take on a particular tensor endpoint (described by
+// a "<node_name>:<output_index>" pair) when pruning the graph.
+//
+// The `AddNode()` method must be overridden to describe this action. The method
+// will be invoked once during `RewriteGraphForExecution()` with tensor endpoint
+// named by `endpoint_name`, and it may either create a single new node, or fail
+// with an error if the resulting graph would be invalid.
+class PruneRewrite {
+ public:
+  // `endpoint_name` and `device_info` must outlive this object.
+  PruneRewrite(const string* endpoint_name, const DeviceAttributes* device_info)
+      : endpoint_name_(endpoint_name), device_info_(device_info) {}
+  virtual ~PruneRewrite() {}
+
+  // Creates a new node whose output replaces the given `tensor` in graph `g`.
+  // The node will be assigned to the device named in `device_info`.
+  virtual Status AddNode(Graph* g, NodeBuilder::NodeOut tensor,
+                         Node** out_node) = 0;
+
+  // Returns the name of the tensor to which this rewrite applies.
+  const string& endpoint_name() { return *endpoint_name_; }
+
+ protected:
+  // The device on which the new node will be created.
+  const DeviceAttributes& device_info() { return *device_info_; }
+
+ private:
+  const string* const endpoint_name_;          // Not owned.
+  const DeviceAttributes* const device_info_;  // Not owned.
+};
+
 // Rewrite the graph structure of "*g" to deal with feeding node
 // outputs, fetching node outputs, and only running a subset of the
 // graph.  "fed_outputs" and "fetch_outputs" are both lists of
@@ -49,7 +81,7 @@ struct RewriteGraphMetadata {
 // In the resulting graph "*g", output edges in "fed_outputs" have
 // been redirected to special "_recv" nodes introduced into the graph.
 // If these fed nodes are not needed in order to compute the effects
-// of the nodes in "targets_nodes" and "fetch_outputs", then these may
+// of the nodes in "target_node_names" and "fetch_outputs", then these may
 // be omitted from the graph.
 //
 // In the resulting graph "*g", additional "_send" nodes are connected
@@ -71,25 +103,61 @@ Status RewriteGraphForExecution(
     const gtl::ArraySlice<string>& target_node_names,
     const DeviceAttributes& device_info, bool use_function_convention,
     RewriteGraphMetadata* out_metadata);
-Status RewriteGraphForExecution(Graph* g,
-                                const CallableOptions& callable_options,
-                                const DeviceAttributes& device_info,
-                                bool use_function_convention,
-                                RewriteGraphMetadata* out_metadata);
-
-typedef std::unordered_map<StringPiece, Node*, StringPieceHasher> NameIndex;
-
-// Augment "*g" by adding special "fetch" nodes that connect to the
-// tensor outputs specified in "fetch_outputs" to retrieve the output
-// of the tensors.  The new nodes added are set up to execute on
-// "client_device_name", and are returned in "*fetch_nodes".
-//
-// Return OK on success.  On error, return false and sets *error to
-// an appropriate error message (and *g is left in an indeterminate
-// state).
-Status FetchOutputs(Graph* g, const DeviceAttributes& device_info,
-                    const gtl::ArraySlice<string>& fetch_outputs,
-                    NameIndex* name_index, std::vector<Node*>* fetch_nodes);
+
+// A more general version of the above function that supports
+// customizable rewriting actions for each fed and fetched tensor.
+Status RewriteGraphForExecution(
+    Graph* g, const std::vector<std::unique_ptr<PruneRewrite>>& feed_rewrites,
+    const std::vector<std::unique_ptr<PruneRewrite>>& fetch_rewrites,
+    const gtl::ArraySlice<string>& target_node_names,
+    RewriteGraphMetadata* out_metadata);
+
+/////////////////////////////////////////////////////////
+// Custom rewrite actions for fed and fetched tensors. //
+/////////////////////////////////////////////////////////
+
+// A rewrite action that adds an _Arg node for a fed tensor.
+class ArgFeedRewrite : public PruneRewrite {
+ public:
+  ArgFeedRewrite(const string* endpoint_name,
+                 const DeviceAttributes* device_info, int32 arg_index)
+      : PruneRewrite(endpoint_name, device_info), arg_index_(arg_index) {}
+  Status AddNode(Graph* g, NodeBuilder::NodeOut feed_tensor,
+                 Node** out_node) override;
+
+ private:
+  const int32 arg_index_;
+};
+
+// A rewrite action that adds a client-terminated _Recv node for a fed tensor.
+class RecvFeedRewrite : public PruneRewrite {
+ public:
+  using PruneRewrite::PruneRewrite;
+  Status AddNode(Graph* g, NodeBuilder::NodeOut feed_tensor,
+                 Node** out_node) override;
+};
+
+// A rewrite action that adds a _Retval node for a fetched tensor.
+class RetvalFetchRewrite : public PruneRewrite {
+ public:
+  RetvalFetchRewrite(const string* endpoint_name,
+                     const DeviceAttributes* device_info, int32 retval_index)
+      : PruneRewrite(endpoint_name, device_info), retval_index_(retval_index) {}
+  Status AddNode(Graph* g, NodeBuilder::NodeOut fetch_tensor,
+                 Node** out_node) override;
+
+ private:
+  const int32 retval_index_;
+};
+
+// A rewrite action that adds a client-terminated _Send node for a
+// fetched tensor.
+class SendFetchRewrite : public PruneRewrite {
+ public:
+  using PruneRewrite::PruneRewrite;
+  Status AddNode(Graph* g, NodeBuilder::NodeOut fetch_tensor,
+                 Node** out_node) override;
+};
 
 }  // namespace subgraph
 }  // namespace tensorflow
-- 
GitLab


From 754c0615c94bbc7f8ede78b8b16cc616104994ef Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 21 Mar 2018 09:19:11 -0700
Subject: [PATCH 295/960] Deleted dead code and fixed compilation warnings

PiperOrigin-RevId: 189918110
---
 .../costs/op_level_cost_estimator_test.cc       | 17 -----------------
 .../grappler/optimizers/dependency_optimizer.h  |  6 ++----
 .../grappler/optimizers/function_optimizer.h    |  5 +----
 .../grappler/optimizers/layout_optimizer.cc     |  4 ----
 4 files changed, 3 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index d5360cba24..a92f230101 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -55,23 +55,6 @@ OpContext DescribeMatMul(int m, int n, int l, int k) {
   return op_context;
 }
 
-// Returns an OpInfo for MatMul with unknown input shapes.
-OpContext DescribeMatMulUnknownShape() {
-  OpContext op_context;
-  SetCpuDevice(&op_context.op_info);
-  op_context.op_info.set_op("MatMul");
-
-  auto input = op_context.op_info.add_inputs();
-  auto shape = input->mutable_shape();
-  shape->set_unknown_rank(true);
-
-  input = op_context.op_info.add_inputs();
-  shape = input->mutable_shape();
-  shape->set_unknown_rank(true);
-
-  return op_context;
-}
-
 // Wrangles the minimum number of proto fields to set up an input of
 // arbitrary rank and type.
 void DescribeArbitraryRankInput(const std::vector<int>& dims, DataType dtype,
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.h b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
index 61ed154793..b4db98125a 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.h
@@ -29,9 +29,8 @@ namespace grappler {
 // optimizations, such as removing nodes that are effectively noops.
 class DependencyOptimizer : public GraphOptimizer {
  public:
-  DependencyOptimizer() : opt_level_(RewriterConfig::ON) {}
-  explicit DependencyOptimizer(RewriterConfig::Toggle opt_level)
-      : opt_level_(opt_level) {}
+  DependencyOptimizer() {}
+  explicit DependencyOptimizer(RewriterConfig::Toggle opt_level) {}
   ~DependencyOptimizer() override {}
 
   string name() const override { return "dependency_optimizer"; };
@@ -63,7 +62,6 @@ class DependencyOptimizer : public GraphOptimizer {
   // Main driver of dependency optimizations.
   Status OptimizeDependencies();
 
-  RewriterConfig::Toggle opt_level_;
   bool fetch_nodes_known_;
   std::unordered_set<string> nodes_to_preserve_;
   std::unique_ptr<NodeMap> node_map_;
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.h b/tensorflow/core/grappler/optimizers/function_optimizer.h
index b124efe01d..41444e4673 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.h
@@ -26,7 +26,7 @@ namespace grappler {
 // operations to make the overall graph more efficient.
 class FunctionOptimizer : public GraphOptimizer {
  public:
-  FunctionOptimizer(RewriterConfig::Toggle opt_level) : opt_level_(opt_level) {}
+  FunctionOptimizer(RewriterConfig::Toggle opt_level) {}
   ~FunctionOptimizer() override {}
 
   string name() const override { return "function_optimizer"; };
@@ -36,9 +36,6 @@ class FunctionOptimizer : public GraphOptimizer {
 
   void Feedback(Cluster* cluster, const GrapplerItem& item,
                 const GraphDef& optimized_graph, double result) override;
-
- private:
-  RewriterConfig::Toggle opt_level_;
 };
 
 }  // end namespace grappler
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index e4af71c40a..18e63f823b 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -301,10 +301,6 @@ bool IsComparisonOp(const NodeDef& node) {
   return is_compare;
 }
 
-bool IsLogicalOp(const NodeDef& node) {
-  return IsLogicalAnd(node) || IsLogicalNot(node) || IsLogicalOr(node);
-}
-
 bool IsReduceOp(const NodeDef& node) {
   return IsSum(node) || IsMean(node) || IsProd(node) || IsMax(node) ||
          IsMin(node) || IsAll(node) || IsAny(node);
-- 
GitLab


From 326bfa618a86c9fd604b8b98be6baff46337b6c6 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 21 Mar 2018 09:20:28 -0700
Subject: [PATCH 296/960] Don't run tensorflow/python:function_test under
 fastbuild.

It gets flaky timeouts.

PiperOrigin-RevId: 189918276
---
 tensorflow/python/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 11195b3565..d11ee6f74c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1034,7 +1034,10 @@ cuda_py_tests(
         "//tensorflow/core:protos_all_py",
     ],
     shard_count = 10,
-    tags = ["noasan"],
+    tags = [
+        "noasan",
+        "optonly",
+    ],
 )
 
 py_test(
-- 
GitLab


From 335c782f5c504e36e496a33180d8243760a4001c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 09:26:16 -0700
Subject: [PATCH 297/960] Deletes sequential_feature_column(|_test).py.

PiperOrigin-RevId: 189919029
---
 .../sequential_feature_column.py              | 325 ------------
 .../sequential_feature_column_test.py         | 471 ------------------
 2 files changed, 796 deletions(-)
 delete mode 100644 tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column.py
 delete mode 100644 tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column_test.py

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column.py b/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column.py
deleted file mode 100644
index 4ed7268e7a..0000000000
--- a/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column.py
+++ /dev/null
@@ -1,325 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Experimental methods for tf.feature_column sequence input."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-
-import abc
-import collections
-
-
-from tensorflow.python.feature_column import feature_column as fc
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import tensor_shape
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import check_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import parsing_ops
-from tensorflow.python.ops import sparse_ops
-from tensorflow.python.ops import variable_scope
-
-# TODO(b/73160931): Fix pydoc.
-# pylint: disable=g-doc-args,missing-docstring,protected-access
-# TODO(b/73827486): Support SequenceExample.
-
-
-def sequence_input_layer(
-    features,
-    feature_columns,
-    weight_collections=None,
-    trainable=True,
-    scope=None):
-  """"Builds input layer for sequence input.
-
-  All `feature_columns` must be sequence dense columns with the same
-  `sequence_length`. The output of this method can be fed into sequence
-  networks, such as RNN.
-
-  The output of this method is a 3D `Tensor` of shape `[batch_size, T, D]`.
-  `T` is the maximum sequence length for this batch, which could differ from
-  batch to batch.
-
-  If multiple `feature_columns` are given with `Di` `num_elements` each, their
-  outputs are concatenated. So, the final `Tensor` has shape
-  `[batch_size, T, D0 + D1 + ... + Dn]`.
-
-  Example:
-
-  ```python
-  rating = sequence_numeric_column('rating')
-  watches = sequence_categorical_column_with_identity(
-      'watches', num_buckets=1000)
-  watches_embedding = embedding_column(watches, dimension=10)
-  columns = [rating, watches]
-
-  features = tf.parse_example(..., features=make_parse_example_spec(columns))
-  input_layer, sequence_length = sequence_input_layer(features, columns)
-
-  rnn_cell = tf.nn.rnn_cell.BasicRNNCell(hidden_size)
-  outputs, state = tf.nn.dynamic_rnn(
-      rnn_cell, inputs=input_layer, sequence_length=sequence_length)
-  ```
-
-  Returns:
-    An `(input_layer, sequence_length)` tuple where:
-    - input_layer: A float `Tensor` of shape `[batch_size, T, D]`.
-        `T` is the maximum sequence length for this batch, which could differ
-        from batch to batch. `D` is the sum of `num_elements` for all
-        `feature_columns`.
-    - sequence_length: An int `Tensor` of shape `[batch_size]`. The sequence
-        length for each example.
-  Raises:
-    ValueError: If any of the `feature_columns` is the wrong type.
-  """
-  feature_columns = fc._clean_feature_columns(feature_columns)
-  for c in feature_columns:
-    if not isinstance(c, _SequenceDenseColumn):
-      raise ValueError(
-          'All feature_columns must be of type _SequenceDenseColumn. '
-          'Given (type {}): {}'.format(type(c), c))
-
-  with variable_scope.variable_scope(
-      scope, default_name='sequence_input_layer', values=features.values()):
-    builder = fc._LazyBuilder(features)
-    output_tensors = []
-    sequence_lengths = []
-    ordered_columns = []
-    for column in sorted(feature_columns, key=lambda x: x.name):
-      ordered_columns.append(column)
-      with variable_scope.variable_scope(
-          None, default_name=column._var_scope_name):
-        dense_tensor, sequence_length = column._get_sequence_dense_tensor(
-            builder,
-            weight_collections=weight_collections,
-            trainable=trainable)
-        # Flattens the final dimension to produce a 3D Tensor.
-        num_elements = column._variable_shape.num_elements()
-        shape = array_ops.shape(dense_tensor)
-        output_tensors.append(
-            array_ops.reshape(
-                dense_tensor,
-                shape=array_ops.concat([shape[:2], [num_elements]], axis=0)))
-        sequence_lengths.append(sequence_length)
-    fc._verify_static_batch_size_equality(output_tensors, ordered_columns)
-    # TODO(b/73160931): Verify sequence_length equality.
-    return array_ops.concat(output_tensors, -1), sequence_lengths[0]
-
-
-# TODO(b/73160931): Add remaining categorical columns.
-def sequence_categorical_column_with_identity(
-    key, num_buckets, default_value=None):
-  return _SequenceCategoricalColumn(
-      fc.categorical_column_with_identity(
-          key=key,
-          num_buckets=num_buckets,
-          default_value=default_value))
-
-
-# TODO(b/73160931): Merge with embedding_column
-def _sequence_embedding_column(
-    categorical_column, dimension, initializer=None, ckpt_to_load_from=None,
-    tensor_name_in_ckpt=None, max_norm=None, trainable=True):
-  if not isinstance(categorical_column, _SequenceCategoricalColumn):
-    raise ValueError(
-        'categorical_column must be of type _SequenceCategoricalColumn. '
-        'Given (type {}): {}'.format(
-            type(categorical_column), categorical_column))
-  return _SequenceEmbeddingColumn(
-      fc.embedding_column(
-          categorical_column,
-          dimension=dimension,
-          initializer=initializer,
-          ckpt_to_load_from=ckpt_to_load_from,
-          tensor_name_in_ckpt=tensor_name_in_ckpt,
-          max_norm=max_norm,
-          trainable=trainable))
-
-
-def sequence_numeric_column(
-    key,
-    shape=(1,),
-    default_value=0.,
-    dtype=dtypes.float32):
-  # TODO(b/73160931): Add validations.
-  return _SequenceNumericColumn(
-      key,
-      shape=shape,
-      default_value=default_value,
-      dtype=dtype)
-
-
-class _SequenceDenseColumn(fc._FeatureColumn):
-  """Represents dense sequence data."""
-
-  __metaclass__ = abc.ABCMeta
-
-  TensorSequenceLengthPair = collections.namedtuple(  # pylint: disable=invalid-name
-      'TensorSequenceLengthPair', ['dense_tensor', 'sequence_length'])
-
-  @abc.abstractproperty
-  def _variable_shape(self):
-    """`TensorShape` without batch and sequence dimensions."""
-    pass
-
-  @abc.abstractmethod
-  def _get_sequence_dense_tensor(
-      self, inputs, weight_collections=None, trainable=None):
-    """Returns a `TensorSequenceLengthPair`."""
-    pass
-
-
-def _sequence_length_from_sparse_tensor(sp_tensor, num_elements=1):
-  with ops.name_scope(None, 'sequence_length') as name_scope:
-    row_ids = sp_tensor.indices[:, 0]
-    column_ids = sp_tensor.indices[:, 1]
-    column_ids += array_ops.ones_like(column_ids)
-    seq_length = (
-        math_ops.segment_max(column_ids, segment_ids=row_ids) / num_elements)
-    # If the last n rows do not have ids, seq_length will have shape
-    # [batch_size - n]. Pad the remaining values with zeros.
-    n_pad = array_ops.shape(sp_tensor)[:1] - array_ops.shape(seq_length)[:1]
-    padding = array_ops.zeros(n_pad, dtype=seq_length.dtype)
-    return array_ops.concat([seq_length, padding], axis=0, name=name_scope)
-
-
-class _SequenceCategoricalColumn(
-    fc._CategoricalColumn,
-    collections.namedtuple(
-        '_SequenceCategoricalColumn', ['categorical_column'])):
-
-  @property
-  def name(self):
-    return self.categorical_column.name
-
-  @property
-  def _parse_example_spec(self):
-    return self.categorical_column._parse_example_spec
-
-  def _transform_feature(self, inputs):
-    return self.categorical_column._transform_feature(inputs)
-
-  @property
-  def _num_buckets(self):
-    return self.categorical_column._num_buckets
-
-  def _get_sparse_tensors(self, inputs, weight_collections=None,
-                          trainable=None):
-    sparse_tensors = self.categorical_column._get_sparse_tensors(inputs)
-    id_tensor = sparse_tensors.id_tensor
-    weight_tensor = sparse_tensors.weight_tensor
-    # Expands final dimension, so that embeddings are not combined during
-    # embedding lookup.
-    check_id_rank = check_ops.assert_equal(
-        array_ops.rank(id_tensor), 2,
-        data=[
-            'Column {} expected ID tensor of rank 2. '.format(self.name),
-            'id_tensor shape: ', array_ops.shape(id_tensor)])
-    with ops.control_dependencies([check_id_rank]):
-      id_tensor = sparse_ops.sparse_reshape(
-          id_tensor,
-          shape=array_ops.concat([id_tensor.dense_shape, [1]], axis=0))
-    if weight_tensor is not None:
-      check_weight_rank = check_ops.assert_equal(
-          array_ops.rank(weight_tensor), 2,
-          data=[
-              'Column {} expected weight tensor of rank 2.'.format(self.name),
-              'weight_tensor shape:', array_ops.shape(weight_tensor)])
-      with ops.control_dependencies([check_weight_rank]):
-        weight_tensor = sparse_ops.sparse_reshape(
-            weight_tensor,
-            shape=array_ops.concat([weight_tensor.dense_shape, [1]], axis=0))
-    return fc._CategoricalColumn.IdWeightPair(id_tensor, weight_tensor)
-
-  def _sequence_length(self, inputs):
-    sparse_tensors = self.categorical_column._get_sparse_tensors(inputs)
-    return _sequence_length_from_sparse_tensor(sparse_tensors.id_tensor)
-
-
-class _SequenceEmbeddingColumn(
-    _SequenceDenseColumn,
-    collections.namedtuple('_SequenceEmbeddingColumn', ['embedding_column'])):
-
-  @property
-  def name(self):
-    return self.embedding_column.name
-
-  @property
-  def _parse_example_spec(self):
-    return self.embedding_column._parse_example_spec
-
-  def _transform_feature(self, inputs):
-    return self.embedding_column._transform_feature(inputs)
-
-  @property
-  def _variable_shape(self):
-    return self.embedding_column._variable_shape
-
-  def _get_sequence_dense_tensor(
-      self, inputs, weight_collections=None, trainable=None):
-    dense_tensor = self.embedding_column._get_dense_tensor(
-        inputs=inputs,
-        weight_collections=weight_collections,
-        trainable=trainable)
-    sequence_length = self.embedding_column.categorical_column._sequence_length(
-        inputs)
-    return _SequenceDenseColumn.TensorSequenceLengthPair(
-        dense_tensor=dense_tensor, sequence_length=sequence_length)
-
-
-class _SequenceNumericColumn(
-    _SequenceDenseColumn,
-    collections.namedtuple(
-        '_SequenceNumericColumn',
-        ['key', 'shape', 'default_value', 'dtype'])):
-
-  @property
-  def name(self):
-    return self.key
-
-  @property
-  def _parse_example_spec(self):
-    return {self.key: parsing_ops.VarLenFeature(self.dtype)}
-
-  def _transform_feature(self, inputs):
-    return inputs.get(self.key)
-
-  @property
-  def _variable_shape(self):
-    return tensor_shape.TensorShape(self.shape)
-
-  def _get_sequence_dense_tensor(
-      self, inputs, weight_collections=None, trainable=None):
-    # Do nothing with weight_collections and trainable since no variables are
-    # created in this function.
-    del weight_collections
-    del trainable
-    sp_tensor = inputs.get(self)
-    dense_tensor = sparse_ops.sparse_tensor_to_dense(
-        sp_tensor, default_value=self.default_value)
-    # Reshape into [batch_size, T, variable_shape].
-    dense_shape = array_ops.concat(
-        [array_ops.shape(dense_tensor)[:1], [-1], self._variable_shape],
-        axis=0)
-    dense_tensor = array_ops.reshape(dense_tensor, shape=dense_shape)
-    sequence_length = _sequence_length_from_sparse_tensor(
-        sp_tensor, num_elements=self._variable_shape.num_elements())
-    return _SequenceDenseColumn.TensorSequenceLengthPair(
-        dense_tensor=dense_tensor, sequence_length=sequence_length)
-
-# pylint: enable=g-doc-args,missing-docstring,protected-access
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column_test.py
deleted file mode 100644
index 59674869a2..0000000000
--- a/tensorflow/contrib/feature_column/python/feature_column/sequential_feature_column_test.py
+++ /dev/null
@@ -1,471 +0,0 @@
-# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for sequential_feature_column."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.feature_column.python.feature_column import sequential_feature_column as sfc
-from tensorflow.python.feature_column.feature_column import _LazyBuilder
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import sparse_tensor
-from tensorflow.python.platform import test
-from tensorflow.python.training import monitored_session
-
-
-class SequenceInputLayerTest(test.TestCase):
-
-  def test_embedding_column(self):
-    vocabulary_size = 3
-    sparse_input_a = sparse_tensor.SparseTensorValue(
-        # example 0, ids [2]
-        # example 1, ids [0, 1]
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(2, 0, 1),
-        dense_shape=(2, 2))
-    sparse_input_b = sparse_tensor.SparseTensorValue(
-        # example 0, ids [1]
-        # example 1, ids [2, 0]
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(1, 2, 0),
-        dense_shape=(2, 2))
-
-    embedding_dimension_a = 2
-    embedding_values_a = (
-        (1., 2.),  # id 0
-        (3., 4.),  # id 1
-        (5., 6.)  # id 2
-    )
-    embedding_dimension_b = 3
-    embedding_values_b = (
-        (11., 12., 13.),  # id 0
-        (14., 15., 16.),  # id 1
-        (17., 18., 19.)  # id 2
-    )
-    def _get_initializer(embedding_dimension, embedding_values):
-      def _initializer(shape, dtype, partition_info):
-        self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
-        self.assertEqual(dtypes.float32, dtype)
-        self.assertIsNone(partition_info)
-        return embedding_values
-      return _initializer
-
-    expected_input_layer = [
-        # example 0, ids_a [2], ids_b [1]
-        [[5., 6., 14., 15., 16.], [0., 0., 0., 0., 0.]],
-        # example 1, ids_a [0, 1], ids_b [2, 0]
-        [[1., 2., 17., 18., 19.], [3., 4., 11., 12., 13.]],
-    ]
-    expected_sequence_length = [1, 2]
-
-    categorical_column_a = sfc.sequence_categorical_column_with_identity(
-        key='aaa', num_buckets=vocabulary_size)
-    embedding_column_a = sfc._sequence_embedding_column(
-        categorical_column_a, dimension=embedding_dimension_a,
-        initializer=_get_initializer(embedding_dimension_a, embedding_values_a))
-    categorical_column_b = sfc.sequence_categorical_column_with_identity(
-        key='bbb', num_buckets=vocabulary_size)
-    embedding_column_b = sfc._sequence_embedding_column(
-        categorical_column_b, dimension=embedding_dimension_b,
-        initializer=_get_initializer(embedding_dimension_b, embedding_values_b))
-
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={
-            'aaa': sparse_input_a,
-            'bbb': sparse_input_b,
-        },
-        # Test that columns are reordered alphabetically.
-        feature_columns=[embedding_column_b, embedding_column_a])
-
-    global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('sequence_input_layer/aaa_embedding/embedding_weights:0',
-         'sequence_input_layer/bbb_embedding/embedding_weights:0'),
-        tuple([v.name for v in global_vars]))
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(embedding_values_a, global_vars[0].eval(session=sess))
-      self.assertAllEqual(embedding_values_b, global_vars[1].eval(session=sess))
-      self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_numeric_column(self):
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[0.], [1]]
-        # example 1, [[10.]]
-        indices=((0, 0), (0, 1), (1, 0)),
-        values=(0., 1., 10.),
-        dense_shape=(2, 2))
-    expected_input_layer = [
-        [[0.], [1.]],
-        [[10.], [0.]],
-    ]
-    expected_sequence_length = [2, 1]
-    numeric_column = sfc.sequence_numeric_column('aaa')
-
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_numeric_column_multi_dim(self):
-    """Tests sequence_input_layer for multi-dimensional numeric_column."""
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[[0., 1.],  [2., 3.]], [[4., 5.],  [6., 7.]]]
-        # example 1, [[[10., 11.],  [12., 13.]]]
-        indices=((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7),
-                 (1, 0), (1, 1), (1, 2), (1, 3)),
-        values=(0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.),
-        dense_shape=(2, 8))
-    # The output of numeric_column._get_dense_tensor should be flattened.
-    expected_input_layer = [
-        [[0., 1., 2., 3.], [4., 5., 6., 7.]],
-        [[10., 11., 12., 13.], [0., 0., 0., 0.]],
-    ]
-    expected_sequence_length = [2, 1]
-    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
-
-    input_layer, sequence_length = sfc.sequence_input_layer(
-        features={'aaa': sparse_input},
-        feature_columns=[numeric_column])
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(expected_input_layer, input_layer.eval(session=sess))
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-
-def _assert_sparse_tensor_value(test_case, expected, actual):
-  test_case.assertEqual(np.int64, np.array(actual.indices).dtype)
-  test_case.assertAllEqual(expected.indices, actual.indices)
-
-  test_case.assertEqual(
-      np.array(expected.values).dtype, np.array(actual.values).dtype)
-  test_case.assertAllEqual(expected.values, actual.values)
-
-  test_case.assertEqual(np.int64, np.array(actual.dense_shape).dtype)
-  test_case.assertAllEqual(expected.dense_shape, actual.dense_shape)
-
-
-class SequenceCategoricalColumnWithIdentityTest(test.TestCase):
-
-  def test_get_sparse_tensors(self):
-    column = sfc.sequence_categorical_column_with_identity(
-        'aaa', num_buckets=3)
-    inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(1, 2, 0),
-        dense_shape=(2, 2))
-    expected_sparse_ids = sparse_tensor.SparseTensorValue(
-        indices=((0, 0, 0), (1, 0, 0), (1, 1, 0)),
-        values=np.array((1, 2, 0), dtype=np.int64),
-        dense_shape=(2, 2, 1))
-
-    id_weight_pair = column._get_sparse_tensors(_LazyBuilder({'aaa': inputs}))
-
-    self.assertIsNone(id_weight_pair.weight_tensor)
-    with monitored_session.MonitoredSession() as sess:
-      _assert_sparse_tensor_value(
-          self,
-          expected_sparse_ids,
-          id_weight_pair.id_tensor.eval(session=sess))
-
-  def test_get_sparse_tensors_inputs3d(self):
-    """Tests _get_sparse_tensors when the input is already 3D Tensor."""
-    column = sfc.sequence_categorical_column_with_identity(
-        'aaa', num_buckets=3)
-    inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0, 0), (1, 0, 0), (1, 1, 0)),
-        values=(1, 2, 0),
-        dense_shape=(2, 2, 1))
-
-    with self.assertRaisesRegexp(
-        errors.InvalidArgumentError,
-        r'Column aaa expected ID tensor of rank 2\.\s*'
-        r'id_tensor shape:\s*\[2 2 1\]'):
-      id_weight_pair = column._get_sparse_tensors(
-          _LazyBuilder({'aaa': inputs}))
-      with monitored_session.MonitoredSession() as sess:
-        id_weight_pair.id_tensor.eval(session=sess)
-
-  def test_sequence_length(self):
-    column = sfc.sequence_categorical_column_with_identity(
-        'aaa', num_buckets=3)
-    inputs = sparse_tensor.SparseTensorValue(
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(1, 2, 0),
-        dense_shape=(2, 2))
-    expected_sequence_length = [1, 2]
-
-    sequence_length = column._sequence_length(_LazyBuilder({'aaa': inputs}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_sequence_length_with_zeros(self):
-    column = sfc.sequence_categorical_column_with_identity(
-        'aaa', num_buckets=3)
-    inputs = sparse_tensor.SparseTensorValue(
-        indices=((1, 0), (3, 0), (3, 1)),
-        values=(1, 2, 0),
-        dense_shape=(5, 2))
-    expected_sequence_length = [0, 1, 0, 2, 0]
-
-    sequence_length = column._sequence_length(_LazyBuilder({'aaa': inputs}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-
-class SequenceEmbeddingColumnTest(test.TestCase):
-
-  def test_get_sequence_dense_tensor(self):
-    vocabulary_size = 3
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, ids [2]
-        # example 1, ids [0, 1]
-        # example 2, ids []
-        # example 3, ids [1]
-        indices=((0, 0), (1, 0), (1, 1), (3, 0)),
-        values=(2, 0, 1, 1),
-        dense_shape=(4, 2))
-
-    embedding_dimension = 2
-    embedding_values = (
-        (1., 2.),  # id 0
-        (3., 5.),  # id 1
-        (7., 11.)  # id 2
-    )
-    def _initializer(shape, dtype, partition_info):
-      self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
-      self.assertEqual(dtypes.float32, dtype)
-      self.assertIsNone(partition_info)
-      return embedding_values
-
-    expected_lookups = [
-        # example 0, ids [2]
-        [[7., 11.], [0., 0.]],
-        # example 1, ids [0, 1]
-        [[1., 2.], [3., 5.]],
-        # example 2, ids []
-        [[0., 0.], [0., 0.]],
-        # example 3, ids [1]
-        [[3., 5.], [0., 0.]],
-    ]
-
-    categorical_column = sfc.sequence_categorical_column_with_identity(
-        key='aaa', num_buckets=vocabulary_size)
-    embedding_column = sfc._sequence_embedding_column(
-        categorical_column, dimension=embedding_dimension,
-        initializer=_initializer)
-
-    embedding_lookup, _ = embedding_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(embedding_values, global_vars[0].eval(session=sess))
-      self.assertAllEqual(expected_lookups, embedding_lookup.eval(session=sess))
-
-  def test_sequence_length(self):
-    vocabulary_size = 3
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, ids [2]
-        # example 1, ids [0, 1]
-        indices=((0, 0), (1, 0), (1, 1)),
-        values=(2, 0, 1),
-        dense_shape=(2, 2))
-    expected_sequence_length = [1, 2]
-
-    categorical_column = sfc.sequence_categorical_column_with_identity(
-        key='aaa', num_buckets=vocabulary_size)
-    embedding_column = sfc._sequence_embedding_column(
-        categorical_column, dimension=2)
-
-    _, sequence_length = embedding_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_sequence_length_with_empty_rows(self):
-    """Tests _sequence_length when some examples do not have ids."""
-    vocabulary_size = 3
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, ids []
-        # example 1, ids [2]
-        # example 2, ids [0, 1]
-        # example 3, ids []
-        # example 4, ids [1]
-        # example 5, ids []
-        indices=((1, 0), (2, 0), (2, 1), (4, 0)),
-        values=(2, 0, 1, 1),
-        dense_shape=(6, 2))
-    expected_sequence_length = [0, 1, 2, 0, 1, 0]
-
-    categorical_column = sfc.sequence_categorical_column_with_identity(
-        key='aaa', num_buckets=vocabulary_size)
-    embedding_column = sfc._sequence_embedding_column(
-        categorical_column, dimension=2)
-
-    _, sequence_length = embedding_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-
-class SequenceNumericColumnTest(test.TestCase):
-
-  def test_get_sequence_dense_tensor(self):
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[0.], [1]]
-        # example 1, [[10.]]
-        indices=((0, 0), (0, 1), (1, 0)),
-        values=(0., 1., 10.),
-        dense_shape=(2, 2))
-    expected_dense_tensor = [
-        [[0.], [1.]],
-        [[10.], [0.]],
-    ]
-    numeric_column = sfc.sequence_numeric_column('aaa')
-
-    dense_tensor, _ = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_dense_tensor, dense_tensor.eval(session=sess))
-
-  def test_get_sequence_dense_tensor_with_shape(self):
-    """Tests get_sequence_dense_tensor with shape !=(1,)."""
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[0., 1., 2.], [3., 4., 5.]]
-        # example 1, [[10., 11., 12.]]
-        indices=((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5),
-                 (1, 0), (1, 1), (1, 2)),
-        values=(0., 1., 2., 3., 4., 5., 10., 11., 12.),
-        dense_shape=(2, 6))
-    expected_dense_tensor = [
-        [[0., 1., 2.], [3., 4., 5.]],
-        [[10., 11., 12.], [0., 0., 0.]],
-    ]
-    numeric_column = sfc.sequence_numeric_column('aaa', shape=(3,))
-
-    dense_tensor, _ = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_dense_tensor, dense_tensor.eval(session=sess))
-
-  def test_get_dense_tensor_multi_dim(self):
-    """Tests get_sequence_dense_tensor for multi-dim numeric_column."""
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[[0., 1.],  [2., 3.]], [[4., 5.],  [6., 7.]]]
-        # example 1, [[[10., 11.],  [12., 13.]]]
-        indices=((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5), (0, 6), (0, 7),
-                 (1, 0), (1, 1), (1, 2), (1, 3)),
-        values=(0., 1., 2., 3., 4., 5., 6., 7., 10., 11., 12., 13.),
-        dense_shape=(2, 8))
-    expected_dense_tensor = [
-        [[[0., 1.], [2., 3.]], [[4., 5.], [6., 7.]]],
-        [[[10., 11.], [12., 13.]], [[0., 0.], [0., 0.]]],
-    ]
-    numeric_column = sfc.sequence_numeric_column('aaa', shape=(2, 2))
-
-    dense_tensor, _ = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_dense_tensor, dense_tensor.eval(session=sess))
-
-  def test_sequence_length(self):
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[0., 1., 2.], [3., 4., 5.]]
-        # example 1, [[10., 11., 12.]]
-        indices=((0, 0), (0, 1), (0, 2), (0, 3), (0, 4), (0, 5),
-                 (1, 0), (1, 1), (1, 2)),
-        values=(0., 1., 2., 3., 4., 5., 10., 11., 12.),
-        dense_shape=(2, 6))
-    expected_sequence_length = [2, 1]
-    numeric_column = sfc.sequence_numeric_column('aaa', shape=(3,))
-
-    _, sequence_length = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_sequence_length_with_shape(self):
-    """Tests _sequence_length with shape !=(1,)."""
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values [[0.], [1]]
-        # example 1, [[10.]]
-        indices=((0, 0), (0, 1), (1, 0)),
-        values=(0., 1., 10.),
-        dense_shape=(2, 2))
-    expected_sequence_length = [2, 1]
-    numeric_column = sfc.sequence_numeric_column('aaa')
-
-    _, sequence_length = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-  def test_sequence_length_with_empty_rows(self):
-    """Tests _sequence_length when some examples do not have ids."""
-    sparse_input = sparse_tensor.SparseTensorValue(
-        # example 0, values []
-        # example 1, values [[0.], [1.]]
-        # example 2, [[2.]]
-        # example 3, values []
-        # example 4, [[3.]]
-        # example 5, values []
-        indices=((1, 0), (1, 1), (2, 0), (4, 0)),
-        values=(0., 1., 2., 3.),
-        dense_shape=(6, 2))
-    expected_sequence_length = [0, 2, 1, 0, 1, 0]
-    numeric_column = sfc.sequence_numeric_column('aaa')
-
-    _, sequence_length = numeric_column._get_sequence_dense_tensor(
-        _LazyBuilder({'aaa': sparse_input}))
-
-    with monitored_session.MonitoredSession() as sess:
-      self.assertAllEqual(
-          expected_sequence_length, sequence_length.eval(session=sess))
-
-
-if __name__ == '__main__':
-  test.main()
-- 
GitLab


From 911225a7eaf2872472484bce5f717d287a0e3224 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 21 Mar 2018 09:59:18 -0700
Subject: [PATCH 298/960] Added an option to run shape analysis assuming the
 shapes of the feed nodes are valid.

PiperOrigin-RevId: 189923541
---
 tensorflow/python/grappler/model_analyzer.cc | 5 +++--
 tensorflow/python/grappler/model_analyzer.h  | 2 +-
 tensorflow/python/grappler/model_analyzer.i  | 8 +++++---
 tensorflow/python/grappler/model_analyzer.py | 5 +++--
 4 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/grappler/model_analyzer.cc b/tensorflow/python/grappler/model_analyzer.cc
index d23eb811ac..5a76cdd8fb 100644
--- a/tensorflow/python/grappler/model_analyzer.cc
+++ b/tensorflow/python/grappler/model_analyzer.cc
@@ -26,9 +26,10 @@ namespace grappler {
 
 ModelAnalyzer::ModelAnalyzer(const GrapplerItem& item) : item_(item) {}
 
-Status ModelAnalyzer::GenerateReport(bool debug, std::ostream& os) {
+Status ModelAnalyzer::GenerateReport(bool debug, bool assume_valid_feeds,
+                                     std::ostream& os) {
   GraphProperties properties(item_);
-  TF_RETURN_IF_ERROR(properties.InferStatically(false));
+  TF_RETURN_IF_ERROR(properties.InferStatically(assume_valid_feeds));
 
   for (const auto& node : item_.MainOpsFanin()) {
     PrintNodeInfo(node, properties, debug, os);
diff --git a/tensorflow/python/grappler/model_analyzer.h b/tensorflow/python/grappler/model_analyzer.h
index 5bc551927d..97ffafabe1 100644
--- a/tensorflow/python/grappler/model_analyzer.h
+++ b/tensorflow/python/grappler/model_analyzer.h
@@ -31,7 +31,7 @@ class GraphProperties;
 class ModelAnalyzer {
  public:
   explicit ModelAnalyzer(const GrapplerItem& item);
-  Status GenerateReport(bool debug, std::ostream& os);
+  Status GenerateReport(bool debug, bool assume_valid_feeds, std::ostream& os);
 
  private:
   void PrintNodeInfo(const NodeDef* node, const GraphProperties& properties,
diff --git a/tensorflow/python/grappler/model_analyzer.i b/tensorflow/python/grappler/model_analyzer.i
index 7c3a692d0e..4955780764 100644
--- a/tensorflow/python/grappler/model_analyzer.i
+++ b/tensorflow/python/grappler/model_analyzer.i
@@ -40,7 +40,8 @@ limitations under the License.
 %}
 
 %{
-string GenerateModelReport(const tensorflow::MetaGraphDef& metagraph, bool debug) {
+string GenerateModelReport(const tensorflow::MetaGraphDef& metagraph,
+                           bool assume_valid_feeds, bool debug) {
   tensorflow::grappler::ItemConfig cfg;
   cfg.apply_optimizations = false;
   std::unique_ptr<tensorflow::grappler::GrapplerItem> item =
@@ -53,10 +54,11 @@ string GenerateModelReport(const tensorflow::MetaGraphDef& metagraph, bool debug
   tensorflow::grappler::ModelAnalyzer analyzer(*item);
 
   std::stringstream os;
-  analyzer.GenerateReport(debug, os);
+  analyzer.GenerateReport(debug, assume_valid_feeds, os);
   return os.str();
 }
 
 %}
 
-string GenerateModelReport(const tensorflow::MetaGraphDef& metagraph, bool debug);
+string GenerateModelReport(const tensorflow::MetaGraphDef& metagraph,
+                           bool assume_valid_feeds, bool debug);
diff --git a/tensorflow/python/grappler/model_analyzer.py b/tensorflow/python/grappler/model_analyzer.py
index 535889e1c4..98cdc57850 100644
--- a/tensorflow/python/grappler/model_analyzer.py
+++ b/tensorflow/python/grappler/model_analyzer.py
@@ -22,11 +22,12 @@ from tensorflow.python import pywrap_tensorflow as tf_wrap
 from tensorflow.python.framework import errors
 
 
-def GenerateModelReport(metagraph, debug=False):
+def GenerateModelReport(metagraph, assume_valid_feeds=True, debug=False):
   """Report what's known statically about each node in the provided metagraph.
 
   Args:
     metagraph: A TensorFlow MetaGraphDef.
+    assume_valid_feeds: If True, assume that the shape of the fed nodes is valid
     debug: Add some information useful for debugging.
 
   Returns:
@@ -34,6 +35,6 @@ def GenerateModelReport(metagraph, debug=False):
   """
   with errors.raise_exception_on_not_ok_status():
     ret_from_swig = tf_wrap.GenerateModelReport(metagraph.SerializeToString(),
-                                                debug)
+                                                assume_valid_feeds, debug)
 
   return ret_from_swig
-- 
GitLab


From d854706bb3ccbcd3808ed5d89cb4b094634614ef Mon Sep 17 00:00:00 2001
From: Jingyue Wu <jingyue@google.com>
Date: Wed, 21 Mar 2018 10:20:25 -0700
Subject: [PATCH 299/960] Collapse adjacent dimensions that have no paddings.

For example,

tf.pad(<4D tensor>, [[0, 0], [0, 0], [0, 0], [0, 1]])

is equivalent to a 2D pad, which is faster.

PiperOrigin-RevId: 189926996
---
 tensorflow/core/kernels/pad_op.cc             | 127 ++++++++++++++++--
 tensorflow/python/kernel_tests/pad_op_test.py |  27 ++++
 2 files changed, 143 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/kernels/pad_op.cc b/tensorflow/core/kernels/pad_op.cc
index a7238ef67b..41494f56c5 100644
--- a/tensorflow/core/kernels/pad_op.cc
+++ b/tensorflow/core/kernels/pad_op.cc
@@ -104,42 +104,147 @@ class PadOp : public OpKernel {
       return;
     }
 
-    Tensor* output = nullptr;
-    OP_REQUIRES_OK(context, context->allocate_output(0, output_shape, &output));
+    TensorShape collapsed_input_shape;
+    TensorShape collapsed_output_shape;
+    Tensor collapsed_paddings;
+    if (fixed_dims > 1 &&
+        CollapseAdjacentNonPaddedDimensions(
+            in0.shape(), in1, output_shape, &collapsed_input_shape,
+            &collapsed_paddings, &collapsed_output_shape)) {
+      Tensor collapsed_input;
+      CHECK(collapsed_input.CopyFrom(in0, collapsed_input_shape));
+      Tensor collapsed_output;
+      AllocatorAttributes alloc_attrs;
+      alloc_attrs.set_on_host(context->input_memory_type(0) == HOST_MEMORY);
+      OP_REQUIRES_OK(context,
+                     context->allocate_temp(collapsed_input.dtype(),
+                                            collapsed_output_shape,
+                                            &collapsed_output, alloc_attrs));
+      const Tensor& collapsed_paddings_ref = collapsed_paddings;
+      typename TTypes<Tpadding>::ConstMatrix collapsed_paddings_matrix =
+          collapsed_paddings_ref.matrix<Tpadding>();
 
+      OperateWithVariableRank(context, collapsed_input_shape.dims(),
+                              collapsed_input, collapsed_paddings_matrix,
+                              pad_value, &collapsed_output);
+
+      Tensor output;
+      CHECK(output.CopyFrom(collapsed_output, output_shape));
+      context->set_output(0, output);
+    } else {
+      Tensor* output = nullptr;
+      OP_REQUIRES_OK(context,
+                     context->allocate_output(0, output_shape, &output));
+      OperateWithVariableRank(context, fixed_dims, in0, paddings, pad_value,
+                              output);
+    }
+  }
+
+ private:
+  // Collapses adjacent dimensions that are not padded to one dimension for
+  // speed. Returns true if any two dimensions are collapsed. For example,
+  //
+  //   Pad(input_shape=[8, 28, 28, 3],
+  //       paddings=[[0, 0], [0, 0], [0, 0], [0, 1]]
+  // is equivalent to
+  //   Pad(input_shape=[6272, 3],
+  //       paddings=[[0, 0], [0, 1]])
+  //
+  // input_shape: the original input shape.
+  // paddings_as_tensor: the original paddings.
+  // output_shape: the original output shape.
+  // collapsed_input_shape: the input shape after collapsing.
+  // collapsed_paddings_as_tensor: the paddings after collapsing.
+  // collapsed_output_shape: the output shape after collapsing.
+  static bool CollapseAdjacentNonPaddedDimensions(
+      const TensorShape& input_shape, const Tensor& paddings_as_tensor,
+      const TensorShape& output_shape, TensorShape* collapsed_input_shape,
+      Tensor* collapsed_paddings_as_tensor,
+      TensorShape* collapsed_output_shape) {
+    bool collapsed = false;
+    typename TTypes<Tpadding>::ConstMatrix paddings =
+        paddings_as_tensor.matrix<Tpadding>();
+    std::vector<std::pair<int, int>> collapsed_paddings;
+    int i = 0;
+    while (i < paddings.dimension(0)) {
+      if (paddings(i, 0) != 0 || paddings(i, 1) != 0) {
+        // If padded, copy the original dimension over.
+        collapsed_input_shape->InsertDim(collapsed_input_shape->dims(),
+                                         input_shape.dim_size(i));
+        collapsed_output_shape->InsertDim(collapsed_output_shape->dims(),
+                                          output_shape.dim_size(i));
+        collapsed_paddings.push_back({paddings(i, 0), paddings(i, 1)});
+        ++i;
+      } else {
+        // If not padded, find the next dimension that is padded and collapse
+        // all dimensions in between to one dimension.
+        int64 collapsed_input_dim_size = input_shape.dim_size(i);
+        int64 collapsed_output_dim_size = output_shape.dim_size(i);
+        ++i;
+        while (i < paddings.dimension(0) && paddings(i, 0) == 0 &&
+               paddings(i, 1) == 0) {
+          collapsed = true;
+          collapsed_input_dim_size *= input_shape.dim_size(i);
+          collapsed_output_dim_size *= output_shape.dim_size(i);
+          ++i;
+        }
+        collapsed_input_shape->InsertDim(collapsed_input_shape->dims(),
+                                         collapsed_input_dim_size);
+        collapsed_output_shape->InsertDim(collapsed_output_shape->dims(),
+                                          collapsed_output_dim_size);
+        collapsed_paddings.push_back({0, 0});
+      }
+    }
+
+    // Copy collapsed_paddings to collapsed_paddings_as_tensor.
+    *collapsed_paddings_as_tensor =
+        Tensor(paddings_as_tensor.dtype(),
+               TensorShape({static_cast<int64>(collapsed_paddings.size()), 2}));
+    auto collapsed_paddings_as_matrix =
+        collapsed_paddings_as_tensor->matrix<Tpadding>();
+    for (size_t i = 0; i < collapsed_paddings.size(); ++i) {
+      collapsed_paddings_as_matrix(i, 0) = collapsed_paddings[i].first;
+      collapsed_paddings_as_matrix(i, 1) = collapsed_paddings[i].second;
+    }
+    return collapsed;
+  }
+
+  void OperateWithVariableRank(OpKernelContext* context, int fixed_dims,
+                               const Tensor& input,
+                               typename TTypes<Tpadding>::ConstMatrix paddings,
+                               T pad_value, Tensor* output) {
     // Invoke the dims-specific implementation.
     switch (fixed_dims) {
       case 0:
-        Operate<0>(context, in0.tensor<T, 0>(), paddings, pad_value, output);
+        Operate<0>(context, input.tensor<T, 0>(), paddings, pad_value, output);
         break;
       case 1:
         // TODO(irving): Once Pad doesn't need a scalar special case,
         // change flat to tensor.  That is, once !allow_legacy_scalars().
-        Operate<1>(context, in0.flat<T>(), paddings, pad_value, output);
+        Operate<1>(context, input.flat<T>(), paddings, pad_value, output);
         break;
       case 2:
-        Operate<2>(context, in0.tensor<T, 2>(), paddings, pad_value, output);
+        Operate<2>(context, input.tensor<T, 2>(), paddings, pad_value, output);
         break;
       case 3:
-        Operate<3>(context, in0.tensor<T, 3>(), paddings, pad_value, output);
+        Operate<3>(context, input.tensor<T, 3>(), paddings, pad_value, output);
         break;
       case 4:
-        Operate<4>(context, in0.tensor<T, 4>(), paddings, pad_value, output);
+        Operate<4>(context, input.tensor<T, 4>(), paddings, pad_value, output);
         break;
       case 5:
-        Operate<5>(context, in0.tensor<T, 5>(), paddings, pad_value, output);
+        Operate<5>(context, input.tensor<T, 5>(), paddings, pad_value, output);
         break;
       case 6:
-        Operate<6>(context, in0.tensor<T, 6>(), paddings, pad_value, output);
+        Operate<6>(context, input.tensor<T, 6>(), paddings, pad_value, output);
         break;
       default:
         OP_REQUIRES(context, false,
                     errors::InvalidArgument("Only ranks up to 6 supported: ",
-                                            in0.shape().DebugString()));
+                                            input.shape().DebugString()));
     }
   }
 
- private:
   template <int Dims>
   void Operate(OpKernelContext* context,
                typename TTypes<T, Dims>::ConstTensor input,
diff --git a/tensorflow/python/kernel_tests/pad_op_test.py b/tensorflow/python/kernel_tests/pad_op_test.py
index 9ed5947aae..361853448c 100644
--- a/tensorflow/python/kernel_tests/pad_op_test.py
+++ b/tensorflow/python/kernel_tests/pad_op_test.py
@@ -336,5 +336,32 @@ class PadOpTest(test.TestCase):
       self.assertAllEqual(inp, out)
       self.assertShapeEqual(inp, tf_val)
 
+  def testCollapseAdjacentNonPaddedDimensions(self):
+    # pyformat: disable
+    paddings_values = [[[0, 0], [0, 0], [0, 0], [0, 1]],
+                       [[0, 0], [2, 3], [0, 0], [0, 0]],
+                       [[0, 0], [0, 0], [0, 0], [0, 0]]]
+    # pyformat: enable
+    for paddings_value in paddings_values:
+      for dtype in [dtypes.float32, dtypes.int32]:
+        inp = constant_op.constant(1, shape=[8, 28, 28, 3], dtype=dtype)
+        paddings = constant_op.constant(paddings_value, dtype=dtypes.int32)
+        padded = array_ops.pad(inp, paddings)
+        middle = array_ops.slice(padded, [row[0] for row in paddings_value],
+                                 [dim.value for dim in inp.shape.dims])
+        left = array_ops.slice(padded, [0, 0, 0, 0],
+                               [row[0] for row in paddings_value])
+        right = array_ops.slice(
+            padded,
+            [paddings_value[i][0] + inp.shape.dims[i].value for i in range(4)],
+            [-1, -1, -1, -1])
+        with self.test_session(use_gpu=True):
+          self.assertAllEqual(inp.eval(), middle.eval())
+          self.assertAllEqual(
+              np.zeros([row[0] for row in paddings_value]), left.eval())
+          self.assertAllEqual(
+              np.zeros([row[1] for row in paddings_value]), right.eval())
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From e50fb3f561f1bfcd0a5fb457c69d50da64c789f8 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 21 Mar 2018 10:26:49 -0700
Subject: [PATCH 300/960] Fix zipfile path for MacOS builds.

For some reason, the zipfile module on Macs appears to work differently and
complains about the whl file we are trying to extract not being found.

PiperOrigin-RevId: 189928007
---
 tensorflow/tools/ci_build/copy_binary.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/copy_binary.py b/tensorflow/tools/ci_build/copy_binary.py
index b5a282b64a..420d390d2b 100755
--- a/tensorflow/tools/ci_build/copy_binary.py
+++ b/tensorflow/tools/ci_build/copy_binary.py
@@ -60,7 +60,7 @@ def copy_binary(directory, origin_tag, new_tag, version, gpu=False):
     package = "tf_nightly"
   origin_binary = BINARY_STRING_TEMPLATE % (package, version, origin_tag)
   new_binary = BINARY_STRING_TEMPLATE % (package, version, new_tag)
-  zip_ref = zipfile.ZipFile(directory + origin_binary, "r")
+  zip_ref = zipfile.ZipFile(os.path.join(directory, origin_binary), "r")
 
   try:
     tmpdir = tempfile.mkdtemp()
@@ -115,6 +115,7 @@ def main():
   args = parser.parse_args()
 
   # Argument checking
+  args.filename = os.path.abspath(args.filename)
   check_existence(args.filename)
   regex_groups = re.search(TF_NIGHTLY_REGEX, args.filename)
   directory = regex_groups.group(1)
-- 
GitLab


From d7cb36a6876e02540c13f31f468a84f54c8591d4 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 21 Mar 2018 10:47:18 -0700
Subject: [PATCH 301/960] [XLA:GPU] Don't crash if a GTE feeds into a bitcast.

GTE and bitcast are sort of "implicitly fused", so we have to handle
them in this way.

PiperOrigin-RevId: 189931422
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 19 +++++++++---
 tensorflow/compiler/xla/tests/BUILD           |  1 +
 .../compiler/xla/tests/hlo_test_base.cc       |  7 +++++
 tensorflow/compiler/xla/tests/hlo_test_base.h |  6 ++++
 tensorflow/compiler/xla/tests/tuple_test.cc   | 29 +++++++++++++++++++
 5 files changed, 58 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 135a607ab9..199e6b7874 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -1979,11 +1979,22 @@ GetHloBufferSlices(const HloInstruction* hlo,
       }
     }
 
-    // If *that* didn't work, check whether instr is a GTE instruction.  If it
-    // is, see if we can get a buffer for its parent, and continue walking up
-    // parents until we find a defined buffer or we hit something that's not a
-    // GTE.
+    // If *that* didn't work, walk up any bitcasts that we might see.  These
+    // must appear before any GTE instructions, because it's illegal to bitcast
+    // to a tuple type.
     const HloInstruction* parent = instr;
+    while (parent->opcode() == HloOpcode::kBitcast) {
+      parent = parent->operand(0);
+
+      auto slice = GetKnownAtRuntimeSlice(parent, {}, buffer_assn);
+      if (slice.has_value()) {
+        return {{*slice, gte_indices}};
+      }
+    }
+
+    // Finally, check whether instr is a GTE instruction.  If it is, see if we
+    // can get a buffer for its parent, and continue walking up parents until we
+    // find a defined buffer or we hit something that's not a GTE.
     while (parent->opcode() == HloOpcode::kGetTupleElement) {
       gte_indices.push_front(parent->tuple_index());
       parent = parent->operand(0);
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 04a9c1ef79..7fb7919674 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1009,6 +1009,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
         "//tensorflow/core:test",
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.cc b/tensorflow/compiler/xla/tests/hlo_test_base.cc
index 5f62c44f25..e574644dea 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.cc
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.cc
@@ -115,6 +115,13 @@ StatusOr<std::unique_ptr<Literal>> HloTestBase::Execute(
   return test_runner_.Execute(std::move(module), arguments);
 }
 
+StatusOr<std::unique_ptr<Literal>> HloTestBase::ExecuteNoHloPasses(
+    std::unique_ptr<HloModule> module,
+    tensorflow::gtl::ArraySlice<Literal*> arguments) {
+  return test_runner_.Execute(std::move(module), arguments,
+                              /*run_hlo_passes=*/false);
+}
+
 std::unique_ptr<Literal> HloTestBase::ExecuteAndTransfer(
     std::unique_ptr<HloModule> module,
     tensorflow::gtl::ArraySlice<Literal*> arguments) {
diff --git a/tensorflow/compiler/xla/tests/hlo_test_base.h b/tensorflow/compiler/xla/tests/hlo_test_base.h
index e375f13a44..3e8e2360bb 100644
--- a/tensorflow/compiler/xla/tests/hlo_test_base.h
+++ b/tensorflow/compiler/xla/tests/hlo_test_base.h
@@ -98,6 +98,12 @@ class HloTestBase : public ::testing::Test {
       std::unique_ptr<HloModule> module,
       tensorflow::gtl::ArraySlice<Literal*> arguments);
 
+  // Same as above, except the module will be executed without running any HLO
+  // passes on it.
+  StatusOr<std::unique_ptr<Literal>> ExecuteNoHloPasses(
+      std::unique_ptr<HloModule> module,
+      tensorflow::gtl::ArraySlice<Literal*> arguments);
+
   std::unique_ptr<Literal> ExecuteAndTransfer(
       std::unique_ptr<HloModule> module,
       tensorflow::gtl::ArraySlice<Literal*> arguments);
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index 2029312f94..fa60af4b6a 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
@@ -514,5 +515,33 @@ XLA_TEST_F(TupleTest, ComplexTuples) {
                          error_spec_);
 }
 
+class TupleHloTest : public HloTestBase {};
+
+// Disabled on CPU parallel because that's broken and will be removed soon.
+// Disabled on the interpreter because bitcast doesn't exist on the interpreter.
+TEST_F(TupleHloTest,
+       DISABLED_ON_INTERPRETER(DISABLED_ON_CPU_PARALLEL(BitcastAfterGTE))) {
+  const char* testcase = R"(
+    HloModule m
+
+    ENTRY test {
+      name.1 = (f32[3]{0}) parameter(0)
+      get-tuple-element.1 = f32[3]{0} get-tuple-element(name.1), index=0
+      bitcast = f32[1,3]{1,0} bitcast(get-tuple-element.1)
+      copy = f32[1,3]{1,0} copy(bitcast)
+      ROOT tuple.4 = (f32[1,3]{1,0}) tuple(copy)
+    }
+  )";
+  auto module =
+      HloRunner::CreateModuleFromString(testcase, GetDebugOptionsForTest())
+          .ValueOrDie();
+  auto param = Literal::MakeTupleOwned(Literal::CreateR1<float>({1, 2, 3}));
+  TF_ASSERT_OK_AND_ASSIGN(auto result,
+                          ExecuteNoHloPasses(std::move(module), {param.get()}));
+  EXPECT_TRUE(LiteralTestUtil::Equal(
+      *result,
+      *Literal::MakeTupleOwned(Literal::CreateR2<float>({{1, 2, 3}}))));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 53f823c1273c7670fb5c337ae7ac2e9647a1fa4f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 11:01:54 -0700
Subject: [PATCH 302/960] Update the doc to reflect the change of replacing
 std::clock with random::New64() as random number generator seed.

PiperOrigin-RevId: 189934377
---
 tensorflow/contrib/tensor_forest/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tensor_forest/README.md b/tensorflow/contrib/tensor_forest/README.md
index 8b24430c71..9e1491ea66 100644
--- a/tensorflow/contrib/tensor_forest/README.md
+++ b/tensorflow/contrib/tensor_forest/README.md
@@ -116,7 +116,7 @@ a different `feature_bagging_fraction * num_features` sized subset of the
 input features.  Defaults to 1.0 (no feature bagging).
 
 * `base_random_seed`.  By default (`base_random_seed = 0`), the random number
-generator for each tree is seeded by the current time (in microseconds) when
+generator for each tree is seeded by a 64-bit random value when
 each tree is first created.  Using a non-zero value causes tree training to
 be deterministic, in that the i-th tree's random number generator is seeded
 with the value `base_random_seed + i`.
-- 
GitLab


From a0d3ce1de30735dd8d0ed8f95a6eb4d0c3e7773b Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Wed, 21 Mar 2018 11:10:13 -0700
Subject: [PATCH 303/960] Adding `drop_remainder` option for the
 `map_and_batch` transformation, which allows the user to express whether they
 wish to drop the last batch if its size is smaller than desired; the default
 is not to drop the smaller batch.

PiperOrigin-RevId: 189936029
---
 .../kernel_tests/batch_dataset_op_test.py     | 42 ++++++++++++-------
 .../contrib/data/python/ops/batching.py       | 30 +++++++++----
 .../kernels/data/map_and_batch_dataset_op.cc  | 17 +++++++-
 .../core/ops/compat/ops_history.v1.pbtxt      |  4 ++
 tensorflow/core/ops/dataset_ops.cc            |  1 +
 5 files changed, 69 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index a2da953c7b..5abb38c2d2 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -311,10 +311,10 @@ class BatchDatasetTest(test.TestCase):
     self.assertEqual([None], dataset.output_shapes[1][0].as_list())
     self.assertEqual([None, 30], dataset.output_shapes[1][1].as_list())
 
-  def _testBatchAndMapDatasetHelper(self, num_parallel_batches=1):
+  def _testMapAndBatchDatasetHelper(self, num_parallel_batches=1):
     """Test a dataset that maps a TF function across its input elements."""
     # The pipeline is TensorSliceDataset ->
-    # RepeatDataset(count) -> BatchAndMapDataset(square_3, batch_size).
+    # RepeatDataset(count) -> MapAndBatchDataset(square_3, batch_size).
     components = (np.arange(7),
                   np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
                   np.array(37.0) * np.arange(7))
@@ -381,26 +381,38 @@ class BatchDatasetTest(test.TestCase):
       with self.assertRaises(errors.InvalidArgumentError):
         sess.run(init_op, feed_dict={count: 14, batch_size: 0})
 
-  def testBatchAndMapDataset(self):
-    return self._testBatchAndMapDatasetHelper()
+  def testMapAndBatchDataset(self):
+    return self._testMapAndBatchDatasetHelper()
 
-  def testBatchAndMapDatasetWithParallelBatching(self):
-    return self._testBatchAndMapDatasetHelper(num_parallel_batches=10)
+  def testMapAndBatchDatasetWithParallelBatching(self):
+    return self._testMapAndBatchDatasetHelper(num_parallel_batches=10)
 
-  def testMapAndBatchYieldsPartialBatch(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .apply(batching.map_and_batch(
-                    lambda x: array_ops.reshape(x * x, [1]), 4))
-                .make_one_shot_iterator())
-    self.assertEqual([None, 1], iterator.output_shapes.as_list())
+  def _testMapAndBatchPartialBatchHelper(self, drop_remainder=False):
+    iterator = (
+        dataset_ops.Dataset.range(10).apply(
+            batching.map_and_batch(
+                lambda x: array_ops.reshape(x * x, [1]),
+                batch_size=4,
+                drop_remainder=drop_remainder)).make_one_shot_iterator())
+    if drop_remainder:
+      self.assertEqual([4, 1], iterator.output_shapes.as_list())
+    else:
+      self.assertEqual([None, 1], iterator.output_shapes.as_list())
     next_element = iterator.get_next()
     with self.test_session() as sess:
       self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
       self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      if not drop_remainder:
+        self.assertAllEqual([[64], [81]], sess.run(next_element))
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  def testMapAndBatchPartialBatch(self):
+    return self._testMapAndBatchPartialBatchHelper()
+
+  def testMapAndBatchPartialBatchDropRemainder(self):
+    return self._testMapAndBatchPartialBatchHelper(drop_remainder=True)
+
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
@@ -425,7 +437,7 @@ class BatchDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(get_next)
 
-  def testBatchAndMapDatasetFails(self):
+  def testMapAndBatchDatasetFails(self):
     """Test a dataset that maps a TF function across its input elements."""
     dataset = dataset_ops.Dataset.from_tensors(
         array_ops.check_numerics(
@@ -439,7 +451,7 @@ class BatchDatasetTest(test.TestCase):
       with self.assertRaisesRegexp(errors.InvalidArgumentError, "oops"):
         sess.run(init_op, feed_dict={batch_size: 14})
 
-  def testBatchAndMapDatasetShapeMismatch(self):
+  def testMapAndBatchDatasetShapeMismatch(self):
     """Test a dataset that maps a TF function across its input elements."""
 
     def generator():
diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index 6463d75750..a212adf6cf 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -348,13 +348,19 @@ class _RestructuredDataset(dataset_ops.Dataset):
 class _MapAndBatchDataset(dataset_ops.MapDataset):
   """A `Dataset` that maps a function over a batch of elements."""
 
-  def __init__(self, input_dataset, map_func, batch_size, num_parallel_batches):
+  def __init__(self, input_dataset, map_func, batch_size, num_parallel_batches,
+               drop_remainder):
     """See `Dataset.map()` for details."""
     super(_MapAndBatchDataset, self).__init__(input_dataset, map_func)
-    self._batch_size = ops.convert_to_tensor(
+    self._batch_size_t = ops.convert_to_tensor(
         batch_size, dtype=dtypes.int64, name="batch_size")
-    self._num_parallel_batches = ops.convert_to_tensor(
+    self._num_parallel_batches_t = ops.convert_to_tensor(
         num_parallel_batches, dtype=dtypes.int64, name="num_parallel_batches")
+    self._drop_remainder_t = ops.convert_to_tensor(
+        drop_remainder, dtype=dtypes.bool, name="drop_remainder")
+
+    self._batch_size = batch_size
+    self._drop_remainder = drop_remainder
 
   def _as_variant_tensor(self):
     # pylint: disable=protected-access
@@ -363,8 +369,9 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
         input_resource,
         self._map_func.captured_inputs,
         f=self._map_func,
-        batch_size=self._batch_size,
-        num_parallel_batches=self._num_parallel_batches,
+        batch_size=self._batch_size_t,
+        num_parallel_batches=self._num_parallel_batches_t,
+        drop_remainder=self._drop_remainder_t,
         output_types=nest.flatten(
             sparse.as_dense_types(self.output_types, self.output_classes)),
         output_shapes=nest.flatten(
@@ -373,8 +380,9 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
 
   @property
   def output_shapes(self):
+    dim = self._batch_size if self._drop_remainder else None
     return nest.pack_sequence_as(self._output_shapes, [
-        tensor_shape.vector(None).concatenate(s)
+        tensor_shape.vector(dim).concatenate(s)
         for s in nest.flatten(self._output_shapes)
     ])
 
@@ -383,7 +391,10 @@ class _MapAndBatchDataset(dataset_ops.MapDataset):
     return self._output_types
 
 
-def map_and_batch(map_func, batch_size, num_parallel_batches=1):
+def map_and_batch(map_func,
+                  batch_size,
+                  num_parallel_batches=1,
+                  drop_remainder=False):
   """Fused implementation of `map` and `batch`.
 
   Maps `map_func` across `batch_size` consecutive elements of this dataset
@@ -403,6 +414,9 @@ def map_and_batch(map_func, batch_size, num_parallel_batches=1):
       number of batches to create in parallel. On one hand, higher values can
       help mitigate the effect of stragglers. On the other hand, higher values
       can increase contention if CPU is scarce.
+    drop_remainder: A `tf.bool` scalar `tf.Tensor`, representing whether the
+      last batch should be dropped in case its size is smaller than desired;
+      the default behavior is not to drop the smaller batch.
 
   Returns:
     A `Dataset` transformation function, which can be passed to
@@ -411,6 +425,6 @@ def map_and_batch(map_func, batch_size, num_parallel_batches=1):
 
   def _apply_fn(dataset):
     return _MapAndBatchDataset(dataset, map_func, batch_size,
-                               num_parallel_batches)
+                               num_parallel_batches, drop_remainder)
 
   return _apply_fn
diff --git a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
index e22200f758..aaf4dc7341 100644
--- a/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/map_and_batch_dataset_op.cc
@@ -66,12 +66,16 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
                 errors::InvalidArgument(
                     "num_parallel_batches must be greater than zero."));
 
+    bool drop_remainder;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument(ctx, "drop_remainder", &drop_remainder));
+
     std::unique_ptr<CapturedFunction> captured_func;
     OP_REQUIRES_OK(ctx, CapturedFunction::Create(
                             func_, std::move(other_arguments), &captured_func));
 
     *output = new Dataset(input, batch_size, num_parallel_batches,
-                          output_types_, output_shapes_,
+                          drop_remainder, output_types_, output_shapes_,
                           std::move(captured_func), &ctx->eigen_cpu_device());
   }
 
@@ -79,13 +83,15 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
   class Dataset : public DatasetBase {
    public:
     Dataset(const DatasetBase* input, int64 batch_size,
-            int64 num_parallel_batches, const DataTypeVector& output_types,
+            int64 num_parallel_batches, bool drop_remainder,
+            const DataTypeVector& output_types,
             const std::vector<PartialTensorShape>& output_shapes,
             std::unique_ptr<CapturedFunction> captured_func,
             const Eigen::ThreadPoolDevice* device)
         : input_(input),
           batch_size_(batch_size),
           num_parallel_batches_(num_parallel_batches),
+          drop_remainder_(drop_remainder),
           output_types_(output_types),
           output_shapes_(output_shapes),
           captured_func_(std::move(captured_func)),
@@ -177,6 +183,12 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
           batch_results_[current_batch_index_].output.clear();
         } else {
           if (num_elements < dataset()->batch_size_) {
+            if (dataset()->drop_remainder_) {
+              // Deallocate tensors allocated for the output.
+              batch_results_[current_batch_index_].output.clear();
+              *end_of_sequence = true;
+              return Status::OK();
+            }
             const std::vector<Tensor>& output =
                 batch_results_[current_batch_index_].output;
             for (size_t i = 0; i < output.size(); ++i) {
@@ -392,6 +404,7 @@ class MapAndBatchDatasetOp : public UnaryDatasetOpKernel {
     const NameAttrList func_;
     const int64 batch_size_;
     const int64 num_parallel_batches_;
+    const bool drop_remainder_;
     const DataTypeVector output_types_;
     const std::vector<PartialTensorShape> output_shapes_;
     const std::unique_ptr<CapturedFunction> captured_func_;
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 992e943966..ddf7627463 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -24383,6 +24383,10 @@ op {
     name: "num_parallel_batches"
     type: DT_INT64
   }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
   output_arg {
     name: "handle"
     type: DT_VARIANT
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index bdbbf6d7c3..f32baee45e 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -166,6 +166,7 @@ REGISTER_OP("MapAndBatchDataset")
     .Input("other_arguments: Targuments")
     .Input("batch_size: int64")
     .Input("num_parallel_batches: int64")
+    .Input("drop_remainder: bool")
     .Output("handle: variant")
     .Attr("f: func")
     .Attr("Targuments: list(type) >= 0")
-- 
GitLab


From 0eaec2864b737b7a278b028a1719d062470f3397 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 21 Mar 2018 11:12:28 -0700
Subject: [PATCH 304/960] Moves KernelAndDevice to common_runtime

PiperOrigin-RevId: 189936396
---
 tensorflow/BUILD                              |   1 +
 tensorflow/c/eager/BUILD                      |   5 +
 tensorflow/c/eager/c_api_internal.h           |   1 +
 tensorflow/c/eager/runtime.cc                 | 133 +++--------------
 tensorflow/c/eager/runtime.h                  |  52 +------
 tensorflow/c/eager/runtime_test.cc            | 129 ----------------
 tensorflow/core/BUILD                         |   5 +-
 tensorflow/core/common_runtime/eager/BUILD    |  86 +++++++++++
 .../common_runtime/eager/kernel_and_device.cc | 132 +++++++++++++++++
 .../common_runtime/eager/kernel_and_device.h  |  85 +++++++++++
 .../eager/kernel_and_device_test.cc           | 140 ++++++++++++++++++
 11 files changed, 474 insertions(+), 295 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/BUILD
 create mode 100644 tensorflow/core/common_runtime/eager/kernel_and_device.cc
 create mode 100644 tensorflow/core/common_runtime/eager/kernel_and_device.h
 create mode 100644 tensorflow/core/common_runtime/eager/kernel_and_device_test.cc

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 057ac79961..e0d86997ff 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -591,6 +591,7 @@ filegroup(
         "//tensorflow/contrib/verbs:all_files",
         "//tensorflow/core:all_files",
         "//tensorflow/core/api_def:all_files",
+        "//tensorflow/core/common_runtime/eager:all_files",
         "//tensorflow/core/debug:all_files",
         "//tensorflow/core/distributed_runtime:all_files",
         "//tensorflow/core/distributed_runtime/rpc:all_files",
diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 73a3450e0e..841ff48a38 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -28,6 +28,8 @@ tf_cuda_library(
             "//tensorflow/c:c_api",
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
+            "//tensorflow/core/common_runtime/eager:eager_executor",
+            "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core:core_cpu_internal",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
@@ -62,6 +64,8 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core/common_runtime/eager:eager_executor",
+        "//tensorflow/core/common_runtime/eager:kernel_and_device",
     ],
 )
 
@@ -96,6 +100,7 @@ tf_cuda_library(
         "//conditions:default": [
             "//tensorflow/c:c_api",
             "//tensorflow/core:core_cpu",
+            "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core:core_cpu_internal",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index cc5ed48b48..a79f8ddd33 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
diff --git a/tensorflow/c/eager/runtime.cc b/tensorflow/c/eager/runtime.cc
index 9b46cf8245..abe2793ce8 100644
--- a/tensorflow/c/eager/runtime.cc
+++ b/tensorflow/c/eager/runtime.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/eager/runtime.h"
 
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/node_def.pb.h"
@@ -95,22 +96,6 @@ Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out) {
   return Status::OK();
 }
 
-Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
-                      TF_AttrType* out, unsigned char* is_list) {
-  auto* t = gtl::FindOrNull(m, attr_name);
-  if (t == nullptr) {
-    return errors::InvalidArgument("Attribute '", attr_name,
-                                   "' does not exist for this operation");
-  }
-  *out = static_cast<TF_AttrType>(*t & ~kIsList);
-  if (*t & kIsList) {
-    *is_list = 1;
-  } else {
-    *is_list = 0;
-  }
-  return Status::OK();
-}
-
 #define DEFINE_SET_ATTR(value_type, value_field)                             \
   template <>                                                                \
   AttrBuilder& AttrBuilder::Set(StringPiece attr_name, value_type&& value) { \
@@ -168,6 +153,22 @@ const NodeDef& AttrBuilder::BuildNodeDef() {
   return *node_def_;
 }
 
+Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
+                      TF_AttrType* out, unsigned char* is_list) {
+  auto* t = gtl::FindOrNull(m, attr_name);
+  if (t == nullptr) {
+    return errors::InvalidArgument("Attribute '", attr_name,
+                                   "' does not exist for this operation");
+  }
+  *out = static_cast<TF_AttrType>(*t & ~kIsList);
+  if (*t & kIsList) {
+    *is_list = 1;
+  } else {
+    *is_list = 0;
+  }
+  return Status::OK();
+}
+
 namespace {
 inline tensorflow::Fprint128 FingerprintCat128(const tensorflow::Fprint128& a,
                                                const tensorflow::Fprint128& b) {
@@ -245,104 +246,4 @@ void AttrBuilder::MayBeInitializeNodeDef() {
   }
 }
 
-// static
-Status KernelAndDevice::InitOp(Device* device, const NodeDef& ndef,
-                               KernelAndDevice* out) {
-  OpKernel* k = nullptr;
-  Status s = CreateOpKernel(device->device_type().c_str(), device,
-                            device->GetAllocator(AllocatorAttributes()),
-                            nullptr, ndef, TF_GRAPH_DEF_VERSION, &k);
-  out->device_ = device;
-  out->kernel_.reset(k);
-  out->flib_ = nullptr;
-  return s;
-}
-
-// static
-Status KernelAndDevice::Init(const NodeDef& ndef, FunctionLibraryRuntime* flib,
-                             KernelAndDevice* out) {
-  OpKernel* k = nullptr;
-  Status s = flib->CreateKernel(ndef, &k);
-  out->device_ = flib->device();
-  out->kernel_.reset(k);
-  out->flib_ = flib;
-  return s;
-}
-
-Status KernelAndDevice::Run(std::vector<Tensor>* input_tensors,
-                            std::vector<Tensor>* output_tensors,
-                            NodeExecStats* stats) {
-  gtl::InlinedVector<TensorValue, 4> inputs;
-  for (Tensor& t : *input_tensors) {
-    inputs.push_back(TensorValue(&t));
-  }
-
-  std::vector<AllocatorAttributes> out_attrs(kernel_->num_outputs());
-  for (size_t i = 0; i < out_attrs.size(); ++i) {
-    out_attrs[i].set_on_host(kernel_->output_memory_types()[i] ==
-                             tensorflow::HOST_MEMORY);
-  }
-
-  OpKernelContext::Params params;
-  params.device = device_;
-  params.frame_iter = FrameAndIter(0, 0);
-  params.inputs = &inputs;
-  params.op_kernel = kernel_.get();
-  params.resource_manager = device_->resource_manager();
-  params.output_attr_array = gtl::vector_as_array(&out_attrs);
-  params.function_library = flib_;
-  params.slice_reader_cache = &slice_reader_cache_;
-  params.rendezvous = rendez_;
-  if (stats != nullptr) {
-    params.track_allocations = true;
-  }
-  // TODO(apassos): use a thread pool.
-  std::function<void(std::function<void()>)> runner =
-      [](std::function<void()> f) { f(); };
-  params.runner = &runner;
-
-  OpKernelContext context(&params);
-
-  if (kernel_->def().op() == "_Recv") {
-    // TODO(apassos) do not special-case _Recv. Currently the GPU device fails
-    // if trying to run _Recv->Compute(), specifically checking for _Recv. To go
-    // around this we call _Recv->ComputeAsync, to mimic graph mode behavior.
-    AsyncOpKernel* async = kernel_->AsAsync();
-    Notification done;
-    device_->ComputeAsync(async, &context, [&done]() { done.Notify(); });
-    done.WaitForNotification();
-  } else {
-    device_->Compute(kernel_.get(), &context);
-  }
-  if (!context.status().ok()) return context.status();
-
-  output_tensors->clear();
-  for (int i = 0; i < context.num_outputs(); ++i) {
-    output_tensors->push_back(Tensor(*context.mutable_output(i)));
-  }
-  if (stats != nullptr) {
-    for (const auto& allocator_pair : context.wrapped_allocators()) {
-      AllocatorMemoryUsed* memory = stats->add_memory();
-      memory->set_allocator_name(allocator_pair.first->Name());
-      auto sizes = allocator_pair.second->GetSizes();
-      memory->set_total_bytes(std::get<0>(sizes));
-      memory->set_peak_bytes(std::get<1>(sizes));
-      memory->set_live_bytes(std::get<2>(sizes));
-
-      AllocatorStats allocator_stats;
-      allocator_pair.first->GetStats(&allocator_stats);
-      memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
-      allocator_pair.second->GetRecordsAndUnRef();
-    }
-    auto* ms = stats->mutable_memory_stats();
-    ms->set_temp_memory_size(context.temp_memory_allocated());
-    for (const auto& alloc_id : context.persistent_alloc_ids()) {
-      ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
-    }
-
-    ms->set_persistent_memory_size(context.persistent_memory_allocated());
-  }
-  return Status::OK();
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/c/eager/runtime.h b/tensorflow/c/eager/runtime.h
index ad16f65495..929b1b8296 100644
--- a/tensorflow/c/eager/runtime.h
+++ b/tensorflow/c/eager/runtime.h
@@ -23,6 +23,7 @@ limitations under the License.
 
 #include "tensorflow/c/c_api.h"
 #include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/types.h"
@@ -45,6 +46,10 @@ Status OpDefForOp(const char* op_name, const OpDef** op_def);
 // Returns the AttrTypeMap for the TensorFlow operation named op_name.
 Status AttrTypeMapForOp(const char* op_name, const AttrTypeMap** out);
 
+// Looks for 'attr_name' in 'm' and sets 'out' and 'is_list'.
+Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
+                      TF_AttrType* out, unsigned char* is_list);
+
 // Looks for 'attr_name' in 'm' and sets 'out' and 'is_list'.
 Status AttrTypeByName(const AttrTypeMap& m, const string& attr_name,
                       TF_AttrType* out, unsigned char* is_list);
@@ -149,53 +154,6 @@ template <>
 AttrBuilder& AttrBuilder::Set(StringPiece attr_name,
                               tensorflow::DataType&& value);
 
-// KernelAndDevice encapsulates an instantiated kernel and the device it is on.
-//
-// Also see:
-// https://www.tensorflow.org/code/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
-// and
-// https://www.tensorflow.org/code/tensorflow/core/kernels/ops_testutil.h
-class KernelAndDevice {
- public:
-  // Populates 'out' with a kernel appropriate for 'ndef'.
-  //
-  // The provided FunctionLibraryRuntime MUST outlive all calls to
-  // Run() on the returned KernelAndDevice.
-  //
-  // TODO(ashankar): Figure out thread-safety concerns around
-  // FunctionLibraryRuntime (in particular, how the underlying
-  // FunctionLibraryDefinition might be mutated by another thread as new
-  // functions are registered with it).  Conservatively, thread-safe usage of
-  // the FunctionLibraryRuntime is pushed on to the caller (see locking in
-  // c_api.cc).
-  static Status Init(const NodeDef& ndef, FunctionLibraryRuntime* flib,
-                     KernelAndDevice* out);
-  // TODO(ashankar): Remove this
-  static Status InitOp(Device* device, const NodeDef& ndef,
-                       KernelAndDevice* out);
-
-  KernelAndDevice(tensorflow::Rendezvous* rendez)
-      : device_(nullptr), flib_(nullptr), rendez_(rendez) {}
-
-  // TODO(ashankar): Handle list-valued inputs.
-  Status Run(std::vector<Tensor>* inputs, std::vector<Tensor>* outputs,
-             NodeExecStats* stats);
-
-  const OpKernel* kernel() const { return kernel_.get(); }
-
-  Device* device() const { return device_; }
-
-  DataTypeVector* mutable_output_dtypes() { return &output_dtypes_; }
-  const DataTypeVector& output_dtypes() { return output_dtypes_; }
-
- private:
-  std::unique_ptr<OpKernel> kernel_;
-  Device* device_;
-  FunctionLibraryRuntime* flib_;
-  checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_;
-  Rendezvous* rendez_;
-  DataTypeVector output_dtypes_;
-};
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/c/eager/runtime_test.cc b/tensorflow/c/eager/runtime_test.cc
index 4f75d27887..27ebeb0508 100644
--- a/tensorflow/c/eager/runtime_test.cc
+++ b/tensorflow/c/eager/runtime_test.cc
@@ -33,27 +33,6 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-class TestEnv {
- public:
-  TestEnv() : flib_def_(OpRegistry::Global(), {}) {
-    Device* device =
-        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0");
-    device_mgr_.reset(new DeviceMgr({device}));
-    flib_runtime_ = NewFunctionLibraryRuntime(device_mgr_.get(), Env::Default(),
-                                              device, TF_GRAPH_DEF_VERSION,
-                                              &flib_def_, nullptr, {}, nullptr);
-  }
-
-  FunctionLibraryRuntime* function_library_runtime() const {
-    return flib_runtime_.get();
-  }
-
- private:
-  FunctionLibraryDefinition flib_def_;
-  std::unique_ptr<DeviceMgr> device_mgr_;
-  std::unique_ptr<FunctionLibraryRuntime> flib_runtime_;
-};
-
 TEST(AttrTypeMap, Lookup) {
   const AttrTypeMap* m = nullptr;
   Status s = AttrTypeMapForOp("ThisOpCannotPossiblyExist", &m);
@@ -79,113 +58,5 @@ TEST(AttrTypeMap, Lookup) {
   EXPECT_NE(is_list, 0);
 }
 
-TEST(KernelAndDevice, Run) {
-  Tensor t(Input({{1.0f, 2.0f}, {3.0f, 4.0f}}).tensor());
-  std::vector<Tensor> inputs;
-  inputs.push_back(t);
-  inputs.push_back(t);
-  NodeDef ndef(AttrBuilder("MatMul")
-                   .Set("T", DT_FLOAT)
-                   .Set("transpose_a", false)
-                   .Set("transpose_b", false)
-                   .NumInputs(inputs.size())
-                   .BuildNodeDef());
-  TestEnv env;
-  KernelAndDevice kernel(nullptr);
-  Status s =
-      KernelAndDevice::Init(ndef, env.function_library_runtime(), &kernel);
-  ASSERT_TRUE(s.ok()) << s;
-  std::vector<Tensor> outputs;
-  s = kernel.Run(&inputs, &outputs, nullptr);
-  ASSERT_TRUE(s.ok()) << s;
-  ASSERT_EQ(1, outputs.size());
-  const Tensor& out = outputs[0];
-  EXPECT_EQ(7, out.matrix<float>()(0, 0));
-  EXPECT_EQ(10, out.matrix<float>()(0, 1));
-  EXPECT_EQ(15, out.matrix<float>()(1, 0));
-  EXPECT_EQ(22, out.matrix<float>()(1, 1));
-}
-
-void BM_CreateGraph(int iters) {
-  for (int i = 0; i < iters; ++i) {
-    Scope root = Scope::NewRootScope();
-    auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
-    auto M = ops::MatMul(root, C, C);
-    TF_CHECK_OK(root.status());
-  }
-}
-BENCHMARK(BM_CreateGraph);
-
-void BM_RunGraph(int iters) {
-  tensorflow::testing::StopTiming();
-  Scope root = Scope::NewRootScope();
-  auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
-  auto M = ops::MatMul(root, C, C);
-  SessionOptions opts;
-  opts.config.set_inter_op_parallelism_threads(1);
-  opts.config.set_intra_op_parallelism_threads(1);
-  ClientSession sess(root, opts);
-  std::vector<Tensor> outputs;
-  tensorflow::testing::StartTiming();
-  for (int i = 0; i < iters; ++i) {
-    outputs.clear();
-    TF_CHECK_OK(sess.Run({M}, &outputs));
-  }
-}
-BENCHMARK(BM_RunGraph);
-
-void BM_CreateAndDestroySession(int iters) {
-  tensorflow::testing::StopTiming();
-  Scope root = Scope::NewRootScope();
-  auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
-  auto M = ops::MatMul(root, C, C);
-  tensorflow::testing::StartTiming();
-  for (int i = 0; i < iters; ++i) {
-    ClientSession sess(root);
-  }
-}
-BENCHMARK(BM_CreateAndDestroySession);
-
-void BM_KernelAndDeviceInit(int iters) {
-  tensorflow::testing::StopTiming();
-  NodeDef ndef(AttrBuilder("MatMul")
-                   .Set("T", DT_FLOAT)
-                   .Set("transpose_a", false)
-                   .Set("transpose_b", false)
-                   .NumInputs(2)
-                   .BuildNodeDef());
-  TestEnv env;
-  KernelAndDevice k(nullptr);
-  tensorflow::testing::StartTiming();
-  for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(
-        KernelAndDevice::Init(ndef, env.function_library_runtime(), &k));
-  }
-}
-BENCHMARK(BM_KernelAndDeviceInit);
-
-void BM_KernelAndDeviceRun(int iters) {
-  tensorflow::testing::StopTiming();
-  Tensor t(Input({{1.0f, 2.0f}, {3.0f, 4.0f}}).tensor());
-  std::vector<Tensor> inputs;
-  inputs.push_back(t);
-  inputs.push_back(t);
-  std::vector<Tensor> outputs;
-  NodeDef ndef(AttrBuilder("MatMul")
-                   .Set("T", DT_FLOAT)
-                   .Set("transpose_a", false)
-                   .Set("transpose_b", false)
-                   .NumInputs(inputs.size())
-                   .BuildNodeDef());
-  TestEnv env;
-  KernelAndDevice kernel(nullptr);
-  TF_CHECK_OK(
-      KernelAndDevice::Init(ndef, env.function_library_runtime(), &kernel));
-  tensorflow::testing::StartTiming();
-  for (int i = 0; i < iters; ++i) {
-    TF_CHECK_OK(kernel.Run(&inputs, &outputs, nullptr));
-  }
-}
-BENCHMARK(BM_KernelAndDeviceRun);
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 8124280914..42d222ff6b 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -794,7 +794,6 @@ tf_cuda_library(
     hdrs = [
         "common_runtime/device.h",
         "common_runtime/device_factory.h",
-        "common_runtime/eager/eager_executor.h",
         "common_runtime/optimization_registry.h",
         "common_runtime/shape_refiner.h",
         "graph/algorithm.h",
@@ -1038,6 +1037,7 @@ filegroup(
             "util/tensor_bundle/*.h",
             "util/tensor_bundle/*.cc",
             "common_runtime/gpu/**/*",
+            "common_runtime/eager/*",
             "common_runtime/gpu_device_factory.*",
         ],
     ),
@@ -1063,6 +1063,7 @@ filegroup(
             "**/*testlib*",
             "**/*main.cc",
             "common_runtime/gpu/**/*",
+            "common_runtime/eager/*",
             "common_runtime/gpu_device_factory.*",
             "graph/dot.*",
         ],
@@ -2150,7 +2151,6 @@ CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/stats_publisher_interface.h",
     "common_runtime/step_stats_collector.h",
     "common_runtime/threadpool_device.h",
-    "common_runtime/eager/eager_executor.h",
     "graph/gradients.h",
     "graph/quantize_training.h",
 ] + if_mkl(["graph/mkl_graph_util.h"])
@@ -2170,7 +2170,6 @@ tf_cuda_library(
         "common_runtime/device_factory.cc",
         "common_runtime/device_mgr.cc",
         "common_runtime/device_set.cc",
-        "common_runtime/eager/eager_executor.cc",
         "common_runtime/executor.cc",
         "common_runtime/function.cc",
         "common_runtime/graph_optimizer.cc",
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
new file mode 100644
index 0000000000..8ba560bef8
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -0,0 +1,86 @@
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+        "//tensorflow_models:__subpackages__",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_cc_test",
+    "tf_cuda_library",
+)
+
+tf_cuda_library(
+    name = "eager_executor",
+    srcs = [
+        "eager_executor.cc",
+    ],
+    hdrs = [
+        "eager_executor.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+tf_cuda_library(
+    name = "kernel_and_device",
+    srcs = [
+        "kernel_and_device.cc",
+    ],
+    hdrs = [
+        "kernel_and_device.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+tf_cc_test(
+    name = "kernel_and_device_test",
+    srcs = ["kernel_and_device_test.cc"],
+    deps = [
+        ":kernel_and_device",
+        "//tensorflow/c/eager:runtime",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:client_session",
+        "//tensorflow/cc:ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
+# -----------------------------------------------------------------------------
+# Google-internal targets.
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.cc b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
new file mode 100644
index 0000000000..0a4895a938
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.cc
@@ -0,0 +1,132 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/public/version.h"
+#include "tensorflow/core/util/tensor_slice_reader_cache.h"
+
+namespace tensorflow {
+
+// static
+Status KernelAndDevice::InitOp(Device* device, const NodeDef& ndef,
+                               KernelAndDevice* out) {
+  OpKernel* k = nullptr;
+  Status s = CreateOpKernel(device->device_type().c_str(), device,
+                            device->GetAllocator(AllocatorAttributes()),
+                            nullptr, ndef, TF_GRAPH_DEF_VERSION, &k);
+  out->device_ = device;
+  out->kernel_.reset(k);
+  out->flib_ = nullptr;
+  return s;
+}
+
+// static
+Status KernelAndDevice::Init(const NodeDef& ndef, FunctionLibraryRuntime* flib,
+                             KernelAndDevice* out) {
+  OpKernel* k = nullptr;
+  Status s = flib->CreateKernel(ndef, &k);
+  out->device_ = flib->device();
+  out->kernel_.reset(k);
+  out->flib_ = flib;
+  return s;
+}
+
+Status KernelAndDevice::Run(std::vector<Tensor>* input_tensors,
+                            std::vector<Tensor>* output_tensors,
+                            NodeExecStats* stats) {
+  gtl::InlinedVector<TensorValue, 4> inputs;
+  for (Tensor& t : *input_tensors) {
+    inputs.push_back(TensorValue(&t));
+  }
+
+  std::vector<AllocatorAttributes> out_attrs(kernel_->num_outputs());
+  for (size_t i = 0; i < out_attrs.size(); ++i) {
+    out_attrs[i].set_on_host(kernel_->output_memory_types()[i] ==
+                             tensorflow::HOST_MEMORY);
+  }
+
+  OpKernelContext::Params params;
+  params.device = device_;
+  params.frame_iter = FrameAndIter(0, 0);
+  params.inputs = &inputs;
+  params.op_kernel = kernel_.get();
+  params.resource_manager = device_->resource_manager();
+  params.output_attr_array = gtl::vector_as_array(&out_attrs);
+  params.function_library = flib_;
+  params.slice_reader_cache = &slice_reader_cache_;
+  params.rendezvous = rendez_;
+  if (stats != nullptr) {
+    params.track_allocations = true;
+  }
+  // TODO(apassos): use a thread pool.
+  std::function<void(std::function<void()>)> runner =
+      [](std::function<void()> f) { f(); };
+  params.runner = &runner;
+
+  OpKernelContext context(&params);
+
+  if (kernel_->def().op() == "_Recv") {
+    // TODO(apassos) do not special-case _Recv. Currently the GPU device fails
+    // if trying to run _Recv->Compute(), specifically checking for _Recv. To go
+    // around this we call _Recv->ComputeAsync, to mimic graph mode behavior.
+    AsyncOpKernel* async = kernel_->AsAsync();
+    Notification done;
+    device_->ComputeAsync(async, &context, [&done]() { done.Notify(); });
+    done.WaitForNotification();
+  } else {
+    device_->Compute(kernel_.get(), &context);
+  }
+  if (!context.status().ok()) return context.status();
+
+  output_tensors->clear();
+  for (int i = 0; i < context.num_outputs(); ++i) {
+    output_tensors->push_back(Tensor(*context.mutable_output(i)));
+  }
+  if (stats != nullptr) {
+    for (const auto& allocator_pair : context.wrapped_allocators()) {
+      AllocatorMemoryUsed* memory = stats->add_memory();
+      memory->set_allocator_name(allocator_pair.first->Name());
+      auto sizes = allocator_pair.second->GetSizes();
+      memory->set_total_bytes(std::get<0>(sizes));
+      memory->set_peak_bytes(std::get<1>(sizes));
+      memory->set_live_bytes(std::get<2>(sizes));
+
+      AllocatorStats allocator_stats;
+      allocator_pair.first->GetStats(&allocator_stats);
+      memory->set_allocator_bytes_in_use(allocator_stats.bytes_in_use);
+      allocator_pair.second->GetRecordsAndUnRef();
+    }
+    auto* ms = stats->mutable_memory_stats();
+    ms->set_temp_memory_size(context.temp_memory_allocated());
+    for (const auto& alloc_id : context.persistent_alloc_ids()) {
+      ms->mutable_persistent_tensor_alloc_ids()->Add(alloc_id);
+    }
+
+    ms->set_persistent_memory_size(context.persistent_memory_allocated());
+  }
+  return Status::OK();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device.h b/tensorflow/core/common_runtime/eager/kernel_and_device.h
new file mode 100644
index 0000000000..46ec550c78
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device.h
@@ -0,0 +1,85 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_KERNEL_AND_DEVICE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_KERNEL_AND_DEVICE_H_
+
+// Support for eager execution of TensorFlow kernels.
+
+#include <memory>
+#include <unordered_map>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/util/tensor_slice_reader_cache.h"
+
+namespace tensorflow {
+
+// KernelAndDevice encapsulates an instantiated kernel and the device it is on.
+//
+// Also see:
+// https://www.tensorflow.org/code/tensorflow/core/common_runtime/kernel_benchmark_testlib.h
+// and
+// https://www.tensorflow.org/code/tensorflow/core/kernels/ops_testutil.h
+class KernelAndDevice {
+ public:
+  // Populates 'out' with a kernel appropriate for 'ndef'.
+  //
+  // The provided FunctionLibraryRuntime MUST outlive all calls to
+  // Run() on the returned KernelAndDevice.
+  //
+  // TODO(ashankar): Figure out thread-safety concerns around
+  // FunctionLibraryRuntime (in particular, how the underlying
+  // FunctionLibraryDefinition might be mutated by another thread as new
+  // functions are registered with it).  Conservatively, thread-safe usage of
+  // the FunctionLibraryRuntime is pushed on to the caller (see locking in
+  // c_api.cc).
+  static Status Init(const NodeDef& ndef, FunctionLibraryRuntime* flib,
+                     KernelAndDevice* out);
+  // TODO(ashankar): Remove this
+  static Status InitOp(Device* device, const NodeDef& ndef,
+                       KernelAndDevice* out);
+
+  KernelAndDevice(tensorflow::Rendezvous* rendez)
+      : device_(nullptr), flib_(nullptr), rendez_(rendez) {}
+
+  // TODO(ashankar): Handle list-valued inputs.
+  Status Run(std::vector<Tensor>* inputs, std::vector<Tensor>* outputs,
+             NodeExecStats* stats);
+
+  const OpKernel* kernel() const { return kernel_.get(); }
+
+  Device* device() const { return device_; }
+
+  DataTypeVector* mutable_output_dtypes() { return &output_dtypes_; }
+  const DataTypeVector& output_dtypes() { return output_dtypes_; }
+
+ private:
+  std::unique_ptr<OpKernel> kernel_;
+  Device* device_;
+  FunctionLibraryRuntime* flib_;
+  checkpoint::TensorSliceReaderCacheWrapper slice_reader_cache_;
+  Rendezvous* rendez_;
+  DataTypeVector output_dtypes_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_KERNEL_AND_DEVICE_H_
diff --git a/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
new file mode 100644
index 0000000000..dd055c3c3e
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/kernel_and_device_test.cc
@@ -0,0 +1,140 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+
+#include <memory>
+#include <vector>
+
+#include "tensorflow/c/eager/runtime.h"
+#include "tensorflow/cc/client/client_session.h"
+#include "tensorflow/cc/framework/ops.h"
+#include "tensorflow/cc/framework/scope.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+namespace {
+
+class TestEnv {
+ public:
+  TestEnv() : flib_def_(OpRegistry::Global(), {}) {
+    Device* device =
+        DeviceFactory::NewDevice("CPU", {}, "/job:a/replica:0/task:0");
+    device_mgr_.reset(new DeviceMgr({device}));
+    flib_runtime_ = NewFunctionLibraryRuntime(device_mgr_.get(), Env::Default(),
+                                              device, TF_GRAPH_DEF_VERSION,
+                                              &flib_def_, nullptr, {}, nullptr);
+  }
+
+  FunctionLibraryRuntime* function_library_runtime() const {
+    return flib_runtime_.get();
+  }
+
+ private:
+  FunctionLibraryDefinition flib_def_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<FunctionLibraryRuntime> flib_runtime_;
+};
+
+void BM_CreateGraph(int iters) {
+  for (int i = 0; i < iters; ++i) {
+    Scope root = Scope::NewRootScope();
+    auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
+    auto M = ops::MatMul(root, C, C);
+    TF_CHECK_OK(root.status());
+  }
+}
+BENCHMARK(BM_CreateGraph);
+
+void BM_RunGraph(int iters) {
+  tensorflow::testing::StopTiming();
+  Scope root = Scope::NewRootScope();
+  auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
+  auto M = ops::MatMul(root, C, C);
+  SessionOptions opts;
+  opts.config.set_inter_op_parallelism_threads(1);
+  opts.config.set_intra_op_parallelism_threads(1);
+  ClientSession sess(root, opts);
+  std::vector<Tensor> outputs;
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    outputs.clear();
+    TF_CHECK_OK(sess.Run({M}, &outputs));
+  }
+}
+BENCHMARK(BM_RunGraph);
+
+void BM_CreateAndDestroySession(int iters) {
+  tensorflow::testing::StopTiming();
+  Scope root = Scope::NewRootScope();
+  auto C = ops::Const(root, {{1.0, 2.0}, {3.0, 4.0}});
+  auto M = ops::MatMul(root, C, C);
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    ClientSession sess(root);
+  }
+}
+BENCHMARK(BM_CreateAndDestroySession);
+
+void BM_KernelAndDeviceInit(int iters) {
+  tensorflow::testing::StopTiming();
+  NodeDef ndef(AttrBuilder("MatMul")
+                   .Set("T", DT_FLOAT)
+                   .Set("transpose_a", false)
+                   .Set("transpose_b", false)
+                   .NumInputs(2)
+                   .BuildNodeDef());
+  TestEnv env;
+  KernelAndDevice k(nullptr);
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    TF_CHECK_OK(
+        KernelAndDevice::Init(ndef, env.function_library_runtime(), &k));
+  }
+}
+BENCHMARK(BM_KernelAndDeviceInit);
+
+void BM_KernelAndDeviceRun(int iters) {
+  tensorflow::testing::StopTiming();
+  Tensor t(Input({{1.0f, 2.0f}, {3.0f, 4.0f}}).tensor());
+  std::vector<Tensor> inputs;
+  inputs.push_back(t);
+  inputs.push_back(t);
+  std::vector<Tensor> outputs;
+  NodeDef ndef(AttrBuilder("MatMul")
+                   .Set("T", DT_FLOAT)
+                   .Set("transpose_a", false)
+                   .Set("transpose_b", false)
+                   .NumInputs(inputs.size())
+                   .BuildNodeDef());
+  TestEnv env;
+  KernelAndDevice kernel(nullptr);
+  TF_CHECK_OK(
+      KernelAndDevice::Init(ndef, env.function_library_runtime(), &kernel));
+  tensorflow::testing::StartTiming();
+  for (int i = 0; i < iters; ++i) {
+    TF_CHECK_OK(kernel.Run(&inputs, &outputs, nullptr));
+  }
+}
+BENCHMARK(BM_KernelAndDeviceRun);
+}  // namespace
+}  // namespace tensorflow
-- 
GitLab


From 0a7f511aff6ef8900a9a56cd3207508e3cd8ec8f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 11:16:13 -0700
Subject: [PATCH 305/960] Allowing dnn tree combined estimator to work with
 core versions of feature columns and losses

PiperOrigin-RevId: 189937063
---
 .../boosted_trees/estimator_batch/BUILD       |   2 +-
 .../dnn_tree_combined_estimator.py            | 213 ++++++++++++++----
 .../dnn_tree_combined_estimator_test.py       |  33 ++-
 3 files changed, 200 insertions(+), 48 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index 289f5bb314..dae402204f 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -149,7 +149,7 @@ py_library(
 
 py_test(
     name = "dnn_tree_combined_estimator_test",
-    size = "small",
+    size = "medium",
     srcs = ["dnn_tree_combined_estimator_test.py"],
     srcs_version = "PY2AND3",
     tags = [
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
index cec3892b57..2e7b8cba05 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator.py
@@ -25,15 +25,20 @@ from __future__ import division
 from __future__ import print_function
 
 import six
-
 from tensorflow.contrib import layers
 from tensorflow.contrib.boosted_trees.estimator_batch import trainer_hooks
 from tensorflow.contrib.boosted_trees.python.ops import model_ops
 from tensorflow.contrib.boosted_trees.python.training.functions import gbdt_batch
 from tensorflow.contrib.layers.python.layers import optimizers
+from tensorflow.contrib.learn.python.learn.estimators import constants
 from tensorflow.contrib.learn.python.learn.estimators import estimator
 from tensorflow.contrib.learn.python.learn.estimators import head as head_lib
 from tensorflow.contrib.learn.python.learn.estimators import model_fn
+from tensorflow.contrib.learn.python.learn.estimators import model_fn as contrib_model_fn_lib
+from tensorflow.contrib.learn.python.learn.estimators import prediction_key
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.estimator.export import export_output
+from tensorflow.python.feature_column import feature_column as feature_column_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import nn
@@ -46,6 +51,52 @@ from tensorflow.python.training import training_util
 
 _DNN_LEARNING_RATE = 0.001
 
+_CORE_MODE_TO_CONTRIB_MODE_ = {
+    model_fn_lib.ModeKeys.TRAIN: contrib_model_fn_lib.ModeKeys.TRAIN,
+    model_fn_lib.ModeKeys.EVAL: contrib_model_fn_lib.ModeKeys.EVAL,
+    model_fn_lib.ModeKeys.PREDICT: contrib_model_fn_lib.ModeKeys.INFER
+}
+
+
+def _core_mode_to_contrib_mode(mode):
+  return _CORE_MODE_TO_CONTRIB_MODE_[mode]
+
+
+def _export_outputs_to_output_alternatives(export_outputs):
+  """Converts EstimatorSpec.export_outputs to output_alternatives.
+
+  Args:
+    export_outputs: export_outputs created by create_estimator_spec.
+  Returns:
+    converted output_alternatives.
+  """
+  output = dict()
+  if export_outputs is not None:
+    for key, value in export_outputs.items():
+      if isinstance(value, export_output.ClassificationOutput):
+        exported_predictions = {
+            prediction_key.PredictionKey.SCORES: value.scores,
+            prediction_key.PredictionKey.CLASSES: value.classes
+        }
+        output[key] = (constants.ProblemType.CLASSIFICATION,
+                       exported_predictions)
+    return output
+  return None
+
+
+def _estimator_spec_to_model_fn_ops(estimator_spec, is_regression):
+  alternatives = []
+  if not is_regression:
+    _export_outputs_to_output_alternatives(estimator_spec.export_outputs)
+
+  return model_fn.ModelFnOps(
+      mode=_core_mode_to_contrib_mode(estimator_spec.mode),
+      predictions=estimator_spec.predictions,
+      loss=estimator_spec.loss,
+      train_op=estimator_spec.train_op,
+      eval_metric_ops=estimator_spec.eval_metric_ops,
+      output_alternatives=alternatives)
+
 
 def _get_optimizer(optimizer):
   if callable(optimizer):
@@ -59,16 +110,26 @@ def _add_hidden_layer_summary(value, tag):
   summary.histogram("%s_activation" % tag, value)
 
 
-def _dnn_tree_combined_model_fn(
-    features, labels, mode, head, dnn_hidden_units,
-    dnn_feature_columns, tree_learner_config, num_trees,
-    tree_examples_per_layer,
-    config=None, dnn_optimizer="Adagrad",
-    dnn_activation_fn=nn.relu, dnn_dropout=None,
-    dnn_input_layer_partitioner=None,
-    dnn_input_layer_to_tree=True, dnn_steps_to_train=10000,
-    tree_feature_columns=None,
-    tree_center_bias=True):
+def _dnn_tree_combined_model_fn(features,
+                                labels,
+                                mode,
+                                head,
+                                dnn_hidden_units,
+                                dnn_feature_columns,
+                                tree_learner_config,
+                                num_trees,
+                                tree_examples_per_layer,
+                                config=None,
+                                dnn_optimizer="Adagrad",
+                                dnn_activation_fn=nn.relu,
+                                dnn_dropout=None,
+                                dnn_input_layer_partitioner=None,
+                                dnn_input_layer_to_tree=True,
+                                dnn_steps_to_train=10000,
+                                tree_feature_columns=None,
+                                tree_center_bias=False,
+                                use_core_versions=False,
+                                is_regression=False):
   """DNN and GBDT combined model_fn.
 
   Args:
@@ -106,6 +167,9 @@ def _dnn_tree_combined_model_fn(
       set to True, these features are in addition to dnn_feature_columns.
     tree_center_bias: Whether a separate tree should be created for
       first fitting the bias.
+    use_core_versions: Whether feature columns and loss are from the core (as
+      opposed to contrib) version of tensorflow.
+    is_regression: Whether the problem is regression or not.
 
   Returns:
     A `ModelFnOps` object.
@@ -135,11 +199,17 @@ def _dnn_tree_combined_model_fn(
         "input_from_feature_columns",
         values=tuple(six.itervalues(features)),
         partitioner=dnn_partitioner) as input_layer_scope:
-      input_layer = layers.input_from_feature_columns(
-          columns_to_tensors=features,
-          feature_columns=dnn_feature_columns,
-          weight_collections=[dnn_parent_scope],
-          scope=input_layer_scope)
+      if use_core_versions:
+        input_layer = feature_column_lib.input_layer(
+            features=features,
+            feature_columns=dnn_feature_columns,
+            weight_collections=[dnn_parent_scope])
+      else:
+        input_layer = layers.input_from_feature_columns(
+            columns_to_tensors=features,
+            feature_columns=dnn_feature_columns,
+            weight_collections=[dnn_parent_scope],
+            scope=input_layer_scope)
     previous_layer = input_layer
     for layer_id, num_hidden_units in enumerate(dnn_hidden_units):
       with variable_scope.variable_scope(
@@ -222,24 +292,51 @@ def _dnn_tree_combined_model_fn(
     del loss
     return control_flow_ops.no_op()
 
-  model_fn_ops = head.create_model_fn_ops(
-      features=features,
-      mode=mode,
-      labels=labels,
-      train_op_fn=_no_train_op_fn,
-      logits=tree_train_logits)
-  dnn_train_op = head.create_model_fn_ops(
-      features=features,
-      mode=mode,
-      labels=labels,
-      train_op_fn=_dnn_train_op_fn,
-      logits=dnn_logits).train_op
-  tree_train_op = head.create_model_fn_ops(
-      features=tree_features,
-      mode=mode,
-      labels=labels,
-      train_op_fn=_tree_train_op_fn,
-      logits=tree_train_logits).train_op
+  if use_core_versions:
+    model_fn_ops = head.create_estimator_spec(
+        features=features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_no_train_op_fn,
+        logits=tree_train_logits)
+    dnn_train_op = head.create_estimator_spec(
+        features=features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_dnn_train_op_fn,
+        logits=dnn_logits)
+    dnn_train_op = _estimator_spec_to_model_fn_ops(dnn_train_op,
+                                                   is_regression).train_op
+
+    tree_train_op = head.create_estimator_spec(
+        features=tree_features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_tree_train_op_fn,
+        logits=tree_train_logits)
+    tree_train_op = _estimator_spec_to_model_fn_ops(tree_train_op,
+                                                    is_regression).train_op
+
+    model_fn_ops = _estimator_spec_to_model_fn_ops(model_fn_ops, is_regression)
+  else:
+    model_fn_ops = head.create_model_fn_ops(
+        features=features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_no_train_op_fn,
+        logits=tree_train_logits)
+    dnn_train_op = head.create_model_fn_ops(
+        features=features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_dnn_train_op_fn,
+        logits=dnn_logits).train_op
+    tree_train_op = head.create_model_fn_ops(
+        features=tree_features,
+        mode=mode,
+        labels=labels,
+        train_op_fn=_tree_train_op_fn,
+        logits=tree_train_logits).train_op
 
   if tree_center_bias:
     num_trees += 1
@@ -277,7 +374,8 @@ class DNNBoostedTreeCombinedClassifier(estimator.Estimator):
                dnn_input_layer_to_tree=True,
                dnn_steps_to_train=10000,
                tree_feature_columns=None,
-               tree_center_bias=True):
+               tree_center_bias=False,
+               use_core_versions=False):
     """Initializes a DNNBoostedTreeCombinedClassifier instance.
 
     Args:
@@ -322,6 +420,8 @@ class DNNBoostedTreeCombinedClassifier(estimator.Estimator):
         set to True, these features are in addition to dnn_feature_columns.
       tree_center_bias: Whether a separate tree should be created for
         first fitting the bias.
+      use_core_versions: Whether feature columns and loss are from the core (as
+        opposed to contrib) version of tensorflow.
     """
     head = head_lib.multi_class_head(
         n_classes=n_classes,
@@ -336,8 +436,8 @@ class DNNBoostedTreeCombinedClassifier(estimator.Estimator):
           tree_learner_config, num_trees, tree_examples_per_layer, config,
           dnn_optimizer, dnn_activation_fn, dnn_dropout,
           dnn_input_layer_partitioner, dnn_input_layer_to_tree,
-          dnn_steps_to_train,
-          tree_feature_columns, tree_center_bias)
+          dnn_steps_to_train, tree_feature_columns, tree_center_bias,
+          use_core_versions)
 
     super(DNNBoostedTreeCombinedClassifier, self).__init__(
         model_fn=_model_fn, model_dir=model_dir,
@@ -366,7 +466,8 @@ class DNNBoostedTreeCombinedRegressor(estimator.Estimator):
                dnn_input_layer_to_tree=True,
                dnn_steps_to_train=10000,
                tree_feature_columns=None,
-               tree_center_bias=True):
+               tree_center_bias=False,
+               use_core_versions=False):
     """Initializes a DNNBoostedTreeCombinedRegressor instance.
 
     Args:
@@ -411,6 +512,8 @@ class DNNBoostedTreeCombinedRegressor(estimator.Estimator):
         set to True, these features are in addition to dnn_feature_columns.
       tree_center_bias: Whether a separate tree should be created for
         first fitting the bias.
+      use_core_versions: Whether feature columns and loss are from the core (as
+        opposed to contrib) version of tensorflow.
     """
     head = head_lib.regression_head(
         label_name=label_name,
@@ -426,11 +529,26 @@ class DNNBoostedTreeCombinedRegressor(estimator.Estimator):
 
     def _model_fn(features, labels, mode, config):
       return _dnn_tree_combined_model_fn(
-          features, labels, mode, head, dnn_hidden_units, dnn_feature_columns,
-          tree_learner_config, num_trees, tree_examples_per_layer, config,
-          dnn_optimizer, dnn_activation_fn, dnn_dropout,
-          dnn_input_layer_partitioner, dnn_input_layer_to_tree,
-          dnn_steps_to_train, tree_feature_columns, tree_center_bias)
+          features,
+          labels,
+          mode,
+          head,
+          dnn_hidden_units,
+          dnn_feature_columns,
+          tree_learner_config,
+          num_trees,
+          tree_examples_per_layer,
+          config,
+          dnn_optimizer,
+          dnn_activation_fn,
+          dnn_dropout,
+          dnn_input_layer_partitioner,
+          dnn_input_layer_to_tree,
+          dnn_steps_to_train,
+          tree_feature_columns,
+          tree_center_bias,
+          use_core_versions,
+          is_regression=True)
 
     super(DNNBoostedTreeCombinedRegressor, self).__init__(
         model_fn=_model_fn, model_dir=model_dir,
@@ -460,7 +578,8 @@ class DNNBoostedTreeCombinedEstimator(estimator.Estimator):
                dnn_input_layer_to_tree=True,
                dnn_steps_to_train=10000,
                tree_feature_columns=None,
-               tree_center_bias=True):
+               tree_center_bias=False,
+               use_core_versions=False):
     """Initializes a DNNBoostedTreeCombinedEstimator instance.
 
     Args:
@@ -500,6 +619,8 @@ class DNNBoostedTreeCombinedEstimator(estimator.Estimator):
         set to True, these features are in addition to dnn_feature_columns.
       tree_center_bias: Whether a separate tree should be created for
         first fitting the bias.
+      use_core_versions: Whether feature columns and loss are from the core (as
+        opposed to contrib) version of tensorflow.
     """
     def _model_fn(features, labels, mode, config):
       return _dnn_tree_combined_model_fn(
@@ -507,8 +628,8 @@ class DNNBoostedTreeCombinedEstimator(estimator.Estimator):
           tree_learner_config, num_trees, tree_examples_per_layer, config,
           dnn_optimizer, dnn_activation_fn, dnn_dropout,
           dnn_input_layer_partitioner, dnn_input_layer_to_tree,
-          dnn_steps_to_train,
-          tree_feature_columns, tree_center_bias)
+          dnn_steps_to_train, tree_feature_columns, tree_center_bias,
+          use_core_versions)
 
     super(DNNBoostedTreeCombinedEstimator, self).__init__(
         model_fn=_model_fn, model_dir=model_dir,
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py
index 83d58c5610..f495edc62f 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/dnn_tree_combined_estimator_test.py
@@ -19,15 +19,17 @@ from __future__ import division
 from __future__ import print_function
 
 import tempfile
-
 from tensorflow.contrib.boosted_trees.estimator_batch import dnn_tree_combined_estimator as estimator
 from tensorflow.contrib.boosted_trees.proto import learner_pb2
 from tensorflow.contrib.layers.python.layers import feature_column
 from tensorflow.contrib.learn.python.learn.estimators import estimator_test_utils
 from tensorflow.contrib.learn.python.learn.estimators import run_config
+from tensorflow.python.estimator.canned import head as head_lib
+from tensorflow.python.feature_column import feature_column_lib as core_feature_column
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import test_util
+from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import googletest
 
 
@@ -100,6 +102,35 @@ class DNNBoostedTreeCombinedTest(test_util.TensorFlowTestCase):
     classifier.fit(input_fn=_train_input_fn, steps=15)
     classifier.evaluate(input_fn=_eval_input_fn, steps=1)
 
+  def testFitAndEvaluateDontThrowExceptionWithCore(self):
+    learner_config = learner_pb2.LearnerConfig()
+    learner_config.num_classes = 2
+    learner_config.constraints.max_tree_depth = 1
+    model_dir = tempfile.mkdtemp()
+    config = run_config.RunConfig()
+
+    # Use core head
+    head_fn = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(
+        loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE)
+
+    classifier = estimator.DNNBoostedTreeCombinedEstimator(
+        head=head_fn,
+        dnn_hidden_units=[1],
+        # Use core feature columns
+        dnn_feature_columns=[core_feature_column.numeric_column("x")],
+        tree_learner_config=learner_config,
+        num_trees=1,
+        tree_examples_per_layer=3,
+        model_dir=model_dir,
+        config=config,
+        dnn_steps_to_train=10,
+        dnn_input_layer_to_tree=True,
+        tree_feature_columns=[],
+        use_core_versions=True)
+
+    classifier.fit(input_fn=_train_input_fn, steps=15)
+    classifier.evaluate(input_fn=_eval_input_fn, steps=1)
+
 
 if __name__ == "__main__":
   googletest.main()
-- 
GitLab


From 6832756b26af035535c9349fec9cad77091584ba Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 11:18:02 -0700
Subject: [PATCH 306/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 189937564
---
 tensorflow/core/ops/ops.pbtxt | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 3beebdc6d4..72326e1137 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -11959,6 +11959,10 @@ op {
     name: "num_parallel_batches"
     type: DT_INT64
   }
+  input_arg {
+    name: "drop_remainder"
+    type: DT_BOOL
+  }
   output_arg {
     name: "handle"
     type: DT_VARIANT
-- 
GitLab


From 7742071501609cf1d5bcf552193ced245e9a290e Mon Sep 17 00:00:00 2001
From: Frank Perbet <fkp@google.com>
Date: Wed, 21 Mar 2018 11:40:25 -0700
Subject: [PATCH 307/960] Make the graph_editor C-API friendly: always
 construct ops with their inputs.

PiperOrigin-RevId: 189941495
---
 .../graph_editor/tests/transform_test.py      |  52 ++++++-
 tensorflow/contrib/graph_editor/transform.py  | 146 ++++++++++++++----
 tensorflow/contrib/graph_editor/util.py       |  29 ++--
 tensorflow/python/framework/ops.py            |   7 +-
 4 files changed, 186 insertions(+), 48 deletions(-)

diff --git a/tensorflow/contrib/graph_editor/tests/transform_test.py b/tensorflow/contrib/graph_editor/tests/transform_test.py
index ca00394388..2603de6407 100644
--- a/tensorflow/contrib/graph_editor/tests/transform_test.py
+++ b/tensorflow/contrib/graph_editor/tests/transform_test.py
@@ -23,6 +23,7 @@ from tensorflow.contrib import graph_editor as ge
 from tensorflow.contrib.graph_editor.tests import match
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -84,9 +85,9 @@ class TransformTest(test.TestCase):
   def test_transform(self):
     transformer = ge.Transformer()
 
-    def my_transform_op_handler(info, op):
+    def my_transform_op_handler(info, op, new_inputs):
       add_noise = op.name.startswith("Add")
-      op_, op_outputs_ = ge.transform.copy_op_handler(info, op)
+      op_, op_outputs_ = ge.transform.copy_op_handler(info, op, new_inputs)
       if not add_noise:
         return op_, op_outputs_
       # add some noise to op
@@ -201,15 +202,56 @@ class TransformTest(test.TestCase):
                         get_operation_by_name("res/grad/mul1_grad/Mul_1"))
 
     # Make sure _original_ops are as expected.
-    self.assertEquals(original_mul1_grad._original_op.name, u"mul1")
-    self.assertEquals(result_mul1_grad._original_op.name, u"res/mul1")
-    self.assertNotEquals(res.name, g.name)
+    self.assertEqual(original_mul1_grad._original_op.name, u"mul1")
+    self.assertEqual(result_mul1_grad._original_op.name, u"res/mul1")
+    self.assertNotEqual(res.name, g.name)
     with session.Session() as sess:
       sess.run(variables.global_variables_initializer())
       g_val, res_val = sess.run([g, res])
     self.assertNear(g_val, 0.0, ERROR_TOLERANCE)
     self.assertNear(res_val, 0.0, ERROR_TOLERANCE)
 
+  def test_graph_while_loop(self):
+    graph = ops.Graph()
+    with graph.as_default():
+      max_index = array_ops.placeholder(dtype=dtypes.int32, shape=tuple())
+      index_start = constant_op.constant(1)
+      sum_start = constant_op.constant(0)
+      _, result = control_flow_ops.while_loop(
+          cond=lambda i, unused_s: i <= max_index,
+          body=lambda i, s: (i + 1, s + i),
+          loop_vars=[index_start, sum_start])
+    copied_graph = ops.Graph()
+    _, copy_info = ge.copy(
+        graph, dst_graph=copied_graph, dst_scope="imported")
+    copied_result = copy_info.transformed(result)
+    copied_max_index = copy_info.transformed(max_index)
+    with copied_graph.as_default():
+      with session.Session() as sess:
+        n = 10
+        sum_val = sess.run(copied_result, feed_dict={copied_max_index: n})
+        self.assertEqual(sum_val, 55)
+
+  def test_graph_cond(self):
+    graph = ops.Graph()
+    with graph.as_default():
+      choice = array_ops.placeholder(shape=(), dtype=dtypes.bool)
+      result = control_flow_ops.cond(
+          choice,
+          lambda: constant_op.constant(1),
+          lambda: constant_op.constant(2))
+    copied_graph = ops.Graph()
+    _, copy_info = ge.copy(
+        graph, dst_graph=copied_graph, dst_scope="imported")
+    copied_result = copy_info.transformed(result)
+    copied_choice = copy_info.transformed(choice)
+    with copied_graph.as_default():
+      with session.Session() as sess:
+        res = sess.run(copied_result, feed_dict={copied_choice: True})
+        self.assertEqual(res, 1)
+        res = sess.run(copied_result, feed_dict={copied_choice: False})
+        self.assertEqual(res, 2)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/graph_editor/transform.py b/tensorflow/contrib/graph_editor/transform.py
index 14ac529665..d8a48387a7 100644
--- a/tensorflow/contrib/graph_editor/transform.py
+++ b/tensorflow/contrib/graph_editor/transform.py
@@ -129,20 +129,26 @@ def transform_op_if_inside_handler(info, op, keep_if_possible=True):
       return None
 
 
-def copy_op_handler(info, op, copy_shape=True):
+def copy_op_handler(info, op, new_inputs, copy_shape=True):
   """Copy a `tf.Operation`.
 
   Args:
     info: Transform._TmpInfo instance.
     op: the `tf.Operation` to be copied.
+    new_inputs: The new inputs for this op.
     copy_shape: also copy the shape of the tensor
   Returns:
     A `(op, op_outputs)` tuple containing the transformed op and its outputs.
   """
+  # The `new_inputs` was added to this function. For compatibility reason,
+  # let's raise an error if `new_inputs` is a boolean.
+  if isinstance(new_inputs, bool):
+    raise TypeError("the `new_inputs` argument must be an iterable.")
+
   # pylint: disable=protected-access
 
   # Clone the node def:
-  node_def_ = deepcopy(op._node_def)
+  node_def_ = deepcopy(op.node_def)
 
   # Transform name:
   name_ = info.new_name(op.name)
@@ -155,10 +161,10 @@ def copy_op_handler(info, op, copy_shape=True):
 
   # Make a copy of the op_def too.
   # Its unique to every _type_ of Operation.
-  op_def_ = deepcopy(op._op_def)
+  op_def_ = deepcopy(op.op_def)
 
   # Initialize a new Operation instance
-  op_ = tf_ops.Operation(node_def_, info.graph_, [], output_types_,
+  op_ = tf_ops.Operation(node_def_, info.graph_, new_inputs, output_types_,
                          [], input_types_, None, op_def_)
 
   # copy the shape over
@@ -170,6 +176,7 @@ def copy_op_handler(info, op, copy_shape=True):
   # attribute to exist, we will create a dummy original_op first and then
   # later finalise it with the actual original_op when all the ops have
   # been copied.
+  # TODO(fkp): Stop worrying about _original_op and remove this code?
   if op._original_op:
     op_._original_op = op._original_op
 
@@ -328,6 +335,14 @@ class _TmpInfo(object):
                             for key in self.graph.get_all_collection_keys())
     self.cyclic_ops = []
     self.transform_original_op_handler = transform_op_if_inside_handler
+    # The graph is transformed op by op, in the same order the original ops
+    # were created. However, this is sometimes not possible due to cycles
+    # (i.e. while loops). So when the transformer creates a new op whose
+    # inputs do not exist yet, temporary placeholders are created and stored
+    # in this `tmp_cyclic_ts` container. During a second pass,
+    # those temporary tensors are replaced by the proper transformed tensors
+    # (see the function `_finalize_cycles`).
+    self.tmp_cyclic_ts = []
 
   def new_name(self, name):
     """Compute a destination name from a source name.
@@ -428,10 +443,10 @@ class Transformer(object):
 
     # Create temporary info used during this transform call
     info = _TmpInfo(sgv, dst_graph, dst_scope, src_scope)
-    info.transform_original_op_handler = self.transform_original_op_handler
 
     self._copy_ops(info)
-    self._connect_ops(info)
+    self._finalize_cycles(info)
+    self._connect_control_inputs(info)
 
     # Compute information about the transformation
     res_info = TransformerInfo(info)
@@ -440,10 +455,10 @@ class Transformer(object):
 
   def _copy_ops(self, info):
     """Copy ops without connecting them."""
-    for op in info.sgv.ops:
-      logging.debug("Copying op: %s", op.name)
-      # TODO(fkp): return a subgraph?
-      op_, op_outputs_ = self.transform_op_handler(info, op)
+    sorted_ops = sorted(info.sgv.ops, key=lambda op: op._id)  # pylint: disable=protected-access
+    for op in sorted_ops:
+      new_inputs = [self._transformed_t(info, t, op) for t in op.inputs]
+      op_, op_outputs_ = self.transform_op_handler(info, op, new_inputs)
       if op is op_:
         raise ValueError("In-place transformation not allowed.")
 
@@ -456,27 +471,36 @@ class Transformer(object):
         info.transformed_ts[op_output] = op_output_
         self.assign_collections_handler(info, op_output, op_output_)
 
-  def _connect_ops(self, info):
+  def _finalize_cycles(self, info):
+    """Reconnects the cyclic tensors."""
+    for t, tmp_t_, consumer_op in info.tmp_cyclic_ts:
+      if t not in info.transformed_ts:
+        raise ValueError("The tensor {} should be transformed by now.".format(
+            t.name))
+      if consumer_op not in info.transformed_ops:
+        raise ValueError("The op {} should be transformed by now.".format(
+            consumer_op.name))
+      t_ = info.transformed_ts[t]
+      consumer_op_ = info.transformed_ops[consumer_op]
+      t_index_ = list(consumer_op_.inputs).index(tmp_t_)
+      consumer_op_._update_input(t_index_, t_, update_dtype=False)  # pylint: disable=protected-access
+
+  def _connect_control_inputs(self, info):
     """Connect the previously copied ops."""
     for op in info.sgv.ops:
-      logging.debug("Finalizing op: %s", op.name)
+      logging.debug("Connecting control inputs of op: %s", op.name)
       op_ = info.transformed_ops[op]
 
-      # pylint: disable=protected-access
-      if op_.inputs:
-        raise ValueError("The newly transformed op should not have "
-                         "any inputs yet: {}".format(op_.name))
-      inputs_ = [self._transformed_t(info, t) for t in op.inputs]
-      for t in inputs_:
-        op_._add_input(t)
-
       # Finalize original op.
+      # TODO(fkp): Stop worrying about _original_op and remove this code?
+      # pylint: disable=protected-access
       if op._original_op:
-        original_op = info.transform_original_op_handler(info, op._original_op)
+        original_op = self.transform_original_op_handler(info, op._original_op)
         if original_op is None:
           logging.debug("Could not find original op for: %s", op_.name)
         else:
           op_._original_op = original_op
+      # pylint: enable=protected-access
 
       # Finalize control inputs:
       control_inputs_ = [self.transform_control_input_handler(info, ci)
@@ -525,19 +549,38 @@ class Transformer(object):
 
     return sgv_.remap(input_map_, output_map_)
 
-  def _transformed_t(self, info, t):
+  def _transformed_t(self, info, t, consumer_op):
     """Return tre transformed tensor of `t`."""
-    if t not in info.transformed_ts:
-      # If op is not in the subgraph.
-      if t in info.sgv_inputs_set:
-        # t is an input of the subgraph.
-        return self.transform_external_input_handler(info, t)
+    if t in info.transformed_ts:
+      # If op is in the subgraph, just return its transformed counterpart.
+      return info.transformed_ts[t]
+
+    if t in info.sgv_inputs_set:
+      # `t` is an input of the subgraph.
+      return self.transform_external_input_handler(info, t)
+    elif t.op in info.ops:
+      # `t` is an internal tensor but is not transformed yet because it
+      # belongs to a graph cycle.
+      logging.debug("Cyclic tensor: t.name = %s", t.name)
+      # Try to find an existing tensor we can use for now,
+      # otherwise create one. We'll rewire this later.
+      if consumer_op.type == "Merge":
+        first_input = consumer_op.inputs[0]
+        tmp_t_ = self._transformed_t(info, first_input, consumer_op)
+      elif t.op.type == "Enter":
+        enter_input = t.op.inputs[0]
+        tmp_t_ = self._transformed_t(info, enter_input, consumer_op)
       else:
-        # t is a hidden input of the subgraph.
-        return self.transform_external_hidden_input_handler(info, t)
+        with info.graph_.as_default():
+          tmp_t_ = util.make_placeholder_from_tensor(t, scope=info.scope_,
+                                                     prefix="geph_tmp")
+        logging.debug("Created temporary placeholder: %s.", tmp_t_.name)
+      # Register as temporary and return.
+      info.tmp_cyclic_ts.append((t, tmp_t_, consumer_op))
+      return tmp_t_
     else:
-      # If op is in the subgraph, just return its transformed.
-      return info.transformed_ts[t]
+      # `t` is a hidden input of the subgraph.
+      return self.transform_external_hidden_input_handler(info, t)
 
 
 def copy(sgv, dst_graph=None, dst_scope="", src_scope="",
@@ -624,6 +667,40 @@ def copy_with_input_replacements(sgv, replacement_ts,
       sgv, dst_graph, dst_scope, src_scope, reuse_dst_scope=reuse_dst_scope)
 
 
+def _add_control_flow_ops(ops, control_ios):
+  """Complete `ops` so that the tranformed graph is valid.
+
+  Partially copying a graph can lead to a malformed graph. For instance,
+  copying half of a while construct is likely to result in an invalid graph.
+  This function attempts to add missing ops so that the transformation result
+  in a valid graph.
+
+  Args:
+    ops: list of ops (modifed in-place).
+    control_ios: object created by a call to `util.ControlOutputs`.
+  """
+  # Find while contexts.
+  control_flow_contexts = set()
+  for op in ops:
+    cfc = op._control_flow_context  # pylint: disable=protected-access
+    if cfc:
+      control_flow_contexts.add(cfc)
+  # Find new ops.
+  new_ops = []
+  for cfc in control_flow_contexts:
+    if cfc.IsWhileContext():
+      new_ops += select.get_walks_intersection_ops(
+          [enter_t.op for enter_t in cfc.loop_enters],
+          [exit_t.op for exit_t in cfc.loop_exits],
+          control_ios=control_ios)
+  # Add new ops.
+  new_ops_set = set(new_ops)
+  ops_set = frozenset(ops)
+  for op in new_ops_set:
+    if op not in ops_set:
+      ops.append(op)
+
+
 def graph_replace(target_ts, replacement_ts, dst_scope="",
                   src_scope="", reuse_dst_scope=False):
   """Create a new graph which compute the targets from the replaced Tensors.
@@ -657,8 +734,13 @@ def graph_replace(target_ts, replacement_ts, dst_scope="",
                                           control_ios=control_ios)
   if not ops:
     raise ValueError("Targets and replacements are not connected!")
+
+  # Complete ops to avoid malformed control flow.
+  # TODO(fkp): Consider moving this function deeper (in the transformer?).
+  _add_control_flow_ops(ops, control_ios)
+
   # Create a copy of the relevant subgraph
-  _, info = copy_with_input_replacements(
+  unused_sgv_, info = copy_with_input_replacements(
       ops, replacement_ts, None, dst_scope, src_scope, reuse_dst_scope)
   # Return the transformed targets but keep the original if the transformed
   # counterpart cannot be found
diff --git a/tensorflow/contrib/graph_editor/util.py b/tensorflow/contrib/graph_editor/util.py
index 30bc33b9ee..584f4509cc 100644
--- a/tensorflow/contrib/graph_editor/util.py
+++ b/tensorflow/contrib/graph_editor/util.py
@@ -38,6 +38,11 @@ __all__ = [
 ]
 
 
+# The graph editor sometimes need to create placeholders, they are named
+# "geph_*". "geph" stands for Graph-Editor PlaceHolder.
+_DEFAULT_PLACEHOLDER_PREFIX = "geph"
+
+
 def concatenate_unique(la, lb):
   """Add all the elements of `lb` to `la` if they are not there already.
 
@@ -405,7 +410,7 @@ def scope_basename(scope):
   return scope[slash + 1:]
 
 
-def placeholder_name(t=None, scope=None):
+def placeholder_name(t=None, scope=None, prefix=_DEFAULT_PLACEHOLDER_PREFIX):
   """Create placeholder name for the graph editor.
 
   Args:
@@ -413,6 +418,7 @@ def placeholder_name(t=None, scope=None):
       on
     scope: absolute scope with which to prefix the placeholder's name. None
       means that the scope of t is preserved. "" means the root scope.
+    prefix: placeholder name prefix.
   Returns:
     A new placeholder name prefixed by "geph". Note that "geph" stands for
       Graph Editor PlaceHolder. This convention allows to quickly identify the
@@ -430,19 +436,20 @@ def placeholder_name(t=None, scope=None):
     if scope is None:
       scope = op_dirname
 
-    if op_basename.startswith("geph__"):
+    if op_basename.startswith("{}__".format(prefix)):
       ph_name = op_basename
     else:
-      ph_name = "geph__{}_{}".format(op_basename, t.value_index)
+      ph_name = "{}__{}_{}".format(prefix, op_basename, t.value_index)
 
     return scope + ph_name
   else:
     if scope is None:
       scope = ""
-    return scope + "geph"
+    return "{}{}".format(scope, prefix)
 
 
-def make_placeholder_from_tensor(t, scope=None):
+def make_placeholder_from_tensor(t, scope=None,
+                                 prefix=_DEFAULT_PLACEHOLDER_PREFIX):
   """Create a `tf.placeholder` for the Graph Editor.
 
   Note that the correct graph scope must be set by the calling function.
@@ -452,17 +459,19 @@ def make_placeholder_from_tensor(t, scope=None):
       (see function placeholder_name).
     scope: absolute scope within which to create the placeholder. None
       means that the scope of `t` is preserved. `""` means the root scope.
+    prefix: placeholder name prefix.
   Returns:
     A newly created `tf.placeholder`.
   Raises:
     TypeError: if `t` is not `None` or a `tf.Tensor`.
   """
   return tf_array_ops.placeholder(
-      dtype=t.dtype, shape=t.get_shape(), name=placeholder_name(
-          t, scope=scope))
+      dtype=t.dtype, shape=t.get_shape(),
+      name=placeholder_name(t, scope=scope, prefix=prefix))
 
 
-def make_placeholder_from_dtype_and_shape(dtype, shape=None, scope=None):
+def make_placeholder_from_dtype_and_shape(dtype, shape=None, scope=None,
+                                          prefix=_DEFAULT_PLACEHOLDER_PREFIX):
   """Create a tf.placeholder for the Graph Editor.
 
   Note that the correct graph scope must be set by the calling function.
@@ -474,11 +483,13 @@ def make_placeholder_from_dtype_and_shape(dtype, shape=None, scope=None):
     shape: the tensor shape (optional).
     scope: absolute scope within which to create the placeholder. None
       means that the scope of t is preserved. "" means the root scope.
+    prefix: placeholder name prefix.
   Returns:
     A newly created tf.placeholder.
   """
   return tf_array_ops.placeholder(
-      dtype=dtype, shape=shape, name=placeholder_name(scope=scope))
+      dtype=dtype, shape=shape,
+      name=placeholder_name(scope=scope, prefix=prefix))
 
 
 _INTERNAL_VARIABLE_RE = re.compile(r"^__\w+__$")
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 50a1d3fe04..b3fa39fdab 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -1916,7 +1916,8 @@ class Operation(object):
     tensor._add_consumer(self)  # pylint: disable=protected-access
     self._recompute_node_def()
 
-  def _update_input(self, index, tensor):
+  # TODO(skyewm): Remove `update_dtype` when we enable the C API.
+  def _update_input(self, index, tensor, update_dtype=True):
     """Update the input to this operation at the given index.
 
     NOTE: This is for TF internal use only. Please don't use it.
@@ -1924,6 +1925,7 @@ class Operation(object):
     Args:
       index: the index of the input to update.
       tensor: the Tensor to be used as the input at the given index.
+      update_dtype: If `False`, the type for this input is not updated.
 
     Raises:
       TypeError: if tensor is not a Tensor,
@@ -1943,7 +1945,8 @@ class Operation(object):
     else:
       self._inputs_val[index].consumers().remove(self)
       self._inputs_val[index] = tensor
-      self._input_types_val[index] = tensor.dtype
+      if update_dtype:
+        self._input_types_val[index] = tensor.dtype
       tensor._add_consumer(self)  # pylint: disable=protected-access
       self._recompute_node_def()
 
-- 
GitLab


From 7a60167ba7718c23b0ed70d079bbb446f63a4fd9 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Wed, 21 Mar 2018 11:41:12 -0700
Subject: [PATCH 308/960] Don't run data_utils_test without optimizations.

PiperOrigin-RevId: 189941645
---
 tensorflow/python/keras/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 3180b9f410..711106d2db 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -613,6 +613,7 @@ py_test(
         "no_windows",
         "noasan",  # times out
         "notsan",
+        "optonly",  # times out
     ],
     deps = [
         ":keras",
-- 
GitLab


From cbede3ea7574b36f429710bc08617d08455bcc21 Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Wed, 21 Mar 2018 12:00:04 -0700
Subject: [PATCH 309/960] Fix compilation error with clang.

Link to breaking CI build:
https://ci.tensorflow.org/view/Nightly/job/nightly-matrix-linux-gpu-clang/232

PiperOrigin-RevId: 189944547
---
 tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
index f86aff47e1..e6811d4ad2 100644
--- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
+++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
@@ -147,7 +147,7 @@ int main(int argc, char** argv) {
   tensorflow::string session_id =
       tensorflow::tpu::GetCurrentTimeStampAsString();
   constexpr char kProfilePluginDirectory[] = "plugins/profile/";
-  string repository_root =
+  tensorflow::string repository_root =
       ::tensorflow::io::JoinPath(FLAGS_logdir, kProfilePluginDirectory);
   while (true) {
     std::cout << "Starting to profile TPU traces for " << duration_ms << " ms. "
-- 
GitLab


From 2d0531d72c7dcbb0e149cafdd3a16ee8c3ff357a Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Wed, 21 Mar 2018 12:07:51 -0700
Subject: [PATCH 310/960] Merge changes from github.

PiperOrigin-RevId: 189945839
---
 README.md                                     |    4 +
 SECURITY.md                                   |   16 +-
 configure.py                                  |    5 +-
 .../xla/service/generic_transfer_manager.cc   |    9 +-
 .../compiler/xla/tests/convolution_test.cc    |    2 +-
 tensorflow/contrib/BUILD                      |    3 +-
 tensorflow/contrib/cmake/README.md            |   12 +-
 tensorflow/contrib/cmake/external/grpc.cmake  |    1 +
 .../contrib/cmake/external/protobuf.cmake     |    2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |    4 +
 tensorflow/contrib/data/__init__.py           |    4 +
 .../contrib/data/python/kernel_tests/BUILD    |   17 +
 .../data/python/kernel_tests/resample_test.py |    4 +-
 .../kernel_tests/slide_dataset_op_test.py     |  242 +++
 tensorflow/contrib/data/python/ops/BUILD      |    1 +
 .../contrib/data/python/ops/resampling.py     |    4 +-
 tensorflow/contrib/data/python/ops/sliding.py |  102 ++
 tensorflow/contrib/factorization/BUILD        |    5 +-
 .../contrib/ffmpeg/default/ffmpeg_lib.cc      |    9 +-
 .../contrib/framework/python/ops/arg_scope.py |    2 +-
 .../eval/python/classifier_metrics_impl.py    |    8 +-
 .../eval/python/sliced_wasserstein_impl.py    |    4 +-
 .../python/conditioning_utils_impl.py         |    2 +-
 .../python/random_tensor_pool_impl.py         |    4 +-
 .../features/python/virtual_batchnorm_test.py |    2 +-
 .../grid_rnn/python/ops/grid_rnn_cell.py      |    2 +-
 tensorflow/contrib/image/BUILD                |    1 +
 tensorflow/contrib/kafka/BUILD                |  108 +-
 .../kafka/kernels/kafka_dataset_ops.cc        |    4 +-
 tensorflow/contrib/kafka/ops/dataset_ops.cc   |   44 +
 .../kafka/python/ops/kafka_dataset_ops.py     |    9 +-
 .../kafka/python/ops/kafka_op_loader.py       |   24 +
 .../contrib/kfac/python/ops/optimizer.py      |    2 +-
 .../layers/python/layers/embedding_ops.py     |    2 +-
 .../contrib/layers/python/layers/encoders.py  |    2 +-
 tensorflow/contrib/learn/BUILD                |    7 +-
 .../python/learn/estimators/estimator.py      |    4 +-
 .../contrib/learn/python/learn/experiment.py  |    2 +-
 .../learn/python/learn/ops/embeddings_ops.py  |    2 +-
 tensorflow/contrib/lite/Makefile              |    9 +-
 tensorflow/contrib/lite/README.md             |    4 +-
 tensorflow/contrib/lite/arena_planner.h       |    2 +-
 tensorflow/contrib/lite/build_rpi_lib.sh      |   22 +
 tensorflow/contrib/lite/error_reporter.h      |    2 +-
 tensorflow/contrib/lite/g3doc/ios.md          |    9 +
 tensorflow/contrib/lite/g3doc/rpi.md          |   50 +
 tensorflow/contrib/lite/interpreter.h         |    2 +-
 tensorflow/contrib/lite/interpreter_test.cc   |    2 +-
 tensorflow/contrib/lite/kernels/conv.cc       |    2 +-
 .../contrib/lite/kernels/depthwise_conv.cc    |    2 +-
 .../contrib/lite/kernels/fully_connected.cc   |    2 +-
 .../lite/kernels/internal/spectrogram.cc      |    1 +
 tensorflow/contrib/lite/kernels/kernel_util.h |    2 +-
 .../contrib/lite/kernels/lsh_projection.cc    |    2 +-
 tensorflow/contrib/lite/kernels/lstm.cc       |    6 +-
 tensorflow/contrib/lite/kernels/reshape.cc    |   12 +-
 .../contrib/lite/kernels/reshape_test.cc      |    2 +-
 tensorflow/contrib/lite/kernels/test_util.cc  |    4 +-
 .../kernels/unidirectional_sequence_lstm.cc   |    2 +-
 tensorflow/contrib/lite/memory_planner.h      |    4 +-
 tensorflow/contrib/lite/model.h               |    2 +-
 .../contrib/lite/nnapi/NeuralNetworksShim.h   |    2 +-
 tensorflow/contrib/lite/rpi_makefile.inc      |   33 +
 .../contrib/lite/schema/upgrade_schema.py     |    8 +-
 .../contrib/lite/simple_memory_arena.cc       |    6 +-
 tensorflow/contrib/lite/simple_memory_arena.h |    6 +-
 tensorflow/contrib/makefile/README.md         |    2 +
 tensorflow/contrib/makefile/build_all_ios.sh  |    5 +-
 .../contrib/metrics/python/ops/metric_ops.py  |    2 +-
 .../model_pruning/python/layers/layers.py     |    2 +-
 .../contrib/model_pruning/python/pruning.py   |    2 +-
 tensorflow/contrib/mpi/mpi_utils.h            |    2 +
 .../contrib/predictor/predictor_factories.py  |    4 +-
 .../contrib/py2tf/converters/single_return.py |    2 +-
 .../quantize/python/fold_batch_norms.py       |    4 +-
 .../contrib/quantize/python/quant_ops.py      |    4 +-
 .../contrib/quantize/python/quantize.py       |    2 +-
 .../contrib/quantize/python/quantize_graph.py |    2 +-
 .../python/quantize_parameterized_test.py     |    8 +-
 .../contrib/quantize/python/quantize_test.py  |    2 +-
 tensorflow/contrib/rnn/ops/gru_ops.cc         |    2 +-
 .../rnn/python/kernel_tests/lstm_ops_test.py  |    2 +-
 tensorflow/contrib/rnn/python/ops/lstm_ops.py |    3 +-
 tensorflow/contrib/rnn/python/ops/rnn_cell.py |   10 +-
 .../kernel_tests/attention_wrapper_test.py    |    3 +
 .../kernel_tests/beam_search_decoder_test.py  |  104 +-
 .../seq2seq/python/ops/attention_wrapper.py   |   19 +-
 .../seq2seq/python/ops/beam_search_decoder.py |  176 +-
 tensorflow/contrib/slim/README.md             |    2 +-
 .../solvers/python/ops/least_squares.py       |    2 +-
 .../solvers/python/ops/linear_equations.py    |    2 +-
 tensorflow/contrib/tensorrt/BUILD             |    2 +
 tensorflow/contrib/tensorrt/README.md         |   23 +-
 tensorflow/contrib/tensorrt/__init__.py       |   18 +-
 .../contrib/tensorrt/convert/convert_graph.cc |  256 ++-
 .../contrib/tensorrt/convert/convert_graph.h  |   10 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 1481 ++++++++++++++---
 .../contrib/tensorrt/convert/convert_nodes.h  |   53 +-
 .../contrib/tensorrt/kernels/trt_calib_op.cc  |   11 +-
 .../contrib/tensorrt/kernels/trt_engine_op.cc |   39 +-
 tensorflow/contrib/tensorrt/log/trt_logger.cc |    8 +-
 tensorflow/contrib/tensorrt/log/trt_logger.h  |    4 +-
 .../contrib/tensorrt/python/__init__.py       |    1 +
 .../contrib/tensorrt/python/trt_convert.py    |   70 +-
 .../tensorrt/resources/trt_int8_calibrator.cc |   56 +-
 .../tensorrt/resources/trt_int8_calibrator.h  |   15 +-
 .../contrib/tensorrt/test/test_tftrt.py       |   57 +-
 tensorflow/contrib/tensorrt/trt_conversion.i  |   63 +-
 .../contrib/tpu/ops/tpu_embedding_ops.cc      |    6 +-
 .../tpu/python/tpu/device_assignment.py       |    4 +-
 .../contrib/tpu/python/tpu/tpu_config.py      |    2 +-
 .../contrib/tpu/python/tpu/tpu_context.py     |    4 +-
 .../contrib/tpu/python/tpu/tpu_estimator.py   |   12 +-
 .../contrib/tpu/python/tpu/training_loop.py   |    2 +-
 tensorflow/core/BUILD                         |    4 +
 .../base_api/api_def_SelfAdjointEig.pbtxt     |    3 +-
 .../base_api/api_def_SelfAdjointEigV2.pbtxt   |    3 +-
 .../base_api/api_def_SlideDataset.pbtxt       |   18 +
 .../core/distributed_runtime/tensor_coding.cc |    4 +-
 tensorflow/core/graph/mkl_layout_pass.cc      |   31 +-
 .../grappler/optimizers/loop_optimizer.cc     |    8 +-
 tensorflow/core/kernels/BUILD                 |   12 +-
 tensorflow/core/kernels/concat_op.cc          |   98 +-
 tensorflow/core/kernels/conv_ops_test.cc      |    2 +-
 tensorflow/core/kernels/data/BUILD            |   14 +
 .../core/kernels/data/slide_dataset_op.cc     |  252 +++
 tensorflow/core/kernels/depthtospace_op.cc    |    3 +
 .../core/kernels/depthtospace_op_gpu.cu.cc    |    6 +
 tensorflow/core/kernels/hexagon/BUILD         |    1 +
 .../core/kernels/mkl_conv_grad_filter_ops.cc  |   81 +-
 .../core/kernels/mkl_conv_grad_input_ops.cc   |   18 +-
 tensorflow/core/kernels/mkl_conv_ops.cc       |  146 +-
 tensorflow/core/kernels/mkl_conv_ops.h        |  117 +-
 .../core/kernels/mkl_input_conversion_op.cc   |    7 +-
 tensorflow/core/kernels/mkl_relu_op.cc        |   23 +-
 .../core/kernels/segment_reduction_ops.h      |    8 +
 tensorflow/core/kernels/spacetodepth_op.cc    |    3 +
 .../core/kernels/spacetodepth_op_gpu.cu.cc    |    6 +
 tensorflow/core/lib/io/record_reader.cc       |    2 +
 tensorflow/core/lib/io/record_reader.h        |    4 +-
 tensorflow/core/ops/dataset_ops.cc            |   12 +-
 tensorflow/core/ops/nn_ops.cc                 |    8 +
 .../platform/windows/windows_file_system.cc   |    3 +-
 tensorflow/core/public/version.h              |    2 +-
 tensorflow/core/util/stat_summarizer.h        |    2 +-
 tensorflow/docs_src/community/welcome.md      |    4 +-
 tensorflow/docs_src/install/install_c.md      |    2 +-
 tensorflow/docs_src/install/install_go.md     |    2 +-
 tensorflow/docs_src/install/install_java.md   |   22 +-
 tensorflow/docs_src/install/install_linux.md  |   66 +-
 tensorflow/docs_src/install/install_mac.md    |   23 +-
 .../docs_src/install/install_sources.md       |    4 +-
 .../docs_src/install/install_windows.md       |    5 +-
 .../docs_src/performance/performance_guide.md |    2 +-
 tensorflow/docs_src/performance/xla/jit.md    |    2 +-
 .../docs_src/programmers_guide/debugger.md    |    3 +-
 tensorflow/docs_src/programmers_guide/faq.md  |    4 +-
 .../docs_src/programmers_guide/saved_model.md |    4 +-
 .../summaries_and_tensorboard.md              |    2 +-
 .../docs_src/programmers_guide/using_tpu.md   |    7 +-
 tensorflow/docs_src/tutorials/deep_cnn.md     |    2 +-
 .../docs_src/tutorials/image_retraining.md    |    2 +-
 .../docs_src/tutorials/kernel_methods.md      |    6 +-
 tensorflow/docs_src/tutorials/layers.md       |   12 +-
 .../docs_src/tutorials/recurrent_quickdraw.md |    3 +-
 tensorflow/docs_src/tutorials/wide.md         |   16 +-
 .../examples/android/AndroidManifest.xml      |    4 +
 .../org/tensorflow/demo/CameraActivity.java   |    7 +-
 .../org/tensorflow/demo/StylizeActivity.java  |   60 +
 tensorflow/examples/ios/README.md             |    6 +-
 tensorflow/examples/learn/mnist.py            |    6 +-
 tensorflow/examples/learn/resnet.py           |   12 +-
 tensorflow/python/BUILD                       |   12 +-
 tensorflow/python/client/timeline_test.py     |    7 +-
 tensorflow/python/estimator/estimator.py      |   34 +-
 tensorflow/python/estimator/run_config.py     |    2 +-
 tensorflow/python/estimator/training.py       |   26 +-
 .../keras/_impl/keras/engine/training.py      |    2 +-
 .../keras/_impl/keras/layers/recurrent.py     |    4 +-
 .../keras/_impl/keras/utils/generic_utils.py  |    4 +-
 .../keras/_impl/keras/utils/vis_utils.py      |    2 +-
 .../python/kernel_tests/concat_op_test.py     |   11 +
 .../python/kernel_tests/conv_ops_test.py      |   20 +-
 .../kernel_tests/depthtospace_op_test.py      |   10 +-
 .../kernel_tests/spacetodepth_op_test.py      |   10 +-
 tensorflow/python/layers/base.py              |    2 +-
 tensorflow/python/layers/normalization.py     |    9 +-
 tensorflow/python/lib/io/file_io_test.py      |    5 +
 tensorflow/python/lib/io/tf_record.py         |   18 +-
 tensorflow/python/ops/linalg_ops.py           |    2 +-
 tensorflow/python/ops/nn_ops.py               |   16 +-
 tensorflow/python/ops/random_ops.py           |    2 +-
 tensorflow/python/ops/rnn.py                  |   17 +-
 tensorflow/python/ops/special_math_ops.py     |    4 +-
 .../python/ops/special_math_ops_test.py       |    5 +
 tensorflow/python/tools/freeze_graph.py       |   36 +-
 tensorflow/python/tools/inspect_checkpoint.py |    4 +-
 tensorflow/python/tools/saved_model_cli.py    |   60 +
 .../python/tools/saved_model_cli_test.py      |   22 +
 tensorflow/python/training/saver.py           |    5 +-
 tensorflow/stream_executor/cuda/cuda_dnn.cc   |   33 +-
 .../tools/api/tests/api_compatibility_test.py |    2 +
 tensorflow/tools/ci_build/Dockerfile.cmake    |    5 +-
 tensorflow/tools/compatibility/tf_upgrade.py  |    6 +-
 tensorflow/tools/dist_test/README.md          |    8 +
 tensorflow/tools/dist_test/local_test.sh      |   22 +-
 .../tools/dist_test/python/mnist_replica.py   |    2 +-
 tensorflow/tools/docker/Dockerfile.gpu        |    9 +-
 tensorflow/tools/git/gen_git_source.py        |    7 +
 tensorflow/tools/graph_transforms/BUILD       |    1 +
 .../graph_transforms/fold_old_batch_norms.cc  |   67 +
 .../fold_old_batch_norms_test.cc              |   97 +-
 tensorflow/tools/pip_package/BUILD            |    1 +
 tensorflow/tools/pip_package/setup.py         |    4 +-
 .../tools/test/upload_test_benchmarks.py      |    9 +-
 third_party/jpeg/jpeg.BUILD                   |    4 +-
 third_party/kafka/BUILD                       |   13 +-
 third_party/py/BUILD.tpl                      |   22 +-
 third_party/tensorrt/tensorrt_configure.bzl   |    4 +
 219 files changed, 4312 insertions(+), 990 deletions(-)
 create mode 100644 tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
 create mode 100644 tensorflow/contrib/data/python/ops/sliding.py
 create mode 100644 tensorflow/contrib/kafka/ops/dataset_ops.cc
 create mode 100644 tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
 create mode 100755 tensorflow/contrib/lite/build_rpi_lib.sh
 create mode 100644 tensorflow/contrib/lite/g3doc/rpi.md
 create mode 100644 tensorflow/contrib/lite/rpi_makefile.inc
 create mode 100644 tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
 create mode 100644 tensorflow/core/kernels/data/slide_dataset_op.cc

diff --git a/README.md b/README.md
index ef5bdc66ef..3cdb6e478d 100644
--- a/README.md
+++ b/README.md
@@ -22,6 +22,10 @@ organization for the purposes of conducting machine learning and deep neural
 networks research.  The system is general enough to be applicable in a wide
 variety of other domains, as well.
 
+Keep up to date with release announcements and security updates by
+subscribing to
+[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
+
 ## Installation
 *See [Installing TensorFlow](https://www.tensorflow.org/get_started/os_setup.html) for instructions on how to install our release binaries or how to build from source.*
 
diff --git a/SECURITY.md b/SECURITY.md
index fea24b2739..378e776967 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -6,7 +6,7 @@ report vulnerabilities in TensorFlow.
 
 ## TensorFlow models are programs
 
-TensorFlow's runtime system interprets and executes programs. What machine 
+TensorFlow's runtime system interprets and executes programs. What machine
 learning practitioners term
 [**models**](https://developers.google.com/machine-learning/glossary/#model) are
 expressed as programs that TensorFlow executes.  TensorFlow programs are encoded
@@ -28,12 +28,12 @@ data you supply to TensorFlow to train a model, or to use a model to run
 inference on the data.
 
 **TensorFlow models are programs, and need to be treated as such from a security
-perspective.** 
+perspective.**
 
 ## Running untrusted models
 
 As a general rule: **Always** execute untrusted models inside a sandbox (e.g.,
-[nsjail](https://github.com/google/nsjail)). 
+[nsjail](https://github.com/google/nsjail)).
 
 There are several ways in which a model could become untrusted. Obviously, if an
 untrusted party supplies TensorFlow kernels, arbitrary code may be executed.
@@ -109,11 +109,11 @@ graphs known to the `ModelServer`. This means that an attacker may run
 graphs using untrusted inputs as described above, but they would not be able to
 execute arbitrary graphs. It is possible to safely expose a `ModelServer`
 directly to an untrusted network, **but only if the graphs it is configured to
-use have been carefully audited to be safe**. 
+use have been carefully audited to be safe**.
 
 Similar to best practices for other servers, we recommend running any
 `ModelServer` with appropriate privileges (i.e., using a separate user with
-reduced permisisons). In the spirit of defense in depth, we recommend
+reduced permissions). In the spirit of defense in depth, we recommend
 authenticating requests to any TensorFlow server connected to an untrusted
 network, as well as sandboxing the server to minimize the adverse effects of
 any breach.
@@ -129,11 +129,11 @@ with specially crafted inputs.
 ### What is a vulnerability?
 
 Given TensorFlow's flexibility, it is possible to specify computation graphs
-which exhibit unexpected or unwanted behaviors. The fact that TensorFlow models
+which exhibit unexpected or unwanted behavior. The fact that TensorFlow models
 can perform arbitrary computations means that they may read and write files,
 communicate via the network, produce deadlocks and infinite loops, or run out
 of memory. It is only when these behaviors are outside the specifications of the
-operations involved that such behavior is a vulnerability. 
+operations involved that such behavior is a vulnerability.
 
 A `FileWriter` writing a file is not unexpected behavior and therefore is not a
 vulnerability in TensorFlow. A `MatMul` allowing arbitrary binary code execution
@@ -168,7 +168,7 @@ below).
 
 Please use a descriptive subject line for your report email. After the initial
 reply to your report, the security team will endeavor to keep you informed of
-the progress being made towards a fix and announcement. 
+the progress being made towards a fix and announcement.
 
 If you believe that an existing (public) issue is security-related, please send
 an email to `security@tensorflow.org`. The email should include the issue ID and
diff --git a/configure.py b/configure.py
index 97f46757ee..7d61c2e5e3 100644
--- a/configure.py
+++ b/configure.py
@@ -1048,7 +1048,10 @@ def set_tf_tensorrt_install_path(environ_cp):
 
     for lib_file in possible_files:
       if is_compatible(lib_file, cuda_ver, cudnn_ver):
-        ver_str = nvinfer_pattern.search(lib_file).group(1)
+        matches = nvinfer_pattern.search(lib_file)
+        if len(matches.groups()) == 0:
+          continue
+        ver_str = matches.group(1)
         ver = convert_version_to_int(ver_str) if len(ver_str) else 0
         if ver > highest_ver[0]:
           highest_ver = [ver, ver_str, lib_file]
diff --git a/tensorflow/compiler/xla/service/generic_transfer_manager.cc b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
index 78dc0ad4fc..a99e2b7794 100644
--- a/tensorflow/compiler/xla/service/generic_transfer_manager.cc
+++ b/tensorflow/compiler/xla/service/generic_transfer_manager.cc
@@ -38,14 +38,7 @@ namespace xla {
 
 GenericTransferManager::GenericTransferManager(se::Platform::Id platform_id,
                                                size_t pointer_size)
-    : platform_id_(platform_id), pointer_size_(pointer_size) {
-  // We currently only support kHostPlatformId for CPU, kCudaPlatformId for
-  // GPU and kInterpreterPlatformId for Interpreter. Before supporting other
-  // platforms, we need to test this transfer manager on them.
-  CHECK(platform_id_ == se::host::kHostPlatformId ||
-        platform_id_ == se::interpreter::kInterpreterPlatformId ||
-        platform_id_ == se::cuda::kCudaPlatformId);
-}
+    : platform_id_(platform_id), pointer_size_(pointer_size) {}
 
 se::Platform::Id GenericTransferManager::PlatformId() const {
   return platform_id_;
diff --git a/tensorflow/compiler/xla/tests/convolution_test.cc b/tensorflow/compiler/xla/tests/convolution_test.cc
index 99640f5bb5..72715398de 100644
--- a/tensorflow/compiler/xla/tests/convolution_test.cc
+++ b/tensorflow/compiler/xla/tests/convolution_test.cc
@@ -723,7 +723,7 @@ INSTANTIATE_TEST_CASE_P(
 );
 #endif
 
-TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
+XLA_TEST_F(ConvolutionTest, Convolve_bf16_1x1x1x2_1x1x1x2_Valid) {
   ComputationBuilder builder(client_, TestName());
   Shape input_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
   Shape filter_shape = ShapeUtil::MakeShape(BF16, {1, 1, 1, 2});
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index bab37e8906..d103da79e3 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -121,6 +121,7 @@ cc_library(
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
+        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -147,7 +148,7 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:kafka_ops_op_lib",
+        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
diff --git a/tensorflow/contrib/cmake/README.md b/tensorflow/contrib/cmake/README.md
index 8f85a75ee4..fe83bb3204 100644
--- a/tensorflow/contrib/cmake/README.md
+++ b/tensorflow/contrib/cmake/README.md
@@ -26,7 +26,7 @@ The CMake files in this directory can build the core TensorFlow runtime, an
 example C++ binary, and a PIP package containing the runtime and Python
 bindings.
 
-### Pre-requisites
+### Prerequisites
 
 * CMake version 3.5 or later.
 
@@ -34,14 +34,16 @@ bindings.
 
 * [SWIG](http://www.swig.org/download.html)
 
-* Additional pre-requisites for Microsoft Windows:
+* Additional prerequisites for Microsoft Windows:
   - Visual Studio 2015
   - Python 3.5
-  - NumPy 1.11.0 or later
 
-* Additional pre-requisites for Linux:
+* Additional prerequisites for Linux:
   - Python 2.7 or later
   - [Docker](https://www.docker.com/) (for automated testing)
+
+* Python dependencies:
+  - wheel
   - NumPy 1.11.0 or later
 
 ### Known-good configurations
@@ -102,7 +104,7 @@ ops or APIs.
 Step-by-step Windows build
 ==========================
 
-1. Install the pre-requisites detailed above, and set up your environment.
+1. Install the prerequisites detailed above, and set up your environment.
 
    * The following commands assume that you are using the Windows Command
      Prompt (`cmd.exe`). You will need to set up your environment to use the
diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index 95106dba1f..cc218e8ab8 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -35,6 +35,7 @@ else()
   set(grpc_STATIC_LIBRARIES
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
+      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
 endif()
 
diff --git a/tensorflow/contrib/cmake/external/protobuf.cmake b/tensorflow/contrib/cmake/external/protobuf.cmake
index aba8a5244e..ab464bc99a 100644
--- a/tensorflow/contrib/cmake/external/protobuf.cmake
+++ b/tensorflow/contrib/cmake/external/protobuf.cmake
@@ -16,7 +16,7 @@ include (ExternalProject)
 
 set(PROTOBUF_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/protobuf/src/protobuf/src)
 set(PROTOBUF_URL https://github.com/google/protobuf.git)
-set(PROTOBUF_TAG 396336eb961b75f03b25824fe86cf6490fb75e3a)
+set(PROTOBUF_TAG b04e5cba356212e4e8c66c61bbe0c3a20537c5b9)
 
 if(WIN32)
   if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index cdf48b3584..237f4fe33a 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -478,6 +478,10 @@ if (tensorflow_BUILD_CC_TESTS)
     "${tensorflow_source_dir}/tensorflow/core/profiler/internal/advisor/*_test.cc"
   )
 
+  list(REMOVE_ITEM tf_test_src_simple
+    ${tf_core_profiler_test_srcs}
+  )
+
   set(tf_test_lib tf_test_lib)
   add_library(${tf_test_lib} STATIC ${tf_src_testlib})
 
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index f09d156832..9212b69700 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -40,6 +40,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@rejection_resample
 @@scan
 @@shuffle_and_repeat
+@@sliding_window_batch
 @@sloppy_interleave
 @@unbatch
 
@@ -72,6 +73,9 @@ from tensorflow.contrib.data.python.ops.readers import SqlDataset
 from tensorflow.contrib.data.python.ops.resampling import rejection_resample
 from tensorflow.contrib.data.python.ops.scan_ops import scan
 from tensorflow.contrib.data.python.ops.shuffle_ops import shuffle_and_repeat
+from tensorflow.contrib.data.python.ops.sliding import sliding_window_batch
+from tensorflow.python.data.ops.iterator_ops import Iterator
+from tensorflow.python.ops.parsing_ops import parse_single_example_v2 as parse_single_example
 # pylint: enable=unused-import
 
 from tensorflow.python.util.all_util import remove_undocumented
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 22418b38e3..2c4d4adfda 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -498,6 +498,23 @@ py_test(
     ],
 )
 
+tf_py_test(
+    name = "slide_dataset_op_test",
+    size = "small",
+    srcs = ["slide_dataset_op_test.py"],
+    additional_deps = [
+        "//tensorflow/contrib/data/python/ops:dataset_ops",
+        "//tensorflow/contrib/data/python/ops:transformation_ops",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:sparse_tensor",
+        "//third_party/py/numpy",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 3c7b46629e..913ab9b9f8 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -45,12 +45,10 @@ class ResampleTest(test.TestCase):
                 target_dist=target_dist,
                 initial_dist=initial_dist,
                 class_func=lambda c, _: c,
-                seed=27)).make_initializable_iterator())
-    init_op = iterator.initializer
+                seed=27)).make_one_shot_iterator())
     get_next = iterator.get_next()
 
     with self.test_session() as sess:
-      sess.run(init_op)
       returned = []
       with self.assertRaises(errors.OutOfRangeError):
         while True:
diff --git a/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
new file mode 100644
index 0000000000..33c48e20be
--- /dev/null
+++ b/tensorflow/contrib/data/python/kernel_tests/slide_dataset_op_test.py
@@ -0,0 +1,242 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the experimental input pipeline ops."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.data.python.ops import sliding
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+class SlideDatasetTest(test.TestCase):
+
+  def testSlideDataset(self):
+    """Test an dataset that maps a TF function across its input elements."""
+    components = (np.arange(7),
+                  np.array([[1, 2, 3]]) * np.arange(7)[:, np.newaxis],
+                  np.array(37.0) * np.arange(7))
+
+    count = array_ops.placeholder(dtypes.int64, shape=[])
+    window_size = array_ops.placeholder(dtypes.int64, shape=[])
+    stride = array_ops.placeholder(dtypes.int64, shape=[])
+
+    def _map_fn(x, y, z):
+      return math_ops.square(x), math_ops.square(y), math_ops.square(z)
+
+    # The pipeline is TensorSliceDataset -> MapDataset(square_3) ->
+    # RepeatDataset(count) -> _SlideDataset(window_size, stride).
+    iterator = (dataset_ops.Dataset.from_tensor_slices(components)
+                .map(_map_fn)
+                .repeat(count)
+                .apply(sliding.sliding_window_batch(window_size, stride))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    self.assertEqual([[None] + list(c.shape[1:]) for c in components],
+                     [t.shape.as_list() for t in get_next])
+
+    with self.test_session() as sess:
+      # Slide over a finite input, where the window_size divides the
+      # total number of elements.
+      sess.run(init_op, feed_dict={count: 20, window_size: 14, stride: 7})
+      # Same formula with convolution layer.
+      num_batches = (20 * 7 - 14) // 7 + 1
+      for i in range(num_batches):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(14):
+            self.assertAllEqual(component[(i*7 + j) % 7]**2,
+                                result_component[j])
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over a finite input, where the window_size does not
+      # divide the total number of elements.
+      sess.run(init_op, feed_dict={count: 20, window_size: 17, stride: 9})
+
+      num_batches = (20 * 7 - 17) // 9 + 1
+      for i in range(num_batches):
+        result = sess.run(get_next)
+        for component, result_component in zip(components, result):
+          for j in range(17):
+            self.assertAllEqual(component[(i*9 + j) % 7]**2,
+                                result_component[j])
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over a finite input, which is less than window_size,
+      # should fail straight away.
+      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 4})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      sess.run(init_op, feed_dict={count: 1, window_size: 10, stride: 8})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Slide over an empty input should fail straight away.
+      sess.run(init_op, feed_dict={count: 0, window_size: 8, stride: 4})
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+      # Empty window_size should be an initialization time error.
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 0, stride: 0})
+
+      # Invalid stride should be an initialization time error.
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 0})
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 3})
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(init_op, feed_dict={count: 14, window_size: 3, stride: 5})
+
+  def assertSparseValuesEqual(self, a, b):
+    self.assertAllEqual(a.indices, b.indices)
+    self.assertAllEqual(a.values, b.values)
+    self.assertAllEqual(a.dense_shape, b.dense_shape)
+
+  def testSlideSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
+        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      num_batches = (10 - 5) // 3 + 1
+      for i in range(num_batches):
+        actual = sess.run(get_next)
+        expected = sparse_tensor.SparseTensorValue(
+            indices=[[0, 0], [1, 0], [2, 0], [3, 0], [4, 0]],
+            values=[i * 3, i * 3 + 1, i * 3 + 2, i * 3 + 3, i * 3 + 4],
+            dense_shape=[5, 1])
+        self.assertTrue(sparse_tensor.is_sparse(actual))
+        self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testSlideSparseWithDifferentDenseShapes(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=array_ops.expand_dims(
+              math_ops.range(i, dtype=dtypes.int64), 1),
+          values=array_ops.fill([math_ops.to_int32(i)], i),
+          dense_shape=[i])
+
+    iterator = dataset_ops.Dataset.range(10).map(_sparse).apply(
+        sliding.sliding_window_batch(5, 3)).make_initializable_iterator()
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      num_batches = (10 - 5) // 3 + 1
+      for i in range(num_batches):
+        actual = sess.run(get_next)
+        expected_indices = []
+        expected_values = []
+        for j in range(5):
+          for k in range(i * 3 + j):
+            expected_indices.append([j, k])
+            expected_values.append(i * 3 + j)
+        expected = sparse_tensor.SparseTensorValue(
+            indices=expected_indices,
+            values=expected_values,
+            dense_shape=[5, i * 3 + 5 - 1])
+        self.assertTrue(sparse_tensor.is_sparse(actual))
+        self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testNestedSlideSparse(self):
+
+    def _sparse(i):
+      return sparse_tensor.SparseTensorValue(
+          indices=[[0]], values=(i * [1]), dense_shape=[1])
+
+    iterator = (dataset_ops.Dataset.range(10)
+                .map(_sparse)
+                .apply(sliding.sliding_window_batch(4, 2))
+                .apply(sliding.sliding_window_batch(3, 1))
+                .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      # Slide: 1st batch.
+      actual = sess.run(get_next)
+      expected = sparse_tensor.SparseTensorValue(
+          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
+                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
+                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
+          values=[0, 1, 2, 3, 2, 3, 4, 5, 4, 5, 6, 7],
+          dense_shape=[3, 4, 1])
+      self.assertTrue(sparse_tensor.is_sparse(actual))
+      self.assertSparseValuesEqual(actual, expected)
+      # Slide: 2nd batch.
+      actual = sess.run(get_next)
+      expected = sparse_tensor.SparseTensorValue(
+          indices=[[0, 0, 0], [0, 1, 0], [0, 2, 0], [0, 3, 0],
+                   [1, 0, 0], [1, 1, 0], [1, 2, 0], [1, 3, 0],
+                   [2, 0, 0], [2, 1, 0], [2, 2, 0], [2, 3, 0]],
+          values=[2, 3, 4, 5, 4, 5, 6, 7, 6, 7, 8, 9],
+          dense_shape=[3, 4, 1])
+      self.assertTrue(sparse_tensor.is_sparse(actual))
+      self.assertSparseValuesEqual(actual, expected)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def testSlideShapeError(self):
+
+    def generator():
+      yield [1.0, 2.0, 3.0]
+      yield [4.0, 5.0, 6.0]
+      yield [7.0, 8.0, 9.0, 10.0]
+
+    iterator = (dataset_ops.Dataset.from_generator(generator, dtypes.float32,
+                                                   output_shapes=[None])
+                .apply(sliding.sliding_window_batch(3, 1))
+                .make_initializable_iterator())
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(iterator.initializer)
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          r"Cannot batch tensors with different shapes in component 0. "
+          r"First element had shape \[3\] and element 2 had shape \[4\]."):
+        sess.run(next_element)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index f03430c5c5..c3331e9636 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -106,6 +106,7 @@ py_library(
         "interleave_ops.py",
         "resampling.py",
         "scan_ops.py",
+        "sliding.py",
         "stats_ops.py",
         "threadpool.py",
         "unique.py",
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index 56f526a330..f4015f19fb 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -54,7 +54,7 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
   def _apply_fn(dataset):
     """Function from `Dataset` to `Dataset` that applies the transformation."""
     dist_estimation_batch_size = 32
-    target_dist_t = ops.convert_to_tensor(target_dist, name="initial_dist")
+    target_dist_t = ops.convert_to_tensor(target_dist, name="target_dist")
     class_values_ds = dataset.map(class_func)
     if initial_dist is not None:
       initial_dist_t = ops.convert_to_tensor(initial_dist, name="initial_dist")
@@ -151,7 +151,7 @@ def _calculate_acceptance_probs(initial_probs, target_probs):
   ```
 
 
-  A solution for a_i in terms of the other variabes is the following:
+  A solution for a_i in terms of the other variables is the following:
     ```a_i = (t_i / p_i) / max_i[t_i / p_i]```
   """
   # Add tiny to initial_probs to avoid divide by zero.
diff --git a/tensorflow/contrib/data/python/ops/sliding.py b/tensorflow/contrib/data/python/ops/sliding.py
new file mode 100644
index 0000000000..19cc3cb89f
--- /dev/null
+++ b/tensorflow/contrib/data/python/ops/sliding.py
@@ -0,0 +1,102 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Sliding dataset transformations."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import sparse
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.ops import gen_dataset_ops
+
+
+class _SlideDataset(dataset_ops.Dataset):
+  """A `Dataset` that passes a sliding window over its input."""
+
+  def __init__(self, input_dataset, window_size, stride=1):
+    """See `sliding_window_batch` for details."""
+    super(_SlideDataset, self).__init__()
+    self._input_dataset = input_dataset
+    self._window_size = ops.convert_to_tensor(
+        window_size, dtype=dtypes.int64, name="window_size")
+    self._stride = ops.convert_to_tensor(
+        stride, dtype=dtypes.int64, name="stride")
+
+  def _as_variant_tensor(self):
+    return gen_dataset_ops.slide_dataset(
+        self._input_dataset._as_variant_tensor(),  # pylint: disable=protected-access
+        window_size=self._window_size,
+        stride=self._stride,
+        output_shapes=nest.flatten(
+            sparse.as_dense_shapes(self.output_shapes, self.output_classes)),
+        output_types=nest.flatten(
+            sparse.as_dense_types(self.output_types, self.output_classes)))
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    input_shapes = self._input_dataset.output_shapes
+    return nest.pack_sequence_as(input_shapes, [
+        tensor_shape.vector(None).concatenate(s)
+        for s in nest.flatten(self._input_dataset.output_shapes)
+    ])
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+
+
+def sliding_window_batch(window_size, stride=1):
+  """A sliding window with size of `window_size` and step of `stride`.
+
+  This transformation passes a sliding window over this dataset. The
+  window size is `window_size` and step size is `stride`. If the left
+  elements cannot fill up the sliding window, this transformation will
+  drop the final smaller element. For example:
+
+  ```python
+  # NOTE: The following examples use `{ ... }` to represent the
+  # contents of a dataset.
+  a = { [1], [2], [3], [4], [5], [6] }
+
+  a.apply(tf.contrib.data.sliding_window_batch(window_size=3, stride=2)) ==
+  {
+      [[1], [2], [3]],
+      [[3], [4], [5]],
+  }
+  ```
+
+  Args:
+    window_size: A `tf.int64` scalar `tf.Tensor`, representing the number of
+      elements in the sliding window.
+    stride: (Optional.) A `tf.int64` scalar `tf.Tensor`, representing the
+      steps moving the sliding window forward for one iteration. The default
+      is `1`. It must be in `[1, window_size)`.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return _SlideDataset(dataset, window_size, stride)
+
+  return _apply_fn
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index 90f10f1fa8..ad8568ad44 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -224,7 +224,10 @@ py_test(
     srcs = ["python/ops/kmeans_test.py"],
     shard_count = 4,
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67512932
+    tags = [
+        "nomac",  # b/73741358
+        "notsan",  # b/67512932
+    ],
     deps = [
         ":factorization_py",
         ":factorization_py_CYCLIC_DEPENDENCIES_THAT_NEED_TO_GO",
diff --git a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
index e61221a6b0..35341406a0 100644
--- a/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
+++ b/tensorflow/contrib/ffmpeg/default/ffmpeg_lib.cc
@@ -256,6 +256,9 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
         if (p != std::string::npos) {
           string rgb24 = line.substr(p + 9, line.find(" ", p + 9));
           rgb24 = rgb24.substr(0, rgb24.find(","));
+          // Strip anything after " ", in case the format is
+          // `640x360 [SAR 1:1 DAR 16:9]`
+          rgb24 = rgb24.substr(0, rgb24.find(" "));
           string rgb24_width = rgb24.substr(0, rgb24.find("x"));
           string rgb24_height = rgb24.substr(rgb24_width.length() + 1);
           if (strings::safe_strtou32(rgb24_width, &width_value) &&
@@ -270,8 +273,10 @@ Status ReadInfoFile(const string& filename, uint32* width, uint32* height,
       // We only look for the first stream mapping to have the number of the
       // frames.
       // Once processed we will not further process stream mapping section.
-      if (line.find("frame=  ") == 0) {
-        string number = line.substr(8, line.find(" ", 8));
+      if (line.find("frame=") == 0) {
+        // The format might be `frame=  166 ` or `frame=12488 `
+        string number = line.substr(6);
+        number = number.substr(number.find_first_not_of(" "));
         number = number.substr(0, number.find(" "));
         if (strings::safe_strtou32(number, &frames_value)) {
           in_mapping = false;
diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py
index 409657fe1d..3cad1fee19 100644
--- a/tensorflow/contrib/framework/python/ops/arg_scope.py
+++ b/tensorflow/contrib/framework/python/ops/arg_scope.py
@@ -142,7 +142,7 @@ def arg_scope(list_ops_or_scope, **kwargs):
   else:
     # Assumes that list_ops_or_scope is a list/tuple of ops with kwargs.
     if not isinstance(list_ops_or_scope, (list, tuple)):
-      raise TypeError('list_ops_or_scope must either be a list/tuple or reused'
+      raise TypeError('list_ops_or_scope must either be a list/tuple or reused '
                       'scope (i.e. dict)')
     try:
       current_scope = current_arg_scope().copy()
diff --git a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
index 7e86d10b64..47e51415fd 100644
--- a/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/classifier_metrics_impl.py
@@ -321,7 +321,7 @@ def classifier_score(images, classifier_fn, num_batches=1):
 
   NOTE: This function consumes images, computes their logits, and then
   computes the classifier score. If you would like to precompute many logits for
-  large batches, use clasifier_score_from_logits(), which this method also
+  large batches, use classifier_score_from_logits(), which this method also
   uses.
 
   Args:
@@ -454,7 +454,7 @@ def frechet_classifier_distance(real_images,
 
   This technique is described in detail in https://arxiv.org/abs/1706.08500.
   Given two Gaussian distribution with means m and m_w and covariance matrices
-  C and C_w, this function calcuates
+  C and C_w, this function calculates
 
               |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
 
@@ -467,7 +467,7 @@ def frechet_classifier_distance(real_images,
   Frechet distance is biased. It is more biased for small sample sizes. (e.g.
   even if the two distributions are the same, for a small sample size, the
   expected Frechet distance is large). It is important to use the same
-  sample size to compute frechet classifier distance when comparing two
+  sample size to compute Frechet classifier distance when comparing two
   generative models.
 
   NOTE: This function consumes images, computes their activations, and then
@@ -659,7 +659,7 @@ def frechet_classifier_distance_from_activations(real_activations,
 
   This technique is described in detail in https://arxiv.org/abs/1706.08500.
   Given two Gaussian distribution with means m and m_w and covariance matrices
-  C and C_w, this function calcuates
+  C and C_w, this function calculates
 
                 |m - m_w|^2 + Tr(C + C_w - 2(C * C_w)^(1/2))
 
diff --git a/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein_impl.py b/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein_impl.py
index 9bebcacbe4..4b10bc0f8e 100644
--- a/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/sliced_wasserstein_impl.py
@@ -212,7 +212,7 @@ def sliced_wasserstein_distance(real_images,
   Args:
       real_images: (tensor) Real images (batch, height, width, channels).
       fake_images: (tensor) Fake images (batch, height, width, channels).
-      resolution_min: (int) Minimum resolution for the Laplacion pyramid.
+      resolution_min: (int) Minimum resolution for the Laplacian pyramid.
       patches_per_image: (int) Number of patches to extract per image per
         Laplacian level.
       patch_size: (int) Width of a square patch.
@@ -221,7 +221,7 @@ def sliced_wasserstein_distance(real_images,
       use_svd: experimental method to compute a more accurate distance.
   Returns:
       List of tuples (distance_real, distance_fake) for each level of the
-      Laplacian pyramid from the highest resoluion to the lowest.
+      Laplacian pyramid from the highest resolution to the lowest.
         distance_real is the Wasserstein distance between real images
         distance_fake is the Wasserstein distance between real and fake images.
   Raises:
diff --git a/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py b/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
index cd31c62667..e2594faf85 100644
--- a/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
+++ b/tensorflow/contrib/gan/python/features/python/conditioning_utils_impl.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Miscellanous utilities for TFGAN code and examples.
+"""Miscellaneous utilities for TFGAN code and examples.
 
 Includes:
 1) Conditioning the value of a Tensor, based on techniques from
diff --git a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
index 4cfae0de44..9e4ec59e70 100644
--- a/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
+++ b/tensorflow/contrib/gan/python/features/python/random_tensor_pool_impl.py
@@ -17,7 +17,7 @@
 We use this to keep a history of values created by a generator, such that
 a discriminator can randomly be trained on some older samples, not just the
 current one. This can help to not let the discriminator get too far ahead of the
-generator and also to keep the system from oscilating, if the discriminator
+generator and also to keep the system from oscillating, if the discriminator
 forgets too fast what past samples from the generator looked like.
 
 See the following papers for more details.
@@ -97,7 +97,7 @@ def tensor_pool(input_values,
         dtypes=[v.dtype for v in input_values],
         shapes=None)
 
-    # In pseudeo code this code does the following:
+    # In pseudo code this code does the following:
     # if not pool_full:
     #   enqueue(input_values)
     #   return input_values
diff --git a/tensorflow/contrib/gan/python/features/python/virtual_batchnorm_test.py b/tensorflow/contrib/gan/python/features/python/virtual_batchnorm_test.py
index 845f89827b..2fe06a2872 100644
--- a/tensorflow/contrib/gan/python/features/python/virtual_batchnorm_test.py
+++ b/tensorflow/contrib/gan/python/features/python/virtual_batchnorm_test.py
@@ -148,7 +148,7 @@ class VirtualBatchnormTest(test.TestCase):
       self.assertAllClose(bn_np[i, ...], vb_np)
 
   def test_minibatch_independent(self):
-    """Test that virtual batch normalized exampels are independent.
+    """Test that virtual batch normalized examples are independent.
 
     Unlike batch normalization, virtual batch normalization has the property
     that the virtual batch normalized value of an example is independent of the
diff --git a/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py b/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py
index 252788140f..bcd2a34c4e 100644
--- a/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py
+++ b/tensorflow/contrib/grid_rnn/python/ops/grid_rnn_cell.py
@@ -110,7 +110,7 @@ class GridRNNCell(rnn.RNNCell):
       logging.warning('%s: Using a concatenated state is slower and will '
                       'soon be deprecated.  Use state_is_tuple=True.', self)
     if not output_is_tuple:
-      logging.warning('%s: Using a concatenated output is slower and will'
+      logging.warning('%s: Using a concatenated output is slower and will '
                       'soon be deprecated.  Use output_is_tuple=True.', self)
 
     if num_dims < 1:
diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 2924aef815..79eb3762ed 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -259,6 +259,7 @@ cuda_py_test(
         "//tensorflow/core:protos_all_py",
     ],
     data = [":sparse_image_warp_test_data"],
+    tags = ["no_pip"],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/kafka/BUILD b/tensorflow/contrib/kafka/BUILD
index efb403462a..1c3974871c 100644
--- a/tensorflow/contrib/kafka/BUILD
+++ b/tensorflow/contrib/kafka/BUILD
@@ -1,66 +1,93 @@
-package(
-    default_visibility = ["//visibility:private"],
-)
+package(default_visibility = ["//tensorflow:internal"])
 
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
-load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
-load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
-load("//tensorflow:tensorflow.bzl", "tf_py_test")
+load(
+    "//tensorflow:tensorflow.bzl",
+    "tf_gen_op_wrapper_py",
+    "tf_kernel_library",
+    "tf_custom_op_library",
+    "tf_custom_op_py_library",
+    "tf_gen_op_libs",
+    "tf_py_test",
+)
 
-tf_kernel_library(
-    name = "kafka_kernels",
+py_library(
+    name = "kafka",
+    srcs = ["__init__.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":dataset_ops",
+    ],
+)
+
+tf_custom_op_library(
+    name = "_dataset_ops.so",
+    srcs = ["ops/dataset_ops.cc"],
+    deps = [":dataset_kernels"],
+)
+
+tf_gen_op_libs(
+    op_lib_names = ["dataset_ops"],
+)
+
+cc_library(
+    name = "dataset_kernels",
     srcs = ["kernels/kafka_dataset_ops.cc"],
-    visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core/kernels:bounds_check_lib",
-        "//tensorflow/core/kernels:dataset",
+        "//tensorflow/core:framework_headers_lib",
         "//third_party/eigen3",
         "@kafka",
+        "@protobuf_archive//:protobuf_headers",
     ],
+    alwayslink = 1,
 )
 
-tf_gen_op_libs(
-    op_lib_names = ["kafka_ops"],
+py_library(
+    name = "dataset_ops",
+    srcs = [
+        "python/ops/kafka_dataset_ops.py",
+    ],
+    srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow/core:lib",
+        ":kafka_op_loader",
+        "//tensorflow/python:dataset_ops_gen",
+        "//tensorflow/python:util",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
     ],
 )
 
 tf_gen_op_wrapper_py(
-    name = "gen_kafka_ops",
-    out = "python/ops/gen_kafka_ops.py",
-    require_shape_functions = True,
-    deps = [":kafka_ops_op_lib"],
+    name = "gen_dataset_ops",
+    out = "python/ops/gen_dataset_ops.py",
+    deps = ["//tensorflow/contrib/kafka:dataset_ops_op_lib"],
 )
 
-py_library(
-    name = "kafka",
-    srcs = [
-        "__init__.py",
-        "python/ops/kafka_dataset_ops.py",
+tf_kernel_library(
+    name = "dataset_ops_kernels",
+    deps = [
+        ":dataset_kernels",
+        "//tensorflow/core:framework",
+    ],
+    alwayslink = 1,
+)
+
+tf_custom_op_py_library(
+    name = "kafka_op_loader",
+    srcs = ["python/ops/kafka_op_loader.py"],
+    dso = ["//tensorflow/contrib/kafka:_dataset_ops.so"],
+    kernels = [
+        ":dataset_ops_kernels",
+        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
     ],
     srcs_version = "PY2AND3",
-    visibility = ["//visibility:public"],
     deps = [
-        ":gen_kafka_ops",
+        ":gen_dataset_ops",
         "//tensorflow/contrib/util:util_py",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:control_flow_ops",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:platform",
-        "//tensorflow/python:state_ops",
-        "//tensorflow/python:training",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/data/ops:iterator_ops",
-        "//tensorflow/python/data/ops:readers",
     ],
 )
 
@@ -88,6 +115,7 @@ tf_py_test(
     ],
     tags = [
         "manual",
+        "no_windows",
         "notap",
     ],
 )
@@ -95,7 +123,9 @@ tf_py_test(
 filegroup(
     name = "all_files",
     srcs = glob(
-        ["**/*"],
+        include = [
+            "**/*",
+        ],
         exclude = [
             "**/METADATA",
             "**/OWNERS",
diff --git a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
index 88ef5f3571..a4cd4a2cc4 100644
--- a/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
+++ b/tensorflow/contrib/kafka/kernels/kafka_dataset_ops.cc
@@ -13,9 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/core/kernels/dataset.h"
-
-#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/dataset.h"
 
 #include "src-cpp/rdkafkacpp.h"
 
diff --git a/tensorflow/contrib/kafka/ops/dataset_ops.cc b/tensorflow/contrib/kafka/ops/dataset_ops.cc
new file mode 100644
index 0000000000..8cdf16103b
--- /dev/null
+++ b/tensorflow/contrib/kafka/ops/dataset_ops.cc
@@ -0,0 +1,44 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+REGISTER_OP("KafkaDataset")
+    .Input("topics: string")
+    .Input("servers: string")
+    .Input("group: string")
+    .Input("eof: bool")
+    .Input("timeout: int64")
+    .Output("handle: variant")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Creates a dataset that emits the messages of one or more Kafka topics.
+
+topics: A `tf.string` tensor containing one or more subscriptions,
+  in the format of [topic:partition:offset:length],
+  by default length is -1 for unlimited.
+servers: A list of bootstrap servers.
+group: The consumer group id.
+eof: If True, the kafka reader will stop on EOF.
+timeout: The timeout value for the Kafka Consumer to wait
+  (in millisecond).
+)doc");
+
+}  // namespace tensorflow
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
index 8e51d27a34..a1624614d1 100644
--- a/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
+++ b/tensorflow/contrib/kafka/python/ops/kafka_dataset_ops.py
@@ -17,8 +17,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.kafka.python.ops import gen_kafka_ops
-from tensorflow.python.data.ops.readers import Dataset
+from tensorflow.contrib.kafka.python.ops import kafka_op_loader  # pylint: disable=unused-import
+from tensorflow.contrib.kafka.python.ops import gen_dataset_ops
+from tensorflow.python.data.ops.dataset_ops import Dataset
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -58,8 +59,8 @@ class KafkaDataset(Dataset):
         timeout, dtype=dtypes.int64, name="timeout")
 
   def _as_variant_tensor(self):
-    return gen_kafka_ops.kafka_dataset(self._topics, self._servers, self._group,
-                                       self._eof, self._timeout)
+    return gen_dataset_ops.kafka_dataset(self._topics, self._servers,
+                                         self._group, self._eof, self._timeout)
 
   @property
   def output_classes(self):
diff --git a/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py b/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
new file mode 100644
index 0000000000..ec2fdea962
--- /dev/null
+++ b/tensorflow/contrib/kafka/python/ops/kafka_op_loader.py
@@ -0,0 +1,24 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Python helper for loading kafka ops and kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.util import loader
+from tensorflow.python.platform import resource_loader
+
+_dataset_ops = loader.load_op_library(
+    resource_loader.get_path_to_datafile("../../_dataset_ops.so"))
diff --git a/tensorflow/contrib/kfac/python/ops/optimizer.py b/tensorflow/contrib/kfac/python/ops/optimizer.py
index dee55cfa39..083da768ec 100644
--- a/tensorflow/contrib/kfac/python/ops/optimizer.py
+++ b/tensorflow/contrib/kfac/python/ops/optimizer.py
@@ -153,7 +153,7 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
       raise ValueError("Unsupported momentum type {}. Must be one of {}."
                        .format(momentum_type, legal_momentum_types))
     if momentum_type != "regular" and norm_constraint is not None:
-      raise ValueError("Update clipping is only supported with momentum"
+      raise ValueError("Update clipping is only supported with momentum "
                        "type 'regular'.")
     if momentum_type not in ["regular", "adam"] and momentum != 0:
       raise ValueError("Momentum must be unspecified if using a momentum_type "
diff --git a/tensorflow/contrib/layers/python/layers/embedding_ops.py b/tensorflow/contrib/layers/python/layers/embedding_ops.py
index b62e3050cd..ffa208540d 100644
--- a/tensorflow/contrib/layers/python/layers/embedding_ops.py
+++ b/tensorflow/contrib/layers/python/layers/embedding_ops.py
@@ -470,7 +470,7 @@ def embedding_lookup_unique(params, ids, name=None):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops.shape(ids)
     ids_flat = array_ops.reshape(
-        ids, math_ops.reduce_prod(shape, keep_dims=True))
+        ids, math_ops.reduce_prod(shape, keepdims=True))
     unique_ids, idx = array_ops.unique(ids_flat)
     unique_embeddings = embedding_ops.embedding_lookup(params, unique_ids)
     embeds_flat = array_ops.gather(unique_embeddings, idx)
diff --git a/tensorflow/contrib/layers/python/layers/encoders.py b/tensorflow/contrib/layers/python/layers/encoders.py
index 89c9d37bd0..f42112206d 100644
--- a/tensorflow/contrib/layers/python/layers/encoders.py
+++ b/tensorflow/contrib/layers/python/layers/encoders.py
@@ -125,7 +125,7 @@ def embed_sequence(ids,
       `reuse` is `None` or `False`.
   """
   if not (reuse or (vocab_size and embed_dim)):
-    raise ValueError('Must specify vocab size and embedding dimension when not'
+    raise ValueError('Must specify vocab size and embedding dimension when not '
                      'reusing. Got vocab_size=%s and embed_dim=%s' % (
                          vocab_size, embed_dim))
   with variable_scope.variable_scope(
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index f837ca3265..9c59150580 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -5,6 +5,8 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+load("//tensorflow:tensorflow.bzl", "py_test")
+
 package(default_visibility = [
     "//engedu/ml/tf_from_scratch:__pkg__",
     "//tensorflow:internal",
@@ -426,7 +428,10 @@ py_test(
     size = "medium",
     srcs = ["python/learn/estimators/kmeans_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["noasan"],
+    tags = [
+        "noasan",  # b/73741358
+        "nomac",
+    ],
     deps = [
         ":learn",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/learn/python/learn/estimators/estimator.py b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
index d8ccb1e7dc..7a026a15e4 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/estimator.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/estimator.py
@@ -917,8 +917,8 @@ class BaseEstimator(sklearn.BaseEstimator, evaluable.Evaluable,
       if feed_fn:
         hooks.append(basic_session_run_hooks.FeedFnHook(feed_fn))
       if steps == 0:
-        logging.warning('evaluation steps are 0. If `input_fn` does not raise'
-                        'OutOfRangeError`, the evaluation will never stop.'
+        logging.warning('evaluation steps are 0. If `input_fn` does not raise '
+                        '`OutOfRangeError`, the evaluation will never stop. '
                         'Use steps=None if intended.')
       if steps:
         hooks.append(
diff --git a/tensorflow/contrib/learn/python/learn/experiment.py b/tensorflow/contrib/learn/python/learn/experiment.py
index 9a7c4cd685..3744abd860 100644
--- a/tensorflow/contrib/learn/python/learn/experiment.py
+++ b/tensorflow/contrib/learn/python/learn/experiment.py
@@ -358,7 +358,7 @@ class Experiment(object):
         self._start_server()
     elif config.cluster_spec and config.master:
       raise ValueError(
-          "For distributed runtime, Experiment class only works with"
+          "For distributed runtime, Experiment class only works with "
           "tf.contrib.learn.RunConfig for now, but provided {}".format(
               type(config)))
 
diff --git a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
index b3b067b8e1..8f9811cf25 100644
--- a/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
+++ b/tensorflow/contrib/learn/python/learn/ops/embeddings_ops.py
@@ -61,7 +61,7 @@ def embedding_lookup(params, ids, name='embedding_lookup'):
     ids = ops.convert_to_tensor(ids)
     shape = array_ops_.shape(ids)
     ids_flat = array_ops_.reshape(
-        ids, math_ops.reduce_prod(shape, keep_dims=True))
+        ids, math_ops.reduce_prod(shape, keepdims=True))
     embeds_flat = nn.embedding_lookup(params, ids_flat, name)
     embed_shape = array_ops_.concat([shape, [-1]], 0)
     embeds = array_ops_.reshape(embeds_flat, embed_shape)
diff --git a/tensorflow/contrib/lite/Makefile b/tensorflow/contrib/lite/Makefile
index 7f31629272..b4504f246a 100644
--- a/tensorflow/contrib/lite/Makefile
+++ b/tensorflow/contrib/lite/Makefile
@@ -27,10 +27,10 @@ LIBDIR := $(MAKEFILE_DIR)/gen/lib/
 GENDIR := $(MAKEFILE_DIR)/gen/obj/
 
 # Settings for the host compiler.
-CXX := $(CC_PREFIX) gcc
+CXX := $(CC_PREFIX)gcc
 CXXFLAGS := --std=c++11 -O3 -DNDEBUG
-CC := $(CC_PREFIX) gcc
-CFLAGS :=
+CC := $(CC_PREFIX)gcc
+CFLAGS := -O3 -DNDEBUG
 LDOPTS :=
 LDOPTS += -L/usr/local/lib
 ARFLAGS := -r
@@ -57,10 +57,11 @@ LIBS := \
 
 # If we're on Linux, also link in the dl library.
 ifeq ($(HOST_OS),LINUX)
-	LIBS += -ldl -lpthread
+	LIBS += -ldl
 endif
 
 include $(MAKEFILE_DIR)/ios_makefile.inc
+include $(MAKEFILE_DIR)/rpi_makefile.inc
 
 # This library is the main target for this makefile. It will contain a minimal
 # runtime that can be linked in to other programs.
diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index df8c1c623c..2680d515eb 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -99,7 +99,7 @@ Similar to the Android demo app, there's an iOS camera app that uses exactly the
 
 This demo app requires a camera so it doesn't work with simulators. It need to be executed on a real iOS device. Follow the instructions to build and run the demo app:
 
-1.   Run `third_party/tensorflow/contrib/lite/examples/ios/download_models.sh` to download the model files used by the demo app.
+1.   Run `tensorflow/contrib/lite/examples/ios/download_models.sh` to download the model files used by the demo app.
 1.   Install [CocoaPods](https://cocoapods.org/) if it wasn't installed yet: `sudo gem install cocoapods`.
 1.   Run `pod install` in `tensorflow/contrib/lite/examples/ios/camera` to generate the workspace file.
 1.   Open the project by running `open tflite_camera_example.xcworkspace`, and build the app in XCode.
@@ -165,7 +165,7 @@ bazel-bin/tensorflow/python/tools/freeze_graph\
     --input_graph=/tmp/mobilenet_v1_224.pb \
     --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
     --input_binary=true --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
-    --output_node_names=MobileNet/Predictions/Reshape_1
+    --output_node_names=MobilenetV1/Predictions/Reshape_1
 ```
 
 The user has to first build the freeze_graph script using bazel and then run the script.  The input_binary flag has to be enabled to ensure that the protobuf is read and written in binary format.  The user has to input the .pb and the .ckpt files to freeze the graph The output_node_names may not be obvious outside of the code that built the model. The easiest way to find them is to visualize the graph, either with
diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index 58bc164619..f84b3dad95 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -33,7 +33,7 @@ class AllocationInfo;
 // each tensor needs to be allocated and deallocated, and preallocates all the
 // necessary memory (the PlanAllocations phase). It then assigns portions of
 // this memory buffer to each tensor (the ExecuteAllocations phase). Tensors may
-// share some of the bufer if a tensor B is to be allocated after another tensor
+// share some of the buffer if a tensor B is to be allocated after another tensor
 // A has been deallocated.
 //
 // If dynamic tensors are used the planning steps can be repeated during model
diff --git a/tensorflow/contrib/lite/build_rpi_lib.sh b/tensorflow/contrib/lite/build_rpi_lib.sh
new file mode 100755
index 0000000000..3824b16412
--- /dev/null
+++ b/tensorflow/contrib/lite/build_rpi_lib.sh
@@ -0,0 +1,22 @@
+#!/bin/bash -x
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+set -e
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+cd "$SCRIPT_DIR/../../.."
+
+CC_PREFIX=arm-linux-gnueabihf- make -j 3 -f tensorflow/contrib/lite/Makefile TARGET=RPI TARGET_ARCH=armv7
diff --git a/tensorflow/contrib/lite/error_reporter.h b/tensorflow/contrib/lite/error_reporter.h
index da193d2586..3c5f805f12 100644
--- a/tensorflow/contrib/lite/error_reporter.h
+++ b/tensorflow/contrib/lite/error_reporter.h
@@ -30,7 +30,7 @@ namespace tflite {
 //  va_list args;
 //  foo.Report("test %d", args); // where args is va_list
 //
-// Sublclass ErrorReporter to provide another reporting destination.
+// Subclass ErrorReporter to provide another reporting destination.
 // For example, if you have a GUI program, you might redirect to a buffer
 // that drives a GUI error log box.
 class ErrorReporter {
diff --git a/tensorflow/contrib/lite/g3doc/ios.md b/tensorflow/contrib/lite/g3doc/ios.md
index a359b8d4b4..e0358a444d 100644
--- a/tensorflow/contrib/lite/g3doc/ios.md
+++ b/tensorflow/contrib/lite/g3doc/ios.md
@@ -22,6 +22,15 @@ Then install
 brew install automake
 brew install libtool
 ```
+If you get an error where either automake or libtool install but do not link correctly, you'll first need to:
+```bash
+sudo chown -R $(whoami) /usr/local/*
+```
+Then follow the instructions to perform the linking:
+```bash
+brew link automake
+brew link libtool
+```
 
 Then you need to run a shell script to download the dependencies you need:
 
diff --git a/tensorflow/contrib/lite/g3doc/rpi.md b/tensorflow/contrib/lite/g3doc/rpi.md
new file mode 100644
index 0000000000..7a3a231626
--- /dev/null
+++ b/tensorflow/contrib/lite/g3doc/rpi.md
@@ -0,0 +1,50 @@
+# TensorFlow Lite for Raspberry Pi
+
+## Cross compiling
+### Installing toolchian
+This has been tested on Ubuntu 16.04.3 64bit and Tensorflow devel docker image [tensorflow/tensorflow:nightly-devel](https://hub.docker.com/r/tensorflow/tensorflow/tags/).
+
+To cross compiling TensorFlow Lite. First you should install the toolchain and libs.
+```bash
+sudo apt-get update
+sudo apt-get install crossbuild-essential-armhf
+```
+> If you are using docker, you may not use `sudo`
+
+### Building
+Clone this Tensorflow repository, Run this script at the root of the repository to download all the dependencies:
+> The Tensorflow repository is in `/tensorflow` if you are using `tensorflow/tensorflow:nightly-devel` docker image, just try it.
+```bash
+./tensorflow/contrib/lite/download_dependencies.sh
+```
+Note than you only need to to this once.
+
+You should then be able to compile:
+```bash
+./tensorflow/contrib/lite/build_rpi_lib.sh
+```
+
+This should compile a static library in:
+`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
+
+## Native compiling
+This has been tested on Raspberry Pi 3b, Raspbian GNU/Linux 9.1 (stretch), gcc version 6.3.0 20170516 (Raspbian 6.3.0-18+rpi1).
+
+Log in to you RPI, install the toolchain.
+```bash
+sudo apt-get instal build-essential
+```
+
+First, clone this TensorFlow repository. Run this at the root of the repository:
+```bash
+./tensorflow/contrib/lite/download_dependencies.sh
+```
+Note than you only need to to this once.
+
+You should then be able to compile:
+```bash
+./tensorflow/contrib/lite/build_rpi_lib.sh
+```
+
+This should compile a static library in:
+`tensorflow/contrib/lite/gen/lib/rpi_armv7/libtensorflow-lite.a`.
diff --git a/tensorflow/contrib/lite/interpreter.h b/tensorflow/contrib/lite/interpreter.h
index 788546fd60..77db178783 100644
--- a/tensorflow/contrib/lite/interpreter.h
+++ b/tensorflow/contrib/lite/interpreter.h
@@ -493,7 +493,7 @@ class Interpreter {
   // During Invoke(), Interpreter will allocate input tensors first, which are
   // known to be fixed size. Then it will allocate outputs from nodes as many
   // as possible. When there is a node that produces dynamic sized tensor.
-  // Intepreter will stop allocating tensors, set the value of next allocate
+  // Interpreter will stop allocating tensors, set the value of next allocate
   // node id, and execute the node to generate the output tensor before continue
   // to allocate successors. This process repeats until all nodes are executed.
   // NOTE: this relies on the order of nodes that is in topological order.
diff --git a/tensorflow/contrib/lite/interpreter_test.cc b/tensorflow/contrib/lite/interpreter_test.cc
index efb29d5c9d..131e088079 100644
--- a/tensorflow/contrib/lite/interpreter_test.cc
+++ b/tensorflow/contrib/lite/interpreter_test.cc
@@ -42,7 +42,7 @@ TEST(BasicInterpreter, InvokeInvalidModel) {
   ASSERT_EQ(interpreter.Invoke(), kTfLiteOk);
 }
 
-// Test size accesser functions.
+// Test size accessor functions.
 TEST(BasicInterpreter, TestSizeFunctions) {
   Interpreter interpreter;
   int base_index;
diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc
index b91ba1a03d..e0cd12f1b4 100644
--- a/tensorflow/contrib/lite/kernels/conv.cc
+++ b/tensorflow/contrib/lite/kernels/conv.cc
@@ -64,7 +64,7 @@ struct OpData {
 
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/depthwise_conv.cc b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
index 15dbfe08c8..cad9ce114c 100644
--- a/tensorflow/contrib/lite/kernels/depthwise_conv.cc
+++ b/tensorflow/contrib/lite/kernels/depthwise_conv.cc
@@ -52,7 +52,7 @@ enum KernelType {
 struct OpData {
   TfLitePaddingValues padding;
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/fully_connected.cc b/tensorflow/contrib/lite/kernels/fully_connected.cc
index a77fe94e49..888e67966c 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected.cc
@@ -48,7 +48,7 @@ enum KernelType {
 
 struct OpData {
   // The scaling factor from input to output (aka the 'real multiplier') can
-  // be represented as a fixed point multipler plus a left shift.
+  // be represented as a fixed point multiplier plus a left shift.
   int32_t output_multiplier;
   int output_shift;
   // The range of the fused activation layer. For example for kNone and
diff --git a/tensorflow/contrib/lite/kernels/internal/spectrogram.cc b/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
index 0e481a9d40..4eddf7bf0a 100644
--- a/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
+++ b/tensorflow/contrib/lite/kernels/internal/spectrogram.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/contrib/lite/kernels/internal/spectrogram.h"
 
+#include <assert.h>
 #include <math.h>
 
 #include "third_party/fft2d/fft.h"
diff --git a/tensorflow/contrib/lite/kernels/kernel_util.h b/tensorflow/contrib/lite/kernels/kernel_util.h
index 28f53b9fbb..21da1daff7 100644
--- a/tensorflow/contrib/lite/kernels/kernel_util.h
+++ b/tensorflow/contrib/lite/kernels/kernel_util.h
@@ -58,7 +58,7 @@ inline bool IsConstantTensor(TfLiteTensor* tensor) {
 }
 
 // Determines whether tensor is dynamic. Note that a tensor can be non-const and
-// not dynamic. This function specificially checks for a dynamic tensor.
+// not dynamic. This function specifically checks for a dynamic tensor.
 inline bool IsDynamicTensor(TfLiteTensor* tensor) {
   return tensor->allocation_type == kTfLiteDynamic;
 }
diff --git a/tensorflow/contrib/lite/kernels/lsh_projection.cc b/tensorflow/contrib/lite/kernels/lsh_projection.cc
index 5f73b56ed9..0ee35775d5 100644
--- a/tensorflow/contrib/lite/kernels/lsh_projection.cc
+++ b/tensorflow/contrib/lite/kernels/lsh_projection.cc
@@ -13,7 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-// LSH Projection projects an input to a bit vector via locality senstive
+// LSH Projection projects an input to a bit vector via locality sensitive
 // hashing.
 //
 // Options:
diff --git a/tensorflow/contrib/lite/kernels/lstm.cc b/tensorflow/contrib/lite/kernels/lstm.cc
index b9255b23a5..8cf1165135 100644
--- a/tensorflow/contrib/lite/kernels/lstm.cc
+++ b/tensorflow/contrib/lite/kernels/lstm.cc
@@ -213,9 +213,9 @@ TfLiteStatus CheckInputTensorDimensions(TfLiteContext* context,
   // present.
   // 2) If projection weight is present, then projection bias is optional.
   // TODO(ghodrat): make sure this is correct.
-  const bool projecton_tensors_consistent =
+  const bool projection_tensors_consistent =
       ((projection_weights != nullptr) || (projection_bias == nullptr));
-  TF_LITE_ENSURE(context, projecton_tensors_consistent == true);
+  TF_LITE_ENSURE(context, projection_tensors_consistent == true);
 
   return kTfLiteOk;
 }
@@ -357,7 +357,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
+  // check the existence of only one to get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/kernels/reshape.cc b/tensorflow/contrib/lite/kernels/reshape.cc
index f3e6ddc9f4..438f70d311 100644
--- a/tensorflow/contrib/lite/kernels/reshape.cc
+++ b/tensorflow/contrib/lite/kernels/reshape.cc
@@ -49,20 +49,20 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   TfLiteIntArray* output_size = TfLiteIntArrayCreate(params->num_dimensions);
   int num_output_elements = 1;
-  int strech_dim = -1;
+  int stretch_dim = -1;
   for (int i = 0; i < params->num_dimensions; ++i) {
     int value = params->shape[i];
     if (value == -1) {
-      TF_LITE_ENSURE_EQ(context, strech_dim, -1);
-      strech_dim = i;
+      TF_LITE_ENSURE_EQ(context, stretch_dim, -1);
+      stretch_dim = i;
     } else {
       num_output_elements *= value;
       output_size->data[i] = value;
     }
   }
-  if (strech_dim != -1) {
-    output_size->data[strech_dim] = num_input_elements / num_output_elements;
-    num_output_elements *= output_size->data[strech_dim];
+  if (stretch_dim != -1) {
+    output_size->data[stretch_dim] = num_input_elements / num_output_elements;
+    num_output_elements *= output_size->data[stretch_dim];
   }
 
   TF_LITE_ENSURE_EQ(context, num_input_elements, num_output_elements);
diff --git a/tensorflow/contrib/lite/kernels/reshape_test.cc b/tensorflow/contrib/lite/kernels/reshape_test.cc
index 0fbcf6e6aa..aecbd0399f 100644
--- a/tensorflow/contrib/lite/kernels/reshape_test.cc
+++ b/tensorflow/contrib/lite/kernels/reshape_test.cc
@@ -60,7 +60,7 @@ TEST(ReshapeOpTest, TooManyDimensions) {
 
 TEST(ReshapeOpTest, TooManySpecialDimensions) {
   EXPECT_DEATH(ReshapeOpModel({1, 2, 4, 1}, {-1, -1, 2, 4}),
-               "strech_dim != -1");
+               "stretch_dim != -1");
 }
 
 TEST(ReshapeOpTest, SimpleTest) {
diff --git a/tensorflow/contrib/lite/kernels/test_util.cc b/tensorflow/contrib/lite/kernels/test_util.cc
index 373310bd87..0bb28b50b2 100644
--- a/tensorflow/contrib/lite/kernels/test_util.cc
+++ b/tensorflow/contrib/lite/kernels/test_util.cc
@@ -141,8 +141,8 @@ void SingleOpModel::SetBuiltinOp(BuiltinOperator type,
 
 void SingleOpModel::SetCustomOp(
     const string& name, const std::vector<uint8_t>& custom_option,
-    const std::function<TfLiteRegistration*()>& registeration) {
-  custom_registrations_[name] = registeration;
+    const std::function<TfLiteRegistration*()>& registration) {
+  custom_registrations_[name] = registration;
   opcodes_.push_back(
       CreateOperatorCodeDirect(builder_, BuiltinOperator_CUSTOM, name.data()));
   operators_.push_back(CreateOperator(
diff --git a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
index 508a570e2e..42941a97db 100644
--- a/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
+++ b/tensorflow/contrib/lite/kernels/unidirectional_sequence_lstm.cc
@@ -360,7 +360,7 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   const int n_output = recurrent_to_output_weights->dims->data[1];
 
   // Since we have already checked that weights are all there or none, we can
-  // check the existense of only one to the get the condition.
+  // check the existence of only one to get the condition.
   const bool use_cifg = (input_to_input_weights == nullptr);
   const bool use_peephole = (cell_to_output_weights != nullptr);
 
diff --git a/tensorflow/contrib/lite/memory_planner.h b/tensorflow/contrib/lite/memory_planner.h
index 5cd6c20850..0294ec815c 100644
--- a/tensorflow/contrib/lite/memory_planner.h
+++ b/tensorflow/contrib/lite/memory_planner.h
@@ -34,8 +34,8 @@ class MemoryPlanner {
   // [first_node, last_node].
   virtual TfLiteStatus ExecuteAllocations(int first_node, int last_node) = 0;
 
-  // Invalidates allocations made earliers. This is called when tensors sizes
-  // have change. All planned allocations remain, but can't be used until
+  // Invalidates allocations made earlier. This is called when tensors sizes
+  // have changed. All planned allocations remain, but can't be used until
   // ExecuteAllocations() is called.
   virtual TfLiteStatus ResetAllocations() = 0;
 };
diff --git a/tensorflow/contrib/lite/model.h b/tensorflow/contrib/lite/model.h
index 0c777760cb..036dc46e03 100644
--- a/tensorflow/contrib/lite/model.h
+++ b/tensorflow/contrib/lite/model.h
@@ -81,7 +81,7 @@ class FlatBufferModel {
       const tflite::Model* model_spec,
       ErrorReporter* error_reporter = DefaultErrorReporter());
 
-  // Releases memory or unmaps mmaped meory.
+  // Releases memory or unmaps mmaped memory.
   ~FlatBufferModel();
 
   // Copying or assignment is disallowed to simplify ownership semantics.
diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
index 76032771af..bd49d327c9 100644
--- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
@@ -569,7 +569,7 @@ enum {
   ANEURALNETWORKS_LOGISTIC = 14,
 
   /**
-   * Projects an input to a bit vector via locality senstive hashing.
+   * Projects an input to a bit vector via locality sensitive hashing.
    *
    * Inputs:
    * * 0: Hash functions. Dim.size == 2, DataType: Float.
diff --git a/tensorflow/contrib/lite/rpi_makefile.inc b/tensorflow/contrib/lite/rpi_makefile.inc
new file mode 100644
index 0000000000..832ef5824b
--- /dev/null
+++ b/tensorflow/contrib/lite/rpi_makefile.inc
@@ -0,0 +1,33 @@
+# Settings for Raspberry Pi.
+ifeq ($(TARGET), RPI)
+	ifeq ($(TARGET_ARCH), armv7)
+		CXXFLAGS += \
+			-march=armv7-a \
+			-mfpu=neon-vfpv4 \
+			-funsafe-math-optimizations \
+			-ftree-vectorize
+
+		CCFLAGS += \
+			-march=armv7-a \
+			-mfpu=neon-vfpv4 \
+			-funsafe-math-optimizations \
+			-ftree-vectorize
+
+		LDFLAGS := \
+			-Wl,--no-export-dynamic \
+			-Wl,--exclude-libs,ALL \
+			-Wl,--gc-sections \
+			-Wl,--as-needed
+	endif
+
+	LIBS := \
+	-lstdc++ \
+	-lpthread \
+	-lm \
+	-ldl
+
+	OBJDIR := $(OBJDIR)rpi_$(TARGET_ARCH)/
+	LIBDIR := $(LIBDIR)rpi_$(TARGET_ARCH)/
+	BINDIR := $(BINDIR)rpi_$(TARGET_ARCH)/
+	DEPDIR := $(DEPDIR)rpi_$(TARGET_ARCH)/
+endif
diff --git a/tensorflow/contrib/lite/schema/upgrade_schema.py b/tensorflow/contrib/lite/schema/upgrade_schema.py
index 94f5730be5..e0b36d3d3e 100644
--- a/tensorflow/contrib/lite/schema/upgrade_schema.py
+++ b/tensorflow/contrib/lite/schema/upgrade_schema.py
@@ -39,8 +39,8 @@ import tensorflow as tf
 from tensorflow.python.platform import resource_loader
 
 parser = argparse.ArgumentParser(
-    description="Script to move TFLite models from pre-release schema to"
-    " new schema.")
+    description="Script to move TFLite models from pre-release schema to "
+    "new schema.")
 parser.add_argument(
     "input",
     type=str,
@@ -48,7 +48,7 @@ parser.add_argument(
 parser.add_argument(
     "output",
     type=str,
-    help="Output json or bin TensorFlow lite model compliant with"
+    help="Output json or bin TensorFlow lite model compliant with "
     "the new schema. Extension must be `.json`, `.bin` or `.tflite`.")
 
 
@@ -258,7 +258,7 @@ class Converter(object):
       # Check if builtin_code is the appropriate string type
       # use type("") instead of str or unicode. for py2and3
       if not isinstance(operator_code["builtin_code"], type(u"")):
-        raise ValueError("builtin_code %r is non-string. this usually means"
+        raise ValueError("builtin_code %r is non-string. this usually means "
                          "your model has consistency problems." %
                          (operator_code["builtin_code"]))
       operator_code["builtin_code"] = (RemapOperator(
diff --git a/tensorflow/contrib/lite/simple_memory_arena.cc b/tensorflow/contrib/lite/simple_memory_arena.cc
index 4aab244989..2f2004f56b 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.cc
+++ b/tensorflow/contrib/lite/simple_memory_arena.cc
@@ -113,21 +113,21 @@ TfLiteStatus SimpleMemoryArena::Commit(TfLiteContext* context) {
     underlying_buffer_size_ = required_size;
     underlying_buffer_aligned_ptr_ = new_underlying_buffer_aligned_ptr;
   }
-  commited_ = true;
+  committed_ = true;
   return underlying_buffer_ != nullptr ? kTfLiteOk : kTfLiteError;
 }
 
 TfLiteStatus SimpleMemoryArena::ResolveAlloc(TfLiteContext* context,
                                              const ArenaAlloc& alloc,
                                              char** output_ptr) {
-  TF_LITE_ENSURE(context, commited_);
+  TF_LITE_ENSURE(context, committed_);
   TF_LITE_ENSURE(context, output_ptr != nullptr);
   *output_ptr = underlying_buffer_aligned_ptr_ + alloc.offset;
   return kTfLiteOk;
 }
 
 TfLiteStatus SimpleMemoryArena::Clear() {
-  commited_ = false;
+  committed_ = false;
   high_water_mark_ = 0;
   allocs_.clear();
   return kTfLiteOk;
diff --git a/tensorflow/contrib/lite/simple_memory_arena.h b/tensorflow/contrib/lite/simple_memory_arena.h
index 0535522374..5faf78b59e 100644
--- a/tensorflow/contrib/lite/simple_memory_arena.h
+++ b/tensorflow/contrib/lite/simple_memory_arena.h
@@ -22,7 +22,7 @@ limitations under the License.
 namespace tflite {
 
 // This little structure holds the offset and the size for a dynamic memory
-// allocation in the memory arena. When the arena is commited and the
+// allocation in the memory arena. When the arena is committed and the
 // underlying buffer is set, the alloc can be resolved into an actual memory
 // pointer.
 struct ArenaAlloc {
@@ -43,7 +43,7 @@ struct ArenaAlloc {
 class SimpleMemoryArena {
  public:
   explicit SimpleMemoryArena(size_t arena_alignment)
-      : commited_(false),
+      : committed_(false),
         arena_alignment_(arena_alignment),
         high_water_mark_(0),
         underlying_buffer_size_(0),
@@ -73,7 +73,7 @@ class SimpleMemoryArena {
   }
 
  private:
-  bool commited_;
+  bool committed_;
   size_t arena_alignment_;
   size_t high_water_mark_;
   std::unique_ptr<char[]> underlying_buffer_;
diff --git a/tensorflow/contrib/makefile/README.md b/tensorflow/contrib/makefile/README.md
index 995230dfa8..6c3b02e12b 100644
--- a/tensorflow/contrib/makefile/README.md
+++ b/tensorflow/contrib/makefile/README.md
@@ -194,6 +194,8 @@ with:
 srcs = glob(["libs/arm64-v8a/*.so"]),
 ```
 
+If you are building for Android TV (Shield TV devices), replace "portrait" with "landscape" for android:screenOrientation in all four activities in tensorflow/examples/android/AndroidManifest.xml
+
 Then run:
 ```bash
 # Create dir for native libs
diff --git a/tensorflow/contrib/makefile/build_all_ios.sh b/tensorflow/contrib/makefile/build_all_ios.sh
index 2d99791839..0a458a27b3 100755
--- a/tensorflow/contrib/makefile/build_all_ios.sh
+++ b/tensorflow/contrib/makefile/build_all_ios.sh
@@ -80,10 +80,9 @@ if [[ ! -z "${OPTIMIZE_FOR_GRAPH}" ]]; then
         fi
     else
         echo "${PRNT_SLCTV_BIN} found. Using it"
-        ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
-
     fi
 
+    ${PRNT_SLCTV_BIN} --graphs=${OPTIMIZE_FOR_GRAPH} > ${TOP_SRCDIR}/tensorflow/core/framework/ops_to_register.h
 fi
 
 if [[ "${ONLY_MAKE_TENSORFLOW}" != "true" ]]; then
@@ -111,7 +110,7 @@ if [[ -z "${BUILD_ARCH}" ]]; then
     TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios`
 else
     # arch specified so build just that
-    TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios -a ${BUILD_ARCH}`
+    TARGET_NSYNC_LIB=`tensorflow/contrib/makefile/compile_nsync.sh -t ios -a "${BUILD_ARCH}"`
 fi
 export HOST_NSYNC_LIB TARGET_NSYNC_LIB
 
diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 0fee584f8e..81f05e7ce5 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -3647,7 +3647,7 @@ def cohen_kappa(labels,
     RuntimeError: If eager execution is enabled.
   """
   if context.executing_eagerly():
-    raise RuntimeError('tf.contrib.metrics.cohen_kappa is not supported'
+    raise RuntimeError('tf.contrib.metrics.cohen_kappa is not supported '
                        'when eager execution is enabled.')
   if num_classes < 2:
     raise ValueError('`num_classes` must be >= 2.'
diff --git a/tensorflow/contrib/model_pruning/python/layers/layers.py b/tensorflow/contrib/model_pruning/python/layers/layers.py
index 988748ad75..466daf204a 100644
--- a/tensorflow/contrib/model_pruning/python/layers/layers.py
+++ b/tensorflow/contrib/model_pruning/python/layers/layers.py
@@ -214,7 +214,7 @@ def masked_convolution(inputs,
     elif data_format == 'NCHW':
       df = 'channels_first'
     else:
-      raise ValueError('Unsupported data fromat', data_format)
+      raise ValueError('Unsupported data format', data_format)
 
     layer = layer_class(
         filters=num_outputs,
diff --git a/tensorflow/contrib/model_pruning/python/pruning.py b/tensorflow/contrib/model_pruning/python/pruning.py
index 86963be4b8..5146a4a2de 100644
--- a/tensorflow/contrib/model_pruning/python/pruning.py
+++ b/tensorflow/contrib/model_pruning/python/pruning.py
@@ -216,7 +216,7 @@ def _partitioned_variable_assign(partitioned_var, new_value):
   """Assign op for partitioned variables.
 
   Args:
-    partitioned_var: A partitioned tensotflow variable
+    partitioned_var: A partitioned tensorflow variable
     new_value: Value to be assigned to the variable var
 
   Returns:
diff --git a/tensorflow/contrib/mpi/mpi_utils.h b/tensorflow/contrib/mpi/mpi_utils.h
index fa297c28cb..df055ff567 100644
--- a/tensorflow/contrib/mpi/mpi_utils.h
+++ b/tensorflow/contrib/mpi/mpi_utils.h
@@ -24,6 +24,8 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/str_util.h"
 
+// Skip MPI C++ bindings support, this matches the usage in other places
+#define OMPI_SKIP_MPICXX
 #include "third_party/mpi/mpi.h"
 #define MPI_CHECK(cmd)                                                \
   do {                                                                \
diff --git a/tensorflow/contrib/predictor/predictor_factories.py b/tensorflow/contrib/predictor/predictor_factories.py
index 04b5d5bdf1..6e77e934fe 100644
--- a/tensorflow/contrib/predictor/predictor_factories.py
+++ b/tensorflow/contrib/predictor/predictor_factories.py
@@ -53,7 +53,7 @@ def from_contrib_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, core_estimator.Estimator):
-    raise TypeError('Espected estimator to be of type '
+    raise TypeError('Expected estimator to be of type '
                     'tf.contrib.learn.Estimator, but got type '
                     'tf.python.estimator.Estimator. You likely want to call '
                     'from_estimator.')
@@ -88,7 +88,7 @@ def from_estimator(estimator,
       `Estimator`.
   """
   if isinstance(estimator, contrib_estimator.Estimator):
-    raise TypeError('Espected estimator to be of type '
+    raise TypeError('Expected estimator to be of type '
                     'tf.python.estimator.Estimator, but got type '
                     'tf.contrib.learn.Estimator. You likely want to call '
                     'from_contrib_estimator.')
diff --git a/tensorflow/contrib/py2tf/converters/single_return.py b/tensorflow/contrib/py2tf/converters/single_return.py
index 90bc22008f..1194b98f5e 100644
--- a/tensorflow/contrib/py2tf/converters/single_return.py
+++ b/tensorflow/contrib/py2tf/converters/single_return.py
@@ -212,7 +212,7 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
 
   def __init__(self):
     self.cant_return = False
-    super(gast.NodeVisitor, self).__init__()
+    super(DetectReturnInUnsupportedControlFlow, self).__init__()
 
   def visit_While(self, node):
     self.cant_return = True
diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index e8a0d41425..5750be6f4c 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -237,7 +237,7 @@ def _FindFusedBatchNorms(graph):
       # The batch variance used during forward and backward prop is biased,
       # i.e it is calculated as: V=sum(x(k)-mu)^2/N. For the moving average
       # calculation, the variance is corrected by the term N/N-1 (Bessel's
-      # correction). The variance tensor read from FuseBatchNorm has bessel's
+      # correction). The variance tensor read from FuseBatchNorm has Bessel's
       # correction applied, so we undo it here.
       scope, sep, _ = bn_op.name.rpartition('/')
       g = ops.get_default_graph()
@@ -306,7 +306,7 @@ def _ComputeBatchNormCorrections(context, match, freeze_batch_norm_delay,
 
   Args:
     context: The scope under which we look for batch norm params
-    match: Object containg required batch norm tensors for correction
+    match: Object containing required batch norm tensors for correction
       computation.
     freeze_batch_norm_delay: Delay in steps at which computation switches
       from regular batch norm to frozen mean and variance.
diff --git a/tensorflow/contrib/quantize/python/quant_ops.py b/tensorflow/contrib/quantize/python/quant_ops.py
index 0a8e35080c..a4f7b1b221 100644
--- a/tensorflow/contrib/quantize/python/quant_ops.py
+++ b/tensorflow/contrib/quantize/python/quant_ops.py
@@ -282,8 +282,8 @@ def _FakeQuantWithMinMaxVars(inputs, min_var, max_var, per_channel, num_bits,
   Args:
     inputs: a tensor containing values to be quantized.
     min_var: a variable containing quantization range lower end(s).
-    max_var: a variable containing quantization range lupper end(s).
-    per_channel: a boolean specifying whether to use per-channel quantizatioh.
+    max_var: a variable containing quantization range upper end(s).
+    per_channel: a boolean specifying whether to use per-channel quantization.
     num_bits: Number of bits to use for quantization, must be between 2 and 8.
     narrow_range: Whether to use the narrow quantization range
       [1; 2^num_bits - 1] or wide range [0; 2^num_bits - 1].
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 2b5b877e8e..33f14e8d0e 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -341,7 +341,7 @@ def _InsertQuantOp(context,
   """Inserts a quant op between a producer op and (multiple) consumer ops.
 
   Args:
-    context: Context w,here producer and consumer operations are nested.
+    context: Context where producer and consumer operations are nested.
     name: Name for the new quantization op within the context.
     producer: Producer operation of the pairs where quantization will be
       inserted.
diff --git a/tensorflow/contrib/quantize/python/quantize_graph.py b/tensorflow/contrib/quantize/python/quantize_graph.py
index d0fb55da74..0b74b438ac 100644
--- a/tensorflow/contrib/quantize/python/quantize_graph.py
+++ b/tensorflow/contrib/quantize/python/quantize_graph.py
@@ -155,7 +155,7 @@ def experimental_create_training_graph(input_graph=None,
   often fail.
 
   Args:
-    input_graph: The tf.Graph to be transformed,if None then defaults to the
+    input_graph: The tf.Graph to be transformed, if None then defaults to the
       default graph.
     weight_bits: Number of bits to use for quantizing weights.
     activation_bits: Number of bits to use for quantizing activations.
diff --git a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
index 0624cc878b..db745aa562 100644
--- a/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_parameterized_test.py
@@ -419,7 +419,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -470,7 +470,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -526,7 +526,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
           normalizer_params=self._BatchNormParams(fused_batch_norm),
           scope=scope)
 
-      # Manually add a bypass (optionaly) and an activation.
+      # Manually add a bypass (optional) and an activation.
       if with_bypass:
         node = math_ops.add(inputs, node, name='test/Add')
 
@@ -565,7 +565,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initialzes with a truncated normal variable.
+      An initialized that initializes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index 216310abe4..bef58bad8d 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -197,7 +197,7 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       stddev: Standard deviation of normal variable.
 
     Returns:
-      An initialized that initialzes with a truncated normal variable.
+      An initialized that initializes with a truncated normal variable.
     """
     return init_ops.truncated_normal_initializer(stddev=stddev)
 
diff --git a/tensorflow/contrib/rnn/ops/gru_ops.cc b/tensorflow/contrib/rnn/ops/gru_ops.cc
index e91d1e8a80..9c8e40851a 100644
--- a/tensorflow/contrib/rnn/ops/gru_ops.cc
+++ b/tensorflow/contrib/rnn/ops/gru_ops.cc
@@ -69,7 +69,7 @@ Element-wise dot product of a and b is represented by ab
 Element-wise dot product is represented by \circ
 Matrix multiplication is represented by *
 
-Baises are initialized with :
+Biases are initialized with :
 `b_ru` - constant_initializer(1.0)
 `b_c` - constant_initializer(0.0)
 
diff --git a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
index 7957edf68c..ffd2421894 100644
--- a/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
+++ b/tensorflow/contrib/rnn/python/kernel_tests/lstm_ops_test.py
@@ -54,7 +54,7 @@ def blocks_match(sess, use_peephole):
   initializer = init_ops.random_uniform_initializer(-0.01, 0.01, seed=19890212)
 
   with variable_scope.variable_scope("test", initializer=initializer):
-    # magic naming so that the cells pick up these variables and resuse them
+    # magic naming so that the cells pick up these variables and reuse them
     if use_peephole:
       wci = variable_scope.get_variable(
           "rnn/lstm_cell/w_i_diag", shape=[cell_size], dtype=dtypes.float32)
diff --git a/tensorflow/contrib/rnn/python/ops/lstm_ops.py b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
index 4eb4fbcd92..9e61fc54d1 100644
--- a/tensorflow/contrib/rnn/python/ops/lstm_ops.py
+++ b/tensorflow/contrib/rnn/python/ops/lstm_ops.py
@@ -480,8 +480,7 @@ class LSTMBlockWrapper(base_layer.Layer):
     """Run this LSTM on inputs, starting from the given state.
 
     Args:
-      inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]`
-        or a list of `time_len` tensors of shape `[batch_size, input_size]`.
+      inputs: `3-D` tensor with shape `[time_len, batch_size, input_size]`.
       initial_state: a tuple `(initial_cell_state, initial_output)` with tensors
         of shape `[batch_size, self._num_units]`. If this is not provided, the
         cell is expected to create a zero initial state of type `dtype`.
diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 73f2607d84..2f6ae9f367 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -534,7 +534,7 @@ class GridLSTMCell(rnn_cell_impl.RNNCell):
       initializer: (optional) The initializer to use for the weight and
         projection matrices, default None.
       num_unit_shards: (optional) int, default 1, How to split the weight
-        matrix. If > 1,the weight matrix is stored across num_unit_shards.
+        matrix. If > 1, the weight matrix is stored across num_unit_shards.
       forget_bias: (optional) float, default 1.0, The initial bias of the
         forget gates, used to reduce the scale of forgetting at the beginning
         of the training.
@@ -993,7 +993,7 @@ class BidirectionalGridLSTMCell(GridLSTMCell):
       initializer: (optional) The initializer to use for the weight and
         projection matrices, default None.
       num_unit_shards: (optional) int, default 1, How to split the weight
-        matrix. If > 1,the weight matrix is stored across num_unit_shards.
+        matrix. If > 1, the weight matrix is stored across num_unit_shards.
       forget_bias: (optional) float, default 1.0, The initial bias of the
         forget gates, used to reduce the scale of forgetting at the beginning
         of the training.
@@ -2133,7 +2133,7 @@ class Conv1DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_1d_lstm_cell", **kwargs):
     """Construct Conv1DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, **kwargs)
+    super(Conv1DLSTMCell, self).__init__(conv_ndims=1, name=name, **kwargs)
 
 
 class Conv2DLSTMCell(ConvLSTMCell):
@@ -2144,7 +2144,7 @@ class Conv2DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_2d_lstm_cell", **kwargs):
     """Construct Conv2DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, **kwargs)
+    super(Conv2DLSTMCell, self).__init__(conv_ndims=2, name=name, **kwargs)
 
 
 class Conv3DLSTMCell(ConvLSTMCell):
@@ -2155,7 +2155,7 @@ class Conv3DLSTMCell(ConvLSTMCell):
 
   def __init__(self, name="conv_3d_lstm_cell", **kwargs):
     """Construct Conv3DLSTM. See `ConvLSTMCell` for more details."""
-    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, **kwargs)
+    super(Conv3DLSTMCell, self).__init__(conv_ndims=3, name=name, **kwargs)
 
 
 def _conv(args, filter_size, num_features, bias, bias_start=0.0):
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index b427dff88b..c4139dde49 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -222,6 +222,9 @@ class AttentionWrapperTest(test.TestCase):
           self.assertEqual(
               (None, batch_size, None),
               tuple(state_alignment_history.get_shape().as_list()))
+        nest.assert_same_structure(
+            cell.state_size,
+            cell.zero_state(batch_size, dtypes.float32))
         # Remove the history from final_state for purposes of the
         # remainder of the tests.
         final_state = final_state._replace(alignment_history=())  # pylint: disable=protected-access
diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
index 9265540317..178328619f 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/beam_search_decoder_test.py
@@ -27,6 +27,7 @@ from tensorflow.contrib.seq2seq.python.ops import beam_search_ops
 from tensorflow.contrib.seq2seq.python.ops import decoder
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.layers import core as layers_core
 from tensorflow.python.ops import array_ops
@@ -70,6 +71,98 @@ class TestGatherTree(test.TestCase):
 
     self.assertAllEqual(expected_result, res_)
 
+  def _test_gather_tree_from_array(self,
+                                   depth_ndims=0,
+                                   merged_batch_beam=False):
+    array = np.array(
+        [[[1, 2, 3], [4, 5, 6], [7, 8, 9], [0, 0, 0]],
+         [[2, 3, 4], [5, 6, 7], [8, 9, 10], [11, 12, 0]]]).transpose([1, 0, 2])
+    parent_ids = np.array(
+        [[[0, 0, 0], [0, 1, 1], [2, 1, 2], [-1, -1, -1]],
+         [[0, 0, 0], [1, 1, 0], [2, 0, 1], [0, 1, 0]]]).transpose([1, 0, 2])
+    expected_array = np.array(
+        [[[2, 2, 2], [6, 5, 6], [7, 8, 9], [0, 0, 0]],
+         [[2, 3, 2], [7, 5, 7], [8, 9, 8], [11, 12, 0]]]).transpose([1, 0, 2])
+    sequence_length = [[3, 3, 3], [4, 4, 3]]
+
+    array = ops.convert_to_tensor(
+        array, dtype=dtypes.float32)
+    parent_ids = ops.convert_to_tensor(
+        parent_ids, dtype=dtypes.int32)
+    expected_array = ops.convert_to_tensor(
+        expected_array, dtype=dtypes.float32)
+
+    max_time = array_ops.shape(array)[0]
+    batch_size = array_ops.shape(array)[1]
+    beam_width = array_ops.shape(array)[2]
+
+    def _tile_in_depth(tensor):
+      # Generate higher rank tensors by concatenating tensor and tensor + 1.
+      for _ in range(depth_ndims):
+        tensor = array_ops.stack([tensor, tensor + 1], -1)
+      return tensor
+
+    if merged_batch_beam:
+      array = array_ops.reshape(
+          array, [max_time, batch_size * beam_width])
+      expected_array = array_ops.reshape(
+          expected_array, [max_time, batch_size * beam_width])
+
+    if depth_ndims > 0:
+      array = _tile_in_depth(array)
+      expected_array = _tile_in_depth(expected_array)
+
+    sorted_array = beam_search_decoder.gather_tree_from_array(
+        array, parent_ids, sequence_length)
+
+    with self.test_session() as sess:
+      sorted_array = sess.run(sorted_array)
+      expected_array = sess.run(expected_array)
+      self.assertAllEqual(expected_array, sorted_array)
+
+  def test_gather_tree_from_array_scalar(self):
+    self._test_gather_tree_from_array()
+
+  def test_gather_tree_from_array_1d(self):
+    self._test_gather_tree_from_array(depth_ndims=1)
+
+  def test_gather_tree_from_array_1d_with_merged_batch_beam(self):
+    self._test_gather_tree_from_array(depth_ndims=1, merged_batch_beam=True)
+
+  def test_gather_tree_from_array_2d(self):
+    self._test_gather_tree_from_array(depth_ndims=2)
+
+
+class TestArrayShapeChecks(test.TestCase):
+
+  def _test_array_shape_dynamic_checks(self, static_shape, dynamic_shape,
+                                       batch_size, beam_width, is_valid=True):
+    t = array_ops.placeholder_with_default(
+        np.random.randn(*static_shape).astype(np.float32),
+        shape=dynamic_shape)
+
+    batch_size = array_ops.constant(batch_size)
+    check_op = beam_search_decoder._check_batch_beam(t, batch_size, beam_width)  # pylint: disable=protected-access
+
+    with self.test_session() as sess:
+      if is_valid:
+        sess.run(check_op)
+      else:
+        with self.assertRaises(errors.InvalidArgumentError):
+          sess.run(check_op)
+
+  def test_array_shape_dynamic_checks(self):
+    self._test_array_shape_dynamic_checks(
+        (8, 4, 5, 10), (None, None, 5, 10), 4, 5, is_valid=True)
+    self._test_array_shape_dynamic_checks(
+        (8, 20, 10), (None, None, 10), 4, 5, is_valid=True)
+    self._test_array_shape_dynamic_checks(
+        (8, 21, 10), (None, None, 10), 4, 5, is_valid=False)
+    self._test_array_shape_dynamic_checks(
+        (8, 4, 6, 10), (None, None, None, 10), 4, 5, is_valid=False)
+    self._test_array_shape_dynamic_checks(
+        (8, 4), (None, None), 4, 5, is_valid=False)
+
 
 class TestEosMasking(test.TestCase):
   """Tests EOS masking used in beam search."""
@@ -319,7 +412,8 @@ class TestLargeBeamStep(test.TestCase):
 
 class BeamSearchDecoderTest(test.TestCase):
 
-  def _testDynamicDecodeRNN(self, time_major, has_attention):
+  def _testDynamicDecodeRNN(self, time_major, has_attention,
+                            with_alignment_history=False):
     encoder_sequence_length = np.array([3, 2, 3, 1, 1])
     decoder_sequence_length = np.array([2, 0, 1, 2, 3])
     batch_size = 5
@@ -359,7 +453,7 @@ class BeamSearchDecoderTest(test.TestCase):
             cell=cell,
             attention_mechanism=attention_mechanism,
             attention_layer_size=attention_depth,
-            alignment_history=False)
+            alignment_history=with_alignment_history)
       cell_state = cell.zero_state(
           dtype=dtypes.float32, batch_size=batch_size_tensor * beam_width)
       if has_attention:
@@ -420,6 +514,12 @@ class BeamSearchDecoderTest(test.TestCase):
   def testDynamicDecodeRNNBatchMajorYesAttention(self):
     self._testDynamicDecodeRNN(time_major=False, has_attention=True)
 
+  def testDynamicDecodeRNNBatchMajorYesAttentionWithAlignmentHistory(self):
+    self._testDynamicDecodeRNN(
+        time_major=False,
+        has_attention=True,
+        with_alignment_history=True)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index f8da5a3e17..9ff8a343f1 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -1278,7 +1278,8 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
         attention_state=self._item_or_tuple(
             a.state_size for a in self._attention_mechanisms),
         alignment_history=self._item_or_tuple(
-            () for _ in self._attention_mechanisms))  # sometimes a TensorArray
+            a.alignments_size if self._alignment_history else ()
+            for a in self._attention_mechanisms))  # sometimes a TensorArray
 
   def zero_state(self, batch_size, dtype):
     """Return an initial (zero) state tuple for this `AttentionWrapper`.
@@ -1318,22 +1319,26 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
         cell_state = nest.map_structure(
             lambda s: array_ops.identity(s, name="checked_cell_state"),
             cell_state)
+      initial_alignments = [
+          attention_mechanism.initial_alignments(batch_size, dtype)
+          for attention_mechanism in self._attention_mechanisms]
       return AttentionWrapperState(
           cell_state=cell_state,
           time=array_ops.zeros([], dtype=dtypes.int32),
           attention=_zero_state_tensors(self._attention_layer_size, batch_size,
                                         dtype),
-          alignments=self._item_or_tuple(
-              attention_mechanism.initial_alignments(batch_size, dtype)
-              for attention_mechanism in self._attention_mechanisms),
+          alignments=self._item_or_tuple(initial_alignments),
           attention_state=self._item_or_tuple(
               attention_mechanism.initial_state(batch_size, dtype)
               for attention_mechanism in self._attention_mechanisms),
           alignment_history=self._item_or_tuple(
-              tensor_array_ops.TensorArray(dtype=dtype, size=0,
-                                           dynamic_size=True)
+              tensor_array_ops.TensorArray(
+                  dtype,
+                  size=0,
+                  dynamic_size=True,
+                  element_shape=alignment.shape)
               if self._alignment_history else ()
-              for _ in self._attention_mechanisms))
+              for alignment in initial_alignments))
 
   def call(self, inputs, state):
     """Perform a step of attention-wrapped RNN.
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 03fe31abf7..a26107b0d7 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -35,6 +35,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import tensor_array_ops
+from tensorflow.python.platform import tf_logging
 from tensorflow.python.util import nest
 
 __all__ = [
@@ -121,14 +122,114 @@ def tile_batch(t, multiplier, name=None):
     return nest.map_structure(lambda t_: _tile_batch(t_, multiplier), t)
 
 
+def gather_tree_from_array(t, parent_ids, sequence_length):
+  """Calculates the full beams for `TensorArray`s.
+
+  Args:
+    t: A stacked `TensorArray` of size `max_time` that contains `Tensor`s of
+      shape `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]`
+      where `s` is the depth shape.
+    parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`.
+    sequence_length: The sequence length of shape `[batch_size, beam_width]`.
+
+  Returns:
+    A `Tensor` which is a stacked `TensorArray` of the same size and type as
+    `t` and where beams are sorted in each `Tensor` according to `parent_ids`.
+  """
+  max_time = parent_ids.shape[0].value or array_ops.shape(parent_ids)[0]
+  batch_size = parent_ids.shape[1].value or array_ops.shape(parent_ids)[1]
+  beam_width = parent_ids.shape[2].value or array_ops.shape(parent_ids)[2]
+
+  # Generate beam ids that will be reordered by gather_tree.
+  beam_ids = array_ops.expand_dims(
+      array_ops.expand_dims(math_ops.range(beam_width), 0), 0)
+  beam_ids = array_ops.tile(beam_ids, [max_time, batch_size, 1])
+
+  mask = array_ops.sequence_mask(
+      sequence_length, maxlen=max_time, dtype=dtypes.int32)
+  mask = array_ops.transpose(mask, perm=[2, 0, 1])
+
+  # Use beam_width + 1 to mark the end of beam.
+  masked_beam_ids = (beam_ids * mask) + (1 - mask) * (beam_width + 1)
+
+  max_sequence_lengths = math_ops.to_int32(
+      math_ops.reduce_max(sequence_length, axis=1))
+  sorted_beam_ids = beam_search_ops.gather_tree(
+      step_ids=masked_beam_ids,
+      parent_ids=parent_ids,
+      max_sequence_lengths=max_sequence_lengths,
+      end_token=beam_width + 1)
+
+  # For out of range steps, simply copy the same beam.
+  sorted_beam_ids = array_ops.where(
+      math_ops.cast(mask, dtypes.bool), x=sorted_beam_ids, y=beam_ids)
+
+  # Generate indices for gather_nd.
+  time_ind = array_ops.tile(array_ops.reshape(
+      math_ops.range(max_time), [-1, 1, 1]), [1, batch_size, beam_width])
+  batch_ind = array_ops.tile(array_ops.reshape(
+      math_ops.range(batch_size), [-1, 1, 1]), [1, max_time, beam_width])
+  batch_ind = array_ops.transpose(batch_ind, perm=[1, 0, 2])
+  indices = array_ops.stack([time_ind, batch_ind, sorted_beam_ids], -1)
+
+  # Gather from a tensor with collapsed additional dimensions.
+  gather_from = t
+  final_shape = array_ops.shape(gather_from)
+  gather_from = array_ops.reshape(
+      gather_from, [max_time, batch_size, beam_width, -1])
+  ordered = array_ops.gather_nd(gather_from, indices)
+  ordered = array_ops.reshape(ordered, final_shape)
+
+  return ordered
+
+
 def _check_maybe(t):
-  if isinstance(t, tensor_array_ops.TensorArray):
-    raise TypeError(
-        "TensorArray state is not supported by BeamSearchDecoder: %s" % t.name)
   if t.shape.ndims is None:
     raise ValueError(
         "Expected tensor (%s) to have known rank, but ndims == None." % t)
 
+def _check_static_batch_beam_maybe(shape, batch_size, beam_width):
+  """Raises an exception if dimensions are known statically and can not be
+  reshaped to [batch_size, beam_size, -1].
+  """
+  reshaped_shape = tensor_shape.TensorShape([batch_size, beam_width, None])
+  if (batch_size is not None and shape[0].value is not None
+      and (shape[0] != batch_size * beam_width
+           or (shape.ndims >= 2 and shape[1].value is not None
+               and (shape[0] != batch_size or shape[1] != beam_width)))):
+    tf_logging.warn("TensorArray reordering expects elements to be "
+                    "reshapable to %s which is incompatible with the "
+                    "current shape %s. Consider setting "
+                    "reorder_tensor_arrays to False to disable TensorArray "
+                    "reordering during the beam search."
+                    % (reshaped_shape, shape))
+    return False
+  return True
+
+def _check_batch_beam(t, batch_size, beam_width):
+  """Returns an Assert operation checking that the elements of the stacked
+  TensorArray can be reshaped to [batch_size, beam_size, -1]. At this point,
+  the TensorArray elements have a known rank of at least 1.
+  """
+  error_message = ("TensorArray reordering expects elements to be "
+                   "reshapable to [batch_size, beam_size, -1] which is "
+                   "incompatible with the dynamic shape of %s elements. "
+                   "Consider setting reorder_tensor_arrays to False to disable "
+                   "TensorArray reordering during the beam search."
+                   % (t.name))
+  rank = t.shape.ndims
+  shape = array_ops.shape(t)
+  if rank == 2:
+    condition = math_ops.equal(shape[1], batch_size * beam_width)
+  else:
+    condition = math_ops.logical_or(
+        math_ops.equal(shape[1], batch_size * beam_width),
+        math_ops.logical_and(
+            math_ops.equal(shape[1], batch_size),
+            math_ops.equal(shape[2], beam_width)))
+  return control_flow_ops.Assert(condition, [error_message])
+
+
 
 class BeamSearchDecoder(decoder.Decoder):
   """BeamSearch sampling decoder.
@@ -173,7 +274,8 @@ class BeamSearchDecoder(decoder.Decoder):
                initial_state,
                beam_width,
                output_layer=None,
-               length_penalty_weight=0.0):
+               length_penalty_weight=0.0,
+               reorder_tensor_arrays=True):
     """Initialize the BeamSearchDecoder.
 
     Args:
@@ -188,6 +290,12 @@ class BeamSearchDecoder(decoder.Decoder):
         `tf.layers.Dense`.  Optional layer to apply to the RNN output prior
         to storing the result or sampling.
       length_penalty_weight: Float weight to penalize length. Disabled with 0.0.
+      reorder_tensor_arrays: If `True`, `TensorArray`s' elements within the cell
+        state will be reordered according to the beam search path. If the
+        `TensorArray` can be reordered, the stacked form will be returned.
+        Otherwise, the `TensorArray` will be returned as is. Set this flag to
+        `False` if the cell state contains `TensorArray`s that are not amenable
+        to reordering.
 
     Raises:
       TypeError: if `cell` is not an instance of `RNNCell`,
@@ -202,6 +310,7 @@ class BeamSearchDecoder(decoder.Decoder):
           "output_layer must be a Layer, received: %s" % type(output_layer))
     self._cell = cell
     self._output_layer = output_layer
+    self._reorder_tensor_arrays = reorder_tensor_arrays
 
     if callable(embedding):
       self._embedding_fn = embedding
@@ -299,12 +408,13 @@ class BeamSearchDecoder(decoder.Decoder):
     """
     finished, start_inputs = self._finished, self._start_inputs
 
+    dtype = nest.flatten(self._initial_cell_state)[0].dtype
     log_probs = array_ops.one_hot(  # shape(batch_sz, beam_sz)
         array_ops.zeros([self._batch_size], dtype=dtypes.int32),
         depth=self._beam_width,
-        on_value=0.0,
-        off_value=-np.Inf,
-        dtype=nest.flatten(self._initial_cell_state)[0].dtype)
+        on_value=ops.convert_to_tensor(0.0, dtype=dtype),
+        off_value=ops.convert_to_tensor(-np.Inf, dtype=dtype),
+        dtype=dtype)
 
     initial_state = BeamSearchDecoderState(
         cell_state=self._initial_cell_state,
@@ -341,6 +451,11 @@ class BeamSearchDecoder(decoder.Decoder):
         outputs.parent_ids,
         max_sequence_lengths=max_sequence_lengths,
         end_token=self._end_token)
+    if self._reorder_tensor_arrays:
+      final_state = final_state._replace(cell_state=nest.map_structure(
+          lambda t: self._maybe_sort_array_beams(
+              t, outputs.parent_ids, final_state.lengths),
+          final_state.cell_state))
     outputs = FinalBeamSearchDecoderOutput(
         beam_search_decoder_output=outputs, predicted_ids=predicted_ids)
     return outputs, final_state
@@ -431,9 +546,10 @@ class BeamSearchDecoder(decoder.Decoder):
       returned unchanged.
 
     Raises:
-      TypeError: If `t` is an instance of `TensorArray`.
       ValueError: If the rank of `t` is not statically known.
     """
+    if isinstance(t, tensor_array_ops.TensorArray):
+      return t
     _check_maybe(t)
     if t.shape.ndims >= 1:
       return self._split_batch_beams(t, s)
@@ -454,15 +570,55 @@ class BeamSearchDecoder(decoder.Decoder):
       A reshaped version of t with shape `[batch_size, beam_width] + s`.
 
     Raises:
-      TypeError: If `t` is an instance of `TensorArray`.
       ValueError:  If the rank of `t` is not statically known.
     """
+    if isinstance(t, tensor_array_ops.TensorArray):
+      return t
     _check_maybe(t)
     if t.shape.ndims >= 2:
       return self._merge_batch_beams(t, s)
     else:
       return t
 
+  def _maybe_sort_array_beams(self, t, parent_ids, sequence_length):
+    """Maybe sorts beams within a `TensorArray`.
+
+    Args:
+      t: A `TensorArray` of size `max_time` that contains `Tensor`s of shape
+        `[batch_size, beam_width, s]` or `[batch_size * beam_width, s]` where
+        `s` is the depth shape.
+      parent_ids: The parent ids of shape `[max_time, batch_size, beam_width]`.
+      sequence_length: The sequence length of shape `[batch_size, beam_width]`.
+
+    Returns:
+      A `TensorArray` where beams are sorted in each `Tensor` or `t` itself if
+      it is not a `TensorArray` or does not meet shape requirements.
+    """
+    if not isinstance(t, tensor_array_ops.TensorArray):
+      return t
+    # pylint: disable=protected-access
+    if (not t._infer_shape or not t._element_shape
+        or t._element_shape[0].ndims is None
+        or t._element_shape[0].ndims < 1):
+      shape = (
+          t._element_shape[0] if t._infer_shape and t._element_shape
+          else tensor_shape.TensorShape(None))
+      tf_logging.warn("The TensorArray %s in the cell state is not amenable to "
+                      "sorting based on the beam search result. For a "
+                      "TensorArray to be sorted, its elements shape must be "
+                      "defined and have at least a rank of 1, but saw shape: %s"
+                      % (t.handle.name, shape))
+      return t
+    shape = t._element_shape[0]
+    # pylint: enable=protected-access
+    if not _check_static_batch_beam_maybe(
+        shape, tensor_util.constant_value(self._batch_size), self._beam_width):
+      return t
+    t = t.stack()
+    with ops.control_dependencies(
+        [_check_batch_beam(t, self._batch_size, self._beam_width)]):
+      return gather_tree_from_array(t, parent_ids, sequence_length)
+
   def step(self, time, inputs, state, name=None):
     """Perform a decoding step.
 
@@ -757,6 +913,8 @@ def _maybe_tensor_gather_helper(gather_indices, gather_from, batch_size,
     output: Gathered tensor of shape tf.shape(gather_from)[:1+len(gather_shape)]
       or the original tensor if its dimensions are too small.
   """
+  if isinstance(gather_from, tensor_array_ops.TensorArray):
+    return gather_from
   _check_maybe(gather_from)
   if gather_from.shape.ndims >= len(gather_shape):
     return _tensor_gather_helper(
diff --git a/tensorflow/contrib/slim/README.md b/tensorflow/contrib/slim/README.md
index 2d9df8f27e..40f484fd78 100644
--- a/tensorflow/contrib/slim/README.md
+++ b/tensorflow/contrib/slim/README.md
@@ -94,7 +94,7 @@ of thin wrapper functions in
 [variables.py](https://www.tensorflow.org/code/tensorflow/contrib/framework/python/ops/variables.py)
 which allow callers to easily define variables.
 
-For example, to create a `weight` variable, initialize it using a truncated
+For example, to create a `weights` variable, initialize it using a truncated
 normal distribution, regularize it with an `l2_loss` and place it on the `CPU`,
 one need only declare the following:
 
diff --git a/tensorflow/contrib/solvers/python/ops/least_squares.py b/tensorflow/contrib/solvers/python/ops/least_squares.py
index fb7c0eb649..6e164f5342 100644
--- a/tensorflow/contrib/solvers/python/ops/least_squares.py
+++ b/tensorflow/contrib/solvers/python/ops/least_squares.py
@@ -33,7 +33,7 @@ def cgls(operator, rhs, tol=1e-6, max_iter=20, name="cgls"):
   r"""Conjugate gradient least squares solver.
 
   Solves a linear least squares problem \\(||A x - rhs||_2\\) for a single
-  righ-hand side, using an iterative, matrix-free algorithm where the action of
+  right-hand side, using an iterative, matrix-free algorithm where the action of
   the matrix A is represented by `operator`. The CGLS algorithm implicitly
   applies the symmetric conjugate gradient algorithm to the normal equations
   \\(A^* A x = A^* rhs\\). The iteration terminates when either
diff --git a/tensorflow/contrib/solvers/python/ops/linear_equations.py b/tensorflow/contrib/solvers/python/ops/linear_equations.py
index d791d46763..9305c6a11c 100644
--- a/tensorflow/contrib/solvers/python/ops/linear_equations.py
+++ b/tensorflow/contrib/solvers/python/ops/linear_equations.py
@@ -41,7 +41,7 @@ def conjugate_gradient(operator,
   r"""Conjugate gradient solver.
 
   Solves a linear system of equations `A*x = rhs` for selfadjoint, positive
-  definite matrix `A` and righ-hand side vector `rhs`, using an iterative,
+  definite matrix `A` and right-hand side vector `rhs`, using an iterative,
   matrix-free algorithm where the action of the matrix A is represented by
   `operator`. The iteration terminates when either the number of iterations
   exceeds `max_iter` or when the residual norm has been reduced to `tol`
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index c832c6f2e0..906cc3f034 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -83,6 +83,7 @@ cc_library(
         "kernels/trt_engine_op.h",
     ],
     copts = tf_copts(),
+    visibility = ["//visibility:public"],
     deps = [
         ":trt_logging",
         ":trt_resources",
@@ -154,6 +155,7 @@ py_library(
     deps = [
         ":trt_convert_py",
         ":trt_ops_py",
+        "//tensorflow/python:errors",
     ],
 )
 
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index dfcce0fd00..461e627e99 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -2,7 +2,8 @@ Using TensorRT in TensorFlow
 ============================
 
 This module provides necessary bindings and introduces TRT_engine_op
-operator that wraps a subgraph in TensorRT.
+operator that wraps a subgraph in TensorRT. This is still a work in progress
+but should be useable with most common graphs.
 
 Compilation
 -----------
@@ -15,26 +16,10 @@ configure script should find the necessary components from the system
 automatically. If installed from tar packages, user has to set path to
 location where the library is installed during configuration.
 
-
-```
+```shell
 bazel build --config=cuda --config=opt //tensorflow/tools/pip_package:build_pip_package
 bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use is shown below.
-
-```python
-import tensorflow as tf
-import tensorflow.contrib.tensorrt as trt
-#... create and train or load model
-gdef = sess.graph.as_graph_def()
-trt_gdef = trt.create_inference_graph(
-    gdef, #original graph_def
-    ["output"], #name of output node(s)
-    max_batch_size, #maximum batch size to run the inference
-    max_workspace_size_bytes) # max memory for TensorRT to use
-tf.reset_default_graph()
-tf.import_graph_def(graph_def=trt_gdef)
-#...... run inference
-```
+will be available. An example use can be found in test/test_tftrt.py directory
diff --git a/tensorflow/contrib/tensorrt/__init__.py b/tensorflow/contrib/tensorrt/__init__.py
index fd551d70b4..140ad48282 100644
--- a/tensorflow/contrib/tensorrt/__init__.py
+++ b/tensorflow/contrib/tensorrt/__init__.py
@@ -18,6 +18,18 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-# pylint: disable=unused-import,wildcard-import
-from tensorflow.contrib.tensorrt.python import *
-# pylint: enable=unused-import,wildcard-import
+from tensorflow.python.framework import errors
+
+# pylint: disable=unused-import,wildcard-import,g-import-not-at-top
+try:
+  from tensorflow.contrib.tensorrt.python import *
+except errors.NotFoundError as e:
+  no_trt_message = (
+      '**** Failed to initialize TensorRT. This is either because the TensorRT'
+      ' installation path is not in LD_LIBRARY_PATH, or because you do not have'
+      ' it installed. If not installed, please go to'
+      ' https://developer.nvidia.com/tensorrt to download and install'
+      ' TensorRT ****')
+  print(no_trt_message)
+  raise e
+# pylint: enable=unused-import,wildcard-import,g-import-not-at-top
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 970f810473..eea8c8efa2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/contrib/tensorrt/convert/convert_graph.h"
 
+#include <list>
 #include <map>
 #include <set>
 #include <unordered_map>
@@ -48,13 +49,29 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-static bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
-      "Identity", "Const", "Conv2D", "MaxPool", "BiasAdd", "Relu",
-      "Add",      "Mul",   "Sub",    "Rsqrt",   "Pad"  // "Placeholder" ,"Mean"
+      "Identity",
+      "Const",
+      "Conv2D",
+      "MaxPool",
+      "BiasAdd",
+      "Relu",
+      "Add",
+      "Mul",
+      "Sub",
+      "Rsqrt",
+      "Pad",
+      "Mean",
+      "AvgPool",
+      "ConcatV2",
+      "DepthwiseConv2dNative",
+      "FusedBatchNorm",
+      "FusedBatchNormV2",
+      // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
   return candidate_ops.count(node_def.op());
@@ -69,6 +86,8 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
       if (!subgraph_node_ids.count(edge->src()->id()) &&
           !edge->src()->IsSource()) {
         incoming_edges->insert(edge);
+      } else {
+        VLOG(2) << edge->src()->name() << " N, ";
       }
     }
   }
@@ -82,7 +101,10 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
           !edge->dst()->IsSink()) {
+        VLOG(2) << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
+      } else {
+        VLOG(2) << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -109,74 +131,150 @@ std::unordered_map<string, std::vector<int>> BuildTensorNameMap(
   }
   return result;
 }
-
-tensorflow::Status ConvertSubGraphToTensorRT(
-    const std::vector<string>& output_names,
-    const std::set<int>& subgraph_node_ids,
-    size_t max_batch_size,  // Max batch size that engine will be created for
-    // Max amount of memory that engine will be allowed to consume, in bytes
-    size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_properties,
-    tensorflow::Graph* graph) {
-  tensorflow::EdgeSet subgraph_incoming_edges;
-  GetSubGraphIncomingEdges(*graph, subgraph_node_ids, &subgraph_incoming_edges);
-
+// TODO(sami): convert references to pointers
+struct ConvertGraphParams {
+  ConvertGraphParams(
+      tensorflow::Graph& inp_graph,
+      const std::vector<string>& output_node_names,
+      const std::set<int>& subgraph_node_id_numbers,
+      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
+      const tensorflow::grappler::GraphProperties& current_graph_properties,
+      std::unordered_map<string, std::pair<int, string>>* output_edges,
+      int engine_precision_mode)
+      : graph(inp_graph),
+        output_names(output_node_names),
+        subgraph_node_ids(subgraph_node_id_numbers),
+        max_batch_size(max_supported_batch_size),
+        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
+        graph_properties(current_graph_properties),
+        output_edge_map(output_edges),
+        precision_mode(engine_precision_mode) {}
+  tensorflow::Graph& graph;
+  const std::vector<string>& output_names;
+  const std::set<int>& subgraph_node_ids;
+  size_t max_batch_size;
+  size_t max_workspace_size_bytes;
+  const tensorflow::grappler::GraphProperties& graph_properties;
+  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
+  int precision_mode;
   std::vector<std::pair<int, int>> subgraph_inputs;
+  std::vector<std::pair<int, int>> subgraph_outputs;
+  tensorflow::EdgeSet subgraph_incoming_edges;
+  tensorflow::EdgeSet subgraph_outgoing_edges;
+};
 
-  // Collect inputs by looking for incoming edges
-  for (const tensorflow::Edge* edge : subgraph_incoming_edges) {
-    subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
+static tensorflow::Status FillSubGraphEdgeSets(ConvertGraphParams* p) {
+  GetSubGraphIncomingEdges(p->graph, p->subgraph_node_ids,
+                           &p->subgraph_incoming_edges);
+  for (const tensorflow::Edge* edge : p->subgraph_incoming_edges) {
+    p->subgraph_inputs.push_back({edge->src()->id(), edge->src_output()});
   }
+  auto output_name_to_index_map = BuildTensorNameMap(p->output_names);
   std::set<std::pair<int, int>> subgraph_outputs_set;
   // Collect outputs referenced from output_names
-  auto output_name_to_index_map = BuildTensorNameMap(output_names);
-  for (int node_id : subgraph_node_ids) {
-    tensorflow::Node* node = graph->FindNodeId(node_id);
+  for (int node_id : p->subgraph_node_ids) {
+    tensorflow::Node* node = p->graph.FindNodeId(node_id);
     if (output_name_to_index_map.count(node->name())) {
       for (int index : output_name_to_index_map.at(node->name())) {
         subgraph_outputs_set.insert({node_id, index});
       }
     }
   }
-  // Collect outputs referenced from outgoing edges
-  tensorflow::EdgeSet subgraph_outgoing_edges;
-  GetSubGraphOutgoingEdges(*graph, subgraph_node_ids, &subgraph_outgoing_edges);
-  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
+  GetSubGraphOutgoingEdges(p->graph, p->subgraph_node_ids,
+                           &p->subgraph_outgoing_edges);
+  for (const tensorflow::Edge* edge : p->subgraph_outgoing_edges) {
     subgraph_outputs_set.insert({edge->src()->id(), edge->src_output()});
   }
-  // Impose an ordering on the outputs
-  std::vector<std::pair<int, int>> subgraph_outputs(
-      subgraph_outputs_set.begin(), subgraph_outputs_set.end());
-  // Build TensorRT node and add it to the graph
+  p->subgraph_outputs.reserve(subgraph_outputs_set.size());
+  p->subgraph_outputs.insert(p->subgraph_outputs.begin(),
+                             subgraph_outputs_set.begin(),
+                             subgraph_outputs_set.end());
+  return tensorflow::Status::OK();
+};
+
+tensorflow::Status GetCalibNode(ConvertGraphParams* params) {
+  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
   tensorflow::NodeDef trt_node_def;
-  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(
-      *graph, subgraph_node_ids, subgraph_inputs, subgraph_outputs,
-      max_batch_size, max_workspace_size_bytes, graph_properties,
-      &trt_node_def));
+  SubGraphParams s(params->graph, params->subgraph_node_ids,
+                   params->subgraph_inputs, params->subgraph_outputs,
+                   params->max_batch_size, params->max_workspace_size_bytes,
+                   params->graph_properties, params->output_edge_map,
+                   &trt_node_def, params->precision_mode);
+  TF_RETURN_IF_ERROR(InjectCalibrationNode(s));
   tensorflow::Status status;
-  tensorflow::Node* trt_node = graph->AddNode(trt_node_def, &status);
+  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
+
+  TF_RETURN_IF_ERROR(status);
+
+  for (auto in_edge :
+       params->subgraph_incoming_edges) {  // loop over incoming edges and
+                                           // attach them to calib node
+    // tensorflow::Node* src_node = in_edge->src();
+    auto src_output = in_edge->src_output();
+    auto dst_node = in_edge->dst();
+    auto dst_input = in_edge->dst_input();
+    VLOG(1) << " update edge " << trt_node->name() << ":" << src_output
+            << " -> " << dst_node->name() << ":" << dst_input;
+    TF_RETURN_IF_ERROR(
+        params->graph.UpdateEdge(trt_node, src_output, dst_node, dst_input));
+  }
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertSubGraphToTensorRT(ConvertGraphParams* params) {
+  TF_RETURN_IF_ERROR(FillSubGraphEdgeSets(params));
+  tensorflow::NodeDef trt_node_def;
+
+  SubGraphParams s(params->graph, params->subgraph_node_ids,
+                   params->subgraph_inputs, params->subgraph_outputs,
+                   params->max_batch_size, params->max_workspace_size_bytes,
+                   params->graph_properties, params->output_edge_map,
+                   &trt_node_def, params->precision_mode);
+  TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRTNodeDef(s));
+  tensorflow::Status status;
+  tensorflow::Node* trt_node = params->graph.AddNode(trt_node_def, &status);
+
+  // AddNode does not wire edges.
+  // Re-map incoming edges to use the new TRT node instead of the orig subgraph
+  std::map<std::pair<int, int>, int> subgraph_edge_to_input_map;
+  for (size_t i = 0; i < params->subgraph_inputs.size(); ++i) {
+    subgraph_edge_to_input_map.insert({params->subgraph_inputs.at(i), i});
+  }
+  for (const tensorflow::Edge* edge : params->subgraph_incoming_edges) {
+    std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
+    int new_src_output = subgraph_edge_to_input_map.at(old_src);
+    params->graph.AddEdge(edge->src(), edge->src_output(), trt_node,
+                          new_src_output);
+    params->graph.RemoveEdge(edge);
+  }
+
+  VLOG(2) << "new wiring edges: " << trt_node->in_edges().size();
+  for (const tensorflow::Edge* edge : trt_node->in_edges()) {
+    VLOG(2) << edge->src()->name() << " port: " << edge->src_output();
+  }
+
   TF_RETURN_IF_ERROR(status);
 
   // Re-map outgoing edges to use the new TRT node instead of the orig subgraph
   std::map<std::pair<int, int>, int> subgraph_edge_to_output_map;
-  for (size_t i = 0; i < subgraph_outputs.size(); ++i) {
-    subgraph_edge_to_output_map.insert({subgraph_outputs.at(i), i});
+  for (size_t i = 0; i < params->subgraph_outputs.size(); ++i) {
+    subgraph_edge_to_output_map.insert({params->subgraph_outputs.at(i), i});
   }
   TF_RETURN_IF_ERROR(status);
-  for (const tensorflow::Edge* edge : subgraph_outgoing_edges) {
+  for (const tensorflow::Edge* edge : params->subgraph_outgoing_edges) {
     std::pair<int, int> old_src = {edge->src()->id(), edge->src_output()};
     int new_src_output = subgraph_edge_to_output_map.at(old_src);
-    TF_RETURN_IF_ERROR(graph->UpdateEdge(trt_node, new_src_output, edge->dst(),
-                                         edge->dst_input()));
+    TF_RETURN_IF_ERROR(params->graph.UpdateEdge(
+        trt_node, new_src_output, edge->dst(), edge->dst_input()));
   }
   // Remove the original subgraph
-  for (int node_id : subgraph_node_ids) {
-    tensorflow::Node* node = graph->FindNodeId(node_id);
+  for (int node_id : params->subgraph_node_ids) {
+    tensorflow::Node* node = params->graph.FindNodeId(node_id);
     // Don't remove the input placeholders
     if (node->type_string() == "Placeholder") {
       continue;
     }
-    graph->RemoveNode(node);
+    params->graph.RemoveNode(node);
   }
   return tensorflow::Status::OK();
 }
@@ -194,12 +292,39 @@ tensorflow::Status BuildNodeMap(
 }
 
 }  // namespace
+tensorflow::Status ConvertCalibGraphToInferGraph(
+    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* infer_graph) {
+  VLOG(0) << "Starting Calib Conversion";
+  tensorflow::Graph graph(tensorflow::OpRegistry::Global());
+  TF_RETURN_IF_ERROR(tensorflow::ConvertGraphDefToGraph(
+      tensorflow::GraphConstructorOptions(), graph_def, &graph));
+  //  get calib nodes
+  std::vector<tensorflow::Node*> calib_nodes;
+  for (auto node : graph.op_nodes()) {
+    if (node->type_string() == "TRTCalibOp") {
+      VLOG(1) << "Found Calib Node";
+      calib_nodes.push_back(node);
+    }
+  }
+  VLOG(0) << "Num Calib nodes in graph= " << calib_nodes.size();
+  if (calib_nodes.size() == 0)
+    return tensorflow::errors::FailedPrecondition(
+        "Graph doesn't contain any calibration nodes!."
+        " Please generate calibration graph and run calibration first");
+  for (auto n : calib_nodes) {
+    TF_RETURN_IF_ERROR(
+        tensorrt::convert::ConvertCalibrationNodeToEngineNode(graph, n));
+  }
+  graph.ToGraphDef(infer_graph);
+  return tensorflow::Status::OK();
+}
 
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def) {
-  // Optimization pass
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
+    int precision_mode = FP32MODE, int minimum_segment_size = 3) {
+  // optimization pass
   tensorflow::grappler::GrapplerItem item;
   item.fetch = output_names;
   tensorflow::GraphDef gdef;
@@ -209,16 +334,23 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   tensorflow::grappler::LayoutOptimizer optimizer;
   tensorflow::grappler::Cluster* cluster;
 
-  // Virtual cluster
+  // virtual cluster
   tensorflow::DeviceProperties device_properties;
+
   device_properties.set_type("GPU");
   device_properties.mutable_environment()->insert({"architecture", "6"});
   cluster =
       new tensorflow::grappler::VirtualCluster({{"/GPU:0", device_properties}});
 
+  // single machine
+  int num_cpu_cores = tensorflow::grappler::GetNumAvailableLogicalCPUCores();
+  int num_gpus = tensorflow::grappler::GetNumAvailableGPUs();
+  VLOG(2) << "cpu_cores: " << num_cpu_cores;
+  VLOG(2) << "gpus: " << num_gpus;
+
   TF_RETURN_IF_ERROR(optimizer.Optimize(cluster, item, &gdef));
 
-  // Constant folding
+  // constant folding
   item.graph = gdef;
   tensorflow::grappler::ConstantFolding fold(nullptr);
   TF_RETURN_IF_ERROR(fold.Optimize(nullptr, item, &gdef));
@@ -226,7 +358,6 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   // AJ refactoring shape inference through grappler/GraphProperties.
   tensorflow::grappler::GraphProperties static_graph_properties(item);
   TF_RETURN_IF_ERROR(static_graph_properties.InferStatically(false));
-
   // Build full graph
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
                                              gdef.library());
@@ -243,7 +374,7 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
 
   // TODO(sami): this should be passed as a knob!!!!
-  segment_options.minimum_segment_size = 2;
+  segment_options.minimum_segment_size = minimum_segment_size;
   tensorflow::tensorrt::segment::SegmentNodesVector segments;
   TF_RETURN_IF_ERROR(tensorrt::segment::SegmentGraph(
       gdef, IsTensorRTCandidate, segment_options, &segments));
@@ -252,14 +383,37 @@ tensorflow::Status ConvertGraphDefToTensorRT(
   }
   std::unordered_map<string, tensorflow::Node*> node_map;
   TF_RETURN_IF_ERROR(BuildNodeMap(graph, &node_map));
+  std::unordered_map<string, std::pair<int, string>> output_edge_map;
+  int count = 0;
+  float total_num_nodes_in_segments = 0.;
+  for (auto s : segments) {
+    total_num_nodes_in_segments += s.size();
+  }
   for (const std::set<string>& subgraph_node_names : segments) {
     std::set<int> subgraph_node_ids;
+    size_t max_mem_per_engine =
+        max_workspace_size_bytes *
+        ((float)subgraph_node_names.size() / total_num_nodes_in_segments);
+    std::stringstream oss;
     for (const string& node_name : subgraph_node_names) {
+      oss << " " << node_name;
       subgraph_node_ids.insert(node_map.at(node_name)->id());
     }
-    TF_RETURN_IF_ERROR(ConvertSubGraphToTensorRT(
-        output_names, subgraph_node_ids, max_batch_size,
-        max_workspace_size_bytes, static_graph_properties, &graph));
+    VLOG(2) << "Subgraph nodes" << oss.str();
+    ConvertGraphParams p(graph, output_names, subgraph_node_ids, max_batch_size,
+                         max_mem_per_engine, static_graph_properties,
+                         &output_edge_map, precision_mode);
+    if (precision_mode == INT8MODE) {
+      TF_RETURN_IF_ERROR(GetCalibNode(&p));
+    } else {
+      tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
+      if (status != tensorflow::Status::OK()) {
+        LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
+                     << " due to: \n"
+                     << status.ToString() << " SKIPPING......";
+      }
+      count++;
+    }
   }
   graph.ToGraphDef(new_graph_def);
   return tensorflow::Status::OK();
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.h b/tensorflow/contrib/tensorrt/convert/convert_graph.h
index 154ad3f2e8..e01e4a5328 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.h
@@ -28,14 +28,20 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
+// This method converts an already generated calibration graph which was used in
+// calibration runs to an inference graph
+tensorflow::Status ConvertCalibGraphToInferGraph(
+    const tensorflow::GraphDef& graph_def, tensorflow::GraphDef* new_graph_def);
+
 // max_batch_size: maximum batch size which can be used for inference for
 //                 optimization targets inference run with max batch size.
-// max_workspace_size_bytes: The upper bound of memory allowence for
+// max_workspace_size_bytes: The upper bound of memory allowance for
 //                 engine building.
 tensorflow::Status ConvertGraphDefToTensorRT(
     const tensorflow::GraphDef& graph_def,
     const std::vector<string>& output_names, size_t max_batch_size,
-    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def);
+    size_t max_workspace_size_bytes, tensorflow::GraphDef* new_graph_def,
+    int precision_mode, int minimum_segment_size);
 
 }  // namespace convert
 }  // namespace tensorrt
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 9ee717dd7f..92a692baa7 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -24,6 +24,10 @@ limitations under the License.
 #include <utility>
 #include <vector>
 
+#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resource_manager.h"
+#include "tensorflow/contrib/tensorrt/resources/trt_resources.h"
+#include "tensorflow/core/framework/node_def.pb.h"  // NOLINT
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"  // NOLINT
 #include "tensorflow/core/framework/types.h"
@@ -32,6 +36,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/tensor_coding.h"
@@ -39,7 +44,6 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "tensorflow/contrib/tensorrt/log/trt_logger.h"
 #include "tensorrt/include/NvInfer.h"
 
 //  Check if the types are equal. Cast to int first so that failure log message
@@ -49,6 +53,7 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrCat;
 
 namespace {
 
@@ -65,7 +70,8 @@ inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
       *trt_dtype = nvinfer1::DataType::kHALF;
       break;
     default:
-      return tensorflow::errors::InvalidArgument("Unsupported data type");
+      return tensorflow::errors::InvalidArgument(
+          "Unsupported data type " + tensorflow::DataTypeString(tf_dtype));
   }
   return tensorflow::Status::OK();
 }
@@ -112,6 +118,18 @@ static std::vector<std::pair<int, int>> CreateSamePadding(
   return padding;
 }
 
+string GetCommonNameScope(const string& op_name_a, const string& op_name_b) {
+  size_t last_scope_separator = 0;
+  for (size_t i = 0; i < std::min(op_name_a.size(), op_name_b.size()); ++i) {
+    if (op_name_a[i] != op_name_b[i]) {
+      break;
+    } else if (op_name_a[i] == '/') {
+      last_scope_separator = i + 1;
+    }
+  }
+  return op_name_a.substr(0, last_scope_separator);
+}
+
 class TRT_ShapedWeights {
  public:
   TRT_ShapedWeights(tensorflow::DataType type, const void* values,
@@ -244,6 +262,11 @@ std::vector<int> TFAttrs::get<std::vector<int>>(string key) const {
   return std::vector<int>(attr.begin(), attr.end());
 }
 
+template <>
+std::vector<string> TFAttrs::get<std::vector<string>>(string key) const {
+  auto attr = this->at(key)->list().s();
+  return std::vector<string>(attr.begin(), attr.end());
+}
 template <>
 nvinfer1::Dims TFAttrs::get<nvinfer1::Dims>(string key) const {
   auto values = this->get<std::vector<int>>(key);
@@ -266,6 +289,17 @@ tensorflow::DataType TFAttrs::get<tensorflow::DataType>(string key) const {
   return this->at(key)->type();
 }
 
+template <>
+float TFAttrs::get<float>(string key) const {
+  return this->at(key)->f();
+}
+
+template <>
+bool TFAttrs::get<bool>(string key) const {
+  return this->at(key)->b();
+}
+
+// TODO(jie): reorder4 & reorder2 should be merged?
 template <typename T>
 void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
               nvinfer1::DimsNCHW istrides, T* odata,
@@ -283,29 +317,87 @@ void Reorder4(nvinfer1::DimsNCHW shape, const T* idata,
   }
 }
 
+template <typename T>
+void Reorder2(nvinfer1::DimsHW shape, const T* idata, nvinfer1::DimsHW istrides,
+              T* odata, nvinfer1::DimsHW ostrides) {
+  for (int h = 0; h < shape.h(); ++h) {
+    for (int w = 0; w < shape.w(); ++w) {
+      odata[h * ostrides.h() + w * ostrides.w()] =
+          idata[h * ostrides.h() + w * ostrides.w()];
+    }
+  }
+}
+
+// TODO(jie): fallback to tensorflow!!
+void ReorderCKtoKC(const TRT_ShapedWeights& iweights,
+                   TRT_ShapedWeights* oweights) {
+  int c = iweights.shape_.d[0];
+  int k = iweights.shape_.d[1];
+  oweights->shape_.d[0] = k;
+  oweights->shape_.d[1] = c;
+  nvinfer1::DimsHW istrides = {1, k};
+  nvinfer1::DimsHW ostrides = {c, 1};
+  switch (iweights.type_) {
+    case tensorflow::DataType::DT_FLOAT: {
+      Reorder2({k, c}, static_cast<float const*>(iweights.GetValues()),
+               istrides,
+               static_cast<float*>(const_cast<void*>(oweights->GetValues())),
+               ostrides);
+      break;
+    }
+    case tensorflow::DataType::DT_HALF: {
+      Reorder2(
+          {k, c}, static_cast<Eigen::half const*>(iweights.GetValues()),
+          istrides,
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
+    default:
+      LOG(FATAL) << "Unsupported type in reorder expected fp32 or fp16 but got "
+                 << DataTypeString(iweights.type_);
+  }
+}
+
 void ReorderRSCKToKCRS(const TRT_ShapedWeights& iweights,
-                       TRT_ShapedWeights* oweights) {
+                       TRT_ShapedWeights* oweights, int num_groups) {
   CHECK_EQ(iweights.type_, oweights->type_);
   CHECK_EQ(iweights.size_bytes(), oweights->size_bytes());
   int r = iweights.shape_.d[0];
   int s = iweights.shape_.d[1];
-  int c = iweights.shape_.d[2];
-  int k = iweights.shape_.d[3];
-  oweights->shape_.d[0] = k;
-  oweights->shape_.d[1] = c;
+  // TRT requires GKcRS, while TF depthwise has RSCK
+  //   where c=1, C=G
+  VLOG(2) << "num_groups: " << num_groups;
+  int c = iweights.shape_.d[2] / num_groups;
+  VLOG(2) << "c" << iweights.shape_.d[2] << " then " << c;
+  int k = iweights.shape_.d[3] * num_groups;
+  VLOG(2) << "k" << iweights.shape_.d[3] << " then " << k;
+  oweights->shape_.d[0] = k / num_groups;
+  oweights->shape_.d[1] = c * num_groups;
   oweights->shape_.d[2] = r;
   oweights->shape_.d[3] = s;
   nvinfer1::DimsNCHW istrides = {1, k, s * k * c, c * k};
   nvinfer1::DimsNCHW ostrides = {c * r * s, r * s, s, 1};
   switch (iweights.type_) {
-    case tensorflow::DataType::DT_FLOAT:
+    case tensorflow::DataType::DT_FLOAT: {
       Reorder4({k, c, r, s}, static_cast<float const*>(iweights.GetValues()),
                istrides,
                static_cast<float*>(const_cast<void*>(oweights->GetValues())),
                ostrides);
       break;
+    }
+    case tensorflow::DataType::DT_HALF: {
+      Reorder4(
+          {k, c, r, s}, static_cast<Eigen::half const*>(iweights.GetValues()),
+          istrides,
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues())),
+          ostrides);
+      break;
+    }
+
     default:
-      LOG(FATAL) << "!!!!!!!!!!!!!!!!!!!!!!!!broke!!!!!!!!!!!!";
+      LOG(FATAL) << "Unsupported type, expected fp32 or fp16 but got "
+                 << DataTypeString(iweights.type_);
   }
 }
 
@@ -323,12 +415,11 @@ inline std::shared_ptr<T> infer_object(T* obj) {
   return std::shared_ptr<T>(obj, InferDeleter());
 }
 
-// Logger for GIE info/warning/errors
 class Converter;
 
 using OpConverter =
     std::function<tensorflow::Status(Converter&, const tensorflow::NodeDef&,
-                                     std::vector<TRT_TensorOrWeights> const&,
+                                     const std::vector<TRT_TensorOrWeights>&,
                                      std::vector<TRT_TensorOrWeights>*)>;
 
 class Converter {
@@ -336,34 +427,57 @@ class Converter {
   std::unordered_map<string, OpConverter> op_registry_;
   nvinfer1::INetworkDefinition* trt_network_;
   std::list<std::vector<uint8_t>> temp_bufs_;
-
+  tensorflow::tensorrt::TRTWeightStore* weight_store_;
+  bool fp16_;
   void register_op_converters();
-
   std::vector<TRT_TensorOrWeights> get_inputs(
       const tensorflow::NodeDef& node_def) {
     std::vector<TRT_TensorOrWeights> inputs;
-    for (const auto& input_name : node_def.input()) {
-      VLOG(2) << "Retrieve input: " << input_name;
-      inputs.push_back(trt_tensors_.at(input_name));
+    for (auto const& input_name : node_def.input()) {
+      /*************************************************************************
+       * TODO(jie) handle case 1) here
+       * Normalizes the inputs and extracts associated metadata:
+       * 1) Inputs can contain a colon followed by a suffix of characters.
+       *    That suffix may be a single number (e.g. inputName:1) or several
+       *    word characters separated from a number by a colon
+       *    (e.g. inputName:foo:1). The
+       *    latter case is used to denote inputs and outputs of functions.
+       * 2) Control dependency inputs contain caret at the beginning and we
+       *    remove this and annotate the edge as a control dependency.
+       ************************************************************************/
+      string name = input_name[0] == '^' ? input_name.substr(1) : input_name;
+      auto first = name.find_first_of(':');
+      if (first != string::npos && first + 2 == name.size() &&
+          name[first + 1] == '0')
+        name.erase(first);
+
+      VLOG(2) << "retrieve input: " << name;
+      if (trt_tensors_.count(name)) {
+        inputs.push_back(trt_tensors_.at(name));
+      } else {
+        LOG(FATAL) << "input: " << name << " not available for node at, "
+                   << node_def.name();
+      }
     }
     return inputs;
   }
 
  public:
-  explicit Converter(nvinfer1::INetworkDefinition* trt_network)
-      : trt_network_(trt_network) {
+  explicit Converter(nvinfer1::INetworkDefinition* trt_network,
+                     tensorflow::tensorrt::TRTWeightStore* ws, bool fp16)
+      : trt_network_(trt_network), weight_store_(ws), fp16_(fp16) {
     this->register_op_converters();
   }
-
+  tensorflow::tensorrt::TRTWeightStore* weight_store() { return weight_store_; }
   TRT_ShapedWeights get_temp_weights(tensorflow::DataType type,
                                      nvinfer1::Dims shape) {
     TRT_ShapedWeights weights(type, nullptr, shape);
     // TODO(jie): check weights size_bytes. 0 means type error
-    temp_bufs_.push_back(std::vector<uint8_t>(weights.size_bytes()));
-    weights.SetValues(temp_bufs_.back().data());
+    weight_store_->store_.push_back(std::vector<uint8_t>(weights.size_bytes()));
+    weights.SetValues(weight_store_->store_.back().data());
     return weights;
   }
-
+  bool isFP16() { return fp16_; };
   TRT_ShapedWeights get_temp_weights_like(const TRT_ShapedWeights& weights) {
     return this->get_temp_weights(weights.type_, weights.shape_);
   }
@@ -382,7 +496,7 @@ class Converter {
       TRT_TensorOrWeights output = outputs.at(i);
       // TODO(jie): tf protobuf seems to be omitting the :0 suffix
       string output_name = node_def.name();
-      if (i != 0) output_name = output_name + ":" + std::to_string(i);
+      if (i != 0) output_name = StrCat(output_name, ":", i);
       if (output.is_tensor()) {
         output.tensor()->setName(output_name.c_str());
       }
@@ -448,7 +562,7 @@ struct LambdaFactory {
     switch (op) {
       case OP_CATEGORY::RSQRT: {
         VLOG(2) << "RSQRT GETS DONE";
-        return [](T t) -> T { return 1.0 / std::sqrt(t); };
+        return [](T t) -> T { return 1.0 / sqrt(t); };
       }
       case OP_CATEGORY::NEG:
         return [](T t) -> T { return -t; };
@@ -534,6 +648,22 @@ struct LambdaFactory {
   }
 };
 
+template <>
+std::function<Eigen::half(Eigen::half)> LambdaFactory::unary<Eigen::half>() {
+  switch (op) {
+    case OP_CATEGORY::RSQRT: {
+      VLOG(2) << "RSQRT GETS DONE";
+      return [](Eigen::half t) -> Eigen::half {
+        return Eigen::half(1.0 / sqrt(float(t)));
+      };
+    }
+    case OP_CATEGORY::NEG:
+      return [](Eigen::half t) -> Eigen::half { return -t; };
+    default:
+      VLOG(2) << "Not supported op for unary: " << static_cast<int>(op);
+      return nullptr;
+  }
+}
 tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
                                 TRT_ShapedWeights* oweights,
                                 LambdaFactory unary_op) {
@@ -545,6 +675,14 @@ tensorflow::Status UnaryCompute(const TRT_ShapedWeights& iweights,
       std::transform(inp, inp + iweights.count(), oup, unary_op.unary<float>());
       break;
     }
+    case tensorflow::DataType::DT_HALF: {
+      auto inp = static_cast<Eigen::half const*>(iweights.GetValues());
+      auto oup =
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
+      std::transform(inp, inp + iweights.count(), oup,
+                     unary_op.unary<Eigen::half>());
+      break;
+    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -588,6 +726,32 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
       }
       break;
     }
+    case tensorflow::DataType::DT_HALF: {
+      auto inp_l = static_cast<const Eigen::half*>(iweights_l.GetValues());
+      auto inp_r = static_cast<const Eigen::half*>(iweights_r.GetValues());
+      auto oup =
+          static_cast<Eigen::half*>(const_cast<void*>(oweights->GetValues()));
+
+      if (iweights_l.count() != iweights_r.count()) {
+        // We only supports broadcast of RankZero
+        if (iweights_l.count() == 1) {
+          VLOG(2) << "I bet it is not working!" << (*inp_l);
+          std::transform(inp_r, inp_r + iweights_r.count(), oup,
+                         binary_op.broadcast_l<Eigen::half>(*inp_l));
+        } else if (iweights_r.count() == 1) {
+          VLOG(2) << "I bet it is not working!" << (*inp_r);
+          std::transform(inp_l, inp_l + iweights_l.count(), oup,
+                         binary_op.broadcast_r<Eigen::half>(*inp_r));
+        } else {
+          return tensorflow::errors::Unimplemented(
+              "Binary op with non-rankZero broadcast not supported");
+        }
+      } else {
+        std::transform(inp_l, inp_l + iweights_l.count(), inp_r, oup,
+                       binary_op.binary<Eigen::half>());
+      }
+      break;
+    }
     default:
       return tensorflow::errors::Unimplemented(
           "Data type not supported: " +
@@ -599,7 +763,7 @@ tensorflow::Status BinaryCompute(const TRT_ShapedWeights& iweights_l,
 
 tensorflow::Status ConstantFoldUnary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input = inputs.at(0).weights();
 
@@ -613,13 +777,12 @@ tensorflow::Status ConstantFoldUnary(
   CHECK_EQ(weights_input.type_,
            TFAttrs(node_def).get<tensorflow::DataType>("T"));
 
-  // Maybe I should do a switch
   LambdaFactory unary_op;
   if (node_def.op() == "Rsqrt") {
     // Compute rsqrt
     unary_op.op = LambdaFactory::OP_CATEGORY::RSQRT;
     auto ret = UnaryCompute(weights_input, &weights_output, unary_op);
-    // PAss the output
+    // Pass the output
     if (ret == tensorflow::Status::OK()) {
       outputs->push_back(TRT_TensorOrWeights(weights_output));
     }
@@ -631,11 +794,11 @@ tensorflow::Status ConstantFoldUnary(
 }
 
 // TODO(jie,ben) broadcast is needed yet not implemented
-// Let's get the simple stuff working first. Maybe we should fall bakc to TF
+// Let's get the simple stuff working first. Maybe we should fall back to TF
 //   approach for constant folding
 tensorflow::Status ConstantFoldBinary(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   TRT_ShapedWeights weights_input_l = inputs.at(0).weights();
   TRT_ShapedWeights weights_input_r = inputs.at(1).weights();
@@ -648,12 +811,12 @@ tensorflow::Status ConstantFoldBinary(
         "Binary op implicit broadcast not supported: " + node_def.op());
 
   // TODO(jie): constant fold should really fall back to TF.
-  int nb_dims = weights_input_l.shape_.nbDims;
+  int num_dims = weights_input_l.shape_.nbDims;
   nvinfer1::Dims output_shape;
-  output_shape.nbDims = nb_dims;
-  VLOG(2) << "nb_dims: " << nb_dims
+  output_shape.nbDims = num_dims;
+  VLOG(2) << "nb_dims: " << num_dims
           << ", the other: " << weights_input_r.shape_.nbDims;
-  for (int i = 0; i < nb_dims; i++) {
+  for (int i = 0; i < num_dims; i++) {
     if (weights_input_l.shape_.d[i] == weights_input_r.shape_.d[i]) {
       output_shape.d[i] = weights_input_l.shape_.d[i];
     } else if (weights_input_l.shape_.d[i] == 1 ||
@@ -678,7 +841,6 @@ tensorflow::Status ConstantFoldBinary(
   // Allocate output weights
   TRT_ShapedWeights weights_output = ctx.get_temp_weights(dtype, output_shape);
 
-  // Maybe I should do a switch
   LambdaFactory binary_op;
   if (node_def.op() == "Sub") {
     binary_op.op = LambdaFactory::OP_CATEGORY::SUB;
@@ -712,48 +874,90 @@ tensorflow::Status BinaryTensorOpWeight(
   // Maybe this part has to be moved into the block of rsqrt later
 
   // Check type consistency
-  auto dtype = TFAttrs(node_def).get<nvinfer1::DataType>("T");
-  CHECK_EQ_TYPE(tensor->getType(), dtype);  // Cast to int for error messages
   nvinfer1::DataType ttype;
   TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
-  CHECK_EQ_TYPE(ttype, dtype);  // Cast to int for error message
 
   // Check scale mode
   auto dims_w = weights.shape_;
   auto dims_t = tensor->getDimensions();
 
-  // Default to channel-wise
+  // default to element-wise
   auto scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
 
+  // TODO(jie): maybe use a permutation instead to support more cases;
+  bool permutation_flag = false;
+
   if (weights.count() == 1) {
     VLOG(2) << "UNIFORM";
     scale_mode = nvinfer1::ScaleMode::kUNIFORM;
   } else {
-    // No broadcasting on Batch dimension;
-    assert(dims_w.d[0] == 1);
-
-    // Broadcasting on Channel dimension only allowed in kUNIFORM
-    assert(dims_w.d[1] == dims_t.d[0]);
-    assert(dims_w.nbDims == dims_t.nbDims);
-
-    // Default is element;
-    for (int i = 2; i < dims_w.nbDims; i++) {
-      if (dims_w.d[i] != dims_t.d[i - 1]) {
-        scale_mode = nvinfer1::ScaleMode::kCHANNEL;
-        break;
+    // no broadcasting on Batch dimension;
+    VLOG(2) << "WEIGHTS DIM: " << dims_w.nbDims
+            << " tensor DIM: " << dims_t.nbDims;
+    if (dims_w.nbDims == dims_t.nbDims + 1) {
+      if (dims_w.d[0] == 1) {
+        for (int i = 1; i < dims_w.nbDims; i++) {
+          dims_w.d[i - 1] = dims_w.d[i];
+        }
+        dims_w.nbDims--;
+      } else {
+        return tensorflow::errors::InvalidArgument(
+            "Binary op cannot operate on batch, " + node_def.name());
       }
     }
-    if (scale_mode == nvinfer1::ScaleMode::kELEMENTWISE) {
+
+    if (dims_w.nbDims == dims_t.nbDims && dims_w.d[0] == dims_t.d[0]) {
       scale_mode = nvinfer1::ScaleMode::kELEMENTWISE;
-      for (int i = 2; i < dims_w.nbDims; i++) {
-        if (dims_w.d[i] != 1)
-          return tensorflow::errors::InvalidArgument(
-              "Weight shape not compatible at, " + node_def.name());
+      // default is element;
+      for (int i = 1; i < dims_w.nbDims; i++) {
+        if (dims_w.d[i] != dims_t.d[i]) {
+          // if dimension does not match, switch back to channel;
+          VLOG(2) << "channel";
+          scale_mode = nvinfer1::ScaleMode::kCHANNEL;
+          break;
+        }
+      }
+      // if channel as candidate, validate it
+      if (scale_mode == nvinfer1::ScaleMode::kCHANNEL) {
+        for (int i = 1; i < dims_w.nbDims; i++) {
+          if (dims_w.d[i] != 1)
+            return tensorflow::errors::InvalidArgument(
+                "Weight shape not compatible at, " + node_def.name());
+        }
+      } else {
+        VLOG(2) << "elementwise";
+      }
+    } else if (dims_w.nbDims == 1 &&
+               dims_w.d[0] == dims_t.d[dims_t.nbDims - 1]) {
+      // channel wise and broadcast required;
+      permutation_flag = true;
+      scale_mode = nvinfer1::ScaleMode::kCHANNEL;
+    } else {
+      return tensorflow::errors::InvalidArgument(
+          "Weight shape not compatible at, " + node_def.name());
+    }
+  }
+
+  // transpose last dimension
+  std::vector<int> permutation(dims_t.nbDims + 1);
+  if (permutation_flag) {
+    if (scale_mode == nvinfer1::ScaleMode::kCHANNEL && dims_t.nbDims > 1) {
+      // we swap the last dimension into channel for trt.
+      // because of tensorflow default broadcasting rules.
+      for (int i = 0; i < static_cast<int>(permutation.size()); i++) {
+        permutation[i] = i;
       }
+      permutation[1] = dims_t.nbDims;
+      permutation[dims_t.nbDims] = 1;
+      tensor = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor),
+                                   permutation);
+    } else {
+      return tensorflow::errors::InvalidArgument(
+          "Transpose cannot be applied, " + node_def.name());
     }
   }
 
-  // Prepare weights
+  // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
   TRT_ShapedWeights power_weights(weights.type_);
@@ -779,88 +983,26 @@ tensorflow::Status BinaryTensorOpWeight(
       scale_weights, power_weights);
 
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  // transpose back dimension
+  if (permutation_flag) {
+    output_tensor = ctx.TransposeTensor(output_tensor, permutation);
+  }
 
   // Pass the output
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status BinaryTensorOpTensor(
-    Converter& ctx, const tensorflow::NodeDef& node_def,
-    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
-      {"Add", nvinfer1::ElementWiseOperation::kSUM},
-      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
-      // {"max", nvinfer1::ElementWiseOperation::kMAX},
-      // {"min", nvinfer1::ElementWiseOperation::kMIN},
-      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
-      {"Div", nvinfer1::ElementWiseOperation::kDIV},
-  };
-
-  // FIXME assume type matches input weights
-  // Get trt type & shape
-  TFAttrs attrs(node_def);
-  // Maybe this part has to be moved into the block of rsqrt later
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
-
-  // Check type consistency
-  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
-  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
-  auto op_pair = ops.find(node_def.op());
-  if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
-
-  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
-      *const_cast<nvinfer1::ITensor*>(tensor_l),
-      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
-
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-
-  // Pass the output
-  outputs->push_back(TRT_TensorOrWeights(output_tensor));
-  return tensorflow::Status::OK();
-}
+enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 
-tensorflow::Status ConvertPlaceholder(
+tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs) {
-  VLOG(2) << "Placeholder should have been replace already";
-  return tensorflow::errors::Unimplemented(", cannot convert Placeholder op");
-  // OK this make sense since we are supposed to replace it with input
-  TFAttrs attrs(node_def);
-  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
-  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
-
-  dims.nbDims--;
-  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
-
-  nvinfer1::ITensor* output =
-      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
-  if (!output) {
-    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
-  }
-  outputs->push_back(TRT_TensorOrWeights(output));
-  return tensorflow::Status::OK();
-}
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs,
+    int group  // group ==0 specifies depthwise conv
+) {
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
-tensorflow::Status ConvertConv2D(Converter& ctx,
-                                 const tensorflow::NodeDef& node_def,
-                                 const std::vector<TRT_TensorOrWeights>& inputs,
-                                 std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  // TODO(jie): handle NHWC/NCHW transpose;
-  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
-  ReorderRSCKToKCRS(weights_rsck, &weights);
-  TRT_ShapedWeights biases(weights.type_);
-  int noutput = weights.shape_.d[0];
-  nvinfer1::DimsHW kernel_size;
-  kernel_size.h() = weights.shape_.d[2];
-  kernel_size.w() = weights.shape_.d[3];
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -874,11 +1016,31 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
     // TODO(jie): transpose it
   }
 
+  // tensor after transpose (NCHW)
+  auto tensor_dim = tensor->getDimensions();
+
+  int num_groups = group;
+  if (num_groups == 0)  // depthwise convolution
+    num_groups = tensor_dim.d[0];
+  VLOG(2) << "groups count: " << num_groups;
+
+  TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
+  ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
+  TRT_ShapedWeights biases(weights.type_);
+  int noutput = weights.shape_.d[0] * num_groups;
+  nvinfer1::DimsHW kernel_size;
+  kernel_size.h() = weights.shape_.d[2];
+  kernel_size.w() = weights.shape_.d[3];
+  VLOG(2) << "kernel size: " << kernel_size.h() << ", " << kernel_size.w();
+
   // TODO(jie): stride. (NHWC/NCHW)
   auto tf_stride = attrs.get<std::vector<int>>("strides");
+  VLOG(2) << "h_INDEX" << h_index << ", w_index " << w_index;
+  VLOG(2) << "stride!!!: " << tf_stride[0] << tf_stride[1] << tf_stride[2]
+          << tf_stride[3];
   nvinfer1::DimsHW stride(tf_stride[h_index], tf_stride[w_index]);
 
-  auto tensor_dim = tensor->getDimensions();
   std::vector<std::pair<int, int>> padding;
   // TODO(jie): padding.
   if (attrs.get<string>("padding") == "SAME") {
@@ -919,10 +1081,11 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
   layer->setStride(stride);
   layer->setPadding({padding[0].first, padding[1].first});
   layer->setName(node_def.name().c_str());
+  layer->setNbGroups(num_groups);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
 
   auto dim_after = output_tensor->getDimensions();
-  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1]
+  VLOG(2) << "TENSOR out: " << dim_after.d[0] << ", " << dim_after.d[1] << ", "
           << dim_after.d[2] << ", " << dim_after.d[3];
 
   if (data_format == "NHWC") {
@@ -935,11 +1098,101 @@ tensorflow::Status ConvertConv2D(Converter& ctx,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertConv2DHelper(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs, ConvolutionType type) {
+  switch (type) {
+    case ConvolutionType::DEFAULT:
+      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 1);
+    case ConvolutionType::DEPTHWISE_CONV:
+      return ConvertConv2DHelper(ctx, node_def, inputs, outputs, 0);
+  }
+  return tensorflow::errors::Unimplemented("unsupported convolution type at, " +
+                                           node_def.name());
+}
+
+tensorflow::Status BinaryTensorOpTensor(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const nvinfer1::ITensor* tensor_l, const nvinfer1::ITensor* tensor_r,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  static const std::unordered_map<string, nvinfer1::ElementWiseOperation> ops{
+      {"Add", nvinfer1::ElementWiseOperation::kSUM},
+      {"Mul", nvinfer1::ElementWiseOperation::kPROD},
+      {"Sub", nvinfer1::ElementWiseOperation::kSUB},
+      {"Div", nvinfer1::ElementWiseOperation::kDIV},
+  };
+
+  // FIXME assume type matches input weights
+  // get trt type & shape
+  TFAttrs attrs(node_def);
+  // maybe this part has to be moved into the block of rsqrt later
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("T");
+
+  // check type consistency
+  CHECK_EQ_TYPE(tensor_l->getType(), dtype);
+  CHECK_EQ_TYPE(tensor_r->getType(), dtype);
+  auto op_pair = ops.find(node_def.op());
+  if (op_pair == ops.end())
+    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
+                                             " not supported at: " +
+                                             node_def.name());
+
+  nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
+      *const_cast<nvinfer1::ITensor*>(tensor_l),
+      *const_cast<nvinfer1::ITensor*>(tensor_r), op_pair->second);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  // pass the output
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertPlaceholder(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  VLOG(2) << "Placeholder should have been replace already";
+  return tensorflow::errors::Unimplemented("cannot convert Placeholder op");
+  // OK this make sense since we are supposed to replace it with input
+  TFAttrs attrs(node_def);
+  nvinfer1::DataType dtype = attrs.get<nvinfer1::DataType>("dtype");
+  nvinfer1::Dims dims = attrs.get<nvinfer1::Dims>("shape");
+
+  dims.nbDims--;
+  for (int i = 0; i < dims.nbDims; i++) dims.d[i] = dims.d[i + 1];
+
+  nvinfer1::ITensor* output =
+      ctx.network()->addInput(node_def.name().c_str(), dtype, dims);
+  if (!output) {
+    return tensorflow::errors::InvalidArgument("Failed to create Input layer");
+  }
+  outputs->push_back(TRT_TensorOrWeights(output));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertConv2D(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
+                             ConvolutionType::DEFAULT);
+}
+
+tensorflow::Status ConvertConv2DDepthwise(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  return ConvertConv2DHelper(ctx, node_def, inputs, outputs,
+                             ConvolutionType::DEPTHWISE_CONV);
+}
+
 tensorflow::Status ConvertPool(Converter& ctx,
                                const tensorflow::NodeDef& node_def,
-                               std::vector<TRT_TensorOrWeights> const& inputs,
+                               const std::vector<TRT_TensorOrWeights>& inputs,
                                std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   TFAttrs attrs(node_def);
 
   int h_index = 2;
@@ -957,6 +1210,8 @@ tensorflow::Status ConvertPool(Converter& ctx,
   // TODO(jie): support other pooling type
   if (node_def.op() == "MaxPool")
     type = nvinfer1::PoolingType::kMAX;
+  else if (node_def.op() == "AvgPool")
+    type = nvinfer1::PoolingType::kAVERAGE;
   else
     return tensorflow::errors::Unimplemented("Only supports Max pool");
 
@@ -1019,9 +1274,9 @@ tensorflow::Status ConvertPool(Converter& ctx,
 
 tensorflow::Status ConvertActivation(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   nvinfer1::IActivationLayer* layer = ctx.network()->addActivation(
       *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ActivationType::kRELU);
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
@@ -1031,14 +1286,14 @@ tensorflow::Status ConvertActivation(
 
 tensorflow::Status ConvertScale(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
     return tensorflow::errors::Unimplemented(
         "Only supports tensor op weight for now, at " + node_def.name());
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
@@ -1055,16 +1310,33 @@ tensorflow::Status ConvertScale(Converter& ctx,
   } else {
     VLOG(2) << "NCHW !!!!";
   }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      weights, empty_weights, empty_weights);
 
-  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
-  if (data_format == "NHWC") {
-    // TODO(jie): transpose it back!
-    output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1});
-  } else {
-    VLOG(2) << "NCHW !!!!";
+  auto dims = tensor->getDimensions();
+  VLOG(2) << "tensor dimensions: " << dims.nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    VLOG(2) << "i: " << dims.d[i];
+  }
+  dims = weights.shape_;
+  VLOG(2) << "tensor dimensions: " << dims.nbDims;
+  for (int i = 0; i < dims.nbDims; i++) {
+    VLOG(2) << "i: " << dims.d[i];
+  }
+
+  nvinfer1::ScaleMode mode = nvinfer1::ScaleMode::kCHANNEL;
+  if (weights.shape_.d[0] == 1) {
+    mode = nvinfer1::ScaleMode::kUNIFORM;
+  }
+
+  nvinfer1::IScaleLayer* layer =
+      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
+                              weights, empty_weights, empty_weights);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  if (data_format == "NHWC") {
+    // TODO(jie): transpose it back!
+    output_tensor = ctx.TransposeTensor(output_tensor, {0, 2, 3, 1});
+  } else {
+    VLOG(2) << "NCHW !!!!";
   }
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -1072,7 +1344,7 @@ tensorflow::Status ConvertScale(Converter& ctx,
 
 tensorflow::Status ConvertConst(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   const auto& weights_tensor = node_def.attr().at("value").tensor();
 
@@ -1091,20 +1363,144 @@ tensorflow::Status ConvertConst(Converter& ctx,
     VLOG(2) << "SCALAR!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
     if (tensor.dims() > 0) {
-      VLOG(2) << "Dimensions: " << tensor.dims();
-      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
-                                  GetTensorShape(tensor));
+      VLOG(2) << "dimensions: " << tensor.dims();
+      VLOG(2) << "size: " << weights_tensor.float_val_size();
+      scalar_shape = GetTensorShape(tensor);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        VLOG(2) << scalar_shape.d[i];
+      if (GetShapeSize(scalar_shape) != weights_tensor.float_val_size()) {
+        if (weights_tensor.float_val_size() == 1 ||
+            scalar_shape.d[0] == weights_tensor.float_val_size()) {
+          scalar_shape.nbDims = 1;
+          // no dimension provided. flatten it
+          scalar_shape.d[0] = weights_tensor.float_val_size();
+          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+        } else {
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
+        }
+      }
     } else {
       VLOG(2) << "Dimensions: " << tensor.dims();
       scalar_shape.nbDims = 1;
-      scalar_shape.d[0] = 1;
+      // no dimension provided. flatten it
+      scalar_shape.d[0] = weights_tensor.float_val_size();
       scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
       for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
         scalar_shape.d[i] = 0;
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
-      weights = TRT_ShapedWeights(dtype, weights_tensor.float_val().data(),
-                                  scalar_shape);
+    }
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      auto half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      half_tensor.device(defd) =
+          tensor.flat<float>().template cast<Eigen::half>();
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<float> tensor_data(
+          weights_tensor.float_val().begin(),
+          weights_tensor.float_val()
+              .end());  //  make a local copy first to flatten
+      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    }
+  } else if (!weights_tensor.int_val().empty()) {
+    VLOG(2) << "int!!!" << node_def.name();
+    nvinfer1::Dims scalar_shape;
+    if (tensor.dims() > 0) {
+      VLOG(2) << "dimensions: " << tensor.dims();
+      scalar_shape = GetTensorShape(tensor);
+      if (GetShapeSize(scalar_shape) != weights_tensor.int_val_size()) {
+        if (weights_tensor.int_val_size() == 1 ||
+            scalar_shape.d[0] == weights_tensor.int_val_size()) {
+          scalar_shape.nbDims = 1;
+          // no dimension provided. flatten it
+          scalar_shape.d[0] = weights_tensor.int_val_size();
+          scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+        } else {
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
+        }
+      }
+    } else {
+      VLOG(2) << "dimensions: " << tensor.dims();
+      scalar_shape.nbDims = 1;
+      // no dimension provided. flatten it
+      scalar_shape.d[0] = weights_tensor.int_val_size();
+      scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
+      for (int i = 1; i < nvinfer1::Dims::MAX_DIMS; i++) {
+        scalar_shape.d[i] = 0;
+        scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
+      }
+    }
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      switch (dtype) {
+        case (tensorflow::DT_INT32): {
+          half_tensor.device(defd) =
+              tensor.flat<int32>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT16): {
+          half_tensor.device(defd) =
+              tensor.flat<int16>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT8): {
+          half_tensor.device(defd) =
+              tensor.flat<int8>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_UINT8): {
+          half_tensor.device(defd) =
+              tensor.flat<uint8>().template cast<Eigen::half>();
+          break;
+        }
+        default:
+          return tensorflow::errors::InvalidArgument(
+              "Datatype " + tensorflow::DataTypeString(dtype) +
+              " for FP16 conversion");
+          break;
+      };
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+      len_data = std::max(len_data, len_tensor);
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<int32> tensor_data(
+          weights_tensor.int_val().begin(),
+          weights_tensor.int_val()
+              .end());  //  make a local copy first to flatten
+                        //  doesn't have to be contiguous
+      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
     }
   } else if (!weights_tensor.tensor_content().empty()) {
     VLOG(2) << "TENSOR!!!" << node_def.name();
@@ -1130,7 +1526,7 @@ tensorflow::Status ConvertConst(Converter& ctx,
 
 tensorflow::Status ConvertIdentity(
     Converter& ctx, const tensorflow::NodeDef& node_def,
-    std::vector<TRT_TensorOrWeights> const& inputs,
+    const std::vector<TRT_TensorOrWeights>& inputs,
     std::vector<TRT_TensorOrWeights>* outputs) {
   outputs->push_back(inputs.at(0));
   return tensorflow::Status::OK();
@@ -1138,7 +1534,7 @@ tensorflow::Status ConvertIdentity(
 
 tensorflow::Status ConvertBinary(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 std::vector<TRT_TensorOrWeights> const& inputs,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2)
     return tensorflow::errors::FailedPrecondition(
@@ -1165,7 +1561,7 @@ tensorflow::Status ConvertBinary(Converter& ctx,
 
 tensorflow::Status ConvertUnary(Converter& ctx,
                                 const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights> const& inputs,
+                                const std::vector<TRT_TensorOrWeights>& inputs,
                                 std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 1)
     return tensorflow::errors::FailedPrecondition(
@@ -1183,7 +1579,7 @@ tensorflow::Status ConvertUnary(Converter& ctx,
 
 tensorflow::Status ConvertReduce(Converter& ctx,
                                  const tensorflow::NodeDef& node_def,
-                                 std::vector<TRT_TensorOrWeights> const& inputs,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
                                  std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1191,7 +1587,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1229,6 +1625,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
       return tensorflow::errors::InvalidArgument("TRT cannot reduce at 0, at" +
                                                  node_def.name());
     if (index_list_data[i] == 1) permuted_index = 1;
+
     idx_set.emplace(index_list_data[i]);
   }
 
@@ -1236,7 +1633,7 @@ tensorflow::Status ConvertReduce(Converter& ctx,
   nvinfer1::DimsHW pool_kernel;
   if (permuted_index == 1) {
     for (int i = 2; i < nb_dims; i++) {
-      if (idx_set.count(i)) {
+      if (idx_set.count(i) == 0) {
         permuted_index = i;
         break;
       }
@@ -1271,12 +1668,13 @@ tensorflow::Status ConvertReduce(Converter& ctx,
     output_tensor = ctx.TransposeTensor(
         const_cast<nvinfer1::ITensor*>(output_tensor), permutation_order);
   }
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
 }
 
 tensorflow::Status ConvertPad(Converter& ctx,
                               const tensorflow::NodeDef& node_def,
-                              std::vector<TRT_TensorOrWeights> const& inputs,
+                              const std::vector<TRT_TensorOrWeights>& inputs,
                               std::vector<TRT_TensorOrWeights>* outputs) {
   if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
       !inputs.at(1).is_weights())
@@ -1284,7 +1682,7 @@ tensorflow::Status ConvertPad(Converter& ctx,
         "Input expects tensor and weights, at" + node_def.name());
 
   // Implement tensor binaryOp weight [channel wise] for now;
-  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
   auto dims = tensor->getDimensions();
   // Restore implicit batch dimension
   int nb_dims = dims.nbDims + 1;
@@ -1371,19 +1769,287 @@ tensorflow::Status ConvertPad(Converter& ctx,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status ConvertConcat(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  // not including the last input (axis) here
+  int input_size = static_cast<int>(inputs.size()) - 1;
+
+  if (!inputs.at(0).is_tensor())
+    return tensorflow::errors::InvalidArgument(
+        "Concat in TRT support only Tensor input, at " + node_def.name());
+
+  // We are retrieving the axis
+  TRT_ShapedWeights axis = inputs.at(input_size).weights();
+
+  TFAttrs attrs(node_def);
+  // auto attr_size = attrs.at("N")->i();
+  // auto data_type = attrs.get<nvinfer1::DataType>("T");
+  auto index_type = attrs.get<tensorflow::DataType>("Tidx");
+
+  // TODO(jie): handle data type
+  // Only expect to handle INT32 as index attributes for now
+  if (index_type != tensorflow::DataType::DT_INT32)
+    return tensorflow::errors::Unimplemented(
+        "Tidx supports only DT_INT32, at " + node_def.name());
+
+  int index = *(static_cast<int*>(const_cast<void*>(axis.GetValues())));
+
+  // TODO(jie): early termination with no-op (attr_size==1)
+
+  auto dim = inputs.at(0).tensor()->getDimensions();
+  // dimension check
+  if (index > dim.nbDims + 1)
+    return tensorflow::errors::InvalidArgument(
+        "Concatenate on axis out of dimension range, at " + node_def.name());
+
+  if (index == 0)
+    return tensorflow::errors::InvalidArgument(
+        "Concatenate on batch dimension not supported, at " + node_def.name());
+
+  // incase we need permutation;
+  std::vector<int> permutation_order(dim.nbDims + 1);
+
+  for (int i = 0; i < dim.nbDims + 1; i++) permutation_order[i] = i;
+
+  if (index != 1) {
+    permutation_order[1] = index - 1;
+    permutation_order[index - 1] = 1;
+  }
+
+  std::vector<nvinfer1::ITensor const*> inputs_vec;
+  // Shap chack (all input tensor should have same shape)
+  // starting from 0 since we are probably also doing transpose here;
+  for (int i = 0; i < input_size; i++) {
+    auto tensor_i = inputs.at(i).tensor();
+    auto dim_i = tensor_i->getDimensions();
+    if (dim_i.nbDims != dim.nbDims)
+      return tensorflow::errors::InvalidArgument(
+          "Concatenate receives inputs with inconsistent dimensions, at " +
+          node_def.name());
+
+    for (int j = 0; j < dim.nbDims; j++) {
+      // check dimension consistency on non-concatenate axis
+      if (j != index - 1 && dim_i.d[j] != dim.d[j])
+        return tensorflow::errors::InvalidArgument(
+            "Concatenate receives inputs with inconsistent shape, at" +
+            node_def.name());
+    }
+
+    // TRT does concatenation only on channel!
+    if (index != 1)
+      tensor_i = ctx.TransposeTensor(const_cast<nvinfer1::ITensor*>(tensor_i),
+                                     permutation_order);
+
+    inputs_vec.push_back(tensor_i);
+  }
+
+  // nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  nvinfer1::IConcatenationLayer* layer = ctx.network()->addConcatenation(
+      const_cast<nvinfer1::ITensor* const*>(inputs_vec.data()),
+      inputs_vec.size());
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+
+  if (index != 1) {
+    output_tensor = ctx.TransposeTensor(output_tensor, permutation_order);
+  }
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertFusedBatchNorm(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  TFAttrs attrs(node_def);
+  float epsilon = attrs.get<float>("epsilon");
+  auto data_format = attrs.get<string>("data_format");
+  if (data_format != "NCHW") {
+    return tensorflow::errors::Unimplemented(
+        "only data_format=NCHW is supported, at " + node_def.name());
+  }
+  bool is_training = attrs.get<bool>("is_training");
+  if (is_training) {
+    return tensorflow::errors::Unimplemented(
+        "only is_training=false is supported, at " + node_def.name());
+  }
+  nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
+  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
+  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
+  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
+  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
+  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
+  TRT_ShapedWeights combined_scale_weights =
+      ctx.get_temp_weights_like(scale_weights);
+  TRT_ShapedWeights combined_offset_weights =
+      ctx.get_temp_weights_like(offset_weights);
+  size_t nweight = scale_weights.count();
+  if ((scale_weights.type_ == offset_weights.type_) &&
+      (mean_weights.type_ == variance_weights.type_) &&
+      (scale_weights.type_ == variance_weights.type_)) {
+    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
+        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+      return tensorflow::errors::Unimplemented(
+          "only float32 or float16 weight data type is supported, for node " +
+          node_def.name() + " got " +
+          tensorflow::DataTypeString(scale_weights.type_));
+    }
+    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
+        float offset =
+            (static_cast<float const*>(offset_weights.GetValues()))[i];
+        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
+        float variance =
+            (static_cast<float const*>(variance_weights.GetValues()))[i];
+        float& combined_scale_ref = const_cast<float*>(
+            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
+        float& combined_offset_ref = const_cast<float*>(
+            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
+        combined_scale_ref = scale / sqrtf(variance + epsilon);
+        combined_offset_ref = offset - mean * combined_scale_ref;
+      }
+    } else {
+      const Eigen::half* scale_vals =
+          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
+      const Eigen::half* off_vals =
+          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
+      const Eigen::half* mean_vals =
+          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
+      const Eigen::half* variance_vals =
+          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
+      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale(scale_vals[i]);
+        float offset(off_vals[i]);
+        float mean(mean_vals[i]);
+        float variance(variance_vals[i]);
+        float combined_scale_ref = scale / sqrtf(variance + epsilon);
+        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
+        float combined_offset_ref = offset - mean * combined_scale_ref;
+        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+      }
+    }
+  }
+  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
+      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
+      combined_offset_weights.GetWeightsForTRT(),
+      combined_scale_weights.GetWeightsForTRT(),
+      dummy_power_weights.GetWeightsForTRT());
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertMatMul(Converter& ctx,
+                                 const tensorflow::NodeDef& node_def,
+                                 const std::vector<TRT_TensorOrWeights>& inputs,
+                                 std::vector<TRT_TensorOrWeights>* outputs) {
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+
+  // TODO(jie): transpose!
+  TFAttrs attrs(node_def);
+
+  TRT_ShapedWeights weights_ck = inputs.at(1).weights();
+  TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_ck);
+  ReorderCKtoKC(weights_ck, &weights);
+  TRT_ShapedWeights biases(weights.type_);
+
+  int noutput = weights.shape_.d[0];
+
+  nvinfer1::IFullyConnectedLayer* layer = ctx.network()->addFullyConnected(
+      *const_cast<nvinfer1::ITensor*>(tensor), noutput, weights, biases);
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status ConvertReshape(
+    Converter& ctx, const tensorflow::NodeDef& node_def,
+    const std::vector<TRT_TensorOrWeights>& inputs,
+    std::vector<TRT_TensorOrWeights>* outputs) {
+  if (inputs.size() != 2 || !inputs.at(0).is_tensor() ||
+      !inputs.at(1).is_weights())
+    return tensorflow::errors::InvalidArgument(
+        "Input expects tensor and weights, at" + node_def.name());
+
+  // implement tensor binaryOp weight [channel wise] for now;
+  const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
+  auto dims = tensor->getDimensions();
+  // restore implicit batch dimension
+
+  TRT_ShapedWeights shape = inputs.at(1).weights();
+
+  TFAttrs attrs(node_def);
+
+  auto padding_type = attrs.get<tensorflow::DataType>("Tshape");
+
+  if (shape.shape_.nbDims != 1)
+    return tensorflow::errors::InvalidArgument(
+        "reshape new shape is not 1 dimensional, at " + node_def.name());
+
+  // Only expect to handle INT32 as attributes for now
+  if (padding_type != tensorflow::DataType::DT_INT32)
+    return tensorflow::errors::Unimplemented(
+        "reshape new shape supports only DT_INT32, at " + node_def.name());
+
+  auto shape_data = static_cast<int*>(const_cast<void*>(shape.GetValues()));
+
+  if (shape_data[0] != -1)
+    return tensorflow::errors::InvalidArgument(
+        "reshape new shape first dimension is not -1, at " + node_def.name());
+
+  auto shape_num_dims = shape.shape_.d[0];
+  VLOG(2) << "shape dimensions: " << shape_num_dims;
+  int volume_w = 1;
+  for (int i = 1; i < shape.shape_.d[0]; i++) volume_w *= shape_data[i];
+
+  int volume_t = 1;
+  for (int i = 0; i < dims.nbDims; i++) volume_t *= dims.d[i];
+
+  VLOG(2) << "volume: " << volume_t << " volume weights: " << volume_w;
+  if (volume_w != volume_t)
+    return tensorflow::errors::InvalidArgument(
+        "volume does not agree between tensor and new shape, at " +
+        node_def.name());
+
+  nvinfer1::IShuffleLayer* layer =
+      ctx.network()->addShuffle(*const_cast<nvinfer1::ITensor*>(tensor));
+
+  nvinfer1::Dims reshape_dims;
+  VLOG(2) << "new dimension: " << shape_num_dims - 1;
+  reshape_dims.nbDims = shape_num_dims - 1;
+  for (int32_t i = 0; i < reshape_dims.nbDims; ++i) {
+    reshape_dims.d[i] = shape_data[i + 1];
+  }
+  layer->setReshapeDimensions(reshape_dims);
+  VLOG(2) << "new dimension: " << shape_num_dims - 1;
+
+  nvinfer1::ITensor* output_tensor = layer->getOutput(0);
+  auto dims_output = output_tensor->getDimensions();
+  VLOG(2) << "output tensor dimension:" << dims_output.nbDims;
+  outputs->push_back(TRT_TensorOrWeights(output_tensor));
+  return tensorflow::Status::OK();
+}
+
 void Converter::register_op_converters() {
   // vgg_16 slim implementation
   op_registry_["Placeholder"] = ConvertPlaceholder;
   op_registry_["Conv2D"] = ConvertConv2D;
+  op_registry_["DepthwiseConv2dNative"] = ConvertConv2DDepthwise;
   op_registry_["Relu"] = ConvertActivation;
   op_registry_["MaxPool"] = ConvertPool;
+  op_registry_["AvgPool"] = ConvertPool;
   // This could be really handled as ConvertBinary
   op_registry_["BiasAdd"] = ConvertScale;
   op_registry_["Const"] = ConvertConst;
-  // op_registry_["MatMul"] = ConvertFullyConnected;  // Not used in vgg
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
-  // op_registry_["AvgPool"] = ConvertPool;
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -1393,26 +2059,364 @@ void Converter::register_op_converters() {
   op_registry_["Mean"] = ConvertReduce;
   op_registry_["Pad"] = ConvertPad;
   // TODO(ben,jie): Add more ops
+
+  op_registry_["ConcatV2"] = ConvertConcat;
+  op_registry_["MatMul"] = ConvertMatMul;
+  op_registry_["Reshape"] = ConvertReshape;
+  op_registry_["FusedBatchNorm"] = ConvertFusedBatchNorm;
+  op_registry_["FusedBatchNormV2"] = ConvertFusedBatchNorm;
 }
 
 }  // namespace
+tensorflow::Status GetTensorRTGraph(tensorrt::convert::SubGraphParams& s) {
+  return tensorflow::errors::Unimplemented("Not implemented yet");
+}
+tensorflow::Status ConvertCalibrationNodeToEngineNode(
+    tensorflow::Graph& graph, tensorflow::Node* c_node) {
+  const auto ndef = c_node->def();
+
+  TFAttrs attrs(ndef);
+  std::vector<string> segment_nodes(
+      attrs.get<std::vector<string>>("segment_nodes"));
+  std::vector<string> output_nodes(
+      attrs.get<std::vector<string>>("segment_output_names"));
+  std::vector<string> input_names(
+      attrs.get<std::vector<string>>("input_names"));
+  string res_name = attrs.get<string>("resource_name");
+  VLOG(1) << "Node name " << c_node->name() << " res_name " << res_name;
+  string engine_name = "my_trt_op";
+  {
+    const auto node_id = tensorflow::str_util::Split(res_name, "_");
+    engine_name += node_id.back();
+  }
+  std::map<string, tensorflow::Node*> node_maps;
+
+  for (auto n : graph.op_nodes()) {
+    node_maps.insert({n->name(), n});
+  }
+  VLOG(1) << "Output Nodes:";
+  std::vector<tensorflow::DataType> out_types;
+  std::vector<const tensorflow::Edge*> out_edges;
+  for (auto& i : output_nodes) {
+    auto node_port = tensorflow::str_util::Split(i, ":");
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+    auto out_node_name = node_port.at(0);
+    if (node_port.size() > 1) {
+      VLOG(1) << "Multi port output" << node_port.at(0) << " "
+              << node_port.at(1) << " size=" << node_port.size();
+    }
+    auto node_it = node_maps.find(out_node_name);
+    if (node_it != node_maps.end()) {
+      tensorflow::Node* out_node = node_it->second;
+      int port = 0;
+      if (node_port.size() == 2) {
+        port = std::strtoul(node_port.at(1).c_str(), nullptr, 10);
+        out_types.push_back(out_node->output_type(port));
+      } else {
+        out_types.push_back(out_node->output_type(0));
+      }
+      for (auto out_edge : out_node->out_edges()) {
+        if (out_edge->src_output() == port) {
+          out_edges.push_back(out_edge);
+          break;
+        }
+      }
+    } else {
+      LOG(WARNING) << " couldn't find output node " << out_node_name;
+    }
+  }
+  VLOG(1) << "Input Nodes:";
+  for (auto& i : input_names) {
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+  }
+  auto trt_rm = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto resmgr = trt_rm->getManager("TRTCalibOps");
+  tensorflow::tensorrt::TRTCalibrationResource* calib_res = nullptr;
+  auto status = resmgr->Lookup(res_name, res_name, &calib_res);
+  if (!status.ok() || !calib_res->calibrator_) {
+    return tensorflow::errors::FailedPrecondition(
+        "You must run calibration"
+        " and inference conversion in the same proces");
+  }
+
+  calib_res->calibrator_->setDone();
+  calib_res->thr_->join();
+  delete calib_res->thr_;
+  if (!calib_res->engine_) {
+    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+                  "calibration graph?";
+  }
+  auto weight_rmgr = trt_rm->getManager("WeightStore");
+  TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
+      res_name, res_name));
+  auto engine_plan = calib_res->engine_->serialize();
+  calib_res->engine_->destroy();
+  calib_res->network_->destroy();
+  calib_res->builder_->destroy();
+  calib_res->thr_ = nullptr;
+  calib_res->engine_ = nullptr;
+  calib_res->builder_ = nullptr;
+  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
+  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  for (const auto in_edge : c_node->in_edges()) {
+    auto src = in_edge->src();
+    int dest_port = in_edge->dst_input();
+    income_edges.emplace_back(src->name(), in_edge->src_output(),
+                              c_node->input_type(dest_port));
+  }
+  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
+      income_edges);
+  op_builder.Input(input_list);
+  tensorflow::NodeDef engine_node;
+  const char* engine_plan_data = static_cast<const char*>(engine_plan->data());
+  string engine_plan_string(engine_plan_data,
+                            engine_plan_data + engine_plan->size());
+  status = op_builder.Attr("serialized_engine", engine_plan_string)
+               .Attr("input_nodes", input_names)
+               .Attr("output_nodes", output_nodes)
+               .Attr("OutT", out_types)
+               .Finalize(&engine_node);
+  if (!status.ok()) {
+    LOG(ERROR) << "Engine Node creation failed";
+    return status;
+  }
+  auto trt_engine_node = graph.AddNode(engine_node, &status);
+  TF_CHECK_OK(status);
+  for (size_t i = 0; i < out_edges.size(); i++) {
+    VLOG(1) << "Connecting trt_engine_node output " << i << " with "
+            << out_edges.at(i)->dst()->name() << " port "
+            << out_edges.at(i)->dst_input();
+    TF_RETURN_IF_ERROR(graph.UpdateEdge(trt_engine_node, i,
+                                        out_edges.at(i)->dst(),
+                                        out_edges.at(i)->dst_input()));
+  }
+  VLOG(1) << "Segment nodes:";
+  for (auto& i : segment_nodes) {
+    VLOG(1) << " " << i << " in graph " << node_maps.count(i);
+    auto it = node_maps.find(i);
+    if (it != node_maps.end()) {
+      graph.RemoveNode(it->second);
+    }
+  }
+  graph.RemoveNode(c_node);
+  return tensorflow::Status::OK();
+}
+
+tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
+  // Visit nodes in reverse topological order and construct the TRT network.
+
+  // Toposort
+  std::vector<tensorflow::Node*> order_vec;
+  tensorflow::GetPostOrder(s.graph, &order_vec);
+  // Select just the subgraph
+  std::list<tensorflow::Node*> order;
+  for (tensorflow::Node* node : order_vec) {
+    if (s.subgraph_node_ids.count(node->id())) {
+      order.push_front(node);  // we want topological order to construct the
+      // network layer by layer
+    }
+  }
+  // topological order is needed to build TRT network
+  static int static_id = 0;
+  string subgraph_name_scope;
+  if (!order.empty()) {
+    subgraph_name_scope = order.front()->name();
+  }
+  for (const tensorflow::Node* node : order) {
+    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
+  }
+  // TODO(sami,ben,jie): proper naming!
+  string calib_op_name =
+      StrCat(subgraph_name_scope, "my_trt_calib_op_", static_id);
+  string engine_name = StrCat(subgraph_name_scope, "my_trt_op", static_id);
+  static_id++;
+  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto op_rmgr = trt_rmgr->getManager("TRTCalibOps");
+  auto op_res = new tensorflow::tensorrt::TRTCalibrationResource();
+  TF_CHECK_OK(op_rmgr->Create(calib_op_name, calib_op_name, op_res));
+  op_res->logger_ = new tensorflow::tensorrt::Logger();
+  op_res->builder_ = nvinfer1::createInferBuilder(*(op_res->logger_));
+
+  if (!op_res->builder_) {
+    return tensorflow::errors::Internal(
+        "failed to create TensorRT builder object");
+  }
+
+  op_res->network_ = op_res->builder_->createNetwork();
+  if (!op_res->network_) {
+    return tensorflow::errors::Internal(
+        "failed to create TensorRT network object");
+  }
+
+  // Build the network
+  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
+  auto ws = new tensorflow::tensorrt::TRTWeightStore();
+  TF_CHECK_OK(weight_rmgr->Create(calib_op_name, calib_op_name, ws));
+  Converter converter(op_res->network_, ws, s.precision_mode == FP16MODE);
+  std::vector<string> input_names;
+  std::vector<tensorflow::DataType> input_dtypes;
+  for (const std::pair<int, int>& input : s.input_inds) {
+    VLOG(2) << "parsing input. Node id= " << input.first;
+    int node_id = input.first;
+    int output_idx = input.second;
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    auto node_name = node->name();
+    input_names.push_back(node_name);  // insert original node name without port
+    // TODO(jie): alternative :)
+    if (!s.graph_properties.HasOutputProperties(node_name))
+      return tensorflow::errors::Internal("failed to find input node: " +
+                                          node_name);
+
+    auto op_info_vec = s.graph_properties.GetOutputProperties(node_name);
+    if (static_cast<int>(op_info_vec.size()) < output_idx)
+      return tensorflow::errors::Internal(
+          "accessing output index of: ", output_idx, ", at node: ", node_name,
+          "with output entry from shape_map: ", op_info_vec.size());
+
+    auto op_info = op_info_vec.at(output_idx);
+
+    tensorflow::DataType tf_dtype = op_info.dtype();
+    input_dtypes.push_back(tf_dtype);
+
+    nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+
+    VLOG(2) << "accessing output index of: " << output_idx
+            << ", at node: " << node_name
+            << "with output entry from shape_map: " << op_info_vec.size();
+
+    // TODO(ben,jie): update TRT input format/dimension
+    nvinfer1::DimsCHW input_dim_psuedo_chw;
+    for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
+
+    for (int i = 1; i < op_info.shape().dim_size(); i++) {
+      VLOG(2) << "dimension: " << i
+              << " , size: " << op_info.shape().dim(i).size();
+      input_dim_psuedo_chw.d[i - 1] = op_info.shape().dim(i).size();
+    }
+
+    // TODO(ben,jie): proper way to restore input tensor name?
+    auto input_tensor_name = node_name;
+    if (output_idx != 0) input_tensor_name = StrCat(node_name, ":", output_idx);
+
+    nvinfer1::ITensor* input_tensor = converter.network()->addInput(
+        input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
+
+    if (!input_tensor)
+      return tensorflow::errors::InvalidArgument(
+          "Failed to create Input layer");
+    VLOG(2) << "input tensor name :" << input_tensor_name;
+
+    if (!converter.insert_input_tensor(input_tensor_name, input_tensor))
+      return tensorflow::errors::AlreadyExists(
+          "output tensor already exists for op: " + input_tensor_name);
+  }
+
+  VLOG(2) << "finished sorting";
+
+  for (const tensorflow::Node* node : order) {
+    const tensorflow::NodeDef& node_def = node->def();
+    VLOG(2) << "converting node: " << node_def.name() << " , " << node_def.op();
+    TF_RETURN_IF_ERROR(converter.convert_node(node_def));
+  }
+
+  VLOG(2) << "finished conversion";
+
+  // Gather output metadata
+  std::vector<string> output_names;
+  std::vector<tensorflow::DataType> output_dtypes;
+  int trt_engine_op_output_idx = 0;
+  for (const std::pair<int, int>& output : s.output_inds) {
+    int node_id = output.first;
+    int output_idx = output.second;
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
+    string op_name = node->name();
+    string tensor_name = op_name;
+
+    s.output_edge_map->insert(
+        {trt_engine_op_output_idx == 0
+             ? engine_name
+             : StrCat(engine_name, ":", trt_engine_op_output_idx),
+         {output_idx, tensor_name}});
+    trt_engine_op_output_idx++;
+    if (output_idx != 0) {
+      tensor_name = StrCat(tensor_name, ":", output_idx);
+    }
+    VLOG(1) << "output tensor name: " << tensor_name;
+    output_names.push_back(tensor_name);
+    auto tensor_or_weights = converter.get_tensor(tensor_name);
+    if (!tensor_or_weights.is_tensor()) {
+      return tensorflow::errors::InvalidArgument(
+          "Output node is weights not tensor");
+    }
+    nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
+    if (!tensor) {
+      return tensorflow::errors::NotFound("Output tensor not found: " +
+                                          tensor_name);
+    }
+    converter.network()->markOutput(*tensor);
+    tensorflow::DataType tf_dtype = node->output_type(output_idx);
+    output_dtypes.push_back(tf_dtype);
+    nvinfer1::DataType trt_dtype = nvinfer1::DataType::kFLOAT;
+    TF_RETURN_IF_ERROR(ConvertDType(tf_dtype, &trt_dtype));
+    tensor->setType(trt_dtype);
+  }
+
+  VLOG(2) << "finished output";
+
+  // Build the engine
+  op_res->builder_->setMaxBatchSize(s.max_batch_size);
+  op_res->builder_->setMaxWorkspaceSize(s.max_workspace_size_bytes);
+
+  // Build the TRT op
+  // TODO(sami,ben,jie): proper naming!
+  tensorflow::NodeDefBuilder op_builder(calib_op_name, "TRTCalibOp");
+  std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  for (size_t i = 0; i < input_names.size(); ++i) {
+    int output_idx = s.input_inds.at(i).second;
+    // we wired up the input here already, it is redundant to do it again in
+    //  ConvertSubGraphToTensorRT(convert_graph.cc)
+    auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
+        input_names.at(i), output_idx, input_dtypes.at(i));
+    VLOG(1) << calib_op_name << " input " << i << " = " << input_names.at(i)
+            << ":" << output_idx
+            << " dType= " << tensorflow::DataTypeString(input_dtypes.at(i));
+    income_edges.push_back(incoming_edge);
+  }
+  tensorflow::gtl::ArraySlice<tensorflow::NodeDefBuilder::NodeOut> input_list(
+      income_edges);
+  op_builder.Input(input_list);
+  std::vector<string> segment_names;
+  segment_names.reserve(s.subgraph_node_ids.size());
+  for (int i : s.subgraph_node_ids) {
+    auto node = s.graph.FindNodeId(i);
+    segment_names.push_back(node->name());
+  }
+  LOG(INFO) << "finished op preparation";
+
+  auto status = op_builder.Attr("segment_nodes", segment_names)
+                    .Attr("input_names", input_names)
+                    .Attr("segment_output_names", output_names)
+                    .Attr("resource_name", calib_op_name)
+                    .Finalize(s.trt_node);
+
+  LOG(INFO) << status.ToString();
+  LOG(INFO) << "finished op building";
+
+  return tensorflow::Status::OK();
+}
 
 tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
-    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
-    const std::vector<std::pair<int, int>>& input_inds,
-    const std::vector<std::pair<int, int>>& output_inds, size_t max_batch_size,
-    size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_properties,
-    tensorflow::NodeDef* trt_node) {
+    tensorrt::convert::SubGraphParams& s) {
   // Visit nodes in reverse topological order and construct the TRT network.
 
   // Toposort
   std::vector<tensorflow::Node*> order_vec;
-  tensorflow::GetPostOrder(graph, &order_vec);
+  tensorflow::GetPostOrder(s.graph, &order_vec);
   // Select just the subgraph
   std::list<tensorflow::Node*> order;
   for (tensorflow::Node* node : order_vec) {
-    if (subgraph_node_ids.count(node->id())) {
+    if (s.subgraph_node_ids.count(node->id())) {
       // We want topological order to contstruct the
       // network layer by layer
       order.push_front(node);
@@ -1434,46 +2438,86 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
         "Failed to create TensorRT network object");
   }
 
+  string subgraph_name_scope;
+  if (!order.empty()) {
+    subgraph_name_scope = order.front()->name();
+  }
+  for (const tensorflow::Node* node : order) {
+    subgraph_name_scope = GetCommonNameScope(subgraph_name_scope, node->name());
+  }
+  static int static_id = 0;
+  // TODO(sami,ben,jie): proper naming!
+  string engine_name = StrCat(subgraph_name_scope, "my_trt_op");
+  engine_name = StrCat(engine_name, static_id++);
+  auto trt_rmgr = tensorflow::tensorrt::TRTResourceManager::instance();
+  auto weight_rmgr = trt_rmgr->getManager("WeightStore");
+  auto ws = new tensorflow::tensorrt::TRTWeightStore();
+  TF_CHECK_OK(weight_rmgr->Create(engine_name, engine_name, ws));
+
   // Build the network
-  Converter converter(trt_network.get());
+  Converter converter(trt_network.get(), ws, s.precision_mode == FP16MODE);
 
   std::vector<string> input_names;
   std::vector<tensorflow::DataType> input_dtypes;
-  for (std::pair<int, int> const& input : input_inds) {
+  for (const std::pair<int, int>& input : s.input_inds) {
+    VLOG(2) << "parsing input!!!!!";
     int node_id = input.first;
     int output_idx = input.second;
-    tensorflow::Node* node = graph.FindNodeId(node_id);
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
     auto node_name = node->name();
-    input_names.push_back(node_name);  // Insert original node name without port
-    // TODO(jie): alternative :)
-    if (!graph_properties.HasOutputProperties(node_name))
-      return tensorflow::errors::Internal("Failed to find input node: " +
-                                          node_name);
+    // input_names should use the node name in the graph
+    // here it should be the input tensor name -> matching the binding
+    // insert original node name without port
+    auto tensor_name = node_name;
+    if (output_idx != 0) {
+      tensor_name = StrCat(tensor_name, ":", output_idx);
+    }
 
-    auto op_info_vec = graph_properties.GetOutputProperties(node_name);
-    if (static_cast<int>(op_info_vec.size()) < output_idx)
-      return tensorflow::errors::Internal(
-          "Accessing output index of: " + std::to_string(output_idx) +
-          ", at node: " + node_name + " with output entry from shape_map: " +
-          std::to_string(op_info_vec.size()));
+    VLOG(2) << "input name: " << node_name << " tensor_name: " << tensor_name
+            << " idx: " << output_idx;
 
-    auto op_info = op_info_vec.at(output_idx);
+    auto shape_inference_node_name = node_name;
+    auto shape_inference_output_idx = output_idx;
+    // rewire the shape inference to original node in the graph
+    if (s.output_edge_map->count(tensor_name)) {
+      shape_inference_node_name = s.output_edge_map->at(tensor_name).second;
+      shape_inference_output_idx = s.output_edge_map->at(tensor_name).first;
+    }
+    if (shape_inference_output_idx < 0) continue;
+    VLOG(2) << "shapeinference name: " << shape_inference_node_name
+            << " idx: " << shape_inference_output_idx;
+
+    if (!s.graph_properties.HasOutputProperties(shape_inference_node_name))
+      return tensorflow::errors::Internal("failed to find input node: " +
+                                          shape_inference_node_name);
+
+    auto op_info_vec =
+        s.graph_properties.GetOutputProperties(shape_inference_node_name);
+    if (static_cast<int>(op_info_vec.size()) <= shape_inference_output_idx)
+      return tensorflow::errors::Internal(
+          "accessing output index of: ", shape_inference_output_idx,
+          ", at node: ", shape_inference_node_name,
+          " with output entry from shape_map: ", op_info_vec.size());
 
+    auto op_info = op_info_vec.at(shape_inference_output_idx);
     tensorflow::DataType tf_dtype = op_info.dtype();
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
-    VLOG(2) << "Accessing output index of: " << std::to_string(output_idx)
+    VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
-            << " with output entry from shape_map: "
-            << std::to_string(op_info_vec.size());
-
+            << " with output entry from shape_map: " << op_info_vec.size();
     // TODO(ben,jie): update TRT input format/dimension
     nvinfer1::DimsCHW input_dim_psuedo_chw;
     for (int i = 0; i < 3; i++) input_dim_psuedo_chw.d[i] = 1;
 
+    // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
+    //            update the code once TRT 4.0 comes out.
+    if (op_info.shape().dim_size() != 4)
+      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
               << " , size: " << op_info.shape().dim(i).size();
@@ -1482,9 +2526,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(ben,jie): proper way to restore input tensor name?
     auto input_tensor_name = node_name;
-    if (output_idx != 0)
-      input_tensor_name = node_name + ":" + std::to_string(output_idx);
+    if (output_idx != 0) {
+      input_tensor_name = StrCat(node_name, ":", output_idx);
+    }
 
+    input_names.push_back(input_tensor_name);
     nvinfer1::ITensor* input_tensor = converter.network()->addInput(
         input_tensor_name.c_str(), dtype, input_dim_psuedo_chw);
 
@@ -1511,14 +2557,22 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   // Gather output metadata
   std::vector<string> output_names;
   std::vector<tensorflow::DataType> output_dtypes;
-  for (std::pair<int, int> const& output : output_inds) {
+  int trt_engine_op_output_idx = 0;
+  for (const std::pair<int, int>& output : s.output_inds) {
     int node_id = output.first;
     int output_idx = output.second;
-    tensorflow::Node* node = graph.FindNodeId(node_id);
+    tensorflow::Node* node = s.graph.FindNodeId(node_id);
     string op_name = node->name();
     string tensor_name = op_name;
+
+    s.output_edge_map->insert(
+        {trt_engine_op_output_idx == 0
+             ? engine_name
+             : StrCat(engine_name, ":", trt_engine_op_output_idx),
+         {output_idx, tensor_name}});
+    trt_engine_op_output_idx++;
     if (output_idx != 0)
-      tensor_name = tensor_name + ":" + std::to_string(output_idx);
+      tensorflow::strings::StrAppend(&tensor_name, ":", output_idx);
     VLOG(2) << "Output tensor name: " << tensor_name;
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
@@ -1540,19 +2594,25 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
 
   VLOG(2) << "Finished output";
-  // TODO(jie): static_id is not thread safe.
-  static int static_id = 0;
 
   // Build the engine
-  trt_builder->setMaxBatchSize(max_batch_size);
-  trt_builder->setMaxWorkspaceSize(max_workspace_size_bytes);
-  VLOG(0) << "Starting build engine " << static_id;
-  // TODO(ben,jie): half2 and int8 mode support
+  trt_builder->setMaxBatchSize(s.max_batch_size);
+  trt_builder->setMaxWorkspaceSize(s.max_workspace_size_bytes);
+  VLOG(0) << "Max batch size= " << s.max_batch_size
+          << " max workspace size= " << s.max_workspace_size_bytes;
+  if (s.precision_mode == FP16MODE) {
+    trt_builder->setHalf2Mode(true);
+    VLOG(0) << "Using FP16 precision mode";
+  }
+  LOG(INFO) << "starting build engine";
   string engine_plan_string;
   {
     auto trt_engine =
         infer_object(trt_builder->buildCudaEngine(*converter.network()));
     VLOG(0) << "Built network";
+    if (trt_engine.get() == nullptr) {
+      return tensorflow::errors::Internal("Engine building failure");
+    }
     auto engine_plan = infer_object(trt_engine->serialize());
     VLOG(0) << "Serialized engine";
     const char* engine_plan_data =
@@ -1560,18 +2620,19 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     engine_plan_string =
         string(engine_plan_data, engine_plan_data + engine_plan->size());
   }
-
-  VLOG(0) << "Finished engine";
+  TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
+      engine_name, engine_name));
+  LOG(INFO) << "finished engine " << engine_name;
 
   // Build the TRT op
-  // TODO(sami,ben,jie): proper naming!
-  tensorflow::NodeDefBuilder op_builder(
-      tensorflow::strings::StrCat("my_trt_op", static_id++), "TRTEngineOp");
+  tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
   std::vector<tensorflow::NodeDefBuilder::NodeOut> income_edges;
+  VLOG(2) << "input edge size: " << input_names.size();
   for (size_t i = 0; i < input_names.size(); ++i) {
-    int output_idx = input_inds.at(i).second;
-    // We wired up the input here already, it is redundant to do it again in
-    // ConvertSubGraphToTensorRT(convert_graph.cc)
+    VLOG(2) << "input edges: " << i << " " << input_names.at(i);
+    int output_idx = s.input_inds.at(i).second;
+    // we wired up the input here already, it is redundant to do it again in
+    //  ConvertSubGraphToTensorRT(convert_graph.cc)
     auto incoming_edge = tensorflow::NodeDefBuilder::NodeOut(
         input_names.at(i), output_idx, input_dtypes.at(i));
     income_edges.push_back(incoming_edge);
@@ -1586,7 +2647,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
                     .Attr("input_nodes", input_names)
                     .Attr("output_nodes", output_names)
                     .Attr("OutT", output_dtypes)
-                    .Finalize(trt_node);
+                    .Finalize(s.trt_node);
 
   VLOG(0) << status.ToString() << " finished op building";
 
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.h b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
index 2e7fd19566..954a1e72f8 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.h
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.h
@@ -17,6 +17,8 @@ limitations under the License.
 #define TENSORFLOW_CONTRIB_TENSORRT_CONVERT_CONVERT_NODES_H_
 
 #include <set>
+#include <string>
+#include <unordered_map>
 #include <utility>
 #include <vector>
 
@@ -32,16 +34,49 @@ namespace tensorflow {
 namespace tensorrt {
 namespace convert {
 
-tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
-    const tensorflow::Graph& graph, const std::set<int>& subgraph_node_ids,
-    const std::vector<std::pair<int, int>>&
-        input_inds,  // {node_id, output_idx}
-    const std::vector<std::pair<int, int>>&
-        output_inds,  // {node_id, output_idx}
-    size_t max_batch_size, size_t max_workspace_size_bytes,
-    const tensorflow::grappler::GraphProperties& graph_prop,
-    tensorflow::NodeDef* trt_node);
+const int FP32MODE = 0;
+const int FP16MODE = 1;
+const int INT8MODE = 2;
 
+struct SubGraphParams {
+  SubGraphParams(
+      tensorflow::Graph& inp_graph,
+      const std::set<int>& subgraph_node_id_numbers,
+      const std::vector<std::pair<int, int>>& input_indices,
+      const std::vector<std::pair<int, int>>& output_indices,
+      size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
+      const tensorflow::grappler::GraphProperties& current_graph_properties,
+      std::unordered_map<string, std::pair<int, string>>* output_edges,
+      tensorflow::NodeDef* constructed_trt_node,
+      int engine_precision_mode = FP32MODE)
+      : graph(inp_graph),
+        subgraph_node_ids(subgraph_node_id_numbers),
+        input_inds(input_indices),
+        output_inds(output_indices),
+        max_batch_size(max_supported_batch_size),
+        max_workspace_size_bytes(max_consumed_workspace_size_bytes),
+        graph_properties(current_graph_properties),
+        output_edge_map(output_edges),
+        trt_node(constructed_trt_node),
+        precision_mode(engine_precision_mode) {}
+
+  tensorflow::Graph& graph;
+  const std::set<int>& subgraph_node_ids;
+  const std::vector<std::pair<int, int>>& input_inds;   // {node_id, output_idx}
+  const std::vector<std::pair<int, int>>& output_inds;  // {node_id, output_idx}
+  size_t max_batch_size;
+  size_t max_workspace_size_bytes;
+  const tensorflow::grappler::GraphProperties& graph_properties;
+  std::unordered_map<string, std::pair<int, string>>* output_edge_map;
+  tensorflow::NodeDef* trt_node;
+  const int precision_mode;
+};
+
+// TODO(sami): Replace references with const reference or pointers
+tensorflow::Status ConvertSubGraphToTensorRTNodeDef(SubGraphParams& params);
+tensorflow::Status InjectCalibrationNode(SubGraphParams& params);
+tensorflow::Status ConvertCalibrationNodeToEngineNode(tensorflow::Graph& graph,
+                                                      tensorflow::Node* c_node);
 }  // namespace convert
 }  // namespace tensorrt
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
index 1dcb87e768..aea44fd8a2 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_calib_op.cc
@@ -21,10 +21,11 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_types.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/platform/stream_executor.h"
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda_runtime_api.h"
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
 
 namespace tensorflow {
@@ -113,7 +114,13 @@ void TRTCalibOp::Compute(tensorflow::OpKernelContext* ctx) {
     ctx->set_output(i, t);
   }
   VLOG(2) << "Filled map for sending";
-  calib_res->calibrator_->setBatch(input_data);
+  // copied from cuda_kernel_helper since it seems only valid in *.cu.cc files
+  const cudaStream_t* stream = CHECK_NOTNULL(
+      reinterpret_cast<const cudaStream_t*>(ctx->op_device_context()
+                                                ->stream()
+                                                ->implementation()
+                                                ->CudaStreamMemberHack()));
+  calib_res->calibrator_->setBatch(input_data, *stream);
   VLOG(2) << "Passed calibration data";
   // TODO(aaroey): make sure we wait for the completion of calibration on the
   // last batch in future PR.
diff --git a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
index 8efdf63ebe..b32371b642 100644
--- a/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
+++ b/tensorflow/contrib/tensorrt/kernels/trt_engine_op.cc
@@ -24,8 +24,12 @@ limitations under the License.
 #include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
-namespace tensorrt {
 static ::tensorflow::tensorrt::Logger logger;
+namespace gpu = ::perftools::gputools;
+using IRuntime = nvinfer1::IRuntime;
+using Dims = nvinfer1::Dims;
+
+namespace tensorrt {
 
 TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // read serialized_engine
@@ -40,10 +44,21 @@ TRTEngineOp::TRTEngineOp(OpKernelConstruction* context) : OpKernel(context) {
   // TODO(samikama) runtime should be taken from a resourcemanager as well.
   // Only engine should be in the op and context and runtime should be taken
   // from resourcemanager
-  nvinfer1::IRuntime* infer = nvinfer1::createInferRuntime(logger);
+  // TODO(jie): cudaSetDevice make sure trt engine is allocated on the same
+  // gpu where the input/output is also located.
+  int gpu_id = context->device()->tensorflow_gpu_device_info()->gpu_id;
+  cudaSetDevice(gpu_id);
+  int device;
+  cudaGetDevice(&device);
+  if (gpu_id != device) LOG(FATAL) << "set device failed!";
+
+  // TODO(samikama) runtime should be taken from a resourcemanager as well.
+  // Only engine should be in the op and context and runtime should be taken
+  // from resourcemanager
+
+  IRuntime* infer = nvinfer1::createInferRuntime(logger);
   trt_engine_ptr_.reset(infer->deserializeCudaEngine(
       serialized_engine.c_str(), serialized_engine.size(), nullptr));
-
   trt_execution_context_ptr_.reset(trt_engine_ptr_->createExecutionContext());
   // Runtime is safe to delete after engine creation
   infer->destroy();
@@ -55,7 +70,6 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
 
   size_t binding_index;
   int num_batch = 0;
-  bool valid = true;
   for (int i = 0; i < context->num_inputs(); i++) {
     // Grab the input tensor
     binding_index = trt_engine_ptr_->getBindingIndex(input_nodes_[i].c_str());
@@ -64,8 +78,12 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     const TensorShape& input_shape = input_tensor.shape();
     if (i == 0) {
       num_batch = input_shape.dim_size(0);
+      if (num_batch > trt_engine_ptr_->getMaxBatchSize()) {
+        LOG(FATAL) << "input tensor batch larger than max_batch_size: "
+                   << trt_engine_ptr_->getMaxBatchSize();
+      }
     } else if (num_batch != input_shape.dim_size(0)) {
-      valid = false;
+      LOG(FATAL) << "input data inconsistent batch size";
       break;
     }
     switch (trt_engine_ptr_->getBindingDataType(binding_index)) {
@@ -81,9 +99,6 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
     }
   }
 
-  // Might want a different way to inform the user of batch size inconsistency
-  if (!valid) LOG(WARNING) << "input data inconsistent batch size";
-
   for (int i = 0; i < static_cast<int>(output_nodes_.size()); i++) {
     // This is bad that we have to reallocate output buffer every run.
     // Create an output tensor
@@ -126,9 +141,11 @@ void TRTEngineOp::Compute(OpKernelContext* context) {
                                                 ->implementation()
                                                 ->CudaStreamMemberHack()));
 
-  // execution handled by TF since we are getting stream from TF.
-  // it is safe for CPU pointer array (buffers) to go out of scope after enqueue
-  trt_execution_context_ptr_->enqueue(num_batch, &buffers[0], *stream, nullptr);
+  // TODO(jie): trt enqueue does not return error
+  auto ret = trt_execution_context_ptr_->enqueue(num_batch, &buffers[0],
+                                                 *stream, nullptr);
+  VLOG(2) << "enqueue returns: " << ret;
+  // sync should be done by TF.
 }
 
 REGISTER_KERNEL_BUILDER(Name("TRTEngineOp").Device(DEVICE_GPU), TRTEngineOp);
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.cc b/tensorflow/contrib/tensorrt/log/trt_logger.cc
index 7add8cb8b3..dda0dc9e71 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.cc
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.cc
@@ -27,19 +27,19 @@ void Logger::log(Severity severity, const char* msg) {
   // Suppress info-level messages
   switch (severity) {
     case Severity::kINFO: {  // Mark TRT info messages as debug!
-      VLOG(2) << msg;
+      VLOG(2) << name_ << " " << msg;
       break;
     }
     case Severity::kWARNING: {
-      LOG(WARNING) << msg;
+      LOG(WARNING) << name_ << " " << msg;
       break;
     }
     case Severity::kERROR: {
-      LOG(ERROR) << msg;
+      LOG(ERROR) << name_ << " " << msg;
       break;
     }
     case Severity::kINTERNAL_ERROR: {
-      LOG(FATAL) << msg;
+      LOG(FATAL) << name_ << " " << msg;
       break;
     }
     // This is useless for now. But would catch it in future if enum changes. It
diff --git a/tensorflow/contrib/tensorrt/log/trt_logger.h b/tensorflow/contrib/tensorrt/log/trt_logger.h
index d71f66b933..7f3544f8cf 100644
--- a/tensorflow/contrib/tensorrt/log/trt_logger.h
+++ b/tensorflow/contrib/tensorrt/log/trt_logger.h
@@ -27,9 +27,11 @@ namespace tensorrt {
 
 // Logger for GIE info/warning/errors
 class Logger : public nvinfer1::ILogger {
- private:
+ public:
+  Logger(string name = "DefaultLogger") : name_(name){};
   void log(nvinfer1::ILogger::Severity severity, const char* msg) override;
 
+ private:
   string name_;
 };
 
diff --git a/tensorflow/contrib/tensorrt/python/__init__.py b/tensorflow/contrib/tensorrt/python/__init__.py
index 7e050a768c..0b2321b5fc 100644
--- a/tensorflow/contrib/tensorrt/python/__init__.py
+++ b/tensorflow/contrib/tensorrt/python/__init__.py
@@ -20,5 +20,6 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 from tensorflow.contrib.tensorrt.python.ops import trt_engine_op
+from tensorflow.contrib.tensorrt.python.trt_convert import calib_graph_to_infer_graph
 from tensorflow.contrib.tensorrt.python.trt_convert import create_inference_graph
 # pylint: enable=unused-import,line-too-long
diff --git a/tensorflow/contrib/tensorrt/python/trt_convert.py b/tensorflow/contrib/tensorrt/python/trt_convert.py
index 9454862f85..338475d90e 100644
--- a/tensorflow/contrib/tensorrt/python/trt_convert.py
+++ b/tensorflow/contrib/tensorrt/python/trt_convert.py
@@ -20,11 +20,17 @@ from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
 import six as _six
+from tensorflow.contrib.tensorrt.wrap_conversion import calib_convert
 from tensorflow.contrib.tensorrt.wrap_conversion import trt_convert
 from tensorflow.core.framework import graph_pb2
+from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import errors_impl as _impl
+from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.grappler import tf_optimizer
+from tensorflow.python.util import compat
+# pylint: enable=unused-import,line-too-long
 
 
 # TODO(skama): get outputs from session when implemented as c++
@@ -32,22 +38,33 @@ from tensorflow.python.framework import ops
 def create_inference_graph(input_graph_def,
                            outputs,
                            max_batch_size=1,
-                           max_workspace_size_bytes=2 << 20):
-  """Python wrapper for the TRT transormation.
-
+                           max_workspace_size_bytes=2 << 20,
+                           precision_mode="FP32",
+                           minimum_segment_size=3):
+  """Python wrapper for the TRT transformation.
 
   Args:
     input_graph_def: GraphDef object containing a model to be transformed.
-    outputs: List of tensors or node names for the model outputs.
+    outputs: list of tensors or node names for the model outputs.
     max_batch_size: max size for the input batch
     max_workspace_size_bytes: parameter to control memory allocation (in Bytes)
+    precision_mode: one of 'FP32', 'FP16' and 'INT8'
+    minimum_segment_size: the minimum number of nodes required for a subgraph to
+      be replaced by TRTEngineOp.
 
   Returns:
     New GraphDef with TRTEngineOps placed in graph replacing subgraphs.
 
   Raises:
+    ValueError: if the provided precision mode is invalid.
     RuntimeError: if the returned status message is malformed.
   """
+  supported_precision_modes = {"FP32": 0, "FP16": 1, "INT8": 2}
+  if precision_mode.upper() not in supported_precision_modes:
+    raise ValueError(("precision mode '{}' is not supported."
+                      "It should be one of {}").format(
+                          precision_mode, "{'FP32', 'FP16', 'INT8'}"))
+  mode = supported_precision_modes[precision_mode.upper()]
 
   def py2bytes(inp):
     return inp
@@ -83,7 +100,7 @@ def create_inference_graph(input_graph_def,
   # pair or strings where first one is encoded status and the second
   # one is the transformed graphs protobuf string.
   out = trt_convert(input_graph_def_str, out_names, max_batch_size,
-                    max_workspace_size_bytes)
+                    max_workspace_size_bytes, mode, minimum_segment_size)
   status = to_string(out[0])
   output_graph_def_string = out[1]
   del input_graph_def_str  # Save some memory
@@ -101,3 +118,46 @@ def create_inference_graph(input_graph_def,
   output_graph_def.ParseFromString(output_graph_def_string)
   del output_graph_def_string  # Save some memory
   return output_graph_def
+
+
+def calib_graph_to_infer_graph(calibration_graph_def):
+  """Convert an existing calibration graph to inference graph.
+
+  Args:
+    calibration_graph_def: the calibration GraphDef object with calibration data
+  Returns:
+    New GraphDef with TRTEngineOps placed in graph replacing calibration nodes.
+  Raises:
+    RuntimeError: if the returned status message is malformed.
+  """
+
+  def py2string(inp):
+    return inp
+
+  def py3string(inp):
+    return inp.decode("utf-8")
+
+  if _six.PY2:
+    to_string = py2string
+  else:
+    to_string = py3string
+
+  graph_str = calibration_graph_def.SerializeToString()
+  out = calib_convert(graph_str)
+  status = to_string(out[0])
+  output_graph_def_string = out[1]
+  del graph_str  # Save some memory
+  if len(status) < 2:
+    raise _impl.UnknownError(None, None, status)
+  if status[:2] != "OK":
+    msg = status.split(";")
+    if len(msg) == 1:
+      raise RuntimeError("Status message is malformed {}".format(status))
+    # pylint: disable=protected-access
+    raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
+                                         int(msg[0]))
+    # pylint: enable=protected-access
+  output_graph_def = graph_pb2.GraphDef()
+  output_graph_def.ParseFromString(output_graph_def_string)
+  del output_graph_def_string  # Save some memory
+  return output_graph_def
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
index 3d5cc76c42..dc7c93f869 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.cc
@@ -23,7 +23,7 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
-#include "cuda_runtime_api.h"
+#include "cuda/include/cuda_runtime_api.h"
 
 namespace tensorflow {
 namespace tensorrt {
@@ -38,22 +38,18 @@ TRTInt8Calibrator::TRTInt8Calibrator(
       done_(false),
       dev_buffers_(dev_buffers),
       calib_running_(false),
+      batch_is_set_(false),
       engine_name_(engine_name) {}
 
-bool TRTInt8Calibrator::setBatch(
-    const std::unordered_map<string, void*>& data) {
-  // TODO(aaroey): make sure that in future PR:
-  // 1. the mutex_lock is outside of the loop
-  // 2. wait() is used instead of wait_for()
-  // 3. done_ is to be protected by the mutex
-  // 4. the first batch is not missed
-  if (done_) return false;
-  while (calib_running_.load(
-      std::memory_order_acquire)) {  // wait while calibration is running
-    tensorflow::mutex_lock l(cond_mtx_);
-    cond_.wait_for(l, std::chrono::milliseconds(50));
-    if (done_) return false;
+bool TRTInt8Calibrator::setBatch(const std::unordered_map<string, void*>& data,
+                                 const cudaStream_t stream) {
+  tensorflow::mutex_lock lock(cond_mtx_);
+  while ((calib_running_ || batch_is_set_) &&
+         !done_) {  // wait while calibration is running
+    cond_.wait(lock);
   }
+  if (done_) return false;
+  CHECK(!calib_running_ && !batch_is_set_);
   VLOG(1) << "Set Batch Waiting finished";
   for (const auto it : data) {
     auto devptr = dev_buffers_.find(it.first);
@@ -65,27 +61,32 @@ bool TRTInt8Calibrator::setBatch(
 
     // TODO(aaroey): we should not use sync copy on default stream. Make sure
     // stream->ThenMemcpy() is used in future PRs.
-    auto status =
-        cudaMemcpy(d.first, it.second, d.second, cudaMemcpyDeviceToDevice);
+    // TODO(sami,aaroey): Need to figure out a way to ensure synchronization
+    // between stream, perhaps using a tensor?
+    auto status = cudaMemcpyAsync(d.first, it.second, d.second,
+                                  cudaMemcpyDeviceToDevice, stream);
     if (status != cudaSuccess) {
       LOG(FATAL) << "cudaMemcpy " << engine_name_ << " for '" << it.first
                  << "' failed with " << status;
     }
   }
-  calib_running_.store(true, std::memory_order_release);  // release builder
+
+  // TODO(Sami, aaorey): Find an alternative way!
+  cudaStreamSynchronize(
+      stream);  // we have to wait for the stream before returning!
+  batch_is_set_ = true;
   cond_.notify_all();
   return true;
 }
 
 bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
                                  int num_bindings) {
-  calib_running_.store(false, std::memory_order_release);  // wait for new batch
+  tensorflow::mutex_lock lock(cond_mtx_);
+  calib_running_ = false;
   cond_.notify_all();
-  while (!calib_running_.load(
-      std::memory_order_acquire)) {  // wait until new batch arrives
-    tensorflow::mutex_lock l(cond_mtx_);
-    cond_.wait_for(l, std::chrono::milliseconds(50));
-    if (done_) return false;
+  while ((!batch_is_set_ && !done_)) {  // wait until new batch arrives
+    cond_.wait(lock);
+
   }
   if (done_) {
     return false;
@@ -100,6 +101,8 @@ bool TRTInt8Calibrator::getBatch(void** bindings, const char** names,
 
     bindings[i] = it->second.first;
   }
+  batch_is_set_ = false;
+  calib_running_ = true;
   return true;
 }
 
@@ -107,6 +110,12 @@ const void* TRTInt8Calibrator::readCalibrationCache(std::size_t& length) {
   return nullptr;
 }
 
+void TRTInt8Calibrator::setDone() {
+  tensorflow::mutex_lock lock(cond_mtx_);
+  done_ = true;
+  cond_.notify_all();
+}
+
 void TRTInt8Calibrator::writeCalibrationCache(const void* ptr,
                                               std::size_t length) {}
 TRTInt8Calibrator::~TRTInt8Calibrator() {
@@ -115,5 +124,6 @@ TRTInt8Calibrator::~TRTInt8Calibrator() {
 
 }  // namespace tensorrt
 }  // namespace tensorflow
+
 #endif
 #endif
diff --git a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
index 8830f7efe7..d77aa2c5ab 100644
--- a/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
+++ b/tensorflow/contrib/tensorrt/resources/trt_int8_calibrator.h
@@ -24,7 +24,10 @@ limitations under the License.
 
 #if GOOGLE_CUDA
 #if GOOGLE_TENSORRT
+
+#include "cuda/include/cuda_runtime_api.h"
 #include "tensorrt/include/NvInfer.h"
+
 namespace tensorflow {
 namespace tensorrt {
 // This class provides a 1 element queue to match TFs push model to
@@ -39,8 +42,9 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   int getBatchSize() const override;
   bool getBatch(void* bindings[], const char* names[],
                 int num_bindings) override;
-  bool setBatch(const std::unordered_map<string, void*>& data);
-  void setDone() { done_ = true; }
+  bool setBatch(const std::unordered_map<string, void*>& data,
+                const cudaStream_t stream);
+  void setDone();
   const void* readCalibrationCache(std::size_t& length) override;
   void writeCalibrationCache(const void* ptr, std::size_t length) override;
   ~TRTInt8Calibrator();
@@ -55,11 +59,14 @@ struct TRTInt8Calibrator : public nvinfer1::IInt8EntropyCalibrator {
   const std::unordered_map<string, std::pair<void*, size_t>>
       dev_buffers_;  // map to keep tensorrt input buffers and sizes keyed with
                      // buffer names
-  std::atomic_bool calib_running_;
+  bool calib_running_;
+  bool batch_is_set_;
   string engine_name_;
 };
+
 }  // namespace tensorrt
 }  // namespace tensorflow
-#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
+
 #endif
 #endif
+#endif  // TENSORFLOW_CONTRIB_TENSORRT_RESOURCES_TRT_INT8_CALIBRATOR_H_
diff --git a/tensorflow/contrib/tensorrt/test/test_tftrt.py b/tensorflow/contrib/tensorrt/test/test_tftrt.py
index c78f6f2224..ad01bedd8f 100644
--- a/tensorflow/contrib/tensorrt/test/test_tftrt.py
+++ b/tensorflow/contrib/tensorrt/test/test_tftrt.py
@@ -60,6 +60,7 @@ def get_simple_graph_def():
 
 
 def run_graph(gdef, dumm_inp):
+  """Run given graphdef once."""
   gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
   ops.reset_default_graph()
   g = ops.Graph()
@@ -74,15 +75,65 @@ def run_graph(gdef, dumm_inp):
   return val
 
 
+# Use real data that is representative of the inference dataset
+# for calibration. For this test script it is random data.
+def run_calibration(gdef, dumm_inp):
+  """Run given calibration graph multiple times."""
+  gpu_options = cpb2.GPUOptions(per_process_gpu_memory_fraction=0.50)
+  ops.reset_default_graph()
+  g = ops.Graph()
+  with g.as_default():
+    inp, out = importer.import_graph_def(
+        graph_def=gdef, return_elements=["input", "output"])
+    inp = inp.outputs[0]
+    out = out.outputs[0]
+  with csess.Session(
+      config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess:
+    # run over real calibration data here, we are mimicking a calibration set of
+    # 30 different batches. Use as much calibration data as you want
+    for _ in range(30):
+      val = sess.run(out, {inp: dumm_inp})
+  return val
+
+
 if "__main__" in __name__:
   inp_dims = (100, 24, 24, 2)
   dummy_input = np.random.random_sample(inp_dims)
-  gdef = get_simple_graph_def()
+  orig_graph = get_simple_graph_def()  # use a frozen graph for inference
   # Get optimized graph
-  trt_graph = trt.create_inference_graph(gdef, ["output"], inp_dims[0])
-  o1 = run_graph(gdef, dummy_input)
+  trt_graph = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="FP32",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  o1 = run_graph(orig_graph, dummy_input)
   o2 = run_graph(trt_graph, dummy_input)
   o3 = run_graph(trt_graph, dummy_input)
   assert np.array_equal(o1, o2)
   assert np.array_equal(o3, o2)  # sanity check
+  fp16_graph = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="FP16",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  int8_calib_gdef = trt.create_inference_graph(
+      input_graph_def=orig_graph,
+      outputs=["output"],
+      max_batch_size=inp_dims[0],
+      max_workspace_size_bytes=1 << 25,
+      precision_mode="INT8",  # TRT Engine precision "FP32","FP16" or "INT8"
+      minimum_segment_size=2  # minimum number of nodes in an engine
+  )
+  o4 = run_graph(fp16_graph, dummy_input)
+  _ = run_calibration(int8_calib_gdef, dummy_input)
+  int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
+  o5 = run_graph(int8_graph, dummy_input)
+  assert np.allclose(o1, o4)
+  assert np.allclose(o1, o5)
   print("Pass")
diff --git a/tensorflow/contrib/tensorrt/trt_conversion.i b/tensorflow/contrib/tensorrt/trt_conversion.i
index d679945d56..46480e99a1 100644
--- a/tensorflow/contrib/tensorrt/trt_conversion.i
+++ b/tensorflow/contrib/tensorrt/trt_conversion.i
@@ -64,13 +64,17 @@ PyObject* pair_helper(std::pair<string, string>* in) {
 %ignoreall
 %unignore tensorflow;
 %unignore trt_convert;
+%unignore calib_convert;
 
 %{
+
 std::pair<string, string> trt_convert(
     string graph_def_string,  // The serialized GraphDef string.
     std::vector<string> output_names,
     size_t max_batch_size,
-    size_t max_workspace_size_bytes
+    size_t max_workspace_size_bytes,
+    int precision_mode,
+    int minimum_segment_size
     // Unfortunately we can't use TF_Status here since it
     // is in c/c_api and brings in a lot of other libraries
     // which in turn declare ops. These ops are included
@@ -90,16 +94,64 @@ std::pair<string, string> trt_convert(
     return std::pair<string, string>{out_status, ""};
   }
 
+  if(precision_mode < 0 || precision_mode > 2){
+    out_status = "InvalidArgument;Invalid precision_mode";
+    return std::pair<string, string>{out_status, ""};
+  }
   if (!output_names.size()) {
     out_status = "InvalidArgument;Size of the output_names vector is 0";
     return std::pair<string, string>{out_status, ""};
-    // return "";
   }
   tensorflow::GraphDef outGraph;
   tensorflow::Status conversion_status =
       tensorflow::tensorrt::convert::ConvertGraphDefToTensorRT(
           graph_def, output_names, max_batch_size, max_workspace_size_bytes,
-          &outGraph);
+          &outGraph, precision_mode, minimum_segment_size);
+  if (!conversion_status.ok()) {
+    auto retCode = (int)conversion_status.code();
+    char buff[2000];
+    snprintf(buff, 2000, "%d;%s", retCode,
+             conversion_status.error_message().c_str());
+    out_status = buff;
+    return std::pair<string, string>{out_status, ""};
+  }
+  string result;
+  if (!outGraph.SerializeToString(&result)) {
+    out_status = "InvalidArgument;Couldn't serialize output as a GraphDef";
+    return std::pair<string, string>{out_status, ""};
+  }
+  out_status = "OK;All good!";
+  return std::pair<string, string>{out_status, result};
+#else
+  // Returns FAILED_PRECONDITION.
+  return std::pair<string, string>{"9;TensorRT is not enabled!", ""};
+#endif  // GOOGLE_CUDA && GOOGLE_TENSORRT
+}
+
+std::pair<string, string> calib_convert(string graph_def_string  //  const tensorflow::GraphDef&
+    // unfortunately we can't use TF_Status here since it
+    // is in c/c_api and brings in a lot of other libraries
+    // which in turn declare ops. These ops are included
+    // statically in our library and cause an abort when
+    // module is loaded due to double registration
+    // until Tensorflow properly exposes these headers
+    // we have to work around this by returning a string
+    // and converting it to exception on python side.
+    //,TF_Status* out_status) {
+) {
+#if GOOGLE_CUDA && GOOGLE_TENSORRT
+  string out_status;
+
+  tensorflow::GraphDef graph_def;
+  if (!graph_def.ParseFromString(graph_def_string)) {
+    out_status = "InvalidArgument;Couldn't interpret input as a GraphDef";
+    return std::pair<string, string>{out_status, ""};
+  }
+
+  tensorflow::GraphDef outGraph;
+  tensorflow::Status conversion_status =
+      tensorflow::tensorrt::convert::ConvertCalibGraphToInferGraph(graph_def,
+                                                                   &outGraph);
   if (!conversion_status.ok()) {
     auto retCode = (int)conversion_status.code();
     char buff[2000];
@@ -122,10 +174,13 @@ std::pair<string, string> trt_convert(
 }
 %}
 
+std::pair<string, string> calib_convert(string graph_def_string);
+
 std::pair<string, string> trt_convert(string graph_def_string,
                                       std::vector<string> output_names,
                                       size_t max_batch_size,
-                                      size_t max_workspace_size_bytes);
+                                      size_t max_workspace_size_bytes,
+                                      int precision_mode, int minimum_segment_size);
 
 
 %unignoreall
diff --git a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
index cc32a26528..72d37f774c 100644
--- a/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
+++ b/tensorflow/contrib/tpu/ops/tpu_embedding_ops.cc
@@ -50,7 +50,7 @@ namespace tensorflow {
 // TPU Embeddings use dedicated ops to enforce Host/TPU consistency in the
 // state of embedding table variables. Before beginning training or inference,
 // the model must Load the optimizer parameters into the TPU memories. Before
-// saving a checkpoint, the model must Retreieve the parameters back into the
+// saving a checkpoint, the model must Retrieve the parameters back into the
 // host CPU memory.
 
 REGISTER_OP("TPUEmbeddingLoadGradientDescentParameters")
@@ -263,7 +263,7 @@ REGISTER_OP("TPUEmbeddingReceiveActivations")
     .SetIsStateful()
     .SetShapeFn(tpu_embedding_config_util::ActivationShapes)
     .Doc(R"doc(
-An op that receives embeddng activations on the TPU.
+An op that receives embedding activations on the TPU.
 
 The TPU system performs the embedding lookups and aggregations specified by
 the arguments to TPUEmbeddingEnqueueSparseBatch. The results of these
@@ -293,7 +293,7 @@ REGISTER_OP("TPUEmbeddingActivations")
 An op enabling differentiation of TPU Embeddings.
 
 This op simply returns its first input, which is assumed to have been sliced
-from the Tensors returnd by TPUEmbeddingDequeueActivations. The presence of this
+from the Tensors returned by TPUEmbeddingDequeueActivations. The presence of this
 op, and its first argument being a trainable Variable, enables automatic
 differentiation of graphs containing embeddings via the TPU Embedding Python
 libraries.
diff --git a/tensorflow/contrib/tpu/python/tpu/device_assignment.py b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
index bdd9b88af5..726b2d248e 100644
--- a/tensorflow/contrib/tpu/python/tpu/device_assignment.py
+++ b/tensorflow/contrib/tpu/python/tpu/device_assignment.py
@@ -191,9 +191,9 @@ class DeviceAssignment(object):
       logical_core: A tuple of three integers which represents a logical core.
     Returns:
       A sorted list of the replicas that are attached to that task and
-      loical_core.
+      logical_core.
     Raises:
-      ValueError: If no replica exisis in the task which contains the logical
+      ValueError: If no replica exists in the task which contains the logical
       core.
     """
     try:
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 009326e3d0..38b5ea2310 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -161,7 +161,7 @@ class RunConfig(run_config_lib.RunConfig):
     self._tpu_config = tpu_config or TPUConfig()
     self._cluster = cluster
 
-    # If user sets master and/or evaluation_master explicilty, including empty
+    # If user sets master and/or evaluation_master explicitly, including empty
     # string '', take it. Otherwise, take the values set by parent class.
     if master is not None:
       if cluster is not None:
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index c5c46ea741..3bac2db77e 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -39,7 +39,7 @@ class _TPUContext(object):
 
   This immutable object holds TPUEstimator config, train/eval batch size, and
   `TPUEstimator.use_tpu`, which is expected to be passed around. It also
-  provides utility functions, basded on the current state, to determine other
+  provides utility functions, based on the current state, to determine other
   information commonly required by TPU computation, such as TPU device names,
   TPU hosts, shard batch size, etc.
 
@@ -218,7 +218,7 @@ class _TPUContext(object):
         model, when mode == PREDICT. Only with this bool, we could
         tell whether user is calling the Estimator.predict or
         Estimator.export_savedmodel, which are running on TPU and CPU
-        respectively. Parent class Estimator does not distingush these two.
+        respectively. Parent class Estimator does not distinguish these two.
 
     Returns:
       bool, whether current input_fn or model_fn should be running on CPU.
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index f61f6bb52e..4354735744 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -137,7 +137,7 @@ def _increase_eval_step_op(iterations_per_loop):
   """Returns an op to increase the eval step for TPU evaluation.
 
   Args:
-    iterations_per_loop: Tensor. The number of eval steps runnining in TPU
+    iterations_per_loop: Tensor. The number of eval steps running in TPU
         system before returning to CPU host for each `Session.run`.
 
   Returns:
@@ -609,17 +609,17 @@ class _StoppingPredictHook(session_run_hook.SessionRunHook):
       # batch. And we append one more batch to signal the system it should stop.
       # The data flow might look like
       #
-      #  batch   0: images, labels, stop = 0  (user provideded)
-      #  batch   1: images, labels, stop = 0  (user provideded)
+      #  batch   0: images, labels, stop = 0  (user provided)
+      #  batch   1: images, labels, stop = 0  (user provided)
       #  ...
-      #  batch  99: images, labels, stop = 0  (user provideded)
+      #  batch  99: images, labels, stop = 0  (user provided)
       #  batch 100: images, labels, stop = 1  (TPUEstimator appended)
       #
       # where the final batch (id = 100) is appended by TPUEstimator, so we
       # should drop it before returning the predictions to user.
       # To achieve that, we throw the OutOfRangeError in after_run. Once
       # Monitored Session sees this error in SessionRunHook.after_run, the
-      # "current" prediciton, i.e., batch with id=100, will be discarded
+      # "current" prediction, i.e., batch with id=100, will be discarded
       # immediately
       raise errors.OutOfRangeError(None, None, 'Stopped by stopping signal.')
 
@@ -758,7 +758,7 @@ class _InputPipeline(object):
   2. (features, labels)
 
   Internally, form 1 is reformed to `(features, None)` as features and labels
-  are passed separatedly to underlying methods. For TPU training, TPUEstimator
+  are passed separately to underlying methods. For TPU training, TPUEstimator
   may expect multiple `features` and `labels` tuples one for each core.
 
   TPUEstimator allows various different structures for inputs (namely `features`
diff --git a/tensorflow/contrib/tpu/python/tpu/training_loop.py b/tensorflow/contrib/tpu/python/tpu/training_loop.py
index 3d7896127a..82a75d0255 100644
--- a/tensorflow/contrib/tpu/python/tpu/training_loop.py
+++ b/tensorflow/contrib/tpu/python/tpu/training_loop.py
@@ -170,7 +170,7 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None):
 
 
 def repeat(n, body, inputs=None, infeed_queue=None, name=None):
-  """Builds a training loop that executes a fixed number of interations.
+  """Builds a training loop that executes a fixed number of iterations.
 
   The set of loop-carried tensors correspond to `inputs`.
   `body` must be a function that takes and returns the values of the
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 42d222ff6b..a14eeed1a5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3338,6 +3338,10 @@ tf_cc_test(
     size = "small",
     srcs = ["common_runtime/function_test.cc"],
     linkstatic = tf_kernel_tests_linkstatic(),
+    tags = [
+        "manual",
+        "no_oss",
+    ],
     deps = [
         ":core",
         ":core_cpu",
diff --git a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
index 51d63eeb56..7be9a958ab 100644
--- a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEig.pbtxt
@@ -19,6 +19,7 @@ form square matrices, with the same constraints as the single matrix
 SelfAdjointEig.
 
 The result is a [..., M+1, M] matrix with [..., 0,:] containing the
-eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
+eigenvalues, and subsequent [...,1:, :] containing the eigenvectors. The eigenvalues
+are sorted in non-decreasing order.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
index 4a5e125258..fae9e84fc8 100644
--- a/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_SelfAdjointEigV2.pbtxt
@@ -31,7 +31,8 @@ END
   summary: "Computes the eigen decomposition of one or more square self-adjoint matrices."
   description: <<END
 Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-`input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+`input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`. The eigenvalues
+are sorted in non-decreasing order.
 
 ```python
 # a is a tensor.
diff --git a/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
new file mode 100644
index 0000000000..9fabe7863e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_SlideDataset.pbtxt
@@ -0,0 +1,18 @@
+op {
+  graph_op_name: "SlideDataset"
+  in_arg {
+    name: "window_size"
+    description: <<END
+A scalar representing the number of elements in the
+sliding window.
+END
+  }
+  in_arg {
+    name: "stride"
+    description: <<END
+A scalar representing the steps moving the sliding window
+forward in one iteration. It must be in `[1, window_size)`.
+END
+  }
+  summary: "Creates a dataset that passes a sliding window over `input_dataset`."
+}
diff --git a/tensorflow/core/distributed_runtime/tensor_coding.cc b/tensorflow/core/distributed_runtime/tensor_coding.cc
index 34a4013547..fe2d1a1293 100644
--- a/tensorflow/core/distributed_runtime/tensor_coding.cc
+++ b/tensorflow/core/distributed_runtime/tensor_coding.cc
@@ -81,7 +81,7 @@ void TensorResponse::InitPartial(const RecvTensorResponse& response) {
 Status TensorResponse::ParseFrom(Source* source) {
   if (!on_host_) {
     protobuf::io::CodedInputStream input(source->contents());
-    input.SetTotalBytesLimit(INT_MAX);  // Unlimited
+    input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
 
     // Pre-parse into local storage, then delegate to device.
     if (!meta_.ParseFromCodedStream(&input) || !input.ConsumedEntireMessage()) {
@@ -217,7 +217,7 @@ bool TensorResponse::ParseTensorSubmessage(
 
 bool TensorResponse::ParseFast(Source* source) {
   protobuf::io::CodedInputStream input(source->contents());
-  input.SetTotalBytesLimit(INT_MAX);  // Unlimited
+  input.SetTotalBytesLimit(INT_MAX, INT_MAX);  // Unlimited
   while (true) {
     auto p = input.ReadTagWithCutoff(127);
     int tag = GetTagFieldNumber(p.first);
diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 02038c5d77..1507b6eae2 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -2492,10 +2492,10 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
                       mkl_op_registry::GetMklOpName(csinfo_.identity),
                       CopyAttrsDataType, AlwaysRewrite});
     rinfo_.push_back({csinfo_.lrn, mkl_op_registry::GetMklOpName(csinfo_.lrn),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.lrn_grad,
                       mkl_op_registry::GetMklOpName(csinfo_.lrn_grad),
-                      CopyAttrsLRN, AlwaysRewrite});
+                      CopyAttrsLRN, LrnRewrite});
     rinfo_.push_back({csinfo_.max_pool,
                       mkl_op_registry::GetMklOpName(csinfo_.max_pool),
                       CopyAttrsPooling, NonDepthBatchWisePoolRewrite});
@@ -2865,6 +2865,28 @@ class MklLayoutRewritePass : public GraphOptimizationPass {
     return false;
   }
 
+  // If the depth_radius of LRN is not 2, then MKL DNN takes unoptimized 
+  // path. The unoptimized path is slow. Thus we dont rewrite the node 
+  // and use default Eigen. But for depth_radius=2, MKL DNN optimized 
+  // path is taken, i.e., eigen node is rewritten by MKl DNN node.
+  static bool LrnRewrite(const Node* n) {
+    CHECK_NOTNULL(n);
+
+    int depth_radius;
+    CHECK_EQ(GetNodeAttr(n->def(), "depth_radius", &depth_radius).ok(), true);
+
+    // if the depth_radius of LRN is not 2, don't rewrite the node by MKL DNN
+    // and use eigen node instead 
+    if (depth_radius == 2) {
+      return true;
+    }
+    VLOG(1) << "LrnRewrite: The model sets depth_radius as not 2 which"
+            << "case is not optimized by Intel MKL, thus using Eigen op"
+            << "for LRN " ; 
+
+    return false;
+  }
+
   static bool AddNRewrite(const Node* n) {
     CHECK_NOTNULL(n);
 
@@ -3528,11 +3550,13 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   string data_format;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   bool use_cudnn_on_gpu;
 
   // Get all attributes from old node.
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "T", &T));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(orig_node->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(orig_node->def(), "data_format", &data_format));
   TF_CHECK_OK(
@@ -3541,6 +3565,7 @@ void MklLayoutRewritePass::CopyAttrsConv2D(const Node* orig_node,
   // Add attributes to new node.
   nb->Attr("T", T);
   nb->Attr("strides", strides);
+  nb->Attr("dilations", dilations);
   nb->Attr("padding", padding);
   nb->Attr("data_format", data_format);
   nb->Attr("use_cudnn_on_gpu", use_cudnn_on_gpu);
@@ -3778,12 +3803,14 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
   DataType T_pred, T_succ;
   string padding;
   std::vector<int32> strides;
+  std::vector<int32> dilations;
   string data_format_pred, data_format_succ;
   bool use_cudnn_on_gnu;
   TF_CHECK_OK(GetNodeAttr(pred->def(), "T", &T_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "T", &T_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "padding", &padding));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "strides", &strides));
+  TF_CHECK_OK(GetNodeAttr(pred->def(), "dilations", &dilations));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "data_format", &data_format_pred));
   TF_CHECK_OK(GetNodeAttr(succ->def(), "data_format", &data_format_succ));
   TF_CHECK_OK(GetNodeAttr(pred->def(), "use_cudnn_on_gpu", &use_cudnn_on_gnu));
diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index f78036d78c..bd0d94b83f 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -413,7 +414,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
       frame_children_[frame_ids[0]].insert(frame_ids[1]);
       frame_parent_[frame_ids.back()] = frame_ids[frame_ids.size() - 2];
     }
-    if (!frame_ids.empty()) {
+    if (frame_ids.size() >= 1) {
       frame_children_.insert(std::make_pair(frame_ids.back(), empty_set_));
       if (node->op() == "LoopCond") {
         if (loop_cond_.count(frame_ids.back())) {
@@ -432,7 +433,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
   }
 
   for (auto it = frame_children_.begin(); it != frame_children_.end(); ++it) {
-    if (it->second.empty()) {
+    if (it->second.size() == 0) {
       worklist.push_back(it->first);
     }
   }
@@ -445,7 +446,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
     if (parent_it != frame_parent_.end()) {
       int parent_id = parent_it->second;
       frame_children_[parent_id].erase(frame_id);
-      if (frame_children_[parent_id].empty()) {
+      if (frame_children_[parent_id].size() == 0) {
         worklist.push_back(parent_id);
       }
     }
@@ -468,6 +469,7 @@ Status LoopOptimizer::LoopInvariantNodeMotion() {
 
 Status LoopOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                GraphDef* optimized_graph) {
+
   TF_RETURN_IF_ERROR(RemoveStackOps(item, optimized_graph));
 
   if (opt_level_ == RewriterConfig::AGGRESSIVE) {
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 48d5955ad1..2e39f25fc1 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5157,7 +5157,6 @@ tf_kernel_library(
     ],
     hdrs = [
         "meta_support.h",
-        "quantization_utils.h",
         "reference_gemm.h",
     ],
     deps = [
@@ -5232,6 +5231,7 @@ tf_cc_test(
     name = "quantization_utils_test",
     srcs = ["quantization_utils_test.cc"],
     deps = [
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5294,6 +5294,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5355,6 +5356,7 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5377,6 +5379,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5441,6 +5444,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5461,6 +5465,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5500,6 +5505,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5556,6 +5562,7 @@ tf_cc_test(
         ":math",
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:client_session",
@@ -5578,6 +5585,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:framework",
@@ -5614,6 +5622,7 @@ tf_cc_test(
     deps = [
         ":ops_testutil",
         ":ops_util",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu",
@@ -5635,6 +5644,7 @@ tf_cc_test(
     deps = [
         ":batch_norm_op",
         ":ops_testutil",
+        ":quantization_utils",
         ":quantized_ops",
         "//tensorflow/core:array_ops_op_lib",
         "//tensorflow/core:core_cpu_internal",
diff --git a/tensorflow/core/kernels/concat_op.cc b/tensorflow/core/kernels/concat_op.cc
index 7011550f7e..f16766315f 100644
--- a/tensorflow/core/kernels/concat_op.cc
+++ b/tensorflow/core/kernels/concat_op.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <limits>
 #include <vector>
 
-#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
@@ -28,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/concat_lib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 
 namespace tensorflow {
 
@@ -53,17 +53,38 @@ class ConcatBaseOp : public OpKernel {
   void Compute(OpKernelContext* c) override {
     const Tensor* concat_dim_tensor;
     const char* axis_attribute_name =
-        AxisArgName == NAME_IS_AXIS
-            ? "axis"
-            : AxisArgName == NAME_IS_CONCAT_DIM ? "concat_dim" : "<invalid>";
+        AxisArgName == NAME_IS_AXIS ? "axis" : AxisArgName == NAME_IS_CONCAT_DIM
+                                                   ? "concat_dim"
+                                                   : "<invalid>";
     OP_REQUIRES_OK(c, c->input(axis_attribute_name, &concat_dim_tensor));
     OP_REQUIRES(c, IsLegacyScalar(concat_dim_tensor->shape()),
                 errors::InvalidArgument(
                     axis_attribute_name,
                     " tensor should be a scalar integer, but got shape ",
                     concat_dim_tensor->shape().DebugString()));
-    const int32 concat_dim =
-        internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
+    int64 concat_dim;
+    // In case of ConcatV2, "axis" could be int32 or int64
+    if (AxisArgName == NAME_IS_AXIS) {
+      OP_REQUIRES(
+          c, (concat_dim_tensor->dtype() == DT_INT32 ||
+              concat_dim_tensor->dtype() == DT_INT64),
+          errors::InvalidArgument(axis_attribute_name,
+                                  " tensor should be int32 or int64, but got ",
+                                  concat_dim_tensor->dtype()));
+    } else {
+      OP_REQUIRES(c, (concat_dim_tensor->dtype() == DT_INT32),
+                  errors::InvalidArgument(axis_attribute_name,
+                                          " tensor should be int32, but got ",
+                                          concat_dim_tensor->dtype()));
+    }
+    if (concat_dim_tensor->dtype() == DT_INT32) {
+      concat_dim =
+          internal::SubtleMustCopy(concat_dim_tensor->scalar<int32>()());
+    } else {
+      concat_dim =
+          internal::SubtleMustCopy(concat_dim_tensor->scalar<int64>()());
+    }
+
     OpInputList values;
     OP_REQUIRES_OK(c, c->input_list("values", &values));
     const int N = values.size();
@@ -154,17 +175,16 @@ using ConcatOp = ConcatBaseOp<Device, T, NAME_IS_CONCAT_DIM>;
 template <typename Device, typename T>
 using ConcatV2Op = ConcatBaseOp<Device, T, NAME_IS_AXIS>;
 
-#define REGISTER_CONCAT(type)                                \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_CPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<CPUDevice, type>)         \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_CPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
-                              .HostMemory("axis"),           \
+#define REGISTER_CONCAT(type)                            \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<CPUDevice, type>)     \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_CPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<CPUDevice, type>)
 
 TF_CALL_POD_STRING_TYPES(REGISTER_CONCAT);
@@ -178,17 +198,16 @@ REGISTER_CONCAT(qint32);
 
 #if GOOGLE_CUDA
 
-#define REGISTER_GPU(type)                                   \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_GPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<GPUDevice, type>)         \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_GPU)            \
-                              .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
-                              .HostMemory("axis"),           \
+#define REGISTER_GPU(type)                               \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_GPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<GPUDevice, type>)     \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_GPU)        \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<GPUDevice, type>)
 
 TF_CALL_GPU_NUMBER_TYPES(REGISTER_GPU);
@@ -212,7 +231,6 @@ REGISTER_KERNEL_BUILDER(Name("Concat")
 REGISTER_KERNEL_BUILDER(Name("ConcatV2")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tidx")
                             .HostMemory("values")
                             .HostMemory("axis")
                             .HostMemory("output"),
@@ -221,17 +239,16 @@ REGISTER_KERNEL_BUILDER(Name("ConcatV2")
 #endif  // GOOGLE_CUDA
 
 #ifdef TENSORFLOW_USE_SYCL
-#define REGISTER_SYCL(type)                                  \
-  REGISTER_KERNEL_BUILDER(Name("Concat")                     \
-                              .Device(DEVICE_SYCL)           \
-                              .TypeConstraint<type>("T")     \
-                              .HostMemory("concat_dim"),     \
-                          ConcatOp<SYCLDevice, type>)        \
-  REGISTER_KERNEL_BUILDER(Name("ConcatV2")                   \
-                              .Device(DEVICE_SYCL)           \
-                              .TypeConstraint<type>("T")     \
-                              .TypeConstraint<int32>("Tidx") \
-                              .HostMemory("axis"),           \
+#define REGISTER_SYCL(type)                              \
+  REGISTER_KERNEL_BUILDER(Name("Concat")                 \
+                              .Device(DEVICE_SYCL)       \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("concat_dim"), \
+                          ConcatOp<SYCLDevice, type>)    \
+  REGISTER_KERNEL_BUILDER(Name("ConcatV2")               \
+                              .Device(DEVICE_SYCL)       \
+                              .TypeConstraint<type>("T") \
+                              .HostMemory("axis"),       \
                           ConcatV2Op<SYCLDevice, type>)
 
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SYCL);
@@ -246,7 +263,6 @@ REGISTER_KERNEL_BUILDER(Name("Concat")
 REGISTER_KERNEL_BUILDER(Name("ConcatV2")
                             .Device(DEVICE_SYCL)
                             .TypeConstraint<int32>("T")
-                            .TypeConstraint<int32>("Tidx")
                             .HostMemory("values")
                             .HostMemory("axis")
                             .HostMemory("output"),
diff --git a/tensorflow/core/kernels/conv_ops_test.cc b/tensorflow/core/kernels/conv_ops_test.cc
index 666bca265c..e2e166c02f 100644
--- a/tensorflow/core/kernels/conv_ops_test.cc
+++ b/tensorflow/core/kernels/conv_ops_test.cc
@@ -401,7 +401,7 @@ class ConvOpTest : public OpsTestBase {
     // (1*0)+(4*5)+(7*6)+(2*0)+(5*9)+(8*10)+(3*0)+(6*0)+(9*0)=187
     // (1*5)+(4*6)+(7*7)+(2*9)+(5*10)+(8*11)+(3*0)+(6*0)+(9*0)=234
     // (1*6)+(4*7)+(7*8)+(2*10)+(5*11)+(8*12)+(3*0)+(6*0)+(9*0)=261
-    // (1*7)+(4*11)+(7*0)+(2*8)+(5*12)+(8*0)+(3*0)+(6*0)+(9*0)=121
+    // (1*7)+(4*8)+(7*0)+(2*11)+(5*12)+(8*0)+(3*0)+(6*0)+(9*0)=121
     // This means we should end up with this matrix:
     // |  105  |  150  |  183  |   95  |
     // |  235  |  312  |  357  |  178  |
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 484d4f88d6..01754ec21a 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -113,6 +113,19 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "slide_dataset_op",
+    srcs = ["slide_dataset_op.cc"],
+    deps = [
+        ":dataset",
+        "//tensorflow/core:dataset_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core/kernels:batch_util",
+    ],
+)
+
 tf_kernel_library(
     name = "padded_batch_dataset_op",
     srcs = ["padded_batch_dataset_op.cc"],
@@ -538,6 +551,7 @@ tf_kernel_library(
         ":scan_dataset_op",
         ":shuffle_dataset_op",
         ":skip_dataset_op",
+        ":slide_dataset_op",
         ":sparse_tensor_slice_dataset_op",
         ":sql_dataset_ops",
         ":stats_aggregator_ops",
diff --git a/tensorflow/core/kernels/data/slide_dataset_op.cc b/tensorflow/core/kernels/data/slide_dataset_op.cc
new file mode 100644
index 0000000000..4f3537b691
--- /dev/null
+++ b/tensorflow/core/kernels/data/slide_dataset_op.cc
@@ -0,0 +1,252 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/partial_tensor_shape.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/kernels/batch_util.h"
+#include "tensorflow/core/kernels/data/dataset.h"
+
+namespace tensorflow {
+
+namespace {
+
+// See documentation in ../ops/dataset_ops.cc for a high-level
+// description of the following op.
+
+class SlideDatasetOp : public UnaryDatasetOpKernel {
+ public:
+  explicit SlideDatasetOp(OpKernelConstruction* ctx)
+      : UnaryDatasetOpKernel(ctx) {}
+
+  void MakeDataset(OpKernelContext* ctx, DatasetBase* input,
+                   DatasetBase** output) override {
+    int64 window_size = 0;
+    int64 stride = 1;
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "window_size", &window_size));
+    OP_REQUIRES_OK(ctx,
+                   ParseScalarArgument<int64>(ctx, "stride", &stride));
+    OP_REQUIRES(
+        ctx, window_size > 0,
+        errors::InvalidArgument("Window size must be greater than zero."));
+    OP_REQUIRES(
+        ctx, stride > 0 && stride < window_size,
+        errors::InvalidArgument("Stride must be in [1, window_size)."));
+
+    *output = new Dataset(ctx, window_size, stride, input);
+  }
+
+ private:
+  class Dataset : public GraphDatasetBase {
+   public:
+    Dataset(OpKernelContext* ctx, int64 window_size, int64 stride, const DatasetBase* input)
+        : GraphDatasetBase(ctx), window_size_(window_size), stride_(stride), input_(input) {
+      input_->Ref();
+
+      const auto& input_shapes = input_->output_shapes();
+      output_shapes_.reserve(input_shapes.size());
+      for (const auto& input_shape : input_shapes) {
+        output_shapes_.emplace_back(
+            PartialTensorShape({-1}).Concatenate(input_shape));
+      }
+    }
+
+    ~Dataset() override { input_->Unref(); }
+
+    std::unique_ptr<IteratorBase> MakeIterator(
+        const string& prefix) const override {
+      return std::unique_ptr<IteratorBase>(new Iterator(
+          Iterator::Params{this, strings::StrCat(prefix, "::Slide")}));
+    }
+
+    const DataTypeVector& output_dtypes() const override {
+      return input_->output_dtypes();
+    }
+
+    const std::vector<PartialTensorShape>& output_shapes() const override {
+      return output_shapes_;
+    }
+
+    string DebugString() override {
+      return strings::StrCat("SlideDatasetOp(", window_size_, ", ", stride_, ")::Dataset");
+    }
+
+   protected:
+    Status AsGraphDefInternal(OpKernelContext* ctx, DatasetGraphDefBuilder* b,
+                              Node** output) const override {
+      Node* input_graph_node = nullptr;
+      TF_RETURN_IF_ERROR(b->AddParentDataset(ctx, input_, &input_graph_node));
+      Node* window_size = nullptr;
+      Node* stride = nullptr;
+      TF_RETURN_IF_ERROR(b->AddScalar(window_size_, &window_size));
+      TF_RETURN_IF_ERROR(b->AddScalar(stride_, &stride));
+      TF_RETURN_IF_ERROR(
+          b->AddDataset(this, {input_graph_node, window_size, stride}, output));
+      return Status::OK();
+    }
+
+   private:
+
+    class Iterator : public DatasetIterator<Dataset> {
+     public:
+      explicit Iterator(const Params& params)
+          : DatasetIterator<Dataset>(params),
+            input_impl_(params.dataset->input_->MakeIterator(params.prefix)) {}
+
+      Status GetNextInternal(IteratorContext* ctx,
+                             std::vector<Tensor>* out_tensors,
+                             bool* end_of_sequence) override {
+        const int64 window_size = dataset()->window_size_;
+        const int64 stride = dataset()->stride_;
+        std::vector<std::vector<Tensor>> batch_elements;
+        {
+          mutex_lock l(mu_);
+          if (!input_impl_) {
+            *end_of_sequence = true;
+            return Status::OK();
+          }
+          batch_elements.reserve(window_size);
+          const bool first_call = cache_.empty();
+          if (first_call) {
+            cache_.reserve(window_size);
+          } else {
+            // Reuse cache in the previous iteration.
+            cache_.swap(batch_elements);
+          }
+          // Fill up with new elements.
+          *end_of_sequence = false;
+          for (size_t i = batch_elements.size(); i < window_size && !*end_of_sequence;
+              ++i) {
+            std::vector<Tensor> batch_element_tuple;
+            TF_RETURN_IF_ERROR(input_impl_->GetNext(ctx, &batch_element_tuple,
+                                                    end_of_sequence));
+            if (!*end_of_sequence) {
+              batch_elements.push_back(std::move(batch_element_tuple));
+            } else {
+              input_impl_.reset();
+            }
+          }
+          // Drop the final smaller blocks.
+          if (batch_elements.size() < window_size) {
+            DCHECK(*end_of_sequence);
+            return Status::OK();
+          }
+          // Cache the data used for the next iteration.
+          for (size_t i = stride; i < window_size; ++i) {
+            cache_.emplace_back(batch_elements[i]);
+          }
+        }
+
+        // Construct output tensors.
+        // Those codes below are copied from batch_dataset_op.cc.
+        const size_t num_tuple_components = batch_elements[0].size();
+        const int64 num_batch_elements = batch_elements.size();
+        for (size_t component_index = 0; component_index < num_tuple_components;
+             ++component_index) {
+          const Tensor& first_element = batch_elements[0][component_index];
+          TensorShape batch_component_shape({num_batch_elements});
+          batch_component_shape.AppendShape(first_element.shape());
+          Tensor batch_component(cpu_allocator(), first_element.dtype(),
+                                 batch_component_shape);
+          // Build the output tuple component by copying one slice
+          // from each input element in the batch.
+          for (size_t i = 0; i < num_batch_elements; ++i) {
+            if (batch_elements[i][component_index].shape() !=
+                first_element.shape()) {
+              return errors::InvalidArgument(
+                  "Cannot batch tensors with different shapes in component ",
+                  component_index, ". First element had shape ",
+                  first_element.shape().DebugString(), " and element ", i,
+                  " had shape ",
+                  batch_elements[i][component_index].shape().DebugString(),
+                  ".");
+            }
+            TF_RETURN_IF_ERROR(batch_util::CopyElementToSlice(
+                std::move(batch_elements[i][component_index]), &batch_component,
+                i));
+          }
+          out_tensors->emplace_back(std::move(batch_component));
+        }
+        *end_of_sequence = false;
+        return Status::OK();
+      }
+
+     protected:
+      Status SaveInternal(IteratorStateWriter* writer) override {
+        mutex_lock l(mu_);
+        if (!input_impl_) {
+          TF_RETURN_IF_ERROR(
+              writer->WriteScalar(full_name("input_impl_empty"), ""));
+        } else {
+          TF_RETURN_IF_ERROR(SaveParent(writer, input_impl_));
+        }
+        // Save cache.
+        TF_RETURN_IF_ERROR(
+            writer->WriteScalar(strings::StrCat("cache_size"), cache_.size()));
+        for (int64 i = 0; i < cache_.size(); i++) {
+          TF_RETURN_IF_ERROR(writer->WriteScalar(
+              strings::StrCat("cache[", i, "]_size"), cache_[i].size()));
+          for (int64 j = 0; j < cache_[i].size(); j++) {
+            TF_RETURN_IF_ERROR(writer->WriteTensor(
+                strings::StrCat("cache[", i, "][", j, "]"), cache_[i][j]));
+          }
+        }
+        return Status::OK();
+      }
+
+      Status RestoreInternal(IteratorContext* ctx,
+                             IteratorStateReader* reader) override {
+        mutex_lock l(mu_);
+        if (!reader->Contains(full_name("input_impl_empty"))) {
+          TF_RETURN_IF_ERROR(RestoreParent(ctx, reader, input_impl_));
+        } else {
+          input_impl_.reset();
+        }
+        // Restore cache.
+        int64 cache_size;
+        TF_RETURN_IF_ERROR(
+            reader->ReadScalar(strings::StrCat("cache_size"), &cache_size));
+        cache_.resize(cache_size);
+        for (int64 i = 0; i < cache_size; i++) {
+          int64 vector_size;
+          TF_RETURN_IF_ERROR(reader->ReadScalar(
+              strings::StrCat("cache[", i, "]_size"), &vector_size));
+          cache_[i].resize(vector_size);
+          for (int64 j = 0; j < vector_size; j++) {
+            TF_RETURN_IF_ERROR(reader->ReadTensor(
+                strings::StrCat("cache[", i, "][", j, "]"), &cache_[i][j]));
+          }
+        }
+        return Status::OK();
+      }
+
+     private:
+      mutex mu_;
+      std::vector<std::vector<Tensor>> cache_ GUARDED_BY(mu_);
+      std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(mu_);
+    };
+
+    const int64 window_size_;
+    const int64 stride_;
+    const DatasetBase* const input_;
+    std::vector<PartialTensorShape> output_shapes_;
+  };
+};
+
+REGISTER_KERNEL_BUILDER(Name("SlideDataset").Device(DEVICE_CPU),
+                        SlideDatasetOp);
+
+}  // namespace
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/depthtospace_op.cc b/tensorflow/core/kernels/depthtospace_op.cc
index 39aa3e9eb0..b74a09e2cb 100644
--- a/tensorflow/core/kernels/depthtospace_op.cc
+++ b/tensorflow/core/kernels/depthtospace_op.cc
@@ -187,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     DepthToSpaceOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
+    DepthToSpaceOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("DepthToSpace").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     DepthToSpaceOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
index 184c703599..0656081177 100644
--- a/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/depthtospace_op_gpu.cu.cc
@@ -238,6 +238,12 @@ struct DepthToSpaceOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// Instantiate the GPU implementations for Eigen::half.
+template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NCHW>;
+template struct functor::DepthToSpaceOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NHWC>;
+
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::DepthToSpaceOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 108d59db2c..7688305019 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -45,6 +45,7 @@ tf_cc_test(
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
         "//tensorflow/core/kernels:cwise_op",
+        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/kernels:reduction_ops",
         "//tensorflow/core/kernels:remote_fused_graph_execute_utils",
diff --git a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
index 1401bc65a4..e0706568b1 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_filter_ops.cc
@@ -444,6 +444,7 @@ class MklConv2DCustomBackpropFilterOp
   ~MklConv2DCustomBackpropFilterOp() {}
 
  private:
+  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -492,7 +493,9 @@ class MklConv2DCustomBackpropFilterOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor, const memory::dims& strides,
+                       Tensor** output_tensor,
+                       const memory::dims& strides,
+                       const memory::dims& dilations,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -518,31 +521,32 @@ class MklConv2DCustomBackpropFilterOp
       bias_grad->SetOpMemDesc(bias_grad_dims, memory::format::x);
     }
 
-    // Create convolution backward weights primitive.
-    auto bwd_desc =
-        (biasEnabled && (bias_grad != nullptr))
-            ? convolution_backward_weights::desc(
-                  convolution_direct, input->GetOpMemDesc(),
-                  output->GetOpMemDesc(), bias_grad->GetOpMemDesc(),
-                  outbackprop->GetOpMemDesc(), strides, padding_l, padding_r,
-                  padding)
-            : convolution_backward_weights::desc(
-                  convolution_direct, input->GetOpMemDesc(),
-                  output->GetOpMemDesc(), outbackprop->GetOpMemDesc(), strides,
-                  padding_l, padding_r, padding);
-
-    auto bwd_pd = convolution_backward_weights::primitive_desc(
-        bwd_desc, cpu_engine, conv_fwd_pd);
-
-    // Allocate output tensor.
-    AllocateOutputTensor(context, bwd_pd, bwd_output_dims, bwd_output_format,
-                         output_tensor);
-
-    CHECK_NOTNULL(*output_tensor);
-    // Set buffer handle using allocated output tensor.
-    output->SetUsrMemDataHandle(*output_tensor);
-
     if (biasEnabled && (bias_grad != nullptr)) {
+      // Create convolution backward weights with bias primitive.
+      // Use dilated convolution in case dilate rates are greater than zero.
+      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  bias_grad->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(), strides,
+                                  dilations, padding_l, padding_r, padding) :
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  bias_grad->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(),
+                                  strides, padding_l, padding_r, padding);
+      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
+                                                            cpu_engine,
+                                                            conv_fwd_pd);
+
+      // Allocate output tensor.
+      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
+                           bwd_output_format, output_tensor);
+
+      CHECK_NOTNULL(*output_tensor);
+      // Set buffer handle using allocated output tensor.
+      output->SetUsrMemDataHandle(*output_tensor);
+
       // Allocate bias_grad tensor
       TensorShape bias_grad_shape({depth});
       Tensor* bias_grad_tensor = nullptr;
@@ -553,11 +557,32 @@ class MklConv2DCustomBackpropFilterOp
           memory::desc({bias_grad_dims}, MklDnnType<T>(), memory::format::x);
       bias_grad->SetUsrMem(bias_grad_md, bias_grad_tensor);
       bias_grad->SetUsrMemDataHandle(bias_grad_tensor);
-    }
 
-    if (biasEnabled && (bias_grad != nullptr)) {
-      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output, bias_grad);
+      PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output,
+                                  bias_grad);
     } else {
+      // Create convolution backward weights primitive.
+      // Use dilated convolution in case dilate rates are greater than zero.
+      auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(), strides,
+                                  dilations, padding_l, padding_r, padding) :
+        convolution_backward_weights::desc(convolution_direct,
+                                  input->GetOpMemDesc(), output->GetOpMemDesc(),
+                                  outbackprop->GetOpMemDesc(),
+                                  strides, padding_l, padding_r, padding);
+      auto bwd_pd = convolution_backward_weights::primitive_desc(bwd_desc,
+                                                            cpu_engine,
+                                                            conv_fwd_pd);
+
+      // Allocate output tensor.
+      AllocateOutputTensor(context, bwd_pd, bwd_output_dims,
+                           bwd_output_format, output_tensor);
+
+      CHECK_NOTNULL(*output_tensor);
+      // Set buffer handle using allocated output tensor.
+      output->SetUsrMemDataHandle(*output_tensor);
       PrepareAndExecutePrimitive(bwd_pd, input, outbackprop, output);
     }
   }
diff --git a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
index eeed009531..d203c04934 100644
--- a/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_grad_input_ops.cc
@@ -369,6 +369,7 @@ class MklConv2DCustomBackpropInputOp
  private:
   const int kInputIndex_Filter = 1, kInputIndex_InputSizes = 0,
             kInputIndex_OutBackProp = 2;
+  const int kDilationH = 0, kDilationW = 1;
   void ValidateMklShapes(const MklDnnShape& input_mkl_shape,
                          const MklDnnShape& filter_mkl_shape,
                          const MklDnnShape& obp_mkl_shape) {
@@ -419,7 +420,9 @@ class MklConv2DCustomBackpropInputOp
                        const convolution_forward::primitive_desc& conv_fwd_pd,
                        MklDnnData<T>* input, MklDnnData<T>* filter,
                        MklDnnData<T>* outbackprop, MklDnnData<T>* output,
-                       Tensor** output_tensor, const memory::dims& strides,
+                       Tensor** output_tensor,
+                       const memory::dims& strides,
+                       const memory::dims& dilations,
                        const memory::dims& padding_l,
                        const memory::dims& padding_r, padding_kind padding,
                        const memory::dims& bwd_output_dims,
@@ -432,9 +435,16 @@ class MklConv2DCustomBackpropInputOp
     CHECK_NOTNULL(output_tensor);
 
     // Create convolution backward data primitive.
-    auto bwd_desc = convolution_backward_data::desc(
-        convolution_direct, output->GetOpMemDesc(), filter->GetOpMemDesc(),
-        outbackprop->GetOpMemDesc(), strides, padding_l, padding_r, padding);
+    // Use dilated convolution in case dilate rates are greater than zero.
+    auto bwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0) ?
+        convolution_backward_data::desc(convolution_direct,
+                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
+                      outbackprop->GetOpMemDesc(), strides,
+                      dilations, padding_l, padding_r, padding):
+        convolution_backward_data::desc(convolution_direct,
+                      output->GetOpMemDesc(), filter->GetOpMemDesc(),
+                      outbackprop->GetOpMemDesc(),
+                      strides, padding_l, padding_r, padding);
 
     auto bwd_pd = convolution_backward_data::primitive_desc(
         bwd_desc, cpu_engine, conv_fwd_pd);
diff --git a/tensorflow/core/kernels/mkl_conv_ops.cc b/tensorflow/core/kernels/mkl_conv_ops.cc
index 1440da8f82..f0818eb96d 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.cc
+++ b/tensorflow/core/kernels/mkl_conv_ops.cc
@@ -493,6 +493,7 @@ class MklConv2DOp : public OpKernel {
   ~MklConv2DOp() {}
 
   explicit MklConv2DOp(OpKernelConstruction* context) : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
     OP_REQUIRES_OK(context, context->GetAttr("strides", &strides_));
     string data_format;
     OP_REQUIRES_OK(context, context->GetAttr("data_format", &data_format));
@@ -509,6 +510,20 @@ class MklConv2DOp : public OpKernel {
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
+    OP_REQUIRES(context, dilations_.size() == 4,
+                errors::InvalidArgument("Sliding window dilations field must "
+                                        "specify 4 dimensions"));
+    const int64 dilation_n = GetTensorDim(dilations_, data_format_, 'N');
+    const int64 dilation_c = GetTensorDim(dilations_, data_format_, 'C');
+    const int64 dilation_h = GetTensorDim(dilations_, data_format_, 'H');
+    const int64 dilation_w = GetTensorDim(dilations_, data_format_, 'W');
+    OP_REQUIRES(context, dilation_n == 1 && dilation_c == 1,
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "dilations in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, dilation_h > 0 && dilation_w > 0,
+        errors::InvalidArgument("Dilated rates should be larger than 0."));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -530,17 +545,19 @@ class MklConv2DOp : public OpKernel {
       MklDnnData<T> filter(&cpu_engine);
       MklDnnData<T> output(&cpu_engine);
 
-      memory::dims src_dims, filter_dims, padding_l, padding_r, strides;
+      memory::dims src_dims, filter_dims, padding_l, padding_r,
+                   dilations, strides;
       memory::dims output_dims_tf_order, output_dims_mkl_order;
 
       // Get shapes of input tensors in MKL-DNN order
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
+                             dilations_);
       auto src_tf_shape = GetTfShape(context, kInputIndex_Src);
       auto filter_tf_shape = GetTfShape(context, kInputIndex_Filter);
       conv_utl.GetConvFwdSizesInMklOrder(
           src_tf_shape, filter_tf_shape, &src_dims, &filter_dims, &strides,
-          &output_dims_tf_order, &output_dims_mkl_order, &padding_l,
-          &padding_r);
+          &dilations, &output_dims_tf_order, &output_dims_mkl_order,
+          &padding_l, &padding_r);
       if (!context->status().ok()) return;
 
       // Check for corner case - if there is nothing to compute, return.
@@ -553,6 +570,7 @@ class MklConv2DOp : public OpKernel {
         //               Need semantics for Null MKL tensor
         MklDnnShape output_mkl_shape;
         output_mkl_shape.SetMklTensor(false);
+
         AllocateOutputSetMklShape(context, kOutputIndex_Dst, &output_tensor,
                                   src_tf_shape, output_mkl_shape);
 
@@ -596,55 +614,79 @@ class MklConv2DOp : public OpKernel {
       filter.SetOpMemDesc(filter_dims, memory::format::any);
       output.SetOpMemDesc(output_dims_mkl_order, memory::format::any);
 
-      // If bias is enabled, then do the same steps as above for bias.
+      // MKLDNN dilation starts from 0.
+      dilations[kDilationH] -= 1;
+      dilations[kDilationW] -= 1;
+
       if (biasEnabled) {
-        MklDnnData<T> bias(&cpu_engine);
-        memory::dims bias_size;
-        conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
-        const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
-        bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
-        bias.SetOpMemDesc(bias_size, memory::format::any);
-
-        // Create convolution primitive with Bias.
-        auto conv_desc = convolution_forward::desc(
-            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
-            filter.GetOpMemDesc(), bias.GetOpMemDesc(), output.GetOpMemDesc(),
-            strides, padding_l, padding_r, TFPaddingToMklDnnPadding(padding_));
-
-        auto conv_prim_desc =
-            convolution_forward::primitive_desc(conv_desc, cpu_engine);
-        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
-                             tf_fmt, &output_tensor);
-        // Set data handle for output.
-        output.SetUsrMemDataHandle(output_tensor);
-
-        Tensor* filter_out_tensor = nullptr;
-        AllocateFilterOutputTensor(context, conv_prim_desc,
-                                   TFShapeToMklDnnDims(filter_tf_shape),
-                                   &filter_out_tensor);
-
-        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
-                             filter_out_tensor);
+          // Create convolution primitive with Bias.
+          MklDnnData<T> bias(&cpu_engine);
+          memory::dims bias_size;
+          conv_utl.GetBiasSizeInMklOrder(kInputIndex_Bias, &bias_size);
+          const Tensor& bias_tensor = MklGetInput(context, kInputIndex_Bias);
+          bias.SetUsrMem(bias_size, memory::format::x, &bias_tensor);
+          bias.SetOpMemDesc(bias_size, memory::format::any);
+
+          // Create convolution primitive with Bias.
+          // Use MKLDNN dilated convolution in case of dilated rate (>0).
+          auto conv_desc = (dilations[kDilationH] > 0 ||
+              dilations[kDilationW] > 0) ?
+              convolution_forward::desc(prop_kind::forward,
+                      convolution_direct, src.GetOpMemDesc(),
+                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
+                      output.GetOpMemDesc(), strides, dilations,
+                      padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_)):
+              convolution_forward::desc(prop_kind::forward,
+                      convolution_direct, src.GetOpMemDesc(),
+                      filter.GetOpMemDesc(), bias.GetOpMemDesc(),
+                      output.GetOpMemDesc(), strides,
+                      padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_));
+
+          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
+                                                                  cpu_engine);
+          AllocateOutputTensor(context, conv_prim_desc,
+                               output_dims_mkl_order, tf_fmt, &output_tensor);
+          // Set data handle for output.
+          output.SetUsrMemDataHandle(output_tensor);
+
+          Tensor* filter_out_tensor = nullptr;
+          AllocateFilterOutputTensor(context, conv_prim_desc,
+                TFShapeToMklDnnDims(filter_tf_shape),
+                &filter_out_tensor);
+
+          PrepareAndExecuteNet(conv_prim_desc, &src, &filter, &bias, &output,
+                               filter_out_tensor);
       } else {
-        // Create convolution primitive without Bias.
-        auto conv_desc = convolution_forward::desc(
-            prop_kind::forward, convolution_direct, src.GetOpMemDesc(),
-            filter.GetOpMemDesc(), output.GetOpMemDesc(), strides, padding_l,
-            padding_r, TFPaddingToMklDnnPadding(padding_));
-
-        auto conv_prim_desc =
-            convolution_forward::primitive_desc(conv_desc, cpu_engine);
-        AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
-                             tf_fmt, &output_tensor);
-        // Set data handle for output.
-        output.SetUsrMemDataHandle(output_tensor);
-
-        Tensor* filter_out_tensor = nullptr;
-        AllocateFilterOutputTensor(context, conv_prim_desc,
-                                   TFShapeToMklDnnDims(filter_tf_shape),
-                                   &filter_out_tensor);
-        PrepareAndExecuteNet(conv_prim_desc, &src, &filter, nullptr, &output,
-                             filter_out_tensor);
+          // Create convolution primitive without Bias.
+          // Use MKLDNN dilated convolution in case of dilated rate (>0).
+          auto conv_desc = (dilations[kDilationH] > 0 ||
+            dilations[kDilationW] > 0) ?
+            convolution_forward::desc(prop_kind::forward,
+              convolution_direct, src.GetOpMemDesc(),
+              filter.GetOpMemDesc(), output.GetOpMemDesc(),
+              strides, dilations, padding_l, padding_r,
+              TFPaddingToMklDnnPadding(padding_)):
+          convolution_forward::desc(prop_kind::forward,
+              convolution_direct, src.GetOpMemDesc(),
+              filter.GetOpMemDesc(), output.GetOpMemDesc(),
+              strides, padding_l, padding_r,
+              TFPaddingToMklDnnPadding(padding_));
+
+          auto conv_prim_desc = convolution_forward::primitive_desc(conv_desc,
+                                                                  cpu_engine);
+          AllocateOutputTensor(context, conv_prim_desc, output_dims_mkl_order,
+                               tf_fmt, &output_tensor);
+          // Set data handle for output.
+          output.SetUsrMemDataHandle(output_tensor);
+
+          Tensor* filter_out_tensor = nullptr;
+          AllocateFilterOutputTensor(context, conv_prim_desc,
+                TFShapeToMklDnnDims(filter_tf_shape),
+                &filter_out_tensor);
+          PrepareAndExecuteNet(conv_prim_desc, &src, &filter,
+                              nullptr, &output, filter_out_tensor);
       }
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -658,10 +700,12 @@ class MklConv2DOp : public OpKernel {
 
  private:
   std::vector<int32> strides_;
+  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
   const int kInputIndex_Src = 0, kInputIndex_Filter = 1, kInputIndex_Bias = 2;
   const int kOutputIndex_Dst = 0, kOutputIndex_Filter = 1;
+  const int kDilationH = 0, kDilationW = 1;
 
   // Allocate output tensor.
   void AllocateOutputTensor(
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 9dd88221a8..7ca10db895 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -58,13 +58,16 @@ class MklDnnConvUtil {
  protected:
   OpKernelContext* context_;  // We don't own this.
   std::vector<int32> strides_;
+  std::vector<int32> dilations_;
   Padding padding_;
   TensorFormat data_format_;
 
  public:
   MklDnnConvUtil(OpKernelContext* context, const std::vector<int32>& strides,
-                 Padding pad, TensorFormat fm)
-      : context_(context), strides_(strides), padding_(pad), data_format_(fm) {}
+                 Padding pad, TensorFormat fm,
+                 const std::vector<int32>& dilations) :
+    context_(context), strides_(strides), padding_(pad),
+    data_format_(fm), dilations_(dilations) {}
 
   virtual ~MklDnnConvUtil() { context_ = nullptr; }
 
@@ -78,6 +81,16 @@ class MklDnnConvUtil {
     *strides = {stride_rows, stride_cols};
   }
 
+  // Calculate Convolution dilations
+  virtual inline void GetDilationsInMklOrder(memory::dims *dilations) {
+    // For now we take the dilation from the second and third dimensions only
+    // (we do not support dilation on the batch or depth dimension).
+    CHECK_NOTNULL(dilations);
+    int dilations_rows = GetTensorDim(dilations_, data_format_, 'H');
+    int dilations_cols = GetTensorDim(dilations_, data_format_, 'W');
+    *dilations = {dilations_rows, dilations_cols};
+  }
+
   // Calculate Convolution input size in MKL-DNN order. MKL-DNN
   // requires input in NCHW format. Function does not return anything.
   // But errors arising from sanity checks are returned in context's
@@ -213,7 +226,8 @@ class MklDnnConvUtil {
   // TODO(nhasabni): Add similar function for input and filter in MklShape.
   virtual inline void GetOutputAndPadSizeInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
-      const memory::dims& strides, memory::dims* output_dims_tf_order,
+      const memory::dims& strides, const memory::dims& dilations,
+      memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -232,6 +246,8 @@ class MklDnnConvUtil {
     // Stride is vector of 2 elements: {s_r, s_c}
     int stride_rows = strides[0];
     int stride_cols = strides[1];
+    int dilation_rows = dilations[0];
+    int dilation_cols = dilations[1];
 
     // Output batch is same as input batch.
     int out_batch = GetTensorDim(input_shape, data_format_, 'N');
@@ -241,11 +257,13 @@ class MklDnnConvUtil {
     int64 out_rows = 0, out_cols = 0;
     int64 pad_top = 0, pad_bottom = 0, pad_left, pad_right;
 
-    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
-                                 input_rows, filter_rows, stride_rows, padding_,
+    OP_REQUIRES_OK(context_,
+            GetWindowedOutputSizeVerboseV2(input_rows, filter_rows,
+                                 dilation_rows, stride_rows, padding_,
                                  &out_rows, &pad_top, &pad_bottom));
-    OP_REQUIRES_OK(context_, GetWindowedOutputSizeVerbose(
-                                 input_cols, filter_cols, stride_cols, padding_,
+    OP_REQUIRES_OK(context_,
+            GetWindowedOutputSizeVerboseV2(input_cols, filter_cols,
+                                 dilation_cols, stride_cols, padding_,
                                  &out_cols, &pad_left, &pad_right));
 
     // Tensorflow output is in data_format order. (NHWC or NCHW)
@@ -271,7 +289,8 @@ class MklDnnConvUtil {
   //
   // Function does not return anything, but sets error in context status.
   inline void GetOutputAndPadSizeInMklOrder(
-      size_t src_index, size_t filter_index, const memory::dims& strides,
+      size_t src_index, size_t filter_index,
+      const memory::dims& strides, const memory::dims& dilations,
       memory::dims* output_dims_tf_order, memory::dims* output_dims_mkl_order,
       memory::dims* pad_l, memory::dims* pad_r) {
     CHECK_NOTNULL(output_dims_tf_order);
@@ -286,9 +305,9 @@ class MklDnnConvUtil {
                 errors::InvalidArgument("input must be 4-dimensional",
                                         input_tf_shape.DebugString()));
 
-    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape, strides,
-                                  output_dims_tf_order, output_dims_mkl_order,
-                                  pad_l, pad_r);
+    GetOutputAndPadSizeInMklOrder(input_tf_shape, filter_tf_shape,
+                                  strides, dilations, output_dims_tf_order,
+                                  output_dims_mkl_order, pad_l, pad_r);
   }
 
   // Wrapper function to calculate input, filter, and output sizes of
@@ -300,12 +319,14 @@ class MklDnnConvUtil {
   inline void GetConvFwdSizesInMklOrder(
       const TensorShape& input_shape, const TensorShape& filter_shape,
       memory::dims* input_dims, memory::dims* filter_dims,
-      memory::dims* strides, memory::dims* output_dims_tf_order,
+      memory::dims* strides, memory::dims *dilations,
+      memory::dims* output_dims_tf_order,
       memory::dims* output_dims_mkl_order, memory::dims* pad_l,
       memory::dims* pad_r) {
     CHECK_NOTNULL(input_dims);
     CHECK_NOTNULL(filter_dims);
     CHECK_NOTNULL(strides);
+    CHECK_NOTNULL(dilations);
     CHECK_NOTNULL(output_dims_tf_order);
     CHECK_NOTNULL(output_dims_mkl_order);
     CHECK_NOTNULL(pad_l);
@@ -316,7 +337,9 @@ class MklDnnConvUtil {
     GetFilterSizeInMklOrder(input_shape, filter_shape, filter_dims);
     if (!context_->status().ok()) return;
     GetStridesInMklOrder(strides);
-    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape, *strides,
+    GetDilationsInMklOrder(dilations);
+    GetOutputAndPadSizeInMklOrder(input_shape, filter_shape,
+                                  *strides, *dilations,
                                   output_dims_tf_order, output_dims_mkl_order,
                                   pad_l, pad_r);
     if (!context_->status().ok()) return;
@@ -344,7 +367,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
         context, (stride_n == 1 && stride_c == 1),
         errors::InvalidArgument("Current implementation does not yet support "
                                 "strides in the batch and depth dimensions."));
-
+    OP_REQUIRES_OK(context, context->GetAttr("dilations", &dilations_));
+    OP_REQUIRES(context, dilations_.size() == 4,
+                errors::InvalidArgument("Sliding window dilations field must "
+                                        "specify 4 dimensions"));
+    int dilation_n = GetTensorDim(dilations_, data_format_, 'N');
+    int dilation_c = GetTensorDim(dilations_, data_format_, 'C');
+    int dilation_h = GetTensorDim(dilations_, data_format_, 'H');
+    int dilation_w = GetTensorDim(dilations_, data_format_, 'W');
+    OP_REQUIRES(context, (dilation_n == 1 && dilation_c == 1),
+                errors::InvalidArgument(
+                    "Current implementation does not yet support "
+                    "dilations in the batch and depth dimensions."));
+    OP_REQUIRES(
+        context, dilation_h > 0 && dilation_w > 0,
+        errors::InvalidArgument("Dilated rates should be larger than 0."));
     OP_REQUIRES_OK(context, context->GetAttr("padding", &padding_));
   }
 
@@ -406,15 +443,16 @@ class MklConv2DBackpropCommonOp : public OpKernel {
       // By default, all dims are in MKL order. Only dims in TF order
       // are those with prefix tf_order.
       memory::dims outbprop_dims, fwd_input_dims, fwd_filter_dims;
-      memory::dims padding_l, padding_r, strides, fwd_output_dims;
+      memory::dims padding_l, padding_r, dilations, strides, fwd_output_dims;
       memory::dims fwd_output_dims_tf_order;
 
       // Get forward convolution parameters.
-      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_);
+      MklDnnConvUtil conv_utl(context, strides_, padding_, data_format_,
+                             dilations_);
       conv_utl.GetConvFwdSizesInMklOrder(
           input_tf_shape, filter_tf_shape, &fwd_input_dims, &fwd_filter_dims,
-          &strides, &fwd_output_dims_tf_order, &fwd_output_dims, &padding_l,
-          &padding_r);
+          &strides, &dilations, &fwd_output_dims_tf_order, &fwd_output_dims,
+          &padding_l, &padding_r);
       if (!context->status().ok()) return;
 
       // Create Convolution forward descriptor since Convolution backward
@@ -437,10 +475,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
                                               memory::format::hwio);
       // Tensorflow Output of Conv2D is in data_format order.
       auto fwd_out_md = memory::desc(fwd_output_dims, MklDnnType<T>(), tf_fmt);
-      auto fwd_desc = convolution_forward::desc(
-          prop_kind::forward, convolution_direct, fwd_input_md, fwd_filter_md,
-          fwd_out_md, strides, padding_l, padding_r,
-          TFPaddingToMklDnnPadding(padding_));
+
+      const int kDilationH = 0, kDilationW = 1;
+      dilations[kDilationH] -= 1;
+      dilations[kDilationW] -= 1;
+      auto fwd_desc = (dilations[kDilationH] > 0 || dilations[kDilationW] > 0)?
+              convolution_forward::desc(prop_kind::forward,
+                     convolution_direct, fwd_input_md,
+                     fwd_filter_md, fwd_out_md,
+                     strides, dilations, padding_l, padding_r,
+                     TFPaddingToMklDnnPadding(padding_)) :
+              convolution_forward::desc(prop_kind::forward,
+                     convolution_direct, fwd_input_md,
+                     fwd_filter_md, fwd_out_md,
+                     strides, padding_l, padding_r,
+                     TFPaddingToMklDnnPadding(padding_));
       auto fwd_pd = convolution_forward::primitive_desc(fwd_desc, cpu_engine);
 
       // Create memory for user data. Describe how the inputs and outputs of
@@ -485,8 +534,9 @@ class MklConv2DBackpropCommonOp : public OpKernel {
 
       // Operator-specific call to create and execute primitive.
       CreatePrimitive(context, cpu_engine, fwd_pd, &input, &filter,
-                      &outbackprop, &output, &output_tensor, strides, padding_l,
-                      padding_r, TFPaddingToMklDnnPadding(padding_),
+                      &outbackprop, &output, &output_tensor,
+                      strides, dilations, padding_l, padding_r,
+                      TFPaddingToMklDnnPadding(padding_),
                       bwd_output_dims, bwd_output_format);
     } catch (mkldnn::error& e) {
       string error_msg = "Status: " + std::to_string(e.status) +
@@ -535,20 +585,21 @@ class MklConv2DBackpropCommonOp : public OpKernel {
   virtual memory::format GetOutputFormat(const memory::format data_format) = 0;
 
   /// Create and execute the primitive storing output in the output_tensor.
-  virtual void CreatePrimitive(
-      OpKernelContext* context, const engine& cpu_engine,
-      const convolution_forward::primitive_desc& conv_fwd_pd,
-      MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
-      MklDnnData<T>* output, Tensor** output_tensor,
-      const memory::dims& strides, const memory::dims& padding_l,
-      const memory::dims& padding_r, padding_kind padding,
-      const memory::dims& bwd_output_dims,
-      memory::format bwd_output_format) = 0;
+  virtual void CreatePrimitive(OpKernelContext* context,
+    const engine& cpu_engine,
+    const convolution_forward::primitive_desc& conv_fwd_pd,
+    MklDnnData<T>* input, MklDnnData<T>* filter, MklDnnData<T>* outbackprop,
+    MklDnnData<T>* output, Tensor** output_tensor, const memory::dims& strides,
+    const memory::dims& dilations, const memory::dims& padding_l,
+    const memory::dims& padding_r, padding_kind padding,
+    const memory::dims& bwd_output_dims,
+    memory::format bwd_output_format) = 0;
 
   // Get the data_format {NCHW, NHWC}
   TensorFormat GetTFDataFormat() { return data_format_; }
 
  private:
+  std::vector<int32> dilations_;
   std::vector<int32> strides_;
   Padding padding_;
   TensorFormat data_format_;
diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc
index e9a2376b54..d91f7107c5 100644
--- a/tensorflow/core/kernels/mkl_input_conversion_op.cc
+++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc
@@ -442,12 +442,11 @@ class MklInputConversionOp : public OpKernel {
       auto input_tf_md = mkl_output_mkl_shape.GetTfLayout();
       tf_input.SetUsrMem(input_tf_md, tf_tensor);
 
-      // Create reorder between tensorflow layout and Mkl layout.
+      // Create reorder between tensorflow layout and Mkl layout if necessary
       std::vector<primitive> net;
-      CHECK_EQ(tf_input.CheckReorderToOpMem(
+      tf_input.CheckReorderToOpMem(
                    memory::primitive_desc(output_mkl_md, cpu_engine),
-                   tensor_out, &net),
-               true);
+                   tensor_out, &net);
       stream(stream::kind::eager).submit(net).wait();
 
       // -- The tensor in MKL format passes through --
diff --git a/tensorflow/core/kernels/mkl_relu_op.cc b/tensorflow/core/kernels/mkl_relu_op.cc
index 267f4f8d12..0a0f69522f 100644
--- a/tensorflow/core/kernels/mkl_relu_op.cc
+++ b/tensorflow/core/kernels/mkl_relu_op.cc
@@ -392,7 +392,7 @@ class MklReluOpBase : public OpKernel {
 
       Tensor* dst_tensor = nullptr;
       if (src_tensor.dims() == 0) {
-        Compute_Scalar(context);
+        Compute_Scalar(context); // scalar case doesn't use in-place operation
         return;
       }
 
@@ -437,11 +437,15 @@ class MklReluOpBase : public OpKernel {
         dnn_shape_dst.SetMklTensor(false);
         tf_shape_dst = src_tensor.shape();
       }
-      AllocateOutputSetMklShape(context, dst_index, &dst_tensor, tf_shape_dst,
-                                dnn_shape_dst);
+      
+      // Allocate output and MklDnnShape tensors separately for possible
+      // in-place operation
+      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                      {src_index}, dst_index, tf_shape_dst, &dst_tensor));
+      AllocateOutputSetMklShape(context, dst_index, dnn_shape_dst);
 
       // Destination memory descriptor is same as source memory descriptor.
-      auto dst_md = src_md;
+      auto &dst_md = src_md;
       dst.SetUsrMem(dst_md, dst_tensor);
 
       // execute net
@@ -492,7 +496,7 @@ class MklReluGradOpBase : public OpKernel {
 
       int src_dims_size = src_tensor.dims();
       if (src_dims_size == 0) {
-        Compute_Scalar(context);
+        Compute_Scalar(context); // scalar case doesn't use in-place operation
         return;
       }
 
@@ -603,8 +607,13 @@ class MklReluGradOpBase : public OpKernel {
         // so it is ok to get TensorFlow shape.
         tf_shape_diff_src = src_tensor.shape();
       }
-      AllocateOutputSetMklShape(context, diff_src_index, &diff_src_tensor,
-                                tf_shape_diff_src, dnn_shape_diff_src);
+
+      // Allocate diff_src and MklDnnShape tensors separately for possible
+      // in-place operation
+      OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                      {diff_dst_index}, diff_src_index, tf_shape_diff_src,
+                                      &diff_src_tensor));
+      AllocateOutputSetMklShape(context, diff_src_index, dnn_shape_diff_src);
 
       // diff_src memory descriptor is same as memory descriptor for both
       // inputs.
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index 4abfbfb1a6..d65692a552 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -16,6 +16,14 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
 #define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
 
+
+// This file requires the following include because it uses CudaAtomicMax:
+// #include "tensorflow/core/util/cuda_kernel_helper.h"
+
+// Unfortunately we can't add the #include, since it breaks compilation for
+// non-GPU targets. This only breaks in clang, because it's more strict for
+// template code and CudaAtomicMax is used in template context.
+
 // This file requires the following include because it uses CudaAtomicMax:
 // #include "tensorflow/core/util/cuda_kernel_helper.h"
 
diff --git a/tensorflow/core/kernels/spacetodepth_op.cc b/tensorflow/core/kernels/spacetodepth_op.cc
index 23df1c35e5..e59adfc6ac 100644
--- a/tensorflow/core/kernels/spacetodepth_op.cc
+++ b/tensorflow/core/kernels/spacetodepth_op.cc
@@ -187,6 +187,9 @@ TF_CALL_ALL_TYPES(REGISTER);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     SpaceToDepthOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<Eigen::half>("T"),
+    SpaceToDepthOp<GPUDevice, Eigen::half>);
 REGISTER_KERNEL_BUILDER(
     Name("SpaceToDepth").Device(DEVICE_GPU).TypeConstraint<qint8>("T"),
     SpaceToDepthOp<GPUDevice, qint8>);
diff --git a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
index db05ca1ed2..f38459724a 100644
--- a/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/spacetodepth_op_gpu.cu.cc
@@ -234,6 +234,12 @@ struct SpaceToDepthOpFunctor<GPUDevice, T, FORMAT_NCHW> {
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NCHW>;
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, float, FORMAT_NHWC>;
 
+// Instantiate the GPU implementations for Eigen::half.
+template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NCHW>;
+template struct functor::SpaceToDepthOpFunctor<GPUDevice, Eigen::half,
+                                               FORMAT_NHWC>;
+
 // NCHW_VECT_C with 4 x qint8 can be treated as NCHW int32.
 template struct functor::SpaceToDepthOpFunctor<GPUDevice, int32, FORMAT_NCHW>;
 
diff --git a/tensorflow/core/lib/io/record_reader.cc b/tensorflow/core/lib/io/record_reader.cc
index 254fdf115d..6de850bb20 100644
--- a/tensorflow/core/lib/io/record_reader.cc
+++ b/tensorflow/core/lib/io/record_reader.cc
@@ -205,7 +205,9 @@ Status RecordReader::SkipNBytes(uint64 offset) {
     if (options_.buffer_size > 0) {
       TF_RETURN_IF_ERROR(input_stream_->SkipNBytes(offset));
     }
+#if !defined(IS_SLIM_BUILD)
   }
+#endif
   return Status::OK();
 }  // namespace io
 
diff --git a/tensorflow/core/lib/io/record_reader.h b/tensorflow/core/lib/io/record_reader.h
index 62dd2efb79..26278e0328 100644
--- a/tensorflow/core/lib/io/record_reader.h
+++ b/tensorflow/core/lib/io/record_reader.h
@@ -16,10 +16,10 @@ limitations under the License.
 #ifndef TENSORFLOW_LIB_IO_RECORD_READER_H_
 #define TENSORFLOW_LIB_IO_RECORD_READER_H_
 
-#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
-#if !defined(IS_SLIM_BUILD)
 #include "tensorflow/core/lib/io/inputstream_interface.h"
+#if !defined(IS_SLIM_BUILD)
 #include "tensorflow/core/lib/io/zlib_compression_options.h"
 #include "tensorflow/core/lib/io/zlib_inputstream.h"
 #endif  // IS_SLIM_BUILD
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index f32baee45e..e2453b9712 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -266,6 +266,16 @@ REGISTER_OP("BatchDataset")
     .Attr("output_shapes: list(shape) >= 1")
     .SetShapeFn(shape_inference::ScalarShape);
 
+// TODO(mrry): move SlideDataset to contrib in the future.
+REGISTER_OP("SlideDataset")
+    .Input("input_dataset: variant")
+    .Input("window_size: int64")
+    .Input("stride: int64")
+    .Output("handle: variant")
+    .Attr("output_types: list(type) >= 1")
+    .Attr("output_shapes: list(shape) >= 1")
+    .SetShapeFn(shape_inference::ScalarShape);
+
 REGISTER_OP("PaddedBatchDataset")
     .Input("input_dataset: variant")
     .Input("batch_size: int64")
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 910fbaca9e..d6a0f38033 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1498,6 +1498,7 @@ REGISTER_OP("_MklConv2D")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn(shape_inference::Conv2DShape)
     .Doc(R"doc(
 MKL version of Conv2D operator. Uses MKL DNN APIs to perform 2D convolution.
@@ -1516,6 +1517,7 @@ REGISTER_OP("__MklDummyConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 Dummy node that enables fusing Conv2D and BiasAdd operator for MKL. This node
 does not perform anything. It is just created as an intermediate output of
@@ -1541,6 +1543,7 @@ REGISTER_OP("_MklConv2DWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2D and BiasAdd operator. Uses MKL DNN APIs to perform
 2D convolution and add Bias to the output of convolution.
@@ -1563,6 +1566,7 @@ REGISTER_OP("_MklConv2DBackpropFilter")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(1, &s));
@@ -1589,6 +1593,7 @@ REGISTER_OP("__MklDummyConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1633,6 +1638,7 @@ REGISTER_OP("_MklConv2DBackpropFilterWithBias")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input_shape;
       // Fetch the data_format attribute, which may not exist.
@@ -1668,6 +1674,7 @@ REGISTER_OP("_MklConv2DWithBiasBackpropBias")
     .Attr("T: {half, float, double}")
     .Attr("strides: list(int)")
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .Doc(R"doc(
 MKL version of Conv2DBackpropBias. Uses MKL DNN APIs to compute the
 gradients of convolution with respect to the bias.
@@ -1690,6 +1697,7 @@ REGISTER_OP("_MklConv2DBackpropInput")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
     .Attr(GetConvnetDataFormatAttrString())
+    .Attr("dilations: list(int) = [1, 1, 1, 1]")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle s;
       TF_RETURN_IF_ERROR(c->MakeShapeFromShapeTensor(0, &s));
diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc
index b6b3722caa..682e46e0fc 100644
--- a/tensorflow/core/platform/windows/windows_file_system.cc
+++ b/tensorflow/core/platform/windows/windows_file_system.cc
@@ -382,7 +382,8 @@ Status WindowsFileSystem::NewReadOnlyMemoryRegionFromFile(
 
 Status WindowsFileSystem::FileExists(const string& fname) {
   constexpr int kOk = 0;
-  if (_access(TranslateName(fname).c_str(), kOk) == 0) {
+  std::wstring ws_translated_fname = Utf8ToWideChar(TranslateName(fname));
+  if (_waccess(ws_translated_fname.c_str(), kOk) == 0) {
     return Status::OK();
   }
   return errors::NotFound(fname, " not found");
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 7405e01e14..22f2c02b78 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX "-rc1"
+#define TF_VERSION_SUFFIX ""
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/core/util/stat_summarizer.h b/tensorflow/core/util/stat_summarizer.h
index f7b63e8686..79fa63723e 100644
--- a/tensorflow/core/util/stat_summarizer.h
+++ b/tensorflow/core/util/stat_summarizer.h
@@ -186,7 +186,7 @@ class StatSummarizer {
   void Reset();
 
   // Returns number of runs.
-  int num_runs() const { return run_total_us_.count(); }
+  int num_runs() const { return static_cast<int>(run_total_us_.count()); }
 
   // Returns stats of total microseconds spent by all nodes in each run.
   const Stat<int64>& run_total_us() const { return run_total_us_; }
diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md
index 9f6fe91b14..6d0458e678 100644
--- a/tensorflow/docs_src/community/welcome.md
+++ b/tensorflow/docs_src/community/welcome.md
@@ -51,6 +51,8 @@ Europe:
 TensorFlow provides multiple communication paths.  To pick the right path,
 please read the following list carefully:
 
+  * For new release announcements and security updates, subscribe to
+    [announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
   * To ask or answer technical questions about TensorFlow, use
     [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).
     For example, ask or search Stack Overflow about a particular error message
@@ -65,5 +67,5 @@ please read the following list carefully:
     on GitHub.  For example, use the issue tracker to request a
     new operation in TensorFlow.
   * To report vulnerabilities, please follow our
-    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/SECURITY.md).
+    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 818798555a..0481c97885 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 4c6dfa8daf..8f89898c92 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 527884863e..0ee9c849e1 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0-rc1</version>
+                 <version>1.6.0</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0-rc1.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0-rc1.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>d
+<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 2741b61bb2..8612762271 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -39,16 +39,9 @@ must be installed on your system:
   * [cuDNN SDK v7](http://developer.nvidia.com/cudnn). For details, see
     [NVIDIA's documentation](http://docs.nvidia.com/deeplearning/sdk/cudnn-install/).
     Ensure that you create the `CUDA_HOME` environment variable as
-    described in NVIDIA's documentation.
-  * [CUPTI](http://docs.nvidia.com/cuda/cupti/) ships with the CUDA Toolkit, but
-    you also need to append its path to the `LD_LIBRARY_PATH` environment
-    variable:
-
-    <pre> $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b> </pre>
-
-In order to run TensorFlow computations on the GPU, you also need:
-
-  * A GPU card with CUDA Compute Capability 3.0 or higher.  See
+    described in the NVIDIA documentation.
+  * GPU card with CUDA Compute Capability 3.0 or higher for building
+    from source and 3.5 or higher for our binaries. See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for
     a list of supported GPU cards.
   * [GPU drivers](http://nvidia.com/driver) supporting your version of the CUDA
@@ -172,7 +165,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -277,7 +270,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -340,24 +333,23 @@ where:
     to 6006.
   * <tt><i>TensorFlowCPUImage</i></tt> is required. It identifies the Docker
     container. Specify one of the following values:
-    * <tt>gcr.io/tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:latest-devel</tt>, which is the latest
+    * <tt>tensorflow/tensorflow</tt>, which is the TensorFlow CPU binary image.
+    * <tt>tensorflow/tensorflow:latest-devel</tt>, which is the latest
       TensorFlow CPU Binary image plus source code.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i></tt>, which is the
+    * <tt>tensorflow/tensorflow:<i>version</i></tt>, which is the
       specified version (for example, 1.1.0rc1) of TensorFlow CPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
+    * <tt>tensorflow/tensorflow:<i>version</i>-devel</tt>, which is
       the specified version (for example, 1.1.0rc1) of the TensorFlow GPU
       binary image plus source code.
 
-    <tt>gcr.io</tt> is the Google Container Registry. Note that some
-    TensorFlow images are also available at
+    TensorFlow images are available at
     [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches the latest TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
+$ <b>docker run -it tensorflow/tensorflow bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow CPU binary image in a
@@ -365,7 +357,7 @@ Docker container. However, in this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b>
+$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -389,14 +381,14 @@ where:
     <tt><i>hostPort</i></tt> and <code><em>containerPort</em></code> to `8888`.
   * <i>TensorFlowGPUImage</i> specifies the Docker container. You must
     specify one of the following values:
-    * <tt>gcr.io/tensorflow/tensorflow:latest-gpu</tt>, which is the latest
+    * <tt>tensorflow/tensorflow:latest-gpu</tt>, which is the latest
       TensorFlow GPU binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:latest-devel-gpu</tt>, which is
+    * <tt>tensorflow/tensorflow:latest-devel-gpu</tt>, which is
       the latest TensorFlow GPU Binary image plus source code.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
+    * <tt>tensorflow/tensorflow:<i>version</i>-gpu</tt>, which is the
       specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image.
-    * <tt>gcr.io/tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
+    * <tt>tensorflow/tensorflow:<i>version</i>-devel-gpu</tt>, which is
       the specified version (for example, 0.12.1) of the TensorFlow GPU
       binary image plus source code.
 
@@ -405,7 +397,7 @@ following command launches the latest TensorFlow GPU binary image in a
 Docker container from which you can run TensorFlow programs in a shell:
 
 <pre>
-$ <b>nvidia-docker run -it gcr.io/tensorflow/tensorflow:latest-gpu bash</b>
+$ <b>nvidia-docker run -it tensorflow/tensorflow:latest-gpu bash</b>
 </pre>
 
 The following command also launches the latest TensorFlow GPU binary image
@@ -413,13 +405,13 @@ in a Docker container. In this Docker container, you can run TensorFlow
 programs in a Jupyter notebook:
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:latest-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:latest-gpu</b>
 </pre>
 
 The following command installs an older TensorFlow version (0.12.1):
 
 <pre>
-$ <b>nvidia-docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow:0.12.1-gpu</b>
+$ <b>nvidia-docker run -it -p 8888:8888 tensorflow/tensorflow:0.12.1-gpu</b>
 </pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
@@ -464,7 +456,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -489,7 +481,7 @@ If you installed through Docker, start a Docker container
 from which you can run bash. For example:
 
 <pre>
-$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b>
+$ <b>docker run -it tensorflow/tensorflow bash</b>
 </pre>
 
 
@@ -631,14 +623,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -650,14 +642,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -669,14 +661,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -688,14 +680,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 99745fcc6d..7207cb4f2b 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -292,24 +292,23 @@ where:
     to 6006.
   * <i>TensorFlowImage</i> is required. It identifies the Docker container.
     You must specify one of the following values:
-    * <code>gcr.io/tensorflow/tensorflow</code>: TensorFlow binary image.
-    * <code>gcr.io/tensorflow/tensorflow:latest-devel</code>: TensorFlow
+    * <code>tensorflow/tensorflow</code>: TensorFlow binary image.
+    * <code>tensorflow/tensorflow:latest-devel</code>: TensorFlow
       Binary image plus source code.
 
-<code>gcr.io</code> is the Google Container Registry. Note that some
-TensorFlow images are also available at
+The TensorFlow images are available at
 [dockerhub](https://hub.docker.com/r/tensorflow/tensorflow/).
 
 For example, the following command launches a TensorFlow CPU binary image
 in a Docker container from which you can run TensorFlow programs in a shell:
 
-<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
 
 The following command also launches a TensorFlow CPU binary image in a
 Docker container. However, in this Docker container, you can run
 TensorFlow programs in a Jupyter notebook:
 
-<pre>$ <b>docker run -it -p 8888:8888 gcr.io/tensorflow/tensorflow</b></pre>
+<pre>$ <b>docker run -it -p 8888:8888 tensorflow/tensorflow</b></pre>
 
 Docker will download the TensorFlow binary image the first time you launch it.
 
@@ -351,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -376,7 +375,7 @@ do the following:
 If you installed through Docker, start a Docker container that runs bash.
 For example:
 
-<pre>$ <b>docker run -it gcr.io/tensorflow/tensorflow bash</b></pre>
+<pre>$ <b>docker run -it tensorflow/tensorflow bash</b></pre>
 
 
 
@@ -519,7 +518,7 @@ The value you specify depends on your Python version.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
 </pre>
 
 
@@ -527,5 +526,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 4e7b07d78b..a7f33819b4 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -350,10 +350,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0rc1 on Linux:
+for TensorFlow 1.6.0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0rc1-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index f0a30ee394..2413bc9cfb 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -17,7 +17,7 @@ You must choose one of the following types of TensorFlow to install:
     NVIDIA® GPU, you must install this version. Note that this version of
     TensorFlow is typically much easier to install (typically,
     in 5 or 10 minutes), so even if you have an NVIDIA GPU, we recommend
-    installing this version first.
+    installing this version first. Prebuilt binaries will use AVX instructions. 
   * **TensorFlow with GPU support**. TensorFlow programs typically run
     significantly faster on a GPU than on a CPU. Therefore, if your
     system has a NVIDIA® GPU meeting the prerequisites shown below
@@ -41,7 +41,8 @@ installed on your system:
     Note that cuDNN is typically installed in a different location from the
     other CUDA DLLs. Ensure that you add the directory where you installed
     the cuDNN DLL to your `%PATH%` environment variable.
-  * GPU card with CUDA Compute Capability 3.0 or higher.  See
+  * GPU card with CUDA Compute Capability 3.0 or higher for building
+    from source and 3.5 or higher for our binaries. See
     [NVIDIA documentation](https://developer.nvidia.com/cuda-gpus) for a
     list of supported GPU cards.
 
diff --git a/tensorflow/docs_src/performance/performance_guide.md b/tensorflow/docs_src/performance/performance_guide.md
index cd47fc2803..580a899ac4 100644
--- a/tensorflow/docs_src/performance/performance_guide.md
+++ b/tensorflow/docs_src/performance/performance_guide.md
@@ -78,7 +78,7 @@ training CIFAR-10 illustrates the use of the `tf.data` API along with
 The `tf.data` API utilizes C++ multi-threading and has a much lower overhead
 than the Python-based `queue_runner` that is limited by Python's multi-threading
 performance. A detailed performance guide for the `tf.data` API can be found
-[here](#datasets_performance).
+[here](@{$datasets_performance}).
 
 While feeding data using a `feed_dict` offers a high level of flexibility, in
 general `feed_dict` does not provide a scalable solution. If only a single GPU
diff --git a/tensorflow/docs_src/performance/xla/jit.md b/tensorflow/docs_src/performance/xla/jit.md
index d4dc3e57c8..d9a979ccbd 100644
--- a/tensorflow/docs_src/performance/xla/jit.md
+++ b/tensorflow/docs_src/performance/xla/jit.md
@@ -157,7 +157,7 @@ to fuse Ops is visible by starting at `hlo_graph_0.dot` and viewing each diagram
 in succession.
 
 To Render the .dot file into a png, install
-[GraphViz](http://www.graphviz.org/Download..php) and run:
+[GraphViz](https://www.graphviz.org/download/) and run:
 
 ```shell
 dot -Tpng hlo_graph_80.dot -o hlo_graph_80.png
diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index 5fb1c2da88..d1399814ee 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -459,7 +459,7 @@ accuracy_score = classifier.evaluate(x=test_set.data,
 
 
 [debug_tflearn_iris.py](https://www.tensorflow.org/code/tensorflow/python/debug/examples/debug_tflearn_iris.py),
-based on {$tflearn$tf-learn's iris tutorial}, contains a full example of how to
+based on [tf-learn's iris tutorial](https://www.tensorflow.org/versions/r1.2/get_started/tflearn), contains a full example of how to
 use the tfdbg with `Estimator`s. To run this example, do:
 
 ```none
@@ -753,6 +753,7 @@ There are three possible workarounds or solutions:
    # For LocalCLIDebugHook
    hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")]
    ```
+
    Make sure that the directory pointed to by dump_root is empty or nonexistent.
    tfdbg cleans up the dump directories before exiting.
 *  Reduce the batch size used during the runs.
diff --git a/tensorflow/docs_src/programmers_guide/faq.md b/tensorflow/docs_src/programmers_guide/faq.md
index 1548d43877..392ac6f7f1 100644
--- a/tensorflow/docs_src/programmers_guide/faq.md
+++ b/tensorflow/docs_src/programmers_guide/faq.md
@@ -159,7 +159,7 @@ available. These operations allow you to build sophisticated
 @{$reading_data$input pipelines}, at the cost of making the
 TensorFlow computation somewhat more complicated. See the how-to documentation
 for
-@{$reading_data#creating-threads-to-prefetch-using-queuerunner-objects$using `QueueRunner` objects to drive queues and readers}
+@{$reading_data#creating_threads_to_prefetch_using_queuerunner_objects$using `QueueRunner` objects to drive queues and readers}
 for more information on how to use them.
 
 ## Variables
@@ -272,7 +272,7 @@ Prefer predefined TensorFlow operations such as @{tf.decode_raw},
 
 If your data is not easily parsable with the built-in TensorFlow operations,
 consider converting it, offline, to a format that is easily parsable, such
-as ${tf.python_io.TFRecordWriter$`TFRecord`} format.
+as @{tf.python_io.TFRecordWriter$`TFRecord`} format.
 
 The more efficient method to customize the parsing behavior is to
 @{$adding_an_op$add a new op written in C++} that parses your
diff --git a/tensorflow/docs_src/programmers_guide/saved_model.md b/tensorflow/docs_src/programmers_guide/saved_model.md
index d01d187e86..55ee42dd64 100644
--- a/tensorflow/docs_src/programmers_guide/saved_model.md
+++ b/tensorflow/docs_src/programmers_guide/saved_model.md
@@ -3,7 +3,7 @@
 The @{tf.train.Saver} class provides methods to save and restore models. The
 @{tf.saved_model.simple_save} function is an easy way to build a
 @{tf.saved_model$saved model} suitable for serving.
-[Estimators](/programmers_guide/estimators) automatically save and restore
+[Estimators](@{$programmers_guide/estimators}) automatically save and restore
 variables in the `model_dir`.
 
 ## Save and restore variables
@@ -400,7 +400,7 @@ defined in:
 
 After training an `Estimator` model, you may want to create a service
 from that model that takes requests and returns a result.  You can run such a
-service locally on your machine or deploy it scalably in the cloud.
+service locally on your machine or deploy it in the cloud.
 
 To prepare a trained Estimator for serving, you must export it in the standard
 SavedModel format. This section explains how to:
diff --git a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
index 79280d246a..fadfa03e78 100644
--- a/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
+++ b/tensorflow/docs_src/programmers_guide/summaries_and_tensorboard.md
@@ -83,7 +83,7 @@ data than you need, though. Instead, consider running the merged summary op
 every `n` steps.
 
 The code example below is a modification of the
-@{$layers$simple MNIST tutorial},
+[simple MNIST tutorial](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/examples/tutorials/mnist/mnist.py),
 in which we have added some summary ops, and run them every ten steps. If you
 run this and then launch `tensorboard --logdir=/tmp/tensorflow/mnist`, you'll be able
 to visualize statistics, such as how the weights or accuracy varied during
diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md
index d74d7f3181..a9c2cb3e33 100644
--- a/tensorflow/docs_src/programmers_guide/using_tpu.md
+++ b/tensorflow/docs_src/programmers_guide/using_tpu.md
@@ -129,10 +129,9 @@ my_tpu_estimator = tf.contrib.tpu.TPUEstimator(
 Typically the `FLAGS` would be set by command line arguments. To switch from
 training locally to training on a cloud TPU you would need to:
 
-  1) Set `FLAGS.use_tpu` to `True`
-  1) Set `FLAGS.tpu_name` so the
-     `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
-  1) Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
+* Set `FLAGS.use_tpu` to `True`
+* Set `FLAGS.tpu_name` so the `tf.contrib.cluster_resolver.TPUClusterResolver` can find it
+* Set `FLAGS.model_dir` to a Google Cloud Storage bucket url (`gs://`).
 
 
 ## Optimizer
diff --git a/tensorflow/docs_src/tutorials/deep_cnn.md b/tensorflow/docs_src/tutorials/deep_cnn.md
index 3692a02f2e..6361079671 100644
--- a/tensorflow/docs_src/tutorials/deep_cnn.md
+++ b/tensorflow/docs_src/tutorials/deep_cnn.md
@@ -268,7 +268,7 @@ in `cifar10_input.py`.
 
 `cifar10_train.py` periodically @{tf.train.Saver$saves}
 all model parameters in
-@{$variables#saving-and-restoring$checkpoint files}
+@{$programmers_guide/saved_model$checkpoint files}
 but it does *not* evaluate the model. The checkpoint file
 will be used by `cifar10_eval.py` to measure the predictive
 performance (see [Evaluating a Model](#evaluating-a-model) below).
diff --git a/tensorflow/docs_src/tutorials/image_retraining.md b/tensorflow/docs_src/tutorials/image_retraining.md
index 246a420400..93d7c86e42 100644
--- a/tensorflow/docs_src/tutorials/image_retraining.md
+++ b/tensorflow/docs_src/tutorials/image_retraining.md
@@ -115,7 +115,7 @@ process is progressing. The training's objective is to make the loss as small as
 possible, so you can tell if the learning is working by keeping an eye on
 whether the loss keeps trending downwards, ignoring the short-term noise.
 
-By default this script will run 4,000 training steps. Each step chooses ten
+By default this script will run 4,000 training steps. Each step chooses 100
 images at random from the training set, finds their bottlenecks from the cache,
 and feeds them into the final layer to get predictions. Those predictions are
 then compared against the actual labels to update the final layer's weights
diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md
index 63f408c2ca..b1f06ce0a3 100644
--- a/tensorflow/docs_src/tutorials/kernel_methods.md
+++ b/tensorflow/docs_src/tutorials/kernel_methods.md
@@ -1,9 +1,9 @@
 # Improving Linear Models Using Explicit Kernel Methods
 
-Note: This document uses a deprecated version of ${tf.estimator},
-which has a ${tf.contrib.learn.estimator$different interface}.
+Note: This document uses a deprecated version of @{tf.estimator},
+which has a different interface (see `tf.contrib.learn Estimator`).
 It also uses other `contrib` methods whose
-${$version_compat#not_covered$API may not be stable}.
+@{$version_compat#not_covered$API may not be stable}.
 
 In this tutorial, we demonstrate how combining (explicit) kernel methods with
 linear models can drastically increase the latters' quality of predictions
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index ee03f440c9..9b17d0d4d5 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -193,7 +193,7 @@ to calculate loss, configure the training op, and generate predictions. If
 you're already experienced with CNNs and @{$get_started/custom_estimators$TensorFlow `Estimator`s},
 and find the above code intuitive, you may want to skim these sections or just
 skip ahead to ["Training and Evaluating the CNN MNIST
-Classifier"](#training-and-evaluating-the-cnn-mnist-classifier).
+Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
 
 ### Input Layer
 
@@ -446,7 +446,7 @@ tf.nn.softmax(logits, name="softmax_tensor")
 
 > Note: We use the `name` argument to explicitly name this operation
 > `softmax_tensor`, so we can reference it later. (We'll set up logging for the
-> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook).
+> softmax values in ["Set Up a Logging Hook"](#set-up-a-logging-hook)).
 
 We compile our predictions in a dict, and return an `EstimatorSpec` object:
 
@@ -534,9 +534,8 @@ if mode == tf.estimator.ModeKeys.TRAIN:
 ```
 
 > Note: For a more in-depth look at configuring training ops for Estimator model
-> functions, see @{$get_started/custom_estimators#defining-the-training-op-for-the-model$"Defining
-> the training op for the model"} in the @{$get_started/custom_estimators$"Creating Estimations in
-> tf.estimator"} tutorial.
+> functions, see @{$get_started/custom_estimators#defining_the_training_op_for_the_model$"Defining the training op for the model"} 
+> in the @{$get_started/custom_estimators$"Creating Estimators in tf.estimator."} tutorial.
 
 ### Add evaluation metrics
 
@@ -625,7 +624,8 @@ operation earlier when we generated the probabilities in `cnn_model_fn`.
 > Note: If you don't explicitly assign a name to an operation via the `name`
 > argument, TensorFlow will assign a default name. A couple easy ways to
 > discover the names applied to operations are to visualize your graph on
-> @{$graph_viz$TensorBoard}) or to enable the @{$debugger$TensorFlow Debugger (tfdbg)}.
+> @{$graph_viz$TensorBoard}) or to enable the
+> @{$programmers_guide/debugger$TensorFlow Debugger (tfdbg)}.
 
 Next, we create the `LoggingTensorHook`, passing `tensors_to_log` to the
 `tensors` argument. We set `every_n_iter=50`, which specifies that probabilities
diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index e22536adb6..7584a76ba5 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -109,7 +109,8 @@ This download will take a while and download a bit more than 23GB of data.
 
 To convert the `ndjson` files to
 @{$python/python_io#tfrecords_format_details$TFRecord} files containing
-${tf.train.Example} protos run the following command.
+[`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
+protos run the following command.
 
 ```shell
    python create_dataset.py --ndjson_path rnn_tutorial_data \
diff --git a/tensorflow/docs_src/tutorials/wide.md b/tensorflow/docs_src/tutorials/wide.md
index 005dc020f9..27ce75a30d 100644
--- a/tensorflow/docs_src/tutorials/wide.md
+++ b/tensorflow/docs_src/tutorials/wide.md
@@ -74,8 +74,8 @@ Here's a list of columns available in the Census Income dataset:
 | relationship   | Categorical | Wife, Own-child, Husband,         |
 :                :             : Not-in-family, Other-relative,    :
 :                :             : Unmarried.                        :
-| race           | Categorical | White, Asian-Pac-Islander,        |
-:                :             : Amer-Indian-Eskimo, Other, Black. :
+| race           | Categorical | Amer-Indian-Eskimo, Asian-Pac-    |
+:                :             : Islander, Black, White, Other.    :
 | gender         | Categorical | Female, Male.                     |
 | capital_gain   | Continuous  | Capital gains recorded.           |
 | capital_loss   | Continuous  | Capital Losses recorded.          |
@@ -247,7 +247,7 @@ hours_per_week = tf.feature_column.numeric_column('hours_per_week')
 ### Making Continuous Features Categorical through Bucketization
 
 Sometimes the relationship between a continuous feature and the label is not
-linear. As an hypothetical example, a person's income may grow with age in the
+linear. As a hypothetical example, a person's income may grow with age in the
 early stage of one's career, then the growth may slow at some point, and finally
 the income decreases after retirement. In this scenario, using the raw `age` as
 a real-valued feature column might not be a good choice because the model can
@@ -361,6 +361,16 @@ The first line of the final output should be something like
 `accuracy: 0.83557522`, which means the accuracy is 83.6%. Feel free to try more
 features and transformations and see if you can do even better!
 
+After the model is evaluated, we can use the model to predict whether an individual has an annual income of over
+50,000 dollars given an individual's information input.
+```python
+  pred_iter = model.predict(input_fn=lambda: input_fn(FLAGS.test_data, 1, False, 1))
+  for pred in pred_iter:
+    print(pred['classes'])
+```
+
+The model prediction output would be like `[b'1']` or `[b'0']` which means whether corresponding individual has an annual income of over 50,000 dollars or not.
+
 If you'd like to see a working end-to-end example, you can download our
 [example code](https://github.com/tensorflow/models/tree/master/official/wide_deep/wide_deep.py)
 and set the `model_type` flag to `wide`.
diff --git a/tensorflow/examples/android/AndroidManifest.xml b/tensorflow/examples/android/AndroidManifest.xml
index bb75431a1f..5c47ce6b67 100644
--- a/tensorflow/examples/android/AndroidManifest.xml
+++ b/tensorflow/examples/android/AndroidManifest.xml
@@ -40,6 +40,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -49,6 +50,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -58,6 +60,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
 
@@ -67,6 +70,7 @@
             <intent-filter>
                 <action android:name="android.intent.action.MAIN" />
                 <category android:name="android.intent.category.LAUNCHER" />
+                <category android:name="android.intent.category.LEANBACK_LAUNCHER" />
             </intent-filter>
         </activity>
     </application>
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
index 8bd4abb154..429138abe5 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/CameraActivity.java
@@ -351,6 +351,10 @@ public abstract class CameraActivity extends Activity
 
   protected void setFragment() {
     String cameraId = chooseCamera();
+    if (cameraId == null) {
+      Toast.makeText(this, "No Camera Detected", Toast.LENGTH_SHORT).show();
+      finish();
+    }
 
     Fragment fragment;
     if (useCamera2API) {
@@ -416,7 +420,8 @@ public abstract class CameraActivity extends Activity
 
   @Override
   public boolean onKeyDown(final int keyCode, final KeyEvent event) {
-    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP) {
+    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP
+            || keyCode == KeyEvent.KEYCODE_BUTTON_L1 || keyCode == KeyEvent.KEYCODE_DPAD_CENTER) {
       debug = !debug;
       requestRender();
       onSetDebug(debug);
diff --git a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
index 6a66ec3927..33ec65e9f7 100644
--- a/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
+++ b/tensorflow/examples/android/src/org/tensorflow/demo/StylizeActivity.java
@@ -16,8 +16,10 @@
 
 package org.tensorflow.demo;
 
+import android.app.UiModeManager;
 import android.content.Context;
 import android.content.res.AssetManager;
+import android.content.res.Configuration;
 import android.graphics.Bitmap;
 import android.graphics.Bitmap.Config;
 import android.graphics.BitmapFactory;
@@ -31,9 +33,11 @@ import android.graphics.Typeface;
 import android.media.ImageReader.OnImageAvailableListener;
 import android.os.Bundle;
 import android.os.SystemClock;
+import android.util.DisplayMetrics;
 import android.util.Size;
 import android.util.TypedValue;
 import android.view.Display;
+import android.view.KeyEvent;
 import android.view.MotionEvent;
 import android.view.View;
 import android.view.View.OnClickListener;
@@ -43,6 +47,7 @@ import android.widget.BaseAdapter;
 import android.widget.Button;
 import android.widget.GridView;
 import android.widget.ImageView;
+import android.widget.RelativeLayout;
 import android.widget.Toast;
 import java.io.IOException;
 import java.io.InputStream;
@@ -381,6 +386,27 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
     grid = (GridView) findViewById(R.id.grid_layout);
     grid.setAdapter(adapter);
     grid.setOnTouchListener(gridTouchAdapter);
+
+    // Change UI on Android TV
+    UiModeManager uiModeManager = (UiModeManager) getSystemService(UI_MODE_SERVICE);
+    if (uiModeManager.getCurrentModeType() == Configuration.UI_MODE_TYPE_TELEVISION) {
+      DisplayMetrics displayMetrics = new DisplayMetrics();
+      getWindowManager().getDefaultDisplay().getMetrics(displayMetrics);
+      int styleSelectorHeight = displayMetrics.heightPixels;
+      int styleSelectorWidth = displayMetrics.widthPixels - styleSelectorHeight;
+      RelativeLayout.LayoutParams layoutParams = new RelativeLayout.LayoutParams(styleSelectorWidth, ViewGroup.LayoutParams.MATCH_PARENT);
+
+      // Calculate number of style in a row, so all the style can show up without scrolling
+      int numOfStylePerRow = 3;
+      while (styleSelectorWidth / numOfStylePerRow * Math.ceil((float) (adapter.getCount() - 2) / numOfStylePerRow) > styleSelectorHeight) {
+        numOfStylePerRow++;
+      }
+      grid.setNumColumns(numOfStylePerRow);
+      layoutParams.addRule(RelativeLayout.ALIGN_PARENT_RIGHT);
+      grid.setLayoutParams(layoutParams);
+      adapter.buttons.clear();
+    }
+
     setStyle(adapter.items[0], 1.0f);
   }
 
@@ -602,4 +628,38 @@ public class StylizeActivity extends CameraActivity implements OnImageAvailableL
 
     borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines);
   }
+
+  @Override
+  public boolean onKeyDown(int keyCode, KeyEvent event) {
+    int moveOffset = 0;
+    switch (keyCode) {
+      case KeyEvent.KEYCODE_DPAD_LEFT:
+        moveOffset = -1;
+        break;
+      case KeyEvent.KEYCODE_DPAD_RIGHT:
+        moveOffset = 1;
+        break;
+      case KeyEvent.KEYCODE_DPAD_UP:
+        moveOffset = -1 * grid.getNumColumns();
+        break;
+      case KeyEvent.KEYCODE_DPAD_DOWN:
+        moveOffset = grid.getNumColumns();
+        break;
+      default:
+        return super.onKeyDown(keyCode, event);
+    }
+
+    // get the highest selected style
+    int currentSelect = 0;
+    float highestValue = 0;
+    for (int i = 0; i < adapter.getCount(); i++) {
+      if (adapter.items[i].value > highestValue) {
+        currentSelect = i;
+        highestValue = adapter.items[i].value;
+      }
+    }
+    setStyle(adapter.items[(currentSelect + moveOffset + adapter.getCount()) % adapter.getCount()], 1);
+
+    return true;
+  }
 }
diff --git a/tensorflow/examples/ios/README.md b/tensorflow/examples/ios/README.md
index 5bdaeb43ce..5d7bd36837 100644
--- a/tensorflow/examples/ios/README.md
+++ b/tensorflow/examples/ios/README.md
@@ -119,11 +119,13 @@ rundown:
    `tensorflow/contrib/makefile/gen/lib` to the Library Search Paths setting.
 
  - You'll also need to add `libprotobuf.a` and `libprotobuf-lite.a` from
-   `tensorflow/contrib/makefile/gen/protobuf_ios/lib` to your _Build Stages_ and
-   _Library Search Paths_.
+   `tensorflow/contrib/makefile/gen/protobuf_ios/lib`
+   and `nsync.a` from `tensorflow/contrib/makefile/downloads/nsync/builds/lipo.ios.c++11` 
+   to your _Build Stages_ and _Library Search Paths_.
 
  - The _Header Search_ paths needs to contain:
    - the root folder of tensorflow,
+   - `tensorflow/contrib/makefile/downloads/nsync/public`
    - `tensorflow/contrib/makefile/downloads/protobuf/src`
    - `tensorflow/contrib/makefile/downloads`,
    - `tensorflow/contrib/makefile/downloads/eigen`, and
diff --git a/tensorflow/examples/learn/mnist.py b/tensorflow/examples/learn/mnist.py
index 98819b20bf..3ead8614b6 100644
--- a/tensorflow/examples/learn/mnist.py
+++ b/tensorflow/examples/learn/mnist.py
@@ -61,8 +61,10 @@ def conv_model(features, labels, mode):
 
   # Densely connected layer with 1024 neurons.
   h_fc1 = tf.layers.dense(h_pool2_flat, 1024, activation=tf.nn.relu)
-  if mode == tf.estimator.ModeKeys.TRAIN:
-    h_fc1 = tf.layers.dropout(h_fc1, rate=0.5)
+  h_fc1 = tf.layers.dropout(
+      h_fc1, 
+      rate=0.5, 
+      training=(mode == tf.estimator.ModeKeys.TRAIN))
 
   # Compute logits (1 per class) and compute loss.
   logits = tf.layers.dense(h_fc1, N_DIGITS, activation=None)
diff --git a/tensorflow/examples/learn/resnet.py b/tensorflow/examples/learn/resnet.py
index 9542e55250..c00de932a8 100755
--- a/tensorflow/examples/learn/resnet.py
+++ b/tensorflow/examples/learn/resnet.py
@@ -53,6 +53,8 @@ def res_net_model(features, labels, mode):
     ndim = int(sqrt(input_shape[1]))
     x = tf.reshape(x, [-1, ndim, ndim, 1])
 
+  training = (mode == tf.estimator.ModeKeys.TRAIN)
+  
   # First convolution expands to 64 channels
   with tf.variable_scope('conv_layer1'):
     net = tf.layers.conv2d(
@@ -60,7 +62,7 @@ def res_net_model(features, labels, mode):
         filters=64,
         kernel_size=7,
         activation=tf.nn.relu)
-    net = tf.layers.batch_normalization(net)
+    net = tf.layers.batch_normalization(net, training=training)
 
   # Max pool
   net = tf.layers.max_pooling2d(
@@ -88,7 +90,7 @@ def res_net_model(features, labels, mode):
             kernel_size=1,
             padding='valid',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       with tf.variable_scope(name + '/conv_bottleneck'):
         conv = tf.layers.conv2d(
@@ -97,7 +99,7 @@ def res_net_model(features, labels, mode):
             kernel_size=3,
             padding='same',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       # 1x1 convolution responsible for restoring dimension
       with tf.variable_scope(name + '/conv_out'):
@@ -108,7 +110,7 @@ def res_net_model(features, labels, mode):
             kernel_size=1,
             padding='valid',
             activation=tf.nn.relu)
-        conv = tf.layers.batch_normalization(conv)
+        conv = tf.layers.batch_normalization(conv, training=training)
 
       # shortcut connections that turn the network into its counterpart
       # residual function (identity shortcut)
@@ -154,7 +156,7 @@ def res_net_model(features, labels, mode):
   loss = tf.losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
 
   # Create training op.
-  if mode == tf.estimator.ModeKeys.TRAIN:
+  if training:
     optimizer = tf.train.AdagradOptimizer(learning_rate=0.01)
     train_op = optimizer.minimize(loss, global_step=tf.train.get_global_step())
     return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index d11ee6f74c..54e944c264 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -86,7 +86,6 @@ py_library(
         ":ops",
         ":platform",
         ":pywrap_tensorflow",
-        ":saver_test_utils",
         ":script_ops",
         ":session_ops",
         ":sets",
@@ -96,14 +95,15 @@ py_library(
         ":standard_ops",
         ":state_ops",
         ":string_ops",
-        ":subscribe",
         ":summary",
         ":tensor_array_ops",
+        ":training",
+        ":saver_test_utils",
+        ":subscribe",
         ":test_ops",  # TODO: Break testing code out into separate rule.
-        ":tf_cluster",
         ":tf_item",
+        ":tf_cluster",
         ":tf_optimizer",
-        ":training",
         ":util",
         ":weights_broadcast_ops",
         "//third_party/py/numpy",
@@ -3971,7 +3971,11 @@ py_test(
     srcs = ["training/checkpoint_utils_test.py"],
     srcs_version = "PY2AND3",
     tags = [
+        "manual",
+        "no_cuda_on_cpu_tap",
+        "no_oss",
         "no_windows",
+        "notap",
     ],
     deps = [
         ":client",
diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index 9641b8b7f2..5e6b5acdb0 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -155,9 +155,12 @@ class TimelineTest(test.TestCase):
     ctf = step_analysis.chrome_trace.format_to_string()
     self._validateTrace(ctf)
     maximums = step_analysis.allocator_maximums
-    self.assertTrue('cpu' in maximums)
+    cpuname = 'cpu'
+    if 'mklcpu' in maximums:
+      cpuname = 'mkl' + cpuname
+    self.assertTrue(cpuname in maximums)
     cpu_max = maximums[
-        'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums['cpu']
+        'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums[cpuname]
     # At least num1 + num2, both float32s (4 bytes each)
     self.assertGreater(cpu_max.num_bytes, 8)
     self.assertGreater(cpu_max.timestamp, 0)
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 9fcbd4ff77..6a4132bca2 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -139,8 +139,8 @@ class Estimator(object):
                  to configure Estimators from hyper parameter tuning.
           * `config`: Optional configuration object. Will receive what is passed
                  to Estimator in `config` parameter, or the default `config`.
-                 Allows updating things in your model_fn based on configuration
-                 such as `num_ps_replicas`, or `model_dir`.
+                 Allows updating things in your `model_fn` based on
+                 configuration such as `num_ps_replicas`, or `model_dir`.
 
         * Returns:
           `EstimatorSpec`
@@ -301,11 +301,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
+          * A tuple (features, labels): Where `features` is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and `labels` is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            features and labels are consumed by `model_fn`. They should satisfy
-            the expectation of `model_fn` from inputs.
+            `features` and `labels` are consumed by `model_fn`. They should
+            satisfy the expectation of `model_fn` from inputs.
 
       hooks: List of `SessionRunHook` subclass instances. Used for callbacks
         inside the training loop.
@@ -381,11 +381,11 @@ class Estimator(object):
 
           * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
             tuple (features, labels) with same constraints as below.
-          * A tuple (features, labels): Where features is a `Tensor` or a
-            dictionary of string feature name to `Tensor` and labels is a
+          * A tuple (features, labels): Where `features` is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and `labels` is a
             `Tensor` or a dictionary of string label name to `Tensor`. Both
-            features and labels are consumed by `model_fn`. They should satisfy
-            the expectation of `model_fn` from inputs.
+            `features` and `labels` are consumed by `model_fn`. They should
+            satisfy the expectation of `model_fn` from inputs.
 
       steps: Number of steps for which to evaluate model. If `None`, evaluates
         until `input_fn` raises an end-of-input exception.
@@ -457,17 +457,17 @@ class Estimator(object):
       checkpoint_path: Path of a specific checkpoint to predict. If `None`, the
         latest checkpoint in `model_dir` is used.
       yield_single_examples: If False, yield the whole batch as returned by the
-        model_fn instead of decomposing the batch into individual elements. This
-        is useful if model_fn return some tensor with first dimension not
-        equal to the batch size
+        `model_fn` instead of decomposing the batch into individual elements.
+        This is useful if `model_fn` returns some tensors whose first dimension
+        is not equal to the batch size.
 
     Yields:
       Evaluated values of `predictions` tensors.
 
     Raises:
-      ValueError: Could not find a trained model in model_dir.
-      ValueError: if batch length of predictions are not same and
-        yield_single_examples is True.
+      ValueError: Could not find a trained model in `model_dir`.
+      ValueError: If batch length of predictions is not the same and
+        `yield_single_examples` is True.
       ValueError: If there is a conflict between `predict_keys` and
         `predictions`. For example if `predict_keys` is not `None` but
         `EstimatorSpec.predictions` is not a `dict`.
@@ -849,7 +849,7 @@ class Estimator(object):
                   'loss': estimator_spec.loss,
                   'step': global_step_tensor
               },
-              every_n_iter=100)
+              every_n_iter=self._config.log_step_count_steps)
       ])
       worker_hooks.extend(estimator_spec.training_hooks)
 
diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 62f035bce5..820fda7765 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -423,7 +423,7 @@ class RunConfig(object):
         to be saved. The default value of 10,000 hours effectively disables
         the feature.
       log_step_count_steps: The frequency, in number of global steps, that the
-        global step/sec will be logged during training.
+        global step/sec and the loss will be logged during training.
 
 
     Raises:
diff --git a/tensorflow/python/estimator/training.py b/tensorflow/python/estimator/training.py
index 2cc3331a15..e38b765da5 100644
--- a/tensorflow/python/estimator/training.py
+++ b/tensorflow/python/estimator/training.py
@@ -128,9 +128,16 @@ class TrainSpec(
     """Creates a validated `TrainSpec` instance.
 
     Args:
-      input_fn: Training input function returning a tuple of:
-          features - `Tensor` or dictionary of string feature name to `Tensor`.
-          labels - `Tensor` or dictionary of `Tensor` with labels.
+      input_fn: A function that provides input data for training as minibatches.
+        See @{$get_started/premade_estimators#create_input_functions} for more
+        information. The function should construct and return one of
+        the following:
+          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
+            tuple (features, labels) with same constraints as below.
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
+            `Tensor` or a dictionary of string label name to `Tensor`.
+            
       max_steps: Int. Positive number of total steps for which to train model.
         If `None`, train forever. The training `input_fn` is not expected to
         generate `OutOfRangeError` or `StopIteration` exceptions. See the
@@ -185,9 +192,16 @@ class EvalSpec(
     """Creates a validated `EvalSpec` instance.
 
     Args:
-      input_fn: Evaluation input function returning a tuple of:
-          features - `Tensor` or dictionary of string feature name to `Tensor`.
-          labels - `Tensor` or dictionary of `Tensor` with labels.
+      input_fn: A function that constructs the input data for evaluation.
+        See @{$get_started/premade_estimators#create_input_functions} for more
+        information. The function should construct and return one of
+        the following:
+          * A 'tf.data.Dataset' object: Outputs of `Dataset` object must be a
+            tuple (features, labels) with same constraints as below.
+          * A tuple (features, labels): Where features is a `Tensor` or a
+            dictionary of string feature name to `Tensor` and labels is a
+            `Tensor` or a dictionary of string label name to `Tensor`.
+            
       steps: Int. Positive number of steps for which to evaluate model. If
         `None`, evaluates until `input_fn` raises an end-of-input exception.
         See `Estimator.evaluate` for details.
diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py
index 57506f9aff..4acb41553e 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training.py
@@ -266,7 +266,7 @@ class Model(Network):
     # initialization for Eager mode execution
     if context.executing_eagerly():
       if target_tensors is not None:
-        raise ValueError('target_tensors are not currently supported in Eager'
+        raise ValueError('target_tensors are not currently supported in Eager '
                          'mode.')
       self.total_loss = None
       self.metrics_tensors = []
diff --git a/tensorflow/python/keras/_impl/keras/layers/recurrent.py b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
index 2910719807..791f9b3113 100644
--- a/tensorflow/python/keras/_impl/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
@@ -546,8 +546,8 @@ class RNN(Layer):
         raise ValueError('The initial state or constants of an RNN'
                          ' layer cannot be specified with a mix of'
                          ' Keras tensors and non-Keras tensors'
-                         '(a "Keras tensor" is a tensor that was'
-                         'returned by a Keras layer, or by `Input`)')
+                         ' (a "Keras tensor" is a tensor that was'
+                         ' returned by a Keras layer, or by `Input`)')
 
     if is_keras_tensor:
       # Compute the full input spec, including state and constants
diff --git a/tensorflow/python/keras/_impl/keras/utils/generic_utils.py b/tensorflow/python/keras/_impl/keras/utils/generic_utils.py
index 5196bf1740..3bbe87f92d 100644
--- a/tensorflow/python/keras/_impl/keras/utils/generic_utils.py
+++ b/tensorflow/python/keras/_impl/keras/utils/generic_utils.py
@@ -490,8 +490,8 @@ def slice_arrays(arrays, start=None, stop=None):
   if arrays is None:
     return [None]
   if isinstance(start, list) and stop is not None:
-    raise ValueError('The stop argument has to be None if the value of start is'
-                     'a list.')
+    raise ValueError('The stop argument has to be None if the value of start '
+                     'is a list.')
   elif isinstance(arrays, list):
     if hasattr(start, '__len__'):
       # hdf5 datasets only support list objects as indices
diff --git a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
index 45c1b92075..4761cece82 100644
--- a/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
+++ b/tensorflow/python/keras/_impl/keras/utils/vis_utils.py
@@ -120,7 +120,7 @@ def model_to_dot(model, show_shapes=False, show_layer_names=True, rankdir='TB'):
     layer_id = str(id(layer))
     for i, node in enumerate(layer._inbound_nodes):
       node_key = layer.name + '_ib-' + str(i)
-      if node_key in model._container_nodes:
+      if node_key in model._network_nodes:  # pylint: disable=protected-access
         for inbound_layer in node.inbound_layers:
           inbound_layer_id = str(id(inbound_layer))
           layer_id = str(id(layer))
diff --git a/tensorflow/python/kernel_tests/concat_op_test.py b/tensorflow/python/kernel_tests/concat_op_test.py
index 81c6a4aa6e..c22934ce47 100644
--- a/tensorflow/python/kernel_tests/concat_op_test.py
+++ b/tensorflow/python/kernel_tests/concat_op_test.py
@@ -606,6 +606,17 @@ class ConcatOpTest(test.TestCase):
           inp_tensors_placeholders, -2, output_shape=[2, 3],
           gather_indexes=[2, 0], feed_dict=feed_dict)
 
+  def testConcatAxisType(self):
+    for dtype in [dtypes.int32, dtypes.int64]:
+      with self.test_session(use_gpu=True):
+        t1 = [[1, 2, 3], [4, 5, 6]]
+        t2 = [[7, 8, 9], [10, 11, 12]]
+
+        c = gen_array_ops.concat_v2([t1, t2],
+                                    constant_op.constant(1, dtype=dtype))
+        self.assertEqual([2, 6], c.get_shape().as_list())
+        output = c.eval()
+        self.assertAllEqual([[1, 2, 3, 7, 8, 9], [4, 5, 6, 10, 11, 12]], output)
 
 class ConcatOffsetTest(test.TestCase):
 
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index f4fe01f868..25525cc128 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -970,7 +970,7 @@ class Conv2DTest(test.TestCase):
       self.assertArrayNear(value_2.flatten(), value.flatten(), err)
 
   def testConv2D2x2Depth3ValidBackpropFilterStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 6, 1],
@@ -984,7 +984,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -998,7 +998,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropFilterDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1012,7 +1012,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 4, 3],
@@ -1026,7 +1026,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropFilterDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropFilterDilation(
             input_sizes=[1, 3, 3, 1],
@@ -1040,7 +1040,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputStride1x1Dilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 6, 1],
@@ -1054,7 +1054,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth1ValidBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 2, 3, 1],
@@ -1068,7 +1068,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2DEmptyBackpropInputDilation1x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[0, 2, 3, 1],
@@ -1082,7 +1082,7 @@ class Conv2DTest(test.TestCase):
             err=1e-5)
 
   def testConv2D2x2Depth3ValidBackpropInputDilation2x1(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         # The GPU version of this test is not very stable. So adjusting the
         # error threshold to 1e-4.
@@ -1098,7 +1098,7 @@ class Conv2DTest(test.TestCase):
             err=1e-4)
 
   def testConv2DKernelSizeMatchesInputSizeBackpropInputDilation2x2(self):
-    if test.is_gpu_available(cuda_only=True):
+    if test.is_gpu_available(cuda_only=True) or test_util.IsMklEnabled():
       for (data_format, use_gpu) in GetTestConfigs():
         self._RunAndVerifyBackpropInputDilation(
             input_sizes=[1, 3, 3, 1],
diff --git a/tensorflow/python/kernel_tests/depthtospace_op_test.py b/tensorflow/python/kernel_tests/depthtospace_op_test.py
index 96c9718b83..f0beabb4e2 100644
--- a/tensorflow/python/kernel_tests/depthtospace_op_test.py
+++ b/tensorflow/python/kernel_tests/depthtospace_op_test.py
@@ -35,8 +35,8 @@ from tensorflow.python.platform import tf_logging
 
 class DepthToSpaceTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs):
-    input_nhwc = math_ops.to_float(inputs)
+  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
+    input_nhwc = math_ops.cast(inputs, dtype)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.depth_to_space(input_nhwc, block_size)
@@ -59,6 +59,12 @@ class DepthToSpaceTest(test.TestCase):
     x_out = [[[[1], [2]], [[3], [4]]]]
     self._testOne(x_np, block_size, x_out)
 
+  def testBasicFloat16(self):
+    x_np = [[[[1, 2, 3, 4]]]]
+    block_size = 2
+    x_out = [[[[1], [2]], [[3], [4]]]]
+    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
+
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testBlockSize2(self):
diff --git a/tensorflow/python/kernel_tests/spacetodepth_op_test.py b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
index b76135764f..cd90d16aac 100644
--- a/tensorflow/python/kernel_tests/spacetodepth_op_test.py
+++ b/tensorflow/python/kernel_tests/spacetodepth_op_test.py
@@ -34,8 +34,8 @@ from tensorflow.python.platform import tf_logging
 
 class SpaceToDepthTest(test.TestCase):
 
-  def _testOne(self, inputs, block_size, outputs):
-    input_nhwc = math_ops.to_float(inputs)
+  def _testOne(self, inputs, block_size, outputs, dtype=dtypes.float32):
+    input_nhwc = math_ops.cast(inputs, dtype)
     with self.test_session(use_gpu=False):
       # test NHWC (default) on CPU
       x_tf = array_ops.space_to_depth(input_nhwc, block_size)
@@ -58,6 +58,12 @@ class SpaceToDepthTest(test.TestCase):
     x_out = [[[[1, 2, 3, 4]]]]
     self._testOne(x_np, block_size, x_out)
 
+  def testBasicFloat16(self):
+    x_np = [[[[1], [2]], [[3], [4]]]]
+    block_size = 2
+    x_out = [[[[1, 2, 3, 4]]]]
+    self._testOne(x_np, block_size, x_out, dtype=dtypes.float16)
+
   # Tests for larger input dimensions. To make sure elements are
   # correctly ordered spatially.
   def testLargerInput2x2(self):
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index e9066d3fda..e4395bea92 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -578,7 +578,7 @@ class Layer(checkpointable.CheckpointableBase):
           if isinstance(variable, tf_variables.PartitionedVariable):
             raise RuntimeError(
                 'Partitioned variable regularization is not yet '
-                'supported when executing eagerly. File a feature request'
+                'supported when executing eagerly. File a feature request '
                 'if this is important to you.')
           # Save a zero-argument lambda which runs the regularizer on the
           # variable, to be executed when `Layer.losses` is requested.
diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 11daf01670..29fb92ccb5 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -664,9 +664,16 @@ def batch_normalization(inputs,
 
   Note: when training, the moving_mean and moving_variance need to be updated.
   By default the update ops are placed in `tf.GraphKeys.UPDATE_OPS`, so they
-  need to be added as a dependency to the `train_op`. For example:
+  need to be added as a dependency to the `train_op`. Also, be sure to add
+  any batch_normalization ops before getting the update_ops collection.
+  Otherwise, update_ops will be empty, and training/inference will not work
+  properly. For example:
 
   ```python
+    x_norm = tf.layers.batch_normalization(x, training=training)
+
+    # ...
+
     update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
     with tf.control_dependencies(update_ops):
       train_op = optimizer.minimize(loss)
diff --git a/tensorflow/python/lib/io/file_io_test.py b/tensorflow/python/lib/io/file_io_test.py
index a751607aaa..223858edfa 100644
--- a/tensorflow/python/lib/io/file_io_test.py
+++ b/tensorflow/python/lib/io/file_io_test.py
@@ -485,6 +485,11 @@ class FileIoTest(test.TestCase):
     f.flush()
     self.assertEqual(content, f.read(len(content) + 1))
 
+  def testUTF8StringPathExists(self):
+    file_path = os.path.join(self._base_dir, "UTF8测试_file_exist")
+    file_io.write_string_to_file(file_path, "testing")
+    v = file_io.file_exists(file_path)
+    self.assertEqual(v, True)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py
index 48ea107a14..6fcf9c91d8 100644
--- a/tensorflow/python/lib/io/tf_record.py
+++ b/tensorflow/python/lib/io/tf_record.py
@@ -75,14 +75,16 @@ def tf_record_iterator(path, options=None):
 
   if reader is None:
     raise IOError("Could not open %s." % path)
-  while True:
-    try:
-      with errors.raise_exception_on_not_ok_status() as status:
-        reader.GetNext(status)
-    except errors.OutOfRangeError:
-      break
-    yield reader.record()
-  reader.Close()
+  try:
+    while True:
+      try:
+        with errors.raise_exception_on_not_ok_status() as status:
+          reader.GetNext(status)
+      except errors.OutOfRangeError:
+        break
+      yield reader.record()
+  finally:
+    reader.Close()
 
 
 @tf_export("python_io.TFRecordWriter")
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 37470e00d7..5b4fb4f7c8 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -341,7 +341,7 @@ def self_adjoint_eig(tensor, name=None):
     name: string, optional name of the operation.
 
   Returns:
-    e: Eigenvalues. Shape is `[..., N]`.
+    e: Eigenvalues. Shape is `[..., N]`. Sorted in non-decreasing order.
     v: Eigenvectors. Shape is `[..., N, N]`. The columns of the inner most
       matrices contain eigenvectors of the corresponding matrices in `tensor`
   """
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index fb3fe77b4d..a74de39eab 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -150,14 +150,12 @@ class _NonAtrousConvolution(object):
                                                               conv_dims))
     if conv_dims == 1:
       # conv1d uses the 2-d data format names
-      if data_format is None or data_format == "NWC":
-        data_format_2d = "NHWC"
-      elif data_format == "NCW":
-        data_format_2d = "NCHW"
-      else:
+      if data_format is None:
+        data_format = "NWC"
+      elif data_format not in {"NCW", "NWC", "NCHW", "NHWC"}:
         raise ValueError("data_format must be \"NWC\" or \"NCW\".")
       self.strides = strides[0]
-      self.data_format = data_format_2d
+      self.data_format = data_format
       self.conv_op = self._conv1d
     elif conv_dims == 2:
       if data_format is None or data_format == "NHWC":
@@ -699,7 +697,7 @@ def convolution(
   `padded_input` is obtained by zero padding the input using an effective
   spatial filter shape of `(spatial_filter_shape-1) * dilation_rate + 1` and
   output striding `strides` as described in the
-  @{tf.nn.convolution$comment here}.
+  @{$python/nn#Convolution$comment here}.
 
   In the case that `data_format` does start with `"NC"`, the `input` and output
   (but not the `filter`) are simply transposed as follows:
@@ -1043,9 +1041,7 @@ def pool(
 
 @tf_export("nn.atrous_conv2d")
 def atrous_conv2d(value, filters, rate, padding, name=None):
-  """Atrous convolution (a.k.a.
-
-  convolution with holes or dilated convolution).
+  """Atrous convolution (a.k.a. convolution with holes or dilated convolution).
 
   This function is a simpler wrapper around the more general
   @{tf.nn.convolution}, and exists only for backwards compatibility. You can
diff --git a/tensorflow/python/ops/random_ops.py b/tensorflow/python/ops/random_ops.py
index db8159579a..6a2dd3f1cd 100644
--- a/tensorflow/python/ops/random_ops.py
+++ b/tensorflow/python/ops/random_ops.py
@@ -209,7 +209,7 @@ def random_uniform(shape,
     maxval: A 0-D Tensor or Python value of type `dtype`. The upper bound on
       the range of random values to generate.  Defaults to 1 if `dtype` is
       floating point.
-    dtype: The type of the output: 'float16`, `float32`, `float64`, `int32`,
+    dtype: The type of the output: `float16`, `float32`, `float64`, `int32`,
       or `int64`.
     seed: A Python integer. Used to create a random seed for the distribution.
       See @{tf.set_random_seed}
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index c59eccc174..42af7f8b27 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -867,7 +867,7 @@ def raw_rnn(cell, loop_fn,
 
   ```python
   time = tf.constant(0, dtype=tf.int32)
-  (finished, next_input, initial_state, _, loop_state) = loop_fn(
+  (finished, next_input, initial_state, emit_structure, loop_state) = loop_fn(
       time=time, cell_output=None, cell_state=None, loop_state=None)
   emit_ta = TensorArray(dynamic_size=True, dtype=initial_state.dtype)
   state = initial_state
@@ -878,7 +878,7 @@ def raw_rnn(cell, loop_fn,
         loop_state=loop_state)
     # Emit zeros and copy forward state for minibatch entries that are finished.
     state = tf.where(finished, state, next_state)
-    emit = tf.where(finished, tf.zeros_like(emit), emit)
+    emit = tf.where(finished, tf.zeros_like(emit_structure), emit)
     emit_ta = emit_ta.write(time, emit)
     # If any new minibatch entries are marked as finished, mark these.
     finished = tf.logical_or(finished, next_finished)
@@ -938,10 +938,15 @@ def raw_rnn(cell, loop_fn,
       and `emit_output`: the output to store for this iteration.
 
       Note that `emit_output` should be a `Tensor` or (possibly nested)
-      tuple of tensors with shapes and structure matching `cell.output_size`
-      and `cell_output` above.  The parameter `cell_state` and output
-      `next_cell_state` may be either a single or (possibly nested) tuple
-      of tensors.  The parameter `loop_state` and
+      tuple of tensors which is aggregated in the `emit_ta` inside the
+      `while_loop`. For the first call to `loop_fn`, the `emit_output`
+      corresponds to the `emit_structure` which is then used to determine the
+      size of the `zero_tensor` for the `emit_ta` (defaults to
+      `cell.output_size`). For the subsequent calls to the `loop_fn`, the
+      `emit_output` corresponds to the actual output tensor
+      that is to be aggregated in the `emit_ta`. The parameter `cell_state`
+      and output `next_cell_state` may be either a single or (possibly nested)
+      tuple of tensors.  The parameter `loop_state` and
       output `next_loop_state` may be either a single or (possibly nested) tuple
       of `Tensor` and `TensorArray` objects.  This last parameter
       may be ignored by `loop_fn` and the return value may be `None`.  If it
diff --git a/tensorflow/python/ops/special_math_ops.py b/tensorflow/python/ops/special_math_ops.py
index 6d7eaababc..5e2146b79f 100644
--- a/tensorflow/python/ops/special_math_ops.py
+++ b/tensorflow/python/ops/special_math_ops.py
@@ -163,7 +163,7 @@ def einsum(equation, *inputs, **kwargs):
     if '...' in equation:
       raise ValueError('Subscripts with ellipses are not yet supported.')
 
-    match = re.match('([a-z,]+)(->[a-z]*)?', equation)
+    match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
     if not match:
       raise ValueError('Indices have incorrect format: %s' % equation)
 
@@ -402,7 +402,7 @@ def _exponential_space_einsum(equation, *inputs):
   if '...' in equation:
     raise ValueError('Subscripts with ellipses are not yet supported.')
 
-  match = re.match('([a-z,]+)(->[a-z]*)?', equation)
+  match = re.match('^([a-zA-Z,]+)(->[a-zA-Z]*)?$', equation)
   if not match:
     raise ValueError('Indices have incorrect format: %s' % equation)
 
diff --git a/tensorflow/python/ops/special_math_ops_test.py b/tensorflow/python/ops/special_math_ops_test.py
index 2c212f4548..d7c3a7e8dc 100644
--- a/tensorflow/python/ops/special_math_ops_test.py
+++ b/tensorflow/python/ops/special_math_ops_test.py
@@ -192,6 +192,9 @@ class EinsumTest(test.TestCase):
       'abc,cba',
       'dba,ead,cad->bce',
       'aef,fbc,dca->bde',
+      'iJ,Jk->ik',
+      'iJ,Ki->JK',
+      'iJk,Jklm->Jk'
   ]
 
   long_cases = [
@@ -208,6 +211,8 @@ class EinsumTest(test.TestCase):
       'ijk ijk',
       'ij.jk->ik',
       'ij...,jk...->ik...',
+      'ij,k ->kji',
+      'ij,k-> kji',
 
       # axis in output that does not exist
       'ij,jk->im',
diff --git a/tensorflow/python/tools/freeze_graph.py b/tensorflow/python/tools/freeze_graph.py
index a52f325ddb..e9f1def48c 100644
--- a/tensorflow/python/tools/freeze_graph.py
+++ b/tensorflow/python/tools/freeze_graph.py
@@ -56,8 +56,6 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.tools import saved_model_utils
 from tensorflow.python.training import saver as saver_lib
 
-FLAGS = None
-
 
 def freeze_graph_with_def_protos(input_graph_def,
                                  input_saver_def,
@@ -256,25 +254,24 @@ def freeze_graph(input_graph,
       checkpoint_version=checkpoint_version)
 
 
-def main(unused_args):
-  if FLAGS.checkpoint_version == 1:
+def main(unused_args, flags):
+  if flags.checkpoint_version == 1:
     checkpoint_version = saver_pb2.SaverDef.V1
-  elif FLAGS.checkpoint_version == 2:
+  elif flags.checkpoint_version == 2:
     checkpoint_version = saver_pb2.SaverDef.V2
   else:
     print("Invalid checkpoint version (must be '1' or '2'): %d" %
-          FLAGS.checkpoint_version)
+          flags.checkpoint_version)
     return -1
-  freeze_graph(FLAGS.input_graph, FLAGS.input_saver, FLAGS.input_binary,
-               FLAGS.input_checkpoint, FLAGS.output_node_names,
-               FLAGS.restore_op_name, FLAGS.filename_tensor_name,
-               FLAGS.output_graph, FLAGS.clear_devices, FLAGS.initializer_nodes,
-               FLAGS.variable_names_whitelist, FLAGS.variable_names_blacklist,
-               FLAGS.input_meta_graph, FLAGS.input_saved_model_dir,
-               FLAGS.saved_model_tags, checkpoint_version)
-
+  freeze_graph(flags.input_graph, flags.input_saver, flags.input_binary,
+               flags.input_checkpoint, flags.output_node_names,
+               flags.restore_op_name, flags.filename_tensor_name,
+               flags.output_graph, flags.clear_devices, flags.initializer_nodes,
+               flags.variable_names_whitelist, flags.variable_names_blacklist,
+               flags.input_meta_graph, flags.input_saved_model_dir,
+               flags.saved_model_tags, checkpoint_version)
 
-if __name__ == "__main__":
+def run_main():
   parser = argparse.ArgumentParser()
   parser.register("type", "bool", lambda v: v.lower() == "true")
   parser.add_argument(
@@ -376,5 +373,10 @@ if __name__ == "__main__":
       separated by \',\'. For tag-set contains multiple tags, all tags \
       must be passed in.\
       """)
-  FLAGS, unparsed = parser.parse_known_args()
-  app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  flags, unparsed = parser.parse_known_args()
+
+  my_main = lambda unused_args: main(unused_args, flags)
+  app.run(main=my_main, argv=[sys.argv[0]] + unparsed)
+
+if __name__ == '__main__':
+  run_main()
diff --git a/tensorflow/python/tools/inspect_checkpoint.py b/tensorflow/python/tools/inspect_checkpoint.py
index dd876cbe7f..6504fbc107 100644
--- a/tensorflow/python/tools/inspect_checkpoint.py
+++ b/tensorflow/python/tools/inspect_checkpoint.py
@@ -30,7 +30,7 @@ FLAGS = None
 
 
 def print_tensors_in_checkpoint_file(file_name, tensor_name, all_tensors,
-                                     all_tensor_names):
+                                     all_tensor_names=False):
   """Prints tensors in a checkpoint file.
 
   If no `tensor_name` is provided, prints the tensor names and shapes
@@ -139,7 +139,7 @@ if __name__ == "__main__":
       const=True,
       type="bool",
       default=False,
-      help="If True, print the values of all the tensors.")
+      help="If True, print the names and values of all the tensors.")
   parser.add_argument(
       "--all_tensor_names",
       nargs="?",
diff --git a/tensorflow/python/tools/saved_model_cli.py b/tensorflow/python/tools/saved_model_cli.py
index b0e9e3e5ed..b88be4ae04 100644
--- a/tensorflow/python/tools/saved_model_cli.py
+++ b/tensorflow/python/tools/saved_model_cli.py
@@ -38,11 +38,15 @@ from tensorflow.core.example import example_pb2
 from tensorflow.core.framework import types_pb2
 from tensorflow.python.client import session
 from tensorflow.python.debug.wrappers import local_cli_wrapper
+from tensorflow.python.framework import meta_graph as meta_graph_lib
 from tensorflow.python.framework import ops as ops_lib
 from tensorflow.python.platform import app  # pylint: disable=unused-import
 from tensorflow.python.saved_model import loader
 from tensorflow.python.tools import saved_model_utils
 
+# Set of ops to blacklist.
+_OP_BLACKLIST = set(['WriteFile', 'ReadFile'])
+
 
 def _show_tag_sets(saved_model_dir):
   """Prints the tag-sets stored in SavedModel directory.
@@ -242,6 +246,27 @@ def get_signature_def_map(saved_model_dir, tag_set):
   return meta_graph.signature_def
 
 
+def scan_meta_graph_def(meta_graph_def):
+  """Scans meta_graph_def and reports if there are ops on blacklist.
+
+  Print ops if they are on black list, or print success if no blacklisted ops
+  found.
+
+  Args:
+    meta_graph_def: MetaGraphDef protocol buffer.
+  """
+  all_ops_set = set(
+      meta_graph_lib.ops_used_by_graph_def(meta_graph_def.graph_def))
+  blacklisted_ops = _OP_BLACKLIST & all_ops_set
+  if blacklisted_ops:
+    # TODO(yifeif): print more warnings
+    print('MetaGraph with tag set %s contains the following blacklisted ops:' %
+          meta_graph_def.meta_info_def.tags, blacklisted_ops)
+  else:
+    print('MetaGraph with tag set %s does not contain blacklisted ops.' %
+          meta_graph_def.meta_info_def.tags)
+
+
 def run_saved_model_with_feed_dict(saved_model_dir, tag_set, signature_def_key,
                                    input_tensor_key_feed_dict, outdir,
                                    overwrite_flag, tf_debug=False):
@@ -609,6 +634,21 @@ def run(args):
                                  args.overwrite, tf_debug=args.tf_debug)
 
 
+def scan(args):
+  """Function triggered by scan command.
+
+  Args:
+    args: A namespace parsed from command line.
+  """
+  if args.tag_set:
+    scan_meta_graph_def(
+        saved_model_utils.get_meta_graph_def(args.dir, args.tag_set))
+  else:
+    saved_model = reader.read_saved_model(args.dir)
+    for meta_graph_def in saved_model.meta_graphs:
+      scan_meta_graph_def(meta_graph_def)
+
+
 def create_parser():
   """Creates a parser that parse the command line arguments.
 
@@ -730,6 +770,26 @@ def create_parser():
            'SavedModel.')
   parser_run.set_defaults(func=run)
 
+  # scan command
+  scan_msg = ('Usage example:\n'
+              'To scan for blacklisted ops in SavedModel:\n'
+              '$saved_model_cli scan --dir /tmp/saved_model\n'
+              'To scan a specific MetaGraph, pass in --tag_set\n')
+  parser_scan = subparsers.add_parser(
+      'scan',
+      description=scan_msg,
+      formatter_class=argparse.RawTextHelpFormatter)
+  parser_scan.add_argument(
+      '--dir',
+      type=str,
+      required=True,
+      help='directory containing the SavedModel to execute')
+  parser_scan.add_argument(
+      '--tag_set',
+      type=str,
+      help='tag-set of graph in SavedModel to scan, separated by \',\'')
+  parser_scan.set_defaults(func=scan)
+
   return parser
 
 
diff --git a/tensorflow/python/tools/saved_model_cli_test.py b/tensorflow/python/tools/saved_model_cli_test.py
index f99c844845..eedc893a38 100644
--- a/tensorflow/python/tools/saved_model_cli_test.py
+++ b/tensorflow/python/tools/saved_model_cli_test.py
@@ -525,6 +525,28 @@ signature_def['serving_default']:
     y_expected = np.array([[2.5], [3.0]])
     self.assertAllClose(y_expected, y_actual)
 
+  def testScanCommand(self):
+    self.parser = saved_model_cli.create_parser()
+    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
+    args = self.parser.parse_args(['scan', '--dir', base_path])
+    with captured_output() as (out, _):
+      saved_model_cli.scan(args)
+    output = out.getvalue().strip()
+    self.assertTrue('does not contain blacklisted ops' in output)
+
+  def testScanCommandFoundBlacklistedOp(self):
+    self.parser = saved_model_cli.create_parser()
+    base_path = test.test_src_dir_path(SAVED_MODEL_PATH)
+    args = self.parser.parse_args(
+        ['scan', '--dir', base_path, '--tag_set', 'serve'])
+    op_blacklist = saved_model_cli._OP_BLACKLIST
+    saved_model_cli._OP_BLACKLIST = set(['VariableV2'])
+    with captured_output() as (out, _):
+      saved_model_cli.scan(args)
+    saved_model_cli._OP_BLACKLIST = op_blacklist
+    output = out.getvalue().strip()
+    self.assertTrue('\'VariableV2\'' in output)
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index 5ef8bd9e9c..ba0d038475 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1135,8 +1135,9 @@ class Saver(object):
   the proliferation of checkpoint files on disk:
 
   * `max_to_keep` indicates the maximum number of recent checkpoint files to
-    keep.  As new files are created, older files are deleted.  If None or 0,
-    all checkpoint files are kept.  Defaults to 5 (that is, the 5 most recent
+    keep.  As new files are created, older files are deleted.   If None or 0,
+    no checkpoints are deleted from the filesystem but only the last one is
+    kept in the `checkpoint` file.  Defaults to 5 (that is, the 5 most recent
     checkpoint files are kept.)
 
   * `keep_checkpoint_every_n_hours`: In addition to keeping the most recent
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 0b3b060fe7..03e3e0857f 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -274,7 +274,8 @@ CUDNN_DNN_ROUTINE_EACH_R6(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
 // clang-format off
 #if CUDNN_VERSION >= 7000
 #define CUDNN_DNN_ROUTINE_EACH_R7(__macro)                    \
-  __macro(cudnnSetConvolutionMathType)
+  __macro(cudnnSetConvolutionMathType)                        \
+  __macro(cudnnSetRNNMatrixMathType)
 
 // clang-format on
 CUDNN_DNN_ROUTINE_EACH_R7(PERFTOOLS_GPUTOOLS_CUDNN_WRAP)
@@ -586,6 +587,19 @@ static bool TensorOpMathEnabled() {
   return is_enabled;
 }
 
+// A helper function to decide whether to enable the TENSOR_OP_MATH math type
+// for RNNs.
+static bool RnnTensorOpMathEnabled() {
+  static bool is_enabled = [] {
+    bool is_disabled = false;
+    TF_CHECK_OK(
+        tensorflow::ReadBoolFromEnvVar("TF_DISABLE_CUDNN_RNN_TENSOR_OP_MATH",
+                                       /*default_val=*/false, &is_disabled));
+    return !is_disabled;
+  }();
+  return is_enabled;
+}
+
 // A helper function to decide whether to use CUDNN_BATCHNORM_SPATIAL_PERSISTENT
 // in batchnorm. This mode can be faster in some tasks because an optimized path
 // may be selected for CUDNN_DATA_FLOAT and CUDNN_DATA_HALF data types, compute
@@ -1124,6 +1138,9 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       SetFailure(cudnn_params_desc_->Status());
       return;
     }
+    if (data_type == CUDNN_DATA_HALF) {
+      set_use_tensor_op_math(true);
+    }
   }
   ~CudnnRnnDescriptor() override {
     if (rnn_desc_) {
@@ -1132,6 +1149,20 @@ class CudnnRnnDescriptor : public CudnnDescriptorCommon<dnn::RnnDescriptor> {
       CUDNN_RETURN_IF_FAIL(status, "Unable to destroy RNN descriptor");
     }
   }
+  void set_use_tensor_op_math(bool use_tensor_op_math) {
+#if CUDNN_VERSION >= 7000
+    cudnnMathType_t math_type =
+        (use_tensor_op_math ? CUDNN_TENSOR_OP_MATH : CUDNN_DEFAULT_MATH);
+    if (RnnTensorOpMathEnabled()) {
+      cudnnStatus_t status =
+          wrap::cudnnSetRNNMatrixMathType(parent_, rnn_desc_, math_type);
+      if (status != CUDNN_STATUS_SUCCESS) {
+        LOG(FATAL) << "could not set cudnn RNN math type: "
+                   << ToString(status);
+      }
+    }
+#endif
+  }
   cudnnRNNDescriptor_t handle() const {
     if (!ok()) return nullptr;
     return rnn_desc_;
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 99c47fd601..96f501e163 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -247,6 +247,8 @@ class ApiCompatibilityTest(test.TestCase):
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.do_not_descend_map['tf'].append('contrib')
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
+    # TODO(annarev): Make slide_dataset available in API.
+    public_api_visitor.private_map['tf'] = ['slide_dataset']
     traverse.traverse(api, public_api_visitor)
 
     proto_dict = visitor.GetProtos()
diff --git a/tensorflow/tools/ci_build/Dockerfile.cmake b/tensorflow/tools/ci_build/Dockerfile.cmake
index ec90c83aac..d5dea4f3e4 100644
--- a/tensorflow/tools/ci_build/Dockerfile.cmake
+++ b/tensorflow/tools/ci_build/Dockerfile.cmake
@@ -23,11 +23,12 @@ RUN /install/install_deb_packages.sh
 
 RUN apt-get update
 RUN apt-get install -y --no-install-recommends python-pip
+RUN pip install --upgrade wheel
 RUN pip install --upgrade astor
 RUN pip install --upgrade gast
 RUN pip install --upgrade numpy
 RUN pip install --upgrade termcolor
 
 # Install golang
-RUN add-apt-repository -y ppa:ubuntu-lxc/lxd-stable
-RUN apt-get install -y golang
+RUN apt-get install -t xenial-backports -y golang-1.9
+ENV PATH=${PATH}:/usr/lib/go-1.9/bin
diff --git a/tensorflow/tools/compatibility/tf_upgrade.py b/tensorflow/tools/compatibility/tf_upgrade.py
index 6e90b286c9..1f8833582a 100644
--- a/tensorflow/tools/compatibility/tf_upgrade.py
+++ b/tensorflow/tools/compatibility/tf_upgrade.py
@@ -662,9 +662,9 @@ class TFAPIChangeSpec(APIChangeSpec):
   def _reverse_handler(file_edit_recorder, node):
     # TODO(aselle): Could check for a literal list of bools and try to convert
     # them to indices.
-    comment = ("ERROR: tf.reverse has had its argument semantics changed\n"
-               "significantly the converter cannot detect this reliably, so you"
-               "need to inspect this usage manually.\n")
+    comment = ("ERROR: tf.reverse has had its argument semantics changed "
+               "significantly the converter cannot detect this reliably, so "
+               "you need to inspect this usage manually.\n")
     file_edit_recorder.add(
         comment,
         node.lineno,
diff --git a/tensorflow/tools/dist_test/README.md b/tensorflow/tools/dist_test/README.md
index c1b1f79bbd..228d5ee35d 100644
--- a/tensorflow/tools/dist_test/README.md
+++ b/tensorflow/tools/dist_test/README.md
@@ -17,6 +17,14 @@ cesnsu model:
 
     ./local_test.sh --model_name CENSUS_WIDENDEEP
 
+You can test specify version of TensorFlow:
+
+```shell
+./local_test.sh ${whl_file_url}
+```
+
+For example, you can find these TensorFlow python package URLs from [here](https://www.tensorflow.org/install/install_linux#the_url_of_the_tensorflow_python_package) for Ubuntu.
+
 **2) Launch a remote k8s cluster on Google Kubernetes Engine (GKE) and run the
 test suite on it**
 
diff --git a/tensorflow/tools/dist_test/local_test.sh b/tensorflow/tools/dist_test/local_test.sh
index 435f9d0dc9..caae7fd530 100755
--- a/tensorflow/tools/dist_test/local_test.sh
+++ b/tensorflow/tools/dist_test/local_test.sh
@@ -16,12 +16,11 @@
 #
 # Tests distributed TensorFlow on a locally running TF GRPC cluster.
 #
-# This script peforms the following steps:
-# 1) Build the docker-in-docker (dind) image capable of running docker and
-#    Kubernetes (k8s) cluster inside.
+# This script performs the following steps:
+# 1) Build the docker image capable of running distributed TensorFlow in docker.
 # 2) Run a container from the aforementioned image and start docker service
 #    in it
-# 3) Call a script to launch a k8s TensorFlow GRPC cluster inside the container
+# 3) Call a script to launch a distributed TensorFlow GRPC cluster inside the container
 #    and run the distributed test suite.
 #
 # Usage: local_test.sh <whl_file_location>
@@ -64,15 +63,9 @@ die() {
 
 # Configurations
 DOCKER_IMG_NAME="tensorflow/tf-dist-test-local-cluster"
-LOCAL_K8S_CACHE=${HOME}/kubernetes
 
-# Helper function
-get_container_id_by_image_name() {
-    # Get the id of a container by image name
-    # Usage: get_docker_container_id_by_image_name <img_name>
-
-    docker ps | grep $1 | awk '{print $1}'
-}
+# Use TensorFlow v1.5.0 for Python 2.7 and CPU only as we set num_gpus to 0 in the below
+DEFAULT_WHL_FILE_LOCATION="https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.5.0-cp27-none-linux_x86_64.whl"
 
 # Parse input arguments
 LEAVE_CONTAINER_RUNNING=0
@@ -84,7 +77,8 @@ SYNC_REPLICAS_FLAG=""
 
 WHL_FILE_LOCATION=${1}
 if [[ -z "${WHL_FILE_LOCATION}" ]]; then
-  die "whl file location is not specified"
+  WHL_FILE_LOCATION=${DEFAULT_WHL_FILE_LOCATION}
+  echo "use default whl file location"
 fi
 
 while true; do
@@ -121,7 +115,7 @@ DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 # Get utility functions
 source ${DIR}/scripts/utils.sh
 
-# Build docker-in-docker image for local k8s cluster.
+# Build docker image for local distributed TensorFlow cluster.
 NO_CACHE_FLAG=""
 if [[ ! -z "${TF_DIST_DOCKER_NO_CACHE}" ]] &&
    [[ "${TF_DIST_DOCKER_NO_CACHE}" != "0" ]]; then
diff --git a/tensorflow/tools/dist_test/python/mnist_replica.py b/tensorflow/tools/dist_test/python/mnist_replica.py
index a2d12442c4..d6e7f317dd 100644
--- a/tensorflow/tools/dist_test/python/mnist_replica.py
+++ b/tensorflow/tools/dist_test/python/mnist_replica.py
@@ -56,7 +56,7 @@ flags.DEFINE_integer("task_index", None,
 flags.DEFINE_integer("num_gpus", 1, "Total number of gpus for each machine."
                      "If you don't use GPU, please set it to '0'")
 flags.DEFINE_integer("replicas_to_aggregate", None,
-                     "Number of replicas to aggregate before parameter update"
+                     "Number of replicas to aggregate before parameter update "
                      "is applied (For sync_replicas mode only; default: "
                      "num_workers)")
 flags.DEFINE_integer("hidden_units", 100,
diff --git a/tensorflow/tools/docker/Dockerfile.gpu b/tensorflow/tools/docker/Dockerfile.gpu
index b6682cd681..625321e123 100644
--- a/tensorflow/tools/docker/Dockerfile.gpu
+++ b/tensorflow/tools/docker/Dockerfile.gpu
@@ -1,11 +1,18 @@
-FROM nvidia/cuda:9.0-cudnn7-runtime-ubuntu16.04
+FROM nvidia/cuda:9.0-base-ubuntu16.04
 
 LABEL maintainer="Craig Citro <craigcitro@google.com>"
 
 # Pick up some TF dependencies
 RUN apt-get update && apt-get install -y --no-install-recommends \
         build-essential \
+        cuda-command-line-tools-9-0 \
+        cuda-cublas-9-0 \
+        cuda-cufft-9-0 \
+        cuda-curand-9-0 \
+        cuda-cusolver-9-0 \
+        cuda-cusparse-9-0 \
         curl \
+        libcudnn7=7.0.5.15-1+cuda9.0 \
         libfreetype6-dev \
         libpng12-dev \
         libzmq3-dev \
diff --git a/tensorflow/tools/git/gen_git_source.py b/tensorflow/tools/git/gen_git_source.py
index 3630dbd740..cbcdbf5b80 100755
--- a/tensorflow/tools/git/gen_git_source.py
+++ b/tensorflow/tools/git/gen_git_source.py
@@ -114,6 +114,13 @@ def configure(src_base_path, gen_path, debug=False):
   for target, src in link_map.items():
     if src is None:
       open(os.path.join(gen_path, target), "w").write("")
+    elif not os.path.exists(src):
+      # Git repo is configured in a way we don't support such as having
+      # packed refs. Even though in a git repo, tf.__git_version__ will not
+      # be accurate.
+      # TODO(mikecase): Support grabbing git info when using packed refs.
+      open(os.path.join(gen_path, target), "w").write("")
+      spec["git"] = False
     else:
       try:
         # In python 3.5, symlink function exists even on Windows. But requires
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index b7d7fac315..6e21aa2846 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -178,6 +178,7 @@ tf_cc_test(
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
+        "//tensorflow/core/kernels:quantization_utils",
         "//tensorflow/core/kernels:quantized_ops",
         "//tensorflow/core/util/tensor_bundle",
     ],
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
index d89afe85c7..d86f65325b 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms.cc
@@ -182,6 +182,36 @@ Status FuseBatchNormWithConv(const NodeMatch& match,
   return Status::OK();
 }
 
+Status FuseBatchNormWithBatchToSpace(const NodeMatch& match,
+                             std::vector<NodeDef>* new_nodes) {
+  // Calculate the scale and offset values to apply.
+  std::vector<float> scale_values;
+  std::vector<float> offset_values;
+  TF_RETURN_IF_ERROR(
+      GetScaleAndOffsetValues(match, &scale_values, &offset_values));
+
+  // Fuse conv weights, and set the final output node name as batch_norm_node.
+  const NodeDef& batch_norm_node = match.node;
+  const NodeMatch& batch_to_space_node_match = match.inputs[0];
+  const NodeMatch& conv_node_match = batch_to_space_node_match.inputs[0];
+  const NodeDef& batch_to_space_node = batch_to_space_node_match.node;
+  const NodeDef& conv_node = conv_node_match.node;
+
+  string biasadd_name = conv_node.name() + "/biasadd";
+  TF_RETURN_IF_ERROR(
+      FuseScaleOffsetToConvWeights(scale_values, offset_values, conv_node_match,
+                                   biasadd_name , new_nodes));
+
+  NodeDef new_batch_to_space_node = batch_to_space_node;
+  // reuse batch_norm node name
+  new_batch_to_space_node.set_name(batch_norm_node.name());
+  new_batch_to_space_node.set_input(0, biasadd_name);
+  new_nodes->push_back(batch_to_space_node_match.inputs[1].node);
+  new_nodes->push_back(batch_to_space_node_match.inputs[2].node);
+  new_nodes->push_back(new_batch_to_space_node);
+  return Status::OK();
+}
+
 Status FuseBatchNormWithConvConcat(const NodeMatch& match,
                                    std::vector<NodeDef>* new_nodes) {
   // Calculate the scale and offset values to apply.
@@ -284,6 +314,43 @@ Status FoldOldBatchNorms(const GraphDef& input_graph_def,
     current_graph_def = replaced_graph_def;
   } while (did_graph_change);
 
+  do {
+    did_graph_change = false;
+    GraphDef replaced_graph_def;
+    TF_RETURN_IF_ERROR(ReplaceMatchingOpTypes(
+        current_graph_def,  // clang-format off
+        {"BatchNormWithGlobalNormalization|FusedBatchNorm",    // batch_norm_node
+         {
+             {"BatchToSpaceND",                  // batch_to_space_node
+              {
+                  {"Conv2D",                     // conv_node
+                   {
+                       {"*"},                    // input_node
+                       {"Const"},                // weights_node
+                   }
+                  },
+                  {"Const"},                     // block_shape
+                  {"Const"},                     // crops
+              }
+             },
+             {"Const"},                          // mean_node
+             {"Const"},                          // variance_node
+             {"Const"},                          // beta_node
+             {"Const"},                          // gamma_node
+         }
+        },  // clang-format on
+        [&did_graph_change](const NodeMatch& match,
+                            const std::set<string>& input_nodes,
+                            const std::set<string>& output_nodes,
+                            std::vector<NodeDef>* new_nodes) {
+          TF_RETURN_IF_ERROR(FuseBatchNormWithBatchToSpace(match, new_nodes));
+          did_graph_change = true;
+          return Status::OK();
+        },
+        {}, &replaced_graph_def));
+    current_graph_def = replaced_graph_def;
+  } while (did_graph_change);
+
   do {
     did_graph_change = false;
     GraphDef replaced_graph_def;
diff --git a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
index b30ba9ac8b..7651a03fe5 100644
--- a/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_old_batch_norms_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/const_op.h"
 #include "tensorflow/cc/ops/image_ops.h"
 #include "tensorflow/cc/ops/nn_ops.h"
+#include "tensorflow/cc/ops/array_ops.h"
 #include "tensorflow/cc/ops/sendrecv_ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
@@ -298,6 +299,96 @@ class FoldOldBatchNormsTest : public ::testing::Test {
   }
 };
 
+void TestFoldFusedBatchNormsWithBatchToSpace() {
+  auto root = tensorflow::Scope::NewRootScope();
+  using namespace ::tensorflow::ops;  // NOLINT(build/namespaces)
+
+  Tensor input_data(DT_FLOAT, TensorShape({2, 1, 3, 2}));
+  test::FillValues<float>(
+      &input_data, {1.0f, 4.0f, 2.0f, 5.0f, 3.0f, 6.0f, -1.0f, -4.0f, -2.0f,
+                    -5.0f, -3.0f, -6.0f});
+  Output input_op =
+      Const(root.WithOpName("input_op"), Input::Initializer(input_data));
+
+  Tensor weights_data(DT_FLOAT, TensorShape({1, 2, 2, 2}));
+  test::FillValues<float>(&weights_data,
+                          {1.0f, 2.0f, 3.0f, 4.0f, 0.1f, 0.2f, 0.3f, 0.4f});
+  Output weights_op =
+      Const(root.WithOpName("weights_op"), Input::Initializer(weights_data));
+
+  Output conv_op = Conv2D(root.WithOpName("conv_op"), input_op, weights_op,
+                          {1, 1, 1, 1}, "VALID");
+
+  Tensor block_shape_data(DT_INT32, TensorShape({2}));
+  test::FillValues<int32>(&block_shape_data, {1, 2});
+  Output block_shape_op =
+      Const(root.WithOpName("block_shape_op"), Input::Initializer(block_shape_data));
+
+  Tensor crops_data(DT_INT32, TensorShape({2, 2}));
+  test::FillValues<int32>(&crops_data, {0, 0, 0, 1});
+  Output crops_op =
+      Const(root.WithOpName("crops_op"), Input::Initializer(crops_data));
+
+  Output batch_to_space_op = BatchToSpaceND(root.WithOpName("batch_to_space_op"),
+                                            conv_op, block_shape_op, crops_data);
+
+  Tensor mean_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&mean_data, {10.0f, 20.0f});
+  Output mean_op =
+      Const(root.WithOpName("mean_op"), Input::Initializer(mean_data));
+
+  Tensor variance_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&variance_data, {0.25f, 0.5f});
+  Output variance_op = Const(root.WithOpName("variance_op"),
+                             Input::Initializer(variance_data));
+
+  Tensor beta_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&beta_data, {0.1f, 0.6f});
+  Output beta_op =
+      Const(root.WithOpName("beta_op"), Input::Initializer(beta_data));
+
+  Tensor gamma_data(DT_FLOAT, TensorShape({2}));
+  test::FillValues<float>(&gamma_data, {1.0f, 2.0f});
+  Output gamma_op =
+      Const(root.WithOpName("gamma_op"), Input::Initializer(gamma_data));
+
+  GraphDef original_graph_def;
+  TF_ASSERT_OK(root.ToGraphDef(&original_graph_def));
+
+  NodeDef batch_norm_node;
+  batch_norm_node.set_op("FusedBatchNorm");
+  batch_norm_node.set_name("output");
+  AddNodeInput("batch_to_space_op", &batch_norm_node);
+  AddNodeInput("gamma_op", &batch_norm_node);
+  AddNodeInput("beta_op", &batch_norm_node);
+  AddNodeInput("mean_op", &batch_norm_node);
+  AddNodeInput("variance_op", &batch_norm_node);
+  SetNodeAttr("T", DT_FLOAT, &batch_norm_node);
+  SetNodeAttr("epsilon", 0.00001f, &batch_norm_node);
+  SetNodeAttr("is_training", false, &batch_norm_node);
+  *(original_graph_def.mutable_node()->Add()) = batch_norm_node;
+
+  std::unique_ptr<Session> original_session(NewSession(SessionOptions()));
+  TF_ASSERT_OK(original_session->Create(original_graph_def));
+  std::vector<Tensor> original_outputs;
+  TF_ASSERT_OK(original_session->Run({}, {"output"}, {}, &original_outputs));
+
+  GraphDef fused_graph_def;
+  TF_ASSERT_OK(FoldOldBatchNorms(original_graph_def, {{}, {"output"}},
+                                 &fused_graph_def));
+
+  std::unique_ptr<Session> fused_session(NewSession(SessionOptions()));
+  TF_ASSERT_OK(fused_session->Create(fused_graph_def));
+  std::vector<Tensor> fused_outputs;
+  TF_ASSERT_OK(fused_session->Run({}, {"output"}, {}, &fused_outputs));
+
+  test::ExpectTensorNear<float>(original_outputs[0], fused_outputs[0], 1e-5);
+
+  for (const NodeDef& node : fused_graph_def.node()) {
+    EXPECT_NE("FusedBatchNormWithBatchToSpace", node.op());
+  }
+}
+
 TEST_F(FoldOldBatchNormsTest, TestFoldOldBatchNorms) {
   TestFoldOldBatchNorms();
 }
@@ -307,7 +398,7 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNorms) {
 }
 
 TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) {
-  // Test axis is not 3, so all weigths and offsets are fused to each of inputs
+  // Test axis is not 3, so all weights and offsets are fused to each of inputs
   // of conv2d.
   TestFoldFusedBatchNormsWithConcat(/*split=*/true);
   // Test axis = 3, BatchNorm weights and offsets will be split before fused
@@ -315,5 +406,9 @@ TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithConcat) {
   TestFoldFusedBatchNormsWithConcat(/*split=*/false);
 }
 
+TEST_F(FoldOldBatchNormsTest, TestFoldFusedBatchNormsWithBatchToSpace) {
+  TestFoldFusedBatchNormsWithBatchToSpace();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 1833d67d82..d55a883df5 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -108,6 +108,7 @@ filegroup(
         "@highwayhash//:LICENSE",
         "@jemalloc//:COPYING",
         "@jpeg//:LICENSE.md",
+        "@kafka//:LICENSE",
         "@libxsmm_archive//:LICENSE",
         "@lmdb//:LICENSE",
         "@local_config_sycl//sycl:LICENSE.text",
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e1a5f091ba..e0152da4df 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0-rc1'
+_VERSION = '1.6.0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -72,7 +72,7 @@ if sys.version_info < (3, 4):
 
 # pylint: disable=line-too-long
 CONSOLE_SCRIPTS = [
-    'freeze_graph = tensorflow.python.tools.freeze_graph:main',
+    'freeze_graph = tensorflow.python.tools.freeze_graph:run_main',
     'toco_from_protos = tensorflow.contrib.lite.toco.python.toco_from_protos:main',
     'toco = tensorflow.contrib.lite.toco.python.toco_wrapper:main',
     'saved_model_cli = tensorflow.python.tools.saved_model_cli:main',
diff --git a/tensorflow/tools/test/upload_test_benchmarks.py b/tensorflow/tools/test/upload_test_benchmarks.py
index 77cc9f75f7..c030575109 100644
--- a/tensorflow/tools/test/upload_test_benchmarks.py
+++ b/tensorflow/tools/test/upload_test_benchmarks.py
@@ -87,6 +87,7 @@ import json
 import os
 import shutil
 
+from six import text_type
 from google.cloud import datastore
 
 
@@ -150,7 +151,7 @@ def upload_benchmark_data(client, data):
   """
   test_result = json.loads(data)
 
-  test_name = unicode(test_result["name"])
+  test_name = text_type(test_result["name"])
   start_time = datetime.datetime.utcfromtimestamp(
       float(test_result["startTime"]))
   batch = []
@@ -162,7 +163,7 @@ def upload_benchmark_data(client, data):
   t_val.update({
       "test": test_name,
       "start": start_time,
-      "info": unicode(data)
+      "info": text_type(data)
   })
   batch.append(t_val)
 
@@ -170,7 +171,7 @@ def upload_benchmark_data(client, data):
   # the attribute to be fetched and displayed.  The full entry information is
   # also stored as a non-indexed JSON blob.
   for ent in test_result["entries"].get("entry", []):
-    ent_name = unicode(ent["name"])
+    ent_name = text_type(ent["name"])
     e_key = client.key("Entry")
     e_val = datastore.Entity(e_key, exclude_from_indexes=["info"])
     e_val.update({
@@ -178,7 +179,7 @@ def upload_benchmark_data(client, data):
         "start": start_time,
         "entry": ent_name,
         "timing": ent["wallTime"],
-        "info": unicode(json.dumps(ent))
+        "info": text_type(json.dumps(ent))
     })
     batch.append(e_val)
 
diff --git a/third_party/jpeg/jpeg.BUILD b/third_party/jpeg/jpeg.BUILD
index 87a23925c4..4418ac32fc 100644
--- a/third_party/jpeg/jpeg.BUILD
+++ b/third_party/jpeg/jpeg.BUILD
@@ -526,12 +526,12 @@ config_setting(
 
 config_setting(
     name = "armeabi-v7a",
-    values = {"android_cpu": "armeabi-v7a"},
+    values = {"cpu": "armeabi-v7a"},
 )
 
 config_setting(
     name = "arm64-v8a",
-    values = {"android_cpu": "arm64-v8a"},
+    values = {"cpu": "arm64-v8a"},
 )
 
 config_setting(
diff --git a/third_party/kafka/BUILD b/third_party/kafka/BUILD
index a61a9e1f6c..a839ca717e 100644
--- a/third_party/kafka/BUILD
+++ b/third_party/kafka/BUILD
@@ -130,12 +130,16 @@ cc_library(
     ],
     hdrs = [
         "config.h",
+        "src-cpp/rdkafkacpp.h",
+        "src-cpp/rdkafkacpp_int.h",
+        "src/lz4.c",
+        "src/snappy_compat.h",
     ],
-    defines = [
+    copts = [
+        "-Iexternal/kafka/src",
+        "-Iexternal/kafka/src-cpp",
     ],
-    includes = [
-        "src",
-        "src-cpp",
+    defines = [
     ],
     linkopts = [
         "-lpthread",
@@ -143,5 +147,6 @@ cc_library(
     visibility = ["//visibility:public"],
     deps = [
         "@boringssl//:ssl",
+        "@zlib_archive//:zlib",
     ],
 )
diff --git a/third_party/py/BUILD.tpl b/third_party/py/BUILD.tpl
index de06ad5f27..1dd8ab433a 100644
--- a/third_party/py/BUILD.tpl
+++ b/third_party/py/BUILD.tpl
@@ -2,20 +2,26 @@ licenses(["restricted"])
 
 package(default_visibility = ["//visibility:public"])
 
+# To build Python C/C++ extension on Windows, we need to link to python import library pythonXY.lib
+# See https://docs.python.org/3/extending/windows.html
+cc_import(
+    name = "python_lib",
+    interface_library = select({
+        ":windows": ":python_import_lib",
+        # A placeholder for Unix platforms which makes --no_build happy.
+        "//conditions:default": "not-existing.lib",
+    }),
+    system_provided = 1,
+)
+
 cc_library(
     name = "python_headers",
     hdrs = [":python_include"],
-    data = select({
-        ":windows": [":python_import_lib"],
+    deps = select({
+        ":windows": [":python_lib"],
         "//conditions:default": [],
     }),
     includes = ["python_include"],
-    linkopts = select({
-        # TODO(pcloudy): Ideally, this should just go into deps after resolving
-        # https://github.com/bazelbuild/bazel/issues/3237,
-        ":windows": ["$(locations :python_import_lib)"],
-        "//conditions:default": [],
-    }),
 )
 
 cc_library(
diff --git a/third_party/tensorrt/tensorrt_configure.bzl b/third_party/tensorrt/tensorrt_configure.bzl
index 8e76e5d02a..9b946505a6 100644
--- a/third_party/tensorrt/tensorrt_configure.bzl
+++ b/third_party/tensorrt/tensorrt_configure.bzl
@@ -57,6 +57,10 @@ def _find_trt_header_dir(repository_ctx, trt_install_path):
     path = "/usr/include/x86_64-linux-gnu"
     if _headers_exist(repository_ctx, path):
       return path
+  if trt_install_path == "/usr/lib/aarch64-linux-gnu":
+    path = "/usr/include/aarch64-linux-gnu"
+    if _headers_exist(repository_ctx, path):
+      return path
   path = str(repository_ctx.path("%s/../include" % trt_install_path).realpath)
   if _headers_exist(repository_ctx, path):
     return path
-- 
GitLab


From e79924eda7285a5aa0cfe908a223aef690b05fdd Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 21 Mar 2018 12:14:30 -0700
Subject: [PATCH 311/960] Made sure to compare all the attributes when
 comparing nodes

PiperOrigin-RevId: 189946858
---
 tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 942724a6ce..bc004df608 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -1037,6 +1037,9 @@ bool UniqueNodes::SameNode(const NodeDef& node1, const NodeDef& node2) const {
   }
 
   // Compare attributes.
+  if (node1.attr().size() != node2.attr().size()) {
+    return false;
+  }
   for (const auto& attr1 : node1.attr()) {
     auto it = node2.attr().find(attr1.first);
     if (it == node2.attr().end()) {
-- 
GitLab


From bdd6f2253a76c707ff2ce2af9b560478891342eb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 12:40:08 -0700
Subject: [PATCH 312/960] [XLA] Adding more tests for ReduceWindow.

PiperOrigin-RevId: 189950361
---
 tensorflow/compiler/xla/tests/reduce_window_test.cc | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index f66fb5cacc..9c317fe579 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -1021,6 +1021,15 @@ struct R2ReduceWindowTestData {
      /*strides=*/{1, 1}, /*pad_low=*/{0, 130}, /*pad_high=*/{0, 0},
      /*layout=*/{1, 0},
      /*reducer=*/Reducer::kAdd},
+// TODO(b/76025683): These tests fail on TPU.
+#if defined(XLA_TEST_BACKEND_CPU) || defined(XLA_TEST_BACKEND_GPU)
+    {/*base_bounds=*/{4096, 4096}, /*window_bounds=*/{1, 4},
+     /*strides=*/{1, 1024}, /*pad_low=*/{0, 0}, /*pad-high=*/{0, 0},
+     /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd},
+    {/*base_bounds=*/{8, 256}, /*window_bounds=*/{1, 4},
+     /*strides=*/{1, 64}, /*pad_low=*/{0, 0}, /*pad_high=*/{0, 0},
+     /*layout=*/{1, 0}, /*reducer=*/Reducer::kAdd},
+#endif
 };
 
 string R2ReduceWindowTestDataToString(
-- 
GitLab


From ee108441201ecb5fa9536573637623d712f9aa33 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Wed, 21 Mar 2018 12:53:04 -0700
Subject: [PATCH 313/960] Further improve accuracy of op_level_cost_estimator
 (Gather, GatherV2, Slice).

PiperOrigin-RevId: 189952132
---
 .../grappler/costs/op_level_cost_estimator.cc | 32 +++++++++++++++----
 .../grappler/costs/op_level_cost_estimator.h  |  2 +-
 .../costs/op_level_cost_estimator_test.cc     | 22 +++++++++++--
 3 files changed, 47 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 84ad8a3e84..d3ffa03fe2 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -48,6 +48,8 @@ constexpr char kSize[] = "Size";
 constexpr char kStopGradient[] = "StopGradient";
 constexpr char kPreventGradient[] = "PreventGradient";
 constexpr char kGather[] = "Gather";
+constexpr char kGatherV2[] = "GatherV2";
+constexpr char kSlice[] = "Slice";
 
 static const Costs::Duration kMinComputeTime(1);
 
@@ -169,7 +171,9 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
 
-      {kGather, wrap(&OpLevelCostEstimator::PredictGather)},
+      {kGather, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
+      {kGatherV2, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
+      {kSlice, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
 
       {kPlaceholder, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kIdentity, wrap(&OpLevelCostEstimator::PredictIdentity)},
@@ -1049,17 +1053,33 @@ Costs OpLevelCostEstimator::PredictMetadata(const OpContext& op_context) const {
   return costs;
 }
 
-Costs OpLevelCostEstimator::PredictGather(const OpContext& op_context) const {
-  // Gather op can have a very large input, but only the size of the output
-  // matters, because indices may select only a very small subset of input.
-
+Costs OpLevelCostEstimator::PredictGatherOrSlice(
+    const OpContext& op_context) const {
+  // Gather & Slice ops can have a very large input, but only access a small
+  // part of it. For these op the size of the output determines the memory cost.
   const auto& op_info = op_context.op_info;
 
   bool unknown_shapes = false;
+
+  // Each output element is a copy of some element from input.
+  // For roofline estimate we assume each copy has a unit cost.
   const int64 op_count =
       CalculateTensorElementCount(op_info.outputs(0), &unknown_shapes);
+
   const double output_size = CalculateOutputSize(op_info, &unknown_shapes);
-  const double total_io = 2 * output_size;
+  double input_size = output_size;
+  if (op_info.op() == "Slice") {
+    // Add 'begin' & 'size' tensors sizes.
+    input_size +=
+        CalculateTensorElementCount(op_info.inputs(1), &unknown_shapes) +
+        CalculateTensorElementCount(op_info.inputs(2), &unknown_shapes);
+  } else {
+    // Assuming this is "Gather" or "GatherV2" op, add 'indices' size.
+    input_size +=
+        CalculateTensorElementCount(op_info.inputs(1), &unknown_shapes);
+  }
+
+  const double total_io = input_size + output_size;
   Costs costs = PredictOpCountBasedCost(op_count, total_io, op_info);
   costs.inaccurate = unknown_shapes;
   costs.max_memory = output_size;
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index e5dd31a7a2..1b3babb206 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -144,7 +144,7 @@ class OpLevelCostEstimator {
   Costs PredictVariable(const OpContext& op_context) const;
   Costs PredictBatchMatMul(const OpContext& op_context) const;
   Costs PredictMetadata(const OpContext& op_context) const;
-  Costs PredictGather(const OpContext& op_context) const;
+  Costs PredictGatherOrSlice(const OpContext& op_context) const;
 
   // Utility function for safe division. Returns 0
   // if rhs is 0 or negative.
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index a92f230101..f2a9615dfb 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -206,9 +206,27 @@ TEST_F(OpLevelCostEstimatorTest, TestGatherCosts) {
   DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
 
   auto cost = estimator_.PredictCosts(op_context);
-  EXPECT_EQ(Costs::Duration(128), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(130), cost.memory_time);
   EXPECT_EQ(Costs::Duration(16), cost.compute_time);
-  EXPECT_EQ(Costs::Duration(144), cost.execution_time);
+  EXPECT_EQ(Costs::Duration(146), cost.execution_time);
+  EXPECT_FALSE(cost.inaccurate);
+}
+
+TEST_F(OpLevelCostEstimatorTest, TestSliceCosts) {
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  op_context.op_info.set_op("Slice");
+
+  // Huge first input shouldn't affect Slice execution and memory costs.
+  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankOutput({10, 10}, DT_FLOAT, &op_context.op_info);
+
+  auto cost = estimator_.PredictCosts(op_context);
+  EXPECT_EQ(Costs::Duration(81), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(10), cost.compute_time);
+  EXPECT_EQ(Costs::Duration(91), cost.execution_time);
   EXPECT_FALSE(cost.inaccurate);
 }
 
-- 
GitLab


From a6a7de3d36a154e4382283a804531f148fca6c9b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 12:53:53 -0700
Subject: [PATCH 314/960] Temporarily disable constant folding past Enter,
 since a few breakages have been observed.

PiperOrigin-RevId: 189952252
---
 tensorflow/core/grappler/optimizers/constant_folding.cc      | 5 ++++-
 tensorflow/core/grappler/optimizers/constant_folding.h       | 2 +-
 tensorflow/core/grappler/optimizers/constant_folding_test.cc | 3 ++-
 3 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 263983584c..bdec73e69e 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1707,7 +1707,9 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
     }
 
     // Move constants past Enter.
-    if (IsEnter(*node) && node->input_size() > 0) {
+    // TODO(rmlarsen): Reenable when we fix the root cause of b/76008022
+    if (opt_level_ == RewriterConfig::AGGRESSIVE && IsEnter(*node) &&
+        node->input_size() > 0) {
       const string& node_name = node->name();
       const NodeDef* input = node_map_->GetNode(node->input(0));
       if (input != nullptr && IsReallyConstant(*input) &&
@@ -1729,6 +1731,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
           NodeDef* new_node = optimized_graph->add_node();
           *new_node = *input;
           new_node->set_name(OptimizedNodeName(*input, "_enter"));
+          new_node->set_device(node->device());
           new_node->clear_input();
           new_node->add_input(AsControlDependency(node_name));
           node_map_->AddNode(new_node->name(), new_node);
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index 13ecfcd281..b6645d335e 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -38,7 +38,7 @@ class ConstantFolding : public GraphOptimizer {
   static string AddControlDependency(const string& input_name, GraphDef* graph,
                                      NodeMap* node_map);
 
-  ConstantFolding(DeviceBase* cpu_device);
+  explicit ConstantFolding(DeviceBase* cpu_device);
   ConstantFolding(RewriterConfig::Toggle opt_level, DeviceBase* cpu_device);
 
   ~ConstantFolding() override {}
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index aeb430b384..914a9257ee 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -2103,7 +2103,8 @@ TEST_F(ConstantFoldingTest, Enter) {
   item.fetch.push_back("id2");
   item.fetch.push_back("id3");
 
-  ConstantFolding optimizer(nullptr /* cpu_device */);
+  ConstantFolding optimizer(RewriterConfig::AGGRESSIVE,
+                            nullptr /* cpu_device */);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
-- 
GitLab


From 6741f81b8216862a83703122191a8632fda333a2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 13:08:02 -0700
Subject: [PATCH 315/960] Make strcat.{h,cc} independent of Eigen.

PiperOrigin-RevId: 189954596
---
 tensorflow/compiler/xla/literal_util.cc      |  5 +++--
 tensorflow/compiler/xla/literal_util_test.cc |  2 +-
 tensorflow/core/framework/tensor.cc          | 18 ++++++++++++++++--
 tensorflow/core/lib/strings/strcat.cc        |  4 ----
 tensorflow/core/lib/strings/strcat.h         |  5 -----
 tensorflow/core/lib/strings/strcat_test.cc   |  6 ------
 tensorflow/stream_executor/stream.cc         |  5 ++++-
 7 files changed, 24 insertions(+), 21 deletions(-)

diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 0a24db046a..20508edaa7 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -929,7 +929,7 @@ string Literal::GetAsString(tensorflow::gtl::ArraySlice<int64> multi_index,
     case U64:
       return StrCat(Get<uint64>(multi_index, shape_index));
     case F16:
-      return StrCat(Get<half>(multi_index, shape_index));
+      return StrCat(static_cast<float>(Get<half>(multi_index, shape_index)));
     case F32:
       return StrCat(Get<float>(multi_index, shape_index));
     case BF16:
@@ -979,7 +979,8 @@ string Literal::GetSparseElementAsString(int64 sparse_element_number,
       return StrCat(
           GetSparseElement<uint64>(sparse_element_number, shape_index));
     case F16:
-      return StrCat(GetSparseElement<half>(sparse_element_number, shape_index));
+      return StrCat(static_cast<float>(
+          GetSparseElement<half>(sparse_element_number, shape_index)));
     case F32:
       return StrCat(
           GetSparseElement<float>(sparse_element_number, shape_index));
diff --git a/tensorflow/compiler/xla/literal_util_test.cc b/tensorflow/compiler/xla/literal_util_test.cc
index 04e45f0049..7627762074 100644
--- a/tensorflow/compiler/xla/literal_util_test.cc
+++ b/tensorflow/compiler/xla/literal_util_test.cc
@@ -1702,7 +1702,7 @@ TEST_F(LiteralUtilTest, GetSparseElementAsString) {
   ASSERT_EQ(Literal::CreateSparse<half>(dimensions, indices,
                                         {half{1.0}, half{2.0}, half{3.0}})
                 ->GetSparseElementAsString(1),
-            tensorflow::strings::StrCat(half{2.0}));
+            tensorflow::strings::StrCat(static_cast<float>(half{2.0})));
   ASSERT_EQ(
       Literal::CreateSparse<complex64>(
           dimensions, indices,
diff --git a/tensorflow/core/framework/tensor.cc b/tensorflow/core/framework/tensor.cc
index 5d32b71628..e2111d6038 100644
--- a/tensorflow/core/framework/tensor.cc
+++ b/tensorflow/core/framework/tensor.cc
@@ -884,6 +884,20 @@ bool Tensor::CanUseDMA() const {
 #undef CASE
 
 namespace {
+
+// StrCat and StrAppend don't support Eigen::half directly at the moment, and
+// we would like to keep them compatible with their absl counterparts, for ease
+// of migration. We could rely on errors::internal::PrepareForStrCat() but the
+// logic is so simple we can just replicate it here, where it is close to its
+// usage and easy to change later. And there's the extra benefit of not
+// accessing an 'internal' namespace.
+inline const strings::AlphaNum& PrintOneElement(const strings::AlphaNum& a) {
+  return a;
+}
+inline float PrintOneElement(const Eigen::half& h) {
+  return static_cast<float>(h);
+}
+
 // Print from left dim to right dim recursively.
 template <typename T>
 void PrintOneDim(int dim_index, const gtl::InlinedVector<int64, 4>& shape,
@@ -896,7 +910,7 @@ void PrintOneDim(int dim_index, const gtl::InlinedVector<int64, 4>& shape,
     for (int64 i = 0; i < element_count; i++) {
       if (*data_index >= limit) return;
       if (i > 0) strings::StrAppend(result, " ");
-      strings::StrAppend(result, data[(*data_index)++]);
+      strings::StrAppend(result, PrintOneElement(data[(*data_index)++]));
     }
     return;
   }
@@ -927,7 +941,7 @@ string SummarizeArray(int64 limit, int64 num_elts,
   if (shape.empty()) {
     for (int64 i = 0; i < limit; ++i) {
       if (i > 0) strings::StrAppend(&ret, " ");
-      strings::StrAppend(&ret, array[i]);
+      strings::StrAppend(&ret, PrintOneElement(array[i]));
     }
     if (num_elts > limit) strings::StrAppend(&ret, "...");
     return ret;
diff --git a/tensorflow/core/lib/strings/strcat.cc b/tensorflow/core/lib/strings/strcat.cc
index 5b1cff486d..f140ec3d26 100644
--- a/tensorflow/core/lib/strings/strcat.cc
+++ b/tensorflow/core/lib/strings/strcat.cc
@@ -20,16 +20,12 @@ limitations under the License.
 #include <stdio.h>
 #include <string.h>
 
-#include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/platform/logging.h"
 
 namespace tensorflow {
 namespace strings {
 
-AlphaNum::AlphaNum(const Eigen::half &f)
-    : piece_(digits_, strlen(FloatToBuffer(static_cast<float>(f), digits_))) {}
-
 AlphaNum::AlphaNum(Hex hex) {
   char *const end = &digits_[kFastToBufferSize];
   char *writer = end;
diff --git a/tensorflow/core/lib/strings/strcat.h b/tensorflow/core/lib/strings/strcat.h
index b681f7398d..b3ec14e448 100644
--- a/tensorflow/core/lib/strings/strcat.h
+++ b/tensorflow/core/lib/strings/strcat.h
@@ -27,10 +27,6 @@ limitations under the License.
 #include "tensorflow/core/platform/macros.h"
 #include "tensorflow/core/platform/types.h"
 
-namespace Eigen {
-struct half;
-}
-
 // The AlphaNum type was designed to be used as the parameter type for StrCat().
 // Any routine accepting either a string or a number may accept it.
 // The basic idea is that by accepting a "const AlphaNum &" as an argument
@@ -122,7 +118,6 @@ class AlphaNum {
   AlphaNum(double f)  // NOLINT(runtime/explicit)
       : piece_(digits_, strlen(DoubleToBuffer(f, digits_))) {}
 
-  AlphaNum(const Eigen::half &f);  // NOLINT(runtime/explicit)
   AlphaNum(Hex hex);               // NOLINT(runtime/explicit)
 
   AlphaNum(const char *c_str) : piece_(c_str) {}   // NOLINT(runtime/explicit)
diff --git a/tensorflow/core/lib/strings/strcat_test.cc b/tensorflow/core/lib/strings/strcat_test.cc
index 7cb186e637..8cc64a6f0a 100644
--- a/tensorflow/core/lib/strings/strcat_test.cc
+++ b/tensorflow/core/lib/strings/strcat_test.cc
@@ -17,7 +17,6 @@ limitations under the License.
 
 #include <string>
 
-#include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
@@ -131,11 +130,6 @@ TEST(StrCat, Basics) {
   result = tensorflow::strings::StrCat("A hundred K and a half squared is ", d);
   EXPECT_EQ(result, "A hundred K and a half squared is 10000100000.25");
 
-  Eigen::half h(10007.0f);
-  result =
-      tensorflow::strings::StrCat("Ten thousand seven is approximately ", h);
-  EXPECT_EQ(result, "Ten thousand seven is approximately 10008");
-
   result = tensorflow::strings::StrCat(1, 2, 333, 4444, 55555, 666666, 7777777,
                                        88888888, 999999999);
   EXPECT_EQ(result, "12333444455555666666777777788888888999999999");
diff --git a/tensorflow/stream_executor/stream.cc b/tensorflow/stream_executor/stream.cc
index 6bbb5f0b2e..1e3afde268 100644
--- a/tensorflow/stream_executor/stream.cc
+++ b/tensorflow/stream_executor/stream.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/stream_executor/platform/port.h"
 
+#include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/stream_executor/blas.h"
 #include "tensorflow/stream_executor/host_buffer.h"
 #include "tensorflow/stream_executor/lib/stacktrace.h"
@@ -117,7 +118,9 @@ string ToVlogString(const DeviceMemoryBase *memory) {
   return ToVlogString(*memory);
 }
 
-string ToVlogString(const Eigen::half &h) { return port::StrCat(h); }
+string ToVlogString(const Eigen::half &h) {
+  return port::StrCat(static_cast<float>(h));
+}
 
 string ToVlogString(int i) { return port::StrCat(i); }
 
-- 
GitLab


From 56e5181f340f855e0eef9a4ce25baea5be1aaebc Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 21 Mar 2018 13:28:11 -0700
Subject: [PATCH 316/960] [TF CriticalSection] Bugfix when Execute() inside a
 while_loop has a dep on a Variable outside of it.

PiperOrigin-RevId: 189957569
---
 .../framework/python/ops/critical_section_ops.py   | 14 +++++++++++++-
 .../framework/python/ops/critical_section_test.py  | 14 ++++++++++++++
 2 files changed, 27 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/framework/python/ops/critical_section_ops.py b/tensorflow/contrib/framework/python/ops/critical_section_ops.py
index 1893d7b466..bd764ed57a 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_ops.py
+++ b/tensorflow/contrib/framework/python/ops/critical_section_ops.py
@@ -308,7 +308,19 @@ class CriticalSection(object):
       all_args_dict.pop(input_.op._id, None)
     all_args_dict.pop(lock_op._id, None)
 
-    lock_op._add_control_inputs(all_args_dict.values())
+    all_args = all_args_dict.values()
+
+    if not all_args:
+      # No control dependencies to add; return early.
+      return
+
+    # This group is important: it ensures that any ops in all_args
+    # outside the control context of the lock_op (and this fn, which
+    # runs in the same context) are added to this context before
+    # being added to the control dependencies of lock_op.
+    all_args = control_flow_ops.group(*all_args)
+
+    lock_op._add_control_input(all_args)
     # pylint: enable=protected-access
 
   def _is_self_handle(self, x):
diff --git a/tensorflow/contrib/framework/python/ops/critical_section_test.py b/tensorflow/contrib/framework/python/ops/critical_section_test.py
index e24140bd72..ba660295cb 100644
--- a/tensorflow/contrib/framework/python/ops/critical_section_test.py
+++ b/tensorflow/contrib/framework/python/ops/critical_section_test.py
@@ -316,6 +316,20 @@ class CriticalSectionTest(test.TestCase):
         ValueError, "requested exclusive resource access"):
       cs1.execute(lambda: v2 + 1)
 
+  def testControlDependencyFromOutsideWhileLoopMixedWithInsideLoop(self):
+    cs = critical_section_ops.CriticalSection()
+    v = resource_variable_ops.ResourceVariable(0, name="v")
+    # Make sure that the control dependencies on v do not cause issues
+    # in the lock_op's automatic control dependency adder.
+    #
+    # Note, here v must be a resource variable (or something similar),
+    # otherwise it gets hoisted into the while_loop by the time we add
+    # control dependencies to the lock_op.
+    out = control_flow_ops.while_loop(
+        lambda i: i < 10, lambda i: cs.execute(lambda j: v + j + 1, i), [0])
+    self.evaluate(v.initializer)
+    self.assertEqual(10, self.evaluate(out))
+
   # TODO(ebrevdo): Re-enable once CriticalSection is in core.
   #
   # def testCriticalSectionAndExecuteOpSaverRoundTrip(self):
-- 
GitLab


From 9bdc24077e4a454a663b2958817bcb5f7d961833 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 13:30:26 -0700
Subject: [PATCH 317/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 189957912
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 31 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 31 +++++++++++++++++++
 2 files changed, 62 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index ddf7627463..99bdc8ccdc 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -51602,6 +51602,37 @@ op {
     }
   }
 }
+op {
+  name: "SlideDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "stride"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "Snapshot"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 72326e1137..02e52b07c4 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -24399,6 +24399,37 @@ op {
     }
   }
 }
+op {
+  name: "SlideDataset"
+  input_arg {
+    name: "input_dataset"
+    type: DT_VARIANT
+  }
+  input_arg {
+    name: "window_size"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "stride"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "handle"
+    type: DT_VARIANT
+  }
+  attr {
+    name: "output_types"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "output_shapes"
+    type: "list(shape)"
+    has_minimum: true
+    minimum: 1
+  }
+}
 op {
   name: "Snapshot"
   input_arg {
-- 
GitLab


From 7dd78367a19e101b45f0cafb5c4fbe6a3c840828 Mon Sep 17 00:00:00 2001
From: Sang Han <jjangsangy@users.noreply.github.com>
Date: Wed, 21 Mar 2018 13:37:52 -0700
Subject: [PATCH 318/960] SetUsrMemDataHandle should return void

---
 tensorflow/core/util/mkl_util.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 34db96075d..9f58e40d94 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -1579,10 +1579,10 @@ class MklDnnData {
   }
 
   /// Set function for data buffer of user memory primitive.
-  inline void* SetUsrMemDataHandle(void* data_buffer) {
+  inline void SetUsrMemDataHandle(void* data_buffer) {
     CHECK_NOTNULL(user_memory_);
     CHECK_NOTNULL(data_buffer);
-    return user_memory_->set_data_handle(data_buffer);
+    user_memory_->set_data_handle(data_buffer);
   }
 
   /// Set function for data buffer of user memory primitive.
-- 
GitLab


From 2533de345fb13ff430eea207d59935b6d4b8fe19 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 13:46:27 -0700
Subject: [PATCH 319/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 189960595

---
 tensorflow/go/op/wrappers.go | 376 +++++++++++++++++------------------
 1 file changed, 188 insertions(+), 188 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index e5256af1e8..16472464db 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -4116,6 +4116,194 @@ func Mod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
+// DepthToSpaceAttr is an optional argument to DepthToSpace.
+type DepthToSpaceAttr func(optionalAttr)
+
+// DepthToSpaceDataFormat sets the optional data_format attribute to value.
+// If not specified, defaults to "NHWC"
+func DepthToSpaceDataFormat(value string) DepthToSpaceAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// DepthToSpace for tensors of type T.
+//
+// Rearranges data from depth into blocks of spatial data.
+// This is the reverse transformation of SpaceToDepth. More specifically,
+// this op outputs a copy of the input tensor where values from the `depth`
+// dimension are moved in spatial blocks to the `height` and `width` dimensions.
+// The attr `block_size` indicates the input block size and how the data is moved.
+//
+//   * Chunks of data of size `block_size * block_size` from depth are rearranged
+//     into non-overlapping blocks of size `block_size x block_size`
+//   * The width the output tensor is `input_depth * block_size`, whereas the
+//     height is `input_height * block_size`.
+//   * The Y, X coordinates within each block of the output image are determined
+//     by the high order component of the input channel index.
+//   * The depth of the input tensor must be divisible by
+//     `block_size * block_size`.
+//
+// The `data_format` attr specifies the layout of the input and output tensors
+// with the following options:
+//   "NHWC": `[ batch, height, width, channels ]`
+//   "NCHW": `[ batch, channels, height, width ]`
+//   "NCHW_VECT_C":
+//       `qint8 [ batch, channels / 4, height, width, 4 ]`
+//
+// It is useful to consider the operation as transforming a 6-D Tensor.
+// e.g. for data_format = NHWC,
+//      Each element in the input tensor can be specified via 6 coordinates,
+//      ordered by decreasing memory layout significance as:
+//      n,iY,iX,bY,bX,oC  (where n=batch index, iX, iY means X or Y coordinates
+//                         within the input image, bX, bY means coordinates
+//                         within the output block, oC means output channels).
+//      The output would be the input transposed to the following layout:
+//      n,iY,bY,iX,bX,oC
+//
+// This operation is useful for resizing the activations between convolutions
+// (but keeping all data), e.g. instead of pooling. It is also useful for training
+// purely convolutional models.
+//
+// For example, given an input of shape `[1, 1, 1, 4]`, data_format = "NHWC" and
+// block_size = 2:
+//
+// ```
+// x = [[[[1, 2, 3, 4]]]]
+//
+// ```
+//
+// This operation will output a tensor of shape `[1, 2, 2, 1]`:
+//
+// ```
+//    [[[[1], [2]],
+//      [[3], [4]]]]
+// ```
+//
+// Here, the input has a batch of 1 and each batch element has shape `[1, 1, 4]`,
+// the corresponding output will have 2x2 elements and will have a depth of
+// 1 channel (1 = `4 / (block_size * block_size)`).
+// The output element shape is `[2, 2, 1]`.
+//
+// For an input tensor with larger depth, here of shape `[1, 1, 1, 12]`, e.g.
+//
+// ```
+// x = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
+// ```
+//
+// This operation, for block size of 2, will return the following tensor of shape
+// `[1, 2, 2, 3]`
+//
+// ```
+//    [[[[1, 2, 3], [4, 5, 6]],
+//      [[7, 8, 9], [10, 11, 12]]]]
+//
+// ```
+//
+// Similarly, for the following input of shape `[1 2 2 4]`, and a block size of 2:
+//
+// ```
+// x =  [[[[1, 2, 3, 4],
+//        [5, 6, 7, 8]],
+//       [[9, 10, 11, 12],
+//        [13, 14, 15, 16]]]]
+// ```
+//
+// the operator will return the following tensor of shape `[1 4 4 1]`:
+//
+// ```
+// x = [[[ [1],   [2],  [5],  [6]],
+//       [ [3],   [4],  [7],  [8]],
+//       [ [9],  [10], [13],  [14]],
+//       [ [11], [12], [15],  [16]]]]
+//
+// ```
+//
+// Arguments:
+//
+//	block_size: The size of the spatial block, same as in Space2Depth.
+func DepthToSpace(scope *Scope, input tf.Output, block_size int64, optional ...DepthToSpaceAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"block_size": block_size}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DepthToSpace",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Conv3DBackpropInputV2Attr is an optional argument to Conv3DBackpropInputV2.
+type Conv3DBackpropInputV2Attr func(optionalAttr)
+
+// Conv3DBackpropInputV2DataFormat sets the optional data_format attribute to value.
+//
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func Conv3DBackpropInputV2DataFormat(value string) Conv3DBackpropInputV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Conv3DBackpropInputV2Dilations sets the optional dilations attribute to value.
+//
+// value: 1-D tensor of length 5.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
+func Conv3DBackpropInputV2Dilations(value []int64) Conv3DBackpropInputV2Attr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of 3-D convolution with respect to the input.
+//
+// Arguments:
+//	input_sizes: An integer vector representing the tensor shape of `input`,
+// where `input` is a 5-D
+// `[batch, depth, rows, cols, in_channels]` tensor.
+//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
+// `in_channels` must match between `input` and `filter`.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3DBackpropInputV2(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropInputV2Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Conv3DBackpropInputV2",
+		Input: []tf.Input{
+			input_sizes, filter, out_backprop,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes square root of x element-wise.
 //
 // I.e., \\(y = \sqrt{x} = x^{1/2}\\).
@@ -24282,194 +24470,6 @@ func StagePeek(scope *Scope, index tf.Output, dtypes []tf.DataType, optional ...
 	return values
 }
 
-// Conv3DBackpropInputV2Attr is an optional argument to Conv3DBackpropInputV2.
-type Conv3DBackpropInputV2Attr func(optionalAttr)
-
-// Conv3DBackpropInputV2DataFormat sets the optional data_format attribute to value.
-//
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func Conv3DBackpropInputV2DataFormat(value string) Conv3DBackpropInputV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Conv3DBackpropInputV2Dilations sets the optional dilations attribute to value.
-//
-// value: 1-D tensor of length 5.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DBackpropInputV2Dilations(value []int64) Conv3DBackpropInputV2Attr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of 3-D convolution with respect to the input.
-//
-// Arguments:
-//	input_sizes: An integer vector representing the tensor shape of `input`,
-// where `input` is a 5-D
-// `[batch, depth, rows, cols, in_channels]` tensor.
-//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
-// `in_channels` must match between `input` and `filter`.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3DBackpropInputV2(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropInputV2Attr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropInputV2",
-		Input: []tf.Input{
-			input_sizes, filter, out_backprop,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// DepthToSpaceAttr is an optional argument to DepthToSpace.
-type DepthToSpaceAttr func(optionalAttr)
-
-// DepthToSpaceDataFormat sets the optional data_format attribute to value.
-// If not specified, defaults to "NHWC"
-func DepthToSpaceDataFormat(value string) DepthToSpaceAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// DepthToSpace for tensors of type T.
-//
-// Rearranges data from depth into blocks of spatial data.
-// This is the reverse transformation of SpaceToDepth. More specifically,
-// this op outputs a copy of the input tensor where values from the `depth`
-// dimension are moved in spatial blocks to the `height` and `width` dimensions.
-// The attr `block_size` indicates the input block size and how the data is moved.
-//
-//   * Chunks of data of size `block_size * block_size` from depth are rearranged
-//     into non-overlapping blocks of size `block_size x block_size`
-//   * The width the output tensor is `input_depth * block_size`, whereas the
-//     height is `input_height * block_size`.
-//   * The Y, X coordinates within each block of the output image are determined
-//     by the high order component of the input channel index.
-//   * The depth of the input tensor must be divisible by
-//     `block_size * block_size`.
-//
-// The `data_format` attr specifies the layout of the input and output tensors
-// with the following options:
-//   "NHWC": `[ batch, height, width, channels ]`
-//   "NCHW": `[ batch, channels, height, width ]`
-//   "NCHW_VECT_C":
-//       `qint8 [ batch, channels / 4, height, width, 4 ]`
-//
-// It is useful to consider the operation as transforming a 6-D Tensor.
-// e.g. for data_format = NHWC,
-//      Each element in the input tensor can be specified via 6 coordinates,
-//      ordered by decreasing memory layout significance as:
-//      n,iY,iX,bY,bX,oC  (where n=batch index, iX, iY means X or Y coordinates
-//                         within the input image, bX, bY means coordinates
-//                         within the output block, oC means output channels).
-//      The output would be the input transposed to the following layout:
-//      n,iY,bY,iX,bX,oC
-//
-// This operation is useful for resizing the activations between convolutions
-// (but keeping all data), e.g. instead of pooling. It is also useful for training
-// purely convolutional models.
-//
-// For example, given an input of shape `[1, 1, 1, 4]`, data_format = "NHWC" and
-// block_size = 2:
-//
-// ```
-// x = [[[[1, 2, 3, 4]]]]
-//
-// ```
-//
-// This operation will output a tensor of shape `[1, 2, 2, 1]`:
-//
-// ```
-//    [[[[1], [2]],
-//      [[3], [4]]]]
-// ```
-//
-// Here, the input has a batch of 1 and each batch element has shape `[1, 1, 4]`,
-// the corresponding output will have 2x2 elements and will have a depth of
-// 1 channel (1 = `4 / (block_size * block_size)`).
-// The output element shape is `[2, 2, 1]`.
-//
-// For an input tensor with larger depth, here of shape `[1, 1, 1, 12]`, e.g.
-//
-// ```
-// x = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
-// ```
-//
-// This operation, for block size of 2, will return the following tensor of shape
-// `[1, 2, 2, 3]`
-//
-// ```
-//    [[[[1, 2, 3], [4, 5, 6]],
-//      [[7, 8, 9], [10, 11, 12]]]]
-//
-// ```
-//
-// Similarly, for the following input of shape `[1 2 2 4]`, and a block size of 2:
-//
-// ```
-// x =  [[[[1, 2, 3, 4],
-//        [5, 6, 7, 8]],
-//       [[9, 10, 11, 12],
-//        [13, 14, 15, 16]]]]
-// ```
-//
-// the operator will return the following tensor of shape `[1 4 4 1]`:
-//
-// ```
-// x = [[[ [1],   [2],  [5],  [6]],
-//       [ [3],   [4],  [7],  [8]],
-//       [ [9],  [10], [13],  [14]],
-//       [ [11], [12], [15],  [16]]]]
-//
-// ```
-//
-// Arguments:
-//
-//	block_size: The size of the spatial block, same as in Space2Depth.
-func DepthToSpace(scope *Scope, input tf.Output, block_size int64, optional ...DepthToSpaceAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"block_size": block_size}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DepthToSpace",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // MapStageAttr is an optional argument to MapStage.
 type MapStageAttr func(optionalAttr)
 
-- 
GitLab


From c8f5b35dd70d103fe8ae6a41ca907ccccaa3ac39 Mon Sep 17 00:00:00 2001
From: Shivani Agrawal <shivaniagrawal@google.com>
Date: Wed, 21 Mar 2018 13:57:29 -0700
Subject: [PATCH 320/960] [docs] Update explicit latest_checkpoint with `path`
 returned while saving for object-based checkpointing.

PiperOrigin-RevId: 189962437
---
 tensorflow/contrib/eager/python/g3doc/guide.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md
index b73dc17e5f..df084e9053 100644
--- a/tensorflow/contrib/eager/python/g3doc/guide.md
+++ b/tensorflow/contrib/eager/python/g3doc/guide.md
@@ -590,14 +590,14 @@ checkpoint = tfe.Checkpoint(x=x, y=y)
 
 # Assign new values to the variables and save.
 x.assign(2.)
-checkpoint.save('/tmp/ckpt')
+save_path = checkpoint.save('/tmp/ckpt')
 
 # Change the variable after saving.
 x.assign(11.)
 assert 16. == (x + y).numpy()  # 11 + 5
 
 # Restore the values in the checkpoint.
-checkpoint.restore('/tmp/ckpt-1')
+checkpoint.restore(save_path)  # save_path='/tmp/ckpt-1'
 
 assert 7. == (x + y).numpy()  # 2 + 5
 ```
-- 
GitLab


From 4c56dcaade7cee9f75740e2f03e30bdf3a6f93be Mon Sep 17 00:00:00 2001
From: Yunxing Dai <yunxing@google.com>
Date: Wed, 21 Mar 2018 14:12:07 -0700
Subject: [PATCH 321/960] [XLA][BF16] Add bf16 rounding function.

We now use truncation to convert a F32 to BF16 by default. This CL adds a rounding method (basically a roll forward of the rounding part in cl/175252067).

PiperOrigin-RevId: 189965138
---
 tensorflow/core/framework/bfloat16_test.cc |  39 +++--
 tensorflow/core/lib/bfloat16/bfloat16.h    | 188 ++++++++++++++++++++-
 2 files changed, 216 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/framework/bfloat16_test.cc b/tensorflow/core/framework/bfloat16_test.cc
index 17e6209f8e..206396a25a 100644
--- a/tensorflow/core/framework/bfloat16_test.cc
+++ b/tensorflow/core/framework/bfloat16_test.cc
@@ -37,19 +37,27 @@ float BinaryToFloat(uint32_t sign, uint32_t exponent, uint32_t high_mantissa,
 
 struct Bfloat16TestParam {
   float input;
-  float expected;
+  float expected_truncation;
+  float expected_rounding;
 };
 
 class Bfloat16Test : public ::testing::Test,
                      public ::testing::WithParamInterface<Bfloat16TestParam> {};
 
 TEST_P(Bfloat16Test, TruncateTest) {
-  bfloat16 a(GetParam().input);
+  bfloat16 truncated(GetParam().input);
   if (std::isnan(GetParam().input)) {
-    EXPECT_TRUE(std::isnan(float(a)) || std::isinf(float(a)));
+    EXPECT_TRUE(std::isnan(float(truncated)) || std::isinf(float(truncated)));
     return;
   }
-  EXPECT_EQ(GetParam().expected, float(a));
+  EXPECT_EQ(GetParam().expected_truncation, float(truncated));
+
+  bfloat16 rounded = bfloat16::round_to_bfloat16((GetParam().input));
+  if (std::isnan(GetParam().input)) {
+    EXPECT_TRUE(std::isnan(float(rounded)) || std::isinf(float(rounded)));
+    return;
+  }
+  EXPECT_EQ(GetParam().expected_rounding, float(rounded));
 }
 
 INSTANTIATE_TEST_CASE_P(
@@ -57,37 +65,48 @@ INSTANTIATE_TEST_CASE_P(
     ::testing::Values(
         Bfloat16TestParam{
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b1111010111000011),
-            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000)},
+            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
+            BinaryToFloat(0, 0b10000000, 0b1001001, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(1, 0b10000000, 0b1001000, 0b1111010111000011),
-            BinaryToFloat(1, 0b10000000, 0b1001000, 0b0000000000000000)},
+            BinaryToFloat(1, 0b10000000, 0b1001000, 0b0000000000000000),
+            BinaryToFloat(1, 0b10000000, 0b1001001, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b1000000000000000),
+            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b11111111, 0b0000000, 0b0000000000000001),
-            BinaryToFloat(0, 0b11111111, 0b0000000, 0b0000000000000000)},
+            BinaryToFloat(0, 0b11111111, 0b0000000, 0b0000000000000000),
+            BinaryToFloat(0, 0b11111111, 0b1000000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b11111111, 0b1111111, 0b1111111111111111),
-            BinaryToFloat(0, 0b11111111, 0b1111111, 0b0000000000000000)},
+            BinaryToFloat(0, 0b11111111, 0b1111111, 0b0000000000000000),
+            BinaryToFloat(0, 0b11111111, 0b1000000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(1, 0b10000000, 0b1001000, 0b1100000000000000),
-            BinaryToFloat(1, 0b10000000, 0b1001000, 0b0000000000000000)},
+            BinaryToFloat(1, 0b10000000, 0b1001000, 0b0000000000000000),
+            BinaryToFloat(1, 0b10000000, 0b1001001, 0b0000000000000000)},
         Bfloat16TestParam{
+            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0100000000000000),
+            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b1000000000000000),
+            BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b10000000, 0b1001000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b00000000, 0b1001000, 0b1000000000000000),
+            BinaryToFloat(0, 0b00000000, 0b1001000, 0b0000000000000000),
             BinaryToFloat(0, 0b00000000, 0b1001000, 0b0000000000000000)},
         Bfloat16TestParam{
             BinaryToFloat(0, 0b00000000, 0b1111111, 0b1100000000000000),
-            BinaryToFloat(0, 0b00000000, 0b1111111, 0b0000000000000000)}));
+            BinaryToFloat(0, 0b00000000, 0b1111111, 0b0000000000000000),
+            BinaryToFloat(0, 0b00000001, 0b0000000, 0b0000000000000000)}));
 
 TEST(Bfloat16Test, Conversion) {
   float a[100];
diff --git a/tensorflow/core/lib/bfloat16/bfloat16.h b/tensorflow/core/lib/bfloat16/bfloat16.h
index 075a8d1430..126e5a17af 100644
--- a/tensorflow/core/lib/bfloat16/bfloat16.h
+++ b/tensorflow/core/lib/bfloat16/bfloat16.h
@@ -165,6 +165,192 @@ struct bfloat16 {
     return complex128(double(*this), double(0.0));
   }
 
+  union FP32 {
+    unsigned int u;
+    float f;
+  };
+
+  // Converts a float point to bfloat16, with round-nearest-to-even as rounding
+  // method.
+  // TODO(b/69266521): Add a truncate_to_bfloat16 function and make this
+  // function as default behavior.
+  // TODO: There is a slightly faster implementation (8% faster on CPU)
+  // than this (documented in cl/175987786), that is exponentially harder to
+  // understand and document. Switch to the faster version when converting to
+  // BF16 becomes compute-bound.
+  B16_DEVICE_FUNC static bfloat16 round_to_bfloat16(float v) {
+    uint32_t input;
+    FP32 f;
+    f.f = v;
+    input = f.u;
+    bfloat16 output;
+
+    if (float_isnan(v)) {
+      // If the value is a NaN, squash it to a qNaN with msb of fraction set,
+      // this makes sure after truncation we don't end up with an inf.
+      //
+      // qNaN magic: All exponent bits set + most significant bit of fraction
+      // set.
+      output.value = 0x7fc0;
+    } else {
+      // Fast rounding algorithm that rounds a half value to nearest even. This
+      // reduces expected error when we convert a large number of floats. Here
+      // is how it works:
+      //
+      // Definitions:
+      // To convert a float 32 to bfloat16, a float 32 can be viewed as 32 bits
+      // with the following tags:
+      //
+      // Sign |  Exp (8 bits) | Frac (23 bits)
+      //  S     EEEEEEEE         FFFFFFLRTTTTTTTTTTTTTTT
+      //
+      //  S: Sign bit.
+      //  E: Exponent bits.
+      //  F: First 6 bits of fraction.
+      //  L: Least significant bit of resulting bfloat16 if we truncate away the
+      //  rest of the float32. This is also the 7th bit of fraction
+      //  R: Rounding bit, 8th bit of fraction.
+      //  T: Sticky bits, rest of fraction, 15 bits.
+      //
+      // To round half to nearest even, there are 3 cases where we want to round
+      // down (simply truncate the result of the bits away, which consists of
+      // rounding bit and sticky bits) and two cases where we want to round up
+      // (truncate then add one to the result).
+      //
+      // The fast converting algorithm simply adds lsb (L) to 0x7fff (15 bits of
+      // 1s) as the rounding bias, adds the rounding bias to the input, then
+      // truncates the last 16 bits away.
+      //
+      // To understand how it works, we can analyze this algorithm case by case:
+      //
+      // 1. L = 0, R = 0:
+      //   Expect: round down, this is less than half value.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input may create any carry, depending on
+      //   whether there is any value set to 1 in T bits.
+      //   - R may be set to 1 if there is a carry.
+      //   - L remains 0.
+      //   - Note that this case also handles Inf and -Inf, where all fraction
+      //   bits, including L, R and Ts are all 0. The output remains Inf after
+      //   this algorithm.
+      //
+      // 2. L = 1, R = 0:
+      //   Expect: round down, this is less than half value.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 1 = 0x8000
+      //   - Adding rounding bias to input doesn't change sticky bits but
+      //   adds 1 to rounding bit.
+      //   - L remains 1.
+      //
+      // 3. L = 0, R = 1, all of T are 0:
+      //   Expect: round down, this is exactly at half, the result is already
+      //   even (L=0).
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input sets all sticky bits to 1, but
+      //   doesn't create a carry.
+      //   - R remains 1.
+      //   - L remains 0.
+      //
+      // 4. L = 1, R = 1:
+      //   Expect: round up, this is exactly at half, the result needs to be
+      //   round to the next even number.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 1 = 0x8000
+      //   - Adding rounding bias to input doesn't change sticky bits, but
+      //   creates a carry from rounding bit.
+      //   - The carry sets L to 0, creates another carry bit and propagate
+      //   forward to F bits.
+      //   - If all the F bits are 1, a carry then propagates to the exponent
+      //   bits, which then creates the minimum value with the next exponent
+      //   value. Note that we won't have the case where exponents are all 1,
+      //   since that's either a NaN (handled in the other if condition) or inf
+      //   (handled in case 1).
+      //
+      // 5. L = 0, R = 1, any of T is 1:
+      //   Expect: round up, this is greater than half.
+      //
+      //   Algorithm:
+      //   - Rounding bias: 0x7fff + 0 = 0x7fff
+      //   - Adding rounding bias to input creates a carry from sticky bits,
+      //   sets rounding bit to 0, then create another carry.
+      //   - The second carry sets L to 1.
+      //
+      // Examples:
+      //
+      //  Exact half value that is already even:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0     1000000000000000
+      //
+      //     This falls into case 3. We truncate the rest of 16 bits and no
+      //     carry is created into F and L:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+      //
+      //  Exact half value, round to next even number:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 0 1     1000000000000000
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     which then propagates into L and F:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 0      0 0 0 0 0 1 0
+      //
+      //
+      //  Max denormal value round to min normal value:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     0 0 0 0 0 0 0 0      1 1 1 1 1 1 1     1111111111111111
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     propagate into L and F, which then propagates into exponent
+      //     bits:
+      //
+      //    Output:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     0 0 0 0 0 0 0 1      0 0 0 0 0 0 0
+      //
+      //  Max normal value round to Inf:
+      //    Input:
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit) | Frac (last 16 bit)
+      //     S     E E E E E E E E      F F F F F F L     RTTTTTTTTTTTTTTT
+      //     0     1 1 1 1 1 1 1 0      1 1 1 1 1 1 1     1111111111111111
+      //
+      //     This falls into case 4. We create a carry from R and T,
+      //     propagate into L and F, which then propagates into exponent
+      //     bits:
+      //
+      //    Sign |  Exp (8 bit)     | Frac (first 7 bit)
+      //     S     E E E E E E E E      F F F F F F L
+      //     0     1 1 1 1 1 1 1 1      0 0 0 0 0 0 0
+      //
+      //
+      // Least significant bit of resulting bfloat.
+      uint32_t lsb = (input >> 16) & 1;
+      uint32_t rounding_bias = 0x7fff + lsb;
+      input += rounding_bias;
+      output.value = static_cast<uint16_t>(input >> 16);
+    }
+    return output;
+  }
+
   static bfloat16 epsilon() {
     bfloat16 x;
     x.value = 0x3c00;  // 0x1.0p-7
@@ -177,7 +363,7 @@ struct bfloat16 {
   static const uint16_t NAN_VALUE = 0x7FC0;
 
  private:
-  B16_DEVICE_FUNC bool float_isnan(const float& x) {
+  B16_DEVICE_FUNC static bool float_isnan(const float& x) {
 #ifdef __CUDA_ARCH__
     return ::isnan(x);
 #else
-- 
GitLab


From c2b346538f8a651bd8adb5fa557bdfac0394c2c3 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 21 Mar 2018 14:26:14 -0700
Subject: [PATCH 322/960] Avoid taking a reference on a temporary value

PiperOrigin-RevId: 189967517
---
 tensorflow/core/grappler/optimizers/loop_optimizer.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/loop_optimizer.cc b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
index bd0d94b83f..a063dc3381 100644
--- a/tensorflow/core/grappler/optimizers/loop_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/loop_optimizer.cc
@@ -368,7 +368,7 @@ Status LoopOptimizer::FindInvariantNodes(NodeDef* node) {
     bool is_invariant = true;
     for (const auto& input : consumer->input()) {
       if (!IsControlInput(input)) {
-        const auto& name = NodeName(input);
+        const string name = NodeName(input);
         auto* producer = node_map_->GetNode(name);
         if (!invariant_nodes_.count(producer)) {
           if (IsConstant(*producer)) {
-- 
GitLab


From a07bd80e27dd41a1b6a3f4c2e1954ae573453cda Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 14:59:06 -0700
Subject: [PATCH 323/960] Add an alternative "no_contrib" BUILD target to
 tensorflow/python to avoid including contrib packages

PiperOrigin-RevId: 189973359
---
 tensorflow/python/BUILD | 28 +++++++++++++++++++---------
 1 file changed, 19 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 54e944c264..079905781d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -58,6 +58,18 @@ py_library(
         "//tensorflow/tools/api/generator:__pkg__",
         "//tensorflow/tools/quantization:__pkg__",  # TODO(b/34059704): remove when fixed
     ],
+    deps = [":no_contrib"] + if_not_windows([
+        "//tensorflow/contrib:contrib_py",
+    ]),
+)
+
+py_library(
+    name = "no_contrib",
+    srcs = ["__init__.py"],
+    srcs_version = "PY2AND3",
+    visibility = [
+        "//tensorflow:__pkg__",
+    ],
     deps = [
         ":array_ops",
         ":bitwise_ops",
@@ -86,6 +98,7 @@ py_library(
         ":ops",
         ":platform",
         ":pywrap_tensorflow",
+        ":saver_test_utils",
         ":script_ops",
         ":session_ops",
         ":sets",
@@ -95,31 +108,28 @@ py_library(
         ":standard_ops",
         ":state_ops",
         ":string_ops",
+        ":subscribe",
         ":summary",
         ":tensor_array_ops",
-        ":training",
-        ":saver_test_utils",
-        ":subscribe",
         ":test_ops",  # TODO: Break testing code out into separate rule.
-        ":tf_item",
         ":tf_cluster",
+        ":tf_item",
         ":tf_optimizer",
+        ":training",
         ":util",
         ":weights_broadcast_ops",
-        "//third_party/py/numpy",
         "//tensorflow/core:protos_all_py",
         "//tensorflow/python/data",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/feature_column:feature_column_py",
         "//tensorflow/python/keras",
-        "//tensorflow/python/ops/losses",
         "//tensorflow/python/ops/distributions",
         "//tensorflow/python/ops/linalg",
+        "//tensorflow/python/ops/losses",
         "//tensorflow/python/profiler",
         "//tensorflow/python/saved_model",
-    ] + if_not_windows([
-        "//tensorflow/contrib:contrib_py",
-    ]),
+        "//third_party/py/numpy",
+    ],
 )
 
 tf_py_build_info_genrule()
-- 
GitLab


From 259525189f41eaa1c08e0baa0dbac8d2cf3a15ef Mon Sep 17 00:00:00 2001
From: Joel Shor <joel-shor@users.noreply.github.com>
Date: Thu, 22 Mar 2018 00:11:33 +0200
Subject: [PATCH 324/960] Fix dataset resampling bug introduced by a bug in
 datasets itself. fixes #16606 (#17896)

* Fixes github issue #16606.

The core issue is that in the case of certain random Tensors, the
following two lines aren't the same:

```
rand_0s_and_1s_ds = ...
gather_ds = rand_0s_and_1s_ds.map(lambda i: tf.gather([0, 1], i))
tup_ds = tf.data.Dataset.zip(gather_ds, rand_0s_and_1s_ds)
```

```
rand_0s_and_1s_ds = ...
tup_ds = rand_0s_and_1s_ds.map(lambda i: (tf.gather([0, 1], i), i))
Note that this does NOT fix the underlying issue of drawing multiple
sampes from the underlying distribution.
```

Tested:
With the new test, bazel test :resample_test fails before and succeeds
after.

* Fixes github issue #16606.

The core issue is that in the case of certain random Tensors, the
following two lines aren't the same:

```
rand_0s_and_1s_ds = ...
gather_ds = rand_0s_and_1s_ds.map(lambda i: tf.gather([0, 1], i))
tup_ds = tf.data.Dataset.zip(gather_ds, rand_0s_and_1s_ds)
```

```
rand_0s_and_1s_ds = ...
tup_ds = rand_0s_and_1s_ds.map(lambda i: (tf.gather([0, 1], i), i))
Note that this does NOT fix the underlying issue of drawing multiple
sampes from the underlying distribution.
```

Tested:
With the new test, bazel test :resample_test fails before and succeeds
after.

* Undo a spurious git-induced change.

* Fix indent issue.

* Fix indent issue.
---
 .../data/python/kernel_tests/resample_test.py | 40 +++++++++++++++++++
 .../contrib/data/python/ops/resampling.py     | 10 +++--
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 913ab9b9f8..527618b231 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -21,8 +21,11 @@ import numpy as np
 
 from tensorflow.contrib.data.python.ops import resampling
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -68,6 +71,43 @@ class ResampleTest(test.TestCase):
     returned_dist = class_counts / total_returned
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
+  def testRandomClasses(self):
+    init_dist = [0.25, 0.25, 0.25, 0.25]
+    target_dist = [0.0, 0.0, 0.0, 1.0]
+    num_classes = len(init_dist)
+    num_samples = 100   # We don't need many samples to test a dirac-delta target distribution
+    data_np = np.random.choice(num_classes, num_samples, p=init_dist)
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
+
+    # Apply a random mapping that preserves the data distribution.
+    def _remap_fn(_):
+      return math_ops.cast(random_ops.random_uniform([1]) * num_classes,
+                           dtypes.int32)[0]
+    dataset = dataset.map(_remap_fn)
+
+    # Reshape distribution.
+    dataset = dataset.apply(
+        resampling.rejection_resample(
+            class_func=lambda x: x,
+            target_dist=target_dist,
+            initial_dist=init_dist))
+
+    get_next = dataset.make_one_shot_iterator().get_next()
+
+
+    with self.test_session() as sess:
+      returned = []
+      with self.assertRaises(errors.OutOfRangeError):
+        while True:
+          returned.append(sess.run(get_next))
+
+    classes, _ = zip(*returned)
+    bincount = np.bincount(
+        np.array(classes),
+        minlength=num_classes).astype(np.float32) / len(classes)
+
+    self.assertAllClose(target_dist, bincount, atol=1e-2)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index f4015f19fb..b465397437 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -101,14 +101,16 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
                                                    initial_dist_ds))
                           .map(maybe_warn_on_large_rejection))
 
-    current_probabilities_ds = dataset_ops.Dataset.zip(
-        (acceptance_dist_ds, class_values_ds)).map(array_ops.gather)
+    def _gather_and_copy(class_val, acceptance_prob, data):
+      return (class_val, array_ops.gather(acceptance_prob, class_val), data)
+    current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
+        (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
     filtered_ds = (
-        dataset_ops.Dataset.zip((class_values_ds, current_probabilities_ds,
-                                 dataset))
+        current_probabilities_and_class_and_data_ds
         .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
+
   return _apply_fn
 
 
-- 
GitLab


From 9cd65e9a9081640934b2b78cf84b6e51ddd69796 Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <ndesaulniers@google.com>
Date: Wed, 21 Mar 2018 15:23:07 -0700
Subject: [PATCH 325/960] [TF:XLA] do not emit bfloat16 sum reductions from
 tf2xla

bfloat16 is a storage format, not a computation format. Doing reductions in
this reduced precision is prone to quickly overflow.  Instead, emit a float32
computation, and wrap the reduce params and result in conversions to and from float32.

PiperOrigin-RevId: 189977590
---
 .../compiler/tf2xla/kernels/batch_norm_op.cc  | 57 +++++------
 .../compiler/tf2xla/kernels/bias_ops.cc       | 13 ++-
 .../compiler/tf2xla/kernels/conv_ops.cc       |  6 +-
 .../tf2xla/kernels/fake_quantize_ops.cc       | 18 +++-
 .../compiler/tf2xla/kernels/image_ops.cc      |  8 +-
 .../compiler/tf2xla/kernels/l2loss_op.cc      | 23 ++---
 tensorflow/compiler/tf2xla/kernels/lrn_ops.cc | 36 ++++---
 .../compiler/tf2xla/kernels/pooling_ops.cc    | 95 ++++++++++---------
 .../compiler/tf2xla/kernels/reduction_ops.cc  | 42 +++++---
 .../compiler/tf2xla/kernels/reduction_ops.h   |  9 +-
 .../tf2xla/kernels/reduction_ops_common.cc    | 46 ++++-----
 .../compiler/tf2xla/kernels/scan_ops.cc       |  8 +-
 .../compiler/tf2xla/kernels/softmax_op.cc     | 51 +++++-----
 tensorflow/compiler/tf2xla/xla_helpers.cc     | 17 ++++
 tensorflow/compiler/tf2xla/xla_helpers.h      | 12 +++
 tensorflow/compiler/xla/literal_util.cc       |  3 +
 tensorflow/compiler/xla/tests/convert_test.cc | 40 ++++++++
 17 files changed, 305 insertions(+), 179 deletions(-)

diff --git a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
index a249b1869f..931175be11 100644
--- a/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/batch_norm_op.cc
@@ -118,30 +118,24 @@ class FusedBatchNormGradOp : public XlaOpKernel {
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
-    xla::ComputationBuilder* b = ctx->builder();
-
-    auto grad_backprop = ctx->Input(0);
-    auto activations = ctx->Input(1);
-    auto scale = ctx->Input(2);
-    auto mean = ctx->Input(3);
-    auto var = ctx->Input(4);
-
-    TensorShape input_shape = ctx->InputShape(0);
-    int feature_index =
-        GetTensorFeatureDimIndex(input_shape.dims(), data_format_);
-
+    xla::ComputationBuilder* const b = ctx->builder();
     DataType input_dtype = ctx->input_type(0);
     DataType scale_dtype = ctx->input_type(2);
-    xla::PrimitiveType input_type;
-    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(input_dtype, &input_type));
-    xla::PrimitiveType scale_type;
-    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(scale_dtype, &scale_type));
 
     // TODO(b/69928690): support mixed precision in the XLA batch normalization
     // operators. For now, cast everything to the statistics type (which
     // may be more precise than the input type).
-    grad_backprop = b->ConvertElementType(grad_backprop, scale_type);
-    activations = b->ConvertElementType(activations, scale_type);
+    auto grad_backprop =
+        XlaHelpers::ConvertElementType(b, ctx->Input(0), scale_dtype);
+    auto activations =
+        XlaHelpers::ConvertElementType(b, ctx->Input(1), scale_dtype);
+    auto scale = ctx->Input(2);
+    auto mean = ctx->Input(3);
+    auto var = ctx->Input(4);
+
+    const int input_dims = ctx->InputShape(0).dims();
+    const int feature_index =
+        GetTensorFeatureDimIndex(input_dims, data_format_);
 
     xla::ComputationDataHandle x_backprop;
     xla::ComputationDataHandle scale_backprop;
@@ -156,7 +150,7 @@ class FusedBatchNormGradOp : public XlaOpKernel {
       offset_backprop = b->GetTupleElement(output, 2);
     } else {
       // Reduce over all dimensions except the feature dim.
-      std::vector<int64> reduction_dims(input_shape.dims() - 1);
+      std::vector<int64> reduction_dims(input_dims - 1);
       std::iota(reduction_dims.begin(), reduction_dims.begin() + feature_index,
                 0);
       std::iota(reduction_dims.begin() + feature_index, reduction_dims.end(),
@@ -165,9 +159,14 @@ class FusedBatchNormGradOp : public XlaOpKernel {
       // scale_backprop = y_backprop * ((x - pop_mean) * rsqrt(pop_var +
       // epsilon))
       // x_backprop = y_backprop * (scale * rsqrt(pop_var + epsilon))
-      offset_backprop =
-          b->Reduce(grad_backprop, XlaHelpers::Zero(b, scale_dtype),
-                    *ctx->GetOrCreateAdd(scale_dtype), reduction_dims);
+      const DataType accumulation_type =
+          XlaHelpers::SumAccumulationType(scale_dtype);
+      auto converted =
+          XlaHelpers::ConvertElementType(b, grad_backprop, accumulation_type);
+      auto reduce =
+          b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                    *ctx->GetOrCreateAdd(accumulation_type), reduction_dims);
+      offset_backprop = XlaHelpers::ConvertElementType(b, reduce, scale_dtype);
 
       // scratch1 = rsqrt(pop_var + epsilon)
       auto neg_half = XlaHelpers::FloatLiteral(b, scale_dtype, -0.5);
@@ -175,17 +174,21 @@ class FusedBatchNormGradOp : public XlaOpKernel {
           b->Pow(b->Add(var, b->ConstantR0<float>(epsilon_)), neg_half);
 
       // scratch2 = sum(y_backprop * (x - mean))
-      auto scratch2 = b->Reduce(
-          b->Mul(grad_backprop, b->Sub(activations, mean, {feature_index})),
-          XlaHelpers::Zero(b, scale_dtype), *ctx->GetOrCreateAdd(scale_dtype),
-          reduction_dims);
+      auto mul =
+          b->Mul(grad_backprop, b->Sub(activations, mean, {feature_index}));
+      converted = XlaHelpers::ConvertElementType(b, mul, accumulation_type);
+      reduce =
+          b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                    *ctx->GetOrCreateAdd(accumulation_type), reduction_dims);
+      auto scratch2 = XlaHelpers::ConvertElementType(b, reduce, scale_dtype);
 
       x_backprop =
           b->Mul(grad_backprop, b->Mul(scratch1, scale), {feature_index});
       scale_backprop = b->Mul(scratch1, scratch2);
     }
 
-    ctx->SetOutput(0, b->ConvertElementType(x_backprop, input_type));
+    ctx->SetOutput(0,
+                   XlaHelpers::ConvertElementType(b, x_backprop, input_dtype));
     ctx->SetOutput(1, scale_backprop);
     ctx->SetOutput(2, offset_backprop);
     ctx->SetConstantOutput(3, Tensor(scale_dtype, {}));
diff --git a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
index c667b4e3e3..ed33b8ed2e 100644
--- a/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/bias_ops.cc
@@ -103,10 +103,15 @@ class BiasAddGradOp : public XlaOpKernel {
     std::iota(reduce_dims.begin(), reduce_dims.begin() + feature_dim, 0);
     std::iota(reduce_dims.begin() + feature_dim, reduce_dims.end(),
               feature_dim + 1);
-    xla::ComputationDataHandle result = ctx->builder()->Reduce(
-        ctx->Input(0), XlaHelpers::Zero(ctx->builder(), input_type(0)),
-        *ctx->GetOrCreateAdd(input_type(0)), reduce_dims);
-    ctx->SetOutput(0, result);
+    xla::ComputationBuilder* const b = ctx->builder();
+    const DataType accumulation_type =
+        XlaHelpers::SumAccumulationType(input_type(0));
+    auto converted =
+        XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type);
+    auto reduce =
+        b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                  *ctx->GetOrCreateAdd(accumulation_type), reduce_dims);
+    ctx->SetOutput(0, XlaHelpers::ConvertElementType(b, reduce, input_type(0)));
   }
 
  private:
diff --git a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
index 81cea6d376..c0ee0c9c2e 100644
--- a/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/conv_ops.cc
@@ -58,7 +58,7 @@ xla::ComputationDataHandle CreateExpandedZero(
 
 // Create a mask for depthwise convolution that will make a normal convolution
 // produce the same results as a depthwise convolution. For a [2, 2, 3, 2]
-// depthwise filter this returns a [2, 2, 3, 6] tesnsor
+// depthwise filter this returns a [2, 2, 3, 6] tensor
 //   1 1 0 0 0 0   1 1 0 0 0 0
 //   0 0 1 1 0 0   0 0 1 1 0 0
 //   0 0 0 0 1 1   0 0 0 0 1 1
@@ -166,6 +166,10 @@ xla::ComputationDataHandle ContractFilterForDepthwiseBackprop(
       CreateExpandedFilterMask(filter_shape, builder), filter_backprop,
       CreateExpandedZero(filter_shape, dtype, builder));
   return builder->Reshape(
+      // This reduce does not need inputs to be converted with
+      // XlaHelpers::SumAccumulationType() since the ExpandedFilterMask with
+      // ExpandedZero guarantees that only one element is non zero, so there
+      // cannot be accumulated precision error.
       builder->Reduce(masked_expanded_filter, XlaHelpers::Zero(builder, dtype),
                       *ctx->GetOrCreateAdd(dtype),
                       {expanded_filter_shape.dims() - 2}),
diff --git a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
index 453a32c494..99470d70e7 100644
--- a/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/fake_quantize_ops.cc
@@ -247,6 +247,8 @@ class FakeQuantWithMinMaxVarsGradOp : public XlaOpKernel {
     const TensorShape gradient_shape = ctx->InputShape(0);
     xla::ComputationDataHandle input = ctx->Input(1);
     const DataType data_type = ctx->input_type(1);
+    const DataType accumulation_type =
+        XlaHelpers::SumAccumulationType(data_type);
     xla::ComputationDataHandle input_min = ctx->Input(2);
     xla::ComputationDataHandle input_max = ctx->Input(3);
 
@@ -265,15 +267,23 @@ class FakeQuantWithMinMaxVarsGradOp : public XlaOpKernel {
     ctx->SetOutput(0, output0);
 
     xla::ComputationDataHandle below_min = b->Lt(input, nudged_input_min);
+    xla::ComputationDataHandle select1 = b->Select(below_min, gradient, zeroes);
+    xla::ComputationDataHandle reduce1 = b->ReduceAll(
+        XlaHelpers::ConvertElementType(b, select1, accumulation_type),
+        XlaHelpers::Zero(b, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type));
     xla::ComputationDataHandle output1 =
-        b->ReduceAll(b->Select(below_min, gradient, zeroes), zero,
-                     *ctx->GetOrCreateAdd(data_type));
+        XlaHelpers::ConvertElementType(b, reduce1, data_type);
     ctx->SetOutput(1, output1);
 
     xla::ComputationDataHandle above_max = b->Gt(input, nudged_input_max);
+    xla::ComputationDataHandle select2 = b->Select(above_max, gradient, zeroes);
+    xla::ComputationDataHandle reduce2 = b->ReduceAll(
+        XlaHelpers::ConvertElementType(b, select2, accumulation_type),
+        XlaHelpers::Zero(b, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type));
     xla::ComputationDataHandle output2 =
-        b->ReduceAll(b->Select(above_max, gradient, zeroes), zero,
-                     *ctx->GetOrCreateAdd(data_type));
+        XlaHelpers::ConvertElementType(b, reduce2, data_type);
     ctx->SetOutput(2, output2);
   }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/image_ops.cc b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
index f22f384256..5eeda79a93 100644
--- a/tensorflow/compiler/tf2xla/kernels/image_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/image_ops.cc
@@ -180,9 +180,13 @@ class AdjustContrastOpV2 : public XlaOpKernel {
 
     DataType type = context->input_type(0);
 
-    auto output = b->Reduce(input, /*init_value=*/XlaHelpers::Zero(b, type),
-                            /*computation=*/*context->GetOrCreateAdd(type),
+    const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
+    auto converted =
+        XlaHelpers::ConvertElementType(b, input, accumulation_type);
+    auto reduce = b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                            *context->GetOrCreateAdd(accumulation_type),
                             {height_dim, width_dim});
+    auto output = XlaHelpers::ConvertElementType(b, reduce, type);
     output = b->Div(output, XlaHelpers::FloatLiteral(b, type, height * width));
 
     std::vector<int64> broadcast_dims(input_shape.dims() - 2);
diff --git a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
index d096415087..c177f08d9c 100644
--- a/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/l2loss_op.cc
@@ -29,21 +29,22 @@ class L2LossOp : public XlaOpKernel {
   explicit L2LossOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {}
 
   void Compile(XlaOpKernelContext* ctx) override {
-    const TensorShape input_shape = ctx->InputShape(0);
+    std::vector<int64> dims(ctx->InputShape(0).dims());
+    std::iota(dims.begin(), dims.end(), 0);
 
     DataType dtype = ctx->input_type(0);
-    xla::ComputationBuilder* b = ctx->builder();
-
-    auto zero = XlaHelpers::Zero(b, dtype);
-    auto two = XlaHelpers::IntegerLiteral(b, dtype, 2);
-    const xla::Computation& add = *ctx->GetOrCreateAdd(dtype);
-
-    std::vector<int64> dims(input_shape.dims());
-    std::iota(dims.begin(), dims.end(), 0);
+    xla::ComputationBuilder* const b = ctx->builder();
 
     //  output = sum(t ** 2) / 2
-    auto x = ctx->Input(0);
-    ctx->SetOutput(0, b->Div(b->Reduce(b->Mul(x, x), zero, add, dims), two));
+    const DataType accumulation_type = XlaHelpers::SumAccumulationType(dtype);
+    auto t =
+        XlaHelpers::ConvertElementType(b, ctx->Input(0), accumulation_type);
+    auto square = b->Mul(t, t);
+    auto reduce = b->Reduce(square, XlaHelpers::Zero(b, accumulation_type),
+                            *ctx->GetOrCreateAdd(accumulation_type), dims);
+    auto deconverted = XlaHelpers::ConvertElementType(b, reduce, dtype);
+    auto two = XlaHelpers::IntegerLiteral(b, dtype, 2);
+    ctx->SetOutput(0, b->Div(deconverted, two));
   }
 };
 
diff --git a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
index 759d1a1a2d..1cfee3070f 100644
--- a/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/lrn_ops.cc
@@ -47,12 +47,17 @@ class LRNOp : public XlaOpKernel {
 
     // We use a window of depth_radius_ * 2 + 1, to account for the current
     // element and a depth_radius_ on either side.
-    auto squared = builder->Mul(input, input);
-    auto sqr_sum = builder->ReduceWindow(
-        squared, XlaHelpers::Zero(builder, input_type(0)),
-        *ctx->GetOrCreateAdd(input_type(0)),
+    auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0));
+    auto converted =
+        XlaHelpers::ConvertElementType(builder, input, accumulation_type);
+    auto squared = builder->Mul(converted, converted);
+    auto reduce = builder->ReduceWindow(
+        squared, XlaHelpers::Zero(builder, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
+    auto sqr_sum =
+        XlaHelpers::ConvertElementType(builder, reduce, input_type(0));
 
     auto scale = builder->Pow(
         builder->Add(builder->ConstantR0<float>(bias_),
@@ -130,12 +135,17 @@ class LRNGradOp : public XlaOpKernel {
     //     dyi *= out_grads[j]
     //     grads[k] += dyi
 
-    auto squared = builder->Mul(in_image, in_image);
-    auto sqr_sum = builder->ReduceWindow(
-        squared, XlaHelpers::Zero(builder, input_type(0)),
-        *ctx->GetOrCreateAdd(input_type(0)),
+    auto accumulation_type = XlaHelpers::SumAccumulationType(input_type(0));
+    auto converted =
+        XlaHelpers::ConvertElementType(builder, in_image, accumulation_type);
+    auto squared = builder->Mul(converted, converted);
+    auto reduce = builder->ReduceWindow(
+        squared, XlaHelpers::Zero(builder, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
+    auto sqr_sum =
+        XlaHelpers::ConvertElementType(builder, reduce, input_type(0));
 
     auto norm =
         builder->Add(builder->ConstantR0<float>(bias_),
@@ -146,11 +156,15 @@ class LRNGradOp : public XlaOpKernel {
                      builder->Div(out_image, norm)),
         in_grads);
 
-    auto dy_reduced = builder->ReduceWindow(
-        dy, XlaHelpers::Zero(builder, input_type(0)),
-        *ctx->GetOrCreateAdd(input_type(0)),
+    auto converted_dy =
+        XlaHelpers::ConvertElementType(builder, dy, accumulation_type);
+    auto dy_reduce = builder->ReduceWindow(
+        converted_dy, XlaHelpers::Zero(builder, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type),
         /* window_dimensions = */ {1, 1, 1, depth_radius_ * 2 + 1},
         /* window_strides = */ {1, 1, 1, 1}, xla::Padding::kSame);
+    auto dy_reduced =
+        XlaHelpers::ConvertElementType(builder, dy_reduce, input_type(0));
 
     xla::ComputationDataHandle gradients = builder->Add(
         builder->Mul(in_image, dy_reduced),
diff --git a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
index 086a9491aa..5f635dd1bc 100644
--- a/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/pooling_ops.cc
@@ -35,8 +35,11 @@ namespace {
 // Superclass of pooling ops.
 class PoolingOp : public XlaOpKernel {
  public:
-  PoolingOp(OpKernelConstruction* ctx, int num_spatial_dims)
-      : XlaOpKernel(ctx), num_spatial_dims_(num_spatial_dims) {
+  PoolingOp(OpKernelConstruction* ctx, int num_spatial_dims,
+            const DataType reduction_type)
+      : XlaOpKernel(ctx),
+        num_spatial_dims_(num_spatial_dims),
+        reduction_type_(reduction_type) {
     if (ctx->num_inputs() == 1) {
       std::vector<int32> ksize_int;
       std::vector<int32> stride_int;
@@ -63,12 +66,10 @@ class PoolingOp : public XlaOpKernel {
   int num_dims() const { return num_spatial_dims_ + 2; }
 
   // Method that builds an initial value to use in reductions.
-  virtual xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b,
-                                               DataType data_type) = 0;
+  virtual xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b) = 0;
 
   // The reduction operation to apply to each window.
-  virtual const xla::Computation* Reduction(XlaOpKernelContext* ctx,
-                                            DataType dtype) = 0;
+  virtual const xla::Computation* Reduction(XlaOpKernelContext* ctx) = 0;
 
   // A post-processing operation to apply on the outputs of the ReduceWindow.
   virtual xla::ComputationDataHandle PostProcessOutput(
@@ -76,9 +77,6 @@ class PoolingOp : public XlaOpKernel {
       DataType dtype, const TensorShape& input_shape) = 0;
 
   void Compile(XlaOpKernelContext* ctx) override {
-    xla::ComputationDataHandle input = ctx->Input(0);
-    const TensorShape input_shape = ctx->InputShape(0);
-
     std::vector<int64> ksize = ksize_;
     std::vector<int64> stride = stride_;
     if (ctx->num_inputs() != 1) {
@@ -106,16 +104,20 @@ class PoolingOp : public XlaOpKernel {
       stride.clear();
       OP_REQUIRES_OK(ctx, ctx->ConstantInputAsIntVector(2, &stride));
     }
+    const TensorShape input_shape = ctx->InputShape(0);
     OP_REQUIRES(ctx, input_shape.dims() == num_dims(),
                 errors::InvalidArgument("Input to ", type_string(),
                                         " operator must have ", num_dims(),
                                         " dimensions"));
 
-    const DataType type = input_type(0);
-    xla::ComputationDataHandle pooled = ctx->builder()->ReduceWindow(
-        input, InitValue(ctx->builder(), type), *Reduction(ctx, type), ksize,
-        stride, padding_);
-    ctx->SetOutput(0, PostProcessOutput(ctx, pooled, type, input_shape));
+    xla::ComputationBuilder* const b = ctx->builder();
+    auto input =
+        XlaHelpers::ConvertElementType(b, ctx->Input(0), reduction_type_);
+    auto reduce = ctx->builder()->ReduceWindow(
+        input, InitValue(b), *Reduction(ctx), ksize, stride, padding_);
+    auto pooled = XlaHelpers::ConvertElementType(b, reduce, input_type(0));
+    ctx->SetOutput(0,
+                   PostProcessOutput(ctx, pooled, input_type(0), input_shape));
   }
 
  protected:
@@ -124,21 +126,21 @@ class PoolingOp : public XlaOpKernel {
   std::vector<int64> stride_;
   xla::Padding padding_;
   TensorFormat data_format_ = FORMAT_NHWC;
+  DataType reduction_type_;
 };
 
 class MaxPoolOp : public PoolingOp {
  public:
   MaxPoolOp(OpKernelConstruction* ctx, int num_spatial_dims)
-      : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims) {}
+      : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims,
+                  /*reduction_type=*/ctx->input_type(0)) {}
 
-  xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b,
-                                       DataType data_type) override {
-    return XlaHelpers::MinValue(b, data_type);
+  xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b) override {
+    return XlaHelpers::MinValue(b, reduction_type_);
   }
 
-  const xla::Computation* Reduction(XlaOpKernelContext* ctx,
-                                    DataType dtype) override {
-    return ctx->GetOrCreateMax(dtype);
+  const xla::Computation* Reduction(XlaOpKernelContext* ctx) override {
+    return ctx->GetOrCreateMax(reduction_type_);
   }
 
   xla::ComputationDataHandle PostProcessOutput(
@@ -209,15 +211,17 @@ static xla::ComputationDataHandle AvgPoolDivideByCount(
     }
 
     // Build a matrix of all 1s, with the same width/height as the input.
+    const DataType accumulation_type = XlaHelpers::SumAccumulationType(dtype);
     auto ones = ctx->builder()->Broadcast(
-        XlaHelpers::One(ctx->builder(), dtype), input_dim_sizes);
+        XlaHelpers::One(ctx->builder(), accumulation_type), input_dim_sizes);
 
     // Perform a ReduceWindow with the same window size, strides, and padding
     // to count the number of contributions to each result element.
-    auto counts = ctx->builder()->ReduceWindow(
-        ones, XlaHelpers::Zero(ctx->builder(), dtype),
-        *ctx->GetOrCreateAdd(dtype), window_ksize, window_stride,
+    auto reduce = ctx->builder()->ReduceWindow(
+        ones, XlaHelpers::Zero(ctx->builder(), accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type), window_ksize, window_stride,
         xla::Padding::kSame);
+    auto counts = XlaHelpers::ConvertElementType(ctx->builder(), reduce, dtype);
 
     return ctx->builder()->Div(output, counts, window_dims);
   }
@@ -226,16 +230,16 @@ static xla::ComputationDataHandle AvgPoolDivideByCount(
 class AvgPoolOp : public PoolingOp {
  public:
   AvgPoolOp(OpKernelConstruction* ctx, int num_spatial_dims)
-      : PoolingOp(ctx, num_spatial_dims) {}
+      : PoolingOp(ctx, /*num_spatial_dims=*/num_spatial_dims,
+                  /*reduction_type=*/
+                  XlaHelpers::SumAccumulationType(ctx->input_type(0))) {}
 
-  xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b,
-                                       DataType data_type) override {
-    return XlaHelpers::Zero(b, data_type);
+  xla::ComputationDataHandle InitValue(xla::ComputationBuilder* b) override {
+    return XlaHelpers::Zero(b, reduction_type_);
   }
 
-  const xla::Computation* Reduction(XlaOpKernelContext* ctx,
-                                    DataType dtype) override {
-    return ctx->GetOrCreateAdd(dtype);
+  const xla::Computation* Reduction(XlaOpKernelContext* ctx) override {
+    return ctx->GetOrCreateAdd(reduction_type_);
   }
 
   xla::ComputationDataHandle PostProcessOutput(
@@ -455,14 +459,12 @@ class AvgPoolGradOp : public XlaOpKernel {
                  gradients_shape, filter_shape, out_backprop_shape, stride_,
                  padding_, data_format_, &dims));
 
+    // The input gradients are computed by a convolution of the output gradients
+    // and the filter, with some appropriate padding. See the comment at the top
+    // of conv_grad_ops.h for details.
+    xla::ComputationBuilder* const b = ctx->builder();
     auto out_backprop = ctx->Input(1);
-
-    // The input gradients are computed by a convolution of the output
-    // gradients
-    // and the filter, with some appropriate padding. See the comment at
-    // the top of conv_grad_ops.h for details.
-    DataType dtype = input_type(1);
-
+    auto dtype = input_type(1);
     xla::Padding xla_padding =
         (padding_ == VALID) ? xla::Padding::kValid : xla::Padding::kSame;
 
@@ -483,17 +485,18 @@ class AvgPoolGradOp : public XlaOpKernel {
       padding->set_interior_padding(dims.spatial_dims[i].stride - 1);
     }
 
-    auto zero = XlaHelpers::Zero(ctx->builder(), dtype);
-    auto padded_gradients =
-        ctx->builder()->Pad(out_backprop_div, zero, padding_config);
+    auto zero = XlaHelpers::Zero(b, dtype);
+    auto padded_gradients = b->Pad(out_backprop_div, zero, padding_config);
 
     // in_backprop = padded_gradients <conv> ones
     std::vector<int64> ones(num_dims(), 1LL);
-    xla::ComputationDataHandle in_backprop = ctx->builder()->ReduceWindow(
-        padded_gradients, zero, *ctx->GetOrCreateAdd(dtype), ksize_,
+    auto accumulation_type = XlaHelpers::SumAccumulationType(dtype);
+    auto in_backprop = b->ReduceWindow(
+        XlaHelpers::ConvertElementType(b, padded_gradients, accumulation_type),
+        XlaHelpers::Zero(b, accumulation_type),
+        *ctx->GetOrCreateAdd(accumulation_type), ksize_,
         /* window_strides=*/ones, xla::Padding::kValid);
-
-    ctx->SetOutput(0, in_backprop);
+    ctx->SetOutput(0, XlaHelpers::ConvertElementType(b, in_backprop, dtype));
   }
 
  protected:
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
index 03b13b2924..812d258cd1 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.cc
@@ -27,7 +27,13 @@ namespace {
 
 class SumOp : public XlaReductionOp {
  public:
-  explicit SumOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit SumOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx,
+                       XlaHelpers::SumAccumulationType(ctx->input_type(0))) {}
+  xla::ComputationDataHandle InitialValue(
+      xla::ComputationBuilder* builder) override {
+    return XlaHelpers::Zero(builder, reduction_type_);
+  }
   void BuildReducer(xla::ComputationBuilder* builder,
                     const xla::ComputationDataHandle& scalar_lhs,
                     const xla::ComputationDataHandle& scalar_rhs) override {
@@ -39,11 +45,13 @@ REGISTER_XLA_OP(Name("Sum").CompileTimeConstInput("reduction_indices"), SumOp);
 
 class ProdOp : public XlaReductionOp {
  public:
-  explicit ProdOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit ProdOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx,
+                       XlaHelpers::SumAccumulationType(ctx->input_type(0))) {}
 
   xla::ComputationDataHandle InitialValue(
       xla::ComputationBuilder* builder) override {
-    return XlaHelpers::One(builder, input_type(0));
+    return XlaHelpers::One(builder, reduction_type_);
   }
 
   void BuildReducer(xla::ComputationBuilder* builder,
@@ -58,13 +66,12 @@ REGISTER_XLA_OP(Name("Prod").CompileTimeConstInput("reduction_indices"),
 
 class MinOp : public XlaReductionOp {
  public:
-  explicit MinOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit MinOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx, ctx->input_type(0)) {}
 
   xla::ComputationDataHandle InitialValue(
       xla::ComputationBuilder* builder) override {
-    xla::PrimitiveType type;
-    TF_CHECK_OK(DataTypeToPrimitiveType(input_type(0), &type));
-    return builder->ConstantLiteral(xla::Literal::MaxValue(type));
+    return XlaHelpers::MaxValue(builder, reduction_type_);
   }
 
   void BuildReducer(xla::ComputationBuilder* builder,
@@ -78,13 +85,12 @@ REGISTER_XLA_OP(Name("Min").CompileTimeConstInput("reduction_indices"), MinOp);
 
 class MaxOp : public XlaReductionOp {
  public:
-  explicit MaxOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit MaxOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx, ctx->input_type(0)) {}
 
   xla::ComputationDataHandle InitialValue(
       xla::ComputationBuilder* builder) override {
-    xla::PrimitiveType type;
-    TF_CHECK_OK(DataTypeToPrimitiveType(input_type(0), &type));
-    return builder->ConstantLiteral(xla::Literal::MinValue(type));
+    return XlaHelpers::MinValue(builder, reduction_type_);
   }
 
   void BuildReducer(xla::ComputationBuilder* builder,
@@ -98,8 +104,14 @@ REGISTER_XLA_OP(Name("Max").CompileTimeConstInput("reduction_indices"), MaxOp);
 
 class MeanOp : public XlaReductionOp {
  public:
-  explicit MeanOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit MeanOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx,
+                       XlaHelpers::SumAccumulationType(ctx->input_type(0))) {}
 
+  xla::ComputationDataHandle InitialValue(
+      xla::ComputationBuilder* builder) override {
+    return XlaHelpers::Zero(builder, reduction_type_);
+  }
   void BuildReducer(xla::ComputationBuilder* builder,
                     const xla::ComputationDataHandle& scalar_lhs,
                     const xla::ComputationDataHandle& scalar_rhs) override {
@@ -121,7 +133,8 @@ REGISTER_XLA_OP(Name("Mean").CompileTimeConstInput("reduction_indices"),
 
 class AllOp : public XlaReductionOp {
  public:
-  explicit AllOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit AllOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx, ctx->input_type(0)) {}
 
   xla::ComputationDataHandle InitialValue(
       xla::ComputationBuilder* builder) override {
@@ -139,7 +152,8 @@ REGISTER_XLA_OP(Name("All").CompileTimeConstInput("reduction_indices"), AllOp);
 
 class AnyOp : public XlaReductionOp {
  public:
-  explicit AnyOp(OpKernelConstruction* ctx) : XlaReductionOp(ctx) {}
+  explicit AnyOp(OpKernelConstruction* ctx)
+      : XlaReductionOp(ctx, ctx->input_type(0)) {}
 
   xla::ComputationDataHandle InitialValue(
       xla::ComputationBuilder* builder) override {
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
index 9aca6d8fed..f3181f0dad 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops.h
@@ -33,12 +33,12 @@ namespace tensorflow {
 // xla::ComputationBuilder.
 class XlaReductionOp : public XlaOpKernel {
  public:
-  explicit XlaReductionOp(OpKernelConstruction* ctx);
+  XlaReductionOp(OpKernelConstruction* ctx, DataType reduction_type);
   ~XlaReductionOp() override {}
 
-  // Return the base case for the reduction. Defaults to zero.
+  // Return the base case for the reduction.
   virtual xla::ComputationDataHandle InitialValue(
-      xla::ComputationBuilder* builder);
+      xla::ComputationBuilder* builder) = 0;
 
   // Implement the (scalar,scalar)->scalar lambda that should be
   // applied to each pair of elements to be reduced. The desired
@@ -63,6 +63,9 @@ class XlaReductionOp : public XlaOpKernel {
  private:
   // True if the number of dimensions should be maintained.
   bool keep_dims_;
+
+ protected:
+  DataType reduction_type_;
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
index 4b5d09eb9f..64fe765ae9 100644
--- a/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
+++ b/tensorflow/compiler/tf2xla/kernels/reduction_ops_common.cc
@@ -24,19 +24,15 @@ limitations under the License.
 
 namespace tensorflow {
 
-XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+XlaReductionOp::XlaReductionOp(OpKernelConstruction* ctx,
+                               DataType reduction_type)
+    : XlaOpKernel(ctx), reduction_type_(reduction_type) {
   const DataType dt = BaseType(input_type(0));
   OP_REQUIRES_OK(ctx, ctx->MatchSignature({dt, DT_INT32}, {dt}));
 
   OP_REQUIRES_OK(ctx, ctx->GetAttr("keep_dims", &keep_dims_));
 }
 
-// Return the base case for the reduction. Defaults to zero.
-xla::ComputationDataHandle XlaReductionOp::InitialValue(
-    xla::ComputationBuilder* builder) {
-  return XlaHelpers::Zero(builder, input_type(0));
-}
-
 // Unless BuildFinalizer is overridden the reduction has no
 // finalizer.
 xla::ComputationDataHandle XlaReductionOp::BuildFinalizer(
@@ -100,36 +96,26 @@ void XlaReductionOp::Compile(XlaOpKernelContext* ctx) {
 
   string desc = ctx->op_kernel().name();
 
-  // Call virtual method to get the initial value.
-  const xla::ComputationDataHandle initial = InitialValue(ctx->builder());
+  xla::ComputationBuilder* const b = ctx->builder();
   // Construct the builder for the reduction lambda.
-  xla::ComputationBuilder r(ctx->builder()->client(),
-                            strings::StrCat(desc, "-reduction"));
+  xla::ComputationBuilder r(b->client(), strings::StrCat(desc, "-reduction"));
   xla::PrimitiveType type;
-  TF_CHECK_OK(DataTypeToPrimitiveType(input_type(0), &type));
-  // Make two scalar parameters of the desired type for the lambda.
-  xla::ComputationDataHandle rx =
-      r.Parameter(0, xla::ShapeUtil::MakeShape(type, {}), "x");
-  xla::ComputationDataHandle ry =
-      r.Parameter(1, xla::ShapeUtil::MakeShape(type, {}), "y");
-
-  auto data = ctx->Input(0);
+  TF_CHECK_OK(DataTypeToPrimitiveType(reduction_type_, &type));
 
+  auto data = b->ConvertElementType(ctx->Input(0), type);
+  // Call virtual method to get the initial value.
+  auto initial = b->ConvertElementType(InitialValue(b), type);
+  // Make two scalar parameters of the desired type for the lambda.
+  auto rx = r.Parameter(0, xla::ShapeUtil::MakeShape(type, {}), "x");
+  auto ry = r.Parameter(1, xla::ShapeUtil::MakeShape(type, {}), "y");
   // Call virtual method to build the reduction lambda.
   BuildReducer(&r, rx, ry);
   xla::Computation reduction_computation = r.Build().ConsumeValueOrDie();
-  xla::ComputationDataHandle reduce =
-      ctx->builder()->Reduce(data, initial, reduction_computation, xla_axes);
 
-  xla::ComputationDataHandle finalized =
-      BuildFinalizer(ctx->builder(), reduce, num_elements_reduced);
-
-  xla::ComputationDataHandle result;
-  if (keep_dims_) {
-    result = ctx->builder()->Reshape(finalized, final_shape);
-  } else {
-    result = finalized;
-  }
+  auto reduce = b->Reduce(data, initial, reduction_computation, xla_axes);
+  auto deconverted = XlaHelpers::ConvertElementType(b, reduce, input_type(0));
+  auto finalized = BuildFinalizer(b, deconverted, num_elements_reduced);
+  auto result = keep_dims_ ? b->Reshape(finalized, final_shape) : finalized;
   ctx->SetOutput(0, result);
 }
 
diff --git a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
index ee4a94164c..4cfa28a0ce 100644
--- a/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
+++ b/tensorflow/compiler/tf2xla/kernels/scan_ops.cc
@@ -66,7 +66,7 @@ class ScanOp : public XlaOpKernel {
                                 -input_shape.dims(), ", ", input_shape.dims(),
                                 "), but got ", axis));
 
-    DataType dtype = ctx->input_type(0);
+    DataType dtype = XlaHelpers::SumAccumulationType(ctx->input_type(0));
 
     if (input_shape.num_elements() == 0) {
       // Exit early if there is nothing to compute.
@@ -91,7 +91,6 @@ class ScanOp : public XlaOpKernel {
       std::swap(padding[axis].first, padding[axis].second);
     }
 
-    xla::ComputationDataHandle input = ctx->Input(0);
     xla::ComputationDataHandle init;
     const xla::Computation* reducer;
     if (sum_) {
@@ -102,7 +101,10 @@ class ScanOp : public XlaOpKernel {
       reducer = ctx->GetOrCreateMul(dtype);
     }
     auto output = builder->ReduceWindowWithGeneralPadding(
-        ctx->Input(0), init, *reducer, window_dims, window_strides, padding);
+        XlaHelpers::ConvertElementType(builder, ctx->Input(0), dtype), init,
+        *reducer, window_dims, window_strides, padding);
+    output =
+        XlaHelpers::ConvertElementType(builder, output, ctx->input_type(0));
 
     // In exclusive mode, we have computed an extra element containing the sum
     // of all the input elements. Slice off this extra "last" element.
diff --git a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
index 750a4c2dec..aa47cb799f 100644
--- a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
@@ -42,9 +42,8 @@ class SoftmaxOp : public XlaOpKernel {
     const DataType type = input_type(0);
     auto logits = ctx->Input(0);
 
-    xla::ComputationBuilder* b = ctx->builder();
+    xla::ComputationBuilder* const b = ctx->builder();
     const xla::Computation& max_func = *ctx->GetOrCreateMax(type);
-    const xla::Computation& add_func = *ctx->GetOrCreateAdd(type);
 
     // Find the max in each batch, resulting in a tensor of shape [batch]
     auto logits_max =
@@ -52,21 +51,20 @@ class SoftmaxOp : public XlaOpKernel {
     // Subtract the max in batch b from every element in batch b. Broadcasts
     // along the batch dimension.
     auto shifted_logits = b->Sub(logits, logits_max, {kBatchDim});
-    xla::ComputationDataHandle softmax;
-    if (log_) {
-      // softmax = shifted_logits - log(sum(exp(shifted_logits)))
-      auto log_sum_exp =
-          b->Log(b->Reduce(b->Exp(shifted_logits), XlaHelpers::Zero(b, type),
-                           add_func, {kClassDim}));
-      softmax = b->Sub(shifted_logits, log_sum_exp, {kBatchDim});
-    } else {
-      // softmax = exp(shifted_logits) / sum(exp(shifted_logits))
-      auto exp_shifted = b->Exp(shifted_logits);
-      auto sum_exp = b->Reduce(exp_shifted, XlaHelpers::Zero(b, type), add_func,
-                               {kClassDim});
-      softmax = b->Div(exp_shifted, sum_exp, {kBatchDim});
-    }
-
+    auto exp_shifted = b->Exp(shifted_logits);
+    const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
+    auto converted =
+        XlaHelpers::ConvertElementType(b, exp_shifted, accumulation_type);
+    auto reduce =
+        b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                  *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
+    auto sum = XlaHelpers::ConvertElementType(b, reduce, type);
+    auto softmax =
+        log_
+            // softmax = shifted_logits - log(sum(exp(shifted_logits)))
+            ? b->Sub(shifted_logits, b->Log(sum), {kBatchDim})
+            // softmax = exp(shifted_logits) / sum(exp(shifted_logits))
+            : b->Div(exp_shifted, sum, {kBatchDim});
     ctx->SetOutput(0, softmax);
   }
 
@@ -82,7 +80,6 @@ CrossEntropyWithLogits(XlaOpKernelContext* ctx, DataType type,
                        const xla::ComputationDataHandle& logits,
                        const xla::ComputationDataHandle& labels) {
   const xla::Computation& max_func = *ctx->GetOrCreateMax(type);
-  const xla::Computation& add_func = *ctx->GetOrCreateAdd(type);
 
   const int kBatchDim = 0;
   const int kClassDim = 1;
@@ -100,8 +97,12 @@ CrossEntropyWithLogits(XlaOpKernelContext* ctx, DataType type,
   auto exp_shifted_logits = b->Exp(shifted_logits);
 
   // sum_{class} (exp(logits - max_logits))
-  auto sum_exp = b->Reduce(exp_shifted_logits, XlaHelpers::Zero(b, type),
-                           add_func, {kClassDim});
+  const DataType accumulation_type = XlaHelpers::SumAccumulationType(type);
+  auto converted =
+      XlaHelpers::ConvertElementType(b, exp_shifted_logits, accumulation_type);
+  auto reduce = b->Reduce(converted, XlaHelpers::Zero(b, accumulation_type),
+                          *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
+  auto sum_exp = XlaHelpers::ConvertElementType(b, reduce, type);
 
   // log(sum(exp(logits - max_logits)))
   auto log_sum_exp = b->Log(sum_exp);
@@ -110,9 +111,13 @@ CrossEntropyWithLogits(XlaOpKernelContext* ctx, DataType type,
   //    ((logits - max_logits) - log(sum(exp(logits - max_logits)))))
   // along classes
   // (The subtraction broadcasts along the batch dimension.)
-  xla::ComputationDataHandle loss = b->Reduce(
-      b->Mul(b->Neg(labels), b->Sub(shifted_logits, log_sum_exp, {kBatchDim})),
-      XlaHelpers::Zero(b, type), add_func, {kClassDim});
+  auto sub = b->Sub(shifted_logits, log_sum_exp, {kBatchDim});
+  auto mul = b->Mul(b->Neg(labels), sub);
+  auto sum =
+      b->Reduce(XlaHelpers::ConvertElementType(b, mul, accumulation_type),
+                XlaHelpers::Zero(b, accumulation_type),
+                *ctx->GetOrCreateAdd(accumulation_type), {kClassDim});
+  auto loss = XlaHelpers::ConvertElementType(b, sum, type);
 
   // backprop: prob - labels, where
   //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.cc b/tensorflow/compiler/tf2xla/xla_helpers.cc
index f048662953..3b0b2f06eb 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.cc
+++ b/tensorflow/compiler/tf2xla/xla_helpers.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/gtl/array_slice.h"
 
 namespace tensorflow {
@@ -273,4 +274,20 @@ Status XlaHelpers::OneHot(xla::ComputationBuilder* builder, int64 depth,
   return Status::OK();
 }
 
+DataType XlaHelpers::SumAccumulationType(const DataType& dtype) {
+  if (dtype == DT_BFLOAT16) {
+    return DT_FLOAT;
+  }
+  return dtype;
+}
+
+xla::ComputationDataHandle XlaHelpers::ConvertElementType(
+    xla::ComputationBuilder* const builder,
+    const xla::ComputationDataHandle& operand,
+    const DataType new_element_type) {
+  xla::PrimitiveType convert_to;
+  TF_CHECK_OK(DataTypeToPrimitiveType(new_element_type, &convert_to));
+  return builder->ConvertElementType(operand, convert_to);
+}
+
 }  // end namespace tensorflow
diff --git a/tensorflow/compiler/tf2xla/xla_helpers.h b/tensorflow/compiler/tf2xla/xla_helpers.h
index 2a027db4c8..68ab93b64a 100644
--- a/tensorflow/compiler/tf2xla/xla_helpers.h
+++ b/tensorflow/compiler/tf2xla/xla_helpers.h
@@ -107,6 +107,18 @@ class XlaHelpers {
                        const xla::ComputationDataHandle& on_value,
                        const xla::ComputationDataHandle& off_value,
                        xla::ComputationDataHandle* one_hot);
+
+  // Certain DataTypes should use increased precision DataTypes when performing
+  // reductions.  This function remaps a given DataType to a higher precision
+  // DataType if needed.
+  static DataType SumAccumulationType(const DataType& dtype);
+
+  // A helper for creating a ConvertElementType xla op given a DataType rather
+  // than the xla::PrimitiveType.
+  static xla::ComputationDataHandle ConvertElementType(
+      xla::ComputationBuilder* const builder,
+      const xla::ComputationDataHandle& operand,
+      const DataType new_element_type);
 };
 
 }  // end namespace tensorflow
diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 20508edaa7..214c2030cd 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -1463,6 +1463,9 @@ StatusOr<std::unique_ptr<Literal>> ConvertIfDestTypeMatches(
 StatusOr<std::unique_ptr<Literal>> Literal::Convert(
     PrimitiveType primitive_dest_type) const {
   TF_RET_CHECK(ShapeUtil::IsArray(shape()));
+  if (shape().element_type() == primitive_dest_type) {
+    return CloneToUnique();
+  }
   switch (shape().element_type()) {
 #define CONVERT_IF_DEST_TYPE_MATCHES(type) \
   case (type):                             \
diff --git a/tensorflow/compiler/xla/tests/convert_test.cc b/tensorflow/compiler/xla/tests/convert_test.cc
index 7926767a4f..9a899b7914 100644
--- a/tensorflow/compiler/xla/tests/convert_test.cc
+++ b/tensorflow/compiler/xla/tests/convert_test.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 #include "tensorflow/core/lib/core/casts.h"
+#include "tensorflow/core/lib/math/math_util.h"
 #include "tensorflow/core/platform/stream_executor_no_cuda.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
@@ -384,5 +385,44 @@ XLA_TEST_F(ConvertTest, ConvertR1F32ToR1F16) {
 
   ComputeAndCompareR1<half>(&builder, expected_output, {dot_lhs_handle.get()});
 }
+
+XLA_TEST_F(ConvertTest, ConvertC64ToC64) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<complex64> x = {{42.0f, 64.0f}};
+  builder.ConvertElementType(builder.ConstantR1<complex64>(x), C64);
+  ComputeAndCompareR1<complex64>(&builder, x, {}, ErrorSpec(0.0001));
+}
+
+XLA_TEST_F(ConvertTest, ConvertS64S64) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<int64> x = {{-42, 64}};
+  builder.ConvertElementType(builder.ConstantR1<int64>(x), S64);
+  ComputeAndCompareR1<int64>(&builder, x, {});
+}
+
+XLA_TEST_F(ConvertTest, ConvertU64U64) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<uint64> x = {{42, 64}};
+  builder.ConvertElementType(builder.ConstantR1<uint64>(x), U64);
+  ComputeAndCompareR1<uint64>(&builder, x, {});
+}
+
+XLA_TEST_F(ConvertTest, ConvertU64S64) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<uint64> unsigned_x = {{42, UINT64_MAX}};
+  builder.ConvertElementType(builder.ConstantR1<uint64>(unsigned_x), S64);
+  std::vector<int64> signed_x = {{42, -1}};
+  ComputeAndCompareR1<int64>(&builder, signed_x, {});
+}
+
+XLA_TEST_F(ConvertTest, ConvertS64U64) {
+  ComputationBuilder builder(client_, TestName());
+  std::vector<int64> signed_x = {{42, -1, INT64_MIN}};
+  builder.ConvertElementType(builder.ConstantR1<int64>(signed_x), U64);
+  std::vector<uint64> unsigned_x = {
+      {42, UINT64_MAX, tensorflow::MathUtil::IPow<uint64>(2, 63)}};
+  ComputeAndCompareR1<uint64>(&builder, unsigned_x, {});
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 942a32bc71291994c14625b6311268319dd27808 Mon Sep 17 00:00:00 2001
From: James Qin <jamesqin@google.com>
Date: Wed, 21 Mar 2018 15:55:30 -0700
Subject: [PATCH 326/960] Change Softmax on CUDA to use fp32 for denominator
 when input/output are fp16.

This avoids potential overflow in the denominator, also makes sure accumulation is done
in high precision.

PiperOrigin-RevId: 189982655
---
 tensorflow/core/kernels/softmax_op_gpu.cu.cc  | 90 ++++++++++++++-----
 tensorflow/python/framework/test_util.py      | 75 +++++++++-------
 tensorflow/python/kernel_tests/BUILD          |  2 +-
 .../python/kernel_tests/softmax_op_test.py    | 38 ++++++--
 4 files changed, 145 insertions(+), 60 deletions(-)

diff --git a/tensorflow/core/kernels/softmax_op_gpu.cu.cc b/tensorflow/core/kernels/softmax_op_gpu.cu.cc
index 1f4a82a733..130d693dbd 100644
--- a/tensorflow/core/kernels/softmax_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/softmax_op_gpu.cu.cc
@@ -33,8 +33,42 @@ namespace tensorflow {
 
 namespace {
 
+template <typename U, typename T>
+__device__ __host__ EIGEN_STRONG_INLINE
+    typename std::enable_if<!std::is_same<T, U>::value, U>::type
+    strict_cast(T t);
+
+template <typename U, typename T>
+__device__ __host__ EIGEN_STRONG_INLINE
+    typename std::enable_if<std::is_same<T, U>::value, U>::type
+    strict_cast(T t) {
+  return t;
+}
+
+template <>
+__device__ __host__ EIGEN_STRONG_INLINE float strict_cast<float, Eigen::half>(
+    Eigen::half t) {
+  return functor::HalfToFloat()(t);
+}
+
+template <>
+__device__ __host__ EIGEN_STRONG_INLINE Eigen::half
+strict_cast<Eigen::half, float>(float t) {
+  return functor::FloatToHalf()(t);
+}
+
 template <typename T>
-__global__ void GenerateNormalizedProb(const T* logits, const T* sum_probs,
+struct softmax_traits {
+  using accumulator_type = T;
+};
+
+template <>
+struct softmax_traits<Eigen::half> {
+  using accumulator_type = float;
+};
+
+template <typename T, typename U>
+__global__ void GenerateNormalizedProb(const T* logits, const U* sum_probs,
                                        const T* max_logits, T* output,
                                        const int num_rows, const int num_cols,
                                        const bool in_log_space) {
@@ -43,25 +77,33 @@ __global__ void GenerateNormalizedProb(const T* logits, const T* sum_probs,
   const int row = tid / num_cols;
   const int col = tid % num_cols;
 
+  // TODO(jamesqin): change to half2 load when inputs are Eigen::half.
+  U input = strict_cast<U>(logits[tid]);
+  U max_val = strict_cast<U>(ldg(max_logits + row));
+  U result;
+
   if (row < num_rows && col < num_cols) {
-    if (in_log_space)
-      output[tid] =
-          logits[tid] - ldg(max_logits + row) - log(ldg(sum_probs + row));
-    else
-      output[tid] =
-          exp(logits[tid] - ldg(max_logits + row)) / ldg(sum_probs + row);
+    if (in_log_space) {
+      result = input - max_val - log(ldg(sum_probs + row));
+    } else {
+      result = exp(input - max_val) / ldg(sum_probs + row);
+    }
+    output[tid] = strict_cast<T>(result);
   }
 }
 
-template <typename T>
+template <typename T, typename U>
 struct SubtractAndExpFunctor {
   __host__ __device__ SubtractAndExpFunctor(const T* logits,
                                             const T* max_logits,
                                             const int num_cols)
       : logits_(logits), max_logits_(max_logits), num_cols_(num_cols) {}
 
-  __host__ __device__ T operator()(const int gid) const {
-    return exp(logits_[gid] - ldg(max_logits_ + gid / num_cols_));
+  __host__ __device__ U operator()(const int gid) const {
+    // TODO(jamesqin): change to half2 load when inputs are Eigen::half.
+    const U diff =
+        strict_cast<U>(logits_[gid] - ldg(max_logits_ + gid / num_cols_));
+    return exp(diff);
   }
 
   const T* logits_;
@@ -80,7 +122,6 @@ void DoRowReduction(OpKernelContext* context, T* output, InputIter input,
   functor::ReduceImpl<T, Op, T*, InputIter, ReductionAxes>(
       context, output, input, 2, rows, cols, 1, 1, constants.kOne, op);
 }
-
 }  // namespace
 
 template <typename T>
@@ -108,8 +149,10 @@ class SoftmaxOpGPU : public OpKernel {
       OP_REQUIRES_OK(context,
                      context->allocate_temp(DataTypeToEnum<T>::value,
                                             softmax_out->shape(), &max_logits));
+
+      typedef typename softmax_traits<T>::accumulator_type acc_type;
       OP_REQUIRES_OK(context,
-                     context->allocate_temp(DataTypeToEnum<T>::value,
+                     context->allocate_temp(DataTypeToEnum<acc_type>::value,
                                             softmax_out->shape(), &sum_probs));
 
       DoRowReduction<T, cub::Max, const T*>(
@@ -120,25 +163,28 @@ class SoftmaxOpGPU : public OpKernel {
       const int numBlocks = Eigen::divup(rows * cols, numThreads);
 
       cub::CountingInputIterator<int> counting_iterator(0);
-      typedef cub::TransformInputIterator<T, SubtractAndExpFunctor<T>,
+      typedef cub::TransformInputIterator<acc_type,
+                                          SubtractAndExpFunctor<T, acc_type>,
                                           cub::CountingInputIterator<int>>
           InputIterType;
 
       InputIterType input_itr(
           counting_iterator,
-          SubtractAndExpFunctor<T>(
+          SubtractAndExpFunctor<T, acc_type>(
               reinterpret_cast<const T*>(logits_in_.flat<T>().data()),
               reinterpret_cast<const T*>(max_logits.flat<T>().data()), cols));
 
-      DoRowReduction<T, cub::Sum, InputIterType>(
-          context, const_cast<T*>(sum_probs.flat<T>().data()), input_itr, rows,
-          cols);
+      DoRowReduction<acc_type, cub::Sum, InputIterType>(
+          context, const_cast<acc_type*>(sum_probs.flat<acc_type>().data()),
+          input_itr, rows, cols);
 
-      GenerateNormalizedProb<<<numBlocks, numThreads, 0, cu_stream>>>(
-          reinterpret_cast<const T*>(logits_in_.flat<T>().data()),
-          reinterpret_cast<const T*>(sum_probs.flat<T>().data()),
-          reinterpret_cast<const T*>(max_logits.flat<T>().data()),
-          const_cast<T*>(softmax_out->flat<T>().data()), rows, cols, log_);
+      GenerateNormalizedProb<T, acc_type>
+          <<<numBlocks, numThreads, 0, cu_stream>>>(
+              reinterpret_cast<const T*>(logits_in_.flat<T>().data()),
+              reinterpret_cast<const acc_type*>(
+                  sum_probs.flat<acc_type>().data()),
+              reinterpret_cast<const T*>(max_logits.flat<T>().data()),
+              const_cast<T*>(softmax_out->flat<T>().data()), rows, cols, log_);
     }
   }
 
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index d8f8569939..43106b6e59 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -53,6 +53,7 @@ from tensorflow.python.eager import tape  # pylint: disable=unused-import
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import importer
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
@@ -201,6 +202,7 @@ def _strip_checkpoint_v2_randomized(graph_def):
 def IsGoogleCudaEnabled():
   return pywrap_tensorflow.IsGoogleCudaEnabled()
 
+
 def CudaSupportsHalfMatMulAndConv():
   return pywrap_tensorflow.CudaSupportsHalfMatMulAndConv()
 
@@ -335,6 +337,8 @@ def _use_c_api_wrapper(fn, use_c_api, *args, **kwargs):
     # Make sure default graph reflects prev_value in case next test doesn't call
     # reset_default_graph().
     ops.reset_default_graph()
+
+
 # pylint: disable=protected-access
 
 
@@ -451,7 +455,8 @@ def with_c_api(cls):
   # If the C API is already enabled, don't do anything. Some tests break if the
   # same test is run twice, so this allows us to turn on the C API by default
   # without breaking these tests.
-  if ops._USE_C_API: return cls
+  if ops._USE_C_API:
+    return cls
 
   for name, value in cls.__dict__.copy().items():
     if callable(value) and name.startswith("test"):
@@ -469,6 +474,7 @@ def assert_no_new_pyobjects_executing_eagerly(f):
   Useful for checking that there are no missing Py_DECREFs in the C exercised by
   a bit of Python.
   """
+
   def decorator(self, **kwargs):
     """Warms up, gets an object count, runs the test, checks for new objects."""
     with context.eager_mode():
@@ -483,8 +489,10 @@ def assert_no_new_pyobjects_executing_eagerly(f):
       new_count = len(gc.get_objects())
       self.assertEqual(previous_count, new_count)
       gc.enable()
+
   return decorator
 
+
 def assert_no_new_tensors(f):
   """Decorator for asserting that no new Tensors persist after a test.
 
@@ -508,17 +516,15 @@ def assert_no_new_tensors(f):
 
     def _is_tensorflow_object(obj):
       try:
-        return isinstance(obj, (
-            ops.Tensor,
-            variables.Variable,
-            tensor_shape.Dimension,
-            tensor_shape.TensorShape))
+        return isinstance(obj,
+                          (ops.Tensor, variables.Variable,
+                           tensor_shape.Dimension, tensor_shape.TensorShape))
       except ReferenceError:
         # If the object no longer exists, we don't care about it.
         return False
 
-    tensors_before = set(id(obj) for obj in gc.get_objects()
-                         if _is_tensorflow_object(obj))
+    tensors_before = set(
+        id(obj) for obj in gc.get_objects() if _is_tensorflow_object(obj))
     outside_graph_key = ops.get_default_graph()._graph_key
     with ops.Graph().as_default():
       # Run the test in a new graph so that collections get cleared when it's
@@ -572,18 +578,18 @@ def assert_no_garbage_created(f):
           "likely due to a reference cycle. New objects in cycle(s):")
       for i, obj in enumerate(gc.garbage[previous_garbage:]):
         try:
-          logging.error(
-              "Object %d of %d" % (i, len(gc.garbage) - previous_garbage))
+          logging.error("Object %d of %d", i,
+                        len(gc.garbage) - previous_garbage)
+
           def _safe_object_str(obj):
             return "<%s %d>" % (obj.__class__.__name__, id(obj))
-          logging.error("  Object type: %s" % (_safe_object_str(obj),))
-          logging.error("  Referrer types: %s" % (
-              ', '.join([_safe_object_str(ref)
-                         for ref in gc.get_referrers(obj)]),))
-          logging.error("  Referent types: %s" % (
-              ', '.join([_safe_object_str(ref)
-                         for ref in gc.get_referents(obj)]),))
-          logging.error("  Object attribute names: %s" % (dir(obj),))
+
+          logging.error("  Object type: %s", _safe_object_str(obj))
+          logging.error("  Referrer types: %s", ", ".join(
+              [_safe_object_str(ref) for ref in gc.get_referrers(obj)]))
+          logging.error("  Referent types: %s", ", ".join(
+              [_safe_object_str(ref) for ref in gc.get_referents(obj)]))
+          logging.error("  Object attribute names: %s", dir(obj))
           logging.error("  Object __str__:")
           logging.error(obj)
           logging.error("  Object __repr__:")
@@ -705,15 +711,23 @@ def is_gpu_available(cuda_only=False, min_cuda_compute_capability=None):
       return 0, 0
     return int(match.group(1)), int(match.group(2))
 
-  for local_device in device_lib.list_local_devices():
-    if local_device.device_type == "GPU":
-      if (min_cuda_compute_capability is None or
-          compute_capability_from_device_desc(local_device.physical_device_desc)
-          >= min_cuda_compute_capability):
+  try:
+    for local_device in device_lib.list_local_devices():
+      if local_device.device_type == "GPU":
+        if (min_cuda_compute_capability is None or
+            compute_capability_from_device_desc(
+                local_device.physical_device_desc) >=
+            min_cuda_compute_capability):
+          return True
+      if local_device.device_type == "SYCL" and not cuda_only:
         return True
-    if local_device.device_type == "SYCL" and not cuda_only:
-      return True
-  return False
+    return False
+  except errors_impl.NotFoundError as e:
+    if not all([x in str(e) for x in ["CUDA", "not find"]]):
+      raise e
+    else:
+      logging.error(str(e))
+      return False
 
 
 @contextlib.contextmanager
@@ -1256,9 +1270,9 @@ class TensorFlowTestCase(googletest.TestCase):
             msg="Mismatched value: a%s is different from b%s." % (path_str,
                                                                   path_str))
       except TypeError as e:
-        msg = "Error: a%s has %s, but b%s has %s" % (
-            path_str, type(a), path_str, type(b))
-        e.args = ((e.args[0] + ' : ' + msg,) + e.args[1:])
+        msg = "Error: a%s has %s, but b%s has %s" % (path_str, type(a),
+                                                     path_str, type(b))
+        e.args = ((e.args[0] + " : " + msg,) + e.args[1:])
         raise
 
   def assertAllClose(self, a, b, rtol=1e-6, atol=1e-6, msg=None):
@@ -1438,8 +1452,7 @@ class TensorFlowTestCase(googletest.TestCase):
     """
     device1 = pydev.canonical_name(device1)
     device2 = pydev.canonical_name(device2)
-    self.assertEqual(device1, device2,
-                     "Devices %s and %s are not equal. %s" % 
+    self.assertEqual(device1, device2, "Devices %s and %s are not equal. %s" %
                      (device1, device2, msg))
 
   # Fix Python 3 compatibility issues
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d9571fa2be..ece1da0332 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1910,7 +1910,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "softmax_op_test",
-    size = "small",
+    size = "medium",
     srcs = ["softmax_op_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
diff --git a/tensorflow/python/kernel_tests/softmax_op_test.py b/tensorflow/python/kernel_tests/softmax_op_test.py
index 2b8e99e18e..981f96b74d 100644
--- a/tensorflow/python/kernel_tests/softmax_op_test.py
+++ b/tensorflow/python/kernel_tests/softmax_op_test.py
@@ -18,14 +18,17 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import unittest
 import numpy as np
 
+
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.platform import test
+from tensorflow.python.platform import tf_logging as logging
 
 
 @test_util.with_c_api
@@ -41,9 +44,10 @@ class SoftmaxTest(test.TestCase):
             features, axis=dim), one_only_on_dim))
     softmax = e / np.reshape(np.sum(e, axis=dim), one_only_on_dim)
     if log:
-      return np.log(softmax)
+      res = np.log(softmax)
     else:
-      return softmax
+      res = softmax
+    return res
 
   def _testSoftmax(self, np_features, dim=-1, log=False, use_gpu=False):
     # A previous version of the code checked the op name rather than the op type
@@ -53,9 +57,9 @@ class SoftmaxTest(test.TestCase):
     np_softmax = self._npSoftmax(np_features, dim=dim, log=log)
     with self.test_session(use_gpu=use_gpu):
       if log:
-        tf_softmax = nn_ops.log_softmax(np_features, dim=dim, name=name)
+        tf_softmax = nn_ops.log_softmax(np_features, axis=dim, name=name)
       else:
-        tf_softmax = nn_ops.softmax(np_features, dim=dim, name=name)
+        tf_softmax = nn_ops.softmax(np_features, axis=dim, name=name)
       out = tf_softmax.eval()
     self.assertAllCloseAccordingToType(np_softmax, out)
     self.assertShapeEqual(np_softmax, tf_softmax)
@@ -117,10 +121,32 @@ class SoftmaxTest(test.TestCase):
     self._testAll(
         np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float32))
 
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testFloatGPU(self):
+    if test.is_gpu_available(cuda_only=True):
+      rows = [2**x + np.random.randint(0, 1024) for x in range(1, 10)]
+      cols = [2**x + np.random.randint(0, 1024) for x in range(1, 10)]
+      for row, col in zip(rows, cols):
+        logging.info("Testing softmax float dtype in shape [%d, %d]", row, col)
+        data = np.random.rand(row, col)
+        self._testAll(data.astype(np.float32))
+
   def testHalf(self):
     self._testAll(
         np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float16))
 
+  @unittest.skipUnless(test.is_built_with_cuda(),
+                       "Test only applicable when running on GPUs")
+  def testHalfGPU(self):
+    if test.is_gpu_available(cuda_only=True):
+      rows = [2**x + np.random.randint(0, 1024) for x in range(1, 8)]
+      cols = [2**x + np.random.randint(0, 1024) for x in range(1, 8)]
+      for row, col in zip(rows, cols):
+        logging.info("Testing softmax half dtype in shape [%d, %d]", row, col)
+        data = np.random.rand(row, col)
+        self._testAll(data.astype(np.float16))
+
   def testDouble(self):
     self._testSoftmax(
         np.array([[1., 1., 1., 1.], [1., 2., 3., 4.]]).astype(np.float64))
@@ -169,7 +195,7 @@ class SoftmaxTest(test.TestCase):
       self.assertEqual(0, array_ops.size(x).eval())
       # reshape would raise if logits is empty
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        nn_ops.softmax(x, dim=0).eval()
+        nn_ops.softmax(x, axis=0).eval()
 
   def testDimTooLarge(self):
     with self.test_session():
@@ -177,7 +203,7 @@ class SoftmaxTest(test.TestCase):
       # inference error.
       dim = array_ops.placeholder_with_default(100, shape=[])
       with self.assertRaises(errors_impl.InvalidArgumentError):
-        nn_ops.softmax([1., 2., 3., 4.], dim=dim).eval()
+        nn_ops.softmax([1., 2., 3., 4.], axis=dim).eval()
 
   def testLargeDims(self):
     # Make sure that we properly handle large inputs. See
-- 
GitLab


From 1f58c96b593c49a97bbfac0665c2628ef9c910cd Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Wed, 21 Mar 2018 16:00:44 -0700
Subject: [PATCH 327/960] Shorter definitions for elementwise_ops in
 op_level_cost_estimator.

PiperOrigin-RevId: 189983460
---
 .../grappler/costs/op_level_cost_estimator.cc | 180 +++++++-----------
 1 file changed, 67 insertions(+), 113 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index d3ffa03fe2..fdbc61f3f1 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -192,121 +192,75 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
       {kShape, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kSize, wrap(&OpLevelCostEstimator::PredictMetadata)}};
 
+#define EIGEN_COST(X) Eigen::internal::functor_traits<Eigen::internal::X>::Cost
+
   // Quantize = apply min and max bounds, multiply by scale factor and round.
   const int quantize_v2_cost =
-      Eigen::internal::functor_traits<
-          Eigen::internal::scalar_product_op<float>>::Cost +
-      Eigen::internal::functor_traits<
-          Eigen::internal::scalar_max_op<float>>::Cost +
-      Eigen::internal::functor_traits<
-          Eigen::internal::scalar_min_op<float>>::Cost +
-      Eigen::internal::functor_traits<
-          Eigen::internal::scalar_round_op<float>>::Cost;
-
-  elementwise_ops_ = {
-      // Unary ops alphabetically sorted
-      {"Acos", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_acos_op<float>>::Cost},
-      {"Asin", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_asin_op<float>>::Cost},
-      {"Atan", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_atan_op<float>>::Cost},
-      {"Atan2", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_quotient_op<float>>::Cost +
-                    Eigen::internal::functor_traits<
-                        Eigen::internal::scalar_atan_op<float>>::Cost},
-      {"Ceil", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_ceil_op<float>>::Cost},
-      {"Cos", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_cos_op<float>>::Cost},
-      {"Dequantize", Eigen::internal::functor_traits<
-                         Eigen::internal::scalar_product_op<float>>::Cost},
-      {"Erf", 1},
-      {"Erfc", 1},
-      {"Exp", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_exp_op<float>>::Cost},
-      {"Expm1", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_expm1_op<float>>::Cost},
-      {"Floor", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_floor_op<float>>::Cost},
-      {"Inv", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_inverse_op<float>>::Cost},
-      {"InvGrad", 1},
-      {"Lgamma", 1},
-      {"Log", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_log_op<float>>::Cost},
-      {"Log1p", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_log1p_op<float>>::Cost},
-      {"Neg", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_opposite_op<float>>::Cost},
-      {"QuantizeV2", quantize_v2_cost},
-      {"Reciprocal", Eigen::internal::functor_traits<
-                         Eigen::internal::scalar_inverse_op<float>>::Cost},
-      {"Rint", 1},
-      {"Round", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_round_op<float>>::Cost},
-      {"Rsqrt", Eigen::internal::functor_traits<
-                    Eigen::internal::scalar_rsqrt_op<float>>::Cost},
-      {"Sqrt", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_sqrt_op<float>>::Cost},
-      {"Square", Eigen::internal::functor_traits<
-                     Eigen::internal::scalar_square_op<float>>::Cost},
-      {"Tanh", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_tanh_op<float>>::Cost},
-      {"Relu", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_max_op<float>>::Cost},
-      {"Sigmoid", Eigen::internal::functor_traits<
-                      Eigen::internal::scalar_sigmoid_op<float>>::Cost},
-      {"Sign", Eigen::internal::functor_traits<
-                   Eigen::internal::scalar_sign_op<float>>::Cost},
-      {"Sin", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_sin_op<float>>::Cost},
-      {"Tan", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_tan_op<float>>::Cost},
-      // Binary ops alphabetically sorted
-      {"Add", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_sum_op<float>>::Cost},
-      {"ApproximateEqual", 1},
-      {"BiasAdd", Eigen::internal::functor_traits<
-                      Eigen::internal::scalar_sum_op<float>>::Cost},
-      {"Div", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_quotient_op<float>>::Cost},
-      {"Equal", 1},
-      {"FloorDiv", Eigen::internal::functor_traits<
-                       Eigen::internal::scalar_quotient_op<float>>::Cost},
-      {"FloorMod", Eigen::internal::functor_traits<
-                       Eigen::internal::scalar_mod_op<float>>::Cost},
-      {"Greater", 1},
-      {"GreaterEqual", 1},
-      {"Less", 1},
-      {"LessEqual", 1},
-      {"LogicalAnd", Eigen::internal::functor_traits<
-                         Eigen::internal::scalar_boolean_and_op>::Cost},
-      {"LogicalNot", 1},
-      {"LogicalOr", Eigen::internal::functor_traits<
-                        Eigen::internal::scalar_boolean_or_op>::Cost},
-      {"Maximum", Eigen::internal::functor_traits<
-                      Eigen::internal::scalar_max_op<float>>::Cost},
-      {"Minimum", Eigen::internal::functor_traits<
-                      Eigen::internal::scalar_min_op<float>>::Cost},
-      {"Mod", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_mod_op<float>>::Cost},
-      {"Mul", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_product_op<float>>::Cost},
-      {"NotEqual", 1},
-      {"QuantizedAdd", Eigen::internal::functor_traits<
-                           Eigen::internal::scalar_sum_op<float>>::Cost},
-      {"QuantizedMul", Eigen::internal::functor_traits<
-                           Eigen::internal::scalar_product_op<float>>::Cost},
-      {"RealDiv", Eigen::internal::functor_traits<
-                      Eigen::internal::scalar_quotient_op<float>>::Cost},
-      {"SquareDifference", 1},
-      {"Sub", Eigen::internal::functor_traits<
-                  Eigen::internal::scalar_difference_op<float>>::Cost},
-      {"TruncateDiv", Eigen::internal::functor_traits<
-                          Eigen::internal::scalar_quotient_op<float>>::Cost},
-      {"TruncateMod", Eigen::internal::functor_traits<
-                          Eigen::internal::scalar_mod_op<float>>::Cost}};
+      EIGEN_COST(scalar_product_op<float>) + EIGEN_COST(scalar_max_op<float>) +
+      EIGEN_COST(scalar_min_op<float>) + EIGEN_COST(scalar_round_op<float>);
+
+  elementwise_ops_ = {// Unary ops alphabetically sorted
+                      {"Acos", EIGEN_COST(scalar_acos_op<float>)},
+                      {"Asin", EIGEN_COST(scalar_asin_op<float>)},
+                      {"Atan", EIGEN_COST(scalar_atan_op<float>)},
+                      {"Atan2", EIGEN_COST(scalar_quotient_op<float>) +
+                                    EIGEN_COST(scalar_atan_op<float>)},
+                      {"Ceil", EIGEN_COST(scalar_ceil_op<float>)},
+                      {"Cos", EIGEN_COST(scalar_cos_op<float>)},
+                      {"Dequantize", EIGEN_COST(scalar_product_op<float>)},
+                      {"Erf", 1},
+                      {"Erfc", 1},
+                      {"Exp", EIGEN_COST(scalar_exp_op<float>)},
+                      {"Expm1", EIGEN_COST(scalar_expm1_op<float>)},
+                      {"Floor", EIGEN_COST(scalar_floor_op<float>)},
+                      {"Inv", EIGEN_COST(scalar_inverse_op<float>)},
+                      {"InvGrad", 1},
+                      {"Lgamma", 1},
+                      {"Log", EIGEN_COST(scalar_log_op<float>)},
+                      {"Log1p", EIGEN_COST(scalar_log1p_op<float>)},
+                      {"Neg", EIGEN_COST(scalar_opposite_op<float>)},
+                      {"QuantizeV2", quantize_v2_cost},
+                      {"Reciprocal", EIGEN_COST(scalar_inverse_op<float>)},
+                      {"Rint", 1},
+                      {"Round", EIGEN_COST(scalar_round_op<float>)},
+                      {"Rsqrt", EIGEN_COST(scalar_rsqrt_op<float>)},
+                      {"Sqrt", EIGEN_COST(scalar_sqrt_op<float>)},
+                      {"Square", EIGEN_COST(scalar_square_op<float>)},
+                      {"Tanh", EIGEN_COST(scalar_tanh_op<float>)},
+                      {"Relu", EIGEN_COST(scalar_max_op<float>)},
+                      {"Sigmoid", EIGEN_COST(scalar_sigmoid_op<float>)},
+                      {"Sign", EIGEN_COST(scalar_sign_op<float>)},
+                      {"Sin", EIGEN_COST(scalar_sin_op<float>)},
+                      {"Tan", EIGEN_COST(scalar_tan_op<float>)},
+                      // Binary ops alphabetically sorted
+                      {"Add", EIGEN_COST(scalar_sum_op<float>)},
+                      {"ApproximateEqual", 1},
+                      {"BiasAdd", EIGEN_COST(scalar_sum_op<float>)},
+                      {"Div", EIGEN_COST(scalar_quotient_op<float>)},
+                      {"Equal", 1},
+                      {"FloorDiv", EIGEN_COST(scalar_quotient_op<float>)},
+                      {"FloorMod", EIGEN_COST(scalar_mod_op<float>)},
+                      {"Greater", 1},
+                      {"GreaterEqual", 1},
+                      {"Less", 1},
+                      {"LessEqual", 1},
+                      {"LogicalAnd", EIGEN_COST(scalar_boolean_and_op)},
+                      {"LogicalNot", 1},
+                      {"LogicalOr", EIGEN_COST(scalar_boolean_or_op)},
+                      {"Maximum", EIGEN_COST(scalar_max_op<float>)},
+                      {"Minimum", EIGEN_COST(scalar_min_op<float>)},
+                      {"Mod", EIGEN_COST(scalar_mod_op<float>)},
+                      {"Mul", EIGEN_COST(scalar_product_op<float>)},
+                      {"NotEqual", 1},
+                      {"QuantizedAdd", EIGEN_COST(scalar_sum_op<float>)},
+                      {"QuantizedMul", EIGEN_COST(scalar_product_op<float>)},
+                      {"RealDiv", EIGEN_COST(scalar_quotient_op<float>)},
+                      {"SquareDifference", 1},
+                      {"Sub", EIGEN_COST(scalar_difference_op<float>)},
+                      {"TruncateDiv", EIGEN_COST(scalar_quotient_op<float>)},
+                      {"TruncateMod", EIGEN_COST(scalar_mod_op<float>)}};
+
+#undef EIGEN_COST
 
   // By default, use sum of memory_time and compute_time for execution_time.
   compute_memory_overlap_ = false;
-- 
GitLab


From 637b090ea0a5029805ba5e1dcf41c3b57d944ae4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 17:34:52 -0700
Subject: [PATCH 328/960] Small convenience changes.

PiperOrigin-RevId: 189996801
---
 tensorflow/contrib/py2tf/pyct/compiler.py    |  2 +-
 tensorflow/contrib/py2tf/pyct/transformer.py | 13 +++++++++++--
 2 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/compiler.py b/tensorflow/contrib/py2tf/pyct/compiler.py
index 507dbc7ed3..24c4517afa 100644
--- a/tensorflow/contrib/py2tf/pyct/compiler.py
+++ b/tensorflow/contrib/py2tf/pyct/compiler.py
@@ -31,7 +31,7 @@ import astor
 import gast
 
 
-def ast_to_source(node, indentation):
+def ast_to_source(node, indentation='  '):
   """Return the source code of given AST."""
   if isinstance(node, gast.AST):
     node = gast.gast_to_ast(node)
diff --git a/tensorflow/contrib/py2tf/pyct/transformer.py b/tensorflow/contrib/py2tf/pyct/transformer.py
index 57016bb4ce..31ef7e1c05 100644
--- a/tensorflow/contrib/py2tf/pyct/transformer.py
+++ b/tensorflow/contrib/py2tf/pyct/transformer.py
@@ -24,6 +24,7 @@ import gast
 import six
 
 from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.py2tf.pyct import compiler
 from tensorflow.contrib.py2tf.pyct import pretty_printer
 
 
@@ -31,6 +32,13 @@ class PyFlowParseError(SyntaxError):
   pass
 
 
+def try_ast_to_source(node):
+  try:
+    return compiler.ast_to_source(node)
+  except AssertionError:
+    return '<could not convert AST to source>'
+
+
 class Base(gast.NodeTransformer):
   """Base class for specialized transformers."""
 
@@ -62,8 +70,9 @@ class Base(gast.NodeTransformer):
       return super(Base, self).visit(node)
     except (ValueError, AttributeError, KeyError, NotImplementedError,
             AssertionError) as e:
-      msg = '%s: %s\nOccurred at node:\n%s' % (
-          e.__class__.__name__, str(e), pretty_printer.fmt(node, color=False))
+      msg = '%s: %s\nOffending source:\n%s\n\nOccurred at node:\n%s' % (
+          e.__class__.__name__, str(e), try_ast_to_source(node),
+          pretty_printer.fmt(node, color=False))
       if source_code:
         line = source_code.splitlines()[self._lineno - 1]
       else:
-- 
GitLab


From 8e4e9f7ceaa78f76b7f0aaa7a607e80e67f0d912 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Wed, 21 Mar 2018 17:38:04 -0700
Subject: [PATCH 329/960] Added an experimental C API to dump TF_Graph in a
 human-readable format, for debugging purposes.

PiperOrigin-RevId: 189997099
---
 tensorflow/c/c_api_experimental.cc | 10 ++++++++++
 tensorflow/c/c_api_experimental.h  |  6 ++++++
 2 files changed, 16 insertions(+)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index eb17e16d3e..34b9dec3ee 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -483,3 +483,13 @@ void TF_ShutdownTPUExecution(TF_Session* session, TF_Output shutdown_node,
                 /*targets*/ &shutdown_node.oper, /*ntargets*/ 1,
                 /*run_metadata*/ nullptr, status);
 }
+
+TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
+                                                      size_t* len) {
+  tensorflow::mutex_lock c(graph->mu);
+  const auto& debug_str = graph->graph.ToGraphDefDebug().DebugString();
+  *len = debug_str.size();
+  char* ret = static_cast<char*>(malloc(*len + 1));
+  memcpy(ret, debug_str.c_str(), *len + 1);
+  return ret;
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 2bad278d63..b95cdfe6aa 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -94,6 +94,12 @@ TF_CAPI_EXPORT extern void TF_ShutdownTPUExecution(TF_Session* session,
                                                    TF_Output shutdown_node,
                                                    TF_Status* status);
 
+// Returns the graph content in a human-readable format, with length set in
+// `len`. The format is subject to change in the future.
+// The returned string is heap-allocated, and caller should call free() on it.
+TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
+                                                      size_t* len);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
-- 
GitLab


From c7334fef9d1173525f6111b8ab50360b6531d76b Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Wed, 21 Mar 2018 18:02:01 -0700
Subject: [PATCH 330/960] [tf.data] Do not crash when combining
 .cache().take().repeat()

Currently, if the .cache() iterator is not fully consumed before
being repeated, it will cause an exception to be raised to Python.
Instead, cache should act as an identity transformation and log
an error, as this will not affect the correctness of the user's
program (at the cost of an unexpected performance cost: i.e. not
actually caching).

PiperOrigin-RevId: 189999552
---
 .../core/kernels/data/cache_dataset_ops.cc      | 17 ++++++++++++++++-
 .../data/kernel_tests/cache_dataset_op_test.py  | 15 +++++++++++++++
 2 files changed, 31 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/data/cache_dataset_ops.cc b/tensorflow/core/kernels/data/cache_dataset_ops.cc
index f0a2192826..4b4728dab6 100644
--- a/tensorflow/core/kernels/data/cache_dataset_ops.cc
+++ b/tensorflow/core/kernels/data/cache_dataset_ops.cc
@@ -308,6 +308,21 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
             input_impl_(params.dataset->input_->MakeIterator(params.prefix)),
             cache_(new std::vector<std::vector<Tensor>>) {}
 
+      ~MemoryWriterIterator() override {
+        mutex_lock l(mu_);
+        if (cache_) {
+          LOG(ERROR)
+              << "The calling iterator did not fully read the dataset we were "
+                 "attempting to cache. In order to avoid unexpected truncation "
+                 "of the sequence, the current [partially cached] sequence "
+                 "will be dropped. This can occur if you have a sequence "
+                 "similar to `dataset.cache().take(k).repeat()`. Instead, swap "
+                 "the order (i.e. `dataset.take(k).cache().repeat()`)";
+          mutex_lock l2(dataset()->mu_);
+          dataset()->writer_iterator_created_ = false;
+        }
+      }
+
       Status GetNextInternal(IteratorContext* ctx,
                              std::vector<Tensor>* out_tensors,
                              bool* end_of_sequence) override {
@@ -318,7 +333,7 @@ class CacheDatasetOp : public UnaryDatasetOpKernel {
           // Guard on cache_ to not crash if GetNext is called a second time
           // after *end_of_sequence == true
           if (cache_) {
-            mutex_lock l2(dataset()->mu_);
+            mutex_lock l(dataset()->mu_);
             DCHECK(dataset()->writer_iterator_created_);
             DCHECK(!dataset()->cache_);
             cache_.swap(dataset()->cache_);
diff --git a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
index 02720a2e98..25269dc810 100644
--- a/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
+++ b/tensorflow/python/data/kernel_tests/cache_dataset_op_test.py
@@ -297,6 +297,21 @@ class MemoryCacheDatasetTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(i2.get_next())
 
+  def testCacheTakeRepeat(self):
+    dataset = dataset_ops.Dataset.range(10).cache().take(5).repeat(2)
+    itr = dataset.make_one_shot_iterator()
+    n = itr.get_next()
+
+    expected_values = [0, 1, 2, 3, 4, 0, 1, 2, 3, 4]
+
+    with self.test_session() as sess:
+      for i, expected in enumerate(expected_values):
+        self.assertEqual(expected, sess.run(n),
+                         "Unexpected value at index %s" % i)
+
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(itr.get_next())
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 7407a35bd5237069b267bb82d08888c97675ab37 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Thu, 22 Mar 2018 09:22:42 +0800
Subject: [PATCH 331/960] Fix the variable typo in the python api example
 (#17900)

---
 tensorflow/contrib/lite/toco/g3doc/python_api.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/g3doc/python_api.md b/tensorflow/contrib/lite/toco/g3doc/python_api.md
index 440f9c367c..36e2d9c372 100644
--- a/tensorflow/contrib/lite/toco/g3doc/python_api.md
+++ b/tensorflow/contrib/lite/toco/g3doc/python_api.md
@@ -28,7 +28,7 @@ val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
 out = tf.identity(val, name="out")
 with tf.Session() as sess:
   tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
-  open("test.tflite", "wb").write(tflite_modeL)
+  open("test.tflite", "wb").write(tflite_model)
 ```
 
 **NOTE** Currently, the TOCO command will cause a fatal error to the Python
-- 
GitLab


From 61aa925ebaa69b9526cc67384fcde3fa42c9e6f1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 21 Mar 2018 18:22:36 -0700
Subject: [PATCH 332/960] Moves TFE_Executor to common_runtime

PiperOrigin-RevId: 190001737
---
 tensorflow/c/eager/BUILD                      |   2 +
 tensorflow/c/eager/c_api.cc                   | 179 ++++++----------
 tensorflow/c/eager/c_api_internal.h           |  84 +-------
 tensorflow/core/common_runtime/eager/BUILD    |  22 ++
 .../core/common_runtime/eager/context.cc      | 142 +++++++++++++
 .../core/common_runtime/eager/context.h       | 193 ++++++++++++++++++
 6 files changed, 428 insertions(+), 194 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/context.cc
 create mode 100644 tensorflow/core/common_runtime/eager/context.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 841ff48a38..bea5a121b3 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -28,6 +28,7 @@ tf_cuda_library(
             "//tensorflow/c:c_api",
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
+            "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core:core_cpu_internal",
@@ -64,6 +65,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
     ],
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index a23015c99e..5d668848ab 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -71,18 +71,6 @@ std::atomic_int_fast64_t func_id_generator(0);
 
 }  // namespace
 
-TFE_ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy) {
-  if (!soft_placement) {
-    return original_policy;
-  }
-  if (original_policy == TFE_DEVICE_PLACEMENT_EXPLICIT ||
-      original_policy == TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32) {
-    return TFE_DEVICE_PLACEMENT_SILENT;
-  }
-  return original_policy;
-}
-
 extern "C" {
 
 TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; }
@@ -104,19 +92,7 @@ void TFE_ContextOptionsSetDevicePlacementPolicy(
 TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
                                                         unsigned char async,
                                                         TF_Status* status) {
-  {
-    tensorflow::mutex_lock l(ctx->async_map_mu);
-    ctx->thread_local_async[std::this_thread::get_id()] = async;
-  }
-  if (async) {
-    ctx->executor.EnableAsync();
-  } else {
-    // TODO(agarwal): Currently we add a wait here to handle cases where a sync
-    // op has a control dependency on an async op, and the latter has not
-    // executed yet. This wait can be removed by storing all the control inputs
-    // and waiting for them when executing ops.
-    status->status = ctx->executor.WaitForAllPendingNodes();
-  }
+  status->status = ctx->context.SetAsyncForThread(async);
 }
 
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
@@ -133,34 +109,26 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
       new tensorflow::DeviceMgr(devices));
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
-  return new TFE_Context(*opts, std::move(device_mgr), r);
+  return new TFE_Context(opts->session_options.options, opts->policy,
+                         opts->async, std::move(device_mgr), r);
 }
 
 void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.WaitForAllPendingNodes();
-  {
-    tensorflow::mutex_lock ml(ctx->cache_mu);
-    tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
-  }
-  ctx->rendezvous->Unref();
   delete ctx;
 }
 
 TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx, TF_Status* status) {
   TF_DeviceList* list = new TF_DeviceList;
-  ctx->device_manager->ListDeviceAttributes(&list->response);
+  ctx->context.device_mgr()->ListDeviceAttributes(&list->response);
   return list;
 }
 
-void TFE_ContextClearCaches(TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->cache_mu);
-  tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
-}
+void TFE_ContextClearCaches(TFE_Context* ctx) { ctx->context.ClearCaches(); }
 
 void TFE_ContextSetThreadLocalDevicePlacementPolicy(
     TFE_Context* ctx, TFE_ContextDevicePlacementPolicy policy) {
-  tensorflow::mutex_lock ml(ctx->policy_map_mu);
-  ctx->thread_local_policies[std::this_thread::get_id()] = policy;
+  ctx->context.SetThreadLocalDevicePlacementPolicy(
+      static_cast<tensorflow::ContextDevicePlacementPolicy>(policy));
 }
 
 // Note: this function looks up a thread local policy. So it should be called in
@@ -168,25 +136,20 @@ void TFE_ContextSetThreadLocalDevicePlacementPolicy(
 // safe to call this function from the async EagerExecutor threads.
 extern TFE_ContextDevicePlacementPolicy TFE_ContextGetDevicePlacementPolicy(
     TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->policy_map_mu);
-  auto policy_map_it =
-      ctx->thread_local_policies.find(std::this_thread::get_id());
-  if (policy_map_it != ctx->thread_local_policies.end()) {
-    return policy_map_it->second;
-  }
-  return ctx->policy;
+  return static_cast<TFE_ContextDevicePlacementPolicy>(
+      ctx->context.GetDevicePlacementPolicy());
 }
 
 void TFE_ContextAsyncWait(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.WaitForAllPendingNodes();
+  status->status = ctx->context.AsyncWait();
 }
 
 void TFE_ContextGetStatus(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
 }
 
 void TFE_ContextAsyncClearError(TFE_Context* ctx) {
-  ctx->executor.ClearError();
+  ctx->context.ClearAsyncError();
 }
 
 TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
@@ -259,7 +222,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   // nullptr.
   tensorflow::Device* src_opd = nullptr;
   TF_RETURN_IF_ERROR(h->TensorAndDevice(&src, &srcd, &src_opd));
-  if (srcd == nullptr) srcd = ctx->devices[0];
+  if (srcd == nullptr) srcd = ctx->context.HostCPU();
   bool is_same_device =
       (srcd == dstd) || (DeviceName(srcd) == DeviceName(dstd));
   const bool dst_cpu = IsCPU(dstd);
@@ -332,8 +295,7 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name,
   status->status = tensorflow::AttrTypeMapForOp(name, &types);
   if (status->status.ok()) return new TFE_Op(ctx, name, types);
   if (TF_GetCode(status) == TF_NOT_FOUND) {
-    tensorflow::mutex_lock l(ctx->functions_mu);
-    if (ctx->func_lib_def.Find(name) != nullptr) {
+    if (ctx->context.FindFunctionByName(name)) {
       status->status = tensorflow::Status::OK();
       return new TFE_Op(ctx, name, nullptr);
     }
@@ -346,20 +308,14 @@ void TFE_DeleteOp(TFE_Op* op) { delete op; }
 void TFE_OpSetDevice(TFE_Op* op, const char* device_name, TF_Status* status) {
   tensorflow::Device* d = nullptr;
   if (device_name != nullptr && strlen(device_name) > 0) {
-    auto it = op->ctx->devices_map.find(device_name);
-    if (it == op->ctx->devices_map.end()) {
-      status->status =
-          tensorflow::errors::InvalidArgument(device_name, " unknown device.");
-      return;
-    }
-    d = it->second;
+    status->status = op->ctx->context.FindDeviceByName(device_name, &d);
   }
   op->device = d;
 }
 
 const char* TFE_OpGetDevice(TFE_Op* op, TF_Status* status) {
   tensorflow::Device* device =
-      (op->device == nullptr) ? op->ctx->devices[0] : op->device;
+      (op->device == nullptr) ? op->ctx->context.HostCPU() : op->device;
   return device->name().c_str();
 }
 
@@ -634,7 +590,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
 tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
                                  TFE_Context* ctx, TF_Status* status) {
   tensorflow::DeviceSet ds;
-  for (tensorflow::Device* d : ctx->devices) {
+  for (tensorflow::Device* d : *ctx->context.devices()) {
     ds.AddDevice(d);
   }
   tensorflow::DeviceTypeVector final_devices;
@@ -648,7 +604,7 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
         "Could not find valid device for node ", ndef.DebugString());
     return nullptr;
   }
-  for (tensorflow::Device* d : ctx->devices) {
+  for (tensorflow::Device* d : *ctx->context.devices()) {
     if (d->device_type() == final_devices[0].type_string()) {
       return d;
     }
@@ -663,9 +619,8 @@ tensorflow::Status Execute(
     const tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4>& op_inputs,
     tensorflow::KernelAndDevice* kernel, tensorflow::NodeExecStats* maybe_stats,
     TFE_TensorHandle** retvals, int num_retvals) {
-  if (!ctx->soft_placement && device == nullptr) {
-    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
-    device = ctx->devices[0];
+  if (!ctx->context.SoftPlacement() && device == nullptr) {
+    device = ctx->context.HostCPU();
   }
 
   if (device == nullptr) {
@@ -697,18 +652,18 @@ tensorflow::Status Execute(
   if (maybe_stats != nullptr) {
     maybe_stats->set_op_end_rel_micros(tensorflow::Env::Default()->NowMicros() -
                                        maybe_stats->all_start_micros());
-    tensorflow::mutex_lock ml(ctx->metadata_mu);
-    if (ctx->should_store_metadata.load()) {
-      auto* step_stats = ctx->run_metadata.mutable_step_stats();
+    tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
+    if (ctx->context.ShouldStoreMetadata()) {
+      auto* step_stats = ctx->context.RunMetadataProto()->mutable_step_stats();
       // Lazily initialize the RunMetadata with information about all devices if
       // this is the first call.
-      while (step_stats->dev_stats_size() < ctx->devices.size()) {
+      while (step_stats->dev_stats_size() < ctx->context.devices()->size()) {
         step_stats->add_dev_stats();
       }
       // Find the current device's index.
       int device_idx = 0;
-      for (int i = 0; i < ctx->devices.size(); ++i) {
-        if (ctx->devices[i] == device) {
+      for (int i = 0; i < ctx->context.devices()->size(); ++i) {
+        if (ctx->context.devices()->at(i) == device) {
           device_idx = i;
           break;
         }
@@ -744,7 +699,7 @@ class ExecuteNode : public tensorflow::EagerNode {
               tensorflow::NodeExecStats* maybe_stats,
               const tensorflow::DataTypeVector& output_dtypes,
               TFE_TensorHandle** retvals, int num_retvals)
-      : tensorflow::EagerNode(op->ctx->executor.NextId()),
+      : tensorflow::EagerNode(op->ctx->context.NextId()),
         ctx_(op->ctx),
         op_device_(op->device),
         inputs_(op->inputs),
@@ -800,7 +755,7 @@ class CopyToDeviceNode : public tensorflow::EagerNode {
  public:
   CopyToDeviceNode(TFE_TensorHandle* src, tensorflow::Device* dstd,
                    TFE_Context* ctx)
-      : tensorflow::EagerNode(ctx->executor.NextId()),
+      : tensorflow::EagerNode(ctx->context.NextId()),
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
@@ -1063,7 +1018,7 @@ extern "C" {
 void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
                  TF_Status* status) {
   TFE_Context* ctx = op->ctx;
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
   if (!status->status.ok()) {
     return;
   }
@@ -1087,7 +1042,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (op->inputs[i]->dtype == tensorflow::DT_RESOURCE &&
         input_op_device != op->device) {
       tensorflow::Device* d =
-          input_op_device == nullptr ? ctx->devices[0] : input_op_device;
+          input_op_device == nullptr ? ctx->context.HostCPU() : input_op_device;
       VLOG(1) << "Changing device of operation " << op->name << " to "
               << d->name() << " because input #" << i
               << " is a resource in this device.";
@@ -1095,40 +1050,35 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     }
   }
   tensorflow::Device* device = op->device;
-  if (!ctx->soft_placement && device == nullptr) {
-    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
-    device = ctx->devices[0];
+  if (!ctx->context.SoftPlacement() && device == nullptr) {
+    device = ctx->context.HostCPU();
   }
 
   tensorflow::Fprint128 cache_key =
       op->attrs.CacheKey(device == nullptr ? "unspecified" : device->name());
-  tensorflow::KernelAndDevice* kernel;
-  {
-    tensorflow::tf_shared_lock l(ctx->cache_mu);
-    kernel = tensorflow::gtl::FindPtrOrNull(ctx->kernel_cache, cache_key);
-  }
+  tensorflow::KernelAndDevice* kernel = ctx->context.GetCachedKernel(cache_key);
   if (kernel == nullptr) {
     const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef();
-    if (ctx->soft_placement && device == nullptr) {
+    if (ctx->context.SoftPlacement() && device == nullptr) {
       device = SelectDevice(ndef, ctx, status);
       if (!status->status.ok()) {
         return;
       }
     }
     CHECK(device != nullptr);
-    if (ctx->log_device_placement) {
+    if (ctx->context.LogDevicePlacement()) {
       LOG(INFO) << "Executing op " << ndef.op() << " in device "
                 << device->name();
     }
-    kernel = new tensorflow::KernelAndDevice(ctx->rendezvous);
+    kernel = new tensorflow::KernelAndDevice(ctx->context.GetRendezvous());
     // Knowledge of the implementation of Init (and in-turn
     // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def
     // will be accessed, so grab on to the lock.
     // See WARNING comment in Execute (before kernel->Run) - would be nice to
     // rework to avoid this subtlety.
-    tensorflow::tf_shared_lock l(ctx->functions_mu);
-    status->status =
-        tensorflow::KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel);
+    tensorflow::tf_shared_lock l(*ctx->context.FunctionsMu());
+    status->status = tensorflow::KernelAndDevice::Init(
+        ndef, ctx->context.func_lib(device), kernel);
     if (!status->status.ok()) {
       delete kernel;
       return;
@@ -1136,7 +1086,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // Update output_dtypes inside `kernel`.
     const tensorflow::OpDef* op_def = nullptr;
     const tensorflow::FunctionDef* function_def =
-        ctx->func_lib_def.Find(ndef.op());
+        ctx->context.FuncLibDef()->Find(ndef.op());
     if (function_def != nullptr) {
       op_def = &(function_def->signature());
     }
@@ -1152,8 +1102,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (!status->status.ok()) {
       return;
     }
-    tensorflow::mutex_lock ml(ctx->cache_mu);
-    tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel);
+    ctx->context.AddKernelToCache(cache_key, kernel);
   }
   const tensorflow::DataTypeVector& output_dtypes = kernel->output_dtypes();
   const int output_dtypes_size = output_dtypes.size();
@@ -1171,11 +1120,11 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // device from the one requested above.
     device = kernel->device();
   }
-  status->status = ValidateInputTypeAndPlacement(ctx, ctx->devices[0], device,
-                                                 op, kernel->kernel());
+  status->status = ValidateInputTypeAndPlacement(ctx, ctx->context.HostCPU(),
+                                                 device, op, kernel->kernel());
   if (!status->status.ok()) return;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats;
-  if (ctx->should_store_metadata.load()) {
+  if (ctx->context.ShouldStoreMetadata()) {
     maybe_stats.reset(new tensorflow::NodeExecStats);
     maybe_stats->set_node_name(op->name);
     maybe_stats->set_all_start_micros(tensorflow::Env::Default()->NowMicros());
@@ -1183,14 +1132,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     maybe_stats->set_scheduled_micros(tensorflow::Env::Default()->NowMicros());
     // TODO(apassos) track referenced tensors
   }
-  if (ctx->Async()) {
+  if (ctx->context.Async()) {
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
     // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
     tensorflow::EagerNode* node =
         new ExecuteNode(op, kernel, maybe_stats.release(), output_dtypes,
                         retvals, *num_retvals);
-    ctx->executor.Add(node);
+    ctx->context.ExecutorAdd(node);
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
@@ -1206,23 +1155,24 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
                                                TFE_Context* ctx,
                                                const char* device_name,
                                                TF_Status* status) {
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
   if (!status->status.ok()) {
     return nullptr;
   }
-  tensorflow::Device* dstd = ctx->devices[0];
+  tensorflow::Device* dstd = ctx->context.HostCPU();
   if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status = ctx->device_manager->LookupDevice(device_name, &dstd);
+    status->status =
+        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
     if (!status->status.ok()) return nullptr;
   }
-  if (ctx->Async()) {
+  if (ctx->context.Async()) {
     // Note that `h` may not be currently ready. However execution order will
     // make sure that `h` is ready before the copy is actually done.
     CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
     TFE_TensorHandle* output = node->dst();
     // Note that calling Add makes `node` accessible by the EagerExecutor
     // thread. So further accesses need to be thread-safe.
-    ctx->executor.Add(node);
+    ctx->context.ExecutorAdd(node);
     return output;
   } else {
     TFE_TensorHandle* output = nullptr;
@@ -1240,24 +1190,20 @@ void TFE_ContextAddFunctionDef(TFE_Context* ctx,
         tensorflow::errors::InvalidArgument("Invalid FunctionDef proto");
     return;
   }
-  tensorflow::mutex_lock l(ctx->functions_mu);
-  status->status = ctx->func_lib_def.AddFunctionDef(function_def);
+  status->status = ctx->context.AddFunctionDef(function_def);
 }
 
 void TFE_ContextAddFunction(TFE_Context* ctx, TF_Function* function,
                             TF_Status* status) {
-  tensorflow::mutex_lock l(ctx->functions_mu);
-  status->status = ctx->func_lib_def.AddFunctionDef(function->fdef);
+  status->status = ctx->context.AddFunctionDef(function->fdef);
 }
 
 void TFE_ContextEnableRunMetadata(TFE_Context* ctx) {
-  ctx->should_store_metadata.store(true);
+  ctx->context.SetShouldStoreMetadata(true);
 }
 
 void TFE_ContextDisableRunMetadata(TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->metadata_mu);
-  ctx->should_store_metadata.store(false);
-  ctx->run_metadata.Clear();
+  ctx->context.SetShouldStoreMetadata(false);
 }
 
 }  // extern "C"
@@ -1286,9 +1232,9 @@ void TFE_ContextExportRunMetadata(TFE_Context* ctx, TF_Buffer* buf,
                                   TF_Status* status) {
   TFE_ContextAsyncWait(ctx, status);
   if (!status->status.ok()) return;
-  tensorflow::mutex_lock ml(ctx->metadata_mu);
-  status->status = MessageToBuffer(ctx->run_metadata, buf);
-  ctx->run_metadata.Clear();
+  tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
+  status->status = MessageToBuffer(*ctx->context.RunMetadataProto(), buf);
+  ctx->context.RunMetadataProto()->Clear();
 }
 
 namespace {
@@ -1363,11 +1309,6 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
 }  // namespace tensorflow
 
 
-bool TFE_Context::Async() const {
-  tensorflow::mutex_lock l(async_map_mu);
-  return tensorflow::gtl::FindWithDefault(
-      thread_local_async, std::this_thread::get_id(), async_default);
-}
 
 bool TFE_TensorHandle::IsReady() {
   if (node_id == 0) return true;
@@ -1381,7 +1322,7 @@ tensorflow::Status TFE_TensorHandle::WaitReady() {
   {
     tensorflow::mutex_lock l(ctx_mutex_);
     if (ctx_ == nullptr) return tensorflow::Status::OK();
-    executor = &ctx_->executor;
+    executor = ctx_->context.Executor();
   }
   return executor->WaitFor(node_id);
 }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index a79f8ddd33..5b29120b40 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
@@ -52,85 +53,18 @@ struct TFE_ContextOptions {
       TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32};
 };
 
-TFE_ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy);
-
 struct TFE_Context {
-  explicit TFE_Context(const TFE_ContextOptions& opts,
+  explicit TFE_Context(const tensorflow::SessionOptions& opts,
+                       TFE_ContextDevicePlacementPolicy default_policy,
+                       bool async,
                        std::unique_ptr<tensorflow::DeviceMgr> device_mgr,
                        tensorflow::Rendezvous* rendezvous)
-      : soft_placement(
-            opts.session_options.options.config.allow_soft_placement()),
-        policy(PlacementPolicy(soft_placement, opts.policy)),
-        device_manager(std::move(device_mgr)),
-        devices(device_manager->ListDevices()),
-        rendezvous(rendezvous),
-        pflr(new tensorflow::ProcessFunctionLibraryRuntime(
-            device_manager.get(), opts.session_options.options.env,
-            TF_GRAPH_DEF_VERSION, &func_lib_def, {})),
-        log_device_placement(
-            opts.session_options.options.config.log_device_placement()),
-        async_default(opts.async) {
-    if (async_default) executor.EnableAsync();
-
-    for (auto* device : devices) {
-      devices_map[tensorflow::StringPiece(device->name())] = device;
-    }
-  }
-
-  const bool soft_placement;
-  const TFE_ContextDevicePlacementPolicy policy;
-
-  // Note: we cannot use C++11 thread_local here as there is no concept of a
-  // thread-local-object-local variable in C++11.
-  tensorflow::mutex policy_map_mu;
-  std::unordered_map<std::thread::id, TFE_ContextDevicePlacementPolicy>
-      thread_local_policies GUARDED_BY(policy_map_mu);
-
-  std::unique_ptr<tensorflow::DeviceMgr> device_manager;
-  // Devices owned by device_manager
-  std::vector<tensorflow::Device*> devices;
-  // All devices are not owned.
-  tensorflow::gtl::FlatMap<tensorflow::StringPiece, tensorflow::Device*,
-                           tensorflow::StringPieceHasher>
-      devices_map;
-  tensorflow::Rendezvous* const rendezvous;
-
-  tensorflow::mutex functions_mu;
-  tensorflow::FunctionLibraryDefinition func_lib_def GUARDED_BY(functions_mu){
-      tensorflow::OpRegistry::Global(), {}};
-
-  // One FunctionLibraryRuntime per device.
-  // func_libs[i] is the FunctionLibraryRuntime corresponding to
-  // session->devices[i].
-  const std::unique_ptr<tensorflow::ProcessFunctionLibraryRuntime> pflr;
-
-  tensorflow::mutex cache_mu;
-  std::unordered_map<tensorflow::Fprint128, tensorflow::KernelAndDevice*,
-                     tensorflow::Fprint128Hasher>
-      kernel_cache GUARDED_BY(cache_mu);
-
-  tensorflow::FunctionLibraryRuntime* func_lib(tensorflow::Device* d) const {
-    return pflr->GetFLR(d->name());
-  }
+      : context(opts,
+                static_cast<tensorflow::ContextDevicePlacementPolicy>(
+                    default_policy),
+                async, std::move(device_mgr), rendezvous) {}
 
-  // Whether we should compute RunMetadata.
-  std::atomic<bool> should_store_metadata{false};
-  tensorflow::mutex metadata_mu;
-  tensorflow::RunMetadata run_metadata GUARDED_BY(metadata_mu);
-  const bool log_device_placement;
-  // EagerExecutor for async execution.
-  tensorflow::EagerExecutor executor;
-
-  // True if running in asynchronous mode.
-  bool Async() const;
-
-  // True if the default value for execution mode is async. Note that this value
-  // can be overridden per thread based on `thread_local_async` overrides.
-  const bool async_default;
-  mutable tensorflow::mutex async_map_mu;
-  std::unordered_map<std::thread::id, bool> thread_local_async
-      GUARDED_BY(async_map_mu);
+  tensorflow::EagerContext context;
 };
 
 struct TFE_TensorHandle : public tensorflow::core::RefCounted {
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 8ba560bef8..de10b10b7e 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -32,6 +32,28 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "context",
+    srcs = [
+        "context.cc",
+    ],
+    hdrs = [
+        "context.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":eager_executor",
+        ":kernel_and_device",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
+    ],
+)
+
 tf_cuda_library(
     name = "kernel_and_device",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
new file mode 100644
index 0000000000..5e8d083cd2
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -0,0 +1,142 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+
+namespace tensorflow {
+
+ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, ContextDevicePlacementPolicy original_policy) {
+  if (!soft_placement) {
+    return original_policy;
+  }
+  if (original_policy == DEVICE_PLACEMENT_EXPLICIT ||
+      original_policy == DEVICE_PLACEMENT_SILENT_FOR_INT32) {
+    return DEVICE_PLACEMENT_SILENT;
+  }
+  return original_policy;
+}
+
+EagerContext::EagerContext(const SessionOptions& opts,
+                           ContextDevicePlacementPolicy default_policy,
+                           bool async, std::unique_ptr<DeviceMgr> device_mgr,
+                           Rendezvous* rendezvous)
+    : soft_placement_(opts.config.allow_soft_placement()),
+      policy_(PlacementPolicy(soft_placement_, default_policy)),
+      device_manager_(std::move(device_mgr)),
+      devices_(device_manager_->ListDevices()),
+      rendezvous_(rendezvous),
+      pflr_(new ProcessFunctionLibraryRuntime(device_manager_.get(), opts.env,
+                                              TF_GRAPH_DEF_VERSION,
+                                              &func_lib_def_, {})),
+      log_device_placement_(opts.config.log_device_placement()),
+      async_default_(async) {
+  if (async_default_) {
+    executor_.EnableAsync();
+  }
+
+  for (auto* device : devices_) {
+    devices_map_[device->name()] = device;
+  }
+}
+
+bool EagerContext::Async() const {
+  mutex_lock l(async_map_mu_);
+  return gtl::FindWithDefault(thread_local_async_, std::this_thread::get_id(),
+                              async_default_);
+}
+
+Status EagerContext::SetAsyncForThread(bool async) {
+  {
+    tensorflow::mutex_lock l(async_map_mu_);
+    thread_local_async_[std::this_thread::get_id()] = async;
+  }
+  if (async) {
+    executor_.EnableAsync();
+  } else {
+    // TODO(agarwal): Currently we add a wait here to handle cases where a
+    // sync op has a control dependency on an async op, and the latter has not
+    // executed yet. This wait can be removed by storing all the control
+    // inputs and waiting for them when executing ops.
+    return executor_.WaitForAllPendingNodes();
+  }
+  return Status::OK();
+}
+
+void EagerContext::ClearCaches() {
+  mutex_lock ml(cache_mu_);
+  gtl::STLDeleteValues(&kernel_cache_);
+}
+
+void EagerContext::SetThreadLocalDevicePlacementPolicy(
+    ContextDevicePlacementPolicy policy) {
+  mutex_lock ml(policy_map_mu_);
+  thread_local_policies_[std::this_thread::get_id()] = policy;
+}
+
+ContextDevicePlacementPolicy EagerContext::GetDevicePlacementPolicy() {
+  mutex_lock ml(policy_map_mu_);
+  auto policy_map_it = thread_local_policies_.find(std::this_thread::get_id());
+  if (policy_map_it != thread_local_policies_.end()) {
+    return policy_map_it->second;
+  }
+  return policy_;
+}
+
+EagerContext::~EagerContext() {
+  executor_.WaitForAllPendingNodes().IgnoreError();
+  ClearCaches();
+  rendezvous_->Unref();
+}
+
+bool EagerContext::FindFunctionByName(const string& name) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.Find(name) != nullptr;
+}
+
+Status EagerContext::FindDeviceByName(const string& name, Device** result) {
+  auto it = devices_map_.find(name);
+  if (it == devices_map_.end()) {
+    return errors::InvalidArgument(name, " unknown device.");
+  }
+  *result = it->second;
+  return Status::OK();
+}
+
+Status EagerContext::AddFunctionDef(const FunctionDef& fdef) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.AddFunctionDef(fdef);
+}
+
+KernelAndDevice* EagerContext::GetCachedKernel(Fprint128 cache_key) {
+  tf_shared_lock l(cache_mu_);
+  return gtl::FindPtrOrNull(kernel_cache_, cache_key);
+}
+
+void EagerContext::AddKernelToCache(Fprint128 cache_key,
+                                    KernelAndDevice* kernel) {
+  mutex_lock ml(cache_mu_);
+  gtl::InsertOrUpdate(&kernel_cache_, cache_key, kernel);
+}
+
+void EagerContext::SetShouldStoreMetadata(bool value) {
+  should_store_metadata_.store(value);
+  if (!value) {
+    mutex_lock ml(metadata_mu_);
+    run_metadata_.Clear();
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
new file mode 100644
index 0000000000..d525d44fe4
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -0,0 +1,193 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+// Note: there's a copy enum in eager/c_api.h. It should be kept in sync.
+enum ContextDevicePlacementPolicy {
+  // Running operations with input tensors on the wrong device will fail. When
+  // soft placement is enabled acts like TFE_DEVICE_PLACEMENT_SILENT.
+  DEVICE_PLACEMENT_EXPLICIT = 0,
+  // Copy the tensor to the right device but log a warning.
+  DEVICE_PLACEMENT_WARN = 1,
+  // Silently copy the tensor, which has a performance cost since the
+  // operation will be blocked till the copy completes.
+  DEVICE_PLACEMENT_SILENT = 2,
+  // Default placement policy which silently copies int32 tensors but not other
+  // dtypes.  When soft placement is enabled acts like
+  // TFE_DEVICE_PLACEMENT_SILENT.
+  DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
+};
+
+ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, ContextDevicePlacementPolicy original_policy);
+
+class EagerContext {
+ public:
+  explicit EagerContext(const SessionOptions& opts,
+                        ContextDevicePlacementPolicy default_policy, bool async,
+                        std::unique_ptr<DeviceMgr> device_mgr,
+                        Rendezvous* rendezvous);
+
+  ~EagerContext();
+
+  // Returns the function library runtime for the given device.
+  FunctionLibraryRuntime* func_lib(Device* d) const {
+    return pflr_->GetFLR(d->name());
+  }
+
+  // True if running in asynchronous mode.
+  bool Async() const;
+
+  EagerExecutor* Executor() { return &executor_; }
+
+  // Sets whether this thread should run in synchronous or asynchronous mode.
+  Status SetAsyncForThread(bool async);
+
+  // TODO(apassos) make this return a constant reference
+  gtl::FlatMap<string, Device*, StringPieceHasher>* device_map() {
+    return &devices_map_;
+  }
+
+  // TODO(apassos) make this return a constant reference
+  std::vector<Device*>* devices() { return &devices_; }
+
+  // Clears the kernel caches.
+  void ClearCaches();
+
+  // Sets the device placement policy for the current thread.
+  void SetThreadLocalDevicePlacementPolicy(ContextDevicePlacementPolicy policy);
+
+  // Returns the device placement policy for the current thread.
+  ContextDevicePlacementPolicy GetDevicePlacementPolicy();
+
+  Status AsyncWait() { return executor_.WaitForAllPendingNodes(); }
+
+  Status GetStatus() { return executor_.status(); }
+
+  void ClearAsyncError() { executor_.ClearError(); }
+
+  bool FindFunctionByName(const string& name);
+
+  Status FindDeviceByName(const string& name, Device** result);
+
+  Device* HostCPU() { return devices_[0]; }
+
+  bool SoftPlacement() { return soft_placement_; }
+
+  uint64 NextId() { return executor_.NextId(); }
+
+  void ExecutorAdd(EagerNode* node) { executor_.Add(node); }
+
+  Status AddFunctionDef(const FunctionDef& fdef);
+
+  KernelAndDevice* GetCachedKernel(Fprint128 cache_key);
+
+  void AddKernelToCache(Fprint128 cache_key, KernelAndDevice* kernel);
+
+  bool LogDevicePlacement() { return log_device_placement_; }
+
+  Rendezvous* GetRendezvous() { return rendezvous_; }
+
+  mutex* FunctionsMu() { return &functions_mu_; }
+
+  tensorflow::DeviceMgr* device_mgr() { return device_manager_.get(); }
+
+  // TODO(apassos) remove the need for this
+  void ReleaseDeviceMgr() { device_manager_.release(); }
+
+  // TODO(apassos) clean up RunMetadata storage.
+  mutex* MetadataMu() { return &metadata_mu_; }
+  bool ShouldStoreMetadata() { return should_store_metadata_.load(); }
+  void SetShouldStoreMetadata(bool value);
+  RunMetadata* RunMetadataProto() { return &run_metadata_; }
+
+  FunctionLibraryDefinition* FuncLibDef() { return &func_lib_def_; }
+
+ private:
+  const bool soft_placement_;
+  const ContextDevicePlacementPolicy policy_;
+
+  // Note: we cannot use C++11 thread_local here as there is no concept of a
+  // thread-local-object-local variable in C++11.
+  mutex policy_map_mu_;
+  std::unordered_map<std::thread::id, ContextDevicePlacementPolicy>
+      thread_local_policies_ GUARDED_BY(policy_map_mu_);
+
+  std::unique_ptr<DeviceMgr> device_manager_;
+  // Devices owned by device_manager
+  std::vector<Device*> devices_;
+  // All devices are not owned.
+  gtl::FlatMap<string, Device*, StringPieceHasher> devices_map_;
+  Rendezvous* const rendezvous_;
+
+  mutex functions_mu_;
+  FunctionLibraryDefinition func_lib_def_ GUARDED_BY(functions_mu_){
+      OpRegistry::Global(), {}};
+
+  // One FunctionLibraryRuntime per device.
+  // func_libs[i] is the FunctionLibraryRuntime corresponding to
+  // session->devices[i].
+  const std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+
+  mutex cache_mu_;
+  std::unordered_map<Fprint128, KernelAndDevice*, Fprint128Hasher> kernel_cache_
+      GUARDED_BY(cache_mu_);
+
+  // Whether we should compute RunMetadata.
+  std::atomic<bool> should_store_metadata_{false};
+  mutex metadata_mu_;
+  RunMetadata run_metadata_ GUARDED_BY(metadata_mu_);
+  const bool log_device_placement_;
+  // EagerExecutor for async execution.
+  EagerExecutor executor_;
+
+  // True if the default value for execution mode is async. Note that this value
+  // can be overridden per thread based on `thread_local_async` overrides.
+  const bool async_default_;
+  mutable mutex async_map_mu_;
+  std::unordered_map<std::thread::id, bool> thread_local_async_
+      GUARDED_BY(async_map_mu_);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
-- 
GitLab


From 73bd57d80111dc957d117b6ae98bc2354f766604 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 19:12:18 -0700
Subject: [PATCH 333/960] Add tensor quantization info to python wrapper

PiperOrigin-RevId: 190005998
---
 tensorflow/contrib/lite/python/interpreter.py   |  2 ++
 .../contrib/lite/python/interpreter_test.py     |  4 ++++
 .../interpreter_wrapper/interpreter_wrapper.cc  | 17 +++++++++++++++++
 .../interpreter_wrapper/interpreter_wrapper.h   |  1 +
 4 files changed, 24 insertions(+)

diff --git a/tensorflow/contrib/lite/python/interpreter.py b/tensorflow/contrib/lite/python/interpreter.py
index accdd04671..b8638007f7 100644
--- a/tensorflow/contrib/lite/python/interpreter.py
+++ b/tensorflow/contrib/lite/python/interpreter.py
@@ -71,6 +71,7 @@ class Interpreter(object):
     tensor_name = self._interpreter.TensorName(tensor_index)
     tensor_size = self._interpreter.TensorSize(tensor_index)
     tensor_type = self._interpreter.TensorType(tensor_index)
+    tensor_quantization = self._interpreter.TensorQuantization(tensor_index)
 
     if not tensor_name or not tensor_type:
       raise ValueError('Could not get tensor details')
@@ -80,6 +81,7 @@ class Interpreter(object):
         'index': tensor_index,
         'shape': tensor_size,
         'dtype': tensor_type,
+        'quantization': tensor_quantization,
     }
 
     return details
diff --git a/tensorflow/contrib/lite/python/interpreter_test.py b/tensorflow/contrib/lite/python/interpreter_test.py
index e85390c56c..bf124410f3 100644
--- a/tensorflow/contrib/lite/python/interpreter_test.py
+++ b/tensorflow/contrib/lite/python/interpreter_test.py
@@ -39,12 +39,14 @@ class InterpreterTest(test_util.TensorFlowTestCase):
     self.assertEqual('input', input_details[0]['name'])
     self.assertEqual(np.float32, input_details[0]['dtype'])
     self.assertTrue(([1, 4] == input_details[0]['shape']).all())
+    self.assertEqual((0.0, 0), input_details[0]['quantization'])
 
     output_details = interpreter.get_output_details()
     self.assertEqual(1, len(output_details))
     self.assertEqual('output', output_details[0]['name'])
     self.assertEqual(np.float32, output_details[0]['dtype'])
     self.assertTrue(([1, 4] == output_details[0]['shape']).all())
+    self.assertEqual((0.0, 0), output_details[0]['quantization'])
 
     test_input = np.array([[1.0, 2.0, 3.0, 4.0]], dtype=np.float32)
     expected_output = np.array([[4.0, 3.0, 2.0, 1.0]], dtype=np.float32)
@@ -67,12 +69,14 @@ class InterpreterTest(test_util.TensorFlowTestCase):
       self.assertEqual('input', input_details[0]['name'])
       self.assertEqual(np.uint8, input_details[0]['dtype'])
       self.assertTrue(([1, 4] == input_details[0]['shape']).all())
+      self.assertEqual((1.0, 0), input_details[0]['quantization'])
 
       output_details = interpreter.get_output_details()
       self.assertEqual(1, len(output_details))
       self.assertEqual('output', output_details[0]['name'])
       self.assertEqual(np.uint8, output_details[0]['dtype'])
       self.assertTrue(([1, 4] == output_details[0]['shape']).all())
+      self.assertEqual((1.0, 0), output_details[0]['quantization'])
 
       test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
       expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
index 14e1190c80..35ad226b78 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.cc
@@ -109,6 +109,13 @@ PyObject* PyArrayFromIntVector(const int* data, npy_intp size) {
   return PyArray_SimpleNewFromData(1, &size, NPY_INT32, pydata);
 }
 
+PyObject* PyTupleFromQuantizationParam(const TfLiteQuantizationParams& param) {
+  PyObject* result = PyTuple_New(2);
+  PyTuple_SET_ITEM(result, 0, PyFloat_FromDouble(param.scale));
+  PyTuple_SET_ITEM(result, 1, PyInt_FromLong(param.zero_point));
+  return result;
+}
+
 }  // namespace
 
 InterpreterWrapper::InterpreterWrapper(
@@ -214,6 +221,16 @@ PyObject* InterpreterWrapper::TensorSize(int i) const {
   return PyArray_Return(reinterpret_cast<PyArrayObject*>(np_array));
 }
 
+PyObject* InterpreterWrapper::TensorQuantization(int i) const {
+  if (!interpreter_ || i >= interpreter_->tensors_size() || i < 0) {
+    Py_INCREF(Py_None);
+    return Py_None;
+  }
+
+  const TfLiteTensor* tensor = interpreter_->tensor(i);
+  return PyTupleFromQuantizationParam(tensor->params);
+}
+
 bool InterpreterWrapper::SetTensor(int i, PyObject* value) {
   if (!interpreter_) {
     LOG(ERROR) << "Invalid interpreter.";
diff --git a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
index 63bdb30f79..0972c57259 100644
--- a/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
+++ b/tensorflow/contrib/lite/python/interpreter_wrapper/interpreter_wrapper.h
@@ -54,6 +54,7 @@ class InterpreterWrapper {
   std::string TensorName(int i) const;
   PyObject* TensorType(int i) const;
   PyObject* TensorSize(int i) const;
+  PyObject* TensorQuantization(int i) const;
   bool SetTensor(int i, PyObject* value);
   PyObject* GetTensor(int i) const;
 
-- 
GitLab


From 212a42a01d7b30fec1d6f8ca34dbf9c095938d4a Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Wed, 21 Mar 2018 22:11:10 -0700
Subject: [PATCH 334/960] Simplified the experimental APIs related to TPU
 execution, by moving the graph rewrite functionality out of it.

PiperOrigin-RevId: 190016936
---
 tensorflow/c/BUILD                 |   1 +
 tensorflow/c/c_api_experimental.cc | 432 ++---------------------------
 tensorflow/c/c_api_experimental.h  |  47 ++--
 3 files changed, 39 insertions(+), 441 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index c178d7f81f..4332f44e5d 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -116,6 +116,7 @@ tf_cuda_library(
         ":c_api",
         ":c_api_internal",
         "//tensorflow/compiler/jit/legacy_flags:mark_for_compilation_pass_flags",
+        "//tensorflow/contrib/tpu:all_ops",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 34b9dec3ee..29caf508e7 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -22,389 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
-using tensorflow::Node;
-using tensorflow::NodeBuilder;
-using tensorflow::NodeDef;
 using tensorflow::Status;
-using tensorflow::string;
-
-namespace {
-
-const char* const DEVICE_TPU_REPLICATED_CORE = "TPU_REPLICATED_CORE";
-const char* const DEVICE_TPU_SYSTEM = "TPU_SYSTEM";
-
-TF_Operation* ToTF_Operation(Node* node) {
-  return static_cast<TF_Operation*>(static_cast<void*>(node));
-}
-
-// Graph rewrite algorithm (modeled after the python TPU graph rewrite path):
-//
-// 1. For each input node I, with C being the consumer node of I's output:
-//
-// a) When infeed is not specified, feed I to a new TPUReplicatedInput node
-// (both running on CPU), which in turn feeds a new Identity node N, and N feeds
-// C (both running on TPU).
-//
-// b) Otherwise, feed I to a new InfeedEnqueueTuple node IE, both running on
-// CPU. Also set an InfeedDequeueTuple node ID to feed C, both running on
-// TPU.
-//
-// In case b), if we have multiple input nodes, they all feed into the same
-// InfeedEnqueueTuple node, so that the graph has a single pair of infeed
-// enqueue and dequeue nodes. The list of output tensors from the dequeue node
-// can go to different consumer nodes. For example, say the original graph has
-// input nodes I1 and I2 respectively feeding nodes C1 and C2. After the rewrite
-// with infeed ops, we will have: I1 and I2 feed a single infeed enqueue node
-// IE, and a corresponding infeed dequeue node ID produces a list of two
-// tensors, respectively feeding C1 and C2.
-//
-// 2. Rewrite all existing graph nodes by adding an attribute on TPU
-// cluster. For each node C reading some input node I, rewire it to read from a
-// new input node generated in step #1 above.
-//
-// 3. For each output node O, feed it to a new Identity node, which in turn
-// feeds a new TPUReplicatedOutput node, which in turn feeds a new Identity node
-// M. Return the set of new output nodes (the "M" nodes) for caller to fetch
-// from.
-//
-// Limitations compared to the python TPU rewrite path:
-// - # replicas is always 1.
-// - Less error checking.
-//
-// TODO(hongm): Simplify the graph rewrite to generating fewer TPUReplicate
-// related nodes.
-class GraphRewriter {
- public:
-  GraphRewriter(TF_Graph* graph, int num_input_nodes,
-                const TF_Output* input_nodes, int num_output_nodes,
-                const TF_Output* output_nodes)
-      EXCLUSIVE_LOCKS_REQUIRED(graph->mu)
-      : graph_(graph), input_nodes_(input_nodes) {
-    for (int i = 0; i < num_input_nodes; ++i) {
-      //  Will fill in the value part later when we create the associated new
-      //  input node.
-      input_node_map_[input_nodes[i].oper->node.name()] =
-          NodeBuilder::NodeOut(nullptr, -1);
-    }
-
-    // Grab all existing nodes for the upcoming rewrite, before mutating the
-    // graph.
-    for (Node* n : graph->graph.nodes()) {
-      nodes_to_rewrite_.push_back(n);
-    }
-
-    for (int i = 0; i < num_output_nodes; ++i) {
-      output_node_map_.emplace(output_nodes[i].oper->node.name(),
-                               PortIndexPair{output_nodes[i].index, i});
-    }
-  }
-
-  // On success, sets `config_op` and `shutdown_op` to the corresponding
-  // "ConfigureDistributedTPU" and "ShutdownDistributedTPU" nodes added to the
-  // graph.
-  tensorflow::Status Rewrite(TF_Output* new_output_nodes,
-                             TF_Operation** infeed_enqueue_node,
-                             TF_Output* config_op, TF_Output* shutdown_op)
-      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
-    TF_RETURN_IF_ERROR(ProcessInputNodes(infeed_enqueue_node));
-
-    return RewriteGraphAndAddOutputNodes(new_output_nodes, config_op,
-                                         shutdown_op);
-  }
-
- private:
-  // Synthesizes new graph nodes (infeed enqueue or TPU replicated input
-  // nodes) for the input nodes, and creates a replicated metadata node.
-  //
-  // When `infeed_enqueue_node` is non-NULL and there are some input nodes,
-  // also adds the infeed dequeue node.
-  tensorflow::Status ProcessInputNodes(TF_Operation** infeed_enqueue_node)
-      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
-    Node* metadata_node;
-    TF_RETURN_IF_ERROR(
-        NodeBuilder(metadata_node_name_.c_str(), "TPUReplicateMetadata")
-            .Attr("num_replicas", 1)
-            .Attr("_tpu_replicate", cluster_name_.c_str())
-            .Finalize(&graph_->graph, &metadata_node));
-
-    Node* dequeue_node = nullptr;
-    // Be deterministic in the corner case where `use_infeed` below is false.
-    if (infeed_enqueue_node) *infeed_enqueue_node = nullptr;
-    const bool use_infeed =
-        infeed_enqueue_node != nullptr && !input_node_map_.empty();
-    if (use_infeed) {
-      std::vector<NodeBuilder::NodeOut> new_input_list;
-      new_input_list.reserve(input_node_map_.size());
-      std::vector<tensorflow::DataType> input_dtypes;
-      input_dtypes.reserve(input_node_map_.size());
-      std::vector<tensorflow::TensorShape> input_shapes;
-      input_shapes.reserve(input_node_map_.size());
-      for (int i = 0; i < input_node_map_.size(); ++i) {
-        Node& input_node = input_nodes_[i].oper->node;
-        new_input_list.push_back(
-            NodeBuilder::NodeOut(&input_node, input_nodes_[i].index));
-        input_dtypes.push_back(input_node.output_type(input_nodes_[i].index));
-        tensorflow::TensorShapeProto shape;
-        TF_RETURN_IF_ERROR(
-            tensorflow::GetNodeAttr(input_node.attrs(), "shape", &shape));
-        VLOG(1) << "Input node " << i << " has shape " << shape.DebugString();
-        input_shapes.push_back(shape);
-      }
-      // Enqueue always runs on CPU.
-      Node* enqueue_node;
-      TF_RETURN_IF_ERROR(NodeBuilder("InfeedEnqueueTuple", "InfeedEnqueueTuple")
-                             .Input(new_input_list)
-                             .Device("/device:CPU:0")
-                             .Attr("device_ordinal", 0)
-                             .Attr("dtypes", input_dtypes)
-                             .Attr("shapes", input_shapes)
-                             .Finalize(&graph_->graph, &enqueue_node));
-      *infeed_enqueue_node = ToTF_Operation(enqueue_node);
-      // The dequeue node should be put onto the "_tpu_replicate" cluster.
-      TF_RETURN_IF_ERROR(
-          NodeBuilder("TPUReplicate/InfeedDequeueTuple", "InfeedDequeueTuple")
-              .ControlInput(metadata_node)
-              .Attr("_tpu_replicate", cluster_name_.c_str())
-              .Attr("dtypes", input_dtypes)
-              .Attr("shapes", input_shapes)
-              .Finalize(&graph_->graph, &dequeue_node));
-    }
-
-    for (int i = 0; i < input_node_map_.size(); ++i) {
-      VLOG(1) << "Handling input node " << input_nodes_[i].oper->node.name();
-      if (use_infeed) {
-        DCHECK(dequeue_node);
-        input_node_map_[input_nodes_[i].oper->node.name()] =
-            NodeBuilder::NodeOut(dequeue_node, i);
-      } else {
-        Node* replicated_input_node;
-        {
-          std::string replicated_input_name("TPUReplicate/input" +
-                                            std::to_string(i));
-          NodeBuilder::NodeOut input(&input_nodes_[i].oper->node,
-                                     input_nodes_[i].index);
-          std::vector<NodeBuilder::NodeOut> input_list;
-          input_list.push_back(input);
-          TF_RETURN_IF_ERROR(
-              NodeBuilder(replicated_input_name.c_str(), "TPUReplicatedInput")
-                  // This op requires an input list.
-                  .Input(input_list)
-                  .Finalize(&graph_->graph, &replicated_input_node));
-        }
-
-        {
-          Node* new_input_node;
-          const std::string new_input_name("TPUReplicate/replicated_input_" +
-                                           std::to_string(i));
-          TF_RETURN_IF_ERROR(NodeBuilder(new_input_name.c_str(), "Identity")
-                                 .Input(replicated_input_node, 0)
-                                 .ControlInput(metadata_node)
-                                 .Attr("_tpu_replicate", cluster_name_.c_str())
-                                 .Finalize(&graph_->graph, &new_input_node));
-          DCHECK_GT(input_node_map_.count(input_nodes_[i].oper->node.name()),
-                    0);
-          input_node_map_[input_nodes_[i].oper->node.name()] =
-              NodeBuilder::NodeOut(new_input_node, 0);
-        }
-      }
-    }
-    return Status::OK();
-  }
-
-  // On success, sets `config_op` and `shutdown_op` to the corresponding
-  // "ConfigureDistributedTPU" and "ShutdownDistributedTPU" nodes added to the
-  // graph.
-  tensorflow::Status RewriteGraphAndAddOutputNodes(TF_Output* new_output_nodes,
-                                                   TF_Output* config_op,
-                                                   TF_Output* shutdown_op)
-      EXCLUSIVE_LOCKS_REQUIRED(graph_->mu) {
-    tensorflow::Status s;
-    // For each non-input node in the input graph, place the node in a "TPU
-    // replicate cluster" via an attribute, and with the above metadata node
-    // as a control dependency.
-    //
-    // Although we have handled the input nodes in ProcessInputNodes(), some
-    // of those nodes may also serve as output nodes, which we will handle
-    // below.
-    for (Node* n : nodes_to_rewrite_) {
-      if (n->IsSource()) continue;
-      VLOG(1) << "Rewriting node " << n->name();
-
-      if (n->IsSink()) {
-        // TODO(hongm): Rewire SINK to be control dependent on the new input
-        // nodes created above?
-        continue;
-      }
-
-      const NodeDef& old_def = n->def();
-      // Let node C be the consumer of `n`'s output in the original graph.
-      // This new node will feed into C in the rewritten graph.
-      NodeBuilder::NodeOut new_node;
-      if (input_node_map_.count(n->name())) {
-        new_node = input_node_map_[n->name()];
-      } else {
-        // This node is to replace `n` in the graph.
-        NodeDef new_def = n->def();
-        const std::string new_node_name = "TPUReplicate/" + n->name();
-        new_def.set_name(new_node_name);
-        new_def.clear_input();
-        for (int i = 0; i < old_def.input_size(); ++i) {
-          const string old_input_name = old_def.input(i);
-          // When there are multiple input nodes that get mapped to the same
-          // infeed dequeue node, use different output ports of the dequeue
-          // node. e.g. Say in the original graph, input I1 feeds C1, and I2
-          // feeds C2. After the rewrite, I1 and I2 both feed a new infeed
-          // enqueue node, and the corresponding dequeue node has its output
-          // port 0 feeding C1, and output port 1 feeding C2. Note C1 and C2
-          // could be the same node (e.g. an Add that takes 2 inputs).
-          const string new_input_name =
-              input_node_map_.count(old_input_name) > 0
-                  ? tensorflow::strings::StrCat(
-                        input_node_map_[old_input_name].node->name(), ":",
-                        input_node_map_[old_input_name].index)
-                  : "TPUReplicate/" + old_input_name;
-          new_def.add_input(new_input_name);
-        }
-        if (old_def.input_size() == 0) {
-          // It is sufficient to only set control dependency of nodes without
-          // input. Other nodes with input(s) with inherit such control
-          // dependency.
-          // e.g. say the graph computes add(x, y). Once we make nodes x and y
-          // control-dependent on the metadata node, node add will inherit
-          // such control dependency indirectly.
-          new_def.add_input(
-              tensorflow::strings::StrCat("^", metadata_node_name_.c_str()));
-        }
-        tensorflow::AddNodeAttr("_tpu_replicate", cluster_name_.c_str(),
-                                &new_def);
-        new_node = NodeBuilder::NodeOut(graph_->graph.AddNode(new_def, &s), 0);
-        if (!s.ok()) {
-          return s;
-        }
-        VLOG(1) << "The rewritten node node is "
-                << new_node.node->DebugString();
-      }
-
-      if (output_node_map_.count(n->name()) > 0) {
-        VLOG(1) << "Handling output node " << n->name();
-        auto range_it = output_node_map_.equal_range(n->name());
-        for (auto it = range_it.first; it != range_it.second; ++it) {
-          const PortIndexPair& pair = it->second;
-          Node* out_identity_node;
-          {
-            // If this output node is also an input, use the input_node_map_'s
-            // stored port, which would also work for an infeed dequeue op.
-            // Otherwise use pair.port.
-            // An example of the former: Say the graph has input nodes I1 and
-            // I2, and the output nodes are also I1 and I2. In the rewritten
-            // graph with infeed, the 2 output nodes will both come from a
-            // single infeed dequeue node ID, with output ports respectively
-            // set to 0 and 1.
-            const int output_port =
-                input_node_map_.count(n->name()) ? new_node.index : pair.port;
-            VLOG(1) << "Handling its output port " << output_port
-                    << " at output index " << pair.index;
-            std::string output_node_name = "TPUReplicate/Identity";
-            if (pair.index > 0) {
-              output_node_name += "_" + std::to_string(pair.index);
-            }
-            TF_RETURN_IF_ERROR(
-                NodeBuilder(output_node_name.c_str(), "Identity")
-                    .Input(new_node.node, output_port)
-                    .Device(!old_def.device().empty()
-                                ? old_def.device()
-                                : tensorflow::strings::StrCat(
-                                      "/device:", DEVICE_TPU_REPLICATED_CORE))
-                    .Attr("_tpu_replicate", cluster_name_.c_str())
-                    .Finalize(&graph_->graph, &out_identity_node));
-            VLOG(1) << "out_identity_node: "
-                    << out_identity_node->DebugString();
-          }
-
-          Node* replicated_output_node;
-          {
-            const std::string replicated_output_node_name =
-                "TPUReplicate/output" + std::to_string(pair.index);
-            TF_RETURN_IF_ERROR(
-                NodeBuilder(replicated_output_node_name.c_str(),
-                            "TPUReplicatedOutput")
-                    .Input(out_identity_node, 0)
-                    .Attr("num_replicas", 1)
-                    .Finalize(&graph_->graph, &replicated_output_node));
-            VLOG(1) << "replicated_output_node: "
-                    << replicated_output_node->DebugString();
-          }
-
-          Node* final_output_node;
-          const std::string final_output_node_name =
-              "TPUReplicate/output_" + std::to_string(pair.index) + "_shard_" +
-              std::to_string(0);
-          TF_RETURN_IF_ERROR(
-              NodeBuilder(final_output_node_name.c_str(), "Identity")
-                  .Input(replicated_output_node, 0)
-                  .Finalize(&graph_->graph, &final_output_node));
-          VLOG(1) << "new_output_node: " << final_output_node->DebugString();
-          auto oper = ToTF_Operation(final_output_node);
-          new_output_nodes[pair.index] = {oper, 0};
-        }
-      }
-
-      if (input_node_map_.count(n->name()) == 0) {
-        graph_->graph.RemoveNode(n);
-      }
-    }
-
-    {
-      Node* config_node;
-      TF_RETURN_IF_ERROR(
-          NodeBuilder("ConfigureDistributedTPU", "ConfigureDistributedTPU")
-              .Device(DEVICE_TPU_SYSTEM)
-              .Finalize(&graph_->graph, &config_node));
-      *config_op = {ToTF_Operation(config_node), 0};
-    }
-
-    {
-      Node* shutdown_node;
-      TF_RETURN_IF_ERROR(
-          NodeBuilder("ShutdownDistributedTPU", "ShutdownDistributedTPU")
-              .Device(DEVICE_TPU_SYSTEM)
-              .Finalize(&graph_->graph, &shutdown_node));
-      *shutdown_op = {ToTF_Operation(shutdown_node), 0};
-    }
-
-    return Status::OK();
-  }
-
-  TF_Graph* const graph_;
-
-  const TF_Output* const input_nodes_;
-
-  const std::string cluster_name_ = "TPUReplicate/cluster";
-  const std::string metadata_node_name_ = "TPUReplicate/TPUReplicateMetadata";
-
-  // Keep mappings from the current input nodes to newly created input nodes,
-  // which we will use to rewrite existing nodes that read these
-  // inputs. e.g. A node that reads input node PlaceHolder could be rewired to
-  // read the created TPUReplicate/replicated_input_0 node or some output port
-  // of the created TPUReplicate/InfeedDequeueTuple node. Because of the latter
-  // case, we the map entries store NodeBuilder::NodeOut, and not just Node*.
-  std::unordered_map<std::string, NodeBuilder::NodeOut> input_node_map_;
-
-  std::vector<Node*> nodes_to_rewrite_;
-
-  // Map from name to set{(output port, output tensor idx)}.
-  // e.g. Say there are 3 output tensors, respectively produced by (node 0,
-  // port 0), (node 0, port 1), (node 1, port 0). Then the mapping entries
-  // are: node 0 -> {(port 0, idx 0), (port 1, idx 1)} node 1 -> {(port 0, idx
-  // 2)} Based on these mappings, we will generate 3 new output nodes.
-  struct PortIndexPair {
-    int port;
-    int index;
-  };
-  std::multimap<std::string, PortIndexPair> output_node_map_;
-};
-
-}  // namespace
 
 void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
   tensorflow::ConfigProto& config = options->options.config;
@@ -425,62 +43,54 @@ void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
   }
 }
 
-TF_Output TF_SetupTPUExecution(TF_Session* session, int num_input_nodes,
-                               const TF_Output* input_nodes,
-                               int num_output_nodes,
-                               const TF_Output* output_nodes,
-                               TF_Output* new_output_nodes,
-                               TF_Operation** infeed_enqueue_node,
-                               TF_Status* status) {
-  TF_Output config_op, shutdown_op;
-  {
-    auto graph = session->graph;
-    tensorflow::mutex_lock c(graph->mu);
-
-    VLOG(1) << "Graph before TPU rewrite: "
-            << graph->graph.ToGraphDefDebug().DebugString();
-    GraphRewriter rewriter(graph, num_input_nodes, input_nodes,
-                           num_output_nodes, output_nodes);
-    status->status = rewriter.Rewrite(new_output_nodes, infeed_enqueue_node,
-                                      &config_op, &shutdown_op);
-    if (!status->status.ok()) {
-      return shutdown_op;
-    }
-    VLOG(1) << "Graph after TPU rewrite: "
-            << graph->graph.ToGraphDefDebug().DebugString();
+void TF_InitializeTPU(TF_Session* session, TF_Status* status) {
+  VLOG(1) << "Initializing TPU";
+  TF_Operation* config_op =
+      TF_GraphOperationByName(session->graph, "ConfigureDistributedTPU");
+  if (config_op == nullptr) {
+    status->status = tensorflow::errors::Internal(
+        "Unable to find node ConfigureDistributedTPU in the TF graph.");
+    return;
   }
 
-  VLOG(1) << "Initializing TPU";
+  TF_Output config_node{config_op, 0};
+
   TF_Tensor* dummy_output;
   TF_SessionRun(session, /*run_options*/ nullptr,
                 // input related parameters
                 /*inputs*/ nullptr, /*input_values*/ nullptr, /*ninputs*/ 0,
                 // output related parameters
-                /*outputs*/ &config_op, /*output_values*/ &dummy_output,
+                /*outputs*/ &config_node, /*output_values*/ &dummy_output,
                 /*noutputs*/ 1,
                 /*targets*/ nullptr, /*ntargets*/ 0,
                 /*run_metadata*/ nullptr, status);
   if (status->status.ok()) {
     TF_DeleteTensor(dummy_output);
   }
-  return shutdown_op;
 }
 
-void TF_ShutdownTPUExecution(TF_Session* session, TF_Output shutdown_node,
-                             TF_Status* status) {
+void TF_ShutdownTPU(TF_Session* session, TF_Status* status) {
   {
     tensorflow::mutex_lock c(session->graph->mu);
     VLOG(1) << "Shutting down TPU, with input graph: "
             << session->graph->graph.ToGraphDefDebug().DebugString();
   }
 
+  TF_Operation* shutdown_op =
+      TF_GraphOperationByName(session->graph, "ShutdownDistributedTPU");
+  if (shutdown_op == nullptr) {
+    status->status = tensorflow::errors::Internal(
+        "Unable to find node ShutdownDistributedTPU in the TF graph.");
+    return;
+  }
+
   TF_SessionRun(session, /*run_options*/ nullptr,
                 // input related parameters
                 /*inputs*/ nullptr, /*input_values*/ nullptr, /*ninputs*/ 0,
                 // output related parameters
                 /*outputs*/ nullptr, /*output_values*/ nullptr,
                 /*noutputs*/ 0,
-                /*targets*/ &shutdown_node.oper, /*ntargets*/ 1,
+                /*targets*/ &shutdown_op, /*ntargets*/ 1,
                 /*run_metadata*/ nullptr, status);
 }
 
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index b95cdfe6aa..f069398bbb 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -60,39 +60,26 @@ extern "C" {
 TF_CAPI_EXPORT extern void TF_EnableXLACompilation(TF_SessionOptions* options,
                                                    unsigned char enable);
 
-// Sets up TPU execution, by rewriting the graph accordingly, and initializing
-// TPU system.
+// Initializes TPU system. Must be called exactly once before TF_SessionRun() is
+// called on a TPU graph.
 //
-// When `infeed_enqueue_node` is non-NULL and there are input tensors, rewrites
-// the graph by adding the relevant infeed enqueue/dequeue ops, and returns the
-// enqueue op in `infeed_enqueue_node` on success, so that user can run that
-// node and feed input tensors. When there are no input tensors,
-// `infeed_enqueue_node` is ignored, and user should not run that node later.
-// TODO(hongm): In this case, we currently only support input tensors of dim 0
-// shape. Lift that constraint.
-//
-// On success, also returns a shutdown node to be used in a subsequent
-// TF_ShutdownTPUExecution(), and sets the new output nodes in
-// `new_output_nodes` for caller to fetch from. Must be called exactly once
-// before TF_SessionRun().
-//
-// The API and logic is modeled after the python counterparts
-// tpu.{initialize_system(), rewrite(), shutdown_system()}.
-//
-// TODO(b/74774824): Create separate APIs for initializing TPU system and graph
-// rewrite.
-TF_CAPI_EXPORT extern TF_Output TF_SetupTPUExecution(
-    TF_Session* session, int num_input_nodes, const TF_Output* input_nodes,
-    int num_output_nodes, const TF_Output* output_nodes,
-    TF_Output* new_output_nodes, TF_Operation** infeed_enqueue_node,
-    TF_Status* status);
-
-// Shuts down TPU system. For any `session` where TF_SetupTPUExecution() has
+// The session graph must contain a node named ConfigureDistributedTPU.
+// TODO(b/74774824): Improve the API on initializing TPU system.
+TF_CAPI_EXPORT extern void TF_InitializeTPU(TF_Session* session,
+                                            TF_Status* status);
+
+// Shuts down TPU system. For any `session` where TF_InitializeTPU() has
 // been successfully called, this call must be made exactly once before the
 // session is closed.
-TF_CAPI_EXPORT extern void TF_ShutdownTPUExecution(TF_Session* session,
-                                                   TF_Output shutdown_node,
-                                                   TF_Status* status);
+// The session graph must contain a node named ShutdownDistributedTPU.
+TF_CAPI_EXPORT extern void TF_ShutdownTPU(TF_Session* session,
+                                          TF_Status* status);
+
+// Returns the graph content in a human-readable format, with length set in
+// `len`. The format is subject to change in the future.
+// The returned string is heap-allocated, and caller should call free() on it.
+TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
+                                                      size_t* len);
 
 // Returns the graph content in a human-readable format, with length set in
 // `len`. The format is subject to change in the future.
-- 
GitLab


From 0e1775355f9d7fe5301bc0d17906453caf970e27 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Wed, 21 Mar 2018 23:04:59 -0700
Subject: [PATCH 335/960] Merge changes from github.

PiperOrigin-RevId: 190020572
---
 .../data/python/kernel_tests/resample_test.py | 40 +++++++++++++++++++
 .../contrib/data/python/ops/resampling.py     |  9 +++--
 .../core/kernels/segment_reduction_ops.h      |  8 ----
 .../docs_src/tutorials/kernel_methods.md      |  4 +-
 .../docs_src/tutorials/recurrent_quickdraw.md |  8 ++--
 5 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/resample_test.py b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
index 913ab9b9f8..5f47dcb339 100644
--- a/tensorflow/contrib/data/python/kernel_tests/resample_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/resample_test.py
@@ -21,7 +21,10 @@ import numpy as np
 
 from tensorflow.contrib.data.python.ops import resampling
 from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
@@ -68,6 +71,43 @@ class ResampleTest(test.TestCase):
     returned_dist = class_counts / total_returned
     self.assertAllClose(target_dist, returned_dist, atol=1e-2)
 
+  def testRandomClasses(self):
+    init_dist = [0.25, 0.25, 0.25, 0.25]
+    target_dist = [0.0, 0.0, 0.0, 1.0]
+    num_classes = len(init_dist)
+    # We don't need many samples to test a dirac-delta target distribution
+    num_samples = 100
+    data_np = np.random.choice(num_classes, num_samples, p=init_dist)
+
+    dataset = dataset_ops.Dataset.from_tensor_slices(data_np)
+
+    # Apply a random mapping that preserves the data distribution.
+    def _remap_fn(_):
+      return math_ops.cast(random_ops.random_uniform([1]) * num_classes,
+                           dtypes.int32)[0]
+    dataset = dataset.map(_remap_fn)
+
+    # Reshape distribution.
+    dataset = dataset.apply(
+        resampling.rejection_resample(
+            class_func=lambda x: x,
+            target_dist=target_dist,
+            initial_dist=init_dist))
+
+    get_next = dataset.make_one_shot_iterator().get_next()
+
+    with self.test_session() as sess:
+      returned = []
+      with self.assertRaises(errors.OutOfRangeError):
+        while True:
+          returned.append(sess.run(get_next))
+
+    classes, _ = zip(*returned)
+    bincount = np.bincount(
+        np.array(classes),
+        minlength=num_classes).astype(np.float32) / len(classes)
+
+    self.assertAllClose(target_dist, bincount, atol=1e-2)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/resampling.py b/tensorflow/contrib/data/python/ops/resampling.py
index f4015f19fb..a182dddd38 100644
--- a/tensorflow/contrib/data/python/ops/resampling.py
+++ b/tensorflow/contrib/data/python/ops/resampling.py
@@ -101,11 +101,12 @@ def rejection_resample(class_func, target_dist, initial_dist=None, seed=None):
                                                    initial_dist_ds))
                           .map(maybe_warn_on_large_rejection))
 
-    current_probabilities_ds = dataset_ops.Dataset.zip(
-        (acceptance_dist_ds, class_values_ds)).map(array_ops.gather)
+    def _gather_and_copy(class_val, acceptance_prob, data):
+      return (class_val, array_ops.gather(acceptance_prob, class_val), data)
+    current_probabilities_and_class_and_data_ds = dataset_ops.Dataset.zip(
+        (class_values_ds, acceptance_dist_ds, dataset)).map(_gather_and_copy)
     filtered_ds = (
-        dataset_ops.Dataset.zip((class_values_ds, current_probabilities_ds,
-                                 dataset))
+        current_probabilities_and_class_and_data_ds
         .filter(lambda _1, p, _2: random_ops.random_uniform([], seed=seed) < p))
     return filtered_ds.map(lambda class_value, _, data: (class_value, data))
 
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index d65692a552..4abfbfb1a6 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -16,14 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
 #define TENSORFLOW_CORE_KERNELS_SEGMENT_REDUCTION_OPS_H_
 
-
-// This file requires the following include because it uses CudaAtomicMax:
-// #include "tensorflow/core/util/cuda_kernel_helper.h"
-
-// Unfortunately we can't add the #include, since it breaks compilation for
-// non-GPU targets. This only breaks in clang, because it's more strict for
-// template code and CudaAtomicMax is used in template context.
-
 // This file requires the following include because it uses CudaAtomicMax:
 // #include "tensorflow/core/util/cuda_kernel_helper.h"
 
diff --git a/tensorflow/docs_src/tutorials/kernel_methods.md b/tensorflow/docs_src/tutorials/kernel_methods.md
index b1f06ce0a3..73e5c51057 100644
--- a/tensorflow/docs_src/tutorials/kernel_methods.md
+++ b/tensorflow/docs_src/tutorials/kernel_methods.md
@@ -1,7 +1,7 @@
 # Improving Linear Models Using Explicit Kernel Methods
 
 Note: This document uses a deprecated version of @{tf.estimator},
-which has a different interface (see `tf.contrib.learn Estimator`).
+which has a @{tf.contrib.learn.Estimator$different interface}.
 It also uses other `contrib` methods whose
 @{$version_compat#not_covered$API may not be stable}.
 
@@ -53,7 +53,7 @@ In order to feed data to a `tf.contrib.learn Estimator`, it is helpful to conver
 it to Tensors. For this, we will use an `input function` which adds Ops to the
 TensorFlow graph that, when executed, create mini-batches of Tensors to be used
 downstream. For more background on input functions, check
-@{$get_started/premade_estimators#input_fn$this section on input functions}.
+@{$get_started/premade_estimators#create_input_functions$this section on input functions}.
 In this example, we will use the `tf.train.shuffle_batch` Op which, besides
 converting numpy arrays to Tensors, allows us to specify the batch_size and
 whether to randomize the input every time the input_fn Ops are executed
diff --git a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
index 7584a76ba5..5d83fbe2a3 100644
--- a/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
+++ b/tensorflow/docs_src/tutorials/recurrent_quickdraw.md
@@ -38,8 +38,8 @@ To try the code for this tutorial:
 1.  [Download the data](#download-the-data) in `TFRecord` format from
     [here](http://download.tensorflow.org/data/quickdraw_tutorial_dataset_v1.tar.gz) and unzip it. More details about [how to
     obtain the original Quick, Draw!
-    data](#optional-download-the-full-quick-draw-data) and [how to convert that
-    to `TFRecord` files](#optional-converting-the-data) is available below.
+    data](#optional_download_the_full_quick_draw_data) and [how to convert that
+    to `TFRecord` files](#optional_converting_the_data) is available below.
 
 1.  Execute the tutorial code with the following command to train the RNN-based
     model described in this tutorial. Make sure to adjust the paths to point to
@@ -108,7 +108,7 @@ This download will take a while and download a bit more than 23GB of data.
 ### Optional: Converting the data
 
 To convert the `ndjson` files to
-@{$python/python_io#tfrecords_format_details$TFRecord} files containing
+@{$python/python_io#TFRecords_Format_Details$TFRecord} files containing
 [`tf.train.Example`](https://www.tensorflow.org/code/tensorflow/core/example/example.proto)
 protos run the following command.
 
@@ -118,7 +118,7 @@ protos run the following command.
 ```
 
 This will store the data in 10 shards of
-@{$python/python_io#tfrecords_format_details$TFRecord} files with 10000 items
+@{$python/python_io#TFRecords_Format_Details$TFRecord} files with 10000 items
 per class for the training data and 1000 items per class as eval data.
 
 This conversion process is described in more detail in the following.
-- 
GitLab


From f83711104b64a108ac43213c92f13827343d09ef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 21 Mar 2018 23:11:40 -0700
Subject: [PATCH 336/960] Automated g4 rollback of changelist 190001737

PiperOrigin-RevId: 190021164
---
 tensorflow/c/eager/BUILD                      |   2 -
 tensorflow/c/eager/c_api.cc                   | 179 ++++++++++------
 tensorflow/c/eager/c_api_internal.h           |  84 +++++++-
 tensorflow/core/common_runtime/eager/BUILD    |  22 --
 .../core/common_runtime/eager/context.cc      | 142 -------------
 .../core/common_runtime/eager/context.h       | 193 ------------------
 6 files changed, 194 insertions(+), 428 deletions(-)
 delete mode 100644 tensorflow/core/common_runtime/eager/context.cc
 delete mode 100644 tensorflow/core/common_runtime/eager/context.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index bea5a121b3..841ff48a38 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -28,7 +28,6 @@ tf_cuda_library(
             "//tensorflow/c:c_api",
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
-            "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core:core_cpu_internal",
@@ -65,7 +64,6 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
-        "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
     ],
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 5d668848ab..a23015c99e 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -71,6 +71,18 @@ std::atomic_int_fast64_t func_id_generator(0);
 
 }  // namespace
 
+TFE_ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy) {
+  if (!soft_placement) {
+    return original_policy;
+  }
+  if (original_policy == TFE_DEVICE_PLACEMENT_EXPLICIT ||
+      original_policy == TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32) {
+    return TFE_DEVICE_PLACEMENT_SILENT;
+  }
+  return original_policy;
+}
+
 extern "C" {
 
 TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; }
@@ -92,7 +104,19 @@ void TFE_ContextOptionsSetDevicePlacementPolicy(
 TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
                                                         unsigned char async,
                                                         TF_Status* status) {
-  status->status = ctx->context.SetAsyncForThread(async);
+  {
+    tensorflow::mutex_lock l(ctx->async_map_mu);
+    ctx->thread_local_async[std::this_thread::get_id()] = async;
+  }
+  if (async) {
+    ctx->executor.EnableAsync();
+  } else {
+    // TODO(agarwal): Currently we add a wait here to handle cases where a sync
+    // op has a control dependency on an async op, and the latter has not
+    // executed yet. This wait can be removed by storing all the control inputs
+    // and waiting for them when executing ops.
+    status->status = ctx->executor.WaitForAllPendingNodes();
+  }
 }
 
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
@@ -109,26 +133,34 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
       new tensorflow::DeviceMgr(devices));
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
-  return new TFE_Context(opts->session_options.options, opts->policy,
-                         opts->async, std::move(device_mgr), r);
+  return new TFE_Context(*opts, std::move(device_mgr), r);
 }
 
 void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status) {
+  status->status = ctx->executor.WaitForAllPendingNodes();
+  {
+    tensorflow::mutex_lock ml(ctx->cache_mu);
+    tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
+  }
+  ctx->rendezvous->Unref();
   delete ctx;
 }
 
 TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx, TF_Status* status) {
   TF_DeviceList* list = new TF_DeviceList;
-  ctx->context.device_mgr()->ListDeviceAttributes(&list->response);
+  ctx->device_manager->ListDeviceAttributes(&list->response);
   return list;
 }
 
-void TFE_ContextClearCaches(TFE_Context* ctx) { ctx->context.ClearCaches(); }
+void TFE_ContextClearCaches(TFE_Context* ctx) {
+  tensorflow::mutex_lock ml(ctx->cache_mu);
+  tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
+}
 
 void TFE_ContextSetThreadLocalDevicePlacementPolicy(
     TFE_Context* ctx, TFE_ContextDevicePlacementPolicy policy) {
-  ctx->context.SetThreadLocalDevicePlacementPolicy(
-      static_cast<tensorflow::ContextDevicePlacementPolicy>(policy));
+  tensorflow::mutex_lock ml(ctx->policy_map_mu);
+  ctx->thread_local_policies[std::this_thread::get_id()] = policy;
 }
 
 // Note: this function looks up a thread local policy. So it should be called in
@@ -136,20 +168,25 @@ void TFE_ContextSetThreadLocalDevicePlacementPolicy(
 // safe to call this function from the async EagerExecutor threads.
 extern TFE_ContextDevicePlacementPolicy TFE_ContextGetDevicePlacementPolicy(
     TFE_Context* ctx) {
-  return static_cast<TFE_ContextDevicePlacementPolicy>(
-      ctx->context.GetDevicePlacementPolicy());
+  tensorflow::mutex_lock ml(ctx->policy_map_mu);
+  auto policy_map_it =
+      ctx->thread_local_policies.find(std::this_thread::get_id());
+  if (policy_map_it != ctx->thread_local_policies.end()) {
+    return policy_map_it->second;
+  }
+  return ctx->policy;
 }
 
 void TFE_ContextAsyncWait(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->context.AsyncWait();
+  status->status = ctx->executor.WaitForAllPendingNodes();
 }
 
 void TFE_ContextGetStatus(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->context.GetStatus();
+  status->status = ctx->executor.status();
 }
 
 void TFE_ContextAsyncClearError(TFE_Context* ctx) {
-  ctx->context.ClearAsyncError();
+  ctx->executor.ClearError();
 }
 
 TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
@@ -222,7 +259,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   // nullptr.
   tensorflow::Device* src_opd = nullptr;
   TF_RETURN_IF_ERROR(h->TensorAndDevice(&src, &srcd, &src_opd));
-  if (srcd == nullptr) srcd = ctx->context.HostCPU();
+  if (srcd == nullptr) srcd = ctx->devices[0];
   bool is_same_device =
       (srcd == dstd) || (DeviceName(srcd) == DeviceName(dstd));
   const bool dst_cpu = IsCPU(dstd);
@@ -295,7 +332,8 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name,
   status->status = tensorflow::AttrTypeMapForOp(name, &types);
   if (status->status.ok()) return new TFE_Op(ctx, name, types);
   if (TF_GetCode(status) == TF_NOT_FOUND) {
-    if (ctx->context.FindFunctionByName(name)) {
+    tensorflow::mutex_lock l(ctx->functions_mu);
+    if (ctx->func_lib_def.Find(name) != nullptr) {
       status->status = tensorflow::Status::OK();
       return new TFE_Op(ctx, name, nullptr);
     }
@@ -308,14 +346,20 @@ void TFE_DeleteOp(TFE_Op* op) { delete op; }
 void TFE_OpSetDevice(TFE_Op* op, const char* device_name, TF_Status* status) {
   tensorflow::Device* d = nullptr;
   if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status = op->ctx->context.FindDeviceByName(device_name, &d);
+    auto it = op->ctx->devices_map.find(device_name);
+    if (it == op->ctx->devices_map.end()) {
+      status->status =
+          tensorflow::errors::InvalidArgument(device_name, " unknown device.");
+      return;
+    }
+    d = it->second;
   }
   op->device = d;
 }
 
 const char* TFE_OpGetDevice(TFE_Op* op, TF_Status* status) {
   tensorflow::Device* device =
-      (op->device == nullptr) ? op->ctx->context.HostCPU() : op->device;
+      (op->device == nullptr) ? op->ctx->devices[0] : op->device;
   return device->name().c_str();
 }
 
@@ -590,7 +634,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
 tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
                                  TFE_Context* ctx, TF_Status* status) {
   tensorflow::DeviceSet ds;
-  for (tensorflow::Device* d : *ctx->context.devices()) {
+  for (tensorflow::Device* d : ctx->devices) {
     ds.AddDevice(d);
   }
   tensorflow::DeviceTypeVector final_devices;
@@ -604,7 +648,7 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
         "Could not find valid device for node ", ndef.DebugString());
     return nullptr;
   }
-  for (tensorflow::Device* d : *ctx->context.devices()) {
+  for (tensorflow::Device* d : ctx->devices) {
     if (d->device_type() == final_devices[0].type_string()) {
       return d;
     }
@@ -619,8 +663,9 @@ tensorflow::Status Execute(
     const tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4>& op_inputs,
     tensorflow::KernelAndDevice* kernel, tensorflow::NodeExecStats* maybe_stats,
     TFE_TensorHandle** retvals, int num_retvals) {
-  if (!ctx->context.SoftPlacement() && device == nullptr) {
-    device = ctx->context.HostCPU();
+  if (!ctx->soft_placement && device == nullptr) {
+    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
+    device = ctx->devices[0];
   }
 
   if (device == nullptr) {
@@ -652,18 +697,18 @@ tensorflow::Status Execute(
   if (maybe_stats != nullptr) {
     maybe_stats->set_op_end_rel_micros(tensorflow::Env::Default()->NowMicros() -
                                        maybe_stats->all_start_micros());
-    tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
-    if (ctx->context.ShouldStoreMetadata()) {
-      auto* step_stats = ctx->context.RunMetadataProto()->mutable_step_stats();
+    tensorflow::mutex_lock ml(ctx->metadata_mu);
+    if (ctx->should_store_metadata.load()) {
+      auto* step_stats = ctx->run_metadata.mutable_step_stats();
       // Lazily initialize the RunMetadata with information about all devices if
       // this is the first call.
-      while (step_stats->dev_stats_size() < ctx->context.devices()->size()) {
+      while (step_stats->dev_stats_size() < ctx->devices.size()) {
         step_stats->add_dev_stats();
       }
       // Find the current device's index.
       int device_idx = 0;
-      for (int i = 0; i < ctx->context.devices()->size(); ++i) {
-        if (ctx->context.devices()->at(i) == device) {
+      for (int i = 0; i < ctx->devices.size(); ++i) {
+        if (ctx->devices[i] == device) {
           device_idx = i;
           break;
         }
@@ -699,7 +744,7 @@ class ExecuteNode : public tensorflow::EagerNode {
               tensorflow::NodeExecStats* maybe_stats,
               const tensorflow::DataTypeVector& output_dtypes,
               TFE_TensorHandle** retvals, int num_retvals)
-      : tensorflow::EagerNode(op->ctx->context.NextId()),
+      : tensorflow::EagerNode(op->ctx->executor.NextId()),
         ctx_(op->ctx),
         op_device_(op->device),
         inputs_(op->inputs),
@@ -755,7 +800,7 @@ class CopyToDeviceNode : public tensorflow::EagerNode {
  public:
   CopyToDeviceNode(TFE_TensorHandle* src, tensorflow::Device* dstd,
                    TFE_Context* ctx)
-      : tensorflow::EagerNode(ctx->context.NextId()),
+      : tensorflow::EagerNode(ctx->executor.NextId()),
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
@@ -1018,7 +1063,7 @@ extern "C" {
 void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
                  TF_Status* status) {
   TFE_Context* ctx = op->ctx;
-  status->status = ctx->context.GetStatus();
+  status->status = ctx->executor.status();
   if (!status->status.ok()) {
     return;
   }
@@ -1042,7 +1087,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (op->inputs[i]->dtype == tensorflow::DT_RESOURCE &&
         input_op_device != op->device) {
       tensorflow::Device* d =
-          input_op_device == nullptr ? ctx->context.HostCPU() : input_op_device;
+          input_op_device == nullptr ? ctx->devices[0] : input_op_device;
       VLOG(1) << "Changing device of operation " << op->name << " to "
               << d->name() << " because input #" << i
               << " is a resource in this device.";
@@ -1050,35 +1095,40 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     }
   }
   tensorflow::Device* device = op->device;
-  if (!ctx->context.SoftPlacement() && device == nullptr) {
-    device = ctx->context.HostCPU();
+  if (!ctx->soft_placement && device == nullptr) {
+    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
+    device = ctx->devices[0];
   }
 
   tensorflow::Fprint128 cache_key =
       op->attrs.CacheKey(device == nullptr ? "unspecified" : device->name());
-  tensorflow::KernelAndDevice* kernel = ctx->context.GetCachedKernel(cache_key);
+  tensorflow::KernelAndDevice* kernel;
+  {
+    tensorflow::tf_shared_lock l(ctx->cache_mu);
+    kernel = tensorflow::gtl::FindPtrOrNull(ctx->kernel_cache, cache_key);
+  }
   if (kernel == nullptr) {
     const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef();
-    if (ctx->context.SoftPlacement() && device == nullptr) {
+    if (ctx->soft_placement && device == nullptr) {
       device = SelectDevice(ndef, ctx, status);
       if (!status->status.ok()) {
         return;
       }
     }
     CHECK(device != nullptr);
-    if (ctx->context.LogDevicePlacement()) {
+    if (ctx->log_device_placement) {
       LOG(INFO) << "Executing op " << ndef.op() << " in device "
                 << device->name();
     }
-    kernel = new tensorflow::KernelAndDevice(ctx->context.GetRendezvous());
+    kernel = new tensorflow::KernelAndDevice(ctx->rendezvous);
     // Knowledge of the implementation of Init (and in-turn
     // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def
     // will be accessed, so grab on to the lock.
     // See WARNING comment in Execute (before kernel->Run) - would be nice to
     // rework to avoid this subtlety.
-    tensorflow::tf_shared_lock l(*ctx->context.FunctionsMu());
-    status->status = tensorflow::KernelAndDevice::Init(
-        ndef, ctx->context.func_lib(device), kernel);
+    tensorflow::tf_shared_lock l(ctx->functions_mu);
+    status->status =
+        tensorflow::KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel);
     if (!status->status.ok()) {
       delete kernel;
       return;
@@ -1086,7 +1136,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // Update output_dtypes inside `kernel`.
     const tensorflow::OpDef* op_def = nullptr;
     const tensorflow::FunctionDef* function_def =
-        ctx->context.FuncLibDef()->Find(ndef.op());
+        ctx->func_lib_def.Find(ndef.op());
     if (function_def != nullptr) {
       op_def = &(function_def->signature());
     }
@@ -1102,7 +1152,8 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (!status->status.ok()) {
       return;
     }
-    ctx->context.AddKernelToCache(cache_key, kernel);
+    tensorflow::mutex_lock ml(ctx->cache_mu);
+    tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel);
   }
   const tensorflow::DataTypeVector& output_dtypes = kernel->output_dtypes();
   const int output_dtypes_size = output_dtypes.size();
@@ -1120,11 +1171,11 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // device from the one requested above.
     device = kernel->device();
   }
-  status->status = ValidateInputTypeAndPlacement(ctx, ctx->context.HostCPU(),
-                                                 device, op, kernel->kernel());
+  status->status = ValidateInputTypeAndPlacement(ctx, ctx->devices[0], device,
+                                                 op, kernel->kernel());
   if (!status->status.ok()) return;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats;
-  if (ctx->context.ShouldStoreMetadata()) {
+  if (ctx->should_store_metadata.load()) {
     maybe_stats.reset(new tensorflow::NodeExecStats);
     maybe_stats->set_node_name(op->name);
     maybe_stats->set_all_start_micros(tensorflow::Env::Default()->NowMicros());
@@ -1132,14 +1183,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     maybe_stats->set_scheduled_micros(tensorflow::Env::Default()->NowMicros());
     // TODO(apassos) track referenced tensors
   }
-  if (ctx->context.Async()) {
+  if (ctx->Async()) {
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
     // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
     tensorflow::EagerNode* node =
         new ExecuteNode(op, kernel, maybe_stats.release(), output_dtypes,
                         retvals, *num_retvals);
-    ctx->context.ExecutorAdd(node);
+    ctx->executor.Add(node);
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
@@ -1155,24 +1206,23 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
                                                TFE_Context* ctx,
                                                const char* device_name,
                                                TF_Status* status) {
-  status->status = ctx->context.GetStatus();
+  status->status = ctx->executor.status();
   if (!status->status.ok()) {
     return nullptr;
   }
-  tensorflow::Device* dstd = ctx->context.HostCPU();
+  tensorflow::Device* dstd = ctx->devices[0];
   if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status =
-        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
+    status->status = ctx->device_manager->LookupDevice(device_name, &dstd);
     if (!status->status.ok()) return nullptr;
   }
-  if (ctx->context.Async()) {
+  if (ctx->Async()) {
     // Note that `h` may not be currently ready. However execution order will
     // make sure that `h` is ready before the copy is actually done.
     CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
     TFE_TensorHandle* output = node->dst();
     // Note that calling Add makes `node` accessible by the EagerExecutor
     // thread. So further accesses need to be thread-safe.
-    ctx->context.ExecutorAdd(node);
+    ctx->executor.Add(node);
     return output;
   } else {
     TFE_TensorHandle* output = nullptr;
@@ -1190,20 +1240,24 @@ void TFE_ContextAddFunctionDef(TFE_Context* ctx,
         tensorflow::errors::InvalidArgument("Invalid FunctionDef proto");
     return;
   }
-  status->status = ctx->context.AddFunctionDef(function_def);
+  tensorflow::mutex_lock l(ctx->functions_mu);
+  status->status = ctx->func_lib_def.AddFunctionDef(function_def);
 }
 
 void TFE_ContextAddFunction(TFE_Context* ctx, TF_Function* function,
                             TF_Status* status) {
-  status->status = ctx->context.AddFunctionDef(function->fdef);
+  tensorflow::mutex_lock l(ctx->functions_mu);
+  status->status = ctx->func_lib_def.AddFunctionDef(function->fdef);
 }
 
 void TFE_ContextEnableRunMetadata(TFE_Context* ctx) {
-  ctx->context.SetShouldStoreMetadata(true);
+  ctx->should_store_metadata.store(true);
 }
 
 void TFE_ContextDisableRunMetadata(TFE_Context* ctx) {
-  ctx->context.SetShouldStoreMetadata(false);
+  tensorflow::mutex_lock ml(ctx->metadata_mu);
+  ctx->should_store_metadata.store(false);
+  ctx->run_metadata.Clear();
 }
 
 }  // extern "C"
@@ -1232,9 +1286,9 @@ void TFE_ContextExportRunMetadata(TFE_Context* ctx, TF_Buffer* buf,
                                   TF_Status* status) {
   TFE_ContextAsyncWait(ctx, status);
   if (!status->status.ok()) return;
-  tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
-  status->status = MessageToBuffer(*ctx->context.RunMetadataProto(), buf);
-  ctx->context.RunMetadataProto()->Clear();
+  tensorflow::mutex_lock ml(ctx->metadata_mu);
+  status->status = MessageToBuffer(ctx->run_metadata, buf);
+  ctx->run_metadata.Clear();
 }
 
 namespace {
@@ -1309,6 +1363,11 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
 }  // namespace tensorflow
 
 
+bool TFE_Context::Async() const {
+  tensorflow::mutex_lock l(async_map_mu);
+  return tensorflow::gtl::FindWithDefault(
+      thread_local_async, std::this_thread::get_id(), async_default);
+}
 
 bool TFE_TensorHandle::IsReady() {
   if (node_id == 0) return true;
@@ -1322,7 +1381,7 @@ tensorflow::Status TFE_TensorHandle::WaitReady() {
   {
     tensorflow::mutex_lock l(ctx_mutex_);
     if (ctx_ == nullptr) return tensorflow::Status::OK();
-    executor = ctx_->context.Executor();
+    executor = &ctx_->executor;
   }
   return executor->WaitFor(node_id);
 }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 5b29120b40..a79f8ddd33 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -30,7 +30,6 @@ limitations under the License.
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
@@ -53,18 +52,85 @@ struct TFE_ContextOptions {
       TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32};
 };
 
+TFE_ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy);
+
 struct TFE_Context {
-  explicit TFE_Context(const tensorflow::SessionOptions& opts,
-                       TFE_ContextDevicePlacementPolicy default_policy,
-                       bool async,
+  explicit TFE_Context(const TFE_ContextOptions& opts,
                        std::unique_ptr<tensorflow::DeviceMgr> device_mgr,
                        tensorflow::Rendezvous* rendezvous)
-      : context(opts,
-                static_cast<tensorflow::ContextDevicePlacementPolicy>(
-                    default_policy),
-                async, std::move(device_mgr), rendezvous) {}
+      : soft_placement(
+            opts.session_options.options.config.allow_soft_placement()),
+        policy(PlacementPolicy(soft_placement, opts.policy)),
+        device_manager(std::move(device_mgr)),
+        devices(device_manager->ListDevices()),
+        rendezvous(rendezvous),
+        pflr(new tensorflow::ProcessFunctionLibraryRuntime(
+            device_manager.get(), opts.session_options.options.env,
+            TF_GRAPH_DEF_VERSION, &func_lib_def, {})),
+        log_device_placement(
+            opts.session_options.options.config.log_device_placement()),
+        async_default(opts.async) {
+    if (async_default) executor.EnableAsync();
+
+    for (auto* device : devices) {
+      devices_map[tensorflow::StringPiece(device->name())] = device;
+    }
+  }
+
+  const bool soft_placement;
+  const TFE_ContextDevicePlacementPolicy policy;
+
+  // Note: we cannot use C++11 thread_local here as there is no concept of a
+  // thread-local-object-local variable in C++11.
+  tensorflow::mutex policy_map_mu;
+  std::unordered_map<std::thread::id, TFE_ContextDevicePlacementPolicy>
+      thread_local_policies GUARDED_BY(policy_map_mu);
+
+  std::unique_ptr<tensorflow::DeviceMgr> device_manager;
+  // Devices owned by device_manager
+  std::vector<tensorflow::Device*> devices;
+  // All devices are not owned.
+  tensorflow::gtl::FlatMap<tensorflow::StringPiece, tensorflow::Device*,
+                           tensorflow::StringPieceHasher>
+      devices_map;
+  tensorflow::Rendezvous* const rendezvous;
+
+  tensorflow::mutex functions_mu;
+  tensorflow::FunctionLibraryDefinition func_lib_def GUARDED_BY(functions_mu){
+      tensorflow::OpRegistry::Global(), {}};
+
+  // One FunctionLibraryRuntime per device.
+  // func_libs[i] is the FunctionLibraryRuntime corresponding to
+  // session->devices[i].
+  const std::unique_ptr<tensorflow::ProcessFunctionLibraryRuntime> pflr;
+
+  tensorflow::mutex cache_mu;
+  std::unordered_map<tensorflow::Fprint128, tensorflow::KernelAndDevice*,
+                     tensorflow::Fprint128Hasher>
+      kernel_cache GUARDED_BY(cache_mu);
+
+  tensorflow::FunctionLibraryRuntime* func_lib(tensorflow::Device* d) const {
+    return pflr->GetFLR(d->name());
+  }
 
-  tensorflow::EagerContext context;
+  // Whether we should compute RunMetadata.
+  std::atomic<bool> should_store_metadata{false};
+  tensorflow::mutex metadata_mu;
+  tensorflow::RunMetadata run_metadata GUARDED_BY(metadata_mu);
+  const bool log_device_placement;
+  // EagerExecutor for async execution.
+  tensorflow::EagerExecutor executor;
+
+  // True if running in asynchronous mode.
+  bool Async() const;
+
+  // True if the default value for execution mode is async. Note that this value
+  // can be overridden per thread based on `thread_local_async` overrides.
+  const bool async_default;
+  mutable tensorflow::mutex async_map_mu;
+  std::unordered_map<std::thread::id, bool> thread_local_async
+      GUARDED_BY(async_map_mu);
 };
 
 struct TFE_TensorHandle : public tensorflow::core::RefCounted {
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index de10b10b7e..8ba560bef8 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -32,28 +32,6 @@ tf_cuda_library(
     ],
 )
 
-tf_cuda_library(
-    name = "context",
-    srcs = [
-        "context.cc",
-    ],
-    hdrs = [
-        "context.h",
-    ],
-    visibility = ["//tensorflow:internal"],
-    deps = [
-        ":eager_executor",
-        ":kernel_and_device",
-        "//tensorflow/core:core_cpu_lib",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:framework_internal",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:protos_all_cc",
-        "//tensorflow/core:session_options",
-    ],
-)
-
 tf_cuda_library(
     name = "kernel_and_device",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
deleted file mode 100644
index 5e8d083cd2..0000000000
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ /dev/null
@@ -1,142 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/common_runtime/eager/context.h"
-
-namespace tensorflow {
-
-ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, ContextDevicePlacementPolicy original_policy) {
-  if (!soft_placement) {
-    return original_policy;
-  }
-  if (original_policy == DEVICE_PLACEMENT_EXPLICIT ||
-      original_policy == DEVICE_PLACEMENT_SILENT_FOR_INT32) {
-    return DEVICE_PLACEMENT_SILENT;
-  }
-  return original_policy;
-}
-
-EagerContext::EagerContext(const SessionOptions& opts,
-                           ContextDevicePlacementPolicy default_policy,
-                           bool async, std::unique_ptr<DeviceMgr> device_mgr,
-                           Rendezvous* rendezvous)
-    : soft_placement_(opts.config.allow_soft_placement()),
-      policy_(PlacementPolicy(soft_placement_, default_policy)),
-      device_manager_(std::move(device_mgr)),
-      devices_(device_manager_->ListDevices()),
-      rendezvous_(rendezvous),
-      pflr_(new ProcessFunctionLibraryRuntime(device_manager_.get(), opts.env,
-                                              TF_GRAPH_DEF_VERSION,
-                                              &func_lib_def_, {})),
-      log_device_placement_(opts.config.log_device_placement()),
-      async_default_(async) {
-  if (async_default_) {
-    executor_.EnableAsync();
-  }
-
-  for (auto* device : devices_) {
-    devices_map_[device->name()] = device;
-  }
-}
-
-bool EagerContext::Async() const {
-  mutex_lock l(async_map_mu_);
-  return gtl::FindWithDefault(thread_local_async_, std::this_thread::get_id(),
-                              async_default_);
-}
-
-Status EagerContext::SetAsyncForThread(bool async) {
-  {
-    tensorflow::mutex_lock l(async_map_mu_);
-    thread_local_async_[std::this_thread::get_id()] = async;
-  }
-  if (async) {
-    executor_.EnableAsync();
-  } else {
-    // TODO(agarwal): Currently we add a wait here to handle cases where a
-    // sync op has a control dependency on an async op, and the latter has not
-    // executed yet. This wait can be removed by storing all the control
-    // inputs and waiting for them when executing ops.
-    return executor_.WaitForAllPendingNodes();
-  }
-  return Status::OK();
-}
-
-void EagerContext::ClearCaches() {
-  mutex_lock ml(cache_mu_);
-  gtl::STLDeleteValues(&kernel_cache_);
-}
-
-void EagerContext::SetThreadLocalDevicePlacementPolicy(
-    ContextDevicePlacementPolicy policy) {
-  mutex_lock ml(policy_map_mu_);
-  thread_local_policies_[std::this_thread::get_id()] = policy;
-}
-
-ContextDevicePlacementPolicy EagerContext::GetDevicePlacementPolicy() {
-  mutex_lock ml(policy_map_mu_);
-  auto policy_map_it = thread_local_policies_.find(std::this_thread::get_id());
-  if (policy_map_it != thread_local_policies_.end()) {
-    return policy_map_it->second;
-  }
-  return policy_;
-}
-
-EagerContext::~EagerContext() {
-  executor_.WaitForAllPendingNodes().IgnoreError();
-  ClearCaches();
-  rendezvous_->Unref();
-}
-
-bool EagerContext::FindFunctionByName(const string& name) {
-  mutex_lock l(functions_mu_);
-  return func_lib_def_.Find(name) != nullptr;
-}
-
-Status EagerContext::FindDeviceByName(const string& name, Device** result) {
-  auto it = devices_map_.find(name);
-  if (it == devices_map_.end()) {
-    return errors::InvalidArgument(name, " unknown device.");
-  }
-  *result = it->second;
-  return Status::OK();
-}
-
-Status EagerContext::AddFunctionDef(const FunctionDef& fdef) {
-  mutex_lock l(functions_mu_);
-  return func_lib_def_.AddFunctionDef(fdef);
-}
-
-KernelAndDevice* EagerContext::GetCachedKernel(Fprint128 cache_key) {
-  tf_shared_lock l(cache_mu_);
-  return gtl::FindPtrOrNull(kernel_cache_, cache_key);
-}
-
-void EagerContext::AddKernelToCache(Fprint128 cache_key,
-                                    KernelAndDevice* kernel) {
-  mutex_lock ml(cache_mu_);
-  gtl::InsertOrUpdate(&kernel_cache_, cache_key, kernel);
-}
-
-void EagerContext::SetShouldStoreMetadata(bool value) {
-  should_store_metadata_.store(value);
-  if (!value) {
-    mutex_lock ml(metadata_mu_);
-    run_metadata_.Clear();
-  }
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
deleted file mode 100644
index d525d44fe4..0000000000
--- a/tensorflow/core/common_runtime/eager/context.h
+++ /dev/null
@@ -1,193 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
-#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
-
-#include <algorithm>
-#include <cstddef>
-#include <map>
-#include <memory>
-#include <queue>
-#include <string>
-#include <vector>
-
-#include "tensorflow/core/common_runtime/device_factory.h"
-#include "tensorflow/core/common_runtime/eager/eager_executor.h"
-#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
-#include "tensorflow/core/common_runtime/function.h"
-#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
-#include "tensorflow/core/framework/rendezvous.h"
-#include "tensorflow/core/lib/core/stringpiece.h"
-#include "tensorflow/core/lib/gtl/inlined_vector.h"
-#include "tensorflow/core/lib/gtl/map_util.h"
-#include "tensorflow/core/lib/gtl/stl_util.h"
-#include "tensorflow/core/platform/fingerprint.h"
-#include "tensorflow/core/platform/mutex.h"
-#include "tensorflow/core/platform/thread_annotations.h"
-#include "tensorflow/core/public/session_options.h"
-#include "tensorflow/core/public/version.h"
-
-namespace tensorflow {
-
-// Note: there's a copy enum in eager/c_api.h. It should be kept in sync.
-enum ContextDevicePlacementPolicy {
-  // Running operations with input tensors on the wrong device will fail. When
-  // soft placement is enabled acts like TFE_DEVICE_PLACEMENT_SILENT.
-  DEVICE_PLACEMENT_EXPLICIT = 0,
-  // Copy the tensor to the right device but log a warning.
-  DEVICE_PLACEMENT_WARN = 1,
-  // Silently copy the tensor, which has a performance cost since the
-  // operation will be blocked till the copy completes.
-  DEVICE_PLACEMENT_SILENT = 2,
-  // Default placement policy which silently copies int32 tensors but not other
-  // dtypes.  When soft placement is enabled acts like
-  // TFE_DEVICE_PLACEMENT_SILENT.
-  DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
-};
-
-ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, ContextDevicePlacementPolicy original_policy);
-
-class EagerContext {
- public:
-  explicit EagerContext(const SessionOptions& opts,
-                        ContextDevicePlacementPolicy default_policy, bool async,
-                        std::unique_ptr<DeviceMgr> device_mgr,
-                        Rendezvous* rendezvous);
-
-  ~EagerContext();
-
-  // Returns the function library runtime for the given device.
-  FunctionLibraryRuntime* func_lib(Device* d) const {
-    return pflr_->GetFLR(d->name());
-  }
-
-  // True if running in asynchronous mode.
-  bool Async() const;
-
-  EagerExecutor* Executor() { return &executor_; }
-
-  // Sets whether this thread should run in synchronous or asynchronous mode.
-  Status SetAsyncForThread(bool async);
-
-  // TODO(apassos) make this return a constant reference
-  gtl::FlatMap<string, Device*, StringPieceHasher>* device_map() {
-    return &devices_map_;
-  }
-
-  // TODO(apassos) make this return a constant reference
-  std::vector<Device*>* devices() { return &devices_; }
-
-  // Clears the kernel caches.
-  void ClearCaches();
-
-  // Sets the device placement policy for the current thread.
-  void SetThreadLocalDevicePlacementPolicy(ContextDevicePlacementPolicy policy);
-
-  // Returns the device placement policy for the current thread.
-  ContextDevicePlacementPolicy GetDevicePlacementPolicy();
-
-  Status AsyncWait() { return executor_.WaitForAllPendingNodes(); }
-
-  Status GetStatus() { return executor_.status(); }
-
-  void ClearAsyncError() { executor_.ClearError(); }
-
-  bool FindFunctionByName(const string& name);
-
-  Status FindDeviceByName(const string& name, Device** result);
-
-  Device* HostCPU() { return devices_[0]; }
-
-  bool SoftPlacement() { return soft_placement_; }
-
-  uint64 NextId() { return executor_.NextId(); }
-
-  void ExecutorAdd(EagerNode* node) { executor_.Add(node); }
-
-  Status AddFunctionDef(const FunctionDef& fdef);
-
-  KernelAndDevice* GetCachedKernel(Fprint128 cache_key);
-
-  void AddKernelToCache(Fprint128 cache_key, KernelAndDevice* kernel);
-
-  bool LogDevicePlacement() { return log_device_placement_; }
-
-  Rendezvous* GetRendezvous() { return rendezvous_; }
-
-  mutex* FunctionsMu() { return &functions_mu_; }
-
-  tensorflow::DeviceMgr* device_mgr() { return device_manager_.get(); }
-
-  // TODO(apassos) remove the need for this
-  void ReleaseDeviceMgr() { device_manager_.release(); }
-
-  // TODO(apassos) clean up RunMetadata storage.
-  mutex* MetadataMu() { return &metadata_mu_; }
-  bool ShouldStoreMetadata() { return should_store_metadata_.load(); }
-  void SetShouldStoreMetadata(bool value);
-  RunMetadata* RunMetadataProto() { return &run_metadata_; }
-
-  FunctionLibraryDefinition* FuncLibDef() { return &func_lib_def_; }
-
- private:
-  const bool soft_placement_;
-  const ContextDevicePlacementPolicy policy_;
-
-  // Note: we cannot use C++11 thread_local here as there is no concept of a
-  // thread-local-object-local variable in C++11.
-  mutex policy_map_mu_;
-  std::unordered_map<std::thread::id, ContextDevicePlacementPolicy>
-      thread_local_policies_ GUARDED_BY(policy_map_mu_);
-
-  std::unique_ptr<DeviceMgr> device_manager_;
-  // Devices owned by device_manager
-  std::vector<Device*> devices_;
-  // All devices are not owned.
-  gtl::FlatMap<string, Device*, StringPieceHasher> devices_map_;
-  Rendezvous* const rendezvous_;
-
-  mutex functions_mu_;
-  FunctionLibraryDefinition func_lib_def_ GUARDED_BY(functions_mu_){
-      OpRegistry::Global(), {}};
-
-  // One FunctionLibraryRuntime per device.
-  // func_libs[i] is the FunctionLibraryRuntime corresponding to
-  // session->devices[i].
-  const std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
-
-  mutex cache_mu_;
-  std::unordered_map<Fprint128, KernelAndDevice*, Fprint128Hasher> kernel_cache_
-      GUARDED_BY(cache_mu_);
-
-  // Whether we should compute RunMetadata.
-  std::atomic<bool> should_store_metadata_{false};
-  mutex metadata_mu_;
-  RunMetadata run_metadata_ GUARDED_BY(metadata_mu_);
-  const bool log_device_placement_;
-  // EagerExecutor for async execution.
-  EagerExecutor executor_;
-
-  // True if the default value for execution mode is async. Note that this value
-  // can be overridden per thread based on `thread_local_async` overrides.
-  const bool async_default_;
-  mutable mutex async_map_mu_;
-  std::unordered_map<std::thread::id, bool> thread_local_async_
-      GUARDED_BY(async_map_mu_);
-};
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
-- 
GitLab


From 585fb74541ed914845eccd3da4b1a2c94a99779e Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 22 Mar 2018 00:26:31 -0700
Subject: [PATCH 337/960] Minor style improvement in TFLite interpreter_test.py

PiperOrigin-RevId: 190027161
---
 .../contrib/lite/python/interpreter_test.py   | 49 ++++++++++---------
 1 file changed, 25 insertions(+), 24 deletions(-)

diff --git a/tensorflow/contrib/lite/python/interpreter_test.py b/tensorflow/contrib/lite/python/interpreter_test.py
index bf124410f3..cd2386f526 100644
--- a/tensorflow/contrib/lite/python/interpreter_test.py
+++ b/tensorflow/contrib/lite/python/interpreter_test.py
@@ -61,30 +61,31 @@ class InterpreterTest(test_util.TensorFlowTestCase):
         'testdata/permute_uint8.tflite')
     with io.open(model_path, 'rb') as model_file:
       data = model_file.read()
-      interpreter = interpreter_wrapper.Interpreter(model_content=data)
-      interpreter.allocate_tensors()
-
-      input_details = interpreter.get_input_details()
-      self.assertEqual(1, len(input_details))
-      self.assertEqual('input', input_details[0]['name'])
-      self.assertEqual(np.uint8, input_details[0]['dtype'])
-      self.assertTrue(([1, 4] == input_details[0]['shape']).all())
-      self.assertEqual((1.0, 0), input_details[0]['quantization'])
-
-      output_details = interpreter.get_output_details()
-      self.assertEqual(1, len(output_details))
-      self.assertEqual('output', output_details[0]['name'])
-      self.assertEqual(np.uint8, output_details[0]['dtype'])
-      self.assertTrue(([1, 4] == output_details[0]['shape']).all())
-      self.assertEqual((1.0, 0), output_details[0]['quantization'])
-
-      test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
-      expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
-      interpreter.set_tensor(input_details[0]['index'], test_input)
-      interpreter.invoke()
-
-      output_data = interpreter.get_tensor(output_details[0]['index'])
-      self.assertTrue((expected_output == output_data).all())
+
+    interpreter = interpreter_wrapper.Interpreter(model_content=data)
+    interpreter.allocate_tensors()
+
+    input_details = interpreter.get_input_details()
+    self.assertEqual(1, len(input_details))
+    self.assertEqual('input', input_details[0]['name'])
+    self.assertEqual(np.uint8, input_details[0]['dtype'])
+    self.assertTrue(([1, 4] == input_details[0]['shape']).all())
+    self.assertEqual((1.0, 0), input_details[0]['quantization'])
+
+    output_details = interpreter.get_output_details()
+    self.assertEqual(1, len(output_details))
+    self.assertEqual('output', output_details[0]['name'])
+    self.assertEqual(np.uint8, output_details[0]['dtype'])
+    self.assertTrue(([1, 4] == output_details[0]['shape']).all())
+    self.assertEqual((1.0, 0), output_details[0]['quantization'])
+
+    test_input = np.array([[1, 2, 3, 4]], dtype=np.uint8)
+    expected_output = np.array([[4, 3, 2, 1]], dtype=np.uint8)
+    interpreter.set_tensor(input_details[0]['index'], test_input)
+    interpreter.invoke()
+
+    output_data = interpreter.get_tensor(output_details[0]['index'])
+    self.assertTrue((expected_output == output_data).all())
 
 
 if __name__ == '__main__':
-- 
GitLab


From 31adaf4361b9b65e382be9633c4d0517d77c29e5 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 22 Mar 2018 00:26:33 -0700
Subject: [PATCH 338/960] TFLite: Ensure only 1 scale/zero_point is in
 QuantizationParam. PiperOrigin-RevId: 190027163

---
 tensorflow/contrib/lite/model.cc | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index f7daa6fc9d..9c619f88e0 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -679,9 +679,27 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
       // but we really only support one value for the whole tensor.
       // TODO(aselle): This breaks as well if these are nullptr's.
       // TODO(aselle): This assumes non per-channel quantization.
-      if (q_params->scale()) quantization.scale = q_params->scale()->Get(0);
-      if (q_params->zero_point())
+
+      if (q_params->scale()) {
+        if (q_params->scale()->size() != 1) {
+          error_reporter_->Report(
+              "QuantizationParam has %d scale values (only 1 is supported).",
+              q_params->scale()->size());
+          return kTfLiteError;
+        }
+        quantization.scale = q_params->scale()->Get(0);
+      }
+
+      if (q_params->zero_point()) {
+        if (q_params->zero_point()->size() != 1) {
+          error_reporter_->Report(
+              "QuantizationParam has %d zero_point values"
+              " (only 1 is supported).",
+              q_params->zero_point()->size());
+          return kTfLiteError;
+        }
         quantization.zero_point = q_params->zero_point()->Get(0);
+      }
     }
 
     TfLiteType type;
-- 
GitLab


From ba97ee847d9baca0ac3b7eab5c6bad93e70a2882 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 22 Mar 2018 00:48:30 -0700
Subject: [PATCH 339/960] Java: Release 1.7.0-rc1

PiperOrigin-RevId: 190028714
---
 tensorflow/java/maven/libtensorflow/pom.xml         | 2 +-
 tensorflow/java/maven/libtensorflow_jni/pom.xml     | 2 +-
 tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml | 2 +-
 tensorflow/java/maven/pom.xml                       | 2 +-
 tensorflow/java/maven/proto/pom.xml                 | 2 +-
 tensorflow/java/maven/tensorflow/pom.xml            | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index 7f3a83b195..0b69a8cbe5 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc0</version>
+    <version>1.7.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index cc436ff840..541876f7f5 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc0</version>
+    <version>1.7.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
index 47f678382a..d8933e5238 100644
--- a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc0</version>
+    <version>1.7.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni_gpu</artifactId>
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index 42d32810a2..6286fd73df 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.tensorflow</groupId>
   <artifactId>parentpom</artifactId>
-  <version>1.7.0-rc0</version>
+  <version>1.7.0-rc1</version>
   <packaging>pom</packaging>
 
   <url>https://www.tensorflow.org</url>
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index 463893ce62..4e881f5a63 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc0</version>
+    <version>1.7.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>proto</artifactId>
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index 60e7f3c199..d512a7eda9 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc0</version>
+    <version>1.7.0-rc1</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>tensorflow</artifactId>
-- 
GitLab


From f9ccb89134d89469ae962bba832e78d1f116b96b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 04:50:42 -0700
Subject: [PATCH 340/960] Add a utility that converts call keyword arguments
 into dicts, in AST space.

PiperOrigin-RevId: 190047495
---
 tensorflow/contrib/py2tf/pyct/ast_util.py      |  9 +++++++++
 tensorflow/contrib/py2tf/pyct/ast_util_test.py | 13 +++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/tensorflow/contrib/py2tf/pyct/ast_util.py b/tensorflow/contrib/py2tf/pyct/ast_util.py
index f916775b9c..6f7e656c26 100644
--- a/tensorflow/contrib/py2tf/pyct/ast_util.py
+++ b/tensorflow/contrib/py2tf/pyct/ast_util.py
@@ -94,3 +94,12 @@ def rename_symbols(node, name_map):
   elif isinstance(node, tuple):
     return tuple(renamer.visit(n) for n in node)
   return renamer.visit(node)
+
+
+def keywords_to_dict(keywords):
+  keys = []
+  values = []
+  for kw in keywords:
+    keys.append(gast.Str(kw.arg))
+    values.append(kw.value)
+  return gast.Dict(keys=keys, values=values)
diff --git a/tensorflow/contrib/py2tf/pyct/ast_util_test.py b/tensorflow/contrib/py2tf/pyct/ast_util_test.py
index a871ccad6f..8d123679e3 100644
--- a/tensorflow/contrib/py2tf/pyct/ast_util_test.py
+++ b/tensorflow/contrib/py2tf/pyct/ast_util_test.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import ast
 
 from tensorflow.contrib.py2tf.pyct import ast_util
+from tensorflow.contrib.py2tf.pyct import compiler
+from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.contrib.py2tf.pyct import qual_names
 from tensorflow.python.platform import test
 
@@ -74,6 +76,17 @@ class AstUtilTest(test.TestCase):
     self.assertFalse(ret is new_node.body[0])
     self.assertFalse(hasattr(new_node.body[0], '__foo'))
 
+  def test_keywords_to_dict(self):
+    keywords = parser.parse_expression('f(a=b, c=1, d=\'e\')').keywords
+    d = ast_util.keywords_to_dict(keywords)
+    # Make sure we generate a usable dict node by attaching it to a variable and
+    # compiling everything.
+    output = parser.parse_str('b = 3')
+    output.body += (ast.Assign([ast.Name(id='d', ctx=ast.Store())], d),)
+    result, _ = compiler.ast_to_object(output)
+    self.assertDictEqual(result.d, {'a': 3, 'c': 1, 'd': 'e'})
+    print(d)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 8cd562c55cb5fa172345f0de0376d9666b2326b4 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Thu, 22 Mar 2018 19:55:34 +0800
Subject: [PATCH 341/960] Fix the inconsistency in the accepted
 shape/data_format of Input Tensor to Conv2D in documentation (#17893)

---
 tensorflow/docs_src/tutorials/layers.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index 9b17d0d4d5..aeb746f29c 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -198,9 +198,9 @@ Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
 ### Input Layer
 
 The methods in the `layers` module for creating convolutional and pooling layers
-for two-dimensional image data expect input tensors to have a shape of
-<code>[<em>batch_size</em>, <em>image_width</em>, <em>image_height</em>,
-<em>channels</em>]</code>, defined as follows:
+for two-dimensional image data expect input tensors to have a `channels_last` shape of
+<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>, <em>channels</em>]</code>
+or a `channels_first` shape of <code>[<em>batch_size</em>, <em>channels</em>, <em>image_height</em>, <em>image_width</em>]</code>, defined as follows:
 
 *   _`batch_size`_. Size of the subset of examples to use when performing
     gradient descent during training.
-- 
GitLab


From 9e651e4571f7b7c2d32bdafe43cc4ced9bb0c750 Mon Sep 17 00:00:00 2001
From: Ilya Biryukov <ibiryukov@google.com>
Date: Thu, 22 Mar 2018 05:33:42 -0700
Subject: [PATCH 342/960] Allow to download clang and use clang for CPU builds.

Previously we only allowed to download clang when doing GPU builds.

The added skylark files use bazel's autoconf scripts, which were only added in
0.10.0. To provide nice error message for older versions of bazel
(i.e. 'version is less than 0.10' vs 'can't load @bazel_tools/cpp/...'), we
move the bazel version check into WORKSPACE file from workspace.bzl.

PiperOrigin-RevId: 190050798
---
 WORKSPACE                                     |  6 +++
 configure.py                                  | 26 ++++-----
 tensorflow/version_check.bzl                  | 48 +++++++++++++++++
 tensorflow/workspace.bzl                      | 53 ++-----------------
 third_party/clang_toolchain/BUILD             |  0
 .../clang_toolchain/cc_configure_clang.bzl    | 27 ++++++++++
 .../download_clang.bzl                        |  0
 third_party/gpus/cuda_configure.bzl           |  2 +-
 third_party/mkl_dnn/mkldnn.BUILD              |  2 +-
 tools/bazel.rc                                |  5 +-
 10 files changed, 104 insertions(+), 65 deletions(-)
 create mode 100644 tensorflow/version_check.bzl
 create mode 100644 third_party/clang_toolchain/BUILD
 create mode 100644 third_party/clang_toolchain/cc_configure_clang.bzl
 rename third_party/{gpus => clang_toolchain}/download_clang.bzl (100%)

diff --git a/WORKSPACE b/WORKSPACE
index 1e38a9a8cd..11c5cdb207 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -14,6 +14,12 @@ load("@io_bazel_rules_closure//closure:defs.bzl", "closure_repositories")
 
 closure_repositories()
 
+# We must check the bazel version before trying to parse any other BUILD
+# files, in case the parsing of those build files depends on the bazel
+# version we require here.
+load("//tensorflow:version_check.bzl", "check_bazel_version_at_least")
+check_bazel_version_at_least("0.10.0")
+
 load("//tensorflow:workspace.bzl", "tf_workspace")
 
 # Uncomment and update the paths in these entries to build the Android demo.
diff --git a/configure.py b/configure.py
index 7d61c2e5e3..ea732c64e2 100644
--- a/configure.py
+++ b/configure.py
@@ -524,7 +524,7 @@ def set_tf_cuda_clang(environ_cp):
 
 def set_tf_download_clang(environ_cp):
   """Set TF_DOWNLOAD_CLANG action_env."""
-  question = 'Do you want to download a fresh release of clang? (Experimental)'
+  question = 'Do you wish to download a fresh release of clang? (Experimental)'
   yes_reply = 'Clang will be downloaded and used to compile tensorflow.'
   no_reply = 'Clang will not be downloaded.'
   set_action_env_var(
@@ -1380,7 +1380,7 @@ def main():
   # environment variables.
   environ_cp = dict(os.environ)
 
-  check_bazel_version('0.5.4')
+  check_bazel_version('0.10.0')
 
   reset_tf_configure_bazelrc(args.workspace)
   cleanup_makefile()
@@ -1397,6 +1397,9 @@ def main():
     environ_cp['TF_NEED_OPENCL'] = '0'
     environ_cp['TF_CUDA_CLANG'] = '0'
     environ_cp['TF_NEED_TENSORRT'] = '0'
+    # TODO(ibiryukov): Investigate using clang as a cpu or cuda compiler on
+    # Windows.
+    environ_cp['TF_DOWNLOAD_CLANG'] = '0'
 
   if is_macos():
     environ_cp['TF_NEED_JEMALLOC'] = '0'
@@ -1444,16 +1447,8 @@ def main():
 
     set_tf_cuda_clang(environ_cp)
     if environ_cp.get('TF_CUDA_CLANG') == '1':
-      if not is_windows():
-        # Ask if we want to download clang release while building.
-        set_tf_download_clang(environ_cp)
-      else:
-        # We use bazel's generated crosstool on Windows and there is no
-        # way to provide downloaded toolchain for that yet.
-        # TODO(ibiryukov): Investigate using clang as a cuda compiler on
-        # Windows.
-        environ_cp['TF_DOWNLOAD_CLANG'] = '0'
-
+      # Ask whether we should download the clang toolchain.
+      set_tf_download_clang(environ_cp)
       if environ_cp.get('TF_DOWNLOAD_CLANG') != '1':
         # Set up which clang we should use as the cuda / host compiler.
         set_clang_cuda_compiler_path(environ_cp)
@@ -1463,6 +1458,13 @@ def main():
       if not is_windows():
         set_gcc_host_compiler_path(environ_cp)
     set_other_cuda_vars(environ_cp)
+  else:
+    # CUDA not required. Ask whether we should download the clang toolchain and
+    # use it for the CPU build.
+    set_tf_download_clang(environ_cp)
+    if environ_cp.get('TF_DOWNLOAD_CLANG') == '1':
+      write_to_bazelrc('build --config=download_clang')
+      write_to_bazelrc('test --config=download_clang')
 
   set_build_var(environ_cp, 'TF_NEED_MPI', 'MPI', 'with_mpi_support', False)
   if environ_cp.get('TF_NEED_MPI') == '1':
diff --git a/tensorflow/version_check.bzl b/tensorflow/version_check.bzl
new file mode 100644
index 0000000000..79e721dab4
--- /dev/null
+++ b/tensorflow/version_check.bzl
@@ -0,0 +1,48 @@
+""" Helpers to check minimum version of bazel."""
+
+def _extract_version_number(bazel_version):
+  """Extracts the semantic version number from a version string
+
+  Args:
+    bazel_version: the version string that begins with the semantic version
+      e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
+
+  Returns:
+    The semantic version string, like "1.2.3".
+  """
+  for i in range(len(bazel_version)):
+    c = bazel_version[i]
+    if not (c.isdigit() or c == "."):
+      return bazel_version[:i]
+  return bazel_version
+
+# Parse the bazel version string from `native.bazel_version`.
+# e.g.
+# "0.10.0rc1 abc123d" => (0, 10, 0)
+# "0.3.0" => (0, 3, 0)
+def _parse_bazel_version(bazel_version):
+  """Parses a version string into a 3-tuple of ints
+
+  int tuples can be compared directly using binary operators (<, >).
+
+  Args:
+    bazel_version: the Bazel version string
+
+  Returns:
+    An int 3-tuple of a (major, minor, patch) version.
+  """
+
+  version = _extract_version_number(bazel_version)
+  return tuple([int(n) for n in version.split(".")])
+
+def check_bazel_version_at_least(minimum_bazel_version):
+  if "bazel_version" not in dir(native):
+    fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
+  elif not native.bazel_version:
+    print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
+    print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
+    return
+
+  if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
+    fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
+        native.bazel_version, minimum_bazel_version))
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 675acbe5f6..ebb9e9412f 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -10,65 +10,18 @@ load("//third_party/sycl:sycl_configure.bzl", "sycl_configure")
 load("//third_party/toolchains/clang6:repo.bzl", "clang6_configure")
 load("//third_party/toolchains/cpus/arm:arm_compiler_configure.bzl", "arm_compiler_configure")
 load("//third_party:repo.bzl", "tf_http_archive")
+load("//third_party/clang_toolchain:cc_configure_clang.bzl", "cc_download_clang_toolchain")
 load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
 
-def _extract_version_number(bazel_version):
-  """Extracts the semantic version number from a version string
-
-  Args:
-    bazel_version: the version string that begins with the semantic version
-      e.g. "1.2.3rc1 abc1234" where "abc1234" is a commit hash.
-
-  Returns:
-    The semantic version string, like "1.2.3".
-  """
-  for i in range(len(bazel_version)):
-    c = bazel_version[i]
-    if not (c.isdigit() or c == "."):
-      return bazel_version[:i]
-  return bazel_version
-
-# Parse the bazel version string from `native.bazel_version`.
-# e.g.
-# "0.10.0rc1 abc123d" => (0, 10, 0)
-# "0.3.0" => (0, 3, 0)
-def _parse_bazel_version(bazel_version):
-  """Parses a version string into a 3-tuple of ints
-
-  int tuples can be compared directly using binary operators (<, >).
-
-  Args:
-    bazel_version: the Bazel version string
-
-  Returns:
-    An int 3-tuple of a (major, minor, patch) version.
-  """
-
-  version = _extract_version_number(bazel_version)
-  return tuple([int(n) for n in version.split(".")])
-
-def check_bazel_version_at_least(minimum_bazel_version):
-  if "bazel_version" not in dir(native):
-    fail("\nCurrent Bazel version is lower than 0.2.1, expected at least %s\n" % minimum_bazel_version)
-  elif not native.bazel_version:
-    print("\nCurrent Bazel is not a release version, cannot check for compatibility.")
-    print("Make sure that you are running at least Bazel %s.\n" % minimum_bazel_version)
-    return
-
-  if _parse_bazel_version(native.bazel_version) < _parse_bazel_version(minimum_bazel_version):
-    fail("\nCurrent Bazel version is {}, expected at least {}\n".format(
-        native.bazel_version, minimum_bazel_version))
 
 # If TensorFlow is linked as a submodule.
 # path_prefix is no longer used.
 # tf_repo_name is thought to be under consideration.
 def tf_workspace(path_prefix="", tf_repo_name=""):
-  # We must check the bazel version before trying to parse any other BUILD
-  # files, in case the parsing of those build files depends on the bazel
-  # version we require here.
-  check_bazel_version_at_least("0.5.4")
+  # Note that we check the minimum bazel version in WORKSPACE.
   clang6_configure(name="local_config_clang6")
+  cc_download_clang_toolchain(name="local_config_download_clang")
   cuda_configure(name="local_config_cuda")
   tensorrt_configure(name="local_config_tensorrt")
   git_configure(name="local_config_git")
diff --git a/third_party/clang_toolchain/BUILD b/third_party/clang_toolchain/BUILD
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/third_party/clang_toolchain/cc_configure_clang.bzl b/third_party/clang_toolchain/cc_configure_clang.bzl
new file mode 100644
index 0000000000..1181110ea9
--- /dev/null
+++ b/third_party/clang_toolchain/cc_configure_clang.bzl
@@ -0,0 +1,27 @@
+""" Downloads clang and configures the crosstool using bazel's autoconf."""
+
+load("@bazel_tools//tools/cpp:cc_configure.bzl", "cc_autoconf_impl")
+load(":download_clang.bzl", "download_clang")
+
+_TF_DOWNLOAD_CLANG = "TF_DOWNLOAD_CLANG"
+_TF_NEED_CUDA = "TF_NEED_CUDA"
+
+def _cc_clang_autoconf(repo_ctx):
+  if repo_ctx.os.environ.get(_TF_DOWNLOAD_CLANG) != "1":
+    return
+  if repo_ctx.os.environ.get(_TF_NEED_CUDA) == "1":
+    # Clang is handled separately for CUDA configs.
+    # See cuda_configure.bzl for more details.
+    return
+
+  download_clang(repo_ctx, out_folder='extra_tools')
+  overriden_tools = {'gcc': 'extra_tools/bin/clang'}
+  cc_autoconf_impl(repo_ctx, overriden_tools)
+
+cc_download_clang_toolchain = repository_rule(
+    environ = [
+        _TF_DOWNLOAD_CLANG,
+        _TF_NEED_CUDA,
+    ],
+    implementation = _cc_clang_autoconf,
+)
diff --git a/third_party/gpus/download_clang.bzl b/third_party/clang_toolchain/download_clang.bzl
similarity index 100%
rename from third_party/gpus/download_clang.bzl
rename to third_party/clang_toolchain/download_clang.bzl
diff --git a/third_party/gpus/cuda_configure.bzl b/third_party/gpus/cuda_configure.bzl
index 6c9c128db6..ede7e31897 100644
--- a/third_party/gpus/cuda_configure.bzl
+++ b/third_party/gpus/cuda_configure.bzl
@@ -96,7 +96,7 @@ NVVM_LIBDEVICE_PATHS = [
   "share/cuda/",
 ]
 
-load(":download_clang.bzl", "download_clang")
+load("//third_party/clang_toolchain:download_clang.bzl", "download_clang")
 
 # TODO(dzc): Once these functions have been factored out of Bazel's
 # cc_configure.bzl, load them from @bazel_tools instead.
diff --git a/third_party/mkl_dnn/mkldnn.BUILD b/third_party/mkl_dnn/mkldnn.BUILD
index 752a0d8498..68f24aabae 100644
--- a/third_party/mkl_dnn/mkldnn.BUILD
+++ b/third_party/mkl_dnn/mkldnn.BUILD
@@ -4,7 +4,7 @@ config_setting(
     name = "clang_linux_x86_64",
     values = {
         "cpu": "k8",
-        "define": "using_cuda_clang=true",
+        "define": "using_clang=true",
     },
 )
 
diff --git a/tools/bazel.rc b/tools/bazel.rc
index 8b8c717561..1c1e6afb65 100644
--- a/tools/bazel.rc
+++ b/tools/bazel.rc
@@ -27,11 +27,14 @@ build --define framework_shared_object=true
 build:mkl --define=using_mkl=true
 build:mkl -c opt
 
+build:download_clang --crosstool_top=@local_config_download_clang//:toolchain
+build:download_clang --define=using_clang=true
+
 build:cuda --crosstool_top=@local_config_cuda//crosstool:toolchain
 build:cuda --define=using_cuda=true --define=using_cuda_nvcc=true
 
 build:cuda_clang --crosstool_top=@local_config_cuda//crosstool:toolchain
-build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true
+build:cuda_clang --define=using_cuda=true --define=using_cuda_clang=true --define=using_clang=true
 
 build:win-cuda --define=using_cuda=true --define=using_cuda_nvcc=true
 
-- 
GitLab


From b559a319411e2d3f2a42f466c18737edd527bb10 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 05:46:07 -0700
Subject: [PATCH 343/960] Update file due to changes in Bazel (PACKAGE_NAME is
 deprecated)

PiperOrigin-RevId: 190051589
---
 tensorflow/compiler/aot/tfcompile.bzl | 2 +-
 tensorflow/tensorflow.bzl             | 4 ++--
 tensorflow/tools/test/performance.bzl | 2 +-
 3 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/aot/tfcompile.bzl b/tensorflow/compiler/aot/tfcompile.bzl
index 9dff1be09f..3a877c5337 100644
--- a/tensorflow/compiler/aot/tfcompile.bzl
+++ b/tensorflow/compiler/aot/tfcompile.bzl
@@ -132,7 +132,7 @@ def tf_library(name, graph, config,
   header_file = name + ".h"
   metadata_object_file = name + "_tfcompile_metadata.o"
   function_object_file = name + "_tfcompile_function.o"
-  ep = ("__" + PACKAGE_NAME + "__" + name).replace("/", "_")
+  ep = ("__" + native.package_name() + "__" + name).replace("/", "_")
   if type(tfcompile_flags) == type(""):
     flags = tfcompile_flags
   else:
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 9b0db8a112..2d3cb415fe 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -34,7 +34,7 @@ def src_to_test_name(src):
   return src.replace("/", "_").split(".")[0]
 
 def full_path(relative_paths):
-  return [PACKAGE_NAME + "/" + relative for relative in relative_paths]
+  return [native.package_name() + "/" + relative for relative in relative_paths]
 
 # List of proto files for android builds
 def tf_android_core_proto_sources(core_proto_sources_relative):
@@ -265,7 +265,7 @@ def _rpath_linkopts(name):
   # deployed. Other shared object dependencies (e.g. shared between contrib/
   # ops) are picked up as long as they are in either the same or a parent
   # directory in the tensorflow/ tree.
-  levels_to_root = PACKAGE_NAME.count("/") + name.count("/")
+  levels_to_root = native.package_name().count("/") + name.count("/")
   return select({
       clean_dep("//tensorflow:darwin"): [
           "-Wl,%s" % (_make_search_paths("@loader_path", levels_to_root),),
diff --git a/tensorflow/tools/test/performance.bzl b/tensorflow/tools/test/performance.bzl
index cee53dd5b6..3486871080 100644
--- a/tensorflow/tools/test/performance.bzl
+++ b/tensorflow/tools/test/performance.bzl
@@ -31,7 +31,7 @@ def tf_cc_logged_benchmark(
       size = "large",
       srcs = ["//tensorflow/tools/test:run_and_gather_logs"],
       args = [
-          "--name=//%s:%s" % (PACKAGE_NAME, name),
+          "--name=//%s:%s" % (native.package_name(), name),
           "--test_name=" + target,
           "--test_args=--benchmarks=%s" % benchmarks,
           "--benchmark_type=%s" % benchmark_type,
-- 
GitLab


From cae614a3300b3befae52d0e076c708450a93d820 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Thu, 22 Mar 2018 05:50:56 -0700
Subject: [PATCH 344/960] Automatically insert api-links.

If the contents of a pair of back-ticks match a public api symbol name insert a link.

PiperOrigin-RevId: 190051941
---
 tensorflow/tools/docs/parser.py | 48 +++++++++++++++++++++++----------
 1 file changed, 34 insertions(+), 14 deletions(-)

diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py
index e758229535..d2a63ecc49 100644
--- a/tensorflow/tools/docs/parser.py
+++ b/tensorflow/tools/docs/parser.py
@@ -34,7 +34,11 @@ from tensorflow.python.util import tf_inspect
 
 
 # A regular expression capturing a python identifier.
-IDENTIFIER_RE = '[a-zA-Z_][a-zA-Z0-9_]*'
+IDENTIFIER_RE = r'[a-zA-Z_]\w*'
+
+
+class TFDocsError(Exception):
+  pass
 
 
 class _Errors(object):
@@ -118,6 +122,8 @@ SYMBOL_REFERENCE_RE = re.compile(
     """,
     flags=re.VERBOSE)
 
+AUTO_REFERENCE_RE = re.compile(r'`([a-zA-Z0-9_.]+?)`')
+
 
 class ReferenceResolver(object):
   """Class for replacing @{...} references with Markdown links.
@@ -240,10 +246,25 @@ class ReferenceResolver(object):
     Returns:
       `string`, with "@{symbol}" references replaced by Markdown links.
     """
-    def one_ref(match):
-      return self._one_ref(match, relative_path_to_root)
 
-    return re.sub(SYMBOL_REFERENCE_RE, one_ref, string)
+    def strict_one_ref(match):
+      try:
+        return self._one_ref(match, relative_path_to_root)
+      except TFDocsError as e:
+        self.add_error(e.message)
+        return 'BAD_LINK'
+
+    string = re.sub(SYMBOL_REFERENCE_RE, strict_one_ref, string)
+
+    def sloppy_one_ref(match):
+      try:
+        return self._one_ref(match, relative_path_to_root)
+      except TFDocsError:
+        return match.group(0)
+
+    string = re.sub(AUTO_REFERENCE_RE, sloppy_one_ref, string)
+
+    return string
 
   def python_link(self, link_text, ref_full_name, relative_path_to_root,
                   code_ref=True):
@@ -307,14 +328,14 @@ class ReferenceResolver(object):
 
     Raises:
       RuntimeError: If `ref_full_name` is not documented.
+      TFDocsError: If the @{} syntax cannot be decoded.
     """
     master_name = self._duplicate_of.get(ref_full_name, ref_full_name)
 
     # Check whether this link exists
     if master_name not in self._all_names:
-      message = 'Cannot make link to "%s": Not in index.' % master_name
-      self.add_error(message)
-      return 'BROKEN_LINK'
+      raise TFDocsError(
+          'Cannot make link to "%s": Not in index.' % master_name)
 
     # If this is a member of a class, link to the class page with an anchor.
     ref_path = None
@@ -369,8 +390,8 @@ class ReferenceResolver(object):
             code_ref=not manual_link_text)
 
     # Error!
-    self.add_error('Did not understand "%s"' % match.group(0))
-    return 'BROKEN_LINK'
+    raise TFDocsError('Did not understand "%s"' % match.group(0),
+                      'BROKEN_LINK')
 
   def _doc_link(self, string, link_text, manual_link_text,
                 relative_path_to_root):
@@ -395,11 +416,10 @@ class ReferenceResolver(object):
     return self._doc_missing(string, hash_tag, link_text, manual_link_text,
                              relative_path_to_root)
 
-  def _doc_missing(self, string, unused_hash_tag, link_text,
+  def _doc_missing(self, string, unused_hash_tag, unused_link_text,
                    unused_manual_link_text, unused_relative_path_to_root):
     """Generate an error for unrecognized @{$...} references."""
-    self.add_error('Unknown Document "%s"' % string)
-    return link_text
+    raise TFDocsError('Unknown Document "%s"' % string)
 
   def _cc_link(self, string, link_text, unused_manual_link_text,
                relative_path_to_root):
@@ -416,8 +436,8 @@ class ReferenceResolver(object):
     elif string == 'tensorflow::ops::Const':
       ret = 'namespace/tensorflow/ops.md#const'
     else:
-      self.add_error('C++ reference not understood: "%s"' % string)
-      return 'TODO_C++:%s' % string
+      raise TFDocsError('C++ reference not understood: "%s"' % string)
+
     # relative_path_to_root gets you to api_docs/python, we go from there
     # to api_docs/cc, and then add ret.
     cc_relative_path = os.path.normpath(os.path.join(
-- 
GitLab


From 69dc403c97f273b750d5927ec1ed26613d90f3ad Mon Sep 17 00:00:00 2001
From: cbockman <c.bockman@gmail.com>
Date: Thu, 22 Mar 2018 06:42:39 -0700
Subject: [PATCH 345/960] spelling fix (#17911)

---
 tensorflow/contrib/data/python/ops/grouping.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py
index a19be22254..ae10d2eb22 100644
--- a/tensorflow/contrib/data/python/ops/grouping.py
+++ b/tensorflow/contrib/data/python/ops/grouping.py
@@ -42,7 +42,7 @@ def group_by_window(key_func,
   This transformation maps each consecutive element in a dataset to a key
   using `key_func` and groups the elements by key. It then applies
   `reduce_func` to at most `window_size_func(key)` elements matching the same
-  key. All execpt the final window for each key will contain
+  key. All except the final window for each key will contain
   `window_size_func(key)` elements; the final window may be smaller.
 
   You may provide either a constant `window_size` or a window size determined by
-- 
GitLab


From 4d5c139fbb831684e58b3875cd253a15c742362d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 08:19:21 -0700
Subject: [PATCH 346/960] Fix QN for Calls.

PiperOrigin-RevId: 190067548
---
 tensorflow/contrib/py2tf/pyct/qual_names.py   | 20 +++++++------------
 .../contrib/py2tf/pyct/qual_names_test.py     | 18 +++++++++++++++++
 .../py2tf/pyct/static_analysis/activity.py    |  4 ++++
 3 files changed, 29 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/qual_names.py b/tensorflow/contrib/py2tf/pyct/qual_names.py
index 6bcbaeb2ae..7dec13db92 100644
--- a/tensorflow/contrib/py2tf/pyct/qual_names.py
+++ b/tensorflow/contrib/py2tf/pyct/qual_names.py
@@ -169,14 +169,6 @@ class QnResolver(gast.NodeTransformer):
   Note: Not using NodeAnnos to avoid circular dependencies.
   """
 
-  def visit_Call(self, node):
-    node = self.generic_visit(node)
-    # This helps treat the following cases uniformly:
-    #   a = b[i]
-    #   a = b()[i]
-    anno.copyanno(node.func, node, anno.Basic.QN)
-    return node
-
   def visit_Name(self, node):
     node = self.generic_visit(node)
     anno.setanno(node, anno.Basic.QN, QN(node.id))
@@ -184,8 +176,9 @@ class QnResolver(gast.NodeTransformer):
 
   def visit_Attribute(self, node):
     node = self.generic_visit(node)
-    anno.setanno(node, anno.Basic.QN,
-                 QN(anno.getanno(node.value, anno.Basic.QN), attr=node.attr))
+    if anno.hasanno(node.value, anno.Basic.QN):
+      anno.setanno(node, anno.Basic.QN,
+                   QN(anno.getanno(node.value, anno.Basic.QN), attr=node.attr))
     return node
 
   def visit_Subscript(self, node):
@@ -201,9 +194,10 @@ class QnResolver(gast.NodeTransformer):
       subscript = QN(StringLiteral(s.value.s))
     else:
       subscript = anno.getanno(node.slice.value, anno.Basic.QN)
-    anno.setanno(node, anno.Basic.QN,
-                 QN(anno.getanno(node.value, anno.Basic.QN),
-                    subscript=subscript))
+    if anno.hasanno(node.value, anno.Basic.QN):
+      anno.setanno(node, anno.Basic.QN,
+                   QN(anno.getanno(node.value, anno.Basic.QN),
+                      subscript=subscript))
     return node
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/qual_names_test.py b/tensorflow/contrib/py2tf/pyct/qual_names_test.py
index f2cd8e98f0..6583fa243b 100644
--- a/tensorflow/contrib/py2tf/pyct/qual_names_test.py
+++ b/tensorflow/contrib/py2tf/pyct/qual_names_test.py
@@ -208,6 +208,24 @@ class QNResolverTest(test.TestCase):
     self.assertQNStringIs(nodes[8], 'a.b[c[d]].e.f')
     self.assertQNStringIs(nodes[9], 'a.b[c[d.e.f].g].h')
 
+  def test_function_calls(self):
+    samples = """
+      a.b
+      a.b()
+      a().b
+      z[i]
+      z[i]()
+      z()[i]
+    """
+    nodes = resolve(parser.parse_str(textwrap.dedent(samples)))
+    nodes = tuple(n.value for n in nodes.body)
+    self.assertQNStringIs(nodes[0], 'a.b')
+    self.assertQNStringIs(nodes[1].func, 'a.b')
+    self.assertQNStringIs(nodes[2].value.func, 'a')
+    self.assertQNStringIs(nodes[3], 'z[i]')
+    self.assertQNStringIs(nodes[4].func, 'z[i]')
+    self.assertQNStringIs(nodes[5].value.func, 'z')
+
 
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py b/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py
index 87fc8c979c..716672a53b 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py
+++ b/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py
@@ -171,6 +171,10 @@ class ActivityAnalizer(transformer.Base):
     self._in_return_statement = False
 
   def _track_symbol(self, node):
+    # This can happen when we have an attribute (or subscript) on a function
+    # call.  Example: a().b
+    if not anno.hasanno(node, anno.Basic.QN):
+      return
     qn = anno.getanno(node, anno.Basic.QN)
 
     if isinstance(node.ctx, gast.Store):
-- 
GitLab


From 4deaf50fd8bb10aa2c96662a106f201b281f57ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 08:51:43 -0700
Subject: [PATCH 347/960] Methods to work with symbolic tensor shapes.

PiperOrigin-RevId: 190071400
---
 tensorflow/core/grappler/optimizers/BUILD     |  30 +++
 .../grappler/optimizers/symbolic_shapes.cc    | 177 ++++++++++++++++++
 .../grappler/optimizers/symbolic_shapes.h     |  60 ++++++
 .../optimizers/symbolic_shapes_test.cc        |  95 ++++++++++
 4 files changed, 362 insertions(+)
 create mode 100644 tensorflow/core/grappler/optimizers/symbolic_shapes.cc
 create mode 100644 tensorflow/core/grappler/optimizers/symbolic_shapes.h
 create mode 100644 tensorflow/core/grappler/optimizers/symbolic_shapes_test.cc

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 96ea8f7a83..ac29edd213 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -5,6 +5,12 @@ load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 
+# Platform specific build config
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_protos_grappler",
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
@@ -586,3 +592,27 @@ tf_cc_test(
         "//tensorflow/core/grappler/utils:grappler_test",
     ],
 )
+
+cc_library(
+    name = "symbolic_shapes",
+    srcs = ["symbolic_shapes.cc"],
+    hdrs = ["symbolic_shapes.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:protos_all_cc",
+    ] + tf_protos_grappler(),
+)
+
+tf_cc_test(
+    name = "symbolic_shapes_test",
+    srcs = ["symbolic_shapes_test.cc"],
+    deps = [
+        ":symbolic_shapes",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
diff --git a/tensorflow/core/grappler/optimizers/symbolic_shapes.cc b/tensorflow/core/grappler/optimizers/symbolic_shapes.cc
new file mode 100644
index 0000000000..cfca2dc0d3
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/symbolic_shapes.cc
@@ -0,0 +1,177 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/symbolic_shapes.h"
+#include "tensorflow/core/util/bcast.h"
+
+namespace tensorflow {
+namespace grappler {
+namespace {
+
+BCast::Vec ShapeDims(const TensorShapeProto& shape) {
+  BCast::Vec dims;
+  dims.reserve(shape.dim_size());
+  for (int i = 0; i < shape.dim_size(); ++i)
+    dims.push_back(shape.dim(i).size());
+  return dims;
+}
+
+}  // namespace
+
+bool IsKnown(const TensorShapeProto::Dim& dim) { return dim.size() >= 0; }
+
+bool IsKnownSymbolically(const TensorShapeProto::Dim& dim) {
+  return dim.size() <= -2;
+}
+
+bool IsUnknown(const TensorShapeProto::Dim& dim) { return dim.size() == -1; }
+
+bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
+  return !shape.unknown_rank() &&
+         std::all_of(
+             shape.dim().begin(), shape.dim().end(),
+             [](const TensorShapeProto::Dim& dim) { return !IsUnknown(dim); });
+}
+
+bool ShapeIsSymbolicallyDefined(const OpInfo::TensorProperties& properties) {
+  return ShapeIsSymbolicallyDefined(properties.shape());
+}
+
+bool ShapesSymbolicallyEqual(const TensorShapeProto& left,
+                             const TensorShapeProto& right) {
+  if (left.unknown_rank() || right.unknown_rank() ||
+      left.dim_size() != right.dim_size()) {
+    return false;
+  }
+  for (int i = 0; i < left.dim_size(); ++i) {
+    const auto& ldim = left.dim(i);
+    const auto& rdim = right.dim(i);
+    if (IsUnknown(ldim) || IsUnknown(rdim) || ldim.size() != rdim.size()) {
+      return false;
+    }
+  }
+  return true;
+}
+
+bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
+                             const OpInfo::TensorProperties& right) {
+  return ShapesSymbolicallyEqual(left.shape(), right.shape());
+}
+
+bool ShapesBroadcastable(const TensorShapeProto& left,
+                         const TensorShapeProto& right) {
+  if (!ShapeIsSymbolicallyDefined(left) || !ShapeIsSymbolicallyDefined(right)) {
+    return false;
+  }
+  BCast bcast(ShapeDims(left), ShapeDims(right),
+              /*fewer_dims_optimization*/ false);
+  return bcast.IsValid();
+}
+
+bool ShapesBroadcastable(const OpInfo::TensorProperties& left,
+                         const OpInfo::TensorProperties& right) {
+  return ShapesBroadcastable(left.shape(), right.shape());
+}
+
+bool CompareSymbolicallyShapedTensorSizes(const TensorShapeProto& left,
+                                          const TensorShapeProto& right) {
+  // if one of the ranks is unknown, it's impossible to compare tensor sizes
+  if (left.unknown_rank() || right.unknown_rank()) {
+    return false;
+  }
+
+  // Tensor size, computed as a product of defined dimensions
+  int64 left_defined_size = 1;
+  int64 right_defined_size = 1;
+
+  // Keep how many times each unknown dimension appeared on the left and right
+  std::unordered_map<int64, int64> left_unknown_dims;
+  std::unordered_map<int64, int64> right_unknown_dims;
+
+  // Assign unique id to every unknown dimension (-1). We are going to
+  // assign positive ids, because negative values are already used by
+  // symbolic dimensions.
+  int64 unknown_dim_id = 1;
+
+  // For each shape dimension update "defined tensor size", if shape is defined,
+  // or increment a counter for unknown dim.
+  auto process_dimensions =
+      [&unknown_dim_id](const TensorShapeProto& shape, int64* defined_size,
+                        std::unordered_map<int64, int64>* unknown_dims) {
+        for (int i = 0; i < shape.dim_size(); ++i) {
+          const auto& dim = shape.dim(i);
+          int64 dim_size = dim.size();
+          if (dim_size > 0) {
+            *defined_size *= dim_size;
+          } else if (IsUnknown(dim)) {
+            ++(*unknown_dims)[unknown_dim_id++];
+          } else if (IsKnownSymbolically(dim)) {
+            ++(*unknown_dims)[dim_size];
+          }
+        }
+      };
+
+  process_dimensions(left, &left_defined_size, &left_unknown_dims);
+  process_dimensions(right, &right_defined_size, &right_unknown_dims);
+
+  // Compute a union of unknown dimension ids appeared in both shapes
+  std::set<int64> unknown_dims;
+  for (const auto& el : left_unknown_dims) unknown_dims.insert(el.first);
+  for (const auto& el : right_unknown_dims) unknown_dims.insert(el.first);
+
+  // Cancel unknown dimensions that appeared in both shapes
+  for (int64 unknown_dim : unknown_dims) {
+    int64 co_occurrence = std::min(left_unknown_dims[unknown_dim],
+                                   right_unknown_dims[unknown_dim]);
+    left_unknown_dims[unknown_dim] -= co_occurrence;
+    right_unknown_dims[unknown_dim] -= co_occurrence;
+  }
+
+  // Count unbalanced unknown dimensions
+  int64 left_unbalanced_unknown_dims = 0;
+  int64 right_unbalanced_unknown_dims = 0;
+  for (const auto& el : left_unknown_dims)
+    left_unbalanced_unknown_dims += el.second;
+  for (const auto& el : right_unknown_dims)
+    right_unbalanced_unknown_dims += el.second;
+
+  if (left_unbalanced_unknown_dims == 0 && right_unbalanced_unknown_dims == 0) {
+    // If unknown dimensions cancelled each other, compare tensor sizes
+    // represented by defined dimensions
+    return left_defined_size < right_defined_size;
+  }
+
+  if (left_defined_size <= right_defined_size &&
+      left_unbalanced_unknown_dims == 0 && right_unbalanced_unknown_dims > 0) {
+    // If size of a 'left" tensor computed from defined dimensions less or
+    // equal, and shape on the right has unbalanced unknown dimensions, we can
+    // guarantee that shape on the left is strictly smaller (assuming that
+    // unknown dimension size is larger than 1)
+    return true;
+  }
+
+  // In every other case, assuming that unknown dimensions can be arbitrary
+  // large in size, we can't guarantee any ordering
+  return false;
+}
+
+bool CompareSymbolicallyShapedTensorSizes(
+    const OpInfo::TensorProperties& left,
+    const OpInfo::TensorProperties& right) {
+  return CompareSymbolicallyShapedTensorSizes(left.shape(), right.shape());
+}
+
+}  // end namespace grappler
+}  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/symbolic_shapes.h b/tensorflow/core/grappler/optimizers/symbolic_shapes.h
new file mode 100644
index 0000000000..a9dcf44e23
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/symbolic_shapes.h
@@ -0,0 +1,60 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_SYMBOLIC_SHAPES_H_
+#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_SYMBOLIC_SHAPES_H_
+
+#include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/grappler/costs/op_performance_data.pb.h"
+
+namespace tensorflow {
+namespace grappler {
+
+bool IsKnown(const TensorShapeProto::Dim& dim);
+bool IsKnownSymbolically(const TensorShapeProto::Dim& dim);
+bool IsUnknown(const TensorShapeProto::Dim& dim);
+
+// Shape is symbolically defined, if it has a known rank, and each dimension is
+// known (dim_size >= 0), or is a symbolic dimension size (dim_size <= -2).
+bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape);
+bool ShapeIsSymbolicallyDefined(const OpInfo::TensorProperties& properties);
+
+// Shapes are symbolically equal, if they have the same rank, they are
+// they are known or symbolically defined, and have matching dimensions.
+bool ShapesSymbolicallyEqual(const TensorShapeProto& left,
+                             const TensorShapeProto& right);
+bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
+                             const OpInfo::TensorProperties& right);
+
+// Check if two shapes can be broadcasted to each other. Both shapes must be at
+// least symbolically defined, and the have valid BCast instance.
+bool ShapesBroadcastable(const TensorShapeProto& left,
+                         const TensorShapeProto& right);
+bool ShapesBroadcastable(const OpInfo::TensorProperties& left,
+                         const OpInfo::TensorProperties& right);
+
+// Return true if can prove, that tensor of size 'left' is smaller than tensor
+// of size 'right'. Return false if it's larger or equal, or it's impossible to
+// compare because of unknown dimensions, or mismatch in symbolic dimensions.
+bool CompareSymbolicallyShapedTensorSizes(const TensorShapeProto& left,
+                                          const TensorShapeProto& right);
+bool CompareSymbolicallyShapedTensorSizes(
+    const OpInfo::TensorProperties& left,
+    const OpInfo::TensorProperties& right);
+
+}  // namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_SYMBOLIC_SHAPES_H_
diff --git a/tensorflow/core/grappler/optimizers/symbolic_shapes_test.cc b/tensorflow/core/grappler/optimizers/symbolic_shapes_test.cc
new file mode 100644
index 0000000000..5ef9f65925
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/symbolic_shapes_test.cc
@@ -0,0 +1,95 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/symbolic_shapes.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace grappler {
+namespace {
+
+class SymbolicShapesTest : public ::testing::Test {
+ protected:
+  TensorShapeProto MakeUnknown() {
+    TensorShapeProto shape;
+    shape.set_unknown_rank(true);
+    return shape;
+  }
+
+  TensorShapeProto MakeShape(std::vector<int> dims) {
+    TensorShapeProto shape;
+    for (int dim_size : dims) {
+      TensorShapeProto::Dim dim;
+      dim.set_size(dim_size);
+      *shape.add_dim() = dim;
+    }
+    return shape;
+  }
+};
+
+bool operator<(const TensorShapeProto& lhs, const TensorShapeProto& rhs) {
+  return CompareSymbolicallyShapedTensorSizes(lhs, rhs);
+}
+
+TEST_F(SymbolicShapesTest, ShapeIsSymbolicallyDefined) {
+  EXPECT_FALSE(ShapeIsSymbolicallyDefined(MakeUnknown()));
+  EXPECT_FALSE(ShapeIsSymbolicallyDefined(MakeShape({-1, 2})));
+
+  EXPECT_TRUE(ShapeIsSymbolicallyDefined(MakeShape({1, 2})));
+  EXPECT_TRUE(ShapeIsSymbolicallyDefined(MakeShape({-2, 2})));
+}
+
+TEST_F(SymbolicShapesTest, ShapesSymbolicallyEqual) {
+  EXPECT_FALSE(ShapesSymbolicallyEqual(MakeUnknown(), MakeUnknown()));
+  EXPECT_FALSE(ShapesSymbolicallyEqual(MakeShape({-1, 2}), MakeShape({-1, 2})));
+  EXPECT_FALSE(ShapesSymbolicallyEqual(MakeShape({-2, 2}), MakeShape({-3, 2})));
+
+  EXPECT_TRUE(ShapesSymbolicallyEqual(MakeShape({1, 2}), MakeShape({1, 2})));
+  EXPECT_TRUE(ShapesSymbolicallyEqual(MakeShape({-2, 2}), MakeShape({-2, 2})));
+}
+
+TEST_F(SymbolicShapesTest, ShapesBroadcastable) {
+  EXPECT_FALSE(ShapesBroadcastable(MakeUnknown(), MakeUnknown()));
+  EXPECT_FALSE(ShapesBroadcastable(MakeShape({-2}), MakeShape({1, -3})));
+  EXPECT_FALSE(ShapesBroadcastable(MakeShape({-1, 2}), MakeShape({-1, 2})));
+  EXPECT_FALSE(ShapesBroadcastable(MakeShape({-2, 2}), MakeShape({-3, 2})));
+  EXPECT_FALSE(ShapesBroadcastable(MakeShape({-2, 4}), MakeShape({-2, 8})));
+
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({1, 2}), MakeShape({1, 2})));
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({-2, 2}), MakeShape({-2, 2})));
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({-2, 32}), MakeShape({-2, 1})));
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({-2, 1}), MakeShape({1, -2})));
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({-2, 1}), MakeShape({1, -3})));
+  EXPECT_TRUE(ShapesBroadcastable(MakeShape({-3}), MakeShape({-2, -3})));
+}
+
+TEST_F(SymbolicShapesTest, CompareSymbolicallyShapedTensorSizes) {
+  EXPECT_TRUE(MakeShape({1, 1, 32}) < MakeShape({32, 32}));
+  EXPECT_TRUE(MakeShape({1, 32, 32}) < MakeShape({2048}));
+  EXPECT_TRUE(MakeShape({1, -2, 32}) < MakeShape({-2, 32, 32}));
+  EXPECT_TRUE(MakeShape({1, 32, 32}) < MakeShape({-2, 32, 32}));
+  EXPECT_TRUE(MakeShape({1, 32, 32}) < MakeShape({-1, 32, 32}));
+  EXPECT_TRUE(MakeShape({1, -2, 32}) < MakeShape({-2, -2, 32}));
+
+  EXPECT_FALSE(MakeShape({1, -2, 32}) < MakeShape({-3, 32, 32}));
+  EXPECT_FALSE(MakeShape({1, -1, 32}) < MakeShape({1, -1, 32}));
+  EXPECT_FALSE(MakeShape({1, -1, 32}) < MakeShape({-1, -1, 32}));
+  EXPECT_FALSE(MakeShape({-1, -1, 32}) < MakeShape({1, -1, 32}));
+}
+
+}  // namespace
+}  // namespace grappler
+}  // namespace tensorflow
-- 
GitLab


From e92c2e0c957bc539fc24dffdceb96f4b3955bbee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 09:14:07 -0700
Subject: [PATCH 348/960] Disable over-aggressive shape inference

PiperOrigin-RevId: 190074445
---
 tensorflow/core/ops/list_ops.cc               |  4 ----
 .../python/kernel_tests/list_ops_test.py      | 22 +++++++++++++++++++
 2 files changed, 22 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/ops/list_ops.cc b/tensorflow/core/ops/list_ops.cc
index 0c16abd369..cad617638f 100644
--- a/tensorflow/core/ops/list_ops.cc
+++ b/tensorflow/core/ops/list_ops.cc
@@ -135,10 +135,6 @@ REGISTER_OP("TensorListStack")
         }
         shape_inference::ShapeHandle ignored;
         TF_RETURN_IF_ERROR(c->Merge(s, list_shape_type.shape, &ignored));
-        if (!c->FullyDefined(list_shape_type.shape)) {
-          return errors::InvalidArgument(
-              "Can only stack a list with fully defined shapes.");
-        }
         s = list_shape_type.shape;
       }
       int expected_num_elements = -1;
diff --git a/tensorflow/python/kernel_tests/list_ops_test.py b/tensorflow/python/kernel_tests/list_ops_test.py
index 8865e165fd..dbbed39c72 100644
--- a/tensorflow/python/kernel_tests/list_ops_test.py
+++ b/tensorflow/python/kernel_tests/list_ops_test.py
@@ -152,6 +152,28 @@ class ListOpsTest(test_util.TensorFlowTestCase):
       s1 = list_ops.tensor_list_stack(t1, element_dtype=dtypes.int32).eval()
       self.assertAllEqual(s1, [0, 1, 2, 3])
 
+  def testGraphStackSwitchDtype(self):
+    with context.graph_mode(), self.test_session():
+      list_ = list_ops.empty_tensor_list(
+          element_shape=constant_op.constant([], dtype=dtypes.int32),
+          element_dtype=dtypes.int32)
+      m = constant_op.constant([1, 2, 3], dtype=dtypes.float32)
+
+      def body(list_, m):
+        list_ = control_flow_ops.cond(
+            math_ops.equal(list_ops.tensor_list_length(list_), 0),
+            lambda: list_ops.empty_tensor_list(m.shape, m.dtype), lambda: list_)
+        list_ = list_ops.tensor_list_push_back(list_, m)
+        return list_, m
+
+      for _ in range(2):
+        list_, m = body(list_, m)
+
+      s1 = list_ops.tensor_list_stack(
+          list_, element_dtype=dtypes.float32).eval()
+      np_s1 = np.array([[1, 2, 3], [1, 2, 3]], dtype=np.float32)
+      self.assertAllEqual(s1, np_s1)
+
   def testGraphStackInLoopSwitchDtype(self):
     with context.graph_mode(), self.test_session():
       t1 = list_ops.empty_tensor_list(
-- 
GitLab


From b6ad189b1a197e454ae527829a01f742d76ba2a2 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Thu, 22 Mar 2018 09:34:29 -0700
Subject: [PATCH 349/960] [XLA] Only overwrite the hlo_profiling flag when it's
 not enabled by default.

This got broken in 504d103a405654f029e8902d97d4dd8f3aa07513

PiperOrigin-RevId: 190077360
---
 .../compiler/xla/client/executable_build_options.cc      | 4 +++-
 .../compiler/xla/client/executable_build_options.h       | 9 +++++----
 tensorflow/compiler/xla/service/local_service.cc         | 6 ++++--
 3 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/client/executable_build_options.cc b/tensorflow/compiler/xla/client/executable_build_options.cc
index 4ff4da6215..6e3c5cb484 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.cc
+++ b/tensorflow/compiler/xla/client/executable_build_options.cc
@@ -103,6 +103,8 @@ ExecutableBuildOptions& ExecutableBuildOptions::set_hlo_profile(bool enabled) {
   return *this;
 }
 
-bool ExecutableBuildOptions::hlo_profile() const { return hlo_profile_; }
+tensorflow::gtl::optional<bool> ExecutableBuildOptions::hlo_profile() const {
+  return hlo_profile_;
+}
 
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/executable_build_options.h b/tensorflow/compiler/xla/client/executable_build_options.h
index 85b2cd96cb..11f1098360 100644
--- a/tensorflow/compiler/xla/client/executable_build_options.h
+++ b/tensorflow/compiler/xla/client/executable_build_options.h
@@ -70,17 +70,18 @@ class ExecutableBuildOptions {
       tensorflow::StringPiece dirpath);
   const tensorflow::gtl::optional<string>& dump_per_pass_hlo_proto_to() const;
 
-  // If set, specifies that we should record an HLO profile during execution and
-  // log it after execution (as in DebugOptions).
+  // If true, specifies that we should record an HLO profile during execution
+  // and log it after execution (as in DebugOptions). If nullopt the default is
+  // used.
   ExecutableBuildOptions& set_hlo_profile(bool enabled);
-  bool hlo_profile() const;
+  tensorflow::gtl::optional<bool> hlo_profile() const;
 
   // Returns a string representation of the build options, suitable for
   // debugging.
   string ToString() const;
 
  private:
-  bool hlo_profile_ = false;
+  tensorflow::gtl::optional<bool> hlo_profile_;
   int device_ordinal_ = -1;
   Shape result_layout_;
   bool result_layout_set_ = false;
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 7fd1ccd1a8..5690a89909 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -119,8 +119,10 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
   }
 
   ExecutionOptions execution_options = CreateDefaultExecutionOptions();
-  execution_options.mutable_debug_options()->set_xla_hlo_profile(
-      build_options.hlo_profile());
+  if (build_options.hlo_profile().has_value()) {
+    execution_options.mutable_debug_options()->set_xla_hlo_profile(
+        *build_options.hlo_profile());
+  }
   if (build_options.generate_hlo_graph().has_value()) {
     execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
         build_options.generate_hlo_graph().value());
-- 
GitLab


From 18c6e42b95dab659aa755242096cda9195db4927 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 09:43:32 -0700
Subject: [PATCH 350/960] Simplifying "is" and "is not" dispatch

PiperOrigin-RevId: 190078959
---
 tensorflow/contrib/py2tf/utils/multiple_dispatch.py | 13 ++++---------
 .../contrib/py2tf/utils/multiple_dispatch_test.py   |  8 ++++----
 2 files changed, 8 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/py2tf/utils/multiple_dispatch.py b/tensorflow/contrib/py2tf/utils/multiple_dispatch.py
index da7a942703..427a936c35 100644
--- a/tensorflow/contrib/py2tf/utils/multiple_dispatch.py
+++ b/tensorflow/contrib/py2tf/utils/multiple_dispatch.py
@@ -22,21 +22,16 @@ import six
 
 from tensorflow.contrib.py2tf.utils.type_check import is_tensor
 from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import math_ops
 
 
 def dynamic_is(left, right):
-  if is_tensor(left, right):
-    return math_ops.equal(left.name, right.name)
-  else:
-    return left is right
+  # TODO(alexbw) if we're sure we should leave 'is' in place,
+  # then change the semantics in converters/logical_expressions.py
+  return left is right
 
 
 def dynamic_is_not(left, right):
-  if is_tensor(left, right):
-    return math_ops.not_equal(left.name, right.name)
-  else:
-    return left is not right
+  return left is not right
 
 
 def run_cond(condition, true_fn, false_fn):
diff --git a/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py b/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py
index 8d89b6898a..75e8fdd5ed 100644
--- a/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py
+++ b/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py
@@ -50,10 +50,10 @@ class MultipleDispatchTest(test.TestCase):
       should_be_false1 = multiple_dispatch.dynamic_is_not(a, also_a)
       should_be_true2 = multiple_dispatch.dynamic_is_not(a, not_actually_a)
       should_be_false2 = multiple_dispatch.dynamic_is(a, not_actually_a)
-      self.assertTrue(should_be_true1.eval())
-      self.assertTrue(should_be_true2.eval())
-      self.assertFalse(should_be_false1.eval())
-      self.assertFalse(should_be_false2.eval())
+      self.assertTrue(should_be_true1)
+      self.assertTrue(should_be_true2)
+      self.assertFalse(should_be_false1)
+      self.assertFalse(should_be_false2)
 
   def test_run_cond_python(self):
     true_fn = lambda: 2.0
-- 
GitLab


From 9816741186cdf327e1ee9fb048f1573356ac1064 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 10:10:18 -0700
Subject: [PATCH 351/960] Template system improvements: allow replacing keyword
 args. Allow using function calls and dicts in name replacements.

PiperOrigin-RevId: 190083700
---
 tensorflow/contrib/py2tf/pyct/templates.py    | 59 ++++++++++++++++++-
 .../contrib/py2tf/pyct/templates_test.py      | 45 ++++++++++++++
 2 files changed, 101 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/templates.py b/tensorflow/contrib/py2tf/pyct/templates.py
index cdd71dc56d..590be68234 100644
--- a/tensorflow/contrib/py2tf/pyct/templates.py
+++ b/tensorflow/contrib/py2tf/pyct/templates.py
@@ -44,8 +44,6 @@ class ReplaceTransformer(gast.NodeTransformer):
     self.replacements = replacements
     self.in_replacements = False
 
-  # TODO(mdan): Make a more detailed pass and clean up if needed.
-
   def visit_Expr(self, node):
     if (isinstance(node.value, gast.Name) and
         node.value.id in self.replacements):
@@ -53,17 +51,57 @@ class ReplaceTransformer(gast.NodeTransformer):
     self.generic_visit(node)
     return node
 
+  def visit_keyword(self, node):
+    if node.arg in self.replacements:
+      repl = self.replacements[node.arg]
+      if isinstance(repl, gast.keyword):
+        return repl
+      elif (isinstance(repl, (list, tuple)) and repl and
+            all(isinstance(r, gast.keyword) for r in repl)):
+        return repl
+      # TODO(mdan): We may allow replacing with a string as well.
+      # For example, if one wanted to replace foo with bar in foo=baz, then
+      # we could allow changing just node arg, so that we end up with bar=baz.
+      raise ValueError(
+          'a keyword argument may only be replaced by another keyword or a '
+          'non-empty list of keywords. Found: %s' % repl)
+    return self.generic_visit(node)
+
   def visit_FunctionDef(self, node):
     node = self.generic_visit(node)
     if node.name in self.replacements:
       repl = self.replacements[node.name]
       if not isinstance(repl, (gast.Name, ast.Name)):
         raise ValueError(
-            'A function name can only be replaced by a Name node. Found: %s' %
+            'a function name can only be replaced by a Name node. Found: %s' %
             repl)
       node.name = repl.id
     return node
 
+  def _check_has_context(self, node):
+    if not node.ctx:
+      raise ValueError('node %s is missing ctx value' % node)
+
+  def _check_inner_children_have_context(self, node):
+    if isinstance(node, gast.Attribute):
+      self._check_inner_children_have_context(node.value)
+      self._check_has_context(node)
+    elif isinstance(node, gast.Tuple):
+      for e in node.elts:
+        self._check_inner_children_have_context(e)
+      self._check_has_context(node)
+    elif isinstance(node, gast.Dict):
+      for e in node.keys:
+        self._check_inner_children_have_context(e)
+      for e in node.values:
+        self._check_inner_children_have_context(e)
+    elif isinstance(node, gast.Name):
+      self._check_has_context(node)
+    elif isinstance(node, (gast.Str, gast.Num)):
+      pass
+    else:
+      raise ValueError('unexpected node type "%s"' % node)
+
   def _set_inner_child_context(self, node, ctx):
     if isinstance(node, gast.Attribute):
       self._set_inner_child_context(node.value, ctx)
@@ -74,6 +112,21 @@ class ReplaceTransformer(gast.NodeTransformer):
       node.ctx = ctx
     elif isinstance(node, gast.Name):
       node.ctx = ctx
+    elif isinstance(node, gast.Call):
+      self._set_inner_child_context(node.func, ctx)
+      # We may be able to override these to Load(), but for now it's simpler
+      # to just assert that they're set.
+      for a in node.args:
+        self._check_inner_children_have_context(a)
+      for k in node.keywords:
+        self._check_inner_children_have_context(k.value)
+    elif isinstance(node, gast.Dict):
+      # We may be able to override these to Load(), but for now it's simpler
+      # to just assert that they're set.
+      for e in node.keys:
+        self._check_inner_children_have_context(e)
+      for e in node.values:
+        self._check_inner_children_have_context(e)
     elif isinstance(node, (gast.Str, gast.Num)):
       pass
     else:
diff --git a/tensorflow/contrib/py2tf/pyct/templates_test.py b/tensorflow/contrib/py2tf/pyct/templates_test.py
index d7835b80a7..af939caf32 100644
--- a/tensorflow/contrib/py2tf/pyct/templates_test.py
+++ b/tensorflow/contrib/py2tf/pyct/templates_test.py
@@ -23,6 +23,7 @@ import imp
 import gast
 
 from tensorflow.contrib.py2tf.pyct import compiler
+from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.contrib.py2tf.pyct import templates
 from tensorflow.python.platform import test
 
@@ -96,6 +97,50 @@ class TemplatesTest(test.TestCase):
     with self.assertRaises(ValueError):
       templates.replace(template, foo=1)
 
+  def test_replace_call_keyword(self):
+    template = """
+      def test_fn():
+        def f(a, d, f):
+          return a + d + f
+        return f(1, kws=None)
+    """
+
+    source = parser.parse_expression('f(d=3, f=5)')
+    node = templates.replace(template, kws=source.keywords)[0]
+    result, _ = compiler.ast_to_object(node)
+    self.assertEquals(9, result.test_fn())
+
+    with self.assertRaises(ValueError):
+      templates.replace(template, kws=[])
+      templates.replace(template, kws=1)
+
+  def test_replace_name_with_call(self):
+    template = """
+      def test_fn():
+        b = 5
+        def g(a):
+          return 3 * a
+        def f():
+          return g
+        return foo
+    """
+
+    source = parser.parse_expression('f()(b)')
+    node = templates.replace(template, foo=source)[0]
+    result, _ = compiler.ast_to_object(node)
+    self.assertEquals(15, result.test_fn())
+
+  def test_replace_name_with_dict(self):
+    template = """
+      def test_fn():
+        return foo['bar']
+    """
+
+    source = parser.parse_expression('{\'bar\': 3}')
+    node = templates.replace(template, foo=source)[0]
+    result, _ = compiler.ast_to_object(node)
+    self.assertEquals(3, result.test_fn())
+
   def replace_as_expression(self):
     template = """
       foo(a)
-- 
GitLab


From aeb9f62e237ae1274482acca2fa09db34aef42d4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 10:17:18 -0700
Subject: [PATCH 352/960] Omit
 tensorflow/python/estimator:replicate_model_fn_test from asan builds. It gets
 flaky timeouts.

PiperOrigin-RevId: 190084932
---
 tensorflow/python/estimator/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 04fcbb0e87..b25f9d2153 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -929,5 +929,6 @@ cuda_py_test(
     ],
     tags = [
         "multi_gpu",
+        "noasan",  # flaky time outs
     ],
 )
-- 
GitLab


From 010e3e401cef883aa0fff334d3f5e56a88e3f5e4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 10:21:39 -0700
Subject: [PATCH 353/960] Mark tensor as const in a function that does not
 mutate a tensor.

PiperOrigin-RevId: 190085757
---
 tensorflow/contrib/lite/kernels/kernel_util.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/kernel_util.h b/tensorflow/contrib/lite/kernels/kernel_util.h
index 21da1daff7..2f407b5da3 100644
--- a/tensorflow/contrib/lite/kernels/kernel_util.h
+++ b/tensorflow/contrib/lite/kernels/kernel_util.h
@@ -53,13 +53,13 @@ inline TfLiteTensor* GetOptionalInputTensor(TfLiteContext* context,
 }
 
 // Determines whether tensor is constant.
-inline bool IsConstantTensor(TfLiteTensor* tensor) {
+inline bool IsConstantTensor(const TfLiteTensor* tensor) {
   return tensor->allocation_type == kTfLiteMmapRo;
 }
 
 // Determines whether tensor is dynamic. Note that a tensor can be non-const and
 // not dynamic. This function specifically checks for a dynamic tensor.
-inline bool IsDynamicTensor(TfLiteTensor* tensor) {
+inline bool IsDynamicTensor(const TfLiteTensor* tensor) {
   return tensor->allocation_type == kTfLiteDynamic;
 }
 
-- 
GitLab


From ed0c4037ec47e3a7d1e5d23514951e5256b8a30f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 10:21:59 -0700
Subject: [PATCH 354/960] Small cleanup StrCat related number formatting

- Resolve inconsistency in return values (pointer to start vs end of buffer)
- Instead, return the number of chars written as this turns out to be most useful to callers
- Removes the need for redundant strlen calls.

PiperOrigin-RevId: 190085812
---
 tensorflow/core/lib/strings/numbers.cc | 49 +++++++++++++++-----------
 tensorflow/core/lib/strings/numbers.h  | 15 ++++----
 tensorflow/core/lib/strings/strcat.h   | 16 ++++-----
 3 files changed, 43 insertions(+), 37 deletions(-)

diff --git a/tensorflow/core/lib/strings/numbers.cc b/tensorflow/core/lib/strings/numbers.cc
index f5822fad8e..516decc3c0 100644
--- a/tensorflow/core/lib/strings/numbers.cc
+++ b/tensorflow/core/lib/strings/numbers.cc
@@ -106,19 +106,22 @@ T locale_independent_strtonum(const char* str, const char** endptr) {
 
 namespace strings {
 
-char* FastInt32ToBufferLeft(int32 i, char* buffer) {
+size_t FastInt32ToBufferLeft(int32 i, char* buffer) {
   uint32 u = i;
+  size_t length = 0;
   if (i < 0) {
     *buffer++ = '-';
+    ++length;
     // We need to do the negation in modular (i.e., "unsigned")
     // arithmetic; MSVC++ apparently warns for plain "-u", so
     // we write the equivalent expression "0 - u" instead.
     u = 0 - u;
   }
-  return FastUInt32ToBufferLeft(u, buffer);
+  length += FastUInt32ToBufferLeft(u, buffer);
+  return length;
 }
 
-char* FastUInt32ToBufferLeft(uint32 i, char* buffer) {
+size_t FastUInt32ToBufferLeft(uint32 i, char* buffer) {
   char* start = buffer;
   do {
     *buffer++ = ((i % 10) + '0');
@@ -126,19 +129,22 @@ char* FastUInt32ToBufferLeft(uint32 i, char* buffer) {
   } while (i > 0);
   *buffer = 0;
   std::reverse(start, buffer);
-  return buffer;
+  return buffer - start;
 }
 
-char* FastInt64ToBufferLeft(int64 i, char* buffer) {
+size_t FastInt64ToBufferLeft(int64 i, char* buffer) {
   uint64 u = i;
+  size_t length = 0;
   if (i < 0) {
     *buffer++ = '-';
+    ++length;
     u = 0 - u;
   }
-  return FastUInt64ToBufferLeft(u, buffer);
+  length += FastUInt64ToBufferLeft(u, buffer);
+  return length;
 }
 
-char* FastUInt64ToBufferLeft(uint64 i, char* buffer) {
+size_t FastUInt64ToBufferLeft(uint64 i, char* buffer) {
   char* start = buffer;
   do {
     *buffer++ = ((i % 10) + '0');
@@ -146,19 +152,18 @@ char* FastUInt64ToBufferLeft(uint64 i, char* buffer) {
   } while (i > 0);
   *buffer = 0;
   std::reverse(start, buffer);
-  return buffer;
+  return buffer - start;
 }
 
 static const double kDoublePrecisionCheckMax = DBL_MAX / 1.000000000000001;
 
-char* DoubleToBuffer(double value, char* buffer) {
+size_t DoubleToBuffer(double value, char* buffer) {
   // DBL_DIG is 15 for IEEE-754 doubles, which are used on almost all
   // platforms these days.  Just in case some system exists where DBL_DIG
   // is significantly larger -- and risks overflowing our buffer -- we have
   // this assert.
   static_assert(DBL_DIG < 20, "DBL_DIG is too big");
 
-  bool full_precision_needed = true;
   if (std::abs(value) <= kDoublePrecisionCheckMax) {
     int snprintf_result =
         snprintf(buffer, kFastToBufferSize, "%.*g", DBL_DIG, value);
@@ -167,18 +172,20 @@ char* DoubleToBuffer(double value, char* buffer) {
     // larger than the precision we asked for.
     DCHECK(snprintf_result > 0 && snprintf_result < kFastToBufferSize);
 
-    full_precision_needed =
-        locale_independent_strtonum<double>(buffer, nullptr) != value;
+    if (locale_independent_strtonum<double>(buffer, nullptr) == value) {
+      // Round-tripping the string to double works; we're done.
+      return snprintf_result;
+    }
+    // else: full precision formatting needed. Fall through.
   }
 
-  if (full_precision_needed) {
-    int snprintf_result =
-        snprintf(buffer, kFastToBufferSize, "%.*g", DBL_DIG + 2, value);
+  int snprintf_result =
+      snprintf(buffer, kFastToBufferSize, "%.*g", DBL_DIG + 2, value);
 
-    // Should never overflow; see above.
-    DCHECK(snprintf_result > 0 && snprintf_result < kFastToBufferSize);
-  }
-  return buffer;
+  // Should never overflow; see above.
+  DCHECK(snprintf_result > 0 && snprintf_result < kFastToBufferSize);
+
+  return snprintf_result;
 }
 
 namespace {
@@ -325,7 +332,7 @@ bool safe_strtod(const char* str, double* value) {
   return *str != '\0' && *endptr == '\0';
 }
 
-char* FloatToBuffer(float value, char* buffer) {
+size_t FloatToBuffer(float value, char* buffer) {
   // FLT_DIG is 6 for IEEE-754 floats, which are used on almost all
   // platforms these days.  Just in case some system exists where FLT_DIG
   // is significantly larger -- and risks overflowing our buffer -- we have
@@ -347,7 +354,7 @@ char* FloatToBuffer(float value, char* buffer) {
     // Should never overflow; see above.
     DCHECK(snprintf_result > 0 && snprintf_result < kFastToBufferSize);
   }
-  return buffer;
+  return snprintf_result;
 }
 
 string FpToString(Fprint fp) {
diff --git a/tensorflow/core/lib/strings/numbers.h b/tensorflow/core/lib/strings/numbers.h
index 3c45b90274..6b7703be37 100644
--- a/tensorflow/core/lib/strings/numbers.h
+++ b/tensorflow/core/lib/strings/numbers.h
@@ -60,19 +60,18 @@ static const int kFastToBufferSize = 32;
 // the output.  The buffer should typically be at least kFastToBufferSize
 // bytes.
 //
-// Returns a pointer to the end of the string (i.e. the null character
-// terminating the string).
+// Returns the number of characters written.
 // ----------------------------------------------------------------------
 
-char* FastInt32ToBufferLeft(int32 i, char* buffer);    // at least 12 bytes
-char* FastUInt32ToBufferLeft(uint32 i, char* buffer);  // at least 12 bytes
-char* FastInt64ToBufferLeft(int64 i, char* buffer);    // at least 22 bytes
-char* FastUInt64ToBufferLeft(uint64 i, char* buffer);  // at least 22 bytes
+size_t FastInt32ToBufferLeft(int32 i, char* buffer);    // at least 12 bytes
+size_t FastUInt32ToBufferLeft(uint32 i, char* buffer);  // at least 12 bytes
+size_t FastInt64ToBufferLeft(int64 i, char* buffer);    // at least 22 bytes
+size_t FastUInt64ToBufferLeft(uint64 i, char* buffer);  // at least 22 bytes
 
 // Required buffer size for DoubleToBuffer is kFastToBufferSize.
 // Required buffer size for FloatToBuffer is kFastToBufferSize.
-char* DoubleToBuffer(double i, char* buffer);
-char* FloatToBuffer(float i, char* buffer);
+size_t DoubleToBuffer(double value, char* buffer);
+size_t FloatToBuffer(float value, char* buffer);
 
 // Convert a 64-bit fingerprint value to an ASCII representation.
 string FpToString(Fprint fp);
diff --git a/tensorflow/core/lib/strings/strcat.h b/tensorflow/core/lib/strings/strcat.h
index b3ec14e448..fb2cd5bc7e 100644
--- a/tensorflow/core/lib/strings/strcat.h
+++ b/tensorflow/core/lib/strings/strcat.h
@@ -101,22 +101,22 @@ class AlphaNum {
   // A bool ctor would also convert incoming pointers (bletch).
 
   AlphaNum(int i32)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastInt32ToBufferLeft(i32, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastInt32ToBufferLeft(i32, digits_)) {}
   AlphaNum(unsigned int u32)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastUInt32ToBufferLeft(u32, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastUInt32ToBufferLeft(u32, digits_)) {}
   AlphaNum(long x)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastInt64ToBufferLeft(x, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastInt64ToBufferLeft(x, digits_)) {}
   AlphaNum(unsigned long x)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastUInt64ToBufferLeft(x, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastUInt64ToBufferLeft(x, digits_)) {}
   AlphaNum(long long int i64)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastInt64ToBufferLeft(i64, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastInt64ToBufferLeft(i64, digits_)) {}
   AlphaNum(unsigned long long int u64)  // NOLINT(runtime/explicit)
-      : piece_(digits_, FastUInt64ToBufferLeft(u64, digits_) - &digits_[0]) {}
+      : piece_(digits_, FastUInt64ToBufferLeft(u64, digits_)) {}
 
   AlphaNum(float f)  // NOLINT(runtime/explicit)
-      : piece_(digits_, strlen(FloatToBuffer(f, digits_))) {}
+      : piece_(digits_, FloatToBuffer(f, digits_)) {}
   AlphaNum(double f)  // NOLINT(runtime/explicit)
-      : piece_(digits_, strlen(DoubleToBuffer(f, digits_))) {}
+      : piece_(digits_, DoubleToBuffer(f, digits_)) {}
 
   AlphaNum(Hex hex);               // NOLINT(runtime/explicit)
 
-- 
GitLab


From 6fa811a94f3da0c49d69db9b15ea424f84a6431f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 10:28:56 -0700
Subject: [PATCH 355/960] Automated g4 rollback of changelist 189819449

PiperOrigin-RevId: 190087074
---
 tensorflow/contrib/distributions/BUILD        |  6 ++-
 .../kernel_tests/statistical_testing_test.py  | 40 +++++++++----------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 4ddec73ec8..e9c827a618 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -486,7 +486,11 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
-    tags = ["noasan"],  # Was found to time out in asan
+    tags = [
+        "manual",
+        "noasan",
+        "noguitar",
+    ],
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
index fc071c273d..3548ac1807 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
@@ -31,34 +31,30 @@ class StatisticalTestingTest(test.TestCase):
   def test_dkwm_design_mean_one_sample_soundness(self):
     numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
-    def check_soundness(ff, fp):
-      sufficient_n = st.min_num_samples_for_dkwm_mean_test(
-          numbers, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-      detectable_d = st.min_discrepancy_of_true_means_detectable_by_dkwm(
-          sufficient_n, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-      return check_ops.assert_less_equal(detectable_d, numbers)
     with self.test_session() as sess:
-      sess.run([check_soundness(ff, fp)
-                for ff in rates
-                for fp in rates])
+      for ff in rates:
+        for fp in rates:
+          sufficient_n = st.min_num_samples_for_dkwm_mean_test(
+              numbers, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
+          detectable_d = st.min_discrepancy_of_true_means_detectable_by_dkwm(
+              sufficient_n, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
+          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
 
   def test_dkwm_design_mean_two_sample_soundness(self):
     numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
-    def check_soundness(ff, fp):
-      (sufficient_n1,
-       sufficient_n2) = st.min_num_samples_for_dkwm_mean_two_sample_test(
-           numbers, 0., 1., 0., 1.,
-           false_fail_rate=ff, false_pass_rate=fp)
-      d_fn = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample
-      detectable_d = d_fn(
-          sufficient_n1, 0., 1., sufficient_n2, 0., 1.,
-          false_fail_rate=ff, false_pass_rate=fp)
-      return check_ops.assert_less_equal(detectable_d, numbers)
     with self.test_session() as sess:
-      sess.run([check_soundness(ff, fp)
-                for ff in rates
-                for fp in rates])
+      for ff in rates:
+        for fp in rates:
+          (sufficient_n1,
+           sufficient_n2) = st.min_num_samples_for_dkwm_mean_two_sample_test(
+               numbers, 0., 1., 0., 1.,
+               false_fail_rate=ff, false_pass_rate=fp)
+          d_fn = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample
+          detectable_d = d_fn(
+              sufficient_n1, 0., 1., sufficient_n2, 0., 1.,
+              false_fail_rate=ff, false_pass_rate=fp)
+          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
 
   def test_true_mean_confidence_interval_by_dkwm_one_sample(self):
     rng = np.random.RandomState(seed=0)
-- 
GitLab


From 1a6752dddf387d280a6a13c2dc7e2bebf69dab2f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 11:12:10 -0700
Subject: [PATCH 356/960] Adds remaining validations in
 sequence_numeric_column.

PiperOrigin-RevId: 190094883
---
 .../feature_column/sequence_feature_column.py | 32 ++++++++++++++++++-
 .../sequence_feature_column_test.py           | 26 +++++++++++++++
 2 files changed, 57 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
index e60116966f..555beddeaa 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column.py
@@ -166,6 +166,10 @@ def sequence_categorical_column_with_identity(
 
   Returns:
     A `_SequenceCategoricalColumn`.
+
+  Raises:
+    ValueError: if `num_buckets` is less than one.
+    ValueError: if `default_value` is not in range `[0, num_buckets)`.
   """
   return fc._SequenceCategoricalColumn(
       fc.categorical_column_with_identity(
@@ -205,6 +209,10 @@ def sequence_categorical_column_with_hash_bucket(
 
   Returns:
     A `_SequenceCategoricalColumn`.
+
+  Raises:
+    ValueError: `hash_bucket_size` is not greater than 1.
+    ValueError: `dtype` is neither string nor integer.
   """
   return fc._SequenceCategoricalColumn(
       fc.categorical_column_with_hash_bucket(
@@ -257,6 +265,13 @@ def sequence_categorical_column_with_vocabulary_file(
 
   Returns:
     A `_SequenceCategoricalColumn`.
+
+  Raises:
+    ValueError: `vocabulary_file` is missing or cannot be opened.
+    ValueError: `vocabulary_size` is missing or < 1.
+    ValueError: `num_oov_buckets` is a negative integer.
+    ValueError: `num_oov_buckets` and `default_value` are both specified.
+    ValueError: `dtype` is neither string nor integer.
   """
   return fc._SequenceCategoricalColumn(
       fc.categorical_column_with_vocabulary_file(
@@ -311,6 +326,12 @@ def sequence_categorical_column_with_vocabulary_list(
 
   Returns:
     A `_SequenceCategoricalColumn`.
+
+  Raises:
+    ValueError: if `vocabulary_list` is empty, or contains duplicate keys.
+    ValueError: `num_oov_buckets` is a negative integer.
+    ValueError: `num_oov_buckets` and `default_value` are both specified.
+    ValueError: if `dtype` is not integer or string.
   """
   return fc._SequenceCategoricalColumn(
       fc.categorical_column_with_vocabulary_list(
@@ -352,8 +373,17 @@ def sequence_numeric_column(
 
   Returns:
     A `_SequenceNumericColumn`.
+
+  Raises:
+    TypeError: if any dimension in shape is not an int.
+    ValueError: if any dimension in shape is not a positive integer.
+    ValueError: if `dtype` is not convertible to `tf.float32`.
   """
-  # TODO(b/73160931): Add validations.
+  shape = fc._check_shape(shape=shape, key=key)
+  if not (dtype.is_integer or dtype.is_floating):
+    raise ValueError('dtype must be convertible to float. '
+                     'dtype: {}, key: {}'.format(dtype, key))
+
   return _SequenceNumericColumn(
       key,
       shape=shape,
diff --git a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
index b64f086376..88f5d53516 100644
--- a/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
+++ b/tensorflow/contrib/feature_column/python/feature_column/sequence_feature_column_test.py
@@ -662,6 +662,32 @@ class SequenceIndicatorColumnTest(test.TestCase):
 
 class SequenceNumericColumnTest(test.TestCase):
 
+  def test_defaults(self):
+    a = sfc.sequence_numeric_column('aaa')
+    self.assertEqual('aaa', a.key)
+    self.assertEqual('aaa', a.name)
+    self.assertEqual('aaa', a._var_scope_name)
+    self.assertEqual((1,), a.shape)
+    self.assertEqual(0., a.default_value)
+    self.assertEqual(dtypes.float32, a.dtype)
+
+  def test_shape_saved_as_tuple(self):
+    a = sfc.sequence_numeric_column('aaa', shape=[1, 2])
+    self.assertEqual((1, 2), a.shape)
+
+  def test_shape_must_be_positive_integer(self):
+    with self.assertRaisesRegexp(TypeError, 'shape dimensions must be integer'):
+      sfc.sequence_numeric_column('aaa', shape=[1.0])
+
+    with self.assertRaisesRegexp(
+        ValueError, 'shape dimensions must be greater than 0'):
+      sfc.sequence_numeric_column('aaa', shape=[0])
+
+  def test_dtype_is_convertible_to_float(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'dtype must be convertible to float'):
+      sfc.sequence_numeric_column('aaa', dtype=dtypes.string)
+
   def test_get_sequence_dense_tensor(self):
     sparse_input = sparse_tensor.SparseTensorValue(
         # example 0, values [[0.], [1]]
-- 
GitLab


From 8991aeea540da49344ceac6e8f5e092778f410a9 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Thu, 22 Mar 2018 11:21:52 -0700
Subject: [PATCH 357/960] Eager/g3doc: Gradients with respect to constants are
 None and not 0.

Same behavior as tf.gradients() for graphs.
Some discussion of this choice in #783

PiperOrigin-RevId: 190096919
---
 tensorflow/contrib/eager/python/g3doc/guide.md | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md
index df084e9053..11064981c6 100644
--- a/tensorflow/contrib/eager/python/g3doc/guide.md
+++ b/tensorflow/contrib/eager/python/g3doc/guide.md
@@ -273,9 +273,9 @@ assert 6 == df(3.)[0].numpy()
 d2f = tfe.gradients_function(lambda x: df(x)[0])
 assert 2 == d2f(3.)[0].numpy()
 
-# Third order derivative.
+# Third order derivative: Will be None
 d3f = tfe.gradients_function(lambda x : d2f(x)[0])
-assert 0 == d3f(3.)[0].numpy()
+assert None == d3f(3.)[0]
 ```
 
 These functions can be used to train models. For example, consider the following
-- 
GitLab


From cfdd61585769188789280e768fc43fdbba799619 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 22 Mar 2018 11:23:26 -0700
Subject: [PATCH 358/960] Run the grappler optimizer tests both on GPU and CPU

PiperOrigin-RevId: 190097168
---
 tensorflow/core/grappler/optimizers/BUILD | 27 +++++++++++------------
 1 file changed, 13 insertions(+), 14 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index ac29edd213..cfb698969c 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -1,7 +1,7 @@
 licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-load("//tensorflow:tensorflow.bzl", "tf_cc_test_gpu")
+load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 
@@ -44,7 +44,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "static_schedule_test",
     srcs = ["static_schedule_test.cc"],
     deps = [
@@ -79,7 +79,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "auto_parallel_test",
     srcs = ["auto_parallel_test.cc"],
     deps = [
@@ -157,7 +157,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "function_optimizer_test",
     srcs = ["function_optimizer_test.cc"],
     deps = [
@@ -223,7 +223,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "graph_optimizer_stage_test",
     size = "small",
     srcs = ["graph_optimizer_stage_test.cc"],
@@ -274,7 +274,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "arithmetic_optimizer_test",
     size = "small",
     srcs = ["arithmetic_optimizer_test.cc"],
@@ -315,7 +315,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "dependency_optimizer_test",
     size = "small",
     srcs = ["dependency_optimizer_test.cc"],
@@ -351,7 +351,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "model_pruner_test",
     srcs = ["model_pruner_test.cc"],
     deps = [
@@ -422,10 +422,9 @@ cc_library(
     ]),
 )
 
-tf_cc_test_gpu(
+tf_cuda_cc_test(
     name = "memory_optimizer_test",
     srcs = ["memory_optimizer_test.cc"],
-    tags = ["no_cuda_on_cpu_tap"],
     deps = [
         ":memory_optimizer",
         "//tensorflow/cc:cc_ops",
@@ -464,7 +463,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "layout_optimizer_test",
     srcs = ["layout_optimizer_test.cc"],
     deps = [
@@ -513,7 +512,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "meta_optimizer_test",
     srcs = ["meta_optimizer_test.cc"],
     deps = [
@@ -542,7 +541,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "custom_graph_optimizer_registry_test",
     size = "small",
     srcs = ["custom_graph_optimizer_registry_test.cc"],
@@ -577,7 +576,7 @@ cc_library(
     ],
 )
 
-tf_cc_test(
+tf_cuda_cc_test(
     name = "loop_optimizer_test",
     srcs = ["loop_optimizer_test.cc"],
     deps = [
-- 
GitLab


From 7c4cdb8bae0e8760ebe4793d49ea5aee68768655 Mon Sep 17 00:00:00 2001
From: Yu-Cheng Ling <ycling@google.com>
Date: Thu, 22 Mar 2018 11:25:49 -0700
Subject: [PATCH 359/960] Supports PReLU in TFLite & Toco.

PiperOrigin-RevId: 190097557
---
 tensorflow/contrib/lite/builtin_ops.h         |   1 +
 .../contrib/lite/kernels/activations.cc       |  64 ++++++++++
 .../contrib/lite/kernels/activations_test.cc  |  43 +++++++
 tensorflow/contrib/lite/kernels/register.cc   |   2 +
 tensorflow/contrib/lite/model.cc              |   1 +
 tensorflow/contrib/lite/nnapi_delegate.cc     |   1 +
 tensorflow/contrib/lite/schema/schema.fbs     |   1 +
 .../contrib/lite/schema/schema_generated.h    |   9 +-
 tensorflow/contrib/lite/testing/BUILD         |   1 +
 .../contrib/lite/testing/generate_examples.py |  49 ++++++++
 .../testing/generated_examples_zip_test.cc    |   4 +
 tensorflow/contrib/lite/toco/BUILD            |   1 +
 .../graph_transformations.h                   |   1 +
 .../graph_transformations/identify_prelu.cc   | 119 ++++++++++++++++++
 .../propagate_fixed_sizes.cc                  |   1 +
 tensorflow/contrib/lite/toco/model.h          |  13 ++
 .../contrib/lite/toco/tflite/operator.cc      |   2 +
 tensorflow/contrib/lite/toco/toco_tooling.cc  |   1 +
 tensorflow/contrib/lite/toco/tooling_util.cc  |   1 +
 19 files changed, 312 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/identify_prelu.cc

diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index e4652a3e70..d7993e60cc 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -78,6 +78,7 @@ typedef enum {
   kTfLiteBuiltinDelegate = 51,
   kTfLiteBuiltinBidirectionalSequenceLstm = 52,
   kTfLiteBuiltinCast = 53,
+  kTfLiteBuiltinPrelu = 54,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/kernels/activations.cc b/tensorflow/contrib/lite/kernels/activations.cc
index 093761c43c..39a54c9396 100644
--- a/tensorflow/contrib/lite/kernels/activations.cc
+++ b/tensorflow/contrib/lite/kernels/activations.cc
@@ -150,6 +150,34 @@ TfLiteStatus SoftmaxPrepare(TfLiteContext* context, TfLiteNode* node) {
                                TfLiteIntArrayCopy(input->dims));
 }
 
+TfLiteStatus PreluPrepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+  TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* output = GetOutput(context, node, 0);
+  TfLiteTensor* alpha = GetInput(context, node, 1);
+
+  output->type = input->type;
+
+  // Currently only Float32 is supported
+  // TODO(ycling): Support other data types.
+  TF_LITE_ENSURE_EQ(context, input->type, kTfLiteFloat32);
+  TF_LITE_ENSURE_EQ(context, alpha->type, kTfLiteFloat32);
+
+  // Currently, only support 4D `input` and 3D `alpha` with shape
+  // (1, 1, channels).
+  // TODO(impjdi): Support other cases where `alpha` is broadcastable
+  // to `input`.
+  TF_LITE_ENSURE_EQ(context, input->dims->size, 4);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->size, 3);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[0], 1);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[1], 1);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[2], input->dims->data[3]);
+
+  return context->ResizeTensor(context, output,
+                               TfLiteIntArrayCopy(input->dims));
+}
+
 TfLiteStatus ReluEval(TfLiteContext* context, TfLiteNode* node) {
   TfLiteTensor* input = GetInput(context, node, 0);
   TfLiteTensor* output = GetOutput(context, node, 0);
@@ -388,6 +416,35 @@ TfLiteStatus LogSoftmaxEval(TfLiteContext* context, TfLiteNode* node) {
   }
 }
 
+TfLiteStatus PreluEval(TfLiteContext* context, TfLiteNode* node) {
+  TfLiteTensor* input = GetInput(context, node, 0);
+  TfLiteTensor* alpha = GetInput(context, node, 1);
+  TfLiteTensor* output = GetOutput(context, node, 0);
+
+  if (input->type != kTfLiteFloat32) {
+    context->ReportError(context, "Only float32 supported currently.");
+    return kTfLiteError;
+  }
+  TF_LITE_ENSURE_EQ(context, input->dims->size, 4);
+  const int batches = input->dims->data[0];
+  const int height = input->dims->data[1];
+  const int width = input->dims->data[2];
+  const int channels = input->dims->data[3];
+
+  TF_LITE_ENSURE_EQ(context, alpha->dims->size, 3);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[0], 1);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[1], 1);
+  TF_LITE_ENSURE_EQ(context, alpha->dims->data[2], channels);
+
+  const int n = batches * height * width * channels;
+  for (int i = 0; i < n; ++i) {
+    const float x = input->data.f[i];
+    output->data.f[i] = x >= 0.0f ? x : alpha->data.f[i % channels] * x;
+  }
+
+  return kTfLiteOk;
+}
+
 }  // namespace activations
 
 TfLiteRegistration* Register_RELU() {
@@ -439,6 +496,13 @@ TfLiteRegistration* Register_LOG_SOFTMAX() {
   return &r;
 }
 
+TfLiteRegistration* Register_PRELU() {
+  static TfLiteRegistration r = {/*init=*/nullptr, /*free=*/nullptr,
+                                 activations::PreluPrepare,
+                                 activations::PreluEval};
+  return &r;
+}
+
 }  // namespace builtin
 }  // namespace ops
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/activations_test.cc b/tensorflow/contrib/lite/kernels/activations_test.cc
index b9a96e3f79..50a84edd47 100644
--- a/tensorflow/contrib/lite/kernels/activations_test.cc
+++ b/tensorflow/contrib/lite/kernels/activations_test.cc
@@ -383,6 +383,49 @@ TEST(FloatActivationsOpTest, LogSoftmax) {
                               })));
 }
 
+class PReluOpModel : public SingleOpModel {
+ public:
+  PReluOpModel(const TensorData& input, const TensorData& alpha) {
+    input_ = AddInput(input);
+    alpha_ = AddInput(alpha);
+    output_ = AddOutput(input);
+    SetBuiltinOp(BuiltinOperator_PRELU, BuiltinOptions_NONE, 0);
+    BuildInterpreter({GetShape(input_), GetShape(alpha_)});
+  }
+  void SetInput(std::initializer_list<float> data) {
+    PopulateTensor(input_, data);
+  }
+  void SetAlpha(std::initializer_list<float> data) {
+    PopulateTensor(alpha_, data);
+  }
+  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+
+ protected:
+  int input_;
+  int alpha_;
+  int output_;
+};
+
+TEST(FloatActivationsOpTest, PRelu) {
+  PReluOpModel m({TensorType_FLOAT32, {1, 2, 2, 3}},
+                 {TensorType_FLOAT32, {1, 1, 3}});
+
+  m.SetInput({
+      0.0f, 0.0f, 0.0f,     // Row 1, Column 1
+      1.0f, 1.0f, 1.0f,     // Row 1, Column 2
+      -1.0f, -1.0f, -1.0f,  // Row 2, Column 1
+      -2.0f, -2.0f, -2.0f,  // Row 1, Column 2
+  });
+  m.SetAlpha({0.0f, 1.0f, 2.0f});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({
+                                 0.0f, 0.0f, 0.0f,    // Row 1, Column 1
+                                 1.0f, 1.0f, 1.0f,    // Row 1, Column 2
+                                 0.0f, -1.0f, -2.0f,  // Row 2, Column 1
+                                 0.0f, -2.0f, -4.0f,  // Row 1, Column 2
+                             }));
+}
+
 }  // namespace
 }  // namespace tflite
 
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 369d3b9886..62045f0a4d 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -75,6 +75,7 @@ TfLiteRegistration* Register_TOPK_V2();
 TfLiteRegistration* Register_LOG_SOFTMAX();
 TfLiteRegistration* Register_CAST();
 TfLiteRegistration* Register_DEQUANTIZE();
+TfLiteRegistration* Register_PRELU();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -131,6 +132,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_LOG_SOFTMAX, Register_LOG_SOFTMAX());
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
   AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
+  AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 9c619f88e0..b7ccdf070b 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -309,6 +309,7 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_LOG_SOFTMAX:
     case BuiltinOperator_CAST:
     case BuiltinOperator_DEQUANTIZE:
+    case BuiltinOperator_PRELU:
       break;
     case BuiltinOperator_LSH_PROJECTION: {
       TfLiteLSHProjectionParams* params =
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index 9d00d965d3..e31b7c03a5 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -349,6 +349,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
       case tflite::BuiltinOperator_DEQUANTIZE:
       case tflite::BuiltinOperator_DELEGATE:
       case tflite::BuiltinOperator_CAST:
+      case tflite::BuiltinOperator_PRELU:
         FATAL("Op code %d is currently not delegated to NNAPI", builtin);
         nn_op_type = -1;  // set to invalid
         break;
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 04387fed33..e1075971e9 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -130,6 +130,7 @@ enum BuiltinOperator : byte {
   DELEGATE = 51,
   BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   CAST = 53,
+  PRELU = 54,
 }
 
 // Options for the builtin operators.
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index b922de2081..86daeaf5cc 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -254,11 +254,12 @@ enum BuiltinOperator {
   BuiltinOperator_DELEGATE = 51,
   BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   BuiltinOperator_CAST = 53,
+  BuiltinOperator_PRELU = 54,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_CAST
+  BuiltinOperator_MAX = BuiltinOperator_PRELU
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[52] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -311,7 +312,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[52] {
     BuiltinOperator_LOG_SOFTMAX,
     BuiltinOperator_DELEGATE,
     BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
-    BuiltinOperator_CAST
+    BuiltinOperator_CAST,
+    BuiltinOperator_PRELU
   };
   return values;
 }
@@ -372,6 +374,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "DELEGATE",
     "BIDIRECTIONAL_SEQUENCE_LSTM",
     "CAST",
+    "PRELU",
     nullptr
   };
   return names;
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index f1b18ad30f..555ea90034 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -39,6 +39,7 @@ gen_zipped_test_files(
         "mean.zip",
         "mul.zip",
         "pad.zip",
+        "prelu.zip",
         "relu.zip",
         "relu1.zip",
         "relu6.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 420bdb41f1..38de9dcf2c 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -617,6 +617,54 @@ def make_relu6_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_prelu_tests(zip_path):
+  """Make a set of tests to do PReLU."""
+
+  test_parameters = [{
+      # The canonical case for image processing is having a 4D `input` (NHWC)
+      # and `shared_axes`=[1, 2], so the alpha parameter is per channel.
+      "input_shape": [[1, 10, 10, 3], [3, 3, 3, 3]],
+      "shared_axes": [[1, 2], [1]],
+  }]
+
+  def build_graph(parameters):
+    """Build the graph for the test case."""
+
+    input_tensor = tf.placeholder(
+        dtype=tf.float32, name="input", shape=parameters["input_shape"])
+    prelu = tf.keras.layers.PReLU(shared_axes=parameters["shared_axes"])
+    out = prelu(input_tensor)
+    return [input_tensor], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    """Build the inputs for the test case."""
+
+    input_shape = parameters["input_shape"]
+    input_values = create_tensor_data(
+        np.float32, input_shape, min_value=-10, max_value=10)
+    shared_axes = parameters["shared_axes"]
+
+    alpha_shape = []
+    for dim in range(1, len(input_shape)):
+      alpha_shape.append(1 if dim in shared_axes else input_shape[dim])
+
+    alpha_values = create_tensor_data(np.float32, alpha_shape)
+
+    with tf.variable_scope("", reuse=True):
+      alpha = tf.get_variable("p_re_lu/alpha")
+      sess.run(alpha.assign(alpha_values))
+
+    return [input_values], sess.run(
+        outputs, feed_dict=dict(zip(inputs, [input_values])))
+
+  make_zip_of_tests(
+      zip_path,
+      test_parameters,
+      build_graph,
+      build_inputs,
+      use_frozen_graph=True)
+
+
 # This function tests various TensorFLow functions that generates Const op,
 # including `tf.ones`, `tf.zeros` and random functions.
 def make_constant_tests(zip_path):
@@ -1911,6 +1959,7 @@ def main(unused_args):
         "relu.zip": make_relu_tests,
         "relu1.zip": make_relu1_tests,
         "relu6.zip": make_relu6_tests,
+        "prelu.zip": make_prelu_tests,
         "l2_pool.zip": make_pool_tests(make_l2_pool),
         "avg_pool.zip": make_pool_tests(tf.nn.avg_pool),
         "max_pool.zip": make_pool_tests(tf.nn.max_pool),
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 5e76e7c510..ba2d259462 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -88,6 +88,9 @@ std::map<string, string> kBrokenTests = {
 
     // Transpose only supports 1D-4D input tensors.
     {R"(^\/transpose.*input_shape=\[.,.,.,.,.\])", "71545879"},
+
+    // PRelu only supports 4D input with (1, 1, channels) 3D alpha now.
+    {R"(^\/prelu.*shared_axes=\[1\])", "75975192"},
 };
 
 // Allows test data to be unzipped into a temporary directory and makes
@@ -253,6 +256,7 @@ INSTANTIATE_TESTS(mul)
 INSTANTIATE_TESTS(pad)
 INSTANTIATE_TESTS(relu)
 INSTANTIATE_TESTS(relu1)
+INSTANTIATE_TESTS(prelu)
 INSTANTIATE_TESTS(relu6)
 INSTANTIATE_TESTS(reshape)
 INSTANTIATE_TESTS(resize_bilinear)
diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 395abc5326..486ff1edcd 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -193,6 +193,7 @@ cc_library(
         "graph_transformations/identify_lstm.cc",
         "graph_transformations/identify_lstm_merge_inputs.cc",
         "graph_transformations/identify_lstm_split_inputs.cc",
+        "graph_transformations/identify_prelu.cc",
         "graph_transformations/identify_relu1.cc",
         "graph_transformations/lstm_utils.cc",
         "graph_transformations/make_initial_dequantize_operator.cc",
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 11e5e19f50..640afc7c74 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -129,6 +129,7 @@ DECLARE_GRAPH_TRANSFORMATION(IdentifyLstmCell)
 DECLARE_GRAPH_TRANSFORMATION(SplitLstmCellInputs)
 DECLARE_GRAPH_TRANSFORMATION(MergeLstmCellInputs)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyRelu1)
+DECLARE_GRAPH_TRANSFORMATION(IdentifyPRelu)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyDilatedConv)
 DECLARE_GRAPH_TRANSFORMATION(MakeInitialDequantizeOperator)
 DECLARE_GRAPH_TRANSFORMATION(PropagateActivationFunctionIntoConstants)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/identify_prelu.cc b/tensorflow/contrib/lite/toco/graph_transformations/identify_prelu.cc
new file mode 100644
index 0000000000..30be4ac0aa
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/identify_prelu.cc
@@ -0,0 +1,119 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+// This transformation rule tries to identify the PRelu structure generated by
+// Keras, and convert it to a single op.
+//
+// The formula of PReLU is:
+// f(x) = alpha * x for x < 0, f(x) = x for x >= 0.
+//
+// `x` is the input, and `alpha` is a trainable tensor which can be broadcasted
+// to the shape of `x`.
+//
+// There's no native PRelu op in TensorFlow, so Keras generates the following
+// structure which does the equivalent calculation:
+// f(x) = Relu(x) + (-alpha * Relu(-x))
+//
+// Practically, alpha is always a constant in the inference graph, and Toco have
+// other graph transformations which fold the activation functions to other ops.
+// Therefore, we're looking for the structure:
+//
+// f(x) = Relu(x) + (negative_alpha * Neg(x, activation=Relu))
+
+namespace toco {
+
+bool IdentifyPRelu::Run(Model* model, std::size_t op_index) {
+  const auto add_op_it = model->operators.begin() + op_index;
+  const auto* add_op = add_op_it->get();
+  if (add_op == nullptr || add_op->type != OperatorType::kAdd ||
+      add_op->inputs.size() != 2 ||
+      add_op->fused_activation_function != FusedActivationFunctionType::kNone) {
+    return false;
+  }
+
+  const auto* relu_input_op = GetOpWithOutput(*model, add_op->inputs[0]);
+  if (relu_input_op == nullptr || relu_input_op->type != OperatorType::kRelu ||
+      relu_input_op->inputs.size() != 1 ||
+      relu_input_op->fused_activation_function !=
+          FusedActivationFunctionType::kNone) {
+    return false;
+  }
+
+  // TODO(ycling): Both Add and Mul are commutative. Support the case where
+  // the position of operands are exchanged.
+  const auto* mul_op = GetOpWithOutput(*model, add_op->inputs[1]);
+  if (mul_op == nullptr || mul_op->type != OperatorType::kMul ||
+      mul_op->inputs.size() != 2 ||
+      mul_op->fused_activation_function != FusedActivationFunctionType::kNone) {
+    return false;
+  }
+
+  const auto neg_alpha_tensor_name = mul_op->inputs[0];
+
+  const auto* relu_neg_input_op = GetOpWithOutput(*model, mul_op->inputs[1]);
+
+  if (relu_neg_input_op == nullptr ||
+      relu_neg_input_op->type != OperatorType::kNeg ||
+      relu_neg_input_op->fused_activation_function !=
+          FusedActivationFunctionType::kRelu ||
+      relu_neg_input_op->inputs.size() != 1) {
+    return false;
+  }
+
+  if (relu_input_op->inputs[0] != relu_neg_input_op->inputs[0]) {
+    return false;
+  }
+
+  const auto input_tensor_name = relu_input_op->inputs[0];
+  const auto output_tensor_name = add_op->outputs[0];
+
+  // Construct a tensor for positive alpha (double negative).
+  const auto alpha_tensor_name =
+      AvailableArrayName(*model, neg_alpha_tensor_name + "_neg");
+  model->GetOrCreateArray(alpha_tensor_name);
+
+  auto* neg_neg_alpha_op = new NegOperator;
+  neg_neg_alpha_op->inputs = {neg_alpha_tensor_name};
+  neg_neg_alpha_op->outputs = {alpha_tensor_name};
+  model->operators.emplace(add_op_it, neg_neg_alpha_op);
+
+  auto* prelu_op = new PReluOperator;
+  prelu_op->inputs = {input_tensor_name, alpha_tensor_name};
+  prelu_op->outputs = {output_tensor_name};
+  model->operators.emplace(add_op_it, prelu_op);
+  AddMessageF("Creating %s replacing equivalent subgraph", LogName(*prelu_op));
+
+  DeleteArrayIfUsedOnce(neg_alpha_tensor_name, model);
+  DeleteArrayIfUsedOnce(add_op->inputs[0], model);
+  DeleteArrayIfUsedOnce(add_op->inputs[1], model);
+  DeleteArrayIfUsedOnce(mul_op->inputs[1], model);
+  // Remove the existing Add op that outputs the final result. If the other
+  // intermediate tensors aren't used by other ops, those will be removed by
+  // other graph transformation rules.
+  model->operators.erase(FindOp(*model, add_op));
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 375848a7d4..676736cfc5 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -1467,6 +1467,7 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
     case OperatorType::kRelu:
     case OperatorType::kRelu1:
     case OperatorType::kRelu6:
+    case OperatorType::kPRelu:
     case OperatorType::kSoftmax:
     case OperatorType::kLogSoftmax:
     case OperatorType::kLogistic:
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 3fa0089cba..5199e292e1 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -65,6 +65,7 @@ enum class OperatorType {
   kRelu,
   kRelu1,
   kRelu6,
+  kPRelu,
   kSoftmax,
   kLogSoftmax,
   kSub,
@@ -566,6 +567,18 @@ struct Relu6Operator : Operator {
   Relu6Operator() : Operator(OperatorType::kRelu6) {}
 };
 
+// PRelu
+//   f(x) = alpha * x for x < 0, f(x) = x for x >= 0.
+//
+// Inputs:
+//   inputs[0]: required: the input array
+//   inputs[1]: required: the alpha array
+//
+// Equivalent to keras.layers.PReLU.
+struct PReluOperator : Operator {
+  PReluOperator() : Operator(OperatorType::kPRelu) {}
+};
+
 // Element-wise Logistic operator:
 //   x -> Logistic(x) = 1 / (1 + exp(-x))
 //
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index f2cc4ef71f..f23249cfa1 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -854,6 +854,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
       new SimpleOperator<Relu1Operator>("RELU_N1_TO_1", OperatorType::kRelu1));
   ops.emplace_back(
       new SimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6));
+  ops.emplace_back(
+      new SimpleOperator<Relu1Operator>("PRELU", OperatorType::kPRelu));
   ops.emplace_back(new SimpleOperator<LogisticOperator>(
       "LOGISTIC", OperatorType::kLogistic));
   ops.emplace_back(
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index ca66110ba3..30dd6fab9e 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -94,6 +94,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new IdentifyL2Normalization);
   transformations->Add(new IdentifyL2Pool);
   transformations->Add(new IdentifyRelu1);
+  transformations->Add(new IdentifyPRelu);
   transformations->Add(new RemoveTrivialBinaryOperator);
   transformations->Add(new ReadFakeQuantMinMax);
   transformations->Add(new ResolveSpaceToBatchNDAttributes);
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 2362206a14..ec1770c129 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -300,6 +300,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(Relu)
     HANDLE_OPERATORTYPENAME_CASE(Relu1)
     HANDLE_OPERATORTYPENAME_CASE(Relu6)
+    HANDLE_OPERATORTYPENAME_CASE(PRelu)
     HANDLE_OPERATORTYPENAME_CASE(ReorderAxes)
     HANDLE_OPERATORTYPENAME_CASE(Softmax)
     HANDLE_OPERATORTYPENAME_CASE(LogSoftmax)
-- 
GitLab


From 282750fee5e2df502436ca9ef6a95283f8adab34 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Thu, 22 Mar 2018 11:25:58 -0700
Subject: [PATCH 360/960] Add new Ops for ScopedAllocator and the associated
 Concat and Split.  The ScopedAllocatorOp allocates a large backing tensor
 whose slices may be concatenated or splitted with ScopedAllocatorConcatOp and
 ScopedAllocatorSplitOp respectively.  These ops should only be added via
 Grappler optimizations on the dataflow graph provided by the user.

PiperOrigin-RevId: 190097586
---
 tensorflow/core/BUILD                         |   3 +
 .../core/common_runtime/gpu/gpu_device.cc     |  11 +
 .../core/common_runtime/gpu/gpu_device.h      |  15 +-
 .../core/common_runtime/scoped_allocator.cc   |   3 +-
 .../common_runtime/scoped_allocator_mgr.cc    |  25 +-
 .../common_runtime/scoped_allocator_mgr.h     |   6 +-
 .../scoped_allocator_mgr_test.cc              |  25 +-
 .../core/common_runtime/threadpool_device.cc  |  16 +-
 .../core/common_runtime/threadpool_device.h   |  12 +-
 tensorflow/core/framework/allocator.h         |  16 +-
 tensorflow/core/framework/device_base.h       |  16 +-
 tensorflow/core/framework/op_kernel.cc        |   9 +-
 tensorflow/core/kernels/BUILD                 |  37 +++
 .../core/kernels/scoped_allocator_ops.cc      | 216 +++++++++++++
 .../core/kernels/scoped_allocator_ops_test.cc | 296 ++++++++++++++++++
 tensorflow/core/ops/scoped_allocator_ops.cc   |  81 +++++
 16 files changed, 742 insertions(+), 45 deletions(-)
 create mode 100644 tensorflow/core/kernels/scoped_allocator_ops.cc
 create mode 100644 tensorflow/core/kernels/scoped_allocator_ops_test.cc
 create mode 100644 tensorflow/core/ops/scoped_allocator_ops.cc

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index a14eeed1a5..15cbba8285 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -634,6 +634,7 @@ tf_gen_op_libs(
         "random_ops",
         "remote_fused_graph_ops",
         "resource_variable_ops",
+        "scoped_allocator_ops",
         "sdca_ops",
         "set_ops",
         "script_ops",
@@ -717,6 +718,7 @@ cc_library(
         ":random_ops_op_lib",
         ":remote_fused_graph_ops_op_lib",
         ":resource_variable_ops_op_lib",
+        ":scoped_allocator_ops_op_lib",
         ":script_ops_op_lib",
         ":sdca_ops_op_lib",
         ":sendrecv_ops_op_lib",
@@ -861,6 +863,7 @@ cc_library(
         "//tensorflow/core/kernels:remote_fused_graph_ops",
         "//tensorflow/core/kernels:required",
         "//tensorflow/core/kernels:resource_variable_ops",
+        "//tensorflow/core/kernels:scoped_allocator_ops",
         "//tensorflow/core/kernels:sdca_ops",
         "//tensorflow/core/kernels:set_kernels",
         "//tensorflow/core/kernels:sparse",
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 8357cc5a72..52fd20e479 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -840,6 +840,17 @@ void BaseGPUDevice::ReinitializeGpuDevice(OpKernelContext* context,
   }
 }
 
+Allocator* BaseGPUDevice::GetScopedAllocator(AllocatorAttributes attr,
+                                             int64 step_id) {
+  if (attr.scope_id > 0) {
+    return scoped_allocator_mgr_->GetContainer(step_id)->GetInstance(
+        attr.scope_id);
+  }
+  LOG(FATAL) << "Unexpected call to BaseGPUDevice::GetScopedAllocator "
+             << "attr.scope_id = " << attr.scope_id;
+  return gpu_allocator_;
+}
+
 const int BaseGPUDeviceFactory::InterconnectMap::kSameDeviceStrength = 1000;
 const int BaseGPUDeviceFactory::InterconnectMap::kStreamExecutorStrength = 1;
 
diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.h b/tensorflow/core/common_runtime/gpu/gpu_device.h
index d817c7dd1f..cc5c3881dd 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.h
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.h
@@ -17,8 +17,8 @@ limitations under the License.
 #error This file must only be included when building with Cuda support
 #endif
 
-#ifndef TENSORFLOW_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
-#define TENSORFLOW_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
 
 #include <memory>
 #include <string>
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/gpu/gpu_id_utils.h"
 #include "tensorflow/core/common_runtime/gpu_device_context.h"
 #include "tensorflow/core/common_runtime/local_device.h"
+#include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/device_base.h"
 #include "tensorflow/core/framework/op_kernel.h"
@@ -95,11 +96,19 @@ class BaseGPUDevice : public LocalDevice {
   // corresponds to the cuda context.
   gpu::StreamExecutor* executor() const { return executor_; }
 
+  Allocator* GetScopedAllocator(AllocatorAttributes attr,
+                                int64 step_id) override;
+
+  ScopedAllocatorMgr* GetScopedAllocatorMgr() const override {
+    return scoped_allocator_mgr_.get();
+  }
+
  protected:
   Allocator* gpu_allocator_;  // not owned
   Allocator* cpu_allocator_;  // not owned
 
   gpu::StreamExecutor* executor_;  // not owned
+  std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 
  private:
   struct StreamGroup {
@@ -205,4 +214,4 @@ class BaseGPUDeviceFactory : public DeviceFactory {
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_GPU_GPU_DEVICE_H_
diff --git a/tensorflow/core/common_runtime/scoped_allocator.cc b/tensorflow/core/common_runtime/scoped_allocator.cc
index 31e7a5e3e2..a26672b79d 100644
--- a/tensorflow/core/common_runtime/scoped_allocator.cc
+++ b/tensorflow/core/common_runtime/scoped_allocator.cc
@@ -75,7 +75,8 @@ void* ScopedAllocator::AllocateRaw(int32 field_index, size_t num_bytes) {
   if (num_bytes != f.bytes) {
     LOG(ERROR) << "ScopedAllocator " << name_ << " got request for "
                << num_bytes << " bytes from field " << field_index
-               << " which has precalculated size " << f.bytes;
+               << " which has precalculated size " << f.bytes << " and offset "
+               << f.offset;
     return nullptr;
   }
 
diff --git a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
index d0d05c6d1b..e1f70404e3 100644
--- a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
+++ b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
@@ -22,7 +22,7 @@ namespace tensorflow {
 Status ScopedAllocatorContainer::AddScopedAllocator(
     const Tensor& backing_tensor, int32 scope_id, const string& scope_name,
     const gtl::ArraySlice<ScopedAllocator::Field>& fields,
-    int32 expected_call_count, ScopedAllocator** sa_ptr) {
+    int32 expected_call_count) {
   VLOG(1) << "AddScopedAllocator " << mgr_->device_name()
           << " step_id_=" << step_id_ << " scope_id=" << scope_id;
   mutex_lock l(mu_);
@@ -41,17 +41,17 @@ Status ScopedAllocatorContainer::AddScopedAllocator(
     }
   }
   VLOG(2) << " container " << this << " step_id " << step_id_;
-  *sa_ptr = new ScopedAllocator(backing_tensor, scope_id, scope_name, fields,
-                                expected_call_count, this);
-  allocators_[scope_id] = ScopedAllocatorContainer::SAField(
-      ScopedAllocator::kBackingIndex, *sa_ptr);
+  ScopedAllocator* sa = new ScopedAllocator(
+      backing_tensor, scope_id, scope_name, fields, expected_call_count, this);
+  allocators_[scope_id] =
+      ScopedAllocatorContainer::SAField(ScopedAllocator::kBackingIndex, sa);
   VLOG(2) << "#fields " << fields.size();
   for (int i = 0; i < fields.size(); ++i) {
     const ScopedAllocator::Field& f = fields[i];
     VLOG(2) << "Adding instance with for " << mgr_->device_name()
             << " scope_id=" << f.scope_id;
     allocators_[f.scope_id] = ScopedAllocatorContainer::SAField(
-        i, new ScopedAllocatorInstance(*sa_ptr, i));
+        i, new ScopedAllocatorInstance(sa, i));
   }
   return Status::OK();
 }
@@ -154,23 +154,26 @@ Status ScopedAllocatorMgr::AddScopedAllocator(
     const Tensor& backing_tensor, int64 step_id, int32 scope_id,
     const string& scope_name,
     const gtl::ArraySlice<ScopedAllocator::Field>& fields,
-    int32 expected_call_count, ScopedAllocator** sa_ptr) {
+    int32 expected_call_count) {
   ScopedAllocatorContainer* sac = GetContainer(step_id);
   return sac->AddScopedAllocator(backing_tensor, scope_id, scope_name, fields,
-                                 expected_call_count, sa_ptr);
+                                 expected_call_count);
 }
 
 void ScopedAllocatorMgr::PopulateFields(
-    int32 scope_id, const gtl::ArraySlice<TensorShape>& shapes, DataType dtype,
-    std::vector<ScopedAllocator::Field>* fields) {
+    int32 scope_id, const gtl::ArraySlice<TensorShape>& shapes,
+    const DataType dtype, std::vector<ScopedAllocator::Field>* fields) {
   const int32 num_fields = static_cast<int32>(shapes.size());
   fields->resize(num_fields);
   size_t offset = 0;
   for (int32 i = 0; i < num_fields; ++i) {
-    size_t bytes = shapes[i].num_elements() * sizeof(dtype);
+    size_t bytes = shapes[i].num_elements() * DataTypeSize(dtype);
     (*fields)[i].scope_id = scope_id + 1 + i;
     (*fields)[i].bytes = bytes;
     (*fields)[i].offset = offset;
+    VLOG(1) << "field=" << i << " scope_id=" << (*fields)[i].scope_id
+            << " bytes=" << (*fields)[i].bytes
+            << " offset=" << (*fields)[i].offset;
     offset += bytes;
     size_t overshoot = offset % Allocator::kAllocatorAlignment;
     if (overshoot > 0) {
diff --git a/tensorflow/core/common_runtime/scoped_allocator_mgr.h b/tensorflow/core/common_runtime/scoped_allocator_mgr.h
index 4d5bc23dd9..effc5f2d77 100644
--- a/tensorflow/core/common_runtime/scoped_allocator_mgr.h
+++ b/tensorflow/core/common_runtime/scoped_allocator_mgr.h
@@ -34,7 +34,7 @@ class ScopedAllocatorContainer : public core::RefCounted {
   Status AddScopedAllocator(
       const Tensor& backing_tensor, int32 scope_id, const string& scope_name,
       const gtl::ArraySlice<ScopedAllocator::Field>& fields,
-      int32 expected_call_count, ScopedAllocator** sa_ptr);
+      int32 expected_call_count);
 
   ScopedAllocatorInstance* GetInstance(int32 scope_id);
   ScopedAllocator* GetAllocator(int32 scope_id);
@@ -83,7 +83,7 @@ class ScopedAllocatorMgr {
       const Tensor& backing_tensor, int64 step_id, int32 scope_id,
       const string& scope_name,
       const gtl::ArraySlice<ScopedAllocator::Field>& fields,
-      int32 expected_call_count, ScopedAllocator** sa_ptr);
+      int32 expected_call_count);
 
   void Cleanup(int64 step_id);
 
@@ -91,7 +91,7 @@ class ScopedAllocatorMgr {
   // consecutive scope_id values following that of the base ScopedAllocator.
   static void PopulateFields(int32 scope_id,
                              const gtl::ArraySlice<TensorShape>& shapes,
-                             DataType dtype,
+                             const DataType dtype,
                              std::vector<ScopedAllocator::Field>* fields);
 
   const string& device_name() const { return device_name_; }
diff --git a/tensorflow/core/common_runtime/scoped_allocator_mgr_test.cc b/tensorflow/core/common_runtime/scoped_allocator_mgr_test.cc
index 81cb3e7979..38e07e47f2 100644
--- a/tensorflow/core/common_runtime/scoped_allocator_mgr_test.cc
+++ b/tensorflow/core/common_runtime/scoped_allocator_mgr_test.cc
@@ -25,7 +25,7 @@ namespace {
 
 class ScopedAllocatorMgrTest : public ::testing::Test {
  public:
-  ScopedAllocatorMgrTest() : sam_("CPU0"), sa_(nullptr) {}
+  ScopedAllocatorMgrTest() : sam_("CPU0") {}
 
   void InitTensor() {
     backing_tensor_ = Tensor(cpu_allocator(), DT_FLOAT, backing_tensor_shape_);
@@ -42,7 +42,7 @@ class ScopedAllocatorMgrTest : public ::testing::Test {
             << " expected_use_count " << expected_use_count;
     return sam_.AddScopedAllocator(backing_tensor_, step_id_, scope_id,
                                    "tensor_shape_599", fields_,
-                                   expected_use_count, &sa_);
+                                   expected_use_count);
   }
 
   Status PrepScopedAllocatorMgr(int expected_use_count) {
@@ -87,7 +87,6 @@ class ScopedAllocatorMgrTest : public ::testing::Test {
   std::vector<TensorShape> fields_shapes_;
   std::vector<ScopedAllocator::Field> fields_;
   ScopedAllocatorMgr sam_;
-  ScopedAllocator* sa_;
   const int step_id_ = 101;
   const int scope_id_ = 599;
   std::vector<ScopedAllocatorInstance*> sa_instances_;
@@ -138,9 +137,9 @@ TEST_F(ScopedAllocatorMgrTest, ContainerAddAllocator) {
 
   // Cleanup the instances by invoking allocate and deallocate.
   void* ptr0 =
-      sa_instances_[0]->AllocateRaw(0 /* alignment */, 512 * sizeof(DT_FLOAT));
+      sa_instances_[0]->AllocateRaw(0 /* alignment */, 512 * sizeof(float));
   void* ptr1 =
-      sa_instances_[1]->AllocateRaw(0 /* alignment */, 512 * sizeof(DT_FLOAT));
+      sa_instances_[1]->AllocateRaw(0 /* alignment */, 512 * sizeof(float));
   sa_instances_[0]->DeallocateRaw(ptr0);
   sa_instances_[1]->DeallocateRaw(ptr1);
 }
@@ -153,7 +152,6 @@ TEST_F(ScopedAllocatorMgrTest, AllocatorSuccess) {
   fields_shapes_ = std::vector<TensorShape>({{512}, {3, 3}, {2, 256}});
   Status s = PrepScopedAllocatorMgr(3);
   other = sac->GetAllocator(scope_id_);
-  EXPECT_EQ(other, sa_);
 
   ScopedAllocatorInstance* inst0 = sac->GetInstance(scope_id_ + 1);
   char* ptr0 = static_cast<char*>(inst0->AllocateRaw(0, 512 * sizeof(float)));
@@ -187,8 +185,7 @@ TEST_F(ScopedAllocatorMgrTest, AllocatorInitFail) {
   fields_.resize(1);
   fields_[0].scope_id = scope_id_ + 1;
   fields_[0].offset = 0;
-  fields_[0].bytes =
-      backing_tensor_shape_.num_elements() * 2 * sizeof(DT_FLOAT);
+  fields_[0].bytes = backing_tensor_shape_.num_elements() * 2 * sizeof(float);
   // fields[0].offset + fields[0].bytes is larger than the size of the backing
   // tensor, so this check should fail
   EXPECT_DEATH(Status s = AddScopedAllocator(1, scope_id_), "");
@@ -208,20 +205,20 @@ TEST_F(ScopedAllocatorMgrTest, AllocatorFail) {
   // so we need to explicitly delete the instances to avoid a memleak.
   SaveInstances(fields_shapes_.size());
 
-  char* ptr0 = static_cast<char*>(
-      sa_instances_[0]->AllocateRaw(0, 512 * sizeof(DT_FLOAT)));
+  char* ptr0 =
+      static_cast<char*>(sa_instances_[0]->AllocateRaw(0, 512 * sizeof(float)));
   VLOG(2) << "Should fail because we deallocate ptr="
           << static_cast<void*>(ptr0 + 8) << " which we never allocated.";
   EXPECT_DEATH(sa_instances_[0]->DeallocateRaw(ptr0 + 8), "");
   VLOG(2) << "Should fail because we allocate smaller than the size of the "
           << "field.";
-  EXPECT_EQ(nullptr, sa_instances_[1]->AllocateRaw(0, 256 * sizeof(DT_FLOAT)));
+  EXPECT_EQ(nullptr, sa_instances_[1]->AllocateRaw(0, 256 * sizeof(float)));
   VLOG(2) << "Should fail because we allocate larger than the size of the "
           << "field.";
-  EXPECT_EQ(nullptr, sa_instances_[1]->AllocateRaw(0, 1024 * sizeof(DT_FLOAT)));
-  void* ptr1 = sa_instances_[1]->AllocateRaw(0, 512 * sizeof(DT_FLOAT));
+  EXPECT_EQ(nullptr, sa_instances_[1]->AllocateRaw(0, 1024 * sizeof(float)));
+  void* ptr1 = sa_instances_[1]->AllocateRaw(0, 512 * sizeof(float));
   VLOG(2) << "Should fail because we exceed expected_use_count.";
-  EXPECT_EQ(nullptr, sa_instances_[0]->AllocateRaw(0, 512 * sizeof(DT_FLOAT)));
+  EXPECT_EQ(nullptr, sa_instances_[0]->AllocateRaw(0, 512 * sizeof(float)));
   sa_instances_[0]->DeallocateRaw(ptr0);
   sa_instances_[1]->DeallocateRaw(ptr1);
 }
diff --git a/tensorflow/core/common_runtime/threadpool_device.cc b/tensorflow/core/common_runtime/threadpool_device.cc
index 5aa01376ab..6d8de6a3c0 100644
--- a/tensorflow/core/common_runtime/threadpool_device.cc
+++ b/tensorflow/core/common_runtime/threadpool_device.cc
@@ -16,6 +16,8 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/threadpool_device.h"
 
 #include "tensorflow/core/common_runtime/local_device.h"
+#include "tensorflow/core/common_runtime/scoped_allocator.h"
+#include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/allocator_registry.h"
 #include "tensorflow/core/framework/device_base.h"
@@ -40,7 +42,8 @@ ThreadPoolDevice::ThreadPoolDevice(const SessionOptions& options,
                                    Allocator* allocator)
     : LocalDevice(options, Device::BuildDeviceAttributes(
                                name, DEVICE_CPU, memory_limit, locality)),
-      allocator_(allocator) {}
+      allocator_(allocator),
+      scoped_allocator_mgr_(new ScopedAllocatorMgr(name)) {}
 
 ThreadPoolDevice::~ThreadPoolDevice() {}
 
@@ -65,6 +68,17 @@ Allocator* ThreadPoolDevice::GetAllocator(AllocatorAttributes attr) {
   return allocator_;
 }
 
+Allocator* ThreadPoolDevice::GetScopedAllocator(AllocatorAttributes attr,
+                                                int64 step_id) {
+  if (attr.scope_id > 0) {
+    return scoped_allocator_mgr_->GetContainer(step_id)->GetInstance(
+        attr.scope_id);
+  }
+  LOG(FATAL) << "Unexpected call to ThreadPoolDevice::GetScopedAllocator "
+             << "attr.scope_id = " << attr.scope_id;
+  return allocator_;
+}
+
 Status ThreadPoolDevice::MakeTensorFromProto(
     const TensorProto& tensor_proto, const AllocatorAttributes alloc_attrs,
     Tensor* tensor) {
diff --git a/tensorflow/core/common_runtime/threadpool_device.h b/tensorflow/core/common_runtime/threadpool_device.h
index 37cb745a0a..afc5d15ebc 100644
--- a/tensorflow/core/common_runtime/threadpool_device.h
+++ b/tensorflow/core/common_runtime/threadpool_device.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
-#define TENSORFLOW_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
 
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/local_device.h"
@@ -31,6 +31,11 @@ class ThreadPoolDevice : public LocalDevice {
 
   void Compute(OpKernel* op_kernel, OpKernelContext* context) override;
   Allocator* GetAllocator(AllocatorAttributes attr) override;
+  Allocator* GetScopedAllocator(AllocatorAttributes attr,
+                                int64 step_id) override;
+  ScopedAllocatorMgr* GetScopedAllocatorMgr() const override {
+    return scoped_allocator_mgr_.get();
+  }
   Status MakeTensorFromProto(const TensorProto& tensor_proto,
                              const AllocatorAttributes alloc_attrs,
                              Tensor* tensor) override;
@@ -39,8 +44,9 @@ class ThreadPoolDevice : public LocalDevice {
 
  private:
   Allocator* allocator_;  // Not owned
+  std::unique_ptr<ScopedAllocatorMgr> scoped_allocator_mgr_;
 };
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_THREADPOOL_DEVICE_H_
diff --git a/tensorflow/core/framework/allocator.h b/tensorflow/core/framework/allocator.h
index 3ce1b61246..2c87156dca 100644
--- a/tensorflow/core/framework/allocator.h
+++ b/tensorflow/core/framework/allocator.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_FRAMEWORK_ALLOCATOR_H_
-#define TENSORFLOW_FRAMEWORK_ALLOCATOR_H_
+#ifndef TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
+#define TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
 
 #include <stdlib.h>
 
@@ -359,7 +359,12 @@ struct AllocatorAttributes {
   bool nic_compatible() const { return value & (0x1 << 1); }
   void set_gpu_compatible(bool v) { value |= (static_cast<int>(v) << 2); }
   bool gpu_compatible() const { return value & (0x1 << 2); }
-  void Merge(AllocatorAttributes other) { value |= other.value; }
+  void Merge(AllocatorAttributes other) {
+    value |= other.value;
+    scope_id = (scope_id > 0 && other.scope_id == 0)
+                   ? scope_id
+                   : ((scope_id == 0) ? other.scope_id : 0);
+  }
   // Returns true if the fields set in *this is a subset of or equal to
   // those set in other.
   bool IsEqualOrLessRestrictiveThan(const AllocatorAttributes& other) const {
@@ -371,6 +376,9 @@ struct AllocatorAttributes {
   // upper 8 bits in device-specific ways, and ops implemented for those
   // devices are responsible for setting those 8 bits appropriately.
   uint32 value = 0;
+  // EXPERIMENTAL: If this is greater than zero, then allocation is delegated to
+  // a named special-purpose allocator on the same device.
+  int32 scope_id = 0;
 };
 
 // Returns a trivial implementation of Allocator which uses the system
@@ -396,4 +404,4 @@ class SubAllocator {
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_FRAMEWORK_ALLOCATOR_H_
+#endif  // TENSORFLOW_CORE_FRAMEWORK_ALLOCATOR_H_
diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h
index fb6d5c69e1..52b9077d8c 100644
--- a/tensorflow/core/framework/device_base.h
+++ b/tensorflow/core/framework/device_base.h
@@ -13,8 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_FRAMEWORK_DEVICE_BASE_H_
-#define TENSORFLOW_FRAMEWORK_DEVICE_BASE_H_
+#ifndef TENSORFLOW_CORE_FRAMEWORK_DEVICE_BASE_H_
+#define TENSORFLOW_CORE_FRAMEWORK_DEVICE_BASE_H_
 
 #include <memory>
 #include <string>
@@ -48,6 +48,7 @@ class Env;
 class EventMgr;
 class OpKernelContext;
 class ResourceMgr;
+class ScopedAllocatorMgr;
 class TensorProto;
 
 namespace thread {
@@ -179,6 +180,15 @@ class DeviceBase {
     return GetAllocator(attr);
   }
 
+  // Return an Allocator prepared for use in particular places by graph
+  // optimization
+  virtual Allocator* GetScopedAllocator(AllocatorAttributes attr,
+                                        int64 step_id) {
+    LOG(FATAL) << "Device does not implement GetScopedAllocator()";
+  }
+
+  virtual ScopedAllocatorMgr* GetScopedAllocatorMgr() const { return nullptr; }
+
   virtual const Eigen::ThreadPoolDevice* eigen_cpu_device() {
     CHECK(eigen_cpu_device_ != nullptr);
     return eigen_cpu_device_;
@@ -243,4 +253,4 @@ class DeviceBase {
 
 }  // namespace tensorflow
 
-#endif  // TENSORFLOW_FRAMEWORK_DEVICE_BASE_H_
+#endif  // TENSORFLOW_CORE_FRAMEWORK_DEVICE_BASE_H_
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 8654437059..9ec1c213c3 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -282,8 +282,13 @@ OpKernelContext::~OpKernelContext() {
 }
 
 Allocator* OpKernelContext::get_allocator(AllocatorAttributes attr) {
-  Allocator* allocator =
-      params_->device->GetStepAllocator(attr, resource_manager());
+  Allocator* allocator = nullptr;
+  if (attr.scope_id > 0) {
+    allocator = params_->device->GetScopedAllocator(attr, step_id());
+    CHECK(allocator);
+  } else {
+    allocator = params_->device->GetStepAllocator(attr, resource_manager());
+  }
   if (track_allocations()) {
     mutex_lock lock(mu_);
     for (const auto& wrapped : wrapped_allocators_) {
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 2e39f25fc1..f6137fb860 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -1666,6 +1666,43 @@ tf_kernel_library(
     ],
 )
 
+tf_kernel_library(
+    name = "scoped_allocator_ops",
+    prefix = "scoped_allocator_ops",
+    deps = [
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:scoped_allocator_ops_op_lib",
+    ],
+)
+
+tf_cuda_cc_test(
+    name = "scoped_allocator_ops_test",
+    srcs = ["scoped_allocator_ops_test.cc"],
+    linkstatic = tf_kernel_tests_linkstatic(),  #Required for benchmarking
+    deps = [
+        ":cwise_op",
+        ":dense_update_ops",
+        ":ops_testutil",
+        ":ops_util",
+        ":scoped_allocator_ops",
+        ":variable_ops",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:core_cpu_internal",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:math_ops_op_lib",
+        "//tensorflow/core:proto_text",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_kernel_library(
     name = "session_ops",
     prefix = "session_ops",
diff --git a/tensorflow/core/kernels/scoped_allocator_ops.cc b/tensorflow/core/kernels/scoped_allocator_ops.cc
new file mode 100644
index 0000000000..d7b25ffad0
--- /dev/null
+++ b/tensorflow/core/kernels/scoped_allocator_ops.cc
@@ -0,0 +1,216 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/dma_helper.h"
+#include "tensorflow/core/common_runtime/scoped_allocator.h"
+#include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+class ScopedAllocatorOp : public OpKernel {
+ public:
+  explicit ScopedAllocatorOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_));
+    OP_REQUIRES_OK(context, context->GetAttr("shapes", &shapes_));
+    OP_REQUIRES_OK(context, context->GetAttr("sa_name", &name_));
+    OP_REQUIRES_OK(context, context->GetAttr("id", &id_));
+    OP_REQUIRES_OK(context, context->GetAttr("expected_call_count",
+                                             &expected_call_count_));
+    device_ = context->device();
+    // Precalculate the size of the backing tensor and the offsets of
+    // the subtensors to be allocated from it, taking into account
+    // alignment considerations.
+    ScopedAllocatorMgr::PopulateFields(id_, shapes_, dtype_, &fields_);
+    size_t num_bytes = fields_.back().offset + fields_.back().bytes;
+    num_elements_ = num_bytes / DataTypeSize(dtype_);
+    OP_REQUIRES(context, num_bytes % DataTypeSize(dtype_) == 0,
+                errors::InvalidArgument(
+                    "Number of bytes ", num_bytes,
+                    " must be divisible by size of datatype ", dtype_));
+  }
+
+  void Compute(OpKernelContext* context) override {
+    ScopedAllocatorMgr* sam = device_->GetScopedAllocatorMgr();
+    if (!sam) {
+      context->SetStatus(errors::Internal(
+          "ScopedAllocatorMgr not supported on device ", device_->name()));
+      return;
+    }
+    Tensor* backing_tensor = nullptr;
+    AllocatorAttributes attr = context->output_alloc_attr(0);
+    Status s =
+        context->allocate_output(0, {num_elements_}, &backing_tensor, attr);
+    VLOG(1) << "_ScopedAllocatorOp new backing tensor size "
+            << backing_tensor->TotalBytes() << " num_elements_ "
+            << num_elements_ << " buffer " << DMAHelper::buffer(backing_tensor)
+            << " base addr " << DMAHelper::base(backing_tensor);
+    if (s.ok()) {
+      s = sam->AddScopedAllocator(*backing_tensor, context->step_id(), id_,
+                                  name_, fields_, expected_call_count_);
+    }
+    if (!s.ok()) {
+      context->SetStatus(s);
+    }
+  }
+
+ private:
+  std::vector<TensorShape> shapes_;
+  DataType dtype_;
+  int64 num_elements_;
+  std::vector<ScopedAllocator::Field> fields_;
+  string name_;
+  int32 id_;
+  int32 expected_call_count_;
+  DeviceBase* device_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocator").Device(DEVICE_CPU),
+                        ScopedAllocatorOp);
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocator").Device(DEVICE_GPU),
+                        ScopedAllocatorOp);
+
+class ScopedAllocatorConcatOp : public OpKernel {
+ public:
+  explicit ScopedAllocatorConcatOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("shape", &shape_));
+    OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_));
+    // This stuff is just for debugging
+    OP_REQUIRES_OK(context, context->GetAttr("sa_name", &name_));
+    OP_REQUIRES_OK(context, context->GetAttr("id", &id_));
+    device_ = context->device();
+  }
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& backing_tensor = context->input(0);
+    // Check that type matches.
+    OP_REQUIRES(
+        context, backing_tensor.dtype() == dtype_,
+        errors::InvalidArgument("Backing tensor type ", backing_tensor.dtype(),
+                                " does not match expected type ", dtype_));
+    // Check that backing tensor is at least as large as the shape of the
+    // output.
+    OP_REQUIRES(context, backing_tensor.NumElements() >= shape_.num_elements(),
+                errors::InvalidArgument("Backing tensor num elements ",
+                                        backing_tensor.NumElements(),
+                                        " is not equal to expected ",
+                                        shape_.num_elements()));
+    VLOG(1) << "_ScopedAllocatorConcatOp outputting backing tensor at "
+            << DMAHelper::base(&backing_tensor);
+    Tensor backing_copy(backing_tensor);
+    context->set_output(0, backing_copy);
+    const TensorBuffer* backing_buf = DMAHelper::buffer(&backing_copy);
+    const void* backing_tensor_lb = backing_buf->data();
+    const void* backing_tensor_ub = static_cast<const void*>(
+        static_cast<const char*>(backing_tensor_lb) + backing_buf->size());
+    // Check that all inputs lie entirely within the backing tensor.
+    for (int i = 1; i < context->num_inputs(); ++i) {
+      const TensorBuffer* input_buf = DMAHelper::buffer(&context->input(i));
+      const void* input_lb = input_buf->data();
+      OP_REQUIRES(
+          context, input_lb >= backing_tensor_lb,
+          errors::InvalidArgument("Lower bound check fail for input ", i,
+                                  " to node ", context->op_kernel().name()));
+      const void* input_ub = static_cast<const void*>(
+          static_cast<const char*>(input_lb) + input_buf->size());
+      OP_REQUIRES(
+          context, input_ub <= backing_tensor_ub,
+          errors::InvalidArgument("Upper bound check fail for input ", i,
+                                  " to node ", context->op_kernel().name()));
+    }
+  }
+
+ private:
+  TensorShape shape_;
+  DataType dtype_;
+  string name_;
+  int32 id_;
+  DeviceBase* device_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocatorConcat").Device(DEVICE_CPU),
+                        ScopedAllocatorConcatOp);
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocatorConcat").Device(DEVICE_GPU),
+                        ScopedAllocatorConcatOp);
+
+class ScopedAllocatorSplitOp : public OpKernel {
+ public:
+  explicit ScopedAllocatorSplitOp(OpKernelConstruction* context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("T", &dtype_));
+    // This stuff is just for debugging
+    OP_REQUIRES_OK(context, context->GetAttr("sa_name", &name_));
+    OP_REQUIRES_OK(context, context->GetAttr("id", &id_));
+    device_ = context->device();
+  }
+
+  void Compute(OpKernelContext* context) override {
+    Tensor backing_copy(context->input(0));
+    // Check that type matches.
+    OP_REQUIRES(
+        context, backing_copy.dtype() == dtype_,
+        errors::InvalidArgument("Backing tensor type ", backing_copy.dtype(),
+                                " does not match expected type ", dtype_));
+    const TensorBuffer* backing_buf = DMAHelper::buffer(&backing_copy);
+    const void* backing_tensor_lb = backing_buf->data();
+    const void* backing_tensor_ub = static_cast<const void*>(
+        static_cast<const char*>(backing_tensor_lb) + backing_buf->size());
+    for (int i = 1; i < context->num_inputs(); ++i) {
+      VLOG(1) << "_ScopedAllocatorSplitOp assigning input " << i
+              << " to output " << i - 1 << " buf addr "
+              << DMAHelper::base(&context->input(i));
+      Tensor copy(context->input(i));
+      OP_REQUIRES(
+          context, copy.dtype() == dtype_,
+          errors::InvalidArgument("Input ", i, " tensor type ", copy.dtype(),
+                                  " does not match expected type ", dtype_));
+      context->set_output(i - 1, copy);
+      const TensorBuffer* input_buf = DMAHelper::buffer(&copy);
+      const void* input_lb = input_buf->data();
+      OP_REQUIRES(
+          context, input_lb >= backing_tensor_lb,
+          errors::InvalidArgument("Lower bound check fail for input ", i,
+                                  " to node ", context->op_kernel().name()));
+      const void* input_ub = static_cast<const void*>(
+          static_cast<const char*>(input_lb) + input_buf->size());
+      OP_REQUIRES(
+          context, input_ub <= backing_tensor_ub,
+          errors::InvalidArgument("Upper bound check fail for input ", i,
+                                  " to node ", context->op_kernel().name()));
+    }
+  }
+
+ private:
+  DataType dtype_;
+  string name_;
+  int32 id_;
+  DeviceBase* device_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocatorSplit").Device(DEVICE_CPU),
+                        ScopedAllocatorSplitOp);
+
+REGISTER_KERNEL_BUILDER(Name("_ScopedAllocatorSplit").Device(DEVICE_GPU),
+                        ScopedAllocatorSplitOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/scoped_allocator_ops_test.cc b/tensorflow/core/kernels/scoped_allocator_ops_test.cc
new file mode 100644
index 0000000000..3d36c8b7d4
--- /dev/null
+++ b/tensorflow/core/kernels/scoped_allocator_ops_test.cc
@@ -0,0 +1,296 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <vector>
+
+#include "tensorflow/core/common_runtime/dma_helper.h"
+#include "tensorflow/core/common_runtime/kernel_benchmark_testlib.h"
+#include "tensorflow/core/common_runtime/scoped_allocator.h"
+#include "tensorflow/core/common_runtime/scoped_allocator_mgr.h"
+#include "tensorflow/core/framework/fake_input.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/graph/testlib.h"
+#include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/platform/test_benchmark.h"
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+class ScopedAllocatorOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(const gtl::ArraySlice<TensorShape>& shapes, DataType dtype,
+              const string& name, int32 id, int32 expected_call_count) {
+    TF_EXPECT_OK(NodeDefBuilder("scoped_allocator_op", "_ScopedAllocator")
+                     .Attr("T", dtype)
+                     .Attr("shapes", shapes)
+                     .Attr("sa_name", name)
+                     .Attr("id", id)
+                     .Attr("expected_call_count", expected_call_count)
+                     .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Allocate and Deallocate the tensors so that memory is not leaked
+    AllocatorAttributes attr;
+    Allocator* allocator;
+    for (size_t i = 0; i < shapes.size(); i++) {
+      attr.scope_id = id + i + 1;
+      allocator = device_->GetScopedAllocator(attr, context_->step_id());
+      Tensor temp(allocator, dtype, shapes[i]);
+    }
+  }
+};
+
+TEST_F(ScopedAllocatorOpTest, Simple) {
+  MakeOp({TensorShape({8})}, DT_FLOAT, "test", 120, 1);
+  MakeOp({TensorShape({32, 32})}, DT_DOUBLE, "test1", 130, 1);
+  MakeOp({TensorShape({64}), TensorShape({3, 3}), TensorShape({5, 5, 5})},
+         DT_HALF, "test2", 140, 3);
+  MakeOp({TensorShape({512}), TensorShape({64, 8})}, DT_UINT32, "test3", 150,
+         2);
+}
+
+// PrepOp is common to ConcatOp tests and SplitOpTests.
+// It allocates a backing tensor that is large enough to hold all slices defined
+// by fields, creates ScopedAllocatorInstances for each field, allocates the
+// tensors, and assigns them as inputs to the op.
+// We won't use the AddInput* suite of functions from ops_testutil.h because
+// they allocate new tensors for each input.  We need to mimic what a
+// ScopedAllocator would do.
+void PrepOp(DataType dtype, int32 id,
+            const std::vector<TensorShape>& fields_shapes,
+            std::vector<ScopedAllocator::Field>* fields,
+            Tensor** backing_tensor, Allocator* allocator,
+            ScopedAllocatorMgr* sam, const string& op_name,
+            std::vector<Tensor>* tensors,
+            gtl::InlinedVector<TensorValue, 4>* inputs,
+            const DataTypeVector& input_types) {
+  ScopedAllocatorMgr::PopulateFields(id, fields_shapes, dtype, fields);
+  // We don't simply allocate a tensor with shape as backing_tensor_shape,
+  // because we need to account for padding in the fields.  We actually need a
+  // tensor of size at least (fields[-1].offset + fields[-1].bytes).
+  size_t num_bytes = fields->back().offset + fields->back().bytes;
+  int32_t num_elements = num_bytes / DataTypeSize(dtype);
+  CHECK_EQ(num_bytes % DataTypeSize(dtype), 0);
+
+  *backing_tensor = new Tensor(allocator, dtype, {num_elements});
+  int64 step_id = 10;
+  Status s = sam->AddScopedAllocator(**backing_tensor, step_id, id,
+                                     "sa_" + op_name + "_test", *fields,
+                                     fields_shapes.size());
+  TF_ASSERT_OK(s);
+
+  ScopedAllocatorContainer* sac = sam->GetContainer(step_id);
+  std::vector<ScopedAllocatorInstance*> sa_instances(fields_shapes.size(),
+                                                     nullptr);
+  for (size_t i = 0; i < fields_shapes.size(); i++) {
+    sa_instances[i] = sac->GetInstance(id + i + 1);
+    tensors->push_back(Tensor(sa_instances[i], dtype, fields_shapes[i]));
+  }
+  // Now add the tensor as an input to ScopedAllocator<op_name>Op.
+  // Order matters here, so first add the backing tensor, then the slices.
+  inputs->reserve(1 + tensors->size());
+  CHECK_GT(input_types.size(), inputs->size());
+  CHECK_EQ(input_types[inputs->size()], dtype);
+  inputs->push_back({nullptr, *backing_tensor});
+  for (size_t i = 0; i < tensors->size(); i++) {
+    CHECK_EQ(input_types[inputs->size()], dtype);
+    inputs->push_back({nullptr, &((*tensors)[i])});
+  }
+}
+
+class ScopedAllocatorConcatOpTest : public OpsTestBase {
+ protected:
+  void MakeOp(const TensorShape& shape, DataType dtype, const string& name,
+              int32 id, int32 num_tensors) {
+    TF_EXPECT_OK(
+        NodeDefBuilder("scoped_allocator_concat_op", "_ScopedAllocatorConcat")
+            .Attr("shape", shape)
+            .Attr("T", dtype)
+            .Attr("N", num_tensors)
+            .Attr("sa_name", name)
+            .Attr("id", id)
+            .Input(FakeInput(dtype))               // backing tensor
+            .Input(FakeInput(num_tensors, dtype))  // list of tensors
+            .Finalize(node_def()));
+    TF_EXPECT_OK(InitOp());
+  }
+
+  void ExecOp(DataType dtype, int32 id,
+              const std::vector<TensorShape>& fields_shapes) {
+    Tensor* backing_tensor = nullptr;
+    std::vector<Tensor> tensors;
+    std::vector<ScopedAllocator::Field> fields;
+    PrepOp(dtype, id, fields_shapes, &fields, &backing_tensor, allocator(),
+           device_->GetScopedAllocatorMgr(), "split", &tensors, &inputs_,
+           input_types_);
+
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Check input and output are same tensor.
+    const Tensor& input = context_->input(0);
+    OpOutputList output_list;
+    Status s = context_->output_list("output", &output_list);
+    TF_ASSERT_OK(s);
+    const Tensor& output = *(output_list[0]);
+    CHECK_EQ(DMAHelper::base(&input), DMAHelper::base(&output));
+    CHECK_EQ(input.dtype(), output.dtype());
+    CHECK_EQ(input.NumElements(), output.NumElements());
+
+    // Free the backing tensor which was allocated in PrepOp.
+    delete backing_tensor;
+  }
+};
+
+TEST_F(ScopedAllocatorConcatOpTest, Success1) {
+  MakeOp({32}, DT_FLOAT, "test", 120, 2);
+  ExecOp(DT_FLOAT, 120, {{16}, {16}});
+}
+
+TEST_F(ScopedAllocatorConcatOpTest, Success2) {
+  MakeOp({2, 2, 2}, DT_DOUBLE, "test", 120, 2);
+  ExecOp(DT_DOUBLE, 120, {{2, 2}, {2, 2}});
+}
+
+TEST_F(ScopedAllocatorConcatOpTest, Success3) {
+  MakeOp({3, 3, 3}, DT_HALF, "test", 120, 3);
+  ExecOp(DT_HALF, 120, {{3, 3}, {3, 3}, {3, 3}});
+}
+
+TEST_F(ScopedAllocatorConcatOpTest, FailDtypeCheck) {
+  MakeOp({8}, DT_FLOAT, "test", 120, 2);
+  EXPECT_DEATH(ExecOp(DT_DOUBLE, 120, {{4}, {4}}), "");
+}
+
+TEST_F(ScopedAllocatorConcatOpTest, FailNumElementsCheck) {
+  MakeOp({32}, DT_FLOAT, "test", 120, 2);
+  AddInputFromArray<float>({8}, {0, 1, 2, 3, 4, 5, 6, 7});
+  AddInputFromArray<float>({4}, {0, 1, 2, 3});
+  AddInputFromArray<float>({4}, {4, 5, 6, 7});
+  Status s = RunOpKernel();
+  EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
+}
+
+// This test should fail because the backing tensor and the input tensors are
+// unrelated, i.e. the inputs are not slices of the backing tensor.
+TEST_F(ScopedAllocatorConcatOpTest, FailBounds) {
+  MakeOp({8}, DT_DOUBLE, "test", 120, 2);
+  AddInputFromArray<double>({8}, {0, 1, 2, 3, 4, 5, 6, 7});
+  AddInputFromArray<double>({4}, {0, 1, 2, 3});
+  AddInputFromArray<double>({4}, {4, 5, 6, 7});
+  Status s = RunOpKernel();
+  EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
+}
+
+class ScopedAllocatorSplitOpTest : public OpsTestBase {
+ protected:
+  void BuildNodeDef(const TensorShape& shape, DataType dtype,
+                    const string& name, int32 id, int32 num_tensors) {
+    TF_EXPECT_OK(
+        NodeDefBuilder("scoped_allocator_split_op", "_ScopedAllocatorSplit")
+            .Attr("T", dtype)
+            .Attr("N", num_tensors)
+            .Attr("sa_name", name)
+            .Attr("id", id)
+            .Input(FakeInput(dtype))  // backing tensor and input
+            .Input(
+                FakeInput(num_tensors, dtype))  // list of subtensors to forward
+            .Finalize(node_def()));
+  }
+
+  void MakeOp(const TensorShape& shape, DataType dtype, const string& name,
+              int32 id, int32 num_tensors) {
+    BuildNodeDef(shape, dtype, name, id, num_tensors);
+    TF_EXPECT_OK(InitOp());
+  }
+
+  // Similar to ConcatOpTest, we add inputs that are allocated from
+  // ScopedAllocator so that the memory lines up nicely.
+  void ExecOp(DataType dtype, int32 id,
+              const std::vector<TensorShape>& fields_shapes) {
+    Tensor* backing_tensor = nullptr;
+    std::vector<Tensor> tensors;
+    std::vector<ScopedAllocator::Field> fields;
+    PrepOp(dtype, id, fields_shapes, &fields, &backing_tensor, allocator(),
+           device_->GetScopedAllocatorMgr(), "split", &tensors, &inputs_,
+           input_types_);
+
+    TF_ASSERT_OK(RunOpKernel());
+
+    // Check that outputs are slices of backing tensor.
+    const Tensor& input = context_->input(0);
+    const void* lower_limit = DMAHelper::base(&input);
+    const char* lower_limit_c =
+        static_cast<const char*>(lower_limit);  // for pointer arithmetic
+    OpOutputList output_list;
+    Status s = context_->output_list("output", &output_list);
+    TF_ASSERT_OK(s);
+    for (int i = 0; i < output_list.size(); i++) {
+      const Tensor& output = *(output_list[i]);
+      const void* expected_base =
+          static_cast<const void*>(lower_limit_c + fields[i].offset);
+      CHECK_EQ(output.dtype(), input.dtype());
+      CHECK_EQ(expected_base, DMAHelper::base(&output));
+      CHECK_EQ(output.NumElements(), fields_shapes[i].num_elements());
+    }
+
+    // Free the backing tensor which was allocated in PrepOp.
+    delete backing_tensor;
+  }
+};
+
+TEST_F(ScopedAllocatorSplitOpTest, Success1) {
+  MakeOp({32}, DT_FLOAT, "test", 120, 2);
+  ExecOp(DT_FLOAT, 120, {{16}, {16}});
+}
+
+TEST_F(ScopedAllocatorSplitOpTest, Success2) {
+  MakeOp({2, 2, 2}, DT_DOUBLE, "test", 120, 2);
+  ExecOp(DT_DOUBLE, 120, {{2, 2}, {2, 2}});
+}
+
+TEST_F(ScopedAllocatorSplitOpTest, Success3) {
+  MakeOp({3, 3, 3}, DT_HALF, "test", 120, 3);
+  ExecOp(DT_HALF, 120, {{3, 3}, {3, 3}, {3, 3}});
+}
+
+TEST_F(ScopedAllocatorSplitOpTest, FailNLessThan2) {
+  BuildNodeDef({4, 4}, DT_FLOAT, "test", 120, 1);
+  Status s = InitOp();
+  EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
+}
+
+TEST_F(ScopedAllocatorSplitOpTest, FailDtypeCheck) {
+  MakeOp({8}, DT_FLOAT, "test", 120, 2);
+  EXPECT_DEATH(ExecOp(DT_HALF, 120, {{4}, {4}}), "");
+}
+
+TEST_F(ScopedAllocatorSplitOpTest, FailBounds) {
+  MakeOp({8}, DT_DOUBLE, "test", 120, 2);
+  AddInputFromArray<double>({8}, {0, 1, 2, 3, 4, 5, 6, 7});
+  AddInputFromArray<double>({4}, {0, 1, 2, 3});
+  AddInputFromArray<double>({4}, {4, 5, 6, 7});
+  Status s = RunOpKernel();
+  EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
+}
+
+}  // end namespace tensorflow
diff --git a/tensorflow/core/ops/scoped_allocator_ops.cc b/tensorflow/core/ops/scoped_allocator_ops.cc
new file mode 100644
index 0000000000..f053a53f4c
--- /dev/null
+++ b/tensorflow/core/ops/scoped_allocator_ops.cc
@@ -0,0 +1,81 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+
+namespace tensorflow {
+
+REGISTER_OP("_ScopedAllocator")
+    .Output("output: T")
+    .Attr("shapes: list(shape)")
+    .Attr("T: type")
+    .Attr("sa_name: string")
+    .Attr("id: int")
+    .Attr("expected_call_count: int")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Allocates a mutable tensor that becomes available to appropriately annotated
+downstream Ops as backing store for their output tensor allocations via the
+ScopedAllocatorMgr.
+Returns a reference to this value.
+
+This is an experimental op for internal use only.  It is possible to use this
+op in unsafe ways.
+)doc");
+
+REGISTER_OP("_ScopedAllocatorConcat")
+    .Output("output: T")
+    .Input("backing: T")
+    .Input("inputs: N * T")
+    .Attr("shape: shape")
+    .Attr("T: type")
+    .Attr("sa_name: string")
+    .Attr("id: int")
+    .Attr("N: int >= 2")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Acts like a Concat Op that merges multple tensors into one, however it must
+only be used in conjunction with a ScopedAllocator which is backing the memory
+of all of its input tensors so that actually it just outputs a read-only
+reference to that ScopedAllocator's backing tensor.
+
+This is an experimental op for internal use only.  It is possible to use this
+op in unsafe ways.
+)doc");
+
+REGISTER_OP("_ScopedAllocatorSplit")
+    .Output("output: N * T")
+    .Input("concat: T")
+    .Input("split: N * T")
+    .Attr("T: type")
+    .Attr("sa_name: string")
+    .Attr("id: int")
+    .Attr("N: int >= 2")
+    .SetIsStateful()
+    .SetShapeFn(shape_inference::ExplicitShape)
+    .Doc(R"doc(
+Acts like a Concat Op that merges multple tensors into one, however it must
+only be used in conjunction with a ScopedAllocator which is backing the memory
+of all of its input tensors so that actually it just outputs a read-only
+reference to that ScopedAllocator's backing tensor.
+
+This is an experimental op for internal use only.  It is possible to use this
+op in unsafe ways.
+)doc");
+
+}  // end namespace tensorflow
-- 
GitLab


From 5f50c1ea7d62d12253b56030110e68c8c1e87e7c Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 22 Mar 2018 11:29:25 -0700
Subject: [PATCH 361/960] Cleanup: replace an errant `in_eager_mode()` with
 `executing_eagerly()`.

PiperOrigin-RevId: 190098277
---
 tensorflow/python/framework/ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index b3fa39fdab..de222e1932 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5411,7 +5411,7 @@ def get_name_scope():
   Returns:
     A string representing the current name scope.
   """
-  if context.in_eager_mode():
+  if context.executing_eagerly():
     return context.context().scope_name.rstrip("/")
   return get_default_graph().get_name_scope()
 
-- 
GitLab


From ebf8abdb4db1f4224ba61cd1d478e5301ff4bfd7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 11:55:11 -0700
Subject: [PATCH 362/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 190102805

---
 tensorflow/go/op/wrappers.go | 2112 +++++++++++++++++-----------------
 1 file changed, 1056 insertions(+), 1056 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 16472464db..92370c4f95 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -605,75 +605,123 @@ func ExtractImagePatches(scope *Scope, images tf.Output, ksizes []int64, strides
 	return op.Output(0)
 }
 
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
+// SpaceToDepthAttr is an optional argument to SpaceToDepth.
+type SpaceToDepthAttr func(optionalAttr)
 
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
+// SpaceToDepthDataFormat sets the optional data_format attribute to value.
+// If not specified, defaults to "NHWC"
+func SpaceToDepthDataFormat(value string) SpaceToDepthAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["data_format"] = value
 	}
 }
 
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// SpaceToDepth for tensors of type T.
 //
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapPeekSharedName(value string) MapPeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified key.  If the
+// Rearranges blocks of spatial data, into depth. More specifically,
+// this op outputs a copy of the input tensor where values from the `height`
+// and `width` dimensions are moved to the `depth` dimension.
+// The attr `block_size` indicates the input block size.
 //
-// underlying container does not contain this key
-// this op will block until it does.
-func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
+//   * Non-overlapping blocks of size `block_size x block size` are rearranged
+//     into depth at each location.
+//   * The depth of the output tensor is `block_size * block_size * input_depth`.
+//   * The Y, X coordinates within each block of the input become the high order
+//     component of the output channel index.
+//   * The input tensor's height and width must be divisible by block_size.
+//
+// The `data_format` attr specifies the layout of the input and output tensors
+// with the following options:
+//   "NHWC": `[ batch, height, width, channels ]`
+//   "NCHW": `[ batch, channels, height, width ]`
+//   "NCHW_VECT_C":
+//       `qint8 [ batch, channels / 4, height, width, 4 ]`
+//
+// It is useful to consider the operation as transforming a 6-D Tensor.
+// e.g. for data_format = NHWC,
+//      Each element in the input tensor can be specified via 6 coordinates,
+//      ordered by decreasing memory layout significance as:
+//      n,oY,bY,oX,bX,iC  (where n=batch index, oX, oY means X or Y coordinates
+//                         within the output image, bX, bY means coordinates
+//                         within the input block, iC means input channels).
+//      The output would be a transpose to the following layout:
+//      n,oY,oX,bY,bX,iC
+//
+// This operation is useful for resizing the activations between convolutions
+// (but keeping all data), e.g. instead of pooling. It is also useful for training
+// purely convolutional models.
+//
+// For example, given an input of shape `[1, 2, 2, 1]`, data_format = "NHWC" and
+// block_size = 2:
+//
+// ```
+// x = [[[[1], [2]],
+//       [[3], [4]]]]
+// ```
+//
+// This operation will output a tensor of shape `[1, 1, 1, 4]`:
+//
+// ```
+// [[[[1, 2, 3, 4]]]]
+// ```
+//
+// Here, the input has a batch of 1 and each batch element has shape `[2, 2, 1]`,
+// the corresponding output will have a single element (i.e. width and height are
+// both 1) and will have a depth of 4 channels (1 * block_size * block_size).
+// The output element shape is `[1, 1, 4]`.
+//
+// For an input tensor with larger depth, here of shape `[1, 2, 2, 3]`, e.g.
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// This operation, for block_size of 2, will return the following tensor of shape
+// `[1, 1, 1, 12]`
+//
+// ```
+// [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
+// ```
+//
+// Similarly, for the following input of shape `[1 4 4 1]`, and a block size of 2:
+//
+// ```
+// x = [[[[1],   [2],  [5],  [6]],
+//       [[3],   [4],  [7],  [8]],
+//       [[9],  [10], [13],  [14]],
+//       [[11], [12], [15],  [16]]]]
+// ```
+//
+// the operator will return the following tensor of shape `[1 2 2 4]`:
+//
+// ```
+// x = [[[[1, 2, 3, 4],
+//        [5, 6, 7, 8]],
+//       [[9, 10, 11, 12],
+//        [13, 14, 15, 16]]]]
+// ```
+//
+// Arguments:
+//
+//	block_size: The size of the spatial block.
+func SpaceToDepth(scope *Scope, input tf.Output, block_size int64, optional ...SpaceToDepthAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{"block_size": block_size}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapPeek",
+		Type: "SpaceToDepth",
 		Input: []tf.Input{
-			key, indices,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapPeek", err)
-		return
-	}
-	return values
+	return op.Output(0)
 }
 
 // Returns (x - y)(x - y) element-wise.
@@ -3383,45 +3431,6 @@ func MatrixDiag(scope *Scope, diagonal tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Says whether the targets are in the top `K` predictions.
-//
-// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
-// prediction for the target class is among the top `k` predictions among
-// all predictions for example `i`. Note that the behavior of `InTopK` differs
-// from the `TopK` op in its handling of ties; if multiple classes have the
-// same prediction value and straddle the top-`k` boundary, all of those
-// classes are considered to be in the top `k`.
-//
-// More formally, let
-//
-//   \\(predictions_i\\) be the predictions for all classes for example `i`,
-//   \\(targets_i\\) be the target class for example `i`,
-//   \\(out_i\\) be the output for example `i`,
-//
-// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
-//
-// Arguments:
-//	predictions: A `batch_size` x `classes` tensor.
-//	targets: A `batch_size` vector of class ids.
-//	k: Number of top elements to look at for computing precision.
-//
-// Returns Computed Precision at `k` as a `bool Tensor`.
-func InTopK(scope *Scope, predictions tf.Output, targets tf.Output, k int64) (precision tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"k": k}
-	opspec := tf.OpSpec{
-		Type: "InTopK",
-		Input: []tf.Input{
-			predictions, targets,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Given a quantized tensor described by (input, input_min, input_max), outputs a
 //
 // range that covers the actual values present in that tensor.  This op is
@@ -5092,45 +5101,46 @@ func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Outp
 	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
+// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
+type AvgPoolGradAttr func(optionalAttr)
 
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+// AvgPoolGradDataFormat sets the optional data_format attribute to value.
 //
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["data_format"] = value
 	}
 }
 
-// Gather specific elements from the TensorArray into output `value`.
-//
-// All elements selected by `indices` must have the same shape.
+// Computes gradients of the average pooling function.
 //
 // Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
+// the output of `avg_pool`.
+//	ksize: The size of the sliding window for each dimension of the input.
+//	strides: The stride of the sliding window for each dimension of the input.
+//	padding: The type of padding algorithm to use.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
+func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "AvgPoolGrad",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			orig_input_shape, grad,
 		},
 		Attrs: attrs,
 	}
@@ -5138,18 +5148,181 @@ func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow
 	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
+
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StageClear",
+
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
+type ComputeAccidentalHitsAttr func(optionalAttr)
+
+// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Computes the ids of the positions in sampled_candidates that match true_labels.
+//
+// When doing log-odds NCE, the result of this op should be passed through a
+// SparseToDense op, then added to the logits of the sampled candidates. This has
+// the effect of 'removing' the sampled labels that match the true labels by
+// making the classifier sure that they are sampled labels.
+//
+// Arguments:
+//	true_classes: The true_classes output of UnpackSparseLabels.
+//	sampled_candidates: The sampled_candidates output of CandidateSampler.
+//	num_true: Number of true labels per context.
+//
+// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
+// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
+// is -FLOAT_MAX.
+func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_true": num_true}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ComputeAccidentalHits",
+		Input: []tf.Input{
+			true_classes, sampled_candidates,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
+
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+//
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Gather specific elements from the TensorArray into output `value`.
+//
+// All elements selected by `indices` must have the same shape.
+//
+// Arguments:
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
+//
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGatherV3",
+		Input: []tf.Input{
+			handle, indices, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
+//
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
+//
+// Arguments:
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
 //
 // Returns A Tensor of the same shape as the input `string_tensor`.
 func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
@@ -8454,6 +8627,81 @@ func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and
 	return tensors
 }
 
+// Computes the maximum along segments of a tensor.
+//
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
+//
+// Computes a tensor such that
+// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the max is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SegmentMax",
+		Input: []tf.Input{
+			data, segment_ids,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Creates a dataset that skips `count` elements from the `input_dataset`.
+//
+// Arguments:
+//
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
+//
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "SkipDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes hyperbolic tangent of `x` element-wise.
+func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Tanh",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Decode web-safe base64-encoded strings.
 //
 // Input may or may not have padding at the end. See EncodeBase64 for padding.
@@ -8901,25 +9149,140 @@ func IRFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Out
 	return op.Output(0)
 }
 
-// Transforms a vector of brain.Example protos (as strings) into typed tensors.
+// DecodeJpegAttr is an optional argument to DecodeJpeg.
+type DecodeJpegAttr func(optionalAttr)
+
+// DecodeJpegChannels sets the optional channels attribute to value.
 //
-// Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	names: A vector containing the names of the serialized protos.
-// May contain, for example, table key (descriptive) names for the
-// corresponding serialized protos.  These are purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty vector if no names are available.
-// If non-empty, this vector must be the same length as "serialized".
-//	sparse_keys: A list of Nsparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: A list of Ndense string Tensors (scalars).
-// The keys expected in the Examples' features associated with dense values.
-//	dense_defaults: A list of Ndense Tensors (some may be empty).
-// dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodeJpegChannels(value int64) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["channels"] = value
+	}
+}
+
+// DecodeJpegRatio sets the optional ratio attribute to value.
+//
+// value: Downscaling ratio.
+// If not specified, defaults to 1
+func DecodeJpegRatio(value int64) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["ratio"] = value
+	}
+}
+
+// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
+//
+// value: If true use a slower but nicer upscaling of the
+// chroma planes (yuv420/422 only).
+// If not specified, defaults to true
+func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["fancy_upscaling"] = value
+	}
+}
+
+// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
+//
+// value: If true try to recover an image from truncated input.
+// If not specified, defaults to false
+func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["try_recover_truncated"] = value
+	}
+}
+
+// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
+//
+// value: The minimum required fraction of lines before a truncated
+// input is accepted.
+// If not specified, defaults to 1
+func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["acceptable_fraction"] = value
+	}
+}
+
+// DecodeJpegDctMethod sets the optional dct_method attribute to value.
+//
+// value: string specifying a hint about the algorithm used for
+// decompression.  Defaults to "" which maps to a system-specific
+// default.  Currently valid values are ["INTEGER_FAST",
+// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
+// jpeg library changes to a version that does not have that specific
+// option.)
+// If not specified, defaults to ""
+func DecodeJpegDctMethod(value string) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["dct_method"] = value
+	}
+}
+
+// Decode a JPEG-encoded image to a uint8 tensor.
+//
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the JPEG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
+//
+// If needed, the JPEG-encoded image is transformed to match the requested number
+// of color channels.
+//
+// The attr `ratio` allows downscaling the image by an integer factor during
+// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+// downscaling the image later.
+//
+//
+// This op also supports decoding PNGs and non-animated GIFs since the interface is
+// the same, though it is cleaner to use `tf.image.decode_image`.
+//
+// Arguments:
+//	contents: 0-D.  The JPEG-encoded image.
+//
+// Returns 3-D with shape `[height, width, channels]`..
+func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeJpeg",
+		Input: []tf.Input{
+			contents,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Transforms a vector of brain.Example protos (as strings) into typed tensors.
+//
+// Arguments:
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	names: A vector containing the names of the serialized protos.
+// May contain, for example, table key (descriptive) names for the
+// corresponding serialized protos.  These are purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty vector if no names are available.
+// If non-empty, this vector must be the same length as "serialized".
+//	sparse_keys: A list of Nsparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples' features associated with dense values.
+//	dense_defaults: A list of Ndense Tensors (some may be empty).
+// dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
 // If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
 // then the shape of dense_defaults[j] must match that of dense_shapes[j].
 // If dense_shapes[j] has an undefined major dimension (variable strides dense
@@ -9073,39 +9436,234 @@ func FFT(scope *Scope, input tf.Output) (output tf.Output) {
 			input,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
+type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
+
+// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
+//
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	gradient_accumulator: Should be from a Variable().
+//	gradient_squared_accumulator: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Learning rate. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	global_step: Training step number. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyAdagradDA",
+		Input: []tf.Input{
+			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// EncodeJpegAttr is an optional argument to EncodeJpeg.
+type EncodeJpegAttr func(optionalAttr)
+
+// EncodeJpegFormat sets the optional format attribute to value.
+//
+// value: Per pixel image format.
+// If not specified, defaults to ""
+func EncodeJpegFormat(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["format"] = value
+	}
+}
+
+// EncodeJpegQuality sets the optional quality attribute to value.
+//
+// value: Quality of the compression from 0 to 100 (higher is better and slower).
+// If not specified, defaults to 95
+func EncodeJpegQuality(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["quality"] = value
+	}
+}
+
+// EncodeJpegProgressive sets the optional progressive attribute to value.
+//
+// value: If True, create a JPEG that loads progressively (coarse to fine).
+// If not specified, defaults to false
+func EncodeJpegProgressive(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["progressive"] = value
+	}
+}
+
+// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
+//
+// value: If True, spend CPU/RAM to reduce size with no quality change.
+// If not specified, defaults to false
+func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["optimize_size"] = value
+	}
+}
+
+// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
+//
+// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
+// If not specified, defaults to true
+func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["chroma_downsampling"] = value
+	}
+}
+
+// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+//
+// value: Unit used to specify `x_density` and `y_density`:
+// pixels per inch (`'in'`) or centimeter (`'cm'`).
+// If not specified, defaults to "in"
+func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["density_unit"] = value
+	}
+}
+
+// EncodeJpegXDensity sets the optional x_density attribute to value.
+//
+// value: Horizontal pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegXDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["x_density"] = value
+	}
+}
+
+// EncodeJpegYDensity sets the optional y_density attribute to value.
+//
+// value: Vertical pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegYDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["y_density"] = value
+	}
+}
+
+// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
+//
+// value: If not empty, embed this XMP metadata in the image header.
+// If not specified, defaults to ""
+func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["xmp_metadata"] = value
+	}
+}
+
+// JPEG-encode an image.
+//
+// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
+//
+// The attr `format` can be used to override the color format of the encoded
+// output.  Values can be:
+//
+// *   `''`: Use a default format based on the number of channels in the image.
+// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
+//     of `image` must be 1.
+// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
+//     of `image` must be 3.
+//
+// If `format` is not specified or is the empty string, a default format is picked
+// in function of the number of channels in `image`:
+//
+// *   1: Output a grayscale image.
+// *   3: Output an RGB image.
+//
+// Arguments:
+//	image: 3-D with shape `[height, width, channels]`.
+//
+// Returns 0-D. JPEG-encoded image.
+func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "EncodeJpeg",
+		Input: []tf.Input{
+			image,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MultinomialAttr is an optional argument to Multinomial.
+type MultinomialAttr func(optionalAttr)
+
+// MultinomialSeed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 is set to be non-zero, the internal random number
+// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// If not specified, defaults to 0
+func MultinomialSeed(value int64) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
 }
 
-// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
-type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
-
-// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
+// MultinomialSeed2 sets the optional seed2 attribute to value.
 //
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func MultinomialSeed2(value int64) MultinomialAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["seed2"] = value
 	}
 }
 
-// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
+// MultinomialOutputDtype sets the optional output_dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["output_dtype"] = value
+	}
+}
+
+// Draws samples from a multinomial distribution.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	gradient_accumulator: Should be from a Variable().
-//	gradient_squared_accumulator: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Learning rate. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	global_step: Training step number. Must be a scalar.
+//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
+// represents the unnormalized log probabilities for all classes.
+//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
 //
-// Returns the created operation.
-func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
+// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
+// contains the drawn class labels with range `[0, num_classes)`.
+func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9114,13 +9672,14 @@ func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumul
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagradDA",
+		Type: "Multinomial",
 		Input: []tf.Input{
-			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
+			logits, num_samples,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
 // Returns the truth value of NOT x element-wise.
@@ -14914,208 +15473,12 @@ func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
 //
 // The inputs must be two-dimensional matrices and the inner dimension of "a" must
 // match the outer dimension of "b". This op is optimized for the case where at
-// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
-// matrix multiply on one platform was 30% zero values in the sparse matrix.
-//
-// The gradient computation of this operation will only take advantage of sparsity
-// in the input gradient when that gradient comes from a Relu.
-func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseMatMul",
-		Input: []tf.Input{
-			a, b,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MultinomialAttr is an optional argument to Multinomial.
-type MultinomialAttr func(optionalAttr)
-
-// MultinomialSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 is set to be non-zero, the internal random number
-// generator is seeded by the given seed.  Otherwise, a random seed is used.
-// If not specified, defaults to 0
-func MultinomialSeed(value int64) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// MultinomialSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func MultinomialSeed2(value int64) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// MultinomialOutputDtype sets the optional output_dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["output_dtype"] = value
-	}
-}
-
-// Draws samples from a multinomial distribution.
-//
-// Arguments:
-//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
-// represents the unnormalized log probabilities for all classes.
-//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
-//
-// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
-// contains the drawn class labels with range `[0, num_classes)`.
-func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Multinomial",
-		Input: []tf.Input{
-			logits, num_samples,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// EncodeJpegAttr is an optional argument to EncodeJpeg.
-type EncodeJpegAttr func(optionalAttr)
-
-// EncodeJpegFormat sets the optional format attribute to value.
-//
-// value: Per pixel image format.
-// If not specified, defaults to ""
-func EncodeJpegFormat(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["format"] = value
-	}
-}
-
-// EncodeJpegQuality sets the optional quality attribute to value.
-//
-// value: Quality of the compression from 0 to 100 (higher is better and slower).
-// If not specified, defaults to 95
-func EncodeJpegQuality(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["quality"] = value
-	}
-}
-
-// EncodeJpegProgressive sets the optional progressive attribute to value.
-//
-// value: If True, create a JPEG that loads progressively (coarse to fine).
-// If not specified, defaults to false
-func EncodeJpegProgressive(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["progressive"] = value
-	}
-}
-
-// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
-//
-// value: If True, spend CPU/RAM to reduce size with no quality change.
-// If not specified, defaults to false
-func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["optimize_size"] = value
-	}
-}
-
-// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
-//
-// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
-// If not specified, defaults to true
-func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["chroma_downsampling"] = value
-	}
-}
-
-// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
-//
-// value: Unit used to specify `x_density` and `y_density`:
-// pixels per inch (`'in'`) or centimeter (`'cm'`).
-// If not specified, defaults to "in"
-func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["density_unit"] = value
-	}
-}
-
-// EncodeJpegXDensity sets the optional x_density attribute to value.
-//
-// value: Horizontal pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegXDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["x_density"] = value
-	}
-}
-
-// EncodeJpegYDensity sets the optional y_density attribute to value.
-//
-// value: Vertical pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegYDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["y_density"] = value
-	}
-}
-
-// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
-//
-// value: If not empty, embed this XMP metadata in the image header.
-// If not specified, defaults to ""
-func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["xmp_metadata"] = value
-	}
-}
-
-// JPEG-encode an image.
-//
-// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
-//
-// The attr `format` can be used to override the color format of the encoded
-// output.  Values can be:
-//
-// *   `''`: Use a default format based on the number of channels in the image.
-// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
-//     of `image` must be 1.
-// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
-//     of `image` must be 3.
-//
-// If `format` is not specified or is the empty string, a default format is picked
-// in function of the number of channels in `image`:
-//
-// *   1: Output a grayscale image.
-// *   3: Output an RGB image.
-//
-// Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
+// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
+// matrix multiply on one platform was 30% zero values in the sparse matrix.
 //
-// Returns 0-D. JPEG-encoded image.
-func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
+// The gradient computation of this operation will only take advantage of sparsity
+// in the input gradient when that gradient comes from a Relu.
+func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15124,9 +15487,9 @@ func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (cont
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeJpeg",
+		Type: "SparseMatMul",
 		Input: []tf.Input{
-			image,
+			a, b,
 		},
 		Attrs: attrs,
 	}
@@ -15513,6 +15876,45 @@ func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, upd
 	return scope.AddOperation(opspec)
 }
 
+// Says whether the targets are in the top `K` predictions.
+//
+// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+// prediction for the target class is among the top `k` predictions among
+// all predictions for example `i`. Note that the behavior of `InTopK` differs
+// from the `TopK` op in its handling of ties; if multiple classes have the
+// same prediction value and straddle the top-`k` boundary, all of those
+// classes are considered to be in the top `k`.
+//
+// More formally, let
+//
+//   \\(predictions_i\\) be the predictions for all classes for example `i`,
+//   \\(targets_i\\) be the target class for example `i`,
+//   \\(out_i\\) be the output for example `i`,
+//
+// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+//
+// Arguments:
+//	predictions: A `batch_size` x `classes` tensor.
+//	targets: A `batch_size` vector of class ids.
+//	k: Number of top elements to look at for computing precision.
+//
+// Returns Computed Precision at `k` as a `bool Tensor`.
+func InTopK(scope *Scope, predictions tf.Output, targets tf.Output, k int64) (precision tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"k": k}
+	opspec := tf.OpSpec{
+		Type: "InTopK",
+		Input: []tf.Input{
+			predictions, targets,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the gradient for the inverse of `x` wrt its input.
 //
 // Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
@@ -16588,243 +16990,75 @@ func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Creates a dataset that skips `count` elements from the `input_dataset`.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
-//
-//
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "SkipDataset",
-		Input: []tf.Input{
-			input_dataset, count,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes hyperbolic tangent of `x` element-wise.
-func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Tanh",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the maximum along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the max is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentMax",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
-type AvgPoolGradAttr func(optionalAttr)
-
-// AvgPoolGradDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Computes gradients of the average pooling function.
-//
-// Arguments:
-//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
-// the output of `avg_pool`.
-//	ksize: The size of the sliding window for each dimension of the input.
-//	strides: The stride of the sliding window for each dimension of the input.
-//	padding: The type of padding algorithm to use.
-//
-// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
-func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "AvgPoolGrad",
-		Input: []tf.Input{
-			orig_input_shape, grad,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
 
-// StageClearCapacity sets the optional capacity attribute to value.
+// MapPeekCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
+func MapPeekCapacity(value int64) MapPeekAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// StageClearContainer sets the optional container attribute to value.
+// MapPeekContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
+func MapPeekContainer(value string) MapPeekAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// StageClearSharedName sets the optional shared_name attribute to value.
+// MapPeekSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
+func MapPeekSharedName(value string) MapPeekAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op removes all elements in the underlying container.
-//
-// Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StageClear",
-
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
-type ComputeAccidentalHitsAttr func(optionalAttr)
-
-// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Computes the ids of the positions in sampled_candidates that match true_labels.
-//
-// When doing log-odds NCE, the result of this op should be passed through a
-// SparseToDense op, then added to the logits of the sampled candidates. This has
-// the effect of 'removing' the sampled labels that match the true labels by
-// making the classifier sure that they are sampled labels.
-//
-// Arguments:
-//	true_classes: The true_classes output of UnpackSparseLabels.
-//	sampled_candidates: The sampled_candidates output of CandidateSampler.
-//	num_true: Number of true labels per context.
+// Op peeks at the values at the specified key.  If the
 //
-// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
-// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
-// is -FLOAT_MAX.
-func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+// underlying container does not contain this key
+// this op will block until it does.
+func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ComputeAccidentalHits",
+		Type: "MapPeek",
 		Input: []tf.Input{
-			true_classes, sampled_candidates,
+			key, indices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapPeek", err)
+		return
+	}
+	return values
 }
 
 // Looks up keys in a table, outputs the corresponding values.
@@ -18790,29 +19024,272 @@ func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_
 	opspec := tf.OpSpec{
 		Type: "DenseToDenseSetOperation",
 		Input: []tf.Input{
-			set1, set2,
+			set1, set2,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// Generate a sharded filename. The filename is printf formatted as
+//
+//    %s-%05d-of-%05d, basename, shard, num_shards.
+func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ShardedFilename",
+		Input: []tf.Input{
+			basename, shard, num_shards,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// BatchToSpace for N-D tensors of type T.
+//
+// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
+// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
+// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
+// the input.  The spatial dimensions of this intermediate result are then
+// optionally cropped according to `crops` to produce the output.  This is the
+// reverse of SpaceToBatch.  See below for a precise description.
+//
+// Arguments:
+//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+// where spatial_shape has M dimensions.
+//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
+//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
+//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
+//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
+//   required that
+//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
+//
+// This operation is equivalent to the following steps:
+//
+// 1. Reshape `input` to `reshaped` of shape:
+//      [block_shape[0], ..., block_shape[M-1],
+//       batch / prod(block_shape),
+//       input_shape[1], ..., input_shape[N-1]]
+//
+// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1], block_shape[0],
+//       ...,
+//       input_shape[M], block_shape[M-1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0],
+//       ...,
+//       input_shape[M] * block_shape[M-1],
+//
+//       input_shape[M+1],
+//       ...,
+//       input_shape[N-1]]
+//
+// 4. Crop the start and end of dimensions `[1, ..., M]` of
+//    `reshaped_permuted` according to `crops` to produce the output of shape:
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
+//       ...,
+//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// Some examples:
+//
+// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 3]` and value:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// The output tensor has shape `[1, 4, 4, 1]` and value:
+//
+// ```
+// x = [[[1],   [2],  [3],  [4]],
+//      [[5],   [6],  [7],  [8]],
+//      [[9],  [10], [11],  [12]],
+//      [[13], [14], [15],  [16]]]
+// ```
+//
+// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [2, 0]]`:
+//
+// ```
+// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+//      [[[0], [2], [4]]], [[[0], [10], [12]]],
+//      [[[0], [5], [7]]], [[[0], [13], [15]]],
+//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
+// ```
+//
+// The output tensor has shape `[2, 2, 4, 1]` and value:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BatchToSpaceND",
+		Input: []tf.Input{
+			input, block_shape, crops,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// UnpackAttr is an optional argument to Unpack.
+type UnpackAttr func(optionalAttr)
+
+// UnpackAxis sets the optional axis attribute to value.
+//
+// value: Dimension along which to unpack.  Negative values wrap around, so the
+// valid range is `[-R, R)`.
+// If not specified, defaults to 0
+func UnpackAxis(value int64) UnpackAttr {
+	return func(m optionalAttr) {
+		m["axis"] = value
+	}
+}
+
+// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
+//
+// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
+// For example, given a tensor of shape `(A, B, C, D)`;
+//
+// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
+//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
+//   dimension unpacked along is gone, unlike `split`).
+//
+// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
+//   and each tensor in `output` will have shape `(A, C, D)`.
+// Etc.
+//
+// This is the opposite of `pack`.
+//
+// Arguments:
+//	value: 1-D or higher, with `axis` dimension size equal to `num`.
+//
+//
+// Returns The list of tensors unpacked from `value`.
+func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num": num}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Unpack",
+		Input: []tf.Input{
+			value,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Unpack", err)
+		return
+	}
+	return output
+}
+
+// Increments variable pointed to by 'resource' until it reaches 'limit'.
+//
+// Arguments:
+//	resource: Should be from a scalar `Variable` node.
+//	limit: If incrementing ref would bring it above limit, instead generates an
+// 'OutOfRange' error.
+//
+//
+// Returns A copy of the input before increment. If nothing else modifies the
+// input, the values produced will all be distinct.
+func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"limit": limit, "T": T}
+	opspec := tf.OpSpec{
+		Type: "ResourceCountUpTo",
+		Input: []tf.Input{
+			resource,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Generate a sharded filename. The filename is printf formatted as
+// Delete the stack from its resource container.
 //
-//    %s-%05d-of-%05d, basename, shard, num_shards.
-func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
+// Arguments:
+//	handle: The handle to a stack.
+//
+// Returns the created operation.
+func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ShardedFilename",
+		Type: "StackCloseV2",
 		Input: []tf.Input{
-			basename, shard, num_shards,
+			handle,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
 // Generate a glob pattern matching all sharded file names.
@@ -19443,121 +19920,6 @@ func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, op
 	return op.Output(0)
 }
 
-// DecodeJpegAttr is an optional argument to DecodeJpeg.
-type DecodeJpegAttr func(optionalAttr)
-
-// DecodeJpegChannels sets the optional channels attribute to value.
-//
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodeJpegChannels(value int64) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["channels"] = value
-	}
-}
-
-// DecodeJpegRatio sets the optional ratio attribute to value.
-//
-// value: Downscaling ratio.
-// If not specified, defaults to 1
-func DecodeJpegRatio(value int64) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["ratio"] = value
-	}
-}
-
-// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
-//
-// value: If true use a slower but nicer upscaling of the
-// chroma planes (yuv420/422 only).
-// If not specified, defaults to true
-func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["fancy_upscaling"] = value
-	}
-}
-
-// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
-//
-// value: If true try to recover an image from truncated input.
-// If not specified, defaults to false
-func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["try_recover_truncated"] = value
-	}
-}
-
-// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
-//
-// value: The minimum required fraction of lines before a truncated
-// input is accepted.
-// If not specified, defaults to 1
-func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["acceptable_fraction"] = value
-	}
-}
-
-// DecodeJpegDctMethod sets the optional dct_method attribute to value.
-//
-// value: string specifying a hint about the algorithm used for
-// decompression.  Defaults to "" which maps to a system-specific
-// default.  Currently valid values are ["INTEGER_FAST",
-// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
-// jpeg library changes to a version that does not have that specific
-// option.)
-// If not specified, defaults to ""
-func DecodeJpegDctMethod(value string) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["dct_method"] = value
-	}
-}
-
-// Decode a JPEG-encoded image to a uint8 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the JPEG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-//
-// If needed, the JPEG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// The attr `ratio` allows downscaling the image by an integer factor during
-// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
-// downscaling the image later.
-//
-//
-// This op also supports decoding PNGs and non-animated GIFs since the interface is
-// the same, though it is cleaner to use `tf.image.decode_image`.
-//
-// Arguments:
-//	contents: 0-D.  The JPEG-encoded image.
-//
-// Returns 3-D with shape `[height, width, channels]`..
-func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeJpeg",
-		Input: []tf.Input{
-			contents,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ExtractJpegShapeAttr is an optional argument to ExtractJpegShape.
 type ExtractJpegShapeAttr func(optionalAttr)
 
@@ -25121,139 +25483,20 @@ func Exit(scope *Scope, data tf.Output) (output tf.Output) {
 			data,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns a copy of the input tensor.
-func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Snapshot",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SpaceToDepthAttr is an optional argument to SpaceToDepth.
-type SpaceToDepthAttr func(optionalAttr)
-
-// SpaceToDepthDataFormat sets the optional data_format attribute to value.
-// If not specified, defaults to "NHWC"
-func SpaceToDepthDataFormat(value string) SpaceToDepthAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// SpaceToDepth for tensors of type T.
-//
-// Rearranges blocks of spatial data, into depth. More specifically,
-// this op outputs a copy of the input tensor where values from the `height`
-// and `width` dimensions are moved to the `depth` dimension.
-// The attr `block_size` indicates the input block size.
-//
-//   * Non-overlapping blocks of size `block_size x block size` are rearranged
-//     into depth at each location.
-//   * The depth of the output tensor is `block_size * block_size * input_depth`.
-//   * The Y, X coordinates within each block of the input become the high order
-//     component of the output channel index.
-//   * The input tensor's height and width must be divisible by block_size.
-//
-// The `data_format` attr specifies the layout of the input and output tensors
-// with the following options:
-//   "NHWC": `[ batch, height, width, channels ]`
-//   "NCHW": `[ batch, channels, height, width ]`
-//   "NCHW_VECT_C":
-//       `qint8 [ batch, channels / 4, height, width, 4 ]`
-//
-// It is useful to consider the operation as transforming a 6-D Tensor.
-// e.g. for data_format = NHWC,
-//      Each element in the input tensor can be specified via 6 coordinates,
-//      ordered by decreasing memory layout significance as:
-//      n,oY,bY,oX,bX,iC  (where n=batch index, oX, oY means X or Y coordinates
-//                         within the output image, bX, bY means coordinates
-//                         within the input block, iC means input channels).
-//      The output would be a transpose to the following layout:
-//      n,oY,oX,bY,bX,iC
-//
-// This operation is useful for resizing the activations between convolutions
-// (but keeping all data), e.g. instead of pooling. It is also useful for training
-// purely convolutional models.
-//
-// For example, given an input of shape `[1, 2, 2, 1]`, data_format = "NHWC" and
-// block_size = 2:
-//
-// ```
-// x = [[[[1], [2]],
-//       [[3], [4]]]]
-// ```
-//
-// This operation will output a tensor of shape `[1, 1, 1, 4]`:
-//
-// ```
-// [[[[1, 2, 3, 4]]]]
-// ```
-//
-// Here, the input has a batch of 1 and each batch element has shape `[2, 2, 1]`,
-// the corresponding output will have a single element (i.e. width and height are
-// both 1) and will have a depth of 4 channels (1 * block_size * block_size).
-// The output element shape is `[1, 1, 4]`.
-//
-// For an input tensor with larger depth, here of shape `[1, 2, 2, 3]`, e.g.
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// This operation, for block_size of 2, will return the following tensor of shape
-// `[1, 1, 1, 12]`
-//
-// ```
-// [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
-// ```
-//
-// Similarly, for the following input of shape `[1 4 4 1]`, and a block size of 2:
-//
-// ```
-// x = [[[[1],   [2],  [5],  [6]],
-//       [[3],   [4],  [7],  [8]],
-//       [[9],  [10], [13],  [14]],
-//       [[11], [12], [15],  [16]]]]
-// ```
-//
-// the operator will return the following tensor of shape `[1 2 2 4]`:
-//
-// ```
-// x = [[[[1, 2, 3, 4],
-//        [5, 6, 7, 8]],
-//       [[9, 10, 11, 12],
-//        [13, 14, 15, 16]]]]
-// ```
-//
-// Arguments:
-//
-//	block_size: The size of the spatial block.
-func SpaceToDepth(scope *Scope, input tf.Output, block_size int64, optional ...SpaceToDepthAttr) (output tf.Output) {
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns a copy of the input tensor.
+func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"block_size": block_size}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SpaceToDepth",
+		Type: "Snapshot",
 		Input: []tf.Input{
 			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -27644,246 +27887,3 @@ func SpaceToBatch(scope *Scope, input tf.Output, paddings tf.Output, block_size
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
-
-// UnpackAttr is an optional argument to Unpack.
-type UnpackAttr func(optionalAttr)
-
-// UnpackAxis sets the optional axis attribute to value.
-//
-// value: Dimension along which to unpack.  Negative values wrap around, so the
-// valid range is `[-R, R)`.
-// If not specified, defaults to 0
-func UnpackAxis(value int64) UnpackAttr {
-	return func(m optionalAttr) {
-		m["axis"] = value
-	}
-}
-
-// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
-//
-// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
-// For example, given a tensor of shape `(A, B, C, D)`;
-//
-// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
-//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
-//   dimension unpacked along is gone, unlike `split`).
-//
-// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
-//   and each tensor in `output` will have shape `(A, C, D)`.
-// Etc.
-//
-// This is the opposite of `pack`.
-//
-// Arguments:
-//	value: 1-D or higher, with `axis` dimension size equal to `num`.
-//
-//
-// Returns The list of tensors unpacked from `value`.
-func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num": num}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Unpack",
-		Input: []tf.Input{
-			value,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Unpack", err)
-		return
-	}
-	return output
-}
-
-// Increments variable pointed to by 'resource' until it reaches 'limit'.
-//
-// Arguments:
-//	resource: Should be from a scalar `Variable` node.
-//	limit: If incrementing ref would bring it above limit, instead generates an
-// 'OutOfRange' error.
-//
-//
-// Returns A copy of the input before increment. If nothing else modifies the
-// input, the values produced will all be distinct.
-func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"limit": limit, "T": T}
-	opspec := tf.OpSpec{
-		Type: "ResourceCountUpTo",
-		Input: []tf.Input{
-			resource,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Delete the stack from its resource container.
-//
-// Arguments:
-//	handle: The handle to a stack.
-//
-// Returns the created operation.
-func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "StackCloseV2",
-		Input: []tf.Input{
-			handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// BatchToSpace for N-D tensors of type T.
-//
-// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
-// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
-// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
-// the input.  The spatial dimensions of this intermediate result are then
-// optionally cropped according to `crops` to produce the output.  This is the
-// reverse of SpaceToBatch.  See below for a precise description.
-//
-// Arguments:
-//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
-// where spatial_shape has M dimensions.
-//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
-//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
-//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
-//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
-//   required that
-//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
-//
-// This operation is equivalent to the following steps:
-//
-// 1. Reshape `input` to `reshaped` of shape:
-//      [block_shape[0], ..., block_shape[M-1],
-//       batch / prod(block_shape),
-//       input_shape[1], ..., input_shape[N-1]]
-//
-// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1], block_shape[0],
-//       ...,
-//       input_shape[M], block_shape[M-1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0],
-//       ...,
-//       input_shape[M] * block_shape[M-1],
-//
-//       input_shape[M+1],
-//       ...,
-//       input_shape[N-1]]
-//
-// 4. Crop the start and end of dimensions `[1, ..., M]` of
-//    `reshaped_permuted` according to `crops` to produce the output of shape:
-//      [batch / prod(block_shape),
-//
-//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
-//       ...,
-//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
-//
-//       input_shape[M+1], ..., input_shape[N-1]]
-//
-// Some examples:
-//
-// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 1]` and value:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 3]` and value:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-//
-// The output tensor has shape `[1, 4, 4, 1]` and value:
-//
-// ```
-// x = [[[1],   [2],  [3],  [4]],
-//      [[5],   [6],  [7],  [8]],
-//      [[9],  [10], [11],  [12]],
-//      [[13], [14], [15],  [16]]]
-// ```
-//
-// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [2, 0]]`:
-//
-// ```
-// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
-//      [[[0], [2], [4]]], [[[0], [10], [12]]],
-//      [[[0], [5], [7]]], [[[0], [13], [15]]],
-//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
-// ```
-//
-// The output tensor has shape `[2, 2, 4, 1]` and value:
-//
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
-func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BatchToSpaceND",
-		Input: []tf.Input{
-			input, block_shape, crops,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From 6ed3cec4ae1a0706abf3c7b82f6b70f6a45a760c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 11:55:34 -0700
Subject: [PATCH 363/960] Make GetLocalTemporaryDirectories() a virtual method
 of Env, that is overriden by the implementations.

PiperOrigin-RevId: 190102851
---
 tensorflow/core/platform/env.h          | 8 ++++++--
 tensorflow/core/platform/posix/env.cc   | 5 ++++-
 tensorflow/core/platform/windows/env.cc | 4 +++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index 2a114d47a8..a7e9fcb17c 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -291,10 +291,10 @@ class Env {
   virtual string FormatLibraryFileName(const string& name,
                                        const string& version) = 0;
 
- private:
   // Returns a possible list of local temporary directories.
-  void GetLocalTempDirectories(std::vector<string>* list);
+  virtual void GetLocalTempDirectories(std::vector<string>* list) = 0;
 
+ private:
   std::unique_ptr<FileSystemRegistry> file_system_registry_;
   TF_DISALLOW_COPY_AND_ASSIGN(Env);
   EnvTime* envTime = EnvTime::Default();
@@ -358,6 +358,10 @@ class EnvWrapper : public Env {
   }
 
  private:
+  void GetLocalTempDirectories(std::vector<string>* list) override {
+    target_->GetLocalTempDirectories(list);
+  }
+
   Env* target_;
 };
 
diff --git a/tensorflow/core/platform/posix/env.cc b/tensorflow/core/platform/posix/env.cc
index 8097624e09..418874d340 100644
--- a/tensorflow/core/platform/posix/env.cc
+++ b/tensorflow/core/platform/posix/env.cc
@@ -118,6 +118,9 @@ class PosixEnv : public Env {
                                const string& version) override {
     return tensorflow::internal::FormatLibraryFileName(name, version);
   }
+
+ private:
+  void GetLocalTempDirectories(std::vector<string>* list) override;
 };
 
 }  // namespace
@@ -131,7 +134,7 @@ Env* Env::Default() {
 }
 #endif
 
-void Env::GetLocalTempDirectories(std::vector<string>* list) {
+void PosixEnv::GetLocalTempDirectories(std::vector<string>* list) {
   list->clear();
   // Directories, in order of preference. If we find a dir that
   // exists, we stop adding other less-preferred dirs
diff --git a/tensorflow/core/platform/windows/env.cc b/tensorflow/core/platform/windows/env.cc
index 41b2644170..2f54f423b2 100644
--- a/tensorflow/core/platform/windows/env.cc
+++ b/tensorflow/core/platform/windows/env.cc
@@ -160,6 +160,8 @@ class WindowsEnv : public Env {
   }
 
  private:
+  void GetLocalTempDirectories(std::vector<string>* list) override;
+
   typedef VOID(WINAPI* FnGetSystemTimePreciseAsFileTime)(LPFILETIME);
   FnGetSystemTimePreciseAsFileTime GetSystemTimePreciseAsFileTime_;
 };
@@ -174,7 +176,7 @@ Env* Env::Default() {
   return default_env;
 }
 
-void Env::GetLocalTempDirectories(std::vector<string>* list) {
+void WindowsEnv::GetLocalTempDirectories(std::vector<string>* list) {
   list->clear();
   // On windows we'll try to find a directory in this order:
   //   C:/Documents & Settings/whomever/TEMP (or whatever GetTempPath() is)
-- 
GitLab


From c7d11e1601d5045f5421c465a438a1d9632df78d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 12:34:02 -0700
Subject: [PATCH 364/960] Merges predict export_outputs in multi_head.

PiperOrigin-RevId: 190108434
---
 tensorflow/contrib/estimator/BUILD            |  1 +
 .../estimator/python/estimator/multi_head.py  |  9 ++++++
 .../python/estimator/multi_head_test.py       | 28 +++++++++++++++----
 3 files changed, 32 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 2f7ed7cd73..676d60231d 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -292,6 +292,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:summary",
+        "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:metric_keys",
         "//tensorflow/python/estimator:model_fn",
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py
index 0346ddc24b..23d3714c53 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head.py
@@ -23,6 +23,7 @@ import six
 from tensorflow.python.estimator import model_fn
 from tensorflow.python.estimator.canned import head as head_lib
 from tensorflow.python.estimator.canned import metric_keys
+from tensorflow.python.estimator.export import export_output as export_output_lib
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -319,6 +320,7 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
             all_estimator_spec[0].export_outputs,
             self._heads[0].name),
     }
+    merged_predict_outputs = {}
     for head, spec in zip(self._heads, all_estimator_spec):
       head_name = head.name
       for k, v in six.iteritems(spec.export_outputs):
@@ -327,8 +329,15 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
         else:
           key = '%s/%s' % (k, head_name)
         export_outputs[key] = v
+        if (k == head_lib._PREDICT_SERVING_KEY and  # pylint:disable=protected-access
+            isinstance(v, export_output_lib.PredictOutput)):
+          for kp, vp in six.iteritems(v.outputs):
+            key = '%s/%s' % (head_name, kp)
+            merged_predict_outputs[key] = vp
       for k, v in six.iteritems(spec.predictions):
         predictions[(head_name, k)] = v
+    export_outputs[head_lib._PREDICT_SERVING_KEY] = (  # pylint:disable=protected-access
+        export_output_lib.PredictOutput(merged_predict_outputs))
 
     return model_fn.EstimatorSpec(
         mode=model_fn.ModeKeys.PREDICT,
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
index 65ea89ba1b..8e788a9ce8 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
@@ -127,8 +127,8 @@ class MultiHeadTest(test.TestCase):
         logits=logits)
 
     self.assertItemsEqual(
-        (_DEFAULT_SERVING_KEY, 'head1', 'classification/head1', 'predict/head1',
-         'head2', 'classification/head2', 'predict/head2'),
+        (_DEFAULT_SERVING_KEY, 'predict', 'head1', 'classification/head1',
+         'predict/head1', 'head2', 'classification/head2', 'predict/head2'),
         spec.export_outputs.keys())
 
     # Assert predictions and export_outputs.
@@ -158,6 +158,22 @@ class MultiHeadTest(test.TestCase):
       self.assertAllClose(
           expected_probabilities['head2'],
           sess.run(spec.export_outputs['head2'].scores))
+      self.assertAllClose(
+          expected_probabilities['head1'],
+          sess.run(
+              spec.export_outputs['predict'].outputs['head1/probabilities']))
+      self.assertAllClose(
+          expected_probabilities['head2'],
+          sess.run(
+              spec.export_outputs['predict'].outputs['head2/probabilities']))
+      self.assertAllClose(
+          expected_probabilities['head1'],
+          sess.run(
+              spec.export_outputs['predict/head1'].outputs['probabilities']))
+      self.assertAllClose(
+          expected_probabilities['head2'],
+          sess.run(
+              spec.export_outputs['predict/head2'].outputs['probabilities']))
 
   def test_predict_two_heads_logits_tensor(self):
     """Tests predict with logits as Tensor."""
@@ -181,8 +197,8 @@ class MultiHeadTest(test.TestCase):
         logits=logits)
 
     self.assertItemsEqual(
-        (_DEFAULT_SERVING_KEY, 'head1', 'classification/head1', 'predict/head1',
-         'head2', 'classification/head2', 'predict/head2'),
+        (_DEFAULT_SERVING_KEY, 'predict', 'head1', 'classification/head1',
+         'predict/head1', 'head2', 'classification/head2', 'predict/head2'),
         spec.export_outputs.keys())
 
     # Assert predictions and export_outputs.
@@ -238,8 +254,8 @@ class MultiHeadTest(test.TestCase):
         logits=logits)
 
     self.assertItemsEqual(
-        (_DEFAULT_SERVING_KEY, 'head1', 'regression/head1', 'predict/head1',
-         'head2', 'regression/head2', 'predict/head2'),
+        (_DEFAULT_SERVING_KEY, 'predict', 'head1', 'regression/head1',
+         'predict/head1', 'head2', 'regression/head2', 'predict/head2'),
         spec.export_outputs.keys())
 
     # Assert predictions and export_outputs.
-- 
GitLab


From 830fc390b76b5eb138a7f59d0e13e83add653870 Mon Sep 17 00:00:00 2001
From: Dan Ringwalt <ringwalt@google.com>
Date: Thu, 22 Mar 2018 12:45:24 -0700
Subject: [PATCH 365/960] Add tf.contrib.framework.argsort, wrapping
 tf.nn.top_k (#288).

Comparable to np.argsort.

PiperOrigin-RevId: 190109968
---
 tensorflow/contrib/framework/__init__.py      |   1 +
 .../contrib/framework/python/ops/sort_ops.py  | 161 +++++++++++++-----
 .../framework/python/ops/sort_ops_test.py     |  34 ++++
 3 files changed, 156 insertions(+), 40 deletions(-)

diff --git a/tensorflow/contrib/framework/__init__.py b/tensorflow/contrib/framework/__init__.py
index 3398b3fd1c..cbb68bd3eb 100644
--- a/tensorflow/contrib/framework/__init__.py
+++ b/tensorflow/contrib/framework/__init__.py
@@ -83,6 +83,7 @@ See the @{$python/contrib.framework} guide.
 @@load_linear_multiclass_bias_initializer
 @@load_variable_slot_initializer
 
+@@argsort
 @@py_func
 @@sort
 
diff --git a/tensorflow/contrib/framework/python/ops/sort_ops.py b/tensorflow/contrib/framework/python/ops/sort_ops.py
index 8f62f0ea7b..1921a77c1e 100644
--- a/tensorflow/contrib/framework/python/ops/sort_ops.py
+++ b/tensorflow/contrib/framework/python/ops/sort_ops.py
@@ -14,6 +14,7 @@
 # ==============================================================================
 """Support for sorting tensors.
 
+@@argsort
 @@sort
 """
 
@@ -21,6 +22,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops as framework_ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
@@ -47,64 +51,141 @@ def sort(values, axis=-1, direction='ASCENDING', name=None):
     ValueError: If axis is not a constant scalar, or the direction is invalid.
   """
   with framework_ops.name_scope(name, 'sort'):
-    if direction not in _SORT_IMPL:
-      raise ValueError('%s should be one of %s' %
-                       (direction, ', '.join(sorted(_SORT_IMPL.keys()))))
-    # Axis must be an integer, not a Tensor.
-    axis = framework_ops.convert_to_tensor(axis, name='axis')
-    axis_static = tensor_util.constant_value(axis)
-    if axis.shape.ndims != 0 or axis_static is None:
-      raise ValueError('axis must be a constant scalar')
-    axis_static = int(axis_static)  # Avoids NumPy casting error
+    return _sort_or_argsort(values, axis, direction, return_argsort=False)
+
+
+def argsort(values, axis=-1, direction='ASCENDING', stable=False, name=None):
+  """Returns the indices of a tensor that give its sorted order along an axis.
+
+  For a 1D tensor, `tf.gather(values, tf.argsort(values))` is equivalent to
+  `tf.sort(values)`. For higher dimensions, the output has the same shape as
+  `values`, but along the given axis, values represent the index of the sorted
+  element in that slice of the tensor at the given position.
+
+  Args:
+    values: 1-D or higher numeric `Tensor`.
+    axis: The axis along which to sort. The default is -1, which sorts the last
+        axis.
+    direction: The direction in which to sort the values (`'ASCENDING'` or
+        `'DESCENDING'`).
+    stable: If True, equal elements in the original tensor will not be
+        re-ordered in the returned order. Unstable sort is not yet implemented,
+        but will eventually be the default for performance reasons. If you
+        require a stable order, pass `stable=True` for forwards compatibility.
+    name: Optional name for the operation.
+
+  Returns:
+    An int32 `Tensor` with the same shape as `values`. The indices that would
+        sort each slice of the given `values` along the given `axis`.
+
+  Raises:
+    ValueError: If axis is not a constant scalar, or the direction is invalid.
+  """
+  del stable  # Unused.
+  with framework_ops.name_scope(name, 'argsort'):
+    return _sort_or_argsort(values, axis, direction, return_argsort=True)
+
+
+def _sort_or_argsort(values, axis, direction, return_argsort):
+  """Internal sort/argsort implementation.
+
+  Args:
+    values: The input values.
+    axis: The axis along which to sort.
+    direction: 'ASCENDING' or 'DESCENDING'.
+    return_argsort: Whether to return the argsort result.
+
+  Returns:
+    Either the sorted values, or the indices of the sorted values in the
+        original tensor. See the `sort` and `argsort` docstrings.
+
+  Raises:
+    ValueError: If axis is not a constant scalar, or the direction is invalid.
+  """
+  if direction not in _SORT_IMPL:
+    raise ValueError('%s should be one of %s' %
+                     (direction, ', '.join(sorted(_SORT_IMPL.keys()))))
+  # Axis must be an integer, not a Tensor.
+  axis = framework_ops.convert_to_tensor(axis, name='axis')
+  axis_static = tensor_util.constant_value(axis)
+  if axis.shape.ndims != 0 or axis_static is None:
+    raise ValueError('axis must be a constant scalar')
+  axis_static = int(axis_static)  # Avoids NumPy casting error
 
-    values = framework_ops.convert_to_tensor(values, name='values')
+  values = framework_ops.convert_to_tensor(values, name='values')
 
-    return _SORT_IMPL[direction](values, axis_static)
+  return _SORT_IMPL[direction](values, axis_static, return_argsort)
 
 
-def _descending_sort(values, axis):
+def _descending_sort(values, axis, return_argsort=False):
   """Sorts values in reverse using `top_k`.
 
   Args:
     values: Tensor of numeric values.
     axis: Index of the axis which values should be sorted along.
+    return_argsort: If False, return the sorted values. If True, return the
+        indices that would sort the values.
 
   Returns:
     The sorted values.
   """
   k = array_ops.shape(values)[axis]
   rank = array_ops.rank(values)
+  static_rank = values.shape.ndims
   # Fast path: sorting the last axis.
   if axis == -1 or axis + 1 == values.get_shape().ndims:
-    return nn_ops.top_k(values, k)[0]
-
-  # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`.
-  if axis < 0:
-    # Make axis a Tensor with the real axis index if needed.
-    axis += rank
-  transposition = array_ops.concat(
-      [
-          # Axes up to axis are unchanged.
-          math_ops.range(axis),
-          # Swap axis and rank - 1.
-          [rank - 1],
-          # Axes in [axis + 1, rank - 1) are unchanged.
-          math_ops.range(axis + 1, rank - 1),
-          # Swap axis and rank - 1.
-          [axis]
-      ],
-      axis=0)
-  top_k_input = array_ops.transpose(values, transposition)
-  values, unused_indices = nn_ops.top_k(top_k_input, k)
-  # transposition contains a single cycle of length 2 (swapping 2 elements),
-  # so it is an involution (it is its own inverse).
-  return array_ops.transpose(values, transposition)
-
-
-def _ascending_sort(values, axis):
+    top_k_input = values
+    transposition = None
+  else:
+    # Otherwise, transpose the array. Swap axes `axis` and `rank - 1`.
+    if axis < 0:
+      # Calculate the actual axis index if counting from the end. Use the static
+      # rank if available, or else make the axis back into a tensor.
+      axis += static_rank or rank
+    if static_rank is not None:
+      # Prefer to calculate the transposition array in NumPy and make it a
+      # constant.
+      transposition = constant_op.constant(
+          np.r_[
+              # Axes up to axis are unchanged.
+              np.arange(axis),
+              # Swap axis and rank - 1.
+              [static_rank - 1],
+              # Axes in [axis + 1, rank - 1) are unchanged.
+              np.arange(axis + 1, static_rank - 1),
+              # Swap axis and rank - 1.
+              [axis]],
+          name='transposition')
+    else:
+      # Generate the transposition array from the tensors.
+      transposition = array_ops.concat(
+          [
+              # Axes up to axis are unchanged.
+              math_ops.range(axis),
+              # Swap axis and rank - 1.
+              [rank - 1],
+              # Axes in [axis + 1, rank - 1) are unchanged.
+              math_ops.range(axis + 1, rank - 1),
+              # Swap axis and rank - 1.
+              [axis]
+          ],
+          axis=0)
+    top_k_input = array_ops.transpose(values, transposition)
+
+  values, indices = nn_ops.top_k(top_k_input, k)
+  return_value = indices if return_argsort else values
+  if transposition is not None:
+    # transposition contains a single cycle of length 2 (swapping 2 elements),
+    # so it is an involution (it is its own inverse).
+    return_value = array_ops.transpose(return_value, transposition)
+  return return_value
+
+
+def _ascending_sort(values, axis, return_argsort=False):
   # Negate the values to get the ascending order from descending sort.
-  values_or_indices = _descending_sort(-values, axis)
-  return -values_or_indices
+  values_or_indices = _descending_sort(-values, axis, return_argsort)
+  # If not argsort, negate the values again.
+  return values_or_indices if return_argsort else -values_or_indices
 
 
 _SORT_IMPL = {
diff --git a/tensorflow/contrib/framework/python/ops/sort_ops_test.py b/tensorflow/contrib/framework/python/ops/sort_ops_test.py
index d08ae502f1..a8fb94b245 100644
--- a/tensorflow/contrib/framework/python/ops/sort_ops_test.py
+++ b/tensorflow/contrib/framework/python/ops/sort_ops_test.py
@@ -24,6 +24,8 @@ from tensorflow.contrib.framework.python.ops import sort_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
@@ -90,6 +92,38 @@ class SortTest(test.TestCase):
               axis=0,
               direction='DESCENDING').eval())
 
+  def testSort_staticallyKnownRank_constantTransposition(self):
+    # The transposition array should be a constant if the rank of "values" is
+    # statically known.
+    tensor = random_ops.random_uniform(
+        # Rank is statically known to be 5, but the dimension lengths are not
+        # known.
+        random_ops.random_uniform(
+            shape=(5,), minval=0, maxval=10, dtype=dtypes.int32))
+    sort_ops.sort(tensor, axis=1)
+    transposition = (
+        ops.get_default_graph().get_tensor_by_name('sort/transposition:0'))
+    self.assertFalse(tensor_util.constant_value(transposition) is None)
+    self.assertAllEqual(
+        # Swaps "1" and "4" to put "1" at the end.
+        tensor_util.constant_value(transposition),
+        [0, 4, 2, 3, 1])
+
+  def testArgsort_1d(self):
+    arr = np.random.random(42)
+    with self.test_session():
+      self.assertAllEqual(
+          np.sort(arr),
+          array_ops.gather(arr, sort_ops.argsort(arr)).eval())
+
+  def testArgsort(self):
+    arr = np.random.random((5, 6, 7, 8))
+    for axis in range(4):
+      with self.test_session():
+        self.assertAllEqual(
+            np.argsort(arr, axis=axis),
+            sort_ops.argsort(arr, axis=axis).eval())
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 3158b499c7c811a5ed4b81a2d8341dd3c8923823 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Thu, 22 Mar 2018 12:52:25 -0700
Subject: [PATCH 366/960] Make api_compatibility_test output verbose by
 default.

PiperOrigin-RevId: 190110866
---
 tensorflow/tools/api/tests/api_compatibility_test.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 96f501e163..603b2a4327 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -58,7 +58,7 @@ _UPDATE_GOLDENS_HELP = """
      have to be authorized by TensorFlow leads.
 """
 
-# DEFINE_boolean, verbose_diffs, default False:
+# DEFINE_boolean, verbose_diffs, default True:
 _VERBOSE_DIFFS_HELP = """
      If set to true, print line by line diffs on all libraries. If set to
      false, only print which libraries have differences.
@@ -286,7 +286,7 @@ if __name__ == '__main__':
   parser.add_argument(
       '--update_goldens', type=bool, default=False, help=_UPDATE_GOLDENS_HELP)
   parser.add_argument(
-      '--verbose_diffs', type=bool, default=False, help=_VERBOSE_DIFFS_HELP)
+      '--verbose_diffs', type=bool, default=True, help=_VERBOSE_DIFFS_HELP)
   FLAGS, unparsed = parser.parse_known_args()
 
   # Now update argv, so that unittest library does not get confused.
-- 
GitLab


From b57af1577f4a6e4181227d105c68463538b8f9ef Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 13:05:30 -0700
Subject: [PATCH 367/960] Disable testing flaky
 tensorflow/contrib/eager/python/examples/spinn:spinn_test under py3

PiperOrigin-RevId: 190112748
---
 tensorflow/contrib/eager/python/examples/spinn/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/eager/python/examples/spinn/BUILD b/tensorflow/contrib/eager/python/examples/spinn/BUILD
index 98d01ad1d5..5966f1d487 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/BUILD
+++ b/tensorflow/contrib/eager/python/examples/spinn/BUILD
@@ -39,6 +39,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
     ],
     tags = [
+        "no-internal-py3",  # flaky
         "no_cuda_on_cpu_tap",
         "no_pip",  # because spinn.py is under third_party/.
     ],
-- 
GitLab


From 3642ae00e9268229db76667150c113b83339d11e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 13:21:57 -0700
Subject: [PATCH 368/960] Allow specifying in the arrays extra info file:  -
 the shape of the array  - the hardcoding of the values of the array as a
 single repeated constant    scalar value, turning an activations array into a
 constant array.

PiperOrigin-RevId: 190115218
---
 .../contrib/lite/toco/model_flags.proto       |  4 +++-
 tensorflow/contrib/lite/toco/tooling_util.cc  | 24 ++++++++++++++++---
 2 files changed, 24 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/model_flags.proto b/tensorflow/contrib/lite/toco/model_flags.proto
index 867b86f31d..42e0f54826 100644
--- a/tensorflow/contrib/lite/toco/model_flags.proto
+++ b/tensorflow/contrib/lite/toco/model_flags.proto
@@ -96,11 +96,13 @@ message RnnState {
 // model that does not already contain such MinMax information.
 message ArraysExtraInfo {
   message Entry {
-    // Next ID to use: 5.
+    // Next ID to use: 7.
     optional string name = 1;
     optional float min = 2;
     optional float max = 3;
     optional IODataType data_type = 4;
+    optional InputArrayShape shape = 5;
+    optional float constant_float_value = 6;
   }
   repeated Entry entries = 1;
 }
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index ec1770c129..f3f50487ff 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -1972,9 +1972,9 @@ void FinishBuildingRNNStates(Model* model) {
 
 void UseArraysExtraInfo(Model* model) {
   for (const auto& entry : model->flags.arrays_extra_info().entries()) {
-    QCHECK(model->HasArray(entry.name()))
-        << "ArraysExtraInfo refers to non-existent array name: "
-        << entry.name();
+    if (!model->HasArray(entry.name())) {
+      continue;
+    }
     auto& array = model->GetArray(entry.name());
     auto& minmax = array.GetOrCreateMinMax();
     if (entry.has_min() || entry.has_max()) {
@@ -1986,6 +1986,24 @@ void UseArraysExtraInfo(Model* model) {
       array.final_data_type =
           ConvertIODataTypeToArrayDataType(entry.data_type());
     }
+    if (entry.has_shape()) {
+      array.clear_shape();
+      // Make sure to create the shape even if there are no dims, to
+      // correctly record 0-D shapes.
+      array.mutable_shape();
+      for (int dim : entry.shape().dims()) {
+        array.mutable_shape()->mutable_dims()->push_back(dim);
+      }
+    }
+    if (entry.has_constant_float_value()) {
+      CHECK(array.has_shape());
+      CHECK(array.data_type == ArrayDataType::kFloat);
+      auto& data = array.GetMutableBuffer<ArrayDataType::kFloat>().data;
+      data.resize(RequiredBufferSizeForShape(array.shape()));
+      for (float& f : data) {
+        f = entry.constant_float_value();
+      }
+    }
   }
 }
 
-- 
GitLab


From 9b9e54538fa766679aaa60b73f352e975c213730 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 22 Mar 2018 13:24:51 -0700
Subject: [PATCH 369/960] Disable all the automatic optimizations when testing,
 to ensure that we can properly compare the results of the original graph
 against that of the hand optimized graph.

PiperOrigin-RevId: 190115606
---
 .../core/grappler/utils/grappler_test.cc      | 22 +++++++++++++++----
 .../core/grappler/utils/grappler_test.h       |  7 ++++++
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 6b6cecebe1..1c15ea65b8 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -17,15 +17,30 @@ limitations under the License.
 #include <memory>
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/rewriter_config.pb.h"
 #include "tensorflow/core/public/session.h"
 
 namespace tensorflow {
 namespace grappler {
 
+GrapplerTest::GrapplerTest() {
+  // Turn off all the automatic optimizations to ensure that we run the graph
+  // exactly as it is given to us. This ensures that we can compare the results
+  // before and after manual optimization, without any of the automatic
+  // optimizations interfering in the comparison.
+  RewriterConfig* cfg =
+      options_.config.mutable_graph_options()->mutable_rewrite_options();
+  cfg->set_constant_folding(RewriterConfig::OFF);
+  cfg->set_arithmetic_optimization(RewriterConfig::OFF);
+  cfg->set_dependency_optimization(RewriterConfig::OFF);
+  cfg->set_loop_optimization(RewriterConfig::OFF);
+  cfg->set_function_optimization(RewriterConfig::OFF);
+  cfg->set_layout_optimizer(RewriterConfig::OFF);
+}
+
 std::vector<Tensor> GrapplerTest::EvaluateNodes(
     const GraphDef& graph, const std::vector<string>& node_names) const {
-  SessionOptions options;
-  std::unique_ptr<tensorflow::Session> session(NewSession(options));
+  std::unique_ptr<tensorflow::Session> session(NewSession(options_));
   TF_CHECK_OK(session->Create(graph));
   RunOptions run_options;
   std::vector<Tensor> output_tensors;
@@ -37,8 +52,7 @@ std::vector<Tensor> GrapplerTest::EvaluateNodes(
 
 std::vector<Tensor> GrapplerTest::EvaluateFetchNodes(
     const GrapplerItem& item) const {
-  SessionOptions options;
-  std::unique_ptr<tensorflow::Session> session(NewSession(options));
+  std::unique_ptr<tensorflow::Session> session(NewSession(options_));
   TF_CHECK_OK(session->Create(item.graph));
   RunOptions run_options;
   if (!item.init_ops.empty()) {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index c7f06557e7..e0c67381a4 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -24,11 +24,15 @@ limitations under the License.
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
 
 namespace tensorflow {
 namespace grappler {
 
 class GrapplerTest : public ::testing::Test {
+ public:
+  GrapplerTest();
+
  protected:
   std::vector<Tensor> EvaluateNodes(
       const GraphDef& graph, const std::vector<string>& node_names) const;
@@ -48,6 +52,9 @@ class GrapplerTest : public ::testing::Test {
 
   // Count nodes of the given op-type in a graph.
   int CountOpNodes(const GraphDef& graph, const string& op);
+
+ private:
+  SessionOptions options_;
 };
 
 }  // end namespace grappler
-- 
GitLab


From f088fa2b1bc010fd4e4396a9f1e6e0868e9890c4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 13:41:13 -0700
Subject: [PATCH 370/960] Code cleanup: rather than storing the
 outside_compilation shape inference graph as a serialized GraphDef in an
 attr, put it into the function library.

PiperOrigin-RevId: 190118116
---
 tensorflow/compiler/jit/BUILD                 |  1 +
 .../jit/encapsulate_subgraphs_pass.cc         | 67 ++++++++-------
 .../jit/encapsulate_subgraphs_pass_test.cc    | 83 +++++++++----------
 3 files changed, 79 insertions(+), 72 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 0475cd9ff2..8e505da622 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -348,6 +348,7 @@ tf_cc_test(
     deps = [
         ":common",
         ":compilation_passes",
+        ":graph_to_functiondef",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/cc:cc_ops_internal",
         "//tensorflow/cc:function_ops",
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 0685036c9d..7fc43fb263 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -334,8 +334,10 @@ class Encapsulator {
     void ConnectSequencerToCallNode(Graph* graph_out);
 
     Status AddShapeInferenceInfo(
+        const string& subgraph_name,
         const string& outside_compilation_subgraph_name,
-        const std::vector<TensorShapeProto>& shapes, GraphDef* inference_graph);
+        const std::vector<TensorShapeProto>& shapes, Graph* inference_graph,
+        FunctionLibraryDefinition* library);
 
     Status ReplaceFunctionDef(FunctionLibraryDefinition* library);
 
@@ -573,7 +575,7 @@ class Encapsulator {
       const std::unordered_set<string>& recv_at_host_nodes, Node* send_node,
       FunctionLibraryDefinition* library,
       std::vector<TensorShapeProto>* static_shape_out,
-      std::unique_ptr<GraphDef>* graphdef_out);
+      std::unique_ptr<Graph>* graph_out);
 
   // Makes a copy of graph containing only nodes that are ancestors of at least
   // one node in send_from_host_nodes and store it in pruned_graph. On exit
@@ -949,8 +951,10 @@ Status Encapsulator::Subgraph::BuildFunctionDef(
 }
 
 Status Encapsulator::Subgraph::AddShapeInferenceInfo(
+    const string& subgraph_name,
     const string& outside_compilation_subgraph_name,
-    const std::vector<TensorShapeProto>& shapes, GraphDef* inference_graph) {
+    const std::vector<TensorShapeProto>& shapes, Graph* inference_graph,
+    FunctionLibraryDefinition* library) {
   OutsideCompilationSubgraph& oc_subgraph =
       outside_compilation_subgraphs_.at(outside_compilation_subgraph_name);
 
@@ -972,14 +976,15 @@ Status Encapsulator::Subgraph::AddShapeInferenceInfo(
     host_compute->AddAttr("shape_inference_graph", "");
     host_compute->AddAttr("shapes", shapes);
   } else {
-    string serialized_graph;
-    if (!inference_graph->SerializeToString(&serialized_graph)) {
-      return errors::Internal(
-          "Failed to serialize graph for outside compilation subgraph ",
-          oc_subgraph.host_compute_name);
-    }
-    host_compute->AddAttr("shape_inference_graph", serialized_graph);
+    string inference_graph_name =
+        strings::StrCat("_outside_compilation_shape_inference_", subgraph_name,
+                        "_", outside_compilation_subgraph_name);
+    FunctionDef fdef;
+    TF_RETURN_IF_ERROR(
+        GraphToFunctionDef(*inference_graph, inference_graph_name, &fdef));
+    host_compute->AddAttr("shape_inference_graph", inference_graph_name);
     host_compute->AddAttr("shapes", std::vector<TensorShapeProto>());
+    TF_RETURN_IF_ERROR(library->AddFunctionDef(fdef));
   }
   return Status::OK();
 }
@@ -1760,7 +1765,7 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
     const std::unordered_set<string>& recv_at_host_nodes, Node* send_node,
     FunctionLibraryDefinition* library,
     std::vector<TensorShapeProto>* static_shape_out,
-    std::unique_ptr<GraphDef>* graphdef_out) {
+    std::unique_ptr<Graph>* graph_out) {
   // Maps from nodes in graph_in to nodes in graph_out.
   //
   // When an edge has fully defined shape the source node in graph_in is
@@ -1777,8 +1782,8 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
   std::unordered_map<Node*, Node*> dummy_node_images;
   std::unordered_map<Node*, Node*> copied_node_images;
 
-  std::unique_ptr<Graph> graph_out(new Graph(graph_in.op_registry()));
-  graph_out->set_versions(graph_in.versions());
+  graph_out->reset(new Graph(graph_in.op_registry()));
+  (*graph_out)->set_versions(graph_in.versions());
   // The final input to the send node is the dynamic key, which we don't include
   // in the static shapes.
   static_shape_out->resize(send_node->num_inputs() - 1);
@@ -1800,7 +1805,7 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
     if (w.leave) {
       TF_RETURN_IF_ERROR(CopyShapeInferenceNodeToGraph(
           n, send_node, dummy_node_images, library, &copied_node_images,
-          graph_out.get()));
+          graph_out->get()));
     } else {
       if (visited[n->id()]) continue;
       visited[n->id()] = true;
@@ -1824,7 +1829,7 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
             context->ShapeHandleToProto(shape, &proto);
             if (dummy_node_images.find(src_node) == dummy_node_images.end()) {
               dummy_node_images[src_node] = AddDummyShapedNode(
-                  src_node->output_type(src_port), proto, graph_out.get());
+                  src_node->output_type(src_port), proto, graph_out->get());
             }
             // The final input to the send node is the dynamic key, which we
             // don't include in the static shapes.
@@ -1849,7 +1854,7 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
           // The shapes of all the inputs to send_node are statically known. We
           // won't have to do any inference at compile time so return now: the
           // shapes were stored in static_shape_out above.
-          graphdef_out->reset();
+          graph_out->reset();
           return Status::OK();
         } else {
           // Any shape that is being processed is either the original send node
@@ -1872,9 +1877,6 @@ Status Encapsulator::DoStaticShapeInferenceForOutsideCompilationSend(
     }
   }
 
-  graphdef_out->reset(new GraphDef());
-  graph_out->ToGraphDef(graphdef_out->get());
-
   return Status::OK();
 }
 
@@ -1997,13 +1999,14 @@ Status Encapsulator::GetShapeInfoForOutsideCompilationSends(
   }
 
   for (auto& subgraph_entry : subgraphs_) {
+    const string& subgraph_name = subgraph_entry.first;
     Subgraph& subgraph = subgraph_entry.second;
     // Find all the recv_at_host nodes in this subgraph.
     std::vector<string> outside_compilation_names;
     subgraph.GetOutsideCompilationSubgraphNames(&outside_compilation_names);
     std::unordered_set<string> recv_at_host_names;
-    for (const auto& name : outside_compilation_names) {
-      Node* recv_node = subgraph.GetRecvAtHostNode(name);
+    for (const auto& oc_name : outside_compilation_names) {
+      Node* recv_node = subgraph.GetRecvAtHostNode(oc_name);
       if (recv_node != nullptr) {
         recv_at_host_names.insert(recv_node->name());
       }
@@ -2012,26 +2015,30 @@ Status Encapsulator::GetShapeInfoForOutsideCompilationSends(
     // without knowing the shape of the recv_at_host nodes, and store the
     // result, along with enough information to complete the job at compile time
     // once the recv_at_host shapes are known.
-    for (const auto& name : outside_compilation_names) {
-      Node* send_node = subgraph.GetSendFromHostNode(name);
+    for (const auto& oc_name : outside_compilation_names) {
+      Node* send_node = subgraph.GetSendFromHostNode(oc_name);
       std::vector<TensorShapeProto> static_shape;
-      std::unique_ptr<GraphDef> graphdef;
+      std::unique_ptr<Graph> graph;
       if (send_node != nullptr) {
         TF_RETURN_IF_ERROR(DoStaticShapeInferenceForOutsideCompilationSend(
             *pruned_graph, shape_refiner, recv_at_host_names,
-            node_images[send_node], library, &static_shape, &graphdef));
-        if (graphdef == nullptr) {
+            node_images[send_node], library, &static_shape, &graph));
+        if (graph == nullptr) {
           VLOG(2) << "Send node  " << send_node->name() << " shapes";
           for (int i = 0; i < static_shape.size(); ++i) {
             VLOG(2) << static_shape[i].DebugString();
           }
         } else {
-          VLOG(2) << "Send node " << send_node->name() << " graph\n"
-                  << graphdef->DebugString();
+          if (VLOG_IS_ON(2)) {
+            GraphDef graphdef;
+            graph->ToGraphDef(&graphdef);
+            VLOG(2) << "Send node " << send_node->name() << " graph\n"
+                    << graphdef.DebugString();
+          }
         }
       }
-      TF_RETURN_IF_ERROR(
-          subgraph.AddShapeInferenceInfo(name, static_shape, graphdef.get()));
+      TF_RETURN_IF_ERROR(subgraph.AddShapeInferenceInfo(
+          subgraph_name, oc_name, static_shape, graph.get(), library));
     }
     if (!outside_compilation_names.empty()) {
       TF_RETURN_IF_ERROR(subgraph.ReplaceFunctionDef(library));
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 711b1424c7..94481a1fde 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -13,12 +13,14 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <memory>
 #include <utility>
 
 #include "tensorflow/compiler/jit/encapsulate_subgraphs_pass.h"
 
 #include "tensorflow/cc/framework/ops.h"
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/compiler/jit/graph_to_functiondef.h"
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
@@ -32,6 +34,24 @@ namespace {
 const char* const kXlaHostTransferSequencerAttr =
     "_xla_host_transfer_sequencer";
 
+Status AddGraphDefToFunctionLibrary(const GraphDefBuilder& graphdef_builder,
+                                    const string& name_suffix,
+                                    FunctionDefLibrary* library) {
+  GraphDef graphdef;
+  TF_RETURN_IF_ERROR(graphdef_builder.ToGraphDef(&graphdef));
+  std::unique_ptr<Graph> graph =
+      std::unique_ptr<Graph>(new Graph(OpRegistry::Global()));
+  GraphConstructorOptions opts;
+  opts.allow_internal_ops = true;
+  TF_RETURN_IF_ERROR(ConvertGraphDefToGraph(opts, graphdef, graph.get()));
+  FunctionDef* fdef = library->add_function();
+  TF_RETURN_IF_ERROR(GraphToFunctionDef(
+      *graph,
+      strings::StrCat("_outside_compilation_shape_inference_", name_suffix),
+      fdef));
+  return Status::OK();
+}
+
 template <class Tkey, class Tvalue>
 bool EqualProtoMap(const ::tensorflow::protobuf::Map<Tkey, Tvalue>& a,
                    const ::tensorflow::protobuf::Map<Tkey, Tvalue>& b,
@@ -115,23 +135,7 @@ bool EqualFunctionNodeDef(const NodeDef& a, const NodeDef& b,
       a.attr(), b.attr(), [](const string& s) { return s; },
       [](const AttrValue& v) { return v.DebugString(); },
       [](const string& key, const AttrValue& av, const AttrValue& bv) {
-        if (key == "shape_inference_graph") {
-          // Default serialization of GraphDef is unstable because maps don't
-          // serialize deterministically. Rather than go through the hoops to
-          // turn on deterministic serialization of this attr just for this
-          // test, add logic here to compare determinstically.
-          GraphDef ga;
-          if (!ga.ParseFromString(av.s())) {
-            return false;
-          }
-          GraphDef gb;
-          if (!gb.ParseFromString(bv.s())) {
-            return false;
-          }
-          return EqualGraphDef(ga, gb, nullptr);
-        } else {
-          return av.DebugString() == bv.DebugString();
-        }
+        return av.DebugString() == bv.DebugString();
       },
       strings::StrCat(diff_preamble, " attr mismatch for node ", a.name()),
       diff);
@@ -848,7 +852,6 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
   FunctionDefLibrary library_expected;
   GraphDef graphdef_expected;
 
-  string shape_string_expected;
   {
     GraphDefBuilder shape(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
@@ -861,9 +864,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
                      shape.opts().WithName("E"));
     SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
                  {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
-    GraphDef shape_graph;
-    TF_EXPECT_OK(shape.ToGraphDef(&shape_graph));
-    EXPECT_TRUE(shape_graph.SerializeToString(&shape_string_expected));
+    TF_EXPECT_OK(
+        AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
 
   *library_expected.add_function() = test::function::XTimesTwo();
@@ -883,7 +885,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"key", "host_compute_channel_F1_O1"},
-            {"shape_inference_graph", shape_string_expected},
+            {"shape_inference_graph",
+             "_outside_compilation_shape_inference_F1_O1"},
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"c"}},
       },
@@ -969,7 +972,6 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
   FunctionDefLibrary library_expected;
   GraphDef graphdef_expected;
 
-  string shape_string_expected_1;
   {
     GraphDefBuilder shape1(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
@@ -982,12 +984,10 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
                      shape1.opts().WithName("E"));
     SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
                  {e}, shape1.opts().WithName("outside_compilation_F1_O1_send"));
-    GraphDef shape1_graph;
-    TF_EXPECT_OK(shape1.ToGraphDef(&shape1_graph));
-    EXPECT_TRUE(shape1_graph.SerializeToString(&shape_string_expected_1));
+    TF_EXPECT_OK(
+        AddGraphDefToFunctionLibrary(shape1, "F1_O1", &library_expected));
   }
 
-  string shape_string_expected_2;
   {
     GraphDefBuilder shape2(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
@@ -1005,9 +1005,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
     Node* h = Binary(ops::NodeOut(recv2, 0), e, shape2.opts().WithName("H"));
     SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2",
                  {h}, shape2.opts().WithName("outside_compilation_F1_O2_send"));
-    GraphDef shape2_graph;
-    TF_EXPECT_OK(shape2.ToGraphDef(&shape2_graph));
-    EXPECT_TRUE(shape2_graph.SerializeToString(&shape_string_expected_2));
+    TF_EXPECT_OK(
+        AddGraphDefToFunctionLibrary(shape2, "F1_O2", &library_expected));
   }
 
   *library_expected.add_function() = FunctionDefHelper::Create(
@@ -1029,7 +1028,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"key", "host_compute_channel_F1_O2"},
-            {"shape_inference_graph", shape_string_expected_2},
+            {"shape_inference_graph",
+             "_outside_compilation_shape_inference_F1_O2"},
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"F"}},
           {{"outside_compilation_O1_host_compute"},
@@ -1038,7 +1038,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"key", "host_compute_channel_F1_O1"},
-            {"shape_inference_graph", shape_string_expected_1},
+            {"shape_inference_graph",
+             "_outside_compilation_shape_inference_F1_O1"},
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"D"}},
       },
@@ -1134,7 +1135,6 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
   FunctionDefLibrary library_expected;
   GraphDef graphdef_expected;
 
-  string shape_string_expected;
   {
     GraphDefBuilder shape(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
@@ -1147,9 +1147,8 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
                      shape.opts().WithName("E"));
     SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
                  {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
-    GraphDef shape_graph;
-    TF_EXPECT_OK(shape.ToGraphDef(&shape_graph));
-    EXPECT_TRUE(shape_graph.SerializeToString(&shape_string_expected));
+    TF_EXPECT_OK(
+        AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
 
   TensorShapeProto shape_proto_expected;
@@ -1172,7 +1171,8 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT, DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"key", "host_compute_channel_F1_O1"},
-            {"shape_inference_graph", shape_string_expected},
+            {"shape_inference_graph",
+             "_outside_compilation_shape_inference_F1_O1"},
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"D"}},
       },
@@ -1661,7 +1661,6 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
   FunctionDefLibrary library_expected;
   GraphDef graphdef_expected;
 
-  string shape_string_expected;
   {
     GraphDefBuilder shape(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
@@ -1673,9 +1672,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
     Node* e = BinaryUnknownShape(known, recv, shape.opts().WithName("E"));
     SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
                  {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
-    GraphDef shape_graph;
-    TF_EXPECT_OK(shape.ToGraphDef(&shape_graph));
-    EXPECT_TRUE(shape_graph.SerializeToString(&shape_string_expected));
+    TF_EXPECT_OK(
+        AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
 
   *library_expected.add_function() = test::function::XTimesTwo();
@@ -1694,7 +1692,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
            {{"Tinputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"Toutputs", gtl::ArraySlice<DataType>({DT_FLOAT})},
             {"key", "host_compute_channel_F1_O1"},
-            {"shape_inference_graph", shape_string_expected},
+            {"shape_inference_graph",
+             "_outside_compilation_shape_inference_F1_O1"},
             {"shapes", gtl::ArraySlice<DataType>({})}},
            {"c"}},
       },
-- 
GitLab


From f2b62548edfd298367bc996fb236ea39f385ff76 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 22 Mar 2018 13:55:48 -0700
Subject: [PATCH 371/960] Fix cases where we export incorrect symbol with
 tf_export. This can happen when both generated op and its python wrapper have
 tf_export decorator. create_python_api.py now checks that we don't export
 different symbols with same name. Also, simplified some logic.

PiperOrigin-RevId: 190120505
---
 .../api_def/python_api/api_def_ArgMax.pbtxt   |  4 +
 .../api_def/python_api/api_def_ArgMin.pbtxt   |  4 +
 .../python_api/api_def_CountUpTo.pbtxt        |  4 +
 .../core/api_def/python_api/api_def_Div.pbtxt |  4 +
 .../core/api_def/python_api/api_def_Erf.pbtxt |  4 +
 .../api_def/python_api/api_def_Identity.pbtxt |  4 +
 .../core/api_def/python_api/api_def_Mod.pbtxt |  4 +
 .../api_def/python_api/api_def_Rank.pbtxt     |  4 +
 .../api_def/python_api/api_def_Round.pbtxt    |  4 +
 .../python_api/api_def_ScatterNdUpdate.pbtxt  |  4 +
 .../python_api/api_def_ScatterUpdate.pbtxt    |  4 +
 .../api_def/python_api/api_def_ShapeN.pbtxt   |  4 +
 .../api_def/python_api/api_def_Sign.pbtxt     |  4 +
 .../api_def/python_api/api_def_Sqrt.pbtxt     |  4 +
 .../api_def/python_api/api_def_Square.pbtxt   |  4 +
 tensorflow/python/framework/python_op_gen.cc  |  8 +-
 tensorflow/python/ops/math_ops.py             |  4 +-
 .../tools/api/generator/create_python_api.py  | 82 ++++++++++++++-----
 18 files changed, 127 insertions(+), 27 deletions(-)
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ArgMax.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ArgMin.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CountUpTo.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Div.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Identity.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Mod.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Rank.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Round.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ScatterNdUpdate.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ScatterUpdate.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ShapeN.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_Square.pbtxt

diff --git a/tensorflow/core/api_def/python_api/api_def_ArgMax.pbtxt b/tensorflow/core/api_def/python_api/api_def_ArgMax.pbtxt
new file mode 100644
index 0000000000..4c23a432f2
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ArgMax.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ArgMax"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ArgMin.pbtxt b/tensorflow/core/api_def/python_api/api_def_ArgMin.pbtxt
new file mode 100644
index 0000000000..daa14f6386
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ArgMin.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ArgMin"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CountUpTo.pbtxt b/tensorflow/core/api_def/python_api/api_def_CountUpTo.pbtxt
new file mode 100644
index 0000000000..f41be2f540
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CountUpTo.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CountUpTo"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Div.pbtxt b/tensorflow/core/api_def/python_api/api_def_Div.pbtxt
new file mode 100644
index 0000000000..8e5537c8bf
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Div.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Div"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt b/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
new file mode 100644
index 0000000000..391167254e
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Erf.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Erf"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Identity.pbtxt b/tensorflow/core/api_def/python_api/api_def_Identity.pbtxt
new file mode 100644
index 0000000000..00f2afde27
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Identity.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Identity"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Mod.pbtxt b/tensorflow/core/api_def/python_api/api_def_Mod.pbtxt
new file mode 100644
index 0000000000..48d828ca72
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Mod.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Mod"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Rank.pbtxt b/tensorflow/core/api_def/python_api/api_def_Rank.pbtxt
new file mode 100644
index 0000000000..05aa12f2fa
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Rank.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Rank"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Round.pbtxt b/tensorflow/core/api_def/python_api/api_def_Round.pbtxt
new file mode 100644
index 0000000000..74428e2f58
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Round.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Round"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ScatterNdUpdate.pbtxt b/tensorflow/core/api_def/python_api/api_def_ScatterNdUpdate.pbtxt
new file mode 100644
index 0000000000..ccf4a9cce8
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ScatterNdUpdate.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ScatterNdUpdate"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ScatterUpdate.pbtxt b/tensorflow/core/api_def/python_api/api_def_ScatterUpdate.pbtxt
new file mode 100644
index 0000000000..e4c41c1226
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ScatterUpdate.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ScatterUpdate"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ShapeN.pbtxt b/tensorflow/core/api_def/python_api/api_def_ShapeN.pbtxt
new file mode 100644
index 0000000000..b2dbe74b09
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ShapeN.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ShapeN"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt b/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
new file mode 100644
index 0000000000..c2ee91dd12
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Sign.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Sign"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt b/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
new file mode 100644
index 0000000000..59e2dfe836
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Sqrt.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Sqrt"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_Square.pbtxt b/tensorflow/core/api_def/python_api/api_def_Square.pbtxt
new file mode 100644
index 0000000000..7b39ae25fa
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_Square.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "Square"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 03721c9a68..9850f0becc 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -78,7 +78,7 @@ bool IsPythonReserved(const string& s) {
 bool IsOpWithUnderscorePrefix(const string& s) {
   static const std::set<string>* const kUnderscoreOps = new std::set<string>(
       {// Lowercase built-in functions and types in Python, from:
-       // [x for x in dir(__builtins__) if x[0].islower()]
+       // [x for x in dir(__builtins__) if x[0].islower()] except "round".
        // These need to be excluded so they don't conflict with actual built-in
        // functions since we use '*' imports.
        "abs", "all", "any", "apply", "bin", "bool", "buffer", "bytearray",
@@ -90,9 +90,9 @@ bool IsOpWithUnderscorePrefix(const string& s) {
        "iter", "len", "license", "list", "locals", "long", "map", "max",
        "memoryview", "min", "next", "object", "oct", "open", "ord", "pow",
        "print", "property", "quit", "range", "raw_input", "reduce", "reload",
-       "repr", "reversed", "round", "set", "setattr", "slice", "sorted",
-       "staticmethod", "str", "sum", "super", "tuple", "type", "unichr",
-       "unicode", "vars", "xrange", "zip",
+       "repr", "reversed", "set", "setattr", "slice", "sorted", "staticmethod",
+       "str", "sum", "super", "tuple", "type", "unichr", "unicode", "vars",
+       "xrange", "zip",
        // These have the same name as ops defined in Python and might be used
        // incorrectly depending on order of '*' imports.
        // TODO(annarev): reduce usage of '*' imports and remove these from the
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index c893bf9b90..4699e05269 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -180,6 +180,8 @@ linspace = gen_math_ops.lin_space
 
 arg_max = deprecation.deprecated(None, "Use `argmax` instead")(arg_max)  # pylint: disable=used-before-assignment
 arg_min = deprecation.deprecated(None, "Use `argmin` instead")(arg_min)  # pylint: disable=used-before-assignment
+tf_export("arg_max")(arg_max)
+tf_export("arg_min")(arg_min)
 
 
 # This is set by resource_variable_ops.py. It is included in this way since
@@ -1196,7 +1198,7 @@ tf_export("floor_div")(floor_div)
 truncatemod = gen_math_ops.truncate_mod
 tf_export("truncatemod")(truncatemod)
 floormod = gen_math_ops.floor_mod
-tf_export("floormod")(floormod)
+tf_export("floormod", "mod")(floormod)
 
 
 def _mul_dispatch(x, y, name=None):
diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py
index bb7c3e77a3..183c4731b8 100644
--- a/tensorflow/tools/api/generator/create_python_api.py
+++ b/tensorflow/tools/api/generator/create_python_api.py
@@ -23,7 +23,6 @@ import collections
 import os
 import sys
 
-from tensorflow import python as tf
 from tensorflow.python.util import tf_decorator
 
 
@@ -39,6 +38,11 @@ Generated by: tensorflow/tools/api/generator/create_python_api.py script.
 """
 
 
+class SymbolExposedTwiceError(Exception):
+  """Raised when different symbols are exported with the same name."""
+  pass
+
+
 def format_import(source_module_name, source_name, dest_name):
   """Formats import statement.
 
@@ -63,6 +67,44 @@ def format_import(source_module_name, source_name, dest_name):
       return 'import %s as %s' % (source_name, dest_name)
 
 
+class _ModuleImportsBuilder(object):
+  """Builds a map from module name to imports included in that module."""
+
+  def __init__(self):
+    self.module_imports = collections.defaultdict(list)
+    self._seen_api_names = set()
+
+  def add_import(
+      self, dest_module_name, source_module_name, source_name, dest_name):
+    """Adds this import to module_imports.
+
+    Args:
+      dest_module_name: (string) Module name to add import to.
+      source_module_name: (string) Module to import from.
+      source_name: (string) Name of the symbol to import.
+      dest_name: (string) Import the symbol using this name.
+
+    Raises:
+      SymbolExposedTwiceError: Raised when an import with the same
+        dest_name has already been added to dest_module_name.
+    """
+    import_str = format_import(source_module_name, source_name, dest_name)
+    if import_str in self.module_imports[dest_module_name]:
+      return
+
+    # Check if we are trying to expose two different symbols with same name.
+    full_api_name = dest_name
+    if dest_module_name:
+      full_api_name = dest_module_name + '.' + full_api_name
+    if full_api_name in self._seen_api_names:
+      raise SymbolExposedTwiceError(
+          'Trying to export multiple symbols with same name: %s.' %
+          full_api_name)
+    self._seen_api_names.add(full_api_name)
+
+    self.module_imports[dest_module_name].append(import_str)
+
+
 def get_api_imports():
   """Get a map from destination module to formatted imports.
 
@@ -73,7 +115,9 @@ def get_api_imports():
           (for e.g. 'from foo import bar') and constant
           assignments (for e.g. 'FOO = 123').
   """
-  module_imports = collections.defaultdict(list)
+  module_imports_builder = _ModuleImportsBuilder()
+  visited_symbols = set()
+
   # Traverse over everything imported above. Specifically,
   # we want to traverse over TensorFlow Python modules.
   for module in sys.modules.values():
@@ -86,6 +130,8 @@ def get_api_imports():
 
     for module_contents_name in dir(module):
       attr = getattr(module, module_contents_name)
+      if id(attr) in visited_symbols:
+        continue
 
       # If attr is _tf_api_constants attribute, then add the constants.
       if module_contents_name == _API_CONSTANTS_ATTR:
@@ -93,36 +139,30 @@ def get_api_imports():
           for export in exports:
             names = export.split('.')
             dest_module = '.'.join(names[:-1])
-            import_str = format_import(module.__name__, value, names[-1])
-            module_imports[dest_module].append(import_str)
+            module_imports_builder.add_import(
+                dest_module, module.__name__, value, names[-1])
         continue
 
       _, attr = tf_decorator.unwrap(attr)
       # If attr is a symbol with _tf_api_names attribute, then
       # add import for it.
       if hasattr(attr, '__dict__') and _API_NAMES_ATTR in attr.__dict__:
-        # The same op might be accessible from multiple modules.
-        # We only want to consider location where function was defined.
-        # Here we check if the op is defined in another TensorFlow module in
-        # sys.modules.
-        if (hasattr(attr, '__module__') and
-            attr.__module__.startswith(tf.__name__) and
-            attr.__module__ != module.__name__ and
-            attr.__module__ in sys.modules and
-            module_contents_name in dir(sys.modules[attr.__module__])):
+        # If the same symbol is available using multiple names, only create
+        # imports for it once.
+        if id(attr) in visited_symbols:
           continue
+        visited_symbols.add(id(attr))
 
         for export in attr._tf_api_names:  # pylint: disable=protected-access
           names = export.split('.')
           dest_module = '.'.join(names[:-1])
-          import_str = format_import(
-              module.__name__, module_contents_name, names[-1])
-          module_imports[dest_module].append(import_str)
+          module_imports_builder.add_import(
+              dest_module, module.__name__, module_contents_name, names[-1])
 
   # Import all required modules in their parent modules.
   # For e.g. if we import 'foo.bar.Value'. Then, we also
   # import 'bar' in 'foo'.
-  imported_modules = set(module_imports.keys())
+  imported_modules = set(module_imports_builder.module_imports.keys())
   for module in imported_modules:
     if not module:
       continue
@@ -135,13 +175,11 @@ def get_api_imports():
         parent_module += ('.' + module_split[submodule_index-1] if parent_module
                           else module_split[submodule_index-1])
         import_from += '.' + parent_module
-      submodule_import = format_import(
-          import_from, module_split[submodule_index],
+      module_imports_builder.add_import(
+          parent_module, import_from, module_split[submodule_index],
           module_split[submodule_index])
-      if submodule_import not in module_imports[parent_module]:
-        module_imports[parent_module].append(submodule_import)
 
-  return module_imports
+  return module_imports_builder.module_imports
 
 
 def create_api_files(output_files):
-- 
GitLab


From 804f98e5bc0a75284f5f92569e5c82fe88b455ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 14:01:10 -0700
Subject: [PATCH 372/960] Supports optimizer arg in head.create_estimator_spec.

PiperOrigin-RevId: 190121386
---
 tensorflow/contrib/estimator/BUILD            |   2 +
 .../estimator/python/estimator/head.py        |  28 +++-
 .../estimator/python/estimator/head_test.py   |  36 +++++
 .../estimator/python/estimator/multi_head.py  |  34 +++--
 .../python/estimator/multi_head_test.py       |  38 ++++++
 tensorflow/python/estimator/BUILD             |   2 +-
 tensorflow/python/estimator/canned/dnn.py     |   9 +-
 .../estimator/canned/dnn_testing_utils.py     |  15 ++-
 tensorflow/python/estimator/canned/head.py    | 126 ++++++++++++------
 .../python/estimator/canned/head_test.py      | 102 ++++++++++++++
 tensorflow/python/estimator/canned/linear.py  |   9 +-
 11 files changed, 322 insertions(+), 79 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 676d60231d..24374266dc 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -175,6 +175,7 @@ py_library(
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:summary",
+        "//tensorflow/python:training",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:metric_keys",
@@ -292,6 +293,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
         "//tensorflow/python:summary",
+        "//tensorflow/python:training",
         "//tensorflow/python/estimator:export_output",
         "//tensorflow/python/estimator:head",
         "//tensorflow/python/estimator:metric_keys",
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index f95fcc8039..42e1b7b68c 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -36,10 +36,12 @@ from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
 
 _DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
 
 
+# TODO(b/65403806): Switch loss_reduction default to SUM_OVER_BATCH_SIZE.
 def multi_class_head(n_classes,
                      weight_column=None,
                      label_vocabulary=None,
@@ -489,8 +491,8 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         processed_labels=processed_labels)
 
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None,
-      regularization_losses=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None, regularization_losses=None):
     """Returns an `EstimatorSpec`.
 
     Args:
@@ -502,8 +504,11 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         with shape `[D0, D1, ... DN, n_classes]` or `SparseTensor` with
         `dense_shape` `[D0, D1, ... DN, ?]`. `labels` is required argument when
         `mode` equals `TRAIN` or `EVAL`.
+      optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
+        Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
+        updates variables and increments `global_step`.
       train_op_fn: Function that takes a scalar loss `Tensor` and returns
-        `train_op`. Required in TRAIN mode.
+        `train_op`. Used if `optimizer` is `None`.
       regularization_losses: A list of additional scalar losses to be added to
         the training loss, such as regularization losses. These losses are
         usually expressed as a batch average, so for best results users need to
@@ -513,7 +518,8 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
     Returns:
       `EstimatorSpec`.
     Raises:
-      ValueError: If `train_op_fn` is `None` in TRAIN mode.
+      ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
+        mode, or if both are set.
     """
     with ops.name_scope(self._name, 'head'):
       logits = head_lib._check_logits_final_dim(logits, self.logits_dimension)  # pylint:disable=protected-access
@@ -565,8 +571,16 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                 regularization_loss=regularization_loss))
 
       # Train.
-      if train_op_fn is None:
-        raise ValueError('train_op_fn can not be None.')
+      if optimizer is not None:
+        if train_op_fn is not None:
+          raise ValueError('train_op_fn and optimizer cannot both be set.')
+        train_op = optimizer.minimize(
+            regularized_training_loss,
+            global_step=training_util.get_global_step())
+      elif train_op_fn is not None:
+        train_op = train_op_fn(regularized_training_loss)
+      else:
+        raise ValueError('train_op_fn and optimizer cannot both be None.')
       # Only summarize mean_loss for SUM reduction to preserve backwards
       # compatibility. Otherwise skip it to avoid unnecessary computation.
       if self._loss_reduction == losses.Reduction.SUM:
@@ -592,7 +606,7 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
         mode=model_fn.ModeKeys.TRAIN,
         predictions=predictions,
         loss=regularized_training_loss,
-        train_op=train_op_fn(regularized_training_loss))
+        train_op=train_op)
 
   def _eval_metric_ops(
       self, labels, probabilities, weights, unreduced_loss,
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index dc30dde877..776f0ee341 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -863,6 +863,42 @@ class MultiLabelHead(test.TestCase):
     self._test_train(
         head=head, logits=logits, labels=labels, expected_loss=expected_loss)
 
+  def test_train_with_optimizer(self):
+    head = head_lib.multi_label_head(n_classes=2)
+    logits = np.array([[-10., 10.], [-15., 10.]], dtype=np.float32)
+    labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
+    # For large logits, sigmoid cross entropy loss is approximated as:
+    # loss = labels * (logits < 0) * (-logits) +
+    #        (1 - labels) * (logits > 0) * logits =>
+    # expected_unweighted_loss = [[10., 10.], [15., 0.]]
+    # Average over classes, sum over weights.
+    expected_loss = 17.5
+    expected_train_result = 'my_train_op'
+
+    class _Optimizer(object):
+
+      def minimize(self, loss, global_step):
+        del global_step
+        return string_ops.string_join(
+            [constant_op.constant(expected_train_result),
+             string_ops.as_string(loss, precision=3)])
+
+    spec = head.create_estimator_spec(
+        features={'x': np.array(((42,),), dtype=np.int32)},
+        mode=model_fn.ModeKeys.TRAIN,
+        logits=logits,
+        labels=labels,
+        optimizer=_Optimizer())
+
+    tol = 1e-3
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      loss, train_result = sess.run((spec.loss, spec.train_op))
+      self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol)
+      self.assertEqual(
+          six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)),
+          train_result)
+
   def test_train_with_regularization_losses(self):
     head = head_lib.multi_label_head(
         n_classes=2, loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE)
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head.py b/tensorflow/contrib/estimator/python/estimator/multi_head.py
index 23d3714c53..bbbc19cc4d 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head.py
@@ -31,6 +31,7 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics as metrics_lib
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
 
 
 _DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
@@ -227,8 +228,10 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
         weights=example_weights_by_head,
         processed_labels=labels_by_head)
 
+  # TODO(b/65403806): Support regularization_losses arg.
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None):
     """See `_Head`."""
     if isinstance(logits, dict):
       logits_dict = logits
@@ -249,9 +252,10 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
               train_op_fn=_no_op_train_fn))
 
     if mode == model_fn.ModeKeys.TRAIN:
-      if train_op_fn is None:
-        raise ValueError('train_op_fn can not be None in TRAIN mode.')
-      spec = self._merge_train(all_estimator_spec, train_op_fn)
+      spec = self._merge_train(
+          all_estimator_spec=all_estimator_spec,
+          optimizer=optimizer,
+          train_op_fn=train_op_fn)
       with ops.name_scope(''):
         summary.scalar(metric_keys.MetricKeys.LOSS, spec.loss)
       return spec
@@ -280,16 +284,21 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
         begin_idx += head.logits_dimension
     return logits_dict
 
-  def _merge_train(self, all_estimator_spec, train_op_fn):
+  def _merge_train(self, all_estimator_spec, optimizer, train_op_fn):
     """Merges list of `EstimatorSpec` for training.
 
     Args:
       all_estimator_spec: list of `EstimatorSpec` for the individual heads.
-      train_op_fn: Function to create train op. See `create_estimator_spec`
-        documentation for more details.
+      optimizer: `Optimizer` instance to create train op. See
+        `create_estimator_spec` documentation for more details.
+      train_op_fn: Function to create train op. Used if `optimizer` is `None`.
 
     Returns:
       `EstimatorSpec` that merges all heads for TRAIN.
+
+    Raises:
+      ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
+        mode.
     """
     losses = []
     metrics = {}
@@ -298,11 +307,20 @@ class _MultiHead(head_lib._Head):  # pylint:disable=protected-access
       # Metric keys already contain head.name.
       metrics.update(spec.eval_metric_ops or {})
     loss = _merge_losses(losses, self._head_weights)
+    if optimizer is not None:
+      if train_op_fn is not None:
+        raise ValueError('train_op_fn and optimizer cannot both be set.')
+      train_op = optimizer.minimize(
+          loss, global_step=training_util.get_global_step())
+    elif train_op_fn is not None:
+      train_op = train_op_fn(loss)
+    else:
+      raise ValueError('train_op_fn and optimizer cannot both be None.')
 
     return model_fn.EstimatorSpec(
         mode=model_fn.ModeKeys.TRAIN,
         loss=loss,
-        train_op=train_op_fn(loss),
+        train_op=train_op,
         eval_metric_ops=metrics)
 
   def _merge_predict(self, all_estimator_spec):
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
index 8e788a9ce8..43cc157a1f 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
@@ -550,6 +550,44 @@ class MultiHeadTest(test.TestCase):
           metric_keys.MetricKeys.LOSS_MEAN + '/head1': expected_loss / 2,
       }, summary_str, tol)
 
+  def test_train_one_head_with_optimizer(self):
+    head1 = head_lib.multi_label_head(n_classes=2, name='head1')
+    multi_head = multi_head_lib.multi_head([head1])
+
+    logits = {'head1': np.array([[-10., 10.], [-15., 10.]], dtype=np.float32)}
+    labels = {'head1': np.array([[1, 0], [1, 1]], dtype=np.int64)}
+    # For large logits, sigmoid cross entropy loss is approximated as:
+    # loss = labels * (logits < 0) * (-logits) +
+    #        (1 - labels) * (logits > 0) * logits =>
+    # expected_unweighted_loss = [[10., 10.], [15., 0.]]
+    # Average over classes, sum over weights.
+    expected_loss = 17.5
+    expected_train_result = 'my_train_op'
+
+    class _Optimizer(object):
+
+      def minimize(self, loss, global_step):
+        del global_step
+        return string_ops.string_join(
+            [constant_op.constant(expected_train_result),
+             string_ops.as_string(loss, precision=3)])
+
+    spec = multi_head.create_estimator_spec(
+        features={'x': np.array(((42,),), dtype=np.int32)},
+        mode=model_fn.ModeKeys.TRAIN,
+        logits=logits,
+        labels=labels,
+        optimizer=_Optimizer())
+
+    tol = 1e-3
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      loss, train_result = sess.run((spec.loss, spec.train_op))
+      self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol)
+      self.assertEqual(
+          six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)),
+          train_result)
+
   def test_train_two_heads_with_weights(self):
     head1 = head_lib.multi_label_head(n_classes=2, name='head1')
     head2 = head_lib.multi_label_head(n_classes=3, name='head2')
diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index b25f9d2153..5afb5a7dd5 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -265,7 +265,6 @@ py_library(
         "//tensorflow/python:nn",
         "//tensorflow/python:partitioned_variables",
         "//tensorflow/python:summary",
-        "//tensorflow/python:training",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/feature_column",
         "//tensorflow/python/ops/losses",
@@ -617,6 +616,7 @@ py_library(
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:summary",
+        "//tensorflow/python:training",
         "//tensorflow/python:weights_broadcast_ops",
         "//tensorflow/python/feature_column",
         "//tensorflow/python/ops/losses",
diff --git a/tensorflow/python/estimator/canned/dnn.py b/tensorflow/python/estimator/canned/dnn.py
index 7043da8de0..6382622e0b 100644
--- a/tensorflow/python/estimator/canned/dnn.py
+++ b/tensorflow/python/estimator/canned/dnn.py
@@ -32,7 +32,6 @@ from tensorflow.python.ops import partitioned_variables
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.summary import summary
-from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
 
 # The default learning rate of 0.05 is a historical artifact of the initial
@@ -183,17 +182,11 @@ def _dnn_model_fn(features,
         input_layer_partitioner=input_layer_partitioner)
     logits = logit_fn(features=features, mode=mode)
 
-    def _train_op_fn(loss):
-      """Returns the op to optimize the loss."""
-      return optimizer.minimize(
-          loss,
-          global_step=training_util.get_global_step())
-
     return head.create_estimator_spec(
         features=features,
         mode=mode,
         labels=labels,
-        train_op_fn=_train_op_fn,
+        optimizer=optimizer,
         logits=logits)
 
 
diff --git a/tensorflow/python/estimator/canned/dnn_testing_utils.py b/tensorflow/python/estimator/canned/dnn_testing_utils.py
index 85b058caf3..44545c058c 100644
--- a/tensorflow/python/estimator/canned/dnn_testing_utils.py
+++ b/tensorflow/python/estimator/canned/dnn_testing_utils.py
@@ -53,7 +53,7 @@ from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import checkpoint_utils
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import monitored_session
-from tensorflow.python.training import optimizer
+from tensorflow.python.training import optimizer as optimizer_lib
 from tensorflow.python.training import saver
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
@@ -134,7 +134,8 @@ def mock_head(testcase, hidden_units, logits_dimension, expected_logits):
       hidden_weights_names + hidden_biases_names +
       [LOGITS_WEIGHTS_NAME + '/part_0:0', LOGITS_BIASES_NAME + '/part_0:0'])
 
-  def _create_estimator_spec(features, mode, logits, labels, train_op_fn):
+  def _create_estimator_spec(
+      features, mode, logits, labels, train_op_fn=None, optimizer=None):
     del features, labels  # Not used.
     trainable_vars = ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
     testcase.assertItemsEqual(expected_var_names,
@@ -144,8 +145,12 @@ def mock_head(testcase, hidden_units, logits_dimension, expected_logits):
         expected_logits, logits, message='Failed for mode={}. '.format(mode))
     with ops.control_dependencies([assert_logits]):
       if mode == model_fn.ModeKeys.TRAIN:
+        if train_op_fn is not None:
+          train_op = train_op_fn(loss)
+        elif optimizer is not None:
+          train_op = optimizer.minimize(loss, global_step=None)
         return model_fn.EstimatorSpec(
-            mode=mode, loss=loss, train_op=train_op_fn(loss))
+            mode=mode, loss=loss, train_op=train_op)
       elif mode == model_fn.ModeKeys.EVAL:
         return model_fn.EstimatorSpec(mode=mode, loss=array_ops.identity(loss))
       elif mode == model_fn.ModeKeys.PREDICT:
@@ -203,8 +208,8 @@ def mock_optimizer(testcase, hidden_units, expected_loss=None):
       return control_flow_ops.no_op()
 
   optimizer_mock = test.mock.NonCallableMagicMock(
-      spec=optimizer.Optimizer,
-      wraps=optimizer.Optimizer(use_locking=False, name='my_optimizer'))
+      spec=optimizer_lib.Optimizer,
+      wraps=optimizer_lib.Optimizer(use_locking=False, name='my_optimizer'))
   optimizer_mock.minimize = test.mock.MagicMock(wraps=_minimize)
 
   return optimizer_mock
diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index f68204a35e..c9635a9c27 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -44,6 +44,7 @@ from tensorflow.python.ops import weights_broadcast_ops
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.saved_model import signature_constants
 from tensorflow.python.summary import summary
+from tensorflow.python.training import training_util
 
 _DEFAULT_SERVING_KEY = signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY
 
@@ -85,40 +86,39 @@ class _Head(object):
     ```python
     def _my_dnn_model_fn(features, labels, mode, params, config=None):
       # Optionally your callers can pass head to model_fn as a param.
-      head = tf.contrib.learn.regression_head(...)
-      input = tf.contrib.layers.input_from_feature_columns(features, ...)
-      last_hidden_layer_out = tf.contrib.layers.stack(
-          input, tf.contrib.layers.fully_connected, [1000, 500])
-      logits = tf.contrib.layers.fully_connected(
-          last_hidden_layer_out, head.logits_dimension, activation_fn=None)
-
-      def _train_op_fn(loss):
-        return optimizer.minimize(loss)
+      head = tf.contrib.estimator.regression_head(...)
+      inputs = tf.feature_column.input_layer(features, ...)
+      hidden_layer0 = tf.layers.dense(
+          inputs, units=1000, activation=tf.nn.relu)
+      hidden_layer1 = tf.layers.dense(
+          hidden_layer0, units=500, activation=tf.nn.relu)
+      logits = tf.layers.dense(
+          hidden_layer1, units=head.logits_dimension, activation=None)
 
       return head.create_estimator_spec(
           features=features,
           labels=labels,
           mode=mode,
           logits=logits,
-          train_op_fn=_train_op_fn)
+          optimizer=optimizer)
     ```
 
   There are cases where computing and applying gradients can not be meaningfully
-  captured with train_op_fn we support (for example, with sync optimizer). In
-  such case, you can take the responsibility on your own. Here is a common
-  use case,
+  captured with optimizer or train_op_fn we support (for example, with sync
+  optimizer). In such case, you can take the responsibility on your own. Here is
+  a common use case,
     ```python
     estimator_spec = head.create_estimator_spec(
         features=features,
         labels=labels,
         mode=mode,
         logits=logits,
-        train_op_fn=tf.contrib.learn.no_op_train_fn)
+        train_op_fn=lambda _: tf.no_op())
     if mode == model_fn.ModeKeys.TRAIN:
       optimizer = ...
       sync = tf.train.SyncReplicasOptimizer(opt=optimizer, ...)
-      update_op = tf.contrib.layers.optimize_loss(optimizer=sync,
-                                                  loss=estimator_spec.loss, ...)
+      update_op = sync.minimize(
+          estimator_spec.loss, global_step=tf.get_global_step())
       hooks = [sync.make_session_run_hook(is_chief)]
       ... update train_op and hooks in EstimatorSpec and return
     ```
@@ -172,10 +172,12 @@ class _Head(object):
     """
     raise NotImplementedError('Calling an abstract method.')
 
+  # TODO(b/65403806): By default, collect regularization_losses from
+  # GraphKeys.REGULARIZATION_LOSSES collection.
   @abc.abstractmethod
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None,
-      regularization_losses=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None, regularization_losses=None):
     """Returns `EstimatorSpec` that a model_fn can return.
 
     Please note that,
@@ -186,10 +188,14 @@ class _Head(object):
       mode: Estimator's `ModeKeys`.
       logits: logits `Tensor` to be used by the head.
       labels: Labels `Tensor`, or `dict` of same.
+      optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
+        Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
+        updates variables and increments `global_step`.
       train_op_fn: Function that takes a scalar loss `Tensor` and returns an op
-        to optimize the model with the loss. This is used in TRAIN mode and
-        must not be None. None is allowed in other modes. If you want to
-        optimize loss yourself you can pass `no_op_train_fn` and then use
+        to optimize the model with the loss in TRAIN mode. Used if `optimizer`
+        is `None`. Exactly one of `train_op_fn` and `optimizer` must be set in
+        TRAIN mode. None is allowed in other modes. If you want to optimize loss
+        yourself you can pass `lambda _: tf.no_op()` and then use
         EstimatorSpec.loss to compute and apply gradients.
       regularization_losses: A list of additional scalar losses to be added to
         the training loss, such as regularization losses.
@@ -694,8 +700,8 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
         processed_labels=label_ids)
 
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None,
-      regularization_losses=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None, regularization_losses=None):
     """Returns an `EstimatorSpec`.
 
     Args:
@@ -706,8 +712,11 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
       labels: Labels integer or string `Tensor` with shape matching `logits`,
         namely `[D0, D1, ... DN, 1]` or `[D0, D1, ... DN]`. `labels` is
         required argument when `mode` equals `TRAIN` or `EVAL`.
+      optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
+        Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
+        updates variables and increments `global_step`.
       train_op_fn: Function that takes a scalar loss `Tensor` and returns
-        `train_op`. Required in TRAIN mode.
+        `train_op`. Used if `optimizer` is `None`.
       regularization_losses: A list of additional scalar losses to be added to
         the training loss, such as regularization losses. These losses are
         usually expressed as a batch average, so for best results users need to
@@ -717,7 +726,8 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
     Returns:
       `EstimatorSpec`.
     Raises:
-      ValueError: If `train_op_fn` is `None` in TRAIN mode.
+      ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
+        mode, or if both are set.
     """
     with ops.name_scope(self._name, 'head'):
       logits = _check_logits_final_dim(logits, self.logits_dimension)
@@ -780,8 +790,16 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
                 regularization_loss=regularization_loss))
 
       # Train.
-      if train_op_fn is None:
-        raise ValueError('train_op_fn cannot be None.')
+      if optimizer is not None:
+        if train_op_fn is not None:
+          raise ValueError('train_op_fn and optimizer cannot both be set.')
+        train_op = optimizer.minimize(
+            regularized_training_loss,
+            global_step=training_util.get_global_step())
+      elif train_op_fn is not None:
+        train_op = train_op_fn(regularized_training_loss)
+      else:
+        raise ValueError('train_op_fn and optimizer cannot both be None.')
       # Only summarize mean_loss for SUM reduction to preserve backwards
       # compatibility. Otherwise skip it to avoid unnecessary computation.
       if self._loss_reduction == losses.Reduction.SUM:
@@ -807,7 +825,7 @@ class _MultiClassHeadWithSoftmaxCrossEntropyLoss(_Head):
         mode=model_fn.ModeKeys.TRAIN,
         predictions=predictions,
         loss=regularized_training_loss,
-        train_op=train_op_fn(regularized_training_loss))
+        train_op=train_op)
 
 
 def _binary_logistic_head_with_sigmoid_cross_entropy_loss(
@@ -1039,8 +1057,8 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
         processed_labels=labels)
 
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None,
-      regularization_losses=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None, regularization_losses=None):
     """Returns an `EstimatorSpec`.
 
     Args:
@@ -1051,8 +1069,11 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
       labels: Labels integer or string `Tensor` with shape matching `logits`,
         namely `[D0, D1, ... DN, 1]` or `[D0, D1, ... DN]`. `labels` is required
         argument when `mode` equals `TRAIN` or `EVAL`.
+      optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
+        Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
+        updates variables and increments `global_step`.
       train_op_fn: Function that takes a scalar loss `Tensor` and returns
-        `train_op`. Required in TRAIN mode.
+        `train_op`. Used if `optimizer` is `None`.
       regularization_losses: A list of additional scalar losses to be added to
         the training loss, such as regularization losses. These losses are
         usually expressed as a batch average, so for best results users need to
@@ -1062,7 +1083,8 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
     Returns:
       `EstimatorSpec`.
     Raises:
-      ValueError: If `train_op_fn` is `None` in TRAIN mode.
+      ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
+        mode, or if both are set.
     """
     # Predict.
     with ops.name_scope(self._name, 'head'):
@@ -1134,8 +1156,16 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
                 regularization_loss=regularization_loss))
 
       # Train.
-      if train_op_fn is None:
-        raise ValueError('train_op_fn can not be None.')
+      if optimizer is not None:
+        if train_op_fn is not None:
+          raise ValueError('train_op_fn and optimizer cannot both be set.')
+        train_op = optimizer.minimize(
+            regularized_training_loss,
+            global_step=training_util.get_global_step())
+      elif train_op_fn is not None:
+        train_op = train_op_fn(regularized_training_loss)
+      else:
+        raise ValueError('train_op_fn and optimizer cannot both be None.')
       # Only summarize mean_loss for SUM reduction to preserve backwards
       # compatibility. Otherwise skip it to avoid unnecessary computation.
       if self._loss_reduction == losses.Reduction.SUM:
@@ -1160,7 +1190,7 @@ class _BinaryLogisticHeadWithSigmoidCrossEntropyLoss(_Head):
         mode=model_fn.ModeKeys.TRAIN,
         predictions=predictions,
         loss=regularized_training_loss,
-        train_op=train_op_fn(regularized_training_loss))
+        train_op=train_op)
 
 
 def _regression_head_with_mean_squared_error_loss(
@@ -1289,8 +1319,8 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
         processed_labels=labels)
 
   def create_estimator_spec(
-      self, features, mode, logits, labels=None, train_op_fn=None,
-      regularization_losses=None):
+      self, features, mode, logits, labels=None, optimizer=None,
+      train_op_fn=None, regularization_losses=None):
     """Returns an `EstimatorSpec`.
 
     Args:
@@ -1302,8 +1332,11 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
         `[D0, D1, ... DN, logits_dimension]`. When `logits_dimension=1`, shape
         `[D0, D1, ... DN]` is also supported. `labels` is required argument when
         `mode` equals `TRAIN` or `EVAL`.
+      optimizer: `Optimizer` instance to optimize the loss in TRAIN mode.
+        Namely, sets `train_op = optimizer.minimize(loss, global_step)`, which
+        updates variables and increments `global_step`.
       train_op_fn: Function that takes a scalar loss `Tensor` and returns
-        `train_op`. Required in TRAIN mode.
+        `train_op`. Used if `optimizer` is `None`.
       regularization_losses: A list of additional scalar losses to be added to
         the training loss, such as regularization losses. These losses are
         usually expressed as a batch average, so for best results users need to
@@ -1313,7 +1346,8 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
     Returns:
       `EstimatorSpec`.
     Raises:
-      ValueError: If `train_op_fn` is `None` in TRAIN mode.
+      ValueError: If both `train_op_fn` and `optimizer` are `None` in TRAIN
+        mode, or if both are set.
     """
     # Predict.
     with ops.name_scope(self._name, 'head'):
@@ -1373,8 +1407,16 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
             eval_metric_ops=eval_metric_ops)
 
       # Train.
-      if train_op_fn is None:
-        raise ValueError('train_op_fn can not be None.')
+      if optimizer is not None:
+        if train_op_fn is not None:
+          raise ValueError('train_op_fn and optimizer cannot both be set.')
+        train_op = optimizer.minimize(
+            regularized_training_loss,
+            global_step=training_util.get_global_step())
+      elif train_op_fn is not None:
+        train_op = train_op_fn(regularized_training_loss)
+      else:
+        raise ValueError('train_op_fn and optimizer cannot both be None.')
       # Only summarize mean_loss for SUM reduction to preserve backwards
       # compatibility. Otherwise skip it to avoid unnecessary computation.
       if self._loss_reduction == losses.Reduction.SUM:
@@ -1399,7 +1441,7 @@ class _RegressionHeadWithMeanSquaredErrorLoss(_Head):
         mode=model_fn.ModeKeys.TRAIN,
         predictions=predictions,
         loss=regularized_training_loss,
-        train_op=train_op_fn(regularized_training_loss))
+        train_op=train_op)
 
 
 def _assert_range(labels, n_classes, message=None):
diff --git a/tensorflow/python/estimator/canned/head_test.py b/tensorflow/python/estimator/canned/head_test.py
index b5d35c9b45..fe6ee07529 100644
--- a/tensorflow/python/estimator/canned/head_test.py
+++ b/tensorflow/python/estimator/canned/head_test.py
@@ -842,6 +842,41 @@ class MultiClassHeadWithSoftmaxCrossEntropyLoss(test.TestCase):
           metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2,
       }, summary_str, tol)
 
+  def test_train_with_optimizer(self):
+    n_classes = 3
+    head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(n_classes)
+
+    logits = np.array(((10, 0, 0), (0, 10, 0),), dtype=np.float32)
+    labels = np.array(((1,), (1,)), dtype=np.int64)
+    features = {'x': np.array(((42,),), dtype=np.int32)}
+    expected_train_result = 'my_train_op'
+
+    class _Optimizer(object):
+
+      def minimize(self, loss, global_step):
+        del global_step
+        return string_ops.string_join(
+            [constant_op.constant(expected_train_result),
+             string_ops.as_string(loss, precision=2)])
+
+    # loss = sum(cross_entropy(labels, logits)) = sum(10, 0) = 10.
+    expected_loss = 10.
+    spec = head.create_estimator_spec(
+        features=features,
+        mode=model_fn.ModeKeys.TRAIN,
+        logits=logits,
+        labels=labels,
+        optimizer=_Optimizer())
+
+    tol = 1e-2
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      loss, train_result = sess.run((spec.loss, spec.train_op))
+      self.assertAllClose(expected_loss, loss, rtol=tol, atol=tol)
+      self.assertEqual(
+          six.b('{0:s}{1:.2f}'.format(expected_train_result, expected_loss)),
+          train_result)
+
   def test_train_summaries_with_head_name(self):
     n_classes = 3
     head = head_lib._multi_class_head_with_softmax_cross_entropy_loss(
@@ -1942,6 +1977,39 @@ class BinaryLogisticHeadWithSigmoidCrossEntropyLossTest(test.TestCase):
           metric_keys.MetricKeys.LOSS_MEAN: 20.5,
       }, summary_str)
 
+  def test_train_with_optimizer(self):
+    head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss()
+
+    logits = np.array(((45,), (-41,),), dtype=np.float32)
+    labels = np.array(((1,), (1,),), dtype=np.float64)
+    expected_train_result = b'my_train_op'
+    features = {'x': np.array(((42,),), dtype=np.float32)}
+    # loss = sum(cross_entropy(labels, logits)) = sum(0, 41) = 41
+    expected_loss = 41.
+
+    class _Optimizer(object):
+
+      def minimize(self, loss, global_step):
+        del global_step
+        with ops.control_dependencies((check_ops.assert_equal(
+            math_ops.to_float(expected_loss), math_ops.to_float(loss),
+            name='assert_loss'),)):
+          return constant_op.constant(expected_train_result)
+
+    # Create estimator spec.
+    spec = head.create_estimator_spec(
+        features=features,
+        mode=model_fn.ModeKeys.TRAIN,
+        logits=logits,
+        labels=labels,
+        optimizer=_Optimizer())
+
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      loss, train_result = sess.run((spec.loss, spec.train_op))
+      self.assertAllClose(expected_loss, loss)
+      self.assertEqual(expected_train_result, train_result)
+
   def test_train_summaries_with_head_name(self):
     head = head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(
         name='some_binary_head')
@@ -3076,6 +3144,40 @@ class RegressionHeadWithMeanSquaredErrorLossTest(test.TestCase):
           metric_keys.MetricKeys.LOSS_MEAN: 6.5,
       }, summary_str)
 
+  def test_train_with_optimizer(self):
+    head = head_lib._regression_head_with_mean_squared_error_loss()
+    self.assertEqual(1, head.logits_dimension)
+
+    # Create estimator spec.
+    logits = np.array(((45,), (41,),), dtype=np.float32)
+    labels = np.array(((43.,), (44.,),), dtype=np.float64)
+    expected_train_result = b'my_train_op'
+    features = {'x': np.array(((42.,),), dtype=np.float32)}
+    # loss = (43-45)^2 + (44-41)^2 = 4 + 9 = 13
+    expected_loss = 13
+
+    class _Optimizer(object):
+
+      def minimize(self, loss, global_step):
+        del global_step
+        with ops.control_dependencies((check_ops.assert_equal(
+            math_ops.to_float(expected_loss), math_ops.to_float(loss),
+            name='assert_loss'),)):
+          return constant_op.constant(expected_train_result)
+
+    spec = head.create_estimator_spec(
+        features=features,
+        mode=model_fn.ModeKeys.TRAIN,
+        logits=logits,
+        labels=labels,
+        optimizer=_Optimizer())
+
+    with self.test_session() as sess:
+      _initialize_variables(self, spec.scaffold)
+      loss, train_result = sess.run((spec.loss, spec.train_op))
+      self.assertAllClose(expected_loss, loss)
+      self.assertEqual(expected_train_result, train_result)
+
   def test_train_summaries_with_head_name(self):
     head = head_lib._regression_head_with_mean_squared_error_loss(
         name='some_regression_head')
diff --git a/tensorflow/python/estimator/canned/linear.py b/tensorflow/python/estimator/canned/linear.py
index a2f24ef270..e7ec417991 100644
--- a/tensorflow/python/estimator/canned/linear.py
+++ b/tensorflow/python/estimator/canned/linear.py
@@ -33,7 +33,6 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.summary import summary
 from tensorflow.python.training import ftrl
-from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -157,17 +156,11 @@ def _linear_model_fn(features, labels, mode, head, feature_columns, optimizer,
         units=head.logits_dimension, feature_columns=feature_columns)
     logits = logit_fn(features=features)
 
-    def _train_op_fn(loss):
-      """Returns the op to optimize the loss."""
-      return optimizer.minimize(
-          loss,
-          global_step=training_util.get_global_step())
-
     return head.create_estimator_spec(
         features=features,
         mode=mode,
         labels=labels,
-        train_op_fn=_train_op_fn,
+        optimizer=optimizer,
         logits=logits)
 
 
-- 
GitLab


From 48b0fb7a524425d57547dc23093d869538b888db Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 22 Mar 2018 14:09:59 -0700
Subject: [PATCH 373/960] Fetch C shapes for ops created by import_graph_def
 with C API enabled.

If _USE_C_API = True, this change makes us always fetch shapes using
the C API after calling TF_ImportGraphDef, even if _USE_C_SHAPES =
False. This is necessary to preserve the shapes specified by the
"_output_shapes" attr on imported NodeDefs (note that this attr isn't
present on the NodeDefs of the imported nodes, so there's no other way
to recover this information after calling TF_ImportGraphDef).

PiperOrigin-RevId: 190122991
---
 tensorflow/python/framework/ops.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index de222e1932..93edaa0cf0 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3455,12 +3455,12 @@ class Graph(object):
     ]
 
     for op in new_ops:
-      # The Python shape inference code does not support imported functions. It
-      # also needs access to op.inputs, which is why we call it here.
+      # Operations created by the C API always retrieve shapes from the C API so
+      # we preserve the shapes of ops created in import_graph_def (from the
+      # "_output_shapes" attr of the imported NodeDef).
       # TODO(b/74620627): move this back to _create_op_helper once _USE_C_SHAPES
       # is removed.
-      if not self._is_function(op.type) or _USE_C_SHAPES:
-        set_shapes_for_outputs(op)
+      _set_shapes_for_outputs_c_api(op)
       new_control_inputs = self._control_dependencies_for_inputs(op.inputs)
       # pylint: disable=protected-access
       op._add_control_inputs(new_control_inputs)
-- 
GitLab


From e3468b56d323783fdfb79fa2d6c24effc58bcaa9 Mon Sep 17 00:00:00 2001
From: Brian Patton <bjp@google.com>
Date: Thu, 22 Mar 2018 14:11:08 -0700
Subject: [PATCH 374/960] Adds float64 support for Conv2d, Conv2dBackpropInput,
 and Conv2dBackpropFilter

PiperOrigin-RevId: 190123191
---
 .../core/kernels/conv_grad_filter_ops.cc      |   7 +
 .../core/kernels/conv_grad_input_ops.cc       |   7 +
 tensorflow/core/kernels/conv_ops.cc           |   7 +-
 tensorflow/core/kernels/conv_ops_gpu_2.cu.cc  |   3 +
 tensorflow/core/kernels/conv_ops_gpu_3.cu.cc  |   3 +
 tensorflow/core/kernels/depthwise_conv_op.cc  |   4 +-
 .../core/kernels/eigen_spatial_convolutions.h | 182 ++++++++++++++++++
 tensorflow/core/ops/nn_ops.cc                 |   6 +-
 .../python/kernel_tests/conv_ops_test.py      |   4 +-
 9 files changed, 215 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/conv_grad_filter_ops.cc b/tensorflow/core/kernels/conv_grad_filter_ops.cc
index e6ae595291..66ee474ca3 100644
--- a/tensorflow/core/kernels/conv_grad_filter_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_filter_ops.cc
@@ -520,6 +520,7 @@ class Conv2DCustomBackpropFilterOp : public OpKernel {
 
 TF_CALL_half(REGISTER_CPU_KERNELS);
 TF_CALL_float(REGISTER_CPU_KERNELS);
+TF_CALL_double(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 // GPU definitions.
@@ -1017,11 +1018,17 @@ namespace functor {
       typename TTypes<T, 4, int>::Tensor out, TensorFormat data_format); \
   extern template struct PadInput<GPUDevice, T, int, 4>;
 
+DECLARE_GPU_SPEC(double);
 DECLARE_GPU_SPEC(float);
 DECLARE_GPU_SPEC(Eigen::half);
 #undef DECLARE_GPU_SPEC
 }  // namespace functor
 
+REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropFilter")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<double>("T")
+                            .HostMemory("filter_sizes"),
+                        Conv2DSlowBackpropFilterOp<GPUDevice, double>);
 REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropFilter")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<float>("T")
diff --git a/tensorflow/core/kernels/conv_grad_input_ops.cc b/tensorflow/core/kernels/conv_grad_input_ops.cc
index 15c55e4d99..71ea0d5d72 100644
--- a/tensorflow/core/kernels/conv_grad_input_ops.cc
+++ b/tensorflow/core/kernels/conv_grad_input_ops.cc
@@ -592,6 +592,7 @@ class Conv2DCustomBackpropInputOp : public OpKernel {
 
 TF_CALL_half(REGISTER_CPU_KERNELS);
 TF_CALL_float(REGISTER_CPU_KERNELS);
+TF_CALL_double(REGISTER_CPU_KERNELS);
 #undef REGISTER_CPU_KERNELS
 
 // GPU definitions.
@@ -1090,11 +1091,17 @@ namespace functor {
       typename TTypes<T, 4, int>::Tensor out, TensorFormat data_format); \
   extern template struct PadInput<GPUDevice, T, int, 4>;
 
+DECLARE_GPU_SPEC(double);
 DECLARE_GPU_SPEC(float);
 DECLARE_GPU_SPEC(Eigen::half);
 #undef DECLARE_GPU_SPEC
 }  // namespace functor
 
+REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropInput")
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<double>("T")
+                            .HostMemory("input_sizes"),
+                        Conv2DSlowBackpropInputOp<GPUDevice, double>);
 REGISTER_KERNEL_BUILDER(Name("Conv2DBackpropInput")
                             .Device(DEVICE_GPU)
                             .TypeConstraint<float>("T")
diff --git a/tensorflow/core/kernels/conv_ops.cc b/tensorflow/core/kernels/conv_ops.cc
index 47f6907c04..88843e4da7 100644
--- a/tensorflow/core/kernels/conv_ops.cc
+++ b/tensorflow/core/kernels/conv_ops.cc
@@ -446,10 +446,11 @@ class Conv2DOp : public BinaryOp<T> {
 #if !defined(USE_GEMM_FOR_CONV)
 TF_CALL_half(REGISTER_CPU);
 TF_CALL_float(REGISTER_CPU);
+TF_CALL_double(REGISTER_CPU);
 #endif  // USE_GEMM_FOR_CONV
 
 // To be used inside depthwise_conv_op.cc.
-template class LaunchConv2DOp<CPUDevice, float>;
+template struct LaunchConv2DOp<CPUDevice, float>;
 
 #if GOOGLE_CUDA
 int64 GetCudnnWorkspaceLimit(const string& envvar_in_mb,
@@ -810,6 +811,7 @@ namespace functor {
       typename TTypes<T, 4, int>::Tensor out, TensorFormat data_format);     \
   extern template struct PadInput<GPUDevice, T, int, 4>
 
+DECLARE_GPU_SPEC(double);
 DECLARE_GPU_SPEC(float);
 DECLARE_GPU_SPEC(Eigen::half);
 #undef DECLARE_GPU_SPEC
@@ -822,6 +824,9 @@ REGISTER_KERNEL_BUILDER(
 REGISTER_KERNEL_BUILDER(
     Name("Conv2D").Device(DEVICE_GPU).TypeConstraint<float>("T"),
     Conv2DOp<GPUDevice, float>);
+REGISTER_KERNEL_BUILDER(
+    Name("Conv2D").Device(DEVICE_GPU).TypeConstraint<double>("T"),
+    Conv2DOp<GPUDevice, double>);
 
 // To be used inside depthwise_conv_op.cc.
 template class LaunchConv2DOp<GPUDevice, float>;
diff --git a/tensorflow/core/kernels/conv_ops_gpu_2.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_2.cu.cc
index b5dd26a9e4..52859af950 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_2.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_2.cu.cc
@@ -25,6 +25,9 @@ limitations under the License.
 namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
+template struct functor::InflatePadAndShuffle<GPUDevice, double, 4, int>;
+template struct functor::InflatePadAndShuffle<GPUDevice, double, 4,
+                                              Eigen::DenseIndex>;
 template struct functor::InflatePadAndShuffle<GPUDevice, float, 4, int>;
 template struct functor::InflatePadAndShuffle<GPUDevice, float, 4,
                                               Eigen::DenseIndex>;
diff --git a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
index a376534bad..2503b475dc 100644
--- a/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
+++ b/tensorflow/core/kernels/conv_ops_gpu_3.cu.cc
@@ -1039,9 +1039,11 @@ template struct functor::SwapDimension0And2InTensor3<GPUDevice, double2,
                                                      /*conjugate=*/true>;
 
 // For 2d ops.
+template struct functor::TransformFilter<GPUDevice, double, int, 4>;
 template struct functor::TransformFilter<GPUDevice, float, int, 4>;
 template struct functor::TransformFilter<GPUDevice, Eigen::half, int, 4>;
 
+template struct functor::ReverseTransformFilter<GPUDevice, double, 4>;
 template struct functor::ReverseTransformFilter<GPUDevice, float, 4>;
 template struct functor::ReverseTransformFilter<GPUDevice, Eigen::half, 4>;
 
@@ -1054,6 +1056,7 @@ template struct functor::NCHWToNHWC<GPUDevice, float, 4>;
 template struct functor::NCHWToNHWC<GPUDevice, Eigen::half, 4>;
 
 template struct functor::PadInput<GPUDevice, int, int, 4>;
+template struct functor::PadInput<GPUDevice, double, int, 4>;
 template struct functor::PadInput<GPUDevice, float, int, 4>;
 template struct functor::PadInput<GPUDevice, Eigen::half, int, 4>;
 
diff --git a/tensorflow/core/kernels/depthwise_conv_op.cc b/tensorflow/core/kernels/depthwise_conv_op.cc
index c060b2e14d..6dedb1a61e 100644
--- a/tensorflow/core/kernels/depthwise_conv_op.cc
+++ b/tensorflow/core/kernels/depthwise_conv_op.cc
@@ -241,7 +241,7 @@ struct LaunchDepthwiseConvOp<CPUDevice, T> {
 };
 
 // Extern template instantiated in conv_ops.cc.
-extern template class LaunchConv2DOp<CPUDevice, float>;
+extern template struct LaunchConv2DOp<CPUDevice, float>;
 
 #if GOOGLE_CUDA
 
@@ -251,7 +251,7 @@ extern template struct LaunchDepthwiseConvOp<GPUDevice, float>;
 extern template struct LaunchDepthwiseConvOp<GPUDevice, double>;
 
 // Extern template instantiated in conv_ops.cc.
-extern template class LaunchConv2DOp<GPUDevice, float>;
+extern template struct LaunchConv2DOp<GPUDevice, float>;
 
 #endif
 
diff --git a/tensorflow/core/kernels/eigen_spatial_convolutions.h b/tensorflow/core/kernels/eigen_spatial_convolutions.h
index 1acbe3a658..a4dff4b91c 100644
--- a/tensorflow/core/kernels/eigen_spatial_convolutions.h
+++ b/tensorflow/core/kernels/eigen_spatial_convolutions.h
@@ -797,6 +797,188 @@ struct gemm_pack_rhs<
   }
 };
 
+// Template specialization for packet_size = 2. We must special-case packet
+// blocks with nr > packet_size, e.g. PacketBlock<Packet2d, 4>.
+template <typename NewDimension, DenseIndex Rows, DenseIndex Cols,
+          typename ArgType, typename Device, typename Scalar, typename Index,
+          typename nocontract_t, typename contract_t, bool inner_dim_contiguous,
+          bool inner_dim_reordered, int Alignment, int nr>
+struct gemm_pack_rhs<
+    Scalar, Index,
+    TensorContractionSubMapper<
+        Scalar, Index, Rhs,
+        TensorEvaluator<
+            const TensorReshapingOp<
+                NewDimension, const TensorImagePatchOp<Rows, Cols, ArgType> >,
+            Device>,
+        nocontract_t, contract_t, 2, inner_dim_contiguous, inner_dim_reordered,
+        Alignment>,
+    nr, ColMajor, false, false> {
+  typedef TensorContractionSubMapper<
+      Scalar, Index, Rhs,
+      TensorEvaluator<
+          const TensorReshapingOp<
+              NewDimension, const TensorImagePatchOp<Rows, Cols, ArgType> >,
+          Device>,
+      nocontract_t, contract_t, 2, inner_dim_contiguous, inner_dim_reordered,
+      Alignment>
+      SubMapper;
+  typedef SubMapper DataMapper;
+
+  EIGEN_DEVICE_FUNC
+  static inline Index ceil_div(Index a, Index b) { return (a + b - 1) / b; }
+
+  EIGEN_DEVICE_FUNC
+  EIGEN_DONT_INLINE void operator()(Scalar* block, const DataMapper& rhs,
+                                    Index depth, Index cols, Index stride = 0,
+                                    Index offset = 0) const {
+    eigen_assert(stride == 0);
+    eigen_assert(offset == 0);
+
+    EIGEN_STATIC_ASSERT((nr == 4), YOU_MADE_A_PROGRAMMING_MISTAKE);
+    typedef typename packet_traits<Scalar>::type Packet;
+
+    const int packet_size = 2;
+    const Index packet_cols4 = (cols / 4) * 4;
+    const Index peeled_k = (depth / packet_size) * packet_size;
+    const bool non_standard_patches = rhs.nonStandardPatches();
+
+    for (Index j2 = 0; j2 < packet_cols4; j2 += 4) {
+      const SubMapper dm0 = rhs.getLinearMapper(0, j2 + 0);
+      const SubMapper dm1 = rhs.getLinearMapper(0, j2 + 1);
+      const SubMapper dm2 = rhs.getLinearMapper(0, j2 + 2);
+      const SubMapper dm3 = rhs.getLinearMapper(0, j2 + 3);
+
+      Index k = 0;
+      if (!non_standard_patches) {
+        const Index patch_depth = rhs.patchDepth();
+        if ((patch_depth % packet_size) == 0) {
+          const Index patch_cols = rhs.patchCols();
+          const Index patch_rows = rhs.patchRows();
+
+          const Index startCol = rhs.colOffset();
+          const Index max_cols = std::min<Index>(
+              ceil_div(peeled_k, patch_rows * patch_depth) + startCol,
+              patch_cols);
+
+          for (Index c = startCol; c < max_cols; ++c) {
+            eigen_assert(k < peeled_k);
+            const Index startRow = (c == startCol) ? rhs.rowOffset() : 0;
+            const Index max_rows = std::min<Index>(
+                ceil_div(peeled_k - c * patch_rows * patch_depth, patch_depth) +
+                    startRow,
+                patch_rows);
+
+            const bool pad_col0 = dm0.padCol(c);
+            const bool pad_col1 = dm1.padCol(c);
+            const bool pad_col2 = dm2.padCol(c);
+            const bool pad_col3 = dm3.padCol(c);
+            for (Index r = startRow; r < max_rows; ++r) {
+              eigen_assert(k < peeled_k);
+              const bool pad0 = pad_col0 || dm0.padRow(r);
+              const bool pad1 = pad_col1 || dm1.padRow(r);
+              const bool pad2 = pad_col2 || dm2.padRow(r);
+              const bool pad3 = pad_col3 || dm3.padRow(r);
+
+              const Index idx0 = dm0.baseIndex(r, c);
+              const Index idx1 = dm1.baseIndex(r, c);
+              const Index idx2 = dm2.baseIndex(r, c);
+              const Index idx3 = dm3.baseIndex(r, c);
+
+              const Index startDepth =
+                  ((c == startCol) && (r == startRow)) ? rhs.depthOffset() : 0;
+              const Index max_depth =
+                  std::min<Index>(peeled_k - c * patch_rows * patch_depth -
+                                      r * patch_depth + startDepth,
+                                  patch_depth);
+              eigen_assert((max_depth - startDepth) % packet_size == 0);
+              for (Index d = startDepth; d < max_depth; d += packet_size) {
+                eigen_assert(k < peeled_k);
+                PacketBlock<Packet, 2> kernel0;
+                PacketBlock<Packet, 2> kernel1;
+                kernel0.packet[0] = pad0 ? pset1<Packet>(Scalar(0))
+                                         : rhs.packetNoPadding(d, idx0);
+                kernel0.packet[1] = pad1 ? pset1<Packet>(Scalar(0))
+                                         : rhs.packetNoPadding(d, idx1);
+                kernel1.packet[0] = pad2 ? pset1<Packet>(Scalar(0))
+                                         : rhs.packetNoPadding(d, idx2);
+                kernel1.packet[1] = pad3 ? pset1<Packet>(Scalar(0))
+                                         : rhs.packetNoPadding(d, idx3);
+                ptranspose(kernel0);
+                ptranspose(kernel1);
+                pstoreu(block + 0 * packet_size, kernel0.packet[0]);
+                pstoreu(block + 1 * packet_size, kernel1.packet[0]);
+                pstoreu(block + 2 * packet_size, kernel0.packet[1]);
+                pstoreu(block + 3 * packet_size, kernel1.packet[1]);
+                block += 4 * packet_size;
+                k += packet_size;
+              }
+            }
+          }
+
+          for (; k < peeled_k; k += packet_size) {
+            PacketBlock<Packet, 2> kernel0;
+            PacketBlock<Packet, 2> kernel1;
+            kernel0.packet[0] = dm0.loadPacketFast(k);
+            kernel0.packet[1] = dm1.loadPacketFast(k);
+            kernel1.packet[0] = dm2.loadPacketFast(k);
+            kernel1.packet[1] = dm3.loadPacketFast(k);
+            ptranspose(kernel0);
+            ptranspose(kernel1);
+            pstoreu(block + 0 * packet_size, kernel0.packet[0]);
+            pstoreu(block + 1 * packet_size, kernel1.packet[0]);
+            pstoreu(block + 2 * packet_size, kernel0.packet[1]);
+            pstoreu(block + 3 * packet_size, kernel1.packet[1]);
+            block += 4 * packet_size;
+          }
+        } else {
+          for (; k < peeled_k; k += packet_size) {
+            PacketBlock<Packet, 2> kernel0;
+            PacketBlock<Packet, 2> kernel1;
+            kernel0.packet[0] = dm0.loadPacketStandard(k);
+            kernel0.packet[1] = dm1.loadPacketStandard(k);
+            kernel1.packet[0] = dm2.loadPacketStandard(k);
+            kernel1.packet[1] = dm3.loadPacketStandard(k);
+            ptranspose(kernel0);
+            ptranspose(kernel1);
+            pstoreu(block + 0 * packet_size, kernel0.packet[0]);
+            pstoreu(block + 1 * packet_size, kernel1.packet[0]);
+            pstoreu(block + 2 * packet_size, kernel0.packet[1]);
+            pstoreu(block + 3 * packet_size, kernel1.packet[1]);
+            block += 4 * packet_size;
+          }
+        }
+      }
+      if (!rhs.nonStandardPatches()) {
+        for (; k < depth; k++) {
+          block[0] = dm0.loadCoeffStandard(k);
+          block[1] = dm1.loadCoeffStandard(k);
+          block[2] = dm2.loadCoeffStandard(k);
+          block[3] = dm3.loadCoeffStandard(k);
+          block += 4;
+        }
+      } else {
+        for (; k < depth; k++) {
+          block[0] = dm0(k);
+          block[1] = dm1(k);
+          block[2] = dm2(k);
+          block[3] = dm3(k);
+          block += 4;
+        }
+      }
+    }
+
+    // copy the remaining columns one at a time (nr==1)
+    for (Index j2 = packet_cols4; j2 < cols; ++j2) {
+      const SubMapper dm0 = rhs.getLinearMapper(0, j2);
+      for (Index k = 0; k < depth; k++) {
+        *block = dm0(k);
+        block += 1;
+      }
+    }
+  }
+};
+
 // Special case for non-vectorized types such as float16.
 template <typename NewDimension, DenseIndex Rows, DenseIndex Cols,
           typename ArgType, typename Device, typename Scalar, typename Index,
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index d6a0f38033..1f4e9753c3 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -266,7 +266,7 @@ REGISTER_OP("Conv2D")
     .Input("input: T")
     .Input("filter: T")
     .Output("output: T")
-    .Attr("T: {half, bfloat16, float}")
+    .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
@@ -279,7 +279,7 @@ REGISTER_OP("Conv2DBackpropInput")
     .Input("filter: T")
     .Input("out_backprop: T")
     .Output("output: T")
-    .Attr("T: {half, bfloat16, float}")
+    .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
@@ -301,7 +301,7 @@ REGISTER_OP("Conv2DBackpropFilter")
     .Input("filter_sizes: int32")
     .Input("out_backprop: T")
     .Output("output: T")
-    .Attr("T: {half, bfloat16, float}")
+    .Attr("T: {half, bfloat16, float, double}")
     .Attr("strides: list(int)")
     .Attr("use_cudnn_on_gpu: bool = true")
     .Attr(GetPaddingAttrString())
diff --git a/tensorflow/python/kernel_tests/conv_ops_test.py b/tensorflow/python/kernel_tests/conv_ops_test.py
index 25525cc128..a291bef0ad 100644
--- a/tensorflow/python/kernel_tests/conv_ops_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_test.py
@@ -159,11 +159,11 @@ class Conv2DTest(test.TestCase):
 
   def _DtypesToTest(self, use_gpu):
     if use_gpu and not test_util.CudaSupportsHalfMatMulAndConv():
-      return [dtypes.float32]
+      return [dtypes.float32, dtypes.float64]
     else:
       # It is important that float32 comes before float16 here,
       # as we will be using its gradients as reference for fp16 gradients.
-      return [dtypes.float32, dtypes.float16]
+      return [dtypes.float32, dtypes.float16, dtypes.float64]
 
   def _SetupValuesForDevice(self, tensor_in_sizes, filter_in_sizes, dilations,
                             strides, padding, data_format, dtype, use_gpu):
-- 
GitLab


From 3a6ca86801093895eb37155b1fe09e43cd4c9f80 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Thu, 22 Mar 2018 14:14:33 -0700
Subject: [PATCH 375/960] Update tf.keras preprocessing to Keras 2.1.5 API

PiperOrigin-RevId: 190123773
---
 .../keras/_impl/keras/preprocessing/image.py  | 211 +++++++++++-----
 .../_impl/keras/preprocessing/image_test.py   | 151 +++++++++++-
 .../_impl/keras/preprocessing/sequence.py     | 232 ++++++++++++++----
 .../keras/preprocessing/sequence_test.py      |  82 ++++++-
 .../keras/_impl/keras/preprocessing/text.py   |  31 ++-
 .../_impl/keras/preprocessing/text_test.py    |  42 +++-
 .../keras/preprocessing/image/__init__.py     |   1 +
 .../keras/preprocessing/sequence/__init__.py  |   1 +
 .../keras/preprocessing/text/__init__.py      |   1 +
 ...processing.image.-directory-iterator.pbtxt |   2 +-
 ...ocessing.image.-image-data-generator.pbtxt |   6 +-
 ...ocessing.image.-numpy-array-iterator.pbtxt |   2 +-
 ...tensorflow.keras.preprocessing.image.pbtxt |   4 +
 ...ssing.sequence.-timeseries-generator.pbtxt |  14 ++
 ...sorflow.keras.preprocessing.sequence.pbtxt |   4 +
 .../tensorflow.keras.preprocessing.text.pbtxt |   4 +
 16 files changed, 665 insertions(+), 123 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.-timeseries-generator.pbtxt

diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/image.py b/tensorflow/python/keras/_impl/keras/preprocessing/image.py
index d12f108639..6299445c34 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/image.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/image.py
@@ -43,6 +43,7 @@ except ImportError:
 
 
 try:
+  from PIL import ImageEnhance
   from PIL import Image as pil_image
 except ImportError:
   pil_image = None
@@ -227,6 +228,32 @@ def random_channel_shift(x, intensity, channel_axis=0):
   return x
 
 
+@tf_export('keras.preprocessing.image.random_brightness')
+def random_brightness(x, brightness_range):
+  """Performs a random adjustment of brightness of a Numpy image tensor.
+
+  Arguments:
+      x: Input tensor. Must be 3D.
+      brightness_range: Tuple of floats; range to pick a brightness value from.
+
+  Returns:
+      Brightness adjusted Numpy image tensor.
+
+  Raises:
+      ValueError: if `brightness_range` isn't a tuple.
+  """
+  if len(brightness_range) != 2:
+    raise ValueError('`brightness_range should be tuple or list of two floats. '
+                     'Received arg: ', brightness_range)
+
+  x = array_to_img(x)
+  x = ImageEnhance.Brightness(x)
+  u = np.random.uniform(brightness_range[0], brightness_range[1])
+  x = x.enhance(u)
+  x = img_to_array(x)
+  return x
+
+
 def transform_matrix_offset_center(matrix, x, y):
   o_x = float(x) / 2 + 0.5
   o_y = float(y) / 2 + 0.5
@@ -265,7 +292,7 @@ def apply_transform(x,
           x_channel,
           final_affine_matrix,
           final_offset,
-          order=0,
+          order=1,
           mode=fill_mode,
           cval=cval) for x_channel in x
   ]
@@ -436,6 +463,7 @@ class ImageDataGenerator(object):
       rotation_range: degrees (0 to 180).
       width_shift_range: fraction of total width, if < 1, or pixels if >= 1.
       height_shift_range: fraction of total height, if < 1, or pixels if >= 1.
+      brightness_range: the range of brightness to apply
       shear_range: shear intensity (shear angle in degrees).
       zoom_range: amount of zoom. if scalar z, zoom will be randomly picked
           in the range [1-z, 1+z]. A sequence of two can be passed instead
@@ -469,6 +497,8 @@ class ImageDataGenerator(object):
           It defaults to the `image_data_format` value found in your
           Keras config file at `~/.keras/keras.json`.
           If you never set it, then it will be "channels_last".
+      validation_split: fraction of images reserved for validation (strictly
+        between 0 and 1).
   """
 
   def __init__(self,
@@ -481,6 +511,7 @@ class ImageDataGenerator(object):
                rotation_range=0.,
                width_shift_range=0.,
                height_shift_range=0.,
+               brightness_range=None,
                shear_range=0.,
                zoom_range=0.,
                channel_shift_range=0.,
@@ -490,7 +521,8 @@ class ImageDataGenerator(object):
                vertical_flip=False,
                rescale=None,
                preprocessing_function=None,
-               data_format=None):
+               data_format=None,
+               validation_split=0.0):
     if data_format is None:
       data_format = K.image_data_format()
     self.featurewise_center = featurewise_center
@@ -502,6 +534,7 @@ class ImageDataGenerator(object):
     self.rotation_range = rotation_range
     self.width_shift_range = width_shift_range
     self.height_shift_range = height_shift_range
+    self.brightness_range = brightness_range
     self.shear_range = shear_range
     self.zoom_range = zoom_range
     self.channel_shift_range = channel_shift_range
@@ -526,6 +559,10 @@ class ImageDataGenerator(object):
       self.channel_axis = 3
       self.row_axis = 1
       self.col_axis = 2
+    if validation_split and not 0 < validation_split < 1:
+      raise ValueError('`validation_split` must be strictly between 0 and 1. '
+                       'Received arg: ', validation_split)
+    self.validation_split = validation_split
 
     self.mean = None
     self.std = None
@@ -574,7 +611,8 @@ class ImageDataGenerator(object):
            seed=None,
            save_to_dir=None,
            save_prefix='',
-           save_format='png'):
+           save_format='png',
+           subset=None):
     return NumpyArrayIterator(
         x,
         y,
@@ -585,7 +623,8 @@ class ImageDataGenerator(object):
         data_format=self.data_format,
         save_to_dir=save_to_dir,
         save_prefix=save_prefix,
-        save_format=save_format)
+        save_format=save_format,
+        subset=subset)
 
   def flow_from_directory(self,
                           directory,
@@ -600,6 +639,7 @@ class ImageDataGenerator(object):
                           save_prefix='',
                           save_format='png',
                           follow_links=False,
+                          subset=None,
                           interpolation='nearest'):
     return DirectoryIterator(
         directory,
@@ -616,6 +656,7 @@ class ImageDataGenerator(object):
         save_prefix=save_prefix,
         save_format=save_format,
         follow_links=follow_links,
+        subset=subset,
         interpolation=interpolation)
 
   def standardize(self, x):
@@ -628,7 +669,7 @@ class ImageDataGenerator(object):
         The inputs, normalized.
     """
     if self.preprocessing_function:
-      x = self.preprocessing_function(x)
+      x = self.image_data_generator.preprocessing_function(x)
     if self.rescale:
       x *= self.rescale
     if self.samplewise_center:
@@ -762,6 +803,9 @@ class ImageDataGenerator(object):
       if np.random.random() < 0.5:
         x = flip_axis(x, img_row_axis)
 
+    if self.brightness_range is not None:
+      x = random_brightness(x, self.brightness_range)
+
     return x
 
   def fit(self, x, augment=False, rounds=1, seed=None):
@@ -828,12 +872,10 @@ class ImageDataGenerator(object):
         raise ImportError('Scipy is required for zca_whitening.')
 
       flat_x = np.reshape(x, (x.shape[0], x.shape[1] * x.shape[2] * x.shape[3]))
-      num_examples = flat_x.shape[0]
-      _, s, vt = linalg.svd(flat_x / np.sqrt(num_examples))
-      s_expand = np.hstack(
-          (s, np.zeros(vt.shape[0] - num_examples, dtype=flat_x.dtype)))
-      self.principal_components = (
-          vt.T / np.sqrt(s_expand**2 + self.zca_epsilon)).dot(vt)
+      sigma = np.dot(flat_x.T, flat_x) / flat_x.shape[0]
+      u, s, _ = linalg.svd(sigma)
+      s_inv = 1. / np.sqrt(s[np.newaxis] + self.zca_epsilon)
+      self.principal_components = (u * s_inv).dot(u.T)
 
 
 @tf_export('keras.preprocessing.image.Iterator')
@@ -947,6 +989,8 @@ class NumpyArrayIterator(Iterator):
           images (if `save_to_dir` is set).
       save_format: Format to use for saving sample images
           (if `save_to_dir` is set).
+      subset: Subset of data (`"training"` or `"validation"`) if
+          validation_split is set in ImageDataGenerator.
   """
 
   def __init__(self,
@@ -959,17 +1003,29 @@ class NumpyArrayIterator(Iterator):
                data_format=None,
                save_to_dir=None,
                save_prefix='',
-               save_format='png'):
+               save_format='png',
+               subset=None):
     if y is not None and len(x) != len(y):
-      raise ValueError('X (images tensor) and y (labels) '
+      raise ValueError('`x` (images tensor) and `y` (labels) '
                        'should have the same length. '
-                       'Found: X.shape = %s, y.shape = %s' %
+                       'Found: x.shape = %s, y.shape = %s' %
                        (np.asarray(x).shape, np.asarray(y).shape))
-
+    if subset is not None:
+      if subset not in {'training', 'validation'}:
+        raise ValueError('Invalid subset name:', subset,
+                         '; expected "training" or "validation".')
+      split_idx = int(len(x) * image_data_generator.validation_split)
+      if subset == 'validation':
+        x = x[:split_idx]
+        if y is not None:
+          y = y[:split_idx]
+      else:
+        x = x[split_idx:]
+        if y is not None:
+          y = y[split_idx:]
     if data_format is None:
       data_format = K.image_data_format()
     self.x = np.asarray(x, dtype=K.floatx())
-
     if self.x.ndim != 4:
       raise ValueError('Input data in `NumpyArrayIterator` '
                        'should have rank 4. You passed an array '
@@ -1032,8 +1088,7 @@ class NumpyArrayIterator(Iterator):
     return self._get_batches_of_transformed_samples(index_array)
 
 
-def _count_valid_files_in_directory(directory, white_list_formats,
-                                    follow_links):
+def _iter_valid_files(directory, white_list_formats, follow_links):
   """Count files with extension in `white_list_formats` contained in directory.
 
   Arguments:
@@ -1043,29 +1098,54 @@ def _count_valid_files_in_directory(directory, white_list_formats,
           the files to be counted.
       follow_links: boolean.
 
-  Returns:
-      the count of files with extension in `white_list_formats` contained in
-      the directory.
+  Yields:
+      tuple of (root, filename) with extension in `white_list_formats`.
   """
 
   def _recursive_list(subpath):
     return sorted(
-        os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])
+        os.walk(subpath, followlinks=follow_links), key=lambda x: x[0])
 
-  samples = 0
-  for _, _, files in _recursive_list(directory):
-    for fname in files:
-      is_valid = False
+  for root, _, files in _recursive_list(directory):
+    for fname in sorted(files):
       for extension in white_list_formats:
+        if fname.lower().endswith('.tiff'):
+          logging.warning(
+              'Using \'.tiff\' files with multiple bands will cause '
+              'distortion. Please verify your output.')
         if fname.lower().endswith('.' + extension):
-          is_valid = True
-          break
-      if is_valid:
-        samples += 1
-  return samples
+          yield root, fname
 
 
-def _list_valid_filenames_in_directory(directory, white_list_formats,
+def _count_valid_files_in_directory(directory, white_list_formats, split,
+                                    follow_links):
+  """Count files with extension in `white_list_formats` contained in directory.
+
+  Arguments:
+      directory: absolute path to the directory
+          containing files to be counted
+      white_list_formats: set of strings containing allowed extensions for
+          the files to be counted.
+      split: tuple of floats (e.g. `(0.2, 0.6)`) to only take into
+          account a certain fraction of files in each directory.
+          E.g.: `segment=(0.6, 1.0)` would only account for last 40 percent
+          of images in each directory.
+      follow_links: boolean.
+
+  Returns:
+      the count of files with extension in `white_list_formats` contained in
+      the directory.
+  """
+  num_files = len(
+      list(_iter_valid_files(directory, white_list_formats, follow_links)))
+  if split:
+    start, stop = int(split[0] * num_files), int(split[1] * num_files)
+  else:
+    start, stop = 0, num_files
+  return stop - start
+
+
+def _list_valid_filenames_in_directory(directory, white_list_formats, split,
                                        class_indices, follow_links):
   """List paths of files in `subdir` with extensions in `white_list_formats`.
 
@@ -1075,6 +1155,10 @@ def _list_valid_filenames_in_directory(directory, white_list_formats,
             `class_indices`.
       white_list_formats: set of strings containing allowed extensions for
           the files to be counted.
+      split: tuple of floats (e.g. `(0.2, 0.6)`) to only take into
+          account a certain fraction of files in each directory.
+          E.g.: `segment=(0.6, 1.0)` would only account for last 40 percent
+          of images in each directory.
       class_indices: dictionary mapping a class name to its index.
       follow_links: boolean.
 
@@ -1084,27 +1168,26 @@ def _list_valid_filenames_in_directory(directory, white_list_formats,
           `directory`'s parent (e.g., if `directory` is "dataset/class1",
           the filenames will be ["class1/file1.jpg", "class1/file2.jpg", ...]).
   """
-
-  def _recursive_list(subpath):
-    return sorted(
-        os.walk(subpath, followlinks=follow_links), key=lambda tpl: tpl[0])
+  dirname = os.path.basename(directory)
+  if split:
+    num_files = len(
+        list(_iter_valid_files(directory, white_list_formats, follow_links)))
+    start, stop = int(split[0] * num_files), int(split[1] * num_files)
+    valid_files = list(
+        _iter_valid_files(directory, white_list_formats,
+                          follow_links))[start:stop]
+  else:
+    valid_files = _iter_valid_files(directory, white_list_formats, follow_links)
 
   classes = []
   filenames = []
-  subdir = os.path.basename(directory)
-  basedir = os.path.dirname(directory)
-  for root, _, files in _recursive_list(directory):
-    for fname in sorted(files):
-      is_valid = False
-      for extension in white_list_formats:
-        if fname.lower().endswith('.' + extension):
-          is_valid = True
-          break
-      if is_valid:
-        classes.append(class_indices[subdir])
-        # add filename relative to directory
-        absolute_path = os.path.join(root, fname)
-        filenames.append(os.path.relpath(absolute_path, basedir))
+  for root, fname in valid_files:
+    classes.append(class_indices[dirname])
+    absolute_path = os.path.join(root, fname)
+    relative_path = os.path.join(dirname,
+                                 os.path.relpath(absolute_path, directory))
+    filenames.append(relative_path)
+
   return classes, filenames
 
 
@@ -1144,6 +1227,8 @@ class DirectoryIterator(Iterator):
           images (if `save_to_dir` is set).
       save_format: Format to use for saving sample images
           (if `save_to_dir` is set).
+      subset: Subset of data (`"training"` or `"validation"`) if
+          validation_split is set in ImageDataGenerator.
       interpolation: Interpolation method used to resample the image if the
           target size is different from that of the loaded image.
           Supported methods are "nearest", "bilinear", and "bicubic".
@@ -1167,6 +1252,7 @@ class DirectoryIterator(Iterator):
                save_prefix='',
                save_format='png',
                follow_links=False,
+               subset=None,
                interpolation='nearest'):
     if data_format is None:
       data_format = K.image_data_format()
@@ -1200,7 +1286,20 @@ class DirectoryIterator(Iterator):
     self.save_format = save_format
     self.interpolation = interpolation
 
-    white_list_formats = {'png', 'jpg', 'jpeg', 'bmp', 'ppm'}
+    if subset is not None:
+      validation_split = self.image_data_generator.validation_split
+      if subset == 'validation':
+        split = (0, validation_split)
+      elif subset == 'training':
+        split = (validation_split, 1)
+      else:
+        raise ValueError('Invalid subset name: ', subset,
+                         '; expected "training" or "validation"')
+    else:
+      split = None
+    self.subset = subset
+
+    white_list_formats = {'png', 'jpg', 'jpeg', 'bmp', 'ppm', 'tif', 'tiff'}
 
     # first, count the number of samples and classes
     self.samples = 0
@@ -1217,7 +1316,8 @@ class DirectoryIterator(Iterator):
     function_partial = partial(
         _count_valid_files_in_directory,
         white_list_formats=white_list_formats,
-        follow_links=follow_links)
+        follow_links=follow_links,
+        split=split)
     self.samples = sum(
         pool.map(function_partial,
                  (os.path.join(directory, subdir) for subdir in classes)))
@@ -1233,14 +1333,15 @@ class DirectoryIterator(Iterator):
     i = 0
     for dirpath in (os.path.join(directory, subdir) for subdir in classes):
       results.append(
-          pool.apply_async(
-              _list_valid_filenames_in_directory,
-              (dirpath, white_list_formats, self.class_indices, follow_links)))
+          pool.apply_async(_list_valid_filenames_in_directory,
+                           (dirpath, white_list_formats, split,
+                            self.class_indices, follow_links)))
     for res in results:
       classes, filenames = res.get()
       self.classes[i:i + len(classes)] = classes
       self.filenames += filenames
       i += len(classes)
+
     pool.close()
     pool.join()
     super(DirectoryIterator, self).__init__(self.samples, batch_size, shuffle,
diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py b/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py
index c0790b5a51..001fee91f9 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/image_test.py
@@ -20,6 +20,7 @@ from __future__ import print_function
 
 import os
 import shutil
+import tempfile
 
 import numpy as np
 
@@ -74,6 +75,7 @@ class TestImage(test.TestCase):
           shear_range=0.5,
           zoom_range=0.2,
           channel_shift_range=0.,
+          brightness_range=(1, 5),
           fill_mode='nearest',
           cval=0.5,
           horizontal_flip=True,
@@ -92,6 +94,47 @@ class TestImage(test.TestCase):
         self.assertEqual(x.shape[1:], images.shape[1:])
         break
 
+  def test_image_data_generator_with_validation_split(self):
+    if PIL is None:
+      return  # Skip test if PIL is not available.
+
+    for test_images in _generate_test_images():
+      img_list = []
+      for im in test_images:
+        img_list.append(keras.preprocessing.image.img_to_array(im)[None, ...])
+
+      images = np.vstack(img_list)
+      generator = keras.preprocessing.image.ImageDataGenerator(
+          validation_split=0.5)
+      seq = generator.flow(
+          images,
+          np.arange(images.shape[0]),
+          shuffle=False,
+          batch_size=3,
+          subset='validation')
+      _, y = seq[0]
+      self.assertEqual(list(y), [0, 1, 2])
+      seq = generator.flow(
+          images,
+          np.arange(images.shape[0]),
+          shuffle=False,
+          batch_size=3,
+          subset='training')
+      _, y2 = seq[0]
+      self.assertEqual(list(y2), [4, 5, 6])
+
+      with self.assertRaises(ValueError):
+        generator.flow(
+            images,
+            np.arange(images.shape[0]),
+            shuffle=False,
+            batch_size=3,
+            subset='foo')
+
+  def test_image_data_generator_with_split_value_error(self):
+    with self.assertRaises(ValueError):
+      keras.preprocessing.image.ImageDataGenerator(validation_split=5)
+
   def test_image_data_generator_invalid_data(self):
     generator = keras.preprocessing.image.ImageDataGenerator(
         featurewise_center=True,
@@ -202,9 +245,80 @@ class TestImage(test.TestCase):
     # check number of classes and images
     self.assertEqual(len(dir_iterator.class_indices), num_classes)
     self.assertEqual(len(dir_iterator.classes), count)
-    self.assertEqual(sorted(dir_iterator.filenames), sorted(filenames))
+    self.assertEqual(set(dir_iterator.filenames), set(filenames))
     _ = dir_iterator.next()
 
+  def directory_iterator_with_validation_split_test_helper(
+      self, validation_split):
+    if PIL is None:
+      return  # Skip test if PIL is not available.
+
+    num_classes = 2
+    tmp_folder = tempfile.mkdtemp(prefix='test_images')
+
+    # create folders and subfolders
+    paths = []
+    for cl in range(num_classes):
+      class_directory = 'class-{}'.format(cl)
+      classpaths = [
+          class_directory,
+          os.path.join(class_directory, 'subfolder-1'),
+          os.path.join(class_directory, 'subfolder-2'),
+          os.path.join(class_directory, 'subfolder-1', 'sub-subfolder')
+      ]
+      for path in classpaths:
+        os.mkdir(os.path.join(tmp_folder, path))
+      paths.append(classpaths)
+
+    # save the images in the paths
+    count = 0
+    filenames = []
+    for test_images in _generate_test_images():
+      for im in test_images:
+        # rotate image class
+        im_class = count % num_classes
+        # rotate subfolders
+        classpaths = paths[im_class]
+        filename = os.path.join(classpaths[count % len(classpaths)],
+                                'image-{}.jpg'.format(count))
+        filenames.append(filename)
+        im.save(os.path.join(tmp_folder, filename))
+        count += 1
+
+    # create iterator
+    generator = keras.preprocessing.image.ImageDataGenerator(
+        validation_split=validation_split)
+
+    with self.assertRaises(ValueError):
+      generator.flow_from_directory(tmp_folder, subset='foo')
+
+    num_validation = int(count * validation_split)
+    num_training = count - num_validation
+    train_iterator = generator.flow_from_directory(
+        tmp_folder, subset='training')
+    self.assertEqual(train_iterator.samples, num_training)
+
+    valid_iterator = generator.flow_from_directory(
+        tmp_folder, subset='validation')
+    self.assertEqual(valid_iterator.samples, num_validation)
+
+    # check number of classes and images
+    self.assertEqual(len(train_iterator.class_indices), num_classes)
+    self.assertEqual(len(train_iterator.classes), num_training)
+    self.assertEqual(
+        len(set(train_iterator.filenames) & set(filenames)), num_training)
+
+    shutil.rmtree(tmp_folder)
+
+  def test_directory_iterator_with_validation_split_25_percent(self):
+    self.directory_iterator_with_validation_split_test_helper(0.25)
+
+  def test_directory_iterator_with_validation_split_40_percent(self):
+    self.directory_iterator_with_validation_split_test_helper(0.40)
+
+  def test_directory_iterator_with_validation_split_50_percent(self):
+    self.directory_iterator_with_validation_split_test_helper(0.50)
+
   def test_img_utils(self):
     if PIL is None:
       return  # Skip test if PIL is not available.
@@ -241,6 +355,41 @@ class TestImage(test.TestCase):
     x = keras.preprocessing.image.img_to_array(img, data_format='channels_last')
     self.assertEqual(x.shape, (height, width, 1))
 
+  def test_batch_standardize(self):
+    if PIL is None:
+      return  # Skip test if PIL is not available.
+
+    # ImageDataGenerator.standardize should work on batches
+    for test_images in _generate_test_images():
+      img_list = []
+      for im in test_images:
+        img_list.append(keras.preprocessing.image.img_to_array(im)[None, ...])
+
+      images = np.vstack(img_list)
+      generator = keras.preprocessing.image.ImageDataGenerator(
+          featurewise_center=True,
+          samplewise_center=True,
+          featurewise_std_normalization=True,
+          samplewise_std_normalization=True,
+          zca_whitening=True,
+          rotation_range=90.,
+          width_shift_range=0.1,
+          height_shift_range=0.1,
+          shear_range=0.5,
+          zoom_range=0.2,
+          channel_shift_range=0.,
+          brightness_range=(1, 5),
+          fill_mode='nearest',
+          cval=0.5,
+          horizontal_flip=True,
+          vertical_flip=True)
+      generator.fit(images, augment=True)
+
+      transformed = np.copy(images)
+      for i, im in enumerate(transformed):
+        transformed[i] = generator.random_transform(im)
+      transformed = generator.standardize(transformed)
+
   def test_img_transforms(self):
     x = np.random.random((3, 200, 200))
     _ = keras.preprocessing.image.random_rotation(x, 20)
diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/sequence.py b/tensorflow/python/keras/_impl/keras/preprocessing/sequence.py
index a423d96d3d..e68c171d9c 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/sequence.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/sequence.py
@@ -22,6 +22,8 @@ import random
 
 import numpy as np
 from six.moves import range  # pylint: disable=redefined-builtin
+
+from tensorflow.python.keras._impl.keras.utils.data_utils import Sequence
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -32,29 +34,40 @@ def pad_sequences(sequences,
                   padding='pre',
                   truncating='pre',
                   value=0.):
-  """Pads each sequence to the same length (length of the longest sequence).
+  """Pads sequences to the same length.
+
+  This function transforms a list of
+  `num_samples` sequences (lists of integers)
+  into a 2D Numpy array of shape `(num_samples, num_timesteps)`.
+  `num_timesteps` is either the `maxlen` argument if provided,
+  or the length of the longest sequence otherwise.
+
+  Sequences that are shorter than `num_timesteps`
+  are padded with `value` at the end.
 
-  If maxlen is provided, any sequence longer
-  than maxlen is truncated to maxlen.
-  Truncation happens off either the beginning (default) or
-  the end of the sequence.
+  Sequences longer than `num_timesteps` are truncated
+  so that they fit the desired length.
+  The position where padding or truncation happens is determined by
+  the arguments `padding` and `truncating`, respectively.
 
-  Supports post-padding and pre-padding (default).
+  Pre-padding is the default.
 
   Arguments:
-      sequences: list of lists where each element is a sequence
-      maxlen: int, maximum length
-      dtype: type to cast the resulting sequence.
-      padding: 'pre' or 'post', pad either before or after each sequence.
-      truncating: 'pre' or 'post', remove values from sequences larger than
-          maxlen either in the beginning or in the end of the sequence
-      value: float, value to pad the sequences to the desired value.
+      sequences: List of lists, where each element is a sequence.
+      maxlen: Int, maximum length of all sequences.
+      dtype: Type of the output sequences.
+      padding: String, 'pre' or 'post':
+          pad either before or after each sequence.
+      truncating: String, 'pre' or 'post':
+          remove values from sequences larger than
+          `maxlen`, either at the beginning or at the end of the sequences.
+      value: Float, padding value.
 
   Returns:
-      x: numpy array with dimensions (number_of_sequences, maxlen)
+      x: Numpy array with shape `(len(sequences), maxlen)`
 
   Raises:
-      ValueError: in case of invalid values for `truncating` or `padding`,
+      ValueError: In case of invalid values for `truncating` or `padding`,
           or in case of invalid shape for a `sequences` entry.
   """
   if not hasattr(sequences, '__len__'):
@@ -92,10 +105,9 @@ def pad_sequences(sequences,
     # check `trunc` has expected shape
     trunc = np.asarray(trunc, dtype=dtype)
     if trunc.shape[1:] != sample_shape:
-      raise ValueError(
-          'Shape of sample %s of sequence at position %s is different from '
-          'expected shape %s'
-          % (trunc.shape[1:], idx, sample_shape))
+      raise ValueError('Shape of sample %s of sequence at position %s '
+                       'is different from expected shape %s' %
+                       (trunc.shape[1:], idx, sample_shape))
 
     if padding == 'post':
       x[idx, :len(trunc)] = trunc
@@ -110,22 +122,26 @@ def pad_sequences(sequences,
 def make_sampling_table(size, sampling_factor=1e-5):
   """Generates a word rank-based probabilistic sampling table.
 
-  This generates an array where the ith element
-  is the probability that a word of rank i would be sampled,
-  according to the sampling distribution used in word2vec.
+  Used for generating the `sampling_table` argument for `skipgrams`.
+  `sampling_table[i]` is the probability of sampling
+  the word i-th most common word in a dataset
+  (more common words should be sampled less frequently, for balance).
 
-  The word2vec formula is:
-      p(word) = min(1, sqrt(word.frequency/sampling_factor) /
-      (word.frequency/sampling_factor))
+  The sampling probabilities are generated according
+  to the sampling distribution used in word2vec:
+
+  `p(word) = min(1, sqrt(word_frequency / sampling_factor) / (word_frequency /
+  sampling_factor))`
 
   We assume that the word frequencies follow Zipf's law (s=1) to derive
   a numerical approximation of frequency(rank):
-     frequency(rank) ~ 1/(rank * (log(rank) + gamma) + 1/2 - 1/(12*rank))
-      where gamma is the Euler-Mascheroni constant.
+
+  `frequency(rank) ~ 1/(rank * (log(rank) + gamma) + 1/2 - 1/(12*rank))`
+  where `gamma` is the Euler-Mascheroni constant.
 
   Arguments:
-      size: int, number of possible words to sample.
-      sampling_factor: the sampling factor in the word2vec formula.
+      size: Int, number of possible words to sample.
+      sampling_factor: The sampling factor in the word2vec formula.
 
   Returns:
       A 1D Numpy array of length `size` where the ith entry
@@ -151,30 +167,37 @@ def skipgrams(sequence,
               seed=None):
   """Generates skipgram word pairs.
 
-  Takes a sequence (list of indexes of words),
-  returns couples of [word_index, other_word index] and labels (1s or 0s),
-  where label = 1 if 'other_word' belongs to the context of 'word',
-  and label=0 if 'other_word' is randomly sampled
+  This function transforms a sequence of word indexes (list of integers)
+  into tuples of words of the form:
+
+  - (word, word in the same window), with label 1 (positive samples).
+  - (word, random word from the vocabulary), with label 0 (negative samples).
+
+  Read more about Skipgram in this gnomic paper by Mikolov et al.:
+  [Efficient Estimation of Word Representations in
+  Vector Space](http://arxiv.org/pdf/1301.3781v3.pdf)
 
   Arguments:
-      sequence: a word sequence (sentence), encoded as a list
+      sequence: A word sequence (sentence), encoded as a list
           of word indices (integers). If using a `sampling_table`,
           word indices are expected to match the rank
           of the words in a reference dataset (e.g. 10 would encode
           the 10-th most frequently occurring token).
           Note that index 0 is expected to be a non-word and will be skipped.
-      vocabulary_size: int. maximum possible word index + 1
-      window_size: int. actually half-window.
-          The window of a word wi will be [i-window_size, i+window_size+1]
-      negative_samples: float >= 0. 0 for no negative (=random) samples.
-          1 for same number as positive samples. etc.
-      shuffle: whether to shuffle the word couples before returning them.
+      vocabulary_size: Int, maximum possible word index + 1
+      window_size: Int, size of sampling windows (technically half-window).
+          The window of a word `w_i` will be
+          `[i - window_size, i + window_size+1]`.
+      negative_samples: Float >= 0. 0 for no negative (i.e. random) samples.
+          1 for same number as positive samples.
+      shuffle: Whether to shuffle the word couples before returning them.
       categorical: bool. if False, labels will be
-          integers (eg. [0, 1, 1 .. ]),
-          if True labels will be categorical eg. [[1,0],[0,1],[0,1] .. ]
+          integers (eg. `[0, 1, 1 .. ]`),
+          if `True`, labels will be categorical, e.g.
+          `[[1,0],[0,1],[0,1] .. ]`.
       sampling_table: 1D array of size `vocabulary_size` where the entry i
           encodes the probability to sample a word of rank i.
-      seed: random seed.
+      seed: Random seed.
 
   Returns:
       couples, labels: where `couples` are int pairs and
@@ -234,9 +257,9 @@ def _remove_long_seq(maxlen, seq, label):
   """Removes sequences that exceed the maximum length.
 
   Arguments:
-      maxlen: int, maximum length
-      seq: list of lists where each sublist is a sequence
-      label: list where each element is an integer
+      maxlen: Int, maximum length of the output sequences.
+      seq: List of lists, where each sublist is a sequence.
+      label: List where each element is an integer.
 
   Returns:
       new_seq, new_label: shortened lists for `seq` and `label`.
@@ -247,3 +270,120 @@ def _remove_long_seq(maxlen, seq, label):
       new_seq.append(x)
       new_label.append(y)
   return new_seq, new_label
+
+
+@tf_export('keras.preprocessing.sequence.TimeseriesGenerator')
+class TimeseriesGenerator(Sequence):
+  """Utility class for generating batches of temporal data.
+
+  This class takes in a sequence of data-points gathered at
+  equal intervals, along with time series parameters such as
+  stride, length of history, etc., to produce batches for
+  training/validation.
+
+  Arguments:
+      data: Indexable generator (such as list or Numpy array)
+          containing consecutive data points (timesteps).
+          The data should be at 2D, and axis 0 is expected
+          to be the time dimension.
+      targets: Targets corresponding to timesteps in `data`.
+          It should have same length as `data`.
+      length: Length of the output sequences (in number of timesteps).
+      sampling_rate: Period between successive individual timesteps
+          within sequences. For rate `r`, timesteps
+          `data[i]`, `data[i-r]`, ... `data[i - length]`
+          are used for create a sample sequence.
+      stride: Period between successive output sequences.
+          For stride `s`, consecutive output samples would
+          be centered around `data[i]`, `data[i+s]`, `data[i+2*s]`, etc.
+      start_index, end_index: Data points earlier than `start_index`
+          or later than `end_index` will not be used in the output sequences.
+          This is useful to reserve part of the data for test or validation.
+      shuffle: Whether to shuffle output samples,
+          or instead draw them in chronological order.
+      reverse: Boolean: if `true`, timesteps in each output sample will be
+          in reverse chronological order.
+      batch_size: Number of timeseries samples in each batch
+          (except maybe the last one).
+
+  Returns:
+      A [Sequence](/utils/#sequence) instance.
+
+  Examples:
+
+  ```python
+  from keras.preprocessing.sequence import TimeseriesGenerator
+  import numpy as np
+
+  data = np.array([[i] for i in range(50)])
+  targets = np.array([[i] for i in range(50)])
+
+  data_gen = TimeseriesGenerator(data, targets,
+                                 length=10, sampling_rate=2,
+                                 batch_size=2)
+  assert len(data_gen) == 20
+
+  batch_0 = data_gen[0]
+  x, y = batch_0
+  assert np.array_equal(x,
+                        np.array([[[0], [2], [4], [6], [8]],
+                                  [[1], [3], [5], [7], [9]]]))
+  assert np.array_equal(y,
+                        np.array([[10], [11]]))
+  ```
+  """
+
+  def __init__(self,
+               data,
+               targets,
+               length,
+               sampling_rate=1,
+               stride=1,
+               start_index=0,
+               end_index=None,
+               shuffle=False,
+               reverse=False,
+               batch_size=128):
+    self.data = data
+    self.targets = targets
+    self.length = length
+    self.sampling_rate = sampling_rate
+    self.stride = stride
+    self.start_index = start_index + length
+    if end_index is None:
+      end_index = len(data) - 1
+    self.end_index = end_index
+    self.shuffle = shuffle
+    self.reverse = reverse
+    self.batch_size = batch_size
+
+  def __len__(self):
+    length = int(
+        np.ceil((self.end_index - self.start_index) /
+                (self.batch_size * self.stride)))
+    return length if length >= 0 else 0
+
+  def _empty_batch(self, num_rows):
+    samples_shape = [num_rows, self.length // self.sampling_rate]
+    samples_shape.extend(self.data.shape[1:])
+    targets_shape = [num_rows]
+    targets_shape.extend(self.targets.shape[1:])
+    return np.empty(samples_shape), np.empty(targets_shape)
+
+  def __getitem__(self, index):
+    if self.shuffle:
+      rows = np.random.randint(
+          self.start_index, self.end_index, size=self.batch_size)
+    else:
+      i = self.start_index + self.batch_size * self.stride * index
+      rows = np.arange(i, min(i + self.batch_size * self.stride,
+                              self.end_index), self.stride)
+
+    samples, targets = self._empty_batch(len(rows))
+    for j in range(len(rows)):
+      indices = range(rows[j] - self.length, rows[j], self.sampling_rate)
+      samples[j] = self.data[indices]
+      targets[j] = self.targets[rows[j]]
+    if self.reverse:
+      return samples[:, ::-1, ...], targets
+    return samples, targets
diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/sequence_test.py b/tensorflow/python/keras/_impl/keras/preprocessing/sequence_test.py
index 4529e6e94f..b9bfdd0004 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/sequence_test.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/sequence_test.py
@@ -84,15 +84,91 @@ class TestSequence(test.TestCase):
     couples, labels = keras.preprocessing.sequence.skipgrams(
         np.arange(3), vocabulary_size=3)
     for couple in couples:
-      assert couple[0] in [0, 1, 2] and couple[1] in [0, 1, 2]
+      self.assertIn(couple[0], [0, 1, 2])
+      self.assertIn(couple[1], [0, 1, 2])
 
     # test window size and categorical labels
     couples, labels = keras.preprocessing.sequence.skipgrams(
         np.arange(5), vocabulary_size=5, window_size=1, categorical=True)
     for couple in couples:
-      assert couple[0] - couple[1] <= 3
+      self.assertLessEqual(couple[0] - couple[1], 3)
     for l in labels:
-      assert len(l) == 2
+      self.assertEqual(len(l), 2)
+
+  def test_TimeseriesGenerator(self):
+    data = np.array([[i] for i in range(50)])
+    targets = np.array([[i] for i in range(50)])
+
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data, targets, length=10, sampling_rate=2, batch_size=2)
+    self.assertEqual(len(data_gen), 20)
+    self.assertAllClose(data_gen[0][0],
+                        np.array([[[0], [2], [4], [6], [8]], [[1], [3], [5],
+                                                              [7], [9]]]))
+    self.assertAllClose(data_gen[0][1], np.array([[10], [11]]))
+    self.assertAllClose(data_gen[1][0],
+                        np.array([[[2], [4], [6], [8], [10]], [[3], [5], [7],
+                                                               [9], [11]]]))
+    self.assertAllClose(data_gen[1][1], np.array([[12], [13]]))
+
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data, targets, length=10, sampling_rate=2, reverse=True, batch_size=2)
+    self.assertEqual(len(data_gen), 20)
+    self.assertAllClose(data_gen[0][0],
+                        np.array([[[8], [6], [4], [2], [0]], [[9], [7], [5],
+                                                              [3], [1]]]))
+    self.assertAllClose(data_gen[0][1], np.array([[10], [11]]))
+
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data, targets, length=10, sampling_rate=2, shuffle=True, batch_size=1)
+    batch = data_gen[0]
+    r = batch[1][0][0]
+    self.assertAllClose(batch[0],
+                        np.array([[[r - 10], [r - 8], [r - 6], [r - 4],
+                                   [r - 2]]]))
+    self.assertAllClose(batch[1], np.array([
+        [r],
+    ]))
+
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data, targets, length=10, sampling_rate=2, stride=2, batch_size=2)
+    self.assertEqual(len(data_gen), 10)
+    self.assertAllClose(data_gen[1][0],
+                        np.array([[[4], [6], [8], [10], [12]], [[6], [8], [10],
+                                                                [12], [14]]]))
+    self.assertAllClose(data_gen[1][1], np.array([[14], [16]]))
+
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data,
+        targets,
+        length=10,
+        sampling_rate=2,
+        start_index=10,
+        end_index=30,
+        batch_size=2)
+    self.assertEqual(len(data_gen), 5)
+    self.assertAllClose(data_gen[0][0],
+                        np.array([[[10], [12], [14], [16], [18]],
+                                  [[11], [13], [15], [17], [19]]]))
+    self.assertAllClose(data_gen[0][1], np.array([[20], [21]]))
+
+    data = np.array([np.random.random_sample((1, 2, 3, 4)) for i in range(50)])
+    targets = np.array([np.random.random_sample((3, 2, 1)) for i in range(50)])
+    data_gen = keras.preprocessing.sequence.TimeseriesGenerator(
+        data,
+        targets,
+        length=10,
+        sampling_rate=2,
+        start_index=10,
+        end_index=30,
+        batch_size=2)
+
+    self.assertEqual(len(data_gen), 5)
+    self.assertAllClose(data_gen[0][0],
+                        np.array(
+                            [np.array(data[10:19:2]),
+                             np.array(data[11:20:2])]))
+    self.assertAllClose(data_gen[0][1], np.array([targets[20], targets[21]]))
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/text.py b/tensorflow/python/keras/_impl/keras/preprocessing/text.py
index 1e3828ccf1..f652f318f3 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/text.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/text.py
@@ -91,6 +91,7 @@ def one_hot(text,
       text, n, hash_function=hash, filters=filters, lower=lower, split=split)
 
 
+@tf_export('keras.preprocessing.text.hashing_trick')
 def hashing_trick(text,
                   n,
                   hash_function=None,
@@ -187,21 +188,27 @@ class Tokenizer(object):
     self.document_count = 0
     self.char_level = char_level
     self.oov_token = oov_token
+    self.index_docs = {}
 
   def fit_on_texts(self, texts):
     """Updates internal vocabulary based on a list of texts.
 
+    In the case where texts contains lists, we assume each entry of the lists
+    to be a token.
+
     Required before using `texts_to_sequences` or `texts_to_matrix`.
 
     Arguments:
         texts: can be a list of strings,
-            or a generator of strings (for memory-efficiency)
+            a generator of strings (for memory-efficiency),
+            or a list of list of strings.
     """
-    self.document_count = 0
     for text in texts:
       self.document_count += 1
-      seq = text if self.char_level else text_to_word_sequence(
-          text, self.filters, self.lower, self.split)
+      if self.char_level or isinstance(text, list):
+        seq = text
+      else:
+        seq = text_to_word_sequence(text, self.filters, self.lower, self.split)
       for w in seq:
         if w in self.word_counts:
           self.word_counts[w] += 1
@@ -226,7 +233,6 @@ class Tokenizer(object):
       if i is None:
         self.word_index[self.oov_token] = len(self.word_index) + 1
 
-    self.index_docs = {}
     for w, c in list(self.word_docs.items()):
       self.index_docs[self.word_index[w]] = c
 
@@ -240,8 +246,7 @@ class Tokenizer(object):
         sequences: A list of sequence.
             A "sequence" is a list of integer word indices.
     """
-    self.document_count = len(sequences)
-    self.index_docs = {}
+    self.document_count += len(sequences)
     for seq in sequences:
       seq = set(seq)
       for i in seq:
@@ -268,7 +273,11 @@ class Tokenizer(object):
     return res
 
   def texts_to_sequences_generator(self, texts):
-    """Transforms each text in texts in a sequence of integers.
+    """Transforms each text in `texts` in a sequence of integers.
+
+    Each item in texts can also be a list, in which case we assume each item of
+    that list
+    to be a token.
 
     Only top "num_words" most frequent words will be taken into account.
     Only words known by the tokenizer will be taken into account.
@@ -281,8 +290,10 @@ class Tokenizer(object):
     """
     num_words = self.num_words
     for text in texts:
-      seq = text if self.char_level else text_to_word_sequence(
-          text, self.filters, self.lower, self.split)
+      if self.char_level or isinstance(text, list):
+        seq = text
+      else:
+        seq = text_to_word_sequence(text, self.filters, self.lower, self.split)
       vect = []
       for w in seq:
         i = self.word_index.get(w)
diff --git a/tensorflow/python/keras/_impl/keras/preprocessing/text_test.py b/tensorflow/python/keras/_impl/keras/preprocessing/text_test.py
index a934e331c4..c6a267e57e 100644
--- a/tensorflow/python/keras/_impl/keras/preprocessing/text_test.py
+++ b/tensorflow/python/keras/_impl/keras/preprocessing/text_test.py
@@ -1,3 +1,4 @@
+# -*- coding: utf-8 -*-
 # Copyright 2016 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -80,17 +81,52 @@ class TestText(test.TestCase):
     x_train = ['This text has only known words']
     x_test = ['This text has some unknown words']  # 2 OOVs: some, unknown
 
-    # Defalut, without OOV flag
+    # Default, without OOV flag
     tokenizer = keras.preprocessing.text.Tokenizer()
     tokenizer.fit_on_texts(x_train)
     x_test_seq = tokenizer.texts_to_sequences(x_test)
-    assert len(x_test_seq[0]) == 4  # discards 2 OOVs
+    self.assertEqual(len(x_test_seq[0]), 4)  # discards 2 OOVs
 
     # With OOV feature
     tokenizer = keras.preprocessing.text.Tokenizer(oov_token='<unk>')
     tokenizer.fit_on_texts(x_train)
     x_test_seq = tokenizer.texts_to_sequences(x_test)
-    assert len(x_test_seq[0]) == 6  # OOVs marked in place
+    self.assertEqual(len(x_test_seq[0]), 6)  # OOVs marked in place
+
+  def test_sequential_fit(self):
+    texts = [
+        'The cat sat on the mat.', 'The dog sat on the log.',
+        'Dogs and cats living together.'
+    ]
+    word_sequences = [['The', 'cat', 'is', 'sitting'],
+                      ['The', 'dog', 'is', 'standing']]
+    tokenizer = keras.preprocessing.text.Tokenizer()
+    tokenizer.fit_on_texts(texts)
+    tokenizer.fit_on_texts(word_sequences)
+
+    self.assertEqual(tokenizer.document_count, 5)
+
+    tokenizer.texts_to_matrix(texts)
+    tokenizer.texts_to_matrix(word_sequences)
+
+  def test_text_to_word_sequence(self):
+    text = 'hello! ? world!'
+    seq = keras.preprocessing.text.text_to_word_sequence(text)
+    self.assertEqual(seq, ['hello', 'world'])
+
+  def test_text_to_word_sequence_unicode(self):
+    text = u'ali! veli? kırk dokuz elli'
+    seq = keras.preprocessing.text.text_to_word_sequence(text)
+    self.assertEqual(seq, [u'ali', u'veli', u'kırk', u'dokuz', u'elli'])
+
+  def test_tokenizer_unicode(self):
+    texts = [
+        u'ali veli kırk dokuz elli', u'ali veli kırk dokuz elli veli kırk dokuz'
+    ]
+    tokenizer = keras.preprocessing.text.Tokenizer(num_words=5)
+    tokenizer.fit_on_texts(texts)
+
+    self.assertEqual(len(tokenizer.word_counts), 5)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/python/keras/preprocessing/image/__init__.py b/tensorflow/python/keras/preprocessing/image/__init__.py
index b96e767552..6aba5fc825 100644
--- a/tensorflow/python/keras/preprocessing/image/__init__.py
+++ b/tensorflow/python/keras/preprocessing/image/__init__.py
@@ -27,6 +27,7 @@ from tensorflow.python.keras._impl.keras.preprocessing.image import img_to_array
 from tensorflow.python.keras._impl.keras.preprocessing.image import Iterator
 from tensorflow.python.keras._impl.keras.preprocessing.image import load_img
 from tensorflow.python.keras._impl.keras.preprocessing.image import NumpyArrayIterator
+from tensorflow.python.keras._impl.keras.preprocessing.image import random_brightness
 from tensorflow.python.keras._impl.keras.preprocessing.image import random_channel_shift
 from tensorflow.python.keras._impl.keras.preprocessing.image import random_rotation
 from tensorflow.python.keras._impl.keras.preprocessing.image import random_shear
diff --git a/tensorflow/python/keras/preprocessing/sequence/__init__.py b/tensorflow/python/keras/preprocessing/sequence/__init__.py
index 112f6af5e5..b7a7149cc4 100644
--- a/tensorflow/python/keras/preprocessing/sequence/__init__.py
+++ b/tensorflow/python/keras/preprocessing/sequence/__init__.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 from tensorflow.python.keras._impl.keras.preprocessing.sequence import make_sampling_table
 from tensorflow.python.keras._impl.keras.preprocessing.sequence import pad_sequences
 from tensorflow.python.keras._impl.keras.preprocessing.sequence import skipgrams
+from tensorflow.python.keras._impl.keras.preprocessing.sequence import TimeseriesGenerator
 
 del absolute_import
 del division
diff --git a/tensorflow/python/keras/preprocessing/text/__init__.py b/tensorflow/python/keras/preprocessing/text/__init__.py
index 5bf1a2fb21..000ad68a0c 100644
--- a/tensorflow/python/keras/preprocessing/text/__init__.py
+++ b/tensorflow/python/keras/preprocessing/text/__init__.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.python.keras._impl.keras.preprocessing.text import hashing_trick
 from tensorflow.python.keras._impl.keras.preprocessing.text import one_hot
 from tensorflow.python.keras._impl.keras.preprocessing.text import text_to_word_sequence
 from tensorflow.python.keras._impl.keras.preprocessing.text import Tokenizer
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-directory-iterator.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-directory-iterator.pbtxt
index 04174bff5f..ec0f3d892d 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-directory-iterator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-directory-iterator.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'directory\', \'image_data_generator\', \'target_size\', \'color_mode\', \'classes\', \'class_mode\', \'batch_size\', \'shuffle\', \'seed\', \'data_format\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'follow_links\', \'interpolation\'], varargs=None, keywords=None, defaults=[\'(256, 256)\', \'rgb\', \'None\', \'categorical\', \'32\', \'True\', \'None\', \'None\', \'None\', \'\', \'png\', \'False\', \'nearest\'], "
+    argspec: "args=[\'self\', \'directory\', \'image_data_generator\', \'target_size\', \'color_mode\', \'classes\', \'class_mode\', \'batch_size\', \'shuffle\', \'seed\', \'data_format\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'follow_links\', \'subset\', \'interpolation\'], varargs=None, keywords=None, defaults=[\'(256, 256)\', \'rgb\', \'None\', \'categorical\', \'32\', \'True\', \'None\', \'None\', \'None\', \'\', \'png\', \'False\', \'None\', \'nearest\'], "
   }
   member_method {
     name: "next"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-image-data-generator.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-image-data-generator.pbtxt
index 41f27d1f74..f5bc04e44c 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-image-data-generator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-image-data-generator.pbtxt
@@ -4,7 +4,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'featurewise_center\', \'samplewise_center\', \'featurewise_std_normalization\', \'samplewise_std_normalization\', \'zca_whitening\', \'zca_epsilon\', \'rotation_range\', \'width_shift_range\', \'height_shift_range\', \'shear_range\', \'zoom_range\', \'channel_shift_range\', \'fill_mode\', \'cval\', \'horizontal_flip\', \'vertical_flip\', \'rescale\', \'preprocessing_function\', \'data_format\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'1e-06\', \'0.0\', \'0.0\', \'0.0\', \'0.0\', \'0.0\', \'0.0\', \'nearest\', \'0.0\', \'False\', \'False\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'featurewise_center\', \'samplewise_center\', \'featurewise_std_normalization\', \'samplewise_std_normalization\', \'zca_whitening\', \'zca_epsilon\', \'rotation_range\', \'width_shift_range\', \'height_shift_range\', \'brightness_range\', \'shear_range\', \'zoom_range\', \'channel_shift_range\', \'fill_mode\', \'cval\', \'horizontal_flip\', \'vertical_flip\', \'rescale\', \'preprocessing_function\', \'data_format\', \'validation_split\'], varargs=None, keywords=None, defaults=[\'False\', \'False\', \'False\', \'False\', \'False\', \'1e-06\', \'0.0\', \'0.0\', \'0.0\', \'None\', \'0.0\', \'0.0\', \'0.0\', \'nearest\', \'0.0\', \'False\', \'False\', \'None\', \'None\', \'None\', \'0.0\'], "
   }
   member_method {
     name: "fit"
@@ -12,11 +12,11 @@ tf_class {
   }
   member_method {
     name: "flow"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'shuffle\', \'seed\', \'save_to_dir\', \'save_prefix\', \'save_format\'], varargs=None, keywords=None, defaults=[\'None\', \'32\', \'True\', \'None\', \'None\', \'\', \'png\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'shuffle\', \'seed\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'subset\'], varargs=None, keywords=None, defaults=[\'None\', \'32\', \'True\', \'None\', \'None\', \'\', \'png\', \'None\'], "
   }
   member_method {
     name: "flow_from_directory"
-    argspec: "args=[\'self\', \'directory\', \'target_size\', \'color_mode\', \'classes\', \'class_mode\', \'batch_size\', \'shuffle\', \'seed\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'follow_links\', \'interpolation\'], varargs=None, keywords=None, defaults=[\'(256, 256)\', \'rgb\', \'None\', \'categorical\', \'32\', \'True\', \'None\', \'None\', \'\', \'png\', \'False\', \'nearest\'], "
+    argspec: "args=[\'self\', \'directory\', \'target_size\', \'color_mode\', \'classes\', \'class_mode\', \'batch_size\', \'shuffle\', \'seed\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'follow_links\', \'subset\', \'interpolation\'], varargs=None, keywords=None, defaults=[\'(256, 256)\', \'rgb\', \'None\', \'categorical\', \'32\', \'True\', \'None\', \'None\', \'\', \'png\', \'False\', \'None\', \'nearest\'], "
   }
   member_method {
     name: "random_transform"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-numpy-array-iterator.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-numpy-array-iterator.pbtxt
index 4ef6e6e99e..42196ddeee 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-numpy-array-iterator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.-numpy-array-iterator.pbtxt
@@ -6,7 +6,7 @@ tf_class {
   is_instance: "<type \'object\'>"
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'x\', \'y\', \'image_data_generator\', \'batch_size\', \'shuffle\', \'seed\', \'data_format\', \'save_to_dir\', \'save_prefix\', \'save_format\'], varargs=None, keywords=None, defaults=[\'32\', \'False\', \'None\', \'None\', \'None\', \'\', \'png\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'image_data_generator\', \'batch_size\', \'shuffle\', \'seed\', \'data_format\', \'save_to_dir\', \'save_prefix\', \'save_format\', \'subset\'], varargs=None, keywords=None, defaults=[\'32\', \'False\', \'None\', \'None\', \'None\', \'\', \'png\', \'None\'], "
   }
   member_method {
     name: "next"
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.pbtxt
index d28fef6965..6b850dd6b7 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.image.pbtxt
@@ -36,6 +36,10 @@ tf_module {
     name: "load_img"
     argspec: "args=[\'path\', \'grayscale\', \'target_size\', \'interpolation\'], varargs=None, keywords=None, defaults=[\'False\', \'None\', \'nearest\'], "
   }
+  member_method {
+    name: "random_brightness"
+    argspec: "args=[\'x\', \'brightness_range\'], varargs=None, keywords=None, defaults=None"
+  }
   member_method {
     name: "random_channel_shift"
     argspec: "args=[\'x\', \'intensity\', \'channel_axis\'], varargs=None, keywords=None, defaults=[\'0\'], "
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.-timeseries-generator.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.-timeseries-generator.pbtxt
new file mode 100644
index 0000000000..d9c3215b55
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.-timeseries-generator.pbtxt
@@ -0,0 +1,14 @@
+path: "tensorflow.keras.preprocessing.sequence.TimeseriesGenerator"
+tf_class {
+  is_instance: "<class \'tensorflow.python.keras._impl.keras.preprocessing.sequence.TimeseriesGenerator\'>"
+  is_instance: "<class \'tensorflow.python.keras._impl.keras.utils.data_utils.Sequence\'>"
+  is_instance: "<type \'object\'>"
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'data\', \'targets\', \'length\', \'sampling_rate\', \'stride\', \'start_index\', \'end_index\', \'shuffle\', \'reverse\', \'batch_size\'], varargs=None, keywords=None, defaults=[\'1\', \'1\', \'0\', \'None\', \'False\', \'False\', \'128\'], "
+  }
+  member_method {
+    name: "on_epoch_end"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.pbtxt
index 1b01935cc5..cf59f8a272 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.sequence.pbtxt
@@ -1,5 +1,9 @@
 path: "tensorflow.keras.preprocessing.sequence"
 tf_module {
+  member {
+    name: "TimeseriesGenerator"
+    mtype: "<type \'type\'>"
+  }
   member_method {
     name: "make_sampling_table"
     argspec: "args=[\'size\', \'sampling_factor\'], varargs=None, keywords=None, defaults=[\'1e-05\'], "
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.text.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.text.pbtxt
index d106429df0..50b54fc7e1 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.text.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.preprocessing.text.pbtxt
@@ -4,6 +4,10 @@ tf_module {
     name: "Tokenizer"
     mtype: "<type \'type\'>"
   }
+  member_method {
+    name: "hashing_trick"
+    argspec: "args=[\'text\', \'n\', \'hash_function\', \'filters\', \'lower\', \'split\'], varargs=None, keywords=None, defaults=[\'None\', \'!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n\', \'True\', \' \'], "
+  }
   member_method {
     name: "one_hot"
     argspec: "args=[\'text\', \'n\', \'filters\', \'lower\', \'split\'], varargs=None, keywords=None, defaults=[\'!\"#$%&()*+,-./:;<=>?@[\\\\]^_`{|}~\\t\\n\', \'True\', \' \'], "
-- 
GitLab


From 1004396a769ad9fdf350ed28083bca5b6ad00402 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 22 Mar 2018 14:24:23 -0700
Subject: [PATCH 376/960] Remove use of deprecated API from RNN Colorbot
 example.

PiperOrigin-RevId: 190125356
---
 .../examples/rnn_colorbot/rnn_colorbot.py     | 27 +++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
index 29f0232454..88fffc962f 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
@@ -60,6 +60,7 @@ import functools
 import os
 import sys
 import time
+import urllib
 
 import six
 import tensorflow as tf
@@ -89,13 +90,35 @@ def parse(line):
   return rgb, chars, length
 
 
+def maybe_download(filename, work_directory, source_url):
+  """Download the data from source url, unless it's already here.
+
+  Args:
+    filename: string, name of the file in the directory.
+    work_directory: string, path to working directory.
+    source_url: url to download from if file doesn't exist.
+
+  Returns:
+    Path to resulting file.
+  """
+  if not tf.gfile.Exists(work_directory):
+    tf.gfile.MakeDirs(work_directory)
+  filepath = os.path.join(work_directory, filename)
+  if not tf.gfile.Exists(filepath):
+    temp_file_name, _ = urllib.request.urlretrieve(source_url)
+    tf.gfile.Copy(temp_file_name, filepath)
+    with tf.gfile.GFile(filepath) as f:
+      size = f.size()
+      print("Successfully downloaded", filename, size, "bytes.")
+  return filepath
+
+
 def load_dataset(data_dir, url, batch_size):
   """Loads the colors data at path into a PaddedDataset."""
 
   # Downloads data at url into data_dir/basename(url). The dataset has a header
   # row (color_name, r, g, b) followed by comma-separated lines.
-  path = tf.contrib.learn.datasets.base.maybe_download(
-      os.path.basename(url), data_dir, url)
+  path = maybe_download(os.path.basename(url), data_dir, url)
 
   # This chain of commands loads our data by:
   #   1. skipping the header; (.skip(1))
-- 
GitLab


From a34a3b2035ca0cfd48488c03bd4b088070bf9a25 Mon Sep 17 00:00:00 2001
From: Mahmoud Abuzaina <mahmoud.abuzaina@intel.com>
Date: Thu, 22 Mar 2018 14:32:12 -0700
Subject: [PATCH 377/960] Fixing the issue where MKL-DNN is getting built when
 not using --config=mkl

---
 tensorflow/tensorflow.bzl | 53 +++++++++++++++++++++++++++------------
 1 file changed, 37 insertions(+), 16 deletions(-)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 9b0db8a112..8549c34691 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -788,7 +788,33 @@ def tf_cc_test_mkl(srcs,
                    tags=[],
                    size="medium",
                    args=None):
-  if_mkl(tf_cc_tests(srcs, deps, name, linkstatic=linkstatic, tags=tags, size=size, args=args, nocopts="-fno-exceptions"))
+  for src in srcs:
+    native.cc_test(
+      name=src_to_test_name(src),
+      srcs=if_mkl([src]) + tf_binary_additional_srcs(),
+      copts=tf_copts(),
+      linkopts=select({
+        clean_dep("//tensorflow:android"): [
+            "-pie",
+          ],
+        clean_dep("//tensorflow:windows"): [],
+        clean_dep("//tensorflow:windows_msvc"): [],
+        "//conditions:default": [
+            "-lpthread",
+            "-lm"
+        ],
+      }) + _rpath_linkopts(src_to_test_name(src)),
+      deps=deps + if_mkl(
+          [
+              "//third_party/mkl:intel_binary_blob",
+          ],
+      ),
+      linkstatic=linkstatic,
+      tags=tags,
+      size=size,
+      args=args,
+      nocopts="-fno-exceptions")
+
 
 def tf_cc_tests_gpu(srcs,
                     deps,
@@ -1006,16 +1032,12 @@ register_extension_info(
 def tf_mkl_kernel_library(name,
                           prefix=None,
                           srcs=None,
-                          gpu_srcs=None,
                           hdrs=None,
                           deps=None,
                           alwayslink=1,
                           copts=tf_copts(),
-                          nocopts="-fno-exceptions",
-                          **kwargs):
+                          nocopts="-fno-exceptions"):
   """A rule to build MKL-based TensorFlow kernel libraries."""
-  gpu_srcs = gpu_srcs  # unused argument
-  kwargs = kwargs  # unused argument
 
   if not bool(srcs):
     srcs = []
@@ -1028,16 +1050,15 @@ def tf_mkl_kernel_library(name,
     hdrs = hdrs + native.glob(
         [prefix + "*.h"])
 
-  if_mkl(
-      native.cc_library(
-          name=name,
-          srcs=srcs,
-          hdrs=hdrs,
-          deps=deps,
-          alwayslink=alwayslink,
-          copts=copts,
-          nocopts=nocopts
-      ))
+  native.cc_library(
+      name=name,
+      srcs=if_mkl(srcs),
+      hdrs=hdrs,
+      deps=deps,
+      alwayslink=alwayslink,
+      copts=copts,
+      nocopts=nocopts
+  )
 
 register_extension_info(
     extension_name = "tf_mkl_kernel_library",
-- 
GitLab


From 1a99109e8832bc94710d2dcfb5d9525688913a50 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Thu, 22 Mar 2018 14:38:41 -0700
Subject: [PATCH 378/960] Merge consecutive broadcast HLO instructions.

As an optimization, replace consecutive broadcast instructions with a single equivalent broadcast in algebraic simplification.

PiperOrigin-RevId: 190127730
---
 .../xla/service/algebraic_simplifier.cc       | 22 ++++++--
 .../xla/service/algebraic_simplifier_test.cc  | 51 +++++++++++++++++++
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 971c2935c8..88f6ff0a07 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1121,10 +1121,10 @@ bool OutputIsSubsetOfOperandElements(HloInstruction* instruction,
 
 Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
   auto operand = broadcast->mutable_operand(0);
+  auto dims = broadcast->dimensions();
   // A degenerate broadcast of a reshape that does not change the number of
   // elements can be replaced by a reshape.
-  if (std::is_sorted(broadcast->dimensions().begin(),
-                     broadcast->dimensions().end()) &&
+  if (std::is_sorted(dims.begin(), dims.end()) &&
       ShapeUtil::ElementsIn(broadcast->shape()) ==
           ShapeUtil::ElementsIn(operand->shape())) {
     VLOG(10) << "transform broadcast(X) -> reshape(X) where "
@@ -1142,8 +1142,8 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     VLOG(10) << "transform broadcast(X) -> transpose(X) where "
                 "n(broadcast(X)) == n(X)";
     return ReplaceWithNewInstruction(
-        broadcast, HloInstruction::CreateTranspose(broadcast->shape(), operand,
-                                                   broadcast->dimensions()));
+        broadcast,
+        HloInstruction::CreateTranspose(broadcast->shape(), operand, dims));
   }
 
   // A broadcast of a reshape which merely inserts 1-sized dimensions can
@@ -1157,7 +1157,6 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     if (merely_inserts_or_deletes_1_sized_dimensions &&
         deleted_indices.empty()) {
       std::reverse(inserted_indices.begin(), inserted_indices.end());
-      auto dims = broadcast->dimensions();
       for (auto inserted_index : inserted_indices) {
         dims.erase(dims.begin() + inserted_index);
       }
@@ -1201,6 +1200,19 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
         return user->ReplaceAllUsesWith(new_broadcast);
       }
     }
+    return Status::OK();
+  }
+
+  // Merge two consecutive broadcasts into a single one.
+  if (operand->opcode() == HloOpcode::kBroadcast) {
+    std::vector<int64> new_dimensions(operand->dimensions().size());
+    for (auto dim : operand->dimensions()) {
+      new_dimensions.push_back(dims[dim]);
+    }
+    return ReplaceWithNewInstruction(
+        broadcast,
+        HloInstruction::CreateBroadcast(
+            broadcast->shape(), operand->mutable_operand(0), new_dimensions));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 451294ef5d..3b80a827bf 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -35,6 +35,8 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 
+using ::testing::ElementsAre;
+
 namespace xla {
 namespace {
 
@@ -2462,6 +2464,55 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
               op::DynamicSlice(op::Parameter(), op::Parameter()));
 }
 
+// Test that two consecutive broadcasts can be merged to one.
+TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) {
+  HloComputation::Builder builder(TestName());
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* input_array = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR1<float>({3, 4})));
+  HloInstruction* inner_bcast = builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r2f32, input_array, {1}));
+  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 2, 2});
+  builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r3f32, inner_bcast, {0, 2}));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_THAT(root, op::Broadcast(op::Constant()));
+  EXPECT_THAT(root->dimensions(), ElementsAre(2));
+}
+
+// Test that two consecutive broadcasts can be merged to one.
+TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) {
+  HloComputation::Builder builder(TestName());
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 3});
+  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 5, 3});
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r2f32, "param0"));
+  // The initial dimensions go to places 0 and 2 in the 3-dim array,
+  // and to places 1 and 3 in the 4-dim array,
+  HloInstruction* inner_bcast = builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r3f32, param0, {0, 2}));
+  Shape r4f32 = ShapeUtil::MakeShape(F32, {4, 2, 5, 3});
+  builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r4f32, inner_bcast, {1, 2, 3}));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_THAT(root, op::Broadcast(op::Parameter(0)));
+  EXPECT_THAT(root->dimensions(), ElementsAre(1, 3));
+}
+
 struct PadReduceWindowEffectiveBroadcastCase {
   std::vector<int64> input_spatials;
   std::vector<int64> symmetric_pad_spatials;
-- 
GitLab


From 730e69519a93a668d97ea298d52365326c00357d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 22 Mar 2018 14:47:22 -0700
Subject: [PATCH 379/960] Automated g4 rollback of changelist 190021164

PiperOrigin-RevId: 190129094
---
 tensorflow/c/eager/BUILD                      |   2 +
 tensorflow/c/eager/c_api.cc                   | 194 ++++++-----------
 tensorflow/c/eager/c_api_internal.h           |  84 +-------
 tensorflow/core/common_runtime/eager/BUILD    |  22 ++
 .../core/common_runtime/eager/context.cc      | 153 ++++++++++++++
 .../core/common_runtime/eager/context.h       | 198 ++++++++++++++++++
 6 files changed, 450 insertions(+), 203 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/context.cc
 create mode 100644 tensorflow/core/common_runtime/eager/context.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 841ff48a38..bea5a121b3 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -28,6 +28,7 @@ tf_cuda_library(
             "//tensorflow/c:c_api",
             "//tensorflow/c:c_api_internal",
             "//tensorflow/core:core_cpu",
+            "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core:core_cpu_internal",
@@ -64,6 +65,7 @@ tf_cuda_library(
         "//tensorflow/core:framework_lite",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
+        "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
     ],
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index a23015c99e..2402a6d044 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -71,18 +71,6 @@ std::atomic_int_fast64_t func_id_generator(0);
 
 }  // namespace
 
-TFE_ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy) {
-  if (!soft_placement) {
-    return original_policy;
-  }
-  if (original_policy == TFE_DEVICE_PLACEMENT_EXPLICIT ||
-      original_policy == TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32) {
-    return TFE_DEVICE_PLACEMENT_SILENT;
-  }
-  return original_policy;
-}
-
 extern "C" {
 
 TFE_ContextOptions* TFE_NewContextOptions() { return new TFE_ContextOptions; }
@@ -104,19 +92,7 @@ void TFE_ContextOptionsSetDevicePlacementPolicy(
 TF_CAPI_EXPORT extern void TFE_ContextSetAsyncForThread(TFE_Context* ctx,
                                                         unsigned char async,
                                                         TF_Status* status) {
-  {
-    tensorflow::mutex_lock l(ctx->async_map_mu);
-    ctx->thread_local_async[std::this_thread::get_id()] = async;
-  }
-  if (async) {
-    ctx->executor.EnableAsync();
-  } else {
-    // TODO(agarwal): Currently we add a wait here to handle cases where a sync
-    // op has a control dependency on an async op, and the latter has not
-    // executed yet. This wait can be removed by storing all the control inputs
-    // and waiting for them when executing ops.
-    status->status = ctx->executor.WaitForAllPendingNodes();
-  }
+  status->status = ctx->context.SetAsyncForThread(async);
 }
 
 void TFE_DeleteContextOptions(TFE_ContextOptions* options) { delete options; }
@@ -133,34 +109,26 @@ TFE_Context* TFE_NewContext(const TFE_ContextOptions* opts, TF_Status* status) {
       new tensorflow::DeviceMgr(devices));
   tensorflow::Rendezvous* r =
       new tensorflow::IntraProcessRendezvous(device_mgr.get());
-  return new TFE_Context(*opts, std::move(device_mgr), r);
+  return new TFE_Context(opts->session_options.options, opts->policy,
+                         opts->async, std::move(device_mgr), r);
 }
 
 void TFE_DeleteContext(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.WaitForAllPendingNodes();
-  {
-    tensorflow::mutex_lock ml(ctx->cache_mu);
-    tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
-  }
-  ctx->rendezvous->Unref();
   delete ctx;
 }
 
 TF_DeviceList* TFE_ContextListDevices(TFE_Context* ctx, TF_Status* status) {
   TF_DeviceList* list = new TF_DeviceList;
-  ctx->device_manager->ListDeviceAttributes(&list->response);
+  ctx->context.device_mgr()->ListDeviceAttributes(&list->response);
   return list;
 }
 
-void TFE_ContextClearCaches(TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->cache_mu);
-  tensorflow::gtl::STLDeleteValues(&ctx->kernel_cache);
-}
+void TFE_ContextClearCaches(TFE_Context* ctx) { ctx->context.ClearCaches(); }
 
 void TFE_ContextSetThreadLocalDevicePlacementPolicy(
     TFE_Context* ctx, TFE_ContextDevicePlacementPolicy policy) {
-  tensorflow::mutex_lock ml(ctx->policy_map_mu);
-  ctx->thread_local_policies[std::this_thread::get_id()] = policy;
+  ctx->context.SetThreadLocalDevicePlacementPolicy(
+      static_cast<tensorflow::ContextDevicePlacementPolicy>(policy));
 }
 
 // Note: this function looks up a thread local policy. So it should be called in
@@ -168,25 +136,20 @@ void TFE_ContextSetThreadLocalDevicePlacementPolicy(
 // safe to call this function from the async EagerExecutor threads.
 extern TFE_ContextDevicePlacementPolicy TFE_ContextGetDevicePlacementPolicy(
     TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->policy_map_mu);
-  auto policy_map_it =
-      ctx->thread_local_policies.find(std::this_thread::get_id());
-  if (policy_map_it != ctx->thread_local_policies.end()) {
-    return policy_map_it->second;
-  }
-  return ctx->policy;
+  return static_cast<TFE_ContextDevicePlacementPolicy>(
+      ctx->context.GetDevicePlacementPolicy());
 }
 
 void TFE_ContextAsyncWait(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.WaitForAllPendingNodes();
+  status->status = ctx->context.AsyncWait();
 }
 
 void TFE_ContextGetStatus(TFE_Context* ctx, TF_Status* status) {
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
 }
 
 void TFE_ContextAsyncClearError(TFE_Context* ctx) {
-  ctx->executor.ClearError();
+  ctx->context.ClearAsyncError();
 }
 
 TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
@@ -259,7 +222,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   // nullptr.
   tensorflow::Device* src_opd = nullptr;
   TF_RETURN_IF_ERROR(h->TensorAndDevice(&src, &srcd, &src_opd));
-  if (srcd == nullptr) srcd = ctx->devices[0];
+  if (srcd == nullptr) srcd = ctx->context.HostCPU();
   bool is_same_device =
       (srcd == dstd) || (DeviceName(srcd) == DeviceName(dstd));
   const bool dst_cpu = IsCPU(dstd);
@@ -332,8 +295,7 @@ TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name,
   status->status = tensorflow::AttrTypeMapForOp(name, &types);
   if (status->status.ok()) return new TFE_Op(ctx, name, types);
   if (TF_GetCode(status) == TF_NOT_FOUND) {
-    tensorflow::mutex_lock l(ctx->functions_mu);
-    if (ctx->func_lib_def.Find(name) != nullptr) {
+    if (ctx->context.FindFunctionByName(name)) {
       status->status = tensorflow::Status::OK();
       return new TFE_Op(ctx, name, nullptr);
     }
@@ -346,20 +308,14 @@ void TFE_DeleteOp(TFE_Op* op) { delete op; }
 void TFE_OpSetDevice(TFE_Op* op, const char* device_name, TF_Status* status) {
   tensorflow::Device* d = nullptr;
   if (device_name != nullptr && strlen(device_name) > 0) {
-    auto it = op->ctx->devices_map.find(device_name);
-    if (it == op->ctx->devices_map.end()) {
-      status->status =
-          tensorflow::errors::InvalidArgument(device_name, " unknown device.");
-      return;
-    }
-    d = it->second;
+    status->status = op->ctx->context.FindDeviceByName(device_name, &d);
   }
   op->device = d;
 }
 
 const char* TFE_OpGetDevice(TFE_Op* op, TF_Status* status) {
   tensorflow::Device* device =
-      (op->device == nullptr) ? op->ctx->devices[0] : op->device;
+      (op->device == nullptr) ? op->ctx->context.HostCPU() : op->device;
   return device->name().c_str();
 }
 
@@ -634,7 +590,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
 tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
                                  TFE_Context* ctx, TF_Status* status) {
   tensorflow::DeviceSet ds;
-  for (tensorflow::Device* d : ctx->devices) {
+  for (tensorflow::Device* d : *ctx->context.devices()) {
     ds.AddDevice(d);
   }
   tensorflow::DeviceTypeVector final_devices;
@@ -648,7 +604,7 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
         "Could not find valid device for node ", ndef.DebugString());
     return nullptr;
   }
-  for (tensorflow::Device* d : ctx->devices) {
+  for (tensorflow::Device* d : *ctx->context.devices()) {
     if (d->device_type() == final_devices[0].type_string()) {
       return d;
     }
@@ -663,9 +619,8 @@ tensorflow::Status Execute(
     const tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4>& op_inputs,
     tensorflow::KernelAndDevice* kernel, tensorflow::NodeExecStats* maybe_stats,
     TFE_TensorHandle** retvals, int num_retvals) {
-  if (!ctx->soft_placement && device == nullptr) {
-    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
-    device = ctx->devices[0];
+  if (!ctx->context.SoftPlacement() && device == nullptr) {
+    device = ctx->context.HostCPU();
   }
 
   if (device == nullptr) {
@@ -684,8 +639,8 @@ tensorflow::Status Execute(
     inputs[i] = *input_tensor;
   }
   // WARNING: kernel->Run utilizes the FunctionLibraryRuntime
-  // (ctx->func_lib(device)), which in turn holds a pointer to func_lib_def,
-  // which is GUARDED_BY(ctx->functions_mu). But knowledge of the implementation
+  // (ctx->func_lib(device)), which in turn holds a pointer to func_lib_def.
+  // But knowledge of the implementation
   // of FunctionLibraryRuntime tells us that func_lib_def is not accessed by
   // FunctionLibraryRuntime::Run(), so there is no thread-safety concern here.
   // This is quite subtle. Re-work things to make this better?  (Would it make
@@ -697,18 +652,18 @@ tensorflow::Status Execute(
   if (maybe_stats != nullptr) {
     maybe_stats->set_op_end_rel_micros(tensorflow::Env::Default()->NowMicros() -
                                        maybe_stats->all_start_micros());
-    tensorflow::mutex_lock ml(ctx->metadata_mu);
-    if (ctx->should_store_metadata.load()) {
-      auto* step_stats = ctx->run_metadata.mutable_step_stats();
+    tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
+    if (ctx->context.ShouldStoreMetadata()) {
+      auto* step_stats = ctx->context.RunMetadataProto()->mutable_step_stats();
       // Lazily initialize the RunMetadata with information about all devices if
       // this is the first call.
-      while (step_stats->dev_stats_size() < ctx->devices.size()) {
+      while (step_stats->dev_stats_size() < ctx->context.devices()->size()) {
         step_stats->add_dev_stats();
       }
       // Find the current device's index.
       int device_idx = 0;
-      for (int i = 0; i < ctx->devices.size(); ++i) {
-        if (ctx->devices[i] == device) {
+      for (int i = 0; i < ctx->context.devices()->size(); ++i) {
+        if (ctx->context.devices()->at(i) == device) {
           device_idx = i;
           break;
         }
@@ -744,7 +699,7 @@ class ExecuteNode : public tensorflow::EagerNode {
               tensorflow::NodeExecStats* maybe_stats,
               const tensorflow::DataTypeVector& output_dtypes,
               TFE_TensorHandle** retvals, int num_retvals)
-      : tensorflow::EagerNode(op->ctx->executor.NextId()),
+      : tensorflow::EagerNode(op->ctx->context.NextId()),
         ctx_(op->ctx),
         op_device_(op->device),
         inputs_(op->inputs),
@@ -800,7 +755,7 @@ class CopyToDeviceNode : public tensorflow::EagerNode {
  public:
   CopyToDeviceNode(TFE_TensorHandle* src, tensorflow::Device* dstd,
                    TFE_Context* ctx)
-      : tensorflow::EagerNode(ctx->executor.NextId()),
+      : tensorflow::EagerNode(ctx->context.NextId()),
         src_(src),
         dstd_(dstd),
         ctx_(ctx),
@@ -866,8 +821,7 @@ const tensorflow::FunctionDef* OpToFunction(
   TFE_Context* ctx = op->ctx;
   const tensorflow::OpRegistrationData* op_data;
   {
-    tensorflow::tf_shared_lock l(ctx->functions_mu);
-    status->status = ctx->func_lib_def.LookUp(op->name, &op_data);
+    status->status = ctx->context.FindFunctionOpData(op->name, &op_data);
     if (!status->status.ok()) {
       return nullptr;
     }
@@ -963,10 +917,9 @@ const tensorflow::FunctionDef* OpToFunction(
   }
   VLOG(1) << "Fixed Output names and all types: " << fdef.DebugString();
 
-  tensorflow::mutex_lock l(ctx->functions_mu);
-  status->status = ctx->func_lib_def.AddFunctionDef(fdef);
+  ctx->context.AddFunctionDef(fdef);
   if (!status->status.ok()) return nullptr;
-  const auto ret = ctx->func_lib_def.Find(signature->name());
+  const auto ret = ctx->context.FindFunctionDef(signature->name());
   DCHECK(ret != nullptr);
   return ret;
 }
@@ -985,8 +938,7 @@ std::unique_ptr<TFE_Op> BuildXlaLaunch(TFE_Op* op, TF_Status* status) {
 
   const tensorflow::FunctionDef* fdef;
   {
-    tensorflow::tf_shared_lock l(op->ctx->functions_mu);
-    fdef = op->ctx->func_lib_def.Find(op->name);
+    fdef = op->ctx->context.FindFunctionDef(op->name);
   }
   std::vector<TF_DataType> const_input_types;
   std::vector<TF_DataType> arg_input_types;
@@ -1063,7 +1015,7 @@ extern "C" {
 void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
                  TF_Status* status) {
   TFE_Context* ctx = op->ctx;
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
   if (!status->status.ok()) {
     return;
   }
@@ -1087,7 +1039,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (op->inputs[i]->dtype == tensorflow::DT_RESOURCE &&
         input_op_device != op->device) {
       tensorflow::Device* d =
-          input_op_device == nullptr ? ctx->devices[0] : input_op_device;
+          input_op_device == nullptr ? ctx->context.HostCPU() : input_op_device;
       VLOG(1) << "Changing device of operation " << op->name << " to "
               << d->name() << " because input #" << i
               << " is a resource in this device.";
@@ -1095,40 +1047,35 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     }
   }
   tensorflow::Device* device = op->device;
-  if (!ctx->soft_placement && device == nullptr) {
-    // TODO(ashankar): ASSUMPTION: ctx->devices[0] is always CPU
-    device = ctx->devices[0];
+  if (!ctx->context.SoftPlacement() && device == nullptr) {
+    device = ctx->context.HostCPU();
   }
 
   tensorflow::Fprint128 cache_key =
       op->attrs.CacheKey(device == nullptr ? "unspecified" : device->name());
-  tensorflow::KernelAndDevice* kernel;
-  {
-    tensorflow::tf_shared_lock l(ctx->cache_mu);
-    kernel = tensorflow::gtl::FindPtrOrNull(ctx->kernel_cache, cache_key);
-  }
+  tensorflow::KernelAndDevice* kernel = ctx->context.GetCachedKernel(cache_key);
   if (kernel == nullptr) {
     const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef();
-    if (ctx->soft_placement && device == nullptr) {
+    if (ctx->context.SoftPlacement() && device == nullptr) {
       device = SelectDevice(ndef, ctx, status);
       if (!status->status.ok()) {
         return;
       }
     }
     CHECK(device != nullptr);
-    if (ctx->log_device_placement) {
+    if (ctx->context.LogDevicePlacement()) {
       LOG(INFO) << "Executing op " << ndef.op() << " in device "
                 << device->name();
     }
-    kernel = new tensorflow::KernelAndDevice(ctx->rendezvous);
+    kernel = new tensorflow::KernelAndDevice(ctx->context.GetRendezvous());
     // Knowledge of the implementation of Init (and in-turn
     // FunctionLibraryRuntime::CreateKernel) tells us that ctx->func_lib_def
     // will be accessed, so grab on to the lock.
     // See WARNING comment in Execute (before kernel->Run) - would be nice to
     // rework to avoid this subtlety.
-    tensorflow::tf_shared_lock l(ctx->functions_mu);
-    status->status =
-        tensorflow::KernelAndDevice::Init(ndef, ctx->func_lib(device), kernel);
+    tensorflow::tf_shared_lock l(*ctx->context.FunctionsMu());
+    status->status = tensorflow::KernelAndDevice::Init(
+        ndef, ctx->context.func_lib(device), kernel);
     if (!status->status.ok()) {
       delete kernel;
       return;
@@ -1136,7 +1083,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // Update output_dtypes inside `kernel`.
     const tensorflow::OpDef* op_def = nullptr;
     const tensorflow::FunctionDef* function_def =
-        ctx->func_lib_def.Find(ndef.op());
+        ctx->context.FuncLibDef()->Find(ndef.op());
     if (function_def != nullptr) {
       op_def = &(function_def->signature());
     }
@@ -1152,8 +1099,7 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     if (!status->status.ok()) {
       return;
     }
-    tensorflow::mutex_lock ml(ctx->cache_mu);
-    tensorflow::gtl::InsertOrUpdate(&(ctx->kernel_cache), cache_key, kernel);
+    ctx->context.AddKernelToCache(cache_key, kernel);
   }
   const tensorflow::DataTypeVector& output_dtypes = kernel->output_dtypes();
   const int output_dtypes_size = output_dtypes.size();
@@ -1171,11 +1117,11 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // device from the one requested above.
     device = kernel->device();
   }
-  status->status = ValidateInputTypeAndPlacement(ctx, ctx->devices[0], device,
-                                                 op, kernel->kernel());
+  status->status = ValidateInputTypeAndPlacement(ctx, ctx->context.HostCPU(),
+                                                 device, op, kernel->kernel());
   if (!status->status.ok()) return;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats;
-  if (ctx->should_store_metadata.load()) {
+  if (ctx->context.ShouldStoreMetadata()) {
     maybe_stats.reset(new tensorflow::NodeExecStats);
     maybe_stats->set_node_name(op->name);
     maybe_stats->set_all_start_micros(tensorflow::Env::Default()->NowMicros());
@@ -1183,14 +1129,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     maybe_stats->set_scheduled_micros(tensorflow::Env::Default()->NowMicros());
     // TODO(apassos) track referenced tensors
   }
-  if (ctx->Async()) {
+  if (ctx->context.Async()) {
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
     // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
     tensorflow::EagerNode* node =
         new ExecuteNode(op, kernel, maybe_stats.release(), output_dtypes,
                         retvals, *num_retvals);
-    ctx->executor.Add(node);
+    ctx->context.ExecutorAdd(node);
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
@@ -1206,23 +1152,24 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
                                                TFE_Context* ctx,
                                                const char* device_name,
                                                TF_Status* status) {
-  status->status = ctx->executor.status();
+  status->status = ctx->context.GetStatus();
   if (!status->status.ok()) {
     return nullptr;
   }
-  tensorflow::Device* dstd = ctx->devices[0];
+  tensorflow::Device* dstd = ctx->context.HostCPU();
   if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status = ctx->device_manager->LookupDevice(device_name, &dstd);
+    status->status =
+        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
     if (!status->status.ok()) return nullptr;
   }
-  if (ctx->Async()) {
+  if (ctx->context.Async()) {
     // Note that `h` may not be currently ready. However execution order will
     // make sure that `h` is ready before the copy is actually done.
     CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
     TFE_TensorHandle* output = node->dst();
     // Note that calling Add makes `node` accessible by the EagerExecutor
     // thread. So further accesses need to be thread-safe.
-    ctx->executor.Add(node);
+    ctx->context.ExecutorAdd(node);
     return output;
   } else {
     TFE_TensorHandle* output = nullptr;
@@ -1240,24 +1187,20 @@ void TFE_ContextAddFunctionDef(TFE_Context* ctx,
         tensorflow::errors::InvalidArgument("Invalid FunctionDef proto");
     return;
   }
-  tensorflow::mutex_lock l(ctx->functions_mu);
-  status->status = ctx->func_lib_def.AddFunctionDef(function_def);
+  status->status = ctx->context.AddFunctionDef(function_def);
 }
 
 void TFE_ContextAddFunction(TFE_Context* ctx, TF_Function* function,
                             TF_Status* status) {
-  tensorflow::mutex_lock l(ctx->functions_mu);
-  status->status = ctx->func_lib_def.AddFunctionDef(function->fdef);
+  status->status = ctx->context.AddFunctionDef(function->fdef);
 }
 
 void TFE_ContextEnableRunMetadata(TFE_Context* ctx) {
-  ctx->should_store_metadata.store(true);
+  ctx->context.SetShouldStoreMetadata(true);
 }
 
 void TFE_ContextDisableRunMetadata(TFE_Context* ctx) {
-  tensorflow::mutex_lock ml(ctx->metadata_mu);
-  ctx->should_store_metadata.store(false);
-  ctx->run_metadata.Clear();
+  ctx->context.SetShouldStoreMetadata(false);
 }
 
 }  // extern "C"
@@ -1286,9 +1229,9 @@ void TFE_ContextExportRunMetadata(TFE_Context* ctx, TF_Buffer* buf,
                                   TF_Status* status) {
   TFE_ContextAsyncWait(ctx, status);
   if (!status->status.ok()) return;
-  tensorflow::mutex_lock ml(ctx->metadata_mu);
-  status->status = MessageToBuffer(ctx->run_metadata, buf);
-  ctx->run_metadata.Clear();
+  tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
+  status->status = MessageToBuffer(*ctx->context.RunMetadataProto(), buf);
+  ctx->context.RunMetadataProto()->Clear();
 }
 
 namespace {
@@ -1363,11 +1306,6 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
 }  // namespace tensorflow
 
 
-bool TFE_Context::Async() const {
-  tensorflow::mutex_lock l(async_map_mu);
-  return tensorflow::gtl::FindWithDefault(
-      thread_local_async, std::this_thread::get_id(), async_default);
-}
 
 bool TFE_TensorHandle::IsReady() {
   if (node_id == 0) return true;
@@ -1381,7 +1319,7 @@ tensorflow::Status TFE_TensorHandle::WaitReady() {
   {
     tensorflow::mutex_lock l(ctx_mutex_);
     if (ctx_ == nullptr) return tensorflow::Status::OK();
-    executor = &ctx_->executor;
+    executor = ctx_->context.Executor();
   }
   return executor->WaitFor(node_id);
 }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index a79f8ddd33..5b29120b40 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/c/c_api_internal.h"
 #include "tensorflow/c/eager/runtime.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/function.h"
@@ -52,85 +53,18 @@ struct TFE_ContextOptions {
       TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32};
 };
 
-TFE_ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, TFE_ContextDevicePlacementPolicy original_policy);
-
 struct TFE_Context {
-  explicit TFE_Context(const TFE_ContextOptions& opts,
+  explicit TFE_Context(const tensorflow::SessionOptions& opts,
+                       TFE_ContextDevicePlacementPolicy default_policy,
+                       bool async,
                        std::unique_ptr<tensorflow::DeviceMgr> device_mgr,
                        tensorflow::Rendezvous* rendezvous)
-      : soft_placement(
-            opts.session_options.options.config.allow_soft_placement()),
-        policy(PlacementPolicy(soft_placement, opts.policy)),
-        device_manager(std::move(device_mgr)),
-        devices(device_manager->ListDevices()),
-        rendezvous(rendezvous),
-        pflr(new tensorflow::ProcessFunctionLibraryRuntime(
-            device_manager.get(), opts.session_options.options.env,
-            TF_GRAPH_DEF_VERSION, &func_lib_def, {})),
-        log_device_placement(
-            opts.session_options.options.config.log_device_placement()),
-        async_default(opts.async) {
-    if (async_default) executor.EnableAsync();
-
-    for (auto* device : devices) {
-      devices_map[tensorflow::StringPiece(device->name())] = device;
-    }
-  }
-
-  const bool soft_placement;
-  const TFE_ContextDevicePlacementPolicy policy;
-
-  // Note: we cannot use C++11 thread_local here as there is no concept of a
-  // thread-local-object-local variable in C++11.
-  tensorflow::mutex policy_map_mu;
-  std::unordered_map<std::thread::id, TFE_ContextDevicePlacementPolicy>
-      thread_local_policies GUARDED_BY(policy_map_mu);
-
-  std::unique_ptr<tensorflow::DeviceMgr> device_manager;
-  // Devices owned by device_manager
-  std::vector<tensorflow::Device*> devices;
-  // All devices are not owned.
-  tensorflow::gtl::FlatMap<tensorflow::StringPiece, tensorflow::Device*,
-                           tensorflow::StringPieceHasher>
-      devices_map;
-  tensorflow::Rendezvous* const rendezvous;
-
-  tensorflow::mutex functions_mu;
-  tensorflow::FunctionLibraryDefinition func_lib_def GUARDED_BY(functions_mu){
-      tensorflow::OpRegistry::Global(), {}};
-
-  // One FunctionLibraryRuntime per device.
-  // func_libs[i] is the FunctionLibraryRuntime corresponding to
-  // session->devices[i].
-  const std::unique_ptr<tensorflow::ProcessFunctionLibraryRuntime> pflr;
-
-  tensorflow::mutex cache_mu;
-  std::unordered_map<tensorflow::Fprint128, tensorflow::KernelAndDevice*,
-                     tensorflow::Fprint128Hasher>
-      kernel_cache GUARDED_BY(cache_mu);
-
-  tensorflow::FunctionLibraryRuntime* func_lib(tensorflow::Device* d) const {
-    return pflr->GetFLR(d->name());
-  }
+      : context(opts,
+                static_cast<tensorflow::ContextDevicePlacementPolicy>(
+                    default_policy),
+                async, std::move(device_mgr), rendezvous) {}
 
-  // Whether we should compute RunMetadata.
-  std::atomic<bool> should_store_metadata{false};
-  tensorflow::mutex metadata_mu;
-  tensorflow::RunMetadata run_metadata GUARDED_BY(metadata_mu);
-  const bool log_device_placement;
-  // EagerExecutor for async execution.
-  tensorflow::EagerExecutor executor;
-
-  // True if running in asynchronous mode.
-  bool Async() const;
-
-  // True if the default value for execution mode is async. Note that this value
-  // can be overridden per thread based on `thread_local_async` overrides.
-  const bool async_default;
-  mutable tensorflow::mutex async_map_mu;
-  std::unordered_map<std::thread::id, bool> thread_local_async
-      GUARDED_BY(async_map_mu);
+  tensorflow::EagerContext context;
 };
 
 struct TFE_TensorHandle : public tensorflow::core::RefCounted {
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 8ba560bef8..de10b10b7e 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -32,6 +32,28 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "context",
+    srcs = [
+        "context.cc",
+    ],
+    hdrs = [
+        "context.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":eager_executor",
+        ":kernel_and_device",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
+    ],
+)
+
 tf_cuda_library(
     name = "kernel_and_device",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
new file mode 100644
index 0000000000..0566329f18
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -0,0 +1,153 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/context.h"
+
+namespace tensorflow {
+
+ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, ContextDevicePlacementPolicy original_policy) {
+  if (!soft_placement) {
+    return original_policy;
+  }
+  if (original_policy == DEVICE_PLACEMENT_EXPLICIT ||
+      original_policy == DEVICE_PLACEMENT_SILENT_FOR_INT32) {
+    return DEVICE_PLACEMENT_SILENT;
+  }
+  return original_policy;
+}
+
+EagerContext::EagerContext(const SessionOptions& opts,
+                           ContextDevicePlacementPolicy default_policy,
+                           bool async, std::unique_ptr<DeviceMgr> device_mgr,
+                           Rendezvous* rendezvous)
+    : soft_placement_(opts.config.allow_soft_placement()),
+      policy_(PlacementPolicy(soft_placement_, default_policy)),
+      device_manager_(std::move(device_mgr)),
+      devices_(device_manager_->ListDevices()),
+      rendezvous_(rendezvous),
+      pflr_(new ProcessFunctionLibraryRuntime(device_manager_.get(), opts.env,
+                                              TF_GRAPH_DEF_VERSION,
+                                              &func_lib_def_, {})),
+      log_device_placement_(opts.config.log_device_placement()),
+      async_default_(async) {
+  if (async_default_) {
+    executor_.EnableAsync();
+  }
+
+  for (auto* device : devices_) {
+    devices_map_[device->name()] = device;
+  }
+}
+
+bool EagerContext::Async() const {
+  mutex_lock l(async_map_mu_);
+  return gtl::FindWithDefault(thread_local_async_, std::this_thread::get_id(),
+                              async_default_);
+}
+
+Status EagerContext::SetAsyncForThread(bool async) {
+  {
+    tensorflow::mutex_lock l(async_map_mu_);
+    thread_local_async_[std::this_thread::get_id()] = async;
+  }
+  if (async) {
+    executor_.EnableAsync();
+  } else {
+    // TODO(agarwal): Currently we add a wait here to handle cases where a
+    // sync op has a control dependency on an async op, and the latter has not
+    // executed yet. This wait can be removed by storing all the control
+    // inputs and waiting for them when executing ops.
+    return executor_.WaitForAllPendingNodes();
+  }
+  return Status::OK();
+}
+
+void EagerContext::ClearCaches() {
+  mutex_lock ml(cache_mu_);
+  gtl::STLDeleteValues(&kernel_cache_);
+}
+
+void EagerContext::SetThreadLocalDevicePlacementPolicy(
+    ContextDevicePlacementPolicy policy) {
+  mutex_lock ml(policy_map_mu_);
+  thread_local_policies_[std::this_thread::get_id()] = policy;
+}
+
+ContextDevicePlacementPolicy EagerContext::GetDevicePlacementPolicy() {
+  mutex_lock ml(policy_map_mu_);
+  auto policy_map_it = thread_local_policies_.find(std::this_thread::get_id());
+  if (policy_map_it != thread_local_policies_.end()) {
+    return policy_map_it->second;
+  }
+  return policy_;
+}
+
+EagerContext::~EagerContext() {
+  executor_.WaitForAllPendingNodes().IgnoreError();
+  ClearCaches();
+  rendezvous_->Unref();
+}
+
+bool EagerContext::FindFunctionByName(const string& name) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.Find(name) != nullptr;
+}
+
+Status EagerContext::FindFunctionOpData(
+    const string& name, const tensorflow::OpRegistrationData** op_data) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.LookUp(name, op_data);
+}
+
+const FunctionDef* EagerContext::FindFunctionDef(const string& name) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.Find(name);
+}
+
+Status EagerContext::FindDeviceByName(const string& name, Device** result) {
+  auto it = devices_map_.find(name);
+  if (it == devices_map_.end()) {
+    return errors::InvalidArgument(name, " unknown device.");
+  }
+  *result = it->second;
+  return Status::OK();
+}
+
+Status EagerContext::AddFunctionDef(const FunctionDef& fdef) {
+  mutex_lock l(functions_mu_);
+  return func_lib_def_.AddFunctionDef(fdef);
+}
+
+KernelAndDevice* EagerContext::GetCachedKernel(Fprint128 cache_key) {
+  tf_shared_lock l(cache_mu_);
+  return gtl::FindPtrOrNull(kernel_cache_, cache_key);
+}
+
+void EagerContext::AddKernelToCache(Fprint128 cache_key,
+                                    KernelAndDevice* kernel) {
+  mutex_lock ml(cache_mu_);
+  gtl::InsertOrUpdate(&kernel_cache_, cache_key, kernel);
+}
+
+void EagerContext::SetShouldStoreMetadata(bool value) {
+  should_store_metadata_.store(value);
+  if (!value) {
+    mutex_lock ml(metadata_mu_);
+    run_metadata_.Clear();
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
new file mode 100644
index 0000000000..bc97219dae
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -0,0 +1,198 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+// Note: there's a copy enum in eager/c_api.h. It should be kept in sync.
+enum ContextDevicePlacementPolicy {
+  // Running operations with input tensors on the wrong device will fail. When
+  // soft placement is enabled acts like TFE_DEVICE_PLACEMENT_SILENT.
+  DEVICE_PLACEMENT_EXPLICIT = 0,
+  // Copy the tensor to the right device but log a warning.
+  DEVICE_PLACEMENT_WARN = 1,
+  // Silently copy the tensor, which has a performance cost since the
+  // operation will be blocked till the copy completes.
+  DEVICE_PLACEMENT_SILENT = 2,
+  // Default placement policy which silently copies int32 tensors but not other
+  // dtypes.  When soft placement is enabled acts like
+  // TFE_DEVICE_PLACEMENT_SILENT.
+  DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
+};
+
+ContextDevicePlacementPolicy PlacementPolicy(
+    bool soft_placement, ContextDevicePlacementPolicy original_policy);
+
+class EagerContext {
+ public:
+  explicit EagerContext(const SessionOptions& opts,
+                        ContextDevicePlacementPolicy default_policy, bool async,
+                        std::unique_ptr<DeviceMgr> device_mgr,
+                        Rendezvous* rendezvous);
+
+  ~EagerContext();
+
+  // Returns the function library runtime for the given device.
+  FunctionLibraryRuntime* func_lib(Device* d) const {
+    return pflr_->GetFLR(d->name());
+  }
+
+  // True if running in asynchronous mode.
+  bool Async() const;
+
+  EagerExecutor* Executor() { return &executor_; }
+
+  // Sets whether this thread should run in synchronous or asynchronous mode.
+  Status SetAsyncForThread(bool async);
+
+  // TODO(apassos) make this return a constant reference
+  gtl::FlatMap<string, Device*, StringPieceHasher>* device_map() {
+    return &devices_map_;
+  }
+
+  // TODO(apassos) make this return a constant reference
+  std::vector<Device*>* devices() { return &devices_; }
+
+  // Clears the kernel caches.
+  void ClearCaches();
+
+  // Sets the device placement policy for the current thread.
+  void SetThreadLocalDevicePlacementPolicy(ContextDevicePlacementPolicy policy);
+
+  // Returns the device placement policy for the current thread.
+  ContextDevicePlacementPolicy GetDevicePlacementPolicy();
+
+  Status AsyncWait() { return executor_.WaitForAllPendingNodes(); }
+
+  Status GetStatus() { return executor_.status(); }
+
+  void ClearAsyncError() { executor_.ClearError(); }
+
+  bool FindFunctionByName(const string& name);
+
+  Status FindFunctionOpData(const string& name,
+                            const tensorflow::OpRegistrationData** op_data);
+
+  const FunctionDef* FindFunctionDef(const string& name);
+
+  Status FindDeviceByName(const string& name, Device** result);
+
+  Device* HostCPU() { return devices_[0]; }
+
+  bool SoftPlacement() { return soft_placement_; }
+
+  uint64 NextId() { return executor_.NextId(); }
+
+  void ExecutorAdd(EagerNode* node) { executor_.Add(node); }
+
+  Status AddFunctionDef(const FunctionDef& fdef);
+
+  KernelAndDevice* GetCachedKernel(Fprint128 cache_key);
+
+  void AddKernelToCache(Fprint128 cache_key, KernelAndDevice* kernel);
+
+  bool LogDevicePlacement() { return log_device_placement_; }
+
+  Rendezvous* GetRendezvous() { return rendezvous_; }
+
+  mutex* FunctionsMu() { return &functions_mu_; }
+
+  tensorflow::DeviceMgr* device_mgr() { return device_manager_.get(); }
+
+  // TODO(apassos) remove the need for this
+  void ReleaseDeviceMgr() { device_manager_.release(); }
+
+  // TODO(apassos) clean up RunMetadata storage.
+  mutex* MetadataMu() { return &metadata_mu_; }
+  bool ShouldStoreMetadata() { return should_store_metadata_.load(); }
+  void SetShouldStoreMetadata(bool value);
+  RunMetadata* RunMetadataProto() { return &run_metadata_; }
+
+  FunctionLibraryDefinition* FuncLibDef() { return &func_lib_def_; }
+
+ private:
+  const bool soft_placement_;
+  const ContextDevicePlacementPolicy policy_;
+
+  // Note: we cannot use C++11 thread_local here as there is no concept of a
+  // thread-local-object-local variable in C++11.
+  mutex policy_map_mu_;
+  std::unordered_map<std::thread::id, ContextDevicePlacementPolicy>
+      thread_local_policies_ GUARDED_BY(policy_map_mu_);
+
+  std::unique_ptr<DeviceMgr> device_manager_;
+  // Devices owned by device_manager
+  std::vector<Device*> devices_;
+  // All devices are not owned.
+  gtl::FlatMap<string, Device*, StringPieceHasher> devices_map_;
+  Rendezvous* const rendezvous_;
+
+  mutex functions_mu_;
+  FunctionLibraryDefinition func_lib_def_ GUARDED_BY(functions_mu_){
+      OpRegistry::Global(), {}};
+
+  // One FunctionLibraryRuntime per device.
+  // func_libs[i] is the FunctionLibraryRuntime corresponding to
+  // session->devices[i].
+  const std::unique_ptr<ProcessFunctionLibraryRuntime> pflr_;
+
+  mutex cache_mu_;
+  std::unordered_map<Fprint128, KernelAndDevice*, Fprint128Hasher> kernel_cache_
+      GUARDED_BY(cache_mu_);
+
+  // Whether we should compute RunMetadata.
+  std::atomic<bool> should_store_metadata_{false};
+  mutex metadata_mu_;
+  RunMetadata run_metadata_ GUARDED_BY(metadata_mu_);
+  const bool log_device_placement_;
+  // EagerExecutor for async execution.
+  EagerExecutor executor_;
+
+  // True if the default value for execution mode is async. Note that this value
+  // can be overridden per thread based on `thread_local_async` overrides.
+  const bool async_default_;
+  mutable mutex async_map_mu_;
+  std::unordered_map<std::thread::id, bool> thread_local_async_
+      GUARDED_BY(async_map_mu_);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_CONTEXT_H_
-- 
GitLab


From 63d46266ba5b2a513244e13321f76e7acd03aba3 Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Thu, 22 Mar 2018 14:53:59 -0700
Subject: [PATCH 380/960] Move cuDNN RNN ops to core, for use in the internal
 TF codebase only (not publicly exposed).

RELNOTES: Moved cuDNN RNN ops to core.
PiperOrigin-RevId: 190130405
---
 tensorflow/contrib/BUILD                      |   2 -
 tensorflow/contrib/cmake/python_modules.txt   |   2 -
 .../contrib/cmake/tf_core_kernels.cmake       |   2 -
 tensorflow/contrib/cmake/tf_core_ops.cmake    |   2 +-
 tensorflow/contrib/cmake/tf_python.cmake      |   3 +-
 tensorflow/contrib/cudnn_rnn/BUILD            |  68 +--------
 .../cudnn_rnn/python/ops/cudnn_rnn_ops.py     |   7 +-
 tensorflow/core/BUILD                         |  47 +++++++
 .../api_def/base_api/api_def_CudnnRNN.pbtxt   |  36 +++++
 .../base_api/api_def_CudnnRNNBackprop.pbtxt   |  45 ++++++
 .../api_def_CudnnRNNCanonicalToParams.pbtxt   |  35 +++++
 .../base_api/api_def_CudnnRNNParamsSize.pbtxt |  27 ++++
 .../api_def_CudnnRNNParamsToCanonical.pbtxt   |  35 +++++
 .../api_def/python_api/api_def_CudnnRNN.pbtxt |   4 +
 .../python_api/api_def_CudnnRNNBackprop.pbtxt |   4 +
 .../api_def_CudnnRNNCanonicalToParams.pbtxt   |   4 +
 .../api_def_CudnnRNNParamsSize.pbtxt          |   4 +
 .../api_def_CudnnRNNParamsToCanonical.pbtxt   |   4 +
 tensorflow/core/kernels/BUILD                 |  17 +++
 .../kernels/cudnn_rnn_ops.cc                  |   0
 .../cudnn_rnn => core}/ops/cudnn_rnn_ops.cc   | 130 ++----------------
 .../ops/cudnn_rnn_ops_test.cc                 |   0
 tensorflow/python/BUILD                       |   8 ++
 tensorflow/python/__init__.py                 |   4 +
 24 files changed, 287 insertions(+), 203 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CudnnRNN.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CudnnRNNBackprop.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CudnnRNNCanonicalToParams.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsSize.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsToCanonical.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CudnnRNN.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CudnnRNNBackprop.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CudnnRNNCanonicalToParams.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsSize.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsToCanonical.pbtxt
 rename tensorflow/{contrib/cudnn_rnn => core}/kernels/cudnn_rnn_ops.cc (100%)
 rename tensorflow/{contrib/cudnn_rnn => core}/ops/cudnn_rnn_ops.cc (53%)
 rename tensorflow/{contrib/cudnn_rnn => core}/ops/cudnn_rnn_ops_test.cc (100%)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index d103da79e3..2d7bbc016f 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -119,7 +119,6 @@ cc_library(
     deps = [
         "//tensorflow/contrib/boosted_trees:boosted_trees_kernels",
         "//tensorflow/contrib/coder:all_kernels",
-        "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
         "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
@@ -143,7 +142,6 @@ cc_library(
     deps = [
         "//tensorflow/contrib/boosted_trees:boosted_trees_ops_op_lib",
         "//tensorflow/contrib/coder:all_ops",
-        "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_ops_op_lib",
         "//tensorflow/contrib/data:dataset_ops_op_lib",
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 0d2a6a23db..f7d3c73b2c 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -147,8 +147,6 @@ tensorflow/contrib/crf
 tensorflow/contrib/crf/python
 tensorflow/contrib/crf/python/ops
 tensorflow/contrib/cudnn_rnn
-tensorflow/contrib/cudnn_rnn/kernels
-tensorflow/contrib/cudnn_rnn/ops
 tensorflow/contrib/cudnn_rnn/python
 tensorflow/contrib/cudnn_rnn/python/layers
 tensorflow/contrib/cudnn_rnn/python/ops
diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index 998f99ecc1..ed018b4fed 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -67,8 +67,6 @@ if(tensorflow_BUILD_CONTRIB_KERNELS)
       "${tensorflow_source_dir}/tensorflow/contrib/coder/kernels/range_coder_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/coder/kernels/range_coder_ops_util.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/coder/ops/coder_ops.cc"
-      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc"
-      "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/ignore_errors_dataset_op.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/prefetching_kernels.cc"
       "${tensorflow_source_dir}/tensorflow/contrib/data/kernels/threadpool_dataset_op.cc"
diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index 59e094812a..d6712aa2b4 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -21,6 +21,7 @@ set(tf_op_lib_names
     "checkpoint_ops"
     "control_flow_ops"
     "ctc_ops"
+    "cudnn_rnn_ops"
     "data_flow_ops"
     "dataset_ops"
     "functional_ops"
@@ -84,7 +85,6 @@ GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_prediction "${tensorflow_source_dir}/t
 GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_quantiles "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/quantile_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(boosted_trees_stats_accumulator "${tensorflow_source_dir}/tensorflow/contrib/boosted_trees/ops/stats_accumulator_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(coder "${tensorflow_source_dir}/tensorflow/contrib/coder/ops/coder_ops.cc")
-GENERATE_CONTRIB_OP_LIBRARY(cudnn_rnn "${tensorflow_source_dir}/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(data_dataset "${tensorflow_source_dir}/tensorflow/contrib/data/ops/dataset_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(factorization_clustering "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/clustering_ops.cc")
 GENERATE_CONTRIB_OP_LIBRARY(factorization_factorization "${tensorflow_source_dir}/tensorflow/contrib/factorization/ops/factorization_ops.cc")
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 1e354bf212..31e715b654 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -326,6 +326,7 @@ GENERATE_PYTHON_OP_LIB("checkpoint_ops")
 GENERATE_PYTHON_OP_LIB("control_flow_ops"
   ADDITIONAL_LIBRARIES $<TARGET_OBJECTS:tf_no_op>)
 GENERATE_PYTHON_OP_LIB("ctc_ops")
+GENERATE_PYTHON_OP_LIB("cudnn_rnn_ops")
 GENERATE_PYTHON_OP_LIB("data_flow_ops")
 GENERATE_PYTHON_OP_LIB("dataset_ops")
 GENERATE_PYTHON_OP_LIB("image_ops")
@@ -367,8 +368,6 @@ GENERATE_PYTHON_OP_LIB("contrib_boosted_trees_stats_accumulator_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/boosted_trees/python/ops/gen_stats_accumulator_ops.py)
 GENERATE_PYTHON_OP_LIB("contrib_coder_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/coder/python/ops/gen_coder_ops.py)
-GENERATE_PYTHON_OP_LIB("contrib_cudnn_rnn_ops"
-  DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/cudnn_rnn/ops/gen_cudnn_rnn_ops.py)
 GENERATE_PYTHON_OP_LIB("contrib_data_dataset_ops"
   DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/data/python/ops/gen_dataset_ops.py)
 GENERATE_PYTHON_OP_LIB("contrib_factorization_clustering_ops"
diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index fec358c4e1..fa86ad38c9 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -9,52 +9,10 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-load("//tensorflow:tensorflow.bzl", "tf_custom_op_library")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_libs")
 load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
-load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
-load("//tensorflow:tensorflow.bzl", "tf_cc_test")
-
-tf_custom_op_library(
-    name = "python/ops/_cudnn_rnn_ops.so",
-    srcs = [
-        "kernels/cudnn_rnn_ops.cc",
-        "ops/cudnn_rnn_ops.cc",
-    ],
-    deps = [
-        "//tensorflow/core/kernels:bounds_check_lib",
-        "@farmhash_archive//:farmhash",
-    ],
-)
-
-tf_kernel_library(
-    name = "cudnn_rnn_kernels",
-    srcs = ["kernels/cudnn_rnn_ops.cc"],
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:lib_internal",
-        "//tensorflow/core:stream_executor",
-        "//tensorflow/core/kernels:bounds_check_lib",
-        "//third_party/eigen3",
-        "@farmhash_archive//:farmhash",
-    ],
-)
-
-tf_gen_op_libs(
-    op_lib_names = ["cudnn_rnn_ops"],
-    deps = [
-        "//tensorflow/core:lib",
-    ],
-)
-
-tf_gen_op_wrapper_py(
-    name = "cudnn_rnn_ops",
-    deps = [":cudnn_rnn_ops_op_lib"],
-)
 
 tf_custom_op_py_library(
     name = "cudnn_rnn_py",
@@ -64,20 +22,13 @@ tf_custom_op_py_library(
         "python/layers/cudnn_rnn.py",
         "python/ops/cudnn_rnn_ops.py",
     ],
-    dso = [
-        ":python/ops/_cudnn_rnn_ops.so",
-    ],
-    kernels = [
-        ":cudnn_rnn_kernels",
-        ":cudnn_rnn_ops_op_lib",
-    ],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        ":cudnn_rnn_ops",
         "//tensorflow/contrib/util:util_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:cudnn_rnn_ops_gen",
         "//tensorflow/python:framework",
         "//tensorflow/python:framework_for_generated_wrappers",
         "//tensorflow/python:init_ops",
@@ -173,23 +124,6 @@ cuda_py_test(
     ],
 )
 
-tf_cc_test(
-    name = "cudnn_rnn_ops_test_cc",
-    size = "small",
-    srcs = [
-        "ops/cudnn_rnn_ops_test.cc",
-    ],
-    deps = [
-        ":cudnn_rnn_ops_op_lib",
-        "//tensorflow/core",
-        "//tensorflow/core:framework",
-        "//tensorflow/core:lib",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-        "//tensorflow/core:testlib",
-    ],
-)
-
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index e87162f0ee..622241a177 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -17,27 +17,22 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.cudnn_rnn.ops import gen_cudnn_rnn_ops
 from tensorflow.contrib.rnn.python.ops import lstm_ops
-from tensorflow.contrib.util import loader
 from tensorflow.python.framework import common_shapes
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.layers import base as base_layer
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_cudnn_rnn_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope as vs
-from tensorflow.python.platform import resource_loader
 from tensorflow.python.training import saver
 
-_cudnn_rnn_ops_so = loader.load_op_library(
-    resource_loader.get_path_to_datafile("_cudnn_rnn_ops.so"))
-
 CUDNN_RNN_UNIDIRECTION = "unidirectional"
 CUDNN_RNN_BIDIRECTION = "bidirectional"
 CUDNN_LSTM = "lstm"
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 15cbba8285..2885a9f823 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -688,6 +688,34 @@ cc_library(
     alwayslink = 1,
 )
 
+cc_library(
+    name = "cudnn_rnn_ops",
+    srcs = [
+        "ops/cudnn_rnn_ops.cc",
+    ],
+    linkstatic = 1,
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/core/kernels:bounds_check_lib",
+        "//third_party/eigen3",
+        "@farmhash_archive//:farmhash",
+    ],
+    alwayslink = 1,
+)
+
+tf_gen_op_libs(
+    op_lib_names = [
+        "cudnn_rnn_ops",
+    ],
+    deps = [
+        ":lib",
+    ],
+)
+
 cc_library(
     name = "ops",
     visibility = ["//visibility:public"],
@@ -700,6 +728,7 @@ cc_library(
         ":checkpoint_ops_op_lib",
         ":control_flow_ops_op_lib",
         ":ctc_ops_op_lib",
+        ":cudnn_rnn_ops_op_lib",
         ":data_flow_ops_op_lib",
         ":dataset_ops_op_lib",
         ":function_ops_op_lib",
@@ -840,6 +869,7 @@ cc_library(
         "//tensorflow/core/kernels:checkpoint_ops",
         "//tensorflow/core/kernels:control_flow_ops",
         "//tensorflow/core/kernels:ctc_ops",
+        "//tensorflow/core/kernels:cudnn_rnn_kernels",
         "//tensorflow/core/kernels:data_flow",
         "//tensorflow/core/kernels:dataset_ops",
         "//tensorflow/core/kernels:fake_quant_ops",
@@ -2914,6 +2944,23 @@ tf_cc_tests(
     ],
 )
 
+tf_cc_test(
+    name = "cudnn_rnn_ops_test_cc",
+    size = "small",
+    srcs = [
+        "ops/cudnn_rnn_ops_test.cc",
+    ],
+    deps = [
+        ":cudnn_rnn_ops",
+        "//tensorflow/core",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
+
 tf_cc_test_mkl(
     name = "mkl_runtime_tests",
     size = "small",
diff --git a/tensorflow/core/api_def/base_api/api_def_CudnnRNN.pbtxt b/tensorflow/core/api_def/base_api/api_def_CudnnRNN.pbtxt
new file mode 100644
index 0000000000..daeb5fe9a2
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CudnnRNN.pbtxt
@@ -0,0 +1,36 @@
+op {
+  graph_op_name: "CudnnRNN"
+  summary: "A RNN backed by cuDNN."
+  description: <<END
+Computes the RNN from the input and initial states, with respect to the params
+buffer.
+
+rnn_mode: Indicates the type of the RNN model.
+input_mode: Indicate whether there is a linear projection between the input and
+  The actual computation before the first layer. 'skip_input' is only allowed
+  when input_size == num_units; 'auto_select' implies 'skip_input' when
+  input_size == num_units; otherwise, it implies 'linear_input'.
+direction: Indicates whether a bidirectional model will be used.
+  dir = (direction == bidirectional) ? 2 : 1
+dropout: dropout probability. When set to 0., dropout is disabled.
+seed: the 1st part of a seed to initialize dropout.
+seed2: the 2nd part of a seed to initialize dropout.
+input: a 3-D tensor with the shape of [seq_length, batch_size, input_size].
+input_h: a 3-D tensor with the shape of [num_layer * dir, batch_size,
+    num_units].
+input_c: For LSTM, a 3-D tensor with the shape of
+    [num_layer * dir, batch, num_units]. For other models, it is ignored.
+params: a 1-D tensor that contains the weights and biases in an opaque layout.
+    The size must be created through CudnnRNNParamsSize, and initialized
+    separately. Note that they might not be compatible across different
+    generations. So it is a good idea to save and restore
+output: a 3-D tensor with the shape of [seq_length, batch_size,
+    dir * num_units].
+output_h: the same shape has input_h.
+output_c: the same shape as input_c for LSTM. An empty tensor for other models.
+is_training: Indicates whether this operation is used for inferenece or
+  training.
+reserve_space: an opaque tensor that can be used in backprop calculation. It
+  is only produced if is_training is false.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CudnnRNNBackprop.pbtxt b/tensorflow/core/api_def/base_api/api_def_CudnnRNNBackprop.pbtxt
new file mode 100644
index 0000000000..075ec52648
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CudnnRNNBackprop.pbtxt
@@ -0,0 +1,45 @@
+op {
+  graph_op_name: "CudnnRNNBackprop"
+  summary: "Backprop step of CudnnRNN."
+  description: <<END
+Compute the backprop of both data and weights in a RNN.
+
+rnn_mode: Indicates the type of the RNN model.
+input_mode: Indicate whether there is a linear projection between the input and
+    The actual computation before the first layer. 'skip_input' is only allowed
+    when input_size == num_units; 'auto_select' implies 'skip_input' when
+    input_size == num_units; otherwise, it implies 'linear_input'.
+direction: Indicates whether a bidirectional model will be used.
+    dir = (direction == bidirectional) ? 2 : 1
+dropout: dropout probability. When set to 0., dropout is disabled.
+seed: the 1st part of a seed to initialize dropout.
+seed2: the 2nd part of a seed to initialize dropout.
+input: a 3-D tensor with the shape of [seq_length, batch_size, input_size].
+input_h: a 3-D tensor with the shape of [num_layer * dir, batch_size,
+    num_units].
+input_c: For LSTM, a 3-D tensor with the shape of
+    [num_layer * dir, batch, num_units]. For other models, it is ignored.
+params: a 1-D tensor that contains the weights and biases in an opaque layout.
+    The size must be created through CudnnRNNParamsSize, and initialized
+    separately. Note that they might not be compatible across different
+    generations. So it is a good idea to save and restore
+output: a 3-D tensor with the shape of [seq_length, batch_size,
+    dir * num_units].
+output_h: the same shape has input_h.
+output_c: the same shape as input_c for LSTM. An empty tensor for other models.
+output_backprop: A 3-D tensor with the same shape as output in the forward pass.
+output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
+    pass.
+output_c_backprop: A 3-D tensor with the same shape as output_c in the forward
+    pass.
+reserve_space: The same reserve_space produced in for forward operation.
+input_backprop: The backprop to input in the forward pass. Has the same shape
+    as input.
+input_h_backprop: The backprop to input_h in the forward pass. Has the same
+    shape as input_h.
+input_c_backprop: The backprop to input_c in the forward pass. Has the same
+    shape as input_c.
+params_backprop: The backprop to the params buffer in the forward pass. Has the
+    same shape as params.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CudnnRNNCanonicalToParams.pbtxt b/tensorflow/core/api_def/base_api/api_def_CudnnRNNCanonicalToParams.pbtxt
new file mode 100644
index 0000000000..abf81a2071
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CudnnRNNCanonicalToParams.pbtxt
@@ -0,0 +1,35 @@
+op {
+  graph_op_name: "CudnnRNNCanonicalToParams"
+  summary: "Converts CudnnRNN params from canonical form to usable form."
+  description: <<END
+Writes a set of weights into the opaque params buffer so they can be used in
+upcoming training or inferences.
+
+Note that the params buffer may not be compatible across different GPUs. So any
+save and restoration should be converted to and from the canonical weights and
+biases.
+
+num_layers: Specifies the number of layers in the RNN model.
+num_units: Specifies the size of the hidden state.
+input_size: Specifies the size of the input state.
+weights: the canonical form of weights that can be used for saving
+    and restoration. They are more likely to be compatible across different
+    generations.
+biases: the canonical form of biases that can be used for saving
+    and restoration. They are more likely to be compatible across different
+    generations.
+num_params: number of parameter sets for all layers.
+    Each layer may contain multiple parameter sets, with each set consisting of
+    a weight matrix and a bias vector.
+rnn_mode: Indicates the type of the RNN model.
+input_mode: Indicate whether there is a linear projection between the input and
+    The actual computation before the first layer. 'skip_input' is only allowed
+    when input_size == num_units; 'auto_select' implies 'skip_input' when
+    input_size == num_units; otherwise, it implies 'linear_input'.
+direction: Indicates whether a bidirectional model will be used.
+    dir = (direction == bidirectional) ? 2 : 1
+dropout: dropout probability. When set to 0., dropout is disabled.
+seed: the 1st part of a seed to initialize dropout.
+seed2: the 2nd part of a seed to initialize dropout.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsSize.pbtxt b/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsSize.pbtxt
new file mode 100644
index 0000000000..31fb85d4fb
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsSize.pbtxt
@@ -0,0 +1,27 @@
+op {
+  graph_op_name: "CudnnRNNParamsSize"
+  summary: "Computes size of weights that can be used by a Cudnn RNN model."
+  description: <<END
+Return the params size that can be used by the Cudnn RNN model. Subsequent
+weight allocation and initialization should use this size.
+
+num_layers: Specifies the number of layers in the RNN model.
+num_units: Specifies the size of the hidden state.
+input_size: Specifies the size of the input state.
+rnn_mode: Indicates the type of the RNN model.
+input_mode: Indicate whether there is a linear projection between the input and
+  The actual computation before the first layer. 'skip_input' is only allowed
+  when input_size == num_units; 'auto_select' implies 'skip_input' when
+  input_size == num_units; otherwise, it implies 'linear_input'.
+direction: Indicates whether a bidirectional model will be used.
+  dir = (direction == bidirectional) ? 2 : 1
+dropout: dropout probability. When set to 0., dropout is disabled.
+seed: the 1st part of a seed to initialize dropout.
+seed2: the 2nd part of a seed to initialize dropout.
+params_size: The size of the params buffer that should be allocated and
+  initialized for this RNN model. Note that this params buffer may not be
+  compatible across GPUs. Please use CudnnRNNParamsWeights and
+  CudnnRNNParamsBiases to save and restore them in a way that is compatible
+  across different runs.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsToCanonical.pbtxt b/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsToCanonical.pbtxt
new file mode 100644
index 0000000000..47753bf8fc
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_CudnnRNNParamsToCanonical.pbtxt
@@ -0,0 +1,35 @@
+op {
+  graph_op_name: "CudnnRNNParamsToCanonical"
+  summary: "Retrieves CudnnRNN params in canonical form."
+  description: <<END
+Retrieves a set of weights from the opaque params buffer that can be saved and
+restored in a way compatible with future runs.
+
+Note that the params buffer may not be compatible across different GPUs. So any
+save and restoration should be converted to and from the canonical weights and
+biases.
+
+num_layers: Specifies the number of layers in the RNN model.
+num_units: Specifies the size of the hidden state.
+input_size: Specifies the size of the input state.
+num_params: number of parameter sets for all layers.
+    Each layer may contain multiple parameter sets, with each set consisting of
+    a weight matrix and a bias vector.
+weights: the canonical form of weights that can be used for saving
+    and restoration. They are more likely to be compatible across different
+    generations.
+biases: the canonical form of biases that can be used for saving
+    and restoration. They are more likely to be compatible across different
+    generations.
+rnn_mode: Indicates the type of the RNN model.
+input_mode: Indicate whether there is a linear projection between the input and
+    The actual computation before the first layer. 'skip_input' is only allowed
+    when input_size == num_units; 'auto_select' implies 'skip_input' when
+    input_size == num_units; otherwise, it implies 'linear_input'.
+direction: Indicates whether a bidirectional model will be used.
+    dir = (direction == bidirectional) ? 2 : 1
+dropout: dropout probability. When set to 0., dropout is disabled.
+seed: the 1st part of a seed to initialize dropout.
+seed2: the 2nd part of a seed to initialize dropout.
+END
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CudnnRNN.pbtxt b/tensorflow/core/api_def/python_api/api_def_CudnnRNN.pbtxt
new file mode 100644
index 0000000000..b13586b63b
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CudnnRNN.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CudnnRNN"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CudnnRNNBackprop.pbtxt b/tensorflow/core/api_def/python_api/api_def_CudnnRNNBackprop.pbtxt
new file mode 100644
index 0000000000..81c4efc60b
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CudnnRNNBackprop.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CudnnRNNBackprop"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CudnnRNNCanonicalToParams.pbtxt b/tensorflow/core/api_def/python_api/api_def_CudnnRNNCanonicalToParams.pbtxt
new file mode 100644
index 0000000000..164a306034
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CudnnRNNCanonicalToParams.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CudnnRNNCanonicalToParams"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsSize.pbtxt b/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsSize.pbtxt
new file mode 100644
index 0000000000..00f97f05b1
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsSize.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CudnnRNNParamsSize"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsToCanonical.pbtxt b/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsToCanonical.pbtxt
new file mode 100644
index 0000000000..841bc0cf55
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_CudnnRNNParamsToCanonical.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "CudnnRNNParamsToCanonical"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index f6137fb860..8d235e79c0 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -920,6 +920,22 @@ tf_kernel_library(
     ]) + ARRAY_DEPS,
 )
 
+tf_kernel_library(
+    name = "cudnn_rnn_kernels",
+    srcs = ["cudnn_rnn_ops.cc"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:cudnn_rnn_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:stream_executor",
+        "//tensorflow/core/kernels:bounds_check_lib",
+        "//third_party/eigen3",
+        "@farmhash_archive//:farmhash",
+    ],
+)
+
 tf_cc_test(
     name = "batch_norm_op_test",
     size = "small",
@@ -5079,6 +5095,7 @@ filegroup(
             # not used on Android. Those ops also do not compile if included,
             # unless we add the additional deps they need.
             "tf_record_reader_op.*",
+            "cudnn_rnn_ops.*",
             "lmdb_reader_op.*",
             "string_to_hash_bucket_op.*",
             "sdca_ops.*",
diff --git a/tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc b/tensorflow/core/kernels/cudnn_rnn_ops.cc
similarity index 100%
rename from tensorflow/contrib/cudnn_rnn/kernels/cudnn_rnn_ops.cc
rename to tensorflow/core/kernels/cudnn_rnn_ops.cc
diff --git a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc b/tensorflow/core/ops/cudnn_rnn_ops.cc
similarity index 53%
rename from tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc
rename to tensorflow/core/ops/cudnn_rnn_ops.cc
index 1a79bf066c..37d70a22ef 100644
--- a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops.cc
+++ b/tensorflow/core/ops/cudnn_rnn_ops.cc
@@ -21,31 +21,6 @@ limitations under the License.
 namespace tensorflow {
 namespace {
 
-constexpr auto kCudnnRNNCommonInputs = R"doc(
-num_layers: Specifies the number of layers in the RNN model.
-num_units: Specifies the size of the hidden state.
-input_size: Specifies the size of the input state.
-)doc";
-
-constexpr auto kCudnnRNNCommonAttrs = R"doc(
-rnn_mode: Indicates the type of the RNN model.
-input_mode: Indicate whether there is a linear projection between the input and
-    The actual computation before the first layer. 'skip_input' is only allowed
-    when input_size == num_units; 'auto_select' implies 'skip_input' when
-    input_size == num_units; otherwise, it implies 'linear_input'.
-direction: Indicates whether a bidirectional model will be used.
-    dir = (direction == bidirectional) ? 2 : 1
-dropout: dropout probability. When set to 0., dropout is disabled.
-seed: the 1st part of a seed to initialize dropout.
-seed2: the 2nd part of a seed to initialize dropout.
-)doc";
-
-constexpr auto kCudnnRNNParamsBuffer = R"doc(
-Note that the params buffer may not be compatible across different GPUs. So any
-save and restoration should be converted to and from the canonical weights and
-biases.
-)doc";
-
 constexpr auto kRNNModeAttrs =
     "rnn_mode: {'rnn_relu', 'rnn_tanh', 'lstm', 'gru'} = 'lstm'";
 
@@ -56,21 +31,13 @@ constexpr auto kRNNInputModeAttrs =
 constexpr auto kRNNDirectionAttrs =
     "direction: {'unidirectional', 'bidirectional'} = 'unidirectional'";
 
-constexpr auto kCudnnRNNParamsCanonical = R"doc(
-weights: the canonical form of weights that can be used for saving
-    and restoration. They are more likely to be compatible across different
-    generations.
-biases: the canonical form of biases that can be used for saving
-    and restoration. They are more likely to be compatible across different
-    generations.
-)doc";
-
 }  // namespace
 
 using shape_inference::DimensionHandle;
 using shape_inference::InferenceContext;
 using shape_inference::ShapeHandle;
 
+
 REGISTER_OP("CudnnRNNParamsSize")
     .Input("num_layers: int32")
     .Input("num_units: int32")
@@ -87,38 +54,8 @@ REGISTER_OP("CudnnRNNParamsSize")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->Vector(1));
       return Status::OK();
-    })
-    .Doc(strings::StrCat(R"doc(
-Return the params size that can be used by the Cudnn RNN model. Subsequent
-weight allocation and initialization should use this size.
-)doc",
-                         kCudnnRNNCommonInputs, kCudnnRNNCommonAttrs,
-                         R"doc(
-params_size: The size of the params buffer that should be allocated and
-    initialized for this RNN model. Note that this params buffer may not be
-    compatible across GPUs. Please use CudnnRNNParamsWeights and
-    CudnnRNNParamsBiases to save and restore them in a way that is compatible
-    across different runs.
-)doc",
-                         kCudnnRNNParamsBuffer));
+    });
 
-static string CudnnRNNForwardTensors() {
-  return R"doc(
-input: a 3-D tensor with the shape of [seq_length, batch_size, input_size].
-input_h: a 3-D tensor with the shape of [num_layer * dir, batch_size,
-    num_units].
-input_c: For LSTM, a 3-D tensor with the shape of
-    [num_layer * dir, batch, num_units]. For other models, it is ignored.
-params: a 1-D tensor that contains the weights and biases in an opaque layout.
-    The size must be created through CudnnRNNParamsSize, and initialized
-    separately. Note that they might not be compatible across different
-    generations. So it is a good idea to save and restore
-output: a 3-D tensor with the shape of [seq_length, batch_size,
-    dir * num_units].
-output_h: the same shape has input_h.
-output_c: the same shape as input_c for LSTM. An empty tensor for other models.
-)doc";
-}
 
 REGISTER_OP("CudnnRNN")
     .Input("input: T")
@@ -160,18 +97,8 @@ REGISTER_OP("CudnnRNN")
       c->set_output(2, output_c_shape);
       c->set_output(3, c->UnknownShape());
       return Status::OK();
-    })
-    .Doc(strings::StrCat(R"doc(
-Computes the RNN from the input and initial states, with respect to the params
-buffer.
-)doc",
-                         kCudnnRNNCommonAttrs, CudnnRNNForwardTensors(),
-                         R"doc(
-is_training: Indicates whether this operation is used for inferenece or
-    training.
-reserve_space: an opaque tensor that can be used in backprop calculation. It
-    is only produced if is_training is false.
-)doc"));
+    });
+
 
 REGISTER_OP("CudnnRNNBackprop")
     .Input("input: T")
@@ -207,27 +134,8 @@ REGISTER_OP("CudnnRNNBackprop")
       c->set_output(2, input_c_shape);
       c->set_output(3, params_shape);
       return Status::OK();
-    })
-    .Doc(strings::StrCat(R"doc(
-Compute the backprop of both data and weights in a RNN.
-)doc",
-                         kCudnnRNNCommonAttrs, CudnnRNNForwardTensors(),
-                         R"doc(
-output_backprop: A 3-D tensor with the same shape as output in the forward pass.
-output_h_backprop: A 3-D tensor with the same shape as output_h in the forward
-    pass.
-output_c_backprop: A 3-D tensor with the same shape as output_c in the forward
-    pass.
-reserve_space: The same reserve_space produced in for forward operation.
-input_backprop: The backprop to input in the forward pass. Has the same shape
-    as input.
-input_h_backprop: The backprop to input_h in the forward pass. Has the same
-    shape as input_h.
-input_c_backprop: The backprop to input_c in the forward pass. Has the same
-    shape as input_c.
-params_backprop: The backprop to the params buffer in the forward pass. Has the
-    same shape as params.
-)doc"));
+    });
+
 
 REGISTER_OP("CudnnRNNParamsToCanonical")
     .Input("num_layers: int32")
@@ -259,17 +167,8 @@ REGISTER_OP("CudnnRNNParamsToCanonical")
         c->set_output(num_params + i, c->Vector(InferenceContext::kUnknownDim));
       }
       return Status::OK();
-    })
-    .Doc(strings::StrCat(R"doc(
-Retrieves a set of weights from the opaque params buffer that can be saved and
-restored in a way compatible with future runs.
-)doc",
-                         kCudnnRNNCommonInputs, kCudnnRNNParamsBuffer, R"doc(
-num_params: number of parameter sets for all layers.
-    Each layer may contain multiple parameter sets, with each set consisting of
-    a weight matrix and a bias vector.
-)doc",
-                         kCudnnRNNParamsCanonical, kCudnnRNNCommonAttrs));
+    });
+
 
 REGISTER_OP("CudnnRNNCanonicalToParams")
     .Input("num_layers: int32")
@@ -289,17 +188,6 @@ REGISTER_OP("CudnnRNNCanonicalToParams")
     .SetShapeFn([](InferenceContext* c) {
       c->set_output(0, c->Vector(InferenceContext::kUnknownDim));
       return Status::OK();
-    })
-    .Doc(strings::StrCat(R"doc(
-Writes a set of weights into the opaque params buffer so they can be used in
-upcoming training or inferences.
-)doc",
-                         kCudnnRNNCommonInputs, kCudnnRNNParamsCanonical,
-                         kCudnnRNNParamsBuffer, R"doc(
-num_params: number of parameter sets for all layers.
-    Each layer may contain multiple parameter sets, with each set consisting of
-    a weight matrix and a bias vector.
-)doc",
-                         kCudnnRNNCommonAttrs));
+    });
 
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc b/tensorflow/core/ops/cudnn_rnn_ops_test.cc
similarity index 100%
rename from tensorflow/contrib/cudnn_rnn/ops/cudnn_rnn_ops_test.cc
rename to tensorflow/core/ops/cudnn_rnn_ops_test.cc
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 079905781d..0e2b980213 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -78,6 +78,7 @@ py_library(
         ":client_testlib",
         ":confusion_matrix",
         ":control_flow_ops",
+        ":cudnn_rnn_ops_gen",
         ":errors",
         ":framework",
         ":framework_for_generated_wrappers",
@@ -1387,6 +1388,13 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "cudnn_rnn_ops_gen",
+    visibility = [
+        "//tensorflow:__subpackages__",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "candidate_sampling_ops_gen",
     visibility = ["//learning/brain/python/ops:__pkg__"],
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 5a9cd7531d..3346937904 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -99,6 +99,10 @@ from tensorflow.python.user_ops import user_ops
 from tensorflow.python.util import compat
 
 
+# Import cudnn rnn ops to make sure their ops are registered.
+from tensorflow.python.ops import gen_cudnn_rnn_ops as _
+
+
 # Import the names from python/training.py as train.Name.
 from tensorflow.python.training import training as train
 
-- 
GitLab


From 580d8c085a1ebcc5f5e6199e1542dbd56d3a42db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 15:14:40 -0700
Subject: [PATCH 381/960] Add kwargs support to wrap_py_func. Also includes
 some fixes to the reference tests and an actual test for wrapping NumPy
 function that uses kwargs.

PiperOrigin-RevId: 190133811
---
 .../contrib/py2tf/converters/call_trees.py    | 32 +++++---
 tensorflow/contrib/py2tf/impl/api.py          |  2 +-
 tensorflow/contrib/py2tf/utils/py_func.py     | 75 +++++++++++++------
 .../contrib/py2tf/utils/py_func_test.py       | 72 ++++++++++--------
 4 files changed, 116 insertions(+), 65 deletions(-)

diff --git a/tensorflow/contrib/py2tf/converters/call_trees.py b/tensorflow/contrib/py2tf/converters/call_trees.py
index 74fbf80677..e3040f09e4 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees.py
+++ b/tensorflow/contrib/py2tf/converters/call_trees.py
@@ -28,6 +28,7 @@ import types
 import gast
 
 from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.py2tf.pyct import ast_util
 from tensorflow.contrib.py2tf.pyct import inspect_utils
 from tensorflow.contrib.py2tf.pyct import parser
 from tensorflow.contrib.py2tf.pyct import templates
@@ -190,23 +191,27 @@ class CallTreeTransformer(transformer.Base):
     return node
 
   def _wrap_to_py_func_no_return(self, node):
-    # TODO(mdan): Properly handle varargs, kwargs, etc.
+    # TODO(mdan): Properly handle varargs, etc.
     template = """
-      py2tf_utils.wrap_py_func(func, None, (original_args,), True)
+      py2tf_utils.wrap_py_func(func, None, (args,), kwargs, True)
     """
-    return templates.replace(template, func=node.func, original_args=node.args)
+    return templates.replace(
+        template,
+        func=node.func,
+        args=node.args,
+        kwargs=ast_util.keywords_to_dict(node.keywords))
 
-  def _wrap_to_py_func_single_return(self, node, fqn):
-    # TODO(mdan): Properly handle varargs, kwargs, etc.
+  def _wrap_to_py_func_single_return(self, node, dtype):
+    # TODO(mdan): Properly handle varargs, etc.
     template = """
-      py2tf_utils.wrap_py_func(func, dtype, (original_args,), False)
+      py2tf_utils.wrap_py_func(func, dtype, (args,), kwargs, False)
     """
-    dtype = KNOWN_NUMPY_FUNCTIONS[fqn].dtype
     return templates.replace_as_expression(
         template,
         func=node.func,
         dtype=parser.parse_expression(dtype),
-        original_args=node.args)
+        args=node.args,
+        kwargs=ast_util.keywords_to_dict(node.keywords))
 
   def _insert_dynamic_conversion(self, node):
     """Inlines a dynamic conversion for a dynamic function."""
@@ -227,10 +232,10 @@ class CallTreeTransformer(transformer.Base):
     # Before we could convert all the time though, we'd need a reasonable
     # caching mechanism.
     template = """
-      py2tf_api.converted_call(func, True, False, {}, original_args)
+      py2tf_api.converted_call(func, True, False, {}, args)
     """
     call_expr = templates.replace(
-        template, func=node.func, original_args=node.args)
+        template, func=node.func, args=node.args)
     new_call = call_expr[0].value
     # TODO(mdan): Improve the template mechanism to better support this.
     new_call.keywords = node.keywords
@@ -273,11 +278,14 @@ class CallTreeTransformer(transformer.Base):
       target_entity = anno.getanno(node.func, 'live_val')
       if anno.hasanno(node.func, 'fqn'):
         target_fqn = anno.getanno(node.func, 'fqn')
+      else:
+        target_fqn = None
       if self._function_is_compilable(target_entity):
         node = self._rename_compilable_function(node)
-      elif target_fqn in KNOWN_NUMPY_FUNCTIONS:
+      elif target_fqn and target_fqn in KNOWN_NUMPY_FUNCTIONS:
         # TODO(mdan): Should we replace these with equivalent TF ops instead?
-        node = self._wrap_to_py_func_single_return(node, target_fqn)
+        node = self._wrap_to_py_func_single_return(
+            node, KNOWN_NUMPY_FUNCTIONS[target_fqn].dtype)
       else:
         raise NotImplementedError(
             'py_func with return values (unknown function)')
diff --git a/tensorflow/contrib/py2tf/impl/api.py b/tensorflow/contrib/py2tf/impl/api.py
index 3a40729e5a..a9e8ea2043 100644
--- a/tensorflow/contrib/py2tf/impl/api.py
+++ b/tensorflow/contrib/py2tf/impl/api.py
@@ -110,7 +110,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
             'RunMode.PY_FUNC does not yet support kwargs')
       # TODO(mdan): Add support for kwargs.
       return py_func.wrap_py_func(
-          f, return_dtypes, args, use_dummy_return=not return_dtypes)
+          f, return_dtypes, args, kwargs, use_dummy_return=not return_dtypes)
 
     if run_as == RunMode.GRAPH:
       wrapper = graph_wrapper
diff --git a/tensorflow/contrib/py2tf/utils/py_func.py b/tensorflow/contrib/py2tf/utils/py_func.py
index 79920cd841..34f2a8b70b 100644
--- a/tensorflow/contrib/py2tf/utils/py_func.py
+++ b/tensorflow/contrib/py2tf/utils/py_func.py
@@ -35,7 +35,7 @@ class MatchDType(namedtuple('MatchDType', ('arg_number',))):
   pass
 
 
-def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
+def wrap_py_func(f, return_dtypes, args, kwargs=None, use_dummy_return=False):
   """Helper that wraps a callable to py_func.
 
   The helper passes tensor arguments through the py_func interface. Non-tensor
@@ -51,29 +51,56 @@ def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
         return values or use_dummy_return is True. Use MatchDType to define a
         dtype identical to that of `i`th argument (argument 0 is the first);
         an argument must of Tensor type if it is to be used with MatchDType.
-    arguments: Arguments for f, as list or tuple.
+    args: Positional arguments for f, as list or tuple.
+    kwargs: Keyword arguments for f, as dict with string keys. May be None.
     use_dummy_return: If True, the function will return a dummy value of 1
         and discard its actual return value.
   Returns:
     The return values of f converted to tensor.
   Raises:
-    ValueError: if the arguments are incorrect.
+    ValueError: if any of the arguments are incorrect.
   """
 
   if return_dtypes and use_dummy_return:
     raise ValueError('if use_dummy_return is True, return_dtypes must be empty')
 
-  n = len(arguments)
-  arg_is_tensor = tuple(map(tensor_util.is_tensor, arguments))
-  index_in_tensor_list = [0] * n
-  i = 0
-  for j in range(n):
-    index_in_tensor_list[j] = i
-    if arg_is_tensor[j]:
-      i += 1
-
-  def match_argument(arg_number):
-    arg = arguments[arg_number]
+  tensor_args = []
+  tensor_args_idx = {}
+
+  # Of the positional arguments, only grab the tensor ones to be passed through
+  # the py_func.
+  n_args = len(args)
+  arg_is_tensor = tuple(map(tensor_util.is_tensor, args))
+  for i in range(n_args):
+    if arg_is_tensor[i]:
+      tensor_args_idx[i] = len(tensor_args)
+      tensor_args.append(args[i])
+
+  # We essentially take the tensor kwargs, if any, and add them to the list of
+  # positional arguments. The kwargs are then reconstructed inside the py_func.
+  #
+  # For example, if
+  #
+  #     args = [Tensor(1), 'foo']
+  #     kwargs = {'a': Tensor(2), 'b': 'bar'}
+  #
+  # Then
+  #
+  #     tensor_args = (Tensor(1), Tensor(2))
+  #     kwarg_keys = ('a', 'b')
+  if kwargs:
+    kwarg_keys = tuple(kwargs.keys())
+    kwarg_is_tensor = {k: tensor_util.is_tensor(kwargs[k]) for k in kwarg_keys}
+    for k in kwarg_keys:
+      if kwarg_is_tensor[k]:
+        tensor_args_idx[k] = len(tensor_args)
+        tensor_args.append(kwargs[k])
+  else:
+    kwarg_keys = ()
+
+  # Set up return dtypes.
+  def match_arg_dtype(arg_number):
+    arg = args[arg_number]
     if not arg_is_tensor[arg_number]:
       raise ValueError(
           'argument %d was used with MatchDType and must be a tf.Tensor, but '
@@ -82,20 +109,24 @@ def wrap_py_func(f, return_dtypes, arguments, use_dummy_return=False):
 
   if return_dtypes:
     if isinstance(return_dtypes, MatchDType):
-      return_dtypes = match_argument(return_dtypes.arg_number)
+      return_dtypes = match_arg_dtype(return_dtypes.arg_number)
     elif isinstance(return_dtypes, (list, tuple)):
       return_dtypes = tuple(
-          match_argument(a.arg_number) if isinstance(a, MatchDType) else a
+          match_arg_dtype(a.arg_number) if isinstance(a, MatchDType) else a
           for a in return_dtypes)
     else:
       assert isinstance(return_dtypes, dtypes.DType)
 
   def f_wrapper(*tensor_args):
-    f_args = tuple(tensor_args[index_in_tensor_list[i]]
-                   if arg_is_tensor[i] else arguments[i] for i in range(n))
-    retval = f(*f_args)
+    f_args = tuple(
+        tensor_args[tensor_args_idx[i]] if arg_is_tensor[i] else a
+        for i, a in enumerate(args))
+    f_kwargs = {
+        k: tensor_args[tensor_args_idx[k]] if kwarg_is_tensor[k] else kwargs[k]
+        for i, k in enumerate(kwarg_keys)
+    }
+    retval = f(*f_args, **f_kwargs)
     return 1 if use_dummy_return else retval
 
-  return script_ops.py_func(
-      f_wrapper, tuple(arguments[i] for i in range(n) if arg_is_tensor[i]),
-      dtypes.int64 if use_dummy_return else return_dtypes)
+  return script_ops.py_func(f_wrapper, tensor_args, dtypes.int64
+                            if use_dummy_return else return_dtypes)
diff --git a/tensorflow/contrib/py2tf/utils/py_func_test.py b/tensorflow/contrib/py2tf/utils/py_func_test.py
index 776b5309c6..3b7a35365a 100644
--- a/tensorflow/contrib/py2tf/utils/py_func_test.py
+++ b/tensorflow/contrib/py2tf/utils/py_func_test.py
@@ -32,19 +32,15 @@ class PyFuncTest(test.TestCase):
       return a + b + c
 
     with self.test_session() as sess:
-      tensor_1 = constant_op.constant(1)
-      self.assertEqual(3,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, dtypes.int64,
-                                                (1, tensor_1, 1))))
-      self.assertEqual(3,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, dtypes.int64,
-                                                (1, 1, 1))))
-      self.assertEqual(3,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, dtypes.int64,
-                                                (tensor_1, 1, tensor_1))))
+      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+                                    (1, constant_op.constant(1), 1))
+      self.assertEqual(3, sess.run(result))
+      result = py_func.wrap_py_func(test_fn, dtypes.int64, (1, 1, 1))
+      self.assertEqual(3, sess.run(result))
+      result = py_func.wrap_py_func(
+          test_fn, dtypes.int64,
+          (constant_op.constant(1), 1, constant_op.constant(1)))
+      self.assertEqual(3, sess.run(result))
 
   def test_wrap_py_func_complex_args(self):
 
@@ -57,15 +53,34 @@ class PyFuncTest(test.TestCase):
       return a * b.foo
 
     with self.test_session() as sess:
-      self.assertEqual(35,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, dtypes.int64,
-                                                (7, TestClass()))))
-      self.assertEqual(
-          35,
-          sess.run(
-              py_func.wrap_py_func(test_fn, dtypes.int64,
-                                   (constant_op.constant(7), TestClass()))))
+      result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass()))
+      self.assertEqual(35, sess.run(result))
+      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+                                    (constant_op.constant(7), TestClass()))
+      self.assertEqual(35, sess.run(result))
+
+  def test_wrap_py_func_kwargs(self):
+
+    class TestClass(object):
+
+      def __init__(self, foo):
+        self.foo = foo
+
+    def test_fn(a, b, c, d):
+      return a * b.foo + c * d.foo
+
+    with self.test_session() as sess:
+      result = py_func.wrap_py_func(test_fn, dtypes.int64, (7, TestClass(5)), {
+          'c': 11,
+          'd': TestClass(13)
+      })
+      self.assertEqual(178, sess.run(result))
+      result = py_func.wrap_py_func(test_fn, dtypes.int64,
+                                    (constant_op.constant(7), TestClass(5)), {
+                                        'c': constant_op.constant(11),
+                                        'd': TestClass(13)
+                                    })
+      self.assertEqual(178, sess.run(result))
 
   def test_wrap_py_func_dummy_return(self):
 
@@ -75,15 +90,12 @@ class PyFuncTest(test.TestCase):
       side_counter[0] += 1
 
     with self.test_session() as sess:
-      self.assertEqual(1,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, None, (5,), True)))
+      result = py_func.wrap_py_func(test_fn, None, (5,), use_dummy_return=True)
+      self.assertEqual(1, sess.run(result))
       self.assertEqual([1], side_counter)
-      self.assertEqual(1,
-                       sess.run(
-                           py_func.wrap_py_func(test_fn, None,
-                                                (constant_op.constant(5),),
-                                                True)))
+      result = py_func.wrap_py_func(
+          test_fn, None, (constant_op.constant(5),), use_dummy_return=True)
+      self.assertEqual(1, sess.run(result))
       self.assertEqual([2], side_counter)
 
 
-- 
GitLab


From 516c2254c3a4042e08652ca3bfd3a288de2f4171 Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Thu, 22 Mar 2018 15:18:25 -0700
Subject: [PATCH 382/960] Explicitly disallow returning variables from a
 tfe.py_func.

PiperOrigin-RevId: 190134382
---
 tensorflow/python/kernel_tests/py_func_test.py | 13 +++++++++++++
 tensorflow/python/ops/script_ops.py            | 16 +++++++++++++---
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/py_func_test.py b/tensorflow/python/kernel_tests/py_func_test.py
index 36142801d6..5b508b7c0e 100644
--- a/tensorflow/python/kernel_tests/py_func_test.py
+++ b/tensorflow/python/kernel_tests/py_func_test.py
@@ -35,6 +35,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import script_ops
 from tensorflow.python.platform import test
 
@@ -480,6 +481,18 @@ class PyFuncTest(test.TestCase):
 
       self._testExceptionHandling(WeirdError, errors.UnknownError, eager=True)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testEagerReturningVariableRaisesError(self):
+    def return_variable():
+      variable = resource_variable_ops.ResourceVariable(0.0)
+      return variable
+
+    with self.assertRaisesRegexp(errors.UnknownError,
+                                 "Attempting to return a variable"):
+      output = script_ops.eager_py_func(
+          return_variable, inp=[], Tout=dtypes.float32)
+      self.evaluate(output)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/script_ops.py b/tensorflow/python/ops/script_ops.py
index fb59bbba5e..1b4111bca6 100644
--- a/tensorflow/python/ops/script_ops.py
+++ b/tensorflow/python/ops/script_ops.py
@@ -36,6 +36,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_script_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
@@ -54,6 +55,16 @@ class EagerFunc(object):
     self._func = func
     self._out_dtypes = Tout
 
+  def _convert(self, value, dtype):
+    if isinstance(value, resource_variable_ops.ResourceVariable):
+      raise RuntimeError(
+          "Attempting to return a variable from an eagerly executed py_func. "
+          "Only numeric data structures like Tensors or NumPy arrays should "
+          "be returned; to return the value of a variable, make sure to obtain "
+          "the Tensor backing it by calling `.read_value()` on the variable in "
+          "question: %s" % value)
+    return ops.convert_to_tensor(value, dtype=dtype)
+
   def __call__(self, on_gpu, args):
     """Passes `args` to `self._func`, which is executed eagerly."""
     with context.eager_mode():
@@ -61,14 +72,13 @@ class EagerFunc(object):
       maybe_copy_to_gpu = lambda x: x if not on_gpu else x.gpu()
       if isinstance(ret, (tuple, list)):
         return [
-            maybe_copy_to_gpu(ops.convert_to_tensor(x, dtype=dtype))
+            maybe_copy_to_gpu(self._convert(x, dtype=dtype))
             for (x, dtype) in zip(ret, self._out_dtypes)
         ]
       elif ret is None:
         return ret
       else:
-        return maybe_copy_to_gpu(
-            ops.convert_to_tensor(ret, dtype=self._out_dtypes[0]))
+        return maybe_copy_to_gpu(self._convert(ret, dtype=self._out_dtypes[0]))
 
 
 class FuncRegistry(object):
-- 
GitLab


From b22aad844669d3d1bd6ffe10fbef3a9a1429e0ad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 15:19:02 -0700
Subject: [PATCH 383/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 190134484
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 803 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 579 +++++++++++++
 2 files changed, 1382 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 99bdc8ccdc..b41826d6eb 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -11610,6 +11610,142 @@ op {
     }
   }
 }
+op {
+  name: "Conv2D"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
+op {
+  name: "Conv2DBackpropFilter"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "filter_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+}
 op {
   name: "Conv2DBackpropFilter"
   input_arg {
@@ -11634,6 +11770,7 @@ op {
     allowed_values {
       list {
         type: DT_HALF
+        type: DT_BFLOAT16
         type: DT_FLOAT
       }
     }
@@ -11672,6 +11809,18 @@ op {
       }
     }
   }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
 }
 op {
   name: "Conv2DBackpropFilter"
@@ -11699,6 +11848,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -11888,6 +12038,83 @@ op {
     }
   }
 }
+op {
+  name: "Conv2DBackpropInput"
+  input_arg {
+    name: "input_sizes"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "filter"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "out_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "strides"
+    type: "list(int)"
+  }
+  attr {
+    name: "use_cudnn_on_gpu"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  attr {
+    name: "padding"
+    type: "string"
+    allowed_values {
+      list {
+        s: "SAME"
+        s: "VALID"
+      }
+    }
+  }
+  attr {
+    name: "data_format"
+    type: "string"
+    default_value {
+      s: "NHWC"
+    }
+    allowed_values {
+      list {
+        s: "NHWC"
+        s: "NCHW"
+      }
+    }
+  }
+  attr {
+    name: "dilations"
+    type: "list(int)"
+    default_value {
+      list {
+        i: 1
+        i: 1
+        i: 1
+        i: 1
+      }
+    }
+  }
+}
 op {
   name: "Conv3D"
   input_arg {
@@ -13280,6 +13507,582 @@ op {
     }
   }
 }
+op {
+  name: "CudnnRNN"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_h"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_c"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "reserve_space"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "is_training"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "CudnnRNNBackprop"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_h_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_c_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reserve_space"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_h_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_c_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "params_backprop"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "CudnnRNNCanonicalToParams"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "weights"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  input_arg {
+    name: "biases"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  output_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "num_params"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
+op {
+  name: "CudnnRNNParamsSize"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "params_size"
+    type_attr: "S"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
+op {
+  name: "CudnnRNNParamsToCanonical"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "weights"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  output_arg {
+    name: "biases"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "num_params"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
 op {
   name: "Cumprod"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 02e52b07c4..af2c563489 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -4814,6 +4814,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -4890,6 +4891,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -4966,6 +4968,7 @@ op {
         type: DT_HALF
         type: DT_BFLOAT16
         type: DT_FLOAT
+        type: DT_DOUBLE
       }
     }
   }
@@ -5721,6 +5724,582 @@ op {
     }
   }
 }
+op {
+  name: "CudnnRNN"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_h"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_c"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "reserve_space"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "is_training"
+    type: "bool"
+    default_value {
+      b: true
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "CudnnRNNBackprop"
+  input_arg {
+    name: "input"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "input_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_h"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_c"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_h_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "output_c_backprop"
+    type_attr: "T"
+  }
+  input_arg {
+    name: "reserve_space"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_h_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "input_c_backprop"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "params_backprop"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "CudnnRNNCanonicalToParams"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "weights"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  input_arg {
+    name: "biases"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  output_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "num_params"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
+op {
+  name: "CudnnRNNParamsSize"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "params_size"
+    type_attr: "S"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "S"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
+op {
+  name: "CudnnRNNParamsToCanonical"
+  input_arg {
+    name: "num_layers"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "num_units"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input_size"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "params"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "weights"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  output_arg {
+    name: "biases"
+    type_attr: "T"
+    number_attr: "num_params"
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_FLOAT
+        type: DT_DOUBLE
+      }
+    }
+  }
+  attr {
+    name: "num_params"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "rnn_mode"
+    type: "string"
+    default_value {
+      s: "lstm"
+    }
+    allowed_values {
+      list {
+        s: "rnn_relu"
+        s: "rnn_tanh"
+        s: "lstm"
+        s: "gru"
+      }
+    }
+  }
+  attr {
+    name: "input_mode"
+    type: "string"
+    default_value {
+      s: "linear_input"
+    }
+    allowed_values {
+      list {
+        s: "linear_input"
+        s: "skip_input"
+        s: "auto_select"
+      }
+    }
+  }
+  attr {
+    name: "direction"
+    type: "string"
+    default_value {
+      s: "unidirectional"
+    }
+    allowed_values {
+      list {
+        s: "unidirectional"
+        s: "bidirectional"
+      }
+    }
+  }
+  attr {
+    name: "dropout"
+    type: "float"
+    default_value {
+      f: 0
+    }
+  }
+  attr {
+    name: "seed"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+  attr {
+    name: "seed2"
+    type: "int"
+    default_value {
+      i: 0
+    }
+  }
+}
 op {
   name: "Cumprod"
   input_arg {
-- 
GitLab


From 0bf45ac29829d30ffdb79f9d8e2df2bc974a316b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 15:32:04 -0700
Subject: [PATCH 384/960] Disable the test
 tensorflow/core/grappler/optimizers:memory_optimizer_test for cuda_on_cpu

PiperOrigin-RevId: 190136546
---
 tensorflow/core/grappler/optimizers/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index cfb698969c..bd9608b369 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -425,6 +425,7 @@ cc_library(
 tf_cuda_cc_test(
     name = "memory_optimizer_test",
     srcs = ["memory_optimizer_test.cc"],
+    tags = ["no_cuda_on_cpu_tap"],  # Do not re-enable again without actually testing.
     deps = [
         ":memory_optimizer",
         "//tensorflow/cc:cc_ops",
-- 
GitLab


From 3870833c52f02298486929f7196558c3ae0da0eb Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Thu, 22 Mar 2018 15:36:27 -0700
Subject: [PATCH 385/960] Instead of depending on ctest to be in PATH, directly
 reference the binary.

PiperOrigin-RevId: 190137278
---
 tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
index b537192a94..97829892b1 100644
--- a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
+++ b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
@@ -28,6 +28,9 @@ IF DEFINED TF_NIGHTLY (ECHO TF_NIGHTLY is set to %TF_NIGHTLY%) ELSE (SET TF_NIGH
 :: Set pip binary location. Do not override if it is set already.
 IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Program Files\Anaconda3\Scripts\pip.exe")
 
+:: Set ctest binary location.
+IF DEFINED CTEST_EXE (ECHO CTEST_EXE is set to %CTEST_EXE%) ELSE (SET CTEST_EXE="C:\Program Files\cmake\bin\ctest.exe")
+
 :: Run the CMAKE build to build the pip package.
 CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\gpu\cmake\run_build.bat
 if %errorlevel% neq 0 exit /b %errorlevel%
@@ -47,4 +50,4 @@ if %errorlevel% neq 0 exit /b %errorlevel%
 
 :: Run all python tests if the installation succeeded.
 echo Running tests...
-ctest -C Release --output-on-failure --jobs 1
+%CTEST_EXE% -C Release --output-on-failure --jobs 1
-- 
GitLab


From 3ab14ecc449c906d85538d7b35700e9c1bb37fcc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 15:46:54 -0700
Subject: [PATCH 386/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 190138848

---
 tensorflow/go/op/wrappers.go | 4592 +++++++++++++++++-----------------
 1 file changed, 2296 insertions(+), 2296 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 92370c4f95..5ddd32ed48 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -724,6 +724,371 @@ func SpaceToDepth(scope *Scope, input tf.Output, block_size int64, optional ...S
 	return op.Output(0)
 }
 
+// SpaceToBatch for 4-D tensors of type T.
+//
+// This is a legacy version of the more general SpaceToBatchND.
+//
+// Zero-pads and then rearranges (permutes) blocks of spatial data into batch.
+// More specifically, this op outputs a copy of the input tensor where values from
+// the `height` and `width` dimensions are moved to the `batch` dimension. After
+// the zero-padding, both `height` and `width` of the input must be divisible by the
+// block size.
+//
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, depth]`.
+//	paddings: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
+//   the padding of the input with zeros across the spatial dimensions as follows:
+//
+//       paddings = [[pad_top, pad_bottom], [pad_left, pad_right]]
+//
+//   The effective spatial dimensions of the zero-padded input tensor will be:
+//
+//       height_pad = pad_top + height + pad_bottom
+//       width_pad = pad_left + width + pad_right
+//
+// The attr `block_size` must be greater than one. It indicates the block size.
+//
+//   * Non-overlapping blocks of size `block_size x block size` in the height and
+//     width dimensions are rearranged into the batch dimension at each location.
+//   * The batch of the output tensor is `batch * block_size * block_size`.
+//   * Both height_pad and width_pad must be divisible by block_size.
+//
+// The shape of the output will be:
+//
+//     [batch*block_size*block_size, height_pad/block_size, width_pad/block_size,
+//      depth]
+//
+// Some examples:
+//
+// (1) For the following input of shape `[1, 2, 2, 1]` and block_size of 2:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// The output tensor has shape `[4, 1, 1, 1]` and value:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// (2) For the following input of shape `[1, 2, 2, 3]` and block_size of 2:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// The output tensor has shape `[4, 1, 1, 3]` and value:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// (3) For the following input of shape `[1, 4, 4, 1]` and block_size of 2:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]],
+//       [[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+//
+// The output tensor has shape `[4, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// (4) For the following input of shape `[2, 2, 4, 1]` and block_size of 2:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+//
+// The output tensor has shape `[8, 1, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [3]]], [[[9], [11]]], [[[2], [4]]], [[[10], [12]]],
+//      [[[5], [7]]], [[[13], [15]]], [[[6], [8]]], [[[14], [16]]]]
+// ```
+//
+// Among others, this operation is useful for reducing atrous convolution into
+// regular convolution.
+//
+func SpaceToBatch(scope *Scope, input tf.Output, paddings tf.Output, block_size int64) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"block_size": block_size}
+	opspec := tf.OpSpec{
+		Type: "SpaceToBatch",
+		Input: []tf.Input{
+			input, paddings,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// SpaceToBatch for N-D tensors of type T.
+//
+// This operation divides "spatial" dimensions `[1, ..., M]` of the input into a
+// grid of blocks of shape `block_shape`, and interleaves these blocks with the
+// "batch" dimension (0) such that in the output, the spatial dimensions
+// `[1, ..., M]` correspond to the position within the grid, and the batch
+// dimension combines both the position within a spatial block and the original
+// batch position.  Prior to division into blocks, the spatial dimensions of the
+// input are optionally zero padded according to `paddings`.  See below for a
+// precise description.
+//
+// Arguments:
+//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+// where spatial_shape has `M` dimensions.
+//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
+//	paddings: 2-D with shape `[M, 2]`, all values must be >= 0.
+//   `paddings[i] = [pad_start, pad_end]` specifies the padding for input dimension
+//   `i + 1`, which corresponds to spatial dimension `i`.  It is required that
+//   `block_shape[i]` divides `input_shape[i + 1] + pad_start + pad_end`.
+//
+// This operation is equivalent to the following steps:
+//
+// 1. Zero-pad the start and end of dimensions `[1, ..., M]` of the
+//    input according to `paddings` to produce `padded` of shape `padded_shape`.
+//
+// 2. Reshape `padded` to `reshaped_padded` of shape:
+//
+//      [batch] +
+//      [padded_shape[1] / block_shape[0],
+//        block_shape[0],
+//       ...,
+//       padded_shape[M] / block_shape[M-1],
+//       block_shape[M-1]] +
+//      remaining_shape
+//
+// 3. Permute dimensions of `reshaped_padded` to produce
+//    `permuted_reshaped_padded` of shape:
+//
+//      block_shape +
+//      [batch] +
+//      [padded_shape[1] / block_shape[0],
+//       ...,
+//       padded_shape[M] / block_shape[M-1]] +
+//      remaining_shape
+//
+// 4. Reshape `permuted_reshaped_padded` to flatten `block_shape` into the batch
+//    dimension, producing an output tensor of shape:
+//
+//      [batch * prod(block_shape)] +
+//      [padded_shape[1] / block_shape[0],
+//       ...,
+//       padded_shape[M] / block_shape[M-1]] +
+//      remaining_shape
+//
+// Some examples:
+//
+// (1) For the following input of shape `[1, 2, 2, 1]`, `block_shape = [2, 2]`, and
+//     `paddings = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// The output tensor has shape `[4, 1, 1, 1]` and value:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// (2) For the following input of shape `[1, 2, 2, 3]`, `block_shape = [2, 2]`, and
+//     `paddings = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// The output tensor has shape `[4, 1, 1, 3]` and value:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// (3) For the following input of shape `[1, 4, 4, 1]`, `block_shape = [2, 2]`, and
+//     `paddings = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]],
+//       [[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+//
+// The output tensor has shape `[4, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// (4) For the following input of shape `[2, 2, 4, 1]`, block_shape = `[2, 2]`, and
+//     paddings = `[[0, 0], [2, 0]]`:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+//
+// The output tensor has shape `[8, 1, 3, 1]` and value:
+//
+// ```
+// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+//      [[[0], [2], [4]]], [[[0], [10], [12]]],
+//      [[[0], [5], [7]]], [[[0], [13], [15]]],
+//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
+// ```
+//
+// Among others, this operation is useful for reducing atrous convolution into
+// regular convolution.
+func SpaceToBatchND(scope *Scope, input tf.Output, block_shape tf.Output, paddings tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SpaceToBatchND",
+		Input: []tf.Input{
+			input, block_shape, paddings,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ListDiffAttr is an optional argument to ListDiff.
+type ListDiffAttr func(optionalAttr)
+
+// ListDiffOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func ListDiffOutIdx(value tf.DataType) ListDiffAttr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Computes the difference between two lists of numbers or strings.
+//
+// Given a list `x` and a list `y`, this operation returns a list `out` that
+// represents all values that are in `x` but not in `y`. The returned list `out`
+// is sorted in the same order that the numbers appear in `x` (duplicates are
+// preserved). This operation also returns a list `idx` that represents the
+// position of each `out` element in `x`. In other words:
+//
+// `out[i] = x[idx[i]] for i in [0, 1, ..., len(out) - 1]`
+//
+// For example, given this input:
+//
+// ```
+// x = [1, 2, 3, 4, 5, 6]
+// y = [1, 3, 5]
+// ```
+//
+// This operation would return:
+//
+// ```
+// out ==> [2, 4, 6]
+// idx ==> [1, 3, 5]
+// ```
+//
+// Arguments:
+//	x: 1-D. Values to keep.
+//	y: 1-D. Values to remove.
+//
+// Returns 1-D. Values present in `x` but not in `y`.1-D. Positions of `x` values preserved in `out`.
+func ListDiff(scope *Scope, x tf.Output, y tf.Output, optional ...ListDiffAttr) (out tf.Output, idx tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ListDiff",
+		Input: []tf.Input{
+			x, y,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Inserts a dimension of 1 into a tensor's shape.
+//
+// Given a tensor `input`, this operation inserts a dimension of 1 at the
+// dimension index `axis` of `input`'s shape. The dimension index `axis` starts at
+// zero; if you specify a negative number for `axis` it is counted backward from
+// the end.
+//
+// This operation is useful if you want to add a batch dimension to a single
+// element. For example, if you have a single image of shape `[height, width,
+// channels]`, you can make it a batch of 1 image with `expand_dims(image, 0)`,
+// which will make the shape `[1, height, width, channels]`.
+//
+// Other examples:
+//
+// ```
+// # 't' is a tensor of shape [2]
+// shape(expand_dims(t, 0)) ==> [1, 2]
+// shape(expand_dims(t, 1)) ==> [2, 1]
+// shape(expand_dims(t, -1)) ==> [2, 1]
+//
+// # 't2' is a tensor of shape [2, 3, 5]
+// shape(expand_dims(t2, 0)) ==> [1, 2, 3, 5]
+// shape(expand_dims(t2, 2)) ==> [2, 3, 1, 5]
+// shape(expand_dims(t2, 3)) ==> [2, 3, 5, 1]
+// ```
+//
+// This operation requires that:
+//
+// `-1-input.dims() <= dim <= input.dims()`
+//
+// This operation is related to `squeeze()`, which removes dimensions of
+// size 1.
+//
+// Arguments:
+//
+//	axis: 0-D (scalar). Specifies the dimension index at which to
+// expand the shape of `input`. Must be in the range
+// `[-rank(input) - 1, rank(input)]`.
+//
+// Returns Contains the same data as `input`, but its shape has an additional
+// dimension of size 1 added.
+func ExpandDims(scope *Scope, input tf.Output, axis tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ExpandDims",
+		Input: []tf.Input{
+			input, axis,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns (x - y)(x - y) element-wise.
 //
 // *NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
@@ -2637,61 +3002,6 @@ func Rsqrt(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Inserts a dimension of 1 into a tensor's shape.
-//
-// Given a tensor `input`, this operation inserts a dimension of 1 at the
-// dimension index `axis` of `input`'s shape. The dimension index `axis` starts at
-// zero; if you specify a negative number for `axis` it is counted backward from
-// the end.
-//
-// This operation is useful if you want to add a batch dimension to a single
-// element. For example, if you have a single image of shape `[height, width,
-// channels]`, you can make it a batch of 1 image with `expand_dims(image, 0)`,
-// which will make the shape `[1, height, width, channels]`.
-//
-// Other examples:
-//
-// ```
-// # 't' is a tensor of shape [2]
-// shape(expand_dims(t, 0)) ==> [1, 2]
-// shape(expand_dims(t, 1)) ==> [2, 1]
-// shape(expand_dims(t, -1)) ==> [2, 1]
-//
-// # 't2' is a tensor of shape [2, 3, 5]
-// shape(expand_dims(t2, 0)) ==> [1, 2, 3, 5]
-// shape(expand_dims(t2, 2)) ==> [2, 3, 1, 5]
-// shape(expand_dims(t2, 3)) ==> [2, 3, 5, 1]
-// ```
-//
-// This operation requires that:
-//
-// `-1-input.dims() <= dim <= input.dims()`
-//
-// This operation is related to `squeeze()`, which removes dimensions of
-// size 1.
-//
-// Arguments:
-//
-//	axis: 0-D (scalar). Specifies the dimension index at which to
-// expand the shape of `input`. Must be in the range
-// `[-rank(input) - 1, rank(input)]`.
-//
-// Returns Contains the same data as `input`, but its shape has an additional
-// dimension of size 1 added.
-func ExpandDims(scope *Scope, input tf.Output, axis tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ExpandDims",
-		Input: []tf.Input{
-			input, axis,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // MatrixInverseAttr is an optional argument to MatrixInverse.
 type MatrixInverseAttr func(optionalAttr)
 
@@ -8111,20 +8421,131 @@ func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr {
 //
 // The resulting update to ref would look like this:
 //
-//     [1, 11, 3, 10, 9, 6, 7, 12]
+//     [1, 11, 3, 10, 9, 6, 7, 12]
+//
+// See @{tf.scatter_nd} for more details about how to make updates to
+// slices.
+//
+// Arguments:
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of updated
+// values to add to ref.
+//
+// Returns the created operation.
+func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterNdUpdate",
+		Input: []tf.Input{
+			ref, indices, updates,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// SqueezeAttr is an optional argument to Squeeze.
+type SqueezeAttr func(optionalAttr)
+
+// SqueezeAxis sets the optional axis attribute to value.
+//
+// value: If specified, only squeezes the dimensions listed. The dimension
+// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
+// be in the range `[-rank(input), rank(input))`.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func SqueezeAxis(value []int64) SqueezeAttr {
+	return func(m optionalAttr) {
+		m["squeeze_dims"] = value
+	}
+}
+
+// Removes dimensions of size 1 from the shape of a tensor.
+//
+// Given a tensor `input`, this operation returns a tensor of the same type with
+// all dimensions of size 1 removed. If you don't want to remove all size 1
+// dimensions, you can remove specific size 1 dimensions by specifying
+// `axis`.
+//
+// For example:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t)) ==> [2, 3]
+// ```
+//
+// Or, to remove specific size 1 dimensions:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
+// ```
+//
+// Arguments:
+//	input: The `input` to squeeze.
+//
+// Returns Contains the same data as `input`, but has one or more dimensions of
+// size 1 removed.
+func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Squeeze",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
+type ResourceApplyAdadeltaAttr func(optionalAttr)
+
+// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
+//
+// value: If True, updating of the var, accum and update_accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the adadelta scheme.
 //
-// See @{tf.scatter_nd} for more details about how to make updates to
-// slices.
+// accum = rho() * accum + (1 - rho()) * grad.square();
+// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
+// update_accum = rho() * update_accum + (1 - rho()) * update.square();
+// var -= update;
 //
 // Arguments:
-//	ref: A resource handle. Must be from a VarHandleOp.
-//	indices: A Tensor. Must be one of the following types: int32, int64.
-// A tensor of indices into ref.
-//	updates: A Tensor. Must have the same type as ref. A tensor of updated
-// values to add to ref.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	accum_update: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay factor. Must be a scalar.
+//	epsilon: Constant factor. Must be a scalar.
+//	grad: The gradient.
 //
 // Returns the created operation.
-func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
+func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8133,9 +8554,9 @@ func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, upd
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterNdUpdate",
+		Type: "ResourceApplyAdadelta",
 		Input: []tf.Input{
-			ref, indices, updates,
+			var_, accum, accum_update, lr, rho, epsilon, grad,
 		},
 		Attrs: attrs,
 	}
@@ -14338,65 +14759,6 @@ func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes [
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ListDiffAttr is an optional argument to ListDiff.
-type ListDiffAttr func(optionalAttr)
-
-// ListDiffOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func ListDiffOutIdx(value tf.DataType) ListDiffAttr {
-	return func(m optionalAttr) {
-		m["out_idx"] = value
-	}
-}
-
-// Computes the difference between two lists of numbers or strings.
-//
-// Given a list `x` and a list `y`, this operation returns a list `out` that
-// represents all values that are in `x` but not in `y`. The returned list `out`
-// is sorted in the same order that the numbers appear in `x` (duplicates are
-// preserved). This operation also returns a list `idx` that represents the
-// position of each `out` element in `x`. In other words:
-//
-// `out[i] = x[idx[i]] for i in [0, 1, ..., len(out) - 1]`
-//
-// For example, given this input:
-//
-// ```
-// x = [1, 2, 3, 4, 5, 6]
-// y = [1, 3, 5]
-// ```
-//
-// This operation would return:
-//
-// ```
-// out ==> [2, 4, 6]
-// idx ==> [1, 3, 5]
-// ```
-//
-// Arguments:
-//	x: 1-D. Values to keep.
-//	y: 1-D. Values to remove.
-//
-// Returns 1-D. Values present in `x` but not in `y`.1-D. Positions of `x` values preserved in `out`.
-func ListDiff(scope *Scope, x tf.Output, y tf.Output, optional ...ListDiffAttr) (out tf.Output, idx tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ListDiff",
-		Input: []tf.Input{
-			x, y,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // Concatenates quantized tensors along one dimension.
 //
 // Arguments:
@@ -16660,53 +17022,6 @@ func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_uppe
 	return op.Output(0)
 }
 
-// DecodeCompressedAttr is an optional argument to DecodeCompressed.
-type DecodeCompressedAttr func(optionalAttr)
-
-// DecodeCompressedCompressionType sets the optional compression_type attribute to value.
-//
-// value: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-// If not specified, defaults to ""
-func DecodeCompressedCompressionType(value string) DecodeCompressedAttr {
-	return func(m optionalAttr) {
-		m["compression_type"] = value
-	}
-}
-
-// Decompress strings.
-//
-// This op decompresses each element of the `bytes` input `Tensor`, which
-// is assumed to be compressed using the given `compression_type`.
-//
-// The `output` is a string `Tensor` of the same shape as `bytes`,
-// each element containing the decompressed data from the corresponding
-// element in `bytes`.
-//
-// Arguments:
-//	bytes: A Tensor of string which is compressed.
-//
-// Returns A Tensor with the same shape as input `bytes`, uncompressed
-// from bytes.
-func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompressedAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeCompressed",
-		Input: []tf.Input{
-			bytes,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes acos of x element-wise.
 func Acos(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -21788,48 +22103,6 @@ func IteratorToStringHandle(scope *Scope, resource_handle tf.Output) (string_han
 	return op.Output(0)
 }
 
-// ShapeNAttr is an optional argument to ShapeN.
-type ShapeNAttr func(optionalAttr)
-
-// ShapeNOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func ShapeNOutType(value tf.DataType) ShapeNAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Returns shape of tensors.
-//
-// This operation returns N 1-D integer tensors representing shape of `input[i]s`.
-func ShapeN(scope *Scope, input []tf.Output, optional ...ShapeNAttr) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ShapeN",
-		Input: []tf.Input{
-			tf.OutputList(input),
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("ShapeN", err)
-		return
-	}
-	return output
-}
-
 // IteratorFromStringHandleAttr is an optional argument to IteratorFromStringHandle.
 type IteratorFromStringHandleAttr func(optionalAttr)
 
@@ -23524,32 +23797,79 @@ func FusedBatchNormGradV2DataFormat(value string) FusedBatchNormGradV2Attr {
 // If not specified, defaults to true
 func FusedBatchNormGradV2IsTraining(value bool) FusedBatchNormGradV2Attr {
 	return func(m optionalAttr) {
-		m["is_training"] = value
+		m["is_training"] = value
+	}
+}
+
+// Gradient for batch normalization.
+//
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+//
+// Arguments:
+//	y_backprop: A 4D Tensor for the gradient with respect to y.
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
+// mean to be reused in gradient computation. When is_training is
+// False, a 1D Tensor for the population mean to be reused in both
+// 1st and 2nd order gradient computation.
+//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
+// variance (inverted variance in the cuDNN case) to be reused in
+// gradient computation. When is_training is False, a 1D Tensor
+// for the population variance to be reused in both 1st and 2nd
+// order gradient computation.
+//
+// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
+// in FusedBatchNorm.
+func FusedBatchNormGradV2(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradV2Attr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "FusedBatchNormGradV2",
+		Input: []tf.Input{
+			y_backprop, x, scale, reserve_space_1, reserve_space_2,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+}
+
+// DecodeCompressedAttr is an optional argument to DecodeCompressed.
+type DecodeCompressedAttr func(optionalAttr)
+
+// DecodeCompressedCompressionType sets the optional compression_type attribute to value.
+//
+// value: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+// If not specified, defaults to ""
+func DecodeCompressedCompressionType(value string) DecodeCompressedAttr {
+	return func(m optionalAttr) {
+		m["compression_type"] = value
 	}
 }
 
-// Gradient for batch normalization.
+// Decompress strings.
 //
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// This op decompresses each element of the `bytes` input `Tensor`, which
+// is assumed to be compressed using the given `compression_type`.
+//
+// The `output` is a string `Tensor` of the same shape as `bytes`,
+// each element containing the decompressed data from the corresponding
+// element in `bytes`.
 //
 // Arguments:
-//	y_backprop: A 4D Tensor for the gradient with respect to y.
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
-// mean to be reused in gradient computation. When is_training is
-// False, a 1D Tensor for the population mean to be reused in both
-// 1st and 2nd order gradient computation.
-//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
-// variance (inverted variance in the cuDNN case) to be reused in
-// gradient computation. When is_training is False, a 1D Tensor
-// for the population variance to be reused in both 1st and 2nd
-// order gradient computation.
+//	bytes: A Tensor of string which is compressed.
 //
-// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
-// in FusedBatchNorm.
-func FusedBatchNormGradV2(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradV2Attr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
+// Returns A Tensor with the same shape as input `bytes`, uncompressed
+// from bytes.
+func DecodeCompressed(scope *Scope, bytes tf.Output, optional ...DecodeCompressedAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -23558,14 +23878,14 @@ func FusedBatchNormGradV2(scope *Scope, y_backprop tf.Output, x tf.Output, scale
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNormGradV2",
+		Type: "DecodeCompressed",
 		Input: []tf.Input{
-			y_backprop, x, scale, reserve_space_1, reserve_space_2,
+			bytes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0)
 }
 
 // Creates a TensorArray for storing the gradients of values in the given handle.
@@ -24723,302 +25043,88 @@ func StageContainer(value string) StageAttr {
 }
 
 // StageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func StageSharedName(value string) StageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Stage values similar to a lightweight Enqueue.
-//
-// The basic functionality of this Op is similar to a queue with many
-// fewer capabilities and options.  This Op is optimized for performance.
-//
-// Arguments:
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
-// Returns the created operation.
-func Stage(scope *Scope, values []tf.Output, optional ...StageAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Stage",
-		Input: []tf.Input{
-			tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// StagePeekAttr is an optional argument to StagePeek.
-type StagePeekAttr func(optionalAttr)
-
-// StagePeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StagePeekCapacity(value int64) StagePeekAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// StagePeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StagePeekMemoryLimit(value int64) StagePeekAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// StagePeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StagePeekContainer(value string) StagePeekAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StagePeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StagePeekSharedName(value string) StagePeekAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op peeks at the values at the specified index.  If the
-//
-// underlying container does not contain sufficient elements
-// this op will block until it does.   This Op is optimized for
-// performance.
-func StagePeek(scope *Scope, index tf.Output, dtypes []tf.DataType, optional ...StagePeekAttr) (values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "StagePeek",
-		Input: []tf.Input{
-			index,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("StagePeek", err)
-		return
-	}
-	return values
-}
-
-// MapStageAttr is an optional argument to MapStage.
-type MapStageAttr func(optionalAttr)
-
-// MapStageCapacity sets the optional capacity attribute to value.
-//
-// value: Maximum number of elements in the Staging Area. If > 0, inserts
-// on the container will block when the capacity is reached.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapStageCapacity(value int64) MapStageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapStageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapStageMemoryLimit(value int64) MapStageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapStageContainer sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container. Otherwise,
-// a default container is used.
-// If not specified, defaults to ""
-func MapStageContainer(value string) MapStageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapStageSharedName sets the optional shared_name attribute to value.
-//
-// value: It is necessary to match this name to the matching Unstage Op.
-// If not specified, defaults to ""
-func MapStageSharedName(value string) MapStageAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Stage (key, values) in the underlying container which behaves like a hashtable.
-//
-// Arguments:
-//	key: int64
-//
-//	values: a list of tensors
-// dtypes A list of data types that inserted values should adhere to.
-//
-//
-// Returns the created operation.
-func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...MapStageAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MapStage",
-		Input: []tf.Input{
-			key, indices, tf.OutputList(values),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// MapUnstageAttr is an optional argument to MapUnstage.
-type MapUnstageAttr func(optionalAttr)
-
-// MapUnstageCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageCapacity(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapUnstageContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapUnstageContainer(value string) MapUnstageAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapUnstageSharedName sets the optional shared_name attribute to value.
+//
+// value: It is necessary to match this name to the matching Unstage Op.
 // If not specified, defaults to ""
-func MapUnstageSharedName(value string) MapUnstageAttr {
+func StageSharedName(value string) StageAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op removes and returns the values associated with the key
+// Stage values similar to a lightweight Enqueue.
 //
-// from the underlying container.   If the underlying container
-// does not contain this key, the op will block until it does.
-func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
+// The basic functionality of this Op is similar to a queue with many
+// fewer capabilities and options.  This Op is optimized for performance.
+//
+// Arguments:
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+// Returns the created operation.
+func Stage(scope *Scope, values []tf.Output, optional ...StageAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapUnstage",
+		Type: "Stage",
 		Input: []tf.Input{
-			key, indices,
+			tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstage", err)
-		return
-	}
-	return values
+	return scope.AddOperation(opspec)
 }
 
-// MapSizeAttr is an optional argument to MapSize.
-type MapSizeAttr func(optionalAttr)
+// StagePeekAttr is an optional argument to StagePeek.
+type StagePeekAttr func(optionalAttr)
 
-// MapSizeCapacity sets the optional capacity attribute to value.
+// StagePeekCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapSizeCapacity(value int64) MapSizeAttr {
+func StagePeekCapacity(value int64) StagePeekAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// StagePeekMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapSizeMemoryLimit(value int64) MapSizeAttr {
+func StagePeekMemoryLimit(value int64) StagePeekAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// MapSizeContainer sets the optional container attribute to value.
+// StagePeekContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func MapSizeContainer(value string) MapSizeAttr {
+func StagePeekContainer(value string) StagePeekAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// MapSizeSharedName sets the optional shared_name attribute to value.
+// StagePeekSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func MapSizeSharedName(value string) MapSizeAttr {
+func StagePeekSharedName(value string) StagePeekAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op returns the number of elements in the underlying container.
-func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
+// Op peeks at the values at the specified index.  If the
+//
+// underlying container does not contain sufficient elements
+// this op will block until it does.   This Op is optimized for
+// performance.
+func StagePeek(scope *Scope, index tf.Output, dtypes []tf.DataType, optional ...StagePeekAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25027,55 +25133,83 @@ func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapSize",
-
+		Type: "StagePeek",
+		Input: []tf.Input{
+			index,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("StagePeek", err)
+		return
+	}
+	return values
 }
 
-// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize.
-type MapIncompleteSizeAttr func(optionalAttr)
+// MapStageAttr is an optional argument to MapStage.
+type MapStageAttr func(optionalAttr)
 
-// MapIncompleteSizeCapacity sets the optional capacity attribute to value.
+// MapStageCapacity sets the optional capacity attribute to value.
+//
+// value: Maximum number of elements in the Staging Area. If > 0, inserts
+// on the container will block when the capacity is reached.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr {
+func MapStageCapacity(value int64) MapStageAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
+// MapStageMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr {
+func MapStageMemoryLimit(value int64) MapStageAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// MapIncompleteSizeContainer sets the optional container attribute to value.
+// MapStageContainer sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container. Otherwise,
+// a default container is used.
 // If not specified, defaults to ""
-func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr {
+func MapStageContainer(value string) MapStageAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// MapIncompleteSizeSharedName sets the optional shared_name attribute to value.
+// MapStageSharedName sets the optional shared_name attribute to value.
+//
+// value: It is necessary to match this name to the matching Unstage Op.
 // If not specified, defaults to ""
-func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr {
+func MapStageSharedName(value string) MapStageAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op returns the number of incomplete elements in the underlying container.
-func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) {
+// Stage (key, values) in the underlying container which behaves like a hashtable.
+//
+// Arguments:
+//	key: int64
+//
+//	values: a list of tensors
+// dtypes A list of data types that inserted values should adhere to.
+//
+//
+// Returns the created operation.
+func MapStage(scope *Scope, key tf.Output, indices tf.Output, values []tf.Output, dtypes []tf.DataType, optional ...MapStageAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25084,48 +25218,49 @@ func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncomp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapIncompleteSize",
-
+		Type: "MapStage",
+		Input: []tf.Input{
+			key, indices, tf.OutputList(values),
+		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// OrderedMapUnstageAttr is an optional argument to OrderedMapUnstage.
-type OrderedMapUnstageAttr func(optionalAttr)
+// MapUnstageAttr is an optional argument to MapUnstage.
+type MapUnstageAttr func(optionalAttr)
 
-// OrderedMapUnstageCapacity sets the optional capacity attribute to value.
+// MapUnstageCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapUnstageCapacity(value int64) OrderedMapUnstageAttr {
+func MapUnstageCapacity(value int64) MapUnstageAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// OrderedMapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// MapUnstageMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapUnstageMemoryLimit(value int64) OrderedMapUnstageAttr {
+func MapUnstageMemoryLimit(value int64) MapUnstageAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// OrderedMapUnstageContainer sets the optional container attribute to value.
+// MapUnstageContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func OrderedMapUnstageContainer(value string) OrderedMapUnstageAttr {
+func MapUnstageContainer(value string) MapUnstageAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// OrderedMapUnstageSharedName sets the optional shared_name attribute to value.
+// MapUnstageSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func OrderedMapUnstageSharedName(value string) OrderedMapUnstageAttr {
+func MapUnstageSharedName(value string) MapUnstageAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
@@ -25135,7 +25270,7 @@ func OrderedMapUnstageSharedName(value string) OrderedMapUnstageAttr {
 //
 // from the underlying container.   If the underlying container
 // does not contain this key, the op will block until it does.
-func OrderedMapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageAttr) (values []tf.Output) {
+func MapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25144,7 +25279,7 @@ func OrderedMapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapUnstage",
+		Type: "MapUnstage",
 		Input: []tf.Input{
 			key, indices,
 		},
@@ -25157,381 +25292,269 @@ func OrderedMapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []
 	var idx int
 	var err error
 	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("OrderedMapUnstage", err)
+		scope.UpdateErr("MapUnstage", err)
 		return
 	}
 	return values
 }
 
-// OrderedMapSizeAttr is an optional argument to OrderedMapSize.
-type OrderedMapSizeAttr func(optionalAttr)
+// MapSizeAttr is an optional argument to MapSize.
+type MapSizeAttr func(optionalAttr)
 
-// OrderedMapSizeCapacity sets the optional capacity attribute to value.
+// MapSizeCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapSizeCapacity(value int64) OrderedMapSizeAttr {
+func MapSizeCapacity(value int64) MapSizeAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// OrderedMapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// MapSizeMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapSizeMemoryLimit(value int64) OrderedMapSizeAttr {
+func MapSizeMemoryLimit(value int64) MapSizeAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// OrderedMapSizeContainer sets the optional container attribute to value.
+// MapSizeContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func OrderedMapSizeContainer(value string) OrderedMapSizeAttr {
+func MapSizeContainer(value string) MapSizeAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
-	}
-}
-
-// OrderedMapSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapSizeSharedName(value string) OrderedMapSizeAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op returns the number of elements in the underlying container.
-func OrderedMapSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapSizeAttr) (size tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "OrderedMapSize",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// CTCLossAttr is an optional argument to CTCLoss.
-type CTCLossAttr func(optionalAttr)
-
-// CTCLossPreprocessCollapseRepeated sets the optional preprocess_collapse_repeated attribute to value.
-//
-// value: Scalar, if true then repeated labels are
-// collapsed prior to the CTC calculation.
-// If not specified, defaults to false
-func CTCLossPreprocessCollapseRepeated(value bool) CTCLossAttr {
-	return func(m optionalAttr) {
-		m["preprocess_collapse_repeated"] = value
-	}
-}
-
-// CTCLossCtcMergeRepeated sets the optional ctc_merge_repeated attribute to value.
-//
-// value: Scalar.  If set to false, *during* CTC calculation
-// repeated non-blank labels will not be merged and are interpreted as
-// individual labels.  This is a simplified version of CTC.
-// If not specified, defaults to true
-func CTCLossCtcMergeRepeated(value bool) CTCLossAttr {
-	return func(m optionalAttr) {
-		m["ctc_merge_repeated"] = value
-	}
-}
-
-// CTCLossIgnoreLongerOutputsThanInputs sets the optional ignore_longer_outputs_than_inputs attribute to value.
-//
-// value: Scalar. If set to true, during CTC
-// calculation, items that have longer output sequences than input sequences
-// are skipped: they don't contribute to the loss term and have zero-gradient.
-// If not specified, defaults to false
-func CTCLossIgnoreLongerOutputsThanInputs(value bool) CTCLossAttr {
-	return func(m optionalAttr) {
-		m["ignore_longer_outputs_than_inputs"] = value
-	}
-}
-
-// Calculates the CTC Loss (log probability) for each batch entry.  Also calculates
-//
-// the gradient.  This class performs the softmax operation for you, so inputs
-// should be e.g. linear projections of outputs by an LSTM.
-//
-// Arguments:
-//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
-//	labels_indices: The indices of a `SparseTensor<int32, 2>`.
-// `labels_indices(i, :) == [b, t]` means `labels_values(i)` stores the id for
-// `(batch b, time t)`.
-//	labels_values: The values (labels) associated with the given batch and time.
-//	sequence_length: A vector containing sequence lengths (batch).
-//
-// Returns A vector (batch) containing log-probabilities.The gradient of `loss`.  3-D, shape:
-// `(max_time x batch_size x num_classes)`.
-func CTCLoss(scope *Scope, inputs tf.Output, labels_indices tf.Output, labels_values tf.Output, sequence_length tf.Output, optional ...CTCLossAttr) (loss tf.Output, gradient tf.Output) {
+	}
+}
+
+// MapSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapSizeSharedName(value string) MapSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of elements in the underlying container.
+func MapSize(scope *Scope, dtypes []tf.DataType, optional ...MapSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "CTCLoss",
-		Input: []tf.Input{
-			inputs, labels_indices, labels_values, sequence_length,
-		},
+		Type: "MapSize",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// CTCGreedyDecoderAttr is an optional argument to CTCGreedyDecoder.
-type CTCGreedyDecoderAttr func(optionalAttr)
+// MapIncompleteSizeAttr is an optional argument to MapIncompleteSize.
+type MapIncompleteSizeAttr func(optionalAttr)
 
-// CTCGreedyDecoderMergeRepeated sets the optional merge_repeated attribute to value.
+// MapIncompleteSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: If True, merge repeated classes in output.
-// If not specified, defaults to false
-func CTCGreedyDecoderMergeRepeated(value bool) CTCGreedyDecoderAttr {
+// REQUIRES: value >= 0
+func MapIncompleteSizeCapacity(value int64) MapIncompleteSizeAttr {
 	return func(m optionalAttr) {
-		m["merge_repeated"] = value
+		m["capacity"] = value
 	}
 }
 
-// Performs greedy decoding on the logits given in inputs.
-//
-// A note about the attribute merge_repeated: if enabled, when
-// consecutive logits' maximum indices are the same, only the first of
-// these is emitted.  Labeling the blank '*', the sequence "A B B * B B"
-// becomes "A B B" if merge_repeated = True and "A B B B B" if
-// merge_repeated = False.
-//
-// Regardless of the value of merge_repeated, if the maximum index of a given
-// time and batch corresponds to the blank, index `(num_classes - 1)`, no new
-// element is emitted.
-//
-// Arguments:
-//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
-//	sequence_length: A vector containing sequence lengths, size `(batch_size)`.
+// MapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Returns Indices matrix, size `(total_decoded_outputs x 2)`,
-// of a `SparseTensor<int64, 2>`.  The rows store: [batch, time].Values vector, size: `(total_decoded_outputs)`,
-// of a `SparseTensor<int64, 2>`.  The vector stores the decoded classes.Shape vector, size `(2)`, of the decoded SparseTensor.
-// Values are: `[batch_size, max_decoded_length]`.Matrix, size `(batch_size x 1)`, containing sequence
-// log-probabilities.
-func CTCGreedyDecoder(scope *Scope, inputs tf.Output, sequence_length tf.Output, optional ...CTCGreedyDecoderAttr) (decoded_indices tf.Output, decoded_values tf.Output, decoded_shape tf.Output, log_probability tf.Output) {
+// REQUIRES: value >= 0
+func MapIncompleteSizeMemoryLimit(value int64) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapIncompleteSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapIncompleteSizeContainer(value string) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapIncompleteSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapIncompleteSizeSharedName(value string) MapIncompleteSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op returns the number of incomplete elements in the underlying container.
+func MapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...MapIncompleteSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "CTCGreedyDecoder",
-		Input: []tf.Input{
-			inputs, sequence_length,
-		},
+		Type: "MapIncompleteSize",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+	return op.Output(0)
 }
 
-// Forwards `data` to the output port determined by `pred`.
-//
-// If `pred` is true, the `data` input is forwarded to `output_true`. Otherwise,
-// the data goes to `output_false`.
-//
-// See also `RefSwitch` and `Merge`.
-//
-// Arguments:
-//	data: The tensor to be forwarded to the appropriate output.
-//	pred: A scalar that specifies which output port will receive data.
+// OrderedMapUnstageAttr is an optional argument to OrderedMapUnstage.
+type OrderedMapUnstageAttr func(optionalAttr)
+
+// OrderedMapUnstageCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// Returns If `pred` is false, data will be forwarded to this output.If `pred` is true, data will be forwarded to this output.
-func Switch(scope *Scope, data tf.Output, pred tf.Output) (output_false tf.Output, output_true tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Switch",
-		Input: []tf.Input{
-			data, pred,
-		},
+// REQUIRES: value >= 0
+func OrderedMapUnstageCapacity(value int64) OrderedMapUnstageAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
 }
 
-// Add all input tensors element wise.
+// OrderedMapUnstageMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	inputs: Must all be the same size and shape.
-func AddN(scope *Scope, inputs []tf.Output) (sum tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AddN",
-		Input: []tf.Input{
-			tf.OutputList(inputs),
-		},
+// REQUIRES: value >= 0
+func OrderedMapUnstageMemoryLimit(value int64) OrderedMapUnstageAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// EnterAttr is an optional argument to Enter.
-type EnterAttr func(optionalAttr)
-
-// EnterIsConstant sets the optional is_constant attribute to value.
-//
-// value: If true, the output is constant within the child frame.
-// If not specified, defaults to false
-func EnterIsConstant(value bool) EnterAttr {
+// OrderedMapUnstageContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapUnstageContainer(value string) OrderedMapUnstageAttr {
 	return func(m optionalAttr) {
-		m["is_constant"] = value
+		m["container"] = value
 	}
 }
 
-// EnterParallelIterations sets the optional parallel_iterations attribute to value.
-//
-// value: The number of iterations allowed to run in parallel.
-// If not specified, defaults to 10
-func EnterParallelIterations(value int64) EnterAttr {
+// OrderedMapUnstageSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapUnstageSharedName(value string) OrderedMapUnstageAttr {
 	return func(m optionalAttr) {
-		m["parallel_iterations"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Creates or finds a child frame, and makes `data` available to the child frame.
-//
-// This op is used together with `Exit` to create loops in the graph.
-// The unique `frame_name` is used by the `Executor` to identify frames. If
-// `is_constant` is true, `output` is a constant in the child frame; otherwise
-// it may be changed in the child frame. At most `parallel_iterations` iterations
-// are run in parallel in the child frame.
-//
-// Arguments:
-//	data: The tensor to be made available to the child frame.
-//	frame_name: The name of the child frame.
+// Op removes and returns the values associated with the key
 //
-// Returns The same tensor as `data`.
-func Enter(scope *Scope, data tf.Output, frame_name string, optional ...EnterAttr) (output tf.Output) {
+// from the underlying container.   If the underlying container
+// does not contain this key, the op will block until it does.
+func OrderedMapUnstage(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"frame_name": frame_name}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Enter",
+		Type: "OrderedMapUnstage",
 		Input: []tf.Input{
-			data,
+			key, indices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Produce a string tensor that encodes the state of a Reader.
-//
-// Not all Readers support being serialized, so this can produce an
-// Unimplemented error.
-//
-// Arguments:
-//	reader_handle: Handle to a Reader.
-func ReaderSerializeStateV2(scope *Scope, reader_handle tf.Output) (state tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "ReaderSerializeStateV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("OrderedMapUnstage", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return values
 }
 
-// Exits the current frame to its parent frame.
-//
-// Exit makes its input `data` available to the parent frame.
+// OrderedMapSizeAttr is an optional argument to OrderedMapSize.
+type OrderedMapSizeAttr func(optionalAttr)
+
+// OrderedMapSizeCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	data: The tensor to be made available to the parent frame.
+// REQUIRES: value >= 0
+func OrderedMapSizeCapacity(value int64) OrderedMapSizeAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapSizeMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Returns The same tensor as `data`.
-func Exit(scope *Scope, data tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// REQUIRES: value >= 0
+func OrderedMapSizeMemoryLimit(value int64) OrderedMapSizeAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Exit",
-		Input: []tf.Input{
-			data,
-		},
+}
+
+// OrderedMapSizeContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapSizeContainer(value string) OrderedMapSizeAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapSizeSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapSizeSharedName(value string) OrderedMapSizeAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns a copy of the input tensor.
-func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
+// Op returns the number of elements in the underlying container.
+func OrderedMapSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Snapshot",
-		Input: []tf.Input{
-			input,
-		},
+		Type: "OrderedMapSize",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AbortAttr is an optional argument to Abort.
-type AbortAttr func(optionalAttr)
-
-// AbortErrorMsg sets the optional error_msg attribute to value.
-//
-// value: A string which is the message associated with the exception.
-// If not specified, defaults to ""
-func AbortErrorMsg(value string) AbortAttr {
-	return func(m optionalAttr) {
-		m["error_msg"] = value
-	}
-}
+// ShapeNAttr is an optional argument to ShapeN.
+type ShapeNAttr func(optionalAttr)
 
-// AbortExitWithoutError sets the optional exit_without_error attribute to value.
-// If not specified, defaults to false
-func AbortExitWithoutError(value bool) AbortAttr {
+// ShapeNOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func ShapeNOutType(value tf.DataType) ShapeNAttr {
 	return func(m optionalAttr) {
-		m["exit_without_error"] = value
+		m["out_type"] = value
 	}
 }
 
-// Raise a exception to abort the process when called.
-//
-// If exit_without_error is true, the process will exit normally,
-// otherwise it will exit with a SIGABORT signal.
-//
-// Returns nothing but an exception.
+// Returns shape of tensors.
 //
-// Returns the created operation.
-func Abort(scope *Scope, optional ...AbortAttr) (o *tf.Operation) {
+// This operation returns N 1-D integer tensors representing shape of `input[i]s`.
+func ShapeN(scope *Scope, input []tf.Output, optional ...ShapeNAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -25540,11 +25563,23 @@ func Abort(scope *Scope, optional ...AbortAttr) (o *tf.Operation) {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Abort",
-
+		Type: "ShapeN",
+		Input: []tf.Input{
+			tf.OutputList(input),
+		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("ShapeN", err)
+		return
+	}
+	return output
 }
 
 // UniformCandidateSamplerAttr is an optional argument to UniformCandidateSampler.
@@ -25620,711 +25655,745 @@ func UniformCandidateSampler(scope *Scope, true_classes tf.Output, num_true int6
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// FixedUnigramCandidateSamplerAttr is an optional argument to FixedUnigramCandidateSampler.
-type FixedUnigramCandidateSamplerAttr func(optionalAttr)
+// CTCLossAttr is an optional argument to CTCLoss.
+type CTCLossAttr func(optionalAttr)
 
-// FixedUnigramCandidateSamplerVocabFile sets the optional vocab_file attribute to value.
+// CTCLossPreprocessCollapseRepeated sets the optional preprocess_collapse_repeated attribute to value.
 //
-// value: Each valid line in this file (which should have a CSV-like format)
-// corresponds to a valid word ID. IDs are in sequential order, starting from
-// num_reserved_ids. The last entry in each line is expected to be a value
-// corresponding to the count or relative probability. Exactly one of vocab_file
-// and unigrams needs to be passed to this op.
-// If not specified, defaults to ""
-func FixedUnigramCandidateSamplerVocabFile(value string) FixedUnigramCandidateSamplerAttr {
+// value: Scalar, if true then repeated labels are
+// collapsed prior to the CTC calculation.
+// If not specified, defaults to false
+func CTCLossPreprocessCollapseRepeated(value bool) CTCLossAttr {
 	return func(m optionalAttr) {
-		m["vocab_file"] = value
+		m["preprocess_collapse_repeated"] = value
 	}
 }
 
-// FixedUnigramCandidateSamplerDistortion sets the optional distortion attribute to value.
+// CTCLossCtcMergeRepeated sets the optional ctc_merge_repeated attribute to value.
 //
-// value: The distortion is used to skew the unigram probability distribution.
-// Each weight is first raised to the distortion's power before adding to the
-// internal unigram distribution. As a result, distortion = 1.0 gives regular
-// unigram sampling (as defined by the vocab file), and distortion = 0.0 gives
-// a uniform distribution.
-// If not specified, defaults to 1
-func FixedUnigramCandidateSamplerDistortion(value float32) FixedUnigramCandidateSamplerAttr {
+// value: Scalar.  If set to false, *during* CTC calculation
+// repeated non-blank labels will not be merged and are interpreted as
+// individual labels.  This is a simplified version of CTC.
+// If not specified, defaults to true
+func CTCLossCtcMergeRepeated(value bool) CTCLossAttr {
 	return func(m optionalAttr) {
-		m["distortion"] = value
+		m["ctc_merge_repeated"] = value
 	}
 }
 
-// FixedUnigramCandidateSamplerNumReservedIds sets the optional num_reserved_ids attribute to value.
+// CTCLossIgnoreLongerOutputsThanInputs sets the optional ignore_longer_outputs_than_inputs attribute to value.
 //
-// value: Optionally some reserved IDs can be added in the range [0,
-// ..., num_reserved_ids) by the users. One use case is that a special unknown
-// word token is used as ID 0. These IDs will have a sampling probability of 0.
-// If not specified, defaults to 0
-func FixedUnigramCandidateSamplerNumReservedIds(value int64) FixedUnigramCandidateSamplerAttr {
+// value: Scalar. If set to true, during CTC
+// calculation, items that have longer output sequences than input sequences
+// are skipped: they don't contribute to the loss term and have zero-gradient.
+// If not specified, defaults to false
+func CTCLossIgnoreLongerOutputsThanInputs(value bool) CTCLossAttr {
 	return func(m optionalAttr) {
-		m["num_reserved_ids"] = value
+		m["ignore_longer_outputs_than_inputs"] = value
 	}
 }
 
-// FixedUnigramCandidateSamplerNumShards sets the optional num_shards attribute to value.
+// Calculates the CTC Loss (log probability) for each batch entry.  Also calculates
 //
-// value: A sampler can be used to sample from a subset of the original range
-// in order to speed up the whole computation through parallelism. This parameter
-// (together with 'shard') indicates the number of partitions that are being
-// used in the overall computation.
-// If not specified, defaults to 1
+// the gradient.  This class performs the softmax operation for you, so inputs
+// should be e.g. linear projections of outputs by an LSTM.
 //
-// REQUIRES: value >= 1
-func FixedUnigramCandidateSamplerNumShards(value int64) FixedUnigramCandidateSamplerAttr {
+// Arguments:
+//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
+//	labels_indices: The indices of a `SparseTensor<int32, 2>`.
+// `labels_indices(i, :) == [b, t]` means `labels_values(i)` stores the id for
+// `(batch b, time t)`.
+//	labels_values: The values (labels) associated with the given batch and time.
+//	sequence_length: A vector containing sequence lengths (batch).
+//
+// Returns A vector (batch) containing log-probabilities.The gradient of `loss`.  3-D, shape:
+// `(max_time x batch_size x num_classes)`.
+func CTCLoss(scope *Scope, inputs tf.Output, labels_indices tf.Output, labels_values tf.Output, sequence_length tf.Output, optional ...CTCLossAttr) (loss tf.Output, gradient tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CTCLoss",
+		Input: []tf.Input{
+			inputs, labels_indices, labels_values, sequence_length,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// CTCGreedyDecoderAttr is an optional argument to CTCGreedyDecoder.
+type CTCGreedyDecoderAttr func(optionalAttr)
+
+// CTCGreedyDecoderMergeRepeated sets the optional merge_repeated attribute to value.
+//
+// value: If True, merge repeated classes in output.
+// If not specified, defaults to false
+func CTCGreedyDecoderMergeRepeated(value bool) CTCGreedyDecoderAttr {
 	return func(m optionalAttr) {
-		m["num_shards"] = value
+		m["merge_repeated"] = value
 	}
 }
 
-// FixedUnigramCandidateSamplerShard sets the optional shard attribute to value.
+// Performs greedy decoding on the logits given in inputs.
 //
-// value: A sampler can be used to sample from a subset of the original range
-// in order to speed up the whole computation through parallelism. This parameter
-// (together with 'num_shards') indicates the particular partition number of a
-// sampler op, when partitioning is being used.
-// If not specified, defaults to 0
+// A note about the attribute merge_repeated: if enabled, when
+// consecutive logits' maximum indices are the same, only the first of
+// these is emitted.  Labeling the blank '*', the sequence "A B B * B B"
+// becomes "A B B" if merge_repeated = True and "A B B B B" if
+// merge_repeated = False.
 //
-// REQUIRES: value >= 0
-func FixedUnigramCandidateSamplerShard(value int64) FixedUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["shard"] = value
+// Regardless of the value of merge_repeated, if the maximum index of a given
+// time and batch corresponds to the blank, index `(num_classes - 1)`, no new
+// element is emitted.
+//
+// Arguments:
+//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
+//	sequence_length: A vector containing sequence lengths, size `(batch_size)`.
+//
+// Returns Indices matrix, size `(total_decoded_outputs x 2)`,
+// of a `SparseTensor<int64, 2>`.  The rows store: [batch, time].Values vector, size: `(total_decoded_outputs)`,
+// of a `SparseTensor<int64, 2>`.  The vector stores the decoded classes.Shape vector, size `(2)`, of the decoded SparseTensor.
+// Values are: `[batch_size, max_decoded_length]`.Matrix, size `(batch_size x 1)`, containing sequence
+// log-probabilities.
+func CTCGreedyDecoder(scope *Scope, inputs tf.Output, sequence_length tf.Output, optional ...CTCGreedyDecoderAttr) (decoded_indices tf.Output, decoded_values tf.Output, decoded_shape tf.Output, log_probability tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "CTCGreedyDecoder",
+		Input: []tf.Input{
+			inputs, sequence_length,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
 }
 
-// FixedUnigramCandidateSamplerUnigrams sets the optional unigrams attribute to value.
+// Forwards `data` to the output port determined by `pred`.
 //
-// value: A list of unigram counts or probabilities, one per ID in sequential
-// order. Exactly one of vocab_file and unigrams should be passed to this op.
-// If not specified, defaults to <>
-func FixedUnigramCandidateSamplerUnigrams(value []float32) FixedUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["unigrams"] = value
+// If `pred` is true, the `data` input is forwarded to `output_true`. Otherwise,
+// the data goes to `output_false`.
+//
+// See also `RefSwitch` and `Merge`.
+//
+// Arguments:
+//	data: The tensor to be forwarded to the appropriate output.
+//	pred: A scalar that specifies which output port will receive data.
+//
+// Returns If `pred` is false, data will be forwarded to this output.If `pred` is true, data will be forwarded to this output.
+func Switch(scope *Scope, data tf.Output, pred tf.Output) (output_false tf.Output, output_true tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Switch",
+		Input: []tf.Input{
+			data, pred,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Add all input tensors element wise.
+//
+// Arguments:
+//	inputs: Must all be the same size and shape.
+func AddN(scope *Scope, inputs []tf.Output) (sum tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "AddN",
+		Input: []tf.Input{
+			tf.OutputList(inputs),
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FixedUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+// EnterAttr is an optional argument to Enter.
+type EnterAttr func(optionalAttr)
+
+// EnterIsConstant sets the optional is_constant attribute to value.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func FixedUnigramCandidateSamplerSeed(value int64) FixedUnigramCandidateSamplerAttr {
+// value: If true, the output is constant within the child frame.
+// If not specified, defaults to false
+func EnterIsConstant(value bool) EnterAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["is_constant"] = value
 	}
 }
 
-// FixedUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// EnterParallelIterations sets the optional parallel_iterations attribute to value.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func FixedUnigramCandidateSamplerSeed2(value int64) FixedUnigramCandidateSamplerAttr {
+// value: The number of iterations allowed to run in parallel.
+// If not specified, defaults to 10
+func EnterParallelIterations(value int64) EnterAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["parallel_iterations"] = value
 	}
 }
 
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// A unigram sampler could use a fixed unigram distribution read from a
-// file or passed in as an in-memory array instead of building up the distribution
-// from data on the fly. There is also an option to skew the distribution by
-// applying a distortion power to the weights.
-//
-// The vocabulary file should be in CSV-like format, with the last field
-// being the weight associated with the word.
-//
-// For each batch, this op picks a single set of sampled candidate labels.
+// Creates or finds a child frame, and makes `data` available to the child frame.
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+// This op is used together with `Exit` to create loops in the graph.
+// The unique `frame_name` is used by the `Executor` to identify frames. If
+// `is_constant` is true, `output` is a constant in the child frame; otherwise
+// it may be changed in the child frame. At most `parallel_iterations` iterations
+// are run in parallel in the child frame.
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+//	data: The tensor to be made available to the child frame.
+//	frame_name: The name of the child frame.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func FixedUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...FixedUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// Returns The same tensor as `data`.
+func Enter(scope *Scope, data tf.Output, frame_name string, optional ...EnterAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	attrs := map[string]interface{}{"frame_name": frame_name}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FixedUnigramCandidateSampler",
+		Type: "Enter",
 		Input: []tf.Input{
-			true_classes,
+			data,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// Elementwise computes the bitwise AND of `x` and `y`.
-//
-// The result will have those bits set, that are set in both `x` and `y`. The
-// computation is performed on the underlying representations of `x` and `y`.
-func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BitwiseAnd",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Elementwise computes the bitwise left-shift of `x` and `y`.
+// Produce a string tensor that encodes the state of a Reader.
 //
-// If `y` is negative, or greater than or equal to the width of `x` in bits the
-// result is implementation defined.
-func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Not all Readers support being serialized, so this can produce an
+// Unimplemented error.
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+func ReaderSerializeStateV2(scope *Scope, reader_handle tf.Output) (state tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LeftShift",
+		Type: "ReaderSerializeStateV2",
 		Input: []tf.Input{
-			x, y,
+			reader_handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Elementwise computes the bitwise right-shift of `x` and `y`.
+// Exits the current frame to its parent frame.
 //
-// Performs a logical shift for unsigned integer types, and an arithmetic shift
-// for signed integer types.
+// Exit makes its input `data` available to the parent frame.
 //
-// If `y` is negative, or greater than or equal to than the width of `x` in bits
-// the result is implementation defined.
-func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Arguments:
+//	data: The tensor to be made available to the parent frame.
+//
+// Returns The same tensor as `data`.
+func Exit(scope *Scope, data tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RightShift",
+		Type: "Exit",
 		Input: []tf.Input{
-			x, y,
+			data,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Adjust the hue of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A delta is then applied all the hue values,
-// and then remapped back to RGB colorspace.
-//
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	delta: A float delta to add to the hue.
-//
-// Returns The hue-adjusted image or images.
-func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
+// Returns a copy of the input tensor.
+func Snapshot(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AdjustHue",
+		Type: "Snapshot",
 		Input: []tf.Input{
-			images, delta,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
-type AvgPool3DGradAttr func(optionalAttr)
+// AbortAttr is an optional argument to Abort.
+type AbortAttr func(optionalAttr)
 
-// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
+// AbortErrorMsg sets the optional error_msg attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+// value: A string which is the message associated with the exception.
+// If not specified, defaults to ""
+func AbortErrorMsg(value string) AbortAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["error_msg"] = value
 	}
 }
 
-// Computes gradients of average pooling function.
+// AbortExitWithoutError sets the optional exit_without_error attribute to value.
+// If not specified, defaults to false
+func AbortExitWithoutError(value bool) AbortAttr {
+	return func(m optionalAttr) {
+		m["exit_without_error"] = value
+	}
+}
+
+// Raise a exception to abort the process when called.
 //
-// Arguments:
-//	orig_input_shape: The original input dimensions.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// If exit_without_error is true, the process will exit normally,
+// otherwise it will exit with a SIGABORT signal.
 //
-// Returns The backprop for input.
-func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
+// Returns nothing but an exception.
+//
+// Returns the created operation.
+func Abort(scope *Scope, optional ...AbortAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AvgPool3DGrad",
-		Input: []tf.Input{
-			orig_input_shape, grad,
-		},
+		Type: "Abort",
+
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
-type ParseSingleSequenceExampleAttr func(optionalAttr)
+// FixedUnigramCandidateSamplerAttr is an optional argument to FixedUnigramCandidateSampler.
+type FixedUnigramCandidateSamplerAttr func(optionalAttr)
 
-// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
-//
-// value: A list of Ncontext_sparse types; the data types of data in
-// each context Feature given in context_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
+// FixedUnigramCandidateSamplerVocabFile sets the optional vocab_file attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// value: Each valid line in this file (which should have a CSV-like format)
+// corresponds to a valid word ID. IDs are in sequential order, starting from
+// num_reserved_ids. The last entry in each line is expected to be a value
+// corresponding to the count or relative probability. Exactly one of vocab_file
+// and unigrams needs to be passed to this op.
+// If not specified, defaults to ""
+func FixedUnigramCandidateSamplerVocabFile(value string) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["context_sparse_types"] = value
+		m["vocab_file"] = value
 	}
 }
 
-// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
-// If not specified, defaults to <>
+// FixedUnigramCandidateSamplerDistortion sets the optional distortion attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// value: The distortion is used to skew the unigram probability distribution.
+// Each weight is first raised to the distortion's power before adding to the
+// internal unigram distribution. As a result, distortion = 1.0 gives regular
+// unigram sampling (as defined by the vocab file), and distortion = 0.0 gives
+// a uniform distribution.
+// If not specified, defaults to 1
+func FixedUnigramCandidateSamplerDistortion(value float32) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["feature_list_dense_types"] = value
+		m["distortion"] = value
 	}
 }
 
-// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
-//
-// value: A list of Ncontext_dense shapes; the shapes of data in
-// each context Feature given in context_dense_keys.
-// The number of elements in the Feature corresponding to context_dense_key[j]
-// must always equal context_dense_shapes[j].NumEntries().
-// The shape of context_dense_values[j] will match context_dense_shapes[j].
-// If not specified, defaults to <>
+// FixedUnigramCandidateSamplerNumReservedIds sets the optional num_reserved_ids attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+// value: Optionally some reserved IDs can be added in the range [0,
+// ..., num_reserved_ids) by the users. One use case is that a special unknown
+// word token is used as ID 0. These IDs will have a sampling probability of 0.
+// If not specified, defaults to 0
+func FixedUnigramCandidateSamplerNumReservedIds(value int64) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["context_dense_shapes"] = value
+		m["num_reserved_ids"] = value
 	}
 }
 
-// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
+// FixedUnigramCandidateSamplerNumShards sets the optional num_shards attribute to value.
 //
-// value: A list of Nfeature_list_sparse types; the data types
-// of data in each FeatureList given in feature_list_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
+// value: A sampler can be used to sample from a subset of the original range
+// in order to speed up the whole computation through parallelism. This parameter
+// (together with 'shard') indicates the number of partitions that are being
+// used in the overall computation.
+// If not specified, defaults to 1
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+// REQUIRES: value >= 1
+func FixedUnigramCandidateSamplerNumShards(value int64) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["feature_list_sparse_types"] = value
+		m["num_shards"] = value
 	}
 }
 
-// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
+// FixedUnigramCandidateSamplerShard sets the optional shard attribute to value.
 //
-// value: A list of Nfeature_list_dense shapes; the shapes of
-// data in each FeatureList given in feature_list_dense_keys.
-// The shape of each Feature in the FeatureList corresponding to
-// feature_list_dense_key[j] must always equal
-// feature_list_dense_shapes[j].NumEntries().
-// If not specified, defaults to <>
+// value: A sampler can be used to sample from a subset of the original range
+// in order to speed up the whole computation through parallelism. This parameter
+// (together with 'num_shards') indicates the particular partition number of a
+// sampler op, when partitioning is being used.
+// If not specified, defaults to 0
 //
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+// REQUIRES: value >= 0
+func FixedUnigramCandidateSamplerShard(value int64) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["feature_list_dense_shapes"] = value
+		m["shard"] = value
 	}
 }
 
-// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
+// FixedUnigramCandidateSamplerUnigrams sets the optional unigrams attribute to value.
 //
-// Arguments:
-//	serialized: A scalar containing a binary serialized SequenceExample proto.
-//	feature_list_dense_missing_assumed_empty: A vector listing the
-// FeatureList keys which may be missing from the SequenceExample.  If the
-// associated FeatureList is missing, it is treated as empty.  By default,
-// any FeatureList not listed in this vector must exist in the SequenceExample.
-//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with context_sparse
-// values.
-//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' context features associated with
-// dense values.
-//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
-// (scalars).  The keys expected in the FeatureLists associated with sparse
-// values.
-//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' feature_lists associated
-// with lists of dense values.
-//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
-// context_dense_defaults[j] provides default values
-// when the SequenceExample's context map lacks context_dense_key[j].
-// If an empty Tensor is provided for context_dense_defaults[j],
-// then the Feature context_dense_keys[j] is required.
-// The input type is inferred from context_dense_defaults[j], even when it's
-// empty.  If context_dense_defaults[j] is not empty, its shape must match
-// context_dense_shapes[j].
-//	debug_name: A scalar containing the name of the serialized proto.
-// May contain, for example, table key (descriptive) name for the
-// corresponding serialized proto.  This is purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty scalar if no name is available.
-func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ParseSingleSequenceExample",
-		Input: []tf.Input{
-			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
+// value: A list of unigram counts or probabilities, one per ID in sequential
+// order. Exactly one of vocab_file and unigrams should be passed to this op.
+// If not specified, defaults to <>
+func FixedUnigramCandidateSamplerUnigrams(value []float32) FixedUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["unigrams"] = value
 	}
-	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
 }
 
-// DecodeWavAttr is an optional argument to DecodeWav.
-type DecodeWavAttr func(optionalAttr)
-
-// DecodeWavDesiredChannels sets the optional desired_channels attribute to value.
+// FixedUnigramCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// value: Number of sample channels wanted.
-// If not specified, defaults to -1
-func DecodeWavDesiredChannels(value int64) DecodeWavAttr {
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func FixedUnigramCandidateSamplerSeed(value int64) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["desired_channels"] = value
+		m["seed"] = value
 	}
 }
 
-// DecodeWavDesiredSamples sets the optional desired_samples attribute to value.
+// FixedUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
 //
-// value: Length of audio requested.
-// If not specified, defaults to -1
-func DecodeWavDesiredSamples(value int64) DecodeWavAttr {
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func FixedUnigramCandidateSamplerSeed2(value int64) FixedUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["desired_samples"] = value
+		m["seed2"] = value
 	}
 }
 
-// Decode a 16-bit PCM WAV file to a float tensor.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// The -32768 to 32767 signed 16-bit values will be scaled to -1.0 to 1.0 in float.
+// A unigram sampler could use a fixed unigram distribution read from a
+// file or passed in as an in-memory array instead of building up the distribution
+// from data on the fly. There is also an option to skew the distribution by
+// applying a distortion power to the weights.
 //
-// When desired_channels is set, if the input contains fewer channels than this
-// then the last channel will be duplicated to give the requested number, else if
-// the input has more channels than requested then the additional channels will be
-// ignored.
+// The vocabulary file should be in CSV-like format, with the last field
+// being the weight associated with the word.
 //
-// If desired_samples is set, then the audio will be cropped or padded with zeroes
-// to the requested length.
+// For each batch, this op picks a single set of sampled candidate labels.
 //
-// The first output contains a Tensor with the content of the audio samples. The
-// lowest dimension will be the number of channels, and the second will be the
-// number of samples. For example, a ten-sample-long stereo WAV file should give an
-// output shape of [10, 2].
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
 // Arguments:
-//	contents: The WAV-encoded audio, usually from a file.
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
 //
-// Returns 2-D with shape `[length, channels]`.Scalar holding the sample rate found in the WAV header.
-func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (audio tf.Output, sample_rate tf.Output) {
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func FixedUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...FixedUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeWav",
+		Type: "FixedUnigramCandidateSampler",
 		Input: []tf.Input{
-			contents,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// UniqueAttr is an optional argument to Unique.
-type UniqueAttr func(optionalAttr)
-
-// UniqueOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueOutIdx(value tf.DataType) UniqueAttr {
-	return func(m optionalAttr) {
-		m["out_idx"] = value
-	}
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Finds unique elements in a 1-D tensor.
-//
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. In other words:
-//
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
-//
-// For example:
-//
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx = unique(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// ```
+// Elementwise computes the bitwise AND of `x` and `y`.
 //
-// Arguments:
-//	x: 1-D.
+// The result will have those bits set, that are set in both `x` and `y`. The
+// computation is performed on the underlying representations of `x` and `y`.
+func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BitwiseAnd",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Elementwise computes the bitwise left-shift of `x` and `y`.
 //
-// Returns 1-D.1-D.
-func Unique(scope *Scope, x tf.Output, optional ...UniqueAttr) (y tf.Output, idx tf.Output) {
+// If `y` is negative, or greater than or equal to the width of `x` in bits the
+// result is implementation defined.
+func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Unique",
+		Type: "LeftShift",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Concatenates a list of `N` tensors along the first dimension.
-//
-// The input tensors are all required to have size 1 in the first dimension.
-//
-// For example:
-//
-// ```
-// # 'x' is [[1, 4]]
-// # 'y' is [[2, 5]]
-// # 'z' is [[3, 6]]
-// parallel_concat([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
-// ```
-//
-// The difference between concat and parallel_concat is that concat requires all
-// of the inputs be computed before the operation will begin but doesn't require
-// that the input shapes be known during graph construction.  Parallel concat
-// will copy pieces of the input into the output as they become available, in
-// some situations this can provide a performance benefit.
+// Elementwise computes the bitwise right-shift of `x` and `y`.
 //
-// Arguments:
-//	values: Tensors to be concatenated. All must have size 1 in the first dimension
-// and same shape.
-//	shape: the final shape of the result; should be equal to the shapes of any input
-// but with the number of input values in the first dimension.
+// Performs a logical shift for unsigned integer types, and an arithmetic shift
+// for signed integer types.
 //
-// Returns The concatenated tensor.
-func ParallelConcat(scope *Scope, values []tf.Output, shape tf.Shape) (output tf.Output) {
+// If `y` is negative, or greater than or equal to than the width of `x` in bits
+// the result is implementation defined.
+func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "ParallelConcat",
+		Type: "RightShift",
 		Input: []tf.Input{
-			tf.OutputList(values),
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the lower regularized incomplete Gamma function `Q(a, x)`.
-//
-// The lower regularized incomplete Gamma function is defined as:
-//
-//
-// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
+// Adjust the hue of one or more images.
 //
-// where
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
 //
-// \\(gamma(a, x) = int_{0}^{x} t^{a-1} exp(-t) dt\\)
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A delta is then applied all the hue values,
+// and then remapped back to RGB colorspace.
 //
-// is the lower incomplete Gamma function.
+// Arguments:
+//	images: Images to adjust.  At least 3-D.
+//	delta: A float delta to add to the hue.
 //
-// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
-// Gamma function.
-func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// Returns The hue-adjusted image or images.
+func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Igamma",
+		Type: "AdjustHue",
 		Input: []tf.Input{
-			a, x,
+			images, delta,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes offsets of concat inputs within its output.
-//
-// For example:
-//
-// ```
-// # 'x' is [2, 2, 7]
-// # 'y' is [2, 3, 7]
-// # 'z' is [2, 5, 7]
-// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
-// ```
+// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
+type AvgPool3DGradAttr func(optionalAttr)
+
+// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
 //
-// This is typically used by gradient computations for a concat operation.
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes gradients of average pooling function.
 //
 // Arguments:
-//	concat_dim: The dimension along which to concatenate.
-//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
+//	orig_input_shape: The original input dimensions.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
 //
-// Returns The `N` int32 vectors representing the starting offset
-// of input tensors within the concatenated output.
-func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
+// Returns The backprop for input.
+func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ConcatOffset",
+		Type: "AvgPool3DGrad",
 		Input: []tf.Input{
-			concat_dim, tf.OutputList(shape),
+			orig_input_shape, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
+	return op.Output(0)
+}
+
+// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
+type ParseSingleSequenceExampleAttr func(optionalAttr)
+
+// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
+//
+// value: A list of Ncontext_sparse types; the data types of data in
+// each context Feature given in context_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["context_sparse_types"] = value
 	}
-	var idx int
-	var err error
-	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
-		scope.UpdateErr("ConcatOffset", err)
-		return
+}
+
+// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_types"] = value
 	}
-	return offset
 }
 
-// Splits a tensor into `num_split` tensors along one dimension.
+// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
 //
-// Arguments:
-//	axis: 0-D.  The dimension along which to split.  Must be in the range
-// `[-rank(value), rank(value))`.
-//	value: The tensor to split.
-//	num_split: The number of ways to split.  Must evenly divide
-// `value.shape[split_dim]`.
+// value: A list of Ncontext_dense shapes; the shapes of data in
+// each context Feature given in context_dense_keys.
+// The number of elements in the Feature corresponding to context_dense_key[j]
+// must always equal context_dense_shapes[j].NumEntries().
+// The shape of context_dense_values[j] will match context_dense_shapes[j].
+// If not specified, defaults to <>
 //
-// Returns They are identically shaped tensors, whose shape matches that of `value`
-// except along `axis`, where their sizes are
-// `values.shape[split_dim] / num_split`.
-func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (output []tf.Output) {
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["context_dense_shapes"] = value
+	}
+}
+
+// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
+//
+// value: A list of Nfeature_list_sparse types; the data types
+// of data in each FeatureList given in feature_list_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_sparse_types"] = value
+	}
+}
+
+// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
+//
+// value: A list of Nfeature_list_dense shapes; the shapes of
+// data in each FeatureList given in feature_list_dense_keys.
+// The shape of each Feature in the FeatureList corresponding to
+// feature_list_dense_key[j] must always equal
+// feature_list_dense_shapes[j].NumEntries().
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_shapes"] = value
+	}
+}
+
+// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
+//
+// Arguments:
+//	serialized: A scalar containing a binary serialized SequenceExample proto.
+//	feature_list_dense_missing_assumed_empty: A vector listing the
+// FeatureList keys which may be missing from the SequenceExample.  If the
+// associated FeatureList is missing, it is treated as empty.  By default,
+// any FeatureList not listed in this vector must exist in the SequenceExample.
+//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with context_sparse
+// values.
+//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' context features associated with
+// dense values.
+//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
+// (scalars).  The keys expected in the FeatureLists associated with sparse
+// values.
+//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' feature_lists associated
+// with lists of dense values.
+//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
+// context_dense_defaults[j] provides default values
+// when the SequenceExample's context map lacks context_dense_key[j].
+// If an empty Tensor is provided for context_dense_defaults[j],
+// then the Feature context_dense_keys[j] is required.
+// The input type is inferred from context_dense_defaults[j], even when it's
+// empty.  If context_dense_defaults[j] is not empty, its shape must match
+// context_dense_shapes[j].
+//	debug_name: A scalar containing the name of the serialized proto.
+// May contain, for example, table key (descriptive) name for the
+// corresponding serialized proto.  This is purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty scalar if no name is available.
+func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Split",
+		Type: "ParseSingleSequenceExample",
 		Input: []tf.Input{
-			axis, value,
+			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
 		},
 		Attrs: attrs,
 	}
@@ -26334,158 +26403,86 @@ func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (outp
 	}
 	var idx int
 	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Split", err)
+	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	return output
-}
-
-// Splits a tensor into `num_split` tensors along one dimension.
-//
-// Arguments:
-//	value: The tensor to split.
-//	size_splits: list containing the sizes of each output tensor along the split
-// dimension. Must sum to the dimension of value along split_dim.
-// Can contain one -1 indicating that dimension is to be inferred.
-//	axis: 0-D.  The dimension along which to split.  Must be in the range
-// `[-rank(value), rank(value))`.
-//
-//
-// Returns Tensors whose shape matches that of `value`
-// except along `axis`, where their sizes are
-// `size_splits[i]`.
-func SplitV(scope *Scope, value tf.Output, size_splits tf.Output, axis tf.Output, num_split int64) (output []tf.Output) {
-	if scope.Err() != nil {
+	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
-	opspec := tf.OpSpec{
-		Type: "SplitV",
-		Input: []tf.Input{
-			value, size_splits, axis,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
+	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("SplitV", err)
+	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	return output
-}
-
-// Gives a guarantee to the TF runtime that the input tensor is a constant.
-//
-// The runtime is then free to make optimizations based on this.
-//
-// Only accepts value typed tensors as inputs and rejects resource variable handles
-// as input.
-//
-// Returns the input tensor without modification.
-func GuaranteeConst(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
+	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "GuaranteeConst",
-		Input: []tf.Input{
-			input,
-		},
+	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns a tensor of zeros with the same shape and type as x.
-//
-// Arguments:
-//	x: a tensor of type T.
-//
-// Returns a tensor of the same shape and type as x but filled with zeros.
-func ZerosLike(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
+	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "ZerosLike",
-		Input: []tf.Input{
-			x,
-		},
+	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
 }
 
-// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
-type QuantizedInstanceNormAttr func(optionalAttr)
+// DecodeWavAttr is an optional argument to DecodeWav.
+type DecodeWavAttr func(optionalAttr)
 
-// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
+// DecodeWavDesiredChannels sets the optional desired_channels attribute to value.
 //
-// value: If True, `given_y_min` and `given_y_min`
-// and `given_y_max` are used as the output range. Otherwise,
-// the implementation computes the output range.
-// If not specified, defaults to false
-func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
+// value: Number of sample channels wanted.
+// If not specified, defaults to -1
+func DecodeWavDesiredChannels(value int64) DecodeWavAttr {
 	return func(m optionalAttr) {
-		m["output_range_given"] = value
+		m["desired_channels"] = value
 	}
 }
 
-// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
+// DecodeWavDesiredSamples sets the optional desired_samples attribute to value.
 //
-// value: Output in `y_min` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
+// value: Length of audio requested.
+// If not specified, defaults to -1
+func DecodeWavDesiredSamples(value int64) DecodeWavAttr {
 	return func(m optionalAttr) {
-		m["given_y_min"] = value
+		m["desired_samples"] = value
 	}
 }
 
-// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
+// Decode a 16-bit PCM WAV file to a float tensor.
 //
-// value: Output in `y_max` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["given_y_max"] = value
-	}
-}
-
-// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
+// The -32768 to 32767 signed 16-bit values will be scaled to -1.0 to 1.0 in float.
 //
-// value: A small float number to avoid dividing by 0.
-// If not specified, defaults to 1e-05
-func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["variance_epsilon"] = value
-	}
-}
-
-// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
+// When desired_channels is set, if the input contains fewer channels than this
+// then the last channel will be duplicated to give the requested number, else if
+// the input has more channels than requested then the additional channels will be
+// ignored.
 //
-// value: Minimum value of `y_max - y_min`
-// If not specified, defaults to 0.001
-func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["min_separation"] = value
-	}
-}
-
-// Quantized Instance normalization.
+// If desired_samples is set, then the audio will be cropped or padded with zeroes
+// to the requested length.
+//
+// The first output contains a Tensor with the content of the audio samples. The
+// lowest dimension will be the number of channels, and the second will be the
+// number of samples. For example, a ten-sample-long stereo WAV file should give an
+// output shape of [10, 2].
 //
 // Arguments:
-//	x: A 4D input Tensor.
-//	x_min: The value represented by the lowest quantized input.
-//	x_max: The value represented by the highest quantized input.
+//	contents: The WAV-encoded audio, usually from a file.
 //
-// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
-func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
+// Returns 2-D with shape `[length, channels]`.Scalar holding the sample rate found in the WAV header.
+func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (audio tf.Output, sample_rate tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26494,337 +26491,241 @@ func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedInstanceNorm",
+		Type: "DecodeWav",
 		Input: []tf.Input{
-			x, x_min, x_max,
+			contents,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1)
 }
 
-// Returns the diagonal part of the tensor.
-//
-// This operation returns a tensor with the `diagonal` part
-// of the `input`. The `diagonal` part is computed as follows:
+// UniqueAttr is an optional argument to Unique.
+type UniqueAttr func(optionalAttr)
+
+// UniqueOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueOutIdx(value tf.DataType) UniqueAttr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Finds unique elements in a 1-D tensor.
 //
-// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
-// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. In other words:
 //
-// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
 //
 // For example:
 //
 // ```
-// # 'input' is [[1, 0, 0, 0]
-//               [0, 2, 0, 0]
-//               [0, 0, 3, 0]
-//               [0, 0, 0, 4]]
-//
-// tf.diag_part(input) ==> [1, 2, 3, 4]
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx = unique(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
 // ```
 //
 // Arguments:
-//	input: Rank k tensor where k is even and not zero.
+//	x: 1-D.
 //
-// Returns The extracted diagonal.
-func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
+// Returns 1-D.1-D.
+func Unique(scope *Scope, x tf.Output, optional ...UniqueAttr) (y tf.Output, idx tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "DiagPart",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the element-wise max of two SparseTensors.
-//
-// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
-//
-// Arguments:
-//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, in the canonical lexicographic ordering.
-//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
-//	a_shape: 1-D.  Shape of the input SparseTensor.
-//	b_indices: counterpart to `a_indices` for the other operand.
-//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
-//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
-//
-// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
-func SparseSparseMaximum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSparseMaximum",
+		Type: "Unique",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
+			x,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1)
 }
 
-// Returns a batched matrix tensor with new batched diagonal values.
+// Concatenates a list of `N` tensors along the first dimension.
 //
-// Given `input` and `diagonal`, this operation returns a tensor with the
-// same shape and values as `input`, except for the main diagonal of the
-// innermost matrices.  These will be overwritten by the values in `diagonal`.
+// The input tensors are all required to have size 1 in the first dimension.
 //
-// The output is computed as follows:
+// For example:
 //
-// Assume `input` has `k+1` dimensions `[I, J, K, ..., M, N]` and `diagonal` has
-// `k` dimensions `[I, J, K, ..., min(M, N)]`.  Then the output is a
-// tensor of rank `k+1` with dimensions `[I, J, K, ..., M, N]` where:
+// ```
+// # 'x' is [[1, 4]]
+// # 'y' is [[2, 5]]
+// # 'z' is [[3, 6]]
+// parallel_concat([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
+// ```
 //
-//   * `output[i, j, k, ..., m, n] = diagonal[i, j, k, ..., n]` for `m == n`.
-//   * `output[i, j, k, ..., m, n] = input[i, j, k, ..., m, n]` for `m != n`.
+// The difference between concat and parallel_concat is that concat requires all
+// of the inputs be computed before the operation will begin but doesn't require
+// that the input shapes be known during graph construction.  Parallel concat
+// will copy pieces of the input into the output as they become available, in
+// some situations this can provide a performance benefit.
 //
 // Arguments:
-//	input: Rank `k+1`, where `k >= 1`.
-//	diagonal: Rank `k`, where `k >= 1`.
+//	values: Tensors to be concatenated. All must have size 1 in the first dimension
+// and same shape.
+//	shape: the final shape of the result; should be equal to the shapes of any input
+// but with the number of input values in the first dimension.
 //
-// Returns Rank `k+1`, with `output.shape = input.shape`.
-func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf.Output) {
+// Returns The concatenated tensor.
+func ParallelConcat(scope *Scope, values []tf.Output, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "MatrixSetDiag",
+		Type: "ParallelConcat",
 		Input: []tf.Input{
-			input, diagonal,
+			tf.OutputList(values),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// EditDistanceAttr is an optional argument to EditDistance.
-type EditDistanceAttr func(optionalAttr)
-
-// EditDistanceNormalize sets the optional normalize attribute to value.
-//
-// value: boolean (if true, edit distances are normalized by length of truth).
-//
-// The output is:
-// If not specified, defaults to true
-func EditDistanceNormalize(value bool) EditDistanceAttr {
-	return func(m optionalAttr) {
-		m["normalize"] = value
-	}
-}
-
-// Computes the (possibly normalized) Levenshtein Edit Distance.
-//
-// The inputs are variable-length sequences provided by SparseTensors
-//   (hypothesis_indices, hypothesis_values, hypothesis_shape)
-// and
-//   (truth_indices, truth_values, truth_shape).
-//
-// The inputs are:
+// Compute the lower regularized incomplete Gamma function `Q(a, x)`.
 //
-// Arguments:
-//	hypothesis_indices: The indices of the hypothesis list SparseTensor.
-// This is an N x R int64 matrix.
-//	hypothesis_values: The values of the hypothesis list SparseTensor.
-// This is an N-length vector.
-//	hypothesis_shape: The shape of the hypothesis list SparseTensor.
-// This is an R-length vector.
-//	truth_indices: The indices of the truth list SparseTensor.
-// This is an M x R int64 matrix.
-//	truth_values: The values of the truth list SparseTensor.
-// This is an M-length vector.
-//	truth_shape: truth indices, vector.
+// The lower regularized incomplete Gamma function is defined as:
 //
-// Returns A dense float tensor with rank R - 1.
 //
-// For the example input:
+// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
 //
-//     // hypothesis represents a 2x1 matrix with variable-length values:
-//     //   (0,0) = ["a"]
-//     //   (1,0) = ["b"]
-//     hypothesis_indices = [[0, 0, 0],
-//                           [1, 0, 0]]
-//     hypothesis_values = ["a", "b"]
-//     hypothesis_shape = [2, 1, 1]
+// where
 //
-//     // truth represents a 2x2 matrix with variable-length values:
-//     //   (0,0) = []
-//     //   (0,1) = ["a"]
-//     //   (1,0) = ["b", "c"]
-//     //   (1,1) = ["a"]
-//     truth_indices = [[0, 1, 0],
-//                      [1, 0, 0],
-//                      [1, 0, 1],
-//                      [1, 1, 0]]
-//     truth_values = ["a", "b", "c", "a"]
-//     truth_shape = [2, 2, 2]
-//     normalize = true
+// \\(gamma(a, x) = int_{0}^{x} t^{a-1} exp(-t) dt\\)
 //
-// The output will be:
+// is the lower incomplete Gamma function.
 //
-//     // output is a 2x2 matrix with edit distances normalized by truth lengths.
-//     output = [[inf, 1.0],  // (0,0): no truth, (0,1): no hypothesis
-//               [0.5, 1.0]]  // (1,0): addition, (1,1): no hypothesis
-func EditDistance(scope *Scope, hypothesis_indices tf.Output, hypothesis_values tf.Output, hypothesis_shape tf.Output, truth_indices tf.Output, truth_values tf.Output, truth_shape tf.Output, optional ...EditDistanceAttr) (output tf.Output) {
+// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
+// Gamma function.
+func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "EditDistance",
+		Type: "Igamma",
 		Input: []tf.Input{
-			hypothesis_indices, hypothesis_values, hypothesis_shape, truth_indices, truth_values, truth_shape,
+			a, x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Gather slices from `params` into a Tensor with shape specified by `indices`.
-//
-// `indices` is an K-dimensional integer tensor, best thought of as a
-// (K-1)-dimensional tensor of indices into `params`, where each element defines a
-// slice of `params`:
-//
-//     output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
-//
-// Whereas in @{tf.gather} `indices` defines slices into the first
-// dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
-// first `N` dimensions of `params`, where `N = indices.shape[-1]`.
-//
-// The last dimension of `indices` can be at most the rank of
-// `params`:
-//
-//     indices.shape[-1] <= params.rank
-//
-// The last dimension of `indices` corresponds to elements
-// (if `indices.shape[-1] == params.rank`) or slices
-// (if `indices.shape[-1] < params.rank`) along dimension `indices.shape[-1]`
-// of `params`.  The output tensor has shape
-//
-//     indices.shape[:-1] + params.shape[indices.shape[-1]:]
-//
-// Some examples below.
-//
-// Simple indexing into a matrix:
-//
-// ```python
-//     indices = [[0, 0], [1, 1]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = ['a', 'd']
-// ```
-//
-// Slice indexing into a matrix:
-//
-// ```python
-//     indices = [[1], [0]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [['c', 'd'], ['a', 'b']]
-// ```
-//
-// Indexing into a 3-tensor:
-//
-// ```python
-//     indices = [[1]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[['a1', 'b1'], ['c1', 'd1']]]
-//
-//
-//     indices = [[0, 1], [1, 0]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [['c0', 'd0'], ['a1', 'b1']]
-//
-//
-//     indices = [[0, 0, 1], [1, 0, 1]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = ['b0', 'b1']
-// ```
+// Computes offsets of concat inputs within its output.
 //
-// Batched indexing into a matrix:
+// For example:
 //
-// ```python
-//     indices = [[[0, 0]], [[0, 1]]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [['a'], ['b']]
 // ```
-//
-// Batched slice indexing into a matrix:
-//
-// ```python
-//     indices = [[[1]], [[0]]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [[['c', 'd']], [['a', 'b']]]
+// # 'x' is [2, 2, 7]
+// # 'y' is [2, 3, 7]
+// # 'z' is [2, 5, 7]
+// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
 // ```
 //
-// Batched indexing into a 3-tensor:
-//
-// ```python
-//     indices = [[[1]], [[0]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[[['a1', 'b1'], ['c1', 'd1']]],
-//               [[['a0', 'b0'], ['c0', 'd0']]]]
-//
-//     indices = [[[0, 1], [1, 0]], [[0, 0], [1, 1]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[['c0', 'd0'], ['a1', 'b1']],
-//               [['a0', 'b0'], ['c1', 'd1']]]
+// This is typically used by gradient computations for a concat operation.
 //
+// Arguments:
+//	concat_dim: The dimension along which to concatenate.
+//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
 //
-//     indices = [[[0, 0, 1], [1, 0, 1]], [[0, 1, 1], [1, 1, 0]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [['b0', 'b1'], ['d0', 'c1']]
-// ```
+// Returns The `N` int32 vectors representing the starting offset
+// of input tensors within the concatenated output.
+func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ConcatOffset",
+		Input: []tf.Input{
+			concat_dim, tf.OutputList(shape),
+		},
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
+		scope.UpdateErr("ConcatOffset", err)
+		return
+	}
+	return offset
+}
+
+// Splits a tensor into `num_split` tensors along one dimension.
 //
 // Arguments:
-//	params: The tensor from which to gather values.
-//	indices: Index tensor.
+//	axis: 0-D.  The dimension along which to split.  Must be in the range
+// `[-rank(value), rank(value))`.
+//	value: The tensor to split.
+//	num_split: The number of ways to split.  Must evenly divide
+// `value.shape[split_dim]`.
 //
-// Returns Values from `params` gathered from indices given by `indices`, with
-// shape `indices.shape[:-1] + params.shape[indices.shape[-1]:]`.
-func GatherNd(scope *Scope, params tf.Output, indices tf.Output) (output tf.Output) {
+// Returns They are identically shaped tensors, whose shape matches that of `value`
+// except along `axis`, where their sizes are
+// `values.shape[split_dim] / num_split`.
+func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "GatherNd",
+		Type: "Split",
 		Input: []tf.Input{
-			params, indices,
+			axis, value,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Split", err)
+		return
+	}
+	return output
 }
 
-// Eagerly executes a python function to compute func(input)->output. The
+// Splits a tensor into `num_split` tensors along one dimension.
 //
-// semantics of the input, output, and attributes are the same as those for
-// PyFunc.
-func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataType) (output []tf.Output) {
+// Arguments:
+//	value: The tensor to split.
+//	size_splits: list containing the sizes of each output tensor along the split
+// dimension. Must sum to the dimension of value along split_dim.
+// Can contain one -1 indicating that dimension is to be inferred.
+//	axis: 0-D.  The dimension along which to split.  Must be in the range
+// `[-rank(value), rank(value))`.
+//
+//
+// Returns Tensors whose shape matches that of `value`
+// except along `axis`, where their sizes are
+// `size_splits[i]`.
+func SplitV(scope *Scope, value tf.Output, size_splits tf.Output, axis tf.Output, num_split int64) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"token": token, "Tout": Tout}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "EagerPyFunc",
+		Type: "SplitV",
 		Input: []tf.Input{
-			tf.OutputList(input),
+			value, size_splits, axis,
 		},
 		Attrs: attrs,
 	}
@@ -26835,39 +26736,26 @@ func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataTy
 	var idx int
 	var err error
 	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("EagerPyFunc", err)
+		scope.UpdateErr("SplitV", err)
 		return
 	}
 	return output
 }
 
-// Stops gradient computation.
-//
-// When executed in a graph, this op outputs its input tensor as-is.
+// Gives a guarantee to the TF runtime that the input tensor is a constant.
 //
-// When building ops to compute gradients, this op prevents the contribution of
-// its inputs to be taken into account.  Normally, the gradient generator adds ops
-// to a graph to compute the derivatives of a specified 'loss' by recursively
-// finding out inputs that contributed to its computation.  If you insert this op
-// in the graph it inputs are masked from the gradient generator.  They are not
-// taken into account for computing gradients.
+// The runtime is then free to make optimizations based on this.
 //
-// This is useful any time you want to compute a value with TensorFlow but need
-// to pretend that the value was a constant. Some examples include:
+// Only accepts value typed tensors as inputs and rejects resource variable handles
+// as input.
 //
-// *  The *EM* algorithm where the *M-step* should not involve backpropagation
-//    through the output of the *E-step*.
-// *  Contrastive divergence training of Boltzmann machines where, when
-//    differentiating the energy function, the training must not backpropagate
-//    through the graph that generated the samples from the model.
-// *  Adversarial training, where no backprop should happen through the adversarial
-//    example generation process.
-func StopGradient(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns the input tensor without modification.
+func GuaranteeConst(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "StopGradient",
+		Type: "GuaranteeConst",
 		Input: []tf.Input{
 			input,
 		},
@@ -26876,13 +26764,18 @@ func StopGradient(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Computes asin of x element-wise.
-func Asin(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns a tensor of zeros with the same shape and type as x.
+//
+// Arguments:
+//	x: a tensor of type T.
+//
+// Returns a tensor of the same shape and type as x but filled with zeros.
+func ZerosLike(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Asin",
+		Type: "ZerosLike",
 		Input: []tf.Input{
 			x,
 		},
@@ -26891,35 +26784,70 @@ func Asin(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// PreventGradientAttr is an optional argument to PreventGradient.
-type PreventGradientAttr func(optionalAttr)
+// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
+type QuantizedInstanceNormAttr func(optionalAttr)
 
-// PreventGradientMessage sets the optional message attribute to value.
+// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
 //
-// value: Will be printed in the error when anyone tries to differentiate
-// this operation.
-// If not specified, defaults to ""
-func PreventGradientMessage(value string) PreventGradientAttr {
+// value: If True, `given_y_min` and `given_y_min`
+// and `given_y_max` are used as the output range. Otherwise,
+// the implementation computes the output range.
+// If not specified, defaults to false
+func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
 	return func(m optionalAttr) {
-		m["message"] = value
+		m["output_range_given"] = value
 	}
 }
 
-// An identity op that triggers an error if a gradient is requested.
+// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
 //
-// When executed in a graph, this op outputs its input tensor as-is.
+// value: Output in `y_min` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["given_y_min"] = value
+	}
+}
+
+// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
 //
-// When building ops to compute gradients, the TensorFlow gradient system
-// will return an error when trying to lookup the gradient of this op,
-// because no gradient must ever be registered for this function.  This
-// op exists to prevent subtle bugs from silently returning unimplemented
-// gradients in some corner cases.
+// value: Output in `y_max` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["given_y_max"] = value
+	}
+}
+
+// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
+//
+// value: A small float number to avoid dividing by 0.
+// If not specified, defaults to 1e-05
+func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["variance_epsilon"] = value
+	}
+}
+
+// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
+//
+// value: Minimum value of `y_max - y_min`
+// If not specified, defaults to 0.001
+func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["min_separation"] = value
+	}
+}
+
+// Quantized Instance normalization.
 //
 // Arguments:
-//	input: any tensor.
+//	x: A 4D input Tensor.
+//	x_min: The value represented by the lowest quantized input.
+//	x_max: The value represented by the highest quantized input.
 //
-// Returns the same input tensor.
-func PreventGradient(scope *Scope, input tf.Output, optional ...PreventGradientAttr) (output tf.Output) {
+// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
+func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26928,96 +26856,185 @@ func PreventGradient(scope *Scope, input tf.Output, optional ...PreventGradientA
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "PreventGradient",
+		Type: "QuantizedInstanceNorm",
 		Input: []tf.Input{
-			input,
+			x, x_min, x_max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Checks a tensor for NaN and Inf values.
+// Returns the diagonal part of the tensor.
 //
-// When run, reports an `InvalidArgument` error if `tensor` has any values
-// that are not a number (NaN) or infinity (Inf). Otherwise, passes `tensor` as-is.
+// This operation returns a tensor with the `diagonal` part
+// of the `input`. The `diagonal` part is computed as follows:
+//
+// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
+// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
+//
+// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
+//
+// For example:
+//
+// ```
+// # 'input' is [[1, 0, 0, 0]
+//               [0, 2, 0, 0]
+//               [0, 0, 3, 0]
+//               [0, 0, 0, 4]]
+//
+// tf.diag_part(input) ==> [1, 2, 3, 4]
+// ```
 //
 // Arguments:
+//	input: Rank k tensor where k is even and not zero.
 //
-//	message: Prefix of the error message.
-func CheckNumerics(scope *Scope, tensor tf.Output, message string) (output tf.Output) {
+// Returns The extracted diagonal.
+func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"message": message}
 	opspec := tf.OpSpec{
-		Type: "CheckNumerics",
+		Type: "DiagPart",
 		Input: []tf.Input{
-			tensor,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Shuffle dimensions of x according to a permutation and conjugate the result.
+// Returns the element-wise max of two SparseTensors.
 //
-// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
-//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
-//   `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
-func ConjugateTranspose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
+// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
+//
+// Arguments:
+//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, in the canonical lexicographic ordering.
+//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
+//	a_shape: 1-D.  Shape of the input SparseTensor.
+//	b_indices: counterpart to `a_indices` for the other operand.
+//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
+//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
+//
+// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
+func SparseSparseMaximum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ConjugateTranspose",
+		Type: "SparseSparseMaximum",
+		Input: []tf.Input{
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Returns a batched matrix tensor with new batched diagonal values.
+//
+// Given `input` and `diagonal`, this operation returns a tensor with the
+// same shape and values as `input`, except for the main diagonal of the
+// innermost matrices.  These will be overwritten by the values in `diagonal`.
+//
+// The output is computed as follows:
+//
+// Assume `input` has `k+1` dimensions `[I, J, K, ..., M, N]` and `diagonal` has
+// `k` dimensions `[I, J, K, ..., min(M, N)]`.  Then the output is a
+// tensor of rank `k+1` with dimensions `[I, J, K, ..., M, N]` where:
+//
+//   * `output[i, j, k, ..., m, n] = diagonal[i, j, k, ..., n]` for `m == n`.
+//   * `output[i, j, k, ..., m, n] = input[i, j, k, ..., m, n]` for `m != n`.
+//
+// Arguments:
+//	input: Rank `k+1`, where `k >= 1`.
+//	diagonal: Rank `k`, where `k >= 1`.
+//
+// Returns Rank `k+1`, with `output.shape = input.shape`.
+func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixSetDiag",
 		Input: []tf.Input{
-			x, perm,
+			input, diagonal,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// UniqueV2Attr is an optional argument to UniqueV2.
-type UniqueV2Attr func(optionalAttr)
+// EditDistanceAttr is an optional argument to EditDistance.
+type EditDistanceAttr func(optionalAttr)
 
-// UniqueV2OutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueV2OutIdx(value tf.DataType) UniqueV2Attr {
+// EditDistanceNormalize sets the optional normalize attribute to value.
+//
+// value: boolean (if true, edit distances are normalized by length of truth).
+//
+// The output is:
+// If not specified, defaults to true
+func EditDistanceNormalize(value bool) EditDistanceAttr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["normalize"] = value
 	}
 }
 
-// Finds unique elements in a 1-D tensor.
+// Computes the (possibly normalized) Levenshtein Edit Distance.
 //
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. In other words:
+// The inputs are variable-length sequences provided by SparseTensors
+//   (hypothesis_indices, hypothesis_values, hypothesis_shape)
+// and
+//   (truth_indices, truth_values, truth_shape).
 //
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+// The inputs are:
 //
-// For example:
+// Arguments:
+//	hypothesis_indices: The indices of the hypothesis list SparseTensor.
+// This is an N x R int64 matrix.
+//	hypothesis_values: The values of the hypothesis list SparseTensor.
+// This is an N-length vector.
+//	hypothesis_shape: The shape of the hypothesis list SparseTensor.
+// This is an R-length vector.
+//	truth_indices: The indices of the truth list SparseTensor.
+// This is an M x R int64 matrix.
+//	truth_values: The values of the truth list SparseTensor.
+// This is an M-length vector.
+//	truth_shape: truth indices, vector.
 //
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx = unique(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// ```
+// Returns A dense float tensor with rank R - 1.
 //
-// Arguments:
-//	x: A `Tensor`.
-//	axis: A `Tensor` of type `int64` (default: 0). The axis of the Tensor to
-// find the unique elements.
+// For the example input:
 //
-// Returns A `Tensor`. Unique elements along the `axis` of `Tensor` x.A 1-D Tensor. Has the same type as x that contains the index of each
-// value of x in the output y.
-func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Attr) (y tf.Output, idx tf.Output) {
+//     // hypothesis represents a 2x1 matrix with variable-length values:
+//     //   (0,0) = ["a"]
+//     //   (1,0) = ["b"]
+//     hypothesis_indices = [[0, 0, 0],
+//                           [1, 0, 0]]
+//     hypothesis_values = ["a", "b"]
+//     hypothesis_shape = [2, 1, 1]
+//
+//     // truth represents a 2x2 matrix with variable-length values:
+//     //   (0,0) = []
+//     //   (0,1) = ["a"]
+//     //   (1,0) = ["b", "c"]
+//     //   (1,1) = ["a"]
+//     truth_indices = [[0, 1, 0],
+//                      [1, 0, 0],
+//                      [1, 0, 1],
+//                      [1, 1, 0]]
+//     truth_values = ["a", "b", "c", "a"]
+//     truth_shape = [2, 2, 2]
+//     normalize = true
+//
+// The output will be:
+//
+//     // output is a 2x2 matrix with edit distances normalized by truth lengths.
+//     output = [[inf, 1.0],  // (0,0): no truth, (0,1): no hypothesis
+//               [0.5, 1.0]]  // (1,0): addition, (1,1): no hypothesis
+func EditDistance(scope *Scope, hypothesis_indices tf.Output, hypothesis_values tf.Output, hypothesis_shape tf.Output, truth_indices tf.Output, truth_values tf.Output, truth_shape tf.Output, optional ...EditDistanceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27026,187 +27043,245 @@ func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Att
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UniqueV2",
+		Type: "EditDistance",
 		Input: []tf.Input{
-			x, axis,
+			hypothesis_indices, hypothesis_values, hypothesis_shape, truth_indices, truth_values, truth_shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Return a slice from 'input'.
+// Gather slices from `params` into a Tensor with shape specified by `indices`.
 //
-// The output tensor is a tensor with dimensions described by 'size'
-// whose values are extracted from 'input' starting at the offsets in
-// 'begin'.
+// `indices` is an K-dimensional integer tensor, best thought of as a
+// (K-1)-dimensional tensor of indices into `params`, where each element defines a
+// slice of `params`:
 //
-// *Requirements*:
-//   0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)
+//     output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
+//
+// Whereas in @{tf.gather} `indices` defines slices into the first
+// dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
+// first `N` dimensions of `params`, where `N = indices.shape[-1]`.
+//
+// The last dimension of `indices` can be at most the rank of
+// `params`:
+//
+//     indices.shape[-1] <= params.rank
+//
+// The last dimension of `indices` corresponds to elements
+// (if `indices.shape[-1] == params.rank`) or slices
+// (if `indices.shape[-1] < params.rank`) along dimension `indices.shape[-1]`
+// of `params`.  The output tensor has shape
+//
+//     indices.shape[:-1] + params.shape[indices.shape[-1]:]
+//
+// Some examples below.
+//
+// Simple indexing into a matrix:
+//
+// ```python
+//     indices = [[0, 0], [1, 1]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = ['a', 'd']
+// ```
+//
+// Slice indexing into a matrix:
+//
+// ```python
+//     indices = [[1], [0]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [['c', 'd'], ['a', 'b']]
+// ```
+//
+// Indexing into a 3-tensor:
+//
+// ```python
+//     indices = [[1]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[['a1', 'b1'], ['c1', 'd1']]]
+//
+//
+//     indices = [[0, 1], [1, 0]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [['c0', 'd0'], ['a1', 'b1']]
+//
+//
+//     indices = [[0, 0, 1], [1, 0, 1]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = ['b0', 'b1']
+// ```
+//
+// Batched indexing into a matrix:
+//
+// ```python
+//     indices = [[[0, 0]], [[0, 1]]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [['a'], ['b']]
+// ```
+//
+// Batched slice indexing into a matrix:
+//
+// ```python
+//     indices = [[[1]], [[0]]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [[['c', 'd']], [['a', 'b']]]
+// ```
+//
+// Batched indexing into a 3-tensor:
+//
+// ```python
+//     indices = [[[1]], [[0]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[[['a1', 'b1'], ['c1', 'd1']]],
+//               [[['a0', 'b0'], ['c0', 'd0']]]]
+//
+//     indices = [[[0, 1], [1, 0]], [[0, 0], [1, 1]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[['c0', 'd0'], ['a1', 'b1']],
+//               [['a0', 'b0'], ['c1', 'd1']]]
+//
+//
+//     indices = [[[0, 0, 1], [1, 0, 1]], [[0, 1, 1], [1, 1, 0]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [['b0', 'b1'], ['d0', 'c1']]
+// ```
 //
 // Arguments:
+//	params: The tensor from which to gather values.
+//	indices: Index tensor.
 //
-//	begin: begin[i] specifies the offset into the 'i'th dimension of
-// 'input' to slice from.
-//	size: size[i] specifies the number of elements of the 'i'th dimension
-// of 'input' to slice. If size[i] is -1, all remaining elements in dimension
-// i are included in the slice (i.e. this is equivalent to setting
-// size[i] = input.dim_size(i) - begin[i]).
-func Slice(scope *Scope, input tf.Output, begin tf.Output, size tf.Output) (output tf.Output) {
+// Returns Values from `params` gathered from indices given by `indices`, with
+// shape `indices.shape[:-1] + params.shape[indices.shape[-1]:]`.
+func GatherNd(scope *Scope, params tf.Output, indices tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Slice",
-		Input: []tf.Input{
-			input, begin, size,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StridedSliceGradAttr is an optional argument to StridedSliceGrad.
-type StridedSliceGradAttr func(optionalAttr)
-
-// StridedSliceGradBeginMask sets the optional begin_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradBeginMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["begin_mask"] = value
-	}
-}
-
-// StridedSliceGradEndMask sets the optional end_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradEndMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["end_mask"] = value
-	}
-}
-
-// StridedSliceGradEllipsisMask sets the optional ellipsis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradEllipsisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["ellipsis_mask"] = value
-	}
-}
-
-// StridedSliceGradNewAxisMask sets the optional new_axis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradNewAxisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["new_axis_mask"] = value
-	}
-}
-
-// StridedSliceGradShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradShrinkAxisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["shrink_axis_mask"] = value
+		Type: "GatherNd",
+		Input: []tf.Input{
+			params, indices,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns the gradient of `StridedSlice`.
-//
-// Since `StridedSlice` cuts out pieces of its `input` which is size
-// `shape`, its gradient will have the same shape (which is passed here
-// as `shape`). The gradient will be zero in any element that the slice
-// does not select.
+// Eagerly executes a python function to compute func(input)->output. The
 //
-// Arguments are the same as StridedSliceGrad with the exception that
-// `dy` is the input gradient to be propagated and `shape` is the
-// shape of `StridedSlice`'s `input`.
-func StridedSliceGrad(scope *Scope, shape tf.Output, begin tf.Output, end tf.Output, strides tf.Output, dy tf.Output, optional ...StridedSliceGradAttr) (output tf.Output) {
+// semantics of the input, output, and attributes are the same as those for
+// PyFunc.
+func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataType) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"token": token, "Tout": Tout}
 	opspec := tf.OpSpec{
-		Type: "StridedSliceGrad",
+		Type: "EagerPyFunc",
 		Input: []tf.Input{
-			shape, begin, end, strides, dy,
+			tf.OutputList(input),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("EagerPyFunc", err)
+		return
+	}
+	return output
 }
 
-// Returns the gradient of `Tile`.
+// Stops gradient computation.
 //
-// DEPRECATED at GraphDef version 3: TileGrad has been replaced with reduce_sum
+// When executed in a graph, this op outputs its input tensor as-is.
 //
-// Since `Tile` takes an input and repeats the input `multiples` times
-// along each dimension, `TileGrad` takes in `multiples` and aggregates
-// each repeated tile of `input` into `output`.
-func TileGrad(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
+// When building ops to compute gradients, this op prevents the contribution of
+// its inputs to be taken into account.  Normally, the gradient generator adds ops
+// to a graph to compute the derivatives of a specified 'loss' by recursively
+// finding out inputs that contributed to its computation.  If you insert this op
+// in the graph it inputs are masked from the gradient generator.  They are not
+// taken into account for computing gradients.
+//
+// This is useful any time you want to compute a value with TensorFlow but need
+// to pretend that the value was a constant. Some examples include:
+//
+// *  The *EM* algorithm where the *M-step* should not involve backpropagation
+//    through the output of the *E-step*.
+// *  Contrastive divergence training of Boltzmann machines where, when
+//    differentiating the energy function, the training must not backpropagate
+//    through the graph that generated the samples from the model.
+// *  Adversarial training, where no backprop should happen through the adversarial
+//    example generation process.
+func StopGradient(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TileGrad",
+		Type: "StopGradient",
 		Input: []tf.Input{
-			input, multiples,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
-type QuantizeAndDequantizeAttr func(optionalAttr)
-
-// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
-
-// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["num_bits"] = value
+// Computes asin of x element-wise.
+func Asin(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
-// If not specified, defaults to false
-func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["range_given"] = value
+	opspec := tf.OpSpec{
+		Type: "Asin",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["input_min"] = value
-	}
-}
+// PreventGradientAttr is an optional argument to PreventGradient.
+type PreventGradientAttr func(optionalAttr)
 
-// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
+// PreventGradientMessage sets the optional message attribute to value.
+//
+// value: Will be printed in the error when anyone tries to differentiate
+// this operation.
+// If not specified, defaults to ""
+func PreventGradientMessage(value string) PreventGradientAttr {
 	return func(m optionalAttr) {
-		m["input_max"] = value
+		m["message"] = value
 	}
 }
 
-// Use QuantizeAndDequantizeV2 instead.
+// An identity op that triggers an error if a gradient is requested.
 //
-// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
-func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
+// When executed in a graph, this op outputs its input tensor as-is.
+//
+// When building ops to compute gradients, the TensorFlow gradient system
+// will return an error when trying to lookup the gradient of this op,
+// because no gradient must ever be registered for this function.  This
+// op exists to prevent subtle bugs from silently returning unimplemented
+// gradients in some corner cases.
+//
+// Arguments:
+//	input: any tensor.
+//
+// Returns the same input tensor.
+func PreventGradient(scope *Scope, input tf.Output, optional ...PreventGradientAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27215,7 +27290,7 @@ func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAn
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantize",
+		Type: "PreventGradient",
 		Input: []tf.Input{
 			input,
 		},
@@ -27225,171 +27300,86 @@ func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAn
 	return op.Output(0)
 }
 
-// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
-type QueueDequeueV2Attr func(optionalAttr)
-
-// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
-//
-// value: If the queue is empty, this operation will block for up to
-// timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
-	}
-}
-
-// Dequeues a tuple of one or more tensors from the given queue.
-//
-// This operation has k outputs, where k is the number of components
-// in the tuples stored in the given queue, and output i is the ith
-// component of the dequeued tuple.
+// Checks a tensor for NaN and Inf values.
 //
-// N.B. If the queue is empty, this operation will block until an element
-// has been dequeued (or 'timeout_ms' elapses, if specified).
+// When run, reports an `InvalidArgument` error if `tensor` has any values
+// that are not a number (NaN) or infinity (Inf). Otherwise, passes `tensor` as-is.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	component_types: The type of each component in a tuple.
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
+//	message: Prefix of the error message.
+func CheckNumerics(scope *Scope, tensor tf.Output, message string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"message": message}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueV2",
+		Type: "CheckNumerics",
 		Input: []tf.Input{
-			handle,
+			tensor,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueV2", err)
-		return
-	}
-	return components
+	return op.Output(0)
 }
 
-// Returns locations of nonzero / true values in a tensor.
-//
-// This operation returns the coordinates of true elements in `condition`. The
-// coordinates are returned in a 2-D tensor where the first dimension (rows)
-// represents the number of true elements, and the second dimension (columns)
-// represents the coordinates of the true elements. Keep in mind, the shape of
-// the output tensor can vary depending on how many true values there are in
-// `condition`. Indices are output in row-major order.
-//
-// For example:
-//
-// ```
-// # 'input' tensor is [[True, False]
-// #                    [True, False]]
-// # 'input' has two true values, so output has two coordinates.
-// # 'input' has rank of 2, so coordinates have two indices.
-// where(input) ==> [[0, 0],
-//                   [1, 0]]
-//
-// # `condition` tensor is [[[True, False]
-// #                     [True, False]]
-// #                    [[False, True]
-// #                     [False, True]]
-// #                    [[False, False]
-// #                     [False, True]]]
-// # 'input' has 5 true values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-//
-// # `condition` tensor is [[[1.5,  0.0]
-// #                     [-0.5, 0.0]]
-// #                    [[0.0,  0.25]
-// #                     [0.0,  0.75]]
-// #                    [[0.0,  0.0]
-// #                     [0.0,  0.01]]]
-// # 'input' has 5 nonzero values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-//
-// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
-// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
-// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-// ```
-func Where(scope *Scope, condition tf.Output) (index tf.Output) {
+// Shuffle dimensions of x according to a permutation and conjugate the result.
+//
+// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+//   `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
+func ConjugateTranspose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Where",
+		Type: "ConjugateTranspose",
 		Input: []tf.Input{
-			condition,
+			x, perm,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
-type DataFormatDimMapAttr func(optionalAttr)
+// UniqueV2Attr is an optional argument to UniqueV2.
+type UniqueV2Attr func(optionalAttr)
 
-// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+// UniqueV2OutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueV2OutIdx(value tf.DataType) UniqueV2Attr {
 	return func(m optionalAttr) {
-		m["src_format"] = value
+		m["out_idx"] = value
 	}
 }
 
-// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+// Finds unique elements in a 1-D tensor.
 //
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["dst_format"] = value
-	}
-}
-
-// Returns the dimension index in the destination data format given the one in
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. In other words:
 //
-// the source data format.
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+//
+// For example:
+//
+// ```
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx = unique(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+// ```
 //
 // Arguments:
-//	x: A Tensor with each element as a dimension index in source data format.
-// Must be in the range [-4, 4).
+//	x: A `Tensor`.
+//	axis: A `Tensor` of type `int64` (default: 0). The axis of the Tensor to
+// find the unique elements.
 //
-// Returns A Tensor with each element as a dimension index in destination data format.
-func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+// Returns A `Tensor`. Unique elements along the `axis` of `Tensor` x.A 1-D Tensor. Has the same type as x that contains the index of each
+// value of x in the output y.
+func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Attr) (y tf.Output, idx tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27398,166 +27388,187 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DataFormatDimMap",
+		Type: "UniqueV2",
 		Input: []tf.Input{
-			x,
+			x, axis,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Return the shape of s0 op s1 with broadcast.
+// Return a slice from 'input'.
 //
-// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
-// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
-func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+// The output tensor is a tensor with dimensions described by 'size'
+// whose values are extracted from 'input' starting at the offsets in
+// 'begin'.
+//
+// *Requirements*:
+//   0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)
+//
+// Arguments:
+//
+//	begin: begin[i] specifies the offset into the 'i'th dimension of
+// 'input' to slice from.
+//	size: size[i] specifies the number of elements of the 'i'th dimension
+// of 'input' to slice. If size[i] is -1, all remaining elements in dimension
+// i are included in the slice (i.e. this is equivalent to setting
+// size[i] = input.dim_size(i) - begin[i]).
+func Slice(scope *Scope, input tf.Output, begin tf.Output, size tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BroadcastArgs",
+		Type: "Slice",
 		Input: []tf.Input{
-			s0, s1,
+			input, begin, size,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
+// StridedSliceGradAttr is an optional argument to StridedSliceGrad.
+type StridedSliceGradAttr func(optionalAttr)
+
+// StridedSliceGradBeginMask sets the optional begin_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradBeginMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["begin_mask"] = value
+	}
+}
+
+// StridedSliceGradEndMask sets the optional end_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradEndMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["end_mask"] = value
+	}
+}
+
+// StridedSliceGradEllipsisMask sets the optional ellipsis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradEllipsisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["ellipsis_mask"] = value
+	}
+}
+
+// StridedSliceGradNewAxisMask sets the optional new_axis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradNewAxisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["new_axis_mask"] = value
+	}
+}
+
+// StridedSliceGradShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradShrinkAxisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["shrink_axis_mask"] = value
+	}
+}
+
+// Returns the gradient of `StridedSlice`.
 //
-// This is typically used by gradient computations for a broadcasting operation.
-func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
+// Since `StridedSlice` cuts out pieces of its `input` which is size
+// `shape`, its gradient will have the same shape (which is passed here
+// as `shape`). The gradient will be zero in any element that the slice
+// does not select.
+//
+// Arguments are the same as StridedSliceGrad with the exception that
+// `dy` is the input gradient to be propagated and `shape` is the
+// shape of `StridedSlice`'s `input`.
+func StridedSliceGrad(scope *Scope, shape tf.Output, begin tf.Output, end tf.Output, strides tf.Output, dy tf.Output, optional ...StridedSliceGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "BroadcastGradientArgs",
+		Type: "StridedSliceGrad",
 		Input: []tf.Input{
-			s0, s1,
+			shape, begin, end, strides, dy,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Pads a tensor with mirrored values.
-//
-// This operation pads a `input` with mirrored values according to the `paddings`
-// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is
-// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many values to add before the contents of `input` in that dimension, and
-// `paddings[D, 1]` indicates how many values to add after the contents of `input`
-// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater
-// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true
-// (if false, respectively).
-//
-// The padded size of each dimension D of the output is:
-//
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
-//
-// For example:
-//
-// ```
-// # 't' is [[1, 2, 3], [4, 5, 6]].
-// # 'paddings' is [[1, 1]], [2, 2]].
-// # 'mode' is SYMMETRIC.
-// # rank of 't' is 2.
-// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2]
-//                       [2, 1, 1, 2, 3, 3, 2]
-//                       [5, 4, 4, 5, 6, 6, 5]
-//                       [5, 4, 4, 5, 6, 6, 5]]
-// ```
+// Returns the gradient of `Tile`.
 //
-// Arguments:
-//	input: The input tensor to be padded.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions
-// do not include the borders, while in symmetric mode the padded regions
-// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings`
-// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and
-// it is `[1, 2, 3, 3, 2]` in symmetric mode.
+// DEPRECATED at GraphDef version 3: TileGrad has been replaced with reduce_sum
 //
-// Returns The padded tensor.
-func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
+// Since `Tile` takes an input and repeats the input `multiples` times
+// along each dimension, `TileGrad` takes in `multiples` and aggregates
+// each repeated tile of `input` into `output`.
+func TileGrad(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"mode": mode}
 	opspec := tf.OpSpec{
-		Type: "MirrorPad",
+		Type: "TileGrad",
 		Input: []tf.Input{
-			input, paddings,
+			input, multiples,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// A placeholder op for a value that will be fed into the computation.
-//
-// DEPRECATED at GraphDef version 23: Placeholder now behaves the same as PlaceholderV2.
-//
-// N.B. This operation will fail with an error if it is executed. It is
-// intended as a way to represent a value that will always be fed, and to
-// provide attrs that enable the fed value to be checked at runtime.
-//
-// Arguments:
-//	dtype: The type of elements in the tensor.
-//	shape: The shape of the tensor. The shape can be any partially-specified
-// shape.  To be unconstrained, pass in a shape with unknown rank.
-//
-// Returns A placeholder tensor that must be replaced using the feed mechanism.
-func PlaceholderV2(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
+type QuantizeAndDequantizeAttr func(optionalAttr)
+
+// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["signed_input"] = value
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
-	opspec := tf.OpSpec{
-		Type: "PlaceholderV2",
+}
 
-		Attrs: attrs,
+// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["num_bits"] = value
+	}
+}
+
+// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
+// If not specified, defaults to false
+func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["range_given"] = value
+	}
+}
+
+// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["input_min"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
-type ResourceApplyAdadeltaAttr func(optionalAttr)
-
-// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
-//
-// value: If True, updating of the var, accum and update_accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
+// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["input_max"] = value
 	}
 }
 
-// Update '*var' according to the adadelta scheme.
-//
-// accum = rho() * accum + (1 - rho()) * grad.square();
-// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
-// update_accum = rho() * update_accum + (1 - rho()) * update.square();
-// var -= update;
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	accum_update: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay factor. Must be a scalar.
-//	epsilon: Constant factor. Must be a scalar.
-//	grad: The gradient.
+// Use QuantizeAndDequantizeV2 instead.
 //
-// Returns the created operation.
-func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
+// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
+func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27566,322 +27577,311 @@ func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdadelta",
+		Type: "QuantizeAndDequantize",
 		Input: []tf.Input{
-			var_, accum, accum_update, lr, rho, epsilon, grad,
+			input,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SqueezeAttr is an optional argument to Squeeze.
-type SqueezeAttr func(optionalAttr)
+// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
+type QueueDequeueV2Attr func(optionalAttr)
 
-// SqueezeAxis sets the optional axis attribute to value.
-//
-// value: If specified, only squeezes the dimensions listed. The dimension
-// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
-// be in the range `[-rank(input), rank(input))`.
-// If not specified, defaults to <>
+// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// REQUIRES: len(value) >= 0
-func SqueezeAxis(value []int64) SqueezeAttr {
+// value: If the queue is empty, this operation will block for up to
+// timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
 	return func(m optionalAttr) {
-		m["squeeze_dims"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Removes dimensions of size 1 from the shape of a tensor.
-//
-// Given a tensor `input`, this operation returns a tensor of the same type with
-// all dimensions of size 1 removed. If you don't want to remove all size 1
-// dimensions, you can remove specific size 1 dimensions by specifying
-// `axis`.
-//
-// For example:
-//
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t)) ==> [2, 3]
-// ```
+// Dequeues a tuple of one or more tensors from the given queue.
 //
-// Or, to remove specific size 1 dimensions:
+// This operation has k outputs, where k is the number of components
+// in the tuples stored in the given queue, and output i is the ith
+// component of the dequeued tuple.
 //
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
-// ```
+// N.B. If the queue is empty, this operation will block until an element
+// has been dequeued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	input: The `input` to squeeze.
+//	handle: The handle to a queue.
+//	component_types: The type of each component in a tuple.
 //
-// Returns Contains the same data as `input`, but has one or more dimensions of
-// size 1 removed.
-func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Squeeze",
+		Type: "QueueDequeueV2",
 		Input: []tf.Input{
-			input,
+			handle,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueV2", err)
+		return
+	}
+	return components
 }
 
-// SpaceToBatch for N-D tensors of type T.
-//
-// This operation divides "spatial" dimensions `[1, ..., M]` of the input into a
-// grid of blocks of shape `block_shape`, and interleaves these blocks with the
-// "batch" dimension (0) such that in the output, the spatial dimensions
-// `[1, ..., M]` correspond to the position within the grid, and the batch
-// dimension combines both the position within a spatial block and the original
-// batch position.  Prior to division into blocks, the spatial dimensions of the
-// input are optionally zero padded according to `paddings`.  See below for a
-// precise description.
-//
-// Arguments:
-//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
-// where spatial_shape has `M` dimensions.
-//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
-//	paddings: 2-D with shape `[M, 2]`, all values must be >= 0.
-//   `paddings[i] = [pad_start, pad_end]` specifies the padding for input dimension
-//   `i + 1`, which corresponds to spatial dimension `i`.  It is required that
-//   `block_shape[i]` divides `input_shape[i + 1] + pad_start + pad_end`.
-//
-// This operation is equivalent to the following steps:
-//
-// 1. Zero-pad the start and end of dimensions `[1, ..., M]` of the
-//    input according to `paddings` to produce `padded` of shape `padded_shape`.
-//
-// 2. Reshape `padded` to `reshaped_padded` of shape:
-//
-//      [batch] +
-//      [padded_shape[1] / block_shape[0],
-//        block_shape[0],
-//       ...,
-//       padded_shape[M] / block_shape[M-1],
-//       block_shape[M-1]] +
-//      remaining_shape
-//
-// 3. Permute dimensions of `reshaped_padded` to produce
-//    `permuted_reshaped_padded` of shape:
-//
-//      block_shape +
-//      [batch] +
-//      [padded_shape[1] / block_shape[0],
-//       ...,
-//       padded_shape[M] / block_shape[M-1]] +
-//      remaining_shape
-//
-// 4. Reshape `permuted_reshaped_padded` to flatten `block_shape` into the batch
-//    dimension, producing an output tensor of shape:
-//
-//      [batch * prod(block_shape)] +
-//      [padded_shape[1] / block_shape[0],
-//       ...,
-//       padded_shape[M] / block_shape[M-1]] +
-//      remaining_shape
-//
-// Some examples:
-//
-// (1) For the following input of shape `[1, 2, 2, 1]`, `block_shape = [2, 2]`, and
-//     `paddings = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// The output tensor has shape `[4, 1, 1, 1]` and value:
-//
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// (2) For the following input of shape `[1, 2, 2, 3]`, `block_shape = [2, 2]`, and
-//     `paddings = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// The output tensor has shape `[4, 1, 1, 3]` and value:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// (3) For the following input of shape `[1, 4, 4, 1]`, `block_shape = [2, 2]`, and
-//     `paddings = [[0, 0], [0, 0]]`:
+// Returns locations of nonzero / true values in a tensor.
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]],
-//       [[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
+// This operation returns the coordinates of true elements in `condition`. The
+// coordinates are returned in a 2-D tensor where the first dimension (rows)
+// represents the number of true elements, and the second dimension (columns)
+// represents the coordinates of the true elements. Keep in mind, the shape of
+// the output tensor can vary depending on how many true values there are in
+// `condition`. Indices are output in row-major order.
 //
-// The output tensor has shape `[4, 2, 2, 1]` and value:
+// For example:
 //
 // ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-//
-// (4) For the following input of shape `[2, 2, 4, 1]`, block_shape = `[2, 2]`, and
-//     paddings = `[[0, 0], [2, 0]]`:
+// # 'input' tensor is [[True, False]
+// #                    [True, False]]
+// # 'input' has two true values, so output has two coordinates.
+// # 'input' has rank of 2, so coordinates have two indices.
+// where(input) ==> [[0, 0],
+//                   [1, 0]]
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
+// # `condition` tensor is [[[True, False]
+// #                     [True, False]]
+// #                    [[False, True]
+// #                     [False, True]]
+// #                    [[False, False]
+// #                     [False, True]]]
+// # 'input' has 5 true values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 //
-// The output tensor has shape `[8, 1, 3, 1]` and value:
+// # `condition` tensor is [[[1.5,  0.0]
+// #                     [-0.5, 0.0]]
+// #                    [[0.0,  0.25]
+// #                     [0.0,  0.75]]
+// #                    [[0.0,  0.0]
+// #                     [0.0,  0.01]]]
+// # 'input' has 5 nonzero values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 //
+// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
+// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
+// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 // ```
-// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
-//      [[[0], [2], [4]]], [[[0], [10], [12]]],
-//      [[[0], [5], [7]]], [[[0], [13], [15]]],
-//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
-// ```
-//
-// Among others, this operation is useful for reducing atrous convolution into
-// regular convolution.
-func SpaceToBatchND(scope *Scope, input tf.Output, block_shape tf.Output, paddings tf.Output) (output tf.Output) {
+func Where(scope *Scope, condition tf.Output) (index tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SpaceToBatchND",
+		Type: "Where",
 		Input: []tf.Input{
-			input, block_shape, paddings,
+			condition,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SpaceToBatch for 4-D tensors of type T.
-//
-// This is a legacy version of the more general SpaceToBatchND.
-//
-// Zero-pads and then rearranges (permutes) blocks of spatial data into batch.
-// More specifically, this op outputs a copy of the input tensor where values from
-// the `height` and `width` dimensions are moved to the `batch` dimension. After
-// the zero-padding, both `height` and `width` of the input must be divisible by the
-// block size.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, height, width, depth]`.
-//	paddings: 2-D tensor of non-negative integers with shape `[2, 2]`. It specifies
-//   the padding of the input with zeros across the spatial dimensions as follows:
-//
-//       paddings = [[pad_top, pad_bottom], [pad_left, pad_right]]
-//
-//   The effective spatial dimensions of the zero-padded input tensor will be:
-//
-//       height_pad = pad_top + height + pad_bottom
-//       width_pad = pad_left + width + pad_right
-//
-// The attr `block_size` must be greater than one. It indicates the block size.
-//
-//   * Non-overlapping blocks of size `block_size x block size` in the height and
-//     width dimensions are rearranged into the batch dimension at each location.
-//   * The batch of the output tensor is `batch * block_size * block_size`.
-//   * Both height_pad and width_pad must be divisible by block_size.
-//
-// The shape of the output will be:
-//
-//     [batch*block_size*block_size, height_pad/block_size, width_pad/block_size,
-//      depth]
-//
-// Some examples:
+// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
+type DataFormatDimMapAttr func(optionalAttr)
+
+// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
 //
-// (1) For the following input of shape `[1, 2, 2, 1]` and block_size of 2:
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["src_format"] = value
+	}
+}
+
+// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
 //
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the dimension index in the destination data format given the one in
 //
-// The output tensor has shape `[4, 1, 1, 1]` and value:
+// the source data format.
 //
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
+// Arguments:
+//	x: A Tensor with each element as a dimension index in source data format.
+// Must be in the range [-4, 4).
 //
-// (2) For the following input of shape `[1, 2, 2, 3]` and block_size of 2:
+// Returns A Tensor with each element as a dimension index in destination data format.
+func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DataFormatDimMap",
+		Input: []tf.Input{
+			x,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Return the shape of s0 op s1 with broadcast.
 //
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
+// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BroadcastArgs",
+		Input: []tf.Input{
+			s0, s1,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
 //
-// The output tensor has shape `[4, 1, 1, 3]` and value:
+// This is typically used by gradient computations for a broadcasting operation.
+func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BroadcastGradientArgs",
+		Input: []tf.Input{
+			s0, s1,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Pads a tensor with mirrored values.
 //
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
+// This operation pads a `input` with mirrored values according to the `paddings`
+// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is
+// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many values to add before the contents of `input` in that dimension, and
+// `paddings[D, 1]` indicates how many values to add after the contents of `input`
+// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater
+// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true
+// (if false, respectively).
 //
-// (3) For the following input of shape `[1, 4, 4, 1]` and block_size of 2:
+// The padded size of each dimension D of the output is:
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]],
-//       [[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
 //
-// The output tensor has shape `[4, 2, 2, 1]` and value:
+// For example:
 //
 // ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
+// # 't' is [[1, 2, 3], [4, 5, 6]].
+// # 'paddings' is [[1, 1]], [2, 2]].
+// # 'mode' is SYMMETRIC.
+// # rank of 't' is 2.
+// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2]
+//                       [2, 1, 1, 2, 3, 3, 2]
+//                       [5, 4, 4, 5, 6, 6, 5]
+//                       [5, 4, 4, 5, 6, 6, 5]]
 // ```
 //
-// (4) For the following input of shape `[2, 2, 4, 1]` and block_size of 2:
+// Arguments:
+//	input: The input tensor to be padded.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions
+// do not include the borders, while in symmetric mode the padded regions
+// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings`
+// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and
+// it is `[1, 2, 3, 3, 2]` in symmetric mode.
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
+// Returns The padded tensor.
+func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"mode": mode}
+	opspec := tf.OpSpec{
+		Type: "MirrorPad",
+		Input: []tf.Input{
+			input, paddings,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// A placeholder op for a value that will be fed into the computation.
 //
-// The output tensor has shape `[8, 1, 2, 1]` and value:
+// DEPRECATED at GraphDef version 23: Placeholder now behaves the same as PlaceholderV2.
 //
-// ```
-// x = [[[[1], [3]]], [[[9], [11]]], [[[2], [4]]], [[[10], [12]]],
-//      [[[5], [7]]], [[[13], [15]]], [[[6], [8]]], [[[14], [16]]]]
-// ```
+// N.B. This operation will fail with an error if it is executed. It is
+// intended as a way to represent a value that will always be fed, and to
+// provide attrs that enable the fed value to be checked at runtime.
 //
-// Among others, this operation is useful for reducing atrous convolution into
-// regular convolution.
+// Arguments:
+//	dtype: The type of elements in the tensor.
+//	shape: The shape of the tensor. The shape can be any partially-specified
+// shape.  To be unconstrained, pass in a shape with unknown rank.
 //
-func SpaceToBatch(scope *Scope, input tf.Output, paddings tf.Output, block_size int64) (output tf.Output) {
+// Returns A placeholder tensor that must be replaced using the feed mechanism.
+func PlaceholderV2(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"block_size": block_size}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
 	opspec := tf.OpSpec{
-		Type: "SpaceToBatch",
-		Input: []tf.Input{
-			input, paddings,
-		},
+		Type: "PlaceholderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-- 
GitLab


From 712539de21d9d122f27253667f84d63c8dcf8aaa Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Thu, 22 Mar 2018 15:49:57 -0700
Subject: [PATCH 387/960] Automated g4 rollback of changelist 190127730

PiperOrigin-RevId: 190139303
---
 .../xla/service/algebraic_simplifier.cc       | 22 ++------
 .../xla/service/algebraic_simplifier_test.cc  | 51 -------------------
 2 files changed, 5 insertions(+), 68 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 88f6ff0a07..971c2935c8 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1121,10 +1121,10 @@ bool OutputIsSubsetOfOperandElements(HloInstruction* instruction,
 
 Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
   auto operand = broadcast->mutable_operand(0);
-  auto dims = broadcast->dimensions();
   // A degenerate broadcast of a reshape that does not change the number of
   // elements can be replaced by a reshape.
-  if (std::is_sorted(dims.begin(), dims.end()) &&
+  if (std::is_sorted(broadcast->dimensions().begin(),
+                     broadcast->dimensions().end()) &&
       ShapeUtil::ElementsIn(broadcast->shape()) ==
           ShapeUtil::ElementsIn(operand->shape())) {
     VLOG(10) << "transform broadcast(X) -> reshape(X) where "
@@ -1142,8 +1142,8 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     VLOG(10) << "transform broadcast(X) -> transpose(X) where "
                 "n(broadcast(X)) == n(X)";
     return ReplaceWithNewInstruction(
-        broadcast,
-        HloInstruction::CreateTranspose(broadcast->shape(), operand, dims));
+        broadcast, HloInstruction::CreateTranspose(broadcast->shape(), operand,
+                                                   broadcast->dimensions()));
   }
 
   // A broadcast of a reshape which merely inserts 1-sized dimensions can
@@ -1157,6 +1157,7 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     if (merely_inserts_or_deletes_1_sized_dimensions &&
         deleted_indices.empty()) {
       std::reverse(inserted_indices.begin(), inserted_indices.end());
+      auto dims = broadcast->dimensions();
       for (auto inserted_index : inserted_indices) {
         dims.erase(dims.begin() + inserted_index);
       }
@@ -1200,19 +1201,6 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
         return user->ReplaceAllUsesWith(new_broadcast);
       }
     }
-    return Status::OK();
-  }
-
-  // Merge two consecutive broadcasts into a single one.
-  if (operand->opcode() == HloOpcode::kBroadcast) {
-    std::vector<int64> new_dimensions(operand->dimensions().size());
-    for (auto dim : operand->dimensions()) {
-      new_dimensions.push_back(dims[dim]);
-    }
-    return ReplaceWithNewInstruction(
-        broadcast,
-        HloInstruction::CreateBroadcast(
-            broadcast->shape(), operand->mutable_operand(0), new_dimensions));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 3b80a827bf..451294ef5d 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -35,8 +35,6 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 
-using ::testing::ElementsAre;
-
 namespace xla {
 namespace {
 
@@ -2464,55 +2462,6 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
               op::DynamicSlice(op::Parameter(), op::Parameter()));
 }
 
-// Test that two consecutive broadcasts can be merged to one.
-TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) {
-  HloComputation::Builder builder(TestName());
-  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2});
-  HloInstruction* input_array = builder.AddInstruction(
-      HloInstruction::CreateConstant(Literal::CreateR1<float>({3, 4})));
-  HloInstruction* inner_bcast = builder.AddInstruction(
-      HloInstruction::CreateBroadcast(r2f32, input_array, {1}));
-  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 2, 2});
-  builder.AddInstruction(
-      HloInstruction::CreateBroadcast(r3f32, inner_bcast, {0, 2}));
-
-  auto computation = module().AddEntryComputation(builder.Build());
-  HloInstruction* root = computation->root_instruction();
-  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
-  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
-  root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Constant()));
-  EXPECT_THAT(root->dimensions(), ElementsAre(2));
-}
-
-// Test that two consecutive broadcasts can be merged to one.
-TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) {
-  HloComputation::Builder builder(TestName());
-  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 3});
-  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 5, 3});
-  HloInstruction* param0 = builder.AddInstruction(
-      HloInstruction::CreateParameter(0, r2f32, "param0"));
-  // The initial dimensions go to places 0 and 2 in the 3-dim array,
-  // and to places 1 and 3 in the 4-dim array,
-  HloInstruction* inner_bcast = builder.AddInstruction(
-      HloInstruction::CreateBroadcast(r3f32, param0, {0, 2}));
-  Shape r4f32 = ShapeUtil::MakeShape(F32, {4, 2, 5, 3});
-  builder.AddInstruction(
-      HloInstruction::CreateBroadcast(r4f32, inner_bcast, {1, 2, 3}));
-
-  auto computation = module().AddEntryComputation(builder.Build());
-  HloInstruction* root = computation->root_instruction();
-  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
-  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
-                                 non_bitcasting_callback());
-  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
-  root = computation->root_instruction();
-  EXPECT_THAT(root, op::Broadcast(op::Parameter(0)));
-  EXPECT_THAT(root->dimensions(), ElementsAre(1, 3));
-}
-
 struct PadReduceWindowEffectiveBroadcastCase {
   std::vector<int64> input_spatials;
   std::vector<int64> symmetric_pad_spatials;
-- 
GitLab


From fd3baace5f494aff716fdb576bdaa6f91b40e16e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 16:05:22 -0700
Subject: [PATCH 388/960] Remove cached tensors from LinearOperator.

CSE should already ensure that tensors are deduped (i.e. multiple Cholesky's for the same matrix are not performed).

This also makes it easier to use LinearOperators in tf.while_loop.

PiperOrigin-RevId: 190141732
---
 .../python/ops/linalg/linear_operator.py      | 106 ++++++------------
 1 file changed, 35 insertions(+), 71 deletions(-)

diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index 957a795918..c7513d5b40 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -204,16 +204,6 @@ class LinearOperator(object):
     self._is_positive_definite = is_positive_definite
     self._name = name or type(self).__name__
 
-    # We will cache some tensors to avoid repeatedly adding shape
-    # manipulation ops to the graph.
-    # Naming convention:
-    #   self._cached_X_tensor is the cached version of self._X_tensor.
-    self._cached_shape_tensor = None
-    self._cached_batch_shape_tensor = None
-    self._cached_domain_dimension_tensor = None
-    self._cached_range_dimension_tensor = None
-    self._cached_tensor_rank_tensor = None
-
   @contextlib.contextmanager
   def _name_scope(self, name=None, values=None):
     """Helper function to standardize op scope."""
@@ -299,15 +289,11 @@ class LinearOperator(object):
       `int32` `Tensor`
     """
     with self._name_scope(name):
-      # Be clean by avoiding adding shape Ops to the graph too many times.
-      if self._cached_shape_tensor is None:
-        # Prefer to use statically defined shape if available.
-        if self.shape.is_fully_defined():
-          self._cached_shape_tensor = linear_operator_util.shape_tensor(
-              self.shape.as_list())
-        else:
-          self._cached_shape_tensor = self._shape_tensor()
-      return self._cached_shape_tensor
+      # Prefer to use statically defined shape if available.
+      if self.shape.is_fully_defined():
+        return linear_operator_util.shape_tensor(self.shape.as_list())
+      else:
+        return self._shape_tensor()
 
   @property
   def batch_shape(self):
@@ -338,14 +324,12 @@ class LinearOperator(object):
     """
     # Derived classes get this "for free" once .shape() is implemented.
     with self._name_scope(name):
-      if self._cached_batch_shape_tensor is None:
-        # Prefer to use statically defined shape if available.
-        if self.batch_shape.is_fully_defined():
-          self._cached_batch_shape_tensor = linear_operator_util.shape_tensor(
-              self.batch_shape.as_list(), name="batch_shape")
-        else:
-          self._cached_batch_shape_tensor = self.shape_tensor()[:-2]
-      return self._cached_batch_shape_tensor
+      # Prefer to use statically defined shape if available.
+      if self.batch_shape.is_fully_defined():
+        return linear_operator_util.shape_tensor(
+            self.batch_shape.as_list(), name="batch_shape")
+      else:
+        return self.shape_tensor()[:-2]
 
   @property
   def tensor_rank(self, name="tensor_rank"):
@@ -378,14 +362,11 @@ class LinearOperator(object):
     """
     # Derived classes get this "for free" once .shape() is implemented.
     with self._name_scope(name):
-      if self._cached_tensor_rank_tensor is None:
-        # Prefer to use statically defined shape if available.
-        if self.tensor_rank is not None:
-          self._cached_tensor_rank_tensor = ops.convert_to_tensor(
-              self.tensor_rank)
-        else:
-          self._cached_tensor_rank_tensor = array_ops.size(self.shape_tensor())
-      return self._cached_tensor_rank_tensor
+      # Prefer to use statically defined shape if available.
+      if self.tensor_rank is not None:
+        return ops.convert_to_tensor(self.tensor_rank)
+      else:
+        return array_ops.size(self.shape_tensor())
 
   @property
   def domain_dimension(self):
@@ -416,14 +397,11 @@ class LinearOperator(object):
     """
     # Derived classes get this "for free" once .shape() is implemented.
     with self._name_scope(name):
-      if self._cached_domain_dimension_tensor is None:
-        # Prefer to use statically defined shape if available.
-        if self.domain_dimension.value is not None:
-          self._cached_domain_dimension_tensor = ops.convert_to_tensor(
-              self.domain_dimension.value)
-        else:
-          self._cached_domain_dimension_tensor = self.shape_tensor()[-1]
-      return self._cached_domain_dimension_tensor
+      # Prefer to use statically defined shape if available.
+      if self.domain_dimension.value is not None:
+        return ops.convert_to_tensor(self.domain_dimension.value)
+      else:
+        return self.shape_tensor()[-1]
 
   @property
   def range_dimension(self):
@@ -454,14 +432,11 @@ class LinearOperator(object):
     """
     # Derived classes get this "for free" once .shape() is implemented.
     with self._name_scope(name):
-      if self._cached_range_dimension_tensor is None:
-        # Prefer to use statically defined shape if available.
-        if self.range_dimension.value is not None:
-          self._cached_range_dimension_tensor = ops.convert_to_tensor(
-              self.range_dimension.value)
-        else:
-          self._cached_range_dimension_tensor = self.shape_tensor()[-2]
-      return self._cached_range_dimension_tensor
+      # Prefer to use statically defined shape if available.
+      if self.range_dimension.value is not None:
+        return ops.convert_to_tensor(self.range_dimension.value)
+      else:
+        return self.shape_tensor()[-2]
 
   def _assert_non_singular(self):
     """Private default implementation of _assert_non_singular."""
@@ -471,8 +446,7 @@ class LinearOperator(object):
     if self._can_use_cholesky():
       return self.assert_positive_definite()
     else:
-      singular_values = linalg_ops.svd(
-          self._get_cached_dense_matrix(), compute_uv=False)
+      singular_values = linalg_ops.svd(self.to_dense(), compute_uv=False)
       # TODO(langmore) Add .eig and .cond as methods.
       cond = (math_ops.reduce_max(singular_values, axis=-1) /
               math_ops.reduce_min(singular_values, axis=-1))
@@ -524,7 +498,7 @@ class LinearOperator(object):
     # and sufficient.
     if self.is_self_adjoint:
       return check_ops.assert_positive(
-          array_ops.matrix_diag_part(self._get_cached_chol()),
+          array_ops.matrix_diag_part(linalg_ops.cholesky(self.to_dense())),
           message="Matrix was not positive definite.")
     # We have no generic check for positive definite.
     raise NotImplementedError("assert_positive_definite is not implemented.")
@@ -547,7 +521,7 @@ class LinearOperator(object):
       return self._assert_positive_definite()
 
   def _assert_self_adjoint(self):
-    dense = self._get_cached_dense_matrix()
+    dense = self.to_dense()
     logging.warn(
         "Using (possibly slow) default implementation of assert_self_adjoint."
         "  Requires conversion to a dense matrix.")
@@ -692,7 +666,7 @@ class LinearOperator(object):
         "Using (possibly slow) default implementation of determinant."
         "  Requires conversion to a dense matrix and O(N^3) operations.")
     if self._can_use_cholesky():
-      diag = array_ops.matrix_diag_part(self._get_cached_chol())
+      diag = array_ops.matrix_diag_part(linalg_ops.cholesky(self.to_dense()))
       return 2 * math_ops.reduce_sum(math_ops.log(diag), reduction_indices=[-1])
     _, log_abs_det = linalg.slogdet(self._matrix)
     return log_abs_det
@@ -726,9 +700,9 @@ class LinearOperator(object):
         "  Requires conversion to a dense matrix and O(N^3) operations.")
     rhs = linalg.adjoint(rhs) if adjoint_arg else rhs
     if self._can_use_cholesky():
-      return linalg_ops.cholesky_solve(self._get_cached_chol(), rhs)
-    return linalg_ops.matrix_solve(
-        self._get_cached_dense_matrix(), rhs, adjoint=adjoint)
+      return linalg_ops.cholesky_solve(
+          linalg_ops.cholesky(self.to_dense()), rhs)
+    return linalg_ops.matrix_solve(self.to_dense(), rhs, adjoint=adjoint)
 
   def solve(self, rhs, adjoint=False, adjoint_arg=False, name="solve"):
     """Solve (exact or approx) `R` (batch) systems of equations: `A X = rhs`.
@@ -866,7 +840,7 @@ class LinearOperator(object):
 
   def _diag_part(self):
     """Generic and often inefficient implementation.  Override often."""
-    return array_ops.matrix_diag_part(self._get_cached_dense_matrix())
+    return array_ops.matrix_diag_part(self.to_dense())
 
   def diag_part(self, name="diag_part"):
     """Efficiently get the [batch] diagonal part of this operator.
@@ -915,7 +889,7 @@ class LinearOperator(object):
 
   def _add_to_tensor(self, x):
     # Override if a more efficient implementation is available.
-    return self._get_cached_dense_matrix() + x
+    return self.to_dense() + x
 
   def add_to_tensor(self, x, name="add_to_tensor"):
     """Add matrix represented by this operator to `x`.  Equivalent to `A + x`.
@@ -936,13 +910,3 @@ class LinearOperator(object):
     # TODO(langmore) Add complex types when tf.cholesky can use them.
     return (not self.dtype.is_complex and self.is_self_adjoint and
             self.is_positive_definite)
-
-  def _get_cached_dense_matrix(self):
-    if not hasattr(self, "_cached_dense_matrix"):
-      self._cached_dense_matrix = self.to_dense()
-    return self._cached_dense_matrix
-
-  def _get_cached_chol(self):
-    if not hasattr(self, "_cached_chol"):
-      self._cached_chol = linalg_ops.cholesky(self._get_cached_dense_matrix())
-    return self._cached_chol
-- 
GitLab


From 7a39f562caca8ec804c0eddaa7b28b64a93f06e7 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 22 Mar 2018 16:42:49 -0700
Subject: [PATCH 389/960] Delete all the name_scopes in the TPU code so that
 tf.Variable() doesn't inherit them.

PiperOrigin-RevId: 190146946
---
 tensorflow/contrib/tpu/python/tpu/tpu.py      | 228 +++++++++---------
 .../contrib/tpu/python/tpu/tpu_estimator.py   |   6 +-
 .../contrib/tpu/python/tpu/training_loop.py   |   8 +-
 3 files changed, 119 insertions(+), 123 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu.py b/tensorflow/contrib/tpu/python/tpu/tpu.py
index d5f54ff4fd..3f2db548ac 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu.py
@@ -201,7 +201,7 @@ def replicate(computation,
       `DeviceAssignment` may be omitted if each replica of the computation uses
       only one core, and there is either only one replica, or the number of
       replicas is equal to the number of cores in the TPU system.
-    name: The name of the operator.
+    name: (Deprecated) Does nothing.
   Returns:
     A list of lists of output tensors, indexed by `[replica_num][output_num]`.
   Raises:
@@ -209,8 +209,7 @@ def replicate(computation,
     ValueError: If the number of inputs per replica does not match
       the number of formal parameters to `computation`.
   """
-  if name is None:
-    name = "TPUReplicate"
+  del name
   inputs = [[]] if inputs is None else inputs
 
   metadata_kwargs = {}
@@ -274,118 +273,117 @@ def replicate(computation,
 
   graph = ops.get_default_graph()
 
-  with ops.name_scope(name, "replicate"):
-    # Fan-in: Builds a TPUReplicatedInput node for each input.
-    computation_inputs = []
-    for i in range(0, input_arity):
-      replicas = [inputs[replica][i] for replica in xrange(num_replicas)]
-      computation_inputs.append(
-          tpu_ops.tpu_replicated_input(replicas, name="input{}".format(i)))
+  # Fan-in: Builds a TPUReplicatedInput node for each input.
+  computation_inputs = []
+  for i in range(0, input_arity):
+    replicas = [inputs[replica][i] for replica in xrange(num_replicas)]
+    computation_inputs.append(
+        tpu_ops.tpu_replicated_input(replicas, name="input{}".format(i)))
+
+  context = TPUReplicateContext(name=graph.unique_name("cluster"))
+  try:
+    context.Enter()
+
+    metadata = tpu_ops.tpu_replicate_metadata(
+        num_replicas=num_replicas, **metadata_kwargs)
+
+    with tpu_function.tpu_shard_context(
+        num_replicas), ops.control_dependencies([metadata]):
+
+      # The EncapsulateTPUComputations rewrite needs to identify the
+      # replicated arguments inside each computation. Adds identity operators
+      # tagged with an attribute _tpu_replicated_input to identify the
+      # replicated inputs.
+      # pylint: disable=protected-access
+      with graph._attr_scope({"_tpu_replicated_input":
+                              attr_value_pb2.AttrValue(b=True)}):
+        computation_inputs = [
+            array_ops.identity(x, name="replicated_input_{}".format(i))
+            for i, x in enumerate(computation_inputs)]
+      # pylint: enable=protected-access
+
+      # If there is an infeed queue, adds the dequeued values to the
+      # computation's inputs.
+      if infeed_queue is not None:
+        infeed_queue.set_number_of_shards(num_replicas)
+        for t in infeed_queue.generate_dequeue_op():
+          computation_inputs.append(t)
+
+      # Only resource variables work inside a TPU computation, so turn on
+      # resource variables for the computation.
+      # TODO(phawkins): consider removing this code. It will
+      # be less confusing to clients if they knowingly choose to use resource
+      # variables.
+      vscope = variable_scope.get_variable_scope()
+      saved_use_resource = vscope.use_resource
+      vscope.set_use_resource(True)
+
+      outputs = computation(*computation_inputs)
+
+      vscope.set_use_resource(saved_use_resource)
+
+    # If the computation only returned one value, makes it a tuple.
+    if not isinstance(outputs, (list, tuple)):
+      outputs = (outputs,)
 
-    context = TPUReplicateContext(name=graph.unique_name("cluster"))
     try:
-      context.Enter()
-
-      metadata = tpu_ops.tpu_replicate_metadata(
-          num_replicas=num_replicas, **metadata_kwargs)
-
-      with tpu_function.tpu_shard_context(
-          num_replicas), ops.control_dependencies([metadata]):
-
-        # The EncapsulateTPUComputations rewrite needs to identify the
-        # replicated arguments inside each computation. Adds identity operators
-        # tagged with an attribute _tpu_replicated_input to identify the
-        # replicated inputs.
-        # pylint: disable=protected-access
-        with graph._attr_scope({"_tpu_replicated_input":
-                                attr_value_pb2.AttrValue(b=True)}):
-          computation_inputs = [
-              array_ops.identity(x, name="replicated_input_{}".format(i))
-              for i, x in enumerate(computation_inputs)]
-        # pylint: enable=protected-access
-
-        # If there is an infeed queue, adds the dequeued values to the
-        # computation's inputs.
-        if infeed_queue is not None:
-          infeed_queue.set_number_of_shards(num_replicas)
-          for t in infeed_queue.generate_dequeue_op():
-            computation_inputs.append(t)
-
-        # Only resource variables work inside a TPU computation, so turn on
-        # resource variables for the computation.
-        # TODO(phawkins): consider removing this code. It will
-        # be less confusing to clients if they knowingly choose to use resource
-        # variables.
-        vscope = variable_scope.get_variable_scope()
-        saved_use_resource = vscope.use_resource
-        vscope.set_use_resource(True)
-
-        outputs = computation(*computation_inputs)
-
-        vscope.set_use_resource(saved_use_resource)
-
-      # If the computation only returned one value, makes it a tuple.
-      if not isinstance(outputs, (list, tuple)):
-        outputs = (outputs,)
-
-      try:
-        with ops.device(core(0)):
-          outputs = [
-              o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
-              for o in outputs
-          ]
-      except Exception as e:
-        raise ValueError(
-            "TPU function return values must all either be Operations or "
-            "convertible to Tensors. Got '%s'" % str(e))
-
-      # Separates the returned Operations and Tensors.
-      output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
-      output_tensors = [o for o in outputs
-                        if not isinstance(o, ops.Operation)]
-
-      if outputs != output_tensors + output_operations:
-        raise ValueError(
-            "TPU functions must return zero-or more Tensor values followed by "
-            "zero or more Operations.")
-      output_arity = len(output_tensors)
-
-      # Wraps outputs in Identity ops. Otherwise a replicated input copied
-      # straight to an output would bypass the replicate(). This would be bad
-      # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
-      # be rewritten away, leading to a runtime error.
-      # TODO(phawkins): extend the rewrite to elide these nodes instead.
-      new_output_tensors = []
-      for t in output_tensors:
-        with ops.device(t.device if t.device else core(0)):
-          new_output_tensors.append(array_ops.identity(t))
-      output_tensors = new_output_tensors
-    finally:
-      context.report_unsupported_operations()
-      context.Exit()
-
-    # Fan-out: Builds a TPUReplicatedOutput node for each output.
-    outputs = [tpu_ops.tpu_replicated_output(output_tensors[i], num_replicas,
-                                             name="output{}".format(i))
-               for i in xrange(output_arity)]
-
-    with ops.control_dependencies(output_operations):
-      if output_arity == 0:
-        # Returns a list of NoOps dependent on the replication Op, indexed by
-        # [replica_num].
-        return [
-            control_flow_ops.no_op(name="%s_shard_%d" % (name, i))
-            for i in range(num_replicas)
-        ]
-      else:
-        # Wraps the outputs in identity operators so the names of any possible
-        # `fetch` nodes are preserved by the replication rewrite.
-        return [
-            [array_ops.identity(outputs[out][replica],
-                                name="output_%d_shard_%d" % (out, replica))
-             for out in xrange(output_arity)]
-            for replica in xrange(num_replicas)
+      with ops.device(core(0)):
+        outputs = [
+            o if isinstance(o, ops.Operation) else ops.convert_to_tensor(o)
+            for o in outputs
         ]
+    except Exception as e:
+      raise ValueError(
+          "TPU function return values must all either be Operations or "
+          "convertible to Tensors. Got '%s'" % str(e))
+
+    # Separates the returned Operations and Tensors.
+    output_operations = [o for o in outputs if isinstance(o, ops.Operation)]
+    output_tensors = [o for o in outputs
+                      if not isinstance(o, ops.Operation)]
+
+    if outputs != output_tensors + output_operations:
+      raise ValueError(
+          "TPU functions must return zero-or more Tensor values followed by "
+          "zero or more Operations.")
+    output_arity = len(output_tensors)
+
+    # Wraps outputs in Identity ops. Otherwise a replicated input copied
+    # straight to an output would bypass the replicate(). This would be bad
+    # because the TPUReplicatedInput/TPUReplicatedOutput operator would not
+    # be rewritten away, leading to a runtime error.
+    # TODO(phawkins): extend the rewrite to elide these nodes instead.
+    new_output_tensors = []
+    for t in output_tensors:
+      with ops.device(t.device if t.device else core(0)):
+        new_output_tensors.append(array_ops.identity(t))
+    output_tensors = new_output_tensors
+  finally:
+    context.report_unsupported_operations()
+    context.Exit()
+
+  # Fan-out: Builds a TPUReplicatedOutput node for each output.
+  outputs = [tpu_ops.tpu_replicated_output(output_tensors[i], num_replicas,
+                                           name="output{}".format(i))
+             for i in xrange(output_arity)]
+
+  with ops.control_dependencies(output_operations):
+    if output_arity == 0:
+      # Returns a list of NoOps dependent on the replication Op, indexed by
+      # [replica_num].
+      return [
+          control_flow_ops.no_op(name="shard_%d" % i)
+          for i in range(num_replicas)
+      ]
+    else:
+      # Wraps the outputs in identity operators so the names of any possible
+      # `fetch` nodes are preserved by the replication rewrite.
+      return [
+          [array_ops.identity(outputs[out][replica],
+                              name="output_%d_shard_%d" % (out, replica))
+           for out in xrange(output_arity)]
+          for replica in xrange(num_replicas)
+      ]
 
 
 def shard(computation,
@@ -450,7 +448,7 @@ def shard(computation,
       `DeviceAssignment` may be omitted if each shard of the computation uses
       only one core, and there is either only one shard, or the number of shards
       is equal to the number of cores in the TPU system.
-    name: The name of the operator.
+    name: (Deprecated) Does nothing.
   Returns:
     A list of output tensors.
   Raises:
@@ -579,7 +577,7 @@ def batch_parallel(computation,
       `DeviceAssignment` may be omitted if each shard of the computation uses
       only one core, and there is either only one shard, or the number of shards
       is equal to the number of cores in the TPU system.
-    name: The name of the operator.
+    name: (Deprecated) Does nothing.
   Returns:
     A list of output tensors.
   Raises:
@@ -613,7 +611,7 @@ def rewrite(computation,
       mapping between logical cores in the computation with physical cores in
       the TPU topology. May be omitted for a single-core computation, in which
       case the core attached to task 0, TPU device 0 is used.
-    name: The name of the operator.
+    name: (Deprecated) Does nothing.
   Returns:
     A list of output tensors.
   """
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 4354735744..aaa6f3c2c1 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2109,8 +2109,7 @@ def _eval_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn):
   def multi_tpu_eval_steps_on_single_shard():
     return training_loop.repeat(
         iterations_per_loop_var,
-        single_tpu_eval_step, [_ZERO_LOSS],
-        name='loop')
+        single_tpu_eval_step, [_ZERO_LOSS])
 
   (loss,) = tpu.shard(
       multi_tpu_eval_steps_on_single_shard,
@@ -2133,8 +2132,7 @@ def _train_on_tpu_system(ctx, model_fn_wrapper, dequeue_fn):
   def multi_tpu_train_steps_on_single_shard():
     return training_loop.repeat(
         iterations_per_loop_var,
-        single_tpu_train_step, [_INITIAL_LOSS],
-        name=b'loop')
+        single_tpu_train_step, [_INITIAL_LOSS])
 
   (loss,) = tpu.shard(
       multi_tpu_train_steps_on_single_shard,
diff --git a/tensorflow/contrib/tpu/python/tpu/training_loop.py b/tensorflow/contrib/tpu/python/tpu/training_loop.py
index 82a75d0255..10a8bccf3b 100644
--- a/tensorflow/contrib/tpu/python/tpu/training_loop.py
+++ b/tensorflow/contrib/tpu/python/tpu/training_loop.py
@@ -44,7 +44,7 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None):
       None (equivalent to an empty list).
     infeed_queue: if not None, the infeed queue from which to append a tuple
       of arguments as inputs to condition.
-    name: an optional name for the loop.
+    name: (Deprecated) Does nothing.
 
   Returns:
     The final values of the loop-carried tensors.
@@ -52,7 +52,7 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None):
   Raises:
     TypeError: if body or condition has the wrong signature.
   """
-
+  del name
   # Converts inputs to Tensors.
   inputs = [] if inputs is None else [ops.convert_to_tensor(x) for
                                       x in inputs]
@@ -166,7 +166,7 @@ def while_loop(condition, body, inputs=None, infeed_queue=None, name=None):
   if input_arity == 0:
     inputs = [array_ops.constant(0)]
   return control_flow_ops.while_loop(condition_wrapper, body_wrapper, inputs,
-                                     name=name)
+                                     name="")
 
 
 def repeat(n, body, inputs=None, infeed_queue=None, name=None):
@@ -183,7 +183,7 @@ def repeat(n, body, inputs=None, infeed_queue=None, name=None):
       None (equivalent to an empty list).
     infeed_queue: if not None, the infeed queue from which to append a tuple
       of arguments as inputs to condition.
-    name: an optional name for the loop.
+    name: (Deprecated) Does nothing.
   Returns:
     The final values of the loop-carried tensors.
   Raises:
-- 
GitLab


From a5c7bb599cfc0b2aaecc054dc761d275fe9bfcbd Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Fri, 23 Mar 2018 08:18:59 +0800
Subject: [PATCH 390/960] Fix minor markdown typo in lite models.md (#17926)

---
 tensorflow/contrib/lite/g3doc/models.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 5b393140d6..48f43d4fc4 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,4 +1,4 @@
-#List of Hosted Models
+# List of Hosted Models
 
 *   [Inception V3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
 *   [Inception V3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
-- 
GitLab


From b95d0733cd55de9e71280fb76cdb418585654d16 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong-512@163.com>
Date: Fri, 23 Mar 2018 08:19:18 +0800
Subject: [PATCH 391/960] Fix the mirror link to replace bazel-mirror by
 mirror.bazel (#17928)

---
 tensorflow/contrib/makefile/download_dependencies.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 4ae18b2cef..8b415e6527 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -34,7 +34,7 @@ PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.
 RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
 FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)"
-CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
+CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
-- 
GitLab


From b24923caba5e3ed64d55c69ef1eb6da560f98865 Mon Sep 17 00:00:00 2001
From: Marvin Richter <marvin.richter@me.com>
Date: Fri, 23 Mar 2018 01:19:36 +0100
Subject: [PATCH 392/960] swapped const and variables (#17918)

The freeze_graph.py is converting Variables to Consts, not the other way around :)
---
 tensorflow/docs_src/mobile/prepare_models.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md
index 360ee302aa..8b22c04d87 100644
--- a/tensorflow/docs_src/mobile/prepare_models.md
+++ b/tensorflow/docs_src/mobile/prepare_models.md
@@ -60,7 +60,7 @@ and serialized as protocol buffers:
   the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
   then we only need a single `GraphDef` file to hold the model architecture and
   the weights. Freezing the graph handles the process of loading the
-  checkpoints, and then converts all Consts to Variables. You can then load the
+  checkpoints, and then converts all Variables to Consts. You can then load the
   resulting file in a single call, without having to restore variable values
   from checkpoints. One thing to watch out for with `GraphDef` files is that
   sometimes they’re stored in text format for easy inspection. These versions
-- 
GitLab


From 58e4b92f2ebc346c6599484959d4263672352e87 Mon Sep 17 00:00:00 2001
From: Sami Kama <samikama@users.noreply.github.com>
Date: Thu, 22 Mar 2018 17:19:59 -0700
Subject: [PATCH 393/960] Update TF-TensorRT segmenter (#17912)

* Do not follow control edges in segmenter and in conversion and gracefully handle some failures

* Run clang-format

* Fix test and include header instead of forward declaration
---
 .../contrib/tensorrt/convert/convert_graph.cc | 19 +++--
 .../contrib/tensorrt/convert/convert_nodes.cc | 82 +++++++++++++------
 .../contrib/tensorrt/segment/segment.cc       | 55 +++++++++----
 tensorflow/contrib/tensorrt/segment/segment.h |  4 +-
 .../contrib/tensorrt/segment/segment_test.cc  |  8 +-
 5 files changed, 109 insertions(+), 59 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index eea8c8efa2..a61d7b0699 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,7 +49,7 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::Node* node) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
@@ -74,7 +74,7 @@ bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
       // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
-  return candidate_ops.count(node_def.op());
+  return candidate_ops.count(node->type_string());
 }
 
 void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
@@ -84,10 +84,10 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->in_edges()) {
       if (!subgraph_node_ids.count(edge->src()->id()) &&
-          !edge->src()->IsSource()) {
+          !edge->src()->IsSource() && !edge->IsControlEdge()) {
         incoming_edges->insert(edge);
       } else {
-        VLOG(2) << edge->src()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->src()->name() << " N, ";
       }
     }
   }
@@ -100,11 +100,11 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
-          !edge->dst()->IsSink()) {
-        VLOG(2) << edge->dst()->name() << " Y, ";
+          !edge->dst()->IsSink() && !edge->IsControlEdge()) {
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
       } else {
-        VLOG(2) << edge->dst()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -409,8 +409,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
       if (status != tensorflow::Status::OK()) {
         LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \n"
-                     << status.ToString() << " SKIPPING......";
+                     << " due to: \"" << status.ToString()
+                     << "\" SKIPPING......( " << subgraph_node_names.size()
+                     << " nodes)";
       }
       count++;
     }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 92a692baa7..935946ad5b 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -53,8 +53,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
-
 namespace {
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
@@ -430,9 +430,8 @@ class Converter {
   tensorflow::tensorrt::TRTWeightStore* weight_store_;
   bool fp16_;
   void register_op_converters();
-  std::vector<TRT_TensorOrWeights> get_inputs(
-      const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs;
+  tensorflow::Status get_inputs(const tensorflow::NodeDef& node_def,
+                                std::vector<TRT_TensorOrWeights>* inputs) {
     for (auto const& input_name : node_def.input()) {
       /*************************************************************************
        * TODO(jie) handle case 1) here
@@ -453,13 +452,17 @@ class Converter {
 
       VLOG(2) << "retrieve input: " << name;
       if (trt_tensors_.count(name)) {
-        inputs.push_back(trt_tensors_.at(name));
+        inputs->push_back(trt_tensors_.at(name));
       } else {
-        LOG(FATAL) << "input: " << name << " not available for node at, "
-                   << node_def.name();
+        string str("Node ");
+        StrAppend(&str, node_def.name(), " should have an input named '", name,
+                  "' but it is not available");
+        LOG(WARNING) << "input: " << name << " not available for node at "
+                     << node_def.name();
+        return tensorflow::errors::InvalidArgument(str);
       }
     }
-    return inputs;
+    return tensorflow::Status::OK();
   }
 
  public:
@@ -483,7 +486,8 @@ class Converter {
   }
 
   tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs = this->get_inputs(node_def);
+    std::vector<TRT_TensorOrWeights> inputs;
+    TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs));
     string op = node_def.op();
     if (!op_registry_.count(op)) {
       return tensorflow::errors::Unimplemented(
@@ -875,7 +879,7 @@ tensorflow::Status BinaryTensorOpWeight(
 
   // Check type consistency
   nvinfer1::DataType ttype;
-  TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
+  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
 
   // Check scale mode
   auto dims_w = weights.shape_;
@@ -1134,9 +1138,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
+    return tensorflow::errors::Unimplemented(
+        "binary op: " + node_def.op() +
+        " not supported at: " + node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1376,8 +1380,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.float_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1432,8 +1439,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.int_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -2143,8 +2153,11 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
   calib_res->thr_->join();
   delete calib_res->thr_;
   if (!calib_res->engine_) {
-    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+    LOG(ERROR) << "Calibration failed!, engine does not exist. Did you run "
                   "calibration graph?";
+    return tensorflow::errors::FailedPrecondition(
+        "Calibration graph needs to be executed on"
+        " calibration data before convertsion to inference graph");
   }
   auto weight_rmgr = trt_rm->getManager("WeightStore");
   TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
@@ -2181,7 +2194,7 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
     return status;
   }
   auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_CHECK_OK(status);
+  TF_RETURN_IF_ERROR(status);
   for (size_t i = 0; i < out_edges.size(); i++) {
     VLOG(1) << "Connecting trt_engine_node output " << i << " with "
             << out_edges.at(i)->dst()->name() << " port "
@@ -2279,6 +2292,12 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Data type conversion for input '" << node_name
+                   << "' failed";
+      return type_status;
+    }
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "accessing output index of: " << output_idx
@@ -2346,8 +2365,8 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node'" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2504,7 +2523,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Type conversion failed for " << node_name;
+      return type_status;
+    }
 
     VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
@@ -2515,8 +2538,12 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
     //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4)
-      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+    if (op_info.shape().dim_size() != 4) {
+      string err_str = "Require 4 dimensional input.";
+      StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ",
+                shape_inference_node_name);
+      return tensorflow::errors::Unimplemented(err_str);
+    }
 
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
@@ -2577,8 +2604,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node '" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2622,7 +2649,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
   TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
       engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name;
+  LOG(INFO) << "finished engine " << engine_name << " containing "
+            << s.subgraph_node_ids.size() << " nodes";
 
   // Build the TRT op
   tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6193f0b0a1..8fc4697c51 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -80,13 +80,20 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> in_edges(dst->in_edges().begin(),
                                                 dst->in_edges().end());
   for (const tensorflow::Edge* in_edge : in_edges) {
-    if (in_edge->src() != src) {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-      if (e->src() == graph->source_node()) {
-        graph->AddEdge(e->src(), e->src_output(), src,
-                       tensorflow::Graph::kControlSlot);
-      } else {
-        graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+    if (in_edge->IsControlEdge()) {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        graph->AddControlEdge(e->src(), src);
+      }
+    } else {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        if (e->src() == graph->source_node()) {
+          graph->AddEdge(e->src(), e->src_output(), src,
+                         tensorflow::Graph::kControlSlot);
+        } else {
+          graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+        }
       }
     }
   }
@@ -94,12 +101,19 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> out_edges(dst->out_edges().begin(),
                                                  dst->out_edges().end());
   for (const tensorflow::Edge* out_edge : out_edges) {
-    tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-    if (e->dst() == graph->sink_node()) {
-      graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
-                     e->dst_input());
+    if (out_edge->IsControlEdge()) {
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      graph->AddControlEdge(src, e->dst());
     } else {
-      graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      if (e->dst() == graph->sink_node()) {
+        VLOG(1) << " edge to sink node " << src->name() << " -> "
+                << e->dst()->name();
+        graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
+                       e->dst_input());
+      } else {
+        graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      }
     }
   }
 
@@ -118,7 +132,7 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
   // Create a Graph representation of the GraphDef.
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
@@ -136,7 +150,7 @@ tensorflow::Status SegmentGraph(
   for (int i = 0; i < graph.num_node_ids(); ++i) {
     tensorflow::Node* node = graph.FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0 ||
-        !candidate_fn(node->def())) {
+        !candidate_fn(node)) {
       node = nullptr;
     }
     node_segments.emplace_back(node);
@@ -155,7 +169,7 @@ tensorflow::Status SegmentGraph(
 
   for (const tensorflow::Node* node : order) {
     // All output nodes of 'node' have been visited...
-    VLOG(2) << "Trying node " << node->name();
+    VLOG(2) << "Trying node " << node->name() << " id=" << node->id();
 
     // 'node' must be a TRT candidate...
     if (node_segments[node->id()].Value() == nullptr) {
@@ -169,8 +183,12 @@ tensorflow::Status SegmentGraph(
     while (true) {
       std::set<const tensorflow::Edge*> contract_edges;
       for (const tensorflow::Edge* out_edge : node->out_edges()) {
-        VLOG(2) << "... out node " << out_edge->dst()->name();
-
+        VLOG(2) << "... out node " << out_edge->dst()->name() << " ( "
+                << out_edge->dst()->id() << " <- " << node->id() << " )";
+        if (out_edge->IsControlEdge()) {
+          VLOG(2) << "... ... Control Edge, Skipping";
+          continue;
+        }
         // Out node must be TRT candidate...
         if (node_segments[out_edge->dst()->id()].Value() == nullptr) {
           VLOG(2) << "... ... not a TRT candidate";
@@ -196,7 +214,8 @@ tensorflow::Status SegmentGraph(
         const tensorflow::Node* src = contract_edge->src();
         const tensorflow::Node* dst = contract_edge->dst();
 
-        VLOG(2) << "Merge " << src->name() << " <- " << dst->name();
+        VLOG(2) << "Merge " << src->name() << " <- " << dst->name() << " ("
+                << src->id() << " <- " << dst->id();
         node_segments[src->id()].Merge(&node_segments[dst->id()]);
 
         // Contracting the edge leaves disconnected graph edges.
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index ee6e2b3ed2..7e8685f44a 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -20,10 +20,12 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+
 namespace tensorrt {
 namespace segment {
 
@@ -46,7 +48,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
 
 }  // namespace segment
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 74cbc5f2b3..7ddabec268 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -35,7 +35,7 @@ class SegmentTest : public ::testing::Test {
   TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph,
                     TF_Status* s, const char* name);
 
-  std::function<bool(const NodeDef&)> MakeCandidateFn(
+  std::function<bool(const Node*)> MakeCandidateFn(
       const std::set<string>& node_names);
 
  protected:
@@ -60,10 +60,10 @@ bool SegmentTest::GetGraphDef(TF_Graph* graph,
   return ret;
 }
 
-std::function<bool(const NodeDef&)> SegmentTest::MakeCandidateFn(
+std::function<bool(const Node*)> SegmentTest::MakeCandidateFn(
     const std::set<string>& node_names) {
-  return [node_names](const NodeDef& node) -> bool {
-    return node_names.find(node.name()) != node_names.end();
+  return [node_names](const Node* node) -> bool {
+    return node_names.find(node->name()) != node_names.end();
   };
 }
 
-- 
GitLab


From e6f5c2f21f990792e7e21729703e89065c4a5464 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 22 Mar 2018 17:29:47 -0700
Subject: [PATCH 394/960] Checkpointable: Fix a bug where SaveableObjects in
 the Saver's var_list were stale when executing eagerly

SaveableObjects were recreated each save(), but the Saver was not recreated and so had an old var_list.

PiperOrigin-RevId: 190153011
---
 .../contrib/eager/python/checkpointable_utils.py |  7 ++++++-
 .../eager/python/checkpointable_utils_test.py    | 16 +++++++++++++---
 2 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index adbb92e43b..91a7aded11 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -631,7 +631,12 @@ class CheckpointableSaver(object):
     named_variables[_OBJECT_GRAPH_PROTO_KEY] = _NoRestoreSaveable(
         tensor=object_graph_tensor,
         name=_OBJECT_GRAPH_PROTO_KEY)
-    if self._last_save_object_graph != graph_proto:
+    if (self._last_save_object_graph != graph_proto
+        # When executing eagerly, we need to re-create SaveableObjects each time
+        # save() is called so they pick up new Tensors passed to their
+        # constructors. That means the Saver needs to be copied with a new
+        # var_list.
+        or context.executing_eagerly()):
       if self._last_save_object_graph is not None:
         self._last_save_saver = _copy_saver_with_new_var_list(
             old_saver=self._last_save_saver, new_var_list=named_variables)
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 690f3ee67a..a4b215bb7a 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -154,14 +154,18 @@ class InterfaceTests(test.TestCase):
     self.assertAllEqual([1., 1., 1.], self.evaluate(v2))
 
 
-class _MirroringSaveable(
-    core_saver.BaseSaverBuilder.ResourceVariableSaveable):
+class _MirroringSaveable(core_saver.BaseSaverBuilder.SaveableObject):
 
   def __init__(self, primary_variable, mirrored_variable, name):
     self._primary_variable = primary_variable
     self._mirrored_variable = mirrored_variable
+    tensor = self._primary_variable.read_value()
+    spec = core_saver.BaseSaverBuilder.SaveSpec(
+        tensor=tensor,
+        slice_spec="",
+        name=name)
     super(_MirroringSaveable, self).__init__(
-        self._primary_variable, "", name)
+        tensor, [spec], name)
 
   def restore(self, restored_tensors, restored_shapes):
     """Restore the same value into both variables."""
@@ -316,6 +320,12 @@ class CheckpointingTests(test.TestCase):
     checkpoint.restore(save_path).assert_consumed().initialize_or_restore()
     self.assertEqual(42., self.evaluate(v.non_dep_variable))
     self.assertEqual(42., self.evaluate(v.mirrored))
+    self.evaluate(v.non_dep_variable.assign(44.))
+    save_path = checkpoint.save(prefix)
+    self.evaluate(v.non_dep_variable.assign(45.))
+    checkpoint.restore(save_path).assert_consumed().initialize_or_restore()
+    self.assertEqual(44., self.evaluate(v.non_dep_variable))
+    self.assertEqual(44., self.evaluate(v.mirrored))
 
   @test_util.run_in_graph_and_eager_modes()
   def testMoreComplexSaveableReturnedWithGlobalName(self):
-- 
GitLab


From 595723d9db550ba22c187cc7268f05e800d03b67 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 17:45:03 -0700
Subject: [PATCH 395/960] Disable test
 tensorflow/contrib/linalg:linear_operator_block_diag_test in asan mode

PiperOrigin-RevId: 190154645
---
 tensorflow/contrib/linalg/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD
index 3bc1427bd2..359255374d 100644
--- a/tensorflow/contrib/linalg/BUILD
+++ b/tensorflow/contrib/linalg/BUILD
@@ -59,6 +59,7 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
     shard_count = 4,
+    tags = ["noasan"],
 )
 
 filegroup(
-- 
GitLab


From 80cac4d951d78660b32ccfde9d91ff4050e5d477 Mon Sep 17 00:00:00 2001
From: Zhixian Yan <zhixianyan@google.com>
Date: Thu, 22 Mar 2018 17:59:19 -0700
Subject: [PATCH 396/960] Convert saved_model to tflite flatbuffer.

PiperOrigin-RevId: 190156133
---
 tensorflow/contrib/lite/python/BUILD          |  35 +++
 .../contrib/lite/python/convert_savedmodel.py | 261 +++++++++++++++++
 .../lite/python/convert_savedmodel_test.py    | 276 ++++++++++++++++++
 tensorflow/tools/pip_package/BUILD            |   1 +
 4 files changed, 573 insertions(+)
 create mode 100644 tensorflow/contrib/lite/python/convert_savedmodel.py
 create mode 100644 tensorflow/contrib/lite/python/convert_savedmodel_test.py

diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index d6f39219ed..ce1a81d06b 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -84,6 +84,41 @@ py_test(
     ],
 )
 
+py_binary(
+    name = "convert_savedmodel",
+    srcs = ["convert_savedmodel.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":lite",
+        "//tensorflow/contrib/saved_model:saved_model_py",
+        "//tensorflow/python:graph_util",
+        "//tensorflow/python/tools:freeze_graph_lib",
+    ],
+)
+
+py_test(
+    name = "convert_savedmodel_test",
+    srcs = ["convert_savedmodel_test.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":convert_savedmodel",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:session",
+        "//tensorflow/python/saved_model",
+    ],
+)
+
+# Transitive dependencies of this target will be included in the pip package.
+py_library(
+    name = "tf_lite_py_pip",
+    deps = [
+        ":convert_savedmodel",
+    ],
+)
+
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/lite/python/convert_savedmodel.py b/tensorflow/contrib/lite/python/convert_savedmodel.py
new file mode 100644
index 0000000000..d39e1a1d98
--- /dev/null
+++ b/tensorflow/contrib/lite/python/convert_savedmodel.py
@@ -0,0 +1,261 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""TensorFlow Lite flatbuffer generation from saved_models.
+
+Example:
+
+bazel run third_party/tensorflow/contrib/lite/python:convert_savedmodel -- \
+  --saved_model_dir=/tmp/test_saved_model/1519865537 \
+  --output_tflite=/tmp/test.lite
+
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.lite.python import lite
+from tensorflow.contrib.saved_model.python.saved_model import reader
+from tensorflow.contrib.saved_model.python.saved_model import signature_def_utils
+from tensorflow.core.framework import types_pb2
+from tensorflow.python.client import session
+from tensorflow.python.framework import graph_util as tf_graph_util
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import app
+from tensorflow.python.platform import flags
+from tensorflow.python.platform import gfile
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.saved_model import loader
+from tensorflow.python.saved_model import signature_constants
+from tensorflow.python.saved_model import tag_constants
+
+flags.DEFINE_string("saved_model_dir", "", "Saved model directory to convert.")
+flags.DEFINE_string("output_tflite", None, "File path to write flatbuffer.")
+flags.DEFINE_string("output_arrays", None,
+                    "List of output tensor names, the default value is None, "
+                    "which means the conversion will keep all outputs.")
+flags.DEFINE_integer("batch_size", 1,
+                     "If input tensor shape has None at first dimension, "
+                     "e.g. (None,224,224,3), replace None with batch_size.")
+flags.DEFINE_string("tag_set", tag_constants.SERVING,
+                    "Group of tag(s) of the MetaGraphDef in the saved_model, "
+                    "in string format, separated by ','. For tag-set contains "
+                    "multiple tags, all tags must be passed in.")
+flags.DEFINE_string("signature_key",
+                    signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
+                    "This is signature key to extract inputs, outputs.")
+
+
+def log_tensor_details(tensor_info):
+  """Log tensor details: name, shape, and type."""
+  for key in tensor_info:
+    val = tensor_info[key]
+    dtype = types_pb2.DataType.Name(val.dtype)
+    if val.tensor_shape.unknown_rank:
+      shape = "unknown_rank"
+    else:
+      dims = [str(dim.size) for dim in val.tensor_shape.dim]
+      shape = "({})".format(", ".join(dims))
+
+    logging.info("Tensor's key in savedmodel's tensor_map: %s", key)
+    logging.info(" tensor name: %s, shape: %s, type: %s", val.name, shape,
+                 dtype)
+
+
+def get_meta_graph_def(saved_model_dir, tag_set):
+  """Validate savedmodel and extract MetaGraphDef.
+
+  Args:
+    saved_model_dir: Savedmodel path to convert.
+    tag_set: Set of tag(s) of the MetaGraphDef to load.
+
+  Returns:
+    The meta_graph_def used for tflite conversion.
+
+  Raises:
+    ValueError: No valid MetaGraphDef for given tag_set.
+  """
+  saved_model = reader.read_saved_model(saved_model_dir)
+  tag_sets = []
+  result_meta_graph_def = None
+  for meta_graph_def in saved_model.meta_graphs:
+    meta_graph_tag_set = set(meta_graph_def.meta_info_def.tags)
+    tag_sets.append(meta_graph_tag_set)
+    if meta_graph_tag_set == tag_set:
+      result_meta_graph_def = meta_graph_def
+  logging.info("The given SavedModel contains the following tags: %s", tag_sets)
+  if result_meta_graph_def is not None:
+    return result_meta_graph_def
+  else:
+    raise ValueError("No valid MetaGraphDef for this tag_set '{}'. Possible "
+                     "values are '{}'. ".format(tag_set, tag_sets))
+
+
+def get_signature_def(meta_graph, signature_key):
+  """Get the signature def from meta_graph with given signature_key.
+
+  Args:
+    meta_graph: meta_graph_def.
+    signature_key: signature_def in the meta_graph_def.
+
+  Returns:
+    The signature_def used for tflite conversion.
+
+  Raises:
+    ValueError: Given signature_key is not valid for this meta_graph.
+  """
+  signature_def_map = meta_graph.signature_def
+  signature_def_keys = set(signature_def_map.keys())
+  logging.info(
+      "The given SavedModel MetaGraphDef contains SignatureDefs with the "
+      "following keys: %s", signature_def_keys)
+  if signature_key not in signature_def_keys:
+    raise ValueError("No '{}' in the saved_model\'s SignatureDefs. Possible "
+                     "values are '{}'. ".format(signature_key,
+                                                signature_def_keys))
+  signature_def = signature_def_utils.get_signature_def_by_key(
+      meta_graph, signature_key)
+  return signature_def
+
+
+def get_inputs_outputs(signature_def):
+  """Get inputs and outputs from signature def.
+
+  Args:
+    signature_def: signatuer def in the meta_graph_def for conversion.
+
+  Returns:
+    The inputs and outputs in the graph for conversion.
+  """
+  inputs_tensor_info = signature_def.inputs
+  outputs_tensor_info = signature_def.outputs
+  logging.info("input tensors info: ")
+  log_tensor_details(inputs_tensor_info)
+  logging.info("output tensors info: ")
+  log_tensor_details(outputs_tensor_info)
+
+  def gather_names(tensor_info):
+    return [tensor_info[key].name for key in tensor_info]
+
+  inputs = gather_names(inputs_tensor_info)
+  outputs = gather_names(outputs_tensor_info)
+  return inputs, outputs
+
+
+def convert(saved_model_dir,
+            output_tflite=None,
+            output_arrays=None,
+            tag_set=None,
+            signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
+            batch_size=1):
+  """Convert a savedmodel to tflite flatbuffer.
+
+  Args:
+    saved_model_dir: Saved model directory to convert.
+    output_tflite: File path to write result flatbuffer.
+    output_arrays: List of output tensor names, the default value is None, which
+      means conversion keeps all output tensors. This is also used to filter
+      tensors that are from Op currently not supported in tflite, e.g., Argmax).
+    tag_set: This is the set of tags to get meta_graph_def in saved_model.
+    signature_key: This is the signature key to extract inputs, outputs.
+    batch_size: If input tensor shape has None at first dimension,
+      e.g. (None,224,224,3), replace None with batch_size.
+
+  Returns:
+    The converted data. For example if tflite was the destination, then
+    this will be a tflite flatbuffer in a bytes array.
+
+  Raises:
+    ValueError: If tag_set does not indicate any meta_graph_def in saved_model,
+      or signature_key is not in relevant meta_graph_def,
+      or input shape has None beyond 1st dimension, e.g., (1,None, None, 3),
+      or given output_arrays are not valid causing empty outputs.
+  """
+  if tag_set is None:
+    tag_set = set([tag_constants.SERVING])
+
+  meta_graph = get_meta_graph_def(saved_model_dir, tag_set)
+  signature_def = get_signature_def(meta_graph, signature_key)
+  inputs, outputs = get_inputs_outputs(signature_def)
+
+  graph = ops.Graph()
+  with session.Session(graph=graph) as sess:
+
+    loader.load(sess, meta_graph.meta_info_def.tags, saved_model_dir)
+
+    in_tensors = [graph.get_tensor_by_name(input_) for input_ in inputs]
+
+    # Users can use output_arrays to filter output tensors for conversion.
+    # If output_arrays is None, we keep all output tensors. In future, we may
+    # use tflite supported Op list and check whether op is custom Op to
+    # automatically filter output arrays.
+    # TODO(zhixianyan): Use tflite supported Op list to filter outputs.
+    if output_arrays is not None:
+      output_arrays = output_arrays.split(",")
+      out_tensors = [
+          graph.get_tensor_by_name(output)
+          for output in outputs
+          if output.split(":")[0] in output_arrays
+      ]
+    else:
+      out_tensors = [graph.get_tensor_by_name(output) for output in outputs]
+
+    output_names = [node.split(":")[0] for node in outputs]
+
+    if not out_tensors:
+      raise ValueError(
+          "No valid output tensors for '{}', possible values are '{}'".format(
+              output_arrays, output_names))
+
+    frozen_graph_def = tf_graph_util.convert_variables_to_constants(
+        sess, graph.as_graph_def(), output_names)
+
+    # Toco requires fully defined tensor shape, for input tensor with None in
+    # their shape, e.g., (None, 224, 224, 3), we need to replace first None with
+    # a given batch size. For shape with more None, e.g. (None, None, None, 3),
+    # still be able to replace and convert, but require further investigation.
+    # TODO(zhixianyan): Add supports for input tensor with more None in shape.
+    for i in range(len(in_tensors)):
+      shape = in_tensors[i].get_shape().as_list()
+      if shape[0] is None:
+        shape[0] = batch_size
+      if None in shape[1:]:
+        raise ValueError(
+            "Only support None shape at 1st dim as batch_size. But tensor "
+            "'{}' 's shape '{}' has None at other dimension. ".format(
+                inputs[i], shape))
+      in_tensors[i].set_shape(shape)
+
+    result = lite.toco_convert(frozen_graph_def, in_tensors, out_tensors)
+
+    if output_tflite is not None:
+      with gfile.Open(output_tflite, "wb") as f:
+        f.write(result)
+      logging.info("Successfully converted to: %s", output_tflite)
+
+    return result
+
+
+def main(_):
+  convert(
+      saved_model_dir=flags.FLAGS.saved_model_dir,
+      output_tflite=flags.FLAGS.output_tflite,
+      output_arrays=flags.FLAGS.output_arrays,
+      batch_size=flags.FLAGS.batch_size,
+      tag_set=set(flags.FLAGS.tag_set.split(",")),
+      signature_key=flags.FLAGS.signature_key)
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/contrib/lite/python/convert_savedmodel_test.py b/tensorflow/contrib/lite/python/convert_savedmodel_test.py
new file mode 100644
index 0000000000..70cff9ef7f
--- /dev/null
+++ b/tensorflow/contrib/lite/python/convert_savedmodel_test.py
@@ -0,0 +1,276 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""TF Lite SavedModel Conversion test cases.
+
+ - test on generated saved_models from simple graphs (sanity check)
+ - test mnist savedmodel generated on-the-fly
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+from tensorflow.contrib.lite.python import convert_savedmodel
+from tensorflow.python import estimator
+from tensorflow.python import keras
+from tensorflow.python import layers
+from tensorflow.python import losses
+from tensorflow.python import nn
+from tensorflow.python import saved_model
+from tensorflow.python import train
+from tensorflow.python.client import session
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import random_ops
+from tensorflow.python.platform import test
+
+
+class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
+
+  def _createSimpleSavedModel(self, shape):
+    """Create a simple savedmodel on the fly."""
+    saved_model_dir = os.path.join(self.get_temp_dir(), "simple_savedmodel")
+    with session.Session() as sess:
+      in_tensor = array_ops.placeholder(shape=shape, dtype=dtypes.float32)
+      out_tensor = in_tensor + in_tensor
+      inputs = {"x": in_tensor}
+      outputs = {"y": out_tensor}
+      saved_model.simple_save(sess, saved_model_dir, inputs, outputs)
+    return saved_model_dir
+
+  def testSimpleSavedModel(self):
+    """Test a simple savedmodel created on the fly."""
+    # Create a simple savedmodel
+    saved_model_dir = self._createSimpleSavedModel(shape=[1, 16, 16, 3])
+    # Convert to tflite
+    result = convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+    self.assertTrue(result)
+
+  def testSimpleSavedModelWithNoneBatchSizeInShape(self):
+    """Test a simple savedmodel, with None in input tensor's shape."""
+    saved_model_dir = self._createSimpleSavedModel(shape=[None, 16, 16, 3])
+    result = convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+    self.assertTrue(result)
+
+  def testSimpleSavedModelWithMoreNoneInShape(self):
+    """Test a simple savedmodel, fail as more None in input shape."""
+    saved_model_dir = self._createSimpleSavedModel(shape=[None, 16, None, 3])
+    # Convert to tflite: this should raise ValueError, as 3rd dim is None.
+    with self.assertRaises(ValueError):
+      convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+
+  def testSimpleSavedModelWithWrongSignatureKey(self):
+    """Test a simple savedmodel, fail as given signature is invalid."""
+    saved_model_dir = self._createSimpleSavedModel(shape=[1, 16, 16, 3])
+    # Convert to tflite: this should raise ValueError, as
+    # signature_key does not exit in the saved_model.
+    with self.assertRaises(ValueError):
+      convert_savedmodel.convert(
+          saved_model_dir=saved_model_dir, signature_key="wrong-key")
+
+  def testSimpleSavedModelWithWrongOutputArray(self):
+    """Test a simple savedmodel, fail as given output_arrays is invalid."""
+    # Create a simple savedmodel
+    saved_model_dir = self._createSimpleSavedModel(shape=[1, 16, 16, 3])
+    # Convert to tflite: this should raise ValueError, as
+    # output_arrays is not valid for the saved_model.
+    with self.assertRaises(ValueError):
+      convert_savedmodel.convert(
+          saved_model_dir=saved_model_dir, output_arrays="wrong-output")
+
+  def testMultipleMetaGraphDef(self):
+    """Test saved model with multiple MetaGraphDef."""
+    saved_model_dir = os.path.join(self.get_temp_dir(), "savedmodel_two_mgd")
+    builder = saved_model.builder.SavedModelBuilder(saved_model_dir)
+    with session.Session(graph=ops.Graph()) as sess:
+      # MetaGraphDef 1
+      in_tensor = array_ops.placeholder(shape=[1, 28, 28], dtype=dtypes.float32)
+      out_tensor = in_tensor + in_tensor
+      sig_input_tensor = saved_model.utils.build_tensor_info(in_tensor)
+      sig_input_tensor_signature = {"x": sig_input_tensor}
+      sig_output_tensor = saved_model.utils.build_tensor_info(out_tensor)
+      sig_output_tensor_signature = {"y": sig_output_tensor}
+      predict_signature_def = (
+          saved_model.signature_def_utils.build_signature_def(
+              sig_input_tensor_signature, sig_output_tensor_signature,
+              saved_model.signature_constants.PREDICT_METHOD_NAME))
+      signature_def_map = {
+          saved_model.signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY:
+              predict_signature_def
+      }
+      builder.add_meta_graph_and_variables(
+          sess,
+          tags=[saved_model.tag_constants.SERVING, "additional_test_tag"],
+          signature_def_map=signature_def_map)
+      # MetaGraphDef 2
+      builder.add_meta_graph(tags=["tflite"])
+      builder.save(True)
+
+    # Convert to tflite
+    convert_savedmodel.convert(
+        saved_model_dir=saved_model_dir,
+        tag_set=set([saved_model.tag_constants.SERVING, "additional_test_tag"]))
+
+
+class Model(keras.Model):
+  """Model to recognize digits in the MNIST dataset.
+
+  Train and export savedmodel, used for testOnflyTrainMnistSavedModel
+
+  Network structure is equivalent to:
+  https://github.com/tensorflow/tensorflow/blob/r1.5/tensorflow/examples/tutorials/mnist/mnist_deep.py
+  and
+  https://github.com/tensorflow/models/blob/master/tutorials/image/mnist/convolutional.py
+
+  But written as a ops.keras.Model using the layers API.
+  """
+
+  def __init__(self, data_format):
+    """Creates a model for classifying a hand-written digit.
+
+    Args:
+      data_format: Either "channels_first" or "channels_last".
+        "channels_first" is typically faster on GPUs while "channels_last" is
+        typically faster on CPUs. See
+        https://www.tensorflow.org/performance/performance_guide#data_formats
+    """
+    super(Model, self).__init__()
+    self._input_shape = [-1, 28, 28, 1]
+
+    self.conv1 = layers.Conv2D(
+        32, 5, padding="same", data_format=data_format, activation=nn.relu)
+    self.conv2 = layers.Conv2D(
+        64, 5, padding="same", data_format=data_format, activation=nn.relu)
+    self.fc1 = layers.Dense(1024, activation=nn.relu)
+    self.fc2 = layers.Dense(10)
+    self.dropout = layers.Dropout(0.4)
+    self.max_pool2d = layers.MaxPooling2D(
+        (2, 2), (2, 2), padding="same", data_format=data_format)
+
+  def __call__(self, inputs, training):
+    """Add operations to classify a batch of input images.
+
+    Args:
+      inputs: A Tensor representing a batch of input images.
+      training: A boolean. Set to True to add operations required only when
+        training the classifier.
+
+    Returns:
+      A logits Tensor with shape [<batch_size>, 10].
+    """
+    y = array_ops.reshape(inputs, self._input_shape)
+    y = self.conv1(y)
+    y = self.max_pool2d(y)
+    y = self.conv2(y)
+    y = self.max_pool2d(y)
+    y = layers.flatten(y)
+    y = self.fc1(y)
+    y = self.dropout(y, training=training)
+    return self.fc2(y)
+
+
+def model_fn(features, labels, mode, params):
+  """The model_fn argument for creating an Estimator."""
+  model = Model(params["data_format"])
+  image = features
+  if isinstance(image, dict):
+    image = features["image"]
+
+  if mode == estimator.ModeKeys.PREDICT:
+    logits = model(image, training=False)
+    predictions = {
+        "classes": math_ops.argmax(logits, axis=1),
+        "probabilities": nn.softmax(logits),
+    }
+    return estimator.EstimatorSpec(
+        mode=estimator.ModeKeys.PREDICT,
+        predictions=predictions,
+        export_outputs={
+            "classify": estimator.export.PredictOutput(predictions)
+        })
+
+  elif mode == estimator.ModeKeys.TRAIN:
+    optimizer = train.AdamOptimizer(learning_rate=1e-4)
+
+    logits = model(image, training=True)
+    loss = losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
+    return estimator.EstimatorSpec(
+        mode=estimator.ModeKeys.TRAIN,
+        loss=loss,
+        train_op=optimizer.minimize(loss, train.get_or_create_global_step()))
+
+  elif mode == estimator.ModeKeys.EVAL:
+    logits = model(image, training=False)
+    loss = losses.sparse_softmax_cross_entropy(labels=labels, logits=logits)
+    return estimator.EstimatorSpec(
+        mode=estimator.ModeKeys.EVAL,
+        loss=loss,
+        eval_metric_ops={
+            "accuracy":
+                ops.metrics.accuracy(
+                    labels=labels, predictions=math_ops.argmax(logits, axis=1)),
+        })
+
+
+def dummy_input_fn():
+  image = random_ops.random_uniform([100, 784])
+  labels = random_ops.random_uniform([100, 1], maxval=9, dtype=dtypes.int32)
+  return image, labels
+
+
+class ConvertSavedModelTestTrainGraph(test_util.TensorFlowTestCase):
+
+  def testTrainedMnistSavedModel(self):
+    """Test mnist savedmodel, trained with dummy data and small steps."""
+    # Build classifier
+    classifier = estimator.Estimator(
+        model_fn=model_fn,
+        params={
+            "data_format": "channels_last"  # tflite format
+        })
+
+    # Train and pred for serving
+    classifier.train(input_fn=dummy_input_fn, steps=2)
+    image = array_ops.placeholder(dtypes.float32, [None, 28, 28])
+    pred_input_fn = estimator.export.build_raw_serving_input_receiver_fn({
+        "image": image,
+    })
+
+    # Export savedmodel
+    saved_model_dir = os.path.join(self.get_temp_dir(), "mnist_savedmodel")
+    classifier.export_savedmodel(saved_model_dir, pred_input_fn)
+
+    # Convert to tflite and test output
+    saved_model_name = os.listdir(saved_model_dir)[0]
+    saved_model_final_dir = os.path.join(saved_model_dir, saved_model_name)
+    output_tflite = os.path.join(saved_model_dir,
+                                 saved_model_final_dir + ".lite")
+    # TODO(zhixianyan): no need to limit output_arrays to `Softmax'
+    # once b/74205001 fixed and argmax implemented in tflite.
+    result = convert_savedmodel.convert(
+        saved_model_dir=saved_model_final_dir,
+        output_arrays="Softmax",
+        output_tflite=output_tflite)
+
+    self.assertTrue(result)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index d55a883df5..8a80d6443b 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -158,6 +158,7 @@ sh_binary(
             "//tensorflow/contrib/keras:keras",
             "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
             "//tensorflow/contrib/lite/python:interpreter_test_data",
+            "//tensorflow/contrib/lite/python:tf_lite_py_pip",
             "//tensorflow/contrib/lite/toco:toco",
             "//tensorflow/contrib/lite/toco/python:toco_wrapper",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
-- 
GitLab


From e07e70a414e83f0b15d3f7f94b9134bd86d4a1c7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 18:05:03 -0700
Subject: [PATCH 397/960] Add asserts for LinearOperatorBlockDiag.

All asserts are only dependent on the constituent operators due to properties of the direct sum.

PiperOrigin-RevId: 190156830
---
 .../linalg/python/ops/linear_operator_block_diag.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
index 5d7a99664d..80649bd52d 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
+++ b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
@@ -24,6 +24,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops.linalg import linear_operator
 from tensorflow.python.ops.linalg import linear_operator_util
 
@@ -333,6 +334,18 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator):
     mat.set_shape(self.shape)
     return mat
 
+  def _assert_non_singular(self):
+    return control_flow_ops.group([
+        operator.assert_non_singular() for operator in self.operators])
+
+  def _assert_self_adjoint(self):
+    return control_flow_ops.group([
+        operator.assert_self_adjoint() for operator in self.operators])
+
+  def _assert_positive_definite(self):
+    return control_flow_ops.group([
+        operator.assert_positive_definite() for operator in self.operators])
+
   def _split_input_into_blocks(self, x, axis=-1):
     """Split `x` into blocks matching `operators`'s `domain_dimension`.
 
-- 
GitLab


From dbea93d7f14be9451f1ec27c0b990fa680dee947 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Thu, 22 Mar 2018 18:20:09 -0700
Subject: [PATCH 398/960] [tf.data] Add `tf.contrib.data.prefetch_to_device()`,
 which supports prefetching to GPU memory.

PiperOrigin-RevId: 190158272
---
 tensorflow/contrib/data/BUILD                 |   1 +
 tensorflow/contrib/data/__init__.py           |   2 +
 .../data/kernels/prefetching_kernels.cc       |  21 +++
 tensorflow/contrib/data/ops/dataset_ops.cc    |   8 ++
 .../contrib/data/python/kernel_tests/BUILD    |   4 -
 .../kernel_tests/prefetching_ops_test.py      |  48 +++++++
 tensorflow/contrib/data/python/ops/BUILD      |   4 +
 .../data/python/ops/prefetching_ops.py        | 130 ++++++++++++++++++
 8 files changed, 214 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index d787ed8a1a..9e25a77d9f 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -21,6 +21,7 @@ py_library(
     deps = [
         "//tensorflow/contrib/data/python/ops:dataset_ops",
         "//tensorflow/contrib/data/python/ops:iterator_ops",
+        "//tensorflow/contrib/data/python/ops:prefetching_ops",
         "//tensorflow/contrib/data/python/ops:readers",
         "//tensorflow/contrib/data/python/ops:shuffle_ops",
         "//tensorflow/contrib/data/python/ops:transformation_ops",
diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 9212b69700..766721d8d2 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -36,6 +36,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@map_and_batch
 @@padded_batch_and_drop_remainder
 @@parallel_interleave
+@@prefetch_to_device
 @@read_batch_features
 @@rejection_resample
 @@scan
@@ -67,6 +68,7 @@ from tensorflow.contrib.data.python.ops.grouping import group_by_window
 from tensorflow.contrib.data.python.ops.interleave_ops import parallel_interleave
 from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave
 from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator
+from tensorflow.contrib.data.python.ops.prefetching_ops import prefetch_to_device
 from tensorflow.contrib.data.python.ops.readers import make_batched_features_dataset
 from tensorflow.contrib.data.python.ops.readers import read_batch_features
 from tensorflow.contrib.data.python.ops.readers import SqlDataset
diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 2f986f2bb1..190a6ee580 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -406,4 +406,25 @@ REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext")
                         FunctionBufferingResourceGetNextOp);
 #endif  // TENSORFLOW_USE_SYCL
 
+class IteratorGetDeviceOp : public OpKernel {
+ public:
+  using OpKernel::OpKernel;
+
+  void Compute(OpKernelContext* ctx) override {
+    // NOTE(mrry): We do not currently Validate that the handle
+    // corresponds to a real IteratorResource, because that symbol is
+    // not exposed from the framework library.
+    Tensor* device_name_t;
+    OP_REQUIRES_OK(ctx,
+                   ctx->allocate_output(0, TensorShape({}), &device_name_t));
+    // NOTE(mrry): Since the operation's input is a resource, we must be
+    // colocated with it, and so we can simply return the current device's
+    // name without looking at the input.
+    device_name_t->scalar<string>()() = ctx->device()->name();
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("IteratorGetDevice").Device(DEVICE_CPU),
+                        IteratorGetDeviceOp);
+
 }  // namespace tensorflow
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index a4c1212da1..74737bbcad 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -37,6 +37,14 @@ REGISTER_OP("UniqueDataset")
 Creates a dataset that contains the unique elements of `input_dataset`.
 )doc");
 
+REGISTER_OP("IteratorGetDevice")
+    .Input("resource: resource")
+    .Output("device: string")
+    .SetShapeFn(shape_inference::ScalarShape)
+    .Doc(R"doc(
+Returns the name of the device on which `resource` has been placed.
+)doc");
+
 REGISTER_OP("FunctionBufferingResource")
     .Input("string_arg: string")
     .Input("target_device: string")
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 2c4d4adfda..f70b29c43b 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -479,10 +479,6 @@ py_test(
     size = "small",
     srcs = ["prefetching_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "manual",
-        "no_oss",  # b/68785503
-    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:prefetching_ops",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index dc3e38db59..1d74afe1e1 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -26,6 +26,7 @@ from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -107,6 +108,53 @@ class StagingAreaOpsTest(test.TestCase):
     self._prefetch_fn_helper("cpu_gpu", "/job:localhost/replica:0/task:0/cpu:0",
                              "/job:localhost/replica:0/task:0/gpu:0")
 
+  def testPrefetchToDevice(self):
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops.prefetch_to_device("/cpu:1"))
+
+    # NOTE(mrry): This device block creates the "host" dataset and iterator on
+    # /cpu:0, and ensures that the prefetching is across devices. In typical use
+    # this would not be necessary, because the GPU device would not support any
+    # of the dataset-related ops.
+    with ops.device("/cpu:0"):
+      iterator = device_dataset.make_one_shot_iterator()
+
+    self.assertEqual(host_dataset.output_types, device_dataset.output_types)
+    self.assertEqual(host_dataset.output_types, iterator.output_types)
+    self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
+    self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
+    self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
+    self.assertEqual(host_dataset.output_classes, iterator.output_classes)
+
+    next_element = iterator.get_next()
+    self.assertEqual(dtypes.int64, next_element.dtype)
+    self.assertEqual([], next_element.shape)
+
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+    with self.test_session(config=worker_config) as sess:
+      for i in range(10):
+        self.assertEqual(i, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+  def testPrefetchToDeviceGpu(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops.prefetch_to_device("/gpu:0"))
+
+    iterator = device_dataset.make_one_shot_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      for i in range(10):
+        self.assertEqual(i, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index c3331e9636..4ecf02825f 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -173,6 +173,10 @@ py_library(
     srcs = ["prefetching_ops.py"],
     deps = [
         ":contrib_op_loader",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
+        "//tensorflow/python/data/util:sparse",
     ],
 )
 
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index b16f12c4ee..e38d53a221 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -17,8 +17,17 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import warnings
+
 from tensorflow.contrib.data.python.ops import contrib_op_loader  # pylint: disable=unused-import
 from tensorflow.contrib.data.python.ops import gen_dataset_ops
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.data.util import nest
+from tensorflow.python.data.util import sparse
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
+from tensorflow.python.framework import ops
 
 
 # TODO(rohanj): Add a python class that constructs resource in the __init__
@@ -51,3 +60,124 @@ def function_buffering_resource_get_next(function_buffer_resource,
       function_buffer_resource=function_buffer_resource,
       output_types=output_types,
       name=name)
+
+
+# pylint: disable=protected-access
+class _PrefetchToDeviceIterator(object):
+  """A replacement for @{tf.data.Iterator} that prefetches to another device."""
+
+  def __init__(self, input_dataset, device, buffer_size):
+    self._input_dataset = input_dataset
+    self._get_next_call_count = 0
+    input_iterator = input_dataset.make_one_shot_iterator()
+    input_iterator_handle = input_iterator.string_handle()
+
+    @function.Defun(dtypes.string)
+    def _prefetch_fn(handle):
+      remote_iterator = iterator_ops.Iterator.from_string_handle(
+          handle, input_iterator.output_types, input_iterator.output_shapes,
+          input_iterator.output_classes)
+      return remote_iterator.get_next()
+
+    with ops.device(device):
+      self._buffering_resource = function_buffering_resource(
+          f=_prefetch_fn,
+          target_device=gen_dataset_ops.iterator_get_device(
+              input_iterator._iterator_resource),
+          string_arg=input_iterator_handle,
+          buffer_size=buffer_size,
+          thread_pool_size=0)
+
+  def get_next(self, name=None):
+    """See @{tf.data.Iterator.get_next}."""
+    self._get_next_call_count += 1
+    if self._get_next_call_count > iterator_ops.GET_NEXT_CALL_WARNING_THRESHOLD:
+      warnings.warn(iterator_ops.GET_NEXT_CALL_WARNING_MESSAGE)
+
+    flat_ret = gen_dataset_ops.function_buffering_resource_get_next(
+        self._buffering_resource,
+        output_types=nest.flatten(sparse.as_dense_types(
+            self.output_types, self.output_classes)), name=name)
+
+    ret = sparse.deserialize_sparse_tensors(
+        nest.pack_sequence_as(self.output_types, flat_ret),
+        self.output_types, self.output_shapes, self.output_classes)
+
+    for tensor, shape in zip(
+        nest.flatten(ret), nest.flatten(self.output_shapes)):
+      if isinstance(tensor, ops.Tensor):
+        tensor.set_shape(shape)
+
+    return ret
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+# pylint: enable=protected-access
+
+
+class _PrefetchToDeviceDataset(dataset_ops.Dataset):
+  """A `Dataset` whose iterator prefetches elements to another device."""
+
+  def __init__(self, input_dataset, device, buffer_size):
+    self._input_dataset = input_dataset
+    self._device = device
+    self._buffer_size = buffer_size if buffer_size is not None else 1
+
+  def make_one_shot_iterator(self):
+    return _PrefetchToDeviceIterator(self._input_dataset, self._device,
+                                     self._buffer_size)
+
+  def make_initializable_iterator(self, shared_name=None):
+    raise NotImplementedError("`prefetch_to_device()` is not currently "
+                              "compatible with initializable iterators. Use "
+                              "`make_one_shot_iterator()` instead.")
+
+  def _as_variant_tensor(self):
+    # TODO(mrry): Raise this error earlier (e.g. when one of the Dataset
+    # transformation methods is called.
+    # TODO(mrry): Investigate support for chaining further transformations after
+    # the prefetch, including GPU support.
+    raise NotImplementedError("`prefetch_to_device()` must be the last "
+                              "transformation in a dataset pipeline.")
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+
+def prefetch_to_device(device, buffer_size=None):
+  """A transformation that prefetches dataset values to the given `device`.
+
+  NOTE: Although the transformation creates a @{tf.data.Dataset}, the
+  transformation must be the final `Dataset` in the input pipeline.
+
+  Args:
+    device: A string. The name of a device to which elements will be prefetched.
+    buffer_size: (Optional.) The number of elements to buffer on `device`.
+      Defaults to an automatically chosen value.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return _PrefetchToDeviceDataset(dataset, device, buffer_size)
+
+  return _apply_fn
-- 
GitLab


From 37f401da65feea7afa95e83c5daf84e721b74f54 Mon Sep 17 00:00:00 2001
From: Yashal Shakti Kanungo <yashalk@amazon.com>
Date: Fri, 23 Mar 2018 07:21:04 +0530
Subject: [PATCH 399/960] Fix incosistencies in the shape of the input Tensor

---
 tensorflow/docs_src/tutorials/layers.md | 54 +++++++++++++------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index aeb746f29c..cadaec391d 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -198,17 +198,23 @@ Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
 ### Input Layer
 
 The methods in the `layers` module for creating convolutional and pooling layers
-for two-dimensional image data expect input tensors to have a `channels_last` shape of
-<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>, <em>channels</em>]</code>
-or a `channels_first` shape of <code>[<em>batch_size</em>, <em>channels</em>, <em>image_height</em>, <em>image_width</em>]</code>, defined as follows:
+for two-dimensional image data expect input tensors to have a shape of
+<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>,
+<em>channels</em>]</code> by default. This behavior can be changed using the <code><em>data_format</em></code> parameter; defined as follows:
+
 
 *   _`batch_size`_. Size of the subset of examples to use when performing
     gradient descent during training.
-*   _`image_width`_. Width of the example images.
 *   _`image_height`_. Height of the example images.
+*   _`image_width`_. Width of the example images.
 *   _`channels`_. Number of color channels in the example images. For color
     images, the number of channels is 3 (red, green, blue). For monochrome
     images, there is just 1 channel (black).
+*   _`image_height`_. Height of the example images.
+*   _`data_format`_. A string, one of `channels_last` (default) or `channels_first`.
+      `channels_last` corresponds to inputs with shape
+      `(batch, ..., channels)` while `channels_first` corresponds to
+      inputs with shape `(batch, channels, ...)`.
 
 Here, our MNIST dataset is composed of monochrome 28x28 pixel images, so the
 desired shape for our input layer is <code>[<em>batch_size</em>, 28, 28,
@@ -247,28 +253,27 @@ conv1 = tf.layers.conv2d(
 ```
 
 The `inputs` argument specifies our input tensor, which must have the shape
-<code>[<em>batch_size</em>, <em>image_width</em>, <em>image_height</em>,
+<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>,
 <em>channels</em>]</code>. Here, we're connecting our first convolutional layer
 to `input_layer`, which has the shape <code>[<em>batch_size</em>, 28, 28,
 1]</code>.
 
 > Note: <code>conv2d()</code> will instead accept a shape of
-> <code>[<em>channels</em>, <em>batch_size</em>, <em>image_width</em>,
-> <em>image_height</em>]</code> when passed the argument
+> <code>[<em>batch_size</em>, <em>channels</em>, <em>image_height</em>, <em>image_width</em>]</code> when passed the argument
 > <code>data_format=channels_first</code>.
 
 The `filters` argument specifies the number of filters to apply (here, 32), and
-`kernel_size` specifies the dimensions of the filters as <code>[<em>width</em>,
-<em>height</em>]</code> (here, <code>[5, 5]</code>).
+`kernel_size` specifies the dimensions of the filters as <code>[<em>height</em>,
+<em>width</em>]</code> (here, <code>[5, 5]</code>).
 
-<p class="tip"><b>TIP:</b> If filter width and height have the same value, you can instead specify a
+<p class="tip"><b>TIP:</b> If filter height and width have the same value, you can instead specify a
 single integer for <code>kernel_size</code>—e.g., <code>kernel_size=5</code>.</p>
 
 The `padding` argument specifies one of two enumerated values
 (case-insensitive): `valid` (default value) or `same`. To specify that the
-output tensor should have the same width and height values as the input tensor,
+output tensor should have the same height and width values as the input tensor,
 we set `padding=same` here, which instructs TensorFlow to add 0 values to the
-edges of the input tensor to preserve width and height of 28. (Without padding,
+edges of the input tensor to preserve height and width of 28. (Without padding,
 a 5x5 convolution over a 28x28 tensor will produce a 24x24 tensor, as there are
 24x24 locations to extract a 5x5 tile from a 28x28 grid.)
 
@@ -277,7 +282,7 @@ output of the convolution. Here, we specify ReLU activation with
 @{tf.nn.relu}.
 
 Our output tensor produced by `conv2d()` has a shape of
-<code>[<em>batch_size</em>, 28, 28, 32]</code>: the same width and height
+<code>[<em>batch_size</em>, 28, 28, 32]</code>: the same height and width
 dimensions as the input, but now with 32 channels holding the output from each
 of the filters.
 
@@ -292,31 +297,30 @@ pool1 = tf.layers.max_pooling2d(inputs=conv1, pool_size=[2, 2], strides=2)
 ```
 
 Again, `inputs` specifies the input tensor, with a shape of
-<code>[<em>batch_size</em>, <em>image_width</em>, <em>image_height</em>,
+<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>,
 <em>channels</em>]</code>. Here, our input tensor is `conv1`, the output from
 the first convolutional layer, which has a shape of <code>[<em>batch_size</em>,
 28, 28, 32]</code>.
 
 > Note: As with <code>conv2d()</code>, <code>max_pooling2d()</code> will instead
-> accept a shape of <code>[<em>channels</em>, <em>batch_size</em>,
-> <em>image_width</em>, <em>image_height</em>]</code> when passed the argument
+> accept a shape of <code>[<em>batch_size</em>, <em>channels</em>, 
+> <em>image_height</em>, <em>image_width</em>]</code> when passed the argument
 > <code>data_format=channels_first</code>.
 
 The `pool_size` argument specifies the size of the max pooling filter as
-<code>[<em>width</em>, <em>height</em>]</code> (here, `[2, 2]`). If both
+<code>[<em>height</em>, <em>width</em>]</code> (here, `[2, 2]`). If both
 dimensions have the same value, you can instead specify a single integer (e.g.,
 `pool_size=2`).
 
 The `strides` argument specifies the size of the stride. Here, we set a stride
 of 2, which indicates that the subregions extracted by the filter should be
-separated by 2 pixels in both the width and height dimensions (for a 2x2 filter,
+separated by 2 pixels in both the height and width dimensions (for a 2x2 filter,
 this means that none of the regions extracted will overlap). If you want to set
-different stride values for width and height, you can instead specify a tuple or
+different stride values for height and width, you can instead specify a tuple or
 list (e.g., `stride=[3, 6]`).
 
 Our output tensor produced by `max_pooling2d()` (`pool1`) has a shape of
-<code>[<em>batch_size</em>, 14, 14, 32]</code>: the 2x2 filter reduces width and
-height by 50% each.
+<code>[<em>batch_size</em>, 14, 14, 32]</code>: the 2x2 filter reduces height and width by 50% each.
 
 ### Convolutional Layer #2 and Pooling Layer #2
 
@@ -338,13 +342,11 @@ pool2 = tf.layers.max_pooling2d(inputs=conv2, pool_size=[2, 2], strides=2)
 
 Note that convolutional layer #2 takes the output tensor of our first pooling
 layer (`pool1`) as input, and produces the tensor `conv2` as output. `conv2`
-has a shape of <code>[<em>batch_size</em>, 14, 14, 64]</code>, the same width
-and height as `pool1` (due to `padding="same"`), and 64 channels for the 64
+has a shape of <code>[<em>batch_size</em>, 14, 14, 64]</code>, the same height and width as `pool1` (due to `padding="same"`), and 64 channels for the 64
 filters applied.
 
 Pooling layer #2 takes `conv2` as input, producing `pool2` as output. `pool2`
-has shape <code>[<em>batch_size</em>, 7, 7, 64]</code> (50% reduction of width
-and height from `conv2`).
+has shape <code>[<em>batch_size</em>, 7, 7, 64]</code> (50% reduction of height and width from `conv2`).
 
 ### Dense Layer
 
@@ -360,7 +362,7 @@ pool2_flat = tf.reshape(pool2, [-1, 7 * 7 * 64])
 
 In the `reshape()` operation above, the `-1` signifies that the *`batch_size`*
 dimension will be dynamically calculated based on the number of examples in our
-input data. Each example has 7 (`pool2` width) * 7 (`pool2` height) * 64
+input data. Each example has 7 (`pool2` height) * 7 (`pool2` width) * 64
 (`pool2` channels) features, so we want the `features` dimension to have a value
 of 7 * 7 * 64 (3136 in total). The output tensor, `pool2_flat`, has shape
 <code>[<em>batch_size</em>, 3136]</code>.
-- 
GitLab


From 0dadbfe118c1d0510deaf4f5bc0a78c6b0dbda9a Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 22 Mar 2018 18:54:27 -0700
Subject: [PATCH 400/960] Merge changes from github.

PiperOrigin-RevId: 190161440
---
 configure.py                                     |  2 +-
 tensorflow/contrib/data/python/ops/grouping.py   |  2 +-
 tensorflow/contrib/lite/toco/g3doc/python_api.md |  2 +-
 tensorflow/core/platform/default/logging.h       |  6 +++---
 tensorflow/docs_src/extend/add_filesys.md        |  2 +-
 tensorflow/docs_src/extend/new_data_formats.md   |  2 +-
 tensorflow/docs_src/tutorials/layers.md          |  6 +++---
 tensorflow/python/client/session.py              |  2 ++
 tensorflow/python/ops/math_ops.py                | 11 +++++++----
 tensorflow/python/ops/sets_impl.py               |  2 +-
 tensorflow/tools/test/upload_test_benchmarks.py  |  1 +
 third_party/sycl/sycl/BUILD.tpl                  |  2 +-
 12 files changed, 23 insertions(+), 17 deletions(-)

diff --git a/configure.py b/configure.py
index ea732c64e2..22b9abedd7 100644
--- a/configure.py
+++ b/configure.py
@@ -40,7 +40,7 @@ _DEFAULT_CUDA_PATH = '/usr/local/cuda'
 _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
 _DEFAULT_CUDA_PATH_WIN = ('C:/Program Files/NVIDIA GPU Computing '
                           'Toolkit/CUDA/v%s' % _DEFAULT_CUDA_VERSION)
-_DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/x86_64-linux-gnu'
+_DEFAULT_TENSORRT_PATH_LINUX = '/usr/lib/%s-linux-gnu' % platform.machine()
 _TF_OPENCL_VERSION = '1.2'
 _DEFAULT_COMPUTECPP_TOOLKIT_PATH = '/usr/local/computecpp'
 _DEFAULT_TRISYCL_INCLUDE_DIR = '/usr/local/triSYCL/include'
diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py
index a19be22254..ae10d2eb22 100644
--- a/tensorflow/contrib/data/python/ops/grouping.py
+++ b/tensorflow/contrib/data/python/ops/grouping.py
@@ -42,7 +42,7 @@ def group_by_window(key_func,
   This transformation maps each consecutive element in a dataset to a key
   using `key_func` and groups the elements by key. It then applies
   `reduce_func` to at most `window_size_func(key)` elements matching the same
-  key. All execpt the final window for each key will contain
+  key. All except the final window for each key will contain
   `window_size_func(key)` elements; the final window may be smaller.
 
   You may provide either a constant `window_size` or a window size determined by
diff --git a/tensorflow/contrib/lite/toco/g3doc/python_api.md b/tensorflow/contrib/lite/toco/g3doc/python_api.md
index 440f9c367c..36e2d9c372 100644
--- a/tensorflow/contrib/lite/toco/g3doc/python_api.md
+++ b/tensorflow/contrib/lite/toco/g3doc/python_api.md
@@ -28,7 +28,7 @@ val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
 out = tf.identity(val, name="out")
 with tf.Session() as sess:
   tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
-  open("test.tflite", "wb").write(tflite_modeL)
+  open("test.tflite", "wb").write(tflite_model)
 ```
 
 **NOTE** Currently, the TOCO command will cause a fatal error to the Python
diff --git a/tensorflow/core/platform/default/logging.h b/tensorflow/core/platform/default/logging.h
index f0efa31d55..2c134f1be9 100644
--- a/tensorflow/core/platform/default/logging.h
+++ b/tensorflow/core/platform/default/logging.h
@@ -64,11 +64,11 @@ class LogMessageFatal : public LogMessage {
 };
 
 #define _TF_LOG_INFO \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::INFO)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::INFO)
 #define _TF_LOG_WARNING \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::WARNING)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::WARNING)
 #define _TF_LOG_ERROR \
-  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, tensorflow::ERROR)
+  ::tensorflow::internal::LogMessage(__FILE__, __LINE__, ::tensorflow::ERROR)
 #define _TF_LOG_FATAL \
   ::tensorflow::internal::LogMessageFatal(__FILE__, __LINE__)
 
diff --git a/tensorflow/docs_src/extend/add_filesys.md b/tensorflow/docs_src/extend/add_filesys.md
index 06f11de4eb..bc0f662f0c 100644
--- a/tensorflow/docs_src/extend/add_filesys.md
+++ b/tensorflow/docs_src/extend/add_filesys.md
@@ -225,7 +225,7 @@ it will use the `FooBarFileSystem` implementation.
 Next, you must build a shared object containing this implementation. An example
 of doing so using bazel's `cc_binary` rule can be found
 [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/BUILD#L244),
-but you may use any build system to do so. See the section on @{$adding_an_op#build-the-op-library$building the op library} for similar
+but you may use any build system to do so. See the section on @{$adding_an_op#build_the_op_library$building the op library} for similar
 instructions.
 
 The result of building this target is a `.so` shared object file.
diff --git a/tensorflow/docs_src/extend/new_data_formats.md b/tensorflow/docs_src/extend/new_data_formats.md
index b3cc968047..10e717c280 100644
--- a/tensorflow/docs_src/extend/new_data_formats.md
+++ b/tensorflow/docs_src/extend/new_data_formats.md
@@ -167,7 +167,7 @@ REGISTER_KERNEL_BUILDER(Name("TextLineReader").Device(DEVICE_CPU),
 ```
 
 The last step is to add the Python wrapper.  You can either do this by
-@{$adding_an_op#building_the_op_library$compiling a dynamic library}
+@{$adding_an_op#build_the_op_library$compiling a dynamic library}
 or, if you are building TensorFlow from source, adding to `user_ops.py`.
 For the latter, you will import `tensorflow.python.ops.io_ops` in
 [`tensorflow/python/user_ops/user_ops.py`](https://www.tensorflow.org/code/tensorflow/python/user_ops/user_ops.py)
diff --git a/tensorflow/docs_src/tutorials/layers.md b/tensorflow/docs_src/tutorials/layers.md
index 9b17d0d4d5..aeb746f29c 100644
--- a/tensorflow/docs_src/tutorials/layers.md
+++ b/tensorflow/docs_src/tutorials/layers.md
@@ -198,9 +198,9 @@ Classifier"](#training_and_evaluating_the_cnn_mnist_classifier).
 ### Input Layer
 
 The methods in the `layers` module for creating convolutional and pooling layers
-for two-dimensional image data expect input tensors to have a shape of
-<code>[<em>batch_size</em>, <em>image_width</em>, <em>image_height</em>,
-<em>channels</em>]</code>, defined as follows:
+for two-dimensional image data expect input tensors to have a `channels_last` shape of
+<code>[<em>batch_size</em>, <em>image_height</em>, <em>image_width</em>, <em>channels</em>]</code>
+or a `channels_first` shape of <code>[<em>batch_size</em>, <em>channels</em>, <em>image_height</em>, <em>image_width</em>]</code>, defined as follows:
 
 *   _`batch_size`_. Size of the subset of examples to use when performing
     gradient descent during training.
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 6e9ce9b080..da5dc6f599 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -889,6 +889,8 @@ class BaseSession(SessionInterface):
       Either a single value if `fetches` is a single graph element, or
       a list of values if `fetches` is a list, or a dictionary with the
       same keys as `fetches` if that is a dictionary (described above).
+      Order in which `fetches` operations are evaluated inside the call
+      is undefined.
 
     Raises:
       RuntimeError: If this `Session` is in an invalid state (e.g. has been
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 4699e05269..276897ab99 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -167,6 +167,7 @@ from tensorflow.python.ops import gen_math_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gen_sparse_ops
 from tensorflow.python.ops import gen_spectral_ops
+from tensorflow.python.platform import tf_logging as logging
 # go/tf-wildcard-import
 # pylint: disable=wildcard-import
 from tensorflow.python.ops.gen_math_ops import *
@@ -775,16 +776,18 @@ def cast(x, dtype, name=None):
   with ops.name_scope(name, "Cast", [x]) as name:
     if isinstance(x, sparse_tensor.SparseTensor):
       values_cast = cast(x.values, base_type, name=name)
-      return sparse_tensor.SparseTensor(x.indices, values_cast, x.dense_shape)
+      x = sparse_tensor.SparseTensor(x.indices, values_cast, x.dense_shape)
     else:
       # TODO(josh11b): If x is not already a Tensor, we could return
       # ops.convert_to_tensor(x, dtype=dtype, ...)  here, but that
       # allows some conversions that cast() can't do, e.g. casting numbers to
       # strings.
       x = ops.convert_to_tensor(x, name="x")
-      if x.dtype.base_dtype == base_type:
-        return x
-      return gen_math_ops.cast(x, base_type, name=name)
+      if x.dtype.base_dtype != base_type:
+        x = gen_math_ops.cast(x, base_type, name=name)
+    if x.dtype.is_complex and base_type.is_floating:
+      logging.warn("Casting complex to real discards imaginary part.")
+    return x
 
 
 @tf_export("saturate_cast")
diff --git a/tensorflow/python/ops/sets_impl.py b/tensorflow/python/ops/sets_impl.py
index b0eecd8a1e..21e08d03d2 100644
--- a/tensorflow/python/ops/sets_impl.py
+++ b/tensorflow/python/ops/sets_impl.py
@@ -247,7 +247,7 @@ def set_difference(a, b, aminusb=True, validate_indices=True):
     #
     # collections.OrderedDict([
     #     ((0, 0, 0), 2),
-    #     ((0, 0, 1), 3),
+    #     ((0, 1, 0), 3),
     # ])
   ```
 
diff --git a/tensorflow/tools/test/upload_test_benchmarks.py b/tensorflow/tools/test/upload_test_benchmarks.py
index c030575109..9c45359ee1 100644
--- a/tensorflow/tools/test/upload_test_benchmarks.py
+++ b/tensorflow/tools/test/upload_test_benchmarks.py
@@ -89,6 +89,7 @@ import shutil
 
 from six import text_type
 from google.cloud import datastore
+from six import text_type
 
 
 def is_real_file(dirpath, fname):
diff --git a/third_party/sycl/sycl/BUILD.tpl b/third_party/sycl/sycl/BUILD.tpl
index 21b1a2bbf7..b7e9aa8edb 100755
--- a/third_party/sycl/sycl/BUILD.tpl
+++ b/third_party/sycl/sycl/BUILD.tpl
@@ -21,7 +21,7 @@ config_setting(
     name = "using_sycl_trisycl",
     define_values = {
         "using_sycl": "true",
-        "using_trisycl": "false",
+        "using_trisycl": "true",
     },
 )
 
-- 
GitLab


From 80383763ed973c40cf06d035cc17afe2189ebd0b Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Thu, 22 Mar 2018 18:59:48 -0700
Subject: [PATCH 401/960] Refactor keras.Sequential and enable building
 Sequential models without specifying an input shape in advance (the model
 gets built on fit/etc, like subclassed models).

PiperOrigin-RevId: 190161771
---
 .../keras/_impl/keras/engine/network.py       |  21 +-
 .../keras/_impl/keras/engine/saving_test.py   |   2 +-
 .../keras/_impl/keras/engine/sequential.py    | 946 +++---------------
 .../_impl/keras/engine/sequential_test.py     | 140 +--
 .../keras/_impl/keras/engine/training.py      |  22 +-
 .../python/keras/_impl/keras/estimator.py     |   5 -
 tensorflow/python/layers/base.py              |  14 -
 .../golden/tensorflow.keras.-sequential.pbtxt |  24 +-
 .../tensorflow.keras.models.-sequential.pbtxt |  24 +-
 9 files changed, 235 insertions(+), 963 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/engine/network.py b/tensorflow/python/keras/_impl/keras/engine/network.py
index bf82390438..ea4be0d293 100644
--- a/tensorflow/python/keras/_impl/keras/engine/network.py
+++ b/tensorflow/python/keras/_impl/keras/engine/network.py
@@ -92,7 +92,9 @@ class Network(base_layer.Layer):
     self._expects_training_arg = False
 
     self.supports_masking = False
-    self.optimizer = None
+    if not hasattr(self, 'optimizer'):
+      # Don't reset optimizer if already set.
+      self.optimizer = None
 
     # Private attributes to implement compatibility with Layer.
     self._updates = []  # Used in symbolic mode only.
@@ -188,17 +190,6 @@ class Network(base_layer.Layer):
     self.built = True
     self._is_graph_network = True
 
-    # # List of initial layers (1 to 1 mapping with self.inputs,
-    # # hence the same layer might appear twice)
-    # self._input_layers = []
-    # self._input_layers_node_indices = []
-    # self._input_layers_tensor_indices = []
-    # # list of layers (1 to 1 mapping with self.inputs,
-    # # hence the same layer might appear twice)
-    # self._output_layers = []
-    # self._output_layers_node_indices = []
-    # self._output_layers_tensor_indices = []
-
     self._input_layers = []
     self._output_layers = []
     self._input_coordinates = []
@@ -1223,9 +1214,6 @@ class Network(base_layer.Layer):
     Returns:
         A JSON string.
     """
-    if not self._is_graph_network:
-      raise NotImplementedError
-
     def get_json_type(obj):
       # If obj is any numpy type
       if type(obj).__module__ == np.__name__:
@@ -1260,9 +1248,6 @@ class Network(base_layer.Layer):
     Raises:
         ImportError: if yaml module is not found.
     """
-    if not self._is_graph_network:
-      raise NotImplementedError
-
     if yaml is None:
       raise ImportError('Requires yaml module installed.')
     return yaml.dump(self._updated_config(), **kwargs)
diff --git a/tensorflow/python/keras/_impl/keras/engine/saving_test.py b/tensorflow/python/keras/_impl/keras/engine/saving_test.py
index 4a18cc2e11..dde0901204 100644
--- a/tensorflow/python/keras/_impl/keras/engine/saving_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/saving_test.py
@@ -340,7 +340,7 @@ class TestWholeModelSaving(test.TestCase):
       model.add(keras.layers.Dense(2, input_shape=(3,)))
       model.add(keras.layers.Dense(3))
       model.compile(loss='mse', optimizer='sgd', metrics=['acc'])
-      model.model._make_train_function()
+      model._make_train_function()
 
       fd, fname = tempfile.mkstemp('.h5')
       keras.models.save_model(model, fname)
diff --git a/tensorflow/python/keras/_impl/keras/engine/sequential.py b/tensorflow/python/keras/_impl/keras/engine/sequential.py
index db5e7754bc..66cef1f5b9 100644
--- a/tensorflow/python/keras/_impl/keras/engine/sequential.py
+++ b/tensorflow/python/keras/_impl/keras/engine/sequential.py
@@ -20,26 +20,17 @@ from __future__ import division
 from __future__ import print_function
 
 import copy
-import os
 
-from tensorflow.python.framework import ops
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras import layers as layer_module
 from tensorflow.python.keras._impl.keras.engine import base_layer
 from tensorflow.python.keras._impl.keras.engine import network
-from tensorflow.python.keras._impl.keras.engine import saving
 from tensorflow.python.keras._impl.keras.engine.input_layer import Input
 from tensorflow.python.keras._impl.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras._impl.keras.engine.training import Model
-from tensorflow.python.keras._impl.keras.utils.io_utils import ask_to_proceed_with_overwrite
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
-try:
-  import h5py  # pylint: disable=g-import-not-at-top
-except ImportError:
-  h5py = None
-
 
 @tf_export('keras.models.Sequential', 'keras.Sequential')
 class Sequential(Model):
@@ -48,77 +39,75 @@ class Sequential(Model):
   Arguments:
       layers: list of layers to add to the model.
 
-  # Note
-      The first layer passed to a Sequential model
-      should have a defined input shape. What that
-      means is that it should have received an `input_shape`
-      or `batch_input_shape` argument,
-      or for some type of layers (recurrent, Dense...)
-      an `input_dim` argument.
-
   Example:
 
-      ```python
-          model = Sequential()
-          # first layer must have a defined input shape
-          model.add(Dense(32, input_dim=500))
-          # afterwards, Keras does automatic shape inference
-          model.add(Dense(32))
-
-          # also possible (equivalent to the above):
-          model = Sequential()
-          model.add(Dense(32, input_shape=(500,)))
-          model.add(Dense(32))
-
-          # also possible (equivalent to the above):
-          model = Sequential()
-          # here the batch dimension is None,
-          # which means any batch size will be accepted by the model.
-          model.add(Dense(32, batch_input_shape=(None, 500)))
-          model.add(Dense(32))
-      ```
+  ```python
+  # Optionally, the first layer can receive an `input_shape` argument:
+  model = Sequential()
+  model.add(Dense(32, input_shape=(500,)))
+  # Afterwards, we do automatic shape inference:
+  model.add(Dense(32))
+
+  # This is identical to the following:
+  model = Sequential()
+  model.add(Dense(32, input_dim=500))
+
+  # And to the following:
+  model = Sequential()
+  model.add(Dense(32, batch_input_shape=(None, 500)))
+
+  # Note that you can also omit the `input_shape` argument:
+  # In that case the model gets built the first time you call `fit` (or other
+  # training and evaluation methods).
+  model = Sequential()
+  model.add(Dense(32))
+  model.add(Dense(32))
+  model.compile(optimizer=optimizer, loss=loss)
+  # This builds the model for the first time:
+  model.fit(x, y, batch_size=32, epochs=10)
+
+  # Note that when using this delayed-build pattern (no input shape specified),
+  # the model doesn't have any weights until the first call
+  # to a training/evaluation method (since it isn't yet built):
+  model = Sequential()
+  model.add(Dense(32))
+  model.add(Dense(32))
+  model.weights  # returns []
+
+  # Whereas if you specify the input shape, the model gets built continuously
+  # as you are adding layers:
+  model = Sequential()
+  model.add(Dense(32, input_shape=(500,)))
+  model.add(Dense(32))
+  model.weights  # returns list of length 4
+
+  When using the delayed-build pattern (no input shape specified), you can
+  choose to manually build your model by calling `build(batch_input_shape)`:
+  model = Sequential()
+  model.add(Dense(32))
+  model.add(Dense(32))
+  model.build((None, 500))
+  model.weights  # returns list of length 4
+  ```
   """
 
   def __init__(self, layers=None, name=None):
-    self._is_graph_network = True
-    self._is_compiled = False
-    self._layers = []  # Stack of layers.
-    self.model = None  # Internal Model instance.
-    self.inputs = []  # List of input tensors
-    self.outputs = []  # List of length 1: the output tensor (unique).
-    self._trainable = True
-    self._initial_weights = None
-    self._input_layers = []
-
-    # Model attributes.
-    self._inbound_nodes = []
-    self._outbound_nodes = []
-    self.built = False
-
-    # Set model name.
-    if not name:
-      prefix = 'sequential_'
-      name = prefix + str(K.get_uid(prefix))
-    self._name = name
-
-    # Used by Layer base class.
-    self._dtype = None
-    self._activity_regularizer = None
-
-    # The following properties are not actually used by Keras;
-    # they exist for compatibility with TF's variable scoping mechanism.
-    self._updates = []
-    self._losses = []
-    self._scope = None
-    self._reuse = None
-    self._base_name = name
-    self._graph = ops.get_default_graph()
+    super(Sequential, self).__init__(name=name)
 
     # Add to the model any layers passed to the constructor.
     if layers:
       for layer in layers:
         self.add(layer)
 
+  @property
+  def layers(self):
+    # Historically, `sequential.layers` only returns layers that were added
+    # via `add`, and omits the auto-generated `InputLayer` that comes at the
+    # bottom of the stack.
+    if self._layers and isinstance(self._layers[0], InputLayer):
+      return self._layers[1:]
+    return self._layers
+
   def add(self, layer):
     """Adds a layer instance on top of the layer stack.
 
@@ -137,11 +126,14 @@ class Sequential(Model):
       raise TypeError('The added layer must be '
                       'an instance of class Layer. '
                       'Found: ' + str(layer))
-    if not self.outputs:
+    self.built = False
+    if not self._layers:
+      set_inputs = False
       # First layer in model: check that it is an input layer.
       if not isinstance(layer, InputLayer):
-        # Create an input layer.
-        # First, we need to infer its expected input shape and dtype.
+        # Create an input tensor and call `layer` on the input tensor.
+        # First, we need to infer the expected input shape and dtype.
+        first_layer = layer
         if isinstance(layer, (Model, Sequential)):
           # We were passed a model as first layer.
           # This requires a specific way to figure out the
@@ -156,42 +148,39 @@ class Sequential(Model):
             first_layer = first_layer.layers[0]
           batch_shape = first_layer._batch_input_shape
           dtype = first_layer.dtype
+
+        if hasattr(first_layer, '_batch_input_shape'):
+          batch_shape = first_layer._batch_input_shape
+          dtype = first_layer.dtype
+          # Instantiate the input layer.
+          x = Input(
+              batch_shape=batch_shape,
+              dtype=dtype,
+              name=layer.name + '_input')
+          # This will build the current layer
+          # and create the node connecting the current layer
+          # to the input layer we just created.
+          layer(x)
+          set_inputs = True
         else:
-          # We were passed a regular layer, and it should
-          # know about its input shape. Otherwise, that's an error.
-          if not hasattr(layer, '_batch_input_shape'):
-            raise ValueError('The first layer in a '
-                             'Sequential model must '
-                             'get an `input_shape` argument.')
-          batch_shape = layer._batch_input_shape
-          dtype = layer.dtype
-        # Instantiate the input layer.
-        x = Input(
-            batch_shape=batch_shape, dtype=dtype, name=layer.name + '_input')
-        # This will build the current layer
-        # and create the node connecting the current layer
-        # to the input layer we just created.
-        layer(x)
-
-      if len(layer._inbound_nodes[-1].output_tensors) != 1:
-        raise ValueError('All layers in a Sequential model '
-                         'should have a single output tensor. '
-                         'For multi-output layers, '
-                         'use the functional API.')
-
-      self.outputs = [layer._inbound_nodes[-1].output_tensors[0]]
-      self.inputs = network.get_source_inputs(self.outputs[0])
-
-      # We create an input node, which we will keep updated
-      # as we add more layers
-      base_layer.Node(
-          outbound_layer=self,
-          inbound_layers=[],
-          node_indices=[],
-          tensor_indices=[],
-          input_tensors=self.inputs,
-          output_tensors=self.outputs)
-    else:
+          # The layer doesn't know about its expected shape. We will have to
+          # build the model lazily on `fit`/etc.
+          batch_shape = None
+      else:
+        # Corner case where the user passes an InputLayer layer via `add`.
+        assert len(layer._inbound_nodes[-1].output_tensors) == 1
+        set_inputs = True
+
+      if set_inputs:
+        if len(layer._inbound_nodes[-1].output_tensors) != 1:
+          raise ValueError('All layers in a Sequential model '
+                           'should have a single output tensor. '
+                           'For multi-output layers, '
+                           'use the functional API.')
+
+        self.outputs = [layer._inbound_nodes[-1].output_tensors[0]]
+        self.inputs = network.get_source_inputs(self.outputs[0])
+    elif self.outputs:
       output_tensor = layer(self.outputs[0])
       if isinstance(output_tensor, list):
         raise TypeError('All layers in a Sequential model '
@@ -199,12 +188,10 @@ class Sequential(Model):
                         'For multi-output layers, '
                         'use the functional API.')
       self.outputs = [output_tensor]
-      # update self._inbound_nodes
-      self._inbound_nodes[0].output_tensors = self.outputs
-      self._inbound_nodes[0].output_shapes = [K.int_shape(self.outputs[0])]
-
-    self._layers.append(layer)
-    self.built = False
+    if self.inputs:
+      self.build()
+    else:
+      self._layers.append(layer)
 
   def pop(self):
     """Removes the last layer in the model.
@@ -215,478 +202,30 @@ class Sequential(Model):
     if not self.layers:
       raise TypeError('There are no layers in the model.')
 
-    self.layers.pop()
+    self._layers.pop()
+    self.built = False
     if not self.layers:
-      self.outputs = []
-      self._inbound_nodes = []
-      self._outbound_nodes = []
-    else:
+      self.outputs = None
+      self.inputs = None
+    elif self.outputs:
       self.layers[-1]._outbound_nodes = []
       self.outputs = [self.layers[-1].output]
-      # update self._inbound_nodes
-      self._inbound_nodes[0].output_tensors = self.outputs
-      self._inbound_nodes[0].output_shapes = [K.int_shape(self.outputs[0])]
-    self.built = False
-
-  def get_layer(self, name=None, index=None):
-    """Retrieve a layer that is part of the model.
-
-    Returns a layer based on either its name (unique)
-    or its index in the graph. Indices are based on
-    order of horizontal graph traversal (bottom-up).
-
-    Arguments:
-        name: string, name of layer.
-        index: integer, index of layer.
-
-    Returns:
-        A layer instance.
-    """
-    if not self.built:
-      self.build()
-    return self.model.get_layer(name, index)
-
-  def call(self, inputs, **kwargs):
-    if not self.built:
       self.build()
-    return self.model.call(inputs, **kwargs)
 
   def build(self, input_shape=None):
-    if not self.inputs or not self.outputs:
-      raise TypeError('Sequential model cannot be built: model is empty.'
-                      ' Add some layers first.')
-    # actually create the model
-    self.model = Model(self.inputs, self.outputs[0], name=self.name + '_model')
-    self.model.trainable = self.trainable
-
-    # mirror model attributes
-    self.supports_masking = self.model.supports_masking
-    self._output_mask_cache = self.model._output_mask_cache
-    self._output_tensor_cache = self.model._output_tensor_cache
-    self._output_shape_cache = self.model._output_shape_cache
-    self._input_layers = self.model._input_layers
-    self._output_layers = self.model._output_layers
-    self._input_coordinates = self.model._input_coordinates
-    self._output_coordinates = self.model._output_coordinates
-    self._nodes_by_depth = self.model._nodes_by_depth
-    self._network_nodes = self.model._network_nodes
-    self.output_names = self.model.output_names
-    self.input_names = self.model.input_names
-    self._feed_input_names = self.model._feed_input_names
-    self._feed_inputs = self.model._feed_inputs
-
-    # Make sure child model callbacks
-    # will call the parent Sequential model.
-    self.model.callback_model = self
-
-    self.built = True
-
-  @property
-  def uses_learning_phase(self):
-    if not self.built:
-      self.build()
-    return self.model.uses_learning_phase
-
-  def _gather_list_attr(self, attr):
-    all_attrs = []
-    for layer in self.layers:
-      all_attrs += getattr(layer, attr, [])
-    return all_attrs
-
-  def _make_train_function(self):
-    self.model._make_train_function()
-
-  def _make_test_function(self):
-    self.model._make_test_function()
-
-  def _make_predict_function(self):
-    self.model._make_predict_function()
-
-  @property
-  def trainable(self):
-    return self._trainable
-
-  @trainable.setter
-  def trainable(self, value):
-    if self.model:
-      self.model.trainable = value
-    self._trainable = value
-
-  @property
-  def trainable_weights(self):
-    if not self.trainable:
-      return []
-    return self._gather_list_attr('trainable_weights')
-
-  @property
-  def non_trainable_weights(self):
-    weights = self._gather_list_attr('non_trainable_weights')
-    if not self.trainable:
-      trainable_weights = self._gather_list_attr('trainable_weights')
-      return trainable_weights + weights
-    return weights
-
-  @property
-  def regularizers(self):
-    if not self.built:
-      self.build()
-    return self.model.regularizers
-
-  def get_weights(self):
-    """Retrieves the weights of the model.
-
-    Returns:
-        A flat list of Numpy arrays
-        (one array per model weight).
-    """
-    if not self.built:
-      self.build()
-    return self.model.get_weights()
-
-  def set_weights(self, weights):
-    """Sets the weights of the model.
-
-    Arguments:
-        weights: Should be a list
-            of Numpy arrays with shapes and types matching
-            the output of `model.get_weights()`.
-    """
-    if not self.built:
-      self.build()
-    self.model.set_weights(weights)
-
-  def load_weights(self, filepath, by_name=False):
-    if h5py is None:
-      raise ImportError('`load_weights` requires h5py.')
-    f = h5py.File(filepath, mode='r')
-    if 'layer_names' not in f.attrs and 'model_weights' in f:
-      f = f['model_weights']
-    layers = self.layers
-    if by_name:
-      saving.load_weights_from_hdf5_group_by_name(f, layers)
-    else:
-      saving.load_weights_from_hdf5_group(f, layers)
-    if hasattr(f, 'close'):
-      f.close()
-
-  def save_weights(self, filepath, overwrite=True):
-    if h5py is None:
-      raise ImportError('`save_weights` requires h5py.')
-    # If file exists and should not be overwritten:
-    if not overwrite and os.path.isfile(filepath):
-      proceed = ask_to_proceed_with_overwrite(filepath)
-      if not proceed:
-        return
-    layers = self.layers
-    f = h5py.File(filepath, 'w')
-    saving.save_weights_to_hdf5_group(f, layers)
-    f.flush()
-    f.close()
-
-  def compile(self,
-              optimizer,
-              loss,
-              metrics=None,
-              sample_weight_mode=None,
-              weighted_metrics=None,
-              target_tensors=None,
-              **kwargs):
-    """Configures the model for training.
-
-    Arguments:
-        optimizer: String (name of optimizer) or optimizer object.
-            See [optimizers](/optimizers).
-        loss: String (name of objective function) or objective function.
-            See [losses](/losses).
-            If the model has multiple outputs, you can use a different loss
-            on each output by passing a dictionary or a list of losses.
-            The loss value that will be minimized by the model
-            will then be the sum of all individual losses.
-        metrics: List of metrics to be evaluated by the model
-            during training and testing.
-            Typically you will use `metrics=['accuracy']`.
-            To specify different metrics for different outputs of a
-            multi-output model, you could also pass a dictionary,
-            such as `metrics={'output_a': 'accuracy'}`.
-        sample_weight_mode: If you need to do timestep-wise
-            sample weighting (2D weights), set this to `"temporal"`.
-            `None` defaults to sample-wise weights (1D).
-            If the model has multiple outputs, you can use a different
-            `sample_weight_mode` on each output by passing a
-            dictionary or a list of modes.
-        weighted_metrics: list of metrics to be evaluated and weighted
-             by `sample_weight` or `class_weight` during training and testing.
-        target_tensors: By default, Keras will create a placeholder for the
-            model's target, which will be fed with the target data during
-            training. If instead you would like to use your own
-            target tensor (in turn, Keras will not expect external
-            Numpy data for these targets at training time), you
-            can specify them via the `target_tensors` argument.
-            It should be a single tensor
-            (for a single-output `Sequential` model).
-        **kwargs: These arguments are passed into `tf.Session.run`.
-
-    Example:
-        ```python
-            model = Sequential()
-            model.add(Dense(32, input_shape=(500,)))
-            model.add(Dense(10, activation='softmax'))
-            model.compile(optimizer='rmsprop',
-                          loss='categorical_crossentropy',
-                          metrics=['accuracy'])
-        ```
-    """
-    # create the underlying model
-    self.build()
-    # call compile method of Model class
-    self.model.compile(
-        optimizer,
-        loss,
-        metrics=metrics,
-        sample_weight_mode=sample_weight_mode,
-        weighted_metrics=weighted_metrics,
-        target_tensors=target_tensors,
-        **kwargs)
-    self.optimizer = self.model.optimizer
-    self.loss = self.model.loss
-    self.metrics = self.model.metrics
-    self.loss_weights = self.model.loss_weights
-    self.sample_weight_mode = self.model.sample_weight_mode
-    self.weighted_metrics = self.model.weighted_metrics
-    self.targets = self.model.targets
-    self.metrics_tensors = self.model.metrics_tensors
-    self.metrics_names = self.model.metrics_names
-    self.sample_weights = self.model.sample_weights
-    self.total_loss = self.model.total_loss
-
-  def fit(self,
-          x=None,
-          y=None,
-          batch_size=None,
-          epochs=1,
-          verbose=1,
-          callbacks=None,
-          validation_split=0.,
-          validation_data=None,
-          shuffle=True,
-          class_weight=None,
-          sample_weight=None,
-          initial_epoch=0,
-          steps_per_epoch=None,
-          validation_steps=None,
-          **kwargs):
-    """Trains the model for a fixed number of epochs.
-
-    Arguments:
-        x: Numpy array of training data.
-            If the input layer in the model is named, you can also pass a
-            dictionary mapping the input name to a Numpy array.
-            `x` can be `None` (default) if feeding from
-            TensorFlow data tensors.
-        y: Numpy array of target (label) data.
-            If the output layer in the model is named, you can also pass a
-            dictionary mapping the output name to a Numpy array.
-            `y` can be `None` (default) if feeding from
-            TensorFlow data tensors.
-        batch_size: Integer or `None`.
-            Number of samples per gradient update.
-            If unspecified, it will default to 32.
-        epochs: Integer. Number of epochs to train the model.
-            An epoch is an iteration over the entire `x` and `y`
-            data provided.
-            Note that in conjunction with `initial_epoch`,
-            `epochs` is to be understood as "final epoch".
-            The model is not trained for a number of iterations
-            given by `epochs`, but merely until the epoch
-            of index `epochs` is reached.
-        verbose: 0, 1, or 2. Verbosity mode.
-            0 = silent, 1 = progress bar, 2 = one line per epoch.
-        callbacks: List of `keras.callbacks.Callback` instances.
-            List of callbacks to apply during training.
-            See [callbacks](/callbacks).
-        validation_split: Float between 0 and 1:
-            Fraction of the training data to be used as validation data.
-            The model will set apart this fraction of the training data,
-            will not train on it, and will evaluate
-            the loss and any model metrics
-            on this data at the end of each epoch.
-            The validation data is selected from the last samples
-            in the `x` and `y` data provided, before shuffling.
-        validation_data: tuple `(x_val, y_val)` or tuple
-            `(x_val, y_val, val_sample_weights)` on which to evaluate
-            the loss and any model metrics at the end of each epoch.
-            The model will not be trained on this data.
-            This will override `validation_split`.
-        shuffle: Boolean (whether to shuffle the training data
-            before each epoch) or str (for 'batch').
-            'batch' is a special option for dealing with the
-            limitations of HDF5 data; it shuffles in batch-sized chunks.
-            Has no effect when `steps_per_epoch` is not `None`.
-        class_weight: Optional dictionary mapping class indices (integers)
-            to a weight (float) value, used for weighting the loss function
-            (during training only).
-            This can be useful to tell the model to
-            "pay more attention" to samples from
-            an under-represented class.
-        sample_weight: Optional Numpy array of weights for
-            the training samples, used for weighting the loss function
-            (during training only). You can either pass a flat (1D)
-            Numpy array with the same length as the input samples
-            (1:1 mapping between weights and samples),
-            or in the case of temporal data,
-            you can pass a 2D array with shape
-            `(samples, sequence_length)`,
-            to apply a different weight to every timestep of every sample.
-            In this case you should make sure to specify
-            `sample_weight_mode="temporal"` in `compile()`.
-        initial_epoch: Epoch at which to start training
-            (useful for resuming a previous training run).
-        steps_per_epoch: Total number of steps (batches of samples)
-            before declaring one epoch finished and starting the
-            next epoch. When training with input tensors such as
-            TensorFlow data tensors, the default `None` is equal to
-            the number of unique samples in your dataset divided by
-            the batch size, or 1 if that cannot be determined.
-        validation_steps: Only relevant if `steps_per_epoch`
-            is specified. Total number of steps (batches of samples)
-            to validate before stopping.
-        **kwargs: Used for backwards compatibility support.
-
-    Returns:
-        A `History` object. Its `History.history` attribute is
-        a record of training loss values and metrics values
-        at successive epochs, as well as validation loss values
-        and validation metrics values (if applicable).
-
-    Raises:
-        RuntimeError: If the model was never compiled.
-        ValueError: In case of mismatch between the provided input data
-            and what the model expects.
-    """
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.fit(
-        x,
-        y,
-        batch_size=batch_size,
-        epochs=epochs,
-        verbose=verbose,
-        callbacks=callbacks,
-        validation_split=validation_split,
-        validation_data=validation_data,
-        shuffle=shuffle,
-        class_weight=class_weight,
-        sample_weight=sample_weight,
-        initial_epoch=initial_epoch,
-        steps_per_epoch=steps_per_epoch,
-        validation_steps=validation_steps)
-
-  def evaluate(self, x, y, batch_size=32, verbose=1, sample_weight=None):
-    """Computes the loss on some input data, batch by batch.
-
-    Arguments:
-        x: input data, as a Numpy array or list of Numpy arrays
-            (if the model has multiple inputs).
-        y: labels, as a Numpy array.
-        batch_size: integer. Number of samples per gradient update.
-        verbose: verbosity mode, 0 or 1.
-        sample_weight: sample weights, as a Numpy array.
-
-    Returns:
-        Scalar test loss (if the model has no metrics)
-        or list of scalars (if the model computes other metrics).
-        The attribute `model.metrics_names` will give you
-        the display labels for the scalar outputs.
-
-    Raises:
-        RuntimeError: if the model was never compiled.
-    """
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.evaluate(
-        x,
-        y,
-        batch_size=batch_size,
-        verbose=verbose,
-        sample_weight=sample_weight)
-
-  def predict(self, x, batch_size=32, verbose=0):
-    """Generates output predictions for the input samples.
-
-    The input samples are processed batch by batch.
-
-    Arguments:
-        x: the input data, as a Numpy array.
-        batch_size: integer.
-        verbose: verbosity mode, 0 or 1.
-
-    Returns:
-        A Numpy array of predictions.
-    """
-    if not self.built:
-      self.build()
-    return self.model.predict(x, batch_size=batch_size, verbose=verbose)
-
-  def predict_on_batch(self, x):
-    """Returns predictions for a single batch of samples.
-
-    Arguments:
-        x: input data, as a Numpy array or list of Numpy arrays
-            (if the model has multiple inputs).
-
-    Returns:
-        A Numpy array of predictions.
-    """
-    if not self.built:
-      self.build()
-    return self.model.predict_on_batch(x)
-
-  def train_on_batch(self, x, y, class_weight=None, sample_weight=None):
-    """Single gradient update over one batch of samples.
-
-    Arguments:
-        x: input data, as a Numpy array or list of Numpy arrays
-            (if the model has multiple inputs).
-        y: labels, as a Numpy array.
-        class_weight: dictionary mapping classes to a weight value,
-            used for scaling the loss function (during training only).
-        sample_weight: sample weights, as a Numpy array.
-
-    Returns:
-        Scalar training loss (if the model has no metrics)
-        or list of scalars (if the model computes other metrics).
-        The attribute `model.metrics_names` will give you
-        the display labels for the scalar outputs.
-
-    Raises:
-        RuntimeError: if the model was never compiled.
-    """
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.train_on_batch(
-        x, y, sample_weight=sample_weight, class_weight=class_weight)
-
-  def test_on_batch(self, x, y, sample_weight=None):
-    """Evaluates the model over a single batch of samples.
-
-    Arguments:
-        x: input data, as a Numpy array or list of Numpy arrays
-            (if the model has multiple inputs).
-        y: labels, as a Numpy array.
-        sample_weight: sample weights, as a Numpy array.
-
-    Returns:
-        Scalar test loss (if the model has no metrics)
-        or list of scalars (if the model computes other metrics).
-        The attribute `model.metrics_names` will give you
-        the display labels for the scalar outputs.
-
-    Raises:
-        RuntimeError: if the model was never compiled.
-    """
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.test_on_batch(x, y, sample_weight=sample_weight)
+    if input_shape and not self.inputs:
+      batch_shape = tuple(input_shape)
+      dtype = K.floatx()
+      x = Input(
+          batch_shape=batch_shape, dtype=dtype, name=self.name + '_input')
+      self.inputs = [x]
+      for layer in self._layers:
+        x = layer(x)
+      self.outputs = [x]
+
+    if self.inputs:
+      self._init_graph_network(self.inputs, self.outputs, name=self.name)
+      self.built = True
 
   def predict_proba(self, x, batch_size=32, verbose=0):
     """Generates class probability predictions for the input samples.
@@ -730,255 +269,6 @@ class Sequential(Model):
     else:
       return (proba > 0.5).astype('int32')
 
-  def fit_generator(self,
-                    generator,
-                    steps_per_epoch=None,
-                    epochs=1,
-                    verbose=1,
-                    callbacks=None,
-                    validation_data=None,
-                    validation_steps=None,
-                    class_weight=None,
-                    max_queue_size=10,
-                    workers=1,
-                    use_multiprocessing=False,
-                    shuffle=True,
-                    initial_epoch=0,
-                    **kwargs):
-    """Fits the model on data generated batch-by-batch by a Python generator.
-
-    The generator is run in parallel to the model, for efficiency.
-    For instance, this allows you to do real-time data augmentation
-    on images on CPU in parallel to training your model on GPU.
-
-    Arguments:
-        generator: A generator.
-            The output of the generator must be either
-            - a tuple (inputs, targets)
-            - a tuple (inputs, targets, sample_weights).
-            All arrays should contain the same number of samples.
-            The generator is expected to loop over its data
-            indefinitely. An epoch finishes when `steps_per_epoch`
-            batches have been seen by the model.
-        steps_per_epoch: Total number of steps (batches of samples)
-            to yield from `generator` before declaring one epoch
-            finished and starting the next epoch. It should typically
-            be equal to the number of samples of your dataset
-            divided by the batch size.
-            Optional for `Sequence`: if unspecified, will use
-            the `len(generator)` as a number of steps.
-        epochs: Integer, total number of iterations on the data.
-            Note that in conjunction with initial_epoch, the parameter
-            epochs is to be understood as "final epoch". The model is
-            not trained for n steps given by epochs, but until the
-            epoch epochs is reached.
-        verbose: Verbosity mode, 0, 1, or 2.
-        callbacks: List of callbacks to be called during training.
-        validation_data: This can be either
-            - A generator for the validation data
-            - A tuple (inputs, targets)
-            - A tuple (inputs, targets, sample_weights).
-        validation_steps: Only relevant if `validation_data`
-            is a generator.
-            Number of steps to yield from validation generator
-            at the end of every epoch. It should typically
-            be equal to the number of samples of your
-            validation dataset divided by the batch size.
-            Optional for `Sequence`: if unspecified, will use
-            the `len(validation_data)` as a number of steps.
-        class_weight: Dictionary mapping class indices to a weight
-            for the class.
-        max_queue_size: Maximum size for the generator queue
-        workers: Maximum number of processes to spin up
-        use_multiprocessing: If True, use process based threading.
-            Note that because
-            this implementation relies on multiprocessing,
-            you should not pass
-            non picklable arguments to the generator
-            as they can't be passed
-            easily to children processes.
-       shuffle: Whether to shuffle the order of the batches at
-              the beginning of each epoch. Only used with instances
-              of `Sequence` (keras.utils.Sequence).
-        initial_epoch: Epoch at which to start training
-            (useful for resuming a previous training run)
-        **kwargs: support for legacy arguments.
-
-    Returns:
-        A `History` object.
-
-    Raises:
-        RuntimeError: if the model was never compiled.
-        ValueError: In case the generator yields
-            data in an invalid format.
-
-    Example:
-
-    ```python
-        def generate_arrays_from_file(path):
-            while 1:
-                f = open(path)
-                for line in f:
-                    # create Numpy arrays of input data
-                    # and labels, from each line in the file
-                    x, y = process_line(line)
-                    yield (x, y)
-                    f.close()
-
-        model.fit_generator(generate_arrays_from_file('/my_file.txt'),
-                            steps_per_epoch=1000, epochs=10)
-    ```
-    """
-    # Legacy support
-    if 'max_q_size' in kwargs:
-      max_queue_size = kwargs.pop('max_q_size')
-      logging.warning('The argument `max_q_size` has been renamed '
-                      '`max_queue_size`. Update your method calls accordingly.')
-    if 'pickle_safe' in kwargs:
-      use_multiprocessing = kwargs.pop('pickle_safe')
-      logging.warning('The argument `pickle_safe` has been renamed '
-                      '`use_multiprocessing`. '
-                      'Update your method calls accordingly.')
-    if kwargs:
-      raise ValueError('Unrecognized keyword arguments: ' + str(kwargs))
-
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.fit_generator(
-        generator,
-        steps_per_epoch,
-        epochs,
-        verbose=verbose,
-        callbacks=callbacks,
-        validation_data=validation_data,
-        validation_steps=validation_steps,
-        class_weight=class_weight,
-        max_queue_size=max_queue_size,
-        workers=workers,
-        use_multiprocessing=use_multiprocessing,
-        shuffle=shuffle,
-        initial_epoch=initial_epoch)
-
-  def evaluate_generator(self,
-                         generator,
-                         steps=None,
-                         max_queue_size=10,
-                         workers=1,
-                         use_multiprocessing=False,
-                         **kwargs):
-    """Evaluates the model on a data generator.
-
-    The generator should return the same kind of data
-    as accepted by `test_on_batch`.
-
-    Arguments:
-        generator: Generator yielding tuples (inputs, targets)
-            or (inputs, targets, sample_weights)
-        steps: Total number of steps (batches of samples)
-            to yield from `generator` before stopping.
-            Optional for `Sequence`: if unspecified, will use
-            the `len(generator)` as a number of steps.
-        max_queue_size: maximum size for the generator queue
-        workers: maximum number of processes to spin up
-        use_multiprocessing: if True, use process based threading.
-            Note that because this implementation
-            relies on multiprocessing, you should not pass
-            non picklable arguments to the generator
-            as they can't be passed easily to children processes.
-        **kwargs: support for legacy arguments.
-
-    Returns:
-        Scalar test loss (if the model has no metrics)
-        or list of scalars (if the model computes other metrics).
-        The attribute `model.metrics_names` will give you
-        the display labels for the scalar outputs.
-
-    Raises:
-        RuntimeError: if the model was never compiled.
-        ValueError: In case the generator yields
-            data in an invalid format.
-    """
-    # Legacy support
-    if 'max_q_size' in kwargs:
-      max_queue_size = kwargs.pop('max_q_size')
-      logging.warning('The argument `max_q_size` has been renamed '
-                      '`max_queue_size`. Update your method calls accordingly.')
-    if 'pickle_safe' in kwargs:
-      use_multiprocessing = kwargs.pop('pickle_safe')
-      logging.warning('The argument `pickle_safe` has been renamed '
-                      '`use_multiprocessing`. '
-                      'Update your method calls accordingly.')
-    if kwargs:
-      raise ValueError('Unrecognized keyword arguments: ' + str(kwargs))
-
-    if not self.built:
-      raise RuntimeError('The model needs to be compiled before being used.')
-    return self.model.evaluate_generator(
-        generator,
-        steps,
-        max_queue_size=max_queue_size,
-        workers=workers,
-        use_multiprocessing=use_multiprocessing)
-
-  def predict_generator(self,
-                        generator,
-                        steps=None,
-                        max_queue_size=10,
-                        workers=1,
-                        use_multiprocessing=False,
-                        verbose=0,
-                        **kwargs):
-    """Generates predictions for the input samples from a data generator.
-
-    The generator should return the same kind of data as accepted by
-    `predict_on_batch`.
-
-    Arguments:
-        generator: generator yielding batches of input samples.
-        steps: Total number of steps (batches of samples)
-            to yield from `generator` before stopping.
-            Optional for `Sequence`: if unspecified, will use
-            the `len(generator)` as a number of steps.
-        max_queue_size: maximum size for the generator queue
-        workers: maximum number of processes to spin up
-        use_multiprocessing: if True, use process based threading.
-            Note that because this implementation
-            relies on multiprocessing, you should not pass
-            non picklable arguments to the generator
-            as they can't be passed easily to children processes.
-        verbose: verbosity mode, 0 or 1.
-        **kwargs: support for legacy arguments.
-
-    Returns:
-        A Numpy array of predictions.
-
-    Raises:
-        ValueError: In case the generator yields
-            data in an invalid format.
-    """
-    # Legacy support
-    if 'max_q_size' in kwargs:
-      max_queue_size = kwargs.pop('max_q_size')
-      logging.warning('The argument `max_q_size` has been renamed '
-                      '`max_queue_size`. Update your method calls accordingly.')
-    if 'pickle_safe' in kwargs:
-      use_multiprocessing = kwargs.pop('pickle_safe')
-      logging.warning('The argument `pickle_safe` has been renamed '
-                      '`use_multiprocessing`. '
-                      'Update your method calls accordingly.')
-    if kwargs:
-      raise ValueError('Unrecognized keyword arguments: ' + str(kwargs))
-
-    if not self.built:
-      self.build()
-    return self.model.predict_generator(
-        generator,
-        steps,
-        max_queue_size=max_queue_size,
-        workers=workers,
-        use_multiprocessing=use_multiprocessing,
-        verbose=verbose)
-
   def get_config(self):
     config = []
     for layer in self.layers:
diff --git a/tensorflow/python/keras/_impl/keras/engine/sequential_test.py b/tensorflow/python/keras/_impl/keras/engine/sequential_test.py
index 166634bd82..c9a47581df 100644
--- a/tensorflow/python/keras/_impl/keras/engine/sequential_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/sequential_test.py
@@ -20,90 +20,110 @@ from __future__ import print_function
 
 import numpy as np
 
+from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras._impl import keras
 from tensorflow.python.platform import test
+from tensorflow.python.training import rmsprop
 
 
 class TestSequential(test.TestCase):
   """Most Sequential model API tests are covered in `training_test.py`.
   """
 
+  @tf_test_util.run_in_graph_and_eager_modes()
   def test_basic_methods(self):
     model = keras.models.Sequential()
     model.add(keras.layers.Dense(1, input_dim=2))
     model.add(keras.layers.Dropout(0.3, name='dp'))
     model.add(keras.layers.Dense(2, kernel_regularizer='l2',
                                  kernel_constraint='max_norm'))
-    model.build()
-    self.assertEqual(model.state_updates, model.model.state_updates)
+    self.assertEqual(len(model.layers), 3)
+    self.assertEqual(len(model.weights), 2 * 2)
     self.assertEqual(model.get_layer(name='dp').name, 'dp')
 
+  @tf_test_util.run_in_graph_and_eager_modes()
   def test_sequential_pop(self):
     num_hidden = 5
     input_dim = 3
     batch_size = 5
     num_classes = 2
-    with self.test_session():
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(num_hidden, input_dim=input_dim))
-      model.add(keras.layers.Dense(num_classes))
-      model.compile(loss='mse', optimizer='sgd')
-      x = np.random.random((batch_size, input_dim))
-      y = np.random.random((batch_size, num_classes))
-      model.fit(x, y, epochs=1)
-      model.pop()
-      self.assertEqual(len(model.layers), 1)
-      self.assertEqual(model.output_shape, (None, num_hidden))
-      model.compile(loss='mse', optimizer='sgd')
-      y = np.random.random((batch_size, num_hidden))
-      model.fit(x, y, epochs=1)
 
-      # Test popping single-layer model
-      model = keras.models.Sequential()
-      model.add(keras.layers.Dense(num_hidden, input_dim=input_dim))
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(num_hidden, input_dim=input_dim))
+    model.add(keras.layers.Dense(num_classes))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    x = np.random.random((batch_size, input_dim))
+    y = np.random.random((batch_size, num_classes))
+    model.fit(x, y, epochs=1)
+    model.pop()
+    self.assertEqual(len(model.layers), 1)
+    self.assertEqual(model.output_shape, (None, num_hidden))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    y = np.random.random((batch_size, num_hidden))
+    model.fit(x, y, epochs=1)
+
+    # Test popping single-layer model
+    model = keras.models.Sequential()
+    model.add(keras.layers.Dense(num_hidden, input_dim=input_dim))
+    model.pop()
+    self.assertEqual(model.layers, [])
+    self.assertEqual(model.outputs, None)
+
+    # Invalid use case
+    model = keras.models.Sequential()
+    with self.assertRaises(TypeError):
       model.pop()
-      self.assertEqual(len(model.layers), 0)
-      self.assertEqual(len(model.outputs), 0)
 
-      # Invalid use case
-      model = keras.models.Sequential()
-      with self.assertRaises(TypeError):
-        model.pop()
+  @tf_test_util.run_in_graph_and_eager_modes()
+  def test_sequential_deferred_build(self):
+    num_hidden = 5
+    input_dim = 3
+    batch_size = 5
+    num_classes = 2
 
+    model = keras.models.Sequential()
+    # We don't specify the input shape.
+    model.add(keras.layers.Dense(num_hidden))
+    model.add(keras.layers.Dense(num_classes))
+    model.compile(loss='mse', optimizer=rmsprop.RMSPropOptimizer(1e-3))
+    self.assertEqual(len(model.layers), 2)
+    self.assertEqual(len(model.weights), 0)
+    self.assertFalse(model.built)
+
+    x = np.random.random((batch_size, input_dim))
+    y = np.random.random((batch_size, num_classes))
+    model.fit(x, y, epochs=1)
+    self.assertTrue(model.built)
+    self.assertEqual(model.inputs[0].get_shape().as_list(), [None, input_dim])
+    self.assertEqual(model.outputs[0].get_shape().as_list(),
+                     [None, num_classes])
+    self.assertEqual(len(model.weights), 2 * 2)
+
+  @tf_test_util.run_in_graph_and_eager_modes()
   def test_invalid_use_cases(self):
-    with self.test_session():
-      # Added objects must be layer instances
-      with self.assertRaises(TypeError):
-        model = keras.models.Sequential()
-        model.add(None)
-
-      # Added layers must have an inputs shape
-      with self.assertRaises(ValueError):
-        model = keras.models.Sequential()
-        model.add(keras.layers.Dense(1))
-
-      # Added layers cannot have multiple outputs
-      class MyLayer(keras.layers.Layer):
-
-        def call(self, inputs):
-          return [3 * inputs, 2 * inputs]
-
-        def compute_output_shape(self, input_shape):
-          return [input_shape, input_shape]
-
-      with self.assertRaises(ValueError):
-        model = keras.models.Sequential()
-        model.add(MyLayer(input_shape=(3,)))
-      with self.assertRaises(TypeError):
-        model = keras.models.Sequential()
-        model.add(keras.layers.Dense(1, input_dim=1))
-        model.add(MyLayer())
-
-      # Building empty model
+    # Added objects must be layer instances
+    with self.assertRaises(TypeError):
       model = keras.models.Sequential()
-      with self.assertRaises(TypeError):
-        model.build()
+      model.add(None)
+
+    # Added layers cannot have multiple outputs
+    class MyLayer(keras.layers.Layer):
+
+      def call(self, inputs):
+        return [3 * inputs, 2 * inputs]
+
+      def compute_output_shape(self, input_shape):
+        return [input_shape, input_shape]
 
+    with self.assertRaises(ValueError):
+      model = keras.models.Sequential()
+      model.add(MyLayer(input_shape=(3,)))
+    with self.assertRaises(TypeError):
+      model = keras.models.Sequential()
+      model.add(keras.layers.Dense(1, input_dim=1))
+      model.add(MyLayer())
+
+  @tf_test_util.run_in_graph_and_eager_modes()
   def test_nested_sequential_trainability(self):
     input_dim = 20
     num_units = 10
@@ -116,6 +136,8 @@ class TestSequential(test.TestCase):
     model.add(inner_model)
     model.add(keras.layers.Dense(num_classes))
 
+    self.assertEqual(len(model.layers), 2)
+
     self.assertEqual(len(model.trainable_weights), 4)
     inner_model.trainable = False
     self.assertEqual(len(model.trainable_weights), 2)
@@ -135,7 +157,6 @@ class TestSequential(test.TestCase):
 
       model.compile('sgd', 'mse')
       assert not model.updates
-      assert not model.model.updates
 
       x1 = model.predict(val_a)
       model.train_on_batch(val_a, val_out)
@@ -145,8 +166,11 @@ class TestSequential(test.TestCase):
       model.trainable = True
       model.compile('sgd', 'mse')
       assert model.updates
-      assert model.model.updates
 
       model.train_on_batch(val_a, val_out)
       x2 = model.predict(val_a)
       assert np.abs(np.sum(x1 - x2)) > 1e-5
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py
index 4acb41553e..08288d353e 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training.py
@@ -874,7 +874,14 @@ class Model(Network):
         whether to build the model's graph in inference mode (False), training
         mode (True), or using the Keras learning phase (None).
     """
-    if context.executing_eagerly():
+    if self.__class__.__name__ == 'Sequential':
+      # Note: we can't test whether the model is `Sequential` via `isinstance`
+      # since `Sequential` depends on `Model`.
+      if isinstance(inputs, list):
+        assert len(inputs) == 1
+        inputs = inputs[0]
+      self.build(input_shape=(None,) + inputs.shape[1:])
+    elif context.executing_eagerly():
       self._eager_set_inputs(inputs)
     else:
       self._symbolic_set_inputs(inputs, training=training)
@@ -1581,9 +1588,9 @@ class Model(Network):
         ValueError: In case the generator yields
             data in an invalid format.
     """
-    if not self._is_graph_network:
+    if not self.built and not self._is_graph_network:
       raise NotImplementedError(
-          '`fit_generator` is not yet enabled for Model subclasses')
+          '`fit_generator` is not yet enabled for unbuilt Model subclasses')
 
     return training_generator.fit_generator(
         self,
@@ -1647,9 +1654,10 @@ class Model(Network):
         ValueError: In case the generator yields
             data in an invalid format.
     """
-    if not self._is_graph_network:
+    if not self.built and not self._is_graph_network:
       raise NotImplementedError(
-          '`evaluate_generator` is not yet enabled for Model subclasses')
+          '`evaluate_generator` is not yet enabled for '
+          'unbuilt Model subclasses')
 
     return training_generator.evaluate_generator(
         self,
@@ -1700,9 +1708,9 @@ class Model(Network):
         ValueError: In case the generator yields
             data in an invalid format.
     """
-    if not self._is_graph_network:
+    if not self.built and not self._is_graph_network:
       raise NotImplementedError(
-          '`predict_generator` is not yet enabled for Model subclasses')
+          '`predict_generator` is not yet enabled for unbuilt Model subclasses')
 
     return training_generator.predict_generator(
         self,
diff --git a/tensorflow/python/keras/_impl/keras/estimator.py b/tensorflow/python/keras/_impl/keras/estimator.py
index 081f25e914..8426d84df9 100644
--- a/tensorflow/python/keras/_impl/keras/estimator.py
+++ b/tensorflow/python/keras/_impl/keras/estimator.py
@@ -296,9 +296,6 @@ def _clone_and_build_model(mode,
         sample_weight_mode=keras_model.sample_weight_mode,
         weighted_metrics=keras_model.weighted_metrics,
         target_tensors=target_tensors)
-
-  if isinstance(model, models.Sequential):
-    model = model.model
   return model
 
 
@@ -396,8 +393,6 @@ def _save_first_checkpoint(keras_model, estimator, custom_objects,
       training_util.create_global_step()
       model = _clone_and_build_model(model_fn_lib.ModeKeys.TRAIN, keras_model,
                                      custom_objects)
-      if isinstance(model, models.Sequential):
-        model = model.model
       # save to checkpoint
       with session.Session(config=estimator._session_config) as sess:
         model.set_weights(keras_weights)
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index e4395bea92..1e5f26a77f 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -892,7 +892,6 @@ class Layer(checkpointable.CheckpointableBase):
         mode.
         ValueError: If the index provided does not match any node.
     """
-    assert not context.executing_eagerly()
     if not self._inbound_nodes:
       raise RuntimeError('The layer has never been called '
                          'and thus has no defined ' + attr_name + '.')
@@ -922,9 +921,6 @@ class Layer(checkpointable.CheckpointableBase):
     Raises:
       RuntimeError: If called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError(
-          'Layer.get_input_shape_at not supported in Eager mode.')
     return self._get_node_attribute_at_index(node_index, 'input_shapes',
                                              'input shape')
 
@@ -985,8 +981,6 @@ class Layer(checkpointable.CheckpointableBase):
     Raises:
       RuntimeError: If called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.get_output_at not supported in Eager mode.')
     return self._get_node_attribute_at_index(node_index, 'output_tensors',
                                              'output')
 
@@ -1008,8 +1002,6 @@ class Layer(checkpointable.CheckpointableBase):
       RuntimeError: If called in Eager mode.
       AttributeError: If no inbound nodes are found.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.input not supported in Eager mode.')
     if not self._inbound_nodes:
       raise AttributeError('Layer ' + self.name +
                            ' is not connected, no input to return.')
@@ -1030,8 +1022,6 @@ class Layer(checkpointable.CheckpointableBase):
         layers.
       RuntimeError: if called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.output not supported in Eager mode.')
     if not self._inbound_nodes:
       raise AttributeError('Layer ' + self.name + ' has no inbound nodes.')
     return self._get_node_attribute_at_index(0, 'output_tensors', 'output')
@@ -1052,8 +1042,6 @@ class Layer(checkpointable.CheckpointableBase):
         AttributeError: if the layer has no defined input_shape.
         RuntimeError: if called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.input_shape not supported in Eager mode.')
     if not self._inbound_nodes:
       raise AttributeError('The layer has never been called '
                            'and thus has no defined input shape.')
@@ -1113,8 +1101,6 @@ class Layer(checkpointable.CheckpointableBase):
         AttributeError: if the layer has no defined output shape.
         RuntimeError: if called in Eager mode.
     """
-    if context.executing_eagerly():
-      raise RuntimeError('Layer.output_shape not supported in Eager mode.')
     if not self._inbound_nodes:
       raise AttributeError('The layer has never been called '
                            'and thus has no defined output shape.')
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
index 0f2428d77a..bf361cf805 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.-sequential.pbtxt
@@ -75,10 +75,6 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "regularizers"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "scope_name"
     mtype: "<type \'property\'>"
@@ -91,10 +87,6 @@ tf_class {
     name: "stateful"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "trainable"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "trainable_variables"
     mtype: "<type \'property\'>"
@@ -153,11 +145,11 @@ tf_class {
   }
   member_method {
     name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'inputs\', \'training\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "compile"
-    argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'loss_weights\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "compute_mask"
@@ -173,11 +165,11 @@ tf_class {
   }
   member_method {
     name: "evaluate"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'32\', \'1\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\', \'steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'1\', \'None\', \'None\'], "
   }
   member_method {
     name: "evaluate_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit"
@@ -185,7 +177,7 @@ tf_class {
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
@@ -245,7 +237,7 @@ tf_class {
   }
   member_method {
     name: "predict"
-    argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
+    argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\', \'steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
   }
   member_method {
     name: "predict_classes"
@@ -253,7 +245,7 @@ tf_class {
   }
   member_method {
     name: "predict_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
   }
   member_method {
     name: "predict_on_batch"
@@ -297,6 +289,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'class_weight\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
 }
diff --git a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
index 102eb32203..be12b0bd2e 100644
--- a/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.keras.models.-sequential.pbtxt
@@ -75,10 +75,6 @@ tf_class {
     name: "output_shape"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "regularizers"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "scope_name"
     mtype: "<type \'property\'>"
@@ -91,10 +87,6 @@ tf_class {
     name: "stateful"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "trainable"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "trainable_variables"
     mtype: "<type \'property\'>"
@@ -153,11 +145,11 @@ tf_class {
   }
   member_method {
     name: "call"
-    argspec: "args=[\'self\', \'inputs\'], varargs=None, keywords=kwargs, defaults=None"
+    argspec: "args=[\'self\', \'inputs\', \'training\', \'mask\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
   member_method {
     name: "compile"
-    argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+    argspec: "args=[\'self\', \'optimizer\', \'loss\', \'metrics\', \'loss_weights\', \'sample_weight_mode\', \'weighted_metrics\', \'target_tensors\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'None\', \'None\', \'None\', \'None\', \'None\'], "
   }
   member_method {
     name: "compute_mask"
@@ -173,11 +165,11 @@ tf_class {
   }
   member_method {
     name: "evaluate"
-    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'32\', \'1\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'batch_size\', \'verbose\', \'sample_weight\', \'steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'1\', \'None\', \'None\'], "
   }
   member_method {
     name: "evaluate_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'10\', \'1\', \'False\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\'], "
   }
   member_method {
     name: "fit"
@@ -185,7 +177,7 @@ tf_class {
   }
   member_method {
     name: "fit_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps_per_epoch\', \'epochs\', \'verbose\', \'callbacks\', \'validation_data\', \'validation_steps\', \'class_weight\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'shuffle\', \'initial_epoch\'], varargs=None, keywords=None, defaults=[\'None\', \'1\', \'1\', \'None\', \'None\', \'None\', \'None\', \'10\', \'1\', \'False\', \'True\', \'0\'], "
   }
   member_method {
     name: "from_config"
@@ -245,7 +237,7 @@ tf_class {
   }
   member_method {
     name: "predict"
-    argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\'], varargs=None, keywords=None, defaults=[\'32\', \'0\'], "
+    argspec: "args=[\'self\', \'x\', \'batch_size\', \'verbose\', \'steps\'], varargs=None, keywords=None, defaults=[\'None\', \'0\', \'None\'], "
   }
   member_method {
     name: "predict_classes"
@@ -253,7 +245,7 @@ tf_class {
   }
   member_method {
     name: "predict_generator"
-    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=kwargs, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
+    argspec: "args=[\'self\', \'generator\', \'steps\', \'max_queue_size\', \'workers\', \'use_multiprocessing\', \'verbose\'], varargs=None, keywords=None, defaults=[\'None\', \'10\', \'1\', \'False\', \'0\'], "
   }
   member_method {
     name: "predict_on_batch"
@@ -297,6 +289,6 @@ tf_class {
   }
   member_method {
     name: "train_on_batch"
-    argspec: "args=[\'self\', \'x\', \'y\', \'class_weight\', \'sample_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
+    argspec: "args=[\'self\', \'x\', \'y\', \'sample_weight\', \'class_weight\'], varargs=None, keywords=None, defaults=[\'None\', \'None\'], "
   }
 }
-- 
GitLab


From 31741634e73d7e75a0285f6977e1f61cd38f754a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 22 Mar 2018 20:02:45 -0700
Subject: [PATCH 402/960] Checkpointable: Add a way for subclasses to manage
 deferred restorations of conditional dependencies. PiperOrigin-RevId:
 190166571

---
 .../eager/python/checkpointable_utils_test.py | 10 ++++++---
 tensorflow/python/training/checkpointable.py  | 21 +++++++++++++++++--
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index a4b215bb7a..a8c47d76d1 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -378,7 +378,11 @@ class CheckpointingTests(test.TestCase):
     if not context.executing_eagerly():
       return  # Restore-on-create is only supported when executing eagerly
     on_create_model = MyModel()
-    on_create_optimizer = adam.AdamOptimizer(0.001)
+    on_create_optimizer = adam.AdamOptimizer(
+        0.001,
+        # Preserve beta1_power and beta2_power when appying gradients so we can
+        # test that they've been restored correctly.
+        beta1=1.0, beta2=1.0)
     on_create_root = checkpointable_utils.Checkpoint(
         optimizer=on_create_optimizer, model=on_create_model)
     # Deferred restoration
@@ -395,8 +399,8 @@ class CheckpointingTests(test.TestCase):
     self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot))
     self.assertAllEqual(optimizer_variables[2:],
                         self.evaluate(on_create_optimizer.variables()))
-    on_create_optimizer._create_slots(
-        [resource_variable_ops.ResourceVariable([1.])])
+    dummy_var = resource_variable_ops.ResourceVariable([1.])
+    on_create_optimizer.minimize(loss=dummy_var.read_value)
     status.assert_consumed()
     beta1_power, beta2_power = on_create_optimizer._get_beta_accumulators()
     self.assertAllEqual(optimizer_variables[0], self.evaluate(beta1_power))
diff --git a/tensorflow/python/training/checkpointable.py b/tensorflow/python/training/checkpointable.py
index 96e3c4821c..d0650eb127 100644
--- a/tensorflow/python/training/checkpointable.py
+++ b/tensorflow/python/training/checkpointable.py
@@ -321,8 +321,10 @@ class CheckpointableBase(object):
     # Maps names -> Checkpointable objects
     self._unconditional_dependency_names = {}
     # Restorations for other Checkpointable objects on which this object may
-    # eventually depend.
-    self._deferred_dependencies = {}  # local name -> _CheckpointPosition list
+    # eventually depend. Maps local name -> _CheckpointPosition list. Optimizers
+    # tack on conditional dependencies, and so need separate management of
+    # deferred dependencies too.
+    self._unconditional_deferred_dependencies = {}
     # The UID of the highest assignment to this object. Used to ensure that the
     # last requested assignment determines the final value of an object.
     if hasattr(self, "_update_uid"):
@@ -344,6 +346,21 @@ class CheckpointableBase(object):
     """
     return self._unconditional_checkpoint_dependencies
 
+  @property
+  def _deferred_dependencies(self):
+    """A dictionary with deferred dependencies.
+
+    Stores restorations for other Checkpointable objects on which this object
+    may eventually depend. May be overridden by sub-classes (e.g. Optimizers use
+    conditional dependencies based the current graph, and so need separate
+    management of deferred dependencies too).
+
+    Returns:
+      A dictionary mapping from local name to a list of _CheckpointPosition
+      objects.
+    """
+    return self._unconditional_deferred_dependencies
+
   def _lookup_dependency(self, name):
     """Look up a dependency by name.
 
-- 
GitLab


From 1c3b7c3639b2d2f694d1b28acbd05daaa6064311 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 22 Mar 2018 20:32:03 -0700
Subject: [PATCH 403/960] Rollback of GCS change. END_PUBLIC

BEGIN_PUBLIC
Automated g4 rollback of changelist 189900977

PiperOrigin-RevId: 190169076
---
 tensorflow/core/platform/cloud/BUILD          |  41 -----
 tensorflow/core/platform/cloud/fake_env.cc    |  62 -------
 tensorflow/core/platform/cloud/fake_env.h     |  60 -------
 .../core/platform/cloud/gce_env_utils.cc      | 159 ------------------
 .../core/platform/cloud/gce_env_utils.h       |  29 ----
 .../core/platform/cloud/gcp_env_utils_test.cc |  53 ------
 .../platform/cloud/google_auth_provider.cc    |  11 --
 .../platform/cloud/google_auth_provider.h     |   7 +-
 .../cloud/google_auth_provider_test.cc        |  39 ++---
 tensorflow/core/platform/env.h                |   4 +-
 10 files changed, 16 insertions(+), 449 deletions(-)
 delete mode 100644 tensorflow/core/platform/cloud/fake_env.cc
 delete mode 100644 tensorflow/core/platform/cloud/fake_env.h
 delete mode 100644 tensorflow/core/platform/cloud/gce_env_utils.cc
 delete mode 100644 tensorflow/core/platform/cloud/gce_env_utils.h
 delete mode 100644 tensorflow/core/platform/cloud/gcp_env_utils_test.cc

diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index e43639e9c7..21636641e7 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -79,18 +79,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "gce_env_utils",
-    srcs = ["gce_env_utils.cc"],
-    hdrs = ["gce_env_utils.h"],
-    copts = tf_copts(),
-    visibility = ["//visibility:private"],
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 cc_library(
     name = "gcs_file_system",
     srcs = ["gcs_file_system.cc"],
@@ -170,7 +158,6 @@ cc_library(
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
         ":curl_http_request",
-        ":gce_env_utils",
         ":oauth_client",
         ":retrying_utils",
         "//tensorflow/core:lib",
@@ -256,21 +243,6 @@ cc_library(
     ],
 )
 
-cc_library(
-    name = "fake_env",
-    srcs = [
-        "fake_env.cc",
-    ],
-    hdrs = [
-        "fake_env.h",
-    ],
-    copts = tf_copts(),
-    deps = [
-        "//tensorflow/core:framework_headers_lib",
-        "//tensorflow/core:lib_internal",
-    ],
-)
-
 tf_cc_test(
     name = "expiring_lru_cache_test",
     size = "small",
@@ -376,7 +348,6 @@ tf_cc_test(
         "testdata/service_account_credentials.json",
     ],
     deps = [
-        ":fake_env",
         ":google_auth_provider",
         ":http_request_fake",
         ":oauth_client",
@@ -423,15 +394,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-tf_cc_test(
-    name = "gce_env_utils_test",
-    size = "small",
-    srcs = ["gcp_env_utils_test.cc"],
-    deps = [
-        ":fake_env",
-        ":gce_env_utils",
-        "//tensorflow/core:test",
-        "//tensorflow/core:test_main",
-    ],
-)
diff --git a/tensorflow/core/platform/cloud/fake_env.cc b/tensorflow/core/platform/cloud/fake_env.cc
deleted file mode 100644
index 221166839e..0000000000
--- a/tensorflow/core/platform/cloud/fake_env.cc
+++ /dev/null
@@ -1,62 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/platform/cloud/fake_env.h"
-
-namespace tensorflow {
-namespace test {
-
-Status FakeEnv::FakeRandomAccessFile::Read(uint64 offset, size_t n,
-                                           StringPiece* result,
-                                           char* scratch) const {
-  CHECK_EQ(offset, 0);
-  CHECK_EQ(n, 256);
-  Status s;
-  string platform;
-  switch (env_type_) {
-    case kGoogle: {
-      platform = "Google\n  ";
-      s = errors::OutOfRange("");
-      break;
-    }
-    case kGce: {
-      platform = "  Google Compute Engine\n  ";
-      s = errors::OutOfRange("");
-      break;
-    }
-    case kLocal: {
-      platform = "HP Linux Workstation";
-      s = Status::OK();
-      break;
-    }
-    case kBad: {
-      platform = "";
-      s = errors::Internal("Expected");
-      break;
-    }
-  }
-  strncpy(scratch, platform.data(), strlen(platform.data()));
-  *result = StringPiece(scratch, platform.length());
-  return s;
-}
-
-Status FakeEnv::NewRandomAccessFile(const string& fname,
-                                    std::unique_ptr<RandomAccessFile>* result) {
-  result->reset(new FakeRandomAccessFile(env_type_));
-  return Status::OK();
-}
-
-}  // namespace test
-}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/fake_env.h b/tensorflow/core/platform/cloud/fake_env.h
deleted file mode 100644
index 7c162d9d66..0000000000
--- a/tensorflow/core/platform/cloud/fake_env.h
+++ /dev/null
@@ -1,60 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
-#define TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
-
-#include "tensorflow/core/platform/env.h"
-
-namespace tensorflow {
-namespace test {
-
-/// Env implementation that stubs out the calls to read a file and time.
-class FakeEnv : public EnvWrapper {
- public:
-  enum EnvType {
-    kGoogle,
-    kGce,
-    kLocal,
-    kBad,
-  };
-
-  FakeEnv(EnvType env_type) : EnvWrapper(Env::Default()), env_type_(env_type) {}
-
-  class FakeRandomAccessFile : public RandomAccessFile {
-   public:
-    FakeRandomAccessFile(EnvType env_type) : env_type_(env_type) {}
-
-    Status Read(uint64 offset, size_t n, StringPiece* result,
-                char* scratch) const override;
-
-   private:
-    EnvType env_type_;
-  };
-
-  Status NewRandomAccessFile(
-      const string& fname, std::unique_ptr<RandomAccessFile>* result) override;
-
-  uint64 NowSeconds() override { return now; }
-  uint64 now = 10000;
-
- private:
-  EnvType env_type_;
-};
-
-}  // namespace test
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_FAKE_ENV_H_
diff --git a/tensorflow/core/platform/cloud/gce_env_utils.cc b/tensorflow/core/platform/cloud/gce_env_utils.cc
deleted file mode 100644
index d78374c4b8..0000000000
--- a/tensorflow/core/platform/cloud/gce_env_utils.cc
+++ /dev/null
@@ -1,159 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/platform/cloud/gce_env_utils.h"
-
-#if defined(PLATFORM_WINDOWS)
-#include <algorithm>
-#include <cctype>
-#include <iostream>
-#include <string>
-
-// The order if these includes is important, windows.h has to come first.
-// clang-format off
-#include <windows.h>   // NOLINT
-#include <tchar.h>     // NOLINT
-#include <shellapi.h>  // NOLINT
-// clang-format on
-#else
-#include "tensorflow/core/lib/gtl/stl_util.h"
-#include "tensorflow/core/lib/strings/str_util.h"
-#endif
-
-namespace tensorflow {
-
-constexpr char kExpectedGoogleProductName[] = "Google";
-constexpr char kExpectedGceProductName[] = "Google Compute Engine";
-
-constexpr char kWinCheckCommand[] = "powershell.exe";
-constexpr char kWinCheckCommandArgs[] =
-    "(Get-WmiObject -Class Win32_BIOS).Manufacturer";
-
-constexpr char kLinuxProductNameFile[] = "/sys/class/dmi/id/product_name";
-
-const size_t kBiosDataBufferSize = 256;
-
-namespace {
-
-#if defined(PLATFORM_WINDOWS)
-
-Status IsRunningOnWinGce(bool* is_running_under_gce) {
-  *is_running_under_gce = FALSE;
-  SECURITY_ATTRIBUTES sa;
-  sa.nLength = sizeof(sa);
-  sa.lpSecurityDescriptor = NULL;
-  sa.bInheritHandle = TRUE;
-
-  // Handles to input and output of the pipe connecting us
-  // to the child process running powershell(). The output of this
-  // child process will be written to 'process_output_in' and read from
-  // 'process_output_in'.
-  HANDLE process_output_out = NULL;
-  HANDLE process_output_in = NULL;
-
-  // Create the actually pipe connecting us to the child process.
-  if (!CreatePipe(&process_output_out, &process_output_in, &sa, 0)) {
-    return errors::Internal("CreatePipe() failed");
-  }
-  if (!SetHandleInformation(process_output_out, HANDLE_FLAG_INHERIT, 0)) {
-    return errors::Internal("SetHandleInformation() failed");
-  }
-
-  PROCESS_INFORMATION pi;
-  STARTUPINFO si;
-  DWORD flags = CREATE_NO_WINDOW;
-  ZeroMemory(&pi, sizeof(pi));
-  ZeroMemory(&si, sizeof(si));
-  si.cb = sizeof(si);
-  si.dwFlags |= STARTF_USESTDHANDLES;
-  si.hStdInput = NULL;
-
-  // Connect the process to pipe's input.
-  si.hStdError = process_output_in;
-  si.hStdOutput = process_output_in;
-  // Execute (and wait for) powershell command to read the product information
-  // out of the registry.
-  TCHAR cmd[kBiosDataBufferSize];
-  snprintf(cmd, kBiosDataBufferSize, "%s %s", _T(kWinCheckCommand),
-           _T(kWinCheckCommandArgs));
-
-  if (!CreateProcess(NULL, cmd, NULL, NULL, TRUE, flags, NULL, NULL, &si,
-                     &pi)) {
-    return errors::Internal("CreateProcess() failed");
-  }
-
-  WaitForSingleObject(pi.hProcess, INFINITE);
-  CloseHandle(pi.hProcess);
-  CloseHandle(pi.hThread);
-
-  // Read data from the pipe. Note that we are reading only kBiosDataBufferSize
-  // chars. There might be technically more data than that but we are looking
-  // for Google product identifiers that are much shorter than
-  // kBiosDataBufferSize.
-  DWORD dwread = 0;
-  CHAR buffer[kBiosDataBufferSize];
-  if (!ReadFile(process_output_out, buffer, kBiosDataBufferSize, &dwread,
-                NULL)) {
-    return errors::Internal("Failed reading from the pipe.");
-  }
-  std::string output(buffer, 0, dwread);
-  // Trim whitespaces
-  output.erase(output.begin(),
-               std::find_if(output.begin(), output.end(),
-                            [](int ch) { return !std::isspace(ch); }));
-  output.erase(std::find_if(output.rbegin(), output.rend(),
-                            [](int ch) { return !std::isspace(ch); })
-                   .base(),
-               output.end());
-  *is_running_under_gce =
-      output == kExpectedGceProductName || output == kExpectedGoogleProductName;
-  return Status::OK();
-}
-
-#else
-
-Status IsRunningOnLinuxGce(Env* env, bool* is_running_under_gce) {
-  std::unique_ptr<RandomAccessFile> file;
-  TF_RETURN_IF_ERROR(env->NewRandomAccessFile(kLinuxProductNameFile, &file));
-  char buf[kBiosDataBufferSize + 1];
-  std::fill(buf, buf + kBiosDataBufferSize + 1, '\0');
-  StringPiece product_name;
-  const Status s = file->Read(0, kBiosDataBufferSize, &product_name, buf);
-  if (!s.ok() && !errors::IsOutOfRange(s)) {
-    // We expect OutOfRange error because bios file doesn't correspond to its
-    // state size,
-    return s;
-  }
-  str_util::RemoveLeadingWhitespace(&product_name);
-  str_util::RemoveTrailingWhitespace(&product_name);
-  *is_running_under_gce = (product_name == kExpectedGceProductName ||
-                           product_name == kExpectedGoogleProductName);
-  return Status::OK();
-}
-
-#endif
-
-}  // namespace
-
-Status IsRunningOnGce(Env* env, bool* is_running_under_gce) {
-  *is_running_under_gce = false;
-#if defined(PLATFORM_WINDOWS)
-  return IsRunningOnWinGce(is_running_under_gce);
-#else
-  return IsRunningOnLinuxGce(env, is_running_under_gce);
-#endif
-}
-
-}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/gce_env_utils.h b/tensorflow/core/platform/cloud/gce_env_utils.h
deleted file mode 100644
index 25aaeb7db3..0000000000
--- a/tensorflow/core/platform/cloud/gce_env_utils.h
+++ /dev/null
@@ -1,29 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
-#define TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
-
-#include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/platform/env.h"
-
-namespace tensorflow {
-
-// Check whether the current process is running under GCE.
-Status IsRunningOnGce(Env* env, bool* is_running_under_gce);
-
-}  // namespace tensorflow
-
-#endif  // TENSORFLOW_CORE_PLATFORM_CLOUD_GCE_ENV_UTILS_H_
diff --git a/tensorflow/core/platform/cloud/gcp_env_utils_test.cc b/tensorflow/core/platform/cloud/gcp_env_utils_test.cc
deleted file mode 100644
index 910397b52b..0000000000
--- a/tensorflow/core/platform/cloud/gcp_env_utils_test.cc
+++ /dev/null
@@ -1,53 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/core/platform/cloud/gce_env_utils.h"
-
-#include "tensorflow/core/lib/core/status_test_util.h"
-#include "tensorflow/core/platform/cloud/fake_env.h"
-#include "tensorflow/core/platform/test.h"
-
-namespace tensorflow {
-
-namespace {
-
-TEST(GcpEnvUtils, IsRunningOnGce) {
-  {
-    test::FakeEnv env(test::FakeEnv::kGoogle);
-    bool is_running_on_gcp = false;
-    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
-    EXPECT_TRUE(is_running_on_gcp);
-  }
-  {
-    test::FakeEnv env(test::FakeEnv::kGce);
-    bool is_running_on_gcp = false;
-    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
-    EXPECT_TRUE(is_running_on_gcp);
-  }
-  {
-    test::FakeEnv env(test::FakeEnv::kLocal);
-    bool is_running_on_gcp = false;
-    TF_EXPECT_OK(IsRunningOnGce(&env, &is_running_on_gcp));
-    EXPECT_FALSE(is_running_on_gcp);
-  }
-  {
-    test::FakeEnv env(test::FakeEnv::kBad);
-    bool is_running_on_gcp = false;
-    EXPECT_TRUE(errors::IsInternal(IsRunningOnGce(&env, &is_running_on_gcp)));
-  }
-}
-
-}  // namespace
-}  // namespace tensorflow
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.cc b/tensorflow/core/platform/cloud/google_auth_provider.cc
index 0e8a620464..7e39b63e3e 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider.cc
@@ -26,7 +26,6 @@ limitations under the License.
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/base64.h"
 #include "tensorflow/core/platform/cloud/curl_http_request.h"
-#include "tensorflow/core/platform/cloud/gce_env_utils.h"
 #include "tensorflow/core/platform/cloud/retrying_utils.h"
 #include "tensorflow/core/platform/env.h"
 
@@ -208,16 +207,6 @@ Status GoogleAuthProvider::GetTokenFromFiles() {
 }
 
 Status GoogleAuthProvider::GetTokenFromGce() {
-  if (!is_running_on_gce_.has_value()) {
-    bool is_running_on_gce = false;
-    TF_RETURN_IF_ERROR(IsRunningOnGce(env_, &is_running_on_gce));
-    is_running_on_gce_ = is_running_on_gce;
-  }
-  if (!is_running_on_gce_.value()) {
-    // Assume bucket is world-accessible. If not, the access will be rejected.
-    current_token_ = "";
-    return Status::OK();
-  }
   const auto get_token_from_gce = [this]() {
     std::unique_ptr<HttpRequest> request(http_request_factory_->Create());
     std::vector<char> response_buffer;
diff --git a/tensorflow/core/platform/cloud/google_auth_provider.h b/tensorflow/core/platform/cloud/google_auth_provider.h
index 79a57ff2a0..00da25a959 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider.h
+++ b/tensorflow/core/platform/cloud/google_auth_provider.h
@@ -17,7 +17,6 @@ limitations under the License.
 #define TENSORFLOW_CORE_PLATFORM_GOOGLE_AUTH_PROVIDER_H_
 
 #include <memory>
-#include "tensorflow/core/lib/gtl/optional.h"
 #include "tensorflow/core/platform/cloud/auth_provider.h"
 #include "tensorflow/core/platform/cloud/oauth_client.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -47,10 +46,7 @@ class GoogleAuthProvider : public AuthProvider {
   /// standard gcloud tool's location.
   Status GetTokenFromFiles() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
-  /// Gets the bearer token from Google Compute Engine environment. May return
-  /// an empty token if the current process is not running under GCE. If that
-  /// happens the caller will try to use the empty token and either succeed
-  /// if the resource is publicly accessible or fail with a permissions error.
+  /// Gets the bearer token from Google Compute Engine environment.
   Status GetTokenFromGce() EXCLUSIVE_LOCKS_REQUIRED(mu_);
 
   /// Gets the bearer token from the systen env variable, for testing purposes.
@@ -61,7 +57,6 @@ class GoogleAuthProvider : public AuthProvider {
   Env* env_;
   mutex mu_;
   string current_token_ GUARDED_BY(mu_);
-  tensorflow::gtl::optional<bool> is_running_on_gce_ GUARDED_BY(mu_);
   uint64 expiration_timestamp_sec_ GUARDED_BY(mu_) = 0;
   // The initial delay for exponential backoffs when retrying failed calls.
   const int64 initial_retry_delay_usec_;
diff --git a/tensorflow/core/platform/cloud/google_auth_provider_test.cc b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
index 55829f84d9..4281c6c737 100644
--- a/tensorflow/core/platform/cloud/google_auth_provider_test.cc
+++ b/tensorflow/core/platform/cloud/google_auth_provider_test.cc
@@ -17,7 +17,6 @@ limitations under the License.
 #include <stdlib.h>
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
-#include "tensorflow/core/platform/cloud/fake_env.h"
 #include "tensorflow/core/platform/cloud/http_request_fake.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -27,6 +26,14 @@ namespace {
 
 constexpr char kTestData[] = "core/platform/cloud/testdata/";
 
+class FakeEnv : public EnvWrapper {
+ public:
+  FakeEnv() : EnvWrapper(Env::Default()) {}
+
+  uint64 NowSeconds() override { return now; }
+  uint64 now = 10000;
+};
+
 class FakeOAuthClient : public OAuthClient {
  public:
   Status GetTokenFromServiceAccountJson(
@@ -82,7 +89,7 @@ TEST_F(GoogleAuthProviderTest, EnvironmentVariable_Caching) {
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> requests;
 
-  test::FakeEnv env(test::FakeEnv::kGoogle);
+  FakeEnv env;
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -116,7 +123,7 @@ TEST_F(GoogleAuthProviderTest, GCloudRefreshToken) {
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> requests;
 
-  test::FakeEnv env(test::FakeEnv::kGoogle);
+  FakeEnv env;
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -162,7 +169,7 @@ TEST_F(GoogleAuthProviderTest, RunningOnGCE) {
                 "token_type":"Bearer"
               })")});
 
-  test::FakeEnv env(test::FakeEnv::kGoogle);
+  FakeEnv env;
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -188,7 +195,7 @@ TEST_F(GoogleAuthProviderTest, OverrideForTesting) {
 
   auto oauth_client = new FakeOAuthClient;
   std::vector<HttpRequest*> empty_requests;
-  test::FakeEnv env(test::FakeEnv::kGoogle);
+  FakeEnv env;
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&empty_requests)),
@@ -208,25 +215,7 @@ TEST_F(GoogleAuthProviderTest, NothingAvailable) {
       "Header Metadata-Flavor: Google\n",
       "", errors::NotFound("404"), 404)});
 
-  test::FakeEnv env(test::FakeEnv::kGoogle);
-  GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
-                              std::unique_ptr<HttpRequest::Factory>(
-                                  new FakeHttpRequestFactory(&requests)),
-                              &env, 0);
-
-  string token;
-  TF_EXPECT_OK(provider.GetToken(&token));
-  EXPECT_EQ("", token);
-}
-
-TEST_F(GoogleAuthProviderTest, AccessingPublicBucket) {
-  setenv("CLOUDSDK_CONFIG",
-         io::JoinPath(testing::TensorFlowSrcRoot(), kTestData).c_str(), 1);
-
-  auto oauth_client = new FakeOAuthClient;
-  std::vector<HttpRequest*> requests;
-
-  test::FakeEnv env(test::FakeEnv::kLocal);
+  FakeEnv env;
   GoogleAuthProvider provider(std::unique_ptr<OAuthClient>(oauth_client),
                               std::unique_ptr<HttpRequest::Factory>(
                                   new FakeHttpRequestFactory(&requests)),
@@ -234,8 +223,6 @@ TEST_F(GoogleAuthProviderTest, AccessingPublicBucket) {
 
   string token;
   TF_EXPECT_OK(provider.GetToken(&token));
-  // We are assuming we are accessing a public bucket (and we are not running
-  // on GCE) so we an empty token is returned.
   EXPECT_EQ("", token);
 }
 
diff --git a/tensorflow/core/platform/env.h b/tensorflow/core/platform/env.h
index a7e9fcb17c..9192f7ba10 100644
--- a/tensorflow/core/platform/env.h
+++ b/tensorflow/core/platform/env.h
@@ -88,8 +88,8 @@ class Env {
   /// The ownership of the returned RandomAccessFile is passed to the caller
   /// and the object should be deleted when is not used. The file object
   /// shouldn't live longer than the Env object.
-  virtual Status NewRandomAccessFile(const string& fname,
-                                     std::unique_ptr<RandomAccessFile>* result);
+  Status NewRandomAccessFile(const string& fname,
+                             std::unique_ptr<RandomAccessFile>* result);
 
   /// \brief Creates an object that writes to a new file with the specified
   /// name.
-- 
GitLab


From 72f48771ea4fce68af893bcda862ca390a0e6b70 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Thu, 22 Mar 2018 20:40:55 -0700
Subject: [PATCH 404/960] Improvements to quantization matching code:

- Guarantee that we do the largest matches first.
- Don't allow matching layers multiple times.
- Don't allow adding quantization ops to the same node multiple times.
- Return a list of match results rather than yielding. This is much easier to reason about.
- Only require ReadVariableOp when matching resource variables, since the input to ReadVariableOps don't necessarily have to be a VarHandleOp.
- Place post activation bypass ops quantization nodes in the same post activation bypass op's name scope for better viewing.

PiperOrigin-RevId: 190169622
---
 .../contrib/quantize/python/graph_matcher.py  |   8 +-
 .../contrib/quantize/python/quantize.py       | 120 +++++++++++++-----
 .../contrib/quantize/python/quantize_test.py  |  59 +++++++++
 3 files changed, 150 insertions(+), 37 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/graph_matcher.py b/tensorflow/contrib/quantize/python/graph_matcher.py
index b458f039df..bacc707a3a 100644
--- a/tensorflow/contrib/quantize/python/graph_matcher.py
+++ b/tensorflow/contrib/quantize/python/graph_matcher.py
@@ -103,7 +103,7 @@ class OneofPattern(Pattern):
 class MatchResult(object):
   r"""Encapsulates the result of a match done by GraphMatcher.
 
-  MatchResult contains a map from OpTypePattern to the matching op and tensor.
+  MatchResult contains a map from Pattern to the matching op and tensor.
   When the matching op has multiple output tensors, the matching tensor is the
   output tensor used by the matching op of the parent pattern. E.g., when we
   match graph
@@ -138,7 +138,7 @@ class MatchResult(object):
       self._name_to_pattern[pattern.name] = pattern
 
   def _to_pattern(self, pattern_or_name):
-    if isinstance(pattern_or_name, OpTypePattern):
+    if isinstance(pattern_or_name, Pattern):
       return pattern_or_name
 
     if isinstance(pattern_or_name, str):
@@ -146,8 +146,8 @@ class MatchResult(object):
         return None
       return self._name_to_pattern[pattern_or_name]
 
-    raise ValueError('pattern_or_name has type %s. Expect OpTypePattern or str.'
-                     % type(pattern_or_name))
+    raise ValueError('pattern_or_name has type %s. Expect Pattern or str.' %
+                     type(pattern_or_name))
 
   def _get_op_tensor(self, pattern_or_name):
     pattern = self._to_pattern(pattern_or_name)
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 33f14e8d0e..019d123a68 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -34,9 +34,6 @@ _QUANTIZABLE_TYPES = {'Conv2D', 'MatMul', 'DepthwiseConv2dNative'}
 # Activations that are supported by the quantization rewrite.
 _ACTIVATION_TYPES = {'Relu', 'Relu6', 'Identity'}
 
-# Weight types that are supported by the quantization rewrite.
-_WEIGHT_TYPES = {'Variable', 'VariableV2', 'VarHandleOp'}
-
 
 def Quantize(graph,
              is_training,
@@ -123,9 +120,12 @@ def Quantize(graph,
           vars_collection=vars_collection,
           bits=activation_bits)
 
+    # Quantize bypass ops that occur after the activation.
     if layer_match.post_activation_bypass_op is not None:
+      post_activation_bypass_context = re.search(
+          r'^(.*)/([^/]+)', layer_match.post_activation_bypass_op.name).group(1)
       _InsertQuantOp(
-          add_context,
+          post_activation_bypass_context,
           'post_activation_bypass_quant',
           layer_match.post_activation_bypass_op,
           input_to_ops_map.ConsumerOperations(
@@ -159,7 +159,7 @@ def _FindLayersToQuantize(graph):
    [post_activation_bypass]
 
   Match replacements:
-    If weight_folded_weight is found, FakeQuant is added afterwards.
+    If weight|folded_weight is found, FakeQuant is added afterwards.
     If bypass is found, FakeQuant is added before and after.
     If activation is found, FakeQuant is added afterwards.
     If post_activation_bypass is found, FakeQuant is added afterwards.
@@ -167,14 +167,14 @@ def _FindLayersToQuantize(graph):
   Args:
     graph: Graph to perform match on.
 
-  Yields:
-    _LayerMatches.
+  Returns:
+    list of _LayerMatches.
   """
   input_pattern = graph_matcher.OpTypePattern('*')
-  weight_var_pattern = graph_matcher.OpTypePattern('|'.join(_WEIGHT_TYPES))
-  weight_pattern = graph_matcher.OpTypePattern(
-      'Identity|ReadVariableOp', inputs=[weight_var_pattern])
-
+  weight_var_pattern = graph_matcher.OpTypePattern('Variable|VariableV2')
+  weight_identity_pattern = graph_matcher.OpTypePattern(
+      'Identity', inputs=[weight_var_pattern])
+  weight_resource_var_pattern = graph_matcher.OpTypePattern('ReadVariableOp')
   folded_weight_pattern = graph_matcher.OpTypePattern('Mul')
 
   # The weights inputs to the layer operation can either be from the Variable or
@@ -183,7 +183,10 @@ def _FindLayersToQuantize(graph):
       '|'.join(_QUANTIZABLE_TYPES),
       inputs=[
           input_pattern,
-          graph_matcher.OneofPattern([weight_pattern, folded_weight_pattern])
+          graph_matcher.OneofPattern([
+              weight_identity_pattern, weight_resource_var_pattern,
+              folded_weight_pattern
+          ])
       ])
 
   folded_bias_mul_pattern = graph_matcher.OpTypePattern(
@@ -232,14 +235,29 @@ def _FindLayersToQuantize(graph):
   post_activation_bypass_pattern_b = graph_matcher.OpTypePattern(
       'Add', inputs=[activation_pattern, '*'])
 
-  layer_matcher = graph_matcher.GraphMatcher(
+  # The order of the following matching blocks is very important. Since matches
+  # aren't guaranteed to be disjoint, we structure matches from largest to
+  # smallest to guarantee that the largest match always wins. Additionally, we
+  # ensure that we don't match layers multiple times.
+
+  layer_matches = []
+  # We use matched_layer_set to ensure that layers aren't matched multiple
+  # times.
+  matched_layer_set = set()
+
+  # First, we match layers that have a post activation bypass. We do this first
+  # to ensure we don't match only the first part of this layer, missing the
+  # post activation bypass node.
+  post_activation_bypass_layer_matcher = graph_matcher.GraphMatcher(
       graph_matcher.OneofPattern([
-          post_activation_bypass_pattern_a, post_activation_bypass_pattern_b,
-          activation_pattern
+          post_activation_bypass_pattern_a,
+          post_activation_bypass_pattern_b,
       ]))
-  for match_result in layer_matcher.match_graph(graph):
+  for match_result in post_activation_bypass_layer_matcher.match_graph(graph):
     layer_op = match_result.get_op(layer_pattern)
-    weight_tensor = match_result.get_tensor(weight_pattern)
+    weight_tensor = match_result.get_tensor(weight_identity_pattern)
+    if weight_tensor is None:
+      weight_tensor = match_result.get_tensor(weight_resource_var_pattern)
     if weight_tensor is None:
       weight_tensor = match_result.get_tensor(folded_weight_pattern)
     activation_op = match_result.get_op(activation_pattern)
@@ -254,30 +272,56 @@ def _FindLayersToQuantize(graph):
     if post_activation_bypass_op is None:
       post_activation_bypass_op = match_result.get_op(
           post_activation_bypass_pattern_b)
-    # If we don't find a post_activation_bypass_op but activation_op has a
-    # bypass following it, then we need to skip this match, since there will be
-    # another match that includes post_activation_bypass_op.
-    if post_activation_bypass_op is None and _HasPostActivationBypass(
-        activation_op):
-      continue
-    yield _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op,
-                      post_activation_bypass_op, bias_add_op)
-
-  # Match the final layer, where there will not be an activation and instead
-  # the output of the final BiasAdd must be quantized, so we treat it as the
-  # 'activation_op' in the _LayerMatch.
-  # TODO(suharshs): Figure out how to quantize this final layer across many
-  # models.
+    if layer_op not in matched_layer_set:
+      matched_layer_set.add(layer_op)
+      layer_matches.append(
+          _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op,
+                      post_activation_bypass_op, bias_add_op))
+
+  # Now, we match the basic layer ending at an activation. We may get duplicate
+  # matches from above, but we don't add them to layer_matches.
+  layer_matcher = graph_matcher.GraphMatcher(activation_pattern)
+  for match_result in layer_matcher.match_graph(graph):
+    layer_op = match_result.get_op(layer_pattern)
+    weight_tensor = match_result.get_tensor(weight_identity_pattern)
+    if weight_tensor is None:
+      weight_tensor = match_result.get_tensor(weight_resource_var_pattern)
+    if weight_tensor is None:
+      weight_tensor = match_result.get_tensor(folded_weight_pattern)
+    activation_op = match_result.get_op(activation_pattern)
+    bias_add_op = match_result.get_op(bias_add_pattern)
+    if bias_add_op is None:
+      bias_add_op = match_result.get_op(folded_bias_add_pattern)
+    bypass_op = match_result.get_op(bypass_pattern_a)
+    if bypass_op is None:
+      bypass_op = match_result.get_op(bypass_pattern_b)
+    if layer_op not in matched_layer_set:
+      matched_layer_set.add(layer_op)
+      layer_matches.append(
+          _LayerMatch(layer_op, weight_tensor, activation_op, bypass_op, None,
+                      bias_add_op))
+
+  # Match the final layer, where there may not be an activation and instead
+  # the output of the final BiasAdd must be quantized. So we treat the BiasAdd
+  # as the 'activation_op' in the _LayerMatch, to ensure that it's output is
+  # quantized.
   final_layer_matcher = graph_matcher.GraphMatcher(bias_add_pattern)
   for match_result in final_layer_matcher.match_graph(graph):
     layer_op = match_result.get_op(layer_pattern)
-    weight_tensor = match_result.get_tensor(weight_pattern)
+    weight_tensor = match_result.get_tensor(weight_identity_pattern)
+    if weight_tensor is None:
+      weight_tensor = match_result.get_tensor(weight_resource_var_pattern)
     if weight_tensor is None:
       weight_tensor = match_result.get_tensor(folded_weight_pattern)
     activation_op = match_result.get_op(bias_add_pattern)
     if activation_op is None:
       activation_op = match_result.get_op(folded_bias_add_pattern)
-    yield _LayerMatch(layer_op, weight_tensor, activation_op, None, None, None)
+    if layer_op not in matched_layer_set:
+      matched_layer_set.add(layer_op)
+      layer_matches.append(
+          _LayerMatch(layer_op, weight_tensor, activation_op, None, None, None))
+
+  return layer_matches
 
 
 def _HasPostActivationBypass(activation_op):
@@ -374,6 +418,16 @@ def _InsertQuantOp(context,
   name_prefix = common.DropStringPrefix(name_prefix, ops.get_name_scope() + '/')
 
   inputs = producer.outputs[0]
+  # Prevent ops from being quantized multiple times. Bypass ops can sometimes
+  # overlap between multiple matches, so we need to ensure that we don't
+  # add duplicate FakeQuant operations.
+  fake_quant_ops = set([
+      'FakeQuantWithMinMaxVars',
+      'FakeQuantWithMinMaxArgs'
+  ])
+  if fake_quant_ops.intersection(set([c.type for c in inputs.consumers()])):
+    return
+
   if moving_avg:
     quant = (
         quant_ops.MovingAvgQuantize(
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index bef58bad8d..98f05c8bfc 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -164,6 +164,65 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       self.assertTrue('FakeQuantWithMinMaxVars' in
                       [i.op.type for i in bypass_tensor.op.inputs])
 
+  def testOverlappingPostActivationBypassQuantized(self):
+    self._RunTestOverParameters(
+        self._TestOverlappingPostActivationBypassQuantized)
+
+  def _TestOverlappingPostActivationBypassQuantized(self, is_training):
+    graph = ops.Graph()
+    with graph.as_default():
+      batch_size, height, width, depth = 5, 128, 128, 3
+      conv_input = array_ops.zeros((batch_size, height, width, depth))
+      conv1 = conv2d(
+          conv_input,
+          32, [5, 5],
+          stride=2,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=array_ops.identity,
+          scope='test/test1')
+
+      # The bypass of this conv is the post activation bypass of the previous
+      # conv.
+      conv2 = conv2d(
+          conv_input,
+          32, [5, 5],
+          stride=2,
+          padding='SAME',
+          weights_initializer=self._WeightInit(0.09),
+          activation_fn=None,
+          scope='test/test2')
+
+      bypass_tensor = math_ops.add(conv1, conv2, name='test/add')
+      _ = array_ops.identity(bypass_tensor, name='test/output')
+
+      quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
+
+      # Ensure that the bypass node is preceded and followed by
+      # FakeQuantWithMinMaxVars operations.
+      self.assertTrue('FakeQuantWithMinMaxVars' in
+                      [c.type for c in bypass_tensor.consumers()])
+      self.assertTrue('FakeQuantWithMinMaxVars' in
+                      [i.op.type for i in bypass_tensor.op.inputs])
+
+      # Ensure that all the convs and activations are quantized.
+      op_names = [op.name for op in graph.get_operations()]
+      self.assertTrue(
+          'test/test1/weights_quant/FakeQuantWithMinMaxVars' in op_names)
+      self.assertTrue(
+          'test/test2/weights_quant/FakeQuantWithMinMaxVars' in op_names)
+      self.assertTrue(
+          'test/test1/act_quant/FakeQuantWithMinMaxVars' in op_names)
+      self.assertTrue('test/act_quant/FakeQuantWithMinMaxVars' in op_names)
+      self.assertEqual(
+          'Identity',
+          graph.get_operation_by_name(
+              'test/test1/act_quant/FakeQuantWithMinMaxVars').inputs[0].op.type)
+      self.assertEqual(
+          'Identity',
+          graph.get_operation_by_name(
+              'test/act_quant/FakeQuantWithMinMaxVars').inputs[0].op.type)
+
   def testWithNameScope(self):
     self._RunTestOverParameters(self._TestWithNameScope)
 
-- 
GitLab


From 0191d264a6e3da12ff7db5ba8002fed6356f071b Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Thu, 22 Mar 2018 20:51:51 -0700
Subject: [PATCH 405/960] Added experimental C APIs to build a hard-coded stack
 of dataset + iterator.

PiperOrigin-RevId: 190170332
---
 tensorflow/c/BUILD                      |  19 ++
 tensorflow/c/c_api_experimental.cc      | 223 ++++++++++++++++++++++++
 tensorflow/c/c_api_experimental.h       |  20 +++
 tensorflow/c/c_api_experimental_test.cc |  66 +++++++
 4 files changed, 328 insertions(+)
 create mode 100644 tensorflow/c/c_api_experimental_test.cc

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 4332f44e5d..d096647558 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -216,6 +216,25 @@ tf_cuda_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "c_api_experimental_test",
+    size = "small",
+    srcs = ["c_api_experimental_test.cc"],
+    linkopts = select({
+        "//tensorflow:darwin": ["-headerpad_max_install_names"],
+        "//conditions:default": [],
+    }),
+    # We must ensure that the dependencies can be dynamically linked since
+    # the shared library must be able to use core:framework.
+    # linkstatic = tf_kernel_tests_linkstatic(),
+    deps = [
+        ":c_api_experimental",
+        ":c_test_util",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_cc_test(
     name = "c_api_function_test",
     size = "small",
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 29caf508e7..8593a8eb50 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -22,7 +22,15 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
+using tensorflow::Node;
+using tensorflow::NodeBuilder;
 using tensorflow::Status;
+using tensorflow::Tensor;
+
+// struct TF_Operation { tensorflow::Node node; };
+static TF_Operation* ToTF_Operation(Node* node) {
+  return static_cast<TF_Operation*>(static_cast<void*>(node));
+}
 
 void TF_EnableXLACompilation(TF_SessionOptions* options, unsigned char enable) {
   tensorflow::ConfigProto& config = options->options.config;
@@ -103,3 +111,218 @@ TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
   memcpy(ret, debug_str.c_str(), *len + 1);
   return ret;
 }
+
+// TODO(hongm): Replace this will a real implementation.
+static tensorflow::Status BuildDatasetTest(TF_Graph* dataset_graph,
+                                           Node** dataset_node) {
+  tensorflow::mutex_lock c(dataset_graph->mu);
+  Tensor const_t(tensorflow::DT_INT32, tensorflow::TensorShape({}));
+  const_t.flat<tensorflow::int32>()(0) = 1;
+
+  Node* const_node;
+  TF_RETURN_IF_ERROR(NodeBuilder("Const", "Const")
+                         .Attr("dtype", tensorflow::DT_INT32)
+                         .Attr("value", const_t)
+                         .Finalize(&dataset_graph->graph, &const_node));
+
+  std::vector<NodeBuilder::NodeOut> input_list;
+  input_list.push_back(NodeBuilder::NodeOut(const_node, 0));
+
+  return NodeBuilder("TensorDataset", "TensorDataset")
+      .Input(input_list)
+      .Attr("Toutput_types", {tensorflow::DT_INT32})
+      .Attr("output_shapes", {tensorflow::TensorShapeProto()})
+      .Finalize(&dataset_graph->graph, dataset_node);
+}
+
+//  On success, returns a newly created TF_Function instance from
+//  `text_proto`. It must be deleted by calling TF_DeleteFunction.
+static TF_Function* CreateFunctionFromTextProto(const char* text_proto,
+                                                TF_Status* status) {
+  tensorflow::FunctionDef fdef;
+  if (!tensorflow::protobuf::TextFormat::ParseFromString(text_proto, &fdef)) {
+    status->status = tensorflow::errors::Internal(
+        "Invalid text proto for FunctionDef: ", text_proto);
+    return nullptr;
+  }
+  std::vector<char> binary_proto_buf(fdef.ByteSizeLong());
+  fdef.SerializeToArray(binary_proto_buf.data(), binary_proto_buf.size());
+  return TF_FunctionImportFunctionDef(binary_proto_buf.data(),
+                                      binary_proto_buf.size(), status);
+}
+
+//  On success, returns a newly created TF_Function instance from `proto_file`,
+//  and sets `dataset_name` to the created dataset name. The returned function
+//  must be deleted by calling TF_DeleteFunction.
+//
+// TODO(hongm): Support reading the file given by `proto_file`.
+static TF_Function* LoadDatasetFunction(const char* proto_file,
+                                        std::string* dataset_name,
+                                        TF_Status* status) {
+  const char* func_def = R"PREFIX(
+signature {
+      name: "_make_dataset_d8de2712"
+      output_arg {
+        name: "TensorSliceDataset"
+        type: DT_VARIANT
+      }
+      is_stateful: true
+    }
+    node_def {
+      name: "TensorSliceDataset/tensors/component_0"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+                size: 3
+              }
+            }
+       tensor_content: "\000\000(B\000\000,B\000\0000B"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "TensorSliceDataset"
+      op: "TensorSliceDataset"
+      input: "TensorSliceDataset/tensors/component_0:output:0"
+      attr {
+        key: "Toutput_types"
+        value {
+          list {
+            type: DT_FLOAT
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    ret {
+      key: "TensorSliceDataset"
+      value: "TensorSliceDataset:handle:0"
+    })PREFIX";
+
+  *dataset_name = "_make_dataset_d8de2712";
+  return CreateFunctionFromTextProto(func_def, status);
+}
+
+// TODO(hongm): Use `file_path` in the implementation.
+TF_Operation* TF_MakeIteratorGetNextWithDatasets(TF_Graph* graph,
+                                                 const char* file_path,
+                                                 TF_Function** dataset_func,
+                                                 TF_Status* status) {
+  tensorflow::Status s;
+
+  // We can parameterize the function name, if we ever need more than 1
+  // iterators in a graph.
+  const std::string dataset_name = "UNIQUE_DATASET";
+
+  std::unique_ptr<TF_Graph, decltype(&TF_DeleteGraph)> dataset_graph(
+      TF_NewGraph(), TF_DeleteGraph);
+  Node* dataset_node = nullptr;
+  s = BuildDatasetTest(dataset_graph.get(), &dataset_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+
+  TF_Output output{ToTF_Operation(dataset_node), 0};
+  std::unique_ptr<TF_Function, decltype(&TF_DeleteFunction)> result_func(
+      TF_GraphToFunction(dataset_graph.get(), dataset_name.c_str(),
+                         /*append_hash_to_fn_name*/ false,
+                         /*num_opers*/ -1,
+                         /*opers*/ nullptr,
+                         /*numinputs*/ 0,
+                         /*inputs*/ nullptr,
+                         /*noutputs*/ 1,
+                         /*outputs*/ &output,
+                         /*outputnames*/ nullptr,
+                         /*functionoptions*/ nullptr, "", status),
+      TF_DeleteFunction);
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  TF_GraphCopyFunction(graph, result_func.get(), /*gradient*/ nullptr, status);
+
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  tensorflow::mutex_lock c(graph->mu);
+
+  tensorflow::NameAttrList func;
+  func.set_name(dataset_name);
+  // Run the iterator node on CPU.
+  Node* oneshot_iterator_node;
+  std::vector<tensorflow::TensorShapeProto> output_shape_list;
+  output_shape_list.push_back(tensorflow::TensorShapeProto());
+  s = NodeBuilder("OneShotIterator", "OneShotIterator")
+          .Device("/device:CPU:0")
+          .Attr("container", "")
+          .Attr("dataset_factory", func)
+          .Attr("output_types", {tensorflow::DT_INT32})
+          .Attr("output_shapes", output_shape_list)
+          .Attr("shared_name", "")
+          .Finalize(&graph->graph, &oneshot_iterator_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+  // Run shape inference function for each newly added node, so that more
+  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
+  s = graph->refiner.AddNode(oneshot_iterator_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+
+  // Run the iterator node on CPU.
+  Node* getnext_node;
+  s = NodeBuilder("IteratorGetNext", "IteratorGetNext")
+          .Input(oneshot_iterator_node)
+          .Device("/device:CPU:0")
+          .Attr("output_types", {tensorflow::DT_INT32})
+          .Attr("output_shapes", output_shape_list)
+          .Finalize(&graph->graph, &getnext_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+  // Run shape inference function for each newly added node, so that more
+  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
+  s = graph->refiner.AddNode(getnext_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+
+  VLOG(1) << "Output graph: " << graph->graph.ToGraphDefDebug().DebugString();
+  *dataset_func = result_func.release();
+  return ToTF_Operation(getnext_node);
+}
+
+void TF_GetAttrScalarTensorShapeProto(TF_Buffer* value, TF_Status* status) {
+  status->status = Status::OK();
+  auto shape = tensorflow::TensorShape({});
+  tensorflow::TensorShapeProto shape_proto;
+  shape.AsProto(&shape_proto);
+  status->status = MessageToBuffer(shape_proto, value);
+}
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index f069398bbb..2fa232878c 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -87,6 +87,26 @@ TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
 TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
                                                       size_t* len);
 
+// Creates a stack of data set + iterator nodes reading the TFRecord files from
+// `file_path`, and outputs the following info on success:
+//
+// 1. Returns the IteratorGetNext node, which caller can run or feed into an
+// node.
+//
+// 2. Sets `dataset_func` to the created function that encapsulates the data set
+// nodes. Caller owns that function, and must call TF_DeleteFunction() on it.
+//
+//
+// The nodes are currently hard-coded to return a single Int32 of value 1.
+// TODO(hongm): Extend the API to allow customization of the nodes created.
+TF_CAPI_EXPORT extern TF_Operation* TF_MakeIteratorGetNextWithDatasets(
+    TF_Graph* graph, const char* file_path, TF_Function** dataset_func,
+    TF_Status* status);
+
+// Returns the shape proto of shape {}.
+TF_CAPI_EXPORT extern void TF_GetAttrScalarTensorShapeProto(TF_Buffer* value,
+                                                            TF_Status* status);
+
 #ifdef __cplusplus
 } /* end extern "C" */
 #endif
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
new file mode 100644
index 0000000000..9ddd65f0c5
--- /dev/null
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -0,0 +1,66 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/c/c_api_experimental.h"
+#include "tensorflow/c/c_test_util.h"
+
+namespace tensorflow {
+namespace {
+
+void TestIteratorStack() {
+  TF_Status* s = TF_NewStatus();
+  TF_Graph* graph = TF_NewGraph();
+
+  TF_Function* dataset_func = nullptr;
+
+  TF_Operation* get_next =
+      TF_MakeIteratorGetNextWithDatasets(graph, "dummy_path", &dataset_func, s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+
+  ASSERT_NE(dataset_func, nullptr);
+  TF_DeleteFunction(dataset_func);
+
+  CSession csession(graph, s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+
+  // Run the graph.
+  for (int i = 0; i < 1; ++i) {
+    csession.SetOutputs({get_next});
+    csession.Run(s);
+    ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+    TF_Tensor* out = csession.output_tensor(0);
+    ASSERT_TRUE(out != nullptr);
+    EXPECT_EQ(TF_INT32, TF_TensorType(out));
+    EXPECT_EQ(0, TF_NumDims(out));  // scalar
+    ASSERT_EQ(sizeof(int32), TF_TensorByteSize(out));
+    int32* output_contents = static_cast<int32*>(TF_TensorData(out));
+    EXPECT_EQ(1, *output_contents);
+  }
+
+  // This should error out since we've exhausted the iterator.
+  csession.Run(s);
+  ASSERT_EQ(TF_OUT_OF_RANGE, TF_GetCode(s)) << TF_Message(s);
+
+  // Clean up
+  csession.CloseAndDelete(s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+  TF_DeleteGraph(graph);
+  TF_DeleteStatus(s);
+}
+
+TEST(CAPI_EXPERIMENTAL, IteratorGetNext) { TestIteratorStack(); }
+
+}  // namespace
+}  // namespace tensorflow
-- 
GitLab


From fc8047525717038d963c31555df01b3d3d53afd2 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 22 Mar 2018 21:45:22 -0700
Subject: [PATCH 406/960] Fixing build issues: removed deletion of os and
 reordered tags

---
 tensorflow/contrib/__init__.py        | 1 -
 tensorflow/contrib/eager/python/BUILD | 2 +-
 2 files changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index cc16e9379d..4f6f539027 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -92,7 +92,6 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
-del os
 del LazyLoader
 
 del absolute_import
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index f9172ef4fd..80176397c0 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -271,9 +271,9 @@ cuda_py_test(
         "//tensorflow/python/keras",
     ],
     tags = [
-        "notsan",
         "no_oss",  # b/74395663
         "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
     ],
 )
 
-- 
GitLab


From ed0defc1ffef77fa27bf360100d983eeee3359db Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 21:46:31 -0700
Subject: [PATCH 407/960] Add cast op support for bfloat16 in CPU.

PiperOrigin-RevId: 190173501
---
 tensorflow/core/kernels/cast_op_impl.h             |  8 ++++++--
 tensorflow/core/kernels/cast_op_impl_bfloat.cc     | 12 +-----------
 tensorflow/core/kernels/cast_op_impl_bool.cc       |  2 +-
 tensorflow/core/kernels/cast_op_impl_complex128.cc |  2 +-
 tensorflow/core/kernels/cast_op_impl_complex64.cc  |  2 +-
 tensorflow/core/kernels/cast_op_impl_double.cc     |  2 +-
 tensorflow/core/kernels/cast_op_impl_float.cc      | 13 -------------
 tensorflow/core/kernels/cast_op_impl_half.cc       |  2 +-
 tensorflow/core/kernels/cast_op_impl_int16.cc      |  2 +-
 tensorflow/core/kernels/cast_op_impl_int32.cc      |  2 +-
 tensorflow/core/kernels/cast_op_impl_int64.cc      |  2 +-
 tensorflow/core/kernels/cast_op_impl_int8.cc       |  2 +-
 tensorflow/core/kernels/cast_op_impl_uint16.cc     |  2 +-
 tensorflow/core/kernels/cast_op_impl_uint8.cc      |  2 +-
 tensorflow/core/kernels/cast_op_test.cc            |  4 +++-
 15 files changed, 21 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/kernels/cast_op_impl.h b/tensorflow/core/kernels/cast_op_impl.h
index 3ae9f2ab4d..382e5440e1 100644
--- a/tensorflow/core/kernels/cast_op_impl.h
+++ b/tensorflow/core/kernels/cast_op_impl.h
@@ -58,10 +58,14 @@ struct CastFunctor<Eigen::SyclDevice, O, I> {
   FN(arg0, arg1, std::complex<float>);       \
   FN(arg0, arg1, std::complex<double>)
 
-#define CURRY_TYPES3(FN, arg0, arg1)   \
-  CURRY_TYPES3_NO_HALF(FN, arg0, arg1) \
+#define CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
+  CURRY_TYPES3_NO_HALF(FN, arg0, arg1)       \
   FN(arg0, arg1, Eigen::half);
 
+#define CURRY_TYPES3(FN, arg0, arg1)   \
+  CURRY_TYPES3_NO_BF16(FN, arg0, arg1) \
+  FN(arg0, arg1, bfloat16);
+
 #define CAST_CASE(DEVICE, IN, OUT)                                         \
   if (DataTypeToEnum<OUT>::value == dst_dtype) {                           \
     return [](OpKernelContext* ctx, const Tensor& inp, Tensor* out) {      \
diff --git a/tensorflow/core/kernels/cast_op_impl_bfloat.cc b/tensorflow/core/kernels/cast_op_impl_bfloat.cc
index a06f815899..bfa7ba0d47 100644
--- a/tensorflow/core/kernels/cast_op_impl_bfloat.cc
+++ b/tensorflow/core/kernels/cast_op_impl_bfloat.cc
@@ -24,17 +24,7 @@ typedef Eigen::GpuDevice GPUDevice;
 
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetCpuCastFromBfloat(DataType dst_dtype) {
-  if (dst_dtype == DT_FLOAT) {
-    return [](OpKernelContext* ctx, const Tensor& inp, Tensor* out) {
-      int64 N = out->NumElements();
-      auto worker_threads = ctx->device()->tensorflow_cpu_worker_threads();
-      auto work = [&inp, &out](int64 start, int64 end) {
-        BFloat16ToFloat(inp.flat<bfloat16>().data() + start,
-                        out->flat<float>().data() + start, end - start);
-      };
-      Shard(worker_threads->num_threads, worker_threads->workers, N, 2, work);
-    };
-  }
+  CURRY_TYPES3(CAST_CASE, CPUDevice, bfloat16);
   return nullptr;
 }
 
diff --git a/tensorflow/core/kernels/cast_op_impl_bool.cc b/tensorflow/core/kernels/cast_op_impl_bool.cc
index 5cd63f2458..c5c7394b43 100644
--- a/tensorflow/core/kernels/cast_op_impl_bool.cc
+++ b/tensorflow/core/kernels/cast_op_impl_bool.cc
@@ -29,7 +29,7 @@ GetCpuCastFromBool(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromBool(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, bool);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, bool);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_complex128.cc b/tensorflow/core/kernels/cast_op_impl_complex128.cc
index c428679d7c..52899d58cd 100644
--- a/tensorflow/core/kernels/cast_op_impl_complex128.cc
+++ b/tensorflow/core/kernels/cast_op_impl_complex128.cc
@@ -29,7 +29,7 @@ GetCpuCastFromComplex128(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromComplex128(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, std::complex<double>);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, std::complex<double>);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_complex64.cc b/tensorflow/core/kernels/cast_op_impl_complex64.cc
index 07b46551b2..617bda53d5 100644
--- a/tensorflow/core/kernels/cast_op_impl_complex64.cc
+++ b/tensorflow/core/kernels/cast_op_impl_complex64.cc
@@ -29,7 +29,7 @@ GetCpuCastFromComplex64(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromComplex64(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, std::complex<float>);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, std::complex<float>);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_double.cc b/tensorflow/core/kernels/cast_op_impl_double.cc
index 1203f066a2..7dc485ddad 100644
--- a/tensorflow/core/kernels/cast_op_impl_double.cc
+++ b/tensorflow/core/kernels/cast_op_impl_double.cc
@@ -29,7 +29,7 @@ GetCpuCastFromDouble(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromDouble(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, double);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, double);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_float.cc b/tensorflow/core/kernels/cast_op_impl_float.cc
index 2ff9af21f2..1c933914fd 100644
--- a/tensorflow/core/kernels/cast_op_impl_float.cc
+++ b/tensorflow/core/kernels/cast_op_impl_float.cc
@@ -25,18 +25,6 @@ typedef Eigen::GpuDevice GPUDevice;
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetCpuCastFromFloat(DataType dst_dtype) {
   CURRY_TYPES3(CAST_CASE, CPUDevice, float);
-  if (dst_dtype == DT_BFLOAT16) {
-    return [](OpKernelContext* ctx, const Tensor& inp, Tensor* out) {
-      int64 N = out->NumElements();
-      auto worker_threads = ctx->device()->tensorflow_cpu_worker_threads();
-      auto work = [&inp, &out](int64 start, int64 end) {
-        FloatToBFloat16(inp.flat<float>().data() + start,
-                        out->flat<bfloat16>().data() + start, end - start);
-      };
-      Shard(worker_threads->num_threads, worker_threads->workers, N, 2, work);
-    };
-  }
-
   return nullptr;
 }
 
@@ -44,7 +32,6 @@ GetCpuCastFromFloat(DataType dst_dtype) {
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromFloat(DataType dst_dtype) {
   CURRY_TYPES3(CAST_CASE, GPUDevice, float);
-  CAST_CASE(GPUDevice, float, bfloat16);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_half.cc b/tensorflow/core/kernels/cast_op_impl_half.cc
index e89d4646d7..ef4b94e326 100644
--- a/tensorflow/core/kernels/cast_op_impl_half.cc
+++ b/tensorflow/core/kernels/cast_op_impl_half.cc
@@ -29,7 +29,7 @@ GetCpuCastFromHalf(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromHalf(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, Eigen::half);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, Eigen::half);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_int16.cc b/tensorflow/core/kernels/cast_op_impl_int16.cc
index f12d852e95..59360f7445 100644
--- a/tensorflow/core/kernels/cast_op_impl_int16.cc
+++ b/tensorflow/core/kernels/cast_op_impl_int16.cc
@@ -29,7 +29,7 @@ GetCpuCastFromInt16(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromInt16(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, int16);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, int16);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_int32.cc b/tensorflow/core/kernels/cast_op_impl_int32.cc
index 2a4b27a12d..a867392fde 100644
--- a/tensorflow/core/kernels/cast_op_impl_int32.cc
+++ b/tensorflow/core/kernels/cast_op_impl_int32.cc
@@ -29,7 +29,7 @@ GetCpuCastFromInt32(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromInt32(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, int32);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, int32);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_int64.cc b/tensorflow/core/kernels/cast_op_impl_int64.cc
index 065defabbb..467a8f6c89 100644
--- a/tensorflow/core/kernels/cast_op_impl_int64.cc
+++ b/tensorflow/core/kernels/cast_op_impl_int64.cc
@@ -29,7 +29,7 @@ GetCpuCastFromInt64(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromInt64(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, int64);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, int64);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_int8.cc b/tensorflow/core/kernels/cast_op_impl_int8.cc
index 8d678c4733..21002a4321 100644
--- a/tensorflow/core/kernels/cast_op_impl_int8.cc
+++ b/tensorflow/core/kernels/cast_op_impl_int8.cc
@@ -29,7 +29,7 @@ GetCpuCastFromInt8(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromInt8(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, int8);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, int8);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_uint16.cc b/tensorflow/core/kernels/cast_op_impl_uint16.cc
index c917aaf7bd..cd829bae2a 100644
--- a/tensorflow/core/kernels/cast_op_impl_uint16.cc
+++ b/tensorflow/core/kernels/cast_op_impl_uint16.cc
@@ -29,7 +29,7 @@ GetCpuCastFromUint16(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromUint16(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, uint16);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, uint16);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_impl_uint8.cc b/tensorflow/core/kernels/cast_op_impl_uint8.cc
index 377c8ca953..2d1a6f3a4e 100644
--- a/tensorflow/core/kernels/cast_op_impl_uint8.cc
+++ b/tensorflow/core/kernels/cast_op_impl_uint8.cc
@@ -29,7 +29,7 @@ GetCpuCastFromUint8(DataType dst_dtype) {
 #if GOOGLE_CUDA
 std::function<void(OpKernelContext*, const Tensor&, Tensor*)>
 GetGpuCastFromUint8(DataType dst_dtype) {
-  CURRY_TYPES3(CAST_CASE, GPUDevice, uint8);
+  CURRY_TYPES3_NO_BF16(CAST_CASE, GPUDevice, uint8);
   return nullptr;
 }
 #endif  // GOOGLE_CUDA
diff --git a/tensorflow/core/kernels/cast_op_test.cc b/tensorflow/core/kernels/cast_op_test.cc
index 057e209a71..7da9d28a3d 100644
--- a/tensorflow/core/kernels/cast_op_test.cc
+++ b/tensorflow/core/kernels/cast_op_test.cc
@@ -75,7 +75,8 @@ class CastOpTest : public OpsTestBase {
   TEST_CAST(in, int64);         \
   TEST_CAST(in, half);          \
   TEST_CAST(in, float);         \
-  TEST_CAST(in, double)
+  TEST_CAST(in, double);        \
+  TEST_CAST(in, bfloat16);
 
 TEST_ALL_CASTS_FROM(uint8)
 TEST_ALL_CASTS_FROM(uint16)
@@ -85,6 +86,7 @@ TEST_ALL_CASTS_FROM(int64)
 TEST_ALL_CASTS_FROM(half)
 TEST_ALL_CASTS_FROM(float)
 TEST_ALL_CASTS_FROM(double)
+TEST_ALL_CASTS_FROM(bfloat16)
 
 #undef TEST_ALL_CASTS_FROM
 #undef TEST_CAST
-- 
GitLab


From 16d92bba68ea6f109c2ca1a4a675d0eb961a8b72 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 22 Mar 2018 23:25:37 -0700
Subject: [PATCH 408/960] Enable fusion of element-wise ops using XLA (Off by
 default, can be enabled by setting the flag tf_xla_fusion_only=true).

PiperOrigin-RevId: 190179459
---
 .../mark_for_compilation_pass_flags.cc        |   6 +-
 .../mark_for_compilation_pass_flags.h         |   4 +
 .../compiler/jit/mark_for_compilation_pass.cc | 162 +++++++++++++++++-
 tensorflow/compiler/tests/jit_test.py         |  51 ++++++
 4 files changed, 221 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
index 51384ac2fe..7277a1d1f8 100644
--- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
+++ b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.cc
@@ -41,6 +41,7 @@ static void AllocateFlags() {
   flags->tf_xla_clustering_debug = false;
   flags->tf_xla_cpu_global_jit = false;
   flags->tf_xla_clustering_fuel = std::numeric_limits<int64>::max();
+  flags->tf_xla_fusion_only = false;
   flag_list = new std::vector<Flag>(
       {Flag("tf_xla_auto_jit", &flags->tf_xla_auto_jit,
             "Control compilation of operators into XLA computations on CPU and "
@@ -59,7 +60,10 @@ static void AllocateFlags() {
             "Enables global JIT compilation for CPU via SessionOptions."),
        Flag("tf_xla_clustering_fuel", &flags->tf_xla_clustering_fuel,
             "Places an artificial limit on the number of ops marked as "
-            "eligible for clustering.")});
+            "eligible for clustering."),
+       Flag("tf_xla_fusion_only", &flags->tf_xla_fusion_only,
+            "enable fusion of element-wise operations only using XLA when "
+            "global_jit_level is ON*.")});
   xla::legacy_flags::ParseFlagsFromEnv(*flag_list);
 }
 
diff --git a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h
index 170b89c987..2affda6ab4 100644
--- a/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h
+++ b/tensorflow/compiler/jit/legacy_flags/mark_for_compilation_pass_flags.h
@@ -51,6 +51,10 @@ typedef struct {
   int64 tf_xla_clustering_fuel;   // "Compiler fuel" for clustering.  Only this
                                   // many ops will be marked as eligible for
                                   // clustering.
+  bool tf_xla_fusion_only;  // This flag is effective only when global_jit_level
+                            // is set to ON* and overrides its behavior. If
+                            // true, enable fusion of element-wise operations
+                            // only using XLA.
 } MarkForCompilationPassFlags;
 
 // Return a pointer to the MarkForCompilationPassFlags struct;
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass.cc b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
index 57fb8d2422..f651768a67 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass.cc
@@ -180,6 +180,158 @@ struct NodeCompare {
 };
 using OrderedNodeSet = std::set<Node*, NodeCompare>;
 
+// Returns true if the op can be decomposed into XLA ops for which
+// there are fusable elemental implementations.
+//
+// TODO(hpucha): Consider a black list instead of a white list as
+// implemented below.
+bool IsXlaFusable(const NodeDef& node) {
+  static const std::unordered_set<std::string>* elementwise_ops =
+      new std::unordered_set<std::string>(
+          {// tf2xla/kernels/aggregate_ops.cc
+           "AddN",
+           // tf2xla/kernels/batchtospace_op.cc
+           "BatchToSpace", "BatchToSpaceND",
+           // tf2xla/kernels/bcast_ops.cc
+           "BroadcastArgs", "BroadcastGradientArgs",
+           // tf2xla/kernels/bias_ops.cc
+           "BiasAdd", "BiasAddV1", "BiasAddGrad" /*(Reduce)*/,
+           // tf2xla/kernels/binary_ops.cc
+           "Add", "Sub", "Mul", "Div", "Atan2", "Complex", "FloorDiv",
+           "FloorMod", "BitwiseAnd", "BitwiseOr", "LeftShift", "RightShift",
+           "LogicalAnd", "LogicalOr", "Mod", "Maximum", "Minimum", "RealDiv",
+           "ReciprocalGrad", "RsqrtGrad", "SqrtGrad", "SquaredDifference",
+           "TruncateDiv", "TruncateMod", "Equal", "NotEqual", "Greater",
+           "GreaterEqual", "Less", "LessEqual", "SigmoidGrad", "SoftplusGrad",
+           "SoftsignGrad", "TanhGrad", "Pow", "ApproximateEqual",
+           // tf2xla/kernels/cast_op.cc
+           "Cast",
+           // tf2xla/kernels/categorical_op.cc
+           "Multinomial" /* (Rng ops are disabled on GPU backend currently)*/,
+           // tf2xla/kernels/concat_op.cc
+           "Concat", "ConcatV2", "ConcatOffset",
+           // tf2xla/kernels/const_op.cc
+           "Const",
+           // tf2xla/kernels/cross_op.cc
+           "Cross",
+           // tf2xla/kernels/depthtospace_op.cc
+           "DepthToSpace",
+           // tf2xla/kernels/diag_op.cc
+           "Diag", "DiagPart", "MatrixDiag", "MatrixDiagPart",
+           // tf2xla/kernels/dynamic_stitch_op.cc
+           "DynamicStitch", "ParallelDynamicStitch",
+           // tf2xla/kernels/elu_op.cc
+           "Elu", "EluGrad", "Selu", "SeluGrad",
+           // tf2xla/kernels/fake_quantize_ops.cc
+           "FakeQuantWithMinMaxArgs", "FakeQuantWithMinMaxArgsGradient",
+           "FakeQuantWithMinMaxVars",
+           "FakeQuantWithMinMaxVarsGradient" /*(Reduce)*/,
+           // tf2xla/kernels/fill_op.cc
+           "Fill",
+           // tf2xla/kernels/gather_op.cc
+           "Gather", "GatherV2", "GatherNd",
+           // tf2xla/kernels/identity_op.cc
+           "Identity", "IdentityN", "PreventGradient", "StopGradient",
+           "Snapshot",
+           // tf2xla/kernels/image_ops.cc
+           "RGBToHSV", "HSVToRGB", "AdjustContrastv2" /*(Reduce)*/,
+           "AdjustSaturation", "AdjustHue",
+           // tf2xla/kernels/index_ops.cc
+           "ArgMax", "ArgMin",
+           // tf2xla/kernels/l2loss_op.cc
+           "L2Loss" /*(Reduce)*/,
+           // tf2xla/kernels/lrn_ops.cc (ReduceWindow)
+           "LRN", "LRNGrad",
+           // tf2xla/kernels/matrix_band_part_op.cc
+           "MatrixBandPart",
+           // tf2xla/kernels/matrix_set_diag_op.cc
+           "MatrixSetDiag",
+           // tf2xla/kernels/mirror_pad_op.cc
+           "MirrorPad",
+           // tf2xla/kernels/no_op.cc
+           "NoOp", "ControlTrigger",
+           // tf2xla/kernels/one_hot_op.cc
+           "OneHot",
+           // tf2xla/kernels/pack_op.cc
+           "Pack",
+           // tf2xla/kernels/pad_op.cc
+           "Pad", "PadV2",
+           // tf2xla/kernels/pooling_ops.cc
+           "MaxPool", "MaxPoolV2", "MaxPool3D", "AvgPool",
+           "AvgPool3D", /*(all the pooling ops use ReduceWindow)*/
+           "MaxPoolGrad", "MaxPoolGradV2", "MaxPool3DGrad", "AvgPoolGrad",
+           "AvgPool3DGrad",
+           // tf2xla/kernels/quantize_and_dequantize_op.cc (Reduce)
+           "QuantizeAndDequantizeV2",
+           // tf2xla/kernels/random_ops.cc (Rng ops are disabled on GPU backend
+           // currently)
+           "RandomUniform", "RandomUniformInt", "RandomStandardNormal",
+           "TruncatedNormal",
+           // tf2xla/kernels/reduction_ops.cc (Reduce)
+           "Sum", "Prod", "Min", "Max", "Mean", "All", "Any",
+           // tf2xla/kernels/relu_op.cc
+           "Relu", "Relu6", "ReluGrad", "Relu6Grad",
+           // tf2xla/kernels/reshape_op.cc
+           "Reshape",
+           // tf2xla/kernels/reverse_op.cc
+           "Reverse", "ReverseV2",
+           // tf2xla/kernels/reverse_sequence_op.cc
+           "ReverseSequence",
+           // tf2xla/kernels/scan_ops.cc (ReduceWindow)
+           "Cumsum", "Cumprod",
+           // tf2xla/kernels/scatter_nd_op.cc (Reduce)
+           "ScatterNd",
+           // tf2xla/kernels/segment_reduction_ops.cc (Reduce)
+           "UnsortedSegmentSum",
+           // tf2xla/kernels/select_op.cc
+           "Select",
+           // tf2xla/kernels/sequence_ops.cc
+           "Range", "LinSpace",
+           // tf2xla/kernels/shape_op.cc
+           "Shape", "ShapeN", "Rank", "Size", "ExpandDims", "Squeeze",
+           "ZerosLike", "OnesLike",
+           // tf2xla/kernels/slice_op.cc
+           "Slice",
+           // tf2xla/kernels/softmax_op.cc (Reduce)
+           "Softmax", "LogSoftmax", "SoftmaxCrossEntropyWithLogits",
+           "SparseSoftmaxCrossEntropyWithLogits",
+           // tf2xla/kernels/spacetobatch_op.cc
+           "SpaceToBatchND", "SpaceToBatch",
+           // tf2xla/kernels/spacetodepth_op.cc
+           "SpaceToDepth",
+           // tf2xla/kernels/split_op.cc
+           "Split", "SplitV",
+           // tf2xla/kernels/stack_ops.cc
+           "StackV2", "StackPushV2", "StackPopV2", "StackCloseV2",
+           // tf2xla/kernels/stateless_random_ops.cc (Rng ops are disabled on
+           // GPU
+           // backend currently)
+           "StatelessRandomUniform",
+           "StatelessRandomNormal"
+           // tf2xla/kernels/strided_slice_op.cc
+           "StridedSlice",
+           "StridedSliceGrad", "ResourceStridedSliceAssign",
+           // tf2xla/kernels/tile_ops.cc
+           "Tile",
+           // tf2xla/kernels/training_ops.cc
+           "ResourceApplyGradientDescent", "ResourceApplyMomentum",
+           "ResourceApplyAdagrad", "ResourceApplyAdam", "ResourceApplyRMSProp",
+           "ResourceApplyFtrl", "ResourceApplyFtrlV2",
+           // tf2xla/kernels/transpose_op.cc
+           "Transpose", "InvertPermutation",
+           // tf2xla/kernels/unary_ops.cc
+           "ComplexAbs", "Angle", "Conj", "Abs", "Acos", "Acosh", "Asin",
+           "Asinh", "Atan", "Atanh", "Ceil", "Cos", "Cosh", "Sin", "Exp",
+           "Expm1", "Floor", "IsFinite", "IsInf", "IsNan", "Inv", "Reciprocal",
+           "Log", "Log1p", "Invert", "LogicalNot", "Neg", "Rint", "Round",
+           "Rsqrt", "Sigmoid", "Sign", "Sinh", "Softplus", "Softsign", "Sqrt",
+           "Square", "Tan", "Tanh", "Real", "Imag",
+           // tf2xla/kernels/unpack_op.cc
+           "Unpack"});
+
+  return elementwise_ops->count(node.op()) > 0;
+}
+
 Status FindCompilationCandidates(
     const Graph& graph, FunctionLibraryDefinition* flib_def, Env* env,
     const std::function<bool(const Node*, const DeviceType&)>& is_compilable_fn,
@@ -338,10 +490,13 @@ Status MarkForCompilationPass::Run(
         static_cast<OptimizerOptions::GlobalJitLevel>(flags->tf_xla_auto_jit);
   }
   bool cpu_global_jit = flags->tf_xla_cpu_global_jit;
+  bool fusion_only = flags->tf_xla_fusion_only;
+
   VLOG(1) << "flags->tf_xla_cpu_global_jit = " << flags->tf_xla_cpu_global_jit;
+  VLOG(1) << "flags->tf_xla_fusion_only = " << flags->tf_xla_fusion_only;
   const FunctionLibraryDefinition* fld = options.flib_def;
 
-  auto is_compilable = [global_jit_level, cpu_global_jit, fld](
+  auto is_compilable = [global_jit_level, cpu_global_jit, fusion_only, fld](
                            const Node* node, const DeviceType& device_type) {
     const XlaOpRegistry::DeviceRegistration* registration;
     if (!XlaOpRegistry::GetCompilationDevice(device_type.type(),
@@ -364,6 +519,11 @@ Status MarkForCompilationPass::Run(
     status = fld->GetAttr(*node, kXlaCompileAttr, &compile);
     if (status.ok()) return compile;
 
+    // Check for fusable ops only if requested.
+    if (global_jit_level > 0 && fusion_only && !IsXlaFusable(node->def())) {
+      return false;
+    }
+
     // Otherwise use the value of global_jit_level.
     // Ignore enable_jit_by_default if global jit compilation for CPU
     // is explicitly requested via tf_xla_cpu_global_jit flag
diff --git a/tensorflow/compiler/tests/jit_test.py b/tensorflow/compiler/tests/jit_test.py
index 2d8236e2cb..f9d87c2d1c 100644
--- a/tensorflow/compiler/tests/jit_test.py
+++ b/tensorflow/compiler/tests/jit_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
 import numpy as np
 
 from tensorflow.contrib.compiler import jit
@@ -436,5 +437,55 @@ class XlaCompilationTest(test.TestCase):
     self.assertTrue(InLabels(labels, "_XlaLaunch"))
 
 
+class ElementWiseFusionTest(test.TestCase):
+
+  # Runs a simple test with the input jit_level and fusion_only flag.
+  def simpleTest(self, arg0, arg1, global_jit_level):
+    config = config_pb2.ConfigProto()
+    config.graph_options.optimizer_options.global_jit_level = global_jit_level
+
+    with session_lib.Session(config=config) as sess:
+      a1 = array_ops.placeholder(dtypes.float32, [2, 2], name="a1")
+      a2 = array_ops.placeholder(dtypes.float32, [2, 2], name="a2")
+      # Two element-wise ops. We need at least two ops since single
+      # element clusters are not passed to XLA in fusion_only mode.
+      a3 = a1 * a2
+      a4 = a3 + a1
+      # A matmul to break XLA clustering.
+      a5 = math_ops.matmul(a4, a1)
+      # Two more element-wise ops.
+      a6 = a5 - a4
+      a7 = a6 + a2
+
+      run_metadata = config_pb2.RunMetadata()
+      output = sess.run(
+          a7, {
+              a1: arg0,
+              a2: arg1
+          },
+          run_metadata=run_metadata,
+          options=config_pb2.RunOptions(
+              trace_level=config_pb2.RunOptions.FULL_TRACE))
+
+      labels = RunMetadataLabels(run_metadata)
+      count = sum("_XlaLaunch(" in x for x in labels)
+
+      return output, count
+
+  def testElementWiseClustering(self):
+    arg0 = np.random.rand(2, 2).astype(np.float32)
+    arg1 = np.random.rand(2, 2).astype(np.float32)
+    os.environ["TF_XLA_FLAGS"] = "--tf_xla_fusion_only=true"
+    tf_op, tf_count = self.simpleTest(arg0, arg1,
+                                      config_pb2.OptimizerOptions.OFF)
+    self.assertEqual(0, tf_count)
+
+    tfef_op, tfef_count = self.simpleTest(arg0, arg1,
+                                          config_pb2.OptimizerOptions.ON_1)
+    self.assertEqual(2, tfef_count)
+
+    self.assertAllClose(tf_op, tfef_op, rtol=1e-1)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 341f906e7b6011de4d4a10380a17040abc8bdf5e Mon Sep 17 00:00:00 2001
From: Sami Kama <samikama@users.noreply.github.com>
Date: Thu, 22 Mar 2018 23:49:47 -0700
Subject: [PATCH 409/960] Do not follow control edges in segmenter and in
 conversion and (#17936)

gracefully handle some failures

(cherry picked from commit 5daa95eeeae66b21fc60e08bf0f7c35b3df517f6)
(cherry picked from commit ee87a13583001dd9b19cb5272f85d227ad59297f)
(cherry picked from commit 9a1e6b0e9ca25da050f5a1866235189e6db528ae)

and squashed
---
 .../contrib/tensorrt/convert/convert_graph.cc | 19 +++--
 .../contrib/tensorrt/convert/convert_nodes.cc | 82 +++++++++++++------
 .../contrib/tensorrt/segment/segment.cc       | 55 +++++++++----
 tensorflow/contrib/tensorrt/segment/segment.h |  4 +-
 .../contrib/tensorrt/segment/segment_test.cc  |  8 +-
 5 files changed, 109 insertions(+), 59 deletions(-)

diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index 90447ee666..ff8cc6374d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,7 +49,7 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::Node* node) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
@@ -75,7 +75,7 @@ bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
       // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
-  return candidate_ops.count(node_def.op());
+  return candidate_ops.count(node->type_string());
 }
 
 void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
@@ -85,10 +85,10 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->in_edges()) {
       if (!subgraph_node_ids.count(edge->src()->id()) &&
-          !edge->src()->IsSource()) {
+          !edge->src()->IsSource() && !edge->IsControlEdge()) {
         incoming_edges->insert(edge);
       } else {
-        VLOG(2) << edge->src()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->src()->name() << " N, ";
       }
     }
   }
@@ -101,11 +101,11 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
-          !edge->dst()->IsSink()) {
-        VLOG(2) << edge->dst()->name() << " Y, ";
+          !edge->dst()->IsSink() && !edge->IsControlEdge()) {
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
       } else {
-        VLOG(2) << edge->dst()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -410,8 +410,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
       if (status != tensorflow::Status::OK()) {
         LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \n"
-                     << status.ToString() << " SKIPPING......";
+                     << " due to: \"" << status.ToString()
+                     << "\" SKIPPING......( " << subgraph_node_names.size()
+                     << " nodes)";
       }
       count++;
     }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 979b5648c2..f22502aaeb 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -53,8 +53,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
-
 namespace {
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
@@ -429,9 +429,8 @@ class Converter {
   tensorflow::tensorrt::TRTWeightStore* weight_store_;
   bool fp16_;
   void register_op_converters();
-  std::vector<TRT_TensorOrWeights> get_inputs(
-      const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs;
+  tensorflow::Status get_inputs(const tensorflow::NodeDef& node_def,
+                                std::vector<TRT_TensorOrWeights>* inputs) {
     for (auto const& input_name : node_def.input()) {
       /*************************************************************************
        * TODO(jie) handle case 1) here
@@ -452,13 +451,17 @@ class Converter {
 
       VLOG(2) << "retrieve input: " << name;
       if (trt_tensors_.count(name)) {
-        inputs.push_back(trt_tensors_.at(name));
+        inputs->push_back(trt_tensors_.at(name));
       } else {
-        LOG(FATAL) << "input: " << name << " not availabled for node at, "
-                   << node_def.name();
+        string str("Node ");
+        StrAppend(&str, node_def.name(), " should have an input named '", name,
+                  "' but it is not available");
+        LOG(WARNING) << "input: " << name << " not available for node at "
+                     << node_def.name();
+        return tensorflow::errors::InvalidArgument(str);
       }
     }
-    return inputs;
+    return tensorflow::Status::OK();
   }
 
  public:
@@ -482,7 +485,8 @@ class Converter {
   }
 
   tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs = this->get_inputs(node_def);
+    std::vector<TRT_TensorOrWeights> inputs;
+    TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs));
     string op = node_def.op();
     if (!op_registry_.count(op)) {
       return tensorflow::errors::Unimplemented(
@@ -887,7 +891,7 @@ tensorflow::Status BinaryTensorOpWeight(
 
   // Check type consistency
   nvinfer1::DataType ttype;
-  TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
+  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
 
   // Check scale mode
   auto dims_w = weights.shape_;
@@ -1152,9 +1156,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
+    return tensorflow::errors::Unimplemented(
+        "binary op: " + node_def.op() +
+        " not supported at: " + node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1397,8 +1401,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.float_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1436,8 +1443,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.int_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -2139,8 +2149,11 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
   calib_res->thr_->join();
   delete calib_res->thr_;
   if (!calib_res->engine_) {
-    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+    LOG(ERROR) << "Calibration failed!, engine does not exist. Did you run "
                   "calibration graph?";
+    return tensorflow::errors::FailedPrecondition(
+        "Calibration graph needs to be executed on"
+        " calibration data before convertsion to inference graph");
   }
   auto weight_rmgr = trt_rm->getManager("WeightStore");
   TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
@@ -2177,7 +2190,7 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
     return status;
   }
   auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_CHECK_OK(status);
+  TF_RETURN_IF_ERROR(status);
   for (size_t i = 0; i < out_edges.size(); i++) {
     VLOG(1) << "Connecting trt_engine_node output " << i << " with "
             << out_edges.at(i)->dst()->name() << " port "
@@ -2275,6 +2288,12 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Data type conversion for input '" << node_name
+                   << "' failed";
+      return type_status;
+    }
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "accessing output index of: " << output_idx
@@ -2342,8 +2361,8 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node'" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2500,7 +2519,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Type conversion failed for " << node_name;
+      return type_status;
+    }
 
     VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
@@ -2511,8 +2534,12 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
     //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4)
-      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+    if (op_info.shape().dim_size() != 4) {
+      string err_str = "Require 4 dimensional input.";
+      StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ",
+                shape_inference_node_name);
+      return tensorflow::errors::Unimplemented(err_str);
+    }
 
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
@@ -2573,8 +2600,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node '" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2618,7 +2645,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
   TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
       engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name;
+  LOG(INFO) << "finished engine " << engine_name << " containing "
+            << s.subgraph_node_ids.size() << " nodes";
 
   // Build the TRT op
   tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6193f0b0a1..8fc4697c51 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -80,13 +80,20 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> in_edges(dst->in_edges().begin(),
                                                 dst->in_edges().end());
   for (const tensorflow::Edge* in_edge : in_edges) {
-    if (in_edge->src() != src) {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-      if (e->src() == graph->source_node()) {
-        graph->AddEdge(e->src(), e->src_output(), src,
-                       tensorflow::Graph::kControlSlot);
-      } else {
-        graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+    if (in_edge->IsControlEdge()) {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        graph->AddControlEdge(e->src(), src);
+      }
+    } else {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        if (e->src() == graph->source_node()) {
+          graph->AddEdge(e->src(), e->src_output(), src,
+                         tensorflow::Graph::kControlSlot);
+        } else {
+          graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+        }
       }
     }
   }
@@ -94,12 +101,19 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> out_edges(dst->out_edges().begin(),
                                                  dst->out_edges().end());
   for (const tensorflow::Edge* out_edge : out_edges) {
-    tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-    if (e->dst() == graph->sink_node()) {
-      graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
-                     e->dst_input());
+    if (out_edge->IsControlEdge()) {
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      graph->AddControlEdge(src, e->dst());
     } else {
-      graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      if (e->dst() == graph->sink_node()) {
+        VLOG(1) << " edge to sink node " << src->name() << " -> "
+                << e->dst()->name();
+        graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
+                       e->dst_input());
+      } else {
+        graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      }
     }
   }
 
@@ -118,7 +132,7 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
   // Create a Graph representation of the GraphDef.
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
@@ -136,7 +150,7 @@ tensorflow::Status SegmentGraph(
   for (int i = 0; i < graph.num_node_ids(); ++i) {
     tensorflow::Node* node = graph.FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0 ||
-        !candidate_fn(node->def())) {
+        !candidate_fn(node)) {
       node = nullptr;
     }
     node_segments.emplace_back(node);
@@ -155,7 +169,7 @@ tensorflow::Status SegmentGraph(
 
   for (const tensorflow::Node* node : order) {
     // All output nodes of 'node' have been visited...
-    VLOG(2) << "Trying node " << node->name();
+    VLOG(2) << "Trying node " << node->name() << " id=" << node->id();
 
     // 'node' must be a TRT candidate...
     if (node_segments[node->id()].Value() == nullptr) {
@@ -169,8 +183,12 @@ tensorflow::Status SegmentGraph(
     while (true) {
       std::set<const tensorflow::Edge*> contract_edges;
       for (const tensorflow::Edge* out_edge : node->out_edges()) {
-        VLOG(2) << "... out node " << out_edge->dst()->name();
-
+        VLOG(2) << "... out node " << out_edge->dst()->name() << " ( "
+                << out_edge->dst()->id() << " <- " << node->id() << " )";
+        if (out_edge->IsControlEdge()) {
+          VLOG(2) << "... ... Control Edge, Skipping";
+          continue;
+        }
         // Out node must be TRT candidate...
         if (node_segments[out_edge->dst()->id()].Value() == nullptr) {
           VLOG(2) << "... ... not a TRT candidate";
@@ -196,7 +214,8 @@ tensorflow::Status SegmentGraph(
         const tensorflow::Node* src = contract_edge->src();
         const tensorflow::Node* dst = contract_edge->dst();
 
-        VLOG(2) << "Merge " << src->name() << " <- " << dst->name();
+        VLOG(2) << "Merge " << src->name() << " <- " << dst->name() << " ("
+                << src->id() << " <- " << dst->id();
         node_segments[src->id()].Merge(&node_segments[dst->id()]);
 
         // Contracting the edge leaves disconnected graph edges.
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index ee6e2b3ed2..7e8685f44a 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -20,10 +20,12 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+
 namespace tensorrt {
 namespace segment {
 
@@ -46,7 +48,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
 
 }  // namespace segment
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 74cbc5f2b3..7ddabec268 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -35,7 +35,7 @@ class SegmentTest : public ::testing::Test {
   TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph,
                     TF_Status* s, const char* name);
 
-  std::function<bool(const NodeDef&)> MakeCandidateFn(
+  std::function<bool(const Node*)> MakeCandidateFn(
       const std::set<string>& node_names);
 
  protected:
@@ -60,10 +60,10 @@ bool SegmentTest::GetGraphDef(TF_Graph* graph,
   return ret;
 }
 
-std::function<bool(const NodeDef&)> SegmentTest::MakeCandidateFn(
+std::function<bool(const Node*)> SegmentTest::MakeCandidateFn(
     const std::set<string>& node_names) {
-  return [node_names](const NodeDef& node) -> bool {
-    return node_names.find(node.name()) != node_names.end();
+  return [node_names](const Node* node) -> bool {
+    return node_names.find(node->name()) != node_names.end();
   };
 }
 
-- 
GitLab


From a3f27a5898a3c1b28e6a3fb2e6e6bd75644c522e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 03:37:05 -0700
Subject: [PATCH 410/960] Internal change

PiperOrigin-RevId: 190199534
---
 tensorflow/BUILD | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index e0d86997ff..c75bf8abab 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -767,7 +767,7 @@ tf_cc_shared_object(
     linkopts = select({
         "//tensorflow:darwin": [
             "-Wl,-exported_symbols_list",  # This line must be directly followed by the exported_symbols.lds file
-            "//tensorflow/c:exported_symbols.lds",
+            "$(location //tensorflow/c:exported_symbols.lds)",
             "-Wl,-install_name,@rpath/libtensorflow.so",
         ],
         "//tensorflow:windows": [],
@@ -776,7 +776,7 @@ tf_cc_shared_object(
             "-z defs",
             "-s",
             "-Wl,--version-script",  #  This line must be directly followed by the version_script.lds file
-            "//tensorflow/c:version_script.lds",
+            "$(location //tensorflow/c:version_script.lds)",
         ],
     }),
     deps = [
@@ -794,7 +794,7 @@ tf_cc_shared_object(
     linkopts = select({
         "//tensorflow:darwin": [
             "-Wl,-exported_symbols_list",  # This line must be directly followed by the exported_symbols.lds file
-            "//tensorflow:tf_exported_symbols.lds",
+            "$(location //tensorflow:tf_exported_symbols.lds)",
         ],
         "//tensorflow:windows": [],
         "//tensorflow:windows_msvc": [],
@@ -802,7 +802,7 @@ tf_cc_shared_object(
             "-z defs",
             "-s",
             "-Wl,--version-script",  #  This line must be directly followed by the version_script.lds file
-            "//tensorflow:tf_version_script.lds",
+            "$(location //tensorflow:tf_version_script.lds)",
         ],
     }),
     deps = [
-- 
GitLab


From 9d77ea7b9568373047b10701576c1234f15bf093 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 03:55:03 -0700
Subject: [PATCH 411/960] Implement operator<< for HloSharding

The new operator makes the error messages coming from gtest more
readable as well as making logging easier.

PiperOrigin-RevId: 190200926
---
 tensorflow/compiler/xla/service/hlo_sharding.cc      | 5 +++++
 tensorflow/compiler/xla/service/hlo_sharding.h       | 2 ++
 tensorflow/compiler/xla/service/hlo_sharding_test.cc | 9 +++++++++
 3 files changed, 16 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index aa9ff89e98..27b4626094 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -374,4 +374,9 @@ HloSharding HloSharding::TransformShardedTileShape(
   return HloSharding::Tile(new_tile_shape, tile_assignment());
 }
 
+std::ostream& operator<<(std::ostream& out, const HloSharding& sharding) {
+  out << sharding.ToString();
+  return out;
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h
index 38273236f9..18d406f370 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.h
+++ b/tensorflow/compiler/xla/service/hlo_sharding.h
@@ -269,6 +269,8 @@ class HloSharding {
   std::vector<HloSharding> tuple_elements_;
 };
 
+std::ostream& operator<<(std::ostream& out, const HloSharding& sharding);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_SERVICE_HLO_SHARDING_H_
diff --git a/tensorflow/compiler/xla/service/hlo_sharding_test.cc b/tensorflow/compiler/xla/service/hlo_sharding_test.cc
index 07fc4687cc..9887096eb5 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding_test.cc
@@ -282,5 +282,14 @@ TEST_F(HloShardingTest, TransformShardedTileShapeTest) {
   EXPECT_EQ(result, expected);
 }
 
+TEST_F(HloShardingTest, OstreamTest) {
+  HloSharding sharding =
+      HloSharding::Tile(ShapeUtil::MakeShape(F32, {3, 5, 7, 11}),
+                        Array4D<int64>({{{{0, 1}, {2, 3}}}}));
+  std::ostringstream oss;
+  oss << sharding;
+  EXPECT_EQ(oss.str(), "{f32[3,5,7,11] devices=(0, 1, 2, 3)}");
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 77ac1fb6fb6b4be8152d4d9972cd2ea3968001e8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 04:41:47 -0700
Subject: [PATCH 412/960] Use int32 GPU kernel for StopGradient

PiperOrigin-RevId: 190204238
---
 tensorflow/core/kernels/identity_op.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/core/kernels/identity_op.cc b/tensorflow/core/kernels/identity_op.cc
index b5603fecd8..dffb4d7171 100644
--- a/tensorflow/core/kernels/identity_op.cc
+++ b/tensorflow/core/kernels/identity_op.cc
@@ -129,6 +129,12 @@ REGISTER_GPU_KERNEL(Variant);
                               .HostMemory("output")           \
                               .TypeConstraint<type>("T"),     \
                           IdentityOp);                        \
+  REGISTER_KERNEL_BUILDER(Name("StopGradient")                \
+                              .Device(DEVICE_GPU)             \
+                              .HostMemory("input")            \
+                              .HostMemory("output")           \
+                              .TypeConstraint<type>("T"),     \
+                          IdentityOp);                        \
   REGISTER_KERNEL_BUILDER(Name("PlaceholderWithDefault")      \
                               .Device(DEVICE_GPU)             \
                               .HostMemory("input")            \
-- 
GitLab


From c1eef98b8fa5fa531c6ddd08a0e5f0f00f31b431 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 05:43:13 -0700
Subject: [PATCH 413/960] Fix HloSharding::ToString to be compatible with the
 HLO text parser

The incompatibility made it impossible to parse HLO modules with
sharding attributes printed from a previous run of XLA. The previous
format generated by this function was ambigious so changing the HLO
parser only wouldn't be feasible.

PiperOrigin-RevId: 190208069
---
 .../compiler/xla/service/hlo_sharding.cc      |  6 ++--
 .../compiler/xla/service/hlo_sharding_test.cc | 32 ++++++++++++++++++-
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_sharding.cc b/tensorflow/compiler/xla/service/hlo_sharding.cc
index 27b4626094..e8e45f1ee9 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding.cc
@@ -20,6 +20,7 @@ limitations under the License.
 
 namespace xla {
 
+using ::tensorflow::str_util::Join;
 using ::tensorflow::strings::StrCat;
 
 HloSharding HloSharding::AssignDevice(int64 device_id) {
@@ -57,8 +58,9 @@ string HloSharding::ToString() const {
     return StrCat(
         "{maximal device=", static_cast<int64>(*tile_assignment_.begin()), "}");
   } else {
-    return StrCat("{", ShapeUtil::HumanString(tile_shape_), " ",
-                  "devices=", VectorString(tile_assignment_), "}");
+    return StrCat("{", ShapeUtil::HumanString(tile_shape_), " ", "devices=[",
+                  Join(tile_assignment_.dimensions(), ","), "]",
+                  Join(tile_assignment_, ","), "}");
   }
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_sharding_test.cc b/tensorflow/compiler/xla/service/hlo_sharding_test.cc
index 9887096eb5..69ea4233e4 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_sharding_test.cc
@@ -282,13 +282,43 @@ TEST_F(HloShardingTest, TransformShardedTileShapeTest) {
   EXPECT_EQ(result, expected);
 }
 
+TEST_F(HloShardingTest, ToStringReplicatedTest) {
+  HloSharding sharding = HloSharding::Replicate();
+  EXPECT_EQ(sharding.ToString(), "{replicated}");
+}
+
+TEST_F(HloShardingTest, ToStringAssignDeviceTest) {
+  HloSharding sharding = HloSharding::AssignDevice(7);
+  EXPECT_EQ(sharding.ToString(), "{maximal device=7}");
+}
+
+TEST_F(HloShardingTest, ToStringTiledTest) {
+  HloSharding sharding =
+      HloSharding::Tile(ShapeUtil::MakeShape(S32, {7, 11, 13}),
+                        Array3D<int64>({{{2, 3}}, {{5, 7}}}));
+  EXPECT_EQ(sharding.ToString(), "{s32[7,11,13] devices=[2,1,2]2,3,5,7}");
+}
+
+TEST_F(HloShardingTest, ToStringTupleTest) {
+  HloSharding sharding = HloSharding::Tuple(
+      ShapeUtil::MakeTupleShape({ShapeUtil::MakeShape(F32, {3, 5}),
+                                 ShapeUtil::MakeShape(U32, {7, 25}),
+                                 ShapeUtil::MakeShape(S32, {9, 11})}),
+      {HloSharding::Replicate(),
+       HloSharding::Tile(ShapeUtil::MakeShape(U32, {7, 13}),
+                         Array2D<int64>({{3, 5}})),
+       HloSharding::AssignDevice(3)});
+  EXPECT_EQ(sharding.ToString(),
+            "{{replicated}, {u32[7,13] devices=[1,2]3,5}, {maximal device=3}}");
+}
+
 TEST_F(HloShardingTest, OstreamTest) {
   HloSharding sharding =
       HloSharding::Tile(ShapeUtil::MakeShape(F32, {3, 5, 7, 11}),
                         Array4D<int64>({{{{0, 1}, {2, 3}}}}));
   std::ostringstream oss;
   oss << sharding;
-  EXPECT_EQ(oss.str(), "{f32[3,5,7,11] devices=(0, 1, 2, 3)}");
+  EXPECT_EQ(oss.str(), "{f32[3,5,7,11] devices=[1,1,2,2]0,1,2,3}");
 }
 
 }  // namespace
-- 
GitLab


From 319f20ad87da6c058f6d84edcc41478519d8e15e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sat, 24 Mar 2018 01:29:22 +0800
Subject: [PATCH 414/960] Remove TF_NEED_KAFKA from configure.py as it is not
 needed anymore. (#17565)

* Remove TF_NEED_KAFKA from configure.py as it is not needed anymore.

With the new update in kafka dataset support, it looks like
TF_NEED_KAFKA is not needed any more as we are loading .so
dynamically. For that I think it makes sense to remove TF_NEED_KAFKA
option from configure.py.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Remove with_kafka_support from tensorflow/BUILD

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Disable kafka module if with_kafka_support is false

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add flag of with_kafka_support_windows_override

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Remove duplication of kafka in bazel BUILD

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 configure.py             |  2 +-
 tensorflow/BUILD         |  7 +++++++
 tensorflow/contrib/BUILD | 27 +++++++++++++++++++++------
 3 files changed, 29 insertions(+), 7 deletions(-)

diff --git a/configure.py b/configure.py
index 22b9abedd7..0f52c0ec99 100644
--- a/configure.py
+++ b/configure.py
@@ -1414,7 +1414,7 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform',
-                'with_kafka_support', False, 'kafka')
+                'with_kafka_support', True, 'kafka')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 611204ac4b..bb9079bf5b 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -240,6 +240,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "with_kafka_support_windows_override",
+    define_values = {"with_kafka_support": "true"},
+    values = {"cpu": "x64_windows"},
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "with_gcp_support_android_override",
     define_values = {"with_gcp_support": "true"},
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 2d7bbc016f..0a955db7da 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -51,7 +51,6 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
-        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -110,7 +109,13 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -120,7 +125,6 @@ cc_library(
         "//tensorflow/contrib/boosted_trees:boosted_trees_kernels",
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -133,7 +137,13 @@ cc_library(
         "//tensorflow/contrib/text:all_kernels",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_cuda([
         "//tensorflow/contrib/nccl:nccl_kernels",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_kernels",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -146,7 +156,6 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
@@ -157,7 +166,13 @@ cc_library(
         "//tensorflow/contrib/tensor_forest:tensor_forest_ops_op_lib",
         "//tensorflow/contrib/text:all_ops",
         "//tensorflow/contrib/tpu:all_ops",
-    ],
+    ] + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_ops_op_lib",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(
-- 
GitLab


From 116b9f34c3b3562192c69bbd045886c541cfffbc Mon Sep 17 00:00:00 2001
From: Anton Matosov <anton.matosov@gmail.com>
Date: Fri, 23 Mar 2018 10:29:48 -0700
Subject: [PATCH 415/960] [WIP] MKL repos download for mac and windows + update
 to MKL(-DNN) 0.12 (#17396)

* Fix MKL build on MacOS

* Fix invalid return type, user_memory_->set_data_handle(buf) returns void.

* Fix type mismatch

* Implement downloading and linking of MKL repo on mac and windows

* Rework the way MKL libs are binded to avoid warnings and simplify code

* Switch to matching version of MKL-DNN

* Add mirror links

* Add renamed license file

* Add -fopenmp flag for linux mkl build as Slice op kernel is optimized using OpenMP

* Use clean_dep in the entire file for consistency

* Fix build. Constants are declrared as static in header outside the class scope now

* Reworked fix for `-openmp` flag using the suggestion from @gunan
Original version here https://github.com/gunan/tensorflow/commit/b4d39c5226e8bbafe084b1dfc35f4644f2d906c2
---
 configure.py                                  |   1 -
 .../core/common_runtime/mkl_cpu_allocator.cc  |   3 -
 .../core/common_runtime/mkl_cpu_allocator.h   |   2 +-
 .../core/kernels/mkl_fused_batch_norm_op.cc   |   2 +-
 tensorflow/core/util/mkl_util.h               |   4 +-
 tensorflow/tensorflow.bzl                     |   4 +-
 tensorflow/tools/lib_package/BUILD            |   2 -
 tensorflow/tools/pip_package/BUILD            |   1 -
 tensorflow/workspace.bzl                      | 133 +++++++++++-------
 third_party/mkl/BUILD                         |  47 ++++++-
 third_party/mkl/{LICENSE => MKL_LICENSE}      |   0
 third_party/mkl/build_defs.bzl                |  12 ++
 third_party/mkl/mkl.BUILD                     |  27 +++-
 13 files changed, 159 insertions(+), 79 deletions(-)
 rename third_party/mkl/{LICENSE => MKL_LICENSE} (100%)

diff --git a/configure.py b/configure.py
index 0f52c0ec99..6744082d5d 100644
--- a/configure.py
+++ b/configure.py
@@ -502,7 +502,6 @@ def set_cc_opt_flags(environ_cp):
   write_to_bazelrc('build --copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
   write_to_bazelrc('build --host_copt=-DGEMMLOWP_ALLOW_SLOW_SCALAR_FALLBACK')
 
-
 def set_tf_cuda_clang(environ_cp):
   """set TF_CUDA_CLANG action_env.
 
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 43a909466e..829c19204a 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-constexpr const char* MklCPUAllocator::kMaxLimitStr;
-constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
-
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index 55c8411ad0..73abf18d97 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -50,7 +50,7 @@ class MklCPUAllocator : public VisitableAllocator {
   // Constructor and other standard functions
 
   /// Environment variable that user can set to upper bound on memory allocation
-  static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
+  static inline constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
 
   /// Default upper limit on allocator size - 64GB
   static constexpr size_t kDefaultMaxLimit = 64LL << 30;
diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 9e564b016f..7c14a46dde 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -933,7 +933,7 @@ class MklFusedBatchNormOp : public OpKernel {
   bool is_training_;
   T* mean_values_;
   T* variance_values_;
-  size_t depth_;  // batch normalization is done for per channel.
+  int depth_;  // batch normalization is done for per channel.
 
   void ExtractParams(OpKernelContext* context) {
     const Tensor& input = MklGetInput(context, 0);
diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 34db96075d..9f58e40d94 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -1579,10 +1579,10 @@ class MklDnnData {
   }
 
   /// Set function for data buffer of user memory primitive.
-  inline void* SetUsrMemDataHandle(void* data_buffer) {
+  inline void SetUsrMemDataHandle(void* data_buffer) {
     CHECK_NOTNULL(user_memory_);
     CHECK_NOTNULL(data_buffer);
-    return user_memory_->set_data_handle(data_buffer);
+    user_memory_->set_data_handle(data_buffer);
   }
 
   /// Set function for data buffer of user memory primitive.
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 2d3cb415fe..fcc57d506e 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -22,6 +22,7 @@ load(
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
+    "if_mkl_lnx_x64"
 )
 
 def register_extension_info(**kwargs):
@@ -202,7 +203,8 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False):
           "-ftemplate-depth=900"])
       + if_cuda(["-DGOOGLE_CUDA=1"])
       + if_tensorrt(["-DGOOGLE_TENSORRT=1"])
-      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML", "-fopenmp",])
+      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"])
+      + if_mkl_lnx_x64(["-fopenmp"])
       + if_android_arm(["-mfpu=neon"])
       + if_linux_x86_64(["-msse3"])
       + if_ios_x86_64(["-msse4.1"])
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 3fbdb5cacd..0ede8c6370 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -138,7 +138,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/c/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
@@ -176,7 +175,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/jni/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index d55a883df5..2c06fa60db 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -126,7 +126,6 @@ filegroup(
         "@org_python_pypi_backports_weakref//:LICENSE",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]) + if_not_windows([
         "@nccl_archive//:LICENSE.txt",
     ]) + tf_additional_license_deps(),
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ebb9e9412f..075c2c849a 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -15,6 +15,11 @@ load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
 
 
+# Sanitize a dependency so that it works correctly from code that includes
+# TensorFlow as a submodule.
+def clean_dep(dep):
+  return str(Label(dep))
+
 # If TensorFlow is linked as a submodule.
 # path_prefix is no longer used.
 # tf_repo_name is thought to be under consideration.
@@ -32,17 +37,37 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   arm_compiler_configure(
       name="local_config_arm_compiler",
       remote_config_repo="../arm_compiler",
-      build_file = str(Label("//third_party/toolchains/cpus/arm:BUILD")))
+      build_file = clean_dep("//third_party/toolchains/cpus/arm:BUILD"))
 
   mkl_repository(
-      name = "mkl",
+      name = "mkl_linux",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+      ],
+      sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f",
+      strip_prefix = "mklml_lnx_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_windows",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip"
+      ],
+      sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4",
+      strip_prefix = "mklml_win_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_darwin",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
-          "https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz"
       ],
-      sha256 = "6b07cb7e5451db67c2e31e785ae458b18f7f363c60a61685488f69e9ae7199d4",
-      strip_prefix = "mklml_lnx_2018.0.1.20171007",
-      build_file = str(Label("//third_party/mkl:mkl.BUILD")),
+      sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f",
+      strip_prefix = "mklml_mac_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
   )
 
   if path_prefix:
@@ -52,12 +77,12 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "mkl_dnn",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
-          "https://github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
+          "https://github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
       ],
-      sha256 = "02e244f63dd95402691a361392504c143eede9a89043426f174836638a9cbf09",
-      strip_prefix = "mkl-dnn-e0bfcaa7fcb2b1e1558f5f0676933c1db807a729",
-      build_file = str(Label("//third_party/mkl_dnn:mkldnn.BUILD")),
+      sha256 = "86fa2a8c12a56e3b725945acedeaa82492746be02545aba6d710f097e013e19e",
+      strip_prefix = "mkl-dnn-0.12",
+      build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"),
   )
 
   tf_http_archive(
@@ -68,7 +93,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
      sha256 = "5996380e3e8b981f55d1c8d58e709c00dbb4806ba367be75d0925a68cc2f6478",
      strip_prefix = "abseil-cpp-720c017e30339fd1786ce4aac68bc8559736e53f",
-     build_file = str(Label("//third_party:com_google_absl.BUILD")),
+     build_file = clean_dep("//third_party:com_google_absl.BUILD"),
   )
 
   tf_http_archive(
@@ -79,8 +104,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0cadb31a35b514bf2dfd6b5d38205da94ef326ec6908fc3fd7c269948467214f",
       strip_prefix = "eigen-eigen-2355b229ea4c",
-      build_file = str(Label("//third_party:eigen.BUILD")),
-      patch_file = str(Label("//third_party:eigen_fix_cuda_compilation.patch"))
+      build_file = clean_dep("//third_party:eigen.BUILD"),
+      patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch")
   )
 
   tf_http_archive(
@@ -93,7 +118,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           # remove the whitelist entry in third_party/repo.bzl.
           # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
       ],
-      build_file = str(Label("//:arm_compiler.BUILD")),
+      build_file = clean_dep("//:arm_compiler.BUILD"),
   )
 
   tf_http_archive(
@@ -104,7 +129,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce",
       strip_prefix = "libxsmm-1.8.1",
-      build_file = str(Label("//third_party:libxsmm.BUILD")),
+      build_file = clean_dep("//third_party:libxsmm.BUILD"),
   )
 
   tf_http_archive(
@@ -117,7 +142,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
       strip_prefix = "or-tools-253f7955c6a1fd805408fba2e42ac6d45b312d15/src",
-      build_file = str(Label("//third_party:ortools.BUILD")),
+      build_file = clean_dep("//third_party:ortools.BUILD"),
   )
 
   tf_http_archive(
@@ -149,7 +174,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0",
       strip_prefix = "farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45",
-      build_file = str(Label("//third_party:farmhash.BUILD")),
+      build_file = clean_dep("//third_party:farmhash.BUILD"),
   )
 
   tf_http_archive(
@@ -160,7 +185,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9",
       strip_prefix = "highwayhash-dfcb97ca4fe9277bf9dc1802dd979b071896453b",
-      build_file = str(Label("//third_party:highwayhash.BUILD")),
+      build_file = clean_dep("//third_party:highwayhash.BUILD"),
   )
 
   tf_http_archive(
@@ -171,7 +196,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324",
       strip_prefix = "nasm-2.12.02",
-      build_file = str(Label("//third_party:nasm.BUILD")),
+      build_file = clean_dep("//third_party:nasm.BUILD"),
   )
 
   tf_http_archive(
@@ -182,7 +207,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77",
       strip_prefix = "libjpeg-turbo-1.5.1",
-      build_file = str(Label("//third_party/jpeg:jpeg.BUILD")),
+      build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"),
   )
 
   tf_http_archive(
@@ -193,7 +218,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2",
       strip_prefix = "libpng-1.2.53",
-      build_file = str(Label("//third_party:png.BUILD")),
+      build_file = clean_dep("//third_party:png.BUILD"),
   )
 
   tf_http_archive(
@@ -204,7 +229,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
       strip_prefix = "sqlite-amalgamation-3200000",
-      build_file = str(Label("//third_party:sqlite.BUILD")),
+      build_file = clean_dep("//third_party:sqlite.BUILD"),
   )
 
   tf_http_archive(
@@ -215,7 +240,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1",
       strip_prefix = "giflib-5.1.4",
-      build_file = str(Label("//third_party:gif.BUILD")),
+      build_file = clean_dep("//third_party:gif.BUILD"),
   )
 
   tf_http_archive(
@@ -226,7 +251,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
       strip_prefix = "six-1.10.0",
-      build_file = str(Label("//third_party:six.BUILD")),
+      build_file = clean_dep("//third_party:six.BUILD"),
   )
 
   tf_http_archive(
@@ -237,7 +262,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d",
       strip_prefix = "astor-0.6.2",
-      build_file = str(Label("//third_party:astor.BUILD")),
+      build_file = clean_dep("//third_party:astor.BUILD"),
   )
 
   tf_http_archive(
@@ -248,7 +273,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "7068908321ecd2774f145193c4b34a11305bd104b4551b09273dfd1d6a374930",
       strip_prefix = "gast-0.2.0",
-      build_file = str(Label("//third_party:gast.BUILD")),
+      build_file = clean_dep("//third_party:gast.BUILD"),
   )
 
   tf_http_archive(
@@ -259,7 +284,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b",
       strip_prefix = "termcolor-1.1.0",
-      build_file = str(Label("//third_party:termcolor.BUILD")),
+      build_file = clean_dep("//third_party:termcolor.BUILD"),
   )
 
   tf_http_archive(
@@ -280,7 +305,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892",
       strip_prefix = "backports.weakref-1.0rc1/src",
-      build_file = str(Label("//third_party:backports_weakref.BUILD")),
+      build_file = clean_dep("//third_party:backports_weakref.BUILD"),
   )
 
   tf_http_archive(
@@ -291,7 +316,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
       strip_prefix = "codegen-1.0",
-      build_file = str(Label("//third_party:codegen.BUILD")),
+      build_file = clean_dep("//third_party:codegen.BUILD"),
   )
 
   filegroup_external(
@@ -376,7 +401,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
       ],
       strip_prefix = "pcre-8.39",
-      build_file = str(Label("//third_party:pcre.BUILD")),
+      build_file = clean_dep("//third_party:pcre.BUILD"),
   )
 
   tf_http_archive(
@@ -388,7 +413,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
       ],
       strip_prefix = "swig-3.0.8",
-      build_file = str(Label("//third_party:swig.BUILD")),
+      build_file = clean_dep("//third_party:swig.BUILD"),
   )
 
   tf_http_archive(
@@ -399,7 +424,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://curl.haxx.se/download/curl-7.49.1.tar.gz",
       ],
       strip_prefix = "curl-7.49.1",
-      build_file = str(Label("//third_party:curl.BUILD")),
+      build_file = clean_dep("//third_party:curl.BUILD"),
   )
 
   tf_http_archive(
@@ -420,7 +445,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
       ],
       strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3",
-      build_file = str(Label("//third_party:linenoise.BUILD")),
+      build_file = clean_dep("//third_party:linenoise.BUILD"),
   )
 
   # TODO(phawkins): currently, this rule uses an unofficial LLVM mirror.
@@ -433,7 +458,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
       strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
-      build_file = str(Label("//third_party/llvm:llvm.BUILD")),
+      build_file = clean_dep("//third_party/llvm:llvm.BUILD"),
   )
 
   tf_http_archive(
@@ -444,7 +469,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326",
       strip_prefix = "lmdb-LMDB_0.9.19/libraries/liblmdb",
-      build_file = str(Label("//third_party:lmdb.BUILD")),
+      build_file = clean_dep("//third_party:lmdb.BUILD"),
   )
 
   tf_http_archive(
@@ -455,7 +480,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2",
       strip_prefix = "jsoncpp-11086dd6a7eba04289944367ca82cea71299ed70",
-      build_file = str(Label("//third_party:jsoncpp.BUILD")),
+      build_file = clean_dep("//third_party:jsoncpp.BUILD"),
   )
 
   tf_http_archive(
@@ -476,7 +501,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
       strip_prefix = "zlib-1.2.8",
-      build_file = str(Label("//third_party:zlib.BUILD")),
+      build_file = clean_dep("//third_party:zlib.BUILD"),
   )
 
   tf_http_archive(
@@ -486,7 +511,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
       ],
       sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296",
-      build_file = str(Label("//third_party/fft2d:fft2d.BUILD")),
+      build_file = clean_dep("//third_party/fft2d:fft2d.BUILD"),
   )
 
   tf_http_archive(
@@ -497,7 +522,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
       strip_prefix = "snappy-1.1.4",
-      build_file = str(Label("//third_party:snappy.BUILD")),
+      build_file = clean_dep("//third_party:snappy.BUILD"),
   )
 
   tf_http_archive(
@@ -508,7 +533,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
       strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7",
-      build_file = str(Label("//third_party:nccl.BUILD")),
+      build_file = clean_dep("//third_party:nccl.BUILD"),
   )
 
   tf_http_archive(
@@ -519,8 +544,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "dd035d57c8f19b0b612dd6eefe6e5eebad76f506e302cccb7c2066f25a83585e",
       strip_prefix = "librdkafka-0.11.1",
-      build_file = str(Label("//third_party:kafka/BUILD")),
-      patch_file = str(Label("//third_party/kafka:config.patch")),
+      build_file = clean_dep("//third_party:kafka/BUILD"),
+      patch_file = clean_dep("//third_party/kafka:config.patch"),
   )
 
   tf_http_archive(
@@ -531,7 +556,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
       strip_prefix = "aws-sdk-cpp-1.3.15",
-      build_file = str(Label("//third_party:aws.BUILD")),
+      build_file = clean_dep("//third_party:aws.BUILD"),
   )
 
   java_import_external(
@@ -567,7 +592,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8",
       strip_prefix = "jemalloc-4.4.0",
-      build_file = str(Label("//third_party:jemalloc.BUILD")),
+      build_file = clean_dep("//third_party:jemalloc.BUILD"),
   )
 
   java_import_external(
@@ -613,7 +638,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4",
       strip_prefix = "pprof-c0fb62ec88c411cc91194465e54db2632845b650",
-      build_file = str(Label("//third_party:pprof.BUILD")),
+      build_file = clean_dep("//third_party:pprof.BUILD"),
   )
 
   tf_http_archive(
@@ -624,7 +649,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
       strip_prefix = "cub-1.8.0",
-      build_file = str(Label("//third_party:cub.BUILD")),
+      build_file = clean_dep("//third_party:cub.BUILD"),
   )
 
   tf_http_archive(
@@ -635,7 +660,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
       ],
       strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
-      build_file = str(Label("//third_party:cython.BUILD")),
+      build_file = clean_dep("//third_party:cython.BUILD"),
       delete = ["BUILD.bazel"],
   )
 
@@ -657,7 +682,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
           "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
       ],
-      build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
+      build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
   )
 
   tf_http_archive(
@@ -668,7 +693,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
           "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
       ],
-      build_file = str(Label("//third_party/flatbuffers:flatbuffers.BUILD")),
+      build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
   )
 
   tf_http_archive(
@@ -678,7 +703,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
       ],
-      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
+      build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
   )
 
   tf_http_archive(
@@ -688,7 +713,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip"
       ],
-      build_file = str(Label("//third_party:tflite_smartreply.BUILD")),
+      build_file = clean_dep("//third_party:tflite_smartreply.BUILD"),
   )
 
   ##############################################################################
@@ -752,7 +777,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # Needed by Protobuf
   native.bind(
       name = "python_headers",
-      actual = str(Label("//util/python:python_headers")),
+      actual = clean_dep("//util/python:python_headers"),
   )
 
   # Needed by Protobuf
diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD
index b27d341404..3262562bcc 100644
--- a/third_party/mkl/BUILD
+++ b/third_party/mkl/BUILD
@@ -1,7 +1,5 @@
 licenses(["notice"])  # 3-Clause BSD
 
-exports_files(["LICENSE"])
-
 config_setting(
     name = "using_mkl",
     values = {
@@ -10,17 +8,52 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "using_mkl_lnx_x64",
+    values = {
+        "cpu": "k8",
+        "define": "using_mkl=true",
+    },
+    visibility = ["//visibility:public"],
+)
+
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
 )
 
+filegroup(
+    name = "LICENSE",
+    visibility = ["//visibility:public"],
+    srcs = ["MKL_LICENSE"] + select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:LICENSE",
+        ]
+    })
+)
+
 cc_library(
     name = "intel_binary_blob",
-    srcs = if_mkl([
-        "@mkl//:libmklml_intel.so",
-        "@mkl//:libiomp5.so",
-    ]),
+
     visibility = ["//visibility:public"],
-    deps = ["@mkl//:mkl_headers"],
+    deps = select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:mkl_headers",
+            "@mkl_linux//:mkl_libs_linux",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:mkl_headers",
+            "@mkl_darwin//:mkl_libs_darwin",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:mkl_headers",
+            "@mkl_windows//:mkl_libs_windows",
+        ]
+    })
 )
diff --git a/third_party/mkl/LICENSE b/third_party/mkl/MKL_LICENSE
similarity index 100%
rename from third_party/mkl/LICENSE
rename to third_party/mkl/MKL_LICENSE
diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl
index 8b73ddabdd..53e02769da 100644
--- a/third_party/mkl/build_defs.bzl
+++ b/third_party/mkl/build_defs.bzl
@@ -24,6 +24,18 @@ def if_mkl(if_true, if_false = []):
         "//conditions:default": if_false
     })
 
+def if_mkl_lnx_x64(if_true, if_false = []):
+    """Shorthand for select()'ing on whether we're building with MKL.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with MKL enabled.  Otherwise, the select statement evaluates to if_false.
+
+    """
+    return select({
+        str(Label("//third_party/mkl:using_mkl_lnx_x64")): if_true,
+        "//conditions:default": if_false
+    })
+
 
 def _enable_local_mkl(repository_ctx):
   return _TF_MKL_ROOT in repository_ctx.os.environ
diff --git a/third_party/mkl/mkl.BUILD b/third_party/mkl/mkl.BUILD
index 8db97232e1..892221ec00 100644
--- a/third_party/mkl/mkl.BUILD
+++ b/third_party/mkl/mkl.BUILD
@@ -17,14 +17,29 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libmklml_intel.so",
-    srcs = ["lib/libmklml_intel.so"],
+cc_library(
+    name = "mkl_libs_linux",
+    srcs = [
+        "lib/libiomp5.so",
+        "lib/libmklml_intel.so"
+    ],
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libiomp5.so",
-    srcs = ["lib/libiomp5.so"],
+cc_library(
+    name = "mkl_libs_darwin",
+    srcs = [
+        "lib/libiomp5.dylib",
+        "lib/libmklml.dylib"
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "mkl_libs_windows",
+    srcs = [
+        "lib/libiomp5md.lib",
+        "lib/mklml.lib"
+    ],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From 39b7787275a407a65d94a47bcbbf1eac5b963ca5 Mon Sep 17 00:00:00 2001
From: Stanislaw Antol <antolstan@gmail.com>
Date: Fri, 23 Mar 2018 12:10:02 -0700
Subject: [PATCH 416/960] Add basic support for Maximum in TensorFlowLite.

---
 tensorflow/contrib/lite/kernels/BUILD         |  13 ++
 .../internal/reference/reference_ops.h        |  25 ++++
 tensorflow/contrib/lite/kernels/maximum.cc    | 106 +++++++++++++++
 .../contrib/lite/kernels/maximum_test.cc      |  81 ++++++++++++
 tensorflow/contrib/lite/kernels/register.cc   |   2 +
 tensorflow/contrib/lite/model.cc              |   3 +
 tensorflow/contrib/lite/nnapi_delegate.cc     |   1 +
 tensorflow/contrib/lite/schema/schema.fbs     |   5 +
 .../contrib/lite/schema/schema_generated.h    | 124 +++++++++++++++++-
 tensorflow/contrib/lite/testing/BUILD         |   1 +
 .../contrib/lite/testing/generate_examples.py |  36 +++++
 .../testing/generated_examples_zip_test.cc    |   2 +-
 .../contrib/lite/toco/tflite/operator.cc      |   2 +
 .../contrib/lite/toco/tflite/operator_test.cc |   2 +
 14 files changed, 396 insertions(+), 7 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/maximum.cc
 create mode 100644 tensorflow/contrib/lite/kernels/maximum_test.cc

diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 956bd35fe6..701993bc43 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -120,6 +120,7 @@ cc_library(
         "local_response_norm.cc",
         "lsh_projection.cc",
         "lstm.cc",
+        "maximum.cc",
         "mean.cc",
         "mul.cc",
         "pad.cc",
@@ -355,6 +356,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "maximum_test",
+    size = "small",
+    srcs = ["maximum_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "mean_test",
     size = "small",
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 53de21697b..d482dc9e32 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -404,6 +404,7 @@ inline void DepthToSpace(const T* input_data, const Dims<4>& input_dims,
           const int in_d =
               out_d + ((out_h % block_size) * block_size + out_w % block_size) *
                           output_depth;
+
           const int in_w = out_w / block_size;
           const int in_h = out_h / block_size;
           const int in_b = out_b;
@@ -3021,6 +3022,30 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+template <typename T>
+void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
+                       const T* input2_data, const Dims<4>& input2_dims,
+                       T* output_data, const Dims<4>& output_dims) {
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          auto out_idx = Offset(output_dims, c, x, y, b);
+          auto in1_idx = SubscriptToIndex(desc1, c, x, y, b);
+          auto in2_idx = SubscriptToIndex(desc2, c, x, y, b);
+          auto in1_val = input1_data[in1_idx];
+          auto in2_val = input2_data[in2_idx];
+          output_data[out_idx] = in1_val > in2_val ? in1_val : in2_val;
+        }
+      }
+    }
+  }
+}
+
 template <typename T1, typename T2, typename T3>
 void ArgMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims,
             T2* output_data, const Dims<4>& output_dims) {
diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum.cc
new file mode 100644
index 0000000000..9fdf2b47ea
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum.cc
@@ -0,0 +1,106 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <string.h>
+#include <vector>
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+#include "tensorflow/contrib/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace maximum {
+
+// This file has a reference implemenation of TFMaximum.
+enum KernelType {
+  kReference,
+};
+
+constexpr int kInputTensor1 = 0;
+constexpr int kInputTensor2 = 1;
+constexpr int kOutputTensor = 0;
+
+struct MaximumContext {
+  MaximumContext(TfLiteContext* context, TfLiteNode* node) {
+    input1 = GetInput(context, node, kInputTensor1);
+    input2 = GetInput(context, node, kInputTensor2);
+    output = GetOutput(context, node, kOutputTensor);
+  }
+  TfLiteTensor* input1;
+  TfLiteTensor* input2;
+  TfLiteTensor* output;
+};
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  MaximumContext op_context(context, node);
+  TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type);
+  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input2->dims);
+  op_context.output->type = op_context.input2->type;
+  return context->ResizeTensor(context, op_context.output, output_dims);
+}
+
+template <KernelType kernel_type>
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  MaximumContext op_context(context, node);
+
+#define TF_LITE_MAXIMUM(kernel_type, data_type)    \
+  kernel_type::TensorFlowMaximum<data_type>(       \
+      GetTensorData<data_type>(op_context.input1), \
+      GetTensorDims(op_context.input1),            \
+      GetTensorData<data_type>(op_context.input2), \
+      GetTensorDims(op_context.input2),            \
+      GetTensorData<data_type>(op_context.output), \
+      GetTensorDims(op_context.output))
+
+  if (kernel_type == kReference) {
+    switch (op_context.output->type) {
+      case kTfLiteFloat32:
+        TF_LITE_MAXIMUM(reference_ops, float);
+        break;
+      default:
+        context->ReportError(context,
+                             "Type %d is currently not supported by Maximum.",
+                             op_context.output->type);
+        return kTfLiteError;
+    }
+  } else {
+    context->ReportError(context,
+                         "Type %d is currently not supported by Maximum.",
+                         op_context.output->type);
+    return kTfLiteError;
+  }
+#undef TF_LITE_MAXIMUM
+  return kTfLiteOk;
+}
+
+}  // namespace maximum
+
+TfLiteRegistration* Register_MAXIMUM_REF() {
+  static TfLiteRegistration r = {nullptr, nullptr, maximum::Prepare,
+                                 maximum::Eval<maximum::kReference>};
+  return &r;
+}
+
+TfLiteRegistration* Register_MAXIMUM() { return Register_MAXIMUM_REF(); }
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc
new file mode 100644
index 0000000000..b3fd7d4e6f
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum_test.cc
@@ -0,0 +1,81 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class MaximumOpModel : public SingleOpModel {
+ public:
+  MaximumOpModel(const TensorData& input1, const TensorData& input2,
+                 const TensorType& output) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_MAXIMUM, BuiltinOptions_MaximumOptions,
+                 CreateMaximumOptions(builder_).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  template <class T>
+  void SetInput1(std::initializer_list<T> data) {
+    PopulateTensor(input1_, data);
+  }
+
+  template <class T>
+  void SetInput2(std::initializer_list<T> data) {
+    PopulateTensor(input2_, data);
+  }
+
+  template <class T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+TEST(MaximumOpTest, FloatTest) {
+  std::initializer_list<float> data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44};
+  std::initializer_list<float> data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43};
+  MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}},
+                   {TensorType_FLOAT32, {3, 1, 2}}, TensorType_FLOAT32);
+  m.SetInput1<float>(data1);
+  m.SetInput2<float>(data2);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2}));
+  EXPECT_THAT(
+      m.GetOutput<float>(),
+      ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43})));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index aea6f8d9d3..ec700e1e18 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -65,6 +65,7 @@ TfLiteRegistration* Register_STRIDED_SLICE();
 TfLiteRegistration* Register_EXP();
 TfLiteRegistration* Register_TOPK_V2();
 TfLiteRegistration* Register_LOG_SOFTMAX();
+TfLiteRegistration* Register_MAXIMUM();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -119,6 +120,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_EXP, Register_EXP());
   AddBuiltin(BuiltinOperator_TOPK_V2, Register_TOPK_V2());
   AddBuiltin(BuiltinOperator_LOG_SOFTMAX, Register_LOG_SOFTMAX());
+  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
 }
 
 TfLiteRegistration* BuiltinOpResolver::FindOp(
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 725f2838c5..187af8af06 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -574,6 +574,9 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
       builtin_data = reinterpret_cast<void*>(params);
       break;
     }
+    case BuiltinOperator_MAXIMUM: {
+      break;
+    }
     case BuiltinOperator_DELEGATE: {
       // TODO(ycling): Revisit when supporting saving delegated models.
       error_reporter->Report("DELEGATE op shouldn't exist in model.");
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index e631ffd845..e6da21664c 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -347,6 +347,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
       case tflite::BuiltinOperator_EXP:
       case tflite::BuiltinOperator_LOG_SOFTMAX:
       case tflite::BuiltinOperator_DELEGATE:
+      case tflite::BuiltinOperator_MAXIMUM:
         FATAL("Op code %d is currently not delegated to NNAPI", builtin);
         nn_op_type = -1;  // set to invalid
         break;
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 98ac0469d1..d97be553aa 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -129,6 +129,7 @@ enum BuiltinOperator : byte {
   // WARNING: Experimental interface, subject to change
   DELEGATE = 51,
   BIDIRECTIONAL_SEQUENCE_LSTM = 52,
+  MAXIMUM = 53,
 }
 
 // Options for the builtin operators.
@@ -169,6 +170,7 @@ union BuiltinOptions {
   TopKV2Options,
   SplitOptions,
   LogSoftmaxOptions,
+  MaximumOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -374,6 +376,9 @@ table StridedSliceOptions {
 table LogSoftmaxOptions {
 }
 
+table MaximumOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 99e1accaa7..b8f3e0bb84 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -139,6 +139,9 @@ struct StridedSliceOptionsT;
 struct LogSoftmaxOptions;
 struct LogSoftmaxOptionsT;
 
+struct MaximumOptions;
+struct MaximumOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -246,11 +249,12 @@ enum BuiltinOperator {
   BuiltinOperator_LOG_SOFTMAX = 50,
   BuiltinOperator_DELEGATE = 51,
   BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM = 52,
+  BuiltinOperator_MAXIMUM = 53,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM
+  BuiltinOperator_MAX = BuiltinOperator_MAXIMUM
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[50] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[51] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -301,7 +305,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[50] {
     BuiltinOperator_SPLIT,
     BuiltinOperator_LOG_SOFTMAX,
     BuiltinOperator_DELEGATE,
-    BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM
+    BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
+    BuiltinOperator_MAXIMUM
   };
   return values;
 }
@@ -361,6 +366,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "LOG_SOFTMAX",
     "DELEGATE",
     "BIDIRECTIONAL_SEQUENCE_LSTM",
+    "MAXIMUM",
     nullptr
   };
   return names;
@@ -409,11 +415,12 @@ enum BuiltinOptions {
   BuiltinOptions_TopKV2Options = 34,
   BuiltinOptions_SplitOptions = 35,
   BuiltinOptions_LogSoftmaxOptions = 36,
+  BuiltinOptions_MaximumOptions = 37,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_LogSoftmaxOptions
+  BuiltinOptions_MAX = BuiltinOptions_MaximumOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[37] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[38] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -451,7 +458,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[37] {
     BuiltinOptions_ExpOptions,
     BuiltinOptions_TopKV2Options,
     BuiltinOptions_SplitOptions,
-    BuiltinOptions_LogSoftmaxOptions
+    BuiltinOptions_LogSoftmaxOptions,
+    BuiltinOptions_MaximumOptions
   };
   return values;
 }
@@ -495,6 +503,7 @@ inline const char **EnumNamesBuiltinOptions() {
     "TopKV2Options",
     "SplitOptions",
     "LogSoftmaxOptions",
+    "MaximumOptions",
     nullptr
   };
   return names;
@@ -653,6 +662,10 @@ template<> struct BuiltinOptionsTraits<LogSoftmaxOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_LogSoftmaxOptions;
 };
 
+template<> struct BuiltinOptionsTraits<MaximumOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -972,6 +985,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_LogSoftmaxOptions ?
       reinterpret_cast<const LogSoftmaxOptionsT *>(value) : nullptr;
   }
+  MaximumOptionsT *AsMaximumOptions() {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<MaximumOptionsT *>(value) : nullptr;
+  }
+  const MaximumOptionsT *AsMaximumOptions() const {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<const MaximumOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -3635,6 +3656,46 @@ inline flatbuffers::Offset<LogSoftmaxOptions> CreateLogSoftmaxOptions(
 
 flatbuffers::Offset<LogSoftmaxOptions> CreateLogSoftmaxOptions(flatbuffers::FlatBufferBuilder &_fbb, const LogSoftmaxOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct MaximumOptionsT : public flatbuffers::NativeTable {
+  typedef MaximumOptions TableType;
+  MaximumOptionsT() {
+  }
+};
+
+struct MaximumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef MaximumOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  MaximumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<MaximumOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct MaximumOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit MaximumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  MaximumOptionsBuilder &operator=(const MaximumOptionsBuilder &);
+  flatbuffers::Offset<MaximumOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<MaximumOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  MaximumOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -3860,6 +3921,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const LogSoftmaxOptions *builtin_options_as_LogSoftmaxOptions() const {
     return builtin_options_type() == BuiltinOptions_LogSoftmaxOptions ? static_cast<const LogSoftmaxOptions *>(builtin_options()) : nullptr;
   }
+  const MaximumOptions *builtin_options_as_MaximumOptions() const {
+    return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast<const MaximumOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -4030,6 +4094,10 @@ template<> inline const LogSoftmaxOptions *Operator::builtin_options_as<LogSoftm
   return builtin_options_as_LogSoftmaxOptions();
 }
 
+template<> inline const MaximumOptions *Operator::builtin_options_as<MaximumOptions>() const {
+  return builtin_options_as_MaximumOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -5512,6 +5580,29 @@ inline flatbuffers::Offset<LogSoftmaxOptions> CreateLogSoftmaxOptions(flatbuffer
       _fbb);
 }
 
+inline MaximumOptionsT *MaximumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new MaximumOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void MaximumOptions::UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<MaximumOptions> MaximumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateMaximumOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateMaximumOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -5836,6 +5927,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const LogSoftmaxOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -5998,6 +6093,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const LogSoftmaxOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -6148,6 +6247,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const LogSoftmaxOptionsT *>(value);
       return CreateLogSoftmaxOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptionsT *>(value);
+      return CreateMaximumOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -6298,6 +6401,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new LogSoftmaxOptionsT(*reinterpret_cast<LogSoftmaxOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      value = new MaximumOptionsT(*reinterpret_cast<MaximumOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -6485,6 +6592,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<MaximumOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 83b9e21427..c8d6e4b43e 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -36,6 +36,7 @@ gen_zipped_test_files(
         "log_softmax.zip",
         "lstm.zip",
         "max_pool.zip",
+        "maximum.zip",
         "mean.zip",
         "mul.zip",
         "pad.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 5488b71fcf..e714f5299d 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -814,6 +814,41 @@ def make_log_softmax_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_maximum_tests(zip_path):
+  """Make a set of tests to do maximum."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the maximum op testing graph."""
+    input_tensor_1 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_1",
+        shape=parameters["input_shape_1"])
+    input_tensor_2 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_2",
+        shape=parameters["input_shape_2"])
+
+    out = tf.maximum(input_tensor_1, input_tensor_2)
+    return [input_tensor_1, input_tensor_2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_1"]),
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_2"])
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_binary_op_tests_func(binary_operator):
   """Return a function that does a test on a binary operator."""
   return lambda zip_path: make_binary_op_tests(zip_path, binary_operator)
@@ -1929,6 +1964,7 @@ def main(unused_args):
         "exp.zip": make_exp_tests,
         "log_softmax.zip": make_log_softmax_tests,
         "lstm.zip": make_lstm_tests,
+        "maximum.zip": make_maximum_tests,
     }
     out = FLAGS.zip_to_output
     bin_path = FLAGS.toco
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 86606d1239..f0e5dcc482 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -254,6 +254,7 @@ INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
+INSTANTIATE_TESTS(maximum)
 INSTANTIATE_TESTS(max_pool)
 INSTANTIATE_TESTS(mul)
 INSTANTIATE_TESTS(pad)
@@ -269,7 +270,6 @@ INSTANTIATE_TESTS(sub)
 INSTANTIATE_TESTS(split)
 INSTANTIATE_TESTS(div)
 INSTANTIATE_TESTS(transpose)
-INSTANTIATE_TESTS(lstm)
 INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(squeeze)
 INSTANTIATE_TESTS(strided_slice)
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index f2cc4ef71f..1d57d2dd7e 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -861,6 +861,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
   ops.emplace_back(new SimpleOperator<ExpOperator>("EXP", OperatorType::kExp));
   ops.emplace_back(new SimpleOperator<LogSoftmaxOperator>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
+  ops.emplace_back(new SimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum));
 
   return ops;
 }
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index 9c19f8d464..f7a213ecfc 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -109,6 +109,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<ExpOperator>("EXP", OperatorType::kExp);
   CheckSimpleOperator<LogSoftmaxOperator>("LOG_SOFTMAX",
                                           OperatorType::kLogSoftmax);
+  CheckSimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
-- 
GitLab


From 8740ef83c660920a17ce259acaee28ec80978aa7 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 23 Mar 2018 14:11:45 -0700
Subject: [PATCH 417/960] Fix broken test

---
 tensorflow/contrib/lite/python/lite.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index 35d224924e..51c302b8fb 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -27,7 +27,7 @@ from __future__ import division
 from __future__ import print_function
 import os
 import subprocess
-import tempfile
+import tempfile as _tempfile
 
 # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import convert_op_hints_to_stubs
@@ -102,10 +102,10 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
     return _toco_python.TocoConvert(
         model_flags_str, toco_flags_str, input_data_str)
 
-  with tempfile.NamedTemporaryFile() as fp_toco, \
-           tempfile.NamedTemporaryFile() as fp_model, \
-           tempfile.NamedTemporaryFile() as fp_input, \
-           tempfile.NamedTemporaryFile() as fp_output:
+  with _tempfile.NamedTemporaryFile() as fp_toco, \
+           _tempfile.NamedTemporaryFile() as fp_model, \
+           _tempfile.NamedTemporaryFile() as fp_input, \
+           _tempfile.NamedTemporaryFile() as fp_output:
     fp_model.write(model_flags_str)
     fp_toco.write(toco_flags_str)
     fp_input.write(input_data_str)
-- 
GitLab


From ad61950fb5db57aa5a4089203a1a4bf48df8c5f4 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Fri, 23 Mar 2018 14:12:34 -0700
Subject: [PATCH 418/960] Instead of depending on ctest to be in PATH, directly
 reference the binary. (#17964)

PiperOrigin-RevId: 190137278
---
 tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
index b537192a94..97829892b1 100644
--- a/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
+++ b/tensorflow/tools/ci_build/windows/gpu/cmake/run_py.bat
@@ -28,6 +28,9 @@ IF DEFINED TF_NIGHTLY (ECHO TF_NIGHTLY is set to %TF_NIGHTLY%) ELSE (SET TF_NIGH
 :: Set pip binary location. Do not override if it is set already.
 IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Program Files\Anaconda3\Scripts\pip.exe")
 
+:: Set ctest binary location.
+IF DEFINED CTEST_EXE (ECHO CTEST_EXE is set to %CTEST_EXE%) ELSE (SET CTEST_EXE="C:\Program Files\cmake\bin\ctest.exe")
+
 :: Run the CMAKE build to build the pip package.
 CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\gpu\cmake\run_build.bat
 if %errorlevel% neq 0 exit /b %errorlevel%
@@ -47,4 +50,4 @@ if %errorlevel% neq 0 exit /b %errorlevel%
 
 :: Run all python tests if the installation succeeded.
 echo Running tests...
-ctest -C Release --output-on-failure --jobs 1
+%CTEST_EXE% -C Release --output-on-failure --jobs 1
-- 
GitLab


From b8a364cc3e0c1e3ba155ab9ac8f35d6178779e74 Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 23 Mar 2018 14:48:22 -0700
Subject: [PATCH 419/960] Fix broken test

---
 tensorflow/contrib/lite/python/lite.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index 51c302b8fb..ed6dd036f9 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -25,8 +25,8 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import os
-import subprocess
+import os as _os
+import subprocess as _subprocess
 import tempfile as _tempfile
 
 # pylint: disable=unused-import
@@ -74,7 +74,7 @@ else:
   _toco_from_proto_bin = _resource_loader.get_path_to_datafile(
       "../toco/python/toco_from_protos")
 
-if _toco_from_proto_bin and not os.path.exists(_toco_from_proto_bin):
+if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin):
   _toco_from_proto_bin = "toco_from_protos"
 
 
@@ -118,11 +118,11 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
         fp_output.name
     ]
     cmdline = " ".join(cmd)
-    proc = subprocess.Popen(
+    proc = _subprocess.Popen(
         cmdline,
         shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
+        stdout=_subprocess.PIPE,
+        stderr=_subprocess.STDOUT,
         close_fds=True)
     stdout, stderr = proc.communicate()
     exitcode = proc.returncode
-- 
GitLab


From 3af26a4b5780832ec7730212d0e6c0237743786a Mon Sep 17 00:00:00 2001
From: Maximilian Mitchell <max@maxis.me>
Date: Fri, 23 Mar 2018 22:28:21 +0000
Subject: [PATCH 420/960] Replace depreciated function variable_op_scope with
 variable_scope. (#17938)

* Replace depreciated function variable_op_scope with variable_scope.

* Fix parameters

* Remove excess spaces

* Format to reduce 80 char line

* Placate the linter
---
 tensorflow/contrib/learn/python/learn/estimators/linear.py   | 4 ++--
 tensorflow/contrib/linear_optimizer/python/sdca_estimator.py | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index 64d7ecc68e..70b70af98c 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -243,8 +243,8 @@ def sdca_model_fn(features, labels, mode, params):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(
-      features.values(), parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
index 05794a42c5..d4e54c82f9 100644
--- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
+++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
@@ -140,8 +140,8 @@ def sdca_model_fn(features, labels, mode, params, config=None):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(features.values(),
-                                        parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
-- 
GitLab


From 94d6690edcdd930c52350625ea2b23fc3eecdbee Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 23 Mar 2018 22:33:19 +0000
Subject: [PATCH 421/960] Fix incorrect rendering in community/documentation.md

This fix tries to fix the incorrect rendering in community/documentation.md.
The issue was caused by the escaping of "```". As could be seen
in the docs, there are "```" (backticks) inside the code block and
the code block was surrended by "```" (backticks) as well. The
result is that backticks confused the inteprater of markdown and
messed the documentation.

This fix uses indent four spaces to wrap the code blocks so that
backticks inside could be rendered correctly.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../docs_src/community/documentation.md       | 38 +++++++++----------
 1 file changed, 18 insertions(+), 20 deletions(-)

diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md
index 003e0a25ec..6f2107ef40 100644
--- a/tensorflow/docs_src/community/documentation.md
+++ b/tensorflow/docs_src/community/documentation.md
@@ -477,31 +477,29 @@ should use Markdown in the docstring.
 
 Here's a simple example:
 
-```python
-def foo(x, y, name="bar"):
-  """Computes foo.
+    def foo(x, y, name="bar"):
+      """Computes foo.
 
-  Given two 1-D tensors `x` and `y`, this operation computes the foo.
+      Given two 1-D tensors `x` and `y`, this operation computes the foo.
 
-  Example:
+      Example:
 
-  ```
-  # x is [1, 1]
-  # y is [2, 2]
-  tf.foo(x, y) ==> [3, 3]
-  ```
-  Args:
-    x: A `Tensor` of type `int32`.
-    y: A `Tensor` of type `int32`.
-    name: A name for the operation (optional).
+      ```
+      # x is [1, 1]
+      # y is [2, 2]
+      tf.foo(x, y) ==> [3, 3]
+      ```
+      Args:
+        x: A `Tensor` of type `int32`.
+        y: A `Tensor` of type `int32`.
+        name: A name for the operation (optional).
 
-  Returns:
-    A `Tensor` of type `int32` that is the foo of `x` and `y`.
+      Returns:
+        A `Tensor` of type `int32` that is the foo of `x` and `y`.
 
-  Raises:
-    ValueError: If `x` or `y` are not of type `int32`.
-  """
-```
+      Raises:
+        ValueError: If `x` or `y` are not of type `int32`.
+      """
 
 ## Description of the docstring sections
 
-- 
GitLab


From 96c4ee44fafa5a428ea4983bb0d028b52d0d5c62 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Sat, 24 Mar 2018 07:37:06 -0700
Subject: [PATCH 422/960] Add kernels/random_op.cc to contrib/makefile

Fixes #17666
---
 tensorflow/contrib/makefile/tf_op_files.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 5a812af4e9..7a7683c953 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -258,6 +258,7 @@ tensorflow/core/kernels/requantize.cc
 tensorflow/core/kernels/remote_fused_graph_execute_op.cc
 tensorflow/core/kernels/remote_fused_graph_execute_utils.cc
 tensorflow/core/kernels/batch_matmul_op_real.cc
+tensorflow/core/kernels/random_op.cc
 tensorflow/core/ops/training_ops.cc
 tensorflow/core/ops/string_ops.cc
 tensorflow/core/ops/state_ops.cc
-- 
GitLab


From 1629161796bccd200e1accd7a83e0485d639cc8d Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Sun, 25 Mar 2018 02:09:12 +0800
Subject: [PATCH 423/960] Fix tf.svd docs issue (#17960)

This fix tries to fix tf.svd docs issue where "````" (four backticks)
were used instead of three backticks. That caused the docs to render
incorrectly.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/ops/linalg_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 5b4fb4f7c8..170861b43f 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -429,7 +429,7 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
   u, s, v_adj = np.linalg.svd(a, full_matrices=False)
   np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj))
   # tf_a_approx and np_a_approx should be numerically close.
-  ````
+  ```
   @end_compatibility
   """
   s, u, v = gen_linalg_ops.svd(
-- 
GitLab


From 5c098710fd63757f2d1a129d1c5f9d11d517fc22 Mon Sep 17 00:00:00 2001
From: naveen marri <naveenkumarmarri6014@gmail.com>
Date: Sat, 24 Mar 2018 14:09:36 -0400
Subject: [PATCH 424/960] BF:Fix for rendering documentation (#17972)

-- 
GitLab


From 80c962a2256f5e1be26b14c470d698bd395c0382 Mon Sep 17 00:00:00 2001
From: brett koonce <koonce@hello.com>
Date: Sat, 24 Mar 2018 19:47:34 -0700
Subject: [PATCH 425/960] contrib/boosted_trees: minor spelling tweaks (#17732)

---
 .../contrib/boosted_trees/kernels/quantile_ops.cc      |  2 +-
 .../contrib/boosted_trees/lib/utils/batch_features.cc  |  2 +-
 .../boosted_trees/lib/utils/batch_features_test.cc     |  2 +-
 .../contrib/boosted_trees/lib/utils/dropout_utils.cc   |  2 +-
 .../contrib/boosted_trees/lib/utils/dropout_utils.h    |  2 +-
 .../lib/utils/sparse_column_iterable_test.cc           |  2 +-
 .../contrib/boosted_trees/proto/tree_config.proto      |  2 +-
 .../python/kernel_tests/prediction_ops_test.py         | 10 +++++-----
 .../python/kernel_tests/quantile_ops_test.py           |  2 +-
 .../contrib/boosted_trees/python/ops/quantile_ops.py   |  2 +-
 10 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
index 0f4c2298f5..0b28f81e7c 100644
--- a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
@@ -253,7 +253,7 @@ class CreateQuantileAccumulatorOp : public OpKernel {
  private:
   float epsilon_;
   int32 num_quantiles_;
-  // An upperbound on the number of enteries that the summaries might have
+  // An upper bound on the number of entries that the summaries might have
   // for a feature.
   int64 max_elements_;
   bool generate_quantiles_;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
index cf4f9a097a..35b059f349 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
@@ -54,7 +54,7 @@ Status BatchFeatures::Initialize(
     TF_CHECK_AND_RETURN_IF_ERROR(
         dense_float_feature.dim_size(1) == 1,
         errors::InvalidArgument(
-            "Dense float features may not be multi-valent: dim_size(1) = ",
+            "Dense float features may not be multivalent: dim_size(1) = ",
             dense_float_feature.dim_size(1)));
     dense_float_feature_columns_.emplace_back(dense_float_feature);
   }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
index 609519e8b1..cfe9101e74 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
@@ -59,7 +59,7 @@ TEST_F(BatchFeaturesTest, DenseFloatFeatures_Multivalent) {
   BatchFeatures batch_features(1);
   auto dense_vec = AsTensor<float>({3.0f, 7.0f}, {1, 2});
   auto expected_error = InvalidArgument(
-      "Dense float features may not be multi-valent: dim_size(1) = 2");
+      "Dense float features may not be multivalent: dim_size(1) = 2");
   EXPECT_EQ(expected_error,
             batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {}));
 }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
index db34db998a..ce67db797d 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
@@ -54,7 +54,7 @@ Status DropoutUtils::DropOutTrees(
   if (probability_of_skipping_dropout < 0 ||
       probability_of_skipping_dropout > 1) {
     return errors::InvalidArgument(
-        "Probability of skiping dropout must be in [0,1] range");
+        "Probability of skipping dropout must be in [0,1] range");
   }
   const auto num_trees = weights.size();
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
index 928bfbfe5c..77c16da541 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
@@ -66,7 +66,7 @@ class DropoutUtils {
       // Current weights and num_updates will be updated as a result of this
       // func
       std::vector<float>* current_weights,
-      // How many weight assignements have been done for each tree already.
+      // How many weight assignments have been done for each tree already.
       std::vector<int32>* num_updates);
 };
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
index 0138aae3db..cc7604745e 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
@@ -34,7 +34,7 @@ TEST_F(SparseColumnIterableTest, Empty) {
 }
 
 TEST_F(SparseColumnIterableTest, Iterate) {
-  // 8 examples having 7 sparse features with the 3rd and 7th multi-valent.
+  // 8 examples having 7 sparse features with the 3rd and 7th multivalent.
   // This can be visualized like the following:
   // Instance | Sparse |
   // 0        |  x     |
diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
index 4407c4d981..81411aa84a 100644
--- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto
+++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
@@ -53,7 +53,7 @@ message DenseFloatBinarySplit {
   // Float feature column and split threshold describing
   // the rule feature <= threshold.
   int32 feature_column = 1;
-  // If feature column is multivalent, this holds the index of the dimensiong
+  // If feature column is multivalent, this holds the index of the dimension
   // for the split. Defaults to 0.
   int32 dimension_id = 5;
   float threshold = 2;
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
index c1acf35160..cf55759aaa 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
@@ -120,8 +120,8 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     """Sets up the prediction tests.
 
     Create a batch of two examples having one dense float, two sparse float
-    single valued, one sparse float multidimensionl and one sparse int features.
-    The data looks like the following:
+    single valued, one sparse float multidimensional and one sparse int
+    features.  The data looks like the following:
     | Instance | Dense0 | SparseF0 | SparseF1 | SparseI0 | SparseM
     | 0        |  7     |    -3    |          |    9,1   | __, 5.0
     | 1        | -2     |          | 4        |          |  3, ___
@@ -810,7 +810,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     # building. This tree should never be dropped.
     num_trees = 10
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 10 trees with some weights.
       for i in range(0, num_trees):
@@ -951,7 +951,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testDropOutZeroProb(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 1000 trees with some weights.
       for i in range(0, 999):
@@ -994,7 +994,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testAveragingAllTrees(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       adjusted_tree_ensemble_config = (
           tree_config_pb2.DecisionTreeEnsembleConfig())
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
index 81f58de28c..074623699d 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
@@ -482,7 +482,7 @@ class QuantilesOpTest(test_util.TensorFlowTestCase):
     """Sets up the quantile op tests.
 
     Create a batch of 4 examples having 2 dense and 4 sparse features.
-    Forth sparse feature is multivalent (3 dimensional)
+    Fourth sparse feature is multivalent (3 dimensional)
     The data looks like this
     | Instance | Dense 0 | Dense 1 | Sparse 0 | Sparse 1 |Sparse 2| SparseM
     | 0        |   -0.1  |  -1     |   -2     |   0.1    |        |_ ,1,_
diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
index 97d57e8b23..1b184d296b 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
@@ -184,7 +184,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
     """Finalizes quantile summary stream and resets it for next iteration.
 
     Args:
-      stamp_token: Exepcted current token.
+      stamp_token: Expected current token.
       next_stamp_token: Next value for the token.
     Returns:
       A list of quantiles or approximate boundaries.
-- 
GitLab


From e5456bbd0ac1d80f0dae4e54407a5001e0fbfba6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 09:15:58 -0700
Subject: [PATCH 426/960] Merged commit includes the following changes:
 190228434  by A. Unique TensorFlower:

    internal change only

--
190215102  by A. Unique TensorFlower:

    Remove placement strategy from Fisher estimator and add it in as Mixin.
    A new Kfac Optimizer sub class which runs covariance and inverse ops periodically. This should deprecate lazykfac.

--
190211603  by A. Unique TensorFlower:

    Internal only

--

PiperOrigin-RevId: 190228434
---
 .../python/kernel_tests/estimator_test.py     | 140 +++++---
 tensorflow/contrib/kfac/python/ops/BUILD      |   2 +
 .../contrib/kfac/python/ops/estimator.py      | 314 +++++++-----------
 .../contrib/kfac/python/ops/optimizer.py      | 157 +++------
 .../contrib/kfac/python/ops/placement.py      | 167 ++++++++++
 5 files changed, 410 insertions(+), 370 deletions(-)
 create mode 100644 tensorflow/contrib/kfac/python/ops/placement.py

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
index 30c5404e03..f22dbcf215 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/estimator_test.py
@@ -23,7 +23,6 @@ import numpy as np
 from tensorflow.contrib.kfac.python.ops import estimator
 from tensorflow.contrib.kfac.python.ops import layer_collection as lc
 from tensorflow.contrib.kfac.python.ops import utils
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -40,30 +39,6 @@ from tensorflow.python.training import training_util
 _ALL_ESTIMATION_MODES = ["gradients", "empirical", "curvature_prop", "exact"]
 
 
-class DeviceContextGeneratorTest(test.TestCase):
-
-  def testNoDevice(self):
-    device_context_generator = estimator._DeviceContextGenerator(None)
-    with ops.device("/device:CPU:0"):  # This is what will be used
-      with device_context_generator():  # Does nothing
-        a = constant_op.constant([2.0], name="a")
-    self.assertEqual("/device:CPU:0", a.op.device)
-
-  def testTwoDevices(self):
-    device_context_generator = estimator._DeviceContextGenerator(
-        ["/device:GPU:0", "/device:GPU:1"])
-    with ops.device("/device:CPU:0"):  # Will be over-ridden by the inner scopes
-      with device_context_generator():
-        a = constant_op.constant([2.0], name="a")
-      with device_context_generator():
-        b = constant_op.constant([2.0], name="b")
-      with device_context_generator():
-        c = constant_op.constant([2.0], name="c")
-    self.assertEqual("/device:GPU:0", a.op.device)
-    self.assertEqual("/device:GPU:1", b.op.device)
-    self.assertEqual("/device:GPU:0", c.op.device)
-
-
 class EstimatorTest(test.TestCase):
 
   def setUp(self):
@@ -90,68 +65,98 @@ class EstimatorTest(test.TestCase):
   def testEstimatorInitManualRegistration(self):
     with self._graph.as_default():
       # We should be able to build an estimator for only the registered vars.
-      estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                self.layer_collection)
+      estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection
+      )
 
       # Check that we throw an error if we try to build an estimator for vars
       # that were not manually registered.
       with self.assertRaises(ValueError):
-        est = estimator.FisherEstimator([self.weights, self.bias], 0.1, 0.2,
-                                        self.layer_collection)
+        est = estimator.FisherEstimatorRoundRobin(
+            variables=[self.weights, self.bias],
+            cov_ema_decay=0.1,
+            damping=0.2,
+            layer_collection=self.layer_collection
+        )
         est.make_ops_and_vars()
 
       # Check that we throw an error if we don't include registered variables,
       # i.e. self.weights
       with self.assertRaises(ValueError):
-        est = estimator.FisherEstimator([], 0.1, 0.2, self.layer_collection)
+        est = estimator.FisherEstimatorRoundRobin(
+            variables=[],
+            cov_ema_decay=0.1,
+            damping=0.2,
+            layer_collection=self.layer_collection)
         est.make_ops_and_vars()
 
   @test.mock.patch.object(utils.SubGraph, "variable_uses", return_value=42)
   def testVariableWrongNumberOfUses(self, mock_uses):
     with self.assertRaises(ValueError):
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection)
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection)
       est.make_ops_and_vars()
 
   def testInvalidEstimationMode(self):
     with self.assertRaises(ValueError):
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection,
-                                      estimation_mode="not_a_real_mode")
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection,
+          estimation_mode="not_a_real_mode")
       est.make_ops_and_vars()
 
   def testGradientsModeBuild(self):
     with self._graph.as_default():
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection,
-                                      estimation_mode="gradients")
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection,
+          estimation_mode="gradients")
       est.make_ops_and_vars()
 
   def testEmpiricalModeBuild(self):
     with self._graph.as_default():
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection,
-                                      estimation_mode="empirical")
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection,
+          estimation_mode="empirical")
       est.make_ops_and_vars()
 
   def testCurvaturePropModeBuild(self):
     with self._graph.as_default():
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection,
-                                      estimation_mode="curvature_prop")
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection,
+          estimation_mode="curvature_prop")
       est.make_ops_and_vars()
 
   def testExactModeBuild(self):
     with self._graph.as_default():
-      est = estimator.FisherEstimator([self.weights], 0.1, 0.2,
-                                      self.layer_collection,
-                                      estimation_mode="exact")
+      est = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          cov_ema_decay=0.1,
+          damping=0.2,
+          layer_collection=self.layer_collection,
+          estimation_mode="exact")
       est.make_ops_and_vars()
 
   def test_cov_update_thunks(self):
     """Ensures covariance update ops run once per global_step."""
     with self._graph.as_default(), self.test_session() as sess:
-      fisher_estimator = estimator.FisherEstimator(
+      fisher_estimator = estimator.FisherEstimatorRoundRobin(
           variables=[self.weights],
           layer_collection=self.layer_collection,
           damping=0.2,
@@ -159,8 +164,8 @@ class EstimatorTest(test.TestCase):
 
       # Construct an op that executes one covariance update per step.
       global_step = training_util.get_or_create_global_step()
-      (cov_variable_thunks, cov_update_op_thunks,
-       _, _) = fisher_estimator.create_ops_and_vars_thunks()
+      (cov_variable_thunks, cov_update_op_thunks, _,
+       _) = fisher_estimator.create_ops_and_vars_thunks()
       for thunk in cov_variable_thunks:
         thunk()
       cov_matrices = [
@@ -198,10 +203,43 @@ class EstimatorTest(test.TestCase):
         sess.run(cov_update_op)
         sess.run(increment_global_step)
 
+  def test_round_robin_placement(self):
+    """Check if the ops and variables are placed on devices correctly."""
+    with self._graph.as_default():
+      fisher_estimator = estimator.FisherEstimatorRoundRobin(
+          variables=[self.weights],
+          layer_collection=self.layer_collection,
+          damping=0.2,
+          cov_ema_decay=0.0,
+          cov_devices=["/cpu:{}".format(i) for i in range(2)],
+          inv_devices=["/cpu:{}".format(i) for i in range(2)])
+
+      # Construct an op that executes one covariance update per step.
+      (cov_update_ops, _, inv_update_ops, _, _,
+       _) = fisher_estimator.make_ops_and_vars(scope="test")
+      self.assertEqual(cov_update_ops[0].device, "/device:CPU:0")
+      self.assertEqual(cov_update_ops[1].device, "/device:CPU:1")
+      self.assertEqual(inv_update_ops[0].device, "/device:CPU:0")
+      self.assertEqual(inv_update_ops[1].device, "/device:CPU:1")
+      cov_matrices = [
+          fisher_factor.get_cov()
+          for fisher_factor in self.layer_collection.get_factors()
+      ]
+      inv_matrices = [
+          matrix
+          for fisher_factor in self.layer_collection.get_factors()
+          for matrix in fisher_factor._matpower_by_exp_and_damping.values()
+      ]
+      self.assertEqual(cov_matrices[0].device, "/device:CPU:0")
+      self.assertEqual(cov_matrices[1].device, "/device:CPU:1")
+      # Inverse matrices need to be explicitly placed.
+      self.assertEqual(inv_matrices[0].device, "")
+      self.assertEqual(inv_matrices[1].device, "")
+
   def test_inv_update_thunks(self):
     """Ensures inverse update ops run once per global_step."""
     with self._graph.as_default(), self.test_session() as sess:
-      fisher_estimator = estimator.FisherEstimator(
+      fisher_estimator = estimator.FisherEstimatorRoundRobin(
           variables=[self.weights],
           layer_collection=self.layer_collection,
           damping=0.2,
diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD
index c26230c2a8..d721ad08af 100644
--- a/tensorflow/contrib/kfac/python/ops/BUILD
+++ b/tensorflow/contrib/kfac/python/ops/BUILD
@@ -171,6 +171,7 @@ py_library(
     name = "fisher_estimator",
     srcs = [
         "estimator.py",
+        "placement.py",
     ],
     srcs_version = "PY2AND3",
     deps = [
@@ -180,6 +181,7 @@ py_library(
         "//tensorflow/python:gradients",
         "//tensorflow/python:util",
         "//third_party/py/numpy",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/contrib/kfac/python/ops/estimator.py b/tensorflow/contrib/kfac/python/ops/estimator.py
index 64755be65c..ced1110676 100644
--- a/tensorflow/contrib/kfac/python/ops/estimator.py
+++ b/tensorflow/contrib/kfac/python/ops/estimator.py
@@ -18,11 +18,11 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import contextlib
-import itertools
-
+import abc
 import numpy as np
+import six
 
+from tensorflow.contrib.kfac.python.ops import placement
 from tensorflow.contrib.kfac.python.ops import utils
 from tensorflow.python.framework import ops as tf_ops
 from tensorflow.python.ops import control_flow_ops
@@ -31,63 +31,46 @@ from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
 
 
-class _DeviceContextGenerator(object):
-  """Class for generating device contexts in a round-robin fashion."""
-
-  def __init__(self, devices):
-    """Creates a _DeviceContextGenerator object.
-
-    Example usage:
+# The linter is confused.
+# pylint: disable=abstract-class-instantiated
+def make_fisher_estimator(placement_strategy=None, **kwargs):
+  """Creates Fisher estimator instances based on the placement strategy.
 
-    ```python
-    dcg = _DeviceContextGenerator(['/gpu:0', 'gpu:1'])
-    with dcg():
-      # All operations in this context will be placed on GPU 0
-      ...
-    with dcg():
-      # All operations in this context will be placed on GPU 1
-      ...
-    ```
-
-    Args:
-      devices: An iterable of device strings (or None). Successive calls to
-          __call__ will give contexts which place devices on these devices in
-          a round-robin fashion.
-    """
-    self._cycle = None if devices is None else itertools.cycle(devices)
+  For example if the `placement_strategy` is 'round_robin' then
+  `FisherEstimatorRoundRobin` instance is returned.
 
-  @contextlib.contextmanager
-  def __call__(self):
-    """Returns a context manager specifying the default device."""
-    if self._cycle is None:
-      yield
-    else:
-      with tf_ops.device(next(self._cycle)):
-        yield
+  Args:
+    placement_strategy: `string`, Strategy to be used for placing covariance
+      variables, covariance ops and inverse ops. Check
+      `placement.FisherEstimatorRoundRobin` for a concrete example.
+   **kwargs: Arguments to be passed into `FisherEstimator` class initializer.
 
+  Returns:
+    An instance of class which inherits from `FisherEstimator` and the mixin
+    which implements specific placement strategy. See,
+    `FisherEstimatorRoundRobin` which inherits from `FisherEstimator` and
+    `RoundRobinPlacementMixin`.
 
-def _make_thunk_on_device(func, device):
-  def thunk():
-    with tf_ops.device(device):
-      return func()
-  return thunk
+  Raises:
+    ValueError: If the `placement_strategy` is not equal to 'round_robin'.
+  """
+  if placement_strategy in [None, "round_robin"]:
+    return FisherEstimatorRoundRobin(**kwargs)
+  else:
+    raise ValueError("Unimplemented vars and ops placement strategy : %s",
+                     placement_strategy)
+# pylint: enable=abstract-class-instantiated
 
 
+@six.add_metaclass(abc.ABCMeta)
 class FisherEstimator(object):
   """Fisher estimator class supporting various approximations of the Fisher.
 
-  Attributes:
-    cov_update_thunks: list of no-arg functions. Executing a function adds
-      covariance update ops for a single FisherFactor to the graph.
-    cov_update_ops: List of Ops. Running an op updates covariance matrices for a
-      single FisherFactor.
-    cov_update_op: Op. Running updates covariance matrices for all
-      FisherFactors.
-    inv_update_thunks: list of no-arg functions.  Executing a function adds
-      inverse update ops for a single FisherFactor to the graph.
-    inv_update_ops: List of Ops. Running an op updates inverse matrices for a
-      single FisherFactor.
-    inv_update_op: Op. Running updates inverse matrices for all FisherFactors.
+  This is an abstract base class which does not implement a strategy for
+  placing covariance variables, covariance update ops and inverse update ops.
+  The placement strategies are implemented in `placement.py`. See
+  `FisherEstimatorRoundRobin` for example of a concrete subclass with
+  a round-robin placement strategy.
   """
 
   def __init__(self,
@@ -184,6 +167,77 @@ class FisherEstimator(object):
   def name(self):
     return self._name
 
+  @abc.abstractmethod
+  def make_ops_and_vars(self, scope=None):
+    """Make ops and vars with a specific placement strategy.
+
+    For each factor, all of that factor's cov variables and their associated
+    update ops will be placed on a particular device.  For example in case of
+    round robin placement a new device is chosen for each factor by cycling
+    through list of devices in the cov_devices argument. If cov_devices is None
+    then no explicit device placement occurs.
+
+    An analogous strategy is followed for inverse update ops, with the list of
+    devices being given by the inv_devices argument.
+
+    Inverse variables on the other hand are not placed on any specific device
+    (they will just use the current the device placement context, whatever
+    that happens to be).  The idea is that the inverse variable belong where
+    they will be accessed most often, which is the device that actually applies
+    the preconditioner to the gradient. The user will be responsible for setting
+    the device context for this.
+
+    Args:
+      scope: A string or None.  If None it will be set to the name of this
+        estimator (given by the name property). All variables will be created,
+        and all ops will execute, inside of a variable scope of the given
+        name. (Default: None)
+
+    Returns:
+      cov_update_ops: List of ops that compute the cov updates. Corresponds
+        one-to-one with the list of factors given by the "factors" property.
+      cov_update_op: cov_update_ops grouped into a single op.
+      inv_update_ops: List of ops that compute the inv updates. Corresponds
+        one-to-one with the list of factors given by the "factors" property.
+      inv_update_op: inv_update_ops grouped into a single op.
+      cov_update_thunks: Thunks that make the ops in cov_update_ops.
+      inv_update_thunks: Thunks that make the ops in inv_update_ops.
+    """
+    pass
+
+  @abc.abstractmethod
+  def make_vars_and_create_op_thunks(self, scope=None):
+    """Make vars and create op thunks with a specific placement strategy.
+
+    For each factor, all of that factor's cov variables and their associated
+    update ops will be placed on a particular device.  A new device is chosen
+    for each factor by cycling through list of devices in the cov_devices
+    argument. If cov_devices is None then no explicit device placement occurs.
+
+    An analogous strategy is followed for inverse update ops, with the list of
+    devices being given by the inv_devices argument.
+
+    Inverse variables on the other hand are not placed on any specific device
+    (they will just use the current the device placement context, whatever
+    that happens to be).  The idea is that the inverse variable belong where
+    they will be accessed most often, which is the device that actually applies
+    the preconditioner to the gradient. The user will be responsible for setting
+    the device context for this.
+
+    Args:
+      scope: A string or None.  If None it will be set to the name of this
+        estimator (given by the name property). All variables will be created,
+        and all thunks will execute, inside of a variable scope of the given
+        name. (Default: None)
+
+    Returns:
+      cov_update_thunks: List of cov update thunks. Corresponds one-to-one with
+        the list of factors given by the "factors" property.
+      inv_update_thunks: List of inv update thunks. Corresponds one-to-one with
+        the list of factors given by the "factors" property.
+    """
+    pass
+
   def _apply_transformation(self, vecs_and_vars, transform):
     """Applies an block-wise transformation to the corresponding vectors.
 
@@ -286,158 +340,6 @@ class FisherEstimator(object):
     self._instantiate_factors()
     self._register_matrix_functions()
 
-  def make_ops_and_vars(self, scope=None):
-    """Make ops and vars with no specific device placement.
-
-    See make_ops_and_vars_round_robin for further details.
-
-    Args:
-      scope: A string or None.  If None it will be set to the name of this
-        estimator (given by the name property). All variables will be created,
-        and all ops will execute, inside of a variable scope of the given
-        name. (Default: None)
-    Returns:
-      cov_update_ops: List of ops that compute the cov updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      cov_update_op: cov_update_ops grouped into a single op.
-      inv_update_ops: List of ops that compute the inv updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      inv_update_op: inv_update_ops grouped into a single op.
-      cov_update_thunks: Thunks that make the ops in cov_update_ops.
-      inv_update_thunks: Thunks that make the ops in inv_update_ops.
-    """
-    return self.make_ops_and_vars_round_robin(scope=scope)
-
-  # TODO(b/70674513): Factor device placement outside of this class.
-  def make_ops_and_vars_round_robin(self, scope=None, cov_devices=None,
-                                    inv_devices=None):
-    """Make ops and vars with a round-robin device placement strategy.
-
-    For each factor, all of that factor's cov variables and their associated
-    update ops will be placed on a particular device.  A new device is chosen
-    for each factor by cycling through list of devices in the cov_devices
-    argument. If cov_devices is None then no explicit device placement occurs.
-
-    An analogous strategy is followed for inverse update ops, with the list of
-    devices being given by the inv_devices argument.
-
-    Inverse variables on the other hand are not placed on any specific device
-    (they will just use the current the device placement context, whatever
-    that happens to be).  The idea is that the inverse variable belong where
-    they will be accessed most often, which is the device that actually applies
-    the preconditioner to the gradient. The user will be responsible for setting
-    the device context for this.
-
-    Args:
-      scope: A string or None.  If None it will be set to the name of this
-        estimator (given by the name property). All variables will be created,
-        and all ops will execute, inside of a variable scope of the given
-        name. (Default: None)
-      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-
-    Returns:
-      cov_update_ops: List of ops that compute the cov updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      cov_update_op: cov_update_ops grouped into a single op.
-      inv_update_ops: List of ops that compute the inv updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      inv_update_op: inv_update_ops grouped into a single op.
-      cov_update_thunks: Thunks that make the ops in cov_update_ops.
-      inv_update_thunks: Thunks that make the ops in inv_update_ops.
-    """
-    (cov_update_thunks,
-     inv_update_thunks) = self.make_vars_and_create_op_thunks_round_robin(
-         scope=scope,
-         cov_devices=cov_devices,
-         inv_devices=inv_devices)
-    cov_update_ops = [thunk() for thunk in cov_update_thunks]
-    inv_update_ops = [thunk() for thunk in inv_update_thunks]
-
-    scope = self.name if scope is None else scope
-    with variable_scope.variable_scope(scope):
-      cov_update_op = control_flow_ops.group(cov_update_ops,
-                                             name="cov_update_op")
-      inv_update_op = control_flow_ops.group(inv_update_ops,
-                                             name="inv_update_op")
-
-    return (cov_update_ops, cov_update_op, inv_update_ops, inv_update_op,
-            cov_update_thunks, inv_update_thunks)
-
-  def make_vars_and_create_op_thunks_round_robin(self,
-                                                 scope=None,
-                                                 cov_devices=None,
-                                                 inv_devices=None):
-    """Make vars and create op thunks w/ a round-robin device placement strat.
-
-    For each factor, all of that factor's cov variables and their associated
-    update ops will be placed on a particular device.  A new device is chosen
-    for each factor by cycling through list of devices in the cov_devices
-    argument. If cov_devices is None then no explicit device placement occurs.
-
-    An analogous strategy is followed for inverse update ops, with the list of
-    devices being given by the inv_devices argument.
-
-    Inverse variables on the other hand are not placed on any specific device
-    (they will just use the current the device placement context, whatever
-    that happens to be).  The idea is that the inverse variable belong where
-    they will be accessed most often, which is the device that actually applies
-    the preconditioner to the gradient. The user will be responsible for setting
-    the device context for this.
-
-    Args:
-      scope: A string or None.  If None it will be set to the name of this
-        estimator (given by the name property). All variables will be created,
-        and all thunks will execute, inside of a variable scope of the given
-        name. (Default: None)
-      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-    Returns:
-      cov_update_thunks: List of cov update thunks. Corresponds one-to-one with
-        the list of factors given by the "factors" property.
-      inv_update_thunks: List of inv update thunks. Corresponds one-to-one with
-        the list of factors given by the "factors" property.
-    """
-
-    (cov_variable_thunks_raw, cov_update_thunks_raw, inv_variable_thunks_raw,
-     inv_update_thunks_raw) = self.create_ops_and_vars_thunks(scope=scope)
-
-    if cov_devices:
-      cov_update_thunks = []
-      for cov_variable_thunk, cov_update_thunk, device in zip(
-          cov_variable_thunks_raw, cov_update_thunks_raw,
-          itertools.cycle(cov_devices)):
-        with tf_ops.device(device):
-          cov_variable_thunk()
-        cov_update_thunks.append(_make_thunk_on_device(cov_update_thunk,
-                                                       device))
-    else:
-      for cov_variable_thunk in cov_variable_thunks_raw:
-        cov_variable_thunk()
-      cov_update_thunks = cov_update_thunks_raw
-
-    for inv_variable_thunk in inv_variable_thunks_raw:
-      inv_variable_thunk()
-
-    if inv_devices:
-      inv_update_thunks = []
-      for inv_update_thunk, device in zip(inv_update_thunks_raw,
-                                          itertools.cycle(inv_devices)):
-        inv_update_thunks.append(_make_thunk_on_device(inv_update_thunk,
-                                                       device))
-    else:
-      inv_update_thunks = inv_update_thunks_raw
-
-    return cov_update_thunks, inv_update_thunks
-
   def create_ops_and_vars_thunks(self, scope=None):
     """Create thunks that make the ops and vars on demand.
 
@@ -582,3 +484,9 @@ class FisherEstimator(object):
               colocate_gradients_with_ops=self._colocate_gradients_with_ops)
           grads_all.append(nest.pack_sequence_as(tensors, grads_flat))
     return zip(*grads_all)
+
+
+class FisherEstimatorRoundRobin(placement.RoundRobinPlacementMixin,
+                                FisherEstimator):
+  """Fisher estimator which provides round robin device placement strategy."""
+  pass
diff --git a/tensorflow/contrib/kfac/python/ops/optimizer.py b/tensorflow/contrib/kfac/python/ops/optimizer.py
index 083da768ec..843aeef7d8 100644
--- a/tensorflow/contrib/kfac/python/ops/optimizer.py
+++ b/tensorflow/contrib/kfac/python/ops/optimizer.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 import warnings
-
 # pylint disable=long-line
 from tensorflow.contrib.kfac.python.ops import curvature_matrix_vector_products as cmvp
 from tensorflow.contrib.kfac.python.ops import estimator as est
@@ -53,8 +52,8 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
                estimation_mode="gradients",
                colocate_gradients_with_ops=True,
                batch_size=None,
-               cov_devices=None,
-               inv_devices=None):
+               placement_strategy=None,
+               **kwargs):
     """Initializes the KFAC optimizer with the given settings.
 
     Args:
@@ -96,14 +95,11 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
           (Default: True)
       batch_size: The size of the mini-batch. Only needed when momentum_type
           == 'qmodel' or when automatic adjustment is used.  (Default: None)
-      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
-          computations will be placed on these devices in a round-robin fashion.
-          Can be None, which means that no devices are specified. Only used
-          with (soon-to-be-depcrecated "convenience" properties).
-      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
-          computations will be placed on these devices in a round-robin fashion.
-          Can be None, which means that no devices are specified. Only used
-          with (soon-to-be-depcrecated "convenience" properties).
+      placement_strategy: string, Device placement strategy used when creating
+        covariance variables, covariance ops, and inverse ops.
+        (Default: `None`)
+      **kwargs: Arguments to be passesd to specific placement
+        strategy mixin. Check `placement.RoundRobinPlacementMixin` for example.
 
     Raises:
       ValueError: If the momentum type is unsupported.
@@ -123,8 +119,6 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
     self._layers = layer_collection
     self._estimation_mode = estimation_mode
     self._colocate_gradients_with_ops = colocate_gradients_with_ops
-    self._cov_devices = cov_devices
-    self._inv_devices = inv_devices
 
     # The below paramaters are required only if damping needs to be adapated.
     # These parameters can be set by calling
@@ -164,16 +158,19 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
     self._momentum_type = momentum_type
     self._norm_constraint = norm_constraint
     self._batch_size = batch_size
+    self._placement_strategy = placement_strategy
 
     with variable_scope.variable_scope(name):
-      self._fisher_est = est.FisherEstimator(
-          self._variables,
-          self._cov_ema_decay,
-          self.damping,
-          self._layers,
+      self._fisher_est = est.make_fisher_estimator(
+          placement_strategy=placement_strategy,
+          variables=self._variables,
+          cov_ema_decay=self._cov_ema_decay,
+          damping=self.damping,
+          layer_collection=self._layers,
           exps=(-1,),
           estimation_mode=self._estimation_mode,
-          colocate_gradients_with_ops=self._colocate_gradients_with_ops)
+          colocate_gradients_with_ops=self._colocate_gradients_with_ops,
+          **kwargs)
 
     super(KfacOptimizer, self).__init__(learning_rate, name=name)
 
@@ -236,6 +233,21 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
       self._damping = variable_scope.get_variable(
           "damping", initializer=self._damping_constant, trainable=False)
 
+  @property
+  def variables(self):
+    return self._variables
+
+  @property
+  def damping(self):
+    if self._damping:
+      return self._damping
+    else:
+      return self._damping_constant
+
+  @property
+  def damping_adaptation_interval(self):
+    return self._damping_adaptation_interval
+
   @property
   def cov_update_thunks(self):
     self._maybe_make_and_save_everything()
@@ -266,37 +278,20 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
     self._maybe_make_and_save_everything()
     return self._inv_update_op
 
-  @property
-  def variables(self):
-    return self._variables
-
-  @property
-  def damping(self):
-    if self._damping:
-      return self._damping
-    else:
-      return self._damping_constant
-
-  @property
-  def damping_adaptation_interval(self):
-    return self._damping_adaptation_interval
-
   def _maybe_make_and_save_everything(self):
     if not self._fisher_est.made_vars():
       warnings.warn("These convenience properties will be depcrecated soon. "
                     "Please use explicit op/thunk creation methods instead "
-                    "(e.g. make_ops_and_vars_round_robin, etc).",
+                    "(e.g. make_ops_and_vars, etc).",
                     DeprecationWarning)
       (self._cov_update_ops, self._cov_update_op, self._inv_update_ops,
        self._inv_update_op, self._cov_update_thunks,
-       self._inv_update_thunks) = self.make_ops_and_vars_round_robin(
-           cov_devices=self._cov_devices,
-           inv_devices=self._inv_devices)
+       self._inv_update_thunks) = self.make_ops_and_vars()
 
   def make_ops_and_vars(self):
-    """Make ops and vars with no specific device placement.
+    """Make ops and vars with device placement `self._placement_strategy`.
 
-    See make_ops_and_vars_round_robin for details.
+    See `FisherEstimator.make_ops_and_vars` for details.
 
     Returns:
       cov_update_ops: List of ops that compute the cov updates. Corresponds
@@ -307,77 +302,11 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
       cov_update_op: cov_update_ops grouped into a single op.
       inv_update_op: inv_update_ops grouped into a single op.
     """
-    with variable_scope.variable_scope(self.get_name()):
-      return self._fisher_est.make_ops_and_vars()
-
-  def make_ops_and_vars_round_robin(self, cov_devices=None, inv_devices=None):
-    """Make ops and vars with a round-robin device placement strategy.
-
-    For each factor, all of that factor's cov variables and their associated
-    update ops will be placed on a particular device.  A new device is chosen
-    for each factor by cycling through list of devices in the cov_devices
-    argument. If cov_devices is None then no explicit device placement occurs.
-
-    An analogous strategy is followed for inverse update ops, with the list of
-    devices being given by the inv_devices argument.
+    return self._fisher_est.make_ops_and_vars(scope=self.get_name())
 
-    Inverse variables on the other hand are not placed on any specific device
-    (they will just use the current the device placement context, whatever
-    that happens to be).  The idea is that the inverse variable belong where
-    they will be accessed most often, which is the device that actually applies
-    the preconditioner to the gradient. The user will be responsible for setting
-    the device context for this.
-
-    Args:
-      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
+  def make_vars_and_create_op_thunks(self):
+    """Make vars and create op thunks.
 
-    Returns:
-      cov_update_ops: List of ops that compute the cov updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      cov_update_op: cov_update_ops grouped into a single op.
-      inv_update_ops: List of ops that compute the inv updates. Corresponds
-        one-to-one with the list of factors given by the "factors" property.
-      cov_update_op: cov_update_ops grouped into a single op.
-      inv_update_op: inv_update_ops grouped into a single op.
-      cov_update_thunks: Thunks that make the ops in cov_update_ops.
-      inv_update_thunks: Thunks that make the ops in inv_update_ops.
-    """
-    with variable_scope.variable_scope(self.get_name()):
-      return self._fisher_est.make_ops_and_vars_round_robin(
-          cov_devices=cov_devices, inv_devices=inv_devices)
-
-  def make_vars_and_create_op_thunks_round_robin(self,
-                                                 cov_devices=None,
-                                                 inv_devices=None):
-    """Make vars and create op thunks w/ a round-robin device placement strat.
-
-    For each factor, all of that factor's cov variables and their associated
-    update ops will be placed on a particular device.  A new device is chosen
-    for each factor by cycling through list of devices in the cov_devices
-    argument. If cov_devices is None then no explicit device placement occurs.
-
-    An analogous strategy is followed for inverse update ops, with the list of
-    devices being given by the inv_devices argument.
-
-    Inverse variables on the other hand are not placed on any specific device
-    (they will just use the current the device placement context, whatever
-    that happens to be).  The idea is that the inverse variable belong where
-    they will be accessed most often, which is the device that actually applies
-    the preconditioner to the gradient. The user will be responsible for setting
-    the device context for this.
-
-    Args:
-      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
-      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
-        computations will be placed on these devices in a round-robin fashion.
-        Can be None, which means that no devices are specified.
     Returns:
       cov_update_thunks: List of cov update thunks. Corresponds one-to-one with
         the list of factors given by the "factors" property.
@@ -385,10 +314,9 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
         the list of factors given by the "factors" property.
     """
     scope = self.get_name() + "/" + self._fisher_est.name
-    return self._fisher_est.make_vars_and_create_op_thunks_round_robin(
-        scope=scope, cov_devices=cov_devices, inv_devices=inv_devices)
+    return self._fisher_est.make_vars_and_create_op_thunks(scope=scope)
 
-  def ops_and_vars_thunks(self):
+  def create_ops_and_vars_thunks(self):
     """Create thunks that make the ops and vars on demand.
 
     This function returns 4 lists of thunks: cov_variable_thunks,
@@ -413,7 +341,7 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
       inv_update_thunks: A list of thunks that make the inv update ops.
     """
     scope = self.get_name() + "/" + self._fisher_est.name
-    return self._fisher_est.ops_and_vars_thunks(scope=scope)
+    return self._fisher_est.create_ops_and_vars_thunks(scope=scope)
 
   def minimize(self, *args, **kwargs):
     # Should this variable scope encompass everything below?  Or will the super-
@@ -462,7 +390,6 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
       An `Operation` that applies the specified gradients.
     """
     self._maybe_make_and_save_everything()
-
     # In Python 3, grads_and_vars can be a zip() object which can only be
     # iterated over once. By converting it to a list, we ensure that it can be
     # iterated over more than once.
@@ -618,7 +545,6 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
     # compute the matrix-vector products with the transposed Fisher factor
     fft_precon_grads = cmvpc.multiply_fisher_factor_transpose(precon_grads)
     fft_prev_updates = cmvpc.multiply_fisher_factor_transpose(prev_updates)
-
     batch_size = math_ops.cast(
         self._batch_size, dtype=fft_precon_grads[0].dtype)
 
@@ -802,7 +728,6 @@ class KfacOptimizer(gradient_descent.GradientDescentOptimizer):
     # Go through variable and update its associated part of the velocity vector.
     return [_update_velocity(vec, var) for vec, var in vecs_and_vars]
 
-  # TODO(b/73448937): Move all update damping code to a separate class/function.
   def _update_damping(self, prev_batch, global_step):
     """Adapts damping parameter. Check KFAC (Section 6.5) for the details.
 
diff --git a/tensorflow/contrib/kfac/python/ops/placement.py b/tensorflow/contrib/kfac/python/ops/placement.py
new file mode 100644
index 0000000000..bf12dbaa9a
--- /dev/null
+++ b/tensorflow/contrib/kfac/python/ops/placement.py
@@ -0,0 +1,167 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Implements placement strategies for cov and inv ops, cov variables."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+
+from tensorflow.python.framework import ops as tf_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import variable_scope
+
+
+def _make_thunk_on_device(func, device):
+  def thunk():
+    with tf_ops.device(device):
+      return func()
+  return thunk
+
+
+class RoundRobinPlacementMixin(object):
+  """Implements round robin placement strategy for ops and variables."""
+
+  def __init__(self, cov_devices=None, inv_devices=None, *args, **kwargs):
+    """Initializes the RoundRobinPlacementMixin class.
+
+    Args:
+      cov_devices: Iterable of device strings (e.g. '/gpu:0'). Covariance
+        computations will be placed on these devices in a round-robin fashion.
+        Can be None, which means that no devices are specified.
+      inv_devices: Iterable of device strings (e.g. '/gpu:0'). Inversion
+        computations will be placed on these devices in a round-robin fashion.
+        Can be None, which means that no devices are specified.
+      *args:
+      **kwargs:
+
+    """
+    super(RoundRobinPlacementMixin, self).__init__(*args, **kwargs)
+    self._cov_devices = cov_devices
+    self._inv_devices = inv_devices
+
+  def make_ops_and_vars(self, scope=None):
+    """Make ops and vars with a round-robin device placement strategy.
+
+    For each factor, all of that factor's cov variables and their associated
+    update ops will be placed on a particular device.  A new device is chosen
+    for each factor by cycling through list of devices in the
+    `self._cov_devices` attribute. If `self._cov_devices` is `None` then no
+    explicit device placement occurs.
+
+    An analogous strategy is followed for inverse update ops, with the list of
+    devices being given by the `self._inv_devices` attribute.
+
+    Inverse variables on the other hand are not placed on any specific device
+    (they will just use the current the device placement context, whatever
+    that happens to be).  The idea is that the inverse variable belong where
+    they will be accessed most often, which is the device that actually applies
+    the preconditioner to the gradient. The user will be responsible for setting
+    the device context for this.
+
+    Args:
+      scope: A string or None.  If None it will be set to the name of this
+        estimator (given by the name property). All variables will be created,
+        and all ops will execute, inside of a variable scope of the given
+        name. (Default: None)
+
+    Returns:
+      cov_update_ops: List of ops that compute the cov updates. Corresponds
+        one-to-one with the list of factors given by the "factors" property.
+      cov_update_op: cov_update_ops grouped into a single op.
+      inv_update_ops: List of ops that compute the inv updates. Corresponds
+        one-to-one with the list of factors given by the "factors" property.
+      inv_update_op: inv_update_ops grouped into a single op.
+      cov_update_thunks: Thunks that make the ops in cov_update_ops.
+      inv_update_thunks: Thunks that make the ops in inv_update_ops.
+    """
+    (cov_update_thunks,
+     inv_update_thunks) = self.make_vars_and_create_op_thunks(scope=scope)
+    cov_update_ops = [thunk() for thunk in cov_update_thunks]
+    inv_update_ops = [thunk() for thunk in inv_update_thunks]
+
+    scope = self.name if scope is None else scope
+    with variable_scope.variable_scope(scope):
+      cov_update_op = control_flow_ops.group(cov_update_ops,
+                                             name="cov_update_op")
+      inv_update_op = control_flow_ops.group(inv_update_ops,
+                                             name="inv_update_op")
+
+    return (cov_update_ops, cov_update_op, inv_update_ops, inv_update_op,
+            cov_update_thunks, inv_update_thunks)
+
+  def make_vars_and_create_op_thunks(self, scope=None):
+    """Make vars and create op thunks w/ a round-robin device placement strat.
+
+    For each factor, all of that factor's cov variables and their associated
+    update ops will be placed on a particular device.  A new device is chosen
+    for each factor by cycling through list of devices in the
+    `self._cov_devices` attribute. If `self._cov_devices` is `Non`e then no
+    explicit device placement occurs.
+
+    An analogous strategy is followed for inverse update ops, with the list of
+    devices being given by the `self._inv_devices` attribute.
+
+    Inverse variables on the other hand are not placed on any specific device
+    (they will just use the current the device placement context, whatever
+    that happens to be).  The idea is that the inverse variable belong where
+    they will be accessed most often, which is the device that actually applies
+    the preconditioner to the gradient. The user will be responsible for setting
+    the device context for this.
+
+    Args:
+      scope: A string or None.  If None it will be set to the name of this
+        estimator (given by the name property). All variables will be created,
+        and all thunks will execute, inside of a variable scope of the given
+        name. (Default: None)
+
+    Returns:
+      cov_update_thunks: List of cov update thunks. Corresponds one-to-one with
+        the list of factors given by the "factors" property.
+      inv_update_thunks: List of inv update thunks. Corresponds one-to-one with
+        the list of factors given by the "factors" property.
+    """
+    # Note: `create_ops_and_vars_thunks` is implemented in `FisherEstimator`.
+    (cov_variable_thunks_raw, cov_update_thunks_raw, inv_variable_thunks_raw,
+     inv_update_thunks_raw) = self.create_ops_and_vars_thunks(scope=scope)
+
+    if self._cov_devices:
+      cov_update_thunks = []
+      for cov_variable_thunk, cov_update_thunk, device in zip(
+          cov_variable_thunks_raw, cov_update_thunks_raw,
+          itertools.cycle(self._cov_devices)):
+        with tf_ops.device(device):
+          cov_variable_thunk()
+        cov_update_thunks.append(_make_thunk_on_device(cov_update_thunk,
+                                                       device))
+    else:
+      for cov_variable_thunk in cov_variable_thunks_raw:
+        cov_variable_thunk()
+      cov_update_thunks = cov_update_thunks_raw
+
+    for inv_variable_thunk in inv_variable_thunks_raw:
+      inv_variable_thunk()
+
+    if self._inv_devices:
+      inv_update_thunks = []
+      for inv_update_thunk, device in zip(inv_update_thunks_raw,
+                                          itertools.cycle(self._inv_devices)):
+        inv_update_thunks.append(_make_thunk_on_device(inv_update_thunk,
+                                                       device))
+    else:
+      inv_update_thunks = inv_update_thunks_raw
+
+    return cov_update_thunks, inv_update_thunks
-- 
GitLab


From 1b732d528ba120a6da32100f9991a353128114e8 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Fri, 23 Mar 2018 10:12:05 -0700
Subject: [PATCH 427/960] Silence compilation warning in allocator.h

Some builds report the following warning (even though LOG(FATAL) is terminal):

```
./tensorflow/core/framework/device_base.h(188): warning: missing return statement at end of non-void function "tensorflow::DeviceBase::GetScopedAllocator"
```

PiperOrigin-RevId: 190235863
---
 tensorflow/core/framework/device_base.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/framework/device_base.h b/tensorflow/core/framework/device_base.h
index 52b9077d8c..8473b228d3 100644
--- a/tensorflow/core/framework/device_base.h
+++ b/tensorflow/core/framework/device_base.h
@@ -185,6 +185,7 @@ class DeviceBase {
   virtual Allocator* GetScopedAllocator(AllocatorAttributes attr,
                                         int64 step_id) {
     LOG(FATAL) << "Device does not implement GetScopedAllocator()";
+    return nullptr;
   }
 
   virtual ScopedAllocatorMgr* GetScopedAllocatorMgr() const { return nullptr; }
-- 
GitLab


From ca8e36cb326540c5f6ea48b79ed209cfb4d43895 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 10:17:52 -0700
Subject: [PATCH 428/960] Check for the existence of an annotation before
 attempting to fetch it.

PiperOrigin-RevId: 190236717
---
 tensorflow/contrib/py2tf/converters/lists.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/py2tf/converters/lists.py b/tensorflow/contrib/py2tf/converters/lists.py
index 12ebd00062..3e62037a50 100644
--- a/tensorflow/contrib/py2tf/converters/lists.py
+++ b/tensorflow/contrib/py2tf/converters/lists.py
@@ -67,6 +67,9 @@ class ListTransformer(transformer.Base):
     node = self.generic_visit(node)
     if isinstance(node.value, gast.Call):
       call_node = node.value
+
+      if not anno.hasanno(call_node.func, anno.Basic.QN):
+        return node
       qn = anno.getanno(call_node.func, anno.Basic.QN)
 
       if qn.qn[-1] == 'append' and (len(call_node.args) == 1):
-- 
GitLab


From 17842715cb46b1f2600912592b7a22f21850eb41 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 10:28:17 -0700
Subject: [PATCH 429/960] Only run GPU version of memory_optimizer_test.

PiperOrigin-RevId: 190238306
---
 tensorflow/core/grappler/optimizers/BUILD | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index bd9608b369..fb13084945 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -3,6 +3,7 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+load("//tensorflow:tensorflow.bzl", "tf_cuda_only_cc_test")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 
 # Platform specific build config
@@ -422,7 +423,7 @@ cc_library(
     ]),
 )
 
-tf_cuda_cc_test(
+tf_cuda_only_cc_test(
     name = "memory_optimizer_test",
     srcs = ["memory_optimizer_test.cc"],
     tags = ["no_cuda_on_cpu_tap"],  # Do not re-enable again without actually testing.
-- 
GitLab


From 1addec9acc81db7cb3fe72217893de28bda1dc26 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 10:34:44 -0700
Subject: [PATCH 430/960] Raise the timeout of tests that are somewhat close to
 the limit, to avoid them timing out.

PiperOrigin-RevId: 190239334
---
 tensorflow/python/keras/BUILD | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 711106d2db..16033e9b8f 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -402,11 +402,10 @@ py_test(
 
 py_test(
     name = "convolutional_recurrent_test",
-    size = "medium",
+    size = "large",
     srcs = ["_impl/keras/layers/convolutional_recurrent_test.py"],
     shard_count = 2,
     srcs_version = "PY2AND3",
-    tags = ["noasan"],  # times out b/63678675
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
-- 
GitLab


From c8e8e6128a82608152fc3559f38eade8336d73ad Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Fri, 23 Mar 2018 10:52:16 -0700
Subject: [PATCH 431/960] Automated g4 rollback of changelist 190139303

PiperOrigin-RevId: 190242291
---
 .../xla/service/algebraic_simplifier.cc       | 22 ++++++--
 .../xla/service/algebraic_simplifier_test.cc  | 51 +++++++++++++++++++
 2 files changed, 68 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 971c2935c8..02b23c2d14 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1121,10 +1121,10 @@ bool OutputIsSubsetOfOperandElements(HloInstruction* instruction,
 
 Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
   auto operand = broadcast->mutable_operand(0);
+  auto dims = broadcast->dimensions();
   // A degenerate broadcast of a reshape that does not change the number of
   // elements can be replaced by a reshape.
-  if (std::is_sorted(broadcast->dimensions().begin(),
-                     broadcast->dimensions().end()) &&
+  if (std::is_sorted(dims.begin(), dims.end()) &&
       ShapeUtil::ElementsIn(broadcast->shape()) ==
           ShapeUtil::ElementsIn(operand->shape())) {
     VLOG(10) << "transform broadcast(X) -> reshape(X) where "
@@ -1142,8 +1142,8 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     VLOG(10) << "transform broadcast(X) -> transpose(X) where "
                 "n(broadcast(X)) == n(X)";
     return ReplaceWithNewInstruction(
-        broadcast, HloInstruction::CreateTranspose(broadcast->shape(), operand,
-                                                   broadcast->dimensions()));
+        broadcast,
+        HloInstruction::CreateTranspose(broadcast->shape(), operand, dims));
   }
 
   // A broadcast of a reshape which merely inserts 1-sized dimensions can
@@ -1157,7 +1157,6 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
     if (merely_inserts_or_deletes_1_sized_dimensions &&
         deleted_indices.empty()) {
       std::reverse(inserted_indices.begin(), inserted_indices.end());
-      auto dims = broadcast->dimensions();
       for (auto inserted_index : inserted_indices) {
         dims.erase(dims.begin() + inserted_index);
       }
@@ -1201,6 +1200,19 @@ Status AlgebraicSimplifierVisitor::HandleBroadcast(HloInstruction* broadcast) {
         return user->ReplaceAllUsesWith(new_broadcast);
       }
     }
+    return Status::OK();
+  }
+
+  // Merge two consecutive broadcasts into a single one.
+  if (operand->opcode() == HloOpcode::kBroadcast) {
+    std::vector<int64> new_dimensions;
+    for (auto dim : operand->dimensions()) {
+      new_dimensions.push_back(dims[dim]);
+    }
+    return ReplaceWithNewInstruction(
+        broadcast,
+        HloInstruction::CreateBroadcast(
+            broadcast->shape(), operand->mutable_operand(0), new_dimensions));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 451294ef5d..3b80a827bf 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -35,6 +35,8 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 
+using ::testing::ElementsAre;
+
 namespace xla {
 namespace {
 
@@ -2462,6 +2464,55 @@ TEST_F(AlgebraicSimplifierTest, TrivialDynamicUpdateSlice) {
               op::DynamicSlice(op::Parameter(), op::Parameter()));
 }
 
+// Test that two consecutive broadcasts can be merged to one.
+TEST_F(AlgebraicSimplifierTest, MergeBroadcasts) {
+  HloComputation::Builder builder(TestName());
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 2});
+  HloInstruction* input_array = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR1<float>({3, 4})));
+  HloInstruction* inner_bcast = builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r2f32, input_array, {1}));
+  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 2, 2});
+  builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r3f32, inner_bcast, {0, 2}));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_THAT(root, op::Broadcast(op::Constant()));
+  EXPECT_THAT(root->dimensions(), ElementsAre(2));
+}
+
+// Test that two consecutive broadcasts can be merged to one.
+TEST_F(AlgebraicSimplifierTest, MergeBroadcasts2) {
+  HloComputation::Builder builder(TestName());
+  Shape r2f32 = ShapeUtil::MakeShape(F32, {2, 3});
+  Shape r3f32 = ShapeUtil::MakeShape(F32, {2, 5, 3});
+  HloInstruction* param0 = builder.AddInstruction(
+      HloInstruction::CreateParameter(0, r2f32, "param0"));
+  // The initial dimensions go to places 0 and 2 in the 3-dim array,
+  // and to places 1 and 3 in the 4-dim array,
+  HloInstruction* inner_bcast = builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r3f32, param0, {0, 2}));
+  Shape r4f32 = ShapeUtil::MakeShape(F32, {4, 2, 5, 3});
+  builder.AddInstruction(
+      HloInstruction::CreateBroadcast(r4f32, inner_bcast, {1, 2, 3}));
+
+  auto computation = module().AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root->opcode(), HloOpcode::kBroadcast);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module()).ValueOrDie());
+  root = computation->root_instruction();
+  EXPECT_THAT(root, op::Broadcast(op::Parameter(0)));
+  EXPECT_THAT(root->dimensions(), ElementsAre(1, 3));
+}
+
 struct PadReduceWindowEffectiveBroadcastCase {
   std::vector<int64> input_spatials;
   std::vector<int64> symmetric_pad_spatials;
-- 
GitLab


From f3f9e6acc2de3094f9e2222498a32fa53cbe40f1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 10:52:57 -0700
Subject: [PATCH 432/960] Slightly tweak the whitelist checking mechanism.

PiperOrigin-RevId: 190242423
---
 tensorflow/contrib/py2tf/converters/call_trees.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/contrib/py2tf/converters/call_trees.py b/tensorflow/contrib/py2tf/converters/call_trees.py
index e3040f09e4..f498b814bf 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees.py
+++ b/tensorflow/contrib/py2tf/converters/call_trees.py
@@ -118,6 +118,12 @@ class CallTreeTransformer(transformer.Base):
 
   def _should_compile(self, node, fqn):
     """Determines whether an entity should be compiled in the context."""
+    # TODO(mdan): Needs cleanup. We should remove the use of fqn altogether.
+    module_name = fqn[0]
+    for mod in self.uncompiled_modules:
+      if module_name.startswith(mod[0] + '.'):
+        return False
+
     for i in range(1, len(fqn)):
       if fqn[:i] in self.uncompiled_modules:
         return False
-- 
GitLab


From 65a9216a776f03f6b1568a827b946ba4b1a1d558 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Fri, 23 Mar 2018 10:53:49 -0700
Subject: [PATCH 433/960] Adds support for SavedModel to TOCO.

PiperOrigin-RevId: 190242571
---
 tensorflow/contrib/lite/toco/BUILD            |  37 +++
 tensorflow/contrib/lite/toco/args.h           |   7 +-
 .../contrib/lite/toco/model_cmdline_flags.cc  |   6 +
 tensorflow/contrib/lite/toco/toco.cc          |  97 ++++---
 .../contrib/lite/toco/toco_cmdline_flags.cc   |  98 +++++--
 .../contrib/lite/toco/toco_saved_model.cc     | 186 ++++++++++++
 .../contrib/lite/toco/toco_saved_model.h      |  53 ++++
 .../lite/toco/toco_saved_model_test.cc        | 274 ++++++++++++++++++
 8 files changed, 690 insertions(+), 68 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/toco_saved_model.cc
 create mode 100644 tensorflow/contrib/lite/toco/toco_saved_model.h
 create mode 100644 tensorflow/contrib/lite/toco/toco_saved_model_test.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 486ff1edcd..102740ee47 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -124,6 +124,7 @@ cc_library(
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/strings",
+        "@com_google_absl//absl/types:optional",
     ],
 )
 
@@ -167,6 +168,41 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "toco_saved_model",
+    srcs = [
+        "toco_saved_model.cc",
+    ],
+    hdrs = [
+        "toco_saved_model.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        ":model_cmdline_flags",
+        ":model_flags_proto_cc",
+        ":toco_flags_proto_cc",
+        ":types_proto_cc",
+        "//tensorflow/cc/tools:freeze_saved_model",
+        "//tensorflow/core:protos_all_cc",
+        "@com_google_absl//absl/strings",
+    ],
+)
+
+tf_cc_test(
+    name = "toco_saved_model_test",
+    srcs = ["toco_saved_model_test.cc"],
+    deps = [
+        ":model_cmdline_flags",
+        ":toco_cmdline_flags",
+        ":toco_saved_model",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:scope",
+        "//tensorflow/core:test",
+        "@com_google_absl//absl/strings",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_library(
     name = "graph_transformations",
     srcs = [
@@ -363,6 +399,7 @@ tf_cc_binary(
         ":toco_cmdline_flags",
         ":toco_flags_proto_cc",
         ":toco_port",
+        ":toco_saved_model",
         ":toco_tooling",
         ":types_proto_cc",
         "//tensorflow/core:lib",
diff --git a/tensorflow/contrib/lite/toco/args.h b/tensorflow/contrib/lite/toco/args.h
index 59a6115920..7b71792ff7 100644
--- a/tensorflow/contrib/lite/toco/args.h
+++ b/tensorflow/contrib/lite/toco/args.h
@@ -190,6 +190,7 @@ struct ParsedModelFlags {
   Arg<string> output_array;
   Arg<string> output_arrays;
   Arg<string> input_shapes;
+  Arg<int> batch_size = Arg<int>(1);
   Arg<float> mean_value = Arg<float>(0.f);
   Arg<string> mean_values;
   Arg<float> std_value = Arg<float>(1.f);
@@ -215,9 +216,11 @@ struct ParsedModelFlags {
 // you want). See toco_cmdline_flags.cc for details.
 struct ParsedTocoFlags {
   Arg<string> input_file;
+  Arg<string> savedmodel_directory;
   Arg<string> output_file;
-  Arg<string> input_format;
-  Arg<string> output_format;
+  Arg<string> input_format = Arg<string>("TENSORFLOW_GRAPHDEF");
+  Arg<string> output_format = Arg<string>("TFLITE");
+  Arg<string> savedmodel_tagset;
   // TODO(aselle): command_line_flags  doesn't support doubles
   Arg<float> default_ranges_min = Arg<float>(0.);
   Arg<float> default_ranges_max = Arg<float>(0.);
diff --git a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
index 4e2dec15a5..4264f21c76 100644
--- a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
+++ b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
@@ -72,6 +72,12 @@ bool ParseModelFlagsFromCommandLineFlags(
            "Shapes corresponding to --input_arrays, colon-separated. For "
            "many models each shape takes the form batch size, input array "
            "height, input array width, input array depth."),
+      Flag("batch_size", parsed_flags.batch_size.bind(),
+           parsed_flags.batch_size.default_value(),
+           "Batch size for the model. Replaces the first dimension of an "
+           "input size array if undefined. Use only with SavedModels when "
+           "--input_shapes flag is not specified. Always use --input_shapes "
+           "flag with frozen graphs."),
       Flag("input_data_type", parsed_flags.input_data_type.bind(),
            parsed_flags.input_data_type.default_value(),
            "Deprecated: use --input_data_types instead. Input array type, if "
diff --git a/tensorflow/contrib/lite/toco/toco.cc b/tensorflow/contrib/lite/toco/toco.cc
index f01ec0ec61..8041aa9e7f 100644
--- a/tensorflow/contrib/lite/toco/toco.cc
+++ b/tensorflow/contrib/lite/toco/toco.cc
@@ -23,40 +23,70 @@ limitations under the License.
 #include "tensorflow/contrib/lite/toco/toco_cmdline_flags.h"
 #include "tensorflow/contrib/lite/toco/toco_flags.pb.h"
 #include "tensorflow/contrib/lite/toco/toco_port.h"
+#include "tensorflow/contrib/lite/toco/toco_saved_model.h"
 #include "tensorflow/contrib/lite/toco/toco_tooling.h"
 #include "tensorflow/contrib/lite/toco/toco_types.h"
 #include "tensorflow/core/platform/logging.h"
 
-#ifndef CHECK_OK
-#define CHECK_OK(val) CHECK_EQ((val).ok(), true)
-#define QCHECK_OK(val) QCHECK_EQ((val).ok(), true)
-#endif
-
 namespace toco {
 namespace {
 
-#define QCHECK_REQUIRE_TOCO_FLAG(arg) \
-  QCHECK(parsed_toco_flags.arg.specified()) << "Missing required flag: " #arg;
-
-void CheckFilePermissions(const ParsedTocoFlags& parsed_toco_flags,
-                          const ParsedModelFlags& parsed_model_flags,
-                          const TocoFlags& toco_flags) {
-  port::CheckInitGoogleIsDone("InitGoogle is not done yet");
-
-  QCHECK_REQUIRE_TOCO_FLAG(input_file)
-  QCHECK_OK(port::file::Exists(parsed_toco_flags.input_file.value(),
-                               port::file::Defaults()))
-      << "Specified input_file does not exist: "
-      << parsed_toco_flags.input_file.value();
-  QCHECK_OK(port::file::Readable(parsed_toco_flags.input_file.value(),
-                                 port::file::Defaults()))
+// Checks the permissions of the output file to ensure it is writeable.
+void CheckOutputFilePermissions(const Arg<string>& output_file) {
+  QCHECK(output_file.specified()) << "Missing required flag --output_file.\n";
+  QCHECK(port::file::Writable(output_file.value()).ok())
+      << "Specified output_file is not writable: " << output_file.value()
+      << ".\n";
+}
+
+// Checks the permissions of the frozen model file.
+void CheckFrozenModelPermissions(const Arg<string>& input_file) {
+  QCHECK(input_file.specified()) << "Missing required flag --input_file.\n";
+  QCHECK(port::file::Exists(input_file.value(), port::file::Defaults()).ok())
+      << "Specified input_file does not exist: " << input_file.value() << ".\n";
+  QCHECK(port::file::Readable(input_file.value(), port::file::Defaults()).ok())
       << "Specified input_file exists, but is not readable: "
-      << parsed_toco_flags.input_file.value();
+      << input_file.value() << ".\n";
+}
 
-  QCHECK_REQUIRE_TOCO_FLAG(output_file);
-  QCHECK_OK(port::file::Writable(parsed_toco_flags.output_file.value()))
-      << "parsed_toco_flags.input_file.value() output_file is not writable: "
-      << parsed_toco_flags.output_file.value();
+// Checks the permissions of the SavedModel directory.
+void CheckSavedModelPermissions(const Arg<string>& savedmodel_directory) {
+  QCHECK(savedmodel_directory.specified())
+      << "Missing required flag --savedmodel_directory.\n";
+  QCHECK(
+      port::file::Exists(savedmodel_directory.value(), port::file::Defaults())
+          .ok())
+      << "Specified savedmodel_directory does not exist: "
+      << savedmodel_directory.value() << ".\n";
+}
+
+// Reads the contents of the GraphDef from either the frozen graph file or the
+// SavedModel directory. If it reads the SavedModel directory, it updates the
+// ModelFlags and TocoFlags accordingly.
+void ReadInputData(const ParsedTocoFlags& parsed_toco_flags,
+                   const ParsedModelFlags& parsed_model_flags,
+                   TocoFlags* toco_flags, ModelFlags* model_flags,
+                   string* graph_def_contents) {
+  port::CheckInitGoogleIsDone("InitGoogle is not done yet.\n");
+
+  bool has_input_file = parsed_toco_flags.input_file.specified();
+  bool has_savedmodel_dir = parsed_toco_flags.savedmodel_directory.specified();
+
+  // Ensure either input_file or savedmodel_directory flag has been set.
+  QCHECK_NE(has_input_file, has_savedmodel_dir)
+      << "Specify either input_file or savedmodel_directory flag.\n";
+
+  // Checks the input file permissions and reads the contents.
+  if (has_input_file) {
+    CheckFrozenModelPermissions(parsed_toco_flags.input_file);
+    CHECK(port::file::GetContents(parsed_toco_flags.input_file.value(),
+                                  graph_def_contents, port::file::Defaults())
+              .ok());
+  } else {
+    CheckSavedModelPermissions(parsed_toco_flags.savedmodel_directory);
+    GetSavedModelContents(parsed_toco_flags, parsed_model_flags, toco_flags,
+                          model_flags, graph_def_contents);
+  }
 }
 
 void ToolMain(const ParsedTocoFlags& parsed_toco_flags,
@@ -67,21 +97,20 @@ void ToolMain(const ParsedTocoFlags& parsed_toco_flags,
   TocoFlags toco_flags;
   ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags, &toco_flags);
 
-  CheckFilePermissions(parsed_toco_flags, parsed_model_flags, toco_flags);
+  string graph_def_contents;
+  ReadInputData(parsed_toco_flags, parsed_model_flags, &toco_flags,
+                &model_flags, &graph_def_contents);
+  CheckOutputFilePermissions(parsed_toco_flags.output_file);
 
-  string input_file_contents;
-  CHECK_OK(port::file::GetContents(parsed_toco_flags.input_file.value(),
-                                   &input_file_contents,
-                                   port::file::Defaults()));
   std::unique_ptr<Model> model =
-      Import(toco_flags, model_flags, input_file_contents);
+      Import(toco_flags, model_flags, graph_def_contents);
   Transform(toco_flags, model.get());
   string output_file_contents;
   Export(toco_flags, *model, toco_flags.allow_custom_ops(),
          &output_file_contents);
-  CHECK_OK(port::file::SetContents(parsed_toco_flags.output_file.value(),
-                                   output_file_contents,
-                                   port::file::Defaults()));
+  CHECK(port::file::SetContents(parsed_toco_flags.output_file.value(),
+                                output_file_contents, port::file::Defaults())
+            .ok());
 }
 
 }  // namespace
diff --git a/tensorflow/contrib/lite/toco/toco_cmdline_flags.cc b/tensorflow/contrib/lite/toco/toco_cmdline_flags.cc
index 0f67c2de72..cc7803dd86 100644
--- a/tensorflow/contrib/lite/toco/toco_cmdline_flags.cc
+++ b/tensorflow/contrib/lite/toco/toco_cmdline_flags.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "absl/strings/str_join.h"
 #include "absl/strings/str_split.h"
 #include "absl/strings/strip.h"
+#include "absl/types/optional.h"
 #include "tensorflow/contrib/lite/toco/toco_cmdline_flags.h"
 #include "tensorflow/contrib/lite/toco/toco_port.h"
 #include "tensorflow/core/platform/logging.h"
@@ -38,6 +39,9 @@ bool ParseTocoFlagsFromCommandLineFlags(
            "Input file (model of any supported format). For Protobuf "
            "formats, both text and binary are supported regardless of file "
            "extension."),
+      Flag("savedmodel_directory", parsed_flags.savedmodel_directory.bind(),
+           parsed_flags.savedmodel_directory.default_value(),
+           "Full path to the directory containing the SavedModel."),
       Flag("output_file", parsed_flags.output_file.bind(),
            parsed_flags.output_file.default_value(),
            "Output file. "
@@ -49,6 +53,11 @@ bool ParseTocoFlagsFromCommandLineFlags(
            parsed_flags.output_format.default_value(),
            "Output file format. "
            "One of TENSORFLOW_GRAPHDEF, TFLITE, GRAPHVIZ_DOT."),
+      Flag("savedmodel_tagset", parsed_flags.savedmodel_tagset.bind(),
+           parsed_flags.savedmodel_tagset.default_value(),
+           "Comma-separated set of tags identifying the MetaGraphDef within "
+           "the SavedModel to analyze. All tags in the tag set must be "
+           "specified."),
       Flag("default_ranges_min", parsed_flags.default_ranges_min.bind(),
            parsed_flags.default_ranges_min.default_value(),
            "If defined, will be used as the default value for the min bound "
@@ -128,47 +137,72 @@ bool ParseTocoFlagsFromCommandLineFlags(
   }
 }
 
+namespace {
+
+// Defines the requirements for a given flag. kUseDefault means the default
+// should be used in cases where the value isn't specified by the user.
+enum class FlagRequirement {
+  kNone,
+  kMustBeSpecified,
+  kMustNotBeSpecified,
+  kUseDefault,
+};
+
+// Enforces the FlagRequirements are met for a given flag.
+template <typename T>
+void EnforceFlagRequirement(const T& flag, const string& flag_name,
+                            FlagRequirement requirement) {
+  if (requirement == FlagRequirement::kMustBeSpecified) {
+    QCHECK(flag.specified()) << "Missing required flag " << flag_name;
+  }
+  if (requirement == FlagRequirement::kMustNotBeSpecified) {
+    QCHECK(!flag.specified())
+        << "Given other flags, this flag should not have been specified: "
+        << flag_name;
+  }
+}
+
+// Gets the value from the flag if specified. Returns default if the
+// FlagRequirement is kUseDefault.
+template <typename T>
+absl::optional<T> GetFlagValue(const Arg<T>& flag,
+                               FlagRequirement requirement) {
+  if (flag.specified()) return flag.value();
+  if (requirement == FlagRequirement::kUseDefault) return flag.default_value();
+  return absl::optional<T>();
+}
+
+}  // namespace
+
 void ReadTocoFlagsFromCommandLineFlags(const ParsedTocoFlags& parsed_toco_flags,
                                        TocoFlags* toco_flags) {
   namespace port = toco::port;
   port::CheckInitGoogleIsDone("InitGoogle is not done yet");
 
-  enum class FlagRequirement { kNone, kMustBeSpecified, kMustNotBeSpecified };
-
-#define ENFORCE_FLAG_REQUIREMENT(name, requirement)                          \
-  do {                                                                       \
-    if (requirement == FlagRequirement::kMustBeSpecified) {                  \
-      QCHECK(parsed_toco_flags.name.specified())                             \
-          << "Missing required flag: " << #name;                             \
-    }                                                                        \
-    if (requirement == FlagRequirement::kMustNotBeSpecified) {               \
-      QCHECK(!parsed_toco_flags.name.specified())                            \
-          << "Given other flags, this flag should not have been specified: " \
-          << #name;                                                          \
-    }                                                                        \
-  } while (false)
-#define READ_TOCO_FLAG(name, requirement)                     \
-  ENFORCE_FLAG_REQUIREMENT(name, requirement);                \
-  do {                                                        \
-    if (parsed_toco_flags.name.specified()) {                 \
-      toco_flags->set_##name(parsed_toco_flags.name.value()); \
-    }                                                         \
+#define READ_TOCO_FLAG(name, requirement)                                \
+  do {                                                                   \
+    EnforceFlagRequirement(parsed_toco_flags.name, #name, requirement);  \
+    auto flag_value = GetFlagValue(parsed_toco_flags.name, requirement); \
+    if (flag_value.has_value()) {                                        \
+      toco_flags->set_##name(flag_value.value());                        \
+    }                                                                    \
   } while (false)
 
-#define PARSE_TOCO_FLAG(Type, name, requirement)               \
-  ENFORCE_FLAG_REQUIREMENT(name, requirement);                 \
-  do {                                                         \
-    if (parsed_toco_flags.name.specified()) {                  \
-      Type x;                                                  \
-      QCHECK(Type##_Parse(parsed_toco_flags.name.value(), &x)) \
-          << "Unrecognized " << #Type << " value "             \
-          << parsed_toco_flags.name.value();                   \
-      toco_flags->set_##name(x);                               \
-    }                                                          \
+#define PARSE_TOCO_FLAG(Type, name, requirement)                         \
+  do {                                                                   \
+    EnforceFlagRequirement(parsed_toco_flags.name, #name, requirement);  \
+    auto flag_value = GetFlagValue(parsed_toco_flags.name, requirement); \
+    if (flag_value.has_value()) {                                        \
+      Type x;                                                            \
+      QCHECK(Type##_Parse(flag_value.value(), &x))                       \
+          << "Unrecognized " << #Type << " value "                       \
+          << parsed_toco_flags.name.value();                             \
+      toco_flags->set_##name(x);                                         \
+    }                                                                    \
   } while (false)
 
-  PARSE_TOCO_FLAG(FileFormat, input_format, FlagRequirement::kMustBeSpecified);
-  PARSE_TOCO_FLAG(FileFormat, output_format, FlagRequirement::kMustBeSpecified);
+  PARSE_TOCO_FLAG(FileFormat, input_format, FlagRequirement::kUseDefault);
+  PARSE_TOCO_FLAG(FileFormat, output_format, FlagRequirement::kUseDefault);
   PARSE_TOCO_FLAG(IODataType, inference_type, FlagRequirement::kNone);
   PARSE_TOCO_FLAG(IODataType, inference_input_type, FlagRequirement::kNone);
   READ_TOCO_FLAG(default_ranges_min, FlagRequirement::kNone);
diff --git a/tensorflow/contrib/lite/toco/toco_saved_model.cc b/tensorflow/contrib/lite/toco/toco_saved_model.cc
new file mode 100644
index 0000000000..91a742b9e0
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/toco_saved_model.cc
@@ -0,0 +1,186 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <string>
+#include <vector>
+
+#include "absl/strings/numbers.h"
+#include "tensorflow/contrib/lite/toco/model_cmdline_flags.h"
+#include "tensorflow/contrib/lite/toco/toco_saved_model.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/tensor_shape.pb.h"
+
+namespace toco {
+namespace {
+
+// Loads a SavedModel from the directory specified in parsed_toco_flags.
+// Returns a SavedModelBundle with the requested MetaGraphDef.
+const tensorflow::SavedModelBundle* LoadSavedModel(
+    const ParsedTocoFlags& parsed_toco_flags) {
+  const string model_path = parsed_toco_flags.savedmodel_directory.value();
+  QCHECK(tensorflow::MaybeSavedModelDirectory(model_path))
+      << "Model is not saved in the supported SavedModel format.\n";
+
+  // Gets the tags identifying the MetaGraphDef from the command line arguments.
+  QCHECK(parsed_toco_flags.savedmodel_tagset.specified())
+      << "Missing required flag --savedmodel_tagset.\n";
+  const string tags_str = parsed_toco_flags.savedmodel_tagset.value();
+  auto tags = absl::StrSplit(tags_str, ',');
+
+  // Loads MetaGraphDef.
+  auto* bundle = new tensorflow::SavedModelBundle;
+  TF_CHECK_OK(tensorflow::LoadSavedModel(tensorflow::SessionOptions(),
+                                         tensorflow::RunOptions(), model_path,
+                                         tags, bundle))
+      << "Failed to load exported model from " << model_path
+      << ". Ensure the model contains the required tags '" << tags_str
+      << "'.\n";
+  return bundle;
+}
+
+// Returns the array name without the postfix.
+//
+// e.g. reduces "input:0" to "input".
+string GetArrayName(const string& name) {
+  const std::vector<string>& names = absl::StrSplit(name, ':');
+  return names[0];
+}
+
+// Returns the list of array names without the postfix sorted alphabetically.
+std::set<string> GetSortedNames(const std::unordered_set<string>& names) {
+  std::vector<string> final_names;
+  final_names.reserve(names.size());
+  for (const auto& name : names) {
+    final_names.push_back(GetArrayName(name));
+  }
+  return std::set<string>(final_names.begin(), final_names.end());
+}
+
+// Gets the final shape after replacing the first dimension with batch size, if
+// it is undefined (containing the value -1). Returns whether the shape is
+// valid.
+bool ReplaceShapeBatchSize(const tensorflow::TensorShapeProto& shape,
+                           int batch_size,
+                           tensorflow::TensorShapeProto* final_shape) {
+  for (int idx = 0; idx < shape.dim().size(); ++idx) {
+    int64 final_dim = shape.dim()[idx].size();
+    if (final_dim == -1) {
+      if (idx > 0) return false;
+      final_dim = batch_size;
+    }
+    final_shape->add_dim()->set_size(final_dim);
+  }
+  return true;
+}
+
+// Updates the input arrays in ModelFlags to contain the shape of the array.
+void ProcessInputShapes(const tensorflow::GraphDef& graph_def, int batch_size,
+                        ModelFlags* model_flags) {
+  // Build map of input array names to input arrays.
+  std::unordered_map<string, InputArray*> input_data_map;
+  for (auto& input : *model_flags->mutable_input_arrays()) {
+    input_data_map[input.name()] = &input;
+  }
+
+  // Adds shapes to the input arrays if the shape is valid.
+  for (const tensorflow::NodeDef& node_def : graph_def.node()) {
+    if (input_data_map.find(node_def.name()) != input_data_map.end()) {
+      const auto shape_it = node_def.attr().find("shape");
+      if (shape_it != node_def.attr().end()) {
+        tensorflow::TensorShapeProto final_shape;
+        bool is_valid = ReplaceShapeBatchSize(shape_it->second.shape(),
+                                              batch_size, &final_shape);
+
+        if (is_valid) {
+          auto* shape = input_data_map.at(node_def.name())->mutable_shape();
+          QCHECK_EQ(shape->dims_size(), 0)
+              << "The shape for the input '" << node_def.name()
+              << "' was previously defined. For clarity please define inputs "
+              << "via --input_arrays and input_shapes flags.\n";
+          for (const auto& dim : final_shape.dim()) {
+            shape->add_dims(dim.size());
+          }
+        }
+      }
+    }
+  }
+
+  // Checks all input arrays have a shape.
+  for (auto const& input : model_flags->input_arrays()) {
+    QCHECK(input.shape().dims_size() > 0)
+        << "A valid input shape was not found for input '" << input.name()
+        << "'. Please define via --input_arrays and --input_shapes flags.\n";
+  }
+}
+
+}  // namespace
+
+void ParseMetaData(const tensorflow::GraphDef& graph_def,
+                   const std::unordered_set<string>& inputs,
+                   const std::unordered_set<string>& outputs,
+                   const ParsedTocoFlags& parsed_toco_flags,
+                   const ParsedModelFlags& parsed_model_flags,
+                   TocoFlags* toco_flags, ModelFlags* model_flags) {
+  if (!parsed_model_flags.input_arrays.specified()) {
+    const std::set<string> sorted_inputs = GetSortedNames(inputs);
+    for (const auto& input_name : sorted_inputs) {
+      model_flags->add_input_arrays()->set_name(input_name);
+    }
+  }
+
+  if (!parsed_model_flags.output_arrays.specified()) {
+    const std::set<string> sorted_outputs = GetSortedNames(outputs);
+    for (const auto& output_name : sorted_outputs) {
+      model_flags->add_output_arrays(GetArrayName(output_name));
+    }
+  }
+
+  if (!parsed_model_flags.input_shapes.specified()) {
+    int batch_size = parsed_model_flags.batch_size.value();
+    ProcessInputShapes(graph_def, batch_size, model_flags);
+  }
+
+  if (!parsed_toco_flags.inference_type.specified()) {
+    toco_flags->set_inference_type(IODataType::FLOAT);
+  }
+}
+
+// TODO(nupurgarg): Add top level tests.
+void GetSavedModelContents(const ParsedTocoFlags& parsed_toco_flags,
+                           const ParsedModelFlags& parsed_model_flags,
+                           TocoFlags* toco_flags, ModelFlags* model_flags,
+                           string* graph_def_contents) {
+  // Loads the MetaGraphDef within a SavedModelBundle.
+  auto bundle = LoadSavedModel(parsed_toco_flags);
+
+  // Converts the MetaGraphDef to frozen GraphDef.
+  tensorflow::GraphDef frozen_graph_def;
+  std::unordered_set<string> inputs;
+  std::unordered_set<string> outputs;
+  TF_CHECK_OK(tensorflow::FreezeSavedModel(*bundle, &frozen_graph_def, &inputs,
+                                           &outputs));
+
+  // Reads the frozen GraphDef into a string.
+  QCHECK(frozen_graph_def.SerializeToString(graph_def_contents))
+      << "Unable to generate serialized GraphDef.\n";
+
+  // Process inputs and outputs and metadata within GraphDef.
+  const tensorflow::GraphDef graph_def = bundle->meta_graph_def.graph_def();
+  ParseMetaData(graph_def, inputs, outputs, parsed_toco_flags,
+                parsed_model_flags, toco_flags, model_flags);
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/toco_saved_model.h b/tensorflow/contrib/lite/toco/toco_saved_model.h
new file mode 100644
index 0000000000..7a0fabd82d
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/toco_saved_model.h
@@ -0,0 +1,53 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CONTRIB_LITE_TOCO_TOCO_SAVED_MODEL_H_
+#define TENSORFLOW_CONTRIB_LITE_TOCO_TOCO_SAVED_MODEL_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/cc/tools/freeze_saved_model.h"
+#include "tensorflow/contrib/lite/toco/args.h"
+#include "tensorflow/contrib/lite/toco/model_flags.pb.h"
+#include "tensorflow/contrib/lite/toco/toco_flags.pb.h"
+#include "tensorflow/contrib/lite/toco/types.pb.h"
+
+namespace toco {
+
+// Parses metadata into `toco_flags` and `model_flags`.
+//
+// Stores `inputs` as input_arrays and `outputs` as output_arrays in
+// `model_flags`. Infers input_shapes from the GraphDef and stores it in
+// `model_flags` as part of the input_arrays. Assumes inference_type is FLOAT
+// and stores it in `toco_flags`.
+void ParseMetaData(const tensorflow::GraphDef& graph_def,
+                   const std::unordered_set<string>& inputs,
+                   const std::unordered_set<string>& outputs,
+                   const ParsedTocoFlags& parsed_toco_flags,
+                   const ParsedModelFlags& parsed_model_flags,
+                   TocoFlags* toco_flags, ModelFlags* model_flags);
+
+// Generates a frozen graph from the SavedModel in the directory specified in
+// `toco_flags`. Reads frozen graph contents into `graph_def_contents`. Parses
+// metadata relating to the GraphDef into `toco_flags` and `model_flags`.
+void GetSavedModelContents(const ParsedTocoFlags& parsed_toco_flags,
+                           const ParsedModelFlags& parsed_model_flags,
+                           TocoFlags* toco_flags, ModelFlags* model_flags,
+                           string* graph_def_contents);
+
+}  // namespace toco
+
+#endif  // TENSORFLOW_CONTRIB_LITE_TOCO_TOCO_SAVED_MODEL_H_
diff --git a/tensorflow/contrib/lite/toco/toco_saved_model_test.cc b/tensorflow/contrib/lite/toco/toco_saved_model_test.cc
new file mode 100644
index 0000000000..5e122afe65
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/toco_saved_model_test.cc
@@ -0,0 +1,274 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/contrib/lite/toco/toco_saved_model.h"
+#include "absl/strings/str_join.h"
+#include "tensorflow/cc/framework/scope.h"
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/contrib/lite/toco/model_cmdline_flags.h"
+#include "tensorflow/contrib/lite/toco/toco_cmdline_flags.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+#include <gmock/gmock.h>
+#include <gtest/gtest.h>
+
+namespace toco {
+namespace {
+
+using tensorflow::ops::Add;
+using tensorflow::ops::Const;
+using tensorflow::ops::FakeQuantWithMinMaxArgs;
+using tensorflow::ops::Placeholder;
+
+class TocoSavedModelTest : public ::testing::Test {
+ protected:
+  // Calls functions to process cmdline arguments and calls ParseMetaData.
+  // ParseMetaData parses input_arrays, output_arrays, and gets metadata from
+  // SavedModel it is not defined in the cmdline arguments.
+  void ProcessGraphDefMetadata(const std::unordered_set<string>& inputs,
+                               const std::unordered_set<string>& outputs,
+                               const tensorflow::GraphDef& graph_def) {
+    ReadTocoFlagsFromCommandLineFlags(parsed_toco_flags_, &toco_flags_);
+    ReadModelFlagsFromCommandLineFlags(parsed_model_flags_, &model_flags_);
+    ParseMetaData(graph_def, inputs, outputs, parsed_toco_flags_,
+                  parsed_model_flags_, &toco_flags_, &model_flags_);
+  }
+
+  // Gets the GraphDef from the SavedModelBundle and processes metadata.
+  void ProcessSavedModelMetadata(const std::unordered_set<string>& inputs,
+                                 const std::unordered_set<string>& outputs) {
+    const tensorflow::GraphDef graph_def = bundle_.meta_graph_def.graph_def();
+    ProcessGraphDefMetadata(inputs, outputs, graph_def);
+  }
+
+  // Returns a GraphDef representing a simple float model with a single input.
+  tensorflow::GraphDef GetFloatGraphDef(const std::vector<int64>& shape) {
+    tensorflow::GraphDef graph_def;
+    tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+    tensorflow::Output input =
+        Placeholder(scope.WithOpName("input"), tensorflow::DT_FLOAT,
+                    Placeholder::Shape(tensorflow::PartialTensorShape(shape)));
+    tensorflow::Output zero = Const(scope.WithOpName("zero"), 0.0f, {});
+    tensorflow::Output add = Add(scope.WithOpName("add"), input, zero);
+
+    TF_EXPECT_OK(scope.ToGraphDef(&graph_def));
+    return graph_def;
+  }
+
+  // Returns a GraphDef representing a simple float model with two inputs.
+  tensorflow::GraphDef GetComplexFloatGraphDef() {
+    tensorflow::GraphDef graph_def;
+    tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+    tensorflow::Output inputA =
+        Placeholder(scope.WithOpName("inputA"), tensorflow::DT_FLOAT,
+                    Placeholder::Shape(tensorflow::TensorShape({1, 3, 3, 1})));
+    tensorflow::Output inputB =
+        Placeholder(scope.WithOpName("inputB"), tensorflow::DT_FLOAT,
+                    Placeholder::Shape(tensorflow::TensorShape({1, 3, 3, 1})));
+    tensorflow::Output add = Add(scope.WithOpName("add"), inputB, inputA);
+
+    TF_EXPECT_OK(scope.ToGraphDef(&graph_def));
+    return graph_def;
+  }
+
+  // Returns a GraphDef representing a simple quantized model.
+  tensorflow::GraphDef GetQuantizedGraphDef() {
+    tensorflow::GraphDef graph_def;
+    tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+    tensorflow::Output input =
+        Placeholder(scope.WithOpName("input"), tensorflow::DT_FLOAT,
+                    Placeholder::Shape(tensorflow::TensorShape({1, 3, 3, 1})));
+    tensorflow::Output zero = Const(scope.WithOpName("zero"), 0.0f, {});
+    tensorflow::Output fake_quant =
+        FakeQuantWithMinMaxArgs(scope.WithOpName("quant"), zero);
+    tensorflow::Output add = Add(scope.WithOpName("add"), input, fake_quant);
+
+    TF_EXPECT_OK(scope.ToGraphDef(&graph_def));
+    return graph_def;
+  }
+
+  // Gets the values in the input_arrays flag.
+  std::vector<string> GetInputArrays() {
+    std::vector<string> actual;
+    for (const auto& input : model_flags_.input_arrays()) {
+      actual.push_back(input.name());
+    }
+    return actual;
+  }
+
+  // Gets the values in the output_arrays flag.
+  std::vector<string> GetOutputArrays() {
+    std::vector<string> actual(model_flags_.output_arrays().begin(),
+                               model_flags_.output_arrays().end());
+    return actual;
+  }
+
+  // Gets the shape of the given input array.
+  string GetInputShape(const string& input_array) {
+    for (const auto& input : model_flags_.input_arrays()) {
+      if (input.name() == input_array) {
+        std::vector<string> dims;
+        for (int idx = 0; idx < input.shape().dims_size(); ++idx) {
+          dims.push_back(std::to_string(input.shape().dims(idx)));
+        }
+        return absl::StrJoin(dims, ",");
+      }
+    }
+    return "";
+  }
+
+  tensorflow::SavedModelBundle bundle_;
+  ParsedTocoFlags parsed_toco_flags_;
+  ParsedModelFlags parsed_model_flags_;
+  TocoFlags toco_flags_;
+  ModelFlags model_flags_;
+};
+
+// Tests if input_arrays, output_arrays, inference_type, and output_arrays are
+// added to ModelFlags if they are not specified in cmdline arguments.
+// Tests if the default batch size replaces a -1 in the first dimension.
+TEST_F(TocoSavedModelTest, NoCmdLine) {
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({-1, 3, 3, 1});
+
+  ProcessGraphDefMetadata({"input"}, {"add"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"input"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add"}));
+  EXPECT_EQ(GetInputShape("input"), "1,3,3,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Tests if the order of input_arrays and output_arrays is deterministic when
+// they are taken from the SavedModel.
+TEST_F(TocoSavedModelTest, NoCmdLineMultipleArrays) {
+  tensorflow::GraphDef graph_def = GetComplexFloatGraphDef();
+
+  // Note: The model does not have two outputs. However, the function does not
+  // need an accurate output_array list. This is only meant to test order.
+  ProcessGraphDefMetadata({"inputB", "inputA"}, {"add", "invalid"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"inputA", "inputB"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add", "invalid"}));
+  EXPECT_EQ(GetInputShape("inputA"), "1,3,3,1");
+  EXPECT_EQ(GetInputShape("inputB"), "1,3,3,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Tests if input_shapes is inferred when input_arrays is passed in via cmdline
+// arguments.
+TEST_F(TocoSavedModelTest, InputNameWithoutInputShape) {
+  parsed_model_flags_.input_arrays.bind()("input");
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({2, 3, 3, 1});
+
+  ProcessGraphDefMetadata({"not_used_input"}, {"add"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"input"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add"}));
+  EXPECT_EQ(GetInputShape("input"), "2,3,3,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Ensures a failure occurs when input_shapes is defined without input_arrays.
+TEST_F(TocoSavedModelTest, InputShapeWithoutInputName) {
+  parsed_model_flags_.input_shapes.bind()("1,224,224,1:9,12");
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({1, 3, 3, 1});
+
+  EXPECT_DEATH(ProcessGraphDefMetadata({"input"}, {"add"}, graph_def),
+               "failed: input_shapes.size\\(\\) == "
+               "model_flags->input_arrays_size\\(\\)");
+}
+
+// Tests if the cmdline values of input_arrays, input_shapes are used when
+// specified with an empty GraphDef.
+TEST_F(TocoSavedModelTest, InputArraysCmdLine) {
+  parsed_model_flags_.input_arrays.bind()("inputA,inputB");
+  parsed_model_flags_.input_shapes.bind()("1,224,224,1:9,12");
+
+  ProcessSavedModelMetadata({"input0", "input1"}, {"output0", "output1"});
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"inputA", "inputB"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"output0", "output1"}));
+  EXPECT_EQ(GetInputShape("inputA"), "1,224,224,1");
+  EXPECT_EQ(GetInputShape("inputB"), "9,12");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Tests if the cmdline values of input_arrays, input_shapes are used when
+// specified even if values exist within the GraphDef.
+TEST_F(TocoSavedModelTest, InputArraysCmdLineWithGraphDef) {
+  parsed_model_flags_.input_arrays.bind()("inputA");
+  parsed_model_flags_.input_shapes.bind()("1,224,224,1");
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({1, 3, 3, 1});
+
+  ProcessGraphDefMetadata({"inputA"}, {"add"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"inputA"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add"}));
+  EXPECT_EQ(GetInputShape("inputA"), "1,224,224,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Tests if the cmdline values of input_arrays, input_shapes, inference_type,
+// and output_arrays are used when specified with an empty GraphDef.
+TEST_F(TocoSavedModelTest, AllParamsCmdLine) {
+  parsed_model_flags_.input_arrays.bind()("inputA,inputB");
+  parsed_model_flags_.output_arrays.bind()("outputA,outputB");
+  parsed_model_flags_.input_shapes.bind()("1,224,224,1:9,12");
+  parsed_toco_flags_.inference_type.bind()("FLOAT");
+
+  ProcessSavedModelMetadata({"input0", "input1"}, {"output0", "output1"});
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"inputA", "inputB"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"outputA", "outputB"}));
+  EXPECT_EQ(GetInputShape("inputA"), "1,224,224,1");
+  EXPECT_EQ(GetInputShape("inputB"), "9,12");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Tests if a quantized graph gives the correct values assuming type is passed
+// in via command line.
+TEST_F(TocoSavedModelTest, QuantizedNoCmdLine) {
+  parsed_toco_flags_.inference_type.bind()("QUANTIZED_UINT8");
+  tensorflow::GraphDef graph_def = GetQuantizedGraphDef();
+
+  ProcessGraphDefMetadata({"input"}, {"add"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"input"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add"}));
+  EXPECT_EQ(GetInputShape("input"), "1,3,3,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::QUANTIZED_UINT8);
+}
+
+// Tests if the provided batch size replaces a -1 in the first dimension of
+// input shape.
+TEST_F(TocoSavedModelTest, MissingShapeParameterValid) {
+  parsed_model_flags_.batch_size.bind()(3);
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({-1, 3, 3, 1});
+
+  ProcessGraphDefMetadata({"input"}, {"add"}, graph_def);
+  EXPECT_EQ(GetInputArrays(), std::vector<string>({"input"}));
+  EXPECT_EQ(GetOutputArrays(), std::vector<string>({"add"}));
+  EXPECT_EQ(GetInputShape("input"), "3,3,3,1");
+  EXPECT_EQ(toco_flags_.inference_type(), IODataType::FLOAT);
+}
+
+// Ensures a failure occurs if there is a -1 in a dimension aside from the first
+// position of input shape.
+TEST_F(TocoSavedModelTest, MissingShapeParameterInvalid) {
+  parsed_model_flags_.batch_size.bind()(3);
+  tensorflow::GraphDef graph_def = GetFloatGraphDef({1, -1, 3, 1});
+
+  EXPECT_DEATH(ProcessGraphDefMetadata({"input"}, {"add"}, graph_def),
+               "A valid input shape was not found for input 'input'.");
+}
+
+}  // namespace
+}  // namespace toco
-- 
GitLab


From 651ebf95adf88924c9dfb9cddac3d96a30dffed3 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Fri, 23 Mar 2018 11:01:25 -0700
Subject: [PATCH 434/960] Override `__str__` and `__repr__` in
 `tf.distributions.Distribution`.

PiperOrigin-RevId: 190243989
---
 .../python/kernel_tests/distribution_test.py  | 141 +++++++++++++++---
 .../python/ops/distributions/distribution.py  |  28 ++++
 2 files changed, 149 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
index 507ceb3585..68e0d9cb82 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
@@ -16,6 +16,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy as np
+
 from tensorflow.contrib import distributions
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
@@ -25,23 +27,23 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import random_ops
 from tensorflow.python.platform import test
 
-ds = distributions
+tfd = distributions
 
 
 class DistributionTest(test.TestCase):
 
   def testParamShapesAndFromParams(self):
     classes = [
-        ds.Normal,
-        ds.Bernoulli,
-        ds.Beta,
-        ds.Chi2,
-        ds.Exponential,
-        ds.Gamma,
-        ds.InverseGamma,
-        ds.Laplace,
-        ds.StudentT,
-        ds.Uniform,
+        tfd.Normal,
+        tfd.Bernoulli,
+        tfd.Beta,
+        tfd.Chi2,
+        tfd.Exponential,
+        tfd.Gamma,
+        tfd.InverseGamma,
+        tfd.Laplace,
+        tfd.StudentT,
+        tfd.Uniform,
     ]
 
     sample_shapes = [(), (10,), (10, 20, 30)]
@@ -63,15 +65,15 @@ class DistributionTest(test.TestCase):
     with self.test_session():
       # Note: we cannot easily test all distributions since each requires
       # different initialization arguments. We therefore spot test a few.
-      normal = ds.Normal(loc=1., scale=2., validate_args=True)
+      normal = tfd.Normal(loc=1., scale=2., validate_args=True)
       self.assertEqual(normal.parameters, normal.copy().parameters)
-      wishart = ds.WishartFull(df=2, scale=[[1., 2], [2, 5]],
-                               validate_args=True)
+      wishart = tfd.WishartFull(df=2, scale=[[1., 2], [2, 5]],
+                                validate_args=True)
       self.assertEqual(wishart.parameters, wishart.copy().parameters)
 
   def testCopyOverride(self):
     with self.test_session():
-      normal = ds.Normal(loc=1., scale=2., validate_args=True)
+      normal = tfd.Normal(loc=1., scale=2., validate_args=True)
       unused_normal_copy = normal.copy(validate_args=False)
       base_params = normal.parameters.copy()
       copy_params = normal.copy(validate_args=False).parameters.copy()
@@ -84,19 +86,19 @@ class DistributionTest(test.TestCase):
       mu = 1.
       sigma = 2.
 
-      normal = ds.Normal(mu, sigma, validate_args=True)
+      normal = tfd.Normal(mu, sigma, validate_args=True)
       self.assertTrue(tensor_util.constant_value(normal.is_scalar_event()))
       self.assertTrue(tensor_util.constant_value(normal.is_scalar_batch()))
 
-      normal = ds.Normal([mu], [sigma], validate_args=True)
+      normal = tfd.Normal([mu], [sigma], validate_args=True)
       self.assertTrue(tensor_util.constant_value(normal.is_scalar_event()))
       self.assertFalse(tensor_util.constant_value(normal.is_scalar_batch()))
 
-      mvn = ds.MultivariateNormalDiag([mu], [sigma], validate_args=True)
+      mvn = tfd.MultivariateNormalDiag([mu], [sigma], validate_args=True)
       self.assertFalse(tensor_util.constant_value(mvn.is_scalar_event()))
       self.assertTrue(tensor_util.constant_value(mvn.is_scalar_batch()))
 
-      mvn = ds.MultivariateNormalDiag([[mu]], [[sigma]], validate_args=True)
+      mvn = tfd.MultivariateNormalDiag([[mu]], [[sigma]], validate_args=True)
       self.assertFalse(tensor_util.constant_value(mvn.is_scalar_event()))
       self.assertFalse(tensor_util.constant_value(mvn.is_scalar_batch()))
 
@@ -126,7 +128,7 @@ class DistributionTest(test.TestCase):
       self.assertFalse(is_scalar.eval(feed_dict={x: [1]}))
 
   def _GetFakeDistribution(self):
-    class FakeDistribution(ds.Distribution):
+    class FakeDistribution(tfd.Distribution):
       """Fake Distribution for testing _set_sample_static_shape."""
 
       def __init__(self, batch_shape=None, event_shape=None):
@@ -188,6 +190,105 @@ class DistributionTest(test.TestCase):
       y = dist._set_sample_static_shape(x, sample_shape)
       self.assertTrue(y.get_shape().ndims is None)
 
+  def testStrWorksCorrectlyScalar(self):
+    normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1))
+    self.assertEqual(
+        ("tf.distributions.Normal("
+         "\"Normal\", "
+         "batch_shape=(), "
+         "event_shape=(), "
+         "dtype=float16)"),  # Got the dtype right.
+        str(normal))
+
+    chi2 = tfd.Chi2(df=np.float32([1., 2.]), name="silly")
+    self.assertEqual(
+        ("tf.distributions.Chi2("
+         "\"silly\", "  # What a silly name that is!
+         "batch_shape=(2,), "
+         "event_shape=(), "
+         "dtype=float32)"),
+        str(chi2))
+
+    exp = tfd.Exponential(rate=array_ops.placeholder(dtype=dtypes.float32))
+    self.assertEqual(
+        ("tf.distributions.Exponential(\"Exponential\", "
+         # No batch shape.
+         "event_shape=(), "
+         "dtype=float32)"),
+        str(exp))
+
+  def testStrWorksCorrectlyMultivariate(self):
+    mvn_static = tfd.MultivariateNormalDiag(
+        loc=np.zeros([2, 2]), name="MVN")
+    self.assertEqual(
+        ("tf.distributions.MultivariateNormalDiag("
+         "\"MVN\", "
+         "batch_shape=(2,), "
+         "event_shape=(2,), "
+         "dtype=float64)"),
+        str(mvn_static))
+
+    mvn_dynamic = tfd.MultivariateNormalDiag(
+        loc=array_ops.placeholder(shape=[None, 3], dtype=dtypes.float32),
+        name="MVN2")
+    self.assertEqual(
+        ("tf.distributions.MultivariateNormalDiag("
+         "\"MVN2\", "
+         "batch_shape=(?,), "  # Partially known.
+         "event_shape=(3,), "
+         "dtype=float32)"),
+        str(mvn_dynamic))
+
+  def testReprWorksCorrectlyScalar(self):
+    normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1))
+    self.assertEqual(
+        ("<tf.distributions.Normal"
+         " 'Normal'"
+         " batch_shape=()"
+         " event_shape=()"
+         " dtype=float16>"),  # Got the dtype right.
+        repr(normal))
+
+    chi2 = tfd.Chi2(df=np.float32([1., 2.]), name="silly")
+    self.assertEqual(
+        ("<tf.distributions.Chi2"
+         " 'silly'"  # What a silly name that is!
+         " batch_shape=(2,)"
+         " event_shape=()"
+         " dtype=float32>"),
+        repr(chi2))
+
+    exp = tfd.Exponential(rate=array_ops.placeholder(dtype=dtypes.float32))
+    self.assertEqual(
+        ("<tf.distributions.Exponential"
+         " 'Exponential'"
+         " batch_shape=<unknown>"
+         " event_shape=()"
+         " dtype=float32>"),
+        repr(exp))
+
+  def testReprWorksCorrectlyMultivariate(self):
+    mvn_static = tfd.MultivariateNormalDiag(
+        loc=np.zeros([2, 2]), name="MVN")
+    self.assertEqual(
+        ("<tf.distributions.MultivariateNormalDiag"
+         " 'MVN'"
+         " batch_shape=(2,)"
+         " event_shape=(2,)"
+         " dtype=float64>"),
+        repr(mvn_static))
+
+    mvn_dynamic = tfd.MultivariateNormalDiag(
+        loc=array_ops.placeholder(shape=[None, 3], dtype=dtypes.float32),
+        name="MVN2")
+    self.assertEqual(
+        ("<tf.distributions.MultivariateNormalDiag"
+         " 'MVN2'"
+         " batch_shape=(?,)"  # Partially known.
+         " event_shape=(3,)"
+         " dtype=float32>"),
+        repr(mvn_dynamic))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/distributions/distribution.py b/tensorflow/python/ops/distributions/distribution.py
index 4071e50e81..c055ca43e8 100644
--- a/tensorflow/python/ops/distributions/distribution.py
+++ b/tensorflow/python/ops/distributions/distribution.py
@@ -1105,6 +1105,34 @@ class Distribution(_BaseDistribution):
     with self._name_scope(name):
       return self._kl_divergence(other)
 
+  def __str__(self):
+    return ("tf.distributions.{type_name}("
+            "\"{self_name}\""
+            "{maybe_batch_shape}"
+            "{maybe_event_shape}"
+            ", dtype={dtype})".format(
+                type_name=type(self).__name__,
+                self_name=self.name,
+                maybe_batch_shape=(", batch_shape={}".format(self.batch_shape)
+                                   if self.batch_shape.ndims is not None
+                                   else ""),
+                maybe_event_shape=(", event_shape={}".format(self.event_shape)
+                                   if self.event_shape.ndims is not None
+                                   else ""),
+                dtype=self.dtype.name))
+
+  def __repr__(self):
+    return ("<tf.distributions.{type_name} "
+            "'{self_name}'"
+            " batch_shape={batch_shape}"
+            " event_shape={event_shape}"
+            " dtype={dtype}>".format(
+                type_name=type(self).__name__,
+                self_name=self.name,
+                batch_shape=self.batch_shape,
+                event_shape=self.event_shape,
+                dtype=self.dtype.name))
+
   @contextlib.contextmanager
   def _name_scope(self, name=None, values=None):
     """Helper function to standardize op scope."""
-- 
GitLab


From 0526238462dc39c7b90733102583eea55a0d62bc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 11:02:14 -0700
Subject: [PATCH 435/960] Changes loss_reduction default to SUM_OVER_BATCH_SIZE
 for multi_label_head.

PiperOrigin-RevId: 190244159
---
 .../estimator/python/estimator/head.py        |   7 +-
 .../estimator/python/estimator/head_test.py   | 108 +++++++++---------
 .../python/estimator/multi_head_test.py       |  50 ++++----
 3 files changed, 81 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index 42e1b7b68c..74da2cbb3f 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -304,7 +304,7 @@ def multi_label_head(n_classes,
                      weight_column=None,
                      thresholds=None,
                      label_vocabulary=None,
-                     loss_reduction=losses.Reduction.SUM,
+                     loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE,
                      loss_fn=None,
                      name=None):
   """Creates a `_Head` for multi-label classification.
@@ -355,7 +355,8 @@ def multi_label_head(n_classes,
       string type and have any value in `label_vocabulary`. Also there will be
       errors if vocabulary is not provided and labels are string.
     loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
-      reduce training loss over batch. Defaults to `SUM`.
+      reduce training loss over batch. Defaults to `SUM_OVER_BATCH_SIZE`, namely
+      weighted sum of losses divided by batch size. See `tf.losses.Reduction`.
     loss_fn: Optional loss function.
     name: name of the head. If provided, summary and metrics keys will be
       suffixed by `"/" + name`. Also used as `name_scope` when creating ops.
@@ -404,7 +405,7 @@ class _MultiLabelHead(head_lib._Head):  # pylint:disable=protected-access
                weight_column=None,
                thresholds=None,
                label_vocabulary=None,
-               loss_reduction=losses.Reduction.SUM,
+               loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE,
                loss_fn=None,
                name=None):
     self._n_classes = n_classes
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index 776f0ee341..8837dfdc6c 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -272,9 +272,9 @@ class MultiLabelHead(test.TestCase):
 
     logits = np.array([[-1., 1.], [-1.5, 1.]], dtype=np.float32)
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
-    # loss = labels * -log(sigmoid(logits)) +
-    #        (1 - labels) * -log(1 - sigmoid(logits))
-    expected_training_loss = np.sum(
+    # loss = (labels * -log(sigmoid(logits)) +
+    #         (1 - labels) * -log(1 - sigmoid(logits))) / 2
+    expected_training_loss = 0.5 * np.sum(
         _sigmoid_cross_entropy(labels=labels, logits=logits))
     actual_training_loss = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
@@ -298,7 +298,7 @@ class MultiLabelHead(test.TestCase):
     # For large logits, this is approximated as:
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits
-    expected_training_loss = np.sum(
+    expected_training_loss = 0.5 * np.sum(
         np.array([[(10. + 10.) / 2.], [(15. + 0.) / 2.]], dtype=np.float32))
     actual_training_loss = head.create_loss(
         features={'x': np.array(((42,),), dtype=np.int32)},
@@ -361,7 +361,7 @@ class MultiLabelHead(test.TestCase):
         labels=labels_input)[0]
     with self.test_session():
       _initialize_variables(self, monitored_session.Scaffold())
-      self.assertAllClose(np.sum(loss), actual_training_loss.eval())
+      self.assertAllClose(np.sum(loss) / 2., actual_training_loss.eval())
 
   def test_eval_create_loss_loss_fn_wrong_shape(self):
     """Tests custom loss_fn that returns Tensor of unexpected shape."""
@@ -438,12 +438,13 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Sum over examples.
-    expected_loss = np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
+    # Sum over examples, divide by batch_size.
+    expected_loss = 0.5 * np.sum(
+        _sigmoid_cross_entropy(labels=labels, logits=logits))
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
-        keys.LOSS_MEAN: expected_loss / 2,
+        keys.LOSS_MEAN: expected_loss,
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.3333,
@@ -468,14 +469,13 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Sum over examples.
-    expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
-    )
+    # Sum over examples, divide by batch_size.
+    expected_loss = 0.5 * np.sum(
+        _sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
-        keys.LOSS_MEAN: expected_loss / 2,
+        keys.LOSS_MEAN: expected_loss,
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.3333,
@@ -533,14 +533,13 @@ class MultiLabelHead(test.TestCase):
     labels_multi_hot = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Sum over examples.
-    expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
-    )
+    # Sum over examples, divide by batch_size.
+    expected_loss = 0.5 * np.sum(
+        _sigmoid_cross_entropy(labels=labels_multi_hot, logits=logits))
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
-        keys.LOSS_MEAN: expected_loss / 2,
+        keys.LOSS_MEAN: expected_loss,
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.3333,
@@ -562,15 +561,14 @@ class MultiLabelHead(test.TestCase):
     labels = np.array([[1, 0], [1, 1]], dtype=np.int64)
     # loss = labels * -log(sigmoid(logits)) +
     #        (1 - labels) * -log(1 - sigmoid(logits))
-    # Sum over examples.
-    expected_loss = (
-        np.sum(_sigmoid_cross_entropy(labels=labels, logits=logits))
-    )
+    # Sum over examples, divide by batch_size.
+    expected_loss = 0.5 * np.sum(
+        _sigmoid_cross_entropy(labels=labels, logits=logits))
 
     keys = metric_keys.MetricKeys
     expected_metrics = {
         # Average loss over examples.
-        keys.LOSS_MEAN: expected_loss / 2,
+        keys.LOSS_MEAN: expected_loss,
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.3333,
@@ -603,8 +601,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, weighted sum over examples.
-    expected_loss = 25.
+    # Average over classes, weighted sum over examples, divide by batch_size.
+    # loss = ( 1 * (10 + 10) / 2 + 2 * (15 + 0) / 2) / 2
+    expected_loss = 12.5
 
     spec = head.create_estimator_spec(
         features={
@@ -617,8 +616,8 @@ class MultiLabelHead(test.TestCase):
 
     keys = metric_keys.MetricKeys
     expected_metrics = {
-        # Average loss over weighted examples.
-        keys.LOSS_MEAN: expected_loss / 3,
+        # Average loss over weighted examples (denominator is sum(weights)).
+        keys.LOSS_MEAN: expected_loss * (2. / 3.),
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.2000,
@@ -663,7 +662,7 @@ class MultiLabelHead(test.TestCase):
     #        (1 - labels) * (logits > 0) * logits
     expected_unreduced_loss = [[(10. + 10.) / 2.], [(15. + 0.) / 2.]]
     expected_weights = [[1.], [2.]]
-    expected_training_loss = 1. * (10. + 10.) / 2. + 2. * (15. + 0.) / 2.
+    expected_training_loss = (1. * (10. + 10.) / 2. + 2. * (15. + 0.) / 2.) / 2.
     training_loss, unreduced_loss, actual_weights, _ = head.create_loss(
         features={
             'x': np.array(((42,),), dtype=np.int32),
@@ -809,11 +808,8 @@ class MultiLabelHead(test.TestCase):
       self.assertEqual(
           six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)),
           train_result)
-      _assert_simple_summaries(self, {
-          metric_keys.MetricKeys.LOSS: expected_loss,
-          # Average loss over examples.
-          metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 2,
-      }, summary_str, tol)
+      _assert_simple_summaries(
+          self, {metric_keys.MetricKeys.LOSS: expected_loss}, summary_str, tol)
 
   def test_train(self):
     head = head_lib.multi_label_head(n_classes=2)
@@ -823,8 +819,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # Average over classes, sum over examples, divide by batch_size.
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2
+    expected_loss = 8.75
     self._test_train(
         head=head, logits=logits, labels=labels, expected_loss=expected_loss)
 
@@ -840,8 +837,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # Average over classes, sum over examples, divide by batch_size.
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2
+    expected_loss = 8.75
     self._test_train(
         head=head, logits=logits, labels=labels, expected_loss=expected_loss)
 
@@ -858,8 +856,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # Average over classes, sum over examples, divide by batch_size.
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2
+    expected_loss = 8.75
     self._test_train(
         head=head, logits=logits, labels=labels, expected_loss=expected_loss)
 
@@ -871,8 +870,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # Average over classes, sum over examples, divide by batch_size.
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2
+    expected_loss = 8.75
     expected_train_result = 'my_train_op'
 
     class _Optimizer(object):
@@ -952,8 +952,9 @@ class MultiLabelHead(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, weighted sum over examples.
-    expected_loss = 25.
+    # Average over classes, weighted sum over examples, divide by batch_size.
+    # loss = ( 1 * (10 + 10) / 2 + 2 * (15 + 0) / 2 ) / 2
+    expected_loss = 12.5
     expected_train_result = 'my_train_op'
     def _train_op_fn(loss):
       return string_ops.string_join(
@@ -987,11 +988,8 @@ class MultiLabelHead(test.TestCase):
       self.assertEqual(
           six.b('{0:s}{1:.3f}'.format(expected_train_result, expected_loss)),
           train_result)
-      _assert_simple_summaries(self, {
-          metric_keys.MetricKeys.LOSS: expected_loss,
-          # Average loss over weighted examples.
-          metric_keys.MetricKeys.LOSS_MEAN: expected_loss / 3,
-      }, summary_str, tol)
+      _assert_simple_summaries(
+          self, {metric_keys.MetricKeys.LOSS: expected_loss,}, summary_str, tol)
 
   def test_multi_dim_weighted_train_create_loss(self):
     """Logits and labels of shape [2, 2, 3], weights [2, 2]."""
@@ -1008,8 +1006,8 @@ class MultiLabelHead(test.TestCase):
     expected_unreduced_loss = [[[20./3.], [10./3.]], [[4.], [8.]]]
     # weights are reshaped to [2, 2, 1] to match logits.
     expected_weights = [[[1.], [1.5]], [[2.], [2.5]]]
-    # weighted_sum_loss = 1*20/3 + 1.5*10/3 + 2*4 + 2.5*8 = 39.6667
-    expected_training_loss = 39.6667
+    # loss = (1*20/3 + 1.5*10/3 + 2*4 + 2.5*8) / 4 = 9.9167
+    expected_training_loss = 9.9167
     training_loss, unreduced_loss, actual_weights, _ = head.create_loss(
         features={'weights': weights},
         mode=model_fn.ModeKeys.TRAIN,
@@ -1035,8 +1033,8 @@ class MultiLabelHead(test.TestCase):
     weights = np.array([[1., 1.5], [2., 2.5]], dtype=np.float32)
     # loss = [[10 + 10 + 0, 0 + 0 + 10], [0 + 0 + 12, 12 + 12 + 0]] / 3
     #      = [[20/3, 10/3], [4, 8]]
-    # weighted_sum_loss = 1*20/3 + 1.5*10/3 + 2*4 + 2.5*8 = 39.6667
-    expected_loss = 39.6667
+    # loss = (1*20/3 + 1.5*10/3 + 2*4 + 2.5*8) / 4 = 9.9167
+    expected_loss = 9.9167
     expected_train_result = 'my_train_op'
     def _train_op_fn(loss):
       return string_ops.string_join(
@@ -1124,11 +1122,11 @@ class MultiLabelHead(test.TestCase):
     weights = np.array([[1., 1.5], [2., 2.5]], dtype=np.float32)
     # loss = [[10 + 10 + 0, 0 + 0 + 10], [0 + 0 + 12, 12 + 12 + 0]] / 3
     #      = [[20/3, 10/3], [4, 8]]
-    # weighted_sum_loss = 1*20/3 + 1.5*10/3 + 2*4 + 2.5*8 = 39.6667
-    expected_loss = 39.6667
+    # loss = (1*20/3 + 1.5*10/3 + 2*4 + 2.5*8) / 4 = 9.9167
+    expected_loss = 9.9167
     keys = metric_keys.MetricKeys
     expected_metrics = {
-        keys.LOSS_MEAN: expected_loss / np.sum(weights),
+        keys.LOSS_MEAN: expected_loss * (4. / np.sum(weights)),
         # auc and auc_pr cannot be reliably calculated for only 4 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC: 0.4977,
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
index 43cc157a1f..74d3d6d728 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
@@ -299,10 +299,11 @@ class MultiHeadTest(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # head1: expected_unweighted_loss = [[10., 10.], [15., 0.]]
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2 = 8.75
     # head2: expected_unweighted_loss = [[20., 20., 20.], [30., 0., 0]]
-    # Average over classes, weighted sum over batch and heads.
-    expected_loss_head1 = 17.5
-    expected_loss_head2 = 30.0
+    # loss = ( (20 + 20 + 20) / 3 + (30 + 0 + 0) / 3 ) / 2 = 15
+    expected_loss_head1 = 8.75
+    expected_loss_head2 = 15.
     expected_loss = 1. * expected_loss_head1 + 2. * expected_loss_head2
 
     spec = multi_head.create_estimator_spec(
@@ -316,8 +317,8 @@ class MultiHeadTest(test.TestCase):
         keys.LOSS + '/head1': expected_loss_head1,
         keys.LOSS + '/head2': expected_loss_head2,
         # Average loss over examples.
-        keys.LOSS_MEAN + '/head1': expected_loss_head1 / 2,
-        keys.LOSS_MEAN + '/head2': expected_loss_head2 / 2,
+        keys.LOSS_MEAN + '/head1': expected_loss_head1,
+        keys.LOSS_MEAN + '/head2': expected_loss_head2,
         # auc and auc_pr cannot be reliably calculated for only 4-6 samples, but
         # this assert tests that the algorithm remains consistent.
         keys.AUC + '/head1': 0.1667,
@@ -363,8 +364,8 @@ class MultiHeadTest(test.TestCase):
     tol = 1e-3
     with self.test_session():
       # Unreduced loss of the head is [[(10 + 10) / 2], (15 + 0) / 2]
-      # (averaged over classes, sum-reduced over examples).
-      self.assertAllClose(17.5, loss.eval(), rtol=tol, atol=tol)
+      # (averaged over classes, averaged over examples).
+      self.assertAllClose(8.75, loss.eval(), rtol=tol, atol=tol)
 
   def test_train_create_loss_two_heads_with_weights(self):
     # Use different example weighting for each head weighting.
@@ -399,18 +400,18 @@ class MultiHeadTest(test.TestCase):
     with self.test_session():
       # loss of the first head is [[(10 + 10) / 2], [(15 + 0) / 2]]
       # = [10, 7.5]
-      # training_loss = 1 * 10 + 2 * 7.5 = 25
+      # training_loss = (1 * 10 + 2 * 7.5) / 2 = 12.5
       # head-weighted unreduced_loss = 1 * [10, 7.5]
       self.assertAllClose(
           [[10.], [7.5]], unreduced_losses['head1'].eval(), rtol=tol, atol=tol)
       # loss of the second head is [[(20 + 20 + 20) / 3], [(30 + 0 + 0) / 3]]
       # = [20, 10]
-      # training_loss = 2 * 20 + 3 * 10 = 70
+      # training_loss = (2 * 20 + 3 * 10) / 2 = 35
       # head-weighted unreduced_loss = 2 * [20, 10]
       self.assertAllClose(
           [[40.], [20.]], unreduced_losses['head2'].eval(), rtol=tol, atol=tol)
-      # head-weighted training_loss = 1 * 25 + 2 * 70 = 165
-      self.assertAllClose(165, training_loss.eval(), rtol=tol, atol=tol)
+      # head-weighted training_loss = 1 * 12.5 + 2 * 35 = 82.5
+      self.assertAllClose(82.5, training_loss.eval(), rtol=tol, atol=tol)
       # head-weighted example weights
       self.assertAllClose(
           [[1.], [2.]], weights['head1'].eval(), rtol=tol, atol=tol)
@@ -447,18 +448,18 @@ class MultiHeadTest(test.TestCase):
     with self.test_session():
       # loss of the first head is [[(10 + 10) / 2], [(15 + 0) / 2]]
       # = [10, 7.5]
-      # training_loss = 1 * 10 + 2 * 7.5 = 25
+      # training_loss = (1 * 10 + 2 * 7.5) / 2 = 12.5
       # head-weighted unreduced_loss = 1 * [10, 7.5]
       self.assertAllClose(
           [[10.], [7.5]], unreduced_losses['head1'].eval(), rtol=tol, atol=tol)
       # loss of the second head is [[(20 + 20 + 20) / 3], [(30 + 0 + 0) / 3]]
       # = [20, 10]
-      # training_loss = 2 * 20 + 3 * 10 = 70
+      # training_loss = (2 * 20 + 3 * 10) / 2 = 35
       # head-weighted unreduced_loss = 2 * [20, 10]
       self.assertAllClose(
           [[40.], [20.]], unreduced_losses['head2'].eval(), rtol=tol, atol=tol)
-      # head-weighted training_loss = 1 * 25 + 2 * 70 = 165
-      self.assertAllClose(165, training_loss.eval(), rtol=tol, atol=tol)
+      # head-weighted training_loss = 1 * 12.5 + 2 * 35 = 82.5
+      self.assertAllClose(82.5, training_loss.eval(), rtol=tol, atol=tol)
       # head-weighted example weights
       self.assertAllClose(
           [[1.], [2.]], weights['head1'].eval(), rtol=tol, atol=tol)
@@ -511,8 +512,8 @@ class MultiHeadTest(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2 = 8.75
+    expected_loss = 8.75
     expected_train_result = 'my_train_op'
     def _train_op_fn(loss):
       return string_ops.string_join(
@@ -546,8 +547,6 @@ class MultiHeadTest(test.TestCase):
       _assert_simple_summaries(self, {
           metric_keys.MetricKeys.LOSS: expected_loss,
           metric_keys.MetricKeys.LOSS + '/head1': expected_loss,
-          # Average loss over examples.
-          metric_keys.MetricKeys.LOSS_MEAN + '/head1': expected_loss / 2,
       }, summary_str, tol)
 
   def test_train_one_head_with_optimizer(self):
@@ -560,8 +559,8 @@ class MultiHeadTest(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # expected_unweighted_loss = [[10., 10.], [15., 0.]]
-    # Average over classes, sum over weights.
-    expected_loss = 17.5
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2 = 8.75
+    expected_loss = 8.75
     expected_train_result = 'my_train_op'
 
     class _Optimizer(object):
@@ -607,10 +606,12 @@ class MultiHeadTest(test.TestCase):
     # loss = labels * (logits < 0) * (-logits) +
     #        (1 - labels) * (logits > 0) * logits =>
     # head1: expected_unweighted_loss = [[10., 10.], [15., 0.]]
+    # loss = ( (10 + 10) / 2 + (15 + 0) / 2 ) / 2 = 8.75
     # head2: expected_unweighted_loss = [[20., 20., 20.], [30., 0., 0]]
+    # loss = ( (20 + 20 + 20) / 3 + (30 + 0 + 0) / 3 ) / 2 = 15
     # Average over classes, weighted sum over batch and heads.
-    expected_loss_head1 = 17.5
-    expected_loss_head2 = 30.0
+    expected_loss_head1 = 8.75
+    expected_loss_head2 = 15.0
     expected_loss = 1. * expected_loss_head1 + 2. * expected_loss_head2
     expected_train_result = 'my_train_op'
     def _train_op_fn(loss):
@@ -646,9 +647,6 @@ class MultiHeadTest(test.TestCase):
           metric_keys.MetricKeys.LOSS: expected_loss,
           metric_keys.MetricKeys.LOSS + '/head1': expected_loss_head1,
           metric_keys.MetricKeys.LOSS + '/head2': expected_loss_head2,
-          # Average loss over examples.
-          metric_keys.MetricKeys.LOSS_MEAN + '/head1': expected_loss_head1 / 2,
-          metric_keys.MetricKeys.LOSS_MEAN + '/head2': expected_loss_head2 / 2,
       }, summary_str, tol)
 
 
-- 
GitLab


From 9560504cc802045f3bea35c184627c3f2328f15d Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Fri, 23 Mar 2018 11:05:14 -0700
Subject: [PATCH 436/960] Move security documentation to the main TensorFlow
 site for better visibility, and leave a stub SECURITY.md pointing users
 there.

PiperOrigin-RevId: 190244853
---
 SECURITY.md                                 | 19 +++++++++++++++----
 tensorflow/docs_src/community/index.md      |  3 +++
 tensorflow/docs_src/community/leftnav_files |  1 +
 tensorflow/docs_src/community/security.md   |  7 +++++++
 4 files changed, 26 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/docs_src/community/security.md

diff --git a/SECURITY.md b/SECURITY.md
index 378e776967..5ca304404d 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -168,7 +168,18 @@ below).
 
 Please use a descriptive subject line for your report email. After the initial
 reply to your report, the security team will endeavor to keep you informed of
-the progress being made towards a fix and announcement.
+the progress being made towards a fix and announcement. 
+
+In addition, please include the following information along with your report:
+
+* Your name and affiliation (if any).
+* A description the technical details of the vulnerabilities. It is very
+  important to let us know how we can reproduce your findings.
+* An explanation who can exploit this vulnerability, and what they gain when
+  doing so -- write an attack scenario. This will help us evaluate your report
+  quickly, especially if the issue is complex.
+* Whether this vulnerability public or known to third parties. If it is, please
+  provide details.
 
 If you believe that an existing (public) issue is security-related, please send
 an email to `security@tensorflow.org`. The email should include the issue ID and
@@ -233,7 +244,7 @@ v//Fw6ZeY+HmRDFdirjD7wXtIuER4vqCryIqR6Xe9X8oJXz9L/Jhslc=
 
 ### Known vulnerabilities
 
-| Type              | Versions affected |        Reported by | Additional Information      |
-|-------------------|:-----------------:|--------------------|-----------------------------|
-| out of bounds read|             <=1.4 | TenCent Blade Team | [issue report](https://github.com/tensorflow/tensorflow/issues/14959) |
+| Type               | Versions affected |        Reported by | Additional Information      |
+|--------------------|:-----------------:|--------------------|-----------------------------|
+| Out Of Bounds Read |             <=1.4 | TenCent Blade Team | [issue report](https://github.com/tensorflow/tensorflow/issues/14959) |
 
diff --git a/tensorflow/docs_src/community/index.md b/tensorflow/docs_src/community/index.md
index b706d9b204..ebeff8493b 100644
--- a/tensorflow/docs_src/community/index.md
+++ b/tensorflow/docs_src/community/index.md
@@ -13,3 +13,6 @@ This section contains the following documents:
     conventions that TensorFlow developers and users should follow.
   * @{$community/benchmarks$Benchmarks}, Benchmarks, a guide for defining and
     running a TensorFlow benchmark.
+  * @{$security$Using TensorFlow Securely}, which explains TensorFlow's security
+    model, a list of recent security reports, and information on how you can
+    report a security vulnerability to the TensorFlow team.
diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files
index fab35024ad..af344506c7 100644
--- a/tensorflow/docs_src/community/leftnav_files
+++ b/tensorflow/docs_src/community/leftnav_files
@@ -4,3 +4,4 @@ roadmap.md
 documentation.md
 style_guide.md
 benchmarks.md
+security.md
diff --git a/tensorflow/docs_src/community/security.md b/tensorflow/docs_src/community/security.md
new file mode 100644
index 0000000000..8d13c7a1ea
--- /dev/null
+++ b/tensorflow/docs_src/community/security.md
@@ -0,0 +1,7 @@
+# Using TensorFlow Securely
+
+Before using TensorFlow, please take a look at our security model, list of
+recent security announcements, and ways you can report security issues to the
+TensorFlow team at the
+[https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md](Using
+TensorFlow Securely) page on GitHub.
-- 
GitLab


From 99f39f590128ca1d5e1d1645c69a147938e481bc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 11:27:56 -0700
Subject: [PATCH 437/960] Move NullFileSystem to its own file.

PiperOrigin-RevId: 190249020
---
 tensorflow/core/BUILD                         |  1 +
 .../common_runtime/constant_folding_test.cc   |  1 +
 .../kernels/immutable_constant_op_test.cc     |  1 +
 tensorflow/core/platform/env_test.cc          |  1 +
 tensorflow/core/platform/file_system.h        | 68 -------------
 tensorflow/core/platform/file_system_test.cc  |  1 +
 tensorflow/core/platform/null_file_system.h   | 98 +++++++++++++++++++
 .../python/framework/test_file_system.cc      |  1 +
 8 files changed, 104 insertions(+), 68 deletions(-)
 create mode 100644 tensorflow/core/platform/null_file_system.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 2885a9f823..1d11410332 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -354,6 +354,7 @@ cc_library(
         "platform/mutex.h",
         "platform/net.h",
         "platform/notification.h",
+        "platform/null_file_system.h",
         "platform/prefetch.h",
         "platform/profile_utils/clock_cycle_profiler.h",
         "platform/profile_utils/cpu_utils.h",
diff --git a/tensorflow/core/common_runtime/constant_folding_test.cc b/tensorflow/core/common_runtime/constant_folding_test.cc
index 6ac9319ad1..16b61315f2 100644
--- a/tensorflow/core/common_runtime/constant_folding_test.cc
+++ b/tensorflow/core/common_runtime/constant_folding_test.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/null_file_system.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session_options.h"
 
diff --git a/tensorflow/core/kernels/immutable_constant_op_test.cc b/tensorflow/core/kernels/immutable_constant_op_test.cc
index b3814331ee..b2dc16d5d7 100644
--- a/tensorflow/core/kernels/immutable_constant_op_test.cc
+++ b/tensorflow/core/kernels/immutable_constant_op_test.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/null_file_system.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/public/session.h"
diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc
index 47ddf0ccb9..9a6ff48069 100644
--- a/tensorflow/core/platform/env_test.cc
+++ b/tensorflow/core/platform/env_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/null_file_system.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 
diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h
index 03c0c5ab51..8f99766e15 100644
--- a/tensorflow/core/platform/file_system.h
+++ b/tensorflow/core/platform/file_system.h
@@ -305,74 +305,6 @@ class ReadOnlyMemoryRegion {
   virtual uint64 length() = 0;
 };
 
-// START_SKIP_DOXYGEN
-
-#ifndef SWIG
-// Degenerate file system that provides no implementations.
-class NullFileSystem : public FileSystem {
- public:
-  NullFileSystem() {}
-
-  ~NullFileSystem() override = default;
-
-  Status NewRandomAccessFile(
-      const string& fname, std::unique_ptr<RandomAccessFile>* result) override {
-    return errors::Unimplemented("NewRandomAccessFile unimplemented");
-  }
-
-  Status NewWritableFile(const string& fname,
-                         std::unique_ptr<WritableFile>* result) override {
-    return errors::Unimplemented("NewWritableFile unimplemented");
-  }
-
-  Status NewAppendableFile(const string& fname,
-                           std::unique_ptr<WritableFile>* result) override {
-    return errors::Unimplemented("NewAppendableFile unimplemented");
-  }
-
-  Status NewReadOnlyMemoryRegionFromFile(
-      const string& fname,
-      std::unique_ptr<ReadOnlyMemoryRegion>* result) override {
-    return errors::Unimplemented(
-        "NewReadOnlyMemoryRegionFromFile unimplemented");
-  }
-
-  Status FileExists(const string& fname) override {
-    return errors::Unimplemented("FileExists unimplemented");
-  }
-
-  Status GetChildren(const string& dir, std::vector<string>* result) override {
-    return errors::Unimplemented("GetChildren unimplemented");
-  }
-
-  Status DeleteFile(const string& fname) override {
-    return errors::Unimplemented("DeleteFile unimplemented");
-  }
-
-  Status CreateDir(const string& dirname) override {
-    return errors::Unimplemented("CreateDir unimplemented");
-  }
-
-  Status DeleteDir(const string& dirname) override {
-    return errors::Unimplemented("DeleteDir unimplemented");
-  }
-
-  Status GetFileSize(const string& fname, uint64* file_size) override {
-    return errors::Unimplemented("GetFileSize unimplemented");
-  }
-
-  Status RenameFile(const string& src, const string& target) override {
-    return errors::Unimplemented("RenameFile unimplemented");
-  }
-
-  Status Stat(const string& fname, FileStatistics* stat) override {
-    return errors::Unimplemented("Stat unimplemented");
-  }
-};
-#endif
-
-// END_SKIP_DOXYGEN
-
 /// \brief A registry for file system implementations.
 ///
 /// Filenames are specified as an URI, which is of the form
diff --git a/tensorflow/core/platform/file_system_test.cc b/tensorflow/core/platform/file_system_test.cc
index abe88ab6c7..e07aad55cb 100644
--- a/tensorflow/core/platform/file_system_test.cc
+++ b/tensorflow/core/platform/file_system_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
+#include "tensorflow/core/platform/null_file_system.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
diff --git a/tensorflow/core/platform/null_file_system.h b/tensorflow/core/platform/null_file_system.h
new file mode 100644
index 0000000000..008e6d54d0
--- /dev/null
+++ b/tensorflow/core/platform/null_file_system.h
@@ -0,0 +1,98 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_NULL_FILE_SYSTEM_H_
+#define TENSORFLOW_CORE_PLATFORM_NULL_FILE_SYSTEM_H_
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system.h"
+
+namespace tensorflow {
+
+// START_SKIP_DOXYGEN
+
+#ifndef SWIG
+// Degenerate file system that provides no implementations.
+class NullFileSystem : public FileSystem {
+ public:
+  NullFileSystem() {}
+
+  ~NullFileSystem() override = default;
+
+  Status NewRandomAccessFile(
+      const string& fname, std::unique_ptr<RandomAccessFile>* result) override {
+    return errors::Unimplemented("NewRandomAccessFile unimplemented");
+  }
+
+  Status NewWritableFile(const string& fname,
+                         std::unique_ptr<WritableFile>* result) override {
+    return errors::Unimplemented("NewWritableFile unimplemented");
+  }
+
+  Status NewAppendableFile(const string& fname,
+                           std::unique_ptr<WritableFile>* result) override {
+    return errors::Unimplemented("NewAppendableFile unimplemented");
+  }
+
+  Status NewReadOnlyMemoryRegionFromFile(
+      const string& fname,
+      std::unique_ptr<ReadOnlyMemoryRegion>* result) override {
+    return errors::Unimplemented(
+        "NewReadOnlyMemoryRegionFromFile unimplemented");
+  }
+
+  Status FileExists(const string& fname) override {
+    return errors::Unimplemented("FileExists unimplemented");
+  }
+
+  Status GetChildren(const string& dir, std::vector<string>* result) override {
+    return errors::Unimplemented("GetChildren unimplemented");
+  }
+
+  Status DeleteFile(const string& fname) override {
+    return errors::Unimplemented("DeleteFile unimplemented");
+  }
+
+  Status CreateDir(const string& dirname) override {
+    return errors::Unimplemented("CreateDir unimplemented");
+  }
+
+  Status DeleteDir(const string& dirname) override {
+    return errors::Unimplemented("DeleteDir unimplemented");
+  }
+
+  Status GetFileSize(const string& fname, uint64* file_size) override {
+    return errors::Unimplemented("GetFileSize unimplemented");
+  }
+
+  Status RenameFile(const string& src, const string& target) override {
+    return errors::Unimplemented("RenameFile unimplemented");
+  }
+
+  Status Stat(const string& fname, FileStatistics* stat) override {
+    return errors::Unimplemented("Stat unimplemented");
+  }
+};
+#endif
+
+// END_SKIP_DOXYGEN
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PLATFORM_NULL_FILE_SYSTEM_H_
diff --git a/tensorflow/python/framework/test_file_system.cc b/tensorflow/python/framework/test_file_system.cc
index 094ea6f658..6e9915adbb 100644
--- a/tensorflow/python/framework/test_file_system.cc
+++ b/tensorflow/python/framework/test_file_system.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/null_file_system.h"
 
 namespace tensorflow {
 
-- 
GitLab


From 1ece83d0cc98d09447e9a142605934328e18d15a Mon Sep 17 00:00:00 2001
From: Yuanzhong Xu <yuanzx@google.com>
Date: Fri, 23 Mar 2018 11:41:05 -0700
Subject: [PATCH 438/960] Using rounding when converting constants in BF16
 propagation.

PiperOrigin-RevId: 190251081
---
 tensorflow/compiler/xla/literal_util.cc       | 24 +++++++++++++++----
 tensorflow/compiler/xla/literal_util.h        |  8 ++++++-
 .../xla/service/bfloat16_propagation.cc       |  6 +++--
 3 files changed, 31 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/literal_util.cc b/tensorflow/compiler/xla/literal_util.cc
index 214c2030cd..13675b7d00 100644
--- a/tensorflow/compiler/xla/literal_util.cc
+++ b/tensorflow/compiler/xla/literal_util.cc
@@ -1385,8 +1385,9 @@ void Literal::EachCellAsString(
 }
 
 namespace {
-template <typename NativeSrcT, typename NativeDestT>
-std::unique_ptr<Literal> ConvertBetweenNativeTypes(const Literal& src_literal) {
+template <typename NativeSrcT, typename NativeDestT, typename ConverterType>
+std::unique_ptr<Literal> ConvertBetweenNativeTypesWithConverter(
+    const Literal& src_literal, const ConverterType& converter) {
   CHECK(ShapeUtil::IsArray(src_literal.shape()));
   auto result_literal = MakeUnique<Literal>(ShapeUtil::ChangeElementType(
       src_literal.shape(),
@@ -1396,11 +1397,18 @@ std::unique_ptr<Literal> ConvertBetweenNativeTypes(const Literal& src_literal) {
   int64 num_elements = src_literal.element_count();
 
   for (int64 i = 0; i < num_elements; ++i) {
-    dest_data[i] = static_cast<NativeDestT>(src_data[i]);
+    dest_data[i] = converter(src_data[i]);
   }
   return result_literal;
 }
 
+template <typename NativeSrcT, typename NativeDestT>
+std::unique_ptr<Literal> ConvertBetweenNativeTypes(const Literal& src_literal) {
+  auto converter = [](NativeSrcT src) { return static_cast<NativeDestT>(src); };
+  return ConvertBetweenNativeTypesWithConverter<NativeSrcT, NativeDestT>(
+      src_literal, converter);
+}
+
 template <PrimitiveType primitive_src_type>
 std::unique_ptr<Literal> ConvertToC64(const Literal& src_literal) {
   CHECK(ShapeUtil::IsArray(src_literal.shape()));
@@ -1492,8 +1500,16 @@ StatusOr<std::unique_ptr<Literal>> Literal::Convert(
 }
 
 StatusOr<std::unique_ptr<Literal>> Literal::ConvertToShape(
-    const Shape& dest_shape) const {
+    const Shape& dest_shape, bool round_f32_to_bf16) const {
   if (!ShapeUtil::IsTuple(dest_shape)) {
+    if (round_f32_to_bf16 && shape().element_type() == F32 &&
+        dest_shape.element_type() == BF16) {
+      auto converter = [](float src) {
+        return tensorflow::bfloat16::round_to_bfloat16(src);
+      };
+      return ConvertBetweenNativeTypesWithConverter<float, bfloat16>(*this,
+                                                                     converter);
+    }
     return Convert(dest_shape.element_type());
   }
   std::vector<Literal> elements;
diff --git a/tensorflow/compiler/xla/literal_util.h b/tensorflow/compiler/xla/literal_util.h
index e24f5285d9..a96a76fbb4 100644
--- a/tensorflow/compiler/xla/literal_util.h
+++ b/tensorflow/compiler/xla/literal_util.h
@@ -340,8 +340,14 @@ class Literal {
 
   // Converts this literal to the given shape. Returns an error is the
   // conversion is not possible.
+  //
+  // round_f32_to_bf16: if true, converting F32 elements to BF16 uses rounding
+  // instead of truncation; otherwise, truncation is used.
+  //
+  // TODO(b/69266521): remove the round_to_bfloat16 flag when rounding becomes
+  // the default behavior.
   StatusOr<std::unique_ptr<Literal>> ConvertToShape(
-      const Shape& dest_shape) const;
+      const Shape& dest_shape, bool round_f32_to_bf16 = false) const;
 
   // Creates a scalar literal value zero of the given primitive type.
   static Literal Zero(PrimitiveType primitive_type);
diff --git a/tensorflow/compiler/xla/service/bfloat16_propagation.cc b/tensorflow/compiler/xla/service/bfloat16_propagation.cc
index 7195c31d9c..c26d2feef5 100644
--- a/tensorflow/compiler/xla/service/bfloat16_propagation.cc
+++ b/tensorflow/compiler/xla/service/bfloat16_propagation.cc
@@ -606,8 +606,10 @@ Status BFloat16Propagation::ResolveInconsistencyOfAliasingBuffers(
         continue;
       }
       if (!ShapeUtil::Equal(hlo->literal().shape(), hlo->shape())) {
-        TF_ASSIGN_OR_RETURN(auto converted_literal,
-                            hlo->literal().ConvertToShape(hlo->shape()));
+        TF_ASSIGN_OR_RETURN(
+            auto converted_literal,
+            hlo->literal().ConvertToShape(hlo->shape(),
+                                          /*round_f32_to_bf16=*/true));
         auto new_constant = computation->AddInstruction(
             HloInstruction::CreateConstant(std::move(converted_literal)));
         TF_RETURN_IF_ERROR(hlo->ReplaceAllUsesWith(new_constant));
-- 
GitLab


From d11dc5f7eefab6eb5590cee616bab8af2786e273 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Fri, 23 Mar 2018 12:02:57 -0700
Subject: [PATCH 439/960] [XLA] Allow reshape mover to move transpose across
 broadcast of a scalar value.

This allows the simplification of pattern "transpose elementwise-ops inversed
transpose" to "elementwise-ops".

Add a test case.

PiperOrigin-RevId: 190254501
---
 .../compiler/xla/service/reshape_mover.cc     | 29 +++++++++++++++++++
 .../xla/service/reshape_mover_test.cc         | 20 +++++++++++++
 2 files changed, 49 insertions(+)

diff --git a/tensorflow/compiler/xla/service/reshape_mover.cc b/tensorflow/compiler/xla/service/reshape_mover.cc
index e62bafc50b..f15117f45c 100644
--- a/tensorflow/compiler/xla/service/reshape_mover.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover.cc
@@ -53,6 +53,14 @@ bool IsReshapeOrTranspose(const HloInstruction* instruction) {
          instruction->opcode() == HloOpcode::kTranspose;
 }
 
+// Returns true if `a` is a broadcast instruction to target shape `shape` and
+// its operand is a scalar.
+bool IsBroadcastScalarToShape(const HloInstruction* a, const Shape& shape) {
+  return a->opcode() == HloOpcode::kBroadcast &&
+         ShapeUtil::SameDimensions(a->shape(), shape) &&
+         ShapeUtil::IsScalar(a->operand(0)->shape());
+}
+
 // Returns true iff `instruction` can change its shape simply by adjusting
 // metadata.
 bool CanTriviallyChangeShape(const HloInstruction* instruction) {
@@ -88,6 +96,7 @@ bool CanTriviallyChangeShape(const HloInstruction* instruction) {
       instruction->user_count() == 1) {
     return true;
   }
+
   return false;
 }
 
@@ -148,6 +157,8 @@ bool AllOperandsHaveEasyShapeChanges(
   //     or
   //    2. Are one of kConstant, kRng, and scalars that can change shape
   //    trivially,
+  //     or
+  //    3. Are broadcast with a scalar operand.
   for (const HloInstruction* operand : instruction->operands()) {
     if (!ShapeUtil::SameDimensions(operand->shape(), instruction->shape())) {
       VLOG(5) << "Operand shape differs from output shape; may be "
@@ -158,6 +169,12 @@ bool AllOperandsHaveEasyShapeChanges(
       return false;
     }
 
+    // Skip the rest checks if the current operand is first_reshape_operand
+    // itself.
+    if (first_reshape_operand == operand) {
+      continue;
+    }
+
     if (AreEquivalentReshapes(first_reshape_operand, operand)) {
       VLOG(5) << "Are equivalent reshapes:\n\tfirst_reshape_operand: "
               << first_reshape_operand->ToString(print_no_metadata)
@@ -171,6 +188,12 @@ bool AllOperandsHaveEasyShapeChanges(
       continue;
     }
 
+    if (IsBroadcastScalarToShape(operand, first_reshape_operand->shape())) {
+      VLOG(5) << "Broadcast scalar to shape: "
+              << operand->ToString(print_no_metadata);
+      continue;
+    }
+
     // TODO(someone): Look into supporting general ops for the operands as
     // well.
     VLOG(5) << "Operand is neither equalivant to the first Reshape operand"
@@ -222,6 +245,12 @@ HloInstruction* UpdateOperand(HloComputation* computation,
       VLOG(5) << "Using existing operand of kReshape or kTranspose";
       return operand->mutable_operand(0);
     }
+    case HloOpcode::kBroadcast:
+      CHECK(IsBroadcastScalarToShape(operand, first_reshape_operand->shape()));
+      VLOG(5) << "Changing broadcast";
+      return computation->AddInstruction(
+          operand->CloneWithNewOperands(new_shape, operand->operands()));
+
     default:
       LOG(FATAL) << "Unexpected operand opcode during update: " << operand;
   }
diff --git a/tensorflow/compiler/xla/service/reshape_mover_test.cc b/tensorflow/compiler/xla/service/reshape_mover_test.cc
index aac8638a54..4e0a0a8832 100644
--- a/tensorflow/compiler/xla/service/reshape_mover_test.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover_test.cc
@@ -560,5 +560,25 @@ TEST_F(ReshapeMoverTest, MultiplePasses) {
       op::Reshape(op::Add(param2, op::Reshape(op::Add(param0, param1)))));
 }
 
+TEST_F(ReshapeMoverTest, SinkTransposeAcrossBroadcastScalar) {
+  const string hlo_string = R"(
+    HloModule TransposeMulInversedTransposeModule
+    ENTRY TransposeMulInversedTranspose {
+      src0 = f32[1,20,8,32]{3,2,1,0} parameter(0)
+      transpose0 = f32[1,8,20,32]{3,2,1,0} transpose(src0), dimensions={0,2,1,3}
+      src1 = f32[] parameter(1)
+      broadcast0 = f32[1,8,20,32]{3,2,1,0} broadcast(src1), dimensions={}
+      ROOT multiply0 = f32[1,8,20,32]{3,2,1,0} multiply(transpose0, broadcast0)
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string.c_str());
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, ReshapeMover().Run(&module()));
+  EXPECT_TRUE(changed);
+
+  EXPECT_THAT(module().entry_computation()->root_instruction(),
+              op::Transpose(op::Multiply()));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 216a023dffacc57b429ef8c01c59fc0ab03c3eee Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Fri, 23 Mar 2018 12:20:12 -0700
Subject: [PATCH 440/960] [XLA:CPU] Task parallel assignment should not
 parallelize kRng, kInfeed, and kOutfeed.

Add two test cases.

PiperOrigin-RevId: 190257131
---
 .../service/cpu/parallel_task_assignment.cc   |  4 ++-
 .../cpu/parallel_task_assignment_test.cc      | 34 +++++++++++++++++++
 2 files changed, 37 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
index 86e8be8461..fb28280fad 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment.cc
@@ -128,6 +128,7 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
   // one of the following properties:
   // *) Internal threading (library calls to kConv, kDot, kFft, kCustomCall).
   // *) Emit custom loops (kSelectAndScatter, FusionKind::kTransposeDot).
+  // *) Operations that are not thread safe (like infeed and rng).
   // *) Tuple-shaped.
   // TODO(b/27458679) Parallelize instructions which are skipped here.
   auto opcode = instruction->opcode();
@@ -135,7 +136,8 @@ int64 ParallelTaskAssignment::GetTargetParallelTaskCount(
       opcode == HloOpcode::kCall || opcode == HloOpcode::kCustomCall ||
       opcode == HloOpcode::kDot || opcode == HloOpcode::kSelectAndScatter ||
       opcode == HloOpcode::kGetTupleElement || opcode == HloOpcode::kBitcast ||
-      opcode == HloOpcode::kFft ||
+      opcode == HloOpcode::kFft || opcode == HloOpcode::kInfeed ||
+      opcode == HloOpcode::kOutfeed || opcode == HloOpcode::kRng ||
       (opcode == HloOpcode::kConvolution &&
        PotentiallyImplementedAsEigenConvolution(*instruction)) ||
       PotentiallyImplementedAsEigenDot(*instruction) ||
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
index 90191221eb..13eb75a572 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/parallel_task_assignment_test.cc
@@ -80,5 +80,39 @@ TEST_F(ParallelTaskAssignmentTest,
   EXPECT_FALSE(changed);
 }
 
+TEST_F(ParallelTaskAssignmentTest, RngOperationNotParallelized) {
+  const string hlo_string = R"(
+    HloModule TestTaskParallel_rng
+    ENTRY Rng {
+      src0 = f32[] parameter(0)
+      src1 = f32[] parameter(1)
+      ROOT rng0 = f32[1234567,2]{1,0} rng(f32[] src0, f32[] src1),
+      distribution=rng_uniform
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string);
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
+                                            max_parallelism_, shape_size_func_)
+                                            .Run(&module()));
+  EXPECT_FALSE(changed);
+}
+
+TEST_F(ParallelTaskAssignmentTest, InfeedOutfeedOperationNotParallelized) {
+  const string hlo_string = R"(
+    HloModule TestTaskParallel_infeed_outfeed
+    ENTRY InfeedOutfeed {
+      infeed0 = u32[12345678,2]{1,0} infeed()
+      ROOT outfeed0 = u32[12345678,2]{1,0} outfeed(infeed0)
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string);
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, cpu::ParallelTaskAssigner(
+                                            max_parallelism_, shape_size_func_)
+                                            .Run(&module()));
+  EXPECT_FALSE(changed);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e284c9fbe69119c452773b007e6c8843d223a8ec Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 12:32:42 -0700
Subject: [PATCH 441/960] [XLA] Redesign: add public methods that create XlaOp.

This is basically copying and pasting the public builder methods from ComputationBuilder and leave the implemention blank.

PiperOrigin-RevId: 190258687
---
 .../compiler/xla/client/xla_client/BUILD      |   1 +
 .../xla/client/xla_client/xla_builder.cc      | 435 ++++++++++++
 .../xla/client/xla_client/xla_builder.h       | 662 +++++++++++++++++-
 3 files changed, 1088 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/BUILD b/tensorflow/compiler/xla/client/xla_client/BUILD
index b912889e26..69df15c988 100644
--- a/tensorflow/compiler/xla/client/xla_client/BUILD
+++ b/tensorflow/compiler/xla/client/xla_client/BUILD
@@ -38,6 +38,7 @@ cc_library(
         "//tensorflow/compiler/xla:types",
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/client:padding",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_proto",
         "//tensorflow/compiler/xla/service:shape_inference",
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 82b61d4d51..ec646cabe9 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -217,6 +217,436 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
   return NoteErrorOrReturn(op());
 }
 
+XlaOp XlaBuilder::Slice(const XlaOp& operand,
+                        tensorflow::gtl::ArraySlice<int64> start_indices,
+                        tensorflow::gtl::ArraySlice<int64> limit_indices,
+                        tensorflow::gtl::ArraySlice<int64> strides) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::SliceInDim(const XlaOp& operand, int64 start_index,
+                             int64 limit_index, int64 stride, int64 dimno) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
+                               tensorflow::gtl::ArraySlice<int64> slice_sizes) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                                     const XlaOp& start_indices) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConcatInDim(tensorflow::gtl::ArraySlice<XlaOp> operands,
+                              int64 dimension) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value,
+                      const PaddingConfig& padding_config) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Reshape(const XlaOp& operand,
+                          tensorflow::gtl::ArraySlice<int64> dimensions,
+                          tensorflow::gtl::ArraySlice<int64> new_sizes) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Reshape(const XlaOp& operand,
+                          tensorflow::gtl::ArraySlice<int64> new_sizes) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Collapse(const XlaOp& operand,
+                           tensorflow::gtl::ArraySlice<int64> dimensions) {
+  return UnimplementedOp();
+}
+
+void XlaBuilder::Trace(const string& tag, const XlaOp& operand) {
+  UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Select(const XlaOp& pred, const XlaOp& on_true,
+                         const XlaOp& on_false) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Eq(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Ne(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Ge(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Gt(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Le(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Lt(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Dot(const XlaOp& lhs, const XlaOp& rhs) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
+                             const DotDimensionNumbers& dimension_numbers) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Conv(const XlaOp& lhs, const XlaOp& rhs,
+                       tensorflow::gtl::ArraySlice<int64> window_strides,
+                       Padding padding) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConvWithGeneralPadding(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> window_strides,
+    tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConvWithGeneralDimensions(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding,
+    const ConvolutionDimensionNumbers& dimension_numbers) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConvGeneral(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> window_strides,
+    tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+    const ConvolutionDimensionNumbers& dimension_numbers) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConvGeneralDilated(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> window_strides,
+    tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+    tensorflow::gtl::ArraySlice<int64> lhs_dilation,
+    tensorflow::gtl::ArraySlice<int64> rhs_dilation,
+    const ConvolutionDimensionNumbers& dimension_numbers) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Fft(const XlaOp& operand, const FftType fft_type,
+                      const tensorflow::gtl::ArraySlice<int64> fft_length) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Infeed(const Shape& shape, const string& config) {
+  return UnimplementedOp();
+}
+
+void XlaBuilder::Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
+                         const string& outfeed_config) {
+  UnimplementedOp();
+}
+
+XlaOp XlaBuilder::CustomCall(const string& call_target_name,
+                             tensorflow::gtl::ArraySlice<XlaOp> operands,
+                             const Shape& shape) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::HostCompute(tensorflow::gtl::ArraySlice<XlaOp> operands,
+                              const string& channel_name,
+                              int64 cost_estimate_ns, const Shape& shape) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Complex(
+    const XlaOp& real, const XlaOp& imag,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Conj(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Sub(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Div(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Rem(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Max(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Min(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::And(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Or(const XlaOp& lhs, const XlaOp& rhs,
+                     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Xor(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Not(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::ShiftLeft(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ShiftRightArithmetic(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ShiftRightLogical(
+    const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Abs(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Atan2(
+    const XlaOp& y, const XlaOp& x,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Exp(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Floor(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Ceil(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Round(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Log(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Sign(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Cos(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Sin(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Tanh(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Real(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Imag(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::IsFinite(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Transpose(const XlaOp& operand,
+                            tensorflow::gtl::ArraySlice<int64> permutation) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Rev(const XlaOp& operand,
+                      tensorflow::gtl::ArraySlice<int64> dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Sort(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::SqrtF32(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Pow(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand,
+                                     PrimitiveType new_element_type) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand,
+                                     PrimitiveType new_element_type) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::SquareF32(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::ReciprocalF32(const XlaOp& operand) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Neg(const XlaOp& operand) { return UnimplementedOp(); }
+
+XlaOp XlaBuilder::Clamp(const XlaOp& min, const XlaOp& operand,
+                        const XlaOp& max) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Map(tensorflow::gtl::ArraySlice<XlaOp> operands,
+                      const XlaComputation& computation,
+                      tensorflow::gtl::ArraySlice<int64> dimensions,
+                      tensorflow::gtl::ArraySlice<XlaOp> static_operands) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::RngNormal(const XlaOp& mu, const XlaOp& sigma,
+                            const Shape& shape) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::RngUniform(const XlaOp& a, const XlaOp& b,
+                             const Shape& shape) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::While(const XlaComputation& condition,
+                        const XlaComputation& body, const XlaOp& init) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Gather(const XlaOp& input, const XlaOp& gather_indices,
+                         const GatherDimensionNumbers& dimension_numbers,
+                         tensorflow::gtl::ArraySlice<int64> window_bounds) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Conditional(const XlaOp& predicate, const XlaOp& true_operand,
+                              const XlaComputation& true_computation,
+                              const XlaOp& false_operand,
+                              const XlaComputation& false_computation) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Reduce(
+    const XlaOp& operand, const XlaOp& init_value,
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<int64> dimensions_to_reduce) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ReduceAll(const XlaOp& operand, const XlaOp& init_value,
+                            const XlaComputation& computation) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ReduceWindow(
+    const XlaOp& operand, const XlaOp& init_value,
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ReduceWindowWithGeneralPadding(
+    const XlaOp& operand, const XlaOp& init_value,
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides,
+    tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::BatchNormTraining(const XlaOp& operand, const XlaOp& scale,
+                                    const XlaOp& offset, float epsilon,
+                                    int64 feature_index) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::BatchNormInference(const XlaOp& operand, const XlaOp& scale,
+                                     const XlaOp& offset, const XlaOp& mean,
+                                     const XlaOp& variance, float epsilon,
+                                     int64 feature_index) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
+                                const XlaOp& batch_mean, const XlaOp& batch_var,
+                                const XlaOp& grad_output, float epsilon,
+                                int64 feature_index) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::CrossReplicaSum(const XlaOp& operand) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::SelectAndScatter(
+    const XlaOp& operand, const XlaComputation& select,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding,
+    const XlaOp& source, const XlaOp& init_value,
+    const XlaComputation& scatter) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::SelectAndScatterWithGeneralPadding(
+    const XlaOp& operand, const XlaComputation& select,
+    tensorflow::gtl::ArraySlice<int64> window_dimensions,
+    tensorflow::gtl::ArraySlice<int64> window_strides,
+    tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+    const XlaOp& source, const XlaOp& init_value,
+    const XlaComputation& scatter) {
+  return UnimplementedOp();
+}
+
+XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits,
+                                  const int mantissa_bits) {
+  return UnimplementedOp();
+}
+
+void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) {
+  UnimplementedOp();
+}
+
+XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
+  return UnimplementedOp();
+}
+
 XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
                                  tensorflow::gtl::ArraySlice<XlaOp> operands) {
   const int64 handle = instructions_.size();
@@ -246,4 +676,9 @@ StatusOr<const HloInstructionProto*> XlaBuilder::LookUpInstruction(
   return &instructions_[op.handle()];
 }
 
+XlaOp XlaBuilder::UnimplementedOp() {
+  NoteError(Unimplemented("Op not yet implemented"));
+  return {};
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index f1d10ecdb9..455ea3d9cc 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -24,6 +24,7 @@ limitations under the License.
 #include <string>
 #include <utility>
 
+#include "tensorflow/compiler/xla/client/padding.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -121,14 +122,6 @@ class XlaBuilder {
     die_immediately_on_error_ = enabled;
   }
 
-  // Enqueues an add instruction onto the computation.
-  XlaOp Add(const XlaOp& lhs, const XlaOp& rhs,
-            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
-
-  // Enqueues a call instruction onto the computation.
-  XlaOp Call(const XlaComputation& computation,
-             tensorflow::gtl::ArraySlice<XlaOp> operands);
-
   // Enqueues a "retrieve parameter value" instruction for a parameter that was
   // passed to the computation.
   XlaOp Parameter(int64 parameter_number, const Shape& shape,
@@ -156,14 +149,590 @@ class XlaBuilder {
   // corresponding native type yet.
   template <typename NativeT>
   XlaOp ConstantR0(NativeT value);
+  template <typename NativeT>
+  XlaOp ConstantR1(tensorflow::gtl::ArraySlice<NativeT> values);
+  XlaOp ConstantR1(const tensorflow::core::Bitmap& values);
+  template <typename NativeT>
+  XlaOp ConstantR2(
+      std::initializer_list<std::initializer_list<NativeT>> values);
+  template <typename NativeT>
+  XlaOp ConstantFromArrayWithLayout(const Array<NativeT>& values,
+                                    const Layout& layout);
+  template <typename NativeT>
+  XlaOp ConstantFromArray(const Array<NativeT>& values);
+  template <typename NativeT>
+  XlaOp ConstantR2FromArray2DWithLayout(const Array2D<NativeT>& values,
+                                        const Layout& layout);
+  template <typename NativeT>
+  XlaOp ConstantR2FromArray2D(const Array2D<NativeT>& values);
+  template <typename NativeT>
+  XlaOp ConstantR3FromArray3DWithLayout(const Array3D<NativeT>& values,
+                                        const Layout& layout);
+  template <typename NativeT>
+  XlaOp ConstantR3FromArray3D(const Array3D<NativeT>& values);
+  template <typename NativeT>
+  XlaOp ConstantR4FromArray4DWithLayout(const Array4D<NativeT>& values,
+                                        const Layout& layout);
+  template <typename NativeT>
+  XlaOp ConstantR4FromArray4D(const Array4D<NativeT>& values);
 
-  // Returns the shape of the given op.
-  StatusOr<Shape> GetShape(const XlaOp& op) const;
+  // Enqueues a rank one constant (vector) onto the computation. The vector has
+  // size 'length' and every element has the value 'value'.
+  template <typename NativeT>
+  XlaOp ConstantR1(int64 length, NativeT value);
+
+  // Adds dimensions to an array by duplicating the data in the array.
+  //
+  // The new dimensions are inserted on the left, i.e. if
+  // broadcast_sizes has values {a0, ..., aN} and the operand shape
+  // has dimensions {b0, ..., bM} then the shape of the output has
+  // dimensions {a0, ..., aN, b0, ..., bM}.
+  //
+  // The new dimensions index into copies of the operand, i.e.
+  //
+  //   output[i0, ..., iN, j0, ..., jM] = operand[j0, ..., jM]
+  XlaOp Broadcast(const XlaOp& operand,
+                  tensorflow::gtl::ArraySlice<int64> broadcast_sizes);
+
+  // Enqueues a pad operation onto the computation that pads the given value on
+  // the edges as well as between the elements of the input. padding_config
+  // specifies the padding amount for each dimension.
+  XlaOp Pad(const XlaOp& operand, const XlaOp& padding_value,
+            const PaddingConfig& padding_config);
+
+  // Enqueues an operation onto the computation that flattens the operand based
+  // on the dimension order (major/slowest-varying to minor/fastest-varying)
+  // given, followed by reshaping it into the shape with the given dimension
+  // sizes (also major to minor). Conceptually, this is a limited form of
+  // "shape casting".
+  XlaOp Reshape(const XlaOp& operand,
+                tensorflow::gtl::ArraySlice<int64> dimensions,
+                tensorflow::gtl::ArraySlice<int64> new_sizes);
+
+  // Enqueues an operation onto the computation that collapses the operand, from
+  // first to last dimension (C order), then reshapes it to the given dimension
+  // sizes. Conceptually, this is a limited form of "shape casting".
+  XlaOp Reshape(const XlaOp& operand,
+                tensorflow::gtl::ArraySlice<int64> new_sizes);
+
+  // Wrapper for Reshape.
+  // Enqueues an operation to collapse the provided dimensions; e.g. an
+  // operand with dimensions {x=256, y=2, z=2, p=32} can be collapsed to
+  // {x=1024, y=32} by collapsing dims {0, 1, 2}. Collapsing dimensions must
+  // be a consecutive, in-order subsequence of the operand dimensions.
+  //
+  // Note that collapsing a single dimension does nothing:
+  //
+  //    {256} collapsing {0} => {256}
+  //    {1} collapsing {0} => {1}
+  //
+  // Collapsing multiple dimensions produces a single result dimension:
+  //
+  //    {256, 2} collapsing {0,1} => {512}
+  //    {256, 2, 3} collapsing {0,1} => {512, 3}
+  //
+  // This could potentially cause data to be moved -- it provides a more
+  // structured form of reshaping than an arbitrary Reshape operation.
+  XlaOp Collapse(const XlaOp& operand,
+                 tensorflow::gtl::ArraySlice<int64> dimensions);
+
+  // Enqueues a slice operation onto the computation that slices the operand
+  // from the start indices to the limit indices; e.g.
+  //
+  //        x
+  //   [ 0 1 2 3 ]
+  // y [ 4 5 6 7 ] => slice(start={1, 1}, limit={2, 3}) => [ 5 6 ]
+  //   [ 8 9 a b ]
+  //
+  // Note that "limit" means up-to-but-not-including; i.e. [start, limit) in 1D
+  // range notation.
+  // The strides parameter determines the stride over the slice
+  XlaOp Slice(const XlaOp& operand,
+              tensorflow::gtl::ArraySlice<int64> start_indices,
+              tensorflow::gtl::ArraySlice<int64> limit_indices,
+              tensorflow::gtl::ArraySlice<int64> strides);
+
+  // Enqueues a slice operation in a given dimension, taking all other
+  // dimensions as they are; e.g. if dimno is 1 from start_index 2 to
+  // limit_index 4 by 1, and the shape is f32[7,8,9], this call is short-hand
+  // for:
+  //
+  //  array[:, 2:4:1, :]
+  XlaOp SliceInDim(const XlaOp& operand, int64 start_index, int64 limit_index,
+                   int64 stride, int64 dimno);
+
+  // Enqueues a slice operation onto the computation that slices the 'operand'
+  // from dynamic start indices which are passed in 'start_indices'.
+  // The size of the slice in each dimension is passed in 'slice_sizes',
+  // which specify the end point of exclusive slice intervals in each
+  // dimension [start, start + size).
+  // The shape of 'start_indices' must be rank == 1, with dimension size
+  // equal to the rank of the 'operand'.
+  // Slice index calculations are computed modulo input dimension sizes to
+  // prevent dynamic start indices from generating out-of-bound array accesses.
+  XlaOp DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
+                     tensorflow::gtl::ArraySlice<int64> slice_sizes);
+
+  // Enqueues a dynamic update slice operation onto the computation, which
+  // updates a slice of 'operand' with 'update' at dynamic 'start_indices'.
+  // The shape of 'update' determines the shape of the slice of 'operand'
+  // which is updated.
+  // The indices specified in 'start_indices' specify the offset of the slice
+  // of 'operand' which is updated.
+  //
+  //               update = {10, 11} // calculated at runtime.
+  //   [1 2 3]     start  = {1, 1}   // calculated at runtime.  [1 2  3 ]
+  //   [4 5 6]  => DynamicUpdateslice(data, update, start)   => [4 10 11]
+  //   [7 8 9]                                                  [7 8  9 ]
+  //
+  // The shape of 'start_indices' must be rank == 1, with dimension size
+  // equal to the rank of the 'operand'.
+  // Slice index calculations are computed modulo update dimension sizes to
+  // prevent dynamic start indices from generating out-of-bound array accesses.
+  XlaOp DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
+                           const XlaOp& start_indices);
+
+  // Enqueues a concatenate instruction onto the computation. 'operands' must
+  // have >= 1 entry.
+  XlaOp ConcatInDim(tensorflow::gtl::ArraySlice<XlaOp> operands,
+                    int64 dimension);
+
+  // Enqueue a tracing operation onto the computation; the computation will emit
+  // a logging message with the operand.
+  void Trace(const string& tag, const XlaOp& operand);
+
+  // Enqueues a conditional-move-like select operation onto the computation;
+  // predicated on pred, selects between on_true and on_false.
+  XlaOp Select(const XlaOp& pred, const XlaOp& on_true, const XlaOp& on_false);
+
+  // Enqueues a tuple-creation instruction onto the computation.
+  XlaOp Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements);
+
+  // Enqueues a tuple-element-get instruction onto the computation.
+  XlaOp GetTupleElement(const XlaOp& tuple_data, int64 index);
+
+  // Enqueues an equal-to comparison instruction onto the computation.
+  XlaOp Eq(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a not-equal comparison instruction onto the computation.
+  XlaOp Ne(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a greater-or-equal comparison instruction onto the computation.
+  XlaOp Ge(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a greater-than comparison instruction onto the computation.
+  XlaOp Gt(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a less-than comparison instruction onto the computation.
+  XlaOp Lt(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a less-or-equal comparison instruction onto the computation.
+  XlaOp Le(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a dot instruction onto the computation.
+  XlaOp Dot(const XlaOp& lhs, const XlaOp& rhs);
+
+  // Enqueues a general dot instruction onto the computation.
+  XlaOp DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
+                   const DotDimensionNumbers& dimension_numbers);
+
+  // Enqueues a convolution instruction onto the computation, which uses the
+  // default convolution dimension numbers.
+  XlaOp Conv(const XlaOp& lhs, const XlaOp& rhs,
+             tensorflow::gtl::ArraySlice<int64> window_strides,
+             Padding padding);
+
+  // Enqueues a convolution instruction onto the computation, with the caller
+  // provided padding configuration in the format returned by MakePadding().
+  XlaOp ConvWithGeneralPadding(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> window_strides,
+      tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding);
+
+  // Enqueues a convolution instruction onto the computation, with the caller
+  // provided dimension numbers configuration.
+  XlaOp ConvWithGeneralDimensions(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> window_strides, Padding padding,
+      const ConvolutionDimensionNumbers& dimension_numbers);
+
+  // Enqueues a convolution instruction onto the computation, with the caller
+  // provided padding configuration as well as the dimension numbers.
+  XlaOp ConvGeneral(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> window_strides,
+      tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+      const ConvolutionDimensionNumbers& dimension_numbers);
+
+  // Enqueues a convolution instruction onto the computation, with the caller
+  // provided padding configuration, dilation factors and dimension numbers.
+  XlaOp ConvGeneralDilated(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> window_strides,
+      tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+      tensorflow::gtl::ArraySlice<int64> lhs_dilation,
+      tensorflow::gtl::ArraySlice<int64> rhs_dilation,
+      const ConvolutionDimensionNumbers& dimension_numbers);
+
+  // Enqueues an FFT instruction onto the computation, of the given type and
+  // with the given FFT length.
+  XlaOp Fft(const XlaOp& operand, FftType fft_type,
+            tensorflow::gtl::ArraySlice<int64> fft_length);
+
+  // Enqueues an infeed instruction onto the computation, which writes data of
+  // the given shape to the infeed buffer of the device.
+  XlaOp Infeed(const Shape& shape, const string& config = "");
+
+  // Enqueues an outfeed instruction onto the computation. This instruction
+  // generates outgoing data transfers for the given data.
+  //
+  // shape_with_layout communicates the laid out shape that we want to outfeed
+  // -- if !ShapeUtil::Compatible(GetShape(operand), shape_with_layout) an error
+  // will occur.
+  void Outfeed(const XlaOp& operand, const Shape& shape_with_layout,
+               const string& outfeed_config);
+
+  // Enqueues a call instruction onto the computation.
+  XlaOp Call(const XlaComputation& computation,
+             tensorflow::gtl::ArraySlice<XlaOp> operands);
+
+  // Enqueues a custom call instruction onto the computation.
+  // During code generation, a call instruction is emitted which targets a
+  // symbol with the name |call_target_name|.  The |operands| are passed to the
+  // call instruction.  |shape| is the resultant shape.
+  XlaOp CustomCall(const string& call_target_name,
+                   tensorflow::gtl::ArraySlice<XlaOp> operands,
+                   const Shape& shape);
+
+  // Enqueues a pseudo-op to represent host-side computation data-dependencies.
+  // During code generation, host send and receive operations will be generated
+  // to transfer |operands| to the host and a single result of |shape| back to
+  // the device.  Host send/recv operations are emitted using |channel_name|.
+  // Dataflow dependencies and the |cost_estimate_ns| field may be used in HLO
+  // instruction scheduling.
+  XlaOp HostCompute(tensorflow::gtl::ArraySlice<XlaOp> operands,
+                    const string& channel_name, int64 cost_estimate_ns,
+                    const Shape& shape);
+
+  // The following methods enqueue element-wise binary arithmetic operations
+  // onto the computation. The shapes of the operands have to match unless one
+  // of the operands is a scalar, or an explicit broadcast dimension is given
+  // (see g3doc for more details).
+
+  // Enqueues a complex compose instruction onto the computation.
+  XlaOp Complex(const XlaOp& real, const XlaOp& imag,
+                tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a complex conjugate instruction onto the computation.
+  XlaOp Conj(const XlaOp& operand);
+
+  // Enqueues an add instruction onto the computation.
+  XlaOp Add(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a subtract instruction onto the computation.
+  XlaOp Sub(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a multiply instruction onto the computation.
+  XlaOp Mul(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a divide instruction onto the computation.
+  XlaOp Div(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a remainder instruction onto the computation.
+  XlaOp Rem(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a max instruction onto the computation.
+  XlaOp Max(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues a min instruction onto the computation.
+  XlaOp Min(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Element-wise logical operators
+  XlaOp And(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  XlaOp Or(const XlaOp& lhs, const XlaOp& rhs,
+           tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  XlaOp Xor(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  XlaOp Not(const XlaOp& operand);
+
+  XlaOp ShiftLeft(const XlaOp& lhs, const XlaOp& rhs,
+                  tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+  XlaOp ShiftRightArithmetic(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+  XlaOp ShiftRightLogical(
+      const XlaOp& lhs, const XlaOp& rhs,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Reduces an array among the provided dimensions, given "computation" as a
+  // reduction operator.
+  XlaOp Reduce(const XlaOp& operand, const XlaOp& init_value,
+               const XlaComputation& computation,
+               tensorflow::gtl::ArraySlice<int64> dimensions_to_reduce);
+
+  // Convenience wrapper around the above that reduces all the dimensions in the
+  // operand shape.
+  XlaOp ReduceAll(const XlaOp& operand, const XlaOp& init_value,
+                  const XlaComputation& computation);
+
+  // Enqueues a windowed reduce instruction onto the computation.
+  XlaOp ReduceWindow(const XlaOp& operand, const XlaOp& init_value,
+                     const XlaComputation& computation,
+                     tensorflow::gtl::ArraySlice<int64> window_dimensions,
+                     tensorflow::gtl::ArraySlice<int64> window_strides,
+                     Padding padding);
+
+  // As ReduceWindow(), but the padding is given in the format
+  // returned by MakePadding().
+  XlaOp ReduceWindowWithGeneralPadding(
+      const XlaOp& operand, const XlaOp& init_value,
+      const XlaComputation& computation,
+      tensorflow::gtl::ArraySlice<int64> window_dimensions,
+      tensorflow::gtl::ArraySlice<int64> window_strides,
+      tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding);
+
+  // Returns the sum of the operand value across all replicas. All replicas
+  // supply one input to the sum and all replicas receive the resulting sum.
+  XlaOp CrossReplicaSum(const XlaOp& operand);
+
+  // Enqueues an operation that scatters the `source` array to the selected
+  // indices of each window.
+  XlaOp SelectAndScatter(const XlaOp& operand, const XlaComputation& select,
+                         tensorflow::gtl::ArraySlice<int64> window_dimensions,
+                         tensorflow::gtl::ArraySlice<int64> window_strides,
+                         Padding padding, const XlaOp& source,
+                         const XlaOp& init_value,
+                         const XlaComputation& scatter);
+
+  // As SelectAndScatter(), but the padding is given in the format
+  // returned by MakePadding().
+  XlaOp SelectAndScatterWithGeneralPadding(
+      const XlaOp& operand, const XlaComputation& select,
+      tensorflow::gtl::ArraySlice<int64> window_dimensions,
+      tensorflow::gtl::ArraySlice<int64> window_strides,
+      tensorflow::gtl::ArraySlice<std::pair<int64, int64>> padding,
+      const XlaOp& source, const XlaOp& init_value,
+      const XlaComputation& scatter);
+
+  // Enqueues an abs instruction onto the computation.
+  XlaOp Abs(const XlaOp& operand);
+
+  // Enqueues a atan2 instruction onto the computation.
+  XlaOp Atan2(const XlaOp& y, const XlaOp& x,
+              tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues an exp instruction onto the computation.
+  XlaOp Exp(const XlaOp& operand);
+
+  // Enqueues a floor instruction onto the computation.
+  XlaOp Floor(const XlaOp& operand);
+
+  // Enqueues a ceil instruction onto the computation.
+  XlaOp Ceil(const XlaOp& operand);
+
+  // Enqueues a round instruction onto the computation, rounding to nearest even
+  // with half-way cases rounding away from zero.
+  XlaOp Round(const XlaOp& operand);
+
+  // Enqueues an log instruction (natural logarithm) onto the computation.
+  XlaOp Log(const XlaOp& operand);
+
+  // Enqueues a sign instruction onto the computation.
+  XlaOp Sign(const XlaOp& operand);
+
+  // Enqueues a cosine instruction onto the computation.
+  XlaOp Cos(const XlaOp& operand);
+
+  // Enqueues a sine instruction onto the computation.
+  XlaOp Sin(const XlaOp& operand);
+
+  // Enqueues a tanh instruction onto the computation.
+  XlaOp Tanh(const XlaOp& operand);
+
+  // Enqueues a real-part instruction onto the computation.
+  XlaOp Real(const XlaOp& operand);
+
+  // Enqueues an imaginary-part instruction onto the computation.
+  XlaOp Imag(const XlaOp& operand);
+
+  // Enqueues a float32 sqrt instruction onto the computation.
+  // (float32 is specified as there is an implicit float32 0.5f constant
+  // exponent).
+  XlaOp SqrtF32(const XlaOp& operand);
+
+  // Enqueues a float32 square instruction onto the computation.
+  // (float32 is specified as there is an implicit float32 2.0f constant
+  // exponent).
+  XlaOp SquareF32(const XlaOp& operand);
+
+  // Enqueues a lhs^rhs computation onto the computation.
+  XlaOp Pow(const XlaOp& lhs, const XlaOp& rhs,
+            tensorflow::gtl::ArraySlice<int64> broadcast_dimensions = {});
+
+  // Enqueues an operator that tests if the operand's values are finite, i.e.,
+  // not Inf or NaN. Defined only for floating-point types. Returns an array of
+  // booleans with the same shape where entries are true iff the corresponding
+  // entry was NaN.
+  XlaOp IsFinite(const XlaOp& operand);
+
+  // Enqueues a convert instruction onto the computation that changes the
+  // element type of the operand array to primitive_type.
+  XlaOp ConvertElementType(const XlaOp& operand,
+                           PrimitiveType new_element_type);
+
+  // Enqueues a no-op instruction onto the computation that changes
+  // the element type of the operand array to primitive_type. The
+  // bit-widths of the source and destination element types must be
+  // identical.
+  XlaOp BitcastConvertType(const XlaOp& operand,
+                           PrimitiveType new_element_type);
+
+  // Enqueues a float32 reciprocal instruction onto the computation.
+  // (float32 is specified as there is an implicit float32 -1.0f constant
+  // exponent).
+  //
+  // TODO(b/34468990) axe F32 suffix, can be determined by reflecting on the
+  // shape of the operand.
+  XlaOp ReciprocalF32(const XlaOp& operand);
+
+  // Enqueues a negate instruction onto the computation.
+  XlaOp Neg(const XlaOp& operand);
+
+  // Enqueues a transpose instruction onto the computation.
+  XlaOp Transpose(const XlaOp& operand,
+                  tensorflow::gtl::ArraySlice<int64> permutation);
+
+  // Enqueues a reverse instruction onto the computation. The order of the
+  // elements in the given dimensions is reversed (i.e., the element at index i
+  // is moved to index dimension_size - 1 - i).
+  XlaOp Rev(const XlaOp& operand,
+            tensorflow::gtl::ArraySlice<int64> dimensions);
+
+  // Enqueues a sort (as increasing order) instruction onto the computation.
+  XlaOp Sort(const XlaOp& operand);
+
+  // Enqueues a clamp instruction onto the computation.
+  XlaOp Clamp(const XlaOp& min, const XlaOp& operand, const XlaOp& max);
+
+  // Enqueues a map instruction onto the computation.
+  XlaOp Map(tensorflow::gtl::ArraySlice<XlaOp> operands,
+            const XlaComputation& computation,
+            tensorflow::gtl::ArraySlice<int64> dimensions,
+            tensorflow::gtl::ArraySlice<XlaOp> static_operands = {});
+
+  // Enqueues a N(mu, sigma) random number generation instruction onto the
+  // computation.
+  XlaOp RngNormal(const XlaOp& mu, const XlaOp& sigma, const Shape& shape);
+
+  // Enqueues a U(a, b) random number generation instruction onto the
+  // computation. Returns values in the semi-open interval [a, b).
+  XlaOp RngUniform(const XlaOp& a, const XlaOp& b, const Shape& shape);
+
+  // Enqueues a while node onto the computation.
+  XlaOp While(const XlaComputation& condition, const XlaComputation& body,
+              const XlaOp& init);
+
+  // Enqueues a conditional node onto the computation.
+  XlaOp Conditional(const XlaOp& predicate, const XlaOp& true_operand,
+                    const XlaComputation& true_computation,
+                    const XlaOp& false_operand,
+                    const XlaComputation& false_computation);
+
+  // Enqueues a ReducePrecision node onto the computation.
+  XlaOp ReducePrecision(const XlaOp& operand, const int exponent_bits,
+                        const int mantissa_bits);
+
+  // Enqueues a Gather node onto the computation.
+  XlaOp Gather(const XlaOp& input, const XlaOp& gather_indices,
+               const GatherDimensionNumbers& dimension_numbers,
+               tensorflow::gtl::ArraySlice<int64> window_bounds);
+
+  // Enqueues a Send node onto the computation, to send the given operand to
+  // a Recv instruction that shares the same channel handle.
+  void Send(const XlaOp& operand, const ChannelHandle& handle);
+
+  // Enqueues a Recv node onto the computation. The data comes from a Send
+  // instruction that shares the same channel handle and its shape must
+  // be the same as the given shape.
+  XlaOp Recv(const Shape& shape, const ChannelHandle& handle);
+
+  // Returns true if 'operand' is a compile-time constant. A compile-time
+  // constant does not depend on parameters with index greater than or equal to
+  // `num_parameters`, or on stateful operators such as `RngNormal` or `Infeed`.
+  // Unlike `ComputeConstant`, `IsConstant` tests whether a computation is a
+  // compile-time constant without evaluating the computation.
+  StatusOr<bool> IsConstant(const XlaOp& operand, int64 num_parameters = 0);
+
+  // Normalizes operand across spatial and batch dimensions for each feature.
+  //
+  // Returns a tuple (normalized, batch_mean, batch_var) where `normalized`
+  // is the normalized result and batch_mean and batch_var are the mean and
+  // variance, respectively, across batch for the operand.
+  XlaOp BatchNormTraining(const XlaOp& operand, const XlaOp& scale,
+                          const XlaOp& offset, float epsilon,
+                          int64 feature_index);
+
+  // Normalizes operand across spatial and batch dimensions for each feature.
+  //
+  // `BatchNormInference` is equivalent to calling `BatchNormTraining` without
+  // computing `mean` and `variance` for each batch inside the operation. It
+  // uses the input `mean` and `variance` instead as estimated values. The
+  // purpose of this op is to reduce latency in inference, hence the name
+  // `BatchNormInference`.
+  //
+  // The output has the same shape as `operand`, and contains the normalized
+  // values for each batch.
+  XlaOp BatchNormInference(const XlaOp& operand, const XlaOp& scale,
+                           const XlaOp& offset, const XlaOp& mean,
+                           const XlaOp& variance, float epsilon,
+                           int64 feature_index);
+
+  // Calculates the gradients of a batch norm op.
+  //
+  // The inputs `batch_mean` and `batch_var` represent the mean and variance
+  // across the batch.
+  //
+  // Returns a tuple of three elements:
+  //   - grad_operand: Gradient with respect to input `operand`
+  //   - grad_offset: Gradient with respect to input `offset`
+  //   - grad_scale: Gradient with respect to input `scale`
+  XlaOp BatchNormGrad(const XlaOp& operand, const XlaOp& scale,
+                      const XlaOp& batch_mean, const XlaOp& batch_var,
+                      const XlaOp& grad_output, float epsilon,
+                      int64 feature_index);
 
   // Builds the computation with the requested operations, or returns a non-ok
   // status.
   StatusOr<XlaComputation> Build();
 
+  // Returns the first error that was encountered while building the
+  // computation. When an error is encountered, by default we return a vacuous
+  // XlaOp and inform the user of the error that occurred while
+  // building the computation when they make a final call to Build().
+  //
+  // See also set_die_immediately_on_error().
+  Status first_error() const { return first_error_; }
+
+  // Returns the shape of the given op.
+  StatusOr<Shape> GetShape(const XlaOp& op) const;
+
  private:
   XlaOp AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
                        tensorflow::gtl::ArraySlice<XlaOp> operands = {});
@@ -182,6 +751,9 @@ class XlaBuilder {
     return op.ConsumeValueOrDie();
   }
 
+  // Helper method that creates an empty op and notes error.
+  XlaOp UnimplementedOp();
+
   StatusOr<const HloInstructionProto*> LookUpInstruction(const XlaOp& op) const;
 
   string name_;  // Name to use for the built computation.
@@ -213,6 +785,76 @@ XlaOp XlaBuilder::ConstantR0(NativeT value) {
   return ConstantLiteral(*Literal::CreateR0<NativeT>(value));
 }
 
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR1(tensorflow::gtl::ArraySlice<NativeT> values) {
+  return ConstantLiteral(*Literal::CreateR1<NativeT>(values));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR1(int64 length, NativeT value) {
+  Literal literal(ShapeUtil::MakeShape(
+      primitive_util::NativeToPrimitiveType<NativeT>(), {length}));
+  literal.PopulateWithValue(value);
+  return ConstantLiteral(literal);
+}
+
+inline XlaOp XlaBuilder::ConstantR1(const tensorflow::core::Bitmap& values) {
+  return ConstantLiteral(*Literal::CreateR1(values));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR2(
+    std::initializer_list<std::initializer_list<NativeT>> values) {
+  return ConstantLiteral(*Literal::CreateR2<NativeT>(values));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantFromArrayWithLayout(const Array<NativeT>& values,
+                                              const Layout& layout) {
+  return ConstantLiteral(
+      *Literal::CreateFromArrayWithLayout<NativeT>(values, layout));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantFromArray(const Array<NativeT>& values) {
+  return ConstantLiteral(*Literal::CreateFromArray<NativeT>(values));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR2FromArray2DWithLayout(
+    const Array2D<NativeT>& values, const Layout& layout) {
+  return ConstantLiteral(
+      *Literal::CreateFromArrayWithLayout<NativeT>(values, layout));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR2FromArray2D(const Array2D<NativeT>& values) {
+  return ConstantLiteral(*Literal::CreateR2FromArray2D<NativeT>(values));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR3FromArray3DWithLayout(
+    const Array3D<NativeT>& values, const Layout& layout) {
+  return ConstantLiteral(
+      *Literal::CreateR3FromArray3DWithLayout<NativeT>(values, layout));
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR3FromArray3D(const Array3D<NativeT>& values) {
+  return ConstantFromArray(values);
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR4FromArray4DWithLayout(
+    const Array4D<NativeT>& values, const Layout& layout) {
+  return ConstantFromArrayWithLayout(values, layout);
+}
+
+template <typename NativeT>
+XlaOp XlaBuilder::ConstantR4FromArray4D(const Array4D<NativeT>& values) {
+  return ConstantFromArray(values);
+}
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_XLA_CLIENT_XLA_BUILDER_H_
-- 
GitLab


From b761b841c4e4880c17bb37148e9f74bc9c05f626 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 13:15:26 -0700
Subject: [PATCH 442/960] * Added links to Machine Learning Crash Course. *
 Removed a link to the script that diagnosed Windows installation errors  
 because that script is now deprecated.

PiperOrigin-RevId: 190264199
---
 .../get_started/get_started_for_beginners.md  | 34 +++++++++++++------
 tensorflow/docs_src/get_started/index.md      |  7 ++++
 tensorflow/docs_src/install/install_linux.md  | 11 ++++--
 tensorflow/docs_src/install/install_mac.md    | 12 +++++--
 .../docs_src/install/install_windows.md       | 14 +++++---
 .../docs_src/programmers_guide/embedding.md   |  3 ++
 6 files changed, 61 insertions(+), 20 deletions(-)

diff --git a/tensorflow/docs_src/get_started/get_started_for_beginners.md b/tensorflow/docs_src/get_started/get_started_for_beginners.md
index b88483be69..f59cebe6c4 100644
--- a/tensorflow/docs_src/get_started/get_started_for_beginners.md
+++ b/tensorflow/docs_src/get_started/get_started_for_beginners.md
@@ -14,6 +14,11 @@ If you are already familiar with basic machine learning concepts
 but are new to TensorFlow, read
 @{$premade_estimators$Getting Started with TensorFlow: for ML Experts}.
 
+If you'd like to learn a lot about the basics of Machine Learning,
+consider taking
+[Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/).
+
+
 ## The Iris classification problem
 
 Imagine you are a botanist seeking an automated way to classify each
@@ -86,6 +91,9 @@ a number.  Here's the representation scheme:
 * 1 represents versicolor
 * 2 represents virginica
 
+For a look at other examples of labels and examples, see the
+[ML Terminology section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/framing/ml-terminology).
+
 
 ## Models and training
 
@@ -371,7 +379,7 @@ There are several categories of neural networks.
 We'll be using a [**fully connected neural
 network**](https://developers.google.com/machine-learning/glossary/#fully_connected_layer),
 which means that the neurons in one layer take inputs from *every* neuron in
-the previous layer.  For example, the following figure illustrates a 
+the previous layer.  For example, the following figure illustrates a
 fully connected neural network consisting of three hidden layers:
 
 *   The first hidden layer contains four neurons.
@@ -385,6 +393,9 @@ fully connected neural network consisting of three hidden layers:
 **A neural network with three hidden layers.**
 <p>&nbsp;</p>
 
+For a more detailed introduction to neural networks, see the
+[Introduction to Neural Nets section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/introduction-to-neural-networks/anatomy).
+
 To specify a model type, instantiate an
 [**Estimator**](https://developers.google.com/machine-learning/glossary/#Estimators)
 class.  TensorFlow provides two categories of Estimators:
@@ -448,9 +459,9 @@ will become very important.
 
 ### Train the model
 
-Instantiating a `tf.Estimator.DNNClassifier` creates a framework for learning 
-the model. Basically, we've wired a network but haven't yet let data flow 
-through it. To train the neural network, call the Estimator object's `train` 
+Instantiating a `tf.Estimator.DNNClassifier` creates a framework for learning
+the model. Basically, we've wired a network but haven't yet let data flow
+through it. To train the neural network, call the Estimator object's `train`
 method. For example:
 
 ```python
@@ -559,15 +570,15 @@ of 0.5.  The following suggests a more effective model:
     <th colspan="1">Label</th>
     <th colspan="1">Prediction</th>
   </tr>
-  <tr> <td>5.9</td> <td>3.0</td> <td>4.3</td> <td>1.5</td> <td>1</td> 
+  <tr> <td>5.9</td> <td>3.0</td> <td>4.3</td> <td>1.5</td> <td>1</td>
           <td style="background-color:green">1</td></tr>
-  <tr> <td>6.9</td> <td>3.1</td> <td>5.4</td> <td>2.1</td> <td>2</td> 
+  <tr> <td>6.9</td> <td>3.1</td> <td>5.4</td> <td>2.1</td> <td>2</td>
           <td style="background-color:green">2</td></tr>
-  <tr> <td>5.1</td> <td>3.3</td> <td>1.7</td> <td>0.5</td> <td>0</td> 
+  <tr> <td>5.1</td> <td>3.3</td> <td>1.7</td> <td>0.5</td> <td>0</td>
           <td style="background-color:green">0</td></tr>
-  <tr> <td>6.0</td> <td>3.4</td> <td>4.5</td> <td>1.6</td> <td>1</td> 
+  <tr> <td>6.0</td> <td>3.4</td> <td>4.5</td> <td>1.6</td> <td>1</td>
           <td style="background-color:red">2</td></tr>
-  <tr> <td>5.5</td> <td>2.5</td> <td>4.0</td> <td>1.3</td> <td>1</td> 
+  <tr> <td>5.5</td> <td>2.5</td> <td>4.0</td> <td>1.3</td> <td>1</td>
           <td style="background-color:green">1</td></tr>
 </table>
 
@@ -631,6 +642,10 @@ Test set accuracy: 0.967
 An accuracy of 0.967 implies that our trained model correctly classified 29
 out of the 30 Iris species in the test set.
 
+To get a deeper understanding of different metrics for evaluating
+models, see the
+[Classification section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/classification).
+
 
 ### Predicting
 
@@ -723,7 +738,6 @@ Prediction is "Virginica" (97.9%), expected "Virginica"
 
 ## Summary
 
-<!--TODO(barryr): When MLCC is released, add pointers to relevant sections.-->
 This document provides a short introduction to machine learning.
 
 Because `premade_estimators.py` relies on high-level APIs, much of the
diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md
index b7bd1286e3..fb83a770a5 100644
--- a/tensorflow/docs_src/get_started/index.md
+++ b/tensorflow/docs_src/get_started/index.md
@@ -1,5 +1,12 @@
 # Getting Started
 
+If you are new to machine learning, we recommend taking the following online
+course prior to diving into TensorFlow documentation:
+
+  * [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/),
+    which introduces machine learning concepts and encourages experimentation
+    with existing TensorFlow code.
+
 TensorFlow is a tool for machine learning. While it contains a wide range of
 functionality, TensorFlow is mainly designed for deep neural network models.
 
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 8612762271..5e9a84bff6 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -506,11 +506,18 @@ TensorFlow programs:
 
 <pre>Hello, TensorFlow!</pre>
 
-If you are new to TensorFlow, see @{$get_started/premade_estimators$Getting Started with TensorFlow}.
-
 If the system outputs an error message instead of a greeting, see [Common
 installation problems](#common_installation_problems).
 
+If you are new to machine learning, we recommend the following:
+
+*  [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course)
+*  @{$get_started/get_started_for_beginners$Getting Started for ML Beginners}
+
+If you are experienced with machine learning but new to TensorFlow, see
+@{$get_started/premade_estimators$Getting Started with TensorFlow}.
+
+
 ## Common installation problems
 
 We are relying on Stack Overflow to document TensorFlow installation problems
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 7207cb4f2b..55b460e189 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -400,12 +400,18 @@ writing TensorFlow programs:
 
 <pre>Hello, TensorFlow!</pre>
 
-If you are new to TensorFlow, see
-@{$get_started/premade_estimators$Getting Started with TensorFlow}.
-
 If the system outputs an error message instead of a greeting, see
 [Common installation problems](#common_installation_problems).
 
+If you are new to machine learning, we recommend the following:
+
+*  [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course)
+*  @{$get_started/get_started_for_beginners$Getting Started for ML Beginners}
+
+If you are experienced with machine learning but new to TensorFlow, see
+@{$get_started/premade_estimators$Getting Started with TensorFlow}.
+
+
 ## Common installation problems
 
 We are relying on Stack Overflow to document TensorFlow installation problems
diff --git a/tensorflow/docs_src/install/install_windows.md b/tensorflow/docs_src/install/install_windows.md
index 2413bc9cfb..86add74da1 100644
--- a/tensorflow/docs_src/install/install_windows.md
+++ b/tensorflow/docs_src/install/install_windows.md
@@ -17,7 +17,7 @@ You must choose one of the following types of TensorFlow to install:
     NVIDIA® GPU, you must install this version. Note that this version of
     TensorFlow is typically much easier to install (typically,
     in 5 or 10 minutes), so even if you have an NVIDIA GPU, we recommend
-    installing this version first. Prebuilt binaries will use AVX instructions. 
+    installing this version first. Prebuilt binaries will use AVX instructions.
   * **TensorFlow with GPU support**. TensorFlow programs typically run
     significantly faster on a GPU than on a CPU. Therefore, if your
     system has a NVIDIA® GPU meeting the prerequisites shown below
@@ -154,13 +154,17 @@ TensorFlow programs:
 
 <pre>Hello, TensorFlow!</pre>
 
-If you are new to TensorFlow, see @{$get_started/premade_estimators$Getting Started with TensorFlow}.
-
 If the system outputs an error message instead of a greeting, see [Common
 installation problems](#common_installation_problems).
 
-There is also a helpful [script](https://gist.github.com/mrry/ee5dbcfdd045fa48a27d56664411d41c)
-for Windows TensorFlow installation issues.
+If you are new to machine learning, we recommend the following:
+
+*  [Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course)
+*  @{$get_started/get_started_for_beginners$Getting Started for ML Beginners}
+
+If you are experienced with machine learning but new to TensorFlow, see
+@{$get_started/premade_estimators$Getting Started with TensorFlow}.
+
 
 ## Common installation problems
 
diff --git a/tensorflow/docs_src/programmers_guide/embedding.md b/tensorflow/docs_src/programmers_guide/embedding.md
index e8027fc12b..d5703e0737 100644
--- a/tensorflow/docs_src/programmers_guide/embedding.md
+++ b/tensorflow/docs_src/programmers_guide/embedding.md
@@ -7,6 +7,9 @@ with the TensorBoard Embedding Projector
 newcomers to machine learning or TensorFlow, and the Embedding Projector how-to
 is for users at all levels.
 
+An alternative tutorial on these concepts is available in the
+[Embeddings section of Machine Learning Crash Course](https://developers.google.com/machine-learning/crash-course/embeddings/video-lecture).
+
 [TOC]
 
 An **embedding** is a mapping from discrete objects, such as words, to vectors
-- 
GitLab


From cf5729fc4710c9e579afa7c1176b00c9c0acec6e Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Fri, 23 Mar 2018 13:20:13 -0700
Subject: [PATCH 443/960] Updates reduce_mean op.

PiperOrigin-RevId: 190264873
---
 tensorflow/contrib/lite/testing/generate_examples.py | 6 +++---
 tensorflow/contrib/lite/toco/import_tensorflow.cc    | 4 +++-
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 38de9dcf2c..e4ef17585f 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -754,7 +754,7 @@ def make_mean_tests(zip_path):
           [-1, -2, -3], [0, 0, 0], [2, 2, 0], [1, 0, -3, -3]
       ],
       "const_axis": [True, False],
-      "keep_dims": [True, False],
+      "keepdims": [True, False],
   }, {
       "input_dtype": [tf.float32, tf.int32, tf.int64],
       "input_shape": [[1, 224, 224, 3]],
@@ -765,7 +765,7 @@ def make_mean_tests(zip_path):
           [2, 2, 3], [-3, -3, -4], [-3, 2, 1]
       ],
       "const_axis": [True, False],
-      "keep_dims": [True, False],
+      "keepdims": [True, False],
   }]
 
   def build_graph(parameters):
@@ -788,7 +788,7 @@ def make_mean_tests(zip_path):
       input_tensors = [input_tensor, axis]
 
     out = tf.reduce_mean(
-        input_tensor, axis=axis, keep_dims=parameters["keep_dims"])
+        input_tensor, axis=axis, keepdims=parameters["keepdims"])
     return input_tensors, [out]
 
   def build_inputs(parameters, sess, inputs, outputs):
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index a7a50e6fc9..b844e0b948 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1541,7 +1541,9 @@ void ConvertMeanOperator(const NodeDef& node,
   op->inputs.push_back(node.input(1));
   op->outputs.push_back(node.name());
   model->operators.emplace_back(op);
-  if (HasAttr(node, "keep_dims")) {
+  if (HasAttr(node, "keepdims")) {
+    op->keep_dims = GetBoolAttr(node, "keepdims");
+  } else if (HasAttr(node, "keep_dims")) {
     op->keep_dims = GetBoolAttr(node, "keep_dims");
   }
 }
-- 
GitLab


From 9b9ca14ae9720b7c28351191a9d9529fc68884b1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 23 Mar 2018 13:23:35 -0700
Subject: [PATCH 444/960] Moves TensorHandle to common_runtime

PiperOrigin-RevId: 190265301
---
 tensorflow/c/eager/BUILD                      |   2 +
 tensorflow/c/eager/c_api.cc                   | 272 ++++++++----------
 tensorflow/c/eager/c_api_internal.h           |  85 +-----
 tensorflow/core/common_runtime/eager/BUILD    |  23 ++
 .../common_runtime/eager/tensor_handle.cc     | 107 +++++++
 .../core/common_runtime/eager/tensor_handle.h | 130 +++++++++
 tensorflow/python/eager/pywrap_tfe_src.cc     |   7 +-
 tensorflow/python/lib/core/py_func.cc         |   2 +-
 8 files changed, 393 insertions(+), 235 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/tensor_handle.cc
 create mode 100644 tensorflow/core/common_runtime/eager/tensor_handle.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index bea5a121b3..d2d8d59323 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -31,6 +31,7 @@ tf_cuda_library(
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
+            "//tensorflow/core/common_runtime/eager:tensor_handle",
             "//tensorflow/core:core_cpu_internal",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
@@ -68,6 +69,7 @@ tf_cuda_library(
         "//tensorflow/core/common_runtime/eager:context",
         "//tensorflow/core/common_runtime/eager:eager_executor",
         "//tensorflow/core/common_runtime/eager:kernel_and_device",
+        "//tensorflow/core/common_runtime/eager:tensor_handle",
     ],
 )
 
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 2402a6d044..59432f2ef8 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -161,29 +161,32 @@ TFE_TensorHandle* TFE_NewTensorHandle(TF_Tensor* t, TF_Status* status) {
 
 void TFE_DeleteTensorHandle(TFE_TensorHandle* h) {
   DCHECK(h);
-  h->Unref();
+  if (h->handle) {
+    h->handle->Unref();
+  }
+  delete h;
 }
 
 TF_DataType TFE_TensorHandleDataType(TFE_TensorHandle* h) {
-  return static_cast<TF_DataType>(h->dtype);
+  return static_cast<TF_DataType>(h->handle->dtype);
 }
 
 int TFE_TensorHandleNumDims(TFE_TensorHandle* h, TF_Status* status) {
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->Tensor(&t);
+  status->status = h->handle->Tensor(&t);
   return t == nullptr ? 0 : t->dims();
 }
 
 int64_t TFE_TensorHandleDim(TFE_TensorHandle* h, int dim_index,
                             TF_Status* status) {
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->Tensor(&t);
+  status->status = h->handle->Tensor(&t);
   return t == nullptr ? 0 : t->dim_size(dim_index);
 }
 
 const char* TFE_TensorHandleDeviceName(TFE_TensorHandle* h, TF_Status* status) {
   tensorflow::Device* d = nullptr;
-  status->status = h->OpDevice(&d);
+  status->status = h->handle->OpDevice(&d);
   return (d == nullptr) ? "/job:localhost/replica:0/task:0/device:CPU:0"
                         : d->name().c_str();
 }
@@ -193,7 +196,7 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
   tensorflow::Device* d = nullptr;
   tensorflow::Device* op_device = nullptr;
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->TensorAndDevice(&t, &d, &op_device);
+  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
   if (!status->status.ok()) return nullptr;
   if (!IsCPU(d)) {
     TF_SetStatus(status, TF_UNIMPLEMENTED,
@@ -212,10 +215,10 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
 
 namespace {
 
-tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
+tensorflow::Status TensorHandleCopyToDevice(tensorflow::TensorHandle* h,
                                             TFE_Context* ctx,
                                             tensorflow::Device* dstd,
-                                            TFE_TensorHandle** output) {
+                                            tensorflow::TensorHandle** output) {
   const tensorflow::Tensor* src = nullptr;
   tensorflow::Device* srcd = nullptr;
   // TODO(agarwal): src_opd is unused. Perhaps allow TensorAndDevice to accept
@@ -232,7 +235,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   const bool both_on_cpu = src_cpu && dst_cpu;
   if (is_same_device || both_on_cpu) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new TFE_TensorHandle(*src, dstd, dstd);
+    *output = new tensorflow::TensorHandle(*src, dstd, dstd);
     return tensorflow::Status::OK();
   }
   if (!dst_cpu && (src->dtype() != tensorflow::DT_VARIANT &&
@@ -249,7 +252,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   tensorflow::Tensor dst(dstd->GetAllocator(attr), src->dtype(), src->shape());
   if (src->shape().num_elements() == 0) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new TFE_TensorHandle(dst, dstd, dstd);
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
     return tensorflow::Status::OK();
   }
   tensorflow::DeviceContext* src_device_context = nullptr;
@@ -280,7 +283,7 @@ tensorflow::Status TensorHandleCopyToDevice(TFE_TensorHandle* h,
   n.WaitForNotification();
   if (status.ok()) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new TFE_TensorHandle(dst, dstd, dstd);
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
   }
   return status;
 }
@@ -335,12 +338,12 @@ void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
     tensorflow::Device* d = nullptr;
     // TODO(agarwal): This call may block if h is not ready. Avoid this if
     // possible.
-    status->status = h->Device(&d);
+    status->status = h->handle->Device(&d);
     if (!status->status.ok()) return;
     if (!IsCPU(d)) op->device = d;
   }
-  h->Ref();
-  op->inputs.push_back(h);
+  h->handle->Ref();
+  op->inputs.push_back(h->handle);
   op->attrs.NumInputs(op->inputs.size());
 }
 
@@ -506,6 +509,79 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 namespace {
 
+class CopyToDeviceNode : public tensorflow::EagerNode {
+ public:
+  CopyToDeviceNode(tensorflow::TensorHandle* src, tensorflow::Device* dstd,
+                   TFE_Context* ctx)
+      : tensorflow::EagerNode(ctx->context.NextId()),
+        src_(src),
+        dstd_(dstd),
+        ctx_(ctx),
+        dst_(new tensorflow::TensorHandle(id, src_->dtype, &ctx->context)) {
+    src_->Ref();
+    dst_->Ref();
+  }
+
+  ~CopyToDeviceNode() override {
+    src_->Unref();
+    dst_->Unref();
+  }
+
+  tensorflow::Status Run() override {
+    tensorflow::TensorHandle* temp = nullptr;
+    TF_RETURN_IF_ERROR(TensorHandleCopyToDevice(src_, ctx_, dstd_, &temp));
+    const tensorflow::Tensor* tensor = nullptr;
+    tensorflow::Device* device = nullptr;
+    tensorflow::Device* op_device = nullptr;
+    tensorflow::Status status =
+        temp->TensorAndDevice(&tensor, &device, &op_device);
+    // `temp` is a ready handle. So the following call should return OK.
+    TF_DCHECK_OK(status) << status.error_message();
+    DCHECK(tensor);
+    dst_->SetTensorAndDevice(*tensor, device, op_device);
+    temp->Unref();
+    return tensorflow::Status::OK();
+  }
+
+  tensorflow::TensorHandle* dst() { return dst_; }
+
+ private:
+  tensorflow::TensorHandle* src_;
+  tensorflow::Device* dstd_;
+  TFE_Context* ctx_;
+  tensorflow::TensorHandle* dst_;
+};
+
+// TODO(apassos) move to TensorHandle
+tensorflow::TensorHandle* TFE_TensorHandleCopyToDevice_Internal(
+    tensorflow::TensorHandle* h, TFE_Context* ctx, const char* device_name,
+    TF_Status* status) {
+  status->status = ctx->context.GetStatus();
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+  tensorflow::Device* dstd = ctx->context.HostCPU();
+  if (device_name != nullptr && strlen(device_name) > 0) {
+    status->status =
+        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
+    if (!status->status.ok()) return nullptr;
+  }
+  if (ctx->context.Async()) {
+    // Note that `h` may not be currently ready. However execution order will
+    // make sure that `h` is ready before the copy is actually done.
+    CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
+    tensorflow::TensorHandle* output = node->dst();
+    // Note that calling Add makes `node` accessible by the EagerExecutor
+    // thread. So further accesses need to be thread-safe.
+    ctx->context.ExecutorAdd(node);
+    return output;
+  } else {
+    tensorflow::TensorHandle* output = nullptr;
+    status->status = TensorHandleCopyToDevice(h, ctx, dstd, &output);
+    return output;
+  }
+}
+
 tensorflow::Status ValidateInputTypeAndPlacement(
     TFE_Context* ctx, tensorflow::Device* host_device,
     tensorflow::Device* op_device, TFE_Op* op,
@@ -518,7 +594,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
   for (int i = 0; i < op->inputs.size(); ++i) {
     const tensorflow::Device* expected_device =
         memtypes[i] == tensorflow::HOST_MEMORY ? host_device : op_device;
-    TFE_TensorHandle* handle = op->inputs[i];
+    tensorflow::TensorHandle* handle = op->inputs[i];
     tensorflow::Device* handle_device = nullptr;
     TF_RETURN_IF_ERROR(handle->Device(&handle_device));
     const tensorflow::Device* actual_device =
@@ -560,8 +636,9 @@ tensorflow::Status ValidateInputTypeAndPlacement(
       // We are only here if the policy is warn or silent copies, so we should
       // trigger a copy.
       TF_Status* s = TF_NewStatus();
-      TFE_TensorHandle* copied_tensor = TFE_TensorHandleCopyToDevice(
-          handle, ctx, expected_device->name().c_str(), s);
+      tensorflow::TensorHandle* copied_tensor =
+          TFE_TensorHandleCopyToDevice_Internal(
+              handle, ctx, expected_device->name().c_str(), s);
       tensorflow::Status status = s->status;
       TF_DeleteStatus(s);
       if (!status.ok()) {
@@ -616,9 +693,10 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
 
 tensorflow::Status Execute(
     TFE_Context* ctx, tensorflow::Device* device,
-    const tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4>& op_inputs,
+    const tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4>&
+        op_inputs,
     tensorflow::KernelAndDevice* kernel, tensorflow::NodeExecStats* maybe_stats,
-    TFE_TensorHandle** retvals, int num_retvals) {
+    tensorflow::TensorHandle** retvals, int num_retvals) {
   if (!ctx->context.SoftPlacement() && device == nullptr) {
     device = ctx->context.HostCPU();
   }
@@ -683,7 +761,7 @@ tensorflow::Status Execute(
       d = nullptr;
     }
     if (retvals[i] == nullptr) {
-      retvals[i] = new TFE_TensorHandle(outputs[i], d, op_device);
+      retvals[i] = new tensorflow::TensorHandle(outputs[i], d, op_device);
     } else {
       retvals[i]->SetTensorAndDevice(outputs[i], d, op_device);
     }
@@ -711,9 +789,10 @@ class ExecuteNode : public tensorflow::EagerNode {
     }
     TFE_Context* ctx = op->ctx;
     for (int i = 0; i < num_retvals; ++i) {
-      TFE_TensorHandle* h = new TFE_TensorHandle(id, output_dtypes[i], ctx);
+      tensorflow::TensorHandle* h =
+          new tensorflow::TensorHandle(id, output_dtypes[i], &ctx->context);
       h->Ref();
-      retvals[i] = h;
+      retvals[i] = new TFE_TensorHandle(h);
       retvals_[i] = h;
     }
   }
@@ -745,54 +824,12 @@ class ExecuteNode : public tensorflow::EagerNode {
  private:
   TFE_Context* ctx_;
   tensorflow::Device* op_device_;
-  tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4> inputs_;
+  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4> inputs_;
   tensorflow::KernelAndDevice* kernel_;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats_;
-  tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 2> retvals_;
+  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> retvals_;
 };
 
-class CopyToDeviceNode : public tensorflow::EagerNode {
- public:
-  CopyToDeviceNode(TFE_TensorHandle* src, tensorflow::Device* dstd,
-                   TFE_Context* ctx)
-      : tensorflow::EagerNode(ctx->context.NextId()),
-        src_(src),
-        dstd_(dstd),
-        ctx_(ctx),
-        dst_(new TFE_TensorHandle(id, src_->dtype, ctx)) {
-    src_->Ref();
-    dst_->Ref();
-  }
-
-  ~CopyToDeviceNode() override {
-    src_->Unref();
-    dst_->Unref();
-  }
-
-  tensorflow::Status Run() override {
-    TFE_TensorHandle* temp = nullptr;
-    TF_RETURN_IF_ERROR(TensorHandleCopyToDevice(src_, ctx_, dstd_, &temp));
-    const tensorflow::Tensor* tensor = nullptr;
-    tensorflow::Device* device = nullptr;
-    tensorflow::Device* op_device = nullptr;
-    tensorflow::Status status =
-        temp->TensorAndDevice(&tensor, &device, &op_device);
-    // `temp` is a ready handle. So the following call should return OK.
-    TF_DCHECK_OK(status) << status.error_message();
-    DCHECK(tensor);
-    dst_->SetTensorAndDevice(*tensor, device, op_device);
-    temp->Unref();
-    return tensorflow::Status::OK();
-  }
-
-  TFE_TensorHandle* dst() { return dst_; }
-
- private:
-  TFE_TensorHandle* src_;
-  tensorflow::Device* dstd_;
-  TFE_Context* ctx_;
-  TFE_TensorHandle* dst_;
-};
 
 #ifdef TENSORFLOW_EAGER_USE_XLA
 // Synthesizes and returns a wrapper function over `op`, which must be a
@@ -1140,11 +1177,14 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
     // allocate it.
+    std::vector<tensorflow::TensorHandle*> handle_retvals(*num_retvals,
+                                                          nullptr);
+    status->status =
+        Execute(op->ctx, op->device, op->inputs, kernel, maybe_stats.get(),
+                handle_retvals.data(), *num_retvals);
     for (int i = 0; i < *num_retvals; ++i) {
-      retvals[i] = nullptr;
+      retvals[i] = new TFE_TensorHandle(handle_retvals[i]);
     }
-    status->status = Execute(op->ctx, op->device, op->inputs, kernel,
-                             maybe_stats.get(), retvals, *num_retvals);
   }
 }
 
@@ -1152,30 +1192,12 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
                                                TFE_Context* ctx,
                                                const char* device_name,
                                                TF_Status* status) {
-  status->status = ctx->context.GetStatus();
-  if (!status->status.ok()) {
-    return nullptr;
-  }
-  tensorflow::Device* dstd = ctx->context.HostCPU();
-  if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status =
-        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
-    if (!status->status.ok()) return nullptr;
-  }
-  if (ctx->context.Async()) {
-    // Note that `h` may not be currently ready. However execution order will
-    // make sure that `h` is ready before the copy is actually done.
-    CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
-    TFE_TensorHandle* output = node->dst();
-    // Note that calling Add makes `node` accessible by the EagerExecutor
-    // thread. So further accesses need to be thread-safe.
-    ctx->context.ExecutorAdd(node);
-    return output;
-  } else {
-    TFE_TensorHandle* output = nullptr;
-    status->status = TensorHandleCopyToDevice(h, ctx, dstd, &output);
-    return output;
+  tensorflow::TensorHandle* handle = TFE_TensorHandleCopyToDevice_Internal(
+      h->handle, ctx, device_name, status);
+  if (status->status.ok()) {
+    return new TFE_TensorHandle(handle);
   }
+  return nullptr;
 }
 
 void TFE_ContextAddFunctionDef(TFE_Context* ctx,
@@ -1214,7 +1236,7 @@ const tensorflow::Tensor* TFE_TensorHandleUnderlyingTensorInHostMemory(
   tensorflow::Device* d = nullptr;
   tensorflow::Device* op_device = nullptr;
   const tensorflow::Tensor* t = nullptr;
-  status->status = h->TensorAndDevice(&t, &d, &op_device);
+  status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
   if (!status->status.ok()) return nullptr;
   if (d != nullptr) {
     status->status = tensorflow::errors::FailedPrecondition(
@@ -1306,70 +1328,8 @@ void SetOpAttrValueScalar(TFE_Context* ctx, TFE_Op* op,
 }  // namespace tensorflow
 
 
-
-bool TFE_TensorHandle::IsReady() {
-  if (node_id == 0) return true;
-  tensorflow::mutex_lock l(ctx_mutex_);
-  return ctx_ == nullptr;
-}
-
-tensorflow::Status TFE_TensorHandle::WaitReady() {
-  if (node_id == 0) return tensorflow::Status::OK();
-  tensorflow::EagerExecutor* executor = nullptr;
-  {
-    tensorflow::mutex_lock l(ctx_mutex_);
-    if (ctx_ == nullptr) return tensorflow::Status::OK();
-    executor = ctx_->context.Executor();
-  }
-  return executor->WaitFor(node_id);
-}
-
-tensorflow::Status TFE_TensorHandle::Tensor(const tensorflow::Tensor** t) {
-  TF_RETURN_IF_ERROR(WaitReady());
-  DCHECK(IsReady());
-  *t = &tensor_;
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status TFE_TensorHandle::Device(tensorflow::Device** d) {
-  TF_RETURN_IF_ERROR(WaitReady());
-  DCHECK(IsReady());
-  *d = device_;
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status TFE_TensorHandle::OpDevice(tensorflow::Device** d) {
-  TF_RETURN_IF_ERROR(WaitReady());
-  DCHECK(IsReady());
-  *d = op_device_;
-  return tensorflow::Status::OK();
-}
-
-tensorflow::Status TFE_TensorHandle::TensorAndDevice(
-    const tensorflow::Tensor** tensor, tensorflow::Device** device,
-    tensorflow::Device** op_device) {
-  TF_RETURN_IF_ERROR(WaitReady());
-  DCHECK(IsReady());
-  *tensor = &tensor_;
-  *device = device_;
-  *op_device = op_device_;
-  return tensorflow::Status::OK();
-}
-
-void TFE_TensorHandle::SetTensorAndDevice(const tensorflow::Tensor& tensor,
-                                          tensorflow::Device* device,
-                                          tensorflow::Device* op_device) {
-  tensorflow::mutex_lock l(ctx_mutex_);
-  DCHECK(node_id > 0 && ctx_) << "SetTensorAndDevice should be only called  "
-                              << "on non-ready handles.";
-  ctx_ = nullptr;
-  tensor_ = tensor;
-  device_ = device;
-  op_device_ = op_device;
-}
-
 TFE_Op::~TFE_Op() {
-  for (TFE_TensorHandle* h : inputs) {
+  for (tensorflow::TensorHandle* h : inputs) {
     h->Unref();
   }
 }
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index 5b29120b40..e6d2ab75ff 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/eager/context.h"
 #include "tensorflow/core/common_runtime/eager/eager_executor.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
@@ -67,84 +68,18 @@ struct TFE_Context {
   tensorflow::EagerContext context;
 };
 
-struct TFE_TensorHandle : public tensorflow::core::RefCounted {
- public:
+struct TFE_TensorHandle {
   TFE_TensorHandle(const tensorflow::Tensor& t, tensorflow::Device* d,
                    tensorflow::Device* op_device)
-      : dtype(t.dtype()),
-        node_id(0),
-        tensor_(t),
-        device_(d),
-        op_device_(op_device),
-        ctx_(nullptr) {}
+      : handle(new tensorflow::TensorHandle(t, d, op_device)) {}
 
   TFE_TensorHandle(tensorflow::uint64 node_id, tensorflow::DataType dtype,
-                   TFE_Context* ctx)
-      : dtype(dtype),
-        node_id(node_id),
-        tensor_(dtype),
-        device_(nullptr),
-        op_device_(nullptr),
-        ctx_(ctx) {
-    DCHECK_GT(node_id, 0);
-  }
-
-  ~TFE_TensorHandle() override {}
-
-  tensorflow::Status Tensor(const tensorflow::Tensor** t);
-
-  tensorflow::Status Device(tensorflow::Device** d);
-
-  tensorflow::Status OpDevice(tensorflow::Device** d);
-
-  tensorflow::Status TensorAndDevice(const tensorflow::Tensor** tensor,
-                                     tensorflow::Device** device,
-                                     tensorflow::Device** op_device);
-
-  // Note that this can be called at most once, and only on non-ready handles,
-  // and makes them ready.
-  void SetTensorAndDevice(const tensorflow::Tensor& tensor,
-                          tensorflow::Device* device,
-                          tensorflow::Device* op_device);
-
-  // dtype for the handle. It must be the same as t.dtype() once the handle is
-  // ready.
-  const tensorflow::DataType dtype;
-
- private:
-  // If the contents of the Tensor pointed to by this handle is yet to be
-  // computed by a EagerNode, this function will block till that compuatation is
-  // done and the handle is "ready".
-  tensorflow::Status WaitReady();
-
-  bool IsReady();
-
-  // Id for the EagerNode that will compute the value pointed to by this handle.
-  // If the value is 0, the handle is already ready, but not vice-versa.
-  const tensorflow::uint64 node_id;
-
-  tensorflow::Tensor tensor_;
-
-  // TODO(ashankar): device_ == nullptr iff local CPU
-  // This was expedient, but perhaps worth revisiting ('device_' should always
-  // be a valid pointer?)
-  // This can be done if TFE_NewOp() and the TFE_TensorHandle constructors are
-  // provided with the appropriate TFE_Context.
-  //
-  // TODO(ashankar): Reference count TFE_Context to ensure that 'device_' of a
-  // TFE_TensorHandle does not outlive the TFE_Context from which it came?
-  tensorflow::Device* device_;
-
-  // Device in which the op producing this tensor was executed. Equals to
-  // device_ for constant tensors.
-  tensorflow::Device* op_device_;
-
-  tensorflow::mutex ctx_mutex_;
-
-  // `ctx` is only guaranteed to be set if the handle is not "ready". This is
-  // typically true when the handle was produced during async execution.
-  // `ctx` object is not owned and should outlive this handle.
-  TFE_Context* ctx_ GUARDED_BY(ctx_mutex_);
+                   tensorflow::EagerContext* ctx)
+      : handle(new tensorflow::TensorHandle(node_id, dtype, ctx)) {}
+
+  TFE_TensorHandle(tensorflow::TensorHandle* handle) : handle(handle) {}
+
+  tensorflow::TensorHandle* handle;
 };
 
 struct TFE_Op {
@@ -161,7 +96,7 @@ struct TFE_Op {
   const tensorflow::string name;
   tensorflow::AttrBuilder attrs;
   const tensorflow::AttrTypeMap* attr_types;
-  tensorflow::gtl::InlinedVector<TFE_TensorHandle*, 4> inputs;
+  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4> inputs;
   tensorflow::Device* device;
   bool use_xla = false;
 };
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index de10b10b7e..02fb83200a 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -54,6 +54,29 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "tensor_handle",
+    srcs = [
+        "tensor_handle.cc",
+    ],
+    hdrs = [
+        "tensor_handle.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        ":eager_executor",
+        ":kernel_and_device",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
+    ],
+)
+
 tf_cuda_library(
     name = "kernel_and_device",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
new file mode 100644
index 0000000000..5bc1700627
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -0,0 +1,107 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+bool TensorHandle::IsReady() {
+  if (node_id == 0) return true;
+  mutex_lock l(ctx_mutex_);
+  return ctx_ == nullptr;
+}
+
+Status TensorHandle::WaitReady() {
+  if (node_id == 0) return Status::OK();
+  EagerExecutor* executor = nullptr;
+  {
+    mutex_lock l(ctx_mutex_);
+    if (ctx_ == nullptr) return Status::OK();
+    executor = ctx_->Executor();
+  }
+  return executor->WaitFor(node_id);
+}
+
+Status TensorHandle::Tensor(const tensorflow::Tensor** t) {
+  TF_RETURN_IF_ERROR(WaitReady());
+  DCHECK(IsReady());
+  *t = &tensor_;
+  return Status::OK();
+}
+
+Status TensorHandle::Device(tensorflow::Device** d) {
+  TF_RETURN_IF_ERROR(WaitReady());
+  DCHECK(IsReady());
+  *d = device_;
+  return Status::OK();
+}
+
+Status TensorHandle::OpDevice(tensorflow::Device** d) {
+  TF_RETURN_IF_ERROR(WaitReady());
+  DCHECK(IsReady());
+  *d = op_device_;
+  return Status::OK();
+}
+
+Status TensorHandle::TensorAndDevice(const tensorflow::Tensor** tensor,
+                                     tensorflow::Device** device,
+                                     tensorflow::Device** op_device) {
+  TF_RETURN_IF_ERROR(WaitReady());
+  DCHECK(IsReady());
+  *tensor = &tensor_;
+  *device = device_;
+  *op_device = op_device_;
+  return Status::OK();
+}
+
+void TensorHandle::SetTensorAndDevice(const tensorflow::Tensor& tensor,
+                                      tensorflow::Device* device,
+                                      tensorflow::Device* op_device) {
+  mutex_lock l(ctx_mutex_);
+  DCHECK(node_id > 0 && ctx_) << "SetTensorAndDevice should be only called  "
+                              << "on non-ready handles.";
+  ctx_ = nullptr;
+  tensor_ = tensor;
+  device_ = device;
+  op_device_ = op_device;
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
new file mode 100644
index 0000000000..97e67e4652
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -0,0 +1,130 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_TENSOR_HANDLE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_TENSOR_HANDLE_H_
+
+#include <algorithm>
+#include <cstddef>
+#include <map>
+#include <memory>
+#include <queue>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/function.h"
+#include "tensorflow/core/common_runtime/rendezvous_mgr.h"
+#include "tensorflow/core/framework/rendezvous.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/platform/fingerprint.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/thread_annotations.h"
+#include "tensorflow/core/public/session_options.h"
+#include "tensorflow/core/public/version.h"
+
+namespace tensorflow {
+
+// Associates a Tensor and a Device, used in the eager runtime. Internal version
+// executor_of the TFE_TensorHandle struct and the python EagerTensor class
+// (unrelated to python TensorHandle).
+class TensorHandle : public core::RefCounted {
+ public:
+  TensorHandle(const Tensor& t, Device* d, Device* op_device)
+      : dtype(t.dtype()),
+        node_id(0),
+        tensor_(t),
+        device_(d),
+        op_device_(op_device),
+        ctx_(nullptr) {}
+
+  TensorHandle(uint64 node_id, DataType dtype, EagerContext* ctx)
+      : dtype(dtype),
+        node_id(node_id),
+        tensor_(dtype),
+        device_(nullptr),
+        op_device_(nullptr),
+        ctx_(ctx) {
+    DCHECK_GT(node_id, 0);
+  }
+
+  ~TensorHandle() override {}
+
+  Status Tensor(const tensorflow::Tensor** t);
+
+  Status Device(tensorflow::Device** d);
+
+  Status OpDevice(tensorflow::Device** d);
+
+  Status TensorAndDevice(const tensorflow::Tensor** tensor,
+                         tensorflow::Device** device,
+                         tensorflow::Device** op_device);
+
+  // Note that this can be called at most once, and only on non-ready handles,
+  // and makes them ready.
+  void SetTensorAndDevice(const tensorflow::Tensor& tensor,
+                          tensorflow::Device* device,
+                          tensorflow::Device* op_device);
+
+  // dtype for the handle. It must be the same as t.dtype() once the handle is
+  // ready.
+  const DataType dtype;
+
+ private:
+  // If the contents of the Tensor pointed to by this handle is yet to be
+  // computed by a EagerNode, this function will block till that compuatation is
+  // done and the handle is "ready".
+  Status WaitReady();
+
+  bool IsReady();
+
+  // Id for the EagerNode that will compute the value pointed to by this handle.
+  // If the value is 0, the handle is already ready, but not vice-versa.
+  const uint64 node_id;
+
+  tensorflow::Tensor tensor_;
+
+  // TODO(ashankar): device_ == nullptr iff local CPU
+  // This was expedient, but perhaps worth revisiting ('device_' should always
+  // be a valid pointer?)
+  // This can be done if TFE_NewOp() and the TFE_TensorHandle constructors are
+  // provided with the appropriate TFE_Context.
+  //
+  // TODO(ashankar): Reference count TFE_Context to ensure that 'device_' of a
+  // TFE_TensorHandle does not outlive the TFE_Context from which it came?
+  tensorflow::Device* device_;
+
+  // Device in which the op producing this tensor was executed. Equals to
+  // device_ for constant tensors.
+  tensorflow::Device* op_device_;
+
+  mutex ctx_mutex_;
+
+  // `ctx` is only guaranteed to be set if the handle is not "ready". This is
+  // typically true when the handle was produced during async execution.
+  // `ctx` object is not owned and should outlive this handle.
+  EagerContext* ctx_ GUARDED_BY(ctx_mutex_);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_TENSOR_HANDLE_H_
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 701f68b8f7..55ba509065 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1013,12 +1013,13 @@ static tensorflow::eager::TapeTensor TapeTensorFromTensor(PyObject* tensor) {
     TFE_TensorHandle* t = EagerTensor_Handle(tensor);
     tensorflow::int64 id = EagerTensor_id(tensor);
     const tensorflow::Tensor* tensor = nullptr;
-    const tensorflow::Status status = t->Tensor(&tensor);
+    const tensorflow::Status status = t->handle->Tensor(&tensor);
     if (MaybeRaiseExceptionFromStatus(status, nullptr)) {
-      return tensorflow::eager::TapeTensor{id, t->dtype,
+      return tensorflow::eager::TapeTensor{id, t->handle->dtype,
                                            tensorflow::TensorShape({})};
     } else {
-      return tensorflow::eager::TapeTensor{id, t->dtype, tensor->shape()};
+      return tensorflow::eager::TapeTensor{id, t->handle->dtype,
+                                           tensor->shape()};
     }
   }
   tensorflow::int64 id = FastTensorId(tensor);
diff --git a/tensorflow/python/lib/core/py_func.cc b/tensorflow/python/lib/core/py_func.cc
index 02eafd42b3..22317a348c 100644
--- a/tensorflow/python/lib/core/py_func.cc
+++ b/tensorflow/python/lib/core/py_func.cc
@@ -166,7 +166,7 @@ bool IsSingleNone(PyObject* obj) {
 // Retrieves a Tensor from `eager_tensor` and stores it in `output_tensor`.
 tensorflow::Status ExtractTensorFromEagerTensor(const PyObject* eager_tensor,
                                                 const Tensor** output_tensor) {
-  return EagerTensor_Handle(eager_tensor)->Tensor(output_tensor);
+  return EagerTensor_Handle(eager_tensor)->handle->Tensor(output_tensor);
 }
 
 // Calls the registered py function through the trampoline.
-- 
GitLab


From 80a878dddcb04512324cb729a4ef5c92510e01a3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 13:47:03 -0700
Subject: [PATCH 445/960] [XLA] Implement the whole graph execution interface
 and make a test use XlaBuilder. - Add Client::ExecuteGraph. - Make
 client_library_test_base also (partially) support XlaBuilder by using
 template. - Make one testcase in the axpy_simple_test use XlaBuilder. The
 test was slightly changed because currently the builder does not expend
 implicit broadcast automatically.

PiperOrigin-RevId: 190268658
---
 tensorflow/compiler/xla/client/BUILD          |   1 +
 tensorflow/compiler/xla/client/client.cc      |  51 ++++
 tensorflow/compiler/xla/client/client.h       |  27 +++
 tensorflow/compiler/xla/client/local_client.h |   2 +
 .../compiler/xla/client/xla_client/BUILD      |  13 +
 .../xla/client/xla_client/xla_builder.cc      | 222 +++++++++++++++---
 .../xla/client/xla_client/xla_builder.h       |  59 +++--
 .../xla/client/xla_client/xla_builder_test.cc |  54 ++++-
 .../xla/client/xla_client/xla_computation.cc  |  26 ++
 .../xla/client/xla_client/xla_computation.h   |  55 +++++
 .../xla/service/compile_only_service.cc       |   2 +-
 tensorflow/compiler/xla/service/hlo_module.cc |   7 +-
 .../compiler/xla/service/local_service.cc     |   7 +-
 tensorflow/compiler/xla/service/service.cc    |  67 +++++-
 tensorflow/compiler/xla/service/service.h     |   4 +-
 tensorflow/compiler/xla/tests/BUILD           |   2 +
 .../compiler/xla/tests/axpy_simple_test.cc    |   9 +-
 .../xla/tests/client_library_test_base.cc     |  74 +++++-
 .../xla/tests/client_library_test_base.h      |  52 ++--
 19 files changed, 619 insertions(+), 115 deletions(-)
 create mode 100644 tensorflow/compiler/xla/client/xla_client/xla_computation.cc
 create mode 100644 tensorflow/compiler/xla/client/xla_client/xla_computation.h

diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 02356699a2..5094e5ce67 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -74,6 +74,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla:xla_proto",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
         "//tensorflow/compiler/xla/service:session_proto",
         "//tensorflow/core:lib",
diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index d15ccb0c28..5ce3c45528 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -177,6 +177,22 @@ StatusOr<std::unique_ptr<Literal>> Client::ExecuteAndTransfer(
   return Transfer(*data, shape_with_output_layout);
 }
 
+StatusOr<std::unique_ptr<Literal>> Client::ExecuteAndTransfer(
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const ExecutionOptions* execution_options,
+    ExecutionProfile* execution_profile) {
+  TF_ASSIGN_OR_RETURN(
+      std::unique_ptr<GlobalData> data,
+      Execute(computation, arguments, execution_options, execution_profile));
+
+  const Shape* shape_with_output_layout = nullptr;
+  if (execution_options && execution_options->has_shape_with_output_layout()) {
+    shape_with_output_layout = &execution_options->shape_with_output_layout();
+  }
+  return Transfer(*data, shape_with_output_layout);
+}
+
 StatusOr<Computation> Client::LoadSnapshot(const SessionModule& module) {
   LoadComputationSnapshotRequest request;
   *request.mutable_module() = module;
@@ -231,6 +247,41 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
   return MakeUnique<GlobalData>(stub_, response.output());
 }
 
+StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const ExecutionOptions* execution_options,
+    ExecutionProfile* execution_profile) {
+  ExecuteGraphRequest request;
+  *request.mutable_computation() = computation.proto();
+
+  if (execution_options == nullptr) {
+    *request.mutable_execution_options() = CreateDefaultExecutionOptions();
+  } else {
+    *request.mutable_execution_options() = *execution_options;
+  }
+  for (GlobalData* argument : arguments) {
+    CHECK(argument != nullptr) << "Argument pointers must not be null.";
+    *request.add_arguments() = argument->handle();
+  }
+
+  ExecuteResponse response;
+  VLOG(1) << "making execute request: " << request.ShortDebugString();
+  Status s = stub_->ExecuteGraph(&request, &response);
+  VLOG(1) << "done with request";
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  if (execution_profile != nullptr) {
+    *execution_profile = response.profile();
+    // TODO(b/74197823): Get execution stats for the graph and VLOG(1) them.
+  }
+
+  return MakeUnique<GlobalData>(stub_, response.output());
+}
+
 StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
     tensorflow::gtl::ArraySlice<ComputationInstance> computations) {
   ExecuteParallelRequest request;
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index c28380b689..ec87646ebf 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/computation.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/session.pb.h"
 #include "tensorflow/compiler/xla/service_interface.h"
@@ -57,6 +58,21 @@ class Client {
       const ExecutionOptions* execution_options = nullptr,
       ExecutionProfile* execution_profile = nullptr);
 
+  // Executes the computation with the given arguments and returns the global
+  // data that was produced from the execution.
+  // * If execution_options is not nullptr, these options are passed to the
+  //   service to affect how it compiles our computation.  (The pointer does not
+  //   need to live beyond this call.)
+  // * If execution_profile is not nullptr then the pointed-to ExecutionProfile
+  //   will be filled with profile data from the execution.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<GlobalData>> Execute(
+      const XlaComputation& computation,
+      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      const ExecutionOptions* execution_options = nullptr,
+      ExecutionProfile* execution_profile = nullptr);
+
   // A struct to represent a computation instance to be executed.
   // * If execution_options.device_handles is not empty, the computation is
   //   executed on the devices associated with the handles by partitioning the
@@ -137,6 +153,17 @@ class Client {
       const ExecutionOptions* execution_options = nullptr,
       ExecutionProfile* execution_profile = nullptr);
 
+  // Executes the computation with the given arguments and transfers the result
+  // to the client as a literal. Parameters are defined the same as for
+  // Execute() and Transfer().
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
+      const XlaComputation& computation,
+      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      const ExecutionOptions* execution_options = nullptr,
+      ExecutionProfile* execution_profile = nullptr);
+
   // Unregister the memory for the given GlobalData on the device.
   Status Unregister(const GlobalData& data);
 
diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h
index de0ed13c43..2e5d85ba68 100644
--- a/tensorflow/compiler/xla/client/local_client.h
+++ b/tensorflow/compiler/xla/client/local_client.h
@@ -123,6 +123,8 @@ class LocalClient : public Client {
       const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
       const ExecutableBuildOptions& options);
 
+  // TODO(b/74197823): Add a overload of Compile for XlaComputation.
+
   // Copy the literal data to the device with the given ordinal and return as a
   // ScopedShapedBuffer. If non-null the given memory allocator is used for
   // device memory allocation. If null, the default memory allocator for the
diff --git a/tensorflow/compiler/xla/client/xla_client/BUILD b/tensorflow/compiler/xla/client/xla_client/BUILD
index 69df15c988..cc5f551c9c 100644
--- a/tensorflow/compiler/xla/client/xla_client/BUILD
+++ b/tensorflow/compiler/xla/client/xla_client/BUILD
@@ -25,12 +25,25 @@ filegroup(
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
+cc_library(
+    name = "xla_computation",
+    srcs = ["xla_computation.cc"],
+    hdrs = ["xla_computation.h"],
+    deps = [
+        "//tensorflow/compiler/xla:status_macros",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/service:hlo_proto",
+        "//tensorflow/core:lib",
+    ],
+)
+
 # TODO(b/74197823): Replace computation_builder with xla_builder.
 cc_library(
     name = "xla_builder",
     srcs = ["xla_builder.cc"],
     hdrs = ["xla_builder.h"],
     deps = [
+        ":xla_computation",
         "//tensorflow/compiler/xla:literal_util",
         "//tensorflow/compiler/xla:shape_util",
         "//tensorflow/compiler/xla:status_macros",
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index ec646cabe9..90f2b2d73a 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 
+#include <numeric>
 #include <string>
 #include <utility>
 
@@ -80,40 +81,32 @@ void XlaBuilder::NoteError(const Status& error) {
   }
 }
 
-StatusOr<XlaComputation> XlaBuilder::Build() {
-  if (!first_error_.ok()) {
-    string backtrace;
-    first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
-    return AppendStatus(first_error_, backtrace);
-  }
-
-  HloComputationProto entry;
-  ProgramShape* program_shape = entry.mutable_program_shape();
-
-  entry.set_name(name_);
+StatusOr<ProgramShape> XlaBuilder::GetProgramShape(int64* root_id) {
+  TF_RET_CHECK(root_id != nullptr);
+  ProgramShape program_shape;
 
   // Not all instructions can be roots. Walk backwards from the last added
   // instruction until a valid root is found.
-  entry.set_root_id(-1);
-  for (int64 i = instructions_.size() - 1; i >= 0; i--) {
+  int64 index = instructions_.size() - 1;
+  for (; index >= 0; index--) {
     TF_ASSIGN_OR_RETURN(HloOpcode opcode,
-                        StringToHloOpcode(instructions_[i].opcode()));
+                        StringToHloOpcode(instructions_[index].opcode()));
     if (CanBeRoot(opcode)) {
-      entry.set_root_id(instructions_[i].id());
-      *program_shape->mutable_result() = instructions_[i].shape();
       break;
     }
   }
-  if (entry.root_id() == -1) {
+  if (index < 0) {
     return FailedPrecondition("no root instruction was found");
   }
+  *root_id = instructions_[index].id();
+  *program_shape.mutable_result() = instructions_[index].shape();
 
   // Check that the parameter numbers are continuous from 0, and add parameter
   // shapes and names to the program shape.
   const int64 param_count = parameter_numbers_.size();
   for (int64 i = 0; i < param_count; i++) {
-    program_shape->add_parameters();
-    program_shape->add_parameter_names();
+    program_shape.add_parameters();
+    program_shape.add_parameter_names();
   }
   for (const HloInstructionProto& instr : instructions_) {
     // Parameter number uniqueness is guaranteed in XlaBuilder::Parameter(). So
@@ -123,10 +116,35 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
       const int64 index = instr.parameter_number();
       TF_RET_CHECK(index >= 0 && index < param_count)
           << "invalid parameter number: " << index;
-      *program_shape->mutable_parameters(index) = instr.shape();
-      *program_shape->mutable_parameter_names(index) = instr.name();
+      *program_shape.mutable_parameters(index) = instr.shape();
+      *program_shape.mutable_parameter_names(index) = instr.name();
     }
   }
+  return program_shape;
+}
+
+StatusOr<ProgramShape> XlaBuilder::GetProgramShape() {
+  int64 root_id;
+  return GetProgramShape(&root_id);
+}
+
+StatusOr<XlaComputation> XlaBuilder::Build() {
+  if (!first_error_.ok()) {
+    string backtrace;
+    first_error_backtrace_.Dump(tensorflow::DebugWriteToString, &backtrace);
+    return AppendStatus(first_error_, backtrace);
+  }
+
+  HloComputationProto entry;
+  entry.set_name(name_);
+
+  {
+    int64 root_id;
+    ProgramShape program_shape;
+    TF_ASSIGN_OR_RETURN(program_shape, GetProgramShape(&root_id));
+    entry.mutable_program_shape()->Swap(&program_shape);
+    entry.set_root_id(root_id);
+  }
 
   for (auto& instruction : instructions_) {
     entry.add_instructions()->Swap(&instruction);
@@ -149,19 +167,120 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
   return std::move(computation);
 }
 
-XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
-                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  auto op = [&]() -> StatusOr<XlaOp> {
+StatusOr<XlaOp> XlaBuilder::InDimBroadcast(
+    const Shape& shape, const XlaOp& operand,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  HloInstructionProto instr;
+  *instr.mutable_shape() = shape;
+  for (int64 dim : broadcast_dimensions) {
+    instr.add_dimensions(dim);
+  }
+  return AddInstruction(std::move(instr), HloOpcode::kBroadcast, {operand});
+}
+
+StatusOr<XlaOp> XlaBuilder::AddBroadcastSequence(const Shape& output_shape,
+                                                 const XlaOp& operand) {
+  TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
+
+  CHECK(ShapeUtil::IsScalar(operand_shape) ||
+        ShapeUtil::Rank(operand_shape) == ShapeUtil::Rank(output_shape));
+  Shape broadcast_shape =
+      ShapeUtil::ChangeElementType(output_shape, operand_shape.element_type());
+
+  // Do explicit broadcast for scalar.
+  if (ShapeUtil::IsScalar(operand_shape)) {
+    return InDimBroadcast(broadcast_shape, operand, {});
+  }
+
+  // Do explicit broadcast for degenerate broadcast.
+  std::vector<int64> broadcast_dimensions;
+  std::vector<int64> reshaped_dimensions;
+  for (int i = 0; i < ShapeUtil::Rank(operand_shape); i++) {
+    if (operand_shape.dimensions(i) == output_shape.dimensions(i)) {
+      broadcast_dimensions.push_back(i);
+      reshaped_dimensions.push_back(operand_shape.dimensions(i));
+    } else {
+      TF_RET_CHECK(operand_shape.dimensions(i) == 1)
+          << "An explicit broadcast sequence requires the broadcasted "
+             "dimensions to be trivial; operand shape: "
+          << operand_shape << "; output_shape: " << output_shape;
+    }
+  }
+  // Eliminate the size one dimensions.
+  TF_ASSIGN_OR_RETURN(XlaOp reshaped_operand,
+                      Reshape(ShapeUtil::MakeShape(operand_shape.element_type(),
+                                                   reshaped_dimensions),
+                              operand));
+  // Broadcast 'reshape' up to the larger size.
+  return InDimBroadcast(broadcast_shape, reshaped_operand,
+                        broadcast_dimensions);
+}
+
+XlaOp XlaBuilder::BinaryOp(
+    HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
+    tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, lhs.GetShape());
     TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, rhs.GetShape());
-    TF_ASSIGN_OR_RETURN(
-        *instr.mutable_shape(),
-        ShapeInference::InferBinaryOpShape(HloOpcode::kAdd, lhs_shape,
-                                           rhs_shape, broadcast_dimensions));
-    return AddInstruction(std::move(instr), HloOpcode::kAdd, {lhs, rhs});
-  };
-  return NoteErrorOrReturn(op());
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferBinaryOpShape(
+                            binop, lhs_shape, rhs_shape, broadcast_dimensions));
+
+    const int64 lhs_rank = ShapeUtil::Rank(lhs_shape);
+    const int64 rhs_rank = ShapeUtil::Rank(rhs_shape);
+
+    XlaOp updated_lhs = lhs;
+    XlaOp updated_rhs = rhs;
+
+    if (!broadcast_dimensions.empty() && lhs_rank != rhs_rank) {
+      const bool should_broadcast_lhs = lhs_rank < rhs_rank;
+      XlaOp from = should_broadcast_lhs ? lhs : rhs;
+      const Shape& from_shape = should_broadcast_lhs ? lhs_shape : rhs_shape;
+
+      std::vector<int64> to_size;
+      for (int64 size : instr.shape().dimensions()) {
+        to_size.push_back(size);
+      }
+      for (int64 from_dim = 0; from_dim < ShapeUtil::Rank(from_shape);
+           from_dim++) {
+        int64 to_dim = broadcast_dimensions[from_dim];
+        to_size[to_dim] = from_shape.dimensions(from_dim);
+      }
+
+      const Shape& broadcasted_shape =
+          ShapeUtil::MakeShape(from_shape.element_type(), to_size);
+      TF_ASSIGN_OR_RETURN(
+          XlaOp broadcasted_operand,
+          InDimBroadcast(broadcasted_shape, from, broadcast_dimensions));
+
+      updated_lhs = should_broadcast_lhs ? broadcasted_operand : lhs;
+      updated_rhs = !should_broadcast_lhs ? broadcasted_operand : rhs;
+    }
+
+    TF_ASSIGN_OR_RETURN(Shape updated_lhs_shape, updated_lhs.GetShape());
+    if (!ShapeUtil::SameDimensions(instr.shape(), updated_lhs_shape)) {
+      TF_ASSIGN_OR_RETURN(updated_lhs,
+                          AddBroadcastSequence(instr.shape(), updated_lhs));
+    }
+    TF_ASSIGN_OR_RETURN(Shape updated_rhs_shape, updated_rhs.GetShape());
+    if (!ShapeUtil::SameDimensions(instr.shape(), updated_rhs_shape)) {
+      TF_ASSIGN_OR_RETURN(updated_rhs,
+                          AddBroadcastSequence(instr.shape(), updated_rhs));
+    }
+
+    return AddInstruction(std::move(instr), binop, {updated_lhs, updated_rhs});
+  }());
+}
+
+XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(HloOpcode::kAdd, lhs, rhs, broadcast_dimensions);
+}
+
+XlaOp XlaBuilder::Mul(const XlaOp& lhs, const XlaOp& rhs,
+                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  return BinaryOp(HloOpcode::kMultiply, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::ConstantLiteral(const Literal& literal) {
@@ -173,7 +292,7 @@ XlaOp XlaBuilder::ConstantLiteral(const Literal& literal) {
 
 XlaOp XlaBuilder::Call(const XlaComputation& computation,
                        tensorflow::gtl::ArraySlice<XlaOp> operands) {
-  auto op = [&]() -> StatusOr<XlaOp> {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     std::vector<const Shape*> operand_shape_ptrs;
     std::vector<Shape> operand_shapes;
@@ -196,13 +315,12 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
     }
 
     return AddInstruction(std::move(instr), HloOpcode::kCall, operands);
-  };
-  return NoteErrorOrReturn(op());
+  }());
 }
 
 XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
                             const string& name) {
-  auto op = [&]() -> StatusOr<XlaOp> {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     if (parameter_numbers_.find(parameter_number) != parameter_numbers_.end()) {
       return InvalidArgument("parameter %lld already registered",
@@ -213,8 +331,37 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
     instr.set_name(name);
     *instr.mutable_shape() = shape;
     return AddInstruction(std::move(instr), HloOpcode::kParameter);
-  };
-  return NoteErrorOrReturn(op());
+  }());
+}
+
+XlaOp XlaBuilder::Broadcast(
+    const XlaOp& operand, tensorflow::gtl::ArraySlice<int64> broadcast_sizes) {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
+    TF_ASSIGN_OR_RETURN(
+        const Shape& shape,
+        ShapeInference::InferBroadcastShape(operand_shape, broadcast_sizes));
+
+    // The client-level broadcast op just appends dimensions on the left (adds
+    // lowest numbered dimensions). The HLO broadcast instruction is more
+    // flexible and can add new dimensions anywhere. The instruction's
+    // dimensions field maps operand dimensions to dimensions in the broadcast
+    // output, so to append dimensions on the left the instruction's dimensions
+    // should just be the n highest dimension numbers of the output shape where
+    // n is the number of input dimensions.
+    const int64 operand_rank = ShapeUtil::Rank(operand_shape);
+    std::vector<int64> dimensions(operand_rank);
+    for (int i = 0; i < operand_rank; ++i) {
+      dimensions[i] = i + ShapeUtil::Rank(shape) - operand_rank;
+    }
+    return InDimBroadcast(shape, operand, dimensions);
+  }());
+}
+
+StatusOr<XlaOp> XlaBuilder::Reshape(const Shape& shape, const XlaOp& operand) {
+  HloInstructionProto instr;
+  *instr.mutable_shape() = shape;
+  return AddInstruction(std::move(instr), HloOpcode::kReshape, {operand});
 }
 
 XlaOp XlaBuilder::Slice(const XlaOp& operand,
@@ -660,6 +807,7 @@ XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
   }
   for (const auto& operand : operands) {
     instr.add_operand_ids(operand.handle());
+    // TODO(b/74197823): Set metadata and sharding.
   }
   instructions_.push_back(instr);
 
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 455ea3d9cc..407b2df274 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -25,6 +25,7 @@ limitations under the License.
 #include <utility>
 
 #include "tensorflow/compiler/xla/client/padding.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/hlo.pb.h"
 #include "tensorflow/compiler/xla/service/hlo_opcode.h"
@@ -65,38 +66,6 @@ class XlaOp {
   XlaBuilder* builder_;  // Not owned.
 };
 
-// The computation graph that the user builds up with the XlaBuilder.
-//
-// TODO(b/74197823): Replace xla::Computation with this one.
-class XlaComputation {
- public:
-  XlaComputation(const XlaComputation&) = delete;
-  XlaComputation& operator=(const XlaComputation&) = delete;
-
-  XlaComputation(XlaComputation&& from) { *this = std::move(from); }
-
-  XlaComputation& operator=(XlaComputation&& from) {
-    proto_ = std::move(from.proto());
-    unique_id_ = from.unique_id_;
-    return *this;
-  }
-
-  // Returns the "program shape" (parameter and return shapes) for this
-  // computation.
-  const ProgramShape& GetProgramShape() const { return proto_.program_shape(); }
-
-  const HloModuleProto& proto() const { return proto_; }
-
- private:
-  // Creates a null Computation.
-  XlaComputation(const int64 unique_id) : unique_id_(unique_id) {}
-  HloModuleProto* mutable_proto() { return &proto_; }
-  friend class XlaBuilder;
-
-  int64 unique_id_;
-  HloModuleProto proto_;
-};
-
 // A convenient interface for building up computations.
 //
 // Thread-compatible.
@@ -733,6 +702,9 @@ class XlaBuilder {
   // Returns the shape of the given op.
   StatusOr<Shape> GetShape(const XlaOp& op) const;
 
+  // Returns the (inferred) result for the current computation's shape.
+  StatusOr<ProgramShape> GetProgramShape();
+
  private:
   XlaOp AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
                        tensorflow::gtl::ArraySlice<XlaOp> operands = {});
@@ -756,6 +728,29 @@ class XlaBuilder {
 
   StatusOr<const HloInstructionProto*> LookUpInstruction(const XlaOp& op) const;
 
+  // Internal helper method that does the building for an arbitrary binary op.
+  // broadcast_dimensions specifies which dimensions to use for broadcasting
+  // when the operation is between tensors of different ranks.
+  XlaOp BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
+                 tensorflow::gtl::ArraySlice<int64> broadcast_dimensions);
+
+  StatusOr<XlaOp> InDimBroadcast(
+      const Shape& shape, const XlaOp& operand,
+      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions);
+
+  // Internal helper method that creates a sequence of instructions that
+  // performs an explicit broadcast of the operand to the target shape.
+  StatusOr<XlaOp> AddBroadcastSequence(const Shape& output_shape,
+                                       const XlaOp& operand);
+
+  // Internal helper method for creating a Reshape op with the already inferred
+  // shape.
+  StatusOr<XlaOp> Reshape(const Shape& shape, const XlaOp& operand);
+
+  // Returns the (inferred) result for the program shape for the current
+  // computation and fills the root_id in the pointer.
+  StatusOr<ProgramShape> GetProgramShape(int64* root_id);
+
   string name_;  // Name to use for the built computation.
 
   // The first error encountered while building the computation.
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
index a400e4e78b..10d8fa1622 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
@@ -57,16 +57,16 @@ TEST_F(XlaBuilderTest, OnePlusTwo) {
   EXPECT_THAT(root, op::Add(op::Constant(), op::Constant()));
 }
 
-TEST_F(XlaBuilderTest, ParamPlusConstant) {
+TEST_F(XlaBuilderTest, ParamPlusConstantHasScalarBroadcast) {
   XlaBuilder b(TestName());
   auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {3, 5}), "x");
   b.Add(x, b.ConstantR0<float>(1.0));
   TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Add(op::Parameter(), op::Constant()));
+  EXPECT_THAT(root, op::Add(op::Parameter(), op::Broadcast(op::Constant())));
 }
 
-TEST_F(XlaBuilderTest, ParamPlusParam) {
+TEST_F(XlaBuilderTest, ParamPlusParamHasBroadcast) {
   XlaBuilder b(TestName());
   const auto& x_shape = ShapeUtil::MakeShape(S32, {2, 4, 6});
   const auto& y_shape = ShapeUtil::MakeShape(S32, {2, 4});
@@ -79,7 +79,7 @@ TEST_F(XlaBuilderTest, ParamPlusParam) {
 
   TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
   auto root = module->entry_computation()->root_instruction();
-  EXPECT_THAT(root, op::Add(op::Parameter(0), op::Parameter(1)));
+  EXPECT_THAT(root, op::Add(op::Parameter(0), op::Broadcast(op::Parameter(1))));
 }
 
 TEST_F(XlaBuilderTest, XPlusX) {
@@ -133,5 +133,51 @@ TEST_F(XlaBuilderTest, Call) {
                             op::Call(op::Constant(), op::Constant())));
 }
 
+TEST_F(XlaBuilderTest, BinopHasDegenerateBroadcast) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {1, 2, 3}), "x");
+  auto y = b.Parameter(1, ShapeUtil::MakeShape(F32, {1, 2, 1}), "y");
+  b.Add(x, y);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+
+  // Expected:
+  //
+  //  x: f32[1,2,3]  y: f32[1,2,1]
+  //      |               |
+  //      |          reshape: f32[1,2]
+  //      |               |
+  //      |          broadcast: f32[1,2,3]
+  //       \             /
+  //            add
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Add(op::Parameter(0),
+                            op::Broadcast(op::Reshape(op::Parameter(1)))));
+}
+
+TEST_F(XlaBuilderTest, BinopHasInDimAndDegenerateBroadcast) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {2, 3}), "x");
+  auto y = b.Parameter(1, ShapeUtil::MakeShape(F32, {2, 1, 4}), "y");
+  b.Add(x, y, /*broadcast_dimensions=*/{0, 1});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+
+  // The binary operation has in-dim broadcast and degenerate broadcast, should
+  // first do the in-dim broadcast then convert the degnerate broadcast into a
+  // reshape and a broadcast.
+  //
+  // Expected:
+  //
+  //  x: f32[2,3]            y: f32[2,1,4]
+  //      |                        |
+  //  broadcast: f32[2,3,4]  reshape: f32[2,4]
+  //      |                        |
+  //      |                  broadcast: f32[2,3,4]
+  //       \                      /
+  //                 add
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Add(op::Broadcast(op::Parameter(0)),
+                            op::Broadcast(op::Reshape(op::Parameter(1)))));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.cc b/tensorflow/compiler/xla/client/xla_client/xla_computation.cc
new file mode 100644
index 0000000000..3681792eee
--- /dev/null
+++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.cc
@@ -0,0 +1,26 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
+
+#include <utility>
+
+namespace xla {
+
+const ProgramShape& XlaComputation::GetProgramShape() const {
+  return proto_.program_shape();
+}
+
+}  // namespace xla
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
new file mode 100644
index 0000000000..5b89747fdd
--- /dev/null
+++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
@@ -0,0 +1,55 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_CLIENT_XLA_CLIENT_XLA_COMPUTATION_H_
+#define TENSORFLOW_COMPILER_XLA_CLIENT_XLA_CLIENT_XLA_COMPUTATION_H_
+
+#include <utility>
+
+#include "tensorflow/compiler/xla/service/hlo.pb.h"
+#include "tensorflow/compiler/xla/status_macros.h"
+#include "tensorflow/compiler/xla/xla_data.pb.h"
+
+namespace xla {
+
+// The computation graph that the user builds up with the XlaBuilder.
+//
+// TODO(b/74197823): Replace xla::Computation with this one.
+class XlaComputation {
+ public:
+  XlaComputation(const XlaComputation&) = delete;
+  XlaComputation& operator=(const XlaComputation&) = delete;
+
+  XlaComputation(XlaComputation&& from) = default;
+
+  XlaComputation& operator=(XlaComputation&& from) = default;
+
+  // Returns the "program shape" (parameter and return shapes) for this
+  // computation.
+  const ProgramShape& GetProgramShape() const;
+  const HloModuleProto& proto() const { return proto_; }
+
+ private:
+  XlaComputation(const int64 unique_id) : unique_id_(unique_id) {}
+  HloModuleProto* mutable_proto() { return &proto_; }
+  friend class XlaBuilder;
+
+  int64 unique_id_;
+  HloModuleProto proto_;
+};
+
+}  // namespace xla
+
+#endif  // TENSORFLOW_COMPILER_XLA_CLIENT_XLA_CLIENT_XLA_COMPUTATION_H_
diff --git a/tensorflow/compiler/xla/service/compile_only_service.cc b/tensorflow/compiler/xla/service/compile_only_service.cc
index 6664496ab6..c83da9eddc 100644
--- a/tensorflow/compiler/xla/service/compile_only_service.cc
+++ b/tensorflow/compiler/xla/service/compile_only_service.cc
@@ -100,7 +100,7 @@ CompileOnlyService::CompileAheadOfTime(
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModuleConfig> module_config,
         CreateModuleConfig(*program_shape, instance.argument_layouts,
-                           &execution_options, *user_computation));
+                           &execution_options, user_computation));
 
     TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> hlo_module,
                         computation_tracker_.BuildHloModule(
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 2037764dae..595c531ccf 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -237,8 +237,8 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   for (int i = 0; i < expected_program_shape.parameters_size(); ++i) {
     const Shape& parameter_shape =
         module_config.entry_computation_layout().parameter_layout(i).shape();
-    TF_RET_CHECK(
-        ShapeUtil::Equal(expected_program_shape.parameters(i), parameter_shape))
+    TF_RET_CHECK(ShapeUtil::Compatible(expected_program_shape.parameters(i),
+                                       parameter_shape))
         << "HloModuleConfig has different shape for parameter " << i
         << " than the HLO module. Expected: "
         << ShapeUtil::HumanStringWithLayout(
@@ -247,7 +247,8 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
   }
   const Shape& result_shape =
       module_config.entry_computation_layout().result_layout().shape();
-  TF_RET_CHECK(ShapeUtil::Equal(expected_program_shape.result(), result_shape))
+  TF_RET_CHECK(
+      ShapeUtil::Compatible(expected_program_shape.result(), result_shape))
       << "HloModuleConfig has different result shape than the HLO module. "
          "Expected: "
       << ShapeUtil::HumanStringWithLayout(expected_program_shape.result())
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 5690a89909..1e2d8eea58 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -146,10 +146,9 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
     LayoutUtil::SetToDefaultLayout(
         execution_options.mutable_shape_with_output_layout());
   }
-  TF_ASSIGN_OR_RETURN(
-      std::unique_ptr<HloModuleConfig> module_config,
-      CreateModuleConfig(*program_shape, argument_layouts, &execution_options,
-                         *user_computation));
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModuleConfig> module_config,
+                      CreateModuleConfig(*program_shape, argument_layouts,
+                                         &execution_options, user_computation));
 
   TF_ASSIGN_OR_RETURN(
       se::StreamExecutor * executor,
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 0becc9d8f8..04487a4795 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -272,7 +272,7 @@ StatusOr<std::unique_ptr<HloModuleConfig>> Service::CreateModuleConfig(
     const ProgramShape& program_shape,
     tensorflow::gtl::ArraySlice<const Shape*> argument_shapes,
     const ExecutionOptions* execution_options,
-    const UserComputation& user_computation) {
+    const UserComputation* user_computation) {
   auto config = MakeUnique<HloModuleConfig>(program_shape);
   auto* computation_layout = config->mutable_entry_computation_layout();
 
@@ -286,8 +286,15 @@ StatusOr<std::unique_ptr<HloModuleConfig>> Service::CreateModuleConfig(
     // ProgramShape.
     if (!ShapeUtil::Compatible(*argument_shapes[i],
                                program_shape.parameters(i))) {
+      if (user_computation == nullptr) {
+        return InvalidArgument(
+            "Argument does not match shape of computation parameter %d: want "
+            "%s, got %s",
+            i, ShapeUtil::HumanString(program_shape.parameters(i)).c_str(),
+            ShapeUtil::HumanString(*argument_shapes[i]).c_str());
+      }
       return InvalidParameterArgument(
-          *user_computation.ParameterMetadata(i).value(),
+          *user_computation->ParameterMetadata(i).value(),
           "Argument does not match shape of computation parameter %d: want %s, "
           "got %s",
           i, ShapeUtil::HumanString(program_shape.parameters(i)).c_str(),
@@ -330,7 +337,7 @@ StatusOr<std::unique_ptr<HloModuleConfig>> Service::CreateModuleConfig(
     const ProgramShape& program_shape,
     tensorflow::gtl::ArraySlice<const ShapedBuffer*> arguments,
     const ExecutionOptions& execution_options,
-    const UserComputation& user_computation) {
+    const UserComputation* user_computation) {
   std::vector<const Shape*> argument_shapes;
   for (const auto* arg : arguments) {
     argument_shapes.push_back(&arg->on_host_shape());
@@ -778,7 +785,7 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
     TF_ASSIGN_OR_RETURN(
         std::unique_ptr<HloModuleConfig> module_config,
         CreateModuleConfig(*program_shape, replicated_arguments.front(),
-                           request.execution_options(), *user_computation));
+                           request.execution_options(), user_computation));
     VLOG(3) << "ExecuteParallel created HloModuleConfig computation layout: "
             << module_config->entry_computation_layout().ToString();
 
@@ -894,7 +901,7 @@ tensorflow::Status Service::Execute(const ExecuteRequest* arg,
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<HloModuleConfig> module_config,
       CreateModuleConfig(*program_shape, replicated_arguments.front(),
-                         arg->execution_options(), *user_computation));
+                         arg->execution_options(), user_computation));
 
   VLOG(3) << "Execute created HloModuleConfig computation layout: "
           << module_config->entry_computation_layout().ToString();
@@ -935,9 +942,49 @@ tensorflow::Status Service::Execute(const ExecuteRequest* arg,
   return tensorflow::Status::OK();
 }
 
-tensorflow::Status Service::ExecuteGraph(const ExecuteGraphRequest* /*arg*/,
-                                         ExecuteResponse* /*result*/) {
-  return Unimplemented("execute-graph is not yet implemented");
+tensorflow::Status Service::ExecuteGraph(const ExecuteGraphRequest* arg,
+                                         ExecuteResponse* result) {
+  VLOG(1) << "running execute-graph request";
+
+  if (!arg->has_computation()) {
+    return InvalidArgument("computations may not be empty");
+  }
+
+  // TODO(b/74197823): Handle partitioning.
+
+  TF_ASSIGN_OR_RETURN(auto replicas, Replicas(*execute_backend_,
+                                              SingleComputationDeviceHandle()));
+  TF_ASSIGN_OR_RETURN(
+      std::vector<std::vector<const ShapedBuffer*>> replicated_arguments,
+      ResolveAndValidateArguments(arg->arguments(), replicas));
+
+  TF_ASSIGN_OR_RETURN(const auto& config,
+                      CreateModuleConfig(arg->computation().program_shape(),
+                                         replicated_arguments.front(),
+                                         arg->execution_options()));
+
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
+                      HloModule::CreateFromProto(arg->computation(), *config));
+  TF_RETURN_IF_ERROR(MaybeDumpHloModule(*module));
+
+  TF_ASSIGN_OR_RETURN(module, execute_backend_->compiler()->RunHloPasses(
+                                  std::move(module),
+                                  execute_backend_->default_stream_executor(),
+                                  /*device_allocator=*/nullptr));
+  TF_ASSIGN_OR_RETURN(
+      std::unique_ptr<Executable> executable,
+      execute_backend_->compiler()->RunBackend(
+          std::move(module), execute_backend_->default_stream_executor(),
+          /*device_allocator=*/nullptr));
+
+  TF_ASSIGN_OR_RETURN(
+      *result->mutable_output(),
+      ExecuteAndRegisterResult(
+          executable.get(), replicated_arguments, execute_backend_.get(),
+          "result of " + arg->computation().name(), result->mutable_profile()));
+
+  VLOG(1) << "successfully completed 'execute-graph' request";
+  return tensorflow::Status::OK();
 }
 
 tensorflow::Status Service::ExecuteAsync(const ExecuteAsyncRequest* arg,
@@ -967,7 +1014,7 @@ tensorflow::Status Service::ExecuteAsync(const ExecuteAsyncRequest* arg,
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<HloModuleConfig> module_config,
       CreateModuleConfig(*program_shape, replicated_arguments.front(),
-                         arg->execution_options(), *user_computation));
+                         arg->execution_options(), user_computation));
 
   VLOG(3) << "ExecuteAsync created HloModuleConfig computation layout: "
           << module_config->entry_computation_layout().ToString();
@@ -1268,7 +1315,7 @@ tensorflow::Status Service::ComputeConstant(const ComputeConstantRequest* arg,
 
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModuleConfig> module_config,
                       CreateModuleConfig(program_shape, {}, execution_options,
-                                         *user_computation));
+                                         user_computation));
 
   // Exclude dead parameter instructions for the purpose of computing constants.
   TF_ASSIGN_OR_RETURN(
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index 96352d9096..a76bdd89c7 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -258,7 +258,7 @@ class Service : public ServiceInterface {
       const ProgramShape& program_shape,
       tensorflow::gtl::ArraySlice<const ShapedBuffer*> arguments,
       const ExecutionOptions& execution_options,
-      const UserComputation& user_computation);
+      const UserComputation* user_computation = nullptr);
 
  protected:
   friend class LocalExecutable;
@@ -286,7 +286,7 @@ class Service : public ServiceInterface {
       const ProgramShape& program_shape,
       tensorflow::gtl::ArraySlice<const Shape*> argument_shapes,
       const ExecutionOptions* execution_options,
-      const UserComputation& user_computation);
+      const UserComputation* user_computation = nullptr);
 
   // Builds an Executable for the given parameters.
   //
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 7fb7919674..e81e862c49 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -190,6 +190,7 @@ cc_library(
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
         "//tensorflow/core:lib",
@@ -386,6 +387,7 @@ xla_test(
     deps = [
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/axpy_simple_test.cc b/tensorflow/compiler/xla/tests/axpy_simple_test.cc
index 3f6fd7c65d..ec3b46acfe 100644
--- a/tensorflow/compiler/xla/tests/axpy_simple_test.cc
+++ b/tensorflow/compiler/xla/tests/axpy_simple_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
 #include "tensorflow/compiler/xla/tests/test_macros.h"
@@ -28,11 +29,11 @@ namespace {
 class AxpySimpleTest : public ClientLibraryTestBase {};
 
 TEST_F(AxpySimpleTest, AxTenValues) {
-  ComputationBuilder builder(client_, "ax_10");
+  XlaBuilder builder("ax_10");
   auto alpha = builder.ConstantR0<float>(3.1415926535);
   auto x = builder.ConstantR1<float>(
       {-1.0, 1.0, 2.0, -2.0, -3.0, 3.0, 4.0, -4.0, -5.0, 5.0});
-  auto ax = builder.Mul(alpha, x);
+  builder.Mul(alpha, x);
 
   std::vector<float> expected = {
       -3.14159265, 3.14159265,  6.28318531,   -6.28318531,  -9.42477796,
@@ -46,7 +47,7 @@ XLA_TEST_F(AxpySimpleTest, AxpyZeroValues) {
   auto x = builder.ConstantR1<float>({});
   auto y = builder.ConstantR1<float>({});
   auto ax = builder.Mul(alpha, x);
-  auto axpy = builder.Add(ax, y);
+  builder.Add(ax, y);
 
   std::vector<float> expected = {};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -60,7 +61,7 @@ TEST_F(AxpySimpleTest, AxpyTenValues) {
   auto y = builder.ConstantR1<float>(
       {5.0, -5.0, -4.0, 4.0, 3.0, -3.0, -2.0, 2.0, 1.0, -1.0});
   auto ax = builder.Mul(alpha, x);
-  auto axpy = builder.Add(ax, y);
+  builder.Add(ax, y);
 
   TF_ASSERT_OK_AND_ASSIGN(ProgramShape shape, builder.GetProgramShape());
 
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index a677986cd9..3cae51576f 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -95,6 +95,20 @@ StatusOr<std::unique_ptr<Literal>> ClientLibraryTestBase::ExecuteAndTransfer(
                                      &execution_options);
 }
 
+StatusOr<std::unique_ptr<Literal>> ClientLibraryTestBase::ExecuteAndTransfer(
+    const XlaComputation& computation,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const Shape* shape_with_output_layout) {
+  ExecutionOptions execution_options = execution_options_;
+  if (shape_with_output_layout != nullptr) {
+    *execution_options.mutable_shape_with_output_layout() =
+        *shape_with_output_layout;
+  }
+  return client_->ExecuteAndTransfer(computation, arguments,
+                                     &execution_options);
+}
+
+template <>
 StatusOr<std::unique_ptr<Literal>> ClientLibraryTestBase::ExecuteAndTransfer(
     ComputationBuilder* builder,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments,
@@ -104,6 +118,15 @@ StatusOr<std::unique_ptr<Literal>> ClientLibraryTestBase::ExecuteAndTransfer(
   return ExecuteAndTransfer(computation, arguments, shape_with_output_layout);
 }
 
+template <>
+StatusOr<std::unique_ptr<Literal>> ClientLibraryTestBase::ExecuteAndTransfer(
+    XlaBuilder* builder, tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const Shape* shape_with_output_layout) {
+  // Build the computation, as a convenience.
+  TF_ASSIGN_OR_RETURN(auto computation, builder->Build());
+  return ExecuteAndTransfer(computation, arguments, shape_with_output_layout);
+}
+
 std::unique_ptr<GlobalData> ClientLibraryTestBase::ExecuteOrDie(
     ComputationBuilder* builder,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
@@ -142,16 +165,18 @@ void ClientLibraryTestBase::ComputeAndCompareR1(
                                                   arguments);
 }
 
+template <typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareLiteral(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments,
     const Shape* shape_with_layout) {
   EXPECT_IS_OK(ComputeAndCompareLiteralWithStatus(builder, expected, arguments,
                                                   shape_with_layout));
 }
 
+template <typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareLiteral(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
     const Shape* shape_with_layout) {
   EXPECT_IS_OK(ComputeAndCompareLiteralWithStatus(builder, expected, arguments,
@@ -249,8 +274,28 @@ ClientLibraryTestBase::ComputeAndCompareLiteralWithAllInputLayouts(
   return choose(0);
 }
 
+tensorflow::Status
+ClientLibraryTestBase::ComputeAndCompareLiteralWithAllOutputLayouts(
+    const xla::XlaComputation& /*computation*/, const Literal& /*expected*/,
+    tensorflow::gtl::ArraySlice<GlobalData*> /*arguments*/,
+    const std::function<void(const Literal& actual,
+                             const string& error_message)>& /*verify_output*/) {
+  return Unimplemented("not yet implemented for XlaComputation");
+}
+
+tensorflow::Status
+ClientLibraryTestBase::ComputeAndCompareLiteralWithAllInputLayouts(
+    const xla::XlaComputation& /*computation*/, const Literal& /*expected*/,
+    tensorflow::gtl::ArraySlice<GlobalData*> /*arguments*/,
+    const std::function<void(const Literal& actual,
+                             const string& error_message)>& /*verify_output*/,
+    const Shape* /*output_with_layout*/) {
+  return Unimplemented("not yet implemented for XlaComputation");
+}
+
+template <typename BuilderT>
 tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments_passed_in,
     const Shape* shape_with_layout) {
   std::vector<GlobalData*> arguments(arguments_passed_in.begin(),
@@ -307,8 +352,9 @@ tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
   return tensorflow::Status::OK();
 }
 
+template <typename BuilderT>
 tensorflow::Status ClientLibraryTestBase::ComputeAndCompareLiteralWithStatus(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments_passed_in,
     ErrorSpec error, const Shape* shape_with_layout) {
   std::vector<GlobalData*> arguments(arguments_passed_in.begin(),
@@ -563,4 +609,24 @@ ComputationDataHandle ClientLibraryTestBase::CreateConstantFromLiteral(
       use_bfloat16_ ? *LiteralTestUtil::ConvertF32ToBF16(literal) : literal);
 }
 
+template void ClientLibraryTestBase::ComputeAndCompareLiteral(
+    ComputationBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const Shape* shape_with_layout);
+
+template void ClientLibraryTestBase::ComputeAndCompareLiteral(
+    XlaBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+    const Shape* shape_with_layout);
+
+template void ClientLibraryTestBase::ComputeAndCompareLiteral(
+    ComputationBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
+    const Shape* shape_with_layout);
+
+template void ClientLibraryTestBase::ComputeAndCompareLiteral(
+    XlaBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
+    const Shape* shape_with_layout);
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index ba0319990b..b553beb01a 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation.h"
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -94,15 +95,22 @@ class ClientLibraryTestBase : public ::testing::Test {
   StatusOr<std::unique_ptr<GlobalData>> Execute(
       ComputationBuilder* builder,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments);
+
+  template <typename BuilderT>
   StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
-      ComputationBuilder* builder,
-      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      BuilderT* builder, tensorflow::gtl::ArraySlice<GlobalData*> arguments,
       const Shape* shape_with_output_layout = nullptr);
+
   StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
       const Computation& computation,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments,
       const Shape* shape_with_output_layout = nullptr);
 
+  StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
+      const XlaComputation& computation,
+      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      const Shape* shape_with_output_layout = nullptr);
+
   // Convenience OrDie variants of above methods.
   std::unique_ptr<GlobalData> ExecuteOrDie(
       ComputationBuilder* builder,
@@ -130,12 +138,12 @@ class ClientLibraryTestBase : public ::testing::Test {
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
 
-  template <typename NativeT>
-  void ComputeAndCompareR1(ComputationBuilder* builder,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR1(BuilderT* builder,
                            tensorflow::gtl::ArraySlice<NativeT> expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
-  template <typename NativeT>
-  void ComputeAndCompareR1(ComputationBuilder* builder,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR1(BuilderT* builder,
                            tensorflow::gtl::ArraySlice<NativeT> expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
@@ -179,22 +187,26 @@ class ClientLibraryTestBase : public ::testing::Test {
   // Build and run the computation and compare the result with the given
   // literal. shape_with_layout indicates the result layout to request when
   // calling Execute.
+  template <typename BuilderT>
   void ComputeAndCompareLiteral(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments,
       const Shape* shape_with_layout = nullptr);
+  template <typename BuilderT>
   void ComputeAndCompareLiteral(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
       const Shape* shape_with_layout = nullptr);
 
   // ComputeAndCompare variant which returns an error status.
+  template <typename BuilderT>
   tensorflow::Status ComputeAndCompareLiteralWithStatus(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments,
       const Shape* shape_with_layout = nullptr);
+  template <typename BuilderT>
   tensorflow::Status ComputeAndCompareLiteralWithStatus(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
       const Shape* shape_with_layout = nullptr);
 
@@ -399,6 +411,18 @@ class ClientLibraryTestBase : public ::testing::Test {
                                const string& error_message)>& verify_output,
       const Shape* output_with_layout = nullptr);
 
+  tensorflow::Status ComputeAndCompareLiteralWithAllOutputLayouts(
+      const xla::XlaComputation& computation, const Literal& expected,
+      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      const std::function<void(const Literal& actual,
+                               const string& error_message)>& verify_output);
+  tensorflow::Status ComputeAndCompareLiteralWithAllInputLayouts(
+      const xla::XlaComputation& computation, const Literal& expected,
+      tensorflow::gtl::ArraySlice<GlobalData*> arguments,
+      const std::function<void(const Literal& actual,
+                               const string& error_message)>& verify_output,
+      const Shape* output_with_layout = nullptr);
+
   // Executes the computation and calculates the expected reference value using
   // the HloEvaluator. Returns two literal in the order of (expected, actual).
   StatusOr<std::pair<std::unique_ptr<Literal>, std::unique_ptr<Literal>>>
@@ -440,9 +464,9 @@ void ClientLibraryTestBase::ComputeAndCompareR0(
                                                   arguments, error);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR1(
-    ComputationBuilder* builder, tensorflow::gtl::ArraySlice<NativeT> expected,
+    BuilderT* builder, tensorflow::gtl::ArraySlice<NativeT> expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR1<NativeT>(expected);
@@ -450,9 +474,9 @@ void ClientLibraryTestBase::ComputeAndCompareR1(
                                                   arguments);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR1(
-    ComputationBuilder* builder, tensorflow::gtl::ArraySlice<NativeT> expected,
+    BuilderT* builder, tensorflow::gtl::ArraySlice<NativeT> expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
                     std::is_same<NativeT, double>::value ||
-- 
GitLab


From 2219b88a3d5154b9158a1902b061cad6cae2d0a8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 14:00:39 -0700
Subject: [PATCH 446/960] Fix behavior of bucket_by_sequence_length with tuple
 Dataset elements Fixes #17932

PiperOrigin-RevId: 190270732
---
 .../python/kernel_tests/bucketing_test.py     | 25 +++++++++++++++++++
 .../contrib/data/python/ops/grouping.py       |  4 +--
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
index 94f800e8a5..d0131896a1 100644
--- a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
@@ -468,6 +468,31 @@ class BucketBySequenceLength(test.TestCase):
     self.assertEqual(sorted(batch_sizes), sorted(batch_sizes_val))
     self.assertEqual(sorted(boundaries), sorted(lengths_val))
 
+  def testTupleElements(self):
+
+    def elements_gen():
+      text = [[1, 2, 3], [3, 4, 5, 6, 7], [1, 2], [8, 9, 0, 2, 3]]
+      label = [1, 2, 1, 2]
+      for x, y in zip(text, label):
+        yield (x, y)
+
+    def element_length_fn(x, y):
+      del y
+      return array_ops.shape(x)[0]
+
+    dataset = dataset_ops.Dataset.from_generator(
+        generator=elements_gen,
+        output_shapes=(tensor_shape.TensorShape([None]),
+                       tensor_shape.TensorShape([])),
+        output_types=(dtypes.int32, dtypes.int32))
+    dataset = dataset.apply(grouping.bucket_by_sequence_length(
+        element_length_func=element_length_fn,
+        bucket_batch_sizes=[2, 2, 2],
+        bucket_boundaries=[0, 8]))
+    shapes = dataset.output_shapes
+    self.assertEqual([None, None], shapes[0].as_list())
+    self.assertEqual([None], shapes[1].as_list())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/grouping.py b/tensorflow/contrib/data/python/ops/grouping.py
index ae10d2eb22..36591c055a 100644
--- a/tensorflow/contrib/data/python/ops/grouping.py
+++ b/tensorflow/contrib/data/python/ops/grouping.py
@@ -140,9 +140,9 @@ def bucket_by_sequence_length(element_length_func,
 
     batch_sizes = constant_op.constant(bucket_batch_sizes, dtype=dtypes.int64)
 
-    def element_to_bucket_id(element):
+    def element_to_bucket_id(*args):
       """Return int64 id of the length bucket for this element."""
-      seq_length = element_length_func(element)
+      seq_length = element_length_func(*args)
 
       boundaries = list(bucket_boundaries)
       buckets_min = [np.iinfo(np.int32).min] + boundaries
-- 
GitLab


From 13ef0af4867477cdda7e0b294e61560c2952df42 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 23 Mar 2018 14:19:37 -0700
Subject: [PATCH 447/960] Fix buffer overflow when fetching resources.

PiperOrigin-RevId: 190273682
---
 .../python/kernel_tests/resource_variable_ops_test.py       | 6 ++++++
 tensorflow/python/lib/core/ndarray_tensor.cc                | 4 +++-
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 2dc993f811..563eeff2a6 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -103,6 +103,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       v = resource_variable_ops.ResourceVariable(False, name="bool_test")
       self.assertAllEqual(bool(v), False)
 
+  def testFetchHandle(self):
+    with self.test_session():
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1], name="foo")
+      self.assertGreater(len(handle.eval()), 0)
+
   def testAssignVariableDtypeMismatchEager(self):
     with context.eager_mode():
       handle = resource_variable_ops.var_handle_op(
diff --git a/tensorflow/python/lib/core/ndarray_tensor.cc b/tensorflow/python/lib/core/ndarray_tensor.cc
index 994af69386..a07e305ffb 100644
--- a/tensorflow/python/lib/core/ndarray_tensor.cc
+++ b/tensorflow/python/lib/core/ndarray_tensor.cc
@@ -267,7 +267,9 @@ gtl::InlinedVector<npy_intp, 4> GetPyArrayDimensionsForTensor(
   const int ndims = TF_NumDims(tensor);
   gtl::InlinedVector<npy_intp, 4> dims(ndims);
   if (TF_TensorType(tensor) == TF_RESOURCE) {
-    dims[0] = TF_TensorByteSize(tensor);
+    CHECK_EQ(ndims, 0)
+        << "Fetching of non-scalar resource tensors is not supported.";
+    dims.push_back(TF_TensorByteSize(tensor));
     *nelems = dims[0];
   } else {
     *nelems = 1;
-- 
GitLab


From 18832ec8497a6acc6f828808e5ea3a2859548efa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 14:41:02 -0700
Subject: [PATCH 448/960] Benchmarker for LPIRC CVPR2018.

PiperOrigin-RevId: 190276899
---
 .../org/tensorflow/ovic/OvicBenchmarker.java  |  197 ++++
 .../org/tensorflow/ovic/OvicClassifier.java   |  209 ++++
 .../ovic/OvicSingleImageResult.java           |   54 +
 .../tensorflow/ovic/OvicClassifierTest.java   |  176 +++
 .../lite/java/ovic/src/testdata/labels.txt    | 1001 +++++++++++++++++
 5 files changed, 1637 insertions(+)
 create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java
 create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java
 create mode 100644 tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java
 create mode 100644 tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java
 create mode 100644 tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt

diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java
new file mode 100644
index 0000000000..d0102883e6
--- /dev/null
+++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicBenchmarker.java
@@ -0,0 +1,197 @@
+/*Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.ovic;
+
+import android.graphics.Bitmap;
+import android.os.SystemClock;
+import android.util.Log;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+
+/**
+ * Class that benchmarks image classifier models.
+ *
+ * <p>===================== General workflow =======================
+ *
+ * <pre>{@code
+ * benchmarker = new OvicBenchmarker();
+ * benchmarker.getReadyToTest(labelInputStream, model);
+ * while (!benchmarker.shouldStop()) {
+ *   Bitmap bitmap = ...
+ *   benchmarker.doTestIteration(bitmap);
+ * }
+ * }</pre>
+ */
+public class OvicBenchmarker {
+  /** Tag for the {@link Log}. */
+  private static final String TAG = "OvicBenchmarker";
+
+  /** Evaluation transformation parameters. */
+  private static final float CENTRAL_FRACTION = 0.875f;
+
+  /** Dimensions of inputs. */
+  private static final int DIM_BATCH_SIZE = 1;
+  private static final int DIM_PIXEL_SIZE = 3;
+  private int imgHeight = 224;
+  private int imgWidth = 224;
+
+  /* Preallocated buffers for storing image data in. */
+  private int[] intValues = null;
+
+  /** A ByteBuffer to hold image data, to be feed into classifier as inputs. */
+  private ByteBuffer imgData = null;
+
+  private OvicClassifier classifier;
+
+  /** Total runtime in ms. */
+  private double totalRuntime = 0.0;
+  /** Total allowed runtime in ms. */
+  private double wallTime = 20000 * 30.0;
+
+  private Boolean benchmarkStarted = null;
+
+  /**
+   * Initializes an {@link OvicBenchmarker}
+   *
+   * @param wallTime: a double number specifying the total amount of time to benchmark.
+   */
+  public OvicBenchmarker(double wallTime) {
+    benchmarkStarted = false;
+    totalRuntime = 0.0;
+    this.wallTime = wallTime;
+  }
+
+  /** Check whether the benchmarker should stop. */
+  public Boolean shouldStop() {
+    if (totalRuntime >= wallTime) {
+      Log.e(
+          TAG,
+          "Total runtime "
+              + Double.toString(totalRuntime)
+              + " exceeded walltime "
+              + Double.toString(wallTime));
+      return true;
+    }
+    return false;
+  }
+
+  /** Check whether the benchmarker is ready to start classifying images. */
+  public Boolean readyToTest() {
+    return (classifier != null);
+  }
+
+  /**
+   * Getting the benchmarker ready for classifying images.
+   *
+   * @param labelInputStream: an {@link InputStream} specifying where the list of labels should be
+   *     read from.
+   * @param model: a {@link MappedByteBuffer} model to benchmark.
+   */
+  public void getReadyToTest(InputStream labelInputStream, MappedByteBuffer model) {
+    try {
+      Log.i(TAG, "Creating classifier.");
+      classifier = new OvicClassifier(labelInputStream, model);
+      int [] inputDims = classifier.getInputDims();
+      imgHeight = inputDims[1];
+      imgWidth = inputDims[2];
+      // Only accept QUANTIZED_UINT8 input.
+      imgData = ByteBuffer.allocateDirect(DIM_BATCH_SIZE * imgHeight * imgWidth * DIM_PIXEL_SIZE);
+      imgData.order(ByteOrder.nativeOrder());
+      intValues = new int[imgHeight * imgWidth];
+    } catch (Exception e) {
+        Log.e(TAG, e.getMessage());
+        Log.e(TAG, "Failed to initialize ImageNet classifier for the benchmarker.");
+    }
+  }
+
+  /** Return how many classes are predicted per image. */
+  public int getNumPredictions() {
+    return classifier.getNumPredictions();
+  }
+
+  /**
+   * Perform test on a single bitmap image.
+   *
+   * @param bitmap: a {@link Bitmap} image to classify.
+   */
+  public OvicSingleImageResult doTestIteration(Bitmap bitmap)
+      throws IOException, InterruptedException {
+    if (shouldStop() || !readyToTest()) {
+      return null;
+    }
+    OvicSingleImageResult iterResult = null;
+    try {
+      Log.i(TAG, "Converting bitmap.");
+      convertBitmapToInput(bitmap);
+      Log.i(TAG, "Classifying image.");
+      iterResult = classifier.classifyByteBuffer(imgData);
+    } catch (RuntimeException e) {
+      Log.e(TAG, e.getMessage());
+      Log.e(TAG, "Failed to classify image.");
+    }
+    if (iterResult == null || iterResult.latency == null) {
+      throw new RuntimeException("Classification result or timing is invalid.");
+    }
+    Log.d(TAG, "Native inference latency: " + iterResult.latency);
+    Log.i(TAG, iterResult.toString());
+
+    if (!benchmarkStarted) {  // Skip the first image to discount warming-up time.
+      benchmarkStarted = true;
+    } else {
+      totalRuntime += (double) iterResult.latency;
+    }
+    return iterResult;
+  }
+
+  /**
+   * Writes Image data into a {@link ByteBuffer}.
+   *
+   * @param bitmap: a {@link Bitmap} source image.
+   */
+  private void convertBitmapToInput(Bitmap bitmap) throws RuntimeException {
+    if (imgData == null) {
+      throw new RuntimeException("Benchmarker is not yet ready to test.");
+    }
+    imgData.rewind();
+    // Perform transformations corresponding to evaluation mode.
+    float width = (float) bitmap.getWidth();
+    float height = (float) bitmap.getHeight();
+    int stWidth = Math.round((width - width * CENTRAL_FRACTION) / 2);
+    int stHeight = Math.round((height - height * CENTRAL_FRACTION) / 2);
+    int newWidth = Math.round(width - stWidth * 2);
+    int newHeight = Math.round(height - stHeight * 2);
+    bitmap = Bitmap.createBitmap(bitmap, stWidth, stHeight, newWidth, newHeight);
+    bitmap = Bitmap.createScaledBitmap(bitmap, imgWidth, imgHeight, true);
+    bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
+
+    // Convert the image to ByteBuffer.
+    int pixel = 0;
+    long startTime = SystemClock.uptimeMillis();
+
+    for (int i = 0; i < imgHeight; ++i) {
+      for (int j = 0; j < imgWidth; ++j) {
+        final int val = intValues[pixel++];
+        imgData.put((byte) ((val >> 16) & 0xFF));
+        imgData.put((byte) ((val >> 8) & 0xFF));
+        imgData.put((byte) (val & 0xFF));
+      }
+    }
+    long endTime = SystemClock.uptimeMillis();
+    Log.d(TAG, "Timecost to put values into ByteBuffer: " + Long.toString(endTime - startTime));
+  }
+}
diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java
new file mode 100644
index 0000000000..b2dfd8f2e7
--- /dev/null
+++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicClassifier.java
@@ -0,0 +1,209 @@
+/*Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.ovic;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.nio.MappedByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.AbstractMap;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import org.tensorflow.lite.Interpreter;
+import org.tensorflow.lite.TestHelper;
+
+/** Benchmark ImageNet Classifier with Tensorflow Lite. */
+public class OvicClassifier {
+
+  /** Tag for the {@link Log}. */
+  private static final String TAG = "OvicClassifier";
+
+  /** Number of results to show (i.e. the "K" in top-K predictions). */
+  private static final int RESULTS_TO_SHOW = 5;
+
+  /** An instance of the driver class to run model inference with Tensorflow Lite. */
+  private Interpreter tflite;
+
+  /** Labels corresponding to the output of the vision model. */
+  private List<String> labelList;
+
+  /** An array to hold inference results, to be feed into Tensorflow Lite as outputs. */
+  private byte[][] inferenceOutputArray = null;
+  /** An array to hold final prediction probabilities. */
+  private float[][] labelProbArray = null;
+
+  /** Input resultion. */
+  private int[] inputDims = null;
+  /** Whether the model runs as float or quantized. */
+  private Boolean outputIsFloat = null;
+
+  private PriorityQueue<Map.Entry<Integer, Float>> sortedLabels =
+      new PriorityQueue<>(
+          RESULTS_TO_SHOW,
+          new Comparator<Map.Entry<Integer, Float>>() {
+            @Override
+            public int compare(Map.Entry<Integer, Float> o1, Map.Entry<Integer, Float> o2) {
+              return (o1.getValue()).compareTo(o2.getValue());
+            }
+          });
+
+  /** Initializes an {@code OvicClassifier}. */
+  OvicClassifier(InputStream labelInputStream, MappedByteBuffer model)
+      throws IOException, RuntimeException {
+    if (model == null) {
+      throw new RuntimeException("Input model is empty.");
+    }
+    labelList = loadLabelList(labelInputStream);
+    // OVIC uses one thread for CPU inference.
+    tflite = new Interpreter(model, 1);
+    inputDims = TestHelper.getInputDims(tflite, 0);
+    if (inputDims.length != 4) {
+      throw new RuntimeException("The model's input dimensions must be 4 (BWHC).");
+    }
+    if (inputDims[0] != 1) {
+      throw new RuntimeException("The model must have a batch size of 1, got "
+          + inputDims[0] + " instead.");
+    }
+    if (inputDims[3] != 3) {
+      throw new RuntimeException("The model must have three color channels, got "
+          + inputDims[3] + " instead.");
+    }
+    int minSide = Math.min(inputDims[1], inputDims[2]);
+    int maxSide = Math.max(inputDims[1], inputDims[2]);
+    if (minSide <= 0 || maxSide > 1000) {
+      throw new RuntimeException("The model's resolution must be between (0, 1000].");
+    }
+    String outputDataType = TestHelper.getOutputDataType(tflite, 0);
+    if (outputDataType.equals("float")) {
+      outputIsFloat = true;
+    } else if (outputDataType.equals("byte")) {
+      outputIsFloat = false;
+    } else {
+      throw new RuntimeException("Cannot process output type: " + outputDataType);
+    }
+    inferenceOutputArray = new byte[1][labelList.size()];
+    labelProbArray = new float[1][labelList.size()];
+  }
+
+  /** Classifies a {@link ByteBuffer} image. */
+  // @throws RuntimeException if model is uninitialized.
+  OvicSingleImageResult classifyByteBuffer(ByteBuffer imgData) throws RuntimeException {
+    if (tflite == null) {
+      throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed.");
+    }
+    if (outputIsFloat == null) {
+      throw new RuntimeException(TAG + ": Classifier output type has not been resolved.");
+    }
+    if (outputIsFloat) {
+      tflite.run(imgData, labelProbArray);
+    } else {
+      tflite.run(imgData, inferenceOutputArray);
+      /** Convert results to float */
+      for (int i = 0; i < inferenceOutputArray[0].length; i++) {
+        labelProbArray[0][i] = (inferenceOutputArray[0][i] & 0xff) / 255.0f;
+      }
+    }
+    OvicSingleImageResult iterResult = computeTopKLabels();
+    iterResult.latency = getLastNativeInferenceLatencyMilliseconds();
+    return iterResult;
+  }
+
+  /** Return the probability array of all classes. */
+  public float[][] getlabelProbArray() {
+    return labelProbArray;
+  }
+
+  /** Return the number of top labels predicted by the classifier. */
+  public int getNumPredictions() {
+    return RESULTS_TO_SHOW;
+  }
+
+  /** Return the four dimensions of the input image. */
+  public int[] getInputDims() {
+    return inputDims;
+  }
+
+  /*
+   * Get native inference latency of last image classification run.
+   *  @throws RuntimeException if model is uninitialized.
+   */
+  public Long getLastNativeInferenceLatencyMilliseconds() {
+    if (tflite == null) {
+      throw new RuntimeException(TAG + ": ImageNet classifier has not been initialized; Failed.");
+    }
+    Long latency = tflite.getLastNativeInferenceDurationNanoseconds();
+    return (latency == null) ? null : (Long) (latency / 1000000);
+  }
+
+  /** Closes tflite to release resources. */
+  public void close() {
+    tflite.close();
+    tflite = null;
+  }
+
+  /** Reads label list from Assets. */
+  private static List<String> loadLabelList(InputStream labelInputStream) throws IOException {
+    List<String> labelList = new ArrayList<String>();
+    try (BufferedReader reader =
+        new BufferedReader(new InputStreamReader(labelInputStream, StandardCharsets.UTF_8))) {
+      String line;
+      while ((line = reader.readLine()) != null) {
+        labelList.add(line);
+      }
+    }
+    return labelList;
+  }
+
+  /** Computes top-K labels. */
+  private OvicSingleImageResult computeTopKLabels() {
+    if (labelList == null) {
+      throw new RuntimeException("Label file has not been loaded.");
+    }
+    for (int i = 0; i < labelList.size(); ++i) {
+      sortedLabels.add(new AbstractMap.SimpleEntry<>(i, labelProbArray[0][i]));
+      if (sortedLabels.size() > RESULTS_TO_SHOW) {
+        sortedLabels.poll();
+      }
+    }
+    OvicSingleImageResult singleImageResult = new OvicSingleImageResult();
+    if (sortedLabels.size() != RESULTS_TO_SHOW) {
+      throw new RuntimeException(
+          "Number of returned labels does not match requirement: "
+              + sortedLabels.size()
+              + " returned, but "
+              + RESULTS_TO_SHOW
+              + " required.");
+    }
+    for (int i = 0; i < RESULTS_TO_SHOW; ++i) {
+      Map.Entry<Integer, Float> label = sortedLabels.poll();
+      // ImageNet model prediction indices are 0-based.
+      singleImageResult.topKIndices.add(label.getKey());
+      singleImageResult.topKClasses.add(labelList.get(label.getKey()));
+      singleImageResult.topKProbs.add(label.getValue());
+    }
+    // Labels with lowest probability are returned first, hence need to reverse them.
+    Collections.reverse(singleImageResult.topKIndices);
+    Collections.reverse(singleImageResult.topKClasses);
+    Collections.reverse(singleImageResult.topKProbs);
+    return singleImageResult;
+  }
+}
diff --git a/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java
new file mode 100644
index 0000000000..4af9a65c2f
--- /dev/null
+++ b/tensorflow/contrib/lite/java/ovic/src/main/java/org/tensorflow/ovic/OvicSingleImageResult.java
@@ -0,0 +1,54 @@
+/*Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.ovic;
+
+import java.util.ArrayList;
+
+/** Result class for inference run on a single image. */
+public class OvicSingleImageResult {
+
+  /** Top K classes and probabilities. */
+  public ArrayList<String> topKClasses;
+  public ArrayList<Float> topKProbs;
+  public ArrayList<Integer> topKIndices;
+
+  /** Latency (ms). */
+  public Long latency;
+
+  OvicSingleImageResult() {
+    topKClasses = new ArrayList<>();
+    topKProbs = new ArrayList<>();
+    topKIndices = new ArrayList<>();
+    latency = -1L;
+  }
+
+  @Override
+  public String toString() {
+    String textToShow = latency + "ms";
+    for (int k = 0; k < topKProbs.size(); ++k) {
+      textToShow +=
+          "\nPrediction ["
+              + k
+              + "] = Class "
+              + Integer.toString(topKIndices.get(k))
+              + " ("
+              + topKClasses.get(k)
+              + ") : "
+              + Float.toString(topKProbs.get(k));
+    }
+    return textToShow;
+  }
+
+}
diff --git a/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java b/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java
new file mode 100644
index 0000000000..4fd23a99d2
--- /dev/null
+++ b/tensorflow/contrib/lite/java/ovic/src/test/java/org/tensorflow/ovic/OvicClassifierTest.java
@@ -0,0 +1,176 @@
+/*Copyright 2018 Google LLC
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    https://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+package org.tensorflow.ovic;
+
+import static com.google.common.truth.Truth.assertThat;
+import static org.junit.Assert.fail;
+
+import java.awt.image.BufferedImage;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.file.Paths;
+import javax.imageio.ImageIO;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+
+/** Unit tests for {@link org.tensorflow.ovic.OvicClassifier}. */
+@RunWith(JUnit4.class)
+public final class OvicClassifierTest {
+
+  private OvicClassifier classifier;
+  private InputStream labelsInputStream = null;
+  private MappedByteBuffer quantizedModel = null;
+  private MappedByteBuffer floatModel = null;
+  private MappedByteBuffer lowResModel = null;
+  private ByteBuffer testImage = null;
+  private ByteBuffer lowResTestImage = null;
+  private OvicSingleImageResult testResult = null;
+  private static final String LABELS_PATH = "testdata/labels.txt";
+  private static final String QUANTIZED_MODEL_PATH = "testdata/quantized_model.lite";
+  private static final String LOW_RES_MODEL_PATH = "testdata/low_res_model.lite";
+  private static final String FLOAT_MODEL_PATH = "testdata/float_model.lite";
+  private static final String TEST_IMAGE_PATH = "testdata/test_image_224.jpg";
+  private static final String TEST_LOW_RES_IMAGE_PATH = "testdata/test_image_128.jpg";
+  private static final int TEST_IMAGE_GROUNDTRUTH = 653; // "military uniform"
+
+  @Before
+  public void setUp() {
+    try {
+      File labelsfile = new File(getTestDir(LABELS_PATH));
+      labelsInputStream = new FileInputStream(labelsfile);
+      quantizedModel = loadModelFile(getTestDir(QUANTIZED_MODEL_PATH));
+      floatModel = loadModelFile(getTestDir(FLOAT_MODEL_PATH));
+      lowResModel = loadModelFile(getTestDir(LOW_RES_MODEL_PATH));
+      File imageFile = new File(getTestDir(TEST_IMAGE_PATH));
+      BufferedImage img = ImageIO.read(imageFile);
+      testImage = toByteBuffer(img);
+      // Low res image and models.
+      imageFile = new File(getTestDir(TEST_LOW_RES_IMAGE_PATH));
+      img = ImageIO.read(imageFile);
+      lowResTestImage = toByteBuffer(img);
+    } catch (IOException e) {
+      System.out.print(e.getMessage());
+    }
+    System.out.println("Successful setup");
+  }
+
+  private static String getTestDir(String testfile) throws IOException {
+    return Paths.get("third_party/tensorflow/contrib/lite/java/ovic/src/", testfile).toString();
+  }
+
+  @Test
+  public void ovicClassifier_quantizedModelCreateSuccess() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, quantizedModel);
+    assertThat(classifier != null).isTrue();
+  }
+
+  @Test
+  public void ovicClassifier_floatModelCreateSuccess() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, floatModel);
+    assertThat(classifier != null).isTrue();
+  }
+
+  @Test
+  public void ovicClassifier_quantizedModelClassifySuccess() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, quantizedModel);
+    testResult = classifier.classifyByteBuffer(testImage);
+    assertCorrectTopK(testResult);
+  }
+
+  @Test
+  public void ovicClassifier_floatModelClassifySuccess() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, floatModel);
+    testResult = classifier.classifyByteBuffer(testImage);
+    assertCorrectTopK(testResult);
+  }
+
+  @Test
+  public void ovicClassifier_lowResModelClassifySuccess() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, lowResModel);
+    testResult = classifier.classifyByteBuffer(lowResTestImage);
+    assertCorrectTopK(testResult);
+  }
+
+  @Test
+  public void ovicClassifier_latencyNotNull() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, floatModel);
+    testResult = classifier.classifyByteBuffer(testImage);
+    assertThat(testResult.latency != null).isTrue();
+  }
+
+  @Test
+  public void ovicClassifier_mismatchedInputResolutionFails() throws Exception {
+    classifier = new OvicClassifier(labelsInputStream, lowResModel);
+    int[] inputDims = classifier.getInputDims();
+    assertThat((inputDims[1] == 128) && (inputDims[2] == 128)).isTrue();
+    try {
+      testResult = classifier.classifyByteBuffer(testImage);
+      fail();
+    } catch (RuntimeException e) {
+      assertThat(e)
+          .hasMessageThat()
+          .contains(
+              "Failed to get input dimensions. 0-th input should have 49152 bytes, "
+                  + "but found 150528 bytes.");
+    }
+  }
+
+  private static ByteBuffer toByteBuffer(BufferedImage image) {
+    ByteBuffer imgData = ByteBuffer.allocateDirect(
+        image.getHeight() * image.getWidth() * 3);
+    imgData.order(ByteOrder.nativeOrder());
+    for (int y = 0; y < image.getHeight(); y++) {
+      for (int x = 0; x < image.getWidth(); x++) {
+        int val = image.getRGB(x, y);
+        imgData.put((byte) ((val >> 16) & 0xFF));
+        imgData.put((byte) ((val >> 8) & 0xFF));
+        imgData.put((byte) (val & 0xFF));
+      }
+    }
+    return imgData;
+  }
+
+  private static void assertCorrectTopK(OvicSingleImageResult testResult) {
+    assertThat(testResult.topKClasses.size() > 0).isTrue();
+    Boolean topKAccurate = false;
+    // Assert that the correct class is in the top K.
+    for (int i = 0; i < testResult.topKIndices.size(); i++) {
+      if (testResult.topKIndices.get(i) == TEST_IMAGE_GROUNDTRUTH) {
+        topKAccurate = true;
+        break;
+      }
+    }
+    System.out.println(testResult.toString());
+    System.out.flush();
+    assertThat(topKAccurate).isTrue();
+  }
+
+  private static MappedByteBuffer loadModelFile(String modelFilePath) throws IOException {
+    File modelfile = new File(modelFilePath);
+    FileInputStream inputStream = new FileInputStream(modelfile);
+    FileChannel fileChannel = inputStream.getChannel();
+    long startOffset = 0L;
+    long declaredLength = fileChannel.size();
+    return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
+  }
+}
diff --git a/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt b/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt
new file mode 100644
index 0000000000..fe811239d8
--- /dev/null
+++ b/tensorflow/contrib/lite/java/ovic/src/testdata/labels.txt
@@ -0,0 +1,1001 @@
+background
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
-- 
GitLab


From 8e0848160a7d135f728dde2519a32876b8a7e3ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 14:52:44 -0700
Subject: [PATCH 449/960] Prepare the XLA and TF code to correctly behave once
 automatic device placement will be enabled by default on computation shapes >
 1.

PiperOrigin-RevId: 190278826
---
 tensorflow/compiler/xla/service/service.cc    | 35 +++++++++++++++----
 tensorflow/compiler/xla/service/service.h     |  6 ++++
 .../contrib/tpu/python/tpu/tpu_estimator.py   |  3 +-
 tensorflow/contrib/tpu/python/tpu/tpu_feed.py | 19 ++++++++--
 4 files changed, 51 insertions(+), 12 deletions(-)

diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 04487a4795..4f6a82333b 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -861,6 +861,33 @@ tensorflow::Status Service::GetDeviceHandles(const GetDeviceHandlesRequest* arg,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status Service::ExecuteOneToN(const ExecuteRequest* arg,
+                                          ExecuteResponse* result) {
+  ExecuteParallelRequest parallel_arg;
+  *parallel_arg.add_requests() = *arg;
+  ExecuteParallelResponse parallel_result;
+  TF_RETURN_IF_ERROR(ExecuteParallel(&parallel_arg, &parallel_result));
+  // The "result device" selection is a bit hacky, but better than assuming it
+  // is device 0. We have b/76035356 for restructuring the client API to clean
+  // up the current asymmetries and support more functionalities.
+  for (int64 i = 0; i < parallel_result.responses_size(); ++i) {
+    TF_ASSIGN_OR_RETURN(const ShapedBuffer* buffer,
+                        allocation_tracker_.ResolveForReplica(
+                            parallel_result.responses(i).output(), 0));
+    const Shape& shape = buffer->on_host_shape();
+    if (!ShapeUtil::IsEmptyTuple(shape)) {
+      *result = parallel_result.responses(i);
+      VLOG(3) << "Fetching result from device " << i << ": "
+              << ShapeUtil::HumanString(shape);
+      return Status::OK();
+    }
+  }
+  TF_RET_CHECK(parallel_result.responses_size() > 0);
+  *result = parallel_result.responses(0);
+  VLOG(1) << "Defaulting to device 0 result";
+  return Status::OK();
+}
+
 tensorflow::Status Service::Execute(const ExecuteRequest* arg,
                                     ExecuteResponse* result) {
   VLOG(1) << "running execute request: " << arg->ShortDebugString();
@@ -877,13 +904,7 @@ tensorflow::Status Service::Execute(const ExecuteRequest* arg,
 
   // If we received multiple device handles, we must partition the module.
   if (arg->execution_options().device_handles_size() > 1) {
-    ExecuteParallelRequest parallel_arg;
-    *parallel_arg.add_requests() = *arg;
-    ExecuteParallelResponse parallel_result;
-    TF_RETURN_IF_ERROR(ExecuteParallel(&parallel_arg, &parallel_result));
-    TF_RET_CHECK(parallel_result.responses_size() > 0);
-    *result = parallel_result.responses(0);
-    return Status::OK();
+    return ExecuteOneToN(arg, result);
   }
 
   TF_ASSIGN_OR_RETURN(
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index a76bdd89c7..3b79920b0a 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -346,6 +346,12 @@ class Service : public ServiceInterface {
       const std::function<StatusOr<ComputationDataHandle>(UserComputation*)>&
           adder);
 
+  // Executes a single computation which has more than one target device.
+  // The N devices are expected to all return an empty tuple, but one, which
+  // will be the result of this computation.
+  tensorflow::Status ExecuteOneToN(const ExecuteRequest* arg,
+                                   ExecuteResponse* result);
+
   // Convenience function which checks whether the given shape_with_layout
   // (presumably passed by the client to set the result layout) is valid for the
   // given computation result shape.
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index aaa6f3c2c1..152f8c8c69 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -931,8 +931,7 @@ class _InputPipeline(object):
       # In the model-parallel case, both the host-side and device-side
       # computations must agree on the core on which infeed takes place. We
       # choose to perform infeed on logical core 0 of each replica.
-      with ops.device(tpu.core(0)):
-        values = self._infeed_queue.generate_dequeue_op()
+      values = self._infeed_queue.generate_dequeue_op(tpu_device=0)
       # The unflatten process uses the structure information recorded above.
       return self._inputs_structure_recorder.unflatten_features_and_labels(
           values)
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
index 42ac6eb680..604e6600c8 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_feed.py
@@ -23,6 +23,7 @@ from __future__ import print_function
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.tpu.python.ops import tpu_ops
+from tensorflow.contrib.tpu.python.tpu import tpu
 from tensorflow.contrib.tpu.python.tpu import tpu_sharding
 
 from tensorflow.python.framework import dtypes
@@ -368,13 +369,20 @@ class InfeedQueue(object):
       policy.freeze()
     self._validate()
 
-  def generate_dequeue_op(self):
+  def generate_dequeue_op(self, tpu_device=0):
     """Generates the device-side Op to dequeue a tuple from the queue.
 
     Implicitly freezes the queue configuration if it is not already
     frozen, which will raise errors if the shapes and types have not
     been fully specified.
 
+    Args:
+      tpu_device: The TPU device ordinal where the infeed instruction should be
+        placed. If None, no explicit placement will be performed, and it is up
+        to the user to call this API from within a proper TPU device scope.
+        The XLA code will fail if the TPU dequeue instruction is not bound to
+        any device.
+
     Returns:
       A list of Outputs corresponding to a shard of infeed dequeued
       into XLA, suitable for use within a replicated block.
@@ -392,8 +400,13 @@ class InfeedQueue(object):
         policy.get_sharded_shape(shape)
         for (shape, policy) in zip(self._tuple_shapes, self._sharding_policies)
     ]
-    return tpu_ops.infeed_dequeue_tuple(
-        dtypes=self._tuple_types, shapes=sharded_shapes, name=full_name)
+    if tpu_device is not None:
+      with ops.device(tpu.core(tpu_device)):
+        return tpu_ops.infeed_dequeue_tuple(
+            dtypes=self._tuple_types, shapes=sharded_shapes, name=full_name)
+    else:
+      return tpu_ops.infeed_dequeue_tuple(
+          dtypes=self._tuple_types, shapes=sharded_shapes, name=full_name)
 
   def _generate_enqueue_op(self,
                            inputs,
-- 
GitLab


From fce07c395d7c3931bc809183031c232651eb0638 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 15:10:56 -0700
Subject: [PATCH 450/960] add EvaluateNodes to OpDedupping test.

PiperOrigin-RevId: 190282163
---
 tensorflow/core/grappler/optimizers/BUILD                  | 1 +
 .../core/grappler/optimizers/arithmetic_optimizer_test.cc  | 7 +++++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index fb13084945..92f7cce502 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -287,6 +287,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/inputs:trivial_test_graph_input_yielder",
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 3876486d80..792f675043 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/arithmetic_optimizer.h"
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/inputs/trivial_test_graph_input_yielder.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
@@ -157,6 +158,8 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
   // Run the optimizer twice to make sure the rewrite is idempotent.
@@ -172,6 +175,10 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   EXPECT_EQ(2, new_div.input_size());
   EXPECT_EQ("c1", new_div.input(0));
   EXPECT_EQ("c1", new_div.input(1));
+
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<double>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, OpDeduppingAssertAndCheckNumerics) {
-- 
GitLab


From db51253fce5882bf766e19b97131d90f0947d0df Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Fri, 23 Mar 2018 15:12:21 -0700
Subject: [PATCH 451/960] Convert the eager SPINN example to use tf.keras.Model
 and object-based checkpointing.

Uses a more recursive/functional tracking style which avoids numbering layers. Maybe this is too magical and we should adapt tf.keras.Sequential first? Let me know what you think.

PiperOrigin-RevId: 190282346
---
 .../eager/python/examples/spinn/spinn_test.py |  24 ++-
 third_party/examples/eager/spinn/spinn.py     | 168 ++++++++++--------
 2 files changed, 108 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 081b0af14f..591d99edcd 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -33,6 +33,7 @@ import tensorflow as tf
 import tensorflow.contrib.eager as tfe
 from tensorflow.contrib.eager.python.examples.spinn import data
 from third_party.examples.eager.spinn import spinn
+from tensorflow.contrib.eager.proto import checkpointable_object_graph_pb2
 from tensorflow.contrib.summary import summary_test_util
 from tensorflow.python.eager import test
 from tensorflow.python.framework import test_util
@@ -172,7 +173,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
         right_in.append(tf.random_normal((1, size * 2)))
         tracking.append(tf.random_normal((1, tracker_size * 2)))
 
-      out = reducer(left_in, right_in, tracking=tracking)
+      out = reducer(left_in, right_in=right_in, tracking=tracking)
       self.assertEqual(batch_size, len(out))
       self.assertEqual(tf.float32, out[0].dtype)
       self.assertEqual((1, size * 2), out[0].shape)
@@ -226,7 +227,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
       self.assertEqual((batch_size, size * 2), stacks[0][0].shape)
 
       for _ in range(2):
-        out1, out2 = tracker(bufs, stacks)
+        out1, out2 = tracker(bufs, stacks=stacks)
         self.assertIsNone(out2)
         self.assertEqual(batch_size, len(out1))
         self.assertEqual(tf.float32, out1[0].dtype)
@@ -259,7 +260,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
       self.assertEqual(tf.int64, transitions.dtype)
       self.assertEqual((num_transitions, 1), transitions.shape)
 
-      out = s(buffers, transitions, training=True)
+      out = s(buffers, transitions=transitions, training=True)
       self.assertEqual(tf.float32, out.dtype)
       self.assertEqual((1, embedding_dims), out.shape)
 
@@ -285,12 +286,15 @@ class SpinnTest(test_util.TensorFlowTestCase):
                                                          vocab_size)
 
       # Invoke model under non-training mode.
-      logits = model(prem, prem_trans, hypo, hypo_trans, training=False)
+      logits = model(
+          prem, premise_transition=prem_trans, hypothesis=hypo,
+          hypothesis_transition=hypo_trans, training=False)
       self.assertEqual(tf.float32, logits.dtype)
       self.assertEqual((batch_size, d_out), logits.shape)
 
       # Invoke model under training model.
-      logits = model(prem, prem_trans, hypo, hypo_trans, training=True)
+      logits = model(prem, premise_transition=prem_trans, hypothesis=hypo,
+                     hypothesis_transition=hypo_trans, training=True)
       self.assertEqual(tf.float32, logits.dtype)
       self.assertEqual((batch_size, d_out), logits.shape)
 
@@ -421,8 +425,14 @@ class SpinnTest(test_util.TensorFlowTestCase):
 
     # 5. Verify that checkpoints exist and contains all the expected variables.
     self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*")))
-    ckpt_variable_names = [
-        item[0] for item in checkpoint_utils.list_variables(config.logdir)]
+    object_graph_string = checkpoint_utils.load_variable(
+        config.logdir, name="_CHECKPOINTABLE_OBJECT_GRAPH")
+    object_graph = checkpointable_object_graph_pb2.CheckpointableObjectGraph()
+    object_graph.ParseFromString(object_graph_string)
+    ckpt_variable_names = set()
+    for node in object_graph.nodes:
+      for attribute in node.attributes:
+        ckpt_variable_names.add(attribute.full_name)
     self.assertIn("global_step", ckpt_variable_names)
     for v in trainer.variables:
       variable_name = v.name[:v.name.index(":")] if ":" in v.name else v.name
diff --git a/third_party/examples/eager/spinn/spinn.py b/third_party/examples/eager/spinn/spinn.py
index 8a1c7db2ea..f8fb6ecb0c 100644
--- a/third_party/examples/eager/spinn/spinn.py
+++ b/third_party/examples/eager/spinn/spinn.py
@@ -51,6 +51,9 @@ import tensorflow.contrib.eager as tfe
 from tensorflow.contrib.eager.python.examples.spinn import data
 
 
+layers = tf.keras.layers
+
+
 def _bundle(lstm_iter):
   """Concatenate a list of Tensors along 1st axis and split result into two.
 
@@ -78,17 +81,16 @@ def _unbundle(state):
   return tf.split(tf.concat(state, 1), state[0].shape[0], axis=0)
 
 
-class Reducer(tfe.Network):
+# pylint: disable=not-callable
+class Reducer(tf.keras.Model):
   """A module that applies reduce operation on left and right vectors."""
 
   def __init__(self, size, tracker_size=None):
     super(Reducer, self).__init__()
-    self.left = self.track_layer(tf.layers.Dense(5 * size, activation=None))
-    self.right = self.track_layer(
-        tf.layers.Dense(5 * size, activation=None, use_bias=False))
+    self.left = layers.Dense(5 * size, activation=None)
+    self.right = layers.Dense(5 * size, activation=None, use_bias=False)
     if tracker_size is not None:
-      self.track = self.track_layer(
-          tf.layers.Dense(5 * size, activation=None, use_bias=False))
+      self.track = layers.Dense(5 * size, activation=None, use_bias=False)
     else:
       self.track = None
 
@@ -123,7 +125,7 @@ class Reducer(tfe.Network):
     return h, c
 
 
-class Tracker(tfe.Network):
+class Tracker(tf.keras.Model):
   """A module that tracks the history of the sentence with an LSTM."""
 
   def __init__(self, tracker_size, predict):
@@ -134,10 +136,10 @@ class Tracker(tfe.Network):
       predict: (`bool`) Whether prediction mode is enabled.
     """
     super(Tracker, self).__init__()
-    self._rnn = self.track_layer(tf.nn.rnn_cell.LSTMCell(tracker_size))
+    self._rnn = tf.nn.rnn_cell.LSTMCell(tracker_size)
     self._state_size = tracker_size
     if predict:
-      self._transition = self.track_layer(tf.layers.Dense(4))
+      self._transition = layers.Dense(4)
     else:
       self._transition = None
 
@@ -182,7 +184,7 @@ class Tracker(tfe.Network):
       return unbundled, None
 
 
-class SPINN(tfe.Network):
+class SPINN(tf.keras.Model):
   """Stack-augmented Parser-Interpreter Neural Network.
 
   See https://arxiv.org/abs/1603.06021 for more details.
@@ -204,9 +206,9 @@ class SPINN(tfe.Network):
     """
     super(SPINN, self).__init__()
     self.config = config
-    self.reducer = self.track_layer(Reducer(config.d_hidden, config.d_tracker))
+    self.reducer = Reducer(config.d_hidden, config.d_tracker)
     if config.d_tracker is not None:
-      self.tracker = self.track_layer(Tracker(config.d_tracker, config.predict))
+      self.tracker = Tracker(config.d_tracker, config.predict)
     else:
       self.tracker = None
 
@@ -248,7 +250,7 @@ class SPINN(tfe.Network):
       trans = transitions[i]
       if self.tracker:
         # Invoke tracker to obtain the current tracker states for the sentences.
-        tracker_states, trans_hypothesis = self.tracker(buffers, stacks)
+        tracker_states, trans_hypothesis = self.tracker(buffers, stacks=stacks)
         if trans_hypothesis:
           trans = tf.argmax(trans_hypothesis, axis=-1)
       else:
@@ -264,7 +266,8 @@ class SPINN(tfe.Network):
           trackings.append(tracking)
 
       if rights:
-        reducer_output = self.reducer(lefts, rights, trackings)
+        reducer_output = self.reducer(
+            lefts, right_in=rights, tracking=trackings)
         reduced = iter(reducer_output)
 
         for transition, stack in zip(trans, stacks):
@@ -273,7 +276,27 @@ class SPINN(tfe.Network):
     return _bundle([stack.pop() for stack in stacks])[0]
 
 
-class SNLIClassifier(tfe.Network):
+class Perceptron(tf.keras.Model):
+  """One layer of the SNLIClassifier multi-layer perceptron."""
+
+  def __init__(self, dimension, dropout_rate, previous_layer):
+    """Configure the Perceptron."""
+    super(Perceptron, self).__init__()
+    self.dense = tf.keras.layers.Dense(dimension, activation=tf.nn.elu)
+    self.batchnorm = layers.BatchNormalization()
+    self.dropout = layers.Dropout(rate=dropout_rate)
+    self.previous_layer = previous_layer
+
+  def call(self, x, training):
+    """Run previous Perceptron layers, then this one."""
+    x = self.previous_layer(x, training=training)
+    x = self.dense(x)
+    x = self.batchnorm(x, training=training)
+    x = self.dropout(x, training=training)
+    return x
+
+
+class SNLIClassifier(tf.keras.Model):
   """SNLI Classifier Model.
 
   A model aimed at solving the SNLI (Standford Natural Language Inference)
@@ -304,29 +327,24 @@ class SNLIClassifier(tfe.Network):
     self.config = config
     self.embed = tf.constant(embed)
 
-    self.projection = self.track_layer(tf.layers.Dense(config.d_proj))
-    self.embed_bn = self.track_layer(tf.layers.BatchNormalization())
-    self.embed_dropout = self.track_layer(
-        tf.layers.Dropout(rate=config.embed_dropout))
-    self.encoder = self.track_layer(SPINN(config))
-
-    self.feature_bn = self.track_layer(tf.layers.BatchNormalization())
-    self.feature_dropout = self.track_layer(
-        tf.layers.Dropout(rate=config.mlp_dropout))
-
-    self.mlp_dense = []
-    self.mlp_bn = []
-    self.mlp_dropout = []
-    for _ in xrange(config.n_mlp_layers):
-      self.mlp_dense.append(self.track_layer(tf.layers.Dense(config.d_mlp)))
-      self.mlp_bn.append(
-          self.track_layer(tf.layers.BatchNormalization()))
-      self.mlp_dropout.append(
-          self.track_layer(tf.layers.Dropout(rate=config.mlp_dropout)))
-    self.mlp_output = self.track_layer(tf.layers.Dense(
+    self.projection = layers.Dense(config.d_proj)
+    self.embed_bn = layers.BatchNormalization()
+    self.embed_dropout = layers.Dropout(rate=config.embed_dropout)
+    self.encoder = SPINN(config)
+
+    self.feature_bn = layers.BatchNormalization()
+    self.feature_dropout = layers.Dropout(rate=config.mlp_dropout)
+
+    current_mlp = lambda result, training: result
+    for _ in range(config.n_mlp_layers):
+      current_mlp = Perceptron(dimension=config.d_mlp,
+                               dropout_rate=config.mlp_dropout,
+                               previous_layer=current_mlp)
+    self.mlp = current_mlp
+    self.mlp_output = layers.Dense(
         config.d_out,
         kernel_initializer=tf.random_uniform_initializer(minval=-5e-3,
-                                                         maxval=5e-3)))
+                                                         maxval=5e-3))
 
   def call(self,
            premise,
@@ -370,10 +388,10 @@ class SNLIClassifier(tfe.Network):
 
     # Run the batch-normalized and dropout-processed word vectors through the
     # SPINN encoder.
-    premise = self.encoder(premise_embed, premise_transition,
-                           training=training)
-    hypothesis = self.encoder(hypothesis_embed, hypothesis_transition,
-                              training=training)
+    premise = self.encoder(
+        premise_embed, transitions=premise_transition, training=training)
+    hypothesis = self.encoder(
+        hypothesis_embed, transitions=hypothesis_transition, training=training)
 
     # Combine encoder outputs for premises and hypotheses into logits.
     # Then apply batch normalization and dropuout on the logits.
@@ -383,15 +401,12 @@ class SNLIClassifier(tfe.Network):
         self.feature_bn(logits, training=training), training=training)
 
     # Apply the multi-layer perceptron on the logits.
-    for dense, bn, dropout in zip(
-        self.mlp_dense, self.mlp_bn, self.mlp_dropout):
-      logits = tf.nn.elu(dense(logits))
-      logits = dropout(bn(logits, training=training), training=training)
+    logits = self.mlp(logits, training=training)
     logits = self.mlp_output(logits)
     return logits
 
 
-class SNLIClassifierTrainer(object):
+class SNLIClassifierTrainer(tfe.Checkpointable):
   """A class that coordinates the training of an SNLIClassifier."""
 
   def __init__(self, snli_classifier, lr):
@@ -450,10 +465,11 @@ class SNLIClassifierTrainer(object):
     """
     with tfe.GradientTape() as tape:
       tape.watch(self._model.variables)
+      # TODO(allenl): Allow passing Layer inputs as position arguments.
       logits = self._model(premise,
-                           premise_transition,
-                           hypothesis,
-                           hypothesis_transition,
+                           premise_transition=premise_transition,
+                           hypothesis=hypothesis,
+                           hypothesis_transition=hypothesis_transition,
                            training=True)
       loss = self.loss(labels, logits)
     gradients = tape.gradient(loss, self._model.variables)
@@ -517,7 +533,9 @@ def _evaluate_on_dataset(snli_data, batch_size, trainer, use_gpu):
       snli_data, batch_size):
     if use_gpu:
       label, prem, hypo = label.gpu(), prem.gpu(), hypo.gpu()
-    logits = trainer.model(prem, prem_trans, hypo, hypo_trans, training=False)
+    logits = trainer.model(
+        prem, premise_transition=prem_trans, hypothesis=hypo,
+        hypothesis_transition=hypo_trans, training=False)
     loss_val = trainer.loss(label, logits)
     batch_size = tf.shape(label)[0]
     mean_loss(loss_val, weights=batch_size.gpu() if use_gpu else batch_size)
@@ -609,29 +627,30 @@ def train_or_infer_spinn(embed,
   with tf.device(device), \
        summary_writer.as_default(), \
        tf.contrib.summary.always_record_summaries():
-    with tfe.restore_variables_on_create(
-        tf.train.latest_checkpoint(config.logdir)):
-      model = SNLIClassifier(config, embed)
-      global_step = tf.train.get_or_create_global_step()
-      trainer = SNLIClassifierTrainer(model, config.lr)
+    model = SNLIClassifier(config, embed)
+    global_step = tf.train.get_or_create_global_step()
+    trainer = SNLIClassifierTrainer(model, config.lr)
+    checkpoint = tfe.Checkpoint(trainer=trainer, global_step=global_step)
+    checkpoint.restore(tf.train.latest_checkpoint(config.logdir))
 
     if inference_sentence_pair:
       # Inference mode.
-      with tfe.restore_variables_on_create(
-          tf.train.latest_checkpoint(config.logdir)):
-        prem, prem_trans = inference_sentence_pair[0]
-        hypo, hypo_trans = inference_sentence_pair[1]
-        hypo_trans = inference_sentence_pair[1][1]
-        inference_logits = model(  # pylint: disable=not-callable
-            tf.constant(prem), tf.constant(prem_trans),
-            tf.constant(hypo), tf.constant(hypo_trans), training=False)
-        inference_logits = inference_logits[0][1:]
-        max_index = tf.argmax(inference_logits)
-        print("\nInference logits:")
-        for i, (label, logit) in enumerate(
-            zip(data.POSSIBLE_LABELS, inference_logits)):
-          winner_tag = " (winner)" if max_index == i else ""
-          print("  {0:<16}{1:.6f}{2}".format(label + ":", logit, winner_tag))
+      prem, prem_trans = inference_sentence_pair[0]
+      hypo, hypo_trans = inference_sentence_pair[1]
+      hypo_trans = inference_sentence_pair[1][1]
+      inference_logits = model(
+          tf.constant(prem),
+          premise_transition=tf.constant(prem_trans),
+          hypothesis=tf.constant(hypo),
+          hypothesis_transition=tf.constant(hypo_trans),
+          training=False)
+      inference_logits = inference_logits[0][1:]
+      max_index = tf.argmax(inference_logits)
+      print("\nInference logits:")
+      for i, (label, logit) in enumerate(
+          zip(data.POSSIBLE_LABELS, inference_logits)):
+        winner_tag = " (winner)" if max_index == i else ""
+        print("  {0:<16}{1:.6f}{2}".format(label + ":", logit, winner_tag))
       return inference_logits
 
     train_len = train_data.num_batches(config.batch_size)
@@ -650,20 +669,15 @@ def train_or_infer_spinn(embed,
           # remain on CPU. Same in _evaluate_on_dataset().
 
         iterations += 1
-        with tfe.restore_variables_on_create(
-            tf.train.latest_checkpoint(config.logdir)):
-          batch_train_loss, batch_train_logits = trainer.train_batch(
-              label, prem, prem_trans, hypo, hypo_trans)
+        batch_train_loss, batch_train_logits = trainer.train_batch(
+            label, prem, prem_trans, hypo, hypo_trans)
         batch_size = tf.shape(label)[0]
         mean_loss(batch_train_loss.numpy(),
                   weights=batch_size.gpu() if use_gpu else batch_size)
         accuracy(tf.argmax(batch_train_logits, axis=1), label)
 
         if iterations % config.save_every == 0:
-          all_variables = trainer.variables + [global_step]
-          saver = tfe.Saver(all_variables)
-          saver.save(os.path.join(config.logdir, "ckpt"),
-                     global_step=global_step)
+          checkpoint.save(os.path.join(config.logdir, "ckpt"))
 
         if iterations % config.dev_every == 0:
           dev_loss, dev_frac_correct = _evaluate_on_dataset(
-- 
GitLab


From a41a2975c4b39ca6026deb46f0343317da165ea6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 15:12:35 -0700
Subject: [PATCH 452/960] [TF:XLA] Fix PotentiallyImplementedAsEigenConvolution
 to use the correct shape as the kernel shape

A small bug is found in accessing the kernel's shape of the convolution instruction in PotentiallyImplementedAsEigenConvolution. The bug was fixed and a new testcase is created to reveal the bug.

PiperOrigin-RevId: 190282385
---
 tensorflow/compiler/xla/service/cpu/BUILD     | 16 +++++++
 .../xla/service/cpu/ir_emission_utils.cc      |  8 ++--
 .../xla/service/cpu/ir_emission_utils_test.cc | 46 +++++++++++++++++++
 3 files changed, 67 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/ir_emission_utils_test.cc

diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 093db020c0..0faa9e9c41 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -670,6 +670,22 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "ir_emission_utils_test",
+    srcs = ["ir_emission_utils_test.cc"],
+    deps = [
+        ":ir_emission_utils",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla/service:hlo",
+        "//tensorflow/compiler/xla/service:hlo_matchers",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/compiler/xla/tools/parser:hlo_parser",
+    ],
+)
+
 cc_library(
     name = "cpu_layout_assignment",
     srcs = ["cpu_layout_assignment.cc"],
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
index 788217aab6..f209a69e3c 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils.cc
@@ -34,14 +34,16 @@ bool PotentiallyImplementedAsEigenConvolution(
   //
   // To be sufficient, certain layout constraints need to be satisfied as well.
   const Shape& input_shape = convolution.operand(0)->shape();
-  const Shape& kernel_shape = convolution.operand(0)->shape();
+  const Shape& kernel_shape = convolution.operand(1)->shape();
   if (ShapeUtil::HasZeroElements(input_shape) ||
       ShapeUtil::HasZeroElements(kernel_shape)) {
     return false;
   }
+  // Make sure input and kernel has the same data type.
+  CHECK(
+      ShapeUtil::SameElementTypeIgnoringFpPrecision(input_shape, kernel_shape));
   // TODO(b/65408531): Explore using Eigen dot for complex64 type.
-  if (ShapeUtil::ElementIsComplex(input_shape) ||
-      ShapeUtil::ElementIsComplex(kernel_shape)) {
+  if (ShapeUtil::ElementIsComplex(input_shape)) {
     return false;
   }
   if (window_util::HasWindowReversal(convolution.window())) {
diff --git a/tensorflow/compiler/xla/service/cpu/ir_emission_utils_test.cc b/tensorflow/compiler/xla/service/cpu/ir_emission_utils_test.cc
new file mode 100644
index 0000000000..215f48c4cc
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/ir_emission_utils_test.cc
@@ -0,0 +1,46 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/cpu/ir_emission_utils.h"
+
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/tools/parser/hlo_parser.h"
+
+namespace xla {
+namespace {
+
+TEST(IrEmitterTest, ConvWithZeroSizedKernelNotImplementedAsEigen) {
+  const char* const hlo_string = R"(
+HloModule ModuleWithConv
+
+ENTRY Conv {
+  input = f32[32,50,28,28]{3,2,1,0} parameter(0)
+  kernel = f32[0,32,5,5]{3,2,1,0} parameter(1)
+  ROOT convolution = f32[64,50,24,24]{3,2,1,0} convolution(input, kernel),
+    window={size=5x5},
+    dim_labels=b01f_01io->b01f
+}
+)";
+  TF_ASSERT_OK_AND_ASSIGN(std::unique_ptr<HloModule> module,
+                          tools::Parse(hlo_string));
+
+  HloComputation* entry_computation = module->entry_computation();
+
+  HloInstruction* conv_instr = entry_computation->root_instruction();
+  EXPECT_FALSE(cpu::PotentiallyImplementedAsEigenConvolution(*conv_instr));
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From bc1dfdf8bc9e3edb4362314a89a23bb2c827bdaa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 15:33:33 -0700
Subject: [PATCH 453/960] Adding support for analyzing assignment info for
 nested tuples.

PiperOrigin-RevId: 190285584
---
 .../py2tf/pyct/static_analysis/type_info.py     | 16 ++++++++++++----
 .../pyct/static_analysis/type_info_test.py      | 17 +++++++++++++++++
 2 files changed, 29 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py b/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py
index 5556a58c02..a969adbeca 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py
+++ b/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py
@@ -168,6 +168,15 @@ class TypeInfoResolver(transformer.Base):
                      anno.getanno(definition, 'element_type'))
     return node
 
+  def _process_tuple_assignment(self, source, t):
+    for i, e in enumerate(t.elts):
+      if isinstance(e, gast.Tuple):
+        self._process_tuple_assignment(source, e)
+      else:
+        self.scope.setval(
+            anno.getanno(e, anno.Basic.QN),
+            gast.Subscript(source, gast.Index(i), ctx=gast.Store()))
+
   def _process_variable_assignment(self, source, targets):
     if isinstance(source, gast.Call):
       func = source.func
@@ -183,10 +192,9 @@ class TypeInfoResolver(transformer.Base):
 
     for t in targets:
       if isinstance(t, gast.Tuple):
-        for i, e in enumerate(t.elts):
-          self.scope.setval(
-              anno.getanno(e, anno.Basic.QN),
-              gast.Subscript(source, gast.Index(i), ctx=gast.Store()))
+        # need to recurse on the case of assigning nested tuples,
+        # ex. a, (b, c) = f()
+        self._process_tuple_assignment(source, t)
       elif isinstance(t, (gast.Name, gast.Attribute)):
         self.scope.setval(anno.getanno(t, anno.Basic.QN), source)
       else:
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py b/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py
index 0d9d5a85f0..8a8956197d 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py
+++ b/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py
@@ -196,6 +196,23 @@ class TypeInfoResolverTest(test.TestCase):
     f_ref = node.body[0].body[1].value
     self.assertEqual(anno.getanno(f_ref, 'element_type'), Foo)
 
+  def test_nested_assignment(self):
+
+    def test_fn(foo):
+      a, (b, c) = foo
+      return a, b, c
+
+    node = self._parse_and_analyze(test_fn, {'foo': (1, 2, 3)})
+    lhs = node.body[0].body[1].value.elts
+    a = lhs[0]
+    b = lhs[1]
+    c = lhs[2]
+    # TODO(mdan): change these once we have the live values propagating
+    # correctly
+    self.assertFalse(anno.hasanno(a, 'live_val'))
+    self.assertFalse(anno.hasanno(b, 'live_val'))
+    self.assertFalse(anno.hasanno(c, 'live_val'))
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 7c57af0c860746e8a91b13bade87bdd1af9dc9e1 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Fri, 23 Mar 2018 15:34:47 -0700
Subject: [PATCH 454/960] [XLA] Don't CSE instructions which have side-effects

PiperOrigin-RevId: 190285774
---
 tensorflow/compiler/xla/service/hlo_cse.cc      |  7 ++++++-
 tensorflow/compiler/xla/service/hlo_cse_test.cc | 13 +++++++------
 2 files changed, 13 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_cse.cc b/tensorflow/compiler/xla/service/hlo_cse.cc
index 279edd4ba8..cd7cbbdd71 100644
--- a/tensorflow/compiler/xla/service/hlo_cse.cc
+++ b/tensorflow/compiler/xla/service/hlo_cse.cc
@@ -109,6 +109,11 @@ StatusOr<bool> HloCSE::Run(HloModule* module) {
         continue;
       }
 
+      // Skip instructions which have side effects.
+      if (instruction->HasSideEffect()) {
+        continue;
+      }
+
       // An instruction is considered to be equivalent to another only if they
       // share the exact same set of operands. So to find equivalent
       // instructions, we just search among instructions which share operand(0)
@@ -118,7 +123,7 @@ StatusOr<bool> HloCSE::Run(HloModule* module) {
       tensorflow::gtl::InlinedVector<HloInstruction*, 8>
           equivalent_instructions;
       for (HloInstruction* user : operand->users()) {
-        if (user != instruction &&
+        if (user != instruction && !user->HasSideEffect() &&
             user->Identical(*instruction, eq_instructions, eq_computations,
                             is_layout_sensitive_)) {
           equivalent_instructions.push_back(user);
diff --git a/tensorflow/compiler/xla/service/hlo_cse_test.cc b/tensorflow/compiler/xla/service/hlo_cse_test.cc
index 3601a790c4..df8853f34f 100644
--- a/tensorflow/compiler/xla/service/hlo_cse_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cse_test.cc
@@ -414,8 +414,7 @@ TEST_F(HloCseTest, DoNotCombineRng) {
   EXPECT_THAT(root, op::Add(rng1, rng2));
 }
 
-// TODO(b/28245743): Handle impure functions correctly in CSE.
-TEST_F(HloCseTest, DISABLED_DoNotCombineCallsToImpureFunctions) {
+TEST_F(HloCseTest, DoNotCombineCallsToImpureFunctions) {
   // Test that two calls to an impure function are not commoned. RNG
   // is the source of the impurity.
 
@@ -458,14 +457,16 @@ TEST_F(HloCseTest, DISABLED_DoNotCombineCallsToImpureFunctions) {
   HloInstruction* root = computation->root_instruction();
   EXPECT_THAT(root, op::Add(op::Map(), op::Map()));
 
+  VLOG(3) << "before: " << module->ToString();
+
   HloCSE cse(/*is_layout_sensitive=*/false);
-  EXPECT_TRUE(cse.Run(module.get()).ValueOrDie());
+  EXPECT_FALSE(cse.Run(module.get()).ValueOrDie());
+
+  VLOG(3) << "after: " << module->ToString();
 
   EXPECT_EQ(4, computation->instruction_count());
   root = computation->root_instruction();
-  auto operand = root->operand(0)->operand(0);
-  EXPECT_THAT(operand, op::Map());
-  EXPECT_THAT(root, op::Add(operand, operand));
+  EXPECT_THAT(root, op::Add(op::Map(op::Constant()), op::Map(op::Constant())));
 }
 
 }  // namespace
-- 
GitLab


From 95a87277174f9fc49b4b5d9c1edbbd149bd0274c Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Fri, 23 Mar 2018 15:52:35 -0700
Subject: [PATCH 455/960] [XLA:CPU] Update calls to IRBuilder::CreateMemCpy to
 the 2-alignment form.

The single alignment version is going away.

PiperOrigin-RevId: 190288581
---
 .../compiler/xla/service/cpu/ir_emitter.cc       | 16 ++++++++++------
 1 file changed, 10 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
index 3b8056d505..3405277d44 100644
--- a/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/ir_emitter.cc
@@ -438,12 +438,14 @@ Status IrEmitter::EmitXfeedTransfer(XfeedKind kind, const Shape& shape,
 
   if (kind == XfeedKind::kInfeed) {
     // Copy to the program buffer address from the acquired buffer.
-    ir_builder_.CreateMemCpy(program_buffer_address, acquired_pointer,
-                             length_32, 1);
+    ir_builder_.CreateMemCpy(program_buffer_address, /*DstAlign=*/1,
+                             acquired_pointer,
+                             /*SrcAlign=*/1, length_32);
   } else {
     // Outfeed -- copy from the in-program address to the acquired buffer.
-    ir_builder_.CreateMemCpy(acquired_pointer, program_buffer_address,
-                             length_32, 1);
+    ir_builder_.CreateMemCpy(acquired_pointer, /*DstAlign=*/1,
+                             program_buffer_address,
+                             /*SrcAlign=*/1, length_32);
   }
 
   ir_builder_.CreateCall(release_func,
@@ -2441,7 +2443,8 @@ void IrEmitter::EmitTransferElements(llvm::Value* target, llvm::Value* source,
     target_array.AnnotateLoadStoreInstructionWithMetadata(store_instruction);
   } else {
     auto* memcpy_instruction = ir_builder_.CreateMemCpy(
-        target, source, element_count * primitive_type_size, element_alignment);
+        target, /*DstAlign=*/element_alignment, source,
+        /*SrcAlign=*/element_alignment, element_count * primitive_type_size);
 
     // The memcpy does the load and the store internally.  The aliasing related
     // metadata has to reflect that.
@@ -2905,7 +2908,8 @@ Status IrEmitter::EmitMemcpy(const HloInstruction& source,
   llvm::Value* destination_value = GetEmittedValueFor(&destination);
   int64 source_size = ByteSizeOf(source.shape());
   // TODO(b/63762267): Be more aggressive about specifying alignment.
-  ir_builder_.CreateMemCpy(destination_value, source_value, source_size, 1);
+  ir_builder_.CreateMemCpy(destination_value, /*DstAlign=*/1, source_value,
+                           /*SrcAlign=*/1, source_size);
   return Status::OK();
 }
 
-- 
GitLab


From 084c10784887d7c4d467416430626cf7eb333cb8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 16:00:14 -0700
Subject: [PATCH 456/960] Extended scatter operations to work with a scalar
 update parameter and added scatter-min and scatter-max operations.

PiperOrigin-RevId: 190289664
---
 .../base_api/api_def_ResourceScatterAdd.pbtxt |   2 +-
 .../base_api/api_def_ResourceScatterDiv.pbtxt |  43 ++++
 .../base_api/api_def_ResourceScatterMax.pbtxt |  43 ++++
 .../base_api/api_def_ResourceScatterMin.pbtxt |  43 ++++
 .../base_api/api_def_ResourceScatterMul.pbtxt |  43 ++++
 .../base_api/api_def_ResourceScatterSub.pbtxt |  43 ++++
 .../api_def/base_api/api_def_ScatterAdd.pbtxt |   2 +-
 .../api_def/base_api/api_def_ScatterDiv.pbtxt |   2 +-
 .../api_def/base_api/api_def_ScatterMax.pbtxt |  60 +++++
 .../api_def/base_api/api_def_ScatterMin.pbtxt |  60 +++++
 .../api_def/base_api/api_def_ScatterMul.pbtxt |   2 +-
 .../api_def/base_api/api_def_ScatterSub.pbtxt |   2 +-
 .../base_api/api_def_ScatterUpdate.pbtxt      |   2 +-
 .../api_def_ResourceScatterDiv.pbtxt          |   4 +
 .../api_def_ResourceScatterMax.pbtxt          |   4 +
 .../api_def_ResourceScatterMin.pbtxt          |   4 +
 .../api_def_ResourceScatterMul.pbtxt          |   4 +
 .../api_def_ResourceScatterSub.pbtxt          |   4 +
 .../core/kernels/resource_variable_ops.cc     |  81 ++++---
 tensorflow/core/kernels/scatter_functor.cc    |  27 ++-
 tensorflow/core/kernels/scatter_functor.h     | 170 +++++++++++++-
 .../core/kernels/scatter_functor_gpu.cu.cc    |   9 +-
 .../core/kernels/scatter_functor_gpu.cu.h     | 108 +++++++--
 tensorflow/core/kernels/scatter_op.cc         | 126 ++++++----
 tensorflow/core/kernels/scatter_op_gpu.cu.cc  |   9 +-
 tensorflow/core/kernels/scatter_op_test.cc    |  26 ++-
 tensorflow/core/ops/resource_variable_ops.cc  |  92 +++++---
 tensorflow/core/ops/state_ops.cc              |  25 +-
 .../docs_src/api_guides/python/state_ops.md   |   2 +
 .../resource_variable_ops_test.py             | 215 ++++++++++++++++++
 .../python/kernel_tests/scatter_ops_test.py   | 145 +++++++++++-
 tensorflow/python/ops/standard_ops.py         |   2 +
 tensorflow/python/ops/state_ops.py            |   2 +
 tensorflow/tools/api/golden/tensorflow.pbtxt  |   8 +
 34 files changed, 1261 insertions(+), 153 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterDiv.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterMax.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterMin.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterMul.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ResourceScatterSub.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ScatterMax.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_ScatterMin.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceScatterDiv.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceScatterMax.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceScatterMin.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceScatterMul.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ResourceScatterSub.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterAdd.pbtxt
index 9e0de08267..4eb6eb4e4d 100644
--- a/tensorflow/core/api_def/base_api/api_def_ResourceScatterAdd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterAdd.pbtxt
@@ -34,7 +34,7 @@ This operation computes
 Duplicate entries are handled correctly: if multiple `indices` reference
 the same location, their contributions add.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterDiv.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterDiv.pbtxt
new file mode 100644
index 0000000000..47148f7b03
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterDiv.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "ResourceScatterDiv"
+  in_arg {
+    name: "resource"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to add to `ref`.
+END
+  }
+  summary: "Divides sparse updates into the variable referenced by `resource`."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] /= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] /= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] /= updates[i, ..., j, ...]
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions multiply.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMax.pbtxt
new file mode 100644
index 0000000000..71f06d9a43
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMax.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "ResourceScatterMax"
+  in_arg {
+    name: "resource"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to add to `ref`.
+END
+  }
+  summary: "Reduces sparse updates into the variable referenced by `resource` using the `max` operation."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] = max(ref[indices, ...], updates[...])
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...])
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions are combined.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterMin.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMin.pbtxt
new file mode 100644
index 0000000000..08e40ee2a8
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMin.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "ResourceScatterMin"
+  in_arg {
+    name: "resource"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to add to `ref`.
+END
+  }
+  summary: "Reduces sparse updates into the variable referenced by `resource` using the `min` operation."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] = min(ref[indices, ...], updates[...])
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] = min(ref[indices[i], ...], updates[i, ...])
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] = min(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions are combined.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterMul.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMul.pbtxt
new file mode 100644
index 0000000000..5c63549d81
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterMul.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "ResourceScatterMul"
+  in_arg {
+    name: "resource"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to add to `ref`.
+END
+  }
+  summary: "Multiplies sparse updates into the variable referenced by `resource`."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] *= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] *= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] *= updates[i, ..., j, ...]
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions multiply.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ResourceScatterSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_ResourceScatterSub.pbtxt
new file mode 100644
index 0000000000..e71e60cbee
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ResourceScatterSub.pbtxt
@@ -0,0 +1,43 @@
+op {
+  graph_op_name: "ResourceScatterSub"
+  in_arg {
+    name: "resource"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to add to `ref`.
+END
+  }
+  summary: "Subtracts sparse updates from the variable referenced by `resource`."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] -= updates[...]
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] -= updates[i, ...]
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] -= updates[i, ..., j, ...]
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions add.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterAdd.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterAdd.pbtxt
index 4b5201f025..9da9d09ea6 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterAdd.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterAdd.pbtxt
@@ -51,7 +51,7 @@ This makes it easier to chain operations that need to use the reset value.
 Duplicate entries are handled correctly: if multiple `indices` reference
 the same location, their contributions add.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterDiv.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterDiv.pbtxt
index 771cf0b591..8e99718c7e 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterDiv.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterDiv.pbtxt
@@ -53,6 +53,6 @@ This makes it easier to chain operations that need to use the reset value.
 Duplicate entries are handled correctly: if multiple `indices` reference
 the same location, their contributions divide.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterMax.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterMax.pbtxt
new file mode 100644
index 0000000000..7b52dad4a1
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterMax.pbtxt
@@ -0,0 +1,60 @@
+op {
+  graph_op_name: "ScatterMax"
+  in_arg {
+    name: "ref"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to reduce into `ref`.
+END
+  }
+  out_arg {
+    name: "output_ref"
+    description: <<END
+= Same as `ref`.  Returned as a convenience for operations that want
+to use the updated values after the update is done.
+END
+  }
+  attr {
+    name: "use_locking"
+    description: <<END
+If True, the update will be protected by a lock;
+otherwise the behavior is undefined, but may exhibit less contention.
+END
+  }
+  summary: "Reduces sparse updates into a variable reference using the `max` operation."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] = max(ref[indices, ...], updates[...])
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] = max(ref[indices[i], ...], updates[i, ...])
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] = max(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions combine.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterMin.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterMin.pbtxt
new file mode 100644
index 0000000000..721ac0ff35
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterMin.pbtxt
@@ -0,0 +1,60 @@
+op {
+  graph_op_name: "ScatterMin"
+  in_arg {
+    name: "ref"
+    description: <<END
+Should be from a `Variable` node.
+END
+  }
+  in_arg {
+    name: "indices"
+    description: <<END
+A tensor of indices into the first dimension of `ref`.
+END
+  }
+  in_arg {
+    name: "updates"
+    description: <<END
+A tensor of updated values to reduce into `ref`.
+END
+  }
+  out_arg {
+    name: "output_ref"
+    description: <<END
+= Same as `ref`.  Returned as a convenience for operations that want
+to use the updated values after the update is done.
+END
+  }
+  attr {
+    name: "use_locking"
+    description: <<END
+If True, the update will be protected by a lock;
+otherwise the behavior is undefined, but may exhibit less contention.
+END
+  }
+  summary: "Reduces sparse updates into a variable reference using the `min` operation."
+  description: <<END
+This operation computes
+
+    # Scalar indices
+    ref[indices, ...] = min(ref[indices, ...], updates[...])
+
+    # Vector indices (for each i)
+    ref[indices[i], ...] = min(ref[indices[i], ...], updates[i, ...])
+
+    # High rank indices (for each i, ..., j)
+    ref[indices[i, ..., j], ...] = min(ref[indices[i, ..., j], ...], updates[i, ..., j, ...])
+
+This operation outputs `ref` after the update is done.
+This makes it easier to chain operations that need to use the reset value.
+
+Duplicate entries are handled correctly: if multiple `indices` reference
+the same location, their contributions combine.
+
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
+
+<div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+<img style="width:100%" src="https://www.tensorflow.org/images/ScatterAdd.png" alt>
+</div>
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterMul.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterMul.pbtxt
index a51f571b00..b9e293ba9e 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterMul.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterMul.pbtxt
@@ -53,6 +53,6 @@ This makes it easier to chain operations that need to use the reset value.
 Duplicate entries are handled correctly: if multiple `indices` reference
 the same location, their contributions multiply.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 END
 }
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterSub.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterSub.pbtxt
index c0d3a4a133..d12b3e68c2 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterSub.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterSub.pbtxt
@@ -51,7 +51,7 @@ This makes it easier to chain operations that need to use the reset value.
 Duplicate entries are handled correctly: if multiple `indices` reference
 the same location, their (negated) contributions add.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/ScatterSub.png" alt>
diff --git a/tensorflow/core/api_def/base_api/api_def_ScatterUpdate.pbtxt b/tensorflow/core/api_def/base_api/api_def_ScatterUpdate.pbtxt
index c44dbbd233..4804908afc 100644
--- a/tensorflow/core/api_def/base_api/api_def_ScatterUpdate.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_ScatterUpdate.pbtxt
@@ -54,7 +54,7 @@ If values in `ref` is to be updated more than once, because there are
 duplicate entries in `indices`, the order at which the updates happen
 for each value is undefined.
 
-Requires `updates.shape = indices.shape + ref.shape[1:]`.
+Requires `updates.shape = indices.shape + ref.shape[1:]` or `updates.shape = []`.
 
 <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
 <img style="width:100%" src="https://www.tensorflow.org/images/ScatterUpdate.png" alt>
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceScatterDiv.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceScatterDiv.pbtxt
new file mode 100644
index 0000000000..56b5a46d10
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceScatterDiv.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceScatterDiv"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceScatterMax.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMax.pbtxt
new file mode 100644
index 0000000000..8119bcc6c6
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMax.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceScatterMax"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceScatterMin.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMin.pbtxt
new file mode 100644
index 0000000000..d874aef3fe
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMin.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceScatterMin"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceScatterMul.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMul.pbtxt
new file mode 100644
index 0000000000..365a37fa0d
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceScatterMul.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceScatterMul"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_ResourceScatterSub.pbtxt b/tensorflow/core/api_def/python_api/api_def_ResourceScatterSub.pbtxt
new file mode 100644
index 0000000000..72dc5bf889
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ResourceScatterSub.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ResourceScatterSub"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index aecad0185f..e134e476f6 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -619,22 +619,35 @@ class ResourceScatterUpdateOp : public OpKernel {
     if (N > 0) {
       auto indices_flat = indices.flat<Index>();
       auto params_flat = params->flat_outer_dims<T>();
-      int64 num_updates = updates.NumElements();
-      OP_REQUIRES(c, num_updates % N == 0,
-                  errors::InvalidArgument(
-                      "shape of indices (", indices.shape().DebugString(),
-                      ") is not compatible with the shape of updates (",
-                      updates.shape().DebugString(), ")"));
-      auto updates_flat = updates.shaped<T, 2>({N, num_updates / N});
-
-      functor::ScatterFunctor<Device, T, Index, op> functor;
-      const Index bad_i = functor(c, c->template eigen_device<Device>(),
-                                  params_flat, updates_flat, indices_flat);
-      OP_REQUIRES(c, bad_i < 0,
-                  errors::InvalidArgument(
-                      "indices", SliceDebugString(indices.shape(), bad_i),
-                      " = ", indices_flat(bad_i), " is not in [0, ",
-                      params->dim_size(0), ")"));
+      if (TensorShapeUtils::IsScalar(updates.shape())) {
+        const auto update = updates.scalar<T>();
+
+        functor::ScatterScalarFunctor<Device, T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<Device>(),
+                                    params_flat, update, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params->dim_size(0), ")"));
+      } else {
+        int64 num_updates = updates.NumElements();
+        OP_REQUIRES(c, num_updates % N == 0,
+                    errors::InvalidArgument(
+                        "shape of indices (", indices.shape().DebugString(),
+                        ") is not compatible with the shape of updates (",
+                        updates.shape().DebugString(), ")"));
+        auto updates_flat = updates.shaped<T, 2>({N, num_updates / N});
+
+        functor::ScatterFunctor<Device, T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<Device>(),
+                                    params_flat, updates_flat, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params->dim_size(0), ")"));
+      }
     }
   }
 };
@@ -652,35 +665,51 @@ class ResourceScatterUpdateOp : public OpKernel {
   REGISTER_SCATTER_KERNEL_INDEX(type, int32, dev, name, op); \
   REGISTER_SCATTER_KERNEL_INDEX(type, int64, dev, name, op);
 
-// TODO(apassos) add the other types here.
-#define REGISTER_SCATTER_ARITHEMTIC(type, dev)                \
+#define REGISTER_SCATTER_ARITHMETIC(type, dev)                \
   REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterAdd",    \
                           scatter_op::UpdateOp::ADD);         \
+  REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterSub",    \
+                          scatter_op::UpdateOp::SUB);         \
+  REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterMul",    \
+                          scatter_op::UpdateOp::MUL);         \
+  REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterDiv",    \
+                          scatter_op::UpdateOp::DIV);         \
   REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterUpdate", \
                           scatter_op::UpdateOp::ASSIGN);
+#define REGISTER_SCATTER_MINMAX(type, dev)                 \
+  REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterMin", \
+                          scatter_op::UpdateOp::MIN);      \
+  REGISTER_SCATTER_KERNEL(type, dev, "ResourceScatterMax", \
+                          scatter_op::UpdateOp::MAX);
 
 // Registers CPU kernels.
-#define REGISTER_SCATTER_ARITHEMTIC_CPU(type) \
-  REGISTER_SCATTER_ARITHEMTIC(type, CPU);
+#define REGISTER_SCATTER_ARITHMETIC_CPU(type) \
+  REGISTER_SCATTER_ARITHMETIC(type, CPU);
+#define REGISTER_SCATTER_MINMAX_CPU(type) REGISTER_SCATTER_MINMAX(type, CPU);
 
-TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ARITHEMTIC_CPU);
+TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ARITHMETIC_CPU);
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_SCATTER_MINMAX_CPU);
 
 REGISTER_SCATTER_KERNEL(string, CPU, "ResourceScatterUpdate",
                         scatter_op::UpdateOp::ASSIGN);
 
 // Registers GPU kernels.
 #if GOOGLE_CUDA
-#define REGISTER_SCATTER_ARITHEMTIC_GPU(type) \
-  REGISTER_SCATTER_ARITHEMTIC(type, GPU);
+#define REGISTER_SCATTER_ARITHMETIC_GPU(type) \
+  REGISTER_SCATTER_ARITHMETIC(type, GPU);
+#define REGISTER_SCATTER_MINMAX_GPU(type) REGISTER_SCATTER_MINMAX(type, GPU);
 
 #define REGISTER_SCATTER_UPDATE_GPU(type) REGISTER_SCATTER_UPDATE(type, GPU);
 
-TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHEMTIC_GPU);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHMETIC_GPU);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_MINMAX_GPU);
 
 #endif  // GOOGLE_CUDA
 
-#undef REGISTER_SCATTER_ARITHEMTIC
-#undef REGISTER_SCATTER_ARITHEMTIC_CPU
+#undef REGISTER_SCATTER_ARITHMETIC
+#undef REGISTER_SCATTER_ARITHMETIC_CPU
+#undef REGISTER_SCATTER_MINMAX
+#undef REGISTER_SCATTER_MINMAX_CPU
 #undef REGISTER_SCATTER_KERNEL
 #undef REGISTER_SCATTER_KERNEL_INDEX
 
diff --git a/tensorflow/core/kernels/scatter_functor.cc b/tensorflow/core/kernels/scatter_functor.cc
index 7eba82899f..cf5408123f 100644
--- a/tensorflow/core/kernels/scatter_functor.cc
+++ b/tensorflow/core/kernels/scatter_functor.cc
@@ -26,21 +26,30 @@ typedef Eigen::GpuDevice GPUDevice;
 namespace functor {
 
 // Forward declarations of the functor specializations for GPU.
-#define DECLARE_GPU_SPECS_OP(T, Index, op)                   \
-  template <>                                                \
-  Index ScatterFunctor<GPUDevice, T, Index, op>::operator()( \
-      OpKernelContext* c, const GPUDevice& d,                \
-      typename TTypes<T>::Matrix params,                     \
-      typename TTypes<T>::ConstMatrix updates,               \
-      typename TTypes<Index>::ConstFlat indices);            \
-  extern template struct ScatterFunctor<GPUDevice, T, Index, op>;
+#define DECLARE_GPU_SPECS_OP(T, Index, op)                         \
+  template <>                                                      \
+  Index ScatterFunctor<GPUDevice, T, Index, op>::operator()(       \
+      OpKernelContext* c, const GPUDevice& d,                      \
+      typename TTypes<T>::Matrix params,                           \
+      typename TTypes<T>::ConstMatrix updates,                     \
+      typename TTypes<Index>::ConstFlat indices);                  \
+  extern template struct ScatterFunctor<GPUDevice, T, Index, op>;  \
+  template <>                                                      \
+  Index ScatterScalarFunctor<GPUDevice, T, Index, op>::operator()( \
+      OpKernelContext* c, const GPUDevice& d,                      \
+      typename TTypes<T>::Matrix params,                           \
+      const typename TTypes<T>::ConstScalar update,                \
+      typename TTypes<Index>::ConstFlat indices);                  \
+  extern template struct ScatterScalarFunctor<GPUDevice, T, Index, op>;
 
 #define DECLARE_GPU_SPECS_INDEX(T, Index)                       \
   DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ASSIGN); \
   DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ADD);    \
   DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::SUB);    \
   DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MUL);    \
-  DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);
+  DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);    \
+  DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MIN);    \
+  DECLARE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MAX);
 
 #define DECLARE_GPU_SPECS(T)         \
   DECLARE_GPU_SPECS_INDEX(T, int32); \
diff --git a/tensorflow/core/kernels/scatter_functor.h b/tensorflow/core/kernels/scatter_functor.h
index 079f15e101..52666645bf 100644
--- a/tensorflow/core/kernels/scatter_functor.h
+++ b/tensorflow/core/kernels/scatter_functor.h
@@ -18,6 +18,8 @@ limitations under the License.
 
 #include <type_traits>
 
+#include "third_party/eigen3/Eigen/Core"
+#include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/bounds_check.h"
 #include "tensorflow/core/platform/types.h"
@@ -33,7 +35,7 @@ typedef Eigen::SyclDevice SYCLDevice;
 
 namespace scatter_op {
 
-enum class UpdateOp { ASSIGN, ADD, SUB, MUL, DIV };
+enum class UpdateOp { ASSIGN, ADD, SUB, MUL, DIV, MIN, MAX };
 
 namespace internal {
 
@@ -45,6 +47,10 @@ struct Assign<scatter_op::UpdateOp::ASSIGN> {
   static void Run(Params p, Update u) {
     p = u;
   }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p.setConstant(u);
+  }
 };
 template <>
 struct Assign<scatter_op::UpdateOp::ADD> {
@@ -52,6 +58,10 @@ struct Assign<scatter_op::UpdateOp::ADD> {
   static void Run(Params p, Update u) {
     p += u;
   }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p + u;
+  }
 };
 template <>
 struct Assign<scatter_op::UpdateOp::SUB> {
@@ -59,6 +69,10 @@ struct Assign<scatter_op::UpdateOp::SUB> {
   static void Run(Params p, Update u) {
     p -= u;
   }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p + static_cast<Update>(-u);
+  }
 };
 template <>
 struct Assign<scatter_op::UpdateOp::MUL> {
@@ -66,6 +80,10 @@ struct Assign<scatter_op::UpdateOp::MUL> {
   static void Run(Params p, Update u) {
     p *= u;
   }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p * u;
+  }
 };
 template <>
 struct Assign<scatter_op::UpdateOp::DIV> {
@@ -73,6 +91,34 @@ struct Assign<scatter_op::UpdateOp::DIV> {
   static void Run(Params p, Update u) {
     p /= u;
   }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p / u;
+  }
+};
+template <>
+struct Assign<scatter_op::UpdateOp::MIN> {
+  // This method requires that Params and Update are tensor types.
+  template <typename Params, typename Update>
+  static void Run(Params p, Update u) {
+    p = p.cwiseMin(u);
+  }
+  // Same thing, but for Update being a scalar type.
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p.cwiseMin(u);
+  }
+};
+template <>
+struct Assign<scatter_op::UpdateOp::MAX> {
+  template <typename Params, typename Update>
+  static void Run(Params p, Update u) {
+    p = p.cwiseMax(u);
+  }
+  template <typename Params, typename Update>
+  static void RunScalar(Params p, Update u) {
+    p = p.cwiseMax(u);
+  }
 };
 
 #ifdef TENSORFLOW_USE_SYCL
@@ -117,6 +163,22 @@ struct AssignSYCL<scatter_op::UpdateOp::DIV> {
     p.device(d) = p / u;
   }
 };
+
+template <>
+struct AssignSYCL<scatter_op::UpdateOp::MIN> {
+  template <typename Device, typename Params, typename Update>
+  static void Run(Device d, Params p, Update u) {
+    p.device(d) = p.cwiseMin(u);
+  }
+};
+
+template <>
+struct AssignSYCL<scatter_op::UpdateOp::MAX> {
+  template <typename Device, typename Params, typename Update>
+  static void Run(Device d, Params p, Update u) {
+    p.device(d) = p.cwiseMax(u);
+  }
+};
 #endif  // TENSORFLOW_USE_SYCL
 
 }  // namespace internal
@@ -241,6 +303,112 @@ struct ScatterFunctorSYCL {
 };
 #endif  // TENSORFLOW_USE_SYCL
 
+template <typename Device, typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctor {
+  Index operator()(OpKernelContext* c, const Device& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::ConstFlat indices);
+};
+
+template <typename Device, typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctorBase {
+  Index operator()(OpKernelContext* c, const Device& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::ConstFlat indices) {
+    // indices and params sizes were validated in DoCompute().
+    const Index N = static_cast<Index>(indices.size());
+    const Index limit = static_cast<Index>(params.dimension(0));
+    for (Index i = 0; i < N; i++) {
+      // Grab the index and check its validity.  An earlier version of the
+      // code checked it and then grabbed it from memory a second time, which
+      // was a security risk since it could have changed in between.
+      const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i));
+      if (!FastBoundsCheck(index, limit)) return i;
+      // Broadcast update to params[index]
+      scatter_op::internal::Assign<op>::RunScalar(
+          params.template chip<0>(index), update());
+    }
+    return -1;
+  }
+};
+
+#ifdef TENSORFLOW_USE_SYCL
+template <typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctorBase<SYCLDevice, T, Index, op> {
+  Index operator()(OpKernelContext* c, const SYCLDevice& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::ConstFlat indices) {
+    // indices and params sizes were validated in DoCompute().
+    const Index N = static_cast<Index>(indices.size());
+    const Index limit = static_cast<Index>(params.dimension(0));
+    for (Index i = 0; i < N; i++) {
+      // Grab the index and check its validity.  An earlier version of the
+      // code checked it and then grabbed it from memory a second time, which
+      // was a security risk since it could have changed in between.
+      const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i));
+      if (!FastBoundsCheck(index, limit)) return i;
+      // Broadcast update to params[index]
+      scatter_op::internal::AssignSYCL<op>::RunScalar(
+          d, params.template chip<0>(index), update);
+    }
+    return -1;
+  }
+};
+#endif  // TENSORFLOW_USE_SYCL
+
+template <typename T, typename Index>
+struct ScatterScalarFunctorBase<CPUDevice, T, Index,
+                                scatter_op::UpdateOp::ASSIGN> {
+  Index operator()(OpKernelContext* c, const CPUDevice& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::ConstFlat indices) {
+    // indices and params sizes were validated in DoCompute().
+    const Index N = static_cast<Index>(indices.size());
+    const Index limit = static_cast<Index>(params.dimension(0));
+    for (Index i = 0; i < N; i++) {
+      // Grab the index and check its validity.  An earlier version of the
+      // code checked it and then grabbed it from memory a second time, which
+      // was a security risk since it could have changed in between.
+      const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i));
+      if (!FastBoundsCheck(index, limit)) return i;
+      // Broadcast update to params[index]
+      scatter_op::internal::Assign<scatter_op::UpdateOp::ASSIGN>::RunScalar(
+          params.template chip<0>(index), update());
+    }
+    return -1;
+  }
+};
+
+template <typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctor<CPUDevice, T, Index, op>
+    : ScatterScalarFunctorBase<CPUDevice, T, Index, op> {};
+
+#ifdef TENSORFLOW_USE_SYCL
+template <typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctorSYCL {
+  Index operator()(OpKernelContext* c, const SYCLDevice& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::Flat indices) {
+    // indices and params sizes were validated in DoCompute().
+    const Index N = static_cast<Index>(indices.size());
+    const Index limit = static_cast<Index>(params.dimension(0));
+    for (Index i = 0; i < N; i++) {
+      const Index index = ::tensorflow::internal::SubtleMustCopy(indices(i));
+      if (!FastBoundsCheck(index, limit)) return i;
+      // Broadcast update to params[index]
+      scatter_op::internal::AssignSYCL<op>::Run(
+          d, params.template chip<0>(index), update());
+    }
+    return -1;
+  }
+};
+#endif  // TENSORFLOW_USE_SYCL
+
 }  // namespace functor
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/scatter_functor_gpu.cu.cc b/tensorflow/core/kernels/scatter_functor_gpu.cu.cc
index 52972997cc..59911bf0d2 100644
--- a/tensorflow/core/kernels/scatter_functor_gpu.cu.cc
+++ b/tensorflow/core/kernels/scatter_functor_gpu.cu.cc
@@ -23,15 +23,18 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
-#define DEFINE_GPU_SPECS_OP(T, Index, op) \
-  template struct functor::ScatterFunctor<GPUDevice, T, Index, op>;
+#define DEFINE_GPU_SPECS_OP(T, Index, op)                           \
+  template struct functor::ScatterFunctor<GPUDevice, T, Index, op>; \
+  template struct functor::ScatterScalarFunctor<GPUDevice, T, Index, op>;
 
 #define DEFINE_GPU_SPECS_INDEX(T, Index)                       \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ASSIGN); \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ADD);    \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::SUB);    \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MUL);    \
-  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);    \
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MIN);    \
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MAX);
 
 #define DEFINE_GPU_SPECS(T)         \
   DEFINE_GPU_SPECS_INDEX(T, int32); \
diff --git a/tensorflow/core/kernels/scatter_functor_gpu.cu.h b/tensorflow/core/kernels/scatter_functor_gpu.cu.h
index be18658543..70809e4dcf 100644
--- a/tensorflow/core/kernels/scatter_functor_gpu.cu.h
+++ b/tensorflow/core/kernels/scatter_functor_gpu.cu.h
@@ -29,12 +29,53 @@ namespace tensorflow {
 
 typedef Eigen::GpuDevice GPUDevice;
 
+namespace scatter_op_gpu {
+
+template <typename T, scatter_op::UpdateOp op>
+struct ScatterOpKernelBody;
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::ASSIGN> {
+  __device__ void operator()(T* dest, T src) const { *dest = src; }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::ADD> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicAdd(dest, src); }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::SUB> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicSub(dest, src); }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::MUL> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicMul(dest, src); }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::DIV> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicDiv(dest, src); }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::MIN> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicMin(dest, src); }
+};
+
+template <typename T>
+struct ScatterOpKernelBody<T, scatter_op::UpdateOp::MAX> {
+  __device__ void operator()(T* dest, T src) const { CudaAtomicMax(dest, src); }
+};
+
 template <typename T, typename Index, scatter_op::UpdateOp op>
 __global__ void ScatterOpCustomKernel(T* params, const T* updates,
                                       const Index* indices,
                                       Index first_dim_size, Index updates_size,
                                       Index indices_size) {
   Index update_block = updates_size / indices_size;
+  ScatterOpKernelBody<T, op> body;
   CUDA_1D_KERNEL_LOOP(i, updates_size) {
     int indices_i = i / update_block;
     int updates_i = i;
@@ -44,31 +85,33 @@ __global__ void ScatterOpCustomKernel(T* params, const T* updates,
       continue;
     }
     int params_i = param_first_index * update_block + (i % update_block);
-    switch (op) {
-      case scatter_op::UpdateOp::ASSIGN: {
-        params[params_i] = ldg(updates + updates_i);
-        break;
-      }
-      case scatter_op::UpdateOp::ADD: {
-        CudaAtomicAdd(params + params_i, ldg(updates + updates_i));
-        break;
-      }
-      case scatter_op::UpdateOp::SUB: {
-        CudaAtomicSub(params + params_i, ldg(updates + updates_i));
-        break;
-      }
-      case scatter_op::UpdateOp::MUL: {
-        CudaAtomicMul(params + params_i, ldg(updates + updates_i));
-        break;
-      }
-      case scatter_op::UpdateOp::DIV: {
-        CudaAtomicDiv(params + params_i, ldg(updates + updates_i));
-        break;
-      }
+    body(&params[params_i], ldg(updates + updates_i));
+  }
+}
+
+template <typename T, typename Index, scatter_op::UpdateOp op>
+__global__ void ScatterScalarOpCustomKernel(T* params, const T* update,
+                                            const Index* indices,
+                                            Index first_dim_size,
+                                            Index indices_size,
+                                            Index synthesized_updates_size) {
+  Index update_block = synthesized_updates_size / indices_size;
+  ScatterOpKernelBody<T, op> body;
+  CUDA_1D_KERNEL_LOOP(i, synthesized_updates_size) {
+    int indices_i = i / update_block;
+    int param_first_index = indices[indices_i];
+    const T update_val = *update;
+    if (!(param_first_index >= 0 && param_first_index < first_dim_size)) {
+      // Ignore indices that are out of range.
+      continue;
     }
+    int params_i = param_first_index * update_block + (i % update_block);
+    body(&params[params_i], update_val);
   }
 }
 
+}  // namespace scatter_op_gpu
+
 namespace functor {
 // Specialization for a GPU device.
 template <typename T, typename Index, scatter_op::UpdateOp op>
@@ -84,7 +127,7 @@ struct ScatterFunctor<GPUDevice, T, Index, op> {
     const Index indices_size = indices.size();
     const Index updates_size = updates.size();
     CudaLaunchConfig config = GetCudaLaunchConfig(updates_size, d);
-    ScatterOpCustomKernel<T, Index, op>
+    scatter_op_gpu::ScatterOpCustomKernel<T, Index, op>
         <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
             params.data(), updates.data(), indices.data(), first_dim_size,
             updates_size, indices_size);
@@ -92,6 +135,27 @@ struct ScatterFunctor<GPUDevice, T, Index, op> {
   }
 };
 
+template <typename T, typename Index, scatter_op::UpdateOp op>
+struct ScatterScalarFunctor<GPUDevice, T, Index, op> {
+  Index operator()(OpKernelContext* c, const GPUDevice& d,
+                   typename TTypes<T>::Matrix params,
+                   const typename TTypes<T>::ConstScalar update,
+                   typename TTypes<Index>::ConstFlat indices) {
+    // TODO(b/31801742): Implement indices range check. The hardest part is
+    // with returning a value after the range check, as we do not want to do
+    // device to host memcpy during a stream.
+    const Index first_dim_size = params.dimension(0);
+    const Index indices_size = indices.size();
+    const Index synthesized_updates_size = indices_size * params.dimension(1);
+    CudaLaunchConfig config = GetCudaLaunchConfig(synthesized_updates_size, d);
+    scatter_op_gpu::ScatterScalarOpCustomKernel<T, Index, op>
+        <<<config.block_count, config.thread_per_block, 0, d.stream()>>>(
+            params.data(), update.data(), indices.data(), first_dim_size,
+            indices_size, synthesized_updates_size);
+    return -1;
+  }
+};
+
 }  // namespace functor
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/scatter_op.cc b/tensorflow/core/kernels/scatter_op.cc
index 282165349f..0fbde764d5 100644
--- a/tensorflow/core/kernels/scatter_op.cc
+++ b/tensorflow/core/kernels/scatter_op.cc
@@ -38,6 +38,7 @@ typedef Eigen::SyclDevice SYCLDevice;
 // Check whether updates.shape = indices.shape + params.shape[1:]
 static bool ValidShapes(const Tensor& params, const Tensor& updates,
                         const Tensor& indices) {
+  if (updates.dims() == 0) return true;
   if (updates.dims() != indices.dims() + params.dims() - 1) return false;
   for (int d = 0; d < indices.dims(); d++) {
     if (updates.dim_size(d) != indices.dim_size(d)) {
@@ -61,11 +62,11 @@ static void DoValidationChecking(OpKernelContext* c, const Tensor& params,
                                       params.shape().DebugString()));
   OP_REQUIRES(
       c, ValidShapes(params, updates, indices),
-      errors::InvalidArgument(
-          "Must have updates.shape = indices.shape + params.shape[1:], got ",
-          "updates.shape ", updates.shape().DebugString(), ", indices.shape ",
-          indices.shape().DebugString(), ", params.shape ",
-          params.shape().DebugString()));
+      errors::InvalidArgument("Must have updates.shape = indices.shape + "
+                              "params.shape[1:] or updates.shape = [], got ",
+                              "updates.shape ", updates.shape().DebugString(),
+                              ", indices.shape ", indices.shape().DebugString(),
+                              ", params.shape ", params.shape().DebugString()));
 }
 
 template <typename Device, typename T, typename Index, scatter_op::UpdateOp op>
@@ -122,16 +123,31 @@ class ScatterUpdateOp : public OpKernel {
     if (N > 0) {
       auto indices_flat = indices.flat<Index>();
       auto params_flat = params.flat_outer_dims<T>();
-      auto updates_flat = updates.shaped<T, 2>({N, updates.NumElements() / N});
-
-      functor::ScatterFunctor<Device, T, Index, op> functor;
-      const Index bad_i = functor(c, c->template eigen_device<Device>(),
-                                  params_flat, updates_flat, indices_flat);
-      OP_REQUIRES(
-          c, bad_i < 0,
-          errors::InvalidArgument(
-              "indices", SliceDebugString(indices.shape(), bad_i), " = ",
-              indices_flat(bad_i), " is not in [0, ", params.dim_size(0), ")"));
+
+      if (TensorShapeUtils::IsScalar(updates.shape()) ||
+          IsLegacyScalar(updates.shape())) {
+        const auto update = updates.scalar<T>();
+        functor::ScatterScalarFunctor<Device, T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<Device>(),
+                                    params_flat, update, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params.dim_size(0), ")"));
+      } else {
+        auto updates_flat =
+            updates.shaped<T, 2>({N, updates.NumElements() / N});
+
+        functor::ScatterFunctor<Device, T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<Device>(),
+                                    params_flat, updates_flat, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params.dim_size(0), ")"));
+      }
     }
   }
 };
@@ -195,16 +211,31 @@ class ScatterUpdateOp<SYCLDevice, T, Index, op> : public OpKernel {
 
       auto indices_flat = indices_host.flat<Index>();
       auto params_flat = params.flat_outer_dims<T>();
-      auto updates_flat = updates.shaped<T, 2>({N, updates.NumElements() / N});
-
-      functor::ScatterFunctorSYCL<T, Index, op> functor;
-      const Index bad_i = functor(c, c->template eigen_device<SYCLDevice>(),
-                                  params_flat, updates_flat, indices_flat);
-      OP_REQUIRES(
-          c, bad_i < 0,
-          errors::InvalidArgument(
-              "indices", SliceDebugString(indices.shape(), bad_i), " = ",
-              indices_flat(bad_i), " is not in [0, ", params.dim_size(0), ")"));
+
+      if (TensorShapeUtils::IsScalar(updates.shape())) {
+        const auto update = updates.scalar<T>();
+
+        functor::ScatterScalarFunctorSYCL<T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<SYCLDevice>(),
+                                    params_flat, update, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params.dim_size(0), ")"));
+      } else {
+        auto updates_flat =
+            updates.shaped<T, 2>({N, updates.NumElements() / N});
+
+        functor::ScatterFunctorSYCL<T, Index, op> functor;
+        const Index bad_i = functor(c, c->template eigen_device<SYCLDevice>(),
+                                    params_flat, updates_flat, indices_flat);
+        OP_REQUIRES(c, bad_i < 0,
+                    errors::InvalidArgument(
+                        "indices", SliceDebugString(indices.shape(), bad_i),
+                        " = ", indices_flat(bad_i), " is not in [0, ",
+                        params.dim_size(0), ")"));
+      }
     }
   }
 };
@@ -221,54 +252,71 @@ class ScatterUpdateOp<SYCLDevice, T, Index, op> : public OpKernel {
   REGISTER_SCATTER_KERNEL_INDEX(type, int32, dev, name, op); \
   REGISTER_SCATTER_KERNEL_INDEX(type, int64, dev, name, op);
 
-#define REGISTER_SCATTER_ARITHEMTIC(type, dev)                                 \
+#define REGISTER_SCATTER_ARITHMETIC(type, dev)                                 \
   REGISTER_SCATTER_KERNEL(type, dev, "ScatterAdd", scatter_op::UpdateOp::ADD); \
   REGISTER_SCATTER_KERNEL(type, dev, "ScatterDiv", scatter_op::UpdateOp::DIV); \
   REGISTER_SCATTER_KERNEL(type, dev, "ScatterMul", scatter_op::UpdateOp::MUL); \
   REGISTER_SCATTER_KERNEL(type, dev, "ScatterSub", scatter_op::UpdateOp::SUB);
 
+#define REGISTER_SCATTER_MINMAX(type, dev)                                     \
+  REGISTER_SCATTER_KERNEL(type, dev, "ScatterMin", scatter_op::UpdateOp::MIN); \
+  REGISTER_SCATTER_KERNEL(type, dev, "ScatterMax", scatter_op::UpdateOp::MAX);
+
 #define REGISTER_SCATTER_UPDATE(type, dev)            \
   REGISTER_SCATTER_KERNEL(type, dev, "ScatterUpdate", \
                           scatter_op::UpdateOp::ASSIGN);
 
 // Registers CPU kernels.
-#define REGISTER_SCATTER_ARITHEMTIC_CPU(type) \
-  REGISTER_SCATTER_ARITHEMTIC(type, CPU);
+#define REGISTER_SCATTER_ARITHMETIC_CPU(type) \
+  REGISTER_SCATTER_ARITHMETIC(type, CPU);
+
+#define REGISTER_SCATTER_MINMAX_CPU(type) REGISTER_SCATTER_MINMAX(type, CPU);
 
 #define REGISTER_SCATTER_UPDATE_CPU(type) REGISTER_SCATTER_UPDATE(type, CPU);
 
-TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ARITHEMTIC_CPU);
+TF_CALL_REAL_NUMBER_TYPES(REGISTER_SCATTER_MINMAX_CPU);
+TF_CALL_NUMBER_TYPES(REGISTER_SCATTER_ARITHMETIC_CPU);
 TF_CALL_ALL_TYPES(REGISTER_SCATTER_UPDATE_CPU);
 
 // Registers GPU kernels.
 #if GOOGLE_CUDA
-#define REGISTER_SCATTER_ARITHEMTIC_GPU(type) \
-  REGISTER_SCATTER_ARITHEMTIC(type, GPU);
+#define REGISTER_SCATTER_ARITHMETIC_GPU(type) \
+  REGISTER_SCATTER_ARITHMETIC(type, GPU);
+
+#define REGISTER_SCATTER_MINMAX_GPU(type) REGISTER_SCATTER_MINMAX(type, GPU);
 
 #define REGISTER_SCATTER_UPDATE_GPU(type) REGISTER_SCATTER_UPDATE(type, GPU);
 
-TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHEMTIC_GPU);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHMETIC_GPU);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_MINMAX_GPU);
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_UPDATE_GPU);
 
 #endif  // GOOGLE_CUDA
 
 // Registers GPU kernels.
 #if TENSORFLOW_USE_SYCL
-#define REGISTER_SCATTER_ARITHEMTIC_SYCL(type) \
-  REGISTER_SCATTER_ARITHEMTIC(type, SYCL);
+#define REGISTER_SCATTER_ARITHMETIC_SYCL(type) \
+  REGISTER_SCATTER_ARITHMETIC(type, SYCL);
+
+#define REGISTER_SCATTER_MINMAX_SYCL(type) REGISTER_SCATTER_MINMAX(type, SYCL);
 
 #define REGISTER_SCATTER_UPDATE_SYCL(type) REGISTER_SCATTER_UPDATE(type, SYCL);
 
-TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHEMTIC_SYCL);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_ARITHMETIC_SYCL);
+TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_MINMAX_SYCL);
 TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_SCATTER_UPDATE_SYCL);
 
-#undef REGISTER_SCATTER_ARITHEMTIC_SYCL
+#undef REGISTER_SCATTER_ARITHMETIC_SYCL
+#undef REGISTER_SCATTER_MINMAX_SYCL
 #undef REGISTER_SCATTER_UPDATE_SYCL
 #endif  // TENSORFLOW_USE_SYCL
 
-#undef REGISTER_SCATTER_ARITHEMTIC
-#undef REGISTER_SCATTER_ARITHEMTIC_CPU
-#undef REGISTER_SCATTER_ARITHEMTIC_GPU
+#undef REGISTER_SCATTER_ARITHMETIC
+#undef REGISTER_SCATTER_ARITHMETIC_CPU
+#undef REGISTER_SCATTER_ARITHMETIC_GPU
+#undef REGISTER_SCATTER_MINMAX
+#undef REGISTER_SCATTER_MINMAX_CPU
+#undef REGISTER_SCATTER_MINMAX_GPU
 #undef REGISTER_SCATTER_UPDATE
 #undef REGISTER_SCATTER_UPDATE_CPU
 #undef REGISTER_SCATTER_UPDATE_GPU
diff --git a/tensorflow/core/kernels/scatter_op_gpu.cu.cc b/tensorflow/core/kernels/scatter_op_gpu.cu.cc
index 0b43704846..0df329310f 100644
--- a/tensorflow/core/kernels/scatter_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/scatter_op_gpu.cu.cc
@@ -24,15 +24,18 @@ namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
 // Instantiates functor specializations for GPU.
-#define DEFINE_GPU_SPECS_OP(T, Index, op) \
-  template struct functor::ScatterFunctor<GPUDevice, T, Index, op>;
+#define DEFINE_GPU_SPECS_OP(T, Index, op)                           \
+  template struct functor::ScatterFunctor<GPUDevice, T, Index, op>; \
+  template struct functor::ScatterScalarFunctor<GPUDevice, T, Index, op>;
 
 #define DEFINE_GPU_SPECS_INDEX(T, Index)                       \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ASSIGN); \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::ADD);    \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::SUB);    \
   DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MUL);    \
-  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::DIV);    \
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MIN);    \
+  DEFINE_GPU_SPECS_OP(T, Index, scatter_op::UpdateOp::MAX);
 
 #define DEFINE_GPU_SPECS(T)         \
   DEFINE_GPU_SPECS_INDEX(T, int32); \
diff --git a/tensorflow/core/kernels/scatter_op_test.cc b/tensorflow/core/kernels/scatter_op_test.cc
index 0b8645a2ae..5b3537b94c 100644
--- a/tensorflow/core/kernels/scatter_op_test.cc
+++ b/tensorflow/core/kernels/scatter_op_test.cc
@@ -185,7 +185,7 @@ TEST_F(ScatterUpdateOpTest, Error_WrongDimsIndices) {
   Status s = RunOpKernel();
   EXPECT_TRUE(StringPiece(s.ToString())
                   .contains("Must have updates.shape = indices.shape + "
-                            "params.shape[1:], got "))
+                            "params.shape[1:] or updates.shape = [], got "))
       << s;
 }
 
@@ -202,7 +202,7 @@ TEST_F(ScatterUpdateOpTest, Error_MismatchedParamsAndUpdateDimensions) {
   Status s = RunOpKernel();
   EXPECT_TRUE(StringPiece(s.ToString())
                   .contains("Must have updates.shape = indices.shape + "
-                            "params.shape[1:], got "))
+                            "params.shape[1:] or updates.shape = [], got "))
 
       << s;
 }
@@ -219,7 +219,7 @@ TEST_F(ScatterUpdateOpTest, Error_MismatchedIndicesAndUpdateDimensions) {
   Status s = RunOpKernel();
   EXPECT_TRUE(StringPiece(s.ToString())
                   .contains("Must have updates.shape = indices.shape + "
-                            "params.shape[1:], got "))
+                            "params.shape[1:] or updates.shape = [], got "))
       << s;
 }
 
@@ -300,6 +300,20 @@ static void BM_ScatterDivInt64(int iters, int embedding_size) {
   BM_ScatterHelper<int64>(iters, embedding_size, "ScatterDiv");
 }
 
+static void BM_ScatterMinInt32(int iters, int embedding_size) {
+  BM_ScatterHelper<int32>(iters, embedding_size, "ScatterMin");
+}
+static void BM_ScatterMinInt64(int iters, int embedding_size) {
+  BM_ScatterHelper<int64>(iters, embedding_size, "ScatterMin");
+}
+
+static void BM_ScatterMaxInt32(int iters, int embedding_size) {
+  BM_ScatterHelper<int32>(iters, embedding_size, "ScatterMax");
+}
+static void BM_ScatterMaxInt64(int iters, int embedding_size) {
+  BM_ScatterHelper<int64>(iters, embedding_size, "ScatterMax");
+}
+
 BENCHMARK(BM_ScatterUpdateInt32)
     ->Arg(1)
     ->Arg(10)
@@ -332,5 +346,11 @@ BENCHMARK(BM_ScatterMulInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
 BENCHMARK(BM_ScatterDivInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
 BENCHMARK(BM_ScatterDivInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
 
+BENCHMARK(BM_ScatterMinInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
+BENCHMARK(BM_ScatterMinInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
+
+BENCHMARK(BM_ScatterMaxInt32)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
+BENCHMARK(BM_ScatterMaxInt64)->Arg(1)->Arg(10)->Arg(64)->Arg(256)->Arg(1024);
+
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/resource_variable_ops.cc b/tensorflow/core/ops/resource_variable_ops.cc
index 0d8cf78cc2..3d0a6c2157 100644
--- a/tensorflow/core/ops/resource_variable_ops.cc
+++ b/tensorflow/core/ops/resource_variable_ops.cc
@@ -167,27 +167,75 @@ REGISTER_OP("ResourceGather")
       return Status::OK();
     });
 
+namespace {
+
+Status ResourceScatterUpdateShape(InferenceContext* c) {
+  ShapeAndType handle_shape_and_type;
+  TF_RETURN_IF_ERROR(ValidateVariableResourceHandle(c, &handle_shape_and_type));
+  ShapeHandle var_shape = handle_shape_and_type.shape;
+  ShapeHandle indices_shape = c->input(1);
+
+  ShapeHandle unused_updates_shape;
+  ShapeHandle concat;
+  ShapeHandle var_subshape;
+  TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape));
+  TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat));
+  TF_RETURN_IF_ERROR(
+      InferenceContext::Rank(c->input(2)) == 0
+          ? Status::OK()
+          : c->Merge(c->input(2), concat, &unused_updates_shape));
+  return Status::OK();
+}
+
+}  // namespace
+
 REGISTER_OP("ResourceScatterAdd")
     .Input("resource: resource")
     .Input("indices: Tindices")
     .Input("updates: dtype")
     .Attr("dtype: numbertype")
     .Attr("Tindices: {int32, int64}")
-    .SetShapeFn([](InferenceContext* c) {
-      ShapeAndType handle_shape_and_type;
-      TF_RETURN_IF_ERROR(
-          ValidateVariableResourceHandle(c, &handle_shape_and_type));
-      ShapeHandle var_shape = handle_shape_and_type.shape;
-      ShapeHandle indices_shape = c->input(1);
+    .SetShapeFn(ResourceScatterUpdateShape);
 
-      ShapeHandle unused_updates_shape;
-      ShapeHandle concat;
-      ShapeHandle var_subshape;
-      TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape));
-      TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat));
-      TF_RETURN_IF_ERROR(c->Merge(c->input(2), concat, &unused_updates_shape));
-      return Status::OK();
-    });
+REGISTER_OP("ResourceScatterSub")
+    .Input("resource: resource")
+    .Input("indices: Tindices")
+    .Input("updates: dtype")
+    .Attr("dtype: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ResourceScatterUpdateShape);
+
+REGISTER_OP("ResourceScatterMul")
+    .Input("resource: resource")
+    .Input("indices: Tindices")
+    .Input("updates: dtype")
+    .Attr("dtype: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ResourceScatterUpdateShape);
+
+REGISTER_OP("ResourceScatterDiv")
+    .Input("resource: resource")
+    .Input("indices: Tindices")
+    .Input("updates: dtype")
+    .Attr("dtype: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ResourceScatterUpdateShape);
+
+REGISTER_OP("ResourceScatterMin")
+    .Input("resource: resource")
+    .Input("indices: Tindices")
+    .Input("updates: dtype")
+    .Attr("dtype: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ResourceScatterUpdateShape);
+
+REGISTER_OP("ResourceScatterMax")
+    .Input("resource: resource")
+    .Input("indices: Tindices")
+    .Input("updates: dtype")
+    .Attr("dtype: numbertype")
+    .Attr("Tindices: {int32, int64}")
+    .SetShapeFn(ResourceScatterUpdateShape);
 
 REGISTER_OP("ResourceScatterUpdate")
     .Input("resource: resource")
@@ -195,21 +243,7 @@ REGISTER_OP("ResourceScatterUpdate")
     .Input("updates: dtype")
     .Attr("dtype: type")
     .Attr("Tindices: {int32, int64}")
-    .SetShapeFn([](InferenceContext* c) {
-      ShapeAndType handle_shape_and_type;
-      TF_RETURN_IF_ERROR(
-          ValidateVariableResourceHandle(c, &handle_shape_and_type));
-      ShapeHandle var_shape = handle_shape_and_type.shape;
-      ShapeHandle indices_shape = c->input(1);
-
-      ShapeHandle unused_updates_shape;
-      ShapeHandle concat;
-      ShapeHandle var_subshape;
-      TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape));
-      TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat));
-      TF_RETURN_IF_ERROR(c->Merge(c->input(2), concat, &unused_updates_shape));
-      return Status::OK();
-    });
+    .SetShapeFn(ResourceScatterUpdateShape);
 
 REGISTER_OP("MutexV2")
     .Attr("container: string = ''")
diff --git a/tensorflow/core/ops/state_ops.cc b/tensorflow/core/ops/state_ops.cc
index 7a524b60c0..664f52452e 100644
--- a/tensorflow/core/ops/state_ops.cc
+++ b/tensorflow/core/ops/state_ops.cc
@@ -122,7 +122,10 @@ Status ScatterUpdateShape(InferenceContext* c) {
   ShapeHandle var_subshape;
   TF_RETURN_IF_ERROR(c->Subshape(var_shape, 1, &var_subshape));
   TF_RETURN_IF_ERROR(c->Concatenate(indices_shape, var_subshape, &concat));
-  TF_RETURN_IF_ERROR(c->Merge(c->input(2), concat, &unused_updates_shape));
+  TF_RETURN_IF_ERROR(
+      InferenceContext::Rank(c->input(2)) == 0
+          ? Status::OK()
+          : c->Merge(c->input(2), concat, &unused_updates_shape));
 
   c->set_output(0, var_shape);
   return Status::OK();
@@ -180,6 +183,26 @@ REGISTER_OP("ScatterDiv")
     .Attr("use_locking: bool = false")
     .SetShapeFn(ScatterUpdateShape);
 
+REGISTER_OP("ScatterMin")
+    .Input("ref: Ref(T)")
+    .Input("indices: Tindices")
+    .Input("updates: T")
+    .Output("output_ref: Ref(T)")
+    .Attr("T: {half, bfloat16, float, double, int32, int64}")
+    .Attr("Tindices: {int32, int64}")
+    .Attr("use_locking: bool = false")
+    .SetShapeFn(ScatterUpdateShape);
+
+REGISTER_OP("ScatterMax")
+    .Input("ref: Ref(T)")
+    .Input("indices: Tindices")
+    .Input("updates: T")
+    .Output("output_ref: Ref(T)")
+    .Attr("T: {half, bfloat16, float, double, int32, int64}")
+    .Attr("Tindices: {int32, int64}")
+    .Attr("use_locking: bool = false")
+    .SetShapeFn(ScatterUpdateShape);
+
 REGISTER_OP("ScatterNdUpdate")
     .Input("ref: Ref(T)")
     .Input("indices: Tindices")
diff --git a/tensorflow/docs_src/api_guides/python/state_ops.md b/tensorflow/docs_src/api_guides/python/state_ops.md
index 0d612ee0c7..ec2d877386 100644
--- a/tensorflow/docs_src/api_guides/python/state_ops.md
+++ b/tensorflow/docs_src/api_guides/python/state_ops.md
@@ -83,6 +83,8 @@ automatically by the optimizers in most cases.
 *   @{tf.scatter_sub}
 *   @{tf.scatter_mul}
 *   @{tf.scatter_div}
+*   @{tf.scatter_min}
+*   @{tf.scatter_max}
 *   @{tf.scatter_nd_update}
 *   @{tf.scatter_nd_add}
 *   @{tf.scatter_nd_sub}
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 563eeff2a6..742564f9bf 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -185,6 +185,204 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
       self.assertEqual(self.evaluate(read), [[3]])
 
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterSub(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[1]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_sub(handle, [0],
+                                                     constant_op.constant(
+                                                         [[2]],
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[-1]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMul(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[1]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_mul(handle, [0],
+                                                     constant_op.constant(
+                                                         [[5]],
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[5]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterDiv(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_div(handle, [0],
+                                                     constant_op.constant(
+                                                         [[3]],
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[2]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMin(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_min(handle, [0],
+                                                     constant_op.constant(
+                                                         [[3]],
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[3]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMax(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_max(handle, [0],
+                                                     constant_op.constant(
+                                                         [[3]],
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[6]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterAddScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[1]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_add(handle, [0],
+                                                     constant_op.constant(
+                                                         2,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[3]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterSubScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[1]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_sub(handle, [0],
+                                                     constant_op.constant(
+                                                         2,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[-1]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMulScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[1]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_mul(handle, [0],
+                                                     constant_op.constant(
+                                                         5,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[5]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterDivScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_div(handle, [0],
+                                                     constant_op.constant(
+                                                         3,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[2]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMinScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_min(handle, [0],
+                                                     constant_op.constant(
+                                                         3,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[3]])
+
+  @test_util.run_in_graph_and_eager_modes(use_gpu=True)
+  def testScatterMaxScalar(self):
+    with ops.device("cpu:0"):
+      handle = resource_variable_ops.var_handle_op(
+          dtype=dtypes.int32, shape=[1, 1])
+      self.evaluate(
+          resource_variable_ops.assign_variable_op(handle,
+                                                   constant_op.constant(
+                                                       [[6]],
+                                                       dtype=dtypes.int32)))
+      self.evaluate(
+          resource_variable_ops.resource_scatter_max(handle, [0],
+                                                     constant_op.constant(
+                                                         3,
+                                                         dtype=dtypes.int32)))
+      read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.int32)
+      self.assertEqual(self.evaluate(read), [[6]])
+
   def testScatterUpdateString(self):
     handle = resource_variable_ops.var_handle_op(
         dtype=dtypes.string, shape=[1, 1])
@@ -196,6 +394,23 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     self.assertEqual(compat.as_bytes(self.evaluate(read)[0][0]),
                      compat.as_bytes("b"))
 
+  def testScatterUpdateStringScalar(self):
+    handle = resource_variable_ops.var_handle_op(
+        dtype=dtypes.string, shape=[1, 1])
+    self.evaluate(
+        resource_variable_ops.assign_variable_op(handle,
+                                                 constant_op.constant(
+                                                     [["a"]],
+                                                     dtype=dtypes.string)))
+    self.evaluate(
+        resource_variable_ops.resource_scatter_update(handle, [0],
+                                                      constant_op.constant(
+                                                          "b",
+                                                          dtype=dtypes.string)))
+    read = resource_variable_ops.read_variable_op(handle, dtype=dtypes.string)
+    self.assertEqual(
+        compat.as_bytes(self.evaluate(read)[0][0]), compat.as_bytes("b"))
+
   # TODO(alive): get this to work in Eager mode.
   def testGPU(self):
     with self.test_session(use_gpu=True):
diff --git a/tensorflow/python/kernel_tests/scatter_ops_test.py b/tensorflow/python/kernel_tests/scatter_ops_test.py
index 7cdf11d884..c70a4ffce7 100644
--- a/tensorflow/python/kernel_tests/scatter_ops_test.py
+++ b/tensorflow/python/kernel_tests/scatter_ops_test.py
@@ -38,38 +38,100 @@ def _NumpyAdd(ref, indices, updates):
     ref[indx] += updates[i]
 
 
+def _NumpyAddScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] += update
+
+
 def _NumpySub(ref, indices, updates):
   for i, indx in np.ndenumerate(indices):
     ref[indx] -= updates[i]
 
 
+def _NumpySubScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] -= update
+
+
 def _NumpyMul(ref, indices, updates):
   for i, indx in np.ndenumerate(indices):
     ref[indx] *= updates[i]
 
 
+def _NumpyMulScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] *= update
+
+
 def _NumpyDiv(ref, indices, updates):
   for i, indx in np.ndenumerate(indices):
     ref[indx] /= updates[i]
 
 
+def _NumpyDivScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] /= update
+
+
+def _NumpyMin(ref, indices, updates):
+  for i, indx in np.ndenumerate(indices):
+    ref[indx] = np.minimum(ref[indx], updates[i])
+
+
+def _NumpyMinScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] = np.minimum(ref[indx], update)
+
+
+def _NumpyMax(ref, indices, updates):
+  for i, indx in np.ndenumerate(indices):
+    ref[indx] = np.maximum(ref[indx], updates[i])
+
+
+def _NumpyMaxScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] = np.maximum(ref[indx], update)
+
+
 def _NumpyUpdate(ref, indices, updates):
   for i, indx in np.ndenumerate(indices):
     ref[indx] = updates[i]
 
 
+def _NumpyUpdateScalar(ref, indices, update):
+  for _, indx in np.ndenumerate(indices):
+    ref[indx] = update
+
+
 _TF_OPS_TO_NUMPY = {
     state_ops.scatter_update: _NumpyUpdate,
     state_ops.scatter_add: _NumpyAdd,
     state_ops.scatter_sub: _NumpySub,
     state_ops.scatter_mul: _NumpyMul,
     state_ops.scatter_div: _NumpyDiv,
+    state_ops.scatter_min: _NumpyMin,
+    state_ops.scatter_max: _NumpyMax,
+}
+
+_TF_OPS_TO_NUMPY_SCALAR = {
+    state_ops.scatter_update: _NumpyUpdateScalar,
+    state_ops.scatter_add: _NumpyAddScalar,
+    state_ops.scatter_sub: _NumpySubScalar,
+    state_ops.scatter_mul: _NumpyMulScalar,
+    state_ops.scatter_div: _NumpyDivScalar,
+    state_ops.scatter_min: _NumpyMinScalar,
+    state_ops.scatter_max: _NumpyMaxScalar,
 }
 
 
 class ScatterTest(test.TestCase):
 
-  def _VariableRankTest(self, tf_scatter, vtype, itype, repeat_indices=False):
+  def _VariableRankTest(self,
+                        tf_scatter,
+                        vtype,
+                        itype,
+                        repeat_indices=False,
+                        updates_are_scalar=False):
     np.random.seed(8)
     with self.test_session(use_gpu=True):
       for indices_shape in (), (2,), (3, 7), (3, 4, 7):
@@ -89,8 +151,11 @@ class ScatterTest(test.TestCase):
                                   indices[np.random.randint(size // 2)])
             np.random.shuffle(indices)
           indices = indices.reshape(indices_shape)
-          updates = _AsType(
-              np.random.randn(*(indices_shape + extra_shape)), vtype)
+          if updates_are_scalar:
+            updates = _AsType(np.random.randn(), vtype)
+          else:
+            updates = _AsType(
+                np.random.randn(*(indices_shape + extra_shape)), vtype)
 
           # Clips small values to avoid division by zero.
           def clip_small_values(x):
@@ -101,7 +166,10 @@ class ScatterTest(test.TestCase):
 
           # Scatter via numpy
           new = old.copy()
-          np_scatter = _TF_OPS_TO_NUMPY[tf_scatter]
+          if updates_are_scalar:
+            np_scatter = _TF_OPS_TO_NUMPY_SCALAR[tf_scatter]
+          else:
+            np_scatter = _TF_OPS_TO_NUMPY[tf_scatter]
           np_scatter(new, indices, updates)
           # Scatter via tensorflow
           ref = variables.Variable(old)
@@ -109,25 +177,35 @@ class ScatterTest(test.TestCase):
           tf_scatter(ref, indices, updates).eval()
           self.assertAllClose(ref.eval(), new)
 
-  def _VariableRankTests(self, tf_scatter, repeat_indices=False):
+  def _VariableRankTests(self,
+                         tf_scatter,
+                         repeat_indices=False,
+                         updates_are_scalar=False):
     for vtype in (np.float32, np.float64):
       for itype in (np.int32, np.int64):
-        self._VariableRankTest(tf_scatter, vtype, itype, repeat_indices)
+        self._VariableRankTest(tf_scatter, vtype, itype, repeat_indices,
+                               updates_are_scalar)
 
   def testVariableRankUpdate(self):
-    self._VariableRankTests(state_ops.scatter_update)
+    self._VariableRankTests(state_ops.scatter_update, False)
 
   def testVariableRankAdd(self):
-    self._VariableRankTests(state_ops.scatter_add)
+    self._VariableRankTests(state_ops.scatter_add, False)
 
   def testVariableRankSub(self):
-    self._VariableRankTests(state_ops.scatter_sub)
+    self._VariableRankTests(state_ops.scatter_sub, False)
 
   def testVariableRankMul(self):
-    self._VariableRankTests(state_ops.scatter_mul)
+    self._VariableRankTests(state_ops.scatter_mul, False)
 
   def testVariableRankDiv(self):
-    self._VariableRankTests(state_ops.scatter_div)
+    self._VariableRankTests(state_ops.scatter_div, False)
+
+  def testVariableRankMin(self):
+    self._VariableRankTests(state_ops.scatter_min, False)
+
+  def testVariableRankMax(self):
+    self._VariableRankTests(state_ops.scatter_max, False)
 
   def testRepeatIndicesAdd(self):
     self._VariableRankTests(state_ops.scatter_add, True)
@@ -141,6 +219,51 @@ class ScatterTest(test.TestCase):
   def testRepeatIndicesDiv(self):
     self._VariableRankTests(state_ops.scatter_div, True)
 
+  def testRepeatIndicesMin(self):
+    self._VariableRankTests(state_ops.scatter_min, True)
+
+  def testRepeatIndicesMax(self):
+    self._VariableRankTests(state_ops.scatter_max, True)
+
+  def testVariableRankUpdateScalar(self):
+    self._VariableRankTests(state_ops.scatter_update, False, True)
+
+  def testVariableRankAddScalar(self):
+    self._VariableRankTests(state_ops.scatter_add, False, True)
+
+  def testVariableRankSubScalar(self):
+    self._VariableRankTests(state_ops.scatter_sub, False, True)
+
+  def testVariableRankMulScalar(self):
+    self._VariableRankTests(state_ops.scatter_mul, False, True)
+
+  def testVariableRankDivScalar(self):
+    self._VariableRankTests(state_ops.scatter_div, False, True)
+
+  def testVariableRankMinScalar(self):
+    self._VariableRankTests(state_ops.scatter_min, False, True)
+
+  def testVariableRankMaxScalar(self):
+    self._VariableRankTests(state_ops.scatter_max, False, True)
+
+  def testRepeatIndicesAddScalar(self):
+    self._VariableRankTests(state_ops.scatter_add, True, True)
+
+  def testRepeatIndicesSubScalar(self):
+    self._VariableRankTests(state_ops.scatter_sub, True, True)
+
+  def testRepeatIndicesMulScalar(self):
+    self._VariableRankTests(state_ops.scatter_mul, True, True)
+
+  def testRepeatIndicesDivScalar(self):
+    self._VariableRankTests(state_ops.scatter_div, True, True)
+
+  def testRepeatIndicesMinScalar(self):
+    self._VariableRankTests(state_ops.scatter_min, True, True)
+
+  def testRepeatIndicesMaxScalar(self):
+    self._VariableRankTests(state_ops.scatter_max, True, True)
+
   def testBooleanScatterUpdate(self):
     if not test.is_gpu_available():
       with self.test_session(use_gpu=False) as session:
diff --git a/tensorflow/python/ops/standard_ops.py b/tensorflow/python/ops/standard_ops.py
index 230b7ef937..e90ff0746a 100644
--- a/tensorflow/python/ops/standard_ops.py
+++ b/tensorflow/python/ops/standard_ops.py
@@ -80,6 +80,8 @@ from tensorflow.python.ops.state_ops import scatter_add
 from tensorflow.python.ops.state_ops import scatter_div
 from tensorflow.python.ops.state_ops import scatter_mul
 from tensorflow.python.ops.state_ops import scatter_sub
+from tensorflow.python.ops.state_ops import scatter_min
+from tensorflow.python.ops.state_ops import scatter_max
 from tensorflow.python.ops.state_ops import scatter_update
 from tensorflow.python.ops.state_ops import scatter_nd_add
 from tensorflow.python.ops.state_ops import scatter_nd_sub
diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index c3ad5831b4..01fc3182bc 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -63,6 +63,8 @@
 @@scatter_nd_update
 @@scatter_sub
 @@scatter_update
+@@scatter_min
+@@scatter_max
 @@sparse_mask
 @@tables_initializer
 @@trainable_variables
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 55b82dd765..937044aece 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -1688,6 +1688,14 @@ tf_module {
     name: "scatter_div"
     argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
   }
+  member_method {
+    name: "scatter_max"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
+  member_method {
+    name: "scatter_min"
+    argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
+  }
   member_method {
     name: "scatter_mul"
     argspec: "args=[\'ref\', \'indices\', \'updates\', \'use_locking\', \'name\'], varargs=None, keywords=None, defaults=[\'False\', \'None\'], "
-- 
GitLab


From dd3adb6165605c28f1a993f9093e8f7c99b357c5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 16:13:13 -0700
Subject: [PATCH 457/960] [XLA] Redesign: implement local client and local
 service interface.

PiperOrigin-RevId: 190291400
---
 .../compiler/xla/client/local_client.cc       |  18 +++
 tensorflow/compiler/xla/client/local_client.h |   9 +-
 tensorflow/compiler/xla/service/BUILD         |   1 +
 .../compiler/xla/service/local_service.cc     | 153 ++++++++++++++----
 .../compiler/xla/service/local_service.h      |  13 ++
 tensorflow/compiler/xla/service/service.cc    |  41 +++--
 tensorflow/compiler/xla/service/service.h     |  11 ++
 7 files changed, 205 insertions(+), 41 deletions(-)

diff --git a/tensorflow/compiler/xla/client/local_client.cc b/tensorflow/compiler/xla/client/local_client.cc
index 91396f055f..30594243dc 100644
--- a/tensorflow/compiler/xla/client/local_client.cc
+++ b/tensorflow/compiler/xla/client/local_client.cc
@@ -265,6 +265,24 @@ StatusOr<std::unique_ptr<LocalExecutable>> LocalClient::Compile(
                                         updated_options));
 }
 
+StatusOr<std::unique_ptr<LocalExecutable>> LocalClient::Compile(
+    const XlaComputation& computation,
+    const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
+    const ExecutableBuildOptions& options) {
+  ExecutableBuildOptions updated_options = options;
+  if (options.device_ordinal() == -1) {
+    updated_options.set_device_ordinal(default_device_ordinal());
+    VLOG(3) << "Set device ordinal to default value of: "
+            << updated_options.device_ordinal();
+  }
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<Executable> executable,
+                      local_service_->CompileExecutable(
+                          computation, argument_layouts, updated_options));
+  return WrapUnique(new LocalExecutable(std::move(executable),
+                                        local_service_->mutable_backend(),
+                                        updated_options));
+}
+
 StatusOr<std::unique_ptr<ScopedShapedBuffer>>
 LocalClient::LiteralToShapedBuffer(const Literal& literal, int device_ordinal,
                                    DeviceMemoryAllocator* allocator) {
diff --git a/tensorflow/compiler/xla/client/local_client.h b/tensorflow/compiler/xla/client/local_client.h
index 2e5d85ba68..98ee7c62c9 100644
--- a/tensorflow/compiler/xla/client/local_client.h
+++ b/tensorflow/compiler/xla/client/local_client.h
@@ -123,7 +123,14 @@ class LocalClient : public Client {
       const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
       const ExecutableBuildOptions& options);
 
-  // TODO(b/74197823): Add a overload of Compile for XlaComputation.
+  // Build and return a LocalExecutable object. The executable is compiled using
+  // the given XlaComputation, argument layouts and options.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<LocalExecutable>> Compile(
+      const XlaComputation& computation,
+      const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
+      const ExecutableBuildOptions& options);
 
   // Copy the literal data to the device with the given ordinal and return as a
   // ScopedShapedBuffer. If non-null the given memory allocator is used for
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index d4d67872cf..da16976d06 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -623,6 +623,7 @@ cc_library(
         "//tensorflow/compiler/xla:util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:executable_build_options",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/core:lib",
         "//tensorflow/core:stream_executor_no_cuda",
     ],
diff --git a/tensorflow/compiler/xla/service/local_service.cc b/tensorflow/compiler/xla/service/local_service.cc
index 1e2d8eea58..499f280211 100644
--- a/tensorflow/compiler/xla/service/local_service.cc
+++ b/tensorflow/compiler/xla/service/local_service.cc
@@ -69,6 +69,68 @@ LocalService::LocalService(const ServiceOptions& options,
                            std::unique_ptr<Backend> execute_backend)
     : Service(options, std::move(execute_backend)) {}
 
+namespace {
+
+// Retrieves the parameter metadata for the given computation and parameter
+// number.
+//
+// If the parameter number is invalid for this computation, nullopt is
+// returned. When the return value has_value(), nullptr will never be
+// the held value.
+tensorflow::gtl::optional<const OpMetadata*> ParameterMetadata(
+    const XlaComputation& computation, int parameter_number) {
+  for (const HloComputationProto& comp : computation.proto().computations()) {
+    if (comp.id() == computation.proto().entry_computation_id()) {
+      for (const HloInstructionProto& instr : comp.instructions()) {
+        if (instr.opcode() == HloOpcodeString(HloOpcode::kParameter) &&
+            instr.parameter_number() == parameter_number) {
+          if (!instr.has_metadata()) {
+            return tensorflow::gtl::nullopt;
+          }
+          return &instr.metadata();
+        }
+      }
+    }
+  }
+  return tensorflow::gtl::nullopt;
+}
+
+ExecutionOptions CreateExecutionOptions(
+    const ExecutableBuildOptions& build_options,
+    const ProgramShape* program_shape) {
+  ExecutionOptions execution_options = CreateDefaultExecutionOptions();
+  if (build_options.hlo_profile().has_value()) {
+    execution_options.mutable_debug_options()->set_xla_hlo_profile(
+        *build_options.hlo_profile());
+  }
+  if (build_options.generate_hlo_graph().has_value()) {
+    execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
+        build_options.generate_hlo_graph().value());
+  }
+  if (build_options.dump_optimized_hlo_proto_to().has_value()) {
+    execution_options.mutable_debug_options()
+        ->set_xla_dump_optimized_hlo_proto_to(
+            build_options.dump_optimized_hlo_proto_to().value());
+  }
+  if (build_options.dump_per_pass_hlo_proto_to().has_value()) {
+    execution_options.mutable_debug_options()
+        ->set_xla_dump_per_pass_hlo_proto_to(
+            build_options.dump_per_pass_hlo_proto_to().value());
+  }
+  if (build_options.result_layout() != nullptr) {
+    *execution_options.mutable_shape_with_output_layout() =
+        *build_options.result_layout();
+  } else {
+    *execution_options.mutable_shape_with_output_layout() =
+        program_shape->result();
+    LayoutUtil::SetToDefaultLayout(
+        execution_options.mutable_shape_with_output_layout());
+  }
+  return execution_options;
+}
+
+}  // namespace
+
 StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
     const ComputationHandle& computation,
     const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
@@ -118,34 +180,8 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
         *build_options.result_layout(), program_shape->result()));
   }
 
-  ExecutionOptions execution_options = CreateDefaultExecutionOptions();
-  if (build_options.hlo_profile().has_value()) {
-    execution_options.mutable_debug_options()->set_xla_hlo_profile(
-        *build_options.hlo_profile());
-  }
-  if (build_options.generate_hlo_graph().has_value()) {
-    execution_options.mutable_debug_options()->set_xla_generate_hlo_graph(
-        build_options.generate_hlo_graph().value());
-  }
-  if (build_options.dump_optimized_hlo_proto_to().has_value()) {
-    execution_options.mutable_debug_options()
-        ->set_xla_dump_optimized_hlo_proto_to(
-            build_options.dump_optimized_hlo_proto_to().value());
-  }
-  if (build_options.dump_per_pass_hlo_proto_to().has_value()) {
-    execution_options.mutable_debug_options()
-        ->set_xla_dump_per_pass_hlo_proto_to(
-            build_options.dump_per_pass_hlo_proto_to().value());
-  }
-  if (build_options.result_layout() != nullptr) {
-    *execution_options.mutable_shape_with_output_layout() =
-        *build_options.result_layout();
-  } else {
-    *execution_options.mutable_shape_with_output_layout() =
-        program_shape->result();
-    LayoutUtil::SetToDefaultLayout(
-        execution_options.mutable_shape_with_output_layout());
-  }
+  ExecutionOptions execution_options =
+      CreateExecutionOptions(build_options, program_shape.get());
   TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModuleConfig> module_config,
                       CreateModuleConfig(*program_shape, argument_layouts,
                                          &execution_options, user_computation));
@@ -159,6 +195,67 @@ StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
                          build_options.device_allocator());
 }
 
+StatusOr<std::unique_ptr<Executable>> LocalService::CompileExecutable(
+    const XlaComputation& computation,
+    const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
+    const ExecutableBuildOptions& build_options) {
+  const HloModuleProto& proto = computation.proto();
+  TF_RET_CHECK(proto.has_program_shape());
+  const ProgramShape& program_shape = proto.program_shape();
+
+  // Validate incoming layouts.
+  if (argument_layouts.size() != program_shape.parameters_size()) {
+    return InvalidArgument(
+        "Invalid number of arguments for computation: expected %d, got %zu.",
+        program_shape.parameters_size(), argument_layouts.size());
+  }
+
+  for (int i = 0; i < argument_layouts.size(); ++i) {
+    const Shape& argument_shape = *argument_layouts[i];
+    TF_RETURN_IF_ERROR(ShapeUtil::ValidateShape(argument_shape));
+    if (!ShapeUtil::Compatible(argument_shape, program_shape.parameters(i))) {
+      tensorflow::gtl::optional<const OpMetadata*> metadata =
+          ParameterMetadata(computation, /*parameter_number=*/i);
+      auto metadata_string = [&metadata]() -> string {
+        if (!metadata.has_value()) {
+          return "";
+        }
+        CHECK(metadata.value() != nullptr);
+        const OpMetadata& m = *metadata.value();
+        if (!m.source_file().empty()) {
+          return tensorflow::strings::Printf(
+              " (%s:%d)", m.source_file().c_str(), m.source_line());
+        }
+        return "";
+      };
+      return InvalidArgument(
+          "Invalid argument shape for argument %d%s, expected %s, got %s.", i,
+          metadata_string().c_str(),
+          ShapeUtil::HumanString(program_shape.parameters(i)).c_str(),
+          ShapeUtil::HumanString(argument_shape).c_str());
+    }
+  }
+  if (build_options.result_layout() != nullptr) {
+    TF_RETURN_IF_ERROR(ValidateResultShapeWithLayout(
+        *build_options.result_layout(), program_shape.result()));
+  }
+
+  ExecutionOptions execution_options =
+      CreateExecutionOptions(build_options, &program_shape);
+
+  TF_ASSIGN_OR_RETURN(
+      std::unique_ptr<HloModuleConfig> module_config,
+      CreateModuleConfig(program_shape, argument_layouts, &execution_options));
+
+  TF_ASSIGN_OR_RETURN(
+      se::StreamExecutor * executor,
+      execute_backend_->stream_executor(build_options.device_ordinal()));
+
+  return BuildExecutable(proto, std::move(module_config),
+                         execute_backend_.get(), executor,
+                         build_options.device_allocator());
+}
+
 StatusOr<int> LocalService::ReplicaNumberToDeviceOrdinal(int replica_number) {
   return backend().computation_placer()->DeviceId(
       replica_number, /*computation=*/0, options_.number_of_replicas(),
diff --git a/tensorflow/compiler/xla/service/local_service.h b/tensorflow/compiler/xla/service/local_service.h
index 15e120685e..06567cabd6 100644
--- a/tensorflow/compiler/xla/service/local_service.h
+++ b/tensorflow/compiler/xla/service/local_service.h
@@ -19,6 +19,7 @@ limitations under the License.
 #include <memory>
 
 #include "tensorflow/compiler/xla/client/executable_build_options.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/service/backend.h"
 #include "tensorflow/compiler/xla/service/compiler.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
@@ -50,6 +51,18 @@ class LocalService : public Service {
       const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
       const ExecutableBuildOptions& options);
 
+  // Builds an Executable with the given XlaComputation, argument layouts and
+  // options. If result_layout is non-null, then the executable is compiled to
+  // produce a result of the given layout.  If device_allocator is non-null,
+  // then the compiler may use it to allocate temp space on the device.  The
+  // compiler is responsible for freeing any memory it allocates this way.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<Executable>> CompileExecutable(
+      const XlaComputation& computation,
+      const tensorflow::gtl::ArraySlice<const Shape*> argument_layouts,
+      const ExecutableBuildOptions& build_options);
+
   // Returns the device ordinal that corresponds to the given replica number.
   //
   // This returns an error if there is not a one-to-one correspondence of
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 4f6a82333b..1d379f0d03 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -963,6 +963,30 @@ tensorflow::Status Service::Execute(const ExecuteRequest* arg,
   return tensorflow::Status::OK();
 }
 
+StatusOr<std::unique_ptr<Executable>> Service::BuildExecutable(
+    const HloModuleProto& module_proto,
+    std::unique_ptr<HloModuleConfig> module_config, Backend* backend,
+    se::StreamExecutor* executor, DeviceMemoryAllocator* device_allocator) {
+  VLOG(1) << Printf(
+      "BuildExecutable on service %p with serialized module proto: %s", this,
+      module_proto.name().c_str());
+
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
+                      HloModule::CreateFromProto(module_proto, *module_config));
+
+  TF_RETURN_IF_ERROR(MaybeDumpHloModule(*module));
+
+  TF_ASSIGN_OR_RETURN(
+      module, backend->compiler()->RunHloPasses(std::move(module), executor,
+                                                device_allocator));
+
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<Executable> executable,
+                      backend->compiler()->RunBackend(
+                          std::move(module), executor, device_allocator));
+
+  return std::move(executable);
+}
+
 tensorflow::Status Service::ExecuteGraph(const ExecuteGraphRequest* arg,
                                          ExecuteResponse* result) {
   VLOG(1) << "running execute-graph request";
@@ -979,24 +1003,17 @@ tensorflow::Status Service::ExecuteGraph(const ExecuteGraphRequest* arg,
       std::vector<std::vector<const ShapedBuffer*>> replicated_arguments,
       ResolveAndValidateArguments(arg->arguments(), replicas));
 
-  TF_ASSIGN_OR_RETURN(const auto& config,
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModuleConfig> module_config,
                       CreateModuleConfig(arg->computation().program_shape(),
                                          replicated_arguments.front(),
                                          arg->execution_options()));
 
-  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
-                      HloModule::CreateFromProto(arg->computation(), *config));
-  TF_RETURN_IF_ERROR(MaybeDumpHloModule(*module));
-
-  TF_ASSIGN_OR_RETURN(module, execute_backend_->compiler()->RunHloPasses(
-                                  std::move(module),
-                                  execute_backend_->default_stream_executor(),
-                                  /*device_allocator=*/nullptr));
   TF_ASSIGN_OR_RETURN(
       std::unique_ptr<Executable> executable,
-      execute_backend_->compiler()->RunBackend(
-          std::move(module), execute_backend_->default_stream_executor(),
-          /*device_allocator=*/nullptr));
+      BuildExecutable(arg->computation(), std::move(module_config),
+                      execute_backend_.get(),
+                      execute_backend_->default_stream_executor(),
+                      /*device_allocator=*/nullptr));
 
   TF_ASSIGN_OR_RETURN(
       *result->mutable_output(),
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index 3b79920b0a..773f0a642d 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -115,6 +115,8 @@ class Service : public ServiceInterface {
   // Executes a computation with the provided global data passed as
   // immutable arguments. The request contains the whole computation graph.
   // Returns global data output and execution timing.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
   tensorflow::Status ExecuteGraph(const ExecuteGraphRequest* arg,
                                   ExecuteResponse* result) override;
 
@@ -299,6 +301,15 @@ class Service : public ServiceInterface {
       perftools::gputools::StreamExecutor* executor,
       DeviceMemoryAllocator* device_allocator = nullptr);
 
+  // Builds an Executable for the given HLO module proto.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<Executable>> BuildExecutable(
+      const HloModuleProto& module_proto,
+      std::unique_ptr<HloModuleConfig> module_config, Backend* backend,
+      perftools::gputools::StreamExecutor* executor,
+      DeviceMemoryAllocator* device_allocator = nullptr);
+
   // Same as BuildExecutable() above, but builds a list of Executables for the
   // given computations that may interact with each other.
   StatusOr<std::vector<std::unique_ptr<Executable>>> BuildExecutables(
-- 
GitLab


From f54f57337078c93877df5c9a1b126e879f5b33a5 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Fri, 23 Mar 2018 16:15:55 -0700
Subject: [PATCH 458/960] Moves TensorHandleCopyToDevice to
 TensorHandle::CopyToDevice.

PiperOrigin-RevId: 190291768
---
 tensorflow/c/eager/BUILD                      |   1 +
 tensorflow/c/eager/c_api.cc                   | 125 +-----------------
 tensorflow/core/common_runtime/eager/BUILD    |  20 +++
 .../eager/copy_to_device_node.h               |  69 ++++++++++
 .../common_runtime/eager/tensor_handle.cc     |  71 ++++++++++
 .../core/common_runtime/eager/tensor_handle.h |   3 +
 6 files changed, 168 insertions(+), 121 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/copy_to_device_node.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index d2d8d59323..8df7b56623 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -32,6 +32,7 @@ tf_cuda_library(
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core/common_runtime/eager:tensor_handle",
+            "//tensorflow/core/common_runtime/eager:copy_to_device_node",
             "//tensorflow/core:core_cpu_internal",
             "//tensorflow/core:framework",
             "//tensorflow/core:framework_internal",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 59432f2ef8..c69635d529 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
+#include "tensorflow/core/common_runtime/eager/copy_to_device_node.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -213,82 +214,6 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
 }
 }  // extern "C"
 
-namespace {
-
-tensorflow::Status TensorHandleCopyToDevice(tensorflow::TensorHandle* h,
-                                            TFE_Context* ctx,
-                                            tensorflow::Device* dstd,
-                                            tensorflow::TensorHandle** output) {
-  const tensorflow::Tensor* src = nullptr;
-  tensorflow::Device* srcd = nullptr;
-  // TODO(agarwal): src_opd is unused. Perhaps allow TensorAndDevice to accept
-  // nullptr.
-  tensorflow::Device* src_opd = nullptr;
-  TF_RETURN_IF_ERROR(h->TensorAndDevice(&src, &srcd, &src_opd));
-  if (srcd == nullptr) srcd = ctx->context.HostCPU();
-  bool is_same_device =
-      (srcd == dstd) || (DeviceName(srcd) == DeviceName(dstd));
-  const bool dst_cpu = IsCPU(dstd);
-  const bool src_cpu = IsCPU(srcd);
-  // both_on_cpu can be true and yet is_same_device is false, if one of src/dst
-  // has device type XLA_CPU, and the other CPU.
-  const bool both_on_cpu = src_cpu && dst_cpu;
-  if (is_same_device || both_on_cpu) {
-    dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(*src, dstd, dstd);
-    return tensorflow::Status::OK();
-  }
-  if (!dst_cpu && (src->dtype() != tensorflow::DT_VARIANT &&
-                   !tensorflow::DataTypeCanUseMemcpy(src->dtype()))) {
-    return tensorflow::errors::InvalidArgument(
-        "Can't copy Tensor with type ",
-        tensorflow::DataTypeString(src->dtype()), " to device ",
-        DeviceName(dstd), ".");
-  }
-  tensorflow::AllocatorAttributes attr;
-  if (src->dtype() == tensorflow::DT_VARIANT) {
-    attr.set_on_host(true);
-  }
-  tensorflow::Tensor dst(dstd->GetAllocator(attr), src->dtype(), src->shape());
-  if (src->shape().num_elements() == 0) {
-    dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
-    return tensorflow::Status::OK();
-  }
-  tensorflow::DeviceContext* src_device_context = nullptr;
-  if (!src_cpu) {
-    src_device_context = srcd->tensorflow_gpu_device_info()->default_context;
-  }
-  tensorflow::DeviceContext* dst_device_context = nullptr;
-  if (!dst_cpu) {
-    dst_device_context = dstd->tensorflow_gpu_device_info()->default_context;
-  }
-  // TODO(ashankar): The Sync() call below may be more aggressive than
-  // necessary. It is based on knowledge of implementation details - that
-  // GPU devices are implemented using 3 streams - one for host->device copies,
-  // one for device->host copies and one for sending operations to the GPU.
-  // With that setup, Sync()ing across all 3 streams should be sufficient
-  // but more than necessary (since it waits for operations that might have
-  // nothing to do with this tensor to complete).
-  TF_RETURN_IF_ERROR(srcd->Sync());
-  tensorflow::Notification n;
-  tensorflow::Status status;
-  tensorflow::CopyTensor::ViaDMA("copy", src_device_context, dst_device_context,
-                                 srcd, dstd, tensorflow::AllocatorAttributes(),
-                                 tensorflow::AllocatorAttributes(), src, &dst,
-                                 [&status, &n](const tensorflow::Status& s) {
-                                   status = s;
-                                   n.Notify();
-                                 });
-  n.WaitForNotification();
-  if (status.ok()) {
-    dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
-  }
-  return status;
-}
-}  // namespace
-
 extern "C" {
 
 TFE_Op* TFE_NewOp(TFE_Context* ctx, const char* op_or_function_name,
@@ -509,49 +434,6 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 namespace {
 
-class CopyToDeviceNode : public tensorflow::EagerNode {
- public:
-  CopyToDeviceNode(tensorflow::TensorHandle* src, tensorflow::Device* dstd,
-                   TFE_Context* ctx)
-      : tensorflow::EagerNode(ctx->context.NextId()),
-        src_(src),
-        dstd_(dstd),
-        ctx_(ctx),
-        dst_(new tensorflow::TensorHandle(id, src_->dtype, &ctx->context)) {
-    src_->Ref();
-    dst_->Ref();
-  }
-
-  ~CopyToDeviceNode() override {
-    src_->Unref();
-    dst_->Unref();
-  }
-
-  tensorflow::Status Run() override {
-    tensorflow::TensorHandle* temp = nullptr;
-    TF_RETURN_IF_ERROR(TensorHandleCopyToDevice(src_, ctx_, dstd_, &temp));
-    const tensorflow::Tensor* tensor = nullptr;
-    tensorflow::Device* device = nullptr;
-    tensorflow::Device* op_device = nullptr;
-    tensorflow::Status status =
-        temp->TensorAndDevice(&tensor, &device, &op_device);
-    // `temp` is a ready handle. So the following call should return OK.
-    TF_DCHECK_OK(status) << status.error_message();
-    DCHECK(tensor);
-    dst_->SetTensorAndDevice(*tensor, device, op_device);
-    temp->Unref();
-    return tensorflow::Status::OK();
-  }
-
-  tensorflow::TensorHandle* dst() { return dst_; }
-
- private:
-  tensorflow::TensorHandle* src_;
-  tensorflow::Device* dstd_;
-  TFE_Context* ctx_;
-  tensorflow::TensorHandle* dst_;
-};
-
 // TODO(apassos) move to TensorHandle
 tensorflow::TensorHandle* TFE_TensorHandleCopyToDevice_Internal(
     tensorflow::TensorHandle* h, TFE_Context* ctx, const char* device_name,
@@ -569,7 +451,8 @@ tensorflow::TensorHandle* TFE_TensorHandleCopyToDevice_Internal(
   if (ctx->context.Async()) {
     // Note that `h` may not be currently ready. However execution order will
     // make sure that `h` is ready before the copy is actually done.
-    CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
+    tensorflow::CopyToDeviceNode* node =
+        new tensorflow::CopyToDeviceNode(h, dstd, &ctx->context);
     tensorflow::TensorHandle* output = node->dst();
     // Note that calling Add makes `node` accessible by the EagerExecutor
     // thread. So further accesses need to be thread-safe.
@@ -577,7 +460,7 @@ tensorflow::TensorHandle* TFE_TensorHandleCopyToDevice_Internal(
     return output;
   } else {
     tensorflow::TensorHandle* output = nullptr;
-    status->status = TensorHandleCopyToDevice(h, ctx, dstd, &output);
+    status->status = h->CopyToDevice(&ctx->context, dstd, &output);
     return output;
   }
 }
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 02fb83200a..a619cac9a4 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -77,6 +77,26 @@ tf_cuda_library(
     ],
 )
 
+tf_cuda_library(
+    name = "copy_to_device_node",
+    hdrs = [
+        "copy_to_device_node.h",
+    ],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":context",
+        ":eager_executor",
+        ":tensor_handle",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:session_options",
+    ],
+)
+
 tf_cuda_library(
     name = "kernel_and_device",
     srcs = [
diff --git a/tensorflow/core/common_runtime/eager/copy_to_device_node.h b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
new file mode 100644
index 0000000000..8a887540b0
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/copy_to_device_node.h
@@ -0,0 +1,69 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_COPY_TO_DEVICE_NODE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_COPY_TO_DEVICE_NODE_H_
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+class CopyToDeviceNode : public EagerNode {
+ public:
+  CopyToDeviceNode(TensorHandle* src, Device* dstd, EagerContext* ctx)
+      : EagerNode(ctx->NextId()),
+        src_(src),
+        dstd_(dstd),
+        ctx_(ctx),
+        dst_(new TensorHandle(id, src_->dtype, ctx)) {
+    src_->Ref();
+    dst_->Ref();
+  }
+
+  ~CopyToDeviceNode() override {
+    src_->Unref();
+    dst_->Unref();
+  }
+
+  Status Run() override {
+    TensorHandle* temp = nullptr;
+    TF_RETURN_IF_ERROR(src_->CopyToDevice(ctx_, dstd_, &temp));
+    const Tensor* tensor = nullptr;
+    Device* device = nullptr;
+    Device* op_device = nullptr;
+    Status status = temp->TensorAndDevice(&tensor, &device, &op_device);
+    // `temp` is a ready handle. So the following call should return OK.
+    TF_DCHECK_OK(status) << status.error_message();
+    DCHECK(tensor);
+    dst_->SetTensorAndDevice(*tensor, device, op_device);
+    temp->Unref();
+    return Status::OK();
+  }
+
+  TensorHandle* dst() { return dst_; }
+
+ private:
+  TensorHandle* src_;
+  Device* dstd_;
+  EagerContext* ctx_;
+  TensorHandle* dst_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_COPY_TO_DEVICE_NODE_H_
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 5bc1700627..328cd5dd5c 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include <string>
 #include <vector>
 
+#include "tensorflow/core/common_runtime/copy_tensor.h"
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
@@ -104,4 +105,74 @@ void TensorHandle::SetTensorAndDevice(const tensorflow::Tensor& tensor,
   op_device_ = op_device;
 }
 
+Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
+                                  TensorHandle** output) {
+  const tensorflow::Tensor* src = nullptr;
+  tensorflow::Device* srcd = nullptr;
+  // TODO(agarwal): src_opd is unused. Perhaps allow TensorAndDevice to accept
+  // nullptr.
+  tensorflow::Device* src_opd = nullptr;
+  TF_RETURN_IF_ERROR(TensorAndDevice(&src, &srcd, &src_opd));
+  if (srcd == nullptr) srcd = ctx->HostCPU();
+  bool is_same_device = (srcd == dstd) || (srcd->name() == dstd->name());
+  const bool dst_cpu = dstd->tensorflow_gpu_device_info() == nullptr;
+  const bool src_cpu = srcd->tensorflow_gpu_device_info() == nullptr;
+  // both_on_cpu can be true and yet is_same_device is false, if one of src/dst
+  // has device type XLA_CPU, and the other CPU.
+  const bool both_on_cpu = src_cpu && dst_cpu;
+  if (is_same_device || both_on_cpu) {
+    dstd = dst_cpu ? nullptr : dstd;
+    *output = new tensorflow::TensorHandle(*src, dstd, dstd);
+    return tensorflow::Status::OK();
+  }
+  if (!dst_cpu && (src->dtype() != tensorflow::DT_VARIANT &&
+                   !tensorflow::DataTypeCanUseMemcpy(src->dtype()))) {
+    return tensorflow::errors::InvalidArgument(
+        "Can't copy Tensor with type ",
+        tensorflow::DataTypeString(src->dtype()), " to device ", dstd->name(),
+        ".");
+  }
+  tensorflow::AllocatorAttributes attr;
+  if (src->dtype() == tensorflow::DT_VARIANT) {
+    attr.set_on_host(true);
+  }
+  tensorflow::Tensor dst(dstd->GetAllocator(attr), src->dtype(), src->shape());
+  if (src->shape().num_elements() == 0) {
+    dstd = dst_cpu ? nullptr : dstd;
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
+    return tensorflow::Status::OK();
+  }
+  tensorflow::DeviceContext* src_device_context = nullptr;
+  if (!src_cpu) {
+    src_device_context = srcd->tensorflow_gpu_device_info()->default_context;
+  }
+  tensorflow::DeviceContext* dst_device_context = nullptr;
+  if (!dst_cpu) {
+    dst_device_context = dstd->tensorflow_gpu_device_info()->default_context;
+  }
+  // TODO(ashankar): The Sync() call below may be more aggressive than
+  // necessary. It is based on knowledge of implementation details - that
+  // GPU devices are implemented using 3 streams - one for host->device copies,
+  // one for device->host copies and one for sending operations to the GPU.
+  // With that setup, Sync()ing across all 3 streams should be sufficient
+  // but more than necessary (since it waits for operations that might have
+  // nothing to do with this tensor to complete).
+  TF_RETURN_IF_ERROR(srcd->Sync());
+  tensorflow::Notification n;
+  tensorflow::Status status;
+  tensorflow::CopyTensor::ViaDMA("copy", src_device_context, dst_device_context,
+                                 srcd, dstd, tensorflow::AllocatorAttributes(),
+                                 tensorflow::AllocatorAttributes(), src, &dst,
+                                 [&status, &n](const tensorflow::Status& s) {
+                                   status = s;
+                                   n.Notify();
+                                 });
+  n.WaitForNotification();
+  if (status.ok()) {
+    dstd = dst_cpu ? nullptr : dstd;
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
+  }
+  return status;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index 97e67e4652..eb69a13c06 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -85,6 +85,9 @@ class TensorHandle : public core::RefCounted {
                           tensorflow::Device* device,
                           tensorflow::Device* op_device);
 
+  Status CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
+                      TensorHandle** output);
+
   // dtype for the handle. It must be the same as t.dtype() once the handle is
   // ready.
   const DataType dtype;
-- 
GitLab


From 97249979d9a76ae05d590f9cbe199c0b47712b4f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 16:16:22 -0700
Subject: [PATCH 459/960] bug fix: evaluate nodes before swap the original
 graph

PiperOrigin-RevId: 190291844
---
 tensorflow/core/grappler/optimizers/constant_folding_test.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 914a9257ee..6340565bcd 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1922,6 +1922,8 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
   item.fetch = {"concat0", "concat1", "concat2", "concat3", "concat4",
                 "concat5", "concat6", "concat7", "concat8", "concat9"};
 
+  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"});
+  EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -1971,9 +1973,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
     }
   }
 
-  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"});
   auto tensors = EvaluateNodes(output, {"concat0"});
-  EXPECT_EQ(1, tensors_expected.size());
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
-- 
GitLab


From 202e4f3b3699e8e40e478402462f76ae853fecbf Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 23 Mar 2018 16:28:16 -0700
Subject: [PATCH 460/960] Make _USE_C_API = True and _USE_C_SHAPES = False work
 with handle data.

This change makes _set_shapes_for_outputs_c_api fetch and set
Tensor._handle_data. This is necessary for running the
Python shape inference code on resource tensors.

PiperOrigin-RevId: 190293303
---
 tensorflow/c/BUILD                           |  2 ++
 tensorflow/c/python_api.cc                   | 26 +++++++++++++++
 tensorflow/c/python_api.h                    |  7 ++++
 tensorflow/python/BUILD                      |  2 ++
 tensorflow/python/client/tf_session.i        |  1 +
 tensorflow/python/framework/importer_test.py | 34 ++++++++++++++++++++
 tensorflow/python/framework/ops.py           |  9 ++++++
 7 files changed, 81 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index d096647558..f4a486d330 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -279,6 +279,8 @@ tf_cuda_library(
     deps = [
         ":c_api",
         ":c_api_internal",
+        # TODO(b/74620627): remove when _USE_C_SHAPES is removed
+        "//tensorflow/python:cpp_shape_inference_proto_cc",
     ],
 )
 
diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc
index cd604538f1..93155998b8 100644
--- a/tensorflow/c/python_api.cc
+++ b/tensorflow/c/python_api.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/python_api.h"
 
 #include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/python/framework/cpp_shape_inference.pb.h"
 
 namespace tensorflow {
 
@@ -109,4 +110,29 @@ void ExtendSession(TF_Session* session, TF_Status* status) {
   session->extend_before_run = false;
 }
 
+std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output) {
+  Node* node = &output.oper->node;
+  CppShapeInferenceResult::HandleData handle_data;
+  handle_data.set_is_set(true);
+  {
+    mutex_lock l(graph->mu);
+    tensorflow::shape_inference::InferenceContext* ic =
+        graph->refiner.GetContext(node);
+    CHECK(ic != nullptr);
+    CHECK_LT(output.index, ic->num_outputs());
+    const auto* shapes_and_types =
+        ic->output_handle_shapes_and_types(output.index);
+    if (shapes_and_types == nullptr) return "";
+
+    for (const auto& p : *shapes_and_types) {
+      auto* out_shape_and_type = handle_data.add_shape_and_type();
+      ic->ShapeHandleToProto(p.shape, out_shape_and_type->mutable_shape());
+      out_shape_and_type->set_dtype(p.dtype);
+    }
+  }
+  string result;
+  handle_data.SerializeToString(&result);
+  return result;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/c/python_api.h b/tensorflow/c/python_api.h
index 13b680b3a2..2d4c8cd9ed 100644
--- a/tensorflow/c/python_api.h
+++ b/tensorflow/c/python_api.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_C_PYTHON_API_H_
 #define TENSORFLOW_C_PYTHON_API_H_
 
+#include <string>
+
 #include "tensorflow/c/c_api.h"
 
 // These functions can be removed without notice. They exist to facilitate some
@@ -51,6 +53,11 @@ void SetRequireShapeInferenceFns(TF_Graph* graph, bool require);
 // the graph after the session has been made aware of them.
 void ExtendSession(TF_Session* session, TF_Status* status);
 
+// Returns the serialized CppShapeInferenceResult::HandleData proto for
+// `output` if its a resource tensor, or otherwise returns the empty string.
+// TODO(b/74620627): remove when _USE_C_SHAPES is removed
+std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_C_PYTHON_API_H_
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 0e2b980213..acfdcd15f7 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3128,6 +3128,8 @@ tf_proto_library(
     srcs = ["framework/cpp_shape_inference.proto"],
     cc_api_version = 2,
     protodeps = tf_additional_all_protos(),
+    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
+    visibility = ["//tensorflow:internal"],
 )
 
 py_test(
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index e88fc0c01a..70a3d032f4 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -723,6 +723,7 @@ def TF_Reset(target, containers=None, config=None):
 %unignore TF_TryEvaluateConstant_wrapper;
 %noexception TF_TryEvaluateConstant_wrapper;
 %unignore ExtendSession;
+%unignore ResourceHandleShapeAndType;
 
 %include "tensorflow/python/client/tf_session_helper.h"
 
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 6593b17184..369669c2e6 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variables
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
@@ -356,6 +357,39 @@ class ImportGraphDefTest(test.TestCase):
       self.assertEqual(d._input_types, [dtypes.int32_ref, dtypes.int32])
       self.assertEqual(d.outputs, [])
 
+  def testResources(self):
+    # Produce GraphDef containing a ops producing and consuming resources.
+    graph = ops.Graph()
+    with graph.as_default():
+      var = resource_variable_ops.ResourceVariable(1.0)
+      var_assign = var.assign(2.0)
+      # Use an op that requires handle shape to be set.
+      var_shape = resource_variable_ops.variable_shape(var.handle)
+      init = variables.global_variables_initializer()
+    graph_def = graph.as_graph_def()
+
+    # Import the GraphDef.
+    with ops.Graph().as_default():
+      # pylint: disable=unused-variable
+      imported_var, imported_assign, imported_shape, imported_init = (
+          importer.import_graph_def(
+              graph_def,
+              return_elements=[var.name, var_assign.name, var_shape.name,
+                               init.name]))
+
+      # Make sure the handle shape is set on the imported variable.
+      new_var_shape = resource_variable_ops.variable_shape(imported_var)
+      # pylint: enable=unused-variable
+
+      # Run the imported graph.
+      # TODO(b/76173421): make this work (currently DCHECKS)
+      # with self.test_session() as sess:
+      #   sess.run(imported_init)
+      #   self.assertEqual(sess.run(imported_var), 1.0)
+      #   self.assertEqual(sess.run(imported_assign), 2.0)
+      #   self.assertEqual(list(sess.run(imported_shape)), [])
+      #   self.assertEqual(list(sess.run(new_var_shape)), [])
+
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
     graph = ops.Graph()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 93edaa0cf0..1fa9285e43 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -42,6 +42,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -295,6 +296,7 @@ class Tensor(_TensorLike):
 
     # Attributes used for C++ shape inference. Not inspected, only forwarded.
     # If set, will be a HandleData object from cpp_shape_inference.proto.
+    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
     self._handle_data = None
     self._id = uid()
 
@@ -2472,6 +2474,13 @@ def _set_shapes_for_outputs_c_api(op):
       shape_vector = [None if d == -1 else d for d in shape_vector]
       output.set_shape(tensor_shape.TensorShape(shape_vector))
 
+    serialized = c_api.ResourceHandleShapeAndType(op._graph._c_graph,
+                                                  output._as_tf_output())
+    if serialized:
+      output._handle_data = (cpp_shape_inference_pb2.CppShapeInferenceResult
+                             .HandleData.FromString(serialized.encode()))
+    else:
+      output._handle_data = None
 
 # TODO(skyewm): remove this when _USE_C_API flag is removed.
 def _set_shapes_for_outputs(op):
-- 
GitLab


From 6e523342d57b175e698bb8379979104e3e0335ac Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 17:19:10 -0700
Subject: [PATCH 461/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 190299240
---
 tensorflow/contrib/estimator/BUILD            |   2 +-
 .../core/ops/compat/ops_history.v1.pbtxt      | 359 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 359 ++++++++++++++++++
 3 files changed, 719 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 24374266dc..c846343d6d 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -358,7 +358,7 @@ cuda_py_test(
     size = "medium",
     srcs = ["python/estimator/replicate_model_fn_test.py"],
     additional_deps = [
-        "//third_party/py/absl/testing:parameterized",
+        "@absl_py//absl/testing:parameterized",
         "//tensorflow/python/estimator",
         "//tensorflow/python/estimator:dnn",
         "//tensorflow/python/estimator:export_export",
diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index b41826d6eb..05d6e02281 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -43705,6 +43705,210 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceScatterDiv"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMax"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMin"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMul"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceScatterNdUpdate"
   input_arg {
@@ -43742,6 +43946,57 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceScatterSub"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceScatterUpdate"
   input_arg {
@@ -48901,6 +49156,110 @@ op {
     }
   }
 }
+op {
+  name: "ScatterMax"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterMin"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ScatterMul"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index af2c563489..274a7fbf75 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -21658,6 +21658,210 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceScatterDiv"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMax"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMin"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "ResourceScatterMul"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceScatterNdUpdate"
   input_arg {
@@ -21695,6 +21899,57 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "ResourceScatterSub"
+  input_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "dtype"
+  }
+  attr {
+    name: "dtype"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_UINT8
+        type: DT_INT16
+        type: DT_INT8
+        type: DT_COMPLEX64
+        type: DT_INT64
+        type: DT_QINT8
+        type: DT_QUINT8
+        type: DT_QINT32
+        type: DT_BFLOAT16
+        type: DT_UINT16
+        type: DT_COMPLEX128
+        type: DT_HALF
+        type: DT_UINT32
+        type: DT_UINT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  is_stateful: true
+}
 op {
   name: "ResourceScatterUpdate"
   input_arg {
@@ -23434,6 +23689,110 @@ op {
     }
   }
 }
+op {
+  name: "ScatterMax"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
+op {
+  name: "ScatterMin"
+  input_arg {
+    name: "ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  input_arg {
+    name: "indices"
+    type_attr: "Tindices"
+  }
+  input_arg {
+    name: "updates"
+    type_attr: "T"
+  }
+  output_arg {
+    name: "output_ref"
+    type_attr: "T"
+    is_ref: true
+  }
+  attr {
+    name: "T"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_HALF
+        type: DT_BFLOAT16
+        type: DT_FLOAT
+        type: DT_DOUBLE
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "Tindices"
+    type: "type"
+    allowed_values {
+      list {
+        type: DT_INT32
+        type: DT_INT64
+      }
+    }
+  }
+  attr {
+    name: "use_locking"
+    type: "bool"
+    default_value {
+      b: false
+    }
+  }
+}
 op {
   name: "ScatterMul"
   input_arg {
-- 
GitLab


From d40c53dd2cb7c0e3ec20ca56f5c3c95038820900 Mon Sep 17 00:00:00 2001
From: Guangda Lai <laigd@google.com>
Date: Fri, 23 Mar 2018 17:28:10 -0700
Subject: [PATCH 462/960] Set the stream in TransformTensor.

PiperOrigin-RevId: 190300166
---
 tensorflow/stream_executor/cuda/cuda_dnn.cc | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 03e3e0857f..ab5e6590e0 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -3157,12 +3157,18 @@ bool CudnnSupport::DoTransformTensor(Stream* stream,
                                      dnn::DataType output_type, float scale,
                                      DeviceMemoryBase* output_data) {
   mutex_lock lock{dnn_handle_mutex_};
+  cudnnStatus_t status = wrap::cudnnSetStream(parent_, ToHandle(dnn_handle_),
+                                              AsCUDAStreamValue(stream));
+  if (status != CUDNN_STATUS_SUCCESS) {
+    LOG(FATAL) << "failed to set stream for cudnn handle: " << ToString(status);
+  }
+
   float beta = 0.0f;
   ScopedTensorDescriptor input_tensor_desc(
       parent_, input_desc, ToCudnnDataType(input_type, input_desc.layout()));
   ScopedTensorDescriptor output_tensor_desc(
       parent_, output_desc, ToCudnnDataType(output_type, output_desc.layout()));
-  cudnnStatus_t status = wrap::cudnnTransformTensor(
+  status = wrap::cudnnTransformTensor(
       parent_, ToHandle(dnn_handle_), &scale, input_tensor_desc.handle(),
       input_data.opaque(), &beta, output_tensor_desc.handle(),
       output_data->opaque());
-- 
GitLab


From c275f2dffb7423328428553f2aafe3b011b48372 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 17:49:47 -0700
Subject: [PATCH 463/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 190302194

---
 tensorflow/go/op/wrappers.go | 3410 +++++++++++++++++-----------------
 1 file changed, 1705 insertions(+), 1705 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 5ddd32ed48..838f4f2301 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1089,184 +1089,190 @@ func ExpandDims(scope *Scope, input tf.Output, axis tf.Output) (output tf.Output
 	return op.Output(0)
 }
 
-// Returns (x - y)(x - y) element-wise.
+// A placeholder op that passes through `input` when its output is not fed.
 //
-// *NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func SquaredDifference(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Arguments:
+//	input: The default value to produce when `output` is not fed.
+//	shape: The (possibly partial) shape of the tensor.
+//
+// Returns A placeholder tensor that defaults to `input` if it is not fed.
+func PlaceholderWithDefault(scope *Scope, input tf.Output, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "SquaredDifference",
+		Type: "PlaceholderWithDefault",
 		Input: []tf.Input{
-			x, y,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Forwards the input to the output.
+// A placeholder op for a value that will be fed into the computation.
 //
-// This operator represents the loop termination condition used by the
-// "pivot" switches of a loop.
+// DEPRECATED at GraphDef version 23: Placeholder now behaves the same as PlaceholderV2.
+//
+// N.B. This operation will fail with an error if it is executed. It is
+// intended as a way to represent a value that will always be fed, and to
+// provide attrs that enable the fed value to be checked at runtime.
 //
 // Arguments:
-//	input: A boolean scalar, representing the branch predicate of the Switch op.
+//	dtype: The type of elements in the tensor.
+//	shape: The shape of the tensor. The shape can be any partially-specified
+// shape.  To be unconstrained, pass in a shape with unknown rank.
 //
-// Returns The same tensor as `input`.
-func LoopCond(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns A placeholder tensor that must be replaced using the feed mechanism.
+func PlaceholderV2(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
 	opspec := tf.OpSpec{
-		Type: "LoopCond",
-		Input: []tf.Input{
-			input,
-		},
+		Type: "PlaceholderV2",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedMulAttr is an optional argument to QuantizedMul.
-type QuantizedMulAttr func(optionalAttr)
+// PlaceholderAttr is an optional argument to Placeholder.
+type PlaceholderAttr func(optionalAttr)
 
-// QuantizedMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
+// PlaceholderShape sets the optional shape attribute to value.
+//
+// value: (Optional) The shape of the tensor. If the shape has 0 dimensions, the
+// shape is unconstrained.
+// If not specified, defaults to <unknown_rank:true >
+func PlaceholderShape(value tf.Shape) PlaceholderAttr {
 	return func(m optionalAttr) {
-		m["Toutput"] = value
+		m["shape"] = value
 	}
 }
 
-// Returns x * y element-wise, working on quantized buffers.
-//
-// Arguments:
-//
+// A placeholder op for a value that will be fed into the computation.
 //
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
+// N.B. This operation will fail with an error if it is executed. It is
+// intended as a way to represent a value that will always be fed, and to
+// provide attrs that enable the fed value to be checked at runtime.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+// Arguments:
+//	dtype: The type of elements in the tensor.
 //
-// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
+// Returns A placeholder tensor that must be replaced using the feed mechanism.
+func Placeholder(scope *Scope, dtype tf.DataType, optional ...PlaceholderAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMul",
-		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
-		},
+		Type: "Placeholder",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
-type QuantizedMatMulAttr func(optionalAttr)
-
-// QuantizedMatMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
+	return op.Output(0)
 }
 
-// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
+// Gradient op for `MirrorPad` op. This op folds a mirror-padded tensor.
 //
-// value: If true, `a` is transposed before multiplication.
-// If not specified, defaults to false
-func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
-	}
-}
-
-// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
+// This operation folds the padded areas of `input` by `MirrorPad` according to the
+// `paddings` you specify. `paddings` must be the same as `paddings` argument
+// given to the corresponding `MirrorPad` op.
 //
-// value: If true, `b` is transposed before multiplication.
-// If not specified, defaults to false
-func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_b"] = value
-	}
-}
-
-// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
+// The folded size of each dimension D of the output is:
 //
-// value: The type of output produced by activation function
-// following this operation.
-// If not specified, defaults to DT_QUINT8
-func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Tactivation"] = value
-	}
-}
-
-// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
+// `input.dim_size(D) - paddings(D, 0) - paddings(D, 1)`
 //
-// The inputs must be two-dimensional matrices and the inner dimension of
-// `a` (after being transposed if `transpose_a` is non-zero) must match the
-// outer dimension of `b` (after being transposed if `transposed_b` is
-// non-zero).
+// For example:
+//
+// ```
+// # 't' is [[1, 2, 3], [4, 5, 6], [7, 8, 9]].
+// # 'paddings' is [[0, 1]], [0, 1]].
+// # 'mode' is SYMMETRIC.
+// # rank of 't' is 2.
+// pad(t, paddings) ==> [[ 1,  5]
+//                       [11, 28]]
+// ```
 //
 // Arguments:
-//	a: Must be a two-dimensional tensor.
-//	b: Must be a two-dimensional tensor.
-//	min_a: The float value that the lowest quantized `a` value represents.
-//	max_a: The float value that the highest quantized `a` value represents.
-//	min_b: The float value that the lowest quantized `b` value represents.
-//	max_b: The float value that the highest quantized `b` value represents.
+//	input: The input tensor to be folded.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	mode: The mode used in the `MirrorPad` op.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
+// Returns The folded tensor.
+func MirrorPadGrad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"mode": mode}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMatMul",
+		Type: "MirrorPadGrad",
 		Input: []tf.Input{
-			a, b, min_a, max_a, min_b, max_b,
+			input, paddings,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// A placeholder op that passes through `input` when its output is not fed.
+// Pads a tensor with mirrored values.
+//
+// This operation pads a `input` with mirrored values according to the `paddings`
+// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is
+// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many values to add before the contents of `input` in that dimension, and
+// `paddings[D, 1]` indicates how many values to add after the contents of `input`
+// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater
+// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true
+// (if false, respectively).
+//
+// The padded size of each dimension D of the output is:
+//
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+//
+// For example:
+//
+// ```
+// # 't' is [[1, 2, 3], [4, 5, 6]].
+// # 'paddings' is [[1, 1]], [2, 2]].
+// # 'mode' is SYMMETRIC.
+// # rank of 't' is 2.
+// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2]
+//                       [2, 1, 1, 2, 3, 3, 2]
+//                       [5, 4, 4, 5, 6, 6, 5]
+//                       [5, 4, 4, 5, 6, 6, 5]]
+// ```
 //
 // Arguments:
-//	input: The default value to produce when `output` is not fed.
-//	shape: The (possibly partial) shape of the tensor.
+//	input: The input tensor to be padded.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions
+// do not include the borders, while in symmetric mode the padded regions
+// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings`
+// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and
+// it is `[1, 2, 3, 3, 2]` in symmetric mode.
 //
-// Returns A placeholder tensor that defaults to `input` if it is not fed.
-func PlaceholderWithDefault(scope *Scope, input tf.Output, shape tf.Shape) (output tf.Output) {
+// Returns The padded tensor.
+func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shape": shape}
+	attrs := map[string]interface{}{"mode": mode}
 	opspec := tf.OpSpec{
-		Type: "PlaceholderWithDefault",
+		Type: "MirrorPad",
 		Input: []tf.Input{
-			input,
+			input, paddings,
 		},
 		Attrs: attrs,
 	}
@@ -1274,38 +1280,78 @@ func PlaceholderWithDefault(scope *Scope, input tf.Output, shape tf.Shape) (outp
 	return op.Output(0)
 }
 
-// Returns the complex conjugate of a complex number.
+// Pads a tensor.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// complex numbers that are the complex conjugate of each element in `input`. The
-// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
-// real part and *b* is the imaginary part.
+// This operation pads `input` according to the `paddings` and `constant_values`
+// you specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is
+// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many padding values to add before the contents of `input` in that dimension,
+// and `paddings[D, 1]` indicates how many padding values to add after the contents
+// of `input` in that dimension. `constant_values` is a scalar tensor of the same
+// type as `input` that indicates the value to use for padding `input`.
 //
-// The complex conjugate returned by this operation is of the form \\(a - bj\\).
+// The padded size of each dimension D of the output is:
+//
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
 //
 // For example:
 //
 // ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
+// # 't' is [[1, 1], [2, 2]]
+// # 'paddings' is [[1, 1], [2, 2]]
+// # 'constant_values' is 0
+// # rank of 't' is 2
+// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+//                       [0, 0, 1, 1, 0, 0]
+//                       [0, 0, 2, 2, 0, 0]
+//                       [0, 0, 0, 0, 0, 0]]
 // ```
-func Conj(scope *Scope, input tf.Output) (output tf.Output) {
+func PadV2(scope *Scope, input tf.Output, paddings tf.Output, constant_values tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Conj",
+		Type: "PadV2",
 		Input: []tf.Input{
-			input,
+			input, paddings, constant_values,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
-type ResourceSparseApplyMomentumAttr func(optionalAttr)
-
+// Returns the complex conjugate of a complex number.
+//
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// complex numbers that are the complex conjugate of each element in `input`. The
+// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
+// real part and *b* is the imaginary part.
+//
+// The complex conjugate returned by this operation is of the form \\(a - bj\\).
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
+// ```
+func Conj(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Conj",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
+type ResourceSparseApplyMomentumAttr func(optionalAttr)
+
 // ResourceSparseApplyMomentumUseLocking sets the optional use_locking attribute to value.
 //
 // value: If `True`, updating of the var and accum tensors will be protected
@@ -2063,6 +2109,47 @@ func LogUniformCandidateSampler(scope *Scope, true_classes tf.Output, num_true i
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// Returns (x - y)(x - y) element-wise.
+//
+// *NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func SquaredDifference(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SquaredDifference",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Forwards the input to the output.
+//
+// This operator represents the loop termination condition used by the
+// "pivot" switches of a loop.
+//
+// Arguments:
+//	input: A boolean scalar, representing the branch predicate of the Switch op.
+//
+// Returns The same tensor as `input`.
+func LoopCond(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LoopCond",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // ApproximateEqualAttr is an optional argument to ApproximateEqual.
 type ApproximateEqualAttr func(optionalAttr)
 
@@ -2391,50 +2478,6 @@ func Sign(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// QuantizedAddAttr is an optional argument to QuantizedAdd.
-type QuantizedAddAttr func(optionalAttr)
-
-// QuantizedAddToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
-}
-
-// Returns x + y element-wise, working on quantized buffers.
-//
-// Arguments:
-//
-//
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
-//
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-//
-// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "QuantizedAdd",
-		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
 // ArgMinAttr is an optional argument to ArgMin.
 type ArgMinAttr func(optionalAttr)
 
@@ -3741,32 +3784,6 @@ func MatrixDiag(scope *Scope, diagonal tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Given a quantized tensor described by (input, input_min, input_max), outputs a
-//
-// range that covers the actual values present in that tensor.  This op is
-// typically used to produce the requested_output_min and requested_output_max for
-// Requantize.
-//
-// Arguments:
-//
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//
-// Returns The computed min output.the computed max output.
-func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RequantizationRange",
-		Input: []tf.Input{
-			input, input_min, input_max,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
 // Returns the truth value of (x <= y) element-wise.
 //
 // *NOTE*: `LessEqual` supports broadcasting. More about broadcasting
@@ -3943,46 +3960,6 @@ func BatchMatMul(scope *Scope, x tf.Output, y tf.Output, optional ...BatchMatMul
 	return op.Output(0)
 }
 
-// Pads a tensor.
-//
-// This operation pads `input` according to the `paddings` and `constant_values`
-// you specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is
-// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many padding values to add before the contents of `input` in that dimension,
-// and `paddings[D, 1]` indicates how many padding values to add after the contents
-// of `input` in that dimension. `constant_values` is a scalar tensor of the same
-// type as `input` that indicates the value to use for padding `input`.
-//
-// The padded size of each dimension D of the output is:
-//
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
-//
-// For example:
-//
-// ```
-// # 't' is [[1, 1], [2, 2]]
-// # 'paddings' is [[1, 1], [2, 2]]
-// # 'constant_values' is 0
-// # rank of 't' is 2
-// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
-//                       [0, 0, 1, 1, 0, 0]
-//                       [0, 0, 2, 2, 0, 0]
-//                       [0, 0, 0, 0, 0, 0]]
-// ```
-func PadV2(scope *Scope, input tf.Output, paddings tf.Output, constant_values tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "PadV2",
-		Input: []tf.Input{
-			input, paddings, constant_values,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Returns which elements of x are NaN.
 //
 // @compatibility(numpy)
@@ -4292,52 +4269,6 @@ func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output
 	return op.Output(0)
 }
 
-// MaxPoolAttr is an optional argument to MaxPool.
-type MaxPoolAttr func(optionalAttr)
-
-// MaxPoolDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolDataFormat(value string) MaxPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
-//
-// Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.
-func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MaxPool",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes gradients of the maxpooling function.
 //
 // Arguments:
@@ -5247,74 +5178,30 @@ func InvertPermutation(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Gradient op for `MirrorPad` op. This op folds a mirror-padded tensor.
+// BiasAddGradAttr is an optional argument to BiasAddGrad.
+type BiasAddGradAttr func(optionalAttr)
+
+// BiasAddGradDataFormat sets the optional data_format attribute to value.
 //
-// This operation folds the padded areas of `input` by `MirrorPad` according to the
-// `paddings` you specify. `paddings` must be the same as `paddings` argument
-// given to the corresponding `MirrorPad` op.
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the bias tensor will be added to the last dimension
+// of the value tensor.
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// The tensor will be added to "in_channels", the third-to-the-last
+//     dimension.
+// If not specified, defaults to "NHWC"
+func BiasAddGradDataFormat(value string) BiasAddGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// The backward operation for "BiasAdd" on the "bias" tensor.
 //
-// The folded size of each dimension D of the output is:
-//
-// `input.dim_size(D) - paddings(D, 0) - paddings(D, 1)`
-//
-// For example:
-//
-// ```
-// # 't' is [[1, 2, 3], [4, 5, 6], [7, 8, 9]].
-// # 'paddings' is [[0, 1]], [0, 1]].
-// # 'mode' is SYMMETRIC.
-// # rank of 't' is 2.
-// pad(t, paddings) ==> [[ 1,  5]
-//                       [11, 28]]
-// ```
-//
-// Arguments:
-//	input: The input tensor to be folded.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	mode: The mode used in the `MirrorPad` op.
-//
-// Returns The folded tensor.
-func MirrorPadGrad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"mode": mode}
-	opspec := tf.OpSpec{
-		Type: "MirrorPadGrad",
-		Input: []tf.Input{
-			input, paddings,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// BiasAddGradAttr is an optional argument to BiasAddGrad.
-type BiasAddGradAttr func(optionalAttr)
-
-// BiasAddGradDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the bias tensor will be added to the last dimension
-// of the value tensor.
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// The tensor will be added to "in_channels", the third-to-the-last
-//     dimension.
-// If not specified, defaults to "NHWC"
-func BiasAddGradDataFormat(value string) BiasAddGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// The backward operation for "BiasAdd" on the "bias" tensor.
-//
-// It accumulates all the values from out_backprop into the feature dimension.
-// For NHWC data format, the feature dimension is the last. For NCHW data format,
-// the feature dimension is the third-to-last.
+// It accumulates all the values from out_backprop into the feature dimension.
+// For NHWC data format, the feature dimension is the last. For NCHW data format,
+// the feature dimension is the third-to-last.
 //
 // Arguments:
 //	out_backprop: Any number of dimensions.
@@ -5411,297 +5298,220 @@ func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Outp
 	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
-type AvgPoolGradAttr func(optionalAttr)
-
-// AvgPoolGradDataFormat sets the optional data_format attribute to value.
+// Returns the rank of a tensor.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Computes gradients of the average pooling function.
+// This operation returns an integer representing the rank of `input`.
 //
-// Arguments:
-//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
-// the output of `avg_pool`.
-//	ksize: The size of the sliding window for each dimension of the input.
-//	strides: The stride of the sliding window for each dimension of the input.
-//	padding: The type of padding algorithm to use.
+// For example:
 //
-// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
-func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// # shape of tensor 't' is [2, 2, 3]
+// rank(t) ==> 3
+// ```
+//
+// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
+// of a tensor is the number of indices required to uniquely select each element
+// of the tensor. Rank is also known as "order", "degree", or "ndims."
+func Rank(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AvgPoolGrad",
+		Type: "Rank",
 		Input: []tf.Input{
-			orig_input_shape, grad,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
-
-// StageClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// Transforms a Tensor into a serialized TensorProto proto.
 //
-// REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// StageClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StageClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes all elements in the underlying container.
+// Arguments:
+//	tensor: A Tensor of type `T`.
 //
-// Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+// Returns A serialized TensorProto proto of the input tensor.
+func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "StageClear",
-
-		Attrs: attrs,
+		Type: "SerializeTensor",
+		Input: []tf.Input{
+			tensor,
+		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
-type ComputeAccidentalHitsAttr func(optionalAttr)
-
-// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+// MatrixSolveAttr is an optional argument to MatrixSolve.
+type MatrixSolveAttr func(optionalAttr)
 
-// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
+// MatrixSolveAdjoint sets the optional adjoint attribute to value.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+// adjoint.
+// If not specified, defaults to false
+func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Computes the ids of the positions in sampled_candidates that match true_labels.
+// Solves systems of linear equations.
 //
-// When doing log-odds NCE, the result of this op should be passed through a
-// SparseToDense op, then added to the logits of the sampled candidates. This has
-// the effect of 'removing' the sampled labels that match the true labels by
-// making the classifier sure that they are sampled labels.
+// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `True` then each output matrix satisfies
+// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
 //
 // Arguments:
-//	true_classes: The true_classes output of UnpackSparseLabels.
-//	sampled_candidates: The sampled_candidates output of CandidateSampler.
-//	num_true: Number of true labels per context.
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
 //
-// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
-// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
-// is -FLOAT_MAX.
-func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+// Returns Shape is `[..., M, K]`.
+func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ComputeAccidentalHits",
+		Type: "MatrixSolve",
 		Input: []tf.Input{
-			true_classes, sampled_candidates,
+			matrix, rhs,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
-
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
-//
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
-	return func(m optionalAttr) {
-		m["element_shape"] = value
+// Computes acos of x element-wise.
+func Acos(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Acos",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Gather specific elements from the TensorArray into output `value`.
+// Real-valued fast Fourier transform.
 //
-// All elements selected by `indices` must have the same shape.
+// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most dimension of `input`.
+//
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+// followed by the `fft_length / 2` positive-frequency terms.
+//
+// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+// Returns A complex64 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
+//   frequency components of its 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft
+// @end_compatibility
+func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "RFFT",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			input, fft_length,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
-//
-// Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
+// DepthwiseConv2dNativeBackpropFilterAttr is an optional argument to DepthwiseConv2dNativeBackpropFilter.
+type DepthwiseConv2dNativeBackpropFilterAttr func(optionalAttr)
+
+// DepthwiseConv2dNativeBackpropFilterDataFormat sets the optional data_format attribute to value.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
-	opspec := tf.OpSpec{
-		Type: "StringToHashBucketFast",
-		Input: []tf.Input{
-			input,
-		},
-		Attrs: attrs,
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func DepthwiseConv2dNativeBackpropFilterDataFormat(value string) DepthwiseConv2dNativeBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
+// DepthwiseConv2dNativeBackpropFilterDilations sets the optional dilations attribute to value.
 //
-// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Maximum",
-		Input: []tf.Input{
-			x, y,
-		},
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func DepthwiseConv2dNativeBackpropFilterDilations(value []int64) DepthwiseConv2dNativeBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Real-valued fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most dimension of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
-// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
-// followed by the `fft_length / 2` positive-frequency terms.
-//
-// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// Computes the gradients of depthwise convolution with respect to the filter.
 //
 // Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
-//   frequency components of its 1D Fourier transform.
+//	input: 4-D with shape based on `data_format`.  For example, if
+// `data_format` is 'NHWC' then `input` is a 4-D `[batch, in_height,
+// in_width, in_channels]` tensor.
+//	filter_sizes: An integer vector representing the tensor shape of `filter`,
+// where `filter` is a 4-D
+// `[filter_height, filter_width, in_channels, depthwise_multiplier]` tensor.
+//	out_backprop: 4-D with shape  based on `data_format`.
+// For example, if `data_format` is 'NHWC' then
+// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution.
+//	padding: The type of padding algorithm to use.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft
-// @end_compatibility
-func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Returns 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
+// the `filter` input of the convolution.
+func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropFilterAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "RFFT",
+		Type: "DepthwiseConv2dNativeBackpropFilter",
 		Input: []tf.Input{
-			input, fft_length,
+			input, filter_sizes, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -6236,6 +6046,79 @@ func Tan(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
+// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
+type ResourceSparseApplyFtrlAttr func(optionalAttr)
+
+// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
+//
+// That is for rows we have grad for, we update var, accum and linear as follows:
+// accum_new = accum + grad * grad
+// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceSparseApplyFtrl",
+		Input: []tf.Input{
+			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns which elements of x are Inf.
+//
+// @compatibility(numpy)
+// Equivalent to np.isinf
+// @end_compatibility
+func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IsInf",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the sum along sparse segments of a tensor divided by the sqrt of N.
 //
 // N is the size of the segment being reduced.
@@ -6918,32 +6801,196 @@ func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output,
 	return scope.AddOperation(opspec)
 }
 
-// CumsumAttr is an optional argument to Cumsum.
-type CumsumAttr func(optionalAttr)
+// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
+type AvgPoolGradAttr func(optionalAttr)
 
-// CumsumExclusive sets the optional exclusive attribute to value.
+// AvgPoolGradDataFormat sets the optional data_format attribute to value.
 //
-// value: If `True`, perform exclusive cumsum.
-// If not specified, defaults to false
-func CumsumExclusive(value bool) CumsumAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
 	return func(m optionalAttr) {
-		m["exclusive"] = value
+		m["data_format"] = value
 	}
 }
 
-// CumsumReverse sets the optional reverse attribute to value.
+// Computes gradients of the average pooling function.
 //
-// value: A `bool` (default: False).
-// If not specified, defaults to false
-func CumsumReverse(value bool) CumsumAttr {
-	return func(m optionalAttr) {
-		m["reverse"] = value
-	}
-}
-
-// Compute the cumulative sum of the tensor `x` along `axis`.
+// Arguments:
+//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
+// the output of `avg_pool`.
+//	ksize: The size of the sliding window for each dimension of the input.
+//	strides: The stride of the sliding window for each dimension of the input.
+//	padding: The type of padding algorithm to use.
 //
-// By default, this op performs an inclusive cumsum, which means that the first
+// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
+func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "AvgPoolGrad",
+		Input: []tf.Input{
+			orig_input_shape, grad,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
+
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StageClear",
+
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
+type ComputeAccidentalHitsAttr func(optionalAttr)
+
+// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Computes the ids of the positions in sampled_candidates that match true_labels.
+//
+// When doing log-odds NCE, the result of this op should be passed through a
+// SparseToDense op, then added to the logits of the sampled candidates. This has
+// the effect of 'removing' the sampled labels that match the true labels by
+// making the classifier sure that they are sampled labels.
+//
+// Arguments:
+//	true_classes: The true_classes output of UnpackSparseLabels.
+//	sampled_candidates: The sampled_candidates output of CandidateSampler.
+//	num_true: Number of true labels per context.
+//
+// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
+// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
+// is -FLOAT_MAX.
+func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_true": num_true}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ComputeAccidentalHits",
+		Input: []tf.Input{
+			true_classes, sampled_candidates,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// CumsumAttr is an optional argument to Cumsum.
+type CumsumAttr func(optionalAttr)
+
+// CumsumExclusive sets the optional exclusive attribute to value.
+//
+// value: If `True`, perform exclusive cumsum.
+// If not specified, defaults to false
+func CumsumExclusive(value bool) CumsumAttr {
+	return func(m optionalAttr) {
+		m["exclusive"] = value
+	}
+}
+
+// CumsumReverse sets the optional reverse attribute to value.
+//
+// value: A `bool` (default: False).
+// If not specified, defaults to false
+func CumsumReverse(value bool) CumsumAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Compute the cumulative sum of the tensor `x` along `axis`.
+//
+// By default, this op performs an inclusive cumsum, which means that the first
 // element of the input is identical to the first element of the output:
 //
 // ```python
@@ -7314,79 +7361,6 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64,
 	return op.Output(0)
 }
 
-// Generates values in an interval.
-//
-// A sequence of `num` evenly-spaced values are generated beginning at `start`.
-// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
-// so that the last one is exactly `stop`.
-//
-// For example:
-//
-// ```
-// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
-// ```
-//
-// Arguments:
-//	start: First entry in the range.
-//	stop: Last entry in the range.
-//	num: Number of values to generate.
-//
-// Returns 1-D. The generated values.
-func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LinSpace",
-		Input: []tf.Input{
-			start, stop, num,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
-type DestroyResourceOpAttr func(optionalAttr)
-
-// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
-//
-// value: whether to ignore the error when the resource
-// doesn't exist.
-// If not specified, defaults to true
-func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
-	return func(m optionalAttr) {
-		m["ignore_lookup_error"] = value
-	}
-}
-
-// Deletes the resource specified by the handle.
-//
-// All subsequent operations using the resource will result in a NotFound
-// error status.
-//
-// Arguments:
-//	resource: handle to the resource to delete.
-//
-// Returns the created operation.
-func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DestroyResourceOp",
-		Input: []tf.Input{
-			resource,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
 // Applies softmax to a batched N-D `SparseTensor`.
 //
 // The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
@@ -7822,24 +7796,97 @@ func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// LRNAttr is an optional argument to LRN.
-type LRNAttr func(optionalAttr)
-
-// LRNDepthRadius sets the optional depth_radius attribute to value.
+// Generates values in an interval.
 //
-// value: 0-D.  Half-width of the 1-D normalization window.
-// If not specified, defaults to 5
-func LRNDepthRadius(value int64) LRNAttr {
-	return func(m optionalAttr) {
-		m["depth_radius"] = value
-	}
-}
-
-// LRNBias sets the optional bias attribute to value.
+// A sequence of `num` evenly-spaced values are generated beginning at `start`.
+// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
+// so that the last one is exactly `stop`.
 //
-// value: An offset (usually positive to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNBias(value float32) LRNAttr {
+// For example:
+//
+// ```
+// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
+// ```
+//
+// Arguments:
+//	start: First entry in the range.
+//	stop: Last entry in the range.
+//	num: Number of values to generate.
+//
+// Returns 1-D. The generated values.
+func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "LinSpace",
+		Input: []tf.Input{
+			start, stop, num,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
+type DestroyResourceOpAttr func(optionalAttr)
+
+// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
+//
+// value: whether to ignore the error when the resource
+// doesn't exist.
+// If not specified, defaults to true
+func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
+	return func(m optionalAttr) {
+		m["ignore_lookup_error"] = value
+	}
+}
+
+// Deletes the resource specified by the handle.
+//
+// All subsequent operations using the resource will result in a NotFound
+// error status.
+//
+// Arguments:
+//	resource: handle to the resource to delete.
+//
+// Returns the created operation.
+func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DestroyResourceOp",
+		Input: []tf.Input{
+			resource,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// LRNAttr is an optional argument to LRN.
+type LRNAttr func(optionalAttr)
+
+// LRNDepthRadius sets the optional depth_radius attribute to value.
+//
+// value: 0-D.  Half-width of the 1-D normalization window.
+// If not specified, defaults to 5
+func LRNDepthRadius(value int64) LRNAttr {
+	return func(m optionalAttr) {
+		m["depth_radius"] = value
+	}
+}
+
+// LRNBias sets the optional bias attribute to value.
+//
+// value: An offset (usually positive to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNBias(value float32) LRNAttr {
 	return func(m optionalAttr) {
 		m["bias"] = value
 	}
@@ -8054,6 +8101,65 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi
 	return op.Output(0)
 }
 
+// Pads a tensor with zeros.
+//
+// This operation pads a `input` with zeros according to the `paddings` you
+// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
+// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many zeros to add before the contents of `input` in that dimension, and
+// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
+// in that dimension.
+//
+// The padded size of each dimension D of the output is:
+//
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+//
+// For example:
+//
+// ```
+// # 't' is [[1, 1], [2, 2]]
+// # 'paddings' is [[1, 1], [2, 2]]
+// # rank of 't' is 2
+// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+//                       [0, 0, 1, 1, 0, 0]
+//                       [0, 0, 2, 2, 0, 0]
+//                       [0, 0, 0, 0, 0, 0]]
+// ```
+func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Pad",
+		Input: []tf.Input{
+			input, paddings,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Checks whether a resource handle-based variable has been initialized.
+//
+// Arguments:
+//	resource: the input resource handle.
+//
+// Returns a scalar boolean which is true if the variable has been
+// initialized.
+func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "VarIsInitializedOp",
+		Input: []tf.Input{
+			resource,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
 type StatelessRandomUniformAttr func(optionalAttr)
 
@@ -8098,6 +8204,38 @@ func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optio
 	return op.Output(0)
 }
 
+// Makes its input available to the next iteration.
+//
+// Arguments:
+//	data: The tensor to be made available to the next iteration.
+//
+// Returns The same tensor as `data`.
+func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "NextIteration",
+		Input: []tf.Input{
+			data,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Output a fact about factorials.
+func Fact(scope *Scope) (fact tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Fact",
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // AngleAttr is an optional argument to Angle.
 type AngleAttr func(optionalAttr)
 
@@ -8672,79 +8810,6 @@ func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output
 	return op.Output(0)
 }
 
-// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
-type ResourceSparseApplyFtrlAttr func(optionalAttr)
-
-// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
-//
-// That is for rows we have grad for, we update var, accum and linear as follows:
-// accum_new = accum + grad * grad
-// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
-//
-// Returns the created operation.
-func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyFtrl",
-		Input: []tf.Input{
-			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Returns which elements of x are Inf.
-//
-// @compatibility(numpy)
-// Equivalent to np.isinf
-// @end_compatibility
-func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IsInf",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
 type ResourceSparseApplyRMSPropAttr func(optionalAttr)
 
@@ -8974,12 +9039,106 @@ func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_box
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns x / y element-wise for integer types.
-//
-// Truncation designates that negative numbers will round fractional quantities
-// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
-// than Python semantics. See `FloorDiv` for a division function that matches
-// Python Semantics.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
+//
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
+//
+// Arguments:
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
+	opspec := tf.OpSpec{
+		Type: "StringToHashBucketFast",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
+//
+// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Maximum",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
+
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+//
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// Gather specific elements from the TensorArray into output `value`.
+//
+// All elements selected by `indices` must have the same shape.
+//
+// Arguments:
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
+//
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGatherV3",
+		Input: []tf.Input{
+			handle, indices, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns x / y element-wise for integer types.
+//
+// Truncation designates that negative numbers will round fractional quantities
+// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
+// than Python semantics. See `FloorDiv` for a division function that matches
+// Python Semantics.
 //
 // *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
@@ -9048,6 +9207,30 @@ func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and
 	return tensors
 }
 
+// Creates a dataset that skips `count` elements from the `input_dataset`.
+//
+// Arguments:
+//
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
+//
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "SkipDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Computes the maximum along segments of a tensor.
 //
 // Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
@@ -9084,30 +9267,6 @@ func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.
 	return op.Output(0)
 }
 
-// Creates a dataset that skips `count` elements from the `input_dataset`.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
-//
-//
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "SkipDataset",
-		Input: []tf.Input{
-			input_dataset, count,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes hyperbolic tangent of `x` element-wise.
 func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
@@ -9861,6 +10020,79 @@ func FFT(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
+// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
+//
+// Arguments:
+//	serialized: A scalar string containing a serialized TensorProto proto.
+//	out_type: The type of the serialized tensor.  The provided type must match the
+// type of the serialized tensor and no implicit conversion will take place.
+//
+// Returns A Tensor of type `out_type`.
+func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"out_type": out_type}
+	opspec := tf.OpSpec{
+		Type: "ParseTensor",
+		Input: []tf.Input{
+			serialized,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
+type MaxPoolWithArgmaxAttr func(optionalAttr)
+
+// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
+// If not specified, defaults to DT_INT64
+func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
+	return func(m optionalAttr) {
+		m["Targmax"] = value
+	}
+}
+
+// Performs max pooling on the input and outputs both max values and indices.
+//
+// The indices in `argmax` are flattened, so that a maximum value at position
+// `[b, y, x, c]` becomes flattened index
+// `((b * height + y) * width + x) * channels + c`.
+//
+// The indices returned are always in `[0, height) x [0, width)` before flattening,
+// even if padding is involved and the mathematically correct answer is outside
+// (either negative or too large).  This is a bug, but fixing it is difficult to do
+// in a safe backwards compatible way, especially due to flattening.
+//
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
+func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MaxPoolWithArgmax",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
 // ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
 type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
 
@@ -11004,104 +11236,37 @@ func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, padd
 	return op.Output(0)
 }
 
-// Transforms a Tensor into a serialized TensorProto proto.
+// Inverse 3D fast Fourier transform.
+//
+// Computes the inverse 3-dimensional discrete Fourier transform over the
+// inner-most 3 dimensions of `input`.
 //
 // Arguments:
-//	tensor: A Tensor of type `T`.
+//	input: A complex64 tensor.
 //
-// Returns A serialized TensorProto proto of the input tensor.
-func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
+//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifftn with 3 dimensions.
+// @end_compatibility
+func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SerializeTensor",
+		Type: "IFFT3D",
 		Input: []tf.Input{
-			tensor,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixSolveAttr is an optional argument to MatrixSolve.
-type MatrixSolveAttr func(optionalAttr)
-
-// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+// Adds `bias` to `value`.
 //
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-// adjoint.
-// If not specified, defaults to false
-func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
-	return func(m optionalAttr) {
-		m["adjoint"] = value
-	}
-}
-
-// Solves systems of linear equations.
-//
-// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
-// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
-// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `True` then each output matrix satisfies
-// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
-//
-// Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
-//
-// Returns Shape is `[..., M, K]`.
-func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MatrixSolve",
-		Input: []tf.Input{
-			matrix, rhs,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Inverse 3D fast Fourier transform.
-//
-// Computes the inverse 3-dimensional discrete Fourier transform over the
-// inner-most 3 dimensions of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
-//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.ifftn with 3 dimensions.
-// @end_compatibility
-func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IFFT3D",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Adds `bias` to `value`.
-//
-// This is a deprecated version of BiasAdd and will be soon removed.
+// This is a deprecated version of BiasAdd and will be soon removed.
 //
 // This is a special case of `tf.add` where `bias` is restricted to be 1-D.
 // Broadcasting is supported, so `value` may have any number of dimensions.
@@ -12025,6 +12190,46 @@ func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_va
 	return op.Output(0)
 }
 
+// Concatenates tensors along one dimension.
+//
+// Arguments:
+//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [-rank(values), rank(values)).
+//
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.
+func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ConcatV2",
+		Input: []tf.Input{
+			tf.OutputList(values), axis,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Reads and outputs the entire contents of the input filename.
+func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReadFile",
+		Input: []tf.Input{
+			filename,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // MinAttr is an optional argument to Min.
 type MinAttr func(optionalAttr)
 
@@ -12088,76 +12293,6 @@ func Transpose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// DepthwiseConv2dNativeBackpropFilterAttr is an optional argument to DepthwiseConv2dNativeBackpropFilter.
-type DepthwiseConv2dNativeBackpropFilterAttr func(optionalAttr)
-
-// DepthwiseConv2dNativeBackpropFilterDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
-// If not specified, defaults to "NHWC"
-func DepthwiseConv2dNativeBackpropFilterDataFormat(value string) DepthwiseConv2dNativeBackpropFilterAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// DepthwiseConv2dNativeBackpropFilterDilations sets the optional dilations attribute to value.
-//
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func DepthwiseConv2dNativeBackpropFilterDilations(value []int64) DepthwiseConv2dNativeBackpropFilterAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of depthwise convolution with respect to the filter.
-//
-// Arguments:
-//	input: 4-D with shape based on `data_format`.  For example, if
-// `data_format` is 'NHWC' then `input` is a 4-D `[batch, in_height,
-// in_width, in_channels]` tensor.
-//	filter_sizes: An integer vector representing the tensor shape of `filter`,
-// where `filter` is a 4-D
-// `[filter_height, filter_width, in_channels, depthwise_multiplier]` tensor.
-//	out_backprop: 4-D with shape  based on `data_format`.
-// For example, if `data_format` is 'NHWC' then
-// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution.
-//	padding: The type of padding algorithm to use.
-//
-// Returns 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
-// the `filter` input of the convolution.
-func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropFilterAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "DepthwiseConv2dNativeBackpropFilter",
-		Input: []tf.Input{
-			input, filter_sizes, out_backprop,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Computes sigmoid of `x` element-wise.
 //
 // Specifically, `y = 1 / (1 + exp(-x))`.
@@ -12888,190 +13023,252 @@ func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...
 	return op.Output(0)
 }
 
-// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
-type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+// DecodeCSVAttr is an optional argument to DecodeCSV.
+type DecodeCSVAttr func(optionalAttr)
 
-// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+// value: char delimiter to separate fields in a record.
+// If not specified, defaults to ","
+func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["field_delim"] = value
 	}
 }
 
-// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+// value: If false, treats double quotation marks as regular
+// characters inside of the string fields (ignoring RFC 4180, Section 2,
+// Bullet 5).
+// If not specified, defaults to true
+func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["use_quote_delim"] = value
 	}
 }
 
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
+// DecodeCSVNaValue sets the optional na_value attribute to value.
 //
-// For each batch, this op picks a single set of sampled candidate labels.
+// value: Additional string to recognize as NA/NaN.
+// If not specified, defaults to ""
+func DecodeCSVNaValue(value string) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["na_value"] = value
+	}
+}
+
+// Convert CSV records to tensors. Each column maps to one tensor.
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+// RFC 4180 format is expected for the CSV records.
+// (https://tools.ietf.org/html/rfc4180)
+// Note that we allow leading and trailing spaces with int or float field.
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+//	records: Each string is a record/row in the csv and all records should have
+// the same format.
+//	record_defaults: One tensor per column of the input record, with either a
+// scalar default value for that column or empty if the column is required.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// Returns Each tensor will have the same shape as records.
+func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ThreadUnsafeUnigramCandidateSampler",
+		Type: "DecodeCSV",
 		Input: []tf.Input{
-			true_classes,
+			records, tf.OutputList(record_defaults),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// MaxPoolV2Attr is an optional argument to MaxPoolV2.
-type MaxPoolV2Attr func(optionalAttr)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("DecodeCSV", err)
+		return
+	}
+	return output
+}
 
-// MaxPoolV2DataFormat sets the optional data_format attribute to value.
+// MapClearAttr is an optional argument to MapClear.
+type MapClearAttr func(optionalAttr)
+
+// MapClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
+// REQUIRES: value >= 0
+func MapClearCapacity(value int64) MapClearAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["capacity"] = value
 	}
 }
 
-// Performs max pooling on the input.
+// MapClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+// REQUIRES: value >= 0
+func MapClearMemoryLimit(value int64) MapClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapClearContainer(value string) MapClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapClearSharedName(value string) MapClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
 //
-// Returns The max pooled output tensor.
-func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
+// Returns the created operation.
+func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"padding": padding}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolV2",
-		Input: []tf.Input{
-			input, ksize, strides,
-		},
+		Type: "MapClear",
+
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Deprecated. Use TensorArrayReadV3
+// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
+type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+
+// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
-func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Generates labels for candidate sampling with a learned unigram distribution.
+//
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
+//
+// For each batch, this op picks a single set of sampled candidate labels.
+//
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
+//
+// Arguments:
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
+//
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV2",
+		Type: "ThreadUnsafeUnigramCandidateSampler",
 		Input: []tf.Input{
-			handle, index, flow_in,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Does nothing. Serves as a control trigger for scheduling.
-//
-// Only useful as a placeholder for control edges.
+// MaxPoolV2Attr is an optional argument to MaxPoolV2.
+type MaxPoolV2Attr func(optionalAttr)
+
+// MaxPoolV2DataFormat sets the optional data_format attribute to value.
 //
-// Returns the created operation.
-func ControlTrigger(scope *Scope) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ControlTrigger",
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Batch normalization.
-//
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
-//
-// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+// Performs max pooling on the input.
 //
 // Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The max pooled output tensor.
+func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	attrs := map[string]interface{}{"padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalization",
+		Type: "MaxPoolV2",
 		Input: []tf.Input{
-			t, m, v, beta, gamma,
+			input, ksize, strides,
 		},
 		Attrs: attrs,
 	}
@@ -13375,99 +13572,40 @@ func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Outp
 	return op.Output(0)
 }
 
-// Checks whether a resource handle-based variable has been initialized.
+// Computes gradients for SparseSegmentMean.
 //
-// Arguments:
-//	resource: the input resource handle.
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
 //
-// Returns a scalar boolean which is true if the variable has been
-// initialized.
-func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
+// Arguments:
+//	grad: gradient propagated to the SparseSegmentMean op.
+//	indices: indices passed to the corresponding SparseSegmentMean op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
+func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "VarIsInitializedOp",
+		Type: "SparseSegmentMeanGrad",
 		Input: []tf.Input{
-			resource,
+			grad, indices, segment_ids, output_dim0,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Pads a tensor with zeros.
-//
-// This operation pads a `input` with zeros according to the `paddings` you
-// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
-// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many zeros to add before the contents of `input` in that dimension, and
-// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
-// in that dimension.
-//
-// The padded size of each dimension D of the output is:
-//
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
-//
-// For example:
+// Returns the truth value of (x >= y) element-wise.
 //
-// ```
-// # 't' is [[1, 1], [2, 2]]
-// # 'paddings' is [[1, 1], [2, 2]]
-// # rank of 't' is 2
-// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
-//                       [0, 0, 1, 1, 0, 0]
-//                       [0, 0, 2, 2, 0, 0]
-//                       [0, 0, 0, 0, 0, 0]]
-// ```
-func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
+// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Pad",
-		Input: []tf.Input{
-			input, paddings,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for SparseSegmentMean.
-//
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
-//
-// Arguments:
-//	grad: gradient propagated to the SparseSegmentMean op.
-//	indices: indices passed to the corresponding SparseSegmentMean op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
-func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanGrad",
-		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the truth value of (x >= y) element-wise.
-//
-// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "GreaterEqual",
+		Type: "GreaterEqual",
 		Input: []tf.Input{
 			x, y,
 		},
@@ -14876,6 +15014,101 @@ func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) {
 	return op.Output(0)
 }
 
+// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
+type MatrixSolveLsAttr func(optionalAttr)
+
+// MatrixSolveLsFast sets the optional fast attribute to value.
+// If not specified, defaults to true
+func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
+	return func(m optionalAttr) {
+		m["fast"] = value
+	}
+}
+
+// Solves one or more linear least-squares problems.
+//
+// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
+// type as `matrix` and shape `[..., M, K]`.
+// The output is a tensor shape `[..., N, K]` where each output matrix solves
+// each of the equations
+// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
+// in the least squares sense.
+//
+// We use the following notation for (complex) matrix and right-hand sides
+// in the batch:
+//
+// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
+// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
+// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
+// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
+//
+// If `fast` is `True`, then the solution is computed by solving the normal
+// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
+// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
+// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
+// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
+// minimum-norm solution to the under-determined linear system, i.e.
+// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
+// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
+// when \\(A\\) is numerically full rank and has a condition number
+// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+// sufficiently large.
+//
+// If `fast` is `False` an algorithm based on the numerically robust complete
+// orthogonal decomposition is used. This computes the minimum-norm
+// least-squares solution, even when \\(A\\) is rank deficient. This path is
+// typically 6-7 times slower than the fast path. If `fast` is `False` then
+// `l2_regularizer` is ignored.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, N]`.
+//	rhs: Shape is `[..., M, K]`.
+//	l2_regularizer: Scalar tensor.
+//
+// @compatibility(numpy)
+// Equivalent to np.linalg.lstsq
+// @end_compatibility
+//
+// Returns Shape is `[..., N, K]`.
+func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MatrixSolveLs",
+		Input: []tf.Input{
+			matrix, rhs, l2_regularizer,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Elementwise computes the bitwise OR of `x` and `y`.
+//
+// The result will have those bits set, that are set in `x`, `y` or both. The
+// computation is performed on the underlying representations of `x` and `y`.
+func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BitwiseOr",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
 type SparseToSparseSetOperationAttr func(optionalAttr)
 
@@ -15174,6 +15407,52 @@ func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
+// MaxPoolAttr is an optional argument to MaxPool.
+type MaxPoolAttr func(optionalAttr)
+
+// MaxPoolDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolDataFormat(value string) MaxPoolAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Performs max pooling on the input.
+//
+// Arguments:
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The max pooled output tensor.
+func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MaxPool",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Says whether the targets are in the top `K` predictions.
 //
 // This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
@@ -16313,106 +16592,29 @@ func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// MfccAttr is an optional argument to Mfcc.
-type MfccAttr func(optionalAttr)
-
-// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
-//
-// value: The highest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 4000
-func MfccUpperFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["upper_frequency_limit"] = value
-	}
-}
-
-// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
-//
-// value: The lowest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 20
-func MfccLowerFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["lower_frequency_limit"] = value
-	}
-}
-
-// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
+// Returns the element-wise sum of a list of tensors.
 //
-// value: Resolution of the Mel bank used internally.
-// If not specified, defaults to 40
-func MfccFilterbankChannelCount(value int64) MfccAttr {
-	return func(m optionalAttr) {
-		m["filterbank_channel_count"] = value
-	}
-}
-
-// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
+// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+// wait for all of its inputs to be ready before beginning to sum. This can
+// save memory if inputs are ready at different times, since minimum temporary
+// storage is proportional to the output size rather than the inputs size.
 //
-// value: How many output channels to produce per time slice.
-// If not specified, defaults to 13
-func MfccDctCoefficientCount(value int64) MfccAttr {
-	return func(m optionalAttr) {
-		m["dct_coefficient_count"] = value
-	}
-}
-
-// Transforms a spectrogram into a form that's useful for speech recognition.
+// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
 //
-// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
-// been effective as an input feature for machine learning. They are created by
-// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
-// higher frequencies that are less significant to the human ear. They have a long
-// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
-// is a good resource to learn more.
+// Returns a `Tensor` of same shape and type as the elements of `inputs`.
 //
 // Arguments:
-//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
-// set to true.
-//	sample_rate: How many samples per second the source audio used.
-func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
+//	inputs: A list of `Tensor` objects, each with same shape and type.
+//	shape: Shape of elements of `inputs`.
+func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "Mfcc",
+		Type: "AccumulateNV2",
 		Input: []tf.Input{
-			spectrogram, sample_rate,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the element-wise sum of a list of tensors.
-//
-// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
-// wait for all of its inputs to be ready before beginning to sum. This can
-// save memory if inputs are ready at different times, since minimum temporary
-// storage is proportional to the output size rather than the inputs size.
-//
-// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
-//
-// Returns a `Tensor` of same shape and type as the elements of `inputs`.
-//
-// Arguments:
-//	inputs: A list of `Tensor` objects, each with same shape and type.
-//	shape: Shape of elements of `inputs`.
-func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"shape": shape}
-	opspec := tf.OpSpec{
-		Type: "AccumulateNV2",
-		Input: []tf.Input{
-			tf.OutputList(inputs),
+			tf.OutputList(inputs),
 		},
 		Attrs: attrs,
 	}
@@ -17022,87 +17224,129 @@ func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_uppe
 	return op.Output(0)
 }
 
-// Computes acos of x element-wise.
-func Acos(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Acos",
-		Input: []tf.Input{
-			x,
-		},
+// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
+type QuantizedMatMulAttr func(optionalAttr)
+
+// QuantizedMatMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["Toutput"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
-type MaxPoolWithArgmaxAttr func(optionalAttr)
+// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
+//
+// value: If true, `a` is transposed before multiplication.
+// If not specified, defaults to false
+func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_a"] = value
+	}
+}
 
-// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
-// If not specified, defaults to DT_INT64
-func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
+// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
+//
+// value: If true, `b` is transposed before multiplication.
+// If not specified, defaults to false
+func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
 	return func(m optionalAttr) {
-		m["Targmax"] = value
+		m["transpose_b"] = value
 	}
 }
 
-// Performs max pooling on the input and outputs both max values and indices.
+// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
 //
-// The indices in `argmax` are flattened, so that a maximum value at position
-// `[b, y, x, c]` becomes flattened index
-// `((b * height + y) * width + x) * channels + c`.
+// value: The type of output produced by activation function
+// following this operation.
+// If not specified, defaults to DT_QUINT8
+func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["Tactivation"] = value
+	}
+}
+
+// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
 //
-// The indices returned are always in `[0, height) x [0, width)` before flattening,
-// even if padding is involved and the mathematically correct answer is outside
-// (either negative or too large).  This is a bug, but fixing it is difficult to do
-// in a safe backwards compatible way, especially due to flattening.
+// The inputs must be two-dimensional matrices and the inner dimension of
+// `a` (after being transposed if `transpose_a` is non-zero) must match the
+// outer dimension of `b` (after being transposed if `transposed_b` is
+// non-zero).
 //
 // Arguments:
-//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	a: Must be a two-dimensional tensor.
+//	b: Must be a two-dimensional tensor.
+//	min_a: The float value that the lowest quantized `a` value represents.
+//	max_a: The float value that the highest quantized `a` value represents.
+//	min_b: The float value that the lowest quantized `b` value represents.
+//	max_b: The float value that the highest quantized `b` value represents.
 //
-// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
-func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolWithArgmax",
+		Type: "QuantizedMatMul",
 		Input: []tf.Input{
-			input,
+			a, b, min_a, max_a, min_b, max_b,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
+// Does nothing. Serves as a control trigger for scheduling.
 //
-// Arguments:
-//	serialized: A scalar string containing a serialized TensorProto proto.
-//	out_type: The type of the serialized tensor.  The provided type must match the
-// type of the serialized tensor and no implicit conversion will take place.
+// Only useful as a placeholder for control edges.
 //
-// Returns A Tensor of type `out_type`.
-func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
+// Returns the created operation.
+func ControlTrigger(scope *Scope) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "ParseTensor",
+		Type: "ControlTrigger",
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Batch normalization.
+//
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+//
+// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+//
+// Arguments:
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	opspec := tf.OpSpec{
+		Type: "BatchNormWithGlobalNormalization",
 		Input: []tf.Input{
-			serialized,
+			t, m, v, beta, gamma,
 		},
 		Attrs: attrs,
 	}
@@ -17110,113 +17354,95 @@ func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (outp
 	return op.Output(0)
 }
 
-// MapClearAttr is an optional argument to MapClear.
-type MapClearAttr func(optionalAttr)
-
-// MapClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// Deprecated. Use TensorArrayReadV3
 //
-// REQUIRES: value >= 0
-func MapClearCapacity(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
+// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
+func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// MapClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapClearMemoryLimit(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
+	attrs := map[string]interface{}{"dtype": dtype}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayReadV2",
+		Input: []tf.Input{
+			handle, index, flow_in,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MapClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapClearContainer(value string) MapClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// QuantizedMulAttr is an optional argument to QuantizedMul.
+type QuantizedMulAttr func(optionalAttr)
 
-// MapClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapClearSharedName(value string) MapClearAttr {
+// QuantizedMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["Toutput"] = value
 	}
 }
 
-// Op removes all elements in the underlying container.
+// Returns x * y element-wise, working on quantized buffers.
 //
-// Returns the created operation.
-func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
+// Arguments:
+//
+//
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+//
+// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapClear",
-
+		Type: "QuantizedMul",
+		Input: []tf.Input{
+			x, y, min_x, max_x, min_y, max_y,
+		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DecodeCSVAttr is an optional argument to DecodeCSV.
-type DecodeCSVAttr func(optionalAttr)
+// QuantizedAddAttr is an optional argument to QuantizedAdd.
+type QuantizedAddAttr func(optionalAttr)
 
-// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
-//
-// value: char delimiter to separate fields in a record.
-// If not specified, defaults to ","
-func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
+// QuantizedAddToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
 	return func(m optionalAttr) {
-		m["field_delim"] = value
+		m["Toutput"] = value
 	}
 }
 
-// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
+// Returns x + y element-wise, working on quantized buffers.
 //
-// value: If false, treats double quotation marks as regular
-// characters inside of the string fields (ignoring RFC 4180, Section 2,
-// Bullet 5).
-// If not specified, defaults to true
-func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["use_quote_delim"] = value
-	}
-}
-
-// DecodeCSVNaValue sets the optional na_value attribute to value.
+// Arguments:
 //
-// value: Additional string to recognize as NA/NaN.
-// If not specified, defaults to ""
-func DecodeCSVNaValue(value string) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["na_value"] = value
-	}
-}
-
-// Convert CSV records to tensors. Each column maps to one tensor.
 //
-// RFC 4180 format is expected for the CSV records.
-// (https://tools.ietf.org/html/rfc4180)
-// Note that we allow leading and trailing spaces with int or float field.
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
 //
-// Arguments:
-//	records: Each string is a record/row in the csv and all records should have
-// the same format.
-//	record_defaults: One tensor per column of the input record, with either a
-// scalar default value for that column or empty if the column is required.
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
 //
-// Returns Each tensor will have the same shape as records.
-func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
+// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17225,84 +17451,117 @@ func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, opt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeCSV",
+		Type: "QuantizedAdd",
 		Input: []tf.Input{
-			records, tf.OutputList(record_defaults),
+			x, y, min_x, max_x, min_y, max_y,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
+	return op.Output(0), op.Output(1), op.Output(2)
+}
+
+// MfccAttr is an optional argument to Mfcc.
+type MfccAttr func(optionalAttr)
+
+// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
+//
+// value: The highest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 4000
+func MfccUpperFrequencyLimit(value float32) MfccAttr {
+	return func(m optionalAttr) {
+		m["upper_frequency_limit"] = value
 	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("DecodeCSV", err)
-		return
+}
+
+// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
+//
+// value: The lowest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 20
+func MfccLowerFrequencyLimit(value float32) MfccAttr {
+	return func(m optionalAttr) {
+		m["lower_frequency_limit"] = value
+	}
+}
+
+// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
+//
+// value: Resolution of the Mel bank used internally.
+// If not specified, defaults to 40
+func MfccFilterbankChannelCount(value int64) MfccAttr {
+	return func(m optionalAttr) {
+		m["filterbank_channel_count"] = value
 	}
-	return output
 }
 
-// Returns the rank of a tensor.
-//
-// This operation returns an integer representing the rank of `input`.
+// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
 //
-// For example:
+// value: How many output channels to produce per time slice.
+// If not specified, defaults to 13
+func MfccDctCoefficientCount(value int64) MfccAttr {
+	return func(m optionalAttr) {
+		m["dct_coefficient_count"] = value
+	}
+}
+
+// Transforms a spectrogram into a form that's useful for speech recognition.
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// # shape of tensor 't' is [2, 2, 3]
-// rank(t) ==> 3
-// ```
+// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
+// been effective as an input feature for machine learning. They are created by
+// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
+// higher frequencies that are less significant to the human ear. They have a long
+// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+// is a good resource to learn more.
 //
-// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
-// of a tensor is the number of indices required to uniquely select each element
-// of the tensor. Rank is also known as "order", "degree", or "ndims."
-func Rank(scope *Scope, input tf.Output) (output tf.Output) {
+// Arguments:
+//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
+// set to true.
+//	sample_rate: How many samples per second the source audio used.
+func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Rank",
+		Type: "Mfcc",
 		Input: []tf.Input{
-			input,
+			spectrogram, sample_rate,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Output a fact about factorials.
-func Fact(scope *Scope) (fact tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Fact",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Makes its input available to the next iteration.
+// Given a quantized tensor described by (input, input_min, input_max), outputs a
+//
+// range that covers the actual values present in that tensor.  This op is
+// typically used to produce the requested_output_min and requested_output_max for
+// Requantize.
 //
 // Arguments:
-//	data: The tensor to be made available to the next iteration.
 //
-// Returns The same tensor as `data`.
-func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//
+// Returns The computed min output.the computed max output.
+func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "NextIteration",
+		Type: "RequantizationRange",
 		Input: []tf.Input{
-			data,
+			input, input_min, input_max,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
 // MapPeekAttr is an optional argument to MapPeek.
@@ -18911,101 +19170,6 @@ func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output t
 	return op.Output(0)
 }
 
-// Elementwise computes the bitwise OR of `x` and `y`.
-//
-// The result will have those bits set, that are set in `x`, `y` or both. The
-// computation is performed on the underlying representations of `x` and `y`.
-func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BitwiseOr",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
-type MatrixSolveLsAttr func(optionalAttr)
-
-// MatrixSolveLsFast sets the optional fast attribute to value.
-// If not specified, defaults to true
-func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
-	return func(m optionalAttr) {
-		m["fast"] = value
-	}
-}
-
-// Solves one or more linear least-squares problems.
-//
-// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
-// type as `matrix` and shape `[..., M, K]`.
-// The output is a tensor shape `[..., N, K]` where each output matrix solves
-// each of the equations
-// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
-// in the least squares sense.
-//
-// We use the following notation for (complex) matrix and right-hand sides
-// in the batch:
-//
-// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
-// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
-// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
-// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
-//
-// If `fast` is `True`, then the solution is computed by solving the normal
-// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
-// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
-// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
-// minimum-norm solution to the under-determined linear system, i.e.
-// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
-// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
-// when \\(A\\) is numerically full rank and has a condition number
-// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
-// sufficiently large.
-//
-// If `fast` is `False` an algorithm based on the numerically robust complete
-// orthogonal decomposition is used. This computes the minimum-norm
-// least-squares solution, even when \\(A\\) is rank deficient. This path is
-// typically 6-7 times slower than the fast path. If `fast` is `False` then
-// `l2_regularizer` is ignored.
-//
-// Arguments:
-//	matrix: Shape is `[..., M, N]`.
-//	rhs: Shape is `[..., M, K]`.
-//	l2_regularizer: Scalar tensor.
-//
-// @compatibility(numpy)
-// Equivalent to np.linalg.lstsq
-// @end_compatibility
-//
-// Returns Shape is `[..., N, K]`.
-func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MatrixSolveLs",
-		Input: []tf.Input{
-			matrix, rhs, l2_regularizer,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // SvdAttr is an optional argument to Svd.
 type SvdAttr func(optionalAttr)
 
@@ -20803,28 +20967,101 @@ func ExtractGlimpseUniformNoise(value bool) ExtractGlimpseAttr {
 // * If the coordinates are not normalized they are interpreted as
 //   numbers of pixels.
 //
-// Arguments:
-//	input: A 4-D float tensor of shape `[batch_size, height, width, channels]`.
-//	size: A 1-D tensor of 2 elements containing the size of the glimpses
-// to extract.  The glimpse height must be specified first, following
-// by the glimpse width.
-//	offsets: A 2-D integer tensor of shape `[batch_size, 2]` containing
-// the y, x locations of the center of each window.
+// Arguments:
+//	input: A 4-D float tensor of shape `[batch_size, height, width, channels]`.
+//	size: A 1-D tensor of 2 elements containing the size of the glimpses
+// to extract.  The glimpse height must be specified first, following
+// by the glimpse width.
+//	offsets: A 2-D integer tensor of shape `[batch_size, 2]` containing
+// the y, x locations of the center of each window.
+//
+// Returns A tensor representing the glimpses `[batch_size,
+// glimpse_height, glimpse_width, channels]`.
+func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Output, optional ...ExtractGlimpseAttr) (glimpse tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExtractGlimpse",
+		Input: []tf.Input{
+			input, size, offsets,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// A container for an iterator resource.
+//
+// Returns A handle to the iterator that can be passed to a "MakeIterator"
+// or "IteratorGetNext" op.
+func Iterator(scope *Scope, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "Iterator",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
+type CropAndResizeGradImageAttr func(optionalAttr)
+
+// CropAndResizeGradImageMethod sets the optional method attribute to value.
+//
+// value: A string specifying the interpolation method. Only 'bilinear' is
+// supported for now.
+// If not specified, defaults to "bilinear"
+func CropAndResizeGradImageMethod(value string) CropAndResizeGradImageAttr {
+	return func(m optionalAttr) {
+		m["method"] = value
+	}
+}
+
+// Computes the gradient of the crop_and_resize op wrt the input image tensor.
+//
+// Arguments:
+//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+// specifies the coordinates of a box in the `box_ind[i]` image and is specified
+// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
+// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
+// `[0, 1]` interval of normalized image height is mapped to
+// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
+// which case the sampled crop is an up-down flipped version of the original
+// image. The width dimension is treated similarly. Normalized coordinates
+// outside the `[0, 1]` range are allowed, in which case we use
+// `extrapolation_value` to extrapolate the input image values.
+//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
+// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
+//	image_size: A 1-D tensor with value `[batch, image_height, image_width, depth]`
+// containing the original image size. Both `image_height` and `image_width` need
+// to be positive.
+//
 //
-// Returns A tensor representing the glimpses `[batch_size,
-// glimpse_height, glimpse_width, channels]`.
-func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Output, optional ...ExtractGlimpseAttr) (glimpse tf.Output) {
+// Returns A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+func CropAndResizeGradImage(scope *Scope, grads tf.Output, boxes tf.Output, box_ind tf.Output, image_size tf.Output, T tf.DataType, optional ...CropAndResizeGradImageAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"T": T}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ExtractGlimpse",
+		Type: "CropAndResizeGradImage",
 		Input: []tf.Input{
-			input, size, offsets,
+			grads, boxes, box_ind, image_size,
 		},
 		Attrs: attrs,
 	}
@@ -20832,24 +21069,6 @@ func ExtractGlimpse(scope *Scope, input tf.Output, size tf.Output, offsets tf.Ou
 	return op.Output(0)
 }
 
-// A container for an iterator resource.
-//
-// Returns A handle to the iterator that can be passed to a "MakeIterator"
-// or "IteratorGetNext" op.
-func Iterator(scope *Scope, shared_name string, container string, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"shared_name": shared_name, "container": container, "output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "Iterator",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // ShuffleDatasetAttr is an optional argument to ShuffleDataset.
 type ShuffleDatasetAttr func(optionalAttr)
 
@@ -21717,47 +21936,6 @@ func CacheDataset(scope *Scope, input_dataset tf.Output, filename tf.Output, out
 	return op.Output(0)
 }
 
-// PlaceholderAttr is an optional argument to Placeholder.
-type PlaceholderAttr func(optionalAttr)
-
-// PlaceholderShape sets the optional shape attribute to value.
-//
-// value: (Optional) The shape of the tensor. If the shape has 0 dimensions, the
-// shape is unconstrained.
-// If not specified, defaults to <unknown_rank:true >
-func PlaceholderShape(value tf.Shape) PlaceholderAttr {
-	return func(m optionalAttr) {
-		m["shape"] = value
-	}
-}
-
-// A placeholder op for a value that will be fed into the computation.
-//
-// N.B. This operation will fail with an error if it is executed. It is
-// intended as a way to represent a value that will always be fed, and to
-// provide attrs that enable the fed value to be checked at runtime.
-//
-// Arguments:
-//	dtype: The type of elements in the tensor.
-//
-// Returns A placeholder tensor that must be replaced using the feed mechanism.
-func Placeholder(scope *Scope, dtype tf.DataType, optional ...PlaceholderAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Placeholder",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Creates a dataset that executes a SQL query and emits rows of the result set.
 //
 // Arguments:
@@ -23339,101 +23517,6 @@ func TensorArrayCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	return scope.AddOperation(opspec)
 }
 
-// CropAndResizeGradImageAttr is an optional argument to CropAndResizeGradImage.
-type CropAndResizeGradImageAttr func(optionalAttr)
-
-// CropAndResizeGradImageMethod sets the optional method attribute to value.
-//
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
-// If not specified, defaults to "bilinear"
-func CropAndResizeGradImageMethod(value string) CropAndResizeGradImageAttr {
-	return func(m optionalAttr) {
-		m["method"] = value
-	}
-}
-
-// Computes the gradient of the crop_and_resize op wrt the input image tensor.
-//
-// Arguments:
-//	grads: A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
-//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
-// specifies the coordinates of a box in the `box_ind[i]` image and is specified
-// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
-// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
-// `[0, 1]` interval of normalized image height is mapped to
-// `[0, image_height - 1] in image height coordinates. We do allow y1 > y2, in
-// which case the sampled crop is an up-down flipped version of the original
-// image. The width dimension is treated similarly. Normalized coordinates
-// outside the `[0, 1]` range are allowed, in which case we use
-// `extrapolation_value` to extrapolate the input image values.
-//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
-// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
-//	image_size: A 1-D tensor with value `[batch, image_height, image_width, depth]`
-// containing the original image size. Both `image_height` and `image_width` need
-// to be positive.
-//
-//
-// Returns A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-func CropAndResizeGradImage(scope *Scope, grads tf.Output, boxes tf.Output, box_ind tf.Output, image_size tf.Output, T tf.DataType, optional ...CropAndResizeGradImageAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"T": T}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "CropAndResizeGradImage",
-		Input: []tf.Input{
-			grads, boxes, box_ind, image_size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Reads and outputs the entire contents of the input filename.
-func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReadFile",
-		Input: []tf.Input{
-			filename,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Concatenates tensors along one dimension.
-//
-// Arguments:
-//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [-rank(values), rank(values)).
-//
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.
-func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ConcatV2",
-		Input: []tf.Input{
-			tf.OutputList(values), axis,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Forwards the value of an available tensor from `inputs` to `output`.
 //
 // `Merge` waits for at least one of the tensors in `inputs` to become available.
@@ -27804,86 +27887,3 @@ func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Outp
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1)
 }
-
-// Pads a tensor with mirrored values.
-//
-// This operation pads a `input` with mirrored values according to the `paddings`
-// you specify. `paddings` is an integer tensor with shape `[n, 2]`, where n is
-// the rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many values to add before the contents of `input` in that dimension, and
-// `paddings[D, 1]` indicates how many values to add after the contents of `input`
-// in that dimension. Both `paddings[D, 0]` and `paddings[D, 1]` must be no greater
-// than `input.dim_size(D)` (or `input.dim_size(D) - 1`) if `copy_border` is true
-// (if false, respectively).
-//
-// The padded size of each dimension D of the output is:
-//
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
-//
-// For example:
-//
-// ```
-// # 't' is [[1, 2, 3], [4, 5, 6]].
-// # 'paddings' is [[1, 1]], [2, 2]].
-// # 'mode' is SYMMETRIC.
-// # rank of 't' is 2.
-// pad(t, paddings) ==> [[2, 1, 1, 2, 3, 3, 2]
-//                       [2, 1, 1, 2, 3, 3, 2]
-//                       [5, 4, 4, 5, 6, 6, 5]
-//                       [5, 4, 4, 5, 6, 6, 5]]
-// ```
-//
-// Arguments:
-//	input: The input tensor to be padded.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	mode: Either `REFLECT` or `SYMMETRIC`. In reflect mode the padded regions
-// do not include the borders, while in symmetric mode the padded regions
-// do include the borders. For example, if `input` is `[1, 2, 3]` and `paddings`
-// is `[0, 2]`, then the output is `[1, 2, 3, 2, 1]` in reflect mode, and
-// it is `[1, 2, 3, 3, 2]` in symmetric mode.
-//
-// Returns The padded tensor.
-func MirrorPad(scope *Scope, input tf.Output, paddings tf.Output, mode string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"mode": mode}
-	opspec := tf.OpSpec{
-		Type: "MirrorPad",
-		Input: []tf.Input{
-			input, paddings,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// A placeholder op for a value that will be fed into the computation.
-//
-// DEPRECATED at GraphDef version 23: Placeholder now behaves the same as PlaceholderV2.
-//
-// N.B. This operation will fail with an error if it is executed. It is
-// intended as a way to represent a value that will always be fed, and to
-// provide attrs that enable the fed value to be checked at runtime.
-//
-// Arguments:
-//	dtype: The type of elements in the tensor.
-//	shape: The shape of the tensor. The shape can be any partially-specified
-// shape.  To be unconstrained, pass in a shape with unknown rank.
-//
-// Returns A placeholder tensor that must be replaced using the feed mechanism.
-func PlaceholderV2(scope *Scope, dtype tf.DataType, shape tf.Shape) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
-	opspec := tf.OpSpec{
-		Type: "PlaceholderV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-- 
GitLab


From e9d6c89aaaf65db2dbaacacdfecdee4a56a3cb7d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 18:39:29 -0700
Subject: [PATCH 464/960] Switch Android C++ compilation mode to "-std=c++11".

PiperOrigin-RevId: 190306256
---
 tensorflow/tools/ci_build/builds/android.sh      | 3 ++-
 tensorflow/tools/ci_build/builds/android_full.sh | 6 ++++--
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/ci_build/builds/android.sh b/tensorflow/tools/ci_build/builds/android.sh
index 564c5aa148..d81793efe0 100755
--- a/tensorflow/tools/ci_build/builds/android.sh
+++ b/tensorflow/tools/ci_build/builds/android.sh
@@ -29,7 +29,8 @@ echo "========== TensorFlow Demo Build Test =========="
 # Enable sandboxing so that zip archives don't get incorrectly packaged
 # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334)
 # TODO(gunan): remove extra flags once sandboxing is enabled for all builds.
-bazel --bazelrc=/dev/null build -c opt --fat_apk_cpu=x86_64 \
+bazel --bazelrc=/dev/null build \
+    --compilation_mode=opt --cxxopt=-std=c++11 --fat_apk_cpu=x86_64 \
     --spawn_strategy=sandboxed --genrule_strategy=sandboxed \
     //tensorflow/examples/android:tensorflow_demo
 
diff --git a/tensorflow/tools/ci_build/builds/android_full.sh b/tensorflow/tools/ci_build/builds/android_full.sh
index 9d449241e8..41dc66dd54 100755
--- a/tensorflow/tools/ci_build/builds/android_full.sh
+++ b/tensorflow/tools/ci_build/builds/android_full.sh
@@ -40,7 +40,8 @@ rm -rf ${AAR_LIB_TMP}
 for CPU in ${CPUS//,/ }
 do
     echo "========== Building native libs for Android ${CPU} =========="
-    bazel build -c opt --config=monolithic --cpu=${CPU} \
+    bazel build --config=monolithic --cpu=${CPU} \
+        --compilation_mode=opt --cxxopt=-std=c++11 \
         --crosstool_top=//external:android/crosstool \
         --host_crosstool_top=@bazel_tools//tools/cpp:toolchain \
         //tensorflow/core:android_tensorflow_lib \
@@ -62,7 +63,8 @@ done
 # in assets/ dir (see https://github.com/bazelbuild/bazel/issues/2334)
 # TODO(gunan): remove extra flags once sandboxing is enabled for all builds.
 echo "========== Building TensorFlow Android Jar and Demo =========="
-bazel --bazelrc=/dev/null build -c opt --config=monolithic --fat_apk_cpu=${CPUS} \
+bazel --bazelrc=/dev/null build --config=monolithic --fat_apk_cpu=${CPUS} \
+    --compilation_mode=opt --cxxopt=-std=c++11 \
     --spawn_strategy=sandboxed --genrule_strategy=sandboxed \
     //tensorflow/contrib/android:android_tensorflow_inference_java \
     //tensorflow/contrib/android:android_tensorflow_inference_java.aar \
-- 
GitLab


From f3f58f4486731faf4137fa62cdf1f885dccfc95b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 18:44:35 -0700
Subject: [PATCH 465/960] When import_scoped_meta_graph is called within a name
 scope, but called without an import_scope, the names of the created variables
 are wrong, resulting in key not found errors when adding these variables to
 their corresponding collections.

PiperOrigin-RevId: 190306555
---
 tensorflow/python/framework/meta_graph.py      |  4 +++-
 tensorflow/python/framework/meta_graph_test.py | 15 +++++++++++++++
 tensorflow/python/framework/ops.py             |  6 ++++++
 3 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/meta_graph.py b/tensorflow/python/framework/meta_graph.py
index 4bb9941bb7..391b17720c 100644
--- a/tensorflow/python/framework/meta_graph.py
+++ b/tensorflow/python/framework/meta_graph.py
@@ -737,7 +737,9 @@ def import_scoped_meta_graph(meta_graph_or_file,
         import_scope or "", mark_as_used=False)
 
     importer.import_graph_def(
-        input_graph_def, name=(import_scope or ""), input_map=input_map,
+        input_graph_def,
+        name=(import_scope or scope_to_prepend_to_names),
+        input_map=input_map,
         producer_op_list=producer_op_list)
 
     # Restores all the other collections.
diff --git a/tensorflow/python/framework/meta_graph_test.py b/tensorflow/python/framework/meta_graph_test.py
index 21963d0bee..5d5fb037fc 100644
--- a/tensorflow/python/framework/meta_graph_test.py
+++ b/tensorflow/python/framework/meta_graph_test.py
@@ -537,6 +537,21 @@ class ScopedMetaGraphTest(test.TestCase):
         self.assertEqual(list(imported_variables.values())[0].name,
                          "foo/bar/myvar:0")
 
+  def testScopedImportUnderNameScopeNoVarScope(self):
+    graph = ops.Graph()
+    with graph.as_default():
+      variables.Variable(initial_value=1.0, trainable=True, name="myvar")
+    meta_graph_def, _ = meta_graph.export_scoped_meta_graph(graph=graph)
+
+    graph = ops.Graph()
+    with graph.as_default():
+      with ops.name_scope("foo"):
+        imported_variables = meta_graph.import_scoped_meta_graph(
+            meta_graph_def)
+        self.assertEqual(len(imported_variables), 1)
+        self.assertEqual(list(imported_variables.values())[0].name,
+                         "foo/myvar:0")
+
   def testImportsUsingSameScopeName(self):
     with ops.Graph().as_default():
       variables.Variable(0, name="v")
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 1fa9285e43..f264e38102 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5872,6 +5872,9 @@ def strip_name_scope(name, export_scope):
     is None.
   """
   if export_scope:
+    if export_scope[-1] == "/":
+      export_scope = export_scope[:-1]
+
     try:
       # Strips export_scope/, export_scope///,
       # ^export_scope/, loc:@export_scope/.
@@ -5897,6 +5900,9 @@ def prepend_name_scope(name, import_scope):
     is None.
   """
   if import_scope:
+    if import_scope[-1] == "/":
+      import_scope = import_scope[:-1]
+
     try:
       str_to_replace = r"([\^]|loc:@|^)(.*)"
       return re.sub(str_to_replace, r"\1" + import_scope + r"/\2",
-- 
GitLab


From 753f99afcd6b814781e19ae44afc6195ff68685d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 19:02:37 -0700
Subject: [PATCH 466/960] Adding support for iterating through a
 tf.data.Dataset for a single epoch.

PiperOrigin-RevId: 190307545
---
 .../contrib/py2tf/converters/for_loops.py     | 30 ++++----
 tensorflow/contrib/py2tf/utils/BUILD          |  1 +
 tensorflow/contrib/py2tf/utils/__init__.py    |  2 +
 tensorflow/contrib/py2tf/utils/builtins.py    | 69 ++++++++++++++++++-
 4 files changed, 86 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/py2tf/converters/for_loops.py b/tensorflow/contrib/py2tf/converters/for_loops.py
index 4297c1cf2a..8d28b149a8 100644
--- a/tensorflow/contrib/py2tf/converters/for_loops.py
+++ b/tensorflow/contrib/py2tf/converters/for_loops.py
@@ -38,19 +38,19 @@ class ForLoopCanonicalizationTransformer(transformer.Base):
     self.generic_visit(node)
     body_scope = anno.getanno(node, NodeAnno.BODY_SCOPE)
     i_var = self.context.namer.new_symbol('i', body_scope.referenced)
-    n_var = self.context.namer.new_symbol('n', body_scope.referenced)
-    iterated_var = self.context.namer.new_symbol('iterated',
-                                                 body_scope.referenced)
+    smart_loop_iter_var = self.context.namer.new_symbol('smart_loop_iter',
+                                                        body_scope.referenced)
+    cont_var = self.context.namer.new_symbol('cont', body_scope.referenced)
     # TODO(mdan): Use TensorListFromTensor(loop_iter) here.
     if anno.hasanno(node, 'extra_cond'):
       template = """
         i = 0
-        iterated = loop_iter
-        n = len(iterated)
-        while i < n and extra_cond:
-          target = iterated[i]
+        smart_loop_iter = py2tf_utils.dynamic_dataset(loop_iter)
+        cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+        while cont and extra_cond:
           body
           i += 1
+          cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
       """
       return templates.replace(
           template,
@@ -58,18 +58,18 @@ class ForLoopCanonicalizationTransformer(transformer.Base):
           target=node.target,
           body=node.body,
           i=i_var,
-          n=n_var,
-          iterated=iterated_var,
+          smart_loop_iter=smart_loop_iter_var,
+          cont=cont_var,
           extra_cond=anno.getanno(node, 'extra_cond'))
     else:
       template = """
         i = 0
-        iterated = loop_iter
-        n = len(iterated)
-        while i < n:
-          target = iterated[i]
+        smart_loop_iter = py2tf_utils.dynamic_dataset(loop_iter)
+        cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+        while cont:
           body
           i += 1
+          cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
       """
       repl = templates.replace(
           template,
@@ -77,8 +77,8 @@ class ForLoopCanonicalizationTransformer(transformer.Base):
           target=node.target,
           body=node.body,
           i=i_var,
-          n=n_var,
-          iterated=iterated_var)
+          smart_loop_iter=smart_loop_iter_var,
+          cont=cont_var)
       return repl
 
   def visit_Continue(self, node):
diff --git a/tensorflow/contrib/py2tf/utils/BUILD b/tensorflow/contrib/py2tf/utils/BUILD
index d029289f5a..b53fbb5c18 100644
--- a/tensorflow/contrib/py2tf/utils/BUILD
+++ b/tensorflow/contrib/py2tf/utils/BUILD
@@ -35,6 +35,7 @@ py_library(
     deps = [
         "//tensorflow/python:list_ops",
         "//tensorflow/python:script_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
         "@six_archive//:six",
     ],
 )
diff --git a/tensorflow/contrib/py2tf/utils/__init__.py b/tensorflow/contrib/py2tf/utils/__init__.py
index d9d8e34689..4e6003c852 100644
--- a/tensorflow/contrib/py2tf/utils/__init__.py
+++ b/tensorflow/contrib/py2tf/utils/__init__.py
@@ -19,6 +19,8 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.py2tf.utils.builtins import dynamic_builtin
+from tensorflow.contrib.py2tf.utils.builtins import dynamic_dataset
+from tensorflow.contrib.py2tf.utils.builtins import dynamic_for_cond
 from tensorflow.contrib.py2tf.utils.builtins import dynamic_print
 from tensorflow.contrib.py2tf.utils.builtins import dynamic_range
 from tensorflow.contrib.py2tf.utils.context_managers import control_dependency_on_returns
diff --git a/tensorflow/contrib/py2tf/utils/builtins.py b/tensorflow/contrib/py2tf/utils/builtins.py
index 3cb62b55d4..251b4ed8ee 100644
--- a/tensorflow/contrib/py2tf/utils/builtins.py
+++ b/tensorflow/contrib/py2tf/utils/builtins.py
@@ -22,8 +22,10 @@ import six
 
 from tensorflow.contrib.py2tf.utils import py_func
 from tensorflow.contrib.py2tf.utils import type_check
+from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import logging_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.util import tf_inspect
@@ -54,7 +56,6 @@ def dynamic_len(list_or_tensor):
       raise ValueError(
           'len requires non-zero rank for tensor "%s"' % list_or_tensor)
     return array_ops.shape(list_or_tensor)[0]
-
   return len(list_or_tensor)
 
 
@@ -97,3 +98,69 @@ def dynamic_print(*values):
   if all(map(is_tf_print_compatible, values)):
     return logging_ops.Print(1, values)
   return py_func.wrap_py_func(print, None, values, use_dummy_return=True)
+
+
+def dynamic_dataset(iterated):
+  """Implementartion of smart tf.data.Dataset epoch wrapping.
+
+  The function checks if the input is a tf.data.Dataset and if so then wraps it
+  so that for each element it returns it also returns the current epoch the
+  dataset iteration is in, for two epochs.  If the input is not a
+  tf.data.Dataset then it just returns the input.
+
+  Args:
+    iterated: The iterable or tf.data.Dataset that is being iterated over.
+  Returns:
+    Either just the untouched input, or in the case of input being a
+    tf.data.Dataset then it returns a wrapped  tf.data.Dataset where for each
+    element it returns it also returns the current epoch the dataset iteration
+    is in.
+  """
+  if not isinstance(iterated, dataset_ops.Dataset):
+    return iterated
+
+  def epoch_dataset_number_helper(i):
+    return dataset_ops.Dataset.zip(
+        (dataset_ops.Dataset.from_tensors(i).repeat(), iterated))
+
+  epoch_numbers = dataset_ops.Dataset.range(2)
+  return epoch_numbers.flat_map(epoch_dataset_number_helper)
+
+
+def dynamic_for_cond(iteration, iterated):
+  """Implementartion of smart while-loop condition using dynamic dispatch.
+
+  The function checks if it is iterating over a tf.data.Dataset or not, and in
+  the case it is not then it simply returns if we are still in range of the
+  iterated and the next element.  If it is iterating over a dataset then it only
+  iterates for a single epoch.
+
+  Args:
+    iteration: The current iteration of the loop.
+    iterated: The iterable or tf.data.Dataset that is being iterated over.
+  Returns:
+    A tuple of a bool that indicates whether the loop should continue, and the
+    next element in iterated.
+  """
+  # TODO(znado): Clean up.
+  # TODO(znado): This won't work for unpacked iterates. Fix.
+  if isinstance(iterated, dataset_ops.Dataset):
+    curr_epoch, next_elem = iterated.make_one_shot_iterator().get_next()
+    return math_ops.less(curr_epoch, 1), next_elem
+  elif tensor_util.is_tensor(iterated):
+    if iterated.shape.ndims > 1:
+      elem_shape = array_ops.shape(iterated)[1:]
+    else:
+      elem_shape = ()
+    if iterated.shape.ndims == 0 or iterated.shape[0] == 0:
+      return False, array_ops.zeros(elem_shape, iterated.dtype)
+    return control_flow_ops.cond(
+        math_ops.less(iteration, dynamic_len(iterated)),
+        lambda: (True, iterated[iteration]),
+        lambda: (False, array_ops.zeros(elem_shape, iterated.dtype)))
+  elif hasattr(iterated, '__len__'):
+    if iteration < len(iterated):
+      return True, iterated[iteration]
+    return False, None
+  else:
+    raise NotImplementedError('Python iterators not yet supported.')
-- 
GitLab


From 759e9f874eb0af7902a586e0efcaf53463816c23 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 19:24:04 -0700
Subject: [PATCH 467/960] Fix loop variable type and status propagation

PiperOrigin-RevId: 190308776
---
 tensorflow/c/eager/c_api.cc | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index c69635d529..eaeb2fd07a 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -837,7 +837,7 @@ const tensorflow::FunctionDef* OpToFunction(
   }
   VLOG(1) << "Fixed Output names and all types: " << fdef.DebugString();
 
-  ctx->context.AddFunctionDef(fdef);
+  status->status = ctx->context.AddFunctionDef(fdef);
   if (!status->status.ok()) return nullptr;
   const auto ret = ctx->context.FindFunctionDef(signature->name());
   DCHECK(ret != nullptr);
@@ -885,7 +885,7 @@ std::unique_ptr<TFE_Op> BuildXlaLaunch(TFE_Op* op, TF_Status* status) {
   // Since input param reordering may have occurred between `op` and `launch_op`
   // via `op_input_to_func_input`, adjust the actual inputs accordingly.
   launch_op->inputs = op->inputs;
-  for (TFE_TensorHandle* h : launch_op->inputs) {
+  for (tensorflow::TensorHandle* h : launch_op->inputs) {
     h->Ref();
   }
   if (!op_input_to_func_input.empty()) {
-- 
GitLab


From 418ae5ed77f1353c794f93a4adfbf7db02fa3191 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 20:18:46 -0700
Subject: [PATCH 468/960] A couple of small device-related utilities.

PiperOrigin-RevId: 190312148
---
 tensorflow/python/BUILD                   |  3 +
 tensorflow/python/training/device_util.py | 68 +++++++++++++++++++++++
 2 files changed, 71 insertions(+)
 create mode 100644 tensorflow/python/training/device_util.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index acfdcd15f7..e6ad564ede 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2884,9 +2884,11 @@ py_library(
         ":client",
         ":control_flow_ops",
         ":data_flow_ops",
+        ":device",
         ":errors",
         ":framework",
         ":framework_for_generated_wrappers",
+        ":framework_ops",
         ":gradients",
         ":init_ops",
         ":io_ops",
@@ -2911,6 +2913,7 @@ py_library(
         ":variable_scope",
         ":variables",
         "//tensorflow/python/eager:backprop",
+        "//tensorflow/python/eager:context",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/python/training/device_util.py b/tensorflow/python/training/device_util.py
new file mode 100644
index 0000000000..f1137e80ab
--- /dev/null
+++ b/tensorflow/python/training/device_util.py
@@ -0,0 +1,68 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Device-related support functions."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import context
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+
+
+def canonicalize(d):
+  d = tf_device.DeviceSpec.from_string(d)
+  assert d.device_type is None or d.device_type == d.device_type.upper(), (
+      "Device type '%s' must be all-caps." % (d.device_type,))
+  # Fill in missing device fields using defaults.
+  result = tf_device.DeviceSpec(
+      job="localhost", replica=0, task=0, device_type="CPU", device_index=0)
+  result.merge_from(d)
+  return result.to_string()
+
+
+class _FakeNodeDef(object):
+  """A fake NodeDef for _FakeOperation."""
+
+  def __init__(self):
+    self.op = ""
+    self.name = ""
+
+
+class _FakeOperation(object):
+  """A fake Operation object to pass to device functions."""
+
+  def __init__(self):
+    self.device = ""
+    self.type = ""
+    self.name = ""
+    self.node_def = _FakeNodeDef()
+
+  def _set_device(self, device):
+    self.device = ops._device_string(device)  # pylint: disable=protected-access
+
+
+def current():
+  """Return a string (not canonicalized) for the current device."""
+  # TODO(josh11b): Work out how this function interacts with ops.colocate_with.
+  ctx = context.context()
+  if ctx.executing_eagerly():
+    d = ctx.device_name
+  else:
+    op = _FakeOperation()
+    ops.get_default_graph()._apply_device_functions(op)  # pylint: disable=protected-access
+    d = op.device
+  return d
-- 
GitLab


From 0a335dc4fd8cae06d331589eab5858fd0a3ffc73 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 20:20:49 -0700
Subject: [PATCH 469/960] [XLA] Prevent using XlaOp from the wrong XlaBuilder.

PiperOrigin-RevId: 190312254
---
 .../xla/client/xla_client/xla_builder.cc      | 19 +++++++++++++------
 .../xla/client/xla_client/xla_builder.h       |  5 +++--
 .../xla/client/xla_client/xla_builder_test.cc | 11 +++++++++++
 3 files changed, 27 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 90f2b2d73a..cbcb747f1c 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -284,10 +284,12 @@ XlaOp XlaBuilder::Mul(const XlaOp& lhs, const XlaOp& rhs,
 }
 
 XlaOp XlaBuilder::ConstantLiteral(const Literal& literal) {
-  HloInstructionProto instr;
-  *instr.mutable_shape() = literal.shape();
-  *instr.mutable_literal() = literal.ToProto();
-  return AddInstruction(std::move(instr), HloOpcode::kConstant);
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    *instr.mutable_shape() = literal.shape();
+    *instr.mutable_literal() = literal.ToProto();
+    return AddInstruction(std::move(instr), HloOpcode::kConstant);
+  }());
 }
 
 XlaOp XlaBuilder::Call(const XlaComputation& computation,
@@ -794,8 +796,9 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
   return UnimplementedOp();
 }
 
-XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
-                                 tensorflow::gtl::ArraySlice<XlaOp> operands) {
+StatusOr<XlaOp> XlaBuilder::AddInstruction(
+    HloInstructionProto&& instr, HloOpcode opcode,
+    tensorflow::gtl::ArraySlice<XlaOp> operands) {
   const int64 handle = instructions_.size();
   instr.set_id(handle);
   instr.set_opcode(HloOpcodeString(opcode));
@@ -806,6 +809,10 @@ XlaOp XlaBuilder::AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
     instr.set_name(StrCat(instr.name(), ".", handle));
   }
   for (const auto& operand : operands) {
+    TF_RET_CHECK(operand.builder_ != nullptr);
+    TF_RET_CHECK(operand.builder_ == this)
+        << "Do not add XlaOp from builder " << operand.builder_->name()
+        << " to builder " << this->name();
     instr.add_operand_ids(operand.handle());
     // TODO(b/74197823): Set metadata and sharding.
   }
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 407b2df274..99d1db7790 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -706,8 +706,9 @@ class XlaBuilder {
   StatusOr<ProgramShape> GetProgramShape();
 
  private:
-  XlaOp AddInstruction(HloInstructionProto&& instr, HloOpcode opcode,
-                       tensorflow::gtl::ArraySlice<XlaOp> operands = {});
+  StatusOr<XlaOp> AddInstruction(
+      HloInstructionProto&& instr, HloOpcode opcode,
+      tensorflow::gtl::ArraySlice<XlaOp> operands = {});
 
   // Notes that the error occurred by:
   // * storing it internally and capturing a backtrace if it's the first error
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
index 10d8fa1622..57dcfc4d4d 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
@@ -179,5 +179,16 @@ TEST_F(XlaBuilderTest, BinopHasInDimAndDegenerateBroadcast) {
                             op::Broadcast(op::Reshape(op::Parameter(1)))));
 }
 
+TEST_F(XlaBuilderTest, OperandFromWrongBuilder) {
+  XlaBuilder b1("b1");
+  auto p0 = b1.Parameter(0, ShapeUtil::MakeShape(F32, {}), "p0");
+  XlaBuilder builder("main");
+  builder.Add(p0, p0);
+  auto statusor = builder.Build();
+  ASSERT_FALSE(statusor.ok());
+  EXPECT_THAT(statusor.status().error_message(),
+              HasSubstr("Do not add XlaOp from builder b1 to builder main"));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 917b79250b0e65aa7856b2418b68292d919cd5dc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 23 Mar 2018 22:27:31 -0700
Subject: [PATCH 470/960] [XLA] Redesign: implement Reshape and Transpose.

Also,
- Templatize ClientLibraryTestBase::CreateParameterAndTransferLiteral. The implementation is moved from .cc to .h because otherewise the linker complains.
- Migrate some reshape tests to use the XlaBuilder.

PiperOrigin-RevId: 190317960
---
 .../xla/client/xla_client/xla_builder.cc      | 30 ++++++++++++--
 .../xla/client/xla_client/xla_builder.h       |  3 +-
 .../xla/client/xla_client/xla_builder_test.cc | 27 +++++++++++++
 tensorflow/compiler/xla/tests/BUILD           |  1 +
 .../xla/tests/client_library_test_base.cc     | 27 -------------
 .../xla/tests/client_library_test_base.h      | 39 +++++++++++++++++--
 tensorflow/compiler/xla/tests/reshape_test.cc | 35 +++++++++--------
 7 files changed, 111 insertions(+), 51 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index cbcb747f1c..596f39b4fd 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -401,12 +401,26 @@ XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value,
 XlaOp XlaBuilder::Reshape(const XlaOp& operand,
                           tensorflow::gtl::ArraySlice<int64> dimensions,
                           tensorflow::gtl::ArraySlice<int64> new_sizes) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
+    TF_ASSIGN_OR_RETURN(const Shape& shape,
+                        ShapeInference::InferReshapeShape(
+                            operand_shape, dimensions, new_sizes));
+    XlaOp transposed = IsIdentityPermutation(dimensions)
+                           ? operand
+                           : Transpose(operand, dimensions);
+    return Reshape(shape, transposed);
+  }());
 }
 
 XlaOp XlaBuilder::Reshape(const XlaOp& operand,
                           tensorflow::gtl::ArraySlice<int64> new_sizes) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(auto shape, operand.GetShape());
+    std::vector<int64> dimensions(shape.dimensions_size());
+    std::iota(dimensions.begin(), dimensions.end(), 0);
+    return Reshape(operand, dimensions, new_sizes);
+  }());
 }
 
 XlaOp XlaBuilder::Collapse(const XlaOp& operand,
@@ -636,7 +650,17 @@ XlaOp XlaBuilder::IsFinite(const XlaOp& operand) { return UnimplementedOp(); }
 
 XlaOp XlaBuilder::Transpose(const XlaOp& operand,
                             tensorflow::gtl::ArraySlice<int64> permutation) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferTransposeShape(operand_shape, permutation));
+    for (int64 dim : permutation) {
+      instr.add_dimensions(dim);
+    }
+    return AddInstruction(std::move(instr), HloOpcode::kTranspose, {operand});
+  }());
 }
 
 XlaOp XlaBuilder::Rev(const XlaOp& operand,
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 99d1db7790..c19eb47165 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -52,10 +52,11 @@ class XlaBuilder;
 // TODO(b/74197823): Replace xla::ComputationDataHandle with this one.
 class XlaOp {
  public:
+  XlaOp() : handle_(0), builder_(nullptr) {}
+
   StatusOr<Shape> GetShape() const;
 
  private:
-  XlaOp() : handle_(0), builder_(nullptr) {}
   XlaOp(int64 handle, XlaBuilder* builder)
       : handle_(handle), builder_(builder) {}
 
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
index 57dcfc4d4d..529287a57a 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
@@ -190,5 +190,32 @@ TEST_F(XlaBuilderTest, OperandFromWrongBuilder) {
               HasSubstr("Do not add XlaOp from builder b1 to builder main"));
 }
 
+TEST_F(XlaBuilderTest, ReshapeDefaultOrder) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {2, 3, 5, 7}), "x");
+  b.Reshape(x, /*new_sizes=*/{6, 35});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Reshape(op::Parameter()));
+}
+
+TEST_F(XlaBuilderTest, ReshapeHasTranspose) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {2, 3, 5, 7}), "x");
+  b.Reshape(x, /*dimensions=*/{3, 2, 1, 0}, /*new_sizes=*/{6, 35});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Reshape(op::Transpose(op::Parameter())));
+}
+
+TEST_F(XlaBuilderTest, Transpose) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(F32, {5, 7}), "x");
+  b.Transpose(x, /*permutation=*/{1, 0});
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  auto root = module->entry_computation()->root_instruction();
+  EXPECT_THAT(root, op::Transpose(op::Parameter()));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index e81e862c49..26022278e5 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1374,6 +1374,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index 3cae51576f..d9bd1ce6eb 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -568,33 +568,6 @@ ClientLibraryTestBase::CreatePatternedMatrixWithZeroPadding(int rows, int cols,
   return array;
 }
 
-std::unique_ptr<GlobalData>
-ClientLibraryTestBase::CreateParameterAndTransferLiteral(
-    int64 parameter_number, const Literal& literal, const string& name,
-    ComputationBuilder* builder, ComputationDataHandle* data_handle) {
-  return CreateParameterAndTransferLiteral(parameter_number, literal, name,
-                                           nullptr, builder, data_handle);
-}
-
-std::unique_ptr<GlobalData>
-ClientLibraryTestBase::CreateParameterAndTransferLiteral(
-    int64 parameter_number, const Literal& literal, const string& name,
-    const DeviceHandle* device_handle, ComputationBuilder* builder,
-    ComputationDataHandle* data_handle) {
-  const Literal* param_literal = &literal;
-  std::unique_ptr<Literal> converted_literal;
-  if (use_bfloat16_) {
-    converted_literal = LiteralTestUtil::ConvertF32ToBF16(literal);
-    param_literal = converted_literal.get();
-  }
-  std::unique_ptr<GlobalData> data =
-      client_->TransferToServer(*param_literal, device_handle)
-          .ConsumeValueOrDie();
-  *data_handle =
-      builder->Parameter(parameter_number, param_literal->shape(), name);
-  return data;
-}
-
 ComputationDataHandle ClientLibraryTestBase::AddParam(
     const Literal& argument, ComputationBuilder* builder) {
   ComputationDataHandle data_handle;
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index b553beb01a..01aa6c756f 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -278,17 +278,19 @@ class ClientLibraryTestBase : public ::testing::Test {
   // server, then stores into "data_handle" the global handle for that
   // parameter. When the use_bfloat16 flag is set but the literal has F32
   // elements, the literal will be converted to BF16 before being transferred.
+  template <typename BuilderT, typename HandleT>
   std::unique_ptr<GlobalData> CreateParameterAndTransferLiteral(
       int64 parameter_number, const Literal& literal, const string& name,
-      ComputationBuilder* builder, ComputationDataHandle* data_handle);
+      BuilderT* builder, HandleT* data_handle);
 
   // As above, but the caller can specify the device that the literal is
   // transferred to. If device_handle is nullptr, the literal will be
   // transferred to the default device.
+  template <typename BuilderT, typename HandleT>
   std::unique_ptr<GlobalData> CreateParameterAndTransferLiteral(
       int64 parameter_number, const Literal& literal, const string& name,
-      const DeviceHandle* device_handle, ComputationBuilder* builder,
-      ComputationDataHandle* data_handle);
+      const DeviceHandle* device_handle, BuilderT* builder,
+      HandleT* data_handle);
 
   // Creates a parameter instruction and sets the value that will be passed to
   // the computation as specified. This function must be used for all parameters
@@ -652,6 +654,37 @@ std::unique_ptr<Array2D<NativeT>> ClientLibraryTestBase::CreatePseudorandomR2(
   return result;
 }
 
+template <typename BuilderT, typename HandleT>
+std::unique_ptr<GlobalData>
+ClientLibraryTestBase::CreateParameterAndTransferLiteral(int64 parameter_number,
+                                                         const Literal& literal,
+                                                         const string& name,
+                                                         BuilderT* builder,
+                                                         HandleT* data_handle) {
+  return CreateParameterAndTransferLiteral(parameter_number, literal, name,
+                                           nullptr, builder, data_handle);
+}
+
+template <typename BuilderT, typename HandleT>
+std::unique_ptr<GlobalData>
+ClientLibraryTestBase::CreateParameterAndTransferLiteral(
+    int64 parameter_number, const Literal& literal, const string& name,
+    const DeviceHandle* device_handle, BuilderT* builder,
+    HandleT* data_handle) {
+  const Literal* param_literal = &literal;
+  std::unique_ptr<Literal> converted_literal;
+  if (use_bfloat16_) {
+    converted_literal = LiteralTestUtil::ConvertF32ToBF16(literal);
+    param_literal = converted_literal.get();
+  }
+  std::unique_ptr<GlobalData> data =
+      client_->TransferToServer(*param_literal, device_handle)
+          .ConsumeValueOrDie();
+  *data_handle =
+      builder->Parameter(parameter_number, param_literal->shape(), name);
+  return data;
+}
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_TESTS_CLIENT_LIBRARY_TEST_BASE_H_
diff --git a/tensorflow/compiler/xla/tests/reshape_test.cc b/tensorflow/compiler/xla/tests/reshape_test.cc
index f7b04debd4..02272d6017 100644
--- a/tensorflow/compiler/xla/tests/reshape_test.cc
+++ b/tensorflow/compiler/xla/tests/reshape_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/reference_util.h"
@@ -207,9 +208,9 @@ XLA_TEST_P(ReshapeTest, Trivial3x1) {
 //
 // Splits an empty vector into an empty matrix.
 XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(R1ToR2_0_To_2x0)) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal = Literal::CreateR1<float>({});
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{0},
@@ -221,10 +222,10 @@ XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(R1ToR2_0_To_2x0)) {
 
 // Splits a vector into a matrix.
 XLA_TEST_P(ReshapeTest, R1ToR2_6_To_2x3) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal =
       Literal::CreateR1<float>({1.0f, 2.0f, 3.0f, 4.0f, 5.0f, 6.0f});
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{0},
@@ -241,9 +242,9 @@ XLA_TEST_P(ReshapeTest, R1ToR2_6_To_2x3) {
 //
 // Transposes a 2x0 array to a 0x2 array.
 XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(Reshape0x2To2x0)) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal = Literal::CreateFromArray(Array2D<float>(0, 2));
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{0, 1},
@@ -255,10 +256,10 @@ XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(Reshape0x2To2x0)) {
 
 // Transposes a 2-dimensional row vector to a column vector.
 XLA_TEST_P(ReshapeTest, ReshapeRowToCol) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto simple = MakeLinspaceArray2D(1.0f, 3.0f, 1, 3);
   auto input_literal = Literal::CreateFromArray(*simple);
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{0, 1},
@@ -272,10 +273,10 @@ XLA_TEST_P(ReshapeTest, ReshapeRowToCol) {
 
 // Transposes a 2-dimensional array.
 XLA_TEST_P(ReshapeTest, TransposeAsReshape) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a4x3 = MakeLinspaceArray2D(1.0f, 12.0f, 4, 3);
   auto input_literal = Literal::CreateFromArray(*a4x3);
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{1, 0},
@@ -291,11 +292,11 @@ XLA_TEST_P(ReshapeTest, TransposeAsReshape) {
 // does not handle zero-sized shapes correctly. Failed last on 2017-11-30
 // with an incorrect result rank.
 //
-// Transposes a 0x4 array with ComputationBuilder::Trans.
+// Transposes a 0x4 array with XlaBuilder::Transpose.
 XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(Transpose0x4)) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal = Literal::CreateFromArray(Array2D<float>(0, 4));
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Transpose(parameter, {1, 0});
@@ -306,10 +307,10 @@ XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(Transpose0x4)) {
 
 // Transposes a 2-dimensional array with ComputationBuilder::Trans.
 XLA_TEST_P(ReshapeTest, Transpose4x3) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a4x3 = MakeLinspaceArray2D(1.0f, 12.0f, 4, 3);
   auto input_literal = Literal::CreateFromArray(*a4x3);
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Transpose(parameter, {1, 0});
@@ -327,9 +328,9 @@ XLA_TEST_P(ReshapeTest, Transpose4x3) {
 // Reshapes an empty 2-dimensional array with dimensions that are not just a
 // rearrangement of the originals (split), but no reordering (no shuffle).
 XLA_TEST_P(ReshapeTest, DISABLED_ON_GPU(ReshapeSplitNoShuffleZeroElements)) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal = Literal::CreateFromArray(Array2D<float>(6, 0));
-  ComputationDataHandle parameter;
+  XlaOp parameter;
   auto input = CreateParameterAndTransferLiteral(0, *input_literal, "input",
                                                  &builder, &parameter);
   builder.Reshape(/*operand=*/parameter, /*dimensions=*/{0, 1},
-- 
GitLab


From f95347a96c431b63183856128bfea3943585f938 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Fri, 23 Mar 2018 23:04:03 -0700
Subject: [PATCH 471/960] Trivial update of layer imports in eager execution
 examples, to reflect recommended practices.

PiperOrigin-RevId: 190319480
---
 .../eager/python/examples/gan/mnist.py        | 21 ++++-----
 .../linear_regression/linear_regression.py    |  4 +-
 .../python/examples/resnet50/resnet50.py      | 43 ++++++++++---------
 .../examples/rnn_colorbot/rnn_colorbot.py     |  6 ++-
 .../eager/python/examples/rnn_ptb/rnn_ptb.py  |  8 ++--
 5 files changed, 46 insertions(+), 36 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/gan/mnist.py b/tensorflow/contrib/eager/python/examples/gan/mnist.py
index 2b7e199fad..b80c909023 100644
--- a/tensorflow/contrib/eager/python/examples/gan/mnist.py
+++ b/tensorflow/contrib/eager/python/examples/gan/mnist.py
@@ -32,6 +32,7 @@ import tensorflow as tf
 import tensorflow.contrib.eager as tfe
 from tensorflow.examples.tutorials.mnist import input_data
 
+layers = tf.keras.layers
 FLAGS = None
 
 
@@ -56,15 +57,15 @@ class Discriminator(tf.keras.Model):
     else:
       assert data_format == 'channels_last'
       self._input_shape = [-1, 28, 28, 1]
-    self.conv1 = tf.layers.Conv2D(
+    self.conv1 = layers.Conv2D(
         64, 5, padding='SAME', data_format=data_format, activation=tf.tanh)
-    self.pool1 = tf.layers.AveragePooling2D(2, 2, data_format=data_format)
-    self.conv2 = tf.layers.Conv2D(
+    self.pool1 = layers.AveragePooling2D(2, 2, data_format=data_format)
+    self.conv2 = layers.Conv2D(
         128, 5, data_format=data_format, activation=tf.tanh)
-    self.pool2 = tf.layers.AveragePooling2D(2, 2, data_format=data_format)
-    self.flatten = tf.layers.Flatten()
-    self.fc1 = tf.layers.Dense(1024, activation=tf.tanh)
-    self.fc2 = tf.layers.Dense(1, activation=None)
+    self.pool2 = layers.AveragePooling2D(2, 2, data_format=data_format)
+    self.flatten = layers.Flatten()
+    self.fc1 = layers.Dense(1024, activation=tf.tanh)
+    self.fc2 = layers.Dense(1, activation=None)
 
   def call(self, inputs):
     """Return two logits per image estimating input authenticity.
@@ -112,16 +113,16 @@ class Generator(tf.keras.Model):
     else:
       assert data_format == 'channels_last'
       self._pre_conv_shape = [-1, 6, 6, 128]
-    self.fc1 = tf.layers.Dense(6 * 6 * 128, activation=tf.tanh)
+    self.fc1 = layers.Dense(6 * 6 * 128, activation=tf.tanh)
 
     # In call(), we reshape the output of fc1 to _pre_conv_shape
 
     # Deconvolution layer. Resulting image shape: (batch, 14, 14, 64)
-    self.conv1 = tf.layers.Conv2DTranspose(
+    self.conv1 = layers.Conv2DTranspose(
         64, 4, strides=2, activation=None, data_format=data_format)
 
     # Deconvolution layer. Resulting image shape: (batch, 28, 28, 1)
-    self.conv2 = tf.layers.Conv2DTranspose(
+    self.conv2 = layers.Conv2DTranspose(
         1, 2, strides=2, activation=tf.nn.sigmoid, data_format=data_format)
 
   def call(self, inputs):
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
index 6ab847cb78..4e1380afb2 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/linear_regression.py
@@ -32,6 +32,8 @@ import tensorflow as tf
 
 import tensorflow.contrib.eager as tfe
 
+layers = tf.keras.layers
+
 
 class LinearModel(tf.keras.Model):
   """A TensorFlow linear regression model."""
@@ -39,7 +41,7 @@ class LinearModel(tf.keras.Model):
   def __init__(self):
     """Constructs a LinearModel object."""
     super(LinearModel, self).__init__()
-    self._hidden_layer = tf.layers.Dense(1)
+    self._hidden_layer = layers.Dense(1)
 
   def call(self, xs):
     """Invoke the linear model.
diff --git a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py
index 6b59413141..a28bc8a43d 100644
--- a/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py
+++ b/tensorflow/contrib/eager/python/examples/resnet50/resnet50.py
@@ -28,6 +28,8 @@ import functools
 
 import tensorflow as tf
 
+layers = tf.keras.layers
+
 
 class _IdentityBlock(tf.keras.Model):
   """_IdentityBlock is the block that has no conv layer at shortcut.
@@ -49,23 +51,23 @@ class _IdentityBlock(tf.keras.Model):
     bn_name_base = 'bn' + str(stage) + block + '_branch'
     bn_axis = 1 if data_format == 'channels_first' else 3
 
-    self.conv2a = tf.layers.Conv2D(
+    self.conv2a = layers.Conv2D(
         filters1, (1, 1), name=conv_name_base + '2a', data_format=data_format)
-    self.bn2a = tf.layers.BatchNormalization(
+    self.bn2a = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2a')
 
-    self.conv2b = tf.layers.Conv2D(
+    self.conv2b = layers.Conv2D(
         filters2,
         kernel_size,
         padding='same',
         data_format=data_format,
         name=conv_name_base + '2b')
-    self.bn2b = tf.layers.BatchNormalization(
+    self.bn2b = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2b')
 
-    self.conv2c = tf.layers.Conv2D(
+    self.conv2c = layers.Conv2D(
         filters3, (1, 1), name=conv_name_base + '2c', data_format=data_format)
-    self.bn2c = tf.layers.BatchNormalization(
+    self.bn2c = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2c')
 
   def call(self, input_tensor, training=False):
@@ -113,34 +115,34 @@ class _ConvBlock(tf.keras.Model):
     bn_name_base = 'bn' + str(stage) + block + '_branch'
     bn_axis = 1 if data_format == 'channels_first' else 3
 
-    self.conv2a = tf.layers.Conv2D(
+    self.conv2a = layers.Conv2D(
         filters1, (1, 1),
         strides=strides,
         name=conv_name_base + '2a',
         data_format=data_format)
-    self.bn2a = tf.layers.BatchNormalization(
+    self.bn2a = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2a')
 
-    self.conv2b = tf.layers.Conv2D(
+    self.conv2b = layers.Conv2D(
         filters2,
         kernel_size,
         padding='same',
         name=conv_name_base + '2b',
         data_format=data_format)
-    self.bn2b = tf.layers.BatchNormalization(
+    self.bn2b = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2b')
 
-    self.conv2c = tf.layers.Conv2D(
+    self.conv2c = layers.Conv2D(
         filters3, (1, 1), name=conv_name_base + '2c', data_format=data_format)
-    self.bn2c = tf.layers.BatchNormalization(
+    self.bn2c = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '2c')
 
-    self.conv_shortcut = tf.layers.Conv2D(
+    self.conv_shortcut = layers.Conv2D(
         filters3, (1, 1),
         strides=strides,
         name=conv_name_base + '1',
         data_format=data_format)
-    self.bn_shortcut = tf.layers.BatchNormalization(
+    self.bn_shortcut = layers.BatchNormalization(
         axis=bn_axis, name=bn_name_base + '1')
 
   def call(self, input_tensor, training=False):
@@ -219,15 +221,15 @@ class ResNet50(tf.keras.Model):
       return _IdentityBlock(
           3, filters, stage=stage, block=block, data_format=data_format)
 
-    self.conv1 = tf.layers.Conv2D(
+    self.conv1 = layers.Conv2D(
         64, (7, 7),
         strides=(2, 2),
         data_format=data_format,
         padding='same',
         name='conv1')
     bn_axis = 1 if data_format == 'channels_first' else 3
-    self.bn_conv1 = tf.layers.BatchNormalization(axis=bn_axis, name='bn_conv1')
-    self.max_pool = tf.layers.MaxPooling2D(
+    self.bn_conv1 = layers.BatchNormalization(axis=bn_axis, name='bn_conv1')
+    self.max_pool = layers.MaxPooling2D(
         (3, 3), strides=(2, 2), data_format=data_format)
 
     self.l2a = conv_block([64, 64, 256], stage=2, block='a', strides=(1, 1))
@@ -250,11 +252,12 @@ class ResNet50(tf.keras.Model):
     self.l5b = id_block([512, 512, 2048], stage=5, block='b')
     self.l5c = id_block([512, 512, 2048], stage=5, block='c')
 
-    self.avg_pool = tf.layers.AveragePooling2D(
+    self.avg_pool = layers.AveragePooling2D(
         (7, 7), strides=(7, 7), data_format=data_format)
 
     if self.include_top:
-      self.fc1000 = tf.layers.Dense(classes, name='fc1000')
+      self.flatten = layers.Flatten()
+      self.fc1000 = layers.Dense(classes, name='fc1000')
     else:
       reduction_indices = [1, 2] if data_format == 'channels_last' else [2, 3]
       reduction_indices = tf.constant(reduction_indices)
@@ -298,7 +301,7 @@ class ResNet50(tf.keras.Model):
     x = self.avg_pool(x)
 
     if self.include_top:
-      return self.fc1000(tf.layers.flatten(x))
+      return self.fc1000(self.flatten(x))
     elif self.global_pooling:
       return self.global_pooling(x)
     else:
diff --git a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
index 88fffc962f..492adbe1d8 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_colorbot/rnn_colorbot.py
@@ -73,6 +73,8 @@ try:
 except ImportError:
   HAS_MATPLOTLIB = False
 
+layers = tf.keras.layers
+
 
 def parse(line):
   """Parse a line from the colors dataset."""
@@ -152,7 +154,7 @@ class RNNColorbot(tf.keras.Model):
 
     self.cells = self._add_cells(
         [tf.nn.rnn_cell.BasicLSTMCell(size) for size in rnn_cell_sizes])
-    self.relu = tf.layers.Dense(
+    self.relu = layers.Dense(
         label_dimension, activation=tf.nn.relu, name="relu")
 
   def call(self, inputs, training=False):
@@ -204,7 +206,7 @@ class RNNColorbot(tf.keras.Model):
 
   def _add_cells(self, cells):
     # "Magic" required for keras.Model classes to track all the variables in
-    # a list of tf.layers.Layer objects.
+    # a list of layers.Layer objects.
     # TODO(ashankar): Figure out API so user code doesn't have to do this.
     for i, c in enumerate(cells):
       setattr(self, "cell-%d" % i, c)
diff --git a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
index 69cd16d12c..a90048d813 100644
--- a/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
+++ b/tensorflow/contrib/eager/python/examples/rnn_ptb/rnn_ptb.py
@@ -38,6 +38,8 @@ import tensorflow as tf
 from tensorflow.contrib.cudnn_rnn.python.layers import cudnn_rnn
 from tensorflow.contrib.eager.python import tfe
 
+layers = tf.keras.layers
+
 
 class RNN(tf.keras.Model):
   """A static RNN.
@@ -74,14 +76,14 @@ class RNN(tf.keras.Model):
 
   def _add_cells(self, cells):
     # "Magic" required for keras.Model classes to track all the variables in
-    # a list of tf.layers.Layer objects.
+    # a list of Layer objects.
     # TODO(ashankar): Figure out API so user code doesn't have to do this.
     for i, c in enumerate(cells):
       setattr(self, "cell-%d" % i, c)
     return cells
 
 
-class Embedding(tf.layers.Layer):
+class Embedding(layers.Layer):
   """An Embedding layer."""
 
   def __init__(self, vocab_size, embedding_dim, **kwargs):
@@ -132,7 +134,7 @@ class PTBModel(tf.keras.Model):
     else:
       self.rnn = RNN(hidden_dim, num_layers, self.keep_ratio)
 
-    self.linear = tf.layers.Dense(
+    self.linear = layers.Dense(
         vocab_size, kernel_initializer=tf.random_uniform_initializer(-0.1, 0.1))
     self._output_shape = [-1, embedding_dim]
 
-- 
GitLab


From d8eda53c488683b37ae60e2ecbdf0bd2fd47c8c1 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Sat, 24 Mar 2018 00:24:50 -0700
Subject: [PATCH 472/960] Misc typo fixes in the XLA sources and docs.

PiperOrigin-RevId: 190322644
---
 tensorflow/compiler/xla/service/algebraic_simplifier.cc      | 2 +-
 tensorflow/compiler/xla/service/algebraic_simplifier.h       | 4 ++--
 tensorflow/compiler/xla/service/compiler.h                   | 2 +-
 tensorflow/compiler/xla/service/while_loop_simplifier.h      | 2 +-
 tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h | 2 +-
 5 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index 02b23c2d14..f9fabd8a35 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -302,7 +302,7 @@ class AlgebraicSimplifierVisitor : public DfsHloVisitorWithDefault {
   // Disable dot strength reduction on platforms where it causes a slowdown.
   bool enable_dot_strength_reduction_;
 
-  // Disable convolution simplication on platforms where it causes a slowdown.
+  // Disable convolution simplification on platforms where it causes a slowdown.
   bool enable_conv_simplification_;
 };
 
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.h b/tensorflow/compiler/xla/service/algebraic_simplifier.h
index f0590943be..c48196e861 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.h
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.h
@@ -57,10 +57,10 @@ class AlgebraicSimplifier : public HloPassInterface {
   bool is_layout_sensitive_;
   ValidBitcastCallback valid_bitcast_callback_;
 
-  // Enable dot simplication on platforms where it is profitable.
+  // Enable dot simplification on platforms where it is profitable.
   bool enable_dot_strength_reduction_;
 
-  // Enable convolution simplication on platforms where it is profitable.
+  // Enable convolution simplification on platforms where it is profitable.
   bool enable_conv_simplification_;
 };
 
diff --git a/tensorflow/compiler/xla/service/compiler.h b/tensorflow/compiler/xla/service/compiler.h
index 33e19efc72..b4b53ae2ed 100644
--- a/tensorflow/compiler/xla/service/compiler.h
+++ b/tensorflow/compiler/xla/service/compiler.h
@@ -127,7 +127,7 @@ class Compiler {
   // Compiles the HLO module for execution on a device given by the executor,
   // and returns an executable object or an error status. No HLO passes are
   // applied to module. Generally a module should be passed through RunHloPasses
-  // prior to calling this method because the some HLO passes are required for
+  // prior to calling this method because some HLO passes are required for
   // correctness. Takes ownership of the HLO module and is free to transform it.
   //
   // The compiler may optionally specialize to the individual device
diff --git a/tensorflow/compiler/xla/service/while_loop_simplifier.h b/tensorflow/compiler/xla/service/while_loop_simplifier.h
index d3d55634c9..3d3e1d60f2 100644
--- a/tensorflow/compiler/xla/service/while_loop_simplifier.h
+++ b/tensorflow/compiler/xla/service/while_loop_simplifier.h
@@ -25,7 +25,7 @@ namespace xla {
 // HLO pass that makes the following transformations on while loops:
 //
 //  - A while loop with static trip count of 0 is deleted.
-//  - A while loops with static trip count of 1 is replaced by its body (sans
+//  - A while loop with static trip count of 1 is replaced by its body (sans
 //    loop).
 //  - Elements of a while loop's tuple that the loop doesn't use are removed
 //    from the tuple.
diff --git a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h
index 063e312df6..8763e588c4 100644
--- a/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h
+++ b/tensorflow/compiler/xla/service/zero_sized_hlo_elimination.h
@@ -19,7 +19,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/service/hlo_pass_interface.h"
 
-// HLO pass that replaces zero sized Hlos with an zero sized constant literal.
+// HLO pass that replaces zero sized Hlos with a zero sized constant literal.
 namespace xla {
 class ZeroSizedHloElimination : public HloPassInterface {
  public:
-- 
GitLab


From 1aa398fe9801bca4dd8e19c255634d93bc9f5456 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Sat, 24 Mar 2018 23:42:08 -0700
Subject: [PATCH 473/960] Build and import rules for distributed strategy

PiperOrigin-RevId: 190367484
---
 tensorflow/tools/docs/generate_lib.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py
index 34dd419f15..d22a465376 100644
--- a/tensorflow/tools/docs/generate_lib.py
+++ b/tensorflow/tools/docs/generate_lib.py
@@ -211,6 +211,7 @@ def _get_default_do_not_descend_map():
       'tf': ['cli', 'lib', 'wrappers'],
       'tf.contrib': [
           'compiler',
+          'distribute',
           'grid_rnn',
           # Block contrib.keras to de-clutter the docs
           'keras',
-- 
GitLab


From 3e4df091fd099170ccb9737be3747b9542a85669 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 25 Mar 2018 09:38:54 -0700
Subject: [PATCH 474/960] Restore dependencies that are needed by the PIP
 package builder

PiperOrigin-RevId: 190387090
---
 .../contrib/boosted_trees/estimator_batch/BUILD | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index dae402204f..dcd235f876 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -13,20 +13,23 @@ load("//tensorflow:tensorflow.bzl", "py_test")
 filegroup(
     name = "all_files",
     srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
+        include = ["**/*"],
+        exclude = ["**/OWNERS"],
     ),
     visibility = ["//tensorflow:__subpackages__"],
 )
 
 py_library(
     name = "init_py",
-    srcs = [
-        "__init__.py",
-    ],
+    srcs = ["__init__.py"],
     srcs_version = "PY2AND3",
+    deps = [
+        "custom_export_strategy",
+        ":custom_loss_head",
+        ":estimator",
+        ":model",
+        ":trainer_hooks",
+    ],
 )
 
 py_library(
-- 
GitLab


From 6c1737e6c8c9e5db405853178fb5e42abc080ba3 Mon Sep 17 00:00:00 2001
From: brett koonce <koonce@hello.com>
Date: Sun, 25 Mar 2018 11:49:51 -0700
Subject: [PATCH 475/960] contrib/factorization: minor spelling tweaks (#17992)

---
 .../factorization/kernels/clustering_ops.cc        |  2 +-
 .../factorization/python/ops/factorization_ops.py  | 14 +++++++-------
 .../python/ops/factorization_ops_test.py           |  8 ++++----
 .../contrib/factorization/python/ops/gmm_ops.py    |  4 ++--
 .../contrib/factorization/python/ops/gmm_test.py   |  2 +-
 .../factorization/python/ops/kmeans_test.py        |  4 ++--
 .../contrib/factorization/python/ops/wals.py       |  2 +-
 7 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
index dd61f59585..2a6c97e8b9 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
@@ -353,7 +353,7 @@ class NearestNeighborsOp : public OpKernel {
     auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
     const int64 num_threads = worker_threads.num_threads;
     // This kernel might be configured to use fewer than the total number of
-    // available CPUs on the host machine. To avoid descructive interference
+    // available CPUs on the host machine. To avoid destructive interference
     // with other jobs running on the host machine, we must only use a fraction
     // of total available L3 cache. Unfortunately, we cannot query the host
     // machine to get the number of physical CPUs. So, we use a fixed per-CPU
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
index 054888e734..8e0ed1d80e 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
@@ -106,7 +106,7 @@ class WALSModel(object):
       # the prep_gramian_op for row(column) can be run.
       worker_init_op = model.worker_init
 
-      # To be run once per interation sweep before the row(column) update
+      # To be run once per integration sweep before the row(column) update
       # initialize ops can be run. Note that in the distributed training
       # situations, this should only be run by the chief trainer. All other
       # trainers need to block until this is done.
@@ -118,9 +118,9 @@ class WALSModel(object):
       init_row_update_op = model.initialize_row_update_op
       init_col_update_op = model.initialize_col_update_op
 
-      # Ops to upate row(column). This can either take the entire sparse tensor
-      # or slices of sparse tensor. For distributed trainer, each trainer
-      # handles just part of the matrix.
+      # Ops to update row(column). This can either take the entire sparse
+      # tensor or slices of sparse tensor. For distributed trainer, each
+      # trainer handles just part of the matrix.
       _, row_update_op, unreg_row_loss, row_reg, _ = model.update_row_factors(
            sp_input=matrix_slices_from_queue_for_worker_shard)
       row_loss = unreg_row_loss + row_reg
@@ -220,7 +220,7 @@ class WALSModel(object):
         in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of
         inner lists matching the number of row factor shards and the elements in
         each inner list are the weights for the rows of the corresponding row
-        factor shard. In this case,  w_ij = unonbserved_weight +
+        factor shard. In this case,  w_ij = unobserved_weight +
                                             row_weights[i] * col_weights[j].
         - If this is a single non-negative real number, this value is used for
         all row weights and w_ij = unobserved_weight + row_weights *
@@ -435,7 +435,7 @@ class WALSModel(object):
       gramian: Variable storing the gramian calculated from the factors.
 
     Returns:
-      A op that updates the gramian with the calcuated value from the factors.
+      A op that updates the gramian with the calculated value from the factors.
     """
     partial_gramians = []
     for f in factors:
@@ -564,7 +564,7 @@ class WALSModel(object):
 
     Note that specifically this initializes the cache of the row and column
     weights on workers when `use_factors_weights_cache` is True. In this case,
-    if these weights are being calcualted and reset after the object is created,
+    if these weights are being calculated and reset after the object is created,
     it is important to ensure this ops is run afterwards so the cache reflects
     the correct values.
     """
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
index c813733915..002f9cfbdd 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
@@ -210,7 +210,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -283,7 +283,7 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 3 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
+      # This is expected to reproduce the same column factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
@@ -385,7 +385,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -462,7 +462,7 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 2 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
+      # This is expected to reproduce the same column factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index 98d6434f47..14d4c733e3 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -280,7 +280,7 @@ class GmmAlgorithm(object):
     self._define_score_samples()
 
   def _define_full_covariance_probs(self, shard_id, shard):
-    """Defines the full covariance probabilties per example in a class.
+    """Defines the full covariance probabilities per example in a class.
 
     Updates a matrix with dimension num_examples X num_classes.
 
@@ -344,7 +344,7 @@ class GmmAlgorithm(object):
   def _define_prior_log_prob_operation(self, shard_id):
     """Computes the prior probability of all samples.
 
-    Updates a vector where each item is the prior probabibility of an
+    Updates a vector where each item is the prior probability of an
     input example.
 
     Args:
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_test.py b/tensorflow/contrib/factorization/python/ops/gmm_test.py
index 00a4734eb6..4fc9c96e9d 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_test.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_test.py
@@ -210,7 +210,7 @@ class GMMTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 0103cc4439..88eb9cf692 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -413,7 +413,7 @@ class KMeansCosineDistanceTest(KMeansTestBase):
     self.assertAllClose(score, self.true_score, atol=1e-2)
 
   def test_predict_kmeans_plus_plus(self):
-    # Most points are concetrated near one center. KMeans++ is likely to find
+    # Most points are concentrated near one center. KMeans++ is likely to find
     # the less populated centers.
     points = np.array(
         [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2],
@@ -604,7 +604,7 @@ class KMeansTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 4fe22ea26e..62db3bb4c4 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -235,7 +235,7 @@ def _wals_factorization_model_function(features, labels, mode, params):
         num_items: An integer, the total number of items of this axis.
         update_fn: A function that takes one argument (`sp_input`), and that
         returns a tuple of
-          * new_factors: A flot Tensor of the factor values after update.
+          * new_factors: A float Tensor of the factor values after update.
           * update_op: a TensorFlow op which updates the factors.
           * loss: A float Tensor, the unregularized loss.
           * reg_loss: A float Tensor, the regularization loss.
-- 
GitLab


From 6645609dffd4bfeb33d4d7250ad8e06935c39e82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 25 Mar 2018 11:51:33 -0700
Subject: [PATCH 476/960] Add skeleton code for DebugStripper.

PiperOrigin-RevId: 190391193
---
 tensorflow/core/grappler/optimizers/BUILD     | 32 ++++++++++++++
 .../grappler/optimizers/debug_stripper.cc     | 36 +++++++++++++++
 .../core/grappler/optimizers/debug_stripper.h | 43 ++++++++++++++++++
 .../optimizers/debug_stripper_test.cc         | 44 +++++++++++++++++++
 .../grappler/optimizers/meta_optimizer.cc     | 14 +++++-
 .../core/grappler/utils/grappler_test.cc      |  1 +
 .../core/protobuf/rewriter_config.proto       |  2 +
 7 files changed, 170 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/core/grappler/optimizers/debug_stripper.cc
 create mode 100644 tensorflow/core/grappler/optimizers/debug_stripper.h
 create mode 100644 tensorflow/core/grappler/optimizers/debug_stripper_test.cc

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 92f7cce502..601984fcfd 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -500,6 +500,7 @@ cc_library(
         ":constant_folding",
         ":custom_graph_optimizer",
         ":custom_graph_optimizer_registry",
+        ":debug_stripper",
         ":dependency_optimizer",
         ":function_optimizer",
         ":graph_optimizer",
@@ -618,3 +619,34 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
+
+cc_library(
+    name = "debug_stripper",
+    srcs = ["debug_stripper.cc"],
+    hdrs = [
+        "debug_stripper.h",
+    ],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/clusters:cluster",
+        "//tensorflow/core/grappler/optimizers:graph_optimizer",
+    ],
+)
+
+tf_cuda_cc_test(
+    name = "debug_stripper_test",
+    size = "small",
+    srcs = ["debug_stripper_test.cc"],
+    deps = [
+        ":debug_stripper",
+        "//tensorflow/cc:cc_ops",
+        "//tensorflow/core:tensorflow",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/utils:grappler_test",
+    ],
+)
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper.cc b/tensorflow/core/grappler/optimizers/debug_stripper.cc
new file mode 100644
index 0000000000..461f1aa2fb
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/debug_stripper.cc
@@ -0,0 +1,36 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/debug_stripper.h"
+#include "tensorflow/core/grappler/clusters/cluster.h"
+#include "tensorflow/core/grappler/grappler_item.h"
+
+namespace tensorflow {
+namespace grappler {
+
+Status DebugStripper::Optimize(Cluster* cluster, const GrapplerItem& item,
+                               GraphDef* output) {
+  // TODO(haoliang): Let's remove assertions here.
+  *output = item.graph;
+  return Status::OK();
+}
+
+void DebugStripper::Feedback(Cluster* cluster, const GrapplerItem& item,
+                             const GraphDef& optimize_output, double result) {
+  // Takes no feedback.
+}
+
+}  // end namespace grappler
+}  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper.h b/tensorflow/core/grappler/optimizers/debug_stripper.h
new file mode 100644
index 0000000000..1fe25aa1c3
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/debug_stripper.h
@@ -0,0 +1,43 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEBUG_STRIPPER_H_
+#define TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEBUG_STRIPPER_H_
+
+#include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// DebugStripper strips off debug-related nodes (e.g.
+// Assert, CheckNumerics, Print) from the graph.
+class DebugStripper : public GraphOptimizer {
+ public:
+  DebugStripper() {}
+  ~DebugStripper() override {}
+
+  string name() const override { return "debug_stripper"; };
+
+  Status Optimize(Cluster* cluster, const GrapplerItem& item,
+                  GraphDef* output) override;
+
+  void Feedback(Cluster* cluster, const GrapplerItem& item,
+                const GraphDef& optimize_output, double result) override;
+};
+
+}  // end namespace grappler
+}  // end namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_OPTIMIZERS_DEBUG_STRIPPER_H_
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
new file mode 100644
index 0000000000..d2cabc0798
--- /dev/null
+++ b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
@@ -0,0 +1,44 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/optimizers/debug_stripper.h"
+
+#include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/utils/grappler_test.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace grappler {
+namespace {
+
+class DebugStripperTest : public GrapplerTest {};
+
+// TODO(haoliang): Add tests for different removal operations.
+TEST_F(DebugStripperTest, OutputEqualToInput) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  auto c = ops::Const(s.WithOpName("c"), 0, {});
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  DebugStripper optimizer;
+  GraphDef output;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+}
+
+}  // namespace
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 6eb2bbc547..47ec16226b 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/auto_parallel.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/custom_graph_optimizer_registry.h"
+#include "tensorflow/core/grappler/optimizers/debug_stripper.h"
 #include "tensorflow/core/grappler/optimizers/dependency_optimizer.h"
 #include "tensorflow/core/grappler/optimizers/function_optimizer.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
@@ -84,6 +85,9 @@ std::unique_ptr<GraphOptimizer> MetaOptimizer::NewOptimizer(
     graph_optimizer.reset(
         new DependencyOptimizer(cfg_.dependency_optimization()));
   }
+  if (optimizer == "debug_stripper") {
+    graph_optimizer.reset(new DebugStripper());
+  }
   return graph_optimizer;
 }
 
@@ -134,10 +138,15 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new AutoParallel(cfg_.auto_parallel().num_replicas())));
     }
+    if (cfg_.debug_stripper() == RewriterConfig::ON) {
+      optimizers.push_back(
+          std::unique_ptr<GraphOptimizer>(new DebugStripper()));
+    }
   } else {
     const std::set<string> available_optimizers = {
-        "pruning",      "function",   "constfold", "layout",    "memory",
-        "autoparallel", "arithmetic", "loop",      "dependency"};
+        "pruning",    "function",      "constfold",  "layout",
+        "memory",     "autoparallel",  "arithmetic", "loop",
+        "dependency", "debug_stripper"};
     std::vector<string> custom_optimizer_names;
     for (const auto& optimizer_name : cfg_.optimizers()) {
       if (available_optimizers.find(optimizer_name) !=
@@ -238,6 +247,7 @@ bool MetaOptimizerEnabled(const RewriterConfig& cfg) {
          cfg.dependency_optimization() != RewriterConfig::OFF ||
          cfg.auto_parallel().enable() ||
          cfg.memory_optimization() != RewriterConfig::NO_MEM_OPT ||
+         cfg.debug_stripper() == RewriterConfig::ON ||
          !cfg.optimizers().empty();
 }
 
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 1c15ea65b8..ee126f4955 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -36,6 +36,7 @@ GrapplerTest::GrapplerTest() {
   cfg->set_loop_optimization(RewriterConfig::OFF);
   cfg->set_function_optimization(RewriterConfig::OFF);
   cfg->set_layout_optimizer(RewriterConfig::OFF);
+  cfg->set_debug_stripper(RewriterConfig::OFF);
 }
 
 std::vector<Tensor> GrapplerTest::EvaluateNodes(
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index fdf16aa1da..bb772460b0 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -46,6 +46,8 @@ message RewriterConfig {
   Toggle loop_optimization = 9;
   // Function optimizations (default is ON).
   Toggle function_optimization = 10;
+  // Strips debug-related nodes from the graph (off by default).
+  Toggle debug_stripper = 11;
   // If true, don't remove unnecessary ops from the graph
   bool disable_model_pruning = 2;
 
-- 
GitLab


From 8561c30ea6538083248b653237754138695702af Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Sun, 25 Mar 2018 19:35:54 -0700
Subject: [PATCH 477/960] Use compat.as_bytes() instead of str.encode().

PiperOrigin-RevId: 190409217
---
 tensorflow/python/framework/ops.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index f264e38102..5e4a884a70 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2477,8 +2477,9 @@ def _set_shapes_for_outputs_c_api(op):
     serialized = c_api.ResourceHandleShapeAndType(op._graph._c_graph,
                                                   output._as_tf_output())
     if serialized:
-      output._handle_data = (cpp_shape_inference_pb2.CppShapeInferenceResult
-                             .HandleData.FromString(serialized.encode()))
+      output._handle_data = (
+          cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData.FromString(
+              compat.as_bytes(serialized)))
     else:
       output._handle_data = None
 
-- 
GitLab


From 9d9ea88abd63d2c317e445e54a4f9c90d747343a Mon Sep 17 00:00:00 2001
From: Petros Mol <pmol@google.com>
Date: Sun, 25 Mar 2018 20:13:13 -0700
Subject: [PATCH 478/960] Minor Error type and documentation fix.

PiperOrigin-RevId: 190411045
---
 tensorflow/python/estimator/canned/head.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/estimator/canned/head.py b/tensorflow/python/estimator/canned/head.py
index c9635a9c27..bb033d3495 100644
--- a/tensorflow/python/estimator/canned/head.py
+++ b/tensorflow/python/estimator/canned/head.py
@@ -887,11 +887,12 @@ def _binary_logistic_head_with_sigmoid_cross_entropy_loss(
   Raises:
     ValueError: If `thresholds` contains a value outside of `(0, 1)`.
     ValueError: If `loss_reduction` is invalid.
+    TypeError: if `label_vocabulary` has invalid type.
   """
   thresholds = tuple(thresholds) if thresholds else tuple()
   if label_vocabulary is not None and not isinstance(label_vocabulary,
                                                      (list, tuple)):
-    raise ValueError(
+    raise TypeError(
         'label_vocabulary should be a list or tuple. Given type: {}'.format(
             type(label_vocabulary)))
 
-- 
GitLab


From 668f182b1fdfc31568a44fe650324fe2ddedbbe1 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Sun, 25 Mar 2018 21:57:09 -0700
Subject: [PATCH 479/960] Always cast `tf.distributions.Distribution`
 `_event_shape`, `_batch_shape`.

PiperOrigin-RevId: 190415923
---
 tensorflow/python/ops/distributions/distribution.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/distributions/distribution.py b/tensorflow/python/ops/distributions/distribution.py
index c055ca43e8..0866fa8b0b 100644
--- a/tensorflow/python/ops/distributions/distribution.py
+++ b/tensorflow/python/ops/distributions/distribution.py
@@ -593,7 +593,7 @@ class Distribution(_BaseDistribution):
     Returns:
       batch_shape: `TensorShape`, possibly unknown.
     """
-    return self._batch_shape()
+    return tensor_shape.as_shape(self._batch_shape())
 
   def _event_shape_tensor(self):
     raise NotImplementedError("event_shape_tensor is not implemented")
@@ -626,7 +626,7 @@ class Distribution(_BaseDistribution):
     Returns:
       event_shape: `TensorShape`, possibly unknown.
     """
-    return self._event_shape()
+    return tensor_shape.as_shape(self._event_shape())
 
   def is_scalar_event(self, name="is_scalar_event"):
     """Indicates that `event_shape == []`.
-- 
GitLab


From c3436d6757a77ab1fefd3f6000a1e961a9ab9881 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Sun, 25 Mar 2018 22:02:09 -0700
Subject: [PATCH 480/960] Disable flaky prefetching_ops_test.

PiperOrigin-RevId: 190416108
---
 tensorflow/contrib/data/python/kernel_tests/BUILD | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index f70b29c43b..8cfe4a727a 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -479,6 +479,10 @@ py_test(
     size = "small",
     srcs = ["prefetching_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = [
+        "manual",
+        "no_oss",
+    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:prefetching_ops",
         "//tensorflow/core:protos_all_py",
-- 
GitLab


From a5a1e9e43131b387395930f38234fc10b02d874b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 08:52:53 -0700
Subject: [PATCH 481/960] Updated test (but not source) of
 https://www.tensorflow.org/api_docs/python/tf/contrib/training/HParams to
 show that it allows '=' in the values.

PiperOrigin-RevId: 190470578
---
 .../training/python/training/hparam_test.py   | 42 ++++++++++++++-----
 1 file changed, 32 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/hparam_test.py b/tensorflow/contrib/training/python/training/hparam_test.py
index 16397622ed..96eff86d8d 100644
--- a/tensorflow/contrib/training/python/training/hparam_test.py
+++ b/tensorflow/contrib/training/python/training/hparam_test.py
@@ -38,40 +38,60 @@ class HParamsTest(test.TestCase):
     self.assertFalse('bar' in hparams)
 
   def testSomeValues(self):
-    hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6')
-    self.assertDictEqual({'aaa': 1, 'b': 2.0, 'c_c': 'relu6'}, hparams.values())
-    expected_str = '[(\'aaa\', 1), (\'b\', 2.0), (\'c_c\', \'relu6\')]'
+    hparams = hparam.HParams(aaa=1, b=2.0, c_c='relu6', d='/a/b=c/d')
+    self.assertDictEqual(
+        {'aaa': 1, 'b': 2.0, 'c_c': 'relu6', 'd': '/a/b=c/d'},
+        hparams.values())
+    expected_str = ('[(\'aaa\', 1), (\'b\', 2.0), (\'c_c\', \'relu6\'), '
+                    '(\'d\', \'/a/b=c/d\')]')
     self.assertEqual(expected_str, str(hparams.__str__()))
     self.assertEqual(expected_str, str(hparams))
     self.assertEqual(1, hparams.aaa)
     self.assertEqual(2.0, hparams.b)
     self.assertEqual('relu6', hparams.c_c)
+    self.assertEqual('/a/b=c/d', hparams.d)
     hparams.parse('aaa=12')
     self.assertDictEqual({
         'aaa': 12,
         'b': 2.0,
-        'c_c': 'relu6'
+        'c_c': 'relu6',
+        'd': '/a/b=c/d'
     }, hparams.values())
     self.assertEqual(12, hparams.aaa)
     self.assertEqual(2.0, hparams.b)
     self.assertEqual('relu6', hparams.c_c)
+    self.assertEqual('/a/b=c/d', hparams.d)
     hparams.parse('c_c=relu4, b=-2.0e10')
     self.assertDictEqual({
         'aaa': 12,
         'b': -2.0e10,
-        'c_c': 'relu4'
+        'c_c': 'relu4',
+        'd': '/a/b=c/d'
     }, hparams.values())
     self.assertEqual(12, hparams.aaa)
     self.assertEqual(-2.0e10, hparams.b)
     self.assertEqual('relu4', hparams.c_c)
+    self.assertEqual('/a/b=c/d', hparams.d)
     hparams.parse('c_c=,b=0,')
-    self.assertDictEqual({'aaa': 12, 'b': 0, 'c_c': ''}, hparams.values())
+    self.assertDictEqual({'aaa': 12, 'b': 0, 'c_c': '', 'd': '/a/b=c/d'},
+                         hparams.values())
     self.assertEqual(12, hparams.aaa)
     self.assertEqual(0.0, hparams.b)
     self.assertEqual('', hparams.c_c)
+    self.assertEqual('/a/b=c/d', hparams.d)
     hparams.parse('c_c=2.3",b=+2,')
     self.assertEqual(2.0, hparams.b)
     self.assertEqual('2.3"', hparams.c_c)
+    hparams.parse('d=/a/b/c/d,aaa=11,')
+    self.assertEqual(11, hparams.aaa)
+    self.assertEqual(2.0, hparams.b)
+    self.assertEqual('2.3"', hparams.c_c)
+    self.assertEqual('/a/b/c/d', hparams.d)
+    hparams.parse('b=1.5,d=/a=b/c/d,aaa=10,')
+    self.assertEqual(10, hparams.aaa)
+    self.assertEqual(1.5, hparams.b)
+    self.assertEqual('2.3"', hparams.c_c)
+    self.assertEqual('/a=b/c/d', hparams.d)
     with self.assertRaisesRegexp(ValueError, 'Unknown hyperparameter'):
       hparams.parse('x=123')
     with self.assertRaisesRegexp(ValueError, 'Could not parse'):
@@ -84,17 +104,19 @@ class HParamsTest(test.TestCase):
       hparams.parse('b=relu')
     with self.assertRaisesRegexp(ValueError, 'Must not pass a list'):
       hparams.parse('aaa=[123]')
-    self.assertEqual(12, hparams.aaa)
-    self.assertEqual(2.0, hparams.b)
+    self.assertEqual(10, hparams.aaa)
+    self.assertEqual(1.5, hparams.b)
     self.assertEqual('2.3"', hparams.c_c)
+    self.assertEqual('/a=b/c/d', hparams.d)
     # Exports to proto.
     hparam_def = hparams.to_proto()
     # Imports from proto.
     hparams2 = hparam.HParams(hparam_def=hparam_def)
     # Verifies that all hparams are restored.
-    self.assertEqual(12, hparams2.aaa)
-    self.assertEqual(2.0, hparams2.b)
+    self.assertEqual(10, hparams2.aaa)
+    self.assertEqual(1.5, hparams2.b)
     self.assertEqual('2.3"', hparams2.c_c)
+    self.assertEqual('/a=b/c/d', hparams2.d)
 
   def testSetFromMap(self):
     hparams = hparam.HParams(a=1, b=2.0, c='tanh')
-- 
GitLab


From 2b078a508b8c6c920db121f676650d7972749bd7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 10:00:20 -0700
Subject: [PATCH 482/960] Automated g4 rollback of changelist 190293303

PiperOrigin-RevId: 190479555
---
 tensorflow/c/BUILD                           |  2 --
 tensorflow/c/python_api.cc                   | 26 ---------------
 tensorflow/c/python_api.h                    |  7 ----
 tensorflow/python/BUILD                      |  2 --
 tensorflow/python/client/tf_session.i        |  1 -
 tensorflow/python/framework/importer_test.py | 34 --------------------
 tensorflow/python/framework/ops.py           | 10 ------
 7 files changed, 82 deletions(-)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index f4a486d330..d096647558 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -279,8 +279,6 @@ tf_cuda_library(
     deps = [
         ":c_api",
         ":c_api_internal",
-        # TODO(b/74620627): remove when _USE_C_SHAPES is removed
-        "//tensorflow/python:cpp_shape_inference_proto_cc",
     ],
 )
 
diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc
index 93155998b8..cd604538f1 100644
--- a/tensorflow/c/python_api.cc
+++ b/tensorflow/c/python_api.cc
@@ -16,7 +16,6 @@ limitations under the License.
 #include "tensorflow/c/python_api.h"
 
 #include "tensorflow/c/c_api_internal.h"
-#include "tensorflow/python/framework/cpp_shape_inference.pb.h"
 
 namespace tensorflow {
 
@@ -110,29 +109,4 @@ void ExtendSession(TF_Session* session, TF_Status* status) {
   session->extend_before_run = false;
 }
 
-std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output) {
-  Node* node = &output.oper->node;
-  CppShapeInferenceResult::HandleData handle_data;
-  handle_data.set_is_set(true);
-  {
-    mutex_lock l(graph->mu);
-    tensorflow::shape_inference::InferenceContext* ic =
-        graph->refiner.GetContext(node);
-    CHECK(ic != nullptr);
-    CHECK_LT(output.index, ic->num_outputs());
-    const auto* shapes_and_types =
-        ic->output_handle_shapes_and_types(output.index);
-    if (shapes_and_types == nullptr) return "";
-
-    for (const auto& p : *shapes_and_types) {
-      auto* out_shape_and_type = handle_data.add_shape_and_type();
-      ic->ShapeHandleToProto(p.shape, out_shape_and_type->mutable_shape());
-      out_shape_and_type->set_dtype(p.dtype);
-    }
-  }
-  string result;
-  handle_data.SerializeToString(&result);
-  return result;
-}
-
 }  // namespace tensorflow
diff --git a/tensorflow/c/python_api.h b/tensorflow/c/python_api.h
index 2d4c8cd9ed..13b680b3a2 100644
--- a/tensorflow/c/python_api.h
+++ b/tensorflow/c/python_api.h
@@ -16,8 +16,6 @@ limitations under the License.
 #ifndef TENSORFLOW_C_PYTHON_API_H_
 #define TENSORFLOW_C_PYTHON_API_H_
 
-#include <string>
-
 #include "tensorflow/c/c_api.h"
 
 // These functions can be removed without notice. They exist to facilitate some
@@ -53,11 +51,6 @@ void SetRequireShapeInferenceFns(TF_Graph* graph, bool require);
 // the graph after the session has been made aware of them.
 void ExtendSession(TF_Session* session, TF_Status* status);
 
-// Returns the serialized CppShapeInferenceResult::HandleData proto for
-// `output` if its a resource tensor, or otherwise returns the empty string.
-// TODO(b/74620627): remove when _USE_C_SHAPES is removed
-std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output);
-
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_C_PYTHON_API_H_
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index e6ad564ede..30ecc477f2 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3131,8 +3131,6 @@ tf_proto_library(
     srcs = ["framework/cpp_shape_inference.proto"],
     cc_api_version = 2,
     protodeps = tf_additional_all_protos(),
-    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
-    visibility = ["//tensorflow:internal"],
 )
 
 py_test(
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 70a3d032f4..e88fc0c01a 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -723,7 +723,6 @@ def TF_Reset(target, containers=None, config=None):
 %unignore TF_TryEvaluateConstant_wrapper;
 %noexception TF_TryEvaluateConstant_wrapper;
 %unignore ExtendSession;
-%unignore ResourceHandleShapeAndType;
 
 %include "tensorflow/python/client/tf_session_helper.h"
 
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 369669c2e6..6593b17184 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -39,7 +39,6 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variables
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
@@ -357,39 +356,6 @@ class ImportGraphDefTest(test.TestCase):
       self.assertEqual(d._input_types, [dtypes.int32_ref, dtypes.int32])
       self.assertEqual(d.outputs, [])
 
-  def testResources(self):
-    # Produce GraphDef containing a ops producing and consuming resources.
-    graph = ops.Graph()
-    with graph.as_default():
-      var = resource_variable_ops.ResourceVariable(1.0)
-      var_assign = var.assign(2.0)
-      # Use an op that requires handle shape to be set.
-      var_shape = resource_variable_ops.variable_shape(var.handle)
-      init = variables.global_variables_initializer()
-    graph_def = graph.as_graph_def()
-
-    # Import the GraphDef.
-    with ops.Graph().as_default():
-      # pylint: disable=unused-variable
-      imported_var, imported_assign, imported_shape, imported_init = (
-          importer.import_graph_def(
-              graph_def,
-              return_elements=[var.name, var_assign.name, var_shape.name,
-                               init.name]))
-
-      # Make sure the handle shape is set on the imported variable.
-      new_var_shape = resource_variable_ops.variable_shape(imported_var)
-      # pylint: enable=unused-variable
-
-      # Run the imported graph.
-      # TODO(b/76173421): make this work (currently DCHECKS)
-      # with self.test_session() as sess:
-      #   sess.run(imported_init)
-      #   self.assertEqual(sess.run(imported_var), 1.0)
-      #   self.assertEqual(sess.run(imported_assign), 2.0)
-      #   self.assertEqual(list(sess.run(imported_shape)), [])
-      #   self.assertEqual(list(sess.run(new_var_shape)), [])
-
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
     graph = ops.Graph()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 5e4a884a70..e579289a8d 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -42,7 +42,6 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
-from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -296,7 +295,6 @@ class Tensor(_TensorLike):
 
     # Attributes used for C++ shape inference. Not inspected, only forwarded.
     # If set, will be a HandleData object from cpp_shape_inference.proto.
-    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
     self._handle_data = None
     self._id = uid()
 
@@ -2474,14 +2472,6 @@ def _set_shapes_for_outputs_c_api(op):
       shape_vector = [None if d == -1 else d for d in shape_vector]
       output.set_shape(tensor_shape.TensorShape(shape_vector))
 
-    serialized = c_api.ResourceHandleShapeAndType(op._graph._c_graph,
-                                                  output._as_tf_output())
-    if serialized:
-      output._handle_data = (
-          cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData.FromString(
-              compat.as_bytes(serialized)))
-    else:
-      output._handle_data = None
 
 # TODO(skyewm): remove this when _USE_C_API flag is removed.
 def _set_shapes_for_outputs(op):
-- 
GitLab


From cc6b2ae837e9c0ce3678671ff5bd59f0f8e53e06 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Mon, 26 Mar 2018 10:25:46 -0700
Subject: [PATCH 483/960] Adding a FunctionBufferingResourceReset Op that
 resets the state of the function buffering resource so that we can start
 using it with re-initializable iterators.

PiperOrigin-RevId: 190484110
---
 .../data/kernels/prefetching_kernels.cc       |  66 +++++++--
 tensorflow/contrib/data/ops/dataset_ops.cc    |   9 ++
 .../kernel_tests/prefetching_ops_test.py      | 137 +++++++++++++++---
 .../data/python/ops/prefetching_ops.py        |   5 +
 4 files changed, 184 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 190a6ee580..79d1fc3494 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -65,12 +65,6 @@ class FunctionBufferingResource : public ResourceBase {
 
   ~FunctionBufferingResource() override {
     Cancel();
-    {
-      mutex_lock l(mu_);
-      while (is_buffering_) {
-        cond_var_.wait(l);
-      }
-    }
     if (thread_pool_ != nullptr) {
       delete thread_pool_;
     }
@@ -107,6 +101,20 @@ class FunctionBufferingResource : public ResourceBase {
   void Cancel() LOCKS_EXCLUDED(mu_) {
     mutex_lock l(mu_);
     cancelled_ = true;
+    while (is_buffering_) {
+      cond_var_.wait(l);
+    }
+  }
+
+  // Cancels all pending operations and then clears out the state.
+  void Reset() LOCKS_EXCLUDED(mu_) {
+    Cancel();
+    mutex_lock l(mu_);
+    buffer_.clear();
+    requests_.clear();
+    is_buffering_ = false;
+    end_of_sequence_ = false;
+    cancelled_ = false;
   }
 
   // If the buffer has anything, runs `callback` on the first element in the
@@ -200,13 +208,12 @@ class FunctionBufferingResource : public ResourceBase {
                   mutex_lock l(mu_);
                   BufferElement buffer_element;
                   buffer_element.status = status;
-                  if (!status.ok()) {
+                  if (status.ok()) {
+                    buffer_element.value.swap(*rets);
+                  } else {
                     end_of_sequence_ = true;
                     is_buffering_ = false;
-                    buffer_.push_back(std::move(buffer_element));
-                    return;
                   }
-                  buffer_element.value.swap(*rets);
                   buffer_.push_back(std::move(buffer_element));
                   if (!requests_.empty()) {
                     buffer_front = std::move(buffer_.front());
@@ -214,7 +221,7 @@ class FunctionBufferingResource : public ResourceBase {
                     callback = std::move(requests_.front());
                     requests_.pop_front();
                   }
-                  if (buffer_.size() < buffer_size_) {
+                  if (buffer_.size() < buffer_size_ && !end_of_sequence_) {
                     restart_buffering = true;
                   } else {
                     is_buffering_ = false;
@@ -406,6 +413,43 @@ REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceGetNext")
                         FunctionBufferingResourceGetNextOp);
 #endif  // TENSORFLOW_USE_SYCL
 
+// Resets the FunctionBufferingResource, cancelling all pending requests and
+// clearing out the buffer.
+class FunctionBufferingResourceResetOp : public OpKernel {
+ public:
+  explicit FunctionBufferingResourceResetOp(OpKernelConstruction* ctx)
+      : OpKernel(ctx) {}
+
+  ~FunctionBufferingResourceResetOp() override {}
+
+  void Compute(OpKernelContext* ctx) override {
+    ResourceHandle handle;
+    OP_REQUIRES_OK(ctx,
+                   HandleFromInput(ctx, "function_buffer_resource", &handle));
+    FunctionBufferingResource* buffer = nullptr;
+    OP_REQUIRES_OK(
+        ctx, LookupResource<FunctionBufferingResource>(ctx, handle, &buffer));
+    core::ScopedUnref s(buffer);
+
+    buffer->Reset();
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceReset")
+                            .Device(DEVICE_CPU)
+                            .HostMemory("function_buffer_resource"),
+                        FunctionBufferingResourceResetOp);
+REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceReset")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("function_buffer_resource"),
+                        FunctionBufferingResourceResetOp);
+#if TENSORFLOW_USE_SYCL
+REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResourceReset")
+                            .Device(DEVICE_SYCL)
+                            .HostMemory("function_buffer_resource"),
+                        FunctionBufferingResourceResetOp);
+#endif  // TENSORFLOW_USE_SYCL
+
 class IteratorGetDeviceOp : public OpKernel {
  public:
   using OpKernel::OpKernel;
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index 74737bbcad..bd96448d64 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -83,6 +83,15 @@ output: A list of return values.
 output_types: The type list for the return values.
 )doc");
 
+REGISTER_OP("FunctionBufferingResourceReset")
+    .Input("function_buffer_resource: resource")
+    .SetShapeFn(shape_inference::UnknownShape)
+    .Doc(R"doc(
+Resets the FunctionBufferingResource.
+
+function_buffer_resource: The FunctionBufferingResource handle.
+)doc");
+
 REGISTER_OP("ThreadPoolDataset")
     .Input("input_dataset: variant")
     .Input("thread_pool: resource")
diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index 1d74afe1e1..a14736ac09 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -17,7 +17,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import itertools
 import threading
 
 from tensorflow.contrib.data.python.ops import prefetching_ops
@@ -39,25 +38,29 @@ class StagingAreaOpsTest(test.TestCase):
   def setUp(self):
     self._event = threading.Event()
 
-  def _prefetch_fn_helper(self, buffer_name, device0, device1):
-    worker_config = config_pb2.ConfigProto()
-    worker_config.device_count["CPU"] = 2
+  def _create_ds_and_iterator(self, device0, initializable=False):
 
     def gen():
-      for i in itertools.count(start=1, step=1):
-        yield [i + 0.0]
+      for i in range(1, 10):
+        yield [float(i)]
         if i == 6:
           self._event.set()
 
     with ops.device(device0):
-      dataset_3 = dataset_ops.Dataset.from_generator(gen, (dtypes.float32))
-      iterator_3 = dataset_3.make_one_shot_iterator()
-      iterator_3_handle = iterator_3.string_handle()
+      ds = dataset_ops.Dataset.from_generator(gen, (dtypes.float32))
+      if initializable:
+        ds_iterator = ds.make_initializable_iterator()
+      else:
+        ds_iterator = ds.make_one_shot_iterator()
+      return (ds, ds_iterator)
+
+  def _create_ops(self, ds, ds_iterator, buffer_name, device0, device1):
+    ds_iterator_handle = ds_iterator.string_handle()
 
     @function.Defun(dtypes.string)
     def _remote_fn(h):
       remote_iterator = iterator_ops.Iterator.from_string_handle(
-          h, dataset_3.output_types, dataset_3.output_shapes)
+          h, ds.output_types, ds.output_shapes)
       return remote_iterator.get_next()
 
     target = constant_op.constant(device0)
@@ -65,7 +68,7 @@ class StagingAreaOpsTest(test.TestCase):
       buffer_resource_handle = prefetching_ops.function_buffering_resource(
           f=_remote_fn,
           target_device=target,
-          string_arg=iterator_3_handle,
+          string_arg=ds_iterator_handle,
           buffer_size=3,
           thread_pool_size=2,
           shared_name=buffer_name)
@@ -74,6 +77,20 @@ class StagingAreaOpsTest(test.TestCase):
       prefetch_op = prefetching_ops.function_buffering_resource_get_next(
           function_buffer_resource=buffer_resource_handle,
           output_types=[dtypes.float32])
+      reset_op = prefetching_ops.function_buffering_resource_reset(
+          function_buffer_resource=buffer_resource_handle)
+      destroy_op = resource_variable_ops.destroy_resource_op(
+          buffer_resource_handle, ignore_lookup_error=True)
+
+    return (prefetch_op, reset_op, destroy_op)
+
+  def _prefetch_fn_helper_one_shot(self, buffer_name, device0, device1):
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+
+    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=False)
+    prefetch_op, _, destroy_op = self._create_ops(ds, ds_iterator, buffer_name,
+                                                  device0, device1)
 
     with self.test_session(config=worker_config) as sess:
       elem = sess.run(prefetch_op)
@@ -87,26 +104,102 @@ class StagingAreaOpsTest(test.TestCase):
       self._event.wait()
       elem = sess.run(prefetch_op)
       self.assertEqual(elem, [5.0])
-      sess.run(
-          resource_variable_ops.destroy_resource_op(
-              buffer_resource_handle, ignore_lookup_error=True))
+      sess.run(destroy_op)
 
   def testSameDeviceCPU(self):
-    self._prefetch_fn_helper("same_device_cpu",
-                             "/job:localhost/replica:0/task:0/cpu:0",
-                             "/job:localhost/replica:0/task:0/cpu:0")
+    self._prefetch_fn_helper_one_shot("same_device_cpu",
+                                      "/job:localhost/replica:0/task:0/cpu:0",
+                                      "/job:localhost/replica:0/task:0/cpu:0")
 
   def testDifferentDeviceCPU(self):
-    self._prefetch_fn_helper("diff_device_cpu",
-                             "/job:localhost/replica:0/task:0/cpu:0",
-                             "/job:localhost/replica:0/task:0/cpu:1")
+    self._prefetch_fn_helper_one_shot("diff_device_cpu",
+                                      "/job:localhost/replica:0/task:0/cpu:0",
+                                      "/job:localhost/replica:0/task:0/cpu:1")
 
   def testDifferentDeviceCPUGPU(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
 
-    self._prefetch_fn_helper("cpu_gpu", "/job:localhost/replica:0/task:0/cpu:0",
-                             "/job:localhost/replica:0/task:0/gpu:0")
+    self._prefetch_fn_helper_one_shot("cpu_gpu",
+                                      "/job:localhost/replica:0/task:0/cpu:0",
+                                      "/job:localhost/replica:0/task:0/gpu:0")
+
+  def testReinitialization(self):
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+
+    device0 = "/job:localhost/replica:0/task:0/cpu:0"
+    device1 = "/job:localhost/replica:0/task:0/cpu:1"
+    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True)
+    prefetch_op, reset_op, destroy_op = self._create_ops(
+        ds, ds_iterator, "reinit", device0, device1)
+
+    with self.test_session(config=worker_config) as sess:
+      sess.run(ds_iterator.initializer)
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [1.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [2.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [3.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [4.0])
+      self._event.wait()
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [5.0])
+      # Lets reset the function buffering resource and reinitialize the
+      # iterator. Should be able to go through this again.
+      self._event.clear()
+      sess.run(reset_op)
+      sess.run(ds_iterator.initializer)
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [1.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [2.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [3.0])
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [4.0])
+      self._event.wait()
+      elem = sess.run(prefetch_op)
+      self.assertEqual(elem, [5.0])
+      sess.run(destroy_op)
+
+  def testReinitializationOutOfRange(self):
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+
+    device0 = "/job:localhost/replica:0/task:0/cpu:0"
+    device1 = "/job:localhost/replica:0/task:0/cpu:1"
+    ds, ds_iterator = self._create_ds_and_iterator(device0, initializable=True)
+    prefetch_op, reset_op, destroy_op = self._create_ops(
+        ds, ds_iterator, "reinit", device0, device1)
+
+    with self.test_session(config=worker_config) as sess:
+      sess.run(ds_iterator.initializer)
+      for i in range(1, 10):
+        elem = sess.run(prefetch_op)
+        self.assertEqual(elem, [float(i)])
+      # Try fetching after its over twice to test out end of sequence.
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(prefetch_op)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(prefetch_op)
+
+      # Now reset everything and try it out again.
+      self._event.clear()
+      sess.run(reset_op)
+      sess.run(ds_iterator.initializer)
+      for i in range(1, 10):
+        elem = sess.run(prefetch_op)
+        self.assertEqual(elem, [float(i)])
+      # Try fetching after its over twice to test out end of sequence.
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(prefetch_op)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(prefetch_op)
+
+      sess.run(destroy_op)
 
   def testPrefetchToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index e38d53a221..1438b5426f 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -62,6 +62,11 @@ def function_buffering_resource_get_next(function_buffer_resource,
       name=name)
 
 
+def function_buffering_resource_reset(function_buffer_resource, name=None):
+  return gen_dataset_ops.function_buffering_resource_reset(
+      function_buffer_resource=function_buffer_resource, name=name)
+
+
 # pylint: disable=protected-access
 class _PrefetchToDeviceIterator(object):
   """A replacement for @{tf.data.Iterator} that prefetches to another device."""
-- 
GitLab


From be917027e37c5e8f21f6ba07f24bdbf072cf6dfd Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Mon, 26 Mar 2018 10:51:21 -0700
Subject: [PATCH 484/960] Added experimental C APIs to build a stack of dataset
 + iterator nodes that reads imagenet TFRecord files.

PiperOrigin-RevId: 190488817
---
 tensorflow/c/BUILD                      |    2 +
 tensorflow/c/c_api_experimental.cc      | 7218 ++++++++++++++++++++++-
 tensorflow/c/c_api_experimental.h       |   31 +-
 tensorflow/c/c_api_experimental_test.cc |   84 +-
 tensorflow/c/testdata/tf_record         |  Bin 0 -> 417114 bytes
 5 files changed, 7155 insertions(+), 180 deletions(-)
 create mode 100644 tensorflow/c/testdata/tf_record

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index d096647558..426f97b844 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -220,6 +220,7 @@ tf_cc_test(
     name = "c_api_experimental_test",
     size = "small",
     srcs = ["c_api_experimental_test.cc"],
+    data = ["testdata/tf_record"],
     linkopts = select({
         "//tensorflow:darwin": ["-headerpad_max_install_names"],
         "//conditions:default": [],
@@ -230,6 +231,7 @@ tf_cc_test(
     deps = [
         ":c_api_experimental",
         ":c_test_util",
+        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 8593a8eb50..1c809cb21e 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -22,10 +22,15 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/protobuf/config.pb.h"
 
+using tensorflow::FunctionDef;
 using tensorflow::Node;
 using tensorflow::NodeBuilder;
 using tensorflow::Status;
-using tensorflow::Tensor;
+
+namespace {
+typedef std::unique_ptr<TF_Function, decltype(&TF_DeleteFunction)>
+    UniqueFuncPtr;
+}
 
 // struct TF_Operation { tensorflow::Node node; };
 static TF_Operation* ToTF_Operation(Node* node) {
@@ -102,8 +107,7 @@ void TF_ShutdownTPU(TF_Session* session, TF_Status* status) {
                 /*run_metadata*/ nullptr, status);
 }
 
-TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
-                                                      size_t* len) {
+const char* TF_GraphDebugString(TF_Graph* graph, size_t* len) {
   tensorflow::mutex_lock c(graph->mu);
   const auto& debug_str = graph->graph.ToGraphDefDebug().DebugString();
   *len = debug_str.size();
@@ -112,55 +116,56 @@ TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
   return ret;
 }
 
-// TODO(hongm): Replace this will a real implementation.
-static tensorflow::Status BuildDatasetTest(TF_Graph* dataset_graph,
-                                           Node** dataset_node) {
-  tensorflow::mutex_lock c(dataset_graph->mu);
-  Tensor const_t(tensorflow::DT_INT32, tensorflow::TensorShape({}));
-  const_t.flat<tensorflow::int32>()(0) = 1;
-
-  Node* const_node;
-  TF_RETURN_IF_ERROR(NodeBuilder("Const", "Const")
-                         .Attr("dtype", tensorflow::DT_INT32)
-                         .Attr("value", const_t)
-                         .Finalize(&dataset_graph->graph, &const_node));
-
-  std::vector<NodeBuilder::NodeOut> input_list;
-  input_list.push_back(NodeBuilder::NodeOut(const_node, 0));
-
-  return NodeBuilder("TensorDataset", "TensorDataset")
-      .Input(input_list)
-      .Attr("Toutput_types", {tensorflow::DT_INT32})
-      .Attr("output_shapes", {tensorflow::TensorShapeProto()})
-      .Finalize(&dataset_graph->graph, dataset_node);
-}
-
-//  On success, returns a newly created TF_Function instance from
-//  `text_proto`. It must be deleted by calling TF_DeleteFunction.
-static TF_Function* CreateFunctionFromTextProto(const char* text_proto,
-                                                TF_Status* status) {
-  tensorflow::FunctionDef fdef;
-  if (!tensorflow::protobuf::TextFormat::ParseFromString(text_proto, &fdef)) {
+// On success, returns a set of TF_Function instances from `text_proto` of
+// GraphDef type. These functions must be deleted by calling TF_DeleteFunction.
+//
+// If `mutate_proto_func` is non-NULL, run it over each FunctionDef proto,
+// before creating a TF_Function out of the possibly mutated proto.
+static std::vector<UniqueFuncPtr> CreateFunctionsFromTextProto(
+    const char* text_proto,
+    std::function<void(FunctionDef*)>* mutate_proto_func, TF_Status* status) {
+  tensorflow::GraphDef gdef;
+  if (!tensorflow::protobuf::TextFormat::ParseFromString(text_proto, &gdef)) {
     status->status = tensorflow::errors::Internal(
-        "Invalid text proto for FunctionDef: ", text_proto);
-    return nullptr;
+        "Invalid text proto for GraphDef: ", text_proto);
+    return {};
+  }
+  const auto& fdef_lib = gdef.library();
+  if (fdef_lib.gradient_size() > 0) {
+    status->status = tensorflow::errors::Internal(
+        "GradientDef is not supported in reading Dataset related functions: ",
+        text_proto);
+    return {};
   }
-  std::vector<char> binary_proto_buf(fdef.ByteSizeLong());
-  fdef.SerializeToArray(binary_proto_buf.data(), binary_proto_buf.size());
-  return TF_FunctionImportFunctionDef(binary_proto_buf.data(),
-                                      binary_proto_buf.size(), status);
+  std::vector<UniqueFuncPtr> ret;
+  for (const auto& fdef : fdef_lib.function()) {
+    // Make a copy so that we can mutate it.
+    FunctionDef fdef_to_load = fdef;
+    if (mutate_proto_func) {
+      (*mutate_proto_func)(&fdef_to_load);
+    }
+    VLOG(1) << "Adding func to graph: " << fdef_to_load.DebugString();
+    std::vector<char> binary_proto_buf(fdef_to_load.ByteSizeLong());
+    fdef_to_load.SerializeToArray(binary_proto_buf.data(),
+                                  binary_proto_buf.size());
+    auto func = TF_FunctionImportFunctionDef(binary_proto_buf.data(),
+                                             binary_proto_buf.size(), status);
+    if (!status->status.ok()) return {};
+    ret.push_back(UniqueFuncPtr(func, TF_DeleteFunction));
+  }
+  return ret;
 }
 
-//  On success, returns a newly created TF_Function instance from `proto_file`,
-//  and sets `dataset_name` to the created dataset name. The returned function
-//  must be deleted by calling TF_DeleteFunction.
-//
-// TODO(hongm): Support reading the file given by `proto_file`.
-static TF_Function* LoadDatasetFunction(const char* proto_file,
-                                        std::string* dataset_name,
-                                        TF_Status* status) {
+//  On success, returns a newly created TF_Function instance encoding a dataset
+//  node stack that returns a sequence of 3 floats, and sets `dataset_name` to
+//  the created dataset name. The returned function must be deleted by calling
+//  TF_DeleteFunction.
+static UniqueFuncPtr CreateFakeDatasetFunction(std::string* dataset_name,
+                                               TF_Status* status) {
   const char* func_def = R"PREFIX(
-signature {
+library {
+  function {
+    signature {
       name: "_make_dataset_d8de2712"
       output_arg {
         name: "TensorSliceDataset"
@@ -217,112 +222,7029 @@ signature {
     ret {
       key: "TensorSliceDataset"
       value: "TensorSliceDataset:handle:0"
-    })PREFIX";
+    }
+  }
+}
+)PREFIX";
 
   *dataset_name = "_make_dataset_d8de2712";
-  return CreateFunctionFromTextProto(func_def, status);
+  auto functions = CreateFunctionsFromTextProto(
+      func_def, /*mutate_proto_func*/ nullptr, status);
+  DCHECK_EQ(functions.size(), 1);
+  return std::move(functions[0]);
 }
 
-// TODO(hongm): Use `file_path` in the implementation.
-TF_Operation* TF_MakeIteratorGetNextWithDatasets(TF_Graph* graph,
-                                                 const char* file_path,
-                                                 TF_Function** dataset_func,
-                                                 TF_Status* status) {
-  tensorflow::Status s;
-
-  // We can parameterize the function name, if we ever need more than 1
-  // iterators in a graph.
-  const std::string dataset_name = "UNIQUE_DATASET";
-
-  std::unique_ptr<TF_Graph, decltype(&TF_DeleteGraph)> dataset_graph(
-      TF_NewGraph(), TF_DeleteGraph);
-  Node* dataset_node = nullptr;
-  s = BuildDatasetTest(dataset_graph.get(), &dataset_node);
-  if (!s.ok()) {
-    status->status = s;
-    return nullptr;
-  }
-
-  TF_Output output{ToTF_Operation(dataset_node), 0};
-  std::unique_ptr<TF_Function, decltype(&TF_DeleteFunction)> result_func(
-      TF_GraphToFunction(dataset_graph.get(), dataset_name.c_str(),
-                         /*append_hash_to_fn_name*/ false,
-                         /*num_opers*/ -1,
-                         /*opers*/ nullptr,
-                         /*numinputs*/ 0,
-                         /*inputs*/ nullptr,
-                         /*noutputs*/ 1,
-                         /*outputs*/ &output,
-                         /*outputnames*/ nullptr,
-                         /*functionoptions*/ nullptr, "", status),
-      TF_DeleteFunction);
-  if (!status->status.ok()) {
-    return nullptr;
-  }
-
-  TF_GraphCopyFunction(graph, result_func.get(), /*gradient*/ nullptr, status);
-
-  if (!status->status.ok()) {
-    return nullptr;
-  }
-
-  tensorflow::mutex_lock c(graph->mu);
-
-  tensorflow::NameAttrList func;
-  func.set_name(dataset_name);
-  // Run the iterator node on CPU.
-  Node* oneshot_iterator_node;
-  std::vector<tensorflow::TensorShapeProto> output_shape_list;
-  output_shape_list.push_back(tensorflow::TensorShapeProto());
-  s = NodeBuilder("OneShotIterator", "OneShotIterator")
-          .Device("/device:CPU:0")
-          .Attr("container", "")
-          .Attr("dataset_factory", func)
-          .Attr("output_types", {tensorflow::DT_INT32})
-          .Attr("output_shapes", output_shape_list)
-          .Attr("shared_name", "")
-          .Finalize(&graph->graph, &oneshot_iterator_node);
-  if (!s.ok()) {
-    status->status = s;
-    return nullptr;
-  }
-  // Run shape inference function for each newly added node, so that more
-  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
-  s = graph->refiner.AddNode(oneshot_iterator_node);
-  if (!s.ok()) {
-    status->status = s;
-    return nullptr;
-  }
-
-  // Run the iterator node on CPU.
-  Node* getnext_node;
-  s = NodeBuilder("IteratorGetNext", "IteratorGetNext")
-          .Input(oneshot_iterator_node)
-          .Device("/device:CPU:0")
-          .Attr("output_types", {tensorflow::DT_INT32})
-          .Attr("output_shapes", output_shape_list)
-          .Finalize(&graph->graph, &getnext_node);
-  if (!s.ok()) {
-    status->status = s;
-    return nullptr;
+//  On success, returns a set of TF_Function instances encoding a dataset
+//  node stack that reads a Imagenet TFRecordFile dataset from `file_path`, and
+//  sets `dataset_name` to the created dataset name. The returned functions must
+//  be deleted by calling TF_DeleteFunction.
+static std::vector<UniqueFuncPtr> CreateImagenetDatasetFunctions(
+    const char* file_path, std::string* dataset_name, TF_Status* status) {
+  const char* func_def = R"PREFIX(
+library {
+  function {
+    signature {
+      name: "tf_map_func_91295dea"
+      input_arg {
+        name: "arg0"
+        type: DT_STRING
+      }
+      output_arg {
+        name: "FlatMapDataset"
+        type: DT_VARIANT
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+      is_stateful: true
+    }
+    node_def {
+      name: "flat_filenames/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: -1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "flat_filenames"
+      op: "Reshape"
+      input: "arg0"
+      input: "flat_filenames/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "TensorSliceDataset"
+      op: "TensorSliceDataset"
+      input: "flat_filenames:output:0"
+      attr {
+        key: "Toutput_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FlatMapDataset"
+      op: "FlatMapDataset"
+      input: "TensorSliceDataset:handle:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+          }
+        }
+      }
+      attr {
+        key: "f"
+        value {
+          func {
+            name: "tf_map_func_0cc8c35b"
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+    }
+    ret {
+      key: "FlatMapDataset"
+      value: "FlatMapDataset:handle:0"
+    }
   }
-  // Run shape inference function for each newly added node, so that more
-  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
-  s = graph->refiner.AddNode(getnext_node);
-  if (!s.ok()) {
-    status->status = s;
-    return nullptr;
+  function {
+    signature {
+      name: "tf_map_func_0cc8c35b"
+      input_arg {
+        name: "arg0"
+        type: DT_STRING
+      }
+      output_arg {
+        name: "TFRecordDataset"
+        type: DT_VARIANT
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+      is_stateful: true
+    }
+    node_def {
+      name: "compression_type"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node_def {
+      name: "buffer_size"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 8388608
+          }
+        }
+      }
+    }
+    node_def {
+      name: "TFRecordDataset"
+      op: "TFRecordDataset"
+      input: "arg0"
+      input: "compression_type:output:0"
+      input: "buffer_size:output:0"
+    }
+    ret {
+      key: "TFRecordDataset"
+      value: "TFRecordDataset:handle:0"
+    }
   }
+  function {
+    signature {
+      name: "tf_map_func_74b6b15c"
+      input_arg {
+        name: "arg0"
+        type: DT_STRING
+      }
+      output_arg {
+        name: "Reshape_1"
+        type: DT_FLOAT
+      }
+      output_arg {
+        name: "sub_1"
+        type: DT_INT32
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+      is_stateful: true
+    }
+    node_def {
+      name: "ParseSingleExample/key_image/class/label"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: -1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape"
+      op: "Reshape"
+      input: "ParseSingleExample/key_image/class/label:output:0"
+      input: "ParseSingleExample/Reshape/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/key_image/class/text"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_1/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_1"
+      op: "Reshape"
+      input: "ParseSingleExample/key_image/class/text:output:0"
+      input: "ParseSingleExample/Reshape_1/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/key_image/encoded"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_2/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_2"
+      op: "Reshape"
+      input: "ParseSingleExample/key_image/encoded:output:0"
+      input: "ParseSingleExample/Reshape_2/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/key_image/format"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "jpeg"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_3/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/Reshape_3"
+      op: "Reshape"
+      input: "ParseSingleExample/key_image/format:output:0"
+      input: "ParseSingleExample/Reshape_3/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "ParseSingleExample/ParseSingleExample"
+      op: "ParseSingleExample"
+      input: "arg0"
+      input: "ParseSingleExample/Reshape:output:0"
+      input: "ParseSingleExample/Reshape_1:output:0"
+      input: "ParseSingleExample/Reshape_2:output:0"
+      input: "ParseSingleExample/Reshape_3:output:0"
+      attr {
+        key: "Tdense"
+        value {
+          list {
+            type: DT_INT64
+            type: DT_STRING
+            type: DT_STRING
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "dense_keys"
+        value {
+          list {
+            s: "image/class/label"
+            s: "image/class/text"
+            s: "image/encoded"
+            s: "image/format"
+          }
+        }
+      }
+      attr {
+        key: "dense_shapes"
+        value {
+          list {
+            shape {
+            }
+            shape {
+            }
+            shape {
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "num_sparse"
+        value {
+          i: 5
+        }
+      }
+      attr {
+        key: "sparse_keys"
+        value {
+          list {
+            s: "image/object/bbox/xmax"
+            s: "image/object/bbox/xmin"
+            s: "image/object/bbox/ymax"
+            s: "image/object/bbox/ymin"
+            s: "image/object/class/label"
+          }
+        }
+      }
+      attr {
+        key: "sparse_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_FLOAT
+            type: DT_FLOAT
+            type: DT_FLOAT
+            type: DT_INT64
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape"
+      op: "Reshape"
+      input: "ParseSingleExample/ParseSingleExample:dense_values:2"
+      input: "Reshape/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/Substr/pos"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/Substr/len"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/Substr"
+      op: "Substr"
+      input: "Reshape:output:0"
+      input: "decode_image/Substr/pos:output:0"
+      input: "decode_image/Substr/len:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/is_jpeg/Substr/pos"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/is_jpeg/Substr/len"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/is_jpeg/Substr"
+      op: "Substr"
+      input: "Reshape:output:0"
+      input: "decode_image/is_jpeg/Substr/pos:output:0"
+      input: "decode_image/is_jpeg/Substr/len:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/is_jpeg/Equal/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "\377\330\377"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/is_jpeg/Equal"
+      op: "Equal"
+      input: "decode_image/is_jpeg/Substr:output:0"
+      input: "decode_image/is_jpeg/Equal/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/Switch"
+      op: "Switch"
+      input: "decode_image/is_jpeg/Equal:z:0"
+      input: "decode_image/is_jpeg/Equal:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/switch_t"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/switch_f"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/pred_id"
+      op: "Identity"
+      input: "decode_image/is_jpeg/Equal:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/check_jpeg_channels/x"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/check_jpeg_channels/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 4
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/check_jpeg_channels"
+      op: "NotEqual"
+      input: "decode_image/cond_jpeg/check_jpeg_channels/x:output:0"
+      input: "decode_image/cond_jpeg/check_jpeg_channels/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/Assert/Const"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 1, 3) when decoding JPEG images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/Assert/Assert/data_0"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 1, 3) when decoding JPEG images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/Assert/Assert"
+      op: "Assert"
+      input: "decode_image/cond_jpeg/check_jpeg_channels:z:0"
+      input: "decode_image/cond_jpeg/Assert/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/DecodeJpeg"
+      op: "DecodeJpeg"
+      input: "decode_image/cond_jpeg/DecodeJpeg/Switch:output_true:0"
+      input: "^decode_image/cond_jpeg/Assert/Assert"
+      attr {
+        key: "acceptable_fraction"
+        value {
+          f: 1.0
+        }
+      }
+      attr {
+        key: "channels"
+        value {
+          i: 3
+        }
+      }
+      attr {
+        key: "dct_method"
+        value {
+          s: ""
+        }
+      }
+      attr {
+        key: "fancy_upscaling"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "ratio"
+        value {
+          i: 1
+        }
+      }
+      attr {
+        key: "try_recover_truncated"
+        value {
+          b: false
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/DecodeJpeg/Switch"
+      op: "Switch"
+      input: "Reshape:output:0"
+      input: "decode_image/cond_jpeg/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/is_png/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "\211PN"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/is_png"
+      op: "Equal"
+      input: "decode_image/cond_jpeg/is_png/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/is_png/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/is_png/Switch"
+      op: "Switch"
+      input: "decode_image/Substr:output:0"
+      input: "decode_image/cond_jpeg/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@decode_image/Substr"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/Switch"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/is_png:z:0"
+      input: "decode_image/cond_jpeg/is_png:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/switch_t"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/cond_png/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/switch_f"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/cond_png/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/pred_id"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/is_png:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/DecodePng"
+      op: "DecodePng"
+      input: "decode_image/cond_jpeg/cond_png/DecodePng/Switch_1:output_true:0"
+      attr {
+        key: "channels"
+        value {
+          i: 3
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/DecodePng/Switch"
+      op: "Switch"
+      input: "Reshape:output:0"
+      input: "decode_image/cond_jpeg/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/DecodePng/Switch_1"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/cond_png/DecodePng/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/is_gif/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "GIF"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/is_gif"
+      op: "Equal"
+      input: "decode_image/cond_jpeg/cond_png/is_gif/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/is_gif/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/is_gif/Switch"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/is_png/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@decode_image/Substr"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Switch"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/cond_png/is_gif:z:0"
+      input: "decode_image/cond_jpeg/cond_png/is_gif:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/pred_id"
+      op: "Identity"
+      input: "decode_image/cond_jpeg/cond_png/is_gif:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels/x"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels"
+      op: "NotEqual"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels/x:output:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1/x"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 4
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1"
+      op: "NotEqual"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1/x:output:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/LogicalAnd"
+      op: "LogicalAnd"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels:z:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_gif_channels_1:z:0"
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert/Const"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 3) when decoding GIF images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert/Assert/data_0"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 3) when decoding GIF images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert/Assert"
+      op: "Assert"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/LogicalAnd:z:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Assert/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif"
+      op: "DecodeGif"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif/Switch_1:output_true:0"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/Assert/Assert"
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif/Switch"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/cond_png/DecodePng/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif/Switch_1"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/pos"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/len"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Substr"
+      op: "Substr"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/pos:output:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/len:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/Switch"
+      op: "Switch"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif/Switch:output_false:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@Reshape"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/is_bmp/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "BM"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/is_bmp"
+      op: "Equal"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Substr:output:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/is_bmp/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_1/Const"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Unable to decode bytes as JPEG, PNG, GIF, or BMP"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_1/Assert/data_0"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Unable to decode bytes as JPEG, PNG, GIF, or BMP"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_1/Assert"
+      op: "Assert"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/is_bmp:z:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_1/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels/x"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels/y"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels"
+      op: "NotEqual"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels/x:output:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_2/Const"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 3) when decoding BMP images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_2/Assert/data_0"
+      op: "Const"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Channels must be in (None, 0, 3) when decoding BMP images"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_2/Assert"
+      op: "Assert"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/check_channels:z:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Assert_2/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeBmp"
+      op: "DecodeBmp"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Substr/Switch:output_false:0"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/Assert_1/Assert"
+      input: "^decode_image/cond_jpeg/cond_png/cond_gif/Assert_2/Assert"
+      attr {
+        key: "channels"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/cond_gif/Merge"
+      op: "Merge"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeBmp:image:0"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/DecodeGif:image:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/cond_png/Merge"
+      op: "Merge"
+      input: "decode_image/cond_jpeg/cond_png/cond_gif/Merge:output:0"
+      input: "decode_image/cond_jpeg/cond_png/DecodePng:image:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "decode_image/cond_jpeg/Merge"
+      op: "Merge"
+      input: "decode_image/cond_jpeg/cond_png/Merge:output:0"
+      input: "decode_image/cond_jpeg/DecodeJpeg:image:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "convert_image/Cast"
+      op: "Cast"
+      input: "decode_image/cond_jpeg/Merge:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "convert_image/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.00392156885937
+          }
+        }
+      }
+    }
+    node_def {
+      name: "convert_image"
+      op: "Mul"
+      input: "convert_image/Cast:y:0"
+      input: "convert_image/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "Const"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+                size: 1
+              }
+              dim {
+                size: 1
+              }
+              dim {
+                size: 4
+              }
+            }
+            tensor_content: "\000\000\000\000\000\000\000\000\000\000\200?\000\000\200?"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "distorted_bounding_box_crop/Shape"
+      op: "Shape"
+      input: "convert_image:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "distorted_bounding_box_crop/sample_distorted_bounding_box/SampleDistortedBoundingBoxV2/min_object_covered"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.10000000149
+          }
+        }
+      }
+    }
+    node_def {
+      name: "distorted_bounding_box_crop/sample_distorted_bounding_box/SampleDistortedBoundingBoxV2"
+      op: "SampleDistortedBoundingBoxV2"
+      input: "distorted_bounding_box_crop/Shape:output:0"
+      input: "Const:output:0"
+      input: "distorted_bounding_box_crop/sample_distorted_bounding_box/SampleDistortedBoundingBoxV2/min_object_covered:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "area_range"
+        value {
+          list {
+            f: 0.0799999982119
+            f: 1.0
+          }
+        }
+      }
+      attr {
+        key: "aspect_ratio_range"
+        value {
+          list {
+            f: 0.75
+            f: 1.33333337307
+          }
+        }
+      }
+      attr {
+        key: "max_attempts"
+        value {
+          i: 1
+        }
+      }
+      attr {
+        key: "seed"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "seed2"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "use_image_if_no_bounding_boxes"
+        value {
+          b: true
+        }
+      }
+    }
+    node_def {
+      name: "distorted_bounding_box_crop/Slice"
+      op: "Slice"
+      input: "convert_image:z:0"
+      input: "distorted_bounding_box_crop/sample_distorted_bounding_box/SampleDistortedBoundingBoxV2:begin:0"
+      input: "distorted_bounding_box_crop/sample_distorted_bounding_box/SampleDistortedBoundingBoxV2:size:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "Shape"
+      op: "Shape"
+      input: "convert_image:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "Shape_1"
+      op: "Shape"
+      input: "distorted_bounding_box_crop/Slice:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "Equal"
+      op: "Equal"
+      input: "Shape:output:0"
+      input: "Shape_1:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "Cast"
+      op: "Cast"
+      input: "Equal:z:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "Const_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Sum"
+      op: "Sum"
+      input: "Cast:y:0"
+      input: "Const_1:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "Tidx"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "keep_dims"
+        value {
+          b: false
+        }
+      }
+    }
+    node_def {
+      name: "GreaterEqual/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "GreaterEqual"
+      op: "GreaterEqual"
+      input: "Sum:output:0"
+      input: "GreaterEqual/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/Switch"
+      op: "Switch"
+      input: "GreaterEqual:z:0"
+      input: "GreaterEqual:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/switch_t"
+      op: "Identity"
+      input: "cond/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/switch_f"
+      op: "Identity"
+      input: "cond/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/pred_id"
+      op: "Identity"
+      input: "GreaterEqual:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/Shape"
+      op: "Shape"
+      input: "cond/Shape/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/Shape/Switch"
+      op: "Switch"
+      input: "convert_image:z:0"
+      input: "cond/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@convert_image"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Cast"
+      op: "Cast"
+      input: "cond/Shape:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice"
+      op: "StridedSlice"
+      input: "cond/Cast:y:0"
+      input: "cond/strided_slice/stack:output:0"
+      input: "cond/strided_slice/stack_1:output:0"
+      input: "cond/strided_slice/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_1/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_1/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_1/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_1"
+      op: "StridedSlice"
+      input: "cond/Cast:y:0"
+      input: "cond/strided_slice_1/stack:output:0"
+      input: "cond/strided_slice_1/stack_1:output:0"
+      input: "cond/strided_slice_1/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/Greater"
+      op: "Greater"
+      input: "cond/strided_slice:output:0"
+      input: "cond/strided_slice_1:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Switch"
+      op: "Switch"
+      input: "cond/Greater:z:0"
+      input: "cond/Greater:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/switch_t"
+      op: "Identity"
+      input: "cond/cond/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/switch_f"
+      op: "Identity"
+      input: "cond/cond/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/pred_id"
+      op: "Identity"
+      input: "cond/Greater:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice/stack"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice/stack_1"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice/stack_2"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice"
+      op: "StridedSlice"
+      input: "cond/cond/strided_slice/Switch:output_true:0"
+      input: "cond/cond/strided_slice/stack:output:0"
+      input: "cond/cond/strided_slice/stack_1:output:0"
+      input: "cond/cond/strided_slice/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice/Switch"
+      op: "Switch"
+      input: "cond/Cast:y:0"
+      input: "cond/cond/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@cond/Cast"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_1/stack"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_1/stack_1"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_1/stack_2"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_1"
+      op: "StridedSlice"
+      input: "cond/cond/strided_slice/Switch:output_true:0"
+      input: "cond/cond/strided_slice_1/stack:output:0"
+      input: "cond/cond/strided_slice_1/stack_1:output:0"
+      input: "cond/cond/strided_slice_1/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/truediv"
+      op: "RealDiv"
+      input: "cond/cond/strided_slice:output:0"
+      input: "cond/cond/strided_slice_1:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/mul/y"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 224.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/mul"
+      op: "Mul"
+      input: "cond/cond/truediv:z:0"
+      input: "cond/cond/mul/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast/x/1"
+      op: "Const"
+      input: "^cond/cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 224.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast/x"
+      op: "Pack"
+      input: "cond/cond/mul:z:0"
+      input: "cond/cond/Cast/x/1:output:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast"
+      op: "Cast"
+      input: "cond/cond/Cast/x:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_2/stack"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_2/stack_1"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_2/stack_2"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_2"
+      op: "StridedSlice"
+      input: "cond/cond/strided_slice_2/Switch:output_false:0"
+      input: "cond/cond/strided_slice_2/stack:output:0"
+      input: "cond/cond/strided_slice_2/stack_1:output:0"
+      input: "cond/cond/strided_slice_2/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_2/Switch"
+      op: "Switch"
+      input: "cond/Cast:y:0"
+      input: "cond/cond/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@cond/Cast"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_3/stack"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_3/stack_1"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_3/stack_2"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/strided_slice_3"
+      op: "StridedSlice"
+      input: "cond/cond/strided_slice_2/Switch:output_false:0"
+      input: "cond/cond/strided_slice_3/stack:output:0"
+      input: "cond/cond/strided_slice_3/stack_1:output:0"
+      input: "cond/cond/strided_slice_3/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/truediv_1"
+      op: "RealDiv"
+      input: "cond/cond/strided_slice_2:output:0"
+      input: "cond/cond/strided_slice_3:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/mul_1/y"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 224.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/mul_1"
+      op: "Mul"
+      input: "cond/cond/truediv_1:z:0"
+      input: "cond/cond/mul_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast_1/x/0"
+      op: "Const"
+      input: "^cond/cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 224.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast_1/x"
+      op: "Pack"
+      input: "cond/cond/Cast_1/x/0:output:0"
+      input: "cond/cond/mul_1:z:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Cast_1"
+      op: "Cast"
+      input: "cond/cond/Cast_1/x:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/cond/Merge"
+      op: "Merge"
+      input: "cond/cond/Cast_1:y:0"
+      input: "cond/cond/Cast:y:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic/images"
+      op: "Pack"
+      input: "cond/Shape/Switch:output_true:0"
+      attr {
+        key: "N"
+        value {
+          i: 1
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic"
+      op: "ResizeBicubic"
+      input: "cond/ResizeBicubic/images:output:0"
+      input: "cond/cond/Merge:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "align_corners"
+        value {
+          b: false
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_2/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_2/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_2/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_2"
+      op: "StridedSlice"
+      input: "cond/ResizeBicubic:resized_images:0"
+      input: "cond/strided_slice_2/stack:output:0"
+      input: "cond/strided_slice_2/stack_1:output:0"
+      input: "cond/strided_slice_2/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/Shape_1"
+      op: "Shape"
+      input: "cond/strided_slice_2:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_3/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_3/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_3/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_3"
+      op: "StridedSlice"
+      input: "cond/Shape_1:output:0"
+      input: "cond/strided_slice_3/stack:output:0"
+      input: "cond/strided_slice_3/stack_1:output:0"
+      input: "cond/strided_slice_3/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/Shape_2"
+      op: "Shape"
+      input: "cond/strided_slice_2:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_4/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_4/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_4/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_4"
+      op: "StridedSlice"
+      input: "cond/Shape_2:output:0"
+      input: "cond/strided_slice_4/stack:output:0"
+      input: "cond/strided_slice_4/stack_1:output:0"
+      input: "cond/strided_slice_4/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/sub/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/sub"
+      op: "Sub"
+      input: "cond/strided_slice_3:output:0"
+      input: "cond/sub/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/add/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/add"
+      op: "Add"
+      input: "cond/sub:z:0"
+      input: "cond/add/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv/Cast"
+      op: "Cast"
+      input: "cond/add:z:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv/Cast_1"
+      op: "Cast"
+      input: "cond/truediv/y:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv"
+      op: "RealDiv"
+      input: "cond/truediv/Cast:y:0"
+      input: "cond/truediv/Cast_1:y:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+    }
+    node_def {
+      name: "cond/sub_1/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/sub_1"
+      op: "Sub"
+      input: "cond/strided_slice_4:output:0"
+      input: "cond/sub_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/add_1/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/add_1"
+      op: "Add"
+      input: "cond/sub_1:z:0"
+      input: "cond/add_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv_1/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv_1/Cast"
+      op: "Cast"
+      input: "cond/add_1:z:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv_1/Cast_1"
+      op: "Cast"
+      input: "cond/truediv_1/y:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/truediv_1"
+      op: "RealDiv"
+      input: "cond/truediv_1/Cast:y:0"
+      input: "cond/truediv_1/Cast_1:y:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+    }
+    node_def {
+      name: "cond/Shape_3"
+      op: "Shape"
+      input: "cond/strided_slice_2:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/Rank"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Equal/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Equal"
+      op: "Equal"
+      input: "cond/Rank:output:0"
+      input: "cond/Equal/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/Assert/Const"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Rank of image must be equal to 3."
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Assert/Assert/data_0"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Rank of image must be equal to 3."
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Assert/Assert"
+      op: "Assert"
+      input: "cond/Equal:z:0"
+      input: "cond/Assert/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_5/stack"
+      op: "Const"
+      input: "^cond/Assert/Assert"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_5/stack_1"
+      op: "Const"
+      input: "^cond/Assert/Assert"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 3
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_5/stack_2"
+      op: "Const"
+      input: "^cond/Assert/Assert"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_5"
+      op: "StridedSlice"
+      input: "cond/Shape_3:output:0"
+      input: "cond/strided_slice_5/stack:output:0"
+      input: "cond/strided_slice_5/stack_1:output:0"
+      input: "cond/strided_slice_5/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/stack/0"
+      op: "Const"
+      input: "^cond/Assert/Assert"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/stack/1"
+      op: "Const"
+      input: "^cond/Assert/Assert"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/stack"
+      op: "Pack"
+      input: "cond/stack/0:output:0"
+      input: "cond/stack/1:output:0"
+      input: "cond/strided_slice_5:output:0"
+      attr {
+        key: "N"
+        value {
+          i: 3
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_6/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_6/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_6/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_6"
+      op: "StridedSlice"
+      input: "cond/Shape_3:output:0"
+      input: "cond/strided_slice_6/stack:output:0"
+      input: "cond/strided_slice_6/stack_1:output:0"
+      input: "cond/strided_slice_6/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/GreaterEqual/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/GreaterEqual"
+      op: "GreaterEqual"
+      input: "cond/strided_slice_6:output:0"
+      input: "cond/GreaterEqual/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_7/stack"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_7/stack_1"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_7/stack_2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_7"
+      op: "StridedSlice"
+      input: "cond/Shape_3:output:0"
+      input: "cond/strided_slice_7/stack:output:0"
+      input: "cond/strided_slice_7/stack_1:output:0"
+      input: "cond/strided_slice_7/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/GreaterEqual_1/y"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 224
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/GreaterEqual_1"
+      op: "GreaterEqual"
+      input: "cond/strided_slice_7:output:0"
+      input: "cond/GreaterEqual_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/LogicalAnd"
+      op: "LogicalAnd"
+      input: "cond/GreaterEqual:z:0"
+      input: "cond/GreaterEqual_1:z:0"
+    }
+    node_def {
+      name: "cond/Assert_1/Const"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Crop size greater than the image size."
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Assert_1/Assert/data_0"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "Crop size greater than the image size."
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/Assert_1/Assert"
+      op: "Assert"
+      input: "cond/LogicalAnd:z:0"
+      input: "cond/Assert_1/Assert/data_0:output:0"
+      attr {
+        key: "T"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "summarize"
+        value {
+          i: 3
+        }
+      }
+    }
+    node_def {
+      name: "cond/stack_1/2"
+      op: "Const"
+      input: "^cond/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_DOUBLE
+            tensor_shape {
+            }
+            double_val: 0.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/stack_1"
+      op: "Pack"
+      input: "cond/truediv:z:0"
+      input: "cond/truediv_1:z:0"
+      input: "cond/stack_1/2:output:0"
+      attr {
+        key: "N"
+        value {
+          i: 3
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/ToInt32"
+      op: "Cast"
+      input: "cond/stack_1:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_DOUBLE
+        }
+      }
+    }
+    node_def {
+      name: "cond/Slice"
+      op: "Slice"
+      input: "cond/strided_slice_2:output:0"
+      input: "cond/ToInt32:y:0"
+      input: "cond/stack:output:0"
+      input: "^cond/Assert_1/Assert"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "cond/Reshape"
+      op: "Reshape"
+      input: "cond/Slice:output:0"
+      input: "cond/stack:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic_1/images"
+      op: "Pack"
+      input: "cond/ResizeBicubic_1/images/Switch:output_false:0"
+      attr {
+        key: "N"
+        value {
+          i: 1
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "axis"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic_1/images/Switch"
+      op: "Switch"
+      input: "distorted_bounding_box_crop/Slice:output:0"
+      input: "cond/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@distorted_bounding_box_crop/Slice"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic_1/size"
+      op: "Const"
+      input: "^cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 2
+              }
+            }
+            tensor_content: "\340\000\000\000\340\000\000\000"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/ResizeBicubic_1"
+      op: "ResizeBicubic"
+      input: "cond/ResizeBicubic_1/images:output:0"
+      input: "cond/ResizeBicubic_1/size:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "align_corners"
+        value {
+          b: false
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_8/stack"
+      op: "Const"
+      input: "^cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_8/stack_1"
+      op: "Const"
+      input: "^cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_8/stack_2"
+      op: "Const"
+      input: "^cond/switch_f"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "cond/strided_slice_8"
+      op: "StridedSlice"
+      input: "cond/ResizeBicubic_1:resized_images:0"
+      input: "cond/strided_slice_8/stack:output:0"
+      input: "cond/strided_slice_8/stack_1:output:0"
+      input: "cond/strided_slice_8/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "cond/Merge"
+      op: "Merge"
+      input: "cond/strided_slice_8:output:0"
+      input: "cond/Reshape:output:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "Const_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+                size: 1
+              }
+              dim {
+                size: 1
+              }
+              dim {
+                size: 3
+              }
+            }
+            tensor_content: "\354Q\370>\325x\351>;\337\317>"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "sub"
+      op: "Sub"
+      input: "cond/Merge:output:0"
+      input: "Const_2:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "Const_3"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+              dim {
+                size: 1
+              }
+              dim {
+                size: 1
+              }
+              dim {
+                size: 3
+              }
+            }
+            tensor_content: "\372~j>B`e>fff>"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "truediv"
+      op: "RealDiv"
+      input: "sub:z:0"
+      input: "Const_3:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/control_dependency"
+      op: "Identity"
+      input: "truediv:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@truediv"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/min"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/max"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 1.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/RandomUniform"
+      op: "RandomUniform"
+      input: "random_flip_left_right/random_uniform/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "seed"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "seed2"
+        value {
+          i: 0
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/sub"
+      op: "Sub"
+      input: "random_flip_left_right/random_uniform/max:output:0"
+      input: "random_flip_left_right/random_uniform/min:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform/mul"
+      op: "Mul"
+      input: "random_flip_left_right/random_uniform/RandomUniform:output:0"
+      input: "random_flip_left_right/random_uniform/sub:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/random_uniform"
+      op: "Add"
+      input: "random_flip_left_right/random_uniform/mul:z:0"
+      input: "random_flip_left_right/random_uniform/min:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/Less/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 0.5
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/Less"
+      op: "Less"
+      input: "random_flip_left_right/random_uniform:z:0"
+      input: "random_flip_left_right/Less/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/Switch"
+      op: "Switch"
+      input: "random_flip_left_right/Less:z:0"
+      input: "random_flip_left_right/Less:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/switch_t"
+      op: "Identity"
+      input: "random_flip_left_right/Switch:output_true:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/switch_f"
+      op: "Identity"
+      input: "random_flip_left_right/Switch:output_false:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/pred_id"
+      op: "Identity"
+      input: "random_flip_left_right/Less:z:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_BOOL
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/ReverseV2/axis"
+      op: "Const"
+      input: "^random_flip_left_right/switch_t"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/ReverseV2"
+      op: "ReverseV2"
+      input: "random_flip_left_right/ReverseV2/Switch:output_true:0"
+      input: "random_flip_left_right/ReverseV2/axis:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tidx"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/ReverseV2/Switch"
+      op: "Switch"
+      input: "random_flip_left_right/control_dependency:output:0"
+      input: "random_flip_left_right/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@truediv"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/Switch_1"
+      op: "Switch"
+      input: "random_flip_left_right/control_dependency:output:0"
+      input: "random_flip_left_right/pred_id:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "_class"
+        value {
+          list {
+            s: "loc:@truediv"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "random_flip_left_right/Merge"
+      op: "Merge"
+      input: "random_flip_left_right/Switch_1:output_false:0"
+      input: "random_flip_left_right/ReverseV2:output:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    node_def {
+      name: "Reshape_1/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 3
+              }
+            }
+            tensor_content: "\340\000\000\000\340\000\000\000\003\000\000\000"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape_1"
+      op: "Reshape"
+      input: "random_flip_left_right/Merge:output:0"
+      input: "Reshape_1/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "Reshape_2/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape_2"
+      op: "Reshape"
+      input: "ParseSingleExample/ParseSingleExample:dense_values:0"
+      input: "Reshape_2/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "Cast_1"
+      op: "Cast"
+      input: "Reshape_2:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    node_def {
+      name: "sub_1/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "sub_1"
+      op: "Sub"
+      input: "Cast_1:y:0"
+      input: "sub_1/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    ret {
+      key: "Reshape_1"
+      value: "Reshape_1:output:0"
+    }
+    ret {
+      key: "sub_1"
+      value: "sub_1:z:0"
+    }
+  }
+  function {
+    signature {
+      name: "tf_predicate_7089b845"
+      input_arg {
+        name: "arg0"
+        type: DT_FLOAT
+      }
+      input_arg {
+        name: "arg1"
+        type: DT_INT32
+      }
+      input_arg {
+        name: "Equal/Placeholder"
+        type: DT_INT64
+      }
+      output_arg {
+        name: "Equal"
+        type: DT_BOOL
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+    }
+    node_def {
+      name: "Shape"
+      op: "Shape"
+      input: "arg0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice"
+      op: "StridedSlice"
+      input: "Shape:output:0"
+      input: "strided_slice/stack:output:0"
+      input: "strided_slice/stack_1:output:0"
+      input: "strided_slice/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "Equal"
+      op: "Equal"
+      input: "strided_slice:output:0"
+      input: "Equal/Placeholder"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    ret {
+      key: "Equal"
+      value: "Equal:z:0"
+    }
+  }
+  function {
+    signature {
+      name: "_make_dataset_5fa5e1f4"
+      output_arg {
+        name: "PrefetchDataset_1"
+        type: DT_VARIANT
+      }
+      is_stateful: true
+    }
+    node_def {
+      name: "TensorSliceDataset/MatchingFiles/pattern"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "$(DATA_DIR)"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "TensorSliceDataset/MatchingFiles"
+      op: "MatchingFiles"
+      input: "TensorSliceDataset/MatchingFiles/pattern:output:0"
+    }
+    node_def {
+      name: "TensorSliceDataset"
+      op: "TensorSliceDataset"
+      input: "TensorSliceDataset/MatchingFiles:filenames:0"
+      attr {
+        key: "Toutput_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/MatchingFiles/pattern"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "$(DATA_DIR)"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/MatchingFiles"
+      op: "MatchingFiles"
+      input: "ShuffleDataset/MatchingFiles/pattern:output:0"
+    }
+    node_def {
+      name: "ShuffleDataset/Shape"
+      op: "Shape"
+      input: "ShuffleDataset/MatchingFiles:filenames:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/strided_slice/stack"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/strided_slice/stack_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/strided_slice/stack_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/strided_slice"
+      op: "StridedSlice"
+      input: "ShuffleDataset/Shape:output:0"
+      input: "ShuffleDataset/strided_slice/stack:output:0"
+      input: "ShuffleDataset/strided_slice/stack_1:output:0"
+      input: "ShuffleDataset/strided_slice/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/Maximum/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/Maximum"
+      op: "Maximum"
+      input: "ShuffleDataset/strided_slice:output:0"
+      input: "ShuffleDataset/Maximum/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/seed"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/seed2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset"
+      op: "ShuffleDataset"
+      input: "TensorSliceDataset:handle:0"
+      input: "ShuffleDataset/Maximum:z:0"
+      input: "ShuffleDataset/seed:output:0"
+      input: "ShuffleDataset/seed2:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "reshuffle_each_iteration"
+        value {
+          b: true
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_1/buffer_size"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 1024
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_1/seed_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_1/seed2_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_1"
+      op: "ShuffleDataset"
+      input: "ShuffleDataset:handle:0"
+      input: "ShuffleDataset_1/buffer_size:output:0"
+      input: "ShuffleDataset_1/seed_1:output:0"
+      input: "ShuffleDataset_1/seed2_1:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "reshuffle_each_iteration"
+        value {
+          b: true
+        }
+      }
+    }
+    node_def {
+      name: "RepeatDataset/count"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: -1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "RepeatDataset"
+      op: "RepeatDataset"
+      input: "ShuffleDataset_1:handle:0"
+      input: "RepeatDataset/count:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset/cycle_length"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 8
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset/block_length"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset/sloppy"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_BOOL
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_BOOL
+            tensor_shape {
+            }
+            bool_val: true
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset/buffer_output_elements"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset/prefetch_input_elements"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 16
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelInterleaveDataset"
+      op: "ParallelInterleaveDataset"
+      input: "RepeatDataset:handle:0"
+      input: "ParallelInterleaveDataset/cycle_length:output:0"
+      input: "ParallelInterleaveDataset/block_length:output:0"
+      input: "ParallelInterleaveDataset/sloppy:output:0"
+      input: "ParallelInterleaveDataset/buffer_output_elements:output:0"
+      input: "ParallelInterleaveDataset/prefetch_input_elements:output:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+          }
+        }
+      }
+      attr {
+        key: "f"
+        value {
+          func {
+            name: "tf_map_func_91295dea"
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_2/buffer_size_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 1024
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_2/seed_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_2/seed2_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset_2"
+      op: "ShuffleDataset"
+      input: "ParallelInterleaveDataset:handle:0"
+      input: "ShuffleDataset_2/buffer_size_1:output:0"
+      input: "ShuffleDataset_2/seed_2:output:0"
+      input: "ShuffleDataset_2/seed2_2:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_STRING
+          }
+        }
+      }
+      attr {
+        key: "reshuffle_each_iteration"
+        value {
+          b: true
+        }
+      }
+    }
+    node_def {
+      name: "ParallelMapDataset/num_parallel_calls"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+            }
+            int_val: 64
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ParallelMapDataset"
+      op: "ParallelMapDataset"
+      input: "ShuffleDataset_2:handle:0"
+      input: "ParallelMapDataset/num_parallel_calls:output:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+          }
+        }
+      }
+      attr {
+        key: "f"
+        value {
+          func {
+            name: "tf_map_func_74b6b15c"
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 224
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 3
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "PrefetchDataset/buffer_size_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 64
+          }
+        }
+      }
+    }
+    node_def {
+      name: "PrefetchDataset"
+      op: "PrefetchDataset"
+      input: "ParallelMapDataset:handle:0"
+      input: "PrefetchDataset/buffer_size_2:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 224
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 3
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "BatchDataset/batch_size"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 64
+          }
+        }
+      }
+    }
+    node_def {
+      name: "BatchDataset"
+      op: "BatchDataset"
+      input: "PrefetchDataset:handle:0"
+      input: "BatchDataset/batch_size:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 3
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FilterDataset/batch_size_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 64
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FilterDataset"
+      op: "FilterDataset"
+      input: "BatchDataset:handle:0"
+      input: "FilterDataset/batch_size_1:output:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+            type: DT_INT64
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 3
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+      attr {
+        key: "predicate"
+        value {
+          func {
+            name: "tf_predicate_7089b845"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "PrefetchDataset_1/buffer_size_3"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 2
+          }
+        }
+      }
+    }
+    node_def {
+      name: "PrefetchDataset_1"
+      op: "PrefetchDataset"
+      input: "FilterDataset:handle:0"
+      input: "PrefetchDataset_1/buffer_size_3:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 64
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 224
+              }
+              dim {
+                size: 3
+              }
+            }
+            shape {
+              dim {
+                size: 64
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    ret {
+      key: "PrefetchDataset_1"
+      value: "PrefetchDataset_1:handle:0"
+    }
+  }
+}
+)PREFIX";
+
+  *dataset_name = "_make_dataset_5fa5e1f4";
+  std::function<void(FunctionDef*)> mutate_proto_func =
+      [dataset_name, file_path](FunctionDef* fdef) {
+        VLOG(1) << "Processsing function " << fdef->DebugString();
+        if (std::string(fdef->signature().name()) != *dataset_name) return;
+        // Change the input file pattern to `file_path`.
+        bool found = false;
+        for (auto& node_def : *fdef->mutable_node_def()) {
+          if (node_def.name() != "TensorSliceDataset/MatchingFiles/pattern" &&
+              node_def.name() != "ShuffleDataset/MatchingFiles/pattern")
+            continue;
+          DCHECK_EQ(node_def.op(), "Const");
+          DCHECK_GT(node_def.attr().count("value"), 0);
+          found = true;
+          DCHECK_EQ(node_def.attr().at("value").tensor().string_val(0),
+                    "$(DATA_DIR)");
+          VLOG(1) << "Setting the value of node_def "
+                     "TensorSliceDataset/MatchingFiles/pattern to "
+                  << file_path;
+          auto* tensor = (*node_def.mutable_attr())["value"].mutable_tensor();
+          tensor->clear_string_val();
+          tensor->add_string_val(file_path);
+        }
+        VLOG(1) << "Rewrote function to " << fdef->DebugString();
+        DCHECK(found);
+      };
+  return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status);
+}
+
+// Adds the input functions to `graph`.  On success, returns the created
+// IteratorGetNext node.
+static TF_Operation* AddDatasetFunctionAndIteratorNodesToGraph(
+    const std::vector<UniqueFuncPtr>& funcs, const std::string& dataset_name,
+    const std::vector<tensorflow::DataType>& output_types,
+    const std::vector<tensorflow::TensorShapeProto>& output_shapes,
+    TF_Graph* graph, TF_Status* status) {
+  DCHECK(!dataset_name.empty());
+  for (auto& func : funcs) {
+    TF_GraphCopyFunction(graph, func.get(), /*gradient*/ nullptr, status);
+    if (!status->status.ok()) {
+      return nullptr;
+    }
+  }
+
+  tensorflow::mutex_lock c(graph->mu);
+
+  tensorflow::NameAttrList func;
+  func.set_name(dataset_name);
+  // Run the iterator node on CPU.
+  Node* oneshot_iterator_node;
+  tensorflow::Status s = NodeBuilder("OneShotIterator", "OneShotIterator")
+                             .Device("/device:CPU:0")
+                             .Attr("container", "")
+                             .Attr("dataset_factory", func)
+                             .Attr("output_types", output_types)
+                             .Attr("output_shapes", output_shapes)
+                             .Attr("shared_name", "")
+                             .Finalize(&graph->graph, &oneshot_iterator_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+  // Run shape inference function for each newly added node, so that more
+  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
+  s = graph->refiner.AddNode(oneshot_iterator_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+
+  // Run the iterator node on CPU.
+  Node* getnext_node;
+  s = NodeBuilder("IteratorGetNext", "IteratorGetNext")
+          .Input(oneshot_iterator_node)
+          .Device("/device:CPU:0")
+          .Attr("output_types", output_types)
+          .Attr("output_shapes", output_shapes)
+          .Finalize(&graph->graph, &getnext_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+  // Run shape inference function for each newly added node, so that more
+  // subsequent nodes can be added to the graph via C API (TF_NewOperation()).
+  s = graph->refiner.AddNode(getnext_node);
+  if (!s.ok()) {
+    status->status = s;
+    return nullptr;
+  }
+
+  VLOG(1) << "Output graph: " << graph->graph.ToGraphDefDebug().DebugString();
+  return ToTF_Operation(getnext_node);
+}
+
+TF_Operation* TF_MakeFakeIteratorGetNextWithDatasets(TF_Graph* graph,
+                                                     TF_Status* status) {
+  tensorflow::Status s;
+
+  std::string dataset_name;
+  UniqueFuncPtr result_func = CreateFakeDatasetFunction(&dataset_name, status);
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  std::vector<UniqueFuncPtr> funcs;
+  funcs.push_back(std::move(result_func));
+  std::vector<tensorflow::TensorShapeProto> output_shape_list;
+  output_shape_list.push_back(tensorflow::TensorShapeProto());
+  auto* getnext_node = AddDatasetFunctionAndIteratorNodesToGraph(
+      funcs, dataset_name, {tensorflow::DT_FLOAT}, output_shape_list, graph,
+      status);
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  return getnext_node;
+}
+
+TF_Operation* TF_MakeImagenetIteratorGetNextWithDatasets(TF_Graph* graph,
+                                                         const char* file_path,
+                                                         int batch_size,
+                                                         TF_Status* status) {
+  tensorflow::Status s;
+
+  std::string dataset_name;
+  const auto& funcs =
+      CreateImagenetDatasetFunctions(file_path, &dataset_name, status);
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  std::vector<tensorflow::TensorShapeProto> output_shape_list;
+  // batch_size X 224 X 224 X 3
+  auto image_shape = tensorflow::TensorShapeProto();
+  image_shape.add_dim()->set_size(batch_size);
+  image_shape.add_dim()->set_size(224);
+  image_shape.add_dim()->set_size(224);
+  image_shape.add_dim()->set_size(3);
+  output_shape_list.push_back(image_shape);
+
+  // batch_size
+  auto label_shape = tensorflow::TensorShapeProto();
+  label_shape.add_dim()->set_size(batch_size);
+  output_shape_list.push_back(label_shape);
+  auto* getnext_node = AddDatasetFunctionAndIteratorNodesToGraph(
+      funcs, dataset_name, {tensorflow::DT_FLOAT, tensorflow::DT_INT32},
+      output_shape_list, graph, status);
+  if (!status->status.ok()) {
+    return nullptr;
+  }
+
+  tensorflow::mutex_lock c(graph->mu);
+  VLOG(1) << "The extended graph: "
+          << graph->graph.ToGraphDefDebug().DebugString();
 
-  VLOG(1) << "Output graph: " << graph->graph.ToGraphDefDebug().DebugString();
-  *dataset_func = result_func.release();
-  return ToTF_Operation(getnext_node);
-}
-
-void TF_GetAttrScalarTensorShapeProto(TF_Buffer* value, TF_Status* status) {
-  status->status = Status::OK();
-  auto shape = tensorflow::TensorShape({});
-  tensorflow::TensorShapeProto shape_proto;
-  shape.AsProto(&shape_proto);
-  status->status = MessageToBuffer(shape_proto, value);
+  return getnext_node;
 }
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index 2fa232878c..a9c551d73e 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -87,25 +87,22 @@ TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
 TF_CAPI_EXPORT extern const char* TF_GraphDebugString(TF_Graph* graph,
                                                       size_t* len);
 
-// Creates a stack of data set + iterator nodes reading the TFRecord files from
-// `file_path`, and outputs the following info on success:
+// Creates a stack of data set + iterator nodes, currently hard-coded to return
+// a sequence of 3 float values <42.0, 43.0, 44.0> over 3 calls. On success,
+// returns the IteratorGetNext node, which caller can run or feed into an node.
 //
-// 1. Returns the IteratorGetNext node, which caller can run or feed into an
-// node.
-//
-// 2. Sets `dataset_func` to the created function that encapsulates the data set
-// nodes. Caller owns that function, and must call TF_DeleteFunction() on it.
-//
-//
-// The nodes are currently hard-coded to return a single Int32 of value 1.
 // TODO(hongm): Extend the API to allow customization of the nodes created.
-TF_CAPI_EXPORT extern TF_Operation* TF_MakeIteratorGetNextWithDatasets(
-    TF_Graph* graph, const char* file_path, TF_Function** dataset_func,
-    TF_Status* status);
-
-// Returns the shape proto of shape {}.
-TF_CAPI_EXPORT extern void TF_GetAttrScalarTensorShapeProto(TF_Buffer* value,
-                                                            TF_Status* status);
+TF_CAPI_EXPORT extern TF_Operation* TF_MakeFakeIteratorGetNextWithDatasets(
+    TF_Graph* graph, TF_Status* status);
+
+// Similar to the above API, except that the returned iterator reads the
+// TFRecord files from `file_path`.
+// The iterators outputs 2 tensors:
+// - A float tensor of shape `batch_size` X 224 X 224 X 3
+// - An int32 tensor of shape `batch_size`
+// TODO(hongm): Extend the API to allow customization of the nodes created.
+TF_CAPI_EXPORT extern TF_Operation* TF_MakeImagenetIteratorGetNextWithDatasets(
+    TF_Graph* graph, const char* file_path, int batch_size, TF_Status* status);
 
 #ifdef __cplusplus
 } /* end extern "C" */
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 9ddd65f0c5..49d64d18bf 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -15,38 +15,36 @@ limitations under the License.
 
 #include "tensorflow/c/c_api_experimental.h"
 #include "tensorflow/c/c_test_util.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
 namespace {
 
-void TestIteratorStack() {
+void TestFakeIteratorStack() {
   TF_Status* s = TF_NewStatus();
   TF_Graph* graph = TF_NewGraph();
 
-  TF_Function* dataset_func = nullptr;
-
-  TF_Operation* get_next =
-      TF_MakeIteratorGetNextWithDatasets(graph, "dummy_path", &dataset_func, s);
+  TF_Operation* get_next = TF_MakeFakeIteratorGetNextWithDatasets(graph, s);
   ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
 
-  ASSERT_NE(dataset_func, nullptr);
-  TF_DeleteFunction(dataset_func);
-
   CSession csession(graph, s);
   ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
 
   // Run the graph.
-  for (int i = 0; i < 1; ++i) {
+  const float base_value = 42.0;
+  for (int i = 0; i < 3; ++i) {
     csession.SetOutputs({get_next});
     csession.Run(s);
     ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
     TF_Tensor* out = csession.output_tensor(0);
     ASSERT_TRUE(out != nullptr);
-    EXPECT_EQ(TF_INT32, TF_TensorType(out));
-    EXPECT_EQ(0, TF_NumDims(out));  // scalar
-    ASSERT_EQ(sizeof(int32), TF_TensorByteSize(out));
-    int32* output_contents = static_cast<int32*>(TF_TensorData(out));
-    EXPECT_EQ(1, *output_contents);
+    ASSERT_EQ(TF_FLOAT, TF_TensorType(out));
+    ASSERT_EQ(0, TF_NumDims(out));  // scalar
+    ASSERT_EQ(sizeof(float), TF_TensorByteSize(out));
+    float* output_contents = static_cast<float*>(TF_TensorData(out));
+    ASSERT_EQ(base_value + i, *output_contents);
   }
 
   // This should error out since we've exhausted the iterator.
@@ -60,7 +58,63 @@ void TestIteratorStack() {
   TF_DeleteStatus(s);
 }
 
-TEST(CAPI_EXPERIMENTAL, IteratorGetNext) { TestIteratorStack(); }
+TEST(CAPI_EXPERIMENTAL, FakeIteratorGetNext) { TestFakeIteratorStack(); }
+
+TEST(CAPI_EXPERIMENTAL, ImagenetIteratorGetNext) {
+  TF_Status* s = TF_NewStatus();
+  TF_Graph* graph = TF_NewGraph();
+
+  const string file_path = tensorflow::io::JoinPath(
+      tensorflow::testing::TensorFlowSrcRoot(), "c/testdata/tf_record");
+  VLOG(1) << "data file path is " << file_path;
+  const int batch_size = 64;
+  TF_Operation* get_next = TF_MakeImagenetIteratorGetNextWithDatasets(
+      graph, file_path.c_str(), batch_size, s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+
+  CSession csession(graph, s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+
+  // Run the graph.
+  // The two output tensors should look like:
+  // Tensor("IteratorGetNext:0", shape=(batch_size, 224, 224, 3), dtype=float32)
+  // Tensor("IteratorGetNext:1", shape=(batch_size, ), dtype=int32)
+  for (int i = 0; i < 3; ++i) {
+    LOG(INFO) << "Running iter " << i;
+    csession.SetOutputs({{get_next, 0}, {get_next, 1}});
+    csession.Run(s);
+    ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+
+    {
+      TF_Tensor* image = csession.output_tensor(0);
+      ASSERT_TRUE(image != nullptr);
+      ASSERT_EQ(TF_FLOAT, TF_TensorType(image));
+      // Confirm shape is 224 X 224 X 3
+      ASSERT_EQ(4, TF_NumDims(image));
+      ASSERT_EQ(batch_size, TF_Dim(image, 0));
+      ASSERT_EQ(224, TF_Dim(image, 1));
+      ASSERT_EQ(224, TF_Dim(image, 2));
+      ASSERT_EQ(3, TF_Dim(image, 3));
+      ASSERT_EQ(sizeof(float) * batch_size * 224 * 224 * 3,
+                TF_TensorByteSize(image));
+    }
+
+    {
+      TF_Tensor* label = csession.output_tensor(1);
+      ASSERT_TRUE(label != nullptr);
+      ASSERT_EQ(TF_INT32, TF_TensorType(label));
+      ASSERT_EQ(1, TF_NumDims(label));
+      ASSERT_EQ(batch_size, TF_Dim(label, 0));
+      ASSERT_EQ(sizeof(int32) * batch_size, TF_TensorByteSize(label));
+    }
+  }
+
+  // Clean up
+  csession.CloseAndDelete(s);
+  ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
+  TF_DeleteGraph(graph);
+  TF_DeleteStatus(s);
+}
 
 }  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/c/testdata/tf_record b/tensorflow/c/testdata/tf_record
new file mode 100644
index 0000000000000000000000000000000000000000..6e16076bfb79ad8151952e96567565e8820b0f5b
GIT binary patch
literal 417114
zcmeC*$jAT!EE5-;;p%?O#wEtZlbM^Co~oabnwg$aBE%-e!o~E2i7SCi03x28lUQ7=
zUy@o;BILv6#pRKlUz(m+q@z$!nwzMjkepbOn44dkSE8c;rV=yr6ml~2^K=vn5;OBk
zQb7t6^GY&647b#r%wmP){JiA+octm#H7+rT4f#n~smUe!NlEz?`jxqf6+%2hTwELr
zItA<-JnHSaKHp>G;)SS8%}dTtNlg)YaF31a&OJ7+8~51$-}wK4LBPw+(~W_Vk&(fJ
z!GnR}|6Ap(fQ<Z-{NjxK0tI6|0|pBxPhS=mkjEJWWSAHjm>8I3m`WH}6ciK`z;db(
zxu7joHi5BAR$P9p7WD9#@m-J{^IM1<$Ycga29O;LAQ~jcq6U`Z1CwCFzJUQ^8!wm(
zvQrMMQWBNV{29#W1d|LP1Hr1ez<frC8R`OH9v1^7&KMY&Kx&x3f<-+T^wZK97#VWX
z(iqslB8*Uxg)?WtMU22kM=>zKMNGjWx(E@lnLofL6fiLS#|oHVfaI}C{ij3@WH%^?
z7(h`12{$H?O$<zs5Mbc|^FZOi$iTql0Krf>Ij|g9JJ=j45T7|2?lTaVfiZ=Ff$;_d
z1A{n&FM~frD1#qE2!kVoCxah@0)r2OCxbtj<-*{~;K~rhpuiB!;KAU}5CRS*un8yx
zEG9tGHyFUCqsYUf3Rx8De2`FHX-*ErvkYuW`K5U&#r{cIASXi&4oWUDWME(bdnzQq
zKp`hTFCC;0gnd%eN|5<ZCHc7!KFE^{3_<A`P<gP2*@|;AlT(Ysa(s$Gu0^qj6)eZV
zz#zoHz`zdXc)CE$0@=jCo?et+T7V+NmS2>ao|%`LSDMSfz`)}g3^LsxEXKgVz?)cF
zlJB0Hms*rql9~cCBc!q*6{?sEq8KCslJU$<R{%v4js4GDT9gyw8i5p(puk5J@hwhI
zMG;|3%qa;;Oh*ypOHR$n2}!Le@htWT@%2f{&(8tH76(iiMKNbaeo>WUPG&kRr1&A8
z_kan50z4%(EwMDGgn@y9vn;i!1dH-8m@ukx?xb{QP~iy*N@(B;Ld<t^hf9FM*e^dX
z6=Zx#et~~!NpUJPVSuC<xN=g{N-#uulJZMR@^dl7xr;#c0-7*5ziEI?;{hcu`ww6?
zBnOLt*`OrM304PE!62ImQpdo+uz-Pqp^|}t@c{$FqX!HOvaSpg42;O7I1{LZ1()Is
zJdP>(NvZM9z6^~2-!brkc?>BHAd~*zW@usH;o#ul<ly1t<PqfN;uaL;=i%WOl@Jyd
z6&98d<N*UH1!iLqjJ(|3ynMU@e0%~Te0+Q&Ac9W>qD2rw@jxjIy<ma=2N(o77%CYS
zFf%GKFbOg;3o`z{%fQ0G_5TQ?Jp%(HD<c?y1B8KrnSqgsg_Vt+gOiJ!5nYmzfr%L=
zDZs$U!pO|T!o<qX#=*+WW6a3FB*@I7C}ha$7?>!mRM=?5CNgp1Mki$z(V&A##wJBg
z7oFKvCk20a_%XTIRBX|vW;K_|At|nh7MGZbUkZJc`e}2^;mf6uKf9?<S<<>?>X9o?
zzAW8(^y<^EVQFP;)0S;JcJ0|WbNBG{^7iS=w;#X${JS6n6B8p73k%py>};&2U?(UV
z3b8l_CKf6Qvl=x{6al&5V9>=6#wtk<i<%}m2a6^bn|xGd137?Q&GgVEm&qZE#Zz2E
z%}P=q!JUBH1?mVV{J+J(!wj}lkinke{L-_@+s<FQ#xap4_dEa7x=*U*HUi&vr*3^E
z$H}+$jNNs!>u>8<M*nAcQGamFpX%B$2EVl(Hu7R0>NoF<J7&E^e8;5P1D(>3Grmod
z-ua*5?0<#_^Oj#;Yjx>s&MS+ZvRd8iHc2M()H+3d_?GikJG=e)UwQw#{~0(IUYr;8
z`{=*vs|$<I*<DV)bIkVYui3MAmz_Pg{@?ryk+$|K7vg=-Da)(2ZqLovm9y^qzH^;!
z{-uBJzf>3Ltry)=JL{r%N%6gB;_O=wF6_x|{w$p>AO9=rum2aXWf2@g440x>TQ43s
zzG~;Oea+!(e0Mw!@Me@dmh&&^mQMZZ@b}lt|A?;U-O;mS$Mu}O{pG8kpWiy^*2llM
zXUDJ4-Tj~8+SV1t>rPbGX)1mTdf0a2&G8LeZ}QkZ+kfTzuPOf-OjcQS-#NYUbK#s*
zS$0cz@=W6C&HZxw&+qy#llqQV?dvGqG{^8wsp}_)V(!%n(}mu~{=EOOTJk?beMpY#
z)CmV}&d`0wZoKUNbGa?Q`iyV;TIwceN|%=Z@_c&l@z2>a-`w2Wt+~~6)3M1rw(na!
z?eo8^`2sucf4XQWc3b`WvPKURvCFa1(yLErtCwy5C;Fek{j-2}LSgZY@;ye!nH;8F
z-?}FG-s|KqdD;8j-~227XZT`jdr#P`+13$^-`Qpgzc22;nE$P|bFWZ#>AUn*58STj
z3hrXQ`?q#>{k8pp5hCIYo!@5ZX3vh!&Gyh+bt&dj_P4ix-}}z@|0iC>Jtsc6`HS6Z
zj!c1PZM$D=-1zg{SGjc?*KAxcTY|HFdd5el=$Cu<oew(opJDDl_m}k#F3h`Td2kJT
zh83q8TYKKmU-dK7_o{4tzxHROa&~R>r#0Ex*$H{YcQ;M@&!GJ~d*=&VdwJodi4P4n
z{?^-aKZo(v4$H^&yJByD{?BkQWW&BSUp7y#{aNjHCyg!W@k+BB&$eI9`OnaDeC_)G
z4BIth>-1DYB3_$|MHkmD5PO|{cXQyE{|vRE=dEp&mp5IivfLZ4yJG^2c=Cm38*lp^
zio2e?ao>WxyKlee-TY$fdh~BH&u)8pt!+|{a?_VpetFf{pS-Q^Z_eMZcfQV>ayx77
z)recNZqFybQ$73a*0*<G{+gRt-+vpE7To3^vi6C<p{!zwXKfQW?`{dZpFDf_i|Bu%
z#=IZ%H{HH^db8X$v*~R@?*m)EPFyp$fBmoBcXsQ|%*!qP%D?IQu4}0`x9)O#-XEUr
z{&k)2RV&%~+2=P+uL?C>Uj0D6G-P(xw^jMCnA-NQ2+%Qox>;UXjqkXHnAG|ucMcqT
zUvhin*Uf*WN|vO`@_(At9sN4a^!f7mJO{gtCsy4r*`3`Sb=y{bYj6J6^*4UoyX~wo
zH(k4dH+qXz{VG}Qy(bTyIlEe``)%>dGmGxK^<KJr)}$&d;{C<zuguG?<>W8-fAQD$
z_4(EOYi#?ZV;tAMvF)&X`7b#5L(MYv^WW<4EVB;&f8lw|^Q0#>vWidpnIjph+rQn2
z+4A~SWui)VxnGMz!G-MQJMZ29m9;LH<<99C+0b%^T8nnaoDGNf-g`Uywd0|^SM5_b
z3hP~uwVnN+Vd7gab%zD7-epVuJy+77P$1+f?|Sy`{wv?!+}-{CKf~*;{|vsz($XA`
z723_QIwa+sdN%J_|C`Na*VkN%`E;A@5=;Ae;g$F8U%#wx`WLxzernllEyZc=d3?`j
z-1%nOy8rcm28k&T)U@AdDcx(@Z7B7sNO#?Hv&q4QISkFAiG4Ts-hU`}?0)z!`{!au
z_xg(jzKVKu_HCh^E-&l(vg6g;%(KqFjJ%(5SZROq3*O9(lDTWMv(xwP_r7(sZry_8
z)!VnZvz+O_*DmmW$?9tL@2g+zzO{jOM{Tz3&RBEhRlm+v$1ty4dSJtq<B{Imru{4b
z;s5!%Y5(dIERNZ`k2mjs6|tAu{LRf7*Fr14YH#6sbE|Bw-?QrW3*XnJuc(bYdGN2D
z^4zK3KGOtP*G_(WM|#`ti2L``|87}!#3CulKUT<ag7r?GMU$S-=HOk^yDLli_Pc-U
zFS~C18u{|_E4DpCGWi*zrtQJ|UmCr(y?r|<xnS+VX<K%l|8BZYIkx`L8@cVf(?3_Q
z-u#D4^49HH+)<w{JdfP(s8*hP!+g`WT2{?a6}4^bzgV~LUBk=JR(4M|`tCp8-i_zK
z-1z%%{ogGOx7Kb*z8G8{U%<ocF{$?azx9`k|1&(8$QPQnK!({jH`}}C$c8r4wTZmR
zv0t>SUhZ4{<(B4(=TDqJ6*+wQ-f=VV>b~jkZcqO5_CG_@m;VgY7N4HrWd2Q8wT7Xp
zII`=wcwlIP#nz41ImLgo>n-;!zVm$lzkgdJQ}c>vWU7dZi*q*b-czeyJ#}$x(!sQA
z)7jq2U!9QruPc3bvEQj<b2$5ct+JY&6Z)w4Yk0EUg^ObR1<D@pch&B9EfOzTJpJoo
zr8%+tdQHMJDvX%tyn1{4_qtzY*5Acr%NG{QhBg=5Tei$FaNgQqbYwBt8u>M1sma@<
zi?tSR>0cE6a_76pFYJ~FMsNSma7_KcMU^tmvYdsj+t_7qADnn#?rz)Dou~84J$~E!
z<+{K5u{*c8Z*6UUan6lzoA3V8{=WO?{qyOcv$tp07VceQxuig~cy93Z{p#j#O}0&J
z?U6a~dGEUVmC?5!zOL)PoxVN&Z=r;IXjn(di(0W*Z>a~bx7~iVbKj-w*Ivy}uAg*0
z;y=U1fBzYNm_DoZmb?+`t}gq`TP8;Kk?i}-xCz{bdM~F19jh|idS&mMv}2PKzS`C<
zev$XDus}X?#<S8tpN{R=xZOf{^|nswlz*@7nkMs2`po`s(u;Th8UA0qdg5jH^OAhu
zFTct!#+GiGab<()TKzxIeoOvxJ#xD*EB~cb#IKXttBkH29xnLq|IYs9djEax+uFho
z#U`&6c;v!pzx(Hz<obi_++Q#EOEXIizj8==W*{GnxbN0l^ZyLTZbz@Yx8UjX^_;Ia
z-S}2y@V+Fsc;R8UUPhLG#j$_qfBw&K@z#2_j*RSjcZQwozO^&3KR2n~`S<>_c^k8D
zNL@c;A7I+Q_~)*=*VC5SxqWabWca<U*Yy4GeeeIAjpROD&u#REy=PX(-Mh<}%l=h!
ztN%Ke{_{VBW9F=xSETODn?E7qxbytJhugFJcYofN`}b>WTCj99XWIQrL)NpCxR>94
zm;I)E`-}e!&3;zRYb<`0Sv=ldUHGwTcH+C_n6k%@N_m3+s*A2GUmkmQ^R;9D8Cw5p
zr8!phullI<k#BcZyZ_6dg7YWIxHxAO@TP6}o>INdu;=d1YY~56FIcU$?b@M}ItCA!
zW0&OJ`Ok27bN;26=3SaEZ}Mzj9cC0KbLjSwIkmsu{1v;Fv$@!I>izgX@hi6Ko>wvc
z(!#uW&iy5vzxe;1zIDC&_OJM7R}zl3)~<e(zdGIi)jz(}b0?4K96M2{vZMFP&co9q
zz5VWo-~98nZqbpmFXlS)h(|{2BwRY3wP6qM9clH{sT=QKiRIIMSNGuUy3AV=7g9R^
ze9!%5_v~NM?tmL{C!}M%yHh5V*V^9RUuom9;)r<5%ekC;gI8a5_?NqT&8dnj+m=nx
z&5r+h?e@+cXR5vwnY(Y!xWxXf!6bqC*TU2<bL8e<T77c0bYbz^ukuzq4EnFW`WIUI
zZ*t!IWnaomt+Lh^&w6(Ledg1jr(~9#a$mq(Im@KV?C$pO&HU^+ezShA5V)=2X}k5x
zi`CaB`aY0NIC5^zO15pa>y6Jns_qC7Ty<95;^5NTxy6i6`0BXK9&cRtwdEa~2g^t4
zkfY&_Liai4t0xt1-1ppp!AJN%L(mnru4Qe>t2W)5cZqZ9r56g`i~WPYZ!<ktY(H&s
z;f!B(Co*+cUbvv_aN+Im^t->$|7Wm`%YMl+fx-Ah{?owP^^+txo77MIP_ENhFSfVT
za)0{DwTfF#B&dHbvs&>l{B!rbe7EDLHd`wj9Zz~3^Jh-^5q1yL@L!hsm+pQEcYhW2
z{Dr4L<z(S^{~4BY+poMkLnF^lI^s@qLgA)^inHguR^0lp`c1a{{=cH~Uzfb_xLfW~
z|Nqh(*7MbmI}{q_VlP-7<9Ib!?~J?qijS{?zB}Abwx3zO{Bz9v3z7dBShHrnl4w1b
zQ1a%&0Ym$rUA;R^B6au*>Q%M}udw&ee`fz#tY9Kbxzxswc9|1gS<h}=o4b2~$u^nm
zd5T~6XWV*U9J4g~4PQYqbIQz2cISux)<w^cs`8NDwUkfn`tn;P(+!{h+&$;wKfe1P
z=D&FRpP?t>V1G43!FOKS<@IO(yuTYIFk_BO`zM`-8{00_Tzj^w+`{v&+)KG{FTd3O
zXINTxMQ6i*hRw4+-ic1Fd*&g;!)|)}w)~QNwf<kL<c#OVsW`iDnQUP8yYK$luuYm9
zLcDo|+%Lb0UA}ki-Hq@6GhEF4vh&Wb*RzbR(pYs>&z%=dv%T-ZwsrsFo7?}Y{*3>@
z_4LErR~FYUoR8Qqcl^qQ<1<Ta?`*w4`^{gyBU#$X+Sxf}-@<OTHKo6_+Mhdn_OI|8
z=d0^4{p(wwWM}HLS#P`G-l;b(dHzza{I<XGSNPufi`V}=a^kUIZ~jgX(QmtVTAn?!
za$ATuqjlT?2R3{Avio<PU+#}!oi^)^Y8%I0W8p8WH-Fxy^L|ZfjM2@$o28}9$unNE
zOrP&Jtv+?_)>HA}y9{43t6q<ay0yYI=-FX!-GZC0&G{F9+no=bzi(!Yp|P#FS4qv>
z&lBevuig>Ctyi#cLF3D<+a!3W<&-_ExBs$#PE>zt-iqt_FPXP89pGz!vGHJf$-cSM
zzpXkRa5y_>(cb?I8KrAB9y`f%+gD`gy}6~o-`@W>{qmQU;p<l(nU;5ZyGN%&|J`5l
z*Fryg2Zk;WI6AF!!--Apu6>QWCzS}UnvnDM#!r?5{rXEp(;h|fU(%cF(5q%6vhVbi
zOL1@a2r*bSKmS+M9lTmr{mYw_tw**8F1%`XTkg_r4uxLRr-s=x4&9bhZhCn^*4sHa
zZ-W)HXL3n#*^DN(v&(<op7<fC)komDe91edl7Fj|i?q`>*R5q@FIN7~pt5>@_5t0L
z{z8Q!j>o@t2bEUK$tGkwFFmkJJ(#!u$zBFik1g*{tZY1|CvYL=kwwGZ{|sMwLpGlM
zCYF-Zoip3J|Jk+!cYEsIpIf&qJp7x$+b=r$3lDJ3j;wwAyzJ4#P@SB2x1Mjl`b++7
zUH2dT7q^98MR$5K9xj%6E_tc!-EH|>MHMXE9&-BsFTc_}z9{C_+iNoMJMye6s-ur@
z4>j&NbV=&^wd+}BTRi*oc8A(T&xkG7trUy4o6PL(z;kuqOx}C%?}WWSy|%$>b?w>f
zmveF%w!W91`}6j^Z#w_@rIST(CVun!>+o=Q#_hDf%-b`Q&9>y+_@>9Z^}e6$Rg<d3
zY5R86EphJ6&3wB5!MAmnx;y!Dr&TYmo^gNiN;!MaqMZxRoIjm7T{ydY?uuzUe)WE<
zEBnt781dBA%XtTDjX>D<i`BW=-K#SVqwl``%9!_E>|gmG<^K#5Cg-YO-ezw8GM8b8
z>B7qiQhV2~zx+*o)5cfK3~eV%m{&V~w_W^w_U@wchU+;BI|P3jt1kGf?>+OcY<}*x
zUtYJq-n;(G^}Y4J`5W&ve%Y)4>(<)I!S~KbSl)KLbMgu|!@bur=a)RMOM7i!8(S#d
ztW%J1r<_Ukh<WL}2k-Aye_MIEw{$9hb@RRBjPEX(o@Y=0cF+FNe+IWnQ6ay6w!}|N
z+;Hu_Yl8rHHpA}zrSJYTu<WW|z3$kdgNpwyXO^xnxG?S9E$6#`R?U8se&JtX@}_Cm
zZkzS4ttg(wIe}UF<@BQW#rtjlGrW}Bvhn?`Tc`a_&5-78erN68@b}v0{|qbv#cT`M
z>u(&ltjqhm^UG}4(j7OR?=ZeqwmWuR_Wt$1|1(^1`nooCzQ^3>x0uV>ruARiRM#E*
z?(VyY8`oyN<yG(~D0Nf(EO#w)_xHWG@AUeGh2*c;vA2!k%EA22SK20B`(6LRG~nj@
z^=a0n^CxG|kA8K<ZwarDiTBFy=H-_{bIor@7RU4ONy(=@%e%e)kNWn{qKflQ8%cfO
ze#>UN$n0PCx9RfB&hC5vXV&K2@6)eoWw%V=cCKXhT{6FlF^t>#<KZ>ByUwnwnf1Us
z>fqb^lkV=DIy?6L(*F#L{2ouaJ?HtJ2_LOFr0o2j8<khbet8obJs~Ii$;nJ>x3&47
zwr&6QlxusN)wTw2wi{|D&F7_S_x)!mp4`E<I%)pmg_9>}TYIXPbmwHwc<`U$;(vzK
zp|3W6=B?hg<z3Fq;_fB850`(M`$eGW-tX4_CBL?AUAXX^E|cd+y??7#oX(qWRBJ9^
z>wG7-DeBtpKj-iK%e4xW39*@ab${R4`&s;qyXQ#?dFtMLD{%SEzvA*2L9?1&Q|5l1
zW-!t20^7B{x6JR{zw3BpXS3Y>g5=xTa#C+&oVS}^J@flb{l5K6|1)r`<i7Lh!*vU>
zeF}dyt{u<IyF35He})5-m&z)+2mY)I^5vEPe`VjF4Kf!qXUE9?vTRD+xcg<T*nbAa
z6FMh$-8PS0Be=lmQgY(O`E#zA$8K>bD7?L6=N<mLY0Lkp|NR)Y_TX%1PMyu+I|R3Q
z+Rj?Zy*Nj2uDL~)0)vvz+Y;GJ?tfJ$UYVV2)%JYuVYO|x`^Bnd%>KQ*w)$M<!ll1%
zq_7LP<l4@$|D?4(l<kz(CXO%jm=FGPzxr+Gtl!(JrW$%S9cle~v4n3S+s)-yN<Itk
z4sw^Sy#3|xOWp(A>5+HUzb6;(oAAqS{jcAbO4mpnU4HAcN=?E$E0ecQ*1gqh8LaR8
zX9y4%Dz~t{8B^CeJNo6D^w0Z0&RewOOpxU}(@7r}|6QXz%{-I$yII-q^1A%ZNuvA-
zEAOq%O?_XGUA8@&VeY;U{~69y2j7l9%spR)*(@vBZTtRr);Iq%NG!XRy-8Q~T+Msc
z!yZrjiX+~a@4oq;VaX+ji;DZM2Tm=X^We8M<1V%nUHyOA@~ifDJ*z5hYw%`vUv}>S
z!~4MZtuv>^+~Q5X_4YQ)=Ra%j{44m+uwC?5^X$Gu$E{+n<)=-MVUB+L%icNvrTOnG
zGUqOSj-H%wd%4nX^+F*>)*~6}vAf%M|LPKnl6c-*ZX<9^l2@R#;?LRL{+n;LhJ7=-
zyrXUF*)KOdt{nVx@Lg`{e}+iW^=6JnpB}HcrntC#4{OAQYim1aC~ot}_U7N6cPZxm
z+pNx4ey0xHVOy7TZh83j`Q=-8OK#=7W_O-(o3rE>hs=N8^{Zwr-*JSq@XZC&>m8SW
zYn1+rpEPN<-ikN3H}*2L?PzefdtR$wwX!}STc=Ed&(bU`?%G?`rG0vLcl_mgQ{4VJ
zKX`*+vP<JG=?Bts_3ppkxg9yVX?pCgUGI)<U3_)z-)FXshri6;yZ>B{Ux)Z>wcSU}
z*R7eG8aC+{N2!C!gh$ud^4o3ua+mI&AH3J=um7|9*Bz!|zuy*XC?0LDeP8^rf!X>^
zJ*elP8n)l$QK{IRs&4IVdzWqByIwj+I_~)1x2bXWpKE`e`JW+A=}K&WcdfemTZ`?t
zSd}ha%s%T}{?h&Pwpz9Q7ydJFZjAbttRMTU^kzNp>x&w7%!RB8cS|hK-;>-r&1%E5
zVE>mgr}v$IWnI5|`<MGWR!PUFPWr5u!aJ#alH2{7>bYNPJG+}XYL`snP)|81KkLT7
z^Dl+NPD$_mvdj5vQ{^S==^8ve+ZifYr~PO6e>Gc%cQ=RP;-GV;>Q2vvcjtc#kkZ_C
zJa^*O3lA=DVb3=Hvuv^Lp##?iRQxWve7t@8KZ7sJ<ZtbJcTH#Oly?z)wrawmncJ9-
zn||<6lC_fjUhw{nVv2>e)Z1Hs#ot=(zRiC7_j#Xd*IqNro>2EYddJ@R&-|*XOMa<N
z<Iq>wQOuip_si_RVT&1y<Pr^~`b|W;c_)0{xZLS&jN_l@nZIXGzPoj&>E6b*#W7OX
zBD^~p>bh_Kp1<qwe}=`I|Ls`D+xu<j6B7o{eAOd)(q{kCpKmVz+@-es#`D`d&z<ZK
z*n7OV?`_oMY_shz{=M1}c_A#$f6L($%GHfZrt-IL|2e<;OUAcbukLO-*><{lr;^>&
zy8BQ5GdvLv`Nq6+60^tLV=m0Qwi(o#nQgP&xBUGI?uKH{@@0Pyamc%vnf=~3fA5*E
zLZNJu=dz=tIleOsJW%dmm|UKd_q?uaYw?n&;bpe27v8&n$-8o(Qf~R87_;9OGP1c(
zw{6Q)+AgeQQ@eZrXI8CW6-R5Y8Gp&E%$vF;gYnzd%|D~(y_S0#Z_>E&W><Q=y4Pdx
z5*|a{3k;s`cW&HzGH3r4Ye!vn_jfXae78d9q+fd1@%!HWOWd)s>#Zyou>H#U=PhtJ
znm3_9@z$2*kDngwm}qaXD^{X;mvMCS!TA^8Hd()$pS{2Q#nQ;zx%qrT7Zvs{Ihi-R
z<Uhm0z1DN9;~qSID_^L{JB{D}zI40o#czgv^Sc#}@|~XcPTIMrcnPojL8nXKnq1y?
zG$i{k{W6=&?N;CBEua4_<+*d6t$C$N^DV~@9!bZy)@goQc#~T*&G)d}^<PQXIV%-%
z8<%TUN`x3)6StiyVAji#!mLwyC;r{H{FP~`cYOCvW~}YAmbhT4^*40WD(UDuGErrF
zCUJZA*WG{hS8Va58iR!k|Mpaxyqdf9P2uhL3_jKYj8EPu|Lc<7wqwV;Lj6h=A^X+8
z_s?4N{Km0&Wqlb!j*n{B^q$b)`FRT;<MT(MVS4@ludUZl=GN}al->FCljl^cTbG>o
zo1Ks@x#fBN*Qt<tgZ~WOA+x{KC4X9VJWV7=pSSom6LaInd)KBN>kZvh6!n@p!tvr_
z{|vof{{I<Hte#eHnf|r@{o9bde^cJS=Ku3UWX9@_{B7M2KWFc9`gT*t$ibRrmxJ-f
z^ZyyxzST{>yV15<GUCeC^C~`W%WwTX+h1&N{%^~|&cFJ~{~37rfB$N(Uvm9p`CsR$
zZNb?=-)H7Mc($QK%G3Dg#xHgApZ#ZevBkY(bID)1)B_%qf0?_~>Mv{U{`25JLr?l&
z=Zp0R`d^3s^zzEN#+q{S_MHO1<-7M!-2HN^^`_N)g(W)-XLqvfko{H5yL;Y~Z?)<*
zxqmwAZ&m$gSabjPrxoSq(Q|L`-dkz-X_NN4Q+k;<tEFUZ?{L@Nzgn7qqv86#WmWkF
zI*H$AUeRX$xvfU}Kf_{Cnf~YeM<b8c-PkYnDfRfKYo!UfI{b4t$l5IETXArQ<-ra9
zcHVRCFTee_ODz4ez5a<STbJeLFTTXL+;#nzq&)V5rX*!I%Z-1x*xXyV@qq2a{<_}E
zi<@tWTVJw&%2VI@r!T5D_~qmO49Au(vJHu~5o?Y-QyizKdpGXeL>+a<nEvN-_b+L)
zOtU?^f!+3bv6N$`;FoB<+uQ$KO<qxJz5lWNe}<5+!RkL3f6RZHwch%XdA<MLY_B)B
zzDIr5+1j`4!n5hGm)*5%yMN|-{husJ#kJEXePegvYk0fFIahwh-8K2oYt0v@)*JeN
zt^cH9@t?u%{A){-9Bak@HzBjHir~q{V9*>XXqK9v!I2?_A)g_MA(cUaA%G!+A)ldy
zA)lcb%qjpc0l=?9fx(#pY#hXV$XXM22GIO2BO}BAhYFWnD>Bm<7(6|-7(mN4xEc5v
znHWF|TP86uFbFU*G4L}mFw6$o#K;U@9|3Zg6eBB`y^4W>K^n^5#K6Fy0aXK924cv_
z#30SUz_5#ffx(E8i2*dRe3gNLL6`|*4oJ<6CME_}1_p+g3=9mJ?JQt3L6gxM%nS@}
zp<bSDo<6<|Zkc(h0htvF!Nvv#1`5!HJPdFeJg=;PSTW=rtPq}=my%yz%wS|-U}|Mx
zYGr7yU}$J%Xkuk#%m7+6#K6Fy#4xJ~;)gya28Ov%_B1922Bi*!`_!Rq@S2DTkYF<~
zGBRME0P)W}CI$u;D0>qV0|Vnkh#E$aA16VU{u~3XSegW}>l_mU12a_o8b~jceUFKO
zffcIe8508o`y_}PKw-lHWrI=%CzK6Zlfng6{|*$83n2Cy7#bKbE`aC-#S_y4NSNMW
zWME*1vQrot7+9ff4rT@hzC{qXePd!^;D@sRF)=U*EP|NB%)r1P4rPO^Vpz-$jwg^f
zBa{uYi*X4=J;<F*NNi>#HX9O~3yIBx#0KdHEu?Z`VqlP9WMBY=JtQJctPG453@oe+
z&8!TKp^=L!=Enq4uLIRL4N8N|1epU97vO`atL9_?`vpdW7BVq1FoCm2kfj9!69Xb#
zL1r*8urV-0(>N$A!SX>FGa?_<F{9=KMkYucv@tU<FhSX_vJlfeKpC9@k(NhSQZS6J
zq<HpibS1^;N{Z2y6r(FCMpsgduA~@UNin*TVss_N=t_#wl@y~ZDMnXPjIN{@T}d&z
zl45iv#pp_k(UlaVD=9`-QjD&o7+py*x{_jaCB^7UiqVx6u$2`5AMr$ZIy=V)1o^vp
z`nWPMGBPmmc=%-I<ueH|Fy!Wy6a~3EDMUm@DX^bo0IhXkV=!c3NK7s+@C|kY9XP<?
z>FTUd9OUi<nWTl0->!k@iWx5JdITtd&Om@k5JL+l7ZjB+Ffax%FfbUWq!uSLFfg_-
zFfinlmlS}+L8~4_lCnW;CeY#xk)nvmC<X>52?hoRk#q=K2gFW-upL0`l-#@&1_maO
zn+sBMQ$Xw^3=9k%Wu>4~9GHC=7#P~hGE>VL7?`gxFfgd(l;&nKFffCT3lYgpO)O?$
zU;%CKS1CzN&R}3*F#sJ6Q4|v7%)r26!@$77la9nrLSlo?lL2|!IlrJ1e87N$MzW@Y
zp{1pTf=6n3PHIVsZa`vkc4ASAf^&XuL1JDd0|Ug*AYBYXps-c&bagf}v@|o*HPQne
zngNypEi=I<2~$LXj);tc#?R+KaGWuUok55pG$X|F7cnqcd}Ux@o`4Wbn#sVhy_JDM
z;ub<oWdQ>Ne+vV{w)05-5d)={jFOT9D}DX)@^Zb*)MPzSfWj>zr~s}KHv`E&J&=9y
zuv2h>oJXMmO4Xplc}j~EiVG5xQx$a46EWEuk(`C(4xOOXwA7;1yyR4cu*}r*%)E33
z=lr~s%#zIfJcZ0WTro_hHY7KIn2>Z0T2Hj(AA?A|9)rMn5eDYBXBb$7xfz&a=7CmL
zz~jw_GmHV0^TOqyL()CS1aQg#GY|w*Kj?gy;!N-vCk)O(AqvT*MP(3qP_|@XW#D4q
zXAohKWRPP}X3${JWiVneXRu~)U~pyd0v|mS$q>ho%#gv5%TUBn&QQb9$k4{n%`kys
zD#I*>`3y@KRx+$(*u=1%VGqMWhGPt;87?qfWw_1ofZ-{_YlaUD-xz)~GBUC=@-hlD
zN;1kbsxoRb8ZlZh+A+E^`Y;ADMlvQarZeU-mNM2bHZyiJPGX$JxR7xr<3`4vj0YJ{
zGG1i7$@q}*CF3W?UrfwQJWL`?vP^1B`b-u~j!fQ6AxyDM=}d)8RZJ~ReM~c$7BQ`1
z+QxK{=`_<-rUy)~nZ7bJFmp4DF)K3bGFvb^Gy5|~F{d#XG1oD7F;8P&#JrAq7xQuE
z%ghg$-!T7VVPg?yQDD(!v1ajL31vxUDP*Z*>0z10vVvtB%Mq5#EDu>eu>51?WtCyo
zX0>AVWQ|}=XDw%KW1YsjjCCvPQP!)hPg%dQv9XD<sk520d9X#YWwBMW^{~xlThDfY
z?IPP_wy*5$?2_!-?6&Ow?1}6p>}~8b+1Ie|XTQY$l>G+>H-|ij5r-Q`6h|IM6UQ`;
zRUG>`E^$2P_{}N6smf`^>Cc(MS;^VYxrB2U=XuVjoWHpQxzxFAxk9+IxEi@;aINDy
z%5{h9GdDN461OFHAa^Er1NRK>4csTVA9DZX5#rI}apsBPDdp+oS<Z8S=O)i*US3``
zUI*SN-V)wE-j%$EdGGQ5;1lN4<MZT8;j8DH&9{y365l(1PJUH>NB%heD*kEwoB7Z4
zzY*XRP!n(#NEE0Om@TkV;JUz9L195dL4Uzq!5+a?f+q!E2(b&P3AqZT2(<_;7CJ2S
zSeRK@S=d=PS-4qviSSY3ry^`3>LQ*ZSt8vcYemk9d=M29H4zOHtq`3px>xjp7_*p~
zn5S5dSijgNv1?+##AU@D#Z$#Q#n*{n68|nCE#V-MD$ymeUgE06FG&SSSIKP2iIUqT
z?@F;qX-Nf2l}pW&Ix6)>T147fI$63)dXw}m873JmnIM^JnZ+_^WWLJE$-2uH$j+2K
zEc-@IOwK_rOKytXKDn3j!t!?VnetQQ_shRh5LIwg$WfS~a75vQqKu-4VyWUn#dC_k
zmDH6&m70{+E8SJ*QnpY|Q=X!HNcn?`yo#?%jmm13TdJI@7OEMl(^ZeFepgdh3s-Ab
z+phLPT|(Vcy-Izx`W+2k4SS73jYS$)G}$yQG_y75YF^M{(lXV`(3-7vUYkkVOgmG1
zuJ%P8Rvk;7Je@^4*L8Vx9d*leSL;616Vvn2Ytq}U_fB6$KT3b1{&D?(2Bro%21^WX
z8wweE88#X2GW=wuWt40*+vtihud$nPz3~p?k0#nCsV4JGZkP(2`kJ<z9y0xFW^Ptw
zw$ALOxr%v$`5g1>79th_7CjaxELkm`EbA@zSpKv!vnsLLWcA)!*E-vJmGui7b(>V1
zB{q+3m24Aj7ur6sQ?N_0TVVIVUeP|$ev$nn2W5v8hh+}W9W@=Z9M?F$b24x$a@y+j
z-Py{y*7<-7vx~b+m&-X<A=gmXS+4iol-$zYR=a(0H+8ReKj6XY;q5We<GQD;XR_x?
z&ktVaUbS9Fym`EXy=Qqp^3n1s^4aCf<m=@-+4qj0s$ZVpc7FzcPyfmOcLUS|3Ilcr
zvIPbN&JKJQWEfNvbUau%I6in)@V5}Bkp7U{p&Fs3p@+iw!(zf#hJ6or37-`HAVNQ)
zHsW-oRAgr4jwtr1u&8BGU!z^4r$#@GF^y@9xfZJ)TM>IQPAV=ZZf`t)d}93O1lEM`
zgjETD68#ewC4NovOq!GQA=x>3dh+WOhm<KPFH>z(C#615vrU_n_9ERjeRBHC4Eu~}
z8E-P3GiPOf%<{;ZpY=W4KYMxh|D5og^|>6m3AsD-gz~cUj^xYbSL9zT&@N~zcu;6n
zIJNLYk$2IuV#eaw;vFR-B?TpCN;OJbOCOfmmCY{uQ65&lxk9iaui{LlR%K`9^D4Kh
zrPVCe$<>Ey6l)r59@aY6F05m$OR76muUy|!|FprqVMQZXV|L@&CWEF)O<$VBn|HOy
zwluUnZgp>6-Nx5e)ONkyx_y2JOGjqMxlZHGnVtW-Qo2rc8+1?W{?n7(bE?;{cSi63
zzO=q`{bv31Ca_J&n{aKS{lpcM1SeHZdOX>8^0p}oQ@W;nnHoR!<TR6M^QUu7FP;8i
zhR=*0GgW3znE7W`#;mKeoo26}BQvLS&iA<~b1%(vn73}e?ELQezZPUJxUtZE;kHF;
zi>57RTU@sI*^;m&$Cp|xUAatZS@*KP%k!2$SP``1$V&5-D_6;^>R-*Yx^(r+H8E>0
ztaVwtW1a50h3m!EcW+?WP`csO#)OU6HhFJ4wAo_wx-F_(=4=(-+O>^wTjjP7+cUO5
z*b%wo;!dxfhjv--+PYhB_wqf8duHzy-P^y9YhTO0|NE=<e?3re;LXAGgO3lzAG&=w
z;_#Itfk)0A^*(yynA@?##~qLFKVf@f&q=G3J5O1h+J4&X^tLmmXSSX-J-hXs*|}}!
z&Cl<+V0mHJMVpKJF4<o?c-iIhu`8ZePG9xAdhuH5wHw!Cu0Obua^uC#oSPqSmEHP%
zyWtM&ovynAcc<Nxxwq)P=Kb{#%pUB0=<@K)qu@ukA16P4{iOKGpQkO)c%MysF8_Sx
z3!@i%Ub?-!@GAP%)7J&Bf4^yaEBJQKJB@c+-aEWM^C9BH(~m_T|9|TFEcto)7n3iC
zz6N}~_bvC^@9$kdB!8^<Y4P*;ukc^bf0zGZ|1;yS=HFfay#L+#pZov+|A&et&~{#4
zv0h?Iep0Gla(=FUMPh-zp`L+0gN=PfL1J=tYKcNpYI<g#t@``j`_&aPQ*714O??e~
z3!GCkGCit_QiH4fLXxYplPy!!?d`a1Dy%AU3vyFS5)~?PbMlI<Ds0ukhFj$oTY-f2
z)fK=>O0sR$BYXoCO7in_vNKCSha>75S?DOF6(#1Tmgg5`D;VmTsoQbc6s4qD1-ZCE
zjVnq?vsH(>8`M56H`dE9O4m2Ew6xSWFw!?N(k)6!(=D#dD@m--%_~+`hw96WfNMkX
zZe|47x9WCWTnb>TTvCgZi!uwq?LLsziAnjTCAR9NrI{&KW=R%iMwXT-y2)l~X}X4n
zDVDlP#>r;7Nof}5DV8SYhN-DZ>Rbv?D^n{n5pG4X6eI()RzblA#I<s*D9A4^ElLf^
z%uTg3G|)G-FtE|bl;MJ@ck?UFO-e1YGqNzyH#C5$g^9wI!Hf+k%1<v!%(XK@GZZF{
zq#-jcEVZaOGe6G`v^*TH9w7`@>ylUkb}!`cy~G?l(As8Q0~1{X;}F=|W?cgdE4cG8
zG+-Fzl9`@al3A6SLbO>39dN^fGfGQJQj3BMQd3jH5_3vZ?My9^LkUwJuGO(1wWy@D
zC>5g3*Z^4@svKO46S&)$ms(s5(PUy^h7=>{vT$v%m~qNXgla^#3sV-Z(KoRI)g`E|
z#E?d4Of5+*%FIjm%}+_SGlfR1FR~z9sSoI=z2N-PqU2ONSlU7sgDZ5)PfpD7NzF?y
z$*?mvLkTWqDTMl*#Nv#C%nEoCLrS}7QgA)s^8|xI-9S4-sH>5L;K~9rD^hbJTrzW0
z^T4Us#2l8`(PdDzL?W~po0>r_0Bb>#folQ#Dj+8@FEt`4wKzYg6jV~!8JQTF>%$y}
zT?rNwA_<#-tOSb*2m?a%GE3}?p{_@`08Ihhh+t44<)!<kmSp6oz)J;GQMh8a%$!tc
zVu0I#5P~ZUPEO5B1sw<pk0yi&T#<8WaY=q|P-<REYEfzmJaM53!<EDG0XWUVQX+~V
zTxoblW=X13VoqWnQldo>gewKR%Q>+Cl+(cuhM9~ZfzXnXSX2PY<Zvek!$lB^O2H=x
zyJQxZfLsxjn3s;oKf%}};aWX|{lVo`P-01DUV5<|mx2PMLJCeT1aly~oJ>1#jsaCo
zIhkMy8+{N5stlR{u$Ts^X52uTOaY{!AhD<z99y>PphN=7>go{lz_pHBCMbH`5_5`E
zp=Lrv!0rO82udv}Ey{xj2b7DD2S*t!3xYWaIk(chWN^kou>dBBP#2V%;#!#s){LwY
zNf;D>ka`mAH9IZ^NO&OC<20z-3y_*2C{=qwN}9f*p0PeCDCy!5*y+;Pn>Ph8X{6=`
zT+GTNv$!O`sM5|xA5)Az;aHNHhNF!Hl0kAbh->AWn4N0p2Hi_%qmK{+B?1U9KP5HC
z4t19uOd(hbuF}6KGc~Uy5u6AOp-Pa&;0lq8d2=IBArH0*Ss1PsxwJ-7i!2OR3(9nm
z+5=p48$sQGCJa{|>73+T6p&vG9+Aw<O9z$NV25Bx&?n|95(}^=Iamh*lmirODiRB<
zoQqNuOHx5?KWL-QRM*fv1h$J2)<8s43Gz5d$TvSFGp&*s%}6TYno~;>QxZ!OiPDa&
z7G?n0>G?$=`T03^xHo6ofO<_3Z$j15Ck~2Bf>Lp0A6P6DgQ_=uP@)1A@Ik3*%7_k1
zMPh-kFaAi(%>|ca;CS@)wQ@-<$}CGwaVyHtB~XWeoo1Dp2dM}>UF^UeC<9Al!_<@%
zGkhH=8+{DD;Q9a}nv$Pfnwy$e;^|@s=~kxXCugQuA<8eP6{f^kfn*pc!a=c!>Pb+2
z2ui{rQ9>3&JBP{mEe40Vl?%c!`lOVK#DXADlfD>#bc1^`#o#EXN9C2040ldJX;BWi
zC!dn6pPG{jisE8@Lp?*VTNG?klC9G6i*gf7>>&3qWEG^Q+vp=iLD83zYz1*YDApjY
zCC8i+FbBfR$prPxt#T6c(rwi%bfIUTtJ{ILQA50o(xQTD23rF+7@`ifaRCtva&fcc
zve5?*Zh!_lAQpqnrX>bt4O*HHHha_(a4--{j0P96Aq7@98eHIDAeI;nE@DFptZp>8
zz`;N)F&bRNh7?%cXmEjpfmmWRxQGoYu)5LU0tW-J#At948&Y6(qrn9Z24acP;377p
z!0JYWiwm)WA~i3?R=r%^-tPZx24@ClCJ+FT%*@QpENm<wz`@4G%ErmT#l^|N$;rjT
z&%?#d$IZ#fE6B^oFCZW&z{MjZEF>VzFCZWQGK3MNlbMBsg@r?ao0D6BWB|S}j4_4r
z1~a1|?82}k46+OiOw7nPg)y+QF|)8UaWHT){y)MHB*4JP%*4vX#Lmjf%EH0I%F4vZ
z%)%<fz$PfHB%)|&#4f7r=oA>Fl9*Ij#9`bxaiN%V)1*bg8#f&kS2amKbn)Sb;^xJd
z)LbS%`uIu0)OGX!BMj1@Q?fueW<pk`g4V1uGqA8Sva$a^!eA=Mz|6qJ$O5{qjG2WA
zbWR%+vyd<gtCF&@h>??`igS>ls7Y`VTVS%GOJWgYaY*AN)C;?gpVBs3a3;$pees*$
zX_5OLB)h6OvOIm@UH<Cuoo0q}>X}MzjG70<?v>26j?Kvz-6I_;RA>{J_u+`cCn<Hu
zw3WV_Y`#_Xemiovc%qqi*0w11gR52)DL&4>+;-4UYwF9-s$SJsZZ4YYSh<r&U{-!!
z^K?y(+~u-c1NR0TS$iPsnp;4%!O?r0X1%Mi(3^Vkg1U4}vr|Z}iJ#Kak9<<6J{N>s
z*emwS?aaNdRabqM7&Few4e-(q(%ISeOu6phgI!k~^%!2RD%@fE>9O{z5C4)XIgT$g
zn)B@L(x*51-|Q(&HVay^?WL8UopW~i%ZCsCGsv18@6KPatZ}6%+pnfszZLYi-Q+I&
z<FLx}!T!R>rr8@$JkwX=%UF6%y4|}<(xE+|Xrca$8yENKy=7qQ>Rz1n=yosLzVnGH
zhkUc<E1NxY%Vbx78?3qZ)Sfl6hsEUzOk%6E`d;4_w<xb{JCi%FEpv;-PO;fsH3^ls
z<CH2N?AWo#V^`TbbMtpcrSB*D3mnV)Qat_svJ)$geydx*bl+{?envIc$+6<THl>@t
zJQRxmEZy1l^v<lvptW&dLN4xATJ$xgb*GXz^T!z53t?X_?~sz1{wVvyr^~DME1wVz
zxuf*t#>bamwN&$1Ed5WcwD_#>Ax4jDiv~-_u5V&n_teebFA{z7vB;`t`Lh;1^J$zG
z-MA-pS{nCXo@YMiT$CMEIf`_57kC}GRlYIyV8NNm6aG2AOWCS)=IZf9=M&#83R$P{
zxnH6Cd0|)P-MouOKe+6^-`h5A%g1i^y`E<-RxO-p#1c36Kf@L?BbFs*FQwORQ`p-v
z{qavNjz_CD&$&|bW&NAS=K}8bW^d|V<=iLUyt_#&b;q{(cN&);idg$`{`bE3nn#<%
z%5;t`N%N?+kxaSO(s|uo-R&XYt*N(uPbtikd=Q>iC37)bWWzcElbkEO8*8`CNx8UK
z$U#>p^x9iq=6-oS>7(oGCv>gQxxQ-RXX$f6&-Q#)wV2(<$N5$`vF`4!MK7P3ykDp5
zZ0FoRX@T0jGf^etT78?o2M7HQ;BNeTk*{I(-RioE3#+po?r6$!8z$^jT&eF`J5fnl
zc+1WjF3y{|7c46`24?B_uXe1eDZ9mSdv=nNd;N?FuFU^zqC;ic7uag1&U$KnF)p@S
z;pKYnS{Cc0w_O*SrP!t3wO)|JwrBsNgPtuZX-}iNr>scj(|p^zyH7fydEcutzmp3(
zmgl!WJbbdMTXn_~|J#ympPRyVgxz}C*M4UzXLaJrCsM0CwU?jWxk@f-rGNV3l-cUB
zIY+*mihJFAdGz3>r@O_)qE2Tm?OE(vrI)grWz~w@;CQWEJ?&|{i(0(ywCl3QE8bnf
z|M9S4;hJ@|8ndU)Sz_(TmR?fixvS-3rss;LEUg#YS{AW<oRqoX2k*AuZ=<`}uCJP`
zeoZx1Yp;3HM7agGFQ**taL{Ahp7Ux|)QSnF51T?AHcDunx%gSOs_@H&7bYEREsYY4
zi+A&y*(fhsCB9wz>z0apetPryR72Sk;?CXc&9qe3eR|6@NIrDN&5pZOn~GG-oDM7~
z$o5~rYjXL1zV?P?;nI;RN97}(&!$>yI;R;Qx4bC!^aP`_+LU95T;pD@s{I$EvT~yP
zr6X1qXSzcgecLZyx^a7=>28kg?z@__Ufs03e<o^G@+3z0GtF_Qzub2($U4Y><%5dd
z{eaU4_vK|c-QQ}+_$0=JUxqgz)~9c+=fal0tvAvQ+KZe_ujpMA`X1^wRXFte$E3tH
zNmH^ezM9l|$~)lT_Gf&z&+|{+u-o<fL!ZMRSgtJI^VUmNZQAEEQ_6mo%jG-W-hEJb
z*Ix<w^a-N&-uu)3GdPvZcfZ&5a?!=Fe%F+EozlK|Jac*7xU6^9hCbEiho_IdsjOQV
z{-ojq)2g@?!4G%OQa-=*SnAA)(Rnrn3QUTb2L-?FV*2J<-23uBL$~BVr!KY?CN`@*
zzcdEC(Dd2EDv=`msUdHNM+ob_JEcGQ<x{NGq$f`c+4)jvzU-~+M){PobaUA$!irlL
z@IJo0Yf)D0o*k@-Ig8#elK2wXK1<u=S>45`mA+NZYj(A$usjr+kg@lojWGY(I!l#9
z{fmETdaMuEx|{A_GHXhi!;<=2?yCg@SY3lezSVS}FSy(MQ{P1`>O9xK3s)jaat>;R
zL<RP$xor1hFsnN;OWac+;I!}g>93Bh_gG)iDwz36OW12xE<@ZyZVeTAPDS;vm!%Hh
z$y#aic+-;TO5s$Ami*lhMQp-Fo*3S#5|wkg+4J5}=(f_Q=_;!K8K!aTCah-klC?<>
zJ|$zHXLQ8l&6bwXlFh#1@#kY=CcTkcQ<eR#ukDP6$)pFLn<d+OH*&i@WmLUr`(l2W
z(d?c%b@|u-xP+GOe39>LdeD-0_GZRua!W7I?i2s_e&da(>_=YH>;FcadnFM(#d~|g
zSLZW<nO7`6EZdXsDO5TAwqDtUT^Z$XSN;C|pW%0P()5JhXXP)eFR_Scz6)&oc7Li=
zwC<AHg}+0aeB%zUUVU_VsiJ)PvyYdfN-qiBIaajSNiO+I**A|fIqzhP7Tou`W0iVX
z!ea5`Z?R%e#RYb%PdmTdC+oBLzX#c&=f%z~?DIH#b5{Go<C#V2m!p(^bojJ#2EUj7
z_~OSsKfOa|B$kU==5z?&5|X{SXWx2$`P1Sl5i)z__KL62-oEaZSiN88qCUaW8{w(E
zHYZvtdn5Qw?jEuauio<Tk71yT)~x2E3bQ5G&9O6b`;nBKK7B{wPjTJCRi|QfgYTqG
z{1#OCcDb_I2M_JPGvc_@?(3)R{i<F4XQQY`TpGW(|CGOcGMN)p3=2)Ga|PxtvYj&d
zmipmkDl-pjTuw>j++A{Os_e8!+pccEp>*w1>xzV`x{oFAKR$dL#3GZjXxr(pl8pw@
zay@Yp{gQ>c0edVfZ4P&aRtfaP6)p8|R$7waaHIRKXWpF~E2U)H9%od=)V>Saez!^S
z6~9xdS6#{J+}+o|1}&QIt{`bGyKx(Peu%Hd%jF;cMtt}i-}IkhS>USI@~c92inMvv
zG;mHVYl#!w6Dt_3a>ZIh>(#vIjT7zk4o^DZJ#p5ibKy&`%dS3Id1}h>6Upv*mhxM}
z9E0ynYk2)J@jt`qZ!sN}|8$+!hMaS9Oo{3_sIhUo)CSjkI@{BBW!|;@ackFdIeAB!
zFVh^ql+WBWIqA=@^#S_a^ONrxGHQ3sE)m%hTYYyySLGhV_P^;j|1h3Nn)+Jd^jH5V
z3C<Ju`_3diiaj$kM=&_vYE{M#j>4t`XNpCOR{EX~+4I`5_nPUHM7KQOW#;F*(pN9)
z+Qq5Y7q{}OS?$N=n%VveTE`wbt~Z{gx#iDUze&Op6LUJ&6@JUr5H73uxhG`5_|L+L
zHP0S*?Of$*78W_v?~H)tT-j%`>X8#p3cGkMd3tKr-y59f6?5i%wv|`gJiGXa-70TA
zYk#>7xBh)SCeZ227=7_jf$MSpNAaH?hrRj1^<L%6(y90AEu5!L_FL+|$fHQrWGRQt
zxB0bCb}-76Z|`ily<^qBxYsR(JI~uECWRKwcoLYo>#4p;*uBDqs}J=XwySP(mXH=K
zye{RZ@Nnwy8a?fnpAX(&k!x8S6P~B*t$+FW#j`rf+*|#(s!V%u=O@>#s4Yuo_ugr^
zx4LxSiF(m9f1PJ`o0Mx5tTdkc+4I$nzLOJF&++xWlW)qrcQk&Q^qI%D!pHbDoFk_C
zZ?jpj;J}2!+gqo{Fez%U`~2PI?YeyuSKEq}nxng({Azv6dairh4aI`O-M+%{yM?~q
zaASOSJ~H36@A_tT`ImQpcy4su^5$pFho!N}1zMYf@^quP`R1+_<e1OjUpd$I-;+DX
z)`+reZcID#l>gG23Zq%<+h$LC^V0g@+vtl6-5xJo))k&t{@21RFlX0MsbjHK$$F-K
z*4~|rr=F=~=gSJ;Z4m4`v{qmL=?;<YD|A_tN@LI76i(_;5MF#@YE0a>Emp~H0w$u4
zUu1TfPHOkfT5`tx@v*ce>3fm_6&Si!<Ygy&W!Ed~Yp2Yp+IH~YhGU`WiW{R=m<IOR
ztkSu%>p_<SU%x|H$uH#vZi^h>T4pZT-H=*PR3Iz*@`C2t-5k1oX9FM2_?!^EPEc^m
z-FI3sR`;KZ{Ighd>e1=YSC36@)m>HczAiFRw`{$lf?zm*eEY<=e`2~nJW`10ELI6w
zsbaJ&-|VG{(KfFK;=KMF)#knX(I=4=R<kDTsn7bfKZm|F7ye$-J}YFh!d7Yi(&lAL
zKixi<Z}rq>-bt^brL$9Qc;>|CeBg+3Npaa-rnaZ#-;6k?^P5{viq6<FYifY9#pT-<
zon*>f7+N^OKC|pt_bfxr_0NWcM>2(HZ}CL`GVH$k)lIoz;~m$n-WN-BbKW_h76{hh
zUtF<x+uLX7ZtPurI5UPlZ<f*zHO86jk9ey#Jn2Zb?Krc&be+xRhV6nY%3O}t9XQ_a
zwy!-ke8IKNUmvnHi}^SfEWVMy@69B(GmO<gS$%eNO<TgaTzli}Gf$Sw{<D}L`Ac;|
zmdL~tUw%5}TfX{n*K+lNmbxRHTlQV~v0KdMZ~pUZy!RtkPs}v+*x8k`koUoAizhGj
zEiztjcduNU5a?PO@-E3~@kcqvb=r!X7P2LrnHZj-Ym_^YSznQ3Pn`cramhF5d|GQR
zhJ7{ZNRCc>Av9-4%Ewy?9$Sn}ugCN6W?1!L(;MR%@6&u%zF7LIEM}(Be5Kj?%}HgP
z5+^PyvEEZ%ez<J)yWRYI%S30+ONy)tNSpp*S(k~`6sy?W>7LuK?)cA;RUqUsDL1A=
zOZ!>z)EzrkH5#`6%4wheIV^Pf)t4rVyRIzzc5lK{-<ACPvYc0Ub!Tp0H+!1+!;q_%
z$<A%_<=?X$D!LOiH7LdB+`;<(LuS3V%Kus1I<dht^*_V*E6d|#-0rTwG$YL2fXnCf
zd;3YRCN(9vOX!+?UbFM>!y>u+&#!-ZSep0IYgs*C@Wh0JGhQ&psy+ObCv;tW$*l{8
zFVB1VhphVPvNt#7WbX!Bm)$H;`Gt8o(ObQqeLMfx`B!1X_WD_B{|<+I(Vn@~>tx|G
z(>?C5ejNJn^UwAzwOhYSB+AAgFUY&w8uK+`h3n!o#xr+n={!%7{-nI1_gVEVskcA9
zewQ9|+7z&A^@hb$vuAc+GK;Ug<;pgjC9>ynzOCX!p-FmWp^v(jbgpe(#i()n*;WJJ
z1$8T1j4GaK$vGFUX%bm>caD#(>i#gxZ;k?6&Wg;{``z($kL|+Rd(kmx46ezkx7Vr!
zz7nie^y$|)_p-0rea^oR{rleSuPQ&j^2Lqd?lqf_<u+KXJ=%ABuH#+V?I%yXd?r27
z)&6;J4}06DoEINYIa-xX?w`7y{pkCD70v<)RmBF1r)C^&d9A#&vVQBH3Hm=P9?e=A
z$!j$0^HI_1$tNxO_Db$#n*MX|E8D4-84G(&lb)&CM~ejIe&*Dft#Y?bJ=5%v!m1a$
zR{da8HZ<f9j+uMiTy$|&NbE*IwzIj#!Pj@KTA%mp<95L;r`eZRiFi$^V70Lg4k?Is
zd&1PQE%Sj!>Z0iK!%vT7P1wk1-Dt)ARbjFm!#<bOzVqdryx1Ns+tzgdoe$UM35ql1
z7Jd~{KCx`i6HcbUX!ThJW=Gv!!X}=zj^KU$dArz^X7fdJUdDZ?ovpd`K?5I8m`vI#
z`CYFbevWkxaddU%4i0_iq}+99W{zWMZ2J;h<@8kxk1xLZY4Vw?3&W(-mYgo1vFmAv
z-`ZI*aZj#q&n`JWJ@x&kIwn(<GiQojv+4?77VoiIYsWn)W=6u-$BGBlO1!NXEr?xI
zc$s6@zWV8ZJXW<-s?M3sZ0Ek>nv=w(Q#?B-Jhqgtd+d89ciG>+ork@>A7wM23_pCr
zErh4a{L99f%bzKolC9i+MfA+qn6(0op#nxh{~7j(M6c5cPP@jrD$HoD*UC3@*V`DL
z{PsR`TN&#)^Xe-*Gwz=L&!FRH>U2hTa;^BK44us};*BT$U2?aG>l(>AR4-bk^`9Yz
zU%g)OKf?{xKz_aBYf|jJBv0u@vG3m~Qq&S!!81$HZKmd;!s9dYU9WBs$@{SD;qDKQ
zo?hRe_Hh<lj?|Ggt3xInU_K+_R3Mi>LpV(6Tgdh2H}?KGo?5vxSls3?zp3Anw6@Qz
zQpsmd-p!f$O}Ws%Yys!eRb5#etanYNH}2T#W7Bf<_<J{lM+chaFR8{nt2%i+w(m-&
z$oGyr<z<f2=d2eoDrBVXeS2|U?9XJWPj#+;Qew_!SU%Q#du`dZ#5c3g9}{z4!8GyB
zJkwWZcO7LzLpb)uY<QQ>`{Con%|<mRXDMEND(I1BBG~U)CBw;cB>Me*aghtxXK6M4
z`|wkJznGZ+Oe16QrDnb-<Yv5mm78$y)9o+I=2`vzxOCc)Uk6V7nk^P!9x(BI=0QON
zeN7>T<vgbO8#<p@emx?y=dsaJo6r5nOSf-J{@I~-(R+XU{71*yrJ}j#q<V?oE8Nv1
z9ntgA?({m5_Bkd`^2HV|`sXEHYf-Nnu(iPY%C*?fZ#KQ26w%l8dGQHP_Wk`SPO+v(
z9(7IlJMB)l+u{@1)l)iiH=XFvn9rvb$Gx!Z@T*;icFlF4s8mz-tkjmL?0)XHnPO91
zJVPe%T;HjXc{l9d{dv-?H?+LmXD>dtY>LH}U8@b_*Sl3oZ%T7FYfn6+E^_z!?5OgO
zGI3$oq5^y>LwR;;x$4bM_YPRT;qlqHb`8gdhgYl`R_IQf&#u3pE9{zu;)`Rle4BlX
zEyA*kV$N)R{>ywjPhMh!LN4QmP~lw{wmtgJ-=-GLc;HEvh}86NO}Tv4=S5CU-7rUe
zO`qCrrP&MeqStn<%L@B3;bES-fVHy!i|E7;zeP<t_bog<OWlE`yyV?J)#F?T6tt&&
zSLHamPQpENW18zZ6Y;3OM=o2fQoGl*z5Y$1fnw;KO;3KsuYI(6=O&lHhi0tBQGM%}
znqGvjKh1UJGq=%GGu>pps8TgQo`urA3z<Go61%r@(fX8F8-pt>ug$~cR`*B*FkfT#
z?C@v4!*$*7*%{d&|7QQLw+4sA_P_QB+8b_PId|fnfNyK16B<_u&RW-5pgb?=&}Vkx
zSyPt9Yy8z=sIsw7`^ed`#p);1e}>0Tw{xW45a3ySMeFN{603+(q1F670fB~!!fLI)
zjy5IlmkHePF}@l3TL0je+!yb4pUyHqZ^<`3ASwBsQ5xHij#tl1e)XMSGP(V7zi{-Q
z?l(q1P2Zb!*_;&k^wZSHTy92%h4@5)?U}O{y<U|i{x4$XIrrIb*w)_gbX@Dqbz!TV
zLDe>nowD2K8R}ke%;XVz8yfVVfiHGv#qFkM-pM&U;hRgOV$Ayj-)xm_yAx74NwFaO
zc&uPS_nn?wXY?<#g`b@&bJX{&>+Gti?7WyuJl-c8y+aqdophDmd(`$wtetJ?`-GEo
zoHoxqzeH=v%g}XAwPD*0S6-YmL-6sD(nXVAeP{Nb=u%oJns?ddal*qMw?I9;$OX@C
z7)#ANQ92<@V8zF?Cv%&fq#Yj5%43pVwrhq0bNNq?z$-Fw>;EJs@qd{cv$AT=rah+)
zn>OY}`t$~O33)C*yj<+!+3VBeylyoyy<TV*zuR$ZNyX&)D{5-|uBj)AI=|;A6-8Gk
ze(3o4X8(EVGlzSnb$?l}sNJ+wxpOc3HJ{B%cbwXs9xR*hHsk!-((bftzOMap>q6h`
zdGjs0E@0bjCdJa@yRJ2H{*3=8ccb^oC#F@(r%tUsTE)v&+oF5*l;tkR0)AVO>ANbD
z&S-BwCNe+qrM}lq*Ij1e8m>&nirz|nKRf#+owF<-)CH*+|7Q^YbTt0eHkqfj$?k8~
z-4zR89LavA(WhPY#3#v|6LSkVd4qH9bn8D%<6r0bw_RJeBl9Z1+lFGE6>mdd?#om*
zcr1L(;b0UKNA`oIH|+V}TRc9qqgCJUM9$^Kwo1b4{+S!Kmuy}koYdgCYgOl>)m7D(
z`6et&<g<%B6Ipz5rvK>)d@h|ATaVAa7k%H9F}mbq&V`B?*AG8xtzu>^f5pB%^3ncV
z%zDq(?4KjoKVNaL?@Ev4RWIkU2<J9h2fJK<>3mVJ`f>GwyPB_cKb%=7(<D?Svw&Ut
ze!h3bpXC1x(`>o69eR*7S@v;N;Jw_p`@+>HA3ywjnX)GH&7IHOb^cDjF=wT2US~~I
zjdoYoB2iy%y(@|*()vC7-#qVKd%Wt9_=&(x%Azhi^+deG_vtHUD?M)3a&_qaoO*Uz
zcZS=6%-<F3p6&g*)w*~8dBH=9%X@>SYiBNBGSy<P?Ci>(BtJ8&jJ@sgyY7?-Om$lF
zHIDo8i!<*-BI|GL_&DK~??;0G2O*(%bxFTJND5f8o1}40xt8L)h;>i$`F_RAJHLr^
zIy4FfcRbP-m}FgYdEs6$VY@4?(`Q8TN9FciEcVYV+MfUL=+q!K$u#b-9!n#wawG(!
z#O_L@y%hIXIH|w)bf-&90r#WJ?DrjbyDzL?!WFCL@KiHc<wWK-nP%|{Jwa97<G#X<
z(()U*-s`^o&%jgZxARz?`}z;sLGJgaXcQaA<!HV?|6AMSQ|^QU2JaHLCz+yxzPA5z
zQVOy@J2Kpv?Aa}{>BUvW*~&@_o)}M*d0W$Zz|5gEdTq#r!t<XG?KG|VGrvQUPc$>K
z{$j`~TdQQlW$qL1Ost4B+a9s&y0nVx^P_?XTtDpC=oYk>?U8lroU5K^4t-u2#Kaw&
zcgfBrW%0swvz9HAu~>h#_3r%sH)_hsr5;=MY&siMc`{8WLdW89%Bhd43>O4V-tGMm
z5I^Bbu<4idIZK?*&3yRlshqD#*41lAd5=f$iTB94ak_KWC$-Dl`}O3^?k<lrPcTh-
z8f@u$YT_%N)MypwDK3tF1*dLh3J9(4U8Jve{hZHo-AeN>4|(H0Y3*YxIv2Wlin!L~
ze|InRWHldAuaDBbv5H~ka_(?f#s3Upy7zflwQbq*y#zjA3|(UCXxVZsWydswdDrf=
zFgbs8300rnm(=^m<?^nD@ogWnW(cbrd^dWt%6V$;+KFY|8T)e2DSoza;$6PiXYWG&
z{|x-h+xN`mx&I~J*znx?9Ani@&1N_JeJb7Lt~b4vQd&FhulAShXK#JiZ%`IpEpTy_
zZ_7^wmxq?wyr0xKJc4JG2Y!C+-?p3i$fSizf!j{Fe$$^U?vb-k_{rY=Z8L7h8ToA5
zbWQacyWE7Yea)8^Ml<-jKf0q7^5KK<8?)X$8;g5&lYg9EvvcY*pEBNpr#=<E+vN&o
zDr|APcXzjbSH%Op-RyVTwi;RNRbuvOX_%tRGrioWyVN`2^1Fxs87}{ccrWSVdDd#a
zYvrn(h(w;JIu^}x7ChBE1-Vvtb#(cOrykTR5jj(IqKC2n_2XW}b+gy>@JZe++LCd@
zBB`T!`FoLdy21PVYcd?&Pl#F0ayeGEDyZt*inxBsmlrS0&(r$I5Udi%VwJJ<^q<8G
z-pZY6*wtJjwbx14D>TX9Tdrg}=k^|t`L`8v1q1e+ESgmyQy`ysCT5;-%xY6P6LYOg
zOLjb8J#WJ<MdO{DABioP`h2+zhwZ}^c5C}g?yR2{*JCuVCOT)E1B?6QzS9A&-;!>v
z5IuV6cBw!2v{SalTF2X`zn+@$f#J!SuQKkhgd)XjCpY-zX|J0%?UO5?oMwKFIU~=y
zV$E4gi$dDw_Y@p+w6wB1dMn|+aE5GRY&@%=u)qG{<1*}fo-LdnGw+P$Pp9fj`>r`#
zDa;gfxqM4*?~xd3>!0rbboBE!%;-t4+UB~}sMP(A_?)}iJ!LH4?u42I{kc5-X_<78
zHbZJqR!{1$?2VB-mU8p-r1RRgwRkXaFN$8br`>z+>paF2M~^F9TQP}sd9r79b?#KB
z@~lhIYaVp%`pNfxn%I$`Ij`cqefyNYzn@!N<D$D|`s4Vk>u2B16P=K@^#%Kdx8f!#
zGCyAm?PRW86{{2@rPN`i*K;r@&iv3_v4<+zQ}2i8q!jC~SzmrFJMz^A)A=qt^FqB$
zH))CdXmPoH^xNbR?$dw8lP!0g+#eaV$?VW7GbSBVzK4Ptjn67$RDW;&eSQAVf<E8G
z`Ny@do-&!f^0{xnSZJT<Im;s~xuQnF_WUagbT%%ic%wXXT5Zwogsa{wB!yg`&6=2S
zhqLJRO*_w?)gO+U+!3B*q?fin*mXsa@2m+MFP*SkxG`gKdujj4&F``pS61%(qaS)G
z`HWs#&dJH2Q<IuaBxhbeHrap5m*N1kqdcvpp^kT+%z6>^a_8-L+Ye7WpHP%|x>)t1
zM&YC83r~I5m>q7EOOO_P=%pmYoOSh8g_7Jyd1;r*3?+*_6RyAJ>|wVH<ysrDbgl$L
zrpTuvZCT4*ju|VnzP2pg<mOm)^J8*+&pKDp%#z)6zAY@=<TK%ql8c(^-MWuHQSl02
zlW#Up&V1{Uvq1XnmT&QbcT;rCPVP2Uaf<KhpH!!l{8@XA)V%zYACfP8W_h)E!x^7#
z_qr7yL@QhMbH(l5Cm4M6rBMQlgl5!Kwv~KK&0G?4qF3|uu0G@19H@OoG*mC|!GDHk
z?&Fb$*D^0}E;YXtzc{j?aNgGi`(JZ@W}Pgk>(Qd)5_F|@(W=*Z#+KUa4%{tc{Lhef
z?X+g->Pu}E&W1~OoRV4mNPkMB+k2<&4RSZ@W?w#bC-TYWOANC_%`ZKxm^$rcNtVx#
zVy$Us`vVG}&q}zV`$&vaR&~P`xu}lmQwpcn$^T$jz2UW4mDB2gqSZIGHH}I=C%7(q
zzVEh8ROWF{-ke;)SfRaVb3guPFcVziw7z2h%)M4=I_u+>Kdo5vrEbHd#~NR~IJYnt
z<t~}KUYa$!*d<7N`qS=7?I&tW_+|UooT$&ynXi>=Czm}t;%jS=_8a$^nr9!GOV!Bf
zvxa2uHu2Nl#@+Vi^q;!$<1+%jXWJ}06BackXlG|$|BQd(>VG!A4ejmmzG68^=_>p3
zZ;4mBv-FA-cb{A)J=0KPbHU`zO2^F`x2c#t6y4+EALQNt@>9~BDSnC_sWOVap5N!)
z&6%N4I?biK-pQRuQ1?N-xK0$W+lgYyl9*-pq*-1UNoKFuAtNexieuHvcUP^G*e5u-
zM6#Do%fIzetJ0<2JI~i-xBsd8ySZ`>9ogyG|91ZtT{q6ja>dJQ_s)NxbKCju<(&QY
z`U>wGLyMAR`uV!}s($O7uRJ)#_xMVOS2{QL90~io<x%UW?@6baCmfAib~y97R;bM3
z>`=FsNv=&Qhqt|wlDK2GsPUG0tzGoS)R37>kqeHN?)a+p<iJ$P8As)u&um$8?chrD
zT`Lx7A8%aA6=gA#!64|l`_k=y=bYK9mSQh=Q)K1Y6FXAcg>LV)n;dGAf4WRNESlA5
zc3Qm5ZhQ5kQ{GHVKGhZSV$%%;CEdHvw&%D;&%SD^&9dfu@`+-F&)No#FGY5lInF44
zx!}y*>#GIl<>)HaZl5*bw&UIEdH1vH<|LaQWvtatwQ5;&R@s8JLM3?dZsBE0bIvSI
znDW(><K0TGWw%yFq$usPFO%o<{&;M4wth;@eCvx3gFf^8I;yqNZ;F%A$<6y0)jz&<
z>+0^{c(eUtTAwWqo>X(3-LP@as+)_}&ejqzdU8_PqsdU+TPc<MR`aYay4#nRxaM-d
zj9ql_@FZ8g)L&aowBjac-duEQf!CU2>M<Lm)c+k6cV=*XyL9ea{r$(Kjh|Y~ni;?3
zmCdou&!Z=2JUGM2ayz9};RH+2*S+;dn;pGfTyl@qpL=xvM)S0n2AOGG*SyvkyCpAH
zn4)rKTj!-4iU!&j1O=vqZ!eWMQ<vtd`p+QvLBi-?#j3O&`KhNCl=AW@YP^}&pz>SV
z&*bsj>F)((Gfgf%&^;b3R#H=<`b6&R6)WF#z0Hvej%|5nWhB>fL-9_u+u9eMXL60V
ztc%^UV|TKh`1?;Hp)tPI<{_(o>hZ<>yX4<4;(C!oCO=2OH6Zky@|iP#3Vv;>o0-mR
zdv>PpGp!4cXKwb^bXIejEb%-!P097~qWK+6h1aEO#Qr_V&^WpyuHW$XZM7sbs|m3M
zO0RakEU!!Nk2n>Rkgaq4YqYUc+Qrna`pxny8_X~7e-UPXETU_%ip=s$$>-AzgOlGJ
zooz5L*K+-4Ro}4x43n0LmNG3m(^b<fDR^t+uJRk5PQTqt@<cY4{9^vczkE+qcd&y}
zd-TGo7Hd+sW$|u%)8=<sY(`+=X<prUE%p7s*Oc+DS*uXLR{3)Km2)SpT2{P!v*)$-
zqJxYa-MQNfBG$fbVxRRU^5ly+@tF0Yn|U)&lonX>PkAn#d_u+HhFC+#+|JW;crDAe
zGdBrNWxmQ4t2Z~`RhPHUmE?}MJOScfAH!N>qbCM#c=jU3Lg$df)2tQu!k2v&xpDb#
zWz5ao`CDW!HbjX{jy2hNSj4vecf|u^c6H%jkGgqZMTZp6)Uws>^tU!zV5`mW^vEk|
z=ln#m1A#ny7N0S>baTen#4cMlSH0$~N>lBuMcd|0jh}J)0E@v7y~_!^a&O#wv}(o#
zy$5ABTvuNxt?><6{PaJAhpC6<wRRpZ*Tb?qFUy%JPuO-+^qNBC*~8&pcdbpI{^T~E
z=*{YV`e(R)Pep04#@2<U+nz;4ub<Vnph|Za=ga(;hxUDxj%yH{8q#;?Rm`%owBXFO
zma@}TBqh4U6BD`njj|tKKYQy=hIWqQs|7bjw#%;eni}#=T_z?)xVk)dLEc@T;~gt>
zXZ>t{bx4LadEGqAOPPy~uqCt<D&CZK`0n*|Z%BaG?dl&7c@Azcvw8L8C-0U&)0c~S
zeiUHbK5g#W-@E6xX@{+B;90&?V_$!ImF3_0S(0Hk9JyCgf>Xa9<UbWGryXOu+n?pO
zY~f?eIdl7hpN1!1%llHV6~5@k7McALr)<S$9c~I*yYpj;Nvwa|-M%+0mUDkI-g-83
zTE^Ooo96OqZ!=fp)l*hknkn|e$hLp~l^JW#e2Pz~Pzb(aQar=|RB!I$Z3*`U6%?-=
z|GA9IC2HY`%b6Qv)RT_gzA5%r@P|uYocyeHyi-qHb+9X1^2}>l>_mR;D>Ii}iP;i0
z`AgS~uC7(Fw;A^yF`5yV@^@}#$(yUwUv5}FH)x8lZNNNXUC#9{KIG4`Q<#5u2aD3;
z(}`#Fd_FT@Q$H@VUupU*qtGJnD?bGv3zvM5JYDJgWZwN;7SAJ>ZVMjF56{^?^~UC7
z;R~{E^P65-xLt0m=W>VMHw)iI?>H6rtaH)*<^9L|y^l@X<k=jVHoG)&+7ja(syg}n
z8n@<7|CPA<3;*;9-<L$?7@htS`KfoBUnuAHd6gTKKbNeouRk4Y`1I%18&B5ve0qJ;
z^khKXRGV)}7oKeNpRdy6o#R)@U+S>^;90}v(>4fXw7<x|Hq+eLXU$2YcHbU<#%&)u
z&CM+3Ug)jZb%$fxAB*53$Cc;b+@7g;-B$SOUZGR-cy>LkIyA4(_&~=T(ZIKh;(x6>
z)*tXV$A3a(*bV*TRjE=tp6K=YH02)5%VC!_Yl>#yy^AaPhNn*Q^N;#{GnUV9h}y8*
z`%AC)m*kfko2Pj#J~P$1xNKgp-wtzq+vsNtpNY79pD*`Ak82sz!pCxZRnmISzLZwH
zrhSuH)7W60r1*)6#lGE&#cf&7PTuNSJnQz4;Jw!#ZTgUC&bq+e`l80unzJvSt;l(E
zWeI;^Sf|W|Svy1K1mwn8&lYqkV6a}h_(!++!j|`Is@844A@_H&(O<j8)1KIS8t1HH
zdaQOnVBIaxtc5c)vQ(Cc{fO&4{^(e>mjCPx%l28nvA2$^JRbLLO}o;r`)#jJ2VQ@7
zUGAKmb8N(3-W?yG8oivm;QZf(?44rwPZnIYIv%`z?i3%Zd`pq~vVGZkOd&@?88Vi4
z>zcpXwdC`*>pJYr%@a?b(_S_sXhPEI(nUP`0*=m|IVGrImO8_UH9<Q!@$;v39xO~{
z>yyqam6g1&xstb3FgC?<gXeRX*qU95mV(iVvW~Mh`*AZcxO^^WyZd<i)96OGr~dVa
zbMHSps43Z|WotWYxwGPp{QQoutQK)E--(?FJ`(79)yaBi|KW~BO(CC_pLw<DS*DR&
zu0u=Ub>%4qdWRR9EY0|#pUb_yy-#N)ldIewR+d$2=lx~dl~)_7SSh&vo@}e*tAD{V
z#{0;IWkD%{GySFF623;9VY{D`*R42VkA$Gn9$yi?4<8@dKRS4_na?(=U#`%kRC7+|
zo8Nv;u}+do@6@K3Z$5N?@yqu!Pqv?EI~W@5vEVsZUd`#;1FO?Hc#aks+W1w6&Fy$4
z7;&*_-8xC$M=UWF^XzI%Yj1tf*wh%^9I)g<R?upllXG2ize`tMi@R%aZre+fyd6#5
zo7fh#w65rk{;_eXnxFd(p%tt7jonvYPIH(U)N`gtKk27O@NK~ltncQ@ZkXpUX`d+R
zYkkqD>MGx)&{wC`<yfAxB+u5(Yi`ia=K1RpzH)hQYvH4Q<NJ@*H*oVC1l&4zbAqva
zP{69GiTy(BPcIhAozQ>h&)Zj-k8}6ene4QD@v%2jdRpMgJlDRvK|05>c+X^by<P3l
zJgxQqVV@M!xOs<GdxW2tYe;$)K4+an%FPwix7<;_#XX@l+*f$pu9^)e!!CdAem^s7
z@{77zZ^Ys!@)=Klx#UZ5O<ZNW*G-j2TGD0T18hZ0oT4|F@BJ)!FTHMpk^Pd6%B~VF
z-rV;=RTq|63Gv*QaB$LZZhIs(Y2l8?+p8}`|B2ml?Ze?0dY)}7e(FsQU$A{z@q&;g
zF}j-tpFO?h!&DY<RoGRwQuWr|9a42RW$AzI4=i>o);$}v;!D>>r*{4ut}U-K_k5Tz
zZOw#xT`O1I+a=#;{;Ou*yn`FgwW;0i&)IZ9a+|)%eQS$}+N$;K(|7%e_<n3=nD&|d
zo)Q5w+2tp4eq`9=vex3G==%B{(PsY{yi+?2Ym!-Yl6Lv|c75M4<Dv(vz!sMflW>zi
zUTa_MdT6fd`cv@H>?sv{POjwdX=^%X?IEW%zcu`C1cyOuz*D`I{z=od^b)ye%y_$W
zo7mLV#}2%IoVETx!*LyL0Yk}Uev7Uu9e<tD+!y?c*C(rF?cVE*8`Umuzba^Q)&Fbl
zkG${fig~tKobG9Iam)4?NbWpf{bS3E6|25qkNZ9U!-FS_Q+BiI?4JJcsxkAW#fSOk
z%`aACEqIrau<YWZ@46+f4st&>cYOOe;hC1Le38y?mENE=N-nQ+CiJjv-`3>a;k)d1
z^37a%`HzVayrF`xE@_r{U3J>*x5j6sbhY4P?ui_+leeAU#Jlaxk-Sow3whTw*BdED
zcQ*JKhie~=Ua(zhld!_&r(#btzb}#d{cg32rtA^#DYJYfE*uT8Xv#dZVXL4jSL&ZD
zmhBZre>kUVgiPbU@|SzvgdIQGmVMI7(qrK7U9fuF>p6RGsKqv#w1ykA1@}G9<v#CM
zu75DkrYOyd;p_~(2~CTWj~=@yV7jN9D^)CX%9&M*O*Q+DRcQ)rmYq?4w{Y#Ed&~Aa
znct`>2s+c!7yL5O@J0B^iGh{Jc0Eu?VcfVO_O{;%t(YI*9+y`aTxS>Bmr%vizPx1f
z-^oYfE;xP8a5xZRB3pfD+U>_(X9U+&ai3HUS{LwF@%)0>7hZl<yjgmf@zsf(`zpTG
zZ_TS#+<tufh+*%!RlY2-{Tb<QiEYs<OpOe?gQN<#TQ0c8U8J|l`_uQw$2PqwHk^6V
z`0$xaMYdV)zG@De{cE>9-n>w-TFKYO_GSCWDQQ=<pUi3uHOh-!eSY%USI#USyERW<
zO%W~i`_CZy_oJm&;}e-zl3P6ZJ32n;>)y&(w13`=9LAHJGAn}*v>Z6lK4H4O-h4j2
z6fqC}i8G|dbut(a7sLnza{F|;hwS%B4{ga8%NAbdwd9J`G9TB8m8WyQEn%`+pBu3D
zigiWOHb0+!(d!yN(@oR3nm>Nd_2N0s64Pt6GSfP!RyR~mL+R#3{nq~s{H4d%Xs=QE
zII}NJ@zl=^nOl{a%5)}m{oGz4oaUwU;M1?tV^6fc@OjCdF1S~=QseQBhh`Th6fi1Y
zSsUbXd7942-RuYcvMf5<b<DZhkA0qEnsN%8-}ZtT-><s_UEh^_`qREmF|J>tr#)_!
zY5Y6UTy>+R)9G^C6+4=qZPz_vRTD3!W6Tw%c1|E>=~J7#Og%h1l}bF9JW1aoq!i{E
zc*M8nM8hNX-J+3mq^^8izffxPg*&T)^Vz&YgG?meZ7)4LX`)q|?ANDTv+6$@zOtPY
zo2>XnYHPAu&w7uGuV$`GzN%5mE+^Tu>BPGoTuWxJZLa%reZNf7+nYaY7~>P?{)$+9
zeb$t-dW#s&G_LaPzP#xYgW$FW8j0JQtnTC~MQN@|_?28H<o0#igEO-yr<sN=`?YU9
zPv(~osSnaiwl6s@yhqAKWQV_tT<mNc@2kd~GfZF3n(f^mbLy*`T2B9(mblExhJ0_W
zx|5SuaNj-l<?*VTk8=;FFP*b?Lbi!xUar`JD_ns)-bnc8#;`c3Ez!F9^1<rtI;-ri
zv*AzdPbmI;{_?Zh(Ube88k@2%b4c9rd$RKC8i$Cg<GZuJDBkW*zofm9d&0VdA|9z8
zX@PrBtli1}Dn6~M{%CJb_31M;26q^9FZZOdIH`!=&TC#(A0x2vRf*-f%MXrb$zEdd
z&X{8&FM9d-=f}S`#<-=-OnuqE!!Wbh?5@Un?w;=(k6-xBk(&{<sDf|)VZOk}rTWG}
z**D89Tzhg)HE(<L<$-G^>+Px^g(i1|M6UCsUK84IBXW_<jV(8}dg{CG-1p$g=F)by
zOQIK-#@ejv{8cF0oUh<yTcy3;_3mUT8LP>2eib}XS&&(@NK>G9t@#5x`_=sM3X84=
zzOoJ6IdA(JwVRgNy6;>B)K3O<nq3X3S)I1%=H34cVipcDg>#iG-Oox^nRE&~GL@CA
zkX4*6f9CdqM+HIKj|jLH9#>Ocv+qo0zqw;exxr(mFN|{&-fvy7sI9BRZ-UUbtERs<
zsw|w-m$hq!qh0^z$dob{X>Qj~f?G?OLYU(I<^L=?&XmUKRd`G$t?5dpKJOW$FNysn
zt808Whg{5Oa8-GqXY-#S^uh9%QzM-vo0i=Ew`;+L^fT>z&VL`fR$Y9Vw^P#W*xR$F
zjGt@2JwClQxKH_7c=xo|eMgu5TG#Qp*XYKFKCZlbDVm#}&HA|})Ipc~`;5B(3={aG
z`JGa$w!HF-T<}C{@06dy60cY3o$B1>6Fk3S3v=J&lv}eV&YK>a&>4Swo0d~F$Fgwo
zi+|1>f3bDWaqrjMt5(DYzFNF|OG|Z-xy*w(0{i0q<hTC#&yfC6dfMg-_j(^c<5_px
zGH!9uOf9yfS2oOCxckjp_s6SOE<MJzY~}G8qA#DS9LTz%zo)LsrBU(LBR|d}{q~US
z?+-ryc=DdR7^izm)XScRj(O#CZ?|k)U;k0-QJlZ?jD`o>^Rm{MUVO9La7CZV{s7jO
z>*fauPZNK5`>562_l7lhUs}0J8+e=)OmB(GSv{Lc!ZxI%H_qvCu5x&6-rcudMVqc2
zK4qcd`0|aUjYjg8h|;cwVavBlUpv3)%C<u)Ct6Z3ugNVByX#V4`DE9FhtG~((VJ5+
zvvj6R*sqGGGHEd^I^8UGL81k-gZ$;C|LIhx=n4hK6l6`QT^3i+xvI2DZBIk;9xtm!
zlkP5_C-%rPL2=T}6&~#tbIn{=W%5X@?Vk}P_^|#=Z}mFCf~+NzHQ)Qsb)Hh#v&-yR
zZ}ah~M>@Fw>2CbDpHGxgi+{?AxlW-HnhO;;+aBL|HqlM-#k+MockJ0BSG!g}J$F~q
zW6h+X8?^z8E<eB0CBwCHyI;z-Th<&c4c3a~F5x2o87dz>Y)MiMi0YPoB-ri37T2co
zrefOd+#~zWOx}LPOuU}YV$<T|UbRV+Ll$p#e7W$rrI4F+VT$AyL*<okcPn`=TcS~S
z@w~ye?|f`)^aB=q=AAllan;P$=WS_MqgDqAJ0yN^x}7;~%Vme3{~31o2~G&rJl8+@
z(TNI;i}N&>yqL1mlr!I2p_3!ZFJsmA$zI#be^2H)c+g2|W9psvVv{FFF|72c=w3hR
z;jX;xB5`+i2uyl%Wk+WIq1X+k0ZXhss+z-Bd1XA7U%66u)9DM66LOtvJhp^gIjSFI
zl)vLY!&_-q-kC>g9)>L0DrdcASFqucRkDExB)&5z>Akp|JG;y8<@wX)`9~XSxNcfF
zZFMi2Zp3SPQ$NnXXJyWgEOQS>a|Y%RzgsnNIi7}$drJzJ9o@dNJ7lKH(v0x?nGE-`
z%l3tMJO8@ddi|I~tlE65y|Xm8NV(*0p7Y#^Z<7p%-{0fC?hmDT_W!JU+4Ii#pN0Fn
zWxVRTsWz+q8hukwsftAQOI>y<>2l23mHv@aV#gBi)8G5#XUkdbnZB_5)4aB&)@JpY
z8QV;GZgU>Kyki<?wUUy{oA^Dt;-_ps&JPth^H}7>`>>>&S8nex&x+^BTweeA;ihoC
zB+ipt%#xnYSnFhPE46o{*t<tcF1On2JG?{UFE?cfJ64|JY4?b&OTL#R&fwoEBzryc
zSD|QJf=)N@(~6Cm)h^#23NLvm`k%qROnOV+qBT-B6~f!g=c&gD-FseeKd*E7(-v)x
zBhqshT|TUPO=<ou&J%iOKK+Nd3ua1bb?MIf8O(E3!{qE=wJOWBm5wu~Z(4sN@p$uf
z?(4Vr<xgI_plxNoAfL^S!e7oFYbG;J`LKT3OWC}nsX@!`E7?REt}R%nyU{^}@oMnH
z4S#QZxofwZ?PRZ1v-ec7kf_CLPAbgMofyTU;;?X0#;W;36^S#mYm@u)QhtWq4?5g0
zJ5%>)?xI}A!`koUPHC<DvEHY0o&WyGGv_zG&Uall-LLE&w_ldxsVAvRK7RjEzg%#g
zLf=a7nPGV;)eP|~+%&fDU~)6tlEE@x@9uwwqmB2NH6K{)^b3nQcmI}EPWqgz?&y@*
zuy;xV&Wd{j1*@~JE{l6pRX0C&!YBU6hA(@*&RO}hV8za+*<SM%e_1!3>G*4DBJFm2
zRqM`0%6tFh=>0hRw;=nmK+)XFH}B`HFur?lDwnL2Psf9sjnZvt-uJHZZVk-3vU^kX
z<J0_CKVImnVv-3gH*h?)F*5a3ubpA&Dlf%M3->n?cS7EuF7n*|pW%Q^^401S$8S#R
z`f{t)Zga98uU$^k#&>3a7nE+ae*N6#KZE||mcsebZ!{O0JbN0PExkcM=T2hV3H{hb
zD)+wEO=p%3y*BH#-R95BFE7sLU!rw+w^7ne-y-d0pLZ`k_T|~Cu6fDdldqQ8FR(M_
zu-2V@gyG|@$4-xO|ESc-pV+u3W24!V-h*`x%q@>oRL)Pk>}Q^o!Zb^KiI=U@v=av`
z75t9QzPu^-KSP~4*Q;4|;r|vLl?YFa5e~`Su+rSl+Hkgb_YH-ayUc%RZCiRb{9|^R
zzELlC%FV;xt1UTupWT{r(bn+2y2_pj0>8hDK0Olrw4KeE*TgTRKI}}Rf!W*4cWx@$
z87p=^%$K!HtYDvTGH9i{Qs3(jO!Lk99_yL-+eq_wuK1qIBH+12g}<FQ^qP-fO4us)
zh$7K6W>Ys63#AXjCk}ADy41EytB_aNSJ+2$#*CJg8(x0=%zV~&rc_C!$oJOi^O%$c
zHV7VgoB4Wi_u|f`Pr^!1#Fc-wYn`1MeOqEXPx`KEFT)A57H91e$Z(O}$7Aa`F=?-r
zt4Hgq&&K-w*|ov>QM^s_Hx*nhy&d<bwZ>lWhSdY7FICYKU8g5ER9#3a?b&l$>7}R2
z?!wKn!Mhe@cWpn^<Q=KHW9yu!-Y@H4Z7X+unJ;?4_q%CmVc7+jVE@(sWGdb*JpAKP
zl>6dk7SBSLU5=e8#j;$@vs3eNfh_wSLjebM!GLw&yjOm@e*SCW;R#&3j@)_5H#w--
zDE)lAE$c7u$6_i?`x81}OuV{Gyy^DI3nJRPKfa0C@KgItoi%&c@+Dld!WV1#_1$?-
z1g&89t-8tJawn)S$1Gvd>e46E`_I}%WQWXn%^6g@Rp07D(wQW0<>)+R(G^Y)J2>_&
z@XT7ax202bsr;h<468-A?U#{}DQTXyviN1;t1H*nWhLjFNT|8a-P@_KZq|#~KjK{b
zRh+{kO`VOLTd#<$v{umlnJ>Nj_VO=36Q<AIpqkqdb8z;G4|3fKt2Qn^)1~i~7-#MJ
zpJDCPkDs^SjxRL-Q9bK^LY~UOFY(hpU#t1guw;9h+f={%8jofCJgn@FwESMUb{+FF
zxsZ=?M-&@&TS!={W@}&FR3mfJbHOw5j;xTGMT)BvbYqfk)XvX$$q&sh_uVMsyI(W$
zPwLG#(v1&qM>q05F5%e~n0Q$FKSRZZ-R1SucKq@Z7xl<A`IHoJId<}<zqZR;*G}Gi
ztJ!Cfl2-Yy>vk1=wjo<5M5--u&sgPtai_P$yT;1+nu?k09|`Zt`~Fb$lY*#(^5Oh7
zRlyIwG%nxea)JNd8ugUyRW(uSM;jjN9T!-oe`5JwqvT$b^L|AvC1O8W4@Ir~JvFp$
z{qH^TE3^F9R4ur4d!qZEne1#bNfXw(maI)+x*8Z7qIx~H{%HQyqaFoqE6%EDep(im
z%(8M4>%KnwfYu!@)md8e`8xLHUu?>k*tO!|)Z^+`(tPIIdAKKOuh}dmKV{h|b06oK
znHeT?@0dI;?LKu^)c1)rtMPu1P<9npg$0L~ZI-jUD_^m_M|Ge6u0E^w6(4qntQCy2
zkO@l8G7S1F|MbqzP45yWzjk+NbJ=Nl@M8Z_PT_c70Tw-nU;F(yxn3Q)c`8^vBF$E6
zTS3$+fj8`)*}T)vbu0c3jEN0gc9g|E^k>qh?wEt6Cw|(lvR0g=<!+Gq#^THLo9t~i
zlZ9D1p7AMcd!e+vnzdffwVNg9%$2OGTF+{94u_<f{j_)*<zkl0@GM-N*Snu5JYj>0
z?~9EYTe(jER#w~|pW1PVP3gd5<s|K<U5f84lWIQv)KKb8O78C|_cBSG@^sB{CnKf2
zYpz>(>x1t9OcYJARL=RL{bK1$$<RHYf>nCcmN?5X>lc{aD4ZMkb-Q=So42=5S=Cg0
zjt!r=;ruhZjfc$wkFM$0+LY69mCHZi=8Xj|hdAQIvi9tGHose9Q*K+$jEZH~6z@11
zKFG@#pT$)5SV@RokM&IlhhOILewn~0KMobXnjbLZO^`Bwf|7FSe6EFh-S^-7Xz@HU
zs+{`Z`9sdQwPoL2S~?hZOcYvQ`r)Il^<!sqJ+F5L!JnBYKef2h-dsLWp?C2?m+A4J
znvz%DW{PTIbG5zkw(Yz6+#N!{EB-UYMEe;9<XwI)DI~{!^w5sG@*2~3i)>+S^xMdj
z>Z&%YI%SU95#D3@iW^J$MZPB(s<B^Qxjt0s#Ih|PICt(+dpn(DqQWZPxf63QGacA8
zfAYWST`R7YOcm*`eyh*Dj`xs?(=3J;T`O1Be^0*rW!W1xId|LK7yB=s^~#GsS?Zqn
zoAb)Y33nJ7c9h*TF}N^g>;ByE7^S!Cr$iQCnyIco)ms13!;q9)xf@UHIi7zYd+Dv|
zIcb~TJmolH$~K)(Sm!AB5$PQP>wi9+&z_J{G^eRB=H(gvt>@werg!y!J|x{XS@G%3
zdA%?9>UzyKOE$SQ=V-!`6H`vy`SH=^BKIqCo`N66rd?A@XYeLpW~(aK+5Y0$`k;Hu
zgP!cVW8P&YD8QJ}UA1UdM)%j`q^Qb9;~d5(5$g?SOJ-?!wjF7Y*t>9oZcUV;)VjI9
z|IObmZrdRwckhd`drrotRbRHxo9H*^Vac;V_FKXVybGmPWj<#=YA3qw;qmFujGwvl
zR0R|*^*n9(#FZ^+`dZEHRu30+r)+XO{BGY{srTg#pN__v%&*%hx3N3TWyO1siOdzh
zrgJXIpUAs1Z~imZBTt_1k7LfNQ2(~zWX`38-YT8xbLvWN7Olyj%)D8l)qcV<b1U7i
zUCnaUTVf~ZHAEfcJ;7A=vH1Akh_9=TZJO_KM=jdOIQMS5q;>!F+3~W5h75OYb<Jh3
zz1ZqyYUX>1bywTkhFc1;4Z-`9_S)$$m%Ni{b0FAuh4VzSJ_X@}#~=T8>P+a~9)D-|
zs<QZpy@E%@{-zY(6py=EC%tBennQVRk!!I3%D@60hNtf9JX*fm)pXk0gtxv>b5>}H
zRyEi#b?esP1BORG38eMQh4S54a_*_=@jC()aR%DwE(L^$dY35wyxpzr6f!M~f5(YS
z2kxx69KXwG?}^(?-`Czt@t>y1x!I+v=K7+m@69C^>_4MC<K~9@;=H_k?+eN<U48mx
znskT7v^gsRU#|CP^?t2%=~Y14L757Vd>7sv*XTME{i7cqR!$a>6|*+26k2}ZWtVl?
zsmcC%%HJk0N>cOOmV9P*-1QjMJ_ebpg<`>aUdpd#_4#fRxSe@P?7P5@mAxy=v+LIH
zmOY^BT*{X!UnIZw<@yp9B~yib=iND_d!2hvFIxZnUrt&npZN*L^56^irY>{+&#>^?
z<Kt-($J11tGFJ+n{qFXJHUD;hZm9UQxqIi&mW!*&`_Hf-WD2{n@)eDn`N~&TT~%+J
z9J0b?YDnTvmY2RY2ks}_iNE{FBy<1AqC<D*&D+9S%<1>V+2ScrlxmVmly}nWi%qT%
z!i(K*yZGy;Wa`O3?Gk<YXBGdC>kIZzs}+0h6<zW}u;_#P%hbu~iYEo%7FkW~+VN;{
zi%V4Q`st5!%x4~a8c<!BmpRj2|5+Dz@!}IRCC^POek`NP$iN}deR9j|j2M?WW^Wba
z^#t}vUw)%;x#vH_hQPB^7p@PNK7DEBn&T^W{1!Z99#b8ZS5a=i<NA)KAJHwI!QX_#
zHkMyn=K5vsjMsq~7e0O9i?1%SF^im7ai_zzaQ7mKuIYc2yC)p(|Hg3U*~E#NJDXJl
zT+}wziiEY!ER@^yqOW(+`s{>{{~0oW<ixk{&3f*nZE@A|QSC9O%>NAk4jzlmJ+LR|
zMVzzVKifG!HmaOAGW|M#b}74DyI4px<MTTi$=$BcCNDj*LjK9|%NF;XT_b)4+KQeO
zj!L;xlX_(iv+|Eg#$n>^NpJr%T$YJCQMKWqbHRd8g+p1!x<{9td*K*kr`LPn)%*{4
z41fPAXbX@@DY7{=C2C8-Z=14{hwi7Yvwrk$g;mxZpErM6N`F-H*d!gDZkuyu#Trr5
zSJ~&elib|elK9IRu0K>2z5XYnx=&)Q<eHss3Mrk_zW3ZSN}jrC<+3jG%tf!d-|uHt
zV>hssOTL_S!GGDLSniJtHeJ5n?89}(Z`ZnV#+?GEJL|#|MU#vgvv~Fzm2@zDnepIe
z$-~QY8;=|>m>qagyhEyT?e_;EUAN9{s!NfX7%;))-Lh-3g*%q=MIU${;JW<1s9tI1
z%dGzlHe!t1CuGfhnQb0E^VZo?P31i=jWRDcY*F(rnqk(t_K^3z(=|7;wrz}R%zd77
zGIH6$)1Ncm`^~tsF;}E^y~?NR4wuvWcIn=5S}E%7Vi>h-bNJc@zg_+oXKxn#^kkGS
zI{0+a-J<NzG8R1xF6HS&_FhW}T6k1y%Cws|wHcxqgnW;h?AU*+OoodoV!~53T{D>#
zdkwAGB5kd_(~|h--oLx*l}qdv|4HT@XEgiwO8>Ig_qe%nSI&YNPpax~FIf}(NJmg;
zo77)7-y8GWCGI+{G*LF&d_Aye&f>*~lO~;TFP$i^`%X3MR{Q#TL9yG8e<MDLufA|<
z|AhE2ot~@goo5_4aGOa`ELOoudvcc)+jjj@$A3FkD+bMO-M_Ig+aUQ`W*W=mLpKdv
z#M%z-VofUl5c0P5-b$n3V=|`-vL`8>=nk<??t3oyqw8Ua$j^t7lbODMGjoZktvF_R
zCwfOp(e16QHg<pQDzY~BhVqI<r`&s8l|8qj#b;-cs+0f2+srzLj_f$Av%uoxh7+RO
zZLZsQ8s^t+INfyTc-*ai6*JEJ3iE|7AG9$j-F-oKNs(5_F6W66YtQ$}EQ{TI#WrJy
zbJ~izf9`x}4aznC&+wmNhfcpzXpqHT^*1WVZW^5w57Oi4npNidz<VM8Y@Wk=#F7tu
z5AePeY^rryUsxc^C-2z57frwC?zMh3t^D|khwHX*ajrdoG2ea5wWOypyNcFV{wVn-
zo!z*>cgtEXhSgcyW7GTpGwhbD`u_Qibl}31A|F!9W^QR+?Q~ByfA_A0&`J3loK{Lt
zV!O`pUU5gsU+;HEpFhjyR;W37(Q#AB-<jNgf)iez*!y?u_s4eIZ?Iij6J^VKy({F$
zLB$)@vok($-r;=yX4gY*#jNFhpRX@2%dC7gTO+bZIcSynmhThWq$)}-9*yVUv-YCl
z=F2x!6U9wa<=^ZsvFbR(>3^zu$79_FPP3R+{jlptJo%++td|Pp%(%9C=4{O~-BVsz
zR;4928%@8&?ezFy_II=Mx?7!IzPCNAE6#fM{MBW(U($qo<z}qDZ2Zi1+0ogKFSByb
zi8&Y9zW;nH^E`{7^o6^k7YhEFCeJ&N{dM-0$iQ<FTfKX}dvCaLEbM$E$2Zsi40`jQ
z9kQG=Bmaike+I{A`)_G+=cIh^dAP~&;n~TR6BPMgAN!QENa^PU5xrk|`SD%1YVB`?
z1|Lwle>;wC$J-qzja2G|3jZ^d*=`rz%`0h`v{1?UorTlsbC$(<Qnglk-R)1zg)c?k
z>h^D+^x|XGjE}N5F)44ibhLV0EqQ<MYQdiyryhPb6L}ch<{r1o?4o;?0HeSyx71AD
z2k*KxxTLo2uM#th>^`&HXqiRMsWbT}t8ebQyCkNi{mII8`!91Qx?hZ|yrFw@)0gnp
zZ7GHcQ*WtHI~wY}`(F3*>cXQRCtT1d)_myO?z{R7Q__Olv-OJ<3phF*^nc!ZRNu{e
zra49I4&SukH75&RF40pt_LJkNzi@Qv&-P#Y<`^`3FZIfOb0}Ey<x_*-YF`?Z#U(P1
z7jD?~mit!w;r#g@6L+#{o>&*0V)4;o)f|a}_|k2U_6np(S21d7_i6E8Xxe3YpQDK9
zyvL%i7eAW>q*(^v?+E@CmcxEy1INMBwU4Zp{<GRv(V?!um$vlGOv{-9d@|x`CQr&6
zCo?jYh+N&98x-ArC%j(9pv>7suwTjf>g0{De8QC7?-YGfEITV*DG(U;SNi?4ANvpO
z)Y~Gocw_hyuM=B&+KL}r=zFF<%yQu7+mT=;#dJUW?d|j}E^j|-Jht1D_j%23e&?W9
zeI7dNnL^*5YW7Y%9Bp&!R!8i=>Wa^I#B{zKy<rd{CpKsQneeEV@XFd7x6;g~^vzas
zJf**l;f`Gri~RSD9ippS13R)>xl@;|uQ;*o{SWt#Wk<IjSlf4?J<>R5ZiYE~f1;i3
zzNZJ*$!~~jvHW_l{%2dR(&NHfd;OOk_kRgH37#rQ+w|B?y2rvgMJ_?{L*V=MGsRtx
zhPBiutnX@=mUOi1vsr$G>W<{v>!J;s_rGlax9FI=`cid;;F2#3AMW>@*H*mEq1&i7
zs#$HN__BhwSo82GWk(UC@~tP={%4T0mFHV&5Z&|H_{I}8WBCp5^Oihri#_2TzL4w1
ztcu+E=fy=G5A3)pyh%EHnORh|p^a8u(AK@nt~>n5-Tdl(8D~>s<EhCdyZrsdZ~2JF
zRUTAe*1fD0%C^PtNz&uwj|r^F8(#4`$wjWPeC=`hPVJ$JxXQzom&8Mk_~v}n7C-*z
z@x#6aC7)z1L;5|IO>jM3T{K60!I3*pf>R{;9)8^3etG%qn=hMEEj7hAvxWCh+H`x$
zoP$BjPMpb|aE?RdgV1g6cIOGtcEu-Yz5JT{^0w$q@3jr<z5BI-i~EkZuHCyoNxVB(
zgYVb+sltx8d0p8jCfKHyX<vG9F-q;tQa6dS{qBB++sit7H(1|wE#Urr^gqMPe>!pw
zn_f$5T?s1+OIdurEWzkaxw)QzQRz<S>J_J7Tq=6NxR9gc=f8Pw+J=+OE$1%Z5xIyp
z%%o$HpShHe#XG?r7tHdq9!}~sOWsk{S-(+!b=MQ^Yx>tT4T`?nar&lLrL?gu)+uj#
zcB19Cl6tMHc*stUjfqp{9k2}Lnk&a^@>p6$#n#y`wzy?qdF<Bk?XUhb)XVrBZG2gB
z=je=LR&)J}nvO;}D^pIpoL1d>XX{4mkG$rh;a?y18?7|B@|WRD^^0=VUnZw>E@>v5
zk9xXEtC;<)>K9p&iPpF5JzgJ|vz~nNCu?+{V219N$7%k|eVfd-y+3ke`Gnw4n<Y;x
zZEAjfvyQiFz82%X+w1t*+b-HasLGmW;k0eTCY94A5{56f><E6~)~+;LBDq`qjt$#|
zi`Cg{XFc3Cd4kxI1xKFpX`1ie#3$UMnm_H5_a4LahwUplVwNB6muWg&@OHc7kBV2T
z{}`Icvh8KFu+a6guIN6Xlx7q3?mxqI(N&8gJrXa+Wq3r*NVzD;t!2~ESsgV`?vKdA
z4PDF6<(Zrm`yepkyZn+3*CuE!>MJ?<pW$|ux51LCEsMUM@hZK^u;ewv<y|Yc-!&=N
zxBg{I(vwt?S90B_nzc>SqRx80YH3<zE~@f1?e#lOBdy1)SXg>@<lR4NDkE&RopH*C
zz1|wnn7%ZIExVLZDyp&m^ed?pj*e}YgD(n)elOg0xA5h{xPzbUo%ixxUG$u%-T91;
zCp%Y2qWID0`6Ve_9Xr@otlFNZ`1|YSyFGCW4_|wnwRmF511F`g*P|5T;$?y}%Q%@%
zR31+;d;7<u?O6NSBF?oZTQ@a?N?5PsyL`LFIrHwqdS8PzlhRG)-78O&T%7WwV0X^p
z2j7-1TDGh9W!w4-@@qc#FVC7%QF&;-Z&u}$A79<(^@wS53O1}M+!%cN{3QLHtdgZw
zTr<QiQfgLDH$Ez7mobCiRLOgiw%mF?6+a6<(<JpPbHDtJ*m!V4R+e0E(fmd79?Dud
zQPyXVzK*V(;ljoeo%1%|s{8lh;5$!OM*q+|P*id2_=Bh~H^sC=Uu=2%Zq8HdYqt4J
zf;}g9v_6ToUE2Jg;mpxIuCBEACTA8$^aq|OiQKZy>A8|y-@W_OOLbHC|65RgxN5#6
z+x-_dtAD2lrUq($k%|aeekSu(%TBI5i3z+XRQ3q6`+4pA_wNt8k3dCVvGR2@m9&R~
z0(We3Rqgd`x7am1u{KpX2e-dD=0BNthv(Bf(pzH<E}ZwU4XRqp`^rVVNwd4p=JJsX
zQ!l#R4Xvws{hvY1=DC!=v0->b_ly-EZq3m@#k|Qm>DhYL1@0feyIkJ2%=E|A8*!5b
zFa3%TnZD=*!$i?Tg$F#@QlhtU>Nmc={WbB;0fWA{4NI>q3y#{mlg}i4XHvD>v?uSs
zb5&G-6x}B1d`0Y(#+0b;OP(uz1)5WrwiZlVBKy_v-d~%$AG{9xO6y*knRqf}`|F~Z
z<Yi*6H=eEk932|@WX^8=Hn|q=;JB{q{wB_OHm9R^ecjmZ^3`}_>F3_=PY0h#@+B;}
zI)A0nUyEPuCzs1`8Y%a0PuTY*UgT=&0rLs%CO`R13_i=UzMSyH`N?0)XX}*q=+-b?
z%~So{5fgivV|ovdU)&s%nLJ1Pc{t>DK6PKb_^rUgN!ppKgkEv3dim_=+J!xQ{2^z~
z&0Brp=!)7!#!O$=7-VKyba0*tv6~wIG>=<JIQXSOCd<e3uhL?!Z!~*bBd=|m^H1qf
zVqr|@)6a=MyV+)K^Zqh7YF&tCan>{5{c_3$xqI@I_&jg_sW_2mRCv0^UUHxNgu8JP
zC*&>qMCTn2^IpTQd8pgK>37S{`j?GioJ|vNONQ+dHaqoB`NQ<5?JE{+HhksF$-ZW0
z0r#sTClBhGeR=H?JL&Wxd(XR_lP7Gv$}gUO`;klMhY8E?whIgOdL=8I54m_tIroVB
z@tv8Ee?HpH65MQ&;&rQp|8~?3iJ3E^n6$c9E^GdMzKk)##^<|T-MOFUmhKbHqzzY^
z?qt!kHk&UK#m?hjAbM<fXNJd|-Qtf9Uwdz~H>G`c@%6=4&-_=~iW?j@OWqk>7VLk&
znD1!V;aN3XE>A!Dqj1r7FUhy;jK^2HmS1D_T^h~D&BuM|Kf}#gj1j#0CA<FBc$E0d
z7zbqCI`N-jE1!F__v*uzdOqqdGc5%ix;#6%+a55jJ<C+t)6z2S{WPT?C(eldXIPRo
zExuxL&x9ElrUwd6tX?ruEpbcWtxvBXs4uGbu~%T+_;M@f<P{&YK1<$TAt=}>#keo2
zA$)$))+NT@`)AY(CQ7z<s%EX#5({jfqI}Ha-|Et5_ev-1xqrv^^8N#Jw68yXDx<IH
zqjx#!>f$r~Ow+4x*MEy%lfhD{RG}}O6TR+WSDK#9*`(<TyDqnV6n>X~qT*Oc_1mqc
zpIPTlKK|+QH0zn7O-E;H3udmp^q}O-mZ!JRaFjgqzx~PAl2hqr`EKqo)+xU<xwm{v
zK0A4HK~L|qd%D+7JMJ~P_SRJ}*n7iqr4{CHKCKc~D^>e2xzFbFxv1-5ymODQo2RyG
z+kF3zNAvzOB&?5o#eZ}9#k1nNf$k<@br-|eajRCh%}?vvwu?zlN^;Ssw@)AUD_;G1
z-u;^Hs{Fu*r^GpC&B(7dRQA?Yy!Ot%&ft^g;|&TfkDUvRl60pF%j+Ng&oEDF4f~px
zFW%m9PCgROx47bV)`LRlC1od4%6ca#ZQW?B7tVWR;i^BsA9lB#s0mFwx?OKZS--sf
zlz($#?yu<$sGh8nS37m`lqHvsKKI_;*M2)D{oc3bYgp44#+&#|F)8nw?fXiV_iX>{
zoAY<nRKITc%o@C6t2*Zl{WrJ#4es=E*q6Opwu<vtxa@6*&~Dy6x7Tsn|6Zu1XZE|>
z*WSi3>v77F;_Y3!Ejg3@co*l-`s<l{xUh=lskn?}xPkcQquZ|PKfYQ2WS_i_)~5HO
z!Dn}tH5u4i+nO!qIVPL5=8tf^cDS&b*0SDJ{X!FtcReodx!b(JqRO^NFMe)RKhx4|
z7lZC4MUGa7A7AFuF9~An@NQZ(efA{PJsXy-Pw>mSa&zKqJ&QyYHfN=KpXKlM?H8SA
zfBdkdxBBd<5gS5wwmoi?eXM65XH)iLQrXXnhq4bI)tFQUKD-eVFK@_Exxa6RO+aSf
zuaGk{Ur1N6tl98ID!b)%vqnQt%gl)f@4Dthzu~jzy?<s(a9ZWJr4j4X{XIjjF;#@D
zx;!V#I=_5JE8p#2aaSRi@5vXwoU}RSaog5DAhOG{Y_oTTRo7C-?XvT3-gqfwl5ykp
zq6G`G%E})9*cje+j?Z((-52p%mx}hqF8j5}CtXEg+mpk}e;>?dd9?iw_m0VH=gXRY
ze^~RZ@O^B;e7_4vj>#o67*04HE1@xSqPQH#idnA<3xEG-INVaWdhIFgCq?o%zZ-ka
z_tsgj<dZ&Q!Q`TudVkg{IjdY1J}~M1m58ga>fJA7t)IwJe>?LetN-H3Ywif1n4MAP
zu~X{x-Rg?Wp13K0pSt}z&vbR>%pC!@?6V#}adG~tnB8Bf_C{Yj+ezXxTkg^7L$Awv
zkJPkGzPI1-=$W>8C%&Y=ST?J!?{(w{{k)V;-HVSl`0Od0C84!vzNhH7&)d)T>wh-A
z|9Ixj9L8sF)`!gWUuF1eb>h)HAG7owC%2Za=ZagfFX`>_>HjKraxJ+Zmwe&W!=n;g
zzH4svkk#HTd!c{At!WEh|MGj2^_z+PT;a@3(o;nkU-G#X1s}MtBwZc<bJZi~6H+T<
zUtX3FnrLQx-Zk&;p-0xKWvdqKid{arO-`U{`V6^*?U_8yN)H7sca*h0a@xLOKd0XG
zgcFuQuY9hox_Cm_=8L6^&&l43#@-VeX=|5WySpO$zHAu3!bRa!@2csFk1JMgw-l6N
zXX$pizKKh{Kete_Nk!_(OMlC_)05?@bi#B?<O~$@F7I>{I`^V!*TP!W=SNbn`FI&G
z7r7O&>#v8j*!`pssg`A)AC;bH3!bg1wye?@`cad+OIuj5iD%X2&uTm?%`Uxa(Kotn
z)%LKk<fcok=joFhLbj~?d?PN$M#xh1Rj%TrrMo_GZqhe9!TW7T)pwSioY8!@^jaI!
zu9<9Q+Y|hh*X?jxNEv(Ke&szZHH^+@oE`|NpWJjv|B0mfwW{EYeg%2!St8WeDaqac
z{h#5!Osdgq(+9ViZ=RB#_@LLbE$^mw-yJ^X>nD^Cul2HYRb1I$?WZFCQT{Yn8tW88
zd1G6~-E(Ia{N8i)_|N3su0rN><z{?tl=EF%edkx9*@T^cw{=zL8mgS=Xb;w3YI}oq
zg=5#%q{%T$w!IHN-h0;ayV(Bs2aR(SleJD&u2}MBaoWc_yDl3tJzb@={(dLp5B3FD
zG`|)qG<*<V`>{HILgy{*yuJG>W(OPcEckPA+RLw}bUH+5SsOQZ=1wRqXnSq1U3(*a
zP5a!>-xo;vvNd1YB9!#>#nLN27Vl2YR7hSp(c*&XLPgD2H|~Dv%q}yjxGr|dOHwZF
z+|1f*k1p=tI4LVSN7!KHhbIgYTjuP$z*)G>N?zMzou7Dy#m-40UpaM8tqxRmUKz4v
zrnK^=I~?kXoMuYCixggX%xb;yKxc8Q+-#PFD{ZbGGE<F=MLG4}HO#bAE%s?z^=g*5
zuIZ;f)+1>TDtoW2T{dM|f~A(;lVUf=*(a|GX}{SU6Y8HKKTS-0!(`p#S{qB2OV#Q;
z-uF4*=Ce}Zkv!efPcNEQXT5S=#}mdXHRZUf;|l+Yar4fY-B){gQ1*$X|C!Ap(VQW-
z6HG7cUwZBF3>k+13>gb|8O>kwOtR;8cH$;+&diKPi|kc5bN@5&vAj#seSRX<w`i`%
zO6jZRbJLFR5MAV**|hx9!s9Zx1eaBDpWLeNVK-^TmYKqGTG`3he8riY-xwBI+;6|S
zNF!N)MeMVtSJhd3p({<E6izu|<TH_P$tlhq7iJcl_}%(u_}AiN&U$xaErC0$oGQ2N
z?AWHdx<9*+>-P7Wjjjtmvql*5ZFqLO$E#PRPut0DU*fx6>N!yj(cZzU1W(M?J~*l9
z(&OdI|2(^`T2hM4CQNDf`C53e!ujFbhm8k}_H<Pj=}x?C6n%8t;*YVl_vDvzq}^~{
zKBu4a>`lSGLxMR|a+P*mcm1CDY2C)ITj%zsib&_*HoCv%iCmuQ4f}PR>$I~NUWKap
zS+|RMPI&h%|Ffg(`#*INYtC>RF(;gw@M5Rzg^C`J*3J6bO-o`fyL`NAa!9oCJ(t;?
zhked_%0EtU{m{w9b^D)A^t4yJF(+J426Q>h?HAVHTM&8b>eJ6Z6W6TMIF)cI^6O{Q
z86V!V-7S1Bx_OJV(Z(gQrcajG{@F2U-^QInRmvwHnd+R>TFU3sR1}?G%Evlw_vRb7
zw+daA{vG|{+mG((-oVA1Cu#A9EcLL+nG)0~kmB+{O>d<X)64Z8@rV2N&%L|oD#H;=
zabxF}z+3n3lyp6t<hm$%;jM>_%NA|>tKDV6JTY<3-6^qizM6dI;rOJy|MF}jIe{JT
zwg#BSa=ZT2lh~8naw7O#<YcX@D@(b2A{Ad(O@1GIN->77d(EFb^*PZ3;d~#re@M($
z^qkAn*8O51|Kj%^7owJ3<XfnC+<e(dF`qM@<rQtKR_B*qo?d5h>EWmPGn&Se>Ml&}
z6rH6bA#mV!>Fn9ur2+{JjdlH{uVy`{D82U7^&7i=`mOf0v)4M5@XYNG&^)wFeDAbq
z7MAPrY}5N@Y(5w4xThmS^0nUO=TGOYI3#hl?{L16i+FXAV!#zv)5q&S9CLbZF0dx8
zuKm;5V9nsjvwI}PrKM_Sv|Q@D(R%Kk(vE#>&mL8jJ?yPZS5M*AI%NO0bcW{ES-f(r
zd=(bVx`!s6p5DFay~%$Dz17b?{rz$F<(%{xJ=-jw%=qcPN@>2*!OT;Q=FhhuF@3Y+
z%bQ!L?;T<N<@6=*ne&>UCsyU|>w}|Ln&)3ylDTF7(anc{T>bo-H><9-?o6Iz<Amn{
zS&mY{Yx~#D<U8WWDZ737i^A36AM0%@<;uTC^Bh$>E2(;FtK59w8Tsahk7nJ<IWF$>
zdGC{V=9*f9TV~aN*OOO!e<L!?X7x*^X=iGlDo8B*VjH}!=UL2^Z@N25_-;)-v36DA
zr^i<dB0uOnO5c!aX7<3LC)v(kZ0csk`=5Gey?pQgHR+4^9#ieT{sqfCvU1GV$gX5!
zO7h#L{QD@2-`%fqH;jbb7hh2@y6gFYw=>4^-gR-kQ!CdCg}iM@RXFOs#;3&1@ypq>
zOK+Vie!ytD&Z={jRz>8qzZNPASJ^h4E12Of=Xois$!}WE*1wk&RnIB?-d(*Rf_*~J
z(vUSFaTOiE#1>6Xj_MWRzh%1YmZ!s2z3s6advgCOcRcnM`PVm_^#uQAC;L!=WmadF
zcPu`u=fQSA@7|XWzyG)t3Or*E<yaS_5^D71R<Pa5<&%XLJLg{6d2If-Z6;N5N}Gh9
zY^|O5AhYd>&Q`H@1I^9Gav7Y)oEz0uSJz$fIwEU3D<kao1J6I*!m^Di;S&SqTuolS
zIArEbzF?oYm#i1l4(Hx>d@UaBySLQ;>X(m)Hp{KvP;{#F<jf$Ks-<tX%gj<LaXEBP
zCH&j<e$gwzXPkdo2Cmu0-|FucSbUUivwPo_lW#hGN_XU_&n~?5!6(FI$+P^IO#MNp
zCUw;?7<E02iB2rt;TmFQY}HmZH~aSb0wJXf4(kN{+kFe86(w3Vw=SA;;@wHcgqj04
zKOH$SqgCcyU{qINg7F#m?T-$oi8-@HFM7t=$Fp+sJw?TxOn*4MGA9)6Ik@b1@tqov
zI00qd)t3*){r#Bdy(aJCrD>0=ZJh(YdA`$PVNxy9==%1bfsHevY34Q|(daDwxc4_B
za?1OQE@|7XWesOL5_IIm#}xP9u1zd<>Gq#>T$LsGm-gTM#+SCzJnp1W<g)9E6Kz&j
zZc*`m<he-cZ=GtI#CG0(%cCcBp6bVYofCc)@JcXBFimBFU(6RL*XZyaKdV0-oAakF
z_4Xg`shJ_)T{26u>~3_woqY1l{F)P-{sE;dmnYpU-6zreL|bd>Y0WE2JzcpTFP(C>
z{Ao`<Jn{I%*%@rR?$&SSXZf|kqW`MV?z20+XXrYt+H>=x`nSmNccpr6H=k^?PT0Qk
zN0f_Qj(reE8tb_wS2dS>7I}4NqD0<F^>x1kMUQAaUe^1{TuXks<L@oUS&TV1Y3ly!
z3-!{FkNVtl_nmw6Z4p@pJJ08bB0VRlbvrM8`l##8%0{`+bt2PFoxOG>H>A|DMAa{|
z>qpC0+mF9RewqI}^Q2Pp(@M!_uOcV@J(790c20;<^zQYY)3(=tn0<^t_?d!f@`Yy6
z!!w>-s(RM_d)iOlJCpB!PZsrgmL%6YQ@oV@P50TtiBWc9KmKg!+IFB&*X&GCPEG~a
zwVk)WbiJBCkI&1;Bi-2hrn>LkhdI%oqfe~6+_q}Ijn^}7=Je2I*0S@aFu0!X5NMwI
z^4+o8s7I1>p3M(f<26I2;NZl&TUpPqKlC;@zD!**rfZ#tUszAwl@lD>3+~vow2H?>
zu4-R-a8v7~318aQY~3U-^mzLuj#btBDgw_q7HO=vQn~CeF}Yl5m--p^zWMdFc5^hF
zt}&+WobYhlamO8RZznxYP!fApXFEOa?}sVEPt^Wh3|V%#NT>c@4tK+TkDrsiT)*FI
z5g&2x!&f<;mx8w^{Jxi{yKLX0xY+*;N4vEPcXa1Gm6~>vrQ*)7Nfj({j7Covl>f1x
zCmMQ#SvGY?X@=OYv=kS;39U~vh1r-^tz5VFyYt=7u)>{uBH!g^eW_T#hvjmT+1=8`
zOP7|n&HK-=T*$|#b<T&oYO;E3<FZUROf@#nEi&$7njpA0|Go6T?=$pg)bBZbYlY(M
z$XUNM7hGMrm1}C$X7{D$T1y2yrxZ>$R&-S<I9z&3Y&(mknf;SP+h4QhHKu*q7rXGP
z<e8*1)1K|P_GJ1zQ#-z>Hm^skCJMb?f4ShBPuOHzR^8IRi_bh7!}6A}7ryIQ`O|mP
z^94Vzz5D+1%f|!%8T?$=ZB)PH7M`{;s!Al{_`bWUFP(0mYrLiP*^Z-O?&-K27pBP_
zI;S$PKO--dT|Ma4)Y#dpm<{Cme=ju1kJj;Z4W1`=Xa2;;{}~n}Pc7Uy)pd<K!{kJ_
znl-`-B^82N-`>CfwP9J@N3R9-Zg2K_e@Qycer4P1l*fH5S(iNI>@v%Iy!P#4ukJqu
zx33!LENuUG+3E9Av77*Fg{>DGb};9zQre}SueYO~b(`IWP#@7#IWHvBrd{bcc&fNR
zGWA}z32$A+)%dy%hvT<Q3tbn!a%bF9$J1AJZ|T~0tUKO!(ayzc*Q!UWlnPxhZ__;V
z()x|Wnw+@aQ!A(56w{bly~aZ2dzj4nBI!1!hJF8jw<o-?U1yVakN<@6&A7Cm)9%cS
zEaG1O-qm7LTKC0!|K48b2rau>t#wOw-}|RsYt<!o&a40Oa>A3)wzc7>uUzj~x&G_H
zZ)FEKTa7Z_d^Fcp&h|fKap&cn>GS(qpC+F*c==d!XOMi#9gQXTo+ug#Z3w9}|Fz}g
z=hzRAuhyT42zu(RyID@-{E{%aDZ;i}*GuuP-=^YtzQ6AHzZjn~fyk;v|JzKxJ1$Jv
zlw=lNyL9=F2@{Spzg9GnWXe6Blw*5ro>0nnp0^6|3rpWVY&^p3u=reCKKJIE6^<K!
z^9JtYyMOA7>8}kb9KmTJvQth6_1vy;yR>QtOUh&KjDOPA^3{6^&m8;S>lY?hHsAg9
zRjnM$$)9d7`eGUWR?=i&+3rOj1Anz_xEeUyVO{Evt?o0;6dSrP#S|HEFnIK6D6Cl9
zY}L;izxd&&B!z8B?zXj_3l6UGU+GhI-6ieJoCI+Jn-@DbWGrgg);^JC!y#=prOUU=
zQY5E{FYu{-oyKXt(8MVzV~LdRqu9AmOYdLi;*5^W()HZ7_}8`xCwKa4TBp>1Yfp5q
zJK}fv?!lbKx`h{dN)E1L^w{{~yPf5<G>+=rj(0yM#BN{Jv1QMMvSUxT@2}onYo2(w
zblTG5%OW;MUC&LKyZnqyP50pse?D|p^0KZwXl`0%y6kZ3%f7=sx@Naq(w=j>PusQX
zUqr9s-lh}3teZ359r*kF*v(rOt1c;?S<~&@?W^f5aXf$8i<#nI4z8KNn=HzINiFtd
zW$w(#Z8d3*lL{v?*>AB+{=z=%>aQ(FPZn-t-X8IA;$Qv1;=R0Q4!)}V@+;ct826?W
zqp+SsrOR%oJGx%=m34Zwcb~Ok)t=nC17R;6H*u$mrMz0aHGG$N>vqQJTeg{XX06ET
zTzab}V)=iDYX;9&#uaIoceZOUF_&2qk^kwU$>r^;0hd{J2(1uGH!#<((sW~&Pro@Q
z<9X@Qvbh(J?EY(E7Q*Vc%wwjVPSsM=XC8V=!4p?7noL(){jPX_Q;hvIky9qI>r-PE
zzZ9}}j5~eaC;O(+nd!>C{kuL0EilVix@h_1$DdUXdK>ySPJXU`Q!aV4s{ORhihGKr
zZ%cgUxh31bxNvQCeLMd><-lpr^d0M_Z91#7;&MiVYk*)M+nI+YEdg5t*{&Qt^!EMj
z9~-_tQ;b}qHa%2sje&CW#ryr=%KM!CIt|yZQ)~Q~xvFT}{UZ^kpEy!2U%WaS6gfj(
zb(YK7la7iTc3wYOE&Iyzht1M5$tMB_c20?SE*9)}>`7qDtgH`0iA#L*T`%wY?)RUe
zecKLkkuzdnlwY)-_n!I6hCQ-!m#^))RW(iA+aB_E&7Eqp=j!U<ZALrqOnKa!dabUz
zRM3s%%fjv3oX(w{)}1tWcjoux!&kRSo{eoe=cg$4e&a@^ou`{#?@PY@>2O)|#LszN
z6Py^6lOE45-zHx4XT#D@yiVo|-{x+3+ro2q)t_+vyYCMdUSl>)-0>uNYwe=2jt@?^
zFY11O{ZUV1P07<~H<gMmGl`u%XD@sIci4wJ0j-w8(wWD%$8G8EiayeQKd<w9_18l)
zyWR#rR6A!e+doFdrtP22x<h|kJ45<(1Z^Lb8QZw$+){p8u5xIa%Sld-$lHb2W~N>G
z#PPG^if8cr{#=6yt~8CQau&jc=Wp|#v#_qIm^gj+yF*_T4Rzg@-BLB#-oo{q@A4D1
z=0#F!+ybXQKHR>^;jp<;bdx;i9q+aG565ULT9;08UcBu}-GV2Z&imP3+0%SK?%&_S
zI)|qlBBx#rUfk0aZEnb4x;;|fQ1bgm@uMCcQn3bWwsGq6I!^bP`{K!QCEe*g4eJD-
z?6==*7`NP5yx64o(oFv9!ex&v9_&r{+4|}7{L=V&R{I=tG`1PWPUZbqG$SP@XKvxs
zC$H4zFuT0Jf4+=q?V2z}y`8f<x><QA&T~4uyJ2mNRP3tCeNmZhr=smXE`65m?_Q)+
zyli&;o{3j@ThBVI=ilF6a;QUO`L@uVb1RK<rc|8Y)Gv6;$?Js{hu%CEova-@kEI;g
zK2iG8yv%cpLKuoHcKJm;h*I({pX_ULsP03N;>?gDTfGG@jQE(V-!yzy$`3KzDjfIE
z;?dO?1(Wp;YQ>uAu*X&I-El6X_eIRw%g>+2v;MKj{nPAk@$ZsklS^etRL-)Lcboj)
zefE~Q<LK7>RdvSomqIs-9NktOUo5wB7muaenX>IMs&08D@4{v&yj}1ke(A4A=kC^A
zeEZmZC9`+G{PNk!$r{nx-Hu8kKPLNHFaLL0qSpEM{4au4ubB^?GMUAz*R9oSx752o
zC~;!#w1bnfx9Ez;r#bH|TCenLkD>hx=IseT9lP1}g4|{a^SP@%72agI%dPg@zgtBg
z<8C+?v#sW+Tr_z~ijj{)l_aCL*i+X$Cys8v7~9)dD}H<bSn#u;UiEKF^uj0eedqN&
z{AG5|_sXWv=c^{|ad}d#EW{?Se{XaBbVplW%NXVU<V!o}E1F)|tM$e>Zs9SRnGEyH
z?&g$BWlj)xX?wlJH~9E1wWaTVCKs|jQ9m<B$5xCvWVP%0+UAU|J6>rk*DGG<T4|)#
zcSmw#obvo{5%sA(y?0!mPrth;^U>n<=Vcl8dY`#8x7%uV>Rm_E8%^(5243+mS|oq|
z;eiy#Q?nIcewqK~@4QWS>KnFYxn@jQdu#dKuSwdxfs5uV>tBm$yY3@fZC6knu{7gb
z;ML8pAAW!Mn(&)hFf*0$#OIehjqU}VZ&$kM3Rj8ewqEh&d&{c#=R?$lK-J9>D~!W3
z_<}`lp6y^M*wS&8_4L%eTSPBE3Ki1-T$tgtG-kf_m8*<WIeCkcQ+}V9KGV?JzsEB<
zchT}KtbSrE3k5@bG@C-Jgwr@fxm4m7g=Fygs^2L(xa;hqonPG=O_gM~`yZIhAhh~S
z_}Q~(Rt0(4M6H?eVTqN_i8*K2?F%^Z&wJ7IwD&u97XA$n+dt8a>DflP_E~!+p6;+b
zF>kW5zJ=56NgkFiO7EB+Ej^jpwaw|-w%>kXy4x#TpS-Cp`NRI9a{e6?pY<=j<X7Dm
zlS@y?Q~j`YQ}omt8*BZe-KWD%y=P9olximueNMv6^@OYdXz|iCosS!yR5O-U&Zw*S
z@^Fq+Mbf2nCKgY3s606S^!WShem^e9nQ)ap+1Ya9>{Z7qx8yVYHbHtjr^h@=z3&|u
zE_QWtUvOF#|GkD{`6YLL#;5gNjjM^7$hN}NvxjrVh9?^w#l8goICa<J(R}O48+IO>
z_;y)($lN?xGgl2Roj$F1-(K`{_wt8)o0YNOCUv!x+@_ArvUgA9oxj&E(z1Bl!!6E|
z-`;9RT@Kw4eyaQF?IXN1wwz5^Sv+U{1!ilzV;&+86bf{A7f6e!%Y6K>rQibh%>2rW
zRXINndRhj>7sm7liaSLLOl#Yh-&nCo{O88w3Q;bnwwgq)^qsGgd^}-hNcY)Jt6N=d
zEnLqQ#BnWMrX;Y@d#%S!iEYwJdXvreWR`Y*tTxrVF*~Mb!Hut{t|e+UK3%D^=}M)t
zX#1TBDSW~y8c%yW7yNjAXT!F1))dAsjW4!Mp7k=Hd*OD?gAdg#81~t;Y+*OKcJ=av
z^|QGineKS9YAL_Y^B1N%?JS#4e^B80cK79DM(JEZ#)V>^a#ty3uTcNbptJFX%59$L
z6OPL|L*`Y?P0<q8&GB4hwDWgQ?F@yxx4-<;)jgCQceY{Eyk~!B)k^<oxHx_BG`G!6
z=IgZ%Uk?$PDSca6FzoTHzYaHlWFDDoa=iOh=}-POYs*a4*(6U0sO1?xJ!0{wUy396
z$Sn8O_tkk9Y|)dwyI=RzF8$vdi#9kVdH8HnebaxlrtIV6u3PP*DSo*t*72RT@o+i)
zSSY$ck~67$a+h|>yRX|hIq#(Ap1l3^)1f<EyhSS>3rY7YPq5LQxUkaY?~DC)qI-IZ
zm5L{=T-)(x!Zz32{Sz(AW9zb84J;O3iE8Lpy6nKD$DPM|Uob|qs9yC*;Hwo;-3n22
z9NsJ4xuc-3(AU(xe*ZS*G}l9|Ct1aqn=UWUZgP6K{pklM-_rM=kHnm~<WT9Q>0GmX
z+wqdEKf10s>6*U%beVOFqsmHQ`95pL;$Z3G$q6yuFAqL(evxOya^JP0le_n~|GOh~
z*+19Gt!ni1@OZG}b#wJG2Pda#AJxA6*!uoH?@^9R4lytG)|5TkDs}l)-p+lhM?8b$
zmmlXi<E*hdu4t{FMxvPSS&_u9hsNhFZg;)>=|J*^HK$k?{JNl-ay4+qndUsZ2i_ZN
zrnk*n*ZnB`+aHT;PxePqdEGNRYS-AF_?^#F`*=r7o}TSQUF|?_k*GYjg<X1)nm(Og
z3~6q+jTnzJUD7|a^8VAN(1L3mp32IXVijNWoN#G9deb_$)W(F@^n1cn-INT)BwjIT
zA>}(CEOa+KDdIjqk&8=h<<HFSQ!5+JrnYM)=~W(l@$Gx+dG+b>{C^A9`Wt-IxUR6{
zs`&E6xtnMF-M#&z&^@8+v33!f-jPha$J4a!v%ckjIeXl9gW&eQ{i+AAPS{ZS{POq3
z-->msHRtlqUUF*M_aMIdP5CDmRZkZCr5lvLfkQehXX)kSUfpZWDGK{8oG#wcDZ6A=
z#m{i|sDIfBr)K&ktxP|2J!&TJ&4=!7J_WtL{kwj0)^Pn)PrLk~WBsF}Hj_G^{PB5a
zqVaD_*;+QaINKK+qi?c%N9RgM=k$E>b5bnZUcOsm2X|=Ur&KkU73)kZPbh8iadSKP
zUDwX_Vf(3QZf$m>TeTi5Caw1M?Nv)zwdCE@89cB2cD!uZFimRt#8Cb-o=bN<c3pTw
z+L2rG^o7%v{}~jnE=ZHOa_LNwMDf*it~=OF+TInI7=4a=_DFaZXU4Pne3F;nf4oyO
z<AZl$Ov~eQ=Z#Yr)-18otDIr7Q9R^__xAd^JAR26ED|?ldBz#?(<Mmx7`IBux`4uq
zTUrk6R-5+vVXLK~#^u{p?p}v^Ztt)z3O(zRe_`@b`=+`GnFbXp>w7nDRUGZ!ZDQ}Y
z#7zD7Mni>z)1G>@2K9$`hg!Vb`^0Dh6HCf6YrjRA%P)7Vay+y$NNr|X@UyRWYj<WJ
zaW59zp|l`#)pFr)wI99)t`d`a7_v;*?{CZA8GCACIc-Y9!=t<Jzl-rX7<5$3`jXZp
zS*r;iTG35zzY|;)yN}FvbchZ;Ys$+JSn{8NPc1RUH0VXq+<Pmft&5X(oIWSBKv7BU
z-W}ef7j2&Kb*)```_+Narl{9!XI2?=@vrAI-KtpIdF7ku1M3KOmskPU%UieiE?rn8
za$sR<bb{#%!?a6xma}@XuCu#i%E|CTbb-psg)dj!Gk>rr>FF1#?QgEXY`&WMpTQ+=
zid>J$cO_wy%@vn34@rONdmt_-DjedXckPzMXW8QA`(Mrs51;+xVEq>JMN!Ka1yx)9
zvg<kV^kGTWiNj}Bu3NjLca_#FpK?F%b@RBpYK>>i+volu`f>3#iJ1OdLcsxEv%{Bt
zR_mC;KBMg8hY!at7u>6J4$rVFp4n{ncGp$SXAz3x6J)HTHr8<a+P?h$NM9vRIV}9|
zyyR<fd0|Cs_iAZOd*qz*HlsbZdYy8SxADXmJgdF?ryBidSn<N(X3pbdxznbn9ortr
znQv?IW3p)IoP>vBdu4A<{AP9fUg@E$(|Y!-nN<F!Nu{MK%Fd#yH-NkJgredpN88IM
z*7_#yY}S0%a3`T5%Ge=S)aB3PZSz+jW^=OH9WvK@ZT-d58=f3ru<i0WnN0>6`oVX7
zE5uW;9l3h&{cY9>cidji+%%ghL~fPrCDwyE6H*xOTvXlOG->TBKc#}*q2C{+Xk;ze
z>}@VH;Z0i1#2|5{T|Uh%d>h*GwpeMX?rMCL;{0&gcF}F8SH6r&{UnfOA-ebdn^m0;
z&lVc-EA<60IC{F@D{Bs;Z09*kt!w8V?zuAMPhHlom#xBz3->Luxc~R2_>9tTGYoj<
ze{1>Tt!;n7YqRsW(yVCK<Ro$KH>#%|EPDNH^I@K-Jx5H0+M<7}{L;@ie6A^ZC?`|d
z`_!Tvhjxf?{_KAE?gwMJ_rH%D4_VE<l=;|5Z(X&-;;Bc>=YHGqZTnA`wb$={Pd<4v
z-q_^H_N6u_XTG_kWLD|Dzy8PW2h$BCJ8tHMl(RX;-92krT+dnl=|6)_jFYyQOvSE7
zj(ugvXZ&a2yZ`EKV=A{%PrF)_xrBv3?;eMIK@r6dzpry-SnO&1q;zVk>`VW>61|U<
z9NsCZt<cU|w!e1&YK~M+j}%Us_U%3=c5+DE&XnrZ%3ig+uEJSUVo}Jfr&cb8na?{L
z#al1FJa%vgd*{vvQSlwC#M+g+eJfY!F>gvaKIQhBPS!bdL}hj3{xj77+9>JScxA=o
z;{47%{@YaaM88Y#Hw^vHa9OZ*a%o_^q{wsMhlXlLy3e>Mm1lH+j&c3Ovc}`Am-M19
zm2OE)O3#m$z0z2I`tWg%BQxfD&6K~o<hA&euyP%>gB&$WcD=sMqo&%H%D?(%=I!@N
z1v#IEQ=HfDj;~`^XZ5S<bd}lp+xF0>PuoK;8k}8VENrf25ta95^1A5K<K^vg{~6|4
zt&wy0v*cOa8Nph-r~B-)e-UYk^WFRTbk8`o@igD@XnS2fQUCsf<;`Kj6BFjVym!~w
z%%x)8{b_Gmt^fXKsAD{=Ty*!4oVZ|-A(OWEe+C@`o^PU$PfXAkII;Ow%>74ug`?Mh
zJY3EBn*FQ4$%mtRADMG>#tI8PY4Ci!d_U*yHLX!CCz%!ew2vt%=?P5NZguyyw^3a8
z;0gCsqg2;zj+f06^QNseezueO$^LGZJ=570CQr?Hx9#>f>4wiqdb0l+l&;;if8H4;
zIYG=>)!r!ZNd(JPlaw9FsfV1xG9TW5eE3J{wZ?<Zq1<Wi5<xsNI}h%O`E;j2rMYA4
zd-*V%=W}drQfn5zv|2k|%dqj4?~<E4S=heG2C%+cH81e;j(<AYA1X3WIC|~2JA5rx
z`i#deoirC|?I=Oheed_yoW3ogtIjj?`Ksue1JX)L!X<m|9uCkki)@`a^@)Y4*H*_<
zy>SfYywhW<=G^XS6f~{t+_`L?)!q9aYa=#YeIsxtGwei3?e4o4!Yt8x61P;;?6`LQ
zc+~Z9Mv#s0Z|BrsbwOVi&S(mp=|1OhX;Yfzir?W5oGuq8zY5l6kDq3H-F4j&6VYuF
z?-Ke>T)rvcE<HJS$Kj<i+jDlDc6s~M>1S=k<@MJj<EHP`()8`-UoNrabpM-~0-tt&
z&zSXcY5V8TiCKJ#FDGdUt=w=|BkhEj?zAZmE=%Xlx-<XXVb9L6;8Kp?O4aA2^S9af
z-?S3vx>dU`Va6i{BiEJYm%DeLH)efUI*Gk!zPjv{=_RIdhw3i#@GEvnUw9R5;`vPQ
zZrhKZa_J57W&atbx8BN{cmCIpY^I>BqlG(yR8508YF+A<PA=d3j^SgFcX-&zKz^6q
z0y-R3jalyATB|Q_`21$4;B!X*f-kCXvi4mNbU1zbsq&#a0&O*Cu2h{_*r=@|W02c^
z%yC<}B$vQDfvl!oyuaHk?kJzt-*(4EVpZ_#m9?&7!92%SNuS$s`oa?%H9vzJMk`ii
zbZD=<e^Z?0)D(``vsWZ9-z)l+ld);>uE&X&_U^hcq1rXqdflU%Y42`K6>khz|5fv4
z>x-_HaydLF)z*6Sh3$1&kvGTu(W<BgjO#j&T-D>X`@M1M$(J9NPkyJl#eQ3r!~XR9
zH<y3wIj#H4@kF}1&Q8N+PpmIA3j6MB$#{~UthFHa@Pp0$H3`?}o_hJus(i)%$sdpI
zx%k4|yh^Fd_~j;NlhkjwAAJc3oG7_KC_d}b$_l0$wTKC?I8T@+p7UaUx%1}aJ|jij
z9lEy<IKAHb!{qJq=TDD*_RX?(oqNUdN@;sV?>5d40!pH9zc20Vdr)Fy+L$H#@%^?-
zRUf&#o_sL%=PlTC_(b4ppG8kkrysT3RLNbK`R;_VQLo~5helJm7dN=#od4Kx^PkB2
z$?jp=ShDi1m4HlH!i1fw(;8al2Z}d*+avB|cfEePu#(A~K3lV$*8`(6ZfXXpZZlqT
z!rCWF@T%i_vD-V6@8%bNuDy}8?NCx%tK!a?se7-dx4iV-xn0gs%JfaoQE7kv_K(@k
zY8qlo4^KIA?D0mm7rciGmTc1LWVG|kUZpqh$&q*k7K4|?il;OKUoCieYEqCY>-5+K
znG2NGZ~nmJrjTWAWEJLkv+bc-xOAV1_|Bi-jwlvRpDmwK-?HP6Rn10@@ROTXns`q2
z+018b=;Zv)@8Uk|cYAZ6>|gG%({4`SlaIT3>=vw(SP?Fr;Cc7$2Hn$npE)~Tt*E~&
z@%fvC);Yavn_I+|3Hw$%=5D)Z?xAbV$?7oC?r-glJ*RY6)jFTO#Aj2<H+NmfgxeK&
zT=q^Z3B7)>P}E~9+nMv17MjI|<jy@FBM`)$x202f{|g^Jb+hK%=bT>jsHZUs^d7Lw
zURzy!e0xXv!Se1ji8*U6q8qL@{b$f*;)^nSv1k|j{YCY&Ii9Ry*{Wk~#c3($>{u*r
zs<3v$jzt-Jch+p2x6kO*l|7H|Iu?H3rs6C5^vy|z8u8dV{&}pkBksNnpYmBH`rW2-
zi|1R-{@gF`|6$PGJW=6rQLx?hn{OsJY2BapY5&oB-rffrZx|at%QR|AiJZSPOJ6i7
z?RERD&vF0ke1yCs4)xgVd*HDv@SXLxz>_Z5E{fQj|NeN;<*@gJ!rR%+X@;$vcAs*X
z6aH(N)vD5vCkyuoNp3RNc5yu4bye2#<ucaRjWVlhozkBBF`Q0PdereJ_Gs`=H5MK-
zVNu&z&dEyIYHByO-G86=L?!#v_lLhs+dIEK_xEqRa+pb-BlmgfF1;D5Aw08q+M@DH
z&Ryd!=$QR$LZ|NKcb5ejbV?_r3ZIRWf4k*Vr?`Ci$0G%sG)~O9?X%|v>)PnUGmH4*
ze<tz<Ml3n-m22`b>yPIPIJWb$pD)x{|9d*;E!~+1*Hs;mDZFgHe&=>6BR<&!p7!&)
z8}cPKxvSWyoIF~6^WCkNX7x;J`_FTod6qD1aZ<FA^4{#--}b1kj4jAc&#&8&ZQvo2
zCY>7`xy{GuUiOS_DK3hqpXclE=9};}!1wIdu$)J8<}SRQyR_nM*~OfE*^1XEOpj)S
zo}9#OH%&FoYs;Z8fm2mls<wKz-&t{1c=v+VqkQhKtl#kcX9zR1DERl_Yw&rK`z~*;
z8mo#4uuPPkbgyuALw}8HlnblVk4~GrANL;KpHN`$ysMMxxobkKOr^vh_p}`c!vyaI
zuW*%QF^n=!i4!b1k@rX|=F8hVQz~9|oRKisKCJQKcA4X`4V^C*>AWmmdqTxG+<*6<
zy&JkbUacs6v8G$;^?wGA4?Bcj=UqR{u*tD~R(*EA^C#m_GyNqtdnM$wWYQy7Mi*KI
zURcN8D1MYVTtDQ4xb2VoFJ;d6{W?5Z&R8lh-dy=;9rHP>2Rq-oT@qCJ_cgR+_otYu
zXTQwE4R5ffz7*KZBJMf6Aztjso89TsCnn#q=M(R+nD0}R7%H>u()V3f*-I}8+<oRO
ze}4Ky|8|y!$<D8|dN-SVpQp*3z055xMad<^H~8$=#02q`?=1MMwltrzJ{5jDvwBDQ
zmVJN3T=;^sdRxMmcr<*TcvJ08)0gu8`i72$uO9wMk)8gZ;dX~fUVDss3a^@Lm7&+x
z8AXRQx20BT>QDANegCMpfX?}q`!nV$&H6c0)clsw>ooOum+vqC>2}Y&HsYzrs^Aqt
zx6hQ+^{&bf49fEqFux)x{Xl$jb#6I#bF9Vm))zaxHhc{%vo)Lay7y?AWKY#Z;hf|B
z3wknRwx(MDwomI_FLF5CAk=%pQ_(1yt`nsb&d#1$ae^hb^y=ry3ubXGnmT8y{ocb4
z#dBWzPE?xh`$~aX<u<R`3Y%Fwez(p2`hoR;XXC^d{5RBeBQ$s3R<`=MEo7&my3yD9
z+-E1uV$$n=!29^j*ApT?C)}v2wo3i3ni}=ge$6+3x01j`+>IN<a^L1pyM5>Q{QMuM
zx^>g%|7XbNnLoKG&vougso7qYdk*_gI&sFTZnlfQmF`TdxfA^7p8WXOd-=PCtB<AI
zJf2dk7xZdk^1IoGZnxNUH}cK<Yy0EIsy$~}lS7t1XNzrmyg~hzU+k(KWw$@aKWyob
zYEkvdy>@%`(RRL)w&OQCB^f5{P|{nqsQN^(>p}joYo60nc{tu)nQN@>;<>%SN22V@
z&Op%^7w=r|ypP8Vq&FUWQ90Yd+BZ+&;*%4KH#Xn6=OcSKZAIt8=-U&1Jh-~G!QPm4
zLYnsGvop3z@=we&O0jNfUfGnnT-N&b--2onx%RMrkCzW+{%%U?=;e6(fxpgb&1A(h
zuP&UIUBunz+{ZcDi0>g^lzdl0)Dr!wi#rNW1;#!0?LDO=KK;|-BE^FAQi13dE$Oop
zkFg$RN@=}#rF3hsk0i^VpJIDG70;9iFsVJ+zV(fN-#n|oN7vUM?mK;Ap3`<Uc}2Dr
zpI4f`DEM5faOl;9&9_cB_&A@){K{{lYy0EalY<uTDy5fgXS-&!rZUKOuh(m@$7(+&
zW@fC;tBa~Jdo=5l*UmZTPQN+Y_0T=#Udv7Hh8s<iH#4TK+S}d5`1_xZ*oO)3=R{^5
z;mX$EtvbDD{+-l?(=Kwir2hJ`D1tGwHX`mr)`B^U&IX%p51PlH`$g%tE_30LS$ay^
zZw3B@_&Z8h|GoC0F5sWJboz!2*?kLA@@3jn%~W*0?=yCjQu?<({Xc`4W!SZ_;}J*h
z7Sv|SE}LnXxow`1H$!=eqPHXaKCO_X4Nf}e_UF%>$n6r&FXvS!e_2$^W{35q*;O{z
z*J{c+D{paGFY3?#PtM|=!)bwg$|;G0GiI9$A9E}a6cj3dH|u5lYMTRU{Sk%NPFZjA
zox4%6(EaqC69ENx7v|~)zB;UZlYN^epXihON3?dm{P)I_T}>r#p@^hf^TZt&Iyj3y
zZV9!Yl-OfB<CjfS?b{CwX|G+jGxwhS@UY?y<GB>S95<b1I*J9+!gBUT{_{@@%ceN+
zRxMPrF=umN6xg{b!SRHzNNZ?)pK9KtR|m?M)%KTm%NXiDJF{oo-E{(MLeEUxIsIti
z+Q-t}?bdFeoL%4leCT!9n^{SE^UQ=No6JqD)51IMEKt3_{YQP6a)?gAmM3NDhubfG
zyz|H*ypY%Q{q61ZS#|`7sdYP6S?tMg$UCuh+qUzk7YKac_SEn7r^BfY*P5rzIlZWe
z^CtTyC!5x@wF|Qsy?poS+o4&GCJk>cdPGimzfm#wlR!?*A+05Q|1+F#p3?G4vDvZU
zNb=bfIi(Emi4$Y0GMaDQJH7IeZ@{ZcCT7pJ055&+ICU0*N{-w4rB`i#sIA-ht8r)3
zY~B@mUKVGz%Wn4yX_vh8P0D-9Nr9=g2j+zyGR@hQ%^oCaW3Jr4_}BC$&MT!&-+T?a
znJ~{T$9+#!+v&z-D-`Z;{qv{r#m`Ca{cMif_hg)^lnaT<i(m2O;u34$Nk47g^s=li
z@Cm6{SmV21{z&Wx-^Y$Wq(x30Hhj?2oZcCoyFcjfkAo+oL*&i|#O5#EcX>-p?A(f%
z?|s9CmGmdHHMz>oa{N6}vnVFV!!p9P{QSpt(g7QJTvu5LS=lX-;ohz<u>EFzjH08}
zl;>+^tIcJYaAbXYBI|axSr%6H@3%?XJbQe#|J;3_nSAbx&U*>1ICy2RckacR?v)3G
z?<9H$u*M2Udk4f+-<uiR5jR<K&zS>frPiETbymYv_mF~E?xVdE1jOFvntuFMxco7T
zrkt$ZiqnQs>3P9=`;$}H6Lu---QzHOxkC23cs4uNNtO+K(o=4*_<c#d+{rQ7Kjx+I
z%A!2;Wit{|rZROIS*;Lqa%L^Iz7|!xPhD7`@sw?p&B=a)cOMR?&GYSyV|mS9x^mH~
zj1HytMTyEqRz)j4t>V{2O}^%vdhMw@_uKm{+v78|tg{z>yJ-DjgPLR3lxI&Z&itDp
zJwrt$UvR?RRoSav%##q4F$tNoCiPTv@ykn9mQ$IMTrw9im#9bYWDKlcx~#jf@Mw+S
zGxe)1sm|>&S>9*Q>u#5;p3!k8$2-vUgZJ_NyLr<3if4Bnn)t-;)T7A4+gz?u-cD&Z
z-+$D;lOOExK`v=O|MY!wr`N<qGw%GAn(@?Yeq_6py!Uqhd13<ZWS^{+o3HI(XkC$*
zbN8V~%HQe57oM#5pRz;brrKR~Q;y4N3Nxjnm$dX|WUbn@cy{(WjZ)JaO|Ny^_iSc0
z+ZuTKYsYfQ=N*eZbC(FKb<DjmQLrR_|KEs@oL?mWKA-!aAz$aDclO1&@RJ`?4hv5V
zkh-jNbM4uqZn62*efxzUt`uu6zuJ4QdtT}e<9UoL@<nY=yk$N!JBQOX#qXV7?8^1|
z3@*o)hTh1CH~M08a;{PK-GHlAD>fdxH}iByN640VSp(C@3a#rlJDs+hJ$uSyh2O{J
zZ^Y)$$ThqoEE2`K^vS%oFY(6LF5WcQ=Tdp(=^vp>s(goU-@Pw#XWQBhm$gHrS~TA5
zXMEoAxI3xQDE3+0`Oy9G(jot}H(p#|@$<ByX-;|Kd3BezM|rnBN-nyF1|OI#b4I3b
z&PrRG<g2fOoIZY=c)XN5httH-)ip5wd7q5svgeJB+RLYW@T+?lc1x^lrWSwK4}KXb
z=9$Y<|J?PScec4YO*#M7;bW_Fc&aY$*(-bU(>$XLk2!l^bLWR#y7IR9gtGYAd((_t
z)7B~{CH81{B!5|7aLa#I^~bQ87mpu$Y4VF{UdLZeTi-K17gwn<GwtvfV9d}pSNhLj
z{xHbpeg1y_{nF3WHlP2>XB7A+`Sa9MxoNHiFN|-l<Pmp0sVVx<Uih8H%Jnsst3SlI
zzp?pRk$+rr)9S05Z%<`ju-AAqc}}jhYl!gs(ETD;w=rwla*Jx^7_H<uv6Rv7-sLk3
zeoCuY@9!`8vEkN3hrOv~k<FJ+^oyCKIUbi$>rcrOR1`AG5s&-P{n&czi8RNyKaEPS
z%3~4_ig)Nd*>-=`?=KfVbpK>X)jGa-s+pxz!hwZ*v(0uclrIvsHF4oDW7lm7)iL*!
z5Zf<v@X(Dry1Vyn^q+pVP(tf;<Vl;)>y3SD-)_`TKOWtpud@3vXc4c5*75U}S~4%*
z^1h4uox8)V@T2I@N31uTw=0~IpSY=_^HF&zht3^Q@5|fUI1AVVO-)Y<J}d1t%9*h1
zVQyuM%f5H_Bt#QbO0Ap=kEB?A-1AakXJRc!)VIq#XUvixt`xBpX0PBd=XxFIEG@8Q
zxBu$*2iAH{o+i1%V~XyEuw3J}dzVg+m^QtSb0)WTP_L8rRQEZV-+L;#n_jmyxp<YW
z+xu_Ie}?XthvZLqP013B$&yaG+H$3^qQ~XfMON981CJC#eq^X>%~P)2mhxb-f=x@u
znW?7lZ=d6A(95t|a_QdG7|8`ZJ%w`~r|{=YdtT&sa{a&4%nM)mF!gNbQ3+Ms%5$e(
z_S?xN`{y?4E!~xUH#}b`==%4Dd+xuOTZ3jQq_Ule`@H1w)6_rPy<NSxhFlH0YkKsg
zcf_~V7k*#k*~;S+S#|7&nB%-BGCljJx5-c9PAz^JB)NO3z)kgztvhr-{9M0Zbk(6#
zZ@1m-DSD>z;E#<<r;YU^ovl-E`p=%=8?Cu~+ku*c9|a3P{)@;@_-Q!fSHKF(I4L`4
z=G$MtI=ChD3u$H7UH#(XvO6e*@$K^2d@tu&|LNjte#4#Cxb*7kkX?&Imc@ys8=dYm
zyFRsX`?gyZFSu{VnjBpj`|O>$ZtE+%gsFauryNs!&mQ)pr)UHB?&sUOS1xPLmy4KP
z#>Xjgd~SQf%n991PVyXG^)Gwal9~;_MDo1td-<+1(oo)Xb;lYj$7MHDZc2P~xTAJU
zmUqJz_du1nr)?W@m9jERnU!pmjb5Id&e6isvFGld;1@?{CMa%JTfyleow(<-5lhG9
zkG#8n)c%m#aP5=C{I`lP&h31ufAOzg2v_!{)mMUc8GPj5JgLh{{-Fx9|3Asv+U-{9
z){mz>j7{%4Wox(SqqXMrITKvBu{>8QxO649?xFL$9p8*<TbF7c7JD(N<51#u&FOhP
z<!iUe-g>q7^7a!u9u#sO4JzE}VtlTC>&v+*PWsxuc?$lI)6|qs3*MY+mLBluc+2d<
zju)5P^KMMYe$aVeSV(N<%Y%nXSGFmfxb>bt#buuPvRMzk+$3gC(Q)r8kybP1^F6%j
z<L!hQs~gIdDzrZSvFMFWUdr3}?3HY&&fKi;iYH!0o84i2&HC5ttaz*lyNm1J2-SoS
zoPn=cW0qdNRdY_(bsEb|CRU-xMUCrLt#DX0pO3Xq^w*)XHI_XGr?)<vljpFbX8W;+
z&+msQZT=@CvL`)e=j+b)S35&JzXrBi-ktWVz30H(q`R!V!4JGm<geZq+uNE@8?a*Q
zBdzl*<<bJS=v<jC;CspRNxtj~bMyO-k8fHht$kwMruF1n%&DJ;+M|~pV{_tlQa;9)
zrC?>FdOzgte+KcpEyg)|@yuVH7DgymXRf$ioxjb!pic1J-oL++pWj*bSuSg9<<6gv
ze|x7L`+R=stEWoBXLkB7yUp61!V;C>$$I^$NNDLU?>@cmBQ?Tux0`l7$}60_;Y`_!
zwZYjzvnpgtH@<M3XrS&L$}U@b?|anVJ?v6DUhAx`X^;)pIx%(f)A0GLX6)9Or8oVQ
z4s*Yhdhe}|+#BK!axK5QW-;3|6~AK^8<RS@bvke9$+oOo`N#H#XrTH+`}FC{*%w=B
zp4z*qy27FTY?h=_{qk*wm0ekdy{o3j<%!xauYLKB*QUeciA~zetXb<;FRxRZvCeje
z<Cn%6%Y&E2drvMn^~hKJ_s2U&(-ii`OcmYc<Z?P9b)lGKjt(2E4XdDaqR*k5jr+g!
zoO|3WQ1U%`*VoR%n)UH@`^$7aT}*wA^{-1?7~btSdRVb`kN@e<`=2h8SNr$ii>X$A
zWccFkVyC@NXUZPVzuNw`BcW<5TM6SM#$&IKIO+<<KWsW>!Z`Jr`g3*es~^2AjEpkB
zGEPvtyKTaX7rVaw-f%UyZqv$_r*iFPPg!_;;ia3mm-9Vaoiy*k-L=P8eTu31KJ(00
z+cwj@r^jMmM&^{|W(7NibKX66?&VK)n~>-~g}!sn&#CYVE{ru6x>NLU|J2oG&E2|%
zJiKiWj!$tYD*II~sp$4oqDye{f>|%`H$IGfBd>78^318e6QA>!Cp`~wdMNbaec3+k
z-Tz`VS8sTqcv|Ir#CQMA2^))#+@I0w_w?i2qjqoj5>jfM%w01tb8kCc==IK_%llpY
zEH$Y!j8{6=o~S&@8{u@b@YmYi-lf;y#kfj$*r%60o*<pK$;P8Hw(#Y{?@861T|z$H
zTG=hRHvGAL=hXrnTJzNYGn{X_^4ZFnjWcJv;Wy_RudDIxXP2(N|L(}wIiWfp-J;8t
zPjySqKGC_I!Dq*;ny6KUK2jT&v7cKy(M#O##P24j1YQ2UXN4K~RFwUvygBqrKTR^v
z?BWD|^_)+&CvSXMF!`96R?hO|9q-rl1qgDtU6i_66np<~T};H0o(0!3zj6f2)UMC?
zo#NtlvXl3eRrTe4YAT_;<_2?j+?tW1CvKF(Qdyy}NK3sbL7H**sodkKH`ZH~A2$>%
zn4R->#}=i~5YSG=`=Z-ki*DG+`l4W6nal(s!L3K~Zg-T3Zg=(PmoZmapj~8@o47r4
zLXOQx;d7G@<bINFIqUpK^>JI_D%&id(6q;!>@(SJ?c7x2@n=Ei;<mlJ)lHA~1jzPz
zeD}ZjZeMkaZ}+jM-=miwZg?|eS<>#SzSp<MGURr99T0eY>&kzII_YT#Bb%T5RA{ab
zHE(FLJ#W=twA)2>d)%$M`qdnXtBzSu{1xxTA7ZO=#^TCNe}T)HJ!YTPe2?aq8oRpe
z<ve5>{MX{nGw+}Eef6EG!jJoeRctQz9O}L%u<3SR$=OY5DN!$)+HNWB*u{Ro=cMv)
z*45%Gw?t0nE@^zu`7qDH{ECNBPRohiuj(%B{vpPG&2Y0!Tc6tLyNokBwpiV%3yeGY
z*U)#0Oi)YV3!S6tvSo+ZIVvXK*<JB_!=y4FMIJAmwM}+ncla#o!*|F1ld$`wX<`<=
zY;wDjvF?<{+(Xu<ZU>h9vwwIvY^9MQPi4>XRpz#<PjKg57Jj5!S@EBNNBc>~3F)9!
ze%)^!er*4}uRM0?t@_8u%XYAL)taZwur#*`mFiYeypY{%X(eHPI_2G^BmPEF&z9Kp
zZ;>thBG-K8C+8EBr?KtxUOCxvY;@b_vOdqqclR2BysNu+y?pw<xBiO0V!XytiOp=T
zZZh+x*BPHPGn@I%US8x!!G`E#Hldmu?@kGRD7$`#9~b+f8<vk(eV1<EpW{%ZbuM+r
z#v`}(v~D|H;9)tj{?X1tj?)_^&RU$qQ=VTr`B&0sv$v1-POUDCwfkM5+YppK<EZ~!
z->DXj%Nw_`&5B;(8!P6&Ge6`?dq}`lhXOCL%MWK7r8G^I`xf7I@5T}FygOZcFGPp9
zUMaP^mAOutB`zo7yjoVU(~F|HVsSgzC-NR)TQzsRe>dxjxb)CND?BF3>3TeN<#_+)
zZ^H3MevPx<o(OeOaT4A3&h`7lhKXmI&Z_MRn7eVNld?$Gnj20RBg~lR?v!}_PrLBG
zq><R?kX4hT+pd}%{m%YYu~NcU#qq+d>xU1$pMSsAjPd)teJA&CEnoa=nd7&e^Nep!
zn?C(^;f+u0vR6+K^qF=z?p!hd)f3fXHs$Ke;>!+OJ6}AnenN1G<BE_7vlz=>6Z37E
zd9Qvu^9tDB|IZ-tRVMYrtjsSWaf^3qO*^aDap25PUXeqF`yR=$tW~HCxw7WS`-Q7j
zdoEapNql{@G9Zyz=TN`iTwn8@f`vOn)?DQ7{m*cEpU#PMVedt16>a^eMojeVR$tM|
zQZDm3&LuoUX6=rxAx93lvE6-96O*iRIm*DyC#E=T&S#G9wplN~Kjxp4b>NbW*h8Dg
zYjZk0WDjSj<dtj@*(v_XpC?=*dEaDXm#E@%c0X^u<mjn(dfi(iDEw@P$+Uwy$HF7l
zr7_QxlV9yP)8A9gce|dR#*H1TAOE;4F5VP;PwtFJ#9L!uOU6xa7S%bgtz6t;bYtq;
zRWDBmGyd4PP*dgKK|i^ZI~Qc8DrlW&*7c}8K2wmpv90Squl+i$V%7E6WM-b-t$dZ^
z@g~9a_wITQ<#&(P$)p=C`P(+7ovSTn$wkdgT2q}Z1r}}J@v*agvwYp&>nuBDjALHT
z-t;&3%&vFN%T5-&_W!ch=&_5+(?emR^H)0_J-W(gb-1nXHCwsVb&Wd$@2G{YH<-3h
z{<<9BKj$-7=geB*v&G2DvS&(}`>f8p*&Z(1%iBvY$J)O)XbX{^c7)-~3Lm-orE?Dz
zUT2xJp=EdU<6i~5GkUx>t2wK41>Si0vq$dQC6Nz$r{;bXH+;2l(U}WRjZT)F*(BZD
zcjKd?Ad~*99d#aj9{g2H?uS}loA;cx@g8F(hsT<WIdRvsTay~fCv5tfbS*_@c1(rK
zw)QisJM-($^1adX-Vk;2kk56=B<l&Xf8U?aXsPiG-u3eH;grMH=NDd^Q~JC+b?)uB
zqC@u65A8@h5RxTxki|V?v9wuti{g#;NqzfYKJLD-O04Kwc*Vl4hA%$m2uv+L#QIgC
zvgDt{^1$4Hn3J-?J)I1evX62%$+`LV%htYJe!MwfQv5_QUwE|PvAN7*(YNmzUVC``
z@KM`a+gr7BxD{F*mW1TiUfy}VrTZh_>UI0eWu%3b%sI*><(6F(RNwOH?lNZO1s7LF
z?^JBKBc1p7=|_<d|13;}&aQcqX0G^dk>2OCC!SopZx=sFF~Q)5Vok&J&5m9LuFD)e
zw^#qTTY6+x_MehNQI3y%wHC^0_wG8rjcH5gITa^!v0X3rY5r$u7mM$i%sGWw_EmE*
zTevkx+ao!fwI-Xl?|N&nLi<eD@{n0AolNdZ$wyvK++Xw{_NccDYsmH2Z+{i|EcD`w
zYoDoZ_oU-Z(9D7|)?-c<;v3#C>g!(KwD6&wlhMwvW+`p6CmdNY>4veuoYK=AE~VUC
zpDxdyYIyFS{RU_4PP-|_4Nq$(TlJV<Q3+XWE~?ADR-5bh^|)Jc_5T?x^2O~Etmd>|
zoSVcet$O(MVc)jTU(DvDx!paGqO;56@rmrL9qNB=f82Rxe&gw}bdxOWk{MsU4hQtV
z`FLdEtd6=52fDu4YfV^K7;k>G=hWS2%}>^e^;=Gl>AGsc5R>CE#YIL~zDRFH_xppJ
zbCfn*xLx2fukW(W(T(@i>ZW|kUZ?daRQv11gVS!Y`5o`RWfimb`f-keqH8rDdcux<
zUn{;=(cfG|wpvZ<o{{E7k<K9LnTm4{<mY|QXVezoRCfLE4Yz2Hqftxjwsn2?O4B<o
zxH0<i4i2m6^u3jTKKw~AH(Age&=z95J*(aJke=bu$s6)6gxxLC=e`~jHKoO9=J_RS
zCOykI{lsV{+p7z87kOrL$Cm8SZmfE}N=m=J{p9WAVih*iAKs2QkygoW$ldEPQ|6Gb
zNynTi3%4k#WQZ*)cYIg%pP^me@tbqX*_Dj9r<~ObXghh!mpdm~{`(`}uEy-|_uH7x
zZm@Kp%NOX}B5;nwSxM98{T)^7t?}h;^ZCSQsPubu37bi=Z=0j-xV`z-&LXb1{f|R0
zv(C7*W6zm#{jQID6*@exsI_>#J>=rQ|ER$p2VHkXE9-(=l`j@WUpSiYxOLIb%tha>
z+K98X=moqAsDHI<;p{Uf#CFwg;%@u+ro7=rR9Cx<JGa48-A${HPSw(IOGy>ZH|P5(
zB+I__?W*s=ey_GIlMrLqckX%<8Tw$6PhL%Q^o$8*2Uw=G3dZDb+M&IC`me`3WVEm9
ze)aC(_1WOp+I_vp4mx%_UFAJn`u^S1{G%N=WE}ZTLM2z(DzGgPQuVl<do9vfJC^T&
z?>GN4ksJJ>d;xK%J{#%S?B`Z?lR2y8DeAWBgX|NgkCRMp>8-eXhvQ16=-!X>SAD5p
zGi#2Gt&PUJ&?gK&t$f839x$svRdJMF;}X5~`nfu}H<?=BWO$7|&xqWrRhu2N_*mNF
z;!=l-Z4F1>-uW1-o3pF!BKwEwmz%`n*KmEYns&#cNU>2g%VXQOdrN+^HR(w<WHpyH
zEK*-|XWkp9YXOU7E@jr|tqJE_Dy74f%_<^QIPr$^v;DhW1+)VX{COu+x;ND<H!uID
zTJy%~H<ugoU7P)_HS?bC-02m+FUM5AzMeemL)@V~hgP{y4Ej3RuuJ3ZlyvtqPM+)9
ze<p3Nl*z6*uYKK^xpMm@mgBLN@22&+$|);xh0ETu_?{B4Hz)jVp6NsVzBbnzvsRoq
zG|_YFYlWP7>WeojILuW$>2Sg1_wEI;{~2!gDs4=%t=by0-hEZrgHm5};mXMs@6|bc
z^51E$y`!tW_i@*mb7E>wf2%*)8Tvvm?6a)q%qi}R?41RRPdF+ZjbU+heY5x|cifGG
zuVc>l{WA<-w^3c=a&h{i_1-g^FIqHS*DTqY7x`@GiPf2A39Enbh}~b^r(cjVr{$Bl
zMQQP+oBSF_*Ll2J7^d<jI(w~6@*A67TQXKZd;hR;x^(1fHCdy1k*Al=Pz_5u&~A`f
zpWODjOm_XMPK#Yizjq|_uecsL^UN-vg5o#Awy84wIqM1p&#AagdofFIzW5)D!}*d^
z`wr+F?3QylP{QNp9JazDNi|Vy&c#sikOw=~Phwsj>H2EQb2sj_MxSTr317VSN%m~+
zp70-*eNRH)i{2|*a^aNF=F)9*ZmC=R6qxE*KXw0VVa2=8b&sxgf2H5^IYoEdx5ecX
z8jd~r`9RE%t(`mA%{nyq=I^_bhTAd|8aWrmESR-?)*ZKi)2tR%63^b8TOcVrciFj%
zLKn7}zS&;!I#1WWm+RGm>#Ku$AGRMz5#J)+(@`GaTKnrqvSm%4Tf$MDIbx?)SQ|by
zJDeB4yWobGXVS7c>-Mhuv9<5uPI>bnh1}ygJ05q6%PA?%ogP~+l-cyPBaJ^e#Y-?R
zd*RvL9X>vJ@~wfTad$u5`pLd(+A2PesosUl7+B^AU1>jm_fkmczY4Bx2PC^bKC!&i
zvuZ+R*&G99>vr$o71{58ChTpF>~Je}NI12sT6|X5OQW;<?{}5`XP7qs;i12&VLAby
zS){&2ciZR)PVwt>*K<Ddc4OUt2ARl~2{{vXhP;T<-{8O`KI=}|d%af*iypjMX}EQQ
z*@D}KnQM=Cs~IOfG*`~?FS)hH)G1W;_aFQB%p8-MS4a9pr{${7+RVP`r$N;5g7@aU
zvfGZlnUjC`w}zNC*VlIb#m#&nZE-aovt9PQ?@?Ova9f61UzZ#AvtRqxXg$7JI%mp?
zMS6L&b(@QBO;9lB%#FxcbZP0~n|9ev$0OEk6KhL5eDPr4(oN|Vix<Wx3arpu)|s~=
z?%xm75Z<itbpbC+A602DI8)H{qMPMJE_2P%+*r$^_}zk2(o-2L8($Q@eD5=#X~~%x
zALnTXOIw`Va-*Sb*(2qa6|TE)Z~C~k;MA&OnP`pOugZ$Q%59y$Dl$K9vQf`pb=~*w
zx$Cr8Ynm?yIvZKLPus>fPonn8oRW{zK6uXgWxs6ajN7?41m5;*o-F0+K9pZO>9N_j
z=8WHW8WQGn-M;_waJFv38Z#+TzW#u2xeyD{ZC<vE9tWAtyDMa56jXMdaj(miFI(#L
zBF;Yx)%?c)*gG+7vRJT_Zdg{x@+CI@FE89mdvHfXy30y+QCODO%D>Mz>mLhjkXjYF
zsoityr7GjwEk3V4F1@=)P*7at)^FQ1Z38~LvSk~dto&p(UGaq2?f&fxrz@%KbpFqv
zqrCqc=ZP)f(sE?f=2ln4l-)l5d(&k<`I8lo?Y3@Lcz4}O>PvG~idygUxjE75JnOgJ
zKD}-39)qCCXJ*IrrKM=_?$F!AV!e;`M9W*%E%slIp4orkUHbi78voX(Rd1FolD>8-
zMP}Pa(Z_ei55NEUVCo!|GwT%=&R_NGzz$UtpI!VJNmdJ0S1z0TBk|Mvv;$MO&a3{r
zWH0v>sXJdbPJL_?t=DlSI_F4d)1yn)i(XBew(okZ#iw6uzsj<&{Qi33#$)RapDsPv
ze8t#%zTvZbMcF=EZYRu+-s`aR*sPBVa@U&v+WXlST)Gx3*x)LDD%|Sf(*Fz&LeZ<1
z-)-5*bkXxAlc{G&`Rvkz+q88*JucL$T>Dw;`(Xoaq16{syxJb~I`3F`Ox!&B+C!0#
zu5TZI|B;j+u`7ssod#P`wA$%{z_VPJpPoMa?$9R&i@Rz|wUQ@ho|lPqxm>hlyUd?m
z{kfatUO!l|Lq*}snRbPs$JcJ9`8+=n-S=_2bO*Qpym|45n;Z`FY!6s{qT6n|jf^Do
zo;8y-Pd(JFexbcS|3U1As}c=02i==B<}D7@7S`M@XmdSw;qe`@Dh9kJTLbNSxD!O4
zJlbondBM-+B;UNxygM&%^Vw8pre)RmH)g}K18TXcD+*qIxwHTJ90sk$=X%0a-c8@U
z@9yE)lK1)hWqD^Fa+d3NTOIV0o%iLLz$ag)GTt<1|G|CY=Io4)T|1BeXLzY&$dVSf
zE-mTo&O=-64Nj(obS-ha(^u%TP1%HXN5XXZ);a9&kDSq3!Iu~v^L6pw>H|Tp8*4ZC
z89LoQbCB=T7QZVy3Zz=J=6ke@8(vO4v}6yzYqg=(9j}~<*5Zb{_b#4$GOatY>eHw1
ze=Mqv5`=!re7U}OSF7r(dT%A;Gd<BWGi8`IxN-!ao3*^@@fqXt*<QB=BERemYZlkK
zGPlvl__F6BwMaRaD7CqN41%T|7cH~7&aQRo@x|w%OKhayapxY*NnS9!oZ;QuSsC-R
z>MU;j=E>T~WA-;VuIs&Tp>smntjeWot(S~9hL&z+wS08YkT0<4rho2<1cuw|J@YJf
z+>j9Nja<EI`Heu&u%|yK&uE<Y=1chG<62d=3!lCW-2b0J=>EpATtVr(>W(qLrj#3g
zKDXoKiQ35;aVs6wFRL$%_mJFfZanv|lv}US;@`&?iL~^+UcA^>l5th;DovB_XP@iV
zM82Hsyg4$xJ=psEyx#iEQ;%~lDyEnvzWFV=-aM@5`lf<b;aQdsy5{*G?TskqD7<{)
z>GCHF_f>Iz$qxHc{^FioRLQTE8(eQ*jVwI(KCgUYm)7!X!QF<>f3?=nvVVGJ^>b&p
z)jW+kUYf@yyb-@G-h8gSGW_^ElLN6HYaX4=a0{t;X!`cK=XQ_V?~e$5@ih(guUtP}
za>l~AnRCu9y?n}ItHR$Uw#Rw&wOcO<?)SUT711@rEH-ha@2*<*!fk!#0l$-!W;ZM?
zP!w7;|Iz!$UlSuI&W`+ivv$Ir8E+ru8Qe~}$jH0w?yW_t{~7A!yZfWw=$hR&mF)L7
zi@PgW;%IZ|hH~o`yWhL3KYTs>mE%;y#Z#MCJu=x9eP+|^TyvRo|CE&OHLSjUrn@;J
zM_Z?RvaQT?H-o47g6~XpJ#&O7hv&|$_<gnha+{7&N~Y+4hRtgaE!?h@V&2&|aT8yL
z>XG04{d%w89TMXcTl7?4v%|6NO-I`8rNznT9=@CV#r4l*=1_hSJzKVrswozZS9V<F
zZuxpF+~Nht&G+wDtgk=XwyN=sUB7?Z^_bUNgM}&&rWtU=<neb^|2;D8Ph1SM7T4FQ
z&n&a^MLtw+WASp@6h3*k*RF~m{~2OjQ~o*>X>9ruIU!E5LTAE*9Vg7n?wEHiov#`i
zU2^E^<q7``xW0Vq7n^06$hb`~zP^IHvvc*by@&5Ues*;B`s0#Y&+n~K*4(is?)jly
zf%@u`JSM*3N9`iQlBcbUI3sOxtESy|rnOIJY=NT0t0*nWMbQr*-WS~@{YG>0q=lOd
z6C!*?(m3ZH-qIanec@v8uH_G-3sYyRu|=I+;=aPUUAB{Fc8ZSK<MWmZ@`lPz(%&u$
z?Cj4K@IKwTO{8Ap+RCe0{aPVSeQ7+CkM8)iO`0L@IV)pzne);88=SSoW{X>iFZ$bf
zZtiwgs|CE@nP0g4n)aVz{vnH)<(uE@G~Uz?@hjN0^O)FDfgSHYE?%a>=6NuxAUr;`
z+w!$~!s?lG&Pv|DR9Iyxaiv`2%-xW+t99mHRmjiuv)Z|@>fW-$DU0)@FN><Rt`VB&
zcxr3V@~5YAomj4=9KG4OIXS2AxSy}UA~WuVFF!ra__J|t)n1+8{i?QU$uGi|Dim>>
zY*@C*D0-F?n_EwHFk6mF;DsMqS+l0@>FzVzd2_nLtqWx{XK^`C`N@#ez4nNDpTmq9
zlebMyT+b+9ekm^Q*JCZOR;y3(eNS3vNHXbfvo*LU|2w%~_3bN`1-H|7tx}x$`pCXD
zyea<~mYK!OO?sEI{LCt!8CO16&C;_|I=hxrIANZd{*?ZAI}PU^43?NQ-&JF4Q<+V&
zQ02v{XZ|m?3iNLMdue)aQ^eMj=f4~%*qW_>VzF~tw$Dk;_ui?;m&E6;6_UC)&q^q}
z>(ikQF&0zJ(DQHpbUORa_$+w$-NV>jDlWRk@47r%`%d{gY!x$_7rwG;TVNW8;G8LO
z7oJRMRNj94mg+yd*E<vAoLi4CzI^ggtCF#vq1JO{jYfmq2Gd11x99I>-h1vDr&zJr
znT~|hhHmmVlPsPdQL&1x$}F$tt$rA{x$#xaiia{QjFvP{cw(3{al(mRm!JKVvfzGr
zr2W-yv9SLPx2<Q-i~jQQxBcN$zME%#S~+Kx%t_tJ4Pg@x7?n50f03^%yIY=9EjRV*
zz5XYrsj+S0aRqr3N+z3kS+|@%^4P`P`rKz%|M{oo_WekRI+D`u|J%k)@npN<N3jz=
z`PzA(e=HN7kuurn&a#D@IdAIRcqsPx<9or~_qsn{t&`W|c*iumMB;f|ny%-A)OMB0
zs&Chd9qDQN?CN*zpX%{a1>MIot0MLF53I5j*N^Zqdc4<qNv`D16QM3P)|K{Gn~pM`
zP!-ob-tU{Y*>{qBa(?b3WhHUz2`lR#9e7|b+xpU3Db@D)OW||VT%3G1%e)igaA{?p
zq?Eh#dvy71mL*@Cv!}jL71NbmK2MXk_x#QTiF;-*q*QNZdDOUU*tPE9q<tHBy|u3H
z^}pa_-1KEx+k#HM!-?Ov+;J#8%krM(a^DP=X&Y{C<C(lE*Q_>b_N!adZ<nz&-ha1l
zLw9@82WjgE8wA#{FZi7MZ2pWquQPlTJ!dD}^5Om6bc0LXddJemAM12{&V6If__FQw
zq+MED4yU<##PprMeMGEx$=W}AUzUG86mIZIQL45o{XawNr>jkWI_|AEt@y@yz?Joi
z;E`Xmck_i7L{2cAv-w%%sV^5#?(5}!G|ShV<@pV@B%A0P374)2s`@?8*Rh?xVKLXv
zB~_AN_T1s}k&kv`dNn(DW<2+zUkvx}26~^FZ<As!X>9tmT*5w+@pTb*-{my6JN;=w
z(NAPrro9aeKFoY9aL;+g=i6RSEH*A$aaUS*ieTclIr)y_PqJ6eTDPlcgPlyqOs-w$
zRtc+!2VCJOs^spoQC*P3(5NJ#^v5ZC(YM2*lM;>g9G=(n=-i5DTHYd3!IybY>ewpj
zd$hlqX;^yn{i^DJ^IHo2XNJs_K34r|GMnY)zsYjno?V$!&|)ced&%yuTmKpCzdSw|
zCe5?b!*qJ`wA&Zv8x&7_<eBqMP<zYf(5JbtT7PY@N!5A0W_eh1(3GP|y>e-1B9dM*
zPVCt5aaE_rveoO`ZS7?DnC{3>3{jn);;}q+S7T@M#4P{rYQI9ohpXqGKizF3@*^ef
zny1l%GZDHQE;tunXMJ>AV8_XehF3~IJenr->CnzIQ<Sc<#?9US=uAz|y$O$VOCH=8
zO8U+Ac-MV_4?d}0@ekY%yuP~5I$(wSREc{Mi=wy9x_425_4W6a`l7!-Y@0mEOJw;4
z*OlQSxwAdWb$30wS`y&*@9B{Uu~{xDr_Z{JKbvj#Zo<2rrygFreE4J0Y|S%5-Aa2*
zE*hwDw!0iZo2j01)!o0>y5`-%ETs)W8@{i)(=VUG!MbqgvzyN9CP5#4gM%JE{aJqW
z*`FP^HZecG((+)ZADcr@eC4r}2^zu-=DU;D*M@ID{pe3bZy?tR&Q)n18#C87M;oz)
z>|1zkHt%gmmQw$+`)cnVf0?Eow@!1Xiea{}s%!4Gg>#;<lqGv8bH96}WjnR{TTFUt
z{td-d4`xc9;x&Dw9>*zsWKQnHy|P>6jFk0GFH%^c@N=JzTe`%eb2;Z+#fmT9Fkcc|
zd_eGUNzBz!mO>LJ(QE%1bayPeV;8P`O>foXt#%>HHmx-`ysEjtFe|LPdGiMW;e=CV
zu^wCAc3*lN-~ZRW;@g?l6Fcqn&U*E*eGWXaJwi55Oz^>FzdPPi+ROW&%$Tq=Db;Lk
z(41P?!`e4H1fpKICp_3``jamrjQy%+k$jhtX1;S<=(1yl4V#k}T@?6p)iW@(bbFwD
z=99;*VV>>RqT&_hV?~~bTCSRGB$fM7$?@;uzU@a@E>;)Dm!ErowQExI**|^3@sdef
zZs!K2I<0q`r{r7OyXQ;yHeQZNA8g&On8qE_%+viWzM(g$<Jp<*(<KX(H>*B>F1}^?
zj+b!<7G?eIzVH9xp{&@-<Gv+-+}?UsofdsHeciqGu%+i2TLZ-IR4LC{D|_yi(kDOP
zKO3_?)qL~eso%lc=Ni4zIP&Y4NiB8>!a>`vsD@2HyiIgMc7V^T<?hy5A!#Z0JTVg<
z_8#JI&J}xTx%2iU^{$$Tte+fLg=*!F=DfPuwdl>{Dv_dI=X?1xCYn7I-nD39<^1}M
ziynQw{czdF?!eW?HfJJqfBo{mnQn5YkbS+{9Np58+)$Ac6~4RsbdnFPpLg<~s#|RS
z7p{NZFN_uhWnKOoEiu3ANTOM4oYRZdO8T?1bMI{V^!Vau?G4u>ZH}#2?=_WWh50ej
z3E`eq$4=yk8gdK9h+ZuHQF1Kknz7c^xQ1CVFGHWDI@B;M%-dO0E_>_I!Y90;*2_0<
zf11Y{Wg^TfC39l4O8Ba&idD~+^Pc5ai#yo=K0wsh)8uc&`PDTA>lL<0)vtKD@8sku
zx?Ao&5}cVQx}<S^$#t%*{qMDPIyK*WP0^ikm~qEih2m&;$IJd-A4FFFNeM0Ze5Go$
z&(}$s4$(=fE9DiXOZ-lLRy}-ceS1&U$%E@%SCpuV*}i?pyI^s_ovXK&*U!(<KH=8f
zcyG})*_RJp+dd!n%$Vc*JkOf%X~6aEyB!a__|ITq+N$)l=4i<B6Eh<#*e6V0JMn%t
zSIf3#yFOhlIQn2Ob0izP-sQW6JA4_8FQ2?uaq`O6<8M@X?fL6@T&HdPp%`u$kg;=?
z&8e7<oHL9~&o^zla5wP!uI&5q@>jd(m|o9c6SQr+z4@yNwSkH!9MxX<p3F%vd8;UB
z#o^Vo>-X<<cGC}ka=7-JdCIQ?ms2eY)M^(=+<I`rcH!<{-#%Wf{Znv;;fRWnR_O61
z!70qwPKitB{A}!a{m#ET^j-7c0>>$@M5-DtKQX!3>SeJ#)8?^ib=8mlS8{c+271Le
zbf3(UIiBXRbbo~P>{}A;ABEj(pULYh{oJ?4g<Ur7Ex&2d%MzglyBYM-u6Z_bdu-j_
zwyRvn)P1$&?%b!pCSUb@8+dlx%D0pAZx?*p-4NlKF=>5{8@JF^zJ<K2Y^$6_JT6uh
zJ(ifaJI8(3tQVd4LwU3fmDv~XYB+VONHL_(WZ_DKcV}wfHVf6AKJRvsC%}b&{+ELb
z&K&tO?`z<yX5;%aid1bbT3in}+sn<Qw=Ict#iDPn?^Iihvcyzh3d=3`$y;GmRh(wL
z-d*sA%aTSj&P76tqSG%oJ~lg0Dje22Z&k9ZSL#WjRcA`dr8;_owmZ6Bv=jZ$5W2&#
zi~CH-uMIwHQugd%^k1_1%Bg8@iq(3y8g5*2wM5>&O>)cHO3p94qICmOLbckr{XTyz
z&*0P6y_*I7`8!z*tzU?-n8dsY+0}eSXi~oA$!95(!+1~anr-(lVsUBZbj5&WNnbjn
zR;F&>^k>zPiG7dxgg+jOjK1p{92+if?6f&eY}u2a6HdN-syZ!kv*dP;wo8T=OX7Z&
z$NCtaVK^NT_-5DVuWH$buhyGyvpJ{j;AUfXMU>-e*rSh+n<Lq-zBl8)S=R6SdUfQf
zV{Wc`#)1)?;<uvDO|tM>v^a0U{r?Ohh3!|ff?tWhv9<J++c(X$as!k0lZQsfy*}Oe
z*TI>k_2W;~<CET&Q&J9|^0hr7kk|L|l-eVal|qt_bYEtcU-zx7wR-Sm*XenOvYL-Z
zw=2o&zT0qQj=F5I|B6uWSfNY13&fvU?46$&I%U<DXW1n?Y;BCy(|K1Nw9VaS#nt*?
z-m^c2{&@!_8g6%el{~RQg~8)aL8_~{%@T9r_Vv46^fZ6Ci)24$o#LxC)k!yF*;(@z
zNwt_heu+D3jd#RDKPvHkPvqX$v_W!jjlr=cm)%%i3On@|?TTBWmDRm&p3HxSFzKpk
z(+#%rZPjvViz?yW?RrQ+LCN}#c5^)6$!AAe<~18u3EKvxv9xQJ?^P03diJ|TCR)8c
z*Yv9G?}bfqej7^@!{*Oh`R<BzBbT(2U?tnV^}AHM7riVJ)NfmOTjX{5@*A2FzseFO
zGUquj%V!a<YE9|8lz5wEE$h!aou3~!@7WoV!?xhMn%63$o0pd#n`%1E`qsyWEtlVY
z*mJ#8ZP$+)3+?x_UyH0!zoPE5>`dR~{|w1}K_|S6A_`TMoc*lcsoS@USuD0@=MTMV
zv{%fgV^hWxW3yu>;Y&2$Z*fw|Sk}?8uhy`%We2<NHTFvl$5#cfFuAd5+x+(H$E}-`
zmt5<tUMDGhZC>j_#jH5)uKSWNjDl{s-3vY^$<`jV^y-t=fLDUs9F+F3<?j;Lzj^tL
zSMS`X2RzNTTr4;``Qz!+>NATbhph}>u#;;u^YT@GrGJm*+^R2Su3T(#(LeZ8MSYo_
zMD6RfHm$k-j%l{pCJyI9+0)KgycOnsxBZz=W^RbSywEXu=@XAOR?cEB?YU?talOe~
z)oj*r?wWHZiHn5PuW9YGxE%9re#BpYr?hm}DS_vfs(eqrb7Doi_Ign#UptQqJ*mC&
zC!RE0Rxj;m*6ZG&wU95-Tgc?58&AWrw`UK!*oN}DpZ%D0^~cek0Pe6AmP$O0Q&sMj
z+1j-6dvVy9s$Ma<94%t|FUB{THL2;uQ>zxMCFXZ0B^@>WyP@*H6DMD;Weuy>&02qE
z<5WHF^EZ^dCV!nabB)cP7jH`yo>Z*0IdxoM1IL9U(JFqApI&}=_~~5R#L)f~2H%#7
z{WX16Q@5r=Mp->7!S&(wT~Xadg<E!3`_}0MJY(gs<=X1LVC$=CtDbM?bL2j7;??V=
zqEk!egcxl(V#@a-#r|W`aVbByiSZZSSk?5jS+-5L&tEZFtvGqlWv9EJbZ=h_t=V_p
zC%=@Bclnz?-lugFGhMyU@g7}veO|kAm3OP0^F=n-B|fu?nXX;jb=K=n;A8Lq3>UVu
zJeiRf^diqxYl+~)skYl^WKK4?W0W2$@afyWFM*H0$F_Z{Y~Cm#z3l5pk6$x&Pu!B`
zh}xqkD3`gu?r5ccO=t%9QHG+G1yN0Yhg0WoHOxLU*+p-A_Axt^X<b_SdkgOOO#a5G
zb}f67)|DH4wJWE$+{-VVn8U!*>g{l`G$#ET2iKJuX~Fj^cR0S~Jt(iQyF2%z-#rP}
zklqj;zlnU)PES%gU~X0^xO2ff!RQmh_m|B`O^FouT55UISvz%FL_wFeZ{L#C{0Bdk
z{{C2Wx_EZX%u`PT<SGvEtW}q>p7lV<PX67|b^jTzFy|f0kFgbL6FAWGF?tWz2LZ;I
z<j2eQH}k9$X4g%PI+oS!JkiQ}`6efw3ET-TdfCUft$cRyH0PwRp7R5<m|yL9>$PD!
zN5`EU7Pl);I=b)sz3kF`T(s&={5gKtJ5v=RUq+?O%5z<w$XVoiRb1>p!xPc(^SIAV
znRR*RtG!~*XEtBHI#*Le!TF4ejd|u*t&APN)V=CYy)pZDDD7p*&2{;w8u-i}x1QXv
zrK6{nQBeHsvRg0ay~$)Ta6G+vsdN6-Qe}bu+}^zvQ&+@Z;6Gb_dY>X!@ha&r+gFNA
zFtjl8URlzRTX=PQC~rXN=bzK=lpfQXaOz}UaCSfQrnt(EcVCaBa4{BD@-UiyeEZ0?
zqa{^)dZd>2RliM!roVT*?LJn+zIW9=@f+e5kJ98Ab#~q9nkuHtesxpTv3FB`&)zdT
z^H6qzpsBuI-}FBpFE6+`dBq0hGu@FrNyaluBC|A#b{<uEGxyJ~t&bwiES`471}!N0
z6;meJd)@g)?uuywRtJR-Xl&EUogg=D!QEEDnr61xJiWVm4F2reDpzJU86G<L<np>3
z4b!?Rd2-~x&G6PZbmc^`VyxDI%2n1|UMx_RX}F^_d1n^;Ug5l>K?l8DL{fd0UwVCs
z^YYDB-)VXV;S)PL)K++w_jl?4DDYYEB&Kra)Y*#;uZhvUbj*3}yXBfJAGzXIuRVQU
zFhuN==xdRkQ?gq1wY%?X2gynX-IV3Hz3r@fsK3#TxyL^~=xEyO6cW|zvBZ^Of9<ZC
zz6AnKS9S`!UVk<(hc9W)WTlJIE3V9*&DCH0cHiSuOR^GvEH8Nz^?Tpyf{o2zy_!R$
z4qs3We0bT((!A)aSaFbO;Kr~=-uvdSsJ*diX->e-fU1<rNg8L}jM$=59NsJPrLXH<
zwr%RF+OK~k^f$ayJNv?QM`gCsv^NvFUeCVjH}{zb=V_Ly8*{F2WL+Rv|7}j*@yXBY
z68~+so*V3Q>gOY!Q`&{E`+ahvx5=-LzN;WM+d;Ildbx_-%X~h&n56rY1-TASi~Q0m
zk`<)lv+TTbpTC8^dqzh*OTk4K>5#WK{@#e$)uLvb7hG*<E4DDG*V$mg*(l>ucRg#i
zza|C2+xu;EK7M{UJ1+AJSDIk4SW1=5iXx>e^P1k2-xgGSWP5Pk!LnTob$^_=zwyXc
zx5PB(zufGV{@-}F8TD2QZ(k+WG0!!0`=1*%3%=+ac(&!N?ZOqI-;yof+C*-MesrLz
zbJ?Z0w|9i!e|9v_H@;8wRYS$jO}w$wR~BY2FwXoqr{G!KOX*&r7t$GBi{{xsmC-DI
zQrjf9T~g`XN{^h_sRs9yc3B#yF1~z2W6ARRT$ii$r@L*^DyL7~f9BWZh?!w)yh;=p
z#eIThEaR?l9cO6V5EAnAzJE8n?w8sUkw<IoCWkCHH>%NKc<%MnLR?~(^sXqz#naW5
z%J=8LJ1z3lMttelNnSB$W}ezoS(9PvA<Pk-F|YR7q00r)`e_;&s}J1`kF$Sy^yzau
zP8*hhHd}q&qpOV!6E}-!O=MrCc7~^OiFEaRr9vC)Pe1mpF`u#WM9G5IpuNe;+y1mK
z@mVU<*stUIw77)DVY1t4$KC!J7k~b)EC`>rU4G#+foA6_R<`-dYuKf}HoP@(o8H02
z`>ZXTW7XQ`+HH@WdsPn1m>j0@(_Pa!-QbyVb=|$(9lVX*UQ%y&EnW61YxTMJ7h6K(
z>Nf06DY0thH@u>-<xTPnHItwPRwv>M&366!YMrs@?)D7Te>sX#{~0!H+3qEr#+G8L
z;1z72d`EqzTucx5-nRKc+wY$}F1jVBUWqHUiu>y;AFDGjp1+bv`qOc4`(Cj};!@&l
z=Pzfh-Lv8LDfwkHxFakLj^|$s%rxM+nbBsP_50F~?FIaEL)h-UtE{YFF6kY8LF>7@
zw9y~4<cmpHnNxSX&);eGyzQgey=}+E?|5}SnDOPWa?{hJDT}x&yMhmkED@P;D9QPa
zm)py3pP%fxmam+a*~OjywV~$7wZPOH+82u1SH1lB+W2vG*^?;&#oyOjiA~z_@a4z$
z-J(U;JS$FL)mpw@&1Y(4&*xtEo@jsfs=M1=T<5VbZ(S$#voLZd!#ajPyM6PveOVbG
z$C%+>_(b8;CzGqypPtNnqZCy6>tLvanpf_!N52-Oq|5eQdz@FOd~3r*mWvyd15B^7
z|C3*Jx}Z$lY|qg<SLN0Qu5!*?W#O8!{h0Tmq=XxiyB;u@b-8FKPp>`vPP>~wL*<Be
z`E*;Z%b9Pg?3=FYnM~?eQt{+-$@%WR;+9}(0QdIZ_T2k?w&sTWoN`P<gD1>0Jhaeo
zd(w7~Rc`GH?U_f|dH4&$%N;iAiR|>33yV4Tw?ZVceuv(vc5m|~fwC)0-k6#5Ogre~
zCj4Efz4UYY)&C5Y_xaa#v7GVrJ|{9gbDdn=yksSvNjEe$I4JL6E?V4?`|@4r<vy+J
zGbRgM)p#n^?)KGtt@)~e?~xOekN@7bUe<AMo|FF7=C$9dxw6ig@KpNk)XMo$RG#^U
z)9i5K5x4&g)?NEKmY){67R~L=zOY#L;F+tQhOc#5(jIjh-+jqpH0RT*{|qMams@q_
zHR#N1R-Wp(;#@bE<C|R%CYrb;xv4E(q@ViX=sNFg>urr%F`fNaSvvx^rSMEF(93jP
z&fya79BU)YF|~{*YG+ls|5dL#7v2MD*PidWc{^`WOmM7S#O6~e3;ymp@x;C9Pxw+p
z<`geuKjD_IXY*E0TBP*ylm9fq9?dsJxi=J^X$r=#34U?@w_|Ekaq3|u-gPd`>vvS%
z{?*R=*ij=aRmXYBg-4g}bbHP_8G1AG*j=T23hDt{1N<}d_W8HJI%qOMbrRE)prqE!
z?N$Ghx9QC+`oQ-qdwp;Fznt_v^Xd$Jzs`&Oa#D6@+_cR{K7YF#yx8zrC1c!Ur$_Sr
zZ99IMd;FUoxmxW;(_Pb7)jMl4Og*1x>xvvZP;=z(zl(x3FFz#5E1d2#yDMzy*XUiL
z=33CYEw0F^V<Jb!ihKJdr>p$@z4ge6hFP%(BQ&SCEa=j+S<^Y8^=aAMtK1>aKPLJ~
zn57(?tt1;G$L6tEbl<Ue3|24JuCFiuRVdI}H!t~P%nP}%5hoN2Z}h%!`?H8Kwr<j8
z|5k$vHiOh_nOko0M*fU!^I5r+!#gZ@PiwT`t+&f_L%DerlluLIB{pp<XK{(H<+><t
zk+&%ObZCg{r%y*%Pp;5CdZuzwXl~Dj!i10+WvM3RJuY6xQH&Y4x>hY@(kgu1%C5#8
za8q;bsVQZa2FX0n&CYn7=<-{*YtgKt+yi`K|6&T+WUh9FoXU+hiNCbL@6bxq3!ir+
zPdi@Syi2)v=Av5Hwv`N}T{@?hMsAwQAepySb;a8g1^Uv_g6bXul|L(f7U(RxenRf_
znx{Xv<{S)KndX+zaIC@L@%t#F$IUH%W_(#qdM_H6{{6eWjoo0Q;(}l8CVpomx8(=(
z{_@pN<=DI1b>*y8>$WXY=+3Z9IgsGHLgm_~J#{-W>vpFs@$0$sd`cthvioKy)?9z0
zr@N=(Va@{4mj%yCMSgsIwa{Pp+p)-e4&mG^r6muvmhZTJoK=-|CT~=!&hqO^Ql~9r
zb($+S>vo^7WP;y*dnXkK*MgRcrPuh69*;ImiSuAzZS*u?Y1l7@`-T6c=cThaw3mu+
zh*`w8YFA~;?!?^3{t7(11@6c!Ix#<_d%F6umu}sx%kC<ca;(XnB({6i3jOA?_SyZf
zwjU^-`{JLm<eol}UBQ9TR-VTsd5_gj?{fS!DLtwD{ZXCRJfFaVl#@<7PE>1!Do;HB
zb@S4U#t9KgDGd9pR=jYnN|aRaZ8?}<pS>hh>DMI|OWE|BkGYEk7N5LrpkQ#z$nW9N
z%GmP$*;3+<i<UKQP%Atsn0;l(vb598I{BH8Zz-u5Uf^|)`_C|M#^EcU>lF>mJa2Y6
zhAlYS7}TS8wROdp^4Z%y8Wl`*6x7>xw0M8@<YI9<bFFzBU+F)Myb`fVZt1C2n+;Rl
z)LY+OyQA{Djr-n}klT-<KK)*IbZ5bThCc@)1KQeOiGNm&(g{vaT54S!*q8if(xZ2G
zA7?E4y)yOgap9houMa#~C9y6rac)3LkT!cx8=HfRFq4^-!**ByvR(Q=-9@e1{XBoF
zJ#7nLAh$7Q+0o@!7WK-n5!%vR<$5&h_H$qE{Vy+bh1M#>t>Zmu*%cbJ;CsAF<qN0i
za}6{3ZcJV@_hLXr>o5L`<vMmUQx3-p#LPTAH7ncrPQMuQF%_E>v-<|cPHIy`1Vc-|
z&C9=fn{k`uwAz-bnIQ#JMSqCgd?#a7owH=RZ1`Kl4ZZt2a^JoSJT^B}W4?pMYu&Vr
zttWk^@a<7L*>*VJ{mus0gXz-`zu*4w(fp@}^ykVXo1NdYIAERoA|CN)$6hJ8s(F?-
z#AddidvwtxL{upBviI8E;tN9#pU`=3Y0FnVX;si_X(QF>%ez@GD(E_gvP(NFmUq5x
zer$HI=LGYsg}04F&CKpvS+6s?^3Znmo>e#BF5O+b^?RkoKb^Evy9IyhqeN$R8GPPo
zx_$cN%aIf1izh~~IE8(8y?*S8fA_1S7Df4sQ(1NIXd0XNSjzCu@(=v|G)GsvLg@SN
zuRm)oE;g$xng#6GUNlGQ%8RWXZynlBw>;S>#Cut6O(mzu#Zo!PT{BWtX8t&}I()&I
zfXBOio87o37+pSN_GP(@q2LCGasgrAu+P`;_6Mr|co9Egdzhfw6L&Gc;7#kM@Y*D(
zJziq9eyZ-%`b1Zy>k`|;&!13?@LlbZqiOh!;opUyr8)u;Ib345<mQB*={|Lb_u&nb
zXY(x<pV$?A-cn|fr9v|2EpZ{Co2!;hF9`b2pntXJd{0n9tJumb9=UBZITm~rKE`~!
zOJiN<m+$sR+YY@j^OGoA%Gb`~aOS!6>_;wVO1Ia%E8NG@F<&b2MN}B;Go6#Ao^O^;
zSF67^i6hlxu69P3u!ofPxtP|oN1PRQ?A!Y3;cDKN)EiM>@_FC?GQYL7<-{kY%eTd%
zC$R8+6}+qVd(q3Kr|w_weP{HhGqqlCp22~0-FCCJ_8wX)7vjAr&m+2h>bX4Tc}Z`x
zzWbzjx6R(yG%xm_W4c+$%FlTVFC0p->fN8LeZ=ulkZAp;*n%IK4?lk{Iu@?>>xik>
zhjfpnr!-_ej5R!FPv+g$vePJvIb%hW9$zo}<==-6dP%bw%Nu=U?h`dLd$2>tCjA79
zYp$!vRhzrl|Lmw+^4D;o&&Qcdd}qr4wvj&TzRmi<o$B60`~B|8beeolTpjU1+i>;8
zShIy9OVhZXKH;65%-e3WC~W$c%P%iGURZn7(kRVL=$cXZ%7UaT3$L$=b4%N<xP0~n
zc_l%S+i%|m#c(?(@ux|tnw_xlbx57tsP=Z-V<jWLUm5fJMV=~YbD4USt_+bn>hWuu
z<J*buIgX6dXZ*K5EnRqfyZ&z8$eeFXmo$D|STQqn#)qbq^{F{0h2)&XSve+{U5=M8
z<B~~Pus&k5wz1c|Cz1;ocZaBRPRhL(?U6FA%=L|R`oI4S-TV))8n8wu92DI(W!2(y
z8@_%1EG_hM_7mQjeXFd#U9^6b;brml<9sj4iP0-e-lWg5cx0wi@adIR-2Ue=%9Hj!
z4E5Le<@dFv-b=J?*^KC@EaL+g3vbI#5HbDu{{5egZ+=PVINcYKI%6eM<<ph3pvvaN
zCeIV69xmNx!+g`x)L-o5rZ}rKXM>Y37xJ83&U#|Kk=>Q{ebdXYZ$I(p@Vg_5-y=mY
z<^`{3*y6+YvTqyT>IL!36qwGmd1+7PQ7-Lut^NH-{CjJ{>!fupP21P+d%s(tnZ+_l
ztEVr&-?3LF#dMpFQAWb;RSmalUwJN`b@$<6wtdbk0#_KFRkCTDvFt08e#b7~cEwDQ
zO8p;~-Wp2u#jQyFvw3E5s@k+G$J>N&8$MA^%gy7sr4dzlN0rqs_BzMC#?|7hF5Esn
zbJAAtQ_<(#n|*6%uxdFqIEX*)d)cHUv{JKt(;KOu9A~21wPw35JJ-l(l0H{5<%>n{
z2?oY?NmhY7TSHx@Z&1DQ@PMw<0#lC?oB1y0-RpX}IOl$A=I`#?^P@Z@Q2YL;?>`DE
z1@_A67T&6rUz@&g*4elkkwDvqyc>tw`q<BW$@f3v`txzas@JSfcjbDmbYFg8!9~fb
zMmwYHl6kg$tS$;P=M{SzsLdemdim8Ki)W1NatnLZQq$)LADeLGRP}_LncD(d_o-bs
zxX7z#YqJ0E%tRK=3q>7QKZ_X6`(m|)%bR)6VI#rC_1+Upcl(NP`dq2#N=n|)ezoo8
z*P54MX9Hix+DdN9oHk32?SUE7<+d!f=KHs%_erc$SC>^f-gluRNOR-7Dvin28=q7w
zm7JJ*K66pmtp!r$A9V{e9dk}(Y`DEDJ;^*Wtu!h$@F%z6_H}|=TF>wHExCTSt7g`c
zAG<Yw%t?D=vhr%Wf=JInyYv4UguTT=xtz_n@qRSE{h6cde!le1il6H~$V|KWWM<pi
zd#v3`g;(d9$r*~z{cSS$f>Lqd<y#qD)A}_3GX!v*u$;9dDD7z4hcw4`IScM@Nl{Ab
z@O<~5AuYz8>&;B*W7-!!D;%hNw@2lu+T*A<PMPc-w-@`x%+eS6bUXLP{)4ab{nIaT
z8+^PL^t#{Zt(o7Ksuo9yGw$XstqnPzHKkWdO|`vW9@^C!Jn^qp+L18DHDR;APUVyE
zobh>)>m}hyJckcde2TM*ShV+T(DrSj+P5d#OG+C_wmgl9R$F*@`EGyttQrTEJ2!6b
zn0t9s2Ip<b?XSdV|Kw<0_BoX$x?66p!JVm!r<Yi6xpsAJdqQwX=<W}XLn}K?KF;M&
z{qx~-sHfj<rk4T9Mmmii8#;28b_j+~Tg0_5|Bpq;<}>fEGfnOfS@9&{<kvN~6~Z%i
zatSOw*0eqI;R6eI_Q{_Fo>w@{-X?JL$EIY*HJeq>-pH>^>7K5+wrhSscA@_F`Eqi5
z^8Pcdd6yijw^B}|KjO+tpHrHy-+rufH`+4oVL+~E$k+ExUw2l9N4jhl^_uY5WT)uP
ziMI-3QbT7S{iwWCV7kkl_lFOQt*?Ign6>Ba&Z`aL%2$@jyz4!byOU4;-C;*zcaxB(
zzYo4&r+Juj%JpL=;hR%l`kY(rY9LW=s^G9BcEg1kFB8A%*PlPz%lbT9I%tm2@++%^
zly+4&iXN9v&o62Gckssb=_23nb{-3nJQkwklg6<v@cFT-X}Q}T>lxjA*PJqUvg`YW
znYSDH<Rer$Pl-pL-m3YvQ`Bd_0ZYg1S5XxVERGCyrY0`eUwwMCH1(5k%+W`luNkMx
zxg5X8d+o`mlI981+e<%uVCVibV|8SO>E#a=LV^{M&2n=RTvVPs_@-WZOkrE*>DaPA
zh08Dh+b^O0wKrJPNNTmQhGgZj1xIc&6#Sk1etp`-&_J8jr_bJ(ySZ?qP3yCd3+G-}
z_mEn0V%@cKuU)p!zR-F0$wgM4lHAhGyVmPe>XpVdUG@4oX~&bQXp@_4d5ms_dvl&9
zvDXTgiuh-6|9q^tzDjWBGe)T?kv@AxI~Z=iy>q5~eGsQwo6PL1q1M?&-`>b>2x#I;
z)vY?PLih0eYk6KXl;xauaA)TV-MxKOWS!fzH!)L#`?!9bczw02WzErx0XI2?<jPx*
z7YFWLyhl9bUE``O`{mOWc4@0}Yd2;s^<RD=OYn(*%E^u-<JCgR&3Ut*EznbYbpNi$
z<kKIv*(+@qwCprG^&sP3*Aow`YU?#KeKh>MO5#inl}uIt-dekQq0)Mps&(A8#S0}{
z%d#dse6F=E@@AXUWBxPW3d+pbTw9)XXfFE9`u#scZI_MdPsvM3hBtJdDXuV#DymsE
z@yqRRza_rSPTC%EY1OT1-?Mk=8C10<&hJ@Uv{rw)?aL25a)x!D*@lnzZrH!+#dP(W
z(7pexxF(ochAcXjmcq<be)Ep3(+vK7e=RGLmQ4H8A3br(h1Gq*<q9X-c_d5E8;Ng{
zw@<woeERXT>r!9xG!O2YJ%2^5{3?}FRSlOI_c>;_GFo**Z)bf{=iWDGI@jaO*>mri
zhOWM{JwxTqiM!6*zV^Jm;iA4^>HVkvJTjIO7Q9{faAokzE7v}=EZ@7|#ZU74-h($;
zSwmd?UavpRbHT8tVb0lkEt_)cgFV@fg*}*Uc#x+ucHOMlbzS=u?{mN6`BE==b`f97
zgy5n#RR(9+s>E;XU|p#0F@eMHv+IX%^FI10uFhQE6ZyqH{6dz<hq75hoZC-0+<Byv
zEivKN<*wIn4Fp_f+9vj_o~e9d&h3C5%_&L=X3gt_+Mn%+Ha;`=i>}XvFYGMJd#?x@
z+_Z9C(J^Zw7gO1vX?N^Z<jzzszdYIGoAH(M8;`C=b@(>xe&$=)l+oWlQ8>r+KZF1M
z{?H5WT3<-+c{n#nF)MJ!Zf_;=7y%*efKG=C><4mx^cL>2H|&~kxXUl4@ZRz_>+PJ{
zZfN9j^v%Be$Mio#?TwZ*>n&#_1T!waBP4tHVg9D*3BT2*SDGF-=4KL?o@nQ?;;qr?
z6jR+9@3|Krkx|TdFTGwUVCq{jUB~BSW3twne!mqw?7kCwP6ypluV#4P@zzXg{f^#%
z>tB96?0(EIn{LYH6<oK`DszY7eMz_L({D0N(trB!-}<8+M~)|$U%SdTtt@$^w8>22
zSw+d|RRSCvUb`G<nzZ!v!TUzF7Hdwmb6=TJyz-pie9d{g7Hdu4E6QfwV&vVWu&ra?
z=^fDpx1Swx5?{30e|Es~Ek`|P@(66uw~1oh$C<M-*McMSsHj5#r}yW>%|~8s4blvK
zvtP^lz~>K-{a0{Vux_|0eXZTrJ??huvk$uk7~0A-&)xO7m%g^z$W!Fy-Dfw1g!yjF
zlCo0WmZZ{ppspe=>8kflHkpcGY1^4fM|nSXbH07Dxy)gA<u;Da(uE1SPvT!mrzIC%
z^PQOa<cNg*E%DwLpOn3;5++|iUO%tn7n7D+gTh1g<V$bdMH^2~sbrk?YO4Fo0L9}I
zeiu9zK3HkyRqgPDZQ5sTQ$fLyqn%8?eDh-@YNZ1dY~?0PzVw!8w7b5r+c$3Y5m%3|
zA^MNok~p(w>Ff{DbgOPTGwGE~?&U46?Vc66vnrd;Xe$XE*2>X&>f0z|8+Gh;pWNL;
zFSBKi?|S)_sr~D*{Fr~;$5i(<T)p@zh;7abnYP)#1B??z4{4>Qb*%pR`1z*R=gfk&
z&BBw6UcdL4fAM60pkPv{p|lZ$c)&p()4fbrFZ?+XadB!eZ=<fqv$V{h)e|!d6x`BR
zZRxt^^3CL<Zm8IO`-h+UZ0`O%f1);F`vm5Iu8(O>jWXu)X`W`^p3A2!E_)?)_W}2P
zDH|Ehe~djMmXTrFMJhoDWTx_T?&RB(St-a_93Zcyv38$5|D&VFpDdUERq~E;M&DYc
zonmc~ZL=9zS2gk&ICkuId1?71_PpMr=`r)gn0xp0FaE*i?%enB;KXak3}=hyL@8GI
zE<YMlcHO^TChc0Z{E1?wV<|IEhl}c&WUM=-lyhWT^J85{(_gJN1t$x))jv@^^K)ji
zvtrF-S=)|v9E+n*IZH0xk;^?b)KaUz|IdcpV~@_?;5OSdd)`DXk(<587kgGdTPFNe
z_hSi%jOa$jtu9-_4~kutDcqrX*k7Wl<BFE3*Oc;?f;0Qx?rv1*^>`<A*&#GYyYyH4
zr(=27Xa8hAQj>2SBAaGo)vNI|WcJQ!cW-Vlc6ukiN@>NSWz*|zB6EKBKbF}!DQvZ9
z#s{z2A(IMkY~6IF?^(O6pku-1T@|s%rHs7K+B9E{3C;_iwA`!fvd#%kzv=#$50>nm
ze8@XVecz>z@5}GEdItBuVV$x!YOdT|Ih#|vjjB49+}OK=IdvQN+>64`w^?KtCH_j-
zEc3LgMLcGv?DamGAaUv76o$M@M}tFr^?v_--EHuRXL8%N<P#}vA>Ws=w4K~Gar->R
zN6%J7si*FqFaNr8{?E2Uzl-D=<(z$Oa!*tqQo3WRRO9Ko#amw`x^BOj^7CJ*F<f7t
z&&vu{w7<FA%DX1|v)DA@>5B^AX=Oe=YWDE<1RwsH6HIQk`^zo)=#ly<^+DBPMG;%Z
z3AZoz?%h}z{`k}6$n$$HYTxY0JZC8;uh*9`ad88So|N94i-Ccag~vC(o^!TF<y!3S
zGS1I3XF{bOPu*yi;#GKVkzwcUZ9l7Byf0nc?sDn=(SCJ3)156X+n7Ixg`VZR-mQOr
z+2(y;($&mTQVMTp`osob+@ch){f_-KflGDPt$hd0WYz4VBtv$czN|d^(vv-1>bnl-
zYH{7K>)aAm7^q$9qph<1Uf6CS_hzm9KrZPsJxXdG@(RA!S$A(W2yzh=y?*Tea$(u}
zXFSbU$}&rCd6z!Ar%=$PtzaJK{`W$AKGWXS`%-&D{q)ytj$thM_PSAKMX7?>-$c`6
zPjgS~u3an;>JZQCxa{rjfA1ydSUJsE-`}X(xVTDkP4IaG*QL9w6HGpCnDbd+LD%~F
zvaWY?H@>NIEwi6czoFgGC#_d5MN9f%h*hM;=XbeLKV5g2*qVlJHM=Oj|Ng_ad;FI#
ze|mB}a{4o`twBqkoVr^$Sz^(3Kf`(EjNh4V2h1)y6dP#y_~er*^@^@jX6yZ$QWQP;
z(Ox6Y6Y6pfQx#I|qAbFc-l}hw4twkG6a6es#4$p!r(Ega#JH}HhbQjdrg+1prFPbf
zj`?2Z>raa+#`nLL?mL`O#21@&>-2^ndt4Q#EzNRqYSG%KwW9I)e+E6lGn0zsQ;w!-
zUD3$hFFLF9%~7_y_qhEt+JmmUUd?-&RQ*A8QlRN%5ebE7UbT9Y3|2iWVse>w@+Nco
z20`1B(4QJf%lX^>6nRW4c^R-$&tayW&+=66l-s_$#4O&nCRyw~{W$+%;dZV!x@8Qt
zwa&`}l6Q+f@2V+loqp)_(G6kUp@p2Dr8{I}EfXusII<T?UtO*$oPVm~glC_60_&pf
zb)Wsscc1OIo%-z2<|6+npW9mT!JGG}e3;SLdL&oqUYp|DyBBH-mx%d0T??&}xW;<>
zOqcZ&GczXf2&Yx6*X?~9{J3S)j@r-sW?xQzS{kyzs8}lZ-duB)f&!M)uKs*+64~BK
zf_vJ`f4dr;-sv_g@Qm``r8jaX*ClbA76gCI-DaK~|JP#u%=G49!-c1ozVle=z&%Uw
zPU-vBL;BX5%MUsAeUmOZzTM>XlpF88<|;`GZeO@f??l_%?=fDp8b8H%-^%>aEzo*g
z#qn#aS@pt<!`nV=o@jVfJmf!v><z&KyDHro`)9U1-zj?bI_tD^j|IXTrGs;QMW%#W
zDiwKe<cR7E*>l{8BRKKR``}}?O)qBbP)=H@C8Rc8=yl&}*)%=lkgh9HM@=uJPMUMj
z=+Fu43hkpSvu6ER#QvXQc~g=$!-Gr7SF%E^T=ST9r<84<Yd1HZ{b+Egf=k>#*F&zk
z{J~ypndbA=1Wz*cs`Wgvy!5!7PpNg;#8_X8>8<x)Zs(h@r%AM9L#m}xUev3EQ-zoE
zQqtuZ1(g^BTwPtm-pl>*etGGQXo{HA?kPJ1pYj&J)#G$3v)rc7Dbc|3&GlmG-&;ky
znLE6LEPJ+luTH%Yvdd<=N^sdzYyZOK4&P%GUDA&Sz9}#J{>W&dXt%)UYiWylUhG@-
zGVxqa497W-in|G|++D9a@B6)OJTS9IGT%6KWxo3&Srg?`{l0ng1@4u)<UQ%Aub*S{
zpTW;UxJR;X<&4!=*)py=+i%+O&`4*Cl2g*{?Gv;<ie5jpIB(w@E2o=Y|8^~Vdg@Ee
z-sqb%t{lpUt9^N%Q}~_ql8z3mS2wv#pMCg#w~cA(DV~UeO?Ntl<R))bytjk(EK9WR
z0aez*EpIEfembc2hH-}X8o%wALW(or?by5YtVL?Yq@9cEmmU4jAZGlleRB62o@!pM
zzORlquhez1N4!~7v&2+xPLsHB+uFrxn(J2GzyEBf;lr%Ev(Cit*%{Vs6|yX~<jmof
zt~s5@=1gGE)V-x7*Rs9##_{JB_ajewsQOk(Z#;TaRJqwK(d^6I4csxeokXtY?>ZWP
zxI1V{+H7vS&9hqd7Mjib8Z>*)1NU!}T6$_5;vaB+S+{zf+?@0oldOE@Y*ro)TWP$l
zy1!@Q=8TS=+}VrNl|KJ?$aV4YZ+6YFjUg*SGB;#y|0>X4?<R9wFyY8tzt{IJ>A!6A
zdfza4MzQ8cFYQh0EwzVoI#~3!OmVEJ{a$0SQuW4jW4VRbG?nFkd|=!#<G8!x#Euho
z50w{%TV_pKdH3<{ZKn4p%`y4dwpH|VqjAjCu(HKGGLrq<9#6Z+q`IJvzjoH5WWMcn
z3;r{_DcrDNx$}}MON{Pt?3uMACe2~blCoQ#cdd(j`47FxIuy-#AY<v}S0Xbnx@b&0
zQT1X)?~FL-XF(yM?$)~h8Tgb7F3nH4<EByB-J3P%O@iFChY7#xa`(x3>-lS~D*Cne
z$Ko1`_m2g3wk|lc?Cg?Fp^}PMEFMPi9@mcHm%H^h{?+e~%a83!d{T7l8E1P;)0?tv
z!^C6#G9GcsH*T?J3ay>*w_@*+4NF6ws85J<sCrsaZ0fMC;Y?*P>!y<1{Q5>e|K2zg
zsVX1;m*Ie=_rjxPA>}D0Nq%m>4f~p$Qkd7RUS9I|xxSe#Kc8pg;=WCLj-3*b_;N4z
zc+NWG+=aW|-W7JfyuD4^#QLbAzmcQFr$v*4%4g+N)v82Smo4kQtNUkz?_-hWq3P0D
zE+@sC^|pn5;&bv{ws-G8`|=vsCwopDzodMye_HfRH-o!sx&~FZPX~W`)iEzlB4+P{
zP2ameJi4l8bmr%>&#fV)FZn*~?$TYg&z^s_w5=b%2mhYkY%kvVoj!9^Gc=-TQP1qz
zlN{PyGhazbWyCW*nlF8<CCsz2m}O<CXW*}n1D`*!-{4MTOD(;B_eb>}cFDDe_bdKu
zTJ?;7y-3mGgAL_VmYw`)THIm2IO@g<b1m`B_O?6boOJ$ja0N@!ltnX{*St7+qG!(q
z-eZ1J(dk{)Uysh4ufi9>wu(RS)><C6#WuQu(dzSlJKYL0JD$`0bm{beYxnRQPAb3I
z5T00PedXq-^%}*u)m#@^=1B>1-oEiCAk;3rvtGtE;`wGq4$CU@1v{>EZB1QZ@iAe;
zld`DF*LQ`?IZtnW`{Cy8507tr`RZpcp14{~q_!fkIC+xha+Rg#DltJn+}*wg{8N53
z??1!oKBFCXW`ENCr}XU&M;g!i$*(@%o>lPcQJbH|8IDaH<s9w)nKx?;!)NR|bVcvf
z^3Bs<zTNl7EZS0fUGC%+>C!L!{=F`ob^WK@MxI@V4qHvW+%dynV8Y=hHph&eE3;<T
z+2vaQn`2ur|3+w-l;oM-6HAXTJ$3rnleS}Tf1i6Ov^l^b?o@sEM=_%c(;u1ZC+J4(
zjmeYD-I{WG-mS`KF2}o9=5@?k^twBs;^H^o<C;ZFHCEnCeIc#(^i0V&n@?f~E6Uz~
zI{Gp^g6Z_kS1U@sJv!WTrXuI;ousgj+;147R_%JVPr6=5XwDg?(+2k!Z5ErZa?h}I
zXMg|kkNW-n`$hF#&L4ZucI;K3`?8(B5<ESHxjU@aXVxD5us**?>HE>RFbRQYlfzyn
z?|H#s;l~+$P*=M8)%|A?W*^(ul-zXW<T+~jaVA&mp-uN#w|I(O{t&%YTcC9NtHV!L
zI_?x-n&$CbEG4*&cfu~023-^8JM3Lqoy$b4zGcspES))jDVML;-YF+$SsK1g+!?T|
z&zy^IVN&^bvx=P;v;LUhbl6d6tb5|CX3)YjFB6+5oUYxTJayqEE*n3y%&+41Uz6`@
zZje~E_kybg-^E`IdF6gf9!`CeeR|XKsp0$f<+v#pb=@^zv*dDx;EuyRlaIWe<KZ$<
zG}NAThGJjTjN4PDxlPD7a^CjZx;0VI%8g}F+uTd9MRt7OF3=IYA~5OlcG;<Mh8mw+
zC#FtzG)kBd-n6KCM-uZSku1TRFUuDHXSf+H_EwT%ddYUF>aYI$xmU_4&RiC{=-~3}
zD{Orhb^T`uP*>n^ZR_+2@OSwZJC`^2ahcMJh9wbOvm$!gRvO$+TJU-+N718m8*bPA
zUgzJ+-Cp#c!HippyVUFQxjl}7jgk*F`l|EV97@{ER_}gw_tf4!)=~2$W;;u%g{TM{
zzBFo_*?(B6@TIg_>ypJgSDDTF9>1GGC2(z++^MYxzclkG?kV(`qh{W*ol{_TkZat3
z22;O?cgZuByPxm9K1VY!)9v9Nr~7l>X>vuq*xzdZQ1Hl^u53}Ab&q%YPRjDJ6l!rj
zzr3S$)%?q>eTQy!e$2i(N&D<ZdEHHZht<CIq!gC-_X~V4_?%RAG-Q3;M6JDZRMHaJ
zGL2&rw<|9x-k*EzK(vX=#frO0Kdbqa4Mp9#eEF7ooLFxnEH|e*CB|V}%eHmS{(Ko#
zb$qVv9<d>=i+A$nFHt*t;kE7Tce2}lHgxQ(4`4IgH8-v%wk2Om>dC1qQcGpe?6~Ur
zlG%TGzuUyJb&}GH*1l6$z5IROe}-;3*R09DZEUqqCOnu@IcaWPilwW_j4IDfNzQj!
z^Y{0k5!mMvvAT}?8K2c*;njyZXMEWelW|$1a=A3qJGc9cBC>)ZA>r}UbYp}07tGAj
zI)1Og;N_9TEuU+H!{#1QUa1v-|8#5IZrhB4w7gKy<&yKcrkp<4cX#%kZD%ICsCxh8
z+x}w3j^(SqaIl9=3`??UPd<Np;+rFpaw<PoPJNl)nee0BtF}ex)VwoniS0hAS1qn@
zd6Hav#A@5F+Sg6_r~l4B@{Db%?@Z6rQ}(TLWs{!7W^lA{V%qPXb>8p2b9Jrt^`7=d
z*2+0IRPtMH`qKGzbM^AuRg(AA^0HD^Cn+sgkkekT_*2Xvx|hrTHw){Uw3h;!@4c2>
zv<lMHtrll`tQh@OIppI9KkfF9J7){9oohWMTj2R(t9!~Gy`zc`x3JhAFZ6c3{`hG8
zoX>@m&Ly+FUKL$1e=_4sW$Dy%zI$HcY^zt5+m{KRnY!YkX6DJcEDM)zm3)*qPh*qC
zKNXkQ>WqE%;rpfPr5_*uds;G^Vajo-wwvy4{LFnuC;O*e+`{rT+s|uP_OrwPi~?Mm
zMLA6DBb}#mudHmJq0Z>Jh`ZIN-k#;4IV-o*Td$3BE{)5&_Fmp|QzZFE>&KmzFQ0~9
zG!*;WV4S0893?hSW8F1L-P10Sdpe?3fBlO%A6NQW)a*U`nF&vtBo<3N;Xc1jPf(IG
zYUNYEdn$W2R!@#i*y_Zymo3*}wZfN7pBU?lYn0eJ99&JWPr9R?-)C$W5tPDoZ8J*;
z?^Hbpqp)fBF5FH%Vi)qq^YJ?V_LJ{#@I}kyEqU+rrCM$+<Aj10pYHS<upE1sd1$Nm
z@jGG3_diWf=T13&?}&KJvh9)EYAPS{*&exbY|h^cZA<n)lSwVyarddEhoaV&+d*8N
z)9PPs&th@<a#HH+x<XOEUoTFG*D3$GE><~bZsxk=n+^HSV(P-aKJ48S*4>}Dc~9Gg
zcp+K4Kb7mBb{cI{G4;~e@+Gh_v{&Znv0cmORjf_E-SMjXyMC_8;p>mi=mab})1m7+
z@mt0v6`o1CkMG783N*)F6b(C&v*BM^G54eswQQ4~ta|A5?Buzcl9Whoj(`$hfq<#O
zVn6I>XrA<4QKWNb=Zk3psV~@r=4*a@b*uSXQ~sUBM?)*rcb4d1?)N_P*w$h{U)pBx
zD9JbS?xj4lr@p$m-qh$!_^aUAedhy`#2Pt;zFn=%n&I=~{^=jPUVi18-V>-9a-;h4
zt$AM}*4yvkIsWYJte4MS-v8P7eBEu^Ms4n7BfDfxzEjny*IW;V-Q!Eqa_R8=ciF|B
z@oCTN(~o~031z-Aaq+J$<-QNQp58s8t8~%y($~<f!SfH_7c+bPO(02Jd&)$y)jB7O
za)NG7JoLIF_X@Lr`!>T$oAl)yRyI~`jYx5c)toqc`|s_%EG~hjdu?h83?9CgSi`u&
zV`WIjQxOJ^G|$w>b*DS_-2eN->2FFJ<4)hXVlpQ;&3pFwiIv%hIOT*Jv$U!=E?oJn
zNwWK<ouunj@$$-zJKsLqcS*tF@~q4ief!1c9~L)kIkq5ZnnBX0C#U*_Dt7g{-ad5B
zv#RE|oYAiCn*K<c-s3Absk_#Ror(6&ouaQgXZPQ`8^faiGX#hxY!yA!;!?CLT|zu2
zC$}WO#O)HdU|{&B-(TPNcIq8`b&%c4P%_B2XVDa=#~u$j-(?@W_wP^St;t_|EGN3=
z?G>H2RQk-6pq#M#UAA|3>-USY9KL?S`E>5#yxtj0Oy!O<r0h7Ym{=KjN#xD8PZxyy
zZ2Kk4q`7Xhb<~!6blzmk`Ra7!xav0X8Mlkm9xhw_QBH23!)bx4PQjRoTn1)E95E>|
z&s;N^UM|-)H8?0+xbu{*9;?$zF<0+vLCVL{7V_SBpu}WlvP^c}RhO`<{~5#-1)QgD
zSK<%JmzdnwnQQ(;*4^*)yN~}FG#(VGF=qC;Pxx?VX|tZ8=)%+&?<+Z%+Sb^VIu-k9
z9^EXi)$FOYx$kVxN5}1Nl^psebTM=%NHKhT|G4~*a(ffwME|O3cdQ&+PEHjoTT`WB
z@}FUog08tq+;!#7RZ%hf*30Ogh)SFGEhcx*MZft*CE-%)Dn|==n-e-be4TeyaJ@)8
zF2k&3bL0NExZn$l{#B`#Z*Qo|_#PDF@$nE94Jj}E<J=^*hIwVxlE=p<ytb9OJ|p$W
zu1A8Zeo`^AU-@NJ_cSW{tDMrxS=XWQ>`D2Y+Q<BNg;N%29=N_UHvOgd2CpTBT?^;R
zXnc}*&3od_uCNI+*stFZExqjjx{al!@Z~0{(}sD*6|1F-%)8z8o!{k|dHbpFfw=H`
z9>;bQ^Lee)qL;gJoaa7zX<>(8$$`J8)~VOU?oP;7tlRiwwZHMS^x8ctC2!3CGdvUf
zWR!OM&D)!-GLN4gPk+oh#qm_ZO{pv8zmB;toolAKI`PE1{XS`>OK%=x+V8kBAl=w^
zm1tUuf%Z0Y@7XaEOPzS_)eRq|+~Z&H`Q_KPRiTSkUt1z_D_GSmd7X1r#fk6~I~(_%
zlh$63;Qsk=ZTb0#$Q<@}A`xG%btt>f+;wBmqOw~a?^fRL&vriD^Yxj%+^vIO=R2;m
z^i6X(QE}(?B&XAdE^tPKE$aHYcixUGE4E5(dYd_IiTJu?&k0wGZr4ua?r`Dh_)!=B
zI;&6k*zOA(g3ma!<%)5y^`D`)Eu5z;O1Y$?;frzH(fdzUSzKN1*7=x0c>3$fZS&<;
znqOPnE1zbQJ}>P;!fhshQD0GRe}lWvj*9ND<XmalxNEVI@_Xm%73EdQM|iv6Pge;E
zjpfu0`M%=&`_{PMGC^$${~4|vGK`q?HA;DQze(yF**z>%nU8GoOFVsCc+$$oM6Y+2
z>?@|qUh~~hd3sjqe$!jo`K1R=PGCw}<=l1JXou08aF1=C6L&Pfj9=|rux!Gaj+cgU
zoIY2X>fJYQ)ADD%-7@#<#lX_vAAAmTU*){`%9N?$&Ggw@7j4+Hr@Qz5!y^x`r{7bY
zvD$gb6|G6@Cgg0oaI)y_v!0$#|LuD%UL>Ao-umNlXkYTqh_%O47o@(f-Pc*wu>Jh@
z`b#_XpKbX2Q1n%ZR15oT^K&vm6Z2kd`zRQ*$aCGS+JA4t;*x@|@!d@id|~j^qjFW`
zraK44B`4)o#$4F<*G?{CQf$)FuM;=jIqC1!I$fcZ$5nUw9N}Pd4ws|s+x7(g4Zq)Q
zJLhz^nn$YLL|dc!3ty{v&95#DOImW!>q5`k<l9kA^TVd;ne9sV@QRM*o;-<FbLXn?
zC3|+iKILlqD9QHh1OZN~8~Sybiwds`&;2*+X~DyF3D?`6&3s{hX|iSLsg|mnEvHI4
z&KavN=RbSeMa8$yyL+zo+z$^w{%rKzI!*J-t!tGNEj~TSbLv(vQZ+6qzax7oSnpTf
z)P2g2dry7%ZIk@PBJfP}{8d3Rt3o#2nUQeYNv()cihIIV*KkgklK%{1Hfw*?n@{-9
zuwiY4h1gQLZqF&M_QnTh_-^rS@yaM|QQ0rF!|wE-wulKYKRi7A`k&H9vAD`Thg{z3
zC=2r54rh!ns^OR?m-_v&_43u19!RCiUzAzt`ufe;&_@53hdfOd&)f|<B{R*Yhq%01
zJ7b+~y7YqM{crww&zFwiH?B%q_{(gr@2PqHn~rZ^C3ZWg=4h5)+spYgT<v*xJvv(1
zdHCx;=3Ab}9c;6$)1JqqZRXpnx@+>`o8O!>a<|`mw#u*kZKm$Zp7`h2mrqLdpW-~<
zRylpuw`eKZC~woGuHVAVc^1uYr<(>9dX{W)KQ5AgHTTRnMXB8^;q6NcPpsd>+aj3b
zns1;eY#4CepLJ`g_MRPG7Z&9#e_&XX*kSN;x#Fn}K6lD9kDZ<7-*Ic#%jLDD{*^_C
z#9p4MP1r1OMQ-DvM)Adkoh-N0POI15pAqom^<8s=Ylpwu3o>5}TiC5~J?>*ElV5cD
zR@s{BSKEHnL>%86<HY%MvDU@UvQE67x7PgF=>F!F;JHg%s_)DZds~0F?c+Lkey{c%
zajwY{O2_+@FBv`AuxIU$EpOs4i+m}cE0`)8rCa2_R=v4hBrk!(GJf426Q3O0`+W@w
zW&y8yPb}|!&k{E8+j;dXVbfMGZ!f)AV<@$o$?2e9>hhUc>^{~_X0s>W+IID4!P~C2
zGi9Dv$uLSh5uB&ac2(?WR5)k!`sb}rCjUO@^rWiUbmoOeXBU;EM7R9ue#3sk-R-E|
z8{?-6M>s?0EZ?tql7VN_g}2?$Zg$)<7kb6Hem3*T{|pzh4Na<+RIcZ6;60SPZKio#
zg}KWWPMa6IYQH`FW8J*v&|j_8A7A=Cg=9<T&e%LFFIo59z1)c=HP>Br#S0AI?W(=h
zk#n+ZW4g!GfEP+<7_$?f-6`2BSlE;uZDT!u-<_EFHPx*eIp@B%EN-ostj9g8vwy<z
zkjuVyIs(pWNzGwqw`Ux%xl@sxaW{LJ-w%z5sET>J`u2ZJ_$WSOlgYNM(6kpO+n4l5
zzU8jEa3jH|<ILrK(-t+=&0*@X%iHMMJ!7wP_<}jlvnIBsDqb}1%ikcrW!G|1={xMX
z`yUEE?6@R$Dzs|l42hNDY6tH1nfy$(oHpk|=Qa_O5NFjO?d>8#x9k4svx|k_S^CvQ
z|Kv=bxJkE-Cir~Y?&33jlh$`>!TM!lPFGenTNb`p>rv(FHBoWd<9FMVk6B2ro_jPf
z;P2vDg`ak=e|jb^LC9yqoxp;$Gq=0tJa(Kp_IO+VJtet|HG9_j)!WNhZ!pM<o$}Oe
z#g_|dmzV9o&OE2Vg<%KZ+$(wi8T3R9Bc3e`Y7)(Au~^frs&%nf_wXK%np|JzFUc#!
z60BN-=8JWi?EG+uXPHZzb3o*y1Iq;h3M#p7O?#wJ*%LBdu~%$qWEsn$+)Zi;cUQbI
zQSrUCy|N^n=il!QU)7S$B6D{zi?86D^3;2m^`WwfVkcH{Nv*2ju1oo&+1aryN%FFa
zsqc*9dz1NUqAJZ4KKoklU-~p_#VU^V#eX80(zGsnxEq-7e!J~;;d(*qb*~u0?|urs
zEVA?drPo~lOibQ~ZM&9Kd~NE6iDC(FTC90I_eCk(U-d)Jt~M@hza0DgH?w<rt}MLt
zL}Z1#>ht6s`U)0V(FuK-H|swpzAj(<hVk`=_%qWzCi1<?*6r@?X*K$`k41K(jrjI_
zGf5uJ+Li<BZan=o<&MVI&Pk2CXLFQ%xUVR(QOCjM<K>4zhZE<oG5y##^<lHzO07l4
zXQCz+Ykg?k^6mxi`mXJZR%!kG&!BFw&d1Jv!b6n`lMR)#mUQ;~c1-a(I`N6tEXh~f
zK0lC|_Qu@8|CmE<&}`mSt>){~UR+YP)HwMvccR(VC+}X(Qd==C_v67V&!3EEmWyPV
zxZA%<NY5%>-(h8_ci&t$?p4&?<^44w>#e7{vl}dBYYtoOA)XMB=E2Fo>%>AMr*P)z
zHrcJ0cQwZTXVCjGz4?>CtT{<*0^H`h7rY7$*wNBnWqoGG#s%f#!Xbv2?;mIUIQe^T
z>NUN~DP{$(3nraDs(JB3%$8#Z-)vK=^{(U3@(D}Hw|UZ3c=3_$K9}S>Z$)?CefPmf
zDSn1k@x?P;zGq4f9+Xqrd!obBGvDR%K|?WlnRkq7kEU-dn;3D&P~5y~twY|9tBkrq
zS9N`&cO7edB2=|EruO*4;EZn@l&ntbr!Zw%X(?#M#M!+G{O2xJ`{>B6wuEM-EuY=5
z*}Oi>S@=Xb<MmsS2{p6Ywk>$sdHCUirIu4QU%t!nubdWN`0cx#!rn`Ye7BUMx^MFz
zwGZB)!+mzM^1{MZu5V9?aj(#uqqoyLPHJKH_vu}ao*#SvQ}~T?qNDGA=MR6C{T5s)
zmQ6TTs~3CCBgMtTe{-*X{tCTg&wo|Vnfx^8iP(de#zAN2HQhe-J$zGqunc=wi+Ka<
zrVm?7L!=Aet=oTFFW)%w!kt+Uz6LyYop*cD<7GMvn4fg7JvM87{b@;=b6jGfL9Iq>
zjQ!T-b=z{kv(IqRWKnSHvf{9qyUxYcWvlD?-I7ABKg$0z6lNZ)ob~u#|I4}wMR%f>
z1>Ku#JE>rrNoSg~>iXVNRh2vWPv0NixB76Mi%tF?)-X43dBc>-kF`-ps|>v=+Lzzs
zZOjpWz*@+5r?UR3%q{z#MW?=)oOh_}y}jo4tS2FhZN1x7kKN7O?s8=A$Bp;yef@BJ
z)yui27B7Fc=p1tG_qlm1WYzI+O3l^R>wf3_XXv}0-~Rf2xBM0R>wiQp#=n!C6BQYB
z@}#VRt(onTCrxSp8B$u;UC>$;-OqY`S5uMx$LIeUzN8jg+AAL`);6lTP{x+PQ&ckV
zmVkHk9=9)k_suTW|EyTf_M`65!pp`hi?jU#m77(wc_*+JZ4u;{U>a|_v~GIrvm@`H
zR2|di+PT@>i(~D1$^Q(gIrEx6`?mi1QYkRm<?-(=KPxvTR-B(dAu(R*sO<4Jo#Sit
z^vvRS3F)&2zhAbvtteqm$)6n-y%%o3U_G8$xIkdX;W=}@EsMId!m~_nyV7*!PM&44
ztBt;dt<8xnh-193`jI#M*j+oF#Ou9PRYwyf+O0~JblM+Qd$(6-?o)d);pIUQasJ27
z@f*Y$uW1V{JgA<Xkn;B0vE37+JlD@Z&bsikmEPggd0|<tZyOqFK1nm)5V#!87;5^x
z;^CY751gLF|JpY1{H;hqrLBkc-?ipmjywJ7z~*d=Q!#D2yJe>*_@srN6_{LpcYE$V
z+l`S^U$_Qkd`K~|ev<G}Fec&Yf!w0Y`}60C>PZ^#^Xf^u7o~Kn-M_hxJ8y^gE5B=x
z9V;?QR{O3IUme%4I4x4`_RGhO44oEENz+$NQm+nr6H%GN7<uSSkD*58BE2tBJwboJ
z6#rNwlJh=(_cX!9f7r_;?Uy}j59;<@eZ1ntJ$0oMl^X=^2svKK_dj&0zkRmUT)uZv
z53U3++H;5F<FgaJJFjq`<L0+5G`m#6pL%^q^7UV5%cWPO@~$xXxA5)pq?2NIZ_atJ
z>#?7v*4DK79P4Mt_%!i{%zpXs%B=pN$wn^aw=4IaWP8~mo?o<JRoTzG-yiPeKH$94
zV<uCI<c8Pm8v}X*;}q}o2_?R=vUt5@g>USEi3|ViOsGxws_V~|bh8X!J#*y)spmm~
zNgm1@BKj71&CdPAxa-5`<e53v4GGyd<Cf0Qp8Ue(-Xa?_iKSar`fmz0&004rwxB=h
z&fBMlUg?W7Z<b%suQcuNn}c1Cb9XB~kna5CRB=`?ko)w*^*U@UN1h}eozIiy$Yf&v
zR;ppsZ3i3k?&Zp^?;qUy&%k4@P^gopRb8BQ<p*zYWFGUO2gd>@W`_$WMSDF|_~63d
z{pgT^r0NY{wn?FGSt8qbxB~4jH7`!!c=<4C{SR^8RL(uAU3#%vTOCiCR?SvPv$585
z&b_$LBlD?)i@v4&?3c$`pVWLcow4QORJ&Pu?s>kKIT~1Qh(0~PU*Y+kt^XO`9uAS|
zKF^z!YqyXq>~qP*Ifsw%v@0b~uRHxUa^3!U3f=*WuRdS(Y0;G^j!R0LC3YnjFfeXX
zmF8p&cYXU_de4sLhRPcYPvj*Yp2u|6vHfP9XO#7<T^St-bK868RI7=dDQAq=zBaGi
z)N%Fh&2lLxCaT%+wlf*l1aNNoJ(*+Iw@SwP1-F+}Y2RPrv$|z+m7~E^hs9I6nJzwA
zG+qC^oVQEHGi$BD6AyRBtaPkg?)13mp?WkoOY^jOc26%$@~k)NvpV%9teE9uj)vjd
zDd!%Vtdm`s8GXm}N_FO!hkGn7Jb0HKmDw3raB7vv_E&wv2c>M9w|T#4Ua`2Tez&+_
z`04IVPnM+|$kWnU6;oB~6*6=3!ZV$MFS#qleSS||y!vhY+QX~XAAPr-_Z9a>_WeEU
zLKTfPo--V6i}|o&t-#ZFAu$Q}_`>xar7A39^G=`dG`k`8GVXzT>6_)xF7wZgXWluV
zr+j5$Oy<hz20`WPs=XsVt>=Fvaps)l-*jfVUEPY)c^<e47H*Qy`Do?*_4l78^WK*(
zzAwJ7i09U$KbFiPLR0?!)v4~f_D)b%=AuRQmg`%0RBLc}_&F|{aDK)K+dVGNH}kP*
z&APR!%kJyXnh5tbec?r06_k#PEz6y*=Oi;<Wc$yD1y61~aN;y7jB5zWEj`w<UM=W3
ztM2u9IXjEZ+V|RvRr;QE1bA5o&TBf{7ghH1iQb*-?>;@67615t<RQ<?N|#gJCtkT)
zb$nLoX7_a)KMK8*S|mPIJkBclayXw&8PAc}DR;9(9ZxO!-aETxM`z=%2Z};7%XV7-
z{y4cZyZ-c?BugQ=m5wiOS(lwCTlYC?<?{ULed1@r51-IIG_|o^PNtyovb)(0#orcD
z`Aq^#Pc51Ir%op3-k<&m3#Q05UZ;(?G>xwsJQR$HZqc3eY4^UlKOc&Q&;Q40p0obt
z#GNj8c&>e!&Stw#X2+YA8y<7r>|4iP>hgZ=x7+2lh3~EEqdkB8WWCe5*)8%*@Vo_I
zXL_4mT6Ahl=lU&zhPNNy-pl;g;{J3soBrtk3<BHcC7(U|YS)&Y{WBy}d@7pP%#tZ*
zf5x?|@IM2e)~44ci6^D^N<8PeC??ah)6pu|<lZ)$%(soqi)>0%b_V3h*4&u%NqkF9
zqq<EM_qmyN^GtU}t6Udi&byuK-)UpGI@C{o&8e^(Vqdre&dyBR@qE?uFMHEBS2kJC
zm|TA7LG_pKiK3sR1VUt%h)8jla_x`c3)gb_cVm-G5?9ehf$7`3wEo7uN%*{O@}t-D
zuLSNbUC^#<?7=R*Y}dKR{T4cVd|yRZ{o}j!pTVZ)=Z=&A846OhzPGT=_vSF-S!Sd6
zQ98#_y7uV0eg7)z)0J;D?5x&1*u<=qV>`3QIJ?98@D=k#ueK@OikCkvzL2+W!_=))
zi&a@CHyc?vr>|9OJ+|29bgt|vt5*z~C#(NN^v2d1?qp8BaB4Ekg5brnHfgz&Z>t~i
zsA*ocyfoy{)c6~MMa~5;FGTlUJALfjLxrDvM8e+Iy<uD19Fel@noPUsvz2N5eD6b&
zl`JGW*Y{0V`p>Xj@^$!3kMQ-ze;2Gadu!%>SoZkQmud>;xvKk%YChcEZQEe<MlW2h
z-E?Ah#hsTPE(|-x6ED57&5{h8bmDyFiA@dCi4&C+s~$d?*1K@ggRJAm3|Cfe)!rU8
z>AS;2i<@RQd*4s^=Ks1O)z4AV`KIl@#<$DnZJ8-_Z|7uTA?=!n%Xj;)7Tv#GX~kUw
z_r>2nKlVs+-z==XcmC1a`}?+)9S(1sQJd~)x~kU8Rd(kFl^JLAZ`rK9;}G-ZgP;9n
zR*oEnMXp+*o+rMlZ=XNm#g4w?Pn5#Gv$~sH3_mQzv!M6Nti_k?Uu{k~CY+r0u0|nu
zlKhuLvyFF7eJ%2zAu2`c2wR`&nq5JR&vo`0sx~c~|602~O#jQbYfsNf_FRmfdHF<=
zje+=0v3X&0ww<2sE4+WU_wFB$ul{X2XU(>%;zZVhFVPIKg-sLKD>yc&J`eR@b-w-A
z11FsjW9O0#rK|7F&9tA2IX_l7_DD#bXQE(rX59`m))}FzEvG)8?j6XSBRyRr=x*Nh
zY?tk7)5Yfo1ls7m?d;h4iEZ(Pdn?m4``_#_owb-(#OzJUf#el$RbP28n0uh9VV-K^
z0bM^2KTGfCzw?$hUs8YH6EGppT{V7jTh_{Lx(haBp4q?qU&1MoDQz|}yJvhBk(uqC
ze%-fJKZGgXnj=8>UdhS8%G{5C^vbuhH|Q)q@inV4boR;+ADQJLGhQur7Jizs(6lJ_
zzN^@6>zZvlZar=~5p(*J@-vOXs2Rzkn~FjLb@hZ?bRDaIlz;zg-_3N-@Iq|U>3r9I
z<*Sz@e}{DD-HUB=PIFVOP%MnD*;D;{gU*4g?HXITmRL<Vv(A%iy1T^Q9W%bVsJ*zM
zXAnCf=-d-IISbW9MoqsoPF8k<voqCnD*4Q>Gpjl2HchN{x%Sp%|J`MB&dXjOd2@VK
zh~-1R5OJ=nnoR#0g0xiDFM7E;^^Cr6{n?h>%E<yb7Vl*amN4gOdTRPCIL;(-?Cz;W
zop)v{eO5|bby$V@>{0o3?Q+&;?>%!$ZwiVtT**=8`!oIQNpGg7Z3(u#(`^$ySG)4x
zxpb$%bHjgz%~E`i;@4mN&(JPf^i$4C^xHS-rw+5tXNzfVipy|`ygPxxPVs)M$ezLn
zR?YVEQ`e=LUf&}<bLPaL<u|=6%uil#ci+LgV%qZd*SooIZwQ{SS8U_9!19@zA8&k9
ztmKY8?IL}-D{G&G?ynQ8!WTcC_E|1L_j*s+@$CV;oMuU`3-UWxMBjU)@^8kD4gQz8
zHx!+_EVFDoUogWL^T@@)$+J@z3e4E{%5p`WuGFXc%|~agzWn|l<JCz`r&Z--4ry#Z
zReDHogO^pr4W*y!uRpziSaj(VA&KQ3jXqPg`h?gsqEkPyD9cs7;Fz_p%cekdlE7mB
zDp#*14^E|=T*)_KOAD97v-$c@yLwd?-jUgB6n0s^B<;NamfHzur|tikwa@j)q}OVE
z?%rZ|Q=+mq<T*E8p7xG0zPzvV_OYked8Wvvb5GT(wZG>2<dF`iW=z;CEB!XMO$vMq
zOzs|^anF2i%4|`yuY%j}t-k&7LAJ{IV@_Yn?ilu|CCV<!o_F?;*xFr-UUe@jI9z;V
zek3QyyxF`bHr`81W^qyZ`mMQT?N8BgAI%DT<!gm057{z{B(|<OtokgKoA=EIt4=$w
z)I_%Kqptc#_r%p-VLA0#aYkU8Mc}u{9JhxCvX)lcUU4r{KUt8iGOOb7iTKlNzkbyy
z%$oezJz>eVlOE}(lnX^)tt^)>(>mno{)wB{?5rAVz~b4~jS5wzmJDn;Sx&1QGFYao
z%}d<Q`>+36xq(g1d}|YrqF%Y0G(Trqr#YQG#~GztuRi-O|GL|te&QQ7=9GhJ0+;Vj
z<X>}B_Qa({o0wukW^EBPQMqvFs!Z4`3BjYkV&!Jr?!1`QcH-_%&509bV%biLl)hh;
z+t;-}T<=Cc-}~cVcM6=SRSdeh<XlRo+{I3Di_g_Zk`63CCZ6RH+yChJ;um}7g$kd@
z{BZN-8}`dZcWSN#WNy~Jl&LCvNpIn7-CNrEelOo&mRr3%Os4!(o$9FrcQ%_9bq6ln
zKBX_^<dhw`+s;@D^?lyr5WMT#$IU!XKATwxHJtQ6SyGkfpSS7Ck24J&JNTz9&7bYh
z_n+Z1`^i7M#I`LtQgtyfwkWi!v%3H4T=6OE?+Q)a!aJi$ud5?Uo!4Hc;@#e3d+w*4
zsY{x3IcaKI@U+J525!%16|VMKyFZYB{(Oo0D{8wWSDoWcE1tB<Rcys+o!KeYhaOb6
zfBGKXrL}!V!p@ne<{K59YM0JwUCwdMLXnf>cGbPTzhCx77*DXV+_X@0y3)QACF?k*
zwSsP{(ayab&tE4RcEjmQ<)m3{cgmJU8=v0de#GU^<CThK2eor}y*|$rn4s`-^A$5E
zg|+S6wZE3%UzJdFP0n5E@dU-~qRS#@99h;f@%gOGMeqMzw0@_4|Fp=%V};yLl#6tt
zjFn|q+TDF!_SVjtOKIi({ilW94s2jJ_wuP%*X7-}{VUJT&MR1Vd3%liwF0TLd~2DK
z4U?a_&v_-Gc72uo-CIlczdE{jhM43!*9Lv}(s>JR3l&bE^XcB)-}N$~le1aRWte9C
zXZTukYsv)ixW@|`8E(b9r-p|7{%|a0p+HgEh8q@vx8|L$b(pARyON<xZF=VN*}S1Q
z9$F@(a-8UvlaMjv@JLxx{YKGAW%vE%ZCvkfK2~D9owmS~X-dxQ#@q!jqnvKui@h9Q
zyF=3WqM0mb=&Ex*-Iou)oLSSc-mGq^qu<MS^1GS2_bYy#Fk{Zlkkdt)YvYuIGS4kp
zuc)N$QhGV=R{x&})pgrt`6YWUL{9XRoS@2jTj{Zx;2qbD>F=(8Ro(IJj@29IB00we
zRb|V#1NJU3K6>3lPqufH3rnnxb>iVqteaV<XX?%gPCYQ?r`J7!otIo<Yk&P`c+<|p
zJ=w3{w<R#q=Xc1*#ADI@Ja4rxZdT^~&#**1|3Aa!106>c1J*j8zFCrH$|QZYswHol
z?eR*pqeoUf>)Ly!=iSp^HHI8In_qL4tSHG3yLf78K&P4Wru4XXHy0neAf@!`dg9}c
zeA~5dA7?OR>xv6K<L<aWO}aGeLC?uaX;NDi5)`Zy4+gqKtGXKPaoIb+{fOcOJ<io>
zFHBFA+?-jgd$Q@MZtn~sxhp4kJW6@ENP2Z8e~Qah@9yM|LsPwvhD{Ax>zWmkao!-%
zyk|~^2gg+ReRD7GT*RIuzG17XRkiM=zljR^78{RUveY^8s=4>%hlWLJyDD4jbi`QG
zc!QHK`&mzDnyq?l-JKa%3KVx-&^6$mDt7zU&z}X^3XGdL-|RSXzI0|^z=>Uz-P*Ui
zc?F|)-CmXQZ12{0D?j{`jc}Mev)Q2P!14>P0yr-fKFrhK#-4IKgvB}IPT=+S_Cxz0
z9g=Cac<<PyAF^xu?0)N85|hniT>mqaXf@4}Qd_j^&W;(pfqF|mD=O)|e6zJMZzAJ8
zrQ5UQ=69Xv(OYBm&uCx#s!*c`2j@pGyX)5#D}DB6!4AH6Ta{h>Kfc(%oKf}KZ=QeW
z@8w>JcyBm4dg7H$AKw<=$!KZ2a_Peb+t>Z-SAOL0e<JeE>1<TUa>?vv6|Rr8PcPqm
zN~O#5@%r<BW}LhJkI7iyWBIw=i(AqhH7{;Bea6YlYZue<ww0&1NiR-t*9*?NwA?kr
z@wxoYiR>NU-FsK8ynlDu@?Y+vqCxX7&kem1d?w_lWS!lcj&$Rkoxg(&3)k9+KlTpZ
zpX2s8{wu?(Y+d)fSwUSnKk7~=xH#l(-}E!)=U)q*wZEB^mrqq+Sd<)mc<BV+qfg3i
zFA};lD`Vx_`L-30&ZvA*NNNt5DYwD+?)-0I<?m9~3h(skG;zIjPw~e_(V+AqtBC>q
zcExiw+$7dcZpuuWC|>A4D{1Mg{|p;uGG+C-+HXIRp1aI*|D8=r%BNjTU0mOH8)aX<
z!90D5%@a*lGZuphLUW!emruzHjeGr|W5)Ss#%DU*&aBA0roys<P1objhJAt;Sxe$i
z`$VSwXJASTpK|-+9l80p+|^FbofnrEm3{yIGQqwBO_6a+j8^v?E&G_!;u)Fm-5q!D
zOI+ORja8D*H`O<2os&8zw=Jo7!u(a6J~MAq{LfIJc{;Wxc<YRvSM)y#N8~;cj}~*%
zbeNZR)6DGMag$B|8TRknai(d{-=>45jQxD>js`sQQ#z-LRc>(6@Z7aM`|;O=P1l^f
zGTp`Qm0qq$-u{Gr2kXiry<Ig|4J!-&I^5W)`CPGfk>sY67QPc3?l@*iDrBra?Ok2D
zQJViU!=$Y<jowF?dTen{Z95THIPI)Y^yMRy54ycQ_j;XnZ2gXMgBY2!VzXu6E{iL&
z=w2c=_kim9y(XWvl&b$!KYF72$u>E(thTA!&~3fq^OVXZv8DU#Z0CzfUHBq2VYPvq
z@DbknvKaRd4>$cfw&AApN^_<8Im>)zZ9je{V*3K`>yhpj>6!su{ZrMy8XJ9e{GPn$
zyjjQA?KuG#OOJc~J7CM#JZIBa&zG{2d!19iors*l;_{>S8tcF8_E!gr@+S8`<GLDn
z$>!yC?ytfXU6vM)ww+vh?f37;&kJ@1Ju6P|G4;qhKIN~%!Q&54-`vyV@mAjU*N-0w
zpNm)}Pkq%nE#@3@Q|I!zotcbhjI$i2T32zc=&JqCU=uli@_St`PUl(If=}+On=JO{
zKLf+EjyF^Mp1heUQ(=DSs{HZt4~JHNtd!fZ+C=Nrl5IyC*?&zGyQNs+a^c9<pK2>x
z*YigPhWrZd*>HDV`@178J8y+6yuN=t{y)QVt^;}pzB;8;&&*ytIqz~_uJDrR#$|i=
zP5iSlO|-H>`NWEF`IWh@41FxtNh#Oa?kZh%4QHOPT|(*j@{27grHw|;x!h^|?!wYn
zT628G`(GV6IH~c+sZ#x|g$azC6rVjmr26@2+`|715|T}~7rZ;AAl|rB@I;HbrJ&G{
zr0snCzxqt4?9`g?DQ?w%=#hNRwYx`LUiIxax$J*hTx+F~)`ZJh0f&NfMeKSaw;isy
z()L8Q`sLbt65ICw;ws!U*@%0Kxr(*qyA`pwcWy7+!8PgU&x+Tdj-+)82nTK9d}cL4
zB1y^3CB;EyiT(85+cd7N{FUxyx^C{h)(!{p!kzD155>=uwpV1Day!9Qf%Dl%flZI1
z$^_aQ`}HRl9@^=BuaxKNcFt8(J%2`@GdQ!o>`w9N4NJUs)z6nZzO<I@xn%#G6~?}O
zrXD`?<NSBO$=B|f!kvGA`F<(pj!a`YrHYk>4YPJHos^WK@T?(gmF&LUSuY#Eaew0z
ziL$u9is!3Ov*DwTdoS5Hsb5`Nt??)JO3uHEJjLxjJ7>yt8%|$-bSZ1pW`+40Z>Ka0
zyg0%dc-U;tn|s?AaQxiB9B{r^;Axawf<U9xwvR_!4(GBoGuK5-zI)cfY;E_cwOg9B
zpNgA?`ZzD-j|-i9x-heH5wqdl36G7mlv9uDI35i9Vz^b<SHzRQuO&mSPV>n6iGF=w
zIzFic$G7j~X??r+k&=^wRg>B-ebwzQzcw5Vm|7;}I5(hGPG+UR&5CD++b$<pMk(=4
z$lCXM+L6aiGU1DVovQViHPy(c;uGg4fjs+7vM!~(o`I|(Yd(vK-80&)apKOQrO&)t
zR(@71S#qHNMy_x{!;v+%y1w~^)&F|TeE9T=`4%cIz4T@Cp<;n!SC^bJEtxOJy0~2R
z)vTGXRHAB?BxXBk9~NJly6vfhvDm^Zk`fbV-`m+X<!d1Gp2<gU`|jtpThCKq@c6v3
zqt;V3on)z)wc8h%d{B^aKm9QDg5Q<f{~7!bKW=+^L#V^@5Kr5#&H$^VGV6YujdKjl
z+8wUi#k~pKEj;NqgI@S*X_FnH^Nh8a%r@PhxlQN`-+j~llHs*!dQoR5t6y8R+q<8+
zLhh!JSa&9WfrCd#O`XS#^LrY<@m{&LR4-(G*soQxj~+~GI`=Yy?^TCXMNRh4M1gf%
zuGxiF_MSb}93)|KZQ<c%Tg-D<%0jxHotWsqbETR0_LF*tma5NwBk{G0`8Dt5N8JjS
z&%1ZI-0kU~ojqY@{JW1oD+{(Y75+`R(y;SlO7NVCzU8-h4(+greX~V2Jht@x+I<!^
zt2f_J?3<Z8d9LV6x6g*ItG(wd>B~BYi2P?r+h-pqy`|};&6BW4hbBg(XOy=}7g>qk
z>2yB)bbbG9maS^4$5#o)t~5^fz5LefzMPB75{-r1_FmX(tiNC6)uJXo$L5QhwQQdR
znRDdXsh*A(@d-P6g7bX8d!v6&f~VQx9y>L=daYFzKOTM7ZhoVeI_>uApzlv@)hsT@
zR_@Q?vTkYdxVinS-i+|WRqQLDMM!!4dSxN|X`z=o>k~QEt;^=utNwgoYHN33qvC3w
z6|A-IEiPKz?iEgO^6yt+_PZ}M|4+n*J&C*SZ1h|aveJ}kmFrUJf|(`B?{X$g=6m77
zuxjPU%yp6rzr0}<%H+MGHCZ*JFZpij;yr$>bAtDGX0ET_&9Pqc%NxrT@<uXer+u5p
zw{l~_+MA9_itq2%{#t&w@kiOhqs<rE7J3$bpSiNqKRNZG@YCu#{||qE?B$!9Roh&r
zJlW!m*n({r_vA_EP2_fdq8L4S*YDqY$|Z53wvFa|{~0zcHeO}vlBgn+InAo#!SiVs
z6$6gMb)T9kzWiRhs>QCd=etg;%6LTY`n#|=x?<J$x3`~mXJ3||_(tdSQVX`?i?Q7s
zIm~Vdsk7aG&uLe2>v5rQ|7(3O&f|*hIYGy|)@@vMWd4n5pL3eZw9lKpy)&ia(Rt~n
zFHBoGoKO9{)bXVusgIAPNLuc*?5bI>+wS*P&fL20Ky9_wHSM#<d9GIH2tRu4UD0Wl
z`BBZVa%!E$N`oMs6$Y~l+tXq%ZhM@%<K^z1H>Mwc<y%wuZ1#*l=M&wwSIIhe`%dNE
zb9PT|bIarOJqh*?FP0qTKB@kiHRRQNmc+vn8`EMgT0J=(SFz{bzyAz%Gx9v_ZwN^q
zda+IJczxf6?WgYA-u+p>n(+o-)WOn>BDIb!4t(~q7j-LcC~dD$v({8_(X-Uj341q*
z=}nQecO!rOJk`kOuNk|FIA3gP%P`_+UB{Kjez<JEjpFMy3`<sglI`<lw$=2?h)a22
z_ItvfSMv{l`FkVxKZE`onNysap{Gvw9$g-?U1`(dq~Ee@3jWovet$GHciwhwV=={}
z1>05pH$E2BzE$vM$F%pX@-i`zHAQ+1QctU7Ej(AgFL_>Y<8Wf-E-kB9E4q&#|M_t1
zB%U1ZDU!TbC5x{XGq_aBG_eXPZ4U{#e)`c2*PE=*#JXI?cshgl3xB(~Iky*bDoo1$
zYtWhKaLV+c_@l$~IcK|WV+rNZeYVK6D0A69m&3P~M>>hjSoBuTrf`PfrMJhO{xfvC
zhyII7TJ4>FPv6MAb>G~>jq25Tm!kWgbp&%qC`3*@GN<Lni7EUXc~KlTJ9+n%9Gx1l
zLElwN<6Rt^;>iuA5B9j6G4ZcVSgqUVnrwTvXL95!TPc~zGkGG!&OVn?RQjC#=kk9B
z{>%N*m!@yH^KjjvE32{<6=%G)-Nv)cDe;+-s`4F!_~{Jq-yLff+o0px6>@!D_g+4o
zAZexm0paG<?HRk&wl{tHnGo>cqVJi0hsEcX-Z*`}JJv=d?{c5>($kv91p_=i{;+gb
z^#3T-;?s02+vvOG&^lW&os*_qxlumzb}e}z&3fsWc9U_AR>p;&b~4HeH$94tuT0pv
zX7;p~8&0kGaZ|&2a*yQe+E=X~@70%Hd)T!=YvS9faeYtIY%BzB<iF8*^TKSx7M_(a
z_LSPn*z3tHW-wlLNj`N~tX|*eNo{eTyo7J@ss3Jl{n$ej8E)<~3MYE2mu4}(cklnU
zpm=$JnN;qD+#E(DCzd;J;{6z^darz3{w#f&J%?OQbYa_fYsP&XdaqtrebalpB|d!X
zpAFN3L_Md^efiWn<$a9PN{<R1_9Jzdn3`6v-*RDg_B6RotSxu8he?&3NK;4-oT9sZ
zr;KIC!aVsB|L=7&%o@5+c$~GK7A*`}dg*Z9tjCKDWD7VrdRy79hqzp<FMswqwryp!
zz*VOOpSKFGK08^ec=96Av!xskHe2@QT@2Y$^!{?k;i*As(Ge>{+E!kXeA{al_gz_A
zhI^J?y?veL)u7g_b%)laEj}@Gru5m3Y0eeNlgpc{?wI}F`lG&2<CT%2w9)(}TD5L#
z*?OlhU@n@cRCr8zbEesg+q?A2KfKv->bdU9{|t>j$M%@IWr}7_c{PDORAJq{O@|6j
zu-e`#+i+cS$7=~=_X!59$Nj1e&plKyJM>{ION(Ie*+TKz&%29O@dougUlo|mdO$j<
z)MQSs%;S?j>s3BJzRe}Ipyr^`3Rf{#%f|)Bgi~?_J4H{VuA3+O?@ei_hSmzH)A=D<
z0baU+4<z1uzPuo3eM@Xl$mF<RS!+LCe|=cI=L>g$o?D>MoS;Q#CY|<k6Dw<ZyXErZ
z>-)@h+|_&$HJ$fGjLbairr*1dToe>&l%MuqB=pi#|Hr+YB`q^6D>9_gtk1YhEUHep
zG2w<?;C0?Snfk=kibBgGYXzkX9~HHCGE6`Ev*vd~X#6+vSu&BoF0{Vb&Rui!R{qI1
zZWmP)<L>KZX#Hm>aQ0HIoa(-4!j>bG@9vz)Ebt@LReb-CkAVqOelzBosJUA$)cAJr
zpR<fdWRR(z>Ybx@;d&N#ECkb9d-g|lnN$isO?~`z!?`4v%(W*BGB@PC<GOK6FKgAl
z9OcQ!`W5Z$6}p*cPF~_wUFJAra*Mveaf6K9`0h>1bGSY{y!>!|pMAG(YmlP4?*3C@
zE9>LGKXweXPgpido~Lk&f5SxH)8&W1Cxz`gJf*))yMD>q{|tQW8Br5feGwFknWDg=
zyX_(GeX)5G&IVT}F+LAUYCYGly;jlbho|k2s&yMh6Y9MluX(rS`?R<0pAQ;_I=-{H
zA@F&(u%g@C7t>tDblT2UHcE6_+*T=@Ju5l8{iNrV^)vp?+N>%Un#E(4clv;ft4jMr
zMrZxksXN?H{$w=`wmy5a$!66=6NZzfyxV2;Qww@mWdCPqmP=k+<gjA1Z%Vn+iR_MT
zKl7%`r)fp4?ObMaMq*7f%SzvCPduy{H!5noeO208`*QvBru7qS8onL2JvZgv4Y|0>
z6M0vz|9a@D?z{~~HzKN<#k8Ak7TEr7ym-lI+tcsM^L}i)<oTQJ@fvZ3G~r~;PDKTC
z5C2DkqPvg(Og8%2bwE07wa2o<X#vt^6S?y@Jw9C+U)I(!z3^>!OzO=y7PX=;zN+1M
z`S8w*$+b@<`K3#P+`hd3^HFb!?DnN0yE^#~r&$Vgo7-FvG`sZe@=rBx`H62N&M2N+
za=72{q=1&Y;<a0M)FPSWoK(O6jVb8*Q2Og=mfDiTbImSuoHtk8^lV9u`_CT<QR`2r
zJaL|DbXLsK=S=bWyhVYX6H@#;g|_DHQk!qR-^SSR%?j32#Y;W9mDf(bF1{%@PT@0i
zg6{Iu4|xJp1siU}YJZH&*t4EH&uo56zW=KodwJAKVprMR3GWg-`ASMroToiLP4jzV
z)z%5KHhiA${xU)KoY~oFpLWNc{`50pvt{%Az?WM!``xFCoYUmE^J(XinkUoh>LeyT
ze8asd@wjA{pc^A&_mh_vDsHXsRf0mV_s!6(I2^{|yx^e3k|O36-MftDa{p(j@>sH<
zbJ=`d!@`0yDWWV=ao@k~3Tfy$(|gj^Vs@L;g{j|uUq7As<MHK>sW);xokJuRzt(u`
z#c}p<)pMEY+BaFB7u|mRR91NV(eB1Hf!8Yw6sA^%n8n@8eU`ZChNQp=LD5`+Kl`k*
z@765-nh<@=Yh|(Sb)y@8>cuAZsoot<f{R)gn<Xsv5ZwFkEHBTgkJ}zz<~p6IvglpS
zS)Qw}{DNK>Wo}ZsoGzS|v#BBI<lp5n*Iv)r!fJN3;@|sy)=gq76sF4QEb6zt`6)*z
z^`&*%TIR%prJecmMU@3-ydEnbb++Oao9>zVs3f&J>-90`3?}|kSJ(G%1;tsG=o-&b
zeySF8#^4tx|A_?88S8D!)$?+FO#(ltT;Klbh}fx8wx+MjT31&VJIxl$vAn4HT%%-%
zn}ZO;%r3>6#3L2!_jAduG2qR}6uI^}Czw%jvg)&|&nhN!Ie0FPTAqD1kf-GOtt1xy
zRYBgG9*d5bJ}yd$TQ=$8djZ}xk`>$9mzVx$2$SMwPPlYu!MO>4V{YbXY;b)&^V=Hk
zO}Q$nrdvx`{0pYd@8(WbYSA<@x~r^q`F^WW#Wj!gy=sOgyF)eJ+wR~qd6fCZ{MnDa
zH@XgN>NZmKm?|f$zbqxa`*+^nqKOtR4fC{i@wKd~pQmiL<r*8SScdYP?JLxib%gz5
z9@m_4<XqyqdE54k`Qlf5R_GtEojGMy#JNZ{?k_(|cFEZ8ocK#Au;5og;l-T5qanc|
zvEg@IdrlO^Px#FxuuC#0Dz^CI=QHQK4UUB@XWc5ux_g(<w?FaakB*)ze<m<JWZ?_1
z+RRt`Vwascb9M7Xla@B8iF2>>m*iXzdFH&R>=<jp=if)S_hc+O(RC$mf3sl<$F|3@
zCu$ZiRC>8?#SF;=p$!lDx|K~rl`dykU)A&ruBzZW!7aNrWP0h5_xb((r=@D_<WCg7
zo)NV3#ot-nI=jt0PfZmwJF)wS`_A2!73INqqxV&O)4uTVxc_%O&)}~t8+TsXVlyvL
zzU_X+gK3;b->i;lMW4^ToxsUI?M|e_rs9`;HYzbc`dg2;UNz&LoNd|lxO)1&4>57s
z8@L~H?rAx*mqp!D<b8$P>uVN;J@E|l>qT`!o~8T8CluwT8!Zivt$I9Ja+1=Pw-3z!
zMXgik^S4j_&ma`KjVa6d@}9Hj^Eh@XA3vQnQUA||(;I?<Cy8^oomb>NT==jk@vYMC
zi!oUayHgiuym~NSI{f{AhRa-qpMu;MZr6z3IYB9>^?+Vn%J&}$vrlNtrmNgptZlPx
z>fu9gqJ(OUWHJx(x9m9;(=M7-TC8$UcJJ*2E;Zo~xAVSPc*xSpk@b1h6PAktQ~%@!
zUXS6{Vp}k)Vf)1$ZXe8adylxbKMMNB@V$j)-6I#<4GhU^;#XNjw?6sLz%QdaL&kA&
z#8mMKd$&K^b-z1)o)TB8+M?&3SFWqry_u{YBfI6tj{gjAg#~ACh-*1ewe(Dd@|S1z
z853?VNl-k|p4093pJAF!yVZMl!*?fV*q*9&yeOWz-sbV|JqI^`>Ob$3zDBOKV5?rb
zyZ6gf*~^i&#~e<))#h*w*ke=qvoJSxP5YT0cQmJ&d_Ft*kQ0Yp;1!#2!SKhwj%mHt
zb6!~*aC>49bKyEG<KGu_S*CBi*I{A(pW%d<!;RvZS?-}?=jI-|$E@)5)?1e^*W+b^
zuWo4gdEwz#o?U^9y<XnQop9&hW4BhpE0_JNDjqD@%`!<mph%{q=kX~;b*|faN^!Qo
zA4GbHq=f{|6gfCkw=wu#dEt|9{fqid-{0R}Yw$yht=K5}+zwUoG|3&RyB@S_hq_Gu
z&tN0Jp<(;7kd<5Z-uT$(&?y-AS^KGz$knSKzW@2K((%Od$c~xw`TrfrZ7EVz5ccMn
zV7gKKc#Q3m#toaNC2W_wDbypmW6ss-{GG9%K72SDa=;^~WRa$;X!ji^?MQh|f!oV}
zZ;aYhe_1h0Q>0@P3-3g;6D>Z?-3cw8w|^#XntbNf>WRm5Uwzoj@GR;`pkkWC))Su6
zqW<-DuiKy5I?XoP=E5U4;dRpEoaxgiOjtCtN$bVleHC?AX6#yFrt7$CNjV?udrqsz
z8mUXKcV?yR-TNbG6GP*3t5qf9a}y^NL}$z?bw72ZQ?Tr>#@>&q;mOmcmhNbJ5_Wgd
z3H4-2iSB$s@i`s)SVQ8ha=#y6Gi6>zp4Mxp&n1Qv&e|<W=V@3h+{9<pD%-&H_Hphh
z$A$Hu4HIqB8FS7RRf=>7@ILH2JXg3mb)BH~-K!VfcP#xE@<zGEe1bTCnwG3ya-Y|l
z`R`i~I*2>4cwCMS-5M*%DC=LzXSq5;>!_8tWxf6tIhjpg(qhx!#(yg?lM(UHD9c`Z
zO?7+hhv{_@?>xj(&IM)(91Ah#Zt&<z_}LT7AmS1~Z~1BeqfKEq<QRVC9A9*O<@Up`
zl#geub^B?h*Ce#?=BJ%D_3u7PZk@xGVx{Lkf99&GyY9+PU%W%l=9I^UJnKgB;3usM
zr54@VUUbO3eS%t>e$|wy%x7*om3cM`6L<Zzn>aUjc}!u*j^2mMWlR~keO9|`lzf}j
zBedZ1{VZLFZ#{FTIV}1zThC>St01eh|K+|3T%WJzT#c9-v~-W)-L@+;uPomG{n_Uu
zpMEUAROh{)Z^t`_2cB~jH($Im|6cFgl_EEuC0+gNX~S}}Yk8qNL#Xh7hUx1pq*&yZ
zUfs1)*nEMpT(^J2nJsD#o`-5qL^$vYaJ%rm{-;ySl)$>Q@2%7Pl7yn?F%^-UPrut)
z->3CiOL@kv<64sol8%bk_LSeLnaAAy{Jwj~?sapgJ0G95HYagw%<9+fvf@(NZ$7IC
z?3D60@7Tt%X!$CoMb-VnJtqx4R5wq|x>|F^bk>u}D-1>3p1bi*YbdYSoZnvjV`F9I
zgePJlMN?hp&dzwp?eJkv`MbnN8#?zJHAgMo&-dVN&$4yrBlah%s&0G9=U!If|LusX
z>~uW=MQ`_8HGd4dPo>W^^qklCcV?V$u|b;TzDt27N5o=(Idse^wLf)3{b_&Yzp0<9
zxvQOKzF0J;v)5Whh10Th`p@djMaj1}XG^U~UFY7d^mes$*6W}*=~-1$T*uG;R@y(K
z_1f#^qtE2h{bOFAZD#y%$0cURL&^S7_H(zFTLkUw*Y;_hwC#<y;%No#X-6wh3hbDB
zyQBA`^z@zo8AM|^54763JnFl>X^rlt-4hgx{za%Vh+D0R4JlIIAy>OG?ZV&qpO1Dt
zxN3V)tzzE2n?>7YuM}plnJ`E9eD;a5_V9ZB-As?KE!REM$FrlTfBEk7XFci<?EAy!
zX=Jq1_{@ZD)|;H)8QPo4ozlJ*SU4@W;0WKv56=&a&Wx#<A@#UV(pIK{>(Lg+13NtG
zmx(PjWAu!!TRwGbBF7P3=7(=2dZacnMb0_xoU-6y?sk5q-7hcu#2R0TU2neOq1E*%
z-R%z#bsXwh>5>#agERVg!CU#~9pU0xmvy$}ANnjJ@uhy+o%Nw@&d;~=&hvU$$uWJ0
z+Wx9UnbS)ytUbAIi|MlD<E!20YB-!++df-AJ9poai5%ADACFHD^EqSs>)_i{dK;%#
z{c|n*xarcwsLnsOe>d#9{yXszZ+BF8$kdA_C#QTn`l#f@e})RHu2-GQJGv?!PKXYe
zF@1?o+H+NF$!GD~uJ7#RpYC_EtNWyvrRX-16cM>;HXWZL`+lE&$6zM2_S4Tlbv$<K
zH|Ti1^SGhdFJ@q)&za>{Im1qC37_h+S$(UY9TI!zB(>yW=1E_tnHdw!7fwB*5~roP
zNWcAZ$3FW#p5fO-8N0r3ONke+%wxD`yJgG2h~$bBVr`G}cHH!u^ig7qs%*?22Tz^}
z%%LIgI=?4=`Zz&B-@U=rw)yrQr3-Vc^8~*u>CbljRhT8gbL8i#sm{eKx^i|LZo08k
z_H6FfknqE?w)g$GuNjMLommldOYPlj-2#u!Lnhz<Om=;HTXfZ<n_IgJZ?6yuRF5<_
z-!3Pt-_lYO{Nvk^4913o?vf&Rce?VZTjvYtxtHJn^!<V8*2zqB6Q?~qrL8zy>A~cb
zJJmfNQ$1^6&A0Jg@3`jpg{KxrlU}@+39M|p6E^45-DTUAYOL;eol~5Vu;i$K?B~6c
zr|1di^=r3G_MX0vjdw%Nl+~__HaoT}y=-v0ceC1HOULxx-OKjt#9T3)vN&SG=QWoy
z3v(t2ci;Xy$Nh$4clWy=e}7zm`6qXt<Db+UPb2+YOHXXsws5v?Y3`;MKYdz4CYFA@
z{`k7TOqI<1Yq{Ys)YiK<8%}4luodf7^1dWhQ>Vsu?Ev#}mskh;BN1v06JM+gNHY#v
zb~#PZZ_ZgU5ANL?Pkj(&=JwS!Fky6(UwOZDhsQM56NM+^zeX?1Ind{4^X>s7H~%U@
znX+|pXO=BI(0%r{Y}Q1lP_3FMSw5XrIWvxmp3L1V<jF9j`|aI@Zzq-KOn9r}e{7!2
zhBe2urmy&<#lJ1oX;F!u*+rYrx%Yg{m|k*D`f-b6*2nLhWmlaiX_-bx@S8cWPCag$
z>YQe*uDCvYTM*NuobLVk{EU~UKRSGMo1&ycp0J|U{AtP8?)BTR7IR6O``yH6Yvrm1
zN>OQd@9bNn+aRfF61!sKr^Wlz`|f)F+dNy><hAc6m+I_CEgp<(xBaO|VxFOsC9D>d
zUMe_GvtQ6$&7VQ`();xK!^>wMyJH}8YD-na#l<^qKmWZmEwasbMz`<x&sp88vTxMs
zsGe(j;!&aE(H~g(e9=yo(v0=W3FjZPT=&zw|L$oUn~Cc>$NAn8p&kc$S4}MCKe=bF
z`CFx1&sMo?({I}sFH?T4+#pxa=HNu76wOAN7`eG&|6B_umVNwi=}lUx)NA9NXAIm|
z<ed|Lw28Od+<ftzmQHuS{|w!S_Zei*?AhnIYQE^qw<p9F%e_vyUAFUtNAQ8mn+o6m
zTN4^u_+!4u@}o>IZ_N|pyR`q|vn6r$AJzC`Vw0~)bA5H1>alPN$FdLaM2~M@u3jSI
z;&;ztdT!mOz2Oo2HHyps?s>Aw<uS_#vyYwJ4Q-iOHQiy?xu)2;Z+J81*8a4Z6OY%0
zX>LBBzoK^eY;WBy0i9d_GyI$J^*_VQW1c;M-Ip%s1unM@Y&$<?oyi~jFxzm|UJJFu
zri-4r{W`AvM_;`Ca@XpDjg>u<ca<mX5dZgP-Iann+t_wCz5Lwz>DGpmw{sQcmy2%J
zU6J@=(VL*I@`>ippC7wFjiqYCVy)>{ZK9)9E~`Y{{hsLDDOYsjPqK7#>9*}19AXQ_
zj?b=d=-73;Ie&_KtbuGpN=&~-)DH88-2V)E{3^`iDpd;=J5pY6+&*{PYw7K`>bxIz
z<rY15SQNZ#X>rN7Q}0gPpW*(jyHz{qH^)BvJug<DH*l8u$-igc8vBe%{!vj%zC}zQ
z<=cNIl(#kP4CPy}=Xvh+EXAis(-N24ZJH{l)2QdX#re-=AHIvfCiKo~v{tD6ZM{x^
zjki=}j*%Ms*~_yNXWy!xSh+E3mEnxDuR_xo|C^-UQ70p8=p*diZ?!idPbAFuaK&2x
z!#^JxEjeJ$_TtN&&EEZ76UydYc2=(beK^|V^p9ya`-7h>S3Fa7J;7^|pQ-;h@#DPP
zcY2rz6yM})lFGh+oMWBTsU6IZ&n-EdXB#DR%wuNhwK7Z2JDu<S&+ot7y718Bf=8!T
zi2V3cpZ6tQ@>5Mre#pD|d^+pZBi?kqxqNS8z{^wV7I`rbcF1^Gd<wn%{%GnH&DU&R
zjB83Bh6%OQo!r(s@vqXneEv$tY4V&o5~tk0o;%SOD>Hr3DRr5YH`;Spm_#qw)a_w?
zxc-KZ!(-pno5520@8`?u9B2IZI#ks+`1mW(aM6~3N|N6Gt|>{IMV_>7yuIr`!~48L
zmL}H<6n(2w-Hps-E8LT=%LwIJaPm$UZ&E6cw!hk)`-CO$-jtjjqRPEW+N$4D`z{n2
zD|zdO>h81qcVj2(HOr7iXUbm1rkY<1Y&l)HsmCGZcU2|#K|`q(#uaj*HfvMwZ8#<(
zojh~f*&`2Q7p+p;_rTX+^{ToLMK_|BpPG|o9&xAQ(c_Fh8*{OhH%eSWKfdqk4HunR
z6lxX{*~e;OX}LT3=+lkPF3gwz{s^<?oME=rv+~C8#Wt__r`<TQ#eT1LjrB6Q)f-d~
zD0-j%@^i}frnRcij%Ut2doIRxVYHxK_Rl{NITP{|i@IM$Z1@{p@BQXR4(~gbpVdc$
zL$4pOo5S?bXw$Y$j_roaTf0h*#r0=QJfZAgpZae`&*9K*+@_q(M#U|^&s$tpGQ3qe
z&+$fg-^$y^<z_FfWn0gFg7dZVDalF8Yzm{q_0+jLCv@>$Q2PGPb;s&F6QlPgMmNmd
z&s>?N&KJKvs40alvu^D~Db0TmE_Xz%*q0DnSzEqNSyTELt6K6Di7yGS&1UaNkI7Q{
zyL@q;Twin%=U=7xmj@nb-(a4qv?2MG(^836owyYq6}`TfY9@!RcMZ7^Fm-SB>t{z&
z9X?piIn(uI)#8PrX{ma%^$R9Wh)>X$6q>YReWCAt#n=|jgELkg=35&uf!|o<cHy)~
zo1WCR&FNd<XRRtQJ-Rv9ktJ9Db9B4x@x*vjOR-GnESnQ0AE)?)NcX6zZ(n|(sxs`z
z<>;FIb*%q388f=M+J#hoSv6%^@|lS_0dC)X7nOwURzDw-TcY^s?m7;Kpamv7c@D30
zHO`6aQB|DwV{!_E@5)75Yj3TH*`csOd)pI<<V&Y|PEY2%s1~wy=dpFybH!im%-b?K
zdLx_QxpvVT#+nBb6i!+=h%boJe6Hg9@v;B|uiZlLrKj41&X}w;o;ATI!mvV6s4>HR
z)$NaeJv$^4XL;wCJ*gDAEPhW>#hl@-*i1oH-kO^4+nbLk@~!w1_QWwQVD{llJ6!nV
z^E<k-(~IH`_G@3wzL)xWCfl)$k{i2=pL(*D9WR*D-`!fEp77^lVN^syaEzK!(C6SO
z9eWuLbIQ19zSmte_h0N~pRgyN<?`1vM0V{>PPjce)h#{CB~dN-al!TbRWG`xD?RL;
zC(RVK=irw+2YqXw9hqmlyk$GvQKP-O(T82)eyP7YDmwXs%$$uHvlrVcuJq5BJ#yRO
z24{@+1IJZa+Kc|$-xJf<dd)aPcUty@qJ2r-(OsX<xr@yczgcnV;iYr;pZ(f$Y`)jN
z?bl30Zf-uZEXySD_21b`R33F!uF-G)QS<SAq_vq8cWnBMtbVTrSKG5ssrK|oR(_UU
zS^DCuRK=F}Tf}XyCmp)dIm7f-%L?JnYkOM43m<o#IeIkUs@;y;i@ubfFD}^Q&9s#}
z@XV^Cebt%@W)lTl=jGP@VYrr%8M<IjUFY&`r(U<T{H(Vz^9f6c^(xLa-zTl&^f<}+
z!|nQi%Nx_2v;{W@NUz)XSbz1Y_`@7Uw>Dn%%3Z$h)6_j8^-tz;cC4DrudaP+YP#7P
zq0e;-e<u1I+Uzuc%{;+^1K;gMA0PkEQ2#OM$kL6h+DtJky~-T6)*g92p)g$TKZC;8
zGYM^XBm%r2)gIh&n&p&r#{PiSxttLncNTmId|alTP#T@+fBl?m(Wf8XGn2kuHfxx5
zEh#Nv>hpsBKGxX@T!(bMTvmQ+vr#bD+jddjF#FSy{|s-$r)`e-Qa=4j>WjbOe2ZLn
zzqh$GfkXF>z;D$b$6wVP`THzf?V;A?E4P*eO}KpC#&=6tbAIgEsk+rK=9~EQByLu@
ze<IlGbkWjKwcxuOj;{E0GtINJljBv@tNHVmpZ@ST*{MkQn1JpK<EB#c_22TQ-C4qY
z=~_p}#1j7pbN4SZ*4r-=@K!ElMV_t6*?GMi^)+ufakXf5&C9Rb^|EWa!g}{P&u&kC
zAgwc3#q+tGLgD4YC4t_e!k>=bGiquuH&!lMns;asOPjP%fk>X5c6sX6;OE^Duayn2
zNKH7k?9@u>GdE@0UDCbV&n{d2_|_TanHmA6$tzvuQcYGpX3qW2zCuZON6VV4zK-HQ
zvb&1n`(JY_%{13Y`gP24M#qHC$<sMw_A&q2Sr?HrJALy<Zoyr8UnM6_c#{^BF<b4(
zpQX>=-{0T1_d|GW{$s($BKtM7nogRkU9FfSkU8DMdBY_2)Sq=a_5!Wj<@&iC&ekQq
z+veN3<GynK*_B$E?_J*BXO2{5+tfa@^3)RR$5$hZSE#i}2smu%DA{PezVu+0W5wM1
zv=<ANe)4Q`(#>N|c`0b&6{TMIZdS74jEKmcJX>X-DV?<Lx+Ey)oK^E7NP+iNcFRra
z(C~TXH(&4-3JJ>QU9~qkTyg#SH@=hHle9&3_KHb1xTRj)UV2h4i1mlb``dy$t&W<p
zT2&W54gM9jck`NCYI%%;V*70r*KdBq&Tx40{#V!h9Gh#cQy!mCXj0MsV_4+X9O`nb
zy5(~5GWQuJ-#S`P+VivO2Iq?Rw?90-@!_(~7aZ#>)aHlW>Qa1lecu)P#31X|&%sJ}
z{En{A-Dk1sl)%M^Gm@gWV%2)NW_{Srr*`J$+mGybMN~tt+Iad0?f#fp=kTiLvCf54
z+Uqu@u)dwA#ImL7Tl>EGCeAVjOnnIxuZC(*-8ai*MJc1;-pTt@x*qNg-kz@}aE3c%
zuj9hQ%l3M=F?@7b5gezf%sQQ6driO<>&4UV-C9`vpMg(_`-HB@?9~?^9yrVJF=X+M
zCl$dh2iJb~p1>F?m=}8C>t6e_-BH^VEV(QfwJR+;-EQYOu{(LI>PhC@u0=MNH!Pcf
zol*6gSTsXwuEnAa2b_yu9NqNequkWpqBX(MQ!6y9kG*k|XylUEYq#Xgci(BZCJM|`
zyJNz8FhI$6%hlYg)oMGK8((p{&vc*YH~(^sE_bgc>l3A=tIFS=_WRMIHCeYgxAC`A
z>P{80PeL^gM;%Rlb9GJQ4>u~hx!R;3mR*&1r*1-uXX<<Hq_c}xSgy)0+EG*TtNM}b
zfs|!eB+L2(rW%-w&GT^-dKS)i_n60i{k!ZxH(X8Vi|l%;xb5uiBV~JMnx8oR<MF|}
z^Cvu?|Flv@(erf4^~A&T-khG1yX{{2s_VC+Gb8p_P1~WGuw<pfl1iR;=5~j37b>Ma
zvP)g?{q6FTB~P0s*EeXnMK0>PtS6S-cIG6{);+pA-^ME{h+Y*54>FOqj+wDlA+m4f
zCXEGWI@iWMQSZ8ONyhf}+IRP6fp(aSaT}$uo(<*XyEe1z#}WRRFa5s{35k6C_%{8#
z=&A>nybeo3GW3IP^=><ItaD>^DR)lC(L8I%`m0UNm4{<qWL%l^;aSF)bg{+uH_!L!
z1;6sC-#q1}z`FhAx%Hl0k6%n$ccS8S)}pwQz9+ZiGi(ljc(gX;a@Y->!!xZIdG`hV
zXYdnP%;xsW&+5cic`?>0IWs0b%zRj4(I+@<v!g@ltBa=Eoge-)_<uemwP4ef+v~-i
zU0xsi<#xAGqK<RWYQD4E+h=&%H@$9s!lqiW_1M9UpJL)zKRj?-Ek0+BhsC3WvOBS-
z&VT>@w(HGpZmr;%Gn6+xn7r9?V$2T9@Ceh81=Y&GoKAmuW+A$xaBY{qtyH0lC13X{
zvD?27O4sQ*#cXt6QFHVDrw4C4_VX>3?WnC;r?<zPWBQE3C96L7KV!|++8MFR<6Gd5
zZ6Bjo*4WQKEz3EjEo8+9rHNJ_gm=e&G-S<Q6S^j}W$`v$1_29mlSTC}4`_9*&SC#5
z9atXw=*G|MXUpVT&61r?{tRm?sa{=u`>SI?<2Q%(z3B~$>hz!X^!a|1{4={<>td?Y
zjKI{=k2g{mUP^f}o6T*y>YfSVN1gBYluZi0k^Y<E=cNrFFUHN}VA$q4eX(@ii6=iV
zIVdgCuZ{+7NZM&5@}!!xv#85-%_djhW!EG%Ty<wU<hgJC&dTn>8ywI5;imrauMH2o
zS}R}327l6Av-8#WquV+f_3sDEz5bXr-_|fX`{UpFoia@K%KC%y7tA|tc(<T-_0fWv
zyvO!lKmPcD<D~v4N<6Ms{Y<l8-N<*FP%I%>`t0!*>%ZO2v-J1O+5Jbkmv!}pJSTs*
zn`!!Q{<g~r_lhhuJL7X?(!0~EzUv?UTkzNAc;dN~Q%gN|U%jeUrX@D{KI?)8@1(X>
z`|_+F7Oe`H8k>52;nmI;MMgW16}IcjvRvSB%)83;<lWqj(!Co*KZ&i}%B7dp_bhIe
zan3Ri4~}_?cP<~SpDuq|xL1FIn6t@q<u7aJl=;LLtP|MZ@xo>0ZU6G!tZZ*Sah>U#
z_V|+0iP!me3OpRVU&iwqtT3-|4eFM&kqnZxJzRKqbLj!4hbt9t-hSlzzWGSng{gK0
zr&6CW`R*?3h)Pk=t#1F3-PU<4ZYmoOSM^o3zUL>(ZWg{%ELh2(YwvnvdHi3FQ=Yy#
z=|_u%9gENL?&_Cr=L%C=!N>0>ygkir=2n(%ZqFw;9eMPyzCHeN`>Ov8jR(HitmZc{
zJD1fRx9n)w*(DtxXIkg&Dra%>`&D-IrM}p_gIz0Fr?O3+7E_eNk@#)-(QVrfSY{Sg
zWPi5N+Inr)p?n2-!?fkg4|Y$#)syp8db-5szZcr})$iHy=s$yN^YaaxIk{9dcM9D3
zdRk|qO{_Cd-qd$dyOcBLFPG~KkKAxZ?zFyF-I9|IRnn6rlODX9wX^MXMsTh`NS*cv
z9oJ8?u50~PUtblNH8rB}fmeO%12xwRg36*s8K)0V*Kz&Wa&*Po!#8)XoRnf+uwC~#
z-v>o;6JGIcv)0%12ph1z+5TqXnLdBpt7a}iktqU>pLXbcFM7naM7>38>W$UY>bE(w
z?pnNnclWgDx_LSI!hYQ`uGRWegYVhJ7%&}6T4~ECeOK_Sz!USUrg9D<-3}d{*CI?p
zZe2c-_hI+;*|J#^)?eY?Vzu<aX~R@2ohfp2Z{OBsnCf4_vS)YI?+unYymcFW&$Sz#
z>-SiAxZPhU_t1RJ=|1gA_ZYgit6pn*I5p@@+lh5S&ks%%)0y5S-4dNA*!$!K`@zFo
z52aaUetE#KYI2b0&bPnb<rwE2{dT+RL7daV(mO%7d)E8EY-HtMCTb^Lnx5*(cE~y0
z#oyp7SJE`L{c+C}4$oM7`NbtGo3jQd+vnzPyD64?Cpm0;RL)fakBPx2mdtpU9@@F$
z%dPCM?)@vxZl!)#DdLXJ@nCm+v}(TIjrAv=EV<zNdX3Iwo!Mu%rf%y_nwQ_n8vIKl
z`B=F9jml4M>9temP5Ik)^~k*B&-3>`zkS@tPR?+qz|^eym$#X^rR=D2UH`k!N-6up
z^rszLud(?X<W-#M3w>fMW|NU}x#`5dB!;p(44QZLy`RRmXZqiUM_<&Ioywa#E#s>O
z+vy9p6K`Lb%V=mAe6glRmwQ7bZ|XJv(~G0lht2uAB%x0rCMn=O>!QH3r5|H$N{<{*
z2%2`_b0yzW?=?0Tu58OZ`>FbNK@HQ<MC*zyn~kb^2ewYQaIbL1)`V`Jo84N<{M_^3
z$~r2B#@6e9d3@oE^`1Lh6%JZ_Je(tIZya%N)hwX^@53zKGh2T|wLB?WC78Uu;G@`|
zyd!Bhq_|6aPHs6V92RAlxL5B#L!I%en3ywv-#j?IS%<xG<F1SH8vhw|!X_NsS!I`y
zd%yO6=#FZGHQQwUR+?)`H>k<)-YWd^up{&3Vv+KsDdqV&M=B<_XY}op5ctjQ`&{E8
z-v!%8g1ZiWu9u&uxv3|5q3@ZCN=s!=ZoK)5?{-J@rPuX3M!sueqZ8H#eQCVHJ@M8g
z!6)1NjC`ig==ZPxvE$L14VrHY4{Kjv=HGWmYqFY=S<Cd|N21$(#eY2bs%qPOb-m*%
zOR<wCiQCM6@JsKvaSiM_RF_eCVsTCL4)$}+#|5W2bC&%1$$pLJ(o>@o8~5i2DHhyb
zH&1qh#(MvVwtWY##V)zN`Co}tPQ>w7x1w7<KYhw1mUq=YF!Q$H?D!39PlnIj5d8Az
zrO?EgUuUU19f_<Bc5*MU(|*vNwf@tmzc)Vjw*337VLDsuD%a_Ab^<qN{EW@^Y3B~!
zy8kp|SMYn$nO|AwoMf7HM$@OIBK}{1+-HH6d`r@I-VC(n+jjf=gP)W9FBeSVj+$sR
z=gBh7&5B1gQe35MpFaz{{@(9eMQqoGfUM;Kk9uQQygRYdA<gAr;c?ctuUoG&g~*Bs
z*51qRG{`;XRn7id&+kiokd^eApjTU}TE5HX%oIyk5N?U*%2W9M$AYuqp?^d+qhGty
zN;zv|*`A#D!c8206M7F8>8)I!>iRw5Yqr*d81?1BtNr}d7l$a@D4mdKxOd~@^`8$m
z-jMonuFK}s!m3LF)fNBBx=%QNf4%6T*UIAsqRjzWQ+@=*N#B$dyz%$MCq~uXhvTR1
zxO{K2<EleBhW(tAX4)Cw&DiB<e&}|c*XniC)@eH?HcWPj<ln62;!^4%^=YM&ruxSN
z<`)XtdS=}&&3OB@q4iDw;qB88b3go>EF$-<U#Yosg`KsE)Y?VnzbD_>vFX81k!Xok
zbG3|hjb+R3hA{Z%{j+=L-E~Iyu)r<OqF>>`O=gdiIzOnj?cMsOd&M%zfC<m0%I>%{
z>8R4y*FjIe*YlJn%zxvq?bpoa>X@{fcdPiE0-xP?R;b^;-yZT@uGfFXbhQInzJ4{l
z$D?}vEb~lGyJoCv{NpiCW81R*mwki|+TYlAVkt|<(V&GEPtE4Uyld*{i07|=*zn}6
zLtx&f+nJ%Ox*dACbrrTXKG0a?vMc+0S91AcX_)|1wMO3+VOfrvi;R{Z%8V+G^Xj^(
z)wTX~pigFBVf)2YK9M!YmAJcJ8C}Zd=*(P@^)UKp7W4OS9}l&iIG=rjLp>nu!qk<j
z$=>U3{PURq^WoK5T`Lz{T)5?1(1ceLyZf6g+_V>Fb=+Lm{b{G^KkcqJ#c#MTC2bBp
z!GC>jj#QoDZiU&o8*kU#&bShKL?txtpOsej`vWHe<gadwK2vbR;p)r_O1pl0btX#f
z@>sI+eW;5|Z0Yr*)!z>qvSw|v7uMexmX`hEkE2Mgo_9z&>uztCE0gX9?^@q~XO`v(
z<Ft&6_m>3EE8z7Coe&dRdFbjy#|yzhu{PmiDs~==L}p&vEECn^)x1Bg;kECU14@yL
z_BiZX^lH_%Nz3FGc;ud)ID5eww#?}!3KtBxy6>=t#|HbeUjI>hBb04N-Fn`{3v4-=
zMbU+FDk|b8%z>p-T?75BR?W;_q~0f_vovn`(N?jgvfJJ>)UFjg7pKIu&%V6=Je%a7
zvqk|gFU@^9(<#Gh>!Vd|89H29U8`^Ex?VJSH*4Jvh8Xh~k9zq^wU!#*KKgF^Ij)Xf
z46G|3KA5hj`87gK{)_QXl`md<eopO7R(*Y;+~i8|14qGsp8d+l%a&hBmpQb2Gh2v;
zk8}sik-4GXL2nfTt^c{jpBIy8nAI2^dF@J|`E4dvKh-y;SA(Cg-_Nu;(ASO6=yING
z-|xqJyv=M%oI@`-ytyju|FV5$nu}!P0;yWvWs@_v$4ql!yb&tt?NIWo<fDI`sB>2>
z&%L^ZO7qswGBQ@&x;!b=`D5r#@%^fo?|x78GIi71JwKsC=?2GM#zQ79Z&tBA>hFIc
z@@F!C!1<?<Yxi6ZS)3>6-mqG9ZlIpqh2KgqX6aA0y>a8Ed}J9bXTR9ti(90nK7D&P
z<4dT%#%9fl*WP-^pXUxT5DJ{S?A5weav||72bh}dYF6wQZOxpRk^fO!=>7AqaNfNC
z4DS;CLefgt?^!=-cIwe^-pG~yW)hmBmQA0<cIN)GN(`Mf?ScH)Im@01EaY1DbnoT6
z4FNO!SA?c*H_8zy51Y{M8@ykh$7I=;1n;d?jaJKyI5#mqa=aNV%ql*c`Pa>iwH9}#
zOB=^-QcxA${v#=2$=Qyp$75oj#~CZT3+Bbve$W1VJx1!Rvx&UR*OzauOx^2vq*ici
zxS*$l+gaVK)k&w$x_Q0-_w2lu^Ad&djHQp4X)jEPTrYX5?Kl6_L%bhjg=5@BQYx$3
zxz^4I6Ioiy5dX#@N__s$gx5dw;~#_^EVga+S#()*r{9Dl91X`^ew)oGGY@_JE;zn3
z`0l6UGoIgl`L+I&_Dr_}+n6TtMOrLrW4PVq<kemN;h3+_2a$&sNw447y_k>})e)us
z<*>ye|F>!zZBJZKK3Vg`>`w1Scc1RSr;qPX|9D+Y&Cb=KU)002hwGv9<iPOIQ141c
zlQ*mL=CE&@_eL~!`Ikux(lsyKn{(#c%$FxmeB%9>8FszP(#UtBV)pWKPf>Qq5SO>N
zrHWk_mCMQ$<$AenoV6}Po#j@U<+fEUtfzNr>Gz8q3$Ba&Q{V7tX`PBCPw*;t!HpBm
zX7PC#y82%BGhdbcEJWmLVS?JMuFKLnPq<fcu3rDi<xTgWjc)Ibs%cz(y6?y`g-s8c
z{XK5my;`;SPw7#wi<`w7e^u++J@%cQ)Uo8oq~%@9?{|Nl^}x_GOHiUlwpYPCaQi&B
z6MLcqKd87oUjO0fJR?2E`TuWnNplHg<|d}6>L=&t<QEkeBqpZ{v2(F<F$cLjacOb!
zK_t^Mb5ipXb5n&RxWu?b@(hd&jZBRV4dV?h4b6@8yaHU^xzxDCAZqiIvQm>v^pleE
zEA%UKGxLOagt)jk8mBtjE$9@m$F8m-H?abuZdN&u{fxXKdoEEfZisp1nJFb1LTpkj
zTue8Zxs<ttAi~KxiN(eG#g%!*sU<?ZT-;ooaA)DN1Ll?)c|msm4$Ssk5?nkG!_xAL
zauZ90IJnrjSiruK<`RU+K&;70OiImxxM4mMmok?)L=xGn5G5Q^Y+Nk!ndUQbiNnG!
zIU_MIFEyuFh*gT2i;<Z<FX?5e3ljqfC|>%wnd@T*Czlw^AsMNe=@}&uo4zw~X~Sa&
z?AwymiV`7lE>SMw+|<Mpg`E7vG#v#H6U5?@!5#yN6+%ox3|xC#Ik|Wt&Q8rs&QD2A
z5n9#C$+fJNlWS2c=l>i3A20}bxp}%VFfuYScrbV{F#NyG;0)q1!T}Q#GZP4~Ff%hT
zv$C+Vva+zSu(GkUv9hwWv9Pdlvaxf304p0O7bgc7hyyYNq@Rh2iG_)Yg@cua6{LbR
z{Qm%hAO~YP;}>Q|K?Wv4MrJ|A|3?_)85o$Dm_dd!f&!C)fq{*km4lgulZ%Owf&2dv
z1{VPaMrIIT0y&e3jggg+iCK_^Rgr;Bh~3apSSgSrQN*aQkyF`eqG-^<jbg?JFMd!d
z;&M)EdiYV?L^XKQqW?!2WEr8>KrLZlU}0ruV`O4y;NS#VBFMnR$jrpd#>~#l%)-RT
z#LmdX%)lZjq{s@kBC)WMZQ?=^qm2hIKK#J0>=cyrQB=jasA&?1n8_k>Rp(8YT>jr;
z;9+KDU=m~&WUy!WVDk9N>OwB>6#iEa@6_!oFV%6~YR4_N%J9qLSt-dKZLBFl6QVi3
zio^vi^_`R$U;Llpy^{zR-=<P|z0#Qq9lZ{zuLD*tQQWFJ^;ctMV1V)O49Cq3md~fU
zsP=kD+?)MP%Jgc~x3k4OyU&U)U9iDJW9FX?MUFu}a-UbeUHe>I-^wD4dA*j_e};`#
zk;lu=t@Dq{`*!=j@0yVBU14`uq#lz>{lFErU(3^@r+MOxIgW2vuE{yvR=uL}b5!h2
zchwWAH~8hEuUp^nF8TV@H^f4$osrwI-~aBUkmT*>Odhu!>y(>Ref-?Ed7@!yJ+F>N
zPcM6SFY9IQuHf5~^0by`-aow3i&5`{s$=bch9%Vv65)xxijU`5zFCr~UUcA#q<zC;
zm7S9c7yCVyl)HLge$grIx<9^g2a?uZO<u8h<F(LBY7f?X>Pcml?pvZ-<P;Wk>xAqT
zkG|tEi+j)UN!=`&X45tI&(D%|eb)?5t=Mo(@Je1yw)>j9{>!bG^=<vtb$!M3{AEis
zZ%i?3yL^yi<_UiRo(7kG!DTi^H^Y)tTFlsv7tYs_x>$Ga{LKlDuc|_2UvK-p;$Asp
zd|@!>isehvIyW^;urRY|;SgpD@z;AAY<XqkX^HvKPuZtNRK%_kl2v&d^W^ok<7L0T
zuCIuSDtof%ROTz@tS3IHTxUFNcASt?7F=GV+#TgsVk~seS@~IEXnn^%xfZ!gR%@nA
zHIEJro+@(imDjf^!HRZIW;WU1$+}pmo}YK>{KSW$-ki(Z{4R+c{J8eit=Crza!a?G
zb4}glCXyog$<99Rm)t!I-E(2OA0M8Z8GEv}z|>lJMdNly_h{xErA<E$6fU0=b<bH*
zrD^Ii2ZiQ4j?DY-A8X$EsO(bKwO#yumDQo|ALchpe48ZX$h^Kazw~+f+=ctXeKIvy
z&McZbwJ?b(sDdZ?+xdy?&t3o6O1mt%vv1EQ(^EUX2$Y@4J$B&ze+H4>!mV2ZPiUz8
zUH&_B%XO_aQi=_kKHJ_c-j~gyF^S<x_3KI%#w}9vo7D>=6*(V?w61kOo^gHV)D6l(
z{!_gd``=&xlH305v&MgUzclAAkxC6>Sg~R2AF&G_jM_72{`|W;i{ojFp83D3x6v}C
z>#wc~o6u7D!R2w>j$Ql;31(4umQCzBvX84_;=I4Vv#d8u%e~ExUs24Wmj2HAl}Fye
zxHDl(CZ5{FuH?s4CcFP~#EyM>*2<^0x9`5Uetz)u&lXcU-|@)b_Lh55dncqpcyGP=
z5y#*O)ny--l?g=e`hLIA*`uXoaqRYe-`TF!PG9@=D|f0*;&lHNml{4E`6j=l;<DtA
zISLs$8|=@nKBRDX@A;)M=lb&fm)b48^X7N``o4U=3D;L1x_D<=S-rH^^%YyZ`!5Th
z)O27u<#|BKxa+!5XVQ^xw>BAXWcYK?<a4{X)cSPAJAbp6{BHgsT<czHof>Pk^h@}@
zUz%UfURKL=ExPma>)Yk)ruYhoEW5jH$NJ!_{wAuPj*;^Yhq^!RRTi6Q<o?Uy_Ho4y
z>(F`MTj!l%Nm#g(=kq$X{C)j-|K{XL@4I|&w|_~F?^%!k3=g&5mUky#nHqA=Kh@xs
z)8reHKa8J;t&{jDc>5JY{SKKwt2S9UAG`bZWsR=yLa$oy)FmqC^;qL($TN9r9aF2n
z9I<Dgo8px9r#9N|I?}kOrt)9X;+Un}QH+dRPq1n;G&9R@<Db7J`}9%fiKp|YW~;7x
z9?U;C%c4Ex)h)%;%=a_qOMDb?GhKT0lyRpk*SfV+6AGUOX|70`qqwAX%36mhO-xHF
zCfj%#H}-nwS1E_(eEzU#$s~yl!lrwcT((+vt5VT-b<^L5so&-qsk6DcAG3R`dc;Cy
zo&R(zi96b7lxHMV@n75Wa^=aDHh&FniG=>yc0hxHL&+_z&gT4Ful%XgjriU>y57FE
z(00MvUB9ftV^>toysGIHvQUeSp`rSbL3R1HMeHg@eD8yUVmFi~xq1Ey3(x6xH4GMI
zJT@nsX=!!H#QO@58w{WS{i+jeGWmu3sjj%t<PN<bwhqsKgm7-L$_n=9+x6-4Nkxs1
z_vJG_8P3V#bXqMi_mOtA$D<i9uhh@`I&I$hFfXSI4>dzHITfZfKmYP}{%={S$wxZZ
zZq@oK^PfR#rf$g`|6SKFt(>(y|5E9-zphS``hpKgzOxpT-}7(t!rnc3yV{Ozop*3+
ztMjStx?8TAd$)>iuvpf-PE~nAgYeVfK8wkXQfIDv3km&HDLU}y<)qkCm&$DVAO8w%
z-(D(Rl+!(R-rGy7uRVJCG<DIsxwlJ?oekTSbv0zm#QzMdzh35YzB6SJhjRNi28GGK
z`)6^^ja_{D{iNHE56g+1vTlvMrt)rk{Lh7A<+&fb+U0z+x_dP=8C^6qJv#!}g>NwV
zm-I9HRg~V{QYhTO-ym}B)vsG&hR2HT*1X(Z`daVst8_PWvBP(24YwNauRof(E@b!A
z?bqZNByt4%E!0&})%KdS&y_o9si)wPrqG_F+4q|n<Yt_=YO{Eox=rg<@!Mb5Q|uqf
z?%X<i`OaL2yUUl&w$8b5G3p?vcV1Yi!Kp)6k6LOhchw46@#kuu1DoH@KXYm}PknP(
z%=$r*ooVi)J$lmCEQ|&3Zn7UJ{nJ~xdxiM5yx-e4?bY8i#jIje`+WT^lTULyFY{{i
ze3bA>ty(hs+xD)!yNm9;Df@Jk)%w<B|IB-@G6kcfG&SQ_=jt!bo#(ssxI=`JQi$^W
zQoeYpO*iiz+Lg@t<#xT)ibC$qIm?~1cs)-|m&<+586&E(!$+W~?pJ00GPi>_{oA#c
zdP_}S^L71`g?^f;Ti1F&3UTzb*)#w4x$YekgMYhEmHd{da7CLdJ2s_d%S(2<7af-t
zG9*uqK3sBfUF*}RmvW01g={^>`FZ|-hN_uUibNR#{8&Adrmeo5nY%E{X}{KygK|DU
z-m&#}cT6&2J*UMWc+&CGl8i}DO-qH7yX3r<s?6Y?p&)r+&dZor(F#lbS$`kn^qcbG
z=8Hm+@3*>Mw6j|K7Z(XR-Q`lO6ynHJsGm~%lwDw=vfnA)X<Hr!yOvk33p}-M-r4)>
zg4JdOS+uR%bYFx~h0k7g_v0_uc^>`IGM})bUH9R`c;^rE6kan(9{LeFXKViAfZP*B
zkABVP2ogM5(0tnR>CZ*>X8aZJXPHZG+QrbDu;4#~{Ijoh8}_!&Ecvur^I~<Ap7phx
z{!tMQ&z3GsRkLW(jGHxqM}Q|?_S~co(x=mB9C@XDyNJ>9SLpml=3!g-=3MEI&sq>4
z5xvuNfB4sv>z0P@I-9m+`z50d_Wzb@`L9}T<QT7Nv86#tsHLIW{gOS0Y0Dh*GNXm+
z67L>-dipe8r#|BFsZ+ZD{u$o;ykyy{`?J%Y7VcPg?clQQE4AX6JWj1!7!a-?e7j}E
zfxiC?0wt;K3|li!>`<J)P?DwkvFZ6Wk0$1RES&T5*j34G<!NVcPWH{u+B!`(*Kh4K
z=@w0g6W*u7ceyYWC49T~`GfJTnQA%x&-Y)NP{`l5r0meX33Xe(Z77{}JK%1W=EcBZ
zpT~~p9Sizhl7)OGzWvYeZ~3{^bCO$XX9{mt=DX|mX!_X$^KS0e40wN<J@NHx$M1E!
z?quYNihK!OyV-|l$<hrAo;*u`Yg7Nb)OK+hS8{5Y!sny*TYp4*F)GXalwCcIVav2R
zQ4(Gg9Od5qXISxh-Mwu~r)+PveVgoZwYRrsgXn98C1#ugYQ+mBXE{&uG~GKTr+;FP
z)QL|~K|)JcF8C0#-pI4@cSg6u-1Xl(MR_kZmFjW6_UD`RPi)z#h&74ZXQyr1tL3%h
zq?beBqv|<*_tZ*Mk3Q)7BT^fAn}tU{{<)RWt)tHxr>vUKGly&I-KgztRi%Lixf!mh
zT2Yl*Ia5Rq`gzO<7g+b7Vf)N0+rwAqmTfq3M7g5)+XAUGi!;*>eo#ryw|d@N5cG~w
z<AF?|zfs!}rQ)CNGe7=ksFh*eb3k|31e=dhCnP_yNysfO{IdGyrM+`^m^W+=zw>%d
z;-P7`GhMZ~qS{3pCUG<~)SO{_>}|39h<#de-p6fsZWPH+KR4^cIdPQ<6IU=lo%1()
zxpmb2@@;RUZk5N%KX#iqA%<J=z!shUKvflY6``q;Csfk>cE*c7E_ZtvcQ^7e`@B8#
z7amwWwLPm#U~bCib9Yv#uev@l&)2kV*UoUU*@>c_EKjG)Jn|}ToGkIleQsp#=1OjF
zO@`*)*%n_9u3WdM&v`))-*WZ*Yb*QytuMTGZ`szXYxS1&CB{^E?GY;CYM1=^^WbG|
z>6uBB-+a_tVEKgi?^)qP58GJ<zV9vkaj7xn`>U){-I7(6Q{2r3f*ypZbTF}TC@b02
z99z8eQ9zW3!bGo19)XOP52m^Bp0bzeKk(i`yt^ph&-m>0{|urs&0QXhOx6<^S&ipk
z@UM+b-JI%MCN*c`B&Pn$@>)|ZBp6IL1o<28_?gzaUOKqo=4-LrXIoSiST$u5{Lf21
z{?FjctS`MoLHiXS+n;S;MXElu=RQu{ZaDX-{~O_$*=Lzw2OZ2>x_!y&Kg&3U{1bd0
z)o)vMC(4si=<)l~V_7#3zPXh#S6y1L=gB9zugYH)i;_OpZjM`Vw<LIXK1b#9Xn(a3
z*$74(PZj~8`kJh}2UP9TZhmdP%))<dNB@y)8*iUFAi!3`w|?E#2$@<gf02dRP6bm|
zEC2hodCkt==BALMf5od@&#dq-2>+az&gtAcpULz1@7Lx5+f#43wbuN+mh0v>Df_41
zHl|ClT7d^_-0QFZ+a<-;qo1LwxL3Pus-xEJ>yIwKVv-6}m@ogm*WL8C-KnF!715K-
z)^A^t^>Wg}fYwbe5)qm|o9wD(Ir>*CdY*5~z7)BsXX6^bs+I}cC-E?Tx60VO<BabH
zo1Y;;$N#yu)-0X&U`I?#nPgMx%Qpt4s}tm0t{7@HcwZB^z$LUuh;#nN3$?S|Zhm<E
zE9^u^<s7cRkG7n;6e9fjID7QB7TJv3{~2tul+2HNotSoIe%0SLl_JmYb+f)G&Ukg;
zr~IkqoXgkQuKs$pRZ}Zywxgk{TIuZWJ!UI*UW?o+_wU}+KVFPMvnsXbc*y=|NM6x;
zl6UHjzWtw1Z5LE}qMahI^(V6?hBZrr<-~WF^gXXERo@z&JFY%AiJhb1aNd-gFU_N^
zc3-%A=h)8|%R{!`_uSr{so{G$_wFj)tBx{FUEFJQwW2tkJe`)RvGnRqy2mbm*WviG
zs=(7#KM$FokUzHOeNcTz`IYCf5(^gm>bUbowlH%#^NQ7bZ{IX$7gFEqov8IG<-7V$
zD=iMu=^LkP`?LK{m9V><s@A#`1%A2Z&wmHyJ=|TDxl=S?Sx)G0!^!WxGwe3!&S++N
zzR^I<NZISz!zf|fm3FT6P1E9xJ*B23xX<OilQTtWqN&*KwX#Wn?K~a*S$h%$XTDqi
zram@F+GxVJnO7tpf4x1|d{V|2jp@lxPh1ds8D*NfZ0EvFoUEKp7g?BA&UpLx`LS(U
zrzLBI`vw2}T@k;jbLpF_-wxg=`I{a4D<nX*MbqZmtAbk-A60*EP$~agA6u#NmO*=K
zW?`y~#;$XVY70NDycP0mNyw&c5eaKpG=Bd~_$Isi@wd4KZHFQh&7(H#o5Fun?9Hng
zjeVh0b#3lCL{EA9%u&&%Na2K#0cYo2EpxxKlM49zG;T*H?7ed5+G$_iuOT`spLVET
zYCB=GnKk0fgXDj+F6zrE1lTcG<j=pfEkjSVKf{P!FI4NR#+Soa`;7g46+RbtaLF&}
zKP@=H(dv}DLjP}%`^uA@ee1kgl;!qsbq%?jAys&3`rXo#TqT=4Jr#U9=S3W3W3QDz
z^rK`K&n6*TH@ih0+XPn2BnZFCC=_44H&7?Y<f^r&=aVT49Vbsby}|i+^U8P2`oq7U
zTP1c^^-820e_P>yhM(b=a{ae0f1NeQYl`qI(dD03f1UT^Kf{R&=G)&YvvN1O9ZWe<
zx;ZmHAme$~1*>20L|6GgydcG+(jfZw)W$g#6OvhupPL>R^UX(b>Is{dYs4m&?OI~{
z@ia^M-i(E}R2NNa)inxOvQWEp)BO$1x2zl1XxZ=24NsY}d-}o4kAt@!-ZLX{$-EiI
zPh3#_Xxe9&Te70Cb;^H+S6q`gIG#S~m+f=unQv8_D&msucH;TIt1ms2y?9lUt(iY9
zGo74#CAw_VELZFE_qQ{5EL)s?p>@00RIfQ6GaqgK8_)56#?_?nkN7Tq-o2%eDO%RQ
zeS7=dm<JEkE~y$d`7F;_cgiN~_Sg1RwjM9_t}MIxd)ak|u*{Vip_N-hn`Rxm?EjMO
zRpuR|?X_kTnl^Sg{A7^+a9n)jqlq&v-u3)_dbwXMr-;7L{%IG={BB(+I`6e$``Xv}
zJGfGU*<a4_3Q6QWoTu?;K}W}bhVxdNBR-Tz+?1Ik*Yf0NeyGN~oNrhB{Cxs?FFp7g
zB+t54LypVv1Z%Ry{M&!+O!t`mlHhskP}t1b({XK)e9A8#lNC>j_Bro!&+l=s|CLoz
zBjl>E{o&FNM^`!LZ1S!Ca?WhJN6v&+hV&=(nSYl@Jkq<w@Y>%ZdB*)s)?L@~^FpSs
z<lpw$D=aF<Ts6tP??L*b()Zif|9BsIPid{l+*$7f&YMV8epxy3q!#CipoEhj?;oEl
zv8HT|#EqP?MP0#hNB7;C?`z*)qVp*_X!5&`nG(&0zbZ}$xdl9#-0a=#EYow~eZ=e)
z%R`fxGE<MN@8T?0;hcAO_m6+K`hp$^uec(lvqG}8`1w}jTv5}jX1*6PGGpr&F4-Kh
zwNOUjwqCKGrpp<Z^OJwg)y~yAExhmS+>P@}>JQ&aoSlB+<8Aq<3mZz)zGm9Z%D5UA
z_Az>n*W@#TQ@;IY2>!TI{YRXn@}Wtbue4tMsa?W2XY$Ka^L`7jH#NO_a;fiJ9~04o
zdPN6(d|V_9XIjjcxGLZEpTR#@p>=kg{NF=s=2$=G<z12N_4D(Wb&VzF>#A#4oleb7
zo>~$r^3rMU%G_3+4-x^7npF;Ju^ahSzwY{Yzxmm=xp{Bzyn3;Nq4LSUtykaJuP74_
zThpFY^&};EUTxOQ($Do*OTAZ0&#ts+3E1#8soB>i)nhqxmq2iX#Zpb52h4`|N^krV
zd8o!6sHtAB^Dg_jf5(s8-j%9HdYI%~dM;hMnfssNT2MpRtYu-}*<`*PWOZ?1^S|?5
zu-~fo$dcdIWo_~A`C6oWjqiAho#U2so*FGX`^){bwQ0T4yB>#}_GsDd?#ASKz-Dft
zOT(s6mBP=@lw;ES{xgKAdX_}H9Sxs(yZGPT{W8}t-RJnv;CyxM*N>V$d6tdM^XHms
z|C_RnH`=uC?cbB7o5OzjNoAEs<@#DIX0%(iKvdC~fr-<gaMf|He=F~Kyt^LwDxkQ}
z$M$E@r{(7JFG^Pgv`%?{VvhW&)qTHy*}dBS<FMN4*lp`d_O)pGe9qMfh&tk~_k?Bj
ze}-K)*$Y10fAyc?aFXI}<JG0Im%py_uKue#F-5WMKf|y2r?Z|J$e)c86BAHlILPv!
z;dyPXOHxT#+7s<*;ygc?^QYD={mL{=WLii+OHcon7a1IG*TfZm2MXQYrJm+s`kVF0
zj$2*R9wm3MS#);%?#(uCP4YNsv!XiVV&jQ}KSS5PVC|asH}TGwyp}B=f*qII?DX<#
zln}oCT!N*3c8Gqj-qJ0*8t3wH-e0iVoa;?Q{bLn=;f?<p`bsZoi#wOEeZL?)_x0X4
z+vo1zdO6^#?6ilGc@-D$#pEXHFEow+)R||gtEI9;SBODVbxus!wO_6o9YJQ7j$7@O
zmY>RAdw8Am_e3ch!-@TJCF)ztj!s>*t?l_u>-rC`TQ7wg`ZVo&!vE^YM6EZB#vSte
zD(3U`-TiITaF1c)*JX3;QWzWsm~HJHudUm6Zre>Q%@;RCc+LGzcQ`R6PsmbnTF~O-
z;W_!Ua>(m@BA>icxDI^1QtT3O>jY!a!ot(c^Y@y**f#ri=iFC<UVpts#GYOc(_P5Q
z)2gan$sjXjWyX|FL)p$lr(L#+OD*8Pr73ww<I~5pcG0ORzjx28eLvl6-R=9iF?u=+
zx0pNb)D&2-WvN@}5tVP6HvJ81MaDg^ZXS6$K{WbG;)*9C41Z1>f9Rq2NPXAo2{x}b
zytkWvG3KoEmC|*&Uo@6p+4f{dtG3b7r0N+z1eJ}KO*&ta9QPvigF4HvS&ezt(hVEm
zdb#FET`~B(G<w@a{ap@GAr-4TJ~ekrus=9HU!br~(KtamaYgN&^8Tmc)3)hUtKazX
zT=~-?tGNEl8zf`h9`IgSx!<?G-GS9lAZ4<I=f97?m=~{ox9&l|mtL<kOZ%nzUl0D7
z9B;q9ch!s!h8N!XpZ~T-{O{&Xx-7r8y*hPjVpD&dgOU;Z#Ap8ZJtN;MpYk<ZuYCTw
z&)X-Fk}mDLAKBZ*UD;ZZJ6$_|r*X!zrTSJ|iZT_%CMYENd9_S%nDg%!m!QT~&Z`Ue
zxD~$l?Va;+Zh+@f|DeKOm%gvC_B**F>t3OF)~&2*l~ulLIn~%|mp?ct;L%iC{a|{z
zjop>b7|(eQIqrLl^E)gAj18u4P5#qvKmV=At9O&`y<B;_s_$ujjn$3nOO>wEEoLgS
z6*E=;XIN)y_3^^Q7gI8?KA(ST=jwvY!W9!gFJ#<oZoFP??rhs!=jEIaCRH9kk~#Ic
zY?}_}vFEb&SxnaEIZqn9)Y6w!zwer}`qp=^r~+G8t7RuO-6tR4!>jk7Va+WezbE|l
zmwKYZ0*Xv4dta+=UtYd@YOO2lyqUkdPZf7-Bre_c(YX1jq{Me0!IdhHKHNT5`rW(e
zy#2ykGa_$zDxSDm^Z8fkwj1?_cfZbF&AA{y>q~KMtj-fr<IqsAs1yf5u7W0)d50NJ
z{=5`?bisDDvVQf`MYU}8R`)(P-Qi?<td;et<okby_Q$WQtESz%dpz;Wx!z1YovWeZ
z*KWuz<!xa3moxA1`CR`046ml@9J|Le`E=m%*Fnp<brz;1?O1(vgZ+=x&`|DI`d3P`
z9%||tr*AcD^6Y4r(dzN|v)u4@ME|4AX}auO$7lRLe0!Q^`(HNpT#uvur`CSE@Sow~
zkzcl(H(k7Q{oU%kqYbI~ck213Tg(58a+hHEb5i|5rs{XOT}fhgFFrn>?-%jwqWqT~
zizNh~{G60^`tAM)?OzzTUO065!Yt{}3qJWj&YHi%!e{=uhg=6ID;rw$a<WV8<O&fw
ze*W97&J2NuAXApgx{`03{xfj<FVxnWSvL9Q{G}O{+U$38oHzNWG8jyppl00TsUB^%
zbF$Hs>RGcssxoL;d|S45|Bw2=^0%eB?3dk}a7mGUo{q?~(1~opMxKE`PO|T~zF_B$
z+@&}87Cc)Lpm<yQN>^Qn-P8-x8o%maO3m?7(Er0UXPyDS=A&Xo^{-M#zMoZONqG4r
zyYlPv0EPK7VYBBu+k54$UAp%I-xMn*Cr7g~Heu$@;|){X?|-#gclT6vqRxVzuWWl%
zW^QTz5yBe(;odKUT>`7lFFV(_Yt!U+Vqs2GR-RC^VXC`qRiE=-;@VC-Nz<1~<tGws
zkJWoc#qMu$|L}W<WBtqeWgYPetFl~8nK;yMGTZEtfBCmm<IyF{-)E#xJpQz>x5aa}
zoAIBQ9&?}m;k0Y`E-cy;_+``6ZL(383(I~lbxu^jQ_=GI`120=r}E71H$Tp9lbtQ(
z@X=CUCnUz{tcpV`!<SWuciaCc{x>6=HU5wI6~EQF?4p0JsMuVMl2T(An!4n-%aZyf
zKUD0+W=>JzpS0Y<W?!l3dj4)!uAG`z^WXRXldgU0vHVq4`B!Dfu;Y53X&;jtru>k(
zRUoS2*m%UV|FY|?8rOpV4F8@mvA65be;d)=a9g=tn!Vm?iLLuTX8X8kSJPD+qpI?+
zlu5NaZ!s4&d9?T9&scW>mj*}sdn$AJWw*@`4ZT$3_@AMR>wTm6q1EmA!druDFMg5#
zy8P0z`PtRAhF|$6E}Ie*?(LQxwAIbu%<J!dHZC6tg>;ws9Z3~mjGor;U6K4@^+xZ8
zUEAu=casZlXx-g<=goC)-R!Iu<A_@`Oa3L>h%hpUtlnUh{@6SBdFz4D)4x9dskh>?
zj-KJ>H6bHv*~`M5%WIFz<zAYn+~pL&l4NNA-T1k$dRKOO7>8h8z15fX$_vU9=NksD
zs=W1l)>_Z}`GRZ}8k{Q0%6kr|OI(?5;}ll+tz<r52#eWUlUWzt+zlT_`6;eis_M#X
z_DiNkRr!~H&Abn8rX2dKy|yvF+?%@gX_?go#wXd#U+1?~9`-sUv@X$ZabFQrwfpwJ
z)v`|C`2I80EjN3Z%IEfP|JLN%Q`c|%nz|M*csxm*!6iXd=6L@0{B3jWc=!FD_%wP~
zZ1QIPt(Ioee8G_tPP6J(Zl4?FbZyBKAtsj@A4HO>Q`}Z<pA%uzDrIo1aDS`t;m(kx
z{|t5&i+*pq?xnrjHYv!qcZq@!Q|$^Co_^Us!pec&pFT0TO<~U6_W7(~>6z?XGRtF%
zKdtPWQ}XZ5uBx-o_fC2p9owV3O!&|1xt{T#UPejIR0<Vfc+VU<Pxr`@nI+LZ!rLkt
zpFa<?V@}~-bG`fTmb(tGAI{pdSMtTy+eNW5ujJkqO>5e*W7><%23C<?Rl92fEQzd|
zPqsYt@?>%fwfWrrCy;f@>(q3~gR*bF+SzZ7?p=IWxOhGHoK|&H)~}uyz5aeqt=o0#
z?YiYfCILT}Gb&tF)YRnQie#$zrvCU#<37<JE7!d5I@6&1Rr1uoEB2y4D(*SoUX*7g
zyZLQQ)$ZHtwuDYK**fj}i&YYVPPd|rPRbtq{h#4}y!)+~B^@ff6RN(f^FH!;m(j$V
zFPWD|O)>L&B^92PC7V0d>~fy27R%~M{abf@Sa<xbmF$it%ZRJxC+~k=%3H3s?!4Q-
z{ZsQV2dh~627ESUDdq}aVepFaO%Trn|2vPjcwX^V)>L#`l``#9N}}Pb!iYC!zpmWR
zzUx@JDk=1p=Aua=Q?(oz%HQ6on{m;6@*^JM=IF?%Gm|$;Cm&xcxHL4Jd%X$k!efVO
zFL65cDF_+MTb#IBV0BPJ;eF`peez|Wd2L>K#@g&qT)pjKDC4^AmrP&pm|ryM%?bwp
zBzI+-Vgvt#(yi-{eEileY5Vx`R^cNSv450f4;=W<5bl*8{N>W!%G+UPd0Z>DhI7|u
zBy3}mS|ae+SD#f;Th;TUjGNo!?MDSW@2os$ICoFh+U%m4##RXx&lToR<-GqZxcK(_
z$aQCyEn7Q1^XHyVf|`6yoFNRg4Vso={=c%~q(8QlJ3hAgeEQT}-*p~+lbC<VZB5u_
z?jE?SDnGn<YwXm-s^{L}E<Y=`s%TDGGB5MHq}LR#g!0GV=juP&;}z}QvSUIoQ}YAg
z!ljd?EAzBA?!EMC?anN7?r#4g@2+PtA?lL?R!*qnDyaLl>E<6lz1v<w>`s<7FQ-IB
z<(=8a@KcK8mSbj(($u_w@HL`}*#V3UOPCS_Y93Tx?fVxIXzecb*Z*zwS;M4Tp&qaJ
zUx!UjeP{OV_nKg-_RJNrxw=aiY?$=QUbFvGr|g1sOJ<vGzZ<3Zzl&LUsmM{Ll~wkd
zqhMl<)7-z`*6Lm_TC_O$WLBq>^pn$l_3^uRaI&<@`2TH-DX?a0{&!<j<C%Hxmk+;o
zjqsba!t+XbW!Cz2y?3>~9}jxDR>XJ5>Qp7)+E^zc4qY<`k29%W3LZi!Pf}XvOC8l+
z&OhzN_nF&d3+L_pu=@RhJ4Vv?3?-H07u*Q<&Uo`qY}N{=t}IT~uL89zn7k%+8r2BT
zV90)Y+WOv^JuE45$%pUldy?tA&P0QGe&?c~fUXpkkgE%SJk)fs34Nfd@a=c)V%e`M
zdwiz3_!!Bx=U+G3s~?cXW2%28B>B|c+(k3HvTk*K*>=NeeysbF<x^FriY#YJt2@Nr
z>vV1M&NGFXauV#XBmH}hBs-fu{`oMjX6{#sLdPQ?L!25u*~eWd{l&Antg7kj$Mp?q
z3FZ^}v;H{E@tX8V`Vs$D1DAjI?WJ^LJy(~WUvO)Y`exZhJZamcp4=?GHA`q`epmG}
zLz#{D_y5X>Ua2y1^NFgEZ@ZZld*=L*2|fMGaf@}y`g<l>iN{yY%9@)M6w0Wwrtuu(
z+dFk_>M@spP1}}aQr)IwCG((W)y)ro=bY=6kmj46UT?T_-Hnpq*Z!}#7sgusXZVqw
zk<Z5N7M9m)o%c3l>AD5ZU%zB6T2Zm>k)@_r#sRtIA}drAn2h5dTu&-tyI0G|b|AU<
z3ER!jrWt*Ph3e@>jSXy`mkRjY?(}`}4ZOB)+E=#kJM*mPFP!b4altl!g}2z-)yHPd
z>)LVItZfdjrUBEu+~apU?au}<G%0Y;exUv)W$GP%)7$q=w+pu%)X7{JBex{sx7?I1
zYpWA(&oMQbdNoY^p890ZR0qWks(OMSmNLyaeo}d7>fwa9?_}S;V?3{#^CPtK?YiWg
z;}cfzGnJ`X@jWBIEqCp8o1^FMU5a0{YP+VX^3~i|<&uiN>wB0cS-QCh%$dXV@a@GX
z9x5H#uXi5&E+=YTSb3VwMZ)IW&mgA9uCd-RdeV8@W`3LS@#}=C)d!wWU_8~wsD503
z+nVyrx1$#Lop^rK^WC2p)ial?OF1}SpURXUy<*v;zlSDXy}Dvbouckk(IPdW;QKQB
z4yC+wXWO=^&)H^@Sn-ula;wk3bS(IN;z)~^`DgjL>e)Je)-HKkiCZ<+x@t!-^a$~}
zs48*VuMwVk`Si}@SSAOF{|ueSCe`}u&yBKh+9Y@Bk=;dp-DB=29xYk<YHr$jw`G6B
z?$ojyPMG*Nc%o7D`MkT!13HiA-rNw>dnNy~@~72P!j0T#ie%^WxlNAVwXbX5o2Q}L
zU;VGFzJ12ZR+Z6asclckvFfrTS=NX1@^(LLt1;{^KG!T(ZshXIlX>Pj|I_wUnU~FL
z2^L>@qcmhyY20jwsbWiG&m42`|IhG+KPRi&PEO3JG(oQ2!tQyNnD*=LiY<5aZYNCQ
zFtO`SEZdmB-`($o*CE;OQ&pW9PMz~VXn4QF>RO~=9t*cHgP-}y^UKPX3dh{>Ub~pN
zE^243#^S6m{VQ#g?_QY}H#evH+uQZ=pDtZxHJa8U@o%YulTD|ZO~=LQd-CpGb6QvW
zJB7b4)W?sv>ij)s(c5+n{~1{9Hr-phMZM0}eZ#D>xmv4MIX8xKR4{XL%zI(PIWPD@
z-_^)}%5U<ng`b(#cH`jXhfhB(yLjukaN+^g%Hr4Oc3r<H|KRF{x>=U_yWYKi_vZfh
zCriEf!a`WKP1sVXu+*YS)Oh}VW~a^~mrMQc^5@NDpRRxU-h#QgR~Ylz46j?>TmRhq
zU*EpDRrXGI)%~sPFTY(O{5H5JI_fKXp10m{J(r~{E1n<ulaU^FdZF_1#Qtl&{OakQ
z#`_x&tmgGi*=Ewv_V!GW2+Ot!(++S7FfjOUEAq5Z{(6MlNYvo5=O<>lLz}hTmC6tP
zsh66bG-aWXw3ifvNP`l8rTU4#HUD-UU3>bFso^zuefy{*KKzEW&OGMy>YfuK!lT1e
z_{O-WT=K2`ijEvT36Ipo^UFM#KdqEAKEu2E&4zidTiLEGZdx#j!zlgRXUU62clMk!
zP0?R@{CaZa;Uk7~UcG!6b=4%`)(z)Fb6-RSEwX<yf1>;qgX)6^8mu3gb*WwAJouUU
z+s`ZAf?8ElfhF(j!tZ|Pdww&zt84Ds<JmQv9v_hRTPh>8WQW6_C9SQ6LIw*vE{coE
zu3>!~rhKCR+O!Y%6xa6T?&eqiad|37rk+{Itp_VqHMAb9oM^E2?ChzWd1*!*$BZWP
z&(?n$mt8u3i-9*Kx5a&T(`3eg082GKAOG9Gvy;y7XZRUM&bsT=ylP)i-zz>7fu&U|
zBYm_yCM}jvR6coZ)91}hYkTD9pZ>C9cYye=a?x#J2ZC+FmW7CFav3Ds+4Sf4y6Ya8
z?zhB3rt|jql@ZeqHJ{ryFO$`4S&+!qB@=3&Fx5Ta`}fG+d0C8G#?+~*IsH6!NsA{&
zUgs%b&9b?yB<o$!(z3#F%K1LKWI?&~uNOAlw@~%4__Csz)B4Djq7}EpS8Z!Ds9k8l
z)SiFM-X}fEXyUvx59Uu?#IkC6fc32IqE?x^kJ>{*Ca~;ORABsIY`gK6iL$@aN-e*I
z=VkV}=g(zRN_z5mu1n~}EtgfVpU9oEWL>yS0ORpJN$&N+pJ!g4U8f|Wb#~5t8Nn|%
z0)=k9kY?<+3sL(#yX?`!Wv;O+KZr2o?3u*pzPbO~f%Ca(zb+npt8};Kz#q#?f5c_4
z9bc-o{rhEo=c=VHZ+F*zS)H6$RnFveG<D;2Pyd-({^ydP)s-&Y^h$O~2haRh&p)wG
z<yrN1^G4Hid28#L`4_%hwmsx)e_H&m^x~kC-rJ;`Zsx3RFF1A5qvODVlW!+3pH|}`
zdRcq3!3#h4>s42Ea<^tr5OTk^)A!o_e=>E8qodcad9glPcXsE7i7$fkwBirz{P=Ej
zrup`Mo)p2q?jHY?rg$#=z~1)YpTt_dw`)bKg9Ul+&DEY*oBL0qKDw4QZ|&db7tY=+
z-)uBloLkZK&3v1^^S*Qj_-HDeztOzf>W`!F!ro*2eHq*xYJzP6UO&=yelFX1SGTlO
z?A2@UDWNBW9KGjE+7^1JRi$C&XBLmk8hNFY8r$wAe_RqSwrSm^V<GdU{%sFlweHDL
zrfAL5EKLUHlUn^dKB==a_#fM=v(E9tV|o2!%O@(UXtD?XyxO^Vfr;jV1<MncnoZlW
zLZem9Dfx{1jt77K{d&%SM9tUh9Y=BxPw0Ya+d^F=<Sfts37k9Yz_nL9!tT5}xZu{c
z{r?$OuL|0`$g^efc~!@^bA+Ee-ualZAY`+H=xsZ7<s(0m<`!f}rcF7mlD<ji<T?I@
zHZN9J=Ci!rWhJ&^hPU~&8(Lc~92d+?SgG~(UeB+kO2u=SSNxEj?QNOxa1ndWJL4z(
zjVu2tJi5iuXUbZ3>_Oev3;!9GaL=8*VlUTr)3P;D>$hcX*Ue@<v}uCv+Z9Y~BCRg!
z6ApedJ@cPo<$d0R;u4vWGdpGPS_FozZ~xIV$wgSo{deV`m-|a^Mb}u#7p=Y$6wINT
z>96sQ<H(QqpC5<pR<7`<Og%hbhX2@{dw%cLQv!q^PpZ#)QhsIIrQ2WT7Da!V60+sm
zsVb?csSCMScWu8ib;_hU9)9X-6?4n?x9pgDq(<B4{3Oo<pT0ib_w{)F3bS&>Q*K2y
zRW(OLULSt7%GY$?wBB4J`Jj81nSA1+PnLMjzv8yeETLV0z50R=>ykg5=ik-$`uCE}
zGRX(3e_q-9>ZadzsnDH_lP(3dYG_P6#9$e6aH9RjpI6MLU%Gw5WwzMM*tmD<8Nb~b
zCjNOE^{vp|JuNi2%lX{CkdVsiDQ*g#lX!S!%70$?e(9vjEqAYAE{BJeuU=2yztu0d
zD(P0%>VV7L-rcHN>ObfFn9TmJqQQpmSM*U;9+&hP>)H?e@ZJAo9ed;7tV21gLc6?*
z!+qW!>U`jJv!rF=CxxkHGn(o>nvEP66h=Q5Y2Tb{@!|fr<)1%9S5HY1>{VW9$$ojM
zT--L-+JEyu+;%-Cow;RtsLRhsQ#4X-Jee5#j!W2Eab14omUOmYIrD6u+N{=N_RA(Q
zr9OOBZU5y*Rr2R0SN}8A-T#&(=k~1dRQUC?(^gKH(i+O=Jjvz!W`?5w40k23etsCo
z$@*sD+mC&fDm=N{ZMYuGc(>m<tmx)r@Sj2F^H-58+ul{3?)RIuN3~?j?1{6?vos@l
zJ~(C0_Gr=3bQgR#i7$wK@~ZcOGfp0@){NPb@Yv$@(@T?T416YUTWMh2yG`=T>$g7r
z%ic{X317YK^;`G&?bjY{zIA`W>by&bP37A>S4S{^oT~RBQpfl|gP||O{P0`zJU!K8
z`EI@9_6#hYcjj&53Hd|2(wW<vPb#ada*XS_A$Qj|)c5jbYq2~38Jf)fZmoB<o%Q1O
zIh(@=mYS^cy0C;rV_8VMbg-Vvs+3P>9IsuM@q45(i__EQyR2Ky9;-`zwmKfmTzbD0
zJ!AYid0$n?xj#3)%DQg8_%3&$f2e!mw&2Y!QSN_sJ$)8*>x7~npO=EX#0S2`5C1a=
zxidtYr)u3i=k%l^_srF(dk>DAe`g4_|GRIB-B)Sf%eN)&h?HAN-HUIpoIcAamesjy
zLha$s_ZG*hn@sD})w2?}rfNQNQ2wwwWV4INs%86E{tE5>m3(vCE0%ZNA2LktM)wB)
z5@HH+P^o56f8?MnP{g-0{i}A@-EZ<K6A%69-Bf$-$FJ*^|N5O7!go$%n(@T_<+*j)
zTRDBr|1;cP>3mSFHl#s`DXo%6`~3DD{~7#blM7i2dp~Rbt2d54-d9*rZE8C&`iN9F
ze{ksKiy=Xl{(+t+Ecop-;txJysSCdGpJ7c#^-Cc!1qS)U3|9)1LK!B`=jlJ6b#ry*
zsj1r@r)|92a=3Mqi=Izwy85;{WgmY52F6<!w<k!4z0c;bNWFh0e10j5-OJRx4UH!_
zd0UQI*FQ7;zG`DC^V%9~87=>`BVkvqxB}9<gr*$m`K02(Q1ku#?~E%m7qdz^*{qly
z!a3>Ap6$Ovo^G4?YyWYjnOucmwyo3fs?<!EZA#($*>|X9QuVqs&1xl+uRQ*^Y{Ffo
zWdZq@e_mR5{gUhQ!ndz%FBg`b4bQ1s>XCo7bSi(~lK%`Y2?8OW{oB}9$;*G#coMp;
zMXkZe)6!l`xjFJwkxJ6~Cxzj&mVK@$3|ppG;%hNUL|j8NXaZkIf>-*Ua(M%`orUv`
zCSTJ1eNW-hI@v834YC_o)US9Sv^$d1X`Woy^;a4z^_Z6Gc{aZPc!T-Z<j<>Z439O$
zcvSvi{1x~)-raO3`|Mhcua{k&-89!tJN7zkN_#Z_$DrsL{&LBN@7~U^Qz}V4yM^)U
zmL9n!5Bn!Ftq%I*s+4p`|4&ft>PK4Bi?&4ty_E~uH8tfVm%$tdPa7pS_nfp`ugIh`
z&i?c5^L0Y5J#)3cBQ5*w!&)xgv!S^w7q4Dgdo)yZ;ekuPS8L8wKA&j2sZ@45@9sCU
z)8cB2e?4p6yI**J+Mf1)-4&CI!}iwk$G_dOHdv~}?DdyZO;h)9HSJi&|4ezCvYOk%
z&}qy!H>KN_{V|-smbbj*JCD4dU`=JfU$NI&doP^5cH3duZ7=VK91oVvd%{!uJALC^
z{$p=LG7{R)t54r*@?ht=Hk&>6Gkyi_F21)`OFx=f&(QCnm%yZioeIyl_}ATS|M!;r
z(DF&o9J=qOzV4~Ds=n^@VEVO#{$X-9FD)xeS504a;nkgZ(}lXqjsM~}=BdA9D!gU=
zU8=H@v$Hef>&|1+*8Fkvy{=lE*}T~$q``i^zSPF~{`;Nw2i*@6Ph<Bf3q5OHuxr8Y
zwI??mRW;(XRP{PNL1o3?#V4%33+&$6vS7AwdTrn7*Oy-g#`<Qkr7GtazI`6oVmGJ0
zale<@D&61B>wmZHU$MJ2<7>>q#qkeURC@HxFK&_XY2bO}p=d9>KXW6uu};Ow!#p(~
z=dTUOx0t2Ua@!^R%$)wIWs8^n6@8iSrR#IEs&-S^vn$(KmMj%9>2Ty=vVG;jTEQT+
z=E$FkOLG+`JoOX*d3mkA_$3ef9g!<{wWv=DyUf@ybpls|g1pvzWy={xb-(P4g_Syv
z9a`k_zBRzX&gS)-g}S;27I38`r7&<d9)I%ujeV}qgN=N5R$R>$lIU2|xOJhPfpboJ
z<MR&TdrIoZ=V}?ulz90xVp5uVozo`u&z~4Q75H?gG^l7yRQUG!@xDb#0h0V)YMLU)
zBd$F)wKb?(=)3E<P>fa}Czr6DO(Dzi_g2>1-#)&!&hV*I3&ZP+TeV|etq!-^)s?$2
zef6D0u18r*HJ5WuP`(n#Fv-$svXTq){a}~V1w1)Fn2#---y;{;!*zG*o!Ni$cD%lA
zH*aa%l@RXG&cc|U2mMCuU;33FJ1XjH9bT=FeregJU76DQmm15{<6oFXuL+3Tw(L#X
zoA+DWT*XyFR`k6r5MoL2IPt^EN;*a8wfgjSeXh3e-Rpc3gy(Zfzc_Yd-<|EIpI6SF
zwQ0_^7`f<InXRpC^;4!zXi)PPVPRVL<b2QT>la_dq$fX~_#<i2!)q<(8Q$hoZNAt2
zxg9OkIP3J)tjxdVwQ7+WMX}!Vf?9f}y7?(Mo|F>Oc${*n<oW6PwHBHG&hNfZcJ#IA
z=Gwf~ZMPS0zMCm<bnmX?@+-UoIP}^Ubk2L`KJT&M3f5VNtbe>OI=N!eBkxkhUaR1u
zWxMWsef9|r54;>|F+repP17Dmfps_f?wPJ(-jsKq*-p8B=F7Os5D%qQYp<Bz3*Pg6
z_LQrkXYbBlwyf;R{D6ij1yfoYIaPK{_4&KoFL&D_<BX-#<Cpzsh>E(jbGFl$56hk%
zc@=u;XZ?SMR{d+UqPUi8+*zG^E#_Ga?@G;pfPa(qYTOK^ysI-JcgMVB>hOLLz2Wt9
zm+DXVckA_By!dwZUfU+FDW4nH<?TvnQDpwKtY*cZW@T^XJuLqjij_L{l!YGRa(r`c
z$vfNq`&Vvb`?2ajgOF=>*RsVcf*4A7O}I6OcfEp!E@!x~ao^30mky~ot5hENxa#ZD
zQ=MOSO6}Wndsk_A{>I>F*R-^!yYjv~3Ol=Y{}Hw`p3~X42-FA}Rr4F%T(_0A_WObF
z%ZtwMOrFCpe`<Bv&->ErSAJ*vwshUCudD7{*<G}-Y^6@<HG$M^Q`my8@Od$u+S|Or
z%A(*hhpwH;yokgu-y=KvbDWN4tY20woDuzA<b2G_d)wcI9pDMRqI~(lza?^~d+NH*
zHSaK7d0zi}+Omh!R`lMQc4nSvnyZn}CnxC!#z)O8K3c*PZ-1S6tSwSD+E(*K?`avX
zbC+FH9W%HXSPu2q2;Zq)E@g7pZS5_dls_;0E*=YP*MD~P`;U?}3%0csdF@oba?(R+
z;{9{Q{~2nN3fFXcO=8U#KEBxF{`I+TAvZO%ZkM*de0k4LQH5vy^a6(Sg}>a|#BZHU
ziAjDLwJ>wSdIhZ+rH@~vzf4~L?fT1KYcKIG+#ymh>)4A~%U4Pqn(&Ds=8ud__SvJU
zeI;DScRw-zcI@lgCx=`aWgF!lzv8VF|ED_NNtXGS>#J|4w|~iczjJlWF5}4P_{`O%
zro7RYnp(C6yni-nxt9}PqH(`i*khjq$DT*n9J?v+C3o+5^z7!%Uro>NHtpJ8c>QwZ
zT26-@p}RD11)n><tcAaNRak=hXU`+^9y2@MDa@{pkT}(>p5VzW`NT4R*4cJ`@nTtN
zH#5s~&resG-Pv9(cX8v&+s6La-)gBli!HHK|C6>yjW@gDcXy`U3wur}*F(7~bC2Kp
zec9}dNZ$*Y=@$1*a~{bArM5)fTKp|$V{fO-`K@8Are3|Lr6;a+yVbY&ef9mqpEB3N
zR7A_!&);%m+oiZYsm--VOYhFVIXkZ!g_q@>N@Mwz-+yaO{N?3g7ccW(jCGHD?ECg~
z)1|vXf0dlG7+O{;Gb)90{bUj1@pyJ$;*Xua?pHnT{@r`6e#G~4OZv!M`26MLy4hD2
zZF}g`-OK9B>MH&6nQSI!P*_KQ;oHY&KAp4n7fspK`@!k>{5@xPehc1PdYgNz;H$t1
zRehncRr@bauWC=NTG@7G@43@4EAKjUI+QUga?d&DG5N;7U6qG^{P|;7IxR=y-0eDx
zm!a=Fc9`ycwc)mwmHe@t!8@b&M(4HNjP%%L{vsw?Zq^m2*#&h4${Sf6uU{-PKdGbj
zc>Q$0smG=+-8$Rv0OR(UUx7;(ZJly!&Z1IH-{2)e9*-V9a4J%8==?e7)y5;%&gz*L
zf)8p*2N-y7k7}>Gw@ohX#=;*BElrOarfMbmB%iQf*ClSd=v#BLON}M7$qwZwMwth<
zm)%xsogNf5<$%*vO%F*H4ViC`Zy$WWDqZl=E%`H7pSfo!@w1uyyz+GCSKEM9SyqiN
z&F1?k&OG>o*`Q`V^Y6^mM{})a<Yk<e(^z4$%=`mO|ApXXXSS`(4EvmGwmnMaynx7#
z=9y=h|FxC6@9Dks;@t7`Sv#B+4Ml^TOBkBx=LEAo&)e`mnma7}?@H6~5|eE&S2cU`
z2d<x}u+USubw#-$`?krhmBopbtIXf^cAPKy)wTXl?1!V%1-7Kl)7c^GX8uLuY>1fj
zzR-JfZ?gLC3@u(6yYko36)c-B#V{T<TOs4aIw|?nk{NH;FSuqMC>57t#<qy<*go@K
zm-2qK*52^ly~q4_TCBc%sX6ag{hiz5#c!+2uHOltSQ@=Wq+BZgSYXrTf0H?)^_r%K
zI7LT#P38))Fer_k!hL)Gws#vDSFqidyZ_YH_Q@rm?K%rq{NC}t=EPBjU7vSWO}&0^
zQ;gp4(|Ngn_dE>^@;&~cWPgiqs)vL3)ul!YcC|AXEwbITKxo1o_K7vYSxGs9+S`?O
zp7{1M=#S&a;PVEJS3LePC%%5`{(8r%>1nrhtxGwUr7qzL`)jiNhJVr4zEf}C1sJQ8
zpLlX@nz8Qhn6r}lXJiCFubT76slYKvLblPqeXIMtIoI;#gW8LtXBAgyPxWOzc<hAf
zJ2iH3P6jRO>dh*0w~A{&=JYFwb;(Pc_05|<*W^J=UdY5;y;JM_p4Xq~(tSC1(){$?
zDY@(ieUHjqdu}7T&(E{jNagwdZR#QGyDfI#^}BRp-t*;W!tdI1mI+3xm>pqqx9LAT
z|Jsbx(RsUOMSZ`ttKUO(<wDI;22aU1MlQVJZ<j{)XCz;K9%lW5BVpn)!%N#{=e26;
zZaB$f#+Wyg>)U!YCI<_qYL^$^9u%F6dt|Zg_@AHimzr*yCaU~a@THxsU8%KifK!Zj
zxYRPH&?8G1+-fvXS+OG3edpi0^w95*zI(R5+1U8!MYg!whkXfcwzk$+n(zK+aGk5P
z-c;?T^Nu&%kr4;=3gY+`)DoB(JbPSTznE!f`RLn*hBce28Gl~q$!|`WZlDk(z4EyS
z_x=mDN&@$Pm<NP0ofpW8&}{XQd~Ez!`N?ac{|piDb=!8I&Pz#VNx0<skJ~j_#>YcW
z=0$&Yt$cp$#;lb)qxG(smcC*P>MCVg!0~Nax?)g`|GW!1d%S;5{njcJ<HT0U*!HvM
z?b}(`|1(^UO13({Q1bD7|8g(G7vFB)7mJ#E(7kf?E}IikQE9P3vobkZR<nPaQ?AA@
zQ+7?y;+M~=WkS;b83IIpJ!d~ZReR%hu~WHAPJNZx*KYEBmD|HaRnd;MDZfs{s1!Y#
zVZ^6WB!Bg>^1fAX=4b4>x?bYt67%b(re~HWpIIFB=jGJxYpf^Qb$IWQDrvtkA!vfs
z<c^sZ44-5V_L?+hU*>DKbo|~n!T9T<v#0mD1ZMoJxTyC%_}a27r@tEpET4AEr}|RE
zMuk;IoI*C`_Uq;r{G9rXlbPd~k*9Qj6`!6wN0VpD-Gg6$g$Az)^%mWJ|HG?qGuP+7
zKeTH>Sd5}qtA?cJOhLP}lo?k~9q&H<E%b`BfPeK$(fDbv_3zfQ^nN>6yDKa-ck8BA
z%f(K`@36Wga5HGbg-HuIKU)|jq@Oe4eZure*+ymJ<(ho&yJ<cXO7)k&kJ?r}_tu&J
z3_oHIcbmMNveGl~(K7`z2MNiJE2l0Sxl66&v7FpIf34-#S)LLa7AvK;EZn_GDsAfh
zLwgpz&d`1m(9(Ej$_!1}#>N{x=}d8j8W*c|)tQf^{BGW{@<<VzuEc?_%dUIun^Lv>
ztNw1;+qb1t_Xm3QF|B%hH1~3lLV0liM3zSNK5NNdoe9(bJ>53JeXi-&@Qz1rS~lTD
z+57%8c-=p^dEsCE72Dt4TNm87GJi>1SYSZYDlr+h1k?WD1MCcqw+$Cgjf-Sf{P%p~
z`O9nbi$AKo{=38Np8CvnM<dLZcIO5J1w_6LxwbphkY~eboey~mr-G)mEMfl4mQr)_
z=2GRr3F$J&jOQN~ji19QZOn5~Tf}FfjAKv-=c5p%AOm*2N6NB&b-%hYZhq9Zn<{JE
zf7o<QQPw-g#vtBp6Bwcx0u)xPbeWKFUito+gEJ@8yb_s`c{+t(sPbaeZnYDyBPUGS
zCi3WT_mmHI9?a9XG<KXkF?sE+a_`~=E7%V_e;MQ~+;+HVNqkV*^C11{UdPsmp4fIf
z;B<}mR94oLj|vY6g(?}jDg0=P@|2X4w><F7$hLhdW4Y*o+`@efzn*RRRlKYG>GtCA
zlFutdLcLdrtP8un&iD7^pj%u*kGPH(S*kj}HC|)8ZN|6B9`;ZCYByfm{^0x0;9N_U
z-WM&UQB&tF+k4q-x9&|*KQr0oYl19I{w!F1$Nj0tn~(zgT-WS$&6d~F6%)5#R)6V}
zyZ*{fpIqlyzoqwfe=fRI@}D8cZgt$f)Fq+!*70reH_@xrO5uCO#h@8bFjM%Qa@@m9
zhcntEPRs4JkK*oh)oI@p7uvag`71Te3Ae2umtBq#u<deQ;l%j%ZK{GwlFbiG;RWk{
z?78^R{>bMwQ6H8kFPRtmIe+gvz8QCKZn&@K^g1ufHt5x*1<PhnS+d%8!9*oLJN3Z5
zx1XqPWVZEx7<~J*O3{n@tm->&!tS1R*kyH0=&5$6*P*6{M@uFI1vs2)RQUA9&nCM%
zS>cK!W4nRn@2bne3^fn(Kdsuf`t8cQzazK5()n)NAJ;Y6+r4;k34f$xOu+I0+a(%(
z&(x<>r?`D8I6b9*>%_)S_KzL@&hl7t!RqIQ@Q82uG1}&-n;vxA%Dr2-Y`dkWcc54Q
zPw(JKkGnD^RIb!?a!ELn?(=3{sL0+U`?u;o3Y&D&Yytnd8UGplekXQ2?7sKzJIngA
z>n3N7R?dm!UYxpVlR%Jz&6KTcz4~)l4j!30{g$l3@;k+ge(0{&ex18=^3~Y(_&uVo
z<s0sZde<-ZIvjIlou0~C6(u)E_B$d*{5M|2$*U~+Ry6PZ^?Nfua)&3c^bOD5ytmBk
z@s@}i_kYiPZTD^a?%1?S-v11`*C$+cbg^(SD(E<2saA3~-*=<N{E`a-CXQRotu{oL
z?$&J#HP7wMc6*w7iPih%8n0DhZcgk5?0i22RE<)e2iks_^u*%QK2w#fj$7CEeU}f-
z&YjYyHGTQ-XU`*~zM0)s<OpeE2zt_BWW+eZ^LM64Pd1au?Y~Ffe_7dk`hEZN^CCSP
zzFaT*np_c>H}`SYj@;kOe07$wFQ-R`Fy``ZW8bT*sCqMg%N)k%;T-Rj8~HrWPqg@>
z$zx*_nQF9OIqcrW?Yh!cjk9YP-Sy4fzGYI|?ZhPu1=5uzEsbg(pDPWSR-SWaxx}AW
z$3?d!B!ou(TDmbXa&4I3f?IRV{+@L$@YV@g8Y8krM`I;dj|Rg{yO#!&7=BJ}T{zQP
zJy)f4)px;dp}8LAi}zmMeQVi`*>2xUZiU;P+O=y|iRP}dDO)eC^Vv~xPH{8y$&O=7
zek+S#*p&74_J_0Pxr&apFZ%0C?XGT^x?OVpf?Zd478ID=S~=^Pl#;!Lyv#ZKz{f68
zAJQw%G^~)5t*gzl%y%nk+<tBrk8_7tS45tG?LrlS#q;hNK7M4fU0|`$f%Lyinm-*)
z6FS-uzx&ft{ebPVztiH^m@m5<dN?O6ROx=C+Y*LlT$7fvPgpp^##p7N@8a4=EjyKX
z-k(>u@6)UonY?jgM4;KXyW!P#7pnd<Z2R{5_rB$m^SR9KK3X2B#hA^$GbmV#Q!<dr
z`nA(J$BuXI+aK<~v~+ioc+e-)$X`pZtbKoZwSBNvW1yS9nOd$1PwsA|nB3`k;;yO(
zmR(y@z*U&L(A$10XF=!VY2Wg0S^LU<JU#t+S9ANCOXlCctrxBjlTF%J`g>LM;<|Uz
zP5Jp2(TkU@SRVABfp1k=*Uo(ho>VNoK4)d6QT@}kD=n@soWI3(O7F_>bw+h{F{|!X
z_*}fHm8q><^mN0|YyB>}Y<KUPz0+*H*RJaAt&dmyK6&h=P|MQHKdgJ2R+Jw3&u}s~
zQ*P0mGb)uGmnIweo9Ajx@3ir}yQ}1-dvvt!+<g(>&&Mr3Qhj}e)PIHtOTR3xcroqt
z%V&?a-JN%T+bf3Y<)xxQD_>>35~<V^NM&qTBsA^8JPx1VQ~YM#a!Z{2yF&85!u%Dl
zL+2?TSDnVo!NI)nkNeuf;8*@Xq)Y!ZEUAAazOFAebJy9eE1xDV*`m3wXQ|ftPetuK
zo{DOIUcooA4x8SY)bLs@{I7la%cmZ1@2U1ozqzOKy5jrW>_u;_*QVXSySwb?w(qP;
zkCydqooe(tvWZKCsYCu1f34w`?VESTYW;mb;b(|;a?$gUK%R={qL=<NEb48nU3Fvj
zu8%vi<o+`-tytqLI&I3D25t?z-5ri{*M4elJElBI_Jir6eU`s<`W9Z++v|An)A``o
z@P`Y_w%y#7BhR(vy6$9`)`hxjpWXMiTXgEsbrvQMIS$3xWl46Xu6-BmRxB-8@P41e
zrzd>(>`Ly+g(aOb*y~!<{W^O2mD};v5!-dGS$*ya{SGzd?aaKjMUYFjQOnUkzj;pf
zx_ZCoVXX_wuP*=a>Acs{n!v*f4x5h|&UxSS_+QpCrIu}xrH+oC9+i&lceX5f{)VH+
zz2fTRt*1B0wD*Pyy?e0fTZ;ZX!}-&$#BSSm)%DuORjXHP`f_(?Zf$s@b44ITSoon0
z*WzE(*o&9`iTk{%VT=C0uNNzXn-08Y+>>=gIY;>epUs2v4=;nfGruZN;fvW*?b!1D
zU9v&)jMGO{dfPlse*blO<;5qp##SrrWU~w+FI;~9;jP83zx9E-A60g^Z+Y@uiHGxq
z?Cuwx@5TM*d|51a$U;O*Blzj*9~uAd=SHvLcU3&nBC4=ZAXHh#bEfTa!Q1CnZ5Nl4
z4l8v3a67tXg1xi*l(T_v=Ps<eSMC)utBQ@)Uqrj{nFv$V6P1}y7<wd>r4DVm{(0%H
zH=NHO2mSFqs?_Pd&&~ew&Q-;mvSe25xnr(+uyo4RS5H25XFL*$u_<N~n#!u-ULaB|
z^-e!sP&4m-@y~1fL>^5#TUuW5A${|?8eZ$`JItfYCdSNO6~Dzk;PI+u&b9NLxSXCe
z`OI@Ed@KAQYk%1VFR3dAq5i!88Dv7%)~l5LXGl@s{n}w_`Ez@|)-b(|q3(D6CU~6^
zJ;D^M<X$-M?`J8c<Lw98CfWby+jlQ!rOY*{3(9gIUO#;5R;^roId^xf^LOr55w(dM
z+759|+NWu-W|_Lm@f%t89Xe@`q>}n8jvJMIdEV9ZbiK|6p_%N7PoKYcyE;EOEOuwv
z7SrObVOKUXe)hY?Jyn2Vs;aBO1ireD9~WDMAGkZ?5No^4CI|NGewo`3Z@ZrM*tu2v
zJlm?T?W=xIPrW;L$trWN)2^<GMYEQzny_tq$krky4#_5#9><BoR*A>#q_*yA+|qb!
zeH;JvxwWhxz8~3qdqV|VNHNn|vAeIX-Pygc)HIx}*WExiL8mzAn8ir}hSuL373#z1
z6;^*R>0}6K{O!NHdB@jt+DoHrf;gmRJox<j%S)Hi+b@@|+w#S1>TRt{KeV)Cen*GM
zm((;{{Pmf<;9Z?~wsf)XsfK&pm;W<l@$tX%J^EZQJEMVPcgkn$z^}Tcb2hYIyQisn
zH)f5g{ijA_*?lSheyw)SuX#OV%gc4k4_jP5&f&Q{Hnd#(O~{K7i%iF8_n#>YGB&pR
zj&weq_x#h0Pp0wbD<9A8NPWV|{PEt3Z?cQFXFZqr&ydR%kh|qVA6M3-h0hM;9+}d|
zuhQ61lh9~;rtX*3y$3Ohos+dHo9?%_nsh8#bwB<1x&xP!mgmH0n!V+o!gE{C#M7%*
z<&w?xy&}tRv&zqunDItM`@yU0Tsm3XF1iV}2YKdHORe!d6nSptjNE7DtEwvs-kyED
z^VX_uv)raln`^h&E7ya2#=OW!X?_YP<V-IpWxUUIWDW>!o@V){IJA4WyQehwoF6@Z
zc5x}B^&Q?dHG5mgm1RBpljfCjd5N?*DzTR`xqnyQk~!y^)6p=of49wbKH2TKH;wgN
zkmAzz&vo;)l5+O7?A`Z$*@aut<psf8i}u~g%(>EfwZ-JF^J{Nk#mb5Af*g<M{OC3E
zKAG(C)MIP&My?{q&+OKf7Y+Y2-1^Uu`-#owKZ8~3-{4y1cl(!}J-q8z{hsWXQunK?
z9+>@oG<VjvRj#_8i}QA>&*GV~(?8H>OQFZX?;n*+mtMYe>&)?h(*F#qhk98)ojmqs
ze>2-0qs5P{n+o?9tL@Lb8u)(F>vLwSg0{Ab3JEQbetT41i{)ceq5ZNoX5tr}-18Qs
z@MOx_*=ks>5nT3Gwf5YRgpxHmK^yemUkeY9of`CMD*p-|VMfhH10lx7#<xGub?@FB
zJ<U=@@Opw>eECc7bxB%BoR0AOT=Tludrx0<O8;Bko36X!*OZuDl{M5}vov18V(q?*
z+J6okDIfE*iR|pUq{^cHGu1)<cffy!q^W5vZHZ5}md$_h`$y0_-J_m)_w57ArEaG#
zW!UMToT()$su5uK{Di{xhW?D1Cwmp@^ouQxtX}4ap0579I`z14i~AxQ{;V&mm;cSm
zm75!xdyRMgcHJ^*W8Z?jdA$qfXG#RV3EQJG$^WaM72mfTJzBk%&)DX^G~&B3lk@&*
zrJ0;jJ1nMHMF+TZhHqg|6MnpLvb>!sPgK%$F9w^+IuqkgJ_)H?UbnqYiFhc+2rW`p
zVvtnvc;r*rzpc;v#<Nq~?Um~JwnX_nEuMR{R%M%t!IrcjK6aLgldQkZP=5Q^DzK2D
z;Z^7<xi$4#s#9ZLDx1z-&~%Aw0@rbg@B9T5kH5FC__)y1S+Og4d*{=Lh*AX|E#-oD
zTC%&d%brJcEzeAGSeCP6kxC~6lar%Y%kkqkUd*;GIjl6n@cI5zOXr@54U%fOZR>Mu
z_r~vmDZ3SqI~{lwetb@)<T;+1CnP%G&d5!h_}fS3tEGI$8fPm57O6RRbi=Mjz2cpG
z(f3R6nhX1ueDac0zExl8XK1@8TU<?Lg7D^w-sAHe*1p$QQF!TBTj(X*@t+}Z-p%r?
zvu~5qHrHKR`Q-KyMYWZRLj0NmoRfv0_#c}Uf9IyG!R-@o+GnRrd^6Dz_?CIg@Jfxw
z{3X`=3-V4gZ@s-fcYSW`g-<KzW%^E8n%w0T{*C?a{o~0CD`vSgyW~%F|FCd%`Ey^Z
zUsEq{4EoJ`#r`1ck%e|&cPUH~n&WApAorx_v1fJU6KAP)OFtg8Z#DFLw_wpq-cN_G
z@h$rrXnN|FsC0Oqv0uier=?zuUd*l8@gh$qt8@r*da!&_FnsteY$0RJhR=%rC*E&W
z)-?89ZCxGuaH0RhmoHx)Ud^M*>7&80^Z=vnMg#j?ooRQEv@lP6dHPOWX-dSWz3<|t
zwB8rzn|D{g`qZR1yKZ~$uDSI}uQV%9I(*%mP5l8K>bE+NR3GViaavNn*Z=wc#~y0C
zJI>Bqd}`)xtCzd-OlS99-o3_SLD9_bqMi{4`3&Rz7A+E)Xm_-w@uW(^LE)v_1%(*W
z&S`#0vCt6-nfg89?djX+_eVRmX(zPUU5xfqpZG?siN7Yzk=aKldW)0V%eP-e6t_G1
znVHyF6~AJ8`N}-)P1!xJlS|4LFfEz*&TAUW@$(bqb49atc)Z-q<tkUW@NSFFI=G|C
zbff+@`5(2nb%jlQnRb<KyWw@CjFDUGq{@?-Guij&R!L89nQHr9>(b*Ex5LQ+bB$is
zzE6zRW83%Fvt-+{`Ii&hqynCJH9p%=Eobq_pi;<BIqvH1o)aNelYfSZx6Ej-d9^(2
zW#*|Mu3M2&ork6<Gu!NYAb0=krOJnf%`#8^oRzw$R=wQx`;W4dr4tq{XLeb$;CaIw
zk4}sCD=JFf9b6PMRW@5|#oES=>!+{0{cXjxE01<+GVD+g{%i5Zhx3EOmAlikCBj)6
zuYX;;wA10eTuH|(bGKWsbYI;Luf0>6cGOqPck2?7ZJTdQa`QBtlxD=hyz|>k=P8Ag
zFH72M`4%3q4qk03a(UUdz~G>*UZ-=nN#rtaaa3?naB-=${c7YXQk3%fi&9tL&4x>N
zT2*zV%j@2pPJ6rS-Rz_G0kNV<>+ao(Y~6O_&rb`L4jYHYW9n9=rhkO_-+r&3C$i7F
ztR;1o!C&u^g(4X%%6=`2^Ze3k{N=UxCikZb&0HT`9!xN4dUZFp+j#x+FDu$5S~*yL
zCtN!8`f7O9&ho_@a=Sx=G%f~eJaboF!OFDsm{7}-6*HBRSkAE7SVbhp&fZf~*uPeP
z)`QrSH`Ul`io+&uxpd+7%gDR{X{Ugx1uPm43{TFK8Gn1f!0xDcR_yKl5BGlvUU1Cf
zOOMdLug{o&^wlQb)7IGjwCn7SS1)>fef{n)jFR)J=UOVNw)EK~`<1g2wk($@d?F*n
z*0p<UpQUw1W0H-ZtN7s*BTb8g|0=G}J?*6Xo%_MsT}F?$RaKN8Te4CsQ@Kmh<M)&W
z?ky^>3hQk$OcZqLk1JG7vb?rOQCDr}TfLN=`Bq<7|B<Vk{x;U^rn&zryA88L_pi3T
zyJ>Z-ZP2C5+p4WPf;^n|KV7w8%K9FO95a=ou%%O@Y}T1usPMMc@_)Eu{$1ltQt|}t
z-CN!Tgf6`Pa#iNt)CjJW)*_38J;L+;{;Lt%bobC^G3k2y*+E+qjQ8EEzET>yMU!vw
zl|t`;snwl<j7~}EYHp?G0^uGf!?;&Z*Inh};O4Mu>-F<X;@WB*Z^mvdbzWY!_QVIr
zta-aH>xKqBV!Bl;_eF4m@!ueh3u50rZpW;!cs<FUIqEcL*cFMFAJ!Fr6@J0>_wwyr
zgO|=F*L~M}v~2ax^0KhHoK!z$vZ9s{n~&#7m6_MAAJ6BEu{o<H*!H~bq+QGVa6k6r
zj{=vhV?Vdz+^3tj+s;R<GCCdhdbiyondLh+?D)|tDKW|CiHx8T-|CG%x=(+rUiw}+
zdEWPB{^ifZp6qD6aZ*~yUvc8k>stSoO}sky_qO%7>lQDYVsc``)kRO9zTCA`wWVd7
z*HrBX7Ax5lJ_Q)mOkTJ0!}PyK3Tj<WZ+_=D)mrb)sVnJ!WS!Z%+9rS62a8{fAxxXz
zt@B-2V%q&+gL9q}tH(r6&N<(Y9AjI5P2h<17T!6Z&;9tmipR(;u<&l)!d?GDbE=Z#
zzOL$QU#Y1jeW&-LQr~F?*4^;|4|Bin@c0*XCC+PS6IZ|~sri*t#IJ7)V(U;__Sw2R
zS@Qe3#?oYs&iGvp&-<nndcTZ(E&HE=rK)Y$m9DNVu8@USEq1j<r(T&gRrs&oEU)DZ
z$3>>_u&QOw7p&3vv@S&6U)cMH>G!my%aS~varN$-VxWKR)#=?Qwk-_(vQ}gJE61ic
z)lDM&bvG@prdr6n<}~J2|M1~#*pKsHldLO!JX(%f%?X)$xFF<GSfFA4_8pV978Wh_
zRAuCpG@2}|`B*~O$jx-N9c$JL*E73+Cd<c_Z2!;j?CbrET@o4}j`F|nTN(9T?Z)bx
zXM@6TtvKskxA=4^pVm@MEeqyPeSwxUKhL>gQGQrASz(^>np*bdb45SqA6xdj?U`w=
z`%L+5KNdurHeGvaTk4s*$}Mzim6xizP2H0NZ*LiYoR;1d(|ogk=XIrh6>XWb+)^$b
zc~eyO-Nea!E7xhCkSj*hoV(rkFiogc5`M?7aNzxI-@H1}X$oihmYCaK|GY^0p=N)P
z^J-b~3E!VbS)c5(niX<~HA_RrXXT?!PKq9Ln2Z$?Y7$;avv-`ac8c7?W?=ZRcSWD{
zp_FOqv5}!mb0>Wcd_Dc`cd^Xapjw4RUZ)o{pKe+r@`uUK{r&3PqqBBxoBGYvUPvt^
zI@G%NT-D=qbAFo_U)uEg%8JEl8}{@cT6=V1)|07y+J6;Q-KT$hyQ<`mW;)BAxfaS2
zUswH+=eZ}n@CY}HU(89-_;2^U%KhHn|Hj4Tz0`Qhn`_hNXctV4UU8^1xOvJ__e;XE
zOBO#fE#LjvtZajz!p(Lc-t_a+v=`osKP;mZaMbB*{HJfym$QpXE=mV2dS1Py+xFLW
zpP#v+9M5(wF<BHebuFVpN;&h!zXw(o^mmFb6}!AL@P*<0xRP>?UE+KvA{*=~?ANAj
zi`MH6k9xU6Pn2WVJ{Q$18ahSqn^aUzu)6PHK6z=McEJX>W6um17f-04xy**UId-0^
zjM#4dAjkXnr%J|H<yI9f*jaeTTzu6P)vP1JY86c>oO7~vryluwOp3$7Z_eu1ef)pz
z<kRKv*#C{%UFyGWRo;o^wMVLQJ7f4fRnOTTN@=h@Y5CXm!1K1~nG*Lro*HksF~4f%
zVG}Luz8jzIx4QmkI4pW?RdjD@uB-J_Uj{Fy-{q3FI}Y^EocQhP)(5`@Bl3z=YBH4*
zY>dp$Us^CNV!KAf<mhc$?yD|Le>r8*XJ3^ak4+huG6X9tu`^8dkYC4N8*MRn>7Tsq
z%sfZ>b|+t!pQ|POGHCURUyrs~EiYPdZ|bz!bwAf`4_P={DreiO*NY9FGD#KaPMI#r
zscjVgE@s7Nl{w6-1=6OT-IXel(7asw*Yi(nxYwo_#w4wN+PqZcZD`y(t@ZyIm=^`-
zEb25}J6&=G*LV46Dt%hV`Afb_?YQIcR&3gYvU|MOFYeEZsz^&?Q<Go5R5l{V>E2(j
z_ur1MUZZ+_)`~w?O?zEG_CH-$A?o>Wg+h5Re@2{zlx6|vhm*=57ChN8?^@}_`^RH)
zox@+%Uc7Sl{0{x=zn7jbxVtV?|LWB{dZuiP_CB}Pd>%V#`CLU#nJXcS<sLI+n?20E
z{qI$1E5nI%rREc#&uafS>$3G7Pc@FGjiFaAXth1urSn}%ZF=>hHKD1Z`7456EYtAv
zR#X?N6g}TCS$J;r4wrzld3qB&W6z!LtNYKeG~mar1JfmRO6B>sGH%{we}#XA-{M=_
z`n}d(w|@6vbwKpW<^E!so(fgn;VDamIRAP!uSk0|U8>~v&V1R8Tlv}9C%R{(DxcbG
z{a3P{Z{NIHv&%twvN!Ik{djdVqfTw#f=_7;9a^qh)f|R;o%0^M?D-@Su{cHk<ITEi
znaj)fmOB(T+4yBI3n^LH!X&Wm%9JBZ11byWXUZNh@T)&PUn)lNZ9$4e;^|Whr`dd;
z+Usq8!fM0Ps(hV8L7Wor9GKNL+3JJ8UE37>qR5lQ{M^Fq=>ZIq=c9haI%!H2Ej%uA
znUP`ACWU6n<2TN~;O~17b!g+c;Q4L$=Vh#T$CF%oLax;J{r*s&{|ueGy|?C0`=06i
zly`T~Cf=xLBFipJ5IxMU!g<m^$@{cKa=`&@woekjo~@qa->X?0C>N_SeYvgQed*=J
zw~t=hb<saJC`NzFS*v!HLYq{tGj+kT=ac)M-f=kPJWcY;pNxOvN29|w@1JTZe64P-
z-pZ24#nIB0d@CC&Kl^+$2vK-^;vDB2lkPW4<%$O+Uhvgx70q#w_z=z3pKCqq=#I>L
zb?1wh?%c%TJg@8LN4A71eb3q7_S<e@Jx~&)d{R!KF7n)!Q<qq0>C8R(GL(IN$h5cJ
zyK;6<IyI|GYSy!?)?}U44YfJSSDo)9S@IW5;Q8dIGS}dtcsY~tEm`kJmA_Z4diy5G
z;ef_!+wJAO+JEh~<Yj;LHsSflXsR!|`>5`cCzDq6Z?F4c^D^h#>VnckMpmM?Uu2zU
zJ<`p$%yi}D=(*1eZs|);da_(%@w(N8r@i&I3%X5F)re#?n)HyXgf&gfc=2)mH-*jn
z8yDs%>8#aQ=5(!T!3H52gTe=elj_pX|7R%tY~`!FCq2>U;pAm=!Y|1esx0Jw)auB<
z_oMTC<M%5UmQB1l_xR_fd7_79=7c@YN)b_uEK!kg6l7@pKJ!f7<!@JtEl<>L<=vT3
zXFB1y`@Q3zFKy%MS2+DTN+e{Vrro1D4+RIK^!_`qyl0jOIZd^*Q2(dVZ7H#mrTS8N
z)%$&OqPM%<d+GNsYwGEJPg7$g3#Y7{K0z#KerzD4YDVL|<RdSO3VZ%Dq-?zU#O&v@
zS?-6VTrwS`!vCy1aY^7-kkA#)MM3>OOhLtW>OH?5DZjY-Fju3}yjuG?D->7k6HkoR
z)SYfQPtk!daDo~~v*n4OSGGG$Tv<8mV$zkPGZLf@wWe%pSkBbI%ymx5#phqfJN36~
zgWN)8XPmBCC9y>><b8auCZEd6R4vs+?e7UG2kh+(Kd)t+*eofoe%Mq$sU^#1^~+R^
z07a%bpM<73+CJZ=Fkim-wv?Q8#ko8EQ*-_^{AB(V7;$Oa?I6nuHw2H`^r!~ZUZ~;t
zY{Z|-7xU_h-&Yf}i)B@9$M#(f2y0?Elz3>#98HS}>wCUmy!P(R&g7--(~dP+mGhX(
zW*Y_;>=22OKl8|Aj%4%O-ye_J8Lv&roROttRQbN^6Kn2`F!sz(9FM26G<ANL_&D$H
ze}-)f1X*)7{`~xH?THJ;v-+cJDt9$a(Vmx5mwDj$<i`F#(x>wleCxdT>1m*$<;k^D
zy9z#u`~8a6W>~Ob#Y*D|juy?o>h_s<9bV_ucaY;n)wPtK9e2_u-d?}VY}Kz@vyW~y
zDG}fL^q#NkwRi)MXOo4hQ&rq|nYeDfbBZT1q_X$?`L&Gp{4siyn;%9^2@S8^b!bKW
zjVp~?KH4o^qTakGQQ_maIecpT=c?V7wcW7VJd^FtF~jrctAswr+I^6eS+(JB@L!jE
zW^(7tE^aqF`*QBl(k`#`RTkTh3WX%>+B2E0qKUnDp2CT{6}77vN|SAiCmH{`VtC$o
z!iw2HuJ`|ESl#>Qe9*7iWqId=z2j4sMNi8vx_NTPTE2R_wSn{U?L=B0$otKmppvp{
z<HPR)?#s$TL+{AEe|oug<<ax}>FqoPb@elrO}{9AY3cU0w#m0|SC>cK+x6vo=GxiY
z-ha9KK;N`(nTEhtH-Co7&zrL>=d{anPx5XqdcR`(k01OoZ$B+~pRcQ%80IQh9lnEi
z<@T!T%8i#}bE@mkbZKq*)fuojXDQ3^EtAw(7@4=rJ?M2+N#0_pEZi?R@$*@Am+hBt
z>6L$4@%rAW(2jK<H_3?Y+SVDo<*0*k2vfbL<DrnGl*-Pzh8}{2x3-0!u3zf*K=hQ5
zwyb^dx1bdod7``fPN%J^onkesYS$LUCo(5Bo;G<zocL&YY{EzG%#<q(CtldkS>>(0
zV)46u{Eyv2`fk6DaCH{t2@5_EC6ntIbKv+n_oPyr0HIkb$``n1u?KlP5Bv&RLmxC{
z<qfTqEZ>x;Crcim+41{DuuN6SO8fTH^E5P{lq%>wJ~m;l=L@qVuS_|X-<ighwW?*o
zl#p%nOifO|;RrncRqhl2oNVzJt6Ohb#Jy+iEpC2%LMlJA)^_7{>7q~1V`ly8{v~iP
zy6kFI+{xwt8T!iRdixs&oJzQ6c4<P;`T(AtO`bvNoD4J0tq}Udt<GC)&YhcK{$BQP
zYxMQ*+xJ{=#k{!}`ZRd8to&VDmEAj&@0m%xsxq<Kwql*0$%3;!GDS~Rxdq%s9a&^t
zWJ~PWkGQ^)RJMw!eQGVAf1_yC?hn-~nvKgZyyJXkxMS(wv_EUM?pPW*Q_mpqUCjzr
zmS(nKBgaXG-#;#&WBu^Nv%EiD?f31jtpCq&HC(`5GKoKP<L$YA7fPG<t-CWjG*vox
z*ID5$z9O0qZF3p*H0EpbF7;Xbr!mX+L9@OEPas>}<!HY6!<TF3OI`2yST^tWt|R>G
z&+X5Zp7^`?&0hbIL$|_B&Agr;f6Cw@;4$+};i@ZtBu<$>k6GUlZM`8k`isRMS97y{
zPh)elyVosVv$9}$bI6~jgFy{4T}h1p8P>mEwO#08i+;Lrrpo)qF8#)vXC_Uyd>3T*
zGUlsn@zPhvB6xQ^a?sZHQjL3Tf5m|R`0?I&-L*0YZ!S(@+;RSa=F;nF3xmYZ_xq$R
zTaowm&E+c>3@)Ej_2TenE&4Y@z&&IB`=z`YpS_mWeP0?@|8jd-@wW*IN&bOb{qp{<
z4o~*o{*TX9?ZL{3!Uv!E*n_6bd*~tfa`iNh@SpBYCWqVaOr3JLva##M9rK5Bnng=Y
zBK|X6&kDZ3_V&FOxhw97Toqa2J#Uu9C%F|`!mrdPs&Fze-acQmZ|fr&OU4YR*RqD^
z7S4V!Pn~H}Fw1BAb0tqUe0kZbHsh+@wATWsLfU3!2HfIunl#U)P)%i$`i>LLOJvIX
zjJ;Op%BS6au9E+-F=Xq8Nh()9Zu{#b<Y!}VrM*2$Q)9*Ih}gtMrEl1;+$_yqp>N<8
z#4&+qa+7~9*Mu0s+dR`>Hg4a!WI~~zrpPl<AD5bt=W@wQjm)NAy1Y@xywzZ7h}F_1
z%NExE`C7)LyuJ8|2b;>P-Gzs*ef!{Dvu9WHRJq&X3m=MC&HZf_D&5;>_;qQk+RwEs
z*R2Ybe`LX0rRkih@I63Li?hRW#=Dwi-HPcP+}W$Q2Jrj5)4DFYb4%RSi|48s-ELQ1
z|EC{u?s?r_z1Ru+IGzQcec&SCa=`9}Yzy<M<41xOwqN+fJaNZ)ojTF$_Dj~J?7f;V
zylq}$8{0oQ(*&=qkZGq*39VD`Up8HZhrgtT!9KU}T9-jVy^!6^_3NH=J>;A4pP}tu
z%<Dzmcdx!<HTjek>AsvbtB9@M`TLvtx&s+EA2-hO_|n9#BjkT3TzTKd?e0OZCskeR
z-S$4Rbm<D`eXC#FoP7TClF=k4w-ENa$7hmVK6vV>mkH}_uuRA^e12|oeeR7TXACDk
ztLF<{GV5!Q=3@7v)OW8n6@GPvr4-u!E_*-kgWozEvwz0TdTW(`UawkyHdAK8nwAyn
zw(QRxet-I2wRYo9NAJLZ!;M+19{qOo<YwX*{h`8Vs8IG}uDrfT=ZsTpx82_7XUq9}
z>+%~aeRl7j{1E&Z@bBiGuOZjwWfVSLvqtw=$QMTil|rHFC$)WR|IWQUwW5=y$o3BZ
z%zM-0Zkl9I*5IqkT0iUD$~~{L)|fq;vm$zx&J}~LtGid(aY%$Gv$+>feDFHX{@ac(
z9!7_A`|Kaz>N9Yc5_)_3@YKwlxW(6Vc0b+rLhP-!?(r3|{%bU6S!ik)pO-zTGNIT~
zg=OQcM>UZNQ*Hmeb6-}zNOwo#B-`!mho?rhl!QIKF1I?jI^HYm)*bF0fg58>cWl`f
zyL{XBci-&-Di*Y?G`O|oiJG<V;?|=&g;TCip6BZRJoJ4>?t(5y_4N5%@7`bTzsz^w
zSJf?@o7;M~P3`WU6SL!v6GPqBODd-)zjgEAJZ9aK9DUJdxyQT42Zg_DRyEcaT;5*t
z;791H+!h74TOkczAwd&4j%Ut!&b<BCE6>GK0}{&j_<acpv+9sqUAtwfg~*bny2al_
zB_CNhO#bEKe?z7>IkL#?#+6P3=cTPWdcV(oc>6IrI{iZ6)w@DdcKtrL(s7o^#6B<P
zNmEh|CDlAqk22hvt}ySzn(he)o)oRwT3x-u)MV<@yAR!d$v92gChq)ae?do}wvbZs
zbK3<1A>qz50~O?|yJBs8b$E=smfm(x&6S;5bLY3;e}>qieeb>0d_6-|Euvnn+<25n
zc~?kFv+jJgVr8El%`q_tml-hpOqx(|nj`VF=RKu4(#|Vo6?O#2zUiv^o$neY>K*Xa
zTRV%9L9cq+pPp};^S4d-yYQius=e~WugkA}`>;Dx-B&*NOVGK69%n^Q%?$gpW&Ys>
zMpGG1PJFA7QpkCtUw!AL7Zz*Ilqg)PDT=n6XOi&Xu+@!6ebcv>i{0M5s6A!t_RLkB
zs!wBL3R04%EVWd>rO?1uA1la{w`=K4hP)Ob{!=qM<P1Ar?SALi@YTBZ-qp&xzh7N>
z_$X30`11M-sWYc4TCy-DXclY9E<SjnP}SpWGVj%$)j^97t7Mw)7P7j0?f#`p4;M>s
z`}Xb8(xo0e(Ua!;b*d!Dy9l4*kO)-%^x)XkwZ|gPh;vl-Jhz*<y!&vt<CWjRjdqjs
zR74hEX3%-6z<h(_<CEHxZ`Z1J#(s5a_|Nd^z+cU?i&&-D?=HBV9=I*{(c_xUk>#e-
zo)&L!tCu*e&#F0P#ad2=%Bf2|e@;0*QM+Aa>y?vi-uec2?Lw^!4*zGE@T59$|Kmwh
zRk<E)KRDIF^UOqp>gR&@x2ed=iXLgs3`qHrw!G9wz`vq+{qgrACBDD5nu=`koW$Ly
zB^1J-lF}*vC9H(??7He33jJS&*k1-7>r-Mga^HAwhxXU}uhXyFc7Ao<m3upW<)g{V
zuXHU<yXc=$|54<qriOO#D`CBr8394nDlGpQX1rQ??48AiDJSy{xVJr+RQqx6>NN%Z
zZ931kN2<k`cdCDQ>ayzBkv;kIc6^z>R=PdwV(#ux#wAOmeb@S%1)e<cxMdy3;nxcC
zrySld-EQM#Sg<|Vm|f=7B=;Sr!ZmifZ&mx)udjaoE4(n{olb~iUT_Fk+XkU6D=em(
z3r*sm$<0#JWLmemeT!7{Q~vq(mv$~@TAaB5)N;vHb{ieODSC$}9<n|1WAbD9m)|Zn
z>`7V1sC*{iyP<OV%#Tl_URQdR&j^%Mab%u;Ctc2{e%s_N#U4l7*ldovOQcF(-kKhK
z@aoFA9o#HJA~TxjJw8AG{<rIE5~rRv%nK<JNxgJfa*fgwO|PgdLmAJ&zIY@5e;2=<
z>uxrfn~=6>o^JZ)^+#>COj@d<d6Ion@MNWg`RZ~-CRWZXIOit@p3J*>!SC^zYYxjT
zO%^THJrI1<#^aG)^6~p`rKV-{^vG7#?4LJv;Y?<kStoWadK9whQCF9nQ_;S|Uo9s8
z{dcRcqB@Mtv8l%1D4UHnmEH8q?LRV6p5C4enx0PWJ7aeLk^9@Wm&=_wGvwFjpZ`L+
zZmqU^yR~NRlND1AItDcHZ)16sygzZqHQhXepU1Q>8M1pmjWX2}QS^QitE%ecuE03S
zy8g25zS3PQWKQTl54z)W!puZH@~GW<bFrzyC#I(OlnWaFI{)p;q|=KN9P=3rG)j6}
zb@i^u@6exejf?B{+Jo0-ByJJi6}FJ`h|3@0{@ZRk!8r`a)>U6<yB)?YbA5jOBk8Z4
zg`0A6&sQ&9Z+7>5RrMdaQx_w=UPh?@e6QlK&=#o`Ir--uHxGj_efDXU9Wes4=j7DY
zyf~30aq+~@e_x;Hox56E_t>>`<<u*8W*2r<r%G_?)hPztwa)abJ*gsiXY%}g>%;{+
zRZl&bU^OY*<3~x*HkIjeSKms%`29jxmibrFjq0E$zjMAOEuQ<iXwM@?@2&5bXkSog
zwVM<&g@a|u31y*>10OkWx%(JCWAm%!4qB(0cUgg<C-m#fYq$2lmF@h`z#nGqqAgvt
z;*##-6DI9qUe<rFen{f@lpI*Ny<k!ezn7fFI_2)ow`F-l(slQTm`Q&X4zoH~TC+lJ
z>F?gN+cj5BH41ys=b5u2g{64{`<-oDCp3H%*!irUch$mz?I$%HZc8<%FO#nN_w3Qu
z!0T7Szv^8K*jTkDqb;;(WntXrexsWzRhiMNa#VU)_$T-rs$6tpIt#z=e+Dm|8;fN&
zy#I0SzW?{I$1ihF{;ImF_c}rUKf|k)EmOBC9A4o$Pm5)yha|&9mgbd<gQ`zCe6{>l
zv_fIWG)}{=eP-3Yi@xVB+Z`=(HFW2Lsfr02st2r3C^<^V?t6aXQdyhP&5j1!*2ogk
zp5&^(RlWLV6Q&rXEIDYiiSggxGyfUZm8$eEGg8hyZ#?JPi(UJ^YxS+(w(XOgqs%N7
ze?x}f%X=PK^HdrxNbGZE-kiko`p;7*=RB6Boj0@QfBpU9@{+$L3wEsgx;=MGbnCTK
z)1!Zc&3&$={&VdL5et=h4_5f}GFBg6n>YKdkb7tN<TvGZTb{SpINRi$y8GnAr}cXO
z8N%%knf+(Dv})GtZ5LyY9ocFX>i668YwmQBrr>u%yp~f{+|nOp#L0eS7kRiMy!%ng
ze+Hr50yZxff2fl`GJz$>@`;VT)}#3WziO}4O^pfl-oI_`<2=y`i-Uq>mRB$pO<m^R
zq?Rmv=k<yY^&PtU6Q`XDxcQ>Ye*e~>AIDktY*8(~^YPdf*UM=ywyQRNiz?mKJ=K2a
zaVF!V75@7S5(FhDseHd=->o|7o@;{1+G0OtjqQorW{au<xjFabexDk4*ELz&F_8BY
z^DJXU=ZzC38W$+<{_XH)GQ+$rNA;C<lt>-pH+<>ZoGyF$hxXK&j~<tD1Zf5=nt#M<
ziK@|b&X!4*416jokDqYwQ`Mai@ic(H<9>ctdTDOnyG=(Pr01kRed?&2d;8h>9andj
zEM2`~+t!!A?u0Qc7k(J~&(E;$mzDR6casGScrRR!|Ka!U+P&1iKMXsbc^s?FoSV7A
zY3{Ne0Uu4eS1eb|h*_etRx?0l#*B&T+w23wcQ-}Mc0Fj-VYjEZY|0K##~p4`?lI41
zOz)0ewCjEG>W!jHS?k-K^^Alx?|b!#%v8U8;L&=U{sUKQ_2x<JHdknVefsMf)ss9b
zHyZyfY^YszB4<|l>b2iYg2ID>F07ikp<PQzV5*IDP?t*Q9G#Z$0c?!SG1nix@ZCN0
zV&tBK|B_$E`Zt}<(q-AK5mUjueHp9C8fnYx+lznu?a$d=x?6ldTU*o@hOd*Bsybem
zQE&F}iU|yvFlE7HMwQlrn!vK0&!U%CO}rq_x8u2Q#dFRL9zGjyE#78STKX%v>EY&W
zN&ectpKiNuySv_(mDRUuw!`{DJ(EjwzZ|OVeD=v_h7F_o#;ZBCyW^%z>{PbYZS41X
zwj$VcjmPR6-#hOxPOxEhw!HkSHP|#Sb@g1;`&ZWHu1bI5tm<=l*<3y0sZKk?MEYMo
zy42G#De>*8hf7%9`y9WWU%r+#|H#|0Q1?!^b<xb;-hH=|W}V-u-_K&5yLe?*SMQdF
zLwcfFo~0_Qo32a?xS^%Mu(fpwlcI~bOzV~z=bq1e=I}|H`RUyDxv@MK?Y}MH>S6OU
zY<##qN`A@hspe+?86+L^@8>N%&>nVmSF5PbCCv=)c@s+g7OZs8RgJsIrBX51tZZBV
zZqrlRRTJmi@a_w-u^02q+1q`4&64^FwF<wP8^iX_7Oh{mMW;??MN5OKibJ0lTg{v^
zKd$z-$HpC1c53&q4dJPEw2yl7{d$t~&H43OO1GoSmR%3{TN@PiIHp!*p{K%vnKM3H
zN=!ChWMOsAqp>rgc*kw)OMc4Ie1Db~p9tCeIxlxayKrcz*Va$oy~})9R(P=Z@v%(2
zG4I#L8y~zY7G2@*E4trT8T8`e)Q@Z3Y6?&P%TV1VWgwS1YZhyjXQ5W`iKqkq$!D|V
zZ!pVlnRSHmmdh!wZHL(dp1%$*c<mj@Eh!q(xAR!-3h&aK=%%2&-%fqE3sX6qJ2F@B
zO<><!qki{h5PyE*$JMUu#l<$QF>jdh=;@Lt^RrgQ9KG#T&2iv(TuFaP=v9UAy}7xj
zTW&<~u8@k5^PeJlu!*N!u1a-7|EAwAjFoA#odrr`HRr~k_fmVA?qq&F=KI{)>2IuD
z+F#v!8uZY^OJkN-LvWu~0)zUy<ZUJOe6x}s1<YWd$(%n|zWlh{CHr@Jh9_&XuUx+z
ze>v~Suc8~*?#|t+al<srq3Ut`%fhM${~7YKRI1%vRAw;v7nL0PsyxlXWk%rn9qw0V
zJ=qiAcec_j<y_6q(4Jl4){Y)uW0lqRE#0Dd;IOOWttO$`73?QxoZl9rt|H67IHYk#
zx%kBMPp?giIeJsphfzMN`^N55?Nx8P<DPE2d?l#;px)(u4n;GstY|fPz;cj_#r=Fj
ziS3h1W|wQvPc_@xXBN|HlCHZ{{i5A0{#}fdPbze3s@t#sozd-<b8N?n4^R8&WpU3=
zh<Nay!S?%7b-yXnd%xOd3r6gI=-d^^vQ6laMMF}>p{w^#hTo1TogB%1-%dLBeeVkk
z-G~&fl>0Z=t=@cBBX=s(yuG=)IxCN?k}B!Xf6Z=C?RjnOAtfEzJI@bKvpsI5{pO9Z
zoT=-t!i#tBX1@yUFO7YnwL*IPx`m~sqI~VF2al?>PDu+rneF+cc7ckTfx^tI4wrVX
zU#qrx<J1%*=AMwV$IIUQ-g@a}(f*va=*;bR6T+gi1g2-*VqsVw-1y*|ZO3Knbqrpd
zf=2c8?t5i(E&l#Qs#l}x{vXG({|s*PHs;z*d%H^a$5U?3(2J{fHG6bdXL@jCUsDoz
zz!B8*sexsJk6iq5I}Nev+}Rv#c{}gu?>HcOZJ+cHCGQnJ?{0-JcvY*xUHYFvd|$`j
z?JqKZrA@tO)#2>6sq4w5o6EK<EOiVF@Z=X+alqry5+MiS<we<(7_HyVmp`>AzBS+X
zVBeb;A4Cprsu$^NySH8LYT~x_(`@$!JU)9TZe1qhu_+ABF;U!sK}=1Z%D<cxgi4nG
zTb{FuWyXJor}I}VdRND{x6e#@&tA=mSNb1GfBCxgZOU8UYsdcG)#q5WctPTYcCiIZ
zmS3q@qN*vV;u#=+xi{mx0rRaR;c^KpeodaA6A)I<AG@pP=<>h!M9-al{3o&e_BZZr
z_iz6-yj{loXlwuByP+|0%a-MZaL;dVSZcxHE~p^js4ft)^vNPN?;VZo6N9(!jecXt
z?4HQ&eaGf(@qdQp)|+;o-5y@#_Hu7f{PdWYg{Q*1vx_G$eEy-8KYY@0qm=4~&-Ydz
zwb*+^CuNE@<N2ihF;D)y`YpR^Mo-qlE8)(0_v5aL*Dtx-y7}_1a@P5)&d0P)dt4Cx
zDmU-qmhD`xm=A?a+7OZ9zQgf5qspxXd&)MwE#W<O=h*yp|4bd4AHH~(&%H@d;$PKu
zo2seTcK^M2D|+%>=eQR;b&sywvOglO@WTv`a3K#*{~VVN+1)RbGr!Gfj*<KDiu3*z
z;cZ$UHrzfwrA<eFsb1V&jh(R?S=zC|Q({E6d4&iq*=-S_`T5<)d!na&GDWrpop`&n
zJ9)+HPq&tDnyt{hL8Nxu?7By9ORaBDTy$LQ%Eg*PSGGM^u~fKI&VhZ;-=i`XPOXm*
zn@_O(HA%zz_RdGOf<4VoWBq=qzAk%u>u%)oThjB3wLWfJ|79J2=-0y~eI_S^^_o06
z1tv8yGI6E*?~~sX(jT%SP<sCK^Hr`N`MG@UGaJhL4YId~lt*Q?1qK{+p0X-eZ`%S(
z^{&YZOFAZ`xK!BJ#;j#jnBd4hz2x84GtYX%Z8koQUbZdxr0RA*ktGY2#Fh7J9<Oh{
zksNg|Vy@ze^9zn<=E)Y_?l-=)U$7t`N;W8Dj?4-N%RkHf8|CjiJ7%14UYt`EvfRo;
zS+36XQ}*<;p5G^*5Lq(8($nI7$H882&WVkp9gA73eLk&ud)>@Y^}EmNDa$9Sv&*a0
z?|-{8C}Xa*nbpRCohh~HYxC+f771~(F*SNj;5cW$|M=^*L7`41JzPi5EY92dY1NTw
zm!@5q?o@pGxPGVlw~3!SEv}_i%nZHhdA$93^t30eb=}Qc)1wuGRs`*EWa((KsY$87
zAK-iQh_hR>`ibVR>UKBQ>|y-9Z0o%>GVS7wkrrx&znPo=Zd`NcprLP;mP+fvL+>M;
zUl?&oJ9l<+`LL{h9vyviOZSQp@tjp`Eoa@YG<=%o#i1JH!lBaSD0n|`@9xdXt3G{R
zDHof)q(^GO{@_~g+ESl4vx5&loe}(Dp?1gtKMod0Bb#KYuir(J<)+>I&){|W&1L~>
zz1W)C)u+_2^@ptbweae7iOpek+WAq@Wv8>W0^%PD+`jbJjpy|GQ$Yr{X3{^E{p;kn
zSDy~LILGi=mZo6He}<CB*XPFlXLzvv%Uu6!b=$9&ZZ8coj=X$t-jyw%buCT?mtMav
z6mp2?q2rcTs|A*IA^oqdta}=B*uRV0#k{Ia>N349seXNG)hDmU6jl};R*m8yu1cMm
zD*qYk{+R~1HpE>xdH#r~+)cH{i5J}eGxSZmo4WYU)@`#rZg=kpx&5C(C^YV<(3GzB
zi3elYw|rLeoWP%?F17JPd8g6OgSIu9p`SNDP`eruTM%fLm*pBQ`o-TYMpVmY!V;ZC
zhZ84{ADpc0w_szYf4PM9ldng$gqwc(8-}qRy(E2f|Elc!WnTk=Zrl&vD$4sQ=x$-Q
z3)?A<{xeLSvOAUwTgCLKy74ZzU&=MdD8252Y-@I@+vd_6`6t(H(z+e&`J+H;`Ko13
zx1twRFLfwljbv$Vuz6{sY0~92ajWOiyTQzBt<3zMFS=`8)b-iZFY~tMmun~gTshaU
zVA)kc87~G23E}To_>RurJe8UMOW%}m-ili?E6<9q)eNf8)GT$F#H41g-+8?MI`i`=
ze<8P90m-M9Ws20kD_u7~Nn}c!(}lvLPV70%{~7APTs&Eo+#e~kf6gk8#XP0@dRsfc
zMFgZim@u_zjmXKz57`+WO#D^%>+6-ji&iT<fBR*{Zl5eoN9oDAk>$MWcHORByfyai
z`2}XXs@j@+?Ksu%Oyc?Z_OtMu+)56GvcpQ7wrj64TF$nU^Q-3{%UipGug6<Or)-PM
zY1NMPyS2_sOn~F$(&&cfDJQ-DGq}uTRG(JJ-_X0I?RC4y>i!Q>-Sr>ehdV#KFQC(T
zqmcE^kD~Q9u@S!*wzaRym(^Ie^UL;9>wvFTE5f#m9QA&vCJ?1iRp2haQN`t(al?Ox
zkoSwGzqu1-aFKVDn?%j$uL~^Be|g^ik-I1{aN@)HC$5U!-)+7&R%uPy>eajDwk^)x
z_UPsQqneV>eLg<&YB|9fRLSJRVPP-zs>behOT77ihRb)0s;WL6E%Gz`;4&@j*J7Ra
zX$2B>l5#)4E=%{Tnz&WFOzijOh*dh_R<GJJr{{XVof>jsM?3#&-c=$~HMP_y)u@yo
zQF-*Af&VPmF+C1-%kvNS2ItJuR>{!&IscP2|E(q8=Bo0&)W4}c<tl5gznE0?yJ@#Q
z)l?>^3R?==wmepdb<SzXJhk2C=a1wS%l3$UT`<|@<tpVX@9fgEq87Hj-Cq54(p(;;
zxn0|BS1>GCsou%JIi<<^$h=>zUya;8rCa<fUcbstsF8KmH!Y8RxwxzQ)7NBN+3+<w
z%iZis+g+zFrC`oeK2x65EqKoG+Ov`K#sssE&%<Y?gf*NjFj;2y`oxdWwJ-VJ+!ebR
zpR=UsZrJS>6HVFF<3VD}?&MEm<}+KeQZu;o$y5*T?#V2#XV!!Z<Z8Ly{(FhNy+iqB
z7$0|jeyz02B@x&Amd7ux-FA8P-E!%+{|vh~J106{z4H8H_lnI|th{1Y<mzoYHM>%V
z>&okz+2JLs&Yw=0Nc=m@qZ;X%cwygs-;MjXM(sFzZd;S+j230)oX2M_&egbUZZ`MU
z>!MB*rG?qClG}}zY?|H_w9KJYD=_t{rh21V*fUNQ=4a7iZ_>?PO;SAOZSLnj&uqz_
zsC$=qJU$ty&o(=_@nO*R>Z6Y)T`t(YXFIpwvYl~Dliz6aUcR!eb<Y+Sfo-=7w|V|c
zU{IK+ysayH%EYaSQNbt6)vsF~{25wxC(>=%(+<P+T5Eq#dR|xcbC;R#^5VD^7j{?c
z$LMU?dT>eHD#=G;6ZWa!Ilpe>Sr(xwD;Uyw)~t;?cD1>&&3e-Qf*-|qYBk?J7usTY
zxKxAjenMsL{Ixoo+x|Kq{B=tA_|BElwa>+tYz=t8(kkNlFJY1s<BU(HFDp(q3+{+%
ze6WA1-XnpVr!~tD$@4tEFM473Dba|skD<#RY@b`2cx0{jlLMT#i`W=?dLF!3y?gb_
zbhCGkSMEPPe}B^Bdn-LFh17UTfA5~Vw)9ro!Lqv^4?Ukuh(6FH$}D6!!TNaS)maCh
z=;<D6nitK}|Llv@{fE_y8NM*9EeLm7Hvf;o;;0D|T3QzhiL7%e61>lFPWfHwnmcw+
zywXb#$St>%SabGBa>jvBmi4w9Kd<J$*w=lbRKDzT?%l(C%<4VOzB)hM)qAi$ezB)p
z#bo_Pl_i`Cj5plW-D)gTrO#DY9Lay&rqFN0^Q8Ex=klE#EjC#JpO2YVr!(7qO;}&=
zDew0?H}8(bKLvaLty6tgOq;p+?caj|tD2lXHO;EC*-&_*(bBN_g2nd3NhVz$B^%Eb
zh%f)O`^L|pJozfgFWhODA01(S+my97->xw?_qx}ry=(W+z8k$zbjr40`d)`)J@Qu9
zOnh<bl8{%>6v;p?)|iBj*85*G6nBa3IQfBD@=yD@ja7FU4)sMokL~{D^($dR347Az
z%PXgZzR!BH?dhc*oLV9O8BXVK|1^<d;#0Sp?4_<Yg$GxL^Vt5|^LqXt$BOSl49BNW
z`_EvtbIRK#I~Hy0jBs%<n0%w-r^4|!rm}O-XDobXp?>|#(zcr+uaDc?T#8<6ws+${
zhGWn7UU_#qdJF%jCF{1b{!^Nn(!lBDIAx}VrusXR+RdAuD4hCn!QAke{qEE9=R&8g
zE{xUKmb<y+?W*k7T2seAp~=T?zufaqg!@!NpTj%@$H`C5E%BHX-LP49zm#rjpOI{Q
zy%)pATBj@9%a?7-JH-~cEp*o{$(0Tbr&~mt91MbI%+X+C+go^GYj5_ie-;a_cWBRL
zV04`8mAxvbRJ$m$tk|60>}j&d_wfG=98Os=Hr`peo7|sF@#?VIEFn~C&hyaq_;>ML
zw)U|v<djt=d{>q{Z&S2zx7?fi*N**X5O;d^ZR3?AW=FF`e$=vA{AW-NO7riTQ#Y&Y
z%Z!5|RyCFrYTXR%N?eyrm*AGT{qwQUxwW;fAz#dw=)CV+oNIe@M>+RZjm24a6PIa4
zu86do?6=ar+xV!_)>TX$LY`OkJ3h8<3AR1JqQ`eQe`)*<_QT)97aUn5vh8M`^L5Gf
z_H&*7seb=HYv-HW@_sM3)jc&8<qh3gx>|qXvTKh%wOwl9e6(be4<|djRblqW>O$^f
z+byw=PfT9&o9o3L;a`rji<wJ}=Hzd%Um8@mur%Y2+0yJSrwT(%ZDvMw@vqs$R<Y8s
zDdqct^T&1uJS_5x%A71=)2L+^ac@`4C973yH(s3=@iLTa*50SPy$kbZuJYQmWvbAM
zAQcT(pOgI-Z&V(~vfUKlb?w1|<CUH#?0?OF_GO>TuFQr~g^J6kmrwupz3k?WDD%8)
zi*+yAOu4<zGh^vfl{w4Y{~VgcqcKUL-|~{4!nV_8DxD|(Gh{xEj?p<5tM|rz-+i6S
zVe`&=$8XZw_WVaT?`??yoojW9i6tik9&K96e8SN~K0}VL`H@Xs`Ob`%ircam%)dSL
zsXMl6Y0Hz!{P~CFqt<<!-rHVintt2-`s)jy=e33&HF@}5`n&e5)820u2qpLUxmjnO
zPS^0Z71wN+<=3C9pBYv#@kmjPuh}n0=Y4Osd)qQ)tz0cwTzWk#dyc2K**SOalJ8y&
zMhlp_QX2SH-tjv6ZV88DXk5#8|7FHs7X65}U87=>{H$KFD=wnE?Ea0r(f78^+mJgu
zKIrAkxHa3CO>27dR@t;=sb|)viT|_|)qnb#pJbjCI-&FEwP$xYw)Ql!6;7Vu-gt~-
z#ezp}$sJ!7R|x*Rbg924G)iVg<tqU(f#uB$=j@ng!!J<4XvC5%SJD}!DZ(Due7<hK
z{#?QGkK)NK6ZkG4pYeThRowD*`|dk*y+3v#W@d?2|71_^?3C{TlJ?5q=Nj&PxHhL@
zdO-U$<B4BZ?C=+x^fF=@qs*7D%h$TS-ekB{cWQo~)GO=1#Z&p!Ixem-n_cCcXsWg_
z?nd;g$^{o#<4<S>wyI>Um?QN_(!}k@mEwM>T?H@rv(!}5@4ws{d2i-DjV!$>;oNag
z{T>%Bb$r#8VixVp>1V+@`GKE%;iHnS{|s%h(hI(G|LtqeKWuwzU5(S4*C$qdy8rQL
zmGZCDUtP(o#j<|;zB`_F)BJVx{I;YUXOqI@mi8yDy#43tmX?oN3VhCwtzMXk2b`EY
zdn)sSt2@4k9<R7Mt?kO%iO<8fo?fc0IAQtitfiM0T(UfGy4+x*QMF9@?_(VqQ^n)f
zG@iCO8vbtnj?#7Oxg<7x&0aox)!&NS(dAXS-RmAMsJV14$S|l~c3GiilVxh$1H&6l
zN~Q7JA2Vh0C~msTR%5F<CG6aV_KSB#WeZ;O`{&krE_kVX?AKkfuWJ{&rkYC4QvaD1
zE}}l+U{HJJkGYxRLjBv{JkhZ=|5F%b&OR&jxA!`c;8^~tOPZFfWcb{)G|@${|BG?I
zRkE~9*7lp3DOn~wym6&Je=HAKvZg^*e#Vj+Dl(tg4>r|1n1t?mGRb4THS7E4S;x8_
z)ta<!Tlauz0+WaOWA<&Po9?aJAS!$=ro7?%+SKqpP2P>33IZN7jq>dF>-=k8sZ5NE
zjYxidYR_>F4`x&Oe+C~5nA$F^P&rg*!&KmMy!hM4m5FmEI5>W`UzXugW_0~@xQn9U
z$;2I8|JDdvcAPvpS5&6&XSr1PQWrIiTj5h#OSM;A*{z>>C!gO!<aMsVrHW5{7WK=2
zKmRtLjrB-JsWIEvmCsrxl+;>YpZ|X4yY=rx-pU?(y6BmvdD}zLEmJ~QvHWLf-8QQ<
zBs|8WXW6uw+hZDy9?fu=v~bbGH&22}*bYkbRle-kmzs7@GR`L9L8;8~wUyu9uHO01
zyVrAvhQ@lI%Q3I#Y`-x(I9u<^6qbHh2IdnB`OHGuxl4G`bPE6Y%LLE3AzCWyd8v4N
zo2KuXy!k&mk{?xeeV_bA_RH7QU#`DmUS4;TSsHk^W4Z6;qEOan%RUC&C|W+v+WhFQ
zlBR-bJt=cT);%pQR^qkdmHsxhj92cA)}8$|IVQ5vFB4WOh^{=)^3A}}cCrD(d^wgE
zMJ|h7dV9k3a%X#36+3*lTV3*PX4=H+%{faQSw8p%i3KR}2^9(o)!FYlr&=!(#L^Zx
zZ+F_#{&ug>*yJ0RF5LemTzAK7{hD1j=I=K7lqC~&>eA`H29F6F1VZ?QZyXf9@0;}F
zm_^fJ)nk+YS;*V`2w4B&%ir}`!VJN;=KaoIp{aB2@-?#n&!{ZR2ZBF)o~3_(yCU_G
zzy_<7EB--cKLh4}c_gth>S*PX#XACaU5zzZndtd#aZ8``4oBPN%+Gy<-Z>p|FklZp
z)wFYSXwNifhrbqn=hB+iJ=}V4<sa+!>{*$o|MV(9zOf-#z3B);{kO+^rcJyb-SOLR
z`LQXtUvl?d*eaC2_WXYa?l)&o@3>huk?+!l-N9#X23al<erJ4V4uj3##g0>jgjZ|v
zD_Kg-dhyL+hv?@`<{o#ev+fpX1m}y0h6Ya&kPduryGTXycjH<;#Yv`&;aYqBquNfr
z@|>{h>r(L`^J)KHe!VsC&8k(>ZTD+dYGxQo@#<MtWryEUm=`Q5WRtm_Pd>}naNop-
zj^~d&Yiz6JD!lRTZJsyly0<P<^`>o_IBVJLAftdSt&GYWJ{zRCZ7h`5H0*yTc=@l%
z$+U343%sujUnPBb)Dpb%fTF?{+XFsYHqSfRkF{4HQDCzF)%*9g$K#+~v*!l)uUNn2
z?%ZouUc2XR)q8vP`3nuDty2YN9XxXUQNRRCmq}s@OU^f>*y$!`+EiRz+8?}cecqJo
zt%W~zBSfEGudBOyaNnA@XWw3${rB?n*pS<9>s~$J@V?r+RAlRw3!WOa9*R%A`kxA|
zILVSetI$vAj>YOJ9A~PR^6}mIS+eeEV!iAa$Eh3kNgVsDJ@;?wFT0oeAtI~a`@c24
zYjXFhzvlHjOFVpyuG(}jZCE2Cd{mXun5iU3{+gcl;dMuX_^#P*GP_eL^W1LH>OaX3
zIPydn7bzECm*n05_+iYSyL07!HN5QD5q4{3_}(`SJEmOSwCMAfm2N>VbLtH@u}FO9
zNf%tV|B#*KgFPvoveoA532x@!PMDslcz@t>wBO!Y;&STKcZlrYlwa-kz<JsCov(Au
zzjH+2@A3Sy>ZIKx83!BBQ=KvV`(_pTF&CY^<tn$|ukNeLe2GHs5^lDs%uD@aSLsjV
zx)t@y>dQN8p$S)`OLdpD3hd|zIAE#d#`s=%Dce!?e|JvbJbBe!@pV|7{9liGPkF*8
zwg-PX8nJI?Y2E429l2R63;l}<j3j~^XSGb4%ClsJ;=y-@D$FXf#qYcJiHEeFa=-U{
z;+*Pa)A|SEFONj~&Us+2AZ`Eg^SasFuS^NOBzvSaR#$xbE$^g-;!<h16Sfz{W|wMk
zae0<ZIwjJou%KmyC#&J1<Vk5$oYlPgmFGnqKfm7I>(w9QhuJ%>&I@BcUbOT40-IG#
z``6rN&$|0cpL1_X-<d5-mutL?UE}4t@rXsMcSfp<W6M;=BOEdb96X9YlD=<xlhSx+
z+s~u!$Mr<tUOMG@K=9L(((2E@LaXooX9$lLy6m(%b#ie)-W0v5tBn2gwyb@2Dr)(X
zR95d&CoW-!=1D0QUz*RAmOre&JiB{}(Y@%5q$18-oAVCFkAH>gy?)duom8yT*k`!3
z^7mD%JMjUt7k$w)HH+O@z*)97lxyWK8)wCvEFXB5+I{j=VR93`WqLKIj%`bu!CH$O
zc?)>{e9D@AeE-^O8&2^UBs8AvJAC`&(naR};aOAOhORGo>z%E)X2-IlOZC0_CoNSx
z<SA40@1F0XYt|eAZX36;|K6YVy#3eyqwhGUDReDgHviUM*|)E<PT$^kuwdK$*-I1o
zY-BY&R8yRyXZ~Ac!^2^D{q!F3HeZ_+D;9jcI!EK%x|6ksZ>gRZs_lOowfow=zri|E
z*|`r{nY;6*wyag@VVY!bGxK1_v69-qNB1tgxLVo%k>URNesv=2PK7Z{l~BHte&brN
zu0v*b^_4Actr}cja}`~7tYDuwalUQ)m+zNP?C6fwU~<~QUc2S@m-DT>MU%Yu9QUj2
z()5kJXtM4jN8yEr*Ij%PEE82GS<ejrE^k$SIRByGi{6t7=7&>$PI5f4a%O;n1DnmU
zdZ~u(wsF4`j6Fkynk*ExG^YwatZhGN`=4Ro5}WehOP3}Ex|ay~Wo6_poRaXyX4|@D
ze+5eTw#}P3Z*9I*$?Q$ty>DNIhXu_NOA`Cx*&tKEG_%pz_^s5Y8tsG1t=5_1y6SgN
z{w)`E|1rD!*z0eq7RQ6H?P_+re8%V!SHLZ&N_Ga$;E4<pl4tn;%#A#m8#_a0598TT
zCgpYi8TzsUJ)`|}w=9^-_?9zIMBzx!V`cl_>8=MRt$vxtVs6~L{iRdP@tcZVm6xO*
z*1a}<AG+%PmbW|sQ>SdXwAwO6mwo$+8INZ^Wte2ipl+z!WB9FC+Dv+x@&1jsCbG}=
zJh5`^)+@WK&n-H?BO^3*cQJEQyU1h##*ham{@m?6XPvS8w8PnnlWH~Ru06Kw(%Gdi
znqFScom{qBoa?sg(pO(rYV#{D=!kOiFzAtx@lRG(e(+*W(zAQEM&CET+?uP~CI04#
zMBxKX=lDHGo2%@dD;KZalP47wE}1m1bE@v`UQ0!fI5vw3K?^({_dS@)FD|%!s!y?5
z;c034=`RD<+?_1gSM%b2Ua8daT{0`vX3hO2J=bVT?6NhE&MTDvGc1`YkkCJqJ<xKN
zD&OH<#SNaz%C1j)!=RA+^SZ>pZB+|*{bzWz_THuU<!U)oOx>n-1-!a+;4z11bOOuD
zp!H|`pD{4Mom&{+QPDCvrsFFE<NX7_i*D`Se<+6UmImvQ_W3%=rl)wfZhf;g@9O<L
z>A2Qc^9l>s20oj7VwKMhe^r^L`WGhgBKuly>(nkxvP_<TCO?+x(a$aQCHod%&Z!f*
z@t=YBhS(|gxZ=#*SuHB3`9wUwOlf7BE<7QKGw`j#%(tuN{0Y!;yD557=Ei3EBX7Nu
zcceM7-J4w}8vbj2+RfjhyXyldZ(DmM#M66?sjttWkgiUb)*UVX;{V=PQ$Dsfo^Nqn
z+6m)t4#lsQ|6TGfgY!p!!l_hC<};VJ_8u&Hb#&6a$6ME~zHJ?K?Q!Nh-lYmVTBlC=
zXe7$sp4|BSv71ED!81Y=pRQK<ec8GC+qAHbhgFZ2O@HoqnSZ)yc249g*|omK9EEub
zRhhMmJZw6fjG6!a@=UB!n)rN1!#?GJFTXCISR8la$erH2+uJskabGjf-?}FFvgVU*
zzb=G@`aNl~T)pcgOVL})3wPYR`_!UbYGjHI-oG{DXqJ=$!_$Y`ANp(aJ>0lIFg`GK
zy;IX9OM}aoIi9hbtTIsfX}kRU**(XkYn9*ie@&6k3Rc@<JFhOiYQ_D~{<mqN+rGC-
zh|Rt9H2z_XB7@q_Eh@jS@XYg+Neq0)!ZCqw-F~*Sx~HvpwnUtJGWXBs%ct+Hyf3iE
z?6cdMCkEgCGt6?2%};%*5_V1Tcu?b?>6V4wSwc*83X?n}W*(UvnUm^q`QI(W$@7<5
z?Ag%ubV3i)o&OAvwr9S(c68}PpSQKHM>QF1=FDVudFUx2+n#yOgxlG8vgxh1o=^9s
zuTF0-_?=z%Nnlrv*}1ai@1BNTT=Syu-0tAqmfPl=^LDK|y;b**-RxHNb<h7ZT)QaJ
z8nmZjo4^IHDbod}i>(M3T=aBXdCC+u4vAyZh5xQSbyfMK{Hp9w#h0I-L=L_<dT!at
z(%X7dcBZXdw<PVY!}f*8UTaSA;9@9q>Jn!-7W{4UnSZKV=PuPLD7^5b=1*qrW}6kE
zw}XB>33po?oBV3Ix7p?CYp3shnH&Br_o~e*P2C=;>jAI28uWKfU8|zfqOwGdf#J-#
z)qj*b?%a-9UTE-KYU3pKy<zVxSa-M?FWY$G$-j`fd;dm#{m-!GD{r3P;=I^%x4->T
z4Z1OZyNSlD6(ZX$R~~86C@OUB@>*7*roQK9#hcCD4IBOBC2y>^SL_mR|DbU|G5J7#
zUEEgIP4Ro*+?skd;Kn-Dtqd|ND&x;N_nA4}(PKWdgX{cmwUl$|p^HBnNpj8jZE1gN
zalXxC`=}jJx1$e>b=EBi-S|T9r>w%A<7-dti2mO8aI?sNhPK>Q$5)9*7DaM7UH#E8
zRc-3By+tpAI=LcRI_D<0#hv7NRHiuP)76dk%lGDgR7tB~=WS25f7-Zx{o7c}7vdjn
zZpNnEj}Bfw{omZXW}&$U?c2;Jm!#e<4*fSXF#nFw7SlKVp)L(xj*@d@*X;OX9y}xE
zP-XM{TRW%A{F`&<miw)2p2Z7T!hNrDK3Nd9A^6QC4>h9#!_V?|CNEQ%4V&j%Enqp7
zdi~bMR>uO(EY3+wJ>NfO{&VB~$F&a|y&l$m&0Bps;P$PpT23B36I!+uvI;Cp^Ev-g
z_}{FvXZLg&Ecy99ZpU<Guim}(R!u8x8XR&nf<zgeYHzB{P~UTY`&X-3wuTeJ?TsBA
z3nVwNeH9Gg3i{N&%E#y9JKN<(Hjn@EEo^g||D*5u{f{ffL;~-Np7&oZw_tZ*X?ptQ
z(AAeyidHZ%A3Zd&xu?*R{Xnba74uVm$-T!mK6Sb6_GI4a>DQmfM7&~kd;GfX`8{d7
z!yZSEXBY*imoC%R?EWqmRjIeHRWVh6+vR?<8$m&fV!U}zC=_0w71pQlNr9z)!ii&b
zQyg~fu<TM0DOLaW=he?z?ON56g|?9|z6WjZy`Ob6^4653eeD^FUqga(Z&m7DQrork
z%R;}{tku4**=MHIYfYOrBmL?V$L)`%U3#iEIdP}Tb2~X_(Y4RHwb&-fo^Q)ub^G<J
zxX!6sr#H-&ZhNy?@6Nq=t+1e?MLlPoo}Uv`c^c`ra!&cL`H9TZ87WeGuG_a-Jd|hK
zJm>rdJNdm~(_`;iuY2(~_1pWxy;d*1rvGR^>DAQU@-Jlh7Di@?x9il8?la#%uUsjh
zhEYP^uxnpbfyPy%?o%_utW|ei_7nN+yHakJS3~$#l}d)n?+GsDKQ4tl3c6kUx>e`S
z2mjYW-5K4P)0cbQSh49+$d2_19z~8%X7rr=_tAL8^4+_+m&Yx&`QZO}){%+Ms~zsh
z=O5eec5l<Bo4LCZrUg0AiFWXOWaMctUo+S0;hngbiQ1Ysifmr7KJ1>uVE8ur%J%59
z`=Yns%fGVHLGIXVu_=p>OR#r-{<ptY`rtqF!dvyFTh=>q{t7X#;S-NBx)NRf`s${4
zPoJGH-x(A$KVECOz{`l*h*nM8<vY%GbpKjiYx6p<$LyKeS=IQaw@X+y$i-^^`n&V{
z+BXZ{Rp0e{zBspTQKr@;sYDfjQ4te{KtF~j9J0S(FEsyJ_sP6xi}vz~mpqTpx2d`s
z-&w;e%*Ly}jn~$y@Y>n^+gHrEy?wQ5>edB|wrd|=dg1Zh{|pKqnjV@SohkzS$?F=~
zkB4nOqrCU&g{H#&z2V9~oFcBrZu{}|&x?OA!!O6}y}EU)XyN|7dD}y|PIZ;d4(8aQ
zqOw5IOJT(p?zVtQlXi4A&g)D#CBCKETYcjF*P%0i<n1)H(7JN+*6sJH@3?Na-MdxJ
z<+W>e(S*xqHgPcvYA__IcYHbJE-ApaXjfmx)MY9Sb?<J^kI9d^9dk&2Q~zAaX*(2U
zE?qqEon?ai&lbi<{NKXsRa{HnD=Dv#{CqUCG$+j9`1`QQZ&`KgKd<ugU=i|EP)kex
z%Rl+swJz1j(}Hupp5?o^W-i~ImBQEa8sGTsd~3CL<-WDPG13uTQEd}!v&0ny9(yZ%
z(-dlMb&<)c)6V6+R6FmH0pouL-A1`TFSm2;+p(?S;0LR|(nEQhYYp?(b>H8Ac)Rr0
zaNB*au61!QwYBuMetU4rE|ryA5BYDIz%R7$aSZ>X^=)}NOYAc`rPO|}vHSS-XvsU?
z<J0OMt@EE7=Jj@Za_qf1h8|n(79U;y@`h)oR;1lk<<I^FoWJ>wxJI|cPm#)H2n*gc
zsa|>3=I$rDb1m#@RyEan|K-hH{rIYR-JGb_?vf6+nN#HtUR%e*Fh%{Rwh^bwlo{`o
zW9uLKzuNnB&LZwdCiB@?eka#dU6t?qqb&QnbJ6Ag_{&+(g2Sc%o!oJ!%syb&j$86u
z4W61Bi*f{*oNF!qGDYFUB(ECx<lC{EzPUaUR6W}%Ve|1y)qe)Vn4)S=#n~P?j1PDG
z3iz!TnHR3HY6bhHfWXQ%0yE@i86_kY-*$DEdDWB__r5h>_OzSyn<dJ8JC9$UD__bs
zX?1G+yXaa|{XNTW=jLiyOm~_YoEe<5!b43=bVg52mYj6aq@5EN2S2Dx{va3f$<Fvl
zoc-_4*v(h$+%LQjs&-ynJ-yg8XZ@-bVzbnD*u<|1eDaLTA-BbG#^iU(+olxu_XQ~g
zPLGsndddF$`mI%y{=|e8tva&dNpVwMY4%%d{lqWdrro=`?bPnICv&IFnz})*^rHNq
z0|hhR${uleJFoJnMepAWp50ISoF$mAAHO&0_FdCEPv;(#`F^`T_l5nc-;<85onE)=
zYShYC8S`>|g1M%!InC45^l+54@!x&>eqZ&==gbezq<cED{(AA$>Gtgt*ZbD^&pePS
z_uSO!w(GNn9I?Evh07RJHMvVf9NtvO-?wbCaoaWf+|BpAe>Qe)J$NwpYC)gFzlz+r
zbGmPhUfFYdM;*2G?os8cyk|4hqhZn%W^GTEd9ghXMr`{Gwr^l;$vPr_UnSXp!MU$r
z+(o)xCSU2i_hH4~1GBPTS>2K=dmfj&bjQ>c!mlJJgdX`_yN~mXKyvi?swl@?20h_<
z<t%?S&H4L0J@r~Pe6a4%iZi>b>#FUYY4Xbb&vfoA&o6T{=Q(*YF-UwU-Mi?n$ys5w
zro##5f&E&AvfDO^tXov|eEOy3W;Yi_uFP8+lP&b7?G)n#HU0vg6Ms!pwmi7bCRn2K
zz9@HB<)!YHj3<^aFU_+Rf7zQK@@sCw#^9CB8OxpGc3v@RJh^OArHk;)sT@oWA168c
z2_B59ev%$@WM0Ysp7=FlOCv<{bzkqe<|VfFOHSa$%VH}7rYz)T{3$T8QQpEo!^r0K
zI#$8eI_X+fMaP$0JquEqv>;8AdF9HDfd@G&J+x$;_8P4_C1H2}bL~f|Bjx8hr?D$9
z_?s0rOP$Hp=8xIa$V!$Lfk(0a`~@l+!u>@j8}AEMH`v$iyR+1Ohnw{A89&3MRC8pW
zTyuD?;N_s8@|%gvYqBsy^81N5R=mzwtEl2Rq3}d+$C?B0%A~m*OP}X1+ny6s&dYCD
zzKAO-WV(e=&!m~hm6;h5Y8M;cI`{CL@#iHres=>515YigoA>=|+Vh(SXWc)x^p*O5
z2Da8+`?7+#>lOutHY>=n2D31J@SOKda*@>9-#edKUe1%ZncG*mzt=0bYjXYl=w)9H
zMT>~!nC4_YJ?Y)AGF87%Nysa>f#saFgMa!8?Q`Zb4Hm~Q^>5qA=eKt6tJSqhrHbcf
z*l2xo_6^A2y<4p}TwS5N)+j~uidp@%<+XPl!|cy0G_HDoEMDZii2dz;mD1P!R=fY`
zekchS61PY&+#dDt-?vw4Uw6M_y=Ok{?e4r)&tA2<Khj>BwKBq!Q;*qTMq~dMsV8&7
zJ5E+=Ssf9z>iNDe^p?_t(=saRhCMv#YZY`&!?oY$p1nQQ$eC%c|Lzz3J{oIlI{f8x
z_!L?ilo@y}ntN~Gc5R6}`=8;C@vKev4jsC~64<x%d9dcjQnpX0&DmG6ZqMK3tE<1}
zme-X<B2H6Wn&#*5$TZ*Hw`j|c#!2i2%x%_l1eagWKeTFF&M{|>*rU3;rf<{Bxj8fM
zd-kf`;a8ki1%xbZ6WOAE<WT{Gp)&hpKb6N~MT`8UXL$X{<NthV%kI`Cu^OBU*UsGh
zUfMrDOmX|souZf8q7oIoQkieR)L6%^!}qs;vO-IZbB}AWHsk%;O@;fX6?a*2D+MM6
zIrFsy&rXV5q4n+R{lfdTQ6j-j&R2c=v~qNq9K*M>RM@etwsHExxwvsD_q-W*9<4QS
zV4G6E_u=Ef+wIvA<$vd`yY%<piI}xDdh>Q_Jewjik@1irt7eB_zg1RG^8D@mo8F(!
z3ptu6#d?T~b#KX{zn89FzjiyXG&F3h*|ZB1IU;*zJP~;0`AzwD|GUD4m-|#dXq<8G
z+jD&9ZFwz?Ek3)R^s_(9a9%N0nz=Nx{@}0fEvsHcuUJ+zS2R~q{~zxm$0<8iJylvJ
zG5nja@bLQK?b9dvsCZU)Rh2H;6a8@iwqr-!Cx(jsy_mh%*LwQd#XB;;t4incAG);f
zR`c2g?jk<_5+*lGeimNWFkkYv`nTn=3iFa{>aDJ7{829zdS<FDD!r{tKSb1H_tWhu
zvB5dcs}is7c3aT2MwC@6#bvpC{k~=U9@}Wx%}iyVW`6tWbN8rQ`;09V_$uz3ynT5+
zz`Whrs`2vK%VjS^r&n!suikdw{^8uyF3Z<OuifRfWcLrRXGf~m=qza6Hr4;>dls>>
zIZhW|m)-f#u-r26h1s8s)#ns8xdd)yY%;R!o5}JdrR2u@3s?3oPcK#L`_uc{d;6Mg
zR$aGFt=?Sr)cnw_UOu}=mu?A6VsOxYFs1Pbhh%T@>hnE6QqLT_6{aK3to^uecIYHM
zd5x`1HjST`v-<tLZ~EPKNB3H3jk=BhLVJx=A5EArPsvY+^H_Z?uj+N)R4ZoI^S);p
za#D*{^xjz<y*IX2b4}vz>?=BJ4+-6x!>1!4{O-w|GRc3pTo38-gu5JyY<y8TKj-rP
zU)fu{KF-~~_O#tY*W62U&7+oHynQ2O>e7};@ex8D=QtF4lD933-d!@wlaVR--}=k)
zb2;wtr0?TSKR4s5Xz8?Vt0tY!T)A`6+->)~6xeHn7R(Gh7$Rx>y_davM{wwg@aqqZ
z>MBKbUJ0@!=aksmYHhn~zVG(tI-%R?wdK-NlcvmCHalnAVZWm9W}yZv4<2!R``oAg
z#i=g605<js%Jo%!pF6jF`{gZEW8V<IasJ|WOWyIVuQe;_Um?9{(tY3BwcA$CYQ2=W
zyp}!GZ@QDJt5!scrl-nBEqj%$=ZEsdyi?T~dOn@+Z@;v|Tvkc?c&|CLT+H#kk<NbM
z=Udh}urfH#Wav|Qyv-zGSHQ#T_a?ifSZy*hUG!J!((;Z}77h9OCY3YilU35s6*n(d
znbT@C_o)5^fx@2oQe7*SJqw$vWAtWHil9-={GQ{w?zvVT?babLy89wbB&$nTm+srN
zZBdZ4(={JqjwXi#{~2n16NSF--u+{4tJ@{98pgQVean6=tlkm3zC=1xdq;luQ}1oJ
zy*-K*kMeO<s{d#BXm-!*_!fzi%$`b*RAm~D*E7zkzASg{DSLT}Nb<%13`=`gZZDg<
zK5uJQ*{)}&q^lmrhy7kxXL?{l(V-75k}Zwgoc#qVNAn+@?+iP1V|K64+7n-H|I0ki
z(Ynqv;`;q#>r1z1?2L~s-t&&@*4x81Yr@w?7p$$>_cv(5?c616v!-flIOlF_`On}L
z?eTFP`*Az<-!Z+VYN39HPhW?h&fv+sal&N&`g8pn-!?N|yX@;HYc$Vc;+^?Bj!%A&
z9%4OD>GS!S(i--ce_h^~uToKDWb=Jx*DCAU`IBU$;>s))*C|X|zk%Jhzt(4to9cwx
zg{N6wxN)4<m$`W8?$px1+O8&|S2e3v{$~i9<u7nu!NKFLo%*&`hh=7`WK-T4_JkPA
zlsAY<hb+^W{(fbRrdFVqW;55v{!@j&IX+6+UQO&xo)`DwXI1*Q1?A~1y|2U;7jCmv
zyCjvnWTIBkZmk-hzgw!$%XEkcEnjHUzyEENpRD{I%P(s(*A@O+dNfOG;`u9%+<_dP
zcIm>CKi?KyvG_-Hm-g1%o^w7aOZ+TdE9VgH7{6BVw(Noeai$3i8jo4lB|kYa?+Am(
z0r^U)CJx=X_cGb*E&elj+9iCEyV{)BlJPXye`9`qhs+L5ms3S*MxK9<@wLr5b6Row
z7R|mazT=0hCa#r8XU=LcTYl-IW{}Vwri%1Q#~lA!&%cuKkK6X!;#-Cqg^g$ayojD1
zv*NtfkE~fQb1S<}X`R}|HSM8d$t%rgDuOB!kClqQ|Llzpf0TDb_X^L)O>+b<AGTQX
zeGjL)uhF_<m*2)E?|3H{tlM!jb4{q}RB!H8MayRVy2{1!WY;{ccPr#oo|Tqt{kO3&
z;;F}JPWPbl{L}u|R&9@Gx~-urad5SG`s>s0rf%J_?O<wFyx<S@D_bW24Y<i^yO?2u
z@#J-r58JQa=&SU1nJwGu>-DVXI&Y<iPt%;Ad_PR~RP28S)}0scXw6pGb4ljGqjcrj
z?ssP%Wa3a($@-mbu~R8RTH)v8mm!wMJPTDM>~;t81|46vboYTt6-}IiZ2T_@*}q67
zFEzS#!e*QGh3|)r%3Yb>`%X;0_pPd%?a{W)vAOlE+)ZvqOXfZ3Fjzn1VvN^4O(mw(
zYy8e9sw=Cm{gLjBaaPRu70yw&_}}dE<8IsMik_P_@%6WZ(XJupW_#C}Jk;u{n)07v
zj>@4B_WiP(*tvh4Zt%=koFRC#Z<S&HvEBTK{v_RdC(JDW?uT5czt$>!)9X_1>!xj(
z?yJ0Z$L&}C%eFQxb*j;*4w*2=M_xU2=d!M9qtCoK`{w&|ZMkvRt^bBm@2;z{udiRW
zjt>y~Vp(?k(82wgFHVbyp55o2`*^bB7qyUWee8K2PYsgz9sbR#Nfezu<A%W6Gs34|
zhUS@n_`3Ael)%4dzrU!9Sf_S>o7(f2-=6Qw`FlHi^`>`Tsw<~Q7OwE_@alLX;=Y%m
z|BO-TmmC>C>lG~Djt9wHXMX%gDrCZK5rrSUM`O8iFXrE^c2CK*J-BVHxb@UsjV3Es
zt<Kf6f630>$h)F2o<*2Z%R*wtu@&Nb#2(#|zISES++)VuS3D_OwRi2(ySp}|=|8dh
z)wko5QoroVO=gE)t(z|8y6u6cS76Yz`4Nw`6x}%+Rvw%i65?$#vnMnEtL!HGxtdln
z9Gz1aC-*tC=PymDI_Mp`W&QPRE1g4+J-r^yjJ=u_ZhHJlv+~UIP3ne{0@7QjeH5J8
zB$qOQeZy<NoJV`-wk(Zx5ANZgXRoz>n%<Q6?^c8vt^K;AxkThp0Mn!ehl?yZ9KT$e
zVdErhw6xr)v|`tfTj$$vXKeP-uuzw?v-MwhRbBki#6v6vhF@RQlveyGS-miQ+nckq
z^OkM6^+H4Ag_Qhhk4)bcKU51Qi7axQ>N&UAPC2=>RCC7NbUU8HAK~`9e)PFaSZ$sD
zchBlIA<?g{dwR46tXg|2<%&q~!eb)y98M)z%=s65<EO<>QSRH7%cr_*%H%xJQ2yn1
z)$c`awmZ+QnQs^zZvB~U%BIU)t5dH`SQim+d`GLWy@fsV^Ptm5!W@#6ohs%ue_HVN
z;iM}Y4<7vd$KOATYxRzR?b$)Ef-Y&vgj5S26!7_d`PY-lwjXbY@rfTew{=HbEKA?w
zB>Q<LnvYISI(V#Z-&ggVwNGPOFL$+v9@(*=b*aWv;h&C^d}PXm_}0pZH|SUXj!j-6
z8Tj?6?zA^2ejLl*a^K^xL2>ezH+kH*GprYV@~yhI?6}(OmS4HwJ5Qc3K9VB1+I3M>
zilG#LyUy>fB_Ge3ml`K*h&9dEE9Z(1<@L=??h<*kWqZdHuO<hsGY6UfGpwp^e(dwO
zmf5d>zwJMRO@8RNor<O#4}4g$cln+*(#KAE?=btD9Xf4S+oe}A61uB6Y}!j!%zI?~
zO*z<h+8ky@&3~8o=F8p-5Ak$W%!*JEpLFB6xM)k`Bb&P56+w4pyB1Bb@s$4aD$s0=
znc-uv*X@yIweu^tZqzf~YW92A-?J+-0x}P+cr@Y3luu_|D%pycP5RLJu;En8RJj>H
zFC6<D_+|3Bwp}l;Si7CSWiIxP&ptRh=+f<_OP1$m6+Zp=on;ZHy2817pI6dxPnz7>
z{U@lu(1=sEdieSJw5zeZG`Da6WF{(icY9#?4e7Oo{&(I@?XWqNz|d9Jp|ZZ8Q;k7o
z<^+b)@}rYN?x(G^D4la@oy5!4iD?_7&89A09lDZtv&SJJ#>XdJ3#YC)veGrkYxi?0
zx8J9@&F!b%D@|xSCK5b(wQP3N{G-8ZOHFpK&b(>1H`94bX8A|H?eBxUuFQ1mc&znU
z?huC&$5rRPn8g#$G^{I}SKs#aavgV*$z8^S>+&xKlq$XTuUc})`Y!jTYyTNuvM+R-
zVzDhuWtk_7_C0~n`|P1|hO+ZZe=EooCC=%eTe9h0NNeq$&GV}oZqGHV&M^5}wSDi^
ze7yxh+w%%1z2en8bS>V0izB0=iqDZstJ`mvJu<wtXy3f{_jOl|Jj><I?)ZHF!&#A7
zy=!kfi#G19Td^Z-)Ba}{8@8%)DO64>vXM#XRh`7ASYUeCpwIL_gWSiT34vyfTu<KA
z6rO9_HTRnD#ut}j-+GmO_xQac{lk`wCl6|~PKg**dNNu6$(SwQt|wJvrYvzXXYRk!
z&Rd2Wj9r(D6Te*i{Z=*B?ak8l3w;w=HZ9azn(4`YDT<MOf|}>m?4HxS2c>2UrSvqP
zm)V*7d3CZS*NPS8um3#tlH2`m!|UmO^Df<q&CXrcWAb*}jHTOymdxOenv|;f_BLDH
ztxW}fOw+EtuGDIiyfLx&KZ8|ZVcEugEtdJZFHN`qTAj2lFLwHd%Z>$0H@01LJ9TA)
zT;>Cg$sIv8qWm*E-h>`MeA>h)PVwsMX_kLAG%5~ynl!JtdRiyXC8z7xs?}3Dw_QHm
zBf5TaS3ZN{p#(1%A<wsuO3&v<U5;$~81c(c{XfHm*I}OTujL4D-x|1X*?X&{SHdD*
zu5^ult-CL5$)x-G+<r!r`0IY2&yD^k{Uh#{snEA0`hNAwJ(pMKDc@`o+*QgF9KOt2
z<z3aPX<4qKonPh$#`WyDCCj0rtLd)E)zTD{r1CuaZlSit9JMm`19N^C{k$UM_Q+Xk
z#yQWQ)tCEA@4PiB6WN}$e8qwtrbkyzoiwRysQ{<q{#UktI65XTVT(07S#nUaDkZt_
zGIN&eQJo75GH?4YJJhn>w|09)$W6}8n|)TP7;O`f(&mWp5mlL}pv7~N<%FEcq3b<s
zpLCsK(!AO5{H?2Fa>le@r?WnvIDcx(OY_;`ychlMF1we<yW#vpZBe^xT|d=t@qbmB
zP-k6vZcV_4ebPK`rHUbMtd}dFpPJqhKOu_Yrs(?8jTdjf{<iI5Zs_(_!w8A03A=)(
zO_<W>^*hm_@jt^Qw^i$N)0Q0MnQ{HPf_#kZ%b3ZV&oId_eqFv-?bI!C-S%DIXY9FV
z7<J0FoN2P|$%ZMa27c9(ir1<>m@eok!+K=K$&<&bO4lylKkIJ6^7;P!KbIW$(+S_Q
zH*iP3qJ~CPx6icY7n3Sl6Xq8-)g;W{xAn332DgP%H(r<~$-G=|-<Nov{j;2^17H3T
zxwAIYTs>Rw<qoN7LI?9N@-8v>ss8I+XiJCuzUJ(wAsM$>zT|LeRahTwPS|c!^7C3;
z!(XMGyxpaTGv404p=a%#wM%Q=)I%3O``QNxvtM#7pOC{}IAzHSMg!|tXFrs_-ShF~
z4&BKo-WYbRE0ojvW{{oH(^K{3*Y(wQ9bXPF->Y`R%(n5;GjY#NYoA{Bn_OVJ+Iw}>
zw1;1>cxVMp@oFhQ!yB_Bp*8;Bebsj3Np(|C?zsEe>sZk2c{~1UUR!pph3Sz<+k;P!
zX3DV(&GGNQ)9ZcYBg>8nweR&Wt)3j+VD#se?G;r;5AROn1_vYeIr2UK8TM^&6-ii~
zA~$u7k%!x)+d=bJ+>L#?-dlRReto_+-=)yvhUk}C_qtXuI{8T0*K5kk>r=Iuq`ekA
zmbw<X+aTCa`tbS<&oUw=8zpQFS-0lvugT?atFK+V@x?sKs{O(;-rP%bE-hVoFd(UN
z@_&Zio$NcL1TE(7KE$%d`-@DgVRM^FM7eb+)3%b|**Q_O?tYKD>h|CcKTEJ-c=XBR
zS}_gI9-a-CIUk<B?2@FiV#OrkLd%6h#&bVTY+hRUzUSwoc_zWW9g`>h&ARv|Iv{Im
ze#n}&Pq)ZbT0D7m@}2*r;@fhjpY`P*Z8taCGI_Vh{Ff~93)js)RFSmA>lD*u|JT7D
znUjQ0Npc1n%sW)gWOrcxAE`x$HA<4%1Z$+FD}Fwk<KA)pyIb^9|7{-@2RUa1FQ{(t
z<O&G+aZcs<w~uS0mu}sXs(gZdp7|e*<=aZ$YhLm9YFV>9CuG^GrB6kd8Y~Kxsr2Kt
zOFsYFaHVpa$n=nz3?=7|+r@?GeoSZ(KG(e3e5&=;`wbgSo}Rq;bB_O?W&O$XU!7n$
zp3m}f>)f+DrYzea@I_hj`(KUpcMNOF*GA>`?pVL>*Vb@#;e*C6H>vlzXa03FS-Cvo
z?kpD8%_ld{&q>>wDbd6D-gQlh*VK~JzW$HqtjQ5&75A80{`AO^gUtKS=Q<Z2c{Wjc
zi-SyZ-iM!cU)LBI*nXb%Z)eHIvTt%;ywMEWnx0zCpJq<{`*r)@u!;|pg<qJjxOcWg
zr}Fx>YcK!CMw@p{pLA$KQ0A@Tj;OtxymmhCnJ8g@sjlSS>wcYg+>2f+vj?y|{?Cv&
zc~)fI<W(zsQm#1Zly|Xe%qj^nYw$SfRWY$6V9t}iTY>qdoN5}6q{0vMB%iK*@pA4L
zhU22je$oX?ET%_AspxJO<<$KBeIi3yzwPIh`pakTw$%Bwknf*`UHsY<hgQS&Di`-|
zy6+kF*4~J_^-!gTzfir-fj7xp{8W<N3to5f^r{GDv;1e6#`v>ttFcRH<&~>>ca|<M
zH04qC)6^5z4z&HuQ1|)UvHrem_wMCy@x7nVeEPn~hFiOtq_$ko@Rt*ss<`<~C+E}q
zoh+QP{pXhK+2!4&63Tp^{pq<y*=F~YZg*+izHQX?@6(}}vfCx8OpE%q&5V&>S|InZ
zLRCdVcYfCHn(P<yFGGL!*6(nTW#cV;-}7dogb~kH^KH5Rwmq7?wd>Dmk?LtT-Y*W5
z@)xn1@h^C7^G&VuZ#WK0y{%6bvf6oo{m_c*#vbe?ftzjzzg6W7Zq)WF5bcZ47gXr^
z`{kOeyA<OZt=ECu*Rs6(yEy6YH}0w3rI8wX+pOA}MMJJ;e3hzXD_F2D^2mcE!=>im
z44<Ak9OAyzTlCI}Uk47#i5{w7`L88EuQiV?`_1{VJ5?n=f=f&%h^%sLTF#w+%Go1)
zOJPjIlI8{%ImKi9INRIWC*8K4`K07Yc>E9l-rI48ea@XV_I08Dt5&~?<66WK&0~A?
zXw^FT=TkYqGtEnJ@JsNVWXDi;-67NJfI<cDtLsN}9QqTt$NyaNpCM?M<;Gpp1M^qE
za=W$l?z$_7d$m_SnR>ZtX}zvC`>Y8*^VEtbFwU)R|IZMh7i89%XKlRCu=z*YYVop-
zI*Id){xM7U%go#PX~k;x%l1xdUVnXMCc9R&Pwak>tz3~1SI|5m{~a?<GJKTn2y5E;
zV}8%QgEw#A+ji%3gT2<Y`>i)ccyDATfBxB3n9JLIdF@)iSGv}9$CmC0^#9?-E#Um#
zCiH}^pC_jWhup=<cjg@YI$d(-j`&Syjb48|p?<lx)n`&!p@^HpkIa8ksY;jgJ@eMx
z_xIb~5xsVWX#1?(n9jfl(uyZI7+t;Jg&0dk<{x-_DRfWkDZz_f+aHy!luK0Nc*aor
za>r*8sqdG!`|fHxs+9e0>%6X=+r4$d7G4PmN%*AsRpQYk(aCzOj|FBjM;H8O=+XDl
z)p3rg+AblToHO}hsGaqP>AbsTCm&$i{>$OO=kxQe0`vG6%)MT;JLF34+wUPq88|)v
zgzm8j*{3jRYE{R#CyZ}p`_>xP_$&2D9Q@C4Wr96l>6!F@lOx=oCQT81p?S6{tNHp;
zXWq+er|z27bY{(m0H5VoA}l1|vOlQ2db|BY_6mjYX#po6Ucd86UhBp5J5G_xg13Lq
zI&t^bwnZPq&Rbb6IKkVpA(VH?!9N!q8tqv={my(Y&}4q(e6zj}PfGQZ%79JxEu<V(
zYPE`{?Yih$y79`psI1kk?N^`twH5Bne7b!4Q?ITNp<<=rzqU(Wypv~{nEY0OC0}0f
z(zF9jVODcAKS}=l8SY=DGJkR1nv#FlFTY)TNp4}pz6H@$^A;`o8d*{JOMO<};>q24
zGc|Op>N<i{v%G#zTIZs|S`o{+#9$3;#D9i~zjqfMkh9r&#l}hV?PjwLAtkRLK0a+1
zdL=toyX1}PwwKPiOBA<knYKf9+krlI2_MF{?-|M^FE87+qw~llR=&!9{&`u`ZFYV7
zqIsp;H1&4!JMYk?f~9p6uWWGtx@KDd3nRO_{EjochTBT7@Nvn1WitPIYtiFC_iu@d
zR;Igbf3>U9H|pxa(9Z2@b2Gj)1U?R#z|PRmWt1{w(&lfBU;i#p4!Gn#HM}DAW{Ygh
zs@u8g+m9YGt=@OdG;2Xr>TGYFJ`Wb3U9XnL{H}JFP~Q2en`4d0I?t!`=1tAnHfL4#
zajW>q@9WOIGW)(WGGChamG9FOl`X4w)r5SA{J!l)Yq_TK^q-UOGyCuq9dp_}SAChi
z-d2?*nMMh@kArJXU#+U@F5MTIx7+CDwsl`@np(a5Mf(EW6OTGHzdh6O{t@>+K^>v)
z-Iadx7QBrvc)jDfQ)FdD@WfkXla_~E&J1$uSMkvd;A%K{K413vwvyeQ8VC4))mB}$
zxF|AVrE~eT@L<<KkE2d2U+q;jnLdGOp<AM(ihGgWGrsBf_H`Fsl-+ae-dB4*7bA^|
zuUela|ExaEe)YWTyWR1{@magNo?cw;`g2uRdw5h#*-E<wfi}BE)+o&Bln~@RFEi=e
zFYaaHRhtg)5-HE#U$A?3-3jiJOZ+~`BI;Lmx$e5W<#+Dv($uJzr!utyHZZ0CdTM-v
z&(Y0VV!mN~oV`xC?P|THu`}c<zpOs>VTahAE$u8nf|#cD7nb~+)cv2~N%ZgC#}<0C
z-P+am*CfPj<s+sbg(H$7e(ZPVb=@h_UdrQi>_L&te}*dO<Xq7W+J$<>r)6G$uMCYV
z|H~Ttxghe*y+zU-Ihv`ymzFm;#xO7~)btR%>+($@Q_H&J>Y2IEUd}%eP~);a^`Fzq
z(34?j7w%m9K5%=?UFE57Lc?}U^|IhUx6aF;$^EON`nh#~j4Jit&hWdTko<1)6Rq7U
z&lJ`@d3<jDd*8Y(W$q!{a}OupcvBi?_0Z?5msVCoz$%RZyNN-5oNSc|0tsi*nm?8w
zlySOtHFaxeL>9{ffg4{|ujl-+>fF;KTz>BDQGQ>OW`<a=`yIYzrG`eiX|POqz+=@5
zzjhxw@GeM!zf*Ym9jT|0-ghhyeF|=1U#kDO{KJ7KOsD*PR{To-y#8Is?YE)2`y=;l
zySi-Ki>qE*@|CYDR|+(Vzts2_GXE@-k=^mAkGpq=cPe^*Vg5I--pjnvQ9es`!sidl
zueV0tJzu}^+NRaqTuXMmala;$p?I=#Np<H`^<ODdp7a~8Vtv}-$tb<yiGO_E)~ekb
zw!Qi7$<4NE@ji1~QDKqlgeWbooMUsJt?hPxy69|3wM<Zlst1F=K*|gumbno-<oSDV
zN2l1w$e(K7`skwlx59SmdCTRdlz#bJu-t2%<_h1^Py0XoYre{>>Da|8I!Ua<M_B$p
z!%Ddt|D$W3O+DkS5Es+-<Mpic;!>f*f3IEQ`?D(UvDbF(rZt@l)|OsRT)MAyg@i%a
zyeBKp-*TJ$;i}n5g$Zp11!lM9;!0L`8lM(=*{@J!qUpWuSAWKRbLqFrQ$^l#<sZDZ
zll9n@NefSEI?A8!oau3_cy;J{!OpdQT>ZK7eV(iC-)br5nCPj#Orz=YmdhFYL#|)C
zx~rpo-j``yS)sO<_$v;~Z>Tew-BGx*S|EDI^1mUETU_26PISBSa>KQNYxc!U_C=js
zx+-YO;fw^`{NEg`s$K$&A#+u>KYZqp_Eu!mSLW$cC6DA76zzGPHtoyWExqS2KVK2I
z$hnc%=J%Q6XRT(U2buUhqKXO>`W{C`Z~3%syGpW!#IFm&F0u^A3NIgPvTDDoZdSXr
zccWw0((MuN7<XkYx@&ZItp~@14e1>YCk{+J(crIZuyNxA=ljS0>Dcpx2wC=foU7N;
zxoO?FT+}yYm-%+tt+{4lyTS@?HQ3DxICbKrlIUq>mTz;Dez$R-U}9JmJmcffq&2=p
zlU6X#wRtPvJ9FpT?PcrCW<S`r;bQKikXc2i7Mv0bS{bM;p;Yhp)nnh~+{yV`tK=Ls
z)4oftoo5zZuxnGT;~zJz+x2|1hp*3@{8c7rb*ZV}n-_a;ZCoa|wDilC46)A(nfC5e
zm>+1vobAJ+sm;#S6#v1$@5!{+8z%_PtB`VYyLs~bpR4YV{M#R!9V?qs)=+yVX)l+=
zAFITpo7C^TVPdIx-!gyx{@bydde%AKeBNK!Ykof4(b@3g)5kwqO|`{;6L0OCvMWA(
z*I^NH+vm}NKX~_MSKpc48feszvSf<-=QT{W3!ZsQm$7a$I(f^dWP98euEP_TORjUO
z<@&0>==c1JclXwP-MunYy8Ttr*Ps`!wGW?~JhhhjyJ+TB`KL`Y895%cUHr3U@lNfE
zoxM*4y-Zd-iB78j8O(96vE%XmD|J&ZoIQG3Y{d&LbH4LCYzr3z_wD=Kw^rkU%12Ji
z=XPI<<QIJAn%y=dI^)}ahWzT+QPHPYeqHzAYWl*mr@PGOuDa^=`_bvF;*Y9)nqGV%
znu<1i_c{2n74|%NJ*z&bPJN!Lw{{KNGw098V?w__+$S(=cl6xZOJnaY-Zs-SXV2|b
znjV~7|5kvOYELk{eQaM%%9Q(i)^Wt-uq!{FeEHfg)upopCzac3Ui5dl&9l~(mqD1N
zT#xVWKJ~XBm+rCSIGo+llk$qa{`p$DYd?aeOTYiRBKv4%m#fXA+mQvzmo*hSB>yuw
zISAgjnsnh1r{VY6r?Z->bCzwW49&O`y=&L|vbxJTQL~o>wtI0rTjSU`FRq~Q*UL3E
z!CQKkIC}movA!ZSy(iD;pUCwa)>UnH&T8icU+QRORTVHO{58jqK|+4l$DUR99!gru
z$nfnmU3_~^_G;DKWk~}2>m?bgIa(et%>2OfcXtNo`cpCA9wj|L!Jm`sqP;qSx&K^;
zEO(Ht!mR@gE3=}4RBaZVzb%)-d^;e%P*dIcmf@AX9M3;3IwJV@t=(6)*LmS~tD@2l
z=eB-~`ViFQc+7r9^X((AtVJ{DNBp~IHK9K0-ZziOKC|X6T#z@Z*Qafs!wKD%gDyn^
zJik2{{OA1pwfXiY6QAUsvh+Xx_v}g|OD`~H{SCX7Tk=(Rr<=%BQH?20Tt4aF-v0eF
zS<3(5vNz27X%Z#M6BaGo=(+pnjZ4RVZ_N~Xu5f!zE6+-;?`r%7%Iw?o_bs0mz#!Zf
z@Hc#&P1y<`!N%oFvo8ed-P+>asm1x2$<pD>k@C0qpS!8Vr#tIZrr%C-*PrUN?S)y#
zlfd^&jt1)#DCjT*@lQ?Yl;8jR#YAD_?A{Ic*gvmyIi<q5`hjss(6&IoN+Hijlc&~i
z`F_5C!?m|ZQ-rd%b?aV{tN&B%d^bwt-&2v2kMsRYE(NuydQ?tbR<Ez>^2ok^VsDxJ
z%jvRdbB?_3*Lrqe^uvoD$6Xi9BX@P?zHARFQ+>B--qu%A+q5+GUZqa8Sd|s3tGGO@
zrTHzNSH+*!<GHnlS?_+!XFa(7*=15<ueEOU`usTELl4vUUAVI%q*C*kmcSPM^9ep$
z4hGeZ^L}L~7pk}%JDflL@hr1TFL~-hc5VIIoO6Hs>)*n8SC;(^y3BVqEbQVM-L7z<
z1-3#PHG}@13hX~EP;qWsTjg&vt=;uD^<GVnyPQ%umRG6Wo$YjZPtMlgdEXuVi#A>}
zf3fS-MTI6C?Oz@W0`5tSEN&ZZXWenT=;7%6%5LVe!hLN*4h8O35AR3U&C`pOx%1|d
z%GK*{Po0`l-x6-t5VB;MXKtpK&lLt4)d&Wrg^kLMHLf4FYzvoKm=L~@h4~#{_PP)C
z9by>_dXAr;zV*mI7_~Y>qj~*Bv*}^>EVt%4INk9Qw-KHye{RntwiKC13t3yQba!|?
znfrUDy<t!AO4Xpo3DXikExfkkrABPl#MWS|Thf_%OK%muk_w(^AmqMk;hA(zF6D_=
zR_7MW<ZM*0DLujcsz}RQ;S7hvk$E-yy^egnb=7UUy#BrE4oz*B?wV>%37Ew2)qcu^
z5C)s6Y-#K}4qQ}jyM0`4XRP_NGRZHdOm~e>E@qi8b<lIqw%zva+q;fuy_~fyLL*vx
zxmIMJ@QHbjMw}td%0le9M$;c2t>`^;;%1)1<1cIHu^HBv&nrCDwMP2Xq+6|Fi{`|v
zipkXdl^e3e<QPMNVv4A!OJ8iCUCAuYM`4Cpiu*6I9lzG#!H~4}MCsp^`-}G6-)_C-
zyOj0hkPL5)M8_*eOD&dkJn@hhFrJe!JA3s?;ou{$YM9<e?VY==?&E)k*P(`|M5nbs
zn!7@G!AIT&odP_}9g|fY<m)%QHcJa{-t?Y1Wa`6%OLRQ#cGqfqzRF%4a@GIVY*$;I
zsk)lpbEbu|unN@ch1xH_9d$S%O6jZui@fmsLrcXrvTC>9>(pJnWKAmDb1&JYUWr*+
zTAYS=joj+vch0xE^(Z@0Z}v=?C%HWnCq9kPW0CENkdI$l+W(_6M@({C-o1raHcYYl
z>Rk4_BI3c4;Pc9p68<yz)o@<j{_uCvwU;{TD$lJ23g5AY?o*q4NGIj}pP%7@67Edf
zJfc!l^gCFVviuM#^0fF{@-nwN{Y|p&^3@fgsocq?f1d1U3O&lz)Dh6xGv9W1vW3+d
zP9cRi^FRFx^ypQ3vgp~$=W*MXHs|H>ZoeXQG*wh<-PxUmshX_2?w>uF?Oz{|;PIZ5
z{cZZg@Z!_Lw|7t4YrycIVb+?s=##PsXE1Qy&I=Ja`k&#|?qY>I+rHgjy8ZUG{?J)f
z%LT#$7!LYxc_Nf@j^h}E)NMzZo|GL|xBu>t*OI;6Wz%}4hllq+gVwFv_XBR<PP_l&
zjcm@%QnRe-Ap*OeMf(diIM{@+tX24=R>1InLB-mRtG65?Ln<1-98~%JXnA|T*Cvis
zcJ=jMyRV(vUHhoByO;Uos->lxK_M)k9l!IJxy;Y=?<s$?eY4Idb`^OZ*?&`WdQLC|
zUfVxEkI^f@Ya$DS%LMiuPd1)@%l99&4$FBmc2`sc2Cih^>n!Tqq_8kG)wz?SX=(b7
z&$c_0_@=KhI_20ip|<?HNRSBsv~?-<f_ANa%S&E8EZ-Or9AC58vGC}MnO`O;c{#8K
za4nep?emiPBDW-!IR)4PA5Z;yU*hMZ*+-AYspiNqSzc$WO0TP4?v<VzAL)2|+OFxc
z-o4wxmfbFNv1m>5-=Wdc(-%0S(eX!r>v5Y&W-nf`7tc^XVp?Y3@?*9)LvQs>=Dnxo
z?q;lC^SgQ3naH)%rLVo*8Xgfe<w~I1+#{MtStY+rn*7f3;{k>>2To4VKQu?Y$Adp>
zsYH?b)R;UmVeyRHo}s({X(Sitw#Jx!7TvP#SJm_hkMekbv0gB8m{hy+neaV#on!M{
zKkS)c(f6>gM&$Wx54}9Oh%d#=r+0srT$OrdR`k7xRj=;Kh6ZlG6uc_n0=J`s(Uhhc
zGL?e=e!a5c(#(0Hxjc8@$Me@#UVrL$s&?n!tgX4+v%m4a-lk`=F7&os-p2W%$AYyV
zzv7Dtycsb+^S#v0Q*SPsP3~#_^Dj4)V_rmlz^dezu`lk}8f}VQk*Rkry}A2K|Bu=z
zp=Gy&dL$lBaQwviZBp_289|?3ajF=&2{7N&DW3E8h0VrUZ;by~sXOW(d-C&wdB8Wt
z^n`ocMR!eGclXMzg4<u7_AI@!&1p)XF_%m*%TL+HYjf19yR$DSDLCA(kgNQ(%)O;{
zzR=EhU0IQD#BR-YJHDc7)lFyT@&&HmQ{q-hKl<qzH0eg*3FAhs{=Qo=-^A-PCaO)C
zBme35mP3!8?BY!~X?SgRBdb62R*qiK<-PHd#lE&uCmAz6o>b0|u<?|sWBmQ{=uX3!
zSM0%zllykp8%7sg>EEW-&zg7erP$%^xAL-AY+Yj#S$pV_lh~t6o{t<^6FxdBan5Xh
z?y3CnyNsWk*~8O9^$)57H^(v>7;l>*-@iOcr@XS{=CfS0xqEB!KAm>4cMUt`z*Xve
zuP;$4z>a5P-_0w+HOfa8s7$e4@Pz&OTc4yyi4oHzl-Ct+pYYQ8ZQR>3{p050#ib#u
z*L>B>47%;TW}CwVu}uLC$3&NveZS-PrX#&Q^~0S%J8LsMEliiZzdgC-_0GQ<?wK(Y
zD|g4|P5X6iosN>oatVhAAGMS_jZz9{SRSwFP5Rt&&meSH?0K7Y8XI0q2rfMIpJDa!
z+e;(w%`BO}ey;oO+3S7HzUszGi<}CTJ@%+&(WG8)r%TEcY^Ja)&G2lR^J0ZXu(z;f
z$5W4UlN)MSxsUC*y2MaRoLScDqHg$yd%u?CZprIC9G`I~l)wE|_RVEGmQJ6vz*EtS
zllN4Qj>zOj_I*np^&WZTTXyY{J_CpPl^@SmPyBdWJjYp)_nC3j)%o7me$&q9?#ZjY
zygTctuF<6>yKa=8D(7*`F&CN;wD4#g&m+dGYOf!8%37~tX?~xQ{M0w+O;p%Lp}+5T
zK7V^@t-JGohBm2R{g+nzeY#njv~ug+`=x%%Ccbi65~O<4Yj#bmrkjIT3!hC$Ldx7I
z-HO~hAG&YK@3Caw@$=H^{tml?<~QFdosF04tmoO=oO5gY+Wl8&hn{-AGdReezhvQ>
z(iUA-C9!h}6CBxRSackD@lW#5%qvHY+9v;X{yBfC{{9~)Z+x5ZW^3gy=`U9||KiPB
zyJ+L}JLasp5xKiot!kLEGLhBG%fWxELXmxu&*YUq+?lFty*w4(?+vz7{Lyevqd;@z
z&b%`>-%jV6)oSe3b}3f-&%Rksj#^q4jmN)8{J47PRBGDQ#)n%&)h6$7u)Ama;PsEm
ze{-(Yy4QNW%lfVA>U>x4_BqS+4QvO3^OtU&@HODbl3nYLZ)Z{Ck&Sz}af);E$vy3_
zooqg{9PIaBZnZ+JZ~6A1w*6llbhW%Ec(4feXUaDA%kOC@HNJ6g*UJe{Z4B(AB<`5>
zu9<fG^4Hb-FWz6vx1!|VogAKb)@v_k?rYulSNfEG>!ijvPG_e4XPC}Xm;88>oZ<nQ
zvdKz}60+rTcim6Tinu>L)m5s!FMC(KPyNH)P1nx7S-Emi*`i06vZhR%#w8upVK-xb
zgZuAu`_zAIIBR>^MA>ih4t4uJ?X6Lp+@;pruZ{n0xsW4qTdgPu?~>N$1MW|K1pD``
z+AcP|DDm^%m;1ENCp}o1!(U~VeKk7f%CtCz8Aq1%@4s(o6>-PPqcLD-+7siTvrDH;
z?ppshV%OgEtyirsw^(cnK4H%2X=GjF*!=DNBax4sa<>dNcyXKGmn!_aa;-&{*Mp=9
zwiXdP3RX-N<GOm%HDSr>9cslYtJrF?t3%D&w|RC1|8t-D{rQ)*7DtR0XRe;U#XEP?
zDJkcze#bXhR3{vKdZ+yDviC=PPA+WrT6_NQ{&OX~yK)z{efhX`yWgkw1x=4RefXZp
z-RQSuciN+MsqBQO<B1~4&tDcT*`Cp%T#&IdEOlj{pT%Xp)iw#Aj1-vEmE6BwxGrEN
zsPr&3fA9X1zL|^k_{1YER<6$N?JB!p`YEE&Wo0m{`;&4uhUVXYw_I-9t@qvX`1H@8
zmQ0`SYj9zGRA^#Di@7hKPfIZalZT4qr+S^vx02_ke$+Dxx-E7znB}3rvFisunO|Ga
zp18s)9sVtvb5-8Fg(*vernG7*_-|8rWXP=W<YhYFz8x6_0&d5P?$&-ke`<|+jF?za
z`-C-Lb1&TyYLj33WRb;3C57+8_YXe(#ZYqU@gs>mDGT$oJ3>X3m9y5~`(P;ddN0GB
zZF^(4ewSPB^h#4`eP9-ocjkM;2OR%y)#e}GP&nt()#)DZd0(4u&6jf5^ekm%zU--1
zznX#P=lOXzE_1BSXMF2;t$AH#*eBPOr6GSs?afb3i1iA&VN~S4<@vOlnSZxzUDI{v
zwf^$gU)M$^UtAK_ntJr@vWI&wZpoULxF}jp^rVe~2a8wnuY@MI-R1|(GrntcrX@dq
z9CP}yr?qxjfq7ia{wThkG5OZZm#3G$^LAY+KK0&<=29NRmt70Grm&RHWD0Mnm=Tiw
zDs@{$;R4~_KZWr&{sJ?_UOBc|u6+HU;qr8!a@KQ;9?W)?RX5hkF}l_CsrJ+p&L0Pl
zb!>ZlX+fssjA;j$<=0<ZY$IfL(56>D*Kf|-wbL$Xntk0G^0>p@KqzDjqv6}eBZYPs
zluu5+m%02x<;ID7KYl)T<!FW54c`@Kxmvc}RQUMkU?jsqiTTW5u3c`QWVqt-d6}1c
zE@t;TzVzC+{LAV5Qv0l!ZF$d@1b;DQXxJe9DtO2IQ=e-NDl{!}pTsYeFd;0yz;qg0
z>(Bku^|QWnO*yPE)&BT&|D5ad@2$@{ecsvpY~1NPUX@+XF24$}yUnmY+FenzcHJl8
zBdk0kGBv`VpFZcGcI?58^QRYD{(MqV5uW@|oaqN=#KT?hdn(f!GQW$?z3|!pjzg}e
zp6b;N>pZ5G{h7h?{Cu);^U`gX4b{vPZ~yW8{iE07!)xDpH<R;z+`c}4eQ#~(!dYEO
zqW?smI>(<XUGjU9$`c)?isJ<g=Z~pJ#TERX`NVK4i@xGqt$mLz*T<Y)&nsYE5fZS)
zEH?P;vYod=TTSMv&#EjJP;ofk_jB^%{2##)x3}y}dQ($0Rntl&Tk`vwt7h66%k{5p
zIQ`Ot@rc!Ft*^Ds_YXW#E3|#W_rvPQAEQ@Sr*1kTlgDuEpY`*X-anS!HeeGl`+4ox
zw!7=*K0j*J6dBaiz`H8=Efb&h<G)*3ud_Z9+MhAKZ^0MI;}6}V?)#nK)Q#d=oVL_t
z#md4^tz3m|6FeAZa<N(WIL!Z*z2dm-L-u{1UsmNVF^D=9=CggK<=<C@t5PnQb(IEN
zU7FJ16|z_{fI)@-bx4zE<MX$RE0PU5AFA^|>_2X^`g)+Y!4(Gk_-9%|iyD`&%9h?_
z@M>wQo`S|9$1@8pp0aPV>HK;5w4b(+GIyy_(aHPyUQypxNX+>8pTRPGVgGX8C3ELq
z+&eLo*Pv@^pyKx>AGx9~iBv!Tv-gvdo!{Dz_c;?-+wNB1{>}5BfwkCA=gf7VGi5US
z=Vl#HJf|ol(yG$p5VT;mg#Q~3yPBHEV%wA73E%x>ec_UE@07Rks~-r>4^f|>BCK`h
zr}FWqlE43cH8~My5bE&FR42@E#rxu=`$NhO&z_wT`*KC*%6q$vmR!5!wPfYD^Ck@w
z5_}vjk9S@aYYI4cvDE65rSj)BF-Kdu{0f6_>;3iJS@w6)Yp<57yls8Ht&A_%)qd>z
z#PNPkg_T_O%+9MuPyYO8c-Gb6!X}Y!+FbpHclQ-<*0Q^I#7;$>N>@`T2sQen%O&a6
z*xawYX^TzyrSzyC`3Vb`Kb&~VpiETu(%YG_8_nkO-P{qr>dmV3Yg4QyZ3t@BD?ZIR
zzhA=VRi(9i&u(j`s-}8j*AFka-W(5<S*5(XHsQux+1qE|<^E?VTD3ir)wyKOt>wYz
zR%Qn<N%`*#C`u|2NobT}`DmehFHNiFKf~z@T^nwG=v*JMbjO{rYq#&-_OH*q#d~to
zcG>l%xwp1_PFvObWyh&0Y)%a}OC(uUm~5W2oRs72FY^7*aJ&Cl``!m{R=FIO3te8b
z^-^e%z26p-w^|c|9u)C(9N%WIBX>N?*lod6Z=U?Q$4|BBeK^KFcQ0$f^Vild#op)E
zSLOekcl&L1d5&6ZsJP0u4<4-ATv`rM3$%@mwPr9gWX(RdaN^>t8D_zA^845Q*`sy)
z!`wHQ?kMTZIIq*^wC7u1?DM=OX-$jz=5;Om9<)_srH_iml=Tl)7$)CU5uX2kNsQUO
zca<}jpX?|(d9>E{_zvE!M<*HD{wbEOGnL%_?)olo*?Vs@qE>4*^?IMxO^p#^_%7m-
za&V^n%FB`Kq;9NT&3k|1zq+px4`d8(dmLoU_+#MklXnGQd*Ri%6?&(a?Npn#^3RTz
zEeo=)PO5Zia4@{3!1&m2hKbQ4ujP)<_g1}cjW+)-z&QW;TAtubIlo@#JzR3fM11{^
z{%-DVyS=9E6?zcm-Bmx~`SHu_{mcIrT38iKH#}x~q{uR1Ui^8h&?6SAGaB|xobxmP
zljwzW{k8r9QM&hg?r(n;o~beCi)(1oq3|H)M7c#lE41Dj)L!C{UGiDJ!;bgud>@5L
z<_DghKXAEEurm8bD^CDlO8>QO-}NKP@`K)pSDjtBYr@xJuHe(zS07B>FlA=AC_~V?
z4@%X^#tYy5lgXHyzUOA!0RjJqk-HuldbH;5ezx-2_j|K@|8Cr7_U=-7QQ*2ezqjg&
zsC*9AJG4|!hEYMJQBci#Zb_VIa?T=86~!sF@AVz(y%rukrYF$KY+CRux%TGVpyaIG
z`{VWm#=W?9>vrmK@A~c%MNd7}iVz{ov`U%m8HL$yTaEnlIN1~X`MLH+i0zni%57eB
zO33*u{C;J7f1OQSyG#3z(yrTaw|Q(tmTvnbws0y#(L*6ehLnnHbwYb*s_SruD6hNy
z>$8Zz#Lhk!j&kM0Y^%<P&fKnaq<3>-)}EM^S(0KKmWs{dJ=Jw(lZOY(zyAz1S(gu}
zPxvS{^@;5?z5>Vl_p8=_%sZKT-B~pC<;49fm+Hp-*d7`fz4ANjsV%W{onFQ#`mT&S
z;J?*tew|}eN>fcw*4=NF31_Bpx7yrJsB>DJr>k~w{wuAECI0`U*SvhYYu4HLs3{>9
zp)r>OvSw-;Eme_WW+*#2@%wet$Gv6wGvb6!RW)z;dEMz@+SRqEj~%`7`P2H){|wGk
zci%7lVs|=l-x{y&yDrbkz1lX{WdEa*r3y<Q9SWKtD)EfPq9}Ri{ML`0sm?qfr<~VY
za9!!gDh;I_+T2g-Ew6se&tsqcN9o1OJBgVZLDOckOqt=)ci47++^@}XksiYC=@Z=T
zQmO=Zd|!3Os5kQb=})WL_wMV}%fFNLO2=)|`|WX22bSg@U9@n^iW5w>OOpF|jyJBe
zwLc&^bE)IR>$;OasjdH3FU)a4=6k#T^Qhty=_S&Klk&K-rt9jzUAyHnU+|t*EukNs
zUKNs082jrB)c8Cu-*5jRv%#}*f{pR^(^GXG-|<aV4m>l@w501&%Mwv02SxV@ih=X)
zF8KcA*e=tp3v+hs>+_$NzN)+Q^2X<ZTSZUi>RBubTJfmjKZED{iMJp4=koKUxn1px
zs}vQS99CMgj;mf$e{EZp`z5b+y>FtoKeTGzb#vun&!v%Hb-&oKKhjcg5tz=LInid0
ze}z@y;hd{;7#b%O-}w4viFwdUh6x2V+GqGLPrvv*)Nc8^-wSQcZ(YB=blQQ#`uAQ*
zczJ9$Sjpj>n8oNFFDkNK&B^of>eDkl6*LZfVE<^k#cajJ>26OiOrEvIx<6xQ+MVyw
z%sQIKSG*3p|48_)Q&S3O<MYSwijsXQLVDaY=R97j?Va=7^YYVK^KOM~U*+BX{z`Dz
zuC^Vz_7_4zYWIGeWY}@w(mCy;LLSyeM^?Uk`qtl0;-Q+SL)_F?yI)*S%(Ip|bla?Y
zuhaeYOP8fwnKmaNm+!+=7U4<uJN`5H)~J58{W3Y;`$kh2-+{)x<-bcjE;{%8yzaW@
z<(kYV+qUX@do5k(d8qEd_s_qwrcHl(V{z8`M(KLqh~xy}+Jno#F3pIQvh_+eKcZ>l
z^nB)j2J^g|C$8)%%URU+PM*Q7wkFHN#Ng1nui^g|WLyt+`f97KTEumQbIRg<3Or@+
zS1lLYy1mW*cTvQ=gvowCO#U<6zdR>2;pMeQTW{X*RW0=nx!PIKV&?eBXNITxP9>h&
z$J-We+tQY8mT9!cO7{G@jYr?8&dN=n8ohUC-5*n1sR^%QSeP2mOlfkz&t`qj=HD!}
z!yS7k2xMv*KH|539wlWRmNUt0wRys>OY1uMxu(m`F!K3yz+Rq9&g7J2(cU>aH69A`
zw;K966@@;%yf(w+yIFvz)6QLmFBZlZaWbDXwqyT(A;#?)$Ay#9vhF7m<a1w~TwCqj
zlE1X(T3GDsOPAB;1aGokFsIO8Mtl2=iT3t_t0vb37pqNC(5_%8|Gvy@j-1A^H9c3Z
zvTnYdIaS>IJx7QCmPwon{PJHfPMkh*>T};58wTcQ8M~)NCbROf`Yip%v_o;Ko;%B;
z>OY*a$Lz2Bsy|{?I}l@dqsZd>0+k&ul7+K+>!z-Jc4py0E(I;l89zSvD)rs^WGQpJ
zjrsVe&ubI+^0hQ}^>VIAt+}JSZ>j)?=cxt9Wy+Q8{xj@bW6$>|@SA6e(e$fl9Zs%#
zF1j^c>a2dA$o%HrVQjMuLS%iGak20-ZxN{9e?KfYXv?`XJ>m<PKZ$7ZxOdIoc4_*&
z%v9aJ%&A2NTPCP@_T87VR5NVUjA6d;_x*G&2ICbcCBLkEmNVshbnl{R%Vy2m$yKRU
zxO~==1|dtQru_zumh&y%uABNn`PB4{Tas=4mrszs>i%ek%eS{HlCL%XTeWh#tiJEu
zRX68#i-wp^xvIzSqIuqPxsZp4vifw!+g87px5RiFPwFYUbFQ%Qa`L%-wU54K-pC9-
zZ5pbxH|G_XXVCYpl{<pv@`LZb)SPytRZ^&Bq36uz2`L^D%3+G>>x>jsx0KqS2@R9e
z`k1`V_V}f>;eJ)SQ(p!9T~f_8-SJY0_t4C(zd9U*wR11Fnk~L@TxWutiN!X<vz@c;
z(ygmQC*6ACw|H0aQ=jAUj4Xwbj*mj@mny$8?)cAe=E3AjPTAuoi93Vp-tpE6IcNmU
ztxdku<7heWKZEe&H*Rfh<+2aFnXk*AUz(&H@u8IG(W>oN{V$(gy#BAr&KnoDL~n1E
zC|h;7NGtr`tQCtpov!dqi<Jr5@=ZWgXySs3?>yoLnfI+9b6@}TeffQ!8n<cNnWJ6>
zpDFpe-degn^(%Y+J9Zs$+vPhi*DhLZ^7YEmrB>5o9gPAWGFVJ}x#oP{F^7Qb=gseD
zZS9@(musVaMY2M5#iw<jQ}y}F-Y!W9|M_WcmF>){;!I+zUGGg@*rI#h%cqk|Wrg5;
zrVx(5COm%<L&KFF;@npUm<vT}C(5v{`D!h_(!)7}J@3-AJM%-Yim<z}FLNrGGUr^)
z`HE}bY?8P4Z=NQ`{iJN=;%ZH4m)tp@PCRu9yF2&FbkSqmi*C%!Wu6hiZ+XJWg=tBr
zno{x?L8*j0m5cL^CaUy*(%tao<J#TP1#4EyZF_yGJ9FE)M<Q?jDOX>y*kLf`#@&b<
z&z4UfjPXY7bEUTLv6ufIthaBgdxf&PW$LH1F3VPay8cM$k&_f_jMMV21_PFIpPqyi
z!OyGT^L2U3yJWFhO)C5m8tP#CU}wJ6w4^2B(Sc#hnKGR$HM2M!HHA3-7Ff$~WcF8j
zFlo!)U24l@{xc-cKelO(r@_Tdy`{!m);=%E@12~quxQy<zgWrStB)20OnOwiWciNw
z0_leHZnfPy^{9HHvHb4y(&;CDJiavTut9H>!_of6&mT-*23>dSimu%>cSGBh!@k>^
z8a#a@H0K>@KH>0V^6E1keLH`zv8=o*l%5+kZ+UI$49`ptuGWte8$7==3NiNmXUO?+
z<t1PEG@GBb*^zn7GAr(FSv^<h;;v89k<qy;4@{VIMn31n%(q5tb!z4t=bke1YTo_y
z{Of4LBdHrhrcE#YI=@1yd#l=$b&Hn-&H3^|_X>|ii-LUClrYXfLqmnS#Xi2fjLI)M
z*e$r46@Aip#T%=T<^JZ+p1gee?YZ~fms9p`E-bOjx^U^%AK~XcJ07zoa?4ceD{4)0
zk##K2SJDtr(7rBfTkP{A_hFK?W8<ZDb8Uk|x4hZw<Gms%$Z*Ph75^fY{|vXiuRVEb
zJ89D4BmCFaTr%1<K|4cL>&r>63l|+cLJrLQx5#?(g1hqXb|{}IT0Dj0hG5M0e~Q(q
zUHd=89@)O`s;0=h0HvvlaWl2QtEN0CY7taF#xRw)czxf4-bdckuAJI4TT=P9(ysC^
z@6y_mjw{R;44MC;e9i62%l?Khcqz-PcegzCoz>Qrud1f%?&`8y$KL&R^>pFW9jq&4
zI%YnZxb~m?+^w2Y1yx^v2KV=_xt5ao<*|I!yHM{Fg^EXa1aYS-v`$)DH`$<uf8(3d
zi*auoFP$>^@yTB6N!WIV9m=fbC)A&YbiXQITg$lc<^6zf`-9(It6sXd?ECH5jYk)(
z+IUe$I%DR)f=GWMl_i~Tmx>!5xA7OeQ+o2?vKq~N-&<RE_0{q#ep;`l_cnIv742n@
zR<2KaynS6>!|eKn9$L{=t7o_``^fFSq**&Ms`8^opIT|u!x_Ct1SS=pUSaoB;%E5f
zmo=LlCbGPhmVWYRs_D1Am$T}NOTL{9-M%_^t7zMt>s5A>R!umsc(z^MJJ00S-lHBt
zj$urbI4-_7-I2T|yYv-bp>y=iu9pg|uluGbdnWFR(k=V7YTB!tTg9h(-7UHqnW(R-
z=&E7J6~H2-BEa+FXWpb#Ar;sD%g<+JAAF;GVMF3#nLV#EZhvF_cHe1tcJ!LOX%UsB
z>(?%kidz0tW$TZE8PS=VPej5E)~QI$70fQZaZK>BY^{51W}%7d#>R=ye<m%?ShH<*
zv3K3;gv<*)j$U3KTHl3@mYiUJ`~2KwIkim_oMoQZz6{vv`BnTd<J<ng-&azz-ke(a
z)$FdU*o_q#LQ(9qvaW0WOEBb|c!cG-rA(ShjNV%dq29W^nn48>Ni~hF9~b3r-k+zj
z+*xD!pW3Jk3pPm|3{MF2<71S&@&5PADIcR-|1<0ep7?NItt;zc9qv=Hm0TP4THe`R
z_O@tK_`9d;uKYEck`+Hy?2tg8b5nc4N<W7uoS*N9`e|m)aOgZzdG21^lxMe|x$s;r
zckj2K-zt%}Z+Sqz)vME{O^<W--M%T)tEla~USXo@<|F>|E-_fnII!WvwT)XH3%FB0
zKcDmI#NStXx4l;j&W-7}xxD<J(2cu+n*;Cb{5G4p`m`sn-m0~qetLHtlRlInRLSJg
z#4|(CZQs6~8z(OAGh8t*dfokiEbn^~GM#$jo-en3X}Z5x$8_5BZEtP2-TK=5Y6(;3
zjHY<*POsz7*gIW#5?)=FKiVy%E%xty8(&?hiH)S1vitR{7yK9bFO)8>k6-=1q;WUv
zu1=rJ*L8K)?D$nNRcEoPYEr-?9(To#OS>;0*|CdhQ)b+PPo``3q{#{K6ediLoAOQX
z@}1jZN4MqZT)lWT@WN$NpTz-_Vv6Q2VJTuYeBWU#;4}GKWu8Uv23IR1GmCSNkDuCk
zTwvC<M@}~m*IQmb)~GvGbIFooE>Axf&lAnJlIpC}{d2F$Zr=R${`}XW&$pM<EI3zV
z74lW1)LTzg%U#J*q(__OB(qQw-=Zc)mDYa4$G_^niu9_Li|!1K*_FoCQP6f?k6DW?
zMS^+ad7<Rv$2upeG-N9Xzl`gWlu7OTS1L5?irE2gehVQ7E#_%1>C@~#@tGWy3-8c)
z{=l;OKf~FyXNh8GjC{l8_LeqE^Bs9@_Qkg3&eu~q7cXsD=N<LqsVDoVrAF!p4b=B-
z%kC64luEZ{zc%@fpS6#fdmmeI9J~3vAFs==r(bJ5dWxI(-S@rv<)IO8r=^E^Z^`-9
z<s=!FprI90=;om6BH?lM@51eywuBV#u1@*&K6JJB@!fM?1g-uRzU=L<^M3WM+b{e*
zy?h(@X1_d>3;!9q^0KFDK6?A>%@6NHjezgYt2q*P+Mmwk3gOe5)t~S^xI4p{^|r<R
z%ZKKD^XM{@zOMP7q3!y`t?XC&gY^EUy<Xa?@#5{m>FdM;*8epQ*V<mGyNv0eK!4Gk
z03YWJ|DSEMKKgepoaevcuw<x%W8yZCm$MYt2-ymIt&07b+f(bSw(EBCjos5<ao3t(
zx*h7fe94#a-=*tXuIySW$Ey`nK5Jos$X3SF3KbStg&&3Yszl`0pY)Kh{Q2o<^gpqU
z+B2u*WM9&E+jf+dbynZ31`U;M^R%1}$QN;ZC|*CoPN`FA#_@u_cl^fNAI|x!^*7bh
z_L?Qrn(B~Hp73e&{LLE}JpyKW^!#VYIhOudYH^`tTJ6lz<UbyFDxZX&y7f)(yx*+C
zfc`_bZ)&b~s(9$dR55>J|IW8-P51c8a@?O+{%ujxySdq3e_v-W-Dw?kHPt%%yL{H<
zsXU&XZ-s203$N!DUby2J$H&`WSN5KXUFKK(GUngDH@~dQUaj3e!Jcc&lBV_fcCnY{
z{M#>QU>dUbVJ1s$7yHSdq06kFY2T{VI{2n`zQ-&PMgGu&7@cjmm)eAIp71H3BO%rG
zpP}XMzDD5<#|<O|f26g0|L{C~%r@(6&Ti=|>r_{DwNLfo@l<G;@y__O>_5|$LI+px
z3Ek6XZ#?VB=VM1Qdi0<B{_?8ws#;PIB=S%zGN?0XiNkxNIb}c9J%70F-6O7+-4r7?
zxBOPizMZxcV-f<VKVEq|cTvTehu)n}7E9Wz8PzAQwtd^%F;%I{O#i&)pVuD7H(x0&
znYo&8+pS9?k4{aMP%NG~U%3B^yy1#GDc68x=a;|J7S34UQ+sPdrK=;$TE!0xHdAF9
z-zi9jznxQY*=76G$2u3Aj!(0yuKU7$U#s_+Yu$zokMk^+hzLBG`ERjcqx|x@@`o4P
zuDwzE*Of~(X*W;!(+EDRm&<ci*47+Zu)0UWs5;&D&nxXq6Hc&XnF^UJ*B{#!>AkMW
ze8JwMenQJ$d&MwnupVJzZ1O+mpa1>pG|wd8a0T9or*l{4p49l|vF6*f3ul9E4(BR&
zP2h3#U@EBpT@qrpX|dMJJ3or$|Lo6_JY8fVH#;~xHBzed)Y^klAzRcUG9<OxTxQG>
z{5=2chr;Xom)|k7T<3daxe5=@=dUcuAJ!Fq?EN;QN%Q<(J*#i6mX}xGb#~NrHDC0(
zHJvk4f7{Zcp2njr#}E9NP}jD-AnC~Zxi)+D6jfc9d?aRdY^lV@kKfj>+Piv7-f36Y
z)4937_XfzHs{CaZ8{y~3n8oIzz5SfwV%_y>p;`@$4|~rq`(oI{8W3qzS8vs{{LXH@
zwhz5)9fKV`3;mfgHE*?S^Go>XFS1JI<Drg!zswR|F&^Zqd{AfW>Mr_3@Z{3yr@sQZ
zrfs_9JZJBc1;UdiYcYAb*YEt#V17Tc@v@t{@deOomT6z+JfDBz+N{_F)kHqkqb!Fm
zKlGZ+<mTsTmog{gcO}!dnGWY42C6idspo6`(sVk!G<9kpm*+&*x@0AF_I*n-lZ`r_
z-H?l(H_y6m>Y_WVRwiyVb=bFIm)!2V(tCq%hrCz&>T|*M>8u>Bl@n_FKOTJ6cvQ<{
zrU%Qvm<#qR{xk6FoXS@`x3Fi`CuQ4O%d4v&>@yZo-!vub<ZWB+>04({y6E<B+4_~+
zFIimh<X%y!P&7k<Q~3TCPc^&JlZz*&^_*B>taf@uZmgR?Sa1u^KD#>6rt1~AuUBo2
z77n=Uyjjt~OWF3?6afy-p7{bSFM>t3o4xt?AmhX9x6a-xgkpDA@P#T(4Olz*WyrN_
zYdtv&j3(SH_@-!cqNCA3eOpb&4vT2J=@Z^r>D~A$xu#Px*l*(N$MGyNucmCz-4;E2
z%H^L^!+)MHa#2V)^Yi_k7ttq{yygEc{_<Bywn~=a_S4^&JnmSb5VmdAq#H&-qMUZt
z<pCdA_-?&@v&`evVo~OKH<a(sT$gzy(PO^hzPeIHKONso3&XQ_&C0y?V4>FIGf#v>
zC%<j2i+KNexs9HQhn(FNc`;?lzsI$1`Yf4Nd;N8o@=O2xH?Pk}mwAi5+I4;PjFn#x
zYK6-uc`b~X(PaH+a*wS3w#sJxo10GYJ!hzszOwyH*c~xtS?vqb-ColbS+>9J(>&O5
zrl;n7-vPPLh5LFs4r|=`eExIQ^YC!R87bFJFI5WIa;noaNa*I2<wh<)=2@Ief9^f$
zDR)^_-+H^y9a~du@9)VLKIL_2Z>d?3|BmBwT200Bg7<?qd9J)Uh3C_~WqVo!zPYZ*
zI=xxfx^C6;P={8NtI-{W9-323)~Qc)_`vb7xl^K7)bvBdff=pxR+G-REvY<Tnc=@W
z+W()@t#bDlLd~Vi68(c$ENDsI?KM%rL5crd^Y&TpxvI{JPcPeBU3zzr$IZoN-xTpL
z*Xvf^k+sQOwP2yEf52B=tC&E>C2B0J{AYwh0{adsluM<T=PL9xEDX<!7X0+w<%au8
zMz%Fp^NiYSgII(d0>3|a!r6SMvRR5LLs~I}skZF<%K6SG8}6*Eyz*||4XgRvwaoTC
zSQ;8?bYZE<gNC<84-VZ9{If*9z<%A-xJX|O!yUno=jzU%8@VH0C_}=^SonE}b3(0P
z#5cDe3+{*SJsrABU+mTUqk*o<I!0RxZG=7tR9YA<ShB*i<ABwg5C6Dx6BUx$Jgt(K
zZ*N%sC;w5~oXlgIy<c;``d|LfuvBl}<-ECXy({jRM|VHFpSd+?N>_KQgU?ja6%B0@
zCWI`Ne8lmHtK;g$d~eOEm(>p+KJa***4o1rp1UV#&E0G7e{Ro?ulcXoSJ)=qeZ4PF
zoA1W$+|Vl)tE^o$^mg=TiOATk;44&gP_5vS=<!(Uf5iFQ++A5VoyU%=GgKF^yL`w_
zT5R>LozkC=@<rZXwC>K^i_*UP`lij?vi4XW&-#liRXjM=x!)V4$X^bvekaYNBFR>G
zqPPEY)<*xj<U}c%<Bznn%U5lD&1-hc?_ScQLo$_~TFY22PE~FQy27q$z~rc?q`q(Q
znF1Hp3&H+@|Kb+Ub<8gk+;Z&3)pM(V&(1y7ow#+aX?E@OTGxfgMQ&#{IZT=N#LGY5
zPRZn!omtYhx1s0$9J}&#Qsfh<zQ^l+{|k}dzUJGl*z*qW9^4U)4!gA6;%dmcJAad+
zJUxS)W_WTddU-i^DhO4~=#8*QI<64(gf-`@{8Zsm!Pdi@Q{-mXMPD!v($2i{b>HXo
zJ32OJbG44fs+zCi<z5-jbnDcGDJxbg=u~9|IT&~Z^jd6qC>XuaT{3ycg1pCb?1~&d
z9DnH@dgq#ez^bEdA31zh1sNOuJ>TmyZOL4Rz|N9Ydz*WjefQ4axvg8e?8D<LQ}=)L
zc5Kf%7u?jM_^$q}sAk}#c}77l6O!2qRIl^YU77Jj^N|sU{alXe)~RfBeh1DC`}u0I
zw{_EAuN7rWzr+Onn(4$8<DWWVN=unMgP{Gg>5C&of15Sl(vSK4y5A}{zcbmu;{4~e
zTnpRZPnUl8a?PI5D|ud<b}iasHLK*a-P0)_8F*Gav~h1RefIdeR@fZ|Wtng6%$Jvn
zW*<zwpxpVP#){v|_N9HCRr`hBgZX!-ij?yn-IO&YV#n>QS7D7|SNgnCk1|QtF}6Cg
z99NlmL%WYT^_b|}c|Jx0cP}sb{OLx&p!bdz^RgEXmi5bDyVUCcnsno~d-ayHZa3~u
zUpIk!a`t})-kq^wU!6_jHU<jJPb?Bi<Cy=M^R)N*wlfo3jhHn)1Q`EkIFavkyYs2+
z_k%yf7tL+{lk%UT-Q9mh<ZbEXywdf#ku!@Y=Wg6Q%Sj}RWfiNZN~5rWPnm^O+wGP;
zo=bTDbezAov*eE0qvAPF_tsnOwO0Ny>95FFs}o1_{r@f8W|e<l|HGAQ(XUT$zP9g6
zbwK1QucT1UuZ)`Z&(2Nzcw=dY<mCYEjqEjjwPtJA*2(X3XL@90^Yfo*bpHNW`G9G5
zi>}|uF3Y=r>#fvOt+uB3dGSlW+Vf}qXAqjY)N77Y8)ulh)zkDxr`vW4`A)T&>h;*-
z^XphG`Np+Tikg$3?u~lCOn)6Wdz5+U^QF6I_07&L4QlHQ4-L5RYkknwfYz3(&Vo}>
z#aFaEGt>M_CePUZLC@~<)2DG~(&ZF>Uv+;J-#+O^(~|s$J^!xeZ?}6^bm5eiZ0WY#
zqpGeSZEl7<;PGmF+b@-S<j3<=vuO;C9&2*0-chuk7s<9N?enkMFW&pS`x<)t{7N(3
z_J|JwuC}FTbuZnW<K^+-R>&gT1dE5ZjVI2nisvbuvGft2uH@GL43qkP*fkc3t+7dd
zI8%9xlyKn3*KsNJ9rLf*{%2Ubd*9DZYfD3vm!Gq^%pu=Z*%~tO-GkU%N5w5uiylu{
z<Z~*i>OAu<!O2U#EG<vxcRi1BZa;ESVv~2hm0air`-gMCtlw(avrD<)*Me8pKJ|ZA
z@6J8DH)q-E9M<TJIV=lQv%dd6$fa$_p}@$+?qhLWDARL#oV!nMliltMoEeJoZ<)6l
zhnm0IcX-FYeRiv?QZL9h@2-D5n|IU7(`_G@FZZ6`ob@L3O>SwXO^kEMOCv$e6y5XX
zrVMO0LD|}T*Y0HPoSgGGFy8*z)ipNq+oqh>Jd(X%Z~ysY>z-f9`?dA2$rrudFJCVg
z3wtZFW&Ot7<vUG23A}r|`wEA1hIi2xP4E0kmliF!%`4L|<!m1JPUXzFH-byj;||-Y
zJWuQkP1(JzE$>ul?(H0&x1UxW-?*>k)%v*{ccq_~<ne#_?J7|DYx=5bQ?$OvtNP6k
z_#HH5#Y&fysZJ>$tyV?GL>IEuDa*9}_H&qE__EnQZNJ#^Hxu56acBKa-n+Fb>GC$;
zEqT_frp;Yd6%w+mp6{avr=GYF)1^Y2iZk>6`Rn~y+^D*x!MSJuJ%vf9Ki~NC@;+;e
zhqrY?@a>oPUstD<Hoeh_p4ail_U2g*FWH)%u3;W0m)F^!Rc<=uIg#aY%%kUx-r0H!
z4mQ_Md(bcIP-_^zJ6l%p+r;wrsF%8W4;L5RYKyM*%<oDI5E0>2vvcS;Uf<sv{^PRJ
zLb1oG_f_5aYRebDT_4$A5_GsQ_Cj~_*$vC(y)H};TPfS$c%Q9!Nm+T5L*(uoujeXM
z7cVPPnIyPea!K5y+b7DNw9n-W5pCG2m1g)>W#*ZVw~rm-CtZ5-q+jl(yv?(orkVu-
z8ov`2zs>iW61LegNX6mje8ERHC$HU6i`Z)HG3&&uCF`#z2i%F8o>bVi=^In*SD_Bx
z?~hzGEB-Sa{=3-Gu=0uR8ODbb4&NzWWAlrFeX3s!=MvrUHFmFrR?cYhIWSqtJ^$;a
zvUwWom@}`i8!)guKl0gB`}nPeqP8BNx2tEW9@*C9JR#6>j-N~AxmhuPZ2vQqZRa`S
zR)1WjHZ$mBtl`ub8q4Ntryem{c46zM2Qws|O1@V(DQ8z_yxRBZ{98=Ng?N52)oXce
z?%xxwwq17D<=ZcXwtM&Oy2Z%5pr|nYjs5=m?eA^hT)g3=lJl)Ohj;et$Y{@Xsh5{C
zHJhz_z4N|)bW`7=dFwj2><agPwQ1V5wib^Vjiv$)yShTD1qJ!tu6JKYYt;0A?Wx&c
zwW-jLVcG_r#_uNQb2PnARx$Mn6|i^o9O;~SX0B<*qFF(zDh&0P?Bz<NW3-iCD46$V
zT?r1p61?a3l7+^ng8Vh!&cAMPChOuM-L=&(Za1I)muWA${Uon?)%IVNTc_8qDeJv{
z>2}H6_P7Ag@0^E%Y?>VGYqA&D=pKAJV@;u1sO7x0ReS3GNw6)CVfNRanAA3PmX^xe
zZPFfI!85$vGkVT9RiF6#_3jRRuD+Y0i81NNlfSNWKI&8C5!p5?I)259btR!&MBl92
z)VGi;Gb2*VlZ(Zz`g{MK?^j>l{B3r6y}n#%#ExT}X;-Y0b3*g)Y+w8M(%jt@b@NVq
z`*VAX&)QRQrkY8s=1!>B-nWu7!ol!_zzX(TKC)43mkY}(@NT#6U+{6s^*>G-4^LRc
z>(xD*Xsf^F&a2#2uf9Fld*OPN+}FI$Wm{EC^F=iiR$r}VjB(%G<kDxUY;d&yR^JqL
z=5_8@p4)3V9)7#@US<sQ({mf+cb(szmunfCUD>-l!Q)9gv*WMj2`;O=nmFDbVV!Se
za=g7@s^)G^{!UAIvwJh|_*pBs`LF+Ma{R@<r}v_prtRAHHR$iXV-wmnva_PSvsX;;
zTEJ&BiK$?Rqy-z_mPeT~K4GRes>0WPDoVW}|6Hqb!;8Y~+(p07P8D6Mny&uozz6p^
zX(p__ud@udt9^O-+MVGBx8SXf;g>I6-yU?k$2j27r7arP?;JdRG_~0rJz0KyoXGG>
z^GWKJ!wI)#dfxt!n)#$uzB)fRM)cSG7w7K3Q2P~XY!ddWc5&R+uv?riA_~<ZXB51;
zy@DKAX0Wn+yUzP#`-9&*-nkU)FqA#UH`l)X^wd&5&gyU7A&k2ZC;z&n`u9J>R-vui
zW^E1*X@0d(LvMKp=ZQ&9K8c<zj&shL#<v}8UR5?T@t+pQ@%6oSn_BNAykYm3x|qBo
zSNFi>5Oaf9(cHOPwm9wJS+X!u<<q+a$^OeMM|77tU0T3im-6cN<+%nj?4G*&<~42W
zy|rk|tSLdZeywF*SA<slRK6zoceh*fe}<)z+pD*Tuxtxhoy`89VdY;fsm56@s>g17
z32m+3*-&t2G80SlZ50MfhwB#t-8qj4zBzklp8J1>nCk|+dlLihyfAv{u}eK}^^(+C
zEh`GXTnp+i{BxPf|LHsb;<ftHJ5J^-*zh7E)84Pdbz!9E#QTyj!_9xxT-Y0Zy}Gvc
z(9G@eX)}*BHC5_-RA5plP|!YjLh!juLAGjzv&1RSf3a`(toGBFa%U@jBCqv+fo5P(
zQ&6-Tr-aI!lYg(?GD>Im`FDXs<HqgtRnLW#cDyV8vgY7X74~T@YqwN0Ff=TxuHT+7
z_&8eU!md|wNu3kbj_v0Xx@*1b=H{2NJMPMtO`NI8uzc2abyf|Z$-kKI95cyyIN2&=
zTfcU|o?GUV=Z4LCw<}M3+LP7!S=Xbc<}Bx|-IaC9wEyT*hesho8(R)2IH-7pyj&8`
z-LhSF{k;!1E5shPFu7}`#ab<NEqb~3a)Ii8ueIC64o~T}nKFIK#5WQ)OHOdTS@?G=
z`;8-6rnwIvzIBoO?cpppRr#Xq+~Cbsl}Veg8GII9?er?^fFsM=ELBg1L=_K~GFJ8H
zQQkQx49}aCu68@YG$%N=ROW_f&MN0+UJj~9%NxJ{t*=#?rc(EZeV^tLBaNGEU&S7O
zk_k6jGC{%o#w7OnYX0{RUJ={;%!%tkh3xsuqBDDrGNd0>wK{t3-IOoRS+V8)A?DBS
z-nb@JvPAcHt5?v16*dBG%dZ4YousT~#Gbt3?HqSS1HHDKclG?wuBwCypDS@Ft(f(R
z|Lgp(j(h4hR9%hUmGkz>?QfxL+4}Qt-S=OzBv4K@l%Ma?PmThGe*shaS?21kv(`?S
z<iXS9Up>uc@9RA^ng`eBCP?+E*t~Ri+_L^_$Cm5wPIqN$&U6mha%R=y%h4x2r>Ji5
zWZ<iDap>p`dcMnwYtaL~>hF8q%6<i(Ug}h!a=JwRqfhVNq{X-MV)C|3&D$HYMa63S
z96c{agSICEQ#$)JndR-&rgm<B^F;RX=`g-~)=AsD?#@W)zkF}o%x!|sF|j-1GIO`O
z?0Iy%{7Uc+H68)KG!Eu3m*ixRq^en4T#>E$RcE=DbwfjIU5Rh<vRh@hvRv-ydKYT<
z`CBYGufTY`lvPsAX0lvq%;Widw^#3sl(2Yt;p!sku!$vY-<C{|TyyT)9p2D|tF*cc
zIBaf32I@BcXE1c&m+hZ&;P<QJAC_E<a@)Iv$K9!F7TY_H>AO$#&)*ty>g?94x~<z1
ze|0&9Y-w|8UFLbf&v=G?&8wIH8A4+Wk8wWTD`e3V+c3q9zw^2K?DS<Jvs)kLZoie4
ztu34I*||e#$`YZl5NpP7Y(6if_jknIEZwg^{o2$me|PAw4*z_*N@>~6=<?j?S6S=U
z#$5NgqWEnVqgVcccqN68md*D!_C4^fQ|Enn<p|pg#{2h8q8Dp0x?lcmx#m0X;h;}i
zYpU%o7wo?!9mr!J?IRWa<{Wo#Q&8V!#)x^p!d16uYumh8{PXhK2*r)%&D!7F?Y1u8
z`1afN@2-ayJ>PazRzsC-s=_Z_0Y?Kr|7Q{wMn-J2&nxd;X}GDO!2DTp(5=FR#hc@j
zX3q^;cl*jU?$V9BqO4n$Ivz~fdcq=&L9<6oMfj1a^TRzB$4ibG|2_K5@nzK68VRFQ
z_F7CkOU}*DzGzrdbp6NYkabFPc|K_5aUN=v-CcCN@p;&9Kl5CpNA8EY;u<8a=eK3Q
zTfB5j^uCGRrTdNsCi0$Yp6dLwPi6TT#yc%@I?pw1d-T~^r=R&-_v47{%90m3`qLQy
z7<%%4yj1!op(W(r?7D@Hjyx@srn=SZ`6cgoFSY+x;YvM*6EWs7{g-osJl`yB?JJj)
zZhU7PZof4Ci1Dk7`{o9g9DAI9b8r65Yg5e5ivRIVbIm<;=a7R(?yL=4?Ai`WOx0rS
zJHV^;qq+HppWRvOspoa_`+Z7m%NM+PubVL8q%>>2&dcv-OY;}VN7eg>hOJt+Fh*Cp
z{iXk^uqna23$|ZTEziz-@ujh=%4*rAIHxO>0;dlu6su^Qyl%4pN2%@;`NS!knRlpj
zu`jdy^YZI@NnPQ?rCg`vSACY7V)-@yPj>l<ol$GzeWcyaXW#PLzHQg<WzV<#Q`>$m
z;$N~yqFxp6wi}8zQ^H?q8!-09>+HzgdM&u)82`WJd;67thx*-abI*B|R%Co4rEcos
zyro}n7nRmM*){iAZm6iP$+XOnu7Z{gOsB3aSyC1>%~O4Uf&#DggZ;b(|D<Q?Bwpa}
zJor^5+4M^8j~=0gI*-Hqd!zLiyqj6VyfW6OX!APhZ*8~E_O`9o*mgP9BkXxVyt%-o
zM;1%^s~tWE%yCq4uT(y687QE2=SKScb^b47W<QQrI=K9D|KG~1-+oQ{wfTN+@GH~x
z%WB0oUb(Y<@p^OT)TDb;W>twkRNQc@c7gf94U0S&T6!3*ZdHqIQ+oHbja}8^PDO6a
zqeH<<IL_PE*@c$>ioVRd@=NNaWpDHIqIJb%&6zuQyqkU5D`!X8>p5qQy#ti*Y;Ex9
zOI1=}T(w>P(cg~WuFHhOZks(`#<2KJ!rK{bxeg^CmzrI9@%QxVz3I!#)-9OLx>9@h
zs+G?!ZOp5zne_MI@erej8H?sFyUeJtVnS+Q@TAY3_XP7cKD0mibMl)qlYhdwX;E+c
zW=bz_P5ZU-wcPFd$L@dmxn<qC+_-tSXL!Ayxh!RV)Ere;p3i?)K5>c(nCDV^s%^#}
z=i=f_8Mh-${)Vl87(~<=9==vCn<O(iAl|*+tJJpY&Q-15L05KP-0hka9`ozeq}j^_
zl_qX2oNRaLfh4Cv(Snjji^}~I<6ai9+vLyp$q(axeSVW-XS~Fx?~DCb=J+n$V6`>0
z;Fh&k*WvuErHUeX3#O=M7z9Nt6nZWAUBe;Uw$uFB9p1AyqKy~2EBM{~Q&*btBfryR
z+K;lU^L8Awf3}Nx!><0#=Da(1brrqcw}NHqxBI7V1%|lmoth)~pTRdob=zhZ%_%8)
zH?7kT1a7+NEqQ&;&nsWw1ca%7><PNG_58mW4<(=GZ*w&s-O*jBloFb(es|flIpI3O
zYnk+>7JSY(KKt(Mnv~-SF7-brnaryW&K0;S7*t~}ua%(A^2F1nY?s*Dl^#)(E<K;<
zIloL)rq1PMHIwL~-YNF$dky=1yM^almA)-|#WZV0<+Q11yaJXo+AfgzXtSW=${tNU
z-DF*()tieBes~^uCuX(#l$SnStDRP_xLvna<jcuG&BvNf^%egag8wtrii)!&@g8~E
z_Hx^OD|g;MYp05eW;k!jD%`gJ>z6;LCcm6ArPH!M&)D*>$uCQVZ2{ArSw0=m{c=`!
z$HQ=O_O1QzE-o%9o!Xtqv8Hg_8M7&Ehxi3=-@h~Yh=4jnrp<o_A(u%rDr#mm7ahCb
z_ci@tjhaQ`{bDzfu(FxCeYvXslPWC&Z5jpYgIqY~GL{xh+Skq#J<%>s;j-FJ!N!SQ
zmx?Z2Hh*$;%jMjq9*Qn1pES7?669IRKF^;hnkRXP<59Xy$LmRjZygG)uTPUTO!)C^
z^M$LML>{VoIi;#}TFy{cv+-m~mR;`m_8og>i)4@RiU+@IGrX%4WgJfTTxrhP95U66
zE6k6vflZ-V#jAZzxk63meJ+pcf&<^(^_gEgesth^>od2}O?OIp)s=bj9l9+azpr%R
zd@_HcmhGMyAJ;GX&#){;P`N9*qfO?=oPQGQY(IvcV!pBSX|(B8)zm9Hwk%O$@OdKh
z>3hWoHDeY3;~^f=5w4d1YL1^<G3j9=Q=2x^oZM&E`m0i=th8~ioS-tt@!$MOrFp_`
z4*k;q8LVcxebCnXYd`O6oYkMJPu^6`veq>{wESxAl8`PJ&3vwe#is;j2DLOAq%hC;
z@JhPYX#3scCocKv<{h$sn|E*i^r-MJ#rZL2`>yY{jOuRx;gIXE*e<Q1x8h`=6TbkX
zT>aaXDz4!drDc9KemVAK)gjg2mQyCLs0`a0?m6R*=aM_pZePx}-Hy4i(nRmFctrjL
zttS&WG~fB~c+5Dr=GvSJ7n73$YaCO47%iE}KGn>sa<lz|xx21z{<wNa$W*JB+WsO_
zHC+w~aWWdJ**z;>_<nuzl<9#D8Lv4~dfD@e-@4_kUCx~moTs8#({YlM<-59C-QO=N
z7qa>dAKS<0>r0u1oMf%%2}^j?H7mu+)?l9u2bcfu6B1{hf3@n9PA-^nUwMA}_PJRm
zZHzxdi+;^~aaTOzS7PPH?H(`hd~yHdc5PXRRMfm2`KQ13q%vOlZK={EdZ3fjYE6yv
zhnqXLX+-b8G_Sx;FL#|w_L8m#UTeawR6|2<X==<`mRX}6QPbvav-xOB<DK(cIJjmW
zT2S*@Ic8Hwd_we|0=N19BG(<yub3pGeyK@cbdAOB(rtIor}6CDcB5$4%y!w*cS=r|
zm=g8W=lshr^5bc8uFj6mC`#rKwVal`fB94iy}ABrX38sVR;OyNDcv^hbGhuXNfQq~
z*L37v{-@(vii)Id%OoF9m6QJ&!XADqKcf7fp*uHV@0OqM&WTi{^<=%wUMUt^x@uzB
zgsH4bLDMezRQIW*Dag+)mF{kHk8YjL^5r1&{jA5IeU+8IT{LEWJ*P`##R?~xRZG_`
z_EL@UUTMf8Fj>K0SVcbPfy{F=N#$OLX}W!GfBrK(+Oy4Y4ohr*?D0$M?tl1`)p~CK
zu5HeX*6z;kog~KQ$mBTj$Ye>|$4BO-H=K$pFRM*IJX3BiFZ)N)Jq1o8b3VU}z3sre
zHbHU2Ca12@x7@!~gavPJpZM7K@r6^SefyuDTdCOXzVX)nu*cc?U2?%0dMs!BWsa*>
zsHty1r&Myst7~p|=Ax~06}UL=%~oozyd|2uqV$&f5{t9WN+BmdxYxxovLEU7KKx^{
ztKH=DE04O@=bSU1`)cQ%U4i>`&xKsMd@)kL;G{;WDPud6y3o`~j1AQi6PI23xY~Wf
z7N?1Q@`cTJ&hLNfYHP4HFI)EAx2wPNLT<f1w~-_0be^iWCsT?L|LeZTQSo=bwK65I
zD|Wvb)f#hd$A|kSbMDnf?Bab{vMn$+@6Ecoj~XOB7?*xhFq@&WGS!AZ`A*%e*d)~*
zMX3P}Pc%d(xlPypo^>&&dfoG#Z?#q9UVBfu{^;b_Nn3tp*}W0!7gCnvd6_TVaMHr+
z1H0GOsPurVp|@W>eie7{%lS*E_kG_Ta>=@?G^y>pPxX~B3w{BiqJT+US8P<(6*5aN
zR(yN+XiW`!^XW=smuBtx{!1(F-Cq;4>-LpdYp+(^tN$4Hb>CH~+1+1Hg_=Kz)6#tK
z)q6tIr6B26^`E)QCmy*?jVoknlJzpGVqL#xhy3Ay=-W#I8&^wDGW))E-t?{h8?D-R
zZtHbeuqW<6!)&i$zu*3wF0TyWlMm2Xc%tK3tA^lDPqvcjJ*rplc0Fd6w0LD->{!@$
zN<2i%$aqed-QBX+Oa0z#yJjDvJ;U4JluVwpw-!U8GN<QvnQwo;Mm(@xQ{Zu#sda<-
zv)Zin=G<MGF0*+mihuqKxp*f;x^}ndQj_nmxaZ!z@HzkX%EOJXLbf0HwaP$f$-2-R
zQ@yGg)B}&EANAVIX|Uw6=N&=6H3=cN{U$!1f4M5=YP`(qXtA_wv3XNtn4}%OLJ|&q
zoRn@~n<=7m?e1(h+seS66AoOzT79E)woX4~wS1M<gj<>s&ZjdkO*<8^SeVOFUZISw
zL}!laG=&Qfw?CicE0q_f*x|<?yWgs7%YTOEyjk66J7-nrUTbV!I(2R9>bc7IT>5v&
zZb_HwOS!15oRXZo{ny29*8=X!JS{Cfas1R&^S4_)Q+L<~=It~saEcAqDAkB^wK@^N
zFj3Lvhq7-@b&zP6h~@1cnY>?TY@2M=)~}WLCaX2q>-N@T`=*w5ma2LL2KeZ)Z;d@8
z<dRlXd09^X@Y!ohsiBEGc{s0rI?E|v@yfZYzw-3C7dv9tKAW1<`-FvEYnF%ma+X$;
zErMJ#-l<E@4Sg?lzJ1xGUGonViJanW_9*^d6?8B()LOsLIQPq|l^S;)^;maIU3y8S
zNkolt-tq5JnTOS)Pw*MOviyDdtwVI#jC*}+^8&-8<5p{aiRCi-&mjDAWeShOrxgGG
zUbVeKO&hl~G|s84`WIr``gWsJY=Qf^jk}Y5?=5N#^|>ST)@bKa7PhAf4W9oQ49>7-
zytcgh_`2MsQl<8t+u9zy@}K_N?{t>ao3E>Lk7=##D-62yLS+RLpG~up!^GR~muhpq
z<-axE<D>mH--_Gv`_?WOIkX{nw(k8a-q&3NEq??&SJhR_)!=;MVfmBck9)5`o@$tO
z)_ezht4XFAmfLn6uDkU+?b*$RS;wxu&OO#7Q@CtzdS+kk%TJRnTmF4!b~vwkZBBwt
z`;jYc!q%K7NvnAeFRuP}v}{+*HrMV_t(jZZujpP_QRNxp6_TLFIY+DJ@wr(MH37N1
zx2u+}k4ygf<5Hjc;iM}m6`!ht%i6QI`+KHuHgydQ4|^qYb7iT~_Gt`<Cab^i=#-a=
zS)XgS)AZ^5^*U9hzk;(Zt~g0fW~dGdUc1}w?bi8u^X{f3ZoeC}UnnEyNqYeMSCzWp
z=9$gSPqHq5xUFn9eKL=kH2-ybFZUyh`hpwmW-d{-wOgD2;kw$|r6KxFs>c&HY*1+R
z-~Co_b?5#!SGKHCIUndK@S-s|&A#HwUcH;^mh8Bxl{@v+?Zl~7E7oxpES-8#)l=hp
zvsQ!q-x3p>(n)Il63*vsUgyb|=}B)cIafLBwfn`oYcHPNd!c)K^|ZGSvo?RdHtTo5
z<t-<7DuhI9TJUj2soKpb5f^`zs`Z9L;oNrDc^tQdXI%MY{rQvWxqr$JKOQZ=sljM#
zdFlEp+uS>M^Y`7&^;j9}I&a?YOQwy-Wy|(DIx0H%upCtQbV8=Cgk5BBc>8a&AMcAT
z0^gsxGRL@uRdun0@XBX3`?gm8ntSzZ@0XpIZeRERW_rIg|MjD}%d$RKttxdDukOqd
z5$k{IYE&t~$#P<n=UnaX?wjBGvyK}j@9k;)bnJ_`z=P|P7Vezw$vn->e&(;B<1f`;
zmap6&6#aA8@%+55@FuNt)0ZX>w_Mn?iG4YvB43E0BBMf+yQ+-m!lkd{V?47?9a5<5
z-T0qjt!>v=-Q3CBBq#0lFWJ3pw)e7XqqCd7)y@@r&B|(#HFryAOv=imYYqmty1IH<
z+O$kEP2IV9dowquy4}q0rhd^k)25u&zi0nJYkh4}iPP%x$lLqhzTG$Lb#rQbx5*l>
zQx?m+PSqQBC|UPR?hUO|dDy$a$c?+_=iEQVjbB&0vp;IGG+&n(y(W6z-M~Q4Xs^HP
z_RYVteC_f|?yn*Xk2jt2U`d!fQGVUN(jQw+%7uAsoL6!G@(=B;5g$9|8F(&>yMDiV
zc82OH)ortTt)>V?uhtG<$W}l7-RIjIe|}s#pZ||{Zp!@hnZ}AwHsybk)tIMxxb+#6
znbv`u-|yYo_fO{XuPIxzb3UX#vN!lH()^_7kB97*Tc1Sco;s|kWhFR&ZMby1XWnAv
z-H#u;Ph(|Y&mR`{E91+xDI2CVC<fk{FFgP9WA8T)CP}Zmme}BSIC0*b7cV7CM7eve
z_T9c>-{qwh`{cBfBLj!fisKdM|NhA`?U41ic=tT@xPGDcng0y81^K4LM&^n>Q7}F>
z(=({TjYIo*=9|L%8)sZi??0*XgsFP*wCDMSbAtErW`sPSxN^7M9bfIlX^gH`3s`tQ
zPAF#a6sXVp(W}#UXM;dz_1x&%eF?#yR~zqFsin<jzZ?`)tFppOgyn?2)<ky8`4db{
zWK6eyTaYDZ;xMPoN^0M<mX<{^iVB?mNe+_*ZxvbBeocxF;QIclMpO2Px<t>V%PXUM
zqYGE-o;)6KnrmHJZw8l_cW0@ijmil@tw05jj)_+8ANfVAIfWj3#K@%AyuXl8`a0<5
zhbNcLsuh`^c#)OVbU)~6^yC-+1SV`P@@JXS<X-cLrEcaG`!6@ulT8?u49|&M?e#cj
zw{`E;GdgolGZ;^P{Jz<jTkh1(I|8>;xfd<=;EF!w>A&*M@1P?843*>et=`OMso_^-
ze3rv+^=wb}Bhe6zwwdcMZp>bo)HIP#bIXL=3KBLB@4wqVQ&xHX;va9uwGR&~>ut=|
zxz`$&oMpMg;OwfP={<Am1V?%H=feHHk-E;?Jf1pu8)`2JxEufJ!qNpRkE(b%`X5?k
zc*CXq{p3Xt!xDSm+3wr2ujolni1Jfc*KF?R%{HllAC><zWVuiLRhiGbdDHj*3^Lay
zE#zL=>ln7_@*2AdhyAO%xs+7aFf0gu_ray}Q;AcIctZH{{hyzE>8;(`yJ}-@-GU1x
zuM6&2CGOFhnzdv}rC*fxv`MqdGp?=k<Tm`X>E@G}eJU=J2eV~lH;YViUQqG&%TmoU
z<y?uQrrXUP-@gCg>Z&z&LUK2J*5ZiOKFQB4WNR?lc%tB=8qaGV3T~I#zmhdz_~Ywa
zSTpx&`csc}6Tiy+ExNP2J?(mU*gGq)D6?#-RT`|v4hFg}dUfFYDu)2A6U{#xv{>VB
z+jud{_A562G1+vulPzgY@x!p(?Lt#m9&DX|s=#*hOgsPj?Z02l2v}n+Q~964UMt<E
z&+X3Y)NHQI38(Hlcd9xj25Jegxj*e-w$I<U*XGT0AMYhUt|?0NYoFU;`}O<9)$abg
z)<*BuSecn&cJGPTx`o=@LBdh3il)8_i~mLKJsZOK;o;*GMN&Uk|C)MGFE^j%qQaZm
zrIyF|mzJGx`LSjDwflD%1g}r4UA5|7){5Ng^|hy7hL<E=J(;yK#pM19F&Pi9NgGuf
znnW3#`rMV~s!VQkF3w#Zk(T@IS*_!N+taN!I82gh@85rEabVH1HJ&R?Ohg)w?s$;v
zwlbzt*URwb0u`nZAI8pJMQ?6VsmREV*BgE@xnG;M>mGA^>|gP(%aT^k;kWczzDmaH
zK+r)ifBp;w_rCuO`B&!F8p{iHvt3r)_|LUb{YJRCkno%*%?~{O7&=E!I`MRg$%I?t
z9`Dry9(gtxc}$)tEIG4z4sZC=ADu}PZX77t_@5zQ@#4L@3JgUr_g`Kbv3~*A*Py?y
z0ZXq%IA!U+Yd$ToDJ4bj(u~Ji?E83LT|YQ;Vcf;*Wv>kTZIX|loAxKLOs#Y3bl%I;
z>ZaabKIxWhP<P!`4Q<(Fc541d8gH3=`?@Hj`Gon0V;-MBu1ol^Cn5KBv`x^o6Rmwp
zoB|aMe04VcH(s>&oeD5bscC=g6JmCh?dlp8s~4<6o(Ag_1Z5hSPtNJPeBVkgyuxk5
zuj38Mc3<q4C9k$L?OI{~Bix&pF(hcomZmLFV;_GzApd(yI-kdkvQ0hd%UlJX-Ys(b
z>SNA#`PJKdzaERV=4b0>&GY}a{A22>v<DrL3Nt;_IsMWFOpa+sC>OonHLtqaJE`r#
zq@A~x>BL&@EID7ez9Qdx@9xsAMVv{S_Le>L`zySJWob=gQ>%m5g5pnp$xiY2woGX4
zEPUDMdb{AKxRu}6L_LPhC+4XhSmZRRW=Rs`gaY>b$qS~xdwBlywK+#T3)Y_7bERG)
z`rV_uzgMS5NID-onjhS1)35SR(WB|np<+gbk7|xjdnTC7@3`N0`OPoK^E#sT2IivI
z>z9YS-PyF`%GoE`{pDKjWuXgiuif`OJR(cuTKhB!ACr{Do!n16EEPHoYO=Pc7upET
zNb%LVmvBAF{LDk2H6cHrzhHkp%lpf(wyo9E?iNPhiJbH(OjX;*N8;`L{@#e_?xlP5
zI(yUC8GGvQX#DNK_vEL!{oL49;(FPOb2Yg0mfm#k;NvqeV(~nv#{akUoa(Nb6DQ7A
zu6bJgRcY-X-N(mN9oYT3er&!Sxq5Yw)?}B{y_zyHTvJ!9mQd+AR{h-LNMVDiaq{}=
zuZwv1XsRVL^h-_N6KuFeKV;2z&!|=Ak^JkHddXzoYQ7^X7gz6hQ>nyvIghaOZB9-*
zcFS)odS2h0U=V&>;_nOP6}wN}PTsCNcW2?2YvDVXELJ|69<8Usa)|Niud+8bnfI9=
zbiHqiP5vxl`Q>P6#oS*BD(r!Fi$#JjX8X<+c^j)W+m|(9Y4&WMm0o#E)4Zl_SpP~j
zvV)1uMEz*Kl6l(u1$^%FmqzX|QB}TheE;+Pr@G%ohi$vJT--nCZQT8pE3_(GbyZXr
zIvlz!q2&2!;rF+z4t+d*tmANzw7c8EpIzZccL=}IJ1!UfK1$bo!&T3Mg-e`*%b2f~
z=rB|=Kk+<asLpU~omkkV(>WY7_OTi6xP2wx)9w_1N%6wfz2|0@@>~%K-F9g{ug#RD
zsxlpFK|&Ud-x)ZMt^Ra-PoG0<yZ^JV>1|i*&aHeHZ+$u|XUV0&wb!nP$81@dRhaG1
zb|pdNxTdGa{PXuG*e(t(c+K6cEa7>{@@-?V>2J3iyxU{jpJ_*@2j%_}-+wqSls6=3
z<@zVvZZ8lERTA6MIpsO?fu2{|t5tT}wwcWT%71S0ysr|iZYONM9kYEo`AW9>k9(Jw
zuD!W++pN_O!iD;rFMIJ>oD550wtX`3#*?WZewj|a*LicURRYgnR+UE|tem3m=l@(0
zyYWARQ0}w0rPs5ptds6%Uk(abqCG>2-{ct|o8P+akIrRsxs>QT9ajG$QqQ($l0^M3
z^A8^v+g{$fw&Y6Am4HBttKkZdPh@@&eAjoN^1br8(o>;4<vc4DFVEd}{@OCDHp$sl
zcW+nElukSL_1@HwQu751BvKDm2!yZ;9^a`T-@MxQJ-6VdXwP<u^SbBCUk06=ILUIp
z)z-Pzcg$i>hnJTA4m~xutL^x@b`dYx*1Eie2TUA4CnWD%?VFv=+!oHb{lqc%eU;Z+
zr>qsLvAn4FtvGw>ZNJ;D+S^_qUaPw+YnilH(375`fH#hBRD3EvU2eH`<o)f&Nj42X
zlQvp=w0e~vc*YwV`*+*Dow+6LXKh3GZjy9$l2#NFb9_?W+}wQUx$W212-QbSF8w@w
z#Xf5-mhQNvIQ_%Z(24PFn_@4fuGzNdcc+ug@<NkW1<eyUl+vopZ1>%&thOm<-P$8~
zg17&2R%ERVo0#{yeZ0Q{?)|tU_BwavY3JExD@79>ms(7_DB#|-G+FYD|M|JH*{LGu
zLYvucyq)uD6-zM3jgAy$S<C15R-bw)Z=RMdza}cz@zebkuQNR?j~mRKQ}6Tr9N!#1
zqacM2Q`RRO%jQU?%__LWxcorR@2l%yN^Sf4zF_g6xWA71(Oir+QxEDbmwREzWcd8p
z#P~wXEs`Qxc@1CAxA#WgZ(lIkkSX{`dDiQ#{+T;22Q9JmFWY?C%Gr0Pzs94<6J|1>
z`JwWsxA>iM#-(FNzcSUB**%{tm%DbJPR7B{$Df9aFWw&(J^$sTTV;87)@=^B`Mz6o
z$?{dHd5gL>%&|L^{&~VW8$QvuM|!<KeEqZSqDNBF>Ia7{zMDqPeps>Y_g(h6rQ2U#
zeyte&Y}dAf*I3@#PIzth{-c@R*HEo(mt&;4IT?Mgh<pqYs#H0&<d93|)b=B}HZ8F)
z9V`WV|7_BldUT(}v^(t$S?o*Wc>DL5g$I}2U0(L(d-2Wfeb?50wP!EbewBS@#X?uD
z2OI6Li1s)NN?ewdJk2S3EP5JW@zX0jLFt{3CS?1ht+;>XxlindcV4xBT|K0?URO2Q
z`pV12=)<Xym2ajpv&=WVS9;2RMU38)2?sRJ2i5z97CeY?uo9U(H&ptosH#@h-6(-+
z*N?0@(Bfx(gmZ3T_U)90thq%t1=6q2zYf10VQOGFr}~1k*|d-w!F)R|pY}HpO3;?w
zEPNyV{r5|G+K28vy1%XKu{U3{eQQLn#g_FQr=D&5rXIWPcJ#8@=~wUHTD_jHPIBuK
zP1mE$Et4j)I{Ex(cxzO+vcl$$V=U{VSIje?uRc6qKWp>7KZ;ek|87j2Wc5kn=k>sE
zttU-QOmu_HPiV!cDl{!xC_HBlzn#ykvjvKMMh@KbW?IVI?khbjtQPmde)^e!_W}Es
z?VGzx+i!c-{Ht%<x1A_i{9)Z4p|4SQ=AW3AX1Vk9@uz~HtY7UvRAGC1v$Vx6pRc!>
zpN3`R_H-|G^Pg(EtMDq{{IB^L7vIaxdv{yy)2;QNoU&d_Rn&HTv}KjQ+O^4jTnv5g
zQlYh}N@9Jb2V+kt^xM8T@x<)qOtY_xuD`kx_E6LN$(jW-nhYG2l?Ci$-OVKgXI#$x
z^FmT-vc-%iVIeEl#l}o!S!TOT&H6!qPQ}MxnHzTN>lM!VbWHoQj`!aa-<Ew{zohK$
z%`LL&1qP=>x8|un<6iOfN6FHlNi5}oLjDJPUayL-_s`8=8@fJZnsv{-#iw?j7yYs1
zsD<HnE$yvePEEQSqNP`wb4Mn(?%D-Kts^J3CM!>9e8e{IgMZ^CO`g7+>dOr0PgUCc
zW9o8t&tn<W9)_Hi)^dvS*3$IpQ#e%jW6q(v{|xV}mV^bh3T83ZvIakyzW1KV?p+}|
z3JWth7*q{*Fg$3sf6M;l)lu##jvP-iIm<5ZxYc|=!clJa9&1K{nx^XVlae2Q`B*e@
zEPS_b?VoMm?AJO>&)$)^|6EH}XJ8a_rt`ifRo+aFYRXfd@jO4@V4`%pwdlm1Uoz>h
zx9Uxu`|sg7yQu=NuD{atYqt$KIp2uKr>42*N3X^99Vexe<t+ZNIwV~&*?4lruZAfL
zR5_PNaXok-z?Q<_;C}n@bKm6c_kX)HudDnWrZ&N;q|xi`&&z99?z`;y&)Pj;)h*Yl
zL02zd(OteW#c9&5%ab_#cWO;pWyF)&&bK#0QE=z={|p=C`>mYo(>Bl9aBsrz^$W_M
zEN%aG>{s2=BO=YdVW&M}y(?5TR(Ww|%rD=uFs^Xfx9@=_JqMj+H(seN_B_uuUE;P}
z%lu1SzoPa|-!^~6%@-T)Y^(cJ(lu*V^~71-T@8~yX{q-*xg6j;<zO`BSn{r(^Lod*
z?6%}iDO~vG+`q*l9}jm2ukx<(_}0YK5O6MepUSx<C7A-p+Eq#xUAy_)w>R(dTHX~a
zLc|{Yp0tFq?~~_y+vRUcRo6U_W%B#pc;HX=Aw%iM0ryW_t^T#}WvO|XmFtc*CziQ}
zKI^rAG;N9I3a#2Dnyw6dM>Dh?<(C&TT=}QAd6vhPDXf0azuD{yx&Dy*%eH+?!L2sm
z*GUCkx}EuQ4zFq2f|cbzd$&ys3R2<VRPT`4S$;g@z0}X^|0YacEG}`Zptx@zGuv(#
zqZsQAjhpy;{VyqX9_#8YHesK%f2#dt`*qW!?m6bn?O@Kgnh<r!taz{RwYghMb5C{W
zTCF`HQ)^W&^Ht==Bel>Ye#LBl4pMJEMmrX)a#?PtW7vOf=cV?5gW^-2u3ZVblaaA@
zYjUe`;)}{Hhk_MS?d7NV{5hWYtxrB%``ATKh4XHg=T!Av(abOK&)~a%E~NU)g~!##
zuk4dVm-5!>9``J?KOHWra$rKGAZz6D<Q|U=1(pl@6nUC0U!RaEeU`<i&XF<m=Z1B^
z_eB-9-7)1TJeo4E#;L!j)6gibaIL;iIbX`woQaoLe*61+)^VLBqUx4w&QJOzw!hj-
zeWH@eisN~oC)HFK&&kg4*=@6+!SeBpZ;OT8Jibhh5L&`2FyTbSDRH($MwJGIXMA<V
z-!2Nry2WS-&)c)@>ss9{Zbp1x*1U2L(5toDdO0rh+wbyIrgl?88iQYGE}1fE>f%Wr
z0St2g8C1SsoS<K^W(T|Eho8j@*H<idRja&q?cA$l=kE1&*Y0{SOZ%eCRoAFVS6wPO
zeVpf{dHS45NO+xn+l_%Ii@m0KPMt{Z)K4A{*d#x!UAHpi()NwtWnHUoFHirF-Lf`K
zy@!>_KBoTnjO5fdr&k3w*DZcsyIy3*+3{TFyn?pVfve8HvyMGIVE43WMdp<Ufp2O}
zI@LGW+4$5RxUiB-XV=xQh99kKHf}#RGg{O_J^69Wi(mJCB`kiW+xBpeF1JpemP)(0
z&z4rD2Le3IonA+Nd%p2G*HW0BFTEqy@3@3*Er;x_O$FLF6?O!O&n@6<PJXNZfvYM_
z&~uk~#H*gG8YY(*x&QekD#ks07slveDD@;>fcwh6x4)TI&1UZT)90@BOYg$zZ($Q-
zQY(L4&9FARn!Mv~-nH&p-D#a!2A5aQ;GM*E=&Jpc4wcHEOeHDXSDGZ<oiklFB3bff
zqsN*z%XYC<wh6y>DGBdV%bNB!G+Tegw&l`=Q~R`<RhSG!KgvsW-ETY6bK1CZ`C*k=
zJDra_+ScyP_x$tIsHD$@D{5!A?wYi2;SsZNo`!YB?E4Q2STxSxw)MAZqq{rf+?~9i
z-)V_G;^lGP*st-geyia&yEnyU2CJGAQuBUa($sdCs;V0KXn&JP^|^_Kbqrk{qE`ee
zUn%@Gp4Id?<Zxi{{P<f_#N}p3zy5yh@f}q?!_*a}tJ<d68_nbV-s&L}<iX{W!Ys@j
zK6{<b#z?mbo~s-T=SxMW9^UG8$)}X@^ulFV5B)NGT=r{~siLmgBd2*@>}+S4+yxby
zncv<Ibw2nmuXt&n^h|Z@pSfEja`Fud{5YQOe;X}&yJ^zKSpm0RubT2qXv^-ySJhcW
z%NaW5+3l_DE;l=ETy<xHj7*XG=X=}t?3|b2yv=_tTind;7x({&iawhvdobHovt;2>
zl}QE7FHg1#Fc{zd{qpSIsdI!)4D|NeoGbZf8ad5{VY-#CvDxA3@Z!tX+u!%a>s+@A
zV=R0os`~s1^9eULAC9@&kA%9N^4_)`zc;fWV|n7eUl;72zxe$-ZtbaGxo5+^X0LvB
zxWC?%OYkYnO!X-Y)fs-WA9Ig9aW$&XUT3B@^KR5Bk;M=C=f8BhEuVThEIPLCZ(w=u
zmCHI?oNc<_o%kjycvMMU-GINMS6ub&{=DRda{eE4Pkg_%b@MhBmK%Xfx95jyPLter
zb;n=j*LIVf<E~9pnLP3BTf--^e)+jinl}edu#~B@{`2AhQ!^)H`&!;SGflnlUo4s*
zu3o$KBK26slu1*C95nwes1)KUmzSD!N@J0G^mK>*GYOAB-kKb{rq^PYfuDBStWDdd
zeGd<=PI@85q4UwZm3P;I$w#dCGgKr6-180xM918SVsq56+R0P&ea#)o6kj)6i_d4(
zHOkrE?T@KHtS9nr|K>}d`$DfocX?!L22Q)SGhp)D<AqWiZy!(Vl6-PmQCg_7B00-h
zUTo$&mPCWa>#~>0YV9le^m0qt@7-2wm#t}CvPrC+Yfc19=j7HyOUnN<WL($S`|x_<
zo%*{xbLM`WP`m0q&zrY*C7IXCFFTyIGp}pa^E}^Gjl64?Pg$XRXrZQ0&B1qu%kS8#
zw|u|tVaxwHE52OWQuF%hFKbu7op#nYWA*k$y|3D|#cp-GGIvHQ6j)fyFF9to#C+Dn
zq(=@CU;e%pbf4H@GGA)>>~|;qv-iekMQ<`rTAs^zB;3rS_4xitjt4xRPP`*2``Bkk
zNz+Xx2Zy(oS@%U$wz(=_GAg_>>-o}O^Nz<~J)LSBTN{-l`d%@kbaQL9pOE!RmGdX1
zUYVJP8-DnnDbpD|Q<mNLuh!ZXdk?Y}@Z0YECAD2_-mEV_wSpu!1cy(%d}Lmhs-q`I
zK!1PX$um!CvgdsKy)#M4Q+6ibgZ2GuW8eI!;gpPMl>ArySM$^S<@=@tT)b5tczfN=
zRRK}C&t9IIlX|$%=E8!c>Jtag7vI{jUpVI5$-F)Z^NxdZmZr^Vx7!`gKHmRumg2&7
z(O0idPoKR#AUMD)Q<__G%g5uUtS<e(Uo^59$rvWxJYMqr`BpQ*=p|Kb$1i_hY3=uN
z@qP2}D{Z5~nN4%9Yz%xQbk{c9?n^<C(WI#i3)~$4-8y@B`Hh6ui$?b>@82n1$Wrhv
zY7ui)c<J<2w>Q`4N$<T}c=tQs)AjCB(XFx97M9j+?z9P-va+ax(O%G)E!k&>>uKHH
zC#~6d>R!ek{3sW)=~{U2f!-o_d&cN!;dOg!U#i`>E5_HEU$-nO^IpRBJ8Q3T9lvmy
zXGKA%^9rG9Cs{ea*6!>K+nai{+M=Q5TlVD1=kLhmx|y?^7tLgP^^@`KywcafFDIwJ
zliq&G+}&(>*!QYmb9cP@cGh@0_g3e3R=W;u`RuhfbSZbFLlFClEv*_$WX{cr{qTOP
zn@jJ?__|7kN*=DK;XKJZOL`03Pd|@&@%nOpNx^;H=sD|Kjf6C(wM<&I?8-U~4G*>u
zPbM}8_BW-Uw~PEx`nzt{t@~ap|1&5R_QzGMe$TM~+%%i>j{CGGzyHs$I<Eb#?u@_L
zw|{PlxtV>jtaDem%npm^qSevMf`WWB>M|cG2)Qk+e>wNdC6%&tq4nnKyWek(w@5uR
z?K@L?%!;SaV|L$|ANf1z@Rc+@(Y+???@hCh%(vLJ>UzYA=#_Gd!lrBzWlEmXtmt*B
z^s36g=JU2Y<TDumGu+<s`Obs+&)3H6Q~S5BtV8$BWQW|@>!&i+`qYceUvY2u)flth
z*~Kxt3NJG>`pp)dcwN&q;iTHygMW^DswAvp+ttqUZ|2SD1IwO@-YM0x=b8JukNd0V
z%`3cni(~$l7r)b<dNo|?_SCbx8y}xrd^Mmjupw;clf{!b1FiPDAAY;A*(J;DWn=Fj
zvqj;(Chr?O>@4r?I{e4TZvC(27tZa=>E>Nr7`J*??#}fk+ist2Jnp%CP2tzJi6K*F
zc&v0cnecqel#cU>$Lv@BXW%$ko^niZk?ZU8$}6mkKE0@Yxh5vg<9M2g`OJHJ51IR(
zne;hVlOxc}y~gvi<<Gz8_SsKvd9q@He({|@`>QnHZ`$26pDX#z-8rjIzquZ6CYb59
zXU=|M2Eo4;LhcpkkL_YV@SlNW=JU7f7(!(^R`nl|sTH!-*z@P*pWWR&ttRcaoF<>N
z37BN<{^jSS8i(_ZR`0)+<wexamyop#*VysMP`JVTwexoCRWnxx_n2MGnBW*VrBn9t
zi4TUWg`$++Jz-mV`OJYV$E}RlukQFY?a^H4rJ7-uS_+aK%(CbI$%=$}o?fdt>&X=T
z6Bq7Ap9+Wy`Q+52p_y(|+<05a&+gU?het|}B}JQy>dwi0HZl72Vag6kFM&xLmd<Ep
z?ChC1`M11Pc6z~0t(z7<gN|)AN|-$Hq4U)iO~x(%88ibzoI+B*Db@XFSl|5n#f|)k
z4H^dQQ<Jh?!g!nZ&vn>;(!2a}#gZld0=0+FB-`&>zGcVUZk2Ye8n>^zJ)Uy)_Gavu
zH0f2)mQP^;t#f#OFw}4PesSZWUnN@egT3z<%YI++b)R(k^iq|=z@P>eewMx?<?=?x
zi}H2djf%f6NL7uVnz3qy;<oDnkF$lsxdc+`lh!>}ejapS=V7BaRv{DjRCNnZs`<9+
z@K@dX+f$@1qT{%VOG0eU|45Q5sMB2Dbiw)T3W4ay3alIk$?xPMqgt6&_a3v``O>qZ
zTT4)ZA^p(8wR;P8zUBH}w(s+%xcAees!Bo@J$9S6GjzimRoA5=hmI7+%RS-Y3tX$0
zv-nDikVfG1o!=Jd)+vQpNeDmpmz{p)?$Z4ct5@8fF4`JwGwoI8{7j#ai#q-JD=uB)
zQuE|G<&!WsQRmUrx^Mqv`d@xstovie?lTv7Ry^PT)@Mo6R>fnpRAcsbYS}$GpLwQ#
zf?x59BO9_+n!+WHzjyT$onmILKDY8e!>`%5)R)y}+?%b|ckzccccXZ8@yd9Cx56*=
zT9yT|ER4CW<ipZG<2vJy$!)hIV?A`IFrHza>u&w|b?ooPICtG9hQh`__g0_HJ+!jD
zxzsE$V4cE}H9J=LcPKdi{Oft$)>G$B)-H$sD+%-0%1zR<d$FwR<-8n|u<comTK>yI
zj~+X&!zyHV#8dM4k=HNO9-QzxS#n0U#(?wNx=<yn*BoUg#_@5Nv;|(Yn61m?RB+(>
zx5nl9`wst#UiU*26?gG`5{vyccdwz#ji%+^W{0+aGkYA`ymGdA(a!%2O0ymu?ptdS
z{*2Xy|5N|jd$pTN?mcepF}YQGe4hLIWoBPjt_#UX>??g4HEp43S#@nep}*p@OTkBu
z8#4D5CH&jxXPtT4aYpmRCG}e>ioUG#IkI?b-O}~3!tYdC&g>N8RQR{7!20ol8CRO4
z*X=aFdL>zDVQ8k*hFw7$9?1r;T_ZF_Go|OF?MAia?YHVmS@-UGu<AMQnYkN3FM6vh
zcv-J}h3?hu8RfZ6`%Z@}=hin_w!DLjq0;3H|0&1!4TW>1(%+Ov78&;M;`}-5-nX-<
zhZ5xDZ|!`4#dhnCZ;RL7e)rx!cZVU@;+<xbE{C3wkcs`z@N^d+$Dzg(6H1<ZWUZKb
zfNRH{2LAIh`>j^SuD`kL#>?Za##a8j^ka)m#C#72NpFv=HP^PZU|Pq=e?rE~fs<2V
zvatH&ZOyZ8?~$`)<2ZkJd5zuUx3O;+`)mHJdb013fT&-{(k(tK?b$qL*zP`YX=fh$
zoe4tw?QgBQ_G(t6dh+8_Q?1{f_uZj$y*kxf@sxXHkjx~9C=Vr(!Uxa4eq6fRVyCmj
z6~3aokL==1?o8pE6<tuiUez|^SEaQ7s^15@s^7kw*4FjpYw)%Qo1Pj4tPBV`5yHr3
zU(X!V_xus}jceYOF$uEWY`i~`f6n?fy<K(7AIpgJaOG{rpI4pTw?1s|+PJ67wp|a8
zcUZF2$LxhyNRv@fipvbyPC?7dtE-F4R$a7I`TFI<#|6JRaza*rFXef8r+>Z{*VcDu
zH<m42x5e5$di!~!B~=;TEkY`CULQ|1IKHtfc@^KP>~l|2@=Eb-`Cc9IS0Tr&Pu=%Q
zyZv(AZM(y+kvXfg9)_IJZ*@<*qa_d^q{hhpmEALWZtsh^%4cfY47WdbKPsLkWN`oH
zE91!g@Y#Qlor+vm+2u8>-PL)u)7}*-q5>?b#YTn-{FejV4_uVFr}RROeg65mg5?Da
zOhOzcxiz(`Uh}G^R=3`I)F2!p!@FdOLdzuKMy2{a8Py>@KHnL#j_Wko?OAbJWui#`
zr773G8keqi-8b>R@71OamjpST8k!h8`hK2&z3k#iE9bS+9`-)J{)McF6*S;6s!IR1
zXt%uo_kUk@T!}Qhx^RiTU(@IJdp>0HF5l5Pzwl(^Bz51*eIIV@sNg$#(_g`yZFy*U
z(UZom)O_D&f9JG?RX1Jz6YuH<E#0-z@7Ins%{Eg!ziVg}-#O#Aa*m$l(bJ~Kt28a!
zSSEg67W83t63ZmHw;J2FF8a?<xOQ>fz7Kb84?ccvAMmkZlDeQx=ZQCN93M<G_o!X%
z3l4Fgt`(aqQfQYXU0L_#_xj8wQ8%~RR{NfBcDS^beTt`ws!HW#d!?EmtKv7iZpqvl
zk(bwUdWrn=t$ER9_nz*)p{=`Z$==d^3$-8J67cRV^jBJ^q^ilXV%|fAzh5RD(V3n&
z*K5NQ$DdF5rMSK+@-R#?e!e$nYwmq(w~IN8?tLq{5_2ba*TLxgl}<NS<YqjXr>OF$
zlUY;I;oH@f@7Ac5Y2H$lId;^4-m#|5LKYIo{vDed{b=!utG^$etoPkDvv=EW`-k>|
z2UVZ!n`{*Ne1h_kM%xXg?>LWIBu86qnpRNvSwp2d>{G!{-8`2pqqVPtMf<};FPZFK
zYHO-BKVyQ!13AkH>pmLF?`xMaJG*5|?rPb>=#QK;r!26(bg%6{gLD4v>$0moW*_Cv
znja7=YikvwRj$&>eCH;U;}3&sw{@{y=bASb9IPq)`7boA%W0yB?~Tu^**BDjE<2cW
z+dAq$gMZTPvZ;ZZr%Mmm|47qv3UTy4GQ)XNb*07eDZw8i)plfFOB2%gJyYVik5IkT
zt`DAq${~6CB3HileYDhiXDDx|-Cb>$6-(O>dfit4Ty$W9|BU13T6))2t7Xm)e8PCp
z<j^OB{;4d@uiZ@VhO94qwszXt(=l0-nr?UL{5r`XGiAn~`}Myw*dGLKnY(4hs=Ma@
zxQ_1Z+|FL*`1RA{z3;@fttdP9+IQB%Yay@uRXKzES~(P+vz#zu|MGXMKKC>WIlX&A
zk560IY+E&7c3MTuqCDQ>lDB_PZa3#HnVzP<?5X>oBRd|aZn)9NmFuQd$??(f^Sb@T
zRXp~o>MpteEDnDESG3A)rgcop^XvIpTmQ=JU0!_GYdv4++p1$zq*7KsU8$cSlvF&k
zFV6qvD)%GbCc9=Yy5}I0-%~fW?7>Stu4$IHXG(l~8UCs5(d{iSZcho{b!yhC1<fz5
z-vu?O_&5qE%kC+=u0QWmdT(T#yG_5mb69uv^4P{Z)&CieZw)=alFhvB%HGRc<~}XW
z&$jiN)}6&GvL&UdRl;ax)6{YS>tpNpbKmPby&>h&&lhZkp>kH&XYEeO-9J~0e@XeQ
zYnB)Hyn3-W?6z}gT#K#-=bzVd;l}UYIlf(J(R9mX-OTrWA@Qtjo>R0_0$u<8^-W#6
za(U=o@z>4kMAt21j@v0Pqr+(ZmA=cXs#LFjJ<ykKWtA!Y{MtJ2OwnBe@AzuAi@xao
zVN{a7_)Dz7uD>o(>%8vP`rhT+n{tO^Rj{6EYTCR7NB8Qg2+WH=UFj9Sv*;d!$Mv9X
z4?aaJ9eaECxc=_XAJ@r$^!;<mN62}K&*S5DmFv#jGml+#;?&iGHCa<vK4DbwVpMR?
zSLk@#_<70S87-T$)YWqYw@<fPALZkDSkL`L*TeXaRd>GUKD@McckN5&klT}1oi7UY
zdyu)QUhFDUV85r}`y5NInCeM(cc;fi9WFLIUZ4Bd-S};+_=9sQ+phC0my69lW%u^g
z-|kA;>*0l4?!?{|)l7(#Sx{o^>$We&MlNHaS0R^Y%f~1#ju{~bc7O114dN(0^{Zy*
zv-L6V9cvn0x1HH*`L{A`ZN9(xvzssP9euU-+WE-5+tRJKPFcIY4AnV%=90Ycw_CYd
zUjt5sEa2|)s(dDr@N(W%)~S6{UoiZsc@eT@lJl))-ley5{!J0x^XgZ4$m}AsbJy>d
zS?^MbarM2r>)1RS2iC@yikXv7x@gS!kn9o}ai=Vy)Aae@9mh}4Z`GQ3+9S+y=ZC);
za@P|>3o1WG`kXfknzVI(A^WE@<!|rrf4g$l%g{p>2V`D*XC#N}pZ=G5{xa|B=*z3$
z>Ym!WyK|{`%H^v8lV4xhk+AJ|*5?BUn{V*Q&pLj@Y-OdiyKr8u`Elom4}I4KK2m13
z{BUpY`~M95{;SV*)@Hx`a=Xm$>7A{==jTOd+}ppgda<i)ZpD{N%|R{OmMZ$Lybu&J
zqxnRk+})a`*?)CsaW8FpS>0UQcf4M%^6S$10(bT<ajO4X`TWDWts%3b!s_{2R?RTG
z!tbNr<zTz-hWz^c&)J)!3OW*OTO1m!-qcq;`R=c5Z7lmoBWUfD)&-LSp2#@JIZFPt
z|M}c+@x#><4t}jjt*m+JGGWOMn^_B2#eS6x@N$~A$nzbCfqy;w&gT14Wgpo3S=QAU
z?mxF6``lR%=|^p0pDxutJg`7m%R`92UjFv>V_Ub+lruX1uIs{?b&iR5SCrq4o`1Uf
zDyPtr1v-6^pMTZwf4yj}(z|WTE%Kt%S~8t;+{+edO^UJ!5$gNkUw7m9x%B<H1|GX@
zW^%4jW4ArSe{4aPz0$R1SC;5Ty4zh8{`S`1PWfDW1ebnBar5JVb4F~7ifpntzf4hm
zD)cKzd@AFS)hFyX*cDE;vTkR0NU@YZZ^abpxmJB{fzZM;94s^bY*;CwRR8zI%VicP
zRI2iShiz<sK1+6I^z2=MVYgq;)>>A}I(;*XTFu1g+vj9gOR5Ck%Fur8eX1>p^WLhn
z<?g>$J3OB9JAIXO+pJ?trfj>oG9Xy<yCavy3<D;Qx2t8hi*)Y3lpem~sAhCzjA-!o
z-t=4l8T_~1vfirp;9hxF@8xp;&l9%q)G%2YA{63i+kSihfs5a3T)Ehp!ws*m<(Ol!
zm{B=p*0epjtJ5C6s^;0zqIKIfrE1>V3<kElr4OPV4HDzdMb1<5n|f=pdX<w2KjY4`
zlB!!GF9_F5^(=pJ^mG3EXtS5M%G*Rf>uIi9l|D!FPpNW~$i<cwq6=EqYAG<P+dW=v
zcZnxe+3v1g%*vn-dB>c#y<O^W7&dL{@+pVHmff`p)%NfgS#YqE`HT9qnmHK)p={Ho
zFJJczWztwzRxgk{?ew<3S*0d{7gw$7S{hcheC~4ZE+^T4*(yc0I~N|{|IZNc*YwPP
zh7~ea<pw7Oq;sdSG`?)8+j?Oi<4z&J1ht(zT&u#KKm5)b%AD%*Au&}${Rzt|`@dPs
zvpyc1CDYSa&HQ!Uk~?NeMa}tv0Y@K4T@u(X<0LfUmNciL`}6k-^WU#t^X+<8(MPe*
z?CQ66TJM(ZDa!3?SXW~<t8LPT$C(b#UvH~j;If9LQ-yP)oT<XRvYr*OK6@Ds%FVN@
zD>-yl>F#Fl=)I-Y{#>)%_p$_Qtjvg-;@P@<i~Kw#mlU3Ja|^mxJZ<UZUngDo_Qz%J
z1Q}7UFL%GZ-#znwt;CJ#*ZdzxS2ebGcTTB(z$$F}_wY952j8!5-l{7Roxs(&^W48j
zJM~2x6=xp&edTKL)#&Ummu}rt`1oCbYsrJv-{q_&Fa0)S40n6`(2H?G-qw)S+x|0L
zu?>B+>dDblN1grJW4>s}t}eAdE#bh*sX6)h&WZ0NPqyF63G<k_skQQXl}FN^j`YoI
zw$*KT_jYo@k}aif7j~CQ#xLnwmw0XNgoT==7D5%1M3~wNN_6iFv_`x1PpT2?dw9q7
z#QPpWpLL7#y1l+lU$INe`uXD~&6N?Aa*MR$3L2(-JfY@cufW{*E81FXhvQw|ZNJlR
zug$Gy@-kL>Vt#33N%Imjxf!>!Ci~rDeY7*X>e)UQv*rBySxg(IERkS-{M;?Z?+i<t
zP2;xG<f+n!Es|VXb*Imd(s-r!eff=R!WDH_YxX*yT$VMfX{P_a*DX^H+4$G25>iN4
z_FU3t^JMYw^8Gx4W#9Y7S_?xSul~;vIhX6kx7|lK{b$%Cb9?trlg7hYtG0)%eReN6
zsBKELWB$CLNzM~z8t}Yi7kYf{XcWgf=j9G=lRrKGx^DV`I?+evAIqj)EN8sG{@M=F
zOKYdr|7w3-_kLTh?CmYzu06W5V&~m?(#-{G*F7U&Y}n4tnx~rEF5}^y>B*qez*BDG
zt-W&zf4=srudE&S?=-)RcT?T5xp~`#k`=KHI>j5F@b6z5n71(U-ko(9<4$DFnt#E2
zW&WfnZ8wf*lhQl_Rc$|hyPPL}{DG@D1Ji>CEAD8@mVI1cah|7GQMKXt`afKrcP_g6
z#~1Diuuj;%GvfWS?o#dil_oF$GZ=-vjM@J9U(ytj>c-oP&(_+W-5ALbdd%ZK`?-~8
zbFJ*0oBP_oEz%Bq^>yvbZAuGYb(zaF$dum@<EgG`oY5ispP^QN+lGeQtKQGovWoR<
zh@QBr_jRVWR@tN1x^0{K*6Pj-=8U@Lldz4=(mm%G<2kpL-&}K+8s3?_bZ7eX>tEN$
z3B6jp>0*4?s_PlwT9++a`>N~7q>zgRt8Y!3_*Q|_t3CNw-7n1@IjXlbxA%NHuBGPs
zX3qD`TYZCrFW#4meRy3<;88TM(G;Q9sZEnO`)@o`;{3Q^jhuGnnX1k5I_5T4J}IAG
z?rsy>v`MM4cd2{v+OkEvUIw&Y-WqhMU8{6yO^T+W?4IWv<+GmN@{~O+!`kt-Z>RL-
z@`^T5|B4{1$2ofqg9|lQuiI&3d%}C&gX8*H;rCc7-~O6YYR#wiDT?U{tNcrb`RlTl
z9#PmoL6$vB>UO#Qhwa*$?@jb9{XQ}tyTj_0(s)$UVSdKsI|m!@I5x;8?(8v`>>;Q9
zM^tpyxo~5-{|u*J`sVI9cRy{}&6>q`%uZ=txw<qnZAn9`&Lp8Gk@ZD<E|Z^!zBAR1
z^}KP!yYUK7`u(YqKWxsmnBM$Zyt=BzH0G`8rFjc?$@#8q5A}H&By16+suseZe8-J%
zwSdzFAMY#+*#fqI6{~9|Ma3v~8PECi_pDZV-_!MZe{<Juy&P~?_l>6RHLG3mnc>P!
z6C94XE6V<UaB^Gtg^sXWsizHJZoj@(XWN6q18aV5m#d%e)xYbjcvjr4X!G0W?k)FL
z_1&JoF!zd_=YIyRikX%wLd9z7QraDV>=F(gluq2cZ|gfT-GAp!DxRO0;k9qs#nQ#w
zHoQ&NN}FL3k(&{qx|K&WsrKT3hRJK<*qd(X*!0gh|1jXrp{GY0a%wCCf@2-FZZBTG
zWyfigt&^m)Rv1hX{oWby#?XHG_bYeo+h$xbh`Sv2Z{8Mtp-uN=B(79mTlXRF<nFa4
z*3pHbTjQf{otph%f%JqT1IP19b(h!Xu78(2`z;Gk(Gy>;ij-o7Yey_TSghg8U1e>%
zW~p@F*RVaNo_>#47AVb?oy1=8e)9co$?lFHW#4R9Rd8^)KVg|cfp~=XwB&v(fg9C7
z=gocBy;U@8{nqN*r;0Z+v&%o6mgyH+maydH+sWT9bGJ`=^J?e2$EPM2TsnFx{XfID
z`%zo3M!#Nk*X!2GH4E=dnfANMU1Z(TrlSsp1{MVYD$Enq_xWV4E<F5ht)9V}V}cK_
z&Af9`rJ2KMpYl|#`O7xvG2OMbUa>=CS(vs~ZK=_GhDiqVGZ<=AGS5Djp2BRLF#o&U
z_Z@#UwEcKgrtg|5r_>Vv^07#N%*vT%JH^EXw#^K7|5WrQ$p00`$41#N_S}!fiwt#)
zmY-Vqs4VE1hs(j2-oJ(Ky52H<bw~Pm#T}_tU#9wO4-A+xEkMLAsF~AK^ZYBndH3%u
z*%8uh@$&S~uJ}yTqYI9lpYvrg@A2yL6=KQpJ6Ep@HQDlxW!?OC`B3FgE9z$)-=Tfb
zPOX%8?Z4@79=z-SnryMWu;Ba7e{X)9JMMq&6vq2Aczd4PtE;lH$8*|Fhb-H=)l~iN
zJWmzTWyva?j~7k4b#-x$p-`!X^q*sOTVKm5{Q3|t%DDSiw|~*Pq=l=(HKSRU-u%{Q
z&v3r}rR_m0{>Bdu9q*am&-r|OZF<kEv{PoamcRd9y~_VE@|xU|FS~AEjh=U-@OQm;
zzW(JaCU-ra#O~Y@a=Y89@kt_=lgMI*3G+MnESgrIUFmU*EoA=G;{6@(LN;ml9xMNw
zy&yL9*Q}kv5&FAl7v0N>+<tM((^RKfQB5b9IXq@)g(X@}VtL@VdX5{1l#k27h`*Vo
zTj%7Nn%BRKVffYh>&+|i=#_cW_PMv$*BakEXL<D2wJV!rqbAg@{bV3CkxPriXr)oI
zRO#V5i7i_S3!Zq~cm0!jIcj5n?4I=e{IvmZx4Y*o*|Fc_uJ^hNn&-T#7HWAm*c(ln
zY2fqtnQ8K_Vy$O>FPWFd*2z3{Ocv(tpLza5(%SDaWw*_u-Q6`J7Htl_9+J}~J~i5P
z|H|*yp?9V|WL@vvmD{MGD*teucaO-56~7y<W<B~?+@@prxTj|0{FSSdPnw)aPM305
zZnB$sZ<XA`+uO_5ajsfzvL*hcN<&kagY6wdCdc<T7+15iKNe&?^yEK-RVaH`WNg$4
z)wf1`clK*dxP8X#l0<GU{}Fa})0OT@g>T;9nEc6Y<Iy~)Qz|}Jm$yZU9m!i<`>C>?
zH~O_}=>FB;r{~rN-Tj@NcYVLhZLzy&3tcPIZoTGI-8SW{Kt%X@#`%Fy4yUk+@|-H&
zb!ANo_bv0g2N|AU7P;K{;ff1?esBCsw#{FpUvaPc&)^-t_Vm@biC@LuUV3}>ht%w(
z`<ufp4xevT5t#1iHM@|@C|EII$%;dMB{Fv;*DV%US0t_Svm(^J)A&G}<>IT7A8(u9
z)nDBIPo?PUq}4gKsX5-)-tFHu%Plwj?acf{Q`46Zgg&jlQTt?R{wL0;u=4`@T(!I^
znv&)}xZYvd@znfTL0$6CtwF~RzbuhB^oqZ&{L8wBf9>jp>lfRvynS|i#+UD_M9pen
zN4-qyI=S<0-mTTzb7U&F94MG}Ct`xd(&Ag{2D>gQb+Amfx9Oi7y5xbl*~6I|9@g+{
z?LKF7D`=j^BS$WuGt7M#<!jvMiWZ7EbTU5mauJ*^#CmY8)YM5{Q`!ZV>Z~X<s`T%v
z>AU#YRa~L1r98{JGfMZomVE8gN0t0%7!_tb;_rX|A?4eZZAaSrWUYL8O)4F`8tZ3;
zOc$AUQKDe-jmo~CJ&&K6L?;>l{p4vUzc0(IAtB|8Z9<Ch%WcXN1E<tE@K63^e?@!S
zw$7<f-dvgZEF;s??cLWE?56_ayq-F02~9F`IK%k+%zuVjvGn)f^6oB8bo_nBY%RMy
zpY9b;qlE_*TAn=rJ^4ZXb)UCWqZ3Ll%+R|bDz|A`;Hy=ZLLu`U7JOpg-hBH%gYlXZ
zE0VZ%cCE7C{EheK!CjYjudMz#Z)Wb)`043;??*kG)*PY~!fAZcOO4@(P|bnQ%WU+U
zJF7R$d%u*sXpzQgu7H@_X!YXI8^6x(C}(?j^UcyNJ7$@$nsQ1g$ipk7`Tmq9C$|*;
zW3$#Yh;0p=WV14M-%O6QWA42X+`8wkmMvbs>&9x`?Oa(cQ@02DZCJ8biSvoQ;X%X4
zZ_>TRd}paWcNI@p4VU=zJm`W{v~TN5&6A!i%RK)x=;u60-d^NCmz%?2-?FDcu`6;|
zp9f9JdbaG!!zRHalO`ys-<~nS|6kc<^^EA`b8Rj~$EMf^&bI$&`occs&Dt%GufC0r
z*>P#1N~_YM8SE1rRUBJ%jSac%jaS`2a!2;r=Ct1Lj`QlLN2Pz*dpr8htJibF=YOnR
zwoKb;{<=W*`2xQZh1A&hZ=ZiZ-q`m)LrB^c`T4)IUue2)e0(OYHE(*cC#Rm~@{T7>
zH~%wCR@M@}mGbZ8^-E_`I|LRpeEG1>yD}*xdwbsd-<8)dM8@hW%lGi9+tfa2{kQzg
z$xrUhH;(dmMxNTv6}V02w$0x?I|IsU9MYXrsvouLomqIQYWr<(UjL_@jzM!HEFXWI
zc|PaW>Fl&m#vX=C9xt~l**;-$E4$^VW46DpXVkaZ2LzYBtUaA#wf$mj<|<8@J1h&g
zHmy`Be{U$PFq7fse+Hg6i`XPouN+@yS($!n#eE5vg}Z$o?TXnVDl|XPx4~iw<D{7u
z^1>>0^L%>E`ZOgT_J%4o>^N7_FLh;A$_h~hk)>Xq!V?N=_tY<`ZnJUtd11P000VQK
zeV^v+J+&$ea}OqM%kwtP-Spn;Xz0!-EguRQ<7PH<X<AH3<CrwVK38OWlkJJ0pBKDa
z8|HCZc)r!rXtTwIZ~v{ey!`#mci+pt8LYoOoC>0&99W-+c?Cr?98{`K@SEWw^w8wQ
zO!-#DhnqJG&A6F){mS2&d(FMxYz<r0Hha6t4eu+AR}QK)pVT@KsVO)oedDz{$=%yp
zZ$>Xy^sDG8S7i(io@;gE_pWmfM5l_Dh|CF*O9>G*duCK2<Z^`b{k*?hP5<8Z=5`WJ
zcDs30NucVwjNRscv#q-K{@T0i^|xjBcHLQ(HQRlu&nlx?QyC^q)m_^1%a6(S^9k)9
zpX}zs_igVVO}%GP?P4p^boh!4|HSo+_A&okcstS|w>jf%vDd1DSNxaVI(sqMy!Z8{
z^;@@Hy}fSFouZjXFWYd1H9ZdS_O=L-F9@1cP}Jsqd$;67Rfj${ueU2wO7E@zX0BOq
zccazi<3FE!&V87;M^kR9?DhPR3yO?<W*7W-8ZZVqFi1RMP)KHRkkUHeds-#RcVFX)
z32wegkG^ba4R|8I{AK9k(rDgw|60wrt(M-bxrDWAd1d>RRP89QAO@BroCZO~ce0lh
z^r@$pvZr%T4Za}z$tJV(I^*0a6Yk!({kJ85r<L!nZ+f?G&Hnc2Xk)Z<kx97ASIwwt
z6K-9SabD=m?NMmSuw<rC#f${0d$(-1u=LBuw+5_HpW|0|f$z@d^S6)R7s=eRIxEo6
zNv3StL=CTnDPCPQ$Ddl9sg$qD+WJx3k%ftS%ZdEm6YP23#xgPA($6lipLaVX^xv|+
zKUW<$#{Opzd%eDBMePzz&HP_pJsnPUZc3&+VO9$x*PUaKz4%Z4Qv1BnIjh``Xg)uk
zpL6Y~>9TDnPUV?&wkmS|@wfDfiBd{es1V{mXOQe1aZknX(uoDHo-9b8>Tb>{uub%n
zt>J$L6VIG`;b$-Vzq*sbDy!|`#60D|LQ%sTD;QZFdgOn<j#CS{`D*ix$@^{`_33js
z&Fg%4s^r_V=Y8{QbXU8Yeq9jrXn9M^>aK;V_EVPmD=F__mpZ!q@CGxLc|P;MG1#S>
z#LSp#z}H_TQC)sJ#?_fWI;KD|t|L14)TH%XTuSNtcbqumR+=!o#Z|e?VB5lf@6S!}
zRd#zEUr}*o_Vl$gixzITj?G)NFJL?G+U){~%BR9uJkI~h=FFMqux9mgn;AuNGtT;}
zewp;2A&+^hYn*HBxeH&-ZU*1>JLk#MCKnZ(yuqtB$T7$z;mF^QYkcdaTtw@4R85ol
zYxwYawC%yx(#GkcpEu~wzMZvqN9L<_@7{HFPc>RO(KG5}i%Xx!438Mep9lYzOz%|*
zPn8N4T=iLD&Yz7gF?T&)NU{~539x&)H1!p8fvc^m(VXJ;W%5%knQbOFRVVb_uZ_$}
znLEWZ=jha$m->PeIp3R2j;hWszH&>i>~-qxf}Q`E!y0a~YCQMaFo$X3yGM1;*-V<g
zojG4nZLsl9{pzI_5BG$maJmJ2`n2HgCF$g%D>jehS8Q5(DKntmA^K#`>M6-K)o1wU
znm#laI>{*{>smSS#IGNhqmOeuy)!NNG<$tj*V5%0tIfq;xB5mnl&W4?>bO5+%8H*&
zJ`P7J->+DhdbLwLXuFLJ|G&Li9?vH;?*Df`aN_)`**BIKl`8!$x&0;S+3u;bx~_-T
znwWjhy8B0&_gCLu-Uq50UU!0~GIDyVG?e7N-E(dA3`T}ew<Uh2ZFzWKj7_J}Bk=k8
zPb+4hdwY8t_t9NTu3NlSx+2uHs$kOZEQ#Wdc?!$B_PosJe^73Gd(+7#|C!47Emq_`
z?~jv7wyRmicd=@#(zaK*IUzP8Ou;7)Y?A3&Z`;P$FL_zd`p}_+B7uD0V_%D9Gi|SU
zw(QQU>S@xsy0#mq-B_b18nWZnN&O}Mi4rGO+;|SYG3n!PU9@A4kaRwC|5}rskG5TS
zdujgc^is>6zbXTFl{0VWs&0O-x$)&93z2F8A;*))zn^%!R3<of?~gXNRQB`spWRfS
z?l~bHX1jS-#J%rocPh8;53gL7H}&z+BfBj1_c8S+sT{MO_i=gcoR%^#pPt(G{cj!O
zmTqhM#M5})eyUFXUf-lQtE$Y_o%zqu8r-nywvv}u8lwT@`NT(cCQ&uEMF%wf{xkGy
zP3~P_x&4WzYxo{n=a;Fcw|bvpKfSM!>%mS*!`l;&@8D++i>UDC_#!x8=hIToqsked
zdtb-?UV7-kr!3L@72doJ%yG+?#R$}-8I-=xlbWbs(L8^~8q+hI8gH)G+BjF}*WCNH
z9A$64vleZ;XI7i^v%WQC*8<HskKabg`0rHTuK&3q%~bYi@0J%yv6atP)XmK*`;o=@
z;nE!8c<IltcCXs&RipOh?Y3FH%TAZP30AxkylLjUfbgSPAGO#Lc(Oycw3?^ua2)$B
zKVjLlKSK8|lm`c_`p*zqwmXk&>($tI{p>pq?REX2Df?q76U#w~D;uL(q!gcIp5e@@
zoGY4Y_B{UES#_7Jy7}+-+TAecQa3j;w~n5Fp(6H1smaobZn92I+$<~Baiw{(xd@00
zKRmPXk@Aj}Z?CBAHe+wRQ?sHt`KH<Dcl-YS6n)eqx^DjE;>*sTW-{NN#MNKm#&|XD
z+b3tS2|tb-_WY}#xb#kX&xYxnWu_mVnj8D_or%uQ%5Cw&QE#{HkP2y2z3#*$GTq<u
z+mb1a%6(h6&$4*VW_mz*p5K-ATRc^H%FjC7e*0ba@s9V>{S|p-6H8Y|f1PU1b>Y!}
z2HBwbg}HpHYOyg7j`&wxjB)#W?m_nVqb0YkJC&Qg=9Q#e<_No6cHc8wJ4#hcQ$f>x
ziqOJG><2%u+Ot;ZPTP!^-_L5cz0kR(@;Ykw{EWTvTfWQP-kN(`R_V@{RUv;JgV$|f
zsy(~mp&Fyzi_V!Rm`u}uRQDZ8Wnr6qhH-xF>lrnrSt&(_XMI}rc>A~3>`;@iuBa_{
zw6460<yq)(cmii$^~p1h%-c^~ecfGX>iU+aGVJ5?_n~s4hfkF+N!szj`ghmH+j;Fp
zk=teG&Ca<y<?P2>+g3|u`Yq*hx@2kgC`K~($cz9^g-?CQLtfu@opgWS(&9Pwme(B~
z3NKW3lQFPPmOS2Dzj(g)JLzfLm(^OVznA}O_tw+A+|xyQ1Vb_}#IP_qD!K-_Prk42
zRIu0S`WEYn+w7lLJuc^4UvS9n80Vpj*6XTcSd1nGg$gqo@K3(I<6ZrI*TY5fnaQ`>
zS(H;<-L8I`^gtszE3Bf4d*USb6DR*3+g@F?X{N}vXR*;za+O-8F|{shlUSCd8b1H_
zw~oIi!+?8VyC_3zV6W3vRnOqA46*b(OU^mWpMT@~@5~tu4_IzB?b+%Q>%7*{MuB1L
zN>zuI2J<t1f16<BQX?;A$H6r>eR=!mk4GIPeb_`Le)lrH4Em|LiAP95h|ML<{tvIv
zBhTAGLWa?63QcBaT`AZ$EhfT?>2l{AHG56wlal@0wr%scTP!x=@vTKu1CIq~3Evg}
z9d~W>_51nxY`R<CuFcof{WPCZLoRf3qfP%glV=Bf3w%5l$je-cespn#`t+!%TNe{v
zrybaOXS>>co|{>WwO#45+QGT4hr&W8D7x_Uy%aK*GO}*pJT<sfd7k;%tc11{E_1dS
zx0Ca3Y>VC2c6s}Y*;{W%PHLN|!W6<8_;ixz<ay6ZUw1y4_*Fw;-u`o;Doz#*9=k$b
zHN|i_U6{J9gMCuUf<Hp+xA)(7@?`0`+cT@{>wkucdvhE7{4dYVzbqC#ZNh}pbDX?S
zGpsn}U-+p0&(A9#RRrcNI95}6CF|~d-B4@){|swNO+vXG7p%z4*&!=;Jz8POGCqq5
z9+M4Bx9nY#uxjocnP-b+0yvEhd=`BsUG?Vmw5xY_PtV<1_(yq)IoB#D3mdMz=F=J$
zzYRIzVXQ8=ezL{X4cYBRdOvNW|1*4Dzv#i^lQ-2DEcBk+J3D*5($cA6Ruh~i%zyfc
zMfmo?j(*vHrg!pwtl>H&;XV0b@88dRi&&>vOFWeB+p{8b`>J_mPwqNDohlG^;!qmH
zp9bOY3XDB}<_f&H<$L5u@AT8f4=0q#Yb77bTYUa|c$n1$gKV!i%U{ms41YK|V9UIq
z8BrTl)#@1D_WfsAwz%v4HrW|dp8kA1efsjP5xR>alizwb_dJ%keQT}xhi|Wrnw@>U
z)OOLT4MEZRy8nbPF*<5i-&s;5<W#33c*EoE>ZMD&qqKHgXjwik{_}B_yYyDii?`of
zJl(P0-OsF@S9#wer?4q&8XAOWSXwd*B-HR<j`rF6*Xws}_6*fsuat_rjy(#B=A9zc
ze1ugfF|eokM#8rPo&CK}X0b~}ht+1bcrQtLAGmZYQw&$?b?=nLcNCKM->XhOaAC9C
z&P(SffBmbmvFMpjt)XY_RDZ9k?knaYCqtd~bF}R6cozL=k-`KXj<>f}PN=(`)Go`E
znpg9)E<{CW)hovG<)*&1y4>=e>sBSroUs09H-kzFr<S@x-F>V0l8cS^r1_s4?v+|o
z@+z<NqvH;pt4EzCJ<Y3Ca~BM0oznODSf#`Jp82;uAMKu*@}^w!RhGwdZZ+?ay{@9x
zYu7%%?5hzn;nfo#!4v0r<j>06p4+4Q+o-5|`I%s&8H$S}*H!<GsnxvrHE-VCJ!Wrb
z-7VM~XtH1e`+tTjW<j+(4juBCz{fcG&g(ffl^@>B31YZtaK--iJ^Mdx&vwNh+%NLC
zElr#u)0Xqk>gK)Szp{7Ux?UQ3?P8>R&JxD+zCLU9t5ya0tY}~`RB`F(w0brzXm5ju
ze&_|BPs}QBuXMe>_>n`eAW3=e!+o}e-u^$NR*IN&>ArN$xD#EqZf@D{($H)^ab2$|
z-dq02$jE;C$Q=;SIpw)u(&GyU1t;B8^K{p3y!z$9pO+uDbPDL4Ic3v!{NB>4(hIux
zcYjx1zw~YR-h$hrUp@Xa@FcCjbJ}l3!&EZ|8;&WT7$liE8WqBSOzyhJ+G%E&!uZL4
zYIbdA!nJ4LnYk*PDyxcD-@RP_(0cc@w{Ls@7AI##Pq{Ta`Hy|`x;+oo?=2OUVPSP@
zP?7j~_0#!>ovPaQBD3S>c`bNSZ}T~3@gw<Hyt|GFx9b(3w+@YYm47ij#!BGtv9|YH
z^Q_mWY}xAisBI=A{}u-iy#=eH!ip3c=ghcLQyjZnVut8(#r?6ziZ^^)^MTKrJ7tH|
zub)>7?^LsW_$qzvmfEdMMVU*jEK4T-b&Qyw!ECku_MbqVAdOI!ACI&4&raFV9GN56
zyx{wizq*^=rsZ6+jrq^8V7UW#q3TxtG6yxL<;UNzzR#2So=ug3-*~I`qzjA6zsVNv
ze|&1`(Mh|e_1=1I6k5KBF=bo$(HuP$=I1I0j~r{6-BCNs*W8)=-vvWg$ApEY3Fd1V
zZ@zdx{nV``ccs>C4OpO=XSD2Rkst#{r|p9I!mE9<8H0`H_Up_MetT`=$t0ECN0vMB
z?b$D7mUY_O>(Gj~>n_hS;8)qvv(R0I$za~y&%eH372Gl9;~nm_D>a?5pZ$2761ElD
z?9Xcd&mg>7bM2PcXy0h9DI3~i9z}b#7pd==bHu$jQtV!A?GlF7>ZU5f``gUyDi`KH
z54z=W{CPl+X-uT)nY*vo&g)*txQ<~8BU3^%%YTM7S$gWFI#1K)I_vwq{k&#*N}q=*
zYxUAqJM8qD)~?WMSae5Mi$#q=b}GyHoLA=${Fr@Aidkw+tj){PyHkohr%2S#SirUG
zck7BpeY1PBn3sA*&7Nv`{GcTRvs_Z~0_CQgSGFEx`OmQY`}075xe%}B{Z~rXh;tv-
z)!e@B+|p%TK~t(7y>sRMZtS~YXZNe=()+;FZBp(_O}aixmp=Vbx5azs+v?EYs#{MB
zR?hmilWV!)w0kze2c3$~Oj>EozJbYPp7+VHgeAF;U3MK2FFjY&eyt@#e&?3IwxObu
ziz>{2R88HsRMDT6$zsw}rUed0la&mmbbrirIC07BU)ttOJx4A!cYP_Y-8r%+_q<*=
zHEizwioLr|^M!^;xruD_yPOoZlx32W&<vGt3~3Bf8$Qf4`=B(bh~If%-IUqA&ng`z
z8oBDfj^3FYm9%W7cA&T2tlJfdOCL0<JpWsA@6=nh*_uuFrmXt+ESuwSa#h#5*z+a1
zUqa2YLJOQuov!N?uoh))maBGK+Aa2ZO5$C6{i%+oSFTU!j9{F%^WNHZ$6tT<Su1Oo
zop~>}^!)WrhdFYm@Nsyr?OCBQr)iO=|B-n=MQ`pCp7iYD(sRcTOD+$K@e^rw@-Qu3
zb^6NhtDV0OWvvb6V_EBX{K5*mM@}sh73ST1FK7Mo`r$ifk7PZ5Kb~l7o9QpMSMX)R
zfxbo4woZ?lefZ$+T<MSw|7W`WnTB`x4xc}MY>h@I-@Aeg>FUnqwsRLHzjmL-yF21_
zd}rPqt5-ro%VU^4&P;jaA^-XpuhOF!&tNV$?L{BXv9&3rN#qM}75UF_xSU-yU|E*r
zquX{Xzp{uXGc<l)IIqHz_x!XQtF5BDi&8q8_}jj6yq&fDgM;cZnbl8Xcr_KY`_Ihz
z30kVL-BYFa$Wf1YnJ4mzj+VZ+MJ2s!S8uzt<8sKF)>QR5Jtt;fDK7Xrvr#=O*7kto
zEyh!eE<Jj;if!dlqf>L9wmf83`1tX6#)59iTU$4E=AGlb?9y{vH#10j`P$k?X(AgK
zEIxl=ueea;tr_bs@!{N;8y`OBS)ZNyuk^Xw6zj)DZ{0F)F5Z1*n&!1oUrt6}jhRnc
z4`>=ss4HF|bkA~zP(yW4+UDLF0&@?4{jzxO%j>qcuU1bjy<M_n)!WPQ@7Jw~;#|3N
zU;WQ>t7^{)FsQIBIbdM(=+zm{YQK7|+`~UKSQ1_y@3+^weydOaL)w+!j`=1bFQ=Xo
z@bG*ll2)KpU3RtV)77POY@F>Jia#B-{_Vo(pM5v=>2v?rwth#KZM$UdUnY90__DrD
z+rpA}k_?=kKCAcDWUW7DQF~D8sW1P<nhf_@0*?ZBE}QrD<=@kJZ>y>oZ~Y?b6{@gW
zWr@GhB#X5U&t>ZPcRY3~;8dNJ)AvBOlKGw1)+1+x*Jspyx+;FhO!nZmyt30b7O!5q
zWQW1&mHrD_)fTFph|W4Mpt4%HbH??=hvA!IoW4EvHu_}o^w+bMX~EMB-_BR`lbd(x
zTAkIw>Sf!n-hOvwD#!Z8doHWR{_&Fe8X$09LG+zhLf?Oe={p&&R$P6bYRp^Zam<nB
zue;WUf^3bO5@$FiUT%LF|04gQ?cqJ&?LDkAcmC?l$-1~tPfW||^j7ml5jt0NmrS>x
zdeW7Bqo&HiCx(J+bz_+%RT3h<v!v9Sp2)8@dg8L2VNU&7xiz}l8O2&oix|I6V4T2z
zdrwcr#Ys+Qc1$g>DVN{s(BydHN~}lF!!06P9Bnq+J>p|O-t%q#bytsB{i$+CLd?yc
zGT46(*y<HDp`cwhh)HPCjK2R2m+Iq=@A-D%`o0TERufBw&!5`RwMoCKeZIodYX_V<
zo+M9Tb#cGHZ~68enek3<?wP81bS+JJ&m6FR*A4B5=R~=_a!%MNyP)r1;Ug1|9jTL@
zY9Bf(GxRVhe0dsZw_(aBEpD#{E_EgA?=#=eXFs;__M-GHNyZPoAE`we_P+M@nzD53
z7LO-OMMWLwUw&(Qyua5t-1zu}9f48KA~P>5-dg=UB;)Gt?`2nR=36b!2o??3TIQU<
z9@Nry+wpVjni)Q4$|g3wU6**wXYRubxn10yQLofkZ7-h*VqR?PysM;b-Q&x1zn?8T
z#np7wG%E1KhE}bh<zk_29VfywReU;h)g~D5thv?i*!)#y)%F9=L;szYi&^z9Ft1wl
z$kN-jkM2imdUpiHdT^cj&v2!p=ie^bsoRwEnLJN^lMZc5jkvADaK3Tr-S?}mZPVXY
zyLQ{Wh5PTlS|#PYcEg9#U12Fx*QO+D^&MEfPg!afr)uB=*%fY&0xd7CZuWb{w2pV-
zIoF1jQ@#bh^ORk_<M;X70e5;6%9>3)tgh$$d6l|cM)b<-N89F1a%yTcaL+uSDNuO(
z`!%6At1GIWS220lOl+KFSHHBd{?)9#)!ys(dTurMFYvWmEf{iX<*NxTx_Tz9=Y>8g
zoa#8xEUZ@g`To(lub=DJDA~U}cx!h(OTxEPXM28M^}Bav?X(rsoaAhF>~NgQ>i*^X
z@nqQtGTH0m3R}B_4zV?!-Q4^`D65n8>}K~{8xLPHv*!&7xaqL^k5f!^<ANm%Cb($v
zRSPqeA3Rp-wfo1s&jH@9>|yuItxNAr^sA^|vO(bR`^v{t-QRInfA>nixh(Oh9?Mh>
zMj>mLqBr~%=WFeLlyPti<{c1uCVk}dQSHZmA&DZ@0lq7LKRa-zto!Y@ttHc~v|8+J
zyEGZOzbo%yz9V;!=hb(9`5mqr>sog)?6j+`(OP~u)@7HRk>{u5*Y;+9v962WHTyE}
z)}1e2WxV3P(zEi&lu4EiT$gMnSf4+(>D6Ro8~wirFT8b#kCs@HuzJJiWk!=0tW@Su
zSWz^|UrzZQOWD6$vQJZcEEzjLtyp}x{&CcW6{*vuybeV&tW;=amb7jEK7Vq5&(Wf%
z?HM;DUe3R@<TPVvruq)~{EYlpvS%J<ip_d5OY7>wplSCe2%hL?Y5DQ*UfH(PzT60@
zs)_s0O>ukReP?NYkh$o&=X)}~9nP$LwQJXuOBW6NKb`8B^zdy<L-C#C$@hJODw<`T
z=j(pAn>V%dUCMkzHO58R^WDEIo8*Vxd@s^}=j0QiN8DN-3=>+`vP?=+tDehccXrK$
zTQ!^#_MI!9#B#1Ys8S+*;Xi}Vug!aPs?U5`y~@Pgt6a|UKLg*&Rjk<sQ{{AjP2?7^
z&tyI~qtM${?1%E}-+_O)dnP?olTe;MH>SGkrRHpBj%A5m`XwwnQ_5B5sc)EJDEW7@
z>)Xdd%{`e<Ja50yditLsu*@m8D6@9InQqg%Me~cKO}DLkbU4Pxj5E5}!0x8{b1DDh
zPimc>X<XSp{Z&=P+b}!Hg6MxYPf8w?Sml1ze(AZHG4HPIe!6yL)+)z{&}`jp6K<cJ
zu)xdO<N3Vf8|Cv1KVAv0U8x^s@Os-)c82;_#rx-7y;djtNcHW!Q%;^wI3<^VcdfF%
zq+Ob{J?+w#OBb(C*r!oww{ZK4<`aut7CiKv_tePl*313;F&|bhQr+gb<N2Ho_FB(t
z67|=}E=p7Mc)d58?QO|P2j`AVpP7%(?@IDi`Oomoe%qQ)K9kn*Imm3>`Tgmv6{)JZ
z$Nn?;PtAzmrgrO2S=1}(r@N-TbbF+lSvW5<$ijU`eIS#E;G>#X(Vkm0xr*n@pUL+*
zkj5D4sC`8yT)ArbADh{C_B!skl>2vUMDPBH{|sjHm#S{`I%lb3wergWoBhibmhc_O
z?u(DS|DWNi^O|J$n0m8ybF)9HcxLSS_v7jQ$B(_^R{v+<oIZEC^KG$re=qfJVpzPh
z&}qVC^^Pi)6_N@wR1#c-SML|tyltA@CFP~ZJ8%37n#cd~$b=Jhcjn%kVd}F!{_y8l
z(aYao+nantV|TQZpknLNqh{4j7Fx#d4j8RCac$Zhs~7$6^|pTc`||AZV`fJ3?}Qeu
zd>>fm_x<JLtI3sbPcQB7;+ng1%_7BZ>_P@1GiLVvkl%4}(`n`A+(N!&na`K6^q=bQ
z?uw2rcdvAxry-wxQ@nn2{Slt+J7YKOd6~Lud+sWclupJ?k30qM2H!Z*ytF@4Ca*L_
z;ir7njKUqkzq8Cgo35_BwR+37Eq9;w|Ey{H+17Pw-X+J+%8z#(Z(gKlr_h(NTf>Oq
zmAQpoU1>|8eyvqqeXmxweCq4Dhq~r!EnS_%I%Vmn$D$m^Mfn^A*oz-OGRb{<wxiD^
zah2ZA90C7rU$>rC@m|5?`Dx|p>X5A=nW2Z~=3FUIJW?DqtKgup!czsIlw`?Ew`X_9
zCaK*Fo^bX2%b?n}WVt{N_UDdM<~J|fwQSb!B=g<wrlMwtxEMY96g28K`8SzvJv$|9
zwT^y&&8#!L`HQ``9bO(km+f7YH}|>yfzF=MFBbZ(v|I4>1?PlTOSu<9#zOA*!zC;>
zR`9v`2lvQNReZF^%{J@Qj-z~+u3U-A(0(n{RqV7>q@_b;9ec^QMnUB>FRC|ZU06~(
zP3C!Rbe*W?-ib^2m)reVGdu0aUEbo-WixO6vOjFc@YPvMRNIrYV1|(p|I>f#H!?f?
zXUMGIa(cyOkGwf%PK|+AZ*MN-QqP_CdTp%L6p^NFbAp_12Ygozs6DAHP*A-1;S?2z
zO)t4Gd|g(y+4jJd)q8c!Wmj!4v3Hn#wrD?t%(DgJKLZRI?d7T;mApIMzjU{;yTr*)
zD^h-_+?g8P(_r~^QB`-Y^|p(dn{Q{W%x?DJj1sd@ah$Z$?&TAXzUNjxcd|TXP8;XW
zbL!mpBTc(RTS9>^IzV@M+LirLq48d^xr=J|-3V%US9nA6Kf}jm>yJ$h5qTZ@n~z=O
zlV&Qj<jZsW%XxK$?nh<iiiqTytyE*w{%gO?-q)93ApQ5AJ=^TQ=J!9G9Vr}q%rHRd
zwy&>zEANCEkxpz3=YN$={&!3Lt<Rm~=ay`HDE94BR%fA0*xVz#L{+!e6*;m@eAho;
z$jD@Whr?yP=W6{kuWq;gb$R8v<zF_go1vb6rR3bC)hjuJ7e@Drdo{$RW(9JZ-2ZfG
zOOcn;<oivY?;M40opZip&2KpCl7dpt2|vdtTeiJQ56Gx$+_`GIso3>f)m&3gRb^#W
z`*yZVE?MU7tGMxq#ZsM>F_oHLK~q9JRbHN5VepjcY1Mn@wZAehgcpBV@H%(u>t?M=
zovO~DGdin`jBnLn-)F*m^hNlEJ%6-Iwz{)aT<P3!b<3d{W&M{t6g>;QSUDKJH-7uP
z#$LE}Iv3w6#gs<37Ux-~-tKCh<^G>R>b1$K3?{J&oLqLvJKrhQnTo`CxhV!ZvY*<r
ze+Gv`jZpnOQR&>B(VDNOdWXF>=PKav%92u4ba4~mZ&IoFo$-6=+($=}nNM8WQgK-1
zKD%szJ@1S4ABtR~q*A9#D<)WQaQX0OO=6gkkbdUG$0av2uA~;()VExg*iobAce%h{
zaqhm7-*<VweC<89{Fl$(+vT_Vf`eCB@0__jZKJpD@~e)%M%`KNtIsGpG%YF6@(5c!
z$LV3{u|*D_Zol?<wmtanuH0S8%lGo{SoT^|G-qp&O?FiIRh?<4b{y(pU^&UG;67Q%
zB<8cjg<hUB{{62*UK~DZ8Og@(x8*@{@!hFu8*;0Ux*l5e=u=GM`M^}2g>Ig~9hyQ#
zQx5bU4=sDdkd-Rwc=^l9+&r~eNjDDt%JryQ^h@WK#>(5xWs3^7v{wqgEK+m!IMkFN
z<eq7-W?KI6`md|XVX7P_mxNn*pF28P>;4mauNV3suD%q#y!L)ztibK}xxZ%rHVJpr
zUZ;3vXTWK(XVYY(-!-@x&C`%c@0#LN{owMCC2zhrTNcje>D`mLUU+wOwUcP&lj5V*
zd%PwsYL!_tKg(&Un~TW5iR=$*w^sJ}E-*~xvSN&}^=eg^Gk>o0j!jc6A~dyw92nH<
zzpeN-e}BDHgHN}?6od2aR#%r^GBaHsVEKv5!Qs&44=x_Yw|{4B*(vm<yZLdzNzKIO
z{f`4oS+56d3UO4J#KRDJgYSZ$|9s|vTU!?1IDYrz=atRN<{snazs69xX|qbxl6eoA
zSSDHjw7TaZlk6Y2xF+a0W5_GX4=cEqd+pq|Y_`{=R;|i3&kyb|*R+S(PBW@0?mx9`
zYDlYwVZg$ZrFm0XmovJ!OjJ2={Eq+2?-%b*&?sq~A9LZQ*0Gz)RfkW7zcLEA@sR5~
zfBH!SABn~f>H9hw)eN5&h5X=Mk@;Yn+o_@@?;d=~o$GaLx>UQ+EUQ!MZ<lXied*ol
zuTZU_((&G|P%2qu#maBBk;i?KUwgfi-dvlf6<_k@eO5mI;j6K_E2^G#h<q)1u=Lxc
zgn8PXj|w%7M2cLV2i^Q=nOiKx+yC@Y2ImeJp=E`8xlAuwt7=IsPI_M8Ew^aGv<of5
zhL(bC9-kRFk39ED-8J!pJeU9WQ&Vr=U{Uv$RX(<8{de8Hm)v!DBEFS=FTI}^%G5V)
z-n}WJAKxt(KPA+nmge8sG<ik$3`f1?%JbVF2mM{AA+1<cn|1WUj)ucB4L%-^R<e{G
ztC=|ehP;)O_on6E#UGXl?^=_*yQ!+~=gMCt3Sx=Zrd<j>(RXyC?efPvW&bneO6_r7
ze9%Y0W=~Ca#lw^{d9Fp1U#I76`OgricXer)m}p?$;!>}bTbC{Mo9}1IDl$*`woJ-F
z&&evUC)I@JZHn8yN44`62lLaem-l%Kr5n?Bg*?8sdh`1)i{4+heO1m{_jJow%_YH>
z+b{Zhv}f2ReE)oVP59QLZB7%Ws!!2b<i>L&_uj1xyGtJ%9#4Cc{B-B{RXexO&)H_%
z8shaT#rSB>uG4;In_8<fC0isP-*V{eXMXH+k)uaetLu3Y-}<R?>OD`26VLzC^0*kd
zZq~ewIaQ_J>`ej_k7_guRUe-?<D?whJjcUtAJ)HHULd<Oci!^DCL3Z_u(_~5_ul(y
zcjybF=w(H#b}U&K6RnwByIlRG?D<#5t9ibKHN29&_RQ>=O~3wHo!Pxxx}0VA82+5Q
zdgcB@r`MkHdUsc8uhl0liIOYP6MlZz6_~Qp?%k7We|_6O2Uf-0KKX6Wammm+mzDQl
zM(ld{;k1-j`1R?J&+YaP*R`8&>#}sK6XOGR{g=YeRVFJxj@Y4OexaUgZCUnuha11R
z`lCX&-Q_cX#pEtp_+8QV@$`(7?-#ybQg`Ode}>1=n>N(4JmHePBj%m=czb9-Oy;Dg
z#tW8wvgva4@d}kw=?lJbe#VszvF>xTZtZDfKW}QDaK<bq`E|(U6=k*uH@*Lz^*3lq
z_`0hh?U}2t`Ttwl*s|q`gI7bq`H9VUJoOVro-i<{&#>QjUn?YMW-3eXsuyo-59Z%7
zi_SfD<3&~Pm8M&q-<#@}MjALCZ=8SG;cTX+?6e-1CuS_aq&Bqm$%s6C8u)wj^56AU
zt4;?;v~G`g4!fEw>zpQ=dqiO4J4wzXf*);<^oq(pF^XndEcbIwpR7b-VUN+L&x^%w
za;I*oS`p;M)ih0nN5G@WroZ37WZI((N4)3cUnySnaAwSg^HB$~u1pW~jC-7@EjKC5
zN7c=!$v-vu>(5KOtDRM5O!i<up7Wo<=GDQQ(`=WdJV-9QG~xEO>-D!nzsnvvko0=n
zDaNX~2`er&)!z8G?(*-l-s1SKH);<*iOrL{#QbkU#P2UZ!y|uG_Z%;laJ)02NPVr!
z+imUTau1f=lQp^@?rqdGTkqoKoh{#4xgF*eRnOnH^FIUs!*an`?tPNVI=TN;of<!_
zUi$UnZ^P}646^fj!=IfE%33+Q`dZ<&6aFR){JVSB-@d`*^HJ)T-1Sazsd$E;%)i3V
z-*Hpz{qo7g@AK-s&f}5Mt%~YRJI)KUA35;O#s6H#tg|MYjV9I9WJKh8Z)?AO-sW|8
z`25v*`@RIf3NN|zo~v>1wncYit*>o+#T`=8;rTCq5re8;2g|vT>Lagb9Q>FReP`1o
z!z*I9Pg^{H8hh^ggXrveG7Eauk00A@ySdz!?bWSoz2{^7-mctr%WSo`zeliU5aXjg
zjK>w8q{qr{>U+j_C$acZy<$w;yDKWom`XEOvfkGH>Jb`hu`G&TB06$Pxs2$Cq<YVD
zC9CdT%jlf=<Wxnmyt31tzG!L2w?8k4J$qSp_Fl=M$%VPvs`n;1c$?~&y$VrbaELfl
zzv5lVwcCAOj_(rqzq~y7?MdxxO{QfMX7R_TUuuwhG4FI<$BoIGx81fb@60VZ)VpMh
zaL1z?GLH63X1-rK$GUHmweVa!FT+nyKP?k)uwt9qEU(!7dDZ!={0r)O@?X~;6uZ0K
zd)w<%Awpldgj$-Cm}VaeV>y%XU0y1)+(x|Msq&@!uO!%?27kDHdv>Mh*J;duJkJ)r
zUn75*z2+vj_RFtY=FwcNtUL=vUVHV~9?Wi<e9xsW<KrreAGMDrRx7CSo|l%F{hItT
zR_XrZh-Lp7zI)!Bc>SMNbKdzS@hfUqhWxdyci&$8r8C0uv5VMsyMvCtljV}1tNgfZ
zz55AYZ*k6pzUuT3XD$05Nt^e_ZfU8U<&!7ZlRtOei}iP;?Tg=B<6;W_@yb0q_ehV%
zq022Nl&4RY-E!`I*8&GAW1;OzZeMhF-S)W|7W__YTb^m)tcxx!HW`!FpJ{0=I%w1Q
zCMiYm^RgIAuc(<1)qLuw+f+41-&y1H%OuCD_hs;oh^z@AyY%`lYifP_mHj}3gGnL5
z$44e}>fVjhu17j9`Sa4VD>g;xdsV<wrcA9_&ilKoCka%xiyldM{LH%UKf~?lv-#yp
zt4=jEp5X02x8$wU4&G0T_Rh^;bca7ix5w<kUSoaMWn5lr3V+;`nNR!`E<AVcoN4y)
zgyj<sp1-zsK2O}Hm<Q@*-EYmWys^Ewt-Ms2bNQZIuWqhiUYPx8iruA2ODv|$Y*qDS
zE1qDJaQ$t%RoBsb^5>;q^B3o8F`Qw{t9a>FJ?m5X$DQw`zrHNpz3y?)-_+1zkr%oC
zcKf`ex`L)G5Y=`0=;hE<XR`L$Ebd8MnbO67SKo2Fr^&QLf2y7RT(!0D)_SkmZgE;a
z)MC=Z^65fnI36ucaXIq+9N*e)Wx9sfx;$lVvgRMDusW(%%G>zV>3ZyrtEaRyq6;=m
zc&@tEVWnH~Be(SZ`ExCd_C&mOPoFP;UVg6r#_e+#+0HnpQFl9aP3=RM-)q>UgLY`9
z2q-`37iQyI@}FUaTUFMtIujr76)$W1=jP|tP0w4qH`3MN?e6~!QNN@rw=B%CTNPrd
z(!|HfB5}m8&Xk}3N1J4)_69k(#(5W)+%lZSxnhgj<PWP>uk?Sk_uf(~2kqS82KN`$
z6;_))F1|c1dzPo7IGgd@;|XiG>#SQg{i8mY+5}-my)~l3eGgLP<z<gebuUXvmU!vr
zdvk&*?{TZ&i+0|LUiQ%EK<N_gSs@EN9qgAU{`!-pnX@RVR7OfCi*uFYe*Yh9U&Wp*
z%l^n6lyT3yQL~WC^UvkG_Qjv)`c@u%Gtcnm9P_>WBEbx)!A{?;r@9{6ShBV6m+x8+
zhp0=I<~@(<ue(J$3LTv)X&Sock<iWyp1%Um`@DS<68~25qHn0*>3^#(eRA^7b}(>1
zUnzLsUh2}Tt518ktUSy&<JZ?E;g^K@c5+_Q`VmtV^*lW9cJ?2u=dQlRc{8sub$i<F
z;%I62P~uQNGvn>Lxapq?=6VY?EuUxpt9Uj4Vc8}7Z>OJ<pRww6>22x#X)&vF%ug+`
z40Mgy6|_XXLq&+|f|rA)g2a&-AD4alsFClq&y9a-)Z4XU?1s;SW|`cYow_w-YjoIO
z?`0tyK4@~WGaJsSDWAORxa6ZEy`+|8nX?|E0Tb9ZKkf^;ymD(-IP0ZJGiNPt5{VXN
z*s;WedB*%#67N@7gf?-6Ro-U!a`4Zpjg<unJ1+Lu|G3q*|Dks7qlepL_TG-TQ#MUY
zIyNfg(S+$I3%qMCb2Icg2F^I4ELDAQsqLx<iSsQaZ8l$D@O_1s+ox&H_vEza{Jy?2
z{=wldZ?}BQ_bWdhw|0fe*4b@aU-QcDSr(YRXlZTyr+a}xSI_#%$Q|UI;3MlR&!uc+
z&GWqG@u%nSL-_mik8E0#QPTMJtLPo3S7IVdCOxiMu%cOJv5@ume-A3>$SO!VTs?nT
zer@r|4rY&Eb)}pkFCINCiq~X$P&nCArDOj7%;UMqTewPIviwT;bG9-!a=NXS?xCgP
zT)}Vq%al7UexGxZwdlNYyzfl&e}=UQH>SQ0$X}y*wBOg!k=5nElgZ5ce>4C4b&oq$
zr}~IIPx4FW6R*EZywj*#YQPx&@XCE7OIBu1<z1n_dj8Ez7h1b#;;uG^<}Yh*O)WdJ
zoALDehXJO1Ci6qL-7R345Ul?BpzUKh`^!GFG8UiotD7BiRwY?M-nZ9f&(fteS9Yp8
z&k|u|XL0zfa?bs>^R3$(RJSe23*RDps$*x&{H*~Y*Q@wLbz3wKoURD^AS_@#`I&v7
z?3QchF1dLn{Yr(O|7KKow{6|dRe3f4?(bjSE6ozOzBS`L>s`N!=aVVVulk2aG_`^*
zZSaV?B;*t_aq@gA%Um%B??i?V@@FoYJ>oXJw0r*ZUL&)mb&tJ5&iX#Pyi+%4t<SoR
znL1$$6IBks?Q}ZBe}>7jyx;2TELJuJlYW^V`K*KGK?dea%3i;=`;+#$a_i-&>)U7N
zUR}Cm*+jivzijj;T@hf~<8u6bZZ!MhTe)ouWaRdo`0`=t^-Wey=Zt-P`y=;#H?@Ab
z`)cUUoE0hRR|9!=h$uANbP#a)Wc;n~XJ~QOL@zNJp5i-`YF7)cm{=w%e_1P9?&d4i
zL%r4ftINE%X6ISVUV3cj>E3+{SDxhiwVJ2Nv1J9%#Ks%P%N9K{c-5T$e63@i0{=}9
ziJz0h?oRvbdbwq)C5!en&cJt-k_KtTH;x~h;x28stg!iV?ca_5c{<ZQxGMQ=t{c|v
zxNvLNwL_CjvL^p$cvIrAwB)_jB6mZTh3yP#)<XOzgy&^{6#aTBcGJs$)w!|8556ZR
zEw0;}J>BimvIeg^wO+g(UMFoDPlc*DHBOvA<5*Ml?2{rw2CL6AtKVOKY4xWwALVZR
zXDH-X{jefx%gS3-ORs2O$W3%v>FSzf_|C{u?uVq{-~JzElKJ@!)2AC$rr%mRDN?C*
z?$n~1vR~J(@`majnm4cW%i5GJOHNs<921#r>Df5H|8i`awd3t?*LkAlA79QF^qBT?
zo$kjCU;bv+mv31&@xtq@+EdFpy<$8ynfssm%Y4`T{OxzPaUpv-U&Dsn2ATGkP9NS$
z3uy{}{m)SLpF!)|%k9D0yZo>CZr``UA^C%;%sf6X@3k5tYc-i|7H{bJsKnB%^SC&s
zY)R}&!#>B;OtuEDCY!hSFo`^=>c6x%?aSfX=&IQp*X-^)ACi@+b<=r;LMxLPoA84t
z6YqZ)URZbG)XWB<eKr4fUp4v`=Do;aQ9-MfN9Cf#G_eJXtUV_h`LQv~`?%JkygsDE
z==i+i`4cTxB+q@Gcb4tZ#t?%o`wxEEHFfd<rU^@y8?t(JSgH&EKJvP{tbNO2**|=m
z6FjeT2Tqqx)jc;i#B{A;yxRk#%PXd=RB@O*=U?7^<pUS_i(by<kf`9RyP|t{)!dW!
zCY8y}$&;HF8@Kx9*XXO!rB_?+lh>5iO$_Xdo7CpmY2YaLNIBr{h9ZtI=YRhhPHoOo
zZF$nuwZUxG^LJvuvaYXKXQFR0`ITY79bK;TTMinrpRjGXB+%6tWb(dL<l?)>i9OdI
z32|_G-O*Na?E9f?zot@3?^G(gGw&)>ht7!+t6xa&3cO<W*=Hr6IhWI!mBNkpjqm^F
zWk2@jcd4LO)AE-u{nzBFHt1~JHh=x4N!PROUfxtKw+=n`aOWD%sk`%57p!mM>a-C(
zB6;Sg!ijTiD`bMW_lfNK9c-dJi=XSIjM~3@AKS}a=5oBM*II8n*KEPvaOt+cFLQ5g
zx>*=?Y1ZkeD-mC0xda?onCuPbtIYe%JR!bDY<u6G;3kh3XaDi}8Gl~4Jo97w&5ITJ
zB92Q+pMPB}yZHL+dFR(Gx_#GY(^Z2PnvUzd4owO)2-#5W-ZUxAsO0v;_pf@kmD_*$
zr_Zu*%il*v`yKeNP5yZP<&$y)>2=cg0>kspmv6tfcFJ__SXbTDjbBtVqP@7h8<tzL
z^uKy%S0h_?+PYI;iB-nLC3ix7)WgZgE_wv4uP9x0@#_2~aWhLjSD#)qZ`tJ?ucp3U
zwzhR8r}qP=7|jy_CsoyjY7R)Py;Gxpc;>ktZMoFS`cLQTW~?sf|8edU$AL)e#WVgh
z%!+z@f6?8?t~rZtUC-Mb`|4NOVU^iRkp<gc?>+TXGkwAYA3^3@3NEjYKb-VHVain5
zP!GAEYBSW22ArI3nA#uP|1|vBMQ88itru>tJnQsg<qco!n!DPHTRIr@ciP=Jf5vZJ
z;L3>B)t^2d4P3vT`&7geey^FCj{E&5_baO3)SRehXJ7Yg>LinQE8mseTI|d6dHs_e
zAr~Ip_R?K)sNz_)$LEQb=QBR9nD;^ay~2U#8y8EZ%HBHgRw>SlDd3ly=Od0eb&0q4
z$iH7O>y~+0iUsfKd9^EU<cA1&JPpXb5E!7DRW@_0=$bz5^RIiH+}NyZzf6)Y%<G;!
z_dmnwxkB3~`+2iUX>l#Pl$FP`XO>iI@QM{8OE@_VYL`4e!hb!;)%N2QUf1gf<extd
z@(L7oR7hK3ZtwqX-h-`{u9df^y<Te67(C^x$ipM6B@eQ=S<i3&9dpT3nKNJMwO#eq
zR3)jePE7n!pPzqS`{ne7SKWJmhTUyCc5&}ik!9<yy<9fIfl1oq%cDvkmG8EDIL!ri
zZfZUB(&_m${_9oSKI}6uWxKmV^Fr=_2GhfOOKa8hxZJg_Z(DbDY8t=%5vQLnK6Bg{
ze#(AcvE;G1fZ*SK7vxea`RDgm#doDniqTNEll*lxCEj7ze9wT@-rMH8H8k0l?6+IK
zrFs%u{q5~JFTdF2X}9w1y(VYPb+_W*oFzZ*KaVZ_rT5Erv8>v@Yw^*UcaO$=>h3OK
z3cBQ@xG639^NMfppFEC{{x$z&zR0{k#&b30bvM<nUiU}An!7e7WnT0J`>XDk<~?~a
z_0Y0K*ORL&a=+hjUa@1{T(6wfm$`xyWDeB`vFA&jXMYr~+OV@W{kW(6+^Fl_u12q<
zlJdD&UyA={IB>mh{yi=8SpoV}R(}cfn)v5?rTd>P6I3NW&i{6;PI2r0%1IIa#kubF
z*ILcDdv})VO+Hf*YV=CA_3z&L9U&37*XU|Ab){eA+tMN9bUgFGkL%t)syjGN%{cet
z#;31q)gOI4QNX}uw?%gQ_R>u^ixyND8y2rf)o0!BBl=>;FP0NumE3F&{x!MWQLs}l
z#6CkpuEO&8iZwS2RTNu%m;AVP=zVq3qGj{eu69)Q*L3g`nECC=zkH#(U;e7Cr+9g8
z#Cz4|T5UJGQnSHc=HlZOTTOOJb=#^QdhGUEvoNzzantmqkSP<2It4hJ(yCv|Oqz6V
z*T=9QmklR51U~e<doXo%Z1C0I3)`lh-|?Tplhbk%$A5;nPo=pVcDu&%-0FNC@=wjA
z{9<9i`nTtghTe`_y>bVC=;Bulon8swJY|2UzbkoVJ(qh)@2h?tAzusjk4Jq?bBm7t
zh}s_PcSxj2#-_V-rqKMC_WAqodqsDatqXO_I-75vQX)4mey-Q`n;Q4@L(HC<T0dc*
zApK^3OHWGumSgFTuGN~SrkIqT-P8M@p`ao2bzg~OyvNm4p;t`;Qw?n0=W=MBO8L87
zf#c`ZCXtGzZ>~HyJ+m(N#+ggro4J%<v3pjn{qlXk?(X`s&<LI1djbREG|%q)>Rgf<
z@>OT)H?di>+~X39ZZ&E=^YT~GVB)yiS3RxOM<u87@wv&4CZFuurOo^9Y<p5`@3rx-
z+UvEl+cqtmU1{u|xGQ9-?zY#SQM0Ek@lg>9JQ?I5A>^KMUE#<4hrc^H%%mG1e&4=&
z`Jw9vJw4fMG{XKf)U1ES_i|^9blT&P8}mIK^g?dA|G9epsr)S#zn%>ncWiCoVJe*5
zI47xp=OZ`EFV9&n3r)|uZ?-lt>%Et7+%2K&ix%<o1r#578sJ}dJTz^$cbi(^`CEP2
z$3w5K30wByUZ_}=xM1G$!s$`bsSZpb0RcgdToblTT6`zXt^eV*$lToj4E@(uWgk_l
z?mOjp@6%JCcRe||53lw9z5H(a+uN?Ex>tl(FEv^zsk^fMRnz7F3<j(Yyh}O1c0D+d
z^ZoMjy*pW!ZhBKGVC3HSb-N^IMqAp|iKpeue?E?T{Y&@K-}M*w&3Mx{<;^RW;2Szu
zbORSO)X&->b1$kv?yJVyl;g#7jZ4#x@EEaA<NIfNL@zIOp4*PPv{|Z;0z|?-g*iM)
z@jRD&;Dr3`APJY|jjosOsWk}YPk$O%$~^I3k|$@(zofuUR?AM?o_{;ul$dyZINd(^
zc~70_k|2?wyIMvop9pZbK9<Oyw1iQe`SJG|SH2e+dro1OTmE(Jp%-_3`9n32OchoA
z@W{V)X84_X4}5yXCm2*pi7vYK^!8f5a69o6^EkNzpKS6_wlKE+-7Bm;f$jIKlf{zn
z=A^a97FevyQ_b@DHbF?$vg5~phVyf?vzGI$zt*s0?vr;}=dwdYg}nPj*Ej@l|6AO|
z@c!}Nl1rv42_a`SIF}T!4OQ7T;r6NEjw!zqGqfHl8!PmjIr!t;mU|A#uk7dSs(#io
zNl+%S?~8Ha?wfB+G}rF95_D0~F(6U#xQJqWo{JDqjePTBixi_JRWrV~YYA0eS+_M*
z)bIQ9xR<_%7MGX&XUMxVy<lln=G|4b8ENxgOjz+{+Wpd{s#%X7Dv8WsQ^}Z-;wC$z
zKj4F%UC6{Ab5a{(Qi_j8$*$bF_V!A>;Luz94}DkFa(}cUo-=sL(h9Gp{cE?*<cvzG
zIe2Bl&6p#eNr6>*Ws7&GXzw+g&A}0nyLGGmX_hA2jW-g`&$Rteck9f$v|5GtpMG7L
zc*5|msJ0XLy9KJcJN6a6Y+`cgQ{ER;U&Ht-``VjR_FKvp`rgU8!Ca-f>?`wHvsc~v
zq6>;WlsJS1Z#=0>e(>$erW2Pm^tbWdt}7|5Y|6cR;(1K>KZWR3*Rx)K*dBT}@7wpN
zjRE{#bH86|X%bt*GDG(3=j+Gk3q-05mCk;jd*yoC5B|pXho{c$>ht@0_M*@8f6G>$
z3cnH)B^|)=jZyM@&zzrfoIkI}%)487`+f62EpfTu&tJye@;v3H`zvf|1n;Hg&0pRz
zg{;%GpQ<2tp?1lHI+n=|%E1xQ?&7t4bACwOdUa7sM&C<tsUnYcP}i*1g$I^2t?&r+
zY)Fv!yN~g&>4hAX?FkE0PRk3&e0rq!ul4s`?T=Tl?wi%t=o{IYqoUF(P{dPyUeR7b
z*7?Y~rVUN%Ji_l8H_A!aZHZ?QvT0S%`|u>wdFQ0ZrCf#6VnWSD<O3S{R61t%-Bfc>
z`Mcby&`u>yd5e7cyqjmEb9dMX`pA5?waMOKe^~le{2#NIu^0EAE?6C`n0ni4nuFFG
z@7#%kPa2=M*Zo>+dqj3)=ZuyY_e|7}$e7A}o2dW!`r7i)Z?l3#UVC}3)#KxuDtAL=
z;?LxhzmuOwZrRql%u-!W@@Tm34H@R|t!^z#UAf~$KKd(6S*fYp@$peH3$L*9vHm6N
zCO)1sKkM#t*#-LhTeEb%Gqybl4DJ+~6s%<P$oTVrhFa#z(&eSK$>-;9_3*q@e5%R(
z>bKY1%KjdDow`-;eQ_?A?xvpBBgwZX_sqMy*h=Z%jTpJxVbX`%r$zjBRI>~y_y5&i
z^J?FfJ%M2t?v%~FyDns%Ri{(XLr+!@t`qjlj7nZ!lUBcAE|PR?yTH>j)76*Gc3#n)
zTCx5=gWTOydn066HeJ-yX`RFn7<7J1@<C(4Uuzy2N1xS8ELvyT*Z%WqVAPt8b9pDR
zw?>qTT7<PSa(dc08Z1<^zf)hoZBl%!<z(jhfscKrWgNT|^8U(~^)vFE!q``M?O<~7
zV|o1Gq~z~c87;hfnjL=y*6wyObt<x{(bAfAF>OKN^|jx=t-M#zx@*}ars)#G^Dq4U
z8aufq`pEM)s{;E2*k8N*A6d48Vcrz&{lDV3?3$ReW?P<BovEsd(H4j3bcK{ToyNz{
zne2Yl-}1I%?*5xkW8_jBFP(mU;m@m^AC4dUUSpK-?Ri+;_oW<P+!OQp=3EYWTh($?
ztL(DTD$z|#TUF*v*>TgRaDKsi!TF;4%@w-O`FMHy<m?=JdiNUkulw@*@%)Y*<w8fc
ztvP8P_|NjwYHhDcx93<+U2u-ckugX$Uj6s$<+qNQG~C%g|Lp2HHs^MmJ-PPVZ`)E;
zKR(aEFIV;{RH?K1x!)=O#5R5H&B$t*a{trczb|@w(d6CEPiwdQa@5v#UYRrhX#?Zu
z+nt}S%<fK{aOlDBPb&_&bQP6cJK!bF_*J)6*MP~ZgXQ4QkCW$QJa<|iS9z_VYXM_?
z@205>oT5WC)(JUHH(=w>-~Pr#V!Qg@HT+sLpQYv`XPujNyL5UW-{M(bJ6Rn4>Q8?%
zvd_(xnQ)?7Okut}*USiw)1@n}XKKn7)?8X7CK4h#<D5$Ueb1#2#cpps_NHD-%X)(N
zF7x%*rM)$Dt8|rSY6|c%39%eF$A16smwU(5gV)S^urTbg?q0`hMH*2Pxg4LKW_a@R
zoQ*`^t+wcnGdr&x|M2acM)ic%yRIwj+ww|Rw0ynmDzVEEcRmN5P@mrT{h;i>n%BbK
zc>()=wg1{~ucdV;l*{w?z4>#CUOh}+vO<f~YtkYYMuP^%6B7NxJ>M^AGsn2e)kXfx
zz0|uk)bi3!^=0)l7TkKfFjw_>3{Mr8f=KhvGfjrKjg9ZiZ(H7V&0?|9Ek55<tv0re
zmJah<W!~6rH@_s6a^v09HvzY_UhNgpe5$%KxLr}}$o!n2o$8_dT|eY+m7a6k{J`>h
z!VmFYXXOq(nGLe6k^i=Q4NuD3H^)3L@7do&J8lb27Y=LLwq##GCEM=LJ`=eP+UFK}
zzf6BqZBe{Hu2Oz&si&Q!v20fMxnn<Wn?z?ljgB}Q{bl;HxshL{OlfUtZsp&<TUo+B
zxBPu;o$9O#m%w#9uJBd-eIciNcyXJJ!TW%74|m7D-}>(A9nHfX9yW8-LK=iD4((<<
zYWRIp;bOrDm0SNFsN|a(v}KddaS!He`(8_wP0q{{({^3Ab9VcpxZtx(pC@uXX=vwA
zl~i&{m~oCJ`+k1t^v%5n^;aypb{_xmM`*<)pI32nH9dS6u96cudsS(z*0ZLB1?qP^
zIGRFLmK!tp%~><)gvp7rl@{%s)6|4M2fCHzzPntbyEP}UD?HLUv+!8uj#HE8XV~s^
z7xFMNWZpN`o~7mKu3z&n%~qZ?>r?Tck4JOj-R_0GjFP-NMLf!NaaY!b(`zq>>UvJP
z=hE>erJVV|nLk$S0u~dJ{k7J7<i2V8ddh=}$5rj?GR?YNgM#nc%e!q6HMk?n=6TZN
zt)bwff|4lKsfWcc)V}b|it(2E{Cdu_1*@~-6~yI&4|+^#=TK;#Gx<J4a^RMT$qKK2
zUODWTbmQLAx2rc>`fS*zvbZT^or~tdk9K_S-;@Jl&uUJv`kpcS^wS7KiIWShpU&Df
z^<3HYJGqK3(<=Nsx&HFFzp#0A{@@csvz5n|e~>x8HZWh0OK(ki6_?QV=#a<x>(;HC
zxO7FmexnEH@roWvBcBufR{2NrrxmZfU3h81!xw9MF7Y1MN_e|k*GlA3_my>bgpX+I
zum0H`B2mbEfW^~@KUZVk$JqLvQ)D^pdo3ScFL)t!f~ThPcW~QDpW9QztxlVUminq{
zb7kK5yl{NgR)tR<1`CB4dnU2p_YjM&P~(gDw>^2%R>RNW#Kw7Nzu%s3SCYLcGX9TU
zcHY<b{}j17y<OKQ^$UeOieYCFFg(Vu8~USnrm^JWRE3}O&oF*n7B*?iuGnJ}tgdnQ
z-C2CcI_kw$*=vsr_DQdrvf5$u%8>Jy*mklX4J@l{<zt<6!;nQcn8{8gG4kTq@B8Y#
zX8k&vdm(paO;&b#Df_xuhgWl(3ohK>e(n9MChZr!tVz?J`rfz`9VBot^~YE5ITJ29
zOj5L*z&GQ*N0YOiQBB|cWq($d%yE)Cxkc{b^XG9N_sx4Ro3#5&{Ico%E4WkpE?!-(
z>$|&h`Qp?JFD*VVvF5kV8(C{?=bU`3&}*=5hgRb|_S*gYhhN9Ice)CD9OB<m89XyH
z;+tNihtW!wNy!^e*gsWxv81*CdvKEdTum+)ftM4?ecfz6XGU-pc^#B-)O^D7r{eE_
z2H)(P9_>+cUp#A-Ni18@wOwe-p+k<EEUhYPK0?X=8J0Xte!J|#ZI8rrSr!onS6un8
z?zf2v4r)0yAwg*6{CUURYb#u*aw<HH%-){&;$~Ilg}cRfEOn-8sJc(z{b`cTyaUe)
zKdzi|;)qH$<EQTno;7by_-obG5p<hRb*Ye0)1n%c9=~+qdB3hbSyNw;|Mb+>K*69B
z*2}ia1Z#SF22E_4%D^)57RT{_0yQrhVz*9_SQ!0g@q5dt%c0J3vpbv*E)Y^SJgLT>
zP<Yecfb-+pwJNtaYgGF!+Zw#^TyyB@m9w2I@@nt8eO=ZzZ`~D*LjrGIog}^WijC$e
zh)xwbz{0?vX}dM4>(M<^+x;a!eq1f?I&RV)@ce~+)W2JMg_bT~yKG0E`Aa=tqm4hR
ztirr9Q`MNRc}^FZ<k`b`kMmq(oZ3#=ruP>vEN?%?b>eaHpO-#oUvFKdrxP5nQ>qh`
zce_Aq!Rlz6uFqVFcN&(3IIzms1u8gabMM}1w0Yuut4al}na{RbZ!Y&$dv*KU?NgE~
zmgGJ=b>-4j)pn*GYq$CoJz5Yvzl_m}>0797M+0xr@p8$Q=hORbsssOWzm<rZv~|~#
zrA(qnz8z?6QkZQ2FUfp@KgXLbPpSjhM2pwUPruZ)HuYl6Wyb>>CJRkG(Lc5B@Pz)}
zs%ua0JbS>Odwy?P<PAmc9{ITilVAV4S#eiB<gQcI-P+x^S8Ci|w0-HU@=GlXw)@;U
z5phywl7~wAGl!p-AKjBb;`@6BSLVxkrC;(7hU?9-GJPMUUmT<B6&ftI+y23%t^7V3
z{3-|3J7o{5NC+!`c_YKz;hZ-~Hun8@#+bT_iEUf$zO3J6cPdk=kdr(2+TT5@j2>^N
z%=|Im-tyQs@7Welj54my{dT9eY^B0x{w+l*6>tADm@utAy=B_h+xIf_0#=Ai&y4N}
zR9Nb%DbtjeQu{otj{V5BM+%de_{!e#uVh+PFI@2Ap`(ufy^jq4O0~CNJA2T5cFJW{
zE)7l_=Jqef%KP&khm}=G#x8Z@komar>ykZIcYAgOG1#2@xx)5h!dfqZ#~sU^Q&@Ff
zQykQ7e|O5+zhAQUh;pwO+k11{oyi9tSG~B_ByvnxF6E!)Q%5oFueTH9-rin!HR!f^
zl~>#R%uMAw3pT9$&LjEp+rcIitGKe&okeReWZeF+@_j{1K6}Os4*N$Ox6kbju2raY
zx@q<E(r(+YtCn3X`?6Q+RFXsCcK0oPh7%iS*!jOOc`E9kcPsir+RE={Np~jMy<BY`
z=(Q~I^2>#bQf)lG_isrvRO!F(eAC?`;lUHRnEh6MNv;RIF09V)(DcYn5t??KPle;i
z_nx1!i=*PFO#R|_EBZv*lD3MATMf=n)h&Pbb5qH`W2>*e6cy!_$lW$0XYKVVOBn<j
z7}!GCYnbY8Nu_V~6bm}e``zRIGuEnSPZ%c^Y+=5<ZtWfOyQlWu_uZIZT{Usq!&A@a
zX0)tXAkfD-GtSRF^Kz%|kp#(uXC1TaHW?n*>fTiHeATwtZ@V8?pSbpRdfc<8Msp$+
zS(p#%PIVNre#Eu>xo^eN<2=9hI&KSneK)DDbm`n9J#Sa5Uz%$^JMTm8e}<2|yK8be
z6unGLMIM%%YElwoh^juWXm95euqZQN73&r0mCrQqgzIpew`Y#JR{KM=&~<WYblk1)
zsoGsm{Z?6fJb0~d@A)0d&Mt9w=Gsf9mlaOfRz6xX?XASa%CE~9YxVnTx;@_s1>bc3
zRd++B@$c@Jxn~5XUE{MYllPh>Ca}(5IX+iJGiY6Qj*`(z&m&9HcfS8_zis*+Ls>)1
zck|M1{`QJ(SMuLu7<p?&`jvpfcPcupf`7eK(vlBY$(?m9W5~TA^Ox1r<^fO0UhTC_
zTUP2W_@+L=NX6&CyXrGPe!mtts&(Rq3cv5Gh$W2GTXk9{EW7?vL)B7|lP$RKKwVqi
z-LG28*_Rhveq5X<v8$n-ZPP-%wMVwyaBO6ZpBd!R{7KFFwTDoo#KHjgzRD0yC6>eQ
zvcE0lvzzD<R=F&QDaEt-{+|Wzzh9;v`LyHGg2q1|7QKBE_@7~7c4d>7$OO5@I}<Ed
zpXoe4*FRyq&+W|P*Ix(S^;z@e(YEH+R0EkxR>8MNw2Xg0GCB59PWAnIyFDu^)zov+
zwl?Z+d#=>7Fz!(jSNp5NkH6esuXwB6@yzwke5si-tS3#EefTnMrT?-OoZ*b`p72ck
z_HWJOFIIAqTlS|dPkyHD7T^}PSUyPPR#U}}84pi!N>1jqxBSnrtb5Cwg~gL6Ki~g0
zP$zr2?zx?(zXo2M`c7!=+0134p_=x;x{4-ocPa>}+8$Bk{2_1El`-?&GO5`Ia@?QB
zZJp(!E+2nC>&dl$i&iaLTiDfA<re8QrQx!MhO>ajpSedG_$L*4PD+uS+st72e4g`_
zi_<=r?>_8lo3*M`eNDA=!nLs5ukWlYymrgG+bi0;C`Dkh8h-`*`FEy!mwwT?+rfS%
z&u*S+`NkiiPdiRrTETwg$=~ehv61(u%dLJ_o@cJL>Q;5oRk2sDJw^?QnvIP;la2gu
znP&GobhHFHKJ)rF`|^)PzYUJ0I(`%>-uSEkinZ<BGOgEZeY1U*H}mb9m$f)0!RH54
zLV5H3n8T0icn`<bvKkzpae(=0czwH_@mxnfH;wG_IIZ_zAFX`#<&L(O(Du7uow}|h
zGpn3GBmd3n(&=W`{QnHysUM=0^z3zSznEP3w5xsLs-o-eu|B6HYg-r3y7r*y&~7=c
z8A^=rn<tdGOrI6&-mbT77gNFQ(%x7{m8yViPuK0dz3zRk^xn3YnY*{lk9~hMcx`)(
z*CZKtwQDMTsw#p)Q~l>gFs->D8QJhNNNq<?3|qTGU%SO$?RR~5v+AzMui)EW?5W}K
zXhG|;bSArxOm5Z3CZAPg_g8%EUw2i(&uO~omMLK$gFXIQu31|u?6<@!_~q&;lQybI
z2wD7R_;GdJxr$k`xpV$%%`MkGboR{45BEQ>3GuUaIIVL%SaDe#*B473&p)Rm-x;v&
zRbH|&Gr&_$q(tbpuW-Aho#a0*<GTALzP;gGtCKFKe+@eRXpZNXX)lBVrhe0Y6j0G@
z;AXN?Pt^U}LH@`88EjV8?!6MYY(n{me{~^due*X`GFPQ*#d=I-QCP9iKw*WW=0QIX
zW@Vpeo8$Mq-M{Y2sz*=SuPkVu7jbKw?$*$0UTv@J4^J#w89iazJkQ|l3Bg)S9G}0>
z=ym5=v3{|#sM4KBVLnM)Uh7}CdV1~hR);Ru#gk$j<L!jz{W;05W+<n8ZqAYoOA=hn
zKc4#<w_|VeYA#c`P}a1h-Xo#cui5+!Ui+Wn(;c;#$R?(w0+xznY}T*bJf5(JU!JQq
z>Enf$%2j#0OSkOLs{EK<cs;ssSDvrR=gdhrWIEVY=KW__a5c}vX<tJG<Kv#%tZIp+
znWj6~uhfYgSg>`~^{p8aTdo<+vf9SnyE1pBzRss}o)eGM{IuUTt>9+plj355+mmzZ
zoaWs-w}$QX_H+AnPCJ`suHBQT^)1Kb^&jhXsw}2wEhRhNsGl!fb6zX9KFdv^g2A+A
zRm3`_UdxNGnx$9rZOy3Lwo*~MaM^`coqZ1frZ8|#m1J^HoE!Q^mv_qECHddo?pglP
zobuhQ=$PYVshZ<|!k6(sjQ?YjT{nNO{}uh{T<`Q%q9qT-w6*4ma&9zwz;b18hiJBk
zz)VGs&st$?PnsS%vng(MmMqsgzS{ls9gkSNm+MQq=<xUa^VV|Cye*<H<}`Q(C9&%r
zU@&Cj|MolMrM`krV(j0I{ijxG1hNS{xMuX}f*c?JzfBJF*jzsTu;@6`^XEmUXky38
zXIYu?0TR!hPZzRsdIULlI(am?E6zJ?c(3jQ|6I+D+G?@K{)w)rc=B%Lr7XRrXI6Sd
zHvjYtlwf#p&QbDj@)@_@)Lg6SDSv8z)~tw3%Q&AF|3XXfSF5VfP3@2(fga8i@8{d!
zG6~pO7APUH_KT7ld&Py<uL@b8OBbro3R$1YJdNSR8Ck<ENz3;X)wch+!Yj4vdjwNW
zmf>-Cb>_88TqZ2l?%}&(U|qkiZQ3KZ$uk%#yQG6BS^FeZ1g%~%``68_D~}3=N9^7k
ztQE|$EJHWL$lX}(;*5izd%la!lNXrEbK}G-`$vs?cecN4JNSkD)2j0PqDSu^zBA`q
zeO33RH+O7ssrU0!{>c^%4$T(FkInJdwCHJI-)2|yZ)?=1J$E+z-M8i3o!H!c7ruPm
z)fVn>?3rt^*{fAsH;LS?>HL>3(-6XxJmC>nhjB0Skq3<&Lf?xoytA8qZByY*VJ-K4
zTG<O_gZI66e|Kei^n(4Hu3DSw*ge&XD7Cq=M3<Y1rBCG>O9GGJ(tzlg!}8_4kNcXJ
z+e=MvkLA44^J#hA`+H$$x2D8KP3W5S$<<cWID5Kwyn)c|O!dz`<p(@f4hXM^`y1-5
zk=$2t&|W9F`k_en=J!!^^=?ls(_Ndp?T*;g@JYM6mR(R#d@dH|@Fqmng{SYp%eias
zZ9U2Qrz7wOqtmSelj^6jFVo!Z)-rvXNKDK8O0KB_hDO#d6V;#mxRA&wH6@|mCS#-3
z%UgZhmKyq&8hZz1F5y#+Y0wl@Kf?Feu6V`sSr&e4T4O5eN~IotS@}FB^i{OT?Bgr<
zdfwftvpXnH{MPl^g|i#Ivvc*PT?!ID!DFVv+*%~jasBL^<c^rrMei0EoVR7$H}OY_
zQk2qm`>3U%I#xfO4`!I0Qcz);5h2@X`$zbM`@Qs7Cuxayu}_j$@)lY?o1x)Q_Tl(*
z=j~l_CP(;VwHP$7G`6U)3R(&p**sz^(UF~=v+ae+sxQe6c5x=oH&!jrT#{0emsuMn
zuxi4=j7i(tSN^#q6nI-+VM!yqp#0uGJJlQJiIV61j$8BR-<xG)z;WQD<?~syHr;-y
zDeyCG@=wo*peb7%QY{Q+8z%m1W)8F1t?5wi|BksKWbWTRKZDAfE~e|fDw#L`VAOZT
zAK&J2KMH)M(dVJ{ZzgB+jX9Dt*jVP>O6Fa)*<_+y=((G>pPqjm=e?!>w@$fDo4v4y
z>{XBJCVbky?h2wOi<TChG@PgK@6^wJtJ%d;4>TtHX#2ZzYxC+20?OKzJy#d}GXB-J
z`>$>OZ{N_^Qs>=HdlSXkx)vC4HER2BD-=rNv-_m|RQFo=&g7m4<$eDdbVAp6{LxEo
zc_q~$^Ref5_^Q84cjnxFpLbh-n|r3V=KNGq-j!RMCMzF2_^7V>k+N^lEw!ADER5&v
z?4-ka9!>Mv^vyXZWcHNPnOV8VE(pjR2%Pd=g3EH^?}WMuihEK|B|JXR^Zdf4FAh7^
zmumf*yuto4x58o`m8)itVx6vt1fTIwp2KOEB2-s8twlpIy!i2nDlWdaJk3=hCCj=V
zZ+y4icW+JNv70Mb=x_SZAb8X@_~+EW4gwA;M;;u>HaB+Ge3Q!Xc>UX1*IT{j7qduh
zUSn-@r%;ApWOb+gzT_|GUoJHd(lHW|&~iJprpv)`o9D-cwWs&$%ubvbSi0id4hz*S
zOkp1t4ha4gsQLD7{@jW^UJSi#{&w?YUsV3+a9rfLRn1&I;#Xm9X4u;Ed29Yiw|rRf
z#d4dG?U5yk4r~02jBlR~+t%aW#*%(ssinWm<-v1ix$1wSrW>oe^B1mLv@!G@TRCg%
zlcgr&3?G9VW++bI$5R(NXClMFYkeD5Jq!>3cW1?KKSgO7vq$qYzg$~(>D9r8xV{3$
z$<Gai@BiJhHJQ6bA|<Z+v3##lQ_8d7*B`mFENSOAp0G%LRx-Q1qRaW_{<#j<KDn!j
z^uD&;<zc;;Y1{2WkBFe_FVwtTb}UdUN>|@F|MuIp>(;8A&{wm!I;GvD6+3;Nmy`dp
ztu2x=UK>xy9W!SC{$4;i&@<P1j?=IH_IN9$YwmnS#vXoaYTa_~O%v!n7_lpD!UI>Q
znTdJYJ^Te;Y{_bcH?rr{xPCl4pF8hR`6YhallJY;W91*NZ(6qBDI~P<cKGkKC!3=4
zuTB5YAeg&$sjIVBltsU<CyR&jtu>wVx2?Y;_BORUbmxg<#XbDVlSAb+dA5CeF?p_k
zE0avn1@VBuW@FpM6F<w#S6taNBl*bYojg^^^~aiwg>K~b&&^qw@ii!9CBp=!PhKv~
zw@;jZZ@b0HyG_L~=hIn_bIz=BTR%Ks{$lIf6DlVrsR-GZOteZ{v3}>xUrE~xS4e+e
zak}|r;`S`LSZg2kJ7G&7IGnU!BKZ52N@Mq%;>b*cLa(z&uli0+Tr9e^{B+gJol{hO
zSLn%{VP%|PyX3hFv(%BmLo1ia9(;NFb+l4@;nHoUNBnlwi{45p?Jv&F-Ja>2?;}-f
z(i=OeqtsO6Nqa1>gI2;C^^QLmW`3566+g1-@{EaYmMQa3e_z+0|JPo~@mp`~!!zNk
z=FhJl`&_(Y_Tz&4@7ODD+^u{Xe936TZBMyJ_xY15mfvDD2=SPy(qy!GO}NxS9)9N&
zzp8?F+DUEwt>qE2^7HGMzkA;QJ$TjX*Xp(Z9_DB5yfyVij^ehPf}9?LHbFD|it6J&
z-rMB3D&t^46<@qoD({vew~cxnueZOC&^EhlvhJU4*DLAOM>*V=`t9H}oarga!hHMf
zf(MTUJZ2Uh_{mdO;<88E?QEOe{JlIe@663bcGqs3vF!!>RnCbg<rsgSDL>cu@Yprw
z+ozUJ`7!_2!bim{{W})anet~AefA9a{GG34=^~aLwi};SR~o8IGW)1*-8lElp<l+Q
z|7GMcFWE3*ef4?emo6sKt-M-WS55Z$Ysq?)X@R5s$|HZ<rkp)=Ri4lE@ru9M8w<K0
z9RIx_p`zY=k=E(kv)312&5M0ry>9Ql`Cl%`?FiAmKCMMTRWo?iq(lW3{+1V!Q%hd@
zPAOTK-spO4hqvGZw=4XmRSVmn^*s)Lv1yVNtBVNp2ZLnwFMs(uw+E_3*5qF5+P!B2
zuS%_b75B=Z+tnWz_8;A~wIJ$l$SMv6_xZN}lKqq}Tzqj-?sv^f8=cVIo)XEYt15S2
z+WW*Rb<?hi6*r_0UA=t!Z0M=B(>vssMJ(KPJ*cX_J8ACYH^IGCYg!o$)zl4GKJMKt
zDS1AvX8$}pJ5htj0g_Q1>MZMwmd`l9Jw|@5?3dTC`YXQNUL@=Dt9C=l<b|h}-u<Gr
zV1J(Vx_hpre+*_dyb|<|dDZ{mki|!?6qPAQPCA85uDdMIv8{;Dif?}4{PlVvyhk)`
zI(5z!%{;))eee0iZR>SDS${qnQ8%;JqttuBU8&o)V)y0!qn*xg();#$^{ZXBTlXJ&
zSL?Xx+OFxBU5nR#Ikaxtv8)NxZ+SID`>8Tm9<<@R6ni8%xZLcSy}ev?vcz+5y&Wcg
z$KHoU$n9=$t9x;d%~9dURi(70PVc>cuw})q{d&7nD@922MBtn11OFNJO<Ci4OIz&5
z*A*6PKU_%RzV~&F^{jidZdEUy$~1A+bFFDhEu?~WdZ_vdKJhDjZoBpQ&Yl>yw9}=`
z3AVqkMhcp&*<oFIY2WD_otRZue!p9F*{k8xl_iZo8fu<ANbp-{6tVf3tOxtLA3rND
z1fOlb`}f7Erk%684ryozyb`$;G=XVK3M=ysj>cow$#Mp)?^8uoW9H1aznxWo@OiQk
z+v|Br^QUK>j=Ou)&ul_qw4YSRgw`vQ!j?T%-oPN=p!_^Eqe5`vVRuEl$Yb_Vw>xKg
z=PYKMY+`=v_1RfZJ#)8ST6-jfb)i57>+yt{txhR@Hzq#jNqKl}!oo~Wi`R4H=W0y4
z<UV=RG4|f_=Rdq_)vtYBUvb-N@m{mnCbG7-&TR}>D$Kcltp}f}*Aym(9EZ|Fj`C9~
zmA!N-%6TPbO1wJfd3EEbgCgul_QY>$ezTzIZuZse_o}`vmzQcq+MHlneb9*im7j{+
zn$58-wNZWR=9ZbBUfLUz%689m!6aJ^&Wx4m6N3CW41XreHoh_P+cHV*L~)N?^gE5%
zEK7Z@?y3bEx(n`$P1<;PQkNIcPs^Dnk2E;4N9i2B?W|W@wm8XQ@53`!wEMPnM2Vd!
zk#Rh=T!20CHUs~bSBXq6*6syT3!CoQT@DVBjh+*9Q!Ch8U6W<<1cxC11cTBiQyZ?Y
zV9L@|+J1Gq<qEB>%2!!fddf})&OGQ>f5!dW+m&l`W73=s@bg>=wpyWP-1w;K3TKI?
zr}tD2uM-SYIu#rQ=X?{qFL`Wnn$=<D<jbEIu3dcP)}54Vle;U}!&)7S0))idt(lD{
z7#KdbSNG9f#&}@jC;rpVgJRdLI=J|ceHGK)-+ZU{O!n!XP`m0@(bUbsKI^y>zDqW7
z1k5RKd@Pjl$nda+*gJ`eqF)!TmmPCqS?9l0wcPH&t!TBWyZ#=o-g9N?ap&C2tJd-=
zW-B;xH)=LEr5rE5FSTKw+Cjb<S87iFyjt|U?OR}5{k5qY_xi7|db;Ubx_i#`Z>u<5
zBdXT_%v;>~-R@DNO%a2-@okk1>pG=Ri``x=mg$i0uiK(}<)RVmv6^#NKbLN}^d&y$
zo%g=0cecy!F8w2Qr|hoZmTN)5YZ<*B&5vZQyPBYq(&3+AlELtwp|U}JqRNBnOPx*!
zTCe9^=6-p*%y3W8s<(%x>q-BcHFbL2>VlP7mCFMaHW^Jj8K4m)Ea335S83YEz$M&m
z(s_)sna9~mD|QDN-P@X<ar;R?%Z3(<iWN<s3^ffk9`?&?g(fH`9GClNx$M$OL)kN2
zO50V~V)&On6KGIXP+1y~-YL%@cz@5|ExKn;B;W1$bTmnl;l@STW1o5#Zus8zr81PW
z>~7A=N>=4brw(!CD)c?`V5<3ZZbISe4&60zuO~7~+MnIG#h<Um_Em=DuW+e#TDQ*b
z^jr1!V9mmn(T-V8EAJLos(+q1VLgjX$GLrl^_@Segx5WI@_t+4kMIi@&M$QO7V0-O
zAa(`g(WTc8^k-cO2^PGkFz-m&j~=((A=C4Vs#O^j{xhiN?h8F_x=HTU_qzJ1j;h|!
zwVIReuRbc{eT0?4^WC}wJx6MPyUqEq*YIFdlSBRK#LvfDU5m9BGC7FyH{QS3tiR1G
zJ2shP$Cj)+47z@W50=c3pS+{-mfi8^fe*Y+a=pAb_viCxme(d6RN(IY`IT*}Z1RVY
z^}*Zvo#q~AaY|Y17{unoqw-yTjZfB@oF@@y3|_{5doPrgt|J|kAm}X0ewoi?l~R$*
z)WR9U?_6BYAFyXqcKK*sCRLqs$^HtP?I-i)0+On>D-Im@`Mhwt+0n4qMYHDJ^gAWC
zlp&;|WAZyKH5CSqc|U%?s^hc~Y~!0<a+v3z$-P6I(L$WehA%jl#y!~bKDN79?Dl4n
zts?&xO$oX6bE2w)s)2#Q`uX?0->G>|Et^{HwEXad&mUK>=i5>-L#Xn{=fB!(?GN97
zZE0(@KdQ=h-+SH5xx20l-Cg)y%)$A>Bx&QHli$Am{8)H}UVgG*n)#c{Im`{k?_<tC
zI^UkQd*YL=PK6J&FDhqd{dBk*%5)<-+d*Yp(*_|=g`QKp8R|1-k1Y`t;5O=+S8?+8
z`ETp&g$>W>*p^HFEM9R{%}Kg+TXb-m09Tj0k^{pJ$<NJ=Yid`#zFGEtrC+PWpVf}1
z%DTMQnV(wfP!!I8rh2N7`<Xdq*KP?MlXrT4Axv4Lcg>D9PC+7zM7DJ~H3-}Esr>od
z({ES$IBy-#jaY@;6X`!Pn(aKwZ2oBN`s)y_w^h3K^wo{~qn5T*Wu|I9KX9V6L(S9n
z5f4vK#&3bi4DU+b?l@ujvuMelLKWqlo=?Zje_rdm|EuWs`dgZF_QrnKefH}yhpDMv
z4425m2lfg+ikwEtZ|3+scCMDP4Dng<`QFwzo=XmF<=@WgRyBUPUB6&S?BS%i)%R)@
zU3sopIA0L933%l2#8B?#@9)=+cW%A-CbG8CzE^G6!(xY`C6_-hs&%QZ$Xy!!?nS8F
z>jn00JB_voEZAtU!^oKVO!?ae@w;h^cNqGQtuqt9*^nLk^8U-E1q-CTY&R|D`Yupp
z!*TrkU%%=5c{A(x9j@E4e8Q5v+cJz1mrn>WNj_nC!zuf(&e~z#e};@J=7QN0j@##D
zES<spM9Xd7T1BtaNlTRHJSo&<IKue&>%Uu!jO-g@m}(g1TeI`6v(=?TZ2vQ)z14l_
z@t5uChRE;Cer8j4-1X*+H+g>0|E<C!zrDwPs0bas;l<4SZz=o7<u=NXqEB#!O!inb
zTXoaT?YYxV`>tx6)OFkELTa0okKz_p1qKES8HbN^lWyJO&~2ULWy08arhh4SVYRY%
z`}@i(mz;#As_I={>mbq>{7(KKC!0ZavTxW;qn$PKLcGj{(#(%)eullyW1Y_J={@15
zO_0J0#wYF__CHEH#iL(No<2SP^1Wr-cvobJHtgExnxnB+#pm1nk`r<ce+}FgPtkFh
z!LK(rW2Q*TWKX*tG4Ed7xV|JpYL#!%&8gL!k8j!|(V*~SvX$Afr%LZF4~IqFZEjP}
z`uz6VyPa1`{+)QWK0JGNR_Rw$Z=Xv~{dOLi$uh&^(PU%6iTAC<K0V@|;lz`5uBiO>
zm8;#YAFj;{b&&|Q`IOzgyJ>%U?8P1Cxwm?sXwAH06~M)jsvWa*lP9YelZu+3%G|JJ
z4-5C6TDpTndUi_w=k-f2EGc_n^km}3gTJmOtlfJ}^;FlaXIC_t{yN-@V0~%)UQWoc
zzd=20_H1nn8P_}`cQfnfmk+IoarZWil=%HGWc#*z{6D%^FLRPy9Kcm5)8WN(BJu-E
zgZlQ`2&ISr88UiWgZp#1Jd+L?mCW6f(R6#4c+|8TTuvF2maM*2`=|btpHZ5_`&C!w
z2r6|hkz8$h<Y&OqHrvekTfB3{<ySdzg*j(Td6=lqd1A)oU;j2PG1ofU<7H4&pdokb
z?hWf6dHGwjeotK`H9IwErBwE;ncO9UZ%S2CRld#n^tbb>(<!rUyDlA&58$)4v0ZgP
z_2#Lkhtn@4)MhQYuX)+))s@p-zD6P*x-%G#Cm5+4tMA+Oo^KLof4bqppU<Cr|CoEh
zc}2>+83%q|xjOx0p5nCGU(_!Zch5cL!(U)md|dwR^E*k_5s$ScSaSDi<p~~Fo$7Vs
z$gF9VU9-<_{I>G;+PZhAQ;%-+_lXN$v(#ozrI&*Zll{s7AvT}P($*f|6YDf?e^+f=
zbY;)-y&)SeU-1YGy7p$vgFp8LWm_Km%x`y@`IgUimC4?+{=lbCd?Rf8udKaR<@M~~
zzL<CZQM%hx@2#2g)ql^H+gY)U{A^O%UuN+&OkG=Jsj*FEhCqL()=D-Z3E94Xa-mcA
zb@XUz-CO+exl>r*@_=Y<J+oE2xJ|QG-PE!8eCE`RIsX|nYY%EM%kJODFJp95@tymc
z`8oAAFC^a|KCyA)4$H60U&ngho9nP-lkrZyP;uLx+jZq1lo%|T)FxC}RpY6~88XTG
zz~@!(rJKyTHXiMlG4(93>fvEd^2@LlUtK!CDCqvXDW?mUF5K^_YyGm-EqOu%BTJJ~
z%CRL6?<n0=4C&U}|HOE5X!nZPbis|cS~aud_ubLUyR|ZMNA!hXbAokkWFMSRnH2hA
z!phDW=g-ZVA)5H+algjB=q{map{{3c>!1Iy;_hSboNdul!(BbLuZnZ<e(CT{brGjb
zkV{cR;hfCcwR<|=Oq%!M=_IqinREAeo!F$!YagX~^P1VWr?V^Xd}r2;lHI=H@V>eA
z^7X46JRINo%NYyH-Ji={n<wuXJd@q|^o=mJJt<R+p0@j+n|ZZNcjc~cr8~Q}Uz;{<
zCfB1$0i53~rV4RQp3~nf<y>xZfmwI))5bfp8y~T^-OajqI3e?H_uqR{p8X2{<+<|p
z{D3ljpR%y~Yx0Y_7CpW8*6VkE^zALx_FqGJSFj5`5?QM1qSM;P>ESsw^Te+0`?ott
z`y8KP-TbmE@?-25v%)OD<Wqa+)S8zBXV;p4S)Et1Jj3ekp?Td`tJXdH%$fUoYSfjG
z*ROMgTBb_0^e{3XoBg3qda03MDPQGX{{`#k|L7N}eD?N)q{)|1=iJ}3rd{DzS;_t;
z$=Xq^dj2+3ou{&2wwGEZB>dH4m55n?YODO0NmB%-s2(|?-m-&_@q9h={`r?9@<dk}
zbKZWH?Wq^$_u|GCUgwuVbG(BhA8Ywr$j{z>Lta^S^KH8fBL?4(XXf8p6g-LP^NLri
znkrVF%T@IXY*>?)#(u(r`I()v>v4hp{|pAqw@o{B@66qK;rDyC6JA|G7A>uUeaF=w
z)YmIVNr=whT*E4}ZK3XmJ;$=R7ECn>d%h~V<xu|tHTOGbJZ+yxpEP-_kigHoF<9&^
z<86->w^{|yCbmpnspY@5#=Ymq;~U@lzh1czr|GoL{L`P0Q3^Z@!)L5bRn5|jo;SNQ
zz`{{M;e_xp^*#S~-Oc$rL4t8U^Y;Bwt<egrOa7IvZxuC?Q89THz<97_QgvGL$#b)g
zbzS0E9Khe_@h2-o+`ls7&qoLA#f5KQJ9Fjz^*ZVispV^R^J~yT_0Osf4c1S-y_Yjx
z+k2w5TGBC~#&gM`SGF#U+vn=O`Rly#j+XxJs>9poZ@Lq;@IQlK28-FWdn{8_?N7hi
zP~^yvqE%FKV8+9QjePIdKlgsJ=Ncaa`;p`irs3PQ=Iyn*JMs9cxK*o4_z!L3TBR*|
zLR5L5P~k~8yOfj{r!B8o+brIF`TW;ab5H6VGfkWvc6@QI;fh-!Q_H<qIPMQ$$+UDs
zrMcG><r|kc>>e~MWm^C0NnxX*$0ZL_*~S+Ib}n_L2@hlJEH1OZue#c>-R$#Hmd`;u
zdic%>3*X+K*uQUDyNZAx|G7EN%!bD@wzf@q6qC_*YKp_jsWJ`{eF~iCCzV>e8#hEV
zHHJtP+&r)4U7P;yRP4%&)^XRopNd}p^<~q`T&?hbQy0Vr8f-|`V2thTJhn3aiu7mE
zZnp;Uyof)qo_^4YO3!3^&My7wY*E$PRpud=rhEN9^?=>f;?DYgi4If#wM_o|^-%64
zH-~elmLK+5w^#oNi%h27omG44teP4HesE4c!CF71|1|Ta^xNvf^FEbc|IyCVn|nln
zY58GMw;gY0+fMiA3vHjfYAg3jQC96Be$ihE^Ma3P2q+pKf8Y2#@Pi*)Qrb*s_h)={
z2~W7z$Ea-gp1dS!diA+&o96OfHvM7M&>%8t;)$KgcjkB{e4lT4!{Oq^_|BNdC7Zq;
zJiqV9ufQ1Pwiu66HU@saE&Lffw#EkWCax~kRN3YAd!@R}<QWsxt+eiaDor|f;PHx=
zZ{6es&P1L&Q2IQ2`lR*?OP7XlEDsV|@ZdP3y0VeSiQn_*3Lnv*S#Vb2_(J=68m}W#
zddl6_XNhfkxBT0*7kTeyiNAgsGND~6Rc=A9%8$+_i{^Qczg|9fx8$hG3+57;&0()4
zl+K;lcYm9f_bOYLm!anlhll+=s=34HQrI+Jqr>ZjCUdF=8nB<?=~pY6c74}d8SkF=
z_mn@cZs%`Ni|OGBHQJfit`(S7)cJJLzShhd?TKHd!+3s&GR7`e-sZRSzT7UqJIaR)
zDic2ae74K}%k##ACPm)ncXC%>eE;FlKSh&T=LHL1ybsRz*}5m=wpZq=S2CP`sxvh`
zIGI&5**H7wS6<!T^6g-hrNktL)44q*GOM4Re|7!K{(m!-V{U60?tDJqer~zk!nW0#
zFJ8U)xvOINq$N!v4&Qzk1W)y_f4^Lub7}FK>kK7^^S(b0SQ5NaBQw<_`|%cad7<Pz
z<>#9htcx^#m8w;Lsp{H>g@#W#?XF7w3Yi{ycACIuPMagg_!v&uPpO-EMLfJgbc)L=
zcEkB6E@wMW`}gSm<EVAM4z=4Sh@Eg`<WoO5$Mb`mg3GZbzaxv%dad_4%g4ogsfFk~
zF}z|`8Rnv5b~|_BrrHG``!hbXDI_VMvv%8SoLYUl=j-zyNeiYOWSYia)xQ7k#jGtc
z>wGSCeKq6m%F!q;U{*^%BU4{^+u2z}&vfOT`rV?6QzsU<W!i5KPTl`*ciN9z{c2qg
z9;?o7HEz_cQ(VN~!Lm~QRK4f1SurOye1pFK-RP~WlrrHI-xOQTe^b;Oq8xHGW;{{8
z;=%sTu(6@oN^*VjB{r_+3o?H*ug+0eSr_>-e$}I;YgLb^h@4tzcV%Yh)7uAF9N$}A
ze9oLPqd)%IvZffbsrU9xd%>v1%D5qvG5)gpmPvN@+uF=s`7@4N{HwmWu`R&!Y5dw<
zO^-c29G3?r@G*CJ?XFK|^LZ`1dGqUI_w|n}?eSnX)jC_hQFs4~)|HOzla?rS{<&v;
z{I7b1xW53$$t8dOGw_5joaD6Qc=@qyx7K(~x#_)LX9cTLQ0;4l&hx*8toeK0OS~=g
z%6|rmoxG*jv7LEs&h7x_JV|Y@et{jQ7V!wq-y%O(V7imqf%_&tmyI^9lv(}cimKz4
zeV4UZgeEy~22NC%e5-K&w$0zxGORk<_M%4H<5BCUH|0zGWAk&O%1Zw|`54_`;uya4
zZ0rJw=G*prUTr*<&b2=ya?|5=w`(<@HX7~lIF^0J{wnXb#g*%>9Nk&6Z>wqfBZ12c
zrmR)9oa6b(c>byM2d@^*343{C<0bab>-CqG#J4B^wzfK9em3y#?X7pWN37evyH%+E
z@XdlvA3V6c!WMXW_I;MO-5<-%dt|#fw?u;Q%5~c-k6hGc)a~I5b$nNLWq;|~wQE-R
z?wgx6W!sd=M?F~*<W-(|{B7GU+k3=Nqsi`D)penrJ1SMu(@xk;(NA)_zUYp~xxCkP
zk9NLYbm5PyEpK#AXh`7<j>61vM%xdb5&|Ad#}BW2(lg_HJeQh;qtoj1_dlM^x@ur%
z`|ExBiluvky2>Bwgqh8c-SWfhkf+bbDD|G6nfB7$iA8O$$L8zRYXxR2yK25?s?nHr
za_8%{eJejLVGDQ=WWisYW_Y8Vq3_rF^I4lbA}g<UZ=E<LA@FV5>s{f&v%HgH)23{j
zV#&gnP+>f$T)ueK#)mtKjkz|t9$V6OH}}B(wQM`4TAcOPFZ!D5>9fo}sIGe6<$trP
zruA(6yKvjrRd2rwUpgo~SLn?;{cBpS9405S|7GnpSm)xt%~;L7UjDYj$8!pYF3e%#
zvzOZTM?B|}pR~z0Tc=m-J~`3vK5keqdfRW^>y#C@r*o|m->VvSJIh`CW!M7sON9!o
z%Ti;CN+Tc0L<w8;UM~(m-|?fw(QT*p{HdjS`+n;g1bA_H?ch7SzwVxUZ|RK#%KBf|
zMVD5bwcEk>*ENzmJm6)!@5^nvao2BtJ9YnKVp-^yTbJ_IpS8%@uxhKM$N{Hj2mbP1
zoW`r(o=)@CSiSsS|Kv?m<j#C=U&?o5SH1t9cY3z2@tK>0wq0NI`u#5HfLyKCm6~cX
z1@lfa`M+J+`=)bh;HEz-xJ487q-TCzmUeMlaKPQnYoB~mC+aP^vcuDRsxRvk$2T0T
z>Vk6ve>@kxewyJw!}hF=a^i*(d2x0kmDg`CU0fIwa_#2rYrjR8ty-8r#bmO`n-wZ%
zXJ$STV32K8bxRlA&Nl6o{h#dnA5Yr1X{tW?JNcDiw*A@HZ13)cy<4!+e@}Vn^rYL{
zd2P4OPTuk@TiI+?XxlB_uJwNNn`|q*D&_1xsr0-$c*J5&kBjmy8^-t5mu^c)G<6g{
z_55Zw_jl2r%h#6Je3kj<qR_gaB3y=HLh_f1zxDETMJmG><az!x@EV3sV6}2;ea0L0
z%eL+B{Op^#)$?vW&7HJqDc8#Si>GEdF+7^!psgLQz~nPws`jzfn?Gtiy7F*t?7nww
z_imn*PWsP~J!2E!tH;xyyPEUO={j{<N!LW{KZ8KsQu&8p-Yi+0tLrTz<yE~lw!wdr
z%kf;L_Rd&eVVgV|-Wg5C^9{SU-Eo%5Uh5(8{PFX+$@T5x7tTHXRj~5z+<;fjUrPQu
z?739mZk(|^Dn{&-!Sj_TuLpVgI?rj)^k^w$@_bwHqc+xW_0&M=qT>ct?=S7RfBZj#
zm}}7^>%>i;KCj!j%}MBm@&ttyeFhv&F8p)KtTyg8$-X)H&Wr3>8!P>mUhX>c)aj|7
z@SlsCtS;5X#-I2nu;;uM^m@fu=qgfVwl;lLW$~k>G85)_20op?|L?~|Q%<l|NyTiO
znlq>RN_5`}RaM@mD-#wNbG>D$asPg7akg4N&z=6gp3JM6*o>bBi26Tc2<AQEHKo8|
zf=b^d1&8ym=X!kqB*raq`{|Eod$ygsIMbDH<(Cx;Q&jnq`<0KSA9*IkX!%)%*GoO9
z*zs6{Q1XiBm71(qx3zwL#N?GADUkd?PI+H}iRTWVEQadOKR^8n=u_j0+;wfy_4b$&
zljVC)hH+JDxy*QDEcuq%!DQ!Y{@vF_XJ<F9Ub)iqO5dipdUK+$hnB3bTD1PMzM;p3
zr=bDLhw7KeSp+JiyB6#$+1BfzTxxSv`_c4M&DXY;`JS(P7nNpi_AXSntMy9j#oG}M
zK{xVL)s|kKsu^Ju#PGfHa`#OWFYZ-SKQjEYJbr26=3P58PEJo*AoaF&?|0p2FE-8c
zmKE+_skHHdTxq6+++>I20vvtr8-&=l%ni|)=(pbYb;Y8IC;1G0WoKUgZFh0$Dqr<g
ze`6ikRxSN}^7sihl^-5YJUlw)-TkT@d*X#mukxkE(JHaq*1py|bbDLv!P+|}TQ1kE
z_HaI3c&?zMkGF}DQ(=-@k;k=dbEiyV;o@Ii7psx-aHDp1PT*&gml-P;NJoiJ;1F2E
zUtnF6@SkD!v9A5|4E{5uUt##bUX`jkYXu+2@u|Ymnv))SJ_wv=bF#nC$p64Wr2_j)
z34M<YHx;i~`-pX$Np9kjSl?i+pt(8Q7OfA|P%<{+d@r!@_lsq7nNsTQDi`MKrA!Y`
zyVGHM=>2;C6}R@@{hC?#a_dy3m)pzEGK;CrNdMXX$5GWkR9A-M>5&~%M9#>&E8j9b
zxX=Dkf3b1&;t2scmRVo&_G|sx-1wit@1A`7;(hu}&(&66mHTivlvgNcTS$w>Hs*I5
zS{n>h4fYu+959^Y&|CEL%o@j8OmF+Lcl`WusdeM#{|uJqE2OvG*S%sOU4K#4lTW>q
zlSQ3V;sgtijQgCq<=UDm(UZc0+vE#AiJnPhXWwd}dudDN+MboBCh8XwzfD^2GnKVf
zW0?_~|CGNUf4yAKKUeQzd2dPMgu}@zehKlPkP5Fl_bIc<_E*vO*WR&5jyj7v%{(fy
zfyvo*Q^>S4<>%CI6|+1KI<TIrv0%r429E<4-_O;}4pudCTk)c$<N&i>ZH7_bv1xgx
zrTS+39X*1A1Q!1}bDnupl4JNC0rfq$KkBsSopNZAjpdNAkNNazncGJxN9lD-W!Z{q
zt7<w|ewcoTbyvC6^1OfvQ_UD>&bxb6!tjyJx69ESYFdc}qS6V5`)XDj?A^U*!p7qh
z|1(IPPQMdUU1OiSV@-g@niVrOb1%6_dc5=OndEtl<yCgKr%MvYWINx+{@mA#KD_(l
z;8fCh;l1a*9l<%rb=`I~Et<Oe%twdJBmWsr1PFP&;S^ArP<P9@RQKwh9c`{*pSc*n
z+`qMEncPL|upXJW^WweMev!7-d%HdB_VU6hnpr`qj$Tog3CxzZ`>!9{*?V6!;{BOF
zuHKCQ=DGZwH{<2wc-I=<13`8Y8(HtXcQDKT`a3aib=||;SEq_)KUJRJxqL<LIaXDz
z$ulRhZ*zI(IDu_;!Rwz(-%1H@eBtjmLw&CCkM$2vXD$%9$LlQmI`~KKe+Gfw>{s}%
zZO^fNTef8JD_!FZzN!fmBPKL7JWhGA%9Ew<Pfw#zZj4o=oW`TMTg9xKH^_bc`Jdsh
z&%bHjhu1}Ai2qjG^)k>$?2`7&pryeM4<@o|+A&PD@i}K?_;L9Y%l`}~S8VTGXPssE
zcX2>hrk2Ytv85UgPuBSG9B(Rqyg%bix6K0n_sbSX76=!fTR83Vs-W3=(?u3JxJWYZ
z7p?sGoAdhG$FquWJV<!Q_@wZ$!|7s$oC&q7IZL+%zvT>&konA1-Q!+)elBleu}p@z
zB+L7!{amS>;Z6}!m)7xwT(b1r@}EKHl%NII#CtAGjLQFJUW{F`ilzHML-(OAJuOZS
ztDFm#?D+D%^jfgNqm`L=>aQ)S@N#@Qsn4N9$+7ci<6N7r7gPRySoOy1g7=!Qt7|uw
zJJ0Sq>>nQ_)ZVXR`}R@D2FCvkw-^OdI40W*%z8RQa{}Y?^K)Z#^VoXT*GlYq?H$13
z5tJgMuFH8KJm<Um*GsdlPdIZ*y4@31FD%%)J~(Qcb19$ItsBfniYlKvxa`#5*_H0L
zHfc+d4!N`^d%M@Qymj*yF17XzxqQN<bw@{pnooa2P2YcpT!R*|juXw?S3<m2DwRv{
zXWh<Pv7n{c_^49w-$&Ky@_RCCTzX$~%sea?I#V?9M5N1t{_=z2dCnV{cl7-<WVib4
zBYOI!<*(rCw*L&ixk@XSG;;E>GR<^2e>{1{M8ma*v#jp$>(n1jy89>F{&ACq!mq|n
zQ#m@H8h2W#KYzP+-Qu^BDS7ke{3`vp-cws~q3!vMMR&ryr><MO?M~9lt%|9UGX?ve
z8PC5^F_Y`jQI(b>hl{^&li62tNOG2#kiOO8ko_KetvBwEjJay+TD{t9uYG9Eo|E!S
zY5eEhliXIuXg`=;9r$<MCpl-2zh%p1oF9qrj4k42+d2Jf^OtDfpu3^Z(}FBo8eeJM
zn)%=%=cMHGlA6r#7~V=<d*6EJcFe0A>ux7Yd|5t+zelHYQP`;;pMG6u+BJ1!^cLG4
zFYYZ{vuwx01e2*vQ&l>&SOt|I3*M_q`_FKoV(Z4*jH5FRC%8T8o1(Hr!S??0ol9ij
zXDEu?u6!O}{>S%H(Eb4C6F#krB?1-x?!O%_clK^i>&Mmg{!<MfD2eK<JyWDUeJktE
z%(|O}!RN1TR8rI^p6K$Q!BgRYlEU--FPCoL?(;yKCw=CBhThk8>KT*duI~FjH7WgS
zX!aNH%wzp>4y)>Q90Qb(r9Z#Da{1P)J11C*26m;qEuYk>GrOzl)`vt>Jq>e>pey?n
z6Zm(Ye^tM)X<qc=P4Dj7utqEH@wgth!13ob-J=z;zU50jUblYzsGaBd@D@jy{l=40
zxrd*>P0%hpQ4~}2^Acy?)j2OuuzdZqa_^Q|X?LB2L%im#C{%RmXGmw>KY3mG$I?w6
z_1o_-FFf|2K{+<~fwaojN|kfRI?DZ?Ec2ZyCerq5YDmKiuOR;=Gft>0tg7gr8*Y)$
zQaZ)ML9R6QM|<bi^Z%4jP5ym;S^kZ0^RB*P-1+9$>fE=t?jJUEFLK_xWq*5A!2Tnb
z)$jU=gm_u{8*m;oy}Cwa>JvdX<G$IsUst(od(}BpiK|wNxqNf^?X?%*?%um?>FSO`
z{-qzIUoTNS78G<Vq$$WRXeFD=#1)ftPTZ(@{PiD~$8;9isBN~EA=fS?mV}CQIO}Z_
zY7kqriQ})eo11;C{^YKX$bP%|2d~Y!?Z;jzvaZJJveW0{%;js>1jlB+(3-GdtGI(!
zfb0e32lDnyWW#cQSnaa*-toMLb*-M>-T8jo;;km!oBr)pWZqoO!b{dI3a2v#7-aV+
z|9X)pm@d~=8u#Jp+}DL=>*lsZ&D@ij@wM&A&)9YMuGT1~`YrO137z=({QU2iJ$7u#
znss-BujRuz(iMl~LK9`uW-tB{9rd4KYq{8?y~ZJuiB8)l?ke3Fc_=HI^Vx=F0bxur
zwhty5u+4~gS!sHLucw5$-mdcYc~N<ZGn+EB_>Rb)f7!KL-p^L_RsEI!3^5vJUV5Q#
zZp%5=zS=HtI_1g~-|FyS4n-}S>zpxtE{~-a-}5a$!hSP%xlY3zw%c-c@mkqOmu`7z
zqpRe4@>tpPH`$l<9shOilD;^rI`&w9_U-R|Uo$zL*6-S)l_g!FA+&7L`cxYMhDS^b
zIE~c()>ya%tUAh_Rk!5pe}=9H^#Xrfi>nr<*1dPIx0l-0p_M7Kl*2<ol=HoDlY8a|
z<J<m|UVLkgRF-p=?+v>%RcZg;fO)!AHJ$1cwGSP%T`rJPH0OK2RomxnF&bC9gi^1}
zT$6hE=marOmyV##-&HD4+Obz$6gM(!4ZQp}`eLDhvgcZ__MiagQy%WA8fA9;+uz!6
zua*5~5a4k*z>b;u`&#vjJ!y+pt@^7jw(OEnWufuAc|U&3{AXBFDK+ob<&BApjEVzp
z-7U*1KD8irRi8pv>q>8yZ;xs|$+>XKueh*EBF!}I{xbI05uGz<2jytHUJqM*q*b@Y
zDP)3?!jXo;2A|sg4ncY0J((dcWyiNaoYk?d({$d}RS&%8s<bR*)#5sz;Qr-&V#oQ$
zwbjmB68H8iN9>#%&QbMq{-r6izZc(~xq5rf-wn6pKe`^7B&#Oq?57kT(sMwC!Oy2A
zv$u8kO5NqoeK8xK|2e9^b7}~SbI9S<w^_|(-^gZ$i?7PgeVG$xvvkMSkakP&F58_#
zMFwgL9G4g7Yjm7h-)iqaPpkjPTZz-R_r213_~rhJX|lI77W!ImDKVcmVcP4p=k%Me
zykZjJF4l6?Y;aWQSKij_CwcE$7uUR<@@pg3zI!~;N$<)D(`n~?wq&imUFs*YgtZ~e
zVP;d^MP?>bk&7=Gm=^rdJZgDumC44lc~S`iMO!`wKAPmSM95lwr+wZ1_bace91P<6
z$s_!(*x`=*O162Se`b|4IkG4)a&X41kYL~`le_nyVck}bROWzV?tOOq_onK8)thr=
zud3DUXs;lDQ8TZO6D)E^(rP4sOT1l_w#&mI<@fXXZv)sTZ{e`y_tH<A6(BNa!lJ2D
zB+j&)N~*hWbz!x^TXw5cWwUl(Ymcy+o$;&iMSkF~{WV&pw{15b*UUVq7#i9wBbCDZ
z*DKQDc&5F&Lt&n(=p5GcD`)@Rb6P9UQe*s|A@osTgp;|7nCau4b5EA8yt`HU+QYEF
z*PiX}dKY?5RC{jny2-CrdU>@gt@E-tf5UGFzwo}uC)pd#w#Yx-koff6ezzaj8Me=E
zHJ$WEgn`e7)hD#n^{CxbXR}=?zmDu!vUZDh=cE~)XF5E7eEv24j?}Tp+{3pVZt?$a
z`&aY&$?Dl*?i%kzdN+1$;k<id$39J4w=B`0Q!4p91tlcspT6_m_<k%CcSJhhb^l5C
zUo!B|v*&tNZneOoCxtIs>e`|`E%%wbolbFZdU<CCJWFVq(b=aN!eL<ZQtQKvhf}sB
zv-EbJe|V;@M4WfW`^}lpKmX0}uh|gGv~$7fPR^j97@w;7PgxZnDl`?}mi_(OTTk3t
z>9oo0^chdz&&oesF{NKQ+h^YF<ZJCZTeoo3hD_!C)q17MZoLJ=Dz5@Hm%huA#<#;F
zEppd<%4D3P_w&4w`EAkl!nxipER5&t;&snv#@u$@`JW+X)zT{({~7ee7;=r-Jk->D
z+*DrqNfw)wGcKR7&hPl)=g|eTmrnd}dh+9dDJK`Nskoq++d6I9R5@2q=TxQWXC6~0
zTFMC}B>Sve^6%5_^)=5`wiUT=yP7=h;o`iDZvNBmx2jrohCJAHEi$(=YuZc|1#`yC
zsSNx|#`il9Dhu|^4Y4@3oM)qp$hx`@H-1TNuQ9I5d+s{jLcMVE*6hIBD_82S37qfA
zz;<PklSg<%ijPM90=DYceY;ldFgoH{cs5$0X4lW~7yRt=u4$|E2$kJ<WB+X0?A7V3
zo@}`qyy>>TCRbs$|CW;tOhNq5)cgydr0=NDnY6xkYn9ymt-i&{(ziF}{`tOO?cCkw
z?ta@<w-@E^UYa}UtJj0VcFAu#2W%z;KN0Y3R;p$yP2075r(sh}Rk+fR=BDo5W!vZ3
zy{+2+Pu~7W*3s+HrhI4ao0gR2ExEMqmP~FQ^N%}%CmAL!7ErVEFuoK0!(Mz=gxj_Q
zFN}Bp`B@!);6KBa_b->8XPy0)YyPdg#Wmp(t3?iPd9-P+V^(3JQAkL-q56R-^Dl9p
zW1HigwDY7>@%g&AsVA3lt)3jvcXcV}muU^t+3aUdQr{54&L%3quFmvj?r%0BhxWa5
zPd@$0b&u0i@@1gj6%~=`bG|Kg|FuN!?!F)Lua|b;X605&biC2~%2Usj*YK^=^eO8e
z=dD@#CpVOTb63d82TOz>%U|(JC_ixI<>}qZZF89_Qr^_A(rlSkFf}-T_VwbZyi)7<
zaxU+~8lJ)Vp++LN*UprmA;2_wLVaw6(i@2^H_I>VAC`!VFI^(wy_s*z<Co8V-&%KT
z)|Oi@U#<<6@$TI=`6Rair(#jc9H*53411Vsg*VNaQz^LDqO0PjGxMp5<@46&<*vQb
z-KDoZYw5P#&AFF@c5@|PY7i2b$TCx<$f$TyfyuU0r?)sQ>D|lz*m3E<m0Zt@-EDLC
zt=nmB*ZOPz;oJAaSABL(icw{a@oI0JJik$@`p}{lMIojgpU%}<1#9nkA|WVKap`tm
zUewy@Wn1s$zFy_a7`{B}G@C<`Tdj?n`<A--dkp6t-`aVK!QbcYkEDfLcWQa<T;0Cq
z``7jggZA95D_5?{@wye|CF_1%A&F&@!jI#xoqubyy}9vD^NWf%hn7Lji<O}sQIn=<
z*fmY^2>kPR_n*G!(SO%@vUN<!l=at|s%EooZE)0rK1Pp6F4~ixNGRW(ccgBXnn`Kk
zocuQ$J5M!m6|bMJr|8YN=?be8C&PjHa)z=eF73>DntV&vy6E1Lnq56%yQj0*2|eU;
z3O}Oh=&)p_CHwij$5Mv6k=CDh|NIR3Xw6q*9vd+|V9`U9cjr|_TW0>7r|@_4B9&Ut
zDHdvrCnxp<Z7#ZO7<1Mu?3S$aCJWIAtJLfszyE&4?2obHCR4eY))(^59avqoj=Nq!
ze)+}ls{af<Z?>KHoBHKrsizBz)x@W<EuAX=8S<IE(zox}o;t;i@qXQ2tI|I!zq9Yn
zJ8|rcn^m&u5xd1V^Te*deRFMJz~kHx**9!oaGh*2s&=s5XA%>1UZ%-YoN?EKf7RhW
zKWf%|nD98v#!)oZT-Eky-oodCfq_1X8FI_FIG#Mm|Esd^R_Wo0*E-F`kE1@eUAT4A
zvO@T0NX4QZaj!F_mOr>|(&02gL2#1)`E%As>eedFO+4ks=HDS2XL#V^w~pP{dSfSs
z?K-+4+g>>0X1m}K{*q@Y2j3aFZJMx?HPWwUe^jJ~k*wigtFYPnx~E#Jw(41jRJ52)
zao=jmXgHsZKY7uDZK~}#T#ZjBJDj+#@*?g-T*;L!cUQ$OHF=TwOl%3mRIyJY^B%gC
zo#@}+c;CaK|8Rh(;$)q}$C*E@oBZKFgY1zs;|a$P?Gw%2dp+>Vp1V==br<ZZeaO0K
zKF3eXH_7Lw9n5PAmhqmn=iBY4Zima&vpYpvZ8M}6t~$O>#Drf%cf&p}O(m~Iwa2PG
z`nO~qOzXKYSFqvm`Y7hELh0@%ffT`K8HYAbd-Y1-sX<i04bK$8ck>Jqe9Gq?;Q05;
zl}Ath-;H|P7nbiD0#;ADC-LXSw%ZQxMgM93XK)dn=Vvm{lcDzaGwXE6*e##D+a&_e
zJpQsQ;BJe9>hq_we(sp?_fg}acCCy#3e#u1G;~^iRH>O^sr-IL+T660>MXgFDu0I<
z*_Eg2*cL|QThE=%`gB*9_@w30rzBQnYI%iB@>$EitLDfYj$YG;8@&bJ&Ed~K)z-&0
z^{bu3SLN5fe<vNgvghsWueY97><r2`zn6H5tI8{6R(Z6x;=ODwt~}d?nqFQ$D<(6C
z)OY&KU0z!-S9{5!XBV7xbhviBzI<t(NvzhcQmxOId#Cz3*WR0+wQ91@E-f#=Qdagm
zi!KGFB}jf`^K5zP&llOVYighWM1`7FJ4@>}mQD6lJX~U3C;D3XMevK)`L$YKPA<E@
zU|IRfa2IyHt0}*hw)kv!Yv&PayR<?j(Mx8ZX>D^;NBjNfPrDx93Ger;)OcTex6L^G
z%JQabQ&g|2H0|?bjA5%eU%$V8Uwd_U&9T`Rl+=Q!-`gL>?O5E*n71IwqOYjHsK|Ow
zn%lOm4<{-AXUHx4tnkmYd&7~`%4?##rtIKUvc0f%v4%q5A^yDQZ>=VU9=PE7=Y?8n
zuaoTAOuLUxm2B4@r8G1!KI}XGq`EF4%j3<Lm6;uhuYdgul{x45uIs|ef?tXHYmcj}
z=zqm^f_;bN=i9=9&!c<uCBo7!UdZpy>aw`n7qvLHM)SzZ9sMU1-L>SZC)y?WKa*1O
zoSbntV*Ny=?FAB<9se0@U%TvRYU_wPA>kFo$YrncN5aB(f2{8^lZ1nXuZq8}jqzB#
zdvD^Gev>KFPI)mX{C(bkr*r=PzFXaII$}0j^X)USIrB-zTrtYO{Z61aqej6KCH6g!
z9GME)l>hybd_K8SId)(A>!?_dpYQY>o_hQiwc?L3@7!g4Y^m8TV>gaJYlIvXJSLxE
za8N$-{o)3fPwuzuO7-=nf;l6Xsy#@WdHw9lv^!UFFU7vNxhf)P$sElcK8Y^h!*d>M
z9(lQR@(KP7f4iy%<tJr}mNW@9**v+F^zzH!q#ak37Oh*m>)G;Iwq7AcZl27O5*}4o
zHu?W772R@DG9~5Ie4Scf<;m@>?-}<RY}@W-w(df7so3jOrFAt<wyE2=uFC#XR$~`Z
z5L|DO>$O%u;`tn@R1t6Sxdu~&H7Z*Bj4C;_RF1HmJZQW8aX`#&7oXm{!B;bm&y1h-
zKHB8%{D8H*OMMwEnLAbZF5W9{{`~#Qwc;ZYBDs;Dj-UU&_VJ8kMzaG;x80j+vwW|)
z*j9hVhbn1>xBoL7>$5Rdo!jTiYttYb@!VbPb#bX^Zt#{{VYh-<RE_)(C|UQ<nagtV
z#qai?Pk&$dyUxF~sD7U5;rvHWzw-O;Fz37ftLWBOlXW-N2o~Sz-}ZvBoS{iN;k(wC
ziGPl!{GK-BOjhg1cCjw|-;xrmC(dm=X3rd-_@Cj&^A9P7(e~2wb3RFm9=W@xPo&TC
z(1vX<p4@N}=J2q)5z}ON=MUdcsVDYn9ex}KJNJaktbYBhHBQjD_TpvbwRYdOyOsa@
zu<=s-wzIQVp7w6d@alE7&J3EgX;Sd3T_P@hA6QPvZcXiec+luuXF*5>>y7Y3^0IrT
z*@Q)S-nNwfVv?QdWa}R1Bp%W4>*c-o${v4VwQ7!;Gv+j|zAyRUf~Q8YCiCCqg!*|?
zPvt)2@Y?XcvM1ohiZ9u-O%p@;R!<0LWodrumMlE+li|b_%e$^@a{bnPtFMCPNyW6M
zr%x@AaXg?}VAFT_!`fr7Vt1HFI<0EIr;$B5xyv(ff~TP{3!}Wk_qk5GdOW!zIWf#v
zQsiq_%+6kxAR1t3f38F=>)sa4=c_X{wPoU%=C-tTHYwR0;rRETp;~qB*)su~&fitZ
zZ9IK%bH$~JH%{z|_;l>csypFQzm{4o{b&%hVcLPI%M4k5o_zc5z{OATv$wQ(E{!Nw
zo?ZR#%CS>d-_91Qebuq_ZmGzb{|qm@d99cx%y3ZjR^pt%z*7BBjpxV3>w8>PYN9u=
zx^aB`d^X~7THI6Tkl^hRmtRNwhuX3HXE5LCFC==>CCSlt^9{*M|MaF!`}Sm4yRDJ`
zz8fDG=~`@Z)LxSETISoQ7a3MBb8FkhR<XanUC5@eLqKFfg2sxDxAq&~F4<;d-M-*$
zTFd@r$Lr;_HW$3_t+>UYtE#{kY4dM(z}oGl@3?EP^2E%~yvBXu`-%x-j>^9#IR>$B
zv-aDVn|fF;^tR3#yLWsJU!TrDwQFws*Hi!QvY1c&HczGGi~UyV*lij61Gg7$|0kJM
z(9q7<5#Hx~-~juX^qAjWoQb>Mq$aTY@V>e&{<ZH|?Xq0u1uL{&?LOSt{yXj2M$O-<
z*Jfr`w!b{TbD|Mf^>>4cpZ2m_{`vo|n3k!ecl_YCKV~a-Oh2+myEEFvoW&ro^hN)R
z;vYT#8IDbv_HSNkWJp=<YrjLm{=w&)Ltn57tqf$*sSBEvRDFE*ex}U4MT(E!H2uA|
z^N;pp{a5@4CLC1uO;<~P!SMI>8o&D*>w4DyF1oZqXj0H5E<g4!-`RIE|1F9AyQRxy
z|K<E^J55qr59PTVo?3K$$7$7E-6z|&?BtlrvP53_jRzb5new@u`6W(el{$AhUVd5h
zNb50|K|R-n6?<8kZmiVqR48sXaI3ptzis>OEgi{P;kMCV|E@}qjP?|n`zp3;c6D^-
z?VMlXZaX#GJz17aIlyK3GtphB-mtmXsJllralyo=e?+4nm8H47m408DAJVnC;9cL`
zn9Kg9i&uL4TdY$|ZcgC;mlP_%p5b}0v)5?W-+7^($0k)3{p9_(Y{EhFq?-v&^Q97E
zt9mb=z3rZS@A~b<C->Z3B_Qf?s`ARUsfoJ3G8v`{tIHR82(J}lzx(xjRpY+GH=1t@
zk8#Yc3zgipZ%NMd@0ly#-)1klrTKE+gVXbNF~4NB44Np%X7PN-WWj}x%KkH)Sk8G_
ze#VOTKei-fr5&r;>^?he-o;(FBW};$+dj`JXlrncW@CAq&!kCQN9G82oGW>{W0w0O
zOY1XM6&H(cd7R?v?GOAF+@1K=>+jO)mHY3mJF%)ZciLpX1CJQgxTdOkH6E++n;2gx
zav*3%^7F}`mQO1@%RN_jzOvu`ADz?gmS#nGu{jt7%d7vbf4y`Ov)nfBiQ8^|T5NZ1
z=9SxF?D~xwLGwJKySyrygzqUFIQj9;kDpikCZ;H`O#ZS`d56@d%CN5Q%qdotTedc(
zEa<PC->8seP+fORRhUI-ne_8$M?=P$d|5_|3lwyZi#!ULcc)sY=0OTm{pWKF=Bm3N
zmg(ox*s>(uV#U>XQM(7X=4iHj`uM2uM(2Nq+}zxpTh_O$FI+NN7BuszbLjRIs}rT~
zR8yuhpHShKedK7Dyr)-ggQ4u2EdM^<H0e3pQbgmcHg4CQedu&-+K$VOUJt%p2|en#
z?}_|QiMOAZ-0}PQ%*Sns!PAMK7b)*l%Dc#Rzqg2e^?jFZyobXct;#p`bP`*-Wr>nZ
zzNX-Ln@xPm$M2l7SgXHqy5aZRXP*2H?QW^@E@n2Hlknwf%-dJ}myZ9DS+!;1t=Fd*
zzKCq)e9%yM<MER@e>X0hCO*k?veg9lt%m)NI+v^LICNEa&8_p1_scrJhx#p8EHf!+
zx#NMJZ?a2jUY%4I=z1h~QHAvn`|F5Gfo&Z(L}k7|b(n5leE#~5t4^#=AuGSKG&Ly*
zsC>V;O+06nfb8G;tmck`MrW5PPR*a1ynNgCEdI5oq8%%P3z{As?D?1Xx25mby*hQn
z0?Ce}Ue|n?lH-iOtli~(R9`Df>)Vahx%<4IpYBwd<k9@A?62oX<9i=D)f}RetUmt=
z))Ox`NoH)bDRf(V{nGvnouF2u#}i+v1~9Ro{P(dpgv(&|)-;`}_f1)+i;r+8IQgxQ
zKh?Bq#j6#0Obbh<istTUE%Z*&=$QGrpZ%O#^1itrK8w5va=&zVt?I6QlT<mLOn$`b
z{cFmXeUtm`Ud`2DzU*bNa-L=Yi{tU%wO^OGi53Z~{J8Lm*ZA8ij<2uZUfMi+>Y*p6
zF17xXYQ3^hD=?=>oxS77U;96=qjy&-IoN$$FQw->&*4(Who?Rh_vVzH-6OTEuxs}0
zf?JwTxhJugI({hr)f=YpaC=(bzn8CjYBi@W`Lt+i+A+J5^b4y?`JUeEC@#wVt@k=<
z;qrQ6=fa$wKCigFykc!U0-{|rHH{gboX^cYs`rFPY1@q-U3QfdatdyK_^Z9_KZ94j
z=c!lcLv()U=id;_x>2I2cW-@;s``gFtnSNhp8p(vg3tEBIm0C}TBn{>8oYXZw$1&q
zx4y-RrW-RO#9#3Hdwtn@>0IlV^d*_8|K^sQPI;SkZ1$J42UnTQn$)0tH(;ONF_kAj
zXHGVI5oTSkFLw6&?(h3{hui*VSaT;vEWN1q+aImt8v|r=M3-pysXRHRe0S!&{|s*h
z7cHNvCmqET5@?mlxO3K|%tKe?J^4g9xjYyIWzT#Qn5gn6{oLX${ky#R-lloTvpbv=
zmG&>mJU(}ECjTezM)yf+hD@CAZy$gCae?kNjvKE;d5`C8OzjYzc|2&TjqH;)cKytV
zCrw7`Z{N(2&zx(VcXYaa+!mqM-mml0m%8nF#V+D>$;&~}i<9GnvEhT69~)|TIdxVn
z$UMpZcgOL4zFFUuBg3qJg)Mbo${jSJa)m&pz-EPVixcmRtley0r@!o(Q~Edi$jhK;
zcAGufOp`o3bX%SJ{~Xj*NlU0a|2w<rP}qfn;FnIvj)v>({`vXa+7Lz7T}_uRZJe{3
zLrpEL@X3$&j{|Me%uh&H<^~-%=`v!iYuOfbdqv6`)zliNv_cmDnaA(3-#1D#t+mwI
zC$TVLa)eU*`|GoOH>a=P9dqq**sHWFpSvD}1vE_Z>OU~S&-k%L+qQ3?&K+WT*FV)Y
zzlqy>!_sx9UiF=QCF^_qpG@%lM<xFm=7a_P@|*U|sqxH`i4P{uzwPFo@Gaf-aPA{^
z`(THoMLowiSzqv-wt2nD@0%ZMmT$gOmROm|w^UbQ;m*E<CK=g_Z*2EWzBBC*Q{$uA
z^^Cv%U1hmm&TsgxJoA+KxeKnjwM);QPFTKX+2Jkkt}f7>X<(`7mKga_*z!{PROauE
z`8H|&LhClfHJXb@==fYS&)N_coTt5O$D_HKy1_mFQq@_5>UECaack~VN-5K+zH6EM
zacQ~usXpsHe?I%Jb`3W7%C0@}t2g+H*@6ih6b<_=9#5A4cdUb3r${Gx;>Y_U=QJHB
z7(A)^5x7@YYKm<9f~azN35!oIlRG{dT25ZuH1CS=7tx(XuOj&U`ET`UtPHs{<+SgD
zNh>%w{~S<fIG)7EaNzHknq^N9ax52}THG_S$$rjD&#$Rpg;Rdn{wldYH)mJSs>eGt
zHDk@XD@qwpbxrEf4hn6!o%!T-yTC1f<t8`XwR&<s9xY6T^UwSYDJz_o^w;L~v9#m+
z?v=Z}d;NEP#ofHy?M0@QVUNtYRy}(qv(~jx-(tI1hr7{=a0!M49+m8A^<fqoZSN!(
zYu=UeEA9`lsagFl|38D&kB&!XpKk8+o0GdezAF1_?)Ka*D_3Pro3*WlA#BO=dzZZ2
zDr6derf+Xt$MYxeQuebxhg)qoly`5t$*i<*@uM&!7nY2}{*4xYdZYg}l%_3D$vO3#
z`|Gr6PI-P-Emy6C7KmJSIpEP?S@idLta#h(84Ed7w5<5Ge(rvJzlrxhgJ9-YK8KpU
zE7eo)Ru@fMHM48iuUBm+RZ<p87_%HOD6-w$ao~E0_s%oV?oMf9D=J+4_2>0tHlBHT
z$&oUz;u-jAzb%$pdv^P)+}uk6mkoY}Y-Nkf+!r%hiN!5J(q^}Va<K2s(-Qr*k2hR6
zoFBmJSbFQYW}kuQ?VQS*<9jvVpW0^dq44+bvgdzI9dbUYB}`WSbcTujMfI!vO?N!@
zPS2=re%EB!FXSqla;7EABJt%`SNk1nM6a9)US51_LcNy2s&$NM3H=2pk2fj^c}m5X
zU3ap%t*MZl;KuhaZDXbFy}e#b=Joc4?B8`OuV-t>v<T5H&%(U(R5LVL<7e==Zz=zA
zu`qti8J7i1nkOhret+unpW#YgNXMK>RkG^&u`6R^4xft=x;3Y&Z{?L%3+eEj2_ord
zj`0*q{C?3b$5^a$C?oCUlLE%0Lf0P|aBq7)HRtB4RhkPIhHcmK2oi{`nDRq?qFU%P
z&bRm7-4Ev}D#j%}c%H*AKiBBticid%6Br*}xsrP`Pq$qpZl(A9DJr@as!Af8`~`1J
za!aW_x1dg<#3**NS|uy9+>?JHF+zHh7dyYabl6@MqB!NM$>&mT`ID-8<>!+o-&qzq
zC*5$VbJN<ElPX@({}}@6`Qo;2J^L?FE5>tvn*3U+lGoMU(a+8n1(Z!y)a0toQuFC^
zk?>>eIDf8iHM{VSkQnv6mThdFHv~T`n@qXqp08hFUjFmiyzAbns@}Rg+9Ll*w`ffG
z$;aa`iN(>lqmkix$UMd5%DhABQo+;iSKpgqE@Z_#!NIn0{#vHZY3sLzOn-aTbj`j?
zmp-TJeTaxkSkkKM#L-Zlq<r%7X>YL-!$VI4YTw(})oYa|y{g|9U$^s4ctG7WyPbMr
zjINt^O)XwsH&eqo<m;=bNx?_mJ0i3uBrycMb9}qVBBx8~!e%per~7L|ExsQLdv$l~
z{`;2a=a#;@yD!)J*nRJ4_qgfTUS}5SZQHQUVFJT34i&lk>=IU%S9dF2+{{!dxa05Z
z$uY_Y6<yiY=iON?eBEo6d(iE^qCCb)>Ybk_-`{Zl%$b*3XN%NRbvWZH@BFRT(#qsx
zWLp%wTj>6Z<vlB!R=l;VDP!sUJnyi->ed}^zOH`$^gn~C=+=E5AI$CQxANUx`65q4
z_s$AU?F&nmYI}EhwI_43@!7}Ebu6v5PuMk^WuCLe_mAJ#OB)=I>Ef^xnR@p(b52l@
zY01Wjw^z1T`TKqov$y03P+?#>zQw1!|Hk>!n&i04^L0=2Fh65Ff2wuI!;8tu`&&hJ
z`fs)BO<A^T$E7X1HY%)G-F;T4=d+Rsd*6qZJ=qcKRl*~^%g+atH2<nAl_?YOn)`9b
zirUSWXUT5g`Lf*Dy6<&>gwv$UT#x2^A2`FbKBr@zf`E1B6(P>3Sl27_1=#9pUbfEd
zPrcu}ebK+=d9R<!o!fo=#(U8c(KCy$<$mkXI+p0t%+O$;_*v@g-2*Nhd*2(Lzj5t)
zU!c)(xsr9aVykzkh8=n5E?l^LrFZJBwrjzMC%MaKvdp)A{Qc5n(L<+J__@b~gk7I_
zYQaCZHKultoUd%@zicPByo6!7Q=(eMBmWA9y8V?Z8@k+EUX=d6TcdgSjHFZJ_TJk6
z4E+KU{C0-|7@uqZX>u3vZ)Dyww~E!!$LQeZmp?CtW#3rIXE>{`yvRD>L3BPt%A^IB
zE;1>?jq`5&yfD3Z_sI!Hwye^t_lfShVtu<~>&qoOriCghYKD9|a#B85ywb*sLCY;e
z<J{sEvM=g37qT@?m_F^hQpJRoTK4%JKkUC+aWQ<bPg`|x=NE<FSKjjEPkiVUUcIq)
ztL1B5KlvP$?|)s=Ps}?!@#jUeZEuxlPfs#=&YbxAb%gK>W``3?^uAOcouZ*WX-Wv2
zMFZ3R3nv8aEw6-6H&XF7v-|n-`|66fY<nxet(v~?t>>!DXRB|=3o!--Jyg}SY@YXb
z?=7LSQYP=Yv2zV}D<~us-7`IJx8lpL`%&$&uF84qpZB=&2=E2f9cKT2t)h0?G-Edg
z^}~sgnVqtCa=)%J+*fuq?Ot({(<@1Zd72X^-cfn{K;ozJbDv4(oy{8==5r+<4BvBi
zo#BPv{|r)5mO*z@wNEZ9JRURO_g&`H72iUhReQW$7H+uWNYC-Wr}IzEIp?h=T~N!)
zzVDFJv_&Be`y50TG|anT=U@58<f!+F5QTfIi`+HdX&$h<I^oWoZF+N${L}o#@x-f9
zrFo7_W4|4btRO#UpUYK^=MOh76T95ew>9UY?5qbetMVuBTbwdAp-|C%Uwz@fS>E+o
z3%e~Y`#n(dF`8pOt82=huwAct)^;`f+}Bh+XOki`!Hwg<N2~6}4-*w{yt91Yf8<)l
zdZENSQ+ne<xdVQiCN5pJCFsJogIeAyLhBUfsk6pkadYFaD)}H{<)~xz^{8I<_Jkcd
zdR2m77v)agJatFRa)-ds;#2Mq54dQ0y_MPc;N-{q9u+N@#0+oGc|5~r-|o<rxmWpb
zG4~|x%KY;B;@b1O)}^O>Id?j;Y*pp4mwDo8GiF7lwDQF*o_JKlL(OGUiu;McT&8Vd
z5Bt_LPW~OVa>tyon^WFgeR=)m;U(7o3+I||$&uaiG`84x#g))Ib2Bxn9tNygdGurB
z6BU*7B8}{RYjRVzOHG;l%+0PjIp$O4(c9V6jM&#k-3p&HPlh@3OQ%4HkO#x@oOze6
zW(Ae9`=nnMmD<<J7|zsrqxL}1ME#Wtg&s-_EIyCF8b2ytaHhEZQn>bIkt{0>=6Ttr
z&lfJKT5mBgU`kU_$^-v96YB3f>n=UKk?ThC<5TTiXZ-xGew*6R?pUzW)iWw&A^SJx
zeMRYeUUcN7N^fS4Gd!lO<T$_T!17B$+kSq?S?VO@;PK<&ndI}gzuya#+3jO}-+rln
zb=;LFW{<W`TM*OfCElMD${3+?@QjRnZ*G28(uwlh%XMnnwPvpSw2RNcuC(>4?mn&7
zRhBEX`{m+pesGjHQa5W>^kymc9C^q7pG%J^u<iKupTTU6mWGqiR9#NSBU3AxtQq)}
z@1&(~`TIrZL0O`3L8bhUKd-7gyBn70N3rfH6;6-HUa31h_jvxYT^7zcmvzqveivnM
zU#1#led0&Ia&Ya0O%tp_(mkf6{@$}<@7B#h6JKgxWj!P$vc`&m{Y?M;w>>YsD~fi#
zo!Y+tsf%-=RC8m*E6s}`S6ywhq}D7C@>j9s_HORc@?%e&(*NG-(ALWyJUcJhXrv{C
z9Q!TL#j^43+sSEr;%?qH`)#}E1N*<FI@?;-+H?f4{A6RCWM6AP>)&htBEOd}e_aSW
zFJ#7XP-OLvTUKAL{b%q^-S$0e#jR5-ST<`g7$pb<GP-<J;RyNNc*J9-o^DC`=UbH;
z$M<EHzWMa}6yIBea(mBj>%PXC=H9#b;_f92wOw-xN>mO$Xy<?CQ*q+k-*eL(kEDkr
z%dxL>e;BuWi+<1C*IoxN^6t*AZ4Zvzv1$g(LS36rwKnBXgeIybq)ad>o@?A9?wd2i
zl&3~dsc~|Cfac@O?S|g|9?boUn)#O}Pgzpf(%&d-_)YM>m)_xT3uh<$sPwwj&dB@t
zLFC}}V6~}=FLICe7kCD=`b>UfX>3<ryEw8--B;)3G?~5s8Km^A`_7bU6*Z*Sg~S~7
z-E}^9=R4nnvA%hm0xqvwv})CY7IU9d>^oLJQ1Mim(cqseE3#hO$fMx#fw`-U-YT_h
zy|K-*=ko`qJr8R0Ydv!7Hb<1V2krW{Zth#X)~oxYw^li<dL8RpnpzaUSZ8lt(sS2<
z{f95@k_+FdYk73p-UCw$&HnK{U7Tjs8MyQFhpXprEt|#hz_wZFE?*hLb-rJfG0*P2
zZo7Xw)clrr-Ls-Ydw16+PGyQ*y{uLFj=&TPS+C00K9|-r9oL2bEv>PxRA<k=(eQ4r
zvwE$q=lKh1`zHNo_}JZXV839;qsi}|+w?E{TK4Ar-Ia1hSAMU#zG)frlqHGBqkSAF
zG8|BuGE+tVdc1DomftQD?`p~}JM!T91$myXzx<7nF}t=cSpKIyzW&sfcVVl%X3aku
zeE0Rs*`<m#K@EGpo!43N$l^z@`Rq<3vzh6x`8=}wrW}3qHr470v-+u4IoD#*T=9Dg
zq7{l&HHGeUdO0z?n{hBizHE2BV76qsp~;WJebF|1J4AmPrdul-yBwK(X>-);9k=gk
zuV1s$Yi{AJqKQY=NOCFsWNB<};BRCOy7A#@_wR2(P5x#tY&uq)GER-(J1^z!kAy2}
zUzcvXeItYGc$U_rJ9ZpiO^$Le1o$;4-jG!G|0mvg?@<^FgS+AB9~sU`0)1yc?6s_$
zxy1Hb->hY;r$0Pc-jV65=<2(|ajNhQ4khd7dmcyZ+iD;>O?b!4htGW<s8*kIW_-o}
z(xpsy^6vi(53Bcm+_JU8wXgMB#H#h)d|I>SUhito+3~`kW#yr%A6Y&rvbell?31Ex
zyuYG;a*^DhRpFNooLIm7UgNsmzn(g*cl+w;x+A*C*8C3ZQiUUOY>X!-&i~XuSMab9
zTi)fjZsxjY1eg42_5IJ_>ol>0&Am5q**$5^SM%MrmP;2oSxPdr$p5|ZpCMx6udMQi
zFD6;Oy}CqZL*<oEA7Ys~+`g)a=k5sJb#>pqd3!saEeK_Gs@Blr=+Au5{C=U{(k;83
zMbETvlR2pVvWrVI#`k7&;(7VG{IR;`Rw-P^HShb{Fx}AU)06wC;OPJKOsBk5bo${I
zk6Z7U7%xmRp1X0*rFCi?(*=^l?uqb*2W3V@MBS`0i!NZB{chnr2YDlrijL#A!?^u~
zdBPrD%G^}E+OLv7itUq=?CyO2=W`?c`tqN@o^o~F&yc1i6WXTn@8z7p*my_qxYVTc
z%?_7ME^TI2pI1CTd6sco{06HihmU{yPtBQn?Uv}iM?0(XPR&tuVw^I;U#Q}sfv3eu
ziDRqxIJcG+rQP%4nbXAa`nB(&!z%5b=cnt0*gw>L7h-bh(rI4}xvWPbOIF*q_Y_Uw
zZ(h5j=Tnr(6pLegzvcOyWwVlNns>-cyxh53^J4vqxmPc#uXsDvv_IBtb+<<_XJpik
z#wL+8$B7EZ7~f7_YGdAOx_4(~X=2|2O^c5<-@Yx8iGTRwiR2aYPxfm|FE5{)ddt>(
z^VWFBqNS^zE%B0x3)E!e;(WsXl$ohCapGH*kU10YJ>AFo^?3;U!%5L63U@4?wan|n
z^vu5@mnSSa8CARYcTL{|zxsV0n}v*ec$F4Sa#$`Z{-ZCOQ)?1~fS(Y<JlXU0zq0ET
z)1LNmYE{mgs$;<GI^%7WzQyvH5t$DhWhym(9y8#O-C*~xgvEwq(-gONKjn{GuGzY;
zXX=y8+`<iUx>paq-a0pN`Q84gd5d*zKUpVCn)leiuXp?IchW^h;+Ox`ZROY$`o7fk
ztLIfNuCSkZVfSx;4XSn7=b4q@!7b6S-oDW4-ks>Ie~+~Sy$rO<HvLZC8*nX9Yv0YV
z!@U-#JDpFeEa^GGW%0Mj__kEr*QpNEJpMCmzwn>osC&uL2~1j3OQrMglzozkHtoOm
zc52h({9vp3L9>i(H(4&;zN3tRU&^}O-=sxG)Gi_UxtG)}mvhqGf1dmN>ng3fU2yCC
z<@@HQd-feZG{fqtgVS^d!MAD-{~2C=o>Z(HeQ&1G!6}San(V9WE(p!NcJ&9pm+mXJ
zSJ@?RpHJSpUDVb!+AA=3+XJQz9Ja@lZTrvH>UD4E<ym!{`NHShQ{`(emnE!_l(Sj=
zdf9X>?cCPx>hGs}eIG2TQ(X1@N}TrshJ^=?3-Bk{S6-ZKbo=mP$+XXZc3xlm?CSj)
zbNaaw_lJb+_w}|{o}f@U$@o|QZO0`+2A7Tpe*CyBSJ3hN%fS55nF>0JDyl9T6OY>e
zjrcA9<^9SVeme?g$n3nfid&<euf#MoXj8}~=^())7Ms}T$lbHy-;*66rQLUecjKhu
zQ|+ESlY3=)+3wEPXWbF}O6Gv(K_Tnf-~SnIJDN`F|2^HR{m0{_GTtd|H@JN7ugSft
zd3=UbOpaqtqfCL%%##P+u&Ma%%{&;;ta{Oke{Nd#WEpk8uMyvmo#xy1=(1ni1c9K1
zk`4iO#%i|rZ=ZQlaBxF=e)6v?A6~xsGUeXN{ein8&K?P!7U`YbvU;*j*FJ^5Q~w^b
zWKP-E>b1a{U;q5qv;0D*W(fZMcxgr4(;K3<y*6#tUJ|`Ulv`2TW73QdPkaRL%=!6u
zbFEpy;u)XMMy?g{*)jY5yyW#OmVUmrU^?%inW@=*Twen3Sx<bVWc^(I$z0#mqdw8;
zY&~WMM@%M9vyEQsbj6^n-;;6qoheJiX6Kyp>f`B`Q*b%9CM?j&t9DUL+TuKcB`<@|
zAKJe%_iKhjNwBEZQdP#f36EZR^c->jm+o}dM(+m8Mwu1u4F_(@ua#eYan-hE_da{G
zcNhgSZ;@#J7kNC<N>FF3-HJKOEPFz>uT$&E`lBVgcBbFcL%p+Drq2#aS?c*+)aAfO
z*)#laS1#Vgb2NGOo|=u@qZph*RvjvyS1I3GlK*n{%eqy+<BmPMXaC47n%Q^hv>7g{
zuO<qui`jI;$o8=1`;Re?#ncKe+}IcXXZEzOFY^uWrde0!#=d>=mvwg9jW=c5vb*|j
zW-iawUC5Ej$#;N3)ctFU#)|K@C!3dD6T2uXsQ3QQhoiyyd8PX+-iAA8#ZOtLdAgL(
zYTgq!;TcLsDu4Ws&DnEqW2=s>{pIMThbFHmS{}=La(OhvE|EK@F0paBRCk{25&rmZ
zm-i9f!2XPAm0$n1xE@wnoc3+etndk)fqaX9RW!4B$nNv2sd=p+alqio<cGg6eO4(n
zIC0c#sVA$BT0oDg0{__=Z;C%vm-V^|r17Mkn7lNfWyKo@o7cu!ncm(D_4F!KCQTA?
ztFJr!$nIAkhY8b>_n&`-d^g~=dEs1f|K0ASv!NARbAR`}T$$?Vy(BSr=~V@fKrXvW
zOP2YR844+t#CEy7-fzEtuXglArdN4eq+|Ez``$A%dQ%;?=>2xRr@mLOu9$Xro1D?4
z*R~HPESdRs@;4)Wp2K2AhE6f@$08Ve|CQWyx)g9nMQ4Ji-Gm^aWZ6A41O;zfMQvPs
zcEL)8icgELNql*(o-f7rO8v*CZEJs;Wo@{cG%F(YzF);TmG5tCo<BdgWRqK8ut@L5
zC+u3Wl?8{4`<zxh_uOV`ckk6BbJerY*R5LZeIO)6E-UrX0;jI2s>;6{CnWLrM&w7P
zb|y{Uy?p-Mur<kP({;+8$FgP~i%(FRKe>0xBuht61Gj`u`TEOmN|xq&DQM3Lo$FD;
zno(T&UUuhZ-B9z}M>CfuDzj!POq|1DRHu@a%W-a*t!w|~>q|2$PIE~vS@rtD@2#&3
z^WQCdm%UiDWbMAE3Yn7}bQY*O6op8J3vqUISez?;6gxBTtg|s+O0rzUl0SNJ_l=sj
ze?GQ$x9-ZkHL4k<K_Lqd6d8E*cbuyE{#5Gh5><g$D)X7I&7Bc4N9v}?zfC(=9@WrP
z?w4TUIl=z^oq>ns-z}zhiabg^zqhM?SR-M&ZsDb+885DEUC1-fb6$<YYC}VH<0tb4
z*RHK-F0&HynKg|gbjpI4k6l8x?2-1G8(A1LX~C50$`yR4el#sm-!b{KmG<V0$)U&I
z+^hJLp)q4pncS6Q8*fZ4UdbCOa@l0!(W|R(1>Bgh*w~26vESlYiORN58B*cuE3clY
z`Zsm6nwyGsO77zrwSVgW8A7J<?YW$He$l<T`PuVs-3+z=&ycw{uez!+Z_yUZE>p8f
zB3~606~8o1l~mL+oS^#HL;sp~+P@pmx8Jjqe7(1p^TU6JFWMb<3fdp}n0WSj+28)p
zAbM=Iyw~J?uO(llU3>N3)to&x=l!j<YjgfS4baHQ4X;v_idtQL_L})W*?_KXH!_bb
znee1x|5+|uxBH#Pg=VKt_DGyCkNrjA`}Z+!-`gjyh-1@zy8q!`J-f9zyPKEX4c~Sp
zZ?>=7l|3Qvx@^DP-L>mg*REM^*%$dQ2T!qzE}GeXIjCjNDUsyMPaeqKb^I95+xNMB
z<Brfbk#*nro*!&0TkZbkdbf0(dfvA)4m{?UyEf!2o>IO1>*}^wr=;2sbEPKe>ahw5
zA8!%qILLC6WzMmEjE}!vd*-%n8<WDk3Wa{=>!!aht+(00HX*UhZi`puM4#=N0hjfc
zE*0t2l3_1kP)g#kJXd;t)`J+iOQ~sjk*lkt@9f~^`d;X@_RC$?b*q+b)%;_nD7QHE
zoYzyC{`~Ud$A-t>%ZDBOFvTfpd+g;O;g4h2Y&!HLTct8xzE$UX&6>1b*LG!jl>{tQ
zVC{T%!BAi-pE8@>r<!ke$D3T4r}j2p-u2w>uhqq4&&0#7Eo_gsI=x3PKSuLbu%B7`
zl@2E_f1x0kLk)pHjnceIvd(Iqz8Se{(aEL=*@H6Q7V2J{)7DtB-d{gT=+?E>^B(Vu
zdSg=dyZ7FQuTc_9mo8ow{Z-UYPe?j*Nz-z%r4xjvv<g-GPrnoTHZPIsU9P@-sD$n-
ztIOHheP_4$w%ggVzkm2V_UA9RUw>}NAHKb~KG5Eyw&tbM_ph<9rh2ZwHJz*Mw%G24
ziMlIyU3w(4;8D;LjRl<(=BcV%^_{x*?w?xR6^@yOyIvl*{djd{y!1oCdxu3%6)Kq>
z`N}Tyr{3%H<EvU;(RZ(HdYHW|cHh*e;;x2a(og1dF>iU~!jyc3ms6kJwg1gW-<9ms
z?55;SnmVI1c5D6iso`D6CSAHPb=!Iu8-IPDKIbPbjRpz+FWh@YzeOi9owl1$$&+G!
zT0-6`llRcKCwB3&{)#W7Mb_-vr51Kod1{_2m&e0-UXNTjSQS?A@6Q#kO+IwQ>*yEt
z4u+jyzbsDm`1)<abM=*ya#upLx4hr<a>b@~uYI_}_Sx)R{b9-ni!dHxX6DC1u19p<
zPV(99wc~bJfi0Vcu*_u7JGK|UZ!q-HvgQxfTKo1+x$6GpI~kgnU$5@=3SRHP;F-hG
z$gF-o?-y5{`r)YTG}j9?j>?}_ofqxpnW>Wdz^|U`&yBn&^V-t|rvF5)c0J6EVqVJE
za(U-TV@-C2#uLB&9A&@e{g~b^(X@x>hRyZEPiL(xc&GGn-%{45(sOprrpasb+j~V7
zcihf;8Ni*pGw`MbgOLCBw<<rbCii^kjx@Ysw&K^T{|w>#v<}Sij|eYbBDcf(&xedv
zFSjRN4Keon>|3?e%Wh_?&Ua12$Hw#S%QRk%DfsauShF^3cc$e5ySOb+wqzV~?2ies
zy6r06v+HO4L#L@nEc!DYPJC2(=6}oPad4S`zC@+0(>%|~`=@Ii34d=a8u#(zhi{Aa
z&cEV&dc*4%>%MW%Soo-7*ZtkiE1SIRxT^o0tM+->yLZnA57j9z<G-x5uKcueNnpV)
zbxS#$jj?7g>aVW5fA__L&GD-;bFbGiuG8{YyBjIM7{aM8yXTvhG0*%?2Z=QfD`mRN
zH~$Lt?yj->Af#knCt7r~?q)^es#WfnVuJz#gxp1cJnj2zdH%qemvcUPbFmkg%xku-
zk6QD0!s1tTrF^eMy_}|=>HK!8FPSNs`SDjPPQQsOwoSMfUHz!D>%qq3t(jI|t!938
zUaL5#!Q+&&{6b0l+U#y8>-GC5+`hN5xFq1!IW41YO_SDMRa-P~;uoEju6pVdv@DyI
z)t?!DUZc*|`i?Pi)t*mn36U@2o^C2@jhgPaTXn0a%9^Gt0)>wyXWCa@yZlk3v~l^n
zAHA<VlUNPdy1Lzc%aUdVpWbl$dSCjLydakPfDXHwvYyAmhm*xVy;2T$KK-PyOHEs;
zb860f;T=E2<{r+=dAl-m-Og^_m0fO8dX_IQFEyHAAe8*x@Vou8<r{xg3EkaTbofN>
zudw@xuD<!_m$z?Na#%Kd`N|zvqS<aO+B%~-V9HXrDHAn<CJQ97DR@ZE`?blQar+Mc
zw+tLFDleFud$>&#IlleNsiP5JO|N}<8uw&xg3_jiS#p>EYI<3;a0XhabNzm`_?ShC
zze4V<#M3G-|7b_MPE%gn7q(>94KuB>=4n$yZ-?Gg+_hANS^byK0of(z=iSm0oyXzw
z?aZ3?`!5Rpm7V^6xBhv>`P<#cxq($T>z3{eRMmXyAJ*i@X0eiany0$4;Nl05e9o?V
z?d1A$U#(k~x{>y4F7sEGA)NPCEUvipE;}~ueq6|`2Cth60`&%OfA0BrdEugMtQ-tX
zl|seaCrJN!u~2Jq7Vn(DTD9LYUvp_|T=mKPwCr?1(1!_03IE!k+go{??shx+rHS1}
z&-nB4s59cb=LF2H-KeLrz=SEiX{V$zTLO=aVCV0O3#+UXZ&_#^o)-N{a`B}2PsLxB
z{=J=T^7E*5W`4h(R-o2IqeV5(|6N=Zr?P`dG-ms+P4!-#Ptz>UDw&7-M}?bPoYwXi
z*s^#Br%L0E%K5j?ZL_h7T9=Y^_R5pTPy41WIv3g#^=i$#f0Mhm$@!)Vx4LS-xx;9?
zQ{wGUhaV?@UP=^O+3XNv_m}^A);(*7wYRj>R;B&?RkEV+c$Uu6+qFqz-f1FH0TVSF
zjO7Jf4qoQgUU|kM@%1+2_vc@_E|;Ci87gTf^V09VPq}a3RL_-LY_=zCV^$0hGh$Fa
zf8fBuD~*fFulb3uP?&#eotR9J#r2qp`JaD<l(PG7=PlZ+mUH!Js%B7wC(8=X@4|i0
ze15!PTmCksecy5I3p^7)Jq?Ph@#Jgpnf<G7$MR$6f2H3EKPqy6Z?xC7#YZ-AUGtYe
zrD}KaPWtbYQuD6;Q&Tx%s9dCwEO%$scKajGw@X;wJ}mFGM^^V&?(N&I*W{LM5e--3
znpf#~T>V$*S%>fEY&Q!p4Jz2{;~kSSuQH&XDbwUml)Gc&wx6q~ZZErRcK^95-;`EO
z^?UOiIMn=;pVYiq$Is=qR%+X#D!EXZjT(wmp8tH_Rhzr3{g&pbkmtq+TKeZRBvtg?
z&%7KVI!|~_OTOc;D>EhTt<k%+b5^<JgGU#h3Cq+x-~X%ZKf|&P0TE_5LkEB3)%E#P
zBWicX*_y1eI3*+FzQIw+X5N7^LG?E#E+1As-I_3;>z2jb-N$BanX0RKKxC<=%L$)v
zJ`<9+sZ{pwThJk}i&N!iSgZ7wAYMPSADlsV3QU-K*37GU@~Eb!Q}*#2mFgCwz7Nth
zFE+frm6PRK<DAvPcth7_IZNRQ=8Y_F%UjrU`OFr^tNQtBzniPK^-ei+^`hp2yxqKK
zU0FSsuU2e7pvJsrRe<}Jj^p+3O#%-jUsm4K*ZrSCeyyJ2<9+RO-Ab7r?lSwZXxB}#
z_{|EMrgq(o+4(1md)y^D?F?_9=C<~j{X6ftE#u38Lhb7fi}W~OJ6jj86?>k2b4%RZ
z<q?G$PTVd_y$v{i3wSb5FG%u{l`cJ~YWS_n{Nsw~ZteX(mBCLnq8~Un7A>0C^8ASU
z{+z7s{~7!<=A4l(zO=-c?fI!`;rnLXTD`YwmU-*ePY?8#Zt(W%>?uF_sIW(pE!oA!
zG9hK5bnB$jl}B0b?8@B}HqqC6cb?g|YYtvBRz@*d9C6>$XK_LyIlyoKGkO2%KYoX>
zY)rnnPcyvY#_ssM&`Yy-Z;$=EPcHmlTj#H>A=+0DdUdFGF*LtZW=h_YaZbAA+){_I
zc&757bz4?dcsHz7^H?o;dF_&SH@19Qy~5Y`*V?OBuUoip*;U>0pJ68FlqE8CafeSB
z&G7z~ede>oQS%K223^H{`$VpZoN8zg+4i47Q>I=)a^kNv6}K9fEQeW*4#r)H#Rap?
zO7;DfgG;u3=eo_Spt@v<P*9(Cgu;Xg$rf*)+b_6w=%(k*W_bql6>&LzFIN8YQktSt
zSSmBSNod8l4?awe3jbz_Dg+subn!Pncbz`vXtR5mwe}M8fNd8}Fr189YQ%mdg;Vyo
zux;D2UXi>%yK+x0PF%_)yZOAVZ*lnR>93vIa{}&LsbBJ5b@a#97x(netQT=~VfMRq
z@IS+xg{LQ;XIRg&B&+{p{;>%M1)I!d*Dt%Ve*J%j1@afncTcl3KbCYxmhn|pb;Wh2
z@1_mmSNB&ZdgVR1&aN`Sh55uMk9Wn2u^;DKJ&ZB#sa^D7lR=pCnchuhxAr}IY|phm
z;g(>4V1M3>PqNxok9Ti-{C)20OKT5*Rli)<_cnX!i~Fy)mA%={zGn8Nwx^zcyZYmn
zdIl`7<<9)i;CaqdQ)_}c=dbGz|0!I$#m>dt*mQ=?f7#cc(tkZ3e!lfFz0sssum16s
zOa8C@Uxt0$_RD?yihr}Ma?AGC+UBxltuN<&)O*=?hgNo_f@h(xuAiab1+NgPOhs1L
zEKPsO%d1QJ9x3(Ruw$OE>P-1E8I$yVOUezOyX|Hyi#@z~o7>ymDQ3HN$6B9Yp6C&o
zsi|gU-8_HJndk9hHI4^z-{h-#s=KMoNw<{#dV6Wz;Xgu$UN=R@=>KAmsnwhs>~z)t
z;r*hmUo4hQ3K7i@Zka!6DbM+O+uvtiwc1?UxG`Mo`0=cr2en1m>*t2w+I1{1L~Cu{
z(qGeJDs&gL6i#r|<T}%oa^elkgzDp0zh7i5Qs0$#`mz!8D>v6VC(on2n^XN9pRg-C
zYi-&k9RGUH;V$P!&8X{|Oq>=>ERF)})!(nSzH{uw1|Bx+x4+-Nb+{9`Dz<F$_R3uq
zzeBQjr#%mMSd}HDwffbxpq4@<(en(NDJ(MsCK-Rc9lgs{i0`~d+|HAhe^yslOCB&f
zX4TX4FEoGO4Bg+8Uq1WRxAyirQL%rEE(Bav&)J#3`U~r}of$!_-hC~a=X5_!S-RCM
zq<XC~tLvxTawi2C&up`|OFk`<<IBCU{6=%r4tC|k>{;P?eslKk-5z?m;;Z}0a^K54
zwO`4s-w_^@?G(Sh*6Yc3naV8ff)z8=>e!EUKIAW&xFpPj@nfy~e}+#gVdptAqsoH&
zzkHFK@igMt)t8OIhh|N^{VBqWGfL>Kv(ADhVY^wj2J;^A^zf>c?4Ra#P4v9qjPLBC
z&!=vjdV>8-P~zOZDQoq*w3GFWE;$BGx*4LBJaOJJuVD2Kl|L_gHg7UIsjRf~`LuJk
zM&&Ed3v}@|&5^gcXuCK&IA(oM1n=XVNNusx`HG>})n2}Ai1@Rru<ptly<mn(oWh+=
z!7ZH9znyjG&X%rxSI@U~=cAiJy>rzUoPT<1`Tl=~JG8$1UA4ZXWL599Lm{*JKTfe~
zU9i$YlW+S8xqHeRe_Z(1RD5em$C)Cx@ciY9ubm?c)qA~pz1V%^&$U&xCokKkST<|#
z_2m6QZ8C-Yx3yvmCo%ang|wZttLgjTBRRh@(p&b5-^JF%6NV=i=pEg1aptE3p>Z!{
z_m+ykzRF#4YDqzeS$Npgl@2D1Ej$iB^?@hPB&jd_lUQyjI3qs!{JZ{jT8#(Zd^G92
zm6`Um=U-{j{6$Oh+V)MK=p-E#G}U6`J0TY#KJ}f<YSr1Z3On-Rbau_rZ<PKW`r?qP
z^41BjK0NpFcKgqecSz%{^^8^PZajA7j0!%pX@RQ3q}l@yeERPG&dSJNyI#WWVX*SF
zzqT6Lt9@lZJrTTY->_`&wUY_EgS%#BWr@G^o)c32yX*Nq>sFCJ9!4rJ^+mQ!lQ<a5
zkp8Fh{++q2Bk~qrkx~4^tEKmwcZ*fSlpxKfFbPGQ5aG%9&oCJ>Z~S@L?UC@Yi^rZY
z{Aoz2t30`$d+Vf_Ij!FoxV+u=_I2jbe~Mh(-YYc?<=%NR#<6_+cq7C4$ZpBs4&ut!
z^8>GSOp1&=wD81-*YCspm;7f4ik_alw2S-fp;a%_qdPrv4<F5CY+{)BhGEWxdh-tE
za6QhWkLSuo#_%(4vu%GJc<+y0-_rYOb}ZrURn^zduHK(HX_fR;-RO6!DjQ-rBf<~0
zXB<3#JFd-R-6zI#mJbr%KRvZ7Tt>^g?i(NfQlsl0m(S{!eV<XfRMTYvCzpzYf}gU2
z{LBmQGd{?;%$9%lMN2mR-HPk8S9^7q&GcRWnW2btvig*t6YaNkR5tJ6nUVN9c-Ln2
zO1ErZKaG_~rVBM5S*i78ou`72f4=g62EL_pC9j+@Q=cI{f33H&_M1mrue?`jKDb@T
zPn=bO&r<k_{C|chChesKQR(4d*GMkz$co!yUn5dlVjA-Gy}@G@tH~!l8<J`gm>cIE
z=(o5W9sT{?V<~stb9SK*HIn$PR*A2h9wWC_b8FD1M_FM)@lJbJc?PK4E)xCw=f~yc
zGkY5E*-wA^`+`yH#4o$9F5GH1JC@Z}ptw0_solGvMH38fJm`?!|8i;h9T%g7x3Art
z^Ax9~q-Lp?y-U5J6UuM2EM{f6*oCR{6ob@6itW{mJbzemNiVDTttK7t-B@_Xs>Fiw
z6~AxoJQWpv_S~kGhgSZ}nsC_jaUX|^e_l}X_DRXEH>34$-gt4Y^6kSvFTEA_?=X7d
zcB3kgm*3^roZtLfi|el5_ByC4H!<dfhp|el#02Xy+0Sdv>nyUnpb%6kWA1;b+he`T
z)Xu{GtQ$*j%uT%MXj&%yblZOh4-v~CyNfxt+$TJ^8Wz8Md*hK`d!Dt*#4_V!5<ga)
ze;>`iMY+Mq=2!5cqk+NY3-cB|%r81!;`rdHbwjIS{1wjw%CblPE_S^CAyV}4vT5oE
zGEJVGQt`F4{JP-m`90t2?(YhY$~Twt;@a-y?RrpV@~5Rf>azPvp0|HEy?W+MCB`kE
z726p8{9Pp=<aTNsf8*L!-M+jB*k^u;aq99`pR+VmnR|YdLE)WD>sza1ws_q=_;}yH
zD^X=#M|O$`R^GUh+RnYDS7jnw;gRY}o+QUPSs@QQI{)tZHqX*#b;gg?tqOC0<!Jgx
zZqjvGJM~JC(n5A!jYD0X2TuO2_poPnj=AYA#8#%jFSvc@qMRqa9uv>6O_o*Jvg1YY
zwxbMFSXQRS{%2UFzO$*o&+f+c^~p&t9k=@`ERGubsdR7J_Ep>YvbL!BvSlwHFxY-@
zC_l4If#a>52YaA%(T4J44t6Q}*P1e{W4;|%>(wY*D=M<>tahHJ!Zz<LyYuR{X9Sc_
z%ztz0hTCz$2Ginm{~1~ta@#g4_cWdt3G~l(a`rL1YwXB8$+MB6-}tY~nZL)j?Rj{u
zkpIiib>&|_d|&(aSwO4fvC4{z#v*ICgfQq%nI!(|{LTm?J3hu^QxE5?;p}x|?^Kyz
zxMA&rHiy<b&qWPO_w3!STWYS=#h!iT0`tXOMOC-@CG#`pocw$K?$otwH|!ETb$a#T
zwVXxqGgo+vU*VPAc`Lbc%e4!arc8)+b>f#~;m=w;z2_V!yRzTfhb<nthZFjG7))pU
z*rex^?)9Hx_x$ryBXieZpEu7kZ_V!QJJ%d+i~bt>Rh}a{$x@?5Sf%4FTTOOGMfHht
z2FA;O_8w)M6KBS<nt8rm{8X#5zV~;l|8>sZyXdj&wz9X*iKpCmsD6JX9mt@u(!EK|
zQ-E^<e{t(umA(6>S{$D!Z*#>^GNx(g@y^$NW@dB$MF*ANdzN}xF-`ga`{TaQh6E4h
zM}9r~ri%2vdp_gO>+tN><jGyf3l`k{UDdbk#oGT2A)iIg{#vKCs<S<$rPDw}Vcz4D
zO4hxsDr;`9u=i?ua$>pFwpG*PzlvV#K3kQ0+tWI$*V}#NkBJ^3Tg+8hS5Mk~LXG8{
z?3ZKbo_-ZOAo;S%|38C1<IB*M`xIySxGzb*F2C1ySL)uX^}F9(H8nkb(_HFS`gNaA
zO|r{obf_?_m@-kwnEAQ$eSguNr<)r3KT68S%ql+oKDA~#GyC%RwPN-ecNCxH`Ij8d
zwhWods`%{Hh1P958CKuvoT$=x<9wx+rNwNc_34V2dpAEnw{_>%ActO#quRTguf{&!
zl6^}|I(hXTyHC-}x362cYRSF_JuD4kjW^`f4<7H$&CWZ&ZO+6~n+vNxHYm!zvcDDn
zZr}Zk8})^`Ya6%A1O=beTv_>P^2`|wYDLKk@|IVOmX+`7S<UBt+TLnL{2jktCHbzW
zZ&}~^)D~QG-#>VR=fBWxQ%p}(+FbE!d6KS>k-u$A!3(L9`7(lE?(JoqJ7dji*V)O5
zZ{IHV`MEV{)9KZZxL0Wf_pSI5)br`oRHI~tA68Naj^>?N8XoxV_<sADE01t<W#>y-
zrrnwHOTjx!Q;=mz<8j5{2WRS*%vW}}zd~<`Kt%d<uUXX|(;VCmTx8YVQ?Gd3V&%?5
zlR~B{{Hd6Ce@m}aQmC)%mdV$rX4(`Qp7p*Qta??GXM%r=%9I|a1fO!HWZRDt*UoEd
zx86Qeyz;@W9`%f7E{$(@Q<g2vJ>7ajX=d9`i|H}(!ZMB%L|K(@r9Visc^%CxHaEz9
z@-s%|dkc2Voavq#^Xk{3U)$GgdRsm1+>58l)4o`|%6Rqez=VBTQ>I+D;Aj={|6C}n
z;3IOqPXF=!<ZZ=#o;I5|{QUGy^iKW4c+Q(2^$g|zIta1-a;vJ<?g+h7%B6QF;M24R
z;ifGDOm_@sa>;(S{p3IA^;12i^_G!O3+Ki(THc>wdHb{IuaAcnnm35O{?4v=SHx<U
zRXJC5pucMM<ug-032>?vF-SHV)o)v4_(x}>vB@T_zC_Ox-|XzR*nV+m*>|Ai*nbAA
zpm(R%F5e}RBkTRr-LGn5-r|Jd;Lz~fQ_kjUYue2A($>6k*GYL+hs~AL*aeL1CV#L$
zE<34A$!FC(AIA892hwkwep-G!{(>Fz(RH(4IVG1_s#ZQf&Yp8=-{i%zqFe9X+g)<K
zr1bF1d%re4lC8U4DxSr@Q(>v&t#o_7R}%y|A8e0l@Mz_n?au#irR&;-(-d2u@3V_D
zJzTN>KSOu1t3sZ%d4~J1-D~Q<ox5`UW^`<C_r{#tdDr*lR{uT49_P7c(W1Iu)%g)?
zbyrRgvJhDs$5ZVwnW@76Sb<}8)%7stfZPw2vfDpDkB|S);IYP{cR7ogjGK+eY~~3?
z#=nAt?r(b+e$6*3U+kC4k`~pa*K`-PeB=_C@y4lf{=S{!qK^a*T6VwLv#0%Kc=Y^^
z9jUv&1qrONO1NsdZE4u6GAFG$rX5bTDT?oeMASPO+%-LBsIjrJ?R&R>W6IXZyg4p^
z%WqWigg=pC%$?w}_O8sX$M?E*%Xf#qoAuh+)8}ficc$yc6K2Z}`Z-OVDd5rf<e;5t
z@U*LHlf8F~z6f|w{nqbM*WEdcP5wdU-#)DGt}Q;f+*MaJc*@tcYb;I)Rj!>di81Kv
ze}+q(J^~y^{AQl$RB5`CI<fqcf5l&W;Xvc9Ynl{yHE%rs_-EKs<LG0*ciqk}FBN*6
zvnhCAZ0WT-QB5qzKh5xJU*oUp<mu%&({^jdwjaxb-FY%LvYvZdD%@WfDd_fVTI^Su
z{^N6P%Z@#HdEbxK_xAF;W?SyAUmL6vbl-flZk(3DD~EvERp#suF6<Ey+F~WZlvFai
zNB7LCPj}ATurGCb`*z1#)7aKB-ln_7=N9Mh4Z2@F@7=R6yYg4Rt$VO4AU64BM5dp{
zf+eEcT3XbZn4A(76$%9uq$_k2`pl-ZeLSfycSD4+J%De<@l%@v-re@ypMAG%mv61q
zDmhcp(|H>eXLlVr7{wrbTuY|^?eoy&e9_YuZN4s_oMxQgm-%9wYuq*CCG!PmvDH1^
z<ZHcc_u6@F3;7jx{@Ai7=%Jz-`<BUu$8WKJmG3p#uyN-!)y7nV%%@kX-=CUvs$aUj
zf9C5;Te9{3YV(?%TJc>~Gc7=d%X{rYg$XGFQyC4vGw_<qnEjo&(3q)sXYBP)$Jb?E
znYPI&UqPov{QH8_XFr`;yS=z@`|ZH*C!<1`T#6?51z3Bq@9Z^=lMy+}x~1#+y0YVs
zz4g}o7Q8oE`S4VwUy8eP+ZLCYZQXM8;Hf1RQ)f=QAl=8(z!ch{R`>Jj@ke=5+qEV5
z#2o)wJb!=sWoUexzKYS#9fzhrS+l#pFpN?B)$+LI>y|A`Jr;bzs(tF2os%aiF|zQS
zX`Dafd5q`&mkU!HCdn%{+?Tao_2ydBb)E$YpAS5Jxwr1xo#4WC1^a_ucY8CQ^YC~O
zwA3N|KSPd$@C0SYNZTI4ndv$<w`<a-$L%)GC@|@ZQ|nt(J?(AQ&P$rzohI2VW+x*Q
zI25Kb-~QFFD0ni?K+3gd^Pz2b92>8GH~gZ0d2aHX)F~X_zCM5Jt8E^zR(jsLRaawt
z_q~$Q2-$tYqM@Lok(1eg#piroAnSLo%0Sr}?7>ZU4$95dl3df0z4T`1x0S!Q-CtC?
zv2gJ!4{aGwubt|Tl{Zz-FHkn@nVTk^n^Sti>=j$#%iy(#cqIAk+Mjzz-S*C1vTVUd
zj*QkFQ%{EQ2-F>z_xxSqlJ$o!KDV7H|GVgCU~$ahiOa)mwy)T&`|b7iX{+{bFJF`T
zElfn#vzftPAc>`ri9NCJR%-5z?~C}n`I-0ceCh7ZQ|=h7(zoDE*4>iruP$v|9=YJ2
zt7Z<1iI(Fxl|;k0|5lz)NGraj8zy|yXGPkK<NdOqe_hS*tWjQGVJ0WPH}34pD{q(Y
zei^xD>5dC`0v1_n@-a9*<x(?p>A&NDEb8#V<Qd;qFst`vEicTQXtw;^U5%s5q-$7%
zL#u;0WI|aT_~$Dz7$wWExMJY8;i=v0iszOK7CGIoU1hYA)5%M$Rg{UT;e_&$N;~x<
z>CUWM0z4&iJ{|Q_;yWr_6_WpM=Ee7?W@%N=z4q8Qcd6=OpF>@OVV3Dvj_;^*WavCs
z^J3{Vk0lqncGXn<yJ8czW>bT7J^#HHtA&a8S0-m~ynO%gC8c=}4&7%wz;x4dQqy9U
zzLWN;b-z>}N8E|gEBlbU{dVmt>BU;749q@$_?xZw<VN+Z#YK_d({A6LYL-4>i*eMY
zr^jAJ*q&!{(KzDy#QogXkZ{|lyz_;B2Xe^V(TmP9yj$kgy>;TTl~YcvbgEpSYcN&Q
z$X&ydsrr@oj5t<dH&f=fzb*=|GI|;1_hr|c*}1oVZe8!k9eY;W>!$O(`=P7~4b6We
z)VB!UIk$Ls<I|2Z9j3?X=TDom^9i~{c$9x!&3ceS+lxs|nX@TaqG{rtZ)$bNw{4x=
zq}l0`<8WLrX@>J8X8mjX=E+=EFj~Z^739$QgZ=G)hR2ax9tPaj%oO#OTWno*ZU3DK
zZ*Frj-xu%NyFO`0+={oR;#$ilO=spPRrF>S+9YhCQoo|-+=9&y4^?bRR+#@OILf<Z
z<6PTJ*2MbsuTNK<iaoeZd&O%FcYEI}8WLJ(G&pClIeth#HnY2Q!x4cy0&5I6+<)%q
z|M&4K-}9%IX;)3(ZYK9&+Z(Udp`}{fxvKmvOSVj$$QfQ!+4<-2lh@Jm?JO5JA5`G6
zd_Vc~v7gURE$3U+XuNY#-lD*3vb&>0vsL-kZ)|4KQs3F&UiV3vrQ(wM!*%&$1wWV%
zJpOcV#u|<Z+bX}TT72Q|X5U@qzE^KwEp?t6@+#DGd1a>E>Z7SzDwTbjXa2Q^OCQvk
zC@^WR`p<K#{upnmlrT=HN?X@FJGR9@NJQj9fxtqe>W{~#9C-itd~S%ol2x_cg$Y*k
z>h-o}W}b6yK7DG#?(G55flnH?R_|z;!N$n4{ITr5e-7%-cMkA)ZA|zluQg9|hS_$R
zS4toM$!}l#{?58k-OE7+H!H7;>}+Hy|5V3+=f#d6#hw=OB@@q>xA$Giy>hW5#*m9Y
zc;l)!eQVy`UhABn+q>SYU8X>7?yTjOJA+#5g4mcGCn`E6Z|mRs@Y*9|mGjf=A4$CL
z7w$i`%J*2vhBq4<&#$X}c`<G2tF_U&(!FWdwkRA3;&y$guJw#Va^~@(<Zas{kE}=z
z5Ot4hzVi6fvWu}UD&fa!|E##UO7_zdO;OX^MH{CUJQ2GzA>ka0u*&!O!o3=Em5WzB
zEbjlr%>Mj$h>^ANt7G@0de_~!Eh+N$O8j2wNH(sZ%`aElCCIG)Yq(-f4$Iw->NWKf
z?|)gKm#aL{Y3hH5x92~twf3F9_>I`Bm23E}USZYbTIp+VG-cvcO|5qZYx?(<7Cx*o
zRr~k&?BDY-=`SBf+4F4=Na|@-pC6U=<~84s8&g;PXK2cwF8XG>=Ze&pmOYvm5<Wy{
zc}{Tb;beF+@7I@4Zj<<~=i2=Gb5<+jz3vUSotlrs-Iv~VzGL1s<yN^Umw3PuInlGM
zs!Ny4U0uA%FIe{XkBJMqOg?27u+>!kn{uI0Y{$Qw5=Sk}V`u+Ux-D)U@g+=dS+Hi4
zlSh!3=dqN7f_=ZfY+$*~9TZpaJfix?@xH8?xzc=3_82@}b$M%+#_Onpi}nG(y>dAV
zHy@a^V=70}jQ<P<^Lve&K8f1SFRk6=zw*4+8ur{R4DL%~{ARsr+^Caw=WEgHby>md
zuI!QKWId`eX^DRW$NSqI%8$L?r%m3X(Y(ve)^_C=K`XT%=a*@`dobax^p&?}%>hev
zHPml0ElQ2-`SZ7Jn)0Gkt8S{)?5$bxaZA;Cd#)u{)-5}EZSVTxJEvo9wPqJhT|PID
z)z)i@!!w^k6+g$B>TgU7r!F#-xcc*VZPsd;b$X8KlB+MeoC=ZFS<31DWu?cr<%SH5
zH|jTD4Sh1vXu-s1{tv_0cg^+SC|`L@Gvu}_!)Kd=jF+m9&w0M%uNvF-JzJ+vFxsF{
zYI9cl+jq{tmv^pA)6EEHWtwN>Rl#8Go~*2X!Y(Dt*TAp!j>dV#(-OCuCW=n>o^$2G
zvgIjNxqEDbDmv8f75SX;E40tIVw@}JlG2}lZnN@{=ex4XZl@IO{B~u*mQNnxf-DnH
zzf(AWKG#Df=wQTyim+8D__e0*kK0!6qr2{2x$3=Hy&NrV-t*5E9AGGvpD|N;!#g9-
zi)MHG_L@E4`}tRH+>T9=(j33s+qCD(M*nKLGdFnI>ctCcZ}Xj<y40liK=dJnl{bVO
z&wo0e!&?=R5_U({ZB~Rz;JSv}Kf6}H%YB{t=)Bw2Q*E=>ZgHs9_6R)Uc;<*xHKRb?
zgT;?z6lJDNycvFYuHC~qS?}BLmR&o)v_0_rbFbc?7guxd&M)q^HUDE?7;4?O;kW+U
zO}BRNsd7%>uwi`6=4hcFdQbAN@k8ArpNY@IJD>jw{iOf;{E=@D40fu!2k#AzoAP$o
z-7`=BUEHxgZ*S=mhpL$|6`GE&PDzZGoCX|kze&x1^q=9SP1!6TE~n&z<rZI0ypAn?
z$p4?gpyG>`*x%$S^{;ozTM6FZw#{tb!&tUQTeMbc$1nE}^|7pKmhtpStKGYC=G%3l
z<_CQ3f{rv;t$X~;zUu8A<NpjN`B=*YK3aGgSLE46hqdzE-E^frV!;MhO|5`JyQWSz
z#>b5eJ^%Lf*=Z|3T&XLw^wLBBprX23yU^=D!jH!=eGGqdNwfIh!a09kT^3E7*7r2f
z@6g+j{JH?<1OX!^kB~`zehwCIB}INDcUfOrshqq}I{A9;tDEWNnQMD=MC5AvPuH$W
zx^`;qm!?T=&(?ct8NcH5;`*W9m(2X_%K7|E#jCgL|ET;evgb<YirVy?{XfHsRnNWd
zFZtcg&8O+!n5q0PQQ?u{<lhed_v1aHZ{LkQaPVos;`huw{d4Oq92408?pyPC;ont)
ztE<+o%v&~5cjcvyB2_Ps_a~P43mEy#7k-eT9jC7nq$9TX!R`|EFQU&kY;m43x$t$6
z&W6?2Ia+hS1)RE6`OA}$JE+0tVFZ)0LBg5L`J((%YwvEFdi6}ognR7FxqCFbH(p@0
z3_k02<VSvF_WLidE02fWn!mn6(Pn~(&<Y<H`Imo}KR?&IWy4hQw@QJx>>DJ%9DMrL
z`ER$wrK4NswM_16-W~gT()M}I_3bQ6r))dj>gCWl)#F_9_MiW5^~ot0T(?@zQaO3!
zSDE&$M!TBC940>LIxy?iuU%S4yQa-r65+|K>Eg6dVWokQqy75&nHTo>w@Ys|I`X<N
zfw}*<i0Rv1Gn}Sc9dCI0=kpqIu3NiSnK^0Ciaw~RKFKpu$&yu2d823Fe+HGxO4(C)
z7A^g}P9(?u;Y24_3k7HG^7e@fXLT*HnHC!R^4Hqc|B_~_3Dh*GA3VpN^W)O&84D++
zPP(KZKHYf#`K4*ni`N`D@Zjr*wZ|{b(~9+8{U<2VLzBIS@5pL}Ny38ib+^8YS2rBG
zCDCx+_CJH)l#pxdQ&(*B`1qe;7Vo|ok#65)-fPq57jl31S9ssd&f@;Nc{Mw8Z;te)
zj4zx2UOsPc<yu&vXZP{;`TC{GRZhOEepaqrTX1>Wt4UkhMT!y<U1|@i`S8?!Eqwg#
zyX~ylV~*S>_@8O<Ns0ve1(w@I|FgcjX2G2`yO{P19lq&wm+?~20{)i%Gb+q8-+sSt
zJj=WD7<U$r--6W*wuR^B=}uX?JK%=;Oobd<?W%46t_JrgY6q7jOc!cZjI?n%5&q?S
z<F7vlnzCoEGiGXa*wbhEt1pE8=G8nV_gz08%WK_QcIRDC+ulWUTUvGY#5@q4_>RHz
z^RM*dwHebTPMuhlYR!}T<I}elm#+R!Ub|sm^*_<2>Fa7&eGIy_a^9Ngm}?>)k?N|c
z364tjPH85&y?c*{JFY8S(C1t@-*K<h9*eSuZIAEm&$j9<-xPK=<l_Cjh0j8STIAwp
zuWDrOnAlX{@&5Z|Z9avE=VmldeB@zw^SR@lQ?mB&#J4D~SF)?V+`N2mfZLNwv%go}
zW~p~M@|iVlyJNHGKg0C>$Nz3!a<a1iey^xVF$>Rp=g7W84`=_*ak!mOFLSx-siWIt
z{+&5lEw?W*o%+vUA~ZuqD0zyH&)+taN#8UTb;J~Q$UnXJ{N?<mf+p{Fg-o*B9~Ez(
z=$Lz5Xv@N@t7pnK9r)lV8Rls?=cPJ-m!<A*R-W=#1@^z@eZSNj8S}t6Z0+iTcd_&1
z1O7IB`YIdnZku?!;mzQtii2l-%08~Pv2nP$JT+fy7hi?t@$FVhucWIxsx`&gkFBlu
zwOjM*WtQgJ**j-tr8Lf3l@w&{Ip<NzWO*y~hWoc)T{SO1>32VB-ADO$h0r6ulb_Do
zUG2T*c6mvV!LqE3td$`%pEOi{QdXG6mSiZzVZUsh!QW4BX7n$V`Fl3_*~W?uGE+Yv
zT&8q%(Ozw{SJS+giAb?4x^VjRJ*i**cF9xaW6NXGSKN7g+hWam<^>GAagR2J1WZxQ
zW^CwZbPD1#{`a5Z&Wp}mljBn)&+$ExKVBte$<y1(uQyM0`Hxbah0B|`3p`uh`Tq8N
z`)lX6{H^MQmIU_eFC%+PcV6Djbx1;6Q@izw&XUe44Q`Ic_m8;W59nN$_U6h1zM1xO
zGeuPWcv<gn)1CD`=xS)La(?)tkh{(m(Fw~qxtZJ(<>mKv&2nd%P}!4kB|OaaiKwx6
zj!4Ogg<rWJ+D_Z7zJ;myoBcZfxfx59FDo1BvE6?uXSsNB+Vz`!PWxkW8ba<1E#jG^
zE-|N4qF;7*=Um5vkd+nJi_h2g`Y`nFHoEgV^x5?zzg~DR^prc~sil6*&n3xtk<ufc
zTS{dY*c(N+Y*?Npx_zz5?H4b-<}FK|AhM)&sRgH?g2?w~HI{j|3{*A<Hm^x8-kM!<
zTJls+zwL{v_YPS?Sub_B-Fo#bY^s9AB(DX+49EEV{~n(jWAx~*<hP85SA~1|CLh;S
zpSbXNdG17$cNg+RY!kG;6|D5+-Tha&yZPW7HDjq+kA7XM*sy8kCrf!9t{c%Ce=k=F
zE!^{lyYI>!-+Z~bcQUgst*_qlvd~~bT1xXf_WgM`B$b^$yk5EXsi4Qq-d(-!m$O)&
z8Js`SmG^eurl?h!hpW%ce7nAcp*=LD={8Fz^X2%7Y<3^-e7m~$&7=DFJLUFow^BOw
za;8CwsDAua+u458UatMBUT}Sf(3Ifl3o%ibJeluIWVT(tzf)?}haK1BBu~iAu6Z%B
z(jxHYHqFreX3K+qwQpUYzwfEuy~WOUdu_LJ>REkJOFWvuXz_r}W=^kl_Oos09`-x$
zb6;k*bWfD_?gbGO?k&!0dKr4}a#rnDdtZN#mTl8k32pVB@`~d-N1scA+qUhI2i}?|
zU-<O=%Yq*^h8h<Y4t`o4cJ$h<WryE}1!*>Q20q~k->odWf5~?PzbR)^jdqq<=sv1%
ze(t}p<gn7Gm2Go8b2+^(1~pDu@t?uVV}jFei8t)GO1o!IQo8VZe%S4oTZQ^6<>I3P
z^Ixm}+Vs}3Z1G}q>u67#8D5Q=F%!>fh7^6iZ~52cdt>Z@Z@-pU)fWCSQ5Wib=6kNz
zY+>>)y=kXziQIW3I&F9IYYjfDmfus>om5q0IbdJE@nX1)(Xrs$(H^GjC*FVRbxW2Z
zx54W2@nz9xBFs(g?k!(axw<xAb;Fd-KgN$j7AQQ_nksJ;QunJ<ty11wAf|+Q{(i5N
z%aY3PWV&|Gu6pYyn|na$TEzn8Uyc?Z8&w|fo3d?5(d2d0Z2BEPJ&)4axFmvmR^F|5
zz9FAI3M@D#+xdSv@}u$F#fw6FHq@?=c-lGpcgw_wu2;?$M{6z5TwbZGKA+=(f?Rwd
zOV78y=ibq7dl=_@l@9b(I1uvmRaV-(`OU2ywQq!vs=73HGS$cYU6382`}_U<r;mSK
zONp^F-s+{+Yj=*pF>sPwWy`nYg}3FuT`6`uweSO9t(K>3`K1j0@-OYV`LD`@3zmJk
z@~D|lcjM`aA$2K&ERD}uYF_M){H?+%^__iQ-IRI%E~>^lJbYce{IA~rH|s)@YhvGT
zGpP>t4>}dLq`~v>{TZDH*bPMI9eL)Q_xC~j=U;E5ZbjQwuibw6>vC_C4zGYEOLiXd
z3Rd|p$3O4>vWZ7E-7c<<{JMDE0wad=8-HGzs=IB|f^AZcsdCpYB%C^^W?<*>_TS$v
zSx4TO=G3mv)RQtWxGiP8@K&y&e_(WR#PTJMTHz`mHCgx$OqNi1e&pXhF@_0GitMH?
z+<kp>mqV%ctJ#^ksv$SbB+?7wE3L!>rmM}G>*#6_<MA(1Kvm&*<vG)(XKtwpHZT2^
zT=*+|7O%VL`P{#|zyJHbvgmWg?jYN+N|E-Mpwqc$7^bMOG^p7<KBpdZzG9<+iloGk
zw?6`#)Oy>@eg+(!H)&zmhwX8SjE|;HQSS&!SMz_`<Y)8airGt@o0lpp<lbLv+hv(>
zdHw{`ZKrF*Uj}O~7k~Nd0D}XA<D5DCmrvSPDqZ^JB4w)Rc=@}j!7e}V=<il6p_f&c
zg-ls_gz-RwkAMN=L4~D}*Q}qs?sQCYGqbJ>Wqo>V>dBuK*DcrdguHpTNy@x6o3m`~
zE%ojW-F{!MCtQs}>?(af1e8p#RrCn?9y)XC;1XqaYr+17Oa5573q6^{_~Dt&#)yB5
zbHDTiUbto(FeSw7L(nA;%>ZMW-!pueKdty_`E)<;fu-|q#jKk9iQ)M9xiLHDtTYMS
zGJDt0i9bT*4&4o73(c#|{b#o3_q70iv5OsA$}BV4zx~eQGd-}7OGRsq^lACOdum_Y
z*r#3@%GJbG)qkkFob^%He}=QWW4A89Q@E5_mvttCV^Dv|1pf)flciRF+~2nLp|ec@
z)9w1{52J6?B$hHJP1k8ZAhy=aw=s{WInZhSkrkhcZ99eKx3Rxnb>%+;`z?=K_jJ$C
zjm_D=HeSS4qF3+PfiHiv=bMU@oL5`;EA7>_1<glNbeA)ns?@6V@DRSyRFm$LpPTfS
z(Tyv9fx^Gg7YxO>=3jqdP`8ymJUHUA-{mWeKgzjwu9ctZ>J%ifse0n=EysIa@@(^)
zEK=P1x`)YM&mlir<b+mu(9h!aS9Y~ed;Va4VXn%j$wvg%u}B!s5jJe>_`T2FMU~z3
zwjW>N^Zik6tjSyadrEFAUpHlYYpu1@?9{baER&oMi3Jp%VMulIDepAA@8T>IwAfSV
zL0|ge+jDc3&K+lc75U`K71c>y%j29XK6^!KYT2IfnZaxk@L{gY(I?w%x6ECV{AbGq
z_ig1XoV?l-m!I2q>94fOs>OvK%d#hyY6QfnT6{bCyWirXe|@Uxf`HwV=N|Xhy7%m2
zRL<@x`L+MHHs%HCwPvS&Ulsj`(>rI%?@84&XPjeN#_}p!PLiW{Nm*#v{mX~HFBRut
zo_cvnzd-SiuPc^m%vzOO(#w7+<bfA+Z{fE&ZgvTL?kDY~rd3xYyJV{JPyX&6HBY^U
zp<z;G<5TDFt5eUKEIk&vBC%;kkXO&A{tV?k$Lv`6CDdZ#l>I`ZH%lC=zx;Wn-a09d
zLQTuuy&9RbgJhTaYM=k7w?Q)Kbc~mRi}r6OLj|7)yrPxNn~rTh(-&Ntxbw?_CCqIz
z=Ci!1pYtnd+5McQ3+{+dV)YL7wz>1HA*#A7L5sca^6hQLz17_{VXE80B1@*+SZ6pt
z%jgiBcUwjE@*`gtEzi80=&`O)<`tuMsiT5s@f{Tp2AKzSv)08##5fzwd)dKWVE5wt
z!bzD=Yo71DKG%EEm#(to)>ExH8MzEL1^F<g26=5L35idc=5#E^@;`&$qgnSkSsZ_@
z|Gc#3+VndsmepTfS>C^4>LjKmYQlC7Y}W4gN;$T0J=f!3u`qGdtVee5je~rw3s)4j
z_`2GQPSwBkgkPav(ZBKUAOFg0(?3t|u6g@lx%}VFQMZpT%R86cUv;zUMy!|YzoZG1
z#9nn(tkqPue7EkNz2>!RKGSt?-ag(Yr|_ia<YM)vaEFTzH15v$th6-uipkrcMGHmw
z#VuKNc-TeigoXUKS^t?YW%tTZkmWx^uG{h2lVAR7SX7iulnp%Lv|Z=(VlUhK#amRn
zs<OJ41T_i1T<aCYpj2J)$?zUq$Fzvwf-;|0%eC|fzTD@(S153j!@8sGF(u1lXRj73
z4ZdQzbH!@ilTMRjf_;`AQB|`&P`7ca|NYP>4>p}CR&F|e;`xndy|dfzcpQFXq5Rs1
z+v;21(`74HZn?YGbakYsNAAhM4h04uW<lo1&-j*X{}HEMGR<Q`*2i=HmpYH0>O1GS
z|MPPDwV9!ByJpvB_Xk~GTFO^Fn_1C7;ZxsZ<$XPEx2|21O|{=q`Jdsq7nAi_qbW&7
zRef96dWV<aE?s*^<KB9wA1bX?r@S1Gm-Gm89<*H??U<b}VI-z(Yvl3$-m*zd;?6}j
zjnDn(PXDUDOLy&beZ7q(3%!KaWy*xhb6Kbcxh(nKxTqrZO7Uiwayc%JKi`)YeK1%(
z!Qjx^br*j1Z-@=)3M|Q8HHjhEe~rR2zR=SfCb=I;-g9hC@lms-T}pg+8X5Xyo=x;U
zm?zqJyyWt|>7_T;z5MFw)OCWni)j+S-u?d!TbRFJxNi1$cC+03&wmWPO{V55&-L5x
za{HiPz}4VgtzN;W*i{98PU2%P{JHp?OU9B&nX(?kI0wP~+X|lszS|lk$D(@R(gn{)
zoGhHj1s{BWV%>3WukJ)4elE>-9_PYY!a9TI%n#H^^-@?Sl%nRw!ZG=%<p+nqe=RyY
zd5oX_4*D{6?ryImyXqr$1l-@7<@scW+`CBOo`fgmR;)^w?>3gm&%eA>ScP@^mb|Io
zKCABPk5zSbDJXO}@=Ymyf4=bAuq&<_iPQ7Fv^TmP+4(eZt4RK~o23~QCnlcgQ&?)d
zvBQ2(`9-69@;es$x@%1N&+u5%>(E=NrjRAuUN-S>mDw@hcKPx7+m<wmM4X(EGXK+a
z@8=#Lr<lh4>bx>LcBTJKsf-Y-BTJv|cMH1DG%tX~@ucj*AGTj)R~_<CnZ#3ih2i+Z
z^<`ZJ?DjFCxjW-D^t5zt=7x9OZr;$c(cSNHR`5O%9)>wG^KM)g%?Nm8enL*tKIW42
z(SsJhWM<WWIlb_U$Tb)Hy+Kbez4@qb{d&i8fhGq=oACb(mye~STV8SBGIz7y^98Z(
z`-**ZEvlzpF?{bYZFY52<n2_kF0QP=)BR>QIgRf*Ef5yqKWF;d+qxoIVcz$J=TA@g
z8GPGf(J`szy_z0@UKWL(8ZwnjIRCO7ubk^N?VWI3^<5*!+qXqBJ3TKMH$U{6d-GE6
z&K>jXwsNoC7Cz<ViiPEuqB;~N%l5HUFX0SZwqe)aCN)m08Rrcue+Q{OoOsm!+sUiX
z1I^y`Uf3A(c<S`qSI;iHGbMP>7t{XF6Ytg3KB~Jdy?SSd*u#R#=kvER-}F>zIk|B;
zcL~?h#4X#xZfAM~F?b5F@Y%B;;s5<YWNU`ZriEAkO>$(vEt--n*`e_~I%9e8QZL;@
zONto0etvIYV}Jco%4F~H#0Jq?^Gh{`0mt(WmajiGOG;~(`-(i1lBo>y&n@w{WZ(Xd
zulhejEpN8U#M=g()35HcdBqXhGb^UtH~Z$cZFe@^sXwBz=F1tYn~Ypq^E5e`RGw;m
zmOUx^*oyPrJC(Hbm`wX}3*Gs`Po2(ZZ9Ls#UAlbkyl2rdQ#ZCQN>^AhqhaN5{!69_
z)Bb$(iY>4)p7r&}Q@`M<;v0IaJaSuieY(xU6p|qN%%`UQ3cv7r&jN{F(<KKouN)S6
zw=rbPBO%#9P6Y-h1<8)T#X@!e8MbZNK2y|dyZ!T1OIlkEUc9`1lFQZUKLd}~jx`z$
z=?~_!Fw_}-UVF7MR#$FAR?#QssnQ{LO_jYeraL$}B_3gAmU9S7^Ee><pW*W71wDpZ
zqNx_LHrh&V^?NMCev5PlxIeu3YQoH*CJ!SQnKEUrP79SkuZvl9<)%v5ZBf>n$++@&
zZQH7q2V_-Q4>1N!*H%%fcCa&Gk-PD?bnTMDMRVhOs~#$PGVMH;v8_8Z_)@#pI)A09
z^W@&82)IwOySvQe!bV4viLLc}>~D8ni{2T?l9KNg`}3>n<+yoji@NSD&C|Xap>Vlj
ziP1x0u9$=0z4mT$`0Lj<ts-!WqNR085!<rvc2C8t`ie0hUoDUSem(f`meTL$IoDMA
za&NB`+i+`1eN^LA!8sj&tzXnB7oFamqHv@6%8!d1mN4&@dvYl}dvBoMa>>%;6MnHP
z^cAQ~;1ujT@OU5Rk(pm&JcDEndIZlOuaCMG9nS9gq*miq+uguIr?(;efh*SYiqGd@
z-t(M^rCFZgPlo&!jmDtDdv^aB0#7vvip}=RyjuLyD5H$|cVDB4nYNeimh)z(vg6kj
zJ&lmn(UbLd)DfL{QlnrgqrnuN85gxV+js-N@7c=SpTPL*;@-~WNnax}R=tkjAG0@Y
z+Jsg%r;|Po2FZ4p)%VOiBD_#}+GEF4O*LMRwoSU}^(5at?ANimi+37^`)Xxq#!qf}
zW|^#3=fhL`C3U@I@02ASr^0Vk2VKq!`TjH1;#b>^@7A%e&hB5a+WXq7CmaQZ6CR2G
z3^52+->786R4g0(@bry&cMiQ-9B|I_yuH@ixWh^(RD+!Qxvt&6W|jMP`>a*V)`_er
zZTVFC%ITq3kIR%McVQ#r`z){LMQeTd-9GbRfuHs6zuF56Ui)u%P~5^$u{kL2^rg1k
z_w5qv_V1eVbivuF=d^7YRK2y>pQ%*z8!66zB<=mh?4{h}zqN;IIZxPY$<;Tcw>f<`
zYPkP9`Q@)?y*k&l%(UkRi@Y=2vU2m0RSrP~D$ERw>U;iFT;lFd^XOUUZs+oT`&9LL
zXWZ2*Qhr<(U6p!y!-@P4r*b{DJP%A%DN~o-U0-u<`}~g66PNHL|26r?J@3K$>l%~J
z2IuS#47n=pH+9;r%$Sx5lO}mNvN5Q5C{^?QShS)i`q|5nA9_FK|1?;vo;bJV?;Q8Q
zSB1{+qjr9Xl3BE5LG(ff-90LNA5SH?OkyiyKel!AzS5ovY{t*(meh%))PyQcmc0Cg
z{c-$?+kWB_(WNJxL?#OyIvDX@B~A3kM23m?3|G`Cdg{rz%HPY27XH!baI(bbtV%?_
zJmbrs*Y+Md^wmn>mbUmA4PD1vjkR1_e-5AT`6*y5Wny}tbK9GxNBSxgg!^L}`*SWB
z-}9Wj^rZTKhKFl6yZp?*vSL-~t;<KbIFmbA-B}vcJ0v6|g?lppGjO)}iWHrcdwV|r
z^fZl`@58l4Hpx%SU#4aK;Z%y%TU)2BOH<A-YZ9BXOj)MDQ)N!&{JH$~hYxhlm8=$d
zH*rbx{QPSTZFg+mB$pX6Ht{`v-POEu?ZV}WU%~@bR<Nwx@W3EQrGZ^fHQx50b<cVE
z!)2eMUrbhK5j^id?XcbNYZnV8#Ej0X#grU)8MS85_ElW7maXviV`Zz{u%%U}le4M7
z&69ltTk&dh{zTpbn|Jh<KGwSBuYYKBdcmYer4qRZpI*PW?aJ4zRfk=6otm&<!PeiC
znNP4K)b_t#8WVOab4h7xfO~vB*T<m6(J~^R*gr4yT6)>Mut;lGu+B;?j{rqoO;7cS
z!gFQ{J}c!t&fa&=b%*r!NzXqX%{`(o;<4$pn)~MkXZ5d5kG`|z%RwH^*6O{b3%xZP
zZ*MfdrBY#)bI0vo#qYQ6A5%{C`=pB~aa_wg9eMop%V^P%n0YHJbEK9wL<e}bPX27?
zp(Vs2Xz_NbjDNf5p3TMy$?rB=?Om0<y1MAC(*C<ox8`p5IJxqRymzm~v@=5IH{YsR
zz0uO3=6nyY%N(T>PI~J?XP*D@etTPG*oBWe%QM&WRi+l1eO&Q-u4xvxSFFSm$EXQ0
zDnec@OBjp|4Ql*zH6C62c<!Rd#ZBq+=Q{sq2+G{lbv!(Vul`rx{H4;jbf%X6>vdRJ
zr>LpQ!py?__&r0DolS4#jyaQbxa5o!<X<gX{P5lB<$0HneE#`m<*_$!SF){q)+_vB
zi-ky&&_zvFB_Yo_E_2fD{8k%$Y%SAW?KUr3@JIS<r@O9N{~0o}^ensPY`bKzimPhc
z{9N}-9)Up>mG(JDcz!AzQ}cKv;C3X2dHs<&8gjE|^_?rW`}Qtmy-nAxmCM)`ZoBR6
zn&EHs>5RZSMTdrZvt;&zz0z6VUQ13CjGX<aQKhG<t2R?QX7cnCUykZ+_sp7gJJ9X+
z3TN%`r~Q?Vf9{?<zT<?QpR&~4Wjm%$abS>sI(w;ScEvQ~o{G;4s*|ooE?t#c8uDI2
zWuXVFl7nWSvPx&*F}6ZuHq$wQp1e<jgcr|{{;R#pTu3#_H757m+-m-n(eq}#%-?2T
zykf;x&1}c4)puO@^LVB*GH|}Bzb-x}Nmy@p=-Y^Q&b1HU6xeN9-2dTp=9(*+m;7x~
zY74_USKpQOx?8lZSMQ35bKjEl8VB5)4T2c{ewp@j?d)K``_&7q`Qxpgt~PNF61(%~
z+pR_Qj43-*uki+-DGE0_)uO4)z{H^-`BTnBr$(WpIr?1iiFbZk@>`d*cw7?f+b8FI
zZOffo@3y?mZ11|^bs%bQe${8??H%XkCEl*&+}pU@+`{pJ-Pg#={gNWuGRe!IyM431
z_U85y>C|(3KOSIN8298rGpFQ-IgN7mvF2NLu_Y=^ivDYH<KL9NM3J2wrt1STpKq^v
zzw_-WXYuNaN)=I6-=$u<oR;7EE~<GnKC`i&_D!fcBzER!`~M8<GE^UIJ1w`hu&-&2
zZu9zYRU4*dy`FHp%QMhvViSX!Tk#`1<!yVun?06#Ci&q>rpm@MOSj6#CCw_UTln_(
zR;yLgt1{MR{b!hSxw&g6r-W@M!}rcPDX%gvs!JtU*r^0fX}ub`xl1c;(e<e42mcvN
zR`ltvOjJ$z7chB}^0qCrl#O0ak<o82neXhQ`%dyQ-_;G%Hk`V$?Vgi}<tmZT56aIU
z2(L(+;dU@yrGn@2okv@%CoT8*SN&zJ$l^OEotWloa(bvtTKAt}ecjzB6}{ZHf4COz
z`)Thb#~I@lyEBYw>H>?_UwsOa{~4yspSI`hpL~PCRQo`a-kDP&ecjLZ=DC%oTNfR5
zv&~vid0b0AY1-1Q3@?HjR9%kDVW|1H@$Z&`&7CnjnOS&`8^3npuDB<ta^ThHH5{9x
zUWQz7<u-XzucKkcAiKbS;{5kZ3qP!Ql5{-za{ZY{3yt-*_V9!(ns8AgT>qJ-k%I6E
zKL)#WE349oyQRf>wOP(3y=t9oS37fq*B-mew?$OdMc4AgV`JOnXX^JY-dL)aQhN32
zX}PO5-8W@)Ys4-*KD9hGqm+|nE$gNbr|Kzxc%C1tyVWI}_vYnt<>Q6@c_F+<5AQFW
z%~1LN)A~M<39C{LYXp4fcruwyVN!vM{}I9O7sGn9IXPB8Ztj;lm06->W?g*Z_^Exe
zi3_-{T-FZqTB>HLuD~H=Q*-|9g8Ut|3*PO&FWuZ1YPxKT)5RRtnx)HZU9As3*;T=m
z8sz@7Em@HH%VoVo3=6*eX9z2Bbv4;>_xt*mZ4u5(y%J|t`0xJQ!NhR9@y4HTA6KYs
zFm#iR=6-zP%5~vy9Zc6wG3d8VSuRpBRbc4~nMs`X>at&jJ=U$gxjlWJ%lrPH&z+~2
zDi%h+kt%6j-dSqHFj09T^X(5Rm4E$j2j21&34B>;YWVDig89l%Yi!PNs&%}OW!2M7
z)wZel#311Dx$hqTji2wAotZTuv*xEzT?^Ndv!`uVPEFsvV2!F#xc!tRlNiHi{<$b<
z@o`1kv>V?WuetloKf`rVNwwP4VtLz@eW4+YFP1(jj{NS~B0QOWr_1?+!u@kSu6y20
zNPjx(?3R{tcdhyxnc1%+6c~OnYO$)ERI_frRsG|@HPQamHN`U9j$hicYeEy#<*e@9
z^|!X)&DBt;-YLW9RMG!d;Y`_2E72c4fg6|;Z0p-Ls%P9;o%ec|RsE(Jp<xg9N3Z=L
zv`Ds8^FW8N<!|Q6_s{iuuUN;nL!@d>gYCw>)o(664lY^sdavEaZ)ZDpdv02*7pvtx
zqx44ZqD34kLY#I_Ch;WM=lXxExbe+(x_$ifW!Y*W=eA28pBj?;@Vf5XE9$bow+j}{
zYd9_?rSdDoXUV^OEv8~Ei}$xff6w+5b&i(ReEitWu~Jg@;^*gHR>vhGze+gOrkWK@
zX@0f*iLg=n-)i2h?_Pb@yRV&ANS^yb>~m3;nDiHyCa;isjsFab_We_i+x@8QNYi7M
z(sNpzGOsS}&G0;Zdy43ptC~lR8XmRm?hguXnN)wLe~)9Yt4ybzsLGXn$@Rs-n+=&y
z&9m0~^Z1-;!PY(Ay&D*U{GYJ1-@W0M($oBL@$=o=dmB5;UTJjX9yeR<zeR6%*n_8H
zw_ovn+!>|gwN-cP)xa%T!G5);z4gq(CJCS8S)#ElWC5Sa5szg)?%&<LxYDLD?Okx<
zX{g_#D^Eoqef4+s`+N1N#qtP~-l~_Sn*K@w8(YnUxONH{sIffG-psP%yqJSbn_YdB
zTi~O~EKllJA2xY(<3EGHZQA|Y@+<GU-QMlDbMMY=D`%L8ToF3`RDbamugFx-XTO>g
z_2wxtiA-f$*{iX+E%U+|MytxcnEwn?Mv;#clZB?KuMOc_T3Ysg<5c%;Yx$*?uPght
z+xSb}B$=#zZA+^BOg@QC^|EAG5Tv51+*f+Bj^{^0A*W-BR>PB5Pq+OJ(o_AY(NN}B
zRGDyT^Rigi#9b*luEE!Cn#W4jZmnBs-=95ytA<Z?*d0gX6HNtt-9Iij=!Ex{on2*i
zv&evd$65b_JJuBGbr#>&*U9&ho_lQBzPTDZ!>;Xk|DWOMvxR3}<jXZ#IAwRs9Y1kh
zZqm)9>Dya2%DFs#-uvog-mwJ7s}r4kqO&w>OEnWcqccjm^geR4_c$GY;UF*koB7C&
z!(Im@8s)72O|f0%@$TUFc(%o_Gd6aGw5M)&)C>x5nq<gO&MezkSG?js!z%;5bLRgU
zZv307eskl0hCk15Pc6K$@6eB5t7`Q{6urC{I08Aj)a-v3exH->=zVm-o0)pzK9B!|
zPv8Gz*8a6;+KZ~UuK4?`_qtRr<NQdM<<plfKH<r}^FPDgd-d1DtTQJr4%R!+=WMY)
zHbz^c<W{?0b=$eyS*6}9(`PkJ6X4(DA~L`Km5^!Njz>;zZ=MvboxseO5Ohy1F*SBW
z`P1zWqqOgYeG8Lf+Iihy!IHstk(!77_Br*r5eW~4Pl-k)ot}Kp=7YUF@5{AEzc?m$
zhO-}Au-ZK7+UYsDIYI$K(NUd@DsIIeRv!HL*z5jd?~v(l7S2ds|28lteoN{ymZp7d
z?Q(aI?%FPtF?(sIccIEQ&#$3JI#g!L+1>nG5>md+oN=jJuF8{s;rG$*1shk)i~FFS
zFZ1j38jd|H4@uiC?l)=T@N#fenO|tcdEofW_Y2=w^zLjnFg~(pn{wfNu6eHSg-x;w
zjOXxN>1F<!x9L}4z?M~9A&(rUHd#M^&usbEZNZP~o@1XpCDICZGaY$;YQp)_oG09g
zZ8tu>zO-F#<?8ZF+f{R?G(|^*g?NU@9C11M@BY^2&O2w7S2k{NIQO67%w<j$MlO%)
z=Q9udXXt8MvBm87iWNl*kIMw-?2^q@v|+U25q>v=f$!#f=FpCcBbS8w3)|%1s()Ov
z;a+r<@7fu2e)3<Mx_#T+O?g(Smls~oyt~TTak*E>qB`{r0_yMQ-P%!5r{}uilBwYJ
z>6fQ+9^0qjp~|MXvOkE`^tz{$^zzDSA-krs7V6D&(DQivn`h=R)8(SAjXK84+aA{{
z)NJ%se#Co2$W8j&+7hE>^U^OS@`+7j?g+DJn&2)of8Hf_nQA}N=q-D9O>}As^>aLV
zwcq(%$Lwx{0Fkh$Q-Us;thPoWmrfn_2%ZppTcz<9gR;tVtNC3H&otgHn(%^MVP5-3
z%eB3qw#l|VKAimduXftS;$^-Gi;JeM)NK%X-K`n<@Lj9Re}+J=pUOfe+_C42HJaQM
zcsKbi{Io2vP~*nmIqS=RUZ3l48gzN7$+jipFPCO=7p7=5sid)Q@t5dle;#$?T#(z=
z>HAI;uAX*p>Y8<nQhU<hJ)imfb?}*m{3h!I6}Kn`3C(+&$<QQ{#>mCKecpY(RPV@C
z*FMAMIn6tsKPdLP+#w#YdgB`XO@~4Rr`jw?sY_aJ{V!R_!&v>~mB_wVmWmxOG*2^&
z)Fduq>|FZn;dZk~jzS-6pHmepEN&c-WMMG=JlTHbrRbOIuAPlM=awxLn;m!4jp3P1
z`GRc|58m2cFRFNCLeNavX%)?e>}(v3cjiW}{V`dh$)i~=y4LOMl{d>%JB;RJeP3x7
z?W>z>8zt>8#xEk$G*7_mfPzL5`}|vWznVL=JPsZ&bKZ9Q`JIc|mA8()T&{fP`N!4e
z={d10GVW$hi~Abs!>8c2qep0^ZKIugFLP;DSg{0y0rLW7yRXOf@^3y(IFOQiBIMGG
zc$0HRAH3i0xH9SGOqt*l$zSL1dC{GHJb3>-=Gs<%(fv2Ccv#B({AW5h<TY203iE%4
zvpEdxJ2a9XDsNAG?)_}P4o{`B9Q*WT8T`MVO-b;-d~WH>Rj-1CewX@BUAQmqslR5U
zvG9$5yRr|x^-uq{SyS*UvsM4jmC+q0Pdw!f-$r*7?bWthYOCE5(z4LQXQiX7t4hZ~
z%l{0YmmK@WxH9e3zJeW3-*=_-q*xp{zhF^!>AKH>r!V<h*e&E+)2XfSDe%<pC+R0V
zJ{dn0oK-Bdv9;9p5vQfB<<Ed{^Vr#C5x2V5ei8kmvp3*w_7v}Up{=4KE1fNDRQPT8
zRlhp&Vb`(q%yUZRTOF<kuDragU)Q?7=)ym_ndar$*On|>x~*8lQ-QT8uv6vVqVywK
zC-f^CT6s<}F<8oezcqF1!-*TTPTbbr>$Nn|Z`UfeYtnpER$f@DDB#C&%+C1sxz*Ec
z{Jf*v`odJceE3<k#>UY*sZ+S*o=8ac${n{g?<{-#N@wAfQwbl{_dJ;>IAh)g(>>cK
zuJO0lJA3-n!mCS`<Q4hWY~4HQ(BaS!)#*Pz1^URjZ&dj4@o`W=bxGHq8E)*x*X^xZ
zXC7`<FAU1fn{E9nP%AoLmn)Fd@eoT=U6K3SwQH9gv3m0C`G=KDW)&+)?cS<%H+0)t
z?O^UoPAv|Pkl#}!fA;<Taw^)`B<jf>eXX0zgx+2VyOnuMrDg4vUz);m)Cw&Aw%OSo
z_Fi-5l26Gt?Ub9FEW?EEs|kd?(9GGv#ib&A+;IMkAAi4^PWe=p<*&ET-mbJWpW(jf
zr&-|^s&?y5K6>>V+PGK#p8w}uXPu(i1jiedTwf<oc$CRt`APKCE~%x{W=&PCahFiy
zD&(|Z#<ysCho@4O-;xrh&K-FdKg|v^xv;lY@yY55-xl<r@(WUbEWGBd*n5?hiCz8$
zlUC|0o{-fR%HZTGIW#+vi;K%h<v54E0snz{&pli3<h0%pVqg9+*tJY#dxp{CsVS~I
zzg-FBaO(8*;Pl@leEN6ozWWLpuTRhMmio`Idb$5}#W?njH;UH0+|KX1h1IOJ<shTc
z(h4rE<M-v?u8)1<l=3R0vGA5ba*)ZkbqS}!UNt>f8NgwDoBep@j|12C7{+<mRoDD$
z+gK2KqibqfU+1z*)5TA>F><gvoRl~qP(5@0w!TT{N~D?}%Zc(#7Ln@rnz{K-?iBv{
zM{M4x`hK*VvwGqIdG=#|+aByOU2;-wYo<&0t>h}R#jnm?+rSo-xms+)?LLLJCo_~M
zo_xbK$<6WP@niX_eK#zx6<7vb^i$@Scp3cmZ#<Wo(ABo%LHpC?!&WBu>#?|ShFLs*
zq~u{CX)(!Te@1Vo^1O`K(YguKY*x4z)Ovn3F1oZUuqt-hJQZWNTY-P}DRBl)ev%;A
z`0I>kx#|(uE3D4%=bs6i<R@5a_9V1w|J~K=rf-*By|QlS(fTu`PpeuFpZ@%N^S$(E
zY$iMPW@$#T+8(|<|8cavNce$c|4i+KTDQGcjXyinso}eOvijp~rYpAgc;4dQf8oM(
z@im=Gd3jbXJQWhWq{!7?)nsW)pO*SbmdPw9I*$0yO?&k%XzsVKi{7l|R@*mq?(W5F
zqff8hQ!f<$mRqHe{TQRfvFiT}+q&jeDrw53OYJOO@kBjXAlvKOVRO@>;BGDMFLP|_
z5(JDV-&SYwFkJbrqNdz2)`7j|?X`xtOcT!UTxff2iRM?;Y14KkZS`a1_HS6JaI#-O
zC0p>r=ez^LdnG0}o}QbKH0i$Y%U!dJzsf9h-1MJeqeGeHiKg@?6YqPTebQa>rm`<2
z`P-=?|FY#8_x48J&6{g#%ILE+=vFA_&zW3u{`dEFZhY*hs!;pb>A*A3O?TRLTRm@o
z-}^l$=4HOw>8sbS7CBdx^?sL%P~W4Wl^ENh(xf85c*Q&>_ra9TlUDoQnH@jX(Y-Z|
z*}5VjWZK(bZ(f~u`4xF()~j=yzp@>@ylZv&L;aAmYyH$VE}F8C_k+Lc(irDlJ}wW9
zDVmNfue{}qE7rNV2fpGwu=w#C^&Q*`*y47qs|_tpuKvn;&1!MQtu?u;o&7ClSH-ST
zdv<BUmK8-+bI)<wOc9%Be9U_CYT4(<gKppEOqlRAf=%K^lV$$HQ0{&5tWQsu9Y`rX
z{`AzW>bMuZZ);ZGo4#B2(!P)30a-F@>z>6_En2LaCscgmqn65?$t&{ySxqYwJX&yO
z?OBi8x0Um=+HN--6nPes#Q9;FZP>%5c1=^=`F4pc@#^sOI5Oj}Lx<G!Y2m_)x_#aE
zpRR3<P@b5an=ohApW=eRJ%?8CuXhht-PEc+o%!t~4nw(zx1Vn~d9inu$)%eXpKm`6
ziaw<>q3fh?sdLDiw%1uwIZaDimLF8J|NO|LKkdEM`?tZnPA%u~v(5g0lS`UQATu{H
zJykzBKPSJaxF9h(RfwI7m5VvZ-HA(+iw`21mYI{9mzbL>B+ezuC7fqqZeea{VQ3I<
zZftI%=M~`U&LzVo22q-yl$DxXqMwwMU!h-_o0%uXB*cJSwjwvN0xT=a#SPJ0o|#gT
zA;c!d!o~E3nM;{V2qK)ElUQ7=UtF12oLVBp%f-#b33mZ5lcDxXaPdHlNXswEO)L@O
z;9}!q0lQb4OAsOhF+3+RDK!UT^I>)_SuSyiB(i59N|>Y=xWr+>lAMv4mzSDTEW|3s
z%*Dw3#`*eBwR9#15XcDnC&{(w94D6;%n2E(nduoN5Iepzae2c79_+i4)QS=zb1qXZ
z<MhNL9fkD7qO{E73^1c0GdopBAt^H_2O{K?T98?sUtE$}TC9*?T%1~3j6D=8Vc~oB
z6ekxi#6zih$@wX%DMGtWadK@t#mTkl6zBgN{~s_2c)5AHF)%VRGI%g}Ffjao#1rA^
z>>M8u<nQL`<I2Fu$iTqk;ggw{&m_RWkegRh6y)xt5D^)rz<!Q_iGh=WjlqzCAu+kQ
zz&F?}1mu5DS7(LdAa^Ga8-c%FV_*dFF6(*(C@6qfBw)ehf}#=z2F3se1_tAl)Z%0Y
z2F4Z!28NvSk^+$U2L=WPk)&)8n~4R)E{ceZVqjpB0I|~{Y#k6g3Bq;&u~TyMQWzMR
zKyEHb$xQ*Vk1#MWbd;5*rZ6xt`!Fytw3TJ1mNPIgUtwTiP{}FH&17I;{sGdPo0?e6
zz`()-5-&+j&R}3*F<@X|;3*0Ta%Ny)v0-4~;7LbfCn2#x@eT63bACZ(5h!6OXe4VY
z7+P9dD0rln=cJaD=msPvXD1e=C^+Zm79{3XGB7~=4AR9Q1PWUPPgiF%LrXI=T_Zh1
zurP$cD+y6Tkc^0og2vD1KyaKfik(4-A+#gJ@)t2MSbSw*V4i>wOPa~Ru)URmLE;ud
zOl1KB1Ahwx!?yEC{t*MEmyD8<0xNy}^73-M%+zE(P=LZMBB%ha5;p_MK0T0q@UT;G
zNli;E%_&g;rD{+?SX!h2E(#TN(GxM*8j+lZ<PM#n)U?#1)V$<Wg|N)j^31$+1?T*{
zl+2RM{5*xsJX|qMrZyxufS8bU&A`AQyyPE)NW30{z<Ch{=C@}UScJJ5m}BNKFfcH}
z<IRUNi~*GM!sVYs(mluoaLNEP5Cl^{0|Nt7ab`L={Wu4OC?uB_l|ke|*^+^kfs28k
zL4-k)L5@M0L4!e;!HB_}!J5H=!Ii;_!Ji?7A(A1EA(<hAA(x?up`4+Hp^>4Dp_^d>
z!&HV@4D%V5Fsx))$FPZEJHsA^gAB(QPBUC!xXN&w;Q_-_hSv-q7``$5W@Kb!XXIrR
zW|U-<XH;d>W;9~7V6<a&W%OYTVvJ-=U`%JsV=QH?VQgmXW}L)0i*X_2O2&<hI~fl$
zo@Bhpc$4uV<4eX*jK7$enRu8)m}HsMnDm(}m>ikBnL?OinbMgGnW~stnEIGzFfC$Q
z!?ca*Ak%54t4t4=UNe1VW?<%K7GqXq)@8O}c4qcxj$%$@E@G}@?qZ(Cyoh-n^DgG&
z%$J!TFu!5`$->4W%%Z@e%VN#q!4k@n%u>iw$I`<xi)97NHkKnSmsuXNd|>&<%F8Ol
zs?BP}>d6|xn$B9z+QvGKbs6hc)}yRfS)a0gV`F0zV^e1{XY*i-WXoc!X6s>_%eJ2F
z0NX{j$82BO+1Vx8wb^ai{n-=QOW51kXR@zh-_L%D{VDqo4sH&44kHdXjwp^ijwX(2
z9IH6?aa`hf&heX5fK!#ziqoGng|m{gpK}T4F3$6uPdR^c3391(*>Z(&WpOof&EQ(c
zb(HH4*Jo~SZY6F@?m+HL?gs7|+#9%0a6jb!$s@$0#pBEq!&A!B$FrR00MAXH&%C_6
zYP=4-QM@I*eY`7q5A)vR{lO>9r^n~Xm%>-iH=A!8-zC0x{G9x%{Eqx_{8jwZ_&4*P
z=YJ!>DWE3cERZNrCoo%Jr@(cAuY$sYhJyZrxq>}{s{~I9z7S#;QWJ6&N)c)iS}b%}
z=&>-fu(GhTaI$c-@Dkyp!cRrmMASt*MY2S?Mb?U(6Zs%2Bx)iWCR!mnTXe7J12JYX
zH8D@I9I<|}O=8!?eu>M9JBp`@cZ#nQza;)$LR!K>B2}VGV!gywiC>ZmlCF~3k`pDj
zOWu`Yk<yY1lq#2+Cv{ZnjkJigwREy{m-Hs-TQW>CS~5X0)iR4^&d7X~m6LUsEs&il
zdsz03oS2-0T$bDvxqWgk<%Q+#<TK@`$nTebr68){sF0&DL*a<R2Spi055-c&g^K4C
ze=Dggg(@{Etyj9M%%yChoTfZQ`H=Dl6?qk3l^T`RDz{WQRV`FARHv&RSN*Q0t`@G=
zuC`t6g}Q{gr+St8YV|uByc+fzg&K=Au4uApT4-i#&ego2#iV7bm7z6T>%2CTwwZRO
z_FV0YI;=XDI(a&abgt|2=sN0_>8{p&s3)f9qt~RjUGJT~ihh*-ME&FX{|rnGatxLj
z+%^<4^fGKR+-3O5NXsbMXtvQ6V_su7<9g#A#ve_zO;SzfncOfHHuW`aH$7zf*Ua3k
z$ZVb2OLG<T1oJuO*DXXW0xWtgPFS*9I$73R?y>x7WoA`kwaMzewXSux^(yNZHtII1
zHcM<C+bY>6+Ag$xV5eZ0V7I{TfxV)AqWvQKM-IvkDGtjVo;zwfW;w2LeCK4~ROGbP
z>ASO)bFK3M7iJfCmoArcu0pP%uCrY4xhc7&yRCNn;BM+(?S8<6)x+CkqQ`YlS<ht8
zm7X8G%)M&8j(GEU2Yb)*e&nO&Q{=PDm&w=5ce3vtKUKdxzwQ1E{+|Am{qF{-2NVYE
z4rB`q2%H`GEXXjZCg^yuaBzI^s^D)SP9gmvw?j2TOG6KZ@rT8Ptql7f?h-yJ{6U0%
zL~X?BNU6xo$Q@DaQDITbqP|AEMo*1?8e<yM7IQ6DJ+>nDWSmr7PTbyj{`kcB%?Ydt
z;R&k}{v`S*E=v5G<e4-l=|i$}^7Q1_DGn)9QeLLorcO$Io@SdiDeXnNZTjT&ml^gM
z(=y&<I%m$x{FvpDH9zZnwtx2W?Eg98IqP#dauafQ<_YCx=N-wH%dg14SfE|dR`8(E
zs&H!Iha&HyWyOrevBf(|L`n)u&Xj7Dww69Dvn!ij_M<$kd~=0hMP9|3O0CMy%I8&X
zRZFW`s*|e^)hO09);z3rtX){gSeI0Hs9w3grT%Gyd&7!GuEy-fvrPs~lbXIXhd1wP
zk!@*cdEDyWy1I?8t*GsKyLJ2g4wjD0j&q&HoijWCb)|Hj>Ne<}*8Qg^x#v``VegFI
z|9xqF=lae1=S^UnkT>DlMEi*=CJ9cen)G<G@8oS$6sB}d`7$+r>d9#))8<d-nqE5n
z!3>`nJ7%iPoG|mxtc+P#XFJVaKSyRx=bZ0zQ|4Zp=P+;GeA)Tk^M5VKTySHd`@(ID
z)D}%!%(l2}@v|jiOO7wKSh{kV)Uxhnf0ySif3PBG#gUcfD_5?PS=GOqX?5x9muq6y
zTv+R}cE>v1bqm*vt?%Bzu%UFrtBna8uWj<)bZE21=5<?Cx6IipytQi^<F?9eAGT*~
zf3PER$HkpqI}h!$-nDhN-tOgl6!*;DE4sITAJ@K?egF4Y?f-h9;J}-M=?5Plia&Jw
zaKzy&M*@$WJL-M(#4)#HhmSiR-+#jP#GaE@CwHE*IJNz>+39U(OwVjRYkGF;IkR)y
z&YPd#al!J!u8TGo_g%8Tbnvpv<zrVoubjT>clF}6&}%oY$6SAKBjv`6n>jZ>-YUEG
z`*y<});nEy1@2C}Cv$Jnea-vpADBJZ`_Sd#nMc8oZa+?b{Q61plRr;ep7B1L_FVq?
z$`?j2_Plg^dEr&`tEaCEUjKg6_EzxioOc@Uw!C+Ef96BPho>KlKK}pI^I7uq@-HS|
z4t)*ydhc8Ax8L8pen|dU@zdhx@n7M;p8qcY!~SQ+U(LU}{(1kq^FR0h|NplcoI(8z
zMmS(%VrBvX7G`E9W>ywfR#p}k7FITPHda=4HWn5(PBwN95MX8F<l^Mu0&zfwfb=sl
zF|jZ)v2d`mu!2;OhW{U65aeJiXZ*s<D9FGh$jB_n`2PrlECT~0E68j}gN1>Cftd;9
z8b%HV&i_XkTm=}I7@1g^SQ%N_SXmgEnOPZ`m{|lFScMb~*&Nx0l>!qBMT{CJE>w2f
zc<|x}4$+{8AH`IRlbpp>i#R2Mn@p06Fa1BlAjb%`1>73~S;fo<wuXU&lM7^xAOj;4
z3kx$F3mX$7GZPylD<hL21GAzbi;!a=YvKnsVWq;xiAD=I9%L7By7=&;a!`_}aglQq
zhnR}$q(z&8lf^H6`hSaohnbOqNsw8P!Ja`Sz{SVw^+HLLNlg#?KfTL6f4%Ydvd&DY
zh@Zw8=D*_Xr)*ubaow)NHy$qi^ku5vlWty>xc>|av#X=)z90NO>zP;d?+cONw<_rz
zJ;Swfd92;F_6^6SH#jUWi@tHY@%Z(>{|X*VV6dv*Z+83a&Lf^(3Td~$w<kFDbbL6r
z?#M*hZ(D!7J^%FHVL6X!{6CHgUCqs2*1Pi3)`m?}j9G*0_65A1G*hKQIO6w$)q3B*
zi#;#M{gJ40GUQ0>l&$%vlTU3`nxG^i>At?=g@;haTVW>Gx5`YMkBk4_W{p`C<hWRt
zZO*>`413Ito1WiZe@|HY@Jltr1+r!eZ$DJu{u#ZG+029Mj@avS*Q;&kacx<-X>XLs
z#$X*iS^jCfjki6-ZZvN{+J9W;MysZ(Qp?}$fcu8exA3gZpQ*O-bjPU;TC3!9*M!tX
zzss$E+P*4g&zZ=>vqE3zg|%|3-IaJVFG?!orf6u}laJN$_5CkvBsCSEF`PYpfA;J=
z(|(m5s&CoUgg$yO2}p=-*q)@DczgfDdp~-FCn}nXe3z@)`*jb8LeHVS+1EaP@?Q5!
z(Cd}f0zM9#<~twvZ)5x!88l~w&{w-%rLS3Lr-`Jkzwxy~tK*hK7)wv2S7UbB{_XO%
z$0a5#nILlY?(3I+TLhRUo%#5oCpa^kiTB&q{$t<n?|<0NvNVpBzv|b$2~KOSwQLAF
zrYvEm>^mhd$F{$^ZiaeEm#9>u-@3b*Jics+ulde2dud%!nU$Y<iT~j3<ob2L^Sj<X
zYmx3tKkq+pz6GNL*NT>9-5V=v=I%{s$niImJude{zr>SO#Zu;9ecGiD%m1l9d}Wjw
zv@mDEuLkz)=-hwxd*7|sb^U6$=P%n?#-Ny|eY+NK$v8SegVA-H`i2cVEF1(xLe5`b
ze)-&nNmm@ytB-q18|yayy0h_44%>m`qne?DfB9cc-%_o}v`n_O^-<Xs{p>q$*snj{
z{;lAqOWG;R>rWl`Yj9|BZMFSnAh71vdamOqt?w}Z*=LaV#DL}3t$hJYxVgSw_my$x
z+*0MBFo{vsf8X!lwU$5j*-li{b3gg@?`yuO(3002Nt@V%R`qYx%@*DJO}hBwHZ~!d
zpsCC6Pd_KBso4FW!G25mx4Nw6B@+*9SNi>*p?-Rfte5HO?|W=iBjYSnIAj_km{0xv
zcwBzk{lh$(o&nN1<?G*bII_0wElD`@;AMzaUe40Cb=Rf)`wsJZ1!S&RcYlwucIf=I
z3sn;<uPp2{V~H#`lfPmArtF~6Q^zyhU%vibe3I+bsWk#R4*O?xmhBXKc0fM={pZ7K
z#w^{uR^NUd7WpNSb#BS&bsCSS=l*B7WS4*Lx^?2qQy~&}F3+F-S;#?g)*@l4V{28i
zB)YlRzFl6pgQY0XUdOIT=&&h!h{38eS@k=fylZIr)YhCN@!j6<`t9x8Sf*Gtwe2$B
zT6IlID9?Y1@)=JilNOzZ_uKy+C_ebd@NxT<3%RTLR<gdkc_`8K(wnp2yI-1Fe^>bP
zZ^yTIZsG51SxmgGGXJ`teLAD3P<kc9+<)&?{?_gH5%7}ytQ}od8f(^as@U?9pw5Dh
z11-vHk3G7(UH$9h$IR1OCOyCZ!fwK*r`FS#DEyS$AZ8hp7ZcOJcH73sKN6R^>g~4Q
z|Mkq_Pn!4EURBUon47fB^~(+6GUc6nw?FQ$F%-HX{HJko^+D0H%c~qtI!fP6Ql6k&
z;JbJC{h$B*RW@At?cSsvp)FGVv(QE9#>BI2zZ-AozwhBRYBpNsutVHl|8_=T)~stH
z+a|Qd3JE!=baQjR&#%7!a+}o7RMj`En!jIuz9wm4`cU`DE{&oS*Z%16OMiP_`0?%L
zr&F3dHkf~TdV1AC;h6_=6H_NW*rIzvrMvg+PkHm_9}Zk#7MLV;ZT<cw{E8P28z^hB
zXh^*A*e`bH^SS%WY)v`rzw^$&HtkpOr%pz#su>|0m#lH{n31=jF*(1#cf+5B3gY?6
zpTFhL-l7z;WbQ0G!%ows7o!dc*0EiCyZ!h3wr4J@r>njD^X>BMLN4AZtDQ`aEnKom
zX>H?X?mK_V-XA|Nk>TWc!t!_Kv)}y|AM|eRpTq9Ze{#Zcp34kj2iN(Y{G;RcpW*HY
zlc_>W)=mBScBjqPJtEUt)Q+(}b5rJWab4$-8d$hry7<=|%c4&!C+U0byzG#@eZf<f
zq<2=oouiMHiZiII^me{0+ixMWQ}U9~gZR@^1QYrNcD!(!6Ppw8@>1FR{N-YI{xfJl
zR9)HNAo|O89rt|mYc>-Y&y>7gePHXPYnQmPzE<TXRf>QA_3*R7rBiNM-CS4OZ%>iv
zve=`iaiBfwpj3lL5^vMnqkZhgyI9f{51YRJWB+yyZ<fmLx4b?D3_&hDx7IQKy()L@
z-rw!tejGY;_=$?zt)KHJ{9Nsu&>ht4%Fwo{Qb8e*N%V$#V)n%3kHyt*Paf!E2xfeE
zZR*<ltIR&C*v4Fw2z3^T=8(Nnm4Eb`+T#y@+Fu=znX^Rv=YIzM{|s-1RDQHR(hG5u
zi|kp_a3?}A_wU*h`_FwpuzQQ~#QL>WW)*W}LS%padnC0;V-ed4maL;5iKQxk{%zmC
zeZ%4EIj7F?-@3D{x@|?phUtbbcRO$AZP~uZAS*$1nd+&nk6rg2`~Ku@+WnW=9~(qe
z4|wxm{qyG7@BJ1Q^M6_W+gB(Y%Hp_`No_9UmAvgL-7yT-1$h?7cTb6&?fCln>e<%8
z-bbg&Wikjm=k5G-O+iaz<yG%BQQ!YA`*m)T<?a2yKQ^59y|O@pE3o16v)tEWMay+A
zy1%_{u{v$atISu+1jUTBv($^$@BQ}2f7_G!b5vet6zW{y%}gtN{ygBjVRhpir%Qb8
zPp|t5UI^YJsH5p<H}wW<+NM8+xBCx^ym)wN_9ON);qSjaVdyMgmL3>1Mdnj~=%t16
z`zEU$4%j!lNPp{Y;r|R4_on`c5IA4W_4Br0Pr=I5tPQrMpW6#pgd{2ZTb35pdR^Zx
z_q)%|AoH}&Q=dPxu6%y&(YIZNmp``Z@UG+)dzZ8sJwL-DcWv+abASF;M<m?X>{K*g
zJG}68K`z5A=g8aZEsCEpzkIOq%96F6cj69jT~lXpQR`;=-MyCAr<!N<2|3iR-(nHu
z#gLS4vs%-pFVDhu+kb`}8y4dWg8LRvcX@HaG`4-KY2W4vk1U#gPTRXS`+btfAGZ7F
z<&0v_HD7so-rfF4#j?J&ib`P<R-5Ggo8)^iI(mQY0p-W@*RdPwEs)*w^^5d|XID2L
zzR=SC`SZ&M9g;eiJiOa37w(%Kalh_y{rP>|CM*38&au0?wZ4A}uTKWM#Sht(7it#w
z-k7tz-5@T1`rZvKM=vi|uL#-wS?-<x8FCd<PRrgb6?pw}J40XjpNby`j2YI(RNa01
zd%lMAROeIw0{jhTJn3l7zR@k<T&~g`e}C@w1L}${tWuNRepO|2WURS={G8y5B~A7Q
zM`aHvFn-FhePcaGPuR{W!aA$;>X+x<GZarbcmMiXAMJi=wa%rO#Z5;&8E#BIdGEGh
z@y|N;sR3M`RpD>1KB$xF-5!5Lm)rK;hcyim#`OmE^9$Sdg*^41V9CFB?NY_1x<UFZ
z`NmUN%#KVGx2|9Q^^f`C0H+gcbDf{pCExdOX_RL#uaulBW#G4<QvK77H+6dwe>P>z
zy4kK5D*k-EMp83_X)EW2C>2$KI|`SJ*ssqq^w7)v^Jn`lk<(eT4^BKe;q}#Qft<Vl
zINy~2+4w|c%7(0}q6^pK%dbf+X<H(y62zb-+4jBcl5T!>PUX+vcQ-z2O;c5EpRnI9
z-a3qJgG2G3-v=|Fc&hX)zN6cyBJauGm;Xb?(DTDH6T7<jeb*b$vnn64EBG51>gjEA
zUT%V<+pqrPiTUpzX!=*gZhG;%=HCa_r!2g$KCn47tn1`|uguNB{>@+RN6wk3No$tR
zx7(gEfiaRzP&-p=O`M6{1c{hF-TLIuAK!LbluTusaDH|4&rcbiiJj)JS2;~Iomy+u
zzCpUXo;&yc?bkJ2QA{3d>yo{C?RSI<u|*lQP6-W&T<XSj{EGgqJ8{?lGt>zDQEhmA
zZQlpIk3YYD?(d0en7L$mpIhK}-`)c^Cf>;1_~)36_`_3k&2GHloAYnO!YIQfnx33X
z12#onR}p21d3$@(?Z*ei&zOF04Hl06^Ec6>!|TST$t$OB+fbai^POJB^BPXC0L%Qa
zCs!|D_FZ+oZKcoS(+hc3qb;4vuO=tgwu{+!{7TqV<IuSLeBAcW6-L~B`wA?YHqF-x
zaQs^PptR0H$TiEZ?t*7`ewXmhUgebBuxEeU_ZQjAc!XS!^!~-wSAO};mp1N6JX~)Y
zrCz`Kd%yeJUwb`+t1JBiOzLJWzkPS%vvt3mj?6v9%@pWyqxkLZ{r2l_d}98wMR#pw
zwA<@+O%^V>f=-#uKXWc^`nd7;>D=Q-*XjRw+t%L@*n2zq@_|bXk}cBT%D(DNdH(KA
z*+20=1xF@1@*S|hwb!IlkJsv7<$=I%oxE4ycSv>IxOb;k&)!Ee+i613<nP<#E2>&=
z-<li7VtVAn!k86Xw~If!#`OKipT8TP`gm<lO}Kvh`Je0OxI&#gU(cyI<GF)J?qJ(g
zbLJxJzx8d(Kh|--%X|Lq)yF4GfA(7hObyIQV$I=*%h5ghclSSeiBAry2NzzSzwK64
zHjmQu-%g^*6P<nu<d&;_kN)6Z{N>xm!%m;Btq*)3RrJ}wvrj9besj{T&I_DUKX>kb
zSorbtF`o7+uC1%TE!ckR!+!<=hWzF68L!yby)&;Y(S2W3@!(%X(sPS56P-`Jtq<S+
zo^!j^!I~|MmySsHDt%M9(|zNcOzt}aRiA*s75Q3Q|K6`Y&~kYRi$j=uid1NXwt$_P
z{PM^BdR4*B&b519o-h91G_!SPKw7<?TZBTzV=jiUucB7ve>iu3d;DB3VVOdAywl%I
z9?mneZX3VOzNDg)w7+)!#+|Z%`=7pTJmb9b+`fHtzJ0$ZZmuux%bQZo5}I1^_8IT`
z{pYvmZkQ~hCN$MRu{H0$o6X9nLaOSzrQ4VA>~WbAEHX*sO4N}YzTe-z{b!gveOJ)2
zlu1H=X8yH3*cudCUGw_S0SR%Y11A#`a^LBnKX=W3x<t__2g&BTThs5quF!~%k9a;k
zJ=EOO@d_vV)x@H@+5G1@(ppbT9-dr1-*)f1M6N%oYjbz~eJHec{T<N<nhMozzsfh-
z9~Znd%Ygsvt*^`8-9KX&Xr^>4t7d`Fk>`#*tA0)G&N2(kJ)z#ecK5@uJ;!Es<@&yl
z?GZFK+I{^sr?yOzzHvI!@$8t}J7oA@{zyJKV}Xo;RDDYOzjv-8Q#Iyn{KV$5S1@ZM
z)AL;41l{eIdn~#2XR7Sw=AK*nG+xVn%95!@3-^5J6xp}b>}%B;rvAgP-?l3yaup{z
zXP*6=S37G%ijTsE*Y}jQ9#=9gYfIm+xORo`iErZD_it13p0#MxlKxuFzy-BmFR7i-
z$=n`yNO?(bfQyXLH19U{21hgV`@P$LJo_P*$}(m1)&28&%^6&lJX1e>eT}inugI^b
zQYL?&{oVJC+x_15`pZ0LAL+Wx)j$1nt<)?VPvb+)4Tqm!`*3a51c9`k1xdBxY;QMw
zxqUy+ZudKbwz@1I@wBX8?{2lPo8_qTi2G@%d(<l{LGEy-<H|Z!OFBzr*S^Y*nf%w{
zK!C@p6%Vd^Up^mgeQR2`j?HuDRW19M>qj)N5ioI^!z#+KS=Q@_V79>FJF{b|3k<%T
zaxP!;qb_MmdX@V8D@S{pHmrGaVSC(MiPg^pTZKh=RC8w?$n)*}ms`Kw-p75$6qe@!
zt8LxpIo$g9NN)odOXGsAfA@DJ%NV+KwFFx*CtYYW`*mM*bAe~DpjTeq$?!A#9z?0G
zs|>98vQ+PR#oT~y4c!gWb9djLe_A?a^&uw?-Q}@a`7&iyZttpoy7aJ?9+R9Yvv%LR
zZ`+R_&;7RH;4{@qr#rIW_D^w$Sa|#V>kD7oUoG~yCOD;=g+YLK@9w|$w|Qr6V_bi3
zp8vM<xxR7Ax_%J>?>S2x&V_c=?_n?U-J#w;M?|I7XzSB$yMJ9hx^T&}gbV)rg_k<S
zIjRUJ>E_Qn-k-mY{q9qJz4Q6oEk5uZWu6{i{I!W`%_{F{Toqz>-rs&<XYf=fqI=sd
z`TPDEoVCpErmcE!k`v-%zt-L}lUM%zoMY!I4~7>e_vbJDuD?Xqvm_ww!ge)X@7lCO
z*EYvze=>Xgvi*EOR>-Sw<@-Z;7c{g?iuZ7F<>usDa718bg46rsFK_RYzB|!nr^o*3
ztBq$DwfpcH{dxP!=HG;GMo!k!(V@5QEv{cCDlFjPxubCH_Gh_*&#W&^=v5V6b0S%N
zs`-Wcr*GF6|5(Mj;@Y3v@i$vr<2Yu{?B4!ac9P@DFi(c+n2331JM5RAJ%3Efaiy#O
z)YrAQpXMg2F4?^7;`y&z`Z~8wti95|{`w31Hc3Oz@E?=o_15z$th(Us5)_u3t7z5t
zy!(CPjrx6Z3s$Kf$n&d=56t?#r`oEK_u_Mwu3wRkZb`SC`W`pdzn@d-`f732e+K2V
z4~j!%7&#`!x_&Rak*VyV{@eEFoy5+W3m4pbWA>%})C~Q+{T-1#Z}>Xe7U_uGTxZ`e
zk>!@pz4N*M{JHIGjQ6}e?#q>QEtSt>gVgSqkL&B-By@U;Fs=VAn{)BWia4IlGaj?f
z40g`qv6MgayZBFm5T|<hm;9+~<I*p->eVQHc*vx*qqcg5g7EQ=k8b~GD66`%bvO6d
z_?e&O3Pjf};!bMWw&WV8l5h0HeUEQX{`oL~LDGMF)X&4!C*;<KIlq`#YqhqzJIDN|
z{m0fot(7b61McT%6m^RAb&BjL)(kkkQD()Rf6Bk+*C+))EYe(eyYh46#<ZQUm$eHB
z#X9b<jk#O@b-&v8C)J6QC;N+<J=6Qopj~}Wtbp;>-O{fa#@YQ2pGCKPe|FubUO0b_
zCZDXs{<mKm(^$4Bo=`og9;ov7Hk0+_w?F<f>=Wb?RWSJc^>Y0j0cMB0l};A}qbwtu
zrfoaGWG;K|-hYPkEOW|tJpcUZ_tgbn(LKAqZqd+sA*6L;bv1Wh$nDo(|GjNI7s#%-
zQnvcN+?m+^x0jux+xop$uc_wK%QN5m{ky&N<D*$}hQZG--{LzH>3W7^NurF4&f@Ie
z?aYO@fB(8K7v8nRLTq`gxkjZH+uoc6o^xDxJ%ij=CxpEJ_5SOT&Xx(q;pgK5+*!<H
z9^^|Js`dID-d}wG(}4iSk5xIhE}yt0BPO@y=b@V-0&7Yp9S_}od;9gq_brjjQrbPX
zUq9ibu;NOT1?z!~@UZV1(MkE$kB{bfg}83}S^82=*|k>rd93=S7q64E`E++~u)BYr
zozwIApZD+mA2_KrY5ffn*3?+$5-wc-_V@kuXa6%i&nQ}MD!Tmi>@61RM~+-#+IToE
z*7bVfPUVjq@Ap5b(_5A;a&fO!)+c`csLnMavxOF9M&G!bn|ObozxBTe&zX&9{`@&_
zZ&lo?)|X%V>-TZT&-qI)hG@t{_paS=C;#vJA`OxGqG#t{TH33<|C;oN36C~RYd^cm
z(#m5utN5)yh5IGq&#KRq&A!l|rumWkkZPxMWnSQI&#Pw-su$LGC*G91&wF&{>*{~^
zwkw3cYO%O*M2Op5<!yoelbXpLm!@s`v9p2scCGR`Ug1PmRXrb%kZ$SXpK|%fpEhY_
z3!D%CeE*ij^fdLif7~S{w;HRpiv=-!lm2#yvr&tIyXfb&ZI@41c(cjc6nZUI==EL^
z#F4<X_TaPoZFj#*Fz?i2I{JR&t6Q1;={%(dTX~$;WNSwkCg+F#{>?w9`ZI&hWXEiK
z_fsDZF^6=0eb4meh1rCk)34|s-+KJxx7$4xOEVJ|di`ezuP<LdMPldPy@@-&G?y1<
z_l8|wQN+HF)yiqwhQ{Rhsablrdvg>HI_EB6VBq9C+BDa<ewR+3`;&URACo(xrUx&M
zyK;Ztf_xU%6$kgvP40ZUn%BISf7SQ99y0Up{b!henK!5}Jhb`c;@jWb&IFps`_*6J
zbh>d`rp<TX$Jo~QACvF*uy_ZztW@Y-7k&0WLnDidSy=x1tB)rt#cp+76`K9(Np}$g
z)3w`kY=cgoS~AV^+jrf4^9p{h4u5I>x<Vr2=|t`wiEFZ=BPRHZ%l@nW$IL7&a!`}!
z!57Q0o-1lxG1eT9k1$NVc8gIfr(F8ef`9Vy59WUIl$^o%<@3^bM`rFcw&&}%wWS;r
zz3<bjEq&+S<2>E`;~(F)2AZ%PG3#~yeudesGlONC+FkpP9G9ZlCoH%#<q|XR8o&F|
zzxnPRXFNOA@fGXW_4c|8Z_X_Lx_r9+T5aBxrStA3hSlHQ_TZcO25H%&f7ky=Fjil(
zL{*H>->&1=OLpM{J73Q+nlvlm^wXpIb9lF8Ze)llyuDu{D{V#l*@w#)?w?w=Z%Xr`
zD(0O_T=w?#d{$!k&rt5&Shn5zWBt1yJZDbMU?@7jf8IRb)A!U`1?Kagk$t4J$+Rq+
zK_s^*ci%Pj8~3-rZr@<1()G#KLS}2XB)4Yrir0$;O!bskeh%5dpkemqc5!0fe})f+
zPgIvaE9X!9WHV!~YvrCuPq&n(%yqRj-@BgX#O2DyWSNR;<T&%X$lmO)-EJdic|@~A
zEz>)s>6+PUwr`*ICOzh~*1F_wyH!j!pLf<Qw=`XWnR+Fr#~%G>xb`FDq0oX;=`#KE
z{yaD_r)}-8=W?yC32RrHE<LUoqJQNb+wJ>L9{a8S!56@AWw+q!2~{S2Zfh27So-kl
zOvloIB{!yqZ3x{d{wwPKato79ye8lEoj?6_NpkI<K3~87oy=ONsk(}S{ecX%ujBjN
zZu6BhBz~G8vOX}*=jgfVdFg?SQM{94USHq8#xmQ+@V0YAmFK<dZ-4%N|KmO*v!>$e
zGX8g0uO}U1pE<>8e}=;4d`l4<-8*alGjNx0`_HgWCM0aJa#Y0YbG?6lIy6|_DXo<g
zmRWw9TWxB()OKle`}ZGngpww$TH58lKjw_R{wt5y>*th)teT?pNq1|k$j!YQpR`9C
z)Uxdsa4o5J5;~&&t?&3tjTOC5Cb-{W>6R{kP=D^)@o7y*T`F2vXUUsCPktKlY1M;S
zKaZD-3r_t0j#KS;-re{6c%}+1Vpy^5x14WAQq;_aw|`FD;K_92JJ(X(_Z#=!kAA=L
zkD+XEamD3VZ*JY5!?I!Ha*J2HW^!w1#d|RJ-xc5X?biLbEpd{WZ2P4D=5D_vlO=J_
za)n{`E8SYBGk%Y^pRYg9=he+~u>bshXF1K~vG#8n_s>|>^y<*JD`rY}%KtOuh#GZx
zcq+`ddhf3twXg8WG?m%Tjj~r5wKdg~>Rx`oKfB&9|HKvljtRg1dMtbHweyxl*s1Mt
zu@dDn-ZDN1cH|wbulp{h@}chCkDNn`7Omd=)?c@r_j|{yfS}ck+|rR#UhmLdIqzs)
z+2iWwwk4~Lmvt5V`MK@4?SggL*F$b`uh8;%b2MavZ|fI<+^yd>iv8SX)6(GJ`sMld
zJ1?2|GVHVeT~PQid-b~Y_ths>|GwQ|Z~Z9fP-FQ*k-IzjYQy9Lmi)0T{p~#A^ctO>
z{0#}LZ%;nnFVSzBmE^F5wc9G8Yn?KKP>-XuoW-5b=U=w@252W2UdW9J_}rq+8rq~e
z)%lcfb8&&K%P01)|8`GU>DqQ*uRv+aqzei)TcVF8vmIzmOfuX5{a2!zu4|Wn{MGsy
zmZ@U<KXbckiA=q+#%%%biHEIg?>z9*@b#Ou)oksq+{A7z)7!T*s%6rzJ(()E$m?zG
zcDZYhAJ?ClEOJywtHgBQF-F6rV;gqrTsSH<c}0_6NOv{&w;OLi*X6c&eV%j6?yvOw
z*Bo6!{;{=+g`z#HT0FYm?`jDCaR2fC$1O8wOwE4sRbH7nDsI&ZTMh$<*P3Oy;k;XS
zzOT-)@87>qV9Eu)Ez5n^O}O(UBJ6%`*{cb%UQ1r3aFt(P_kHKKBX9Rj+<yER+tjoV
z4KlS2*RRk1^YDlmk6nC1f>leG^M)Hj94fE3e?NG;M&j&B^L5u=*Cx4_Pg~i*>7lp$
zs^Sivx=Qmu=O4DtnB<qsf89HGjfFSUE@w+gXZBjwV~by8XZ_jzpFvtir)8zP#=1HC
z3)ws_zm{dV!BP^!At4*KUFF+9`I(197!|ozmTmjC{r$@|S|JxdGsLc82u)bS`QQ)V
z-SfvKcqUC_G}Vpz^Jhz4wexL_b$i1^l{#*D|MFnJGELF(PsQWJ+j5Sc3pRfKm$*Xh
z`csA^=T43b*8*~HJ>P!p@z+DlPV+AZ-QId^-}P_JihsYqUgfa0)oO8q@4C|7w&Q*E
z_AgtH82{MwGH&%--o$AeT-j8#l6aagd73fE=l9S1{h%>1`;70G=YM~?Ph0-vZx~C&
znt)fVIfp~fo!*~N*ZiFC9K)CPlrtXtE<AFrxH+*g;Lwxi4g0={zujT|`+@BP6<yW8
zwpw@oB}q*)y(U~X@peqkk(lVvDHHE^|L*U9)bK>vsp{_HQ;aj!Cbajg>59@_yQ8ou
ze|!GZ9}jtsoZ*{xZEJ4TJ(<?FgsQ&}74|K?yy~~;?d{o(x9;7y_;fPqiq-A<DT<7%
zIaTf+l;6aeKf#{wKf`gZuu~$%pKR_u|JDAKg+nt*NbzY~kH8V;qsRWpA7^lIoDuRS
zOTKO`PtdBU{|s9MUa6*ORtp{9zWinGae4VOFXN`G?*0Dxsh2}jvq-ZuWBIi>hIOlJ
z51Sr6c>lxWIfe`>osFSSB(DFs9Wg=3z2~0YETfB4T{p{is!#CW_wU>BrM_-4EBDR$
z`SWw*qO_>5*EU?R2&oih+;PcUdj4$w-VdfL*8JGN=?nAuD3=A^+hZ1l3rn1u8XBCm
z+w|?9{g2rl1GwC`$Eyhh8Z15jPKqUEV$7zyV*eRxec$B2-FNqc(vp>?IyS4nyl~#V
z;GWiT<vus|fNbGhs~@t*zkhC6o5^)@-&!xWs#?8V{-^^yy4;g~IT?gWx2eplc<_ne
z>}}h^3=Y*hrC+uf#2$}7(b{pa+*c*)0F&pHvM2TDFW>#-5XicE{j|erQZHX~ga(v0
zZri+4WfSjKp-P6i+uv`y_ABvn$b_3AG5OzC{`%ac@=E*q=ZMVQvcwk(*_*oG<rjXc
zpMITdWy2E2daE5*&ui`b@a59<zl)!&jF{%e$lJltz_In8<=eL>?j05s)zP@{_pR6b
zGP$emoq0yR^%-efoq~mI^z?4cKFz}V<M!+7KlgczoKsyj<Y)ffyQfDqXj41eoJV$*
zF+vLuH*HJ3;t)MC|3pOmg9JBk!3n0fPkh?He2w1}A@|rg3ICW&n!i1?S2R6Xa5c~K
z>iG*Xyyui2KGB$bmt8zpKfCK^h{Qxa2YL4EZ@+fE`Ltz{h_fN9<0S_xzi&I=zQ6zR
zwvg9M$!X_)W$!({K*!Q1%FE%@(`~O83S2t&ZG!r=Ro^zt-w!Ik{p{MG+`5NqBCH{k
zf@4jV+U)h)x+Zd2Qc$eF|FnA-xmU%Ob>_WdRy$&#^1DtxzRuw0r0h8t?|!-X{>!tE
zNghhezHfHsRG;f%Xjn0WY2U&PTi2dHCT&y|P_*iw+xqZMj_I0fKmAj#ZI4Nr6qXw2
zSUpwV{P`raV{Ow+{xjT?j?U7rtqVN0jAcbmdEU0BEj_0`PMsR`(D#H@?K?Z&qyHIt
zKPv3Vb<KFg{_TUx663G$`M8!(`<1$bW%dmL*EJlrKOaBdC#@98nb6Je@I2l=y<w`%
zwVwy4JAS&e!uOye?^KT)49EZOE^gPk(0J}^f$uNf*UWxtd9zP1-eVEHY5x@I-FJjM
zZvClaXL4G){7+Z#{5;paweO1(T>aM+d!)@ez3A{2#vA!ZcmHRQ@$gV!dHMODeOHuB
z3Crt?O|7+8lN4PK-8**V`}5v@i%;?zsl}h}uYbnkeCO-+&{olagPr0mLgx0)_0{po
zAK$jA3j}cqabJ77{@S;ea+zJPCVp(#vsP+>R;|{yyJgSkCz<~(c=JMob2t0))0Zb)
zlS!Mi{;wl5ulDyDeB}!LZ|bWbhUqob>8s^d8K{}}FXel<Wwj-jx3qk~51IJHwo^q6
zp{_Q^w!Xe(tJ}(%yU=@OGMBTo@b2&X^PB(uXYjEMzcux@*t(C8&Yzy5sZy#f`uNiF
z4Lj7kyMN#Rkx(qeXl(N8@znneGo&U-8#e7}a&`5%D8l=n;oJQmjD`OhB>EaAEL>q<
zw(ggcDy!P|_8^vs10nq$u`0K2`M%xu``fSlrjx9xo!_mpe%-&vG%>8vM$KF(LX7#6
zBlE4>eb@ItZ}XhNp%U}!LVlEN<>AYQm)z#oJl;M@d1dk@{(D=0-~0Wb$7@q-@PvB*
z_zW+>s)Hi?r|50lx%KG5t#9=Bu08Iq*D=&!f3$2X!_`*LoxcxrB%S-P%%!l_SLqJ-
zn{DF%)bHPB=3qT4yX)^mmP<2=mo_sS>=3ib;lI33rKKYJ?<(ojm%cw`<l=q!`di=b
z6J;9`_^xeGPdvZ#@#o`Hgzj0D7JmPyXvAf?rdHB0D>m=U;q2z)$sg)@j8a`$&2O(S
zZw>F-?I@<g)g>}zhqVCPeVIPSw=Kbj8n5pAuc_XnGAaLh)WOA9mvXBViau!Gc<cA=
z?EL&3MGuClVa>OLpM7n3&R1Hu=kdlBmtHj{>v(nN>DDLy=<y7iaovCFzWH9cOi``p
zV{0CKdMAEq>A9yKTfcw*Azf2v(bh2O=}+dce_N&Ab9C=~eR=&9UJ=&xo;6uI9xA+U
zcjQmcpTADj$Y_Px?3#UUw+~-8_@lM@tH)!bS4OQ2>{q{+Z9o1(ui%M~YSX&XsyA`J
zUh-+?^O#QbT9~hMqiUVM+0RG+8K%jFPPz7H>+4tFn!Rt#f5&uM{msQI0vtOPr27w7
z=M)^c8Xx*o=--8aEn5}LLX);_;7!Wmd@y(K`TD*a1rF+(s^{n0<yam*y!jqmL+m_*
zlmt(6>+N6U&ogOjrR<v>wLBwK$TgEw{Ig@FWa}EqhTHod|K49C=>37`0K09y!G8u0
z)`fY3PTI25p6LB&V9#InpW)sQsqmRDjZxfDe>1+dEj{~^Gc`o2X|v(peYNk;-TwCT
z+rgIWp%QyFpZcyj)^_2;_9fh2tKE`&_OC3n?QMPkrhc7haYNDKptsevnQIpYZJzSX
zOu&~#f1#6MY;UJ@b*)}KcS6Pt@2Q#BH&^Yc_}rqw<Zwzj(2T2I@c09J8P0455vk&D
z>(^aa9jLgab;}vSkl8D*I4E!z?)<}kOh#a;>hxN(`>(lDd#)@gG*w;4oca6q`<*iV
z6L0g(Tkvke<3)dN*H=5oXx`bv;dZc=@ts)!yV&<X{~6jh9OOEeAbM(V_`6Ey&FuX0
z9={G58S>YvOf0I-sb|V<Q&{4CN~dmm^6R~h&k~&^?}<q?r?VLxc=?(4_>TS7Z~rrx
z9T2-Y_v_`;dD{iL^qz!n`|Z@$#^jr|O6<twyx;M4_1+1Mm7X4Udi~L_6xTWTR;|(V
z%;DQo8k{6DapSLVd;6ca=|vSc7N0u4aNmWM9*aw_Tm0Bq^eCXq<M-L@2tK>~{ZCuM
z8yPjLxQjnYz5Lpysle`I&a0@mT9I=}uhLTowr_v>etfzh)T+oHAG=rb*BXzM4KEJN
zGg$Fkq~LzR?2S8aZ~M>i{x(bC%79y6zW1kp47#rse@dET>qf?Iy|oTmZ|~33egE~_
z;R#z>m@*D-ySu;aYxBxnw>6^4j10S&!mjOYJNE7Ujz|9)BshapRjS_o6@GS>H>l(L
z>x^F}4rLL?uE+bX-7hcpZ~Fl?eFN^S(8lfS_Sy($6+Wt9^;*4oLiE+!0nZ=Q$uGN+
zv-4nN(5ijYe_cN(=H<b_Yqdx7mUXYt#^SUq#|yvaXa8r=F*Wv^E!Vz8qd?|AgWQE%
zl{)*khDFXgGAS>jTui?2hf%byjqkD}KVO}B@3_P&s{Y!&3ul|yeT!JFoYvk>I$q@e
z%)fZwj~>x75z|jM`)}WfecC!{vilnIiQPwRPIz@kJ(!psKX=ESHwLLJ-+MyytAEGW
zdsKK!m0n%AdGCd+LY1~_sT=R#I==08*<Zsaih41zf88GcT)DwqAYsCn*Xy@11x-?R
zeeH20vN?x4x}iEhasNJL18z~)`F>k2$uup9ds1t4`<n7{Lz9iI>K@|WZ<v|){r<k=
z*Ss4!mnIy$AH*N}-MsVCr9I2$>^14gx?U8^wPzCJlFR@`*&p|}%S(Kk{L6#)_ovNl
zHj9q)%~|;Nc|=c&Dc>ofO8q-g4|vZVyCxm?gXN^7f@IIp@0a>=l(JivOp0FnZ=z_(
zRzb;bCdPmh{>!uH9(#PJIwG3S+sCp~&+Z!Q-AiAVpL~=vS*M{?yrc74u%f;~=i2YC
z-+sLm+b1EasyKDZyu8O#B9*eD`rk{h+AXp{?P2-Wb+7CGGpv95BiZMu<5`*i49TaH
z&shd~l)sSMU#qY$IqLS`4<7#+0>y50$K-y#FC;KwLx|4Vzp^%e`4+KHU3vX_h2Dwn
zU-={M%ys#AV&A>{*O`J@Q+++wvhJOKrg=rOlZ*KyliR0wxE8%kRB|&9@J!ItUw!-0
ze}=<+##~b$*_CbGc8w+I+4R3XR$S5n38}G7yLtQWe?I@^z*-lnYlr=IeqQ6AT<Cj5
zS3D$3RD#LD?$~|%rG2*(I%bF(r2SaAx!<?)&n%h!9rO0CWo7)PRk~2jO!}{x{Jk4r
z&a{Zk=bruZeprL5=e7+y8Bz^5lq)*0$hrNFss7JkV;e7^cCUB$?Wz2yY?>SkuYYCY
znk<&)o3Lv8>g&}XYxnz@MY6DRY3I+CcTdVmtno-Q;XnQUbJJ?=rdwjps#jvyMBlrA
z_~RkTlH7fLxzDRQzPhXRi6)8H-+WtS{yc%>=Y}8B4>nEHVt0?v{deJz^Y*w8&1;OU
zCT8D=U|#ZvyOw>Q(u#>m@5^q!4BobHUcuKD0!PART^f%q=vnZqY3uf_?~iZ)dVE4-
z(1ZP7KQmlv{TZVj9B}B*lB5!^)Ctlrm5y$EBFDLP=b?q%P9B^~v!AD5zcp{;qqRB8
zPg1X|J(R9h55Ied$$t7duAK`Qe#OrZ|GQA+SLr#~NYSZg0gI0A)H?S2uWtT%-k9rG
zfAR00zsHSRLCDsGrS3hm<MLqpS*y&Wd)L0x$xr+#!|8h0O19j}ZvDPRkK}gEGJeh0
z@v<P!U$Nm5$M)^p#U9<iarha-MDM?U|NeBC;>mOVx<9*ylzVR4lWsAaKlk_N*yudr
zQaYEro8NxpH31=$=3~2fqIc`fjxJ4bd~~}$rRG7FUdSX5J(Hz7dH1Ykn4R7~|1@KW
zo7BH4tlQ<|uK$y$C=pm|np<CfOC+t_ZvT;*rO}Rz@Aj8&jqFd(pZopSL#IvGpWOay
zUl*`ctL*+QZjNiRTSetdCf*USw=1xmEc95lf9}8YPkADhJv5W0478e#y>h!IwmrYb
zR`Ci)wX@K4Q-ie*s_#117Jt8<SeJYIXEfhVwYT4;O7uCV2(12_n5|K-Q`z>1yYD|k
zbpF<T1(uV#WG2?lzO=94-Wz)j|Ct36rmgFf>inKvzjy7Q#V4*QbGly5o0}=8ssE?m
ziB(yaflVtUukn5T0=xUC<%G7cc<%r7@1BL1nx`o->$C@-`oul&N#oYv_5L}gMhD9L
zx0_wR^Yh?@Hbu?Ls}6@}&(643@>}pn^X=}K-w$4<Ojx+u{OfAzjtLDX_pm-uJv&RW
zK+oydl>EKB`<F@T1$wuzUE3Rc{glW8feUPMu_v^TD>iPwbo-H2xZzy&+FuzHnnEjH
zCheOjwP})bY-{V@?K^M$XE?{PsQIDJm1X<()+R2T5_@x7Ys`wUSXGJH`QO(T9)Fx&
z-CSqzjJs6#%NMDvdudYKU$4)pRi3<5U{n9R`(pQNc(saWhJO0D^~GlArl^mnBNt9p
znDA>yn3d=Kug5#$e2#3)p0;#rtJe*YN&hB(&gv|g63Z2#vG>HI`~8=HCfe)u?32EH
zYR<2phghElA9CteoE$TIu4{POAO5G?ZX`Ieww_;HEdI6XX7llBYgfwX9Z_1u(7SQR
zzCYE`_0Q^A9at;R`W~(McSufk`_V$K&nD|89gGYv{d>FjUG0B{=Nv&28saRo?^yq3
zH*fLEddBu|!?8EhE;-A2>}UV{=id&=C|BXY_}ZG=m)dW>i16J0U53e2m22H>hTre=
zmp`)GFY)P;mq*%_s-O4cW*pM>%DMPSd)<+Y?3}|1=5If4Q}K!N<YL%hTlp*RJxA0t
zp{FyiCKj)e&OP?`{q6qOjnm|xZ2tN6i@?himTQ)4R$e-FTEcG9v0Fdh-@CSrdCH2X
zjzwi#ZM~}kUN2E(z5Dw5cloT;Y4Q#|y|eT8`0lj#-S+Kxs}O_UmruDhU)R_y(2%;k
z*1qD=iPjqovsh0sf3JO?_ift`!C69WYFEDBe!qQ+#lm(A<`YjNX2!(anOekX+wp$C
z-S3A%TMWEh8Fat=`sLXApP|3P*<@0r)a&ZGXS=%vv*Ujy%)KP^^Tv+LwKe-bT%5vl
z_}8lo)93J-JFMe>Fa68iy5{yjv*Xj2hA{fJ+SU6i{ylr^ez9!EljX~&xhkJY$YYjn
z&R=%8{!P-loR3R0_|GqVxx`jjXe;9uo-J*z;e2_ecce4F%U%0d{l4S!Mn_GRs5^Tr
zKkp9WII#5BQ{E%*TzY?dG-#b$oo@4vf5+C=z8l}`CT;lsNoLpcJ(58|lCMAgn<afM
z#OvLuX+?*xEIZQ2cK`l1-GZeG44VV;a<@kvUUJgo;;aid(}g&>Pbw?%d+)rv&iZzq
z4&R;RL#{ziwc6=RzgpF@2o`mf>@nL}!0>6&jk&!?|FK*Do`0O3Lr!7RSGn}_X`TGB
zzh1xPn7eq>+`@fZ%@t}7R_EoO6qEh=r)_G}v-{pvd*im1ZK(=N%zuB0>5|9Msqb7H
z)dMzel;J%ofA2=uPR&!E&p-eEYkBz00`9*3Gg1}YFRj<oVsKzBQ<(eq<I(qjJ}j!_
zIx`{Wx-YBc-|d}2p7I=PoX$y}KGxE5>s!*X+u!6HzaLtu#lfZ8cR%Vg(`}mw?-_?S
z99yu{Xmw`@zc;JWtK6eM%)i!GCvdasZcGi&4$h6weJgBmMxx*S^_z?GVF#PDgN2Ox
z!)GkIU41{#Tz%)^368<DChpD2cCMbr*1{O86ejJq<8#-WWh*0eO$t0E7_#<m%{_nq
zI#0+NizSaAUAw>iYp&8oE$@nh6}Kn3UOK@L(fI86<-KpN-H<Q7|M}>rrPK7j{kqc>
z_k{cAI^|jAq3rSg`zBP}T$I4JF}a}Z!S9LZ)xRFu##QLG|MOc9uA81(%PxE}D}1_%
z;mCHaC6muTz0c(2s8z*VvOM0{*Tpr0|B>C7e~+Iggr>OpD%XbZaL_CNX7B&FMoD1G
zgjOb3*HZOYmD#66tD5WYzvg>r7s;gYb=lfO6U^Cr*MHo;Phi)iVr}0)tC!l$zs0iY
z;gu+{O=|mfw_0ewV(H5(|JB&GUP{POvGIDWZS-4?nJHHdp3SZK(kK2|$*Sqjn<byC
zzdfIK?}pVhr_Brcowxf7Zsa~y%yrx}-a%)f`A+tUTRXo0_*3zmal)3RR~VJ<-9PtD
z%0$$4-qcrfvpLHHOa(&io_;OdfBkX&<8wTnie?O@wSQK>l=BGKqtnXBw4!9c${XK*
z*3}6!561fX$L;q>zH{UF(Z-2aG_EX|{Ay?Lk8ghsm>d*6O}?zYch|B-e~R744;<#c
zJlF2XnQyyh{XR!PI5^%R=GX0evpYUsSiOFWrs=5@$`9V<Zrr+l>-TpBMtioH{LcKA
z@#zwSYQV#&Psf&f_H5i?x1e0Qe_r|Fkgy<+g|hjd?k!WEExd+jgOH)h3-isgeG|9t
z7i>RuZd&%`*<$zKFI&gZs=MIz4}-6=k2fu!vL|9T)1DhOpMUh-5V`bdy4{^Wb8ayN
zaTfh8YIRjmbC|zLN6&1><u7k6w%14>5fHH}|5>_6=|be$`U;b*HPZwQ!ps!?+5R&u
z-FYC?aEiy`BHnFZS2O-T?5P~>D0pM*e})g!9%$@8f9ZFfgqCWWbCc)amv5gl2Z-&F
zOzsur(%u|npu2<V&!qbO^L~W*eoW2kUw?k}q4YqN`5He3WOEkIkiMTiUHsqnpMO4R
zo=y{!_bUCe$KXto6~mzm$Ch6Zy>);!ur_|_zWImw^gl+}pXxs+n5p(R`|`n4`LahA
z9mv~u|M&NQGh7s>L|mVLdi8;j^_*L49!wTGH?d>>8QmX+$6wZ8=ke4m)xTZuc5iCG
zcgmGaPGu+dYo|o=css?d<8S}|eo$3Y;r8og7ats%F0eB`;P0Xd-@i^_kh}JCzjZ>9
zF~iY28NQ}%-VSUN)SOtRg|G<kU@yEg`TW;yk~4G+w%ae)Uq44GuK)TJmd%z0XKJGv
z{FOia*mifLQsTKq(y{V>>%X<D@-Q{LUYk%A(x<##`b|XP*1zZTAGYa;2nA)8zW8hN
zVCR>%Pm?z6J$T9W#;5$c$M^HxZ09`lR6YMONa5wmEo><XYfDrenCtG{{{5f9M?+I}
zvfFy+r|T}h%$D0AsmCSVy!ZR92lHzGGw@VuGy1Il`u;VGSF=0Em1|*U_NAWBpMO1)
zB5-blMCISg4fm&1E!yB}lu@9X=3pwFU;X>v?t1BmM;W~mWM);C?z<PzrPL?!@50^G
zOU^8uic<XtnKwRpe5zh9=~qB)`Q^(Vl@4~&y$_DP=rGe*Wi9yQevM_S2CLOP$M$(|
zU!RhmA<V2-U+Tboh40AkKi>|mxqc+e)&0`_{A}j~OB3^z1(n%D19CTY7vH<{dH;3Z
zNz(#7_%hdDpQ^dpa*vW=bdc4isZR5jB<bY-z9YE(3jcYoBiaVfx4$j^(sJT?{TALS
z3n!g_`aZ#P+w1~{w|74N$Zv6Z&bMUW>|f#+F8^9%ZM;EXM}$Gdq0JG6)w3H9{<#0+
zZO3O--GgQ8Kc9bI5h|d)fBTZ!T-8vf)go74+|c|}yUyTe(W$oD>~ph<z6e~(`xkgV
zS~PIYjtxPl&wt<kym6_5V7Nkv{q3I%pYp{8JofOsCfjUNxPj?=?7<)6_Y+%!C!Lf(
z(9V`{w<ICwcGHsMLXCw5u1ELls65#evuxS@%eUw6O*~+kT+DlS$;6p%EjHSgH~4?<
z-=E_ab1Cxm?wXgM8GaqS#2~-zKSTYrjbHTMd3$P1wA{MJ{lxnpk59Y*XPAEC&wqyX
zbIh`sa<s4evV`-CBu+7%k(n*F@!N*&`Pm<v7Ebc6wVAc#@>`Ce1&euYu7>3X{?tv{
z{y3@f*?)%nkDJT0f@0+3_E*VD@m%5B|DO4}%q)|gD)$cdHj3Xa{QLIDLk?BFFPEok
zS?+t(&#$R?P;L>|<n=R@ON$&8a*K~Mzy7mPuBMe`$IjZi-&`DyI!{%kzkX`lDjM=>
zqW)5+rikXGYw|CD#l&yj`$5G!k}2ruo7=a3HJ^y%ZJ)5^ht#*Ww1B*gS)p7@w*P8)
zTzqir`%jfJ55uPAT(PQ(F#Yr8YbTc@i)B&#a~_qcUJd)E_C4&BpMUAVe+IuBMpxqX
zKOOe3iwfA{;lvR<x&HRFk7tcCRU`spn7Ep9{9nJ#D?idSLFA!&_Qhkr4mW2#`Fia_
z$<p45>@&{rUE90P?AHCb9}IJ+Jd3Em6MgwDld*@{_Vw%IBhp?fEjrb>grnrm?mYjx
z<N4jUpYtiI2(-pu5AfBGzu_`7MdR%nbLQaOmj%0D-OfE}Blk`EKZ8xR0q3c`Uw7X8
zmi7B}gj#^j!YP+_>t0kzI+C~Sy1n?lKXW%6ZI~te=vRIH9<d<it9$F-`d9SkGVGj@
zm9XeQPJocR`2FO6-z0dR3cc{{+&2G!V3KTz{CzixD<>u%ei3nKRTe`;MAV}{S<H#`
z{XLGyAEhnwn)Xk5_3O{qUke+o(2Dhs&sZLGHRQ_v={DTCVY)k1-qnUZ$>F&F<1y#W
zdG?)0eCO&5Cq%wH`Es>zz=MltqfCDMVN$8T9sG5<?Zd*?C45;*P2bIea?1Z5f7sWd
zy6(WOY3k2Dik6rNZd%ORJ2&KKmznwgONU=rSAQ@pRtqd?nO0w4`*+UeDaD?h#XDZF
z*L`?EQ+~?fkgS{>zsUvp+p`m1sov_354h9n_wMQvj!BE+mek$*5FpZYW@+@?D&6Gj
z8~k(EpA+y6(9x>Dl4Vk_^(iAM_C~e+Ihhr`I?j<>Ufp40uie}E-CicQMMIlU$JT$^
zKi|)Sk|#w!HNWnD^RZRQ`u^mkYcdbdE!8*pzI^-juxk?cl)k=x%i5~<;I!NnqX-87
zy4-y~VnUv9U+LG+b!FeTp*wPe>GieBPYRYcOufJD`MXKiZtxfAb~2<T1f9~&-Ff-;
z(+pRYSM1J<Sv@@&%Zehz9_N4G`76olsk%YZ{qoCs(X*Y?yw8TWz6$4hAz$6^%@B6|
zcJe-+NQdr;r`m4)^|-~VlWO|+S4Pn0HPI*2cJ8#Fd;WDp61Vp1Pw7v&B32$)Qp3{r
zkf-nVqS@W=Z~bRzf7!k&NodWAM0dSkQB$w&l3&8JJhb%sJ;5c$%xig1UQYbJyS~0p
zUgxrwyS{$>77-2R2y500Uh^HlMs}G0y1(y-$jXOal?^*CUAx*FeRhF!a^1G3lV!VR
zax><>t-F5v{`0>Z&Yxjr^SHL?-Y%{`e-~ulT;Q?Fe?j||JB$Za%KmISaAj)G3dPyC
zr+>?2(^@Q1ab!V|{f9Od20iKO_dn+ubk3S0bADQTRptwu*DnR{+Z*#mEL|fpHCsbf
zkL~XJvz_1nS=elO#&+L+`ZeX2>7V{RxX3ZrRQ0bz_aFXK{~7K-&^g7xni=@)jok0w
z*H}#^)a={vk#EPuPL2IFLU;E$KB@W7ka+oukY8Zju55>C(?WNBZd$dZ<CjWW;wy!Q
z{I`E=cN|}OPNC=J^0WH_Ocj=F6qwFx>Y=o0a~0e1k9TSfPPSSG>s+q>bNgG9H$%_*
z>_c2t5tWTgcH}YcROpue&%j?J!5}-?o6q-5RrNjLJ*y}3*(~wik)d2Gw&U;pY`N#J
zTb!<CJiqnj`dg;=oOiCT_juVBaD<I3_x;D6@|SDOq8lDu+ADugc%SD={;T_Eyqn3q
zY6qXk4aV;k?|;lODw;gi-89en-Yni-6^_%jyPTMf1<t6jMi+GNjDJ#{+pc4gQ2TSM
zS^m?9@1!+v39jU3<$36MBStnlhwuKvpGPVehAJJ&zSzX*v~2(K_$|7;OA?iiESS5F
z@A*yteTRP}bVQ11{R)2lbdRZnLHg^$Z*4*UE@Y)%RY)w7x^Mj_H~+Yh>XN1Ej$bN_
zn8iGoaGcnrk(m_4ktBA!zxe&%0#nrvA#ZEDt-t>Kd9cE{!m;fepR7lchgANKy>0&)
zqRY1l?=$*0eckqd4`z0}T*xQ+E_7)S>w%5G;=b>E{^76;)2$1${yn(;oNJQM`3#=I
zxsGdevo&lc=I-17vEOIj$<@85OTV^HI?$UaJn2a6DbqFXsek^>zi&~pRO>tYozL|e
z8Ud_5JXLdM`ksyuI=nym_>MmxK7~whD*1Ign)!T2!GDGTg*VrFuWV-6w^VFbbA5mE
z-)~2PEjt)OKJWj}FkSik7LK?HrW4}QuDp^ub}o<eet+(}LWNC}y(W5@hkf~Vkjqt(
z^@Q55<i0=Ce&2U~|F^Edpou~0YuR^Y|93VFpA9@!tb~HKnvUe;Z#};Ar-jqh6s5FT
z_vfF_aGAB)+Q~j6&t&fOJpUDfCx85#e!RnR6>n*6|0OPmdm92(<|Va<B<ATS)Z5*^
z@^A&KhW)+1gGV1(Ep1!a+L`yjcIl=3BFFs?`xR#&%r$@gHX^9Y;+nvj7>=tmV}5Pl
zuHI+P{`h&1wgSK80;`fY{_A!*PioLg<k1s5a(il?-gR^Fx5s5}GsPTBs4f4>&+oaK
zajycOp0m&?){Y%-cz<k{pKotbbUEitOz@4sbHAFdEfLg~ij$rhrgZzX(&xSJ9Dd$^
z%n`JdA>?M?z4{0ZzrQv{zg=&s>NzqPez`GwcAlk#)Mgct*UxJ;lA=;CJ~Z_D&M&ul
zqq1*fws19%)=7m)Rm(p=eaf*i^M{XCRs-Mi-YZR<8+JbWzWw;?UkRS)I$N06J$}9T
z^S%i$gJ#X%V=nwcM>>3(Ez_fe4(}Mu_kG*`C!zZ=)6<ptZ*EQbx^Rtt*~8YyE0*2g
zGDV!FvY~-PUS7HOZNqA@r;`J-ci-#Nj9*ffwc2s1)$Zy8%ddQz@;T6Lf@|ON+;7$m
z_WgD@T+|PER^99Wlx4drLw9A<X$6L;dlxQMGDisA;$WOIG56@n4fpG7-+w(aWre-H
zR`Yo`ma2oP3!7^m@LTrmxVCgg!>aIr=hb|N6YOOa<hZN)l5>BCPi$TkseAg8fEq(%
zR_eW{5#1}^<>xQh|Gd$|QE}e>*ljD?Iem=7?q8Af>J%`Yv+L#E-0F!t73$P~?~@3U
zYCc-N?fX|g#T5>^ZU*<yP0-X8P~rZq`S#BH+xzQ#Dj78cI7FWNm3^*IbSOIh{h#7w
z&!@S<Z9SH|Rw##`zjFW9oy1R5=Y|$#o?mA6+i}wLAM1IZ7TuC)+byIM9`^6r``f=Q
zq+*1wE!%H<?mvTjr-x&LeMg#Si~UTQV-Kc(X|5~#DZ9^g!>VcAs=CjfelfWn{cDQ{
zi$)V~)z2fjF0C(*)aBmwo$vS1JzMV1e+Ju+;(~2HM<#69<+=FQo>gCey%qlOT;)`k
z#5yMCQ!Dxwuqgdrx$yzxLHXW|+>D2&s{EMgrN_UUecAqii2P#15H9sPH{pPk-XmGJ
z8pQT3e&0FwRA`X%Yv=m?a}2pWMSr{PW$5(Kc(9o_+&upT-|_sX$4oq;j+%%imgT=V
zZZ%6}m7t?X-B|_a6^l+VsP=x@_Mc(dk$)9pCmogEeqTR_@nHIjUye}*jZ<{GU$IPQ
zPW}1gKSPdG%u&{<m!>S+cX0m8eH%Co!d-KxdTnRFyrk@L(tWeHEnZok3%;(rK7*w?
zvm!xLb^F&B`4L5%I1YPA9x%VU?MZ+0{rc-HAx%$pSO0a5I<S3-d(QJyc6SU@Q>K}k
zbMM%H;@SP^8W}BBgM?|OH~yCVx?ji_>MU^N>xEVw->0uXT|cd!Juy4_^S`4XE_WrC
z>(t)+=eEe!F!TKMxV?(L>ns^}a@A^a?A<Oszt5nuSYX3T-KPoWGTWasC%Be0xg4(+
z5xQNLxbe@A$BcRl7&i<0e!KDS&g0V_XIHzIvR~S+bFlo@lm5ev$ImfMa5$Hn7uP=R
z!_CAFUXO^WtS20#9$z|B>wN3m`yU@{H=OJ+X7b!<p1*DFg1xVKV>aj7&3UMJX2k=&
z83%7nu-Lyr{B6@p1r4FNeKGy<(T64oWf%VrzGSB-;;}uA?bx-gt-Jrl?)`b6B}gcE
zhT!>AUp`&=o5;%%zTIwrhsipxV=f)hNsB$L+<wQ;F8))<=d!d$x9ZA$|K$tz7)2C4
zo8ZwM_27it<l@}7d3B7;%}u`EOFKWUkJsXI7T>n`HDAE0DWAfeuJ1kj_WsV>k9iU&
zJX746dG+vr1_6VoCv|0oWPO+wq9=)eH<zBg{o}!lmM4|JTW|REwQ&kt{rf4Nu6i9U
z_32?e3x2hIlea(dJN`cNtgRsnT}<3!CcXRC?5+^-VA3Vtw~0G$d_MP|;oc1guT{G5
z|2Uj-W)u1P`jpJ{P6ySNph>j~j4yxe++Kg3tBJpn`Qh$&R=<xFwF;kbS|V`6Rc3+6
zjT>*OZ}gvj&hAlK(Y*WD%WKc4REV|*USru5)A+K5UCXxdhx|O{nGF0Km*$^;%lGc(
zUWFAamhL&4C}92K&aLm;Wn}m7{qXg1kVIBpETeCtq}PI#&oc_Y%@7Mq2r;ytRQBU`
zlmF)<A>SKa-5)Z|{+)Tg_JP-wHES6^Cuim@jqH41!2Z4dap&Y^8f<3yv2UOAzq@j#
z<mZ8Ie)YicfK#H3){Xh~@8=oHB^7B*jJo!p;e^%S2bT^<-8$?Uvu+B@iIR7g+u48B
z&9D+{?RgUOCHgIkli!r};eFy<HrIAI<)`h~c|UjWx$B1mVy67g-aeIKwM0nH?8C)6
zeq6mw-|Q>y%>HKI{<77Jd*|!i*<pLcHAVa9MY}JvWwX9ryY0>H;(bc%^74ge{d`><
zzIX4=L(5`gx(-ad*pVqBA{KD{4)^c+`aZd-78<T!`Ho!rSF_W^W=VI<&ZitYOF5W#
zn0=GJTYmdz?&nsoFp+=X#P06eW2v~R%D_}`V?oD5JyEx{*3tbx4o!Y|?(6;c?)$9|
zS|4R)f4#2ozTSt|tbetbBi{L4+rR(wVPlasY596l8@_$Lc}A^jqrgoD#<{gxp5ixN
ztH0j+pJ5%Jv@x&AtyypWGh}F8`Tq2!EPutc6QQ{$_cr{HUw0!Zb-Kd)uiHPKvT==K
zm@KBqn3#9yKSL0Ibzy6O?+x4X@{9BOu76r$Su{iVy56={H$A~~&6{p|?mhc$$G^HC
zRmTE?_`AP7|7Kcm5!%FkJVirr>h<9M-CzF2?wg&!S^jD9#P__8DtlW>C1YmXOj5|<
zYT)17Cy{2LB`$q??YD0&Mz<DC{63>Jf}6=x`)%$U`@P%0A76ff>Fb5Ludhvbb?12G
z+=vY-dIy?xr2p<XKB?#QDd%F*Pq+4;sZi@@6|(kJ>^~FlxpQknfa4#w$8vt5>JPW>
zIq~y?vq+cW<@dZNw3Al|`LDkB<ac%1_8NQRm5s^EPyO1<uA$u|(YI&uWWS!kS#A@1
z7hA2n&QO2+S%Hek#A)BH<$bNbw4ZX6D&F@|F8G3qN@dP1e}+%yd%xY_{dQk2KvJ@d
zH~93_?5IC4g#>D1e*Jy8>b#aV@2Qv@3_-Wwe!e}Ys!%KV#@BbHJC!{QJPIRnm5n8J
zQhLjO-`~CA*1g}kZ(GhkRJk{&|BLB86-9;O){|?m8RP`62@N!J{CDG9!E6h=r2h=g
z7h0U71NZPAnHK8c5IsS6b@lJ!`UUUo3T4Em-KwrN5tn%w5R_Hgcv3}3yvuZBS<r%w
zU*4Ym{ZHQR#zY;aQ`y!38N%|n2wGk;mfPv9J~5QREb+TX0&DyA`qTGc=9qd0|5&^5
z&UX8G*SE+jO!jzWWf?XhbKAGXz3Xy(_nx%Kx#OU<SLb~GqX}M14`p*Er%EY@6<259
ziRkXjt*&+de0U0DSkl`vx$EUS^f$DzTCRNk^J~MZyARVnmUMj+GvBFpfWP|BpFa;%
zJyf+W+`F1(AKtvukAt;GacSAcph(}X+6LB*iS^IiI8v6LVvyPM_x{P3TrIsB7ccow
zGcj~Z-pImKX8ZQ!y?@6U&P)p7T~THJ)agKRi&u7F@}I-y6P7Jfjw*X6F#Daq-QD-C
zr)TP%n>WYc_rXaGrS*^Vw-`-r)LY}Tafh^!yW0ID^*l;O0v=L3Z_PJ4aJgd2gk?)P
z+TI;qvWtaba@y66H@cZ`<bK_{;b=v`=ll~jZ=?R+<7wHk#`WC0cV2FxOgGn=Gw|+b
zV!Yjdp3yJxQDD-SwfDb0Ig+6-tg5K|$S$_zqO?w?^!8Z`Hn4BqbGBjU_WJ(DkNuXR
zv6t4`R$iHF`s|nQl&u0M{%w8zRPd_tiPSBZ&hj)cgjq5$zus>l^FU-#z*LRAV|(6S
zpW`uAMPwJNZE}1_@Pao7Eq48Vn0xlP^!&%i^a@r*vi9=w*7?*P*H1h@)xEYZ&i~qn
zZ`vmn1KPd0rz#!(`@Y$8np1IevD>m;k*l5G3tp4py5hXO^P0B>->Kcg1-j?%-_K5*
zoSs_7<NLOJ`L%x+d9^=mxW>_>v@0bn`*)u19p3f1?cZ|*jYYbe(=TmhPX6Ph>7soo
zdiBqPjw<bZBDceyJuNredbE#0<V);|U(C<9>waBh(x9iw>L!&OFrDQk`{o<BuKn2l
zq@Z+~Wo!o@``!29KM!;0*UE7%F)&luV6`<qsoehh@B2LOgT&I<tt($&wk~R1Y4Wjj
z3CrFqO{b&|ykfh<$N7ijw$M^<h8?@3&)h!~e|nCj2J`8ByZsq)`<=C!740T)^;K6F
z7ui2%J{O?9G`cqXe7Mvvhbxo5ToT%((m3l@(C<n41#G9vZ_l%S_~|p#xi8D+ZL5yR
zuU!~2X;${1d!h=Tw)T3gaP7>Vt8{x}{-pZbf&ueB{0v=LmGL)U&TDh@?<VC5US8)?
zZuYhw{kPv*evZYYDON%UUzjdf_x0A6{Tf^oe1kMIepxPda=oFr_2dTa;_O<p<C}H;
z;*W-1-Ma2bkgX|8cj<fCe1mC0Yqr@os=U85zq_8tEXYQo`}&Qd=j$f4ES=-LW6L5>
zX|_*?Cxv=$u9mLl|GrO#;q(kPUB8`|SdEV?Shw?d;DNoTSLp6wn9ZHzzjd4V&*Mv%
zc7>e(lj-~FJXh>X)@Y_}>0M^l6O;Gv{8Rny&&S6zHl&9g`O`oB`V^KG8?T9}a2(K3
zeE(9bQsM7w-u3yt{~2Pe!W%4unf>SPeJFNJ_OM3MEYDAGN*cFY-+9=esh(GUb>_QY
z7q@kM4y^yy`gf^ocjVdx&Rf5$<t)wqy{l|gxN_<HtN9U`Qy%VmU13u!%4d4xj#zY_
z$e)i#w;gc}^~<qM+#J9uv+Knny&m6YQ60vha<*?b_pUMfHAgNi@l*0%Q>I;}i8Vfl
zL{<7$b?7cTdeC~ix$ODv9}bIb2u`<GK7T$UDTsT;J(jJRwMDrrCgz@hp!@Om%XWXS
zO)aNa{49QTA=D#_Z`ZF(Hld?S8<~#Rifu@|A^(03hwAOv?a!|}csc&7db#-ND!!Tp
z2UhQ0yZzq39g5CYef_8AX2$Bv2}ML5lHRFys#GWZirDR`wmS}c*Ceg_{o4K2M^!zW
zhdXsj%{oQi@7-ax?LR}~zqjoT%))^$>h8@Be9)S7@#7@jG<Tz{hc`Bu*+{>AQ^0f7
zQ|Kh)+V@`E+kHN?XwJGIsoJthVei4aYb+V!yXAgJc&@s8r?%wwul5!VrqaGnVUN=E
zu*AKyi}<eJpLaj{VK=Mm*@OS=<}DKK)N$SEcrp1?iq1vh<q3{F+jeX{cKzp{Lh=2Y
z42<i)L|nhe=GE2wpMi~0XX(C&DOMh0-U8Pj*S^iWANOO{$}^rfe)s<R+P;t>;BSJd
zP^MJhOabnuHRdv(?B!20dv$(jj7yGRV&~<kyQV1C&s&>K?Qvb@uRnhcGF>%)G<UuB
z*LG^U-ge1$a++X+>)-XY@BjA5W-GNn+-mkLVbi5AR>#j=3sulsE#E!2_xJ663~8&4
zwN{DWnp<^f$z*2^qhAd?M;pH8*?Ql&fB)}?b8mDu-#eb~%)-I*tM)%bhDzV{uQwMn
z9VlY3w@wsUrkHtozW#>iOZcLkWj21vkuu$Hw(rodozitN4_U4XT(2wpY5q3j%aw}G
zmg%Zpoq7vh*Hnl<uV4T9K1b{2qx*KvUHRL&DQYdZk=+87f?aJ}wGK=ZzjJ%v-!?(x
zprbFCxqiLfJOAe5n~a_c(eGs=k1fxBB~|MnXYubp19O{((Dby|@yq8ZPC7JYjp;0x
zbfFjNst%Wb@;~0&e*Q9x-WiUiHxFmN-Qzh`<LT*){Iw@Sv%PNlo_&{B-C#F=orva?
z1(UbdO*PBiv+#wN^R^GS&w2cH_9!yDQg$OZ`OogRjc4VQu4d2P?l$MavPcuNr#x5A
zu99VKYE8JlZ(&i{{_pRbL>5?iH(y-i5f`<iez}vtkr;*)wg;?CZ>nSN)O<VKb3k;4
zxRr2r>FbD~NzU?*1a&UGy87%@ujAAF<l_Aw|7?^Cx-?yj@0a0i|EQ?lhc8WVH|CXF
z<Ph3j>6=u_J-Pb#`;UJLLYX!>Hzz*(b))*=g_g~+3)l+Qn3~kZuXHfus@IlL^vvSw
znwxoMhJKM`>%+}EjW+H4bU(rEd4cWDC-Z7I9A(v<VI20hE?&>+qf!8a#hO}`r8hR)
ztW`+5Rr2@kc569*`{PeH32=ETFz)>;^UAKY&D+DH@aea{B}aq=_HJ4G`+jm=8GD_z
z+zkz9nVT8f;TJB4-<sgl_J;S@zl4puMU9RJmOtSuE#F}#_UzAph7E_bCwA_ieC0;y
z%iQg4dJ!ANzWn{Oa7QXbqwLj_T%Pwo*nGFHxy|BrZ(+Cb8>#oZO&@G#*l6~8F-LyL
z(h0lv-@CnIgZfwHmyJt3<z%JiTfP0`9Mm9W9Q~5##?Qu!L9vc|efX8{@K65R!<jfm
z<PFEyPfzaaE{gCA)B7+<CA8E?Z*zfH_iN_I#kEKG%N0zTHs_!Mo7u`MZ0jDav%JL-
z!ddqECDY7{Vd-0TMH%TY_v_PPe0KNmKi59FmS+*C?#Gwy{>wchGB0wA-QVi!BPAWb
zH%}1wdqIb_?a#-p-^BNu9cDN)Nzm=e-f!P_UQ<)`WSai^_1it}r)3)BFAHm`2pqX}
z?Dw77vL}B$zbzN?bgIgNiPv}J&Rg_RE09n2<<*6q(<1ECO+wDA-%|_9%isI2?fajC
z<$SFTYi{nlx^>IHgi{RDJ(o|(4t#Sj!IO0kt3ut2`$_ETb=CL9@)$Xm_FoSd$@;Uc
zN9D?{$ooG#Eqbm6L`>3SjuAc}`>#EJ8Q03pRUY~8e_nnokfs&=EaL~8(P{Nr&#q})
zXIXo+_x+Cl40Q!_ZwH5~-MVn$PyeY4#?v)CWi#1yUYR)x?2zib!#}(1pM}$^RHb<(
zE6Q3txK`TDi?54vo?39|V1r`$rrE#0KmNVlBInASWk*+72JY4Oy7Y`8XhEFF*;S`j
zdA)p>o8SDm&fsa9hrY|-y}3KvR24$QUoUGswaJ5_HZE~ftCk1b?T5vwdC&fIeA>Qz
zw)EjAElclRUH#ipP&W3G%iPB@H}^f*Zu2%j|8<KcV@be&2K94KkH$Zh3%bKvQolpv
zvy8Im^HsAMEWh0N&rp;5Be5&Mi!W-f`RkdDOggjwK3;UKgVndHJfQN<U-R4C2GbQL
zN$s1n{dc8vNPbl5g+3vj)8cPMZU`<r67l}W`=7rbOlQ?LHT@}I`#xSHQ|D;Nzr@1T
z9&cV8dgJL7KJm-X_XqAX$9&Xb=S~0m>+<zWBD2iyJlGedIbnmtk`Fohxvl%^7w+FD
zd2g!BLCcP(U%%~GV(N0C^x6mIWxjf~h7sFW?p6Qs`;n)vlk6PZ{&kCPhY5)3uCaDl
z)4k$Xhr)#G>T`Q{etW$0h-P=xLEq28S=<v!tQ=0Ha{MYw+BMZNNO$?BT6;Y{6@@N_
z+~D(Tql7|VSNx8f>*~#`>e{*QKKsw;9!u3#8;se1*8EKjlWUGpNGw{oX4}@8<-a28
zt8eprsG9PX%SOLt&gT(QQx0m<V`NVdmy1l&^xgUT>mHR0&-R}_I4z-Dsv$S`WSrQ)
ze>Y?fOw|xyHg&CQ%^5>Smr03o1w0~6A&keBU$9^Km(Cp=%6;j%|C5^45k)}{W&}>K
z(ET1#wPWjc`S~($+c;abRpnw{$=`o-NP6l@uBvH_xl0>WA_SMrE@R&M<5%L#Nz;G*
zd(Z2$Zl?pwG9>|V-X$xQZvU>gKVDOru=aG=p5y!eELh4PEH8P<XkVn>ZNr`W78fVk
z&A-2mBgFGmL~YyRibPMn4{j^u3dJ1-7|Y}L<X7Lm-Mha{>L1Up$?2EZDh6vG`2KQ1
zzJ|)C1uAcJ82++<&*9aszEFJonMeNo39T+$ohwsSGWF6F*mm6By1rx8BilQd8YF7>
z&rR}D5j&glVydaZRfQzeoQgNOzY70+2tBR9|MmWQ4M~<|6UtVG1VqM&swnl{Z~M=1
zy@oA%n&Pfs^13w-7e%Vh``%{k*LpQC{)X_5Ux}Bx7!_T&KdW23>PTmhr&!;saI>A+
z{dc!7-~Qp>-)WHnD*Kma?_BTCbLtS^q;7^clAZgW9juPe&;EWmRAtWb3zuKMJiBn!
zwAfa=5Bz2?lioybmE%32Qa$nf%YGeAt+Fe1*IwAZ<o6QcRphxP8tOUWlkVoi`_CWz
zXRwjfE)`d3d|TcAz5S2t`S%Q&QVwTM9X()n<MZbq4<aL0s9bz|UF)DT7su~_q!p)p
z|7zuxS<4;X_vnYy;*?ojFPFy^zU37BsB_D6-)-6Lzv}DmJQOxmFur#6zVqe7P38Sl
zC!giGZ0N-Ht8>Gjdj2_;nZd!nfA`gE<XrHIU1jy!LwV;0tBd;E*^`sZZr#q^C&9_7
z7^)nVpS*wW!yx7xU-!>3FP|3c<EUj3`-Y$8_UFC(r+yIZ;7sSK`Lg%lEk~AZhDUbo
zm?(L8b&PO!-~INzADlH(OU3H9=6pANa>a1tw|~hmQy8P1N*dpo`~G0w|Mf_VYKY^0
zt<q<QWQ1fJUaz0RcxU#KSusqKt!vXZNZ)>ad>^Z&--0C-Zc|_VXSmR$%I+bU!ntH(
zc5mC$_Uf3weD{9j91)fKIA>N!@Pwri8Ho=jEm)DMJb7)mfTeVUgLKrr{|vt0549}y
zIwk$>``#XpDGeMI44+zjf4gxfY-c+1dE4)@?fYcs>?=55_4!q^M+JA8@+DD?U3)J(
zrYeR+y|K$b_-{vF=V1%kXuo><3I`qg_0J@|b#ES@A;}e4{c8Jh`TOUdO<S|xz+&_D
z>25p?3<BqZy46lU&9A<lwC&b^hPnF+etMV8xtbOBXscYw_cfl=XLLoc+v!{#`*i}#
zf*X@JuzfSX{W`~BW`?B0!YbYs_o{+ab}+WE8F~NAS^2H))^!HKy7K4q3y<g66|dFf
z=B=vFP<a$JO}47qxy|g-(Pnj(Y1ZGEd;7o0TclSkU9r-1^V@TC{&s$EXv&p|G?~38
zWooWD?`G*wIp(*2<kzTNQCkzA|7c&qnn!JaYaT9rGWFiAhU5i8$FFX;i7EeiXodKx
zMc>q#i_R~1;`0iY6*6F1H_0^WZQ_!D*5AY=w5G^ROtAYUzxwAPgK`5-*~jT_E<4_^
zo!x)@Y5wvDmvvoLM6E;K@7rs9FrdMCk7~&J`%Gsz-XE{4U4Q(<VZNl8s;FJ;R%VYc
zduWC&_Rv1j5n#P(cEqdq`wleTI^5&V=jrA8N;-<)FK#WfSH>f0uQh8tXW8<tIUxIe
zhjne`-Zi?d!d?f@X#K01JzGrr_BpPmmkYMXYrE8KI;(oB<G`g5;eB`h^wskp=F~TH
z(r~@kbL(eD=BsBasSIqQZ9>K-s~S6amfUIlxP9xt9Ys?l&K7Yz&D;B0=n`|-<8`x?
z)jH+e$_!onez{mR{F46Ik#&3ngVoB@H!l5WxDuf%vj3bJ#|0MK%U2eEOMAsA`*(-f
zjhb&q56qh6EHA2e=hdyf?z%n2GrQ~Gzr8-i&a?3o#|4RKZy_@kz619^e%o+xnNK>;
zb-69q=gzpdeu;~w;<*p0DOXgsH5aZhDq_$7{!Mnn%^UN%SDx6t<Mrj*g{@zI9h&6r
zCF!lQ;BJ{%LZ$L&W$qb<0cE=Bbt{wIpUP=kO8xsc!MWmErl5<Tbl3szx!bPZ`uCsV
zW0P7&AfrpeedX&n4lMoAAugNwtyMC>St69x!OCxKd)|MB_s3+KLL%;Pn_azrZ%y_Z
zUd_NrmCakE-+C=w6x5~1t(m7|-KT%<Kf_@Lmjyuwgv;!{nO69{m@l+%ZgnQJtAnUS
zv*>mmo!sR6`!DzY({DPV&10(>|2lp}>x|`=t(K+V+t&xRPBB|L<x|<tU)BHipBCf_
z`xzYm-0k-Kg`)SCDK865y4J|)l%*xua8vidxBaqjYh)GETJMCv_ujtwQ$@|m{EzVw
zPc9o*Nkubm`sMJe{+#mDGn2M0HC=DJa^L;1Qx2lc&-s+6PP#aiA(Mq&dSdcP+3)|}
zwwz-04v@;vwV&^iHF4G=nV%B{H>9vu#-7-oWIz8L%gjcFS*xe6{r#3v)7;=@ljIFH
zqsawgPvURyyAv<;)IUn-!PZ^R`B_?an^;V|IAw~$%AYs-^tTtbDKhQ}P5Sn##o^Mo
z5HlSWSFTyH&Z61>s%s|yO!8*u)GD{Do0s7Dxn<@3yuFGJlh*uVjGoOae#wD_|Jr|s
zyBi|2PA~`DzVM$x{)j>2^*ORCy$)&GxzW7Jm-lX95Ghs+HT!boYlO@Nuh|(73%2vL
zXncA5^8M`>2A#8vEWhuccYVFCB3Jy{e;+0)@dwx?zZ7S+SNog$vpV5bber_Oxh2mg
zIL$I#@M_}qsohnz`qlCi>L>jA7m=WNV#oC}Rqs#VvzhWLTb56PS7-Gb-R~X@AMEP)
ze)y`vb3*AqLtSm9Ggrhl2AQ9;&P`*x{rk84WsW0FvrTVT7uzkKV}IIjZ$jf8WwxFM
zzqKk265qBTw=Ud%j!8{Z`^=;xaV_s(Z_WGGs>srH?QU_4HvgW;YfhDyx2>y}U&d)y
z)D$vh!j;dj=V;#g_<dH$WSONZtPY_vAGh5pKF6f?jNS2N?yof_-dV<7U#~8HXxjBl
z+Wp_z+V%VXGsMUr(R$;4r)K4{MH`-aKYz<H@o{SOW=q|)ozFLHyC47Nk>ufp=@yq`
zkL{Z^LrG+zgTiY~m+NZ<7<4>u{AZ~B&%mD}=%t#!dw)n{d7|6YMw3+`YZ3};56aKo
z{hwi*+`9=Px##v<_s`p~&?K2jl(jjlCzCnNZ2N=S_dk-X(oz(PH++6QM|u}i(=#qL
z`ORMraqQi}{OS9_Ph7p{x6inBPQlAF>2K8`DPQ+XVvpY!94Wrf#O)P4*DNbWu*&=^
zpBLwnn)hoYb<BS+_-6ag`eVPMD;Gn%)Vy8GEkacHs0n>~A@#nt?aANE8}|PGThPU9
z;8`5aZ8rP<<$E$yEc15#Ti_cg(ZO%Yr1{m*?9SZx_p58mjxW>9e<(N4@5}2`8@`;f
zT&2z06e74;STK3-{{H08zY@5VGI?^>&AZU;)X4LeX;$FBiTVE-6tn-<>mUDku;odE
zSByZ}rTbdHvRS5UyvpPdHkzEkvg~O7-i~j!JM4QuK5DtKto@SImNdTHQ@kpVPnTxO
z9qE?NS^V<v=Y}-Cm)`Ne*<U_A`*7u?MGG7RIL;I><Qr|sp1W6}*7nD*_YLcurb+(f
z;x{V)>3G-MH0|f1^clu?Hn}ak&8on_f8a>Vx(GRafy=k=zudR+rDveb&;JYt;wy`j
z1$NKB$3FW%!v_0#9AX<L`N}V|3vlri4{R`Xj?;(^n-_O~?~U9Wi7TUOygWW;#?7C+
zMR;d=2m@%;?N)G6?s@xZf4~1{sIg!bH9AqaY|rcq9hxjL=CMK2l?$c9gm$_rc4uxq
z{ww!ab%LW~#<^wJ>$ck+;+mN@<=*`ITP$i}JWHmq_y2WW$6-@`Fldv|SIwVN^Mm~>
zs+LX+l<6uKHVk8p_R31gSy0`3_U!uY`;@21tO}C3{`zdQ*}KHo(?VEv<IVprFcoEN
z*%vxRu)th>@{av7iK$CO^Wy&2EI8q~_(=7cGhL_pYj1ZqR5KQDFK#~Lw7q`)950t`
zp%3`AHN(pyr9BmxzxVDCf0z5G;K*53o!MPqKS{m!P)c2L>29s^)0GJaUEXmhu(Q0_
z`hCOpm;2>olqNW6dGMHA{rPf>(o(h`^%X9Ca=BrjZp0l85AC>hJAQxhecqrsKX0B6
zuHVe7`03J#jKB#UOL{Mg&TVGfDV@hJ^U3_j&wWyo+UE--_r5=U>i87p5P{qaEeBG1
zJ2nc><GlUCbNkW$r|p`I{l6Md^*NaQ{PT68=#)E`7*!p6Z@&{)udaQa_1*69{n^n%
z3|<Xqyel()ZTqubV8yftMXx=WHU_QGlURFT$K|uVyMNawGaNOxyzgHB<kQb<98t?e
zWm*KD*_#FM*PbujyHi}8{V~6e47Xfd$zJ&hJJ;wO>2JNgZqGua-3k#;Zu&3j6PZxw
zclhJC!$CnYODv@C?prdOO?-=%qvFv7PlK;2U9zrcJ8o<}X`X-h_daGrEsg&S(bu+Z
z{G5D>XN&M7|F{X7oGUs7z5IUf?v8EUJ@MXyzpO_$Y^vc>EZe?U>dLzJ>Fp}1dKo6S
zb`^DOJ|)R$*e!dswO?U={*OcJ@=_Z2pXm#;nYZrv6u*G)KYux=Ykl%KlgGK^c4MB!
z(VVX=8}hSbJ~u5&bDYAr_i=2KrL1x<v(gmR5V`2u-GayEEq44l75r$iWzucieGTu-
z`l@ybm3Epo8HlXxJfQxcA+GFp?%O&$fAPcn4Lm>c-nw?<+LY_}l)ks^lE1zBu&2DY
zr_>_t+kT6Dw@DX%`BVPw;D#e-0~DiX2cM5uJSDFe-L5vbszkB8SZcF{_r6JY-Y0*4
z|NiIq!#7tP-IiR=TfLz!>X52wfywKI+&r(CGQNBed%*UqkN5mGW#6E%She;=@8c&l
zvTWO1yLi_$UDa@}-YvJjJ>UPlQ?gOAod0*}r`A+q?xN%!35&e@zB51gJ%9aWZe34q
zF11-fzs=t-QLx<JJTt#u*K4K1r2A@5{#w|#NzH83N-SG8E1>c}LjgnEss%4qXU<vW
z=KC$SQC{xw{(VvbOim7kmp^~n93`>EZg25(UgzCb9XCswvkjOI{P{ii+pjrZ?=zgX
zZ@V9rxkRF2%B_u8N<KC7d(VzxE?V@w`tx5ucB2eUg|4vO(g)6L6+PTKK}T!V8{dXa
zpZKqTKHhU@iRJ6bl?S?FOXNN*Hf@U8BxW6R@3;K<+ck`is-L)Ociy_be2RsaUfkMi
zSNe3FZB98e+}bL7$wFSbKX-SX#iu8X`ih@^zN=s1dXkgHY{E`eRZ0FCU3{~nQ}_1$
zXSlol@Y2*1LZ-G=fs0SLcbmJfuXwXbHj96$*Zb_^x_dXWt}L1mB(*i^ZuE~vFISm3
zyYh{by??QG?0a<m{TqYStuisW&VO^RU$PP6;8X5;BkGk^xH-~yeRl32gUl>1N&WLr
z-~M)5DO&NvbKSqmA_?~s52$ZHc7th(Q&vp$t!egGzqd=e1U+N9{HrrrLE%RC$K>rF
zw|_jelvCl^^5x;*a#!;Som`N&Kf}v<$Fz_#{nd%t^PWAr`@5F^99QS;t9xy;^CPMz
zFmhU#Ui%lYe98vvjoB~N3yaSk-*J0h*@5XADo=NR+HUsi#hu3|d7oZC$Gc{BY)E|d
zBt746k3alp$l=S{y+w4zFXsGx(>6P+DxT#L%Ul#*z{#_3b`Hnw2J8O!HOxFqb*C?^
zx3~RzOF~71>j106wwAw}1vh?|>Hg19^T_9n0oR$jEyde)wU=Aem#$pU(kXg1IW4wa
zdU7^*-Tv#$vvpQHb8XbU9yj4pn?A>9rY4cf=8e|3KmXbOr{KBb>1$i=*$D6|w*{H5
zeZT!>p6<^)gO~m@-`uLV4cq0&WgnDyYQn2+mE3|!p5otk{QG%eiqYiR7Y|<d3SZ@t
zb}EL8BUr~v^1yB;&&@H6A6Wl0>_5!tbz@Wa{AFkJ^|u(Rx*ukm<XX0oHMpphuk3H_
z{&mtyfnJQ}7nr{O-TA)4yuA4OSB|6wV!Db6rcCU0zpE3ErU*{?&%pL?!h>&%oouct
zPFZzl-E8Tm{p_FYdpDG-G%cOYZhe3KEx7<0@$Ku^6y6NKa4*o6aUJUkx&I97UliCp
zSunA-H&@O<@8PdYY?nmY->t9~@a+)${{3Is<C<y%&iogLFSYOCeY>FT-f_>pZS(Z&
z^KN|92t30f_4%{ZulW&G`LexJl(p}AO-*1>_jvsALEVf5Mh2F4`Lp|~-KXlF?%Vb6
zq0yN}ucnad`nSi9Z#X{Dpy7z6y#F38)ycsXKd$IConT;}E&cubewl~H6-|!oqntWJ
z8IB*kvr;j(!$xqYOn?8k`+QyjQyC_mUw61(f4|1Jusuwk%65r6Cmm?Lydp;S#~+S;
z0-A@p>_0}`o%?UXms1K~uP@PDrBreD)Q&qZ&A0BeoAaQksp+8S{`cR1nO)fQRQHwL
zgwA=cE<!e`w{A^!mwv-{`1XtZwx>?9&c43;!fO>TE#1|fx&J$F%}vg-J&j_gL|xzB
zI(_U<&F6DAYC(?;zsSswpDdv?$xgOvk6C4Nmj6xB=vJ#WjMo1djQ3w>o#s)dGMC+#
z{hdcrX~Sj_huFhx$<=vokNz`=Gt`A07UKFRcqZ)O+N}Q!0xk{SefvHzMVS_?JLmSs
z_2?Q7=I(^KpRO&pPpV2}Wf0Q1&_1zyL#DtahwW+Kq-*Pa%#ws+=JNgf5Te-J_@}-=
zO)Y&Q>r+P0D;&FZcgWq{{<eixQE!#}(x`paPE%Ui*?+(G$ho&x^-9YQ1;f2>824X(
z{_<^;qSq<sD@A|*E_h<PaQO^wCS|+OQ%%07>g-SbD4MijzTo1F?<@CDyBARRrqMZW
zSH;7GkF7~7!b^64`(ychj?x5UgN>KxzC1m**@-jpW$eSm!`iYpBW^ZIIcyaB@%G2V
z$!?+MuRgOo@87B&@Zqsr<DRab0Jh%Q-yJqc_n!aw{WjaIm!hkA|DIs~RZ-yi?RTlK
z!h$=zeDR+59#{7rD7;ZOqxXKQUhghj-UPQ<rO$bvZQ$~j*T}QnaDRgJ2eZ8&9|Wyu
z6s`L5`*r*lS)Dgm+s@S{oanoy$DqL|X0~^K{Q3K==a@RZH<!MS%HJc(@O0sR`-m5u
zatmS#7-gEwx7XR(KWCb?Xc-ryefYonil9Xv4aPaHcU?PG_&jXCzki!wckR%5PsLet
zi)uB#HMRWydVLFTNJGu$or?bNGlK8jcxPt5y*~S4Tg{f$d)Rtvzioc)QRqD_JpPP|
zM+c|g#}iSfLT=3d!1(sueTz`0O#$A0rQhx1(oeYeEM8l+{xh%YicBfv6&n&9vm>H+
zs{DRu&v)mM#fhonVYe>c+-oj<WM+|u<BPA~J0hQNR+y;DqQRg%TjSg9><8V4uKj!b
zbK9Loky9qc)GF8XWc4&e`yG3m`~Lm?+uL)PVy^HDCPbgQmjC{g<))+Ab-VWHK2uIl
zT9v<dyNq=F?RU3@QW%36)vwKY`Rd<=gF2>GUw$1<)0CX#p7g?0es@B-`93vYZk1-o
zXPVxPf88f}%}`?&nZ@;rEwpgs)~)r*cm6X}Ps@-Kd6s0e;?C@+xes5trEHzWuktU5
zb%OQWwW(Y8RsNFQ`>#Mn!6&?h{mQ3(U%u%x`>Z*7)auoTnR7iuwR(=dvNW@~>;C=6
z!%tI|@GOb=&yc!Jm!oCc!tIyW>uW@9Nj<hK!R_^O|7DMNzCZf@uL0*t&((%$D{SsI
zUwrwL_anpMWlfx-VXj{?**&<oSKoiWPo-r#C)?7$^Ow~x=ua);Xf<_|-@NJMxzq2;
z_U_KF&wt)1vTJTt(WR;7{~mTu;W(pt-TAq^${G>#CEF(1`Cp0u@|bnTw$taczn<Ik
z_^9For+`aZnnBARUDqzM-7kOo!-1eRJEugf{FxJdwYT;?pQeIP!fVZo1*=b*>sHIx
zzfXMIuiGhk{_FSsa}0wTZA-sryqsb^Vb-IvyR+lY=-0omJIv~J<)7B}rT4G(es0^y
zct*wR=q!_>2fBN;x2{dn{Z!v4)zZbJlNp~?b!b%t)0S3~UJ1E`XvVx+ow|LuA1>fl
zu~_%+>f-R{p9Rjn(U4jC>(}v8trIH*?|aIg(6ZnBGn!p>634Pbvo}2b*3V+0Wp&8s
z-eRpR{py$ZCvUWmf88?8FGxhg_ow}p!!4@}UuP(Jtnr%o^xp@o_g|UkS3jJ5YMo!=
zQjNQ{QUXm9_g6{C&wl0+Av)7x>p^Sz+P3>|%nvPjy2_#a`ZL@7)7NAtG)~+2Y5U6A
zXZRD#Z0^rJAur#?CL%Iv@%ro2?E)4|l3ag1+C3~(S~KB0)7vEXhd<xe2qm1`E~<O^
z-R%Qsw(s3p`mNQbf3@q)zURd|<PUs)+wZM2VXLv(-JFA`w<ZQnU(vLXIhai=C+EN&
zv)_$hZ|C<YJ$rs?X~b5|^UE)>X+()}t%wb0yY;SqpZ$G~PDee*w9EOkf4fif{N*6!
zq|6nks&Gg4Z0Ga*hktuKf|~kLnV+8bs$I=B<;tvUcDfZe_EhENu)gE}bN|N!VX=G8
zcmF)tnRQ51<pJC7MZKys=JDO;Kl7iVI>F<tX=U_O&GPN1BT{y{Hpj8ZMz}7&);zyR
zFaO;28*e}NYiw=}eCy6Wxn{nv;ww*<Xa5uvyR*dO4Px}xp1*c~p3Ta2+PUs)3=<e6
zf&`y29K0zV5WV)K-QV5P4_Z$-ur+<FuZ!F8X4$<DTlVCX-etJ=Adi24ef58ab1X|)
z1NSdHvGF-a$n(Q8#+Atq$;GwTp4@+Zj(w`Puk$qXpK?9wCC^XUbrqal_<h?^uXvBg
z_q$`tk56f7oLyV_@85-|r-ChheZ97Ub7_Ki-Py?2chd8Z-Ca|5dz(~`m*}JGX<MXX
zR;8M~UYlrF`FHWl;sOV=JLQijf0en<q@ZHdA$s3`+l9S17QDXyy+J&^<m!r@Dp~&-
zA~yV+F++`g^_QIg3>H;E=Q%zzu_@e`eBkZ&?a%7hNu=$bsy64|)$4N?{**c7a>Q=2
zqN`1Aa(?dm`X2|@MIPR-ANJ`}duo=6>e4{byNg{DS0`@zxOZ(k=MtgmD_*|MpRW-x
zDdd2)lgla()`Ke3WCYBf-@g-4{I*TBX|tQ=>)6GorbXVm&_2uj3P(j3ljq+540;t?
z-+ydhvSJqd2Hw+WXK~&Yx%Qq@=u8N2RLb6W0Rrr@=l(PFzigYzW|{lj=?TLvmdgU}
zZ*6V3MdjOm&wcx&*5K2WrwQ}x*MH@jV(-4%;g(a>Cr_D844$<v-9qJd_uv0`Ap0u7
zXo<-F{TF}jkyv`)z2U?w5rvGKSKnv9G5>u#;@iJt+@_~jNX@#rGD>)fh4RrEH+P9P
zo>Y!<V`vC_bbI!n+&VW=W2TIAl_hoO`KRkF`TDt0<;|g2qI!(t*1acpetY}(KEsTu
zD|2Jrrbuo39UOhIOFF#8=cwf4u$0ujZ+HH>|NCF-iB)NQvr6tfOugFICSCdPnoEFB
z@5bj>EN<QR&(Bf0&U*K&$?sVK2ahaM+V*<YA)aHl8mpyNb<6&EbpLQ||I1d_gXQV@
z4}QKppApk1^Y!{1o~y=9`*wy0NVyl+FZ*|=`0IgGt%sS(>lUwHV!!jF<^EgJp324x
zGE3YJy}2Vjp?;p4A)mwglMT#IS3I+M@afWl4Mhs~s%Gjnmx?hx5q!Jt`tynVn5VS#
zTw#6fD|dFiHb;7yQ+D3245^<-1!LB(-IyK}(FhumefhcHGay3M@z@rtuS+G*R7S=h
zvYj{YLc|l*mroK+R^6U-Ajf<IdsB5yBJ-4)c?Zn)$3=b4FPp%jboTe}!%j;**j$1b
zY+LtDe~|yH&%k-A%YvRet9SDHh5S1rV<=mC-NI?M^70w6j#qq-9f>>q_~*mTTI+r^
zK9UYj<o_G`>GjXXlZG1#lC+L}T=%^BhWznQ3V*D!Y+SeOtoJq$U%K@cM`u-TMOyE&
zfL*u3cqEv17v}wWn^(Zt;hCJx8}~c<`EQq2rr;S_f2$8I{d6+^mfMkAXZK#awo~@+
ze+K#Ue8N|}HV1As`M$e2{W<TnN$09=FU;UJ-n9L#^2BosYNIEge02Cm{W-Ro4JuQ1
zNL6p%Ed5HSbc=*STD|@wmRdg1nJ)PWD_qxA3$Q1?{Pz4le^jTlEo)ZE_Aj*(9SU~h
z;!^jN(tGb5JG81)mG$Vc<F9i+Ts=9>ds(FN<Pw{08PldJXWU&8t8BP$nTqZe*|mGu
zuzr71x1X6IRg;fdMe3*7!u9@vuVt<aGzYNXU%y9mO68_CzDgob^+K4pZ$Hk?@1XOJ
z`%-`G_pH6V3YH!oXJr3PZS!I7<P5lOou%rLSZB}tQ}BnkvBRRwfbx9bd%de2gI+4H
z3Su~-*0E<<diL!1E8UJv%wO>5gU`uettpr0GCqIeW6NPLVAjN~=xMTPLxQdKr2J#w
zem%bQQ8#4zncrr|F9gkYxHF&cLy_LQ9y#Y#Pnvv>n>~NdI^R<7q^QWvj(cmD_49CB
z#$+<}x#>w6tWC~kVXj^K<;S0bM?yAQ->ss(ue}YsyZE%?jNgY_Cq^c-9(`l7shsUV
zt^NLW5>u81x#qup_Wb@FUeDn9b4sIwekL6G9{XkOiOsU>s*ChLzU|k`3Rtq1rOl)|
z;%(IHv<g=?(+hvADum9i6yRJG;^Z`;G}9*j?>WhtN}W~r<{zyxZ`YO6wm&82C4P02
zlU{WA2Kl$=4}3d3+0>JF#rF00rzlU5dN%Rsq+b&jUhusg{%-E>f8zVkpA&L!<?H79
z$-6@SmPp%*9X}7vJ>AB#lZUIp?KWTT&$nL>2lXrwOTNhZ_K8nb024Q7k4@t(Cr+)B
zzS`Xd9(y-zjxZEA-MeqwJ+|$Mr|Z@;nerS^NqBWhk71)Z`^NpppEh?oZqCcxzH;Ng
zO6Q(wdQ}-dW*73GORUjdX?{Dq<^#V)Xnl{&@x0v6*S0E6c-eSvZs<d|NWl~#3oenE
zx$pAN=lAyKeppe)(6xK}gd5fNlMh^)u=ruq=2dOM&!hfL+4}x-{ef?{d0dW#Rmu80
zM||+K=XOt?r^&sgb!pJ9>)sg`DxTl2aXR8PCuGX0w5le~!qxnX4}DJw{@F0Sw_*3%
zj>{|VU1z#oH~)>HflpAhF6+1DpZ^};G+mdAM|8&V(%XK=zTN)#K6k@|MKO%oF@JAe
zxbW;>y@i)SayiR}q+N|W{g40rljw42;>_FY_Fag3yOK?)A>ve&lDV1L4)z~KxoxVB
zK0&5S&7WS^kn;-Ke@0O4QcNee!tLJmpzUt)*LfvpPIkV0A$*mysP0i0b5W^vt;uP-
zx|=us=H2}<M^vkT$^Y<z^)cU@zB|rwcX{VBJ!nmic;59#*Ec*VY+IZ7^Yi6*w>dJV
zX_)kJb@$(T$#O#^c3qC$+tv+-xEfF9?ccb@;<Ubh#ltc+Glov5sR8`QU*G?g5GP<0
z_4Vt<>mH68jHd+D9$DO!t`BhVyYv0ue}>ry(gL%E<WEmkd_BF$rOfh<McU-EzOy~H
z{rQ-qUvq-fcXimWKMO@Q1irP_E@|oY%w%m_J@>rK`=f6Y8_TuJ#4oR&<eJE`;N#KR
z7lk&kFs~5Y{xH9;;AFF+DqqRAedYOcG^Yo@Wiqx>YFcryGike+^gI18|0Hq_EY;e-
zw{HI);oY8VxV+jlN>1I1V9399_w4s)*KI#IIn`-bezAS{C_-zii|UsRdPmwW?_@jb
zA@k4vM1o^6<AMcdX4xC>pSrM;g=gg!E?<>jbMDMNu(j_`^#1Rko0esIo^j>B6UH+u
zfJ3I|<twHaYdAKrXj(^nmw(&nm2<OUrEI;x3hy7u&&-{dsr7Kbt>2};{nMcdrxq5T
zFWY0wJcl#IZlYqUIm^3h-hI<<^UPS#&3*mUT<Oc#<aQ>1IrMl{xY-@W9hb`4=hX|=
zKWyt<r8G5fSFWPC=IZOQ5}Z?WbXK)*U^@J|c7J>Ahx962*IzGJ^Lg>@w{V{+nSNbM
zfn{#^jnBvTe*fzys3*XceO;r@>D+`wo;RT$rPsHvvv|AFzF+Rg$2kHIYRuREl!#dn
zcfrtXD$iN9J`OjA?%cvVpBtAZh$+W!wLLe%)m!EBb9v7|MitlfXOulvZWxMf1Dyms
zX_dx>{qJ9&+Hl1?I!t<RpvM8N0~3>SYxjP8Ji+iytGA-kyf0suO=;@ni^*jC?31W>
z$ob8J3A}%QfBz>@<e)7X!+ieK;mg09L|W~7dGFpZn`z3tq~VOY+Vkw<y80N!Sw+v;
z`UAA{cIIYiPT^f3sv7CYx-9f-@p0x)_vNNrA2`AleJ1?=6e$nAYYkEpdd{5wHo<*G
zbp6u*3>$wXHfZ{3h}7+i>6PQ^<YT@hqE#ENt+IO6fvm4;-)`jYyCLQO%D?6J{+-W1
z_e<tF`!D>vz)-v($MNqquEojwp6|52{p-Pu#)*;a6IR#Uy4`DYvPq+2*VC&}TuZyP
z7)yc{Y+YMem%rhGj8xEJ>kIq$ZCSbUTH}JOJFlMZJQzAv&{TBN_2S|Kj}zbTIHY}K
z&9AR7UvA-bnVj}I;<2&ysuQUiwN;(_-`Cr@UwFv#pkLtf$=AWB*A+bRzSjSzc3;7n
z+18B8ip-&Fa#(-ZuQ;?YOIeHg&)%}yZOOj<LB@jqbF|OPMK3aR{?G7;S@-<qXZIy!
zj;>MBsGe=ORZ7%ou3y%lU$>OwN@lqw8;CsIp0;m&adlyJ^T%@>8w93_fBI(iGXIv^
zvPiGo`02|-TYjEi*)8C)Y5R(p#@aQNHr2nsA1PV$j4k=ot>v-2rpqTgeNX8sz5bE+
zY)J14uV)YLXPVu*E%*ID!yM5{0j}&bf!~)kzqlgc_k`i`^FN2g+;!C&UObt!pu6s+
z?XkNXu4)))XPH~w{9d?RqjH+9{k&~$m(DOR32JC$YI3|&6H)y2$8EWk1yfsvpV!;@
z<>-FuG+HXeHF-^;%(lhPT5?_FZ?f;)Zyvj!hcD%*V)N{hKMy&i-S+!OJ>_5JDeJ(!
zdgqU~Z-15@S`;;D@eG5-(dV5w?mlKU^We!mzxv>;rM}-LSSDD0Y0Q0NXJ@uiro3dS
zuJd8H{Cc}tQ`We1F6FhKzyFl9rqOH-D`)Ac%2z+sP5bx#Nb6!v%gK*KK3%wc{XKsI
z_vDHvK0OY>+8eB+-sXP4BUf(z?;Nke+vL4kcDFxU9bVRy$MAYtGrQ}KjKjAcUr*xw
zw*7MMw)ZWZ!NQ5_)waE_&R*hvDz5z&x6Yb3tF>OSyx^)-DYIYx<)BL2(&D&XeQWn^
zIO)L<@FjVr(p-hz89@w;>$q#?zi#VXXDqVw%C@<sOH7OYmG;dP6%Yz~kQr^d^*8hO
z;@qEy7&g81>vh}q>>BfY?c=*QEw%mrPtmobQgr6qUAJz{xb^J*2kSYuDO=gLUXZnA
zwLR?K<mh1+7b|b!eP)5!HIaxs-UIplKW;NCx}@+uzU<dOrCQOh@N&T7xM@M=+N;0k
zRJ`+NzEOVYyH?Gy+xh47Z(ZqE3%<X8LF<}{m4#;_C0%dadHaE5zwLe=t4zhnn5O9K
zWl!>-a-<4G$L-${IKjK<$+Ye23HSZ;^nZQZC#>Hga(VgX`_r$sD%$fd=nHpti|l&6
z)9l&(_Tw)<wk`E3R=FM@uWR#gq3X-ahoY96iUcn`JT>=iePQCkf9cF8KAoC*JpA+P
zi0>ut6Z>KoS%w)JW@o<RVBWv~dts;1vWgk}`{U+Ke8y$UFRNzsN+RCrSL4(A*naux
z*BQK~I0l-RE#=oL&G=BDxu)bq2-B)_vuDq~@lA#Q=ih>#4MNYJ^{%y<`JW-<qw|FW
z`!3v?buKlDcdK;F_YLp=Ge{^-Ir^q*zur8~yS>h(mydJSM#c0S+<WkT#qI6+HEKnN
z&E~A#fBsuPgVcNtjcHF>X3n{tJzF~B-tWA6yXr)*d5@MqKR>z3SybOiLFdhNwigRT
z?{>1>?&kaRx1OicQDOSE1-0k(7dEkMHW9FDOc0G=@@IQ{?AU$(b!=gq76rXo9bJ`m
zj4i12Md@4C$kQ(aKI(+st`B_t^ZSoOLJnu#=YL<LFzM<wmZsxD6GRxl_nZH^o%><c
zjEB9f{~6Zbj`%cnVav~37q%psZ&dpuyYq70ow_{_SE^XeX&1kI-RtlviA^i7Z4j-t
zW;n?9_D=lv?T@z~KgJQ8<59Qj{^`S~oJ+TeoDe>3CVsJwQ|9-7hT0FNI#<ueCtfaY
zY_V8=OGL|QOV4RVuI<Xz6a3A$fB)IT>S5`?!Wi$@H~&b*$%6A0+}?s|iibB!cZ=@h
zViw!*G(LWA;z_493%PzfgeP2gV|f3jw(WPhMAq!H-%RZM7xYW2bUDn5<>Bd)deS}b
z?zwBX?mTSlD2NGOpHUZ7C*>r1WmQXn^!te4Wzywr&wt+y-DuC<w{^3lNR#INxC@@L
z2g|D`96Ox-$$lQ|M5$Sf*FPjK>~hZJ{qyPDrH{uCOBb+j_<p=)_3qu7*CrHe_U1E0
zhP`QxY!=|p4U=a4ddNHKicCnqmax~aT9c-kJ~CpBC#^WegLfTIT+4Up`<wgoq;>fi
zee2h6oqbDYX1eD6wF>tVwfD8C|7VE4^`GJRb3UGXar3@?sn<wWec-F5e#hhR#P`+z
z8U8ak^J;6|zy0*@;U-Vh*WVh97d4jqrf{>~J3M*&_uF#9VZl=(-sgqBjxaCtpKcMd
zNo~`<W~GTc)a#$N1l=&2xBiXvk2PiyfszZ<asxdz5=GnZ|7Y;uc>6~Ue~mVa^xoQ^
zjzvZddAoicE}tG?xbv%V)PcGE`%m7wfA3YA%-u)645ISR^>Y_YnsoF?L5N1Z<JT-3
z?i+H~|JeThd3dVM^u(!e({)e%i@#F6YiUA|U&}mo--G66N`K6=9~3iPuFCrt_x7@*
zphm;92{U7~t=T;}ccg!p_x^b6KZB2eiq9g)4a@HRx_?hb*g=<Z;$EwkSqH6h59d_g
z-Y2iqWb}5soloYrdlP<UY{<y_q_y|N6vHR+)9xql-|^t#>Zq<cvx3&V-T3#`txpDy
znt8JXQn(y4WgRZ*aNdcpKa!}rQ8MrS^3B`t+>;UT&}Eu2b(4o9_s+Kkz8!zw?)WC5
zvUCHh(%RxM(|xlXFUj5dyTEk{i_1pqSLQ0q%{D&apTCY}YI9WJo}K#_{xVH`sJ7&n
z!|BN6*c$HX%0;^W8RX~5{1b9LyGrwE)PD1{iXeHvT@@2rjP|TpnjD>Z>4;eN!~@dr
zzr1gmZeV?Pr(F5{d1pP^O_ntH{#_Ks;E-{Z?RkL&?>F|B_wVz}wDj`c%RX<{o7zQO
z9j1XDCz~TqL~QJ7au)Ku_n+bTkHlknp{h##%hfC9tCoJgeS6j6%w&^4rcP%Lyk_Y2
zI(CQocyj*o2PZ4${Wj7%uzka?>!&`X$Q%&kT|UKFZ@o#v@)?qWTWcSlKVG}jx+d{6
zr=quSj4bcWRor<wZ`U62@@5fRTzYNa1h$EHCY!HvI+K&*!SrXt_Ok5Bw{PbdMP;3F
zWmoci^3S2}nEV8_E3%cp9OfSIj{Vx8cd&EE{vwMzb6Y>0EOr)P-K)F(>e&Y?9aZe(
zoFkY|uupsFywY0KdUF2d+aDh9v*{MuS-4~QC;ppjE__j$eVkcm;`#FFE<GF@wjHiN
z-+$)uQ3l-!vF&lQUU_F$#B{M;lP!J9I3?4a+rY|Ev2`0unezSI-_dfV6Qh2a`<DGY
zIKf!JV}I1!?`^>$SKPj3ELQ0aSD2K)_xw4PhiNISCs*3-vOmSqsS%L4R5_%r+HGmh
z!}Z@CRqAYbltMhU!_{kl$|YAE)|FG+zNJjE%Ps1>>W)hc3=@ynAAk9=y*x9>&nxzR
z;rSnJdcl|1Ta=w&I3rf%jd%1F)@?txe|gUDby`$bGu~Z&wF3i-?Z5f=_gDyV<{Jd@
z9{noBUZ+=i<KxHnl`0da#?+@z+bE%|-*Wx6yt2;x9V)SE-x>H{G2Z`tXqIa4!5jT|
z-+sEd@O7ZyuCLEIGfx&u?&)zozUJQT#^m?kxBtBVpy=8(mW<M;CtuhvZ(kI__@5zv
z{wc0?r@v3V(4bp8-QVWhgZdzY%*~wB9_)T!6*T9|Z--B9iPuAao9L>BC_7msc{AlK
z|M<7wr!qA&fXigTKi0ja3k|L=nHcO}|H{s8_Lfey^`S@KCT2J0<^I`lbnOL^Neq{N
zzA$T!2v^x)pls7R`%1{f?5vG>{~6Z)XV_;@y6mX30dL&CZ5e^8I?uQ^StK+FDuit>
zs=jqw$n1|nXRXKH+M6%0?=j6_wd`FfW7H6MZb8FTu{(Trb=40~zGw1spTBVVH?t+m
z()rw<_ZporS)kBxHi7+jM3Tw$f@@24?%l3kkg9q!w)9i0-|14hMHVu5n6sNd>2AL~
zN7Q2S>3R3+;=SCLENWdZ8ynsm+Iyy9((K-LrMqkDy&p;lEXgu?F}L!N#<Hc(e!KoY
zn0%t=wAMmC*Eg@Eo9nOFpYO2>RFHYc@a@z6w>(oMb-z59jpW;@?BdFN#mr14dg8}F
z?^_ETSuVbO{rS%4>_bz881D(I^k+ONwR&3}J^4rR{k(4nS}urqdCy(F{wr6*bPWN9
zXYUTyr(L_`a6`KMhuw{wLt(OBd263PXVqMsd;OG{ik0u>G~Nwt{~6>j*O^NgO%AK#
z-YFaR%qBo)W1p|`=AA)aZQGvwz5PCU>;7pBt;`R3WNL2vFSkh9V!+S*wO?Z=&#rG)
z<=d-&*IFK&YO0d%%a~mmd~r)N+e#+R@+8S-<9Xjy_}^XMCdI+7JA3^rrUOyU>*fi4
zsylnrL90?ZTh3np{hi0B^rsZPDE?X8#F=p6{<;>`S6`aBmSpgK+p%-+Hre0uM-pV;
z1$p(d{*pDZl*rb4W^sv=VY1uW_2=F&{N?@6pi`9*<=-B^e~uD!zzokMkKIfX5zYJ7
z>F(cNzf2}g;QYt@*Ly7cV@r~of~Aa2rcJv&!8|#4J3CXP3QNPgUq4@-5)3*t)zQQI
zMuu|HqTAA+|JcpDQ8?$xS_i>D0f#4kZQXOwYfaz7?V5>2zI%Uj7v*ld_aoF>%P;%-
ziI87aPD<YGOalE+m-HNa9?lS9u=DxzJ_F+!YgA?#hy1i%c-k|N$wF0i(yNPWJ8oz9
zeg9ivvWCe)KVGkQrJTkot38@e1C$uIOHFCoE&MtA?S6}zjWZ7ipT9ppF7Tn=(x251
zladxrc<FAp>`HD?_5BZj3mzKnY_Tl*&k$@s?c#wavf^P*TsQRdmv4Xgxj{_X==g-C
z8uu-#Ca&ASDKo+QRO*xN{Mxg}>+2u4m+$mAW%Mij+p8%}DuHWduAJI!sJramYR2sc
z9``?d-|sW$i7fm6H5I*UuWe}Z$uZ4aQ1@}8Omy<Qg2^?Z;%haRW-m4W*3PnbLBgs{
zQ(VKc|Jv^}xN=sZVR72e(g<ZS(~8K1gX=`^aGM@|m%CG;{Pxd>PE({c^9zgf)s>`#
zSQwWuEbwTK`o6a#xb8ng=abx~G%cN7%<+b&e<rhfu`QGK*3O(*s&HrJzHhSs=6+CF
zp|SIF>^Ix0gMpeIjqO|)#h!cax;?M_hjl#PY-_6;rH1~;bF4KLws1bNikxJxq1(9e
zPJCVM?!Wr|UY2+Dr2g{XlbQI4FP|xCm22CP_~f@YZofRoJ>O_W&EC@AzXBgVy1J)0
zFjx1tsG^zDZN48uTtP2SuUj-{rC-fLr>N!%Zh2F^HLFXLx9*QKdp-|zK+?%j-*;b@
zx~>1zEOfMo%}r{__LZgFOm+1uq#vwSVBg7i|BNKV)7nI~fX2iL2DjfWs#+`^_08tn
zuf)|e-$(5|_coF5UCc9i`{y1^W)r2%o=hzA{qe@_Kg0VUk1sEqvSOv^-<?tS3R2l?
zcU=<{b!L}(H~ICxzwO)Y-yRbzIw>dgaAx1$>f_UOjJX|a&g+WKjc}_~d3itYKST87
z)^kCdg3Ev3+_HR&(u9BI?OzfqFSst5J3X2^_v7cC|K4^?R+DtL)URL4AKLL_`YHb{
z5t55JrH||6IPRCque<*K{_E`$eLGw0gxAMC(u&wwD7xx;=+veiT%2OkucQAh&d%?=
zbc#cAz5mvq_w)=FOyt?~v@J#FUZj2H1paL9?mP?o(>0Y|XQqXIVPAjmqPp_ZnGg5O
z3!G%}r1Fwq%{CUM@~y{<a{n_}3mo;0Vob3&sI^LKRuL;WQhNO@@2${^ct>|B&#!BD
zyg&DwM^t$CB4eI4caG-1%X@ljf}`rxN4sD0oax#ZS+4OXayOHi`9|>{KV=d+V{M#|
z+*_yb6?}TClE>6A#^Z07)T&H8adt^-a+%_+U)9pLZ{1&aLtuu}rX_#pHvF8c`{d)x
zOii(;mk%nfS{}S_`p#Xm8Dwhxv&(-R$#J>LAe6bb`_Hzee$S_HrO8%ZzBli}lk@KO
z_d1;Tmxg`Ypz{0v4aR%lneSL?1*~c;$X>~QZJM3$0)Zx<D}Pjf$>#1coe+HNfbY}O
zic;NKg0J@1n%{nTyYEKLR&9-y-|FX|o6=N1+1_>Uzry;q#h1J`s@yts`{NGVx5;1k
z*{br#`*!?uTe2fpS>DLI^lIXX*ombETXG_-YcKfzuIaPeF{N{>%I@Bv+f}mpykU>5
zMS7OpXgXcZ$1Py}{q5KLd|r1Mw3_dq_pN-R;5{u$cFCy_x0OqSezkquF7~*8p7q0+
z1r2YP$8WsWq`X2PzFy<!lHZ#+pLR+IhAAghGv%>vll}XT&qk&~G)#2u&-fX&i4U0{
zre`0Qy<(B%+`L^}>+rlkMFokE%eMW>xi_ma-y_R$>;C+Eyn>gXHXJd!?|JXQe}-4N
ziU0ooHcI*|nl;yd+4-lhxDWYU$*B5u=<v?OMQ#)Q{+>N{c>m-4JcF5BL9c%_+`fMQ
z7R!MclbYD6DJt4}M+6Q%u3!G=`{7fu-P83K%Dn$9ukpb_#q`(fmyBJm`8T@_cQPlY
zG4D5fcK1W4TGZwS_xP&+3<9nTSLIxIni85iHE?lak^L3(?XUNLI==MewCPGm;vWBc
zaX@6M;MCAxPTt<nUh%Bn;GB2*^Znn?`C?8c)ZBUfX^FkY>45(X`!!;Mre3*Vwz$^0
z&VB3p;}$t5pESi^_Ft*X9J1id!t3v4!(3G-c}ytCPde~^`}f<=d8fs8sCF*5wvB%+
zcPhtu#=}_$8?L#nb%}oWx4Js&PQ=3#5#{_Fa_gpiy(N2h=NbVIT`glH?)Qlai+9S~
zw~2Z)P5X7J=dhQ<DV_Xl8$wnDd=D*Scy+b5^M1D6vHuLmd8bVK&|01Qn|}>&(HwJM
znVF51j+2ZS%v0?qa7y=Q|NhIzHp%F3<-`-!+b+D!RA8EreYNT-gI!Xz?^oHrMS70!
zi{Do#c}<x7aoOUvX8#l#_bPOFE7xB0HNSOl@qXE#7F<(`t>(N=vrFCn?CUjl!+F~q
zgkF>iF1?zc{O50*jGAh~GxO&we;wSkxA*Z=O}Y8IoK|e)J@M_^zPmpTge(npy7y=8
zg{;F<CgrXv4SVCZVNbu~zH8EV-tW8pCsAD|K#Dv3$}h9#x4da0E0@&hJYhJ!=*T+{
zor>+BekFO%S@5*EtM0qq+QrUiwXVLpV7A$+J&iqk;=aQ_3LB<O@C!_T`swS}R)>@y
zl?TqW=+4};!9(ow?H6_R`+j^ZFpfX}bC3P*Yfs!y@6cKJ*oi}{g1IpFyL$28_Z^=d
z^?cj4_G-WVyhL$fhS#Q|7G>t-?ehNb|DM`;nfvSWKYta&nt$~a7)`x-sQaDCo!-Ca
zbJu?Ccs+yt@E_w^=i^NtX07`sUDIBwe<06pcioK287GUMy}kbC!j&C<Us{g&uX>(6
zA-}*uetC|@j8)n%9b<QXYWH#L<+xV5f@jt7{vYow|9pJ3VB<aeh0nk1M%|gK7-ScC
zX$hxtz@k?S4APB9*Vy;jtbCd~>)w>QRf&v>osCwadiz$q{XOUHe&yT!=b7J6G1hvz
z({kJHuI*byG&FmJohRu_RVK*kvA#X~ySQ;`<Ac>-;^!(K%nf9@;Md0Spm+A2_{aNS
zpA!ld;okAODmZ=$XYeW)Qx5iNOFSjCm)vdK@aXpUx9w{ard>aM<*SET!I2rkN4l15
z2$!CHKj8k(_X+uJ-m^lE{mS_QTH3xcs)zMZ8~d3JD({xRFR1T0KG9Gm)!w`Ad`4B6
zgW>Vbk`hKM#r7*bzg_$H`+*4yW@fU8-3VVS=g`(Fk@H&Oa>mJB?|*#%^`JvhQ<5_^
z@2~YQS<_D02ba&WDwMKmGge0($OR3T|9o(AZoK-FukxFJ9$IqKeeD9HrBkP@DGMsI
z={xq#`jh8>hB>@iD^#xB+`lkZ-tj5>l;z3SmlP&Va#J$Xdvs)hbvg6P^XF_tIHU#6
zzmlK7_C429#*YEc3fCu04G`cH`EJAfv#@zxs?>A$t#bGMH70CN`r!S7b=%(Q-^-+{
z%WUfTZuF!reWg`Z==Et~%fjVzOeW7{op+7ZF~R-Pe}?$=$GBR~h^n8T_QvvcVDvQ}
z5&8QbqRy5RC$Oce^hWhwd;a-(K|K#wB&S2vS%+`=lY3XQOgh82H~h`LjVB@$CeGE_
zl@oP+`}aEixBdQ>Gc_4Ds(en&x6i1$^mYxS_pTczZStw_tY_bz-~X~{>N87@^rwq+
zKW%;ZQ%FzZ>*WJy*7SXnx0teF@7lr}{MVU74*2Er*3Gx(o3TSdfYs{N2a7C+B*u-G
zO72#_Kf7)3@0#xi)x1=vl;rZVPfXt{eU5LZYIDbwWve}IojYK4>tM;;b~C#_%wLW~
ze)`N7w!ZMQo$KtUqMtv!(+uQWGB2%A%57T$&l<LK)*t`fNu1np!Ylv$+60bCZACS!
zGm@X(6<N1J!0h{%3FZ&}?Y@2czL3hJ5C)#dtKTl&xmPK^jPcGE_i5!}nI_w2$KB47
zUBhovCnj%EIYUAtYL^A?yE|vRI2kTAvFpvaBYkSYg8cBkYk%-R&k<yqrPc3{omie*
zv+lx6zXgXRy#ppHX0KUwf?MF&gXovrer<R<vpZPm6w9ZTpQVbE{52+Zx`y#a@Jw}?
z7O$kJ)Ube+dj+58{-Y<~<<)+e%;CX!UOBibYP;F3en(cHSxx@2Q36*zmUi6@%`utV
zF1G!^og~94OMXn^dw2Gk<tvjl0v<|BO0R!p65e!bo7Ms0lWgDa&n|LM|L`ke0aJtg
z{OLyVzO2&cWbZO5@`ptSt!g~d+PXe@hs?YB_4{sAr7}M9EX;G7`g8aG&CX9xGR{A}
zs^+1t?fF&nGW&l`W$pXVp!T2P`0EafsV!@FpD)$?7t->sW^rI&qLPiK`l{~_BCX{g
ze|auE(a?og?#TUmUHyf+62%&=On2ko@~MVKHOlRtn$U0~vTfqq^H&~B`slLp%HF-D
zR&!g|GkERMyJGjff5|)rrWXplDl2W|-u|)Ouw>FH)}Jp9|Fn7eo~tuUd41kIj!moc
zJa+7Or<-4Y>~=kmhRTMB`uX2{zdq&B$OxIVl7pj&Q89nho&M}Qf;n1hLQ6#(_Z#Rg
zy?0=~hg4wnzmH$~IE#)MG3;Mjnz`+F|AizkHJJs{m#cZrKC9VGxisZxweyaXsfGe2
zjM)v2S5w~?*WD>T_M`vaG@Y1>tN#6Gh}s*)wBYEkZ!Ow)RC?Aw`LJsn_rbTfbAK1U
zZP#QFm}R+h{jv=YngbNp8YVRyIko-zPI2k)-+yesUSq1wQIzYO`FZ=j1<ov!=4R(@
z>z@|cooVDD6WPT2d)eO~`|D40KJZj!z8;@lANW*(<yM=B3Tth-R;9xHTe&;-f4|Mg
z#i6o+_igpB_b+(^RyW2k;eFbBvs>ZPEx*&>_w748rz**TVYNfm--SU<8VV;Bl3tXa
z;+nr~HFI@5!?d6vk==Z==I=2%y{3k1S}@=1#QcDyvir4rL3gmNT&-Pp->-^y%daq@
zld0|}%@<^Zt9z)t+P3)p`P}{7#ph&ZTXjcnUApTRhwJIpa{KqqvXVWwh^=ksMrpBc
z+qc)B|5TPT&D>^<`R7mlB27nbIPxt`<$0$fmY6Lq=lAdX``b*5)7+<K?lygAdy8$h
zn26y~j-^ek+m6WBzdWwKPsA_KyVq*z+AS6nlOtp{Gp7BW+xa};4&(XD#cih*e)LWG
z{`=}-ev2vrUml&qhOP-~uH8G(t-Isae}?$$a??#(jaF&d+z)^GSJ8OsE#*r~pXiqJ
zl*X)<ZNHxV%_5A|vT^gRf46=eeCEL^(roI?a9VC!kbd@e<%#E|zkfUYbi$(INz&66
zZT-#R_x|s~q{RUjoK`pPeWzplkNxxa0~{Y^?w|i2C9Ami+J>bo>H{1icUZqqFc<sd
zCpSUBEBN*G`*WkDQhAft2P_fOTa}-CJNfOG=Nuu9C+_^ZyXRq`9-~T28w=al-1?_?
z`d_v!RM@%gKf}G|SBeiz6htlwpXJT|`+jZj-VK}6b&a{USZD5Ee5i4y;C<_W@bphN
zbhkZ_-64O!hF6%wVUpYKpZCAFzMC<vQKjQ!`_$0LX-9;!-&f~;`S-TnCu+i?cWx6_
zd9t=hK27CY<~?_P?pEo0-*4}KRnNoYb!Nk|<rnW4er@q$w&?r$VEUb`?_%QY)$zA;
zOtnA!$l&YEwYASW?6cY9<qQVN<SA_mQ{(T*tLM+%;8`q|v25|@3pZ-k$_ee<evjq7
zM#8Txi{9tjpQ*Y3^{}dsuj4}7`k03=J<?iq`u0U?B!8{y{kHwffsb60(Z<Wp+zq=o
z;S*zFF<)zRcJHYZM+9=JZ^Rw>CjYUaNqT<Bv&`QPEX!G3szVIpcQ1)8PyXI$w*T1i
zACH2TJz4kIfAQ<Hj~u43pX0c?WYf(FdcWhIR2T2xro7a1Rqna-do&LRo>4!WrZQbF
z;L9Zcjo*IpR6OFeWbV#B{cCTWOn~IpmaTz|rDn4yc_^2CtuC<r(8F2ibZ@@;y7_l1
zjaKhr%Gz-5=`B+mUYFkW`&ZQK7q=T_&Jf(U@15<|X0M5U`zDCZ*^tJv^Vv1&?oW2*
z-~McT<d;|T?^hm+rO=E9#Who{Vvd=<VqiFMN3goiCi?SXpMw*$gBNq(ub8ao+~m9K
z-vi$nOVVB_%kFh}CGTW6<CCn*B+lg@a~1zHOjaqmaPf8Xj7Qn46@GUe_?cdO{FC*A
z(=*PAOqcU{kmIOVyLD~Ci^ImNYs(u?21;v8oU&$`c6HSAZu#rn6PhO*|NNqPv0cgY
z%pswxo+?T2Rg{jja+#W|{W~(ZfB*Bgm08MKk?nDZ=Y4pnu;l%hwna-eO^*zjVE6v)
zn!4?Wbp5I~Y3`3QUF0x*%Es5<`e&tOuIHOM<JPz29Xr%t=U@MF_~{u-*$|Iw-xmkh
z-;<f?z-IRQ>sy)M*=J=kRtlC`?`x23XD&W)`^$lF3zdpjW-Igen%@$%D46u*-+hf_
znQNS~>qKHKS=R6U`Qz=zgFGK%eWSXHmft_8Dyj45@w|jk4`-!qM{{#iK30FeU445S
z2ZxHr7rVQeTt{~Gwtf57r?NmmI;79_oAtdr_wM}7Z&>IuspaN3-cK*jRC+ACzeOi>
z>yfS_Z}+cs*!J&l-|znn$9P(;pZ%y=^{>n}FR^O^d&}QPVY3|id~cdCgnUzeo%{3G
zpGR^UtvpMDv*Y&PyC_t}wTk(`>gQ5jN}C0Dy}Whz?U#SGA7T_7RaR{LdOv>pE%~;|
z?pvH~eZrj9z0oU;TvPUI{`~8oCK@HpnHrJ(eA|D9v~H2a(_44_Q{F1PLgDB1o*S0T
zvui82HhzE1>vI1Rvn*@m*Zg3)>ALsk1<HKoRG+a*Wbe5J-=g`mCl((6{rE&wz^rNO
ze$R{I?wF)%z<+%UlVzu7rHSpiXMg5?m(D+S_~W4z#{wJYI|BE&zh@2EURWVBHH)#g
zxjpYm#G=Uu%%AVwR}iv<$tm}FT}u2ETO+Ms9AZ0dwqJZU;d;W7_&(QvCX;{tNiw;4
zZgJRJ->>D#J2%{rF!1j@UHbQc>fD?6+;Y=Q83cIW@UDGRDXx4cVP@!bjfHhCyKANH
zn>Be)bi45NC1d4EPPPn{iEmH+YTdWv(`U<%PrrK9<>YQ(BFZu29=F@E{xqpsx^E61
z*S<YFBCq1lxf>DzSwCxEUbme+X9}<U`uS_b&;4Dt?27E{#8nTtrTb6aNeY@cm)~jj
zoqPM{ZMEBAsMBz0LFv~fca;@V#+HG(;R&1xdio!CJSg(K&3wei!Lnh)`ni95`_(EE
zy+V%e-Lm-5mCY}DXE_#07Z#X5epz??b&Jmh9j$Bo4a$nl42-6IEd6t6&n3BEXWB!z
zD%@>PvX}jkvQqDbNNz^dl=8J2iER-~^Dj34{G=G<rE-nMVE^hT2KMpi<-9Ju>1F)q
zEWK^Qy|?P8EVN`znw6$K6{!l(t7phP{`f|c*NHPP7GJvERm2}3;p)kD_<Yqt-{g*?
z9ddJ>bMO6UkiL_DeB+-(DJqlZ-)dppn!R^>n>ydCbsvnaC9X8ddDq^4x2ODVyCMTq
zlSEai1Mm4|8=iQr7WCWo>tI-*z-3_pr%B>Xcfb8vU%cPKRa8Ut?An^cFSj23a%p-?
z$fAXhu5nxqGM?1BBv)~JbgTHE-3g8i8%=z>za~3>YAkmMIOK9*&&HRZ^>eqG-QIux
zx{$ELRIcLd6LaTB6i)Sec~!CMKuD<8vEQ=MbI-0X{*?b!UgCqI%Jh8qmHQ2|th2oi
zT$wfbdUVyHs4!`R3%}Y9cCg$#!gpuhorm2F46(<yZEat0=U0%ZhM3f+7l%5zBw6}4
zWl4P%(7&_t{hh!1In_o}idB{7?c0}ncin}?7ydDi8{hppEphA7?OTsuS={MA%{09|
zNqXJ7du<y&%ra<dPZYWx5pm#>rMY_d_vicW{iwdMbiLa@%PO&L8&sC06<vLON$6|c
zhFLcoq;I^mef#&%99{*DwnE>M*Z1URvZy{@wqWkj$zJd7Jv%VTUco*_@)L7>{PMUD
zt-7+mgI%AkEW2<zIaMt7{l+`-O?EptCronq`|ImH8&&Hdrq9-j(XCFt2dw$np1eQ)
zeH%;AOvi&Oc;cgixMn(No?)<BGOglq%7)~<z1#2nXV4KfzO;csCuY|5ITk7tow+8t
ziwl`dym$NCvu}3C=bw1k=qaB2edXeMm5UFBrt@iEza%WImHBtoN|oGf=Is1G1>Z`=
zcG@d_U$$^1w}I^9eYqDUSVQ;SmOX!do5aj11@HL&Y<d2E!{+0=*9+Sw@M>#vaj;#p
znZ#gIYk8Q-k#*gF2FGu|irUK)S1B@uN3XmS_sZ<ge+K@#+Yeo~FWB)d@zcK}B8qAs
z=5OVElb4#m=k`0+AN%(o-^Xrz=z;w7t-n6^b8`9GE=*d%xOUIsBH!Nk+0hBs`G3C)
zxz%%DcfOh%@x3stne(sbZ`TRs=5IH={qay__2m4?{~1ggHF>z(S}LDuACr3Yoi%ss
zdAn~v4z~p@uWSE&O5`LrKTB2XHHo|PbKN)oF&Dq_py(Axb^i6Jxr&_=iZ*Pmc_^2A
z<8{xC$nDaGe{J8sZCcPY{r-i2AD$}pDcWT<F>5jiSpLy}f1fYpLY>0l^~=x6rOZ`+
zHn&M!*wA*$1be~T{|q^cxsJM;OBd_9^_QEbwRoGJTdwGOs#PKXcK)H?4_?fe_F-0$
zT~(e!=kx_z%#Exz1c{tjT)SiI{@C;TWLo$2@b3D3_>*m%rD*`yZH*u^hUOjL{=IFT
zZm5~K+^#NQ(}rh)w%%+LlS7q$YsV+wo<Dd0IwlSUjle&vqkiA7O_-F@C6u`K;*}|`
zWhw%k8<}tacvyXD@*@-L?c23#4&ON##NAM3FmrBl#Gfl&k{!X{x0!Eb&;5PB{y2x4
zbWHr)SHENvI$36glwQB2{Crh_+chzzW6!Stv*BO6A=PM#_XhsmKlkpRbbZgtAWzY@
z*Fmj;k~4Wv-ni4<T>aNm-bd$<v1Y!T-0jz_A&Z`Lef{kqv{3Stw0qiD!LR#YZkJyt
zmes-Nay|L&^@UB6>e?NW(=S#}iinYp_@~@mThCW;Xvtnz)_L1xW=uXMqbi>K^x6eO
zWdqN}dXCY}-y7}4-`Oj+#$1z(-1n<8J~m;d^r7UgPYg;s6`DLa-2UCJuTB)%FQ`?x
zdEV^PS%;onzw|-TFuWxApoVkJ$L)1@s%sLPDmcq>HJ#Y!ww+x+MKI`a(SA3pbDfKR
z1u1@?_GIn3-!XZNe+sT>ta-QU{pYpv>G>9cGp_&m)F9p$(i89ZyQxC9?Kk7!dVw!3
z<x5@fpK990@Ha}OJVm^*Yiik~hkdIj>|J~N(Y0+qK6!d@)(gnH*T=aDoKRq7HrDA#
zRq57We)-t#_xrzW_<5v3a`MW5^>g)~a;a`iTqBx(UCSgY;8)otr^Zj#@(-&MIA%<k
zBpRl|zfn}}*`9ALMhtS=;pbEy%~kZMX-m>ydZns5$IjyCky4SBDY`mm-<4K$tmF=U
z_Ur3qC#~5Tb0(~EZJWn;o5}q9{)e3!M^0~>E5EtYeW}7;yLyX`$_0w$M$)riv%T@S
z_x(YRy^oi1*Q;RnZTqHlo^blJ_PRyBX14yc$cPCGEX408RXo0W;(>f&+nkl5Ht#Z5
zeEs$N$jr?9Pd|PA-61fs<5P}OE{j&C+`_~J#(m#!J$`vkYRb|Ky;7NG+bp|j`5HwI
zcGh2CE-{<9sE2KH<u*Q#poPb)qy96<Pdwx~V@<2p4`02vXD-f(cHAS#sa3o7=@u@}
z+I*vFom-mnSA4U(-|;W(J|A}g&#tX&KULW~$#p6=<#XsMvrJ_UJrFA8b))y6p6=V+
zz3W@1^&CjkyS{$;w)<xU*iA04+ndRlxo7vJhq^C<7?lo4&o0}r|99f00Kae@y|~{D
z8c%ZAw(GL`%(=Jy^*M=<RpNolmI>WArzhzjf84Q|Pd44;U|nt2N2Qpp0x#2gH>!Qv
zB=oDKv21rlaqWknNl5{W>7q-upRdmQUgd5UrQ>4hsivS9)@vFQ)A;h+-qxSr{(Mx~
z$#c$3gtzFr+}CSu%4eK1@~Sd?CJ9~tdTiS6X{vs!-{o06|7zjt&yZ?b)wlKfEuoNA
zhHG@ZKA9|!W>c8PBKtRd@7~@28O}*gWYqK&xpe>PzC1aRB@Eo9zrQXKKBLjtyinmk
z!|L3Qt>1smuRqTo6c##B^Xcgem0g!leYmsW!t3|)s-8O;erDCW&(kmb5r6wf;-|2u
zmxE0AR$caRRt~P)cOg)D-@RlN_e-VBDbhE-KfaMDZs-xIQg1c!{F&<+c>(k7*A#~~
z6i7y{-^p|!>cRW_w|_n8oW;Pj_Vca&7avajv8~zlrN5_h$(FSx3I^iVpSR79Z~n94
zrm<5<(3P88|JAIP6FNKj`s(+5p&eQeh13}1o0Du0{B!trys7^Xr>=K+{H;Gfoh4Uf
z*Y(t-2`;v3-#0;j_4o3fKW&(QB@|s-s@ZQB^{(XU_qI-#Y1aQPv?v{D6Ic<P9{6j1
z?tb098$$gqh^y}0ksq6Q+JB|N#(;J26+#>s)c4IU(m(c}VI89`i-xM&^WU#CTu&+L
zUak7l-!tv<!j`7%y9?`d>g@L&W}VhrqhfYtb?l$B`#v1#^;MX7GOfpIqmx5{y`Sa#
zpNtL5g`y2se!BePc5I8G$5LBGZ<B73M2`u)jhCxuf8TdUu)45mt*5#8$`|>Y?};o_
znPncor&R7#1Y1CK_9wGv_wL&>GX-5*^!)Vcssm?sZG3(GEyHimtEGI~)V|wvzpYQM
z&(V51iRDnt@Ba+BpPR~cBwk-tEIhMmO^WtzF|!RD<@;G8l23E7>`Y*>O4&3y{*l0^
zNoOV*X0$fzt-jrT=bJn;-vVXRdw*ko{ysiw0)yuMu;<SiCQf&@*kI)$Jv;t4^P@Wt
ztIvO}E!qC&-fzdC$qsdM>prxtELQ!p>7{wVk<4ee|0&ewx2f*lRwYonfBKtTexC)m
z-*Zihn3Sm)IrZAFhSc4Bw;S*MexRl5$*9y^JLl)&fXNdy+P?mM+?L6dp{o>g?|Amy
z9sAYuuOvKG)KXRb_2l3D3};on@2{trO-p#Os^^U7t>0|F%I-71KN0K9cy9mU_d=@5
zx`#{O%ex1N-gB50cKhYVJ8y4&`*C1X;FRSy^B149*{Rm_%<l8dWG3s#Z7dB7`;R~V
za**X%aE8f^(%gqf8D_m?HeGX~K|6={Kf@JI>6ni{A8DLgR4kQouWp&BMVjh=h7M&v
zmz*=d996@PG#tBqd-nT3`mHHbjdHyw_=jGyQRO)_uQo{NYHIG<Cp~xcp6|5#&(JP+
z`$5-~2NPGWUvT^V+SME?LFYMtdxquaaOat7sIIMT{PDhFKDYS3KYvzVoyU=Ck)SDK
z$`d*#i$TCP_Waf#|0D`#y<$z9m;33(VU}e&D;{5qoHJLoL2mX3-JSnxKN#vwUwM7r
zU5=24TYmmkR9fdYF;u|x`g_yfcV&BbzSG?%x5+3y)H3iT|2vJ4M7N!>j*J&MrAlUt
z-8=T+pZq+Qrk19umG)nr?LVV%SVNXSW}(rUp7Tm478&F*JfEF^>$h;>_NV<SY@Ao#
zf4dU)@rlaqwGZ?|8oOU9^oe_Jy7T+b!^t6ziXzp&%rAazQu9=tCHQUOGN;pWho0ST
z^KX9t>pugJpf*GEANklyF6Sdn9(g@AU%6L`G^%vn+NRLQp8WOkK8X;~u!T!&|FqB1
z*{RZ0Vq5&ZZIV~dEyX=mt?O4;f86`yPHo}K;-JYheV-N<2Jd8=kaAjSl~~)QkcL~=
zZoPlC=|dgkAOGvz|Lj}+4u`LQAI-Knn{B$nEcx|sKQ)@XVsK*ExLKeye*ga0tudFf
zc>Vq}Xy<<2BjRN*(9r2M_h#OL$;H*TubzMU<Kg-#*OtD#etjy7$O7romyB6U^u;DB
zs9#|CYWjEW+b>6Aw63Ua{FwHn^4iwg`59JA#n!)-SReM|{)Jjw-BMP)q`SUz9TuNm
za@y;}mL;-BF5hux-GAxq>iivra~|K_^Kg!7>@l^p%>6Igg7)T2*{S?U>000HoA(c`
z)9rRj_i-=Q3=6s=TK}l++<t>;{~0E`&t6@BEH$?_{bzl9oqXt5)q+bZ>B1fzfB$+f
zd%X4SyV{M%Ebc$HGs@bOn7S;|tX}f+1jkDUHf}V$wPF7s^AnTx_dfcuQCFI0(Y*Cv
z`=h_82bG!!m9+amw~f90cv)4`su`#4Vy163+?~lF<e{@L?#SEKveh5%_dY6|qbbo-
z{`b&Q_Ovr{E!+%l_bz@up1s44o%Kk1;*Oeo--1LAR^NQX+P&o6#r({hEXhGD?wBp`
zwm<oO>-x_%`CN^GJp9v|-JfMff9|=Z+VrIV_WC7CUe^_8g<fg;E!(;_zk56L^ACrG
zmb6EQPi8(dtI{^d_Rbu=PSe*<zjVDTzWP(l`N`$ylC4@{UXl-%-hHL!XR4!JcA?^G
z!uI!Xy^mLWHauKm{Bg6(mg(sWgSnoH2iC5P*!OVmtYgWHVX`bsO|&*SK3jC~=*|06
zDua*4e{|>NOANm_m(_QE+T5gd8+|f07p+{eRPD%$AUTUWIks_WeJ?n7JpNPu@leRl
z+UtB%&dukV#{7oU?a$O#l8ayP9r(vQb5ih;icOPdit~j`nU`|WOa8}{XLma0ekeZL
zFZ=88$_J;d?l1b}GkJ;9@@bBNLHEC34=j~S$oBHIZtY<#D%;v8yfMDR##biDXXoag
zlf|J^9rN=3Gpq<yJ#5UoQdGWke*52P5C7N(|48|&FW|v5zhUQE=|i`K7WuA~TN&9p
z>3B$T(XBkcd*6Q^zn<zD)NCMECA;0Mf4=o1nb&+rj+$y_#<|Jw3dv*NukiWv{%2p#
zD0ZeaxZbZ_b8qeI2##qcJS?@MKF@N!H-1?dzst_i?)QV@fSyJLgCfsQYpjxlnB^_f
zl|6a_jE<|Fa(W}t_vB90o%lS5!%qa(ckQ~jz`EvJbDL>i!J-R{0cz6sv-4N(*s%3Y
zx%lC<mR6?u+^@dzWiL^3IxX_WHtMWL##F;4=hn&uU7p6e_$$M|>+02iy6Y7tpDS6S
z^5bUz<Vn$9pD$fh%UL{a*@iQnqH?AaPsb?Sh?)KQLxp|Y@5IzIX)_iaxU%u}?CncL
zQUh66Uz=OhAapBsUTTYo4%eM+cRT)G-d$gQP{c7RK&H_<f9>H58*ATsSn)O3Z@(Ux
zzo%x+{vUT|obzT*FyXIlmzMt6I%U(rl%Q!}HZIt!r>LfKC;GX(pYnA@jujmxQ=O|<
z$S>Tw{zvlN$2Fzu(c#a2XWX(eb5K><YW?m%gYN>D^Lq1j)1=&*^lDYg4)m95bzfU}
z$@J#o&3in8oV+EjghjI6GR;l(7P_`=ed)I+4?0yg)%!IkZa-09*c`+ls=W5$CvPPs
zlO;xPBx2&8+>tLTW3GSR^u5vN(A8hjX(Cs01*c8$<ZSKyvYEZQJ6ku`wLzwSf5G*`
zs%N8)cODSgzkK#jXP#XK`{!EyG7(jsFjL8HW9{zw)xUrDmF>I!oIPfdhndopr7Nq8
z*Y6Ne*5MbAEj`7z`9vv`iC*_4j*V~1wtW|S{xiv{LC)~$_w`Y4nT&#%zh1Z|EHZI%
zUSQ{~2fYWs-Ts+-KF4B3%cNNoz4l)}b>ZivldspWk<E&5+m@iTBkkS;({qO#{Ox5w
zWM)Qna;;!JzTxY7t11VV2Hw>fo->`M1>Fm4OibV~e^CAX_TLT9wEUd3D(8D;zRcGs
zoEm0c;jz+F=cDa|0KVUA`+mpI`@1_aXllb{=1B+gubrt~Xg$eJd){9A8Mb#O@!G$+
z64`q6$NnYdd%qnxbYX(l`*T;Hp8whSM5HwCZf)|TPo`HNB{R#u(eqc%zx?&vVa_DS
zob%B?EkB7Y(PW$>)e$~heMRNEGl%_IWdAeV;XlnH%;*s}{nMpu^$S{r8g#C-R=!`_
z#j^EaarMvYBHQ=puRQ2k#1+P`cz)*kYwUM3*84wyp(vNUbA#gg0!F3R`;UG9%D?i!
zk*gZd19N#_UH`JgMnxq`<kE_Wizg!*H;Ac6itRi8^HIp@3o>`*r~P(j*>!38vQ~F(
zY4!;cjhh8ORZ3Uix&L7w(*i94+0}o4zKr+y^puy?|MvPvc9}QYtRh(1uho71`~5~y
zz43jHhHESBu59?*xACz~X7TdaDkqnQw$wz0B*wt4C+~>Kx7j@E_pNyu_IKZvc9Hkj
zr|6uK*yVM*iFN&f??2w}W51X7QK|R(+t<84%W}3E7(I@gf6LH%!sAPN*5%*qEnNNd
zjAngWxpw}wX}Vvk0zI--o&J7YdZF&yp~HE0(|$cnwVpD2_T81=cOQIuBG1k8*}fAJ
zy0f0W@wblqcf0W8IVp9^GZXlI-Ikxc_u<KDk5ajv4vXio`RlN5z4joo<MF)PfBF7y
zlsEO%%iK5LR`Ypm;@!*rqE!`-PcM4Ik^1+wy>j;ZzxR9epPe=oHC=K2=@u=4P~U|5
z%uPc4(^}r?C!hV#u=!(u$cnc0AFusc`1fdL@P!7c8FQw|MP;doS@nM3`=8-CvsPFD
z-`2&!@BcWmOl4*?EnKnN%34Uj$dUa{-=BZthtre%64+n<c4U){b`o3bAh@Dq-)!S7
z`-WfjF{Z(OX|eMf7=MR2_eq{U-0(>w%%xKKrrU(*&tm`H_e(BKoE*H{_>+LCs>+-z
zZJR@9oZES`EMSwS*u^B@4zt@^|IO%H<K?{7=B(}aEBkNp>T~!XetAyPJm|?pp{bGg
zVue4IUUO-?QxNrh-oO70eXPbE9uAVrRD?<cWZwN}*ea#c$JrfFSv196WyPOc5@)a7
z)#W<!e5F;*F%PcOk(}EPF!p_8e=fSYVjGXTdDX<(|D5mr{(a<T?bJuyB{RAIGkgr+
z+v0Hj$LfdQyk$Zw`(wTw3-#9A;vX2cVqLv#@7m+n?+Y&bHoZNv_=3rg)~_q`TB09X
zGafTkS>z|06|h_NU--HYCLiM$Xl=GE^77mAa@U!=652t|K`OJ3#61<S&VReT&fsM2
z#)pYX5%b#rGn~k<^|ARUSJ-k=VrpWTK+d9Ra?d8*-4p-eL&)E0aobL=%-*Y|Jolbh
z_13*o2d?gI)UG{m5n6V<Wof0KqMq};sb5}482mAN99i6>HD&Grma2`qd!uGKrF@)x
zxTCoK6WeQ^MKW*wwk(+%XBjhj2@Bui$ej=MJ38M!=QuktVoLMsXFGB^e#Jh%^q+yp
z;?{@%46`IsCEaE=3+|1W_F4BmpH@xa3!iQC4d!iMsi~>BEc(jsg8?jJ@2>T~F5=%V
z;rVy>!*b(;%oCGr8E^XsTsSG;{wJ>J!ptB}?XAJ_>fxbn7b<o>oEtLb%Fgt-FLz@@
zIyN1ZF->xt8M#-dYj5X<?dDtmZhUzApIB7N884~t%V%Hzwsbw~+#iL<Q(JxJw?qZW
z2nC(FUY>u8m%VE)XYlOMCx)eUem}p6Rc0)maa1L-?}$zp=dTs^%Ver^5B@&XniM2s
z>n)_A6Md$r@!TKVM=vHAY8}?z6*`4MVe2bS=BtaW)*sX7&6pRu?sZOIgRcLL)X(0>
z|1%W19*LUEa5eY4*|zWBbl%<guuLUM;6Y=M1IyL9TbEx`^ZaA_xR}A{!>Zt(gEM~Q
zyy4#b>r?ZUADx#(PU>v_$`_M)P0?~<STO$-%gJXs0yJ({-{zZmJ5Fq|k5g=KR$b%H
zc>y<L+a6^rYED>QmSz2R>Y+Jr&hx&u3I1@eN#eEEuIEcNGkMn=uIo7#vG&+>@mQDq
zW#6}bd-Cwp@ASi7PAy*yuN!>+#J^zo!3}pC9|wn|F6K6vyJ`Az?d@h64X^G?SOqVC
zxvZ?}&a2fvK}$jmUpY&^wAyp|#kRTcI5*rnGCQfiqfYh19G(pujrZuhnSV9t;Q1E2
zhDlMnziodcA1R#7py2K6sAXF4<&<_+==Kf$ieIPqPQ6<w8s@e0j>(jHiR`*dCO+s~
zeD8Fi=H}B|JDr#6l?QG-&hpm%(KUOya4o0ZON1S!pIKNuvwYo39!pdIi?uWV7#{wx
zXpY0wnYkZVPKgg{{IEQLgEd-MF1ml+uibj*rnR*%%wBdnwP@l|oqU$4y!@wAr9{iE
zSN@t)Y*X?qv+<xv?bn6*%CX5ieBYd7Dw8=kvy(At$&SFUMV{97Te4+6U9Vd%bM<t-
zwDqUE*P%$CUnwTt%d1yRbj%hiw=TcK6PZ}(?cXZ@T{J55<hdo)$D~{QA8s<#n%V2L
zY}Que=@&LP=9}HJUHw4h>y&w-U!KLz{mLsI^jzdn#Vf7M*L7Exe_OlMTlTM(!~NQ}
zZ^tjkH%n~glQ-tuIkO{bhF#8rjf}UyXB|I1zk8GFna7uxCq9)7x;mw0%2RueqH8N}
z&D*&4S>;=?K)0X0rO~QFyN(x1PL650nH;cP=M8I9gXbUpzTP7ibLN;#^*^%0+e2La
z?b;RPb~pHB52+_D<1`9*wXbQq%5Da>C&$eFAGHNvzbaciv7|g8ZKkZ5f5xiMCtlcG
z`E2VtZF1O(o6~)}cZh90pmyZLyc_>4X9`blcxd$bjN)g32g3Q>c8T?(G9f`$o9g-1
zJ}gPxSmPBiW93z!HD|pHqm5?lZdvwgcE`tQ(V>s_$DO^h^M1?5^-f=Sb)FX2Z8~Sn
zvOKDt>0WAHdccwCPp8e2En>)Fmw5Omdf%}}VM~QO=CA7HW812(#5%iHhox4&cj|#3
zNh%L_JpcJoOaAfU(~iOXU#F`wU1Ll7w7gh)$$y4N{s-#jG;6<<=lkVu{qgiFF%wat
zr;+JDQ$>9p0^}Sb<(4flRGF3$_WgNrN6`{Vp-G05KkH;j>?p4}m45Mb&0%+Q`Igv4
zM|CUSY1&#y>N{-x-p#q1>8R0#!w#En2QgoWReTg5S<yN9D}P69<eVSY(`U_iw)kYm
z{Wo8AIm`uL)T$^ls4c$|Bb=b$t=u)!c(+T-6mON#*TDtL)@FT^OOn3TXMXTUIs44E
zhZQA{!>6?R1e}?krh52F?4+9+&ur(iy1%)9rkACa^Ag($f42KG|5QJI_eNwJrx#}y
z`yJD7D=#(KMa<P&uw+Nwu3wH1<_E6{WIB5OUEH&4Px$@_*WF(_>x!L0QRbQA>Po@&
zTX+_1OI{v--QvZ{lz!(E8;<#&c(UO^QJv)>P3@5DS^Tf|&A-t0UE!*!f!eea$(f!L
z8n!FMwkzE4yWb~rCh&sUp~Yc#N52YuS$ewLx$+rf-|XW88+Wqz<+$56{;3!G8Pt&A
zTE6t(e};Gun=Qg;zFzlb5S8|@*(=0pDm{sH=Z5Xq?)Uzdv5kM6+9bjJ^u*-oTvnqj
z?j5h!m0e@gV#!wenRCkbAbaDE9n5vyObpxZZrIFm?);zr*zar?{;ZRa+tYS^4PWLg
z&-5wGHb=L8SO3qDpM9tB==--FKbx0ubIafM-<foTqe^fJLx$-;MNW&3R(;7SrZd*I
zO{x&vdj9g`{T5&DIL;`VwW!B3tBf~3ukd9@xvu~Ex6c_f&uy4j?Dy!6txA6G`Muku
zbI<X0p0o^e^<`UQSvzmvbd5<8)AIB7CURWbpky|Y!z>}?@_w6$>i+t5Qk{;Dmo%?$
zpCWM5Uw{6)x_f~uXQqUx);dOg&*Qvx_;GTLqgP)}mrkwwJM-EtJrdI<Maei8oZ`GN
zUF>#3?#JBY7E{fVc5f(p_T}aAsXc53a(gSkE|D@-ImU5i(eb@&${84M&;7acP-iQ{
z^kv%?)}G&7cY#%_eSL*Rd0CIegxQl{i78!wo4dcfI=7v(D?)>H%hRfeo$|2{yc~*x
zudfm1mbkJwIw_>~(w#j0zaQIXE)e>&*U};}-|H|(YPf$yfr{4Ad<oXxIi_FU=62ke
z&AV^zhL^%zlZ={7_wM^*^0{%PRrlNL+t<`Sn>N|2W1j5E1b(^y4A%b{WVD_)Nv<lt
z*U}a7w>-;zlR@!yjVGH|Z`766zF&IHZQ{Rm1}>|$ikIwIzV_=*rF$QrcrJb5Cu8h#
zZrARYEVr}o#C-qupP|N5YuAaSzx7GCofbN7HWA1Ql61J_zCt+r^ZBoGe?J^iJzw^Q
zb>(}1>!Ljxp-0rz3!)uzeHd9{rpZ0e%l(o2<)Fxt#)tY<7r1KoJY40Ww72M<$fQ#n
zkF}gRpUqu8&wgL!LB1f*u(@@=qHLD1n=Glh^eTDTy+G^L1*|)Mz0ZEQC@DzW{rZm=
z94@CBid&g>IU6L_p2&T7C%Vu4{g=0mD<<_U`Ezgj&edN%k|u9=zP~;~W8z~Y;opw;
zjvbNy&#*r`VMVBO_m}tAwU2i(&D<rMFZ0>!#g+3bCPn0DR|{B2?=XMP6!O*3Y3<v*
zs;Wz}cPHoV*>|49lbKW1o8#c#twp<;++XOopI18dM0$pvec9z}(uR_L!4;l!{ic)#
zF+AX_*kM2a!}0DV0r%&w(!Q#Aa_Wly-jF4h9A~<C4}SYDZ~g5M%K{Dc;Meh4wY86r
zc{woNUcZEKRsLFLw<Wh{{4m?vy5sq;4~7C~I5iXNlWY%~?U5AXy~U!%bII+X&-(o<
zeZOzFj=v+HBdDdQ_WbhvOLD2HtS?3PoL{qvXVoGtt2e)OmaLnwO2^1=UvY7QGJAOP
z?FogR=36Xu?AuBXaNN04&1?Dd=s)&NKkhj3oD2$C((B}%xsrF=##z@Z{xdv$QW@)4
z>^kdNtUJ>#m)_}@Hgm0vyOrFqZ=&|Lhb8xZMOSwxD|i0!eJFJP$F-%x|5g@Ma@)Pn
zzSd^;BmUq%qm?Ho8gfqaP(QZp6zi#(pBh)+JQAY)pW(<lkM6na;xF%Pxp3jK&ZqP7
znoE~YnSLoy)o`j+OiQKyQi%pX<BI*qVuUq=+|qPIO|zq=xULu!oDc6c<NjbSRK3}%
zfHkE-z<5RAot}epO*zv8wxs<${$t^*Td5~rB)ycGt$Y5pcF<If=aI|VzJ29PE)DJN
zKK}I71La;9Ztlm{9e)aD&+wk8sh`BX<BoM(_O;(`w;6wMAD)ut8>i?srDbZGWbl86
zSB@%emXr8WZcpJh_;~)v<$2Fa4$ap;wJVY5;^okm<&kMqgM;0*c5iknOJZhzmbpQ}
z^rNHo$Mv0>XSS*^20r+;)9mK_Q`}8|)E;CWR!lc~E~cutjbR##0b|UKU9mSdyM<1z
z@30rBnm7N=PV>%4zsf6H4;-Env?g<!$r<S@>HVIcmU8)ZTPxqc!xQN9C-TuMN44_)
z3;bs3wX0(PEUxYTBlqBjY5M#ojmdhe_0r64djFW@f8-^XcThjC)}Dx0GUw#xEm--{
zd8N_3MQOizw)Drpnzy0$$qL7WjgureFINi|7S=c&t5f)>x=`VKbeF)JH}{U8zNT{W
zkL^LvkeeL`76$E8ShLOTrRqsDRkk^=c`~gIo1Wh_eMMHn>y~pxD;`bKIX81%R?)I>
z#USYut8J<d>CgW@h4<KVk&1J7{pT$THCXxQ&cxUQpLZ_toqO|+nK-*v<l&U)rq*Ds
zJF#&>0g>mtg_Awn_-8+_JhNAIc1dbl<ot`5?UF6p?OmsJO<;KSMQGBoFsnvQ%_+HC
z8;{1EySUbqWv|qc9lzgv&+6Fh(_<&+aO~mpR<B4-evJuBZ+ves3tYTJ;oTFbm0F+9
zd|bI}w(>`nz>w6EgtMa2*{u&bLciv&vf3uK*F8WbHa9y-XvdwW6E@Y>+?+js;V<Wp
z5|KN~4?Z&Jo4HCS*y+iRtc}ym7BW3s&vSWIm9oLBcN5pYd|4zBZ&t6gXx-Y>M$P87
zo~0#*3%WdO(|x{$J-DAe?cC#H;n=yi8mbw%yXE(Y#)x-#9q^yt$z^+?V9KV>tUX&b
zo^dV>cv93Zsp}UiR(?g6)A!0e^^#}VD*akdzZRZ2>l!Gq>6}Vx+B~5Z`qJy~UUEIh
zvhIS<kC=^$ckZry=AECraE?>Ry+`4~F)OtlcxS5coD98mjw$4urg@=|No0J`Gr{z^
zhL7r^-LFo)Qn)u)G-I9N>6x0b)7ml~^DNNRN#4mKop~VqxY&ZHe!QLXXH8x{lG7G#
znSM9vht-)SYbPyvTC!yCGgYsoXAK*aE7qM1y%Osf|M0uhb6evJJKx35IQ}g4>Xo0}
zYjQdMp301<JG9p1-EPC&nRx}1D^BfRBzw?))wJdF*FJh^QKKdnr`>U4t|8a7$V^SX
z-)rsHMOI&7xpDNm-KkIeCiUsf`53)kyQ*Jv)$;38+-C%5naD+Mc2(&xJs85k7$JM$
zN}0m+`fF;O4(-ax%fqU#SDgK@C*@7$pZhF_I*WzsPXt99UlHJMEjqc9CD?6?v)9ek
znpnxXKVq+~mk+pl+}1AKH8^Tw{Y+;rMpcuKlR~vl-En(%D4RP!yHzHIrGYO%gYSAm
zeZ9xL&f?TLE`5gWm6=If8T_0UKaBENIqhqZ^OM?n>Yvt4I={#7#L7o!tfca$FQ2t)
z>858<T_r1*Pg7o%&JktI8^7dc&c57ru5SvK_dU^aI=nou`sG%>T^;L>eOzPCab&W#
zh}s0<xre$u!Vk$#ofVsvJ-Jrt%sl?Dwt4Q;14^&vSoVe}zS=bB-K>nDEHB6Ha|JUU
zKVNMWI#9m#-18#E<9$i<JH^(%|8z&NUSpZ<u0!5Ow95HBJ%U+JE4pa9J26FT$}_F`
zqPsMC*^58!=T`r^e)vCwkizND&d&Za8?Wry;OiXd6Txw4Zq{apHz&OHJ~BnT+$b@j
zV$aFq0Ip82=IdE;i*8q5e000p=b=SrM4jd#Pro8ZW`l_ES5-n&8@QQ2ZVqcxefHt|
zieJs<r+2MB>~kUYZ!*tX=NT9F#_)R2ILQ+}&-iyOTgD^qg-*9hwoP6k;LxWzDPY>v
z)EOM7r(8TyvOBP0<LaMBG-FpBN?fv@{piJ6PHidLTANige)u1Fv$V$bF4vA^v7XaP
zAFeMqw%ntb^@=mAIACi0(<$EsLQV!Swne0VZaTI}R%+722ZcwcZpfUIS#;pYicO)R
z7K>J{ik*9JiqgYHtC^2j>@)JJxE17-YH=?1lHf;Ho^{b*X0<k#D=8b#=d0@0IjLC{
zzW8&^<)y|p8>QcTE!|S&&YI9Ob)rf^A6sXA!ktF-Y5y4t-w6s$V03yhDKE#IrG4Ad
zxvP}?*tbSR?pT^!ctSY!tf^NBr}tGA)uWdmiGA2rr*dSy^R*wjkNJH{`2w%S@|w9n
z-saEh5-O=Rby3Zk<2xs@Oj%MG^6H}C5|u5p=DC_n8S=%PoTSd9rnELG<IkK~24B=R
zif_3Qr+Hwb(MB740rA7jrffOiqioitcx8$4yf1+|LJO|;Yp2|lF1;S(eOaoi|M01k
zNs+Fixy~n<Hn2>Koc?WIBy)p=?mj--&KWaVSa*3|`xp9a^WXO5<ByN?Jv=NnsmIAN
z=OlyJv&rr<H|9r7Uh;u`&F6EA*Sy^SZ_34>i<i9rGt|#|Hs$k)m13(ooo2qf_@Pu|
zZ|~b`9<e$TZ&e+&wSDWCO=Y+o(QE(Fx;?R=^O#Jgs`H^1k*@8BKLmWT3w*hr$4YMT
zvK8&#70K6TDyOVBK5G+c%cC6_7~pW_S>}_86LlIxOI}~j-@$3I_rd;-9p`v8i#9kK
zU0KA{7hc57rZT(IY;nb7*{zR_HeRl`K2~tbn1MmEJ1k&!KbKI0_pIrseogkc^vX)F
z{ZPHcFXN;0Kl(qOa##GKTwu4>%R3wI#T+&YV4u;*HsfLcHSRNx*%KJperf;qzs09{
ztl&`6lr<eXVT(A`{>lF|ed*x%`P9Q7pMIQYI8tDDwBX!t0q%8KH@w{z>@3v`u?*ti
z4J!XU{YUpl>D9Y#AKi1ds{Qbp!|VR2Pt%;fv`Slb-Mnt?(=TW6tM=azOqtuj<i@{w
z<DIF@>_^T_OF0$B>8tdg;ZtOqY6{bGb@#IDIIH~+-O~(Y>>HOx<anhdH_CofN($Q9
z6<}vzeJN<-uORnBv)Oyumle;K-F14|C%ZZF5At<i`%OQ+h*^L6t5-LR&a^d7`<C&t
z=9Q_LxO&Vw-;*kzJ1Q)dyqFauLxUI<<d<*Wxw2=!*nw}^j6Jh^ZI>9O``RfhYR<Iu
zFOTqf{HCSR;aYcmu;)LcU0Ji=&fMQA|4{mm$``&?|BYY1RZmSdG2D8RbFsdtX=%{=
z!+ToZ-ny~v(9G2gr>0HmniRt&-6|A6BRR;lYo0(gbKlIn4?oPfr`=iKm9z75SmmB~
z?}CKATb9&3W?m5g;p3F&A(hwTD<;qS@$ixKb<S0<KHc2RzC28byCG$HS9*5u-taeO
z4}bLi%1YB0G}P8P*tu}`Ebd&#wNh4g*8dr1r9GV5`f!HHgAI*36ZXB(TJWD?)s+VG
z#aY>M{~0QSoOM2k8H(xcIv6Cf?P&W>mfUxBhTrAWm$xRS`fNDGEnar&_iwIw)qgTn
zHSI+21WjEbpUlSN)4#~!GJj~Jcw+XuYT3ryzaG8Ib7`D)?)>Vxl^6f4ba=$Xs9>=?
z?!#uQbt`>m=A`iO-<7UEZ~li(O+{o*m)WwZfA?&;#ClgOH~0G2&wa*CJn5&WUtMvx
zLAJN|cb<p*u^%j3txlFLynk`Vr`1<xbve2{n)3Q*qt7d5F*Xx3mD#cf-&O9|FK-dD
zUZWtUA=>*^%O~~A5s{uc%-6jS$fZXzGn~`2_0rL?W3c|dcfa-nm8A?zeXg#ot@$oJ
zD^qBqj=;B{hclTrzv405$i=z+>-XpVb3cU6S;DH?xOm#~nKy1tU&-RUbQ*8n-UY2H
zz4QHg6+5DP&vrg3ym$Le**A-%3#}>)Pq*BjP#>M`kyhZa{Q7#^#K}z`cPnPR<L$b4
zw^90Yt=Z4_i6?^%{pScAe^tA8r^}Yu-m2wSqAng*w#w=0E;=I+Up>#!-v4n!=V~sc
z%=*QZHS^tSqh!w3nm2a*x$AOWg2Srhm3DUS@jQc*lb>3v-Rb<fv%F>bufsk2*Gfq!
z)|{KUuy?|X70sn$dplX)=vPbUe%xpL$9juq%=y!2B*UFb_r8rc-`W0nx1QMIpQY9r
z`RBbZyMOIvn{3py%w>}1%2apNsh5sNY`FFK#@p@3vmg6xX*u<uLHPPy#=4zFJ!#5p
zI}?^PaV%tCyYP=GTR6{3f#jmRZTt70-`n|k?%Rfym7M4QzC1tu!$-lQ%!t}=w;DEF
z$w-M7%3)->_k{oZpAR=JV;#Dtv9<15EBf+~;wj%>FAko0yHm+E>}%GYo4F^yThH;B
zvOHjIu-Aq!GW}oW-bKx|Q8J(I>|C(quA?~1`}_Bwe>gZvXz_zfYlBbquf4t8vr*-_
zK>QXD_nQY(m4h>!&wqV%?{IyNUDUDka<gCe-ag-}xc6|ydzs2FYowlZ%-<tn_V(6)
z2Id5_?GM{d%?a`I`uBb5oJn!FH_s?aUFq_){+3-+#KEf-UyEk3zPYx6eSYu$WfxB^
z+u&ic!0v0-+mqW{W=zg_G3VDc;}sEaRHSbwm8m!1|5vD6%yhn?DS!7n-t}+gjG`Yc
zU%JnULn`COl1`f(x4l<hSu>T!Z+rZ_{;=TmPSwhrjZZ)2%??VODdgPsbxF--8;9Lh
zAvepTGZ}jO^ZtB4*mQXg^YiBy7Y7L%Id1*;VZQZgMxV1M6ISi+PyV`3c%PQ{(rx8(
z>2K@;KB~UDTX(@RL}WD!-~C;}?>^_%ez$P8u)TTn?#q4CE-|fC*&kKQQgcZmSuw#j
z$EkPA?7ZB!Uw%#QSF1~5WlPfg*I@SjOM9p4rk_r%E?F+AU7KaSa}UIoef!0qQ&sY5
z=FBY%F5bzDzCSfj`Ix6^mUCp*pB3{W_pC_jb$v2nmBLrKH#yet{A|09Dn<F+Y+-qS
zqSSl;DSmz4nK|vn3yN}2*aRA{7pu}={&+%~<x`6rOQ*yz7x1s_T4=S+`kUE4gE{^m
z`CA`OIW<dUQO<-AzpH6KH>}vv+t2gI;?aylJS*L+{vEt>_LMQt=|3*1zB8|-nut{9
zgc=+db~EO=B_S7Zj?vDkt%SK+c%wCcP4Pd)A5RsHPEOQ4p}JY?%MDAvuX|e`FG#Rw
zRsXJe#4j?*`-xKrkLDv4reANg%DVo{GbpV~oL$@2y6O1R-&$8r{)_fWjmV2^>YCLS
zQvT_3WS_yA-P1G{*QhCf+~0B5t7pN?vQ4Gg6}{1A%w7_EC;x~)oa*sDE$!@#CF#M|
z21PAQ3yj5BU6>PR+&)!$&-iOvm2u4EHM^b~zge|ZrR~ZGky)&1R;4yP6D=8TZdt)u
z5c12<SjK<5jlWpq{c~C#Hzr2UZ=LtnLHm^THt`Sl+aGcn+L&|SU8`I4VCxMHO~<p&
z^F^m<^J*AZ$*owuZ25`eUwyM`)<ry7zA|Q?#*vU&w<KMRPu9Aw=N8^9WcumIoF(^k
z&5~Lk)KnX=?AObY>E($?Dlj(42-+A_ebGd~tyDZX%`c@bXDidgJvs+W=Gx3Wn;!I~
zq&EGMq*{L5RMD;Oh6^fLWUakT+E46BY+cfqY9+Z>!R+Wx(V~xD5vr~;b@M|S-rcrk
z*fx<}pvPv9oWcWjO-@B-rG~Sno^n<jOIEU^F)iuZx}fq|?3Ks`r#Er=99MpA_^&W<
z&U4P$Q6FcNP3vU*(o(Q5AvC~~;R7F&;fj<qjF-+Gd-~<GG~2xk|780QojBhaC8~a4
zX3;5)pZg@+XVfg2bZBu#{gt_qwrcg$E<QQ!7<p++h<4=sLsuPBecH<8eDsU-p9&qw
z{2G5+YK`x$O`G^DVrQPTUp3QPdUxfoJ?;->M70mCEuP~S)s-r<<^2R3FCNV~`EF+-
zkJS23yzpe^_L$Oh)mLurtqF}%SkvdLe14f?-DwlP16c|!-ZvUQ*SJeOnC`JJ>DiWW
zrP9^i*>W?FJWEP>xFb!<D=0W*)`kAxjSf<QXJ0)!%Uk7dd~$P5_X?}mD?Vvhm75yc
zvIjX`>6Y2J_T5y0Y0RFvTHD2U?mc$k;bx``?iYV%y}Z<NiTBut$xj}o7TnRfY!v-$
zO0kgTQs1)j^?M)8m5#k^8|pacNAKLH%N5Q%?0iu?e|vDlRT15ks2%Hc)R}!xeR7n3
zCDGY2qu$eM|MlBm@y~hQvh<{JrR)=K53iVXYWkE}Iw1?Ao|>j#*ej)E^lsC#(?(ji
z=FNK1x8RJ`a=kTCHmlZd+MsgQXq(g8fPdY_QOCn~_OAWDVYAmXul_?NRbS8bC7qT&
zFx6UQ%A=1QKAoLfQN<U984D&x@kOUG>1}YFQW1WsGOm2awP&)WCMs9IYz@Ak6z&{!
zUElA(TvzvXugapW-f>8XSDPhuC^f$Na%uL~_e%mKFaJ1aT2bjK#Q5QyN1%qLSnSNR
z5?!;TkEUt)EH?kopkK4x_`H?*;;L&QE7ura`k8+@@{5aT?`+?r##^`l-gc>4AgYk7
zNZ`Qr*!rY{=e1lnJwG==Z01X!mp>GOrU!rDBwED$pWzPk6>+JK>zn)*dg+{Au_s{N
zE4x{%yEQIe{->If?IFf<VsU2EwwVl!-dX1C4p%nm-e#F?vR08R@Y$Pht20-M_NWU!
zbW52TBP%q8=lSfGnKMtxSUp{Ht-0!AH%If!ozg3;y3JnQ+cn$Z!}X*0CU<_0SS`13
z`Lgp?mHdkY@=8U+Lq3bV>sh<=K-%`QnO1j-o<yDfb9B9!|5Jk#2R>Xdw2%or5p*+D
zUm$OF(83!xnv~rCGpzWzEPu}Po%uc`Hd`vggMxdGAC{c8ful8R>(V~Yh?_C36C`^7
zW-hx{{m{Q*!P*;a>()fw{`*M&KZE9p!w;XHJ(lDcC-LNH!>lLyoSdQx7ODJ~;vO!t
z*8Hf|w0vb%{=ZEhE_~VF9vrePcIF;avz-2we_3BMuzyq7)^>I4)pe}bnPyCyvTNRS
zvD(_CFyl>=1>0u-XYf1vea8Z0#iMG8Tb%!`3f#`)!KU)*S3dj8UE0SZe@$NV>;9@u
z6~)(1|C-fqbmqdAuc>!6jk$ss2V8!|+MaPoz^z9rV#dr>fuh&8tdIN1FWLQg)$tD(
zTPEs-9Q8h{>EW0#fpMm0Y1ZA<1!p66rvCMw`6K^9Uhdo<{g*8xUitJ{ygVs7<x@~;
zs<6P8sV3)zW_t=--JR~4djI>jG+E6#TNTd*uGQT>W3lPitYh_}hn5!BT~y{hq^`Yd
z-t&h6A+23!oSJ&4Feb!gE_Hr*y)(A*(e}2v$?=gBLmjU?W7hSk>R7b=Y`WCGlhMq(
z?w{?Pk-c4c+TM;{ts<9yFa2)aT2Ooc>3f#9;i(qstdBmaEXpv7ZDtB-xx8oE0&b(j
zo7<~7KC2a9lCw<xIx{}J)IOlJ%r&a2No&uydb@cUFSG6*=-W5VL{V<eih%pOmgMD}
z?kxV;H+AJ72VRGQD`_``Y~`%pv2T%`Ggr4lMrMhRe~HV6Q=Xd4M|D%$z3zV6>o=9n
zHu{R3O4?lYNB4p+*hVuynQN@@+s=8GuCKByTgoe*TN74t`UbCPYC0giZS8G_S(`dJ
zqBsK<-)cP@b#1Yf*)Dgnhp|(p_B|-Jn8xsYj<zAM;VH%s8w<Ri8aJ0o1-Qxk{!?68
zv~OSYa&0Snt;q}1*NPNe36k>PcHC{#-M}TEw(OaBvMfwNXt9TaU}wb3OegP<@TlGc
zIvJ1VD=05K^G7G-pXARa{^8Gr<{Z(sbK0ckaOR<wFUN|NIv*1jy_nDV(#HMV>f(8O
zR=twj(QWPdX!-h8iz1Ge^grX~&a8-!2=l))L5M%cxWjL@$&@qRDh+uYcPcM+iQHYZ
zUienrW&M`ZE<GC4+l)@ADkWM9NuSX$RM_5W_c8W$|3`0euZpR2Q%^pe@Asc!dSJ*;
zK~pc6t3{V}w@5PmJ8>dpYHio4S$TSl!fFpa6gBju=B(XZsqVH|P;BnZKQb2g1oi5l
zU7ed*#y>$jVEV_@s%7h@S_jF0b?^SmZtr!u;`PIPfBAolf)`!7zIdv?YUFywHG-$t
z<%&DAx}Dh;J$ZTJ3CAOylb&Bz-hB1W?O9rx1*Nz2+27a;yV-op@A3Q3@KWrRBVSB^
z`))07#aC(yJFFN!2EBa$G5q1x*s{p0vXu`*b85X=9xScp2#>h3cIwIL!N!KFOL~5(
zJYn!WQhI8(NvhVFhU1G1pRZ+heJ=7x{L!YTt62|yJiIGlRjB(~XVnJgFRNZ|yu>@H
z{&(BCU-z~j(P!FaZ~9UG@Skg$y`sE)J6FCqDOT}kt>eli(sPv#9k~(Be)R02Gc%^1
z(pj>(tLf35y}6$n3pMvnJzVhQ!@A3lgxijKF&j-e=(Uz3SkTbQ)+6C-YkK`d{*_f`
zT7293kL1n&u=b|VqGj`{R=UhA_55n&s@1ZChkeQ1Xx@8=S&c9E2I~cjT3QFcSfuy;
z9_!)#avN$Ky&mqL5~mZX!f@KRG01y!*WR6~kF`(la`-CimiZ{X<8ONSqoqEIFE@3{
zZM`*bm&f|#U`@7=Hsj<48hU5S@67Q$we+l?%>0ik8&}IE#LjJ=wPE9u+>LV={+Z#<
z{?MYZg)7xZGDxIh^-~W^uc<rj1jDujzcAJpuX=BFKI_wZ!5XU<d(JvmP3<=5*KrGc
zsbuT!t-<_m?Y6hYoePUYTxM1}U!23&%XZ|w_>VR6j1Ma+Gv>F<wq$F(kT&=A3ad%A
zkFxeJu|G1SscPlol^?y;wk=(Aa`j8C1HZQm&DiQAzU|xn{X3rd&D1swR6Sam?R?@6
zcV8eo@A2IVVFJ?*oxN%-v?V|1O#8L@fqX^lmVNZri!6HiNA#uFP4BFzs$DC?f<G?p
zC|CRY^GKngrhYW9U(Y#P?hIQN_hSX?xXu5Ze8lZ>S5fy93&$7!A0;p63x3X9u;$gd
zg)ij#U%Z>LK*8b4lx0?pK}T2wCqxK-@6J-(##*`KdYOHba_2|Oh*FE`AN96aW--6>
z4m)<tQvR{J^h1jrfdh$#!RkqB@kVnkop)adJ7AFe_!gILveag#-8a8m7x-_xyKlb5
z?;ec}p3E9|s#C?5wzYly^f1@QgENJ5;a7%zhg;uQ|M_R#BPz^Ts_C$^`WKt-yEAO-
zKFGZhwY~qE(P(#Oc)^+njQe*ypLh82-aeIulVr=~?c&m}7epLx;PvL)SHS7nJNryX
zi1c<9{@ml|U;Z+3I>;iLru*&7#lHzBJ~CW2us*3daVq<)Z~OlK+<M;bM&v2Rmh)bf
zhUxj$0{8M*RM<E4&-COwxzh1!(y`-z|1-?}aIw>?Xvu1gT82-H|Nd2USaCMmxhPpf
zB_X+fPi}Exo_pc@kI7BGGq!BvygKjD+WNWMH(Xg+5K+6hEwp%PM5NQNJX__tYyaMv
zocm4YrKiKGt0ww;TffMM<<$w@xF#*hoyn@=<e68Ya%WzC{jU19+X<gT9o8H+)Gcz%
z-BSC=!&&^(KU;;o3#*llRvXx@mS(81{vG{s9|O;krX_7GwytMi9Q*yk%%GEdgD&sf
zP!3+5b$5iibT@9hKTSvV5hGvr+n1j*wB^5c%=*u;@#)X7uCAyFtPRN;MG-O4^KQ%i
zc(~cn{;6N^Bh{B_ru7pS{(bbFZ|d12RwAYsckVU+a!Y*rv!JIRnmv`fI+o}gNI9MA
z;cX0ZG~392JN!y=_~9QfCoSL=&~Ur9{_6U;0G%=hKl>*l0ZZ@xd0l;M>g4Akk(0EK
zYOmhR+quz+(~PU~)&l|azlJ;gR4>oGV|Zob>!jPqggXmXZsHbrQ1x>E6=~P0Ns~g9
zo@_gIlw-sF`SU$(>Sb(vrWt*2ZLRswFl$=6U7*L-C1S5<J9oTdIdfD>*F}@<chsHS
z?aVcmLS24kE51LCU-os2`%HC(B=gtTWTmFKmDC1_sZadRFyTY_gSkJBFIjHjAy%JI
z8Sfwa$i85$#(ll*4YS%d9XaGa^Xyjc33nUi{_g(v^Y9hMxZ(}3?pJ7~w4Ytjx~K2O
zf=aiyD>VN;@Vnjnrrv&?V5W$uw?(~UK>m|kbLFJ<qQzgYpJH(4g|yMq<;#EPmc_i(
z30^b7((TF`4)^W%dm5Llf6r9MasFF(<LO61T_Rgm=dH9}(!kfkJ)y49zU}ev&b9=v
zum^1c1xI3XTo3NBKFYNG(7s74X3g5wxp_{zQD5o0_>HqNGoH-RbSqmjOV{~=1BaT{
znxzda_hpJDe}o@RSuszx`@=^6p6PrhnvAP|t#WNuu5SC|er!?HQa`UWm7aY%2NTU1
zbhWZ~uQ*+#;VIJa^q;i5vtG=Uy}^OstgJE<y4X+6=<JzaasKJGx@!)OT2ExHW*z!=
z)aBmsZ<`!cxr4e5IA?xn$hf&_!lLghl~nxSK3Taka<)QRRHM4VrJHeqr_34WHeY;X
zum8es=i+6jGM2u6vu(rWYkR(m3Z+Hv5xRE7)WFlD@j&#1SwXiI#BU3n+8wK)S{!WE
zsq~KX!jAIln{(&5oN>?kVRBS2so<#PBnC+pfyoXpKQ{TwUin@u82Hq-+Lx8pHb3H`
z$Jbg%zRs7s&S~7^Qe!<6#QDgji*d<fk5bQ9GWTvQv)%lbO)6!}`59ZLJUct9sbR|Q
ze(oKonBP9NdGy2R%2w~g7lnFxnjB-9n_4wix*p_Ql#!fu%0}s9@(S%&=RB|7H4UEZ
zEfa7#=;DvY%U4YeL&W|r-Z(GC$fYZ!<?jWFd)_k9r{2Do5$elv2<lwuwRD=d%$0&C
z(^nfTOw#Q;^6?*|mWS~YlOvNtLX`y?f>;?iJsDkg&-LpR?KzZHFI3ciUgOvC)x|aQ
zmN9F0_WulM)tFvy7WwbuQ&ZQ&wTH8Bmdx$bn*L+wQI?j~dpQGHZkkvb@6`Ls|3vPW
z$wLd3cW-O#`KB)Qb)WMz-DSC2LjRxD#j}i$ysBGxX8NLO%|}*QwXNe0*0p-ow`hUp
zj;M;mvCXQd@3b-Ge%~@b*<R$wrYXYrr+Y}w2+8AMsBqrbyRz47x8>m{b?JK#nwLA|
zP58BDGZU}S!R=)gN3D(@(b?8)(rr9{?IydG9>&b#H(xKBw9Qh*^Ut%uwcT!SL=*L#
z-KEwq%&$rMICs_2B&M{pK~o+c4Ag(PSyYPY!G!EI!^_pyH&xg97xfq|dFH(<l7(r`
z!W#?=v=%UHy8P%iiTS*4*N6WMy03Cpt+fi)Iyq@kM`&82M%}fWcVkTE|4>|0@XWmR
zsot7Zwl3Kl#7(*n`_F8hYvL=Jbzo(0(Q@NYE3(-5yr*^9N#FdZY0kDpt7ux#niZv~
z?=CF;<28+|n6;(BQT_0T-52kL|5Ny5exk~Fv4+vU&bkffv^2xDS1(<3(Q1*i%7oSa
z;o*`Ku3BvTx|nH``3h;CKwsfyXHW5RNM$(-a4y`MQzF1q|FB&AXrYYD%^mZeMR`VO
zl}rj@>Q;(sa@q1pp^3SpBW}am6PY!7CyVvEPy8`h({FyZS1#V|%rc!>9=Db`YNVTa
zrlsvj;Jm=FA(}($w8istc5Jn5K^~&3_eMvBsrx)waa3QrV{&JhrmE1~to+{#9T_{A
zRys!>DDs~4@!7p4D_-~uu2}V}ds!7%T-T(2qpZ#Lb8LF9X32>6UT=6XyJ0)ma^H<g
z4s$i9&T2nA<?5=3+pT2ftgdrTKe8j_$V0;&^ZqkfMb3EU=ICT8>$x&CY3`%w@C!%&
z%KLj7EU!ATYGvu#Q^$fmoNu)5y77i%yKHA`kXKvL(X+>vPkr>CVN>jtLx!h9Cpms*
zJt<e6k(6w6_EPtoEtOVRpGY1{wK4g+bF1-$Sv6Y%g^w-%HgAhgseFTh>-owBr<b0e
z{paml(Ld&FD|aqCpL%)<rx*WBJx-IwT>Qr3ciIhJ`b|8qU6nT3=B%;It)<h=SFKpT
zJZQ?*jZC*L9#mat`m66+?lEokNh@m~MQdf5+U>7>Y<^@(^2Qq}kJY?6r_E;yniT61
zD$%QTMXLLN5m(N(-3vG8ZTP4)ec5NOuw+wNFSjP2Q*-ZM=e)41Tl05EZl3F*#wI^5
zR*irkQL}rwKV7=g{?W(fWsRo<pTN?kVF9ZZ9Rhv44gb2>?=(z!{jY!ZM|QiDyNnkv
z_FM2`-Tc!lmrT>yx76nIw2+&ZBEH@@r=;%o{c^!>9<G3yGrecTsBBc)w()v>_&u%P
z#<xTY*65s5KEhw#aYWNWw0Zho_m(A3Yj(b0mR5aq@iCuf>#pU?cCj0u*u3WbQeLjW
z>**#FIa6l|FL4Z3nOGI07;4qAk?DB#96p~flhS;D^zGYXw=eFZ|Bw2k1xJN<%vrh8
z<<C*iX^RfXUuzHzc$c~K&4thF_SnBVb>{T4Jtn7~Eq!&$XTDB^@sYNbyt%n73pi5Q
z1H&{^>}KD)xS97u=4Z8vsTRA9r(axAzbZ(%EPtzoW>I=!<$+TnPbX_0JTbF#)#Y@_
z)zy|vdbZ{d?|0jpUb-uP>96hlr<QH`)2`Rgd~((J{F1bv3C`>Fr*0Kmv+CWxh<it;
zPnF=W3Ay`tD&tyC`TgfKc5KhCkkgm8m+qOeWKmL)4}+a&pHx<c><QMhIyp(Juf(zM
z5A*mNV0CM`>VkFWmrhohsP1dYmi1|BR=LVeffF4Iw$6~2doFaoV)K;t+xL_6b$`0v
z=lJ6(A^21ObnCj9h4%vfGfWE*iMZ1;DI##*S+PBzb|1;AdAzo`Z}#Gsd;T-1OxF0%
zpm@&cR9W;>%N`G|P#u@qn=7ZQyiZrIRTg-%R#Dx8?a!W-B74l@Ib&u@<RnekOPG1@
zbR_$qNmsI(DveE7p3|C;YHNS_pHL}V?c;Qplb37bPO}BhauQuQ-*)-cl2x(h;S(}d
zHr_FcZ@(LA=o<TEv7@usywWc#uj!>He~dV1t<8V)j>qhFzAaLkMpv#nRyCA~v#xN-
zjrG4ERPw#K^Rg|sTJ$5wrAxg2&0cv%q0jBCX4#~y1hX_H*(~NF(T2B)UeiK5j84tj
z-1IiK-}=vy;(z?NHg`S#&oJkaf7=<aw52Lb6yv$VO8B<v>3BRB-7{J3L)qd5Rq{)U
z_S`o;wab3(YTsO)B8{?_J#QH}pBkDwKT}%vgt^r0KSS{J_A`@m_)lG1_vJ!(#<oW`
z&Rdu}lL|klww^n%Tr;culJ9-zhx+bW*YxY;F8^n^DjR(5p32gc%V{BAH`!xTu3APN
z(Q46SDgC%G-`S|?8`J&DiU*gOu1vhUL{!C)?MR-P-TsJkW$T51>d48rXzI+I|MajA
zx4K?oWXdZ>g^>NPo=R?5@!`Eryx6a%^@n{nZC_c$wY1GE<P_JzD?61>nVea1JKI%Z
znhc|G^+REeXYZa)=j59kW4wK7ZozH01zwLX?^kS_-mN*c!@E`V$C7iLdXwB=2C}v-
zH%ba$_p4d+oWJjn>W6$WbN#2Mm`r?jGc7Q1TGp~xIon+S?v=Gj%DbpMIf3<E?w+LM
zArqzu@P-EZmdYLd`Cf3h@!|A`J@2%Shu_r~Sgc#jbTcST<%gQ)8nKM`Y4sh)+QTmk
zKbKSZ5q@Ex%=1kZ*3+Fmem-;LOP|8-b98A~<fl*5gI5`xtvK~5rtxKrdz@tLVF9+}
z;sR%b7rWhpxsP3Le>mfw`q8ET3SFH-F12oS@D}kcGYS0b?4TVis&i=dAJ-quesj&|
z{V1!6dU?Z`wPw3zzy9u9>DRt2J1~X)3d7}u=<J&X#)rcM?`^$t`||Bqzg*7=lm&0)
z{&P=S{HNzBZ@tJ^S;eNq+j7swG%ihA{3yNmFYBxyol`2}ugv2<IO|Kv)QyY11E=X&
zHeK)%ojGZ)+tx{HX{ikNy>2HJyu4)HV;Pm@aqB<B8}=LbeII?__@AM@#?PNCUh+^O
zQ|Cjq#(bT%DjT#+Gy@p4-OY8S-O~TbZPIBz(ldQkhE=leD}Da#MSWcV8LT9dvzTS)
zP1`po>dljeO)+;H_gg0#OHK;1Q_#5fq3)$s&Eq)_?Zpf&)(IN3niVZgdO9iSsJmH~
z&y=+dqHf2hr0uh-Z<{y!qv8FvRp~2@>?XXr9kpoILQ#bkldj)6*6wY<u=<_;?V5-a
z5m7ulSFUSa_j_;KZ&T0S^oQ+_<+NMszr9JB>NjywqpCxtMmuxJ-9_7-<=cMRJ3jeT
z@qO|aU;mgOjfOP8k1tla{AW1y<b~AIfZFWe*Hqr*c3SY*99Z|(*<^$3x0_$C?b@q!
zT-?4psbb2JZ&4{1+hU7CFZmylJgxF?@{)j8yJjD_H|tc@G=H8VE4`VodQKU8`$kGs
zoe)YBn`9_^Xix45|GWDiHhD~%Z2wvK>^6mVwYH|?`BrZ0!nxk+nG32E&TLEROgEbB
zqPZowtCibUcG2X^brz*<kr&$!zZd+YID4Jywi($PpXLUI9dhKF%Ir4bK*6EA>v!(7
z<BB<>bI>S3QQx2Qjs<fz*Sm`gAIrDhko@EPZ|%X4{~4xBE%i>A?BwN_uJB5G-IU(u
zg|BV$KD<7B>&@%~Hr<P>yp_)$Dcvd%n)WPk4L92)A(NR?B4Q@*YTO}l<HPyGK5kaw
zf1QGaA1~IdjM}ED>@B$Tp5^^a`=8leA5+e~4S6|vnP*AU%v|RM!TW;0Uzl<DPxQ<C
zQdQ><&)WWRvd)>$HrJ-bZ*`BIEb(gUWAXj-nGCLo7q1L^{9{`E!}k)0mL}!&bJtB0
ziHK^L{(N(U#LK-6kLS)hP*}2j^J=evjnj3Gd)EJ0(Hy+du(2fJ%%?N+joO#)@o0*h
zeoip+wF0k>P*&)s`OX5?4Zgko>=HF{I$kqouGCp_-z#ikyKz<T;<aV1LCgJ|HqHo}
ztFp^iMB+{e`=>i`+?VX6QzYxdgZwA{=)8Ev)>)u;_VQgDQx<m{@4TnFlHsUH`rcbg
zjyKjTRIChownDSmXnKK%ZdC{q^XGpFHs#wSR?cAM@e@i~D*m5Ar!CUmsQHnnoHs+8
z!YzH-;*0mVp6qoO)$jNx^kd~e>CZd=Gh}+tNfAvqKCKlo!TIR3&Eh@pw-gx{?$7Z#
zdRp7PVyDBRsovLTUyt4$bWYjw=DZ8vx`kV`A4<HG@rue=SsZ-V$K%FR2G;0D4)PI8
zN)tcYFR3b>zSRDRUH<id64&D-r_I_|sI*YV+c_j}@#Unz8M!;fC1!PcIQ%)r>uj)N
zfl^Q2gKq2UIFmiRG19W#HThhA@)BJ=Cp}p{&D1g6y{X{|XT|sMAB7)YFZt!axai#S
z6UH{zSF9@T3YnL+Yi_`@*|U`{PZIT-aA4zZImQjgv;K&``6RCLG4s`b21Qk62L{E1
zOf8f2)0pgn{$#vuvzI?|z454`(PfnhLK5?hR!_({U?A1r@40D9q1Vx=r473ds$YHo
z_0NxQ$1g6J_eDi2q%Y?F+HHLrQ=1n2a-MXGY42m_f;_!)_9i=>ii6cpc6#uguL`))
zc75XgbIMQ6)_>o^%e9b2{??@Y1U*LU=vUUWi;u~gWldS+G$U#oOLo?q=vN<Hy{ozS
zw-^@%tTs9kFgxx$OZ?6s`|WSq*WWYUT(n$?mG|}YKp`gmFJFEiKQ(Kjt@@?>1n=y;
zisy+RkI5B!Ix32rKfmhzDp72kUEQ1olZ92X0$!>J`7d3=ap&z$nLp8jDQBJBd$#3U
z?Va)I_pif3bCxuJzx+gTHp9E=Y|cfFTYvYyDSK0|qyJnY)oJ~yhW;S^Po0XEjCFex
zQ<wA^HMCy3mAgFb!ME*acZk{jkXpue=H1t&ji#bD>&%|>a<J^z+huUm!+GVFP5qHy
zV%Pb$=Px|Wa<ns{i7(#k-x2Xo%`2p%Uvjhhtl{z2RM61UUv_=Ncbopxe8N|ZR){e*
zd%b<W@4kj~MTFSDC>E{7{WH9$-Fq#y!FHSNu^ZhTtX`$Yp9^aDXSvLK^IBAK(pOR2
z$dzqYQM;TDpK`jWGi|^Air#J3fB0(>Km5@u@|d<{&b2CI(|`Gii)}LMUsY&!oY*`y
zYQCUcuh=ZVxl4L|CZCPSPGw!DR+ygqK7n<o^0q%846FY$oZiIbIC)m<I*nsrHzxWo
z==rDJ>l4%!#1`o2YkPn39>1qEgP(3P+2Ai)WZE4Za(VGV)9nl|w_o7xKXGKshpTDl
zLS0Q?96i9+dT*;<{LARxM~5;v15RBF@?@#rzi!T=i_b2Y82o3rb>2}^#n?S4Mw&U_
zxjFG{>-WdoR1@3uf;Ek86k>L+?I>eE^WwW{mu2USr>DDObw9JzDrVcHpXqZ?msXmb
zSk1We4dean-|xFus;KPnzuRaf=N>zA9z&|;UB&fZw@9inn652ecHo=Y{f}?kQut1-
zw*PQz``oNd#&@gknn;A|Ja(`O^zFMQyKhO{_d}|hY*Rly{eJm5gGH%|@XFVx1TMGV
zvzWN(_bm^0&wCHPsTAkg%_v&7)Y!&OII8_={^J1ig_AY7=fC1o?=m>zt#oqP<Gp9s
z*B!gN&#vrnQ0b1(`M!C+{{j?sy%YrfS`<#@ae2A8Zc~Z6ey8l`;~Lrg*;>sPn0{@(
zem#q$!mUeGKErToh+D+c)4B)6_CL;FcK7?YBPAM5A<UPHoHL()&2|V@nf|qH<Mqby
zPzN!OEpD51TDk=yn#}$_D4G48_hZ+~iVrI<3$S-8Snm8-89Hy;9?v-5r)s%(Hdpbl
zzs0rrb-Izyy0zO^+>Nd6J@=o%G4FTp$BYd#_pvesXKlR0{r5{++xN&E$whW~Q(mq)
z;B|Gb?c_6Si;kZNnQ%k?p0|+e-oM*#OZ;(9`4RGDir<8$oCl`2b8WVEZu;%sKZU*A
z(sAb`yOO+F3ZM0wUjEu;wTdzL$`{F!&ZnlPcDeRbRZ<un-P3{#w>^4Zm)~<v`$Kz^
zzk*hzXZfNP+;5hfE_MFtdW?J9ji156CQpJ6mInzv=ni4XEDU?1v^(@+UbJqx&oj^Y
z>B84ij{j%avNhM#w&>~i4ct;E<xVH;>%LK>drr&HzBxiv^z3v)M{gD(2bH_`j_jV*
zRw!O%FKDO~@=`6xDZQpLWJ}t$HLI_^eJ;R!NT;h=`;5^G8`}@Yrd?i>qf&EPS7!M-
zY;+S+-nBw@iiX_cmv`nQY_xBTxYDwiyF*UCt;PI5!wGq{1a0xmKi)^q&g$$wniO<c
zyS&l!bU2TL>enJ)ebK;+d-JEJZrN$;66-np$pnWjr6!ei<szD~qTeH#e&#raU2@ng
zz_d|0CvD^YmOq&smW)BX4&KsvjBHD+P2Y<z-ulO=<Y|(?67Ly{LZ<3Pb$UMWODuR7
z!yLf4hTHgDR#DyTGe$PkmwfJ-Y!t@pr)6aMXI0{={|sefU!@~Ug|6NC<hs$gTK!1S
zl9`QOX$1kxH&0s}xAv2uGJEUvj-$(kJC3L&@!racnP#;jxM&*xOOd>dvlnUYaXwx3
zi_@yBPowDmM*rz)C6|=0oP4@VvO(!|vhqu}edpZPXHEU!<FnaR?Uc<OZsq_+UB^@X
zQg!F&DfJ2L6FH&keMYR6>EOZB-WS*v*JiBwDsZCMZ?>hs|MgUZr;eV#wS(8#d0vxr
zTl~GkCjY+R^Mai(WIrgZFi7Y9yqft-cb~AasIuhGo~V%U-BWEjOs?LKUz1R${^aMs
zz$+7<%}t;F)z4*X$cu>~SDjY7M?9GOa8>Utb0@xQic?|)3!4)Rb_!2*TXpx~`k7xA
zuGL9iU1lxWek5V{I=Ll2r#uye=kWhjbkGo7dvJ%L<Eqt*)?f6TJuTGv+l*&@#uZ%K
zxsy1voAPc8aj+yD=39QoTR=bf<xZP(vo4u@D!7~~_sKP`MY#O_^r@{;`g@hVS;NE9
zX5KN~GS~Hq!kX|aa$ad%wn2g$!?t$ZVKY;neQ3VAmeESJf9=2dbPgHKsj6C<AlMef
zv*x(ofh)IX?4P>m{^ayZRdcsezvTHTSAI@+J{hL9>6KN5i!SFCmJO#5_AnOdt}e>W
zexNm>d8*Qb%SpViqa$i&N5v^N9MSJMF}Y>tbP=J`fqK7|I{n-tWwCehg{^Z_Fa1^a
zeRx;D@vilR(rxwbjPra$&Z@fn^f`4zr|s3dm745-y<f_g78td%PH;=mczF9>+mxiz
z?|reY4=mmkee`~-bJb(YrbBBcnyOfeGfep<eUEoi#l`D?c_)|fm0p)+%Q&l-9XkCv
zuTGBg+EcF_Q)(5a1WKCoOxrp+`JAj#%vQt5wT|o0u>Z<Ard+6h_?_gD<&TyWoC$vJ
z6sp@%a>n)Wgy#kq?w*a)yXo#x@-F$G!mjq!d#qkrNv*l9xwQ4wr(4q}tV?#);*SZ^
zl<U1Bl)L<L;kHM9l9`h{_-0FAUKOOV@$Q17-fy<7;TN6avh$Q>7{do$wNq^Ci&tOU
z+VQX9V{Ng;o}}j0zp56i<t$s~^IYrG)%__UFHK$?5O}+7wRHTo{En9g(|)*jEwQzq
zeZBe`|E<15l3EX69q{9R-!N@^){cTrCbz>D^ypdzEncay`Q7Z7JBmAgX|Fi&%5T=n
z$k;~<&I)Xv6}fY>kep%R_a?K|vd{b8sw)Ny9kIV7wtlI{t%;XIj|;ZU)E4j2{BunH
zPw=FJS2Zr^S~ogHJ2?kUmbJdN*Y?6(=Gg0%VqrhSJ8xd~PKi7h8)@B@9LBiiOy9Qt
zqBRnNyG3urB$>R-RC$wm_V3DA6LCZK?y`E(BR^(Uq-j=eC~~jV&15$7|5_09#&I)q
z#fDP*ps#%C505un&hMV9I{jI#{={o@cCTKUr4*JmE1bbOtj);mf^>l+zij7M9=CTv
z?mrIP`ZqtVmLqr3@}`14D(m!*sC<xoYP?)U&#8G+n9Qok!rhr6D-SHG@?T%W`nqe*
z;e*cq<gK<f@y$6I`gz^#g=<$&cIhmc=C5<(uB6n_@(pSCZs=@XtE?0=A*M2BGvDe!
znc18EGjz_e6#CKeHl!)-d}i^KDeYGk-v+q8@MJi8{XfI4rS3tc=Odra4fXg^CtMoS
zZhGD2rD(iIdREAdxC3ryORwm0XyxALo>}=>CD7qzzC&j9XPL!j_R>oqR+P!raqP3^
zFshidtY-4lUk$6JLu?HGGl>2ZyjW2!=X~u`X<OlxOKE`zFWJ`W+MJklHOO*{>6-2p
z?3>Isl)So_9l5sYVahq7GY2>{Wg3&+lefnEt`@du`CVz;IdhIl;-#(LTJzgj#JJ@}
zHh7EuX9x~c{UZ1xs-w#Nk6wLS#N&_5D~kGsHH%&`|7Y;OdeG|i8BNd~0sd-dix<A(
zJ^tpnbW%9q!L`bJgBrv=%eFF@U!S*Z%6dJQGx<5YWBhrh6r7r|{u7hNRP{#@n?o#l
zKgyMC`4RrG|3}8TUCxL0ZGBOtRTVe$x8vEWut>QDr!Q~K^PFVsyMyn==_j+B4mQXw
zZdUWJ)_E!?&K~5nruOy=mVj^9x5chIoc!^Qb;p{SXD1a%MrFA&CTJaKo@2<+<g3i{
zG5L?ne}=aGVtXP_?2%g3Y;w}7&+2;<--?CH1Jyj|x@t|y@?1XoQi|c0^jw80V&)q*
z9X&r?SN6=6_E26OZ=>dERgAeVCG(EfiGJJM_`!4T!nn{=n#HPKx<+1;ADx@BY^UQH
zQ&THvy(J&bS6S&FSE*b0yDa&^zd}}9zU(;VmP=1ePi^a4C-InHrO<-;?A2|{R0M?O
zj(K_oYi4{3Ot>@U>WaTnyxh@uv^PocsovaVux3upq@{}N1|c`oUIePBO#CIaMoZ>s
z@jCk>{S7zOyLP%cP76$(>pqK<+55_sFB?zu#Rug+vAV(^v^Z~Rt)l0h=o<H<i9%C3
zi#y_O8(yEfis6>-ZtEZJw`$y(XLik6>g6`6t1D#7zm1d5>3yx{_;kK^pXyil(2Cwm
zH>bDV)4zP_kk*o@KkaXQO|8!?{?<2f{r$8c(J6EHeZ6vDOrl5Vva(jlDVCV#sEKd%
zOf%lpxBZ=PFky$wPn-DmNku)Ai&|DNCxql2J030GuTt@M{@3L}>5{JI{@y`Red&q`
zt1^}>PjGE%zP#xri;y1wYv~RzpPjKcf2?!9c02d9$GiKjGTQAAK3rByI`FYKh(qCR
z^(#|B<@te6?s{JQX|qo?H2C?<MT<SUWs5HBx;nja@jKUb_C~|4{#2zK>O7P0dA(b<
zvhmlS=H8X>l=L}gb{_mF+1#wY+fg@Us!>S+qe8CF?iH><{p&u}Ojc-Wo4>&7v}Q*0
z!bK}09{K8+Pm+{AotE}BcIx{Ld*5&T@M^lolWCF;Z7fWlS+h1Q>y5YAGxuhXo4Q!q
znWNsp+dC}vOadeun5Jm<Jlbyb;kJpYG0*iQ@0wSo>+}b!x~w>R<HnZh@-j6Kiqofx
zM=H#o?^mCjoT<)w^vSi5oyM0}vozg1*?+PB!}sY|F8oNI`gir3N3W}_+N*YX?br7d
z6kdM%&Lof2;{AKKDa$+l<2gNJj?B%*ys0`{xqB8^?_2Wb&a33d{>NlecFrkEFK;sv
zY4e)&K6>9Q!8adY)XiJ8blv$+HM`a=%98Thbwe?$!)^I;w}}Be*sg0y{qhvwcH`xq
zWU-Y-;&u_G-h5X@8E;Kxn|{CINyW`oE>nGcy@E1I6@-_)-I8!O|DTkV`dZCjyA4hU
zEnZ}Gi&=VYSv1!gV}lt>)&}}-I>CGW>dV4+`F**!_8ogr_<2c^(f71muFLCN1VVKD
zH#a@Ax}I7uwPQZ7QOYA}!FIhDk)@1NFA1!gaJ!;iQO@9_@}amjFJm^<Y(K)Y!F|e+
z@6Gm$8lNud{2^B*pt^ML_iHB6-}@~2W&YSciqST@;Bv>3@8xRKnC|Xvhbw-0Z@k|j
zW8ZO3_=n5+PtvCXIV10D7R~Hk7W6nk!Pa`_kBJY>of!kK>@@GVC$W>+<mIX#b(j5=
zbW`WCOqeoNr`*uUO7;!=ibMmi8O8c6dsY=i|83Z>6!qx0%RK#t2Y<>tpZQEp+_?C`
zoXJ!DSA{r>%#zX&Svm9M64xh^dg1fF#zoJpV4an+tn#U-=v@8y86rz3io8k?T69k6
zK$GYag&g_nq^u>YMJ_#Ata9n32aBQZ#_hW+UDg>Md7fyXJ^faT<|GcECrht!E?A}U
zTS#!HPu;Sq69ao1ldbYD9{nD`&8uDerSI-HiQoF0ZbVKpC{od7)M8X<Oo&=j^WIna
z!BSnzj=gz#e$U<S$%u8zrS{I0h>81_P}}jg_MB<)GI17xKXYe(O%C{#`NW~)fzZTR
z_cfHy)okdNu-lrlwRPLp{|tWC_xUDIU=pc`+FP@yIdt{Q2uBsWq>?pe*FK5+>Ku9?
zb~SgOhus6-#v^N-P9%s<d2{}%c5PXc$kU{^*WYru7np7m>uJrN{c7LiKE@mU^{3M~
zwSv4%R#yl9XUJX2A8l=0tNb)sf<^FY@79=G*IEAN=5ahpTsddynkEL-6W8yb_cl=q
z`o91B@1M?IJC{XO`f&1G+WCdkyG6GyV#O79a|P-0XFs|-j;-+)PGvLODzV$T+Jd2f
z%^8&`qCf61mZkrk_K8p4+ac0st#inPvnw`F;9%Wm{!2P>hve%ZwldWyuD)`q;@x=;
zO*Nqjw<_xe4{P#-JqzXDVSej@T)p@GPlul+8P7R>c5hDIzEZ;{UW;eh6l~J^S?4`<
zM|Hc(?A)a4`G<czd>J%HS8eH+<XGt>zP2S237$oZDvQ4KF|S(adLe87s^f*#b#i~c
z{WSaZY|UwnK>Hs%zUunV->T&x$#XaUn#JcG<r|k>t=_;;<-c9tM<mIpXYGH6C-1-f
zOH7%{(l9+^dRsuq>UQ&uDmNzN*~^*9-TwVR!$gw*kh_B8ovuGyyG~BZdhqr79@{YI
z;<OI_UB@1~*ET$Ux%F-0+s;ef6_cZcpRaZGeDcM-@yGnSeG`=ySQrYI2;R)C*7&pb
z_wu}o_Z>Nn|6HV}E8g3G{F-)W{-^A(FMkOhHjtVjaK=>D<qmgl_V2m<vLBQ0F3_}W
z&Zyk^<>#+B0WY&pfB&`!rcFLQ+xzG0`rQYV{kMNQbV@`0xbwD!D+^aj{bvxktZCWY
z^`F5YP*Z>P8FyB8<pAyt4|4Ck{oJPJ#g=kfWZS`a`k%6AUy}$pnIQf2&%u8e!b~?7
ztxn>2W5H0i^Y*91qLy3*RaX1$-3{bkp5(u_@k+(N3%~X}*lr%(p#LMU`o_b+PDVSy
zJU@n~FP&@_pLW{B?4c~ZZqbs3tiC^=7gXP_e|UDo%am4D#$|u@?Y`5SXk=JeyD&32
zH#vZ@Bl=xLO+BCE--1<Bf}FaaGVS{Ei>uOBL3^@y^zzodzRe<^Rg{jMPw;#Eac}#V
z10P-${&Rb}<NNB;QzD69Oa$)e1f7nIHqicUF_me-<z;W8j^yzk_%1!~cd~i&l!6;S
zo1*rqEP3@#<KP#@71yn{cxeSpo9TIH|Au_~HKL^g=aVjL?BG71Twu2C?)QBrOY?%K
z?)<#h_Gi_R$}WZtTf9tT0#`7757Sz;?BBY>+wRou_c1CGpRiQ*P3+p2yK^g=dJOIM
z%187RP3ld}^JcLMdouT=-2G`>#SE`%4;F^!PyA`@a>UB_*OJnEsv(|{DzVaR(M`6;
zAK3>so|(33&h;eI^e6mIznp{k(go+ISpLnK8-Lx_bjB5_nVG7~*DU7Ux^hWto?VDX
zkM-Y=`@S7ZZ!8qF{%lcs^r~h`z`v<R6AoY5adC0_F$<d|AMWT`1te@(vid)R%8Khz
zQzpJi`s!{PvO+FR*hQ`4ajbS}h#7Y^ckhfllh2zmJeZ@I<rTK{)mGzKj58auBDdVX
zWE^p5$_8I<=J(;hs-wiYBwh;WhB~*3e>Ro%$*X)2Q(%3AuTt@W@zjn(r;R<e^!T1M
z<$ft&!!NRZ_J^*mCL2EQ+jE8~W2<JGVTOt(fBLs4a^><Xk9_Sko;0DxcZcY$NA2?t
zE=)P2u;K5%iBW4`+D)5PY#Q$S<;!L1*2<zE-I{yPzk8GC_n*OH>HM2<A|WAbjJQ_(
z6Ufgs`Iq#b`<9%-lvJ0mjT@q7=ko99Ivd_Lm+#{hcRok|$<7K}9(B%Qt>b^26z?*9
z`+Lnh!d*Q657)EJ^Ep|xDf^*{@XNq59oMLj^{sE8i#biIezZw<25ZqmQ@NHPBcA^Z
ze{?FU=W%teFphb?WL54`Ull`d{sQOA+Zs;q_}w$7z|UY$@P}5h!@dFPt}2>#ex{%6
zlC136{?t6$5$5wVbI!a}$LNdB#?p7R&n}vMIgY86y>(_;uK(rI$ogpinHe>wV+<}c
zOx&2#r&8<Zw?5cxEBhsr*)j<q_qW<)1=hXMRoHPVd%@DaB@?sRmSpbO|L7ij@0n?Z
z2`7Z^tA?zaz^JlxlBR>vw3TOM+>I}@m<XrNb)E8DR@day(V#SweQ_%<9gUftF-3rB
z!^4J+cX*ON&XFvN-tReO!F_=Qzja@d%s#g~TphKH^M_^U90tYB_APhfjiMghkq%;<
ztmgcmAw2c2^R%E#DW5K0W@Pp8=@-p1+A(R`Le?$ntAmW<uG@ZOv{B9Q<P4s~@mb_6
zON8y81wXzf^)XghJ~_i}$|4Zt$g^eE&Wnyde~v_Mw%c_6*W=}{UuOQQnp!Lswb?&<
zYItQ`T&A1*+vKL`8;W5(>vT%l9z0@MC$YU~R+oTlW{}&jpIepYvmV&6$5^Cua@w3O
zkDxTwyWD?O9p@WgbUM2uBhLNduaaqnm-ohB_<8I2+(7;5JeI8;>lfVhm390cP_jCI
z1JkhskB)!3tg+<B9J_7z&ld)7+b}11>vLJ9Jnqg@{~2TsJz4za%>IgbZcl>NT{kRz
zd1}t*<yQ4;t-Q2qw%d0_EMNQb<|)=oF_XR&^@cfdT1hqraYxj>Q|{sK*j3|Tp|o&D
zZ|M`?p7b5-pKh#JV`k}PbVAeTVyvz~(E48uYh~UG|K+_pPp%{=b+5eR6g}@tl23}R
zT|PQvamy)V5#}??bXQ~+EtsGe_OfuJMYDNp&tb))9R4$vujKAcjJ=h|d9<Q4&Mxcs
z$xj?!n*?4@s}tQE5|MXot%QAS+D!8vNfWJ`YKbwM!uB|An>IIevn<caXN!6JCca_V
zd;C3*Y!FkLz#^~SlC=@_Nz?ll-drQ5-jL+~lPOYvs_Um6HA{m2y?rWWmiQ<>w&c9;
zm-T)prw3PbPtuyPc}o3e&d*ZHViULwzFJt%?PO)&r}p8Ws7bSl;-6Vplg{TqcTiSr
zf78!<?6IN5hRtoWvh^RIn!0<M!i|>!5;N9@Jh{p5SNhzowB7T^b$QR!oeF)+wU(XT
ztZyV$t6Sb#vcvG$ftQy!Z`Z!jRdY`_WL4Hk`_GVSQ0U!$@f~w>#P;b&KSrFB?0ER&
zom9GBb?ZuwRPjyS%a1r)Z~a!WrBwaLqSyX{yOtl3n|N}M--llsRekd`js7!i3Jp20
z7ah*URJvuF)RN7-&H=ktGB7uvKe*Jms8#jK0mC!?##!@qwMA{zKNK?_DzJ!C>^OK%
zoyQ|CWdZBL)D_GZO_p*C@g^M>%f5Q^KSN+(seR|)_{r%%YvaP@{IC4AIw^HHTqxe<
z!%2aAzV2tG);Nnx?KsTOIm>ws1B1c_xwWs3?2wjNHoN`6!-B4ZA7ZSQuH`mXIk$Sr
z?4^$JOWUF>6uVv9kID-_K4YVK+D7N4&)M{oMP9G6W^ruJJR1^fa?#*F!{!y)yqQS?
zhF4QVr6<-%By93}H{obn<5|sL{<`uRE8fJXOxN#S-<y2lvied-w}?|tQ~xupXk1ve
zRc2Dg$L`Yd4qJP6_cUAHi#zYeyF3+o7Wmv`%Z2^(c9s@RIDRtRTG?pr0qF@Ptyc;(
zFKzfyS!|@OmtZULa;5ZV!HX8`sT;eFFYf#(nN)SDS=Z%9@q#8B7p@7MD>q-6CMbF7
zt7J>?bIbj%KXO$n+Y*#ad)eZ@WkoIx@~k-0z;Q4s>h#Cj14qutvm8k}X_T_k%RFUT
zRixYe-FrMAubg`K^x=;(*^8o|vQ|!Eo$=H((d#Fd+msJKF3;by+;Gi_qR^=(UYo9&
z{5rVz>k@~idil%JJ^T-^i^cyGYB_EHDd5YZV&nR#W7?l<XCLhUz<MUz$f;(=Dy>NN
zh7BoOt{UzS*~)9Zags;TE{&Iu%5_g0Pt(0qUAO&mf<4EnzRVj9ocls%%{Q%C`P=on
z_@C%7rGy}rG^K{Mt}bjB7#O~H2mR2#<S%MdoM!8?FH$+glv`Rf!I{@nLc5yx;2Xm_
zdu{_orzEjWvHux7iq{tXnS1Nv{QnGp<{xqD6uJ=a#JOVYm809b^Rh3<8qWRlC02j-
z)8{6qLRqp^dQRK3WDDzdWY4`(RKCsR)SEtzhisxnpLx$8dmk^rKl{y<o_FlqB;M#5
zJl^o?#HQHh)K8~k1UHxoKHj(PKLeMn<MeQ`53koqtyW&u^LJUlajMWxrbpl7wH8+V
zXE-6+c6&~d+{J&QNlUGM6m=cGs@k-wJVEsAjbHAUZ0G!`J?KAudiTRmXB`(!c*37N
zll?z~kYi8xVvVYOU5gg$ygauquls!F*RC~vtEPEoq$Zx6tUo={Re4&6fw<kobCxwn
z)_MJ&s=izAjB~M^@anf`8fDeDNUW1LWGd5huh;aQmKDNu;Hjz3lbl7I*^^mTysCMv
z{d(Pmg&&m{{4!rw6}N1iRMx6l9ydSTT9V1M=OSB~gM-?~Nxb_@{h1#Xt9c(zJ7ucJ
z-~R5*ug$qzN(y{T-br@MJoqtK?VOjVM_^_{!$YkEk>jq0?8WjcHUy;3OyQjV)UsAA
zdWzFA4b6v*N_`s5vR2AWrx?sH**|~7cRpchVnNZ8ht5WhlCiyV%UX6;{k?JgQ1Qpf
z&(9cd<kp(8YLchQwAIts&$0FF(TNolbA0V|_PdAO_wBmJ|0Lbb<nq?IJZ(X5P3wP#
z9UVcT#i!SM?0c>@Z}%#zz9^UeH#@h8imq5;vsS58R4i}<gVhG+hOoo8C1z@KJp43a
z!?B=Avu+=X{?G7u?arfO;zkCSj+{QN5VoK5^z(r8Cw4aem>sEBKKrSxv$3#V?!?s6
z$p$X*R)Ibny!>WoUDZ|DzC%38bKj2*Z>FrARPf9#o@eP=#+B}NN4uw2yX<3F)N;mY
zy>2$s7s<`)Cnqg9yMR|N^z0smlMHKDy|}f~%4*lLPa(n19#bQIi-q^kpLW7#@0$Z#
z)YNPyRbMG<{&2f3`TEb!`iJwlpF1%ATw&q)nNj6+#akKekLDsqryphhIV9}h)upad
zf2^n`S}Y>s(!_`t?-{L_RbT7uUX`^+Kd0Gl`QJ6BH`&9CQ@z9Ne@5%9UM?!A%=F;M
z<b9KK%_W}R@IQ1;Q(0CwV%dCE6?sN4&!h(vwyjv)Sn%lk<+AA?v*o)Vb{{x(?D@1+
zB4;Q53!44({ZgM3i!@G!Y}v8feetA186&0#ZFUO_uFEudJSn~}@7<Yf<WcZVs;}<t
zxrv9=yi1u(u0MUbY=X0q*8UZup=R2*=T+Qhb2+-#Ei}vO^5s)(sZusIS)tArpG$ZT
zikrvj<}W?E&*1Cxz|+6_ms`*Gma96vyUE$P(Cn$xmM#vd+#Pba%YI1L>NlM`syan%
zRpnm0^Jlu=vsl!=y!!ar)Yz*_uDr_gy<xUGCi?c{kMcf(!CtZ)Yb9d1Ld?I18NPeb
z=>78gnW}?}=Lj7><FH20WB>BJ{5O7o=3f?E=B=foBXleFz`DafGF(q99lFq3aPpb9
zHiNgAmj2oAkMI9X;GCm9`;gg6z5{{HpKedR8P>LN50j>l<lScDX_u9(_HK=dzp~)_
zpZ85`o-=BjUdathzqDU#_cdNaR!?az^S3-H`cHn&s!v^YqC{}Nb$#r%<5Rmm=A82s
z|G9P5+tn^p+H}{?kGpTonbp!_5zrtz?UWwl{RG}MX43Ef6e=ci`K4Z3cz1TW-Bfn(
zIPRZCA^#cns2IKGmWkh><IJkkd-lNX{|v7-eEXHia^ay__}q-E%teR(U2S`!Q!J)?
z{qs52Z@YHod;AVb-0R?SIzRWXdhOo78&prc?@LvgWGQcZ?6-4==fC|kHfXlHEc3s)
z?%2V-ojZQbUEl9HLu}50&GzT+#dO=9Nm}X9%JzDWO;hI8Ng7j24O#9T*!Jw$@x;X1
z4}M=c9gDo;-enw|d-UcjxhaBDNx?xmRxg}d!>4g}c6*dvE`0T$q5Ip9gDy|CAFs`J
zySevlRe`bMJKgK&>?SR|%FuIKfu;4*5yK7ELYC5Ve=_jbNadV4t$2_jJN-_`)xGyb
zg@m|Qn8yYk<C}D<<XF1CA%l?E*^STs-1@%lj&S>gC1-*roy|S`*KXN`Lk;Jy$h>@d
zvFYc*wvMYeOJj4UDNo!W@7%BH$*z>?b$zO?zrSumq{K5_Hc_oxP5TaC-t=;Z+V5{N
z1=h6&pH&@A`JMNg@2yS9ny}zV-%cmxDLWNsy<zyzaQu}u^SRse>Nj@0N+{m1wLD&n
z!K9$_@KoJ>@^fT;7fOj3v2D=Z_~>DEZjHbru4!CnO_uHXp1*FF!|bEcpML#3e({9Q
zz8MjlBc`cH_s`GsKf6DKYswUH9aq!6{MY_71Xp}H@*|(G*Gt}TVoFa;o`dd^`0R*=
z@*VH&dF(RvrrkRDr`J2??rOow?E#7FPd;+n*0)`AwMprRHN2`yA-}kGon~g4W*#2o
ztgRdqy;I@FN@lf+S!zj7&rb|kuw<_8V%b?!H$LY%yizzd&9Q3x#G*-jM+ENdY~~GG
z>^aGe<I8KwRPjrBIU;wOe0Of&@O+={vm`OKMG=d(>qW4s&U){_Eq%OY%EW0$|1<2b
zf5#|t+N9_03;B|}9$dP$=f3^=YiR2|W8%5rlm0U(<vwSdsi9)fth{X!voYI){|t-k
zJnfag?$6O$_Mf4n)tX=Hw{^`LZKj|&wv;_dPATErcPix7ecdmAnPH~6R;S19Ge0v9
zU3k#7J?yzeS=y(kb!K0-r9VF_BiR?MDzY@>=3N!_{cI2G=WaZ!@~BUI%jeV$nU{{Q
z^f3AN=dgR|!xs{Lr~M}#m>JB}uHbO%<JQ2dQ`+X7=v?BKS6@;!<Hp*ZZ@<b1_nI>u
zo!H`a(sHi*+_USW-~Bpnd@S)`B=hvS4qGf%F-^<gxNpKmB}E1K=v#jT-+Aoc#(L9L
zq~K@d;WRNbhUw`tGmkD%nEK<`et(Z;rFN+^gX^arI2$=N^>UcAuU?ADQK`yykBA%1
zX^*$QJ9H#zd)LRLso#aVMVaJU_Ij)T`1?5gK!H%0*f#E{AP-m1WlM6u3FvH7zC0y#
z%2v^t>(c$Up7(cKD}3d=(KN1&OR`#TOuNQ(cb#5-@`fGZl78|mQ~jf-3j1HVZ~thP
z^T+eO^QJ0ldY<B1Jyogi$)tnLmML2o&JPdyDi-dgxms)COPN;-<KI-qWdBQ?eQw&$
zL~qechQE(&OxXQu+hNg|=&h?nJUFYx&(!@{yE<R!2OB%PW#t9ymzl~&=YKqDPCK`1
zbr!>y^gUh|!cN-7PQM!B8rG{4xLempF63Yb+b8$mYwxU(j)_j}{U(+*T|;Dc!(*qn
z`?bR3BbGmsC^<cEi$e31+2V5Tfg5WdKFO^&*laRQwPfo7GxIyu)k#r@PoMIBeLg=b
zbNxBRDHE3zRa;3FExdGXlF_oWFaJK=`#bmer;eSRF<X}<luTck#J}+VOTH;<R-Kzw
z+*MZiFy8h0QSaFJ%U41+`Kbt=^og5!(>Xd)J5_syQIr0zt#A8M)}1NfDO0Nx3Yiu%
zXXhy!1yP;@Y&ssIik=IUi<Cnf-kukk_;9+#1e0Km$gjm|Up98d{NqWPxKU)T<CPGv
zW4~&CJZC$e{5s$c^Xc-VVd@{2vv|H+zEY)##oF|Sb%w1YlSxs%@JE;WX<j0Cy6Rf@
zxhXCGP;UI`NtEB-v&ZiBn{|3`-}dV9<Jz?y!v7gK(-|KgwOD&`-P7Kx=fX?>sd}6d
zHRhO?vxxn|-v{gm|1Aqk`Ool5Xlis!ZNtkO?>s*9%l-Ry!2O|_uFJ}Z2}fs6%6{=q
zcl|Xkd!Yg!j)PbJGo0K1pFtus^1*+GkZVDwJ<Ve-J`>5WH`Lm6*IRW5OSbvjtsCXi
zAL{8J`g~#ee}<IRflDjjKS_!c|ESKW>6me7%8H1kIk#W_Iha`^cW&h;v6or<7A#!)
za_gkZm5)ySc@@57^#lb|E|IBeo1&c~J#@2Wk4V-x-)I%}em_C@$-*c>=Z}nGp4M9*
zpZ?=_Xtn6iE1p`j9N6tbSyfdwc`6=Qd(<)e_}r@dfAoHIP7m~qob)`hyd&#jj?&6$
z(+&FbSd8WhaZju@5__EcasH$K4DB}IY^`TJ-Un&CD1LTQXMOsDEoFRX{#h2pxE$zl
z;*;n-6sq%|;hjg}s|mJ?Pu*PZ7m`xBG%a}61u30Nl02qUu9ozuEUW%|Gbz_d^U0kU
zx$4JEL6e@o`nvb*$~yOqcRTEKdVHpwpS0(M;-#Prx0lL?YSu6Dttgyu{mR$>3_ee9
zrpASPMyuACd=%PsMd;L?skzG4@!OAy3QbGnEV^VY#G0k$aO?K$Eur6bcY3*V&$TKF
z2@qIRvO(QR{&Za7FX!n$Ow%%r?KYp&k`e0K;4<M(+~YDm4|$7sf9yZZXkoM$@|1ry
zG2y|zFS!z3Gk%<w{xM7QPU0lfBF|suZ^e4HAAHx^_C9?2Gs`pGWop~Dc`|Oi^D=1l
z!DEZrf5ab}uzXgl&TOmh;swmL>=P~-|G0l-p3W-osMP)+OZaE99KEi8qDW`zo;~xG
zPH9NLd)36^z3#Ty)|0pI{jN!lZg_K_@9@sUC!Kir34GbTqwI5%Pg>!ec=1mDj#*22
zJIz0BKe;ILo*QqaRFLZpvG>~!{rz1Ru{TJ3j@s|an*ANCJea3&oL112ofew9^{_xp
z44>PbzXc}B8j>rXpW*jcKE*v%*Z+aj>~DGUeot<vZcwR_JO1@>r-yT|+5R*9k0)-p
zB&)kU?&GJM9bXqsiir4lqdR(H{p*7V!zRtr&YQc-RNtfMyx!I(&ZBz{?3(S78}=aT
zfZV<R46VN(%#1pB>|WL9C#DM;X2vi1di|Q@nTDxuDu!Y#-J-cCZp8h&fA8Wm)3c94
zTE))jSUnZ7+xS9n!t1Ki#jkpr`b`Aiv-<XwnxrtMwUsD&ocJDO{kcm{|K=GvyU0JM
zn6}nB@7vBieMfJ8+V8|86-m3#@H}eYl@YYg%d=#Dz{eS1rZ-N#t9<p#UdO3lza`Ht
zS&{J7(yZQL>!Sq|vzIz5d&ggqS$HY@`|eNs6o15axM>A_C=Obg=&;&)hlids!>OAq
zUn$og|IZ+%n*5{c!{!f*V!nMiGi6iTwUt*gf<yx{F1~Py*W}u^TB}KN*7u+L_dTxr
z_P(ZUsrCUQevSmzhEF##PhYawbuQK7r}Lz9JsY)D(!N`FZ@c~R{HJ}~hB;iT{MT<S
zy?-N!O|q*$w%YZzde?Hp(`sv)0*Z6ynXIjt9XV-H!^z)HbJubk6fHXLa>-tB>&A+o
z%ls6BPQ`_MxHD;k<{8myD~GF#<_k~dKDE9r)iN#1{-0CBy9s(O&d(p8=~pzG)o2tb
zA+dL_tNK(2<_4x4F?~<wo;{;?^bvn`msh9exd*v38DEyG2;DMODid@!ITSfZzGV3`
z&zCF1RT;|WT+wNbO#XC1;E3R6S%($d#dvoH)E+x6+$;MiWvOM?xhV>awd=0zj?UQV
zRHHX-acT0m1r?92em%bQ$<5Jjs#V|b3CsIDk4{;7M*az7`{k#f>MdNF9^Cl4evW#C
z!ZD>JNAA?<TldAKzx^q2U6<(T+CJN|?0UZSpXr)E8XHbqX16#v&0fh~`u4{=KR)3p
z?YqURtKS`y>(t4wpZ}e`@<4CXhSYt_HmXN%uYdfXL0$0a$}2Tz{iY~a+4iqJ?GUK9
zRkpN(bE0dPLBX})6_Yoxy{>*^@qW97`tyksRQ5GzU0NH}op&LpPkZ@2hSb;_DlK<r
zzg7|4a5*W5QM&xr^BR?ts}x$Zz1p9@Q+O7A=klue{%aLh2b>W6BB*zL=kFtv^nb?B
zWB7JBJ*hKj&ZP5CuIp9WU0wRXYXXOYRH@(A{$96^86GJ?o($XNSMJ?u;i_$PTU)ll
zI&hu#)}!|xiJU5axW;hC%G4bbr}TS&xVByP?cSpwR?Oi@ZF=-_b@uvKR*i{{LQk0(
z?+Lw270|tZ(lJ{+cU>j-aqBwU51bFu&rjFt*unFk;rFaFPrtN!ZQ5y{pTEaEESKw<
z`UQc=9Sjk_axAvKue(2wUt6e`_r<xmuV(epx%>)|udi0^VLp>7BhI&de_?{d&i#*a
zn^$FV6)m~cll5KaZ1^b+*>ew?6ka=cEd7-~HF=`dCjB?fj2o0}qig*?-DW&;y7W(F
zWm^8k?GqAD-V;iS*kEt5jVaHIZQ}ld><xRrDbJQOvwM)_CZd+$p?pL>-K6O4)3uyk
zLQ!qEF8})8u3CChCGTi#jPmN)ayfa`g+G};wtE~^*JQbSdV{Xlv}?O3p17pEk&{bq
zZfPO2k;$ra?lYt(_`YK<Sn{Fr;FU#Frc7fxyM4iGe#3Jd!c$th%tQMY@5$L{x;Q;}
zr&b<&0)Jh78-pQ71XphExi7kbpTzf=dYp*~-Sg0?k|F7{yLI*s&EHG7`xiXVeiX8m
zP15!7^lI6*7H;VYdhdCnSQ&V%>bI1>Qj=m`!m_r5yXb9k@%iujC4|xxbtmi)TbcIR
zZuaGBr|%xNQC6EPx4UR9&b`6HQsjT&`KR}tUbBR>YhB~3a%x^a&v5lVb^iCkniGEx
zs419siX}M3Kgo0dAmQAR7RjT-`1I4ZDRLVwby?MYZ9N+G=>((JlN@H*DIaTh$K3w9
zPmb#V>z4Fu>)lM*r#6d89Tso*;J+SqJSbt8r;_ol+cD+){?r;WoK4d^An0|u_@zVK
z@pDou%df9hJiKSa^kp+8RYMsT+??@}hx6O-18?6qDP{k7vh?V&{viFpfam@iQmRXS
zTu%P9ZvL5Q%~Z={p1d;ry0N}Y9v9opG`+PZTr@v);^GP2mw(r^6!)+T9ZfmzJtsqC
z;eUo_ru!DPy!Kc-tM6H1)QYLw|1(T5mrK07m-%Rg-^Po<1{{$mr=D(#=-*|>@%z5@
zw{4~oK3>;5W}Bw<ocuX?b<@AKj)lV0b!N`=cp`QCSEh{ceV4{%GY>}{ukQ2zbmPyX
z;weoHQNOQEGq0+(beO6U!X{h$>z3FiV~rG-Yn~g;RpRf|M!)=eaAWX^;HC`68>ZQ(
zul+JjTomh3x96|ohIz)TF7B3|svtX~`o@2T<T`^HD>n;eWIugz#WcMxSBPbmfnnc|
z?wNM6v%m1=D1JT@IrWLT^t<JLw<qM4{jg5%|MsZ7@Ysfk#UI&4t1V>q-`KWw-Nnn*
zg3S`05_>FlXNvs1eo05NJ;F9DbK;kKE{W5&ZAw|%`fcNsD-v6}=guzNFSokMFuvoh
zd|~quuPGVL2Ese^(|4?pzNGbG?re_9#^POz0%ZCe_22$>)?B*keAW7upE*@UU#cYs
zZEQIs9CmrWPv+`u-2=>%w#h_`#;8l0f7+zls_URCwIg{o*JslOtlzqomBOUiwLByv
z73MSkXNX?$^X}dGvyYt@cP%|0m?F6?Uf%U$`jyvOVTZJ}<D@E9^{klWv`O%pILD<E
zlk^v~g-Z8`da30sS$X&HhSlQ5%R8d(94UUer@lS0Fk@aJTV&GYsfu2jNkNMk(^Mxd
znJ?VrtL*sEZnDM8yw>@PjQe_DYjLKYU$yMyieFOck;OjCECL0deLZupE$Pjvt`i4K
zW4Plv_9^NnvZWmATv;w&@Mc!*&9ld*^_)+7_V8nkL!peBQdGl~M#G2+lER)U--QmW
zU|{fSYScM$%f@%^^Ph!Xt91J2`SqQ2xEwj<$;?*eD}E1};*F-w^IWakGci}^-ygv?
z9TDZdk(|YAITwe9TF2hFc{KS&p6p@uG|wXwtx_-T?1=DSiSi1Y;>Eh^q4G78J2RD+
zocc7&*wx!rrsNch!C94)5?jA4KKVLv=kIGuk8jIvZq-cZ*U`AbSoCaKt>5<|lfL4k
zVc{J!%?$(9o-mXZ2|pCOq!Th}!o?d?nXad?+<HE_?!MgjgT<Qn9_(evt18awKB}k4
z$Nlx%hn-q7$5k#KeP-30z*uk4ee2pg-yh%Kx9R#!P20#HYs+cIm!Ew&^%O(PERKtR
zLM8hGue`o!*=tjBspMU}_r$J}IVZ$CCdWEYlizbyIApgzSFQQ&yx1S1ogwFBDtewY
ztnoQ|=1#++lPs?vdgMAR3HerQw9MnromJmgX;sH{$~h`3xFucI-YxX8EqQz58$IjX
zKh^hT!>xXA3h|Zxsq|~>x_oh`K+cUzYWgmpmRh#h`oxL`#$1<f{R<!e6lmVPB@mV|
zrOC7G^H&qQEv-Qedt3kg+YqeH$y1ke>DaC<cQRQ`>#Q8ie*G;JXOIgFda-oQ@i5b^
zTb9kYaB6T%VEB9c*$2t=&*D*&cHj2<{Vw<Yjt8}VD&JRfEl7VonYVO}R={zcB1^YT
zOZyJIS^RGMmG~T;gAXiMUAuhlKSRRDrwbb1zGo0pRN37iGF9RDcINIfv#tB@^F@?K
zZC^Y~eE&ywj?jkvc5CYj{a)+!zWC1|kg;gVYnirgTTSulEbmiRH(D@BP5kW|c#~7Z
z;&M-V#%}(`NAoOKroLG6$Kl$8``4D4%FcZ2FZ@urxZvkfhUuESIIkP=$`?t-Jl6ei
z`;YCH^7gsj=3${>X|=JlgY5q^q?iV@`0VWb&u~_e_qS)zzXqmds}0!>`bjtc6IP3}
zG~QS&Y{;oMK_e#Gc*BhiMUS>+ypY}WC+Yad9R4S%OFW&9|CHDn@Sh>ji?wd`v<3If
zf4sTYqvfgkHi`G`oBG}TIb}18g${ZyUmZUCYX7-i?vk-GH=HLvn<;vvL7RKyzO(s#
z$L|Y?y-He`U-i|aPg-+H;hz(ytd}({*pPhmxAgoUN0J)mGpyJw=lZtDW-UL@0*7yn
z>=S1;^jK#pdu%-^{Wkym!2ltqDOcoXm7HDrwN=wQPSiBYTS;i`tDW1;?)MkozTLm}
zu3B8+6!GWjucv6KXeu-^6>D>bZ0vQoqjf{R&eHn#_XE2>ggjf)9Q$upNZa$O!<U36
zNo~9~v&qAG*G-Q%CZCJn{<~2ytvFSx=5FhMhWXWplp+exKbP0p6q^=w@Z|3PuYA{j
z{jPba;-5I>im9IW{n>mW^%b8kN$WLT;Jle<v#NC_=Z2&`^4W4a`gu2!_<PdIf7jhN
z-*M;CB=;i{$743Oo^%$t>)N^P4|o6f1J8U`sNDRvZ!WKkAxm2J*K^Vifm|o<MGKZb
zZ>aeG{hx)NCil6M=D)d_*Uz7KQ7y@nCHhB`%h6<h#VeAYTDxD~Zz{k4Mc%eaNXU`>
zXTiS<M^&u3%wuaFKKZig@?q}?fmxB<9Z#Zf&de>9ju4QDDNb5m;CMjwBWHHdQiUHA
zuIT9}NB1)F$sS1d;1c8Nl3Td1-u~&sw`_@ek&)if4wt)@B)vQ5*yp<?^OE0-1DlNI
z&Wz~$ak`MxEl;&beL-h&&~=4pu34_VF@jM+@*NXqZrZeV)}6!Ges?z)M5*j*Hd%JY
z%_pJv<%%G7K_?ZVEu7DGSEg6bmi-(3zV7=m|BVurJM`2$CfY_$TWY*pcnT}ipF{l<
zna&G*urSuQTQcF<xd`?v)vjR)iCWKGH5dBd`+cTdee&^T_uIFb|FIVl@1AS;hx0@4
z&g<oif5+)ue)OUH@3t0SCC%WBE}=8GJj6EMC=+|qpJZ4ivh`2uBuk5BvoAl-ND_Ky
zy7bR3_KGFWS8o=cfAz`op2e%h{~3;P?w_Gs#!{A7_P*`0VaJp$rvojgovEDPHf`Zj
zmn^lis=`C+A`>5)KaG-0ektnnpJDB+`Kw=Oyj;z2@=<8t8r@ST3-}h+XE*4c-EZ+H
zyhAU@DInHuU9Gd&(mDSgZCv!wb{hA?Q!z*WGfdd&{o-Eq(WboZFRNUN?F#*ucBFCe
zH2#|=e|I;JP?}L}@e0Q0A8tQ0-}`7*c<RQ>9;bDFU4E6=^<(<wAZzuX&245%7c4Hn
z|KQI4@%YrE1)JL389tX*Mk+m=)AUi}o4OSH0o&s-yVgu)k0|+@@^Mkx&BCMut>vH2
zt8ZpJ+IGB0s9bo85X;uSEyeW?+?$X1_@@^ZP4F+PW&U&g#F6*6uBAmfFW7pZy4&FW
zzQ=byXD5Ypu5?rgHMLuOCI4%dYxvVm7ydH@UYU|3<|vTB5M2D~-m}Bq#UDFP%~+z*
z!gJ)>vGoBpD}VD|^f2FDtC+1YF=llegGkY`hm6XF@9Nyu`Ae48v5W2ASbnj8iK&M|
z`)8S3LUBea2}if)iJ6-#%sVpAL;34%7KxT6D!M9-zOPtg*By!2Dm_QTaly}tzw0iX
zTvEigH8i_<#ca`SzZxIhxL>?qA@StVrU@e4(l7X~u+84^O<yQz!aJ)cHOFr;PWRUN
z5^%+5(^hZRi6?ie_I+30Z&S-=EIa+!sx;YU8f$hn&z-Yv%hH>(pRzv^x-|2YlZ1*8
zTjTz`x($EMs%)#c7ZSvF`|UP;b=FBQXSML9u&+O&nY^$g_Exa#J)K!+vPGA;rrt9O
z`)0oJ#=HFN+xPE_8Knz5et2bTUvxfKSL7g{huQVFe-Emr_g;@#x>V&i+tnn8@B3fq
zc4p2BnLPdB;-74n&q+)W`LcX{e8uC_8jdR$Mo$o(=*_xg>-S&f2N>P!zMAh8QCj;v
z>UKrW7S%^)yVe|^erlqXtl?$@1?IvNa({&QUlw-BnIADa>(#m0G%HK{<vUlM?9A4~
z%_hO|>;5zN&6DTaI?<)Y(rx3`Z`mL7{vP<VZ^p$HE9Yv-?P_1Q@8Bb*4No*03uUVo
zE3f`_y6(Huqd)y;{(Qg98{nNj{nXj2`G(auXH640_i*xbzg@RIysr4YS!S)>y>Fcp
z<DHt%+aEV~p5#asyCHDp^8B@N(?r|LH5jJ`UVL%*#4D9gi+MA03mmMx<{o`ldv?3n
z;V>c9Et|`eH(#CEkbC@_@^?)x-MF2#Jlf}l99MaZiA$TFeOIwx`By^skts_yNA_<!
zbalI3!GX2_=JmEc<!||`k1zYX^@GIy-fz$UevCeH^3O(&u&9vk<}YIF=B3H0wy#|j
zwrSa-Wvj|pm1a$Fnh?;uLaX<<*0lh8{TlX`b4pHZE!DesK0mwdd)rJ84^D@~>wBw~
zl*!yJx~P|<60fCkWYX&-XV>3HCU9<$zCXLxu>I|pMKjd}buBkNRiB#wGMa6z;>iOu
zPe~M)it+B7_4*~3S4&lGRAWHtkpuTm=dOQ${$+1H=ahw-Mh8E#TtCxWSHH(?rbg#X
zk0(`MS_CCRRquNyaGTxYlzHcOhyQp@JWtfcpc<E`bCna<9a_4zy=g_Uw`X)rsz}MD
zFdM<WK3&^%CMxK>O{xgo`TqWW(TYBWPUT}o&k|&BtakKEcYMn`W6P3Fr)_P0_de1~
zo9=BbDVDn9YIW1?-~W_vvkPAnn6pH_ihuU!vZq2q6Z{?=Gk+|@AAI>|O-GL6nn;^N
zI!US#Z0z0l^Jo8I?$iHq(ELwC<;ju>t9tL)vM;|Q@p6A#@uZ%lLt8z0_r{-}ufg6v
zvu0*r@}<DQcCU`lvz(-y@@-YF3c7i0=joStAYHlm#$)MQ%7&arwgwwTD%77;xFVOZ
z*V}b|=0?rQT7@$MCx%`0^fayI2>;KpvZJ#%#`1mmQ^}GIz8dRR^jy1^`6=m>O04g)
z?~F55@(uD=y1p~i-hB6&Qs6v0p#w~^pBnU?9z8m+lUJB=TEUIV%X4^ES_Zv;%-Z~k
zg=bk$(#4CG>*C`><Nu{PdtcpBVst#*fAVRmlioqi0bkh|y{kCfqNna;VJN?HZmwxi
z&ymww8Vb+UI&U=}@%v?vo7B9e!g5mT;@y#x(wHV~@CZ=t>x<f_5Fy<x*MBCvNN??Y
zvsW4CLxM`p5<_M#F}+!r8?rNJ>q3(yMJh#`4lX&w5#F=xXeoo)$;UtTyv%%8$hb`R
z6hrXNKtAVPxr=X9f3^{m7Hkyyuvl@jhKQ#2w7w4tvnF^shUHvUj9m4`bDi?s0QGQx
zZcqRD$IWlg@>*#;XJ40lgg0-<m77yJ90Pi8tyN@TmA>2d>zwGJ%Vt&KdIx^=#k9OS
z5~A_>y4#+-f*FqwUvdbN)>iS3IaVvS(zsYeK>Co(-WgJnqFu6FEFOzJFXz7fW1YLb
zzNU1|Nd~j?{!?}R%DxCr(&XaHs{9&J%^4}?bXg^@!Gk&X8ShqB^&RY?yBm%a6nb~Q
z6*Bm7C1Fa_QXP4*KCOG_ZWUL2oZ!WEd~)`Qud8h`>?Ff3KCKP)zFIleR4kFt-QTQY
z%Vjq?&1X%==1O`pPPk%gx8%vX^>b4HGYCHOo!#um<TNv)by>JsXy|{27gKGspCnr!
zpJ;HxNM+U;MV2?BAM1ay8s1&3GvDgm<~5&<)~`DAO6!7$t4|NJX>Rspl}o!P_$`Qu
zVcu;md_ZFRgBz;KVu3nOC#!H~y0-OsKA9lI_jBu-?ET8^haU#2YNTo{_E{m~<#1!8
zO@s8|W|LWWR$i>Gz4z$bPL;QBd$Ts?6;EFt^(x=rtb=pPG=_r@SNEw*e74r0ZBy%p
z-CH|DWe@D)uI1CJco{gUGvSXH(=7|LyL108zI#$@QS6*w5q}d6HyKG=PW|!i>fZ1F
z8A6YLGfdgpz37ei@7ftJOrA0Yd0gD}(?tF3%q8i$(~donzJKHPr+*S3PdM9ZUT9L)
zSXA};p51j7$B8DZqfT$!X;b!}p~rjbBXxtidh1Wwhf=wc5{yLqPxJK3*Y<AzZr}f+
zUdB*GYQJ^PmE3*RjuWQWK3-P*>}LDjJHNK=P<!-z@_w7TitafoGE2|?4DH(;f6Hd2
z<NW$tma7glKFegdu~C8dcmJ6;_A#C=#}rTGa^{+ursu{j_<1OG^(B7A1otV-^VpM*
z*RI{hs8_-HQEK{*?{*7%k3R6=Obd?<<0-8yJgGJ{Et|RER^N5?{eS-4=AV+9erm4O
z*Sz}6A?F-g_RP9_fX%_2h1YTW+xyS{GguUAdg@5t58(KD`xSH0hfcq!+fNxbck*6c
z8fM+9BBJp9-_N)0QzzLt_DaoP5^?!uc=p|%2~uw^gz4zWaxmt-ihkEnX0P_=Kf^xB
zpA+8feIHe#*c)@;XYF6bJ)7K8*c6xfDz3fjD06$eg{#iw$KGmNo^Ln0e|S>Dr3*#V
zw`ia7^eM8Nn)tnO|LWh*nF<&gVz-EVTbrYI-P*lF^FxP3*=M^Kb_-e?Zhv}X9shD4
z$I7|4x~@N5{BrsAlkM7vUVd)7<mDl>KtVXSpLuJWp#78JHJ@L7d;XMdx@KP>C+m{*
zuuP#d`YYlNec$?f_U!6^5iV1iqXl%|F1sa`t9*(_hX37#biR2pa~N~qE)h&F`+h(1
zecRePsjKsoPws4d%Htfm>nU^33f7Z`ze+wH?a#mRH@7{BYctQW{-t{7pXT4<G*p?k
z|FiJVyp*J<H~Xs1;sY0-bDGtXr@t(fxl{aV65E4K(rw#}=47{@pV5+J@am*p<cG<r
zF;iA#Of_G)A~~Yj%BjZKRP0W#`^29$fvV1v&K2MImHKqyv`uTT&Ew>r%)MnnrO|4>
z!1L1_xAX5^<EF9Wy7-@CtTt|59x;nVr!~e2IbA3%;A##KV76#Hb7ZNJ&##6F&kL@z
z&sbp>7bvdlK68SOo{;*|)e6dQ4q3WAn0!#kZSjAG*UB&1dmjn6KZ>Xc%IfTqc_)*`
zJS(K~Lg-qhko}*<RxkT=T(`^fTbt#C%eD@cH4CL*Siet9%)Xt+pOf|5Xv=Dz5E;>_
zvg^FFi>8S*d9XAbyr+HXq+<CcZgDdP*}dCr%EkW4JoZt4meBMd=*jE%Cw^Yj4b(AE
z_xQwCIAgC{_RLV9)V>9wyESrs7TkHm$*|(l@vZxsCG||ZgEnSO@m4*sR%7<1DGLi1
zU)uP^pYM>WL}k&@O?~T*pN?o+b+*{@$;xT@w?fVwIC1p&irce4__p@`j`-ZQ^wFu#
zS*{tD+ooOYwL6n^p7-z*MMcd8zWdWIPqz!?$zGKb9yT>OZ9kXT*=iwE`FH;rELct+
z)>b~8wbyKM<@AMLyX8+WRK5A@=^inupYg%djD$B_y1hVLhI!+^inf`nS-d@Vls!Lr
z)?M-B<I4sMk4pTlx+dv4b3*G<1|}vq&P$%(c`V*pH{a(w;vW#y6#QwcMB@ojyOy|Z
zokkzFCSG3>@oC?Jvzr!8wKT0u_2jbb{Svbv*Zk6hUw*f{yNf$^)I^?{!g^IfgI#~K
zah6`0UZ$wTJwc@k%_B>-YTunNT<=_Q%gX!6t(<FKS~_Qwm#@39LM3cVuz~ti`_i?y
zzP+u<D)_tDEuqI#a$c{)ue1vx7hdFNJd0iQ#^)qsa7rkjcT7Xw;@Z;QFrRCE>?NWC
zhaL)Uv03u&)RMRd0(ytDWp{r+dY=3E%vVv*86WCAT3>$uh>lcqphRebnk=XEw&QLp
z_V!aWS10sM)sD+bP?>zvlVugdM34P4Ik)5I{_T_NQ&$zf+J3`ArvFXep+0tvPX$`)
z8ti4cR==J1^sl_FG_P*K;xmeUSz3{=7701?eB3fG(&Nf;Bbyv<4@RD)wP8<~6w3M{
zjEa~;1!My5zI%2{cAu7wx!@t5t7?w^$CfmQG9B5qibL<fW`WH2v8Pupj<RxG<eu8A
zC()dgo9iNN%*}kehksLX+Vb{AC5M=<c-q{&)iiZCm*o<_V-KgKt?lKszphYUQSo!F
zso|&RUTHIa&5m*|tUZ3;YRAU=_XU3(&-t*!@|mZ#-2As5Sr#!@8<sHL%lggM{nS^>
zW!?Qv0TTT;?_Jm0z2e(Wrtf>--<Il;Gt7#r@$=aId*A-BEh(vIi=^M4`*6nZnx=xT
zrr4$3{72^9zW?#h+6|wt&b+*E*SgA=?oXEpoMd6rJ^%LFgvKDAN$&D94s4M6E-q#-
z`@a75KB1bMwYOL5No^_ipRrzX@x_JD`5i*vimqF@I<n|i$I}qk89x&y7BQMBJ-K&{
z{nm};kALi0DYQ?O&8qU^zFk+s^eXIRm$zDexW0aizq5(kB!LqGI@wEZJvvhUU0F(T
zR;0?sSGiw`j&9pFG2AMu=aA#h-+vZ7b9e6zJE@?sT6FIY#`W1n^}A)X3^g=0O%rUR
zmc*X>J<p|sy=W5Wj#m>zRvNaZmsg6s+OWO)j&Pp+z8|jsf5WfOj(@uTy@%%HnFrdP
zDrRIRDhBB-&(^zkv-2JM%dZxnKH5oN3Hy3G`%}lUK+$dAe>pB(^_e48>B$@Wl8@Q{
zRPMii-}oqCrH`lL40)^Hz5BMd`AnYF?qBb}{Vl&S>umi~E3#jUrY>9M$&)6o<IT?T
zal_WbOd)?|{ds%V3q8JcE6=|pU~|C4hhLt%UE128uv9JOv0kjv=6%=e)qXZ_v0Sx2
z>ej92pCxacS14RDjko^ZPV>Dx@4VCBr?bQ5s6N-Erz_-oqSn6n)8cB8_m=Y$@8+V{
zPkGg*fAiZlK~QdC!kX2IC*^K`JojVbk!NX#eWqPeXg6OsXU|UKD~Fe!`WPN`t8d>#
z<FJNjq7ygUWbbKET`TpHcR%y@9lvsWzRCCei*l2Y7o6o}8?kTMl=QD`%BNOk)|HpP
z+Hmoi(~_#Gd9EjSvS&wsvAF#3qiP*jnp}Qy`6u0NAD^67KDs6%i|gVOO_eJvvR`*h
z5-q)@m2vW`?!il!><-*d6s})g71DnuxhD1L$?N~JF1()*lec31v^h80xIQdu-5PlL
zq|&|=*VJdBY5^D7F29s2XV3o6z}$Uva)&CrZ!7=Ih2L1WbPH;}eE+R0KK#heR>lQg
z606dZOLSJbsQ-2OU}nGa*W;=G)IU7zII#AT*V7-fwq#AWzWdcA<LbAt%){s3PmAEU
zI%>3B=heKvv(q-KE?=^)NlC4Z;qir1ws`M5Sv#ZUc-tQ~JeuL-@=wgr+`;$$mlXoV
z0io+GO>$I|H&}d;w3rm+%bC3+*ZfMrQOPRvRavi#mFHgkcg-_Pbw$X;fD`AU&q#^q
zdFR;8*z@K)%iQ+Er$0>8<;a>C6=PU^_2y5vw*1Tu8eVR)VW(JSXEv<-(kp#fXVz+s
zldsawJXp2h-0O6qGm{mApSCEOY3YQBZWeGonLW?VTz1Z6y)sz`A<va}GS@W-r>%Qr
zICtkG&iS2QssgE}c_i08SYaE}?7QNS?$w!9{*g1>{R4x2r-dBp58G0zX0j$(@r#GZ
zuhzz$4AOR5?F>`9A2IJ`mAw?z)VIC*_M!Ruf?bw2PAMxn0~yu`3D2<fV0!pAI&@By
z{>`vpr<wY}VaNSh_=PS7U3S~>A|x|Kr-YR;>?HRakG&_w&5tb%4%944J;BpF`v&9d
zqUWzJPCS<UOsi(eT9<uE-n*ajhOb;HuNTM=W4B?GaM{-P2UU32<S^X-e5_^nXS?(}
z@o!8jrm4;nUVcl(==rjkYm+RW+If3yFpm1p`@&jIclG}53Xf)Q-O<^3aAwD%rJKI3
z^3Bd_m*2xxvbEexQ_8yL;aS_-u-GZ3!6%rmXgQ0VGE;Wxd%}3)>mlp=8=p(oUHqtH
zvQNEbuEmdlnVZ?popj3I5xOS+lI>4HtCA&40w&#Cz`yW!w`;IogR8SQgU7S^o)fS1
zE}Pg{p1l88=bi7z9=9g`XV6vf5))5)qH*P~zrR?;&6&?uhB6(Vz{BvZJ@33o-$Dj)
zsm2@cH|{@qw5>8#a+TwYs{1c)9SN9op!Bj+@{BFZ!upbhrs=HSwn#tE@xlB53}4=F
zGv1R{=J#Z;Q^Vp3p1%E7Z5bw8jJz0E{>px*m~W{fzqw(-FR!`QIr`^*Kl;dJX|!bH
z=g+^|ug|RDFD~G}XSaOHT%LrCW8ahVm*=164?5tOzCJEDaE0B3$rp4!1S>?EY9>r$
zF5daZ{BY`7VfL2xpSAJVEAreI?)PUl71Cl+3r(v%)yMmL^4o0rby8|e8dfpa=3M{Y
z>N92X41SLvyH=E|2RFastq)XA{CcC_Mo^e*#%3n-?|XkORX*uG!Q}7ahe|eGPuZo7
z-buXiyZ8LlVXmuF+SjhGtL1k3o}$vXOYTF}qLT%WFB}ed9>|j8`R&h!NLS0Q$%SXk
zrq$N|ba=8v@abEg$FqDndNvtuU|hrckK<oOl~?DPRde&7f8Q*)N8;4VP`lbHN9H{#
z8&Vl~#l`pjv)Au&iF!JLJLKxRU-Pvx-oNBcaCQ%z$~NhU-rg|gcjjBqC!eqTQF8O7
z6Q`4>7LUw*fBQ!hR`JDfZQuX-YGTlt>!p1QIaD2)=ULv~Ui+V+PiiXDk~1C)*88vP
z(d3z`HM8UOx>-V-*{{jQtlV7vcEi^H42-`X21rht{-I6&`0bZ`MrE%)1ncB&V>}@(
zyg2Z20sr>1dq2MH<O<80%JrY2YW}thS5%#UXGmL=Y@gP8blSI_Nqpb!ZvPWMlGO6U
zWAW=3T6cQe_G=V5vL0N1IU~=fsBy{4+eLaz6aW6Rm~59Uv`E`=@2h*s<x6EGblvwz
zoLZ&85o%K9EtFT!XMX?o*CY8Yn~vUJ?;W?jkjIHreU4_zwaHo0hF0D6t6$bX?=NrU
zWRVK~RlmdTL;E2|+0ws<N_TR-kq%qrcHqYE=nv258Qh#H{*d+7x;J;eteX{lPj;$a
z^tx?J3jG#Jbei5Q+qrLX-gldM<_C+`teI{5^JmuC?-^xFRvhV7OEpV%mDP>S=?rGz
zp6+!?Ltxw9G+UP#-P1<zm`>K2ZnHd|@#M_I&DZ;;>a6o>{}~{AVuiNMA!or~&1@>}
z%RIcqUh@5C5Y?01-@L1)#q(=krvEJWIi*^9G0#m~!p|<KbmH1F|KBT7$2v9l0&DH{
zGBVTjB4kV+E;*|eCh7OEd-0=b>l{ui2w5K3p~o4tY&lm<dAi2cOTiD+&#Lx4k!XFD
z@TKxvGWTh2wZ(>Cv*KfyF#T?RS-N+d?2lu&H|+6?`6sNK!SceK@kD{;i~AP({ES77
z9&7i`sp&9(E+EP;#dL7WtH77$;@cD4zR9h9S9$DK&81cCEY-!^_RL`G)(pDpx6f@}
z;4@1rrCqyM7i(oiMQpl}uB&qF*o$Ki9k#uFr4aY+zGR)brq|IaOaC|?jZiV3HQREd
z;qBmCokcsU#Fo7mk>I)$c=t4a#>%g0v+|al`P3?S@t|bHg^gAL3_92Ce#`#cr}R<B
zW{=e&hNK|b7oMhd`KKMl=P-yqxY%K*6f>`G&SI;nt}d63P4c#8$l(%LSN2mu&fnf*
z)r_FjwU&E4^;a`T3+B(TlDK*LgNnw5$}6=ex>?PfXL@)WUs}TwbEkRv2bs7B@gKh*
z5SVOwNQhyr#GR({-InFqyz}Oji%ilCe;pU?%yRk_YtF=wi|0i=?!OA}3*G;#QEcDu
z=<NA5j<Y*8Jf>@0o%`qTwEN4mIZqv3yo<}L(JoYOcI7?&EhVRCe0i2X!QE!ryt&g&
znpC}K-0%=`Tv_Y%$o0v@jW?2(a{2Ul%+&P>;o0QA(D&!vt@|d<I+Db*#%GVGXn=<5
zy9X1i0=Mu-^etJqe(lP|%YV!Iy8LR%PPn*hbGh`bZ+5re6{Z{LsHP}X8)pXT>C{h*
zH2zq$bg35?gYo<P`MzRar{*oHI<+(Nxu?Ltqseo)*JLqB%O1SMdi3|B>n$@c6};f!
zW8AkcJ6h;iZ*Eebmq8+@*W~COSEm1-KP7&}t71(Xn=_i9wW=)k-1#?iveUaYTQ)J3
z#=VlTU@8Cd*xo>-aA&a^TX@;>1uL}J8ed#r_r1z3TUgJjVv^4!Z|~q=YciA0`bRXD
z@lM*Sq;ob@*>IYQ+t*_v4IX=E)V}+^V}DI_VN>64r4>_}jy*em<K2;7U9n9U+>|YU
zI2%@nJ1?6RvBlJQhA5AUgLFWUYv;GE+r+k>u={;~j{log*XD>!<=W)1Gf8`e`pMQC
z{q|ZBk(RIS*8bX}Sv6_lSJ#=kDVLAbY{^&p&#?Ka+dAJyrsubB-FUut;Sv8oU+%rv
zw)oMYwm>+`cvf!4jC~Rfj1$f{&%Ph=)hsiV>9nm5cgU`EeVaA*pH3dw@SnkJg_!JD
zLz~;O)0-Zim^O2B#sdSkMOVzee(7Uwo3@|p%C!Yj{CDPO@hTa{J$tp{;BBu~>!{}w
zkN*DiI85!d&IGTu0apU*qyEf!&lbgPY1Ve>^ec{kK`T|u<mS&kUb{a!$xW`TAYAWp
z^>z2PY)fzOXQ=F~`ue#;=9=Xz<s~lure}S!LUYUFIOP?~w_Uqk{l2TlWeJbZk~JGj
zg)$B>{A>Krkk(x%YbbQ8YlYF=m)m!KuGey%x-?Si*3<tCLAMNF^5h=)_qMw;amtaN
z1JYY>J=y%me#a7@MJ95;D-Wp}Du=9}akO!RuX8YeHUIJ75=n(KKQ%p#`giyO*WXNT
z9Wh_!37<9ZI4x=TcB3j!=kC$}4AVHfA8DRqwq{qj5U_E*RaHl1=Yl&g`D?CDo5s~~
zrjWhCP>SQTschqa22Zma{SWsU|B>%oZlvWIs1o?8G2iTVF|UzVSWt=e*J~f9UP|Ta
zt~h&q34hp=JAC(LKi1CHC>2-8OnAT@w!L`U1*xYjGqzp`4F9SY8hY`|)brD>o>jh{
zx>h4{VzzQ;N9V+!o)>I%R)lP>bje%vr+E7e%^55`dAts)s?#Q?s!UvaEI1(ROo^J(
za!GA&ON)~$&P^~{{bkOQ3r(ySY8)mwNX~0BjyY|8rpnu5`tqv#+LElr9cfF}HaQ2n
z#`0`jv(aZ#=B_sPu#LBl<T)H&pZ1?Y$UrznRahux(w6|95N?@O3D5dUlYSlBY9kb8
z<#4oM>-WypOL|s&vMgEBzLG^cd-2gpU8}fG9Qkm?QN^|Qp4zU4@|{Y*`;)Wd?@LGS
zaTQ)Nr$O`Ps^@-bU#gB(i?__w*mTK##@}2288WtC^W_PantCB~zDCzVZ?k6~+*cj$
zzGun0`(5F=Cy#?VEmD(0gl2jNuR7S47`vh_X?xckomLIWs%Og2>I<Lp{7Nt_jaK<;
zS~4@>s%c>7)mY8suf`0y<;w5+j$TjRXS!pZ#zI|-rcFzImTbs*RkJ?bex=)4FNu0_
z_cEgft~WBVZarTUOs}<Vh`Db$*>e57T^_zVM%?aP!cQ+dE#P<*|2NQq;rrI#bD|4Q
z%1@bix6LxkZ|NP4U$ghRnlLVUSfJ#xQ73er?@2vRn=APOlftw-jYKakx*qm)`~4Tz
zHGDTVxIFiYyWK8&Y0Ap4Y&vW*8*EFzcC3t4o%L>;&c1EDs{`IA)iH`a`u<p;up-6B
zX~H=XTQj}*`(eq+o|haw_HNm_@LX2iMQ_)L3kQzAoEtgm6tkNorvh&$L)rU#-;Z0r
z*~aPhu=q#fstI9T!H@Kec2vv0d-NeiTj!MU(x2CU9X@lf%l`A7`!4ort?8e)v~6vf
zR=Sz_&Evw}^0dOe-}w)p6Aub9G4p1xy}5`_T41p%U(Dv}fXkPZk4CTB(-v0bsV8Z!
zQ}^*sO>y=2qYtMvEhsg+*Bo8BUCa4*!%z2UmpHGtwsDz3sUbpPKdZiUXS%oq)#<OU
z&q@E0r}>}3&{nhNwDN2PmiuX*Hs>y;A6?qwJykzt|Ef;)Cl_lf^Cz5~etgO9usxe(
zr3EIg(y%?%tUP!3<j(k+{?Cq_7CNIDn|rpwZsT+g`<d_ke>K*gp1(janpbFo%c?j2
z3kqvn*WYh4|2N}e`74b;#>Vo+TYclccN@=m>09;NVaX&**~K#jj=gceYhThG6Fu(+
z&(Q@7mU>>;Z|mJAlKbyHpO=UhuZ!*ZSvHsTR;|cj4ZoGJY|;j!R!5bfW9#nSo_X=x
z&OOs*-yT~oGh?Tay7q(fDt1RbVm5CL;;8?zq3OesRUsFIqBX-}eS=oj|C;zjL2yTG
zZl-ef<m&r(Zrr~;zxLzJ3$2qLAF172q9Rwi^pVXG``Ee*OJ_PX#hYJad!wIxN3P<#
z><2kj+u~G>lZ(#IJuUpXE+L!OizARFcwNBVl<O;lJ}JZ|a2FPCu9mKoDXvN1(Wl}Y
zChB6|dDe52cg=r>j+u9#mVbJ^`kM9gqsks_>r4_jx$<nzeS7qup>yloAIXmQ7HV@$
zne=q$oz1VztsR0j9rc9kcI{CzpO`XnzVFuD?e}iH{c@-Jlc8Rk;-u4S|6R4tU-|QR
z7b64r;h!BQZ%y)zt#TFAe${S!!@nZ`PEPqY?Z9rf@~8h9f`b+We3lm5_V3^{?%sa)
z^*0|b@)9{Mz4dy-_AA%69lF0v)hALx_lD_<e+f7C-fY--PcDFyrC#vroITB#dK%l-
zRx|1T>a$mFpXRYv=521B@0Ya?MO7!CpB~a0yrJ>5UF+AZeG|C(yUVt={#z_J(_{^+
z^64P<bN6C;%*s2AlKTVBY!6t)9rLz+A$$M(=j<s<Hj2tCE?+if-In%F#n&%+j;2f!
zl5AcnU-tOwxg)pl*U#bR)iipOrD0mHz4vR`w@%p;%yR@ns#5+gOxjh>T##FRd++vt
zH3<z97S1tHnZz1@Y2KH#M#-?HzkVKkr2XvBCZDqjr5a+7pZ&XcBkp6OYIE*tTTREW
zZ8`=j{2I!;PpLWFOFf}<;7$JbZO<qFecxFA!mH^7qe^z#*XKofk0hpSmM+aY6jzqv
zC4Y1B$y;yAHf(r*?mvUs;UKAnT^*l4UAtxc`JSjyLs(+`mf|x`HHTI$@7)pSXn*`)
zLFW-471ft7bLZD3-*I|kwnzO;MpSzL8KJ+w<GycXxOe+M1CRS`bF0`F62_5V)?Lrq
zn00s)(}7#kXD!2A98UF=Us+l&_`X1o|MNbXnVqw^yn1bB{JX_=L&9RBlYeYBzrqT4
zSB1)=sC#SI?Y{q?;n=mm1)4Kj4OQO1ei~p|X?>*f((M_)|1%h<&77IKymNxy?C$w{
z7<X*D-~X`FGcHJ^E9mh2^~<lxO<|d^wRB01TG?lj#%DJea(^GcQ)W|d!<2KNNh|v9
zcl|$yBpnPduiH1_<|@ga&P4*>wjLH>-Fg4sgFl7Mol6#~-gxmKeu7zc*wWSPMhB9Y
z)ND<3dfl`7@3)9+rw(t}_~`dPGr1p)c17K)?^i0`kFGf!B=YcN=->C(Z(mZeN;X`2
zCfMSos`23&j<K>?RaY;0MJjO{tJF-Fi%D&;ezAAE*~7VuCRoIa7;UoYaoNhYkt<7O
zE~Dk_v#rguJ0D!v@p`1TQd85M-6-TG%bS?Zz0qeX_vYN3mQ<|Op1EvAuWH%lY{R|p
zL>M?V%9svI)gQJA3Ov*%<#l^!q{o$)j#1gQt*1Lg<|s_+duhU1pL^%!=Z=$Vcc(~u
zT-W-|zx4R?{U0S>o}4>{$#ri@+Rv`5^DcN6eKnKYC~ndbo7D9^`{nMMcRD)OzyCAH
zxH>AHT9tD@A*=Gw*NizWnsd6{@~?!*nzQEm70$YJFg(<tS5)Se;7gWRck_)hAMDr^
zlTzk5aL-OK+8x`ucJ5ZYGJfek{+?s&f>IK%1Wnml=w7z)(^JW1r4hx6ye1|lSGI7g
zPEz%i`Q*9n^e6u0_g$WwRl{7CFwB*Soz?L6#Kk{;7dJoLk=Ul16{w~3{byFC%XIHA
zOZD=k!gwS7W@|bHczm<IpT{omKgZEekSW7AXjy5<4k@nar0j~;;}(C;Sf2i{;@z?S
z*J~dY^)J};b;ZiQO;WQu7tN_mO-l{yX$Vi;_=Nd>-i_cxsy_1q7CCVgu->_2_eRGy
zX|v$1oi~M+2r#u~CTZm7KKjov;i-rSlgEpfN==%{6<b_iW=gDNXue-xKTp5CFi%=7
zX3<VfmL;tblO+1XKYd%0@}tkL<PpzNA=m6gd0!q~Po+~6H=QuK?Yqum|G#awf4o1Y
z{@5>*Yevn?FJ|>scP3nTX*@-(!es5g*!br>DW{75dTIx`_|0`$sIYS9*I3_^PrkqX
z+|5~$qG-uzCnmN!B0Kl&jlBw+ezaVW(7yjNtG>|OF5Ih1s9R_0=ao;VTUU1f+;xK6
ztwY>*9pg1m>-SqXPM_QQpW(gV;+e8Hr0RFB-I_IfT5(0Meuw_Q1?fI~mu{Uco%DXM
zM0CX&<BlwiUdOh!$OjQp5~rekeV09r`_HiBc-({E(L9G9rfCL;-0Tdj+BRkJ<$3?o
z%>EeFY+aESEza_CEypuvTLppCom*plO?3mOwapAojpQkJ-tgs@Yx_O1ANwzh&Ycx<
zwBV%zf7kiM>CR8wwpPtP{88ej(4lE<L37%!eP3I5q3h4Sb*tZ$vfG_{$?UKzx!~4S
zM};gc^BoLL<(I;Kob-4rl<GTIefRmPS&|J4bKP~7kCm*<3bLN6+k14c_R@aye#d78
zdwFw1b$yJSl}=|09Pw`A-7<k;<Gy>xj=V9vyW6~Pde>pEm_<J)?=@Lk`O5p{DNCD)
zGZj7cq^8O}u6-!9Z_>F4lg36B%RB!WelmZ(Qy-JTb*LeT^^HuhlS%qY(+jpQ<)Y%6
zF6diqI-8J~aI(O4joZ<^?{oh%+~)~XS3Irr`pLtSrY2%5`1B@vR8ELVTl;t0_eO)O
zrY1d)J<5JqZF$4+ZRgvw8zr@;8l`Oe5$Sl}IqQ?v*41vSC)8-k#a$>iy6v~bc?omz
z-t$H7AHFI5dE^;2p<VUft(Vzc0pdS{SZ3-+_wD+;#b{Q+g7+-*wm3LNY`^kx`}rg{
z*8ZbQB}1m`osG8qB6js1<5Qt03-)&Gv0F0p&oQf~p$v0V=K1bqetrAaeF@JWO*`35
zoz}lQlJRiv-+Rh_QyslN{g&J5`go;sR@Rv|%jMIoQ*LH!q-YjohOW=f|K7o)!?Ep~
zvdrTRjKNF&XDv8#X4Zr!vsg1%th>?`!*b@*^eOgz>Hitd^m|7wk;rzmRkdY}b^gyV
z)$fz$Vz(5%SH9f_>izBodYsX6KQ2Dn$Th#qPcC-yqhKk!POIr_WumV3`%T;Pt3i8#
z)Q0cRa~0=A?l`Ue>CUmXmDdg0N>+4x|NCVyk@wr>SE1I`@i~r<V{9#^tmgCO3VDCl
zZeG%C#)yeOl?}OE4xCizcapdqC1f{o$MY+a_FWs@FWt1!p7Ax7*J83_<KCcBwNQnu
z*3*~1U5)m;bn>a%v4~q9)33^K{rTLcD`en#^iklfpu5I;USiYyE20(%NNhiJV6IDQ
zMVL}OpZttjMXIS&Tv>KDv`m$qu#GQg>ZblyFXeNw<qy*QFG-%&I$~Tm{j95sQ0-#<
z^{IP~nSOQ*@c7~tXS9y(<HWolvv2wdw;oZfcqeTebb6W|N4id=V~2t@yL!Y+8TOCH
zqTPpgp0<jfaLU7kWov}g(kYJ%T&&Lb2c^czmz3p)^8fsO*)%TTtGY{&NvFXqLnY?r
z-i}cx*K#l%xgMn7vp>^tM~nBdf;aIUGOM^<_c$zd)|Hr6>*~DO;YU<~@sBw#i_Vvb
zPOZ#x7I~_ZCz(+epg+T?vD@aX@8rM<6D1N>I$c~nQPCrWJ#Y=Tw(yDgFK_Ql_nrJ`
z+$UkV&cAHJd{dDhD;`uh^BYyV)eGttFJI~ZVZORr`(n}aW)?GNN?A<pWSDwKI_5tE
z|AohElqF{}9-nu7?txleuL&QOcLp-VbO`y}O1r#fQfp^&lIKSC`AKf~&$G|9oH9vj
zzpm-_({9Opv$eLqURQ8qV~UZ{YmTN*7MD{Wf4X=7-p5dT-C&p9mkz#*d2#M{#1>DE
zTT>T27yfuq`N>s7ex1EGtBt1J_*oPwBj0dZ@;}3`{+#5?9n<wYr|1~W&7Esy{C4A|
zZ5?iDnGNqazKZgmxhDB6G;rg>70x=#yrimMO_;nk`O~S1#RvQSEzFZer!(Hs%n4Yk
z^8R6{SvlLBFU?+ux24E(Zn!YLR_{HJ=zGT9RS}g{P4}Mse*g9M+>eWeFY2WKRqU(_
zU0jqu$I_wI;PdOJr<AP9QgSw*Z!>CJ`)n0Uj;_iY*YkzT?#F)gKgjj$;nS%rPt6TA
z{V04oocVHix1FU<kDIG}gR1>Mh1y5&%@UnWb<HQe+r(>Ev}&qCD|7zJP<wV!i<Y&v
zAC|WSu&>qdyb(TM%dIL-Dl1Lq@ag*t&nZ4HiRu)5#HR2hqFa8p+~Z$=l9StKEmhEU
zTf?vR{K6%^nRznts}Ces`>kDOp5Q6|#(nSG#Qo|wa__Uw*wpFs=KEUvqVLkXGg*Te
zJ!R_+Zn3RClcms`b<Eq>HrJ%sS0U{p`z#^14G9hx-dFco2IuZrpJ`$8^>nxtt7DMJ
z)Ry^II#aB<HmzDT*|*y!MlEHfvs%n^tyzbUI5#{IU%`7mqSi#%F*@AY;*+|kN}J1-
zS|d$`2G?&_mhjgfmt7X87IV@mD@)|4*$Oke(z^1Ck7jDIa?a3o`qRHP;>*RO(>*g8
zJEkVAP2$+P&$f3R|2~<O4LVPLH0b2|+t*JqSIBzJvwE|}N~yWGgBjL&v+kd^Anta#
z83UjHJ|pcT-jmG&W(IJbS4&=QvEk5zGZ&UWeI37TxktXY&aL`)%r@uF>YO;B!MgUP
z^48YY^|z&LWY)b4nrdLwSsZjOrg80-C2Fdge?^#@FNZ&`D3tcNxPR#>%^NGF`<_4k
z`!QoqQux#8JO3F>cV4{_#I%Ly&ci2>YxUy4O+WK^;-(M1-{$MCIjCdn#dmA^rEBkk
z?#Lup`*y_1bN;-kcvWu!L+;uGi)|0AHqq%(TDC%wZOP^I=cm87Z~d8;Jkej|(pg`d
zoLbS6%{m)*Gag7)_d6o1yMtZqk97Y-!xLN+oMz4uXIng5T>AGlCGV)TB}+S=II`c^
zzGG)w{!&ImlZJ?U2WD%U{Y!rS`RJ-N4b5j#4f1CuIyi6m*SJS3=3=abwP=#`nTd0l
z-sU}t&wsdG`eE4C3rh^d3QAuX-~Drj?@C_ZhKpA{!izSpbSgE!n|$&o!~4JH2PfLH
zb$Zrrz5V?8Yj*$WuiqNh&wP4fVq1D(abb4Z<4=eFW$JNt+gg75wDjMGRSqh`T)nO*
z14Mgvg=Y&MSH2;fJ;&MQfSqvkz223+=O<_Lsjgh??mY8aX87sAe9xcTZr6I+cO02{
z+Ebz1-rVLtL*qFK!HE;9UQdy;tXa~tsB#U*4hQoc_xtYtXW(Hi+B|vg&;6%=3YhjB
z?Myr;%Cy>1<;<&`Uu<i8PwahvbnUO-kL<LvLW2G?2=IMX`V}>+>1)3ai-ro{JT@)m
z&vO?ibH<%H@GHE|bzSb4JAV=;Hv~CNl3@;ZI3E2o+rwz-w2$od`}g0wuw1Zq;wzRS
ziLY;y>!faf+;}JEPfgN9iwB*@wyC=%FsRtBtSh^5SYYb&(<|fyJ{-|uopU1n`ik95
z^;LJ`x9-?3U7g?j;pz_$Z&tl8QnP~-4%hZ7IbB;|qjTn7|L58V7hhd#a&>2E63t(7
zzrn6-=YNJ8|4BwFEBW33<-c7%hc83)RPOaTdY@Jquiie*JTbR%|MtS;$7~EmR&HOp
ze*W*!j{1rOFRz)a<wH&!OrDyQuh8o6DP34l{^aw|hsTOyn%(wo*?xQa%ziJARi10i
zS53WnAm8Fxt=qBddAu7YP4V@Lbw1bf+H1O$jg(gq>uYA?MMj({VHXZgoe~l8e*4~^
zZx6n2(Dh87s<!^@t@A4sVz~c2I20K0;q~=vnwGCj?E_LL>Cadj%zRnrq+(cC3gctN
zZI}Gl<{zEW$9c?rn~hE(r>F3|mhP~vTw<G6@9ryH_uw?wAswlf2@$U*yH*GMY$$1c
z{x@)zp7Xbvn%CM*mS-fjxUE{0TJE%F<qeU^)%TuA$9=cBcj(M)Rg0&-%GRt=I~z1D
z8<IQ}eqKH&+xQ?zw3~z7NQ_hOX_sSx-x5LZfZHB*GpDSd;py`)$!!B?i3ZExZ_?(M
z?!Vn<RkZQ+c@5s++PKy)&x1cN3DswtsBV>2Y<^0is(aOC?SL-3i!*MpNNriwRQGY)
z0p9n`k}D_7`{?=M<ujAb-B;R|mKmlW{2@3$*d^P4N@~&B=o#1iRbwV!HB2~qXp(>8
zfej3L*Z)<lKlVu2bgJo$-*Ma4a<1FA^l`0T$V#DxX$)DV-!rT(pWuj!xWuWd#mM3O
z;?0sft+(rE*F4%9vGi2*^s?-^wx(P4WN$98{h;_vJpEJc+xRV+J6#1&Yd&7)>CIrO
zUz>jWSNz<K_WU}hrv)2Wb*{L1``Yze%~6Rz+O+pYM#>+L+LCFh?(*zz#uk%U-f7p<
ze;({|o_S$<tK7`REHj-?oZ{h`6(6ZMp;L6<tBvO$R|hMzY?V`3(`nDBS8TbN!LJ}A
zs>ep}nZ`yNzc_)ZkFE&JN)0Vln!-BeOTvR4N>8|@e^@@^5%9LLRH-}>apaiHv|atC
z69YD|%y{v9Lhm<o@$cIdPW*9wc=_6;u7(ZTo(`%fb~XIYTzu5wZ631$S6ukwwc68c
z<|PF$H|x2!ZSH~!n@K9%*6vD{mvwC#q}@KsUg@5Y|GD#zedo+hKe4RPnJTsnEZpKE
zYfX!tE-snC$Q8IbaE;s2uTObG-|bs(l`1zaNq1%AulU;^x9>BppB)&?CR&)rf8ytS
z=Zd0<EAsVred+x5W^eei3}eqP9?Om9&0$Qa|98*q@tyY@x8L1nsNOlVCR@`olHr&2
zrza8?R$H77``OPj3jDX=(~pkV=QK?c)7oT%Iv!o|TRowRbMM>aYT>^YN1wCI*sAEQ
zSvqME>sepfTT5HtzLnw$ipp9Uo&USEVvYOk^Zu-EJ;&yy8Tw2-^>OoNjWC4`VwX&&
zDimi+zfYe2P-4pIW4?z^+uqr*>72~92TZ(-XXMtEW?Vk|aA9$f7w;vH0*MRqR=Y1x
zkm}YgbxJc#4ZRtbJ1f*=N<i28&+@vdeJQc5&dV;TPUPA*#Y{nT*Q=cONjs9BCd?L9
z)d~z$XjQQ=Woe3$JzDeF)+{b^v#?d{lxVfIq_|t+s)y&^TVtzvz2vLpeV?b2NAo=A
zrNnwqchcOfRkcDjOr&d-;)<ZdSA{eqX5M<juk~d1Zut-QdzNz^F_d65@Nt^ynPjK@
zktz3)DUaSG%X~&9H`StmEvte~FKFWRo~rgmWmkj9kJ*V6c=P7P?d+S)7JGuVx8cXW
z$F=`vygPN&n>DgSa&4|@k?hRIr1MXuN)@MsIG0}k%xiPtQCNpo!z!l9C#~iDj?X_M
zcuuV3SCgkHds$N5ng<*B3)zJ_5)yB1HJs;f;mLB;+vVcy7V*EY-{*(t9<=>&)L-~f
z_u;cnmqVsZs$^gL;y**++69hW8i|2Pzdv1kxmJpQN$@oL&7bp31zv^J%+Af0p1<O;
z*e1iynfiOCf0(tm)vaw`$c^SGhcJ=#Tx+kdpW>3ny>4EqU|Yc=U!A<|v2P3V_iumL
zbxyOJbDEfh;Me*jhI=2anHGu3T)OwA+tvHPhZM=CMQcnYbBs3dCTcP)Hb~r{vfesA
zIk9Ix!&{b<GdMS^iHB^}(U>6DcFSVp`lU%bjruAlZT0Hdy6SPUL+<Bp`_Pho=@XuB
znHlPRRjBBd=Hsj4(|mM&rQfbDTCkfTKW+PVk=UlUF-r=rCULGSWs(V*I^p)Nl?4y?
z90@7dr>!pB{(chsk%fDmrkHGPd@AxQ>gI{CS!pwGWL!Ao!Bb}~#Cl%(V@}B)l_giR
z{d?VwdgTLdvH1$wo}MiB`NIijEoa8-@#dX@Q=PP5rmQ^kV$-y;ZS6c>(-_((G?eHa
zdzYJgc5O{Md&V+#ktGUg%t;#V*PWwph_@{l@R>A;tKr%-))~Bhhqiy|R(g6>bgtvY
zy~4A%mif+hj28=ynY@Cn%WRe2tk!pnY>RTrHUx3*v)pp-vgh>nqcdG6O-&bJSTSXS
zqx5Vk<G_0xgH0<9HgEhiGc(Amxo(jFV^+AS?rW*(abK_dR$RSexvV%&v&wpP+``4D
zv(;w1%B4DOkH~zjlKh*iHG1kQyH3;0`Hq*(^r|%vZM5O$5^6|0X*$C`C}f#dkU3kY
z#?nryj#D4_SUVovFi8xuzL_bfz`RE5$r8rLp632XCdf)QFEd*8itCn4^O{}qD-%|o
z+MJ=b<lMq#D;(B7cZkxQ5_6-iY2}Q}9rY>^w`<(q&Qpu|`6DMy=DE}6g)TWuRyxdO
zEX;0>xbom5hsDZ>H=1XfM4H@AeN=hxx>&Qm-LT+e*K?+Ai>+6PFdEJI6Sy=+R7owX
zYDVu?N3HlmnRN%a+XPST^0<^g^9{p1l?4eNp3H_^s>_%nr1k~5a`Fal?3>vhGv%S<
z+<%^tn<l(jGFNQ!e+HHAE21hp^dxJouNgiQElL&rurKh+hfJYM8+M(n{`DaH-zB3;
zwKMwri~HUlt`650YM3-<iCB<`*5sG3jvZ<^nlJU{<g~`lNyR5Z`|eEtx9X?#(T!R+
zrfjI+nQ)3jIs2F6uUmId9{p6PY`FS*wCtA^A}YV;J~msUaphO(*T#^L;7h@KBd6A9
zz36KY5PMqtxBqF!%)6PBCh_`yPdf1G_}qt|mg*%P*|tGN-uYW+%x0sgnnT7R$KUA5
zaWyqxeY$I^j(&AR$@X&-*e#Yk{voteV}{1g6zvOAx3<-0Wi!lf{UE6KT{_t5-n~5w
z^PhEOYOZ=R&GY56c|9vacde*5V6~Z8xn$!{yTbyz?K{_L#q8qwdFRcRre_DFGONQj
zK5g(z>24^!*}s-mNqptK)!L^vXB`(6WLbWl+elb)V%wCjg*yw_9@qFhstJlYBXqSr
z$meK<vjF$1`N2!M@AZnw@N?expRUtA>)nB!KbeyLGd%q6_hntBr@8x;XYI=8|Lk~p
zGFwZF>EfP+C9|TI<^OT=-Qr)X@i6pQyUx^_^o~asRjOwW_k9Z4rKxj7g};~A$zi2;
zNZ7||Z|y~ndCva%WU<XA&lwCK8T^m6{n+$xNvvnle1W9vhNnOCEIB*<xyjd?xxQ-Y
zrXm+Z^rkPb*DY0)sVe#Y_<3?ik;SUChu5w7oF=6{D@=OQU!Zrdt(R@0Rm`)Qs#8Q-
zR+ue3>-hX~l#;fuiH?t{T2*gqU+*KwqMV3(Z)(+Z+ly=4_O}!on+1jbirl)Aam5j_
z+w~sPb-7If{{Fe0xAVXxky%YO$6C+E3S~rfiV3w%?Y}Fmc-J}V)bk6!BRkEG=<y!k
z@Z;oiepAb|_C?J?GCwAn@`v-U=oEgklAAO1%L;{eM{>T3`z>6}RBI&p^5!YY&<Tdo
zIUy?(D);G4<9W%d&7iPDsqx;!V?EgcQP&!l9@b8?J0jNIAAL~p%%(N(X33h&+!pCM
zYo*$)lv2TS+aEnzB-Q%w?8o@FJr1i|>%6MwrOA3tDgEKeu-|0LwcbS08rdw4n5j~y
zqb9%LUVfKVP4koIbDdrZ)tpe**5mfG>-Qi1#j}j(tBC&Zw`=!&?>1Smq<wk3N5dIr
zyYwlX=~J9G&r}Wl&+vp}H}jAAdDphrq;H&b*;-gD@jt`l-`1~J?!Kj0kd-N1)bPaM
z$+P|XYrJ>z9bL1s$oGxM_Kh~P-+%d)^x`b*gz4v^w&x|3{40#s5SUiC{mF&K2#K6)
zR$GN^cRZh+W4BzWTjZJNh1r+pRc%a6``&8dCb~YhDo)&{(4>f=F(lGyZo72j8?)!n
z1vh5+JQ2~b3X|x*bEDs@I3s6f7guf78jm$w+aw-O%u-qCab-!r#MJ<uqvhsG2RpXD
z$t~P_;Q8OlQ|<hP=eL`3dS&lR)!4YANOI-=3GGR{H%-{GAyD;L(}Cab4*XeAq_ydI
znNviQjPwMZ@*BMS_inuKR8#HujfmcN=ghx;YMB%>@hwwWGPAJSrUUQ%X6Ntie1H7)
z+fJcpDr=TX%?f|!{nFjc=D}*`RgDXIB)872x$0e5oIUT}@9NuJhE{4<#TJNPuC)tz
z(xx-}SLsWcrS2D_6E4-Ao!NLl=nfOdpWS6g<n&EBPb`sHZn1SM?;q#qQ|2tXpSR~h
z;ciaRD34RCrDGl0`~EX@NN9y|ROUHz@qV{zPtK}z4rBEGdTqlumru$O2Q$8QPpEsr
zdviwqj=#BWoiBqY<-Xovwzn=&F23X0?YA5H9A3<?J&}90_l<t;j{A>qe>i;R(4vFe
z*>l~tY*_okv?8+cpWye`uQ``aQ4G`b)kxiYqlxiGcP;y$B!0nJTUC8OimPARZ!Ng7
z!sN)#t=Yf#a4j-#R59`pX!~?-VfDn#@q2gLuYF_ixHITbNV`hAh|*Wq)%6pr*jpz)
z4mN)6Ke_nJ&m*NGSCzgRsZaNb_^G*3k@I%w_Itm!{jj^S_>f}pwLPy>S}k~;4j+wW
zR6Niv6BNMVl)v)%>6d(A3!i+N<S5AQ5wv^m59Qmp>w7Yu`#3rX-4CeTaP!7HlZuX&
z3YI=rX(p#-rtjq17iA<^P-b?!?)l66OfJXG3ZJjuzirQ22Cw2t?0Zyp+Ltcu6DsA-
zFRYgCp8NC9hnuMiKe8;%rapB{|7}vC!tf?e)>`7wt``{{9L{%Ittz;0<ni5?V2IS?
zW)90LHofkh$xyD7|K~kV%%Yb`i%!KEN(6D3ZCkwW_WdtEkF2!ld4E@Uq3XXYU#pAN
zR29$i>%V?^*+D*9!a;FuF5C4DKfnEu5InZf`}D%ER(CdU+}68SDL~vdS7hH7<MWe!
zgZ=E^y;$b8V~yQMCvOI}-_7@LB=OmEsGSI6EIQg@$`T^==B)Q?DZ8^NekzLbJO_em
zbKidMc(-`PvR8cbPOo|)WT~B$m31<J+hk|`jO*&%Yl2Qn+8*+(Jv7PUx$vxlimgdY
zSL^Hv3TcvCn5h`XI5)H4*3;^V;xc?bXUr6fp8c^?X<k;u{-5Ej!heQ$yu|`jBX_0l
zTx#-_Dbr_WwdS2W+ZN<-{#5^@E-TF0dsfS9ndUw%;|UT${~1=tuiL2S#&CYZl0`lH
zz1k|4@J4p0S#C=;QJbgcGU=4JY*kk`pGYRt^GUC)AMTiEk)Cv^HKxh!%q&K)&+D)D
zWPajw6e@VM^!CO1S9bqomR-{LT03Z3kGtC?H!qVGy~Xc1-k<yXQ8r>_=BeOEtcNfC
zirezc*0k+W_}$a*)k6D2y3a*VI5tr-*uWqphHU~{qQbM^g+Ff3`(d-k=<u@KgIlHU
z<<)nber&NX)o9Yo+j*hCT^H-m+BS<bQ_7K9NN}eAx+LC%)ddIc_u9li77Eqg8^rTf
zXkt(XtK+gKvgRxuOO7mynj$u<`?c@wg4;)QCHl=@v2>d)uK1-j_s^;a2MpJkJ&8N?
z!fVIXjTvtbS}c0&Ft^k3;ST}huBit3e`-!hdNTfO7W*8{+j3T8pJL+W{|x;98D0cU
zP4c^#=qj04I3u-GAjmz~^XaOoy{@;ro6P0Q12*0$zb*Dd$7SY@5=9QlkXdo_w7T}a
ziNE@&E6MsK=bWa<ysx&|;oTV@+5fFxw{F=-IjJ{q7yWIqT)iUaS_cD1y=nj7J7QWs
z!iI@UUQE-!cSxUU54*6?4`WrGzc>FgY}?Bhwo2sS95%KkrmiLD&hhB9=(kOnb5*l!
z+UmB%5bb{%Jn=_bGudZeunUc4iO*SlCUYrURMf0YZs$neqw~xXc-NUbF(11g7OHf-
z`mn_(Lq$(5%cGKhT%iGuJ9IQeQ$rml3Z*=<KAGxj>9NDy^G4%|3;kTJK|ZyrF`K@8
zOPFxt>B7~^SGc8Rzsz{%bX~fBN!<5a_r9I{Hm}Iwj+3A2x|y9iSFQwy?@SCj&o{A+
zS!j~OihysL3iWau=cG+`=T(W!_E!<Qy0X<nx-mhr{e^X&?~bEO_Eb)8oURwmx`ttT
zQcQdKj|?r=V;|Q(e4#a?kmsT3=Nq+uwMF0UK6_bz@uUr0URI^c3?|E{pYD6%v@tNb
zFlotebGE}GHr_4zybV5YbVIeaubLdr<N3qap-TJQ_Pdi0&iq_G>6OY}-y2u#ZZ+=s
z+;{0KTV|}Uf9RB_TNd?R^!OdqWWIO%rU`uWlggFrz8zIh`?2e4)3k{iKb$T2xU)F-
z9d~E5?RaRE>Kn83RA7MmLze!6Wwo~^zS`GwR<p`VXZz~R&0TZ)SN*%N@#2-W2J<u@
zYPfn`xtzIW%RA|lCFYT#H$==VZhQ<q6&2h#T|?LO+$_fb3_PnC5A`2C(aySB#=9qB
z*<&?ro}HCT4OOfKmVXe@e6eI(%IqcWX9exl{q9P~dq2we4mhmdzSZ}e-lCVUtd^}>
zwru&tDS}?_f4g#(>~Zq;6xKTStHD!#e!^~FTZ<zf?o<XhKZt795VQ19xcX{pgGcC#
zINjZ{x-xEFKjKujY07B^dlgMu@calXhbi}|qLrUj=PWgd+{*Gra+kCCtyL@D)vTN(
z7HA~8YZBKR4~5&`eUImq&67TM*-|4QpiFPhp*M`YC)!vhE);yDb=k<#ENpSo*XzDt
zckC%@Htg$tH7_bcJ>q2w57U$lW>4;a?`=K0yZiUWvV|2>eSCI$DaiTfGCa{<Jxwo{
z+nRR{!<*;pGq#_ca_P$vfz$%`;Avhd^&3O2x9<7tIy2>p-b#guxz#NRciPMjlm%}&
zZ4vi1Z>Q&?$xhC~k9ZDr?^u|m?a|sfZ%?4aq+->!#MKPUZKj$^``<{;=(?P}d6hw@
zU8<<9YOJgK6kZ!|!Fdk9yhQd*&0e{afqBWDa%-35BZq<&T^76u@jZOEQ#0v+T-4Dv
z>9#Yg0z;PAW*wY+KQLN6e@f)%kS@JxFMT#Ht|;c&x%$PDsT15T9ewt5qx^pcrAarM
z-!Z7?6!)Dgn`!j)@r31?s;sNsB}0CP1bixzeYBicXSJkNfnMj_TeFt-#jL7}oLnDe
z*<Ifo8^{_SdDKZu^VO4PwS|jTm|PN;GIH6qm_6az4!6?VLEFvusnrQSO!Jv0zUkVQ
zz`O3zhNom(yC?BJn!KS#(&ORX*T!qJ_#`$4DtxML$U0)|7VI6Y!sMhjF?riE@e3Yr
z*0}E2fA4YP=f3A1hgF->Lj9a04E0^E&y_!A8a!d@29ck?{xb+ztvNK0LnUgm&4w8_
zcDYGtZcv}#Xn(w}aQ5S6QL3Fk3MYCd@OG?_w%&Q|+a7uDwwZH81h~8ZtmF9pty|3b
zYUNY+kS~ipWkYL3E?MimD_`DcG0WqcukJcWsUuDkOc_cVu8BXMyYlcVpR4{yo49UU
zs!adLJfl}ecE*OSkMso|T7)E|E9y*Iwyb3;L*!v~LsOQGrUg$IZLU~l^s?BBtF7g#
zlS}F(heJ-kyrvZBE}eSu>h6{732xSD!u_J2E-%?8YsmGScanGFP&|8U!;Z|%NUx5C
zYdbbKJ-B+1Mc|7`i)qQ(;7MmqYoh}$UO5`1SGve$&C)5#Ir|!(9roD!bD#0uRS!C1
zS|6!gO|bNB)XDyue@o1ZDcI1Gd5z%4qBk*dYon%p>5$tqZ}+m5RtHtr^3Qm(RdY5k
zZ`rElQQMoA>i%o%TieN;|0ku);%SU^mp)_9p-1<X7*08EFE?CKY8f)YeX3cndRoe<
zV|pIUhb<&aH-!X<>-=YUBGM(ixc8C#id|{*R~W7Gni+1qVCEvz%D9A%<e7|nm-c*e
zv~PdhclqcKpFP=0AA;@A>-Fg_6)pF?c}M!S+(VnRJZ^y{Z2@Ibyg|{<+pjTCD3x{f
zSwA)1pQZAw0lzl??nuF#S<1J&??%>_uCq`Uml5t)m(&fO{BQ=drm3W-RQ=I#Z!Ym`
z^2aQMQkQ?W2=G}NT{3%$TvhXmXCKqgOkA>Xnxdww_FCW8Mi%bRxohuld~{>O46h*J
zDU%|P&#hf~Zn3w_KbDrWTzUJax4%?fFlm{2<|(6fv+u+-@7%F>uX>J;M9WIkt);uO
z*ss49xUM6b8Y=rs%B57V*2QSgZH=|1U0zYGrRh6$nT~}_%m2@C{mGpgGs{IEf|fmc
z#uF9EDrdDpl{?|>Vu8ia^u>jDlw3B7Qwx7Bm9eP)KSO%HNB?yDX_IX}{OZz@(pqKz
z*<+O)OOKn{vJOv^?WO-2p6qgCo>2E=hth-R73)(j>PxgHir!tfw6kQ1N}$uab62iD
zI?ma0w(aPdpbkYArmWM!DgiHp9th^@OTUY}xKljJX39P3bbh^3EwPwpi%A-<Uhz#{
zW#y@-wCZu{mz_JLJdbXC+h@<fWbtHzYQx<x8@wu~cC9p%?)53^$(Nqrs2JnR(@_6T
zL8viGw7)C%=khsTr)PRzskAi<pYb#(xT1OLshXLpy<eYAIL^V%^5WW77e|kvyh|FF
z@<QU}JB}spNZNf;>wL=`rqD+$3Typm>i+WW*kH=S(X7q<GxJ%=)_qb58`2!LR!wq$
z5w=U^O2B1>v#DNH7nZs#ir%->D%>;Rz_%x++n@bs=zP6*>Z7hOt{`77XM@!u(?0DG
z72JN<v&1UntoOD1YHQf~4W>;hoz}G|E9#*1f`VP^dN;3LdCTM0za@zaZr@>2`?!x=
zT~goV&}EI&zL%UgiY>n$p|o_8O@iRYty<DaCQ6IGb*@^GxN3RKVI8l7)4UaXFHR4?
z@ti@J_x!!!?-%>;d;j~AXTZmLJlrVq(4!e=r+>_G`0LdF=(e**lj=;b6Pt|9C-?q2
zI>p4MqdTUse(sM#)d-KRDUuVGPBOFG=pM6H$>K-Utjufw8FpobPdJcld+1|uK}}Ve
zT!Ny+lBEf=)7)FjmNxmDnRd-=@Hoye`9DLc%Kkq~H{?fVv2Hz)b8)`Etd7S;i6v9F
zniuHYZ~y+s@RI7I)55>{l|Ox!s!e)otfm<`PvKfsrqrYbR#zN)RXcK>9tc$2nC~gi
zF1Y+7%gUf-3mrSv3$BW<H~)4?H?(cWgq10Vy5I6w|8#zF=VzJT0i(O8O`F=@l<hxu
z-@nE${_vbG<|n0lGyXH&af#BZ+G3F_6>{R}l6`Y^58QtF^`AshA*X|a!1T{|x^{eH
zKBwHUn&Z_KcS9j1tDV*Iq3gE2alE}>CbvzpRV#h_e(g)MozE;)**3qTQe|ChbD&h}
z1b3xPi!-eaHYy0$@vGlDbmRDksY?o5H@&{gbMF4KM435O`?U8>5m+_t(_zh<$8GtR
zzp+iIlFC}C(BgM?X$PaX@J6!@;(Hta=v5~t6=qHm+T_fn?<I8W;@0I;7!@aQEfm?R
zb5diPufXO?$G6q1K4<^4Kla1*Q9#g|t&IEkt@-fs(~Koc!u~D0@G0@hL7`pKi_Wa!
zZSYS{y#MlD?S}btc)T}UoH5DltIf{Of&s=l6PKU=(I#3sD|Jgz$d0`x^EXX%5RnLP
zORAmrx>SC5l418-bK%)aF?-TqPFXLd<rb8&@r+@h(o5CXVl({Yf3|*iezfD+HMy<H
zUKb8s+IU*|NTL|iDc+-NZ}8>6vu`t<Br;Wf59`y-O=hB1@9f-^&m7vk)c;TKx2Dsf
z$|hO|ml(E&GPg75{byiio7ki2)zw?~ug>k?1eL@CtSkOAsQ*ilE8F2M85^?pz~4(2
z+s_@T+wUVNEUK5O!>V8Y<@2|8Ne9iHzrJ2*duiDD&}jecv<+N!ujW0FV?D4x|1{Gh
z%b?f;oXXd$`Hi0`MWsITyuAJv>y^++{~4y*a$T|E4K^v%n;&$yL5$(^%e~v)_nb;o
z@p>=p%EitQ8&VguV2?Zx%fAopoGhQ0)~#ike5$o(qf)Q3W0H4P=uhcSdVKfWbSkPu
z_E<UavD#0&{UTrR)uIsB*YB@e#F{!yTG#YZy18(3H}{+ReTK!>f;#JG2d?7}<*h#C
zH}RL>ybn9o|1RWKn%UNn$9H&wqfIyOe}?Cx9x;!MLe-Yd+Q}1NzBs&iiNfTE#aUl}
zyQe2BZ$7XvQdYF_NmKbRY2iNwck*mJomUv@KFTfsCne;vfi=hdLP^3UuUMWkL#Gu^
z8>DtKF8R$re}3;1ucHh_&#HFIz7GE#x$2xk=CrG(CbhP^{z-TSy}A_TUUEIx?a+PZ
zPqy2S%$(6UMPv5`Zt2(0{>)Nc=8(924_l;=%LK7KOPd-j0~2y=d%JGGZv0o66&C!~
zOVC$8wj<ZGeNiM!>H2RUxMm(&b<%HRC70f>DZi%OZnB=+cRNOgU(jHo-@@Q{UIx8$
zY)!lPq7HueQS?;X)GT@Z+jkcyq)weVE%NHmst|>hpBB#3DlOnBnO^F&``a#qZ^`?9
z7s?+w=~&@%!dF!_DZ%u^^p#?KTG5vyUT#kDoZA>R(OpyhthxSYnTOv)gH8Q|Q{$7%
z`ZZ^GeQBIx^5tQOyypAtUGJ9sVUsp9iSSc37FHEfSDn(zvU0^cjcxp{Y??PNTye2u
zw>0_?EV65@&eK1tCnejJ>;E%Ecg#HH?C2RDYML^&(q$^Qzeq6S(-6kAotql;Ug_Qu
ztp0vfPSsP!FUZ)!Ej`#a>eh~zEe?-ZRxAq&I?Z@eT*~yD-GZa)%Z;C(+pKfn&PwSN
z`>n9nxa$!@K?}0Pc774A-SNk8#lb>jbD5Q#9`BaUS$~EhWT}No(V3EKQ};jKqbiYU
z^r5rnpUm-puKH)!XI$2paMaz`)M<gxijs_T-*q^zRDS<=%w&e<p$w(4<<YZWuDHP;
zC{g3zl{8uB=F!?k`etkTw5)D*Ea}pkEUYDDk}+ZJl#S1)$<O`$sOlqw%8EH>WUhO+
z|6BQ7*rHBmRhGblmsPJGhyD7q;AT#~tGln?tH~!;P4Mv8w=6(^gT3=frR)2*#xdwx
zl&s50{wUi$`@Ngunk)Uu*YEiTP4W;`KJn$w!*i)oQ!A&1&IqhKm%nkA)zT92>w;gT
z=e9nYWFPq4xv(bSp~F;*htJsbLTq}ya|1lGxn9amY}GitD&|zr&BiAS<Nq@xWS(*L
z4qlRfWood+sT;G5)virisCVu1+ofwb#P-dey7#QT_*<SOA2mG}7~R!MIy1G#I->ip
zOZ+mo1cMAYD~%%0rGMUkDqDJgk+GT2vSqVZ_U6`kMhASoDzd>KGE;TZ0e}D1sRdmk
zE5Do&_l~~w(q2?APhX^PhLWIxf#*d}krS&7`2$x^XLvp5t)8G-fklAO&IA?VRK}8(
z>o`|jHTPR^T2<&`=#{6F!`wqBKMQ#hbls_EWyDG?gH$(hw_s_e-d&~#-$WbU^0U<{
z<1D_^vw6ewV@>BK#YkP45Z4mud*TSwW<#YSq0`f{CYS8;J+VV!heAc`R%Yd8T34rR
zz4~ZrR$QS(MuUP7r%L?9Y~8loH946Tn<r{}oKkJscJ<p1R=<n8XKQ?S3t8X78(+U}
z_tDFS-Agr$*jBu<(m8BBZLLV~#m;@F0yt~7X3CbDub8b7pZ&&nqj}$BpC>*?RBLag
zHi@u??6Q?>T=h<7vdA4>iKCBN))-A{nDo?l*2^O14NFh|bTu~pD12CMPPgikvwLjL
zyvkNx<TaP`ifHqj7iVq>yzH)C5x(=@IsKmF43B?I^~f=~&*aT?Xbo$f@YAJhlp`D8
z%ojbXm2$>R)zYb>Sl3j6-Cpd=XEyV`MQ1j9Y+kXM=lQbFeAAbT8eR?#4hyy^+>&rh
zm*)W|x5n;?{~1oM?|zid_=t1zp;Ln9d*)1)d9zH_?d6qwE0+pMTyfVq@E}#sE3kc*
zT1Cv13E8DW*Z4C3Gl+?1g?-9e^IlE#(#14U$<okTlPjfavVU$>GWNc_BjBtyE!Dbw
zhr*3VueZj|o!<L!m19BB=>k@cBn~si0}D)&lG5yAI=lT+S~;`0yk9H~SeRx~80Y%%
zU9`KXx<|>g^vi!8=Z0Lm>>KFt@rw2F*eRzrF@N*ACOtjC+uiJ6iK|TH)T>3cQy+WJ
zwT!3<3JNJ&ZD7aR#1S-k>rJPg1qU_?yi+xFmo+th$}*K@RZqYI_v1Q&ll+g&N{L=_
zv2&BRZmw9Yu3N^|&w^2zbI!B{oL(?JNXB$))=uSJzlD3{r4D<zJe&N`OGt!cA>*M8
z&nkAEJky>p`(@s2@<=OOES{LQ{lYt?dYzW^;4kw$!+CB_4!;s~#o6CSU~alklwjkI
z-ajj+ZoPHe^nGEU?V%q_&M2v!^`4@zR!e2uQcsok^_?$w@?5Y<dG{>j=8CYOh6SEw
zi&vh$Y*eh$vt1<P_~+>>&sg>A28%lRdQB}gHE5A~Fd@{JC&O_gYl*=|#<sb?c}o3`
z777|jb}1~?<n&Z&KPMD>Fk3qE-hw?j8$Mmu<~H0P=;fMcq?%=Q_vjq)Wgm+?uhmUB
zBV&AOOJu9Gk6@@v$Y-|J8Pn_<)=a;~DE-d8M$Te2@3CcxSC;u)(VEIIrQJ8`>WXsf
z_p^MKs!L6omG1P;^P}Pv-Ey|Z<BCfhN*6tuncfxoEP7g3d*f%V7b|bM@%UPKl-$YR
z5E@u~Qhleo^tt27&%GzNFhADtG+&tSR&M8fcG2yo<$~7Di*jBGv<UxanAA}9dF7{k
z4cXcn69Gr-V4Df2&Mo=*Z^=gKiTO#DH@Y9Mn5VJWC@O4`WcOwjS?%9dZv{5|5Iq;@
zeWmF2Q&G+0wpsI}G8XTe)!Kbl{gOxGrrxhH%=@Kpzp@DzcKKwzdAdQ7xA_fm<=k6q
zr7UeUyd!5UdLz5%=(&rlO1{jz?)Eh-(k3KZF5NA2vzPk}w!X5^&yNaoXWB|n+`e+k
z2ZzJoo|VaWJ-E`epK0pEq@y=fXRhC{Z)5kPpeK(DT(3TiOPRK#ctZ2WwX84rj;a0$
zJ(XGH^Pge4_9w2&<8IeoJ&bnTGR;?;<<axSGxhf_fsnx2Z>05p9=HBu|M;z8jj_PC
zcb_$y9_)(LO0K$?^PgeG)q9*bb)q&#c&?mRf3D|d-xHb5ym_xO8@J3prI=CF`Rd`i
zm1ngyKIu%1Oper6N&D@v^`ysRo-@-os4#J|*XeJadFy2{%Sns!S)067l@9RiQ;JY8
zcruCG`J=={okcMV=cS(T4tC(wk$<M?eJuT-Qn6O(;urr`z5JE+YNujf7T2C*6;qYZ
zEci5ubCJQyT*E0@#pz##U)_2vQ!ci~W$BrTUCO%^6cz|OEDzBCcc95Rpt$$?LzUKN
zM-n!w@Ep<9<nt1F>KQz()GpDB+pYL=q2qN~(WyM=cCD<M)fukew!x?Fvab;LkzH+X
z7E0z=SU=wNz{3BpJG*x0L-9$Oa;j>5OD5SkEqL+HbV<gJJ6AvKQ`@Yla5BDU=^ich
zM|V5J*Mz+?*4y;r&%LIcZ+$Y~rimM=mI_VPS^Kg!d54zBghPi8G;!?H{IDmloc{>t
zj4M-Cozq#6Cu4l=$OMOFAFEprez>RH9#ydF-Kl1g6|b~xX6zO$U(wsI{A$ncmAA}J
zXNh?{j!iQ&n>g!9M4t}FPK6uy^B&8Zv)|H`XqnTqHmdKtv{dGYou4Y6)kJTe(Zl6m
zGy8Sn^KT5d)U~w3W~M9?zV|DjL3iB~2L0N%+3%0(PPN$N)o|(dwez3lGnP+of6rHT
z?>~dZs!LaG;?&kfn=W2<)~M{0$lQ4uGiKL%{i=69W?3&YW8%@zm$e%zTYK8w%1@Qt
zaGP1g^h0u1e^<z<pBi;vyPn0*+H&(qWG&C3rJFLA^>NPa@O(VY<4EG{`hNx&C%<D;
zEjXvJ{%mUeaYa>?;G>TgvaTvxdieF8sK*5giDzCmwtSkjK=PoM_vQZ#Y5{VEo4$N2
zoc3(8)8?yJf(|a^-qGUmaqE}vWlQhGx+{EtJ~vKTL``8?utqAouFgH}=GRrXwmrA;
zZ}put>15Hf{@T2wUoS4cEb=#b;#anC|4C{0)Hyz{D0QFq{9mlEXpDN2(z+l?t%pxI
zOMjj$ID94``8da;7``&8s*Oe;8cynjYX~e6XW13EFy!)~oQU?mxP9y$hkcDFw%Xpz
z`E*yy@4W7+)~uUFTK9w>bsa0%vh%4}Zg}ve%Ch9EH#2&kn{3%!SseOvlbF{n&xx}v
zlV&}>bn4V5P0oZ@J7(Y9XZ=%h{^Y49-?{!XZ1YfIEY|ff<?_8Fu6***ZMM!CnH|&8
zMNZ84dsC@dBUv}0At-pqN>9mElagi6W)_{Q&^CUv*edH(m9na<Pw<Pbiihffq0E;R
zqPa8f?+mQ&JKX-*Ytp1Wen(I6afqd_unIlaFD)ZD+tiuYP}1wgv?7_n?|fhDi+O*g
zuTY!R{pYml;?7GIky)CH!*rC@FI*I}J}Eo#=)aHQ(Ts{m>@?PRE%AON6m+A7X%W*L
zkx5V0t}0l6>WqnQJMHqH;q<9cMjr2&*xRllK?j!XJS}zH<(Y}e%+Dp?l4GS(_WF7m
zX+HJ%st|d~R5fA8wi&CEm&|+8Br1{PDRJ`K_TG~-Tc-CN(tYNom>kbh^We?Hdy6hk
zd~_pnk=Jr>|1QUK=YywyUi3Z4YqI-FnZ;Tw&!jDxZ?x>p1If}@k<G#3!G`8-6}@xK
z|1;dQ{Auyh#=8IX%vL``Gmo=gJ4CB8Tph0nN(CNv<al>3VZ)@!J3{+6u6DMYbL#bK
z%_5$@Nk%_=9YZ;vwPr5SNYUV3yK?p={RB_xk6lk6Eq!1xN!+5P<*C;4-`3Nf_-1@n
z-P$P=EUe=@w<Gd{>X!u!4g3>CZ5AHBs^s=wd6!%Fv+c`$+c(PnXQ=+T&GbcF=ngx%
zP+QqIw_H?Cs~?&**>Beti{H+tQ+uy2){>ZDDbGB={xn~hMoX(K|F*u!*>4W9q&%&;
zzD3L{ePx?v=2W4Il+O%1E*;=D{g>mv&G=8rqhRGd6Bf<v)MM12*&FQp#$-iCnN(H4
zqM2J+40rd=k~t`A<E!d9ZSTaDdk*ma+0c04hV0b7&HLZS?oOWbp-x-jqe+s=Ws8U)
znO$O%UcSe+-+K_p+Zl9K$6M!M+_pOzhcCAHez_kS`lUTIMb>}1!ld2$>z1$ecwcYj
zlA5dZ?dBAN?QG9}%arW1ZL_!|a>x9)O0TA~zyD_OC4nFL_<GWW(!>%|)~f1nQ2N)}
zzr<qo%G0~D*T$qwxa>LYVj8br+$34Y1(Bt064I|?RWzmB#Gn0G{lLcI@eePbqejza
zO{}@kncID3`m$rv?UT&~o7e^XN>$P>A8{7DwN=`+Znat4x~P?zi%cfkY&x-V+xi<#
z9$HJfH?0@{roD5Y`nTe)Ic}0#vkoPESW$mcZC~81qf5?7q^w|yG5GrV#{3lqhxoPD
z`V`krS=Vbe?Wk;nLi6XH+b?gEJa%}`VSNJ?%e`__%6>{obJ%yq20YpoyFX)XYN@wl
zXy~L6??A3Y`?Ho@VVyT+<Cpu@B5Jp}Cw>t>UzRLs+<rEmZB~uarp~G*lj|~7w`2vj
z#1-;b&egTp(hz@W{hx;Y|6U(mv}oBhlVvNKDz!DHT@>n0>r32xBtPMZtlA&ZRNJ?`
zdeVIhi_;S8zXq)7`J&obR?gVHZ)!EONXEf6lcx5pGJSIF_~h%eo41&lmi2DY+LC(4
z>$LjP_`P>z_xA4Cc=Y}AparjZ7jE@!l@Px(H>0#)o5ykO&9f$-u2~l@T=-SCw|^r`
zQ~mUF$C-~LF$EoIFt2|-^HTv!wQh0g_0NK$qH4>wYjovn9A@@6b~DYqS<=pa|C;%~
zX-74uJn;@Z`*_`(>B^@b?AX|CX|#K1u*m$YQ!-c9Z}Geoc6R2)%YK@VT?>NLTtZwF
zUTxi>n_VsZxvik`q*G+d=E+fW)@A$KInI17xUs_J<cuYI_xfhO`ByM~WxZWg#iNQ;
zc86;%5g&Kh{mIY$X0dn5$~>XwS*<5bSHE0qRl!woSR<(W{q>lIoih({P1`?RIQI7Y
z%rLW!>^E+Ivis3j$mPXybBg-JRT<owS(ld1JET_b{O`i9m9MnF{LW=rcc$rrPvrH}
zy6d~w9_HP*JiGC}=$(Mp#A~Y45_d=J{3377ZqoMPgUeao`bR2fYaez@lsgw0a_^zX
z-}e*mcbCb2)JYPxw4Chz?AoWR-+sjjZCIeuH^)J3?G?53X{X$7t*x&=ocmE(Kvu_L
z<*(TLH&?`m?Y`KicQ&FuGbMCo+x$~qt30?5McI07>h7Mv_j@Y)_8iyf)sheO=LnR4
zm_3_qy;8PV#*|4xOZNptY`ir8bmft>=quWDzI=<FVc~IU(X>yRzc)_a+~m$+pq+el
zqj*c9{2327#gga=%PuV0C|<yK=g*=v>y|lYMk$RktCrh3S9Z;}IJK#3)~+{)cn&Rm
z*crRn<LZq`7xPx0eD*5p@EzW7$E_aiSmJZE$5LH`l{tX1Ami4>ZA?~bRv|lsc$ToI
z7oThS#r|uSTT}X)v&M7QwN08Pb4h>m`B1Y?k^1Sk48oGGUl!CZRqWVdm0N4pZr-Vq
zV6bDRw(T#j%vH?-j?W%UnZ&?8CG|<-Wo3TDVoui}=6|cM>XwFfBsxc@^?Gey_RDGe
z0h{xNFHfHS`k%o|*K=L<mcp;RJP%X)JQkbFXDrJ%EUogsd}CsX#Ib@r>n?xsX?wB`
zPPP$U!vEJ)rD8&iZM1>>+nFma&YOB<WAYY`sh!>)N2YA;bejB7VgA>d?xKt(B5F2U
zr*y5E*RH&vOLX#ohV4tI<hU)@37MZCUbsWO_?+q8`A7G$N=%jS^$uLO-u-Kq_2rCL
zH6Gqa{N`Gw?n#erwyCmS;_m<a%T(d#viX-|c_yBEaq+s&=g{m}#jeW-`Zw%5+I7EL
zc$@ym*tIb;=TDj37IL%sKZAfod%DW+u55=WOLggqorlhFPE{<P!^Bn8vayWSOH(ev
zuAk-nCwtyUIeWv8<Sks+=E>}#@G@7>$F%Homh(#Wevh4j)yz50#|qEswfH!xEM!z+
zdR-;(rpB5v>Qhs}oOq$Igo2e^O#Nny|1%ur_02ky8Ie6BHPU8gt!%nqsafb1!#!#K
z3Ws%7;}%3GvYcWzySB-C>-(+8uGd&^VQDhVI2~55IzxJ5V6wA!c0`$Mam!U@L7meI
zTMtcsK2zm#xI%DP&!^TYGB;PNPn)jY#=kApKR<@;npVU7KmBE{aeAz8b5~XS9`%#;
zIKrkKDl|=0H0c)u^K{;|_e@t!o};W$Y8PN)e_*F|l-AAm_@xCa+l?27zRapx>app@
zUB^C#UH+4f2&M2Gy)k#@lr|RGjXUc2wt64&jjUWVRn%<V6WJ^csYUbL-CN$=db(0A
zYX6xff6~hrH~xxh;AVRFQ9F9B>-3Vp#V6+;*G&=l8F)&hIg~ke!QA?*rAZubUK~z5
z`#p8Xo|UaeM^z^j*-Fkou=w9eJ=Qw{xgKk$9}+bdnK08wB+LG)Q<vurTensf7nRuh
z(7+gtO(zqx^lh^*aR$7oYP}@pvj47nu%_AFGV@y(c6+w^St=Tx|H-pys)k^|lZ{VT
zTz#dK^;@sA$oL$)acaRip}0A39z;mBnXfrJ+uu0IpEuOiRBMZ&2g|IOQ<nra)L*f-
zZ)CrdWS?HB+i)UpLdfQV42irW^$OVqa~Pbaw5o{890}@SZ{7Frw#(gw{|pwh_1e$P
zJmk%Dv~**_tWMqJjSPw#Wg-gqsg(7L>lDm<<)IlgX?4s_-jk~Z3U+R|dCg&?+*-#I
zlj4h%tv_16Dwf-NZnLYv)#D+LDsF6?bi#ECb8SNa>+WRnn>|4jn_MO=e(>Q&W_@v7
z-mhc3H$Ii9lh<=pRW&iW9$f32e8ogI@oI^%ukYk%rh-qWrBnsEZ22sl7Haao;sz6=
z((&2l>Qa4I3;r0Yr5xZ6+ArKSi6c2H_WP2;<O3g46@xE7zITf0NGn@tj`o=-U6Y^d
zys!Su@^<=J@A>?^;6%+uQx3a_z1e+S*ziBYwn;AzXm6i(B*LV8<D5A)Cxu*Ao;QmW
zXL;ta*JD+ybMqrnW>cTgqz%F!icie=th)cmffa@e6>1VJiYCtBTA|t5)!tScsqRy1
z9)6)zmo53qU0c)go11lZEy@&%cs$G2YT}I7eHkfSr&kn7`Y*lp<DX)yv4K#_)~B;3
zEfATuLvMD#k}d(?)*PO=Z<||W(!}}Axfbdr1-uMkXbQ@4;90UJZrew$MyaEI?xI$g
z<YqS52Q7IU?He!rGt{zI`G}~LnR?(QkxA3<UoG&OW8C~P$1byL)`Xi}9IGaYyn7;E
zQ0uT)&T!Jl?x>J8Ui%L@u&S-}YMRreqbITLOyr{aB~>j~(|6XT2J$MErkI3s>IemL
z95&wUmU1y$=k%)Bv;(qPxdnxdN0SAATK=dknY6UECX{uHwGcDYZx=qx!<!zSSrq3P
zkR<Z7fwl8LL$J@T3)c*N?ypSwD7Sxc_OuOQrC+|DpXwSUIW2B!Yu;;ry|w3(7?d>4
zJ|A&X(^Yi3RmXgUo#)89K<~t5d>?aT+M2B*YQ(gCBk!<Pyx8F<_@BYoPbp~9jS>~z
zEmIah)RHQ<f1`bOUzOsgJs&PxFM4Hseap-_TW-!uyD1wLc-U^~uFL(F9=XlJ%Tmp6
zuqm{@mELwJDfQHB6~i-=JoG<S)H0_gnTi+&-{zAyXDisO($x1XsaW88e9NUJTua(k
zDoDMamTtMIq`gCOzt>5Xna}dQSwbSCPaCzZ><SZRn|hF8hg-*7Ytu$Cm7dtL{BGal
z4?pWXo#SyPIC1Ko#?4on<jf;@#Q!?Co;t2lUUbE4n#2#U8XeXf6Bfu_+aP%7wbgXv
z-?ltKXU<(zxN_{)?df}eb5G7QS8mKHz5i>1aP<~rl_uYV1@E^VJhn{nfkjfvvpE}A
zez|@BY4wq}B3lFRZF4DixL46L@oDQNol_3mo)mFz{hgHYSYt-<-Z{*x_cCXF`|@10
zqEjQ-%QHEB^-uFDIkv{LJg;1-)c0}t^ei^g>FK8vha;Z_wVdYLdm=jaYL0_K|Gncj
zZ`-R|kMLw=G<ifQOj8JSiD^&Q;g$<*el)}6p_<Q;HK`N!cf8=<<eit}tM|)wQ`U=L
z(--jH-Yh6|Fk0PzR_O!@&8b(eKK;CL>($l}(+eB!)Zf)TzFwxi*K@XVXPU~onWx!`
zY_ypR0+y^=dM-frNce*%bJPVtu%DS|c>759-E$rndb*}<Jj=RPqAzg%m1kc}jh{vJ
zB`rB{>SXbGwcp$k%%ALqzwooZvsd7m(jc|VGS7kO$>)vfz1PJa7Jc}9N%fEU>(>?a
z^8zoX&lSCDDf6*%ep*YK$BBhK*>W>pIXte{T{O8V-~XJE?Aim&Q=L8wPYyd<(q{7_
zw>F8N^N7#kLe7Ym`5jM_B2Ud({GZ{5$bW|SJ@vY$jO}@LJvM#Wbu(eb(yD2vpRU-*
znCj$Glz$>MamO3SpGVUlyw4R{I!Efje+KnVrGEwz%dg!_TrDfynVj-*o>))J2B(H|
z?8+axqy*~fE*78lTs<j~B|y1x?TpDw*7hD-c&9P>>+#%2!l`RDy)HZm4AYyv+1@yP
zYrbR*!@J`>vu=O?(k@x&KIIf&l19+HD0ky1O_$tn`3_f>@O$e_6W`9Kf9%IR<(r-A
z!v7gAuQ|2!i}}K<9T(3!XS2>dn!eQbT>2TQg?~c77lz#l(ib^-X>WX})Y+JuVcBa`
zf_*|bQ_aLv9MhB@oqcI@Ur;5<;u$CNXMR(Dw+o-HWG?)>B5k#B^YjlpuFh**!?~ht
zt4rEEgQu3qT}tP-Mb69hW(m9;X5zNcbJD!{9TC~v9yHb}7cIH<{iwUdTRriPb0<}P
zba7}dOPLUpBW>#WBD<dd*{S0Ne{yE3er9mkD4tdO=A-!aNS~X0DOYEPO!l05*!f$N
zcyZ|T)7nP6Ch6ozM&$TjyZ2*m(_8+ZPag>|OJ)i)uoyM!c}oc_vvd8jX5QIkHZ}Ls
zjvrT+$xNHlaQxhYdHGwsQh%O}3k{x@ZqoDVlE2@Ix7-?h*LXJRU1MyzasRK|1DV_%
z4IGwgY)<!QaM{d{esgW5n)XHs7C)~iI-4fN?J`_pz4iF1iL0K)xm;N}JGJCfNSMyM
zs|GH)E7-J_y0$iV3%}ieUD8kf7U!fZ1~-;k=rFDE4)+#Y@bj2AzqZktoh>Q`>Ca?-
zhwfeU`Qpzf!Tt_At&f}LN{4uQ`_Eu_^Cd6(tz_{jrkR;BmYSRP6s>q1wa=h-$swke
zWRXdy<&-pI4(l9{c^VaR;A&Uw!Hn~T=VYA3W?u37)D-l9wei6$#<B?IqXkQZ%ajy<
z{a(B&Z}F)~%NO;grn<Bi|9YW5Jvc2g#VYh!zi-DZ&CsNI*0Byky-PN1eQ3L3?(9!{
z9Gx|FmMzx)vTXjs`lZqP9xnNo+~arHFQ;0yampn2!-jqb_8P6^TbUmGU{}n=m%FAG
z&x#FQH~+bYc=UYd6P{-J{}#EfRevwM@Au;^AO5(xbExk&ujc=@^u1emj60i=SP?(t
zw#DwBR{tpJJkxEo_?1udpWbT=o)=H+*H~UD?U(fSKf`4Go_Ueo!h93|GfX%w@YQ45
zbK{K`IU+|@Ll67QsmxyJc69%hF4=Xm3u2txeJ6Ije>qV!GU26^kDc%1`pZ+pJj?3T
zzbcn*nVBDz`uTWh@D-8!Pa}W3F!p}VQ0=YxT)k0L<Xsx?ww0{`J2?|)JeaY#eZFzi
znHh_O9K0izdT|!JSvo4uZjL^>JoQIZfoGUl#_2orKJpjpdaCPc80oOg-o&$c^^BL<
z&p1U>wK?t>e3}_|)I)V!>JIkOZQsRz$&^*@cv<}6$Qj8dzEHJy=Qu<+u8sC`QF(A6
z(CO<+-j}bGqSw57vUZWJtBAkE8KruAr(UHMqM!Y4b59pyu)mr=$MMZ_5u+5Jho8C6
zOgJDIbLsqn$_r1|%0wxAkTFU<`)AXChA$`5_eE{L*lFz|cqXN3p2f;I$+WF=Hb?Br
zP0W3ByMMjhb7|Gb9n(Z6JX}*5vfV2Cr7#<>R$S^1sUqv2F4KIkn0#IT#3f2<Zt^#y
z#cb@=ck)*!xgJ!W<1+WilT}VrIP^HvWjdB>KCM;K7kaQ+<;PjKNB55=)y?r#`69gM
zqMa|xEiHi)0z0agHvYZ6!TC6M?b^P~+xGva3-u|tGVdu^>?OE(uK$ORa$W&~M&Yjm
zmd{v!@vnE7;^rH#!&dFuth;E|))k5AZF6-}=U8mA^nR81bO!TMzXyyp@gM%(IAwI%
zu4tw45!O|Q59u8@s1NU!lU9oKm}>HrTk%}us*>djU(JdIFQk=PF<<pqKdUlplBZN=
zcna5&TfdncHuc>}l27@lIpK%pe1Vt)S2s7+%r?LK;6{&>yvZ)_D&DzYrFqNSu9{x)
z{1>jn*BSc!R%pqtGjiYZv%6QA%P`a))0x=)XTDIJWa6QjnOE<m2Hb0%`nCDwp))Z%
zTSFOwwx#{Lx^r6hy6lLrlIbpKhOSIv3q^`|{c@Zn>BXkNd$Q*Ux9r(Yix@Lo?>KW#
zvnARFI}?`d`|{C(X_rabh94!%QeI}6oOw_bP-H9e#^jPltZQIkyf=@@&A6zY_u>Pe
zUko$0$qQintohXZ!bcC0#uaQqjJ(;++*>9sKjTy}XR6#s1`RI5G}RRk-<*GP%ybX0
z(33TtUA9*aoD$K=I3?Ap`bEciM@9R|&mNmuUq!9rY?*X4*f&^MBU>jpxYxru+NbN|
z)ZE>2hb7OQ{kC}Dqh*N+ohKKRr!s!tnwP=s+AjHU)ufq1&-gjLCNI6TO3J#4|JE}t
zNwv~QomyA3P(Lk=MQ2}MKPe<K$;9ZchmME&zVF|j{kT+MXWM1_&OyYfQ}OkTCE`q{
z_}($sF8bb6aFelUvc*QDr&s4huDbduFQk*F+jQD?gY!&RtUa!t`4`ju)Jy!duS-*<
z*uDF4kHzm8_nn;EHBT(VLnq^&)Mf8*cbP}x;ty9&R|?gc-PK-nKYoj^)s<^kgAN@1
z&+tJ~C2iM)U5#3H{tMo3xG&_Ua%SFi9T9HXg9?Igblop$JNT^Bin!alD*E@<hwFWO
z<74CJc(QtRm8kXE$1PN5yLN`*d3lP=t+VgD@?<+_n|Jz#`s}O|ODkGDqc81%`5Nio
zMKLRbF1dI7=vO#n=OCXbUbU-h{l+l8bnc_GE*@AfJoVnDId|O_RHkTMOWYWmYF{kO
zXT5XI5`{pY_rH8=8>Irb+<2aHYRb|Dr+<if*i1b=k!`V)qr6DC=BmwEyHuC7sJiS6
zh;CT+uYXtW#ys774|2N$k1Z|Fe9!%i@s+diC%!yyvyG`8haNsXdu|u|^SytzKlvyb
zJ}cz<5toZ@A1m38G^bB#D7}3rqC0->2cDECkCx2L_`#K3HEjX!g7RetR|{|0A#yUf
zIb5#sMEa`<QZ<M6Ez*i=uk2t@=8g<)UB~u!`)%PL;R5p0y(WLyT)APb{icsEZ%BFu
zO=7q|w`z~)=_^yNoVQg}@aSQ?RB+^%*S$m6WY^{&dZwq`BjB@hx6s?VxcZNGk7f!k
zEihf9miA}gUI`n^DW+S$^h_-=t>2iI+E<$9yy{lO2e|`gM;h#9+zuCR@eK%KEbx`=
z)G^r|+ju2%LGTBziK-4wS|--<r~NI?xK&8?Tb|S`vG6$+vFB*<i^jf1`8V=hJ8w%q
z?w82%Q|2{j*J3qWmKbzRMX+pJn>p9R_$Sl5!d7@D=`Hk+&zSJiE9%W-vnHOdshOKu
zSM_av+tZP}q)mx)l56TcQ})p9S8`ZSZnJ!!X_&lUC}_G)P~+;kd0M4vw{FQFO*=hN
zVYR?=Z;_{~SFAkDW^VSh=zvYG=+1J#XR-DZzPQB)^E;NZeBI<N>vH7i)UR`1J!PEw
zJ<F?UX^GpJ4{@)5t6N_w_*16d-V=0EIKVf=PDSNv^IiKl4qUq7ef7$s-EMm60<)}t
zdU||lGd6h8Zt1#A@s38r%-(pZ#QRNK?kQa;UGHBa%kOmk`Q4zu(XLajJSknZam$7i
z$E0RuhF<xZmAGP>ztPcT&S$k(_ZiggoZmLRAzn~eLgbk<yIMk0)5Kq0Qy=8U#GbL=
zW?T_-N9UZv9tW3s53{*_=kT0}P-wTDz#Oph(}kny$Lwam_}BI^+~8!DzTMJS`?^01
z@ys?~&Y&A>{BGCc{|pyiMNfXs)so|9{&)6UmY*|R>Jnw|{LB$)&skM3<i9fA<c90?
zWW$I%<?L?C(mJQ-xTP-mH%UMKC!5Wjjuj=pu9aMwsUP?>v_UvJT2<wMyNM~sE)F*H
zpe05-c{b^rvzA$0^?DRPXSP|$E1`2Td)IiZ&|_uHx7hmVphUwSlZ!Dc3IiPVSLi#*
zh|JEG+LwDJRyOwXO|{%Bw%T4l&-x|X8eMtfW%P5)A{Vi+;C1tYqQb*sxeNAJ`e}=|
z9O?YLByst?mm2P^F4N0(G#)H5{jxM!zVX0g!LFIg{01$m3|r0%tPIsS?ak84JmcOm
zqc@*cE4OVvd8#<$Gta7(XM!DVczuI?gSHuIySPo4FPPMGy4v9KYT+IA;-_0AD$h+_
zQlqqB$scv&tF3%%3oC9U7D!B0-ZP^u$hnCxWy<WZmr83mSXaa@n)CB)eOS6sD*v3z
zPY+2ooi^Fm@j69IS<u&D-?R;1tncQPZ02q|8fGsy+5R)fv==-KRSRsr!w=OSzBEND
zwBSj?hC>OCkxC0<LO-jTl$iv0i>+W=WW458_n)S|#oFsnM%AsHB$OU0qJ7D&{L1q>
zhbs%q3LM$4seR|Ri+Z9jYJ7CgYOPWo3E7$RwlM6;C@>D~n4`Wd+0%1zh{hY;C-D!>
z*9a@k)HYhynb@~#lfCXM6|1dRjvdI3H2=!+gyWS1qqB!~Pg1(h0@ju3X0zGMa%IhT
z&)R*^qD!Q6$xhDIyo~IJ-2XWqyz{hOX|+z%V*R#=SE*}e?e?4->r!B#{4KS?sxdmt
z@qlu3ck9_%E=Qjyuw7+n4${%eJnfrgwDg>2Ad6AZ+i;s>S2teWD#-M8YRRYPQvwxD
z?Vn9vV>m%Vd+{sIz3=$%GT(VPM_K&tiT8{jo_IV>@YJm}UlOsLUpI5A!5>SZe=f$q
zvRAvT^-H%Asm$_LfBUfG)Fiz{vTJ-hH*VO@^6`rWo4M#*Rbw6h%@Z<gO*&@OD5dG9
za7jtm?E8J>oU60{l}f$vSewZw1155$XqZN<v^HB<kmGjzvE)D1Bk7(;^rED-FaElx
z__1J*oubF|<fy+p-Ckdo^ffDF3A}Wc`4s28tVte$FR%E%-N6!ZyVvRavGsk)mo>TG
z9n5cGK7Z?1;5v!pI*)DK(qvtO8M7E>zs|n!=Fb_Y^q;EvzUiW|MpLJ3{bG7iFWX-u
zG<eyY$!;O{F0Iy%%{{ThqWPxXMRie4PnG4n3d)!taXmA6qQ7I;fsdROcQlS`>a0vz
z(3jZdt2c>b{leC#H@Eye{p?EH+}W$cwLR`_T#{CEPW7hp#D3#rp)BdsLNE9pDPME^
zgwMQBC5!(VY+31?bl|z)N&6L%=2bKJ)R|`5-nz2>L%PniMVvc%W4d;=Hd?lFEl&UN
zd+nw_k5!huyg6^?W?e6%TkT<;E0<0YkGS!^;>q7tY|>dFEP01TViz9J=Ism9=l`&3
z6(5Ik^s6IM84rSP#9jQrc_Suhqp8sOV|ot)Rzx3}&)xl0b&bMZ^Q$J7p|QLnS>0Bp
zB9_6&JWb=Ak4Ku!in?}z<C259y5s$t#N`jFA1!GO^q8^y;-w(wzVe%mg70(>q@8hE
zI$JXK!GDIB1E)Q{W_1}0X?lDL@3^U(<~;k$G>3_2<0GbiH4X6OmUyBmv{604+*7pE
zs_}_7b4|${alymqk8wU|^V#r5CL-K@`QFYgW&ch-ewvh$Way=!uvAmwiq*~lkrlxw
z5B6?WTV)#?dAZK;?3@&XlYIHI@=>)*R85jHeddLRXmyE(tuqXY%vt?yM;7Z_EuMgn
zrE`qb3T`HBx=_7HFM^+u@3~^m_7aVNr-tW#Y?J>qDZRktbZFo#XRA)mbgxcRqp1^^
zUfG^nI=fZk^dGkuyJqQplWAU)HQm11C}Cz^#imuKr(bhhCb2TMwRf6E!o^kB0+%)_
zB*~}knExzwYC?N)*MwKPaUaA~u7}s0c`~bQO-{A?Eft?bbK?2@%TjBKR-JO{OVwfC
zHD%2iU+<=48dGMr2d6|n*{k_Tmo0h9S60K6iCPanJz4qOd)o$1m6eL^OLxEHvuaPT
zj&DA;_%k2ZfeqUdogUscwG(zYxY)hx;SV1_>Cbmm9gC(MKNQ1zAn9J)Vy9!X*85%i
zyRNtB7+axnZ^64+oW(2pK5jj6yLs>RoZQs&PEKkH4VqKt?tGD@oA2@bPdZ=fnfEz|
zzF)fY_L;`9jk(L$NxEk5RV|xyi&3%5!6U0u=!n(rg6kWlDt{cSmp&@oIb+IM{~uRY
z?UfO@sy6F-)buR@j|`Fpb<~pTk1Z=G<+||c(zYANF2<Ytf7Pjt+NtYn^q=9%PubKG
zlaO=uZ$Ep5b}hNd_P}OJ{?pA-&lWc9o5Zg6=u&#(q|V~z@5VyMrcPPp%D2;LM(-q^
z<PDlf6Q?RXa{SNma+#`DpuEV=nG=gQg&8;gYrOfGf5n-o#Vg!ueRV{B))k81TGFM_
zH8F^9=Jn#;SzqgaoZ9#8*zJaorVo~AIj!po*=zDG?K7|UvDcp(68<yjZFn^)YUv+y
z&u<H#O<ohXsmo+m=y{E6Jolu6ZA`xio?MmIdUf%%kM|2MZstENwkN+&@<*apl^0jT
zDPN&4YP&S1nXi>yE!n4Pu<*lcRaT#%#f~!smTbJE!56o2M#XfKFQT=Y!L!e=*FU2u
zDKj-+^-;g^;)n-*VflAG7ntznZ}>d5af8Lv{|pnA3TL)?`?6c4tq4?Ek{48LT5``^
zws%tJrsqmtkwNDx4n2!G6y}`Q`n|#Y*97OoTrX3iWY#6E7dawtu$cR2qlenvHwS(f
zzwKXWaHr<c6V7uF69Tv*gmk?xbT_`1o2aN%bZnNbhIjvDlZYarnQ`vnSAV9M-aM2#
zmCtxpLdS;MSxJHdp0z$HdrlqTc5_LYJ+bRdxTs^0!DR8WQ?4EXRrM>gGz^~{+8BML
zy}HwnE0e>R;oa%!XC^$^Y~Ft8Tx_y28=LB{{^c)??wpEpo8)s`XRFPZ$mWHSTA8Xn
zTZ*Phzw>%!)5c?G;KMH}9Ky+Ks3*dhI74u`)mGhrU(OF_yquiU+4*6`GcLc*0G*&a
z4LYVg$tLG*yk7IQp4e;^@oe$AU3pn6UTSaZIX5#aV;OT`NW!(0gs_K?wNCGy-C%6H
zV(Q^%=clAC&~&}R@nMZ|h=!%;!ri^fsRw^+Ha?P=vZkqXp{i#Tr`NG(Qxuv576g?E
zt}f{h_%gr4w92?R{L&Y7oq19fQcLzsk(#UGx_n~e=a5&`%1oglanGJh%BY;2A$fUv
zo8(c>0(tHE9f2nbAMrfCk^JL71NYGnN1jb!xjaWyEMb+%q=mnHnI~lVot0!uv<tp?
zH*j`|jpbGEl*<QoeHmAue%9tOr)n0fSgOX^S57Uh=E2!3k2@atxVh`_9IFm39!D;=
z)*G6)AAZv`Uy}PIF+KT1(T_<JtdC3(ImIHlsBaxBcS8tsd$PEd-jemFiZhf~_MiF8
z66CUmeR{mK+r8=gv+I_`-B=mLr{-Q{rx5Nfe&Ooj?(QP7F3vrAQBPN9m=#35)Gxf+
zn6V?|`OLQHCt<7i{7hf9Fs8LqbJ1I#&dnYnJA%RsR<+D=KH&f7%wo0tBNAc@g7<fr
zANr=cj(@38Ma;?rOHD$VLp!fD2QVf%zw$PHXe)R<a$Ee2lD}@z@!_6x{e!+%>c(1W
z{<h0!l0J6o-?X+>wi++)P1kAMw$FH{YI#@4ljo<n!lwOL`iix?yGh%<zpQ_Akeb)5
z)+Le&EN48cPDd_k4$%slv!ttWMn~H@jYqGnIBlX&J~_40%4*xXC5I<p(supN;CrPn
zbAF_{M-Y$79V24_#xUm4n|)S0v*V=dJ3See`u}GrR1n#z5)*jul)}!k&L5u3IW`_x
z&Q!?Z6_=QNZ^_NXt?K`tEocnl=!(rNj@OHyShAN}{A`wJ>|9T_Hi7Gt!nSr9=I?T3
zd-mXtO#U)YpCjrVE$zC6ZcP6fvg4*NiF~-DX3M$R{j)tY3+}&Jq%q-(j-A4>DVtpn
z8=c?mIFt2rhIZVt(wsw<i$aBi7r%<y&YAeEbBVU{3YN8w6St+OKC-E1GLV>=8NDe&
zVaMj(HER=YUs5jkC$Cvi<e_=WM#VD8HGs1!m2b){rk<t)vhr%Dqv|=VrmfndVj3d3
z)hw%VZ;;iw&%u$b@7}JnaEx>CzI~x^?cr|=yl1Nmntt*vTD)Lg*5wNZ{EiayRC~NC
z-t06w-<lC*SrIh#x`V{58@rhUMQc85o-;Z(duhMXs;F63yA)?#)N$E$TqSo|#;n7E
zGkH=|#I~IX5lvOu5qDwB%Ziif+V#Dme9r04vkmjz(<`<=ys=aGW0z3SDW^}`MIvmW
zIYN#luWRlpIHYXeed(x1q~Nu9Lsfs@t5dY~)_-XWOw!N(xX<d&KBeU!8j6?B+Ag}_
zj?nd_3#y)8VZkqsPpmKd^dxELl;^yvyrRA-I+F#y85KWg({ekUkmGmXqOmPzO-zgL
ztRwHAypIxc+_0j`bkXz94{tVK>r9)n?aY6MBk@nyE?U{w#kFQv>FN&77-#iq(GhG0
zuNch|wqG%qo4b9Rv6jotO=&8Vj)-i$9woM8N3GkGo=*m+r-kgy`mn0;@U1tQ0dJ0d
zZJW9`+i~`@zxJ$Zp79gRqEgl@H~D;0ICPESofqk`QH!rzvuWjpyt&;|ypFSby1K|p
zi<$F8S56T*Gc_|Swe0MZciLhdZ=T$g$ehe|=*TtC0=}4|s~GKtRRkV=QPSc4&)`?l
z9V@ceHtNgX*o9{TLW6aUpD`%vM3#!^2`KXE)_zrrI+Z7VWLZH^eWl6db}qdmb6VP1
zE@&)k{K=awk}9!JQenB+^oi~Y&p6jgY|wefVKdXK>iNyJd<my@xh{s+&s;QL%U<Q@
zqL73K8NZMGJJZek-Rzt7zZ;L1PD%WD>87K$=w+?hvCf-66-()f+DdkHN`_Wl>`?l4
z_4TQZn=h0s|LcBnXU$#9K-I8PnNJ=?zm|FI`7+V8{^rhH;mJ=oXRxMhnh-KUPx#iw
zhDV=+dlI#hf*<kK2}P`JW!CIWGOc)iW1Z&DoTQ1JygH4K_#&2U?_qek+p^^99gY*m
zx3pGHl9CFHbe|*DDYR|sWuGZuE;4;Gkm)m-svub1+jqUa_t_NH1w5OJmS{)`d0yL^
z#UwRl@An$5k}Z?1l0_IFnzMQ>c#`*M?cS`E8ts=SitUA0Te)UyW@s0$;5CbvKKb<W
z^^hAjC6{(=Yk2z9W%75GiSBoG43Gbg?wfh|<FPyut74|v7J9O+bB~s$uc_X8Cp2h>
zmf=xV&q=X9vX1s#Q`pupXvweCR9nJvdb8ZrGrP=JT6ImcRQ8&wowj1B(2CxkQ_I|z
zPT+ny-P<H*!35{kE45w~mf!fNm$oQtjfbabx@p>zh$VZoqr<bcyH6fZR9PA7;pH@I
ziwQS-$PO;g1V>i21s%`aO=Tv2mCO6GX{zPMNtTBeHyl{69FV43u~B;BzQa;Kf2@?a
zd0E_W|H<SeNyZb_yumBiJ~FfOd7|YfZT=!j%P!~8M)kR+`oR~RFJ1j>`aET>^UMQV
zPWnnrek-!(wALnuM{mFX`*cSzG55sb9rLp`dra?i$~g11d&0sbUX$gz3>`7XjS0`R
zYW9TxR6V8Y@3XYa!(_qMZo%4~Dr<$8=Vse#PY*3w7By4XRxe)W=T?!=Hm@d5m}r*B
z)As&b_*7ku-L-R{vHUyzF7#V=|2(OB)vk|C%g;In84FbYTTmasqosY~$8o`<tED#f
zgcU6nXk}oHS>YA1y<*BA5!qKkhmU_uxtse)w$xNs%;6^wTf?&pU9M9$%o5gKQslC9
zjoo>@9o}w-XB=kYv|4X*&B{^xk&$Oo)8sR}Gh%g}csotk<XP+9w9Jm$sFry~pxD^(
z6em|=%*)jpvK|U;sh%^IIeW_Ty8D;11P2=jpLphy;;FoMyHIYnfV0NNFy_MM4}BLr
zEsCb@aO9ITQTWX7Zhdnt=YIzOKUW@`#at2C)o}ELUFKJwQo)0lP49+JmRd7S^{nyt
z)~8dsJ6>rEHf_vER59M5$H-Zh;C5WVB0Y0*`<yj;iaM?AUadh|uBChq<u{JBe6*~{
zd??%(nb2VBq4**+LDT09V^zngO;#N-*Pl)mT+`?KQ0qw+@9y4hW@_I+i|m*?eNX=r
zeZei&w0P^++g^rivUjYRvr~Q9txIPeYAaph1GmNsbieuaXvsW#-z4QI$A7cuMICHE
zxc|!0a4GH?Y1V7Sw=ypEtaWdnZgq4z*Hm4XXRF#@Ru_NTqSkHHt-t)Y(rS+tXO;$<
z+GH=e!TRjm-XB{(bj_N{;a=ttwEyO;DOFjXH@_zG9(gd;vi9M^#MANj6sqiYX=+<d
z+O+8Y)2%^&r!94U*~7-<xZ?d`t%=pOm&(kJX?G>3E*3Uew3@-Va@(Bsi;teqDCib?
za@ze^!{cBt*`536otI7vn5mm&?*Am#)&KI}cvC0O*?Q>)5vxyVgw5j25nv5f(PWM+
ze6{PbW!t<tR+qcxOxUzK_<{GVH_x6lc{7|1l(4B;`Ea8|r)yD=jihVPyH^2wea}X&
zIi|QN?A)iCb}PxHLRW(>KjW55cAR;cbBf!&XqEHKOzS08ub**`6xj4@dek-<k*UdP
zOK(hg(7bx%Uhi7%L@^r|&$(UG?`n2UdKu)+E;)yl>A_|(g{2NsVoJ98)4#3{{<6Kn
z*7%yp7LiK+sIT=u*ZY+`%b)#>+a@$p)?}h<G;^qh(Hfr=0hh}LTHQvHYqN{u6kB5M
zsakrTDYy~J6cK2AimAz|B*2qpVycdmpr3u$n`^?tw=`#1wyfq?@SbhT;mP6~5jf3>
zRrd1CM8zpZbNW6${kQg1vCd|dv%7>>7Il1jxBBI!PsjaKRsQx^Z=bY;Kf>v;BTJ`p
zNoqFh-et_a>;Ek><bQa?An-rK#C+kXwc#79b{~&=aK|a;=cSK7N~e0wchA4(usY*G
z#r%f>3cZ{9rk+v#&+yXt`ROS(lhlK*Rem+{{_TCzlcz%@z9MwOqyxG6p=<YDZu*!o
z&)Qt8n?==8>5RydNv2v0mVcM(vu|SW<uR)&*G#zbVXec?{|rgI%-6jlcAgFOdRnQ<
zv-z~jgv*ulrfX-WO#9Vo&URgF+qJ+u$(aq0xGc8nOy|1zlgn^3`}!sB2Tpp<HOuh$
zD0Z)t=l1o0Qgx5i@GGXA-lh)cKK)rY+rWd_Q%hb?(sAQwX13jIk*wF==Bm^$7x=Mc
z&n2&E6Skf@W3uAHy$6ERlirqh=cx1fS5DN>(ong#M0S;iqwkA%U)Q)jzf$Oz^=en<
zrj2tJscgO+5UT5{d};EMkfYf~tqHdlBuH9b5|K67@W(YsrNysaAZ1pX`d98re#dsU
z>I>~C`qZ*qgugt1Ym!sb`YC%k6&oT{G;iK*WU2M`51n}Z$<A4(0n3|Gvs!n1davA?
z)gh;Bu9viTYxd<ObDunT+Hy|l{4w7(97!5-?>KIMm8klW=%y;X<l>2i8?y}Wbf;w-
z?+O3ZCTUt0Ben7Q@@cNuHh7&<Z4OjjlzMlo;{F{Q*{gekAAjW12ny0J@U6YOXZtPP
zjSaq2mWphe9q=YPcE;srift)co6cO@tX1i0aP^>xYCx-50W(vM-iu$I4;zn8y0SF!
zu1cut>FZ7=3)!X?I{iNUVWrDWrvh;<wmoZCewc6|ll83goWsTqo*MmsZdE*1(3<dC
zw~6}`W8&Vu=11bVJ2VcdANt4e>rVLGnAulXGCurt`L1I8e})|4C&9rN*F^s6^Af!L
zdgjJ?lU8MYV%C4_Rhi-Z%Wsl+@B%aS+j}=gg&SsZDjrC$;=X+1Dc@~ACv6KOwYuz`
zJP!VopP8(UYuK>#OUBAn%O@owpAKxka>nM+tp#O)J7k*JSKr^JQmB$Pb6#AS;<uls
zR)!0YBye@;E?53s@%?yE>gN*H9FvW<x_O?F5)rS&B%U<5>nOO19@yKEqPRXucR#!2
zKfzlng)$}=4NR_NI2Jgro?Vomwx#Cyp^qAmf_g4+L~%dW<8O6YdC=;L=~0)o*O#x_
z`p)~R7Cpn+e{%W+7di2gl4V(6i_R=PclsG`&D&K+ZX~U87qm!G7H%tDc&GiNh+tmM
zzuc%>rpr4QbGA*kiqxLsE!A_7C1A6C!}g_fjuxvmFZC-96w8dC{bgzJrAkkejhzdY
zPF2x*<a;UNW<+(Y-|r?q^Bp-QCs)o|$>MW%%EpBX%X+eO-E_Aec3mzkvF@SPw23CH
zAxuY_7<o%p9=x|W>*y|?`4*>)xn{k&6y+ak9o(rfLqa>o@YB=ID>>hDb9zJ@cW&*w
z$rEN5rxaht$kr#q-Fxstb2>|Z#Ns1KD*h8@7R6kVYI?~VvO-qlKs(!1T>;75=;yVG
z6Vtw4<lM}$*{Xgk)A5*PP6|crjI&<dDPCQqV;FVJzoJuh(vlUtGrQUzt~l$Q?;YdQ
zsNQ~dqRu8R(P>vsKW4hhwCG!U+um4@tUZA*?>t}mXhqE~o|enTPF?e2gRYob8}t2M
zRL6VoxX^AzrH~^@OIAG9($r9Su|S0RKf_N`z4J@No7V4susP+yiE}I0hp#GP?Vo6O
zG5q9e#g}}W&uU~Ttvt8K@NClR(^@Lac)Di?xxMPw@h+@<IPqmxa*)Th<_X81gk0=7
zEZRJ&W|K&xKUa6lUS1JLPw`Vp_hsASjN2DgNm}fT3}M_QvcSWuw?^9|W~*4u(l5M~
zAM-Wm>%Z`suD^Pv=u{)WP1kpCT6{9(*W#k-`(~^c2~xQ6vif~dq3g|;d%qb5Z#XxV
z?L^+$z_OjH>JGP}jBPuPd(S?q?l(nRBW3f%2_bf`l7d!Ec(_>T+|qNq9&0XKbhfza
z&3U8Kp)Ulkdc_=bjx8}Xk9B&|P<FXM&-cr#11Dxo?mM(VHH+CeV%EOIm1Uo=9Cvy6
z>BAqbs%0M21Uly_9WD&dUBjDq?2ytf=T)zE)&DuH_Nq$fP$ff%sK`07tk~oC+`gPz
zp7-d`5ji(g3%-jBGdX9w@?UWXKY8$5;fW*fE);N9oSD%h)SQ!3)3M5bVX+hI<IA#^
z6U@@5hWJid7CqN#%VkB)sHjM9Ch>0`v!-@Nmw%0Z-CsCQRYx*w)}$#b+7l!K@7}*T
z`?XAUZ_@9T2Ygl84R%EYeO2ucf1-5w!PbzOvYuz3eERhD%xCp=ryi|vyQp$SsP5R@
z+wV0dX}nT?5`T98KI7PB9P?X+>r_>&Iurz^vZ?)NSbQjR`<k$Siitf>$~7f-Z)wqK
zlHpvC79sXw&C)P??p&VDc9Yv=9<SIGH^Efa+1oYlXRlJwp>r>N_&$lM7_V%7P`&Oy
z!{*@Jjr^iV(&{3GH)?TeJ9t*7-Td;gD(pjzym-`-Q*--1h%Qle*&w;@utPB`n@N%L
zQE~k<tBdv=Ha+{|6|cw%X^-CCn;|!sI+ZIu=<VFPj`51&6^lBpm^YTfI{z7XZ<sCG
zUS)5-Cq8*Y(DPF+oy`U+*6*C3T1z!=+}ze1Z1}P(U);vx{HImUE?2j<Mhc#EntH+M
zoQ|I3Z&!sW%(Y7v1Sa>n-zrsaeIz>dKf@7EwtqsgbNdf0GuaW?lh5<$^n;qHkedsh
zy02cC#q^)yKSN$3pQ+FF&xdSO=S@HIX1V^$oI~r47Q9{<EGy}H|E*QnY0Yd|zI#*E
z|9byt*!pqL(L&Wa5C4M`@3u^GEcc$Sdt$oc8=0cMxwkvro0G(Qyqq?i3Hcp%c7>?Y
znMq$WF71*xUjO0Z<};gD8v9LLa`JPO<tJ0gE%}10J#QUNS==#MxGmtN^^9wezTLj!
zlXYJ3usV<GoD-*(Oc7vj51AS=ZCAwJgH>+!TlQGFiwk{}Fgatas%dP)BD0jKMdzYt
z(kX{h?yN(1nJzskdM+zC=gX1UKz)B@h2Nn*w^SN5lAl(lh{vUb?D&1A{B8lKOs@9K
zpNEzzoKe)`WDv-S{mR3-P3F<0yd1C8#~(gdZ|u-_kkNSe?zmILY_2$)uaEvEJI?wt
zub8LOSIlGOEY+Y@*`KbQObhU7?NHNX+^HfhEqg*_)r6(Ke;!Q<sS^y>kvBVXbxYOW
z<mKPE+0FcHg^j1~GJ3f%>CAy2ZH-4|Z=Uh<pYUvFU$pmS!6R2qKBrEpoOD*kZ%tR4
zSx-Axsr)Uq>%pZP74CBycW8y_Xk<BgiY~U$(>R{a_U2-8|0AK4vwt3S3v6B3DbBf<
zlQ%icypVU-eB=KNW>tI>Uftiy=oja+sB)>cO3R&$Yvl|ER^0r%a=asCd?gxIE;Vus
zds^!%clayg<|{Mo*$>o3YCV!*u<=o2(2x$h|9AFRL#9V6nyXj8ShVbA&9k*HjdyV^
z7Gb=y`bzLFS-rMziqQdP>T_+|bc6eYrl|`Uv$I64UAL7vT|`i<PAKN6wxpuIUQ*1~
zrp6UEXZdCy+v+|mD70j4sA|d<<KWsAMpqad7o~maw{@MXtit%L;Xw7rtTjQ?l``f{
znLO+3qOA-95C1bfiu_mDF~2d&u+fvHB6oM_r7tS{mm)s(Y|~qE$|`+X)v;L{i$&IS
zdD`qq>+pOM)|~KCsei{2*_rNl#IHyvo_7qna&{pX&(q@ud<}bNykF6~<Q%`fXz#=c
zZ8}c^FP161vXm{!xW7xOS!?l!e9e=qlUAM4G~TJ`$&<A&Cp6Bx(#UDqt!JXXLX7_z
zKJ3}mA%0ZloT9D7LaWGADoq++<yVM_1yp4%OY4igW#jU5?!uWJ(@q*WrmJ=p^BlN4
z`R-2p&zuQw<qBVUZjM=QoYNKP>O9}qIOqiTq@pLR&4)g{i9Q`z>%3F!){PfG^bA%u
zN-SBmN<{CU5L@_e0qyr`eJc+wIXPeZvx2CX#~J%)8BPuBkNEaYuR8W|Myjmuqc-1*
zH~XE#pNjl^7O4|*RrpGXj-ahU(8e<{&HE;OG`f9dcbURxKg%AMH8p2+KHl@5^^`~E
z-Lg-nwRVN)E9wOfeaQU$<fFlw$&LytqT&T1({7#iyznwyO4?Q^<*%(%M$xHXD>P?o
zPMS4o-p40DyRPU=%1(+{U1@lt@!cuWjvc%mbNGb~i&>VgS-v|!@$^d1oUM834|THg
zF80TCHj7$ig*b&2H~EG&O`32^a88pmlSi3~Yx*+dDw*?&uXGl#niR#eX>QcMNhxU&
zFJ2`x?8#XW8RT*C@y=VXCI4RTWzQ}MHu%i+VXD{9;Ock0uJ2qTKGv|@w3ulzhriS1
zX~04)&yD3pL5nz^_yoGetGQ2ow%K>)tUXt=?u1shtn}6K$?3=u)3ux5RBruv$%Bff
zQvzEnrYsP#zuMh2O-vzp_MI7**9-n!`NB)}sauq}VYuR5|9FiR3k?IGmAp6;HE-`>
zm&fn5Hxyj|#^z-CM#uM~HtUhAKO;A>hCJz&n6^}V{mIi3d*(HyElmHa5K_eZ&Y}L6
z{$@wsu;hQ)dX>}DJLBzdJ!_k_MCJ1O{|txbWjJhC_b|V~!1gO|N1oogH=Z-v!lrQX
zu4eGm(GHX>QCSvWGBZ{7#wkud#{Ue-Q~!Kf`?W)E-PtT-xdoq}oK1M?x#{QCmkGb6
zPksw}nI4vOT{&dJ&fs&)%<gVHTd|JK$<S-2O1jt0)3YXg*~-cOgIE0Syn~$)HZyrH
zY>3&x$G9+UVZs8A+<6!E=Y3gMwR639X}OoH=M3?cZCjI@wpj#C%eeYz%92S2Y}PMR
z?bgm&oX2s*|JSQ!x*11yGfkPqSj6SAR`1x(nRmZ-#29x!+&%M$PNHVQl1Yv#O-_>@
zDt%`=vLPt%<MOZ5oTION+1bnMYARy+$78GD)x%LsmqasVrlu6x+&DkyqW7w&p&{>+
z+fTgZIb+8f5#V38WI9WYij<%ildG$C#EB0+DSNaoo2q;-<#A>I)1Xih>ab-i_qIiv
zUw_UGI9<$h-MVYta_g|JoMW-vtJbhtJ)P*e{g%iE9S*4rY{~&?);#(me-16tH~%qZ
zl7!R^i`<}h?pq>HA5sl&naQ9k$|=Pz!f;=6Lw?JT9OsCHk4n>57_Yc|q%zWKTH7Sn
z_LYm+qJ9XjU6{qN=~^%gXVQ_g@+<QWG+YQi{Cnjp36*CaUYosC7#!_|STC(Mo3m(D
zeTr!7p=S?ECmVWIx=p$huX|?eq%S<J0T;T$_XK=+w_m%-Zo~4kn`Rk)I~DP;&$xcO
ztHI2$tjx9PEuK7?xif2?T-`OXXPbK8YDrxk&X5g`7nU?}6l94wDxV3`s}{XAvv^v8
zwxda){x7ehT_Gl6p~4$aMLptt#FjZFeV^i&w)CqeQ!k~={1!eXbj_8iS(3#O4@_37
z<kY?PFgb92*R~yp-!9x3fAHffFCo=6OFdMixLQ_B%#C;*<+AR~gXbq68ZSu<u|CN)
zW17N$hFe<ag33?bJZ#A0RVw&=&n@?pRms+;g91$^p8l3PvHaS)GWFWd{?oBGD;R|9
zs?3DNAIvs&U@P9uWb(86mX(@Z#Qfgr1~=OCl3$0byn3g}nYVad&GE(aws{&~p5p84
z>*yO)>Z){1%9C;L(XCp|xo>!Tzl&B}YG?>J>5;D3sih*k_P}E6@TiWE4?B~>lr>JM
zvx*8<ZSdVGRP)n)-6y_{^HNS!H7}X_;@5h;$svIu=Cjl%g(yW#IUyY}G1`67lh<s=
z+_TkKyQegpEGpLIV!U^TSLfdMqyr(xg&xK5>Kk+S@@plm6iJx;&`vCAa?HaS?WdS$
zoY~{LG;Y<)IkQ(4S;%hc$X@v}%VpzFw|V6bhkDn(l8fL@x~6KOa_pA7cCV^ZKL4x*
zA<j!zKiqun=X$T+l3w?eDQm>c()b1LKAN;hVuvQ{?gRfB-o)NK5PDd{=6YEA{jb~o
zpVj(J>a=Pyx>;7RHa9eAQ`@rDN(Y3EVr6%HPD-x7_2_uh`9Cuy%t|hLnk<Sw(zbfz
zb|ssm=FAWNGkm#|WRP>ITQBYamoEP@=SL3tjSh9QQh&BptHc^#j$HF;Zpgs}BB^uz
zo_l$(^;><H@3yGft>4`~Vy4f|PM&?JfqScWNLM%eC)3r(Zu1_Fmo{?{36{BXX^G*o
zeOudN9p|U*btrnbuusb6+|0$1+D{(s3-7+9rMq!r1A}+sniYJWimiRZ-KP&f3EAk+
zdHHaV)ULcMBGpIk2_M?<$?3yn&J(he>^)fQ;(xTOYhSSRWtNr;zZz1R<~#Arzv@Yj
z^S_k3wXQv#6?6HtXk4q>s#%LYO{OX^?K+TBuq(w=<gJsz+%Gq~oKA&!ge9JR;cUos
ze)*R0#}+;(H&5?Ayk=2Vyw>p!k)0oO^-epQCOI|l;_@hVdtEo@j8(lsv%!M1n|Y2e
zGnPn+nYXq3z`K%OPb21xXFbwZY>5Su5v!TIW8?*=+A4CHPCLiQHkJ9p#HuJ0+xd@z
zJ{t9GU7(i5ac&w5Z<lwd>K?6x5(R+^d9z>s`E0j%wr1Pb*P9h3&qxU?7;0&{_GzjV
zxSmM8y!D`BDfeTAHA(h{_s$edPyBau?)lTjUWY!$X|^Bs5uP&5Rp3qBI<E%NW&7SF
zDO=8b-D>{xs#RcmsloPJr&fgZWpwPAq?}x>llIDS@3-6yA-X}&Rb7Ox_^#sXGpnEC
z`S8+YofRG`r#&T0S+r9tCuuvGwa?YQ8hXWYS#<b}Yk}U+E-!Kpd;hH?YjMztq}hJw
zT7=eYYTh8#ux$ZP%%+z@C$si8hs+XQQGH{#t>T^ZmQ!;zGn{;R*DRSObMpB{Ptnb{
zZ2vR-Gu>qVvYyxe5S!~|Tf-|tU({S5{PUf0<tz_xAYW|YjIFLA%QD4&nQq*+((KD8
z-}mu1|9JT)eONVP!^|B!Z=Aep{-5EMNc+vEfC;9IbGJU)-*(wgyZ58zjF~6iKl9@h
zWmw_u<iU`2G)q-;MX?vhhc?HnlcnbMzudp+Y30SW(<WIlhpS9K&1o0&a&730%T{_9
zH>AYQb=oxPrrkv^_mZtm4yPO!teSht`9`15Gix#Xrb}fSQ`Hp$!vZGHZC<cZfpy7V
zN7i_*6`8s-f23X&tSG;BHP&`|SeoY4uRMF5G*|N9QeWh<xZ+iq$789MFgL{uMnc<l
zY`xnG*4gUi3;Jx;);ZM1c3_3((bzrCW%m>lt%I9q9#v7Qnz1fmnr8v`8CDiypR0V_
z0=&IE$st#pHnr>bwHq#7&Mgt;8GJ?EJy6B@@~I644e#{KpXW_I^zos#u)z1)3|{V>
zBjSczjwD{y%Q|yp(tiel8%<7oXPf(qHa|P)DjJv0YZg2CtjXlddvg}4tOyo#cj90V
z$k*xSeOICMW8)(kZ<D7Cp56?LSY-r#d%3T4UDy-5Tx<H>Q^ke~%u}pG1Zq0U*WHp(
z_x~C*S6bIM%=OC8T8BxYnTvHEpN;-^?%@Ttp8Sd%_jhcRzK|rWRrFcsN&vfvxK7-=
zdj;<_54zbdeWWtsz{aItHx`uHXWN`BZa<YJwRwrLR94J-<K>gj_Hjy>s%-RJapZTy
zE0&`ZCKmNs_jn&pe!4EHP}oD8rOWWgBo2iaoT4&24{WG0-{&yB&r@rPv;8X;<K9)0
zp^F-=j?_f%wUzbpoOyEHT>m+nT^OpY|GqTg-6Fe{amS6@wX2`43=Rod;N|+I?B+Vx
zGPc<+oqKG~IQ2|8a<O~UHvYZ0Z*gDg{5H#`ZBfSEWuac~yPKDJL~>;*Z+5yUnE!=a
z_?+&UM?yO@Op?zYYUR5U+&_i6ReSerLxl}`wIbZx4K~D|yO?rTap{?=`=-lJ-&4!V
zdbxCxTEcqux7=p4->Pf>XV`x1`myTfwLv?If;Qcp{NhUb_0y*`3XGR|J=}EF^wk7*
zErb6I$K#^*IDEZuMzbniZ&};4Wlg(G=B)_YoV_t=!E5$+lf<q`%QrVaw4S+nO0o4*
zhG`dq)0oyi-xB*R(`b56(6m@P0pGQ0zdn8Hy6)@fJ|{IWT(zv;ug631(#6VG)3zQq
zTbQ=@+jg<<2h={gdRvMVt%xc9X1@2)=WeYsxyos(OQUw0|BPIGLyoPq^VPfF`HQN$
zDl$LkObUtFa`xrShu3Z`3bC)yW|!Ko7#nhjb(?hW%t?iu#}ltUoFiFnkrudA!+k|z
z#KZlawsZcp3;cN}V6(?T)i>vi#J>KrrH%nBl}yhyoqlPYxA8US=e$KPXQ;JtY74ck
zicFk%M{ilegXq&5mqaxpru{u%?Wb;+WZ@+?GjEcUQ=^#rSN4Tvy8~Yxag*P&W9h>U
z79}22HcfIWX5dJ2V4VNx40F@jJ%OKn`X*bgJG=Um#HW1=n%3#-EfdpvHsg*#X^}Hy
zszXHHZaK!?7Jg=VVjeCZmZa*5*w&~!7ggSz^Q>1)So*G~%gQIGHTCouFLtcB@}qRa
zYGb>`u1RtolaB<>cxL)@kF9I;^q{9xC37xccAj=zs^4nSLWb6Rkv<!LdL4P$V|HtO
zue;=wiTZpqS2<0zij0~h;xzejO6&{)uI8`@7h9~i{C%|Xc}vQvW=$?_uSp_l8k47L
z>TM91v)oWTm_4n1XTR?gwy3<FFSB+Qg|Ao<y7JWA_d+M+&v>p_mLIZmW!Q(E`m$%Y
z>TaBK#Am+7_h8AQ8=Lx%NWR!`rZlzTRMvzQ-R+Mmrh400OU*f|sk5<2TGQvqO|7IF
z=R?abT++I-^IHD3TKlUv_w1}O4VnM!tLYWDMFOSHI;+%NOSK{eZ<%eoU)g_5A*tY#
zLD7?y>c;}7hVJHP>X5qV`T0(mUfPG7GWMcM4#C>Hy}X4>mLKv^*cjx<f3nB*YS35S
z*!ZT6JiAqzHP@Ve{b>1;Wmc=0-CC@@$_!>o8HBq271Oz{=z5N;W^u<+fvIU;!R36j
zjPr!P-uSe}f2uL3p1(j?YI#Q#*OZL;g3AJbH96|#&T26A+c@h&x^48ODN}Z?dFyEx
z_8~OnvQ7Akl`Y$vPPPAMP<CBmtg+ifJ^N?6WDn1k#E-2BJd>Gbv@;0F_AS(6oV#L~
z+Q*gyeuon_dK(G-eGuST&!O|?o8*?u7VD2FpWo|yN%Fbx*Sv6#i>cjmIdx~6yk6Y3
zyRtaQ&|E5XYU_-bJJ{>otk}~(c3K=g;pNNR&snx<SG~q78Rg!GA1hc_o<6xUW%{Z7
zr5;M#b6vdqTFyLNx61On|5shr@af{yGG6k$dbM!p#Jq)N($aUhAN=u(6BhfdG9h&4
z=^wrhmtU?v>Q(KltyHgVsO3HV%$lu*3s>5mc`K#fzVnPEZ@o{zm8D^hGsB*(W))te
z<`_^q<HW?W3;rv3x4a6_(cxG)zxCaQ&(>R8mudzoPZmGDGkL8?@$*@4^LQRD@tZMi
z#>}9*p*dU1*|se=yRzHu{-wK>lhW&RPo*wia&FVCoL9><f;KJ=o_uA^C6Ul6%qjsp
z3KKXTH*o2>DL?vA)F&0_H8U+L*6~kbgF<(jwA944ED06o(+wrk=7|Js_4ZWNf9&ZM
z^Yll^0@v8Sb@?Jo*Ve}B&N=wxaInZ@)iu|p*?W$fOz2ddW)Oarq3d_!3T_78jq%+l
zdXzStR_W~2<~exMdjiuIE<RRo*~p`hKIo)-MsF_fO!iu$*`SlLJvZuGms+Ings{|`
zTXp^YoTF|g8~nD}yocA<@8evd<<rv=I6@QNnM_#TeLK1Rk<fm<siIR>CNLeaP@Ewy
zTJS_{hhDCfPRyC0v_&~9Z@OGt&BV!)=2^}hBw@)Po?d%FPVrKz=}x1kccm}QwKNX7
z9MrYTWJQ<Tnl{%AIoC>A&rGL<R&EmmHFg9iN8aj_>YP-4Q#JL@3Qb4PQ;uf~9v;vQ
zSFsLKC|kVg^^wyv)RV-G4=tYZR9R_~Q$tF#%a4Y7F_+ob49{&lv00zLCU5=OqFJfS
z)=q2>3Ai-%y7Xml&7_{`yi0TLC3R=InVAQ;6>pkT8nhuH)YI|Z6`hAG;@vmczd0`}
z-H`mWu<l%s&7+s{4<k2nd8D(Q%bS#!?<~AGbWXt6b-6RrcE-hC{!(H&XVUr5$*yZt
z7StR45n$=y=9UfD66qFjvohz6aFgT=y0Y+B*uMr1_9UsLE22GCKd6x0W6jqo9JXku
z1IrnmQ@lHV_-cEJv&=raTmQ7?FY*5jOMmverT_ev8y-IG%H>j4)dHTIp`ohY8li>q
ztLAyAEt@F1bxV4H%G!oCk1USr20uwUpr^W$i^1{?SJ4XlBU70J9D=);gB`n$`#e&Y
zRF~J(;E3{Bs@9a0rXZe_u|-$gtK*c2M8lJ#&NiouV`eR0{i}cBGtrrR>C9;b{;C>P
zJsXZoWF}9H4?3>3*?@;RCSs-7fekahoI0e=-sQnPZNY)7S}YABCd%a-_heR-skwYP
zarkjt<fJ}V_LP+;PdSQeDp-oF6|Fc^rMqkOD&r?z>}Rc}uSj1y!JslObD0+B%})IS
zk9ybh)&jasA9=EZCL7j?G-z)(W1Q{qW!c14k6#{qsA4rMX3L5U2EOZJE0$)Skve3|
z=r~W?KiJ>f+v`eQOl!oZSyv`5t~F|WX*I?6RIB>s#Dap8k?MZ7cW+wyEj_;R8QVMV
zg9R(&B)(^;tkAr5kBO0QZbnq=fwf7qJ!gDP^LKUndDT`gk(Vz)^U=IAp}jf>O-)TY
ze9!RNIwx^5G}l-ZhV(du9GNP__t=2<&60`LeM$Py9vTEMTakD!Xv<EfhW3OF^*16u
zEfmc2+os7<8Dkj}S6Z@LWh;w2i}h;_Po4GEjLF5HcsDMbv*aAB(VPu=$>Fte{kuaK
z+pIgswAt2s5nI!9_bkCLPsOYL%r<x>Ri2UgdjGkiUsY0FGt#x<_UlaD!g+;(`Nam_
zg9@vY_4Wj${kXJL(0$P>_v`m&9hDQ8R_0nD;$CjBMs@ySL(@}bXP5O!J_=t_ykya9
zpKWcsCVpOJv0AKIM~Y{W)JiTbodCvVDJrW(6kdlH@vrBdwP~lThwh|h%NE4mTEE~%
z$C}4CSbZM8o~;-u5<IiY%6X^OE^kYghz6;NS+DQSJEQgb)vv`XjX0~4SFBvU+UVA=
zsP(o@$Al~wW;1*XT63*Tr^HRa_MG+)bAiLUn$uh3GQC7hd}BKpuDh8Pi`ve3G$~Ng
zB1Yuix1dR_OFd7QYX+<ikzT?W?AGyb(nsch^H*i%En2knOp)fSP3^X)7iU}(_J6W6
zSbFlV^@|+NR<G${6hG2uc_`eLzx~4*|4z1@lNJ_u{}S7nG4U4P={Y$Y_GopbEmvBq
zv(s|U?%uOOt{e(NCf$ch-l_zAO>;N34-MR_7ps>$$=}}~+<kfHS|z>L#xJIpuG(Pa
zF5SeTESkQBp<H5()&VU`HLHw2Um330M*8@qT|Szob8=p@uA?`{435Pb?QBOCkGITH
zbdfTt?3=!7Gta@zpu>0XhRW*3-%e(U)VwCT!NF|XBH!MhEW$OVmByi+(?TMebki92
ztqq81c1ycv_2(bEn#Xj#0NH~<MgJL|gyg?;?=?3#^J(X1E}u(PE3)g3Phacou9>g5
z^rVv2GXHxVA<Ay9%|_dPZCihs$0=ow(m$3Jm%So_4zBt=t>VPPg5b%jL6eN8uALLZ
zSi-ze?`7D5F5@F5rIW7MvV2L3$Xul<a-hxn%&!o;2_Bc0XiwmKvWhve#`j>-r}F{6
z-Jx@W?woy<drIZZVHQ=7*h`DPE~p9*4&Xa5N$X<P@|7pEGc<SQTP@0&Zs~F4%hs!^
z*VJ<T7lde<x#a$5$XV=l`x?jYg%y2jSzc+UW`s=%*4*o1VwXH!Ib-3>_9IG(M^9Ql
zopj10DYALW#3K7ESN$LAoiupqeMR_EdZk$P)HKU2l`*=JIz?`qPCIokOzAV<xyfuf
z`xL3EM;rEO{g|{=&o9X#s+Tuw;nSeJBRjKP*VngAHHuSmI#hjlN=U<^cul3LBL5jC
zCQW)~?H!b%Z6tN)`Et!-z3IzbmM>eEV*hQ`(<we{*UWfUee6t`;`DCW&o*vJYF?Gb
zTw#)a=0zL|U0ue~*?dPHPFdpX(s@)kG-%?lnR`@H89hZAviKfNk9n<f%D7ABy!nZQ
zRg<K=wk$ro(r8M=Ya^+p8B<@a`PaF4D%VD>Lq{}sNjHkKA5Z6rFYHQ4GYgd$2;3mO
zG~R7-$C(EfQ{LEI@i=C1ZqmxE=qEEKi8yJ7tmG;<BIMjJdS=et=(&Bfx>sZwdu~3n
z{G`9tye*ZdT{X;th3DT=7Tq8f{hp7*`t-x2!eT0#mT9aSX;;_Dt~|B<enihaZspJQ
zQ(g9S7aa*UxY6IzzOpcD_tCWGwZ$^?ygD}h+j;(l;)H3ly2^zw2fyf9{j$wB>%_L5
zEVb8nN(<>+?6zoqHd9qAkhR-TYImE``Kk-HNuLF8-ZXAF-|l%jm7ObiR;Z4n_1?pk
z2h!zKJ!f1mDGR=8DcHRD;kxx_jAiz#&dfZq)n=l}2ew`AR_*+$9Ft!7dKRu1sJO=z
z{?zA;q2`PTrWzNPIcH+KZF?ivM{GXu&DKaYNj>42ox)O1r)TVc9UZmO0zH$oye2%I
zy6{F#)ypccdj1-l)5+_iKL3jiX9-Q1GWAUH%ib?BGaA{{G|M<H&JUFGGuXo|BHlOK
zGI2u48{LqmZ}%2YRi67#`s3993|9+{yS!>LRn6s8cJlk24G?u&u$t+S&HGtbQ}6ov
zPM)S4Jo`%E(a<18O@T?PEVY6n=ZJ2e<a9@Ea;kdw#BEbM;-q4Z`slnrAH{v4SUt<8
zv@>XWOUO|!Rh1>y)4ZCv9B!EZw)M7M#I>$%QS(ZT7pq<^U*NOH<I>S*Y!h!@Epyzy
zqUo`xxsBfI<>C@D&WjwwU+HhPo%?!A#KUJwWxQTf7>y-+Tlx01YO$IwWHYtu^s!Zq
zj56B!B;Wk}$_P#0sgHtW8WS!~@4dN2c*(?<woTt>PIcbF_idqk^>rt?jXl$K^kr11
zwx#i{Zj~z1S-_ScyUx_iZpKH;O|#~_;+){`IQQc6pOdcmFic^+&g<44xxDj4s<n_K
z+pjyOJpbg>R~om5&1!z?R$jo7b3IGysJqOYjw4D@mm-TqiVQZE1;xIY+NJg8{UVQf
zE4!C&id*=qQd4eP-Hg=>ykbsGI%a5UESk1*$xe-n3qPs4@UDBHH*wXWs*S=Mg)SPJ
zM6o5fwXK`<l}|>ai}9&wLb9Pnq@thtLYIZ%sqB(sS4(+L2IefWEEjAlP5(JNt>ALl
z)0<0KWi(e#J)G%ODlJ)&#&YL(M9pKzLmNA1>YSdiz<KBP4%K<nlWqlFyj-C%MP+73
zK+u%P!=W<*>@NJ9ZnpBoFMIu^XEn5pV`gpf)Ow*MtrM|0bmF2kbFW@ux1+vOr%t(D
z>-E)7SR{T@^xrv?YhAhdZ!w#<dmkxeT^H0O)+Smn88pX&u`5P^*SkrsMSQM>)~%-9
zM(dX>KXb|QvcbgkEBCh8Zwuq`R66keW{UK!;{xfAyq1?|slMuUC~y|e?(Ls^V8@B+
zbB$7E`na5P>Vzbln?H+HWKR1!v#Htk<mWqE)lRH>Ic-r&nbyUtE~c*Gg63QgUU?V1
z`n2$7+sqmB-Z~l^JZV!q_VV_vrK?Pzz5K)fXVSC8tN&c4N$v9dsIc|eo(`{8*W8=(
zkx%dOR73_p@()W5u{@T0rS$&OFEeWnw60jP{O{`1ON@TiJnzgo?78*rjSB^9Y}Vww
zT(@av5VM8-$`a-O3@S@(Wu}XItO@yOch2YWpI94ZrcEsmFVFQ7P1Qd%;Z61wVXv3x
zjL+@)$kx3+dU@40o#i?&&s<tG^WRM#K`WJFrsdvZVo3~ID&kkR?O^+{;n1U{tAc_x
z^@P?eJUvZeqjg}1T;KQ0-|9DAwn(pT(l!>DWHiZ9#ZW4`XmZkbUcp%X*eP5+Q?t%$
zJ^grLcZkQDh_%bF%Jk;8G@ngXxPQ5z@`z35jF#t9Y*Jk+yDS!OUHM!rCd}u;)G52C
zvWdtq?T}gV;Jy3;`$wmCuQ2v&`_I7ixAf9qzapPU(+_FcbX9e%`gc4ySS@_UmB61H
z1suvbl+1$itObvjGaNZR`)137rsA2x8#5GFFwR=Jl11#p6d#`QJ%#TMCvE6@Booiw
zHASS^QY&ZYB;(T?7jJKmYF_tGX6>2&ps#H4Gt>SvB>JuB>6*Va;9^(mERQ>rHn}vY
z&)h6;WUDeex40(Qd)m~87g|EK7Vz=@XPEZsl2xBPSHX@_?<xFctBm^6xWu{iqW@_X
z$Y<O$lKH;kbLOu}>y0h%#a{iXm+BkHm$H>fyLJ_)a@QiuBreyJJn3QYqC-y}%NFiA
zbn3`SJ7K*f$DpDsPr}y4{uKDP@$8RzTy7F4eI`xTDhlC>{Lk=qg{Ehc$I?xL{~5#*
z<zmlleskG&#hG2jN9<<xb+29NGvUhfFBiGIelGRoR=<@L^4m69SaDCI@{Ha;&$jj*
z=Mq-TZ;Uj$k}<n2pw=(p=nbK$1;;ZECl-8CT`qq3`YfjzOIe<IaPbsPTJu6gMAbv=
zgu)hCak1JNPk-)Jt@nC<@wmeJwS9{=pI)&zG>FHIbIbPWE7zB*%IG#vGTZpb;^pPQ
zJ7>JQraIi`T<iV9W8dO50~>cqi_EM^3$~<51_e!LnWnHS<Bin9V+$9t>D+2JS+oB1
zu3OE9Ss(1*EjfK^@!tM_iPBXZ3e5@O&CA~>1x%T8hGWu6wgyx7z2)r1!fGFWPD)*t
z$LOrr=<oky$6Bec&okCFhkIWUi4;|smvbq#UdO5Fty;s1gMQab)7+wq4t(q=-tb9f
z!MQUG1&-g<mjCsw>D!oBw3=^?*NcX8ziqQmY-FEOCA_FAeDUXBnV)8U(z?@c(Z$SH
z>looMNoirp^#E~(-h+_}Wq;o%t=V!|)XwqG!GP%L9vhTHrd{pO)QPh!mwRlnN2}$b
zfzK50(?OO08D5@AujLA3-1d%F-7G7vFY7%2XMO%(+6I#*9LvaFJa7H!GcVUAM_!J;
zYx0h30@tQp6HIQ4eM(V0rr;CjYAMDWuBq4@ddJ=%*|TNZo&p&!Kdq$VeJPt{xpI!G
zw#NG&WQkM#HG4%B`?Afe-YiyJ*tPt~s^!;hb6s+EEVGU7ye-NrjShOZ_aIA8&<9Jc
z+07Dbk~*eM=bXlEEySoEa3FZ@JuM4<-bWE<jMN{APUo2D(!>~DWh!~&!po>*k=G)x
zx_Qn#7(M%JXkhr0ms1#Df0nX4lDl->@}Re8x3eUDXP9zs($tEemI;;Sx}I6)dvbQj
z^Z03bPFG-<zU+$KJ2TTShMW@ugVmyC9%UG<R8KZOw0TQa#M@oJ-xxB>>-p&wEj!~h
zC3fx>v7c_;o&Gb_QoYwOoEO)!Ic)G^z3}H<JQubUo(z6wlHp+}$+k74o40EchfPfW
zHpK$VNrn=Fx`|gbo;yuWl8@ba$vEPS%0A_TPtQ$~X?dMI<*8+ZRj=K|_w~G=|4ljn
zxo5N0`A@%Ut_AW;+1WHB#ALz=ef1fuj~$QDKCC-sZp87AIV?XdDtBaRoywcUxAI5%
z#)OjOU}t%gvd!(u1~-!153Sc=(KP+*bmq&G*cUMk7oYKbIe&5I`=D!2OAG9Br@!v_
zSzoqR@aR(S;(~w;%x!G91<Zf=<Ujl1@<{k$1nc@Kvr_dUm|}bH9@}{BM}?m~zgo<d
z37tJPo${PFyBp6KH#xtM6aLz&AO58+aawArh^_q9ul)Y;Tt}@hD|^dr-IB`s?$sHt
z&5Ua&Ot>Z{FBkGhW7^7On<-Wv3w0dIS?@&7Ft#_AymzZWx%v2VK^w_UK`&Gq79Cyp
zEp3Mwi^aA1*Gs>C5iz|obFEo)ybhaJkc+R#2QRnvrU!n1(mlmi-B`x(F(X}Ct!B#_
zo|O@PLEOuxn5;iqoXhR{JUzub=D<!@&$EwKvIzcLoHBdE--kzIZS&$U-4&g9;UL3l
zwVuQOQXU*>3#&*;o#;FzSMLOO6W=7}+S?B|FZYZyO4%-X$nlhplOtoZMZp&ntrdG*
zCmOlDiBlFfe$4tQU;;ysqe#I6sjvl8?kOIbSzUTAdiHbfsm?YgpRa_xPHSnsx=_)n
zapruVq-!i<Y02Au<TKA$T7EO6XbDU6jF?%|!lsJcdMosM=lm~E6F>Z%F=a{JTA5iB
zB+vb4;9#DTwepJTM9=Bzwc>M2_5Hq@sv4eo(ep3n(W1gv($d<^*@l<68<RUGFJ;ka
zHMyDE$RaX#Rot8zo3}L?%759JqR)Rsby>>N)k-rj9lCQu;Y!F;)gytsUai-9{jPn<
zM|R~?8k^Q{JoVMG*QCt&>E_k^TMo)S+%vV@ScB=HEc^0k=43@-gNT=vQ@p%cG`d9o
zO<I>w5#q@gJGW74-8t{Mhb_8x#A&8R9`N$~5hi_L@v>9~p_rWk<_Sq&(@j>t+;wF6
zbp5DPnF^7iPba3ft&-o9+M{5#_7#hws)Ko8_T%V|xmsBwO<o}!>~5ja(Nc!WCywT+
zGxenGIhYf>f<xhEr`eMb=4nZ>i&I|wS+4PV&gF$qPrTNUn`!*xm0!Hgd+lJhODYj+
zldtTGO`M`_92w_(gZuHEZ!0V2$O%;*S)zGi&1#-Ex-&I(-p*9(&CQf~-k)43W6a;_
zr>br^#Vvv1%=sxRH5V3yOmg6J*r+R)`C-1bsBHc<;VV~pHtQ@otG(pxVYf^F8NOYc
z=4c&J9;@sxo+%O|WVq?@8LuBU3l2t3n83V|n?c0z^S*Cu%KUB@eAaJ2;j(d7(uGFd
z4*4gi<}5zxALnrF`Q=*wDd~5UW%WFHzHB@`>*x$;+5WeQJ?=VNCihO*ePB~-ZR0G1
z^3RONvbzdiPFj+7L__F^z%NUWAj>5y#QrnHTw^k@@o)A&{Nd9YN69X(_*9vL+MfFY
zHcgjO`Ok3Vh+5uU%PAGjk4?S^r&qeVO0Jy0Y}GSfndzb(>zOqe)7`ov8RCOB%?-Qt
zWAVX4$xjtNJ5NUW9G)sHB(c;pKw+Ahrh+(+W48?BQQwCVj25bil`a<~`hVw%@Lq6a
z*vnzECi&9!)SG4Drfh|t!PBp-%3d-lvuka#?4Id8P2Suauih=0mz%qvcfP_yN2j~K
zkE$<7?Y;RT@^!3FPH@v)qk_2#ttuCN?8N>$yghx!*zwo5-KMW^?y-JlH}lhfhSxef
z<*ZIkD|dRcOH4Ja{?ERSvnQTxjt^H?))d+hoTgIobmhuZ>vZ(KuP#dDd2@`-pW9EL
zz1#l5EJu%pi9xIQ+m_99oM9YMY}0Tw)b~oKE1$2whTGc3k?en0#s^2XzF9hB)3)cH
z3Uk+`e~?rBS+vG1=hBs@Ds!Xy&oG=?Q6(6$_HZH3r#?Zib3s2t|DAXA3T>7vTaxy8
z_2#NQd$nG5>HA21Ti~H^%VJ?gPpSvg>D4|dhC7pVJZ~h2x6W6qxiWR}ku7^`4*JHt
zlANFIwp#GO6SeYkz0FNdbMCCrJh*#W(Sv}Di{=xbZ|U`(5t4Rj-li_kS5X;BbGw;3
zpRyS?Z)r2|-QBo)!i{g%-1k-Qt0g)w3(EP;bY(+TkkX?aMVGcL^H6k9?PLxLIltm;
zZfEWyz2+t2TCJbCDvdLZZq0mZsh8+=E7~R4#4XKcAx}$V!GZ4Eg>4_^X>3`Wuy)!6
z<Dk~pelF~*FHdNGyyJMyzPFLZDtsTc{xeK++VmzyFsiNU66>t`8DF1AJ}b3UOZJ}r
zTy!dLT)$S`sbwpt{5)yeuwicA(cKn1@4Ss{jd=KEq2`%sIlEF+Z(66CGUvL;v!xXL
zlh=z=RrPS-Tzbg;kn`lov^kfvxGw+NzU1>Mo5Q*%&PB(1&kDJ8<fM1B+{}3yYmcp+
zky)+D^2+1<Wg$N+m#-SlbL9n<XJp5HFDy6fzQ=pi;FZ(zG*(U(p-C@OE29>kNH}F^
zA3OO%U}-5^@Ql>2n$yCb?)1x7DY)Em$}Pd)hBbAEw$YWf#}2*y5nk{os%Wjz)&C5+
zF3e^8UccJq)4HA{r>$p6I~8!I)yLJ-^i0u>S96vZi^ps>$}E1h>&>goU1j_BEk3ns
z>f+CKr>F1km^Am^18pVN8U6RU#U3QpDLkKaNn-M{XYVuv_zjLV8OJ#Dun8J{IN;;N
zyu{k-z?l$cYmFcA?qM727N0g^e>L%8U#{QVD|>ANd4Br56Ll-;Ua?m81cTQ_jwh{O
z86re(Oto0<b99;iUq=&9o(=Mc++Y7^xRtTs*u+^!lM9`h|1*S7sXyAm#m)Sm!Dm|O
z#k-1mGaFY(Mjzbc@j5bmg_hRmx`nT5{&n@QNSmxb^I5!IR>Xmc99Nfc+ALSDlMVVF
z<i>rdVdg`pp!wWlBIODfV&DC07W^%L^JV=+j>D&1w(13N@jm3zoAl=CW?spIDizJg
ztRERUUf#K2)ysq_vgX0QD!SjQ*Z6C_J8^J*u2X2Ob>?o>w&w16(j8%;o=2`G|6TpN
zGJI;b^t=;3aeCq9eII=t*|wj`an3nn8`o8}C(}+U?YQ|%EzXypcYf;Ia$3>r&MHa!
zjHv<ZrcF@#xP1ky+R@T(*#l=ko-w&8%yUygE9kblYUVS`dyOw8^{zeW6PH`C)3G9R
z$+AQdMYqoSsSf2?9p_%Z*kkna<f{JQ3tx}L$NEij*!q)a%U8{Bnn_!d7ELM;iobbU
zLRGcreRhG{akiD4FLa)qGbN3!+iMChgX$%x$;k_)j_q?(?tawZ$8$PJBwpvhk~t54
zY+9=q{D^ffm&ooNddojFZQHk4|5ZHKuWijoMSt78%gkJyrE_m4*VHRFOP-%CT9Z>V
zZE9`bVO|-xI+>LfPmFSBs_&UrIcdV<2|>=XiK^c&1u<=?N%K4`E~=szsP|>Yd<K?i
z3&)EK1HbI6o$^%V&~p7(KHJt!S-nSC*=p6XMmbler)8Hn>m9f~FKv16shPUM0vnth
zE?@d)*FKkpwI-b3u4>X$MbEf}>%1H4%FNeYxDo8@eQ9s>%rwzV-kg%vx>k%kW;8Kt
zNUyvf+S<5Z?8zMGj<k6#s-Gr&-{7${=fH*LmvS>rWJ)@PF1*xIm=`Vk?VicH=2s8>
z5+)ljS+sIp)4U9gER~{DuaZjVO<HA|tINFlOxL<)-^3VLl=y`X^!O~&5_He9WQlmE
z8Q|(%`SiE%fnugkFGZE61<b}8BDyT?+@1`RCc9ftl-$b8>X&#~bguSviO)Q5W87sG
z940v{c|2fv<3D4P)?Ay+>V^BZSS)eVQA~_qvLn67mwV#Bd*Mluk5jEhbtNWt&oVl_
zFT3eMX#1fc-*QnggN~|y{`~wNS5j{-EtT~S4cdM6(iK%-))-g2^Pw^qr*7HA@uWA>
zoBvOhZ(p0(jWsNDT(+K?!#v@Q!`U!}y@#b`?+5b6rv@Kw`EuSNDAI>RGs^q9qvtNS
z1P|TST^3gRLmkb1g6ij##HxzA`&Eigxw%zVTQaeUz44ajBR$3pn-lL^dc7o$JbJw^
zzR^Es=X3tvRo#YWq1piv8f+@f@m<p-=CLe!v}}&KV4(0^wx}tRE=^367RXLJIz_gd
zby=HX-sI_vRvU9!T20E^+Hcg<66XCa^Fix^Rl6)d-B>L5q5Gv+%69Q>i{hk;o-R?!
za=!jhwQWklms|e|FYY*1(Xvy^kHh;^$YF!Pk}HOM_D|P3R(H%ez3Pwt>z>C3>zlP-
z>ol8o&2n%Gan|wRYWh^EWS}yY>+T_=Y_ZbV8#*y%d<h;dJFn}9WM@rS<UZjHbJmyj
z#}a?+V{%f}47%2;k+GmOapq~2ZVOBQmf3cjXFM~BbZ^`%{OsAr@L<OoX<nPo7N|!q
z(HG))lM-2<@ya2Wcgc;}+)oO2l%9Pw+4sd{@4!aA2ZlErHwZQ#N>od#_Ut&-#5Gx~
z*=fQlm3=|8n7A#<6>N?A=DxhXI9~W=!qOFIUT#vIwW~ApReiqSWQkQPU%5_u)AV`5
zwT))IO|Q)_J`xkR^9=ma>o+Y)-(7j`_ul)$kDAh-JQEU1-xTtwWx2`L<q36Cw(K3&
zuGMQkw=ZsLUaFD#vM(*nYR;qR*P^C+YfRoJDYeQlR9;uyxVO)wLbO%q5m&KKEW-tr
zh?pzel1dhI-eG4DH;g#h(!$Omc4~rarqj}iZ=|QVq}<$=v~csY3I7>pI7||ow#?ms
z+d?+$?))|Py;;Ssm_}62m^7v5?n#v?6O$cS|ID~w{8r4yol|3*44c=iPUb`LX4e+C
zuMYg9q&x9SP{@<gO;4_Va`*I}emT^gXT_SRlE8Ha#|y>Zs#xdO?k{_`pNm05_{K^<
zxnGxVZMfDi5~=2WgyoU&qo64=vYZQ_t>TS3Ch0#bEplJDX-(<OrfZjvg>i7OYCLvv
zYLUOIYqaZj^HPV~Ir7|HYFQ~7d~Yrk>p0Cmk$U}}ygIX%%ZDbmP;vE8ArHHpY5GfJ
z7ySHJeeLY!$cwY~^z>YH){Rx^^O`kZK{F+AOY)UPw}l)Nbe0S6Fw+W{v8drlQ|uPD
z2W{&Ad@|m!*tZtElbI8<rR6`v)DEueYDo@Ov*op=CJM>Q&V4#FKPbdL<f&v{@no5W
zMz3xtg}Se8%2kh;T+ADj7_#J{2d_k==%R89&rQ`ggN`ho+E?H`;eq+I=r)tJQ6J-P
z@kyJN@$g7p>704$Qr5|!xL${qSt_jx6P>&`B77o#g{`XVKKF3J$}`GqU;a$%H(t6Y
zaN3zn&rh*`wx$_cbS%v1mKMn?xpTGfj7QA9ZxO2f28@13`mZMY&eGm-Wd5%<i8EeB
z=NJ}-+OhIWtYkJg^E2d{(s?ibkn5LH?^@1Good-txwAHB%c?g#y(&(zM|=}*d2s!T
z`ewEEzWC$|!G@8l7us*moniesFZb+Y?}Hys&+c})qBL2EJAyU!N_aS*o>sE?%dT1f
z_N-a{dcMHobDP&qI<fMoShTauu4xjFb)Qa_``)OMeKPm7h5BRNBhz*sDA>c*B*3X<
zndLQO)&4iD%nnWKT`oFNyY;jMpHxui=X0qVU6Ynoy;wBAJ%K}TP3gCK|0|Y9O~XQ+
zFB_x^Ub*b(J)L3Wr41S0l?#vd&E%W-N?~PwkC21Q94*zQDuGw>mb0$Nej{b@ckaXI
zyc(zHg@rkFO^rD?g;_A&*O!rX@5`+p0-uUTE)fx#_;jLZX|GnQ!XieQ1C|9|Rz@4%
zzuIvpzw=>Y7AwQr&?}~n(!36$o9|7mZH;=qkS%F*)ZuW=4LcNGo;beh(Yoay&Hsr1
zunx1DX0ql5--cIl^8`(F&TL2v`SNqAj*)<0$Jc{#qOC_S^zKglV7u67pJMOBOizy)
zJBx4naFomx)?DAoS*H7l&C*RT>W6=BVNlPmWEIcoy>ebxHB*?2PCF_#uD%|x>wV??
zmBUu&f6cdCw5n<5re9KBrTzb&zi~3_%v!W{&v}jGB7XYo&I|6m(=AyM-0D>DBP3AK
z^-IC>q+1i>bCp*o{%|-rCE>%Sj<feP<S&VN1UlX}Q1!fcs8u=kMctlerI(LoeM2nQ
z`3BAmy8OD!Wuntw(~uyEt%9Yh-ZN&rtlg2sXZ&};L$;DbA=82aJU*Y{*g7Yo`+COV
zviX~{jwVTRp4iYjO;2ZK7T=+k8q0<PzXcLozTC@ozxq!t<jb+bsaK!f4fK`qof_cv
zLbIx=_rWa7m%57KeOuQ}x^zzJQN|_vdjS_W3a5D=^wJFQGG|&D6~Y_!QmLD*LFV>W
zd4YpI2MZE=CT*DMrNte<vMT7%q@WGXOHO;1oOd=a^P1PvZ~m;$$mY?aCZ7i~Z?ig2
zuYUS>%2duJTk@`LpC;j1ApLGf>5k2e7nW)Gq-xqpS(Wg)x6d~9xVSpvb&XA3n!_x?
z2d5u-tYB+sEKl39E8>;Xwq@H_#cNfGShj~Yg{@Lu#__f5)|Snya*nhx&dxhBw^7q5
z$>80}u7t@CP49L{wOy{*Geh;EapyisDe;2IUY1*5*lue|+i4&8<w|k7v-wG5o5#9s
z&NB{s$EeMk#MSMx=1HVPn^mOXY`-nm-BY(u^3#h7zB^^fge6U^MO;7X6l`wHx*WkL
zF57l=-h-u{E{BY!sw~m;u<VduwZKA!J;|)xR{xQ2ovT{>m6^UdS8GG3du2V!T6E^2
zgxQXzLAg4q(Z1hzy$)mbI2xzTF{diD?TGNigy@2+?B{d0H58sZar!80;&Rm`XC(c?
z(iaQVd$KT}*wn}+nEWbVOyiHsIpbtjUUs)He?u<$U;ZK-DmDL;rksHk$MVuho9%@O
z5z>))(N~TMPS)nti<xzXlX;I0TaxD7_SWybxwEu>s3)lvtkKz`n^tnAK}|<ReGkL5
zph=t_0pi(LXEmhX-1Fh7<+t=Xp@COJo_!8JbZO0DZ<T#c8bPe<IKzTZu$0{v+)=Xb
zBd1g3ByZ!*Oz%vp>jf7cPz?TX;_#A&mX$uY)AV1-$ZIsc;#<FJ{h9kZtCuZazi@fw
zi%swC-2O9cwVQi^?XEMsf^4VxyFyiNCsl<B4l1HtG0mq>{`2{mduWH0>cb;Xgv6Ew
z-H7|Xl-r;%?NcXD+s64?i^En=*`zmnH5>1}@A5H5m70O9k(H+V0{$ttc+T<4;MZ^u
z4t*~AbFJ0$HjT|IM1oH0u<lK)d$;z)hP?;bu32;*{4r%_F`u4Sjh4d3{P<ee`w{7^
zhZC1Ry!x~1v?;@j?_V#@e0)t-@R7TJx<FuhLRd;|+C*2LkC#1EmYi8+HF4{aoWO(M
zR3ozFX1^Ej|G+Twy;efZ%(jKjN2c>}g~&`?EY&M#(Vp>Jxod)-$fhSdH3M06xfY5h
zaaeC|y53uSbY0V_7rRz%wm7{xEA_;(C0#4dd-{EiEAf|o$SI+4w=?7WhTxSEb30Fb
z&?)NUOOqD<s4(G_jwkEX01h7QWi26FlC1e}b#C6^G1ZjC*Q-M_kTZ!}XYz*K?S&z4
zY`P~sv-I<xa`Cu(=*)vk+QL2)Tlc8BE1Jd_XgRWHH|_1Ss(JS{#m8HSfj=mAhP0{U
zlYfD?f6SAs`T0uQW98l(4KkZe<4<kOJNN6p?W=7sRn8h!nZJ5fExGE->UGINtrgA7
zu1{s!B~+HXe%p;R^_M<uQa=;q|D!=&^RL#X*^@d8(^k96^SpT{&U(K*n7?4P%I*cz
zxsv~_Dsa2yT(d9d@|vpnl}2y&c}3aG>l2$)v5F&8d)X`(O_9P6yCha`W~^;^^jNU)
zvrCW9&Qqm#H3K$qvP}Bb#$ejnpY-(Pf#fG=W^pak*`U;@sIaSZXQ2b<og=dP(P{}Z
zU1sjBJiU4QvZlV7PaMKS!g(gyu9w?%+w9SQhKdQ-k1b|AblI%r<ctENm98mln}hXQ
zmhEiYCYWlzd6{Q?NzVz-pzheI0S;B0LjoGJI<`Nr-2D6|!?J0SBFwTnRTC3TUcA|t
zqH}`vG{d9g?`BueK5+P9^~{c#MRH3vU!Btwd!y-Ab~MMLhcgUYkBC0KGkfKUnpF&Y
zSH?Q7U6ie}B5L1`<u6W&FF37Twa&_Q*^0_TPB%}jc5ps*En@L%HjBBlPOa6lRB$uo
z73TVI>fNN!2`gH*7HKUpwqDQ_bo7LG)<te%9p!dydDCliG{X|E3eC|m*wwRY!a1*?
zAVw{n>-Oh2ubcj2-Rzeuv+Pb^DYcv%D}H0^rjsF8KJ7Kke7gTlW>QM_3WXCVZgp-m
zv51K{e#YlYL%<T1+eeDSrkGsXA2IRp4mWwxBkD<q>W@rW8ab;$S201QZs`i?`7Hux
zrOX?)MDF@=`r%ccnR3xHgHyM@$zGj#|8&3_bwlxncaEuC9KV<J9y>1a@Ai40dz{@t
zT^B-{7))6m=O0~|^vgDCy+r;|&r9|{%SBC%y+RF@wLHpmW;O&|?Vjgf^zoTg&3l8F
z@3wsOS6b8)8u+aAva5>Dqht14Oio2PCxu80Ex6es{y?z&Acu`r$foHLULIzvoaN%2
z{UhEATjcbF^B+uFcHxwlhs`IJY0{s~ShmL58eXnbNS`U^?0fl2=*2Hz>Jy??CN0Uh
zc+X9N<6d^mtKQ>F-hOX;a-Zqwhm&cE^8&<{tO)6Py7<Cejs%B3@1xb$N_85KrcBx|
z7T~>s_an=TQ*8U(^h7PM`iERkz3c8h<JrcmaXLE!cljxLt}Qmw*)_@IiCa+6y=lz(
z0qTn80x6H$1S`WPd9Dk7$p27>vGCuyCxLfwb$`k@cq&XNWy^-rXOc1PIm@+0R;nJb
zYA-Gn^_MZa)3y8Xi#?8~vR-Cwwpuy;&!wzTZJzs^OZb8lx!HQ6OssOQ+cY<Rh|HRN
zB)s*=(g_p1m~Wm4VC*b#G^sLpF8-gPy<(43#l(k`4{~}~GVb8xw)oLnryTyRNByu}
z`?~X+R%-ocIAdI;b7_TtaK+OUlXHzc9*wy>#FJ`oBwgPYla|Kd9pRC^nJu?g``dHr
zy4z**;)JpS1g0ul1_#(U><HA~xqM07AB&G%H7~p;{g@dybFGG)kLlFRMOkJOryYI1
z_oi>x%LJzz^B3v2&HNGfpMh<Hn1+J_E1Om1ts`c~CeD9ro;F#b^N*~h?<rTO8R97~
zN(*X2*cARayg2sal>2F;Z%fylQeGN$E2`t^wW)y~N0^_j+`U0j+Vl1$#yb)B%ZfYG
zr~h+sy6nWu5&lVjZl=ffH_xTr1!mqqQ?h2YCX1mb)56|HmiC?NCiaIfwPo|ue!9$y
zZ_T9g*)hQu;SBHAMDApeZQOa~`m-BXz31$l|LyeBwS_M<4*wS1b4b)@{@o>eCNQkv
z(=0d~pXj?d{ljUSQ)w?>)y+P;EOMpR4Z{U9^%uF`@ppLr%IKHJq`7O?I*TS<W41g}
z@LAM&&*51MOic;{_FnSsz2xgGEc1Qq9le4hb3)p=oPV@-*e^@ioHez-WrG5HmwQ0!
zO3&HPZEYv%`jrIAWh(Wa)N0&%H`eYy!@@OO4YwybZ2e#v(J)!B=8;eYZ}+SB`AcU$
zc0aT+wP@+1k1UEO7{BVAId+vlDu5yJx%ZFjdg<QsZeJfAziMl2>+;c)HCl{Ql7X$y
z_nr~MjtQpkSPzOH{bt|laa32Lv;Og{D?xeu7j|6Sw?R+$hM`j8@elVu`Z9?7E!(2G
zDC9rGt9$Fk-wW<a;?kS?;j#7U&*o?94*yG@8W3W#D0A6VHO~iKLG?f4W^GuhaI-}x
zhxg#If;S-#`;Ks~{8$l^;Siv()Uo|l$)zc^$(A?s^wJ(1MuaX_tLY9%W6)I9()`ab
zDb0J?%?3RwgMUlTt$tN>W~S1tt)EZrytizbylQe<#OZ#ORdY`n83f+Z-EnL=YnNtZ
zL_&k+iAh>==Om;mv!(Be9)7x5V5<(JF0+Q-SDj+ZI|9EgD|SD#4Vrpa|KaiQG=b+I
zj&1prvN>&&!ln?1(6u}(xD6tnx@juC?s__9+v1jqy6ocid`*)UKW1FVtHrCw8|=Jp
z;>-gD!P=7D4?lJJcldC~zuGR!)_7$SqvxsJVj0Er(q@*2e|f4`e{Hf=_)($DVJuF^
zvJMLNzG1z}(onhTz|`ad>!kIpp#7oK6`oWzcxS!2uB!GpJHUNM9#bB>$Qm_<3ePjM
z4ZNSpob*5BxMJT&kJsx~tUZ@_+WW%OD?e*>tq!gCk#sq`K)k%T=u$%t`@8PG+jkS6
zdk38_%P+ql>}&G#t@z!U4>o;x8mAPt=uHW~16${ncM(7Nml?0QuQOe3%}LE$KDT-n
zaw=ZwTB0|*n@zDX#q5UQtsnKmQ~I7Qnc%?t`_uJ$i+YKNlhl?asZX4kpS?GJs^sx|
z<~iq<b(_8X)3j-=g5?xZ?@8f~!76T(7D*g73J%p!Y`wTF;-=^Bpc|QrxsxAWR#PlC
zIJ8n>Rl_ruSCdc1i_ev@DV}4c=6TfDiN9gFl?RK4u+tW4c>&J|1%v46d_VK*CtiLg
z8ym@dK2+4y*DNu5Qi!v1_VP2Ck3aM3^x4<w`0Q$(Q5E;i?AZ3!Wc5YSOQx)3u<M(i
zKhw-=NAGr?M_g{KDoaEa78)}$c8Nq=NQYh9aq(QYcx1<Bnb{^Exfa{KVh)|Ab=Llt
z^jq$gQ%vR+B%BE?<vn{+p)F{^Wc7VBrfTy!Wd|Ibe{%L}X}7R^#!BzQ9}di{6nD5M
z6~eQ|q3BM-4!y*$g&$TM{g(Gik1f5PC4P3&blX*ZGUw(kOqPp@&x|+}!kx8I%9HJP
zbWPY=iBGbMjP8%TB@=otIWNf%j0$il*z{w|hBbNeyn1VRwoKM_e6?m~r{m_P2U1}l
z8Rve<OTTn6b+1#_r=Is}^CYgmE_&5gA+VWgQ{^n@rd@e_llL}q<ryZoKhBJqtS`XD
zEhWUNcTU#kmD}zk@8pj!O%_?B;Wc4{x2glH*pBoxy<-cuEpa&_%Q`o3d2s1}hS)i$
z7kyfpZ0Mr0Jbzo-B=3ZGS1x5{Ojso;nxdVZmHd)f;*Yzq%g<GZKRgnj()8vfmpr$?
zH?<|PFV}E<Yv!L7&U<jn)vj4T0)!K1SOgYNneg~a_a0%()i2L9D=%2T=0)7pSMz49
zo2H?id}ejrq{TiAn>m)LCNG`2c;C!W`GCja(;Ox?AND=M9Lh93XNg&EM)?(u%V7pS
zOob9XK}$~0*%>UrDH$Zjmla~D%^1$oW}b5~`=hu`>!w{<T{?G-Yv#qQTenW;>Vi-I
z8P2>s)5PlTc2~`aRmk~_(UBuDMjKUnw9<GMT82%PVooe_dz^c>p-b-khoG4^!zN8-
zU6NXU%0WxTc$Y)b)W}T5R}z)8ZG?iJ-_^?MNt0SqzPzF{-;7`TlGoqcYo7$&nahyh
z-J_`OtbE9k)A~!zEvsb=%hJT^%d(mnSywDyTV|r{b<}_9D>={NGiOfP`9}tPtnFIF
zWOF3n!f4mTUEG>h{~5mSxWnUflp%eIf(64JvCUD(PhZmgab&_ot%#2;`x0_LY?L`Y
zi?6hIk=**}t7;Tyu2dFyHnTj~^_h;~Tv3tvsngHo>j;=kdgUR%du7|C-6_Y!cRZSV
z%Ynn_^uC|_?ngX-`5=Ds;|HrYJ@Yi*5mvIbk|FYILX3pCYx)NFYZppQ^^$`pT?~D)
zv{=)08rMgU3uomO%vL<v*t<==u;@V2X(9EQnbRb_%b2EIu|HoVc}9Fojj4OPjMBxq
z9y1?Y_|XvL_n?(a>z7zk!e7zLUrmcH|7Z;jp6)y8xv6N$*Dsr$zMWo>{#i55ZS$rz
zYh@=naCovr%~+|hBPsCbnwc3I%}I-xg-n*M%ZbmBmwS{kjh~C5^(2R;Lcd^Igc!>b
z@qmJmNz3>cJI*ax7!|X~Z~3aVE6(2VHoEBVCb?*RM@CxzU&Hx2ni2m#d8~Y?ohHQ-
zbF<4bh^eS4V#3+1ytlRfRxam+?M>dFR8-_+OIW?+YTxrCe#uGGlV=`@dbQ-F(YlEz
zimgOWXFZ;^sb&(-v1u0%&k0&27OAadeLPgAh;c>ZGj}E7Vo6;G=k!Q*Ef3v-S1UNJ
zg_nKVeALXx<xk4ZU{1-PnXPKeqB@fVasrmJr+-+f66vG4Mb?uq+HbGj&fLv&9)?+N
zov0ibti-lcY_qUV!^<bD`W_1#{BT*odAdm0fmdd_mic1Uc5@cjYsxL6t3RvOahgh{
z1*({<d6-_9y|HcGgi==XsMNbw#yJO$2a1Gpo2=?d>s`CHB3|#-lvj5>|FB+J<8o7r
zDUmgjx6vdts;}7g#Qr&=85WT{DsQY=S>Tc2%hx_{%bxg8c`Yj!%v)7+e8n#F<*Qa{
zJzBG1#boPGN17CD3tqVYUA6bGjMZ+<4IA_)_RR5^GSl1AS>Pn^`d3Pg_q{HCoc~C6
zk;?&Ly^xcQ3QHZl%gPtc>F4B)zvQ4CEc)t&Po!&n*wdfOytIr?b)-z|)0%x!hJ{hF
zOJV+xrc;*trn$XgQ+)jF`w>qKQ<fKu%5G=gxxTsg(VE}b<jaHQDqAi@y3{R9@c!dB
zE05#KG2W{)ujWUUW}e)%d}`)a%at03LbyDyoI0hOE%!L7cG{1*M}9OF{3!8s{5Wmt
znzKd6rtX`4$*P~>A;*D){|tr}mWyq<|1;Emw2s(0Eq2O<FLO;Tmt<Uy4TuapwAk>%
zuGWLeWh}iNW^Wr3yi&Zir-shjyXWl76aPL2w+Sut53|ZF$Sn-einzfQcjl39<diR8
zVr8Y$iUSYpGAvkD>azFc%`JwttsU2Tc)#Dj_haQu{aq|eRdxjJ6nf`SyrS@_$PUZ2
zG~1_4E@z(Ye{)XyX5TG|rC(F?lH+FumYj{bVCyTwrqy{m=EQ+bD^<AXdj!|MPb|K3
zZhD7P3g=4o<AEn?OP-v4E#=4AHmy_Z=mG(;7`Cm9R`1;wJe7O<p>GvW{~ybWwXPSZ
zPEEI4_@6=lRaT|_=`+o5L#E0_@`Sw9SQ#5IP3>EO*^cA9kDsXbd(7-eyr&p+HL4-f
zb7>5#`qb0z=W2Ix&%L#V<xElik<CV)lD7{V>btMd*HxQRc*=(Ps%YhrzrM3i&rN4q
zta~oCa?!k{mqSe*>~<XZeJk*0zztrHz%`7_vO6B1I$kx?L4ZMljX8PQ)Vk{9mt&OK
z+ov~AGdPjz$zZ^?GE02(#DH(X?K&UVf0?T0H&<QPBy?(|o~!Tgs$W%|JpCTdD@-|;
zmR7n<|GRe15zZaC3Nmf#l7_17y;(Zn%2>7=Nf<0ppV(tAJKHki%9<s*S&KXtx792?
zZEfCqAhwY&srbk#E-%07i&rjRc}BacTf3@eU)J*E=T)(49Y4EDc4S;Kb9V~ux|3-)
zk-bUhUvAMd0o5~}0gUT{PSvh@pDa?_cVunF*HaNpA$Lx5URtoU#FSM_MML1>t4AAE
zAGyVbKhpPo`cE}2RAfu!Syd~uIt`r({wq8C)+%4I^*MHlK}Xd(!n9@I*>gr=Yq(;k
z#Io$(u*=b^ZE0+)tK!@VN0nQXMGq}uN?kGUwP}dtT$VTNhrXWD4wT4Co?kzqcAA!Y
zSk@H-Z{^F2yiUu`Pz-+68}UNv_|+wow>E5_e)!P5%;1&k<;=!cvQNw_>FaHNcvvj-
zp0(1#PQ}yhWvL=p?_HBVB3C;9q$>M#PZ@(*J5pD!bie1!#JEs@`?5D@boU*QW7FA@
zb!BR2Yu9EL{(xe+Xz}M6TMrgI>D*`J;m9B>bLVx~@|9Lc<<-w@dhz5$%G%TCvXa_e
zHcY#u(j%M}DPCIm?mF9{KRJJbrYzCa42zm%x%a@HjKUqJPiG0NSunlU(Y!FtX6Ay*
z{r=B(u2l=3eVAu+%2#2NfKX1xUWpR|Yu4C*+%$<hYx|N{_l`693rv~!RjIf3L>A-1
zTFb*9YTQCv*Lmt}Y4AvT)@RiGkt=ZPlAKzPsdF8tJdt&r?JRS%)!23NvrSGl2R^wS
zGnp~BY5Ii&vt?c4_ZipdOBuYF-Qd0KmG_$kuXuN5oR|5}&?XsCc{wHVMXSvbW-q}R
z%XZtcO*8AMw?3==N@e94Wk<(YAE{+(&MT*V;=LWf^iARIuQ}1*g|4(bJnOjlCojW`
z%45$N+onts@Jm_Mo-JLpqj$CL*;!d!nh7u0HO`OGRQ~xvLRNB$>6S$=m9KCpZ<M>S
z`k*qeA#1R_fr^;wis;|ZCHAEARj%TiZF*4o@FNl3;&{V7al78D^)1#~an`7G;eQ5$
z{?pT<e3u$nI<E9ozADXq<IYRX+?^&7H6c5N8Rqf1o31#+`SjPM{M~yOo8G*qIH}<9
z`I=Q_p^eGTC!GWGejTg7Y`I1LSxCQA`l)?aIhU_%oai0fHs6r7Z>ibl8u`<H3ypU~
zF>O};tn%7x<8RxWtEKGpn*%4U`;dMjR&7DBQ;Fiso&E97vU;%(*E^Qll}Ak#*?BfJ
z&ehj8=GWqd>y~BQ2{x9t(wT5-Nv66Z%RA1?(_>EFn5VGE_SzJ+WbU<E1&g$tmpTUq
zdbzW@JU#Q+VhvBjaXvwlmadh;Ol@T%I$I`W2Ku=x&a&s3Y;)#Q+?JP8w`}%a*(zGf
zmi6JJ@7j*CTCZ34HaTlEEV$dIS76~2bH^f#tJ>F>D`Y~@0_J5$1R`I_Y*nrjKgQFN
zw841NCOegteB2BN1RQy$DTb;991U93y!7!ao6U+J?yoVHIkt-}qwCp(r7~M4t(xuP
zVHdLMsM*vA<_%l>R1?*bJ!aaA>u9oAzB`w&WZ&^4lT6uu$tt~z`4f0ZU-0OMhn&Ik
zoZ4J$ufzh<SB1#+Jvq8>-@N_Go|mMo)=fOWN<%7B^Hs?4CtfPOTQwPO*`8j#Zh_;i
zXFry5PY4rg<(zbB!Li1};+|#eTBP>Pe%^IV(bHH&-HSWHHAY~4@uJ2g)1LXOKE0~B
zyu2vq-YR{)s9P%)R~l|w*J&ETdhgM3-*@Hazh=&zd(>xg@60yGRg#`{iO0U*h%&Pg
zmRV`Bb4Ni&14p^Z)t8&M-Of<Y{MD~*(mf&U%GS72Z;rmDQrcb1S>0zXzQbMnxSGu-
z*=xhf)y7X51eRspp76x`Ms`HYK2Zz#Gv2(qsYi3{ul#2a+0ZL)@N(Up*Y~EKS^1(^
z{?or*eSMi3n`X`R{g<S?iP<|SCoa}Y{6vA7vbUV!j3d5B9;)%gG|y#dSj2H=lBc@C
zU(c*(GI!j(+{KT6I1?pqE4g7w!_uXD^y8)24^7jbYf|0byML+Cy586S83cB<tGsTB
z{IzXb|G$~%H}BN0dA?*ty!M$hD=%NzD6#ZE!}i%%+<Jw1t)3mxQHV0=I{IMxRAbJV
zwL%A0|H(?X472f2T=imG*s;QzGV9iYm%%&g|F&xHb=feD%duwDlp5E_iI1mc_;<=L
z74-jL$0*(}$7OWw>CdNvXR;och9sYgI@<4&vFO{SBO4W@-|uIUVc#Ngb4H(Nzi{9L
zt)P(OPFW9TNm=|9_~3u!lYi=t4o-~#FK?+SyC-G5lX^DAdPl!$MPs0hoyoHa2VG@^
zRGkA&N;dwqx@BH*wqL3<xX3KICevEgM4xA>eQPG$(i;N2i#L3WWUZYuEo3>XWn*BK
z29vF}`xY<#h~U$+pItS)_~h%>bLWFyE*-k0EYg4a(kk92Zsm8T?14L3cw)Q{Ej2nY
z<4eu~&3#Jm&#ikCxviu7fwoY<q@}8gla}dZ#h%=KM`|PcnPaObmDZNJNXEr_2J#%z
zxs~;+FyOnJu1s9i%O@)|RxdmDIJZGv;+^qf{)(NeS(}W^wgj3kJ9os*T>gNnYOuNw
zOWHJ{h_F{@;?5oYDRuaU;@MyOjn3JKo+-{K%A3%(Z21bcxepb?3WX&Warqu8I&;eR
zeZs!U|JE0Be)>H##wn`U*Mafgg9#E7u6nS0FfB{`U^MX-r-#;@1v3-uUlz{v(_6A2
zv&i|=h972ME_`+ON{L*vf#;q_w$c<$t<?%Dn>NTya<J-q5cT|4|3^cv*@t((RCwxb
zy4I_I_Pb>LA5-U6aNd}yd|=Wf-j_RWeY<9`Z{evGE30m=D5^QMYEj$FtS6qSlV+TH
zw{YggrhgypOKyC)`|ZfRGapSQjiyavh;rN=_2}s0WgmQMju!kBe!BFeht8XG42rjA
zw@bNQ3{<g=f0(Q1emV4KYHC>6OI4do8v`!PZuZwq*jBvc=Jq#sg~dkUh5ZsUJAbsg
zYO!{tKjD8DWq6~c&b?c!PDg{Y?*U`|H=Zw2{l;<`uXfp7n>KIu($!TJPrptGHm#Vp
zPBLiG(l9rj3Ho;n>i+hx=-8|KdwHmGhsSJ%GZWSvedRPq^>9V&;jG-iC8c$|R!&n2
zCp}B^>hM-+n()9daL3KJQH>RiCyZmZtF89gZN9YKdSUOSHB;v<p2etoZbfQz<|Pm7
zd)M9;6eb(<J=Izz;%t!|#qiSn`6-<r)3{2We80r!DiECDc-k$%R8J=8&*P|bo4IC5
zXj;u)zHHWXE1v@)>(Y6;HhalT{j3st!+eJF-u*RVkGzxBZ4-rEcJ|!o^yHg$h4JLv
z#swmm3@c(xcD6kD$RO#eJE7@7n-1s2xm&ZimTBrvJb1l7bjFvzpZ+abJ}qqRf`~8f
zvyQ!A|BaFHqC&8e(R@d3*)1P@LJB$*olR!3i~G-1Tkm8l`doa|j;0-TkrC@fwlWB<
z5^7?uxc6Y1rt;CC_k}A<?z*MUsVG$U3l4wM?d=n6e|q7%DaXEUHQW?<La{)1i*d}6
zk~XgLD~-No>SB|--4sp-UH!1qQ>mfn;6byJ3ti36CG9lzLi)4{i-p#l`p+P>slh3z
z{={_tRg*u4ESYUI&!BHn+j4{RS*j<q6c%P=-4zme`_$xx$<3JlJrUwEVmut;x5am8
zKJwY1cJ=TLp`)jR9p2QmYrKfjSQI9rF2FTcX-mQ6sKyNoZ?qXaT>i@Du=4p|S)2M+
z%~~aMi0ej%lSiNyPnAl>V`;IZMX#(E^}fv!4SvF`my{d0c<bTW8!rEBIoUe3b85FE
z7ni4JsLT>^5B0?87k=-B?pn?ay%g|tX0T0tY9HIe)uH(}+&P1Go!VfQanm`vI=1ad
zJkRlEky<mh{OAjs6|<H5&i=WMY}z}v2?b5tdH9jKQQni?8<hVxrY~P%$baRoW57+e
z*h^oyo^4yS&DqyqtI0&jZTSY3RkAJy{}}>hH%wYmt`+if>VnVefx$vM^tn8au9q}^
z$jRsSI4^5)U(Ca4j-Jy}Bd^-jf0bA{(RB09*867@S~jG!-CL^=Q?^lJ!x0&$R1MRb
ze~rCmm()I5PELDxs%2T(;)VlrcZOdx>zlUz<h^AzJ11#nJU6%&?A9l%vU>UKw}HE}
zo;^O-k-L9xZimI0kBLSx=l-hR>rzjx$Y0hn|Bk!uV<YwNNf|1em%6kma{HIL+J9R;
zEq0dRrC#ruu2*liq%JtG`D@mqb)uExzKcK2vP^I;NQvHL@o@2!=>H6oI}4v?%$h3j
zf^mK>huV}-j(-VvGPNhH6zUB~S*o(J>KUunW^R$$OC5H8<nYT|zT#!3LS{zd`l?$q
zQo6J%Q&X3IHHs7vSN3sWjeGJjC+^|LAk}qircAI8U=VzsA6~_E&_Maq<&Z5+K|+&c
z8Xqk%HQBYwZ(gD0#V@wHmkyh(eC_kosApT|(&hc?%NC19wp^^@y&uTwy|8KDPKguC
zjwr|6>lg1ha(3=yzV-zlS4@z-)ey3Og5T3c_Htp`B2(1abyj9FOr4tQV7ee{?ZGsU
zB)!$H?{z-TTbH5GHGf&+sa<Zb*?&$|+3Y&wnNZWoZ#%P?Zpt(XMw#^Va{ca|rS-#S
zsYTSuxCK+D?9`v4bp7*M`J89k8$F%6u9};Cj#;9zP@(;V(7Od+6;_5T7<Mi1x>B1w
z|4E!&>P)_hzmb<MeZvwaOPxL(^VOJ5HJi~Yb<RQt5rti`SEM%{eJ0%5$z^nH!c^V9
zlq(tx=dOE5hcK6Q`yKN?khuKOa{uXfeP;={MJ=3k)m28_pkCm^>O&q&l0{<OgF~n7
zbC36$`OZ6-|3-G^nQ5y&`F;`Qu{Cy8*i#)N6!I(fV95ia;-;EV7DMlT-LF>HtVG_4
zw44=wI`3T)U&{PTzq;zC_IFozEiqrTN99E7Yn#REYDCwXWKSy<mbtvNnA>ia&aG*!
zs~)Uy-mOwPG1ydP<HTHFX~A_fk_~UgByCtKaV69vW!-kKD?*hOo?PqZES^zzxa(iA
zVEbduA4@9ImIt-2-yP&PbE&3NlE-Q$ogJV1vags#?)U!^s<S26HJG<TM6@vY=hDDM
zY_XZ!Ho44dYTiG6>&bh!`P;q3R<3);mQX1r71?UuH@*3x*VOJKx?BZIDkgbH{D}54
z7RvdvXz~@2%}-5lwgpZPbC2eA33-0Lv%I2&X_|Uzn!R&!ongLOqKD7RlgVMLZ){X}
zed?_E9@C5?-fl`+5ldFhIk<j?-=3exc3$4;l(J9m*-qAwt&w)QzO#a_+f3A5;u*@#
zDfZ;vtuLSR3vWx#)D|{8F|WVXq2FKQP@$gk&pvH^kp{6=MbEV}rsYowP?@xBYuc%o
z8mev6jjD=O3N$YYE8Pp_vNr3u6P+z&UtTMxmX*qEu=)t2-16|RmrPqVrU_eSt=V&Z
zZ=HO)a&^9RT=;d#UU8Syn|<EylP-h>U9O$Ct8ZCv&6-CuBy}bmN?p-9uA{fgwTUzA
zOiMyUo~ZJ_8+^WwXS_o~O(h*o)%IIFzp?I&*UY|<HI9K^-Af%qomOyquy#)pnG}0p
zanXXEuBxVX@mDJUIZtzqp14(1ta!@bWtV1V>^aP{Xz_oB&&S>+s3*jF#4d?;7d~Ir
z>AiXqN0F@4mP^zBzGOO{#I?-pa8lT1rm4EQZqaFe@jq5D?9B`gnQ@}%&%OByuR2~8
zd7YYBbgSl+=F6!ktsYD=Ryy#Y_9=_E*yB>`#v()YF2$@#E4_kQ4VU#q-a2O&xcBIT
zshtN!9toWe^3eTvQpaEa(!!0M_hz5|T)DDp@==k-SDNugk`W@t(`RcH{aQS8(bj+G
z-pCm$-3^`6<amjzCuM4~u<oZ8_M$Vf%~_5ji=!?^AARoWwd6xnL`(WdHg4XKXa5-z
zBwf`WG_UNC=Ps6;wdTdSz5f|j6>XmRbkeh2ov9(Aa$(5=yK=>LA2o`2t??=^;Mb(@
zW<RE!v+*oWU9r@6waC^y%_k3M>HJP?J97H?!#0skO%hD9V&?fmehtkI&P^AKCRNp}
zUY=FdGugI(MOE0URRZZI$L8hi?3NWa$vJZQjDz-OBijS~I~Y1#t}Na=WB#WlQtS#l
z(~6gNu<frB@ea8YG1n@-+*H$~=Ah%M>1jcw8v=HUH2he3_3^?gla#sUVZ0_El!8N@
zj{Bv`JzN!i*JIa={Xd?#g)!$?w=7XgoZp$M!~A)Z;|}RrLPrjjSvS;QDZD@9XxEbI
zo`2O{_>vyRuVfMMNwt-Xo)B14mORhrYRO&qZ(F-&S#Z0lHZb{Y%5!AtRme?ZyzjZ!
zsGxghSKuZGK98O15BX#&f96Zq+02RB_~OvQsoI(<6W%P}aJzi1S;d{I;uTiWD;6z1
zReWcO$r+zX?ncK_dQb2~iHju{H2q#FCcbkYW5;2?!^`(f%V4&3wY)kzG2`$@8!>&p
zP%EdIN=K|s_c-hjwU*80-Q)k^oz3!vo98V*@wvh{XX)bp^Pu^Klvuu&oC6lW`hzAm
zf4KQs%_n5h{7W8pXDwP1Q!VB2Tlto$&m*G^o0mR`p7tbShh6e_Hk(H~uS}hKDedB-
zrK%RLy<Wdc8B9&LKRcfM@qPQ5nKLyeEsEj#E4A^8JCB!uihReuU7_3h3o4?cXI`$b
zzw-4zLzbt8c3+}b=9vS__Hr=KSiiDE(bM{d#oL-9yQ_~TtTWY>%$n1B>)`j*%vo`E
zrEllV<bJ#<)x0mhY<aHCB;yF%*#*hsFZ`x1c)g0}@ZqSBy1sRVPp+j3A2C()j+pYv
zWAlmqmBC(HjHdo)czETBysGYn!ip~^=Ks7Z<`k>7)I%#hv1aKN^A*>u(q$eCThv@B
zGzm1?teS8ysaQp%m$P^#!@q6JkMxc&w14$(;`w6tS$&H&&u1F#Ij=W~tK;A#tHr`~
z=RJ2+dot{hHrlvxCp**Qh{|9lRi=L?t0!CA-dZQME0^W1kwL^8EupEbr=y-~2<Gf;
z%?x}PDt9C#Q*4FtoXvJi*S|coYm&yMR}(Ugo=(}Bvnf<pN95Oye%E&wh1iZhR`0(2
z_z7pE`r1&(Qo|i~qGC6?=Jq-_U3Bey_<6(KdqzonGAyT@F>X2;HEV0y`KnE`K1yDi
zr+nq2+@i{&m8+IcYJIe(vullIHyhW-Gt1v?ZdMXvot@-*MJDPapV+Y|ubG|!N`0;x
zHEVcvizB8V+pVvcILWEWAwtu%q)A4glyl+ku5MvT{*IX`nrvZ3Ry#lYwB8B3=&u~v
zYrXN_9Wmz@*##$ybR3WMw<jHRI&$P`;Et>&lh<y7yWi|nt(^LEN{dg<D&_*a{|o|m
z#01^Wq%G^(#-uG0=*!R;#b%Throx!J?)ZM&p9hV6-%S#V?B2_|&Tsu+w`O*QwJd@G
zaSK;5)MvD>Uf!=~^2s?`_QL9;#Y;|@N%?+fkzG@E`@`|2qB@$sUjG@aOt*D&246M(
zy!_p*%l_}wGCo&ks4O{UbnEG9r751@X02D1ZMuKuf!u$FUG~xw3?-F!PFeZdBdU`(
zh+X~Z%1__d&usF!u<~>8jJA}Wa?am>xo@gT`0!rg!}2vnIn$RJ{W;gZRI8|G)uMG)
zt6$6r4{#3BQr3zQiJW?3b97;LOiz*BnU6DP{NXCR942mi?d`&8TfH^cz05MbWnrGM
zJGA?;gyrSJQ(SXqPImH0mpyY-S#wdu6aikw`xPJcL!MrKY8v`Qa%#+`uwy(f8$!5G
z+Wl(Wp>+G5?Z@i{U0PmY{S9xcF8?s=Usz$MDjYC#lC#S7+I_Q*t-bKY<mYwwlE>Er
zuAdF|_x~1jkfouuOzFBcx54eczaJZq6|y863ae_o`gVt}dQa+ZM{7ZKS(C&mLX%>5
zPLO{tHgDQl<5OKS+a}GkIkwF!vgDLf&q<k$+ZQx3+^yrg^TAEOVo4`gRL>!XpmM&+
z&Q4ihE`;CHw7AFT5Ov+4<(Wv8fNSaE!(LrE`%I@B^z|@%y_ITK(3v<XqUwN|^{xL5
zV&9L{Np>2gKMh#a8c`)>w>>ve_3W&_7lT{bpPrj<v4dYq%iSd{PU5;|&OE;b-S$gX
zujuxfa<%aC7QH^+143?xUjJJXEiK~_&pj>8>$tavu&Q^IW~yQq*V4|^$Q4KK9LtPv
zKVcJSljVQ>uB4ZTiDuAdl^t129ony|ycLb=`L=y^Raw@YdsdBee(Hzq%HcU#dq5`Z
z@}hrF9iL3@*4z;pv}|e6X~o0uI9K2PmK|jxR8g`=<fE47e+KENPx!B{W7mps6<;lp
zdZoZq)NAo15xvzK6TN;l%lX^e&r>PrY)J|_IB);$=!?g<#D6H6*>!5mrGKq2?p@sZ
zL^k~D<R_LsvEd;*|1)?@uedaA&C6L`iq%&F_C2!}Q=PbOeeuf&9X9^$51;vHz5C>&
zG?}%=|HB#nM_g%aEn@mI{~0<8Vw4QI|8!OFyrR;zH07o-+mg$gJVB8sSHIsYeZg{F
z?!~{=7cXD%pY%0XEo^V#<XJbAJ$-Toq!+U2<cqviV86Z5OvZYPOu~bTHER;|TD5mZ
z@EzFA|Mz6T>0swmmg*V5BbE7&8qDzuI`rXne4(+XibY`7#Vb=)mi(3am-Vuyd%1C!
z%(A+P^L+Y`a7?qRniX?vmD%$1>e-V%T(z6Je9kNGr`|7;mz`U3<NKMEotroV{xj&w
zG<h@MQ(*kJf31FzmYLGVTNc;FjwU|yaTk#MaBQImyUCVbk%pfa?C+W~y;NH4`QgR$
z%Rgp6toI6i`qnk1_({L%qDd;9kNY&0r&Xxv|7Yks$<w-i)8E&zj>qgf9|`r?sSE0M
z6<LHv9Z~t5<Q%lDKTy){iZ<Uo(Vn9V{do2>HL3_L2%5^Ev1*0cw~gyIPW#dLpW)h_
z%eM2H6;2lUOQ$U~4Vl@s>!AM?qdiA2rY7a6WIUNA9{h4D@0=UbVQ1g`=@V{L=AT(n
z5G1PoNQ39Jru?f24MSF~6-z&S{%!i7;q<1ZotvkulM^|rek7$~0=L12sGudY7hF}^
zI_ZP{(q%99ixt%^YF_^E<qM6f+4@;~CY;+EoLVwJi>Z{S)PC|ase;LDRu5}$&N5Tu
z`BSsUpDSeQ`aMn;HmC{Q+{3c)Fu!1G8jr-)RZ8NIj*I)*Z~nYt)y%oxlZ|x_7%j~@
z;l(SFc6!E?V6B4*r<e@<Kic|6UfO4Wa^}_GrxW&?2VQsgcbOzUt76fbYio@JQzC5+
zC*;+Av}rK8-zMqKd${vJXV$by{tG>`Wo4weqpZ{)?0H#n?vz@Ph+n3MRv-^!$Tmgw
zpgFq&0@ozokFK~qN2TCGimdANFRiMry5=`CC$5r;Wb5VD`o+xaxF)3b_P-bE-t7<O
zm@K{cD3Iq>naquC*UvaqOVp>=^FOg#_9$KQ)Xq|wHCj)kHf}w8J7RH(TOzMo{QPyn
zaVI0ErJlH1>YAg_th70M%I!<9ix?8??!UC?KE@c~^=yf*RsdJ-%-IE{{~o*Ek}+C#
zW}3SCVN-VtPcx;at?WU{ZfAErOsd&zzhFt<JegNhj>m6b6||9ob@%b#=~{=Yo2$E%
zHa9*{{a$RaY*Ex4>F$eP*`{qt*^!>!-_LEZbHZ}(jOWgK83Hy-cl3O?zAo#9P2dZi
z^Ty>%I<8mpS_eg@8!c9G51OmEmHWmAljdzV*t34b$yV^Vi83uz*<i{Wmb}<@<(m(F
zcgm7t&ark+)>^LJIP0eOs<t&p_nx>X`{SU~<R25SO7_}ZnR4cl<h8lW|1RD<iPQCa
zu!L^@SLUr7yWbqtc(7@i$VD!NOIONj-TK!}WZxissKR8X7t7}Mtbo8j_vX2{*XX?D
zIp^}4W!>3b_L?thuIsH>`a3M~O39Q-og%#qOZrrFBvxJ#Z}Pjo<E5}+aa!VKEuW=l
zqPW6195rvw*&r>Dl%7-Zry)g?GwP$!l*v<#rY$)3l~Xf`wR8WLjOQA!&+OMf`K#XY
zRbBJzKl?oOjH=drk~-{Yu<wHx$27HXsRcQ?$<-2<pVuY27H@d%)UoJNgUI2UElO*O
zS=CQWmu&8NSS)TV-a6HoZ=ui2X-iu9f2wZMp52#c^0__ghegJ&No$^jNIr`VS{Suh
zW!09ciXv=fOJ=&wn!*q<fmuUbd5+)1KczlAPgzyO&L&OzHe=S5>DyXnxd*@b&yctK
zj+UFIXMjD=k&7zRnJr8m)y_rC&J>*aE^W$%i$NEi{T`q8s$~*$pBVLIMfR;DuWVwB
z*8PrVo7!fqGF8+tsB_`Br%p1JN2dw1&h!d8dG8&&*{8)>oC~}3gMFhN!y~U$>N>dw
zi)vlla&dyUw@h;|1Mi!|&bjYyZ1Q@=)ZXgCq-Aq_cfVTqJ3+meE9#sbB4>hbGilgr
zf9j7ec^`i9{H!m_JFZBUiimu<bTz0{RwRGXeEqg<YEkW>r3o8CTrNePTd$RLcu&gb
zJASN+!UhVvLS*@aUfPM=J?S)C*LmqS!=n|`*`iX;KinMVHAN_O#kSO$UaV#cQx)12
z%u>StHXO5Bt@G+V%N~dGzx?f&o_5PPRHe0}H6*xm-n_+rE5+0mQ%myqzPvg0La#&b
ze6FJ>A6bN)^WyND>3C3_^^_;SmqA&<G4ng}(oK)=m`dDH>T{gxY3Y&kqDqv(%TeZV
z)6@{>S5sD<xorL_uH~$@meG{3{x^rxGG9oFR0cDBFY9yZ3tau4T`ajS+Cox8eilPh
zhIe?$Y@T!RNlHpQI};`Rlp54zqYo{}n`LljlB@2tMH8bJtzW1==jvCMW!ofP+_>^g
z&}ma_)B2^Jj1kkcBqruhQ@Q6fAx14$g!}0Q-V<^UxNCI^B`TL1U2Ns?YF53~GV9X9
z<_&TOOaeCf&7Zt-(qh|XvyIN}nwmJTYR>#O?}GYux-4$^n%vs!@T$y^L1)Dp-`+;U
z8ArVWVy^PNYY_h^oNe9`wd)3B@$yGD;h%Eb0yri;;OvtP7V$VWWvf(!#;i#WBAsVu
z>zyx-`8Ho+NzU>=&sHs2W7x6sQCmVzgBU}%q0Ry!kL>*BDW31Hnr7%+TbDm`*`G&z
z=F;tx+bue0Zc+)^)hTsrTDr-Kib*GuR-a5>pP9C1(HYI`%vFhQ0dliedhKieH(S<m
zx2$f5jhj=tf~Kigv)x0dKWA5DeOVG-XtSerp$4O8P{`zso-FouafMZ9KJDu|RqS=@
zTf5=<Q`*Z!KONZG%W7(B(y(d5MR~(x4+M6(PPs8bx9gm=xv){dl&Pr^i#MoE(ep@O
zZnh=(!N(mpHwEi#h++#U+NmXC{-Ik)_LZx);NJL0`5~!y4QE|^dU@X5)7ng(D?V*X
zXbf#%@hL94Eq(JwxeXp_H)bCE;nHlNl~B6usOyQ_6XYChd>_<KKkRdKVe$SGORP0R
zZQ6e=T_MB1B-P`@1fHF0Zp*!<>8vbOlN6kzx|98K@>k~j;;&qpdnM-Io}2ORmfKcO
z)!2t}O6)rngnw7sWJbDGU-=t-xpJMm<J@%qZya?Od09k*))-x$u=I-VUsZ<S?e~HX
z9auHTZN`#^SEf$Oy<zd}&F|flcl69NZ-4N?VoQ=(%YTLm8v_{+ZCST2rbs8}kyi6j
zyQO{emusd(JY%Sz7JkvDXUf-!%rDcJODA7i;;~I}&EXqo|Lj?+nL5Wp=}4=r)0Ug7
z{Q`@srCZKgCTxH0xlmK>anh_msxKaV+Mc%4#nyS+Q`_h%r3?|LYo~6Ox*BL}a*D%c
zbpq>wTV~U4P2IE2JZQP(LyKjZla?^~I*73CVyM&eIeu<qaNBVildBJwd#{k(q<TW(
z$`tm!C%iXGHY9QMUjC!<Y_fSs=+}A4wrTOxe&qYl{F;_@L2rv*W=hkn$qshww}$x(
z9b5EWDYwJoWZ9#n>xwdV@yEM@f}c!Suyyi^6D{)#IQwdU9Bj%lf10YI8RKwz()5LL
z220m|dG~b5RH37ru1$Q*SuE8zZ|zI3Su3B<nkChtwZ&a&byj0!S>mRxTXf&vDAMg~
z)bk9P+R}FMcG0_ApI#g<ncm{37V~5QgO^u!S|Ho8U-K)rKf4(h>O6OQjI++p<zCl9
zq&l7!KG{>syL0&j4Xw2jxobDFJ#I1(Nssum>9kx}Vv@|Pxh??@4JW51?tl48GF`#g
zx@SR+!|c}V6&o)cJ3IetD_iXJXFJcj`Oo_Daz%#hqba&>ZJ*pGG_0Dij>|wLPclMU
zCe0wmz~_j!(Da$grv7cm);V7ZzqnH7bMptz?%Dc$GaD<NS~<Octomx|S?I;|puOMj
z$<>{GnX^<(`J*QumNl$Q?Q-Z-h!nHF-<Yg=;)&kcb7m5YbyQ3pgV{yOs&ZeiDYdET
zw5ZgZ7O9oMQ8=?}&zFT8bseQ*eM?nSSz`5yUnS4-y8rCw;*eF0GEHYbwyPby{w*&e
z=Wv3jn9+tEnT!AIbPPJld9mGbRc5hdW<{y@tU{+qPX9)pUpF?0net?De4F{<@ao>Q
z;A?laLq%1?eL|kxToWR&gvaa3NtGb0>Cu&5Q3j_tHr%=OZL7ZUVb0*CQ!XscRNG}7
zTzp1;Nod*Oh9~pek9({5`J8lh{m<a#)Swe@%yevh!pDhX?P?;~s=mBYzJj0TCKf2p
zjPU3#+o+}UisSaTcHtjwGCCS7%<d^<<h*rXZp8bzV(J{fGEbf?nGk{FWu~nCzdrZ=
z>R-3Hrv0$ZqUDQ?ZO(i-t2tfse9@~NLM}~_vdaRG%@taGA}BOeWs&l)#jLL@^4P*_
zLTyyN)eV%s$va6fd5AMuyPMCy`O;#Y&gMRc_adjKE$}E{3qCVtL&+ppPhU}=>j(c$
zI$bsUux8QLv*-M;M9wV@4!)cw*yuf@!z;reWaS>yjI&xVC$LOa)n4hH<Z|<~aNlf8
z22IWE8pnfbUxIQXqD*6VJ-TeLC(_u<*n7&9oxYc6-Z%0;(!`)QUx+KVLy&*o!O+V|
zeSP~To1EI!*H>BUXrq{=afrF}evzKo;nI7L^YRNWUp5O%(GXeLyfgRhJO2fpbF7LA
zRlTMySn0*I;KJVcXJsa968E~!yq<p7QI|#NsHzT6;*wT<nY8fi=^UORSI&4c9nL+b
z`ccL!CGn7mGlS=yTTSzvtmN4bo<8)@<!X)Bs@~I#={pl0*FMfUT~()*X|zl0W>Hnm
zv{@1_O@gmUW@#Ork<BZS+kK~jAx^(0RrTR>NtaWeXBPeJW_YI=ym6uF#zYC8a*?1z
z&%C&dMZ8u`5SpE_$RxeU?<|94o0IYNHK#t?EnZZtFl)-bfXx34SpjW=zrL$IdA~zk
zZgqa9E|-_EYNZU<Q<WR9AAViBv%{!h$NbYQeqJ6){6CEY@7P+J+8*)vY97T|#cy=-
zQ_nF^W3E^8H$4+Ls(nOzUE`kb%T*Q|TngE+k!Mxjt?e5QEG<g&iCQD<Eg*Z?lJBZo
zf31=_`<tIKzQW5@Cp5fJVV<_1BlLA)!8wf!&t~aOT=U^SgZ@eFj0`Jp!!C(GYwnnA
z3Npx@{?;MxXtzMBd~x=j91ROMS=Owm&NJI}JZ`P^V%)lOMvTjyq>4!!IRa-jc`s~;
zwZC*l{#BjUW~rmU7kplGe6f}K3ahfk>YhneJF<EdeJ<_WASq_<|7U|mzwf0dZ7k*c
zZQl0tb3YPVzA{owXu%ieB!7c17x*nsmQ8Lu`AEhuOv5L6%F;K?VwR1YCV6knHe@rL
zb>`ou?iIVb+NK+<TF@prPb;G4`DG?fdCwr<E36EfPa+C)N@A3no_a?!afl{7nZRv7
zH~yCSjXZh1G`^6dDw+pWHmqgd_bz|onyUO2kIxz3xqet)?9-ilKg9!r+`Z@diagRZ
zdh^J{;0iB8Uyge0ch<MZj3aW2+Jy4hQnt_3bYVF6qTEsa1jE`HmVJ>;F`Yt%HF8<@
zClw=?Hpys3T271$b==#0&*q`;)w}85?w(igM$RZrtDkf<P*!gFr&(&NxQqQFZY*w;
z%+@#C_iJN)x8{+R5hhO@8FnjnrzNJ9tFEs1c=`E<9jkX3GusrUq{guIQ~oW`dU)D$
z^R|ilS1R*@XWRRI_4C#ZdUEw@mzJ2aw8nhbDHf_;Vs5vmCapbqG>>0kXHdAV`>_dH
zj&`!#T|!a|-dv15_))mvQ5wI<v6L+r??0@XaE@DEiBZZmZd%Q*<x7)ZN@X4Won`Cu
z#pj!6%|S(e@f89Gr|8#qeZSgWBgM1FYa>frP;1#zZZ@l}x=+PlPdBtUe{MyEq@r)y
zY1g9UQ^%*IeBD`o^Yx}Ko4smH{vn&3RT}G#{o>@1d2=9U_6I%o<jRVf&v$Cis&Q%b
zEh$^;cg!rJ@?p?2pBXU`0(x>wx_futyYOD~%!*87o}F2WQ9qn(XHLoptH=_&HN$8l
z%Z>M?=CNx#X6)pOS<07O;5>^fI(!MUc<Zd4UK*CZFFceT1NbLPA9<HP@u^6y{fw{s
zU7j(nNb;0gu*Fx`Ey}6(!kiZ>LbBU8Zd9*vJG|1Zs^G_hC&uh4v+m6OS+i&}=TY@D
zM;5m~zI00G8fR$v-HrmA1A=etKhCv@wSM&7wZduRGh=<<ABh*g)J4u*vsLivOYNy)
zdmBZI48^41CZ4nS&+yP>W%rva>oiS$f3CdzEO6#@L*KOJEe6|mx%2+vpI~UCE`FSm
z`Lo2GzzsFa(j1-?J1}j0cxO}9oqG-8;i)EHgE{t{mkKfb&+u3*sQSL=2KCv|+pHPi
z1zKtpPl>r4b-C(qzP!uhR5hP-@4RkUU);L?Gt-N_)n|;}OkA<@)>I|Wdj}V>bl-Sa
zFBA4LN2&Vr!(V^9UHZ%qF7=%wwm69G;v1dgi$6!b={A3*&;R-}-|`nTG%qdMC9|n4
z=I}BP^)sSZR=Ga-w$l6fvHiMza{}r=8!gdEG~EAi#!`Vv&Wm!{Vpi=4Fg=yMxxJ5H
zeT$J`=k(@BnnD{@Cd_$IUcCRxsYrRTPp9=07DgO0T60a!@>txnm`h8iROZU8SSvM&
z%h+nRIRkTYVe#kgIHg6KI#=-=TVQQbkhxUr+=Cm(U5+P;u<Y#ousob?jn=_A`KM1!
zc>m|#{ME*G8&6p^&DUADZWik*ug;`QIj<d;UJ2Z-*>85OI_^ntn}AZzN~1!Hs!bgU
z(T5g?=YPF4k+VW_j-8uanXH!vyCy?}k;X=~)=zcQvuu-$?3SJl+tsdqCUxl~FWZof
zkq!atN=38tuB%Uen|D}BMoIO&%gF|Lf#RLYfl<NH#xv#{w|E&%wQ~M3)BQ})>|nPq
zYrGEY@CHZEdKP@?Kg0Bc0+U>(RL)B_{G!x$W0Hrt+}#bFny01(lv=ax?s~cOens1c
z4T4G$mYoS#(suvK=h^gc*%{+I*LJOa`J`=P!bHI^r_)b2$1GZ)T6THo);F~q^#!MD
zct)B_F{bCu&rSGoPh6i@Q<F<_PIa&6-m)t97x6#5K1%Jj`Z#^r%c}jxYhE#4E!q4f
zB;ZQqtiE|xj-rab+j$}%X%}ucKYrf6Z;xbgMVzvv2BT5n`;$@=_Dh7$T(SIq_VJVT
z{daq(w!QiC^ux=f_$OE9I4aGYva&Up@zbKlwV858ukQ4%Uskn$?XS?HpIJq(YL16^
z<}S;WdStDy()g9pI(gH#;;zMljfWqd%xvLmzYu$Nd)fMPc0F(YGYC%YefT3MFD>d~
z@V0<+PgijNoV=p<_o2ASCMI)zugJPf1zvZYoK|X?SJnGWEk4#(`QpEO2HrO1tGfd>
zmTGNc$Z<SVwyeHlNn*;rR9!|7J(eTSxJ?t7_OLX&OwaW$_Pbqp&?u$HjAxh6$;_D5
zqKe$YvaB=Y8M7MVwQn4Az9!N?rIbDJV#(cv1*=YLtW;f|mC0f7vXtHXO)c9sdzSm+
zf7CmDHMvha8!Tz#aV%nFYd>{y$Ais+org}x8S1I6OlakjlGXlL`}mHn;I&Jo^S!S8
zxOmy%%U|=(662t!vYFF$G&Wpg*7`f!+<`+wB{%SdyKa)>HUpXLmgQ}af)9uBKY9Kv
zWNFO8wAE8GjxauQ*c>#ie^#~g&5MT%_y5`2r)}Z7H2bjbU58Z5C*q=-Jc0ii#P8Yk
z|I;_}$h1E)y>qR<_jLDY`@oa&GRxBRlrk1+u&vJQ5qOY$Zv8U-{XB;E|4!!;TdJ{C
zGIaZqjKfoJd%KtXVLHCHw(p=?qNU4(*$gtGX<rrRez_cR(paZ!(RRDDR<m;6Yt(N(
z={Yk@h2en0@61X6LQ-aPJlVIn_+_azZ@a^Uot=kyEFDi3F9<!{AbD<QL*k#~P6rD`
z7W4ENrLMdpGG({;yV;MF|4z<|sb0M#!~U$suV(c#kLNrKd2``RK+Gm9k-h!@PUs)s
zvE5?b<Kok&=c}FOx##-AujhlEq0PyedL{E4L-}{;f6@Kgx#rh)!__aVuAO}%S(Z8N
zH1lkM+%=U+4|=EkOl)B2)zR(vA$ekQ_gUo!GkCe#lb&#Gd%u2<<HH+~L4_}cY&_<O
ze)<?(95w0a!8S?$u2M_O1t*Oo#7$p@Pr1S|cdahV^y{lQIaED0wu>n?Y*#VU`_a!m
zwa3w8(SZ*uOe;0&7RF}jaWsGQJJ#b6r{s9ja?{0|PWKe_&P6Tgy)|)d#lab=hZN_B
z1*CpHAL;J@&8n^YI?J@6R@-jY&BYrwib*`Z+c(Kx+nn#8ipE~w<{h7^>rJ1_w?|~I
zDVQ|H!D;FYTaJy@yQKr-ruS5vR;~=2ZrIkXw`z%)(aV5T7ir5+?_3&=<v56ou~*Av
z|IpJmvYy!|>05O0t(V~1xbGP+VpLkwCM^|Mm^<HJtY@{Y*3o&Uek)#acxL&3me~{J
z?(Nqm)p2S^>*6mWli!_M64|!eVB<!m$6^sCx?Pth&k+vXeWSbSo6@^0J=@tIW_#-_
zndGU*-7a4E%lqye*=&(Nr)^)!H0wkyTf1_;=HjVwS(=$!4~i5qt@`k*S(~w-<VCJ`
z!I5~emXk9LIM+PA^!^dsbcIlx(~UXn#dpNXwj4R9;l&VjRD@e&%5=q$B)M}U5;uaB
zKJl$S`T5<7mp`(eRdHoyJz3Ed)mX7v;zCoh-Zf2wT^pkr9&}f=o_Lnvv&ZVls@0mT
zx*^T((r=@>kGwNJsQ$1Zc-q7nD^&#ansOqnm)wxQ6#G(Qd8o6i=d9AXtWukf$W>_s
zc-9uIy{V&T#;{)Q-h&5wEWN`7B@b`lKXLKblKdY!SI<mtpKbMKs#I1+UG??5i$h=J
zEnauZSZkMx$dq?e@>$(y3rw<_@$a?UtJ@|y46M~Rq6>6=%~@Jivl({pSnFEC94ImG
zLctc_X)_y7uMiAVxO-5P?M>aePj_7}+tyzTD*4Y4<2vc=mB^S+&m<M5s`M_Kv}m4%
zihzgy+9#iu^iJ5m@?zA1sfn!*|D^M(W=}M|u#9_S&XS39#1bEgCLE3s=X$iqoLw)S
z|2@lA_1&?t4UqwbZp;P>TZKGC3PmN1b*?YVK7aFN_NMLACT6}~u`H}-Rm|=q2O3^T
zI|Lm%?wMbc!KdbPQe~;5N0q>_Z9HEJo;GfYJb%#hsMp5{o2FWcNL)McChCFsYUj9R
zU5nRlo^xIMMVHu_U3FH8^Fl5MEi`fzo02T#!naaPTgZv|rPGs0Pk{h?wMMO?8T(?6
z-kh^BFpDWKGw8O<KE5Rh1}|p!S*A_CyJXdooqWEHVSTP19(qafOLi^XCADh)%1__t
z`iM`mTG6y|(wS17hS^rT=IU6lmM$=O6DJgNvunC#^UH<2pVAhGh;_{~Js5aoQbBR{
ztd~|fZVQ%NRd>=BS<}~-<<qZyM!nzkYS6wC(~wOat|E&(8M$~*8|`$;%@#cOSm?u#
zezAaMK~WE;e>O?Kwd+`a=NX?fP8X(qxX{W}%)~yKN$%aFUT5j@PSN1>ooW7$dahh@
z48J0<<m;}^K;4=IcSA0IAw#3A4R<W$4(YK4Ot+0N3FOpcy&=t=@kK1-`QDj_du*P5
zD5x>w<h(ke#$(+}5#^lq5$z|Rw0_E2wp8`ZYb~Q+CqEri)C-cdf2YVe>*3|9qvG!t
zZ_^g(|F_hGi`n3ehDX7J{WFrE3L8!eD-;$u$vI=n`m@dlR&mQWT&{f(n>_7l@zeJ+
zeNA&#uFVRPY<qh2!q0}7tk^K~=cgEcKXPLhHVkHE&4_B;bL+UjwM!pk*Swi~PcTSZ
z-1=g!@N)Ap!)c*cw9<{Q{=CGPdGkt!V#v+4KO%1TpBF!^Y~aOkd)=y!7YZ#(ez0u2
z!F}ER^-H1oU&Aslm&)qT`|`D9!?nw2v=wJKf4{54wCmncNzbso$5-6Wma+5MqHZzc
zh*pd3@f*L*J~G+}`Dq$2NxJkhl%tk!!3#$|-E+EeYEL6$&5|$84WE!&99Z%!-!JQ-
z_b<(-(VYoA`MErPNB-XC-5A|ChpkdOc%^z&h5w=WQ}d>G<()`Q_nmCb*%f_uO3~aC
zf3=>^sCiU<nCn;1V!Jb%kCt}ZysSC<blI9k;e1;!c>cCETy2r8dSlZByHAbP5gj$D
z8b-#t2VG-(IrRRlxxTgeh~Cu)N8T85TB>^|O|4%w>FHTsjZ?K+*L$BI_jwf0SLLkx
zLHW>%X^S;aW+%Sdx9$~J(rWeabRqq}JD6@A73DB;<=UulDt*?ywHxP7pV0T?W1OOo
z%ZGr*u<4p+UjDx|dM<N{E{bjFQBU}=>Blnje_N0C_hp5u`D}XPHO<cG+k15b?Q8{A
zemTc;tCC)^EneHi!E18a+bDv2Wr{+R|J}O|+p}|znFl@5RCK!aIqOF4d)el84~vvl
zQT-hZwjJM>@5$P$9eJ_j(e}l={xht&z3fuNpIud5TP9`29w@D9?GST_xEo!|>XB=H
zwLeMaQQA_iLyS=yEj@J?hHsEM7T*1E<ElKx!%NG1gP2)!oQmV7OYN1rSo%@=u-@_y
z{}~kGR-IjX_Sbr?GwJR=7nx2^JF24kxJ#<4<w}o`SWou^w+GxB7H13I8S9JpNn|eY
z(v)YN;>xLFuv5yFJ+0cg{gK4{Qzt#9q&6AdO5{$}SgATi)}hP7MCD4y=Q-*IOWL%4
z71W)5HfPf!l{+Uz+Jkl9L>OGIp1Ae6a!zjAxoHb7GjDaROTJ{~bYy1B>Db#A)*=^V
zcU_W9mt6ex!nM?rW9wF{Wi46Vvgx>l*NeSU>zB@0;67D{t0^T+`2w4ox5GlM(kWN2
z-q_!AJvPts$QwRh;}EsAEyon@{16S>!RNtt<U;$(-r0xOEonFXD6RC0z4eLL^WUtw
zwuXj#Hl3F2KN}X1zvj&JTff#8II@e%WJOGE=ll`b*S2WQu_>F6+rG``h*>9Y%%j&9
z<hg=TMO#~Vf#>2|$Bgqft?tsCr#?k;h3Zt5w5RjFiObGtRiEeiKDkrd>5`{duvDe}
z{kKx<&Obc0^cmA+iMCt+4*r|vdT!pzO?`7;Yh`TqQd_-#sVT2VbNpnkiHXY0NfD-B
z-i13^D0C;pXkJtGH9F13yi#e~&BK=0eUql|65ePO5)c)cbF8m4p+x%DQWf!z`#f<u
zx0h-BVp_K0lH&}k(7<Ur%99rC+{ozqWmmiNtv7nM7FEy6wZe=7O8L2F_1K&}J)u@9
zKD#L;>FSZCOV4aL#q=|Ju1rAx)Q9Uk=Bb80+nExg{p=dgu4EZeQL#xGi5D(vnY6t*
zb)ZdfQc2a`14pNEv&>Yt&<i<!MO<x4Vp5)u^k&(~J~Qt=TsT9f(ZRb_OspU;YJ1P-
z!z(n;x>e5!^PF+&%9NIiGrk2zD{@Pwy<_^>o4Y;moWe(em6uabc`{ykRW+&cTbaz`
z_*+YTgr1(9a@c65|G%bH`9=qS9{YN6bKj1aR{Q2onh;jX<WMZLsVGn~#vpRB-ZeIJ
z)fDFMDXQC8*WHO(a%$#1c1{zWe_z(Fe|pODzU-6&i-(=8ONFN<ZA<rL$kLjZnYf^>
za=v#(<K^OGm%q#pOr3Js<!rFZmUAE6JXM;rTcoqtCvM%bPib{$g1Sd#m*if@rL4F9
z?F-cAK59{>AvR~K$OZ<1{|x)*P3#K0x->lC_|&JnmMrcJe8s#$^={rL;f)M2%=;7c
zHyml*|MXt^k>v{AVU7J$@(XSh&C#4|)unfK<APpW<2Or-erjs3X!E~fuwa>FkM4hl
zToqTQpOf8YcLqJ^6fm;-`*jCj!b8<%5$uMFou^ja*~y<6`_EtEYQfLDFM>qp1?o@v
z<M-~QT2Is|h2{^FG<0GauUQ`R4Gs?$velV5b&`5(N1y`p=@jV|QfuyYSKpqKUY&k;
z!;Lvs0{ewx+MYO8#XlE~UHaxvkCV{k@P*YIe>APh^%s6CB>zZUt1n;j^Q&7C!iwGY
z9=rOcPI5J<6y3NsK;_k!?sE?#WDhdD({KJK|9HwientJc8UGn%^Fo>epDws#%l1M3
zuuh^3SIo|K%fpj3?%s>kxV27*llM%VZk$wb>d*P@a|<6HjjOk1bw2Jn`)PjI8l|fe
zYHH2}JWakbaj&&kZWKFWr+fe5#!Vj`0uz=x1btDtc!257<rPbu-da6V?s)j4XQ!is
zsG_6JKQ-2$snxH9T-?8w%EgCY+T=b**T-a2X<oj`q@YcUy<Bx=Rtk4VyfnJ?ggMvc
z-VJrGxuzB~Cp}#`^{$sFpTtxq->5ZT^b##(Zf@GRc*jSP6H_Gp96p}$R6fJaTqfA{
zB`)7HI8@EwRQRc-_F1`AmFksE=F`=u<!5*AG~4^8!fe8$_x)ZjTQrS~MFna?Sgu$X
zDZfyzKK`Lt_4J=pewhto3XDb5f0`TFe{7q&WpeP>xa66s6OZ~kObt5A5w@Z83FFb(
z`zAe+`z~)`GV@_Y$x>%S%lLV7^H~ok2^DBewAgip!DJ=p@(UJ)8Ozf}BU@c7&&-mF
zS{1P{e1=xH%aIN2fp?=X<tHbxv)4qNoy4-`Bx9$|m-OFTGAv3uJ*VE;$ucYcym$Yy
zkCL)(fwN|1)f}5RYnq4^uUV3!;E8XnTQU;sp4YxTzgbi_=+a9Sjp-(ASNWda_Ly!I
z^D=U}#*eAH<Qh-gCBF2{m1=hhdRAiVKP9zNugK4LmN#2qz{E@Q^(vlE74|vU!E@wB
z|K2n%*Nsyje->0{Hy2uP+9_<|$`4EC%I<mjR?_am9Gg2z&mIN-^Yq)`)!fwJy{zx{
ztiXG=#s3*zG0soD693rySl{APMsF4^ah>Hp#n)u*xAx^CS2hM`^nXugR8scc$}Ld#
z-1bNM!O7Yi=a{p1da4=;MSfa(Qj2S8?CI-a%Qhai|B>==vRWBCbK9IvLJL+l@N++(
zv9hl;)I&d6QZ0G<mwUbs#r?iM%bfJ|=CbN5vi^}%pB}B*x_tSP#mhUF8aYjy;QjpE
z3f{L%?2oB_SQB{kqvws3U8x=}n-)GQ+PP#yd6E%()s<`J5jJUh<+3vmr|K*fQdyRC
z_tcb?-9bT&dUJLPc`n`^wed+*%zuWZOR|oycq#Eh=8A_)Kw7bq-rq(Zx9{%?w?3K_
zd|xbRUwp})vL1h)qdbfEYb{vm6=yPM{<>eQ7Z}gkyrRnA?98rc^Tku+G+(A>JaK){
zbUMkA?F_$lb75Q3jxu>+-H4nhVY8s4L0OZx&&{&@EI8|t?5snDI#wTc+)T;})5$XD
z-B{J6rN^Nw*ZFeK^@H)^pI>g4IktTH9>I@}6O&i0XfqV||0>dx|2slJc)Fj%e}<Dn
zGq|Ip+>3r3D$G0g(ND1DKf?>IjS=B?tYJCZ{xkf_vi}~uN%S$}rxlysl^#drT`bU1
zm^5Ke8SiP!8kYr@^MAjP*62J?qZMtE7N=LjmVMDS{fbECyujHhmXlo8KM8SNI>~G^
zTM#GT64S#gwGK3K%A9>uF7Ic}GIP?kso}M*9187cPqm1%nayo|Yco&v@RKvs4UGlj
zk8n4++8kUgGOOu*curea#I3AMZS_^l*G9%n2(X`-yt=3&`^KGrbH7)9(zeo_wjkq6
z?>S{YCk_u*^<0z1^@58-zVs!}e0sHXOGiqU#I;DxKj&|GOE22ABlaZs`r-rc?@N9u
zOjKz!JQU)=?%<dtcj(&Z-EP6W$CjL$r>g&TlDvbXLi>iY1zqPHXXxl`d^%-g0H@&7
zvfQXIjZ5CY^Hi8!pqtozCdpOY{K$m+*8DsEK6>7n@R@yHv4F$Hb^jR@1Lc<eXW$Kf
zW~ttLPr5W#R5UGr(o@O1vc4`8Z<#C)o2D$X=6K3(tH}IUT8HjDp8Zc{^}dJ3OLSVa
zQnspCs2ylBDLD6Mb;QH@y_(ZI-&`|iHgptL_ljV+GL4JNJKMc@@$TKOkN=dHRF|0l
znD}bnw9|6yWEy9rnQ(l&(rFygKbtM^Kf@z7WBCS~l*c<*)&%~z_`o_i&S~m%5&J)e
zA3q=YE!%$VEW?>qp_`W-UmM5%P(9#jt^6g~q@trsR-R=`o%+<2Q?FBLb+~Xyh_-fq
zfu>UQO18JzH*QPsQ|ydWm)7z34B4udD7|E{hF2qhP{^+)lg_=HA70*g$H~3wp{1IS
zCm)x`iI#|~pI2;L%-)q+%CYeE)YGe_I%k`oisMk?T)5|0UvYZun~+|GJNNU-kMKzu
zC!}BaA@VLeX?mXfiJ*^_jYq!E;QJQ6Wo2L7mJ(~(-Y*Fs^gsN*W>j*&GgdQl^1k#~
z|9LJy{mMNpc&>_sYA$j~viop#>Ei6zl->%3p4hVOtzJh9PX4G|#iGKxpq<xHMPS+L
zpceL$_Y3|rh*{hJ+{fSH^FSy??o(9r#iEl6y>kK+E-abd);<5n#<n$I_ln$Hvf)bF
z#gn4XTrXw4yfw`-D9Lj-n^^YTzwzEl)(m@Qggbe#NZB%SD=fSHc1!wy2Ie%c{|rl>
zPLPUfO!&{hwRy@)n-0a<Qz}m{({L-ETpCw(dCk|&ewkS-&Q5T5OXBEXQgknE;gZ_Q
z<Gzm)Jvb)CC|nV6KgMagVzGPlqZ#K<KYEfla}slF?6r69ic`CPx%K~(erdGkw$+*V
z6;X>g9=a?!oAOy?+Gkb4<GQ^aj4k2HO%jv188UY#?c8keBBbfbBv+jUPrCASr1njm
z`X_R}Id7Z8VFpiC58e%>{AOpLNvC(a_sx}zy)^S_)lq-V#b-Yq^xLd6>p|46)T=s$
z{mJXpnG4IRrFl|*+aGb$iZ~Z@I(Retsa4GHI81I$^!!nNeCgv2lb-4PShDu^*ObVV
z$xj3P72OS=YQ>z^jL2GWw&=xX&J-a>$#jK_CreW&M#y_Ja6ITX=FZ<9>c^X)Iwfeq
z(U)<d3t8eXY$^MB!}Dk3gk`6W^nVj&F%GXXEa0h1Ha%~&=KR7}FWX9Hsu&*ASkSag
zY_;ishAqw_JijY*t2dp$?Z8sidEE5iL$$@?rizMrUMrj;dyhRYc^$l^Fi-Mudg705
zdGD=%bb4}fE<CgBud`EMO1HtWVz0=QQ_H+6@7b$G+N^ZoQn{2<&hlYR`471t+a6jR
zRu;2=!uG4T-&VQ4X<kc`TjEECM{oW!oai{W<+Yb~$=Og_oydI&KXayB-Lk3af`^i-
z;EhSTNk>~d)RXHB*ZWMGxjEP{PRLO-mC3FBV*lDj*Bic7B?t#?o~W3xq&aW~llyA(
z&w@LbmCkw+s~YXMbBWL;Z>Mgbgx)=~j+SkkA9~8j(mb|n>g_aUwmGSO@x44t?w+0<
z(6+<+^)sa~{&tU}$&-zhHMR8KyiBYzQ1rZZMCyv0mtTsern1I{qX8>t=xy0J{d8G>
zlzgq~`|OwnGLKGL>=(@nuJhA=nLpd@p^`^b_fG$YZGDW%ug*UESoTs;KhS;liJ+`m
zFBb1Dot|;Z_Q<hA-xAlv@OIpv*sxMFX_14cp8Q;nzxxYCWKxxR?>0Di{;1sX!`-0v
zOZW7fT3K4!FJA<0HkffhN71e5^cq>`$;oHW@6%qkR%pu7b728*Uthe?RD0iI<(igt
z$tw1W%eMV=d;YL3eqMS(`aiLBlhB{xanE)dKeM&<cy;mozx-)OZDYKJmfc#FS$EfW
z>0X~W<?I#5!h0S+eVFF`Xh+efCo88r{%6QbR`nAN5?a$$xH0><_1iQfH_@UWN2anS
zrk>%LwR-ukc}q?QPY#=EygRy!UtpubMXN`PmM<(i^ZclVRsE_>oS`pYEc>0}J?Dt&
zmYfT#el3v^3*WF|hxPmAM;;h_-sryDYsr~~+RT5_a}Lb)Fl9V^Z=z-ltLE|5{D(wV
z1~ZHDEIDOz##rpbC#CgnU1#l`FO)oU^}HIXA9Ox`;;hH9)1I4@yxD4ES{Kz4$z_nS
zRDti&F$LFkj-D1XKW)^S-r?bO>q*Zl-);*<&nw(lnsl!H=(}*{=p*sgx$0V<o`!4{
zoa%WtNXDq$?5|Hr*Zk<`mGMDOgD*ebnPYM{=<8{@)L>Wtr<b37+Q{IwG47+^S=A*W
z@p{f_0m}n)Wo$hDGbnkhG%#_`nccGD-`vJhwR5g(ciSIUPfTw;8yGTSx<h2CLj6j$
zE)V7dp66yYJ?yA{*fQm~{`sPw{|uTZKYOW8^O~ix`gw+v#FZr*+k2aj-CkVX7PxWS
zqem>|lder#z|9nN&7O00Q{0o6g4Ri5>fZBPqb4jeTCs$=YRRT%K_{E7{Z3!b#^3s-
zHGR28Rh-qDNm5no7M;l4Z2Yb3){jM*jC~VUt+*(edBfx5-7h%}vTpW*A@jc5Tz#f#
z8L~i9cX8J<eOG~^_I-EG_jtbB(RuXjoXaA?T8ZwhoLc!pcP70`xuAXMYuLo+|CYYC
zIlS`oyS0nYBrjb)^~JIYj;?-+U2csnO)o4Mlc$#aduc8)sl{ueuCOYb$V$~4m;N){
zDdiJ9pqf~4NzJ>6si9O{+2B<6N|mNx-AD2QXE?j+viZ)jHCTGa`=IvHufgZF{+*d?
z{W8^h-Z{2EkLraEugnZO8ZztdQSR+`cvqfMNiJA5L)^~!*|a0PT#l=Y*$#)yXlT~X
z@?5R)WbT7?oq_5svRS8Gr`%6m>9(L;`hE6WrI0&MRm97`U4LF()Scs}y~itrz2$aR
zf0b0eh*bS{#%Z-X17H3*wnxz7GbhWo3415R#O@GRu8BzZ3W^Z;Qr)jsH7{`StW=pp
z;YYn@=e%EOmHkxw%%s^SuQXnKvUnwxk*T>V>&44kWmyrOJ^#)Ie2VN$db?58{n~!P
z4v$owfLU#<E_TbGPBAuA+c@WuaLU8tB#)e@imo%=w%0O<F<tHUciyY-b#TUy#ilA-
zO(knNb2eCa+z8JUPh)zrrEtq8?l%#Ne=iw^TLmrmk#L#jxoV};>}`!x@?VSoDbW^L
z@?pa=?qKbBw&}||8u%7iT+_cNo|b#{uI<c{yD`uFmM&bj=5&(rzYE^ZidQ96`fu43
zEnwckm-~H<@!XXTQ+8Gy_~|ssU#-CNiTm!b=|{_6a}^jreRN4G=t}Ff#X1I|YeXi!
zU0u{~pK3Z&E_T|pkkE9)c7wL%UO#<hwwzwMSM}JDwTG*=-&~*DXVx{@^w4Lvv?Tql
zuKlO1)|vS!m0Zz_7MZxZH6U~GwmCCji|?C!VpsL5s^m4omZ#m~<J|+a)X(^|KJ8WN
z580i=VSXh<Fx%XkC#cZ!@_vh^nJv$Fju;eWo^4pR`zXJ}@uY$oom`XFoQO!6mJoAk
zYueWCOIN$MxBWBv`n+{#dsNC#yMmql-l3PB&5|cwSGx9e=7Gyk#3oIP_t25d)#`P6
zAe6_(dE`gToCKFOMaEg&Mw9Yh24(TD7AwwJ`qn<lAjU7G^Rw`@B2Uh!8ezd-4QI4m
z@2Xxd%kI7-Yk46~JN4$$^v235cX=Nr2%HVQ_*Il!rm{OzYPQyzTGbBL>2Eg4IbL#G
zogj2fnc=cw{Id!2N6u}@;JTb-9>BjRJ^V=F9nFmC$0w*goa<<Opk{aZ3cmY4m-U=a
z?@?D?ddApp?edfV8BVQKJu9?HtiqV9jAN4f(ip#OD))?~PA2g4X0OvL+;LKw|DLFY
z;$g9J_4rHw8FE)j&r`RHStGRknWCg`kn_eOqgj!DLI1)Ga?b0@GF~eU(Pmg8@^kmX
z2A@=seLfvhW;&~)oAk5iD!i|rbguH#v^E{qZcE)06Z7v$YK#A82tM$9_kz%UItMl~
zo}XR3Z(`U-d7~4b=Pmviw_Sg2!;+s@u2zOMJXC3wJuFyQ5Fy?0?VI&&-b0C^nnoAi
zPgM{%54K#lHkx<Q>rFF!{N&l2rdq0A)?wXkJwZA<>-;I%Yu9{?OwNifZsiMqGGT@1
zR23)D-UbFHl_1ak%l8)+bT)b#9oTe!xnA7`9yv+d{|sf$bAG)D%6=!gWlh8;ll2Qf
z+-o;24?FYn!oC0z_KZd4u}+TXr>%=}$lW-z@y31r!)Js;++VUx`ZagY`3tV;LLocn
zga!#os{A;+iZRW4g5=hOYd!0aX!&fLZ?I_LX=9G;po(kDmaTlIlQOeWb<PGBp*)WV
z!in9P-gEEF+`P}!?#!Kr%l_`Y)6)utmn{rdQ(3O5z;@}3R{e|{Rmz8}e(sWZy3*S<
zG(PI}%*$7wew}c|$cyRLtct_zu3EY))?PjP=9KT|8#xhfDv#1wz1a^w@=UAcb^g1}
zu<NaU=2G<z&Y2xvY(XMi&eAJY+Ej!CzWQn;tG#@4Xtna$Ord%^%>}zo`<+hr6o^^r
zyyKg~?c3W-W=whR85BA9(2ANpJC7aiPMm1xwJAjF_{8U*S)xp3?yq|NE3kNFQQpF{
zD>j|q>>BFJVp_Cn<>dbiJ=}lqHeSll7QFGuXk+nn$pitf6`sl2cT967?x?d(?J>G+
z7vyzk$)pKiuK(ou?W(NvQQvsY(pmf0YZR^9IOW(y*XK@CzTOMF^(EA0VdK6Drfgf!
z9m#Dxcj&}i>pj!@UZ|~`#cuXt(h>gLWZ#}QTej$UJUkV$fcwr$h8HhXdhd387Vq{;
zj=yL*&DHzLmEbF>A|@L4$x%zQx|9VL&eWB8v;Fcj_cUp?9&TY~-sV|iDJSAeCI+X8
ztFF2iD(&hs$6R1)heha*E1L^H>`YJps3N^-Wm(`Fv&Bo!71>-{zr1+PY?G5(DekMY
zxK`YfOwd^r5$$>Rn#T>R4N@GbS{~Vty1UL#w&qDs<Oqu4x#z?Agt5T=o@{T-%u8)*
zUY9mVTDCf_*2s~#(A}mOCipU6WRJ<Y;{Ck#hwpbw3a9jX%?wTVNk1Dr<G8QC%#^A9
zDm<SzUh2s#mY8(xz#7+bE&Zl9@)<2_+^pNqcuk#_5XIm<X=W$i;lR0CS62thOx{?#
z{_u%Me}bDHc{OVLy!4r>>Eh$S&8T42Ua2!HYe5E&<p<}3J-cQ#udX_8c)lus^{@2h
zMwYL&ADk}Ekn+@8l_kby5#H}?wvuo1{+sizt0aBj_9t1~EM=bjUA?GhXO_2~N?j;9
zWtEWZO;a9Y_U5wB9~rY^=J2e{nrba*sNoqUax-Ak%C2<_0+{_J%DVKs7cYCUM}JAX
z&KaZdm2u10t**-6e=BHS&bo;%v$pGUR;Vz<Oij(pGq`thvA{Z|r7L9S-V*vfWo@3#
zjjqkE*OTil_L+rRY9*_3p7q%|*(x#ZKSNqTl_5__zhcRkuS#22+<sH~p|<1HrDF#w
zJ4?07H{NX6!Su>~<31}sgB2<Q@19-#EM41E_D)w*Exv-&^URs@jd!As$6KrnyME=k
zdo+{JiT#=8aqd&}t}FfFv(D|-QT*7Xci%d^?EaLPlZRBjovzsXpSxS!@^tG~&dZja
zQ=j)O%IdiEdc($@s(TM?U?|cn=$IvHvtpKt{0)O&uX3OKNtyC5RX1Ql{<r7*K7M!a
znze7~>Bq|>1bmK)s`VJzP26z**`q(r`xCq78QK~>Nz?b<{F>+SPt}K_v%N0Q54&Y{
zZdQl)wSTPPp`5X^R~hZHeEjK)^&ZoY)3n@G^rr4Nzwzx@^p@f)>v)f(p0@v0Y|Atu
zI$CPJ#(~1fr8f2*r``WET%55Wu4koJ`jfApI`cn@M%?`VqwV8yyG7Hyvb4^vJ^M=K
z^E10d=aBbl?^o1X-WJ+zv01-4;c4@1otc>h5;Y9bS;14n{|P+olipNg&Tn8fSt)==
z<0nI*)c(H6V~!I~KAhhaxpGm~yy-7Hp08fDRM{(C^vJa<lI*)?KJ4gy`}A$k@q`^U
zFK1-liB)OrnYD;<`Hu@MTg(@Et^9ZEb^jmBgO3c^C2wi!-I#e)(`(+QCI1;Zd7n?s
zl79A7POnNNtLs#U)SROxZSmeWm>hP9-MY>$eW$?k)86LH-^VX#9yJP@*|3B6cg^Cf
zawcE(UB0%5l)6l@G(5ffrRpSp%j69!Q=>N<-Jd->=Kap+lJ5i$hc_Qymb2An=~mv`
zzPDyYU$a=BXb~=OAW@}rvazCr*bVWtFPCgCPI@MCW$NTBwG*YLcB?H@ne?AQ_Sl2u
zjDTiMuYixn{mbuJ-81J}Iq4JAky^WtZ{()Gw*P#u@08KIRr<5Ptcl;q^JHsSyvV#|
zCVlR2j$ZBTxGQn%@jlCA=FL+dUt43}R<5NoaZ&p#QPCf*_dOoUgg5l)S*85pIB6ND
z_E%eT!8y6*seumJOH%G>ROHY6;_E6}!|OS#WNN9_)u)kDgaoD*NcZ}(G<XU(aHJ@7
z_s-mT^ghemkj|wQl`Bu%MW!xkPl~wcc&P1mj&kXZbJrjKm@-vCXy&D>Q!|uShTb+P
zSz+2?mk=eJvh&T&5)sLxI-JG_{T-qn@al*@-tnFN^v2)<*Z1dm?YSmR^=4~XHM@yv
z_Qe+^_Y?~GMATPCq>2@7`&3Z(MX|&1%(7{VE!UjUQtX?mGE+vR*F<Ti#2fRyTbj2s
zaM#9n%~LtW;jq@uIrdSRTERpeiyzUkt#2zTr~1qJ&q=-V)BaXcmb3S_XL_Y)Rs2`j
zTUF*N@QKzHh^|ZdsbwtkGE&R?#lzKW^~$^U)GRr7c+I1TXJ@BP;}YAvZ+TJjm*W2n
zSC(rpUG-``=dV+<7A7v6y*MLn<*HPDl?=g9&;2WJ%)b&dv8Lwje}=>F3^V-9nOIq#
z1|2STOn>Ti%g*gT!-M-BYZ46qSSDWSak^>7GRbo*L;h5TEfxC9g)D0H&-~h~&o+0_
z`Gh5I8<N#_Rmz+@^8T5yAVbsv0qc0#_nn1HPd&;ww7*@^>f)3MXR{@KP3}4SpW(gc
z0soaDn--s(!r-fMKzy(N7Vf2+&m1Vt_bvUKQYsqjES8nlr#?Ysvd5mxvrQBHZ3Oq9
zonJIjFHQ6hw<G(t_NRFT|I80Bt-t*2!^zYoD|y~V-Ff%>ovujK&0-F#Ym=5Q`K)zk
z--@(Pda+?GZWm6SQ3+MPk=+w_?RM|l&K(L%5>FX2Xrz2(4p?BLQnNqflBdho4^QhV
z6+KTFH-|Hb%wL#z&UUU{vi*-mABtCfj<l+i*?cswv1yjoio~fdmA%639U8@{@p_>X
z9d=r!`JU*PncaDE@&Ut7>L((@grycS|C`FDG;3GHEaPQY`{xzRF!`pw#mJ!cSq;BH
z(5V<pmjkCvc5-_QpH=Hy@mb6}Jow_5P;F19t5-sUJ!OKJ1e}8=xx8~Q|FR%V`LEBZ
zvdewTdo|`PIX$aJQ(?k&QHHA5Uozg>_;bz4y0o%&QY%xbs)`2Ve}<ih-!O_9&P$2>
zV4HCHS#B^}e7Mb&uSQFIW^prkY_AWR_~rY-qKTS88@>j)C72Wjynnjv<I+bKLUY(6
zO@kN|q;2byc<N?XO!S}eWA#6gUFk<EGas%z`y%<b&s3=`=C@W#u3WWP+`i29+OGz2
z)w<jhhDE9c|LpIcTBK(1pW(IvtIE&T;IjTn)|z(iEN^@M2|ZCv*N#7I{AbctU&oY<
z7k|1xd34~)VgrUI!L3vGz0nJqkh}5UjJKD>ST?L|{m(G*bN|-QtWVVgf6MJ#WMVh%
z9AEmyoh?Sak5&fF&)9v_(alEo?2dgJJKALaGh7K`Sk-8<?BBfVW5#_S?mh2P=)W>K
zE6;w%yTXV}V-v%pOSNBKOHuhC_TUDe<u*mX2`-zK%KuLPx$MruPaAGN+0~mm?fVv?
z<$9+#<xN<8GBb0v>grh>pDSNw#i|(joK@Q3z;IbouyM(xJkFgvj;^_|)W_%L(ivt~
z6tvd7%ZvNvbmGX_BaeB$3J7sJFdk-{w!D;m$I|9F<9(~!7Bwp@TfAhF*@;)HT-I{)
zhuXa03=h5?G_gQWZ}Ds%-^o4NGw=HGEU8g1SQrrPkY)3|;@XFTsgHb){%5%3kWlWq
z!+IlY=Y^jKR;Ei%O?Q5><r1@u$khp(JhmHd3^?QXpv_F7%=&XT=cT8Mtr}M{u3u^w
zZ{_^l<6Y__LvEgsAZO{V?31(iE=ty#t+2pn^D3hg%RP%aTUBe0OwIb0Znb#kltQH?
zyEs_xdsu%joU&!YoPUxFJF-~0f+sFa&gkxy=$!dsa{X5uPx)61o#X>N3O7%bs`q#r
zYX9|YE{m;$Ro-r0uT|Sucb~OOpTIpeR!mD)I()m9<KB%fJu&ZIoS8CVQADcQO64dq
zh3(f2ZHlEeBUpSw*1Vh*G+|b-t=1QvRhc_lO6{^|e3{-cPrfwIb(;Gn(_b%*|17IA
z3$2>gw=BXnn$@h&t*WSIW5PlQF}5Fhl17CdbK7SuHe@yupLzDuPv^S|7E&GhJP$pN
z7p(DP7TNbzMK9>kDh5rjP3)o!+K==?-eoB}tX;hH70<i8{z>z7PVGuJ|FmneTA#!X
zy};l|jr3NAJ8qL417)`}Foiulx|#E+WDv`xSF@E5E%HCoV51kcw8Q#+)>GA_>n(@>
zGhA`hnyn!j%$YBh%EZ+Frz>RA_j?QW#a^20JEJ0d#<QIvL5?#n)~?Fpdc|qlku8|2
znW<C#X-Uk=``dosd@XU3<Nfi*1WlD#mG1Vkbh#wcKjAHlyS=A(C~GwZEm_q!U8<IM
z{uK4~MLtbUv-Ow0)H?avsA|UJl^tG7)fe$ZeVn#|CuF5m>pKVYZ|y4QCal@9zE>qA
z>zdX5_$AVt=eP@6o1BTr>{O9;nz`y+M%m4m;!!#0qTa1uS7~?G^V-k+{|rT$%*Wo&
zyX?-iTfA05)uY>c^K4Fs0*?ur3Fi)<`ZK+v?cJ1vj{OajZcdVac-CiET7l$|^z;Xt
z9!zN2wBd7&LI`Kdip}gQ3syDMHJ#y-T{A`Ux%-#rv3i;R>aUx8J@GGEXM<y;cc@5L
z$dS-fyB3CTP!Eir+b>bHVaLfEDa#Tay(2pmJAQZ+Jh<+VG++4e#vl8{Qf3;R@+kM3
z(#kaH=ZXi@`IojPD}5B#^PBRKeTnvehF4j7OG;Lngl0_nbmW89&+@LY#NYldZx%LY
zGv+ZI-Y_|BUC`#!C#I_2Jy;z+efv4H*a^pa^10@;dKkT&Y3f;CJX5E})O#ay?E}l@
zPG3v4&(FM?`td)*^aGzRtabKlomJ7<qjz?tgUUO0<~j20M?ZX?rqXm;L30(eY^K+|
zmWTw4Z=Cgyoc`p9oQa&mc(ecWDfYmKsW*KjryMJszQ@-wQX)TmM%P-AuF9RkOK$W|
zX+EL&DRf$z7n66>z5fhRHgbOQyXK^{^c;BR`NXrJrsL-&#m`(`Z#JteI`=wh$*R?J
zre)2|U7GC1`D9VI#1}aZwm)~GD=R8H-t$VfA6{_v*!$IgzD!oiS-E3!>m(1al4sd7
z=Ei54&Qh&0J?rb2&6n(}bLDGiyiBOq?=>q-t6cXUR<vOX-}+%rmcu&^1##)<_+``9
zE@IE#py|`BnRw7s`Dy%$Cm;9zXXv=sd)mr;$sg<PWp{$EU%sH1Ki}<?&Hk-v0iBNp
z8O$&GU0AtO>FFAGX3iV)l+6W=sx~r+XjTT8p4Pn_^lRmx#q;aL%9`>8k36b3{vbND
z==*7vr<QxJiN4|N*D2`_GxvP{Gd;*UWKFlS-yv-_LxH7h>OutH@#o}y+~L?YWhdXU
z`y2k%DzE!k$5D7AbB4=ifvcOZt~@p2^N}<2Cab>UIn}Nkt{odHvtZ+{os}#nPMKXY
zIkD(=)RCm!ZB|PKO|CNZZ!X@#zVVM^MURrW`F{o<^9`pQ@7>$p?@*DjQ)G&4^rAhf
z*<n{pIhag(pPqI!{t&SKUSRb%6~=v6EKW)^c2CyVOO?I3PWE+LeZ<q)&9gg&H@tlJ
zIrWs@!ONz~<r|k<%{cNaE6VVCNyt^+U{$ts$8b-bb<Q@8i&eZ>_o`U*9{RT7+p*bV
zOTX7X*&rkM^0fOu<>iH(+BT-H49vQo0Z)$IdEucR-Z|&I`q~pKi@HwDdTDg3!|twW
zYz#|#_;24wDVOs2=JIgd%`0c;Y(6IWc^RkTsgCpwDed3X=WDr!2}|qMoSbxN&PhwA
zpqv^x^}0L`^QA}fx0-DFactx9_<%3_OfFsux$Yb*5F+_kq;PJww$_TM9U<(I5{(K@
z5={<A-FSryR{uHE<ykx<WBTF-&ywjY@)e>i|A`e@m<miksI%14w|c9v8_WEUD|UJC
z1-)Yw^nINbvwTfe()6QVyR4?4&D2!#xn^1*w9YleV^#V6@Vjn-C$y?%q6&UaW>fyO
zS>)`O<Ro*sE!}a9R%PrG>$qBZR;epyEfLppyFF#T;fj94CnanitBW~jxT;+1S)C{O
z=irL&`1|{Fe2zS-_i}EGe}0AI^^BiCLsZqjT)#NO_0p=o$X)V!I#brLD9y}W;UT+~
zLBCeD)M;+xo%bCQI$2E5c6D&-l`fridty?ub=$ltA2}OrPhOm1zCy?EUx1d)w?!UB
zZg!=1vF=md-DO<A$};9_?%yr@Plr9n|B$Yxhf`>f*BrL;`zLB2%vdMavMPCc)b$xZ
z=eOUexieA3^3lpyJ(tbS#CdPE@eeyyU!l4~rZ~y%w(Q;wIc16(0h7Gdu3X6e+{@Ft
zm{V2gz)t2XU+O+CcAxybcjB|3Uxb`xL{4m4KV4JU!IW1|>hOPtgq-|M6Wbqde8DYI
zvM;)5gQ@)eYevgE6YGt73{_Ulb>({;Z@Z;iul7-V;Pc8lXZ4t;S7NiLw?(Xc^+ZYM
zSi`CncA*h}7krs&6|qYoLR30RXN{PkV)itity=5s!;X6X@XbztE?D88`r*=ZAAi2c
zNxP@qi<)!Uh`;4};MAzS_r#Cb%+oopdusKksIHw=dCTT`X8yb8e#>S??3Be53!=<)
zw!be-I*@Zm&g4JCZN)2Jl^XWO*VUhiEX%4|ayH74fpewe?T4k@jmr|_q8eg9-pV@p
zbY9iTh_28XE)yp3_D_@kq)`2zLAYCUwe0n>mUgD?_ZQvzd2C<Fl9uoBE86S)?pMq<
zJ+{hAXVKbelRCAQok@y)_iKVkP}sMxYR~6><nz$*YFY5g($UJGTqon}*AC4osn6!?
ztvV(AG-}o-!*yTEyGm+9eJ7XbcfL0c4!E-Zr&@*Z+1V;ZZ!1z9Onsexo!jBa>S^Sj
z&ZD~WQH@c|LbEi1RNcS?himg!T@}9~alG)|iPh})3i-s&8|v&V6J`)mxM}Rnvnoyb
z&~oq8FS%vq*J?HY$?r*gnNd9Rm-otF_A86-{F@WBXiwc@z36!6ttTeR*ebWCrqpYf
z2KV{iyy9WIaLUrfMukGl6rQatF}X6|W@+i(hP6As$+vxYlb&T@v2)Uqn-M#MQ^gB1
zr`0TFOXBi~C~)-4(&$Kkq*gqk<ob`uml0O8HBSGkws`fQL11;(vUR649t)aSx`q0Q
zoc3V|SZkMQ9i`NMY3-7U$;;kMY3_bJV~v}vOOMUWq62{@#?!X0?A}pw+N0{ozq18z
zLjK+Kwa9EsGKdIiXkD={$K@%L(5^{NDh;pCXazkG?@=~b`=4Rg@>LZpmY>P=n5UXm
zI&IR@9L})Lx1|YZN;Yhraqr0NNUy_^#XX&klh$^!*sYWQ-Z%U3lgWC0LaU@UHP&)e
zwPjULV7_QvGk?*p!zL$R{3@IIOMdm7`5P}SK4}&n^=n<SZ`9IdNqG!*Db>B#t6woK
zdpK#q(lmzY?2q{LL^R6RCjDn<Pu}>$vY1cj`zObbTAedy8Ba@D!xgQyQ{s^En@!7=
z&Kc`WTFMjI<8#rcr72nKiGYNV#qH#7A2UmDj~Q40nf}hrcrk;=Dc#HG${%x?J0*#w
zhbGO}?@8Ud)%cRhBNt)!rTgAZTC+XzWkPsz@_F_&#Z<ke?aS}&-(%&Krdhdc39F%q
z%^hj(4!vNVnW<?}?-pqaUOU%*<zUB7yZde0hA#e&oEurLDLqzQRwQ=STl&+lU&j|2
zdENT6-QZ>BqLpmRRX;Md8#=5xu<F!@*9x<{?}_Gp|Mj`$(I<ODWq}YI;|;I;rh5cO
zZVAqn*)6RWb22z+$&d7hX$qe;xze9LQudO#`lsN-)tB{0*aJ)T>f@(fK0UW|YsweP
ztYhzrwOm(EeLD3?w2s0HErGjztLNGzZQfM7#9|-Q%!&{J)ss#kiyRnTIXM|ZR?l1@
z^O*NX@~tOl-yG=?6J(ibk;1Aofk`A_!j#5R<DDiRSvD1q*I6~2oUhuwa@TRgHJQdy
zT|c#DW3qP5w0rp`r|cL1GOK06e5#()o2N-AD%vfGe=7TW^Bglbac1@QQ~r&boL`y#
zZU5Z%(=KbRXtG4wy@j_9tey7FOuFU~pAD;q;D&A+H&%y+Ll^(w<dWtR$jnVlPt{M(
z&&e+;E=WvH6=LUN<zfzUcjD6G;)6)0W#**jCFZ6INpOjAiR2j=nVA?Gn_I>knphYc
z>v;vZx^u~Ji9yunCuOB3m*^)Y<yYue=4R#zF$pnXm#xT6tN_c3a&bfSmS?7vWC*cI
zv2ZbcVdhfi5`qXP=Oh*v>lat%6{nU6@p5r<al)N|%Vem%5?nkGBhvDVauZ90IJnrj
zSitU;<`RU+Kn%}GOiImx*xbX+CCeoakwo?kL<y4=1D7}~T#_>q^YT)2iiKFEn7J64
JT@R)11ONnn*>V5?

literal 0
HcmV?d00001

-- 
GitLab


From 9d8779eebdf0e813748fa1b81b975f443f84f73a Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Mon, 26 Mar 2018 10:52:46 -0700
Subject: [PATCH 485/960] [tf.data] Usability improvements to
 `tf.contrib.data.make_csv_dataset`.

PiperOrigin-RevId: 190489086
---
 .../contrib/data/python/kernel_tests/BUILD    |   2 -
 .../kernel_tests/reader_dataset_ops_test.py   | 268 +++++++++++++++---
 tensorflow/contrib/data/python/ops/BUILD      |   4 +
 tensorflow/contrib/data/python/ops/readers.py | 250 +++++++++++++---
 4 files changed, 438 insertions(+), 86 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 8cfe4a727a..d7cc2f14a4 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -294,9 +294,7 @@ py_test(
         "//tensorflow/python:errors",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:lib",
-        "//tensorflow/python:math_ops",
         "//tensorflow/python:parsing_ops",
-        "//tensorflow/python:string_ops",
         "//tensorflow/python:util",
         "//tensorflow/python/data/ops:iterator_ops",
         "//third_party/py/numpy",
diff --git a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
index 699e8e7865..6ee1b572f1 100644
--- a/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/reader_dataset_ops_test.py
@@ -35,9 +35,7 @@ from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.lib.io import python_io
 from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
-from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 from tensorflow.python.util import compat
 
@@ -568,12 +566,20 @@ class MakeCsvDatasetTest(test.TestCase):
       dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float64, dtypes.string
   ]
   COLUMNS = ["col%d" % i for i in range(len(COLUMN_TYPES))]
+  DEFAULT_VALS = [[], [], [], [], ["NULL"]]
+  DEFAULTS = [
+      constant_op.constant([], dtype=dtypes.int32),
+      constant_op.constant([], dtype=dtypes.int64),
+      constant_op.constant([], dtype=dtypes.float32),
+      constant_op.constant([], dtype=dtypes.float64),
+      constant_op.constant(["NULL"], dtype=dtypes.string)
+  ]
   LABEL = COLUMNS[0]
 
   def setUp(self):
     super(MakeCsvDatasetTest, self).setUp()
     self._num_files = 2
-    self._num_records = 7
+    self._num_records = 11
     self._test_filenames = self._create_files()
 
   def _csv_values(self, fileno, recordno):
@@ -588,49 +594,63 @@ class MakeCsvDatasetTest(test.TestCase):
   def _csv_record(self, fileno, recordno):
     return ",".join(str(v) for v in self._csv_values(fileno, recordno))
 
+  def _create_file(self, fileno, header=True, comment=True):
+    fn = os.path.join(self.get_temp_dir(), "csv_file%d.csv" % fileno)
+    f = open(fn, "w")
+    if header:
+      f.write(",".join(self.COLUMNS) + "\n")
+    for recno in range(self._num_records):
+      f.write(self._csv_record(fileno, recno) + "\n")
+      if comment:
+        f.write("# Some comment goes here. Should be ignored!\n")
+    f.close()
+    return fn
+
   def _create_files(self):
     filenames = []
     for i in range(self._num_files):
-      fn = os.path.join(self.get_temp_dir(), "csv_file%d.csv" % i)
-      filenames.append(fn)
-      f = open(fn, "w")
-      f.write(",".join(self.COLUMNS) + "\n")  # header line
-      for j in range(self._num_records):
-        f.write(self._csv_record(i, j) + "\n")
-        f.write("# Some comment goes here. Should be ignored!\n")
-      f.close()
+      filenames.append(self._create_file(i))
     return filenames
 
-  def _make_csv_dataset(self,
-                        filenames,
-                        defaults,
-                        label_key=LABEL,
-                        batch_size=1,
-                        num_epochs=1,
-                        shuffle=False,
-                        shuffle_seed=None):
+  def _make_csv_dataset(
+      self,
+      filenames,
+      defaults,
+      column_names=COLUMNS,
+      label_name=LABEL,
+      batch_size=1,
+      num_epochs=1,
+      shuffle=False,
+      shuffle_seed=None,
+      header=True,
+      comment="#",
+      na_value="",
+      default_float_type=dtypes.float32,
+  ):
     return readers.make_csv_dataset(
         filenames,
-        column_keys=self.COLUMNS,
-        column_defaults=defaults,
-        label_key=label_key,
         batch_size=batch_size,
+        column_names=column_names,
+        column_defaults=defaults,
+        label_name=label_name,
         num_epochs=num_epochs,
         shuffle=shuffle,
         shuffle_seed=shuffle_seed,
-        skip=1,
-        filter_fn=
-        lambda line: math_ops.not_equal(string_ops.substr(line, 0, 1), "#"),
+        header=header,
+        comment=comment,
+        na_value=na_value,
+        default_float_type=default_float_type,
     )
 
-  def _next_actual_batch(self, file_indices, batch_size, num_epochs):
+  def _next_actual_batch(self, file_indices, batch_size, num_epochs, defaults):
     features = {col: list() for col in self.COLUMNS}
     for _ in range(num_epochs):
       for i in file_indices:
         for j in range(self._num_records):
           values = self._csv_values(i, j)
-          if not values[-1]:
-            values[-1] = "NULL"  # null values in csv are interpreted as default
+          for n, v in enumerate(values):
+            if v == "":  # pylint: disable=g-explicit-bool-comparison
+              values[n] = defaults[n][0]
           values[-1] = values[-1].encode("utf-8")
 
           # Regroup lists by column instead of row
@@ -651,7 +671,8 @@ class MakeCsvDatasetTest(test.TestCase):
       sess,
       dataset,
       file_indices,
-      label_key=LABEL,
+      defaults=tuple(DEFAULT_VALS),
+      label_name=LABEL,
       batch_size=1,
       num_epochs=1,
   ):
@@ -659,11 +680,11 @@ class MakeCsvDatasetTest(test.TestCase):
     get_next = iterator.get_next()
 
     for expected_features in self._next_actual_batch(file_indices, batch_size,
-                                                     num_epochs):
+                                                     num_epochs, defaults):
       actual_features = sess.run(get_next)
 
-      if label_key is not None:
-        expected_labels = expected_features.pop(label_key)
+      if label_name is not None:
+        expected_labels = expected_features.pop(label_name)
         # Compare labels
         self.assertAllEqual(expected_labels, actual_features[1])
         actual_features = actual_features[0]  # Extract features dict from tuple
@@ -676,10 +697,7 @@ class MakeCsvDatasetTest(test.TestCase):
       sess.run(get_next)
 
   def test_make_csv_dataset(self):
-    defaults = [
-        constant_op.constant([], dtype=d) for d in self.COLUMN_TYPES[:-1]
-    ]
-    defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
+    defaults = self.DEFAULTS
 
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
@@ -705,11 +723,26 @@ class MakeCsvDatasetTest(test.TestCase):
         self._verify_records(
             sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
 
+  def test_make_csv_dataset_with_bad_columns(self):
+    """Tests that exception is raised when input is malformed.
+    """
+    dupe_columns = self.COLUMNS[:-1] + self.COLUMNS[:1]
+    defaults = self.DEFAULTS
+
+    # Duplicate column names
+    with self.assertRaises(ValueError):
+      self._make_csv_dataset(
+          self._test_filenames, defaults, column_names=dupe_columns)
+
+    # Label key not one of column names
+    with self.assertRaises(ValueError):
+      self._make_csv_dataset(
+          self._test_filenames, defaults, label_name="not_a_real_label")
+
   def test_make_csv_dataset_with_no_label(self):
-    defaults = [
-        constant_op.constant([], dtype=d) for d in self.COLUMN_TYPES[:-1]
-    ]
-    defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
+    """Tests that CSV datasets can be created when no label is specified.
+    """
+    defaults = self.DEFAULTS
     with ops.Graph().as_default() as g:
       with self.test_session(graph=g) as sess:
         # Read from both files. Make sure this works with no label key supplied.
@@ -718,16 +751,64 @@ class MakeCsvDatasetTest(test.TestCase):
             defaults,
             batch_size=2,
             num_epochs=10,
-            label_key=None)
+            label_name=None)
         self._verify_records(
             sess,
             dataset,
             range(self._num_files),
             batch_size=2,
             num_epochs=10,
-            label_key=None)
+            label_name=None)
+
+  def test_make_csv_dataset_with_no_comments(self):
+    """Tests that datasets can be created from CSV files with no header line.
+    """
+    defaults = self.DEFAULTS
+    file_without_header = self._create_file(
+        len(self._test_filenames), comment=False)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            file_without_header,
+            defaults,
+            batch_size=2,
+            num_epochs=10,
+            comment=None,
+        )
+        self._verify_records(
+            sess,
+            dataset,
+            [len(self._test_filenames)],
+            batch_size=2,
+            num_epochs=10,
+        )
+
+  def test_make_csv_dataset_with_no_header(self):
+    """Tests that datasets can be created from CSV files with no header line.
+    """
+    defaults = self.DEFAULTS
+    file_without_header = self._create_file(
+        len(self._test_filenames), header=False)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            file_without_header,
+            defaults,
+            batch_size=2,
+            num_epochs=10,
+            header=False,
+        )
+        self._verify_records(
+            sess,
+            dataset,
+            [len(self._test_filenames)],
+            batch_size=2,
+            num_epochs=10,
+        )
 
   def test_make_csv_dataset_with_types(self):
+    """Tests that defaults can be a dtype instead of a Tensor for required vals.
+    """
     defaults = [d for d in self.COLUMN_TYPES[:-1]]
     defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
     with ops.Graph().as_default() as g:
@@ -735,10 +816,109 @@ class MakeCsvDatasetTest(test.TestCase):
         dataset = self._make_csv_dataset(self._test_filenames, defaults)
         self._verify_records(sess, dataset, range(self._num_files))
 
+  def test_make_csv_dataset_with_no_col_names(self):
+    """Tests that datasets can be created when column names are not specified.
+
+    In that case, we should infer the column names from the header lines.
+    """
+    defaults = self.DEFAULTS
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        # Read from both files. Exercise the `batch` and `num_epochs` parameters
+        # of make_csv_dataset and make sure they work.
+        dataset = self._make_csv_dataset(
+            self._test_filenames,
+            defaults,
+            column_names=None,
+            batch_size=2,
+            num_epochs=10)
+        self._verify_records(
+            sess, dataset, range(self._num_files), batch_size=2, num_epochs=10)
+
+  def test_make_csv_dataset_type_inference(self):
+    """Tests that datasets can be created when no defaults are specified.
+
+    In that case, we should infer the types from the first N records.
+    """
+    # Test that it works with standard test files (with comments, header, etc)
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            self._test_filenames, defaults=None, batch_size=2, num_epochs=10)
+        self._verify_records(
+            sess,
+            dataset,
+            range(self._num_files),
+            batch_size=2,
+            num_epochs=10,
+            defaults=[[], [], [], [], [""]])
+
+    # Test on a deliberately tricky file
+    fn = os.path.join(self.get_temp_dir(), "file.csv")
+    expected_dtypes = [
+        dtypes.int32, dtypes.int64, dtypes.float32, dtypes.float32,
+        dtypes.string, dtypes.string
+    ]
+    rows = [[0, 0, 0, "NAN", "", "a"], [1, 2**31 + 1, 2**64, 123, "NAN", ""],
+            ['"123"', 2, 2**64, 123.4, "NAN", '"cd,efg"']]
+    expected = [[0, 0, 0, 0, "", "a"], [1, 2**31 + 1, 2**64, 123, "", ""],
+                [123, 2, 2**64, 123.4, "", "cd,efg"]]
+    for row in expected:
+      row[-1] = row[-1].encode("utf-8")  # py3 expects byte strings
+      row[-2] = row[-2].encode("utf-8")  # py3 expects byte strings
+    col_names = ["col%d" % i for i in range(len(expected_dtypes))]
+    with open(fn, "w") as f:
+      f.write(",".join(col_names))
+      f.write("\n")
+      for row in rows:
+        f.write(",".join([str(v) if v else "" for v in row]) + "\n")
+
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            batch_size=1,
+            num_epochs=1,
+            label_name=None,
+            na_value="NAN",
+            default_float_type=dtypes.float32,
+        )
+        features = dataset.make_one_shot_iterator().get_next()
+        # Check that types match
+        for i in range(len(expected_dtypes)):
+          assert features["col%d" % i].dtype == expected_dtypes[i]
+        for i in range(len(rows)):
+          assert sess.run(features) == dict(zip(col_names, expected[i]))
+
+    # With float64 as default type for floats
+    expected_dtypes = [
+        dtypes.int32, dtypes.int64, dtypes.float64, dtypes.float64,
+        dtypes.string, dtypes.string
+    ]
+    with ops.Graph().as_default() as g:
+      with self.test_session(graph=g) as sess:
+        dataset = self._make_csv_dataset(
+            fn,
+            defaults=None,
+            column_names=None,
+            batch_size=1,
+            num_epochs=1,
+            label_name=None,
+            na_value="NAN",
+            default_float_type=dtypes.float64,
+        )
+        features = dataset.make_one_shot_iterator().get_next()
+        # Check that types match
+        for i in range(len(expected_dtypes)):
+          assert features["col%d" % i].dtype == expected_dtypes[i]
+        for i in range(len(rows)):
+          assert sess.run(features) == dict(zip(col_names, expected[i]))
+
   def test_make_csv_dataset_with_shuffle(self):
     total_records = self._num_files * self._num_records
-    defaults = [d for d in self.COLUMN_TYPES[:-1]]
-    defaults.append(constant_op.constant(["NULL"], dtype=dtypes.string))
+    defaults = self.DEFAULTS
     for batch_size in [1, 2]:
       with ops.Graph().as_default() as g:
         with self.test_session(graph=g) as sess:
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 4ecf02825f..647620eb84 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -72,14 +72,18 @@ py_library(
         "//tensorflow/python:dataset_ops_gen",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:lib",
+        "//tensorflow/python:math_ops",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
         "//tensorflow/python:sparse_tensor",
+        "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_shape",
         "//tensorflow/python:util",
         "//tensorflow/python/data/ops:dataset_ops",
         "//tensorflow/python/data/ops:readers",
         "//tensorflow/python/data/util:nest",
+        "//third_party/py/numpy",
     ],
 )
 
diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index f70f9c881d..95edca6cdd 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -17,6 +17,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import csv
+
+import numpy as np
+
 from tensorflow.contrib.data.python.ops import interleave_ops
 from tensorflow.contrib.data.python.ops import shuffle_ops
 from tensorflow.python.data.ops import dataset_ops
@@ -26,8 +30,11 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.lib.io import file_io
 from tensorflow.python.ops import gen_dataset_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import gfile
 from tensorflow.python.util import deprecation
 
@@ -35,21 +42,142 @@ _ACCEPTABLE_CSV_TYPES = (dtypes.float32, dtypes.float64, dtypes.int32,
                          dtypes.int64, dtypes.string)
 
 
+def _is_valid_int32(str_val):
+  try:
+    # Checks equality to prevent int32 overflow
+    return dtypes.int32.as_numpy_dtype(str_val) == dtypes.int64.as_numpy_dtype(
+        str_val)
+  except (ValueError, OverflowError):
+    return False
+
+
+def _is_valid_int64(str_val):
+  try:
+    dtypes.int64.as_numpy_dtype(str_val)
+    return True
+  except (ValueError, OverflowError):
+    return False
+
+
+def _is_valid_float(str_val, float_dtype):
+  try:
+    return float_dtype.as_numpy_dtype(str_val) < np.inf
+  except ValueError:
+    return False
+
+
+def _infer_type(str_val, na_value, prev_type, float_dtype):
+  """Given a string, infers its tensor type.
+
+  Infers the type of a value by picking the least 'permissive' type possible,
+  while still allowing the previous type inference for this column to be valid.
+
+  Args:
+    str_val: String value to infer the type of.
+    na_value: Additional string to recognize as a NA/NaN CSV value.
+    prev_type: Type previously inferred based on values of this column that
+      we've seen up till now.
+    float_dtype: Either `tf.float32` or `tf.float64`. Denotes what float type
+      to parse float strings as.
+  Returns:
+    Inferred dtype.
+  """
+  if str_val in ("", na_value):
+    return prev_type
+
+  if _is_valid_int32(str_val) and prev_type in (None, dtypes.int32):
+    return dtypes.int32
+
+  if _is_valid_int64(str_val) and prev_type in (None, dtypes.int32,
+                                                dtypes.int64):
+    return dtypes.int64
+
+  if _is_valid_float(str_val, float_dtype) and prev_type != dtypes.string:
+    return float_dtype
+
+  return dtypes.string
+
+
+def _next_csv_row(filenames, num_cols, field_delim, use_quote_delim, header,
+                  comment):
+  for fn in filenames:
+    with file_io.FileIO(fn, "r") as f:
+      rdr = csv.reader(
+          f,
+          delimiter=field_delim,
+          quoting=csv.QUOTE_MINIMAL if use_quote_delim else csv.QUOTE_NONE)
+      if header:
+        next(rdr)  # Skip header lines
+
+      for csv_row in rdr:
+        if comment is not None and csv_row[0].startswith(comment):
+          continue  # Skip comment lines
+
+        if len(csv_row) != num_cols:
+          raise ValueError(
+              "Problem inferring types: CSV row has different number of fields "
+              "than expected.")
+        yield csv_row
+
+
+def _infer_column_defaults(filenames, num_cols, field_delim, use_quote_delim,
+                           na_value, header, comment, float_dtype,
+                           rows_for_inference):
+  """Infers column types from the first N valid CSV records of files."""
+  inferred_types = [None] * num_cols
+
+  for rows_read, csv_row in enumerate(
+      _next_csv_row(filenames, num_cols, field_delim, use_quote_delim, header,
+                    comment)):
+    if rows_for_inference is not None and rows_read >= rows_for_inference:
+      break
+    for i, str_val in enumerate(csv_row):
+      inferred_types[i] = _infer_type(str_val, na_value, inferred_types[i],
+                                      float_dtype)
+
+  # Replace None's with a default type
+  inferred_types = [t or dtypes.string for t in inferred_types]
+  # Default to 0 or '' for null values
+  return [
+      constant_op.constant([0 if t is not dtypes.string else ""], dtype=t)
+      for t in inferred_types
+  ]
+
+
+def _infer_column_names(filenames, field_delim, use_quote_delim):
+  """Infers column names from first rows of files."""
+  csv_kwargs = {
+      "delimiter": field_delim,
+      "quoting": csv.QUOTE_MINIMAL if use_quote_delim else csv.QUOTE_NONE
+  }
+  with file_io.FileIO(filenames[0], "r") as f:
+    column_names = next(csv.reader(f, **csv_kwargs))
+
+  for name in filenames[1:]:
+    with file_io.FileIO(name, "r") as f:
+      if next(csv.reader(f, **csv_kwargs)) != column_names:
+        raise ValueError("Files have different column names in the header row.")
+  return column_names
+
+
 def make_csv_dataset(
     file_pattern,
     batch_size,
-    column_keys,
-    column_defaults,
-    label_key=None,
+    column_names=None,
+    column_defaults=None,
+    label_name=None,
     field_delim=",",
     use_quote_delim=True,
-    skip=0,
-    filter_fn=None,
+    na_value="",
+    header=True,
+    comment=None,
     num_epochs=None,
     shuffle=True,
     shuffle_buffer_size=10000,
     shuffle_seed=None,
     prefetch_buffer_size=1,
+    default_float_type=dtypes.float32,
+    num_rows_for_inference=100,
 ):
   """Reads CSV files into a dataset.
 
@@ -63,27 +191,36 @@ def make_csv_dataset(
       records. See @{tf.gfile.Glob} for pattern rules.
     batch_size: An int representing the number of consecutive elements of this
       dataset to combine in a single batch.
-    column_keys: A list of strings that corresponds to the CSV columns, in
-      order. One per column of the input record.
-    column_defaults: A list of default values for the CSV fields. One item per
-      column of the input record. Each item in the list is either one of the
-      following dtypes: float32, float64, int32, int64, or string, or a
-      `Tensor` with one of the aforementioned types. One item per column of
-      the input record, with either scalar default value for that column if it
-      is required, or, if the column is required, an empty `Tensor` or a dtype.
-    label_key: A optional string corresponding to the label column. If provided,
-      the data for this column is returned as a separate `Tensor` from the
-      features dictionary, so that the dataset complies with the format expected
-      by a `tf.Estimator.train` or `tf.Estimator.evaluate` input function.
+    column_names: An optional list of strings that corresponds to the CSV
+      columns, in order. One per column of the input record. If this is not
+      provided, infers the column names from the first row of the records.
+      These names will be the keys of the features dict of each dataset element.
+    column_defaults: A optional list of default values for the CSV fields. One
+      item per column of the input record. Each item in the list is either a
+      valid CSV dtype (float32, float64, int32, int64, or string), or a
+      `Tensor` with one of the aforementioned types. The tensor can either be
+      a scalar default value (if the column is optional), or an empty tensor (if
+      the column is required). If a dtype is provided instead of a tensor, the
+      column is also treated as required. If this list is not provided, tries
+      to infer types based on reading the first num_rows_for_inference rows of
+      files specified, and assumes all columns are optional, defaulting to `0`
+      for numeric values and `""` for string values.
+    label_name: A optional string corresponding to the label column. If
+      provided, the data for this column is returned as a separate `Tensor` from
+      the features dictionary, so that the dataset complies with the format
+      expected by a `tf.Estimator.train` or `tf.Estimator.evaluate` input
+      function.
     field_delim: An optional `string`. Defaults to `","`. Char delimiter to
       separate fields in a record.
     use_quote_delim: An optional bool. Defaults to `True`. If false, treats
       double quotation marks as regular characters inside of the string fields.
-    skip: An integer that corresponds to the number of lines to skip at the
-      head of each CSV file. Defaults to 0.
-    filter_fn: A callable function that takes in a CSV string and returns a
-      boolean that corresponds to whether the record should be included. If
-      None, does not filter records.
+    na_value: Additional string to recognize as NA/NaN.
+    header: A bool that indicates whether the first rows of provided CSV files
+      correspond to header lines with column names, and should not be included
+      in the data.
+    comment: An optional character string that marks lines that should not be
+      parsed as csv records. If this is provided, all lines that start with
+      this character will not be parsed.
     num_epochs: An int specifying the number of times this dataset is repeated.
       If None, cycles through the dataset forever.
     shuffle: A bool that indicates whether the input should be shuffled.
@@ -94,50 +231,83 @@ def make_csv_dataset(
     prefetch_buffer_size: An int specifying the number of feature batches to
       prefetch for performance improvement. Recommended value is the number of
       batches consumed per training step.
+    default_float_type: Either `tf.float32` or `tf.float64`. If defaults are
+      not provided, float-like strings are interpreted to be this type.
+    num_rows_for_inference: Number of rows of a file to use for type inference
+      if record_defaults is not provided. If None, reads all the rows of all
+      the files. Defaults to 100.
 
   Returns:
     A dataset, where each element is a (features, labels) tuple that corresponds
     to a batch of `batch_size` CSV rows. The features dictionary maps feature
     column names to `Tensor`s containing the corresponding column data, and
     labels is a `Tensor` containing the column data for the label column
-    specified by `label_key`.
+    specified by `label_name`.
+
+  Raises:
+    ValueError: If any of the arguments is malformed.
   """
-  filenames = _get_file_names(file_pattern, False)
-  column_defaults = [
-      constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x
-      for x in column_defaults
-  ]
+  filenames = _get_file_names(file_pattern, shuffle)
+  if comment is not None and len(comment) != 1:
+    raise ValueError("`comment` arg must be a single-character string or None")
+
+  # Clean arguments; figure out column names and defaults
+  if column_names is None:
+    if not header:
+      raise ValueError("Cannot infer column names without a header line.")
+    # If column names are not provided, infer from the header lines
+    column_names = _infer_column_names(filenames, field_delim, use_quote_delim)
+  if len(column_names) != len(set(column_names)):
+    raise ValueError("Cannot have duplicate column names.")
+
+  if column_defaults is not None:
+    column_defaults = [
+        constant_op.constant([], dtype=x) if x in _ACCEPTABLE_CSV_TYPES else x
+        for x in column_defaults
+    ]
+  else:
+    # If column defaults are not provided, infer from records at graph
+    # construction time
+    column_defaults = _infer_column_defaults(
+        filenames, len(column_names), field_delim, use_quote_delim, na_value,
+        header, comment, default_float_type, num_rows_for_inference)
 
   dataset = dataset_ops.Dataset.from_tensor_slices(filenames)
-  if label_key is not None:
-    assert label_key in column_keys
+  if label_name is not None and label_name not in column_names:
+    raise ValueError("`label_name` provided must be one of the columns.")
+
+  # Define map and filter functions
+  def filter_fn(line):
+    return math_ops.not_equal(string_ops.substr(line, 0, 1), comment)
 
   def filename_to_dataset(filename):
     ds = core_readers.TextLineDataset(filename)
-    if skip > 0:
-      ds = ds.skip(skip)
-    if filter_fn is not None:
+    if header:
+      ds = ds.skip(1)
+    if comment is not None:
       ds = ds.filter(filter_fn)
     return ds
 
   def decode_csv(line):
-    """Decodes csv line into features.
+    """Decodes CSV line into features.
 
     Args:
       line: String tensor corresponding to one csv record.
     Returns:
       A dictionary of feature names to values for that particular record. If
-      label_key is provided, extracts the label feature to be returned as the
+      label_name is provided, extracts the label feature to be returned as the
       second element of the tuple.
     """
     columns = parsing_ops.decode_csv(
         line,
         column_defaults,
         field_delim=field_delim,
-        use_quote_delim=use_quote_delim)
-    features = dict(zip(column_keys, columns))
-    if label_key is not None:
-      label = features.pop(label_key)
+        use_quote_delim=use_quote_delim,
+        na_value=na_value,
+    )
+    features = dict(zip(column_names, columns))
+    if label_name is not None:
+      label = features.pop(label_name)
       return features, label
     return features
 
@@ -287,7 +457,7 @@ def make_batched_features_dataset(file_pattern,
       lambda x: parsing_ops.parse_example(x, features),
       num_parallel_calls=parser_num_threads)
 
-  # TODO(rachelim): Add an optional label_key argument for extracting the label
+  # TODO(rachelim): Add an optional label_name argument for extracting the label
   # from the features dictionary, to comply with the type expected by the
   # input_fn to a `tf.Estimator.train` or `tf.Estimator.evaluate` function.
   dataset = dataset.prefetch(prefetch_buffer_size)
-- 
GitLab


From 005b8aa42c273a0152642279d0c57aa9e08ccbe0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 11:05:37 -0700
Subject: [PATCH 486/960] Fixes an issue with calling
 tf.contrib.seq2seq.dynamic_decode with an extended BasicDecoder which for
 example returns a tf.contrib.seq2seq.AttentionWrapperState. In this case the
 internal while-loop fails when trying to store an instance
 tf.contrib.seq2seq.AttentionWrapperState in the internal TensorArray.

PiperOrigin-RevId: 190491787
---
 tensorflow/contrib/seq2seq/python/ops/decoder.py | 15 +++++----------
 tensorflow/python/ops/rnn.py                     | 15 ++++++---------
 2 files changed, 11 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/ops/decoder.py b/tensorflow/contrib/seq2seq/python/ops/decoder.py
index f14974b9d5..898493662d 100644
--- a/tensorflow/contrib/seq2seq/python/ops/decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/decoder.py
@@ -30,6 +30,7 @@ from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import rnn
+from tensorflow.python.ops import rnn_cell_impl
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.util import nest
@@ -39,6 +40,7 @@ __all__ = ["Decoder", "dynamic_decode"]
 
 
 _transpose_batch_time = rnn._transpose_batch_time  # pylint: disable=protected-access
+_zero_state_tensors = rnn_cell_impl._zero_state_tensors  # pylint: disable=protected-access
 
 
 @six.add_metaclass(abc.ABCMeta)
@@ -133,16 +135,8 @@ class Decoder(object):
 
 def _create_zero_outputs(size, dtype, batch_size):
   """Create a zero outputs Tensor structure."""
-  def _t(s):
-    return (s if isinstance(s, ops.Tensor) else constant_op.constant(
-        tensor_shape.TensorShape(s).as_list(),
-        dtype=dtypes.int32,
-        name="zero_suffix_shape"))
-
   def _create(s, d):
-    return array_ops.zeros(
-        array_ops.concat(
-            ([batch_size], _t(s)), axis=0), dtype=d)
+    return _zero_state_tensors(s, batch_size, d)
 
   return nest.map_structure(_create, size, dtype)
 
@@ -212,7 +206,8 @@ def dynamic_decode(decoder,
     initial_time = constant_op.constant(0, dtype=dtypes.int32)
 
     def _shape(batch_size, from_shape):
-      if not isinstance(from_shape, tensor_shape.TensorShape):
+      if (not isinstance(from_shape, tensor_shape.TensorShape) or
+          from_shape.ndims == 0):
         return tensor_shape.TensorShape(None)
       else:
         batch_size = tensor_util.constant_value(
diff --git a/tensorflow/python/ops/rnn.py b/tensorflow/python/ops/rnn.py
index 42af7f8b27..1dd464d51d 100644
--- a/tensorflow/python/ops/rnn.py
+++ b/tensorflow/python/ops/rnn.py
@@ -49,24 +49,21 @@ _concat = rnn_cell_impl._concat
 
 
 def _transpose_batch_time(x):
-  """Transpose the batch and time dimensions of a Tensor.
+  """Transposes the batch and time dimensions of a Tensor.
 
-  Retains as much of the static shape information as possible.
+  If the input tensor has rank < 2 it returns the original tensor. Retains as
+  much of the static shape information as possible.
 
   Args:
-    x: A tensor of rank 2 or higher.
+    x: A Tensor.
 
   Returns:
     x transposed along the first two dimensions.
-
-  Raises:
-    ValueError: if `x` is rank 1 or lower.
   """
   x_static_shape = x.get_shape()
   if x_static_shape.ndims is not None and x_static_shape.ndims < 2:
-    raise ValueError(
-        "Expected input tensor %s to have rank at least 2, but saw shape: %s" %
-        (x, x_static_shape))
+    return x
+
   x_rank = array_ops.rank(x)
   x_t = array_ops.transpose(
       x, array_ops.concat(
-- 
GitLab


From 7e4432ca4da28621deb20b8f3ce7cec6aa0e8e67 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Mon, 26 Mar 2018 11:15:37 -0700
Subject: [PATCH 487/960] BUGFIX: Fix failure-to-broadcast in Wishart.sample.

PiperOrigin-RevId: 190493969
---
 .../python/kernel_tests/wishart_test.py       | 20 +++++++++++++++++++
 .../distributions/python/ops/wishart.py       |  5 ++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
index 9044aa2850..dcecce981f 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/wishart_test.py
@@ -390,6 +390,26 @@ class WishartCholeskyTest(test.TestCase):
                 chol_scale, dtype=np.int32),
             validate_args=False)
 
+  def testSampleBroadcasts(self):
+    dims = 2
+    batch_shape = [2, 3]
+    sample_shape = [2, 1]
+    scale = np.float32([
+        [[1., 0.5],
+         [0.5, 1.]],
+        [[0.5, 0.25],
+         [0.25, 0.75]],
+    ])
+    scale = np.reshape(np.concatenate([scale, scale, scale], axis=0),
+                       batch_shape + [dims, dims])
+    wishart = distributions.WishartFull(df=5, scale=scale)
+    x = wishart.sample(sample_shape, seed=42)
+    with self.test_session() as sess:
+      x_ = sess.run(x)
+    expected_shape = sample_shape + batch_shape + [dims, dims]
+    self.assertAllEqual(expected_shape, x.shape)
+    self.assertAllEqual(expected_shape, x_.shape)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/wishart.py b/tensorflow/contrib/distributions/python/ops/wishart.py
index e4ac65012b..5a8c94dabf 100644
--- a/tensorflow/contrib/distributions/python/ops/wishart.py
+++ b/tensorflow/contrib/distributions/python/ops/wishart.py
@@ -228,9 +228,12 @@ class _WishartLinearOperator(distribution.Distribution):
     # Complexity: O(nbk)
     # This parametrization is equivalent to Chi2, i.e.,
     # ChiSquared(k) == Gamma(alpha=k/2, beta=1/2)
+    expanded_df = self.df * array_ops.ones(
+        self.scale_operator.batch_shape_tensor(),
+        dtype=self.df.dtype.base_dtype)
     g = random_ops.random_gamma(shape=[n],
                                 alpha=self._multi_gamma_sequence(
-                                    0.5 * self.df, self.dimension),
+                                    0.5 * expanded_df, self.dimension),
                                 beta=0.5,
                                 dtype=self.dtype,
                                 seed=distribution_util.gen_new_seed(
-- 
GitLab


From a7588a70a5de8ece6920f4eb8b877104ede898f7 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 26 Mar 2018 11:18:26 -0700
Subject: [PATCH 488/960] tf.GradientTape: Clearly say that tf.while_loop and
 tf.cond are not supported by tf.GradientTape.gradient() at this time.

PiperOrigin-RevId: 190494436
---
 tensorflow/python/eager/BUILD              |  1 +
 tensorflow/python/eager/backprop.py        |  8 ++++
 tensorflow/python/eager/backprop_test.py   | 44 ++++++++++++++++++++++
 tensorflow/python/ops/control_flow_grad.py |  5 +--
 4 files changed, 55 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 5bedf9c6fd..0e089a26eb 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -105,6 +105,7 @@ cuda_py_test(
         ":test",
         "//tensorflow/python:embedding_ops",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:resource_variable_ops",
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 06e11f6ef9..cdcce65c52 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -86,6 +86,14 @@ class _MockOp(object):
         return make_attr(typ, self.attrs[i + 1])
     raise KeyError(attr)
 
+  def _get_control_flow_context(self):
+    raise NotImplementedError(
+        "tf.GradientTape.gradients() does not support graph control flow "
+        "operations like tf.cond or tf.while at this time. Use tf.gradients() "
+        "instead. If you need this feature, please file a feature request at "
+        "https://github.com/tensorflow/tensorflow/issues/new"
+    )
+
 
 def _magic_gradient_function(op_name, attr_tuple, num_inputs,
                              inputs, outputs, out_grads):
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index bca2928708..f04d89a6d9 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -31,6 +31,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import custom_gradient
 from tensorflow.python.ops import embedding_ops
 from tensorflow.python.ops import gradients
@@ -384,6 +385,49 @@ class BackpropTest(test.TestCase):
     grad = g.gradient(y, [x])[0]
     self.assertEqual(self.evaluate(grad), 6.0)
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testGradientTapeWithCond(self):
+    x = constant_op.constant(3.0)
+
+    def true_fn():
+      return x
+
+    def false_fn():
+      return x * x
+
+    with backprop.GradientTape() as g:
+      g.watch(x)
+      y = control_flow_ops.cond(x < x, true_fn, false_fn)
+
+    if not context.executing_eagerly():
+      with self.assertRaisesRegexp(NotImplementedError, 'tf.gradients'):
+        dy = g.gradient(y, [x])[0]
+    else:
+      dy = g.gradient(y, [x])[0]
+      self.assertEqual(self.evaluate(dy), 6.0)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testGradientTapeWithWhileLoop(self):
+    i = constant_op.constant(1)
+    x = constant_op.constant(2.)
+
+    def cond(i, _):
+      return i < 3
+
+    def body(i, x):
+      return i + 1, x * 2
+
+    with backprop.GradientTape() as g:
+      g.watch([x])
+      _, y = control_flow_ops.while_loop(cond, body, [i, x])
+
+    if not context.executing_eagerly():
+      with self.assertRaisesRegexp(NotImplementedError, 'tf.gradients'):
+        dy = g.gradient(y, [x])[0]
+    else:
+      dy = g.gradient(y, [x])[0]
+      self.assertEqual(self.evaluate(dy), 4.0)
+
   @test_util.assert_no_new_tensors
   def testGradientTapeGradientCalledMultipleTimes(self):
     with backprop.GradientTape() as g:
diff --git a/tensorflow/python/ops/control_flow_grad.py b/tensorflow/python/ops/control_flow_grad.py
index 21354b5ae8..45955554ca 100644
--- a/tensorflow/python/ops/control_flow_grad.py
+++ b/tensorflow/python/ops/control_flow_grad.py
@@ -142,6 +142,7 @@ def _ExitGrad(op, grad):
   """Gradients for an exit op are calculated using an Enter op."""
   graph = ops.get_default_graph()
   # pylint: disable=protected-access
+  op_ctxt = op._get_control_flow_context()
   grad_ctxt = graph._get_control_flow_context()
   # pylint: enable=protected-access
   if not grad_ctxt.back_prop:
@@ -150,10 +151,8 @@ def _ExitGrad(op, grad):
     # no gradient computation.
     return None
 
-  # pylint: disable=protected-access
-  if op._get_control_flow_context().grad_state:
+  if op_ctxt.grad_state:
     raise TypeError("Second-order gradient for while loops not supported.")
-  # pylint: enable=protected-access
 
   if isinstance(grad, ops.Tensor):
     grad_ctxt.AddName(grad.name)
-- 
GitLab


From 73937a7096908a9ae01dd7da2d76932a7fed194b Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 26 Mar 2018 11:38:15 -0700
Subject: [PATCH 489/960] Made the NumElements function more accurate

PiperOrigin-RevId: 190497916
---
 tensorflow/core/framework/shape_inference.cc | 16 +++++++++++++---
 1 file changed, 13 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/framework/shape_inference.cc b/tensorflow/core/framework/shape_inference.cc
index 641681973a..54ecaa5dd4 100644
--- a/tensorflow/core/framework/shape_inference.cc
+++ b/tensorflow/core/framework/shape_inference.cc
@@ -298,13 +298,23 @@ bool InferenceContext::FullyDefined(ShapeHandle s) {
 DimensionHandle InferenceContext::NumElements(ShapeHandle s) {
   const auto rank = Rank(s);
   if (rank == kUnknownRank) return UnknownDim();
+  bool found_unknown = false;
   int64 size = 1;
   for (int i = 0; i < rank; ++i) {
     int64 dim_val = Value(Dim(s, i));
-    if (dim_val == kUnknownDim) return UnknownDim();
-    size *= dim_val;
+    if (dim_val == kUnknownDim) {
+      found_unknown = true;
+    } else if (dim_val == 0) {
+      return MakeDim(0);
+    } else {
+      size *= dim_val;
+    }
+  }
+  if (found_unknown) {
+    return UnknownDim();
+  } else {
+    return MakeDim(size);
   }
-  return MakeDim(size);
 }
 
 string InferenceContext::DebugString(ShapeHandle s) {
-- 
GitLab


From 6d46c21e9f300d07e30a2185671f07d34fac3999 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 11:44:19 -0700
Subject: [PATCH 490/960] Make the CSE ("node deduping") pass in
 ArithmeticOptimizer more robust in the presence of ops that modify their
 inputs in-place: Do not dedup nodes if the underlying buffers for their
 outputs may be passed to an in-place op.

PiperOrigin-RevId: 190499037
---
 tensorflow/core/grappler/op_types.cc          | 13 ++++--
 tensorflow/core/grappler/op_types.h           |  4 ++
 .../optimizers/arithmetic_optimizer.cc        | 42 ++++++++++++++++---
 3 files changed, 50 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 259168bb33..1a6751befc 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -396,12 +396,17 @@ bool IsFreeOfSideEffect(const NodeDef& node) {
       return false;
     }
   }
+  return !ModifiesInputsInPlace(node);
+}
+
+bool ModifiesInputsInPlace(const NodeDef& node) {
   // Some nodes do in-place updates on regular tensor inputs.
-  if (GetBoolAttr(node, "in_place") || GetBoolAttr(node, "inplace") ||
-      StringPiece(op_name).starts_with("Inplace")) {
-    return false;
+  string op_name = node.op();
+  std::transform(op_name.begin(), op_name.end(), op_name.begin(), ::tolower);
+  if (StringPiece(op_name).contains("inplace")) {
+    return true;
   }
-  return true;
+  return GetBoolAttr(node, "in_place") || GetBoolAttr(node, "inplace");
 }
 
 bool ModifiesFrameInfo(const NodeDef& node) {
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 49e01f68e3..1ec1cd46e3 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -154,8 +154,12 @@ bool IsCommutative(const NodeDef& node);
 bool IsPersistent(const NodeDef& node);
 
 bool IsFreeOfSideEffect(const NodeDef& node);
+
 bool ModifiesFrameInfo(const NodeDef& node);
 
+// Returns true if the op is known to write to one or more of its inputs.
+bool ModifiesInputsInPlace(const NodeDef& node);
+
 // Returns true if the op is an element-wise involution, i.e. if it is its
 // own inverse such that f(f(x)) == x.
 bool IsInvolution(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index bc004df608..23e21855c8 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -1085,6 +1085,24 @@ bool ArithmeticOptimizer::OptimizedNodeExists(const NodeDef& node,
   return node_map_->NodeExists(OptimizedNodeName(node, suffix));
 }
 
+namespace {
+
+bool FeedsInPlaceOp(const SimpleGraphView& graph_view, const NodeDef& node) {
+  const std::unordered_set<string> op_types_to_traverse = {
+      node.op(), "Identity", "IdentityN", "Reshape"};
+  int node_idx = graph_view.index(node.name());
+  std::set<int> node_fanout;
+  graph_view.DepthFirstSearch(op_types_to_traverse, node_idx, &node_fanout);
+  for (int fanout : node_fanout) {
+    if (ModifiesInputsInPlace(graph_view.graph()->node(fanout))) {
+      return true;
+    }
+  }
+  return false;
+}
+
+}  // namespace
+
 bool ArithmeticOptimizer::CanDedup(const NodeDef& node) const {
   if (nodes_to_preserve_.find(node.name()) != nodes_to_preserve_.end()) {
     return false;
@@ -1104,6 +1122,11 @@ bool ArithmeticOptimizer::CanDedup(const NodeDef& node) const {
 
 void ArithmeticOptimizer::DedupComputations() {
   bool stop = true;
+  SimpleGraphView graph_view;
+  if (!graph_view.Initialize(*optimized_graph_).ok()) {
+    LOG(WARNING) << "Failed to build SimpleGraphView.";
+    return;
+  }
   std::set<int> duplicates;
   do {
     stop = true;
@@ -1120,19 +1143,28 @@ void ArithmeticOptimizer::DedupComputations() {
       if (rep == node) {
         continue;
       }
+      // If either node feeds an inplace op, deduping them may cause data races.
+      // For example: If we dedup nodes initializing two independent inplace
+      // accumulations, they will write to the same buffer, clobbering each
+      // other's results.
+      if (FeedsInPlaceOp(graph_view, *rep) ||
+          FeedsInPlaceOp(graph_view, *node)) {
+        continue;
+      }
       const std::set<NodeDef*>& fanouts = node_map_->GetOutputs(node->name());
       for (NodeDef* fanout : fanouts) {
-        for (string& name : *fanout->mutable_input()) {
+        for (int i = 0; i < fanout->input_size(); ++i) {
+          string* name = fanout->mutable_input(i);
           int position;
-          const string nodename = ParseNodeName(name, &position);
+          const string nodename = ParseNodeName(*name, &position);
           if (nodename == node->name()) {
             // Update name in-place.
             if (position > 0) {
-              name = StrCat(rep->name(), ":", position);
+              *name = StrCat(rep->name(), ":", position);
             } else if (position == 0) {
-              name = rep->name();
+              *name = rep->name();
             } else {
-              name = StrCat("^", rep->name());
+              *name = StrCat("^", rep->name());
             }
             node_map_->AddOutput(rep->name(), fanout->name());
           }
-- 
GitLab


From 3fbdba0c84941f34782a5e074b691916bca61a93 Mon Sep 17 00:00:00 2001
From: Sami Kama <skama@nvidia.com>
Date: Mon, 26 Mar 2018 11:49:03 -0700
Subject: [PATCH 491/960] update GPU installation instructions

---
 tensorflow/docs_src/install/install_linux.md | 14 ++++++++++++--
 1 file changed, 12 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 378946b459..3c5db9bced 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -33,7 +33,7 @@ must be installed on your system:
 
   * CUDA® Toolkit 9.0. For details, see
     [NVIDIA's documentation](http://docs.nvidia.com/cuda/cuda-installation-guide-linux/#axzz4VZnqTJ2A).
-    Ensure that you append the relevant Cuda pathnames to the
+    Ensure that you append the relevant CUDA pathnames to the
     `LD_LIBRARY_PATH` environment variable as described in the
     NVIDIA documentation.
   * The NVIDIA drivers associated with CUDA Toolkit 9.0.
@@ -56,7 +56,7 @@ must be installed on your system:
     and add its path to your `LD_LIBRARY_PATH` environment variable:
 
     <pre>
-    $ <b>export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/extras/CUPTI/lib64</b>
+    $ <b>export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}/usr/local/cuda/extras/CUPTI/lib64</b>
     </pre>
 
     For CUDA Toolkit <= 7.5 do:
@@ -64,6 +64,16 @@ must be installed on your system:
     <pre>
     $ <b>sudo apt-get install libcupti-dev</b>
     </pre>
+  * **[OPTIONAL]**  For optimized inferencing performance, you can also install
+    NVIDIA TensorRT 3.0. For details, see
+    [NVIDIA's TensorRT documentation](http://docs.nvidia.com/deeplearning/sdk/tensorrt-install-guide/index.html#installing-tar).
+    Only steps 1-4 in the TensorRT Tar File installation instructions are
+    required for compatibility with TensorFlow; the Python package installation
+    in steps 5 and 6 can be omitted. Detailed installation instructions can be found at [package documentataion](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/tensorrt#installing-tensorrt-304)
+
+    **IMPORTANT:** For compatibility with the pre-built `tensorflow-gpu`
+    package, please use the Ubuntu **14.04** tar file package of TensorRT
+    even when installing onto an Ubuntu 16.04 system.   
 
 If you have an earlier version of the preceding packages, please upgrade to
 the specified versions. If upgrading is not possible, then you may still run
-- 
GitLab


From d2604f8dcb8a63ca063f712c24ce5aa63403b0aa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 11:47:50 -0700
Subject: [PATCH 492/960] Revert to initializing number of threads when
 SetNumThreads is called. Requiring it to happen before OpInit() is way too
 confusing for users.

PiperOrigin-RevId: 190499644
---
 tensorflow/contrib/lite/BUILD                    | 2 +-
 tensorflow/contrib/lite/interpreter.cc           | 6 ++++++
 tensorflow/contrib/lite/kernels/conv.cc          | 5 ++---
 tensorflow/contrib/lite/kernels/eigen_support.cc | 7 +++++++
 tensorflow/contrib/lite/kernels/eigen_support.h  | 3 +++
 tensorflow/contrib/lite/kernels/gemm_support.cc  | 6 ++++++
 tensorflow/contrib/lite/kernels/gemm_support.h   | 3 +++
 7 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/BUILD b/tensorflow/contrib/lite/BUILD
index dafe6f136e..18efa64507 100644
--- a/tensorflow/contrib/lite/BUILD
+++ b/tensorflow/contrib/lite/BUILD
@@ -133,10 +133,10 @@ cc_library(
         ":schema_fbs_version",
         ":simple_memory_arena",
         ":util",
+        "//tensorflow/contrib/lite/kernels:eigen_support",
         "//tensorflow/contrib/lite/kernels:gemm_support",
         "//tensorflow/contrib/lite/nnapi:nnapi_lib",
         "//tensorflow/contrib/lite/schema:schema_fbs",
-        "//tensorflow/core:lib_platform",
     ],
 )
 
diff --git a/tensorflow/contrib/lite/interpreter.cc b/tensorflow/contrib/lite/interpreter.cc
index 937c185b0a..4575fe884d 100644
--- a/tensorflow/contrib/lite/interpreter.cc
+++ b/tensorflow/contrib/lite/interpreter.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/contrib/lite/context.h"
 #include "tensorflow/contrib/lite/error_reporter.h"
 #include "tensorflow/contrib/lite/graph_info.h"
+#include "tensorflow/contrib/lite/kernels/eigen_support.h"
 #include "tensorflow/contrib/lite/kernels/gemm_support.h"
 #include "tensorflow/contrib/lite/memory_planner.h"
 #include "tensorflow/contrib/lite/nnapi_delegate.h"
@@ -762,6 +763,11 @@ void Interpreter::UseNNAPI(bool enable) {
 
 void Interpreter::SetNumThreads(int num_threads) {
   context_.recommended_num_threads = num_threads;
+
+  // TODO(ahentz): find a way to avoid this. It causes gemmlowp and eigen to
+  // be required in order to compile the framework.
+  gemm_support::SetNumThreads(&context_, num_threads);
+  eigen_support::SetNumThreads(&context_, num_threads);
 }
 
 TfLiteStatus Interpreter::ModifyGraphWithDelegate(TfLiteDelegate* delegate,
diff --git a/tensorflow/contrib/lite/kernels/conv.cc b/tensorflow/contrib/lite/kernels/conv.cc
index e0cd12f1b4..18ff33bf9f 100644
--- a/tensorflow/contrib/lite/kernels/conv.cc
+++ b/tensorflow/contrib/lite/kernels/conv.cc
@@ -89,9 +89,6 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   auto* data = new OpData;
   gemm_support::IncrementUsageCounter(context);
   eigen_support::IncrementUsageCounter(context);
-
-  data->run_multithreaded_kernel = context->recommended_num_threads != 1;
-
   return data;
 }
 
@@ -176,6 +173,8 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   auto* params = reinterpret_cast<TfLiteConvParams*>(node->builtin_data);
   OpData* data = reinterpret_cast<OpData*>(node->user_data);
 
+  data->run_multithreaded_kernel = context->recommended_num_threads != 1;
+
   TF_LITE_ENSURE_STATUS(AllocateTemporaryTensorsIfRequired(context, node));
 
   bool hasBias = node->inputs->size == 3;
diff --git a/tensorflow/contrib/lite/kernels/eigen_support.cc b/tensorflow/contrib/lite/kernels/eigen_support.cc
index 213e465552..f1fdb42624 100644
--- a/tensorflow/contrib/lite/kernels/eigen_support.cc
+++ b/tensorflow/contrib/lite/kernels/eigen_support.cc
@@ -46,8 +46,15 @@ void DecrementUsageCounter(TfLiteContext* context) {
   }
   if (--ptr->num_references == 0) {
     delete ptr;
+    context->eigen_context = nullptr;
   }
 }
 
+void SetNumThreads(TfLiteContext* context, int num_threads) {
+  IncrementUsageCounter(context);
+  Eigen::setNbThreads(num_threads);
+  DecrementUsageCounter(context);
+}
+
 }  // namespace eigen_support
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/eigen_support.h b/tensorflow/contrib/lite/kernels/eigen_support.h
index d47e691123..aa8c351fd8 100644
--- a/tensorflow/contrib/lite/kernels/eigen_support.h
+++ b/tensorflow/contrib/lite/kernels/eigen_support.h
@@ -28,6 +28,9 @@ void IncrementUsageCounter(TfLiteContext* context);
 // usages all temporary Eigen objects will be deleted.
 void DecrementUsageCounter(TfLiteContext* context);
 
+// Set the number of threads that can be used by Eigen.
+void SetNumThreads(TfLiteContext* context, int num_threads);
+
 }  // namespace eigen_support
 }  // namespace tflite
 
diff --git a/tensorflow/contrib/lite/kernels/gemm_support.cc b/tensorflow/contrib/lite/kernels/gemm_support.cc
index 76a5165d14..95f45ea768 100644
--- a/tensorflow/contrib/lite/kernels/gemm_support.cc
+++ b/tensorflow/contrib/lite/kernels/gemm_support.cc
@@ -61,5 +61,11 @@ gemmlowp::GemmContext* GetFromContext(TfLiteContext* context) {
   return ptr->gemm_context_;
 }
 
+void SetNumThreads(TfLiteContext* context, int num_threads) {
+  IncrementUsageCounter(context);
+  GetFromContext(context)->set_max_num_threads(num_threads);
+  DecrementUsageCounter(context);
+}
+
 }  // namespace gemm_support
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/gemm_support.h b/tensorflow/contrib/lite/kernels/gemm_support.h
index 37af772c68..f033501cb6 100644
--- a/tensorflow/contrib/lite/kernels/gemm_support.h
+++ b/tensorflow/contrib/lite/kernels/gemm_support.h
@@ -45,6 +45,9 @@ void IncrementUsageCounter(TfLiteContext* context);
 // 'context'. If there are no more usages the GemmContext will be deleted.
 void DecrementUsageCounter(TfLiteContext* context);
 
+// Set the number of threads that can be used by gemmlowp.
+void SetNumThreads(TfLiteContext* context, int num_threads);
+
 }  // namespace gemm_support
 }  // namespace tflite
 
-- 
GitLab


From f9cfb9e917c8937152b248c300b095798d79501a Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Mon, 26 Mar 2018 11:50:19 -0700
Subject: [PATCH 493/960] Extended experimental C API with MNIST
 dataset/iterators support.

PiperOrigin-RevId: 190500020
---
 tensorflow/c/c_api_experimental.cc      | 1151 ++++++++++++++++++++++-
 tensorflow/c/c_api_experimental.h       |   11 +-
 tensorflow/c/c_api_experimental_test.cc |    4 +-
 3 files changed, 1149 insertions(+), 17 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index 1c809cb21e..f411efc941 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -138,7 +138,7 @@ static std::vector<UniqueFuncPtr> CreateFunctionsFromTextProto(
     return {};
   }
   std::vector<UniqueFuncPtr> ret;
-  for (const auto& fdef : fdef_lib.function()) {
+  for (const FunctionDef& fdef : fdef_lib.function()) {
     // Make a copy so that we can mutate it.
     FunctionDef fdef_to_load = fdef;
     if (mutate_proto_func) {
@@ -148,8 +148,8 @@ static std::vector<UniqueFuncPtr> CreateFunctionsFromTextProto(
     std::vector<char> binary_proto_buf(fdef_to_load.ByteSizeLong());
     fdef_to_load.SerializeToArray(binary_proto_buf.data(),
                                   binary_proto_buf.size());
-    auto func = TF_FunctionImportFunctionDef(binary_proto_buf.data(),
-                                             binary_proto_buf.size(), status);
+    TF_Function* func = TF_FunctionImportFunctionDef(
+        binary_proto_buf.data(), binary_proto_buf.size(), status);
     if (!status->status.ok()) return {};
     ret.push_back(UniqueFuncPtr(func, TF_DeleteFunction));
   }
@@ -7120,6 +7120,1130 @@ library {
   return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status);
 }
 
+//  On success, returns a set of TF_Function instances encoding a dataset
+//  node stack that reads an MNIST file dataset from `file_path`, and
+//  sets `dataset_name` to the created dataset name. The returned functions must
+//  be deleted by calling TF_DeleteFunction.
+static std::vector<UniqueFuncPtr> CreateMNISTDatasetFunctions(
+    const char* file_path, std::string* dataset_name, TF_Status* status) {
+  const char* func_def = R"PREFIX(
+library {
+  function {
+    signature {
+      name: "tf_map_func_521bfd08"
+      input_arg {
+        name: "arg0"
+        type: DT_STRING
+      }
+      output_arg {
+        name: "truediv"
+        type: DT_FLOAT
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+    }
+    node_def {
+      name: "DecodeRaw"
+      op: "DecodeRaw"
+      input: "arg0"
+      attr {
+        key: "little_endian"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "Cast"
+      op: "Cast"
+      input: "DecodeRaw:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "Reshape/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 784
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape"
+      op: "Reshape"
+      input: "Cast:y:0"
+      input: "Reshape/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "truediv/y"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_FLOAT
+            tensor_shape {
+            }
+            float_val: 255.0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "truediv"
+      op: "RealDiv"
+      input: "Reshape:output:0"
+      input: "truediv/y:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+    }
+    ret {
+      key: "truediv"
+      value: "truediv:z:0"
+    }
+  }
+  function {
+    signature {
+      name: "tf_map_func_9a08860d"
+      input_arg {
+        name: "arg0"
+        type: DT_STRING
+      }
+      output_arg {
+        name: "ToInt32"
+        type: DT_INT32
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+    }
+    node_def {
+      name: "DecodeRaw"
+      op: "DecodeRaw"
+      input: "arg0"
+      attr {
+        key: "little_endian"
+        value {
+          b: true
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    node_def {
+      name: "Reshape/shape"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+              }
+            }
+          }
+        }
+      }
+    }
+    node_def {
+      name: "Reshape"
+      op: "Reshape"
+      input: "DecodeRaw:output:0"
+      input: "Reshape/shape:output:0"
+      attr {
+        key: "T"
+        value {
+          type: DT_UINT8
+        }
+      }
+      attr {
+        key: "Tshape"
+        value {
+          type: DT_INT32
+        }
+      }
+    }
+    node_def {
+      name: "ToInt32"
+      op: "Cast"
+      input: "Reshape:output:0"
+      attr {
+        key: "DstT"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "SrcT"
+        value {
+          type: DT_UINT8
+        }
+      }
+    }
+    ret {
+      key: "ToInt32"
+      value: "ToInt32:y:0"
+    }
+  }
+  function {
+    signature {
+      name: "tf_predicate_7089b845"
+      input_arg {
+        name: "arg0"
+        type: DT_FLOAT
+      }
+      input_arg {
+        name: "arg1"
+        type: DT_INT32
+      }
+      input_arg {
+        name: "Equal/Placeholder"
+        type: DT_INT64
+      }
+      output_arg {
+        name: "Equal"
+        type: DT_BOOL
+      }
+      description: "A wrapper for Defun that facilitates shape inference."
+    }
+    node_def {
+      name: "Shape"
+      op: "Shape"
+      input: "arg0"
+      attr {
+        key: "T"
+        value {
+          type: DT_FLOAT
+        }
+      }
+      attr {
+        key: "out_type"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice/stack_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT32
+            tensor_shape {
+              dim {
+                size: 1
+              }
+            }
+            int_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "strided_slice"
+      op: "StridedSlice"
+      input: "Shape:output:0"
+      input: "strided_slice/stack:output:0"
+      input: "strided_slice/stack_1:output:0"
+      input: "strided_slice/stack_2:output:0"
+      attr {
+        key: "Index"
+        value {
+          type: DT_INT32
+        }
+      }
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "begin_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "ellipsis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "end_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "new_axis_mask"
+        value {
+          i: 0
+        }
+      }
+      attr {
+        key: "shrink_axis_mask"
+        value {
+          i: 1
+        }
+      }
+    }
+    node_def {
+      name: "Equal"
+      op: "Equal"
+      input: "strided_slice:output:0"
+      input: "Equal/Placeholder"
+      attr {
+        key: "T"
+        value {
+          type: DT_INT64
+        }
+      }
+    }
+    ret {
+      key: "Equal"
+      value: "Equal:z:0"
+    }
+  }
+  function {
+    signature {
+      name: "_make_dataset_2451e43a"
+      output_arg {
+        name: "FilterDataset"
+        type: DT_VARIANT
+      }
+      is_stateful: true
+    }
+    node_def {
+      name: "FixedLengthRecordDataset/filenames"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "$(DATA_DIR)/train-images-idx3-ubyte"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset/header_bytes"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 16
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset/record_bytes"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 784
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset/footer_bytes"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset/buffer_size"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 262144
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset"
+      op: "FixedLengthRecordDataset"
+      input: "FixedLengthRecordDataset/filenames:output:0"
+      input: "FixedLengthRecordDataset/header_bytes:output:0"
+      input: "FixedLengthRecordDataset/record_bytes:output:0"
+      input: "FixedLengthRecordDataset/footer_bytes:output:0"
+      input: "FixedLengthRecordDataset/buffer_size:output:0"
+    }
+    node_def {
+      name: "MapDataset"
+      op: "MapDataset"
+      input: "FixedLengthRecordDataset:handle:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+          }
+        }
+      }
+      attr {
+        key: "f"
+        value {
+          func {
+            name: "tf_map_func_521bfd08"
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1/filenames_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: "$(DATA_DIR)/train-labels-idx1-ubyte"
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1/header_bytes_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 8
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1/record_bytes_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1/footer_bytes_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1/buffer_size_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 262144
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FixedLengthRecordDataset_1"
+      op: "FixedLengthRecordDataset"
+      input: "FixedLengthRecordDataset_1/filenames_1:output:0"
+      input: "FixedLengthRecordDataset_1/header_bytes_1:output:0"
+      input: "FixedLengthRecordDataset_1/record_bytes_1:output:0"
+      input: "FixedLengthRecordDataset_1/footer_bytes_1:output:0"
+      input: "FixedLengthRecordDataset_1/buffer_size_1:output:0"
+    }
+    node_def {
+      name: "MapDataset_1"
+      op: "MapDataset"
+      input: "FixedLengthRecordDataset_1:handle:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+          }
+        }
+      }
+      attr {
+        key: "f"
+        value {
+          func {
+            name: "tf_map_func_9a08860d"
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ZipDataset"
+      op: "ZipDataset"
+      input: "MapDataset:handle:0"
+      input: "MapDataset_1:handle:0"
+      attr {
+        key: "N"
+        value {
+          i: 2
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "CacheDataset/filename"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_STRING
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_STRING
+            tensor_shape {
+            }
+            string_val: ""
+          }
+        }
+      }
+    }
+    node_def {
+      name: "CacheDataset"
+      op: "CacheDataset"
+      input: "ZipDataset:handle:0"
+      input: "CacheDataset/filename:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "RepeatDataset/count"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: -1
+          }
+        }
+      }
+    }
+    node_def {
+      name: "RepeatDataset"
+      op: "RepeatDataset"
+      input: "CacheDataset:handle:0"
+      input: "RepeatDataset/count:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/buffer_size_2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 50000
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/seed"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset/seed2"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 0
+          }
+        }
+      }
+    }
+    node_def {
+      name: "ShuffleDataset"
+      op: "ShuffleDataset"
+      input: "RepeatDataset:handle:0"
+      input: "ShuffleDataset/buffer_size_2:output:0"
+      input: "ShuffleDataset/seed:output:0"
+      input: "ShuffleDataset/seed2:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: 784
+              }
+            }
+            shape {
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+      attr {
+        key: "reshuffle_each_iteration"
+        value {
+          b: true
+        }
+      }
+    }
+    node_def {
+      name: "BatchDataset/batch_size"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 128
+          }
+        }
+      }
+    }
+    node_def {
+      name: "BatchDataset"
+      op: "BatchDataset"
+      input: "ShuffleDataset:handle:0"
+      input: "BatchDataset/batch_size:output:0"
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 784
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FilterDataset/batch_size_1"
+      op: "Const"
+      attr {
+        key: "dtype"
+        value {
+          type: DT_INT64
+        }
+      }
+      attr {
+        key: "value"
+        value {
+          tensor {
+            dtype: DT_INT64
+            tensor_shape {
+            }
+            int64_val: 128
+          }
+        }
+      }
+    }
+    node_def {
+      name: "FilterDataset"
+      op: "FilterDataset"
+      input: "BatchDataset:handle:0"
+      input: "FilterDataset/batch_size_1:output:0"
+      attr {
+        key: "Targuments"
+        value {
+          list {
+            type: DT_INT64
+          }
+        }
+      }
+      attr {
+        key: "output_shapes"
+        value {
+          list {
+            shape {
+              dim {
+                size: -1
+              }
+              dim {
+                size: 784
+              }
+            }
+            shape {
+              dim {
+                size: -1
+              }
+            }
+          }
+        }
+      }
+      attr {
+        key: "output_types"
+        value {
+          list {
+            type: DT_FLOAT
+            type: DT_INT32
+          }
+        }
+      }
+      attr {
+        key: "predicate"
+        value {
+          func {
+            name: "tf_predicate_7089b845"
+          }
+        }
+      }
+    }
+    ret {
+      key: "FilterDataset"
+      value: "FilterDataset:handle:0"
+    }
+  }
+}
+)PREFIX";
+
+  *dataset_name = "_make_dataset_2451e43a";
+  std::function<void(FunctionDef*)> mutate_proto_func =
+      [dataset_name, file_path](FunctionDef* fdef) {
+        VLOG(1) << "Processsing function " << fdef->DebugString();
+        if (std::string(fdef->signature().name()) != *dataset_name) return;
+        // Change the input file pattern to `file_path`.
+        bool found = false;
+        // `node_def` may be mutated.
+        for (auto& node_def : *fdef->mutable_node_def()) {
+          if (node_def.name() != "FixedLengthRecordDataset/filenames" &&
+              node_def.name() != "FixedLengthRecordDataset_1/filenames_1")
+            continue;
+          DCHECK_EQ(node_def.op(), "Const");
+          DCHECK_GT(node_def.attr().count("value"), 0);
+          found = true;
+          // Replace $(DATA_DIR)/foo with <file_path>/foo
+          // TODO(hongm): Use StringPiece manipulation for better efficiency.
+          const std::string cur_value =
+              node_def.attr().at("value").tensor().string_val(0);
+          const std::string pattern = "$(DATA_DIR)";
+          DCHECK_EQ(cur_value.compare(0, pattern.length(), pattern), 0);
+          const std::string new_value =
+              file_path + cur_value.substr(pattern.length());
+          VLOG(1) << "Setting the value of node_def " << node_def.name()
+                  << " to " << new_value;
+          auto* tensor = (*node_def.mutable_attr())["value"].mutable_tensor();
+          tensor->clear_string_val();
+          tensor->add_string_val(new_value);
+        }
+        VLOG(1) << "Rewrote function to " << fdef->DebugString();
+        DCHECK(found);
+      };
+  return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status);
+}
+
 // Adds the input functions to `graph`.  On success, returns the created
 // IteratorGetNext node.
 static TF_Operation* AddDatasetFunctionAndIteratorNodesToGraph(
@@ -7209,15 +8333,16 @@ TF_Operation* TF_MakeFakeIteratorGetNextWithDatasets(TF_Graph* graph,
   return getnext_node;
 }
 
-TF_Operation* TF_MakeImagenetIteratorGetNextWithDatasets(TF_Graph* graph,
-                                                         const char* file_path,
-                                                         int batch_size,
-                                                         TF_Status* status) {
+TF_Operation* TF_MakeFileBasedIteratorGetNextWithDatasets(
+    TF_Graph* graph, const char* file_path, int batch_size,
+    unsigned char is_mnist, TF_Status* status) {
   tensorflow::Status s;
 
   std::string dataset_name;
   const auto& funcs =
-      CreateImagenetDatasetFunctions(file_path, &dataset_name, status);
+      is_mnist
+          ? CreateMNISTDatasetFunctions(file_path, &dataset_name, status)
+          : CreateImagenetDatasetFunctions(file_path, &dataset_name, status);
   if (!status->status.ok()) {
     return nullptr;
   }
@@ -7226,9 +8351,13 @@ TF_Operation* TF_MakeImagenetIteratorGetNextWithDatasets(TF_Graph* graph,
   // batch_size X 224 X 224 X 3
   auto image_shape = tensorflow::TensorShapeProto();
   image_shape.add_dim()->set_size(batch_size);
-  image_shape.add_dim()->set_size(224);
-  image_shape.add_dim()->set_size(224);
-  image_shape.add_dim()->set_size(3);
+  if (is_mnist) {
+    image_shape.add_dim()->set_size(784);
+  } else {
+    image_shape.add_dim()->set_size(224);
+    image_shape.add_dim()->set_size(224);
+    image_shape.add_dim()->set_size(3);
+  }
   output_shape_list.push_back(image_shape);
 
   // batch_size
diff --git a/tensorflow/c/c_api_experimental.h b/tensorflow/c/c_api_experimental.h
index a9c551d73e..ebcec8176b 100644
--- a/tensorflow/c/c_api_experimental.h
+++ b/tensorflow/c/c_api_experimental.h
@@ -96,13 +96,16 @@ TF_CAPI_EXPORT extern TF_Operation* TF_MakeFakeIteratorGetNextWithDatasets(
     TF_Graph* graph, TF_Status* status);
 
 // Similar to the above API, except that the returned iterator reads the
-// TFRecord files from `file_path`.
+// file based dataset from `file_path`.
+// If `is_mnist` is 0, the dataset corresponds to ImageNet.
 // The iterators outputs 2 tensors:
-// - A float tensor of shape `batch_size` X 224 X 224 X 3
+// - A float tensor of shape `batch_size` X 784 when `is_mnist` is non-zero, or
+// `batch_size` X 224 X 224 X 3 otherwise.
 // - An int32 tensor of shape `batch_size`
 // TODO(hongm): Extend the API to allow customization of the nodes created.
-TF_CAPI_EXPORT extern TF_Operation* TF_MakeImagenetIteratorGetNextWithDatasets(
-    TF_Graph* graph, const char* file_path, int batch_size, TF_Status* status);
+TF_CAPI_EXPORT extern TF_Operation* TF_MakeFileBasedIteratorGetNextWithDatasets(
+    TF_Graph* graph, const char* file_path, int batch_size,
+    unsigned char is_mnist, TF_Status* status);
 
 #ifdef __cplusplus
 } /* end extern "C" */
diff --git a/tensorflow/c/c_api_experimental_test.cc b/tensorflow/c/c_api_experimental_test.cc
index 49d64d18bf..30fcfd401d 100644
--- a/tensorflow/c/c_api_experimental_test.cc
+++ b/tensorflow/c/c_api_experimental_test.cc
@@ -68,8 +68,8 @@ TEST(CAPI_EXPERIMENTAL, ImagenetIteratorGetNext) {
       tensorflow::testing::TensorFlowSrcRoot(), "c/testdata/tf_record");
   VLOG(1) << "data file path is " << file_path;
   const int batch_size = 64;
-  TF_Operation* get_next = TF_MakeImagenetIteratorGetNextWithDatasets(
-      graph, file_path.c_str(), batch_size, s);
+  TF_Operation* get_next = TF_MakeFileBasedIteratorGetNextWithDatasets(
+      graph, file_path.c_str(), batch_size, /*is_mnist*/ false, s);
   ASSERT_EQ(TF_OK, TF_GetCode(s)) << TF_Message(s);
 
   CSession csession(graph, s);
-- 
GitLab


From af0fe569f48f3d5e8405eab76e14abde3c4e3d36 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 12:14:05 -0700
Subject: [PATCH 494/960] LSTM support: Support fused activation functions in
 int16 Add ops.

PiperOrigin-RevId: 190503823
---
 .../kernels/internal/optimized/optimized_ops.h  | 17 +++++++++++++++--
 .../kernels/internal/reference/reference_ops.h  | 17 +++++++++++++++--
 2 files changed, 30 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index f7840258ec..d7a0005f27 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -1660,11 +1660,21 @@ template <FusedActivationFunctionType Ac>
 inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
                 int input1_shift, const int16* input2_data,
                 const Dims<4>& input2_dims, int input2_shift,
+                int16 output_activation_min, int16 output_activation_max,
                 int16* output_data, const Dims<4>& output_dims) {
   gemmlowp::ScopedProfilingLabel label("Add/Int16");
   // This is a copy of the reference implementation. We do not currently have a
   // properly optimized version.
-  static_assert(Ac == FusedActivationFunctionType::kNone, "");
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, -32768);
+    TFLITE_DCHECK_EQ(output_activation_max, 32767);
+  }
 
   const int flat_size = RequiredBufferSizeForDims(output_dims);
   TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input1_dims), flat_size);
@@ -1685,7 +1695,10 @@ inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
     F0 scaled_input =
         F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_shift));
     F0 result = gemmlowp::SaturatingAdd(scaled_input, input_ready_scaled);
-    output_data[i] = result.raw();
+    const int16 raw_output = result.raw();
+    const int16 clamped_output = std::min(
+        output_activation_max, std::max(output_activation_min, raw_output));
+    output_data[i] = clamped_output;
   }
 }
 
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 472ddc60df..ce12fad95d 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -956,8 +956,18 @@ template <FusedActivationFunctionType Ac>
 inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
                 int input1_shift, const int16* input2_data,
                 const Dims<4>& input2_dims, int input2_shift,
+                int16 output_activation_min, int16 output_activation_max,
                 int16* output_data, const Dims<4>& output_dims) {
-  static_assert(Ac == FusedActivationFunctionType::kNone, "");
+  static_assert(Ac == FusedActivationFunctionType::kNone ||
+                    Ac == FusedActivationFunctionType::kRelu ||
+                    Ac == FusedActivationFunctionType::kRelu6 ||
+                    Ac == FusedActivationFunctionType::kRelu1,
+                "");
+  TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
+  if (Ac == FusedActivationFunctionType::kNone) {
+    TFLITE_DCHECK_EQ(output_activation_min, -32768);
+    TFLITE_DCHECK_EQ(output_activation_max, 32767);
+  }
 
   const int flat_size = RequiredBufferSizeForDims(output_dims);
   TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input1_dims), flat_size);
@@ -978,7 +988,10 @@ inline void Add(const int16* input1_data, const Dims<4>& input1_dims,
     F0 scaled_input =
         F0::FromRaw(gemmlowp::RoundingDivideByPOT(shift_input[i], input_shift));
     F0 result = gemmlowp::SaturatingAdd(scaled_input, input_ready_scaled);
-    output_data[i] = result.raw();
+    const int16 raw_output = result.raw();
+    const int16 clamped_output = std::min(
+        output_activation_max, std::max(output_activation_min, raw_output));
+    output_data[i] = clamped_output;
   }
 }
 
-- 
GitLab


From 04b1e736897505ccf5b483379289d02a274ea586 Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Mon, 26 Mar 2018 12:16:34 -0700
Subject: [PATCH 495/960] tfdbg CLI: Allow node exclusion with tensor filters

Fixes: #16619

See the referred GitHub issue for details, but users want to be
able to skip certain nodes when searching for inf/nans, because
some nodes generate inf/nans even in nominal conditions.

This CL adds a new optional flag `--filter_exclude_node_names`
(or `-fenn` for short), which allows users to do exactly that,
by using a regex for node names.

RELNOTES: tfdbg CLI: Allow exclusion of nodes by regular expressions
during tensor filter-enabled Session runs: see the new flags
`--filter_exclude_node_names` (or `-fenn` for short).
PiperOrigin-RevId: 190504225
---
 .../docs_src/programmers_guide/debugger.md    | 16 ++++++
 tensorflow/python/debug/cli/analyzer_cli.py   | 22 ++++++++-
 .../python/debug/cli/analyzer_cli_test.py     | 26 ++++++++++
 tensorflow/python/debug/lib/debug_data.py     | 14 +++++-
 .../python/debug/lib/session_debug_testlib.py | 49 +++++++++++++++++++
 .../debug/wrappers/local_cli_wrapper.py       | 39 +++++++++++++--
 .../debug/wrappers/local_cli_wrapper_test.py  | 36 +++++++++++++-
 7 files changed, 196 insertions(+), 6 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index d1399814ee..d1cd7e7c06 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -155,6 +155,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | | `-n <name_pattern>` | List dumped tensors with names matching given regular-expression pattern. | `lt -n Softmax.*` |
 | | `-t <op_pattern>` | List dumped tensors with op types matching given regular-expression pattern. | `lt -t MatMul` |
 | | `-f <filter_name>` | List only the tensors that pass a registered tensor filter. | `lt -f has_inf_or_nan` |
+| | `-f <filter_name> -fenn <regex>` | List only the tensors that pass a registered tensor filter, excluding nodes with names matching the regular expression. | `lt -f has_inf_or_nan` `-fenn .*Sqrt.*` |
 | | `-s <sort_key>` | Sort the output by given `sort_key`, whose possible values are `timestamp` (default), `dump_size`, `op_type` and `tensor_name`. | `lt -s dump_size` |
 | | `-r` | Sort in reverse order. | `lt -r -s dump_size` |
 | **`pt`** | | **Print value of a dumped tensor.** | |
@@ -200,6 +201,7 @@ Try the following commands at the `tfdbg>` prompt (referencing the code at
 | | `-n` | Execute through the next `Session.run` without debugging, and drop to CLI right before the run after that. | `run -n` |
 | | `-t <T>` | Execute `Session.run` `T - 1` times without debugging, followed by a run with debugging. Then drop to CLI right after the debugged run. | `run -t 10` |
 | | `-f <filter_name>` | Continue executing `Session.run` until any intermediate tensor triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan` |
+| | `-f <filter_name> -fenn <regex>` | Continue executing `Session.run` until any intermediate tensor whose node names doesn't match the regular expression triggers the specified Tensor filter (causes the filter to return `True`). | `run -f has_inf_or_nan -fenn .*Sqrt.*` |
 | | `--node_name_filter <pattern>` | Execute the next `Session.run`, watching only nodes with names matching the given regular-expression pattern. | `run --node_name_filter Softmax.*` |
 | | `--op_type_filter <pattern>` | Execute the next `Session.run`, watching only nodes with op types matching the given regular-expression pattern. | `run --op_type_filter Variable.*` |
 | | `--tensor_dtype_filter <pattern>` | Execute the next `Session.run`, dumping only Tensors with data types (`dtype`s) matching the given regular-expression pattern. | `run --tensor_dtype_filter int.*` |
@@ -813,6 +815,20 @@ sess.run(b)
 the constant-folding would not occur and `tfdbg` should show the intermediate
 tensor dumps.
 
+
+**Q**: I am debugging a model that generates unwanted infinities or NaNs. But
+       there are some nodes in my model that are known to generate infinities
+       or NaNs in their output tensors even under completely normal conditions.
+       How can I skip those nodes during my `run -f has_inf_or_nan` actions?
+
+**A**: Use the `--filter_exclude_node_names` (`-fenn` for short) flag. For
+       example, if you known you have a node with name matching the regular
+       expression `.*Sqrt.*` that generates infinities or NaNs regardless
+       of whether the model is behaving correctly, you can exclude the nodes
+       from the infinity/NaN-finding runs with the command
+       `run -f has_inf_or_nan -fenn .*Sqrt.*`.
+
+
 **Q**: Is there a GUI for tfdbg?
 
 **A**: Yes, the **TensorBoard Debugger Plugin** is the GUI of tfdbg.
diff --git a/tensorflow/python/debug/cli/analyzer_cli.py b/tensorflow/python/debug/cli/analyzer_cli.py
index 156afdfd4c..9a47cd12b4 100644
--- a/tensorflow/python/debug/cli/analyzer_cli.py
+++ b/tensorflow/python/debug/cli/analyzer_cli.py
@@ -185,6 +185,15 @@ class DebugAnalyzer(object):
         type=str,
         default="",
         help="List only Tensors passing the filter of the specified name")
+    ap.add_argument(
+        "-fenn",
+        "--filter_exclude_node_names",
+        dest="filter_exclude_node_names",
+        type=str,
+        default="",
+        help="When applying the tensor filter, exclude node with names "
+        "matching the regular expression. Applicable only if --tensor_filter "
+        "or -f is used.")
     ap.add_argument(
         "-n",
         "--node_name_filter",
@@ -484,6 +493,10 @@ class DebugAnalyzer(object):
 
     Returns:
       Output text lines as a RichTextLines object.
+
+    Raises:
+      ValueError: If `--filter_exclude_node_names` is used without `-f` or
+        `--tensor_filter` being used.
     """
 
     # TODO(cais): Add annotations of substrings for dumped tensor names, to
@@ -520,8 +533,15 @@ class DebugAnalyzer(object):
         _add_main_menu(output, node_name=None, enable_list_tensors=False)
         return output
 
-      data_to_show = self._debug_dump.find(filter_callable)
+      data_to_show = self._debug_dump.find(
+          filter_callable,
+          exclude_node_names=parsed.filter_exclude_node_names)
     else:
+      if parsed.filter_exclude_node_names:
+        raise ValueError(
+            "The flag --filter_exclude_node_names is valid only when "
+            "the flag -f or --tensor_filter is used.")
+
       data_to_show = self._debug_dump.dumped_tensor_data
 
     # TODO(cais): Implement filter by lambda on tensor value.
diff --git a/tensorflow/python/debug/cli/analyzer_cli_test.py b/tensorflow/python/debug/cli/analyzer_cli_test.py
index 6b110fda9e..55231954d1 100644
--- a/tensorflow/python/debug/cli/analyzer_cli_test.py
+++ b/tensorflow/python/debug/cli/analyzer_cli_test.py
@@ -820,6 +820,32 @@ class AnalyzerCLISimpleMulAddTest(test_util.TensorFlowTestCase):
         op_type_regex="(Add|MatMul)")
     check_main_menu(self, out, list_tensors_enabled=False)
 
+  def testListTensorWithFilterAndNodeNameExclusionWorks(self):
+    # First, create and register the filter.
+    def is_2x1_vector(datum, tensor):
+      del datum  # Unused.
+      return list(tensor.shape) == [2, 1]
+    self._analyzer.add_tensor_filter("is_2x1_vector", is_2x1_vector)
+
+    # Use shorthand alias for the command prefix.
+    out = self._registry.dispatch_command(
+        "lt", ["-f", "is_2x1_vector", "--filter_exclude_node_names", ".*v.*"])
+
+    # If the --filter_exclude_node_names were not used, then the matching
+    # tensors would be:
+    #   - simple_mul_add/v:0
+    #   - simple_mul_add/v/read:0
+    #   - simple_mul_add/matmul:0
+    #   - simple_mul_add/add:0
+    #
+    # With the --filter_exclude_node_names option, only the last two should
+    # show up in the result.
+    assert_listed_tensors(
+        self,
+        out, ["simple_mul_add/matmul:0", "simple_mul_add/add:0"],
+        ["MatMul", "Add"], tensor_filter_name="is_2x1_vector")
+    check_main_menu(self, out, list_tensors_enabled=False)
+
   def testListTensorsFilterNanOrInf(self):
     """Test register and invoke a tensor filter."""
 
diff --git a/tensorflow/python/debug/lib/debug_data.py b/tensorflow/python/debug/lib/debug_data.py
index 8d355aa27f..8a65ad087b 100644
--- a/tensorflow/python/debug/lib/debug_data.py
+++ b/tensorflow/python/debug/lib/debug_data.py
@@ -23,6 +23,7 @@ import glob
 import json
 import os
 import platform
+import re
 
 import numpy as np
 import six
@@ -1411,7 +1412,11 @@ class DebugDumpDir(object):
 
     return self._watch_key_to_datum[device_name].get(debug_watch_key, [])
 
-  def find(self, predicate, first_n=0, device_name=None):
+  def find(self,
+           predicate,
+           first_n=0,
+           device_name=None,
+           exclude_node_names=None):
     """Find dumped tensor data by a certain predicate.
 
     Args:
@@ -1430,17 +1435,24 @@ class DebugDumpDir(object):
         time order) for which the predicate returns True. To return all the
         `DebugTensotDatum` instances, let first_n be <= 0.
       device_name: optional device name.
+      exclude_node_names: Optional regular expression to exclude nodes with
+        names matching the regular expression.
 
     Returns:
       A list of all `DebugTensorDatum` objects in this `DebugDumpDir` object
        for which predicate returns True, sorted in ascending order of the
        timestamp.
     """
+    if exclude_node_names:
+      exclude_node_names = re.compile(exclude_node_names)
 
     matched_data = []
     for device in (self._dump_tensor_data if device_name is None
                    else (self._dump_tensor_data[device_name],)):
       for datum in self._dump_tensor_data[device]:
+        if exclude_node_names and exclude_node_names.match(datum.node_name):
+          continue
+
         if predicate(datum, datum.get_tensor()):
           matched_data.append(datum)
 
diff --git a/tensorflow/python/debug/lib/session_debug_testlib.py b/tensorflow/python/debug/lib/session_debug_testlib.py
index f4fac14019..070d9c4cd7 100644
--- a/tensorflow/python/debug/lib/session_debug_testlib.py
+++ b/tensorflow/python/debug/lib/session_debug_testlib.py
@@ -669,6 +669,55 @@ class SessionDebugTestBase(test_util.TensorFlowTestCase):
       self.assertEqual(1, len(first_bad_datum))
       self.assertEqual(x_name, first_bad_datum[0].node_name)
 
+  def testFindInfOrNanWithOpNameExclusion(self):
+    with session.Session() as sess:
+      u_name = "testFindInfOrNanWithOpNameExclusion/u"
+      v_name = "testFindInfOrNanWithOpNameExclusion/v"
+      w_name = "testFindInfOrNanWithOpNameExclusion/w"
+      x_name = "testFindInfOrNanWithOpNameExclusion/x"
+      y_name = "testFindInfOrNanWithOpNameExclusion/y"
+      z_name = "testFindInfOrNanWithOpNameExclusion/z"
+
+      u_init = constant_op.constant([2.0, 4.0])
+      u = variables.Variable(u_init, name=u_name)
+      v_init = constant_op.constant([2.0, 1.0])
+      v = variables.Variable(v_init, name=v_name)
+
+      # Expected output: [0.0, 3.0]
+      w = math_ops.subtract(u, v, name=w_name)
+
+      # Expected output: [inf, 1.3333]
+      x = math_ops.div(u, w, name=x_name)
+
+      # Expected output: [nan, 4.0]
+      y = math_ops.multiply(w, x, name=y_name)
+
+      z = math_ops.multiply(y, y, name=z_name)
+
+      u.initializer.run()
+      v.initializer.run()
+
+      _, dump = self._debug_run_and_get_dump(
+          sess, z,
+          expected_partition_graph_count=self._expected_partition_graph_count)
+
+      # Find all "offending tensors".
+      bad_data = dump.find(debug_data.has_inf_or_nan,
+                           exclude_node_names=".*/x$")
+
+      # Verify that the nodes with bad values are caught through running find
+      # on the debug dump.
+      self.assertEqual(2, len(bad_data))
+      # Assert that the node `x` should have been excluded.
+      self.assertEqual(y_name, bad_data[0].node_name)
+      self.assertEqual(z_name, bad_data[1].node_name)
+
+      first_bad_datum = dump.find(
+          debug_data.has_inf_or_nan, first_n=1, exclude_node_names=".*/x$")
+
+      self.assertEqual(1, len(first_bad_datum))
+      self.assertEqual(y_name, first_bad_datum[0].node_name)
+
   def _session_run_for_graph_structure_lookup(self):
     with session.Session(config=no_rewrite_session_config()) as sess:
       u_name = "testDumpGraphStructureLookup/u"
diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper.py b/tensorflow/python/debug/wrappers/local_cli_wrapper.py
index 1465cb7295..c8625655e5 100644
--- a/tensorflow/python/debug/wrappers/local_cli_wrapper.py
+++ b/tensorflow/python/debug/wrappers/local_cli_wrapper.py
@@ -115,6 +115,7 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
     #   unavailable (i.e., is None), the run-start CLI will be launched to ask
     #   the user. This is the case, e.g., right before the first run starts.
     self._active_tensor_filter = None
+    self._active_filter_exclude_node_names = None
     self._active_tensor_filter_run_start_response = None
     self._run_through_times = 1
     self._skip_debug = False
@@ -148,6 +149,15 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
         type=str,
         default="",
         help="Run until a tensor in the graph passes the specified filter.")
+    ap.add_argument(
+        "-fenn",
+        "--filter_exclude_node_names",
+        dest="filter_exclude_node_names",
+        type=str,
+        default="",
+        help="When applying the tensor filter, exclude node with names "
+        "matching the regular expression. Applicable only if --tensor_filter "
+        "or -f is used.")
     ap.add_argument(
         "--node_name_filter",
         dest="node_name_filter",
@@ -324,9 +334,11 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
       debug_dump.set_python_graph(self._sess.graph)
 
       passed_filter = None
+      passed_filter_exclude_node_names = None
       if self._active_tensor_filter:
         if not debug_dump.find(
-            self._tensor_filters[self._active_tensor_filter], first_n=1):
+            self._tensor_filters[self._active_tensor_filter], first_n=1,
+            exclude_node_names=self._active_filter_exclude_node_names):
           # No dumped tensor passes the filter in this run. Clean up the dump
           # directory and move on.
           self._remove_dump_root()
@@ -334,10 +346,14 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
         else:
           # Some dumped tensor(s) from this run passed the filter.
           passed_filter = self._active_tensor_filter
+          passed_filter_exclude_node_names = (
+              self._active_filter_exclude_node_names)
           self._active_tensor_filter = None
+          self._active_filter_exclude_node_names = None
 
       self._prep_debug_cli_for_run_end(
-          debug_dump, request.tf_error, passed_filter)
+          debug_dump, request.tf_error, passed_filter,
+          passed_filter_exclude_node_names)
 
       self._run_start_response = self._launch_cli()
 
@@ -358,7 +374,11 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
     if os.path.isdir(self._dump_root):
       shutil.rmtree(self._dump_root)
 
-  def _prep_debug_cli_for_run_end(self, debug_dump, tf_error, passed_filter):
+  def _prep_debug_cli_for_run_end(self,
+                                  debug_dump,
+                                  tf_error,
+                                  passed_filter,
+                                  passed_filter_exclude_node_names):
     """Prepare (but not launch) CLI for run-end, with debug dump from the run.
 
     Args:
@@ -368,6 +388,9 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
         (if any).
       passed_filter: (None or str) Name of the tensor filter that just passed
         and caused the preparation of this run-end CLI (if any).
+      passed_filter_exclude_node_names: (None or str) Regular expression used
+        with the tensor filter to exclude ops with names matching the regular
+        expresssion.
     """
 
     if tf_error:
@@ -383,6 +406,9 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
       if passed_filter is not None:
         # Some dumped tensor(s) from this run passed the filter.
         self._init_command = "lt -f %s" % passed_filter
+        if passed_filter_exclude_node_names:
+          self._init_command += (" --filter_exclude_node_names %s" %
+                                 passed_filter_exclude_node_names)
         self._title_color = "red_on_white"
 
     self._run_cli = analyzer_cli.create_analyzer_ui(
@@ -496,6 +522,11 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
     parsed.op_type_filter = parsed.op_type_filter or None
     parsed.tensor_dtype_filter = parsed.tensor_dtype_filter or None
 
+    if parsed.filter_exclude_node_names and not parsed.till_filter_pass:
+      raise ValueError(
+          "The --filter_exclude_node_names (or -feon) flag is valid only if "
+          "the --till_filter_pass (or -f) flag is used.")
+
     if parsed.profile:
       raise debugger_cli_common.CommandLineExit(
           exit_token=framework.OnRunStartResponse(
@@ -525,6 +556,8 @@ class LocalCLIDebugWrapperSession(framework.BaseDebugWrapperSession):
       if parsed.till_filter_pass in self._tensor_filters:
         action = framework.OnRunStartAction.DEBUG_RUN
         self._active_tensor_filter = parsed.till_filter_pass
+        self._active_filter_exclude_node_names = (
+            parsed.filter_exclude_node_names)
         self._active_tensor_filter_run_start_response = run_start_response
       else:
         # Handle invalid filter name.
diff --git a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
index 490812c96d..b06fa26a93 100644
--- a/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
+++ b/tensorflow/python/debug/wrappers/local_cli_wrapper_test.py
@@ -87,7 +87,11 @@ class LocalCLIDebuggerWrapperSessionForTest(
   def _prep_cli_for_run_start(self):
     pass
 
-  def _prep_debug_cli_for_run_end(self, debug_dump, tf_error, passed_filter):
+  def _prep_debug_cli_for_run_end(self,
+                                  debug_dump,
+                                  tf_error,
+                                  passed_filter,
+                                  passed_filter_exclude_op_names):
     self.observers["debug_dumps"].append(debug_dump)
     self.observers["tf_errors"].append(tf_error)
 
@@ -451,6 +455,36 @@ class LocalCLIDebugWrapperSessionTest(test_util.TensorFlowTestCase):
     self.assertEqual(2, len(wrapped_sess.observers["debug_dumps"]))
     self.assertEqual([None, None], wrapped_sess.observers["tf_errors"])
 
+  def testRunTillFilterPassesWithExcludeOpNames(self):
+    wrapped_sess = LocalCLIDebuggerWrapperSessionForTest(
+        [["run", "-f", "greater_than_twelve",
+          "--filter_exclude_node_names", "inc_v.*"],
+         ["run"], ["run"]],
+        self.sess,
+        dump_root=self._tmp_dir)
+
+    def greater_than_twelve(datum, tensor):
+      del datum  # Unused.
+      return tensor > 12.0
+
+    # Verify that adding the same tensor filter more than once is tolerated
+    # (i.e., as if it were added only once).
+    wrapped_sess.add_tensor_filter("greater_than_twelve", greater_than_twelve)
+
+    # run five times.
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+    wrapped_sess.run(self.inc_v)
+
+    self.assertAllClose(14.0, self.sess.run(self.v))
+
+    self.assertEqual([1], wrapped_sess.observers["run_start_cli_run_numbers"])
+
+    # Due to the --filter_exclude_op_names flag, the run-end CLI should show up
+    # not after run 3, but after run 4.
+    self.assertEqual([4], wrapped_sess.observers["run_end_cli_run_numbers"])
+
   def testRunTillFilterPassesWorksInConjunctionWithOtherNodeNameFilter(self):
     """Test that --.*_filter flags work in conjunction with -f.
 
-- 
GitLab


From 8158adc21db1612c42607dff41c083dd3a435e58 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 26 Mar 2018 12:20:32 -0700
Subject: [PATCH 496/960] Internal change.

PiperOrigin-RevId: 190504933
---
 tensorflow/python/kernel_tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index ece1da0332..dbe1bd437e 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1089,6 +1089,7 @@ cuda_py_test(
     tags = [
         "no_windows",
         "noasan",
+        "noguitar",
         "notap",
     ],
 )
-- 
GitLab


From b704d1488d5c15d9e8497843e0bbc667117383ae Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 26 Mar 2018 12:40:56 -0700
Subject: [PATCH 497/960] Internal change.

PiperOrigin-RevId: 190507631
---
 tensorflow/contrib/data/python/kernel_tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index d7cc2f14a4..0b3bf63f79 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -480,6 +480,7 @@ py_test(
     tags = [
         "manual",
         "no_oss",
+        "notap",
     ],
     deps = [
         "//tensorflow/contrib/data/python/ops:prefetching_ops",
-- 
GitLab


From 5890401336c149f49892579bb1a7f4e7c6a52fea Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 26 Mar 2018 12:47:47 -0700
Subject: [PATCH 498/960] Clarify doc strings on gradient methods

PiperOrigin-RevId: 190508614
---
 tensorflow/python/eager/backprop.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index cdcce65c52..a7837b8a7f 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -172,7 +172,7 @@ def implicit_val_and_grad(f):
 
   The wrapped function returns the value and the gradient of f when called with
   the same arguments. The gradient is with respect to all TFE variables which
-  have `variable.watch()` called on them by f.
+  are either trainable or have `variable.watch()` called on them by f.
 
   This function is useful when the exact set of variables to differentiate with
   is not known ahead of time.
@@ -249,8 +249,8 @@ def implicit_grad(f):
   """Returns a function which differentiates f with respect to variables.
 
   The wrapped function returns the gradient of f when called with the same
-  arguments. The gradient is with respect to all TFE variables which have
-  `variable.watch()` called on them by f.
+  arguments. The gradient is with respect to all TFE variables which are
+  either trainable or have `variable.watch()` called on them by f.
 
   This function is useful when the exact set of variables to differentiate with
   is not known ahead of time.
@@ -653,10 +653,10 @@ class GradientTape(object):
   Operations are recorded if they are executed within this context manager and
   at least one of their inputs is being "watched".
 
-  Variables (created by `tf.contrib.eager.Variable` or @{tf.get_variable})
-  are automatically watched. Tensors can be manually watched by invoking the
-  `watch`
-  method on this context manager.
+  Trainable variables (created by `tf.contrib.eager.Variable` or
+  @{tf.get_variable}, trainable=True is default in both cases) are automatically
+  watched. Tensors can be manually watched by invoking the `watch` method on
+  this context manager.
 
   For example, consider the function `y = x * x`. The gradient at `x = 3.0` can
   be computed as:
-- 
GitLab


From ea644ac0783537a6ac8a2c8a2432829b3db69aeb Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 26 Mar 2018 13:05:52 -0700
Subject: [PATCH 499/960] Disabling the state_management_test.

For non-pip builds also.
---
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 64f5cd8357..d72cc1b8a2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -233,6 +233,7 @@ py_test(
     ],
     srcs_version = "PY2AND3",
     tags = [
+        "manual",
         "no_pip",  # b/64527635
         "no_pip_gpu",  # b/63391119
     ],
-- 
GitLab


From 383ce820e5221511cb57904ebd9c32d42d797ac9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 13:08:54 -0700
Subject: [PATCH 500/960] Optimized ops, move code to early, common, section so
 that it can be shared.

PiperOrigin-RevId: 190511964
---
 .../internal/optimized/optimized_ops.h        | 384 +++++++++---------
 1 file changed, 192 insertions(+), 192 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index d7a0005f27..f08d9d6d57 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -324,6 +324,198 @@ void Gemm(const Eigen::MatrixBase<Lhs>& lhs, const Eigen::MatrixBase<Rhs>& rhs,
   }
 }
 
+#ifdef GEMMLOWP_NEON
+// In the common case of batch size 1, a fully-connected node degenerates
+// to a matrix*vector product. LSTM cells contain a fully-connected node;
+// when quantized, this becomes a special type of GEMV operation where
+// the output is 16bit-quantized, thus needs its own special path.
+inline void GEMVForLstmCell(const uint8* input_data, const Dims<4>& input_dims,
+                            const uint8* weights_data,
+                            const Dims<4>& weights_dims,
+                            uint8 weights_zero_point, const int32* bias_data,
+                            const Dims<4>& bias_dims, int32 accum_multiplier,
+                            int accum_shift, int16* output_data,
+                            const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("GEMVForLstmCell");
+  TFLITE_DCHECK(IsPackedWithoutStrides(input_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(weights_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(bias_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
+  TFLITE_DCHECK_EQ(ArraySize(output_dims, 1) * ArraySize(output_dims, 2) *
+                       ArraySize(output_dims, 3),
+                   1);
+  const int input_size = input_dims.strides[3];
+  const int output_size = MatchingArraySize(weights_dims, 1, output_dims, 0);
+  // This special fast path for quantized LSTM cells does not try to support
+  // odd sizes that we haven't encountered in any LSTM cell, that would
+  // require special code (that would go untested until any LSTM cell
+  // exercises it). We just guard our assumptions about size evenness with
+  // the following assertions.
+  TFLITE_DCHECK(!(output_size % 4));
+  TFLITE_DCHECK(!(input_size % 8));
+  const int32* bias_ptr = bias_data;
+  int16* output_ptr = output_data;
+  for (int out = 0; out < output_size; out += 4) {
+    int32x4_t acc_0 = vdupq_n_s32(0);
+    int32x4_t acc_1 = vdupq_n_s32(0);
+    int32x4_t acc_2 = vdupq_n_s32(0);
+    int32x4_t acc_3 = vdupq_n_s32(0);
+    const int16x8_t input_offset_vec = vdupq_n_s16(-128);
+    const int16x8_t weights_offset_vec = vdupq_n_s16(-weights_zero_point);
+    int in = 0;
+    // Handle 16 levels of depth at a time.
+    for (; in <= input_size - 16; in += 16) {
+      const uint8x16_t input_val_u8 = vld1q_u8(input_data + in);
+      const uint8* weights_ptr = weights_data + in + out * input_size;
+      uint8x16_t weights_val_u8_0 = vld1q_u8(weights_ptr + 0 * input_size);
+      uint8x16_t weights_val_u8_1 = vld1q_u8(weights_ptr + 1 * input_size);
+      uint8x16_t weights_val_u8_2 = vld1q_u8(weights_ptr + 2 * input_size);
+      uint8x16_t weights_val_u8_3 = vld1q_u8(weights_ptr + 3 * input_size);
+      int16x8_t input_val_0, input_val_1;
+      const uint8x8_t low = vget_low_u8(input_val_u8);
+      const uint8x8_t high = vget_high_u8(input_val_u8);
+      input_val_0 = vreinterpretq_s16_u16(vmovl_u8(low));
+      input_val_1 = vreinterpretq_s16_u16(vmovl_u8(high));
+      input_val_0 = vaddq_s16(input_val_0, input_offset_vec);
+      input_val_1 = vaddq_s16(input_val_1, input_offset_vec);
+      int16x8_t weights_val_0_0, weights_val_1_0, weights_val_2_0,
+          weights_val_3_0;
+      int16x8_t weights_val_0_1, weights_val_1_1, weights_val_2_1,
+          weights_val_3_1;
+      weights_val_0_0 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_0))),
+          weights_offset_vec);
+      weights_val_0_1 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_0))),
+          weights_offset_vec);
+      weights_val_1_0 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_1))),
+          weights_offset_vec);
+      weights_val_1_1 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_1))),
+          weights_offset_vec);
+      weights_val_2_0 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_2))),
+          weights_offset_vec);
+      weights_val_2_1 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_2))),
+          weights_offset_vec);
+      weights_val_3_0 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_3))),
+          weights_offset_vec);
+      weights_val_3_1 = vaddq_s16(
+          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_3))),
+          weights_offset_vec);
+      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0_0),
+                        vget_low_s16(input_val_0));
+      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1_0),
+                        vget_low_s16(input_val_0));
+      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2_0),
+                        vget_low_s16(input_val_0));
+      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3_0),
+                        vget_low_s16(input_val_0));
+      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0_0),
+                        vget_high_s16(input_val_0));
+      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1_0),
+                        vget_high_s16(input_val_0));
+      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2_0),
+                        vget_high_s16(input_val_0));
+      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3_0),
+                        vget_high_s16(input_val_0));
+      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0_1),
+                        vget_low_s16(input_val_1));
+      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1_1),
+                        vget_low_s16(input_val_1));
+      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2_1),
+                        vget_low_s16(input_val_1));
+      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3_1),
+                        vget_low_s16(input_val_1));
+      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0_1),
+                        vget_high_s16(input_val_1));
+      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1_1),
+                        vget_high_s16(input_val_1));
+      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2_1),
+                        vget_high_s16(input_val_1));
+      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3_1),
+                        vget_high_s16(input_val_1));
+    }
+    // Handle 8 levels of depth at a time.
+    for (; in < input_size; in += 8) {
+      const uint8x8_t input_val_u8 = vld1_u8(input_data + in);
+      const uint8* weights_ptr = weights_data + in + out * input_size;
+      uint8x8_t weights_val_u8_0 = vld1_u8(weights_ptr + 0 * input_size);
+      uint8x8_t weights_val_u8_1 = vld1_u8(weights_ptr + 1 * input_size);
+      uint8x8_t weights_val_u8_2 = vld1_u8(weights_ptr + 2 * input_size);
+      uint8x8_t weights_val_u8_3 = vld1_u8(weights_ptr + 3 * input_size);
+      int16x8_t input_val;
+      input_val = vreinterpretq_s16_u16(vmovl_u8(input_val_u8));
+      input_val = vaddq_s16(input_val, input_offset_vec);
+      int16x8_t weights_val_0, weights_val_1, weights_val_2, weights_val_3;
+      weights_val_0 =
+          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_0)),
+                    weights_offset_vec);
+      weights_val_1 =
+          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_1)),
+                    weights_offset_vec);
+      weights_val_2 =
+          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_2)),
+                    weights_offset_vec);
+      weights_val_3 =
+          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_3)),
+                    weights_offset_vec);
+      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0),
+                        vget_low_s16(input_val));
+      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1),
+                        vget_low_s16(input_val));
+      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2),
+                        vget_low_s16(input_val));
+      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3),
+                        vget_low_s16(input_val));
+      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0),
+                        vget_high_s16(input_val));
+      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1),
+                        vget_high_s16(input_val));
+      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2),
+                        vget_high_s16(input_val));
+      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3),
+                        vget_high_s16(input_val));
+    }
+    // Horizontally reduce accumulators
+    int32x2_t pairwise_reduced_acc_0, pairwise_reduced_acc_1,
+        pairwise_reduced_acc_2, pairwise_reduced_acc_3;
+    pairwise_reduced_acc_0 =
+        vpadd_s32(vget_low_s32(acc_0), vget_high_s32(acc_0));
+    pairwise_reduced_acc_1 =
+        vpadd_s32(vget_low_s32(acc_1), vget_high_s32(acc_1));
+    pairwise_reduced_acc_2 =
+        vpadd_s32(vget_low_s32(acc_2), vget_high_s32(acc_2));
+    pairwise_reduced_acc_3 =
+        vpadd_s32(vget_low_s32(acc_3), vget_high_s32(acc_3));
+    const int32x2_t reduced_lo =
+        vpadd_s32(pairwise_reduced_acc_0, pairwise_reduced_acc_1);
+    const int32x2_t reduced_hi =
+        vpadd_s32(pairwise_reduced_acc_2, pairwise_reduced_acc_3);
+    int32x4_t reduced = vcombine_s32(reduced_lo, reduced_hi);
+    // Add bias values.
+    int32x4_t bias_vec = vld1q_s32(bias_ptr);
+    bias_ptr += 4;
+    reduced = vaddq_s32(reduced, bias_vec);
+    int left_shift = accum_shift > 0 ? accum_shift : 0;
+    int right_shift = accum_shift > 0 ? 0 : -accum_shift;
+    reduced = vshlq_s32(reduced, vdupq_n_s32(left_shift));
+    // Multiply by the fixed-point multiplier.
+    reduced = vqrdmulhq_n_s32(reduced, accum_multiplier);
+    // Rounding-shift-right.
+    using gemmlowp::RoundingDivideByPOT;
+    reduced = RoundingDivideByPOT(reduced, right_shift);
+    // Narrow values down to 16 bit signed.
+    const int16x4_t res16 = vqmovn_s32(reduced);
+    vst1_s16(output_ptr, res16);
+    output_ptr += 4;
+  }
+}
+#endif
+
 inline void FullyConnected(const float* input_data, const Dims<4>& input_dims,
                            const float* weights_data,
                            const Dims<4>& weights_dims, const float* bias_data,
@@ -2478,198 +2670,6 @@ inline void LstmCell(const float* input_data, const Dims<4>& input_dims,
       output_state_map.tanh();
 }
 
-#ifdef GEMMLOWP_NEON
-// In the common case of batch size 1, a fully-connected node degenerates
-// to a matrix*vector product. LSTM cells contain a fully-connected node;
-// when quantized, this becomes a special type of GEMV operation where
-// the output is 16bit-quantized, thus needs its own special path.
-inline void GEMVForLstmCell(const uint8* input_data, const Dims<4>& input_dims,
-                            const uint8* weights_data,
-                            const Dims<4>& weights_dims,
-                            uint8 weights_zero_point, const int32* bias_data,
-                            const Dims<4>& bias_dims, int32 accum_multiplier,
-                            int accum_shift, int16* output_data,
-                            const Dims<4>& output_dims) {
-  gemmlowp::ScopedProfilingLabel label("GEMVForLstmCell");
-  TFLITE_DCHECK(IsPackedWithoutStrides(input_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(weights_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(bias_dims));
-  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
-  TFLITE_DCHECK_EQ(ArraySize(output_dims, 1) * ArraySize(output_dims, 2) *
-                       ArraySize(output_dims, 3),
-                   1);
-  const int input_size = input_dims.strides[3];
-  const int output_size = MatchingArraySize(weights_dims, 1, output_dims, 0);
-  // This special fast path for quantized LSTM cells does not try to support
-  // odd sizes that we haven't encountered in any LSTM cell, that would
-  // require special code (that would go untested until any LSTM cell
-  // exercises it). We just guard our assumptions about size evenness with
-  // the following assertions.
-  TFLITE_DCHECK(!(output_size % 4));
-  TFLITE_DCHECK(!(input_size % 8));
-  const int32* bias_ptr = bias_data;
-  int16* output_ptr = output_data;
-  for (int out = 0; out < output_size; out += 4) {
-    int32x4_t acc_0 = vdupq_n_s32(0);
-    int32x4_t acc_1 = vdupq_n_s32(0);
-    int32x4_t acc_2 = vdupq_n_s32(0);
-    int32x4_t acc_3 = vdupq_n_s32(0);
-    const int16x8_t input_offset_vec = vdupq_n_s16(-128);
-    const int16x8_t weights_offset_vec = vdupq_n_s16(-weights_zero_point);
-    int in = 0;
-    // Handle 16 levels of depth at a time.
-    for (; in <= input_size - 16; in += 16) {
-      const uint8x16_t input_val_u8 = vld1q_u8(input_data + in);
-      const uint8* weights_ptr = weights_data + in + out * input_size;
-      uint8x16_t weights_val_u8_0 = vld1q_u8(weights_ptr + 0 * input_size);
-      uint8x16_t weights_val_u8_1 = vld1q_u8(weights_ptr + 1 * input_size);
-      uint8x16_t weights_val_u8_2 = vld1q_u8(weights_ptr + 2 * input_size);
-      uint8x16_t weights_val_u8_3 = vld1q_u8(weights_ptr + 3 * input_size);
-      int16x8_t input_val_0, input_val_1;
-      const uint8x8_t low = vget_low_u8(input_val_u8);
-      const uint8x8_t high = vget_high_u8(input_val_u8);
-      input_val_0 = vreinterpretq_s16_u16(vmovl_u8(low));
-      input_val_1 = vreinterpretq_s16_u16(vmovl_u8(high));
-      input_val_0 = vaddq_s16(input_val_0, input_offset_vec);
-      input_val_1 = vaddq_s16(input_val_1, input_offset_vec);
-      int16x8_t weights_val_0_0, weights_val_1_0, weights_val_2_0,
-          weights_val_3_0;
-      int16x8_t weights_val_0_1, weights_val_1_1, weights_val_2_1,
-          weights_val_3_1;
-      weights_val_0_0 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_0))),
-          weights_offset_vec);
-      weights_val_0_1 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_0))),
-          weights_offset_vec);
-      weights_val_1_0 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_1))),
-          weights_offset_vec);
-      weights_val_1_1 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_1))),
-          weights_offset_vec);
-      weights_val_2_0 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_2))),
-          weights_offset_vec);
-      weights_val_2_1 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_2))),
-          weights_offset_vec);
-      weights_val_3_0 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_low_u8(weights_val_u8_3))),
-          weights_offset_vec);
-      weights_val_3_1 = vaddq_s16(
-          vreinterpretq_s16_u16(vmovl_u8(vget_high_u8(weights_val_u8_3))),
-          weights_offset_vec);
-      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0_0),
-                        vget_low_s16(input_val_0));
-      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1_0),
-                        vget_low_s16(input_val_0));
-      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2_0),
-                        vget_low_s16(input_val_0));
-      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3_0),
-                        vget_low_s16(input_val_0));
-      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0_0),
-                        vget_high_s16(input_val_0));
-      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1_0),
-                        vget_high_s16(input_val_0));
-      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2_0),
-                        vget_high_s16(input_val_0));
-      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3_0),
-                        vget_high_s16(input_val_0));
-      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0_1),
-                        vget_low_s16(input_val_1));
-      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1_1),
-                        vget_low_s16(input_val_1));
-      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2_1),
-                        vget_low_s16(input_val_1));
-      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3_1),
-                        vget_low_s16(input_val_1));
-      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0_1),
-                        vget_high_s16(input_val_1));
-      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1_1),
-                        vget_high_s16(input_val_1));
-      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2_1),
-                        vget_high_s16(input_val_1));
-      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3_1),
-                        vget_high_s16(input_val_1));
-    }
-    // Handle 8 levels of depth at a time.
-    for (; in < input_size; in += 8) {
-      const uint8x8_t input_val_u8 = vld1_u8(input_data + in);
-      const uint8* weights_ptr = weights_data + in + out * input_size;
-      uint8x8_t weights_val_u8_0 = vld1_u8(weights_ptr + 0 * input_size);
-      uint8x8_t weights_val_u8_1 = vld1_u8(weights_ptr + 1 * input_size);
-      uint8x8_t weights_val_u8_2 = vld1_u8(weights_ptr + 2 * input_size);
-      uint8x8_t weights_val_u8_3 = vld1_u8(weights_ptr + 3 * input_size);
-      int16x8_t input_val;
-      input_val = vreinterpretq_s16_u16(vmovl_u8(input_val_u8));
-      input_val = vaddq_s16(input_val, input_offset_vec);
-      int16x8_t weights_val_0, weights_val_1, weights_val_2, weights_val_3;
-      weights_val_0 =
-          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_0)),
-                    weights_offset_vec);
-      weights_val_1 =
-          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_1)),
-                    weights_offset_vec);
-      weights_val_2 =
-          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_2)),
-                    weights_offset_vec);
-      weights_val_3 =
-          vaddq_s16(vreinterpretq_s16_u16(vmovl_u8(weights_val_u8_3)),
-                    weights_offset_vec);
-      acc_0 = vmlal_s16(acc_0, vget_low_s16(weights_val_0),
-                        vget_low_s16(input_val));
-      acc_1 = vmlal_s16(acc_1, vget_low_s16(weights_val_1),
-                        vget_low_s16(input_val));
-      acc_2 = vmlal_s16(acc_2, vget_low_s16(weights_val_2),
-                        vget_low_s16(input_val));
-      acc_3 = vmlal_s16(acc_3, vget_low_s16(weights_val_3),
-                        vget_low_s16(input_val));
-      acc_0 = vmlal_s16(acc_0, vget_high_s16(weights_val_0),
-                        vget_high_s16(input_val));
-      acc_1 = vmlal_s16(acc_1, vget_high_s16(weights_val_1),
-                        vget_high_s16(input_val));
-      acc_2 = vmlal_s16(acc_2, vget_high_s16(weights_val_2),
-                        vget_high_s16(input_val));
-      acc_3 = vmlal_s16(acc_3, vget_high_s16(weights_val_3),
-                        vget_high_s16(input_val));
-    }
-    // Horizontally reduce accumulators
-    int32x2_t pairwise_reduced_acc_0, pairwise_reduced_acc_1,
-        pairwise_reduced_acc_2, pairwise_reduced_acc_3;
-    pairwise_reduced_acc_0 =
-        vpadd_s32(vget_low_s32(acc_0), vget_high_s32(acc_0));
-    pairwise_reduced_acc_1 =
-        vpadd_s32(vget_low_s32(acc_1), vget_high_s32(acc_1));
-    pairwise_reduced_acc_2 =
-        vpadd_s32(vget_low_s32(acc_2), vget_high_s32(acc_2));
-    pairwise_reduced_acc_3 =
-        vpadd_s32(vget_low_s32(acc_3), vget_high_s32(acc_3));
-    const int32x2_t reduced_lo =
-        vpadd_s32(pairwise_reduced_acc_0, pairwise_reduced_acc_1);
-    const int32x2_t reduced_hi =
-        vpadd_s32(pairwise_reduced_acc_2, pairwise_reduced_acc_3);
-    int32x4_t reduced = vcombine_s32(reduced_lo, reduced_hi);
-    // Add bias values.
-    int32x4_t bias_vec = vld1q_s32(bias_ptr);
-    bias_ptr += 4;
-    reduced = vaddq_s32(reduced, bias_vec);
-    int left_shift = accum_shift > 0 ? accum_shift : 0;
-    int right_shift = accum_shift > 0 ? 0 : -accum_shift;
-    reduced = vshlq_s32(reduced, vdupq_n_s32(left_shift));
-    // Multiply by the fixed-point multiplier.
-    reduced = vqrdmulhq_n_s32(reduced, accum_multiplier);
-    // Rounding-shift-right.
-    using gemmlowp::RoundingDivideByPOT;
-    reduced = RoundingDivideByPOT(reduced, right_shift);
-    // Narrow values down to 16 bit signed.
-    const int16x4_t res16 = vqmovn_s32(reduced);
-    vst1_s16(output_ptr, res16);
-    output_ptr += 4;
-  }
-}
-#endif
-
 // Quantized LSTM cell. Currently just a copy of the reference impl in
 // reference_ops.h. See the big function comment there, not replicating it
 // here.
-- 
GitLab


From 4c909d283d7efab3e0dde68eb27d31d68407e207 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 13:11:18 -0700
Subject: [PATCH 501/960] Add header guard to lstm_utils.

PiperOrigin-RevId: 190512302
---
 .../contrib/lite/toco/graph_transformations/lstm_utils.h     | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/lstm_utils.h b/tensorflow/contrib/lite/toco/graph_transformations/lstm_utils.h
index 881c2d4dc8..4a9974ed4e 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/lstm_utils.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/lstm_utils.h
@@ -12,6 +12,9 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+#ifndef TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_LSTM_UTILS_H_
+#define TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_LSTM_UTILS_H_
+
 #include <iostream>
 #include <string>
 #include <vector>
@@ -100,3 +103,5 @@ bool GetMatchingRnnArray(Model* model, const string& back_edge_source_array,
                          string* rnn_array);
 
 }  // namespace toco
+
+#endif  // TENSORFLOW_CONTRIB_LITE_TOCO_GRAPH_TRANSFORMATIONS_LSTM_UTILS_H_
-- 
GitLab


From 3f708534f7fa5d548c2ccd0a77a229a815868e8f Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 26 Mar 2018 13:15:53 -0700
Subject: [PATCH 502/960] Internal change.

PiperOrigin-RevId: 190512928
---
 tensorflow/contrib/distributions/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index e9c827a618..1c381cc354 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -490,6 +490,7 @@ cuda_py_test(
         "manual",
         "noasan",
         "noguitar",
+        "optonly",
     ],
 )
 
-- 
GitLab


From 5427f60f69c3f22bc5e40b3c51a484dd3af504fb Mon Sep 17 00:00:00 2001
From: Jiri Simsa <jsimsa@google.com>
Date: Mon, 26 Mar 2018 13:25:58 -0700
Subject: [PATCH 503/960] Add additional protobuf imports.

PiperOrigin-RevId: 190514839
---
 tensorflow/core/platform/default/protobuf.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/platform/default/protobuf.h b/tensorflow/core/platform/default/protobuf.h
index 03d8b6c238..c732c76ff7 100644
--- a/tensorflow/core/platform/default/protobuf.h
+++ b/tensorflow/core/platform/default/protobuf.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "google/protobuf/arena.h"
 #include "google/protobuf/compiler/importer.h"
 #include "google/protobuf/descriptor.h"
+#include "google/protobuf/dynamic_message.h"
 #include "google/protobuf/io/coded_stream.h"
 #include "google/protobuf/io/zero_copy_stream.h"
 #include "google/protobuf/io/zero_copy_stream_impl_lite.h"
-- 
GitLab


From 54cc8b35f1062f385f0e97c397e1ae96c91c9f62 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 13:30:17 -0700
Subject: [PATCH 504/960] Global rename of py2tf to autograph

PiperOrigin-RevId: 190515509
---
 tensorflow/BUILD                              | 12 ++--
 tensorflow/contrib/BUILD                      |  2 +-
 tensorflow/contrib/{py2tf => autograph}/BUILD |  8 +--
 .../contrib/{py2tf => autograph}/README.md    |  2 +-
 .../contrib/{py2tf => autograph}/__init__.py  | 20 +++---
 .../{py2tf => autograph}/converters/BUILD     | 15 +++--
 .../converters/__init__.py                    |  2 +-
 .../converters/asserts.py                     |  4 +-
 .../converters/asserts_test.py                |  4 +-
 .../converters/break_statements.py            |  8 +--
 .../converters/break_statements_test.py       |  4 +-
 .../converters/builtin_functions.py           |  8 +--
 .../converters/builtin_functions_test.py      |  4 +-
 .../converters/call_trees.py                  | 21 +++---
 .../converters/call_trees_test.py             |  4 +-
 .../converters/continue_statements.py         |  8 +--
 .../converters/continue_statements_test.py    |  4 +-
 .../converters/control_flow.py                | 16 ++---
 .../converters/control_flow_test.py           |  4 +-
 .../converters/converter_test_base.py         | 22 +++----
 .../converters/decorators.py                  |  4 +-
 .../converters/decorators_test.py             |  6 +-
 .../converters/for_loops.py                   | 20 +++---
 .../converters/for_loops_test.py              |  4 +-
 .../{py2tf => autograph}/converters/ifexp.py  |  6 +-
 .../converters/ifexp_test.py                  | 18 ++---
 .../converters/list_comprehension.py          |  6 +-
 .../converters/list_comprehension_test.py     |  4 +-
 .../{py2tf => autograph}/converters/lists.py  |  8 +--
 .../converters/lists_test.py                  |  6 +-
 .../converters/logical_expressions.py         | 12 ++--
 .../converters/logical_expressions_test.py    |  4 +-
 .../converters/name_scopes.py                 |  4 +-
 .../converters/name_scopes_test.py            |  4 +-
 .../converters/side_effect_guards.py          | 18 ++---
 .../converters/side_effect_guards_test.py     |  4 +-
 .../converters/single_return.py               | 12 ++--
 .../converters/single_return_test.py          |  4 +-
 .../contrib/{py2tf => autograph}/impl/BUILD   | 10 +--
 .../contrib/{py2tf => autograph}/impl/api.py  | 16 ++---
 .../{py2tf => autograph}/impl/api_test.py     | 18 +++--
 .../{py2tf => autograph}/impl/config.py       | 16 ++---
 .../{py2tf => autograph}/impl/conversion.py   | 66 +++++++++----------
 .../impl/conversion_test.py                   |  4 +-
 .../{py2tf => autograph}/impl/naming.py       |  2 +-
 .../{py2tf => autograph}/impl/naming_test.py  |  2 +-
 .../contrib/{py2tf => autograph}/pyct/BUILD   |  0
 .../{py2tf => autograph}/pyct/__init__.py     |  0
 .../contrib/{py2tf => autograph}/pyct/anno.py |  0
 .../{py2tf => autograph}/pyct/anno_test.py    |  2 +-
 .../{py2tf => autograph}/pyct/ast_util.py     |  2 +-
 .../pyct/ast_util_test.py                     |  8 +--
 .../{py2tf => autograph}/pyct/compiler.py     |  0
 .../pyct/compiler_test.py                     |  4 +-
 .../{py2tf => autograph}/pyct/context.py      |  0
 .../pyct/inspect_utils.py                     |  0
 .../pyct/inspect_utils_test.py                |  2 +-
 .../{py2tf => autograph}/pyct/parser.py       |  0
 .../{py2tf => autograph}/pyct/parser_test.py  |  2 +-
 .../pyct/pretty_printer.py                    |  0
 .../pyct/pretty_printer_test.py               |  2 +-
 .../{py2tf => autograph}/pyct/qual_names.py   |  2 +-
 .../pyct/qual_names_test.py                   | 10 +--
 .../pyct/static_analysis/BUILD                | 10 +--
 .../pyct/static_analysis/__init__.py          |  0
 .../pyct/static_analysis/activity.py          |  8 +--
 .../pyct/static_analysis/activity_test.py     | 14 ++--
 .../pyct/static_analysis/annos.py             |  0
 .../pyct/static_analysis/live_values.py       |  6 +-
 .../pyct/static_analysis/live_values_test.py  | 14 ++--
 .../pyct/static_analysis/type_info.py         |  4 +-
 .../pyct/static_analysis/type_info_test.py    | 16 ++---
 .../{py2tf => autograph}/pyct/templates.py    |  6 +-
 .../pyct/templates_test.py                    |  6 +-
 .../{py2tf => autograph}/pyct/transformer.py  | 12 ++--
 .../contrib/{py2tf => autograph}/utils/BUILD  |  0
 .../contrib/autograph/utils/__init__.py       | 36 ++++++++++
 .../{py2tf => autograph}/utils/builtins.py    |  4 +-
 .../utils/builtins_test.py                    |  2 +-
 .../utils/context_managers.py                 |  0
 .../utils/context_managers_test.py            |  2 +-
 .../{py2tf => autograph}/utils/misc.py        |  0
 .../{py2tf => autograph}/utils/misc_test.py   |  2 +-
 .../utils/multiple_dispatch.py                |  4 +-
 .../utils/multiple_dispatch_test.py           |  2 +-
 .../{py2tf => autograph}/utils/py_func.py     |  5 +-
 .../utils/py_func_test.py                     |  2 +-
 .../{py2tf => autograph}/utils/tensor_list.py |  0
 .../utils/tensor_list_test.py                 |  4 +-
 .../{py2tf => autograph}/utils/testing.py     |  0
 .../{py2tf => autograph}/utils/type_check.py  |  2 +-
 .../utils/type_check_test.py                  |  2 +-
 .../{py2tf => autograph}/utils/type_hints.py  |  0
 tensorflow/contrib/py2tf/utils/__init__.py    | 36 ----------
 tensorflow/tools/pip_package/BUILD            | 12 ++--
 95 files changed, 346 insertions(+), 349 deletions(-)
 rename tensorflow/contrib/{py2tf => autograph}/BUILD (75%)
 rename tensorflow/contrib/{py2tf => autograph}/README.md (87%)
 rename tensorflow/contrib/{py2tf => autograph}/__init__.py (64%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/BUILD (92%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/__init__.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/asserts.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/asserts_test.py (90%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/break_statements.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/break_statements_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/builtin_functions.py (92%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/builtin_functions_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/call_trees.py (94%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/call_trees_test.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/continue_statements.py (94%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/continue_statements_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/control_flow.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/control_flow_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/converter_test_base.py (85%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/decorators.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/decorators_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/for_loops.py (80%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/for_loops_test.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/ifexp.py (88%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/ifexp_test.py (86%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/list_comprehension.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/list_comprehension_test.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/lists.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/lists_test.py (90%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/logical_expressions.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/logical_expressions_test.py (92%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/name_scopes.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/name_scopes_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/side_effect_guards.py (91%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/side_effect_guards_test.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/single_return.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/converters/single_return_test.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/BUILD (82%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/api.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/api_test.py (92%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/config.py (79%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/conversion.py (84%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/conversion_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/naming.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/impl/naming_test.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/BUILD (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/__init__.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/anno.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/anno_test.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/ast_util.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/ast_util_test.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/compiler.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/compiler_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/context.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/inspect_utils.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/inspect_utils_test.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/parser.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/parser_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/pretty_printer.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/pretty_printer_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/qual_names.py (99%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/qual_names_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/BUILD (83%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/__init__.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/activity.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/activity_test.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/annos.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/live_values.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/live_values_test.py (89%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/type_info.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/static_analysis/type_info_test.py (93%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/templates.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/templates_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/pyct/transformer.py (89%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/BUILD (100%)
 create mode 100644 tensorflow/contrib/autograph/utils/__init__.py
 rename tensorflow/contrib/{py2tf => autograph}/utils/builtins.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/builtins_test.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/context_managers.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/context_managers_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/misc.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/misc_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/multiple_dispatch.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/multiple_dispatch_test.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/py_func.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/py_func_test.py (98%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/tensor_list.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/tensor_list_test.py (97%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/testing.py (100%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/type_check.py (95%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/type_check_test.py (96%)
 rename tensorflow/contrib/{py2tf => autograph}/utils/type_hints.py (100%)
 delete mode 100644 tensorflow/contrib/py2tf/utils/__init__.py

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index c75bf8abab..b073adfee9 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -448,6 +448,12 @@ filegroup(
         "//tensorflow/contrib:all_files",
         "//tensorflow/contrib/all_reduce:all_files",
         "//tensorflow/contrib/android:all_files",
+        "//tensorflow/contrib/autograph:all_files",
+        "//tensorflow/contrib/autograph/converters:all_files",
+        "//tensorflow/contrib/autograph/impl:all_files",
+        "//tensorflow/contrib/autograph/pyct:all_files",
+        "//tensorflow/contrib/autograph/pyct/static_analysis:all_files",
+        "//tensorflow/contrib/autograph/utils:all_files",
         "//tensorflow/contrib/batching:all_files",
         "//tensorflow/contrib/bayesflow:all_files",
         "//tensorflow/contrib/boosted_trees:all_files",
@@ -541,12 +547,6 @@ filegroup(
         "//tensorflow/contrib/opt:all_files",
         "//tensorflow/contrib/periodic_resample:all_files",
         "//tensorflow/contrib/predictor:all_files",
-        "//tensorflow/contrib/py2tf:all_files",
-        "//tensorflow/contrib/py2tf/converters:all_files",
-        "//tensorflow/contrib/py2tf/impl:all_files",
-        "//tensorflow/contrib/py2tf/pyct:all_files",
-        "//tensorflow/contrib/py2tf/pyct/static_analysis:all_files",
-        "//tensorflow/contrib/py2tf/utils:all_files",
         "//tensorflow/contrib/quantize:all_files",
         "//tensorflow/contrib/receptive_field:all_files",
         "//tensorflow/contrib/reduce_slice_ops:all_files",
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 2d7bbc016f..bdbd738906 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -79,7 +79,7 @@ py_library(
         "//tensorflow/contrib/predictor",
         "//tensorflow/contrib/quantization:quantization_py",
         "//tensorflow/contrib/quantize:quantize_graph",
-        "//tensorflow/contrib/py2tf",
+        "//tensorflow/contrib/autograph",
         "//tensorflow/contrib/receptive_field:receptive_field_py",
         "//tensorflow/contrib/reduce_slice_ops:reduce_slice_ops_py",
         "//tensorflow/contrib/remote_fused_graph/pylib:remote_fused_graph_ops_py",
diff --git a/tensorflow/contrib/py2tf/BUILD b/tensorflow/contrib/autograph/BUILD
similarity index 75%
rename from tensorflow/contrib/py2tf/BUILD
rename to tensorflow/contrib/autograph/BUILD
index d91220f6dd..30dd846893 100644
--- a/tensorflow/contrib/py2tf/BUILD
+++ b/tensorflow/contrib/autograph/BUILD
@@ -15,16 +15,16 @@ filegroup(
 )
 
 py_library(
-    name = "py2tf",
+    name = "autograph",
     srcs = [
         "__init__.py",
     ],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/contrib/py2tf/impl",
-        "//tensorflow/contrib/py2tf/pyct",
-        "//tensorflow/contrib/py2tf/utils",
+        "//tensorflow/contrib/autograph/impl",
+        "//tensorflow/contrib/autograph/pyct",
+        "//tensorflow/contrib/autograph/utils",
         "@gast_archive//:gast",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/py2tf/README.md b/tensorflow/contrib/autograph/README.md
similarity index 87%
rename from tensorflow/contrib/py2tf/README.md
rename to tensorflow/contrib/autograph/README.md
index cd50675ad5..7e84f237dc 100644
--- a/tensorflow/contrib/py2tf/README.md
+++ b/tensorflow/contrib/autograph/README.md
@@ -1,4 +1,4 @@
-# Py2TF
+# Autograph
 
 A compiler for generating TensorFlow numeric and control flow ops from Python
 code.
diff --git a/tensorflow/contrib/py2tf/__init__.py b/tensorflow/contrib/autograph/__init__.py
similarity index 64%
rename from tensorflow/contrib/py2tf/__init__.py
rename to tensorflow/contrib/autograph/__init__.py
index a4b62a0976..a39f44b21a 100644
--- a/tensorflow/contrib/py2tf/__init__.py
+++ b/tensorflow/contrib/autograph/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Py2TF compiles Python code into equivalent TensorFlow code.
+"""Autograph compiles Python code into equivalent TensorFlow code.
 
 Equivalent here means that they have the same effect when executed.
 """
@@ -21,19 +21,19 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.impl.api import convert
-from tensorflow.contrib.py2tf.impl.api import converted_call
-from tensorflow.contrib.py2tf.impl.api import do_not_convert
-from tensorflow.contrib.py2tf.impl.api import RunMode
-from tensorflow.contrib.py2tf.impl.api import to_code
-from tensorflow.contrib.py2tf.impl.api import to_graph
-from tensorflow.contrib.py2tf.pyct.transformer import PyFlowParseError
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.impl.api import convert
+from tensorflow.contrib.autograph.impl.api import converted_call
+from tensorflow.contrib.autograph.impl.api import do_not_convert
+from tensorflow.contrib.autograph.impl.api import RunMode
+from tensorflow.contrib.autograph.impl.api import to_code
+from tensorflow.contrib.autograph.impl.api import to_graph
+from tensorflow.contrib.autograph.pyct.transformer import AutographParseError
 from tensorflow.python.util.all_util import remove_undocumented
 
 _allowed_symbols = [
     'utils', 'convert', 'converted_call', 'do_not_convert', 'RunMode',
-    'to_code', 'to_graph', 'PyFlowParseError'
+    'to_code', 'to_graph', 'AutographParseError'
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/py2tf/converters/BUILD b/tensorflow/contrib/autograph/converters/BUILD
similarity index 92%
rename from tensorflow/contrib/py2tf/converters/BUILD
rename to tensorflow/contrib/autograph/converters/BUILD
index f624c42686..608bd82722 100644
--- a/tensorflow/contrib/py2tf/converters/BUILD
+++ b/tensorflow/contrib/autograph/converters/BUILD
@@ -49,9 +49,9 @@ py_library(
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
         ":converters",
-        "//tensorflow/contrib/py2tf/pyct",
-        "//tensorflow/contrib/py2tf/pyct/static_analysis",
-        "//tensorflow/contrib/py2tf/utils",
+        "//tensorflow/contrib/autograph/pyct",
+        "//tensorflow/contrib/autograph/pyct/static_analysis",
+        "//tensorflow/contrib/autograph/utils",
         "@gast_archive//:gast",
         "@six_archive//:six",
     ],
@@ -89,11 +89,12 @@ py_test(
 
 py_test(
     name = "call_trees_test",
+    size = "large",
     srcs = ["call_trees_test.py"],
     srcs_version = "PY2AND3",
     deps = [
         ":test_lib",
-        "//tensorflow/contrib/py2tf/impl",
+        "//tensorflow/contrib/autograph/impl",
         "//tensorflow/python:client_testlib",
     ],
 )
@@ -143,7 +144,7 @@ py_test(
     srcs = ["name_scopes_test.py"],
     deps = [
         ":test_lib",
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/python:client_testlib",
     ],
 )
@@ -199,7 +200,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":test_lib",
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/python:client_testlib",
     ],
 )
@@ -210,7 +211,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":test_lib",
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/python:client_testlib",
     ],
 )
diff --git a/tensorflow/contrib/py2tf/converters/__init__.py b/tensorflow/contrib/autograph/converters/__init__.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/__init__.py
rename to tensorflow/contrib/autograph/converters/__init__.py
index ca10896ee5..e4e8eda42f 100644
--- a/tensorflow/contrib/py2tf/converters/__init__.py
+++ b/tensorflow/contrib/autograph/converters/__init__.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Code converters used by Py2TF."""
+"""Code converters used by Autograph."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/py2tf/converters/asserts.py b/tensorflow/contrib/autograph/converters/asserts.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/asserts.py
rename to tensorflow/contrib/autograph/converters/asserts.py
index 5b9b8e772b..f011a97ade 100644
--- a/tensorflow/contrib/py2tf/converters/asserts.py
+++ b/tensorflow/contrib/autograph/converters/asserts.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 class AssertsTransformer(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/converters/asserts_test.py b/tensorflow/contrib/autograph/converters/asserts_test.py
similarity index 90%
rename from tensorflow/contrib/py2tf/converters/asserts_test.py
rename to tensorflow/contrib/autograph/converters/asserts_test.py
index 6611f2777a..cc913febe8 100644
--- a/tensorflow/contrib/py2tf/converters/asserts_test.py
+++ b/tensorflow/contrib/autograph/converters/asserts_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.converters import asserts
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import asserts
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/break_statements.py b/tensorflow/contrib/autograph/converters/break_statements.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/break_statements.py
rename to tensorflow/contrib/autograph/converters/break_statements.py
index bfb709c5e3..721bc0ccd0 100644
--- a/tensorflow/contrib/py2tf/converters/break_statements.py
+++ b/tensorflow/contrib/autograph/converters/break_statements.py
@@ -20,10 +20,10 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class BreakCanonicalizationTransformer(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/converters/break_statements_test.py b/tensorflow/contrib/autograph/converters/break_statements_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/break_statements_test.py
rename to tensorflow/contrib/autograph/converters/break_statements_test.py
index 095fcdff07..dd4914a022 100644
--- a/tensorflow/contrib/py2tf/converters/break_statements_test.py
+++ b/tensorflow/contrib/autograph/converters/break_statements_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import break_statements
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import break_statements
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/builtin_functions.py b/tensorflow/contrib/autograph/converters/builtin_functions.py
similarity index 92%
rename from tensorflow/contrib/py2tf/converters/builtin_functions.py
rename to tensorflow/contrib/autograph/converters/builtin_functions.py
index f1129ef153..0349ce29ce 100644
--- a/tensorflow/contrib/py2tf/converters/builtin_functions.py
+++ b/tensorflow/contrib/autograph/converters/builtin_functions.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 class BuiltinFunctionTransformer(transformer.Base):
@@ -38,13 +38,13 @@ class BuiltinFunctionTransformer(transformer.Base):
 
   def _convert_builtin(self, node):
     template = """
-      py2tf_utils.dynamic_builtin(func, args)
+      autograph_utils.dynamic_builtin(func, args)
     """
     return templates.replace(template, func=node.func, args=node.args)[0].value
 
   def _convert_print(self, node):
     template = """
-      py2tf_utils.dynamic_print(args)
+      autograph_utils.dynamic_print(args)
     """
     return templates.replace(template, args=node.args)[0].value
 
diff --git a/tensorflow/contrib/py2tf/converters/builtin_functions_test.py b/tensorflow/contrib/autograph/converters/builtin_functions_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/converters/builtin_functions_test.py
rename to tensorflow/contrib/autograph/converters/builtin_functions_test.py
index eb60a1d8ae..ac7e756c47 100644
--- a/tensorflow/contrib/py2tf/converters/builtin_functions_test.py
+++ b/tensorflow/contrib/autograph/converters/builtin_functions_test.py
@@ -22,8 +22,8 @@ import sys
 
 import six
 
-from tensorflow.contrib.py2tf.converters import builtin_functions
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import builtin_functions
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import logging_ops
diff --git a/tensorflow/contrib/py2tf/converters/call_trees.py b/tensorflow/contrib/autograph/converters/call_trees.py
similarity index 94%
rename from tensorflow/contrib/py2tf/converters/call_trees.py
rename to tensorflow/contrib/autograph/converters/call_trees.py
index f498b814bf..61f6bfd7e7 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees.py
+++ b/tensorflow/contrib/autograph/converters/call_trees.py
@@ -27,12 +27,12 @@ import types
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import inspect_utils
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import inspect_utils
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 from tensorflow.python.util import tf_inspect
 
 
@@ -199,7 +199,7 @@ class CallTreeTransformer(transformer.Base):
   def _wrap_to_py_func_no_return(self, node):
     # TODO(mdan): Properly handle varargs, etc.
     template = """
-      py2tf_utils.wrap_py_func(func, None, (args,), kwargs, True)
+      autograph_utils.wrap_py_func(func, None, (args,), kwargs, True)
     """
     return templates.replace(
         template,
@@ -210,7 +210,7 @@ class CallTreeTransformer(transformer.Base):
   def _wrap_to_py_func_single_return(self, node, dtype):
     # TODO(mdan): Properly handle varargs, etc.
     template = """
-      py2tf_utils.wrap_py_func(func, dtype, (args,), kwargs, False)
+      autograph_utils.wrap_py_func(func, dtype, (args,), kwargs, False)
     """
     return templates.replace_as_expression(
         template,
@@ -238,10 +238,9 @@ class CallTreeTransformer(transformer.Base):
     # Before we could convert all the time though, we'd need a reasonable
     # caching mechanism.
     template = """
-      py2tf_api.converted_call(func, True, False, {}, args)
+      autograph_api.converted_call(func, True, False, {}, args)
     """
-    call_expr = templates.replace(
-        template, func=node.func, args=node.args)
+    call_expr = templates.replace(template, func=node.func, args=node.args)
     new_call = call_expr[0].value
     # TODO(mdan): Improve the template mechanism to better support this.
     new_call.keywords = node.keywords
diff --git a/tensorflow/contrib/py2tf/converters/call_trees_test.py b/tensorflow/contrib/autograph/converters/call_trees_test.py
similarity index 97%
rename from tensorflow/contrib/py2tf/converters/call_trees_test.py
rename to tensorflow/contrib/autograph/converters/call_trees_test.py
index 1106432da6..c666dcb73b 100644
--- a/tensorflow/contrib/py2tf/converters/call_trees_test.py
+++ b/tensorflow/contrib/autograph/converters/call_trees_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.py2tf.converters import call_trees
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import call_trees
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/py2tf/converters/continue_statements.py b/tensorflow/contrib/autograph/converters/continue_statements.py
similarity index 94%
rename from tensorflow/contrib/py2tf/converters/continue_statements.py
rename to tensorflow/contrib/autograph/converters/continue_statements.py
index 4069a678b1..4299a8a9d5 100644
--- a/tensorflow/contrib/py2tf/converters/continue_statements.py
+++ b/tensorflow/contrib/autograph/converters/continue_statements.py
@@ -18,10 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class ContinueCanonicalizationTransformer(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/converters/continue_statements_test.py b/tensorflow/contrib/autograph/converters/continue_statements_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/continue_statements_test.py
rename to tensorflow/contrib/autograph/converters/continue_statements_test.py
index a598dcd1ae..bcbb316d74 100644
--- a/tensorflow/contrib/py2tf/converters/continue_statements_test.py
+++ b/tensorflow/contrib/autograph/converters/continue_statements_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import continue_statements
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import continue_statements
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/control_flow.py b/tensorflow/contrib/autograph/converters/control_flow.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/control_flow.py
rename to tensorflow/contrib/autograph/converters/control_flow.py
index 762c26f0c7..49d932026f 100644
--- a/tensorflow/contrib/py2tf/converters/control_flow.py
+++ b/tensorflow/contrib/autograph/converters/control_flow.py
@@ -20,11 +20,11 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class SymbolNamer(object):
@@ -82,7 +82,7 @@ class ControlFlowTransformer(transformer.Base):
   def _create_cond_expr(self, results, test, body_name, orelse_name):
     if results is not None:
       template = """
-        results = py2tf_utils.run_cond(test, body_name, orelse_name)
+        results = autograph_utils.run_cond(test, body_name, orelse_name)
       """
       return templates.replace(
           template,
@@ -92,7 +92,7 @@ class ControlFlowTransformer(transformer.Base):
           orelse_name=orelse_name)
     else:
       template = """
-        py2tf_utils.run_cond(test, body_name, orelse_name)
+        autograph_utils.run_cond(test, body_name, orelse_name)
       """
       return templates.replace(
           template, test=test, body_name=body_name, orelse_name=orelse_name)
@@ -204,7 +204,7 @@ class ControlFlowTransformer(transformer.Base):
       def body_name(state_ssf):
         body
         return state_ssf,
-      state_ast_tuple = py2tf_utils.run_while(test_name, body_name, [state])
+      state_ast_tuple = autograph_utils.run_while(test_name, body_name, [state])
     """
     node = templates.replace(
         template,
diff --git a/tensorflow/contrib/py2tf/converters/control_flow_test.py b/tensorflow/contrib/autograph/converters/control_flow_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/control_flow_test.py
rename to tensorflow/contrib/autograph/converters/control_flow_test.py
index b785b284a7..86fed51f27 100644
--- a/tensorflow/contrib/py2tf/converters/control_flow_test.py
+++ b/tensorflow/contrib/autograph/converters/control_flow_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import control_flow
-from tensorflow.contrib.py2tf.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import control_flow
+from tensorflow.contrib.autograph.converters import converter_test_base
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/converters/converter_test_base.py b/tensorflow/contrib/autograph/converters/converter_test_base.py
similarity index 85%
rename from tensorflow/contrib/py2tf/converters/converter_test_base.py
rename to tensorflow/contrib/autograph/converters/converter_test_base.py
index 8c08c5492a..3ea2cfd668 100644
--- a/tensorflow/contrib/py2tf/converters/converter_test_base.py
+++ b/tensorflow/contrib/autograph/converters/converter_test_base.py
@@ -21,15 +21,15 @@ from __future__ import print_function
 import contextlib
 import imp
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import context
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import pretty_printer
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.static_analysis import activity
-from tensorflow.contrib.py2tf.pyct.static_analysis import live_values
-from tensorflow.contrib.py2tf.pyct.static_analysis import type_info
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import context
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import pretty_printer
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.static_analysis import activity
+from tensorflow.contrib.autograph.pyct.static_analysis import live_values
+from tensorflow.contrib.autograph.pyct.static_analysis import type_info
 from tensorflow.python.platform import test
 
 
@@ -75,8 +75,8 @@ class TestCase(test.TestCase):
     try:
       result, source = compiler.ast_to_object(node)
       result.tf = self.make_fake_mod('fake_tf', *symbols)
-      result.py2tf_utils = utils
-      result.py2tf_api = self.make_fake_mod('fake_api', converted_call)
+      result.autograph_utils = utils
+      result.autograph_api = self.make_fake_mod('fake_api', converted_call)
       yield result
     except Exception:  # pylint:disable=broad-except
       if source is None:
diff --git a/tensorflow/contrib/py2tf/converters/decorators.py b/tensorflow/contrib/autograph/converters/decorators.py
similarity index 96%
rename from tensorflow/contrib/py2tf/converters/decorators.py
rename to tensorflow/contrib/autograph/converters/decorators.py
index 68bf241ef3..92445f3174 100644
--- a/tensorflow/contrib/py2tf/converters/decorators.py
+++ b/tensorflow/contrib/autograph/converters/decorators.py
@@ -24,8 +24,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import pretty_printer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import pretty_printer
 
 
 class DecoratorsTransformer(gast.NodeTransformer):
diff --git a/tensorflow/contrib/py2tf/converters/decorators_test.py b/tensorflow/contrib/autograph/converters/decorators_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/decorators_test.py
rename to tensorflow/contrib/autograph/converters/decorators_test.py
index c75e546174..e67ab1cd6a 100644
--- a/tensorflow/contrib/py2tf/converters/decorators_test.py
+++ b/tensorflow/contrib/autograph/converters/decorators_test.py
@@ -20,9 +20,9 @@ from __future__ import print_function
 
 from functools import wraps
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import decorators
-from tensorflow.contrib.py2tf.pyct import compiler
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import decorators
+from tensorflow.contrib.autograph.pyct import compiler
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/for_loops.py b/tensorflow/contrib/autograph/converters/for_loops.py
similarity index 80%
rename from tensorflow/contrib/py2tf/converters/for_loops.py
rename to tensorflow/contrib/autograph/converters/for_loops.py
index 8d28b149a8..4999c47bdc 100644
--- a/tensorflow/contrib/py2tf/converters/for_loops.py
+++ b/tensorflow/contrib/autograph/converters/for_loops.py
@@ -22,10 +22,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class ForLoopCanonicalizationTransformer(transformer.Base):
@@ -45,12 +45,12 @@ class ForLoopCanonicalizationTransformer(transformer.Base):
     if anno.hasanno(node, 'extra_cond'):
       template = """
         i = 0
-        smart_loop_iter = py2tf_utils.dynamic_dataset(loop_iter)
-        cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+        smart_loop_iter = autograph_utils.dynamic_dataset(loop_iter)
+        cont, target = autograph_utils.dynamic_for_cond(i, smart_loop_iter)
         while cont and extra_cond:
           body
           i += 1
-          cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+          cont, target = autograph_utils.dynamic_for_cond(i, smart_loop_iter)
       """
       return templates.replace(
           template,
@@ -64,12 +64,12 @@ class ForLoopCanonicalizationTransformer(transformer.Base):
     else:
       template = """
         i = 0
-        smart_loop_iter = py2tf_utils.dynamic_dataset(loop_iter)
-        cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+        smart_loop_iter = autograph_utils.dynamic_dataset(loop_iter)
+        cont, target = autograph_utils.dynamic_for_cond(i, smart_loop_iter)
         while cont:
           body
           i += 1
-          cont, target = py2tf_utils.dynamic_for_cond(i, smart_loop_iter)
+          cont, target = autograph_utils.dynamic_for_cond(i, smart_loop_iter)
       """
       repl = templates.replace(
           template,
diff --git a/tensorflow/contrib/py2tf/converters/for_loops_test.py b/tensorflow/contrib/autograph/converters/for_loops_test.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/for_loops_test.py
rename to tensorflow/contrib/autograph/converters/for_loops_test.py
index b6e3e8c8d8..943f52de55 100644
--- a/tensorflow/contrib/py2tf/converters/for_loops_test.py
+++ b/tensorflow/contrib/autograph/converters/for_loops_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import for_loops
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import for_loops
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/ifexp.py b/tensorflow/contrib/autograph/converters/ifexp.py
similarity index 88%
rename from tensorflow/contrib/py2tf/converters/ifexp.py
rename to tensorflow/contrib/autograph/converters/ifexp.py
index 5fd6f348af..aff94d2b79 100644
--- a/tensorflow/contrib/py2tf/converters/ifexp.py
+++ b/tensorflow/contrib/autograph/converters/ifexp.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 class IfExp(transformer.Base):
@@ -27,7 +27,7 @@ class IfExp(transformer.Base):
 
   def visit_IfExp(self, node):
     template = """
-        py2tf_utils.run_cond(test, lambda: body, lambda: orelse)
+        autograph_utils.run_cond(test, lambda: body, lambda: orelse)
     """
     desugared_ifexp = templates.replace_as_expression(
         template, test=node.test, body=node.body, orelse=node.orelse)
diff --git a/tensorflow/contrib/py2tf/converters/ifexp_test.py b/tensorflow/contrib/autograph/converters/ifexp_test.py
similarity index 86%
rename from tensorflow/contrib/py2tf/converters/ifexp_test.py
rename to tensorflow/contrib/autograph/converters/ifexp_test.py
index 9c357ef35b..ac6849dcb4 100644
--- a/tensorflow/contrib/py2tf/converters/ifexp_test.py
+++ b/tensorflow/contrib/autograph/converters/ifexp_test.py
@@ -18,9 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import ifexp
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import ifexp
 from tensorflow.python.platform import test
 
 
@@ -38,7 +38,7 @@ class IfExpTest(converter_test_base.TestCase):
       return 1 if x else 0
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       for x in [0, 1]:
         self.assertEqual(test_fn(x), result.test_fn(x))
 
@@ -52,7 +52,7 @@ class IfExpTest(converter_test_base.TestCase):
       return y
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       result.f = f
       for x in [-2, 2]:
         self.assertEqual(test_fn(x), result.test_fn(x))
@@ -63,7 +63,7 @@ class IfExpTest(converter_test_base.TestCase):
       return x * x if x > 0 else x
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       for x in [-2, 2]:
         self.assertEqual(test_fn(x), result.test_fn(x))
 
@@ -73,7 +73,7 @@ class IfExpTest(converter_test_base.TestCase):
       return x * x if x > 0 else x if x else 1
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       for x in [-2, 0, 2]:
         self.assertEqual(test_fn(x), result.test_fn(x))
 
@@ -85,7 +85,7 @@ class IfExpTest(converter_test_base.TestCase):
       return -x
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       for x in [-2, 2, 5]:
         self.assertEqual(test_fn(x), result.test_fn(x))
 
@@ -97,7 +97,7 @@ class IfExpTest(converter_test_base.TestCase):
       return x
 
     with self.compiled_fn(test_fn) as result:
-      result.py2tf_util = utils
+      result.autograph_util = utils
       for x in [-2, 2, 5]:
         self.assertEqual(test_fn(x), result.test_fn(x))
 
diff --git a/tensorflow/contrib/py2tf/converters/list_comprehension.py b/tensorflow/contrib/autograph/converters/list_comprehension.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/list_comprehension.py
rename to tensorflow/contrib/autograph/converters/list_comprehension.py
index e874483110..d7f2920151 100644
--- a/tensorflow/contrib/py2tf/converters/list_comprehension.py
+++ b/tensorflow/contrib/autograph/converters/list_comprehension.py
@@ -31,9 +31,9 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 class ListCompCanonicalizationTransformer(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/converters/list_comprehension_test.py b/tensorflow/contrib/autograph/converters/list_comprehension_test.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/list_comprehension_test.py
rename to tensorflow/contrib/autograph/converters/list_comprehension_test.py
index 025fac11e4..4758671f5e 100644
--- a/tensorflow/contrib/py2tf/converters/list_comprehension_test.py
+++ b/tensorflow/contrib/autograph/converters/list_comprehension_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import list_comprehension
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import list_comprehension
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/converters/lists.py b/tensorflow/contrib/autograph/converters/lists.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/lists.py
rename to tensorflow/contrib/autograph/converters/lists.py
index 3e62037a50..234a0a7487 100644
--- a/tensorflow/contrib/py2tf/converters/lists.py
+++ b/tensorflow/contrib/autograph/converters/lists.py
@@ -32,9 +32,9 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 from tensorflow.python.framework import dtypes
 
 
@@ -74,7 +74,7 @@ class ListTransformer(transformer.Base):
 
       if qn.qn[-1] == 'append' and (len(call_node.args) == 1):
         template = """
-          target = py2tf_utils.dynamic_list_append(target, element)
+          target = autograph_utils.dynamic_list_append(target, element)
         """
         node = templates.replace(
             template,
diff --git a/tensorflow/contrib/py2tf/converters/lists_test.py b/tensorflow/contrib/autograph/converters/lists_test.py
similarity index 90%
rename from tensorflow/contrib/py2tf/converters/lists_test.py
rename to tensorflow/contrib/autograph/converters/lists_test.py
index 671a1cc7b1..749ba14347 100644
--- a/tensorflow/contrib/py2tf/converters/lists_test.py
+++ b/tensorflow/contrib/autograph/converters/lists_test.py
@@ -18,9 +18,9 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import lists
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import lists
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/converters/logical_expressions.py b/tensorflow/contrib/autograph/converters/logical_expressions.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/logical_expressions.py
rename to tensorflow/contrib/autograph/converters/logical_expressions.py
index e0abf74ebc..3a795a315a 100644
--- a/tensorflow/contrib/py2tf/converters/logical_expressions.py
+++ b/tensorflow/contrib/autograph/converters/logical_expressions.py
@@ -23,10 +23,10 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 # TODO(mdan): Properly extrack boolean ops according to lazy eval rules.
@@ -57,8 +57,8 @@ class LogicalExpressionTransformer(transformer.Base):
         gast.NotEq: 'tf.not_equal',
         gast.Or: 'tf.logical_or',
         gast.USub: 'tf.negative',
-        gast.Is: 'py2tf_utils.dynamic_is',
-        gast.IsNot: 'py2tf_utils.dynamic_is_not'
+        gast.Is: 'autograph_utils.dynamic_is',
+        gast.IsNot: 'autograph_utils.dynamic_is_not'
     }
 
   def _expect_simple_symbol(self, operand):
diff --git a/tensorflow/contrib/py2tf/converters/logical_expressions_test.py b/tensorflow/contrib/autograph/converters/logical_expressions_test.py
similarity index 92%
rename from tensorflow/contrib/py2tf/converters/logical_expressions_test.py
rename to tensorflow/contrib/autograph/converters/logical_expressions_test.py
index eb28c309a4..2814060c4d 100644
--- a/tensorflow/contrib/py2tf/converters/logical_expressions_test.py
+++ b/tensorflow/contrib/autograph/converters/logical_expressions_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import logical_expressions
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import logical_expressions
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/py2tf/converters/name_scopes.py b/tensorflow/contrib/autograph/converters/name_scopes.py
similarity index 93%
rename from tensorflow/contrib/py2tf/converters/name_scopes.py
rename to tensorflow/contrib/autograph/converters/name_scopes.py
index c702823fcf..2a3f474360 100644
--- a/tensorflow/contrib/py2tf/converters/name_scopes.py
+++ b/tensorflow/contrib/autograph/converters/name_scopes.py
@@ -21,8 +21,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
 
 
 class FunctionNameScopeTransformer(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/converters/name_scopes_test.py b/tensorflow/contrib/autograph/converters/name_scopes_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/converters/name_scopes_test.py
rename to tensorflow/contrib/autograph/converters/name_scopes_test.py
index a8ca341602..61e5db2af8 100644
--- a/tensorflow/contrib/py2tf/converters/name_scopes_test.py
+++ b/tensorflow/contrib/autograph/converters/name_scopes_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import name_scopes
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import name_scopes
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/converters/side_effect_guards.py b/tensorflow/contrib/autograph/converters/side_effect_guards.py
similarity index 91%
rename from tensorflow/contrib/py2tf/converters/side_effect_guards.py
rename to tensorflow/contrib/autograph/converters/side_effect_guards.py
index 30976b3ec6..1c1293d2c4 100644
--- a/tensorflow/contrib/py2tf/converters/side_effect_guards.py
+++ b/tensorflow/contrib/autograph/converters/side_effect_guards.py
@@ -36,12 +36,12 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class SymbolNamer(object):
@@ -160,8 +160,8 @@ class SideEffectGuardTransformer(transformer.Base):
               [alias_map.get(s, s).ast() for s in guarded_args], None)
 
         template = """
-          with py2tf_utils.control_dependency_on_returns(call):
-            aliased_guarded_args = py2tf_utils.alias_tensors(guarded_args)
+          with autograph_utils.control_dependency_on_returns(call):
+            aliased_guarded_args = autograph_utils.alias_tensors(guarded_args)
         """
         control_deps_guard = templates.replace(
             template,
@@ -172,7 +172,7 @@ class SideEffectGuardTransformer(transformer.Base):
         alias_map = {}
 
         template = """
-          with py2tf_utils.control_dependency_on_returns(call):
+          with autograph_utils.control_dependency_on_returns(call):
             pass
         """
         control_deps_guard = templates.replace(template, call=node.value)[-1]
diff --git a/tensorflow/contrib/py2tf/converters/side_effect_guards_test.py b/tensorflow/contrib/autograph/converters/side_effect_guards_test.py
similarity index 97%
rename from tensorflow/contrib/py2tf/converters/side_effect_guards_test.py
rename to tensorflow/contrib/autograph/converters/side_effect_guards_test.py
index 463db2e770..ce0ce33243 100644
--- a/tensorflow/contrib/py2tf/converters/side_effect_guards_test.py
+++ b/tensorflow/contrib/autograph/converters/side_effect_guards_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import side_effect_guards
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import side_effect_guards
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
diff --git a/tensorflow/contrib/py2tf/converters/single_return.py b/tensorflow/contrib/autograph/converters/single_return.py
similarity index 96%
rename from tensorflow/contrib/py2tf/converters/single_return.py
rename to tensorflow/contrib/autograph/converters/single_return.py
index 1194b98f5e..bcc9ca9dfe 100644
--- a/tensorflow/contrib/py2tf/converters/single_return.py
+++ b/tensorflow/contrib/autograph/converters/single_return.py
@@ -20,11 +20,11 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import templates
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import templates
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 # TODO(mdan): Move this logic into transformer_base.
@@ -232,7 +232,7 @@ class DetectReturnInUnsupportedControlFlow(gast.NodeVisitor):
   def visit_Return(self, node):
     if self.cant_return:
       raise ValueError(
-          'Pyflow currently does not support `return` statements in loops. '
+          '`return` statements are not supported in loops. '
           'Try assigning to a variable in the while loop, and returning '
           'outside of the loop')
 
diff --git a/tensorflow/contrib/py2tf/converters/single_return_test.py b/tensorflow/contrib/autograph/converters/single_return_test.py
similarity index 97%
rename from tensorflow/contrib/py2tf/converters/single_return_test.py
rename to tensorflow/contrib/autograph/converters/single_return_test.py
index 2ea7a9d6d3..d483005a09 100644
--- a/tensorflow/contrib/py2tf/converters/single_return_test.py
+++ b/tensorflow/contrib/autograph/converters/single_return_test.py
@@ -18,8 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.converters import converter_test_base
-from tensorflow.contrib.py2tf.converters import single_return
+from tensorflow.contrib.autograph.converters import converter_test_base
+from tensorflow.contrib.autograph.converters import single_return
 from tensorflow.python.framework.ops import name_scope
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/py2tf/impl/BUILD b/tensorflow/contrib/autograph/impl/BUILD
similarity index 82%
rename from tensorflow/contrib/py2tf/impl/BUILD
rename to tensorflow/contrib/autograph/impl/BUILD
index cc49d71b78..e468176da1 100644
--- a/tensorflow/contrib/py2tf/impl/BUILD
+++ b/tensorflow/contrib/autograph/impl/BUILD
@@ -25,10 +25,10 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
-        "//tensorflow/contrib/py2tf/converters",
-        "//tensorflow/contrib/py2tf/pyct",
-        "//tensorflow/contrib/py2tf/pyct/static_analysis",
-        "//tensorflow/contrib/py2tf/utils",
+        "//tensorflow/contrib/autograph/converters",
+        "//tensorflow/contrib/autograph/pyct",
+        "//tensorflow/contrib/autograph/pyct/static_analysis",
+        "//tensorflow/contrib/autograph/utils",
         "@gast_archive//:gast",
         "@six_archive//:six",
     ],
@@ -40,7 +40,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":impl",
-        "//tensorflow/contrib/py2tf/utils",
+        "//tensorflow/contrib/autograph/utils",
         "//tensorflow/python:client_testlib",
         "//third_party/py/numpy",
     ],
diff --git a/tensorflow/contrib/py2tf/impl/api.py b/tensorflow/contrib/autograph/impl/api.py
similarity index 95%
rename from tensorflow/contrib/py2tf/impl/api.py
rename to tensorflow/contrib/autograph/impl/api.py
index a9e8ea2043..1c4fcaa622 100644
--- a/tensorflow/contrib/py2tf/impl/api.py
+++ b/tensorflow/contrib/autograph/impl/api.py
@@ -27,13 +27,13 @@ import gast
 import six
 # pylint:enable=g-bad-import-order
 
-from tensorflow.contrib.py2tf.impl import config
-from tensorflow.contrib.py2tf.impl import conversion
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import inspect_utils
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.utils import builtins
-from tensorflow.contrib.py2tf.utils import py_func
+from tensorflow.contrib.autograph.impl import config
+from tensorflow.contrib.autograph.impl import conversion
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import inspect_utils
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.utils import builtins
+from tensorflow.contrib.autograph.utils import py_func
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_inspect
 
@@ -89,7 +89,7 @@ def do_not_convert(run_as=RunMode.GRAPH, return_dtypes=None):
   Args:
     run_as: RunMode value. Whether to run the function as-is, or wrap it into
         a py_func.
-    return_dtypes: See py2tf.utils.py_func.wrap_py_func. Setting to None or
+    return_dtypes: See autograph.utils.py_func.wrap_py_func. Setting to None or
         empty list or tuple will create a dummy return value that can be used
         to set control dependencies.
 
diff --git a/tensorflow/contrib/py2tf/impl/api_test.py b/tensorflow/contrib/autograph/impl/api_test.py
similarity index 92%
rename from tensorflow/contrib/py2tf/impl/api_test.py
rename to tensorflow/contrib/autograph/impl/api_test.py
index a7b1aba852..ee2d301d75 100644
--- a/tensorflow/contrib/py2tf/impl/api_test.py
+++ b/tensorflow/contrib/autograph/impl/api_test.py
@@ -20,11 +20,11 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.impl import api
-from tensorflow.contrib.py2tf.impl import config
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.utils import py_func
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.impl import api
+from tensorflow.contrib.autograph.impl import config
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
 from tensorflow.python.platform import test
 
@@ -37,10 +37,8 @@ class ApiTest(test.TestCase):
   def setUp(self):
     config.COMPILED_IMPORT_STATEMENTS = (
         'from __future__ import print_function',
-        'from tensorflow.contrib.py2tf import utils as '
-        'py2tf_utils',
-        'tf = py2tf_utils.fake_tf()'
-    )
+        'from tensorflow.contrib.autograph import utils as '
+        'autograph_utils', 'tf = autograph_utils.fake_tf()')
 
   def test_decorator_recurses(self):
 
@@ -200,7 +198,7 @@ class ApiTest(test.TestCase):
     compiled_code = api.to_code(test_fn)
 
     # Just check for some key words and that it is parseable Python code.
-    self.assertRegexpMatches(compiled_code, 'py2tf_utils\\.run_while')
+    self.assertRegexpMatches(compiled_code, 'autograph_utils\\.run_while')
     self.assertIsNotNone(parser.parse_str(compiled_code))
 
 
diff --git a/tensorflow/contrib/py2tf/impl/config.py b/tensorflow/contrib/autograph/impl/config.py
similarity index 79%
rename from tensorflow/contrib/py2tf/impl/config.py
rename to tensorflow/contrib/autograph/impl/config.py
index bdbc6663dd..543c1486e6 100644
--- a/tensorflow/contrib/py2tf/impl/config.py
+++ b/tensorflow/contrib/autograph/impl/config.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf import utils
+from tensorflow.contrib.autograph import utils
 
 
 PYTHON_LITERALS = {
@@ -35,16 +35,16 @@ DEFAULT_UNCOMPILED_MODULES = set((
     # All of tensorflow's subpackages. Unlike the root tf module, they don't
     # have well-known names. Not refering to the module directly to avoid
     # circular imports.
-    (utils.__name__[:-len('.contrib.py2tf.utils')],),
+    (
+        utils.__name__[:-len('.contrib.autograph.utils')],),
 ))
 
 NO_SIDE_EFFECT_CONSTRUCTORS = set(('tensorflow',))
 
 # TODO(mdan): Also allow controlling the generated names (for testability).
 COMPILED_IMPORT_STATEMENTS = (
-    'from __future__ import print_function',
-    'import tensorflow as tf',
-    'from tensorflow.contrib.py2tf.impl import api as '
-    'py2tf_api',
-    'from tensorflow.contrib.py2tf import utils as '
-    'py2tf_utils')
+    'from __future__ import print_function', 'import tensorflow as tf',
+    'from tensorflow.contrib.autograph.impl import api as '
+    'autograph_api',
+    'from tensorflow.contrib.autograph import utils as '
+    'autograph_utils')
diff --git a/tensorflow/contrib/py2tf/impl/conversion.py b/tensorflow/contrib/autograph/impl/conversion.py
similarity index 84%
rename from tensorflow/contrib/py2tf/impl/conversion.py
rename to tensorflow/contrib/autograph/impl/conversion.py
index 37b24ab55f..62a49cd92d 100644
--- a/tensorflow/contrib/py2tf/impl/conversion.py
+++ b/tensorflow/contrib/autograph/impl/conversion.py
@@ -20,31 +20,31 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.converters import asserts
-from tensorflow.contrib.py2tf.converters import break_statements
-from tensorflow.contrib.py2tf.converters import builtin_functions
-from tensorflow.contrib.py2tf.converters import call_trees
-from tensorflow.contrib.py2tf.converters import continue_statements
-from tensorflow.contrib.py2tf.converters import control_flow
-from tensorflow.contrib.py2tf.converters import decorators
-from tensorflow.contrib.py2tf.converters import for_loops
-from tensorflow.contrib.py2tf.converters import ifexp
-from tensorflow.contrib.py2tf.converters import lists
-from tensorflow.contrib.py2tf.converters import logical_expressions
-from tensorflow.contrib.py2tf.converters import name_scopes
-from tensorflow.contrib.py2tf.converters import side_effect_guards
-from tensorflow.contrib.py2tf.converters import single_return
-from tensorflow.contrib.py2tf.impl import config
-from tensorflow.contrib.py2tf.impl import naming
-from tensorflow.contrib.py2tf.pyct import context
-from tensorflow.contrib.py2tf.pyct import inspect_utils
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.static_analysis import activity
-from tensorflow.contrib.py2tf.pyct.static_analysis import live_values
-from tensorflow.contrib.py2tf.pyct.static_analysis import type_info
-from tensorflow.contrib.py2tf.utils import type_hints
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.converters import asserts
+from tensorflow.contrib.autograph.converters import break_statements
+from tensorflow.contrib.autograph.converters import builtin_functions
+from tensorflow.contrib.autograph.converters import call_trees
+from tensorflow.contrib.autograph.converters import continue_statements
+from tensorflow.contrib.autograph.converters import control_flow
+from tensorflow.contrib.autograph.converters import decorators
+from tensorflow.contrib.autograph.converters import for_loops
+from tensorflow.contrib.autograph.converters import ifexp
+from tensorflow.contrib.autograph.converters import lists
+from tensorflow.contrib.autograph.converters import logical_expressions
+from tensorflow.contrib.autograph.converters import name_scopes
+from tensorflow.contrib.autograph.converters import side_effect_guards
+from tensorflow.contrib.autograph.converters import single_return
+from tensorflow.contrib.autograph.impl import config
+from tensorflow.contrib.autograph.impl import naming
+from tensorflow.contrib.autograph.pyct import context
+from tensorflow.contrib.autograph.pyct import inspect_utils
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.static_analysis import activity
+from tensorflow.contrib.autograph.pyct.static_analysis import live_values
+from tensorflow.contrib.autograph.pyct.static_analysis import type_info
+from tensorflow.contrib.autograph.utils import type_hints
 from tensorflow.python.util import tf_inspect
 
 
@@ -213,19 +213,19 @@ def class_to_graph(c, conversion_map):
 def _add_self_references(namespace, api_module):
   """Self refs are only required for analysis and are not used directly."""
   # Manually add the utils namespace which may be used from generated code.
-  if 'py2tf_util' not in namespace:
-    namespace['py2tf_utils'] = utils
-  elif namespace['py2tf_utils'] != utils:
+  if 'autograph_util' not in namespace:
+    namespace['autograph_utils'] = utils
+  elif namespace['autograph_utils'] != utils:
     raise ValueError(
-        'The module name "py2tf_utils" is reserved and may not be used.')
+        'The module name "autograph_utils" is reserved and may not be used.')
 
   # We also make reference to the api module for dynamic conversion, but
   # to avoid circular references we don't import it here.
-  if 'py2tf_api' not in namespace:
-    namespace['py2tf_api'] = api_module
-  elif namespace['py2tf_api'] != api_module:
+  if 'autograph_api' not in namespace:
+    namespace['autograph_api'] = api_module
+  elif namespace['autograph_api'] != api_module:
     raise ValueError(
-        'The module name "py2tf_api" is reserved and may not be used.')
+        'The module name "autograph_api" is reserved and may not be used.')
 
 
 def function_to_graph(f, conversion_map, arg_values, arg_types,
diff --git a/tensorflow/contrib/py2tf/impl/conversion_test.py b/tensorflow/contrib/autograph/impl/conversion_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/impl/conversion_test.py
rename to tensorflow/contrib/autograph/impl/conversion_test.py
index 9ff256aace..7066739eb8 100644
--- a/tensorflow/contrib/py2tf/impl/conversion_test.py
+++ b/tensorflow/contrib/autograph/impl/conversion_test.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.impl import conversion
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.impl import conversion
 from tensorflow.python.framework import constant_op
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/py2tf/impl/naming.py b/tensorflow/contrib/autograph/impl/naming.py
similarity index 98%
rename from tensorflow/contrib/py2tf/impl/naming.py
rename to tensorflow/contrib/autograph/impl/naming.py
index 51326091de..1facaa0ca0 100644
--- a/tensorflow/contrib/py2tf/impl/naming.py
+++ b/tensorflow/contrib/autograph/impl/naming.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.pyct import qual_names
+from tensorflow.contrib.autograph.pyct import qual_names
 
 
 class Namer(object):
diff --git a/tensorflow/contrib/py2tf/impl/naming_test.py b/tensorflow/contrib/autograph/impl/naming_test.py
similarity index 98%
rename from tensorflow/contrib/py2tf/impl/naming_test.py
rename to tensorflow/contrib/autograph/impl/naming_test.py
index beb4e54937..73fc089465 100644
--- a/tensorflow/contrib/py2tf/impl/naming_test.py
+++ b/tensorflow/contrib/autograph/impl/naming_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.impl import naming
+from tensorflow.contrib.autograph.impl import naming
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/BUILD b/tensorflow/contrib/autograph/pyct/BUILD
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/BUILD
rename to tensorflow/contrib/autograph/pyct/BUILD
diff --git a/tensorflow/contrib/py2tf/pyct/__init__.py b/tensorflow/contrib/autograph/pyct/__init__.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/__init__.py
rename to tensorflow/contrib/autograph/pyct/__init__.py
diff --git a/tensorflow/contrib/py2tf/pyct/anno.py b/tensorflow/contrib/autograph/pyct/anno.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/anno.py
rename to tensorflow/contrib/autograph/pyct/anno.py
diff --git a/tensorflow/contrib/py2tf/pyct/anno_test.py b/tensorflow/contrib/autograph/pyct/anno_test.py
similarity index 97%
rename from tensorflow/contrib/py2tf/pyct/anno_test.py
rename to tensorflow/contrib/autograph/pyct/anno_test.py
index 6c29918fdf..1d4d9d119e 100644
--- a/tensorflow/contrib/py2tf/pyct/anno_test.py
+++ b/tensorflow/contrib/autograph/pyct/anno_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import ast
 
-from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.autograph.pyct import anno
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/ast_util.py b/tensorflow/contrib/autograph/pyct/ast_util.py
similarity index 98%
rename from tensorflow/contrib/py2tf/pyct/ast_util.py
rename to tensorflow/contrib/autograph/pyct/ast_util.py
index 6f7e656c26..5a41b5e4a9 100644
--- a/tensorflow/contrib/py2tf/pyct/ast_util.py
+++ b/tensorflow/contrib/autograph/pyct/ast_util.py
@@ -22,7 +22,7 @@ import ast
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.autograph.pyct import anno
 
 
 class CleanCopier(gast.NodeVisitor):
diff --git a/tensorflow/contrib/py2tf/pyct/ast_util_test.py b/tensorflow/contrib/autograph/pyct/ast_util_test.py
similarity index 93%
rename from tensorflow/contrib/py2tf/pyct/ast_util_test.py
rename to tensorflow/contrib/autograph/pyct/ast_util_test.py
index 8d123679e3..8faf92c705 100644
--- a/tensorflow/contrib/py2tf/pyct/ast_util_test.py
+++ b/tensorflow/contrib/autograph/pyct/ast_util_test.py
@@ -20,10 +20,10 @@ from __future__ import print_function
 
 import ast
 
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/compiler.py b/tensorflow/contrib/autograph/pyct/compiler.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/compiler.py
rename to tensorflow/contrib/autograph/pyct/compiler.py
diff --git a/tensorflow/contrib/py2tf/pyct/compiler_test.py b/tensorflow/contrib/autograph/pyct/compiler_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/compiler_test.py
rename to tensorflow/contrib/autograph/pyct/compiler_test.py
index 243f4c8153..98cdc1506b 100644
--- a/tensorflow/contrib/py2tf/pyct/compiler_test.py
+++ b/tensorflow/contrib/autograph/pyct/compiler_test.py
@@ -22,8 +22,8 @@ import textwrap
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import parser
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import parser
 from tensorflow.python.platform import test
 from tensorflow.python.util import tf_inspect
 
diff --git a/tensorflow/contrib/py2tf/pyct/context.py b/tensorflow/contrib/autograph/pyct/context.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/context.py
rename to tensorflow/contrib/autograph/pyct/context.py
diff --git a/tensorflow/contrib/py2tf/pyct/inspect_utils.py b/tensorflow/contrib/autograph/pyct/inspect_utils.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/inspect_utils.py
rename to tensorflow/contrib/autograph/pyct/inspect_utils.py
diff --git a/tensorflow/contrib/py2tf/pyct/inspect_utils_test.py b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py
similarity index 98%
rename from tensorflow/contrib/py2tf/pyct/inspect_utils_test.py
rename to tensorflow/contrib/autograph/pyct/inspect_utils_test.py
index 5528ac851f..ddca6f963b 100644
--- a/tensorflow/contrib/py2tf/pyct/inspect_utils_test.py
+++ b/tensorflow/contrib/autograph/pyct/inspect_utils_test.py
@@ -22,7 +22,7 @@ from functools import wraps
 
 import six
 
-from tensorflow.contrib.py2tf.pyct import inspect_utils
+from tensorflow.contrib.autograph.pyct import inspect_utils
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/parser.py b/tensorflow/contrib/autograph/pyct/parser.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/parser.py
rename to tensorflow/contrib/autograph/pyct/parser.py
diff --git a/tensorflow/contrib/py2tf/pyct/parser_test.py b/tensorflow/contrib/autograph/pyct/parser_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/parser_test.py
rename to tensorflow/contrib/autograph/pyct/parser_test.py
index c58ffc7e0c..007a4c6fb0 100644
--- a/tensorflow/contrib/py2tf/pyct/parser_test.py
+++ b/tensorflow/contrib/autograph/pyct/parser_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import textwrap
 
-from tensorflow.contrib.py2tf.pyct import parser
+from tensorflow.contrib.autograph.pyct import parser
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/pretty_printer.py b/tensorflow/contrib/autograph/pyct/pretty_printer.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/pretty_printer.py
rename to tensorflow/contrib/autograph/pyct/pretty_printer.py
diff --git a/tensorflow/contrib/py2tf/pyct/pretty_printer_test.py b/tensorflow/contrib/autograph/pyct/pretty_printer_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/pretty_printer_test.py
rename to tensorflow/contrib/autograph/pyct/pretty_printer_test.py
index 81e3f47b80..0cb48f3576 100644
--- a/tensorflow/contrib/py2tf/pyct/pretty_printer_test.py
+++ b/tensorflow/contrib/autograph/pyct/pretty_printer_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import ast
 
-from tensorflow.contrib.py2tf.pyct import pretty_printer
+from tensorflow.contrib.autograph.pyct import pretty_printer
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/qual_names.py b/tensorflow/contrib/autograph/pyct/qual_names.py
similarity index 99%
rename from tensorflow/contrib/py2tf/pyct/qual_names.py
rename to tensorflow/contrib/autograph/pyct/qual_names.py
index 7dec13db92..4d5764a974 100644
--- a/tensorflow/contrib/py2tf/pyct/qual_names.py
+++ b/tensorflow/contrib/autograph/pyct/qual_names.py
@@ -29,7 +29,7 @@ import collections
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
+from tensorflow.contrib.autograph.pyct import anno
 
 
 class Symbol(collections.namedtuple('Symbol', ['name'])):
diff --git a/tensorflow/contrib/py2tf/pyct/qual_names_test.py b/tensorflow/contrib/autograph/pyct/qual_names_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/qual_names_test.py
rename to tensorflow/contrib/autograph/pyct/qual_names_test.py
index 6583fa243b..103bd25aa3 100644
--- a/tensorflow/contrib/py2tf/pyct/qual_names_test.py
+++ b/tensorflow/contrib/autograph/pyct/qual_names_test.py
@@ -20,11 +20,11 @@ from __future__ import print_function
 
 import textwrap
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.qual_names import QN
-from tensorflow.contrib.py2tf.pyct.qual_names import resolve
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.qual_names import QN
+from tensorflow.contrib.autograph.pyct.qual_names import resolve
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/BUILD b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD
similarity index 83%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/BUILD
rename to tensorflow/contrib/autograph/pyct/static_analysis/BUILD
index 2799b56a00..d192bc7aab 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/BUILD
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD
@@ -25,7 +25,7 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "@gast_archive//:gast",
     ],
 )
@@ -36,7 +36,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":static_analysis",
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/python:client_testlib",
         "@gast_archive//:gast",
     ],
@@ -48,7 +48,7 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":static_analysis",
-        "//tensorflow/contrib/py2tf/pyct",
+        "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/python:client_testlib",
     ],
 )
@@ -59,8 +59,8 @@ py_test(
     srcs_version = "PY2AND3",
     deps = [
         ":static_analysis",
-        "//tensorflow/contrib/py2tf/pyct",
-        "//tensorflow/contrib/py2tf/utils",
+        "//tensorflow/contrib/autograph/pyct",
+        "//tensorflow/contrib/autograph/utils",
         "//tensorflow/python:client_testlib",
     ],
 )
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/__init__.py b/tensorflow/contrib/autograph/pyct/static_analysis/__init__.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/__init__.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/__init__.py
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py
similarity index 97%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/activity.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/activity.py
index 716672a53b..da6a2f6f05 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/activity.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity.py
@@ -22,10 +22,10 @@ import copy
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.qual_names import QN
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.qual_names import QN
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 # TODO(mdan): Add support for PY3 (e.g. Param vs arg).
 
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/activity_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py
similarity index 95%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/activity_test.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py
index b16d15b39d..37c28872bb 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/activity_test.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/activity_test.py
@@ -20,13 +20,13 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import context
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.qual_names import QN
-from tensorflow.contrib.py2tf.pyct.static_analysis import activity
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import context
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.qual_names import QN
+from tensorflow.contrib.autograph.pyct.static_analysis import activity
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/annos.py b/tensorflow/contrib/autograph/pyct/static_analysis/annos.py
similarity index 100%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/annos.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/annos.py
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
index ac5697900a..5f813355e6 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
@@ -25,9 +25,9 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import transformer
-from tensorflow.contrib.py2tf.pyct.static_analysis.annos import NodeAnno
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import transformer
+from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class LiveValueResolver(transformer.Base):
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
similarity index 89%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
index a56dff824e..b66439624e 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/live_values_test.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
@@ -18,13 +18,13 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import context
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.static_analysis import activity
-from tensorflow.contrib.py2tf.pyct.static_analysis import live_values
-from tensorflow.contrib.py2tf.pyct.static_analysis import type_info
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import context
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.static_analysis import activity
+from tensorflow.contrib.autograph.pyct.static_analysis import live_values
+from tensorflow.contrib.autograph.pyct.static_analysis import type_info
 from tensorflow.python.framework import constant_op
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py b/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py
similarity index 98%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/type_info.py
index a969adbeca..203aa3c3d1 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/type_info.py
@@ -43,8 +43,8 @@ from __future__ import print_function
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import transformer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import transformer
 from tensorflow.python.util import tf_inspect
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py
similarity index 93%
rename from tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py
rename to tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py
index 8a8956197d..c0de4a6043 100644
--- a/tensorflow/contrib/py2tf/pyct/static_analysis/type_info_test.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/type_info_test.py
@@ -18,14 +18,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf import utils
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import context
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
-from tensorflow.contrib.py2tf.pyct.static_analysis import activity
-from tensorflow.contrib.py2tf.pyct.static_analysis import live_values
-from tensorflow.contrib.py2tf.pyct.static_analysis import type_info
+from tensorflow.contrib.autograph import utils
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import context
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
+from tensorflow.contrib.autograph.pyct.static_analysis import activity
+from tensorflow.contrib.autograph.pyct.static_analysis import live_values
+from tensorflow.contrib.autograph.pyct.static_analysis import type_info
 from tensorflow.python.client import session
 from tensorflow.python.platform import test
 from tensorflow.python.training import training
diff --git a/tensorflow/contrib/py2tf/pyct/templates.py b/tensorflow/contrib/autograph/pyct/templates.py
similarity index 98%
rename from tensorflow/contrib/py2tf/pyct/templates.py
rename to tensorflow/contrib/autograph/pyct/templates.py
index 590be68234..fb99e0d4e5 100644
--- a/tensorflow/contrib/py2tf/pyct/templates.py
+++ b/tensorflow/contrib/autograph/pyct/templates.py
@@ -26,9 +26,9 @@ import textwrap
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import ast_util
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import qual_names
+from tensorflow.contrib.autograph.pyct import ast_util
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import qual_names
 
 
 class ReplaceTransformer(gast.NodeTransformer):
diff --git a/tensorflow/contrib/py2tf/pyct/templates_test.py b/tensorflow/contrib/autograph/pyct/templates_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/pyct/templates_test.py
rename to tensorflow/contrib/autograph/pyct/templates_test.py
index af939caf32..a01f8bf04c 100644
--- a/tensorflow/contrib/py2tf/pyct/templates_test.py
+++ b/tensorflow/contrib/autograph/pyct/templates_test.py
@@ -22,9 +22,9 @@ import imp
 
 import gast
 
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import parser
-from tensorflow.contrib.py2tf.pyct import templates
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import parser
+from tensorflow.contrib.autograph.pyct import templates
 from tensorflow.python.platform import test
 
 
diff --git a/tensorflow/contrib/py2tf/pyct/transformer.py b/tensorflow/contrib/autograph/pyct/transformer.py
similarity index 89%
rename from tensorflow/contrib/py2tf/pyct/transformer.py
rename to tensorflow/contrib/autograph/pyct/transformer.py
index 31ef7e1c05..35f114b6e1 100644
--- a/tensorflow/contrib/py2tf/pyct/transformer.py
+++ b/tensorflow/contrib/autograph/pyct/transformer.py
@@ -23,12 +23,12 @@ import sys
 import gast
 import six
 
-from tensorflow.contrib.py2tf.pyct import anno
-from tensorflow.contrib.py2tf.pyct import compiler
-from tensorflow.contrib.py2tf.pyct import pretty_printer
+from tensorflow.contrib.autograph.pyct import anno
+from tensorflow.contrib.autograph.pyct import compiler
+from tensorflow.contrib.autograph.pyct import pretty_printer
 
 
-class PyFlowParseError(SyntaxError):
+class AutographParseError(SyntaxError):
   pass
 
 
@@ -77,8 +77,8 @@ class Base(gast.NodeTransformer):
         line = source_code.splitlines()[self._lineno - 1]
       else:
         line = '<no source available>'
-      six.reraise(PyFlowParseError,
-                  PyFlowParseError(
+      six.reraise(AutographParseError,
+                  AutographParseError(
                       msg,
                       (source_file, self._lineno, self._col_offset + 1, line)),
                   sys.exc_info()[2])
diff --git a/tensorflow/contrib/py2tf/utils/BUILD b/tensorflow/contrib/autograph/utils/BUILD
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/BUILD
rename to tensorflow/contrib/autograph/utils/BUILD
diff --git a/tensorflow/contrib/autograph/utils/__init__.py b/tensorflow/contrib/autograph/utils/__init__.py
new file mode 100644
index 0000000000..22898b17e9
--- /dev/null
+++ b/tensorflow/contrib/autograph/utils/__init__.py
@@ -0,0 +1,36 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility module that contains APIs usable in the generated code."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.autograph.utils.builtins import dynamic_builtin
+from tensorflow.contrib.autograph.utils.builtins import dynamic_dataset
+from tensorflow.contrib.autograph.utils.builtins import dynamic_for_cond
+from tensorflow.contrib.autograph.utils.builtins import dynamic_print
+from tensorflow.contrib.autograph.utils.builtins import dynamic_range
+from tensorflow.contrib.autograph.utils.context_managers import control_dependency_on_returns
+from tensorflow.contrib.autograph.utils.misc import alias_tensors
+from tensorflow.contrib.autograph.utils.multiple_dispatch import dynamic_is
+from tensorflow.contrib.autograph.utils.multiple_dispatch import dynamic_is_not
+from tensorflow.contrib.autograph.utils.multiple_dispatch import run_cond
+from tensorflow.contrib.autograph.utils.multiple_dispatch import run_while
+from tensorflow.contrib.autograph.utils.py_func import wrap_py_func
+from tensorflow.contrib.autograph.utils.tensor_list import dynamic_list_append
+from tensorflow.contrib.autograph.utils.testing import fake_tf
+from tensorflow.contrib.autograph.utils.type_check import is_tensor
+from tensorflow.contrib.autograph.utils.type_hints import set_element_type
diff --git a/tensorflow/contrib/py2tf/utils/builtins.py b/tensorflow/contrib/autograph/utils/builtins.py
similarity index 98%
rename from tensorflow/contrib/py2tf/utils/builtins.py
rename to tensorflow/contrib/autograph/utils/builtins.py
index 251b4ed8ee..4ab32ee47d 100644
--- a/tensorflow/contrib/py2tf/utils/builtins.py
+++ b/tensorflow/contrib/autograph/utils/builtins.py
@@ -20,8 +20,8 @@ from __future__ import print_function
 
 import six
 
-from tensorflow.contrib.py2tf.utils import py_func
-from tensorflow.contrib.py2tf.utils import type_check
+from tensorflow.contrib.autograph.utils import py_func
+from tensorflow.contrib.autograph.utils import type_check
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
diff --git a/tensorflow/contrib/py2tf/utils/builtins_test.py b/tensorflow/contrib/autograph/utils/builtins_test.py
similarity index 98%
rename from tensorflow/contrib/py2tf/utils/builtins_test.py
rename to tensorflow/contrib/autograph/utils/builtins_test.py
index 59b3573d38..d9f7913d89 100644
--- a/tensorflow/contrib/py2tf/utils/builtins_test.py
+++ b/tensorflow/contrib/autograph/utils/builtins_test.py
@@ -22,7 +22,7 @@ import sys
 
 import six
 
-from tensorflow.contrib.py2tf.utils import builtins
+from tensorflow.contrib.autograph.utils import builtins
 from tensorflow.python.framework import constant_op
 from tensorflow.python.platform import test
 
diff --git a/tensorflow/contrib/py2tf/utils/context_managers.py b/tensorflow/contrib/autograph/utils/context_managers.py
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/context_managers.py
rename to tensorflow/contrib/autograph/utils/context_managers.py
diff --git a/tensorflow/contrib/py2tf/utils/context_managers_test.py b/tensorflow/contrib/autograph/utils/context_managers_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/utils/context_managers_test.py
rename to tensorflow/contrib/autograph/utils/context_managers_test.py
index 404f6e44e5..42e27724b9 100644
--- a/tensorflow/contrib/py2tf/utils/context_managers_test.py
+++ b/tensorflow/contrib/autograph/utils/context_managers_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.utils import context_managers
+from tensorflow.contrib.autograph.utils import context_managers
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import tensor_array_ops
diff --git a/tensorflow/contrib/py2tf/utils/misc.py b/tensorflow/contrib/autograph/utils/misc.py
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/misc.py
rename to tensorflow/contrib/autograph/utils/misc.py
diff --git a/tensorflow/contrib/py2tf/utils/misc_test.py b/tensorflow/contrib/autograph/utils/misc_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/utils/misc_test.py
rename to tensorflow/contrib/autograph/utils/misc_test.py
index 8aedd4cd64..71e358c33e 100644
--- a/tensorflow/contrib/py2tf/utils/misc_test.py
+++ b/tensorflow/contrib/autograph/utils/misc_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.utils.misc import alias_tensors
+from tensorflow.contrib.autograph.utils.misc import alias_tensors
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.ops.variables import Variable
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/utils/multiple_dispatch.py b/tensorflow/contrib/autograph/utils/multiple_dispatch.py
similarity index 95%
rename from tensorflow/contrib/py2tf/utils/multiple_dispatch.py
rename to tensorflow/contrib/autograph/utils/multiple_dispatch.py
index 427a936c35..b756ccfaee 100644
--- a/tensorflow/contrib/py2tf/utils/multiple_dispatch.py
+++ b/tensorflow/contrib/autograph/utils/multiple_dispatch.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utilities for type-dependent behavior used in py2tf-generated code."""
+"""Utilities for type-dependent behavior used in autograph-generated code."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import six
 
-from tensorflow.contrib.py2tf.utils.type_check import is_tensor
+from tensorflow.contrib.autograph.utils.type_check import is_tensor
 from tensorflow.python.ops import control_flow_ops
 
 
diff --git a/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py b/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
similarity index 98%
rename from tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py
rename to tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
index 75e8fdd5ed..8c7daa6ded 100644
--- a/tensorflow/contrib/py2tf/utils/multiple_dispatch_test.py
+++ b/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy as np
 
-from tensorflow.contrib.py2tf.utils import multiple_dispatch
+from tensorflow.contrib.autograph.utils import multiple_dispatch
 from tensorflow.python.client.session import Session
 from tensorflow.python.framework.constant_op import constant
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/utils/py_func.py b/tensorflow/contrib/autograph/utils/py_func.py
similarity index 97%
rename from tensorflow/contrib/py2tf/utils/py_func.py
rename to tensorflow/contrib/autograph/utils/py_func.py
index 34f2a8b70b..11ebfb2e49 100644
--- a/tensorflow/contrib/py2tf/utils/py_func.py
+++ b/tensorflow/contrib/autograph/utils/py_func.py
@@ -118,9 +118,8 @@ def wrap_py_func(f, return_dtypes, args, kwargs=None, use_dummy_return=False):
       assert isinstance(return_dtypes, dtypes.DType)
 
   def f_wrapper(*tensor_args):
-    f_args = tuple(
-        tensor_args[tensor_args_idx[i]] if arg_is_tensor[i] else a
-        for i, a in enumerate(args))
+    f_args = tuple(tensor_args[tensor_args_idx[i]] if arg_is_tensor[i] else a
+                   for i, a in enumerate(args))
     f_kwargs = {
         k: tensor_args[tensor_args_idx[k]] if kwarg_is_tensor[k] else kwargs[k]
         for i, k in enumerate(kwarg_keys)
diff --git a/tensorflow/contrib/py2tf/utils/py_func_test.py b/tensorflow/contrib/autograph/utils/py_func_test.py
similarity index 98%
rename from tensorflow/contrib/py2tf/utils/py_func_test.py
rename to tensorflow/contrib/autograph/utils/py_func_test.py
index 3b7a35365a..2468263142 100644
--- a/tensorflow/contrib/py2tf/utils/py_func_test.py
+++ b/tensorflow/contrib/autograph/utils/py_func_test.py
@@ -18,7 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.utils import py_func
+from tensorflow.contrib.autograph.utils import py_func
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/utils/tensor_list.py b/tensorflow/contrib/autograph/utils/tensor_list.py
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/tensor_list.py
rename to tensorflow/contrib/autograph/utils/tensor_list.py
diff --git a/tensorflow/contrib/py2tf/utils/tensor_list_test.py b/tensorflow/contrib/autograph/utils/tensor_list_test.py
similarity index 97%
rename from tensorflow/contrib/py2tf/utils/tensor_list_test.py
rename to tensorflow/contrib/autograph/utils/tensor_list_test.py
index 110e4d105e..d58489eb68 100644
--- a/tensorflow/contrib/py2tf/utils/tensor_list_test.py
+++ b/tensorflow/contrib/autograph/utils/tensor_list_test.py
@@ -12,13 +12,13 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Tests for PyFlow list."""
+"""Tests for Autograph lists."""
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.py2tf.utils import tensor_list as tl
+from tensorflow.contrib.autograph.utils import tensor_list as tl
 from tensorflow.python.client.session import Session
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
diff --git a/tensorflow/contrib/py2tf/utils/testing.py b/tensorflow/contrib/autograph/utils/testing.py
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/testing.py
rename to tensorflow/contrib/autograph/utils/testing.py
diff --git a/tensorflow/contrib/py2tf/utils/type_check.py b/tensorflow/contrib/autograph/utils/type_check.py
similarity index 95%
rename from tensorflow/contrib/py2tf/utils/type_check.py
rename to tensorflow/contrib/autograph/utils/type_check.py
index b9b2b451a4..8748abc47b 100644
--- a/tensorflow/contrib/py2tf/utils/type_check.py
+++ b/tensorflow/contrib/autograph/utils/type_check.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Utilities used in py2tf-generated code."""
+"""Utilities used in autograph-generated code."""
 
 from __future__ import absolute_import
 from __future__ import division
diff --git a/tensorflow/contrib/py2tf/utils/type_check_test.py b/tensorflow/contrib/autograph/utils/type_check_test.py
similarity index 96%
rename from tensorflow/contrib/py2tf/utils/type_check_test.py
rename to tensorflow/contrib/autograph/utils/type_check_test.py
index 7d0428e9cc..3b67b7194c 100644
--- a/tensorflow/contrib/py2tf/utils/type_check_test.py
+++ b/tensorflow/contrib/autograph/utils/type_check_test.py
@@ -20,7 +20,7 @@ from __future__ import print_function
 
 import numpy
 
-from tensorflow.contrib.py2tf.utils import type_check
+from tensorflow.contrib.autograph.utils import type_check
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import test
diff --git a/tensorflow/contrib/py2tf/utils/type_hints.py b/tensorflow/contrib/autograph/utils/type_hints.py
similarity index 100%
rename from tensorflow/contrib/py2tf/utils/type_hints.py
rename to tensorflow/contrib/autograph/utils/type_hints.py
diff --git a/tensorflow/contrib/py2tf/utils/__init__.py b/tensorflow/contrib/py2tf/utils/__init__.py
deleted file mode 100644
index 4e6003c852..0000000000
--- a/tensorflow/contrib/py2tf/utils/__init__.py
+++ /dev/null
@@ -1,36 +0,0 @@
-# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Utility module that contains APIs usable in the generated code."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-from tensorflow.contrib.py2tf.utils.builtins import dynamic_builtin
-from tensorflow.contrib.py2tf.utils.builtins import dynamic_dataset
-from tensorflow.contrib.py2tf.utils.builtins import dynamic_for_cond
-from tensorflow.contrib.py2tf.utils.builtins import dynamic_print
-from tensorflow.contrib.py2tf.utils.builtins import dynamic_range
-from tensorflow.contrib.py2tf.utils.context_managers import control_dependency_on_returns
-from tensorflow.contrib.py2tf.utils.misc import alias_tensors
-from tensorflow.contrib.py2tf.utils.multiple_dispatch import dynamic_is
-from tensorflow.contrib.py2tf.utils.multiple_dispatch import dynamic_is_not
-from tensorflow.contrib.py2tf.utils.multiple_dispatch import run_cond
-from tensorflow.contrib.py2tf.utils.multiple_dispatch import run_while
-from tensorflow.contrib.py2tf.utils.py_func import wrap_py_func
-from tensorflow.contrib.py2tf.utils.tensor_list import dynamic_list_append
-from tensorflow.contrib.py2tf.utils.testing import fake_tf
-from tensorflow.contrib.py2tf.utils.type_check import is_tensor
-from tensorflow.contrib.py2tf.utils.type_hints import set_element_type
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 8a80d6443b..e01306f953 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -164,12 +164,12 @@ sh_binary(
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
             "//tensorflow/contrib/nn:nn_py",
             "//tensorflow/contrib/predictor:predictor_pip",
-            "//tensorflow/contrib/py2tf:py2tf",
-            "//tensorflow/contrib/py2tf/converters:converters",
-            "//tensorflow/contrib/py2tf/converters:test_lib",
-            "//tensorflow/contrib/py2tf/impl:impl",
-            "//tensorflow/contrib/py2tf/pyct:pyct",
-            "//tensorflow/contrib/py2tf/pyct/static_analysis:static_analysis",
+            "//tensorflow/contrib/autograph:autograph",
+            "//tensorflow/contrib/autograph/converters:converters",
+            "//tensorflow/contrib/autograph/converters:test_lib",
+            "//tensorflow/contrib/autograph/impl:impl",
+            "//tensorflow/contrib/autograph/pyct:pyct",
+            "//tensorflow/contrib/autograph/pyct/static_analysis:static_analysis",
             "//tensorflow/contrib/receptive_field:receptive_field_pip",
             "//tensorflow/contrib/session_bundle:session_bundle_pip",
             "//tensorflow/contrib/signal:signal_py",
-- 
GitLab


From 710ba88846c9aca71ad1f83000255db4d3bb17e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 13:41:35 -0700
Subject: [PATCH 505/960] Quick fix to assign_moving_average documentation
 formatting.

PiperOrigin-RevId: 190517622
---
 tensorflow/python/training/moving_averages.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/training/moving_averages.py b/tensorflow/python/training/moving_averages.py
index b9ecb27df1..61fc828a84 100644
--- a/tensorflow/python/training/moving_averages.py
+++ b/tensorflow/python/training/moving_averages.py
@@ -52,16 +52,19 @@ def assign_moving_average(variable, value, decay, zero_debias=True, name=None):
   they were created in and the scope of the variables they debias. They are also
   given a uniqifying-suffix.
 
-  Ex:
+  E.g.:
+
+  ```
     with tf.variable_scope('scope1'):
       with tf.variable_scope('scope2'):
         var = tf.get_variable('foo')
-        assign_moving_average(var, 0.0, 1.0)
-        assign_moving_average(var, 0.0, 0.9)
+        tf.assign_moving_average(var, 0.0, 1.0)
+        tf.assign_moving_average(var, 0.0, 0.9)
 
-    var.name: 'scope1/scope2/foo'
-    shadow var names: 'scope1/scope2/scope1/scope2/foo/biased'
-                      'scope1/scope2/scope1/scope2/foo/biased_1'
+    # var.name: 'scope1/scope2/foo'
+    # shadow var names: 'scope1/scope2/scope1/scope2/foo/biased'
+    #                   'scope1/scope2/scope1/scope2/foo/biased_1'
+  ```
 
   Args:
     variable: A Variable.
-- 
GitLab


From 1fcef75aaa1989376324ff8dfc25033b443a69df Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Mon, 26 Mar 2018 13:48:00 -0700
Subject: [PATCH 506/960] Update BUILD

---
 tensorflow/contrib/timeseries/python/timeseries/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index d72cc1b8a2..67ee644d3b 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -233,7 +233,7 @@ py_test(
     ],
     srcs_version = "PY2AND3",
     tags = [
-        "manual",
+        "no_oss",
         "no_pip",  # b/64527635
         "no_pip_gpu",  # b/63391119
     ],
-- 
GitLab


From 72ed3c3b743e5feef99e37058dbd2f4344bcc5e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 14:04:35 -0700
Subject: [PATCH 507/960] Add description of shapes and a pointer to external
 tutorial notebook in `tf.distributions.Distribution`.

PiperOrigin-RevId: 190521666
---
 .../python/ops/distributions/distribution.py  | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tensorflow/python/ops/distributions/distribution.py b/tensorflow/python/ops/distributions/distribution.py
index 0866fa8b0b..7c43bf54fc 100644
--- a/tensorflow/python/ops/distributions/distribution.py
+++ b/tensorflow/python/ops/distributions/distribution.py
@@ -338,6 +338,27 @@ class Distribution(_BaseDistribution):
   cum_prob_invalid = u.cdf([4.0, 5.0, 6.0])
   ```
 
+  #### Shapes
+
+  There are three important concepts associated with TensorFlow Distributions
+  shapes:
+  - Event shape describes the shape of a single draw from the distribution;
+    it may be dependent across dimensions. For scalar distributions, the event
+    shape is `[]`. For a 5-dimensional MultivariateNormal, the event shape is
+    `[5]`.
+  - Batch shape describes independent, not identically distributed draws, aka a
+    "collection" or "bunch" of distributions.
+  - Sample shape describes independent, identically distributed draws of batches
+    from the distribution family.
+
+  The event shape and the batch shape are properties of a Distribution object,
+  whereas the sample shape is associated with a specific call to `sample` or
+  `log_prob`.
+
+  For detailed usage examples of TensorFlow Distributions shapes, see
+  [this tutorial](
+  https://github.com/tensorflow/probability/blob/master/tensorflow_probability/examples/jupyter_notebooks/Understanding%20TensorFlow%20Distributions%20Shapes.ipynb)
+
   #### Parameter values leading to undefined statistics or distributions.
 
   Some distributions do not have well-defined statistics for all initialization
-- 
GitLab


From 2ff8e913ad000d379405c284857e7fc81eef9fed Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 26 Mar 2018 14:33:10 -0700
Subject: [PATCH 508/960] Clarify eager gradient doc strings

PiperOrigin-RevId: 190526387
---
 tensorflow/python/eager/backprop.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index a7837b8a7f..c54a5a1445 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -171,8 +171,8 @@ def implicit_val_and_grad(f):
   """Returns a function which differentiates f with respect to variables.
 
   The wrapped function returns the value and the gradient of f when called with
-  the same arguments. The gradient is with respect to all TFE variables which
-  are either trainable or have `variable.watch()` called on them by f.
+  the same arguments. The gradient is with respect to all trainable TFE
+  variables accessed by `f`.
 
   This function is useful when the exact set of variables to differentiate with
   is not known ahead of time.
@@ -249,8 +249,8 @@ def implicit_grad(f):
   """Returns a function which differentiates f with respect to variables.
 
   The wrapped function returns the gradient of f when called with the same
-  arguments. The gradient is with respect to all TFE variables which are
-  either trainable or have `variable.watch()` called on them by f.
+  arguments. The gradient is with respect to all trainable TFE variables
+  accessed by `f`.
 
   This function is useful when the exact set of variables to differentiate with
   is not known ahead of time.
-- 
GitLab


From 0a86c23860968c66e95b9b6e930d14fac2699889 Mon Sep 17 00:00:00 2001
From: Jayaram Bobba <jayaram.bobba@intel.com>
Date: Mon, 26 Mar 2018 14:52:32 -0700
Subject: [PATCH 509/960] reverting mkl allocator inline modifier from #17396.
 causes build issues on linux systems (#18006)

---
 tensorflow/core/common_runtime/mkl_cpu_allocator.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.h b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
index 73abf18d97..55c8411ad0 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.h
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.h
@@ -50,7 +50,7 @@ class MklCPUAllocator : public VisitableAllocator {
   // Constructor and other standard functions
 
   /// Environment variable that user can set to upper bound on memory allocation
-  static inline constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
+  static constexpr const char* kMaxLimitStr = "TF_MKL_ALLOC_MAX_BYTES";
 
   /// Default upper limit on allocator size - 64GB
   static constexpr size_t kDefaultMaxLimit = 64LL << 30;
-- 
GitLab


From 2c548819707bdafc8057cdd9c997f2a7b420d577 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 15:07:10 -0700
Subject: [PATCH 510/960] Fix some compiler warnings in MKL-DNN build.

PiperOrigin-RevId: 190532168
---
 tensorflow/core/graph/mkl_layout_pass.cc           | 13 ++++++-------
 tensorflow/core/kernels/BUILD                      |  5 +++++
 tensorflow/core/kernels/mkl_concat_op.cc           |  6 ++++--
 tensorflow/core/kernels/mkl_conv_ops.h             |  9 ++++++---
 tensorflow/core/kernels/mkl_fused_batch_norm_op.cc | 12 ++++++------
 tensorflow/core/kernels/mkl_lrn_op.cc              |  7 ++++---
 tensorflow/core/kernels/mkl_reshape_op.cc          |  3 ++-
 tensorflow/core/kernels/mkl_softmax_op.cc          |  1 -
 tensorflow/core/util/mkl_util.h                    |  4 ++--
 9 files changed, 35 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/graph/mkl_layout_pass.cc b/tensorflow/core/graph/mkl_layout_pass.cc
index 1507b6eae2..5368774f2d 100644
--- a/tensorflow/core/graph/mkl_layout_pass.cc
+++ b/tensorflow/core/graph/mkl_layout_pass.cc
@@ -3103,8 +3103,7 @@ void MklLayoutRewritePass::GetDummyMklTensorNode(std::unique_ptr<Graph>* g,
   TensorProto proto;
   proto.set_dtype(dt);
   uint8 zero[8] = {0, 0, 0, 0, 0, 0, 0, 0};
-  proto.set_tensor_content(const_cast<const void*>(static_cast<void*>(&zero)),
-                           8);
+  proto.set_tensor_content(string(reinterpret_cast<char*>(&zero), 8));
   TensorShape dummy_shape({8});
   dummy_shape.AsProto(proto.mutable_tensor_shape());
   TF_CHECK_OK(NodeBuilder((*g)->NewName("DMT"), "Const")
@@ -3219,7 +3218,8 @@ int MklLayoutRewritePass::SetUpContiguousInputs(
     // For that let's first find filter node that is 2nd input (slot 1)
     // of BackpropInput.
     Node* filter_node = nullptr;
-    old_node->input_node(kConv2DBackpropInputFilterInputSlotIdx, &filter_node);
+    TF_CHECK_OK(old_node->input_node(kConv2DBackpropInputFilterInputSlotIdx,
+                                     &filter_node));
     CHECK_NOTNULL(filter_node);
 
     // Now check which nodes receive from filter_node. Filter feeds as
@@ -3399,8 +3399,7 @@ void MklLayoutRewritePass::GetDummyWorkspaceTensorNode(
   TensorProto proto;
   proto.set_dtype(dt);
   float zero[1] = {0};
-  proto.set_tensor_content(const_cast<const void*>(static_cast<void*>(&zero)),
-                           4);
+  proto.set_tensor_content(string(reinterpret_cast<char*>(&zero), 4));
   TensorShape dummy_shape({1});
   dummy_shape.AsProto(proto.mutable_tensor_shape());
   TF_CHECK_OK(NodeBuilder((*g)->NewName("DMT"), "Const")
@@ -3876,7 +3875,7 @@ Status MklLayoutRewritePass::MergeConv2DWithBiasAdd(std::unique_ptr<Graph>* g,
 
   // Create node.
   Node* new_node;
-  nb.Finalize(&**g, &new_node);
+  TF_CHECK_OK(nb.Finalize(&**g, &new_node));
   CHECK_NOTNULL(new_node);
 
   // Incoming data edges from 'pred' node and 'succ' node to new 'new_node'
@@ -3987,7 +3986,7 @@ Status MklLayoutRewritePass::MergeConv2DBackpropFilterWithBiasAddGrad(
 
   // Create node.
   Node* new_node;
-  nb.Finalize(&**g, &new_node);
+  TF_CHECK_OK(nb.Finalize(&**g, &new_node));
   CHECK_NOTNULL(new_node);
 
   // Incoming data edges from BiasAddGrad node and Conv2DBackpropFilter node to
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 8d235e79c0..9bb80eb892 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -5959,6 +5959,7 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn",
     ],
 )
 
@@ -5979,6 +5980,7 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn",
     ],
 )
 
@@ -6010,6 +6012,7 @@ tf_mkl_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:nn_ops_op_lib",
+        "//third_party/eigen3",
         "//third_party/mkl:intel_binary_blob",
         "@mkl_dnn",
     ],
@@ -6029,6 +6032,7 @@ tf_mkl_kernel_library(
     prefix = "mkl_aggregate_ops",
     deps = MATH_DEPS + [
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn",
     ],
 )
 
@@ -6046,6 +6050,7 @@ tf_mkl_kernel_library(
     prefix = "mkl_reshape_op",
     deps = ARRAY_DEPS + [
         "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn",
     ],
 )
 
diff --git a/tensorflow/core/kernels/mkl_concat_op.cc b/tensorflow/core/kernels/mkl_concat_op.cc
index aa3ea890b0..9ab95d765c 100644
--- a/tensorflow/core/kernels/mkl_concat_op.cc
+++ b/tensorflow/core/kernels/mkl_concat_op.cc
@@ -803,8 +803,10 @@ class MklConcatOp : public OpKernel {
     Tensor* output_tensor = nullptr;
     TensorShape tf_shape_output;
     tf_shape_output.AddDim(dnn_shape_output.GetSerializeBufferSize());
-    context->allocate_output(GetTensorMetaDataIndex(0, context->num_outputs()),
-                             tf_shape_output, &output_tensor);
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(
+                       GetTensorMetaDataIndex(0, context->num_outputs()),
+                       tf_shape_output, &output_tensor));
     dnn_shape_output.SerializeMklDnnShape(
         output_tensor->flat<uint8>().data(),
         output_tensor->flat<uint8>().size() * sizeof(uint8));
diff --git a/tensorflow/core/kernels/mkl_conv_ops.h b/tensorflow/core/kernels/mkl_conv_ops.h
index 7ca10db895..8333a09316 100644
--- a/tensorflow/core/kernels/mkl_conv_ops.h
+++ b/tensorflow/core/kernels/mkl_conv_ops.h
@@ -65,9 +65,12 @@ class MklDnnConvUtil {
  public:
   MklDnnConvUtil(OpKernelContext* context, const std::vector<int32>& strides,
                  Padding pad, TensorFormat fm,
-                 const std::vector<int32>& dilations) :
-    context_(context), strides_(strides), padding_(pad),
-    data_format_(fm), dilations_(dilations) {}
+                 const std::vector<int32>& dilations)
+      : context_(context),
+        strides_(strides),
+        dilations_(dilations),
+        padding_(pad),
+        data_format_(fm) {}
 
   virtual ~MklDnnConvUtil() { context_ = nullptr; }
 
diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 9e564b016f..333a6570dc 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -817,8 +817,8 @@ class MklFusedBatchNormOp : public OpKernel {
       // set weights primitive
       // MKL-DNN packs scale & shift as "weights":
       // <scale>...<scale><shift>...<shift>
-      auto weights_desc =
-          memory::desc({2, depth_}, MklDnnType<T>(), memory::format::nc);
+      auto weights_desc = memory::desc({2, static_cast<int>(depth_)},
+                                       MklDnnType<T>(), memory::format::nc);
       auto weights_pd = memory::primitive_desc(weights_desc, cpu_engine);
       auto weights_m = memory(weights_pd);
       T* weights_data = reinterpret_cast<T*>(weights_m.get_data_handle());
@@ -833,8 +833,8 @@ class MklFusedBatchNormOp : public OpKernel {
       }
 
       // set mean primitive
-      auto mean_desc =
-          memory::desc({1, depth_}, MklDnnType<T>(), memory::format::nc);
+      auto mean_desc = memory::desc({1, static_cast<int>(depth_)},
+                                    MklDnnType<T>(), memory::format::nc);
       auto mean_pd = memory::primitive_desc(mean_desc, cpu_engine);
       char* saved_mean_data_tf =
           reinterpret_cast<char*>(saved_mean_tensor->flat<T>().data());
@@ -844,8 +844,8 @@ class MklFusedBatchNormOp : public OpKernel {
           memory(mean_pd, reinterpret_cast<void*>(saved_mean_data_tf));
 
       // set variance primitive
-      auto variance_desc =
-          memory::desc({1, depth_}, MklDnnType<T>(), memory::format::nc);
+      auto variance_desc = memory::desc({1, static_cast<int>(depth_)},
+                                        MklDnnType<T>(), memory::format::nc);
       auto variance_pd = memory::primitive_desc(variance_desc, cpu_engine);
       char* saved_variance_data_tf =
           reinterpret_cast<char*>(saved_variance_tensor->flat<T>().data());
diff --git a/tensorflow/core/kernels/mkl_lrn_op.cc b/tensorflow/core/kernels/mkl_lrn_op.cc
index 282012c719..eef254cdad 100644
--- a/tensorflow/core/kernels/mkl_lrn_op.cc
+++ b/tensorflow/core/kernels/mkl_lrn_op.cc
@@ -752,7 +752,8 @@ class MklLRNOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_));
     OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_));
     workspace_enabled_ = false;
-    context->GetAttr("workspace_enabled", &workspace_enabled_);
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("workspace_enabled", &workspace_enabled_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -1001,7 +1002,8 @@ class MklLRNGradOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("alpha", &alpha_));
     OP_REQUIRES_OK(context, context->GetAttr("beta", &beta_));
     workspace_enabled_ = false;
-    context->GetAttr("workspace_enabled", &workspace_enabled_);
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("workspace_enabled", &workspace_enabled_));
   }
 
   void Compute(OpKernelContext* context) override {
@@ -1043,7 +1045,6 @@ class MklLRNGradOp : public OpKernel {
       // Naming: diff_dst is input_gradient_tensor; src is orig_input_tensor.
       const Tensor& input_grad_tensor = MklGetInput(context, kIdxGradient);
       const Tensor& orig_input_tensor = MklGetInput(context, kIdxOrigInput);
-      const Tensor& orig_output_tensor = MklGetInput(context, kIdxOrigOutput);
 
       // Get input sizes in MKL-DNN required NCHW format.
       // LRN does not have data_format attribute. But by default it has
diff --git a/tensorflow/core/kernels/mkl_reshape_op.cc b/tensorflow/core/kernels/mkl_reshape_op.cc
index 5dbc4a2709..e12f6f437a 100644
--- a/tensorflow/core/kernels/mkl_reshape_op.cc
+++ b/tensorflow/core/kernels/mkl_reshape_op.cc
@@ -266,7 +266,8 @@ class MklReshapeOp : public OpKernel {
                                                    &net)) {
               stream(stream::kind::eager).submit(net).wait();
             } else {
-              output_tensor->CopyFrom(input_tensor, shape_to);
+              OP_REQUIRES(context,
+                          output_tensor->CopyFrom(input_tensor, shape_to));
             }
             return;
           } else {
diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl_softmax_op.cc
index aceef1e234..170523b5b4 100644
--- a/tensorflow/core/kernels/mkl_softmax_op.cc
+++ b/tensorflow/core/kernels/mkl_softmax_op.cc
@@ -27,7 +27,6 @@ limitations under the License.
 
 #include "mkldnn.h"
 #include "mkldnn_types.h"
-#include "tensorflow/core/platform/default/logging.h"
 #include "tensorflow/core/util/mkl_util.h"
 
 #include "mkldnn.hpp"
diff --git a/tensorflow/core/util/mkl_util.h b/tensorflow/core/util/mkl_util.h
index 34db96075d..9f58e40d94 100644
--- a/tensorflow/core/util/mkl_util.h
+++ b/tensorflow/core/util/mkl_util.h
@@ -1579,10 +1579,10 @@ class MklDnnData {
   }
 
   /// Set function for data buffer of user memory primitive.
-  inline void* SetUsrMemDataHandle(void* data_buffer) {
+  inline void SetUsrMemDataHandle(void* data_buffer) {
     CHECK_NOTNULL(user_memory_);
     CHECK_NOTNULL(data_buffer);
-    return user_memory_->set_data_handle(data_buffer);
+    user_memory_->set_data_handle(data_buffer);
   }
 
   /// Set function for data buffer of user memory primitive.
-- 
GitLab


From 3a00d79b16348f0a53379e81b8e98bdd93d4833e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 15:19:29 -0700
Subject: [PATCH 511/960] [XLA] Redesign: implement and test unary and binary
 ops.

Also,
- Templatized ComputeAndCompareRX and CreateRXParameter so that they accept XlaBuilder and XlaOp.
- Clear data held by an XlaBuilder when Build() is called, otherwise errors will occur when the builder is reused.

PiperOrigin-RevId: 190534245
---
 .../xla/client/xla_client/xla_builder.cc      | 136 +++--
 .../xla/client/xla_client/xla_builder.h       |   3 +
 .../xla/client/xla_client/xla_builder_test.cc |  14 +
 .../compiler/xla/service/shape_inference.cc   |   9 +-
 .../compiler/xla/service/shape_inference.h    |   2 +
 tensorflow/compiler/xla/tests/BUILD           |   1 +
 .../xla/tests/array_elementwise_ops_test.cc   | 496 +++++++++---------
 .../xla/tests/client_library_test_base.h      | 123 ++---
 8 files changed, 430 insertions(+), 354 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 596f39b4fd..bf91efcfd6 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -164,6 +164,11 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
   }
   module->add_computations()->Swap(&entry);
 
+  // Clear data held by this builder.
+  this->instructions_.clear();
+  this->embedded_.clear();
+  this->parameter_numbers_.clear();
+
   return std::move(computation);
 }
 
@@ -216,6 +221,16 @@ StatusOr<XlaOp> XlaBuilder::AddBroadcastSequence(const Shape& output_shape,
                         broadcast_dimensions);
 }
 
+XlaOp XlaBuilder::UnaryOp(HloOpcode unop, const XlaOp& operand) {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferUnaryOpShape(unop, operand_shape));
+    return AddInstruction(std::move(instr), unop, {operand});
+  }());
+}
+
 XlaOp XlaBuilder::BinaryOp(
     HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
@@ -447,32 +462,32 @@ XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
 
 XlaOp XlaBuilder::Eq(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kEq, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Ne(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kNe, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Ge(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kGe, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Gt(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kGt, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Le(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kLe, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Lt(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kLt, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Dot(const XlaOp& lhs, const XlaOp& rhs) {
@@ -551,102 +566,134 @@ XlaOp XlaBuilder::HostCompute(tensorflow::gtl::ArraySlice<XlaOp> operands,
 XlaOp XlaBuilder::Complex(
     const XlaOp& real, const XlaOp& imag,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kComplex, real, imag, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Conj(const XlaOp& operand) { return UnimplementedOp(); }
 
 XlaOp XlaBuilder::Sub(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kSubtract, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Div(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kDivide, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Rem(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kRemainder, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Max(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kMaximum, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Min(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kMinimum, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::And(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kAnd, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::Or(const XlaOp& lhs, const XlaOp& rhs,
                      tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kOr, lhs, rhs, broadcast_dimensions);
 }
 
+// TODO(b/65209188): Create a dedicated lowering for Xor.
 XlaOp XlaBuilder::Xor(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return Or(And(Not(lhs), rhs, broadcast_dimensions),
+            And(lhs, Not(rhs), broadcast_dimensions));
 }
 
-XlaOp XlaBuilder::Not(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Not(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kNot, operand);
+}
 
 XlaOp XlaBuilder::ShiftLeft(
     const XlaOp& lhs, const XlaOp& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kShiftLeft, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::ShiftRightArithmetic(
     const XlaOp& lhs, const XlaOp& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kShiftRightArithmetic, lhs, rhs,
+                  broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::ShiftRightLogical(
     const XlaOp& lhs, const XlaOp& rhs,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kShiftRightLogical, lhs, rhs,
+                  broadcast_dimensions);
 }
 
-XlaOp XlaBuilder::Abs(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Abs(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kAbs, operand);
+}
 
 XlaOp XlaBuilder::Atan2(
     const XlaOp& y, const XlaOp& x,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kAtan2, y, x, broadcast_dimensions);
 }
 
-XlaOp XlaBuilder::Exp(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Exp(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kExp, operand);
+}
 
-XlaOp XlaBuilder::Floor(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Floor(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kFloor, operand);
+}
 
-XlaOp XlaBuilder::Ceil(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Ceil(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kCeil, operand);
+}
 
-XlaOp XlaBuilder::Round(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Round(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kRoundNearestAfz, operand);
+}
 
-XlaOp XlaBuilder::Log(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Log(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kLog, operand);
+}
 
-XlaOp XlaBuilder::Sign(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Sign(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kSign, operand);
+}
 
-XlaOp XlaBuilder::Cos(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Cos(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kCos, operand);
+}
 
-XlaOp XlaBuilder::Sin(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Sin(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kSin, operand);
+}
 
-XlaOp XlaBuilder::Tanh(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Tanh(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kTanh, operand);
+}
 
-XlaOp XlaBuilder::Real(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Real(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kReal, operand);
+}
 
-XlaOp XlaBuilder::Imag(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Imag(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kImag, operand);
+}
 
-XlaOp XlaBuilder::IsFinite(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::IsFinite(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kIsFinite, operand);
+}
 
 XlaOp XlaBuilder::Transpose(const XlaOp& operand,
                             tensorflow::gtl::ArraySlice<int64> permutation) {
@@ -668,13 +715,18 @@ XlaOp XlaBuilder::Rev(const XlaOp& operand,
   return UnimplementedOp();
 }
 
-XlaOp XlaBuilder::Sort(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Sort(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kSort, operand);
+}
 
-XlaOp XlaBuilder::SqrtF32(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::SqrtF32(const XlaOp& operand) {
+  return BinaryOp(HloOpcode::kPower, operand, ConstantR0<float>(0.5),
+                  /*broadcast_dimensions=*/{});
+}
 
 XlaOp XlaBuilder::Pow(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kPower, lhs, rhs, broadcast_dimensions);
 }
 
 XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand,
@@ -687,13 +739,19 @@ XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand,
   return UnimplementedOp();
 }
 
-XlaOp XlaBuilder::SquareF32(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::SquareF32(const XlaOp& operand) {
+  return BinaryOp(HloOpcode::kPower, operand, ConstantR0<float>(2.0),
+                  /*broadcast_dimensions=*/{});
+}
 
 XlaOp XlaBuilder::ReciprocalF32(const XlaOp& operand) {
-  return UnimplementedOp();
+  return BinaryOp(HloOpcode::kPower, operand, ConstantR0<float>(-1.0),
+                  /*broadcast_dimensions=*/{});
 }
 
-XlaOp XlaBuilder::Neg(const XlaOp& operand) { return UnimplementedOp(); }
+XlaOp XlaBuilder::Neg(const XlaOp& operand) {
+  return UnaryOp(HloOpcode::kNegate, operand);
+}
 
 XlaOp XlaBuilder::Clamp(const XlaOp& min, const XlaOp& operand,
                         const XlaOp& max) {
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index c19eb47165..22cf094512 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -730,6 +730,9 @@ class XlaBuilder {
 
   StatusOr<const HloInstructionProto*> LookUpInstruction(const XlaOp& op) const;
 
+  // Internal helper method that does the building for an arbitrary unary op.
+  XlaOp UnaryOp(HloOpcode unop, const XlaOp& operand);
+
   // Internal helper method that does the building for an arbitrary binary op.
   // broadcast_dimensions specifies which dimensions to use for broadcasting
   // when the operation is between tensors of different ranks.
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
index 529287a57a..85d4227ba4 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
@@ -217,5 +217,19 @@ TEST_F(XlaBuilderTest, Transpose) {
   EXPECT_THAT(root, op::Transpose(op::Parameter()));
 }
 
+// TODO(b/65209188): Create a dedicated lowering for Xor.
+TEST_F(XlaBuilderTest, Xor) {
+  XlaBuilder b(TestName());
+  auto x = b.Parameter(0, ShapeUtil::MakeShape(PRED, {}), "x");
+  auto y = b.Parameter(1, ShapeUtil::MakeShape(PRED, {}), "y");
+  b.Xor(x, y);
+  TF_ASSERT_OK_AND_ASSIGN(auto module, BuildHloModule(&b));
+  auto root = module->entry_computation()->root_instruction();
+  LOG(ERROR) << module->ToString();
+  EXPECT_THAT(root,
+              op::Or(op::And(op::Not(op::Parameter(0)), op::Parameter(1)),
+                     op::And(op::Parameter(0), op::Not(op::Parameter(1)))));
+}
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 8c8bd6d73a..2a70ea0354 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -304,12 +304,17 @@ StatusOr<Shape> InferWindowOutputShape(const Shape& base_shape,
 
 /* static */ StatusOr<Shape> ShapeInference::InferUnaryOpShape(
     HloOpcode opcode, const HloInstruction* operand) {
+  return InferUnaryOpShape(opcode, operand->shape());
+}
+
+/* static */ StatusOr<Shape> ShapeInference::InferUnaryOpShape(
+    HloOpcode opcode, const Shape& shape) {
   // There is no copy operation at the proto level, so handle copy explicitly.
   if (opcode == HloOpcode::kCopy) {
-    return operand->shape();
+    return shape;
   }
 
-  return InferUnaryOpShape(OpcodeToUnaryOperation(opcode), operand->shape());
+  return InferUnaryOpShape(OpcodeToUnaryOperation(opcode), shape);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferUnaryOpShape(
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index 085fdac60c..b6552a34ae 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -48,6 +48,8 @@ class ShapeInference {
   // given input shape.
   static StatusOr<Shape> InferUnaryOpShape(UnaryOperation operation,
                                            const Shape& arg);
+  static StatusOr<Shape> InferUnaryOpShape(HloOpcode opcode,
+                                           const Shape& shape);
   static StatusOr<Shape> InferUnaryOpShape(HloOpcode opcode,
                                            const HloInstruction* operand);
 
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 26022278e5..3705d6c271 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -598,6 +598,7 @@ xla_test(
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index 6e21dda25d..fa7ac3ca9b 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/layout_util.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -50,28 +51,28 @@ class ArrayElementwiseOpTestParamCount
       public ::testing::WithParamInterface<int> {};
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantZeroElementF32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
-  auto result = builder.Neg(a);
+  builder.Neg(a);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantF32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 3.14f, 2.25f, -10.0f, 6.0f});
-  auto result = builder.Neg(a);
+  builder.Neg(a);
 
   ComputeAndCompareR1<float>(&builder, {2.5f, -3.14f, -2.25f, 10.0f, -6.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantS32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({-1, 0, 1, 324,
                                       std::numeric_limits<int32>::min(),
                                       std::numeric_limits<int32>::max()});
-  auto result = builder.Neg(a);
+  builder.Neg(a);
 
   // -min == min for int32 due to an overflow. In C++ it is undefined behavior
   // to do this calculation. For XLA we have not specified that, so it
@@ -83,18 +84,18 @@ XLA_TEST_F(ArrayElementwiseOpTest, NegConstantS32) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantZeroElementC64) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>({});
-  auto result = builder.Neg(a);
+  builder.Neg(a);
 
   ComputeAndCompareR1<complex64>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantC64) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>(
       {{-2.5f, 1.0f}, {0.0f, 3.14f}, {2.25f, -1.0f}, {-10.0f, 0.0f}});
-  auto result = builder.Neg(a);
+  builder.Neg(a);
 
   ComputeAndCompareR1<complex64>(
       &builder, {{2.5f, -1.0f}, {0.0f, -3.14f}, {-2.25f, 1.0f}, {10.0f, 0.0f}},
@@ -102,7 +103,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, NegConstantC64) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NegConstantS64) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int64>({
       -1,
       1,
@@ -112,7 +113,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, NegConstantS64) {
       static_cast<int64>(0x8000000000000000LL),
       static_cast<int64>(0x8000000000000001LL),
   });
-  auto result = builder.Neg(a);
+  builder.Neg(a);
   LOG(INFO) << -static_cast<int64>(0x7FFFFFFFFFFFFFFFLL);
 
   ComputeAndCompareR1<int64>(&builder,
@@ -129,9 +130,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, NegConstantS64) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, IsFiniteZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
-  auto result = builder.IsFinite(a);
+  builder.IsFinite(a);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
@@ -140,64 +141,63 @@ XLA_TEST_F(ArrayElementwiseOpTest, IsFiniteZeroElementF32s) {
 static const float kNonCanonicalNaN = tensorflow::bit_cast<float>(0x7FD01234);
 
 XLA_TEST_F(ArrayElementwiseOpTest, IsFiniteScalarF32) {
-  ComputationBuilder builder(client_, TestName());
-  auto result = builder.IsFinite(builder.ConstantR0<float>(NAN));
+  XlaBuilder builder(TestName());
+  builder.IsFinite(builder.ConstantR0<float>(NAN));
   ComputeAndCompareR0<bool>(&builder, false, {});
 
   EXPECT_TRUE(std::isnan(kNonCanonicalNaN));
-  auto result_non_canonical =
-      builder.IsFinite(builder.ConstantR0<float>(kNonCanonicalNaN));
+  builder.IsFinite(builder.ConstantR0<float>(kNonCanonicalNaN));
   ComputeAndCompareR0<bool>(&builder, false, {});
 
   const float inf = std::numeric_limits<float>::infinity();
-  auto result_inf = builder.IsFinite(builder.ConstantR0<float>(inf));
+  builder.IsFinite(builder.ConstantR0<float>(inf));
   ComputeAndCompareR0<bool>(&builder, false, {});
 
-  auto result_neg_inf = builder.IsFinite(builder.ConstantR0<float>(-inf));
+  builder.IsFinite(builder.ConstantR0<float>(-inf));
   ComputeAndCompareR0<bool>(&builder, false, {});
 
-  auto result_zero = builder.IsFinite(builder.ConstantR0<float>(0.0f));
+  builder.IsFinite(builder.ConstantR0<float>(0.0f));
   ComputeAndCompareR0<bool>(&builder, true, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, IsFiniteR1F32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   const float inf = std::numeric_limits<float>::infinity();
   EXPECT_TRUE(std::isnan(kNonCanonicalNaN));
   auto a = builder.ConstantR1<float>(
       {{NAN, 7.0f, kNonCanonicalNaN, -1.0f, inf, -inf}});
-  auto result = builder.IsFinite(a);
+  builder.IsFinite(a);
 
   ComputeAndCompareR1<bool>(&builder, {false, true, false, true, false, false},
                             {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 3.14f, 2.25f, -10.0f, 6.0f});
   auto b = builder.ConstantR1<float>({100.0f, 3.13f, 2.75f, 10.5f, -999.0f});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   ComputeAndCompareR1<float>(&builder, {97.5f, 6.27f, 5.0f, 0.5f, -993.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>(
       {{-2.5f, 0.0f}, {0.0f, 3.14f}, {2.25f, 0.0f}, {1.0f, -10.0f}});
   auto b = builder.ConstantR1<complex64>(
       {{100.0f, 0.0f}, {3.13f, 0.0f}, {2.75f, 1.0f}, {-2.0f, 10.5f}});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   ComputeAndCompareR1<complex64>(
       &builder, {97.5f, {3.13f, 3.14f}, {5.0f, 1.0f}, {-1.0f, 0.5f}}, {},
@@ -205,10 +205,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantC64s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantZeroElementC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>({});
   auto b = builder.ConstantR1<complex64>({});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   ComputeAndCompareR1<complex64>(&builder, {}, {}, error_spec_);
 }
@@ -295,7 +295,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantS64s) {
 
 TEST_P(ArrayElementwiseOpTestParamCount, AddManyValues) {
   const int count = GetParam();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::vector<float> a_values;
   std::vector<float> b_values;
   for (int i = 0; i < count; ++i) {
@@ -334,49 +334,49 @@ TEST_P(ArrayElementwiseOpTestParamCount, AddManyValues) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 3.14f, 2.25f, -10.0f, 6.0f});
   auto b = builder.ConstantR1<float>({100.0f, 3.13f, 2.75f, 10.5f, -999.0f});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<float>(&builder, {-102.5f, 0.01f, -0.5f, -20.5f, 1005.0f},
                              {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({-1, 0, 2, 1000000000});
   auto b = builder.ConstantR1<int32>({-1, 2, 1, -1});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {0, -2, 1, 1000000001}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantZeroElementS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({});
   auto b = builder.ConstantR1<int32>({});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>(
       {{-2.5f, 0.0f}, {0.0f, 3.14f}, {3.0f, 2.25f}});
   auto b = builder.ConstantR1<complex64>(
       {{0.0f, 10.0f}, {3.13f, 0.0f}, {2.75f, -0.25f}});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<complex64>(
       &builder, {{-2.5f, -10.0f}, {-3.13f, 3.14f}, {0.25f, 2.5f}}, {},
@@ -384,29 +384,29 @@ XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantC64s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SubTwoConstantZeroElementC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>({});
   auto b = builder.ConstantR1<complex64>({});
-  auto add = builder.Sub(a, b);
+  builder.Sub(a, b);
 
   ComputeAndCompareR1<complex64>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, DivTwoConstantF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, -10.0f, 6.0f});
   auto b = builder.ConstantR1<float>({10.0f, 5.1f, 1.0f, 10.0f, -6.0f});
-  auto add = builder.Div(a, b);
+  builder.Div(a, b);
 
   ComputeAndCompareR1<float>(&builder, {-0.25f, 5.0f, 2.25f, -1.0f, -1.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, DivTwoConstantZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto add = builder.Div(a, b);
+  builder.Div(a, b);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
@@ -436,9 +436,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivS32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
-    ComputationDataHandle divisor;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
+    XlaOp divisor;
     auto dividend_data =
         CreateR1Parameter<int32>(dividends, 0, "dividend", &builder, &dividend);
     auto divisor_data =
@@ -451,8 +451,8 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivS32s) {
 
   // Test with a compile-time constant divisor.
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
     auto dividend_data =
         CreateR1Parameter<int32>(dividends, 0, "dividend", &builder, &dividend);
     builder.Div(dividend, builder.ConstantR1<int32>(divisors));
@@ -461,9 +461,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivS32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
-    ComputationDataHandle divisor;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
+    XlaOp divisor;
     auto dividend_data =
         CreateR1Parameter<int32>(dividends, 0, "dividend", &builder, &dividend);
     auto divisor_data =
@@ -476,8 +476,8 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivS32s) {
 
   // Test with a compile-time constant divisor.
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
     auto dividend_data =
         CreateR1Parameter<int32>(dividends, 0, "dividend", &builder, &dividend);
     builder.Rem(dividend, builder.ConstantR1<int32>(divisors));
@@ -507,9 +507,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivU32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
-    ComputationDataHandle divisor;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
+    XlaOp divisor;
     auto dividend_data = CreateR1Parameter<uint32>(dividends, 0, "dividend",
                                                    &builder, &dividend);
     auto divisor_data =
@@ -521,8 +521,8 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivU32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
     auto dividend_data = CreateR1Parameter<uint32>(dividends, 0, "dividend",
                                                    &builder, &dividend);
     builder.Div(dividend, builder.ConstantR1<uint32>(divisors));
@@ -531,9 +531,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivU32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
-    ComputationDataHandle divisor;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
+    XlaOp divisor;
     auto dividend_data = CreateR1Parameter<uint32>(dividends, 0, "dividend",
                                                    &builder, &dividend);
     auto divisor_data =
@@ -545,8 +545,8 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivU32s) {
   }
 
   {
-    ComputationBuilder builder(client_, TestName());
-    ComputationDataHandle dividend;
+    XlaBuilder builder(TestName());
+    XlaOp dividend;
     auto dividend_data = CreateR1Parameter<uint32>(dividends, 0, "dividend",
                                                    &builder, &dividend);
     builder.Rem(dividend, builder.ConstantR1<uint32>(divisors));
@@ -556,33 +556,33 @@ XLA_TEST_F(ArrayElementwiseOpTest, DivU32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, DivTwoConstantC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>(
       {{-2.5f, 1.0f}, {-25.5f, 0.0f}, {2.0f, -1.0f}});
   auto b = builder.ConstantR1<complex64>(
       {{10.0f, 0.0f}, {0.0f, 1.0f}, {2.0f, -1.0f}});
-  auto div = builder.Div(a, b);
+  builder.Div(a, b);
 
   ComputeAndCompareR1<complex64>(
       &builder, {{-0.25f, 0.1f}, {0.0f, 25.5f}, {1.0f, 0.0f}}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, DivTwoConstantZeroElementC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>({});
   auto b = builder.ConstantR1<complex64>({});
-  auto div = builder.Div(a, b);
+  builder.Div(a, b);
 
   ComputeAndCompareR1<complex64>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, RemF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>(
       {-2.5f, 25.5f, 2.25f, -10.0f, 6.0f, 3.0f, 3.0f, -1.0f, -8.0f});
   auto b = builder.ConstantR1<float>(
       {10.0f, 5.1f, 1.0f, 10.0f, -6.0f, 2.0f, -2.0f, 7.0f, -4.0f});
-  auto add = builder.Rem(a, b);
+  builder.Rem(a, b);
 
   ComputeAndCompareR1<float>(
       &builder, {-2.5f, 0.0f, 0.25f, 0.0f, -0.0f, 1.0f, 1.0f, -1.0f, -0.0f}, {},
@@ -590,21 +590,21 @@ XLA_TEST_F(ArrayElementwiseOpTest, RemF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, RemZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto add = builder.Rem(a, b);
+  builder.Rem(a, b);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, RemF64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<double>(
       {-2.5, 25.5, 2.25, -10.0, 6.0, 3.0, 3.0, -1.0, -8.0});
   auto b = builder.ConstantR1<double>(
       {10.0, 5.1, 1.0, 10.0, -6.0, 2.0, -2.0, 7.0, -4.0});
-  auto add = builder.Rem(a, b);
+  builder.Rem(a, b);
 
   ComputeAndCompareR1<double>(
       &builder, {-2.5, 0.0, 0.25, 0.0, -0.0, 1.0, 1.0, -1.0, -0.0}, {},
@@ -612,20 +612,20 @@ XLA_TEST_F(ArrayElementwiseOpTest, RemF64s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, -10.0f, 6.0f});
   auto b = builder.ConstantR1<float>({10.0f, 5.0f, 1.0f, 10.0f, -6.0f});
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<float>(&builder, {-25.0f, 127.5f, 2.25f, -100.0f, -36.0f},
                              {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
@@ -648,19 +648,19 @@ XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantS32s) {
     }
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>(a_data);
   auto b = builder.ConstantR1<int32>(b_data);
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<int32>(&builder, expected, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantZeroElementS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({});
   auto b = builder.ConstantR1<int32>({});
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {}, {});
 }
@@ -679,21 +679,21 @@ XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantU32s) {
     }
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>(a_data);
   auto b = builder.ConstantR1<uint32>(b_data);
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<uint32>(&builder, expected, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>(
       {{-2.5f, 0.0f}, {0.0f, 25.5f}, {2.0f, -10.0f}});
   auto b = builder.ConstantR1<complex64>(
       {{0.0f, 10.0f}, {5.0f, 1.0f}, {10.0f, -6.0f}});
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<complex64>(
       &builder, {{0.0f, -25.0f}, {-25.5f, 127.5f}, {-40.0f, -112.0}}, {},
@@ -701,264 +701,264 @@ XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantC64s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MulTwoConstantZeroElementC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<complex64>({});
   auto b = builder.ConstantR1<complex64>({});
-  auto add = builder.Mul(a, b);
+  builder.Mul(a, b);
 
   ComputeAndCompareR1<complex64>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {false, false, false, true}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndPredR2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<bool>({{false, false}, {true, true}});
   auto b = builder.ConstantR2<bool>({{false, true}, {false, true}});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   Array2D<bool> expected_array({{false, false}, {false, true}});
   ComputeAndCompareR2<bool>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({0, -1, -8});
   auto b = builder.ConstantR1<int32>({5, -7, 12});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {0, -7, 8}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndS32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<int32>({{0, -5}, {-1, 5}});
   auto b = builder.ConstantR2<int32>({{1, -6}, {4, 5}});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   Array2D<int32> expected_array({{0, -6}, {4, 5}});
   ComputeAndCompareR2<int32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({});
   auto b = builder.ConstantR1<int32>({});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({0, 1, 8});
   auto b = builder.ConstantR1<int32>({5, 7, 12});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {0, 1, 8}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndU32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<uint32>({{0, 1}, {3, 8}});
   auto b = builder.ConstantR2<uint32>({{1, 0}, {7, 6}});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   Array2D<uint32> expected_array({{0, 0}, {3, 0}});
   ComputeAndCompareR2<uint32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AndZeroElementU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>({});
   auto b = builder.ConstantR1<uint32>({});
-  auto out = builder.And(a, b);
+  builder.And(a, b);
 
   ComputeAndCompareR1<uint32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({false, false, true, true});
   auto b = builder.ConstantR1<bool>({false, true, false, true});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {false, true, true, true}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrPredR2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<bool>({{false, false}, {true, true}});
   auto b = builder.ConstantR2<bool>({{false, true}, {false, true}});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   Array2D<bool> expected_array({{false, true}, {true, true}});
   ComputeAndCompareR2<bool>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({});
   auto b = builder.ConstantR1<bool>({});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({0, -1, 8});
   auto b = builder.ConstantR1<int32>({5, -7, 4});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {5, -1, 12}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrS32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<int32>({{0, -1}, {8, 8}});
   auto b = builder.ConstantR2<int32>({{5, -7}, {4, 1}});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   Array2D<int32> expected_array({{5, -1}, {12, 9}});
   ComputeAndCompareR2<int32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({});
   auto b = builder.ConstantR1<int32>({});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>({0, 1, 8});
   auto b = builder.ConstantR1<uint32>({5, 7, 4});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<uint32>(&builder, {5, 7, 12}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrU32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<uint32>({{0, 1}, {8, 8}});
   auto b = builder.ConstantR2<uint32>({{5, 7}, {4, 1}});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   Array2D<uint32> expected_array({{5, 7}, {12, 9}});
   ComputeAndCompareR2<uint32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, OrZeroElementU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>({});
   auto b = builder.ConstantR1<uint32>({});
-  auto out = builder.Or(a, b);
+  builder.Or(a, b);
 
   ComputeAndCompareR1<uint32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({false, true, true, false});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<bool>(&builder, {true, false, false, true}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotPredR2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<bool>({{false, true}, {true, false}});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   Array2D<bool> expected_array({{true, false}, {false, true}});
   ComputeAndCompareR2<bool>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementPredR1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<bool>({});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({-1, 0, 1});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<int32>(&builder, {0, -1, -2}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotS32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<int32>({{-1, 0}, {1, 8}});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   Array2D<int32> expected_array({{0, -1}, {-2, -9}});
   ComputeAndCompareR2<int32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementS32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<int32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>({0, 4294967295});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<uint32>(&builder, {4294967295, 0}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotU32R2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2<uint32>({{0, 4294967295}, {1, 4294967294}});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   Array2D<uint32> expected_array({{4294967295, 0}, {4294967294, 1}});
   ComputeAndCompareR2<uint32>(&builder, expected_array, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NotZeroElementU32R1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>({});
-  auto out = builder.Not(a);
+  builder.Not(a);
 
   ComputeAndCompareR1<uint32>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftLeftS32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({static_cast<int32>(0x12345678),
                                       static_cast<int32>(0xF0001000), 1, 3, 77,
                                       1, -3, 77});
   auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 15, 32, 100, -1});
-  auto out = builder.ShiftLeft(a, b);
+  builder.ShiftLeft(a, b);
 
   ComputeAndCompareR1<int32>(&builder,
                              {static_cast<int32>(0x23456780), 0x00100000, 0x4,
@@ -967,12 +967,12 @@ XLA_TEST_F(ArrayElementwiseOpTest, ShiftLeftS32) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightArithmeticS32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({static_cast<int32>(0x92345678),
                                       static_cast<int32>(0x10001000), 1, 3, 77,
                                       1, -3, 77});
   auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 2, 32, 100, -1});
-  auto out = builder.ShiftRightArithmetic(a, b);
+  builder.ShiftRightArithmetic(a, b);
 
   ComputeAndCompareR1<int32>(
       &builder,
@@ -982,45 +982,45 @@ XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightArithmeticS32) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightLogicalS32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({static_cast<int32>(0x92345678),
                                       static_cast<int32>(0x10001000), 1, 3, 77,
                                       1, -3, 77});
   auto b = builder.ConstantR1<int32>({4, 8, 2, 7, 5, 32, 100, -1});
-  auto out = builder.ShiftRightLogical(a, b);
+  builder.ShiftRightLogical(a, b);
 
   ComputeAndCompareR1<int32>(&builder,
                              {0x09234567, 0x00100010, 0, 0, 2, 0, 0, 0}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftLeftU32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>(
       {0x12345678, 0xF0001000, 1, 3, 77, 1, ~3u, 77});
   auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 15, 32, 100, ~0u});
-  auto out = builder.ShiftLeft(a, b);
+  builder.ShiftLeft(a, b);
 
   ComputeAndCompareR1<uint32>(
       &builder, {0x23456780, 0x00100000, 0x4, 0x180, 2523136, 0, 0, 0}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightArithmeticU32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>(
       {0x92345678, 0x10001000, 1, 3, 77, 1, ~3u, 77});
   auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 2, 32, 100, ~0u});
-  auto out = builder.ShiftRightArithmetic(a, b);
+  builder.ShiftRightArithmetic(a, b);
 
   ComputeAndCompareR1<uint32>(
       &builder, {0xF9234567, 0x00100010, 0, 0, 19, 0, ~0u, 0}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightLogicalU32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<uint32>(
       {0x92345678, 0x10001000, 1, 3, 77, 1, ~3u, 77});
   auto b = builder.ConstantR1<uint32>({4, 8, 2, 7, 5, 32, 100, ~0u});
-  auto out = builder.ShiftRightLogical(a, b);
+  builder.ShiftRightLogical(a, b);
 
   ComputeAndCompareR1<uint32>(&builder,
                               {0x09234567, 0x00100010, 0, 0, 2, 0, 0, 0}, {});
@@ -1028,59 +1028,59 @@ XLA_TEST_F(ArrayElementwiseOpTest, ShiftRightLogicalU32) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 5.0f, 2.25f, 10.0f, NAN});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {false, false, true, false, false}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({});
   auto rhs = builder.ConstantR1<float>({});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGeF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 5.0f, 1.0f, 10.0f, NAN});
-  auto compare = builder.Ge(lhs, rhs);
+  builder.Ge(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {false, true, true, false, false}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGtF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 5.0f, 1.0f, 10.0f, NAN});
-  auto compare = builder.Gt(lhs, rhs);
+  builder.Gt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {false, true, true, false, false}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLeF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 5.0f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 5.0f, 1.0f, 10.0f, NAN});
-  auto compare = builder.Le(lhs, rhs);
+  builder.Le(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {true, true, false, false, false}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLtF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 5.0f, 1.0f, 10.0f, NAN});
-  auto compare = builder.Lt(lhs, rhs);
+  builder.Lt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {true, false, false, false, false}, {});
 }
@@ -1088,10 +1088,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareLtF32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, false, false, false, true, false, false, false, true},
@@ -1099,17 +1099,17 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareEqS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqZeroElementS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({});
   auto rhs = builder.ConstantR1<int32>({});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqC64s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<complex64>({{-2.5f, 10.0f},
                                             {1.0f, 25.5f},
                                             {2.25f, -3.0f},
@@ -1120,16 +1120,16 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareEqC64s) {
                                             {2.25f, -3.0f},
                                             {10.0f, 0.0f},
                                             {1.0f, NAN}});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {false, false, true, false, false}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqZeroElementC64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<complex64>({});
   auto rhs = builder.ConstantR1<complex64>({});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {}, {});
 }
@@ -1138,7 +1138,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeC64s) {
   // Disable fast-math because we're operating on NaNs.
   SetFastMathDisabled(true);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<complex64>({{-2.5f, 10.0f},
                                             {1.0f, 25.5f},
                                             {2.25f, -3.0f},
@@ -1149,7 +1149,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeC64s) {
                                             {2.25f, -3.0f},
                                             {10.0f, 0.0f},
                                             {1.0f, NAN}});
-  auto compare = builder.Ne(lhs, rhs);
+  builder.Ne(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {true, true, false, true, true}, {});
 }
@@ -1158,10 +1158,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeF32s) {
   // Disable fast-math because we're operating on NaNs.
   SetFastMathDisabled(true);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.5f, 25.5f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({10.0f, 25.5f, 1.0f, 10.0f, NAN});
-  auto compare = builder.Ne(lhs, rhs);
+  builder.Ne(lhs, rhs);
 
   ComputeAndCompareR1<bool>(&builder, {true, false, true, true, true}, {});
 }
@@ -1169,10 +1169,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeF32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareNeS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Ne(lhs, rhs);
+  builder.Ne(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, true, true, true, false, true, true, true, false}, {});
@@ -1181,10 +1181,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeS32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGeS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Ge(lhs, rhs);
+  builder.Ge(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, false, false, true, true, false, true, true, true}, {});
@@ -1193,10 +1193,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGeS32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGtS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Gt(lhs, rhs);
+  builder.Gt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, false, false, true, false, false, true, true, false},
@@ -1206,10 +1206,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtS32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLeS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Le(lhs, rhs);
+  builder.Le(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, true, true, false, true, true, false, false, true}, {});
@@ -1218,10 +1218,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareLeS32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLtS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<int32>({min, min, min, 0, 0, 0, max, max, max});
   auto rhs = builder.ConstantR1<int32>({min, 0, max, -1, 0, 1, min, 0, max});
-  auto compare = builder.Lt(lhs, rhs);
+  builder.Lt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, true, true, false, false, true, false, false, false},
@@ -1230,10 +1230,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareLtS32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareEqU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Eq(lhs, rhs);
+  builder.Eq(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, false, false, false, true, false, false, false, true},
@@ -1242,10 +1242,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareEqU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareNeU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Ne(lhs, rhs);
+  builder.Ne(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, true, true, true, false, true, true, true, false}, {});
@@ -1253,10 +1253,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareNeU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGeU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Ge(lhs, rhs);
+  builder.Ge(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, false, false, true, true, false, true, true, true}, {});
@@ -1264,10 +1264,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGeU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGtU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Gt(lhs, rhs);
+  builder.Gt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, false, false, true, false, false, true, true, false},
@@ -1276,10 +1276,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLeU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Le(lhs, rhs);
+  builder.Le(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {true, true, true, false, true, true, false, false, true}, {});
@@ -1287,10 +1287,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareLeU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, CompareLtU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<uint32>({0, 0, 0, 5, 5, 5, max, max, max});
   auto rhs = builder.ConstantR1<uint32>({0, 1, max, 4, 5, 6, 0, 1, max});
-  auto compare = builder.Lt(lhs, rhs);
+  builder.Lt(lhs, rhs);
 
   ComputeAndCompareR1<bool>(
       &builder, {false, true, true, false, false, true, false, false, false},
@@ -1299,12 +1299,12 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareLtU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, PowF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs =
       builder.ConstantR1<float>({4.0f, 2.0f, 2.0f, NAN, 6.0f, -2.0f, -2.0f});
   auto rhs =
       builder.ConstantR1<float>({2.0f, -2.0f, 3.0f, 10.0f, NAN, 3.0f, 4.0f});
-  auto minimum = builder.Pow(lhs, rhs);
+  builder.Pow(lhs, rhs);
 
   ComputeAndCompareR1<float>(
       &builder, {16.0f, 0.25f, 8.0f, NAN, NAN, -8.0f, 16.0f}, {}, error_spec_);
@@ -1312,20 +1312,20 @@ XLA_TEST_F(ArrayElementwiseOpTest, PowF32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, PowNonIntegerF32s) {
   SetFastMathDisabled(true);
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({-2.0f, -0.6f, -0.6f, 0.0f});
   auto rhs = builder.ConstantR1<float>({0.5f, 0.6f, -0.6f, -0.6f});
-  auto minimum = builder.Pow(lhs, rhs);
+  builder.Pow(lhs, rhs);
 
   ComputeAndCompareR1<float>(&builder, {NAN, NAN, NAN, INFINITY}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, PowZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({});
   auto rhs = builder.ConstantR1<float>({});
-  auto minimum = builder.Pow(lhs, rhs);
+  builder.Pow(lhs, rhs);
 
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
@@ -1599,14 +1599,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, Div4F32) {
 
 TEST_P(ArrayElementwiseOpTestParamCount, SquareManyValues) {
   const int count = GetParam();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::vector<float> values;
   values.reserve(count);
   for (int i = 0; i < count; ++i) {
     values.push_back(i / static_cast<float>(count));
   }
   auto x = builder.ConstantR1<float>(values);
-  auto exp = builder.Pow(x, builder.ConstantR0<float>(2.0f));
+  builder.Pow(x, builder.ConstantR0<float>(2.0f));
 
   std::vector<float> expected;
   expected.reserve(values.size());
@@ -1618,7 +1618,7 @@ TEST_P(ArrayElementwiseOpTestParamCount, SquareManyValues) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SquareIn4D) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array4D<float> values(2, 2, 2, 2);
 
   std::vector<float> values_vector;
@@ -1632,77 +1632,77 @@ XLA_TEST_F(ArrayElementwiseOpTest, SquareIn4D) {
   Array4D<float> expected(2, 2, 2, 2, expected_vector);
 
   auto x = builder.ConstantR4FromArray4D<float>(values);
-  auto exp = builder.Pow(x, builder.ConstantR0<float>(2.0f));
+  builder.Pow(x, builder.ConstantR0<float>(2.0f));
 
   ComputeAndCompareR4<float>(&builder, expected, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SquareIn4DZeroElements) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array4D<float> values(2, 2, 0, 2);
   Array4D<float> expected(2, 2, 0, 2);
 
   auto x = builder.ConstantR4FromArray4D<float>(values);
-  auto exp = builder.Pow(x, builder.ConstantR0<float>(2.0f));
+  builder.Pow(x, builder.ConstantR0<float>(2.0f));
 
   ComputeAndCompareR4<float>(&builder, expected, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MinF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   SetFastMathDisabled(true);
   auto lhs = builder.ConstantR1<float>({1.0f, 1.0f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({2.0f, -5.0f, 1.0f, 10.0f, NAN});
-  auto minimum = builder.Min(lhs, rhs);
+  builder.Min(lhs, rhs);
 
   ComputeAndCompareR1<float>(&builder, {1.0f, -5.0f, 1.0f, NAN, NAN}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MinZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({});
   auto rhs = builder.ConstantR1<float>({});
-  auto minimum = builder.Min(lhs, rhs);
+  builder.Min(lhs, rhs);
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MinF64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   SetFastMathDisabled(true);
   auto lhs = builder.ConstantR1<double>({1.0, 1.0, 2.25, NAN, 6.0});
   auto rhs = builder.ConstantR1<double>({2.0, -5.0, 1.0, 10.0, NAN});
-  auto minimum = builder.Min(lhs, rhs);
+  builder.Min(lhs, rhs);
 
   ComputeAndCompareR1<double>(&builder, {1.0, -5.0, 1.0, NAN, NAN}, {},
                               error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   SetFastMathDisabled(true);
   auto lhs = builder.ConstantR1<float>({1.0f, 1.0f, 2.25f, NAN, 6.0f});
   auto rhs = builder.ConstantR1<float>({2.0f, -5.0f, 1.0f, 10.0f, NAN});
-  auto maximum = builder.Max(lhs, rhs);
+  builder.Max(lhs, rhs);
 
   ComputeAndCompareR1<float>(&builder, {2.0f, 1.0f, 2.25f, NAN, NAN}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto lhs = builder.ConstantR1<float>({});
   auto rhs = builder.ConstantR1<float>({});
-  auto minimum = builder.Max(lhs, rhs);
+  builder.Max(lhs, rhs);
   ComputeAndCompareR1<float>(&builder, {}, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxF64s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   SetFastMathDisabled(true);
   auto lhs = builder.ConstantR1<double>({1.0, 1.0, 2.25, NAN, 6.0});
   auto rhs = builder.ConstantR1<double>({2.0, -5.0, 1.0, 10.0, NAN});
-  auto maximum = builder.Max(lhs, rhs);
+  builder.Max(lhs, rhs);
 
   ComputeAndCompareR1<double>(&builder, {2.0, 1.0, 2.25, NAN, NAN}, {},
                               error_spec_);
@@ -1711,7 +1711,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxF64s) {
 XLA_TEST_F(ArrayElementwiseOpTest, MaxS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<int32>(
       {min, min, min, -1, -1, 0, 0, 0, 1, 1, max, max, max});
   auto y = builder.ConstantR1<int32>(
@@ -1726,7 +1726,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxS32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, MinS32s) {
   const int32 min = std::numeric_limits<int32>::min();
   const int32 max = std::numeric_limits<int32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<int32>(
       {min, min, min, -1, -1, 0, 0, 0, 1, 1, max, max, max});
   auto y = builder.ConstantR1<int32>(
@@ -1740,7 +1740,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MinS32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<uint32>({0, 0, 1, 1, 1, max, max, max});
   auto y = builder.ConstantR1<uint32>({0, 1, 0, 1, 10, 0, 234234, max});
   builder.Max(x, y);
@@ -1751,7 +1751,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxU32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, MinU32s) {
   const uint32 max = std::numeric_limits<uint32>::max();
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<uint32>({0, 0, 1, 1, 1, max, max, max});
   auto y = builder.ConstantR1<uint32>({0, 1, 0, 1, 10, 0, 234234, max});
   builder.Min(x, y);
@@ -1761,7 +1761,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MinU32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxTenF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<float>(
       {-0.0, 1.0, 2.0, -3.0, -4.0, 5.0, 6.0, -7.0, -8.0, 9.0});
   auto y = builder.ConstantR1<float>(
@@ -1774,7 +1774,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxTenF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxR1S1AndR1S0F32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto u = builder.ConstantR1<float>({3.5});
   auto v = builder.ConstantR1<float>({});
   builder.Max(u, v);
@@ -1784,7 +1784,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxR1S1AndR1S0F32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxR1S0AndR2S0x2F32s) {
   for (int broadcast_dim : {0, 1}) {
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto u = builder.ConstantR1<float>({3.5});
     auto v = builder.ConstantR2FromArray2D<float>(Array2D<float>(0, 2));
     builder.Max(u, v, /*broadcast_dimensions=*/{broadcast_dim});
@@ -1794,7 +1794,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxR1S0AndR2S0x2F32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Max1DAnd2DF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<float>({2.0f, 3.0f, 4.0f});
   auto m =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
@@ -1805,7 +1805,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Max1DAnd2DF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Max1DAnd2DZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<float>({});
   auto m = builder.ConstantR2<float>({{}, {}});
   builder.Max(v, m, /*broadcast_dimensions=*/{1});
@@ -1815,7 +1815,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Max1DAnd2DZeroElementF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Max3DAndScalarS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto scalar = builder.ConstantR0<int32>(2);
   Array3D<int32> a_3d({{{3, 9, -1}, {2, -10, 3}}, {{-2, 2, 8}, {12, 10, 4}}});
   auto array = builder.ConstantR3FromArray3D<int32>(a_3d);
@@ -1826,7 +1826,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Max3DAndScalarS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Max3DAndScalarZeroElementS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto scalar = builder.ConstantR0<int32>(2);
   Array3D<int32> a_3d(2, 0, 3);
   auto array = builder.ConstantR3FromArray3D<int32>(a_3d);
@@ -1837,7 +1837,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Max3DAndScalarZeroElementS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo1DF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto m =
       builder.ConstantR2<float>({{-10.4f, 64.0f, 6.0f}, {0.1f, 32.0f, 16.1f}});
   auto v = builder.ConstantR1<float>({-10.2f, 16.4f});
@@ -1848,7 +1848,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo1DF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo1DZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto m = builder.ConstantR2<float>({{}, {}});
   auto v = builder.ConstantR1<float>({-10.2f, 16.4f});
   builder.Min(m, v, /*broadcast_dimensions=*/{0});
@@ -1858,7 +1858,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo1DZeroElementF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo4DF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto array2d =
       builder.ConstantR2<float>({{-12.2f, 64.3f, 6.1f}, {0.0f, 32.2f, 2.5f}});
   auto array4d = builder.ConstantR4FromArray4D<float>(
@@ -1873,7 +1873,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo4DF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo4DZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto array2d =
       builder.ConstantR2<float>({{-12.2f, 64.3f, 6.1f}, {0.0f, 32.2f, 2.5f}});
   Array4D<float> arg(2, 2, 0, 3);
@@ -1885,7 +1885,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Min2DTo4DZeroElementF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MinTenS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<int32>({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
   auto y = builder.ConstantR1<int32>({9, 8, 7, 6, 5, 4, 3, 2, 1, 0});
   builder.Min(x, y);
@@ -1895,7 +1895,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, MinTenS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, MaxTenS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.ConstantR1<int32>({0, 1, 2, 3, 4, 5, 6, 7, 8, 9});
   auto y = builder.ConstantR1<int32>({9, 8, 7, 6, 5, 4, 3, 2, 1, 0});
   builder.Max(x, y);
@@ -1905,10 +1905,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, MaxTenS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, RemTwoConstantS32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<int32>({-3, 26, 2, -1, 1});
   auto b = builder.ConstantR1<int32>({10, 5, 1, 10, -10});
-  auto add = builder.Rem(a, b);
+  builder.Rem(a, b);
 
   ComputeAndCompareR1<int32>(&builder, {-3, 1, 0, -1, 1}, {});
 }
@@ -2635,7 +2635,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtR3F32sWithDegenerateDim2) {
   Array3D<float> b_3d({{{7.0f, 1.0f}, {3.0f, 10.0f}, {15.0f, 6.0f}}});
   auto b = builder.ConstantR3FromArray3D<float>(b_3d);
 
-  auto compare = builder.Gt(a, b);
+  builder.Gt(a, b);
 
   Array3D<int> expected_3d(
       {{{0, 1}, {0, 0}, {0, 0}}, {{0, 1}, {1, 0}, {0, 1}}});
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 01aa6c756f..c39597c4e1 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -96,6 +96,9 @@ class ClientLibraryTestBase : public ::testing::Test {
       ComputationBuilder* builder,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments);
 
+  // TODO(b/74197823): Remove the template type 'BuilderT' in all methods once
+  // the migration to XlaBuilder is complete.
+
   template <typename BuilderT>
   StatusOr<std::unique_ptr<Literal>> ExecuteAndTransfer(
       BuilderT* builder, tensorflow::gtl::ArraySlice<GlobalData*> arguments,
@@ -127,14 +130,14 @@ class ClientLibraryTestBase : public ::testing::Test {
   // Convenience methods for building and running a computation, transferring
   // the result, and comparing it to the expected value(s). Methods are
   // templated on the native host type which maps to specific XLA types (See
-  // ComputationBuilder for details). For each rank, two forms are provided: one
-  // for floating point types with an ErrorSpec parameter, and one for integral
-  // types without the ErrorSpec parameter.
-  template <typename NativeT>
-  void ComputeAndCompareR0(ComputationBuilder* builder, NativeT expected,
+  // ComputationBuilder/XlaBuilder for details). For each rank, two forms are
+  // provided: one for floating point types with an ErrorSpec parameter, and one
+  // for integral types without the ErrorSpec parameter.
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR0(BuilderT* builder, NativeT expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
-  template <typename NativeT>
-  void ComputeAndCompareR0(ComputationBuilder* builder, NativeT expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR0(BuilderT* builder, NativeT expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
 
@@ -154,33 +157,27 @@ class ClientLibraryTestBase : public ::testing::Test {
                            const tensorflow::core::Bitmap& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
 
-  template <typename NativeT>
-  void ComputeAndCompareR2(ComputationBuilder* builder,
-                           const Array2D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR2(BuilderT* builder, const Array2D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
-  template <typename NativeT>
-  void ComputeAndCompareR2(ComputationBuilder* builder,
-                           const Array2D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR2(BuilderT* builder, const Array2D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
 
-  template <typename NativeT>
-  void ComputeAndCompareR3(ComputationBuilder* builder,
-                           const Array3D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR3(BuilderT* builder, const Array3D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
-  template <typename NativeT>
-  void ComputeAndCompareR3(ComputationBuilder* builder,
-                           const Array3D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR3(BuilderT* builder, const Array3D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
 
-  template <typename NativeT>
-  void ComputeAndCompareR4(ComputationBuilder* builder,
-                           const Array4D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR4(BuilderT* builder, const Array4D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments);
-  template <typename NativeT>
-  void ComputeAndCompareR4(ComputationBuilder* builder,
-                           const Array4D<NativeT>& expected,
+  template <typename NativeT, typename BuilderT>
+  void ComputeAndCompareR4(BuilderT* builder, const Array4D<NativeT>& expected,
                            tensorflow::gtl::ArraySlice<GlobalData*> arguments,
                            ErrorSpec error);
 
@@ -337,10 +334,12 @@ class ClientLibraryTestBase : public ::testing::Test {
   //
   // When the use_bfloat16 flag is set but NativeT is float, the data will be
   // converted to bfloat16.
-  template <typename NativeT>
-  std::unique_ptr<GlobalData> CreateR0Parameter(
-      NativeT value, int64 parameter_number, const string& name,
-      ComputationBuilder* builder, ComputationDataHandle* data_handle);
+  template <typename NativeT, typename BuilderT, typename HandleT>
+  std::unique_ptr<GlobalData> CreateR0Parameter(NativeT value,
+                                                int64 parameter_number,
+                                                const string& name,
+                                                BuilderT* builder,
+                                                HandleT* data_handle);
 
   // Creates a parameter instruction that wraps the given values and then stores
   // into "data_handle" the global handle for that parameter.
@@ -350,11 +349,10 @@ class ClientLibraryTestBase : public ::testing::Test {
   //
   // When the use_bfloat16 flag is set but NativeT is float, the data will be
   // converted to bfloat16.
-  template <typename NativeT>
+  template <typename NativeT, typename BuilderT, typename HandleT>
   std::unique_ptr<GlobalData> CreateR1Parameter(
       tensorflow::gtl::ArraySlice<NativeT> values, int64 parameter_number,
-      const string& name, ComputationBuilder* builder,
-      ComputationDataHandle* data_handle);
+      const string& name, BuilderT* builder, HandleT* data_handle);
 
   // Creates a parameter instruction that wraps the given constant array
   // "array_2d" and then stores to "data_handle" the global handle for that
@@ -365,11 +363,10 @@ class ClientLibraryTestBase : public ::testing::Test {
   //
   // When the use_bfloat16 flag is set but NativeT is float, the data will be
   // converted to bfloat16.
-  template <typename NativeT>
+  template <typename NativeT, typename BuilderT, typename HandleT>
   std::unique_ptr<GlobalData> CreateR2Parameter(
       const Array2D<NativeT>& array_2d, int64 parameter_number,
-      const string& name, ComputationBuilder* builder,
-      ComputationDataHandle* data_handle);
+      const string& name, BuilderT* builder, HandleT* data_handle);
 
   // Creates a parameter instruction that wraps the given constant array
   // "array_3d" and then stores to "data_handle" the global handle for that
@@ -380,11 +377,10 @@ class ClientLibraryTestBase : public ::testing::Test {
   //
   // When the use_bfloat16 flag is set but NativeT is float, the data will be
   // converted to bfloat16.
-  template <typename NativeT>
+  template <typename NativeT, typename BuilderT, typename HandleT>
   std::unique_ptr<GlobalData> CreateR3Parameter(
       const Array3D<NativeT>& array_3d, int64 parameter_number,
-      const string& name, ComputationBuilder* builder,
-      ComputationDataHandle* data_handle);
+      const string& name, BuilderT* builder, HandleT* data_handle);
 
   // Getter and setter for the use_bfloat16 flag, which indicates whether to run
   // tests with all float-type input/output converted to bfloat16.
@@ -440,9 +436,9 @@ class ClientLibraryTestBase : public ::testing::Test {
   std::vector<std::unique_ptr<GlobalData>> arguments_;
 };
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR0(
-    ComputationBuilder* builder, NativeT expected,
+    BuilderT* builder, NativeT expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR0<NativeT>(expected);
@@ -450,9 +446,9 @@ void ClientLibraryTestBase::ComputeAndCompareR0(
                                                   arguments);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR0(
-    ComputationBuilder* builder, NativeT expected,
+    BuilderT* builder, NativeT expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
                     std::is_same<NativeT, double>::value ||
@@ -492,9 +488,9 @@ void ClientLibraryTestBase::ComputeAndCompareR1(
                                                   arguments, error);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR2(
-    ComputationBuilder* builder, const Array2D<NativeT>& expected,
+    BuilderT* builder, const Array2D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR2FromArray2D<NativeT>(expected);
@@ -502,9 +498,9 @@ void ClientLibraryTestBase::ComputeAndCompareR2(
                                                   arguments);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR2(
-    ComputationBuilder* builder, const Array2D<NativeT>& expected,
+    BuilderT* builder, const Array2D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
                     std::is_same<NativeT, double>::value ||
@@ -518,9 +514,9 @@ void ClientLibraryTestBase::ComputeAndCompareR2(
                                                   arguments, error);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR3(
-    ComputationBuilder* builder, const Array3D<NativeT>& expected,
+    BuilderT* builder, const Array3D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR3FromArray3D<NativeT>(expected);
@@ -528,9 +524,9 @@ void ClientLibraryTestBase::ComputeAndCompareR3(
                                                   arguments);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR3(
-    ComputationBuilder* builder, const Array3D<NativeT>& expected,
+    BuilderT* builder, const Array3D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
                     std::is_same<NativeT, double>::value ||
@@ -544,9 +540,9 @@ void ClientLibraryTestBase::ComputeAndCompareR3(
                                                   arguments, error);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR4(
-    ComputationBuilder* builder, const Array4D<NativeT>& expected,
+    BuilderT* builder, const Array4D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   std::unique_ptr<Literal> expected_literal =
       Literal::CreateR4FromArray4D<NativeT>(expected);
@@ -554,9 +550,9 @@ void ClientLibraryTestBase::ComputeAndCompareR4(
                                                   arguments);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareR4(
-    ComputationBuilder* builder, const Array4D<NativeT>& expected,
+    BuilderT* builder, const Array4D<NativeT>& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   static_assert(std::is_same<NativeT, float>::value ||
                     std::is_same<NativeT, double>::value ||
@@ -570,10 +566,10 @@ void ClientLibraryTestBase::ComputeAndCompareR4(
                                                   arguments, error);
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT, typename HandleT>
 std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR0Parameter(
     NativeT value, int64 parameter_number, const string& name,
-    ComputationBuilder* builder, ComputationDataHandle* data_handle) {
+    BuilderT* builder, HandleT* data_handle) {
   std::unique_ptr<Literal> literal = Literal::CreateR0(value);
   if (use_bfloat16_ && literal->shape().element_type() == F32) {
     literal = LiteralTestUtil::ConvertF32ToBF16(*literal);
@@ -584,11 +580,10 @@ std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR0Parameter(
   return data;
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT, typename HandleT>
 std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR1Parameter(
     tensorflow::gtl::ArraySlice<NativeT> values, int64 parameter_number,
-    const string& name, ComputationBuilder* builder,
-    ComputationDataHandle* data_handle) {
+    const string& name, BuilderT* builder, HandleT* data_handle) {
   std::unique_ptr<Literal> literal = Literal::CreateR1(values);
   if (use_bfloat16_ && literal->shape().element_type() == F32) {
     literal = LiteralTestUtil::ConvertF32ToBF16(*literal);
@@ -599,11 +594,10 @@ std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR1Parameter(
   return data;
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT, typename HandleT>
 std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR2Parameter(
     const Array2D<NativeT>& array_2d, int64 parameter_number,
-    const string& name, ComputationBuilder* builder,
-    ComputationDataHandle* data_handle) {
+    const string& name, BuilderT* builder, HandleT* data_handle) {
   std::unique_ptr<Literal> literal = Literal::CreateR2FromArray2D(array_2d);
   if (use_bfloat16_ && literal->shape().element_type() == F32) {
     literal = LiteralTestUtil::ConvertF32ToBF16(*literal);
@@ -614,11 +608,10 @@ std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR2Parameter(
   return data;
 }
 
-template <typename NativeT>
+template <typename NativeT, typename BuilderT, typename HandleT>
 std::unique_ptr<GlobalData> ClientLibraryTestBase::CreateR3Parameter(
     const Array3D<NativeT>& array_3d, int64 parameter_number,
-    const string& name, ComputationBuilder* builder,
-    ComputationDataHandle* data_handle) {
+    const string& name, BuilderT* builder, HandleT* data_handle) {
   std::unique_ptr<Literal> literal = Literal::CreateR3FromArray3D(array_3d);
   if (use_bfloat16_ && literal->shape().element_type() == F32) {
     literal = LiteralTestUtil::ConvertF32ToBF16(*literal);
-- 
GitLab


From eee15c1f8ea56dbb516fa9e35392e0a224e99966 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 15:34:21 -0700
Subject: [PATCH 512/960] Update recompute_grad for TPU

PiperOrigin-RevId: 190536468
---
 .../layers/python/layers/rev_block_lib.py     | 105 +++++++++++++++++-
 .../python/layers/rev_block_lib_test.py       |  61 +++++++---
 2 files changed, 146 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py
index 123275e1fd..0b38c0c3fd 100644
--- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py
+++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py
@@ -29,6 +29,7 @@ from __future__ import print_function
 import functools
 import re
 
+import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.framework.python import ops as contrib_framework_ops
@@ -37,6 +38,7 @@ from tensorflow.python.framework import ops as framework_ops
 from tensorflow.python.layers import base
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import control_flow_util
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
@@ -46,6 +48,7 @@ from tensorflow.python.util import nest
 __all__ = ["rev_block", "RevBlock", "recompute_grad"]
 
 LAYER_RE = re.compile(".*revlayer_([0-9]*)/([fg])/.*")
+_USE_DEFAULT = "__rev_block_lib_default"
 
 
 def _acc_grads(*lists_of_grads):
@@ -219,7 +222,13 @@ class RevBlock(base.Layer):
 
   def _efficient_grad_fn(self, inputs, variables, ys, grad_ys):
     """Custom gradient fn for a block of reversible residual layers."""
+    # Inputs have passed through an Identity. Recover the original Tensors to
+    # be able to match up side inputs.
+    assert [u"Identity"] == list(set([x.op.type for x in inputs]))
+    inputs = [x.op.inputs[0] for x in inputs]
     side_inputs = inputs[2:]
+    del inputs
+
     f_side_idxs = [None] * len(self.f_side_input)
     g_side_idxs = [None] * len(self.g_side_input)
     assert len(side_inputs) == len(self.f_side_input) + len(self.g_side_input)
@@ -405,12 +414,36 @@ def rev_block(x1,
   return block.forward(x1, x2)
 
 
-def recompute_grad(fn):
+def enable_with_args(dec):
+  """A decorator for decorators to enable their usage with or without args."""
+
+  @functools.wraps(dec)
+  def new_dec(*args, **kwargs):
+    if len(args) == 1 and not kwargs and callable(args[0]):
+      # Used as decorator without args
+      fn = args[0]
+      return dec(fn)
+    else:
+      return lambda fn: dec(fn, *args, **kwargs)
+
+  return new_dec
+
+
+@enable_with_args
+def recompute_grad(fn, use_data_dep=_USE_DEFAULT, tupleize_grads=False):
   """Decorator that recomputes the function on the backwards pass.
 
   Args:
     fn: a function that takes Tensors (all as positional arguments) and returns
       a tuple of Tensors.
+    use_data_dep: `bool`, if `True` will use a dummy data dependency to force
+      the recompute to happen. If `False` will use a control dependency. By
+      default will be `True` if in an XLA context and `False` otherwise. XLA
+      ignores control dependencies and so this data dependency is necessary.
+    tupleize_grads: `bool`, if `True` will use control dependencies to ensure
+      that all gradients are produced before any are consumed by downstream ops.
+      If `use_data_dep` is also `True`, will use a data dependency instead of
+      a control dependency.
 
   Returns:
     A wrapped fn that is identical to fn when called, but its activations will
@@ -420,13 +453,25 @@ def recompute_grad(fn):
 
   @functools.wraps(fn)
   def wrapped(*args):
-    return _recompute_grad(fn, args)
+    return _recompute_grad(
+        fn, args, use_data_dep=use_data_dep, tupleize_grads=tupleize_grads)
 
   return wrapped
 
 
-def _recompute_grad(fn, args):
+def _is_on_tpu():
+  ctxt = framework_ops.get_default_graph()._get_control_flow_context()  # pylint: disable=protected-access
+  return control_flow_util.GetContainingXLAContext(ctxt) is not None
+
+
+def _recompute_grad(fn, args, use_data_dep=_USE_DEFAULT, tupleize_grads=False):
   """See recompute_grad."""
+  for arg in args:
+    if not isinstance(arg, framework_ops.Tensor):
+      raise ValueError("All inputs to function must be Tensors")
+  use_data_dep_ = use_data_dep
+  if use_data_dep_ == _USE_DEFAULT:
+    use_data_dep_ = _is_on_tpu()
 
   cached_vs = []
   cached_arg_scope = []
@@ -436,6 +481,8 @@ def _recompute_grad(fn, args):
     del outputs
     # Recompute outputs
     with framework_ops.control_dependencies(output_grads):
+      if use_data_dep_:
+        inputs = _force_data_dependency(output_grads, inputs)
       with contrib_framework_ops.arg_scope(cached_arg_scope[0]):
         with variable_scope.variable_scope(cached_vs[0], reuse=True):
           outputs = fn(*inputs)
@@ -444,6 +491,13 @@ def _recompute_grad(fn, args):
       outputs = [outputs]
     outputs = list(outputs)
     grads = gradients_impl.gradients(outputs, inputs + variables, output_grads)
+
+    if tupleize_grads:
+      if use_data_dep_:
+        grads = _tuple_with_data_dep(grads)
+      else:
+        grads = control_flow_ops.tuple(grads)
+
     grad_inputs = grads[:len(inputs)]
     grad_vars = grads[len(inputs):]
     return grad_inputs, grad_vars
@@ -532,7 +586,7 @@ def _fn_with_custom_grad_internal(fn, inputs, grad_fn, use_global_vars=False):
   get_vars_fn = (
       vs.global_variables if use_global_vars else vs.trainable_variables)
   len_before_vars = len(get_vars_fn())
-  inputs = list(inputs)
+  inputs = [array_ops.identity(x) for x in inputs]
   outputs = fn(*inputs)
   train_vars = get_vars_fn()[len_before_vars:]
 
@@ -581,3 +635,46 @@ def _fn_with_custom_grad_internal(fn, inputs, grad_fn, use_global_vars=False):
   flat_inputs = nest.flatten(defun_inputs)
   id_out = identity(*flat_inputs)
   return id_out
+
+
+def _force_data_dependency(first_compute, then_compute):
+  """Force all of `then_compute` to depend on all of `first_compute`.
+
+  Uses a dummy data dependency, which is useful when running on TPUs because
+  XLA ignores control dependencies. Only supports float arguments.
+
+  Args:
+    first_compute: `list<Tensor>`. These will be made to run before the
+      `Tensor`s `then_compute`.
+    then_compute: `list<Tensor>`. These will run after all the `Tensor`s in
+      `first_compute`.
+
+  Returns:
+    `list<Tensor>`, same length as `then_compute`.
+
+  Raises:
+    ValueError: if ranks are unknown or types are not floating.
+  """
+
+  def _first_element(x):
+    if x.get_shape().ndims is None:
+      raise ValueError("Rank of Tensor %s must be known" % x)
+    ndims = x.get_shape().ndims
+    return array_ops.reshape(array_ops.slice(x, [0] * ndims, [1] * ndims), [])
+
+  first_compute_sum = math_ops.add_n(
+      [_first_element(x) for x in first_compute if x is not None])
+  dtype = first_compute_sum.dtype
+  if not dtype.is_floating:
+    raise ValueError("_force_data_dependency only supports floating dtypes.")
+  epsilon = np.finfo(dtype.as_numpy_dtype).tiny
+  zero = array_ops.stop_gradient(epsilon * first_compute_sum)
+
+  return [
+      array_ops.identity(x) + zero if x is not None else None
+      for x in then_compute
+  ]
+
+
+def _tuple_with_data_dep(tensors):
+  return _force_data_dependency(tensors, tensors)
diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py
index cbcbcd7511..d1ad4e8c98 100644
--- a/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py
+++ b/tensorflow/contrib/layers/python/layers/rev_block_lib_test.py
@@ -154,7 +154,7 @@ class RevBlockTest(test.TestCase):
       y_val, yd_val, gd_val, g_val = sess.run([y, y_rev, grads_rev, grads])
       self.assertAllClose(y_val, yd_val)
       for g1, g2 in zip(gd_val, g_val):
-        self.assertAllClose(g1, g2)
+        self.assertAllClose(g1, g2, rtol=1e-5)
 
   def testRevBlock(self):
     self._testRevBlock()
@@ -255,25 +255,54 @@ class RecomputeTest(test.TestCase):
     def fn_recompute(x):
       return fn(x)
 
+    @rev_block_lib.recompute_grad(use_data_dep=True)
+    def fn_use_data_dep(x):
+      return fn(x)
+
+    @rev_block_lib.recompute_grad(tupleize_grads=True)
+    def fn_tupleize(x):
+      return fn(x)
+
+    @rev_block_lib.recompute_grad(use_data_dep=True, tupleize_grads=True)
+    def fn_both(x):
+      return fn(x)
+
     x = random_ops.random_uniform((3, 1, 3))
-    recompute_vars = None
-    with variable_scope.variable_scope("recompute") as vs:
-      out1 = math_ops.reduce_sum(fn_recompute(x))
-      recompute_vars = vs.trainable_variables()
-    reg_vars = None
-    with variable_scope.variable_scope("regular") as vs:
-      out2 = math_ops.reduce_sum(fn(x))
-      reg_vars = vs.trainable_variables()
-
-    grad1 = gradients_impl.gradients(out1, recompute_vars)
-    grad2 = gradients_impl.gradients(out2, reg_vars)
+
+    names_and_fns = [
+        ("recompute", fn_recompute),
+        ("regular", fn),
+        ("use_data_dep", fn_use_data_dep),
+        ("tupleize", fn_tupleize),
+        ("tuple_and_data_dep", fn_both),
+    ]
+    outputs_and_vars = []
+    for name, wrapped_fn in names_and_fns:
+      with variable_scope.variable_scope(name) as vs:
+        out = math_ops.reduce_sum(wrapped_fn(x))
+        outputs_and_vars.append((out, vs.trainable_variables()))
+
+    all_grads = []
+    for out, scope_vars in outputs_and_vars:
+      all_grads.append(gradients_impl.gradients(out, scope_vars))
 
     with self.test_session() as sess:
       sess.run(variables.global_variables_initializer())
-      outs = sess.run([out1, out2, grad1, grad2])
-      self.assertAllClose(outs[0], outs[1])
-      for g1, g2 in zip(outs[2], outs[3]):
-        self.assertAllClose(g1, g2)
+      outputs = list(zip(*outputs_and_vars))[0]
+      outs, all_grads_val = sess.run([outputs, all_grads])
+
+      # All outputs are the same
+      current = outs[0]
+      for out in outs[1:]:
+        self.assertAllClose(current, out)
+        current = out
+
+      # All gradients are the same
+      for grads in zip(all_grads_val):
+        current = grads[0]
+        for g in grads[1:]:
+          self.assertAllClose(current, g)
+          current = g
 
 
 class FnWithCustomGradTest(test.TestCase):
-- 
GitLab


From 290632966fae0619db30c1ba777634db9a43b757 Mon Sep 17 00:00:00 2001
From: Mingsheng Hong <hongm@google.com>
Date: Mon, 26 Mar 2018 15:37:40 -0700
Subject: [PATCH 513/960] In the experimental C API, parametrized batch_size
 for the generate dataset / iterator stack.

PiperOrigin-RevId: 190536945
---
 tensorflow/c/c_api_experimental.cc | 67 ++++++++++++++++++------------
 1 file changed, 41 insertions(+), 26 deletions(-)

diff --git a/tensorflow/c/c_api_experimental.cc b/tensorflow/c/c_api_experimental.cc
index f411efc941..bea9378571 100644
--- a/tensorflow/c/c_api_experimental.cc
+++ b/tensorflow/c/c_api_experimental.cc
@@ -7125,7 +7125,8 @@ library {
 //  sets `dataset_name` to the created dataset name. The returned functions must
 //  be deleted by calling TF_DeleteFunction.
 static std::vector<UniqueFuncPtr> CreateMNISTDatasetFunctions(
-    const char* file_path, std::string* dataset_name, TF_Status* status) {
+    const char* file_path, int batch_size, std::string* dataset_name,
+    TF_Status* status) {
   const char* func_def = R"PREFIX(
 library {
   function {
@@ -8089,7 +8090,7 @@ library {
             dtype: DT_INT64
             tensor_shape {
             }
-            int64_val: 128
+            int64_val: -123
           }
         }
       }
@@ -8145,7 +8146,7 @@ library {
             dtype: DT_INT64
             tensor_shape {
             }
-            int64_val: 128
+            int64_val: -123
           }
         }
       }
@@ -8211,35 +8212,48 @@ library {
 
   *dataset_name = "_make_dataset_2451e43a";
   std::function<void(FunctionDef*)> mutate_proto_func =
-      [dataset_name, file_path](FunctionDef* fdef) {
+      [dataset_name, file_path, batch_size](FunctionDef* fdef) {
         VLOG(1) << "Processsing function " << fdef->DebugString();
         if (std::string(fdef->signature().name()) != *dataset_name) return;
         // Change the input file pattern to `file_path`.
-        bool found = false;
+        bool found_file_path = false, found_batch_size = false;
         // `node_def` may be mutated.
         for (auto& node_def : *fdef->mutable_node_def()) {
-          if (node_def.name() != "FixedLengthRecordDataset/filenames" &&
-              node_def.name() != "FixedLengthRecordDataset_1/filenames_1")
-            continue;
-          DCHECK_EQ(node_def.op(), "Const");
-          DCHECK_GT(node_def.attr().count("value"), 0);
-          found = true;
-          // Replace $(DATA_DIR)/foo with <file_path>/foo
-          // TODO(hongm): Use StringPiece manipulation for better efficiency.
-          const std::string cur_value =
-              node_def.attr().at("value").tensor().string_val(0);
-          const std::string pattern = "$(DATA_DIR)";
-          DCHECK_EQ(cur_value.compare(0, pattern.length(), pattern), 0);
-          const std::string new_value =
-              file_path + cur_value.substr(pattern.length());
-          VLOG(1) << "Setting the value of node_def " << node_def.name()
-                  << " to " << new_value;
-          auto* tensor = (*node_def.mutable_attr())["value"].mutable_tensor();
-          tensor->clear_string_val();
-          tensor->add_string_val(new_value);
+          if (node_def.name() == "FixedLengthRecordDataset/filenames" ||
+              node_def.name() == "FixedLengthRecordDataset_1/filenames_1") {
+            DCHECK_EQ(node_def.op(), "Const");
+            DCHECK_GT(node_def.attr().count("value"), 0);
+            found_file_path = true;
+            // Replace $(DATA_DIR)/foo with <file_path>/foo
+            // TODO(hongm): Use StringPiece manipulation for better efficiency.
+            const std::string cur_value =
+                node_def.attr().at("value").tensor().string_val(0);
+            const std::string pattern = "$(DATA_DIR)";
+            DCHECK_EQ(cur_value.compare(0, pattern.length(), pattern), 0);
+            const std::string new_value =
+                file_path + cur_value.substr(pattern.length());
+            VLOG(1) << "Setting the value of node_def " << node_def.name()
+                    << " to " << new_value;
+            auto* tensor = (*node_def.mutable_attr())["value"].mutable_tensor();
+            tensor->clear_string_val();
+            tensor->add_string_val(new_value);
+          } else if (node_def.name() == "BatchDataset/batch_size" ||
+                     node_def.name() == "FilterDataset/batch_size_1") {
+            DCHECK_EQ(node_def.op(), "Const");
+            DCHECK_GT(node_def.attr().count("value"), 0);
+            found_batch_size = true;
+            // Replace $(BATCH_SIZE) with `batch_size`
+            DCHECK_EQ(node_def.attr().at("value").tensor().int64_val(0), -123);
+            VLOG(1) << "Setting the batch size attr value of node_def "
+                    << node_def.name() << " to " << batch_size;
+            auto* tensor = (*node_def.mutable_attr())["value"].mutable_tensor();
+            tensor->clear_int64_val();
+            tensor->add_int64_val(batch_size);
+          }
         }
         VLOG(1) << "Rewrote function to " << fdef->DebugString();
-        DCHECK(found);
+        DCHECK(found_file_path);
+        DCHECK(found_batch_size);
       };
   return CreateFunctionsFromTextProto(func_def, &mutate_proto_func, status);
 }
@@ -8341,7 +8355,8 @@ TF_Operation* TF_MakeFileBasedIteratorGetNextWithDatasets(
   std::string dataset_name;
   const auto& funcs =
       is_mnist
-          ? CreateMNISTDatasetFunctions(file_path, &dataset_name, status)
+          ? CreateMNISTDatasetFunctions(file_path, batch_size, &dataset_name,
+                                        status)
           : CreateImagenetDatasetFunctions(file_path, &dataset_name, status);
   if (!status->status.ok()) {
     return nullptr;
-- 
GitLab


From c83a54adcface7d4bb666d7c4fd3968ba980a50d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 26 Mar 2018 15:39:54 -0700
Subject: [PATCH 514/960] Makes tf.gather not silently snapshot resource
 variables.

PiperOrigin-RevId: 190537320
---
 .../kernel_tests/attention_wrapper_test.py    | 29 +++++++++++--------
 tensorflow/python/ops/array_ops.py            | 17 +++++++----
 tensorflow/python/ops/embedding_ops.py        | 29 ++++---------------
 3 files changed, 33 insertions(+), 42 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index c4139dde49..07b3ad71d4 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -785,26 +785,31 @@ class AttentionWrapperTest(test.TestCase):
         wrapper.BahdanauAttention, wrapper.LuongAttention)
 
     expected_final_output = BasicDecoderOutput(
-        rnn_output=ResultSummary(
-            shape=(5, 3, 20), dtype=dtype('float32'), mean=0.11798714846372604),
-        sample_id=ResultSummary(
-            shape=(5, 3), dtype=dtype('int32'), mean=7.933333333333334))
+        rnn_output=ResultSummary(shape=(5, 3, 20),
+                                 dtype=dtype('float32'),
+                                 mean=0.11723966),
+        sample_id=ResultSummary(shape=(5, 3),
+                                dtype=dtype('int32'),
+                                mean=9.2666666666666675))
     expected_final_state = AttentionWrapperState(
         cell_state=LSTMStateTuple(
-            c=ResultSummary(
-                shape=(5, 9), dtype=dtype('float32'), mean=-0.0036486709),
-            h=ResultSummary(
-                shape=(5, 9), dtype=dtype('float32'), mean=-0.0018835809)),
-        attention=ResultSummary(
-            shape=(5, 20), dtype=dtype('float32'), mean=0.11798714846372604),
+            c=ResultSummary(shape=(5, 9),
+                            dtype=dtype('float32'),
+                            mean=-0.003545674),
+            h=ResultSummary(shape=(5, 9),
+                            dtype=dtype('float32'),
+                            mean=-0.0018327223)),
+        attention=ResultSummary(shape=(5, 20),
+                                dtype=dtype('float32'),
+                                mean=0.11728073),
         time=3,
         alignments=(
             ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125),
             ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)),
+        alignment_history=(),
         attention_state=(
             ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125),
-            ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)),
-        alignment_history=())
+            ResultSummary(shape=(5, 8), dtype=dtype('float32'), mean=0.125)))
     expected_final_alignment_history = (
         ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125),
         ResultSummary(shape=(3, 5, 8), dtype=dtype('float32'), mean=0.125))
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index ec7c14f7d8..9106461c60 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -2691,12 +2691,17 @@ reverse_sequence.__doc__ = deprecation.rewrite_argument_docstring(
 
 @tf_export("gather")
 def gather(params, indices, validate_indices=None, name=None, axis=0):
-  # TODO(rjryan): Remove "Gather" creation in favor of GatherV2 once the forward
-  # compatibility 3 week period has passed.
-  if axis == 0:
-    return gen_array_ops.gather(
-        params, indices, validate_indices=validate_indices, name=name)
-  else:
+  del validate_indices
+  if axis != 0:
+    # Note that we do a sparse_read here to avoid snapshotting the entire
+    # resource variable and doing a gather, which can be inefficient and lead to
+    # subtle race conditions. TODO(apassos) implement axis != 0 on sparse_read
+    return gen_array_ops.gather_v2(params, indices, axis, name=name)
+  try:
+    # TODO(apassos) find a less bad way of detecting resource variables without
+    # introducing a circular dependency.
+    return params.sparse_read(indices, name=name)
+  except AttributeError:
     return gen_array_ops.gather_v2(params, indices, axis, name=name)
 
 
diff --git a/tensorflow/python/ops/embedding_ops.py b/tensorflow/python/ops/embedding_ops.py
index 20e4a28b9c..f0120f2957 100644
--- a/tensorflow/python/ops/embedding_ops.py
+++ b/tensorflow/python/ops/embedding_ops.py
@@ -35,34 +35,14 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
 
-def _gather(params, ids, name=None):
-  """Helper function for _embedding_lookup_and_transform.
-
-  This function gathers embeddings from a single tensor. The gather deals with
-  resource variables specially.
-
-  Args:
-    params: A `Tensor` of embeddings.
-    ids: A `Tensor` indexing the embeddings to be retrieved from `params`.
-    name: A name for the operation (optional).
-
-  Returns:
-    A `Tensor` with the same type as `params`.
-  """
-  if isinstance(params, resource_variable_ops.ResourceVariable):
-    return params.sparse_read(ids, name=name)
-  else:
-    return array_ops.gather(params, ids, name=name)
-
-
 def _clip(params, ids, max_norm):
   """Helper function for _embedding_lookup_and_transform.
 
   This function optionally clips embeddings to an l2-norm of max_norm.
 
   Args:
-    params: A `Tensor` of embeddings retrieved by `_gather`.
-    ids: The `ids` argument that was passed to `_gather`.
+    params: A `Tensor` of embeddings retrieved by `gather`.
+    ids: The `ids` argument that was passed to `gather`.
     max_norm: If provided, the embeddings are l2-normalized to the value of
       max_norm.
 
@@ -148,7 +128,8 @@ def _embedding_lookup_and_transform(params,
     ids = ops.convert_to_tensor(ids, name="ids")
     if np == 1 and (not transform_fn or ids.get_shape().ndims == 1):
       with ops.colocate_with(params[0]):
-        result = _clip(_gather(params[0], ids, name=name), ids, max_norm)
+        result = _clip(array_ops.gather(params[0], ids, name=name),
+                       ids, max_norm)
         if transform_fn:
           result = transform_fn(result)
         return result
@@ -212,7 +193,7 @@ def _embedding_lookup_and_transform(params,
       for p in xrange(np):
         pids = gather_ids[p]
         with ops.colocate_with(params[p]):
-          result = _gather(params[p], pids)
+          result = array_ops.gather(params[p], pids)
           if transform_fn:
             # If transform_fn is provided, the clip_by_norm precedes
             # the transform and hence must be co-located. See below
-- 
GitLab


From db076ca01f12368c9476fa4db9d87756f22f9670 Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Mon, 26 Mar 2018 15:52:12 -0700
Subject: [PATCH 515/960] Rename convert_savedmodel to convert_saved_model

to be consistent with export_saved_model

PiperOrigin-RevId: 190539064
---
 tensorflow/contrib/lite/python/BUILD             | 12 ++++++------
 ...vert_savedmodel.py => convert_saved_model.py} | 15 ++++++++-------
 ...model_test.py => convert_saved_model_test.py} | 16 ++++++++--------
 3 files changed, 22 insertions(+), 21 deletions(-)
 rename tensorflow/contrib/lite/python/{convert_savedmodel.py => convert_saved_model.py} (96%)
 rename tensorflow/contrib/lite/python/{convert_savedmodel_test.py => convert_saved_model_test.py} (96%)

diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index ce1a81d06b..411d5c0d27 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -85,8 +85,8 @@ py_test(
 )
 
 py_binary(
-    name = "convert_savedmodel",
-    srcs = ["convert_savedmodel.py"],
+    name = "convert_saved_model",
+    srcs = ["convert_saved_model.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
@@ -98,12 +98,12 @@ py_binary(
 )
 
 py_test(
-    name = "convert_savedmodel_test",
-    srcs = ["convert_savedmodel_test.py"],
+    name = "convert_saved_model_test",
+    srcs = ["convert_saved_model_test.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
-        ":convert_savedmodel",
+        ":convert_saved_model",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:platform_test",
         "//tensorflow/python:session",
@@ -115,7 +115,7 @@ py_test(
 py_library(
     name = "tf_lite_py_pip",
     deps = [
-        ":convert_savedmodel",
+        ":convert_saved_model",
     ],
 )
 
diff --git a/tensorflow/contrib/lite/python/convert_savedmodel.py b/tensorflow/contrib/lite/python/convert_saved_model.py
similarity index 96%
rename from tensorflow/contrib/lite/python/convert_savedmodel.py
rename to tensorflow/contrib/lite/python/convert_saved_model.py
index d39e1a1d98..a2b5ef488e 100644
--- a/tensorflow/contrib/lite/python/convert_savedmodel.py
+++ b/tensorflow/contrib/lite/python/convert_saved_model.py
@@ -16,7 +16,7 @@ r"""TensorFlow Lite flatbuffer generation from saved_models.
 
 Example:
 
-bazel run third_party/tensorflow/contrib/lite/python:convert_savedmodel -- \
+bazel run third_party/tensorflow/contrib/lite/python:convert_saved_model -- \
   --saved_model_dir=/tmp/test_saved_model/1519865537 \
   --output_tflite=/tmp/test.lite
 
@@ -68,16 +68,16 @@ def log_tensor_details(tensor_info):
       dims = [str(dim.size) for dim in val.tensor_shape.dim]
       shape = "({})".format(", ".join(dims))
 
-    logging.info("Tensor's key in savedmodel's tensor_map: %s", key)
+    logging.info("Tensor's key in saved_model's tensor_map: %s", key)
     logging.info(" tensor name: %s, shape: %s, type: %s", val.name, shape,
                  dtype)
 
 
 def get_meta_graph_def(saved_model_dir, tag_set):
-  """Validate savedmodel and extract MetaGraphDef.
+  """Validate saved_model and extract MetaGraphDef.
 
   Args:
-    saved_model_dir: Savedmodel path to convert.
+    saved_model_dir: saved_model path to convert.
     tag_set: Set of tag(s) of the MetaGraphDef to load.
 
   Returns:
@@ -94,7 +94,8 @@ def get_meta_graph_def(saved_model_dir, tag_set):
     tag_sets.append(meta_graph_tag_set)
     if meta_graph_tag_set == tag_set:
       result_meta_graph_def = meta_graph_def
-  logging.info("The given SavedModel contains the following tags: %s", tag_sets)
+  logging.info("The given saved_model contains the following tags: %s",
+               tag_sets)
   if result_meta_graph_def is not None:
     return result_meta_graph_def
   else:
@@ -118,7 +119,7 @@ def get_signature_def(meta_graph, signature_key):
   signature_def_map = meta_graph.signature_def
   signature_def_keys = set(signature_def_map.keys())
   logging.info(
-      "The given SavedModel MetaGraphDef contains SignatureDefs with the "
+      "The given saved_model MetaGraphDef contains SignatureDefs with the "
       "following keys: %s", signature_def_keys)
   if signature_key not in signature_def_keys:
     raise ValueError("No '{}' in the saved_model\'s SignatureDefs. Possible "
@@ -159,7 +160,7 @@ def convert(saved_model_dir,
             tag_set=None,
             signature_key=signature_constants.DEFAULT_SERVING_SIGNATURE_DEF_KEY,
             batch_size=1):
-  """Convert a savedmodel to tflite flatbuffer.
+  """Convert a saved_model to tflite flatbuffer.
 
   Args:
     saved_model_dir: Saved model directory to convert.
diff --git a/tensorflow/contrib/lite/python/convert_savedmodel_test.py b/tensorflow/contrib/lite/python/convert_saved_model_test.py
similarity index 96%
rename from tensorflow/contrib/lite/python/convert_savedmodel_test.py
rename to tensorflow/contrib/lite/python/convert_saved_model_test.py
index 70cff9ef7f..d87fbeb91c 100644
--- a/tensorflow/contrib/lite/python/convert_savedmodel_test.py
+++ b/tensorflow/contrib/lite/python/convert_saved_model_test.py
@@ -24,7 +24,7 @@ from __future__ import division
 from __future__ import print_function
 
 import os
-from tensorflow.contrib.lite.python import convert_savedmodel
+from tensorflow.contrib.lite.python import convert_saved_model
 from tensorflow.python import estimator
 from tensorflow.python import keras
 from tensorflow.python import layers
@@ -60,13 +60,13 @@ class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
     # Create a simple savedmodel
     saved_model_dir = self._createSimpleSavedModel(shape=[1, 16, 16, 3])
     # Convert to tflite
-    result = convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+    result = convert_saved_model.convert(saved_model_dir=saved_model_dir)
     self.assertTrue(result)
 
   def testSimpleSavedModelWithNoneBatchSizeInShape(self):
     """Test a simple savedmodel, with None in input tensor's shape."""
     saved_model_dir = self._createSimpleSavedModel(shape=[None, 16, 16, 3])
-    result = convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+    result = convert_saved_model.convert(saved_model_dir=saved_model_dir)
     self.assertTrue(result)
 
   def testSimpleSavedModelWithMoreNoneInShape(self):
@@ -74,7 +74,7 @@ class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
     saved_model_dir = self._createSimpleSavedModel(shape=[None, 16, None, 3])
     # Convert to tflite: this should raise ValueError, as 3rd dim is None.
     with self.assertRaises(ValueError):
-      convert_savedmodel.convert(saved_model_dir=saved_model_dir)
+      convert_saved_model.convert(saved_model_dir=saved_model_dir)
 
   def testSimpleSavedModelWithWrongSignatureKey(self):
     """Test a simple savedmodel, fail as given signature is invalid."""
@@ -82,7 +82,7 @@ class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
     # Convert to tflite: this should raise ValueError, as
     # signature_key does not exit in the saved_model.
     with self.assertRaises(ValueError):
-      convert_savedmodel.convert(
+      convert_saved_model.convert(
           saved_model_dir=saved_model_dir, signature_key="wrong-key")
 
   def testSimpleSavedModelWithWrongOutputArray(self):
@@ -92,7 +92,7 @@ class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
     # Convert to tflite: this should raise ValueError, as
     # output_arrays is not valid for the saved_model.
     with self.assertRaises(ValueError):
-      convert_savedmodel.convert(
+      convert_saved_model.convert(
           saved_model_dir=saved_model_dir, output_arrays="wrong-output")
 
   def testMultipleMetaGraphDef(self):
@@ -124,7 +124,7 @@ class ConvertSavedModelTestBasicGraph(test_util.TensorFlowTestCase):
       builder.save(True)
 
     # Convert to tflite
-    convert_savedmodel.convert(
+    convert_saved_model.convert(
         saved_model_dir=saved_model_dir,
         tag_set=set([saved_model.tag_constants.SERVING, "additional_test_tag"]))
 
@@ -264,7 +264,7 @@ class ConvertSavedModelTestTrainGraph(test_util.TensorFlowTestCase):
                                  saved_model_final_dir + ".lite")
     # TODO(zhixianyan): no need to limit output_arrays to `Softmax'
     # once b/74205001 fixed and argmax implemented in tflite.
-    result = convert_savedmodel.convert(
+    result = convert_saved_model.convert(
         saved_model_dir=saved_model_final_dir,
         output_arrays="Softmax",
         output_tflite=output_tflite)
-- 
GitLab


From 73f40467bde137e2e2b31297b73944cc2830bdb7 Mon Sep 17 00:00:00 2001
From: Ou Changkun <hi@changkun.us>
Date: Tue, 27 Mar 2018 00:57:52 +0200
Subject: [PATCH 516/960] Fix missing interpretation of document (#17990)

* Fix missing interpretation of document

* Rephrase the sentence of missing interpretation
---
 tensorflow/docs_src/mobile/optimizing.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md
index ca9cb043e9..778e4d3a62 100644
--- a/tensorflow/docs_src/mobile/optimizing.md
+++ b/tensorflow/docs_src/mobile/optimizing.md
@@ -233,6 +233,8 @@ order by how long they took. From left to right, the columns are:
 - The cumulative total time of this and the previous ops in the table. This is
   handy for understanding what the distribution of work is across the layers, to
   see if just a few of the nodes are taking up most of the time.
+  
+- The amount of memory consumed by outputs of this type of op.
 
 - Name of the node.
 
-- 
GitLab


From e5dcaf921cf9feefd42b2ab176590c696b3b0285 Mon Sep 17 00:00:00 2001
From: Jerry Liu <twairball@yahoo.com>
Date: Tue, 27 Mar 2018 07:21:54 +0800
Subject: [PATCH 517/960] Fix #15900 (#16154)

- Added `save_checkpoint_steps` attribute to `MonitoredTrainingSession`.
If both `save_checkpoint_steps` and `save_checkpoint_secs` are both `None` then default saver is disabled. Default is `save_checkpoint_secs=600`
- Added `test_save_checkpoint_steps`
- Updated golden file
---
 .../python/training/monitored_session.py      | 33 +++++++++++++----
 .../python/training/monitored_session_test.py | 36 +++++++++++++++++++
 .../tools/api/golden/tensorflow.train.pbtxt   |  2 +-
 3 files changed, 64 insertions(+), 7 deletions(-)

diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 6c5c9e01a7..2d4f09a60a 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -281,13 +281,14 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
                              scaffold=None,
                              hooks=None,
                              chief_only_hooks=None,
-                             save_checkpoint_secs=600,
+                             save_checkpoint_secs=USE_DEFAULT,
                              save_summaries_steps=USE_DEFAULT,
                              save_summaries_secs=USE_DEFAULT,
                              config=None,
                              stop_grace_period_secs=120,
                              log_step_count_steps=100,
-                             max_wait_secs=7200):
+                             max_wait_secs=7200,
+                             save_checkpoint_steps=USE_DEFAULT):
   """Creates a `MonitoredSession` for training.
 
   For a chief, this utility sets proper session initializer/restorer. It also
@@ -310,8 +311,10 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
     chief_only_hooks: list of `SessionRunHook` objects. Activate these hooks if
       `is_chief==True`, ignore otherwise.
     save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved
-      using a default checkpoint saver. If `save_checkpoint_secs` is set to
-      `None`, then the default checkpoint saver isn't used.
+      using a default checkpoint saver. If both `save_checkpoint_steps` and
+      `save_checkpoint_secs` are set to `None`, then the default checkpoint
+      saver isn't used. If both are provided, then only `save_checkpoint_secs`
+      is used. Default 600.
     save_summaries_steps: The frequency, in number of global steps, that the
       summaries are written to disk using a default summary saver. If both
       `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
@@ -330,6 +333,11 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       become available. This should be kept relatively short to help detect
       incorrect code, but sometimes may need to be increased if the chief takes
       a while to start up.
+    save_checkpoint_steps: The frequency, in number of global steps, that a
+      checkpoint is saved using a default checkpoint saver. If both
+      `save_checkpoint_steps` and `save_checkpoint_secs` are set to `None`, then
+      the default checkpoint saver isn't used. If both are provided, then only
+      `save_checkpoint_secs` is used. Default not enabled.
 
   Returns:
     A `MonitoredSession` object.
@@ -342,6 +350,15 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
   elif save_summaries_steps == USE_DEFAULT:
     save_summaries_steps = None
 
+  if save_checkpoint_steps == USE_DEFAULT and \
+    save_checkpoint_secs == USE_DEFAULT:
+    save_checkpoint_steps = None
+    save_checkpoint_secs = 600
+  elif save_checkpoint_secs == USE_DEFAULT:
+    save_checkpoint_secs = None
+  elif save_checkpoint_steps == USE_DEFAULT:
+    save_checkpoint_steps = None
+
   scaffold = scaffold or Scaffold()
   if not is_chief:
     session_creator = WorkerSessionCreator(
@@ -374,9 +391,13 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
           save_steps=save_summaries_steps,
           save_secs=save_summaries_secs,
           output_dir=checkpoint_dir))
-    if save_checkpoint_secs and save_checkpoint_secs > 0:
+    if (save_checkpoint_secs and save_checkpoint_secs > 0) or (
+        save_checkpoint_steps and save_checkpoint_steps > 0):
       all_hooks.append(basic_session_run_hooks.CheckpointSaverHook(
-          checkpoint_dir, save_secs=save_checkpoint_secs, scaffold=scaffold))
+          checkpoint_dir,
+          save_steps=save_checkpoint_steps,
+          save_secs=save_checkpoint_secs,
+          scaffold=scaffold))
 
   if hooks:
     all_hooks.extend(hooks)
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 159b2d5c16..3806056f01 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -282,6 +282,42 @@ class MonitoredTrainingSessionTest(test.TestCase):
           is_chief=True, checkpoint_dir=logdir) as session:
         self.assertEqual(2, session.run(gstep))
 
+  def test_save_checkpoint_steps(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_steps')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_steps=100,
+          log_step_count_steps=10) as session:
+        for _ in range(100):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(100, session.run(gstep))
+
+  def test_save_checkpoint_secs(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_secs')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_secs=0.1,
+          log_step_count_steps=10) as session:
+        session.run(new_gstep)
+        time.sleep(0.2)
+        for _ in range(10):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(11, session.run(gstep))
+
   def test_summaries_steps(self):
     logdir = _test_dir(self.get_temp_dir(), 'test_summaries_steps')
     with ops.Graph().as_default():
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index c75ee474aa..bec72e1e60 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -238,7 +238,7 @@ tf_module {
   }
   member_method {
     name: "MonitoredTrainingSession"
-    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'600\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\'], "
+    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\', \'save_checkpoint_steps\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'<object object instance>\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\', \'<object object instance>\'], "
   }
   member_method {
     name: "NewCheckpointReader"
-- 
GitLab


From 307cfe7ab7e2c475b2741fc2a2f7663b46223e6d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 16:19:50 -0700
Subject: [PATCH 518/960] Save the last loss reduction method (for future use).

PiperOrigin-RevId: 190543066
---
 tensorflow/python/framework/ops.py          | 3 +++
 tensorflow/python/ops/losses/losses_impl.py | 5 +++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index e579289a8d..25a951a2de 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2788,6 +2788,9 @@ class Graph(object):
     # being called inside function definitions behave as if they were seeing the
     # actual outside graph).
     self._graph_key = "grap-key-%d/" % (uid(),)
+    # A string with the last reduction method passed to
+    # losses.compute_weighted_loss(), or None.
+    self._last_loss_reduction = None
     self._container = ""
     self._registered_ops = op_def_registry.get_registered_ops()
 
diff --git a/tensorflow/python/ops/losses/losses_impl.py b/tensorflow/python/ops/losses/losses_impl.py
index 0840760810..34ca1adc3e 100644
--- a/tensorflow/python/ops/losses/losses_impl.py
+++ b/tensorflow/python/ops/losses/losses_impl.py
@@ -194,6 +194,11 @@ def compute_weighted_loss(
   """
   Reduction.validate(reduction)
   with ops.name_scope(scope, "weighted_loss", (losses, weights)):
+    # Save the `reduction` argument for loss normalization when distributing
+    # to multiple towers.
+    # TODO(josh11b): Associate it with the returned op for more precision.
+    ops.get_default_graph()._last_loss_reduction = reduction  # pylint: disable=protected-access
+
     with ops.control_dependencies((
         weights_broadcast_ops.assert_broadcastable(weights, losses),)):
       losses = ops.convert_to_tensor(losses)
-- 
GitLab


From eda7aa3f7e763734f5f3550bed8b044a384b2ce8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 17:02:55 -0700
Subject: [PATCH 519/960] Add missing parameter to OP_REQUIRES call.

PiperOrigin-RevId: 190548854
---
 tensorflow/core/kernels/mkl_reshape_op.cc | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_reshape_op.cc b/tensorflow/core/kernels/mkl_reshape_op.cc
index e12f6f437a..2cfde1f6fd 100644
--- a/tensorflow/core/kernels/mkl_reshape_op.cc
+++ b/tensorflow/core/kernels/mkl_reshape_op.cc
@@ -266,8 +266,9 @@ class MklReshapeOp : public OpKernel {
                                                    &net)) {
               stream(stream::kind::eager).submit(net).wait();
             } else {
-              OP_REQUIRES(context,
-                          output_tensor->CopyFrom(input_tensor, shape_to));
+              OP_REQUIRES(
+                  context, output_tensor->CopyFrom(input_tensor, shape_to),
+                  errors::InvalidArgument("invalid input tensor shape"));
             }
             return;
           } else {
-- 
GitLab


From 931f6d553172ddfc9ec4a7a94ea2c6233bf33cb0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 17:39:51 -0700
Subject: [PATCH 520/960] [XLA] Redesign: handle metadata and sharding.

- Add a xla.OpSharding field to the HloInstructionProto.
- Metatdata handling is tested.

PiperOrigin-RevId: 190553731
---
 .../xla/client/xla_client/xla_builder.cc      |  7 +++-
 .../xla/client/xla_client/xla_builder.h       | 32 +++++++++++++++++++
 tensorflow/compiler/xla/service/hlo.proto     |  2 ++
 tensorflow/compiler/xla/tests/BUILD           |  3 +-
 .../compiler/xla/tests/hlo_metadata_test.cc   |  9 +++---
 5 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index bf91efcfd6..1b90b45bfb 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -896,8 +896,13 @@ StatusOr<XlaOp> XlaBuilder::AddInstruction(
         << "Do not add XlaOp from builder " << operand.builder_->name()
         << " to builder " << this->name();
     instr.add_operand_ids(operand.handle());
-    // TODO(b/74197823): Set metadata and sharding.
   }
+
+  *instr.mutable_metadata() = metadata_;
+  if (sharding_) {
+    *instr.mutable_sharding() = *sharding_;
+  }
+
   instructions_.push_back(instr);
 
   XlaOp op(handle, this);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index 22cf094512..cc33356cc1 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -85,6 +85,29 @@ class XlaBuilder {
   // Returns the computation name.
   const string& name() const { return name_; }
 
+  // Sets OpMetadata that will be added to all instructions until cleared.
+  //
+  // OpMetadata is often applied to a series of XLA HLO instructions. As a
+  // result, OpMetadata is set on the Computation Builder. All subsequent
+  // instructions generated via this Computation Builder will have the same
+  // OpMetadata attached until a call to ClearOpMetadata.
+  void SetOpMetadata(const OpMetadata& metadata) { metadata_ = metadata; }
+
+  // Clears the HloMetadata state.
+  void ClearOpMetadata() { metadata_.Clear(); }
+
+  // Sets an OpSharding that will be attached to all instructions until cleared.
+  void SetSharding(const OpSharding& sharding) { sharding_ = sharding; }
+
+  // Clears the sharding. Ops will be sharded according to the default placement
+  // policy.
+  void ClearSharding() { sharding_ = tensorflow::gtl::nullopt; }
+
+  // Returns the OpSharding that will be attached to all instructions.
+  const tensorflow::gtl::optional<OpSharding>& sharding() const {
+    return sharding_;
+  }
+
   // Sets the builder to a mode where it will die immediately when an error is
   // encountered, rather than producing it in a deferred fashion when Build() is
   // called (which is the default).
@@ -776,6 +799,15 @@ class XlaBuilder {
   // The unique parameter numbers.
   tensorflow::gtl::FlatSet<int64> parameter_numbers_;
 
+  // The metadata to attach to each op. This is structured as a "modal"-like
+  // operation, in order to simplify client code (and not sprinkle this metadata
+  // throughout the TensorFlow op kernel implementations).
+  OpMetadata metadata_;
+
+  // Sharding for this operator. This is structured as a "model"-like operation,
+  // in order to simplify client code, similar to metadata_.
+  tensorflow::gtl::optional<OpSharding> sharding_;
+
   // Mode bit that indicates whether to die when a first error is encountered.
   bool die_immediately_on_error_ = false;
 };
diff --git a/tensorflow/compiler/xla/service/hlo.proto b/tensorflow/compiler/xla/service/hlo.proto
index 406feadfd4..0b446c6547 100644
--- a/tensorflow/compiler/xla/service/hlo.proto
+++ b/tensorflow/compiler/xla/service/hlo.proto
@@ -141,6 +141,8 @@ message HloInstructionProto {
   repeated int64 operand_ids = 36;
   repeated int64 control_predecessor_ids = 37;
   repeated int64 called_computation_ids = 38;
+
+  xla.OpSharding sharding = 40;
 }
 
 // Serialization of HloComputation.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 3705d6c271..5ab25f2264 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1810,9 +1810,8 @@ tf_cc_test(
     deps = [
         ":local_client_test_base",
         "//tensorflow/compiler/xla:test_helpers",
-        "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
-        "//tensorflow/compiler/xla/service:computation_tracker",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/service:cpu_plugin",
         "//tensorflow/compiler/xla/service:local_service",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/compiler/xla/tests/hlo_metadata_test.cc b/tensorflow/compiler/xla/tests/hlo_metadata_test.cc
index eded2077fc..cf971dd61b 100644
--- a/tensorflow/compiler/xla/tests/hlo_metadata_test.cc
+++ b/tensorflow/compiler/xla/tests/hlo_metadata_test.cc
@@ -13,9 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
-#include "tensorflow/compiler/xla/service/computation_tracker.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/service/local_service.h"
 #include "tensorflow/compiler/xla/test_helpers.h"
 #include "tensorflow/compiler/xla/tests/local_client_test_base.h"
@@ -30,7 +29,7 @@ class HloMetadataTest : public LocalClientTestBase {
     metadata_.set_op_name("my_sum_op");
   }
 
-  void BuildAddComputation(ComputationBuilder* builder) {
+  void BuildAddComputation(XlaBuilder* builder) {
     auto x = builder->Parameter(0, ShapeUtil::MakeShape(F32, {}), "x");
     auto y = builder->Parameter(1, ShapeUtil::MakeShape(F32, {}), "y");
     builder->Add(x, y);
@@ -40,7 +39,7 @@ class HloMetadataTest : public LocalClientTestBase {
 };
 
 TEST_F(HloMetadataTest, MetadataPropagation) {
-  ComputationBuilder builder(local_client_, "add");
+  XlaBuilder builder("add");
   builder.SetOpMetadata(metadata_);
   BuildAddComputation(&builder);
   builder.ClearOpMetadata();
@@ -61,7 +60,7 @@ TEST_F(HloMetadataTest, MetadataPropagation) {
 }
 
 TEST_F(HloMetadataTest, MetadataClearing) {
-  ComputationBuilder builder(local_client_, "add");
+  XlaBuilder builder("add");
   builder.SetOpMetadata(metadata_);
   // Some other pretend computation here.
   builder.ClearOpMetadata();
-- 
GitLab


From 0be974c423f6e5c363db2d95ed335dde4cb4e69b Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Mon, 26 Mar 2018 18:50:27 -0700
Subject: [PATCH 521/960] Finish deprecation of
 tf.contrib.bayesflow.{HMC,MetropolisHastings}. New home:
 https://github.com/tensorflow/probability/tree/master/tensorflow_probability/python/mcmc

PiperOrigin-RevId: 190560180
---
 tensorflow/contrib/bayesflow/BUILD            |  41 -
 tensorflow/contrib/bayesflow/README.md        |  17 +
 tensorflow/contrib/bayesflow/__init__.py      |   8 -
 .../bayesflow/python/kernel_tests/hmc_test.py | 737 --------------
 .../kernel_tests/metropolis_hastings_test.py  | 340 -------
 .../contrib/bayesflow/python/ops/hmc.py       |  30 -
 .../contrib/bayesflow/python/ops/hmc_impl.py  | 961 ------------------
 .../python/ops/metropolis_hastings.py         |  34 -
 .../python/ops/metropolis_hastings_impl.py    | 527 ----------
 9 files changed, 17 insertions(+), 2678 deletions(-)
 create mode 100644 tensorflow/contrib/bayesflow/README.md
 delete mode 100644 tensorflow/contrib/bayesflow/python/kernel_tests/hmc_test.py
 delete mode 100644 tensorflow/contrib/bayesflow/python/kernel_tests/metropolis_hastings_test.py
 delete mode 100644 tensorflow/contrib/bayesflow/python/ops/hmc.py
 delete mode 100644 tensorflow/contrib/bayesflow/python/ops/hmc_impl.py
 delete mode 100644 tensorflow/contrib/bayesflow/python/ops/metropolis_hastings.py
 delete mode 100644 tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py

diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD
index c6feec68e0..a55029b314 100644
--- a/tensorflow/contrib/bayesflow/BUILD
+++ b/tensorflow/contrib/bayesflow/BUILD
@@ -37,25 +37,6 @@ py_library(
     ],
 )
 
-cuda_py_test(
-    name = "metropolis_hastings_test",
-    size = "large",
-    srcs = ["python/kernel_tests/metropolis_hastings_test.py"],
-    additional_deps = [
-        ":bayesflow_py",
-        "//third_party/py/numpy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_ops",
-        "//tensorflow/python:variable_scope",
-        "//tensorflow/python:variables",
-    ],
-)
-
 cuda_py_test(
     name = "monte_carlo_test",
     size = "small",
@@ -77,28 +58,6 @@ cuda_py_test(
     ],
 )
 
-cuda_py_test(
-    name = "hmc_test",
-    size = "large",
-    srcs = ["python/kernel_tests/hmc_test.py"],
-    additional_deps = [
-        ":bayesflow_py",
-        "//third_party/py/numpy",
-        "//tensorflow/contrib/distributions:distributions_py",
-        "//tensorflow/contrib/layers:layers_py",
-        "//tensorflow/python/ops/distributions",
-        "//tensorflow/python:client_testlib",
-        "//tensorflow/python:framework",
-        "//tensorflow/python:framework_for_generated_wrappers",
-        "//tensorflow/python:framework_test_lib",
-        "//tensorflow/python:gradients",
-        "//tensorflow/python:math_ops",
-        "//tensorflow/python:platform_test",
-        "//tensorflow/python:random_seed",
-    ],
-    tags = ["nomsan"],
-)
-
 filegroup(
     name = "all_files",
     srcs = glob(
diff --git a/tensorflow/contrib/bayesflow/README.md b/tensorflow/contrib/bayesflow/README.md
new file mode 100644
index 0000000000..10323dc6d5
--- /dev/null
+++ b/tensorflow/contrib/bayesflow/README.md
@@ -0,0 +1,17 @@
+# Notice
+
+`tf.contrib.bayesflow` has moved!
+
+See new code at [github.com/tensorflow/probability](
+https://github.com/tensorflow/probability).
+
+Switch imports with:
+
+```python
+# old
+import tensorflow as tf
+tfp = tf.contrib.bayesflow
+
+# new
+import tensorflow_probability as tfp
+```
diff --git a/tensorflow/contrib/bayesflow/__init__.py b/tensorflow/contrib/bayesflow/__init__.py
index f868203826..41a8c920fc 100644
--- a/tensorflow/contrib/bayesflow/__init__.py
+++ b/tensorflow/contrib/bayesflow/__init__.py
@@ -21,8 +21,6 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long
-from tensorflow.contrib.bayesflow.python.ops import hmc
-from tensorflow.contrib.bayesflow.python.ops import metropolis_hastings
 from tensorflow.contrib.bayesflow.python.ops import monte_carlo
 # pylint: enable=unused-import,line-too-long
 
@@ -30,13 +28,7 @@ from tensorflow.python.util.all_util import remove_undocumented
 
 
 _allowed_symbols = [
-    'entropy',
-    'hmc',
-    'metropolis_hastings',
     'monte_carlo',
-    'special_math',
-    'stochastic_variables',
-    'variational_inference',
 ]
 
 remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/bayesflow/python/kernel_tests/hmc_test.py b/tensorflow/contrib/bayesflow/python/kernel_tests/hmc_test.py
deleted file mode 100644
index dabadfc7b6..0000000000
--- a/tensorflow/contrib/bayesflow/python/kernel_tests/hmc_test.py
+++ /dev/null
@@ -1,737 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for Hamiltonian Monte Carlo."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-import numpy as np
-from scipy import stats
-
-from tensorflow.contrib.bayesflow.python.ops import hmc
-from tensorflow.contrib.bayesflow.python.ops.hmc_impl import _compute_energy_change
-from tensorflow.contrib.bayesflow.python.ops.hmc_impl import _leapfrog_integrator
-
-from tensorflow.contrib.distributions.python.ops import independent as independent_lib
-from tensorflow.python.framework import ops
-from tensorflow.python.framework import random_seed
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import gen_linalg_ops
-from tensorflow.python.ops import gradients_impl as gradients_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.ops.distributions import gamma as gamma_lib
-from tensorflow.python.ops.distributions import normal as normal_lib
-from tensorflow.python.platform import test
-from tensorflow.python.platform import tf_logging as logging_ops
-
-
-def _reduce_variance(x, axis=None, keepdims=False):
-  sample_mean = math_ops.reduce_mean(x, axis, keepdims=True)
-  return math_ops.reduce_mean(
-      math_ops.squared_difference(x, sample_mean), axis, keepdims)
-
-
-class HMCTest(test.TestCase):
-
-  def setUp(self):
-    self._shape_param = 5.
-    self._rate_param = 10.
-
-    random_seed.set_random_seed(10003)
-    np.random.seed(10003)
-
-  def assertAllFinite(self, x):
-    self.assertAllEqual(np.ones_like(x).astype(bool), np.isfinite(x))
-
-  def _log_gamma_log_prob(self, x, event_dims=()):
-    """Computes log-pdf of a log-gamma random variable.
-
-    Args:
-      x: Value of the random variable.
-      event_dims: Dimensions not to treat as independent.
-
-    Returns:
-      log_prob: The log-pdf up to a normalizing constant.
-    """
-    return math_ops.reduce_sum(self._shape_param * x -
-                               self._rate_param * math_ops.exp(x),
-                               event_dims)
-
-  def _integrator_conserves_energy(self, x, independent_chain_ndims, sess,
-                                   feed_dict=None):
-    step_size = array_ops.placeholder(np.float32, [], name="step_size")
-    hmc_lf_steps = array_ops.placeholder(np.int32, [], name="hmc_lf_steps")
-
-    if feed_dict is None:
-      feed_dict = {}
-    feed_dict[hmc_lf_steps] = 1000
-
-    event_dims = math_ops.range(independent_chain_ndims,
-                                array_ops.rank(x))
-
-    m = random_ops.random_normal(array_ops.shape(x))
-    log_prob_0 = self._log_gamma_log_prob(x, event_dims)
-    grad_0 = gradients_ops.gradients(log_prob_0, x)
-    old_energy = -log_prob_0 + 0.5 * math_ops.reduce_sum(m**2., event_dims)
-
-    new_m, _, log_prob_1, _ = _leapfrog_integrator(
-        current_momentums=[m],
-        target_log_prob_fn=lambda x: self._log_gamma_log_prob(x, event_dims),
-        current_state_parts=[x],
-        step_sizes=[step_size],
-        num_leapfrog_steps=hmc_lf_steps,
-        current_target_log_prob=log_prob_0,
-        current_grads_target_log_prob=grad_0)
-    new_m = new_m[0]
-
-    new_energy = -log_prob_1 + 0.5 * math_ops.reduce_sum(new_m * new_m,
-                                                         event_dims)
-
-    x_shape = sess.run(x, feed_dict).shape
-    event_size = np.prod(x_shape[independent_chain_ndims:])
-    feed_dict[step_size] = 0.1 / event_size
-    old_energy_, new_energy_ = sess.run([old_energy, new_energy],
-                                        feed_dict)
-    logging_ops.vlog(1, "average energy relative change: {}".format(
-        (1. - new_energy_ / old_energy_).mean()))
-    self.assertAllClose(old_energy_, new_energy_, atol=0., rtol=0.02)
-
-  def _integrator_conserves_energy_wrapper(self, independent_chain_ndims):
-    """Tests the long-term energy conservation of the leapfrog integrator.
-
-    The leapfrog integrator is symplectic, so for sufficiently small step
-    sizes it should be possible to run it more or less indefinitely without
-    the energy of the system blowing up or collapsing.
-
-    Args:
-      independent_chain_ndims: Python `int` scalar representing the number of
-        dims associated with independent chains.
-    """
-    with self.test_session(graph=ops.Graph()) as sess:
-      x_ph = array_ops.placeholder(np.float32, name="x_ph")
-      feed_dict = {x_ph: np.random.rand(50, 10, 2)}
-      self._integrator_conserves_energy(x_ph, independent_chain_ndims,
-                                        sess, feed_dict)
-
-  def testIntegratorEnergyConservationNullShape(self):
-    self._integrator_conserves_energy_wrapper(0)
-
-  def testIntegratorEnergyConservation1(self):
-    self._integrator_conserves_energy_wrapper(1)
-
-  def testIntegratorEnergyConservation2(self):
-    self._integrator_conserves_energy_wrapper(2)
-
-  def testIntegratorEnergyConservation3(self):
-    self._integrator_conserves_energy_wrapper(3)
-
-  def testSampleChainSeedReproducibleWorksCorrectly(self):
-    with self.test_session(graph=ops.Graph()) as sess:
-      num_results = 10
-      independent_chain_ndims = 1
-
-      def log_gamma_log_prob(x):
-        event_dims = math_ops.range(independent_chain_ndims,
-                                    array_ops.rank(x))
-        return self._log_gamma_log_prob(x, event_dims)
-
-      kwargs = dict(
-          target_log_prob_fn=log_gamma_log_prob,
-          current_state=np.random.rand(4, 3, 2),
-          step_size=0.1,
-          num_leapfrog_steps=2,
-          num_burnin_steps=150,
-          seed=52,
-      )
-
-      samples0, kernel_results0 = hmc.sample_chain(
-          **dict(list(kwargs.items()) + list(dict(
-              num_results=2 * num_results,
-              num_steps_between_results=0).items())))
-
-      samples1, kernel_results1 = hmc.sample_chain(
-          **dict(list(kwargs.items()) + list(dict(
-              num_results=num_results,
-              num_steps_between_results=1).items())))
-
-      [
-          samples0_,
-          samples1_,
-          target_log_prob0_,
-          target_log_prob1_,
-      ] = sess.run([
-          samples0,
-          samples1,
-          kernel_results0.current_target_log_prob,
-          kernel_results1.current_target_log_prob,
-      ])
-      self.assertAllClose(samples0_[::2], samples1_,
-                          atol=1e-5, rtol=1e-5)
-      self.assertAllClose(target_log_prob0_[::2], target_log_prob1_,
-                          atol=1e-5, rtol=1e-5)
-
-  def _chain_gets_correct_expectations(self, x, independent_chain_ndims,
-                                       sess, feed_dict=None):
-    counter = collections.Counter()
-    def log_gamma_log_prob(x):
-      counter["target_calls"] += 1
-      event_dims = math_ops.range(independent_chain_ndims,
-                                  array_ops.rank(x))
-      return self._log_gamma_log_prob(x, event_dims)
-
-    num_results = array_ops.placeholder(
-        np.int32, [], name="num_results")
-    step_size = array_ops.placeholder(
-        np.float32, [], name="step_size")
-    num_leapfrog_steps = array_ops.placeholder(
-        np.int32, [], name="num_leapfrog_steps")
-
-    if feed_dict is None:
-      feed_dict = {}
-    feed_dict.update({num_results: 150,
-                      step_size: 0.05,
-                      num_leapfrog_steps: 2})
-
-    samples, kernel_results = hmc.sample_chain(
-        num_results=num_results,
-        target_log_prob_fn=log_gamma_log_prob,
-        current_state=x,
-        step_size=step_size,
-        num_leapfrog_steps=num_leapfrog_steps,
-        num_burnin_steps=150,
-        seed=42)
-
-    self.assertAllEqual(dict(target_calls=2), counter)
-
-    expected_x = (math_ops.digamma(self._shape_param)
-                  - np.log(self._rate_param))
-
-    expected_exp_x = self._shape_param / self._rate_param
-
-    log_accept_ratio_, samples_, expected_x_ = sess.run(
-        [kernel_results.log_accept_ratio, samples, expected_x],
-        feed_dict)
-
-    actual_x = samples_.mean()
-    actual_exp_x = np.exp(samples_).mean()
-    acceptance_probs = np.exp(np.minimum(log_accept_ratio_, 0.))
-
-    logging_ops.vlog(1, "True      E[x, exp(x)]: {}\t{}".format(
-        expected_x_, expected_exp_x))
-    logging_ops.vlog(1, "Estimated E[x, exp(x)]: {}\t{}".format(
-        actual_x, actual_exp_x))
-    self.assertNear(actual_x, expected_x_, 2e-2)
-    self.assertNear(actual_exp_x, expected_exp_x, 2e-2)
-    self.assertAllEqual(np.ones_like(acceptance_probs, np.bool),
-                        acceptance_probs > 0.5)
-    self.assertAllEqual(np.ones_like(acceptance_probs, np.bool),
-                        acceptance_probs <= 1.)
-
-  def _chain_gets_correct_expectations_wrapper(self, independent_chain_ndims):
-    with self.test_session(graph=ops.Graph()) as sess:
-      x_ph = array_ops.placeholder(np.float32, name="x_ph")
-      feed_dict = {x_ph: np.random.rand(50, 10, 2)}
-      self._chain_gets_correct_expectations(x_ph, independent_chain_ndims,
-                                            sess, feed_dict)
-
-  def testHMCChainExpectationsNullShape(self):
-    self._chain_gets_correct_expectations_wrapper(0)
-
-  def testHMCChainExpectations1(self):
-    self._chain_gets_correct_expectations_wrapper(1)
-
-  def testHMCChainExpectations2(self):
-    self._chain_gets_correct_expectations_wrapper(2)
-
-  def testKernelResultsUsingTruncatedDistribution(self):
-    def log_prob(x):
-      return array_ops.where(
-          x >= 0.,
-          -x - x**2,  # Non-constant gradient.
-          array_ops.fill(x.shape, math_ops.cast(-np.inf, x.dtype)))
-    # This log_prob has the property that it is likely to attract
-    # the flow toward, and below, zero...but for x <=0,
-    # log_prob(x) = -inf, which should result in rejection, as well
-    # as a non-finite log_prob.  Thus, this distribution gives us an opportunity
-    # to test out the kernel results ability to correctly capture rejections due
-    # to finite AND non-finite reasons.
-    # Why use a non-constant gradient?  This ensures the leapfrog integrator
-    # will not be exact.
-
-    num_results = 1000
-    # Large step size, will give rejections due to integration error in addition
-    # to rejection due to going into a region of log_prob = -inf.
-    step_size = 0.1
-    num_leapfrog_steps = 5
-    num_chains = 2
-
-    with self.test_session(graph=ops.Graph()) as sess:
-
-      # Start multiple independent chains.
-      initial_state = ops.convert_to_tensor([0.1] * num_chains)
-
-      states, kernel_results = hmc.sample_chain(
-          num_results=num_results,
-          target_log_prob_fn=log_prob,
-          current_state=initial_state,
-          step_size=step_size,
-          num_leapfrog_steps=num_leapfrog_steps,
-          seed=42)
-
-      states_, kernel_results_ = sess.run([states, kernel_results])
-      pstates_ = kernel_results_.proposed_state
-
-      neg_inf_mask = np.isneginf(kernel_results_.proposed_target_log_prob)
-
-      # First:  Test that the mathematical properties of the above log prob
-      # function in conjunction with HMC show up as expected in kernel_results_.
-
-      # We better have log_prob = -inf some of the time.
-      self.assertLess(0, neg_inf_mask.sum())
-      # We better have some rejections due to something other than -inf.
-      self.assertLess(neg_inf_mask.sum(), (~kernel_results_.is_accepted).sum())
-      # We better have accepted a decent amount, even near end of the chain.
-      self.assertLess(
-          0.1, kernel_results_.is_accepted[int(0.9 * num_results):].mean())
-      # We better not have any NaNs in states or log_prob.
-      # We may have some NaN in grads, which involve multiplication/addition due
-      # to gradient rules.  This is the known "NaN grad issue with tf.where."
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isnan(kernel_results_.proposed_target_log_prob))
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isnan(states_))
-      # We better not have any +inf in states, grads, or log_prob.
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isposinf(kernel_results_.proposed_target_log_prob))
-      self.assertAllEqual(
-          np.zeros_like(states_),
-          np.isposinf(kernel_results_.proposed_grads_target_log_prob[0]))
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isposinf(states_))
-
-      # Second:  Test that kernel_results is congruent with itself and
-      # acceptance/rejection of states.
-
-      # Proposed state is negative iff proposed target log prob is -inf.
-      np.testing.assert_array_less(pstates_[neg_inf_mask], 0.)
-      np.testing.assert_array_less(0., pstates_[~neg_inf_mask])
-
-      # Acceptance probs are zero whenever proposed state is negative.
-      acceptance_probs = np.exp(np.minimum(
-          kernel_results_.log_accept_ratio, 0.))
-      self.assertAllEqual(
-          np.zeros_like(pstates_[neg_inf_mask]),
-          acceptance_probs[neg_inf_mask])
-
-      # The move is accepted ==> state = proposed state.
-      self.assertAllEqual(
-          states_[kernel_results_.is_accepted],
-          pstates_[kernel_results_.is_accepted],
-      )
-      # The move was rejected <==> state[t] == state[t - 1].
-      for t in range(1, num_results):
-        for i in range(num_chains):
-          if kernel_results_.is_accepted[t, i]:
-            self.assertNotEqual(states_[t, i], states_[t - 1, i])
-          else:
-            self.assertEqual(states_[t, i], states_[t - 1, i])
-
-  def _kernel_leaves_target_invariant(self, initial_draws,
-                                      independent_chain_ndims,
-                                      sess, feed_dict=None):
-    def log_gamma_log_prob(x):
-      event_dims = math_ops.range(independent_chain_ndims, array_ops.rank(x))
-      return self._log_gamma_log_prob(x, event_dims)
-
-    def fake_log_prob(x):
-      """Cooled version of the target distribution."""
-      return 1.1 * log_gamma_log_prob(x)
-
-    step_size = array_ops.placeholder(np.float32, [], name="step_size")
-
-    if feed_dict is None:
-      feed_dict = {}
-
-    feed_dict[step_size] = 0.4
-
-    sample, kernel_results = hmc.kernel(
-        target_log_prob_fn=log_gamma_log_prob,
-        current_state=initial_draws,
-        step_size=step_size,
-        num_leapfrog_steps=5,
-        seed=43)
-
-    bad_sample, bad_kernel_results = hmc.kernel(
-        target_log_prob_fn=fake_log_prob,
-        current_state=initial_draws,
-        step_size=step_size,
-        num_leapfrog_steps=5,
-        seed=44)
-
-    [
-        log_accept_ratio_,
-        bad_log_accept_ratio_,
-        initial_draws_,
-        updated_draws_,
-        fake_draws_,
-    ] = sess.run([
-        kernel_results.log_accept_ratio,
-        bad_kernel_results.log_accept_ratio,
-        initial_draws,
-        sample,
-        bad_sample,
-    ], feed_dict)
-
-    # Confirm step size is small enough that we usually accept.
-    acceptance_probs = np.exp(np.minimum(log_accept_ratio_, 0.))
-    bad_acceptance_probs = np.exp(np.minimum(bad_log_accept_ratio_, 0.))
-    self.assertGreater(acceptance_probs.mean(), 0.5)
-    self.assertGreater(bad_acceptance_probs.mean(), 0.5)
-
-    # Confirm step size is large enough that we sometimes reject.
-    self.assertLess(acceptance_probs.mean(), 0.99)
-    self.assertLess(bad_acceptance_probs.mean(), 0.99)
-
-    _, ks_p_value_true = stats.ks_2samp(initial_draws_.flatten(),
-                                        updated_draws_.flatten())
-    _, ks_p_value_fake = stats.ks_2samp(initial_draws_.flatten(),
-                                        fake_draws_.flatten())
-
-    logging_ops.vlog(1, "acceptance rate for true target: {}".format(
-        acceptance_probs.mean()))
-    logging_ops.vlog(1, "acceptance rate for fake target: {}".format(
-        bad_acceptance_probs.mean()))
-    logging_ops.vlog(1, "K-S p-value for true target: {}".format(
-        ks_p_value_true))
-    logging_ops.vlog(1, "K-S p-value for fake target: {}".format(
-        ks_p_value_fake))
-    # Make sure that the MCMC update hasn't changed the empirical CDF much.
-    self.assertGreater(ks_p_value_true, 1e-3)
-    # Confirm that targeting the wrong distribution does
-    # significantly change the empirical CDF.
-    self.assertLess(ks_p_value_fake, 1e-6)
-
-  def _kernel_leaves_target_invariant_wrapper(self, independent_chain_ndims):
-    """Tests that the kernel leaves the target distribution invariant.
-
-    Draws some independent samples from the target distribution,
-    applies an iteration of the MCMC kernel, then runs a
-    Kolmogorov-Smirnov test to determine if the distribution of the
-    MCMC-updated samples has changed.
-
-    We also confirm that running the kernel with a different log-pdf
-    does change the target distribution. (And that we can detect that.)
-
-    Args:
-      independent_chain_ndims: Python `int` scalar representing the number of
-        dims associated with independent chains.
-    """
-    with self.test_session(graph=ops.Graph()) as sess:
-      initial_draws = np.log(np.random.gamma(self._shape_param,
-                                             size=[50000, 2, 2]))
-      initial_draws -= np.log(self._rate_param)
-      x_ph = array_ops.placeholder(np.float32, name="x_ph")
-
-      feed_dict = {x_ph: initial_draws}
-
-      self._kernel_leaves_target_invariant(x_ph, independent_chain_ndims,
-                                           sess, feed_dict)
-
-  def testKernelLeavesTargetInvariant1(self):
-    self._kernel_leaves_target_invariant_wrapper(1)
-
-  def testKernelLeavesTargetInvariant2(self):
-    self._kernel_leaves_target_invariant_wrapper(2)
-
-  def testKernelLeavesTargetInvariant3(self):
-    self._kernel_leaves_target_invariant_wrapper(3)
-
-  def testNanRejection(self):
-    """Tests that an update that yields NaN potentials gets rejected.
-
-    We run HMC with a target distribution that returns NaN
-    log-likelihoods if any element of x < 0, and unit-scale
-    exponential log-likelihoods otherwise. The exponential potential
-    pushes x towards 0, ensuring that any reasonably large update will
-    push us over the edge into NaN territory.
-    """
-    def _unbounded_exponential_log_prob(x):
-      """An exponential distribution with log-likelihood NaN for x < 0."""
-      per_element_potentials = array_ops.where(
-          x < 0.,
-          array_ops.fill(array_ops.shape(x), x.dtype.as_numpy_dtype(np.nan)),
-          -x)
-      return math_ops.reduce_sum(per_element_potentials)
-
-    with self.test_session(graph=ops.Graph()) as sess:
-      initial_x = math_ops.linspace(0.01, 5, 10)
-      updated_x, kernel_results = hmc.kernel(
-          target_log_prob_fn=_unbounded_exponential_log_prob,
-          current_state=initial_x,
-          step_size=2.,
-          num_leapfrog_steps=5,
-          seed=46)
-      initial_x_, updated_x_, log_accept_ratio_ = sess.run(
-          [initial_x, updated_x, kernel_results.log_accept_ratio])
-      acceptance_probs = np.exp(np.minimum(log_accept_ratio_, 0.))
-
-      logging_ops.vlog(1, "initial_x = {}".format(initial_x_))
-      logging_ops.vlog(1, "updated_x = {}".format(updated_x_))
-      logging_ops.vlog(1, "log_accept_ratio = {}".format(log_accept_ratio_))
-
-      self.assertAllEqual(initial_x_, updated_x_)
-      self.assertEqual(acceptance_probs, 0.)
-
-  def testNanFromGradsDontPropagate(self):
-    """Test that update with NaN gradients does not cause NaN in results."""
-    def _nan_log_prob_with_nan_gradient(x):
-      return np.nan * math_ops.reduce_sum(x)
-
-    with self.test_session(graph=ops.Graph()) as sess:
-      initial_x = math_ops.linspace(0.01, 5, 10)
-      updated_x, kernel_results = hmc.kernel(
-          target_log_prob_fn=_nan_log_prob_with_nan_gradient,
-          current_state=initial_x,
-          step_size=2.,
-          num_leapfrog_steps=5,
-          seed=47)
-      initial_x_, updated_x_, log_accept_ratio_ = sess.run(
-          [initial_x, updated_x, kernel_results.log_accept_ratio])
-      acceptance_probs = np.exp(np.minimum(log_accept_ratio_, 0.))
-
-      logging_ops.vlog(1, "initial_x = {}".format(initial_x_))
-      logging_ops.vlog(1, "updated_x = {}".format(updated_x_))
-      logging_ops.vlog(1, "log_accept_ratio = {}".format(log_accept_ratio_))
-
-      self.assertAllEqual(initial_x_, updated_x_)
-      self.assertEqual(acceptance_probs, 0.)
-
-      self.assertAllFinite(
-          gradients_ops.gradients(updated_x, initial_x)[0].eval())
-      self.assertAllEqual([True], [g is None for g in gradients_ops.gradients(
-          kernel_results.proposed_grads_target_log_prob, initial_x)])
-      self.assertAllEqual([False], [g is None for g in gradients_ops.gradients(
-          kernel_results.proposed_grads_target_log_prob,
-          kernel_results.proposed_state)])
-
-      # Gradients of the acceptance probs and new log prob are not finite.
-      # self.assertAllFinite(
-      #     gradients_ops.gradients(acceptance_probs, initial_x)[0].eval())
-      # self.assertAllFinite(
-      #     gradients_ops.gradients(new_log_prob, initial_x)[0].eval())
-
-  def _testChainWorksDtype(self, dtype):
-    with self.test_session(graph=ops.Graph()) as sess:
-      states, kernel_results = hmc.sample_chain(
-          num_results=10,
-          target_log_prob_fn=lambda x: -math_ops.reduce_sum(x**2., axis=-1),
-          current_state=np.zeros(5).astype(dtype),
-          step_size=0.01,
-          num_leapfrog_steps=10,
-          seed=48)
-      states_, log_accept_ratio_ = sess.run(
-          [states, kernel_results.log_accept_ratio])
-      self.assertEqual(dtype, states_.dtype)
-      self.assertEqual(dtype, log_accept_ratio_.dtype)
-
-  def testChainWorksIn64Bit(self):
-    self._testChainWorksDtype(np.float64)
-
-  def testChainWorksIn16Bit(self):
-    self._testChainWorksDtype(np.float16)
-
-  def testChainWorksCorrelatedMultivariate(self):
-    dtype = np.float32
-    true_mean = dtype([0, 0])
-    true_cov = dtype([[1, 0.5],
-                      [0.5, 1]])
-    num_results = 2000
-    counter = collections.Counter()
-    with self.test_session(graph=ops.Graph()) as sess:
-      def target_log_prob(x, y):
-        counter["target_calls"] += 1
-        # Corresponds to unnormalized MVN.
-        # z = matmul(inv(chol(true_cov)), [x, y] - true_mean)
-        z = array_ops.stack([x, y], axis=-1) - true_mean
-        z = array_ops.squeeze(
-            gen_linalg_ops.matrix_triangular_solve(
-                np.linalg.cholesky(true_cov),
-                z[..., array_ops.newaxis]),
-            axis=-1)
-        return -0.5 * math_ops.reduce_sum(z**2., axis=-1)
-      states, _ = hmc.sample_chain(
-          num_results=num_results,
-          target_log_prob_fn=target_log_prob,
-          current_state=[dtype(-2), dtype(2)],
-          step_size=[0.5, 0.5],
-          num_leapfrog_steps=2,
-          num_burnin_steps=200,
-          num_steps_between_results=1,
-          seed=54)
-      self.assertAllEqual(dict(target_calls=2), counter)
-      states = array_ops.stack(states, axis=-1)
-      self.assertEqual(num_results, states.shape[0].value)
-      sample_mean = math_ops.reduce_mean(states, axis=0)
-      x = states - sample_mean
-      sample_cov = math_ops.matmul(x, x, transpose_a=True) / dtype(num_results)
-      [sample_mean_, sample_cov_] = sess.run([
-          sample_mean, sample_cov])
-      self.assertAllClose(true_mean, sample_mean_,
-                          atol=0.05, rtol=0.)
-      self.assertAllClose(true_cov, sample_cov_,
-                          atol=0., rtol=0.1)
-
-
-class _EnergyComputationTest(object):
-
-  def testHandlesNanFromPotential(self):
-    with self.test_session(graph=ops.Graph()) as sess:
-      x = [1, np.inf, -np.inf, np.nan]
-      target_log_prob, proposed_target_log_prob = [
-          self.dtype(x.flatten()) for x in np.meshgrid(x, x)]
-      num_chains = len(target_log_prob)
-      dummy_momentums = [-1, 1]
-      momentums = [self.dtype([dummy_momentums] * num_chains)]
-      proposed_momentums = [self.dtype([dummy_momentums] * num_chains)]
-
-      target_log_prob = ops.convert_to_tensor(target_log_prob)
-      momentums = [ops.convert_to_tensor(momentums[0])]
-      proposed_target_log_prob = ops.convert_to_tensor(proposed_target_log_prob)
-      proposed_momentums = [ops.convert_to_tensor(proposed_momentums[0])]
-
-      energy = _compute_energy_change(
-          target_log_prob,
-          momentums,
-          proposed_target_log_prob,
-          proposed_momentums,
-          independent_chain_ndims=1)
-      grads = gradients_ops.gradients(energy, momentums)
-
-      [actual_energy, grads_] = sess.run([energy, grads])
-
-      # Ensure energy is `inf` (note: that's positive inf) in weird cases and
-      # finite otherwise.
-      expected_energy = self.dtype([0] + [np.inf]*(num_chains - 1))
-      self.assertAllEqual(expected_energy, actual_energy)
-
-      # Ensure gradient is finite.
-      self.assertAllEqual(np.ones_like(grads_).astype(np.bool),
-                          np.isfinite(grads_))
-
-  def testHandlesNanFromKinetic(self):
-    with self.test_session(graph=ops.Graph()) as sess:
-      x = [1, np.inf, -np.inf, np.nan]
-      momentums, proposed_momentums = [
-          [np.reshape(self.dtype(x), [-1, 1])]
-          for x in np.meshgrid(x, x)]
-      num_chains = len(momentums[0])
-      target_log_prob = np.ones(num_chains, self.dtype)
-      proposed_target_log_prob = np.ones(num_chains, self.dtype)
-
-      target_log_prob = ops.convert_to_tensor(target_log_prob)
-      momentums = [ops.convert_to_tensor(momentums[0])]
-      proposed_target_log_prob = ops.convert_to_tensor(proposed_target_log_prob)
-      proposed_momentums = [ops.convert_to_tensor(proposed_momentums[0])]
-
-      energy = _compute_energy_change(
-          target_log_prob,
-          momentums,
-          proposed_target_log_prob,
-          proposed_momentums,
-          independent_chain_ndims=1)
-      grads = gradients_ops.gradients(energy, momentums)
-
-      [actual_energy, grads_] = sess.run([energy, grads])
-
-      # Ensure energy is `inf` (note: that's positive inf) in weird cases and
-      # finite otherwise.
-      expected_energy = self.dtype([0] + [np.inf]*(num_chains - 1))
-      self.assertAllEqual(expected_energy, actual_energy)
-
-      # Ensure gradient is finite.
-      g = grads_[0].reshape([len(x), len(x)])[:, 0]
-      self.assertAllEqual(np.ones_like(g).astype(np.bool), np.isfinite(g))
-
-      # The remaining gradients are nan because the momentum was itself nan or
-      # inf.
-      g = grads_[0].reshape([len(x), len(x)])[:, 1:]
-      self.assertAllEqual(np.ones_like(g).astype(np.bool), np.isnan(g))
-
-
-class EnergyComputationTest16(test.TestCase, _EnergyComputationTest):
-  dtype = np.float16
-
-
-class EnergyComputationTest32(test.TestCase, _EnergyComputationTest):
-  dtype = np.float32
-
-
-class EnergyComputationTest64(test.TestCase, _EnergyComputationTest):
-  dtype = np.float64
-
-
-class _HMCHandlesLists(object):
-
-  def testStateParts(self):
-    with self.test_session(graph=ops.Graph()) as sess:
-      dist_x = normal_lib.Normal(loc=self.dtype(0), scale=self.dtype(1))
-      dist_y = independent_lib.Independent(
-          gamma_lib.Gamma(concentration=self.dtype([1, 2]),
-                          rate=self.dtype([0.5, 0.75])),
-          reinterpreted_batch_ndims=1)
-      def target_log_prob(x, y):
-        return dist_x.log_prob(x) + dist_y.log_prob(y)
-      x0 = [dist_x.sample(seed=1), dist_y.sample(seed=2)]
-      samples, _ = hmc.sample_chain(
-          num_results=int(2e3),
-          target_log_prob_fn=target_log_prob,
-          current_state=x0,
-          step_size=0.85,
-          num_leapfrog_steps=3,
-          num_burnin_steps=int(250),
-          seed=49)
-      actual_means = [math_ops.reduce_mean(s, axis=0) for s in samples]
-      actual_vars = [_reduce_variance(s, axis=0) for s in samples]
-      expected_means = [dist_x.mean(), dist_y.mean()]
-      expected_vars = [dist_x.variance(), dist_y.variance()]
-      [
-          actual_means_,
-          actual_vars_,
-          expected_means_,
-          expected_vars_,
-      ] = sess.run([
-          actual_means,
-          actual_vars,
-          expected_means,
-          expected_vars,
-      ])
-      self.assertAllClose(expected_means_, actual_means_, atol=0.05, rtol=0.16)
-      self.assertAllClose(expected_vars_, actual_vars_, atol=0., rtol=0.25)
-
-
-class HMCHandlesLists32(_HMCHandlesLists, test.TestCase):
-  dtype = np.float32
-
-
-class HMCHandlesLists64(_HMCHandlesLists, test.TestCase):
-  dtype = np.float64
-
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/bayesflow/python/kernel_tests/metropolis_hastings_test.py b/tensorflow/contrib/bayesflow/python/kernel_tests/metropolis_hastings_test.py
deleted file mode 100644
index f508e5b114..0000000000
--- a/tensorflow/contrib/bayesflow/python/kernel_tests/metropolis_hastings_test.py
+++ /dev/null
@@ -1,340 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Tests for Metropolis-Hastings."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import numpy as np
-
-from tensorflow.contrib.bayesflow.python.ops import metropolis_hastings_impl as mh
-from tensorflow.contrib.distributions.python.ops import mvn_tril as mvn_tril_lib
-from tensorflow.python.framework import constant_op
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import init_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import variable_scope
-from tensorflow.python.ops import variables
-from tensorflow.python.ops.distributions import normal as normal_lib
-from tensorflow.python.platform import test
-
-
-class MetropolisHastingsTest(test.TestCase):
-
-  def testKernelStateTensor(self):
-    """Test that transition kernel works with tensor input to `state`."""
-    loc = variable_scope.get_variable("loc", initializer=0.)
-
-    def target_log_prob_fn(loc):
-      return normal_lib.Normal(loc=0.0, scale=0.1).log_prob(loc)
-
-    new_state, _ = mh.kernel(
-        target_log_prob_fn=target_log_prob_fn,
-        proposal_fn=mh.proposal_normal(scale=0.05),
-        current_state=loc,
-        seed=231251)
-    loc_update = loc.assign(new_state)
-
-    init = variables.initialize_all_variables()
-    with self.test_session() as sess:
-      sess.run(init)
-      loc_samples = []
-      for _ in range(2500):
-        loc_sample = sess.run(loc_update)
-        loc_samples.append(loc_sample)
-    loc_samples = loc_samples[500:]  # drop samples for burn-in
-
-    self.assertAllClose(np.mean(loc_samples), 0.0, rtol=1e-5, atol=1e-1)
-    self.assertAllClose(np.std(loc_samples), 0.1, rtol=1e-5, atol=1e-1)
-
-  def testKernelStateList(self):
-    """Test that transition kernel works with list input to `state`."""
-    num_chains = 2
-    loc_one = variable_scope.get_variable(
-        "loc_one", [num_chains],
-        initializer=init_ops.zeros_initializer())
-    loc_two = variable_scope.get_variable(
-        "loc_two", [num_chains], initializer=init_ops.zeros_initializer())
-
-    def target_log_prob_fn(loc_one, loc_two):
-      loc = array_ops.stack([loc_one, loc_two])
-      log_prob = mvn_tril_lib.MultivariateNormalTriL(
-          loc=constant_op.constant([0., 0.]),
-          scale_tril=constant_op.constant([[0.1, 0.1], [0.0, 0.1]])).log_prob(
-              loc)
-      return math_ops.reduce_sum(log_prob, 0)
-
-    def proposal_fn(loc_one, loc_two):
-      loc_one_proposal = mh.proposal_normal(scale=0.05)
-      loc_two_proposal = mh.proposal_normal(scale=0.05)
-      loc_one_sample, _ = loc_one_proposal(loc_one)
-      loc_two_sample, _ = loc_two_proposal(loc_two)
-      return [loc_one_sample, loc_two_sample], None
-
-    new_state, _ = mh.kernel(
-        target_log_prob_fn=target_log_prob_fn,
-        proposal_fn=proposal_fn,
-        current_state=[loc_one, loc_two],
-        seed=12415)
-    loc_one_update = loc_one.assign(new_state[0])
-    loc_two_update = loc_two.assign(new_state[1])
-
-    init = variables.initialize_all_variables()
-    with self.test_session() as sess:
-      sess.run(init)
-      loc_one_samples = []
-      loc_two_samples = []
-      for _ in range(10000):
-        loc_one_sample, loc_two_sample = sess.run(
-            [loc_one_update, loc_two_update])
-        loc_one_samples.append(loc_one_sample)
-        loc_two_samples.append(loc_two_sample)
-
-    loc_one_samples = np.array(loc_one_samples)
-    loc_two_samples = np.array(loc_two_samples)
-    loc_one_samples = loc_one_samples[1000:]  # drop samples for burn-in
-    loc_two_samples = loc_two_samples[1000:]  # drop samples for burn-in
-
-    self.assertAllClose(np.mean(loc_one_samples, 0),
-                        np.array([0.] * num_chains),
-                        rtol=1e-5, atol=1e-1)
-    self.assertAllClose(np.mean(loc_two_samples, 0),
-                        np.array([0.] * num_chains),
-                        rtol=1e-5, atol=1e-1)
-    self.assertAllClose(np.std(loc_one_samples, 0),
-                        np.array([0.1] * num_chains),
-                        rtol=1e-5, atol=1e-1)
-    self.assertAllClose(np.std(loc_two_samples, 0),
-                        np.array([0.1] * num_chains),
-                        rtol=1e-5, atol=1e-1)
-
-  def testKernelResultsUsingTruncatedDistribution(self):
-    def log_prob(x):
-      return array_ops.where(
-          x >= 0.,
-          -x - x**2,
-          array_ops.fill(x.shape, math_ops.cast(-np.inf, x.dtype)))
-    # The truncated distribution has the property that it is likely to attract
-    # the flow toward, and below, zero...but for x <=0,
-    # log_prob(x) = -inf, which should result in rejection, as well
-    # as a non-finite log_prob.  Thus, this distribution gives us an opportunity
-    # to test out the kernel results ability to correctly capture rejections due
-    # to finite AND non-finite reasons.
-
-    num_results = 1000
-    # Large step size, will give rejections due to going into a region of
-    # log_prob = -inf.
-    step_size = 0.3
-    num_chains = 2
-
-    with self.test_session(graph=ops.Graph()) as sess:
-
-      # Start multiple independent chains.
-      initial_state = ops.convert_to_tensor([0.1] * num_chains)
-
-      states = []
-      is_accepted = []
-      proposed_states = []
-      current_state = initial_state
-      for _ in range(num_results):
-        current_state, kernel_results = mh.kernel(
-            target_log_prob_fn=log_prob,
-            proposal_fn=mh.proposal_uniform(step_size=step_size),
-            current_state=current_state,
-            seed=42)
-        states.append(current_state)
-        proposed_states.append(kernel_results.proposed_state)
-        is_accepted.append(kernel_results.is_accepted)
-
-      states = array_ops.stack(states)
-      proposed_states = array_ops.stack(proposed_states)
-      is_accepted = array_ops.stack(is_accepted)
-      states_, pstates_, is_accepted_ = sess.run(
-          [states, proposed_states, is_accepted])
-
-      # We better have accepted a decent amount, even near end of the chain.
-      self.assertLess(
-          0.1, is_accepted_[int(0.9 * num_results):].mean())
-      # We better not have any NaNs in states.
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isnan(states_))
-      # We better not have any +inf in states.
-      self.assertAllEqual(np.zeros_like(states_),
-                          np.isposinf(states_))
-
-      # The move is accepted ==> state = proposed state.
-      self.assertAllEqual(
-          states_[is_accepted_],
-          pstates_[is_accepted_],
-      )
-
-      # The move was rejected <==> state[t] == state[t - 1].
-      for t in range(1, num_results):
-        for i in range(num_chains):
-          if is_accepted_[t, i]:
-            self.assertNotEqual(states_[t, i], states_[t - 1, i])
-          else:
-            self.assertEqual(states_[t, i], states_[t - 1, i])
-
-  def testDensityIncreasingStepAccepted(self):
-    """Tests that if a transition increases density, it is always accepted."""
-    target_log_density = lambda x: - x * x
-    state = variable_scope.get_variable("state", initializer=10.)
-    state_log_density = variable_scope.get_variable(
-        "state_log_density",
-        initializer=target_log_density(state.initialized_value()))
-    log_accept_ratio = variable_scope.get_variable(
-        "log_accept_ratio", initializer=0.)
-
-    get_next_proposal = lambda x: (x - 1., None)
-    step = mh.evolve(state, state_log_density, log_accept_ratio,
-                     target_log_density, get_next_proposal, seed=1234)
-    init = variables.initialize_all_variables()
-    with self.test_session() as sess:
-      sess.run(init)
-      for j in range(9):
-        sess.run(step)
-        sample = sess.run(state)
-        sample_log_density = sess.run(state_log_density)
-        self.assertAlmostEqual(sample, 9 - j)
-        self.assertAlmostEqual(sample_log_density, - (9 - j) * (9 - j))
-
-  def testSampleProperties(self):
-    """Tests that the samples converge to the target distribution."""
-
-    def target_log_density(x):
-      """Log-density corresponding to a normal distribution with mean = 4."""
-      return - (x - 2.0) * (x - 2.0) * 0.5
-
-    # Use the uniform random walker to generate proposals.
-    proposal_fn = mh.proposal_uniform(
-        step_size=1.0, seed=1234)
-
-    state = variable_scope.get_variable("state", initializer=0.0)
-    state_log_density = variable_scope.get_variable(
-        "state_log_density",
-        initializer=target_log_density(state.initialized_value()))
-    log_accept_ratio = variable_scope.get_variable(
-        "log_accept_ratio", initializer=0.)
-
-    # Random walk MCMC converges slowly so need to put in enough iterations.
-    num_iterations = 5000
-    step = mh.evolve(state, state_log_density, log_accept_ratio,
-                     target_log_density, proposal_fn, seed=4321)
-
-    init = variables.global_variables_initializer()
-
-    sample_sum, sample_sq_sum = 0.0, 0.0
-    with self.test_session() as sess:
-      sess.run(init)
-      for _ in np.arange(num_iterations):
-        # Allow for the mixing of the chain and discard these samples.
-        sess.run(step)
-      for _ in np.arange(num_iterations):
-        sess.run(step)
-        sample = sess.run(state)
-        sample_sum += sample
-        sample_sq_sum += sample * sample
-
-    sample_mean = sample_sum / num_iterations
-    sample_variance = sample_sq_sum / num_iterations - sample_mean * sample_mean
-    # The samples have large autocorrelation which reduces the effective sample
-    # size.
-    self.assertAlmostEqual(sample_mean, 2.0, delta=0.1)
-    self.assertAlmostEqual(sample_variance, 1.0, delta=0.1)
-
-  def testProposalNormal(self):
-    """Tests that the normal proposals are correctly distributed."""
-
-    initial_points = array_ops.ones([10000], dtype=dtypes.float32)
-    proposal_fn = mh.proposal_normal(
-        scale=2.0, seed=1234)
-    proposal_points, _ = proposal_fn(initial_points)
-
-    with self.test_session() as sess:
-      sample = sess.run(proposal_points)
-
-    # It is expected that the elements in proposal_points have the same mean as
-    # initial_points and have the standard deviation that was supplied to the
-    # proposal scheme.
-    self.assertAlmostEqual(np.mean(sample), 1.0, delta=0.1)
-    self.assertAlmostEqual(np.std(sample), 2.0, delta=0.1)
-
-  def testDocstringExample(self):
-    """Tests the simplified docstring example with multiple chains."""
-
-    n = 2  # dimension of the problem
-
-    # Generate 300 initial values randomly. Each of these would be an
-    # independent starting point for a Markov chain.
-    state = variable_scope.get_variable(
-        "state", initializer=random_ops.random_normal(
-            [300, n], mean=3.0, dtype=dtypes.float32, seed=42))
-
-    # Computes the log(p(x)) for the unit normal density and ignores the
-    # normalization constant.
-    def log_density(x):
-      return  - math_ops.reduce_sum(x * x, reduction_indices=-1) / 2.0
-
-    # Initial log-density value
-    state_log_density = variable_scope.get_variable(
-        "state_log_density",
-        initializer=log_density(state.initialized_value()))
-
-    # A variable to store the log_acceptance_ratio:
-    log_acceptance_ratio = variable_scope.get_variable(
-        "log_acceptance_ratio",
-        initializer=array_ops.zeros([300], dtype=dtypes.float32))
-
-    # Generates random proposals by moving each coordinate uniformly and
-    # independently in a box of size 2 centered around the current value.
-    # Returns the new point and also the log of the Hastings ratio (the
-    # ratio of the probability of going from the proposal to origin and the
-    # probability of the reverse transition). When this ratio is 1, the value
-    # may be omitted and replaced by None.
-    def random_proposal(x):
-      return (x + random_ops.random_uniform(
-          array_ops.shape(x), minval=-1, maxval=1,
-          dtype=x.dtype, seed=12)), None
-
-    #  Create the op to propagate the chain for 100 steps.
-    stepper = mh.evolve(
-        state, state_log_density, log_acceptance_ratio,
-        log_density, random_proposal, n_steps=100, seed=123)
-    init = variables.initialize_all_variables()
-    with self.test_session() as sess:
-      sess.run(init)
-      # Run the chains for a total of 1000 steps.
-      for _ in range(10):
-        sess.run(stepper)
-      samples = sess.run(state)
-      covariance = np.eye(n)
-      # Verify that the estimated mean and covariance are close to the true
-      # values.
-      self.assertAlmostEqual(
-          np.max(np.abs(np.mean(samples, 0)
-                        - np.zeros(n))), 0,
-          delta=0.1)
-      self.assertAlmostEqual(
-          np.max(np.abs(np.reshape(np.cov(samples, rowvar=False), [n**2])
-                        - np.reshape(covariance, [n**2]))), 0,
-          delta=0.2)
-
-if __name__ == "__main__":
-  test.main()
diff --git a/tensorflow/contrib/bayesflow/python/ops/hmc.py b/tensorflow/contrib/bayesflow/python/ops/hmc.py
deleted file mode 100644
index c8a5a195d3..0000000000
--- a/tensorflow/contrib/bayesflow/python/ops/hmc.py
+++ /dev/null
@@ -1,30 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Hamiltonian Monte Carlo, a gradient-based MCMC algorithm."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# go/tf-wildcard-import
-from tensorflow.contrib.bayesflow.python.ops.hmc_impl import *  # pylint: disable=wildcard-import,unused-wildcard-import,g-importing-member
-from tensorflow.python.util import all_util
-
-_allowed_symbols = [
-    "sample_chain",
-    "kernel",
-]
-
-all_util.remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/bayesflow/python/ops/hmc_impl.py b/tensorflow/contrib/bayesflow/python/ops/hmc_impl.py
deleted file mode 100644
index 66afcc7497..0000000000
--- a/tensorflow/contrib/bayesflow/python/ops/hmc_impl.py
+++ /dev/null
@@ -1,961 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Hamiltonian Monte Carlo, a gradient-based MCMC algorithm.
-
-@@sample_chain
-@@kernel
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-import numpy as np
-
-from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import functional_ops
-from tensorflow.python.ops import gradients_impl as gradients_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.ops.distributions import util as distributions_util
-
-__all__ = [
-    "sample_chain",
-    "kernel",
-]
-
-
-KernelResults = collections.namedtuple(
-    "KernelResults",
-    [
-        "log_accept_ratio",
-        "current_grads_target_log_prob",  # "Current result" means "accepted".
-        "current_target_log_prob",  # "Current result" means "accepted".
-        "is_accepted",
-        "proposed_grads_target_log_prob",
-        "proposed_state",
-        "proposed_target_log_prob",
-    ])
-
-
-def _make_dummy_kernel_results(
-    dummy_state,
-    dummy_target_log_prob,
-    dummy_grads_target_log_prob):
-  return KernelResults(
-      log_accept_ratio=dummy_target_log_prob,
-      current_grads_target_log_prob=dummy_grads_target_log_prob,
-      current_target_log_prob=dummy_target_log_prob,
-      is_accepted=array_ops.ones_like(dummy_target_log_prob, dtypes.bool),
-      proposed_grads_target_log_prob=dummy_grads_target_log_prob,
-      proposed_state=dummy_state,
-      proposed_target_log_prob=dummy_target_log_prob,
-  )
-
-
-def sample_chain(
-    num_results,
-    target_log_prob_fn,
-    current_state,
-    step_size,
-    num_leapfrog_steps,
-    num_burnin_steps=0,
-    num_steps_between_results=0,
-    seed=None,
-    current_target_log_prob=None,
-    current_grads_target_log_prob=None,
-    name=None):
-  """Runs multiple iterations of one or more Hamiltonian Monte Carlo chains.
-
-  Hamiltonian Monte Carlo (HMC) is a Markov chain Monte Carlo (MCMC) algorithm
-  that takes a series of gradient-informed steps to produce a Metropolis
-  proposal. This function samples from an HMC Markov chain at `current_state`
-  and whose stationary distribution has log-unnormalized-density
-  `target_log_prob_fn()`.
-
-  This function samples from multiple chains in parallel. It assumes that the
-  the leftmost dimensions of (each) `current_state` (part) index an independent
-  chain.  The function `target_log_prob_fn()` sums log-probabilities across
-  event dimensions (i.e., current state (part) rightmost dimensions). Each
-  element of the output of `target_log_prob_fn()` represents the (possibly
-  unnormalized) log-probability of the joint distribution over (all) the current
-  state (parts).
-
-  The `current_state` can be represented as a single `Tensor` or a `list` of
-  `Tensors` which collectively represent the current state. When specifying a
-  `list`, one must also specify a list of `step_size`s.
-
-  Note: `target_log_prob_fn` is called exactly twice.
-
-  Since HMC states are correlated, it is sometimes desirable to produce
-  additional intermediate states, and then discard them, ending up with a set of
-  states with decreased autocorrelation.  See [1].  Such "thinning" is made
-  possible by setting `num_steps_between_results > 0`.  The chain then takes
-  `num_steps_between_results` extra steps between the steps that make it into
-  the results.  The extra steps are never materialized (in calls to `sess.run`),
-  and thus do not increase memory requirements.
-
-  [1]: "Statistically efficient thinning of a Markov chain sampler."
-       Art B. Owen. April 2017.
-       http://statweb.stanford.edu/~owen/reports/bestthinning.pdf
-
-  #### Examples:
-
-  ##### Sample from a diagonal-variance Gaussian.
-
-  ```python
-  tfd = tf.contrib.distributions
-
-  def make_likelihood(true_variances):
-    return tfd.MultivariateNormalDiag(
-        scale_diag=tf.sqrt(true_variances))
-
-  dims = 10
-  dtype = np.float32
-  true_variances = tf.linspace(dtype(1), dtype(3), dims)
-  likelihood = make_likelihood(true_variances)
-
-  states, kernel_results = hmc.sample_chain(
-      num_results=1000,
-      target_log_prob_fn=likelihood.log_prob,
-      current_state=tf.zeros(dims),
-      step_size=0.5,
-      num_leapfrog_steps=2,
-      num_burnin_steps=500)
-
-  # Compute sample stats.
-  sample_mean = tf.reduce_mean(states, axis=0)
-  sample_var = tf.reduce_mean(
-      tf.squared_difference(states, sample_mean),
-      axis=0)
-  ```
-
-  ##### Sampling from factor-analysis posteriors with known factors.
-
-  I.e.,
-
-  ```none
-  for i=1..n:
-    w[i] ~ Normal(0, eye(d))            # prior
-    x[i] ~ Normal(loc=matmul(w[i], F))  # likelihood
-  ```
-
-  where `F` denotes factors.
-
-  ```python
-  tfd = tf.contrib.distributions
-
-  def make_prior(dims, dtype):
-    return tfd.MultivariateNormalDiag(
-        loc=tf.zeros(dims, dtype))
-
-  def make_likelihood(weights, factors):
-    return tfd.MultivariateNormalDiag(
-        loc=tf.tensordot(weights, factors, axes=[[0], [-1]]))
-
-  # Setup data.
-  num_weights = 10
-  num_factors = 4
-  num_chains = 100
-  dtype = np.float32
-
-  prior = make_prior(num_weights, dtype)
-  weights = prior.sample(num_chains)
-  factors = np.random.randn(num_factors, num_weights).astype(dtype)
-  x = make_likelihood(weights, factors).sample(num_chains)
-
-  def target_log_prob(w):
-    # Target joint is: `f(w) = p(w, x | factors)`.
-    return prior.log_prob(w) + make_likelihood(w, factors).log_prob(x)
-
-  # Get `num_results` samples from `num_chains` independent chains.
-  chains_states, kernels_results = hmc.sample_chain(
-      num_results=1000,
-      target_log_prob_fn=target_log_prob,
-      current_state=tf.zeros([num_chains, dims], dtype),
-      step_size=0.1,
-      num_leapfrog_steps=2,
-      num_burnin_steps=500)
-
-  # Compute sample stats.
-  sample_mean = tf.reduce_mean(chains_states, axis=[0, 1])
-  sample_var = tf.reduce_mean(
-      tf.squared_difference(chains_states, sample_mean),
-      axis=[0, 1])
-  ```
-
-  Args:
-    num_results: Integer number of Markov chain draws.
-    target_log_prob_fn: Python callable which takes an argument like
-      `current_state` (or `*current_state` if it's a list) and returns its
-      (possibly unnormalized) log-density under the target distribution.
-    current_state: `Tensor` or Python `list` of `Tensor`s representing the
-      current state(s) of the Markov chain(s). The first `r` dimensions index
-      independent chains, `r = tf.rank(target_log_prob_fn(*current_state))`.
-    step_size: `Tensor` or Python `list` of `Tensor`s representing the step size
-      for the leapfrog integrator. Must broadcast with the shape of
-      `current_state`. Larger step sizes lead to faster progress, but too-large
-      step sizes make rejection exponentially more likely. When possible, it's
-      often helpful to match per-variable step sizes to the standard deviations
-      of the target distribution in each variable.
-    num_leapfrog_steps: Integer number of steps to run the leapfrog integrator
-      for. Total progress per HMC step is roughly proportional to `step_size *
-      num_leapfrog_steps`.
-    num_burnin_steps: Integer number of chain steps to take before starting to
-      collect results.
-      Default value: 0 (i.e., no burn-in).
-    num_steps_between_results: Integer number of chain steps between collecting
-      a result. Only one out of every `num_steps_between_samples + 1` steps is
-      included in the returned results.  The number of returned chain states is
-      still equal to `num_results`.  Default value: 0 (i.e., no thinning).
-    seed: Python integer to seed the random number generator.
-    current_target_log_prob: (Optional) `Tensor` representing the value of
-      `target_log_prob_fn` at the `current_state`. The only reason to specify
-      this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    current_grads_target_log_prob: (Optional) Python list of `Tensor`s
-      representing gradient of `target_log_prob` at the `current_state` and wrt
-      the `current_state`. Must have same shape as `current_state`. The only
-      reason to specify this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    name: Python `str` name prefixed to Ops created by this function.
-      Default value: `None` (i.e., "hmc_sample_chain").
-
-  Returns:
-    next_states: Tensor or Python list of `Tensor`s representing the
-      state(s) of the Markov chain(s) at each result step. Has same shape as
-      input `current_state` but with a prepended `num_results`-size dimension.
-    kernel_results: `collections.namedtuple` of internal calculations used to
-      advance the chain.
-  """
-  with ops.name_scope(
-      name, "hmc_sample_chain",
-      [num_results, current_state, step_size, num_leapfrog_steps,
-       num_burnin_steps, num_steps_between_results, seed,
-       current_target_log_prob, current_grads_target_log_prob]):
-    with ops.name_scope("initialize"):
-      [
-          current_state,
-          step_size,
-          current_target_log_prob,
-          current_grads_target_log_prob,
-      ] = _prepare_args(
-          target_log_prob_fn,
-          current_state,
-          step_size,
-          current_target_log_prob,
-          current_grads_target_log_prob)
-      num_results = ops.convert_to_tensor(
-          num_results,
-          dtype=dtypes.int32,
-          name="num_results")
-      num_leapfrog_steps = ops.convert_to_tensor(
-          num_leapfrog_steps,
-          dtype=dtypes.int32,
-          name="num_leapfrog_steps")
-      num_burnin_steps = ops.convert_to_tensor(
-          num_burnin_steps,
-          dtype=dtypes.int32,
-          name="num_burnin_steps")
-      num_steps_between_results = ops.convert_to_tensor(
-          num_steps_between_results,
-          dtype=dtypes.int32,
-          name="num_steps_between_results")
-
-    def _run_chain(num_steps, current_state, kernel_results):
-      """Runs the chain(s) for `num_steps`."""
-      def _loop_body(iter_, current_state, kernel_results):
-        return [iter_ + 1] + list(kernel(
-            target_log_prob_fn,
-            current_state,
-            step_size,
-            num_leapfrog_steps,
-            seed,
-            kernel_results.current_target_log_prob,
-            kernel_results.current_grads_target_log_prob))
-      while_loop_kwargs = dict(
-          cond=lambda iter_, *args: iter_ < num_steps,
-          body=_loop_body,
-          loop_vars=[
-              np.int32(0),
-              current_state,
-              kernel_results,
-          ],
-      )
-      if seed is not None:
-        while_loop_kwargs["parallel_iterations"] = 1
-      return control_flow_ops.while_loop(
-          **while_loop_kwargs)[1:]  # Lop-off "iter_".
-
-    def _scan_body(args_list, iter_):
-      """Closure which implements `tf.scan` body."""
-      current_state, kernel_results = args_list
-      return _run_chain(
-          1 + array_ops.where(math_ops.equal(iter_, 0),
-                              num_burnin_steps,
-                              num_steps_between_results),
-          current_state,
-          kernel_results)
-
-    scan_kwargs = dict(
-        fn=_scan_body,
-        elems=math_ops.range(num_results),  # iter_: used to choose burnin.
-        initializer=[
-            current_state,
-            _make_dummy_kernel_results(
-                current_state,
-                current_target_log_prob,
-                current_grads_target_log_prob),
-        ])
-    if seed is not None:
-      scan_kwargs["parallel_iterations"] = 1
-    return functional_ops.scan(**scan_kwargs)
-
-
-def kernel(target_log_prob_fn,
-           current_state,
-           step_size,
-           num_leapfrog_steps,
-           seed=None,
-           current_target_log_prob=None,
-           current_grads_target_log_prob=None,
-           name=None):
-  """Runs one iteration of Hamiltonian Monte Carlo.
-
-  Hamiltonian Monte Carlo (HMC) is a Markov chain Monte Carlo (MCMC)
-  algorithm that takes a series of gradient-informed steps to produce
-  a Metropolis proposal. This function applies one step of HMC to
-  randomly update the variable `x`.
-
-  This function can update multiple chains in parallel. It assumes that all
-  leftmost dimensions of `current_state` index independent chain states (and are
-  therefore updated independently). The output of `target_log_prob_fn()` should
-  sum log-probabilities across all event dimensions. Slices along the rightmost
-  dimensions may have different target distributions; for example,
-  `current_state[0, :]` could have a different target distribution from
-  `current_state[1, :]`. This is up to `target_log_prob_fn()`. (The number of
-  independent chains is `tf.size(target_log_prob_fn(*current_state))`.)
-
-  #### Examples:
-
-  ##### Simple chain with warm-up.
-
-  ```python
-  tfd = tf.contrib.distributions
-
-  # Tuning acceptance rates:
-  dtype = np.float32
-  target_accept_rate = 0.631
-  num_warmup_iter = 500
-  num_chain_iter = 500
-
-  x = tf.get_variable(name="x", initializer=dtype(1))
-  step_size = tf.get_variable(name="step_size", initializer=dtype(1))
-
-  target = tfd.Normal(loc=dtype(0), scale=dtype(1))
-
-  next_x, other_results = hmc.kernel(
-      target_log_prob_fn=target.log_prob,
-      current_state=x,
-      step_size=step_size,
-      num_leapfrog_steps=3)[:4]
-
-  x_update = x.assign(next_x)
-
-  step_size_update = step_size.assign_add(
-      step_size * tf.where(
-          tf.exp(tf.minimum(other_results.log_accept_ratio), 0.) >
-              target_accept_rate,
-          0.01, -0.01))
-
-  warmup = tf.group([x_update, step_size_update])
-
-  tf.global_variables_initializer().run()
-
-  sess.graph.finalize()  # No more graph building.
-
-  # Warm up the sampler and adapt the step size
-  for _ in xrange(num_warmup_iter):
-    sess.run(warmup)
-
-  # Collect samples without adapting step size
-  samples = np.zeros([num_chain_iter])
-  for i in xrange(num_chain_iter):
-    _, x_, target_log_prob_, grad_ = sess.run([
-        x_update,
-        x,
-        other_results.target_log_prob,
-        other_results.grads_target_log_prob])
-    samples[i] = x_
-
-  print(samples.mean(), samples.std())
-  ```
-
-  ##### Sample from more complicated posterior.
-
-  I.e.,
-
-  ```none
-    W ~ MVN(loc=0, scale=sigma * eye(dims))
-    for i=1...num_samples:
-        X[i] ~ MVN(loc=0, scale=eye(dims))
-      eps[i] ~ Normal(loc=0, scale=1)
-        Y[i] = X[i].T * W + eps[i]
-  ```
-
-  ```python
-  tfd = tf.contrib.distributions
-
-  def make_training_data(num_samples, dims, sigma):
-    dt = np.asarray(sigma).dtype
-    zeros = tf.zeros(dims, dtype=dt)
-    x = tfd.MultivariateNormalDiag(
-        loc=zeros).sample(num_samples, seed=1)
-    w = tfd.MultivariateNormalDiag(
-        loc=zeros,
-        scale_identity_multiplier=sigma).sample(seed=2)
-    noise = tfd.Normal(
-        loc=dt(0),
-        scale=dt(1)).sample(num_samples, seed=3)
-    y = tf.tensordot(x, w, axes=[[1], [0]]) + noise
-    return y, x, w
-
-  def make_prior(sigma, dims):
-    # p(w | sigma)
-    return tfd.MultivariateNormalDiag(
-        loc=tf.zeros([dims], dtype=sigma.dtype),
-        scale_identity_multiplier=sigma)
-
-  def make_likelihood(x, w):
-    # p(y | x, w)
-    return tfd.MultivariateNormalDiag(
-        loc=tf.tensordot(x, w, axes=[[1], [0]]))
-
-  # Setup assumptions.
-  dtype = np.float32
-  num_samples = 150
-  dims = 10
-  num_iters = int(5e3)
-
-  true_sigma = dtype(0.5)
-  y, x, true_weights = make_training_data(num_samples, dims, true_sigma)
-
-  # Estimate of `log(true_sigma)`.
-  log_sigma = tf.get_variable(name="log_sigma", initializer=dtype(0))
-  sigma = tf.exp(log_sigma)
-
-  # State of the Markov chain.
-  weights = tf.get_variable(
-      name="weights",
-      initializer=np.random.randn(dims).astype(dtype))
-
-  prior = make_prior(sigma, dims)
-
-  def joint_log_prob_fn(w):
-    # f(w) = log p(w, y | x)
-    return prior.log_prob(w) + make_likelihood(x, w).log_prob(y)
-
-  weights_update = weights.assign(
-      hmc.kernel(target_log_prob_fn=joint_log_prob,
-                 current_state=weights,
-                 step_size=0.1,
-                 num_leapfrog_steps=5)[0])
-
-  with tf.control_dependencies([weights_update]):
-    loss = -prior.log_prob(weights)
-
-  optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
-  log_sigma_update = optimizer.minimize(loss, var_list=[log_sigma])
-
-  sess.graph.finalize()  # No more graph building.
-
-  tf.global_variables_initializer().run()
-
-  sigma_history = np.zeros(num_iters, dtype)
-  weights_history = np.zeros([num_iters, dims], dtype)
-
-  for i in xrange(num_iters):
-    _, sigma_, weights_, _ = sess.run([log_sigma_update, sigma, weights])
-    weights_history[i, :] = weights_
-    sigma_history[i] = sigma_
-
-  true_weights_ = sess.run(true_weights)
-
-  # Should converge to something close to true_sigma.
-  plt.plot(sigma_history);
-  plt.ylabel("sigma");
-  plt.xlabel("iteration");
-  ```
-
-  Args:
-    target_log_prob_fn: Python callable which takes an argument like
-      `current_state` (or `*current_state` if it's a list) and returns its
-      (possibly unnormalized) log-density under the target distribution.
-    current_state: `Tensor` or Python `list` of `Tensor`s representing the
-      current state(s) of the Markov chain(s). The first `r` dimensions index
-      independent chains, `r = tf.rank(target_log_prob_fn(*current_state))`.
-    step_size: `Tensor` or Python `list` of `Tensor`s representing the step size
-      for the leapfrog integrator. Must broadcast with the shape of
-      `current_state`. Larger step sizes lead to faster progress, but too-large
-      step sizes make rejection exponentially more likely. When possible, it's
-      often helpful to match per-variable step sizes to the standard deviations
-      of the target distribution in each variable.
-    num_leapfrog_steps: Integer number of steps to run the leapfrog integrator
-      for. Total progress per HMC step is roughly proportional to `step_size *
-      num_leapfrog_steps`.
-    seed: Python integer to seed the random number generator.
-    current_target_log_prob: (Optional) `Tensor` representing the value of
-      `target_log_prob_fn` at the `current_state`. The only reason to
-      specify this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    current_grads_target_log_prob: (Optional) Python list of `Tensor`s
-      representing gradient of `current_target_log_prob` at the `current_state`
-      and wrt the `current_state`. Must have same shape as `current_state`. The
-      only reason to specify this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    name: Python `str` name prefixed to Ops created by this function.
-      Default value: `None` (i.e., "hmc_kernel").
-
-  Returns:
-    next_state: Tensor or Python list of `Tensor`s representing the state(s)
-      of the Markov chain(s) at each result step. Has same shape as
-      `current_state`.
-    kernel_results: `collections.namedtuple` of internal calculations used to
-      advance the chain.
-
-  Raises:
-    ValueError: if there isn't one `step_size` or a list with same length as
-      `current_state`.
-  """
-  with ops.name_scope(
-      name, "hmc_kernel",
-      [current_state, step_size, num_leapfrog_steps, seed,
-       current_target_log_prob, current_grads_target_log_prob]):
-    with ops.name_scope("initialize"):
-      [current_state_parts, step_sizes, current_target_log_prob,
-       current_grads_target_log_prob] = _prepare_args(
-           target_log_prob_fn, current_state, step_size,
-           current_target_log_prob, current_grads_target_log_prob,
-           maybe_expand=True)
-      independent_chain_ndims = distributions_util.prefer_static_rank(
-          current_target_log_prob)
-      current_momentums = []
-      for s in current_state_parts:
-        current_momentums.append(random_ops.random_normal(
-            shape=array_ops.shape(s),
-            dtype=s.dtype.base_dtype,
-            seed=seed))
-        seed = distributions_util.gen_new_seed(
-            seed, salt="hmc_kernel_momentums")
-
-      num_leapfrog_steps = ops.convert_to_tensor(
-          num_leapfrog_steps,
-          dtype=dtypes.int32,
-          name="num_leapfrog_steps")
-    [
-        proposed_momentums,
-        proposed_state_parts,
-        proposed_target_log_prob,
-        proposed_grads_target_log_prob,
-    ] = _leapfrog_integrator(current_momentums,
-                             target_log_prob_fn,
-                             current_state_parts,
-                             step_sizes,
-                             num_leapfrog_steps,
-                             current_target_log_prob,
-                             current_grads_target_log_prob)
-
-    energy_change = _compute_energy_change(current_target_log_prob,
-                                           current_momentums,
-                                           proposed_target_log_prob,
-                                           proposed_momentums,
-                                           independent_chain_ndims)
-    log_accept_ratio = -energy_change
-
-    # u < exp(log_accept_ratio),  where u~Uniform[0,1)
-    # ==> log(u) < log_accept_ratio
-    random_value = random_ops.random_uniform(
-        shape=array_ops.shape(energy_change),
-        dtype=energy_change.dtype,
-        seed=seed)
-    random_negative = math_ops.log(random_value)
-    is_accepted = random_negative < log_accept_ratio
-
-    accepted_target_log_prob = array_ops.where(is_accepted,
-                                               proposed_target_log_prob,
-                                               current_target_log_prob)
-
-    next_state_parts = [_choose(is_accepted,
-                                proposed_state_part,
-                                current_state_part,
-                                independent_chain_ndims)
-                        for current_state_part, proposed_state_part
-                        in zip(current_state_parts, proposed_state_parts)]
-
-    accepted_grads_target_log_prob = [
-        _choose(is_accepted,
-                proposed_grad,
-                grad,
-                independent_chain_ndims)
-        for proposed_grad, grad
-        in zip(proposed_grads_target_log_prob, current_grads_target_log_prob)]
-
-    maybe_flatten = lambda x: x if _is_list_like(current_state) else x[0]
-    return [
-        maybe_flatten(next_state_parts),
-        KernelResults(
-            log_accept_ratio=log_accept_ratio,
-            current_grads_target_log_prob=accepted_grads_target_log_prob,
-            current_target_log_prob=accepted_target_log_prob,
-            is_accepted=is_accepted,
-            proposed_grads_target_log_prob=proposed_grads_target_log_prob,
-            proposed_state=maybe_flatten(proposed_state_parts),
-            proposed_target_log_prob=proposed_target_log_prob,
-        ),
-    ]
-
-
-def _leapfrog_integrator(current_momentums,
-                         target_log_prob_fn,
-                         current_state_parts,
-                         step_sizes,
-                         num_leapfrog_steps,
-                         current_target_log_prob=None,
-                         current_grads_target_log_prob=None,
-                         name=None):
-  """Applies `num_leapfrog_steps` of the leapfrog integrator.
-
-  Assumes a simple quadratic kinetic energy function: `0.5 ||momentum||**2`.
-
-  #### Examples:
-
-  ##### Simple quadratic potential.
-
-  ```python
-  tfd = tf.contrib.distributions
-
-  dims = 10
-  num_iter = int(1e3)
-  dtype = np.float32
-
-  position = tf.placeholder(np.float32)
-  momentum = tf.placeholder(np.float32)
-
-  [
-      next_momentums,
-      next_positions,
-  ] = hmc._leapfrog_integrator(
-      current_momentums=[momentum],
-      target_log_prob_fn=tfd.MultivariateNormalDiag(
-          loc=tf.zeros(dims, dtype)).log_prob,
-      current_state_parts=[position],
-      step_sizes=0.1,
-      num_leapfrog_steps=3)[:2]
-
-  sess.graph.finalize()  # No more graph building.
-
-  momentum_ = np.random.randn(dims).astype(dtype)
-  position_ = np.random.randn(dims).astype(dtype)
-
-  positions = np.zeros([num_iter, dims], dtype)
-  for i in xrange(num_iter):
-    position_, momentum_ = sess.run(
-        [next_momentums[0], next_position[0]],
-        feed_dict={position: position_, momentum: momentum_})
-    positions[i] = position_
-
-  plt.plot(positions[:, 0]);  # Sinusoidal.
-  ```
-
-  Args:
-    current_momentums: Tensor containing the value(s) of the momentum
-      variable(s) to update.
-    target_log_prob_fn: Python callable which takes an argument like
-      `*current_state_parts` and returns its (possibly unnormalized) log-density
-      under the target distribution.
-    current_state_parts: Python `list` of `Tensor`s representing the current
-      state(s) of the Markov chain(s). The first `independent_chain_ndims` of
-      the `Tensor`(s) index different chains.
-    step_sizes: Python `list` of `Tensor`s representing the step size for the
-      leapfrog integrator. Must broadcast with the shape of
-      `current_state_parts`.  Larger step sizes lead to faster progress, but
-      too-large step sizes make rejection exponentially more likely. When
-      possible, it's often helpful to match per-variable step sizes to the
-      standard deviations of the target distribution in each variable.
-    num_leapfrog_steps: Integer number of steps to run the leapfrog integrator
-      for. Total progress per HMC step is roughly proportional to `step_size *
-      num_leapfrog_steps`.
-    current_target_log_prob: (Optional) `Tensor` representing the value of
-      `target_log_prob_fn(*current_state_parts)`. The only reason to specify
-      this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    current_grads_target_log_prob: (Optional) Python list of `Tensor`s
-      representing gradient of `target_log_prob_fn(*current_state_parts`) wrt
-      `current_state_parts`. Must have same shape as `current_state_parts`. The
-      only reason to specify this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    name: Python `str` name prefixed to Ops created by this function.
-      Default value: `None` (i.e., "hmc_leapfrog_integrator").
-
-  Returns:
-    proposed_momentums: Updated value of the momentum.
-    proposed_state_parts: Tensor or Python list of `Tensor`s representing the
-      state(s) of the Markov chain(s) at each result step. Has same shape as
-      input `current_state_parts`.
-    proposed_target_log_prob: `Tensor` representing the value of
-      `target_log_prob_fn` at `next_state`.
-    proposed_grads_target_log_prob: Gradient of `proposed_target_log_prob` wrt
-      `next_state`.
-
-  Raises:
-    ValueError: if `len(momentums) != len(state_parts)`.
-    ValueError: if `len(state_parts) != len(step_sizes)`.
-    ValueError: if `len(state_parts) != len(grads_target_log_prob)`.
-    TypeError: if `not target_log_prob.dtype.is_floating`.
-  """
-  def _loop_body(step,
-                 current_momentums,
-                 current_state_parts,
-                 ignore_current_target_log_prob,  # pylint: disable=unused-argument
-                 current_grads_target_log_prob):
-    return [step + 1] + list(_leapfrog_step(current_momentums,
-                                            target_log_prob_fn,
-                                            current_state_parts,
-                                            step_sizes,
-                                            current_grads_target_log_prob))
-
-  with ops.name_scope(
-      name, "hmc_leapfrog_integrator",
-      [current_momentums, current_state_parts, step_sizes, num_leapfrog_steps,
-       current_target_log_prob, current_grads_target_log_prob]):
-    if len(current_momentums) != len(current_state_parts):
-      raise ValueError("`momentums` must be in one-to-one correspondence "
-                       "with `state_parts`")
-    num_leapfrog_steps = ops.convert_to_tensor(num_leapfrog_steps,
-                                               name="num_leapfrog_steps")
-    current_target_log_prob, current_grads_target_log_prob = (
-        _maybe_call_fn_and_grads(
-            target_log_prob_fn,
-            current_state_parts,
-            current_target_log_prob,
-            current_grads_target_log_prob))
-    return control_flow_ops.while_loop(
-        cond=lambda iter_, *args: iter_ < num_leapfrog_steps,
-        body=_loop_body,
-        loop_vars=[
-            np.int32(0),  # iter_
-            current_momentums,
-            current_state_parts,
-            current_target_log_prob,
-            current_grads_target_log_prob,
-        ],
-        back_prop=False)[1:]  # Lop-off "iter_".
-
-
-def _leapfrog_step(current_momentums,
-                   target_log_prob_fn,
-                   current_state_parts,
-                   step_sizes,
-                   current_grads_target_log_prob,
-                   name=None):
-  """Applies one step of the leapfrog integrator."""
-  with ops.name_scope(
-      name, "_leapfrog_step",
-      [current_momentums, current_state_parts, step_sizes,
-       current_grads_target_log_prob]):
-    proposed_momentums = [m + 0.5 * ss * g for m, ss, g
-                          in zip(current_momentums,
-                                 step_sizes,
-                                 current_grads_target_log_prob)]
-    proposed_state_parts = [x + ss * m for x, ss, m
-                            in zip(current_state_parts,
-                                   step_sizes,
-                                   proposed_momentums)]
-    proposed_target_log_prob = target_log_prob_fn(*proposed_state_parts)
-    if not proposed_target_log_prob.dtype.is_floating:
-      raise TypeError("`target_log_prob_fn` must produce a `Tensor` "
-                      "with `float` `dtype`.")
-    proposed_grads_target_log_prob = gradients_ops.gradients(
-        proposed_target_log_prob, proposed_state_parts)
-    if any(g is None for g in proposed_grads_target_log_prob):
-      raise ValueError(
-          "Encountered `None` gradient. Does your target `target_log_prob_fn` "
-          "access all `tf.Variable`s via `tf.get_variable`?\n"
-          "  current_state_parts: {}\n"
-          "  proposed_state_parts: {}\n"
-          "  proposed_grads_target_log_prob: {}".format(
-              current_state_parts,
-              proposed_state_parts,
-              proposed_grads_target_log_prob))
-    proposed_momentums = [m + 0.5 * ss * g for m, ss, g
-                          in zip(proposed_momentums,
-                                 step_sizes,
-                                 proposed_grads_target_log_prob)]
-    return [
-        proposed_momentums,
-        proposed_state_parts,
-        proposed_target_log_prob,
-        proposed_grads_target_log_prob,
-    ]
-
-
-def _compute_energy_change(current_target_log_prob,
-                           current_momentums,
-                           proposed_target_log_prob,
-                           proposed_momentums,
-                           independent_chain_ndims,
-                           name=None):
-  """Helper to `kernel` which computes the energy change."""
-  with ops.name_scope(
-      name, "compute_energy_change",
-      ([current_target_log_prob, proposed_target_log_prob,
-        independent_chain_ndims] +
-       current_momentums + proposed_momentums)):
-    # Abbreviate lk0=log_kinetic_energy and lk1=proposed_log_kinetic_energy
-    # since they're a mouthful and lets us inline more.
-    lk0, lk1 = [], []
-    for current_momentum, proposed_momentum in zip(current_momentums,
-                                                   proposed_momentums):
-      axis = math_ops.range(independent_chain_ndims,
-                            array_ops.rank(current_momentum))
-      lk0.append(_log_sum_sq(current_momentum, axis))
-      lk1.append(_log_sum_sq(proposed_momentum, axis))
-
-    lk0 = -np.log(2.) + math_ops.reduce_logsumexp(array_ops.stack(lk0, axis=-1),
-                                                  axis=-1)
-    lk1 = -np.log(2.) + math_ops.reduce_logsumexp(array_ops.stack(lk1, axis=-1),
-                                                  axis=-1)
-    lp0 = -current_target_log_prob   # potential
-    lp1 = -proposed_target_log_prob  # proposed_potential
-    x = array_ops.stack([lp1, math_ops.exp(lk1), -lp0, -math_ops.exp(lk0)],
-                        axis=-1)
-
-    # The sum is NaN if any element is NaN or we see both +Inf and -Inf.
-    # Thus we will replace such rows with infinite energy change which implies
-    # rejection. Recall that float-comparisons with NaN are always False.
-    is_sum_determinate = (
-        math_ops.reduce_all(math_ops.is_finite(x) | (x >= 0.), axis=-1) &
-        math_ops.reduce_all(math_ops.is_finite(x) | (x <= 0.), axis=-1))
-    is_sum_determinate = array_ops.tile(
-        is_sum_determinate[..., array_ops.newaxis],
-        multiples=array_ops.concat([
-            array_ops.ones(array_ops.rank(is_sum_determinate),
-                           dtype=dtypes.int32),
-            [4],
-        ], axis=0))
-    x = array_ops.where(is_sum_determinate,
-                        x,
-                        array_ops.fill(array_ops.shape(x),
-                                       value=x.dtype.as_numpy_dtype(np.inf)))
-
-    return math_ops.reduce_sum(x, axis=-1)
-
-
-def _choose(is_accepted,
-            accepted,
-            rejected,
-            independent_chain_ndims,
-            name=None):
-  """Helper to `kernel` which expand_dims `is_accepted` to apply tf.where."""
-  def _expand_is_accepted_like(x):
-    with ops.name_scope("_choose"):
-      expand_shape = array_ops.concat([
-          array_ops.shape(is_accepted),
-          array_ops.ones([array_ops.rank(x) - array_ops.rank(is_accepted)],
-                         dtype=dtypes.int32),
-      ], axis=0)
-      multiples = array_ops.concat([
-          array_ops.ones([array_ops.rank(is_accepted)], dtype=dtypes.int32),
-          array_ops.shape(x)[independent_chain_ndims:],
-      ], axis=0)
-      m = array_ops.tile(array_ops.reshape(is_accepted, expand_shape),
-                         multiples)
-      m.set_shape(x.shape)
-      return m
-  with ops.name_scope(name, "_choose", values=[
-      is_accepted, accepted, rejected, independent_chain_ndims]):
-    return array_ops.where(_expand_is_accepted_like(accepted),
-                           accepted,
-                           rejected)
-
-
-def _maybe_call_fn_and_grads(fn,
-                             fn_arg_list,
-                             fn_result=None,
-                             grads_fn_result=None,
-                             description="target_log_prob"):
-  """Helper which computes `fn_result` and `grads` if needed."""
-  fn_arg_list = (list(fn_arg_list) if _is_list_like(fn_arg_list)
-                 else [fn_arg_list])
-  if fn_result is None:
-    fn_result = fn(*fn_arg_list)
-  if not fn_result.dtype.is_floating:
-    raise TypeError("`{}` must be a `Tensor` with `float` `dtype`.".format(
-        description))
-  if grads_fn_result is None:
-    grads_fn_result = gradients_ops.gradients(
-        fn_result, fn_arg_list)
-  if len(fn_arg_list) != len(grads_fn_result):
-    raise ValueError("`{}` must be in one-to-one correspondence with "
-                     "`grads_{}`".format(*[description]*2))
-  if any(g is None for g in grads_fn_result):
-    raise ValueError("Encountered `None` gradient.")
-  return fn_result, grads_fn_result
-
-
-def _prepare_args(target_log_prob_fn, state, step_size,
-                  target_log_prob=None, grads_target_log_prob=None,
-                  maybe_expand=False, description="target_log_prob"):
-  """Helper which processes input args to meet list-like assumptions."""
-  state_parts = list(state) if _is_list_like(state) else [state]
-  state_parts = [ops.convert_to_tensor(s, name="state")
-                 for s in state_parts]
-  target_log_prob, grads_target_log_prob = _maybe_call_fn_and_grads(
-      target_log_prob_fn,
-      state_parts,
-      target_log_prob,
-      grads_target_log_prob,
-      description)
-  step_sizes = list(step_size) if _is_list_like(step_size) else [step_size]
-  step_sizes = [
-      ops.convert_to_tensor(
-          s, name="step_size", dtype=target_log_prob.dtype)
-      for s in step_sizes]
-  if len(step_sizes) == 1:
-    step_sizes *= len(state_parts)
-  if len(state_parts) != len(step_sizes):
-    raise ValueError("There should be exactly one `step_size` or it should "
-                     "have same length as `current_state`.")
-  maybe_flatten = lambda x: x if maybe_expand or _is_list_like(state) else x[0]
-  return [
-      maybe_flatten(state_parts),
-      maybe_flatten(step_sizes),
-      target_log_prob,
-      grads_target_log_prob,
-  ]
-
-
-def _is_list_like(x):
-  """Helper which returns `True` if input is `list`-like."""
-  return isinstance(x, (tuple, list))
-
-
-def _log_sum_sq(x, axis=None):
-  """Computes log(sum(x**2))."""
-  return math_ops.reduce_logsumexp(2. * math_ops.log(math_ops.abs(x)), axis)
diff --git a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings.py b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings.py
deleted file mode 100644
index e7fcbc65ef..0000000000
--- a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings.py
+++ /dev/null
@@ -1,34 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Functions to create a Markov Chain Monte Carlo Metropolis step."""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# go/tf-wildcard-import
-# pylint: disable=wildcard-import
-from tensorflow.contrib.bayesflow.python.ops.metropolis_hastings_impl import *
-# pylint: enable=wildcard-import
-from tensorflow.python.util.all_util import remove_undocumented
-
-_allowed_symbols = [
-    'kernel',
-    'evolve',
-    'proposal_uniform',
-    'proposal_normal',
-]
-
-remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
deleted file mode 100644
index 05aa134ed5..0000000000
--- a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
+++ /dev/null
@@ -1,527 +0,0 @@
-# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-"""Metropolis-Hastings and proposal distributions.
-
-@@kernel
-@@evolve
-@@proposal_uniform
-@@proposal_normal
-"""
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-import collections
-
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
-from tensorflow.python.ops import control_flow_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import random_ops
-from tensorflow.python.ops import state_ops
-
-__all__ = [
-    "kernel",
-    "evolve",
-    "proposal_uniform",
-    "proposal_normal",
-]
-
-
-KernelResults = collections.namedtuple(
-    "KernelResults",
-    [
-        "log_accept_ratio",
-        "current_target_log_prob",  # "Current result" means "accepted".
-        "is_accepted",
-        "proposed_state",
-    ])
-
-
-def kernel(target_log_prob_fn,
-           proposal_fn,
-           current_state,
-           seed=None,
-           current_target_log_prob=None,
-           name=None):
-  """Runs the Metropolis-Hastings transition kernel.
-
-  This function can update multiple chains in parallel. It assumes that all
-  leftmost dimensions of `current_state` index independent chain states (and are
-  therefore updated independently). The output of `target_log_prob_fn()` should
-  sum log-probabilities across all event dimensions. Slices along the rightmost
-  dimensions may have different target distributions; for example,
-  `current_state[0, :]` could have a different target distribution from
-  `current_state[1, :]`. This is up to `target_log_prob_fn()`. (The number of
-  independent chains is `tf.size(target_log_prob_fn(*current_state))`.)
-
-  Args:
-    target_log_prob_fn: Python callable which takes an argument like
-      `current_state` (or `*current_state` if it's a list) and returns its
-      (possibly unnormalized) log-density under the target distribution.
-    proposal_fn: Python callable which takes an argument like `current_state`
-      (or `*current_state` if it's a list) and returns a tuple of proposed
-      states of same shape as `state`, and a log ratio `Tensor` of same shape
-      as `current_target_log_prob`. The log ratio is the log-probability of
-      `state` given proposed states minus the log-probability of proposed
-      states given `state`. If the proposal is symmetric, set the second value
-      to `None`: this enables more efficient computation than explicitly
-      supplying a tensor of zeros.
-    current_state: `Tensor` or Python `list` of `Tensor`s representing the
-      current state(s) of the Markov chain(s). The first `r` dimensions index
-      independent chains, `r = tf.rank(target_log_prob_fn(*current_state))`.
-    seed: Python integer to seed the random number generator.
-    current_target_log_prob: (Optional) `Tensor` representing the value of
-      `target_log_prob_fn` at the `current_state`. The only reason to
-      specify this argument is to reduce TF graph size.
-      Default value: `None` (i.e., compute as needed).
-    name: A name of the operation (optional).
-
-  Returns:
-    next_state: Tensor or Python list of `Tensor`s representing the state(s)
-      of the Markov chain(s) at each result step. Has same shape as
-      `current_state`.
-    kernel_results: `collections.namedtuple` of internal calculations used to
-      advance the chain.
-
-  #### Examples
-
-  We illustrate Metropolis-Hastings on a Normal likelihood with
-  unknown mean.
-
-  ```python
-  tfd = tf.contrib.distributions
-  tfp = tf.contrib.bayesflow
-
-  loc = tf.get_variable("loc", initializer=1.)
-  x = tf.constant([0.0] * 50)
-
-  def make_target_log_prob_fn(x):
-    def target_log_prob_fn(loc):
-      prior = tfd.Normal(loc=0., scale=1.)
-      likelihood = tfd.Independent(
-        tfd.Normal(loc=loc, scale=0.1),
-        reinterpreted_batch_ndims=1)
-      return prior.log_prob(loc) + likelihood.log_prob(x)
-    return target_log_prob_fn
-
-  next_state, kernel_results = tfp.metropolis_hastings.kernel(
-      target_log_prob_fn=make_target_log_prob_fn(x),
-      proposal_fn=tfp.metropolis_hastings.proposal_normal(),
-      current_state=loc)
-  loc_update = loc.assign(next_state)
-  ```
-
-  We illustrate Metropolis-Hastings on a Normal likelihood with
-  unknown mean and variance. We apply 4 chains.
-
-  ```python
-  tfd = tf.contrib.distributions
-  tfp = tf.contrib.bayesflow
-
-  num_chains = 4
-  loc = tf.get_variable("loc", shape=[num_chains],
-                        initializer=tf.random_normal_initializer())
-  scale = tf.get_variable("scale", shape=[num_chains],
-                          initializer=tf.ones_initializer())
-  x = tf.constant([0.0] * 50)
-
-  def make_target_log_prob_fn(x):
-    data = tf.reshape(x, shape=[-1, 1])
-    def target_log_prob_fn(loc, scale):
-      prior_loc = tfd.Normal(loc=0., scale=1.)
-      prior_scale = tfd.InverseGamma(concentration=1., rate=1.)
-      likelihood = tfd.Independent(
-        tfd.Normal(loc=loc, scale=scale),
-        reinterpreted_batch_ndims=1)
-      return (prior_loc.log_prob(loc) +
-              prior_scale.log_prob(scale) +
-              likelihood.log_prob(data))
-    return target_log_prob_fn
-
-  def proposal_fn(loc, scale):
-    loc_proposal = tfp.metropolis_hastings.proposal_normal()
-    scale_proposal = tfp.metropolis_hastings.proposal_uniform(minval=-1.)
-    proposed_loc, _ = loc_proposal(loc)
-    proposed_scale, _ = scale_proposal(scale)
-    proposed_scale = tf.maximum(proposed_scale, 0.01)
-    return [proposed_loc, proposed_scale], None
-
-  next_state, kernel_results = tfp.metropolis_hastings.kernel(
-      target_log_prob_fn=make_target_log_prob_fn(x),
-      proposal_fn=proposal_fn,
-      current_state=[loc, scale])
-  train_op = tf.group(loc.assign(next_state[0]),
-                      scale.assign(next_state[1]))
-  ```
-
-  """
-  with ops.name_scope(
-      name, "metropolis_hastings_kernel",
-      [current_state, seed, current_target_log_prob]):
-    with ops.name_scope("initialize"):
-      maybe_expand = lambda x: list(x) if _is_list_like(x) else [x]
-      current_state_parts = maybe_expand(current_state)
-      if current_target_log_prob is None:
-        current_target_log_prob = target_log_prob_fn(*current_state_parts)
-
-    proposed_state, log_transit_ratio = proposal_fn(*current_state_parts)
-    proposed_state_parts = maybe_expand(proposed_state)
-
-    proposed_target_log_prob = target_log_prob_fn(*proposed_state_parts)
-
-    with ops.name_scope(
-        "accept_reject",
-        [current_state_parts, proposed_state_parts,
-         current_target_log_prob, proposed_target_log_prob]):
-      log_accept_ratio = proposed_target_log_prob - current_target_log_prob
-      if log_transit_ratio is not None:
-        # If the log_transit_ratio is None, then assume the proposal is
-        # symmetric, i.e.,
-        #   log p(old | new) - log p(new | old) = 0.
-        log_accept_ratio += log_transit_ratio
-
-      # u < exp(log_accept_ratio),  where u~Uniform[0,1)
-      # ==> log(u) < log_accept_ratio
-      random_value = random_ops.random_uniform(
-          array_ops.shape(log_accept_ratio),
-          dtype=log_accept_ratio.dtype,
-          seed=seed)
-      random_negative = math_ops.log(random_value)
-      is_accepted = random_negative < log_accept_ratio
-      next_state_parts = [array_ops.where(is_accepted,
-                                          proposed_state_part,
-                                          current_state_part)
-                          for proposed_state_part, current_state_part in
-                          zip(proposed_state_parts, current_state_parts)]
-      accepted_log_prob = array_ops.where(is_accepted,
-                                          proposed_target_log_prob,
-                                          current_target_log_prob)
-    maybe_flatten = lambda x: x if _is_list_like(current_state) else x[0]
-    return [
-        maybe_flatten(next_state_parts),
-        KernelResults(
-            log_accept_ratio=log_accept_ratio,
-            current_target_log_prob=accepted_log_prob,
-            is_accepted=is_accepted,
-            proposed_state=maybe_flatten(proposed_state_parts),
-        ),
-    ]
-
-
-def evolve(initial_sample,
-           initial_log_density,
-           initial_log_accept_ratio,
-           target_log_prob_fn,
-           proposal_fn,
-           n_steps=1,
-           seed=None,
-           name=None):
-  """Performs `n_steps` of the Metropolis-Hastings update.
-
-  Given a probability density function, `f(x)` and a proposal scheme which
-  generates new points from old, this `Op` returns a tensor
-  which may be used to generate approximate samples from the target distribution
-  using the Metropolis-Hastings algorithm. These samples are from a Markov chain
-  whose equilibrium distribution matches the target distribution.
-
-  The probability distribution may have an unknown normalization constan.
-  We parameterize the probability density as follows:
-
-  ```none
-  f(x) = exp(L(x) + constant)
-  ```
-
-  Here `L(x)` is any continuous function with an (possibly unknown but finite)
-  upper bound, i.e. there exists a number beta such that
-  `L(x)< beta < infinity` for all x. The constant is the normalization needed
-  to make `f(x)` a probability density (as opposed to just a finite measure).
-
-  Although `initial_sample` can be arbitrary, a poor choice may result in a
-  slow-to-mix chain. In many cases the best choice is the one that maximizes
-  the target density, i.e., choose `initial_sample` such that
-  `f(initial_sample) >= f(x)` for all `x`.
-
-
-  If the support of the distribution is a strict subset of R^n (but of non zero
-  measure), then the unnormalized log-density `L(x)` should return `-infinity`
-  outside the support domain. This effectively forces the sampler to only
-  explore points in the regions of finite support.
-
-  Usage:
-  This function is meant to be wrapped up with some of the common proposal
-  schemes (e.g. random walk, Langevin diffusion etc) to produce a more user
-  friendly interface. However, it may also be used to create bespoke samplers.
-
-  The following example, demonstrates the use to generate a 1000 uniform random
-  walk Metropolis samplers run in parallel for the normal target distribution.
-
-  ```python
-  n = 3  # dimension of the problem
-
-  # Generate 1000 initial values randomly. Each of these would be an
-  # independent starting point for a Markov chain.
-  state = tf.get_variable(
-      "state",
-      initializer=tf.random_normal([1000, n],
-                                   mean=3.0,
-                                   dtype=tf.float64,
-                                   seed=42))
-
-  # Computes the log(p(x)) for the unit normal density and ignores the
-  # normalization constant.
-  def log_density(x):
-    return -tf.reduce_sum(x * x, reduction_indices=-1) / 2.0
-
-  # Initial log-density value
-  state_log_density = tf.get_variable(
-      "state_log_density",
-      initializer=log_density(state.initialized_value()))
-
-  # A variable to store the log_acceptance_ratio:
-  log_acceptance_ratio = tf.get_variable(
-      "log_acceptance_ratio",
-      initializer=tf.zeros([1000], dtype=tf.float64))
-
-  # Generates random proposals by moving each coordinate uniformly and
-  # independently in a box of size 2 centered around the current value.
-  # Returns the new point and also the log of the Hastings ratio (the
-  # ratio of the probability of going from the proposal to origin and the
-  # probability of the reverse transition). When this ratio is 1, the value
-  # may be omitted and replaced by None.
-  def random_proposal(x):
-    return (x + tf.random_uniform(tf.shape(x), minval=-1, maxval=1,
-                                  dtype=x.dtype, seed=12)), None
-
-  #  Create the op to propagate the chain for 100 steps.
-  stepper = mh.evolve(
-      state, state_log_density, log_acceptance_ratio,
-      log_density, random_proposal, n_steps=100, seed=123)
-  init = tf.initialize_all_variables()
-  with tf.Session() as sess:
-    sess.run(init)
-    # Run the chains for a total of 1000 steps and print out the mean across
-    # the chains every 100 iterations.
-    for n_iter in range(10):
-      # Executing the stepper advances the chain to the next state.
-      sess.run(stepper)
-      # Print out the current value of the mean(sample) for every dimension.
-      print(np.mean(sess.run(state), 0))
-    # Estimated covariance matrix
-    samples = sess.run(state)
-    print(np.cov(samples, rowvar=False))
-  ```
-
-  Args:
-    initial_sample: A float-like `tf.Variable` of any shape that can
-      be consumed by the `target_log_prob_fn` and `proposal_fn`
-      callables.
-    initial_log_density: Float-like `tf.Variable` with `dtype` and shape
-      equivalent  to `target_log_prob_fn(initial_sample)`, i.e., matching
-        the result of `target_log_prob_fn` invoked at `current_state`.
-    initial_log_accept_ratio: A `tf.Variable` with `dtype` and shape matching
-      `initial_log_density`. Stands for the log of Metropolis-Hastings
-      acceptance ratio after propagating the chain for `n_steps`.
-    target_log_prob_fn: A Python callable evaluated at
-      `current_state` and returning a float-like `Tensor` of log target-density
-      up to a normalizing constant. In other words,
-      `target_log_prob_fn(x) = log(g(x))`, where
-      `target_density = g(x)/Z` for some constant `A`. The shape of the input
-      tensor is the same as the shape of the `current_state`. The shape of the
-      output tensor is either
-        (a). Same as the input shape if the density being sampled is one
-          dimensional, or
-        (b). If the density is defined for `events` of shape
-          `event_shape = [E1, E2, ... Ee]`, then the input tensor should be of
-          shape `batch_shape + event_shape`, here `batch_shape = [B1, ..., Bb]`
-          and the result must be of shape [B1, ..., Bb]. For example, if the
-          distribution that is being sampled is a 10 dimensional normal,
-          then the input tensor may be of shape [100, 10] or [30, 20, 10]. The
-          last dimension will then be 'consumed' by `target_log_prob_fn`
-          and it should return tensors of shape [100] and [30, 20] respectively.
-    proposal_fn: A callable accepting a real valued `Tensor` of current sample
-      points and returning a tuple of two `Tensors`. The first element of the
-      pair should be a `Tensor` containing the proposal state and should have
-      the same shape as the input `Tensor`. The second element of the pair gives
-      the log of the ratio of the probability of transitioning from the
-      proposal points to the input points and the probability of transitioning
-      from the input points to the proposal points. If the proposal is
-      symmetric, i.e.
-      Probability(Proposal -> Current) = Probability(Current -> Proposal)
-      the second value should be set to None instead of explicitly supplying a
-      tensor of zeros. In addition to being convenient, this also leads to a
-      more efficient graph.
-    n_steps: A positive `int` or a scalar `int32` tensor. Sets the number of
-      iterations of the chain.
-    seed: `int` or None. The random seed for this `Op`. If `None`, no seed is
-      applied.
-    name: A string that sets the name for this `Op`.
-
-  Returns:
-    forward_step: an `Op` to step the Markov chain forward for `n_steps`.
-  """
-
-  with ops.name_scope(name, "metropolis_hastings", [initial_sample]):
-    current_state = initial_sample
-    current_target_log_prob = initial_log_density
-    log_accept_ratio = initial_log_accept_ratio
-
-    def step(i, current_state, current_target_log_prob, log_accept_ratio):
-      """Wrap single Markov chain iteration in `while_loop`."""
-      next_state, kernel_results = kernel(
-          target_log_prob_fn=target_log_prob_fn,
-          proposal_fn=proposal_fn,
-          current_state=current_state,
-          current_target_log_prob=current_target_log_prob,
-          seed=seed)
-      accepted_log_prob = kernel_results.current_target_log_prob
-      log_accept_ratio = kernel_results.log_accept_ratio
-      return i + 1, next_state, accepted_log_prob, log_accept_ratio
-
-    (_, accepted_state, accepted_target_log_prob, accepted_log_accept_ratio) = (
-        control_flow_ops.while_loop(
-            cond=lambda i, *ignored_args: i < n_steps,
-            body=step,
-            loop_vars=[
-                0,  # i
-                current_state,
-                current_target_log_prob,
-                log_accept_ratio,
-            ],
-            parallel_iterations=1 if seed is not None else 10,
-            # TODO(b/73775595): Confirm optimal setting of swap_memory.
-            swap_memory=1))
-
-    forward_step = control_flow_ops.group(
-        state_ops.assign(current_target_log_prob, accepted_target_log_prob),
-        state_ops.assign(current_state, accepted_state),
-        state_ops.assign(log_accept_ratio, accepted_log_accept_ratio))
-
-    return forward_step
-
-
-def proposal_uniform(step_size=1.,
-                     seed=None,
-                     name=None):
-  """Returns a callable that adds a random uniform tensor to the input.
-
-  This function returns a callable that accepts one `Tensor` argument of any
-  shape and a real data type (i.e. `tf.float32` or `tf.float64`). It adds a
-  sample from a random uniform distribution drawn from [-stepsize, stepsize]
-  to its input. It also returns the log of the ratio of the probability of
-  moving from the input point to the proposed point, but since this log ratio is
-  identically equal to 0 (because the probability of drawing a value `x` from
-  the symmetric uniform distribution is the same as the probability of drawing
-  `-x`), it simply returns None for the second element of the returned tuple.
-
-  Args:
-    step_size: A positive `float` or a scalar tensor of real dtype
-      controlling the scale of the uniform distribution.
-      If step_size = a, then draws are made uniformly from [-a, a].
-    seed: `int` or None. The random seed for this `Op`. If `None`, no seed is
-      applied.
-    name: A string that sets the name for this `Op`.
-
-  Returns:
-    proposal_fn:  A callable accepting one float-like `Tensor` and returning a
-      2-tuple. The first value in the tuple is a `Tensor` of the same shape and
-      dtype as the input argument and the second element of the tuple is None.
-  """
-
-  with ops.name_scope(name, "proposal_uniform", [step_size]):
-    step_size = ops.convert_to_tensor(step_size, name="step_size")
-
-    def proposal_fn(input_state, name=None):
-      """Adds a uniform perturbation to the input state.
-
-      Args:
-        input_state: A `Tensor` of any shape and real dtype.
-        name: A string that sets the name for this `Op`.
-
-      Returns:
-        proposal_state:  A float-like `Tensor` with `dtype` and shape matching
-          `input_state`.
-        log_transit_ratio: `None`. Proposal is symmetric.
-      """
-      with ops.name_scope(name, "proposer", [input_state]):
-        input_state = ops.convert_to_tensor(input_state, name="input_state")
-        return input_state + random_ops.random_uniform(
-            array_ops.shape(input_state),
-            minval=-step_size,
-            maxval=step_size,
-            seed=seed), None
-    return proposal_fn
-
-
-def proposal_normal(scale=1.,
-                    seed=None,
-                    name=None):
-  """Returns a callable that adds a random normal tensor to the input.
-
-  This function returns a callable that accepts one `Tensor` argument of any
-  shape and a real data type (i.e. `tf.float32` or `tf.float64`). The callable
-  adds a sample from a normal distribution with the supplied standard deviation
-  and zero mean to its input argument (called the proposal point).
-  The callable returns a tuple with the proposal point as the first element.
-  The second element is identically `None`. It is included so the callable is
-  compatible with the expected signature of the proposal scheme argument in the
-  `metropolis_hastings` function. A value of `None` indicates that the
-  probability of going from the input point to the proposal point is equal to
-  the probability of going from the proposal point to the input point.
-
-  Args:
-    scale: A positive `float` or a scalar tensor of any real dtype controlling
-      the scale of the normal distribution.
-    seed: `int` or None. The random seed for this `Op`. If `None`, no seed is
-      applied.
-    name: A string that sets the name for this `Op`.
-
-  Returns:
-    proposal_fn: A callable accepting one float-like `Tensor` and returning a
-      2-tuple. The first value in the tuple is a `Tensor` of the same shape and
-      dtype as the input argument and the second element of the tuple is None.
-  """
-
-  with ops.name_scope(name, "proposal_normal", [scale]):
-    scale = ops.convert_to_tensor(scale, name="scale")
-
-    def proposal_fn(input_state, name=None):
-      """Adds a normal perturbation to the input state.
-
-      Args:
-        input_state: A `Tensor` of any shape and real dtype.
-        name: A string that sets the name for this `Op`.
-
-      Returns:
-        proposal_state:  A float-like `Tensor` with `dtype` and shape matching
-          `input_state`.
-        log_transit_ratio: `None`. Proposal is symmetric.
-      """
-
-      with ops.name_scope(name, "proposer", [input_state]):
-        input_state = ops.convert_to_tensor(input_state, name="input_state")
-        return input_state + random_ops.random_normal(
-            array_ops.shape(input_state),
-            mean=0.,
-            stddev=scale,
-            dtype=scale.dtype,
-            seed=seed), None
-    return proposal_fn
-
-
-def _is_list_like(x):
-  """Helper which returns `True` if input is `list`-like."""
-  return isinstance(x, (tuple, list))
-- 
GitLab


From 56d1cfde15c04ebe27fe31409a724a56e7051b15 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 19:08:15 -0700
Subject: [PATCH 522/960] [XLA] Redesign: implement and test ternary ops.

PiperOrigin-RevId: 190561679
---
 .../xla/client/xla_client/xla_builder.cc      |  42 +++-
 .../xla/client/xla_client/xla_builder.h       |   4 +
 .../compiler/xla/service/shape_inference.cc   |   8 +-
 .../compiler/xla/service/shape_inference.h    |   3 +
 .../xla/tests/array_elementwise_ops_test.cc   | 205 +++++++++---------
 .../xla/tests/client_library_test_base.cc     |  21 +-
 .../xla/tests/client_library_test_base.h      |   2 +
 7 files changed, 175 insertions(+), 110 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 1b90b45bfb..fcaf393b6b 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -288,6 +288,44 @@ XlaOp XlaBuilder::BinaryOp(
   }());
 }
 
+XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
+                            const XlaOp& ehs) {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, lhs.GetShape());
+    TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, rhs.GetShape());
+    TF_ASSIGN_OR_RETURN(const Shape& ehs_shape, ehs.GetShape());
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferTernaryOpShape(
+                            triop, lhs_shape, rhs_shape, ehs_shape));
+    XlaOp updated_lhs = lhs;
+    XlaOp updated_rhs = rhs;
+    XlaOp updated_ehs = ehs;
+    if (!ShapeUtil::IsTuple(instr.shape())) {
+      if (!ShapeUtil::IsTuple(lhs_shape) &&
+          !ShapeUtil::SameDimensions(instr.shape(), lhs_shape)) {
+        // lhs is being implicitly broadcasted. Change to explicit.
+        TF_ASSIGN_OR_RETURN(updated_lhs,
+                            AddBroadcastSequence(instr.shape(), lhs));
+      }
+      if (!ShapeUtil::IsTuple(rhs_shape) &&
+          !ShapeUtil::SameDimensions(instr.shape(), rhs_shape)) {
+        // rhs is being implicitly broadcasted. Change to explicit.
+        TF_ASSIGN_OR_RETURN(updated_rhs,
+                            AddBroadcastSequence(instr.shape(), rhs));
+      }
+      if (!ShapeUtil::IsTuple(ehs_shape) &&
+          !ShapeUtil::SameDimensions(instr.shape(), ehs_shape)) {
+        // ehs is being implicitly broadcasted. Change to explicit.
+        TF_ASSIGN_OR_RETURN(updated_ehs,
+                            AddBroadcastSequence(instr.shape(), ehs));
+      }
+    }
+    return AddInstruction(std::move(instr), triop,
+                          {updated_lhs, updated_rhs, updated_ehs});
+  }());
+}
+
 XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
                       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
   return BinaryOp(HloOpcode::kAdd, lhs, rhs, broadcast_dimensions);
@@ -449,7 +487,7 @@ void XlaBuilder::Trace(const string& tag, const XlaOp& operand) {
 
 XlaOp XlaBuilder::Select(const XlaOp& pred, const XlaOp& on_true,
                          const XlaOp& on_false) {
-  return UnimplementedOp();
+  return TernaryOp(HloOpcode::kSelect, pred, on_true, on_false);
 }
 
 XlaOp XlaBuilder::Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements) {
@@ -755,7 +793,7 @@ XlaOp XlaBuilder::Neg(const XlaOp& operand) {
 
 XlaOp XlaBuilder::Clamp(const XlaOp& min, const XlaOp& operand,
                         const XlaOp& max) {
-  return UnimplementedOp();
+  return TernaryOp(HloOpcode::kClamp, min, operand, max);
 }
 
 XlaOp XlaBuilder::Map(tensorflow::gtl::ArraySlice<XlaOp> operands,
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index cc33356cc1..c5c35159e0 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -762,6 +762,10 @@ class XlaBuilder {
   XlaOp BinaryOp(HloOpcode binop, const XlaOp& lhs, const XlaOp& rhs,
                  tensorflow::gtl::ArraySlice<int64> broadcast_dimensions);
 
+  // Internal helper method that does the building for an arbitrary ternary op.
+  XlaOp TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
+                  const XlaOp& ehs);
+
   StatusOr<XlaOp> InDimBroadcast(
       const Shape& shape, const XlaOp& operand,
       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions);
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 2a70ea0354..36456d552d 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1038,8 +1038,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
 /* static */ StatusOr<Shape> ShapeInference::InferTernaryOpShape(
     HloOpcode opcode, const HloInstruction* lhs, const HloInstruction* rhs,
     const HloInstruction* ehs) {
-  return InferTernaryOpShape(OpcodeToTernaryOperation(opcode), lhs->shape(),
-                             rhs->shape(), ehs->shape());
+  return InferTernaryOpShape(opcode, lhs->shape(), rhs->shape(), ehs->shape());
+}
+
+/* static */ StatusOr<Shape> ShapeInference::InferTernaryOpShape(
+    HloOpcode opcode, const Shape& lhs, const Shape& rhs, const Shape& ehs) {
+  return InferTernaryOpShape(OpcodeToTernaryOperation(opcode), lhs, rhs, ehs);
 }
 
 /* static */ StatusOr<Shape> ShapeInference::InferTernaryOpShape(
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index b6552a34ae..88830e6d25 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -70,6 +70,9 @@ class ShapeInference {
   static StatusOr<Shape> InferTernaryOpShape(TernaryOperation operation,
                                              const Shape& lhs, const Shape& rhs,
                                              const Shape& ehs);
+  static StatusOr<Shape> InferTernaryOpShape(HloOpcode opcode, const Shape& lhs,
+                                             const Shape& rhs,
+                                             const Shape& ehs);
   static StatusOr<Shape> InferTernaryOpShape(HloOpcode opcode,
                                              const HloInstruction* lhs,
                                              const HloInstruction* rhs,
diff --git a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
index fa7ac3ca9b..03c91745b9 100644
--- a/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/array_elementwise_ops_test.cc
@@ -244,7 +244,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoConstantU64s) {
   std::unique_ptr<GlobalData> rhs_data =
       client_->TransferToServer(*rhs_literal).ConsumeValueOrDie();
 
-  auto add = b.Add(lhs_param, rhs_param);
+  b.Add(lhs_param, rhs_param);
 
   std::vector<uint64> expected(lhs.size());
   for (int64 i = 0; i < lhs.size(); ++i) {
@@ -1914,101 +1914,98 @@ XLA_TEST_F(ArrayElementwiseOpTest, RemTwoConstantS32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NonNanClampF32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto minimum = builder.ConstantR1<float>({1.0f, -6.5f, 1.0f, 2.25f, 0.0f});
   auto argument = builder.ConstantR1<float>({2.0f, 10.0f, -5.0f, 1.0f, 10.0f});
   auto maximum = builder.ConstantR1<float>({3.0f, 0.5f, 25.5f, 5.0f, 123.0});
-  auto clamp = builder.Clamp(minimum, argument, maximum);
+  builder.Clamp(minimum, argument, maximum);
 
   ComputeAndCompareR1<float>(&builder, {2.0f, 0.5f, 1.0f, 2.25f, 10.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampF32Scalar) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto minimum = builder.ConstantR0<float>(0.0f);
   auto argument = builder.ConstantR1<float>({2.0f, 10.0f, -5.0f, 1.0f, 4.0f});
   auto maximum = builder.ConstantR0<float>(5.0f);
-  auto clamp = builder.Clamp(minimum, argument, maximum);
+  builder.Clamp(minimum, argument, maximum);
 
   ComputeAndCompareR1<float>(&builder, {2.0f, 5.0f, 0.0f, 1.0f, 4.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampF32ScalarVector) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto min_scalar = builder.ConstantR0<float>(0.0f);
   auto min_vector = builder.ConstantR1<float>({1.0f, -6.5f, 1.0f, 2.25f, 0.0f});
   auto arg_vector = builder.ConstantR1<float>({2.0f, 10.0f, -5.0f, 1.0f, 4.0f});
   auto max_scalar = builder.ConstantR0<float>(3.0f);
   auto max_vector = builder.ConstantR1<float>({3.0f, 0.5f, 25.5f, 5.0f, 123.0});
   // Perform clamp with broadcasted scalar and vector.
-  auto clamp = builder.Add(
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
-                  builder.Clamp(min_scalar, arg_vector, max_vector)),
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
-                  builder.Clamp(min_scalar, arg_vector, max_scalar)));
+  builder.Add(builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
+                          builder.Clamp(min_scalar, arg_vector, max_vector)),
+              builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
+                          builder.Clamp(min_scalar, arg_vector, max_scalar)));
 
   ComputeAndCompareR1<float>(&builder, {8.0f, 7.0f, 2.0f, 6.5f, 14.0f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampS32Vector) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto min_vector = builder.ConstantR1<int32>({1, -6, 1, 2, 0, -5});
   auto arg_vector = builder.ConstantR1<int32>({2, 10, -5, 1, 4, 10});
   auto max_vector = builder.ConstantR1<int32>({3, 0, 25, 5, 123, -1});
-  auto clamp = builder.Clamp(min_vector, arg_vector, max_vector);
+  builder.Clamp(min_vector, arg_vector, max_vector);
 
   ComputeAndCompareR1<int32>(&builder, {2, 0, 1, 2, 4, -1}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampS32ScalarVector) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto min_scalar = builder.ConstantR0<int32>(0);
   auto min_vector = builder.ConstantR1<int32>({1, -6, 1, 2, 0});
   auto arg_vector = builder.ConstantR1<int32>({2, 10, -5, 1, 4});
   auto max_scalar = builder.ConstantR0<int32>(3);
   auto max_vector = builder.ConstantR1<int32>({3, 1, 25, 5, 123});
   // Perform clamp with broadcasted scalar and vector.
-  auto clamp = builder.Add(
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
-                  builder.Clamp(min_scalar, arg_vector, max_vector)),
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
-                  builder.Clamp(min_scalar, arg_vector, max_scalar)));
+  builder.Add(builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
+                          builder.Clamp(min_scalar, arg_vector, max_vector)),
+              builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
+                          builder.Clamp(min_scalar, arg_vector, max_scalar)));
 
   ComputeAndCompareR1<int32>(&builder, {8, 8, 2, 6, 14}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampU32Vector) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto min_vector = builder.ConstantR1<uint32>({1, 2, 1, 2, 0, ~0u - 4});
   auto arg_vector = builder.ConstantR1<uint32>({2, 10, 5, 1, 4, 10});
   auto max_vector = builder.ConstantR1<uint32>({3, 5, 25, 5, 123, ~0u});
-  auto clamp = builder.Clamp(min_vector, arg_vector, max_vector);
+  builder.Clamp(min_vector, arg_vector, max_vector);
 
   ComputeAndCompareR1<uint32>(&builder, {2, 5, 5, 2, 4, ~0u - 4}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, ClampU32ScalarVector) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto min_scalar = builder.ConstantR0<uint32>(0);
   auto min_vector = builder.ConstantR1<uint32>({1, 0, 1, 2, 0});
   auto arg_vector = builder.ConstantR1<uint32>({2, 10, 0, 1, 4});
   auto max_scalar = builder.ConstantR0<uint32>(3);
   auto max_vector = builder.ConstantR1<uint32>({3, 1, 25, 5, 123});
   // Perform clamp with broadcasted scalar and vector.
-  auto clamp = builder.Add(
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
-                  builder.Clamp(min_scalar, arg_vector, max_vector)),
-      builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
-                  builder.Clamp(min_scalar, arg_vector, max_scalar)));
+  builder.Add(builder.Add(builder.Clamp(min_vector, arg_vector, max_scalar),
+                          builder.Clamp(min_scalar, arg_vector, max_vector)),
+              builder.Add(builder.Clamp(min_vector, arg_vector, max_vector),
+                          builder.Clamp(min_scalar, arg_vector, max_scalar)));
 
   ComputeAndCompareR1<uint32>(&builder, {8, 8, 2, 6, 14}, {});
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Literal> param0_literal =
       Literal::CreateR1<float>({1.1f, 2.2f, 3.3f, 5.5f});
@@ -2022,7 +2019,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersF32s) {
 
   auto p0 = builder.Parameter(0, param0_literal->shape(), "param0");
   auto p1 = builder.Parameter(1, param1_literal->shape(), "param1");
-  auto add = builder.Add(p0, p1);
+  builder.Add(p0, p1);
 
   ComputeAndCompareR1<float>(&builder, {8.3f, 4.5f, 6.7f, 11.1f},
                              {param0_data.get(), param1_data.get()},
@@ -2030,7 +2027,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersZeroElementF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Literal> param0_literal =
       Literal::CreateR3FromArray3D<float>(Array3D<float>(0, 7, 0));
@@ -2044,7 +2041,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersZeroElementF32s) {
 
   auto p0 = builder.Parameter(0, param0_literal->shape(), "param0");
   auto p1 = builder.Parameter(1, param1_literal->shape(), "param1");
-  auto add = builder.Add(p0, p1);
+  builder.Add(p0, p1);
 
   Array3D<float> expected(0, 7, 0);
   ComputeAndCompareR3<float>(
@@ -2052,7 +2049,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddTwoParametersZeroElementF32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, AddParameterToConstantF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Literal> param0_literal =
       Literal::CreateR1<float>({1.1f, 2.2f, 3.3f, 5.5f});
@@ -2061,35 +2058,35 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddParameterToConstantF32s) {
 
   auto a = builder.ConstantR1<float>({1.1f, 2.2f, 3.3f, 4.4f});
   auto p = builder.Parameter(0, param0_literal->shape(), "param0");
-  auto add = builder.Add(a, p);
+  builder.Add(a, p);
 
   ComputeAndCompareR1<float>(&builder, {2.2f, 4.4f, 6.6f, 9.9f},
                              {param0_data.get()}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, CosF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({3.14159f, 0.0f, 1.570796f, -0.78539f});
-  auto result = builder.Cos(a);
+  builder.Cos(a);
 
   ComputeAndCompareR1<float>(&builder, {-1.0f, 1.0f, 0.0f, 0.707107f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, SinF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({3.14159f, 0.0f, 1.570796f, -0.78539f});
-  auto result = builder.Sin(a);
+  builder.Sin(a);
 
   ComputeAndCompareR1<float>(&builder, {0.0f, 0.0f, 1.0f, -0.707107f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Atan2F32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({0.0f, 5.0f, 0.0f, -3.0f, 2.0f, -8.0f});
   auto b = builder.ConstantR1<float>({6.0f, 0.0f, -4.0f, 0.0f, 2.0f, 8.0f});
-  auto atan = builder.Atan2(a, b);
+  builder.Atan2(a, b);
 
   ComputeAndCompareR1<float>(
       &builder,
@@ -2098,9 +2095,9 @@ XLA_TEST_F(ArrayElementwiseOpTest, Atan2F32s) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, TanhF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({-2.5f, 3.14f, 2.25f});
-  auto result = builder.Tanh(a);
+  builder.Tanh(a);
 
   ComputeAndCompareR1<float>(&builder, {-0.986614f, 0.996260f, 0.978026}, {},
                              error_spec_);
@@ -2110,7 +2107,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, TanhF32sVector) {
   // This is like the test ArrayElementwiseOpTest.TanhF32s above, except that
   // the input tensor is large enough to exercise the vectorized tanh
   // implementation on XLA CPU.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto input_literal = Literal::CreateR1<float>(
       {1.02,  -0.32, 0.85,  0.90,  1.23,  -0.91, -0.49, 0.80,  -0.67, 0.16,
        -0.07, 0.39,  -0.41, 0.04,  1.36,  1.25,  0.41,  0.65,  -1.08, 0.32,
@@ -2149,7 +2146,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, TanhF32sVector) {
 XLA_TEST_F(ArrayElementwiseOpTest, ExpF32sVector) {
   // The input tensor is large enough to exercise the vectorized exp
   // implementation on XLA CPU.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   // Just to help make sense of the scales here -- exp(89) saturates float32 and
   // exp(-10) is smaller than our error spec.
@@ -2185,7 +2182,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, ExpF32sVector) {
 XLA_TEST_F(ArrayElementwiseOpTest, LogF32sVector) {
   // The input tensor is large enough to exercise the vectorized exp
   // implementation on XLA CPU.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Literal> input_literal = Literal::CreateR1<float>(
       {-1.29,    -1.41,    -1.25,    -13.5,    -11.7,    -17.9,    -198,
@@ -2225,14 +2222,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddChainFoldLeft) {
   //         /               /
   // b -----/               /
   // c---------------------/
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   auto a = builder.ConstantR1<float>({1.1f, 2.2f, 3.3f, 4.4f});
   auto b = builder.ConstantR1<float>({2.1f, 3.2f, 4.3f, 5.4f});
   auto c = builder.ConstantR1<float>({-3.3f, -15.5f, -7.7f, -29.9f});
 
   auto add = builder.Add(a, b);
-  auto add2 = builder.Add(add, c);
+  builder.Add(add, c);
 
   ComputeAndCompareR1<float>(&builder, {-0.1f, -10.1f, -0.1f, -20.1f}, {},
                              error_spec_);
@@ -2243,14 +2240,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddChainFoldRight) {
   //         /               /
   // c -----/               /
   // a---------------------/
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   auto a = builder.ConstantR1<float>({91.1f, 2.2f, 3.3f, 4.4f});
   auto b = builder.ConstantR1<float>({2.1f, 3.2f, 4.3f, 5.4f});
   auto c = builder.ConstantR1<float>({-3.3f, -15.5f, -7.7f, -29.9f});
 
   auto add = builder.Add(b, c);
-  auto add2 = builder.Add(a, add);
+  builder.Add(a, add);
 
   ComputeAndCompareR1<float>(&builder, {89.9f, -10.1f, -0.1f, -20.1f}, {},
                              error_spec_);
@@ -2260,14 +2257,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddWithNeg) {
   // a ----- (neg) ----- (add)
   //                    /
   // b ----- (neg) ----/
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   auto a = builder.ConstantR1<float>({91.1f, 2.2f, 3.3f, 4.4f});
   auto b = builder.ConstantR1<float>({2.1f, 3.2f, 4.3f, 5.4f});
 
   auto neg_a = builder.Neg(a);
   auto neg_b = builder.Neg(b);
-  auto result = builder.Add(neg_a, neg_b);
+  builder.Add(neg_a, neg_b);
 
   ComputeAndCompareR1<float>(&builder, {-93.2f, -5.4f, -7.6f, -9.8f}, {},
                              error_spec_);
@@ -2281,7 +2278,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddChainTwoSide) {
   // c ------ (add) ------------/
   //         /
   // d -----/
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   auto a = builder.ConstantR1<float>({91.1f, 2.2f, 3.3f, 4.4f});
   auto b = builder.ConstantR1<float>({2.1f, 3.2f, 4.3f, 5.4f});
@@ -2290,19 +2287,19 @@ XLA_TEST_F(ArrayElementwiseOpTest, AddChainTwoSide) {
 
   auto add_ab = builder.Add(a, b);
   auto add_cd = builder.Add(c, d);
-  auto add_all = builder.Add(add_ab, add_cd);
+  builder.Add(add_ab, add_cd);
 
   ComputeAndCompareR1<float>(&builder, {70.9f, -0.1f, -40.1f, 0.1f}, {},
                              error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, 2DBinaryOpF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto b =
       builder.ConstantR2<float>({{-1.5f, 8.14f, 42.0}, {-1.0f, -4.0f, 5.55f}});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   Array2D<float> expected_array(
       {{-4.0f, 11.28f, 43.0f}, {1.25f, -14.0f, 8.88f}});
@@ -2311,11 +2308,11 @@ XLA_TEST_F(ArrayElementwiseOpTest, 2DBinaryOpF32s) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, ScalarPlus2DF32) {
   // Add a scalar + matrix.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto scalar = builder.ConstantR0<float>(3.0f);
-  auto add = builder.Add(scalar, a);
+  builder.Add(scalar, a);
 
   Array2D<float> expected_array({{0.5f, 6.14f, 4.0f}, {5.25f, -7.0f, 6.33f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
@@ -2323,11 +2320,11 @@ XLA_TEST_F(ArrayElementwiseOpTest, ScalarPlus2DF32) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, 2DPlusScalarF32) {
   // Add a matrix + scalar.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto scalar = builder.ConstantR0<float>(3.0f);
-  auto add = builder.Add(a, scalar);
+  builder.Add(a, scalar);
 
   Array2D<float> expected_array({{0.5f, 6.14f, 4.0f}, {5.25f, -7.0f, 6.33f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
@@ -2336,14 +2333,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, 2DPlusScalarF32) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo2DF32) {
   // Test simple broadcasting of a R1F32 over R2F32. The vector's size matches
   // only dim 0 of the matrix.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<float>({20.0f, 40.0f, 60.0f});
   // clang-format off
   auto m = builder.ConstantR2<float>({
     {-2.5f, 3.14f, 1.0f},
     {2.25f, -10.0f, 3.33f}});
   // clang-format on
-  auto add = builder.Add(v, m, /*broadcast_dimensions=*/{1});
+  builder.Add(v, m, /*broadcast_dimensions=*/{1});
   Array2D<float> expected_array(
       {{17.5f, 43.14f, 61.0f}, {22.25f, 30.0f, 63.33f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
@@ -2369,10 +2366,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Eq) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Ne) {
   // Test broadcasting in Ne comparison.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<int32>({42, 73});
   auto m = builder.ConstantR2<int32>({{42, 73}, {42, 52}});
-  auto cmp = builder.Ne(v, m, /*broadcast_dimensions=*/{1});
+  builder.Ne(v, m, /*broadcast_dimensions=*/{1});
 
   const string expected = R"(pred[2,2] {
   { 00 },
@@ -2383,10 +2380,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Ne) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Ge) {
   // Test broadcasting in Ge comparison.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<int32>({1, 2, 3, 4});
   auto m = builder.ConstantR2<int32>({{1, 0, 5, 6}, {42, 52, 10, 4}});
-  auto cmp = builder.Ge(v, m, /*broadcast_dimensions=*/{1});
+  builder.Ge(v, m, /*broadcast_dimensions=*/{1});
 
   const string expected = R"(pred[2,4] {
   { 1100 },
@@ -2397,10 +2394,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Ge) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Gt) {
   // Test broadcasting in Gt comparison.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<int32>({1, 2, 3, 4});
   auto m = builder.ConstantR2<int32>({{1, 0, 5, 6}, {42, 52, 10, 4}});
-  auto cmp = builder.Gt(v, m, /*broadcast_dimensions=*/{1});
+  builder.Gt(v, m, /*broadcast_dimensions=*/{1});
 
   const string expected = R"(pred[2,4] {
   { 0100 },
@@ -2411,10 +2408,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Gt) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Le) {
   // Test broadcasting in Le comparison.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<int32>({1, 2, 3, 4});
   auto m = builder.ConstantR2<int32>({{1, 0, 5, 6}, {42, 52, 10, 4}});
-  auto cmp = builder.Le(v, m, /*broadcast_dimensions=*/{1});
+  builder.Le(v, m, /*broadcast_dimensions=*/{1});
 
   const string expected = R"(pred[2,4] {
   { 1011 },
@@ -2425,10 +2422,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Le) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Lt) {
   // Test broadcasting in Lt comparison.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<int32>({1, 2, 3, 4});
   auto m = builder.ConstantR2<int32>({{1, 0, 5, 6}, {42, 52, 10, 4}});
-  auto cmp = builder.Lt(v, m, /*broadcast_dimensions=*/{1});
+  builder.Lt(v, m, /*broadcast_dimensions=*/{1});
 
   const string expected = R"(pred[2,4] {
   { 0011 },
@@ -2440,24 +2437,24 @@ XLA_TEST_F(ArrayElementwiseOpTest, Compare1DTo2DS32Lt) {
 XLA_TEST_F(ArrayElementwiseOpTest, Mul2Dby1DF32) {
   // Test simple broadcasting of a R1F32 over R2F32 when the order of binary op
   // arguments is reversed.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto m = builder.ConstantR2<float>({{1.5f, 2.5f, 3.5f}, {4.5f, 5.5f, 6.5f}});
   auto v = builder.ConstantR1<float>({2.0f, 4.0f, 6.0f});
-  auto add = builder.Mul(m, v, /*broadcast_dimensions=*/{1});
+  builder.Mul(m, v, /*broadcast_dimensions=*/{1});
   Array2D<float> expected_array({{3.0f, 10.0f, 21.0f}, {9.0f, 22.0f, 39.0f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo2DWithDegenerateDim1) {
   // Tests broadcasting for arrays with degenerate (size == 1) dimensions.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // m's shape in XLA notation is {3, 2}
   // md's shape in XLA notation is {3, 1}
   // The result has shape {3, 2}, where md is broadcast over m
   auto m =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto md = builder.ConstantR2<float>({{10.0f, 20.0f, 30.0f}});
-  auto add = builder.Add(m, md);
+  builder.Add(m, md);
   Array2D<float> expected_array(
       {{7.5f, 23.14f, 31.0f}, {12.25f, 10.0f, 33.33f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
@@ -2465,14 +2462,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo2DWithDegenerateDim1) {
 
 XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo2DWithDegenerateDim0) {
   // Tests broadcasting for arrays with degenerate (size == 1) dimensions.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // m's shape in XLA notation is {3, 2}
   // md's shape in XLA notation is {1, 2}
   // The result has shape {3, 2}, where md is broadcast over m
   auto m =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto md = builder.ConstantR2<float>({{10.0f}, {20.0f}});
-  auto add = builder.Add(m, md);
+  builder.Add(m, md);
   Array2D<float> expected_array(
       {{7.5f, 13.14f, 11.0f}, {22.25f, 10.0f, 23.33f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
@@ -2483,13 +2480,13 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add2DsWithDegenerateDimsOuterProduct) {
   // effectively creates an "outer product" operation.
   // This is taken from the Numpy docs example at:
   // http://docs.scipy.org/doc/numpy-1.10.1/user/basics.broadcasting.html
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // a's shape in XLA notation is {1, 4}
   // b's shape in XLA notation is {3, 1}
   // The result has shape {3, 4}.
   auto a = builder.ConstantR2<float>({{0.0f}, {10.0f}, {20.0f}, {30.0f}});
   auto b = builder.ConstantR2<float>({{1.0f, 2.0f, 3.0f}});
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
   Array2D<float> expected_array({{1.0f, 2.0f, 3.0f},
                                  {11.0f, 12.0f, 13.0f},
                                  {21.0f, 22.0f, 23.0f},
@@ -2500,10 +2497,10 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add2DsWithDegenerateDimsOuterProduct) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo2DF32TwoWaysOver1) {
   // Add together a (2,2) array and a (2) array, using dimension 0 for
   // broadcasting (though there are two ways to broadcast these shapes).
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<float>({20.0f, 40.0f});
   auto m = builder.ConstantR2<float>({{10.0f, 50.0f}, {77.0f, 88.0f}});
-  auto add = builder.Add(v, m, /*broadcast_dimensions=*/{1});
+  builder.Add(v, m, /*broadcast_dimensions=*/{1});
   Array2D<float> expected_array({{30.0f, 90.0f}, {97.0f, 128.0f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
 }
@@ -2511,17 +2508,17 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo2DF32TwoWaysOver1) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo2DF32TwoWaysOver0) {
   // Add together a (2,2) array and a (2) array, using dimension 1 for
   // broadcasting (though there are two ways to broadcast these shapes).
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto v = builder.ConstantR1<float>({20.0f, 40.0f});
   auto m = builder.ConstantR2<float>({{10.0f, 50.0f}, {77.0f, 88.0f}});
-  auto add = builder.Add(v, m, /*broadcast_dimensions=*/{0});
+  builder.Add(v, m, /*broadcast_dimensions=*/{0});
   Array2D<float> expected_array({{30.0f, 70.0f}, {117.0f, 128.0f}});
   ComputeAndCompareR2<float>(&builder, expected_array, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, 3DBinaryOpF32s) {
   // Binary add of two R3s together
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array3D<float> a_3d({{{1.0f, 2.0f}, {3.0f, 4.0f}, {5.0f, 6.0f}},
                        {{7.0f, 8.0f}, {9.0f, 10.0f}, {11.0f, 12.0f}}});
   auto a = builder.ConstantR3FromArray3D<float>(a_3d);
@@ -2529,7 +2526,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, 3DBinaryOpF32s) {
   Array3D<float> b_3d({{{2.0f, 4.0f}, {6.0f, 8.0f}, {10.0f, 12.0f}},
                        {{14.0f, 16.0f}, {18.0f, 20.0f}, {22.0f, 24.0f}}});
   auto b = builder.ConstantR3FromArray3D<float>(b_3d);
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   Array3D<float> expected_3d(
       {{{3.0f, 6.0f}, {9.0f, 12.0f}, {15.0f, 18.0f}},
@@ -2540,7 +2537,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, 3DBinaryOpF32s) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver2) {
   // Add together a (2, 3, 2) array with a (2) array, using dimension 0 for
   // broadcasting (though there are two ways to broadcast these shapes).
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // clang-format off
   Array3D<float> a_3d({
     {{1.0f, 2.0f},
@@ -2553,7 +2550,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver2) {
   // clang-format on
   auto a = builder.ConstantR3FromArray3D<float>(a_3d);
   auto v = builder.ConstantR1<float>({10.0f, 20.0f});
-  auto add = builder.Add(a, v, /*broadcast_dimensions=*/{2});
+  builder.Add(a, v, /*broadcast_dimensions=*/{2});
 
   Array3D<float> expected_3d(
       {{{11.0f, 22.0f}, {13.0f, 24.0f}, {15.0f, 26.0f}},
@@ -2564,7 +2561,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver2) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver0) {
   // Add together a (2, 3, 2) array with a (2) array, using dimension 2 for
   // broadcasting (though there are two ways to broadcast these shapes).
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // clang-format off
   Array3D<float> a_3d({
     {{1.0f, 2.0f},
@@ -2577,7 +2574,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver0) {
   // clang-format on
   auto a = builder.ConstantR3FromArray3D<float>(a_3d);
   auto v = builder.ConstantR1<float>({10.0f, 20.0f});
-  auto add = builder.Add(a, v, /*broadcast_dimensions=*/{0});
+  builder.Add(a, v, /*broadcast_dimensions=*/{0});
 
   // clang-format off
   Array3D<float> expected_3d({
@@ -2595,7 +2592,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add1DTo3DTwoWaysOver0) {
 XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo3D) {
   // Add together a (2, 3, 2) array with a (3, 2) array, using dimensions {1,2}
   // for broadcasting.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   // clang-format off
   Array3D<float> a_3d({
     {{1.0f, 2.0f},
@@ -2610,7 +2607,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo3D) {
     {10.0f, 20.0f, 30.0f},
     {40.0f, 50.0f, 60.0f},
   });
-  auto add = builder.Add(a, m, /*broadcast_dimensions=*/{0, 1});
+  builder.Add(a, m, /*broadcast_dimensions=*/{0, 1});
 
   Array3D<float> expected_3d({
     {{11.0f, 12.0f},
@@ -2627,7 +2624,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, Add2DTo3D) {
 XLA_TEST_F(ArrayElementwiseOpTest, CompareGtR3F32sWithDegenerateDim2) {
   // Comparison between two 3D arrays of compatible shapes:
   // (2, 3, 2) and (2, 3, 1): expected to produce a (2, 3, 2) shape of PREDs.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array3D<float> a_3d({{{1.0f, 2.0f}, {3.0f, 4.0f}, {5.0f, 6.0f}},
                        {{7.0f, 8.0f}, {9.0f, 10.0f}, {11.0f, 12.0f}}});
   auto a = builder.ConstantR3FromArray3D<float>(a_3d);
@@ -2651,7 +2648,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, CompareGtR3F32sWithDegenerateDim2) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, 4DBinaryOpF32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Array4D<float>> operand_a_4d(new Array4D<float>(2, 3, 4, 5));
   std::unique_ptr<Array4D<float>> operand_b_4d(new Array4D<float>(2, 3, 4, 5));
@@ -2672,13 +2669,13 @@ XLA_TEST_F(ArrayElementwiseOpTest, 4DBinaryOpF32s) {
 
   auto a = builder.ConstantR4FromArray4D<float>(*operand_a_4d);
   auto b = builder.ConstantR4FromArray4D<float>(*operand_b_4d);
-  auto add = builder.Add(a, b);
+  builder.Add(a, b);
 
   ComputeAndCompareR4<float>(&builder, *expected_4d, {}, error_spec_);
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, R4PlusR1InDim1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::unique_ptr<Array4D<float>> operand_a_4d(new Array4D<float>(2, 3, 4, 5));
   std::unique_ptr<Array4D<float>> expected_4d(new Array4D<float>(2, 3, 4, 5));
@@ -2700,7 +2697,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, R4PlusR1InDim1) {
 
   auto a = builder.ConstantR4FromArray4D<float>(*operand_a_4d);
   auto b = builder.ConstantR1<float>(operand_b_1d);
-  auto add = builder.Add(a, b, {1});
+  builder.Add(a, b, {1});
 
   ComputeAndCompareR4<float>(&builder, *expected_4d, {}, error_spec_);
 }
@@ -2715,7 +2712,7 @@ XLA_TEST_F(ArrayElementwiseOpTest, R4_16x16x2x2_Plus_R1_16) {
   std::vector<float> r1(d1);
   std::iota(r1.begin(), r1.end(), 1.0);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::unique_ptr<Literal> a_literal = Literal::CreateR4FromArray4DWithLayout(
       r4, LayoutUtil::MakeLayout({0, 1, 2, 3}));
   auto a = builder.ConstantLiteral(*a_literal);
@@ -2736,11 +2733,11 @@ XLA_TEST_F(ArrayElementwiseOpTest, R4_16x16x2x2_Plus_R1_16) {
 
 // Show that we can't add two opaques.
 XLA_TEST_F(ArrayElementwiseOpTest, CannotAddOpaques) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto shape = ShapeUtil::MakeOpaqueShape();
   auto x = builder.Parameter(0, shape, "x");
-  auto concatenated = builder.Add(x, x);
-  StatusOr<Computation> computation_status = builder.Build();
+  builder.Add(x, x);
+  auto computation_status = builder.Build();
   ASSERT_FALSE(computation_status.ok());
   EXPECT_THAT(computation_status.status().ToString(),
               ::testing::ContainsRegex(
@@ -2748,12 +2745,12 @@ XLA_TEST_F(ArrayElementwiseOpTest, CannotAddOpaques) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, IdentityBroadcastOfSameRankIsAllowed) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto b =
       builder.ConstantR2<float>({{-1.5f, 8.14f, 42.0}, {-1.0f, -4.0f, 5.55f}});
-  auto add = builder.Add(a, b, /*broadcast_dimensions=*/{0, 1});
+  builder.Add(a, b, /*broadcast_dimensions=*/{0, 1});
 
   Array2D<float> expected_array(
       {{-4.0f, 11.28f, 43.0f}, {1.25f, -14.0f, 8.88f}});
@@ -2761,14 +2758,14 @@ XLA_TEST_F(ArrayElementwiseOpTest, IdentityBroadcastOfSameRankIsAllowed) {
 }
 
 XLA_TEST_F(ArrayElementwiseOpTest, NonIdentityBroadcastOfSameRankIsDisallowed) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a =
       builder.ConstantR2<float>({{-2.5f, 3.14f, 1.0f}, {2.25f, -10.0f, 3.33f}});
   auto b =
       builder.ConstantR2<float>({{-1.5f, 8.14f, 42.0}, {-1.0f, -4.0f, 5.55f}});
-  auto add = builder.Add(a, b, /*broadcast_dimensions=*/{1, 0});
+  builder.Add(a, b, /*broadcast_dimensions=*/{1, 0});
 
-  StatusOr<Computation> computation_status = builder.Build();
+  auto computation_status = builder.Build();
   ASSERT_FALSE(computation_status.ok());
   EXPECT_THAT(computation_status.status().error_message(),
               ::testing::ContainsRegex("must.*be the identity"));
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index d9bd1ce6eb..ec95a68ead 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -139,14 +139,31 @@ std::unique_ptr<Literal> ClientLibraryTestBase::ExecuteAndTransferOrDie(
   return ExecuteAndTransfer(builder, arguments).ConsumeValueOrDie();
 }
 
+string ClientLibraryTestBase::ExecuteToString(
+    XlaBuilder* builder, tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
+  auto computation_status = builder->Build();
+  if (!computation_status.ok()) {
+    return computation_status.status().ToString();
+  }
+  auto computation = computation_status.ConsumeValueOrDie();
+
+  auto result =
+      client_->ExecuteAndTransfer(computation, arguments, &execution_options_);
+  if (!result.ok()) {
+    return result.status().ToString();
+  } else {
+    return result.ValueOrDie()->ToString();
+  }
+}
+
 string ClientLibraryTestBase::ExecuteToString(
     ComputationBuilder* builder,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
-  StatusOr<Computation> computation_status = builder->Build();
+  auto computation_status = builder->Build();
   if (!computation_status.ok()) {
     return computation_status.status().ToString();
   }
-  Computation computation = computation_status.ConsumeValueOrDie();
+  auto computation = computation_status.ConsumeValueOrDie();
 
   auto result =
       client_->ExecuteAndTransfer(computation, arguments, &execution_options_);
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index c39597c4e1..5ff200be03 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -124,6 +124,8 @@ class ClientLibraryTestBase : public ::testing::Test {
 
   // Run a computation and return its value as a string. If an error
   // occurs, then instead return the error as a string.
+  string ExecuteToString(XlaBuilder* builder,
+                         tensorflow::gtl::ArraySlice<GlobalData*> arguments);
   string ExecuteToString(ComputationBuilder* builder,
                          tensorflow::gtl::ArraySlice<GlobalData*> arguments);
 
-- 
GitLab


From 071e32d7334b0ff6452111c83ae0b139f28b36ff Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 19:29:48 -0700
Subject: [PATCH 523/960] Add "distribute" argument to tf.estimator.RunConfig()
 in anticipation of upcoming DistributionStrategy support in Estimator.

PiperOrigin-RevId: 190563074
---
 tensorflow/python/estimator/run_config.py     | 20 +++++++++++++++----
 .../tensorflow.estimator.-run-config.pbtxt    |  6 +++++-
 2 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 820fda7765..141eaeff64 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -43,7 +43,8 @@ _DEFAULT_REPLACEABLE_LIST = [
     'session_config',
     'keep_checkpoint_max',
     'keep_checkpoint_every_n_hours',
-    'log_step_count_steps'
+    'log_step_count_steps',
+    'distribute'
 ]
 
 _SAVE_CKPT_ERR = (
@@ -300,7 +301,8 @@ class RunConfig(object):
                session_config=None,
                keep_checkpoint_max=5,
                keep_checkpoint_every_n_hours=10000,
-               log_step_count_steps=100):
+               log_step_count_steps=100,
+               distribute=None):
     """Constructs a RunConfig.
 
     All distributed training related properties `cluster_spec`, `is_chief`,
@@ -424,7 +426,10 @@ class RunConfig(object):
         the feature.
       log_step_count_steps: The frequency, in number of global steps, that the
         global step/sec and the loss will be logged during training.
-
+      distribute: an optional instance of
+        `tf.contrib.distribute.DistributionStrategy`. If specified,
+        then Estimator will distribute the user's model according to the policy
+        specified by that strategy.
 
     Raises:
       ValueError: If both `save_checkpoints_steps` and `save_checkpoints_secs`
@@ -460,7 +465,8 @@ class RunConfig(object):
         session_config=session_config,
         keep_checkpoint_max=keep_checkpoint_max,
         keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
-        log_step_count_steps=log_step_count_steps)
+        log_step_count_steps=log_step_count_steps,
+        distribute=distribute)
 
     self._init_distributed_setting_from_environment_var(tf_config)
 
@@ -671,6 +677,12 @@ class RunConfig(object):
     """Returns the platform defined (in TF_CONFIG) service dict."""
     return self._service
 
+  @property
+  def distribute(self):
+    """Returns the optional `tf.contrib.distribute.DistributionStrategy` object.
+    """
+    return self._distribute
+
   def replace(self, **kwargs):
     """Returns a new instance of `RunConfig` replacing specified properties.
 
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
index 091b1be0c8..759ff752b0 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
@@ -6,6 +6,10 @@ tf_class {
     name: "cluster_spec"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "distribute"
+    mtype: "<type \'property\'>"
+  }
   member {
     name: "evaluation_master"
     mtype: "<type \'property\'>"
@@ -80,7 +84,7 @@ tf_class {
   }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\', \'distribute\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\', \'None\'], "
   }
   member_method {
     name: "replace"
-- 
GitLab


From 41982886efaa2ab9cc75d0d5ab6c27368468d061 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 19:30:26 -0700
Subject: [PATCH 524/960] Fix inconsistency in run_cond.

PiperOrigin-RevId: 190563114
---
 .../contrib/autograph/converters/ifexp.py       |  2 +-
 .../autograph/utils/multiple_dispatch.py        | 11 +++++++++--
 .../autograph/utils/multiple_dispatch_test.py   | 17 ++++++++---------
 3 files changed, 18 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/autograph/converters/ifexp.py b/tensorflow/contrib/autograph/converters/ifexp.py
index aff94d2b79..bb0c0a36a7 100644
--- a/tensorflow/contrib/autograph/converters/ifexp.py
+++ b/tensorflow/contrib/autograph/converters/ifexp.py
@@ -27,7 +27,7 @@ class IfExp(transformer.Base):
 
   def visit_IfExp(self, node):
     template = """
-        autograph_utils.run_cond(test, lambda: body, lambda: orelse)
+        autograph_utils.run_cond(test, lambda: (body,), lambda: (orelse,))
     """
     desugared_ifexp = templates.replace_as_expression(
         template, test=node.test, body=node.body, orelse=node.orelse)
diff --git a/tensorflow/contrib/autograph/utils/multiple_dispatch.py b/tensorflow/contrib/autograph/utils/multiple_dispatch.py
index b756ccfaee..47049255f3 100644
--- a/tensorflow/contrib/autograph/utils/multiple_dispatch.py
+++ b/tensorflow/contrib/autograph/utils/multiple_dispatch.py
@@ -55,10 +55,17 @@ def run_cond(condition, true_fn, false_fn):
 
 
 def py_cond(condition, true_fn, false_fn):
+  """Functional version of Python's conditional."""
   if condition:
-    return true_fn()
+    results = true_fn()
   else:
-    return false_fn()
+    results = false_fn()
+
+  # The contract for the branch functions is to return tuples, but they should
+  # be collapsed to a single element when there is only one output.
+  if len(results) == 1:
+    return results[0]
+  return results
 
 
 def run_while(cond_fn, body_fn, init_args):
diff --git a/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py b/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
index 8c7daa6ded..e6a41bb416 100644
--- a/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
+++ b/tensorflow/contrib/autograph/utils/multiple_dispatch_test.py
@@ -56,20 +56,19 @@ class MultipleDispatchTest(test.TestCase):
       self.assertFalse(should_be_false2)
 
   def test_run_cond_python(self):
-    true_fn = lambda: 2.0
-    false_fn = lambda: 3.0
-    self.assertEqual(multiple_dispatch.run_cond(True, true_fn, false_fn), 2.0)
-    self.assertEqual(multiple_dispatch.run_cond(False, true_fn, false_fn), 3.0)
+    true_fn = lambda: (2,)
+    false_fn = lambda: (3,)
+    self.assertEqual(multiple_dispatch.run_cond(True, true_fn, false_fn), 2)
+    self.assertEqual(multiple_dispatch.run_cond(False, true_fn, false_fn), 3)
 
   def test_run_cond_tf(self):
-
-    true_fn = lambda: constant([2.0])
-    false_fn = lambda: constant([3.0])
+    true_fn = lambda: (constant(2),)
+    false_fn = lambda: (constant(3),)
     with Session() as sess:
       out = multiple_dispatch.run_cond(constant(True), true_fn, false_fn)
-      self.assertEqual(sess.run(out), 2.0)
+      self.assertEqual(sess.run(out), 2)
       out = multiple_dispatch.run_cond(constant(False), true_fn, false_fn)
-      self.assertEqual(sess.run(out), 3.0)
+      self.assertEqual(sess.run(out), 3)
 
   def test_run_while_python(self):
     cond_fn = lambda x, t, s: x > t
-- 
GitLab


From 591b6a7709fa05d490b0c718253492dfad35557f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 19:46:51 -0700
Subject: [PATCH 525/960] Include additional cases for evaluating the fqn
 annotation.

PiperOrigin-RevId: 190564036
---
 .../contrib/autograph/pyct/static_analysis/live_values.py | 8 +++++++-
 .../autograph/pyct/static_analysis/live_values_test.py    | 8 +++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
index 5f813355e6..53ae154590 100644
--- a/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values.py
@@ -59,9 +59,15 @@ class LiveValueResolver(transformer.Base):
           obj = self.context.namespace[node.id]
           anno.setanno(node, 'live_val', obj)
           if hasattr(obj, '__name__'):
+            anno.setanno(node, 'fqn', (obj.__name__,))
+          elif hasattr(obj, '__class__'):
+            obj_class = obj.__class__
+            anno.setanno(node, 'fqn',
+                         (obj_class.__module__, obj_class.__name__))
+          else:
             # If the symbol value is for example a primitive, then it will not
             # have a name.
-            anno.setanno(node, 'fqn', (obj.__name__,))
+            pass
         else:
           pass
           # TODO(mdan): Should we raise an error here?
diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
index b66439624e..69e428bde1 100644
--- a/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/live_values_test.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import six
+
 from tensorflow.contrib.autograph.pyct import anno
 from tensorflow.contrib.autograph.pyct import context
 from tensorflow.contrib.autograph.pyct import parser
@@ -75,7 +77,11 @@ class LiveValuesResolverTest(test.TestCase):
 
     node = self._parse_and_analyze(test_fn, {'a': True})
     retval_node = node.body[0].body[0].value
-    self.assertFalse(anno.hasanno(retval_node, 'fqn'))
+    if six.PY2:
+      self.assertEqual(
+          anno.getanno(retval_node, 'fqn'), ('__builtin__', 'bool'))
+    else:
+      self.assertEqual(anno.getanno(retval_node, 'fqn'), ('builtins', 'bool'))
 
   def test_namespace(self):
 
-- 
GitLab


From c6bc514ffcc601abb7018721c2518cf91a39eeb1 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Mon, 26 Mar 2018 19:53:09 -0700
Subject: [PATCH 526/960] Updating documentation of supported ops.

PiperOrigin-RevId: 190564365
---
 .../lite/g3doc/tf_ops_compatibility.md        | 183 +++++++++++++++++-
 1 file changed, 175 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
index b1bbb7c670..61ea5231e3 100644
--- a/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
+++ b/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md
@@ -30,13 +30,18 @@ quantized training is necessary before conversion.
 ## Data Format and Broadcasting
 
 At the moment TensorFlow Lite supports only TensorFlow's "NHWC" format, and
-broadcasting in operations like tf.add and tf.mul is generally not supported.
+broadcasting is only support in a limited number of ops (tf.add, tf.mul, tf.sub,
+and tf.div).
 
 ## Compatible Operations
 
 The following TensorFlow operations are usually mapped to their TensorFlow Lite
 counterparts:
 
+*   [tf.batch_to_space_nd](https://www.tensorflow.org/api_docs/python/tf/batch_to_space_nd) -
+    *as long as the input tensor is 4D (1 batch + 2 spatial + 1 other) and the
+    crops attribute is not used*
+*   [tf.exp](https://www.tensorflow.org/api_docs/python/tf/exp)
 *   [tf.matmul](https://www.tensorflow.org/api_docs/python/tf/matmul) - *as long
     as the second argument is constant and transposition is not used*
 *   [tf.nn.avg_pool](https://www.tensorflow.org/api_docs/python/tf/nn/avg_pool)
@@ -47,12 +52,30 @@ counterparts:
 *   [tf.nn.l2_normalize](https://www.tensorflow.org/api_docs/python/tf/nn/l2_normalize) -
     *as long as normalization is done along the last dimension*
 *   [tf.nn.local_response_normalization](https://www.tensorflow.org/api_docs/python/tf/nn/local_response_normalization)
+*   [tf.nn.log_softmax](https://www.tensorflow.org/api_docs/python/tf/nn/log_softmax) -
+    *as long as axis is not provided*
 *   [tf.nn.max_pool](https://www.tensorflow.org/api_docs/python/tf/nn/max_pool)
 *   [tf.nn.softmax](https://www.tensorflow.org/api_docs/python/tf/nn/softmax) -
     *as long as tensors are 2D and axis is the last dimension*
+*   [tf.nn.top_k](https://www.tensorflow.org/api_docs/python/tf/nn/top_k)
+*   [tf.pad](https://www.tensorflow.org/api_docs/python/tf/pad) - *as long as
+    mode and constant_values are not used*
+*   [tf.reduce_mean](https://www.tensorflow.org/api_docs/python/tf/reduce_mean) -
+    *as long as the reduction_indices attribute is not used*
 *   [tf.reshape](https://www.tensorflow.org/api_docs/python/tf/reshape)
 *   [tf.sigmoid](https://www.tensorflow.org/api_docs/python/tf/sigmoid)
+*   [tf.space_to_batch_nd](https://www.tensorflow.org/api_docs/python/tf/space_to_batch_nd) -
+    *as long as the input tensor is 4D (1 batch + 2 spatial + 1 other)*
 *   [tf.space_to_depth](https://www.tensorflow.org/api_docs/python/tf/space_to_depth)
+*   [tf.split](https://www.tensorflow.org/api_docs/python/tf/split) - *as long
+    as num is not provided and num_or_size_split contains number of splits as a
+    0D tensor*
+*   [tf.squeeze](https://www.tensorflow.org/api_docs/python/tf/squeeze) - *as
+    long as axis is not provided*
+*   [tf.strided_slice](https://www.tensorflow.org/api_docs/python/tf/strided_slice) -
+    *as long as ellipsis_mask and new_axis_mask are not used*
+*   [tf.transpose](https://www.tensorflow.org/versions/master/api_docs/python/tf/transpose) -
+    *as long as conjugate is not used*
 
 ## Straightforward Conversions, Constant-Folding and Fusing
 
@@ -91,7 +114,6 @@ Here is a list of TensorFlow operations that are usually removed from the graph:
 *   [tf.shape](https://www.tensorflow.org/api_docs/python/tf/shape)
 *   [tf.sqrt](https://www.tensorflow.org/api_docs/python/tf/sqrt)
 *   [tf.square](https://www.tensorflow.org/api_docs/python/tf/square)
-*   [tf.squeeze](https://www.tensorflow.org/api_docs/python/tf/squeeze)
 *   [tf.subtract](https://www.tensorflow.org/api_docs/python/tf/subtract)
 *   [tf.tile](https://www.tensorflow.org/api_docs/python/tf/tile)
 *   [tf.nn.batch_norm_with_global_normalization](https://www.tensorflow.org/api_docs/python/tf/nn/batch_norm_with_global_normalization)
@@ -109,17 +131,11 @@ fused.
 TensorFlow operation not listed above are likely unsupported. Notably, the
 following common ops are not supported at the moment:
 
-*   [tf.batch_to_space_nd](https://www.tensorflow.org/api_docs/python/tf/batch_to_space_nd)
 *   [tf.depth_to_space](https://www.tensorflow.org/api_docs/python/tf/depth_to_space)
 *   [tf.floor](https://www.tensorflow.org/api_docs/python/tf/floor)
 *   [tf.gather](https://www.tensorflow.org/api_docs/python/tf/gather)
 *   [tf.image.resize_bilinear](https://www.tensorflow.org/api_docs/python/tf/image/resize_bilinear)
-*   [tf.pad](https://www.tensorflow.org/api_docs/python/tf/pad)
-*   [tf.reduce_mean](https://www.tensorflow.org/api_docs/python/tf/reduce_mean)
 *   [tf.slice](https://www.tensorflow.org/api_docs/python/tf/slice)
-*   [tf.space_to_batch_nd](https://www.tensorflow.org/api_docs/python/tf/space_to_batch_nd)
-*   [tf.split](https://www.tensorflow.org/api_docs/python/tf/split)
-*   [tf.strided_slice](https://www.tensorflow.org/api_docs/python/tf/strided_slice)
 *   [tf.tanh](https://www.tensorflow.org/api_docs/python/tf/tanh)
 
 ## TensorFlow Lite Operations
@@ -160,6 +176,20 @@ Options {
 }
 ```
 
+**BATCH_TO_SPACE_ND**
+
+```
+Inputs {
+  0: 4D tensor
+  1: 1D tensor
+  2: 2D tensor
+}
+Outputs {
+  0: tensor rearranged using block_shape. See tf.batch_to_space_nd for
+     details.
+}
+```
+
 **CONCATENATION**
 
 ```
@@ -213,6 +243,17 @@ Options {
 }
 ```
 
+**EXP**
+
+```
+Inputs {
+  0: tensor
+}
+Outputs {
+  0: result of computing element-wise exponential of the input tensor
+}
+```
+
 **FULLY_CONNECTED**
 
 ```
@@ -289,6 +330,17 @@ Outputs {
 }
 ```
 
+**LOG_SOFTMAX**
+
+```
+Inputs {
+  0: tensor
+}
+Outputs {
+  0: tensor equivalent to logits - log(reduce_sum(exp(logits), -1))
+}
+```
+
 **MAX_POOL_2D**
 
 ```
@@ -322,6 +374,34 @@ Options {
 }
 ```
 
+**PAD**
+
+```
+Inputs {
+  0: tensor
+  1: tensor
+}
+Outputs {
+  0: tensor where additional values are added before and after the contents of
+     each dimension
+}
+```
+
+**MEAN (tf.reduce_mean)**
+
+```
+Inputs {
+  0: tensor
+  1: tensor
+}
+Outputs {
+  0: tensor containing the mean of the elements
+}
+Options {
+  keep_dims: whether to retain reduced dimensions
+}
+```
+
 **RELU**
 
 ```
@@ -399,6 +479,93 @@ Options {
 }
 ```
 
+**SPACE_TO_BATCH_ND**
+
+```
+Inputs {
+  0: 4D tensor
+  1: 1D tensor
+  2: 2D tensor
+}
+Outputs {
+  0: a tensor rearranged using block_shape. See tf.space_to_batch_nd for
+     details.
+}
+```
+
+**SPLIT**
+
+```
+Inputs {
+  0: 0D tensor (axis)
+  1: tensor (input)
+}
+Outputs {
+  0-N: subtensors built from the input tensors
+}
+Options {
+  num_splits: Specifies number of outputs
+}
+```
+
+**SQUEEZE**
+
+```
+Inputs {
+  0: tensor
+}
+Outputs {
+  0: tensor without any dimensions of size 1
+}
+Options {
+  squeeze_dims
+}
+```
+
+**STRIDED_SLICE**
+
+```
+Inputs {
+  0: tensor
+  1: 1D tensor
+  2: 1D tensor
+  3: 1D tensor
+}
+Outputs {
+  0: slice of the input tensor of the given size
+}
+Options {
+  begin_mask: mask for begin indicies
+  end_mask: mask for end indices
+  shrink_axis_mask: mask that indicates which dimensions to remove
+}
+```
+
+**TOP_K**
+
+```
+Inputs {
+  0: tensor
+  1: OD tensor
+}
+Outputs {
+  0: k largest element along each last dimensional slice
+  1: indicies of values within the last dimension of the input ensor
+}
+```
+
+**TRANSPOSE**
+
+```
+Inputs {
+  0: tensor
+  1: tensor
+}
+Outputs {
+  0: tensor permuted according to perm
+}
+```
+
 And these are TensorFlow Lite operations that are present but not ready for
 custom models yet:
 
-- 
GitLab


From 63cfd006fa1e848daeaf9ac74e2c9f8c42e401b1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 20:01:35 -0700
Subject: [PATCH 527/960] Do not assume Attribute nodes always have a QN - it
 may be missing for attributes of dynamic objects, like function calls.

PiperOrigin-RevId: 190564784
---
 tensorflow/contrib/autograph/pyct/ast_util.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/pyct/ast_util.py b/tensorflow/contrib/autograph/pyct/ast_util.py
index 5a41b5e4a9..4f76a69522 100644
--- a/tensorflow/contrib/autograph/pyct/ast_util.py
+++ b/tensorflow/contrib/autograph/pyct/ast_util.py
@@ -84,7 +84,10 @@ class SymbolRenamer(gast.NodeTransformer):
     return self._process(node)
 
   def visit_Attribute(self, node):
-    return self._process(node)
+    if anno.hasanno(node, anno.Basic.QN):
+      return self._process(node)
+    # Attributes of dynamic objects will not have a QN.
+    return self.generic_visit(node)
 
 
 def rename_symbols(node, name_map):
-- 
GitLab


From 8bcc574711b8770e8341f77d1a9b8370d72d7477 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 20:02:24 -0700
Subject: [PATCH 528/960] Include subscripts in the list of nodes accepted for
 replacement.

PiperOrigin-RevId: 190564824
---
 tensorflow/contrib/autograph/pyct/templates.py | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/tensorflow/contrib/autograph/pyct/templates.py b/tensorflow/contrib/autograph/pyct/templates.py
index fb99e0d4e5..baf7923fff 100644
--- a/tensorflow/contrib/autograph/pyct/templates.py
+++ b/tensorflow/contrib/autograph/pyct/templates.py
@@ -95,6 +95,15 @@ class ReplaceTransformer(gast.NodeTransformer):
         self._check_inner_children_have_context(e)
       for e in node.values:
         self._check_inner_children_have_context(e)
+    elif isinstance(node, gast.Subscript):
+      self._check_inner_children_have_context(node.value)
+      self._check_inner_children_have_context(node.slice)
+    elif isinstance(node, gast.Slice):
+      self._check_inner_children_have_context(node.lower)
+      if node.upper:
+        self._check_inner_children_have_context(node.upper)
+      if node.step:
+        self._check_inner_children_have_context(node.step)
     elif isinstance(node, gast.Name):
       self._check_has_context(node)
     elif isinstance(node, (gast.Str, gast.Num)):
@@ -127,6 +136,9 @@ class ReplaceTransformer(gast.NodeTransformer):
         self._check_inner_children_have_context(e)
       for e in node.values:
         self._check_inner_children_have_context(e)
+    elif isinstance(node, gast.Subscript):
+      self._set_inner_child_context(node.value, ctx)
+      self._check_inner_children_have_context(node.slice)
     elif isinstance(node, (gast.Str, gast.Num)):
       pass
     else:
-- 
GitLab


From 2f208bb73054109390ef9565e8038c14329f73ad Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Mon, 26 Mar 2018 20:18:03 -0700
Subject: [PATCH 529/960] [XLA] Add tests for R1 PRED Slices.

PiperOrigin-RevId: 190566036
---
 tensorflow/compiler/xla/tests/slice_test.cc | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc
index fe36df160d..a14a365bd0 100644
--- a/tensorflow/compiler/xla/tests/slice_test.cc
+++ b/tensorflow/compiler/xla/tests/slice_test.cc
@@ -193,7 +193,9 @@ class SliceR1Test : public ClientLibraryTestBase,
  protected:
   template <typename NativeT>
   void Run(const R1Spec& spec) {
-    std::vector<NativeT> input(spec.input_dim0);
+    // This can't be an std::vector, since you can't grab an ArraySlice of a
+    // vector<bool>.
+    tensorflow::gtl::InlinedVector<NativeT, 1> input(spec.input_dim0);
     std::iota(input.begin(), input.end(), NativeT());
 
     ComputationBuilder builder(client_, TestName());
@@ -201,7 +203,8 @@ class SliceR1Test : public ClientLibraryTestBase,
     builder.Slice(original, {spec.slice_start}, {spec.slice_limit},
                   {spec.slice_stride});
 
-    std::vector<NativeT> expected;
+    // Ditto.
+    tensorflow::gtl::InlinedVector<NativeT, 1> expected;
     for (int i = spec.slice_start; i < spec.slice_limit;
          i += spec.slice_stride) {
       expected.push_back(i);
@@ -230,6 +233,8 @@ XLA_TEST_P(SliceR1Test, DoIt_U64) { Run<uint64>(GetParam()); }
 
 XLA_TEST_P(SliceR1Test, DoIt_S64) { Run<int64>(GetParam()); }
 
+XLA_TEST_P(SliceR1Test, DoIt_PRED) { Run<bool>(GetParam()); }
+
 // Tests for R1 slice ops.
 // The format for each testcase is {input size, start, limit, stride}.
 // clang-format off
-- 
GitLab


From 574303015eb2b6fc4e002f5d2400c3e7f512ae82 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 21:30:56 -0700
Subject: [PATCH 530/960] Add class DistributionStrategy to python/training/,
 though not part of the exposed TF API.

PiperOrigin-RevId: 190570489
---
 tensorflow/python/BUILD                       |   13 +
 tensorflow/python/training/distribute.py      | 1118 +++++++++++++++++
 tensorflow/python/training/distribute_test.py |  104 ++
 3 files changed, 1235 insertions(+)
 create mode 100644 tensorflow/python/training/distribute.py
 create mode 100644 tensorflow/python/training/distribute_test.py

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 30ecc477f2..20d7e81045 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -2914,6 +2914,7 @@ py_library(
         ":variables",
         "//tensorflow/python/eager:backprop",
         "//tensorflow/python/eager:context",
+        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -2943,6 +2944,18 @@ py_test(
     ],
 )
 
+py_test(
+    name = "distribute_test",
+    size = "small",
+    srcs = ["training/distribute_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":client_testlib",
+        ":training",
+        ":variable_scope",
+    ],
+)
+
 py_test(
     name = "evaluation_test",
     size = "small",
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
new file mode 100644
index 0000000000..9261e13230
--- /dev/null
+++ b/tensorflow/python/training/distribute.py
@@ -0,0 +1,1118 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class DistributionStrategy, TowerContext, and supporting APIs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import threading
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses_impl
+from tensorflow.python.training import device_util
+from tensorflow.python.util import nest
+
+
+# ------------------------------------------------------------------------------
+# Internal API for setting the current thread mode as being either in a
+# tower or cross-tower context for a particular distribution strategy.
+
+
+class _ThreadMode(object):
+
+  def __init__(self, dist, cross, tower):
+    self.distribution_strategy = dist
+    self.cross_tower_context = cross
+    self.tower_context = tower
+
+
+class _CrossTowerThreadMode(_ThreadMode):
+
+  def __init__(self, distribution_strategy):
+    _ThreadMode.__init__(
+        self, distribution_strategy, distribution_strategy, None)
+
+
+class _InTowerThreadMode(_ThreadMode):
+
+  def __init__(self, tower_ctx):
+    _ThreadMode.__init__(
+        self, tower_ctx.distribution_strategy, None, tower_ctx)
+
+
+_per_thread_mode = threading.local()
+
+
+def _push_per_thread_mode(context):
+  if not hasattr(_per_thread_mode, "stack"):
+    _per_thread_mode.stack = []
+  _per_thread_mode.stack.append(context)
+
+
+def _pop_per_thread_mode():
+  _per_thread_mode.stack.pop(-1)
+
+
+class _DefaultTowerThreadMode(_ThreadMode):
+  """Type of default value returned by `_get_per_thread_mode()`.
+
+  Used when the thread-local stack is empty.
+  """
+
+  def __init__(self):
+    # _default_distribution_strategy and _default_tower_context are
+    # defined at the bottom of this file.
+    _ThreadMode.__init__(
+        self, _default_distribution_strategy, None, _default_tower_context)
+
+
+def _get_per_thread_mode():
+  try:
+    return _per_thread_mode.stack[-1]
+  except (AttributeError, IndexError):
+    # _default_tower_mode is defined at the bottom of this file.
+    return _default_tower_mode
+
+
+# ------------------------------------------------------------------------------
+# Context tracking whether in a distribution.update() or .update_non_slot()
+# call.
+
+
+_update_device = threading.local()
+
+
+def get_update_device():
+  try:
+    return _update_device.current
+  except AttributeError:
+    return None
+
+
+class UpdateContext(object):
+  """Context manager when you are in `update()` or `update_non_slot()`."""
+
+  def __init__(self, device):
+    self._device = device
+    self._old_device = None
+
+  def __enter__(self):
+    self._old_device = get_update_device()
+    _update_device.current = self._device
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    del exception_type, exception_value, traceback
+    _update_device.current = self._old_device
+
+
+# ------------------------------------------------------------------------------
+# Public API for accessing the current thread mode
+
+
+def get_tower_context():
+  """Returns the current TowerContext or None.
+
+  Note that execution:
+  1. starts in the default (single-tower) tower context;
+  2. switches to cross-tower context when entering a
+     `with DistributionStrategy.scope():` block;
+  3. switches to a (non-default) tower context inside
+     `call_for_each_tower(fn, ...)`;
+  4. if `fn` calls `get_tower_context()->merge_call(merge_fn, ...)`, then
+     inside `merge_fn` you are back in the cross-tower context.
+
+  Note that you can also go directly from step 1 to 4 to switch to a
+  cross-tower context for the default `DistributionStrategy`. You may
+  also switch from the cross-tower context of 4 to a tower context by
+  calling `call_for_each_tower()`, jumping back to step 3.
+
+  Most `DistributionStrategy` methods may only be executed in
+  a cross-tower context, in a tower context you should use the
+  `TowerContext` API instead.
+
+  Returns:
+    The current `TowerContext` object when in a tower context scope, else None.
+
+    Exactly one of `get_tower_context()` and `get_cross_tower_context()`
+    will return None in a particular block.
+  """
+  return _get_per_thread_mode().tower_context
+
+
+def get_cross_tower_context():
+  """Returns the current DistributionStrategy if in a cross-tower context.
+
+  Note that execution:
+  1. starts in the default (single-tower) tower context;
+  2. switches to cross-tower context when entering a
+     `with DistributionStrategy.scope():` block;
+  3. switches to a (non-default) tower context inside
+     `call_for_each_tower(fn, ...)`;
+  4. if `fn` calls `get_tower_context()->merge_call(merge_fn, ...)`, then
+     inside `merge_fn` you are back in the cross-tower context.
+
+  Note that you can also go directly from step 1 to 4 to switch to a
+  cross-tower context for the default `DistributionStrategy`. You may
+  also switch from the cross-tower context of 4 to a tower context by
+  calling `call_for_each_tower()`, jumping back to step 3.
+
+  Most `DistributionStrategy` methods may only be executed in
+  a cross-tower context.
+
+  Returns:
+    Returns the current `DistributionStrategy` object in a cross-tower
+    context, or None.
+
+    Exactly one of `get_tower_context()` and `get_cross_tower_context()`
+    will return None in a particular block.
+  """
+  return _get_per_thread_mode().cross_tower_context
+
+
+def get_distribution_strategy():
+  """Returns the current `DistributionStrategy` object.
+
+  Returns:
+    A `DistributionStrategy` object. Inside a
+    `with distribution_strategy.scope()` block, it returns
+    `distribution_strategy`, otherwise it returns the default
+    (single-tower) `DistributionStrategy` object.
+  """
+  return _get_per_thread_mode().distribution_strategy
+
+
+def has_distribution_strategy():
+  """Return if there is a current non-default `DistributionStrategy`.
+
+  Returns:
+    True if inside a `with distribution_strategy.scope():`.
+  """
+  return get_distribution_strategy() is not _default_distribution_strategy
+
+
+# ------------------------------------------------------------------------------
+# Public utility functions.
+
+
+def get_loss_reduction():
+  """Reduce `method_string` corresponding to the last loss reduction."""
+  loss_reduction = ops.get_default_graph()._last_loss_reduction  # pylint: disable=protected-access
+  if loss_reduction == losses_impl.Reduction.SUM:
+    return "sum"
+  return "mean"
+
+
+# ------------------------------------------------------------------------------
+# Internal API for validating the current thread mode
+
+
+def _require_cross_tower_context(distribution_strategy):
+  """Verify in cross-tower context for `distribution_strategy`."""
+  context = _get_per_thread_mode()
+  if context.cross_tower_context is distribution_strategy: return
+  # We have an error to report, figure out the right message.
+  if context.distribution_strategy is not distribution_strategy:
+    if context.distribution_strategy is _default_distribution_strategy:
+      raise RuntimeError(
+          'Need to be inside "with distribution_strategy.scope()" for %s' %
+          (distribution_strategy,))
+    else:
+      raise RuntimeError(
+          "Mixing different DistributionStrategy objects: %s is not %s" %
+          (context.distribution_strategy, distribution_strategy))
+  assert context.cross_tower_context is None
+  raise RuntimeError("Method requires being in cross-tower context, use "
+                     "get_tower_context().merge_call()")
+
+
+def require_tower_context(tower_ctx):
+  """Verify in `tower_ctx` tower context."""
+  context = _get_per_thread_mode()
+  if context.tower_context is tower_ctx: return
+  # We have an error to report, figure out the right message.
+  if context.tower_context is None:
+    raise RuntimeError("Need to be inside `call_for_each_tower()`")
+  if context.distribution_strategy is tower_ctx.distribution_strategy:
+    # Two different TowerContexts with the same DistributionStrategy.
+    raise RuntimeError("Mismatching tower context.")
+  raise RuntimeError(
+      "Mismatching DistributionStrategy objects: %s is not %s." %
+      (context.distribution_strategy, tower_ctx.distribution_strategy))
+
+
+def _require_distribution_strategy_scope(distribution_strategy):
+  """Verify in a `distribution_strategy.scope()` in this thread."""
+  context = _get_per_thread_mode()
+  if context.distribution_strategy is distribution_strategy: return
+  # We have an error to report, figure out the right message.
+  if context.distribution_strategy is _default_distribution_strategy:
+    raise RuntimeError(
+        'Need to be inside "with distribution_strategy.scope()" for %s' %
+        (distribution_strategy,))
+  else:
+    raise RuntimeError(
+        "Mixing different DistributionStrategy objects: %s is not %s" %
+        (context.distribution_strategy, distribution_strategy))
+
+
+# ------------------------------------------------------------------------------
+# Internal context managers used to implement the DistributionStrategy
+# base class
+
+
+class _CurrentDistributionContext(object):
+  """Context manager for setting the `DistributionStrategy` and var creator."""
+
+  def __init__(self, distribution_strategy, var_creator_scope):
+    self._context = _CrossTowerThreadMode(distribution_strategy)
+    self._var_creator_scope = var_creator_scope
+
+  def __enter__(self):
+    _push_per_thread_mode(self._context)
+    self._var_creator_scope.__enter__()
+    return self._context.distribution_strategy
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    self._var_creator_scope.__exit__(exception_type, exception_value, traceback)
+    _pop_per_thread_mode()
+
+
+class _SameScopeAgainContext(object):
+  """Trivial context manager when you are already in `scope()`."""
+
+  def __init__(self, distribution_strategy):
+    self._distribution_strategy = distribution_strategy
+
+  def __enter__(self):
+    return self._distribution_strategy
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    del exception_type, exception_value, traceback
+
+
+# ------------------------------------------------------------------------------
+# Base classes for all distribution strategies.
+
+
+class DistributionStrategy(object):
+  """A list of devices with a state & compute distribution policy.
+
+  The intent is that you can write an algorithm in a stylized way and
+  it will be usable with a variety of different `DistributionStrategy`
+  implementations. Each descendant will implement a different strategy
+  for distributing the algorithm across multiple devices/machines.
+  Furthermore, these changes can be hidden inside the specific layers
+  and other library classes that need special treatment to run in a
+  distributed setting, so that most users' model definition code can
+  run unchanged. The `DistributionStrategy` API works the same way
+  with eager and graph execution.
+
+  First let's introduce a few high-level concepts:
+
+  * _Data parallelism_ is where we run multiple copies of the model
+    on different slices of the input data. This is in contrast to
+    _model parallelism_ where we divide up a single copy of a model
+    across multiple devices.
+    Note: for now we only support data parallelism at this time, but
+    hope to add support for model parallelism in the future.
+  * A _tower_ is one copy of the model, running on one slice of the
+    input data.
+  * _Synchronous_, or more commonly _sync_, training is when the
+    updates from each tower are aggregated together before updating
+    the model variables. This is in contrast to _asynchronous_, or
+    _async_ training where each tower updates the model variables
+    independently.
+  * Furthermore you might run your computation on multiple devices
+    on one machine (or "host"), or on multiple machines/hosts.
+    If you are running on multiple machines, you might have a
+    single master host that drives computation across all of them,
+    or you might have multiple clients driving the computation
+    asynchronously.
+
+  To distribute an algorithm, we might use some of these ingredients:
+
+  * Parameter servers: These are hosts that hold a single copy of
+    parameters/variables. All towers that want to operate on a variable
+    retrieve it at the beginning of a step and send an update to be
+    applied at the end of the step. Can support either sync or async
+    training.
+  * Mirrored variables: These are variables that are copied to multiple
+    devices, where we keep the copies in sync by applying the same
+    updates to every copy. Normally would only be used with sync training.
+  * Reductions and Allreduce: A _reduction_ is some method of
+    aggregating multiple values into one value, like "sum" or
+    "mean". If doing sync training, we will perform a reduction on the
+    gradients to a parameter from each tower before applying the
+    update. Allreduce is an algorithm for performing a reduction on
+    values from multiple devices and making the result available on
+    all of those devices.
+  * TODO(josh11b): Future: partitioned variables
+
+  We have then a few approaches we want to support:
+  * Code written (as if) with no knowledge of class `DistributionStrategy`.
+    This code should work as before, even if some of the layers, etc.
+    used by that code are written to be distribution-aware. This is done
+    by having a default `DistributionStrategy` that gives ordinary behavior,
+    and by default being in a single tower context.
+  * Ordinary model code that you want to run using a specific
+    `DistributionStrategy`. This can be as simple as:
+
+    ```
+    with my_distribution.scope():
+      iterator = my_distribution.distribute_dataset(dataset)
+      # TODO(josh11b): iterator = dataset.make_one_shot_iterator()
+      tower_train_ops = my_distribution.call_for_each_tower(
+          tower_fn, iterator.get_next())
+      train_op = tf.group(my_distribution.unwrap(tower_train_ops))
+    ```
+
+    This takes an ordinary `dataset` and `tower_fn` and runs it
+    distributed using a particular `DistributionStrategy` in
+    `my_distribution`. Any variables created in `tower_fn` are created
+    using `my_distribution`'s policy, and library functions called by
+    `tower_fn` can use the `get_tower_context()` API to get enhanced
+    behavior in this case.
+  * If you want to write a distributed algorithm, you may use any of
+    the `DistributionStrategy` APIs inside a
+    `with my_distribution.scope():` block of code.
+
+  Lower-level concepts:
+
+  * Wrapped values: In order to represent values parallel across devices
+    (either towers or the devices associated with a particular value), we
+    wrap them in a "PerDevice" or "Mirrored" object that contains a map
+    from device to values. "PerDevice" is used when the value may be
+    different across devices, and "Mirrored" when the value are the same.
+  * Unwrapping and merging: Consider calling a function `fn` on
+    multiple devices, like `call_for_each_tower(fn, w)` with an
+    argument `w that is a wrapped value. This means `w` will have a
+    map taking tower device `d0` to `w0`, tower device `d1` to `w1`,
+    etc. `call_for_each_tower()` unwraps `w` before calling `fn`, so
+    it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges
+    the return values from `fn()`, which can possibly result in
+    wrapped values. For example, let's say `fn()` returns a tuple with
+    three components: (x, a, v0) from tower 0, (x, b, v1) on tower 1,
+    etc. If the first component is the same object `x` from every
+    tower, then the first component of the merged result will also be
+    `x`. If the second component is different (`a`, `b`, ...)  from
+    each tower, then the merged value will have a wrapped map from
+    tower device to the different values. If the third component is
+    the members of a mirrored variable (`v` maps `d0` to `v0, `d1` to
+    `v1`, etc.), then the merged result will be that mirrored variable
+    (`v`).
+  * Tower context vs. Cross-tower context: _tower context_ is when we
+    are in some function that is being called once for each tower.
+    Otherwise we are in cross-tower context, which is useful for
+    calling `DistributionStrategy` methods which operate across the
+    towers (like `reduce()`). By default you start in a tower context
+    (the default "single tower context") and then some methods can
+    switch you back and forth, as described below.
+  * Worker devices vs. parameter devices: Most tower computations will
+    happen on worker devices. Since we don't yet support model
+    parallelism, there will be one worker device per tower. When using
+    parameter servers (see above), the set of devices holding
+    variables may be different, otherwise the parameter devices might
+    match the worker devices.
+  * Non-slot devices are some subset of the parameter devices where we
+    put all the non-slot variables. We need to ensure that all
+    non-slot variables are allocated on the same device, or mirrored
+    across the same set of devices. If you have some variable you want
+    to colocate all the non-slot variables with, you can use
+    `colocate_vars_with()` to get the remaining non-slot variables on
+    the same device.  Otherwise you can use `non_slot_devices()` to
+    pick a consistent set of devices to pass to both
+    `colocate_vars_with()` and `update_non_slot()`.
+
+  When using a `DistributionStrategy`, we have a new type dimension
+  called _locality_ that says what values are compatible with which
+  APIs:
+
+  * T: different value for each tower (e.g. a PerDevice-wrapped value).
+  * M: value is "mirrored" across towers, i.e. there are copies with the
+    same value on each tower (e.g. a Mirrored-wrapped value).
+  * V(`v`): value is "mirrored" across all the devices which have a
+    copy of variable `v` (also a Mirrored-wrapped value, but over
+    parameter devices instead of worker devices).
+  * N: value is "mirrored" across all the "non-slot" devices
+
+  Rules for methods with respect to locality and single-tower vs.
+  cross-tower context:
+
+  * `with d.scope()`: default single-tower context -> cross-tower context for
+    `d`
+  * `with d.colocate_vars_with(v)`: in tower/cross-tower context, variables
+    will be created with locality V(`v`). That is, if we write
+    `with d.colocate_vars_with(v1): v2 = tf.get_variable(...)`, then
+    `v2` will have locality V(`v1`), i.e. locality V(`v2`) will equal
+    V(`v1`).
+  * `with d.colocate_vars_with(d.non_slot_devices(...))`: in
+    tower/cross-tower context, variables will be created with locality N
+  * `v = tf.get_variable(...)`: in tower/cross-tower context, creates
+    a variable (which by definition will have locality V(`v`), though
+    will match another locality if inside a `colocate_vars_with`
+    scope).
+  * `d.distribute_dataset(dataset)`: in cross-tower context, produces an
+    iterator with locality T
+  * `d.broadcast(t)`: in cross-tower context, produces a value with locality M
+  * `d.broadcast(t, v)`: in cross-tower context, produces a value with
+    locality V(`v`)
+  * `d.call_for_each_tower(fn, ...)`: in cross-tower context, runs
+    `fn()` in a tower context (and so may call `get_tower_context()` and
+    use its API, including `merge_call()` to get back to cross-tower
+    context), once for each tower. May use values with locality T or
+    M, and any variable.
+  * `d.reduce(m, t)`: in cross-tower context, accepts t with locality T
+    and produces a value with locality M.
+  * `d.reduce(m, t, v)`: in cross-tower context, accepts t with
+    locality T and produces a value with locality V(`v`).
+  * `d.batch_reduce(m, [(t, v)]): see `d.reduce()`
+  * `d.update(v, fn, ...)`: in cross-tower context, runs `fn()` once
+    for each device `v` is copied to, all inputs should have locality
+    V(`v`), output will have locality V(`v`) as well.
+  * `d.update_non_slot(d.non_slot_devices(), fn)`: in cross-tower
+    context, like `d.update()` except with locality N.
+  * `d.fetch(t)`: Copy `t` with any locality to the client's CPU device.
+
+  The standard pattern for updating variables is to:
+
+  1. Wrap your input dataset in `d.distribute_dataset()`.
+  2. Define each tower `d.call_for_each_tower()` up to the point of
+     getting a list of gradient, variable pairs.
+  3. Call `d.reduce("sum", t, v)` or `d.batch_reduce()` to sum the
+     gradients (with locality T) into values with locality V(`v`).
+  4. Call `d.update(v)` for each variable to update its value.
+
+  Steps 3 and 4 are done automatically by class `Optimizer` if you call
+  its `apply_gradients` method in a tower context. Otherwise you can
+  manually call its `distributed_apply` method in a cross-tower context.
+
+  Another thing you might want to do in the middle of your tower function
+  is an all-reduce of some intermediate value, using `d.reduce()` or
+  `d.batch_reduce()` without supplying a variable as the destination.
+
+  Layers should expect to be called in a tower context, and can use
+  the `get_tower_context()` function to get a `TowerContext` object.  The
+  `TowerContext` object has a `merge_call()` method for entering
+  cross-tower context where you can use `reduce()` (or
+  `batch_reduce()`) and then optionally `update()` to update state.
+
+  You may use this API whether or not a `DistributionStrategy` is
+  being used, since there is a default implementation of
+  `TowerContext` and `DistributionStrategy`. Or you can use the
+  `get_tower_context().is_single_tower` property to run different code
+  in the distributed vs. single tower cases.
+  """
+
+  # TODO(josh11b): Raise an exception if variable paritioning requested before
+  #   we add support.
+  # TODO(josh11b): Also `parameter_device_index` property?
+  # TODO(josh11b): `map()`
+  # TODO(josh11b): ClusterSpec/ClusterResolver
+  # TODO(josh11b): Partitioned computations, state; sharding
+  # TODO(josh11b): Model parallelism: "towers" with multiple devices; shuffling
+  # TODO(josh11b): Tower-local variables
+  # TODO(josh11b): List of towers with their worker and parameter devices
+  #   (where the parameter devices may overlap in the ps case).
+
+  def scope(self):
+    """Returns a context manager selecting this DistributionStrategy as current.
+
+    Inside a `with distribution_strategy.scope():` code block, this thread
+    will use a variable creator set by `distribution_strategy`, and will
+    enter its "cross-tower context".
+
+    Returns:
+      A context manager.
+    """
+    if has_distribution_strategy():
+      _require_cross_tower_context(self)
+      return _SameScopeAgainContext(self)
+
+    def creator_with_resource_vars(*args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      kwargs["use_resource"] = True
+      return self._create_variable(*args, **kwargs)
+
+    return _CurrentDistributionContext(
+        self, variable_scope.variable_creator_scope(creator_with_resource_vars))
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    # Note: should support "colocate_with" argument.
+    raise NotImplementedError("must be implemented in descendants")
+
+  def colocate_vars_with(self, colocate_with_variable):
+    """Controls which devices variables will be created on.
+
+    Note this may only be used inside `self.scope()`.
+
+    Example usage:
+
+    ```
+    with distribution_strategy.scope():
+      var1 = tf.get_variable(...)
+      with distribution_strategy.colocate_vars_with(v1):
+        # var2 and var3 will be created on the same device(s) as var1
+        var2 = tf.get_variable(...)
+        var3 = tf.get_variable(...)
+
+      def fn(v1, v2, v3):
+        # operates on v1 from var1, v2 from var2, and v3 from var3
+
+      # `fn` runs on every device `v1` is on, `v2` and `v3` will be there too.
+      distribution_strategy.update(v1, fn, v2, v3)
+    ```
+
+    Args:
+      colocate_with_variable: A created in `self.scope()`. Variables created
+        while in the returned context manager will be on the same set of
+        devices as `colocate_with_variable`.
+
+    Returns:
+      A context manager.
+    """
+    def create_colocated_variable(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      kwargs["use_resource"] = True
+      kwargs["colocate_with"] = colocate_with_variable
+      return next_creator(*args, **kwargs)
+
+    _require_distribution_strategy_scope(self)
+    return variable_scope.variable_creator_scope(create_colocated_variable)
+
+  # TODO(josh11b): Currently this returns an iterator, but should return
+  # something implementing (a subset of) the Dataset API.
+  def distribute_dataset(self, dataset):
+    """Return an iterator into `dataset` split across all towers.
+
+    Suitable for providing input to for `call_for_each_tower()`, as in:
+
+    ```
+    with distribution_strategy.scope():
+      iterator = distribution_strategy.distribute_dataset(dataset)
+      tower_results = distribution_strategy.call_for_each_tower(
+          tower_fn, iterator.get_next())
+    ```
+
+    Args:
+      dataset: A `tf.data.Dataset`.
+
+    Returns:
+      A Dataset iterator that will produce separate splits for each tower.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  def broadcast(self, tensor, destinations=None):
+    """Mirror a tensor on one device to all worker devices.
+
+    Args:
+      tensor: A Tensor value to broadcast.
+      destinations: An optional mirrored variable, device string, or
+        list of device strings, specifying the destination devices
+        to copy `tensor` to. Defaults to `self.worker_devices`.
+
+    Returns:
+      A value mirrored to `destinations` devices.
+    """
+    # TODO(josh11b): More docstring
+    _require_cross_tower_context(self)
+    return self._broadcast(tensor, destinations)
+
+  def _broadcast(self, tensor, destinations):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def call_for_each_tower(self, fn, *args, **kwargs):
+    """Run `fn` once per tower.
+
+    `fn` may call `tf.get_tower_context()` to access methods such as
+    `tower_id()` and `merge_call()`.
+
+    `merge_call()` is used to communicate betwen the towers and
+    re-enter the cross-tower context. All towers pause their execution
+    having encountered a `merge_call()` call. After that the
+    `merge_fn`-function is executed. Its results are then unwrapped and
+    given back to each tower call. After that execution resumes until
+    `fn` is complete or encounters another `merge_call()`.  Example:
+
+    ```python
+    # Called once in "cross-tower" context.
+    def merge_fn(distribution, three_plus_tower_id):
+      # sum the values across towers
+      return sum(distribution.unwrap(three_plus_tower_id))
+
+    # Called once per tower in `distribution`, in a "tower" context.
+    def fn(three):
+      tower_ctx = tf.get_tower_context()
+      v = three + tower_ctx.tower_id
+      # Computes the sum of the `v` values across all towers.
+      s = tower_ctx.merge_call(merge_fn, v)
+      return s + v
+
+    with distribution.scope():
+      # in "cross-tower" context
+      ...
+      merged_results = distribution.call_for_each_tower(fn, 3)
+      # merged_results has the values from every tower execution of `fn`.
+      print(distribution.unwrap(merged_results))  # Prints a list
+    ```
+
+    Args:
+      fn: function to run (will be run once per tower).
+      *args: positional arguments for `fn`
+      **kwargs: keyword arguments for `fn`.
+          `"run_concurrently"`: Boolean indicating whether executions of `fn`
+             can be run concurrently (under eager execution only), defaults to
+             `True`.
+
+    Returns:
+      Merged return value of `fn` across all towers.
+    """
+    _require_cross_tower_context(self)
+    return self._call_for_each_tower(fn, *args, **kwargs)
+
+  def _call_for_each_tower(self, fn, *args, **kwargs):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def reduce(self, method_string, value, destinations=None):
+    """Combine (via e.g. sum or mean) values across towers.
+
+    Args:
+      method_string: A string indicating how to combine values, either
+        "sum" or "mean".
+      value: A per-device value with one value per tower.
+      destinations: An optional mirrored variable, a device string,
+        list of device strings. The return value will be copied to all
+        destination devices (or all the devices where the mirrored
+        variable resides). If `None` or unspecified, the destinations
+        will match the devices `value` resides on.
+
+    Returns:
+      A value mirrored to `destinations`.
+    """
+    # TODO(josh11b): More docstring
+    # TODO(josh11b): Return an unwrapped value if colocate_with is a
+    # single device.
+    _require_cross_tower_context(self)
+    return self._reduce(method_string, value, destinations)
+
+  def _reduce(self, method_string, value, destinations):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def batch_reduce(self, method_string, value_destination_pairs):
+    """Combine multiple `reduce` calls into one for faster execution.
+
+    Args:
+      method_string: A string indicating how to combine values, either
+        "sum" or "mean".
+      value_destination_pairs: A sequence of (value, destinations)
+        pairs. See `reduce()` for a description.
+
+    Returns:
+      A list of mirrored values, one per pair in `value_destination_pairs`.
+    """
+    # TODO(josh11b): More docstring
+    _require_cross_tower_context(self)
+    assert method_string in ("sum", "mean")
+    return self._batch_reduce(method_string, value_destination_pairs)
+
+  def _batch_reduce(self, method_string, value_destination_pairs):
+    return [self.reduce(method_string, t, destinations=v)
+            for t, v in value_destination_pairs]
+
+  def update(self, var, fn, *args, **kwargs):
+    """Run `fn` to update `var` using inputs mirrored to the same devices.
+
+    If `var` is mirrored across multiple devices, then this implements
+    logic like:
+
+    ```
+    results = {}
+    for device, v in var:
+      with tf.device(device):
+        # *args and **kwargs will be unwrapped if they are mirrored.
+        results[device] = fn(v, *args, **kwargs)
+    return merged(results)
+    ```
+
+    Otherwise this returns `fn(var, *args, **kwargs)` colocated with `var`.'
+
+    Neither *args nor **kwargs may contain per-device values.
+    If they contain mirrored values, they will be unwrapped before
+    calling `fn`.
+
+    Args:
+      var: Variable, possibly mirrored to multiple devices, to operate on.
+      fn: Function to call. Should take the variable as the first argument.
+      *args: Additional positional arguments to pass to `fn()`.
+      **kwargs: Keyword arguments to pass to `fn()`.
+
+    Returns:
+      Merged return value of `fn` across all towers.
+    """
+    _require_cross_tower_context(self)
+    return self._update(var, fn, *args, **kwargs)
+
+  def _update(self, var, fn, *args, **kwargs):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    """Runs `fn(*args, **kwargs)` on `colocate_with` devices.
+
+    Args:
+      colocate_with: The return value of `non_slot_devices()`.
+      fn: Function to execute.
+      *args: Positional arguments to pass to `fn()`.
+      **kwargs: Keyword arguments to pass to `fn()`.
+
+    Returns:
+      Return value of `fn`, possibly merged across devices.
+    """
+    _require_cross_tower_context(self)
+    return self._update_non_slot(colocate_with, fn, *args, **kwargs)
+
+  def _update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def fetch(self, val, destination="/device:CPU:0", fn=lambda x: x):
+    """Return a copy of `val` or `fn(val)` on `destination`.
+
+    This is useful for getting a mirrored value onto a device.  It
+    will attempt to avoid a copy by checking if the value is already
+    on the destination device.
+
+    Args:
+      val: Value (which may be mirrored) to copy.
+      destination: A device string to copy the value to.
+      fn: An optional function to apply to the value on the source
+          device, before copying.
+
+    Returns:
+      A `Tensor` on `destination`.
+    """
+    _require_cross_tower_context(self)
+    return self._fetch(val, destination, fn)
+
+  def _fetch(self, val, destination, fn):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def unwrap(self, value):
+    """Returns the list of all per-device values contained in `value`.
+
+    Args:
+      value: A value returned by `call_for_each_tower()` or a variable
+        created in `scope()`.
+
+    Returns:
+      A list of values contained in `value`. If `value` represents a single
+      value, this returns `[value].`
+    """
+    _require_cross_tower_context(self)
+    return self._unwrap(value)
+
+  def _unwrap(self, distributed_value):
+    raise NotImplementedError("must be implemented in descendants")
+
+  def group(self, value, name=None):
+    """Shortcut for `tf.group(distribution.unwrap(value))`."""
+    value = nest.flatten(self.unwrap(value))
+
+    if len(value) != 1 or name is not None:
+      return control_flow_ops.group(value, name=name)
+    # Special handling for the common case of one op.
+    v, = value
+    if isinstance(v, ops.Tensor):
+      v = v.op
+    return v
+
+  @property
+  def is_single_tower(self):
+    """Returns whether there is a single tower or multiple.
+
+    Returns:
+      A boolean. If `True`, `call_for_each_tower(fn)` will only call `fn` once.
+      If `False`, `call_for_each_tower(fn)` may call `fn` multiple times.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def num_towers(self):
+    """Returns number of towers, for purposes of averaging across towers."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def worker_devices(self):
+    """Returns the list of devices used to run `call_for_each_tower()` calls."""
+    # TODO(josh11b): More docstring
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def parameter_devices(self):
+    """Returns the list of devices used for variable and `update` placement."""
+    # TODO(josh11b): More docstring
+    raise NotImplementedError("must be implemented in descendants")
+
+  def non_slot_devices(self, var_list):
+    """Device(s) for non-slot variables.
+
+    Create variables on these devices in a
+    `with colocate_vars_with(non_slot_devices(...)):` block.
+    Update those using `update_non_slot()`.
+
+    Args:
+      var_list: The list of variables being optimized, needed with the
+        default `DistributionStrategy`.
+    """
+    raise NotImplementedError("must be implemented in descendants")
+
+  @property
+  def worker_device_index(self):
+    """An object mapping worker device to an id.
+
+    This might be passed as an argument to `call_for_each_tower()`, as in:
+
+    ```
+    with distribution_strategy.scope():
+
+      def fn(device_id):
+        # device_id is an integer. `fn` is being executed on device:
+        #    distribution_strategy.worker_devices[device_id].
+
+      distribution_strategy.call_for_each_tower(
+          fn, distribution_strategy.worker_device_index)
+    ```
+
+    Returns:
+      An index object, or the integer 0 if there is only a single tower.
+    """
+    _require_cross_tower_context(self)
+    return self._worker_device_index()
+
+  def _worker_device_index(self):
+    raise NotImplementedError("must be implemented in descendants")
+
+
+# A note about the difference between the context managers
+# `TowerContext` (defined here) and `_CurrentDistributionContext`
+# (defined above) used by `DistributionStrategy.scope()`:
+#
+# * a TowerContext is only present during a `call_for_each_tower()`
+#   call (except during a `merge_run` call) and in such a scope it
+#   will be returned by calls to `get_tower_context()`.  Implementers of new
+#   DistributionStrategy descendants will frequently also need to
+#   define a descendant of TowerContext, and are responsible for
+#   entering and exiting this context.
+#
+# * DistributionStrategy.scope() sets up a variable_creator scope that
+#   changes variable creation calls (e.g. to make mirrored
+#   variables). This is intended as an outer scope that users enter once
+#   around their model creation and graph definition. There is no
+#   anticipated need to define descendants of _CurrentDistributionContext.
+#   It sets the current DistributionStrategy for purposes of
+#   `get_distribution_strategy()` and `has_distribution_strategy()`
+#   and switches the thread mode to a "cross-tower context".
+class TowerContext(object):
+  """DistributionStrategy API inside a `call_for_each_tower()` call."""
+
+  def __init__(self, distribution_strategy, tower_id):
+    self._distribution_strategy = distribution_strategy
+    self._thread_context = _InTowerThreadMode(self)
+    self._tower_id = tower_id
+
+  def __enter__(self):
+    _push_per_thread_mode(self._thread_context)
+
+  def __exit__(self, exception_type, exception_value, traceback):
+    _pop_per_thread_mode()
+
+  def merge_call(self, merge_fn, *args, **kwargs):
+    """Merge args across towers and run `merge_fn` in a cross-tower context.
+
+    This allows communication and coordination when there are multiple calls
+    to a model function triggered by a call to
+    `distribution.call_for_each_tower(model_fn, ...)`.
+
+    See `MirroredDistribution.call_for_each_tower()` for an explanation.
+
+    Otherwise, this is equivalent to:
+
+    ```
+    distribution = get_distribution_strategy()
+    with cross-tower-context(distribution):
+      return merge_fn(distribution, *args, **kwargs)
+    ```
+
+    Args:
+      merge_fn: function that joins arguments from threads that are given as
+        PerDevice. It accepts `DistributionStrategy` object as the first
+        argument.
+      *args: positional per-thread arguments for `merge_fn`
+      **kwargs: keyword per-thread arguments for `merge_fn`.
+
+    Returns:
+      The return value of `merge_fn`, except for `PerDevice` values which are
+      unpacked.
+    """
+    require_tower_context(self)
+    return self._merge_call(merge_fn, *args, **kwargs)
+
+  def _merge_call(self, merge_fn, *args, **kwargs):
+    """Default implementation for single tower."""
+    _push_per_thread_mode(  # thread-local, so not needed with multiple threads
+        _CrossTowerThreadMode(self._distribution_strategy))
+    try:
+      return merge_fn(self._distribution_strategy, *args, **kwargs)
+    finally:
+      _pop_per_thread_mode()
+
+  @property
+  def is_single_tower(self):
+    """Returns whether there is a single tower or multiple."""
+    require_tower_context(self)
+    return self._distribution_strategy.is_single_tower
+
+  @property
+  def num_towers(self):
+    """Returns number of towers, for purposes of averaging across towers."""
+    return self._distribution_strategy.num_towers
+
+  @property
+  def tower_id(self):
+    """Which tower is being defined, a number from 0 to `num_towers - 1`."""
+    require_tower_context(self)
+    return self._tower_id
+
+  @property
+  def distribution_strategy(self):
+    """The current `DistributionStrategy` object."""
+    return self._distribution_strategy
+
+  @property
+  def device(self):
+    """The device this tower is to be executed on, as a string."""
+    require_tower_context(self)
+    return device_util.current()
+
+  # TODO(josh11b): Implement `start_all_reduce(method, t)` that returns
+  # a function returning the result of reducing `t` across all
+  # towers. Most likely can be implemented in terms of `merge_call()`
+  # and `batch_reduce()`.
+
+# ------------------------------------------------------------------------------
+
+
+class _DefaultDistributionStrategy(DistributionStrategy):
+  """Default `DistributionStrategy` if none is explicitly selected."""
+
+  def scope(self):
+    """Context manager setting a variable creator and `self` as current."""
+    if has_distribution_strategy():
+      raise RuntimeError("Must not nest DistributionStrategy scopes.")
+
+    def creator(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      return next_creator(*args, **kwargs)
+
+    return _CurrentDistributionContext(
+        self, variable_scope.variable_creator_scope(creator))
+
+  def colocate_vars_with(self, colocate_with_variable):
+    """Does not require `self.scope`."""
+    def create_colocated_variable(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      with ops.colocate_with(colocate_with_variable):
+        return next_creator(*args, **kwargs)
+
+    _require_distribution_strategy_scope(self)
+    return variable_scope.variable_creator_scope(create_colocated_variable)
+
+  def distribute_dataset(self, dataset):
+    # TODO(josh11b): Support for this when executing eagerly is currently only
+    # in contrib.
+    return dataset.make_one_shot_iterator()
+
+  def _broadcast(self, tensor, destinations):
+    if destinations is None:
+      return tensor
+    else:
+      raise NotImplementedError("TODO")
+
+  def _call_for_each_tower(self, fn, *args, **kwargs):
+    # We don't run `fn` in multiple threads in _DefaultDistributionStrategy.
+    kwargs.pop("run_concurrently", None)
+    with TowerContext(self, tower_id=0):
+      return fn(*args, **kwargs)
+
+  def _reduce(self, method_string, value, destinations):
+    # TODO(josh11b): Use destinations?
+    del method_string, destinations
+    return value
+
+  def _update(self, var, fn, *args, **kwargs):
+    # TODO(josh11b): Figure out what we should be passing to UpdateContext()
+    # once that value is used for something.
+    with ops.colocate_with(var), UpdateContext(var):
+      return fn(var, *args, **kwargs)
+
+  def _update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    # TODO(josh11b): Figure out what we should be passing to UpdateContext()
+    # once that value is used for something.
+    with ops.colocate_with(colocate_with), UpdateContext(colocate_with):
+      return fn(*args, **kwargs)
+
+  def _fetch(self, var, destination, fn):
+    with ops.colocate_with(var):
+      var = fn(var)
+    with ops.device(destination):
+      return array_ops.identity(var)
+
+  def _unwrap(self, distributed_value):
+    return [distributed_value]
+
+  @property
+  def is_single_tower(self):
+    return True
+
+  @property
+  def num_towers(self):
+    return 1
+
+  @property
+  def worker_devices(self):
+    raise RuntimeError(
+        "worker_devices() method unsupported by _DefaultDistributionStrategy.")
+
+  @property
+  def parameter_devices(self):
+    raise RuntimeError("parameter_devices() method unsupported by "
+                       "_DefaultDistributionStrategy.")
+
+  def non_slot_devices(self, var_list):
+    return min(var_list, key=lambda x: x.name)
+
+  def _worker_device_index(self):
+    raise RuntimeError("worker_device_index() method unsupported by "
+                       "_DefaultDistributionStrategy.")
+
+
+# ------------------------------------------------------------------------------
+# Singletons
+
+_default_distribution_strategy = _DefaultDistributionStrategy()
+_default_tower_context = TowerContext(
+    _default_distribution_strategy, tower_id=0)
+_default_tower_mode = _DefaultTowerThreadMode()
diff --git a/tensorflow/python/training/distribute_test.py b/tensorflow/python/training/distribute_test.py
new file mode 100644
index 0000000000..0a4f19c31f
--- /dev/null
+++ b/tensorflow/python/training/distribute_test.py
@@ -0,0 +1,104 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Test DistributionStrategy, TowerContext, and supporting APIs."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import test
+from tensorflow.python.training import distribute
+
+
+class _TestTowerContext(distribute.TowerContext):
+
+  def merge_call(self, fn, *args, **kwargs):
+    return kwargs["test_arg"]
+
+
+class _TestStrategy(distribute.DistributionStrategy):
+
+  def _call_for_each_tower(self, fn, *args, **kwargs):
+    with _TestTowerContext(self, tower_id=0):
+      return fn(*args, **kwargs)
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    return kwargs["name"]
+
+
+def _assert_in_default_state(t):
+  t.assertIs(distribute._default_tower_context,
+             distribute.get_tower_context())
+  t.assertIs(None, distribute.get_cross_tower_context())
+  t.assertIs(distribute._default_distribution_strategy,
+             distribute.get_distribution_strategy())
+  t.assertFalse(distribute.has_distribution_strategy())
+
+
+class TestStrategyTest(test.TestCase):
+
+  def testCallForEachTower(self):
+    _assert_in_default_state(self)
+    dist = _TestStrategy()
+
+    def run_fn():
+      tower_context = distribute.get_tower_context()
+      self.assertTrue(tower_context is not None)
+      self.assertIs(None, distribute.get_cross_tower_context())
+      self.assertTrue(distribute.has_distribution_strategy())
+      self.assertIs(dist, distribute.get_distribution_strategy())
+      self.assertEqual("foo", tower_context.merge_call(None, test_arg="foo"))
+      self.assertEqual("bar", variable_scope.variable(1.0, name="bar"))
+
+    with self.assertRaises(RuntimeError):
+      dist.call_for_each_tower(run_fn)
+    with dist.scope():
+      dist.call_for_each_tower(run_fn)
+    _assert_in_default_state(self)
+
+  def testScope(self):
+    _assert_in_default_state(self)
+    dist = _TestStrategy()
+    with dist.scope():
+      self.assertIs(None, distribute.get_tower_context())
+      self.assertIs(dist, distribute.get_cross_tower_context())
+      self.assertTrue(distribute.has_distribution_strategy())
+      self.assertIs(dist, distribute.get_distribution_strategy())
+      self.assertEqual("baz", variable_scope.variable(1.0, name="baz"))
+    _assert_in_default_state(self)
+
+
+class DefaultDistributionStrategyTest(test.TestCase):
+
+  def testMergeCall(self):
+    _assert_in_default_state(self)
+
+    def merge_fn(dist, s):
+      self.assertIs(distribute._default_distribution_strategy, dist)
+      self.assertIs(None, distribute.get_tower_context())
+      self.assertIs(dist, distribute.get_cross_tower_context())
+      self.assertIs(dist, distribute.get_distribution_strategy())
+      self.assertFalse(distribute.has_distribution_strategy())
+      return "foo_" + s
+
+    tower_ctx = distribute.get_tower_context()
+    self.assertIs(distribute._default_tower_context, tower_ctx)
+    self.assertEqual("foo_bar", tower_ctx.merge_call(merge_fn, "bar"))
+    _assert_in_default_state(self)
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From b16ec315e7e9d41645634398da202629c3baa5af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 26 Mar 2018 22:30:15 -0700
Subject: [PATCH 531/960] Use is_resource_variable() in
 train.assert_gloabl_step.

PiperOrigin-RevId: 190573872
---
 tensorflow/python/training/training_util.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/python/training/training_util.py b/tensorflow/python/training/training_util.py
index 4f1abccc96..d05e1d2c83 100644
--- a/tensorflow/python/training/training_util.py
+++ b/tensorflow/python/training/training_util.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-
 from tensorflow.python.eager import context
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import graph_io
@@ -31,7 +30,6 @@ from tensorflow.python.ops import variables
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
-
 # Picked a long key value to minimize the chance of collision with user defined
 # collection keys.
 GLOBAL_STEP_READ_KEY = 'global_step_read_op_cache'
@@ -170,8 +168,7 @@ def assert_global_step(global_step_tensor):
   """
   if not (isinstance(global_step_tensor, variables.Variable) or
           isinstance(global_step_tensor, ops.Tensor) or
-          isinstance(global_step_tensor,
-                     resource_variable_ops.ResourceVariable)):
+          resource_variable_ops.is_resource_variable(global_step_tensor)):
     raise TypeError(
         'Existing "global_step" must be a Variable or Tensor: %s.' %
         global_step_tensor)
-- 
GitLab


From 307794e156bc21b2f122bf5e7d907299392023c5 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Mon, 26 Mar 2018 22:44:27 -0700
Subject: [PATCH 532/960] [XLA:CPU] Allow the shape partition algorithm to
 partition the most minor dimension.

The current shape paritition algorithm does not partition the most minor
dimension, because doing so causes dynamic loop bounds for the inner loop and
used to prohibit LLVM vectorization. This constraint has been removed with
revision 328478 and LLVM can now vectorize loops with dynamic bounds. Allow
partitioning the most minor dimension is also necessary to support the
parallelization of matrix-vector multiplication.

Adjust shape_partition_test to reflect this change in the shape partition
algorithm.

PiperOrigin-RevId: 190574615
---
 .../xla/service/cpu/shape_partition.cc        |   5 +-
 .../xla/service/cpu/shape_partition_test.cc   | 116 ++++++------------
 .../exhaustive_f32_elementwise_op_test.cc     |   4 +-
 3 files changed, 43 insertions(+), 82 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/shape_partition.cc b/tensorflow/compiler/xla/service/cpu/shape_partition.cc
index 61b408b8c2..42fe955f19 100644
--- a/tensorflow/compiler/xla/service/cpu/shape_partition.cc
+++ b/tensorflow/compiler/xla/service/cpu/shape_partition.cc
@@ -20,12 +20,13 @@ namespace cpu {
 
 std::vector<int64> ShapePartitionAssigner::Run(int64 target_partition_count) {
   // Gather outer-most dims where dim_size >= 'target_partition_count'.
-  // Note: always leave inner-dim static for vectorization/optimizations.
+  // This may include the inner-dim as LLVM can vectorize loops with dynamic
+  // bounds.
   std::vector<int64> outer_dims;
   int64 outer_dim_size = 1;
   // TODO(b/27458679) Consider reserving enough minor dimensions (based on
   // target vector register width) to enable vector instructions.
-  for (int i = shape_.layout().minor_to_major_size() - 1; i >= 1; --i) {
+  for (int i = shape_.layout().minor_to_major_size() - 1; i >= 0; --i) {
     const int64 dimension = shape_.layout().minor_to_major(i);
     outer_dims.push_back(dimension);
     outer_dim_size *= shape_.dimensions(dimension);
diff --git a/tensorflow/compiler/xla/service/cpu/shape_partition_test.cc b/tensorflow/compiler/xla/service/cpu/shape_partition_test.cc
index ee0c53fa6d..ae80a6f497 100644
--- a/tensorflow/compiler/xla/service/cpu/shape_partition_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/shape_partition_test.cc
@@ -30,105 +30,65 @@ class ShapePartitionAssignerTest : public HloTestBase {
  protected:
   typedef std::vector<int64> Vec;
 
-  void RunR2Test(const Shape& shape, const int64 expected_max_partition_count) {
+  void RunR2Test(const Shape& shape, int64 max_target_partition_count,
+                 const std::vector<int64>* expected_partitions) {
     ShapePartitionAssigner assigner(shape);
-    // Check all partitions of outer dimension.
-    for (int64 i = 1; i <= expected_max_partition_count; ++i) {
-      EXPECT_TRUE(ContainersEqual(Vec({i}),
-                                  assigner.Run(/*target_partition_count=*/i)));
+    // Iterate through 1..max_target_partition_count.
+    for (int64 i = 1; i <= max_target_partition_count; ++i) {
+      std::vector<int64> actual_partitions =
+          assigner.Run(/*target_partition_count=*/i);
+      EXPECT_THAT(actual_partitions, expected_partitions[i - 1]);
     }
-    // Check target_partition_count > outer dimension size.
-    EXPECT_TRUE(ContainersEqual(
-        Vec({expected_max_partition_count}),
-        assigner.Run(
-            /*target_partition_count=*/expected_max_partition_count + 1)));
   }
 };
 
 TEST_F(ShapePartitionAssignerTest, Shape13WithLayout10) {
-  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {1, 3}, {1, 0}), 1);
+  std::vector<int64> expected_partitions[] = {{1} /* 1 */, {1, 2} /* 2 */};
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {1, 3}, {1, 0}), 2,
+            expected_partitions);
 }
 
 TEST_F(ShapePartitionAssignerTest, Shape31WithLayout01) {
-  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {3, 1}, {0, 1}), 1);
+  std::vector<int64> expected_partitions[] = {
+      {1} /* 1 */, {1, 2} /* 2 */
+  };
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {3, 1}, {0, 1}), 2,
+            expected_partitions);
 }
 
 TEST_F(ShapePartitionAssignerTest, Shape53WithLayout10) {
-  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3}, {1, 0}), 5);
+  std::vector<int64> expected_partitions[] = {{1} /* 1 */, {2} /* 2 */,
+                                              {3} /* 3 */, {4} /* 4 */,
+                                              {5} /* 5 */, {3, 2} /* 6 */};
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3}, {1, 0}), 6,
+            expected_partitions);
 }
 
 TEST_F(ShapePartitionAssignerTest, Shape53WithLayout01) {
-  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3}, {0, 1}), 3);
+  std::vector<int64> expected_partitions[] = {
+      {1} /* 1 */, {2} /* 2 */, {3} /* 3 */, {2, 2} /* 4 */};
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3}, {0, 1}), 4,
+            expected_partitions);
 }
 
 TEST_F(ShapePartitionAssignerTest, Shape532WithLayout210) {
-  Shape shape = ShapeUtil::MakeShapeWithLayout(F32, {5, 3, 2}, {2, 1, 0});
-  ShapePartitionAssigner assigner(shape);
-
-  for (int64 i = 1; i <= 5; ++i) {
-    EXPECT_TRUE(ContainersEqual(Vec({i}), assigner.Run(
-                                              /*target_partition_count=*/i)));
-  }
-
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 2}), assigner.Run(/*target_partition_count=*/6)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 2}), assigner.Run(/*target_partition_count=*/7)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({4, 2}), assigner.Run(/*target_partition_count=*/8)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 3}), assigner.Run(/*target_partition_count=*/9)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 3}),
-                              assigner.Run(/*target_partition_count=*/10)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 3}),
-                              assigner.Run(/*target_partition_count=*/11)));
-  EXPECT_TRUE(ContainersEqual(Vec({4, 3}),
-                              assigner.Run(/*target_partition_count=*/12)));
-  EXPECT_TRUE(ContainersEqual(Vec({4, 3}),
-                              assigner.Run(/*target_partition_count=*/13)));
-  EXPECT_TRUE(ContainersEqual(Vec({4, 3}),
-                              assigner.Run(/*target_partition_count=*/14)));
-  EXPECT_TRUE(ContainersEqual(Vec({5, 3}),
-                              assigner.Run(/*target_partition_count=*/15)));
-  EXPECT_TRUE(ContainersEqual(Vec({5, 3}),
-                              assigner.Run(/*target_partition_count=*/16)));
+  std::vector<int64> expected_partitions[] = {
+      {1} /* 1 */,     {2} /* 2 */,     {3} /* 3 */,     {4} /* 4 */,
+      {5} /* 5 */,     {3, 2} /* 6 */,  {3, 2} /* 7 */,  {4, 2} /* 8 */,
+      {3, 3} /* 9 */,  {3, 3} /* 10 */, {3, 3} /* 11 */, {4, 3} /* 12 */,
+      {4, 3} /* 13 */, {4, 3} /* 14 */, {5, 3} /* 15 */, {4, 2, 2} /* 16 */};
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3, 2}, {2, 1, 0}), 16,
+            expected_partitions);
 }
 
 TEST_F(ShapePartitionAssignerTest, Shape532WithLayout201) {
-  Shape shape = ShapeUtil::MakeShapeWithLayout(F32, {5, 3, 2}, {2, 0, 1});
-  ShapePartitionAssigner assigner(shape);
-
-  for (int64 i = 1; i <= 3; ++i) {
-    EXPECT_TRUE(ContainersEqual(Vec({i}), assigner.Run(
-                                              /*target_partition_count=*/i)));
-  }
-
-  EXPECT_TRUE(
-      ContainersEqual(Vec({2, 2}), assigner.Run(/*target_partition_count=*/4)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({2, 2}), assigner.Run(/*target_partition_count=*/5)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 2}), assigner.Run(/*target_partition_count=*/6)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 2}), assigner.Run(/*target_partition_count=*/7)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 2}), assigner.Run(/*target_partition_count=*/8)));
-  EXPECT_TRUE(
-      ContainersEqual(Vec({3, 3}), assigner.Run(/*target_partition_count=*/9)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 3}),
-                              assigner.Run(/*target_partition_count=*/10)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 3}),
-                              assigner.Run(/*target_partition_count=*/11)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 4}),
-                              assigner.Run(/*target_partition_count=*/12)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 4}),
-                              assigner.Run(/*target_partition_count=*/13)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 4}),
-                              assigner.Run(/*target_partition_count=*/14)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 5}),
-                              assigner.Run(/*target_partition_count=*/15)));
-  EXPECT_TRUE(ContainersEqual(Vec({3, 5}),
-                              assigner.Run(/*target_partition_count=*/16)));
+  std::vector<int64> expected_partitions[] = {
+      {1} /* 1 */,     {2} /* 2 */,     {3} /* 3 */,     {2, 2} /* 4 */,
+      {2, 2} /* 5 */,  {3, 2} /* 6 */,  {3, 2} /* 7 */,  {3, 2} /* 8 */,
+      {3, 3} /* 9 */,  {3, 3} /* 10 */, {3, 3} /* 11 */, {3, 4} /* 12 */,
+      {3, 4} /* 13 */, {3, 4} /* 14 */, {3, 5} /* 15 */, {3, 2, 2} /* 16 */};
+  RunR2Test(ShapeUtil::MakeShapeWithLayout(F32, {5, 3, 2}, {2, 0, 1}), 16,
+            expected_partitions);
 }
 
 class ShapePartitionIteratorTest : public HloTestBase {
diff --git a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
index 6fe7737de7..b28fe0c15a 100644
--- a/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
+++ b/tensorflow/compiler/xla/tests/exhaustive_f32_elementwise_op_test.cc
@@ -71,8 +71,8 @@ XLA_TEST_P(ExhaustiveF32ElementwiseOpTest, LogF32) {
 #ifdef XLA_TEST_BACKEND_CPU
   // TODO(b/73141998): The vectorized Log implementation gives results outside
   // our error spec in this range (these numbers are bitwise representations of
-  // floats expressed as a zero extended int64):
-  std::pair<int64, int64> known_incorrect_range = {1, 8315654};
+  // floats expressed as a zero extended int64).
+  std::pair<int64, int64> known_incorrect_range = {1, 8388608};
 #else
   std::pair<int64, int64> known_incorrect_range = {0, 0};
 #endif
-- 
GitLab


From 1c38584cb9793642928bf888be1a98698d3b8c44 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Mon, 26 Mar 2018 23:34:05 -0700
Subject: [PATCH 533/960] Fix acknowledgment to say "Blade Team of Tencent" in
 security.md file.

Team is incorrectly referred to as "TenCent Blade Team"

PiperOrigin-RevId: 190577449
---
 SECURITY.md | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/SECURITY.md b/SECURITY.md
index 5ca304404d..a5ce3a62ee 100644
--- a/SECURITY.md
+++ b/SECURITY.md
@@ -244,7 +244,7 @@ v//Fw6ZeY+HmRDFdirjD7wXtIuER4vqCryIqR6Xe9X8oJXz9L/Jhslc=
 
 ### Known vulnerabilities
 
-| Type               | Versions affected |        Reported by | Additional Information      |
-|--------------------|:-----------------:|--------------------|-----------------------------|
-| Out Of Bounds Read |             <=1.4 | TenCent Blade Team | [issue report](https://github.com/tensorflow/tensorflow/issues/14959) |
+| Type               | Versions affected | Reported by           | Additional Information      |
+|--------------------|:-----------------:|-----------------------|-----------------------------|
+| Out Of Bounds Read |             <=1.4 | Blade Team of Tencent | [issue report](https://github.com/tensorflow/tensorflow/issues/14959) |
 
-- 
GitLab


From c66b2ed3c23240be3d6a4a609e5b87c109fb0cea Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Tue, 27 Mar 2018 00:02:42 -0700
Subject: [PATCH 534/960] Remove broken ibiblio url

I suspect ibiblio selectively mirrors or perhaps only mirrors
highly popular artifacts.

PiperOrigin-RevId: 190578860
---
 tensorflow/workspace.bzl | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ebb9e9412f..206a5a3d99 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -576,7 +576,6 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       jar_urls = [
           "http://mirror.bazel.build/repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
           "http://repo1.maven.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
-          "http://maven.ibiblio.org/maven2/com/google/testing/compile/compile-testing/0.11/compile-testing-0.11.jar",
       ],
       licenses = ["notice"],  # New BSD License
       testonly_ = True,
-- 
GitLab


From 7555534be3c6138cbcca138556fe4dbf4cc6b8ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 00:30:02 -0700
Subject: [PATCH 535/960] Handle out of range values when casting from floating
 point to integer in quantize.

PiperOrigin-RevId: 190580805
---
 .../lite/kernels/internal/quantization_util.h |  69 +++++++++-
 .../internal/quantization_util_test.cc        | 126 ++++++++++++++++++
 .../toco/graph_transformations/quantize.cc    |   7 +-
 3 files changed, 195 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util.h b/tensorflow/contrib/lite/kernels/internal/quantization_util.h
index f7706c7938..9a04b76e56 100644
--- a/tensorflow/contrib/lite/kernels/internal/quantization_util.h
+++ b/tensorflow/contrib/lite/kernels/internal/quantization_util.h
@@ -97,6 +97,71 @@ QuantizationParams ChooseQuantizationParams(double rmin, double rmax) {
   return quantization_params;
 }
 
+// Converts a floating-point number to an integer. For all inputs x where
+// static_cast<IntOut>(x) is legal according to the C++ standard, the result
+// is identical to that cast (i.e. the result is x with its fractional part
+// truncated whenever that is representable as IntOut).
+//
+// static_cast would cause undefined behavior for the following cases, which
+// have well-defined behavior for this function:
+//
+//  1. If x is NaN, the result is zero.
+//
+//  2. If the truncated form of x is above the representable range of IntOut,
+//     the result is std::numeric_limits<IntOut>::max().
+//
+//  3. If the truncated form of x is below the representable range of IntOut,
+//     the result is std::numeric_limits<IntOut>::min().
+//
+// Note that cases #2 and #3 cover infinities as well as finite numbers.
+//
+// The range of FloatIn must include the range of IntOut, otherwise
+// the results are undefined.
+// TODO(sfeuz): Replace by absl::SafeCast once available.
+template <class IntOut, class FloatIn>
+IntOut SafeCast(FloatIn x) {
+  static_assert(!std::numeric_limits<FloatIn>::is_integer,
+                "FloatIn is integer");
+  static_assert(std::numeric_limits<IntOut>::is_integer,
+                "IntOut is not integer");
+  static_assert(std::numeric_limits<IntOut>::radix == 2, "IntOut is base 2");
+
+  // Special case NaN, for which the logic below doesn't work.
+  if (std::isnan(x)) {
+    return 0;
+  }
+
+  // Negative values all clip to zero for unsigned results.
+  if (!std::numeric_limits<IntOut>::is_signed && x < 0) {
+    return 0;
+  }
+
+  // Handle infinities.
+  if (std::isinf(x)) {
+    return x < 0 ? std::numeric_limits<IntOut>::min()
+                 : std::numeric_limits<IntOut>::max();
+  }
+
+  // Set exp such that x == f * 2^exp for some f with |f| in [0.5, 1.0),
+  // unless x is zero in which case exp == 0. Note that this implies that the
+  // magnitude of x is strictly less than 2^exp.
+  int exp = 0;
+  std::frexp(x, &exp);
+
+  // Let N be the number of non-sign bits in the representation of IntOut. If
+  // the magnitude of x is strictly less than 2^N, the truncated version of x
+  // is representable as IntOut. The only representable integer for which this
+  // is not the case is kMin for signed types (i.e. -2^N), but that is covered
+  // by the fall-through below.
+  if (exp <= std::numeric_limits<IntOut>::digits) {
+    return x;
+  }
+
+  // Handle numbers with magnitude >= 2^N.
+  return x < 0 ? std::numeric_limits<IntOut>::min()
+               : std::numeric_limits<IntOut>::max();
+}
+
 // Decompose a double multiplier into a Q0.31 int32 representation of its
 // significand, and shift representation of NEGATIVE its exponent ---
 // this is intended as a RIGHT-shift.
@@ -135,8 +200,8 @@ void PreprocessSoftmaxScaling(double beta, double input_scale,
 // Calculate the largest input that will result in a within-bounds intermediate
 // result within MultiplyByQuantizedMultiplierGreaterThanOne.  In other words,
 // it must not overflow before we reduce the value by multiplication by the
-// input multiplier.  The negative radius is used as the minimum difference
-// in Softmax.
+// input multiplier.  The negative radius is used as the minimum difference in
+// Softmax.
 int CalculateInputRadius(int input_integer_bits, int input_left_shift);
 
 }  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/internal/quantization_util_test.cc b/tensorflow/contrib/lite/kernels/internal/quantization_util_test.cc
index 4ae2085c30..3e9a3c29ee 100644
--- a/tensorflow/contrib/lite/kernels/internal/quantization_util_test.cc
+++ b/tensorflow/contrib/lite/kernels/internal/quantization_util_test.cc
@@ -22,6 +22,132 @@ namespace {
 
 using ::testing::Pair;
 
+template <class FloatIn, class IntOut>
+void RunSafeCastTests() {
+  const IntOut imax = std::numeric_limits<IntOut>::max();
+  EXPECT_GT(imax, 0);
+  const IntOut imin = std::numeric_limits<IntOut>::min();
+  const bool s = std::numeric_limits<IntOut>::is_signed;
+  if (s) {
+    EXPECT_LT(imin, 0);
+  } else {
+    EXPECT_EQ(0, imin);
+  }
+
+  // Some basic tests.
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(0.0)), 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-0.0)), 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(0.99)), 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(1.0)), 1);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(1.01)), 1);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(1.99)), 1);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(2.0)), 2);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(2.01)), 2);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-0.99)), 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-1.0)), s ? -1 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-1.01)), s ? -1 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-1.99)), s ? -1 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-2.0)), s ? -2 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-2.01)), s ? -2 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(117.9)), 117);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(118.0)), 118);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(118.1)), 118);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-117.9)), s ? -117 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-118.0)), s ? -118 : 0);
+  EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(-118.1)), s ? -118 : 0);
+
+  // Some edge cases.
+  EXPECT_EQ(SafeCast<IntOut>(std::numeric_limits<FloatIn>::max()), imax);
+  EXPECT_EQ(SafeCast<IntOut>(std::numeric_limits<FloatIn>::lowest()), imin);
+  EXPECT_EQ(SafeCast<IntOut>(std::numeric_limits<FloatIn>::infinity()), imax);
+  EXPECT_EQ(SafeCast<IntOut>(-std::numeric_limits<FloatIn>::infinity()), imin);
+  EXPECT_EQ(SafeCast<IntOut>(std::numeric_limits<FloatIn>::quiet_NaN()), 0);
+
+  // Some larger numbers.
+  if (sizeof(IntOut) >= 4 && sizeof(FloatIn) > 4) {
+    EXPECT_EQ(SafeCast<IntOut>(static_cast<FloatIn>(0x76543210)), 0x76543210);
+  }
+
+  if (sizeof(FloatIn) > sizeof(IntOut)) {
+    // Check values near imax.
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) + 0.1)),
+              imax);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) + 0.99)),
+              imax);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) + 1.0)),
+              imax);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) + 1.99)),
+              imax);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) + 2.0)),
+              imax);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 0.1)),
+              imax - 1);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 0.99)),
+              imax - 1);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 1.0)),
+              imax - 1);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 1.01)),
+              imax - 2);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 1.99)),
+              imax - 2);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 2.0)),
+              imax - 2);
+    EXPECT_EQ(SafeCast<IntOut>(
+                  static_cast<FloatIn>(static_cast<FloatIn>(imax) - 2.01)),
+              imax - 3);
+  }
+
+  // Check values considerably larger in magnitude than imin and imax
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imax) * 2)),
+      imax);
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imax) * 20)),
+      imax);
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imax) * 100)),
+      imax);
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imin) * 2)),
+      imin);
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imin) * 20)),
+      imin);
+  EXPECT_EQ(
+      SafeCast<IntOut>(static_cast<FloatIn>(static_cast<FloatIn>(imin) * 100)),
+      imin);
+}
+
+TEST(QuantizationUtilTest, SafeCast) {
+  RunSafeCastTests<float, int8>();
+  RunSafeCastTests<double, int8>();
+  RunSafeCastTests<float, int16>();
+  RunSafeCastTests<double, int16>();
+  RunSafeCastTests<float, int32>();
+  RunSafeCastTests<double, int32>();
+  RunSafeCastTests<float, int64>();
+  RunSafeCastTests<double, int64>();
+  RunSafeCastTests<float, uint8>();
+  RunSafeCastTests<double, uint8>();
+  RunSafeCastTests<float, uint16>();
+  RunSafeCastTests<double, uint16>();
+  RunSafeCastTests<float, uint32>();
+  RunSafeCastTests<double, uint32>();
+  RunSafeCastTests<float, uint64>();
+  RunSafeCastTests<double, uint64>();
+}
+
 // Example taken from http://www.tensorflow.org/performance/quantization
 //
 //  Quantized | Float
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
index ad3f05274b..9679ea0a77 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
@@ -65,8 +65,6 @@ std::unique_ptr<GenericBuffer> QuantizeBuffer(
       static_cast<const Buffer<ArrayDataType::kFloat>&>(buffer);
   auto* quantized_buffer = new Buffer<A>;
   quantized_buffer->data.resize(float_buffer.data.size());
-  const auto qmin = static_cast<int32>(std::numeric_limits<DataType<A>>::min());
-  const auto qmax = static_cast<int32>(std::numeric_limits<DataType<A>>::max());
   for (std::size_t i = 0; i < float_buffer.data.size(); i++) {
     const float src_val = float_buffer.data[i];
     double scaled_val;  // Astonishingly, using 'float' degrades accuracy just
@@ -78,9 +76,8 @@ std::unique_ptr<GenericBuffer> QuantizeBuffer(
     } else {
       scaled_val = quantization_params.zero_point + inverse_scale * src_val;
     }
-    const auto rounded_val = static_cast<int32>(std::round(scaled_val));
-    const auto clamped_val = std::min(qmax, std::max(qmin, rounded_val));
-    quantized_buffer->data[i] = static_cast<DataType<A>>(clamped_val);
+    quantized_buffer->data[i] =
+        tflite::SafeCast<DataType<A>>(std::round(scaled_val));
   }
   return std::unique_ptr<GenericBuffer>(quantized_buffer);
 }
-- 
GitLab


From 1c055f0679ea6cdae28b3c78c3bf98cb40f00e13 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 03:23:58 -0700
Subject: [PATCH 536/960] Avoid reading the input file twice for
 InitializableLookupTable in combination with HashTable.

Before this cl, TextFileLineIterator::total_size() was called for HashTable::DoPrepare, even though HashTable::DoPrepare ignores the size parameter.
In order to have a result ready for TextFileLineIterator::total_size(), Init() called GetNumLinesInTextFile(), which read the whole file. Just to throw away the result :-/

This cl:
- adds a DoLazyPrepare, that gets a functor to get the size, only if needed.
- add HashTable::DoLazyPrepare which does not call this functor.
- modify TextFileLineIterator::Init() to not call GetNumLinesInTextFile() anymore, when vocab_size was given as -1.
- modify TextFileLineIterator::total_size() to call GetNumLinesInTextFile() lazily on the first call, if vocab_size_ was passed as -1.

PiperOrigin-RevId: 190593744
---
 .../kernels/initializable_lookup_table.cc     |  2 +-
 .../core/kernels/initializable_lookup_table.h | 12 ++++++++++
 tensorflow/core/kernels/lookup_table_op.h     |  5 ++++
 tensorflow/core/kernels/lookup_util.cc        | 24 +++++++++++++------
 4 files changed, 35 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/kernels/initializable_lookup_table.cc b/tensorflow/core/kernels/initializable_lookup_table.cc
index 9c428cdedc..06d53eba30 100644
--- a/tensorflow/core/kernels/initializable_lookup_table.cc
+++ b/tensorflow/core/kernels/initializable_lookup_table.cc
@@ -44,7 +44,7 @@ Status InitializableLookupTable::Initialize(InitTableIterator& iter) {
     return errors::FailedPrecondition("Table already initialized.");
   }
 
-  TF_RETURN_IF_ERROR(DoPrepare(iter.total_size()));
+  TF_RETURN_IF_ERROR(DoLazyPrepare([&iter]() { return iter.total_size(); }));
   while (iter.Valid()) {
     TF_RETURN_IF_ERROR(DoInsert(iter.keys(), iter.values()));
     iter.Next();
diff --git a/tensorflow/core/kernels/initializable_lookup_table.h b/tensorflow/core/kernels/initializable_lookup_table.h
index e9eae9f863..b16c76dc7f 100644
--- a/tensorflow/core/kernels/initializable_lookup_table.h
+++ b/tensorflow/core/kernels/initializable_lookup_table.h
@@ -114,6 +114,7 @@ class InitializableLookupTable : public LookupInterface {
     virtual Status status() const = 0;
 
     // Returns the total number of elements that the iterator will produce.
+    // It might return -1 in case of error.
     virtual int64 total_size() const = 0;
 
    private:
@@ -129,6 +130,17 @@ class InitializableLookupTable : public LookupInterface {
   // number of expected elements.
   virtual Status DoPrepare(size_t expected_num_elements) = 0;
 
+  // Same as DoPrepare() but derived implementations might choose to skip
+  // calling get_expected_num_elements if size is not needed for DoPrepare.
+  virtual Status DoLazyPrepare(
+      std::function<int64(void)> get_expected_num_elements) {
+    int64 expected_num_elements = get_expected_num_elements();
+    if (expected_num_elements < 0) {
+      return errors::FailedPrecondition("Got negative expected_num_elements.");
+    }
+    return DoPrepare(expected_num_elements);
+  }
+
   // Populates the table in batches given keys and values as tensors into the
   // underlying data structure.
   virtual Status DoInsert(const Tensor& keys, const Tensor& values) = 0;
diff --git a/tensorflow/core/kernels/lookup_table_op.h b/tensorflow/core/kernels/lookup_table_op.h
index 5ba9b936e4..3657fd5b6a 100644
--- a/tensorflow/core/kernels/lookup_table_op.h
+++ b/tensorflow/core/kernels/lookup_table_op.h
@@ -191,6 +191,11 @@ class HashTable : public InitializableLookupTable {
     return Status::OK();
   };
 
+  Status DoLazyPrepare(std::function<int64(void)> unused) override {
+    constexpr size_t kUnusedSize = 0;
+    return DoPrepare(kUnusedSize);
+  }
+
   Status DoInsert(const Tensor& keys, const Tensor& values) override {
     if (!table_) {
       return errors::FailedPrecondition("HashTable is not prepared.");
diff --git a/tensorflow/core/kernels/lookup_util.cc b/tensorflow/core/kernels/lookup_util.cc
index c7ce1c3747..27031d9216 100644
--- a/tensorflow/core/kernels/lookup_util.cc
+++ b/tensorflow/core/kernels/lookup_util.cc
@@ -75,9 +75,6 @@ class TextFileLineIterator
   Status Init(const string& filename, int64 vocab_size, char delimiter,
               DataType key_dtype, int64 key_index, DataType value_dtype,
               int64 value_index, Env* env) {
-    if (vocab_size == -1) {
-      TF_RETURN_IF_ERROR(GetNumLinesInTextFile(env, filename, &vocab_size));
-    }
     filename_ = filename;
     vocab_size_ = vocab_size;
     delimiter_ = delimiter;
@@ -85,6 +82,7 @@ class TextFileLineIterator
     value_ = Tensor(value_dtype, TensorShape({}));
     key_index_ = key_index;
     value_index_ = value_index;
+    env_ = env;
 
     status_ = env->NewRandomAccessFile(filename_, &file_);
     if (!status_.ok()) return status_;
@@ -103,15 +101,15 @@ class TextFileLineIterator
     string line;
     status_ = input_buffer_->ReadLine(&line);
     if (!status_.ok()) {
-      if (errors::IsOutOfRange(status_) && next_id_ != vocab_size_) {
+      if (errors::IsOutOfRange(status_) && next_id_ != total_size()) {
         status_ = errors::InvalidArgument("Invalid vocab_size in ", filename_,
-                                          ": expected ", vocab_size_,
+                                          ": expected ", total_size(),
                                           " but got ", next_id_);
       }
       valid_ = false;
       return;
     }
-    if (next_id_ >= vocab_size_) {
+    if (vocab_size_ != -1 && next_id_ >= vocab_size_) {
       LOG(WARNING) << "Truncated " << filename_ << " before its end at "
                    << vocab_size_ << " records.";
       LOG(WARNING) << "next_id_  : " << next_id_;
@@ -162,7 +160,18 @@ class TextFileLineIterator
 
   Status status() const override { return status_; }
 
-  int64 total_size() const override { return vocab_size_; }
+  int64 total_size() const override {
+    if (vocab_size_ == -1) {
+      int64 new_size;
+      Status status = GetNumLinesInTextFile(env_, filename_, &new_size);
+      if (!status.ok()) {
+        LOG(WARNING) << "Unable to get line count: " << status;
+        new_size = -1;
+      }
+      *const_cast<int64*>(&vocab_size_) = new_size;
+    }
+    return vocab_size_;
+  }
 
  private:
   Tensor key_;
@@ -170,6 +179,7 @@ class TextFileLineIterator
   bool valid_;  // true if the iterator points to an existing range.
   int64 key_index_;
   int64 value_index_;
+  Env* env_;
   int64 next_id_;
   int64 vocab_size_;
   string filename_;
-- 
GitLab


From 3d9f820ff2b4c7e79f9e3239b2a09472e99448e2 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 03:48:57 -0700
Subject: [PATCH 537/960] Don't flush denormals when calling
 Eigen::SelfAdjointEigenSolver.

PiperOrigin-RevId: 190595222
---
 tensorflow/BUILD                              |  1 +
 tensorflow/contrib/cmake/python_modules.txt   |  1 +
 tensorflow/core/kernels/BUILD                 |  4 +--
 .../core/kernels/self_adjoint_eig_op.cc       |  4 +++
 .../kernels/self_adjoint_eig_v2_op_impl.h     |  4 +++
 tensorflow/python/kernel_tests/BUILD          |  1 +
 .../kernel_tests/self_adjoint_eig_op_test.py  | 17 ++++++++++
 tensorflow/python/kernel_tests/testdata/BUILD | 24 ++++++++++++++
 ...lf_adjoint_eig_fail_if_denorms_flushed.txt | 32 +++++++++++++++++++
 tensorflow/tools/pip_package/BUILD            |  1 +
 10 files changed, 87 insertions(+), 2 deletions(-)
 create mode 100644 tensorflow/python/kernel_tests/testdata/BUILD
 create mode 100644 tensorflow/python/kernel_tests/testdata/self_adjoint_eig_fail_if_denorms_flushed.txt

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index b073adfee9..6ab43638ba 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -654,6 +654,7 @@ filegroup(
         "//tensorflow/python/kernel_tests/distributions:all_files",
         "//tensorflow/python/kernel_tests/linalg:all_files",
         "//tensorflow/python/kernel_tests/random:all_files",
+        "//tensorflow/python/kernel_tests/testdata:all_files",
         "//tensorflow/python/ops/distributions:all_files",
         "//tensorflow/python/ops/linalg:all_files",
         "//tensorflow/python/ops/losses:all_files",
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index f7d3c73b2c..112b690511 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -82,6 +82,7 @@ tensorflow/python/kernel_tests
 tensorflow/python/kernel_tests/distributions
 tensorflow/python/kernel_tests/linalg
 tensorflow/python/kernel_tests/random
+tensorflow/python/kernel_tests/testdata
 tensorflow/python/layers
 tensorflow/python/lib
 tensorflow/python/lib/core
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index 9bb80eb892..b469c01881 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -2528,13 +2528,13 @@ tf_kernel_library(
 tf_kernel_library(
     name = "self_adjoint_eig_op",
     prefix = "self_adjoint_eig_op",
-    deps = LINALG_DEPS,
+    deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"],
 )
 
 tf_kernel_library(
     name = "self_adjoint_eig_v2_op",
     prefix = "self_adjoint_eig_v2_op",
-    deps = LINALG_DEPS + if_cuda([
+    deps = LINALG_DEPS + ["//tensorflow/core:lib_internal"] + if_cuda([
         ":cast_op",
         ":cwise_op",
     ]),
diff --git a/tensorflow/core/kernels/self_adjoint_eig_op.cc b/tensorflow/core/kernels/self_adjoint_eig_op.cc
index bcd8877390..cea5883db7 100644
--- a/tensorflow/core/kernels/self_adjoint_eig_op.cc
+++ b/tensorflow/core/kernels/self_adjoint_eig_op.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/kernels/linalg_ops_common.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/denormal.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -55,6 +56,9 @@ class SelfAdjointEigOp : public LinearAlgebraOp<Scalar> {
       return;
     }
 
+    // This algorithm relies on denormals, so switch them back on locally.
+    port::ScopedDontFlushDenormal dont_flush_denormals;
+
     Eigen::SelfAdjointEigenSolver<
         Eigen::Matrix<Scalar, Eigen::Dynamic, Eigen::Dynamic, Eigen::RowMajor>>
         es(inputs[0]);
diff --git a/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h b/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h
index 8c0633f422..271dd2c485 100644
--- a/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h
+++ b/tensorflow/core/kernels/self_adjoint_eig_v2_op_impl.h
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/kernels/linalg_ops_common.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/platform/denormal.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -61,6 +62,9 @@ class SelfAdjointEigV2Op : public LinearAlgebraOp<Scalar> {
       return;
     }
 
+    // This algorithm relies on denormals, so switch them back on locally.
+    port::ScopedDontFlushDenormal dont_flush_denormals;
+
     Eigen::SelfAdjointEigenSolver<Matrix> eig(
         inputs[0],
         compute_v_ ? Eigen::ComputeEigenvectors : Eigen::EigenvaluesOnly);
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index dbe1bd437e..228d1c2452 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2724,6 +2724,7 @@ cuda_py_test(
         "//tensorflow/python:linalg_ops",
         "//tensorflow/python:math_ops",
     ],
+    data = ["//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files"],
     shard_count = 20,
 )
 
diff --git a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
index 4de5f4e4db..d2647088c5 100644
--- a/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
+++ b/tensorflow/python/kernel_tests/self_adjoint_eig_op_test.py
@@ -71,6 +71,23 @@ class SelfAdjointEigTest(test.TestCase):
       self.assertAllEqual(val[4], val[5])
       self.assertAllEqual(val[1], val[3])
 
+  def testMatrixThatFailsWhenFlushingDenormsToZero(self):
+    # Test a 32x32 matrix which is known to fail if denorm floats are flushed to
+    # zero.
+    matrix = np.genfromtxt(
+        test.test_src_dir_path(
+            "python/kernel_tests/testdata/"
+            "self_adjoint_eig_fail_if_denorms_flushed.txt")).astype(np.float32)
+    self.assertEqual(matrix.shape, (32, 32))
+    matrix_tensor = constant_op.constant(matrix)
+    with self.test_session(use_gpu=True) as sess:
+      (e, v) = sess.run(linalg_ops.self_adjoint_eig(matrix_tensor))
+      self.assertEqual(e.size, 32)
+      self.assertAllClose(
+          np.matmul(v, v.transpose()), np.eye(32, dtype=np.float32), atol=2e-3)
+      self.assertAllClose(matrix,
+                          np.matmul(np.matmul(v, np.diag(e)), v.transpose()))
+
 
 def SortEigenDecomposition(e, v):
   if v.ndim < 2:
diff --git a/tensorflow/python/kernel_tests/testdata/BUILD b/tensorflow/python/kernel_tests/testdata/BUILD
new file mode 100644
index 0000000000..a4a0dfc139
--- /dev/null
+++ b/tensorflow/python/kernel_tests/testdata/BUILD
@@ -0,0 +1,24 @@
+# Data files for kernel tests.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+filegroup(
+    name = "self_adjoint_eig_op_test_files",
+    srcs = ["self_adjoint_eig_fail_if_denorms_flushed.txt"],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/python/kernel_tests/testdata/self_adjoint_eig_fail_if_denorms_flushed.txt b/tensorflow/python/kernel_tests/testdata/self_adjoint_eig_fail_if_denorms_flushed.txt
new file mode 100644
index 0000000000..d56a690a79
--- /dev/null
+++ b/tensorflow/python/kernel_tests/testdata/self_adjoint_eig_fail_if_denorms_flushed.txt
@@ -0,0 +1,32 @@
+2.60986303e-17 -9.66826148e-21 -1.68610775e-24 -9.16104778e-17 -1.1039539e-18 -1.66460338e-25 -2.12362492e-23 1.90946688e-21 -3.34190535e-22 1.2000634e-18 -7.31782583e-20 2.57851762e-20 -2.55509e-20 -9.54284927e-20 -1.04248315e-17 -5.32450516e-22 -1.81712853e-17 6.0044594e-18 3.96602716e-11 2.89077487e-25 -2.47461475e-25 1.77941757e-24 -7.30388687e-21 -3.84350041e-16 -3.88532388e-21 -4.29928618e-21 4.13551131e-16 -2.63408791e-25 -2.84830375e-21 -1.6450072e-16 -2.8585296e-21 -3.65413296e-21
+-9.66826148e-21 5.03939189e-22 9.17361108e-26 5.17304053e-20 1.99338895e-20 1.25259775e-28 -8.70441942e-26 9.91474109e-25 -5.80960164e-24 -1.19022314e-21 3.90467165e-22 -1.38179098e-22 1.79253406e-22 2.23977705e-22 1.1864143e-19 7.16291934e-24 4.10159639e-20 -2.16798529e-20 -4.95460504e-14 -2.6881406e-27 5.32861213e-27 -4.54567085e-28 1.99794328e-23 1.26854541e-17 -1.92916739e-23 8.60632417e-24 -1.04721097e-18 -7.00607669e-28 6.86771954e-23 8.65173173e-19 1.24469175e-22 6.03883081e-24
+-1.68610775e-24 9.17361108e-26 1.34889529e-26 2.65059e-22 2.39713735e-23 -2.00915344e-30 -1.135692e-27 -6.46049964e-26 -1.03607712e-26 -1.57623654e-23 -1.63805162e-24 -5.95741642e-25 3.24984759e-25 6.49561204e-24 2.28504969e-21 2.8319611e-25 3.96494845e-22 -2.1988623e-22 6.26027228e-16 1.2418479e-30 2.1016041e-30 6.22813846e-30 -1.0708067e-25 6.90778045e-21 1.86361622e-25 7.08789674e-26 -9.23628499e-21 1.65335067e-30 -1.12173032e-26 8.2257321e-22 -4.72686764e-27 -2.58501275e-26
+-9.16104778e-17 5.17304053e-20 2.65059e-22 2.69965968e-14 7.06005733e-17 1.69851446e-22 -2.75994304e-21 -6.61589523e-20 3.8682048e-20 -1.69253147e-17 -2.68580354e-18 -7.74994098e-19 -9.75466696e-19 2.13537585e-18 2.13185342e-16 6.89417478e-21 1.35805044e-16 -3.48309239e-16 1.0448622e-09 -2.17287918e-23 7.41749185e-24 -7.36683057e-23 -1.31083094e-20 1.574e-14 5.72646592e-19 -9.85673749e-21 -1.0654985e-14 2.70679318e-23 4.0943479e-20 -3.42938568e-15 8.57373804e-20 -2.18094505e-20
+-1.1039539e-18 1.99338895e-20 2.39713735e-23 7.06005733e-17 1.83801666e-17 1.09735975e-24 -5.73058223e-24 7.2227645e-22 -8.94843118e-22 -2.30558605e-19 -7.84892038e-20 -1.88692532e-20 -1.02217713e-20 2.95458834e-20 2.42873413e-17 8.89161401e-22 1.21669872e-17 -6.85317731e-18 -7.345906e-12 -3.1158751e-25 1.36359449e-24 -1.57981417e-24 3.89633371e-21 9.94580899e-16 1.45732115e-20 6.92065325e-22 -1.86114433e-16 6.00601346e-26 3.26844e-21 4.38573742e-17 1.06803444e-20 4.60203933e-22
+-1.66460338e-25 1.25259775e-28 -2.00915344e-30 1.69851446e-22 1.09735975e-24 5.75549306e-30 4.74050864e-29 -5.99239043e-28 -1.5784658e-27 -1.74631273e-25 -1.22702975e-25 -1.03371979e-26 -1.96967552e-26 -1.56446725e-26 -3.06462576e-25 -6.33857393e-28 -6.08829397e-24 -7.07478859e-24 -4.82614847e-18 -2.7324345e-31 1.23830207e-31 -7.96172e-31 -1.9034503e-27 -3.82709848e-22 -2.69257733e-26 -3.84934809e-27 -1.48572725e-22 4.14585761e-31 2.5611404e-28 -2.77402858e-24 3.10373361e-28 -5.09669241e-28
+-2.12362492e-23 -8.70441942e-26 -1.135692e-27 -2.75994304e-21 -5.73058223e-24 4.74050864e-29 6.28162e-26 -3.30076462e-25 -3.30065418e-25 -1.1370873e-23 -8.97722764e-24 -1.03190629e-24 -9.52908672e-25 -3.27285413e-24 1.36216664e-22 -8.0549564e-26 -1.94826821e-22 -3.64999226e-22 -2.92500975e-15 -3.00986528e-29 2.39712646e-29 -1.02470704e-28 -4.99034099e-25 -1.32277916e-19 -5.05595e-24 -3.04012473e-25 -1.44724215e-20 5.04614184e-30 -4.12370105e-26 4.20735765e-21 -1.02818953e-25 3.41267575e-26
+1.90946688e-21 9.91474109e-25 -6.46049964e-26 -6.61589523e-20 7.2227645e-22 -5.99239043e-28 -3.30076462e-25 1.8948059e-22 1.83367373e-23 1.06616038e-21 -2.81616502e-22 1.18347412e-22 8.3458038e-23 9.67703245e-24 -1.37445558e-20 2.11412652e-24 2.64820742e-21 8.02510339e-20 4.39926334e-13 9.58727772e-27 2.9838033e-28 1.29183353e-26 1.78626483e-22 3.03531056e-19 9.62612316e-23 1.33722715e-23 2.92905627e-18 -9.42286262e-28 3.23170971e-24 4.10885529e-19 -8.38673724e-25 -8.63732285e-25
+-3.34190535e-22 -5.80960164e-24 -1.03607712e-26 3.8682048e-20 -8.94843118e-22 -1.5784658e-27 -3.30065418e-25 1.83367373e-23 9.30693173e-23 1.48929558e-21 1.83278606e-21 1.08468362e-22 2.61703785e-22 4.42441537e-23 1.23906316e-20 2.55235433e-24 8.36323349e-20 1.2152038e-19 9.83332204e-14 5.14523933e-27 -3.28220159e-28 8.22099066e-27 3.34939233e-23 4.3309476e-19 5.82711129e-22 1.14299394e-22 3.25240717e-18 5.84184241e-28 -1.76991199e-24 5.5568966e-20 -2.80294941e-24 4.59071175e-24
+1.2000634e-18 -1.19022314e-21 -1.57623654e-23 -1.69253147e-17 -2.30558605e-19 -1.74631273e-25 -1.1370873e-23 1.06616038e-21 1.48929558e-21 2.05547703e-18 2.01471341e-20 2.65473229e-20 1.36331708e-20 -2.19777252e-20 -3.09825792e-18 -1.93365673e-22 -2.25608735e-18 7.98997246e-18 1.45582661e-11 6.29004356e-25 -1.14866332e-25 -5.51419319e-26 2.97082139e-21 -2.39052259e-16 1.48920411e-20 1.28589326e-21 4.27717466e-16 -4.44694851e-26 -1.80270052e-22 3.29932795e-18 -5.11645591e-22 5.53091711e-23
+-7.31782583e-20 3.90467165e-22 -1.63805162e-24 -2.68580354e-18 -7.84892038e-20 -1.22702975e-25 -8.97722764e-24 -2.81616502e-22 1.83278606e-21 2.01471341e-20 4.38037939e-19 -4.46678177e-21 3.48516266e-20 7.32592348e-21 1.11928135e-18 8.58541052e-23 8.80645183e-18 4.80109643e-21 -1.7163557e-11 1.92262335e-26 -2.78003951e-26 5.48322572e-25 8.95330117e-23 -1.11570766e-17 3.13666242e-20 4.47195205e-21 -1.09014604e-17 7.69340111e-26 1.64649306e-22 1.71054085e-17 1.33471053e-23 6.40747815e-22
+2.57851762e-20 -1.38179098e-22 -5.95741642e-25 -7.74994098e-19 -1.88692532e-20 -1.03371979e-26 -1.03190629e-24 1.18347412e-22 1.08468362e-22 2.65473229e-20 -4.46678177e-21 5.22731861e-21 1.06412616e-21 -8.0508039e-22 -1.68829721e-19 -2.7699538e-23 -2.15173717e-19 7.46895651e-19 1.71858101e-12 5.41956e-26 -6.15013064e-27 1.54884457e-26 2.54028029e-22 -1.50009535e-18 1.11920465e-21 1.05890428e-22 3.6487132e-17 -2.06798384e-27 -5.5143889e-23 -1.71529414e-18 -7.38099094e-23 -6.5250472e-24
+-2.55509e-20 1.79253406e-22 3.24984759e-25 -9.75466696e-19 -1.02217713e-20 -1.96967552e-26 -9.52908672e-25 8.3458038e-23 2.61703785e-22 1.36331708e-20 3.48516266e-20 1.06412616e-21 4.08927657e-20 -2.76503659e-21 -6.81059804e-20 5.13487959e-23 1.80612902e-18 5.32462054e-19 -3.89327199e-12 3.60012729e-26 -2.5575456e-26 3.14316426e-25 4.56614351e-22 -1.24545392e-17 9.14707146e-21 7.97421952e-22 2.84371096e-17 2.98359736e-26 1.33439467e-23 1.00242743e-17 -4.94476664e-23 3.28816461e-22
+-9.54284927e-20 2.23977705e-22 6.49561204e-24 2.13537585e-18 2.95458834e-20 -1.56446725e-26 -3.27285413e-24 9.67703245e-24 4.42441537e-23 -2.19777252e-20 7.32592348e-21 -8.0508039e-22 -2.76503659e-21 5.02409342e-20 1.57549297e-18 2.63027228e-22 6.11241908e-19 -2.71906856e-19 1.41003203e-12 2.66730019e-26 2.25679315e-26 1.00596535e-25 3.02875382e-22 3.85539387e-17 6.79708607e-22 1.60452617e-22 -2.08440846e-17 -5.40071056e-28 4.56236979e-23 -1.00868521e-17 1.22265047e-22 -1.81997389e-23
+-1.04248315e-17 1.1864143e-19 2.28504969e-21 2.13185342e-16 2.42873413e-17 -3.06462576e-25 1.36216664e-22 -1.37445558e-20 1.23906316e-20 -3.09825792e-18 1.11928135e-18 -1.68829721e-19 -6.81059804e-20 1.57549297e-18 2.5311263e-15 9.97996576e-20 2.26115975e-16 -3.86907114e-17 3.68487445e-12 8.23669787e-24 1.00324064e-23 3.38722042e-24 8.64234911e-21 2.46521189e-15 1.72823337e-19 9.24995431e-20 -3.16903295e-15 5.94130048e-25 1.73965082e-20 1.17371651e-15 2.26718703e-20 4.16709318e-21
+-5.32450516e-22 7.16291934e-24 2.8319611e-25 6.89417478e-21 8.89161401e-22 -6.33857393e-28 -8.0549564e-26 2.11412652e-24 2.55235433e-24 -1.93365673e-22 8.58541052e-23 -2.7699538e-23 5.13487959e-23 2.63027228e-22 9.97996576e-20 2.88326168e-23 1.35358898e-20 5.43364968e-21 4.24011412e-14 1.88486064e-27 8.93106076e-29 4.5748278e-27 2.48573168e-24 5.81165621e-19 1.96505062e-23 5.84813631e-24 -2.46866108e-20 1.912471e-29 2.0243857e-24 -2.88983463e-20 1.35761502e-24 1.40424791e-27
+-1.81712853e-17 4.10159639e-20 3.96494845e-22 1.35805044e-16 1.21669872e-17 -6.08829397e-24 -1.94826821e-22 2.64820742e-21 8.36323349e-20 -2.25608735e-18 8.80645183e-18 -2.15173717e-19 1.80612902e-18 6.11241908e-19 2.26115975e-16 1.35358898e-20 3.66013906e-15 1.35652384e-17 -1.97764849e-09 4.16586597e-24 1.28936031e-24 6.96597122e-23 2.43147439e-21 -1.25627342e-15 1.52711738e-18 2.61025243e-19 -2.00782109e-15 9.75835691e-24 4.0203e-21 1.40790259e-15 -7.8869e-21 8.51983e-20
+6.0044594e-18 -2.16798529e-20 -2.1988623e-22 -3.48309239e-16 -6.85317731e-18 -7.07478859e-24 -3.64999226e-22 8.02510339e-20 1.2152038e-19 7.98997246e-18 4.80109643e-21 7.46895651e-19 5.32462054e-19 -2.71906856e-19 -3.86907114e-17 5.43364968e-21 1.35652384e-17 1.19795414e-15 1.18472676e-09 2.74214961e-23 -7.6305178e-26 1.25969175e-23 1.68466447e-19 1.33873166e-15 1.0739288e-18 1.02533716e-19 2.73480291e-14 -1.87024011e-24 -9.73944425e-21 2.74769918e-16 -1.48632788e-20 1.69142815e-21
+3.96602716e-11 -4.95460504e-14 6.26027228e-16 1.0448622e-09 -7.345906e-12 -4.82614847e-18 -2.92500975e-15 4.39926334e-13 9.83332204e-14 1.45582661e-11 -1.7163557e-11 1.71858101e-12 -3.89327199e-12 1.41003203e-12 3.68487445e-12 4.24011412e-14 -1.97764849e-09 1.18472676e-09 0.0257282555 5.64106473e-17 5.83845666e-18 -1.72409096e-16 1.02886027e-12 1.42563525e-08 -1.57067415e-12 -4.61972799e-13 3.30651737e-08 -5.20615037e-17 -1.71347193e-14 2.87764201e-10 5.03749196e-14 -1.97989316e-13
+2.89077487e-25 -2.6881406e-27 1.2418479e-30 -2.17287918e-23 -3.1158751e-25 -2.7324345e-31 -3.00986528e-29 9.58727772e-27 5.14523933e-27 6.29004356e-25 1.92262335e-26 5.41956e-26 3.60012729e-26 2.66730019e-26 8.23669787e-24 1.88486064e-27 4.16586597e-24 2.74214961e-23 5.64106473e-17 1.2555855e-29 -1.30304595e-31 8.42884087e-31 1.75222077e-26 -2.89058862e-23 3.0225144e-26 6.67962117e-27 8.54181718e-22 -1.2385176e-32 -5.78078369e-28 3.34704626e-23 -2.00599605e-27 2.05674681e-28
+-2.47461475e-25 5.32861213e-27 2.1016041e-30 7.41749185e-24 1.36359449e-24 1.23830207e-31 2.39712646e-29 2.9838033e-28 -3.28220159e-28 -1.14866332e-25 -2.78003951e-26 -6.15013064e-27 -2.5575456e-26 2.25679315e-26 1.00324064e-23 8.93106076e-29 1.28936031e-24 -7.6305178e-26 5.83845666e-18 -1.30304595e-31 2.26490979e-30 -4.25637053e-31 1.40697e-27 5.91197152e-22 -2.08475892e-26 -5.64982671e-28 -3.97199197e-23 -5.06794406e-32 1.11993943e-27 -2.94280711e-23 2.65858181e-27 -2.23093754e-28
+1.77941757e-24 -4.54567085e-28 6.22813846e-30 -7.36683057e-23 -1.57981417e-24 -7.96172e-31 -1.02470704e-28 1.29183353e-26 8.22099066e-27 -5.51419319e-26 5.48322572e-25 1.54884457e-26 3.14316426e-25 1.00596535e-25 3.38722042e-24 4.5748278e-27 6.96597122e-23 1.25969175e-23 -1.72409096e-16 8.42884087e-31 -4.25637053e-31 1.40764294e-28 1.38735442e-26 -1.93810515e-22 1.93660175e-25 1.97417449e-26 1.62145272e-22 2.52533191e-31 -3.42833345e-28 6.34130774e-22 -2.01859e-27 6.1781768e-27
+-7.30388687e-21 1.99794328e-23 -1.0708067e-25 -1.31083094e-20 3.89633371e-21 -1.9034503e-27 -4.99034099e-25 1.78626483e-22 3.34939233e-23 2.97082139e-21 8.95330117e-23 2.54028029e-22 4.56614351e-22 3.02875382e-22 8.64234911e-21 2.48573168e-24 2.43147439e-21 1.68466447e-19 1.02886027e-12 1.75222077e-26 1.40697e-27 1.38735442e-26 1.18400807e-21 1.40670976e-18 2.40320429e-22 3.69528133e-23 4.81603371e-18 -1.49322683e-27 -2.70670724e-25 1.59463723e-19 6.40406749e-24 1.17170599e-23
+-3.84350041e-16 1.26854541e-17 6.90778045e-21 1.574e-14 9.94580899e-16 -3.82709848e-22 -1.32277916e-19 3.03531056e-19 4.3309476e-19 -2.39052259e-16 -1.11570766e-17 -1.50009535e-18 -1.24545392e-17 3.85539387e-17 2.46521189e-15 5.81165621e-19 -1.25627342e-15 1.33873166e-15 1.42563525e-08 -2.89058862e-23 5.91197152e-22 -1.93810515e-22 1.40670976e-18 4.40677789e-12 7.86017934e-19 7.73466606e-19 1.96690791e-15 -1.65941347e-22 2.63659933e-18 -3.0624544e-14 5.87194631e-18 -3.46291098e-19
+-3.88532388e-21 -1.92916739e-23 1.86361622e-25 5.72646592e-19 1.45732115e-20 -2.69257733e-26 -5.05595e-24 9.62612316e-23 5.82711129e-22 1.48920411e-20 3.13666242e-20 1.11920465e-21 9.14707146e-21 6.79708607e-22 1.72823337e-19 1.96505062e-23 1.52711738e-18 1.0739288e-18 -1.57067415e-12 3.0225144e-26 -2.08475892e-26 1.93660175e-25 2.40320429e-22 7.86017934e-19 1.80741048e-20 9.85491491e-22 5.08456938e-17 1.08072265e-26 -1.75036654e-23 4.36436952e-18 -1.77728563e-23 1.01268548e-22
+-4.29928618e-21 8.60632417e-24 7.08789674e-26 -9.85673749e-21 6.92065325e-22 -3.84934809e-27 -3.04012473e-25 1.33722715e-23 1.14299394e-22 1.28589326e-21 4.47195205e-21 1.05890428e-22 7.97421952e-22 1.60452617e-22 9.24995431e-20 5.84813631e-24 2.61025243e-19 1.02533716e-19 -4.61972799e-13 6.67962117e-27 -5.64982671e-28 1.97417449e-26 3.69528133e-23 7.73466606e-19 9.85491491e-22 3.68332283e-22 1.76753773e-18 2.6167718e-27 3.55918682e-25 1.95786374e-19 -2.60077304e-24 1.84790635e-23
+4.13551131e-16 -1.04721097e-18 -9.23628499e-21 -1.0654985e-14 -1.86114433e-16 -1.48572725e-22 -1.44724215e-20 2.92905627e-18 3.25240717e-18 4.27717466e-16 -1.09014604e-17 3.6487132e-17 2.84371096e-17 -2.08440846e-17 -3.16903295e-15 -2.46866108e-20 -2.00782109e-15 2.73480291e-14 3.30651737e-08 8.54181718e-22 -3.97199197e-23 1.62145272e-22 4.81603371e-18 1.96690791e-15 5.08456938e-17 1.76753773e-18 1.57092991e-12 -4.31425852e-23 -3.78241e-19 -1.15899865e-14 -7.61890782e-19 -1.15344546e-19
+-2.63408791e-25 -7.00607669e-28 1.65335067e-30 2.70679318e-23 6.00601346e-26 4.14585761e-31 5.04614184e-30 -9.42286262e-28 5.84184241e-28 -4.44694851e-26 7.69340111e-26 -2.06798384e-27 2.98359736e-26 -5.40071056e-28 5.94130048e-25 1.912471e-29 9.75835691e-24 -1.87024011e-24 -5.20615037e-17 -1.2385176e-32 -5.06794406e-32 2.52533191e-31 -1.49322683e-27 -1.65941347e-22 1.08072265e-26 2.6167718e-27 -4.31425852e-23 1.5576233e-30 -6.14697676e-29 -5.39097603e-24 -8.01112167e-29 1.81063126e-27
+-2.84830375e-21 6.86771954e-23 -1.12173032e-26 4.0943479e-20 3.26844e-21 2.5611404e-28 -4.12370105e-26 3.23170971e-24 -1.76991199e-24 -1.80270052e-22 1.64649306e-22 -5.5143889e-23 1.33439467e-23 4.56236979e-23 1.73965082e-20 2.0243857e-24 4.0203e-21 -9.73944425e-21 -1.71347193e-14 -5.78078369e-28 1.11993943e-27 -3.42833345e-28 -2.70670724e-25 2.63659933e-18 -1.75036654e-23 3.55918682e-25 -3.78241e-19 -6.14697676e-29 2.71732416e-23 2.4136621e-19 2.38938648e-23 1.21468477e-24
+-1.6450072e-16 8.65173173e-19 8.2257321e-22 -3.42938568e-15 4.38573742e-17 -2.77402858e-24 4.20735765e-21 4.10885529e-19 5.5568966e-20 3.29932795e-18 1.71054085e-17 -1.71529414e-18 1.00242743e-17 -1.00868521e-17 1.17371651e-15 -2.88983463e-20 1.40790259e-15 2.74769918e-16 2.87764201e-10 3.34704626e-23 -2.94280711e-23 6.34130774e-22 1.59463723e-19 -3.0624544e-14 4.36436952e-18 1.95786374e-19 -1.15899865e-14 -5.39097603e-24 2.4136621e-19 2.10373291e-13 4.84257897e-20 2.71571227e-19
+-2.8585296e-21 1.24469175e-22 -4.72686764e-27 8.57373804e-20 1.06803444e-20 3.10373361e-28 -1.02818953e-25 -8.38673724e-25 -2.80294941e-24 -5.11645591e-22 1.33471053e-23 -7.38099094e-23 -4.94476664e-23 1.22265047e-22 2.26718703e-20 1.35761502e-24 -7.8869e-21 -1.48632788e-20 5.03749196e-14 -2.00599605e-27 2.65858181e-27 -2.01859e-27 6.40406749e-24 5.87194631e-18 -1.77728563e-23 -2.60077304e-24 -7.61890782e-19 -8.01112167e-29 2.38938648e-23 4.84257897e-20 7.77486414e-23 -7.38542574e-25
+-3.65413296e-21 6.03883081e-24 -2.58501275e-26 -2.18094505e-20 4.60203933e-22 -5.09669241e-28 3.41267575e-26 -8.63732285e-25 4.59071175e-24 5.53091711e-23 6.40747815e-22 -6.5250472e-24 3.28816461e-22 -1.81997389e-23 4.16709318e-21 1.40424791e-27 8.51983e-20 1.69142815e-21 -1.97989316e-13 2.05674681e-28 -2.23093754e-28 6.1781768e-27 1.17170599e-23 -3.46291098e-19 1.01268548e-22 1.84790635e-23 -1.15344546e-19 1.81063126e-27 1.21468477e-24 2.71571227e-19 -7.38542574e-25 3.49516247e-23
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index e01306f953..16c47f7555 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -190,6 +190,7 @@ sh_binary(
             "//tensorflow/python:util_example_parser_configuration",
             "//tensorflow/python/debug:debug_pip",
             "//tensorflow/python/eager:eager_pip",
+            "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files",
             "//tensorflow/python/saved_model:saved_model",
             "//tensorflow/python/tools:tools_pip",
             "//tensorflow/python:test_ops",
-- 
GitLab


From cd98c3ac0e4ab094f00dcb2dfc1188c0c5ee08e0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 08:01:25 -0700
Subject: [PATCH 538/960] - Added support a different strategy for cov
 computations in the multi-tower scenario.  In this strategy we do the cov
 computations locally on each tower and then sum the results, as opposed to
 concatenating everything onto a single device.  This other strategy can be
 enabled by setting the global variable TOWER_STRATEGY to "separate" (default
 value is "concat", which implements the old strategy).  We might change this
 to use "separate" by default if this turns out to be the best default. - The
 code and documentation now no longer refer to the towers as computing
 different "mini-batches", since this was a confusing use of terminology.  The
 best way to think about things is that the combine data over all the towers
 forms the mini-batch.  Note however when factors process multiple towers
 using the "separate" strategy their batch_size variable will still refer to
 the amount of data in a single tower. - Fixed a bug in how the "option 1" and
 "option 2" RNN Fisher approximations were computed in the multi-tower
 scenario. - The "time-folded-into-batch" feature recently added has now
 changed in terms of what format it uses. Time is now the first dimension
 before the reshape, not the second, which is consistent with the convention
 used in other codebases.

PiperOrigin-RevId: 190615398
---
 .../python/kernel_tests/fisher_blocks_test.py |  72 ++--
 .../kernel_tests/fisher_factors_test.py       |  77 +++--
 .../kernel_tests/layer_collection_test.py     |   8 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  | 269 ++++++++++-----
 .../contrib/kfac/python/ops/fisher_factors.py | 317 ++++++++++++------
 .../kfac/python/ops/layer_collection.py       |  46 +--
 tensorflow/contrib/kfac/python/ops/utils.py   |  12 +-
 7 files changed, 525 insertions(+), 276 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
index b70c700f09..6eda6c31e3 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_blocks_test.py
@@ -63,7 +63,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       self.assertAllEqual(params, block.tensors_to_compute_grads())
 
@@ -72,7 +72,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       self.assertAllEqual(params, block.tensors_to_compute_grads())
 
@@ -81,7 +81,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       grads = (params[0]**2, math_ops.sqrt(params[1]))
       block.instantiate_factors(grads, 0.5)
@@ -91,7 +91,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = (params[0]**2, math_ops.sqrt(params[1]))
       block.instantiate_factors((grads,), 0.5)
       block._factor.instantiate_cov_variables()
@@ -112,7 +112,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = array_ops.constant([[1.], [2.]])
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = params**2
       block.instantiate_factors((grads,), 0.5)
       block._factor.instantiate_cov_variables()
@@ -133,7 +133,7 @@ class FullFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.FullFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = (array_ops.constant([2., 3.]), array_ops.constant(4.))
       damping = 0.5
       block.instantiate_factors((grads,), damping)
@@ -163,7 +163,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       self.assertAllEqual(params, block.tensors_to_compute_grads())
 
@@ -172,7 +172,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       self.assertAllEqual(params, block.tensors_to_compute_grads())
 
@@ -181,7 +181,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
 
       grads = (params[0]**2, math_ops.sqrt(params[1]))
       block.instantiate_factors(grads, 0.5)
@@ -191,7 +191,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = (params[0]**2, math_ops.sqrt(params[1]))
       block.instantiate_factors((grads,), 0.5)
       block._factor.instantiate_cov_variables()
@@ -210,7 +210,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = array_ops.constant([[1.], [2.]])
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = params**2
       block.instantiate_factors((grads,), 0.5)
       block._factor.instantiate_cov_variables()
@@ -228,7 +228,7 @@ class NaiveDiagonalFBTest(test.TestCase):
       random_seed.set_random_seed(200)
       params = (array_ops.constant([1., 2.]), array_ops.constant(3.))
       block = fb.NaiveDiagonalFB(lc.LayerCollection(), params)
-      block.register_additional_minibatch(32)
+      block.register_additional_tower(32)
       grads = (params[0]**2, math_ops.sqrt(params[1]))
       damping = 0.5
       block.instantiate_factors((grads,), damping)
@@ -324,8 +324,8 @@ class FullyConnectedDiagonalFBTest(test.TestCase):
 
     self.assertAllClose(expected_result, result)
 
-  def testRegisterAdditionalMinibatch(self):
-    """Ensure 1 big minibatch and 2 small minibatches are equivalent."""
+  def testRegisterAdditionalTower(self):
+    """Ensure 1 big tower and 2 small towers are equivalent."""
     multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps(
         self.w, [self.inputs], [self.outputs], [self.output_grads])
     multiply_result_small, multiply_inverse_result_small = (
@@ -376,7 +376,7 @@ class FullyConnectedDiagonalFBTest(test.TestCase):
       block = fb.FullyConnectedDiagonalFB(
           lc.LayerCollection(), has_bias=isinstance(params, (tuple, list)))
       for (i, o) in zip(inputs, outputs):
-        block.register_additional_minibatch(i, o)
+        block.register_additional_tower(i, o)
 
       block.instantiate_factors((output_grads,), damping=0.0)
       block._factor.instantiate_cov_variables()
@@ -402,7 +402,7 @@ class EmbeddingKFACFBTest(test.TestCase):
       # Add some examples.
       inputs = array_ops.constant([[0, 1], [1, 2], [2, 3]])
       outputs = array_ops.constant([[0.], [1.], [2.]])
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       # Instantiate factor's variables. Ensure it doesn't fail.
       grads = outputs**2.
@@ -420,7 +420,7 @@ class EmbeddingKFACFBTest(test.TestCase):
       # Add some examples.
       inputs = array_ops.constant([[0, 1], [1, 2], [2, 3]])
       outputs = array_ops.constant([[0.], [1.], [2.]])
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       # Instantiate factor's variables. Ensure it doesn't fail.
       grads = outputs**2.
@@ -461,7 +461,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.constant([1., 2.])
       outputs = array_ops.constant([3., 4.])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection())
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       self.assertAllEqual([outputs], block.tensors_to_compute_grads())
 
@@ -471,7 +471,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=True)
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
@@ -482,7 +482,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
@@ -493,7 +493,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.constant([[1., 2., 3.], [3., 4., 5.], [5., 6., 7.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
 
@@ -525,7 +525,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       inputs = array_ops.constant([[1., 2.], [3., 4.]])
       outputs = array_ops.constant([[3., 4.], [5., 6.]])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
       block._input_factor.instantiate_cov_variables()
@@ -553,7 +553,7 @@ class FullyConnectedKFACBasicFBTest(test.TestCase):
       outputs = array_ops.zeros([32, output_dim])
       params = array_ops.zeros([input_dim, output_dim])
       block = fb.FullyConnectedKFACBasicFB(lc.LayerCollection(), has_bias=False)
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       damping = 0.  # This test is only valid without damping.
       block.instantiate_factors(((grads,),), damping)
@@ -689,8 +689,8 @@ class ConvDiagonalFBTest(test.TestCase):
 
     self.assertAllClose(expected_result, result, atol=1e-3)
 
-  def testRegisterAdditionalMinibatch(self):
-    """Ensure 1 big minibatch and 2 small minibatches are equivalent."""
+  def testRegisterAdditionalTower(self):
+    """Ensure 1 big tower and 2 small towers are equivalent."""
     multiply_result_big, multiply_inverse_result_big = self.runFisherBlockOps(
         self.w, [self.inputs], [self.outputs], [self.output_grads])
     multiply_result_small, multiply_inverse_result_small = (
@@ -751,7 +751,7 @@ class ConvDiagonalFBTest(test.TestCase):
       block = fb.ConvDiagonalFB(
           lc.LayerCollection(), params, strides=[1, 1, 1, 1], padding='SAME')
       for (i, o) in zip(inputs, outputs):
-        block.register_additional_minibatch(i, o)
+        block.register_additional_tower(i, o)
 
       block.instantiate_factors((output_grads,), damping=0.0)
       block._factor.instantiate_cov_variables()
@@ -775,7 +775,7 @@ class DepthwiseConvKFCBasicFBTest(test.TestCase):
       layer_collection = lc.LayerCollection()
       block = fb.DepthwiseConvKFCBasicFB(
           layer_collection, params=params, strides=[1, 1, 1, 1], padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       block.instantiate_factors(([grads],), 0.5)
 
@@ -788,7 +788,7 @@ class DepthwiseConvKFCBasicFBTest(test.TestCase):
       layer_collection = lc.LayerCollection()
       block = fb.DepthwiseConvKFCBasicFB(
           layer_collection, params=params, strides=[1, 1, 1, 1], padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       block.instantiate_factors(([grads],), 0.5)
       block._input_factor.instantiate_cov_variables()
@@ -825,7 +825,7 @@ class ConvKFCBasicFBTest(test.TestCase):
       outputs = random_ops.random_normal((2, 2, 2))
       block = fb.ConvKFCBasicFB(
           lc.LayerCollection(), params=params, padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
 
       self.assertAllEqual([outputs], block.tensors_to_compute_grads())
 
@@ -843,7 +843,7 @@ class ConvKFCBasicFBTest(test.TestCase):
       outputs = random_ops.random_normal((2, 2, 2, 2))
       block = fb.ConvKFCBasicFB(
           lc.LayerCollection(), params=params, padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
       block._input_factor.instantiate_cov_variables()
@@ -874,7 +874,7 @@ class ConvKFCBasicFBTest(test.TestCase):
       outputs = random_ops.random_normal((2, 2, 2, 2))
       block = fb.ConvKFCBasicFB(
           lc.LayerCollection(), params=params, padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       self.assertFalse(block._has_bias)
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
@@ -902,7 +902,7 @@ class ConvKFCBasicFBTest(test.TestCase):
       outputs = random_ops.random_normal((2, 2, 2, 2))
       block = fb.ConvKFCBasicFB(
           lc.LayerCollection(), params=params, padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       self.assertTrue(block._has_bias)
       grads = outputs**2
       block.instantiate_factors(((grads,),), 0.5)
@@ -930,7 +930,7 @@ class ConvKFCBasicFBTest(test.TestCase):
       outputs = array_ops.zeros((2, 2, 2, 2))
       block = fb.ConvKFCBasicFB(
           lc.LayerCollection(), params=params, padding='SAME')
-      block.register_additional_minibatch(inputs, outputs)
+      block.register_additional_tower(inputs, outputs)
       grads = outputs**2
       damping = 0.  # This test is only valid without damping.
       block.instantiate_factors(((grads,),), damping)
@@ -964,7 +964,7 @@ class FullyConnectedSeriesFBTest(test.TestCase):
       inputs = array_ops.constant([1., 2.])
       outputs = array_ops.constant([3., 4.])
       block = fb.FullyConnectedSeriesFB(lc.LayerCollection())
-      block.register_additional_minibatch([inputs], [outputs])
+      block.register_additional_tower([inputs], [outputs])
       self.assertAllEqual([[outputs]], block.tensors_to_compute_grads())
 
   def testInstantiateFactorsHasBias(self):
@@ -975,7 +975,7 @@ class FullyConnectedSeriesFBTest(test.TestCase):
       block = fb.FullyConnectedSeriesFB(
           lc.LayerCollection(),
           has_bias=True)
-      block.register_additional_minibatch([inputs], [outputs])
+      block.register_additional_tower([inputs], [outputs])
       grads = outputs**2
       block.instantiate_factors((((grads,),),), 0.5)
 
@@ -987,7 +987,7 @@ class FullyConnectedSeriesFBTest(test.TestCase):
       block = fb.FullyConnectedSeriesFB(
           lc.LayerCollection(),
           has_bias=False)
-      block.register_additional_minibatch([inputs], [outputs])
+      block.register_additional_tower([inputs], [outputs])
       grads = outputs**2
       block.instantiate_factors((((grads,),),), 0.5)
 
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
index e007f70939..2a3592c53f 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/fisher_factors_test.py
@@ -85,6 +85,12 @@ class FisherFactorTestingDummy(ff.FisherFactor):
   def instantiate_inv_variables(self):
     return NotImplementedError
 
+  def _num_towers(self):
+    raise NotImplementedError
+
+  def _get_data_device(self):
+    raise NotImplementedError
+
 
 class InverseProvidingFactorTestingDummy(ff.InverseProvidingFactor):
   """Dummy class to test the non-abstract methods on ff.InverseProvidingFactor.
@@ -116,6 +122,12 @@ class InverseProvidingFactorTestingDummy(ff.InverseProvidingFactor):
   def instantiate_covariance(self):
     pass
 
+  def _num_towers(self):
+    raise NotImplementedError
+
+  def _get_data_device(self):
+    raise NotImplementedError
+
 
 class NumericalUtilsTest(test.TestCase):
 
@@ -430,7 +442,7 @@ class EmbeddingInputKroneckerFactorTest(test.TestCase):
     with tf_ops.Graph().as_default():
       input_ids = array_ops.constant([[0], [1], [4]])
       vocab_size = 5
-      factor = ff.EmbeddingInputKroneckerFactor(input_ids, vocab_size)
+      factor = ff.EmbeddingInputKroneckerFactor((input_ids,), vocab_size)
       factor.instantiate_cov_variables()
       cov = factor.get_cov_var()
       self.assertEqual(cov.shape.as_list(), [vocab_size])
@@ -439,7 +451,7 @@ class EmbeddingInputKroneckerFactorTest(test.TestCase):
     with tf_ops.Graph().as_default():
       input_ids = array_ops.constant([[0], [1], [4]])
       vocab_size = 5
-      factor = ff.EmbeddingInputKroneckerFactor(input_ids, vocab_size)
+      factor = ff.EmbeddingInputKroneckerFactor((input_ids,), vocab_size)
       factor.instantiate_cov_variables()
       cov_update_op = factor.make_covariance_update_op(0.0)
 
@@ -477,8 +489,8 @@ class ConvDiagonalFactorTest(test.TestCase):
       ]
 
       factor = ff.ConvDiagonalFactor(
-          inputs,
-          outputs_grads,
+          (inputs,),
+          (outputs_grads,),
           self.kernel_shape,
           self.strides,
           self.padding,
@@ -508,7 +520,8 @@ class ConvDiagonalFactorTest(test.TestCase):
           self.out_channels)
 
       factor = ff.ConvDiagonalFactor(
-          constant_op.constant(inputs), [constant_op.constant(outputs_grad)],
+          (constant_op.constant(inputs),),
+          ((constant_op.constant(outputs_grad),),),
           self.kernel_shape,
           strides=[1, 1, 1, 1],
           padding='VALID')
@@ -537,8 +550,8 @@ class ConvDiagonalFactorTest(test.TestCase):
       ]
 
       factor = ff.ConvDiagonalFactor(
-          inputs,
-          outputs_grads,
+          (inputs,),
+          (outputs_grads,),
           self.kernel_shape,
           self.strides,
           self.padding,
@@ -569,7 +582,7 @@ class FullyConnectedKroneckerFactorTest(test.TestCase):
     with tf_ops.Graph().as_default():
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c')
-      factor = ff.FullyConnectedKroneckerFactor((tensor,), has_bias=has_bias)
+      factor = ff.FullyConnectedKroneckerFactor(((tensor,),), has_bias=has_bias)
       factor.instantiate_cov_variables()
       cov = factor.get_cov()
       self.assertEqual(cov.dtype, dtype)
@@ -587,7 +600,7 @@ class FullyConnectedKroneckerFactorTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      factor = ff.FullyConnectedKroneckerFactor((tensor,), has_bias=True)
+      factor = ff.FullyConnectedKroneckerFactor(((tensor,),), has_bias=True)
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -598,7 +611,7 @@ class FullyConnectedKroneckerFactorTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      factor = ff.FullyConnectedKroneckerFactor((tensor,))
+      factor = ff.FullyConnectedKroneckerFactor(((tensor,),))
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -629,8 +642,8 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
       out_channels = 4
 
       factor = ff.ConvInputKroneckerFactor(
-          inputs=random_ops.random_uniform(
-              (batch_size, width, width, width, in_channels), seed=0),
+          inputs=(random_ops.random_uniform(
+              (batch_size, width, width, width, in_channels), seed=0),),
           filter_shape=(width, width, width, in_channels, out_channels),
           padding='SAME',
           strides=(2, 2, 2),
@@ -661,8 +674,8 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
       out_channels = 4
 
       factor = ff.ConvInputKroneckerFactor(
-          inputs=random_ops.random_uniform(
-              (batch_size, width, width, in_channels), seed=0),
+          inputs=(random_ops.random_uniform(
+              (batch_size, width, width, in_channels), seed=0),),
           filter_shape=(1, 1, in_channels, out_channels),
           padding='SAME',
           strides=(1, 1, 1, 1),
@@ -691,8 +704,8 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
       out_channels = 4
 
       factor = ff.ConvInputKroneckerFactor(
-          inputs=random_ops.random_uniform(
-              (batch_size, width, width, in_channels), seed=0),
+          inputs=(random_ops.random_uniform(
+              (batch_size, width, width, in_channels), seed=0),),
           filter_shape=(1, 1, in_channels, out_channels),
           padding='SAME',
           strides=(1, 2, 1, 1),
@@ -716,8 +729,8 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
       out_channels = 4
 
       factor = ff.ConvInputKroneckerFactor(
-          inputs=random_ops.random_uniform(
-              (batch_size, width, width, in_channels), seed=0),
+          inputs=(random_ops.random_uniform(
+              (batch_size, width, width, in_channels), seed=0),),
           filter_shape=(3, 3, in_channels, out_channels),
           padding='SAME',
           extract_patches_fn='extract_image_patches',
@@ -739,7 +752,7 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
     with tf_ops.Graph().as_default():
       tensor = array_ops.ones((64, 1, 2, 3), name='a/b/c')
       factor = ff.ConvInputKroneckerFactor(
-          inputs=tensor,
+          inputs=(tensor,),
           filter_shape=(1, 2, 3, 4),
           padding='SAME',
           has_bias=False)
@@ -751,7 +764,7 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
     with tf_ops.Graph().as_default():
       tensor = array_ops.ones((64, 1, 2, 3), name='a/b/c')
       factor = ff.ConvInputKroneckerFactor(
-          tensor, filter_shape=(1, 2, 3, 4), padding='SAME', has_bias=True)
+          (tensor,), filter_shape=(1, 2, 3, 4), padding='SAME', has_bias=True)
       factor.instantiate_cov_variables()
       self.assertEqual([1 * 2 * 3 + 1, 1 * 2 * 3 + 1],
                        factor.get_cov().get_shape().as_list())
@@ -761,7 +774,7 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
       dtype = dtypes.float64_ref
       tensor = array_ops.ones((64, 1, 2, 3), name='a/b/c', dtype=dtypes.float64)
       factor = ff.ConvInputKroneckerFactor(
-          tensor, filter_shape=(1, 2, 3, 4), padding='SAME', has_bias=True)
+          (tensor,), filter_shape=(1, 2, 3, 4), padding='SAME', has_bias=True)
       factor.instantiate_cov_variables()
       cov = factor.get_cov()
       self.assertEqual(cov.dtype, dtype)
@@ -775,7 +788,7 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
           np.arange(1, 1 + np.prod(input_shape)).reshape(input_shape).astype(
               np.float32))
       factor = ff.ConvInputKroneckerFactor(
-          tensor, filter_shape=(1, 1, 1, 1), padding='SAME', has_bias=True)
+          (tensor,), filter_shape=(1, 1, 1, 1), padding='SAME', has_bias=True)
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -794,7 +807,7 @@ class ConvInputKroneckerFactorTest(ConvFactorTestCase):
           np.arange(1, 1 + np.prod(input_shape)).reshape(input_shape).astype(
               np.float32))
       factor = ff.ConvInputKroneckerFactor(
-          tensor, filter_shape=(1, 1, 1, 1), padding='SAME')
+          (tensor,), filter_shape=(1, 1, 1, 1), padding='SAME')
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -810,10 +823,10 @@ class ConvOutputKroneckerFactorTest(ConvFactorTestCase):
       width = 3
       out_channels = width**3
 
-      factor = ff.ConvOutputKroneckerFactor(outputs_grads=[
+      factor = ff.ConvOutputKroneckerFactor(outputs_grads=([
           random_ops.random_uniform(
               (batch_size, width, width, width, out_channels), seed=0)
-      ])
+      ],))
       factor.instantiate_cov_variables()
 
       with self.test_session() as sess:
@@ -829,7 +842,7 @@ class ConvOutputKroneckerFactorTest(ConvFactorTestCase):
     with tf_ops.Graph().as_default():
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3, 4, 5), name='a/b/c')
-      factor = ff.ConvOutputKroneckerFactor((tensor,))
+      factor = ff.ConvOutputKroneckerFactor(((tensor,),))
       factor.instantiate_cov_variables()
       self.assertEqual([5, 5], factor.get_cov().get_shape().as_list())
 
@@ -838,7 +851,7 @@ class ConvOutputKroneckerFactorTest(ConvFactorTestCase):
       dtype = dtypes.float64_ref
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3, 4, 5), dtype=dtype, name='a/b/c')
-      factor = ff.ConvOutputKroneckerFactor((tensor,))
+      factor = ff.ConvOutputKroneckerFactor(((tensor,),))
       factor.instantiate_cov_variables()
       cov = factor.get_cov()
       self.assertEqual(cov.dtype, dtype)
@@ -848,7 +861,7 @@ class ConvOutputKroneckerFactorTest(ConvFactorTestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = np.arange(1, 17).reshape(2, 2, 2, 2).astype(np.float32)
-      factor = ff.ConvOutputKroneckerFactor((array_ops.constant(tensor),))
+      factor = ff.ConvOutputKroneckerFactor(((array_ops.constant(tensor),),))
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -862,7 +875,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default():
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3), name='a/b/c')
-      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=False)
+      factor = ff.FullyConnectedMultiKF(((tensor,),), has_bias=False)
       factor.instantiate_cov_variables()
       self.assertEqual([3, 3], factor.get_cov().get_shape().as_list())
 
@@ -871,7 +884,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
       dtype = dtypes.float64_ref
       random_seed.set_random_seed(200)
       tensor = array_ops.ones((2, 3), dtype=dtype, name='a/b/c')
-      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=False)
+      factor = ff.FullyConnectedMultiKF(((tensor,),), has_bias=False)
       factor.instantiate_cov_variables()
       cov = factor.get_cov()
       self.assertEqual(cov.dtype, dtype)
@@ -881,7 +894,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      factor = ff.FullyConnectedMultiKF((tensor,), has_bias=True)
+      factor = ff.FullyConnectedMultiKF(((tensor,),), has_bias=True)
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
@@ -892,7 +905,7 @@ class FullyConnectedMultiKFTest(test.TestCase):
     with tf_ops.Graph().as_default(), self.test_session() as sess:
       random_seed.set_random_seed(200)
       tensor = array_ops.constant([[1., 2.], [3., 4.]], name='a/b/c')
-      factor = ff.FullyConnectedMultiKF((tensor,))
+      factor = ff.FullyConnectedMultiKF(((tensor,),))
       factor.instantiate_cov_variables()
 
       sess.run(tf_variables.global_variables_initializer())
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
index ba22099340..cb80fca370 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
+++ b/tensorflow/contrib/kfac/python/kernel_tests/layer_collection_test.py
@@ -35,7 +35,7 @@ from tensorflow.python.platform import test
 class MockFisherBlock(object):
   """A fake FisherBlock."""
 
-  num_registered_minibatches = 2
+  num_registered_towers = 2
 
   def __init__(self, name='MockFisherBlock'):
     self.name = name
@@ -468,13 +468,13 @@ class LayerCollectionTest(test.TestCase):
       b = variable_scope.get_variable('b', [3])
       lc = layer_collection.LayerCollection()
       lc.register_fully_connected(w, inputs, outputs)
-      self.assertEqual(lc.fisher_blocks[w].num_registered_minibatches, 1)
+      self.assertEqual(lc.fisher_blocks[w].num_registered_towers, 1)
       with self.assertRaises(KeyError):
         lc.register_fully_connected((w, b), inputs, outputs, reuse=True)
       self.assertNotIn((w, b), lc.fisher_blocks)
-      self.assertEqual(lc.fisher_blocks[w].num_registered_minibatches, 1)
+      self.assertEqual(lc.fisher_blocks[w].num_registered_towers, 1)
       lc.register_fully_connected(w, inputs, outputs, reuse=True)
-      self.assertEqual(lc.fisher_blocks[w].num_registered_minibatches, 2)
+      self.assertEqual(lc.fisher_blocks[w].num_registered_towers, 2)
 
   def testMakeOrGetFactor(self):
     with ops.Graph().as_default():
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index f517e3148f..b04bf76a88 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -75,37 +75,6 @@ def set_global_constants(normalize_damping_power=None, pi_type=None):
     PI_TYPE = pi_type
 
 
-def _make_partitionedtensors_inputs(inputs):
-  """Constructs PartitionedTensor for inputs.
-
-  The purpose of this method is to package up the towers/minibatch dimension
-  of these arrays into PartitionedTensor objects.
-
-  Args:
-    inputs: a 1-D list of Tensors. Index is tower/mini-batch.
-
-  Returns:
-    A PartitionedTensor.
-  """
-  return utils.PartitionedTensor(inputs)
-
-
-def _make_partitionedtensors_grads(grads_list):
-  """Constructs PartitionedTensor for grads_list.
-
-  The purpose of this method is to package up the towers/minibatch dimension
-  of these arrays into PartitionedTensor objects.
-
-  Args:
-    grads_list: 2-D list of Tensors. First index is for source, second
-      index for tower.
-
-  Returns:
-    Tuple of PartitionedTensors, one per source.
-  """
-  return tuple(utils.PartitionedTensor(grads) for grads in grads_list)
-
-
 def normalize_damping(damping, num_replications):
   """Normalize damping after adjusting scale by NORMALIZE_DAMPING_POWER."""
   if NORMALIZE_DAMPING_POWER:
@@ -191,7 +160,7 @@ class FisherBlock(object):
   """Abstract base class for objects modeling approximate Fisher matrix blocks.
 
   Subclasses must implement register_matpower, multiply_matpower,
-  instantiate_factors, tensors_to_compute_grads, and num_registered_minibatches
+  instantiate_factors, tensors_to_compute_grads, and num_registered_towers
   methods.
   """
 
@@ -266,8 +235,8 @@ class FisherBlock(object):
     pass
 
   @abc.abstractproperty
-  def num_registered_minibatches(self):
-    """Number of minibatches registered for this FisherBlock.
+  def num_registered_towers(self):
+    """Number of towers registered for this FisherBlock.
 
     Typically equal to the number of towers in a multi-tower setup.
     """
@@ -319,8 +288,8 @@ class FullFB(FisherBlock):
   def tensors_to_compute_grads(self):
     return self._params
 
-  def register_additional_minibatch(self, batch_size):
-    """Register an additional minibatch.
+  def register_additional_tower(self, batch_size):
+    """Register an additional tower.
 
     Args:
       batch_size: The batch size, used in the covariance estimator.
@@ -328,7 +297,7 @@ class FullFB(FisherBlock):
     self._batch_sizes.append(batch_size)
 
   @property
-  def num_registered_minibatches(self):
+  def num_registered_towers(self):
     return len(self._batch_sizes)
 
   @property
@@ -381,8 +350,8 @@ class NaiveDiagonalFB(FisherBlock):
   def tensors_to_compute_grads(self):
     return self._params
 
-  def register_additional_minibatch(self, batch_size):
-    """Register an additional minibatch.
+  def register_additional_tower(self, batch_size):
+    """Register an additional tower.
 
     Args:
       batch_size: The batch size, used in the covariance estimator.
@@ -390,7 +359,7 @@ class NaiveDiagonalFB(FisherBlock):
     self._batch_sizes.append(batch_size)
 
   @property
-  def num_registered_minibatches(self):
+  def num_registered_towers(self):
     return len(self._batch_sizes)
 
   @property
@@ -398,24 +367,78 @@ class NaiveDiagonalFB(FisherBlock):
     return math_ops.reduce_sum(self._batch_sizes)
 
 
-class InputOutputMultiMinibatch(object):
+class InputOutputMultiTower(object):
   """Mix-in class for blocks with inputs & outputs and multiple mini-batches."""
 
   def __init__(self, *args, **kwargs):
     self.__inputs = []
     self.__outputs = []
-    super(InputOutputMultiMinibatch, self).__init__(*args, **kwargs)
+    super(InputOutputMultiTower, self).__init__(*args, **kwargs)
+
+  def _process_data(self, grads_list):
+    """Process data into the format used by the factors.
+
+    This function takes inputs and grads_lists data and processes it into
+    one of the formats expected by the FisherFactor classes (depending on
+    the value of the global configuration variable TOWER_STRATEGY).
+
+    The initial format of self._inputs is expected to be a list of Tensors
+    over towers. Similarly grads_lists is expected to be a list over sources
+    of such lists.
+
+    If TOWER_STRATEGY is "concat", 'inputs' becomes a tuple containing a single
+    tensor (represented as a PartitionedTensor object) equal to the
+    concatenation (across towers) of all of the elements of self._inputs. And
+    similarly grads_list is formatted into a tuple (over sources) of such
+    tensors (also represented as PartitionedTensors).
+
+    If TOWER_STRATEGY is "separate", formatting of inputs and grads_list
+    remains unchanged from the initial format (although possibly converting
+    from lists into tuples).
+
+    Args:
+      grads_list: grads_list in its initial format (see above).
+
+    Returns:
+      inputs: self._inputs transformed into the appropriate format (see
+        above).
+      grads_list: grads_list transformed into the appropriate format (see
+        above).
+
+    Raises:
+      ValueError: if TOWER_STRATEGY is not one of "separate" or "concat".
+    """
+    inputs = self._inputs
+    # inputs is a list over towers of Tensors
+    # grads_list is a list of list with the first index being sources and the
+    # second being towers.
+    if fisher_factors.TOWER_STRATEGY == "concat":
+      # Merge towers together into a PartitionedTensor. We package it in
+      # a singleton tuple since the factors will expect a list over towers
+      inputs = (utils.PartitionedTensor(inputs),)
+      # Do the same for grads_list but preserve leading sources dimension
+      grads_list = tuple((utils.PartitionedTensor(grads),)
+                         for grads in grads_list)
+    elif fisher_factors.TOWER_STRATEGY == "separate":
+      inputs = tuple(inputs)
+      grads_list = tuple(grads_list)
+
+    else:
+      raise ValueError("Global config variable TOWER_STRATEGY must be one of "
+                       "'concat' or 'separate'.")
+
+    return inputs, grads_list
 
   def tensors_to_compute_grads(self):
     """Tensors to compute derivative of loss with respect to."""
-    return self._outputs
+    return tuple(self._outputs)
 
-  def register_additional_minibatch(self, inputs, outputs):
+  def register_additional_tower(self, inputs, outputs):
     self._inputs.append(inputs)
     self._outputs.append(outputs)
 
   @property
-  def num_registered_minibatches(self):
+  def num_registered_towers(self):
     result = len(self._inputs)
     assert result == len(self._outputs)
     return result
@@ -429,7 +452,7 @@ class InputOutputMultiMinibatch(object):
     return self.__outputs
 
 
-class FullyConnectedDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
+class FullyConnectedDiagonalFB(InputOutputMultiTower, FisherBlock):
   """FisherBlock for fully-connected (dense) layers using a diagonal approx.
 
   Estimates the Fisher Information matrix's diagonal entries for a fully
@@ -466,8 +489,7 @@ class FullyConnectedDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
     super(FullyConnectedDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    inputs = _make_partitionedtensors_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     self._factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedDiagonalFactor,
@@ -500,7 +522,7 @@ class FullyConnectedDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
     return utils.mat2d_to_layer_params(vector, reshaped_out)
 
 
-class ConvDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
+class ConvDiagonalFB(InputOutputMultiTower, FisherBlock):
   """FisherBlock for 2-D convolutional layers using a diagonal approx.
 
   Estimates the Fisher Information matrix's diagonal entries for a convolutional
@@ -580,11 +602,10 @@ class ConvDiagonalFB(InputOutputMultiMinibatch, FisherBlock):
     super(ConvDiagonalFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
-    inputs = _make_partitionedtensors_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     # Infer number of locations upon which convolution is applied.
-    self._num_locations = num_conv_locations(inputs.shape.as_list(),
+    self._num_locations = num_conv_locations(inputs[0].shape.as_list(),
                                              self._strides)
 
     self._factor = self._layer_collection.make_or_get_factor(
@@ -691,7 +712,7 @@ class KroneckerProductFB(FisherBlock):
                                                         right_factor)
 
 
-class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
+class EmbeddingKFACFB(InputOutputMultiTower, KroneckerProductFB):
   """K-FAC FisherBlock for embedding layers.
 
   This FisherBlock is similar to FullyConnectedKFACBasicFB, except that its
@@ -723,8 +744,7 @@ class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
       damping: 0-D Tensor or float. 'damping' * identity is approximately added
         to this FisherBlock's Fisher approximation.
     """
-    inputs = _make_partitionedtensors_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.EmbeddingInputKroneckerFactor,
@@ -734,7 +754,7 @@ class EmbeddingKFACFB(InputOutputMultiMinibatch, KroneckerProductFB):
     self._setup_damping(damping)
 
 
-class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
+class FullyConnectedKFACBasicFB(InputOutputMultiTower, KroneckerProductFB):
   """K-FAC FisherBlock for fully-connected (dense) layers.
 
   This uses the Kronecker-factorized approximation from the original
@@ -764,8 +784,7 @@ class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
       damping: 0-D Tensor or float. 'damping' * identity is approximately added
         to this FisherBlock's Fisher approximation.
     """
-    inputs = _make_partitionedtensors_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.FullyConnectedKroneckerFactor,
@@ -776,7 +795,7 @@ class FullyConnectedKFACBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
     self._setup_damping(damping)
 
 
-class ConvKFCBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
+class ConvKFCBasicFB(InputOutputMultiTower, KroneckerProductFB):
   """FisherBlock for convolutional layers using the basic KFC approx.
 
   Estimates the Fisher Information matrix's blog for a convolutional
@@ -846,8 +865,7 @@ class ConvKFCBasicFB(InputOutputMultiMinibatch, KroneckerProductFB):
     self._num_locations = num_conv_locations(self._inputs[0].shape.as_list(),
                                              self._strides)
 
-    inputs = _make_partitionedtensors_inputs(self._inputs)
-    grads_list = _make_partitionedtensors_grads(grads_list)
+    inputs, grads_list = self._process_data(grads_list)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvInputKroneckerFactor,
@@ -1122,22 +1140,67 @@ def num_conv_locations(input_shape, strides):
   return spatial_input_locations // spatial_strides_divisor
 
 
-class InputOutputMultiMinibatchMultiUse(InputOutputMultiMinibatch):
-  """Adds methods for multi-use/time-step case to InputOutputMultiMinibatch."""
+class InputOutputMultiTowerMultiUse(InputOutputMultiTower):
+  """Adds methods for multi-use/time-step case to InputOutputMultiTower."""
 
   def __init__(self, num_uses=None, *args, **kwargs):
     self._num_uses = num_uses
-    super(InputOutputMultiMinibatchMultiUse, self).__init__(*args, **kwargs)
+    super(InputOutputMultiTowerMultiUse, self).__init__(*args, **kwargs)
 
   def _process_data(self, grads_list):
-    """Process temporal/multi-use data into a standard format."""
+    """Process temporal/multi-use data into the format used by the factors.
+
+    This function takes inputs and grads_lists data and processes it into
+    one of the formats expected by the FisherFactor classes (depending on
+    the value of the global configuration variable TOWER_STRATEGY).
+
+    It accepts the data in one of two initial formats. The first possible
+    format is where self._inputs is a list of list of Tensors. The first index
+    is tower, the second is use/time-step. grads_list, meanwhile, is a list
+    over sources of such lists of lists.
+
+    The second possible data format is where self._inputs is a Tensor with
+    uses/times-steps folded into the batch dimension.  i.e. it is a Tensor
+    of shape [num_uses * size_batch, ...] which represents a reshape of a
+    Tensor of shape [num_uses, size_batch, ...].  And similarly grads_list is
+    a list over sources of such Tensors.
+
+    There are two possible formats which inputs and grads_list are transformed
+    into.
+
+    If TOWER_STRATEGY is "concat", 'inputs' becomes a tuple containing
+    a single tensor (represented as a PartitionedTensor object) with all of
+    the data from the towers, as well as the uses/time-steps, concatenated
+    together. In this tensor the leading dimension is the batch and
+    use/time-step dimensions folded together (with 'use' being the major of
+    these two, so that the tensors can be thought of as reshapes of ones of
+    shape [num_uses, batch_size, ...]). grads_list is similarly formatted as a
+    tuple over sources of such tensors.
+
+    If TOWER_STRATEGY is "separate" the inputs are formatted into lists of
+    tensors over towers. Each of these tensors has a similar format to
+    the tensor produced by the "concat" option, except that each contains
+    only the data from a single tower.  grads_list is similarly formatted
+    into a tuple over sources of such tuples.
+
+    Args:
+      grads_list: grads_list in its initial format (see above).
+
+    Returns:
+      inputs: self._inputs transformed into the appropriate format (see
+        above).
+      grads_list: grads_list transformed into the appropriate format (see
+        above).
+
+    Raises:
+      ValueError: If TOWER_STRATEGY is not one of "separate" or "concat".
+      ValueError: If the given/initial format of self._inputs and grads_list
+        isn't recognized, or doesn't agree with self._num_uses.
+    """
 
     inputs = self._inputs
 
-    # The first possible data format is where inputs is a list of tensors,
-    # one for each use/time-step.
     if isinstance(inputs[0], (list, tuple)):
-      # The first index is tower/minibatch, the second is use/time-step
       num_uses = len(inputs[0])
       if self._num_uses is not None and self._num_uses != num_uses:
         raise ValueError("num_uses argument doesn't match length of inputs.")
@@ -1147,15 +1210,29 @@ class InputOutputMultiMinibatchMultiUse(InputOutputMultiMinibatch):
       # Check that all mini-batches/towers have the same number of uses
       if not all(len(input_) == num_uses for input_ in inputs):
         raise ValueError("Length of inputs argument is inconsistent across "
-                         "mini-batches/towers.")
-      # Fold uses/time-step and towers/minibatches dimensions together
-      inputs = nest.flatten(inputs)
+                         "towers.")
 
-      inputs = _make_partitionedtensors_inputs(inputs)
-    # If inputs is not a tuple then we assume that inputs is a tensor
-    # with 'uses' folded into the batch dimension. (And grads_list is a list
-    # across sources of such Tensors.)  This is the native format that the
-    # factor will take as arguments.
+      if fisher_factors.TOWER_STRATEGY == "concat":
+        # Reverse the tower and use/time-step indices, so that use is now first,
+        # and towers is second
+        inputs = tuple(zip(*inputs))
+
+        # Flatten the two dimensions
+        inputs = nest.flatten(inputs)
+
+        # Merge everything together into a PartitionedTensor. We package it in
+        # a singleton tuple since the factors will expect a list over towers
+        inputs = (utils.PartitionedTensor(inputs),)
+
+      elif fisher_factors.TOWER_STRATEGY == "separate":
+        # Merge together the uses/time-step dimension into PartitionedTensors,
+        # but keep the leading dimension (towers) intact for the factors to
+        # process individually.
+        inputs = tuple(utils.PartitionedTensor(input_) for input_ in inputs)
+
+      else:
+        raise ValueError("Global config variable TOWER_STRATEGY must be one of "
+                         "'concat' or 'separate'.")
 
     # Now we perform the analogous processing for grads_list
     if isinstance(grads_list[0][0], (list, tuple)):
@@ -1170,10 +1247,34 @@ class InputOutputMultiMinibatchMultiUse(InputOutputMultiMinibatch):
       if not all(len(grad) == num_uses for grads in grads_list
                  for grad in grads):
         raise ValueError("Length of outputs argument is inconsistent across "
-                         "mini-batches/towers.")
+                         "towers.")
+
+      if fisher_factors.TOWER_STRATEGY == "concat":
+        # Reverse the tower and use/time-step indices, so that use is now first,
+        # and towers is second
+        grads_list = tuple(tuple(zip(*grads)) for grads in grads_list)
+
+        # Flatten the two dimensions, leaving the leading dimension (source)
+        # intact
+        grads_list = tuple(nest.flatten(grads) for grads in grads_list)
+
+        # Merge inner dimensions together into PartitionedTensors. We package
+        # them in a singleton tuple since the factors will expect a list over
+        # towers
+        grads_list = tuple((utils.PartitionedTensor(grads),)
+                           for grads in grads_list)
+
+      elif fisher_factors.TOWER_STRATEGY == "separate":
+        # Merge together the uses/time-step dimension into PartitionedTensors,
+        # but keep the leading dimension (towers) intact for the factors to
+        # process individually.
+        grads_list = tuple(tuple(utils.PartitionedTensor(grad)
+                                 for grad in grads)
+                           for grads in grads_list)
 
-      grads_list = tuple(nest.flatten(grads) for grads in grads_list)
-      grads_list = _make_partitionedtensors_grads(grads_list)
+      else:
+        raise ValueError("Global config variable TOWER_STRATEGY must be one of "
+                         "'concat' or 'separate'.")
 
     if self._num_uses is None:
       raise ValueError("You must supply a value for the num_uses argument if "
@@ -1184,7 +1285,7 @@ class InputOutputMultiMinibatchMultiUse(InputOutputMultiMinibatch):
     return inputs, grads_list
 
 
-class FullyConnectedMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+class FullyConnectedMultiIndepFB(InputOutputMultiTowerMultiUse,
                                  KroneckerProductFB):
   """FisherBlock for fully-connected layers that share parameters.
 
@@ -1228,7 +1329,7 @@ class FullyConnectedMultiIndepFB(InputOutputMultiMinibatchMultiUse,
     return float(self._num_uses)
 
 
-class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+class ConvKFCBasicMultiIndepFB(InputOutputMultiTowerMultiUse,
                                KroneckerProductFB):
   """FisherBlock for 2D convolutional layers using the basic KFC approx.
 
@@ -1309,7 +1410,7 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiMinibatchMultiUse,
     return self._num_locations * self._num_uses
 
 
-class EmbeddingKFACMultiIndepFB(InputOutputMultiMinibatchMultiUse,
+class EmbeddingKFACMultiIndepFB(InputOutputMultiTowerMultiUse,
                                 KroneckerProductFB):
   """K-FAC FisherBlock for embedding layers used multiple times in the graph.
 
@@ -1320,7 +1421,7 @@ class EmbeddingKFACMultiIndepFB(InputOutputMultiMinibatchMultiUse,
   Does not support bias parameters.
   """
 
-  def __init__(self, layer_collection, vocab_size, num_uses):
+  def __init__(self, layer_collection, vocab_size, num_uses=None):
     """Creates a EmbeddingKFACMultiIndepFB block.
 
     Args:
@@ -1368,7 +1469,7 @@ class SeriesFBApproximation(enum.IntEnum):
   option2 = 2
 
 
-class FullyConnectedSeriesFB(InputOutputMultiMinibatchMultiUse,
+class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse,
                              KroneckerProductFB):
   """FisherBlock for fully-connected layers that share parameters across time.
 
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index f521363536..353e1c6abb 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 import abc
+import contextlib
 
 import numpy as np
 import six
@@ -35,6 +36,8 @@ from tensorflow.python.ops import special_math_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import moving_averages
+from tensorflow.python.util import nest
+
 
 # Whether to initialize covariance estimators at a zero matrix (or the identity
 # matrix).
@@ -52,16 +55,25 @@ EIGENVALUE_DECOMPOSITION_THRESHOLD = 2
 # matrix powers. Must be nonnegative.
 EIGENVALUE_CLIPPING_THRESHOLD = 0.0
 
+# TOWER_STRATEGY can be one of "concat" or "separate".  If "concat", the data
+# passed to the factors from the blocks will be concatenated across towers
+# (lazilly via PartitionedTensor objects).  Otherwise a tuple of tensors over
+# towers will be passed in, and the factors will iterate over this and do the
+# cov computations separately for each one, averaging the results together.
+TOWER_STRATEGY = "concat"
+
 
 def set_global_constants(init_covariances_at_zero=None,
                          zero_debias=None,
                          eigenvalue_decomposition_threshold=None,
-                         eigenvalue_clipping_threshold=None):
+                         eigenvalue_clipping_threshold=None,
+                         tower_strategy=None):
   """Sets various global constants used by the classes in this module."""
   global INIT_COVARIANCES_AT_ZERO
   global ZERO_DEBIAS
   global EIGENVALUE_DECOMPOSITION_THRESHOLD
   global EIGENVALUE_CLIPPING_THRESHOLD
+  global TOWER_STRATEGY
 
   if init_covariances_at_zero is not None:
     INIT_COVARIANCES_AT_ZERO = init_covariances_at_zero
@@ -71,6 +83,8 @@ def set_global_constants(init_covariances_at_zero=None,
     EIGENVALUE_DECOMPOSITION_THRESHOLD = eigenvalue_decomposition_threshold
   if eigenvalue_clipping_threshold is not None:
     EIGENVALUE_CLIPPING_THRESHOLD = eigenvalue_clipping_threshold
+  if tower_strategy is not None:
+    TOWER_STRATEGY = tower_strategy
 
 
 def inverse_initializer(shape, dtype, partition_info=None):  # pylint: disable=unused-argument
@@ -89,6 +103,15 @@ def diagonal_covariance_initializer(shape, dtype, partition_info):  # pylint: di
   return array_ops.ones(shape, dtype)
 
 
+@contextlib.contextmanager
+def place_on_device(device):
+  if device is not None and len(device):
+    with tf_ops.device(device):
+      yield
+  else:
+    yield
+
+
 def compute_cov(tensor, tensor_right=None, normalizer=None):
   """Compute the empirical second moment of the rows of a 2D Tensor.
 
@@ -255,6 +278,10 @@ class FisherFactor(object):
     """
     pass
 
+  @abc.abstractproperty
+  def _num_towers(self):
+    pass
+
   @abc.abstractproperty
   def _dtype(self):
     """dtype for variable backing this factor."""
@@ -277,12 +304,14 @@ class FisherFactor(object):
           dtype=self._dtype)
 
   @abc.abstractmethod
-  def _compute_new_cov(self, idx=0):
+  def _compute_new_cov(self, source, tower):
     """Computes minibatch-estimated covariance for a single source.
 
     Args:
-      idx: int in [0, self._num_sources). Which source to use when estimating
-        covariance.
+      source: int in [0, self._num_sources). Which source to use when computing
+        the cov update.
+      tower: int in [0, self._num_towers). Which tower to use when computing
+        the cov update.
 
     Returns:
       Tensor of same shape as self.get_cov_var().
@@ -297,15 +326,29 @@ class FisherFactor(object):
     Returns:
       An Op for updating the covariance Variable referenced by _cov.
     """
-    new_cov_contribs = tuple(self._compute_new_cov(idx)
-                             for idx in range(self._num_sources))
-    new_cov = math_ops.add_n(new_cov_contribs)
+    new_cov_contribs = []
+    for source in range(self._num_sources):
+      for tower in range(self._num_towers):
+        device = (self._get_data_device(tower)
+                  if TOWER_STRATEGY == "separate" else None)
+        with place_on_device(device):
+          new_cov_contribs.append(self._compute_new_cov(source, tower))
+
+    new_cov = math_ops.add_n(new_cov_contribs) / float(self._num_towers)
+
+    # I have no idea if the TPU code below is still correct since I don't know
+    # what it actually does.  Also, this code is not present in some of the
+    # other versions of make_covariance_update_op.  Does it matter?
     # Synchronize value across all TPU cores.
     if utils.on_tpu():
       new_cov = utils.cross_replica_mean(new_cov)
     return moving_averages.assign_moving_average(
         self._cov, new_cov, ema_decay, zero_debias=ZERO_DEBIAS)
 
+  @abc.abstractmethod
+  def _get_data_device(self, tower):
+    pass
+
   @abc.abstractmethod
   def instantiate_inv_variables(self):
     """Makes the internal "inverse" variable(s)."""
@@ -596,17 +639,26 @@ class FullFactor(InverseProvidingFactor):
   def _num_sources(self):
     return len(self._params_grads)
 
+  @property
+  def _num_towers(self):
+    return 1
+
   @property
   def _dtype(self):
     return self._params_grads[0][0].dtype
 
-  def _compute_new_cov(self, idx=0):
+  def _compute_new_cov(self, source, tower):
+    assert tower == 0
+
     # This will be a very basic rank 1 estimate
-    params_grads_flat = utils.tensors_to_column(self._params_grads[idx])
+    params_grads_flat = utils.tensors_to_column(self._params_grads[source])
     return ((params_grads_flat * array_ops.transpose(
         params_grads_flat)) / math_ops.cast(self._batch_size,
                                             params_grads_flat.dtype))
 
+  def _get_data_device(self, tower):
+    return None
+
 
 class DiagonalFactor(FisherFactor):
   """A base class for FisherFactors that use diagonal approximations.
@@ -691,15 +743,24 @@ class NaiveDiagonalFactor(DiagonalFactor):
   def _num_sources(self):
     return len(self._params_grads)
 
+  @property
+  def _num_towers(self):
+    return 1
+
   @property
   def _dtype(self):
     return self._params_grads[0][0].dtype
 
-  def _compute_new_cov(self, idx=0):
-    params_grads_flat = utils.tensors_to_column(self._params_grads[idx])
+  def _compute_new_cov(self, source, tower):
+    assert tower == 0
+
+    params_grads_flat = utils.tensors_to_column(self._params_grads[source])
     return (math_ops.square(params_grads_flat) / math_ops.cast(
         self._batch_size, params_grads_flat.dtype))
 
+  def _get_data_device(self, tower):
+    return None
+
 
 class EmbeddingInputKroneckerFactor(DiagonalFactor):
   r"""FisherFactor for input to an embedding layer.
@@ -719,8 +780,8 @@ class EmbeddingInputKroneckerFactor(DiagonalFactor):
     """Instantiate EmbeddingInputKroneckerFactor.
 
     Args:
-      input_ids: Tensor of shape [batch_size, input_size] and dtype int32.
-        Indices into embedding matrix.
+      input_ids: List of Tensors of shape [batch_size, input_size] and dtype
+        int32. Indices into embedding matrix. List index is tower.
       vocab_size: int or 0-D Tensor. Maximum value for entries in 'input_ids'.
       dtype: dtype for covariance statistics. Must be a floating point type.
         Defaults to float32.
@@ -743,15 +804,18 @@ class EmbeddingInputKroneckerFactor(DiagonalFactor):
   def _num_sources(self):
     return 1
 
+  @property
+  def _num_towers(self):
+    return len(self._input_ids)
+
   @property
   def _dtype(self):
     return self._cov_dtype
 
-  def _compute_new_cov(self, idx=0):
-    if idx != 0:
-      raise ValueError("EmbeddingInputKroneckerFactor only supports idx = 0")
+  def _compute_new_cov(self, source, tower):
+    assert source == 0
 
-    input_ids = self._input_ids
+    input_ids = self._input_ids[tower]
 
     if len(input_ids.shape) > 2:
       raise ValueError(
@@ -781,6 +845,9 @@ class EmbeddingInputKroneckerFactor(DiagonalFactor):
 
     return new_cov
 
+  def _get_data_device(self, tower):
+    return self._input_ids[tower].device
+
 
 class FullyConnectedDiagonalFactor(DiagonalFactor):
   r"""FisherFactor for a diagonal approx of a fully-connected layer's Fisher.
@@ -800,10 +867,11 @@ class FullyConnectedDiagonalFactor(DiagonalFactor):
     """Instantiate FullyConnectedDiagonalFactor.
 
     Args:
-      inputs: Tensor of shape [batch_size, input_size]. Inputs to this layer.
+      inputs: List of Tensors of shape [batch_size, input_size]. Inputs to this
+        layer.  List index is towers.
       outputs_grads: List of Tensors, each of shape [batch_size, output_size],
         which are the gradients of the loss with respect to the layer's
-        outputs. One Tensor for each "source".
+        outputs. First index is source, second is tower.
 
       has_bias: bool. If True, append '1' to each input.
     """
@@ -817,47 +885,58 @@ class FullyConnectedDiagonalFactor(DiagonalFactor):
   @property
   def _var_scope(self):
     return "ff_diagfc_" + scope_string_from_params(
-        (self._inputs,) + tuple(self._outputs_grads))
+        tuple(self._inputs) + tuple(nest.flatten(self._outputs_grads)))
 
   @property
   def _cov_shape(self):
-    input_size = self._inputs.shape[1] + self._has_bias
-    output_size = self._outputs_grads[0].shape[1]
+    input_size = self._inputs[0].shape[1] + self._has_bias
+    output_size = self._outputs_grads[0][0].shape[1]
     return [input_size, output_size]
 
   @property
   def _num_sources(self):
     return len(self._outputs_grads)
 
+  @property
+  def _num_towers(self):
+    return len(self._inputs)
+
   @property
   def _dtype(self):
-    return self._outputs_grads[0].dtype
+    return self._outputs_grads[0][0].dtype
 
   def make_covariance_update_op(self, ema_decay):
-    inputs = self._inputs
 
-    if self._has_bias:
-      inputs = append_homog(inputs)
-    self._squared_inputs = math_ops.square(inputs)
+    self._squared_inputs = []
+    for tower in range(self._num_towers):
+      inputs = self._inputs[tower]
+
+      with place_on_device(self._get_data_device(tower)):
+        if self._has_bias:
+          inputs = append_homog(inputs)
+        self._squared_inputs.append(math_ops.square(inputs))
 
     return super(FullyConnectedDiagonalFactor, self).make_covariance_update_op(
         ema_decay)
 
-  def _compute_new_cov(self, idx=0):
-    batch_size = array_ops.shape(self._squared_inputs)[0]
-    outputs_grad = self._outputs_grads[idx]
+  def _compute_new_cov(self, source, tower):
+    batch_size = array_ops.shape(self._squared_inputs[tower])[0]
+    outputs_grad = self._outputs_grads[source][tower]
 
     # The well-known special formula that uses the fact that the entry-wise
     # square of an outer product is the outer-product of the entry-wise squares.
     # The gradient is the outer product of the input and the output gradients,
     # so we just square both and then take their outer-product.
     new_cov = math_ops.matmul(
-        self._squared_inputs,
+        self._squared_inputs[tower],
         math_ops.square(outputs_grad),
         transpose_a=True)
     new_cov /= math_ops.cast(batch_size, new_cov.dtype)
     return new_cov
 
+  def _get_data_device(self, tower):
+    return self._inputs[tower].device
+
 
 class ConvDiagonalFactor(DiagonalFactor):
   """FisherFactor for a diagonal approx of a convolutional layer's Fisher."""
@@ -874,11 +953,12 @@ class ConvDiagonalFactor(DiagonalFactor):
     """Creates a ConvDiagonalFactor object.
 
     Args:
-      inputs: Tensor of shape [batch_size, height, width, in_channels].
-        Input activations to this layer.
+      inputs: List of Tensors of shape [batch_size, height, width, in_channels].
+        Input activations to this layer.  List index is towers.
       outputs_grads: List of Tensors, each of shape [batch_size,
         height, width, out_channels], which are the gradients of the loss
-        with respect to the layer's outputs. One Tensor for each "source".
+        with respect to the layer's outputs.  First index is source, second
+        index is tower.
       filter_shape: Tuple of 4 ints: (kernel_height, kernel_width, in_channels,
         out_channels). Represents shape of kernel used in this layer.
       strides: The stride size in this layer (1-D Tensor of length 4).
@@ -896,14 +976,15 @@ class ConvDiagonalFactor(DiagonalFactor):
     """
     if not utils.is_data_format_channel_last(data_format):
       raise ValueError("Channel must be last.")
-    if inputs.shape.ndims != 4:
-      raise ValueError("inputs must be 4-D Tensor.")
-    if inputs.shape.as_list()[-1] != filter_shape[-2]:
+    if any(input_.shape.ndims != 4 for input_ in inputs):
+      raise ValueError("inputs must be a list of 4-D Tensors.")
+    if any(input_.shape.as_list()[-1] != filter_shape[-2] for input_ in inputs):
       raise ValueError("inputs and filter_shape must agree on in_channels.")
     for i, outputs_grad in enumerate(outputs_grads):
-      if outputs_grad.shape.ndims != 4:
+      if any(output_grad.shape.ndims != 4 for output_grad in outputs_grad):
         raise ValueError("outputs[%d] must be 4-D Tensor." % i)
-      if outputs_grad.shape.as_list()[-1] != filter_shape[-1]:
+      if any(output_grad.shape.as_list()[-1] != filter_shape[-1]
+             for output_grad in outputs_grad):
         raise ValueError(
             "outputs[%d] and filter_shape must agree on out_channels." % i)
     if len(strides) != 4:
@@ -926,7 +1007,7 @@ class ConvDiagonalFactor(DiagonalFactor):
   @property
   def _var_scope(self):
     return "ff_convdiag_" + scope_string_from_params(
-        (self._inputs,) + tuple(self._outputs_grads))
+        tuple(self._inputs) + tuple(nest.flatten(self._outputs_grads)))
 
   @property
   def _cov_shape(self):
@@ -940,9 +1021,13 @@ class ConvDiagonalFactor(DiagonalFactor):
   def _num_sources(self):
     return len(self._outputs_grads)
 
+  @property
+  def _num_towers(self):
+    return len(self._inputs)
+
   @property
   def _dtype(self):
-    return self._outputs_grads[0].dtype
+    return self._inputs[0].dtype
 
   def make_covariance_update_op(self, ema_decay):
     filter_height, filter_width, _, _ = self._filter_shape
@@ -953,25 +1038,30 @@ class ConvDiagonalFactor(DiagonalFactor):
       rates = (1, 1, 1, 1)
     else:
       rates = tuple(self._dilations)
-    patches = array_ops.extract_image_patches(
-        self._inputs,
-        ksizes=[1, filter_height, filter_width, 1],
-        strides=self._strides,
-        rates=rates,
-        padding=self._padding)
 
-    if self._has_bias:
-      patches = append_homog(patches)
+    self._patches = []
+    for tower in range(self._num_towers):
+      with place_on_device(self._get_data_device(tower)):
+        patches = array_ops.extract_image_patches(
+            self._inputs[tower],
+            ksizes=[1, filter_height, filter_width, 1],
+            strides=self._strides,
+            rates=rates,
+            padding=self._padding)
+
+        if self._has_bias:
+          patches = append_homog(patches)
 
-    self._patches = patches
+        self._patches.append(patches)
 
     return super(ConvDiagonalFactor, self).make_covariance_update_op(ema_decay)
 
-  def _compute_new_cov(self, idx=0):
-    batch_size = array_ops.shape(self._patches)[0]
-    outputs_grad = self._outputs_grads[idx]
+  def _compute_new_cov(self, source, tower):
+    patches = self._patches[tower]
+    batch_size = array_ops.shape(patches)[0]
+    outputs_grad = self._outputs_grads[source][tower]
 
-    new_cov = self._convdiag_sum_of_squares(self._patches, outputs_grad)
+    new_cov = self._convdiag_sum_of_squares(patches, outputs_grad)
     new_cov /= math_ops.cast(batch_size, new_cov.dtype)
 
     return new_cov
@@ -984,6 +1074,9 @@ class ConvDiagonalFactor(DiagonalFactor):
                                                   outputs_grad)
     return math_ops.reduce_sum(math_ops.square(case_wise_gradients), axis=0)
 
+  def _get_data_device(self, tower):
+    return self._inputs[tower].device
+
 
 class FullyConnectedKroneckerFactor(InverseProvidingFactor):
   """Kronecker factor for the input or output side of a fully-connected layer.
@@ -995,9 +1088,9 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor):
     """Instantiate FullyConnectedKroneckerFactor.
 
     Args:
-      tensors: List of Tensors, each of shape [batch_size, n], one for each
-      source.  The Tensors are typically either a layer's inputs or its
-      output's gradients.
+      tensors: List of list of Tensors, each of shape [batch_size, n]. The
+        Tensors are typically either a layer's inputs or its output's gradients.
+        The first list index is source, the second is tower.
       has_bias: bool. If True, append '1' to each row.
     """
     # The tensor argument is either a tensor of input activations or a tensor of
@@ -1009,27 +1102,34 @@ class FullyConnectedKroneckerFactor(InverseProvidingFactor):
   @property
   def _var_scope(self):
     return "ff_fckron_" + scope_string_from_params(
-        tuple(self._tensors) + (self._has_bias,))
+        tuple(nest.flatten(self._tensors)) + (self._has_bias,))
 
   @property
   def _cov_shape(self):
-    size = self._tensors[0].shape[1] + self._has_bias
+    size = self._tensors[0][0].shape[1] + self._has_bias
     return [size, size]
 
   @property
   def _num_sources(self):
     return len(self._tensors)
 
+  @property
+  def _num_towers(self):
+    return len(self._tensors[0])
+
   @property
   def _dtype(self):
-    return self._tensors[0].dtype
+    return self._tensors[0][0].dtype
 
-  def _compute_new_cov(self, idx=0):
-    tensor = self._tensors[idx]
+  def _compute_new_cov(self, source, tower):
+    tensor = self._tensors[source][tower]
     if self._has_bias:
       tensor = append_homog(tensor)
     return compute_cov(tensor)
 
+  def _get_data_device(self, tower):
+    return self._tensors[0][tower].device
+
 
 class ConvInputKroneckerFactor(InverseProvidingFactor):
   r"""Kronecker factor for the input side of a convolutional layer.
@@ -1053,8 +1153,8 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
     """Initializes ConvInputKroneckerFactor.
 
     Args:
-      inputs: Tensor of shape [batch_size, ..spatial_input_size.., in_channels].
-        Inputs to layer.
+      inputs: List of Tensors of shape [batch_size, ..spatial_input_size..,
+        in_channels]. Inputs to layer. List index is tower.
       filter_shape: List of ints. Contains [..spatial_filter_size..,
         in_channels, out_channels]. Shape of convolution kernel.
       padding: str. Padding method for layer. "SAME" or "VALID".
@@ -1083,10 +1183,10 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
 
   @property
   def _var_scope(self):
-    return "ff_convinkron_" + scope_string_from_params([
-        self._inputs, self._filter_shape, self._strides, self._padding,
-        self._dilation_rate, self._data_format, self._has_bias
-    ])
+    return "ff_convinkron_" + scope_string_from_params(
+        tuple(self._inputs) +
+        tuple((self._filter_shape, self._strides, self._padding,
+               self._dilation_rate, self._data_format, self._has_bias)))
 
   @property
   def _cov_shape(self):
@@ -1099,19 +1199,24 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
   def _num_sources(self):
     return 1
 
+  @property
+  def _num_towers(self):
+    return len(self._inputs)
+
   @property
   def _dtype(self):
-    return self._inputs.dtype
+    return self._inputs[0].dtype
 
-  def _compute_new_cov(self, idx=0):
-    if idx != 0:
-      raise ValueError("ConvInputKroneckerFactor only supports idx = 0")
+  def _compute_new_cov(self, source, tower):
+    assert source == 0
+
+    inputs = self._inputs[tower]
 
     # TODO(b/64144716): there is potential here for a big savings in terms of
     # memory use.
     if self._extract_patches_fn in [None, "extract_convolution_patches"]:
       patches = utils.extract_convolution_patches(
-          self._inputs,
+          inputs,
           self._filter_shape,
           padding=self._padding,
           strides=self._strides,
@@ -1119,7 +1224,7 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
           data_format=self._data_format)
 
     elif self._extract_patches_fn == "extract_image_patches":
-      assert self._inputs.shape.ndims == 4
+      assert inputs.shape.ndims == 4
       assert len(self._filter_shape) == 4
       assert len(self._strides) == 4, self._strides
       if self._dilation_rate is None:
@@ -1129,7 +1234,7 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
         assert len(rates) == 4
         assert rates[0] == rates[-1] == 1
       patches = array_ops.extract_image_patches(
-          self._inputs,
+          inputs,
           ksizes=[1] + list(self._filter_shape[0:-2]) + [1],
           strides=self._strides,
           rates=rates,
@@ -1139,7 +1244,7 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
       assert self._strides in [None, [1, 1, 1, 1], (1, 1, 1, 1)]
       assert self._filter_shape[0] == self._filter_shape[1] == 1
       patches = utils.extract_pointwise_conv2d_patches(
-          self._inputs, self._filter_shape, data_format=None)
+          inputs, self._filter_shape, data_format=None)
 
     else:
       raise NotImplementedError(self._extract_patches_fn)
@@ -1164,6 +1269,9 @@ class ConvInputKroneckerFactor(InverseProvidingFactor):
     # (Tilde omitted over A for clarity.)
     return compute_cov(patches_flat)
 
+  def _get_data_device(self, tower):
+    return self._inputs[tower].device
+
 
 class ConvOutputKroneckerFactor(InverseProvidingFactor):
   r"""Kronecker factor for the output side of a convolutional layer.
@@ -1180,9 +1288,9 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor):
     """Initializes ConvOutputKroneckerFactor.
 
     Args:
-      outputs_grads: list of Tensors. Each Tensor is of shape
-          [batch_size, ..spatial_input_size.., out_channels]. One Tensor per
-          source.
+      outputs_grads: List of list of Tensors. Each Tensor is of shape
+          [batch_size, ..spatial_input_size.., out_channels].  First list index
+          is source, the second is tower.
       data_format: None or str. Format of outputs_grads.
 
     Raises:
@@ -1190,13 +1298,14 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor):
     """
     if not utils.is_data_format_channel_last(data_format):
       raise ValueError("Channel must be last.")
-    self._out_channels = outputs_grads[0].shape.as_list()[-1]
+    self._out_channels = outputs_grads[0][0].shape.as_list()[-1]
     self._outputs_grads = outputs_grads
     super(ConvOutputKroneckerFactor, self).__init__()
 
   @property
   def _var_scope(self):
-    return "ff_convoutkron_" + scope_string_from_params(self._outputs_grads)
+    return "ff_convoutkron_" + scope_string_from_params(
+        nest.flatten(self._outputs_grads))
 
   @property
   def _cov_shape(self):
@@ -1207,12 +1316,16 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor):
   def _num_sources(self):
     return len(self._outputs_grads)
 
+  @property
+  def _num_towers(self):
+    return len(self._outputs_grads[0])
+
   @property
   def _dtype(self):
-    return self._outputs_grads[0].dtype
+    return self._outputs_grads[0][0].dtype
 
-  def _compute_new_cov(self, idx=0):
-    outputs_grad = self._outputs_grads[idx]
+  def _compute_new_cov(self, source, tower):
+    outputs_grad = self._outputs_grads[source][tower]
 
     # reshaped_tensor below is the matrix DS_l defined in the KFC paper
     # (tilde omitted over S for clarity). It has shape M|T| x I, where
@@ -1225,6 +1338,9 @@ class ConvOutputKroneckerFactor(InverseProvidingFactor):
     # (Tilde omitted over S for clarity.)
     return compute_cov(reshaped_tensor)
 
+  def _get_data_device(self, tower):
+    return self._outputs_grads[0][tower].device
+
 
 class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
   """Kronecker factor for a fully connected layer used multiple times."""
@@ -1236,9 +1352,11 @@ class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
     """Constructs a new `FullyConnectedMultiKF`.
 
     Args:
-      tensors: List of Tensors of shape, each of shape [batch_size, n]. Each of
-        these tensors is usually a layer's inputs or its output's gradients.
-        The list is over sources.
+      tensors: List of list of Tensors of shape, each of shape
+        [num_uses * batch_size, n], and is a reshape version of a Tensor of
+        shape [num_uses, batch_size, n]. Each of these tensors is usually a
+        layer's inputs or its output's gradients. The first list index is
+        sources, the second is towers.
       num_uses: int. The number of time-steps / uses.
       has_bias: bool. If True, '1' is appended to each row.
     """
@@ -1262,16 +1380,24 @@ class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
   @property
   def _var_scope(self):
     return "ff_fc_multi_" + scope_string_from_params(
-        tuple(self._tensors) + (self._num_timesteps, self._has_bias,))
+        tuple(nest.flatten(self._tensors))
+        + (self._num_timesteps, self._has_bias,))
 
   def make_covariance_update_op(self, ema_decay):
 
     op = super(FullyConnectedMultiKF, self).make_covariance_update_op(ema_decay)
 
     if self._cov_dt1 is not None:
-      new_cov_dt1_contribs = tuple(self._compute_new_cov_dt1(idx)
-                                   for idx in range(self._num_sources))
-      new_cov_dt1 = math_ops.add_n(new_cov_dt1_contribs)
+      new_cov_dt1_contribs = []
+      for source in range(self._num_sources):
+        for tower in range(self._num_towers):
+          with place_on_device(self._get_data_device(tower)):
+            new_cov_dt1_contribs.append(self._compute_new_cov_dt1(source,
+                                                                  tower))
+
+      new_cov_dt1 = (math_ops.add_n(new_cov_dt1_contribs)
+                     / float(self._num_towers))
+
       op2 = moving_averages.assign_moving_average(
           self._cov_dt1, new_cov_dt1, ema_decay, zero_debias=ZERO_DEBIAS)
 
@@ -1284,8 +1410,8 @@ class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
 
     return op
 
-  def _compute_new_cov_dt1(self, idx=0):  # pylint: disable=missing-docstring
-    tensor = self._tensors[idx]
+  def _compute_new_cov_dt1(self, source, tower):  # pylint: disable=missing-docstring
+    tensor = self._tensors[source][tower]
     if self._has_bias:
       # This appending is technically done twice (the other time is for
       # _compute_new_cov())
@@ -1303,9 +1429,12 @@ class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
     return compute_cov(
         tensor_future, tensor_right=tensor_present, normalizer=total_len)
 
+  def _get_data_device(self, tower):
+    return self._tensors[0][tower].device
+
   @property
   def _vec_shape(self):
-    size = self._tensors[0].shape[1] + self._has_bias
+    size = self._tensors[0][0].shape[1] + self._has_bias
     return [size]
 
   def get_option1quants(self, damping_func):
diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 7727c607db..586a004f88 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -390,7 +390,7 @@ class LayerCollection(object):
       if name in self._loss_dict:
         raise KeyError(
             "Loss function named {} already exists. Set reuse=True to append "
-            "another minibatch/tower.".format(name))
+            "another tower.".format(name))
 
       loss_list = []
       self._loss_dict[name] = loss_list
@@ -596,7 +596,7 @@ class LayerCollection(object):
     vocab_size = int(params.shape[0])
     block = self.register_block(
         params, block_type(self, vocab_size), reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, 1)
 
@@ -637,7 +637,7 @@ class LayerCollection(object):
     has_bias = isinstance(params, (tuple, list))
     block = self.register_block(params, block_type(self, has_bias=has_bias),
                                 reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, 1)
 
@@ -716,7 +716,7 @@ class LayerCollection(object):
     else:
       raise NotImplementedError(approx)
 
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, 1)
 
@@ -774,7 +774,7 @@ class LayerCollection(object):
             dilation_rate=dilation_rate,
             data_format=data_format),
         reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, 1)
 
@@ -830,7 +830,7 @@ class LayerCollection(object):
             rate=rate,
             data_format=data_format),
         reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, 1)
 
@@ -913,7 +913,7 @@ class LayerCollection(object):
 
     Args:
       params: Tensor or tuple of Tensors corresponding to the parameters.
-      batch_size: 0-D Tensor. Size of the minibatch.
+      batch_size: 0-D Tensor. Size of the minibatch (for this tower).
       approx: str or None. It not None, must be one of "full" or "diagonal".
         The Fisher approximation to use. If None the default value is used.
         (Default: None)
@@ -932,7 +932,7 @@ class LayerCollection(object):
         _GENERIC_APPROX_TO_BLOCK_TYPES)
 
     block = self.register_block(params, block_type(self, params), reuse=reuse)
-    block.register_additional_minibatch(batch_size)
+    block.register_additional_tower(batch_size)
 
     self._add_uses(params, float("inf"))
 
@@ -952,14 +952,14 @@ class LayerCollection(object):
       inputs: A list of Tensors, each of shape [batch_size, input_size]. Inputs
         to layer. The list indexes each use in the graph (which might
         correspond to a "time-step" in an RNN). OR, can be single Tensor, of
-        shape [batch_size * num_uses, input_size], which is a reshaped version
-        of a Tensor of shape [batch_size, num_uses, input_size].
+        shape [num_uses * batch_size , input_size], which is a reshaped version
+        of a Tensor of shape [num_uses, batch_size, input_size].
       outputs: A list of Tensors, the same length as 'inputs', each of shape
         [batch_size, output_size]. Outputs produced by layer. The list indexes
         each use in the graph (which might correspond to a "time-step" in an
         RNN). Needs to correspond with the order used in 'inputs'.  OR, can be
-        a single Tensor of shape [batch_size * num_uses, output_size], which is
-        a reshaped version of a Tensor of shape [batch_size, num_uses,
+        a single Tensor of shape [num_uses * batch_size, output_size], which is
+        a reshaped version of a Tensor of shape [num_uses, batch_size,
         output_size].
       num_uses: int or None. The number uses/time-steps in the graph where the
         layer appears. Only needed if both inputs and outputs are given in the
@@ -989,7 +989,7 @@ class LayerCollection(object):
     block = self.register_block(params, block_type(self, has_bias=has_bias,
                                                    num_uses=num_uses),
                                 reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     assert len(inputs) == len(outputs)
     self._add_uses(params, len(inputs))
@@ -1017,16 +1017,16 @@ class LayerCollection(object):
       inputs: A list of Tensors, each of shape [batch_size, height, width,
         in_channels]. Inputs to layer. The list indexes each use in the graph
         (which might correspond to a "time-step" in an RNN). OR, can be single
-        Tensor, of shape [batch_size * num_uses, height, width, in_channels],
-        which is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        Tensor, of shape [num_uses * batch_size, height, width, in_channels],
+        which is a reshaped version of a Tensor of shape [num_uses, batch_size,
         height, width, in_channels].
       outputs: A list of Tensors, each of shape [batch_size, height, width,
         out_channels]. Output produced by layer. The list indexes each use
         in the graph (which might correspond to a "time-step" in an RNN).
         Needs to correspond with the order used in 'inputs'.  OR, can be a
-        single Tensor, of shape [batch_size*num_uses, height, width,
+        single Tensor, of shape [num_uses * batch_size, height, width,
         out_channels], which is a reshaped version of a Tensor of shape
-        [batch_size, num_uses, height, width, out_channels].
+        [num_uses, batch_size, height, width, out_channels].
       num_uses: int or None. The number uses/time-steps in the graph where the
         layer appears. Only needed if both inputs and outputs are given in the
         single Tensor format. (Default: None)
@@ -1065,7 +1065,7 @@ class LayerCollection(object):
             num_uses=num_uses),
         reuse=reuse)
 
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     assert len(inputs) == len(outputs)
     self._add_uses(params, len(inputs))
@@ -1088,15 +1088,15 @@ class LayerCollection(object):
       inputs: A list of Tensors, each of shape [batch_size, input_size] and
         dtype int32. Indices into embedding matrix. The list indexes each use
         in the graph (which might correspond to a "time-step" in an RNN).
-        OR, can be single Tensor, of shape [batch_size * num_uses, input_size],
-        which is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        OR, can be single Tensor, of shape [num_uses, batch_size, input_size],
+        which is a reshaped version of a Tensor of shape [num_uses, batch_size,
         input_size].
       outputs: A list of Tensors, each of shape [batch_size, embedding_size].
         Outputs produced by layer. The list indexes each use in the graph
         (which might correspond to a "time-step" in an RNN). Needs to
         correspond with the order used in 'inputs'. OR, can be a
-        single Tensor, of shape [batch_size*num_uses, embedding_size], which
-        is a reshaped version of a Tensor of shape [batch_size, num_uses,
+        single Tensor, of shape [num_uses * batch_size, embedding_size], which
+        is a reshaped version of a Tensor of shape [num_uses, batch_size,
         embedding_size].
       num_uses: int or None. The number uses/time-steps in the graph where the
         layer appears. Only needed if both inputs and outputs are given in the
@@ -1127,7 +1127,7 @@ class LayerCollection(object):
 
     block = self.register_block(
         params, block_type(self, vocab_size, num_uses=num_uses), reuse=reuse)
-    block.register_additional_minibatch(inputs, outputs)
+    block.register_additional_tower(inputs, outputs)
 
     self._add_uses(params, len(inputs))
 
diff --git a/tensorflow/contrib/kfac/python/ops/utils.py b/tensorflow/contrib/kfac/python/ops/utils.py
index c9de0c7270..b6f42815e7 100644
--- a/tensorflow/contrib/kfac/python/ops/utils.py
+++ b/tensorflow/contrib/kfac/python/ops/utils.py
@@ -649,9 +649,6 @@ class PartitionedTensor(object):
   def dtype(self):
     return self.tensors[0].dtype
 
-  def devices(self):
-    return set(tensor.device for tensor in self.tensors)
-
   def __str__(self):
     return "PartitionedTensor([%s, ...], dtype=%s, shape=%s)" % (
         self.tensors[0].name, self.dtype.name, tuple(self.shape.as_list()))
@@ -681,6 +678,15 @@ class PartitionedTensor(object):
         self._concats[result.device] = result
       return self._concats[result.device]
 
+  @property
+  def device(self):
+    # PartitionedTensors in general do not live on a single device.  If the
+    # device cannot be determined unambiguously this property will return None.
+    device = self.tensors[0].device
+    if all(tensor.device == device for tensor in self.tensors):
+      return device
+    return None
+
 
 ops.register_tensor_conversion_function(
     PartitionedTensor,
-- 
GitLab


From 85bec0af5e4bd036a9cb922c794bbe7191f7b76d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 08:16:59 -0700
Subject: [PATCH 539/960] Optimized quantized fully-connected op for LSTMs.

PiperOrigin-RevId: 190617310
---
 .../internal/optimized/optimized_ops.h        | 82 +++++++++++--------
 1 file changed, 50 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index f08d9d6d57..e079ff3f4c 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -802,21 +802,20 @@ inline void FullyConnected(const uint8* input_data, const Dims<4>& input_dims,
       input_offset, output_pipeline);
 }
 
-inline void FullyConnected(const uint8* input_data, const Dims<4>& input_dims,
-                           int32 input_offset, const uint8* filter_data,
-                           const Dims<4>& filter_dims, int32 filter_offset,
-                           const int32* bias_data, const Dims<4>& bias_dims,
-                           int32 output_offset, int32 output_multiplier,
-                           int output_shift, int32 output_activation_min,
-                           int32 output_activation_max, int16* output_data,
-                           const Dims<4>& output_dims,
-                           gemmlowp::GemmContext* gemm_context) {
+inline void FullyConnected(
+    const uint8* input_data, const Dims<4>& input_dims, int32 input_offset,
+    const uint8* filter_data, const Dims<4>& filter_dims, int32 filter_offset,
+    const int32* bias_data_int32, const Dims<4>& bias_dims, int32 output_offset,
+    int32 output_multiplier, int output_shift, int32 output_activation_min,
+    int32 output_activation_max, int16* output_data, const Dims<4>& output_dims,
+    gemmlowp::GemmContext* gemm_context) {
   gemmlowp::ScopedProfilingLabel label("FullyConnected/Uint8Int16");
   // This is a copy of the reference implementation. We do not currently have a
   // properly optimized version.
   (void)gemm_context;  // only used in properly optimized code.
   TFLITE_DCHECK_LE(output_activation_min, output_activation_max);
   TFLITE_DCHECK_EQ(output_offset, 0);
+
   // TODO(benoitjacob): This really should be:
   //     const int batches = ArraySize(output_dims, 1);
   // but the current --variable_batch hack consists in overwriting the 3rd
@@ -828,30 +827,49 @@ inline void FullyConnected(const uint8* input_data, const Dims<4>& input_dims,
   const int accum_depth = ArraySize(filter_dims, 0);
   TFLITE_DCHECK(IsPackedWithoutStrides(input_dims));
   TFLITE_DCHECK(IsPackedWithoutStrides(filter_dims));
-  for (int b = 0; b < batches; ++b) {
-    for (int out_c = 0; out_c < output_depth; ++out_c) {
-      // Internal accumulation.
-      // Initialize accumulator with the bias-value.
-      int32 accum = bias_data[out_c];
-      // Accumulation loop.
-      for (int d = 0; d < accum_depth; ++d) {
-        int16 input_val = input_data[b * accum_depth + d] + input_offset;
-        int16 filter_val = filter_data[out_c * accum_depth + d] + filter_offset;
-        accum += filter_val * input_val;
-      }
-      // Down-scale the final int32 accumulator to the scale used by our
-      // (16-bit, typically 3 integer bits) fixed-point format. The quantized
-      // multiplier and shift here have been pre-computed offline
-      // (e.g. by toco).
-      accum = MultiplyByQuantizedMultiplier(accum, output_multiplier,
-                                            -output_shift);
-      // Saturate, cast to int16, and store to output array.
-      accum = std::max(accum, output_activation_min - output_offset);
-      accum = std::min(accum, output_activation_max - output_offset);
-      accum += output_offset;
-      output_data[out_c + output_depth * b] = accum;
-    }
+
+  // Implementation of the fully connected node suited to the inside of an LSTM
+  // cell. The operands are 8-bit integers, the accumulators are internally
+  // 32bit integers, and the output is 16-bit fixed-point with 3 integer bits so
+  // the output range is [-2^3, 2^3] == [-8, 8]. The rationale for that
+  // is explained in the function comment above.
+#ifdef GEMMLOWP_NEON
+  if (batches == 1 && !(output_depth % 4) && !(accum_depth % 8) &&
+      input_offset == -128 && output_activation_min == -32768 &&
+      output_activation_max == 32767) {
+    GEMVForLstmCell(input_data, input_dims, filter_data, filter_dims,
+                    filter_offset, bias_data_int32, bias_dims,
+                    output_multiplier, -output_shift, output_data, output_dims);
+    return;
   }
+#endif
+  gemmlowp::MatrixMap<const uint8, gemmlowp::MapOrder::RowMajor> weights_matrix(
+      filter_data, output_depth, accum_depth);
+  gemmlowp::MatrixMap<const uint8, gemmlowp::MapOrder::ColMajor> input_matrix(
+      input_data, accum_depth, batches);
+  gemmlowp::MatrixMap<int16, gemmlowp::MapOrder::ColMajor> output_matrix(
+      output_data, output_depth, batches);
+  typedef gemmlowp::VectorMap<const int32, gemmlowp::VectorShape::Col>
+      ColVectorMap;
+  ColVectorMap bias_vector(bias_data_int32, output_depth);
+  gemmlowp::OutputStageBiasAddition<ColVectorMap> bias_addition_stage;
+  bias_addition_stage.bias_vector = bias_vector;
+  gemmlowp::OutputStageScaleInt32ByFixedPointAndExponent scale_stage;
+  scale_stage.result_offset_after_shift = 0;
+  scale_stage.result_fixedpoint_multiplier = output_multiplier;
+  // Note that this shift is negated wrt ordinary FC.
+  scale_stage.result_exponent = -output_shift;
+  gemmlowp::OutputStageClamp clamp_stage;
+  clamp_stage.min = output_activation_min;
+  clamp_stage.max = output_activation_max;
+  gemmlowp::OutputStageSaturatingCastToInt16 saturating_cast_int16_stage;
+  auto output_pipeline =
+      std::make_tuple(bias_addition_stage, scale_stage, clamp_stage,
+                      saturating_cast_int16_stage);
+  gemmlowp::GemmWithOutputPipeline<uint8, int16,
+                                   gemmlowp::L8R8WithLhsNonzeroBitDepthParams>(
+      gemm_context, weights_matrix, input_matrix, &output_matrix, filter_offset,
+      input_offset, output_pipeline);
 }
 
 // legacy, for compatibility with old checked-in code
-- 
GitLab


From cf24990855b4418f86ffc5cfe65b502cd0d8b924 Mon Sep 17 00:00:00 2001
From: Peter Hawkins <phawkins@google.com>
Date: Tue, 27 Mar 2018 08:33:22 -0700
Subject: [PATCH 540/960] Automated g4 rollback of changelist 188385868

PiperOrigin-RevId: 190618988
---
 tensorflow/compiler/tests/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index bbb6089ea8..1c5a8f8e69 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -542,7 +542,6 @@ tf_xla_py_test(
     size = "medium",
     srcs = ["spacetobatch_op_test.py"],
     shard_count = 3,
-    tags = ["notsan"],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
-- 
GitLab


From ff9040f645a042ef62782be0eed8b4597e80ce6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 09:20:54 -0700
Subject: [PATCH 541/960] Flush the output of print (fixes out-of-order prints
 in public colab)

PiperOrigin-RevId: 190624708
---
 tensorflow/contrib/autograph/utils/builtins.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/utils/builtins.py b/tensorflow/contrib/autograph/utils/builtins.py
index 4ab32ee47d..c6af0e4d13 100644
--- a/tensorflow/contrib/autograph/utils/builtins.py
+++ b/tensorflow/contrib/autograph/utils/builtins.py
@@ -18,6 +18,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import sys
+
 import six
 
 from tensorflow.contrib.autograph.utils import py_func
@@ -97,7 +99,13 @@ def dynamic_print(*values):
 
   if all(map(is_tf_print_compatible, values)):
     return logging_ops.Print(1, values)
-  return py_func.wrap_py_func(print, None, values, use_dummy_return=True)
+
+  def flushed_print(*vals):
+    print(*vals)
+    sys.stdout.flush()
+
+  return py_func.wrap_py_func(
+      flushed_print, None, values, use_dummy_return=True)
 
 
 def dynamic_dataset(iterated):
-- 
GitLab


From 2ad47da4fb9896290eb9bc87fe809a4138269f2c Mon Sep 17 00:00:00 2001
From: brett koonce <koonce@hello.com>
Date: Tue, 27 Mar 2018 09:24:40 -0700
Subject: [PATCH 542/960] Seq2seq minorsp (#18010)

* contrib/seq2seq: minor spelling tweaks

* contrib/timeseries: minor spelling tweaks

* contrib/slim: minor spelling tweaks
---
 tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc     | 2 +-
 .../contrib/seq2seq/python/ops/attention_wrapper.py       | 8 ++++----
 .../contrib/seq2seq/python/ops/beam_search_decoder.py     | 6 +++---
 .../contrib/slim/python/slim/data/parallel_reader.py      | 4 ++--
 .../contrib/slim/python/slim/data/prefetch_queue.py       | 4 ++--
 .../contrib/slim/python/slim/data/tfexample_decoder.py    | 2 +-
 .../contrib/timeseries/python/timeseries/ar_model.py      | 2 +-
 .../contrib/timeseries/python/timeseries/math_utils.py    | 2 +-
 .../python/timeseries/state_space_models/varma.py         | 4 ++--
 9 files changed, 17 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index dfa12e873a..a9a32b7b25 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -74,7 +74,7 @@ class GatherTreeOp : public OpKernel {
         ctx,
         step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0),
         errors::InvalidArgument("batch size dimensions step_ids.shape[1] and "
-                                "max_seqeuence_lengths.shape[0] must match.  "
+                                "max_sequence_lengths.shape[0] must match.  "
                                 "but shapes are: ",
                                 step_ids_shape.DebugString(), " and ",
                                 max_sequence_lengths.shape().DebugString()));
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 9ff8a343f1..be53779826 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -736,7 +736,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
   """Base attention mechanism for monotonic attention.
 
   Simply overrides the initial_alignments function to provide a dirac
-  distribution,which is needed in order for the monotonic attention
+  distribution, which is needed in order for the monotonic attention
   distributions to have the correct behavior.
   """
 
@@ -763,7 +763,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -867,7 +867,7 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -1133,7 +1133,7 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
       output_attention: Python bool.  If `True` (default), the output at each
         time step is the attention value.  This is the behavior of Luong-style
         attention mechanisms.  If `False`, the output at each time step is
-        the output of `cell`.  This is the beahvior of Bhadanau-style
+        the output of `cell`.  This is the behavior of Bhadanau-style
         attention mechanisms.  In both cases, the `attention` tensor is
         propagated to the next time step via the state and is used there.
         This flag only controls whether the attention mechanism is propagated
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index a26107b0d7..184144f64a 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -821,9 +821,9 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight):
   Returns:
     The scores normalized by the length_penalty.
   """
-  length_penality_ = _length_penalty(
+  length_penalty_ = _length_penalty(
       sequence_lengths=sequence_lengths, penalty_factor=length_penalty_weight)
-  return log_probs / length_penality_
+  return log_probs / length_penalty_
 
 
 def _length_penalty(sequence_lengths, penalty_factor):
@@ -860,7 +860,7 @@ def _mask_probs(probs, eos_token, finished):
   unfinished beams remain unchanged.
 
   Args:
-    probs: Log probabiltiies of shape `[batch_size, beam_width, vocab_size]`
+    probs: Log probabilities of shape `[batch_size, beam_width, vocab_size]`
     eos_token: An int32 id corresponding to the EOS token to allocate
       probability to.
     finished: A boolean tensor of shape `[batch_size, beam_width]` that
diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
index b3343aef47..99ad487630 100644
--- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
+++ b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
@@ -115,8 +115,8 @@ class ParallelReader(io_ops.ReaderBase):
     reader needs to start reading from a new file since it has finished with
     the previous file).
 
-    A queue runner for enqueing in the `common_queue` is automatically added to
-    the TF QueueRunners collection.
+    A queue runner for enqueuing in the `common_queue` is automatically added
+    to the TF QueueRunners collection.
 
     Args:
       queue: A Queue or a mutable string Tensor representing a handle
diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
index 37e9c4754c..62bd200361 100644
--- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
+++ b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
@@ -36,9 +36,9 @@ def prefetch_queue(tensors,
                    dynamic_pad=False,
                    shared_name=None,
                    name=None):
-  """Creates a queue to prefetech tensors from `tensors`.
+  """Creates a queue to prefetch tensors from `tensors`.
 
-  A queue runner for enqueing tensors into the prefetch_queue is automatically
+  A queue runner for enqueuing tensors into the prefetch_queue is automatically
   added to the TF QueueRunners collection.
 
   Example:
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index b3b61e1dfe..f2d31dc8db 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -124,7 +124,7 @@ class BoundingBox(ItemHandler):
     super(BoundingBox, self).__init__(self._full_keys)
 
   def tensors_to_item(self, keys_to_tensors):
-    """Maps the given dictionary of tensors to a contatenated list of bboxes.
+    """Maps the given dictionary of tensors to a concatenated list of bboxes.
 
     Args:
       keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index ff140efd48..4f6527a546 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -70,7 +70,7 @@ class ARModel(model.TimeSeriesModel):
       input_window_size: Number of past time steps of data to look at when doing
         the regression.
       output_window_size: Number of future time steps to predict. Note that
-        setting it to > 1 empiricaly seems to give a better fit.
+        setting it to > 1 empirically seems to give a better fit.
       num_features: number of input features per time step.
       num_time_buckets: Number of buckets into which to divide (time %
         periodicity) for generating time based features.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index 23452a81c3..26793c80bf 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -185,7 +185,7 @@ def batch_matrix_pow(matrices, powers):
                     { matmul(A, power(matmul(A, A), (p - 1) / 2)) for odd p
       power(A, 0) = I
 
-    The power(A, 0) = I case is handeled by starting with accumulator set to the
+    The power(A, 0) = I case is handled by starting with accumulator set to the
     identity matrix; matrices with zero residual powers are passed through
     unchanged.
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
index 1afc58cfb2..6746dd7b43 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
@@ -107,7 +107,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state transition matrix. It has shape
-        [self.state_dimendion, self.state_dimension].
+        [self.state_dimension, self.state_dimension].
     """
     # Pad any unused AR blocks with zeros. The extra state is necessary if
     # ma_order >= ar_order.
@@ -127,7 +127,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state noise transform matrix. It has shape
-        [self.state_dimendion, self.num_features].
+        [self.state_dimension, self.num_features].
     """
     # Noise is broadcast, through the moving average coefficients, to
     # un-observed parts of the latent state.
-- 
GitLab


From fdec18588d7f8b5f6383601f1030ed71f634d1c0 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 27 Mar 2018 09:36:52 -0700
Subject: [PATCH 543/960] Prevent warning every time someone imports
 contrib.learn.datasets.base

Everything in contrib/learn/python/learn/datasets/base.py has been deprecated. One of the function in there is a decorator, retry. Because another function in that file is decorated with retry, the function is called upon import, which prints a warning.

I have fixed this by adding a private function, _internal_retry, which is used internally, and redefining retry to simply call this. That way, using retry in user-code will still print the deprecated warning, but it's not printed upon every import.

I also cleaned up the docstrings slightly.

PiperOrigin-RevId: 190626717
---
 .../learn/python/learn/datasets/base.py       | 35 ++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/datasets/base.py b/tensorflow/contrib/learn/python/learn/datasets/base.py
index 3b5c9b97c0..4676eedb20 100644
--- a/tensorflow/contrib/learn/python/learn/datasets/base.py
+++ b/tensorflow/contrib/learn/python/learn/datasets/base.py
@@ -139,15 +139,48 @@ def retry(initial_delay,
 
   Args:
     initial_delay: the initial delay.
+    max_delay: the maximum delay allowed (actual max is
+        max_delay * (1 + jitter).
     factor: each subsequent retry, the delay is multiplied by this value.
         (must be >= 1).
     jitter: to avoid lockstep, the returned delay is multiplied by a random
         number between (1-jitter) and (1+jitter). To add a 20% jitter, set
         jitter = 0.2. Must be < 1.
+    is_retriable: (optional) a function that takes an Exception as an argument
+        and returns true if retry should be applied.
+
+  Returns:
+    A function that wraps another function to automatically retry it.
+  """
+  return _internal_retry(
+      initial_delay=initial_delay,
+      max_delay=max_delay,
+      factor=factor,
+      jitter=jitter,
+      is_retriable=is_retriable)
+
+
+def _internal_retry(initial_delay,
+                    max_delay,
+                    factor=2.0,
+                    jitter=0.25,
+                    is_retriable=None):
+  """Simple decorator for wrapping retriable functions, for internal use only.
+
+  Args:
+    initial_delay: the initial delay.
     max_delay: the maximum delay allowed (actual max is
         max_delay * (1 + jitter).
+    factor: each subsequent retry, the delay is multiplied by this value.
+        (must be >= 1).
+    jitter: to avoid lockstep, the returned delay is multiplied by a random
+        number between (1-jitter) and (1+jitter). To add a 20% jitter, set
+        jitter = 0.2. Must be < 1.
     is_retriable: (optional) a function that takes an Exception as an argument
         and returns true if retry should be applied.
+
+  Returns:
+    A function that wraps another function to automatically retry it.
   """
   if factor < 1:
     raise ValueError('factor must be >= 1; was %f' % (factor,))
@@ -195,7 +228,7 @@ def _is_retriable(e):
 
 
 @deprecated(None, 'Please use urllib or similar directly.')
-@retry(initial_delay=1.0, max_delay=16.0, is_retriable=_is_retriable)
+@_internal_retry(initial_delay=1.0, max_delay=16.0, is_retriable=_is_retriable)
 def urlretrieve_with_retry(url, filename=None):
   return urllib.request.urlretrieve(url, filename)
 
-- 
GitLab


From bba3c8f13516b4d4df83f179913376ab36807f9f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 10:04:41 -0700
Subject: [PATCH 544/960]   import tpu profiler analysis grpc python stub to
 tensorflow.

PiperOrigin-RevId: 190630641
---
 tensorflow/contrib/tpu/BUILD                                  | 2 ++
 tensorflow/contrib/tpu/profiler/BUILD                         | 4 +---
 .../contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py    | 2 +-
 tensorflow/contrib/tpu/python/profiler/__init__.py            | 1 +
 4 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index eea19e9465..95dc6f5ced 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -119,6 +119,8 @@ py_library(
     srcs = ["python/profiler/__init__.py"],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow/contrib/tpu/profiler:tpu_profiler_analysis_pb2_grpc",
+        "//tensorflow/contrib/tpu/profiler:tpu_profiler_analysis_proto_py",
         "//tensorflow/contrib/tpu/profiler:trace_events_proto_py",
         "//tensorflow/python:util",
     ],
diff --git a/tensorflow/contrib/tpu/profiler/BUILD b/tensorflow/contrib/tpu/profiler/BUILD
index 0a52d0b13b..56ddd7eff1 100644
--- a/tensorflow/contrib/tpu/profiler/BUILD
+++ b/tensorflow/contrib/tpu/profiler/BUILD
@@ -127,7 +127,5 @@ py_library(
     srcs = ["tpu_profiler_analysis_pb2_grpc.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
-    deps = [
-        ":tpu_profiler_analysis_proto_py",
-    ],
+    deps = [":tpu_profiler_analysis_proto_py"],
 )
diff --git a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py
index c28fef22a9..8f51488288 100644
--- a/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py
+++ b/tensorflow/contrib/tpu/profiler/tpu_profiler_analysis_pb2_grpc.py
@@ -22,7 +22,7 @@ from __future__ import print_function
 
 import grpc
 
-from third_party.tensorflow.contrib.tpu.profiler import tpu_profiler_analysis_pb2 as third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2
+from tensorflow.contrib.tpu.profiler import tpu_profiler_analysis_pb2 as third__party_dot_tensorflow_dot_contrib_dot_tpu_dot_profiler_dot_tpu__profiler__analysis__pb2
 
 
 class TPUProfileAnalysisStub(object):
diff --git a/tensorflow/contrib/tpu/python/profiler/__init__.py b/tensorflow/contrib/tpu/python/profiler/__init__.py
index bde13f0527..15ce6aceec 100644
--- a/tensorflow/contrib/tpu/python/profiler/__init__.py
+++ b/tensorflow/contrib/tpu/python/profiler/__init__.py
@@ -20,6 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=wildcard-import,unused-import
+from tensorflow.contrib.tpu.profiler.tpu_profiler_analysis_pb2 import *
 from tensorflow.contrib.tpu.profiler.trace_events_pb2 import *
 # pylint: enable=wildcard-import,unused-import
 
-- 
GitLab


From 1712002ad02f044f7569224bf465e0ea00e6a6c4 Mon Sep 17 00:00:00 2001
From: Nick Felt <nfelt@users.noreply.github.com>
Date: Tue, 27 Mar 2018 10:11:49 -0700
Subject: [PATCH 545/960] Update tb-nightly dep to >= 1.8.0a0, < 1.9.0a0
 (#18009)

Synchronize tf-nightly dep on current tb-nightly.
---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index ff30016cc2..3e4f9b0fdd 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -62,7 +62,7 @@ else:
 if 'tf_nightly' in project_name:
   for i, pkg in enumerate(REQUIRED_PACKAGES):
     if 'tensorboard' in pkg:
-      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.7.0a0, < 1.8.0a0'
+      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.8.0a0, < 1.9.0a0'
       break
 
 # weakref.finalize and enum were introduced in Python 3.4
-- 
GitLab


From bdaa9a0ce84798eb13b97de664451cd87c3f8210 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 10:20:05 -0700
Subject: [PATCH 546/960] Internal cleanup.

PiperOrigin-RevId: 190633067
---
 tensorflow/contrib/lite/testing/BUILD                          | 2 +-
 tensorflow/contrib/lite/testing/generated_examples_zip_test.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 555ea90034..dc9492f5e2 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -29,7 +29,7 @@ gen_zipped_test_files(
         "exp.zip",
         "fully_connected.zip",
         "fused_batch_norm.zip",
-        "gather.zip",
+        # "gather.zip",  #TODO(b/76437794)
         "global_batch_norm.zip",
         "l2_pool.zip",
         "l2norm.zip",
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index ba2d259462..08354b762c 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -244,7 +244,7 @@ INSTANTIATE_TESTS(div)
 INSTANTIATE_TESTS(exp)
 INSTANTIATE_TESTS(fully_connected)
 INSTANTIATE_TESTS(fused_batch_norm)
-INSTANTIATE_TESTS(gather)
+// INSTANTIATE_TESTS(gather)  //TODO(b/76437794)
 INSTANTIATE_TESTS(global_batch_norm)
 INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
-- 
GitLab


From f04822a1bb5b1bc50e8b41d4bc3a04d0641d93e1 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 27 Mar 2018 11:09:50 -0700
Subject: [PATCH 547/960] Match behavior of py_func in graph and eager.

PiperOrigin-RevId: 190641841
---
 tensorflow/python/ops/script_ops.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/script_ops.py b/tensorflow/python/ops/script_ops.py
index 1b4111bca6..96fb024715 100644
--- a/tensorflow/python/ops/script_ops.py
+++ b/tensorflow/python/ops/script_ops.py
@@ -334,7 +334,11 @@ def py_func(func, inp, Tout, stateful=True, name=None):
     result = func(*[x.numpy() for x in inp])
     result = nest.flatten(result)
 
-    return [x if x is None else ops.convert_to_tensor(x) for x in result]
+    result = [x if x is None else ops.convert_to_tensor(x) for x in result]
+    if len(result) == 1:
+      # Mimic the automatic unwrapping in graph-mode py_func
+      result, = result
+    return result
 
   return _internal_py_func(
       func=func, inp=inp, Tout=Tout, stateful=stateful, eager=False, name=name)
-- 
GitLab


From 5c1ad16bfd265da2268ab1820d411dfaeaca5e05 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 27 Mar 2018 11:27:11 -0700
Subject: [PATCH 548/960] Fix: Clamp takes three arguments after computation,
 not arbitrarily many.

PiperOrigin-RevId: 190644837
---
 tensorflow/docs_src/performance/xla/operation_semantics.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 5e39e710a0..4d12c7ab6d 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -241,7 +241,7 @@ See also
 
 Clamps an operand to within the range between a minimum and maximum value.
 
-<b> `Clamp(computation, args...)` </b>
+<b> `Clamp(computation, min, operand, max)` </b>
 
 | Arguments     | Type                    | Semantics                        |
 | ------------- | ----------------------- | -------------------------------- |
-- 
GitLab


From a185f4a4c203853506b0b1989f2322210ef27660 Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Tue, 27 Mar 2018 11:35:18 -0700
Subject: [PATCH 549/960] Trying to fix libtensorflow GPU build.

CUDNN path error.
Invalid path to cuDNN 7 toolkit. None of the following files can be found:
C:/tools/cuda\lib/x64/cudnn.lib
C:/tools/cuda\lib/x64/cudnn.lib
---
 tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
index 7b2d7e1a56..d654b433e7 100644
--- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
@@ -120,7 +120,9 @@ function run_configure_for_gpu_build {
   export TF_CUDA_VERSION=9.0
   export CUDA_TOOLKIT_PATH="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0"
   export TF_CUDNN_VERSION=7.0
-  export CUDNN_INSTALL_PATH="C:/tools/cuda"
+  if [ -z "$CUDNN_INSTALL_PATH" ]; then
+    export CUDNN_INSTALL_PATH="C:/tools/cuda"
+  fi
   export TF_CUDA_COMPUTE_CAPABILITIES="3.7"
   if [ -z "$TF_ENABLE_XLA" ]; then
     export TF_ENABLE_XLA=0
-- 
GitLab


From aec2496567a7bfd508fc487dec474263b6a7481f Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 27 Mar 2018 11:54:26 -0700
Subject: [PATCH 550/960] Exclude Python C extension from tensorflow/c:srcs
 target.

The Python extensions aren't part of the official C API.

PiperOrigin-RevId: 190649576
---
 tensorflow/c/BUILD | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 426f97b844..7f03e40d38 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -34,6 +34,8 @@ filegroup(
         exclude = [
             "c_api_experimental.cc",
             "c_api_experimental.h",
+            "python_api.cc",
+            "python_api.h",
             "*test*",
         ],
     ),
-- 
GitLab


From fd77211de17bf053cc8f5a82c8eff1818451120c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 12:00:44 -0700
Subject: [PATCH 551/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 190650553
---
 tensorflow/cc/framework/cc_op_gen_test.cc     |  5 +-
 tensorflow/cc/framework/scope.cc              |  3 +-
 tensorflow/compiler/aot/codegen_test.cc       |  3 +-
 tensorflow/compiler/aot/tfcompile_main.cc     |  2 +-
 .../compiler/xla/service/hlo_graph_dumper.cc  |  6 +-
 .../compiler/xla/service/user_computation.cc  |  4 +-
 .../xla/tests/compute_constant_test.cc        |  9 +--
 .../xla/tests/xla_hlo_profile_test.cc         |  4 +-
 tensorflow/contrib/cloud/kernels/BUILD        |  1 +
 .../kernels/bigquery_table_accessor_test.cc   |  5 +-
 .../session_bundle/session_bundle_test.cc     | 30 ++++----
 .../contrib/session_bundle/signature_test.cc  | 68 ++++++++++---------
 tensorflow/core/grappler/costs/BUILD          |  1 +
 .../core/grappler/costs/graph_properties.cc   |  4 +-
 tensorflow/core/lib/io/inputbuffer_test.cc    |  3 +-
 tensorflow/core/lib/io/recordio_test.cc       |  3 +-
 tensorflow/python/framework/python_op_gen.cc  |  2 +-
 .../python/framework/python_op_gen_main.cc    |  4 +-
 tensorflow/stream_executor/kernel.cc          |  3 +-
 tensorflow/stream_executor/lib/str_util.h     |  2 +-
 20 files changed, 92 insertions(+), 70 deletions(-)

diff --git a/tensorflow/cc/framework/cc_op_gen_test.cc b/tensorflow/cc/framework/cc_op_gen_test.cc
index 1e0f2d241b..5d9dfd95a5 100644
--- a/tensorflow/cc/framework/cc_op_gen_test.cc
+++ b/tensorflow/cc/framework/cc_op_gen_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_gen_lib.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -61,12 +62,12 @@ op {
 )";
 
 void ExpectHasSubstr(StringPiece s, StringPiece expected) {
-  EXPECT_TRUE(s.contains(expected))
+  EXPECT_TRUE(str_util::StrContains(s, expected))
       << "'" << s << "' does not contain '" << expected << "'";
 }
 
 void ExpectDoesNotHaveSubstr(StringPiece s, StringPiece expected) {
-  EXPECT_FALSE(s.contains(expected))
+  EXPECT_FALSE(str_util::StrContains(s, expected))
       << "'" << s << "' contains '" << expected << "'";
 }
 
diff --git a/tensorflow/cc/framework/scope.cc b/tensorflow/cc/framework/scope.cc
index 7164249262..c143b97833 100644
--- a/tensorflow/cc/framework/scope.cc
+++ b/tensorflow/cc/framework/scope.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 
@@ -218,7 +219,7 @@ std::unordered_set<string> Scope::Impl::GetColocationConstraints(
   if (GetNodeAttr(attrs, kColocationAttrName, &node_constraints).ok()) {
     for (const string& entry : node_constraints) {
       StringPiece s(entry);
-      if (s.Consume(kColocationGroupPrefix)) {
+      if (str_util::ConsumePrefix(&s, kColocationGroupPrefix)) {
         current_constraints.insert(s.ToString());
       }
     }
diff --git a/tensorflow/compiler/aot/codegen_test.cc b/tensorflow/compiler/aot/codegen_test.cc
index 972b7d51ec..2642536c4f 100644
--- a/tensorflow/compiler/aot/codegen_test.cc
+++ b/tensorflow/compiler/aot/codegen_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -33,7 +34,7 @@ namespace {
 
 void ExpectErrorContains(const Status& status, StringPiece str) {
   EXPECT_NE(Status::OK(), status);
-  EXPECT_TRUE(StringPiece(status.error_message()).contains(str))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(), str))
       << "expected error: " << status.error_message() << " to contain: " << str;
 }
 
diff --git a/tensorflow/compiler/aot/tfcompile_main.cc b/tensorflow/compiler/aot/tfcompile_main.cc
index e2f01179d4..8ea014c2ee 100644
--- a/tensorflow/compiler/aot/tfcompile_main.cc
+++ b/tensorflow/compiler/aot/tfcompile_main.cc
@@ -55,7 +55,7 @@ const char kUsageHeader[] =
     "\n";
 
 Status ReadProtoFile(const string& fname, protobuf::Message* proto) {
-  if (StringPiece(fname).ends_with(".pbtxt")) {
+  if (str_util::EndsWith(fname, ".pbtxt")) {
     return ReadTextProto(Env::Default(), fname, proto);
   } else {
     return ReadBinaryProto(Env::Default(), fname, proto);
diff --git a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
index 1dc72355cf..25702dc65e 100644
--- a/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
+++ b/tensorflow/compiler/xla/service/hlo_graph_dumper.cc
@@ -823,7 +823,7 @@ string HloDotDumper::GetInstructionNodeInlinedOperands(
 
     // Otherwise, print e.g. "%constant.42 (s32[100])".
     string constant_name;
-    if (tensorflow::StringPiece(constant->name()).starts_with("constant")) {
+    if (tensorflow::str_util::StartsWith(constant->name(), "constant")) {
       constant_name = constant->name();
     } else {
       constant_name = StrCat("constant ", constant->name());
@@ -1041,8 +1041,8 @@ string HloDotDumper::GetInstructionNodeLabel(const HloInstruction* instr) {
 
   // The HLO instruction name contains usually the opcode, e.g. "%add.42" is
   // an add instruction.  In this case we render just the name.
-  if (tensorflow::StringPiece(instr->name())
-          .starts_with(HloOpcodeString(instr->opcode()))) {
+  if (tensorflow::str_util::StartsWith(instr->name(),
+                                       HloOpcodeString(instr->opcode()))) {
     return Printf("<b>%s</b>", HtmlLikeStringSanitize(instr->name()));
   }
   string extended_opcode =
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index 0dca30a804..fcdb2e01fb 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -1284,8 +1284,8 @@ StatusOr<ComputationDataHandle> UserComputation::AddCustomCallInstruction(
     TF_RETURN_IF_ERROR(LookUpRequest(handle).status());
   }
 
-  if (tensorflow::StringPiece(custom_call_request.call_target_name())
-          .starts_with("$")) {
+  if (tensorflow::str_util::StartsWith(custom_call_request.call_target_name(),
+                                       "$")) {
     return InvalidArgument(
         "Invalid custom_call_target \"%s\": Call targets that start with '$' "
         "are reserved for internal use.",
diff --git a/tensorflow/compiler/xla/tests/compute_constant_test.cc b/tensorflow/compiler/xla/tests/compute_constant_test.cc
index ec2c580670..e5a03b49ad 100644
--- a/tensorflow/compiler/xla/tests/compute_constant_test.cc
+++ b/tensorflow/compiler/xla/tests/compute_constant_test.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/test_macros.h"
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace xla {
@@ -167,8 +168,8 @@ TEST_F(ComputeConstantTest, DirectParamMissing) {
     EXPECT_FALSE(IsConstant(computation, &b));
 
     auto value = ComputeConstantScalar<float>(client, computation, &b);
-    EXPECT_TRUE(tensorflow::StringPiece(value.status().ToString())
-                    .contains("depends on a parameter"))
+    EXPECT_TRUE(tensorflow::str_util::StrContains(value.status().ToString(),
+                                                  "depends on a parameter"))
         << value.status();
   }
 }
@@ -183,8 +184,8 @@ TEST_F(ComputeConstantTest, IndirectParamMissing) {
     EXPECT_FALSE(IsConstant(computation, &b));
 
     auto value = ComputeConstantScalar<float>(client, computation, &b);
-    EXPECT_TRUE(tensorflow::StringPiece(value.status().ToString())
-                    .contains("depends on a parameter"))
+    EXPECT_TRUE(tensorflow::str_util::StrContains(value.status().ToString(),
+                                                  "depends on a parameter"))
         << value.status();
   }
 }
diff --git a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
index 24b9f37a80..ff3418a128 100644
--- a/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
+++ b/tensorflow/compiler/xla/tests/xla_hlo_profile_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/tests/test_utils.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/regexp.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
@@ -294,7 +295,8 @@ XLA_TEST_F(HloProfileTest,
   auto while_body_profile_start =
       std::find_if(profile_output_lines.begin(), profile_output_lines.end(),
                    [](tensorflow::StringPiece s) {
-                     return s.starts_with("Execution profile for body");
+                     return tensorflow::str_util::StartsWith(
+                         s, "Execution profile for body");
                    });
 
   ASSERT_NE(while_body_profile_start, profile_output_lines.end());
diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD
index 56f930a9a8..d5fc604de9 100644
--- a/tensorflow/contrib/cloud/kernels/BUILD
+++ b/tensorflow/contrib/cloud/kernels/BUILD
@@ -73,6 +73,7 @@ tf_cc_test(
     ],
     deps = [
         ":bigquery_table_accessor",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
diff --git a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc
index e9b79a066d..7416eb19d3 100644
--- a/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc
+++ b/tensorflow/contrib/cloud/kernels/bigquery_table_accessor_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/example/feature.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/cloud/http_request_fake.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -28,8 +29,8 @@ constexpr char kTestProject[] = "test-project";
 constexpr char kTestDataset[] = "test-dataset";
 constexpr char kTestTable[] = "test-table";
 
-bool HasSubstr(const string& base, const string& substr) {
-  bool ok = StringPiece(base).contains(substr);
+bool HasSubstr(StringPiece base, StringPiece substr) {
+  bool ok = str_util::StrContains(base, substr);
   EXPECT_TRUE(ok) << base << ", expected substring " << substr;
   return ok;
 }
diff --git a/tensorflow/contrib/session_bundle/session_bundle_test.cc b/tensorflow/contrib/session_bundle/session_bundle_test.cc
index 6d997bac9e..612623ae30 100644
--- a/tensorflow/contrib/session_bundle/session_bundle_test.cc
+++ b/tensorflow/contrib/session_bundle/session_bundle_test.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/session.h"
@@ -239,8 +240,8 @@ TEST(LoadSessionBundleFromPath, BasicTestRunOptionsThreadPoolInvalid) {
 
   // Expect failed session run calls with invalid run-options.
   EXPECT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Invalid inter_op_thread_pool: 2"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Invalid inter_op_thread_pool: 2"))
       << status.error_message();
 }
 
@@ -314,8 +315,8 @@ TEST_F(SessionBundleTest, ServingGraphEmpty) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(StringPiece(status_.error_message())
-                  .contains("Expected exactly one serving GraphDef"))
+  EXPECT_TRUE(str_util::StrContains(status_.error_message(),
+                                    "Expected exactly one serving GraphDef"))
       << status_.error_message();
 }
 
@@ -330,8 +331,9 @@ TEST_F(SessionBundleTest, ServingGraphAnyIncorrectType) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(StringPiece(status_.error_message())
-                  .contains("Expected Any type_url for: tensorflow.GraphDef"))
+  EXPECT_TRUE(
+      str_util::StrContains(status_.error_message(),
+                            "Expected Any type_url for: tensorflow.GraphDef"))
       << status_.error_message();
 }
 
@@ -347,7 +349,8 @@ TEST_F(SessionBundleTest, ServingGraphAnyValueCorrupted) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(StringPiece(status_.error_message()).contains("Failed to unpack"))
+  EXPECT_TRUE(
+      str_util::StrContains(status_.error_message(), "Failed to unpack"))
       << status_.error_message();
 }
 
@@ -362,9 +365,9 @@ TEST_F(SessionBundleTest, AssetFileAnyIncorrectType) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(
-      StringPiece(status_.error_message())
-          .contains("Expected Any type_url for: tensorflow.serving.AssetFile"))
+  EXPECT_TRUE(str_util::StrContains(
+      status_.error_message(),
+      "Expected Any type_url for: tensorflow.serving.AssetFile"))
       << status_.error_message();
 }
 
@@ -380,7 +383,8 @@ TEST_F(SessionBundleTest, AssetFileAnyValueCorrupted) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(StringPiece(status_.error_message()).contains("Failed to unpack"))
+  EXPECT_TRUE(
+      str_util::StrContains(status_.error_message(), "Failed to unpack"))
       << status_.error_message();
 }
 
@@ -395,8 +399,8 @@ TEST_F(SessionBundleTest, InitOpTooManyValues) {
   });
   status_ = LoadSessionBundleFromPath(options_, path, &bundle_);
   EXPECT_FALSE(status_.ok());
-  EXPECT_TRUE(StringPiece(status_.error_message())
-                  .contains("Expected exactly one serving init op"))
+  EXPECT_TRUE(str_util::StrContains(status_.error_message(),
+                                    "Expected exactly one serving init op"))
       << status_.error_message();
 }
 
diff --git a/tensorflow/contrib/session_bundle/signature_test.cc b/tensorflow/contrib/session_bundle/signature_test.cc
index 741b7fde9b..b1ff55552e 100644
--- a/tensorflow/contrib/session_bundle/signature_test.cc
+++ b/tensorflow/contrib/session_bundle/signature_test.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session.h"
 
@@ -33,8 +34,8 @@ namespace tensorflow {
 namespace serving {
 namespace {
 
-static bool HasSubstr(const string& base, const string& substr) {
-  bool ok = StringPiece(base).contains(substr);
+static bool HasSubstr(StringPiece base, StringPiece substr) {
+  bool ok = str_util::StrContains(base, substr);
   EXPECT_TRUE(ok) << base << ", expected substring " << substr;
   return ok;
 }
@@ -69,8 +70,8 @@ TEST(GetClassificationSignature, MissingSignature) {
   ClassificationSignature signature;
   const Status status = GetClassificationSignature(meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected a classification signature"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected a classification signature"))
       << status.error_message();
 }
 
@@ -86,8 +87,8 @@ TEST(GetClassificationSignature, WrongSignatureType) {
   ClassificationSignature signature;
   const Status status = GetClassificationSignature(meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected a classification signature"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected a classification signature"))
       << status.error_message();
 }
 
@@ -122,8 +123,8 @@ TEST(GetNamedClassificationSignature, MissingSignature) {
   const Status status =
       GetNamedClassificationSignature("foo", meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Missing signature named \"foo\""))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Missing signature named \"foo\""))
       << status.error_message();
 }
 
@@ -141,9 +142,9 @@ TEST(GetNamedClassificationSignature, WrongSignatureType) {
   const Status status =
       GetNamedClassificationSignature("foo", meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(
-      StringPiece(status.error_message())
-          .contains("Expected a classification signature for name \"foo\""))
+  EXPECT_TRUE(str_util::StrContains(
+      status.error_message(),
+      "Expected a classification signature for name \"foo\""))
       << status.error_message();
 }
 
@@ -176,8 +177,8 @@ TEST(GetRegressionSignature, MissingSignature) {
   RegressionSignature signature;
   const Status status = GetRegressionSignature(meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected a regression signature"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected a regression signature"))
       << status.error_message();
 }
 
@@ -193,8 +194,8 @@ TEST(GetRegressionSignature, WrongSignatureType) {
   RegressionSignature signature;
   const Status status = GetRegressionSignature(meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected a regression signature"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected a regression signature"))
       << status.error_message();
 }
 
@@ -227,8 +228,8 @@ TEST(GetNamedSignature, MissingSignature) {
   Signature signature;
   const Status status = GetNamedSignature("foo", meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Missing signature named \"foo\""))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Missing signature named \"foo\""))
       << status.error_message();
 }
 
@@ -370,7 +371,7 @@ TEST(RunClassification, RunNotOk) {
   const Status status = RunClassification(signature, input_tensor, &session,
                                           &classes_tensor, nullptr);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("Data is gone"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(), "Data is gone"))
       << status.error_message();
 }
 
@@ -386,7 +387,8 @@ TEST(RunClassification, TooManyOutputs) {
   const Status status = RunClassification(signature, input_tensor, &session,
                                           &classes_tensor, nullptr);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("Expected 1 output"))
+  EXPECT_TRUE(
+      str_util::StrContains(status.error_message(), "Expected 1 output"))
       << status.error_message();
 }
 
@@ -402,8 +404,9 @@ TEST(RunClassification, WrongBatchOutputs) {
   const Status status = RunClassification(signature, input_tensor, &session,
                                           &classes_tensor, nullptr);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Input batch size did not match output batch size"))
+  EXPECT_TRUE(
+      str_util::StrContains(status.error_message(),
+                            "Input batch size did not match output batch size"))
       << status.error_message();
 }
 
@@ -449,7 +452,7 @@ TEST_F(RunRegressionTest, RunNotOk) {
   const Status status =
       RunRegression(signature_, input_tensor_, &session_, &output_tensor_);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("Data is gone"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(), "Data is gone"))
       << status.error_message();
 }
 
@@ -460,8 +463,9 @@ TEST_F(RunRegressionTest, MismatchedSizeForBatchInputAndOutput) {
   const Status status =
       RunRegression(signature_, input_tensor_, &session_, &output_tensor_);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Input batch size did not match output batch size"))
+  EXPECT_TRUE(
+      str_util::StrContains(status.error_message(),
+                            "Input batch size did not match output batch size"))
       << status.error_message();
 }
 
@@ -488,7 +492,7 @@ TEST(GetSignatures, MissingSignature) {
   const auto status = GetSignatures(meta_graph_def, &read_signatures);
   EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Expected exactly one"))
+      str_util::StrContains(status.error_message(), "Expected exactly one"))
       << status.error_message();
 }
 
@@ -502,9 +506,9 @@ TEST(GetSignatures, WrongProtoInAny) {
   Signatures read_signatures;
   const auto status = GetSignatures(meta_graph_def, &read_signatures);
   EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected Any type_url for: "
-                            "tensorflow.serving.Signatures"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected Any type_url for: "
+                                    "tensorflow.serving.Signatures"))
       << status.error_message();
 }
 
@@ -519,7 +523,7 @@ TEST(GetSignatures, JunkInAny) {
   Signatures read_signatures;
   const auto status = GetSignatures(meta_graph_def, &read_signatures);
   EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("Failed to unpack"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(), "Failed to unpack"))
       << status.error_message();
 }
 
@@ -567,7 +571,7 @@ TEST(GetSignatures, MultipleSignaturesNotOK) {
   const auto status = GetSignatures(meta_graph_def, &read_signatures);
   EXPECT_EQ(tensorflow::error::FAILED_PRECONDITION, status.code());
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Expected exactly one"))
+      str_util::StrContains(status.error_message(), "Expected exactly one"))
       << status.error_message();
 }
 
@@ -641,8 +645,8 @@ TEST(GetGenericSignature, WrongSignatureType) {
   const Status status =
       GetGenericSignature("generic_bindings", meta_graph_def, &signature);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Expected a generic signature:"))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(),
+                                    "Expected a generic signature:"))
       << status.error_message();
 }
 
diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index 5336df1f51..df5a26f475 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -55,6 +55,7 @@ cc_library(
         ":utils",
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler:utils",
diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 817247e379..a5fd79447d 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/grappler/costs/utils.h"
 #include "tensorflow/core/grappler/utils.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -251,8 +252,7 @@ typename DisjointSet<Handle>::Rep* DisjointSet<Handle>::Find(Handle value) {
 }
 
 bool IsQueue(const Node& node) {
-  StringPiece type(node.type_string());
-  return type.ends_with("QueueV2");
+  return str_util::EndsWith(node.type_string(), "QueueV2");
 }
 
 // Returns true if the node is an Enter op AND its input is a Queue.
diff --git a/tensorflow/core/lib/io/inputbuffer_test.cc b/tensorflow/core/lib/io/inputbuffer_test.cc
index 6be1f819c2..3608008b30 100644
--- a/tensorflow/core/lib/io/inputbuffer_test.cc
+++ b/tensorflow/core/lib/io/inputbuffer_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
@@ -287,7 +288,7 @@ TEST(InputBuffer, Seek) {
     EXPECT_TRUE(errors::IsOutOfRange(in.ReadNBytes(1, &read)));
 
     EXPECT_TRUE(
-        StringPiece(in.Seek(-1).ToString()).contains("negative position"));
+        str_util::StrContains(in.Seek(-1).ToString(), "negative position"));
   }
 }
 
diff --git a/tensorflow/core/lib/io/recordio_test.cc b/tensorflow/core/lib/io/recordio_test.cc
index b7e51256a2..63235761d9 100644
--- a/tensorflow/core/lib/io/recordio_test.cc
+++ b/tensorflow/core/lib/io/recordio_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/record_reader.h"
 #include "tensorflow/core/lib/io/record_writer.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -218,7 +219,7 @@ TEST_F(RecordioTest, RandomRead) {
 
 // Tests of all the error paths in log_reader.cc follow:
 static void AssertHasSubstr(StringPiece s, StringPiece expected) {
-  EXPECT_TRUE(StringPiece(s).contains(expected))
+  EXPECT_TRUE(str_util::StrContains(s, expected))
       << s << " does not contain " << expected;
 }
 
diff --git a/tensorflow/python/framework/python_op_gen.cc b/tensorflow/python/framework/python_op_gen.cc
index 9850f0becc..e5e3b82199 100644
--- a/tensorflow/python/framework/python_op_gen.cc
+++ b/tensorflow/python/framework/python_op_gen.cc
@@ -448,7 +448,7 @@ string AttrValueToPython(const string& type, const AttrValue& value,
     return TensorToPython(value.tensor());
   } else if (type == "func") {
     return StringToPython(value.func().name());
-  } else if (StringPiece(type).starts_with("list(")) {
+  } else if (str_util::StartsWith(type, "list(")) {
     return strings::StrCat("[", AttrListToPython(value, dtype_module), "]");
   } else {
     return "?";
diff --git a/tensorflow/python/framework/python_op_gen_main.cc b/tensorflow/python/framework/python_op_gen_main.cc
index bc5ca195da..ca6ed42bee 100644
--- a/tensorflow/python/framework/python_op_gen_main.cc
+++ b/tensorflow/python/framework/python_op_gen_main.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/lib/io/inputbuffer.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/scanner.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
@@ -95,7 +96,8 @@ string InferSourceFileName(const char* argv_zero) {
   // operators defined in <op type>_ops.cc
   const char* kExecPrefix = "gen_";
   const char* kExecSuffix = "_py_wrappers_cc";
-  if (command_str.Consume(kExecPrefix) && command_str.ends_with(kExecSuffix)) {
+  if (str_util::ConsumePrefix(&command_str, kExecPrefix) &&
+      str_util::EndsWith(command_str, kExecSuffix)) {
     command_str.remove_suffix(strlen(kExecSuffix));
     return strings::StrCat(command_str, ".cc");
   } else {
diff --git a/tensorflow/stream_executor/kernel.cc b/tensorflow/stream_executor/kernel.cc
index 81e531efb3..636199cfa2 100644
--- a/tensorflow/stream_executor/kernel.cc
+++ b/tensorflow/stream_executor/kernel.cc
@@ -21,6 +21,7 @@ limitations under the License.
 
 #include "tensorflow/stream_executor/platform/port.h"
 
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/stream_executor/lib/demangle.h"
 #include "tensorflow/stream_executor/platform.h"
 #include "tensorflow/stream_executor/platform/logging.h"
@@ -96,7 +97,7 @@ static const char *kStubPrefix = "__device_stub_";
 void KernelBase::set_name(port::StringPiece name) {
   name_ = name.ToString();
   port::StringPiece stubless_name = name;
-  if (name.starts_with(kStubPrefix)) {
+  if (tensorflow::str_util::StartsWith(name, kStubPrefix)) {
     stubless_name.remove_prefix(strlen(kStubPrefix));
   }
   demangled_name_ = port::Demangle(stubless_name.data());
diff --git a/tensorflow/stream_executor/lib/str_util.h b/tensorflow/stream_executor/lib/str_util.h
index 4dd6f3b0cc..5dd3d06aff 100644
--- a/tensorflow/stream_executor/lib/str_util.h
+++ b/tensorflow/stream_executor/lib/str_util.h
@@ -29,7 +29,7 @@ using tensorflow::str_util::Split;
 // Returns a copy of the input string 'str' with the given 'suffix'
 // removed. If the suffix doesn't match, returns a copy of the original string.
 inline string StripSuffixString(port::StringPiece str, port::StringPiece suffix) {
-  if (str.ends_with(suffix)) {
+  if (tensorflow::str_util::EndsWith(str, suffix)) {
     str.remove_suffix(suffix.size());
   }
   return str.ToString();
-- 
GitLab


From 7c06ae2fd9b933e83aea0e5088c0b32b7c1fcaaf Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 27 Mar 2018 12:06:19 -0700
Subject: [PATCH 552/960] Remove warnings for initialize_variables (#18023)

The `initialize_variables` has been deprecated and replaced with
`tf.variables_initializer`.

This fix makes the change and fixes the following warning in array_ops_test.py:
```
WARNING:tensorflow:From /private/var/tmp/_bazel_ytang/48f7de64c479bcefe5e55c65866b55a6/execroot/org_tensorflow/bazel-out/darwin-opt/bin/tensorflow/python/kernel_tests/array_ops_test.runfiles/org_tensorflow/tensorflow/python/util/tf_should_use.py:118: initialize_variables (from tensorflow.python.ops.variables) is deprecated and will be removed after 2017-03-02.
Instructions for updating:
Use `tf.variables_initializer` instead.
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/kernel_tests/array_ops_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index d0ba8020c1..b82aa47ebe 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -890,7 +890,7 @@ class StridedSliceAssignChecker(object):
         var = resource_variable_ops.ResourceVariable(self.x)
       else:
         var = variables.Variable(self.x)
-      sess.run(variables.initialize_variables([var]))
+      sess.run(variables.variables_initializer([var]))
       val = sess.run(var[index].assign(value))
       # val_copy is used to check that tf.assign works equivalently to the
       # assign method above.
-- 
GitLab


From 4a24413ee92c23727e11108bfd9b823ac09ef209 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 27 Mar 2018 12:11:39 -0700
Subject: [PATCH 553/960] Validate axis in shape function of tf.reverse
 (#18024)

* Validate axis in shape function of tf.reverse

tf.reverse requires the axis to be in the range of
`[-rank(tensor), rank(tensor))`. Previously the validation
is only done in runtime though it is possible to validate
axis inside the shape function if the shape of the input tensor
is already known.

This fix add the validation in the shape function.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Replace with temp variable

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Sanitize with clang-foramt -i

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Validate multiple specification of axis in tf.reverse as well

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for axis validation in shape function for tf.reverse

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update existing test cases

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/ops/array_ops.cc              | 26 ++++++++++++++++++-
 .../python/kernel_tests/array_ops_test.py     | 24 ++++++++++++++---
 2 files changed, 46 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 39b92464cb..f97f1645a6 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -752,11 +752,35 @@ REGISTER_OP("ReverseV2")
       ShapeHandle input = c->input(0);
       ShapeHandle axis;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &axis));
-      // TODO(aselle): if input(0)'s dimension is known we could validate axis
       if (c->Rank(input) > 8) {
         return errors::InvalidArgument(
             "reverse does not work on tensors with more than 8 dimensions");
       }
+      const Tensor* axis_tensor = c->input_tensor(1);
+      if (axis_tensor != nullptr && c->RankKnown(input)) {
+        int32 rank = c->Rank(input);
+        std::vector<int64> axis_value;
+        if (axis_tensor->dtype() == DT_INT32) {
+          axis_value = AsInt64<int32>(axis_tensor, axis_tensor->NumElements());
+        } else {
+          axis_value = AsInt64<int64>(axis_tensor, axis_tensor->NumElements());
+        }
+        std::vector<bool> axes_dense(c->Rank(input), false);
+        for (int i = 0; i < axis_value.size(); i++) {
+          int64 canonical_axis =
+              axis_value[i] < 0 ? rank + axis_value[i] : axis_value[i];
+          if (canonical_axis < 0 || canonical_axis >= rank) {
+            return errors::InvalidArgument("'axis'[", i, "] = ", axis_value[i],
+                                           " is out of valid range [", 0, ", ",
+                                           rank - 1);
+          }
+          if (axes_dense[canonical_axis]) {
+            return errors::InvalidArgument("axis ", canonical_axis,
+                                            " specified more than once.");
+          }
+          axes_dense[canonical_axis] = true;
+        }
+      }
       c->set_output(0, input);
       return Status::OK();
     });
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index b82aa47ebe..64c1760d5e 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -315,21 +315,39 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
             self.assertAllEqual(x_tf_4, np.asarray(x_np)[:, ::-1])
             self.assertAllEqual(x_tf_5, np.asarray(x_np)[::-1, ::-1])
 
+  # This test covers the axis validation in the shape function
+  # (no eval())
+  def testInvalidAxis(self):
+    x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [-30])
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [2])
+    with self.assertRaisesRegexp(ValueError,
+                                 "axis 0 specified more than once"):
+      array_ops.reverse_v2(x_np, [0, -2])
+
   # This is the version of reverse that uses axis indices rather than
   # bool tensors
   # TODO(b/32254538): Change this test to use array_ops.reverse
+  #
+  # Note: this test passes placeholder as constant axis is validated
+  # in shape function (see testInvalidAxis)
   def testInvalid(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    axis = array_ops.placeholder(dtypes.int32)
     with self.test_session():
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [-30]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [-30]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [2]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "axis 0 specified more than once"):
-        array_ops.reverse_v2(x_np, [0, -2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]})
 
   def testReverse1DimAuto(self):
     for dtype in [
-- 
GitLab


From 5da1cdcf0032f63c22afb41a460fd44c52ada048 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 27 Mar 2018 12:09:59 -0700
Subject: [PATCH 554/960] Improved shape inference for reshape

PiperOrigin-RevId: 190651873
---
 .../python/kernel_tests/shape_ops_test.py     |   5 +-
 .../contrib/signal/python/ops/shape_ops.py    |   2 +
 tensorflow/core/ops/array_ops.cc              | 104 ++++++++++++------
 tensorflow/core/ops/array_ops_test.cc         |   6 +-
 4 files changed, 84 insertions(+), 33 deletions(-)

diff --git a/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py b/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
index 1c052354b8..bc4663fbb0 100644
--- a/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
@@ -338,7 +338,10 @@ class FrameTest(test.TestCase):
 
   def test_constant_folding(self):
     """frame should be constant foldable for constant inputs."""
-    for pad_end in [False, True]:
+    # Padding is incorrectly defined in shape_ops.py (the rank of the padding
+    # tensor should be equal to the rank of the input tensor + 1): only test
+    # with padding set to False to avoid this.
+    for pad_end in [False]:
       g = ops.Graph()
       with g.as_default():
         frame_length, frame_step = 32, 16
diff --git a/tensorflow/contrib/signal/python/ops/shape_ops.py b/tensorflow/contrib/signal/python/ops/shape_ops.py
index 1ddc2941ec..97fe20866b 100644
--- a/tensorflow/contrib/signal/python/ops/shape_ops.py
+++ b/tensorflow/contrib/signal/python/ops/shape_ops.py
@@ -139,6 +139,8 @@ def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1,
            [[0, pad_samples]],
            array_ops.zeros([num_inner_dimensions, 2], dtype=pad_samples.dtype)],
           0)
+      # TODO(rjryan): the paddings tensor must of rank tf.rank(signal) + 1. This
+      # isn't the case here and should be fixed.
       signal = array_ops.pad(signal, paddings, constant_values=pad_value)
 
       signal_shape = array_ops.shape(signal)
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 39b92464cb..88d2aa3f41 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -178,46 +178,88 @@ Status SetOutputShapeForReshape(InferenceContext* c) {
     c->set_output(0, out);
     return Status::OK();
   }
-  DimensionHandle num_in_elems = c->NumElements(in);
-  if (c->FullyDefined(out)) {
-    DimensionHandle num_out_elems = c->NumElements(out);
-    if (c->ValueKnown(num_in_elems) &&
-        c->Value(num_in_elems) != c->Value(num_out_elems)) {
-      return errors::InvalidArgument(
-          "Cannot reshape a tensor with ", c->DebugString(num_in_elems),
-          " elements to shape ", c->DebugString(out), " (",
-          c->DebugString(num_out_elems), " elements)");
-    }
-    c->set_output(0, out);
-    return Status::OK();
-  }
 
-  if (c->ValueKnown(num_in_elems)) {
+  if (c->RankKnown(out) && c->RankKnown(in)) {
     // We don't know the number of output elements, but we can try to infer
     // the missing dimension.
-    int32 unknown_idx = -1;
     bool too_many_unknown = false;
-    DimensionHandle known_elems = c->MakeDim(1);
-    for (int32 i = 0; i < c->Rank(out); ++i) {
-      DimensionHandle dim = c->Dim(out, i);
-      if (!c->ValueKnown(dim)) {
-        if (unknown_idx >= 0) {
-          too_many_unknown = true;
-          break;
+    int32 out_unknown_idx = -1;
+
+    DimensionHandle known_out_elems = c->NumElements(out);
+    if (!c->ValueKnown(known_out_elems)) {
+      known_out_elems = c->MakeDim(1);
+      for (int32 i = 0; i < c->Rank(out); ++i) {
+        DimensionHandle dim = c->Dim(out, i);
+        if (!c->ValueKnown(dim)) {
+          if (out_unknown_idx >= 0) {
+            too_many_unknown = true;
+            break;
+          }
+          out_unknown_idx = i;
+        } else {
+          TF_RETURN_IF_ERROR(
+              c->Multiply(known_out_elems, dim, &known_out_elems));
         }
-        unknown_idx = i;
-      } else {
-        TF_RETURN_IF_ERROR(c->Multiply(known_elems, dim, &known_elems));
       }
     }
-    if (!too_many_unknown && c->Value(known_elems) != 0) {
-      DimensionHandle inferred_dim;
-      TF_RETURN_IF_ERROR(c->Divide(num_in_elems, c->Value(known_elems),
-                                   true /* evenly_divisible */, &inferred_dim));
-      TF_RETURN_IF_ERROR(c->ReplaceDim(out, unknown_idx, inferred_dim, &out));
+    int32 in_unknown_idx = -1;
+    DimensionHandle known_in_elems = c->NumElements(in);
+    if (!c->ValueKnown(known_in_elems)) {
+      known_in_elems = c->MakeDim(1);
+      for (int32 i = 0; i < c->Rank(in); ++i) {
+        DimensionHandle dim = c->Dim(in, i);
+        if (!c->ValueKnown(dim)) {
+          if (in_unknown_idx >= 0) {
+            too_many_unknown = true;
+            break;
+          }
+          in_unknown_idx = i;
+        } else {
+          TF_RETURN_IF_ERROR(c->Multiply(known_in_elems, dim, &known_in_elems));
+        }
+      }
     }
-  }
 
+    if (!too_many_unknown) {
+      if (in_unknown_idx < 0 && out_unknown_idx < 0) {
+        // Just check that the dimensions match.
+        if (c->Value(known_in_elems) != c->Value(known_out_elems)) {
+          return errors::InvalidArgument(
+              "Cannot reshape a tensor with ", c->DebugString(known_in_elems),
+              " elements to shape ", c->DebugString(out), " (",
+              c->DebugString(known_out_elems), " elements)");
+        }
+      } else if (in_unknown_idx < 0 && out_unknown_idx >= 0 &&
+                 c->Value(known_out_elems) > 0) {
+        // Input fully known, infer the one missing output dim
+        DimensionHandle inferred_dim;
+        TF_RETURN_IF_ERROR(c->Divide(known_in_elems, c->Value(known_out_elems),
+                                     true /* evenly_divisible */,
+                                     &inferred_dim));
+        TF_RETURN_IF_ERROR(
+            c->ReplaceDim(out, out_unknown_idx, inferred_dim, &out));
+
+      } else if (in_unknown_idx >= 0 && out_unknown_idx < 0 &&
+                 c->Value(known_in_elems) != 0) {
+        // Output fully known, infer the one missing input dim
+        DimensionHandle inferred_dim;
+        TF_RETURN_IF_ERROR(c->Divide(known_out_elems, c->Value(known_in_elems),
+                                     true /* evenly_divisible */,
+                                     &inferred_dim));
+        DimensionHandle unknown_in_dim = c->Dim(in, in_unknown_idx);
+        TF_RETURN_IF_ERROR(
+            c->Merge(unknown_in_dim, inferred_dim, &unknown_in_dim));
+      } else if (in_unknown_idx >= 0 && out_unknown_idx >= 0) {
+        // Exactly one unknown dimension in both input and output. These 2 are
+        // equal iff the known elements are equal.
+        if (c->Value(known_in_elems) == c->Value(known_out_elems)) {
+          DimensionHandle unknown_in_dim = c->Dim(in, in_unknown_idx);
+          TF_RETURN_IF_ERROR(
+              c->ReplaceDim(out, out_unknown_idx, unknown_in_dim, &out));
+        }
+      }
+    }
+  }
   c->set_output(0, out);
   return Status::OK();
 }
diff --git a/tensorflow/core/ops/array_ops_test.cc b/tensorflow/core/ops/array_ops_test.cc
index cf5bb5ad84..b1463338fb 100644
--- a/tensorflow/core/ops/array_ops_test.cc
+++ b/tensorflow/core/ops/array_ops_test.cc
@@ -838,7 +838,7 @@ TEST(ArrayOpsTest, Reshape_ShapeFn) {
   // Unknown dimensions.
   // Flatten:
   new_shape = test::AsTensor<int32>({-1});
-  INFER_OK(op, "[?];[1]", "[?]");
+  INFER_OK(op, "[?];[1]", "[d0_0]");
   INFER_OK(op, "[2,2];[1]", "[4]");
   // The first dimension is inferred:
   new_shape = test::AsTensor<int32>({2, -1});
@@ -851,6 +851,10 @@ TEST(ArrayOpsTest, Reshape_ShapeFn) {
   new_shape = test::AsTensor<int32>({-1, -1, 2});
   INFER_OK(op, "[8];[3]", "[?,?,2]");
 
+  // Symbolic shape propagation
+  new_shape = test::AsTensor<int32>({-1, 2, 3});
+  INFER_OK(op, "[?,2,3];[3]", "[d0_0,2,3]");
+
   // Reshaping to a scalar.
   new_shape = test::AsTensor<int32>({});
   INFER_OK(op, "[1];[0]", "[]");
-- 
GitLab


From 3771b5b0d9cbd5a9d34f1d579454b78012cb0bb4 Mon Sep 17 00:00:00 2001
From: Rasmus Munk Larsen <rmlarsen@google.com>
Date: Tue, 27 Mar 2018 12:22:49 -0700
Subject: [PATCH 555/960] Update BUILD

---
 tensorflow/python/kernel_tests/testdata/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/testdata/BUILD b/tensorflow/python/kernel_tests/testdata/BUILD
index a4a0dfc139..45264c773a 100644
--- a/tensorflow/python/kernel_tests/testdata/BUILD
+++ b/tensorflow/python/kernel_tests/testdata/BUILD
@@ -1,7 +1,7 @@
 # Data files for kernel tests.
 
 package(
-    default_visibility = ["//tensorflow:internal"],
+    default_visibility = ["//visibility:public"],
 )
 
 licenses(["notice"])  # Apache 2.0
-- 
GitLab


From 7fd3ca7ab6e96af7b867c7ae56ac74a3f3393b26 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 12:34:17 -0700
Subject: [PATCH 556/960] Updating test so that it evaluates the optimized and
 original graph and checks whether the output tensors produced by them are the
 same.

PiperOrigin-RevId: 190655831
---
 .../core/grappler/optimizers/constant_folding_test.cc       | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 6340565bcd..dc9c1053d2 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -614,7 +614,8 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
   GrapplerItem item;
   item.fetch.push_back("e");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
-
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -641,6 +642,9 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
     }
   }
   EXPECT_EQ(1, found);
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<int>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
-- 
GitLab


From 2700e87f0e5fbc3aa7fe3a6a7ffb7152b894da4a Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Wed, 28 Mar 2018 03:54:20 +0800
Subject: [PATCH 557/960] Fix the incorect rendering of math equation in
 monte_carlo api guides (#18018)

* Fix the math equation in monte_carlo api guides

* Replace \( \) with \\(  \\) according to guideline
---
 .../python/contrib.bayesflow.monte_carlo.md   | 36 +++++++++----------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
index 956dccb64f..f3db5857ae 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
@@ -6,42 +6,42 @@ Monte Carlo integration and helpers.
 ## Background
 
 Monte Carlo integration refers to the practice of estimating an expectation with
-a sample mean.  For example, given random variable `Z in R^k` with density `p`,
+a sample mean.  For example, given random variable `Z in \\(R^k\\)` with density `p`,
 the expectation of function `f` can be approximated like:
 
 ```
-E_p[f(Z)] = \int f(z) p(z) dz
-          ~ S_n
-          := n^{-1} \sum_{i=1}^n f(z_i),  z_i iid samples from p.
+$$E_p[f(Z)] = \int f(z) p(z) dz$$
+$$          ~ S_n
+          := n^{-1} \sum_{i=1}^n f(z_i),  z_i\ iid\ samples\ from\ p.$$
 ```
 
-If `E_p[|f(Z)|] < infinity`, then `S_n --> E_p[f(Z)]` by the strong law of large
-numbers.  If `E_p[f(Z)^2] < infinity`, then `S_n` is asymptotically normal with
-variance `Var[f(Z)] / n`.
+If `\\(E_p[|f(Z)|] < infinity\\)`, then `\\(S_n\\) --> \\(E_p[f(Z)]\\)` by the strong law of large
+numbers.  If `\\(E_p[f(Z)^2] < infinity\\)`, then `\\(S_n\\)` is asymptotically normal with
+variance `\\(Var[f(Z)] / n\\)`.
 
 Practitioners of Bayesian statistics often find themselves wanting to estimate
-`E_p[f(Z)]` when the distribution `p` is known only up to a constant.  For
+`\\(E_p[f(Z)]\\)` when the distribution `p` is known only up to a constant.  For
 example, the joint distribution `p(z, x)` may be known, but the evidence
-`p(x) = \int p(z, x) dz` may be intractable.  In that case, a parameterized
-distribution family `q_lambda(z)` may be chosen, and the optimal `lambda` is the
-one minimizing the KL divergence between `q_lambda(z)` and
-`p(z | x)`.  We only know `p(z, x)`, but that is sufficient to find `lambda`.
+`\\(p(x) = \int p(z, x) dz\\)` may be intractable.  In that case, a parameterized
+distribution family `\\(q_\lambda(z)\\)` may be chosen, and the optimal `\\(\lambda\\)` is the
+one minimizing the KL divergence between `\\(q_\lambda(z)\\)` and
+`\\(p(z | x)\\)`.  We only know `p(z, x)`, but that is sufficient to find `\\(\lambda\\)`.
 
 
 ## Log-space evaluation and subtracting the maximum
 
 Care must be taken when the random variable lives in a high dimensional space.
-For example, the naive importance sample estimate `E_q[f(Z) p(Z) / q(Z)]`
-involves the ratio of two terms `p(Z) / q(Z)`, each of which must have tails
-dropping off faster than `O(|z|^{-(k + 1)})` in order to have finite integral.
+For example, the naive importance sample estimate `\\(E_q[f(Z) p(Z) / q(Z)]\\)`
+involves the ratio of two terms `\\(p(Z) / q(Z)\\)`, each of which must have tails
+dropping off faster than `\\(O(|z|^{-(k + 1)})\\)` in order to have finite integral.
 This ratio would often be zero or infinity up to numerical precision.
 
 For that reason, we write
 
 ```
-Log E_q[ f(Z) p(Z) / q(Z) ]
-   = Log E_q[ exp{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C} ] + C,  where
-C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].
+$$Log E_q[ f(Z) p(Z) / q(Z) ]$$
+$$   = Log E_q[ \exp\{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C\} ] + C,$$  where
+$$C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].$$
 ```
 
 The maximum value of the exponentiated term will be 0.0, and the expectation
-- 
GitLab


From 5d76c7db2ab72f9b0cc70ce12ba0a3395dcc20d3 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Wed, 28 Mar 2018 03:55:28 +0800
Subject: [PATCH 558/960] Fix minor spelling typos in contrib (#18015)

---
 .../contrib/bayesflow/python/ops/metropolis_hastings_impl.py    | 2 +-
 .../contrib/estimator/python/estimator/replicate_model_fn.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
index 05aa134ed5..fdee0a8da6 100644
--- a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
+++ b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
@@ -238,7 +238,7 @@ def evolve(initial_sample,
   using the Metropolis-Hastings algorithm. These samples are from a Markov chain
   whose equilibrium distribution matches the target distribution.
 
-  The probability distribution may have an unknown normalization constan.
+  The probability distribution may have an unknown normalization constant.
   We parameterize the probability density as follows:
 
   ```none
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index e0fae2c992..fa2697800e 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replice only on the subset of available GPUs.
+      argument can be used to replicate only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
-- 
GitLab


From 0ef36a5de45486ccbc0d6237f86280c2ac22f52e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 27 Mar 2018 12:56:14 -0700
Subject: [PATCH 559/960] Add broadcast support for
 softmax_cross_entropy_with_logits (#16784)

* Add broadcast support for softmax_cross_entropy_with_logits

This fix tries to address the issue raised in 11534 where
there was no broadcast support for SoftmaxCrossEntropyWithLogits.
This fix adds the broadcast support for SoftmaxCrossEntropyWithLogits,
and adds test cases for it.

This fix fixes 11534.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add BroadcastBinaryOpOutputShapeFn for shape function

This commit adds BroadcastBinaryOpOutputShapeFn,
so that the implementation of BroadcastBinaryOpShapeFn
coule be reused in SoftmaxCrossEntropyWithLogits.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Update the shape function of SoftmaxCrossEntropyWithLogits

so that broadcast could be supported.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add broadcast support for SoftmaxCrossEntropyWithLogits

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add broadcast support for SoftmaxCrossEntropyWithLogits with GPU

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Reformat with clang-format

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix shape test issues

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Remove `_` for gen_nn_ops._softmax_cross_entropy_with_logits

as `_` is not needed anymore with the recent changes

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Sanitize nn_ops.cc with clang-format

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add broadcast examples for SoftmaxCrossEntropyWithLogits shape function

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add benchmark tests for trival cases

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix pylint issue

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/core/framework/common_shape_fns.cc |  4 +-
 tensorflow/core/framework/common_shape_fns.h  |  8 +-
 tensorflow/core/framework/shape_inference.h   |  1 +
 tensorflow/core/kernels/xent_op.cc            | 65 ++++++++++-----
 tensorflow/core/kernels/xent_op.h             | 35 +++++---
 tensorflow/core/kernels/xent_op_gpu.cu.cc     |  9 ++-
 tensorflow/core/ops/nn_ops.cc                 | 30 ++++---
 tensorflow/core/ops/nn_ops_test.cc            | 16 +++-
 .../python/kernel_tests/xent_op_test.py       | 81 ++++++++++++++++++-
 9 files changed, 197 insertions(+), 52 deletions(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 623248b6ce..2fb17c2b02 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1210,7 +1210,7 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
-Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index) {
   ShapeHandle shape_x = c->input(0);
   ShapeHandle shape_y = c->input(1);
   if (!c->RankKnown(shape_x) || !c->RankKnown(shape_y)) {
@@ -1272,7 +1272,7 @@ Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
     }
   }
 
-  c->set_output(0, c->MakeShape(dims));
+  c->set_output(output_index, c->MakeShape(dims));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 293c40e04d..7230e0f09c 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -265,9 +265,15 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
+// Shape function for binary operators that broadcast their inputs
+// and with output to output_index.
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index);
+
 // Shape function for binary operators that broadcast their inputs.
 // Tested by ops/math_ops_test.cc.
-Status BroadcastBinaryOpShapeFn(InferenceContext* c);
+inline Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+    return BroadcastBinaryOpOutputShapeFn(c, 0);
+}
 
 // Shape function for random operations.
 Status RandomShape(shape_inference::InferenceContext* c);
diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h
index e3cc848a16..accc587000 100644
--- a/tensorflow/core/framework/shape_inference.h
+++ b/tensorflow/core/framework/shape_inference.h
@@ -317,6 +317,7 @@ class InferenceContext {
     input_tensors_as_shapes_ = input_tensors_as_shapes;
   }
 
+  ShapeHandle output(int64 idx) const { return outputs_[idx]; }
   void set_output(int idx, ShapeHandle shape) { outputs_[idx] = shape; }
   Status set_output(StringPiece output_name,
                     const std::vector<ShapeHandle>& shapes);
diff --git a/tensorflow/core/kernels/xent_op.cc b/tensorflow/core/kernels/xent_op.cc
index a6a71fdfaf..ebd19c3d35 100644
--- a/tensorflow/core/kernels/xent_op.cc
+++ b/tensorflow/core/kernels/xent_op.cc
@@ -17,12 +17,14 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
-#include "tensorflow/core/kernels/xent_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/xent_op.h"
+#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
@@ -41,37 +43,56 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& logits_in = context->input(0);
     const Tensor& labels_in = context->input(1);
-    OP_REQUIRES(context, logits_in.IsSameSize(labels_in),
-                errors::InvalidArgument(
-                    "logits and labels must be same size: logits_size=",
-                    logits_in.shape().DebugString(),
-                    " labels_size=", labels_in.shape().DebugString()));
-    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits_in.shape()),
-                errors::InvalidArgument("logits must be 2-dimensional"));
-    // As we already tested that both inputs have the same shape no need to
-    // check that "labels" is a matrix too.
+
+    TensorShape shape_in = logits_in.shape();
+
+    BCast bcast(BCast::FromShape(logits_in.shape()),
+                BCast::FromShape(labels_in.shape()));
+    if (!logits_in.IsSameSize(labels_in)) {
+      OP_REQUIRES(context, bcast.IsValid(),
+                  errors::InvalidArgument(
+                      "logits and labels must be broadcastable: logits_size=",
+                      logits_in.shape().DebugString(), " labels_size=",
+                      labels_in.shape().DebugString()));
+      shape_in = BCast::ToShape(bcast.output_shape());
+    }
+    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
+                errors::InvalidArgument("logits and labels must be beither "
+                                        "2-dimensional, or roadcasted to "
+                                        "2-dimensional"));
 
     // loss is 1-D (one per example), and size is batch_size.
 
     Tensor scratch;
     OP_REQUIRES_OK(
         context, context->allocate_temp(DataTypeToEnum<T>::value,
-                                        TensorShape({logits_in.dim_size(0), 1}),
+                                        TensorShape({shape_in.dim_size(0), 1}),
                                         &scratch));
 
     Tensor* loss_out = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(
-                       0, TensorShape({logits_in.dim_size(0)}), &loss_out));
+                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
     Tensor* back_out = nullptr;
     // Try to reuse the logits_in buffer for the backprop output.
     OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 1, logits_in.shape(), &back_out));
-    if (logits_in.dim_size(0) > 0) {
+                                {0}, 1, shape_in, &back_out));
+    if (shape_in.dim_size(0) > 0) {
       functor::XentFunctor<Device, T> functor;
-      functor(context->eigen_device<Device>(), logits_in.matrix<T>(),
-              labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
-              back_out->matrix<T>());
+      if (logits_in.IsSameSize(labels_in)) {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1},
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1}, logits_in.matrix<T>(),
+                labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
+                back_out->matrix<T>());
+      } else {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                BCast::ToIndexArray<2>(bcast.x_bcast()),
+                BCast::ToIndexArray<2>(bcast.y_bcast()),
+                logits_in.template shaped<T, 2>(bcast.x_reshape()),
+                labels_in.template shaped<T, 2>(bcast.y_reshape()),
+                scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
+      }
     }
   }
 };
@@ -81,13 +102,17 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename Device, typename T>
 struct XentFunctorBase {
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device& d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2>& shape,
+                  const Eigen::array<Eigen::DenseIndex, 2>& logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2>& labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<Device, T>::Compute(d, logits, labels, scratch, loss,
-                                      backprop);
+    XentEigenImpl<Device, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                      logits, labels, scratch, loss, backprop);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op.h b/tensorflow/core/kernels/xent_op.h
index e689fca7ff..87be17fca9 100644
--- a/tensorflow/core/kernels/xent_op.h
+++ b/tensorflow/core/kernels/xent_op.h
@@ -18,6 +18,7 @@ limitations under the License.
 // Functor definition for XentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor_types.h"
 
 namespace tensorflow {
@@ -33,7 +34,11 @@ struct XentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
@@ -45,7 +50,11 @@ struct XentFunctor {
 // specializations for both device types.
 template <typename Device, typename T>
 struct XentEigenImpl {
-  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  static void Compute(const Device &d,
+                      const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                      const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                      const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                      typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<T>::ConstMatrix labels,
                       typename TTypes<T>::Matrix scratch,
                       typename TTypes<T>::Vec loss,
@@ -57,8 +66,8 @@ struct XentEigenImpl {
     const int kBatchDim = 0;
     const int kClassDim = 1;
 
-    const int batch_size = logits.dimension(kBatchDim);
-    const int num_classes = logits.dimension(kClassDim);
+    const int batch_size = shape[kBatchDim];
+    const int num_classes = shape[kClassDim];
 
 // These arrays are used to reduce along the class dimension, and broadcast
 // the resulting value to all classes.
@@ -84,10 +93,12 @@ struct XentEigenImpl {
 #endif
 
     // max_logits along classes.
-    scratch.reshape(batch_only).device(d) = logits.maximum(along_class);
+    scratch.reshape(batch_only).device(d) =
+        logits.broadcast(logits_bcast).maximum(along_class);
 
     // logits - max_logits.
-    backprop.device(d) = logits - scratch.broadcast(one_by_class);
+    backprop.device(d) =
+        logits.broadcast(logits_bcast) - scratch.broadcast(one_by_class);
 
     // sum(exp(logits - max_logits)) along classes.
     scratch.reshape(batch_only).device(d) = backprop.exp().sum(along_class);
@@ -99,15 +110,15 @@ struct XentEigenImpl {
     //  sum(-labels *
     //     ((logits - max_logits) - log(sum(exp(logits - max_logits)))))
     //  along classes
-    loss.device(d) =
-        (labels * (scratch.log().eval().broadcast(one_by_class) - backprop))
-            .eval()
-            .sum(along_class);
+    loss.device(d) = (labels.broadcast(labels_bcast) *
+                      (scratch.log().eval().broadcast(one_by_class) - backprop))
+                         .eval()
+                         .sum(along_class);
 
     // backprop: prob - labels, where
     //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
-    backprop.device(d) =
-        (backprop.exp() / scratch.broadcast(one_by_class)) - labels;
+    backprop.device(d) = (backprop.exp() / scratch.broadcast(one_by_class)) -
+                         labels.broadcast(labels_bcast);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op_gpu.cu.cc b/tensorflow/core/kernels/xent_op_gpu.cu.cc
index 05ee7da490..2c0c0b3a02 100644
--- a/tensorflow/core/kernels/xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/xent_op_gpu.cu.cc
@@ -31,12 +31,17 @@ typedef Eigen::GpuDevice GPUDevice;
 namespace functor {
 template <typename T>
 struct XentFunctor<GPUDevice, T> {
-  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const GPUDevice &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<GPUDevice, T>::Compute(d, logits, labels, scratch, loss,
+    XentEigenImpl<GPUDevice, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                         logits, labels, scratch, loss,
                                          backprop);
   }
 };
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 1f4e9753c3..b9d5104857 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1062,12 +1062,22 @@ REGISTER_OP("SoftmaxCrossEntropyWithLogits")
     .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input));
-      TF_RETURN_IF_ERROR(c->Merge(input, c->input(1), &input));
+      if (c->WithRank(c->input(0), 2, &input) == Status::OK() &&
+          c->Merge(input, c->input(1), &input) == Status::OK()) {
+        DimensionHandle batch_size = c->Dim(input, 0);
+        c->set_output(0, c->Vector(batch_size));
+        c->set_output(1, input);
+        return Status::OK();
+      }
+      TF_RETURN_IF_ERROR(BroadcastBinaryOpOutputShapeFn(c, 1));
 
-      DimensionHandle batch_size = c->Dim(input, 0);
+      if (!c->RankKnown(c->output(1)) || c->Rank(c->output(1)) != 2) {
+        return errors::InvalidArgument(
+            "Shape must be broadcasted with rank 2, but is rank ",
+            c->Rank(c->output(1)));
+      }
+      DimensionHandle batch_size = c->Dim(c->output(1), 0);
       c->set_output(0, c->Vector(batch_size));
-      c->set_output(1, input);
       return Status::OK();
     });
 
@@ -1155,9 +1165,9 @@ Status TopKShapeFn(InferenceContext* c) {
   DimensionHandle last_dim = c->Dim(input, -1);
   if (c->ValueKnown(last_dim) && c->ValueKnown(k_dim) &&
       c->Value(last_dim) < c->Value(k_dim)) {
-    return errors::InvalidArgument(
-        "input must have last dimension >= k = ", c->Value(k_dim), " but is ",
-        c->Value(last_dim));
+    return errors::InvalidArgument("input must have last dimension >= k = ",
+                                   c->Value(k_dim), " but is ",
+                                   c->Value(last_dim));
   }
 
   // Replace last_dim with k_dim.
@@ -1211,9 +1221,9 @@ REGISTER_OP("NthElement")
       DimensionHandle last_dim = c->Dim(input, -1);
       if (c->ValueKnown(last_dim) && c->ValueKnown(n_dim) &&
           c->Value(last_dim) <= c->Value(n_dim)) {
-        return errors::InvalidArgument(
-            "Input must have last dimension > n = ", c->Value(n_dim),
-            " but is ", c->Value(last_dim));
+        return errors::InvalidArgument("Input must have last dimension > n = ",
+                                       c->Value(n_dim), " but is ",
+                                       c->Value(last_dim));
       }
 
       // Reduce last_dim for output tensor
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 1b17a7cda6..289b953055 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -410,10 +410,18 @@ TEST(NNOpsTest, SoftmaxCrossEntropyWithLogits_ShapeFn) {
   INFER_OK(op, "[1,?];[?,2]", "[d0_0];[d0_0,d0_1|d1_1]");
   INFER_OK(op, "[?,2];[1,2]", "[d1_0];in1");
 
-  INFER_ERROR("Dimension 0 in both shapes must be equal, but are 1 and 2", op,
-              "[1,?];[2,?]");
-  INFER_ERROR("Shape must be rank 2 but is rank 3", op, "[1,2,3];?");
-  INFER_ERROR("Shapes must be equal rank, but are 2 and 3", op, "?;[1,2,3]");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "[1,2,3];?");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "?;[1,2,3]");
+
+  // Broadcast example
+  // [1,4] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,4];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [2,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[2,1]", "[d0_0];[d0_0|d1_0,d0_1]");
+  // [1,?] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,?];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [?,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[?,1]", "[d0_0];[d0_0|d1_0,d0_1]");
 }
 
 TEST(NNOpsTest, SparseSoftmaxCrossEntropyWithLogits_ShapeFn) {
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index e3e120a4eb..60c726d54c 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -18,10 +18,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import itertools
+import sys
+
 import numpy as np
 
+from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -88,7 +94,7 @@ class XentTest(test.TestCase):
                                                     4.]]]).astype(dtype)
       np_labels = np.array([[[0., 0., 0., 1.]], [[0., .5, .5,
                                                   0.]]]).astype(dtype)
-      self.assertRaisesRegexp(ValueError, "must be rank 2",
+      self.assertRaisesRegexp(ValueError, "rank 2, but is rank 3",
                               gen_nn_ops.softmax_cross_entropy_with_logits,
                               np_features, np_labels)
 
@@ -128,6 +134,24 @@ class XentTest(test.TestCase):
     self.assertAllClose(
         np.array([1.3862, 1.9401]), np_loss, rtol=1.e-3, atol=1.e-3)
 
+  def testShapeBroadcast(self):
+    np_f = np.array([[1., 2., 3., 4.],
+                     [1., 2., 3., 4.]]).astype(np.float32)
+    np_l = np.array([[0., 0., 0., 1.],
+                     [0., .5, .5, 0.]]).astype(np.float32)
+    np_loss, np_backprop = self._npXent(np_f, np_l)
+    tf_f = constant_op.constant(
+        np.array([[1., 2., 3., 4.]]).astype(np.float32))
+    tf_l = constant_op.constant(
+        np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32))
+    for use_gpu in [False, True]:
+      with self.test_session(use_gpu=use_gpu) as sess:
+        loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
+            tf_f, tf_l)
+        tf_loss, tf_backprop = sess.run([loss, backprop])
+      self.assertAllCloseAccordingToType(np_loss, tf_loss)
+      self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
+
   def testShapeMismatch(self):
     with self.test_session():
       with self.assertRaises(ValueError):
@@ -260,5 +284,60 @@ class XentTest(test.TestCase):
     self.assertAllEqual(np_loss, tf_loss)
 
 
+class XentBenchmark(test.Benchmark):
+
+  def benchmarkZeroDimension(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "zero_dimension_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          logits = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+  def benchmarkSingleClass(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "single_class_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = constant_op.constant([[1.], [-1.], [0.]],
+                                        dtype=dtypes.float32)
+          logits = constant_op.constant([[-1.], [0.], [1.]],
+                                        dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From 31232f29daffe0e496bea22dffeda9e7945d344c Mon Sep 17 00:00:00 2001
From: Nick Desaulniers <ndesaulniers@google.com>
Date: Tue, 27 Mar 2018 12:55:56 -0700
Subject: [PATCH 560/960] [TF:XLA] Force DebugOptions to be specified when
 calling HloModule::CreateModuleConfigFromProto

Otherwise it's easy to forget that you likely want the DebugOptions to be `legacy_flags::GetDebugOptionsFromFlags()`.

PiperOrigin-RevId: 190659046
---
 tensorflow/compiler/xla/client/xla_client/BUILD            | 1 +
 .../compiler/xla/client/xla_client/xla_builder_test.cc     | 4 +++-
 tensorflow/compiler/xla/service/hlo_module.cc              | 3 ++-
 tensorflow/compiler/xla/service/hlo_module.h               | 2 +-
 tensorflow/compiler/xla/service/hlo_runner.cc              | 7 +++----
 5 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/BUILD b/tensorflow/compiler/xla/client/xla_client/BUILD
index cc5f551c9c..60f13e04cb 100644
--- a/tensorflow/compiler/xla/client/xla_client/BUILD
+++ b/tensorflow/compiler/xla/client/xla_client/BUILD
@@ -70,6 +70,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/legacy_flags:debug_options_flags",
         "//tensorflow/compiler/xla/service:hlo",
         "//tensorflow/compiler/xla/service:hlo_matchers",
         "//tensorflow/core:test",
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
index 85d4227ba4..ce984564d0 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder_test.cc
@@ -17,6 +17,7 @@ limitations under the License.
 
 #include <string>
 
+#include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
 #include "tensorflow/compiler/xla/service/hlo_matchers.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -39,7 +40,8 @@ class XlaBuilderTest : public ::testing::Test {
     TF_ASSIGN_OR_RETURN(XlaComputation computation, b->Build());
     const HloModuleProto& proto = computation.proto();
     TF_ASSIGN_OR_RETURN(const auto& config,
-                        HloModule::CreateModuleConfigFromProto(proto));
+                        HloModule::CreateModuleConfigFromProto(
+                            proto, legacy_flags::GetDebugOptionsFromFlags()));
     return HloModule::CreateFromProto(proto, config);
   }
 
diff --git a/tensorflow/compiler/xla/service/hlo_module.cc b/tensorflow/compiler/xla/service/hlo_module.cc
index 595c531ccf..08b9a29aed 100644
--- a/tensorflow/compiler/xla/service/hlo_module.cc
+++ b/tensorflow/compiler/xla/service/hlo_module.cc
@@ -295,12 +295,13 @@ StatusOr<std::unique_ptr<HloModule>> HloModule::CreateFromProto(
 
 /* static */
 StatusOr<HloModuleConfig> HloModule::CreateModuleConfigFromProto(
-    const HloModuleProto& module) {
+    const HloModuleProto& module, const DebugOptions& debug_options) {
   TF_RET_CHECK(module.has_program_shape())
       << "No program shape found in the proto";
   const auto& program_shape = module.program_shape();
 
   HloModuleConfig module_config(program_shape);
+  module_config.set_debug_options(debug_options);
 
   // The module config is constructed with default layouts regardless of what is
   // passed in via the ProgramShape. Set the layouts to the appropriate values.
diff --git a/tensorflow/compiler/xla/service/hlo_module.h b/tensorflow/compiler/xla/service/hlo_module.h
index 755bbd359f..9f7f25202b 100644
--- a/tensorflow/compiler/xla/service/hlo_module.h
+++ b/tensorflow/compiler/xla/service/hlo_module.h
@@ -172,7 +172,7 @@ class HloModule {
   // Creates and returns an HloModuleConfig with an appropriate program shape
   // for the HLO module in the given proto.
   static StatusOr<HloModuleConfig> CreateModuleConfigFromProto(
-      const HloModuleProto& module);
+      const HloModuleProto& module, const DebugOptions& debug_options);
 
   // Outlines the given expression from the given computation.
   // instructions_to_outline contains the instructions that form the expression.
diff --git a/tensorflow/compiler/xla/service/hlo_runner.cc b/tensorflow/compiler/xla/service/hlo_runner.cc
index e5b1c2efa3..ec7d8210a7 100644
--- a/tensorflow/compiler/xla/service/hlo_runner.cc
+++ b/tensorflow/compiler/xla/service/hlo_runner.cc
@@ -52,10 +52,9 @@ namespace {
 // Creates an HloModule from the given proto.
 StatusOr<std::unique_ptr<HloModule>> HloProtoToModule(
     const HloProto& proto, const DebugOptions& debug_options) {
-  TF_ASSIGN_OR_RETURN(
-      HloModuleConfig config,
-      HloModule::CreateModuleConfigFromProto(proto.hlo_module()));
-  config.set_debug_options(debug_options);
+  TF_ASSIGN_OR_RETURN(HloModuleConfig config,
+                      HloModule::CreateModuleConfigFromProto(proto.hlo_module(),
+                                                             debug_options));
   TF_ASSIGN_OR_RETURN(auto module,
                       HloModule::CreateFromProto(proto.hlo_module(), config));
   return std::move(module);
-- 
GitLab


From 502237976694764897e31bad606ec7d02320d5d3 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Wed, 28 Mar 2018 04:19:36 +0800
Subject: [PATCH 561/960]  Fix the math equation format issue in
 tf.matrix_solve_ls (#18022)

* Fix the math equation format issue in tf.matrix_solve_ls

* Fix math equation to avoid break new line within \\( and \\)

* Fix minor indent
---
 .../core/api_def/base_api/api_def_MatrixSolveLs.pbtxt       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
index 51d91399f8..e667c328ae 100644
--- a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
@@ -49,14 +49,14 @@ in the batch:
 If `fast` is `True`, then the solution is computed by solving the normal
 equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
 \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-\lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). 
+If \\(m \lt n\\) then `output` is computed as
 \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
 minimum-norm solution to the under-determined linear system, i.e.
 \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
 subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
 when \\(A\\) is numerically full rank and has a condition number
-\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is
 sufficiently large.
 
 If `fast` is `False` an algorithm based on the numerically robust complete
-- 
GitLab


From 40d12e552f67600ddfd3f2937092c0fda157b8a8 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 27 Mar 2018 14:06:44 -0700
Subject: [PATCH 562/960] Fast path for calling pack when the list is full of
 eager tensors.

FastPathExecute function also allows inputs to be sequences instead of just lists.

PiperOrigin-RevId: 190670587
---
 tensorflow/python/eager/pywrap_tfe_src.cc | 36 ++++++++++++-----------
 tensorflow/python/ops/array_ops.py        |  5 ++++
 2 files changed, 24 insertions(+), 17 deletions(-)

diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 55ba509065..30ef6781ec 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1405,10 +1405,10 @@ bool CheckInputsOk(PyObject* seq, int start_index,
     PyObject* item = PyTuple_GET_ITEM(seq, i + start_index);
     if (!op_def.input_arg(i).number_attr().empty() ||
         !op_def.input_arg(i).type_list_attr().empty()) {
-      // This item should be a list input.
-      if (!PyList_Check(item)) return false;
-      for (Py_ssize_t j = 0; j < PyList_Size(item); j++) {
-        PyObject* inner_item = PyList_GET_ITEM(item, j);
+      // This item should be a seq input.
+      if (!PySequence_Check(item)) return false;
+      for (Py_ssize_t j = 0; j < PySequence_Fast_GET_SIZE(item); j++) {
+        PyObject* inner_item = PySequence_Fast_GET_ITEM(item, j);
         if (!EagerTensor_CheckExact(inner_item) &&
             !CheckResourceVariable(inner_item)) {
           return false;
@@ -1726,11 +1726,11 @@ const char* GetDeviceName(PyObject* py_device_name) {
   return nullptr;
 }
 
-bool RaiseIfNotPyList(PyObject* list, const string& attr_name) {
-  if (!PyList_Check(list)) {
+bool RaiseIfNotPySequence(PyObject* seq, const string& attr_name) {
+  if (!PySequence_Check(seq)) {
     PyErr_SetString(PyExc_TypeError,
-                    Printf("expected a list for attr %s, got %s instead",
-                           attr_name.data(), list->ob_type->tp_name)
+                    Printf("expected a sequence for attr %s, got %s instead",
+                           attr_name.data(), seq->ob_type->tp_name)
                         .data());
 
     return false;
@@ -1940,8 +1940,8 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
         PyTuple_GET_ITEM(args, kFastPathExecuteInputStartIndex + i);
     if (!input_arg.number_attr().empty()) {
       // The item is a homogeneous list.
-      if (!RaiseIfNotPyList(input, input_arg.number_attr())) return nullptr;
-      Py_ssize_t len = PyList_Size(input);
+      if (!RaiseIfNotPySequence(input, input_arg.number_attr())) return nullptr;
+      Py_ssize_t len = PySequence_Fast_GET_SIZE(input);
 
       TFE_OpSetAttrInt(op, input_arg.number_attr().data(), len);
       if (op_exec_info.run_callbacks) {
@@ -1953,15 +1953,15 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
 
       if (len > 0) {
         // First item adds the type attr.
-        if (!AddInputToOp(op_exec_info, PyList_GET_ITEM(input, 0), &input_arg,
-                          flattened_attrs.get(), flattened_inputs.get(), op,
-                          status)) {
+        if (!AddInputToOp(op_exec_info, PySequence_Fast_GET_ITEM(input, 0),
+                          &input_arg, flattened_attrs.get(),
+                          flattened_inputs.get(), op, status)) {
           return nullptr;
         }
 
         for (Py_ssize_t j = 1; j < len; j++) {
           // Since the list is homogeneous, we don't need to re-add the attr.
-          if (!AddInputToOp(op_exec_info, PyList_GET_ITEM(input, j),
+          if (!AddInputToOp(op_exec_info, PySequence_Fast_GET_ITEM(input, j),
                             nullptr /* input_arg */,
                             nullptr /* flattened_attrs */,
                             flattened_inputs.get(), op, status)) {
@@ -1971,16 +1971,18 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
       }
     } else if (!input_arg.type_list_attr().empty()) {
       // The item is a heterogeneous list.
-      if (!RaiseIfNotPyList(input, input_arg.type_list_attr())) return nullptr;
+      if (!RaiseIfNotPySequence(input, input_arg.type_list_attr())) {
+        return nullptr;
+      }
       const string& attr_name = input_arg.type_list_attr();
-      Py_ssize_t len = PyList_Size(input);
+      Py_ssize_t len = PySequence_Fast_GET_SIZE(input);
       tensorflow::gtl::InlinedVector<TF_DataType, 4> attr_value(len);
       PyObject* py_attr_value = nullptr;
       if (op_exec_info.run_callbacks) {
         py_attr_value = PyTuple_New(len);
       }
       for (Py_ssize_t j = 0; j < len; j++) {
-        PyObject* py_input = PyList_GET_ITEM(input, j);
+        PyObject* py_input = PySequence_Fast_GET_ITEM(input, j);
         tensorflow::Safe_PyObjectPtr py_eager_tensor;
         if (!ConvertToTensor(op_exec_info, py_input, &py_eager_tensor,
                              status)) {
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 9106461c60..207866610b 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -957,6 +957,11 @@ def _autopacking_helper(list_or_tuple, dtype, name):
   Returns:
     A `tf.Tensor` with value equivalent to `list_or_tuple`.
   """
+  if context.executing_eagerly():
+    # NOTE: Fast path when all the items are tensors, this doesn't do any type
+    # checking.
+    if all(ops.is_dense_tensor_like(elem) for elem in list_or_tuple):
+      return gen_array_ops.pack(list_or_tuple, name=name)
   must_pack = False
   converted_elems = []
   with ops.name_scope(name) as scope:
-- 
GitLab


From cc4103dacacc10edb93224c276f68f172922734a Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Tue, 27 Mar 2018 14:12:00 -0700
Subject: [PATCH 563/960] Fix documentation of Clamp; it does not take a
 computation at all.

See:
https://github.com/tensorflow/tensorflow/blob/r1.6/tensorflow/compiler/xla/client/computation_builder.h#L668
PiperOrigin-RevId: 190671530
---
 tensorflow/docs_src/performance/xla/operation_semantics.md | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 4d12c7ab6d..32f249cf10 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -241,13 +241,10 @@ See also
 
 Clamps an operand to within the range between a minimum and maximum value.
 
-<b> `Clamp(computation, min, operand, max)` </b>
+<b> `Clamp(min, operand, max)` </b>
 
 | Arguments     | Type                    | Semantics                        |
 | ------------- | ----------------------- | -------------------------------- |
-| `computation` | `Computation`           | computation of type `T_0, T_1,   |
-:               :                         : ..., T_N -> S` with N parameters :
-:               :                         : of arbitrary type                :
 | `min`         | `ComputationDataHandle` | array of type T                  |
 | `operand`     | `ComputationDataHandle` | array of type T                  |
 | `max`         | `ComputationDataHandle` | array of type T                  |
-- 
GitLab


From 12accfe7d26617797f6f9d63352e09697c6d9455 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Tue, 27 Mar 2018 14:14:01 -0700
Subject: [PATCH 564/960] Add "serve" as a default value for savedmodel_tagset.

PiperOrigin-RevId: 190671867
---
 tensorflow/contrib/lite/toco/BUILD               | 1 +
 tensorflow/contrib/lite/toco/args.h              | 3 ++-
 tensorflow/contrib/lite/toco/toco_saved_model.cc | 9 ++++++---
 3 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 102740ee47..051fa8de3c 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -143,6 +143,7 @@ cc_library(
         ":toco_graphviz_dump_options",
         ":toco_port",
         ":types_proto_cc",
+        "//tensorflow/cc/saved_model:tag_constants",
         "//tensorflow/core:framework_internal",
         "//tensorflow/core:lib",
         "@com_google_absl//absl/strings",
diff --git a/tensorflow/contrib/lite/toco/args.h b/tensorflow/contrib/lite/toco/args.h
index 7b71792ff7..52c789293c 100644
--- a/tensorflow/contrib/lite/toco/args.h
+++ b/tensorflow/contrib/lite/toco/args.h
@@ -26,6 +26,7 @@ limitations under the License.
 #endif
 #include "absl/strings/numbers.h"
 #include "absl/strings/str_split.h"
+#include "tensorflow/cc/saved_model/tag_constants.h"
 #include "tensorflow/contrib/lite/toco/toco_port.h"
 #include "tensorflow/contrib/lite/toco/toco_types.h"
 
@@ -220,7 +221,7 @@ struct ParsedTocoFlags {
   Arg<string> output_file;
   Arg<string> input_format = Arg<string>("TENSORFLOW_GRAPHDEF");
   Arg<string> output_format = Arg<string>("TFLITE");
-  Arg<string> savedmodel_tagset;
+  Arg<string> savedmodel_tagset = Arg<string>(tensorflow::kSavedModelTagServe);
   // TODO(aselle): command_line_flags  doesn't support doubles
   Arg<float> default_ranges_min = Arg<float>(0.);
   Arg<float> default_ranges_max = Arg<float>(0.);
diff --git a/tensorflow/contrib/lite/toco/toco_saved_model.cc b/tensorflow/contrib/lite/toco/toco_saved_model.cc
index 91a742b9e0..26f55a66c7 100644
--- a/tensorflow/contrib/lite/toco/toco_saved_model.cc
+++ b/tensorflow/contrib/lite/toco/toco_saved_model.cc
@@ -35,9 +35,12 @@ const tensorflow::SavedModelBundle* LoadSavedModel(
       << "Model is not saved in the supported SavedModel format.\n";
 
   // Gets the tags identifying the MetaGraphDef from the command line arguments.
-  QCHECK(parsed_toco_flags.savedmodel_tagset.specified())
-      << "Missing required flag --savedmodel_tagset.\n";
-  const string tags_str = parsed_toco_flags.savedmodel_tagset.value();
+  string tags_str;
+  if (parsed_toco_flags.savedmodel_tagset.specified()) {
+    tags_str = parsed_toco_flags.savedmodel_tagset.value();
+  } else {
+    tags_str = parsed_toco_flags.savedmodel_tagset.default_value();
+  }
   auto tags = absl::StrSplit(tags_str, ',');
 
   // Loads MetaGraphDef.
-- 
GitLab


From 95f2304c1faf24e9ae2e15cc6dc4b6f7114d1dad Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Tue, 27 Mar 2018 14:17:16 -0700
Subject: [PATCH 565/960] Improve error message when users forget to pass toco
 cmdline args for quantization, but have a model that has FAKE_QUANT
 operations.

PiperOrigin-RevId: 190672414
---
 tensorflow/contrib/lite/toco/tflite/export.cc | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/tflite/export.cc b/tensorflow/contrib/lite/toco/tflite/export.cc
index 2771959970..335b496dcc 100644
--- a/tensorflow/contrib/lite/toco/tflite/export.cc
+++ b/tensorflow/contrib/lite/toco/tflite/export.cc
@@ -300,6 +300,17 @@ void Export(const Model& model, bool allow_custom_ops,
   std::set<string> error_summary;
   auto op_codes = ExportOperatorCodes(model, ops_by_type, operators_map,
                                       &builder, &error_summary);
+  const string fake_quant_operation_name = "FAKE_QUANT";
+  if (error_summary.count(fake_quant_operation_name) != 0) {
+    LOG(ERROR)
+        << fake_quant_operation_name
+        << " operation was not converted. If running quantized make sure you "
+           "are passing --inference_type=QUANTIZED_UINT8 and values for "
+           "--std_values and --mean_values.";
+    // Remove the fake quant operation from the errors, since it shouldn't
+    // be provided a custom implementation.
+    error_summary.erase(fake_quant_operation_name);
+  }
   if (!allow_custom_ops && !error_summary.empty()) {
     LOG(QFATAL) << "Some of the operators in the model are not supported by "
                    "the standard TensorFlow Lite runtime. If you have a custom "
-- 
GitLab


From 6bb4f7abb03a7904fecc5b61e3ed37e9b663d6b0 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 27 Mar 2018 14:23:28 -0700
Subject: [PATCH 566/960] [tf.data] Raise error when window size is 0 in
 `tf.contrib.data.group_by_window()`.

PiperOrigin-RevId: 190673466
---
 .../data/python/kernel_tests/bucketing_test.py    | 15 +++++++++++++++
 .../kernels/data/group_by_window_dataset_op.cc    |  5 +++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
index d0131896a1..6002cc73c8 100644
--- a/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/bucketing_test.py
@@ -104,6 +104,21 @@ class GroupByWindowTest(test.TestCase):
       self.assertAllEqual([0, 0, 0], sess.run(get_next))
       self.assertAllEqual([1], sess.run(get_next))
 
+  def testEmpty(self):
+    iterator = (
+        dataset_ops.Dataset.range(4).apply(
+            grouping.group_by_window(lambda _: 0, lambda _, xs: xs, 0))
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(init_op)
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "Window size must be greater than zero, but got 0."):
+        print(sess.run(get_next))
+
   def testReduceFuncError(self):
     components = np.random.randint(100, size=(200,)).astype(np.int64)
 
diff --git a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
index 834c06bb93..46f43dd1b1 100644
--- a/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
+++ b/tensorflow/core/kernels/data/group_by_window_dataset_op.cc
@@ -263,6 +263,11 @@ class GroupByWindowDatasetOp : public UnaryDatasetOpKernel {
                 }
                 const int64 window_size =
                     window_size_func_output[0].scalar<int64>()();
+                if (window_size <= 0) {
+                  return errors::InvalidArgument(
+                      "Window size must be greater than zero, but got ",
+                      window_size, ".");
+                }
                 window_sizes_[key] = window_size;
               }
 
-- 
GitLab


From 11d895b4bc2f1820767a5bae65ed2ca251d03017 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@gmail.com>
Date: Tue, 27 Mar 2018 14:54:40 -0700
Subject: [PATCH 567/960] Revert "Fix minor typos in contrib files" (#18029)

* Revert " Fix the math equation format issue in tf.matrix_solve_ls (#18022)"

This reverts commit 502237976694764897e31bad606ec7d02320d5d3.

* Revert "Add broadcast support for softmax_cross_entropy_with_logits (#16784)"

This reverts commit 0ef36a5de45486ccbc0d6237f86280c2ac22f52e.

* Revert "Fix minor spelling typos in contrib (#18015)"

This reverts commit 5d76c7db2ab72f9b0cc70ce12ba0a3395dcc20d3.
---
 .../contrib/bayesflow/python/ops/metropolis_hastings_impl.py    | 2 +-
 .../contrib/estimator/python/estimator/replicate_model_fn.py    | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
index fdee0a8da6..05aa134ed5 100644
--- a/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
+++ b/tensorflow/contrib/bayesflow/python/ops/metropolis_hastings_impl.py
@@ -238,7 +238,7 @@ def evolve(initial_sample,
   using the Metropolis-Hastings algorithm. These samples are from a Markov chain
   whose equilibrium distribution matches the target distribution.
 
-  The probability distribution may have an unknown normalization constant.
+  The probability distribution may have an unknown normalization constan.
   We parameterize the probability density as follows:
 
   ```none
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index fa2697800e..e0fae2c992 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replicate only on the subset of available GPUs.
+      argument can be used to replice only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
-- 
GitLab


From 5baa43999877aeb7ec070374e579b291b1efe0cf Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Tue, 27 Mar 2018 15:01:51 -0700
Subject: [PATCH 568/960] Fix issue caused by None in batch dimension for
 tf.layers.conv3d (#18027)

* Fix issue caused by None in batch dimension for tf.layers.conv3d

This fix tries to address the issue raised in 15655 where
error returns when the batch dimension for tf.layers.conv3d
is None with "channels_first" format.

This fix cast None to `-1` to address the issue

This fix fixes 15655.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for batch dimension = None in tf.layers.conv3d

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Address review feedback

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/python/layers/convolutional.py      | 2 ++
 tensorflow/python/layers/convolutional_test.py | 6 ++++++
 2 files changed, 8 insertions(+)

diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 74e7c63fb3..2d99b1688f 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -180,6 +180,8 @@ class _Conv(base.Layer):
           # bias_add when computing gradients. To use bias_add, we collapse Z
           # and Y into a single dimension to obtain a 4D input tensor.
           outputs_shape = outputs.shape.as_list()
+          if outputs_shape[0] is None:
+            outputs_shape[0] = -1
           outputs_4d = array_ops.reshape(outputs,
                                          [outputs_shape[0], outputs_shape[1],
                                           outputs_shape[2] * outputs_shape[3],
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index 160e732b67..cdb42f5bd1 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -325,6 +325,12 @@ class ConvTest(test.TestCase):
     self.assertEqual(conv3d.kernel_constraint, k_constraint)
     self.assertEqual(conv3d.bias_constraint, b_constraint)
 
+  def testConv3DChannelsFirst(self):
+    # Test case for GitHub issue 15655
+    images = array_ops.placeholder(
+        dtype=dtypes.float32, shape=[None, 1, 32, 32, 32])
+    conv_layers.conv3d(images, 32, 9, data_format='channels_first')
+
 
 @test_util.with_c_api
 class SeparableConv1DTest(test.TestCase):
-- 
GitLab


From 8eab17db63deb80c7c21b92d5c92dc48e852de96 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Wed, 28 Mar 2018 06:03:14 +0800
Subject: [PATCH 569/960] Fix the contrib losses function bullet lists
 rendering (#18021)

---
 .../api_guides/python/contrib.losses.md       | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/docs_src/api_guides/python/contrib.losses.md b/tensorflow/docs_src/api_guides/python/contrib.losses.md
index d7f862625e..8b7442216c 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.losses.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.losses.md
@@ -107,19 +107,19 @@ weighted average over the individual prediction errors:
   loss = tf.contrib.losses.mean_squared_error(predictions, depths, weight)
 ```
 
-@{tf.contrib.losses.absolute_difference}
-@{tf.contrib.losses.add_loss}
-@{tf.contrib.losses.hinge_loss}
-@{tf.contrib.losses.compute_weighted_loss}
-@{tf.contrib.losses.cosine_distance}
-@{tf.contrib.losses.get_losses}
-@{tf.contrib.losses.get_regularization_losses}
-@{tf.contrib.losses.get_total_loss}
-@{tf.contrib.losses.log_loss}
-@{tf.contrib.losses.mean_pairwise_squared_error}
-@{tf.contrib.losses.mean_squared_error}
-@{tf.contrib.losses.sigmoid_cross_entropy}
-@{tf.contrib.losses.softmax_cross_entropy}
-@{tf.contrib.losses.sparse_softmax_cross_entropy}
+* @{tf.contrib.losses.absolute_difference}
+* @{tf.contrib.losses.add_loss}
+* @{tf.contrib.losses.hinge_loss}
+* @{tf.contrib.losses.compute_weighted_loss}
+* @{tf.contrib.losses.cosine_distance}
+* @{tf.contrib.losses.get_losses}
+* @{tf.contrib.losses.get_regularization_losses}
+* @{tf.contrib.losses.get_total_loss}
+* @{tf.contrib.losses.log_loss}
+* @{tf.contrib.losses.mean_pairwise_squared_error}
+* @{tf.contrib.losses.mean_squared_error}
+* @{tf.contrib.losses.sigmoid_cross_entropy}
+* @{tf.contrib.losses.softmax_cross_entropy}
+* @{tf.contrib.losses.sparse_softmax_cross_entropy}
 
 
-- 
GitLab


From 736e055a756cf0f99d59b67284aade01baec9799 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 27 Mar 2018 15:07:05 -0700
Subject: [PATCH 570/960] Make _USE_C_API = True and _USE_C_SHAPES = False work
 with handle data, take 2.

This change makes _set_shapes_for_outputs_c_api fetch and set
Tensor._handle_data. This is necessary for running the
Python shape inference code on resource tensors.

PiperOrigin-RevId: 190681459
---
 tensorflow/c/BUILD                           |  2 ++
 tensorflow/c/python_api.cc                   | 26 +++++++++++++++
 tensorflow/c/python_api.h                    |  7 ++++
 tensorflow/python/BUILD                      |  2 ++
 tensorflow/python/client/tf_session.i        |  1 +
 tensorflow/python/framework/importer_test.py | 34 ++++++++++++++++++++
 tensorflow/python/framework/ops.py           | 10 ++++++
 7 files changed, 82 insertions(+)

diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 7f03e40d38..249135f728 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -283,6 +283,8 @@ tf_cuda_library(
     deps = [
         ":c_api",
         ":c_api_internal",
+        # TODO(b/74620627): remove when _USE_C_SHAPES is removed
+        "//tensorflow/python:cpp_shape_inference_proto_cc",
     ],
 )
 
diff --git a/tensorflow/c/python_api.cc b/tensorflow/c/python_api.cc
index cd604538f1..93155998b8 100644
--- a/tensorflow/c/python_api.cc
+++ b/tensorflow/c/python_api.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/c/python_api.h"
 
 #include "tensorflow/c/c_api_internal.h"
+#include "tensorflow/python/framework/cpp_shape_inference.pb.h"
 
 namespace tensorflow {
 
@@ -109,4 +110,29 @@ void ExtendSession(TF_Session* session, TF_Status* status) {
   session->extend_before_run = false;
 }
 
+std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output) {
+  Node* node = &output.oper->node;
+  CppShapeInferenceResult::HandleData handle_data;
+  handle_data.set_is_set(true);
+  {
+    mutex_lock l(graph->mu);
+    tensorflow::shape_inference::InferenceContext* ic =
+        graph->refiner.GetContext(node);
+    CHECK(ic != nullptr);
+    CHECK_LT(output.index, ic->num_outputs());
+    const auto* shapes_and_types =
+        ic->output_handle_shapes_and_types(output.index);
+    if (shapes_and_types == nullptr) return "";
+
+    for (const auto& p : *shapes_and_types) {
+      auto* out_shape_and_type = handle_data.add_shape_and_type();
+      ic->ShapeHandleToProto(p.shape, out_shape_and_type->mutable_shape());
+      out_shape_and_type->set_dtype(p.dtype);
+    }
+  }
+  string result;
+  handle_data.SerializeToString(&result);
+  return result;
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/c/python_api.h b/tensorflow/c/python_api.h
index 13b680b3a2..2d4c8cd9ed 100644
--- a/tensorflow/c/python_api.h
+++ b/tensorflow/c/python_api.h
@@ -16,6 +16,8 @@ limitations under the License.
 #ifndef TENSORFLOW_C_PYTHON_API_H_
 #define TENSORFLOW_C_PYTHON_API_H_
 
+#include <string>
+
 #include "tensorflow/c/c_api.h"
 
 // These functions can be removed without notice. They exist to facilitate some
@@ -51,6 +53,11 @@ void SetRequireShapeInferenceFns(TF_Graph* graph, bool require);
 // the graph after the session has been made aware of them.
 void ExtendSession(TF_Session* session, TF_Status* status);
 
+// Returns the serialized CppShapeInferenceResult::HandleData proto for
+// `output` if its a resource tensor, or otherwise returns the empty string.
+// TODO(b/74620627): remove when _USE_C_SHAPES is removed
+std::string ResourceHandleShapeAndType(TF_Graph* graph, TF_Output output);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_C_PYTHON_API_H_
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 20d7e81045..4f61c01f65 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3144,6 +3144,8 @@ tf_proto_library(
     srcs = ["framework/cpp_shape_inference.proto"],
     cc_api_version = 2,
     protodeps = tf_additional_all_protos(),
+    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
+    visibility = ["//tensorflow:internal"],
 )
 
 py_test(
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index e88fc0c01a..70a3d032f4 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -723,6 +723,7 @@ def TF_Reset(target, containers=None, config=None):
 %unignore TF_TryEvaluateConstant_wrapper;
 %noexception TF_TryEvaluateConstant_wrapper;
 %unignore ExtendSession;
+%unignore ResourceHandleShapeAndType;
 
 %include "tensorflow/python/client/tf_session_helper.h"
 
diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 6593b17184..369669c2e6 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -39,6 +39,7 @@ from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import random_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variables
 import tensorflow.python.ops.nn_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
@@ -356,6 +357,39 @@ class ImportGraphDefTest(test.TestCase):
       self.assertEqual(d._input_types, [dtypes.int32_ref, dtypes.int32])
       self.assertEqual(d.outputs, [])
 
+  def testResources(self):
+    # Produce GraphDef containing a ops producing and consuming resources.
+    graph = ops.Graph()
+    with graph.as_default():
+      var = resource_variable_ops.ResourceVariable(1.0)
+      var_assign = var.assign(2.0)
+      # Use an op that requires handle shape to be set.
+      var_shape = resource_variable_ops.variable_shape(var.handle)
+      init = variables.global_variables_initializer()
+    graph_def = graph.as_graph_def()
+
+    # Import the GraphDef.
+    with ops.Graph().as_default():
+      # pylint: disable=unused-variable
+      imported_var, imported_assign, imported_shape, imported_init = (
+          importer.import_graph_def(
+              graph_def,
+              return_elements=[var.name, var_assign.name, var_shape.name,
+                               init.name]))
+
+      # Make sure the handle shape is set on the imported variable.
+      new_var_shape = resource_variable_ops.variable_shape(imported_var)
+      # pylint: enable=unused-variable
+
+      # Run the imported graph.
+      # TODO(b/76173421): make this work (currently DCHECKS)
+      # with self.test_session() as sess:
+      #   sess.run(imported_init)
+      #   self.assertEqual(sess.run(imported_var), 1.0)
+      #   self.assertEqual(sess.run(imported_assign), 2.0)
+      #   self.assertEqual(list(sess.run(imported_shape)), [])
+      #   self.assertEqual(list(sess.run(new_var_shape)), [])
+
   def testWhileLoop(self):
     # Produce GraphDef containing while loop.
     graph = ops.Graph()
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 25a951a2de..4b0f3f3a20 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -42,6 +42,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.eager import core
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import c_api_util
+from tensorflow.python.framework import cpp_shape_inference_pb2
 from tensorflow.python.framework import device as pydev
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -295,6 +296,7 @@ class Tensor(_TensorLike):
 
     # Attributes used for C++ shape inference. Not inspected, only forwarded.
     # If set, will be a HandleData object from cpp_shape_inference.proto.
+    # TODO(b/74620627): remove when _USE_C_SHAPES is removed
     self._handle_data = None
     self._id = uid()
 
@@ -2472,6 +2474,14 @@ def _set_shapes_for_outputs_c_api(op):
       shape_vector = [None if d == -1 else d for d in shape_vector]
       output.set_shape(tensor_shape.TensorShape(shape_vector))
 
+    serialized = c_api.ResourceHandleShapeAndType(op._graph._c_graph,
+                                                  output._as_tf_output())
+    if serialized:
+      output._handle_data = (
+          cpp_shape_inference_pb2.CppShapeInferenceResult.HandleData.FromString(
+              compat.as_bytes(serialized)))
+    else:
+      output._handle_data = None
 
 # TODO(skyewm): remove this when _USE_C_API flag is removed.
 def _set_shapes_for_outputs(op):
-- 
GitLab


From 016a79b371061e060ad85b0dbac4aa27d07f0239 Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@gmail.com>
Date: Tue, 27 Mar 2018 15:09:19 -0700
Subject: [PATCH 571/960] Refix spelling error reverted in #18015 (#18031)

---
 .../contrib/estimator/python/estimator/replicate_model_fn.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index e0fae2c992..fa2697800e 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replice only on the subset of available GPUs.
+      argument can be used to replicate only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
-- 
GitLab


From 9a0b91023d8444cd4691be10b36ce469ca08058d Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 27 Mar 2018 15:08:12 -0700
Subject: [PATCH 572/960] Moves Execute() from c_api.cc

PiperOrigin-RevId: 190681610
---
 tensorflow/c/eager/BUILD                      |   1 +
 tensorflow/c/eager/c_api.cc                   |  90 ++------------
 tensorflow/c/eager/c_api_test.cc              |   4 +-
 tensorflow/core/common_runtime/eager/BUILD    |  15 +++
 .../core/common_runtime/eager/execute.cc      | 110 ++++++++++++++++++
 .../core/common_runtime/eager/execute.h       |  37 ++++++
 6 files changed, 172 insertions(+), 85 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/execute.cc
 create mode 100644 tensorflow/core/common_runtime/eager/execute.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index 8df7b56623..e57011a08b 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -30,6 +30,7 @@ tf_cuda_library(
             "//tensorflow/core:core_cpu",
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
+            "//tensorflow/core/common_runtime/eager:execute",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core/common_runtime/eager:tensor_handle",
             "//tensorflow/core/common_runtime/eager:copy_to_device_node",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index eaeb2fd07a..ac7114f71e 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -33,6 +33,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_mgr.h"
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/eager/copy_to_device_node.h"
+#include "tensorflow/core/common_runtime/eager/execute.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -574,83 +575,6 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
   return nullptr;
 }
 
-tensorflow::Status Execute(
-    TFE_Context* ctx, tensorflow::Device* device,
-    const tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4>&
-        op_inputs,
-    tensorflow::KernelAndDevice* kernel, tensorflow::NodeExecStats* maybe_stats,
-    tensorflow::TensorHandle** retvals, int num_retvals) {
-  if (!ctx->context.SoftPlacement() && device == nullptr) {
-    device = ctx->context.HostCPU();
-  }
-
-  if (device == nullptr) {
-    // TODO(apassos) debug how the assignment below might return a different
-    // device from the one requested above.
-    device = kernel->device();
-  }
-
-  std::vector<tensorflow::Tensor> outputs(1);
-  const tensorflow::MemoryTypeVector* output_memory_types = nullptr;
-  output_memory_types = &kernel->kernel()->output_memory_types();
-  std::vector<tensorflow::Tensor> inputs(op_inputs.size());
-  for (int i = 0; i < op_inputs.size(); ++i) {
-    const tensorflow::Tensor* input_tensor = nullptr;
-    TF_RETURN_IF_ERROR(op_inputs[i]->Tensor(&input_tensor));
-    inputs[i] = *input_tensor;
-  }
-  // WARNING: kernel->Run utilizes the FunctionLibraryRuntime
-  // (ctx->func_lib(device)), which in turn holds a pointer to func_lib_def.
-  // But knowledge of the implementation
-  // of FunctionLibraryRuntime tells us that func_lib_def is not accessed by
-  // FunctionLibraryRuntime::Run(), so there is no thread-safety concern here.
-  // This is quite subtle. Re-work things to make this better?  (Would it make
-  // sense for FunctionLibraryRuntime to ensure thread-safe access to
-  // FunctionLibraryDefinition?).  TODO(apassos) figure out how to record stats
-  // for ops which are a part of functions.
-  // TODO(agarwal): change Run to take vector of handles ?
-  TF_RETURN_IF_ERROR(kernel->Run(&inputs, &outputs, maybe_stats));
-  if (maybe_stats != nullptr) {
-    maybe_stats->set_op_end_rel_micros(tensorflow::Env::Default()->NowMicros() -
-                                       maybe_stats->all_start_micros());
-    tensorflow::mutex_lock ml(*ctx->context.MetadataMu());
-    if (ctx->context.ShouldStoreMetadata()) {
-      auto* step_stats = ctx->context.RunMetadataProto()->mutable_step_stats();
-      // Lazily initialize the RunMetadata with information about all devices if
-      // this is the first call.
-      while (step_stats->dev_stats_size() < ctx->context.devices()->size()) {
-        step_stats->add_dev_stats();
-      }
-      // Find the current device's index.
-      int device_idx = 0;
-      for (int i = 0; i < ctx->context.devices()->size(); ++i) {
-        if (ctx->context.devices()->at(i) == device) {
-          device_idx = i;
-          break;
-        }
-      }
-      // Populate the device stats for this device.
-      auto* dev_stats = step_stats->mutable_dev_stats(device_idx);
-      dev_stats->set_device(device->name());
-      *dev_stats->add_node_stats() = *maybe_stats;
-    }
-  }
-  DCHECK_EQ(num_retvals, outputs.size());
-  tensorflow::Device* op_device = IsCPU(device) ? nullptr : device;
-  for (int i = 0; i < num_retvals; ++i) {
-    tensorflow::Device* d = op_device;
-    if (d != nullptr && output_memory_types != nullptr &&
-        (*output_memory_types)[i] == tensorflow::HOST_MEMORY) {
-      d = nullptr;
-    }
-    if (retvals[i] == nullptr) {
-      retvals[i] = new tensorflow::TensorHandle(outputs[i], d, op_device);
-    } else {
-      retvals[i]->SetTensorAndDevice(outputs[i], d, op_device);
-    }
-  }
-  return tensorflow::Status::OK();
-}
 
 // TODO(agarwal): move EagerExecutor and EagerNode related code to a separate
 // file.
@@ -690,9 +614,9 @@ class ExecuteNode : public tensorflow::EagerNode {
   }
 
   tensorflow::Status Run() override {
-    const tensorflow::Status status =
-        Execute(ctx_, op_device_, inputs_, kernel_, maybe_stats_.get(),
-                retvals_.begin(), retvals_.size());
+    const tensorflow::Status status = tensorflow::EagerExecute(
+        &ctx_->context, op_device_, inputs_, kernel_, maybe_stats_.get(),
+        retvals_.begin(), retvals_.size());
     if (status.ok()) {
       return status;
     } else {
@@ -1062,9 +986,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // allocate it.
     std::vector<tensorflow::TensorHandle*> handle_retvals(*num_retvals,
                                                           nullptr);
-    status->status =
-        Execute(op->ctx, op->device, op->inputs, kernel, maybe_stats.get(),
-                handle_retvals.data(), *num_retvals);
+    status->status = tensorflow::EagerExecute(
+        &op->ctx->context, op->device, op->inputs, kernel, maybe_stats.get(),
+        handle_retvals.data(), *num_retvals);
     for (int i = 0; i < *num_retvals; ++i) {
       retvals[i] = new TFE_TensorHandle(handle_retvals[i]);
     }
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index 2268aba90d..d88a6c1dda 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -688,12 +688,12 @@ TEST(CAPI, Execute_Min_CPU) {
   TFE_DeleteOp(minOp);
   TFE_DeleteTensorHandle(input);
   TFE_DeleteTensorHandle(axis);
-  TFE_DeleteContext(ctx, status);
-  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   ASSERT_EQ(1, num_retvals);
 
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteTensorHandle(retvals[0]);
+  TFE_DeleteContext(ctx, status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   float output[2] = {0};
   EXPECT_EQ(sizeof(output), TF_TensorByteSize(t));
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index a619cac9a4..56e2e1094f 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -149,3 +149,18 @@ filegroup(
     ),
     visibility = ["//tensorflow:__subpackages__"],
 )
+
+cc_library(
+    name = "execute",
+    srcs = ["execute.cc"],
+    hdrs = ["execute.h"],
+    deps = [
+        ":context",
+        ":kernel_and_device",
+        ":tensor_handle",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
new file mode 100644
index 0000000000..9e6dddaa02
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -0,0 +1,110 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/common_runtime/eager/execute.h"
+
+#include <vector>
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+
+Status EagerExecute(EagerContext* ctx, Device* device,
+                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                    TensorHandle** retvals, int num_retvals) {
+  if (!ctx->SoftPlacement() && device == nullptr) {
+    device = ctx->HostCPU();
+  }
+
+  if (device == nullptr) {
+    // TODO(apassos) debug how the assignment below might return a different
+    // device from the one requested above.
+    device = kernel->device();
+  }
+
+  std::vector<Tensor> outputs(1);
+  const MemoryTypeVector* output_memory_types = nullptr;
+  output_memory_types = &kernel->kernel()->output_memory_types();
+  std::vector<Tensor> inputs(op_inputs.size());
+  for (int i = 0; i < op_inputs.size(); ++i) {
+    const Tensor* input_tensor = nullptr;
+    TF_RETURN_IF_ERROR(op_inputs[i]->Tensor(&input_tensor));
+    inputs[i] = *input_tensor;
+  }
+  // WARNING: kernel->Run utilizes the FunctionLibraryRuntime
+  // (ctx->func_lib(device)), which in turn holds a pointer to func_lib_def.
+  // But knowledge of the implementation
+  // of FunctionLibraryRuntime tells us that func_lib_def is not accessed by
+  // FunctionLibraryRuntime::Run(), so there is no thread-safety concern here.
+  // This is quite subtle. Re-work things to make this better?  (Would it make
+  // sense for FunctionLibraryRuntime to ensure thread-safe access to
+  // FunctionLibraryDefinition?).  TODO(apassos) figure out how to record stats
+  // for ops which are a part of functions.
+  // TODO(agarwal): change Run to take vector of handles ?
+  TF_RETURN_IF_ERROR(kernel->Run(&inputs, &outputs, maybe_stats));
+  if (maybe_stats != nullptr) {
+    maybe_stats->set_op_end_rel_micros(Env::Default()->NowMicros() -
+                                       maybe_stats->all_start_micros());
+    mutex_lock ml(*ctx->MetadataMu());
+    if (ctx->ShouldStoreMetadata()) {
+      auto* step_stats = ctx->RunMetadataProto()->mutable_step_stats();
+      // Lazily initialize the RunMetadata with information about all devices if
+      // this is the first call.
+      while (step_stats->dev_stats_size() < ctx->devices()->size()) {
+        step_stats->add_dev_stats();
+      }
+      // Find the current device's index.
+      int device_idx = 0;
+      for (int i = 0; i < ctx->devices()->size(); ++i) {
+        if (ctx->devices()->at(i) == device) {
+          device_idx = i;
+          break;
+        }
+      }
+      // Populate the device stats for this device.
+      auto* dev_stats = step_stats->mutable_dev_stats(device_idx);
+      dev_stats->set_device(device->name());
+      *dev_stats->add_node_stats() = *maybe_stats;
+    }
+  }
+  DCHECK_EQ(num_retvals, outputs.size());
+  Device* op_device = device;
+  for (int i = 0; i < num_retvals; ++i) {
+    Device* d = op_device;
+    if (d != nullptr && output_memory_types != nullptr &&
+        (*output_memory_types)[i] == HOST_MEMORY) {
+      d = nullptr;
+    }
+    if (retvals[i] == nullptr) {
+      retvals[i] = new TensorHandle(outputs[i], d, op_device);
+    } else {
+      retvals[i]->SetTensorAndDevice(outputs[i], d, op_device);
+    }
+  }
+  return Status::OK();
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h
new file mode 100644
index 0000000000..1c0ea6bcde
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/execute.h
@@ -0,0 +1,37 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+
+namespace tensorflow {
+
+// Low-level utility to execute the kernel specified by kernel on device device,
+// with the inputs op_inputs, in the context ctx.
+Status EagerExecute(EagerContext* ctx, Device* device,
+                    const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
+                    KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+                    TensorHandle** retvals, int num_retvals);
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_
-- 
GitLab


From f9fd69a5236f20b8e90e0898f5b2928cd6591ebb Mon Sep 17 00:00:00 2001
From: Nick Felt <nfelt@users.noreply.github.com>
Date: Tue, 27 Mar 2018 15:35:37 -0700
Subject: [PATCH 573/960] Update tensorboard dependency to 1.7.0+

---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 3e4f9b0fdd..365e8d6b08 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -39,7 +39,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.13.3',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorboard >= 1.6.0, < 1.7.0',
+    'tensorboard >= 1.7.0, < 1.8.0',
     'termcolor >= 1.1.0',
 ]
 
-- 
GitLab


From 05ddf373980fae94a2c73cf93161332484e102fd Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Tue, 27 Mar 2018 15:37:35 -0700
Subject: [PATCH 574/960] [XLA] Fold reduce-window(convert(pad(X))) into
 reduce-window(convert(X))

ReduceWindow operations are done in higher precision to avoid accumulation
error. Convert operations can find their way between a ReduceWindow and a Pad
which can prevent a Pad from combining with a ReduceWindow.

Fix this by looking past the Convert while also checking that the Convert'd
Pad's init value is identical to the reduce-window value.

PiperOrigin-RevId: 190686175
---
 .../xla/service/algebraic_simplifier.cc       | 57 ++++++++++---
 .../xla/service/algebraic_simplifier_test.cc  | 85 +++++++++++++++++++
 2 files changed, 132 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier.cc b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
index f9fabd8a35..0e4624fd69 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier.cc
@@ -1731,18 +1731,29 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
                                   function));
   }
 
-  VLOG(10) << "Considering folding Pad: " << operand->ToString()
-           << "\ninto reduce-window: " << reduce_window->ToString();
-
   // This optimization folds a pad op into reduce_window.
-  if (operand->opcode() != HloOpcode::kPad) {
+  HloInstruction* pad;
+  const HloInstruction* convert = nullptr;
+  if (operand->opcode() == HloOpcode::kPad) {
+    pad = operand;
+  } else if (operand->opcode() == HloOpcode::kConvert &&
+             operand->operand(0)->opcode() == HloOpcode::kPad) {
+    convert = operand;
+    pad = operand->mutable_operand(0);
+  } else {
     VLOG(10) << "Not folding pad into reduce-window as there is no pad.";
     return Status::OK();
   }
 
+  VLOG(10) << "Considering folding Pad: " << pad->ToString()
+           << "\ninto reduce-window: " << reduce_window->ToString()
+           << (convert != nullptr ? tensorflow::strings::StrCat(
+                                        "\nvia convert: ", convert->ToString())
+                                  : "");
+
   // Do not fold interior padding into ReduceWindow since the backends do not
   // support it.
-  const PaddingConfig& pad_config = operand->padding_config();
+  const PaddingConfig& pad_config = pad->padding_config();
   if (HasInteriorPadding(pad_config)) {
     VLOG(10) << "Not folding pad into reduce-window due to interior padding.";
     return Status::OK();
@@ -1750,14 +1761,27 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
 
   // If reduce_window already has padding, the pad value of the pad op and the
   // init value of reduce_window must match to allow folding the pad.
-  const HloInstruction* pad_value = operand->operand(1);
+  const HloInstruction* pad_value = pad->operand(1);
   const HloInstruction* reduce_init_value = reduce_window->operand(1);
   if (pad_value != reduce_init_value) {
+    auto literals_are_equivalent = [&] {
+      auto& pad_literal = pad_value->literal();
+      auto& reduce_init_literal = reduce_init_value->literal();
+      if (pad_literal == reduce_init_literal) {
+        return true;
+      }
+      auto converted_pad_literal = pad_literal.ConvertToShape(
+          reduce_init_value->shape(), /*round_f32_to_bf16=*/true);
+      if (!converted_pad_literal.ok()) {
+        return false;
+      }
+      return *converted_pad_literal.ValueOrDie() == reduce_init_literal;
+    };
     // The pad value is usually a constant, so we handle that case and do not
     // try to get more fancy about proving equivalence in cases beyond that.
     if (pad_value->opcode() != HloOpcode::kConstant ||
         reduce_init_value->opcode() != HloOpcode::kConstant ||
-        pad_value->literal() != reduce_init_value->literal()) {
+        !literals_are_equivalent()) {
       VLOG(10) << "Not folding pad into reduce-window due to different pad "
                   "values.";
       return Status::OK();
@@ -1766,7 +1790,7 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
 
   // If the pad puts a single non-identity value in each window that we're
   // reducing, then this is a broadcast.
-  HloInstruction* pad_operand = operand->mutable_operand(0);
+  HloInstruction* pad_operand = pad->mutable_operand(0);
   auto is_effective_broadcast = [&] {
     if (window_util::HasStride(window)) {
       VLOG(10) << "Window has stride.";
@@ -1810,6 +1834,18 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
     VLOG(10) << "Found window covers a single unpadded element.";
     return true;
   };
+
+  HloInstruction* new_reduce_window_operand;
+  if (convert != nullptr) {
+    new_reduce_window_operand =
+        computation_->AddInstruction(HloInstruction::CreateConvert(
+            ShapeUtil::ChangeElementType(pad_operand->shape(),
+                                         convert->shape().element_type()),
+            pad_operand));
+  } else {
+    new_reduce_window_operand = pad_operand;
+  }
+
   if (is_effective_broadcast()) {
     VLOG(10) << "Replacing pad/reduce-window with (implicit) broadcast.";
     auto fadd = [this](std::unique_ptr<HloInstruction> x) {
@@ -1818,7 +1854,7 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
     return ReplaceWithNewInstruction(
         reduce_window, HloInstruction::CreateBroadcastSequence(
                            /*output_shape=*/reduce_window->shape(),
-                           /*operand=*/pad_operand, fadd));
+                           /*operand=*/new_reduce_window_operand, fadd));
   }
 
   // Carry out the folding of the pad into reduce_window.
@@ -1835,10 +1871,11 @@ Status AlgebraicSimplifierVisitor::HandleReduceWindow(
     window_dim.set_padding_high(window_dim.padding_high() +
                                 pad_dim.edge_padding_high());
   }
+
   return ReplaceWithNewInstruction(
       reduce_window, HloInstruction::CreateReduceWindow(
                          /*shape=*/reduce_window->shape(),
-                         /*operand=*/pad_operand,
+                         /*operand=*/new_reduce_window_operand,
                          /*init_value=*/reduce_window->mutable_operand(1),
                          /*window=*/new_window,
                          /*reduce_computation=*/function));
diff --git a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
index 3b80a827bf..20c549562d 100644
--- a/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
+++ b/tensorflow/compiler/xla/service/algebraic_simplifier_test.cc
@@ -2338,6 +2338,91 @@ TEST_F(AlgebraicSimplifierTest, FoldPadIntoReduceWindow) {
   EXPECT_EQ(root->window().dimensions(3).padding_high(), 102);
 }
 
+// Test that ReduceWindow(Convert(Pad(op, x)), y) can simplify to
+// ReduceWindow(Convert(op), x).
+TEST_F(AlgebraicSimplifierTest, FoldConvertedPadIntoReduceWindow) {
+  HloModule module(TestName());
+  HloComputation::Builder builder(TestName());
+
+  // Create operand to the pad.
+  HloInstruction* parameter =
+      builder.AddInstruction(HloInstruction::CreateParameter(
+          0, ShapeUtil::MakeShape(BF16, {1, 2, 3, 4}), "p0"));
+
+  // Create the pad.
+  PaddingConfig padding = MakeNoPaddingConfig(4);
+  padding.mutable_dimensions(1)->set_edge_padding_low(1);
+  padding.mutable_dimensions(3)->set_edge_padding_high(2);
+
+  HloInstruction* pad_value = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(5.0f)));
+  HloInstruction* pad = builder.AddInstruction(HloInstruction::CreatePad(
+      ShapeUtil::MakeShape(BF16, {1, 3, 3, 5}), parameter, pad_value, padding));
+
+  HloInstruction* convert =
+      builder.AddInstruction(HloInstruction::CreateConvert(
+          ShapeUtil::ChangeElementType(pad->shape(), F32), pad));
+
+  // Create add computation.
+  HloComputation* add_computation = nullptr;
+  {
+    HloComputation::Builder builder(TestName() + ".add");
+    const Shape scalar_shape = ShapeUtil::MakeShape(F32, {});
+    HloInstruction* p0 = builder.AddInstruction(
+        HloInstruction::CreateParameter(0, scalar_shape, "p0"));
+    HloInstruction* p1 = builder.AddInstruction(
+        HloInstruction::CreateParameter(1, scalar_shape, "p1"));
+    builder.AddInstruction(
+        HloInstruction::CreateBinary(scalar_shape, HloOpcode::kAdd, p0, p1));
+    add_computation = module.AddEmbeddedComputation(builder.Build());
+  }
+
+  // Create the reduce-window.
+  Window window;
+  for (int64 i = 0; i < ShapeUtil::Rank(pad->shape()); ++i) {
+    auto* dim = window.add_dimensions();
+    dim->set_size(1);
+    dim->set_padding_low(10);
+    dim->set_padding_high(100);
+    dim->set_window_dilation(1);
+    dim->set_base_dilation(1);
+  }
+  const Shape reduce_window_shape =
+      ShapeUtil::MakeShape(F32, {111, 113, 113, 115});
+  HloInstruction* reduce_init_value = builder.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(5.0f)));
+  HloInstruction* reduce_window =
+      builder.AddInstruction(HloInstruction::CreateReduceWindow(
+          reduce_window_shape, convert, reduce_init_value, window,
+          add_computation));
+
+  // Build the computation and run the simplifier.
+  auto computation = module.AddEntryComputation(builder.Build());
+  HloInstruction* root = computation->root_instruction();
+  EXPECT_EQ(root, reduce_window);
+  AlgebraicSimplifier simplifier(/*is_layout_sensitive=*/false,
+                                 non_bitcasting_callback());
+  ASSERT_TRUE(simplifier.Run(&module).ValueOrDie());
+
+  // Running simplification again should not result in any further changes.
+  ASSERT_FALSE(simplifier.Run(&module).ValueOrDie());
+
+  // Verify the result
+  root = computation->root_instruction();
+  EXPECT_THAT(root, op::ReduceWindow(op::Convert(parameter), op::Constant()));
+  EXPECT_TRUE(ShapeUtil::Equal(root->shape(), reduce_window_shape))
+      << ShapeUtil::HumanString(root->shape()) << " vs "
+      << ShapeUtil::HumanString(reduce_window_shape);
+  EXPECT_EQ(root->window().dimensions(0).padding_low(), 10);
+  EXPECT_EQ(root->window().dimensions(1).padding_low(), 11);
+  EXPECT_EQ(root->window().dimensions(2).padding_low(), 10);
+  EXPECT_EQ(root->window().dimensions(3).padding_low(), 10);
+  EXPECT_EQ(root->window().dimensions(0).padding_high(), 100);
+  EXPECT_EQ(root->window().dimensions(1).padding_high(), 100);
+  EXPECT_EQ(root->window().dimensions(2).padding_high(), 100);
+  EXPECT_EQ(root->window().dimensions(3).padding_high(), 102);
+}
+
 TEST_F(AlgebraicSimplifierTest, ReversalOfTrivialDimensionsToBitcast) {
   HloComputation::Builder builder(TestName());
   const Shape shape = ShapeUtil::MakeShape(F32, {448, 2048, 1, 1});
-- 
GitLab


From b4742b76c386409c96c60172e6ca1c1534e2b4af Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 15:47:23 -0700
Subject: [PATCH 575/960] Add node types for DFS traversal to catch more issues
 with deduping inputs to in-place ops.

PiperOrigin-RevId: 190687820
---
 tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 23e21855c8..5dd0b6f4b0 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -1089,7 +1089,8 @@ namespace {
 
 bool FeedsInPlaceOp(const SimpleGraphView& graph_view, const NodeDef& node) {
   const std::unordered_set<string> op_types_to_traverse = {
-      node.op(), "Identity", "IdentityN", "Reshape"};
+      node.op(),    "Identity", "IdentityN", "Reshape",
+      "ExpandDims", "Enter",    "Switch",    "Merge"};
   int node_idx = graph_view.index(node.name());
   std::set<int> node_fanout;
   graph_view.DepthFirstSearch(op_types_to_traverse, node_idx, &node_fanout);
-- 
GitLab


From a16761483ec55095158b1b11118d93ea00a538f4 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 27 Mar 2018 15:55:04 -0700
Subject: [PATCH 576/960] TFTS: Fix a bug in the SavedModel cold-start export

It now correctly broadcasts start state across whatever batch dimension it is
passed rather than sqishing it down to a batch dimension of 1.

PiperOrigin-RevId: 190688855
---
 .../python/timeseries/estimators_test.py      | 21 +++++++++++++++++++
 .../timeseries/python/timeseries/head.py      |  6 ++++--
 2 files changed, 25 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
index f4304f2560..51d0c0ca3f 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
@@ -126,6 +126,27 @@ class TimeSeriesRegressorTest(test.TestCase):
             signatures=signatures,
             session=sess)
 
+        # Test cold starting
+        batch_numpy_times = numpy.tile(
+            numpy.arange(30, dtype=numpy.int64)[None, :], (10, 1))
+        batch_numpy_values = numpy.ones([10, 30, 1])
+        state = saved_model_utils.cold_start_filter(
+            signatures=signatures,
+            session=sess,
+            features={
+                feature_keys.FilteringFeatures.TIMES: batch_numpy_times,
+                feature_keys.FilteringFeatures.VALUES: batch_numpy_values
+            }
+        )
+        predict_times = numpy.tile(
+            numpy.arange(30, 45, dtype=numpy.int64)[None, :], (10, 1))
+        predictions = saved_model_utils.predict_continuation(
+            continue_from=state,
+            times=predict_times,
+            signatures=signatures,
+            session=sess)
+        self.assertAllEqual([10, 15, 1], predictions["mean"].shape)
+
   def test_fit_restore_fit_ar_regressor(self):
     def _estimator_fn(model_dir):
       return estimators.ARRegressor(
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py
index 3d7e615290..4cf6bbcfd4 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head.py
@@ -154,8 +154,10 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
       no_state_features = {
           k: v for k, v in features.items()
           if not k.startswith(feature_keys.State.STATE_PREFIX)}
-      cold_filtering_outputs = self.create_loss(
-          no_state_features, estimator_lib.ModeKeys.EVAL)
+      # Ignore any state management when cold-starting. The model's default
+      # start state is replicated across the batch.
+      cold_filtering_outputs = self.model.define_loss(
+          features=no_state_features, mode=estimator_lib.ModeKeys.EVAL)
     return estimator_lib.EstimatorSpec(
         mode=estimator_lib.ModeKeys.PREDICT,
         export_outputs={
-- 
GitLab


From 496840acbdd8b8b7688c257793e09a02229d21f6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 16:26:12 -0700
Subject: [PATCH 577/960] Test all TFLite kernel implementations for fully
 connected.

PiperOrigin-RevId: 190693455
---
 tensorflow/contrib/lite/kernels/BUILD         |  1 +
 .../lite/kernels/fully_connected_test.cc      | 57 +++++++++++++++----
 2 files changed, 48 insertions(+), 10 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 200cb3075b..1450c1e14b 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -710,6 +710,7 @@ tf_cc_test(
         ":builtin_ops",
         "//tensorflow/contrib/lite:framework",
         "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_absl//absl/memory",
         "@com_google_googletest//:gtest",
     ],
 )
diff --git a/tensorflow/contrib/lite/kernels/fully_connected_test.cc b/tensorflow/contrib/lite/kernels/fully_connected_test.cc
index a0f766c4f4..87413000a9 100644
--- a/tensorflow/contrib/lite/kernels/fully_connected_test.cc
+++ b/tensorflow/contrib/lite/kernels/fully_connected_test.cc
@@ -19,12 +19,25 @@ limitations under the License.
 
 #include <gmock/gmock.h>
 #include <gtest/gtest.h>
+#include "absl/memory/memory.h"
 #include "tensorflow/contrib/lite/interpreter.h"
 #include "tensorflow/contrib/lite/kernels/register.h"
 #include "tensorflow/contrib/lite/kernels/test_util.h"
 #include "tensorflow/contrib/lite/model.h"
 
 namespace tflite {
+
+namespace ops {
+namespace builtin {
+
+TfLiteRegistration* Register_FULLY_CONNECTED_REF();
+TfLiteRegistration* Register_FULLY_CONNECTED_NEON_OPT();
+TfLiteRegistration* Register_FULLY_CONNECTED_GENERIC_OPT();
+TfLiteRegistration* Register_FULLY_CONNECTED_PIE();
+
+}  // namespace builtin
+}  // namespace ops
+
 namespace {
 
 using ::testing::ElementsAre;
@@ -119,7 +132,8 @@ static float fully_connected_golden_output[] = {
 class BaseFullyConnectedOpModel : public SingleOpModel {
  public:
   // TODO(ahentz): test different activation types too.
-  BaseFullyConnectedOpModel(int units, int batches, const TensorData& input,
+  BaseFullyConnectedOpModel(TfLiteRegistration* registration, int units,
+                            int batches, const TensorData& input,
                             const TensorData& output = {TensorType_FLOAT32})
       : batches_(batches), units_(units) {
     int total_input_size = 1;
@@ -149,6 +163,8 @@ class BaseFullyConnectedOpModel : public SingleOpModel {
         BuiltinOperator_FULLY_CONNECTED, BuiltinOptions_FullyConnectedOptions,
         CreateFullyConnectedOptions(builder_, ActivationFunctionType_RELU)
             .Union());
+    resolver_ = absl::make_unique<SingleOpResolver>(
+        BuiltinOperator_FULLY_CONNECTED, registration);
     BuildInterpreter({GetShape(input_), GetShape(weights_), GetShape(bias_)});
   }
 
@@ -208,10 +224,25 @@ class QuantizedFullyConnectedOpModel : public BaseFullyConnectedOpModel {
   }
 };
 
+const auto kKernelMap = new std::map<string, TfLiteRegistration*>({
+    {"Reference", ops::builtin::Register_FULLY_CONNECTED_REF()},
+    {"NeonOptimized", ops::builtin::Register_FULLY_CONNECTED_NEON_OPT()},
+    {"GenericOptimized", ops::builtin::Register_FULLY_CONNECTED_GENERIC_OPT()},
+    {"Pie", ops::builtin::Register_FULLY_CONNECTED_PIE()},
+});
+
+class FullyConnectedOpTest : public SingleOpTest {
+ protected:
+  const std::map<string, TfLiteRegistration*>& GetKernelMap() override {
+    return *kKernelMap;
+  }
+};
+
 // TODO(ahentz): add more small tests like this one, focused on making sure the
 // calculations are correct.
-TEST(FullyConnectedOpTest, SimpleTest) {
-  FloatFullyConnectedOpModel m(3, 2, {TensorType_FLOAT32, {2, 10}});
+TEST_P(FullyConnectedOpTest, SimpleTest) {
+  FloatFullyConnectedOpModel m(GetRegistration(), 3, 2,
+                               {TensorType_FLOAT32, {2, 10}});
   m.SetWeights({
       1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 0
       1, 2, 3, 4, 5, 6, 7, 8, 9, 10,  // u = 1
@@ -229,9 +260,9 @@ TEST(FullyConnectedOpTest, SimpleTest) {
   EXPECT_THAT(m.GetOutput(), ElementsAre(24, 25, 26, 58, 59, 60));
 }
 
-TEST(FullyConnectedOpTest, SimpleTestQuantized) {
+TEST_P(FullyConnectedOpTest, SimpleTestQuantized) {
   QuantizedFullyConnectedOpModel m(
-      3, 2,
+      GetRegistration(), 3, 2,
       /*input=*/{TensorType_UINT8, {2, 10}, -63.5, 64},
       /*output=*/{TensorType_UINT8, {}, -127, 128});
 
@@ -261,7 +292,8 @@ TEST(FullyConnectedOpTest, SimpleTest4DInput) {
   // Note that it is not required that the first dimension be the number of
   // batches. All we care is that the input can be evenly distributed in
   // batches. In this case, we need the input to have multiples of '2'.
-  FloatFullyConnectedOpModel m(/*units=*/3,
+  FloatFullyConnectedOpModel m(ops::builtin::Register_FULLY_CONNECTED_PIE(),
+                               /*units=*/3,
                                /*batches=*/2,
                                /*input=*/{TensorType_FLOAT32, {4, 1, 5, 1}});
   m.SetWeights({
@@ -284,9 +316,9 @@ TEST(FullyConnectedOpTest, SimpleTest4DInput) {
                              }));
 }
 
-TEST(FullyConnectedOpTest, SimpleTest4dInputQuantized) {
+TEST_P(FullyConnectedOpTest, SimpleTest4dInputQuantized) {
   QuantizedFullyConnectedOpModel m(
-      3, 2,
+      GetRegistration(), 3, 2,
       /*input=*/{TensorType_UINT8, {4, 1, 5, 1}, -63.5, 64},
       /*output=*/{TensorType_UINT8, {}, -127, 128});
 
@@ -312,10 +344,15 @@ TEST(FullyConnectedOpTest, SimpleTest4dInputQuantized) {
   EXPECT_THAT(m.GetOutput(), ElementsAre(151, 152, 153, 185, 186, 187));
 }
 
+INSTANTIATE_TEST_CASE_P(
+    FullyConnectedOpTest, FullyConnectedOpTest,
+    ::testing::ValuesIn(SingleOpTest::GetKernelTags(*kKernelMap)));
+
 // TODO(ahentz): Reconsider this test. Having arbitrary weights makes it hard
 // to debug errors and doesn't necessarily test all the important details.
-TEST(FullyConnectedOpTest, BlackBoxTest) {
-  FloatFullyConnectedOpModel m(16, 2, {TensorType_FLOAT32, {2, 8}});
+TEST_P(FullyConnectedOpTest, BlackBoxTest) {
+  FloatFullyConnectedOpModel m(GetRegistration(), 16, 2,
+                               {TensorType_FLOAT32, {2, 8}});
   m.SetWeights(
       {0.091327,  0.103366,  -0.316505, -0.083120, 0.149366,  -0.196636,
        -0.123672, 0.062800,  0.063031,  0.191670,  -0.062001, -0.061504,
-- 
GitLab


From 9da8636553c4e6713d974fb7e9694198a530425a Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@andyselle.com>
Date: Tue, 27 Mar 2018 16:46:34 -0700
Subject: [PATCH 578/960] Add link to raspberry pi demo (#18034)

---
 tensorflow/contrib/lite/README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index 2680d515eb..c15ae3f233 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -126,6 +126,9 @@ The above pre-trained models have been trained on the ImageNet data set, which c
 
 The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
 
+# Getting started with RaspberryPi
+
+Using RaspberryPi can be accomplished by following the [Makefile instructions](g3doc/rpi.md). That will give a you a static library (.a) that you can build your app against. Python bindings will be coming soon as well as a demo app.
 
 ### Train a custom model
 A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
-- 
GitLab


From 71593602d95385fbd8c3dde361dab09d381b5ac6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 16:43:48 -0700
Subject: [PATCH 579/960] Fixed a bug in ConvKFCBasicMultiIndepFB introduced in
 the last CL

PiperOrigin-RevId: 190695737
---
 tensorflow/contrib/kfac/python/ops/fisher_blocks.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index b04bf76a88..e0d9cb5ea9 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -861,12 +861,12 @@ class ConvKFCBasicFB(InputOutputMultiTower, KroneckerProductFB):
     super(ConvKFCBasicFB, self).__init__(layer_collection)
 
   def instantiate_factors(self, grads_list, damping):
+    inputs, grads_list = self._process_data(grads_list)
+
     # Infer number of locations upon which convolution is applied.
-    self._num_locations = num_conv_locations(self._inputs[0].shape.as_list(),
+    self._num_locations = num_conv_locations(inputs[0].shape.as_list(),
                                              self._strides)
 
-    inputs, grads_list = self._process_data(grads_list)
-
     self._input_factor = self._layer_collection.make_or_get_factor(
         fisher_factors.ConvInputKroneckerFactor,
         (inputs, self._filter_shape, self._padding, self._strides,
@@ -1391,7 +1391,7 @@ class ConvKFCBasicMultiIndepFB(InputOutputMultiTowerMultiUse,
     inputs, grads_list = self._process_data(grads_list)
 
     # Infer number of locations upon which convolution is applied.
-    self._num_locations = num_conv_locations(inputs.shape.as_list(),
+    self._num_locations = num_conv_locations(inputs[0].shape.as_list(),
                                              self._strides)
 
     self._input_factor = self._layer_collection.make_or_get_factor(
-- 
GitLab


From 719b0f6f900aca51b59e3bfc6fbf73d3b4a0af63 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 16:48:31 -0700
Subject: [PATCH 580/960] Improve support for DT_HALF and DT_BFLOAT16 in
 Grappler graph optimizations.

Update GrapplerTest::EvaluateNodes to take feeds as an argument, to make it easier to write tests with placeholders.

PiperOrigin-RevId: 190696386
---
 .../optimizers/arithmetic_optimizer_test.cc   |   4 +-
 .../grappler/optimizers/constant_folding.cc   |  84 +++++++-------
 .../grappler/optimizers/constant_folding.h    |   2 +-
 .../optimizers/constant_folding_test.cc       | 109 +++++++++++++-----
 .../optimizers/function_optimizer_test.cc     |   9 +-
 .../optimizers/memory_optimizer_test.cc       |   2 +-
 tensorflow/core/grappler/utils.cc             |   8 +-
 .../core/grappler/utils/grappler_test.cc      |   5 +-
 .../core/grappler/utils/grappler_test.h       |   3 +-
 9 files changed, 141 insertions(+), 85 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 792f675043..ad3edc144a 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -158,7 +158,7 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {});
   EXPECT_EQ(1, tensors_expected.size());
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -176,7 +176,7 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   EXPECT_EQ("c1", new_div.input(0));
   EXPECT_EQ("c1", new_div.input(1));
 
-  auto tensors = EvaluateNodes(output, item.fetch);
+  auto tensors = EvaluateNodes(output, item.fetch, {});
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorNear<double>(tensors_expected[0], tensors[0], 1e-6);
 }
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index bdec73e69e..22ede19493 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -109,33 +109,18 @@ class DeviceSimple : public DeviceBase {
 };
 
 template <typename T>
-bool AllValuesAre(const TensorProto& tensor, const T& value) {
-  // TensorProto represents the content of the tensor in either <type>_val or
-  // tensor_content.
-  typename checkpoint::SaveTypeTraits<T>::RepeatedField* tensor_values =
-      checkpoint::MutableTensorProtoData<T>(const_cast<TensorProto*>(&tensor));
-  if (!tensor_values->empty()) {
-    for (const T& tensor_value : *tensor_values) {
-      if (tensor_value != value) {
-        return false;
-      }
-    }
-    return true;
+bool AllValuesAre(const TensorProto& proto, const T& value) {
+  Tensor tensor;
+  if (!tensor.FromProto(proto)) {
+    return false;
   }
-  const auto tensor_content_size = tensor.tensor_content().size();
-  if (tensor_content_size > 0) {
-    CHECK_EQ(0, tensor_content_size % sizeof(T));
-    std::vector<T> raw_values(tensor_content_size / sizeof(T));
-    port::CopyToArray(tensor.tensor_content(),
-                      reinterpret_cast<char*>(raw_values.data()));
-    for (int i = 0; i < tensor_content_size / sizeof(T); ++i) {
-      if (raw_values[i] != value) {
-        return false;
-      }
+  auto values = tensor.flat<T>();
+  for (int i = 0; i < tensor.NumElements(); ++i) {
+    if (values(i) != value) {
+      return false;
     }
-    return true;
   }
-  return false;
+  return true;
 }
 
 // Add new_input as a control input to node if it does not already depend on it.
@@ -825,17 +810,23 @@ Status CreateConstantTensorAttrValue(DataType type, double value,
   t->set_dtype(type);
   *t->mutable_tensor_shape() = shape;
   switch (type) {
-    SET_TENSOR_VAL_CASE(DT_FLOAT, float, float);
-    SET_TENSOR_VAL_CASE(DT_DOUBLE, double, double);
-    SET_TENSOR_VAL_CASE(DT_INT64, int64, int64);
-    SET_TENSOR_VAL_CASE(DT_UINT64, int64, int64);
-    SET_TENSOR_VAL_CASE(DT_INT32, int32, int);
-    SET_TENSOR_VAL_CASE(DT_UINT32, int32, int);
-    SET_TENSOR_VAL_CASE(DT_INT16, int32, int);
-    SET_TENSOR_VAL_CASE(DT_UINT16, int32, int);
-    SET_TENSOR_VAL_CASE(DT_INT8, int32, int);
-    SET_TENSOR_VAL_CASE(DT_UINT8, int32, int);
-    SET_TENSOR_VAL_CASE(DT_BOOL, bool, bool);
+    case DT_HALF:
+      t->add_half_val(static_cast<Eigen::half>(value).x);
+      break;
+    case DT_BFLOAT16:
+      t->add_half_val(static_cast<bfloat16>(value).value);
+      break;
+      SET_TENSOR_VAL_CASE(DT_FLOAT, float, float);
+      SET_TENSOR_VAL_CASE(DT_DOUBLE, double, double);
+      SET_TENSOR_VAL_CASE(DT_INT64, int64, int64);
+      SET_TENSOR_VAL_CASE(DT_UINT64, int64, int64);
+      SET_TENSOR_VAL_CASE(DT_INT32, int32, int);
+      SET_TENSOR_VAL_CASE(DT_UINT32, int32, int);
+      SET_TENSOR_VAL_CASE(DT_INT16, int32, int);
+      SET_TENSOR_VAL_CASE(DT_UINT16, int32, int);
+      SET_TENSOR_VAL_CASE(DT_INT8, int32, int);
+      SET_TENSOR_VAL_CASE(DT_UINT8, int32, int);
+      SET_TENSOR_VAL_CASE(DT_BOOL, bool, bool);
     default:
       return errors::InvalidArgument("Unsupported type: ", type);
   }
@@ -1388,8 +1379,8 @@ bool ConstantFolding::IsOnes(const NodeDef& node) const {
   }
   const auto dtype = node.attr().at("dtype").type();
   switch (dtype) {
-    // TODO(rmlarsen): Make DT_HALF case compile.
-    //    IS_ONES_CASE(DT_HALF);
+    IS_ONES_CASE(DT_HALF);
+    IS_ONES_CASE(DT_BFLOAT16);
     IS_ONES_CASE(DT_FLOAT);
     IS_ONES_CASE(DT_DOUBLE);
     IS_ONES_CASE(DT_COMPLEX64);
@@ -1423,8 +1414,8 @@ bool ConstantFolding::IsZeros(const NodeDef& node) const {
   }
   const auto dtype = node.attr().at("dtype").type();
   switch (dtype) {
-    // TODO(rmlarsen): Make DT_HALF case compile.
-    //    IS_ZEROS_CASE(DT_HALF);
+    IS_ZEROS_CASE(DT_HALF);
+    IS_ZEROS_CASE(DT_BFLOAT16);
     IS_ZEROS_CASE(DT_FLOAT);
     IS_ZEROS_CASE(DT_DOUBLE);
     IS_ZEROS_CASE(DT_COMPLEX64);
@@ -1511,9 +1502,8 @@ void ConstantFolding::ReplaceSubtractionFromZeroByNegation(NodeDef* node,
 }
 
 Status ConstantFolding::ReplaceOperationWithConstant(
-    double value, const TensorShapeProto& shape, NodeDef* node,
-    GraphDef* graph) {
-  AttrValue dtype_attr = node->attr().at("T");
+    double value, const AttrValue& dtype_attr, const TensorShapeProto& shape,
+    NodeDef* node, GraphDef* graph) {
   AttrValue tensor_attr;
   TF_RETURN_IF_ERROR(CreateConstantTensorAttrValue(dtype_attr.type(), value,
                                                    shape, &tensor_attr));
@@ -1947,8 +1937,14 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
           (is_mul || is_matmul || optimize_zeros_divided_by_y)) {
         const PartialTensorShape shp(output_shape);
         if (shp.IsFullyDefined()) {
-          TF_RETURN_IF_ERROR(ReplaceOperationWithConstant(0, output_shape, node,
-                                                          optimized_graph));
+          AttrValue dtype_attr;
+          if (node->op() == "SparseMatMul") {
+            dtype_attr.set_type(DT_FLOAT);
+          } else {
+            dtype_attr = node->attr().at("T");
+          }
+          TF_RETURN_IF_ERROR(ReplaceOperationWithConstant(
+              0, dtype_attr, output_shape, node, optimized_graph));
           continue;
         }
         // Even if an input shape is only partially known, we may known that it
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.h b/tensorflow/core/grappler/optimizers/constant_folding.h
index b6645d335e..f8a9e90d62 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.h
+++ b/tensorflow/core/grappler/optimizers/constant_folding.h
@@ -83,7 +83,7 @@ class ConstantFolding : public GraphOptimizer {
   void ReplaceOperationWithSnapshot(int input_to_forward, NodeDef* node,
                                     GraphDef* graph);
   void ReplaceSubtractionFromZeroByNegation(NodeDef* node, GraphDef* graph);
-  Status ReplaceOperationWithConstant(double value,
+  Status ReplaceOperationWithConstant(double value, const AttrValue& dtype_attr,
                                       const TensorShapeProto& shape,
                                       NodeDef* node, GraphDef* graph);
   void ReplaceDivisionOfOnesByReciprocal(NodeDef* node, GraphDef* graph);
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index dc9c1053d2..85f877883c 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -28,7 +28,59 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-class ConstantFoldingTest : public GrapplerTest {};
+class ConstantFoldingTest : public GrapplerTest {
+ protected:
+  template <DataType DTYPE>
+  void SimpleNeutralElementTest() {
+    typedef typename EnumToDataType<DTYPE>::Type T;
+    tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+    Output x = ops::Placeholder(s.WithOpName("x"), DTYPE,
+                                ops::Placeholder::Shape(TensorShape({2, 2})));
+    Tensor zeros_t(DTYPE, TensorShape({2, 2}));
+    Tensor ones_t(DTYPE, TensorShape({2, 2}));
+    Tensor x_t(DTYPE, TensorShape({2, 2}));
+    for (int i = 0; i < 4; ++i) {
+      zeros_t.flat<T>()(i) = T(0);
+      ones_t.flat<T>()(i) = T(1);
+      x_t.flat<T>()(i) = T(i + 1);
+    }
+    Output zeros = ops::Const(s.WithOpName("zeros"), zeros_t);
+    Output ones = ops::Const(s.WithOpName("ones"), ones_t);
+    Output mul1 = ops::Mul(s.WithOpName("mul1"), x, zeros);
+    Output mul2 = ops::Mul(s.WithOpName("mul2"), x, ones);
+
+    GrapplerItem item;
+    TF_CHECK_OK(s.ToGraphDef(&item.graph));
+    item.fetch = {"mul1", "mul2"};
+    ConstantFolding optimizer(nullptr /* cpu_device */);
+    GraphDef output;
+    Status status = optimizer.Optimize(nullptr, item, &output);
+    TF_EXPECT_OK(status);
+    LOG(INFO) << output.DebugString();
+    EXPECT_EQ(5, output.node_size());
+    for (int i = 0; i < output.node_size(); ++i) {
+      const NodeDef& node = output.node(i);
+      const string& name = node.name();
+      if (name == "mul1") {
+        EXPECT_EQ("Const", node.op());
+        EXPECT_EQ("^x", node.input(0));
+        EXPECT_EQ("^zeros", node.input(1));
+      } else if (name == "mul2") {
+        EXPECT_EQ("Snapshot", node.op());
+        EXPECT_EQ("x", node.input(0));
+        EXPECT_EQ("^ones", node.input(1));
+      }
+    }
+    auto tensors_expected =
+        EvaluateNodes(item.graph, {"mul1", "mul2"}, {{"x", x_t}});
+    auto tensors = EvaluateNodes(output, {"mul1", "mul2"}, {{"x", x_t}});
+    EXPECT_EQ(2, tensors_expected.size());
+    EXPECT_EQ(2, tensors.size());
+    for (int i = 0; i < 2; ++i) {
+      test::ExpectTensorEqual<T>(tensors_expected[i], tensors[i]);
+    }
+  }
+};
 
 TEST_F(ConstantFoldingTest, SimpleFolding) {
   // Build a simple graph with a few trivially prunable ops.
@@ -55,8 +107,8 @@ TEST_F(ConstantFoldingTest, SimpleFolding) {
   EXPECT_EQ("Const", node_d.op());
 
   std::vector<string> fetch = {"d"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch);
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(1, tensors_expected.size());
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -141,10 +193,10 @@ TEST_F(ConstantFoldingTest, AddTree) {
 
   // Check that the result nodes have the expected value.
   std::vector<string> fetch = {"c3", "c20"};
-  auto tensor_expected = EvaluateNodes(item.graph, fetch);
+  auto tensor_expected = EvaluateNodes(item.graph, fetch, {});
   EXPECT_EQ(fetch.size(), tensor_expected.size());
   fetch = {"add_child", "mul_child"};
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
     test::ExpectTensorEqual<float>(tensor_expected[i], tensors[i]);
@@ -322,6 +374,11 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
   }
 }
 
+TEST_F(ConstantFoldingTest, NeutralElement_ShortFloats) {
+  SimpleNeutralElementTest<DT_HALF>();
+  SimpleNeutralElementTest<DT_BFLOAT16>();
+}
+
 TEST_F(ConstantFoldingTest, StrengthReduce_Reciprocal) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output cf_half = ops::Const(s.WithOpName("cf_half"), 0.5f, {1});
@@ -379,10 +436,10 @@ TEST_F(ConstantFoldingTest, StrengthReduce_Reciprocal) {
 
   // Check that the reciprocals have the expected value.
   std::vector<string> fetch = {"cf_half"};
-  auto tensor_expected = EvaluateNodes(item.graph, fetch);
+  auto tensor_expected = EvaluateNodes(item.graph, fetch, {});
   EXPECT_EQ(fetch.size(), tensor_expected.size());
   fetch = {"ConstantFolding/div_f_recip", "ConstantFolding/realdiv_recip"};
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
     test::ExpectTensorEqual<float>(tensor_expected[0], tensors[i]);
@@ -590,8 +647,8 @@ TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) {
   EXPECT_EQ("Const", new_d.op());
 
   std::vector<string> fetch = {"e", "f"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch);
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(fetch.size(), tensors_expected.size());
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
@@ -614,7 +671,7 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
   GrapplerItem item;
   item.fetch.push_back("e");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
-  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {});
   EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
@@ -631,8 +688,8 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
     if (node.name() == "e") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"e"});
-      auto expected = EvaluateNodes(item.graph, {"e"});
+      auto folded = EvaluateNodes(output, {"e"}, {});
+      auto expected = EvaluateNodes(item.graph, {"e"}, {});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -642,7 +699,7 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
     }
   }
   EXPECT_EQ(1, found);
-  auto tensors = EvaluateNodes(output, item.fetch);
+  auto tensors = EvaluateNodes(output, item.fetch, {});
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorEqual<int>(tensors_expected[0], tensors[0]);
 }
@@ -678,8 +735,8 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
     if (node.name() == "i1") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"i1"});
-      auto expected = EvaluateNodes(item.graph, {"i1"});
+      auto folded = EvaluateNodes(output, {"i1"}, {});
+      auto expected = EvaluateNodes(item.graph, {"i1"}, {});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -689,8 +746,8 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
     if (node.name() == "i2") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"i2"});
-      auto expected = EvaluateNodes(item.graph, {"i2"});
+      auto folded = EvaluateNodes(output, {"i2"}, {});
+      auto expected = EvaluateNodes(item.graph, {"i2"}, {});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -808,8 +865,8 @@ TEST_F(ConstantFoldingTest, VariableNumberOfOutputs) {
   }
   EXPECT_EQ(8, constant_folded);
 
-  auto expected = EvaluateNodes(item.graph, outputs);
-  auto optimized = EvaluateNodes(output, outputs);
+  auto expected = EvaluateNodes(item.graph, outputs, {});
+  auto optimized = EvaluateNodes(output, outputs, {});
   ASSERT_EQ(expected.size(), optimized.size());
   for (int i = 0; i < expected.size(); ++i) {
     test::ExpectTensorEqual<int>(expected[i], optimized[i]);
@@ -1236,7 +1293,7 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   EXPECT_EQ(6, found_nodes);
 
   std::vector<string> fetch = {"out1", "idx1"};
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(2, tensors.size());
   const Tensor& out_value = tensors[0];
   EXPECT_EQ(3 * 5, out_value.NumElements());
@@ -1891,8 +1948,8 @@ TEST_F(ConstantFoldingTest, PartialFolding_AssociativeAndCommutative) {
     }
 
     std::vector<string> fetch = {"acc0"};
-    auto tensors_expected = EvaluateNodes(item.graph, fetch);
-    auto tensors = EvaluateNodes(output, fetch);
+    auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
+    auto tensors = EvaluateNodes(output, fetch, {});
     EXPECT_EQ(1, tensors_expected.size());
     EXPECT_EQ(1, tensors.size());
     test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
@@ -1926,7 +1983,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
   item.fetch = {"concat0", "concat1", "concat2", "concat3", "concat4",
                 "concat5", "concat6", "concat7", "concat8", "concat9"};
 
-  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"});
+  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"}, {});
   EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
@@ -1977,7 +2034,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
     }
   }
 
-  auto tensors = EvaluateNodes(output, {"concat0"});
+  auto tensors = EvaluateNodes(output, {"concat0"}, {});
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
@@ -2075,8 +2132,8 @@ TEST_F(ConstantFoldingTest, TrivialPack) {
   }
 
   std::vector<string> fetch = {"stack"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch);
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(1, tensors_expected.size());
   EXPECT_EQ(1, tensors.size());
   EXPECT_EQ(tensors_expected[0].shape(), tensors[0].shape());
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index 52a1118080..deb2fabded 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -414,8 +414,9 @@ TEST_F(FunctionOptimizerTest, SymbolicGradients) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  std::vector<Tensor> expected = EvaluateNodes(item.graph, {"out1", "out2"});
-  std::vector<Tensor> optimized = EvaluateNodes(output, {"out1", "out2"});
+  std::vector<Tensor> expected =
+      EvaluateNodes(item.graph, {"out1", "out2"}, {});
+  std::vector<Tensor> optimized = EvaluateNodes(output, {"out1", "out2"}, {});
   test::ExpectTensorEqual<float>(expected[0], optimized[0]);
   test::ExpectTensorEqual<float>(expected[1], optimized[1]);
 }
@@ -478,8 +479,8 @@ TEST_F(FunctionOptimizerTest, SymbolicGradientsIdentity) {
     EXPECT_EQ("Identity", output.node(i).op());
   }
 
-  std::vector<Tensor> expected = EvaluateNodes(item.graph, {"out"});
-  std::vector<Tensor> optimized = EvaluateNodes(output, {"out"});
+  std::vector<Tensor> expected = EvaluateNodes(item.graph, {"out"}, {});
+  std::vector<Tensor> optimized = EvaluateNodes(output, {"out"}, {});
   test::ExpectTensorEqual<float>(expected[0], optimized[0]);
 }
 
diff --git a/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc b/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
index 9595936e9e..a1f80802dd 100644
--- a/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/memory_optimizer_test.cc
@@ -426,7 +426,7 @@ TEST_F(MemoryOptimizerTest, AccumulationRewrites) {
   EXPECT_EQ(4, count);
 
   std::vector<string> fetch = {"a", "b", "c", "e"};
-  auto tensors = EvaluateNodes(output, fetch);
+  auto tensors = EvaluateNodes(output, fetch, {});
   EXPECT_EQ(4, tensors.size());
 
   for (int i = 0; i < tensors[0].NumElements(); ++i) {
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index 829bfe9e31..86a6d5000d 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -33,8 +33,8 @@ namespace {
 template <typename T>
 bool SafeSetScalarTensorValue(double value, Tensor* tensor) {
   using RealType = typename Eigen::NumTraits<T>::Real;
-  if (value > std::numeric_limits<RealType>::max() ||
-      value < std::numeric_limits<RealType>::min()) {
+  if (value > static_cast<double>(std::numeric_limits<RealType>::max()) ||
+      value < static_cast<double>(std::numeric_limits<RealType>::min())) {
     return false;
   }
   tensor->flat<T>()(0) = static_cast<T>(value);
@@ -473,8 +473,8 @@ Status SetTensorValue(DataType dtype, int value, Tensor* tensor) {
         "Expected scalar tensor, got num_elements = ", tensor->NumElements());
   }
   switch (dtype) {
-    // TODO(rmlarsen): Handle DT_HALF.
-    //    HANDLE_CASE(DT_HALF);
+    HANDLE_CASE(DT_HALF);
+    HANDLE_CASE(DT_BFLOAT16);
     HANDLE_CASE(DT_BOOL);
     HANDLE_CASE(DT_FLOAT);
     HANDLE_CASE(DT_DOUBLE);
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index ee126f4955..5c96359867 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -40,12 +40,13 @@ GrapplerTest::GrapplerTest() {
 }
 
 std::vector<Tensor> GrapplerTest::EvaluateNodes(
-    const GraphDef& graph, const std::vector<string>& node_names) const {
+    const GraphDef& graph, const std::vector<string>& node_names,
+    const std::vector<std::pair<string, Tensor>>& inputs) const {
   std::unique_ptr<tensorflow::Session> session(NewSession(options_));
   TF_CHECK_OK(session->Create(graph));
   RunOptions run_options;
   std::vector<Tensor> output_tensors;
-  TF_CHECK_OK(session->Run(run_options, {}, node_names, node_names,
+  TF_CHECK_OK(session->Run(run_options, inputs, node_names, node_names,
                            &output_tensors, nullptr));
   TF_CHECK_OK(session->Close());
   return output_tensors;
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index e0c67381a4..4b160e7f16 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -35,7 +35,8 @@ class GrapplerTest : public ::testing::Test {
 
  protected:
   std::vector<Tensor> EvaluateNodes(
-      const GraphDef& graph, const std::vector<string>& node_names) const;
+      const GraphDef& graph, const std::vector<string>& node_names,
+      const std::vector<std::pair<string, Tensor>>& inputs) const;
 
   std::vector<Tensor> EvaluateFetchNodes(const GrapplerItem& item) const;
 
-- 
GitLab


From b2644288ebfb8a4cc52231c3ca93a968397c860a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 16:53:13 -0700
Subject: [PATCH 581/960]   Change the host-op result per TPU step from a
 single value to a collection of values.

PiperOrigin-RevId: 190696953
---
 .../contrib/tpu/profiler/tf_op_stats.proto       | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
index 20ed7419fd..590db2c376 100644
--- a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
+++ b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
@@ -199,10 +199,22 @@ message HostOpsPerTpuStep {
   map<int32, int32> step_diffs = 5;
 }
 
+message HostOpsDetailsPerCore {
+  // Map from core id to HostOpsPerTpuStep.
+  map<int32, HostOpsPerTpuStep> core_map = 1;
+}
+
+message HostOpsDetailsPerHost {
+  // Map from hostname to a map from core id to HostOpsPerTpuStep.
+  map<string, HostOpsDetailsPerCore> host_map = 1;
+}
+
 // Result proto for the host ops for all TPU steps.
 message HostOpsResult {
-  // A sequence of HostOpsPerTpuStep (one for each TPU step)
-  repeated HostOpsPerTpuStep host_op_sequence = 1;
+  reserved 1;  // (was repeated HostOpsPerTpuStep host_op_sequence)
+  // A sequence of records with one for each TPU step. Each record
+  // is a map from hostname to a map from core id to HostOpsPerTpuStep.
+  repeated HostOpsDetailsPerHost hostops_details = 2;
 }
 
 // Result proto for TfStatsHelper.
-- 
GitLab


From 6f8b85d301140ce42c0aa4871750ee0aec758105 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 17:02:59 -0700
Subject: [PATCH 582/960] [XLA] Redesign: implement Tuple and GetTupleElement.

PiperOrigin-RevId: 190698245
---
 .../xla/client/xla_client/xla_builder.cc      | 33 ++++++-
 .../compiler/xla/service/shape_inference.cc   |  6 ++
 .../compiler/xla/service/shape_inference.h    |  3 +
 tensorflow/compiler/xla/tests/BUILD           |  2 +
 .../xla/tests/client_library_test_base.cc     | 22 ++++-
 .../xla/tests/client_library_test_base.h      |  6 +-
 tensorflow/compiler/xla/tests/tuple_test.cc   | 99 +++++++++----------
 7 files changed, 115 insertions(+), 56 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index fcaf393b6b..7d39701b10 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -491,11 +491,40 @@ XlaOp XlaBuilder::Select(const XlaOp& pred, const XlaOp& on_true,
 }
 
 XlaOp XlaBuilder::Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    std::vector<const Shape*> operand_shape_ptrs;
+    std::vector<Shape> operand_shapes;
+    for (const XlaOp& e : elements) {
+      TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(e));
+      operand_shapes.push_back(shape);
+    }
+    c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
+                [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferVariadicOpShape(
+                            HloOpcode::kTuple, operand_shape_ptrs));
+    return AddInstruction(std::move(instr), HloOpcode::kTuple, elements);
+  }());
 }
 
 XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& tuple_shape, GetShape(tuple_data));
+    if (!ShapeUtil::IsTuple(tuple_shape)) {
+      return InvalidArgument(
+          "Operand to GetTupleElement() is not a tuple; got %s",
+          ShapeUtil::HumanString(tuple_shape).c_str());
+    }
+    *instr.mutable_shape() =
+        ShapeUtil::GetTupleElementShape(tuple_shape, index);
+
+    instr.set_tuple_index(index);
+
+    return AddInstruction(std::move(instr), HloOpcode::kGetTupleElement,
+                          {tuple_data});
+  }());
 }
 
 XlaOp XlaBuilder::Eq(const XlaOp& lhs, const XlaOp& rhs,
diff --git a/tensorflow/compiler/xla/service/shape_inference.cc b/tensorflow/compiler/xla/service/shape_inference.cc
index 36456d552d..77e12d3602 100644
--- a/tensorflow/compiler/xla/service/shape_inference.cc
+++ b/tensorflow/compiler/xla/service/shape_inference.cc
@@ -1070,6 +1070,12 @@ ShapeInference::InferDegenerateDimensionBroadcastShape(
   for (const HloInstruction* operand : operands) {
     operand_shapes.push_back(&operand->shape());
   }
+  return InferVariadicOpShape(opcode, operand_shapes);
+}
+
+/* static */ StatusOr<Shape> ShapeInference::InferVariadicOpShape(
+    HloOpcode opcode,
+    tensorflow::gtl::ArraySlice<const Shape*> operand_shapes) {
   return InferVariadicOpShape(OpcodeToVariadicOperation(opcode),
                               operand_shapes);
 }
diff --git a/tensorflow/compiler/xla/service/shape_inference.h b/tensorflow/compiler/xla/service/shape_inference.h
index 88830e6d25..9da2c99b41 100644
--- a/tensorflow/compiler/xla/service/shape_inference.h
+++ b/tensorflow/compiler/xla/service/shape_inference.h
@@ -83,6 +83,9 @@ class ShapeInference {
   static StatusOr<Shape> InferVariadicOpShape(
       VariadicOperation operation,
       tensorflow::gtl::ArraySlice<const Shape*> operand_shapes);
+  static StatusOr<Shape> InferVariadicOpShape(
+      HloOpcode opcode,
+      tensorflow::gtl::ArraySlice<const Shape*> operand_shapes);
   static StatusOr<Shape> InferVariadicOpShape(
       HloOpcode opcode,
       tensorflow::gtl::ArraySlice<const HloInstruction*> operands);
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 5ab25f2264..2fd97fa38e 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1011,6 +1011,8 @@ xla_test(
         "//tensorflow/compiler/xla/client:computation",
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:hlo_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index ec95a68ead..4a9faef1dc 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -441,8 +441,9 @@ void ClientLibraryTestBase::ComputeAndCompareR1U8(
   EXPECT_EQ(expected, actual->GetR1U8AsString());
 }
 
+template <typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareTuple(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments) {
   auto actual_status = ExecuteAndTransfer(builder, arguments);
   EXPECT_IS_OK(actual_status.status());
@@ -453,8 +454,9 @@ void ClientLibraryTestBase::ComputeAndCompareTuple(
   LiteralTestUtil::ExpectEqual(expected, *actual);
 }
 
+template <typename BuilderT>
 void ClientLibraryTestBase::ComputeAndCompareTuple(
-    ComputationBuilder* builder, const Literal& expected,
+    BuilderT* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error) {
   auto actual_status = ExecuteAndTransfer(builder, arguments);
   EXPECT_IS_OK(actual_status.status());
@@ -619,4 +621,20 @@ template void ClientLibraryTestBase::ComputeAndCompareLiteral(
     tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error,
     const Shape* shape_with_layout);
 
+template void ClientLibraryTestBase::ComputeAndCompareTuple(
+    ComputationBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments);
+
+template void ClientLibraryTestBase::ComputeAndCompareTuple(
+    XlaBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments);
+
+template void ClientLibraryTestBase::ComputeAndCompareTuple(
+    ComputationBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error);
+
+template void ClientLibraryTestBase::ComputeAndCompareTuple(
+    XlaBuilder* builder, const Literal& expected,
+    tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error);
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index 5ff200be03..be90f14c8e 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -217,11 +217,13 @@ class ClientLibraryTestBase : public ::testing::Test {
 
   // Convenience method for running a built computation, transferring the
   // result, and comparing it to the expected tuple literal.
+  template <typename BuilderT>
   void ComputeAndCompareTuple(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments);
+  template <typename BuilderT>
   void ComputeAndCompareTuple(
-      ComputationBuilder* builder, const Literal& expected,
+      BuilderT* builder, const Literal& expected,
       tensorflow::gtl::ArraySlice<GlobalData*> arguments, ErrorSpec error);
 
   // Convenience method for running a built computation and comparing the result
diff --git a/tensorflow/compiler/xla/tests/tuple_test.cc b/tensorflow/compiler/xla/tests/tuple_test.cc
index fa60af4b6a..098be6d7aa 100644
--- a/tensorflow/compiler/xla/tests/tuple_test.cc
+++ b/tensorflow/compiler/xla/tests/tuple_test.cc
@@ -20,6 +20,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation.h"
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -41,7 +43,7 @@ class TupleTest : public ClientLibraryTestBase {
 
 // Tests a tuple-shaped constant.
 XLA_TEST_F(TupleTest, TupleConstant) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   const float constant_scalar = 7.3f;
   std::initializer_list<float> constant_vector = {1.1f, 2.0f, 3.3f};
@@ -54,13 +56,13 @@ XLA_TEST_F(TupleTest, TupleConstant) {
                           Literal::CreateR1<float>(constant_vector).get(),
                           Literal::CreateR2<float>(constant_matrix).get()});
 
-  auto result = builder.ConstantLiteral(*value);
+  builder.ConstantLiteral(*value);
   ComputeAndCompareTuple(&builder, *value, {}, error_spec_);
 }
 
 // Tests a tuple made of scalar constants.
 XLA_TEST_F(TupleTest, TupleScalarConstant) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   const float constant_scalar1 = 7.3f;
   const float constant_scalar2 = 1.2f;
@@ -68,13 +70,13 @@ XLA_TEST_F(TupleTest, TupleScalarConstant) {
       Literal::MakeTuple({Literal::CreateR0<float>(constant_scalar1).get(),
                           Literal::CreateR0<float>(constant_scalar2).get()});
 
-  auto result = builder.ConstantLiteral(*value);
+  builder.ConstantLiteral(*value);
   ComputeAndCompareTuple(&builder, *value, {}, error_spec_);
 }
 
 // Tests the creation of tuple data.
 XLA_TEST_F(TupleTest, TupleCreate) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   const float constant_scalar = 7.3f;
   std::initializer_list<float> constant_vector = {1.1f, 2.0f, 3.3f};
@@ -82,9 +84,9 @@ XLA_TEST_F(TupleTest, TupleCreate) {
       {1.1f, 2.2f, 3.5f},  // row 0
       {4.8f, 5.0f, 6.7f},  // row 1
   };
-  auto result = builder.Tuple({builder.ConstantR0<float>(constant_scalar),
-                               builder.ConstantR1<float>(constant_vector),
-                               builder.ConstantR2<float>(constant_matrix)});
+  builder.Tuple({builder.ConstantR0<float>(constant_scalar),
+                 builder.ConstantR1<float>(constant_vector),
+                 builder.ConstantR2<float>(constant_matrix)});
 
   auto expected =
       Literal::MakeTuple({Literal::CreateR0<float>(constant_scalar).get(),
@@ -95,9 +97,9 @@ XLA_TEST_F(TupleTest, TupleCreate) {
 
 // Tests the creation of tuple data.
 XLA_TEST_F(TupleTest, TupleCreateWithZeroElementEntry) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
-  auto result = builder.Tuple(
+  builder.Tuple(
       {builder.ConstantR0<float>(7.0), builder.ConstantR1<float>({})});
 
   auto expected = Literal::MakeTuple({Literal::CreateR0<float>(7.0).get(),
@@ -107,15 +109,15 @@ XLA_TEST_F(TupleTest, TupleCreateWithZeroElementEntry) {
 
 // Tests the creation of an empty tuple.
 XLA_TEST_F(TupleTest, EmptyTupleCreate) {
-  ComputationBuilder builder(client_, TestName());
-  auto result = builder.Tuple({});
+  XlaBuilder builder(TestName());
+  builder.Tuple({});
   auto expected = Literal::MakeTuple({});
   ComputeAndCompareTuple(&builder, *expected, {}, error_spec_);
 }
 
 // Trivial test for extracting a tuple element with GetTupleElement.
 XLA_TEST_F(TupleTest, GetTupleElement) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::initializer_list<float> constant_vector = {1.f, 2.f, 3.f};
   std::initializer_list<std::initializer_list<float>> constant_matrix = {
       {1.f, 2.f, 3.f},  // row 0
@@ -123,23 +125,23 @@ XLA_TEST_F(TupleTest, GetTupleElement) {
   };
   auto tuple_data = builder.Tuple({builder.ConstantR1<float>(constant_vector),
                                    builder.ConstantR2<float>(constant_matrix)});
-  auto matrix_element = builder.GetTupleElement(tuple_data, 1);
+  builder.GetTupleElement(tuple_data, 1);
   ComputeAndCompareR2<float>(&builder, Array2D<float>(constant_matrix), {},
                              error_spec_);
 }
 
 // Trivial test for extracting a tuple element with GetTupleElement.
 XLA_TEST_F(TupleTest, GetTupleElementWithZeroElements) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto tuple_data = builder.Tuple(
       {builder.ConstantR1<float>({}),
        builder.ConstantR2FromArray2D<float>(Array2D<float>(0, 101))});
-  auto matrix_element = builder.GetTupleElement(tuple_data, 1);
+  builder.GetTupleElement(tuple_data, 1);
   ComputeAndCompareR2<float>(&builder, Array2D<float>(0, 101), {}, error_spec_);
 }
 
 XLA_TEST_F(TupleTest, GetTupleElementOfNonTupleFailsGracefully) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto value = builder.ConstantR1<float>({4.5f});
   builder.GetTupleElement(value, 1);
   auto result_status = builder.Build();
@@ -152,7 +154,7 @@ XLA_TEST_F(TupleTest, GetTupleElementOfNonTupleFailsGracefully) {
 // Extracts both elements from a tuple with GetTupleElement and then adds them
 // together.
 XLA_TEST_F(TupleTest, AddTupleElements) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::initializer_list<float> constant_vector = {1.f, 2.f, 3.f};
   std::initializer_list<std::initializer_list<float>> constant_matrix = {
       {1.f, 2.f, 3.f},  // row 0
@@ -164,22 +166,22 @@ XLA_TEST_F(TupleTest, AddTupleElements) {
   auto matrix_element = builder.GetTupleElement(tuple_data, 1);
   auto vector_shape = builder.GetShape(vector_element).ConsumeValueOrDie();
   auto matrix_shape = builder.GetShape(matrix_element).ConsumeValueOrDie();
-  auto result = builder.Add(matrix_element, vector_element,
-                            /*broadcast_dimensions=*/{1});
+  builder.Add(matrix_element, vector_element,
+              /*broadcast_dimensions=*/{1});
 
   Array2D<float> expected({
       {2.f, 4.f, 6.f},  // row 0
       {5.f, 7.f, 9.f},  // row 1
   });
-  ASSERT_TRUE(ShapeUtil::ShapeIs(*vector_shape, F32, {3}));
-  ASSERT_TRUE(ShapeUtil::ShapeIs(*matrix_shape, F32, {/*y=*/2, /*x=*/3}));
+  ASSERT_TRUE(ShapeUtil::ShapeIs(vector_shape, F32, {3}));
+  ASSERT_TRUE(ShapeUtil::ShapeIs(matrix_shape, F32, {/*y=*/2, /*x=*/3}));
   ComputeAndCompareR2<float>(&builder, expected, {}, error_spec_);
 }
 
 // Extracts both elements from a tuple and then puts them into a new tuple in
 // the opposite order.
 XLA_TEST_F(TupleTest, TupleGTEToTuple) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::initializer_list<float> constant_vector = {1.f, 2.f, 3.f};
   std::initializer_list<std::initializer_list<float>> constant_matrix = {
       {1.f, 2.f, 3.f},  // row 0
@@ -187,8 +189,8 @@ XLA_TEST_F(TupleTest, TupleGTEToTuple) {
   };
   auto tuple_data = builder.Tuple({builder.ConstantR1<float>(constant_vector),
                                    builder.ConstantR2<float>(constant_matrix)});
-  auto new_tuple = builder.Tuple({builder.GetTupleElement(tuple_data, 1),
-                                  builder.GetTupleElement(tuple_data, 0)});
+  builder.Tuple({builder.GetTupleElement(tuple_data, 1),
+                 builder.GetTupleElement(tuple_data, 0)});
   auto expected =
       Literal::MakeTuple({Literal::CreateR2<float>(constant_matrix).get(),
                           Literal::CreateR1<float>(constant_vector).get()});
@@ -196,8 +198,8 @@ XLA_TEST_F(TupleTest, TupleGTEToTuple) {
 }
 
 XLA_TEST_F(TupleTest, SelectBetweenPredTuples) {
-  ComputationBuilder b(client_, TestName());
-  ComputationDataHandle v1, v2;
+  XlaBuilder b(TestName());
+  XlaOp v1, v2;
 
   for (bool direction : {false, true}) {
     std::unique_ptr<GlobalData> v1_data =
@@ -210,7 +212,7 @@ XLA_TEST_F(TupleTest, SelectBetweenPredTuples) {
     auto v2_gt = b.Gt(v2, v1);             // true
     auto v1_v2 = b.Tuple({v1_gt, v2_gt});  // {false, true}
     auto v2_v1 = b.Tuple({v2_gt, v1_gt});  // {true, false}
-    auto select = b.Select(direction ? v1_gt : v2_gt, v1_v2, v2_v1);
+    b.Select(direction ? v1_gt : v2_gt, v1_v2, v2_v1);
     auto expected =
         Literal::MakeTuple({Literal::CreateR0<bool>(direction).get(),
                             Literal::CreateR0<bool>(!direction).get()});
@@ -237,7 +239,7 @@ XLA_TEST_F(TupleTest, TupleGTEToTupleToGTEAdd) {
   //              \                (tuple10)--                     /
   //               \              /           \                   /
   //                -----(GTE 0)--             --(GTE 1)----------
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   std::initializer_list<float> constant_vector = {1.f, 2.f, 3.f};
   std::initializer_list<std::initializer_list<float>> constant_matrix = {
       {1.f, 2.f, 3.f},  // row 0
@@ -257,8 +259,8 @@ XLA_TEST_F(TupleTest, TupleGTEToTupleToGTEAdd) {
   auto addvectors = builder.Add(vector_from_01, vector_from_10);
   auto addmatrices = builder.Add(matrix_from_01, matrix_from_10);
 
-  auto result = builder.Add(addmatrices, addvectors,
-                            /*broadcast_dimensions=*/{1});
+  builder.Add(addmatrices, addvectors,
+              /*broadcast_dimensions=*/{1});
 
   Array2D<float> expected({
       {4.f, 8.f, 12.f},    // row 0
@@ -269,7 +271,7 @@ XLA_TEST_F(TupleTest, TupleGTEToTupleToGTEAdd) {
 
 XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesOnFalse)) {
   // Tests a selection between tuples with "false" path taken.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::initializer_list<float> vec1 = {1.f, 2.f, 3.f};
   std::initializer_list<float> vec2 = {2.f, 4.f, 6.f};
@@ -278,8 +280,7 @@ XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesOnFalse)) {
   auto tuple21 = builder.Tuple(
       {builder.ConstantR1<float>(vec2), builder.ConstantR1<float>(vec1)});
 
-  auto select =
-      builder.Select(builder.ConstantR0<bool>(false), tuple12, tuple21);
+  builder.Select(builder.ConstantR0<bool>(false), tuple12, tuple21);
   auto expected = Literal::MakeTuple({Literal::CreateR1<float>(vec2).get(),
                                       Literal::CreateR1<float>(vec1).get()});
   ComputeAndCompareTuple(&builder, *expected, {}, error_spec_);
@@ -314,7 +315,7 @@ XLA_TEST_F(TupleTest, TuplesInAMap) {
 
 XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesOnTrue)) {
   // Tests a selection between tuples with "true" path taken.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::initializer_list<float> vec1 = {1.f, 2.f, 3.f};
   std::initializer_list<float> vec2 = {2.f, 4.f, 6.f};
@@ -323,8 +324,7 @@ XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesOnTrue)) {
   auto tuple21 = builder.Tuple(
       {builder.ConstantR1<float>(vec2), builder.ConstantR1<float>(vec1)});
 
-  auto select =
-      builder.Select(builder.ConstantR0<bool>(true), tuple12, tuple21);
+  builder.Select(builder.ConstantR0<bool>(true), tuple12, tuple21);
   auto expected = Literal::MakeTuple({Literal::CreateR1<float>(vec1).get(),
                                       Literal::CreateR1<float>(vec2).get()});
   ComputeAndCompareTuple(&builder, *expected, {}, error_spec_);
@@ -333,7 +333,7 @@ XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesOnTrue)) {
 XLA_TEST_F(TupleTest, SelectBetweenTuplesElementResult) {
   // Tests a selection between tuples but the final result is an element of the
   // tuple, not the whole tuple.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::initializer_list<float> vec1 = {1.f, 2.f, 3.f};
   std::initializer_list<float> vec2 = {2.f, 4.f, 6.f};
@@ -344,7 +344,7 @@ XLA_TEST_F(TupleTest, SelectBetweenTuplesElementResult) {
 
   auto select =
       builder.Select(builder.ConstantR0<bool>(false), tuple12, tuple21);
-  auto element = builder.GetTupleElement(select, 0);
+  builder.GetTupleElement(select, 0);
 
   ComputeAndCompareR1<float>(&builder, vec2, {}, error_spec_);
 }
@@ -368,7 +368,7 @@ XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesCascaded)) {
   //                                /             --(GTE 1)--
   //                               /
   //                          (tuple 21)
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::initializer_list<float> vec1 = {1.f, 2.f, 3.f};
   std::initializer_list<float> vec2 = {2.f, 4.f, 6.f};
@@ -384,8 +384,8 @@ XLA_TEST_F(TupleTest, DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesCascaded)) {
       builder.Select(builder.GetTupleElement(pred_tuple, 0), tuple12, tuple21);
   auto select2 =
       builder.Select(builder.GetTupleElement(pred_tuple, 1), tuple21, select1);
-  auto result = builder.Add(builder.GetTupleElement(select2, 0),
-                            builder.GetTupleElement(select2, 1));
+  builder.Add(builder.GetTupleElement(select2, 0),
+              builder.GetTupleElement(select2, 1));
 
   ComputeAndCompareR1<float>(&builder, {3.f, 6.f, 9.f}, {}, error_spec_);
 }
@@ -394,7 +394,7 @@ XLA_TEST_F(TupleTest,
            DISABLED_ON_CPU_PARALLEL(SelectBetweenTuplesReuseConstants)) {
   // Similar to SelectBetweenTuples, but the constants are shared between the
   // input tuples.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   std::initializer_list<float> vec1 = {1.f, 2.f, 3.f};
   std::initializer_list<float> vec2 = {2.f, 4.f, 6.f};
@@ -403,19 +403,18 @@ XLA_TEST_F(TupleTest,
   auto tuple12 = builder.Tuple({c1, c2});
   auto tuple21 = builder.Tuple({c2, c1});
 
-  auto select =
-      builder.Select(builder.ConstantR0<bool>(false), tuple12, tuple21);
+  builder.Select(builder.ConstantR0<bool>(false), tuple12, tuple21);
+
   auto expected = Literal::MakeTuple({Literal::CreateR1<float>(vec2).get(),
                                       Literal::CreateR1<float>(vec1).get()});
   ComputeAndCompareTuple(&builder, *expected, {}, error_spec_);
 }
 
 XLA_TEST_F(TupleTest, NestedTuples) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto inner_tuple = builder.Tuple(
       {builder.ConstantR1<float>({1.0, 2.0}), builder.ConstantR0<float>(42.0)});
-  auto outer_tuple =
-      builder.Tuple({inner_tuple, builder.ConstantR1<float>({22.0, 44.0})});
+  builder.Tuple({inner_tuple, builder.ConstantR1<float>({22.0, 44.0})});
 
   auto expected_v1 = Literal::CreateR1<float>({1.0, 2.0});
   auto expected_s = Literal::CreateR0<float>(42.0);
@@ -429,7 +428,7 @@ XLA_TEST_F(TupleTest, NestedTuples) {
 }
 
 XLA_TEST_F(TupleTest, GetTupleElementOfNestedTuple) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   Shape data_shape = ShapeUtil::MakeShape(F32, {3});
   Shape inner_tuple_shape = ShapeUtil::MakeTupleShape({data_shape, data_shape});
@@ -460,7 +459,7 @@ XLA_TEST_F(TupleTest, GetTupleElementOfNestedTuple) {
 }
 
 XLA_TEST_F(TupleTest, ComplexTuples) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   {
     Shape c64r0 = ShapeUtil::MakeShape(C64, {});
     Shape c64r1 = ShapeUtil::MakeShape(C64, {2});
-- 
GitLab


From 3551d41e36c569a200d3cbaeb1074b4165fb8f0a Mon Sep 17 00:00:00 2001
From: Nick Felt <nfelt@users.noreply.github.com>
Date: Tue, 27 Mar 2018 15:35:37 -0700
Subject: [PATCH 583/960] Update tensorboard dependency to 1.7.0+

---
 tensorflow/tools/pip_package/setup.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 7a3184d64d..8b83257887 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -39,7 +39,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.13.3',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorboard >= 1.6.0, < 1.7.0',
+    'tensorboard >= 1.7.0, < 1.8.0',
     'termcolor >= 1.1.0',
 ]
 
-- 
GitLab


From 3dc861e754ffb86286038ef9c78327f59384eaad Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 17:13:22 -0700
Subject: [PATCH 584/960] K-FAC: Bugfixes for TPU compatibility with covariance
 update ops.

PiperOrigin-RevId: 190699635
---
 .../contrib/kfac/python/ops/fisher_factors.py    | 16 ++++++++++++----
 1 file changed, 12 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/ops/fisher_factors.py b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
index 353e1c6abb..0d40d265a1 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_factors.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_factors.py
@@ -336,12 +336,16 @@ class FisherFactor(object):
 
     new_cov = math_ops.add_n(new_cov_contribs) / float(self._num_towers)
 
-    # I have no idea if the TPU code below is still correct since I don't know
-    # what it actually does.  Also, this code is not present in some of the
-    # other versions of make_covariance_update_op.  Does it matter?
-    # Synchronize value across all TPU cores.
+    # Compute average of 'new_cov' across all TPU cores. On a TPU, each
+    # instance of 'new_cov' will be based on a different minibatch. This ensures
+    # that by the end of assign_moving_average(), all TPU cores see the same
+    # value for self._cov.
+    #
+    # Other implementations of make_covariance_update_op() that accumulate
+    # statistics in other variables should mimic this behavior.
     if utils.on_tpu():
       new_cov = utils.cross_replica_mean(new_cov)
+
     return moving_averages.assign_moving_average(
         self._cov, new_cov, ema_decay, zero_debias=ZERO_DEBIAS)
 
@@ -1398,6 +1402,10 @@ class FullyConnectedMultiKF(FullyConnectedKroneckerFactor):
       new_cov_dt1 = (math_ops.add_n(new_cov_dt1_contribs)
                      / float(self._num_towers))
 
+      # See comments in FisherFactor.make_covariance_update_op() for details.
+      if utils.on_tpu():
+        new_cov_dt1 = utils.cross_replica_mean(new_cov_dt1)
+
       op2 = moving_averages.assign_moving_average(
           self._cov_dt1, new_cov_dt1, ema_decay, zero_debias=ZERO_DEBIAS)
 
-- 
GitLab


From 52271c7c2ae3905bb557ebf355a4eca44a930c1f Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 27 Mar 2018 17:14:50 -0700
Subject: [PATCH 585/960] Make tf.keras.Sequential (properly) Checkpointable

Just numbers Layers like "layer-N". It may also make sense to track them by
"ClassName-M", but that's a backwards-compatible change.

Special-cases all of the dependency collection, since Layers can be added and
removed from Sequential.

PiperOrigin-RevId: 190699818
---
 .../eager/python/checkpointable_utils_test.py | 33 +++++++++++++++++++
 .../keras/_impl/keras/engine/sequential.py    | 31 +++++++++++++++++
 tensorflow/python/training/checkpointable.py  |  1 +
 3 files changed, 65 insertions(+)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index a8c47d76d1..5e1b64728a 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -30,6 +30,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
+from tensorflow.python.keras._impl.keras.engine import sequential
 from tensorflow.python.keras._impl.keras.engine import training
 from tensorflow.python.layers import core
 from tensorflow.python.ops import control_flow_ops
@@ -1036,6 +1037,38 @@ class CheckpointingTests(test.TestCase):
         beta1_power, _ = optimizer._get_beta_accumulators()
         self.assertAllEqual(3., self.evaluate(beta1_power))
 
+  @test_util.run_in_graph_and_eager_modes()
+  def test_sequential(self):
+    model = sequential.Sequential()
+    checkpoint = checkpointable_utils.Checkpoint(model=model)
+    model.add(core.Dense(4))
+    second_dense = core.Dense(5)
+    model.add(second_dense)
+    model(constant_op.constant([[1.]]))
+    checkpoint.restore(None).initialize_or_restore()
+    self.evaluate(second_dense.bias.assign(
+        constant_op.constant([1., 2., 3., 4., 5.])))
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    save_path = checkpoint.save(checkpoint_prefix)
+    self.evaluate(second_dense.bias.assign(
+        constant_op.constant([5., 6., 7., 8., 9.])))
+    checkpoint.restore(save_path).assert_consumed().run_restore_ops()
+    self.assertAllEqual([1., 2., 3., 4., 5.], self.evaluate(second_dense.bias))
+
+    deferred_sequential = sequential.Sequential()
+    deferred_sequential_checkpoint = checkpointable_utils.Checkpoint(
+        model=deferred_sequential)
+    status = deferred_sequential_checkpoint.restore(save_path)
+    deferred_sequential.add(core.Dense(4))
+    deferred_sequential(constant_op.constant([[1.]]))
+    deferred_second_dense = core.Dense(5)
+    deferred_sequential.add(deferred_second_dense)
+    deferred_sequential(constant_op.constant([[1.]]))
+    status.run_restore_ops()
+    self.assertAllEqual([1., 2., 3., 4., 5.],
+                        self.evaluate(deferred_second_dense.bias))
+
 
 class TemplateTests(test.TestCase):
 
diff --git a/tensorflow/python/keras/_impl/keras/engine/sequential.py b/tensorflow/python/keras/_impl/keras/engine/sequential.py
index 66cef1f5b9..2ef99d5ab3 100644
--- a/tensorflow/python/keras/_impl/keras/engine/sequential.py
+++ b/tensorflow/python/keras/_impl/keras/engine/sequential.py
@@ -29,6 +29,7 @@ from tensorflow.python.keras._impl.keras.engine.input_layer import Input
 from tensorflow.python.keras._impl.keras.engine.input_layer import InputLayer
 from tensorflow.python.keras._impl.keras.engine.training import Model
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import checkpointable
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -192,6 +193,36 @@ class Sequential(Model):
       self.build()
     else:
       self._layers.append(layer)
+    # In implementing Checkpointable, Sequential does not track its Layers
+    # normally, since they may be added and removed (in pop()). Instead, it
+    # names everything on demand (gathering dependencies in
+    # _checkpoint_dependencies, and looking them up in
+    # _lookup_dependency). _handle_deferred_dependencies just checks whether an
+    # existing checkpoint load targets this Layer, it does not create a
+    # dependency on the Layer.
+    self._handle_deferred_dependencies(
+        name='layer-%d' % (len(self._layers) - 1), checkpointable=layer)
+
+  @property
+  def _checkpoint_dependencies(self):
+    """For implementing Checkpointable. Layers which should be saved."""
+    return super(Sequential, self)._checkpoint_dependencies + [
+        checkpointable.CheckpointableReference(
+            name='layer-%d' % layer_index, ref=layer)
+        for layer_index, layer in enumerate(self._layers)]
+
+  def _lookup_dependency(self, name):
+    """For implementing Checkpointable. Looks up a Layer."""
+    super_lookup = super(Sequential, self)._lookup_dependency(name=name)
+    if super_lookup is not None:
+      return super_lookup
+    if name.startswith('layer-'):
+      try:
+        return self._layers[int(name[6:])]
+      except IndexError:
+        return None
+    else:
+      return None
 
   def pop(self):
     """Removes the last layer in the model.
diff --git a/tensorflow/python/training/checkpointable.py b/tensorflow/python/training/checkpointable.py
index d0650eb127..bbbe1e8ac5 100644
--- a/tensorflow/python/training/checkpointable.py
+++ b/tensorflow/python/training/checkpointable.py
@@ -560,6 +560,7 @@ class CheckpointableBase(object):
       checkpointable: The Checkpointable object to restore (inheriting from
         `CheckpointableBase`).
     """
+    self._maybe_initialize_checkpointable()
     deferred_dependencies_list = self._deferred_dependencies.pop(name, ())
     for checkpoint_position in sorted(
         deferred_dependencies_list,
-- 
GitLab


From e4b367cef5f89d6741dac223c91a11fda9ca63ae Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 17:15:14 -0700
Subject: [PATCH 586/960] Make slot_creator use DistributionStrategy for
 co-locating variables. Make DistributionStrategy.colocate_vars_with() match
 the existing behavior of ops.colocate_with() by default, for compatibility.

PiperOrigin-RevId: 190699882
---
 tensorflow/python/training/distribute.py   | 16 ++++++++--------
 tensorflow/python/training/slot_creator.py |  8 +++++---
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 9261e13230..757ba71c4a 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -557,9 +557,14 @@ class DistributionStrategy(object):
     raise NotImplementedError("must be implemented in descendants")
 
   def colocate_vars_with(self, colocate_with_variable):
-    """Controls which devices variables will be created on.
+    """Scope that controls which devices variables will be created on.
 
-    Note this may only be used inside `self.scope()`.
+    No operations should be added to the graph inside this scope, it
+    should only be used when creating variables (some implementations
+    work by changing variable creation, others work by using a
+    tf.colocate_with() scope).
+
+    This may only be used inside `self.scope()`.
 
     Example usage:
 
@@ -1032,13 +1037,8 @@ class _DefaultDistributionStrategy(DistributionStrategy):
 
   def colocate_vars_with(self, colocate_with_variable):
     """Does not require `self.scope`."""
-    def create_colocated_variable(next_creator, *args, **kwargs):
-      _require_distribution_strategy_scope(self)
-      with ops.colocate_with(colocate_with_variable):
-        return next_creator(*args, **kwargs)
-
     _require_distribution_strategy_scope(self)
-    return variable_scope.variable_creator_scope(create_colocated_variable)
+    return ops.colocate_with(colocate_with_variable)
 
   def distribute_dataset(self, dataset):
     # TODO(josh11b): Support for this when executing eagerly is currently only
diff --git a/tensorflow/python/training/slot_creator.py b/tensorflow/python/training/slot_creator.py
index 9ac52dd071..258a6f045d 100644
--- a/tensorflow/python/training/slot_creator.py
+++ b/tensorflow/python/training/slot_creator.py
@@ -40,12 +40,12 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.eager import context
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
+from tensorflow.python.training import distribute as distribute_lib
 
 
 def _create_slot_var(primary, val, scope, validate_shape, shape, dtype):
@@ -112,7 +112,8 @@ def create_slot(primary, val, name, colocate_with_primary=True):
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      with ops.colocate_with(primary):
+      distribution_strategy = distribute_lib.get_distribution_strategy()
+      with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, val, "", validate_shape, None, None)
     else:
       return _create_slot_var(primary, val, "", validate_shape, None, None)
@@ -148,7 +149,8 @@ def create_slot_with_initializer(primary, initializer, shape, dtype, name,
     prefix = primary.op.name
   with variable_scope.variable_scope(None, prefix + "/" + name):
     if colocate_with_primary:
-      with ops.colocate_with(primary):
+      distribution_strategy = distribute_lib.get_distribution_strategy()
+      with distribution_strategy.colocate_vars_with(primary):
         return _create_slot_var(primary, initializer, "", validate_shape, shape,
                                 dtype)
     else:
-- 
GitLab


From 50e1888fa89bce621e988a92ede3dc362e37b248 Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Tue, 27 Mar 2018 17:16:31 -0700
Subject: [PATCH 587/960] [XLA] Assert that all buffers and sub-buffers passed
 to XLA have an explicit pointer.

In the past, we allowed sub-buffers to be null if the top-level tuple
was non-null.

This doesn't actually work well on the GPU: For ops that are implemented
using cudnn or cublas, we have to have a pointer to the sub-buffer on
the host in order to make the call.  Retrieving it from the GPU in an
efficient manner is complicated, and the best we can come up with isn't
all that efficient (fundamentally having to pull data down from the GPU
blocks the ability of the CPU to "run ahead" of the GPU).

Since TF wasn't making use of our flexibility *anyway*, we add the
requirement that XLA be given non-null pointers to all sub-buffers.

Changes to the XLA:GPU backend to take advantage of this will come
separately.

PiperOrigin-RevId: 190700021
---
 .../xla/service/gpu/gpu_executable.cc         | 24 ++++++++++++-------
 1 file changed, 15 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
index 04b37d913e..28f9344795 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.cc
@@ -267,16 +267,22 @@ StatusOr<std::unique_ptr<ShapedBuffer>> GpuExecutable::ExecuteOnStream(
        ++i) {
     const BufferAllocation& allocation = assignment_->GetAllocation(i);
     if (allocation.is_entry_computation_parameter()) {
-      // The caller must give us a buffer for ShapeIndex {} of every parameter.
-      // It can optionally give us a buffer for other ShapeIndices, but we
-      // ignore them: Because we can't rely on these sub-buffers' addresses
-      // being available, our generated code can't use them.  Instead, it must
-      // chase pointers starting at the tuple root.
-      if (allocation.param_shape_index().empty()) {
-        auto param_no = allocation.parameter_number();
-        buffer_allocations_builder.RegisterBuffer(
-            i, arguments[param_no]->root_buffer());
+      auto param_no = allocation.parameter_number();
+      se::DeviceMemoryBase buffer =
+          arguments[param_no]->buffer(allocation.param_shape_index());
+
+      // All top-level buffers and sub-buffers must have an explicit, non-null
+      // pointer, except for zero-sized buffers, which may be null.
+      if (buffer.is_null() && buffer.size() > 0) {
+        return FailedPrecondition(
+            "Cannot run XLA computation because pointer to (sub-)buffer at "
+            "index %s of parameter %lld was null.  All pointers to "
+            "(sub-)buffers must not be null, unless the (sub-)buffer has zero "
+            "elements.",
+            allocation.param_shape_index().ToString().c_str(), param_no);
       }
+
+      buffer_allocations_builder.RegisterBuffer(i, buffer);
     }
   }
   se::StreamExecutor* executor = run_options->stream()->parent();
-- 
GitLab


From ec3123979cd622a70fc809ee98b5a7b35772f369 Mon Sep 17 00:00:00 2001
From: David Majnemer <majnemer@google.com>
Date: Tue, 27 Mar 2018 17:35:14 -0700
Subject: [PATCH 588/960] [XLA] Accurately measure FLOPs for base-dilated
 convolutions

We incorrectly counted FLOPs when the output and kernel line up to access the
padding or the dilated area. These should not be accounted as contributing to
the FLOP count.

PiperOrigin-RevId: 190702384
---
 tensorflow/compiler/xla/service/BUILD         |   1 +
 .../compiler/xla/service/hlo_cost_analysis.cc | 108 +++++++++++++++---
 .../xla/service/hlo_cost_analysis_test.cc     |  37 +++++-
 3 files changed, 129 insertions(+), 17 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index da16976d06..f0bf68ae57 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -1580,6 +1580,7 @@ cc_library(
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:window_util",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
index 4ec2ef27bf..44e4f75f75 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/util.h"
+#include "tensorflow/compiler/xla/window_util.h"
 #include "tensorflow/core/lib/core/bits.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
@@ -379,20 +380,101 @@ Status HloCostAnalysis::HandleTranspose(const HloInstruction*) {
 }
 
 Status HloCostAnalysis::HandleConvolution(const HloInstruction* convolution) {
-  auto rhs_instruction = convolution->operand(1);
+  auto lhs = convolution->operand(0);
+  auto rhs = convolution->operand(1);
+  Window window = convolution->window();
+  const auto& result_shape = convolution->shape();
+  const Shape& lhs_shape = lhs->shape();
+  const Shape& rhs_shape = rhs->shape();
+
   const auto& dnums = convolution->convolution_dimension_numbers();
-  const int64 output_features =
-      convolution->shape().dimensions(dnums.output_feature_dimension());
-
-  // For each output element, we do one fma per element in the kernel at some
-  // given output feature index.
-  const int64 fmas_per_output_element =
-      output_features > 0
-          ? ShapeUtil::ElementsIn(rhs_instruction->shape()) / output_features
-          : 0;
-  const int64 output_elements = ShapeUtil::ElementsIn(convolution->shape());
-  current_properties_[kFlopsKey] =
-      output_elements * fmas_per_output_element * kFmaFlops;
+
+  const int64 input_batch_dim = dnums.input_batch_dimension();
+  const int64 input_feature_dim = dnums.input_feature_dimension();
+  const int64 output_feature_dim = dnums.output_feature_dimension();
+  const int64 input_feature =
+      ShapeUtil::GetDimension(lhs_shape, input_feature_dim);
+  const int64 output_feature =
+      ShapeUtil::GetDimension(result_shape, output_feature_dim);
+  const int64 batch = ShapeUtil::GetDimension(lhs_shape, input_batch_dim);
+
+  DimensionVector kernel_limits;
+  DimensionVector output_limits;
+  DimensionVector input_limits;
+  if (window.dimensions().empty()) {
+    window = window_util::MakeWindow({1});
+    kernel_limits.push_back(1);
+    output_limits.push_back(1);
+    input_limits.push_back(1);
+  } else {
+    for (int64 spatial_dimension = 0;
+         spatial_dimension < window.dimensions_size(); ++spatial_dimension) {
+      // Spatial dimension number for kernel (rhs).
+      const int64 kernel_spatial_dim =
+          dnums.kernel_spatial_dimensions(spatial_dimension);
+      const int64 kernel_limit = rhs_shape.dimensions(kernel_spatial_dim);
+      kernel_limits.push_back(kernel_limit);
+
+      // Spatial dimension number for output.
+      const int64 output_spatial_dim =
+          dnums.output_spatial_dimensions(spatial_dimension);
+      const int64 output_limit = result_shape.dimensions(output_spatial_dim);
+      output_limits.push_back(output_limit);
+
+      // Spatial dimension number for input (lhs).
+      const int64 input_spatial_dim =
+          dnums.input_spatial_dimensions(spatial_dimension);
+      const int64 input_limit = lhs_shape.dimensions(input_spatial_dim);
+      input_limits.push_back(input_limit);
+    }
+  }
+
+  DimensionVector valid_position_counts;
+
+  // Loop over each spatial dimension.
+  for (int64 spatial_dimension = 0;
+       spatial_dimension < window.dimensions_size(); ++spatial_dimension) {
+    int64 valid_position_count = 0;
+    // Loop over each point in the kernel.
+    for (int64 kernel_idx = 0; kernel_idx < kernel_limits[spatial_dimension];
+         ++kernel_idx) {
+      // Loop over each point in the output.
+      for (int64 output_idx = 0; output_idx < output_limits[spatial_dimension];
+           ++output_idx) {
+        // Calculate lhs (input) index without taking base dilation into
+        // account.
+        const auto& window_dim = window.dimensions(spatial_dimension);
+        const int64 undilated_index = output_idx * window_dim.stride() -
+                                      window_dim.padding_low() +
+                                      kernel_idx * window_dim.window_dilation();
+
+        // Calculate the actual lhs (input) index after dilation. Avoid the
+        // division as an optimization.
+        const int64 lhs_spatial_index =
+            window_dim.base_dilation() > 1
+                ? undilated_index / window_dim.base_dilation()
+                : undilated_index;
+
+        // Skip if the lhs (input) index is to be dilated.
+        if (undilated_index != lhs_spatial_index * window_dim.base_dilation()) {
+          continue;
+        }
+
+        // Skip if input index is not in bound.
+        if (lhs_spatial_index < 0 ||
+            lhs_spatial_index >= input_limits[spatial_dimension]) {
+          continue;
+        }
+
+        valid_position_count += 1;
+      }
+    }
+    valid_position_counts.push_back(valid_position_count);
+  }
+
+  const int64 fma_count =
+      input_feature * output_feature * batch * Product(valid_position_counts);
+  current_properties_[kFlopsKey] = fma_count * kFmaFlops;
   return Status::OK();
 }
 
diff --git a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
index 3b289c240a..3d055b327e 100644
--- a/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_cost_analysis_test.cc
@@ -186,12 +186,14 @@ TEST_F(HloCostAnalysisTest, Map) {
 TEST_F(HloCostAnalysisTest, Convolution) {
   ComputationBuilder builder(client_, "convolution");
   auto input = builder.Parameter(
-      0, ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/10,
-                                    /*x_dim=*/20}),
+      0,
+      ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/10,
+                                 /*x_dim=*/20}),
       "input");
   auto kernel = builder.Parameter(
-      1, ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/3,
-                                    /*x_dim=*/3}),
+      1,
+      ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/3,
+                                 /*x_dim=*/3}),
       "kernel");
   auto result = builder.Conv(input, kernel, {1, 1}, Padding::kValid);
 
@@ -440,5 +442,32 @@ TEST_F(HloCostAnalysisTest, TupleCost) {
   EXPECT_EQ(analysis.bytes_accessed(), kPointerSize * 2);
 }
 
+TEST_F(HloCostAnalysisTest, BaseDilatedConvolution) {
+  ComputationBuilder builder(client_, "BaseDilatedConvolution");
+  auto input = builder.Parameter(
+      0,
+      ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/10,
+                                 /*x_dim=*/20}),
+      "input");
+  auto kernel = builder.Parameter(
+      1,
+      ShapeUtil::MakeShape(F32, {/*p_dim=*/1, /*z_dim=*/1, /*y_dim=*/3,
+                                 /*x_dim=*/3}),
+      "kernel");
+
+  auto result = builder.ConvGeneralDilated(
+      input, kernel, /*window_strides=*/{1, 1}, /*padding=*/{{1, 1}, {1, 1}},
+      /*lhs_dilation=*/{3, 5}, /*rhs_dilation=*/{7, 11},
+      ComputationBuilder::CreateDefaultConvDimensionNumbers(2));
+
+  // Run HLO cost analysis.
+  auto hlo_module = BuildHloGraph(&builder);
+  HloCostAnalysis analysis(ShapeSize);
+  ASSERT_IS_OK(
+      hlo_module->entry_computation()->root_instruction()->Accept(&analysis));
+
+  EXPECT_EQ(analysis.flop_count(), 1472);
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From e7ce062d5f1144b0ab5b45a43a5e86d118941bf9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 17:35:55 -0700
Subject: [PATCH 589/960] Support GatherV2 (using Gather)

PiperOrigin-RevId: 190702442
---
 tensorflow/contrib/lite/testing/BUILD                    | 2 +-
 tensorflow/contrib/lite/testing/generate_examples.py     | 3 +--
 .../contrib/lite/testing/generated_examples_zip_test.cc  | 5 ++++-
 tensorflow/contrib/lite/toco/import_tensorflow.cc        | 9 ++++++---
 4 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index dc9492f5e2..555ea90034 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -29,7 +29,7 @@ gen_zipped_test_files(
         "exp.zip",
         "fully_connected.zip",
         "fused_batch_norm.zip",
-        # "gather.zip",  #TODO(b/76437794)
+        "gather.zip",
         "global_batch_norm.zip",
         "l2_pool.zip",
         "l2norm.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index e4ef17585f..cb5c500136 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -874,12 +874,11 @@ def make_gather_tests(zip_path):
       # TODO(mgubin): add string tests when they are supported by Toco.
       # TODO(mgubin): add tests for Nd indices when they are supported by
       # TfLite.
-      # TODO(mgubin): add tests for axis != 0 when it is supported by TfLite.
       "params_dtype": [tf.float32, tf.int32],
       "params_shape": [[10], [1, 2, 20]],
       "indices_dtype": [tf.int32],
       "indices_shape": [[3], [5]],
-      "axis": [0],  # axis!=0 is GatherV2
+      "axis": [0, 1],
   }]
 
   def build_graph(parameters):
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 08354b762c..a4a7283508 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -91,6 +91,9 @@ std::map<string, string> kBrokenTests = {
 
     // PRelu only supports 4D input with (1, 1, channels) 3D alpha now.
     {R"(^\/prelu.*shared_axes=\[1\])", "75975192"},
+
+    // No support for axis!=0 in GatherV2.
+    {R"(^\/gather.*axis=1)", "76910444"},
 };
 
 // Allows test data to be unzipped into a temporary directory and makes
@@ -244,7 +247,7 @@ INSTANTIATE_TESTS(div)
 INSTANTIATE_TESTS(exp)
 INSTANTIATE_TESTS(fully_connected)
 INSTANTIATE_TESTS(fused_batch_norm)
-// INSTANTIATE_TESTS(gather)  //TODO(b/76437794)
+INSTANTIATE_TESTS(gather)
 INSTANTIATE_TESTS(global_batch_norm)
 INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index b844e0b948..c26e4bddff 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -1343,13 +1343,16 @@ void ConvertFloorOperator(const NodeDef& node,
 void ConvertGatherOperator(const NodeDef& node,
                            const TensorFlowImportFlags& tf_import_flags,
                            Model* model) {
-  CHECK_EQ(node.op(), "Gather");
-  CheckInputsCount(node, tf_import_flags, 2);
+  CHECK(node.op() == "Gather" || node.op() == "GatherV2");
+  if (node.op() == "Gather") CheckInputsCount(node, tf_import_flags, 2);
+  if (node.op() == "GatherV2") CheckInputsCount(node, tf_import_flags, 3);
   const auto indices_data_type = GetDataTypeAttr(node, "Tindices");
   CHECK(indices_data_type == DT_INT32 || indices_data_type == DT_INT64);
   auto* op = new GatherOperator;
   op->inputs.push_back(node.input(0));
   op->inputs.push_back(node.input(1));
+  // TODO(ahentz): we currently ignore the third tensor in GatherV2 but we
+  // should read it an pass it on to the TF Lite Interpreter.
   op->outputs.push_back(node.name());
   model->operators.emplace_back(op);
 }
@@ -2119,7 +2122,7 @@ std::unique_ptr<Model> ImportTensorFlowGraphDef(
       ConvertCastOperator(node, tf_import_flags, model);
     } else if (node.op() == "Floor") {
       ConvertFloorOperator(node, tf_import_flags, model);
-    } else if (node.op() == "Gather") {
+    } else if (node.op() == "Gather" || node.op() == "GatherV2") {
       ConvertGatherOperator(node, tf_import_flags, model);
     } else if (node.op() == "ResizeBilinear") {
       ConvertResizeBilinearOperator(node, tf_import_flags, model);
-- 
GitLab


From 4979e1d55783d05520fda56fd89641f817daf119 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 27 Mar 2018 18:06:30 -0700
Subject: [PATCH 590/960] Disable new Gather/Slice estimators for now to fix
 the crashes during some TF graphs optimizations.

PiperOrigin-RevId: 190705686
---
 .../grappler/costs/op_level_cost_estimator.cc |  3 +
 .../costs/op_level_cost_estimator_test.cc     | 61 ++++++++++---------
 2 files changed, 35 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index fdbc61f3f1..905cc2a215 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -171,9 +171,12 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
 
+      // TODO(76227186): re-enable with output size check & test
+      /*
       {kGather, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
       {kGatherV2, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
       {kSlice, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
+      */
 
       {kPlaceholder, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kIdentity, wrap(&OpLevelCostEstimator::PredictIdentity)},
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index f2a9615dfb..99bf28f21b 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -195,40 +195,43 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
   OpLevelCostEstimator estimator_;
 };
 
+// TODO(76227186): re-enable with output size check & test
+/*
 TEST_F(OpLevelCostEstimatorTest, TestGatherCosts) {
-  OpContext op_context;
-  SetCpuDevice(&op_context.op_info);
-  op_context.op_info.set_op("Gather");
-
-  // Huge first input shouldn't affect Gather execution and memory costs.
-  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
-  DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
-  DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
-
-  auto cost = estimator_.PredictCosts(op_context);
-  EXPECT_EQ(Costs::Duration(130), cost.memory_time);
-  EXPECT_EQ(Costs::Duration(16), cost.compute_time);
-  EXPECT_EQ(Costs::Duration(146), cost.execution_time);
-  EXPECT_FALSE(cost.inaccurate);
+OpContext op_context;
+SetCpuDevice(&op_context.op_info);
+op_context.op_info.set_op("Gather");
+
+// Huge first input shouldn't affect Gather execution and memory costs.
+DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
+DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
+
+auto cost = estimator_.PredictCosts(op_context);
+EXPECT_EQ(Costs::Duration(130), cost.memory_time);
+EXPECT_EQ(Costs::Duration(16), cost.compute_time);
+EXPECT_EQ(Costs::Duration(146), cost.execution_time);
+EXPECT_FALSE(cost.inaccurate);
 }
 
 TEST_F(OpLevelCostEstimatorTest, TestSliceCosts) {
-  OpContext op_context;
-  SetCpuDevice(&op_context.op_info);
-  op_context.op_info.set_op("Slice");
-
-  // Huge first input shouldn't affect Slice execution and memory costs.
-  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
-  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
-  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
-  DescribeArbitraryRankOutput({10, 10}, DT_FLOAT, &op_context.op_info);
-
-  auto cost = estimator_.PredictCosts(op_context);
-  EXPECT_EQ(Costs::Duration(81), cost.memory_time);
-  EXPECT_EQ(Costs::Duration(10), cost.compute_time);
-  EXPECT_EQ(Costs::Duration(91), cost.execution_time);
-  EXPECT_FALSE(cost.inaccurate);
+OpContext op_context;
+SetCpuDevice(&op_context.op_info);
+op_context.op_info.set_op("Slice");
+
+// Huge first input shouldn't affect Slice execution and memory costs.
+DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+DescribeArbitraryRankOutput({10, 10}, DT_FLOAT, &op_context.op_info);
+
+auto cost = estimator_.PredictCosts(op_context);
+EXPECT_EQ(Costs::Duration(81), cost.memory_time);
+EXPECT_EQ(Costs::Duration(10), cost.compute_time);
+EXPECT_EQ(Costs::Duration(91), cost.execution_time);
+EXPECT_FALSE(cost.inaccurate);
 }
+*/
 
 TEST_F(OpLevelCostEstimatorTest, BiasAddExecutionTime) {
   auto cost = PredictCosts(DescribeBiasAdd(1000, 10));
-- 
GitLab


From 94b2d2db576a6cce878aee92d6b1f90ded4278b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 18:09:37 -0700
Subject: [PATCH 591/960] [XLA] Remove CheckShape and CheckSameShape in
 ComputationBuilder, they are not/rarely used.

PiperOrigin-RevId: 190706088
---
 .../xla/client/computation_builder.cc         | 20 --------------
 .../compiler/xla/client/computation_builder.h |  9 -------
 .../xla/tests/batch_normalization_test.cc     | 26 ++++++++++++++-----
 3 files changed, 20 insertions(+), 35 deletions(-)

diff --git a/tensorflow/compiler/xla/client/computation_builder.cc b/tensorflow/compiler/xla/client/computation_builder.cc
index 39d02f0863..4d3b0ee0d6 100644
--- a/tensorflow/compiler/xla/client/computation_builder.cc
+++ b/tensorflow/compiler/xla/client/computation_builder.cc
@@ -253,26 +253,6 @@ StatusOr<ProgramShape> ComputationBuilder::GetProgramShape() {
   return std::move(*response.mutable_program_shape());
 }
 
-ComputationDataHandle ComputationBuilder::CheckShape(
-    const ComputationDataHandle& operand, const Shape& expected_shape) {
-  std::unique_ptr<Shape> actual_shape = GetShape(operand).ConsumeValueOrDie();
-  CHECK(ShapeUtil::Equal(expected_shape, *actual_shape))
-      << "want " << ShapeUtil::HumanString(expected_shape) << " got "
-      << ShapeUtil::HumanString(*actual_shape);
-  return operand;
-}
-
-void ComputationBuilder::CheckSameShape(const ComputationDataHandle& lhs,
-                                        const ComputationDataHandle& rhs) {
-  std::unique_ptr<Shape> lhs_shape = GetShape(lhs).ConsumeValueOrDie();
-  std::unique_ptr<Shape> rhs_shape = GetShape(rhs).ConsumeValueOrDie();
-  VLOG(2) << "checking " << ShapeUtil::HumanString(*lhs_shape) << " equals "
-          << ShapeUtil::HumanString(*rhs_shape);
-  CHECK(ShapeUtil::Equal(*lhs_shape, *rhs_shape))
-      << "lhs " << ShapeUtil::HumanString(*lhs_shape) << " rhs "
-      << ShapeUtil::HumanString(*rhs_shape);
-}
-
 ComputationDataHandle ComputationBuilder::Slice(
     const ComputationDataHandle& operand,
     tensorflow::gtl::ArraySlice<int64> start_indices,
diff --git a/tensorflow/compiler/xla/client/computation_builder.h b/tensorflow/compiler/xla/client/computation_builder.h
index 2141ebc206..019c6f3afb 100644
--- a/tensorflow/compiler/xla/client/computation_builder.h
+++ b/tensorflow/compiler/xla/client/computation_builder.h
@@ -104,15 +104,6 @@ class ComputationBuilder {
   // Retrieves the (inferred) result for the current computation's shape.
   StatusOr<ProgramShape> GetProgramShape();
 
-  // Checks that the operand has the given expected shape. Returns the operand
-  // if yes, fails with a CHECK error if no.
-  ComputationDataHandle CheckShape(const ComputationDataHandle& operand,
-                                   const Shape& expected_shape);
-
-  // Checks that the lhs and rhs results have the same shape.
-  void CheckSameShape(const ComputationDataHandle& lhs,
-                      const ComputationDataHandle& rhs);
-
   // Enqueues a constant with the value of the given literal onto the
   // computation.
   ComputationDataHandle ConstantLiteral(const Literal& literal);
diff --git a/tensorflow/compiler/xla/tests/batch_normalization_test.cc b/tensorflow/compiler/xla/tests/batch_normalization_test.cc
index 28ab965499..af8af99c79 100644
--- a/tensorflow/compiler/xla/tests/batch_normalization_test.cc
+++ b/tensorflow/compiler/xla/tests/batch_normalization_test.cc
@@ -69,6 +69,17 @@ class BatchNormalizationTest
     CHECK_EQ(kY, input_array_.width());
   }
 
+  ComputationDataHandle CheckShape(ComputationBuilder* b,
+                                   const ComputationDataHandle& operand,
+                                   const Shape& expected_shape) const {
+    std::unique_ptr<Shape> actual_shape =
+        b->GetShape(operand).ConsumeValueOrDie();
+    CHECK(ShapeUtil::Equal(expected_shape, *actual_shape))
+        << "want " << ShapeUtil::HumanString(expected_shape) << " got "
+        << ShapeUtil::HumanString(*actual_shape);
+    return operand;
+  }
+
   static constexpr int64 kSamples = 3;
   static constexpr int64 kX = 1;
   static constexpr int64 kY = 1;
@@ -164,14 +175,15 @@ XLA_TEST_P(BatchNormalizationTest, VarianceToStddev) {
 XLA_TEST_P(BatchNormalizationTest, SpecComparisonForward) {
   ComputationBuilder builder(client_, "batch_normalize_per_spec");
   auto input_activations =
-      builder.CheckShape(builder.ConstantLiteral(input_literal_),
-                         ShapeUtil::MakeShape(F32, {3, 2, 1, 1}));
+      CheckShape(&builder, builder.ConstantLiteral(input_literal_),
+                 ShapeUtil::MakeShape(F32, {3, 2, 1, 1}));
   auto gamma = builder.ConstantR1<float>({1.0, 1.0});
   auto beta = builder.ConstantR1<float>({0.0, 0.0});
   Computation add = CreateScalarAddComputation(F32, &builder);
   // Reduce all dimensions except dimension 1.
   Shape TwoElementVectorF32 = ShapeUtil::MakeShape(F32, {2});
-  auto sum = builder.CheckShape(
+  auto sum = CheckShape(
+      &builder,
       builder.Reduce(input_activations, builder.ConstantR0<float>(0.0f), add,
                      /*dimensions_to_reduce=*/{0, 2, 3}),
       TwoElementVectorF32);
@@ -187,14 +199,16 @@ XLA_TEST_P(BatchNormalizationTest, SpecComparisonForward) {
   auto activation_deviations = builder.Sub(input_activations, set_means,
                                            /*broadcast_dimensions=*/{1});
   auto dev_squares = builder.SquareF32(activation_deviations);
-  auto sum_of_squares = builder.CheckShape(
+  auto sum_of_squares = CheckShape(
+      &builder,
       builder.Reduce(dev_squares, builder.ConstantR0<float>(0.0f), add,
                      /*dimensions_to_reduce=*/{0, 2, 3}),
       TwoElementVectorF32);
   auto variance = builder.Div(sum_of_squares, count);
   auto standard_deviation = builder.SqrtF32(variance);
-  auto standard_deviation_above_epsilon = builder.CheckShape(
-      builder.Gt(standard_deviation, epsilon), ShapeUtil::MakeShape(PRED, {2}));
+  auto standard_deviation_above_epsilon =
+      CheckShape(&builder, builder.Gt(standard_deviation, epsilon),
+                 ShapeUtil::MakeShape(PRED, {2}));
   auto gt_eps = builder.Select(standard_deviation_above_epsilon,
                                standard_deviation, epsilon2);
   auto normalization_factors = builder.ReciprocalF32(gt_eps);
-- 
GitLab


From 9d0b5ba5cfdf0f1c35a228abac9c84c11bb7d695 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 27 Mar 2018 18:18:33 -0700
Subject: [PATCH 592/960] Pass options to TFE_ContextOptionsSetAsync

PiperOrigin-RevId: 190707017
---
 tensorflow/python/eager/context.py   | 2 +-
 tensorflow/python/eager/core_test.py | 8 ++++++++
 2 files changed, 9 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 6c9a14730c..8c1bb06bc3 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -232,7 +232,7 @@ class Context(object):
             pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy(
                 opts, self._device_policy)
           if self._execution_mode == ASYNC:
-            pywrap_tensorflow.TFE_ContextOptionsSetAsync(True)
+            pywrap_tensorflow.TFE_ContextOptionsSetAsync(opts, True)
           self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
       finally:
         pywrap_tensorflow.TFE_DeleteContextOptions(opts)
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 6ebf5b2481..5f19f64846 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -97,6 +97,14 @@ class TFETest(test_util.TensorFlowTestCase):
     self.assertTrue(has_cpu_device)
     del ctx
 
+  def testAsyncBasic(self):
+    ctx = context.Context(execution_mode=context.ASYNC)
+    has_cpu_device = False
+    for x in ctx.devices():
+      has_cpu_device = has_cpu_device or 'CPU' in x
+    self.assertTrue(has_cpu_device)
+    del ctx
+
   def testRunMetadata(self):
     context.enable_run_metadata()
     t = constant_op.constant(1.0)
-- 
GitLab


From 9b5411a13c2d983e6709e6dff4f82b1779389ece Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Tue, 27 Mar 2018 18:31:55 -0700
Subject: [PATCH 593/960] Fix problem with HandleElementwiseUnary/Binary in
 DfsHloVisitorWithDefault.

DfsHloVisitorWithDefault incorrectly included some overrides for handling
several elementwise binary and unary opcodes. These overrides explicitly
called DefaultAction which meant that these opcodes were not handled by
HandleElementwiseUnary/Binary. This CL removes these overrides and adds a
comment describing the potential problem. Unfortunately, I don't see a way
of automatically catching these issues when new opcodes are added, so the
comment will have to do.

PiperOrigin-RevId: 190708245
---
 tensorflow/compiler/xla/service/BUILD         | 17 ++++
 .../service/dfs_hlo_visitor_with_default.h    | 15 ++--
 .../dfs_hlo_visitor_with_default_test.cc      | 90 +++++++++++++++++++
 3 files changed, 113 insertions(+), 9 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index f0bf68ae57..bde749d317 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -285,6 +285,23 @@ cc_library(
     ],
 )
 
+tf_cc_test(
+    name = "dfs_hlo_visitor_with_default_test",
+    srcs = ["dfs_hlo_visitor_with_default_test.cc"],
+    deps = [
+        ":hlo",
+        ":hlo_runner",
+        "//tensorflow/compiler/xla:shape_util",
+        "//tensorflow/compiler/xla:test",
+        "//tensorflow/compiler/xla:test_helpers",
+        "//tensorflow/compiler/xla:util",
+        "//tensorflow/compiler/xla:xla_data_proto",
+        "//tensorflow/compiler/xla/tests:hlo_test_base",
+        "//tensorflow/compiler/xla/tests:xla_internal_test_main",
+        "//tensorflow/core:test",
+    ],
+)
+
 cc_library(
     name = "hlo_reachability",
     srcs = ["hlo_reachability.cc"],
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
index ecda5288ee..240faebe62 100644
--- a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h
@@ -35,6 +35,12 @@ class HloInstruction;
 // DfsHloVisitor with default action based on the HloInstruction being visited.
 // Users should not use this class directly, but use the type aliases
 // DfsHloVisitorWithDefault/ConstDfsHloVisitorWithDefault instead.
+//
+// Do *not* add an override to this class if the opcode is covered by
+// HandleElementwiseUnary/Binary. These opcode handlers dispatch to
+// HandleElementwiseUnary/Binary in DfsHloVisitorBase. Adding such a handler
+// here will break passes which rely on the HandleElementwiseUnary/Binary
+// handling these opcodes.
 template <typename HloInstructionPtr>
 class DfsHloVisitorWithDefaultBase
     : public DfsHloVisitorBase<HloInstructionPtr> {
@@ -70,12 +76,6 @@ class DfsHloVisitorWithDefaultBase
   Status HandleConcatenate(HloInstructionPtr concatenate) override {
     return DefaultAction(concatenate);
   }
-  Status HandleConvert(HloInstructionPtr convert) override {
-    return DefaultAction(convert);
-  }
-  Status HandleCopy(HloInstructionPtr copy) override {
-    return DefaultAction(copy);
-  }
   Status HandleSelect(HloInstructionPtr select) override {
     return DefaultAction(select);
   }
@@ -91,9 +91,6 @@ class DfsHloVisitorWithDefaultBase
   Status HandleCrossReplicaSum(HloInstructionPtr crs) override {
     return DefaultAction(crs);
   }
-  Status HandleCompare(HloInstructionPtr compare) override {
-    return DefaultAction(compare);
-  }
   Status HandleRng(HloInstructionPtr random) override {
     return DefaultAction(random);
   }
diff --git a/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc
new file mode 100644
index 0000000000..825e1436f0
--- /dev/null
+++ b/tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default_test.cc
@@ -0,0 +1,90 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/xla/service/dfs_hlo_visitor_with_default.h"
+
+#include "tensorflow/compiler/xla/service/hlo_computation.h"
+#include "tensorflow/compiler/xla/service/hlo_instruction.h"
+#include "tensorflow/compiler/xla/service/hlo_module.h"
+#include "tensorflow/compiler/xla/service/hlo_opcode.h"
+#include "tensorflow/compiler/xla/service/hlo_runner.h"
+#include "tensorflow/compiler/xla/shape_util.h"
+#include "tensorflow/compiler/xla/test.h"
+#include "tensorflow/compiler/xla/test_helpers.h"
+#include "tensorflow/compiler/xla/tests/hlo_test_base.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+
+namespace xla {
+namespace {
+
+class DfsHloVisitorWithDefaultTest : public HloTestBase {};
+
+TEST_F(DfsHloVisitorWithDefaultTest, DefaultElementwiseTest) {
+  // Verify that HandleElementwiseBinary and HandleElementwiseUnary are called
+  // on the appropriate HLO ops (elementwise binary/unary ops).
+
+  class ElementwiseTestVisitor : public DfsHloVisitorWithDefault {
+   public:
+    Status DefaultAction(HloInstruction* hlo) override {
+      // The HLO should be neither an elementwise unary nor binary op. These
+      // cases are handled in HandleElementwiseBinary/Unary.
+      TF_RET_CHECK(!(hlo->IsElementwise() && hlo->operand_count() == 2))
+          << hlo->ToString();
+      TF_RET_CHECK(!(hlo->IsElementwise() && hlo->operand_count() == 1))
+          << hlo->ToString();
+      return Status::OK();
+    }
+
+    Status HandleElementwiseBinary(HloInstruction* hlo) override {
+      // HLO should be elementwise binary.
+      TF_RET_CHECK(hlo->IsElementwise() && hlo->operand_count() == 2)
+          << hlo->ToString();
+      return Status::OK();
+    }
+    Status HandleElementwiseUnary(HloInstruction* hlo) override {
+      // HLO should be elementwise unary.
+      TF_RET_CHECK(hlo->IsElementwise() && hlo->operand_count() == 1)
+          << hlo->ToString();
+      return Status::OK();
+    }
+  };
+
+  // HLO module contains are arbitrary mix of elementwise and non-elementwise
+  // operations.
+  const string& hlo_string = R"(
+HloModule TestModule
+
+ENTRY TestComputation {
+  arg = f32[] parameter(0)
+  tuple = (f32[]) tuple(arg)
+  gte = f32[] get-tuple-element(tuple), index=0
+  abs = f32[] abs(arg)
+  add = f32[] add(arg, gte)
+  broadcast = f32[42] broadcast(add), dimensions={}
+  slice = f32[0] slice(broadcast), slice={[1:2]}
+  copy = f32[] copy(arg)
+  eq = pred[] equal-to(arg, gte)
+  neg = f32[] negate(arg)
+  ROOT convert = f64[] convert(f32[] arg)
+})";
+  std::unique_ptr<HloModule> module =
+      HloRunner::CreateModuleFromString(hlo_string, GetDebugOptionsForTest())
+          .ConsumeValueOrDie();
+  ElementwiseTestVisitor visitor;
+  TF_EXPECT_OK(module->entry_computation()->Accept(&visitor));
+}
+
+}  // namespace
+}  // namespace xla
-- 
GitLab


From f656b7f3e07fc3a6a51cb6083d27abebcc6212bb Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 27 Mar 2018 19:24:40 -0700
Subject: [PATCH 594/960] Fixed the interaction between virtual cluster and
 measuring cost estimator.

PiperOrigin-RevId: 190712404
---
 .../costs/measuring_cost_estimator.cc         | 23 ++++++++++++++++---
 tensorflow/core/grappler/costs/utils.cc       |  4 ++--
 tensorflow/python/grappler/cluster_test.py    | 16 ++++++++-----
 3 files changed, 32 insertions(+), 11 deletions(-)

diff --git a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
index ea4320687a..833205ac6f 100644
--- a/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/measuring_cost_estimator.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <limits>
 
 #include "tensorflow/core/framework/cost_graph.pb.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/costs/robust_stats.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -52,6 +53,8 @@ Status MeasuringCostEstimator::Initialize(const GrapplerItem& item) {
 Status MeasuringCostEstimator::PredictCosts(const GraphDef& optimized_graph,
                                             CostGraphDef* cost_graph,
                                             Costs* costs) const {
+  const bool running_simulation = (cluster_->type() == "virtual");
+
   std::vector<double> times(measurement_steps_);
   BlockingCounter barrier(measurement_steps_);
 
@@ -80,9 +83,23 @@ Status MeasuringCostEstimator::PredictCosts(const GraphDef& optimized_graph,
     }
 
     const Costs::MicroSeconds finish = Env::Default()->NowMicros();
-    const double time = (finish - start).count() * 1e3;
-    times[step] = time;
-
+    if (running_simulation) {
+      // When running simulation, return the estimated runtime, not the time it
+      // takes to run the simulation.
+      double time = 0.0;
+      for (const DeviceStepStats& stepstats :
+           metadata.step_stats().dev_stats()) {
+        for (const NodeExecStats& node_stats : stepstats.node_stats()) {
+          const double completion_time =
+              node_stats.all_end_rel_micros() + node_stats.all_start_micros();
+          time = std::max(time, completion_time * 1e3);
+        }
+      }
+      times[step] = time;
+    } else {
+      const double time = (finish - start).count() * 1e3;
+      times[step] = time;
+    }
     if (cost_graph && (step + 1 == measurement_steps_)) {
       metadata.mutable_cost_graph()->Swap(cost_graph);
     }
diff --git a/tensorflow/core/grappler/costs/utils.cc b/tensorflow/core/grappler/costs/utils.cc
index 076945d5c6..f318e3911c 100644
--- a/tensorflow/core/grappler/costs/utils.cc
+++ b/tensorflow/core/grappler/costs/utils.cc
@@ -212,8 +212,8 @@ DeviceProperties GetDeviceInfo(const string& device_str) {
       CudaGpuId cuda_gpu_id;
       Status s = GpuIdManager::TfToCudaGpuId(tf_gpu_id, &cuda_gpu_id);
       if (!s.ok()) {
-        LOG(ERROR) << s;
-        return unknown;
+        // We are probably running simulation without linking cuda libraries.
+        cuda_gpu_id = CudaGpuId(parsed.id);
       }
       return GetLocalGPUInfo(cuda_gpu_id);
     } else if (parsed.type == "CPU") {
diff --git a/tensorflow/python/grappler/cluster_test.py b/tensorflow/python/grappler/cluster_test.py
index a3c4c2bbeb..26c6f22d34 100644
--- a/tensorflow/python/grappler/cluster_test.py
+++ b/tensorflow/python/grappler/cluster_test.py
@@ -87,9 +87,10 @@ class ClusterTest(test.TestCase):
 
   def testVirtualCluster(self):
     with ops.Graph().as_default() as g:
-      a = random_ops.random_uniform(shape=())
-      b = random_ops.random_uniform(shape=())
-      c = a + b
+      with ops.device('/device:GPU:0'):
+        a = random_ops.random_uniform(shape=[1024, 1024])
+        b = random_ops.random_uniform(shape=[1024, 1024])
+        c = a + b
       train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
       train_op.append(c)
       mg = meta_graph.create_meta_graph_def(graph=g)
@@ -102,10 +103,13 @@ class ClusterTest(test.TestCase):
               'architecture': '7'
           })
       named_device = device_properties_pb2.NamedDevice(
-          properties=device_properties, name='/GPU:0')
-      grappler_cluster = cluster.Cluster(devices=[named_device])
+          properties=device_properties, name='/device:GPU:0')
+      grappler_cluster = cluster.Cluster(
+          disable_detailed_stats=False,
+          disable_timeline=False,
+          devices=[named_device])
       op_perfs, run_time, _ = grappler_cluster.MeasureCosts(grappler_item)
-      self.assertGreater(run_time, 0)
+      self.assertEqual(run_time, 0.000545)
       self.assertEqual(len(op_perfs), 15)
 
       estimated_perf = grappler_cluster.EstimatePerformance(named_device)
-- 
GitLab


From 756dc39c83136cc3518e20993be4382fe77f9013 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 19:42:55 -0700
Subject: [PATCH 595/960] Implement strip assert in DebugStripper.

PiperOrigin-RevId: 190713919
---
 tensorflow/core/grappler/optimizers/BUILD     |  5 +-
 .../grappler/optimizers/debug_stripper.cc     | 19 ++++-
 .../optimizers/debug_stripper_test.cc         | 71 +++++++++++++++++--
 .../grappler/optimizers/meta_optimizer.cc     |  8 +--
 4 files changed, 93 insertions(+), 10 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 601984fcfd..19ff788aba 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -630,7 +630,10 @@ cc_library(
     deps = [
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler:op_types",
+        "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/clusters:cluster",
         "//tensorflow/core/grappler/optimizers:graph_optimizer",
     ],
@@ -638,7 +641,6 @@ cc_library(
 
 tf_cuda_cc_test(
     name = "debug_stripper_test",
-    size = "small",
     srcs = ["debug_stripper_test.cc"],
     deps = [
         ":debug_stripper",
@@ -646,6 +648,7 @@ tf_cuda_cc_test(
         "//tensorflow/core:tensorflow",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
         "//tensorflow/core/grappler:grappler_item",
         "//tensorflow/core/grappler/utils:grappler_test",
     ],
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper.cc b/tensorflow/core/grappler/optimizers/debug_stripper.cc
index 461f1aa2fb..0e058e3435 100644
--- a/tensorflow/core/grappler/optimizers/debug_stripper.cc
+++ b/tensorflow/core/grappler/optimizers/debug_stripper.cc
@@ -14,16 +14,33 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/optimizers/debug_stripper.h"
+
+#include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
+#include "tensorflow/core/grappler/op_types.h"
+#include "tensorflow/core/grappler/utils.h"
 
 namespace tensorflow {
 namespace grappler {
 
 Status DebugStripper::Optimize(Cluster* cluster, const GrapplerItem& item,
                                GraphDef* output) {
-  // TODO(haoliang): Let's remove assertions here.
   *output = item.graph;
+  for (NodeDef& node : *output->mutable_node()) {
+    if (IsAssert(node)) {
+      // Convert this node into a no-op.
+      node.set_op("NoOp");
+      node.clear_attr();
+      // Convert all its inputs into control dependency, which will then
+      // be optimized away by dependency optimizer.
+      for (string& inp : *node.mutable_input()) {
+        if (!IsControlInput(inp)) {
+          inp = AsControlDependency(inp);
+        }
+      }
+    }
+  }
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
index d2cabc0798..aacd55f136 100644
--- a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
+++ b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/debug_stripper.h"
 
 #include "tensorflow/cc/ops/standard_ops.h"
+#include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
@@ -27,16 +28,78 @@ namespace {
 
 class DebugStripperTest : public GrapplerTest {};
 
-// TODO(haoliang): Add tests for different removal operations.
 TEST_F(DebugStripperTest, OutputEqualToInput) {
-  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  auto c = ops::Const(s.WithOpName("c"), 0, {});
+  constexpr char device[] = "/device:CPU:0";
   GrapplerItem item;
-  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  item.graph = test::function::GDef(
+      {test::function::NDef("x", "Placeholder", {}, {{"dtype", DT_FLOAT}},
+                            device),
+       test::function::NDef("y", "XTimesTwo", {"x"}, {}, device),
+       test::function::NDef("z", "Identity", {"y"}, {{"T", DT_FLOAT}}, device)},
+      {});
 
   DebugStripper optimizer;
   GraphDef output;
   TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+  CompareGraphs(item.graph, output);
+}
+
+TEST_F(DebugStripperTest, StripAssertFromGraph) {
+  constexpr char device[] = "/device:CPU:0";
+  GrapplerItem item;
+  item.graph = test::function::GDef(
+      {test::function::NDef("x", "Placeholder", {}, {{"dtype", DT_FLOAT}},
+                            device),
+       test::function::NDef("y", "Placeholder", {}, {{"dtype", DT_FLOAT}},
+                            device),
+       test::function::NDef("GreaterEqual", "GreaterEqual", {"x", "y"},
+                            {{"T", DT_FLOAT}}, device),
+       test::function::NDef("Assert", "Assert", {"GreaterEqual"},
+                            {{"T", DT_FLOAT}}, device),
+       test::function::NDef("z", "Add", {"x", "y", "^Assert"}, {}, device)},
+      {});
+
+  DebugStripper optimizer;
+  GraphDef output;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int count = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "x") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(device, node.device());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "y") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(device, node.device());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "GreaterEqual") {
+      count++;
+      EXPECT_EQ("GreaterEqual", node.op());
+      EXPECT_EQ(device, node.device());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+    } else if (node.name() == "Assert") {
+      count++;
+      EXPECT_EQ("NoOp", node.op());
+      EXPECT_EQ(device, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^GreaterEqual", node.input(0));
+      EXPECT_EQ(0, node.attr_size());
+    } else if (node.name() == "z") {
+      count++;
+      EXPECT_EQ("Add", node.op());
+      EXPECT_EQ(device, node.device());
+      EXPECT_EQ(3, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+      EXPECT_EQ("^Assert", node.input(2));
+    }
+  }
+  EXPECT_EQ(5, count);
 }
 
 }  // namespace
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index 47ec16226b..ad655db727 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -102,6 +102,10 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new FunctionOptimizer(cfg_.function_optimization())));
     }
+    if (cfg_.debug_stripper() == RewriterConfig::ON) {
+      optimizers.push_back(
+          std::unique_ptr<GraphOptimizer>(new DebugStripper()));
+    }
     if (cfg_.constant_folding() != RewriterConfig::OFF) {
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new ConstantFolding(cfg_.constant_folding(), cpu_device_)));
@@ -138,10 +142,6 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       optimizers.push_back(std::unique_ptr<GraphOptimizer>(
           new AutoParallel(cfg_.auto_parallel().num_replicas())));
     }
-    if (cfg_.debug_stripper() == RewriterConfig::ON) {
-      optimizers.push_back(
-          std::unique_ptr<GraphOptimizer>(new DebugStripper()));
-    }
   } else {
     const std::set<string> available_optimizers = {
         "pruning",    "function",      "constfold",  "layout",
-- 
GitLab


From 078793e131ecec0eae3e7a0549eb13a321993dfb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 19:58:53 -0700
Subject: [PATCH 596/960] Fix _force_data_dependency for scalar inputs

PiperOrigin-RevId: 190715033
---
 tensorflow/contrib/layers/python/layers/rev_block_lib.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/layers/python/layers/rev_block_lib.py b/tensorflow/contrib/layers/python/layers/rev_block_lib.py
index 0b38c0c3fd..e49589ddf6 100644
--- a/tensorflow/contrib/layers/python/layers/rev_block_lib.py
+++ b/tensorflow/contrib/layers/python/layers/rev_block_lib.py
@@ -33,6 +33,7 @@ import numpy as np
 from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.contrib.framework.python import ops as contrib_framework_ops
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops as framework_ops
 from tensorflow.python.layers import base
@@ -660,7 +661,9 @@ def _force_data_dependency(first_compute, then_compute):
     if x.get_shape().ndims is None:
       raise ValueError("Rank of Tensor %s must be known" % x)
     ndims = x.get_shape().ndims
-    return array_ops.reshape(array_ops.slice(x, [0] * ndims, [1] * ndims), [])
+    begin = framework_ops.convert_to_tensor([0] * ndims, dtype=dtypes.int32)
+    size = framework_ops.convert_to_tensor([1] * ndims, dtype=dtypes.int32)
+    return array_ops.reshape(array_ops.slice(x, begin, size), [])
 
   first_compute_sum = math_ops.add_n(
       [_first_element(x) for x in first_compute if x is not None])
-- 
GitLab


From 9bc9b63bf8319eb1224d0d3331668f4e0913af54 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 21:03:41 -0700
Subject: [PATCH 597/960] Fix non-uniformity of orthogonal matrices. Add test
 code for this purpose.

PiperOrigin-RevId: 190719729
---
 .../python/kernel_tests/init_ops_test.py      | 24 +++++++++++++++++++
 tensorflow/python/ops/init_ops.py             |  9 ++++---
 2 files changed, 30 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index c1755985ee..36a86a25cc 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -649,6 +649,30 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
               sess.run(outputs_2norm)/(np.sqrt(np.prod(shape))*np.sqrt(3.14)),
               rtol=tol, atol=tol)
 
+  def testNonuniformity(self):
+    value = 0
+    abs_value = 0
+    shape = [3, 3, 10, 10]
+    count = 70
+    tol = 1e-5
+    with self.test_session(use_gpu=True):  # as sess:
+      for i in range(count):
+        x = variable_scope.get_variable("{}".format(i), shape=shape,
+                                        initializer=
+                                        init_ops.convolutional_delta_orthogonal)
+        x.initializer.run()
+        y = x.eval()[1, 1, :, :]
+        determinant = np.linalg.det(y)
+        value += determinant
+        abs_value += np.abs(determinant)
+
+      # Check there is some variation in the signs of the determinants
+      self.assertLess(value, count - tol)
+      self.assertLess(-count + tol, value)
+      # Check all determinants have absolute value 1
+      # Compute the sum of the absolute values of 'count' determinants
+      self.assertAllClose(abs_value, count, rtol=tol, atol=tol)
+
 
 class IdentityInitializerTest(test.TestCase):
 
diff --git a/tensorflow/python/ops/init_ops.py b/tensorflow/python/ops/init_ops.py
index 40ab22951b..9dfe5ffbf4 100644
--- a/tensorflow/python/ops/init_ops.py
+++ b/tensorflow/python/ops/init_ops.py
@@ -532,8 +532,7 @@ class Orthogonal(Initializer):
     q, r = linalg_ops.qr(a, full_matrices=False)
     # Make Q uniform
     d = array_ops.diag_part(r)
-    ph = d / math_ops.abs(d)
-    q *= ph
+    q *= math_ops.sign(d)
     if num_rows < num_cols:
       q = array_ops.matrix_transpose(q)
     return self.gain * array_ops.reshape(q, shape)
@@ -579,7 +578,11 @@ class ConvolutionDeltaOrthogonal(Initializer):
     a = random_ops.random_normal([shape[-1], shape[-1]],
                                  dtype=dtype, seed=self.seed)
     # Compute the qr factorization
-    q, _ = linalg_ops.qr(a, full_matrices=False)
+    q, r = linalg_ops.qr(a, full_matrices=False)
+    # Make Q uniform
+    d = array_ops.diag_part(r)
+    # ph = d / math_ops.abs(d)
+    q *= math_ops.sign(d)
     q = q[:shape[-2], :]
     q *= math_ops.sqrt(math_ops.cast(self.gain, dtype=dtype))
     if len(shape) == 3:
-- 
GitLab


From 0a451b1aa0baaa3f7abbf8d90dfe58193cf1533e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 27 Mar 2018 21:22:54 -0700
Subject: [PATCH 598/960] Speed up statistical_testing_test by consolidating
 sess.run calls.

PiperOrigin-RevId: 190721153
---
 tensorflow/contrib/distributions/BUILD        |  1 +
 .../kernel_tests/statistical_testing_test.py  | 82 +++++++++++++------
 2 files changed, 60 insertions(+), 23 deletions(-)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 1c381cc354..682448b84b 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -486,6 +486,7 @@ cuda_py_test(
         "//third_party/py/numpy",
         "//tensorflow/python:client_testlib",
     ],
+    shard_count = 4,
     tags = [
         "manual",
         "noasan",
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
index 3548ac1807..c0e7bdd259 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
@@ -22,39 +22,75 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import statistical_testing as st
 from tensorflow.python.framework import errors
-from tensorflow.python.ops import check_ops
 from tensorflow.python.platform import test
 
 
 class StatisticalTestingTest(test.TestCase):
 
   def test_dkwm_design_mean_one_sample_soundness(self):
-    numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
+    thresholds = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
-    with self.test_session() as sess:
-      for ff in rates:
-        for fp in rates:
-          sufficient_n = st.min_num_samples_for_dkwm_mean_test(
-              numbers, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-          detectable_d = st.min_discrepancy_of_true_means_detectable_by_dkwm(
-              sufficient_n, 0., 1., false_fail_rate=ff, false_pass_rate=fp)
-          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
+    false_fail_rates, false_pass_rates = np.meshgrid(rates, rates)
+    false_fail_rates = false_fail_rates.flatten().astype(np.float32)
+    false_pass_rates = false_pass_rates.flatten().astype(np.float32)
+
+    detectable_discrepancies = []
+    for false_pass_rate, false_fail_rate in zip(
+        false_pass_rates, false_fail_rates):
+      sufficient_n = st.min_num_samples_for_dkwm_mean_test(
+          thresholds, low=0., high=1., false_fail_rate=false_fail_rate,
+          false_pass_rate=false_pass_rate)
+      detectable_discrepancies.append(
+          st.min_discrepancy_of_true_means_detectable_by_dkwm(
+              sufficient_n, low=0., high=1., false_fail_rate=false_fail_rate,
+              false_pass_rate=false_pass_rate))
+
+    detectable_discrepancies_ = self.evaluate(detectable_discrepancies)
+    for discrepancies, false_pass_rate, false_fail_rate in zip(
+        detectable_discrepancies_, false_pass_rates, false_fail_rates):
+      below_threshold = discrepancies <= thresholds
+      self.assertAllEqual(
+          np.ones_like(below_threshold, np.bool), below_threshold,
+          msg='false_pass_rate({}), false_fail_rate({})'.format(
+              false_pass_rate, false_fail_rate))
 
   def test_dkwm_design_mean_two_sample_soundness(self):
-    numbers = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
+    thresholds = [1e-5, 1e-2, 1.1e-1, 0.9, 1., 1.02, 2., 10., 1e2, 1e5, 1e10]
     rates = [1e-6, 1e-3, 1e-2, 1.1e-1, 0.2, 0.5, 0.7, 1.]
-    with self.test_session() as sess:
-      for ff in rates:
-        for fp in rates:
-          (sufficient_n1,
-           sufficient_n2) = st.min_num_samples_for_dkwm_mean_two_sample_test(
-               numbers, 0., 1., 0., 1.,
-               false_fail_rate=ff, false_pass_rate=fp)
-          d_fn = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample
-          detectable_d = d_fn(
-              sufficient_n1, 0., 1., sufficient_n2, 0., 1.,
-              false_fail_rate=ff, false_pass_rate=fp)
-          sess.run(check_ops.assert_less_equal(detectable_d, numbers))
+    false_fail_rates, false_pass_rates = np.meshgrid(rates, rates)
+    false_fail_rates = false_fail_rates.flatten().astype(np.float32)
+    false_pass_rates = false_pass_rates.flatten().astype(np.float32)
+
+    detectable_discrepancies = []
+    for false_pass_rate, false_fail_rate in zip(
+        false_pass_rates, false_fail_rates):
+      [
+          sufficient_n1,
+          sufficient_n2
+      ] = st.min_num_samples_for_dkwm_mean_two_sample_test(
+          thresholds, low1=0., high1=1., low2=0., high2=1.,
+          false_fail_rate=false_fail_rate,
+          false_pass_rate=false_pass_rate)
+
+      detectable_discrepancies.append(
+          st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample(
+              n1=sufficient_n1,
+              low1=0.,
+              high1=1.,
+              n2=sufficient_n2,
+              low2=0.,
+              high2=1.,
+              false_fail_rate=false_fail_rate,
+              false_pass_rate=false_pass_rate))
+
+    detectable_discrepancies_ = self.evaluate(detectable_discrepancies)
+    for discrepancies, false_pass_rate, false_fail_rate in zip(
+        detectable_discrepancies_, false_pass_rates, false_fail_rates):
+      below_threshold = discrepancies <= thresholds
+      self.assertAllEqual(
+          np.ones_like(below_threshold, np.bool), below_threshold,
+          msg='false_pass_rate({}), false_fail_rate({})'.format(
+              false_pass_rate, false_fail_rate))
 
   def test_true_mean_confidence_interval_by_dkwm_one_sample(self):
     rng = np.random.RandomState(seed=0)
-- 
GitLab


From 6861e22c4cd68f57a408b2b51af834990b7a881a Mon Sep 17 00:00:00 2001
From: Dustin Tran <trandustin@google.com>
Date: Tue, 27 Mar 2018 23:15:08 -0700
Subject: [PATCH 599/960] Have TensorFlow Distributions share name scopes
 across method calls.

PiperOrigin-RevId: 190728742
---
 .../python/kernel_tests/distribution_test.py     | 16 ++++++++++++++++
 .../distributions/python/ops/autoregressive.py   |  2 +-
 .../contrib/distributions/python/ops/binomial.py |  2 +-
 .../contrib/distributions/python/ops/cauchy.py   |  2 +-
 .../contrib/distributions/python/ops/chi2.py     |  4 ++--
 .../distributions/python/ops/deterministic.py    |  2 +-
 .../distributions/python/ops/geometric.py        |  2 +-
 .../contrib/distributions/python/ops/gumbel.py   |  2 +-
 .../distributions/python/ops/half_normal.py      |  2 +-
 .../distributions/python/ops/independent.py      |  2 +-
 .../distributions/python/ops/inverse_gamma.py    |  4 ++--
 .../distributions/python/ops/kumaraswamy.py      |  9 +++++----
 .../contrib/distributions/python/ops/logistic.py |  2 +-
 .../contrib/distributions/python/ops/mixture.py  |  2 +-
 .../python/ops/mixture_same_family.py            |  2 +-
 .../contrib/distributions/python/ops/mvn_diag.py |  4 ++--
 .../python/ops/mvn_diag_plus_low_rank.py         |  2 +-
 .../python/ops/mvn_full_covariance.py            |  2 +-
 .../python/ops/mvn_linear_operator.py            |  2 +-
 .../contrib/distributions/python/ops/mvn_tril.py |  2 +-
 .../python/ops/negative_binomial.py              |  2 +-
 .../python/ops/onehot_categorical.py             |  2 +-
 .../contrib/distributions/python/ops/poisson.py  |  2 +-
 .../python/ops/poisson_lognormal.py              |  2 +-
 .../python/ops/quantized_distribution.py         |  2 +-
 .../python/ops/relaxed_bernoulli.py              |  2 +-
 .../python/ops/relaxed_onehot_categorical.py     |  2 +-
 .../distributions/python/ops/sinh_arcsinh.py     |  3 ++-
 .../python/ops/vector_diffeomixture.py           |  2 +-
 .../python/ops/vector_exponential_diag.py        |  2 +-
 .../ops/vector_exponential_linear_operator.py    |  2 +-
 .../python/ops/vector_sinh_arcsinh_diag.py       |  2 +-
 .../distributions/python/ops/vector_student_t.py |  2 +-
 .../contrib/distributions/python/ops/wishart.py  | 10 +++++-----
 tensorflow/python/ops/distributions/bernoulli.py |  2 +-
 tensorflow/python/ops/distributions/beta.py      |  6 +++---
 .../python/ops/distributions/categorical.py      |  2 +-
 tensorflow/python/ops/distributions/dirichlet.py |  2 +-
 .../ops/distributions/dirichlet_multinomial.py   |  2 +-
 .../python/ops/distributions/distribution.py     | 10 ++++++++--
 .../python/ops/distributions/exponential.py      |  4 ++--
 tensorflow/python/ops/distributions/gamma.py     |  4 ++--
 tensorflow/python/ops/distributions/laplace.py   |  4 ++--
 .../python/ops/distributions/multinomial.py      |  2 +-
 tensorflow/python/ops/distributions/normal.py    |  4 ++--
 tensorflow/python/ops/distributions/student_t.py |  4 ++--
 .../distributions/transformed_distribution.py    |  2 +-
 tensorflow/python/ops/distributions/uniform.py   |  2 +-
 48 files changed, 89 insertions(+), 65 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
index 68e0d9cb82..750fe00d63 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
@@ -190,6 +190,22 @@ class DistributionTest(test.TestCase):
       y = dist._set_sample_static_shape(x, sample_shape)
       self.assertTrue(y.get_shape().ndims is None)
 
+  def testNameScopeWorksCorrectly(self):
+    x = tfd.Normal(loc=0., scale=1., name="x")
+    x_duplicate = tfd.Normal(loc=0., scale=1., name="x")
+    with ops.name_scope("y") as name:
+      y = tfd.Bernoulli(logits=0., name=name)
+    x_sample = x.sample(name="custom_sample")
+    x_sample_duplicate = x.sample(name="custom_sample")
+    x_log_prob = x.log_prob(0., name="custom_log_prob")
+
+    self.assertEqual(x.name, "x")
+    self.assertEqual(x_duplicate.name, "x_1")
+    self.assertEqual(y.name, "y")
+    self.assertTrue(x_sample.name.startswith("x/custom_sample"))
+    self.assertTrue(x_sample_duplicate.name.startswith("x/custom_sample_1"))
+    self.assertTrue(x_log_prob.name.startswith("x/custom_log_prob"))
+
   def testStrWorksCorrectlyScalar(self):
     normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1))
     self.assertEqual(
diff --git a/tensorflow/contrib/distributions/python/ops/autoregressive.py b/tensorflow/contrib/distributions/python/ops/autoregressive.py
index 69f3d57ff0..88ed012784 100644
--- a/tensorflow/contrib/distributions/python/ops/autoregressive.py
+++ b/tensorflow/contrib/distributions/python/ops/autoregressive.py
@@ -145,7 +145,7 @@ class Autoregressive(distribution_lib.Distribution):
       ValueError: if `num_steps < 1`.
     """
     parameters = locals()
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       self._distribution_fn = distribution_fn
       self._sample0 = sample0
       self._distribution0 = (distribution_fn() if sample0 is None
diff --git a/tensorflow/contrib/distributions/python/ops/binomial.py b/tensorflow/contrib/distributions/python/ops/binomial.py
index 6a1bb39ab2..12d1603178 100644
--- a/tensorflow/contrib/distributions/python/ops/binomial.py
+++ b/tensorflow/contrib/distributions/python/ops/binomial.py
@@ -164,7 +164,7 @@ class Binomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]):
+    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
       self._total_count = self._maybe_assert_valid_total_count(
           ops.convert_to_tensor(total_count, name="total_count"),
           validate_args)
diff --git a/tensorflow/contrib/distributions/python/ops/cauchy.py b/tensorflow/contrib/distributions/python/ops/cauchy.py
index 6f5d724a2a..daacfe657f 100644
--- a/tensorflow/contrib/distributions/python/ops/cauchy.py
+++ b/tensorflow/contrib/distributions/python/ops/cauchy.py
@@ -121,7 +121,7 @@ class Cauchy(distribution.Distribution):
       TypeError: if `loc` and `scale` have different `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)]
                                     if validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/chi2.py b/tensorflow/contrib/distributions/python/ops/chi2.py
index e610f469e5..c77c5fd208 100644
--- a/tensorflow/contrib/distributions/python/ops/chi2.py
+++ b/tensorflow/contrib/distributions/python/ops/chi2.py
@@ -88,7 +88,7 @@ class Chi2(gamma.Gamma):
     # not true in the parent class "gamma."  therefore, passing
     # allow_nan_stats=True
     # through to the parent class results in unnecessary asserts.
-    with ops.name_scope(name, values=[df]):
+    with ops.name_scope(name, values=[df]) as name:
       with ops.control_dependencies([
           check_ops.assert_positive(df),
       ] if validate_args else []):
@@ -120,7 +120,7 @@ class Chi2WithAbsDf(Chi2):
                allow_nan_stats=True,
                name="Chi2WithAbsDf"):
     parameters = locals()
-    with ops.name_scope(name, values=[df]):
+    with ops.name_scope(name, values=[df]) as name:
       super(Chi2WithAbsDf, self).__init__(
           df=math_ops.floor(
               math_ops.abs(df, name="abs_df"),
diff --git a/tensorflow/contrib/distributions/python/ops/deterministic.py b/tensorflow/contrib/distributions/python/ops/deterministic.py
index 8049522e9f..a42350430e 100644
--- a/tensorflow/contrib/distributions/python/ops/deterministic.py
+++ b/tensorflow/contrib/distributions/python/ops/deterministic.py
@@ -87,7 +87,7 @@ class _BaseDeterministic(distribution.Distribution):
       ValueError:  If `loc` is a scalar.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, atol, rtol]):
+    with ops.name_scope(name, values=[loc, atol, rtol]) as name:
       loc = ops.convert_to_tensor(loc, name="loc")
       if is_vector and validate_args:
         msg = "Argument loc must be at least rank 1."
diff --git a/tensorflow/contrib/distributions/python/ops/geometric.py b/tensorflow/contrib/distributions/python/ops/geometric.py
index 8f190e48a7..53dd42f4c8 100644
--- a/tensorflow/contrib/distributions/python/ops/geometric.py
+++ b/tensorflow/contrib/distributions/python/ops/geometric.py
@@ -86,7 +86,7 @@ class Geometric(distribution.Distribution):
     """
 
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]):
+    with ops.name_scope(name, values=[logits, probs]) as name:
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits, probs, validate_args=validate_args, name=name)
 
diff --git a/tensorflow/contrib/distributions/python/ops/gumbel.py b/tensorflow/contrib/distributions/python/ops/gumbel.py
index 8d05ad6b80..2c261073ee 100644
--- a/tensorflow/contrib/distributions/python/ops/gumbel.py
+++ b/tensorflow/contrib/distributions/python/ops/gumbel.py
@@ -125,7 +125,7 @@ class _Gumbel(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/half_normal.py b/tensorflow/contrib/distributions/python/ops/half_normal.py
index fc0751a6e0..d0df2befd6 100644
--- a/tensorflow/contrib/distributions/python/ops/half_normal.py
+++ b/tensorflow/contrib/distributions/python/ops/half_normal.py
@@ -106,7 +106,7 @@ class HalfNormal(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[scale]):
+    with ops.name_scope(name, values=[scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._scale = array_ops.identity(scale, name="scale")
diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py
index 7dcb3e3ac4..d7d0b0c78b 100644
--- a/tensorflow/contrib/distributions/python/ops/independent.py
+++ b/tensorflow/contrib/distributions/python/ops/independent.py
@@ -119,7 +119,7 @@ class Independent(distribution_lib.Distribution):
     parameters = locals()
     name = name or "Independent" + distribution.name
     self._distribution = distribution
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       if reinterpreted_batch_ndims is None:
         reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims(
             distribution)
diff --git a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
index 51ac61dcf6..502bd4f493 100644
--- a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
+++ b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
@@ -126,7 +126,7 @@ class InverseGamma(distribution.Distribution):
       TypeError: if `concentration` and `rate` are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]):
+    with ops.name_scope(name, values=[concentration, rate]) as name:
       with ops.control_dependencies([
           check_ops.assert_positive(concentration),
           check_ops.assert_positive(rate),
@@ -281,7 +281,7 @@ class InverseGammaWithSoftplusConcentrationRate(InverseGamma):
                allow_nan_stats=True,
                name="InverseGammaWithSoftplusConcentrationRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]):
+    with ops.name_scope(name, values=[concentration, rate]) as name:
       super(InverseGammaWithSoftplusConcentrationRate, self).__init__(
           concentration=nn.softplus(concentration,
                                     name="softplus_concentration"),
diff --git a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
index 192dede6ff..66682b2ff5 100644
--- a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
+++ b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
@@ -151,10 +151,11 @@ class Kumaraswamy(transformed_distribution.TransformedDistribution):
         more of the statistic's batch members are undefined.
       name: Python `str` name prefixed to Ops created by this class.
     """
-    concentration1 = ops.convert_to_tensor(
-        concentration1, name="concentration1")
-    concentration0 = ops.convert_to_tensor(
-        concentration0, name="concentration0")
+    with ops.name_scope(name, values=[concentration1, concentration0]) as name:
+      concentration1 = ops.convert_to_tensor(
+          concentration1, name="concentration1")
+      concentration0 = ops.convert_to_tensor(
+          concentration0, name="concentration0")
     super(Kumaraswamy, self).__init__(
         distribution=uniform.Uniform(
             low=array_ops.zeros([], dtype=concentration1.dtype),
diff --git a/tensorflow/contrib/distributions/python/ops/logistic.py b/tensorflow/contrib/distributions/python/ops/logistic.py
index 68e6bca5a5..c83b5bc2e3 100644
--- a/tensorflow/contrib/distributions/python/ops/logistic.py
+++ b/tensorflow/contrib/distributions/python/ops/logistic.py
@@ -120,7 +120,7 @@ class Logistic(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/mixture.py b/tensorflow/contrib/distributions/python/ops/mixture.py
index cef6a143fc..2ef294af2e 100644
--- a/tensorflow/contrib/distributions/python/ops/mixture.py
+++ b/tensorflow/contrib/distributions/python/ops/mixture.py
@@ -145,7 +145,7 @@ class Mixture(distribution.Distribution):
           "none of the components provide a static number of ndims")
 
     # Ensure that all batch and event ndims are consistent.
-    with ops.name_scope(name, values=[cat.logits]):
+    with ops.name_scope(name, values=[cat.logits]) as name:
       num_components = cat.event_size
       static_num_components = tensor_util.constant_value(num_components)
       if static_num_components is None:
diff --git a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
index b93bdc5ab4..0b1301e551 100644
--- a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
+++ b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
@@ -131,7 +131,7 @@ class MixtureSameFamily(distribution.Distribution):
         `components_distribution` rightmost batch shape.
     """
     parameters = locals()
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       self._mixture_distribution = mixture_distribution
       self._components_distribution = components_distribution
       self._runtime_assertions = []
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag.py b/tensorflow/contrib/distributions/python/ops/mvn_diag.py
index e862552880..e3236c2db9 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_diag.py
@@ -194,7 +194,7 @@ class MultivariateNormalDiag(
       ValueError: if at most `scale_identity_multiplier` is specified.
     """
     parameters = locals()
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier]):
         # No need to validate_args while making diag_scale.  The returned
@@ -225,7 +225,7 @@ class MultivariateNormalDiagWithSoftplusScale(MultivariateNormalDiag):
                allow_nan_stats=True,
                name="MultivariateNormalDiagWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[scale_diag]):
+    with ops.name_scope(name, values=[scale_diag]) as name:
       super(MultivariateNormalDiagWithSoftplusScale, self).__init__(
           loc=loc,
           scale_diag=nn.softplus(scale_diag),
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
index 413e88f03a..2f6a6f198c 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
@@ -218,7 +218,7 @@ class MultivariateNormalDiagPlusLowRank(
     parameters = locals()
     def _convert_to_tensor(x, name):
       return None if x is None else ops.convert_to_tensor(x, name=name)
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier, scale_perturb_factor,
           scale_perturb_diag]):
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
index 4bea99fbb7..86fcd4db54 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
@@ -159,7 +159,7 @@ class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL):
     parameters = locals()
 
     # Convert the covariance_matrix up to a scale_tril and call MVNTriL.
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[loc, covariance_matrix]):
         if covariance_matrix is None:
           scale_tril = None
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
index a739979289..44c92312c7 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
@@ -176,7 +176,7 @@ class MultivariateNormalLinearOperator(
     if not scale.dtype.is_floating:
       raise TypeError("`scale` parameter must have floating-point dtype.")
 
-    with ops.name_scope(name, values=[loc] + scale.graph_parents):
+    with ops.name_scope(name, values=[loc] + scale.graph_parents) as name:
       # Since expand_dims doesn't preserve constant-ness, we obtain the
       # non-dynamic value if possible.
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
index 6c7dc4ca7a..d6f8b731cb 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
@@ -184,7 +184,7 @@ class MultivariateNormalTriL(
       return None if x is None else ops.convert_to_tensor(x, name=name)
     if loc is None and scale_tril is None:
       raise ValueError("Must specify one or both of `loc`, `scale_tril`.")
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[loc, scale_tril]):
         loc = _convert_to_tensor(loc, name="loc")
         scale_tril = _convert_to_tensor(scale_tril, name="scale_tril")
diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
index 3a58df80da..eeaf9c0a5e 100644
--- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py
+++ b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
@@ -91,7 +91,7 @@ class NegativeBinomial(distribution.Distribution):
     """
 
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]):
+    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits, probs, validate_args=validate_args, name=name)
       with ops.control_dependencies(
diff --git a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
index 46c2cc8b7a..fd5e8b10b0 100644
--- a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
@@ -116,7 +116,7 @@ class OneHotCategorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]):
+    with ops.name_scope(name, values=[logits, probs]) as name:
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           name=name, logits=logits, probs=probs, validate_args=validate_args,
           multidimensional=True)
diff --git a/tensorflow/contrib/distributions/python/ops/poisson.py b/tensorflow/contrib/distributions/python/ops/poisson.py
index 02e97c0a2f..a84aad6fc9 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson.py
@@ -94,7 +94,7 @@ class Poisson(distribution.Distribution):
       TypeError: if `log_rate` is not a float-type.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[rate]):
+    with ops.name_scope(name, values=[rate]) as name:
       if (rate is None) == (log_rate is None):
         raise ValueError("Must specify exactly one of `rate` and `log_rate`.")
       elif log_rate is None:
diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
index 92f2bba182..ea5514d345 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
@@ -256,7 +256,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
         `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       if loc is not None:
         loc = ops.convert_to_tensor(loc, name="loc")
       if scale is not None:
diff --git a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py b/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
index 8aebb79b91..1ef7651d03 100644
--- a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
+++ b/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
@@ -217,7 +217,7 @@ class QuantizedDistribution(distributions.Distribution):
     values = (
         list(distribution.parameters.values()) +
         [low, high])
-    with ops.name_scope(name, values=values):
+    with ops.name_scope(name, values=values) as name:
       self._dist = distribution
 
       if low is not None:
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
index b525809015..00c1ede5c1 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
@@ -166,7 +166,7 @@ class RelaxedBernoulli(transformed_distribution.TransformedDistribution):
       ValueError: If both `probs` and `logits` are passed, or if neither.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs, temperature]):
+    with ops.name_scope(name, values=[logits, probs, temperature]) as name:
       with ops.control_dependencies([check_ops.assert_positive(temperature)]
                                     if validate_args else []):
         self._temperature = array_ops.identity(temperature, name="temperature")
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
index ff33f327c7..b60a4872aa 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
@@ -163,7 +163,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs, temperature]):
+    with ops.name_scope(name, values=[logits, probs, temperature]) as name:
 
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           name=name, logits=logits, probs=probs, validate_args=validate_args,
diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
index 0d8a192691..7a8f5bbfbe 100644
--- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
+++ b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
@@ -134,7 +134,8 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
     """
     parameters = locals()
 
-    with ops.name_scope(name, values=[loc, scale, skewness, tailweight]):
+    with ops.name_scope(name,
+                        values=[loc, scale, skewness, tailweight]) as name:
       loc = ops.convert_to_tensor(loc, name="loc")
       dtype = loc.dtype
       scale = ops.convert_to_tensor(scale, name="scale", dtype=dtype)
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 971d65c4a6..026de97be7 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -396,7 +396,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
       ValueError: if `not distribution.is_scalar_event`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[mix_loc, temperature]):
+    with ops.name_scope(name, values=[mix_loc, temperature]) as name:
       if not scale or len(scale) < 2:
         raise ValueError("Must specify list (or list-like object) of scale "
                          "LinearOperators, one for each component with "
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
index 526fe2d39a..e265b5d0f7 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
@@ -176,7 +176,7 @@ class VectorExponentialDiag(
       ValueError: if at most `scale_identity_multiplier` is specified.
     """
     parameters = locals()
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier]):
         # No need to validate_args while making diag_scale.  The returned
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
index 9d5fd9ac41..89136d6760 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
@@ -181,7 +181,7 @@ class VectorExponentialLinearOperator(
     if not scale.dtype.is_floating:
       raise TypeError("`scale` parameter must have floating-point dtype.")
 
-    with ops.name_scope(name, values=[loc] + scale.graph_parents):
+    with ops.name_scope(name, values=[loc] + scale.graph_parents) as name:
       # Since expand_dims doesn't preserve constant-ness, we obtain the
       # non-dynamic value if possible.
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
index 003c66b941..b8ac7c10ca 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
@@ -169,7 +169,7 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
         name,
         values=[
             loc, scale_diag, scale_identity_multiplier, skewness, tailweight
-        ]):
+        ]) as name:
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
       tailweight = 1. if tailweight is None else tailweight
       has_default_skewness = skewness is None
diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
index 8c67647a61..23a01cc95c 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
@@ -178,7 +178,7 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution):
     parameters = locals()
     graph_parents = [df, loc, scale_identity_multiplier, scale_diag,
                      scale_tril, scale_perturb_factor, scale_perturb_diag]
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=graph_parents):
         # The shape of the _VectorStudentT distribution is governed by the
         # relationship between df.batch_shape and affine.batch_shape. In
diff --git a/tensorflow/contrib/distributions/python/ops/wishart.py b/tensorflow/contrib/distributions/python/ops/wishart.py
index 5a8c94dabf..91453fed5d 100644
--- a/tensorflow/contrib/distributions/python/ops/wishart.py
+++ b/tensorflow/contrib/distributions/python/ops/wishart.py
@@ -109,7 +109,7 @@ class _WishartLinearOperator(distribution.Distribution):
     """
     parameters = locals()
     self._cholesky_input_output_matrices = cholesky_input_output_matrices
-    with ops.name_scope(name) as ns:
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[df, scale_operator]):
         if not scale_operator.dtype.is_floating:
           raise TypeError(
@@ -163,7 +163,7 @@ class _WishartLinearOperator(distribution.Distribution):
         parameters=parameters,
         graph_parents=([self._df, self._dimension] +
                        self._scale_operator.graph_parents),
-        name=ns)
+        name=name)
 
   @property
   def df(self):
@@ -531,7 +531,7 @@ class WishartCholesky(_WishartLinearOperator):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[scale]):
+    with ops.name_scope(name, values=[scale]) as name:
       with ops.name_scope("init", values=[scale]):
         scale = ops.convert_to_tensor(scale)
         if validate_args:
@@ -647,7 +647,7 @@ class WishartFull(_WishartLinearOperator):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name) as ns:
+    with ops.name_scope(name) as name:
       with ops.name_scope("init", values=[scale]):
         scale = ops.convert_to_tensor(scale)
         if validate_args:
@@ -666,5 +666,5 @@ class WishartFull(_WishartLinearOperator):
         cholesky_input_output_matrices=cholesky_input_output_matrices,
         validate_args=validate_args,
         allow_nan_stats=allow_nan_stats,
-        name=ns)
+        name=name)
     self._parameters = parameters
diff --git a/tensorflow/python/ops/distributions/bernoulli.py b/tensorflow/python/ops/distributions/bernoulli.py
index 68aaf3815e..2c9f0e9a32 100644
--- a/tensorflow/python/ops/distributions/bernoulli.py
+++ b/tensorflow/python/ops/distributions/bernoulli.py
@@ -72,7 +72,7 @@ class Bernoulli(distribution.Distribution):
       ValueError: If p and logits are passed, or if neither are passed.
     """
     parameters = locals()
-    with ops.name_scope(name):
+    with ops.name_scope(name) as name:
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits=logits,
           probs=probs,
diff --git a/tensorflow/python/ops/distributions/beta.py b/tensorflow/python/ops/distributions/beta.py
index 469bcadb8e..8beab99bf8 100644
--- a/tensorflow/python/ops/distributions/beta.py
+++ b/tensorflow/python/ops/distributions/beta.py
@@ -151,7 +151,7 @@ class Beta(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration1, concentration0]):
+    with ops.name_scope(name, values=[concentration1, concentration0]) as name:
       self._concentration1 = self._maybe_assert_valid_concentration(
           ops.convert_to_tensor(concentration1, name="concentration1"),
           validate_args)
@@ -323,7 +323,7 @@ class BetaWithSoftplusConcentration(Beta):
                name="BetaWithSoftplusConcentration"):
     parameters = locals()
     with ops.name_scope(name, values=[concentration1,
-                                      concentration0]) as ns:
+                                      concentration0]) as name:
       super(BetaWithSoftplusConcentration, self).__init__(
           concentration1=nn.softplus(concentration1,
                                      name="softplus_concentration1"),
@@ -331,7 +331,7 @@ class BetaWithSoftplusConcentration(Beta):
                                      name="softplus_concentration0"),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats,
-          name=ns)
+          name=name)
     self._parameters = parameters
 
 
diff --git a/tensorflow/python/ops/distributions/categorical.py b/tensorflow/python/ops/distributions/categorical.py
index 9161e3fa9f..66fa9e110c 100644
--- a/tensorflow/python/ops/distributions/categorical.py
+++ b/tensorflow/python/ops/distributions/categorical.py
@@ -183,7 +183,7 @@ class Categorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]):
+    with ops.name_scope(name, values=[logits, probs]) as name:
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits=logits,
           probs=probs,
diff --git a/tensorflow/python/ops/distributions/dirichlet.py b/tensorflow/python/ops/distributions/dirichlet.py
index 25afeec936..eafcd5c78f 100644
--- a/tensorflow/python/ops/distributions/dirichlet.py
+++ b/tensorflow/python/ops/distributions/dirichlet.py
@@ -155,7 +155,7 @@ class Dirichlet(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration]):
+    with ops.name_scope(name, values=[concentration]) as name:
       self._concentration = self._maybe_assert_valid_concentration(
           ops.convert_to_tensor(concentration, name="concentration"),
           validate_args)
diff --git a/tensorflow/python/ops/distributions/dirichlet_multinomial.py b/tensorflow/python/ops/distributions/dirichlet_multinomial.py
index 03a98c56ba..fe0ed7e07d 100644
--- a/tensorflow/python/ops/distributions/dirichlet_multinomial.py
+++ b/tensorflow/python/ops/distributions/dirichlet_multinomial.py
@@ -192,7 +192,7 @@ class DirichletMultinomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, concentration]):
+    with ops.name_scope(name, values=[total_count, concentration]) as name:
       # Broadcasting works because:
       # * The broadcasting convention is to prepend dimensions of size [1], and
       #   we use the last dimension for the distribution, whereas
diff --git a/tensorflow/python/ops/distributions/distribution.py b/tensorflow/python/ops/distributions/distribution.py
index 7c43bf54fc..b6e6a01fc8 100644
--- a/tensorflow/python/ops/distributions/distribution.py
+++ b/tensorflow/python/ops/distributions/distribution.py
@@ -434,13 +434,19 @@ class Distribution(_BaseDistribution):
     for i, t in enumerate(graph_parents):
       if t is None or not tensor_util.is_tensor(t):
         raise ValueError("Graph parent item %d is not a Tensor; %s." % (i, t))
+    if name is None:
+      with ops.name_scope(type(self).__name__) as name:
+        pass
+    # If a name ends with a '/' it is a "name scope" and we use it as-is, after
+    # removing the trailing '/'.
+    name = name[:-1] if (name and name[-1] == "/") else name
     self._dtype = dtype
     self._reparameterization_type = reparameterization_type
     self._allow_nan_stats = allow_nan_stats
     self._validate_args = validate_args
     self._parameters = parameters or {}
     self._graph_parents = graph_parents
-    self._name = name or type(self).__name__
+    self._name = name
 
   @classmethod
   def param_shapes(cls, sample_shape, name="DistributionParamShapes"):
@@ -1157,7 +1163,7 @@ class Distribution(_BaseDistribution):
   @contextlib.contextmanager
   def _name_scope(self, name=None, values=None):
     """Helper function to standardize op scope."""
-    with ops.name_scope(self.name):
+    with ops.name_scope(self.name + "/"):  # use absolute name scope
       with ops.name_scope(name, values=(
           ([] if values is None else values) + self._graph_parents)) as scope:
         yield scope
diff --git a/tensorflow/python/ops/distributions/exponential.py b/tensorflow/python/ops/distributions/exponential.py
index 6345a76d48..cf0e729e1a 100644
--- a/tensorflow/python/ops/distributions/exponential.py
+++ b/tensorflow/python/ops/distributions/exponential.py
@@ -95,7 +95,7 @@ class Exponential(gamma.Gamma):
     # true in the parent class "Gamma."  Therefore, passing
     # allow_nan_stats=True
     # through to the parent class results in unnecessary asserts.
-    with ops.name_scope(name, values=[rate]):
+    with ops.name_scope(name, values=[rate]) as name:
       self._rate = ops.convert_to_tensor(rate, name="rate")
     super(Exponential, self).__init__(
         concentration=array_ops.ones([], dtype=self._rate.dtype),
@@ -144,7 +144,7 @@ class ExponentialWithSoftplusRate(Exponential):
                allow_nan_stats=True,
                name="ExponentialWithSoftplusRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[rate]):
+    with ops.name_scope(name, values=[rate]) as name:
       super(ExponentialWithSoftplusRate, self).__init__(
           rate=nn.softplus(rate, name="softplus_rate"),
           validate_args=validate_args,
diff --git a/tensorflow/python/ops/distributions/gamma.py b/tensorflow/python/ops/distributions/gamma.py
index adb1f4f9a8..d39f7c56d3 100644
--- a/tensorflow/python/ops/distributions/gamma.py
+++ b/tensorflow/python/ops/distributions/gamma.py
@@ -127,7 +127,7 @@ class Gamma(distribution.Distribution):
       TypeError: if `concentration` and `rate` are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]):
+    with ops.name_scope(name, values=[concentration, rate]) as name:
       with ops.control_dependencies([
           check_ops.assert_positive(concentration),
           check_ops.assert_positive(rate),
@@ -262,7 +262,7 @@ class GammaWithSoftplusConcentrationRate(Gamma):
                allow_nan_stats=True,
                name="GammaWithSoftplusConcentrationRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]):
+    with ops.name_scope(name, values=[concentration, rate]) as name:
       super(GammaWithSoftplusConcentrationRate, self).__init__(
           concentration=nn.softplus(concentration,
                                     name="softplus_concentration"),
diff --git a/tensorflow/python/ops/distributions/laplace.py b/tensorflow/python/ops/distributions/laplace.py
index e98ac855c5..3ccfc618d1 100644
--- a/tensorflow/python/ops/distributions/laplace.py
+++ b/tensorflow/python/ops/distributions/laplace.py
@@ -101,7 +101,7 @@ class Laplace(distribution.Distribution):
       TypeError: if `loc` and `scale` are of different dtype.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
@@ -218,7 +218,7 @@ class LaplaceWithSoftplusScale(Laplace):
                allow_nan_stats=True,
                name="LaplaceWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       super(LaplaceWithSoftplusScale, self).__init__(
           loc=loc,
           scale=nn.softplus(scale, name="softplus_scale"),
diff --git a/tensorflow/python/ops/distributions/multinomial.py b/tensorflow/python/ops/distributions/multinomial.py
index 4ae67a009b..ab77f5c1f8 100644
--- a/tensorflow/python/ops/distributions/multinomial.py
+++ b/tensorflow/python/ops/distributions/multinomial.py
@@ -183,7 +183,7 @@ class Multinomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]):
+    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
       self._total_count = ops.convert_to_tensor(total_count, name="total_count")
       if validate_args:
         self._total_count = (
diff --git a/tensorflow/python/ops/distributions/normal.py b/tensorflow/python/ops/distributions/normal.py
index 32e8a49c81..20d4420e91 100644
--- a/tensorflow/python/ops/distributions/normal.py
+++ b/tensorflow/python/ops/distributions/normal.py
@@ -132,7 +132,7 @@ class Normal(distribution.Distribution):
       TypeError: if `loc` and `scale` have different `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]):
+    with ops.name_scope(name, values=[loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
@@ -244,7 +244,7 @@ class NormalWithSoftplusScale(Normal):
                allow_nan_stats=True,
                name="NormalWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[scale]):
+    with ops.name_scope(name, values=[scale]) as name:
       super(NormalWithSoftplusScale, self).__init__(
           loc=loc,
           scale=nn.softplus(scale, name="softplus_scale"),
diff --git a/tensorflow/python/ops/distributions/student_t.py b/tensorflow/python/ops/distributions/student_t.py
index 9d9e65b4e8..961b07a7bd 100644
--- a/tensorflow/python/ops/distributions/student_t.py
+++ b/tensorflow/python/ops/distributions/student_t.py
@@ -158,7 +158,7 @@ class StudentT(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[df, loc, scale]):
+    with ops.name_scope(name, values=[df, loc, scale]) as name:
       with ops.control_dependencies([check_ops.assert_positive(df)]
                                     if validate_args else []):
         self._df = array_ops.identity(df, name="df")
@@ -350,7 +350,7 @@ class StudentTWithAbsDfSoftplusScale(StudentT):
                allow_nan_stats=True,
                name="StudentTWithAbsDfSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[df, scale]):
+    with ops.name_scope(name, values=[df, scale]) as name:
       super(StudentTWithAbsDfSoftplusScale, self).__init__(
           df=math_ops.floor(math_ops.abs(df)),
           loc=loc,
diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py
index 1efcf9d32e..6290297153 100644
--- a/tensorflow/python/ops/distributions/transformed_distribution.py
+++ b/tensorflow/python/ops/distributions/transformed_distribution.py
@@ -258,7 +258,7 @@ class TransformedDistribution(distribution_lib.Distribution):
     parameters = locals()
     name = name or (("" if bijector is None else bijector.name) +
                     distribution.name)
-    with ops.name_scope(name, values=[event_shape, batch_shape]):
+    with ops.name_scope(name, values=[event_shape, batch_shape]) as name:
       # For convenience we define some handy constants.
       self._zero = constant_op.constant(0, dtype=dtypes.int32, name="zero")
       self._empty = constant_op.constant([], dtype=dtypes.int32, name="empty")
diff --git a/tensorflow/python/ops/distributions/uniform.py b/tensorflow/python/ops/distributions/uniform.py
index ec623b55eb..b8acd0ded6 100644
--- a/tensorflow/python/ops/distributions/uniform.py
+++ b/tensorflow/python/ops/distributions/uniform.py
@@ -103,7 +103,7 @@ class Uniform(distribution.Distribution):
       InvalidArgumentError: if `low >= high` and `validate_args=False`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[low, high]):
+    with ops.name_scope(name, values=[low, high]) as name:
       with ops.control_dependencies([
           check_ops.assert_less(
               low, high, message="uniform not defined when low >= high.")
-- 
GitLab


From 7a212edc6b3ed6200158fe51acf4694a62ca6938 Mon Sep 17 00:00:00 2001
From: Karl Lessard <karllessard@users.noreply.github.com>
Date: Wed, 28 Mar 2018 02:55:01 -0400
Subject: [PATCH 600/960] Utility classes for writing Java source code from a
 C++ process (part 3) (#16120)

* Add output streams for writing Java source code.

* Rename variable 'declared_generics_names' to 'generic_namespace'

* Merge Java streams into Source writer

* Second pass of code review

* Replace vectors and deques with lists
---
 tensorflow/java/BUILD                         |   4 +-
 tensorflow/java/src/gen/cc/java_defs.h        |  45 +--
 tensorflow/java/src/gen/cc/source_writer.cc   | 297 +++++++++++++-
 tensorflow/java/src/gen/cc/source_writer.h    | 191 +++++++--
 .../java/src/gen/cc/source_writer_test.cc     | 367 +++++++++++++++++-
 .../java/src/gen/resources/test.snippet.java  |   2 +
 6 files changed, 824 insertions(+), 82 deletions(-)
 create mode 100644 tensorflow/java/src/gen/resources/test.snippet.java

diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 9dee1aa72b..7296205e24 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -311,9 +311,11 @@ tf_cc_test(
     srcs = [
         "src/gen/cc/source_writer_test.cc",
     ],
+    data = [
+        "src/gen/resources/test.snippet.java",
+    ],
     deps = [
         ":java_op_gen_lib",
-        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/java/src/gen/cc/java_defs.h b/tensorflow/java/src/gen/cc/java_defs.h
index 615cdc165b..59f8beaee7 100644
--- a/tensorflow/java/src/gen/cc/java_defs.h
+++ b/tensorflow/java/src/gen/cc/java_defs.h
@@ -17,10 +17,7 @@ limitations under the License.
 #define TENSORFLOW_JAVA_SRC_GEN_CC_JAVA_DEFS_H_
 
 #include <string>
-#include <vector>
-#include <deque>
-
-#include "tensorflow/core/platform/env.h"
+#include <list>
 
 namespace tensorflow {
 namespace java {
@@ -104,17 +101,17 @@ class Type {
     description_ = description;
     return *this;
   }
-  const std::vector<Type>& parameters() const { return parameters_; }
+  const std::list<Type>& parameters() const { return parameters_; }
   Type& add_parameter(const Type& parameter) {
     parameters_.push_back(parameter);
     return *this;
   }
-  const std::vector<Annotation>& annotations() const { return annotations_; }
+  const std::list<Annotation>& annotations() const { return annotations_; }
   Type& add_annotation(const Annotation& annotation) {
     annotations_.push_back(annotation);
     return *this;
   }
-  const std::deque<Type>& supertypes() const { return supertypes_; }
+  const std::list<Type>& supertypes() const { return supertypes_; }
   Type& add_supertype(const Type& type) {
     if (type.kind_ == CLASS) {
       supertypes_.push_front(type);  // keep superclass at the front of the list
@@ -141,9 +138,9 @@ class Type {
   string name_;
   string package_;
   string description_;
-  std::vector<Type> parameters_;
-  std::vector<Annotation> annotations_;
-  std::deque<Type> supertypes_;
+  std::list<Type> parameters_;
+  std::list<Annotation> annotations_;
+  std::list<Type> supertypes_;
 };
 
 // Definition of a Java annotation
@@ -223,16 +220,12 @@ class Method {
     return_description_ = description;
     return *this;
   }
-  const std::vector<Variable>& arguments() const { return arguments_; }
-  Method& add_arguments(const std::vector<Variable>& args) {
-    arguments_.insert(arguments_.cend(), args.cbegin(), args.cend());
-    return *this;
-  }
+  const std::list<Variable>& arguments() const { return arguments_; }
   Method& add_argument(const Variable& var) {
     arguments_.push_back(var);
     return *this;
   }
-  const std::vector<Annotation>& annotations() const { return annotations_; }
+  const std::list<Annotation>& annotations() const { return annotations_; }
   Method& add_annotation(const Annotation& annotation) {
     annotations_.push_back(annotation);
     return *this;
@@ -244,29 +237,13 @@ class Method {
   bool constructor_;
   string description_;
   string return_description_;
-  std::vector<Variable> arguments_;
-  std::vector<Annotation> annotations_;
+  std::list<Variable> arguments_;
+  std::list<Annotation> annotations_;
 
   Method(const string& name, const Type& return_type, bool constructor)
     : name_(name), return_type_(return_type), constructor_(constructor) {}
 };
 
-// A piece of code to read from a file.
-class Snippet {
- public:
-  static Snippet Create(const string& fname, Env* env = Env::Default()) {
-    return Snippet(fname, env);
-  }
-  const string& data() const { return data_; }
-
- private:
-  string data_;
-
-  Snippet(const string& fname, Env* env) {
-    TF_CHECK_OK(ReadFileToString(env, fname, &data_));
-  }
-};
-
 }  // namespace java
 }  // namespace tensorflow
 
diff --git a/tensorflow/java/src/gen/cc/source_writer.cc b/tensorflow/java/src/gen/cc/source_writer.cc
index 2da81f2911..214999af9a 100644
--- a/tensorflow/java/src/gen/cc/source_writer.cc
+++ b/tensorflow/java/src/gen/cc/source_writer.cc
@@ -14,49 +14,318 @@ limitations under the License.
 ==============================================================================*/
 
 #include <string>
+#include <algorithm>
+#include <deque>
 
 #include "tensorflow/java/src/gen/cc/source_writer.h"
 
 namespace tensorflow {
+namespace java {
 
-SourceWriter& SourceWriter::Append(const StringPiece& str) {
-  if (!str.empty()) {
-    if (newline_) {
-      DoAppend(left_margin_ + line_prefix_);
-      newline_ = false;
-    }
-    DoAppend(str);
-  }
+SourceWriter::SourceWriter() {
+  // push an empty generic namespace at start, for simplification
+  generic_namespaces_.push(new GenericNamespace());
+}
+
+SourceWriter& SourceWriter::Indent(int tab) {
+  left_margin_.resize(
+      std::max(static_cast<int>(left_margin_.size() + tab), 0), ' ');
+  return *this;
+}
+
+SourceWriter& SourceWriter::Prefix(const char* line_prefix) {
+  line_prefix_ = line_prefix;
   return *this;
 }
 
-SourceWriter& SourceWriter::Write(const string& str) {
+SourceWriter& SourceWriter::Write(const StringPiece& str) {
   size_t line_pos = 0;
   do {
     size_t start_pos = line_pos;
     line_pos = str.find('\n', start_pos);
     if (line_pos != string::npos) {
       ++line_pos;
-      Append(StringPiece(str.data() + start_pos, line_pos - start_pos));
+      Append(str.substr(start_pos, line_pos - start_pos));
       newline_ = true;
     } else {
-      Append(StringPiece(str.data() + start_pos, str.size() - start_pos));
+      Append(str.substr(start_pos, str.size() - start_pos));
     }
   } while (line_pos != string::npos && line_pos < str.size());
 
   return *this;
 }
 
+SourceWriter& SourceWriter::WriteFromFile(const string& fname, Env* env) {
+  string data_;
+  TF_CHECK_OK(ReadFileToString(env, fname, &data_));
+  return Write(data_);
+}
+
+SourceWriter& SourceWriter::Append(const StringPiece& str) {
+  if (!str.empty()) {
+    if (newline_) {
+      DoAppend(left_margin_ + line_prefix_);
+      newline_ = false;
+    }
+    DoAppend(str);
+  }
+  return *this;
+}
+
+SourceWriter& SourceWriter::AppendType(const Type& type) {
+  if (type.kind() == Type::Kind::GENERIC && type.name().empty()) {
+    Append("?");
+  } else {
+    Append(type.name());
+  }
+  if (!type.parameters().empty()) {
+    Append("<");
+    for (const Type& t : type.parameters()) {
+      if (&t != &type.parameters().front()) {
+        Append(", ");
+      }
+      AppendType(t);
+    }
+    Append(">");
+  }
+  return *this;
+}
+
 SourceWriter& SourceWriter::EndLine() {
   Append("\n");
   newline_ = true;
   return *this;
 }
 
-SourceWriter& SourceWriter::Indent(int tab) {
-  left_margin_.resize(std::max(static_cast<int>(left_margin_.size() + tab), 0),
-                      ' ');
+SourceWriter& SourceWriter::BeginMethod(const Method& method, int modifiers) {
+  GenericNamespace* generic_namespace = PushGenericNamespace(modifiers);
+  if (!method.constructor()) {
+    generic_namespace->Visit(method.return_type());
+  }
+  for (const Variable& v : method.arguments()) {
+    generic_namespace->Visit(v.type());
+  }
+  EndLine();
+  WriteDoc(method.description(), method.return_description(),
+      &method.arguments());
+  if (!method.annotations().empty()) {
+    WriteAnnotations(method.annotations());
+  }
+  WriteModifiers(modifiers);
+  if (!generic_namespace->declared_types().empty()) {
+    WriteGenerics(generic_namespace->declared_types());
+    Append(" ");
+  }
+  if (!method.constructor()) {
+    AppendType(method.return_type()).Append(" ");
+  }
+  Append(method.name()).Append("(");
+  for (const Variable& v : method.arguments()) {
+    if (&v != &method.arguments().front()) {
+      Append(", ");
+    }
+    AppendType(v.type()).Append(v.variadic() ? "... " : " ").Append(v.name());
+  }
+  return Append(")").BeginBlock();
+}
+
+SourceWriter& SourceWriter::EndMethod() {
+  EndBlock();
+  PopGenericNamespace();
   return *this;
 }
 
+SourceWriter& SourceWriter::BeginType(const Type& type,
+    const std::list<Type>* dependencies, int modifiers) {
+  if (!type.package().empty()) {
+    Append("package ").Append(type.package()).Append(";").EndLine();
+  }
+  if (dependencies != nullptr && !dependencies->empty()) {
+    TypeImporter type_importer(type.package());
+    for (const Type& t : *dependencies) {
+      type_importer.Visit(t);
+    }
+    EndLine();
+    for (const string& s : type_importer.imports()) {
+      Append("import ").Append(s).Append(";").EndLine();
+    }
+  }
+  return BeginInnerType(type, modifiers);
+}
+
+SourceWriter& SourceWriter::BeginInnerType(const Type& type, int modifiers) {
+  GenericNamespace* generic_namespace = PushGenericNamespace(modifiers);
+  generic_namespace->Visit(type);
+  EndLine();
+  WriteDoc(type.description());
+  if (!type.annotations().empty()) {
+    WriteAnnotations(type.annotations());
+  }
+  WriteModifiers(modifiers);
+  CHECK_EQ(Type::Kind::CLASS, type.kind()) << ": Not supported yet";
+  Append("class ").Append(type.name());
+  if (!generic_namespace->declared_types().empty()) {
+    WriteGenerics(generic_namespace->declared_types());
+  }
+  if (!type.supertypes().empty()) {
+    bool first_interface = true;
+    for (const Type& t : type.supertypes()) {
+      if (t.kind() == Type::CLASS) {  // superclass is always first in list
+        Append(" extends ");
+      } else if (first_interface) {
+        Append(" implements ");
+        first_interface = false;
+      } else {
+        Append(", ");
+      }
+      AppendType(t);
+    }
+  }
+  return BeginBlock();
+}
+
+SourceWriter& SourceWriter::EndType() {
+  EndBlock();
+  PopGenericNamespace();
+  return *this;
+}
+
+SourceWriter& SourceWriter::WriteFields(const std::list<Variable>& fields,
+    int modifiers) {
+  EndLine();
+  for (const Variable& v : fields) {
+    WriteModifiers(modifiers);
+    AppendType(v.type()).Append(" ").Append(v.name()).Append(";");
+    EndLine();
+  }
+  return *this;
+}
+
+SourceWriter& SourceWriter::WriteModifiers(int modifiers) {
+  if (modifiers & PUBLIC) {
+    Append("public ");
+  } else if (modifiers & PROTECTED) {
+    Append("protected ");
+  } else if (modifiers & PRIVATE) {
+    Append("private ");
+  }
+  if (modifiers & STATIC) {
+    Append("static ");
+  }
+  if (modifiers & FINAL) {
+    Append("final ");
+  }
+  return *this;
+}
+
+SourceWriter& SourceWriter::WriteDoc(const string& description,
+    const string& return_description, const std::list<Variable>* parameters) {
+  if (description.empty() && return_description.empty()
+      && (parameters == nullptr || parameters->empty())) {
+    return *this;  // no doc to write
+  }
+  bool do_line_break = false;
+  Append("/**").EndLine().Prefix(" * ");
+  if (!description.empty()) {
+    Write(description).EndLine();
+    do_line_break = true;
+  }
+  if (parameters != nullptr && !parameters->empty()) {
+    if (do_line_break) {
+      EndLine();
+      do_line_break = false;
+    }
+    for (const Variable& v : *parameters) {
+      Append("@param ").Append(v.name());
+      if (!v.description().empty()) {
+        Append(" ").Write(v.description());
+      }
+      EndLine();
+    }
+  }
+  if (!return_description.empty()) {
+    if (do_line_break) {
+      EndLine();
+      do_line_break = false;
+    }
+    Append("@return ").Write(return_description).EndLine();
+  }
+  return Prefix("").Append(" **/").EndLine();
+}
+
+SourceWriter& SourceWriter::WriteAnnotations(
+    const std::list<Annotation>& annotations) {
+  for (const Annotation& a : annotations) {
+    Append("@" + a.name());
+    if (!a.attributes().empty()) {
+      Append("(").Append(a.attributes()).Append(")");
+    }
+    EndLine();
+  }
+  return *this;
+}
+
+SourceWriter& SourceWriter::WriteGenerics(
+    const std::list<const Type*>& generics) {
+  Append("<");
+  for (const Type* pt : generics) {
+    if (pt != generics.front()) {
+      Append(", ");
+    }
+    Append(pt->name());
+    if (!pt->supertypes().empty()) {
+      Append(" extends ").AppendType(pt->supertypes().front());
+    }
+  }
+  return Append(">");
+}
+
+SourceWriter::GenericNamespace* SourceWriter::PushGenericNamespace(
+    int modifiers) {
+  GenericNamespace* generic_namespace;
+  if (modifiers & STATIC) {
+    generic_namespace = new GenericNamespace();
+  } else {
+    generic_namespace = new GenericNamespace(generic_namespaces_.top());
+  }
+  generic_namespaces_.push(generic_namespace);
+  return generic_namespace;
+}
+
+void SourceWriter::PopGenericNamespace() {
+  GenericNamespace* generic_namespace = generic_namespaces_.top();
+  generic_namespaces_.pop();
+  delete generic_namespace;
+}
+
+void SourceWriter::TypeVisitor::Visit(const Type& type) {
+  DoVisit(type);
+  for (const Type& t : type.parameters()) {
+    DoVisit(t);
+  }
+  for (const Annotation& t : type.annotations()) {
+    DoVisit(t);
+  }
+  for (const Type& t : type.supertypes()) {
+    DoVisit(t);
+  }
+}
+
+void SourceWriter::GenericNamespace::DoVisit(const Type& type) {
+  // ignore non-generic parameters, wildcards and generics already declared
+  if (type.kind() == Type::GENERIC
+      && !type.IsWildcard()
+      && generic_names_.find(type.name()) == generic_names_.end()) {
+    declared_types_.push_back(&type);
+    generic_names_.insert(type.name());
+  }
+}
+
+void SourceWriter::TypeImporter::DoVisit(const Type& type) {
+  if (!type.package().empty() && type.package() != current_package_) {
+    imports_.insert(type.package() + '.' + type.name());
+  }
+}
+
+}  // namespace java
 }  // namespace tensorflow
diff --git a/tensorflow/java/src/gen/cc/source_writer.h b/tensorflow/java/src/gen/cc/source_writer.h
index bff26eb185..6abe13b5d2 100644
--- a/tensorflow/java/src/gen/cc/source_writer.h
+++ b/tensorflow/java/src/gen/cc/source_writer.h
@@ -17,45 +17,23 @@ limitations under the License.
 #define TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_
 
 #include <string>
+#include <stack>
+#include <list>
+#include <set>
 
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/java/src/gen/cc/java_defs.h"
 
 namespace tensorflow {
+namespace java {
 
-// A utility class for writing source code, normally generated at
-// compile-time.
-//
-// Source writers are language-agnostic and therefore only expose generic
-// methods common to most languages. Extend or wrap this class to implement
-// language-specific features.
-//
-// Note: if you are looking to reuse this class for generating code in another
-// language than Java, please do by moving it at the '//tensorflow/core/lib/io'
-// level.
+// A class for writing Java source code.
 class SourceWriter {
  public:
+  SourceWriter();
   virtual ~SourceWriter() = default;
 
-  // Returns true if the writer is at the beginnig of a new line
-  bool newline() const { return newline_; }
-
-  // Appends a piece of code or text.
-  //
-  // It is expected that no newline character is present in the data provided,
-  // otherwise Write() must be used.
-  SourceWriter& Append(const StringPiece& str);
-
-  // Writes a block of code or text.
-  //
-  // The data might potentially contain newline characters, therefore it will
-  // be scanned to ensure that each line is indented and prefixed properly,
-  // making it a bit slower than Append().
-  SourceWriter& Write(const string& text);
-
-  // Appends a newline character and start writing on a new line.
-  SourceWriter& EndLine();
-
   // Indents following lines with white spaces.
   //
   // Indentation is cumulative, i.e. the provided tabulation is added to the
@@ -75,18 +53,166 @@ class SourceWriter {
   // Indent(2)->Prefix("//") will result in prefixing lines with "  //".
   //
   // An empty value ("") will remove any line prefix that was previously set.
-  SourceWriter& Prefix(const char* line_prefix) {
-    line_prefix_ = line_prefix;
-    return *this;
+  SourceWriter& Prefix(const char* line_prefix);
+
+  // Writes a source code snippet.
+  //
+  // The data might potentially contain newline characters, therefore it will
+  // be scanned to ensure that each line is indented and prefixed properly,
+  // making it a bit slower than Append().
+  SourceWriter& Write(const StringPiece& text);
+
+  // Writes a source code snippet read from a file.
+  //
+  // All lines of the file at the provided path will be read and written back
+  // to the output of this writer in regard of its current attributes (e.g.
+  // the indentation, prefix, etc.)
+  SourceWriter& WriteFromFile(const string& fname, Env* env = Env::Default());
+
+  // Appends a piece of source code.
+  //
+  // It is expected that no newline character is present in the data provided,
+  // otherwise Write() must be used.
+  SourceWriter& Append(const StringPiece& str);
+
+  // Appends a type to the current line.
+  //
+  // The type is written in its simple form (i.e. not prefixed by its package)
+  // and followed by any parameter types it has enclosed in brackets (<>).
+  SourceWriter& AppendType(const Type& type);
+
+  // Appends a newline character.
+  //
+  // Data written after calling this method will start on a new line, in respect
+  // of the current indentation.
+  SourceWriter& EndLine();
+
+  // Begins a block of source code.
+  //
+  // This method appends a new opening brace to the current data and indent the
+  // next lines according to Google Java Style Guide. The block can optionally
+  // be preceded by an expression (e.g. Append("if(true)").BeginBlock();)
+  SourceWriter& BeginBlock() {
+    return Append(newline_ ? "{" : " {").EndLine().Indent(2);
+  }
+
+  // Ends the current block of source code.
+  //
+  // This method appends a new closing brace to the current data and outdent the
+  // next lines back to the margin used before BeginBlock() was invoked.
+  SourceWriter& EndBlock() {
+    return Indent(-2).Append("}").EndLine();
   }
 
+  // Begins to write a method.
+  //
+  // This method outputs the signature of the Java method from the data passed
+  // in the 'method' parameter and starts a new block. Additionnal modifiers can
+  // also be passed in parameter to define the accesses and the scope of this
+  // method.
+  SourceWriter& BeginMethod(const Method& method, int modifiers = 0);
+
+  // Ends the current method.
+  //
+  // This method ends the block of code that has begun when invoking
+  // BeginMethod() prior to this.
+  SourceWriter& EndMethod();
+
+  // Begins to write the main type of a source file.
+  //
+  // This method outputs the declaration of the Java type from the data passed
+  // in the 'type' parameter and starts a new block. Additionnal modifiers can
+  // also be passed in parameter to define the accesses and the scope of this
+  // type.
+  //
+  // If not null, all types found in the 'dependencies' list will be imported
+  // before declaring the new type.
+  SourceWriter& BeginType(const Type& clazz,
+      const std::list<Type>* dependencies, int modifiers = 0);
+
+  // Begins to write a new inner type.
+  //
+  // This method outputs the declaration of the Java type from the data passed
+  // in the 'type' parameter and starts a new block. Additionnal modifiers can
+  // also be passed in parameter to define the accesses and the scope of this
+  // type.
+  SourceWriter& BeginInnerType(const Type& type, int modifiers = 0);
+
+  // Ends the current type.
+  //
+  // This method ends the block of code that has begun when invoking
+  // BeginType() or BeginInnerType() prior to this.
+  SourceWriter& EndType();
+
+  // Writes a list of variables as fields of a type.
+  //
+  // This method must be called within the definition of a type (see BeginType()
+  // or BeginInnerType()). Additional modifiers can also be passed in parameter
+  // to define the accesses and the scope of those fields.
+  SourceWriter& WriteFields(const std::list<Variable>& fields,
+      int modifiers = 0);
+
  protected:
   virtual void DoAppend(const StringPiece& str) = 0;
 
  private:
+  // A utility base class for visiting elements of a type.
+  class TypeVisitor {
+   public:
+    virtual ~TypeVisitor() = default;
+    void Visit(const Type& type);
+
+   protected:
+    virtual void DoVisit(const Type& type) = 0;
+  };
+
+  // A utility class for keeping track of declared generics in a given scope.
+  class GenericNamespace : public TypeVisitor {
+   public:
+    GenericNamespace() = default;
+    explicit GenericNamespace(const GenericNamespace* parent)
+      : generic_names_(parent->generic_names_) {}
+    std::list<const Type*> declared_types() {
+      return declared_types_;
+    }
+   protected:
+    virtual void DoVisit(const Type& type);
+
+   private:
+    std::list<const Type*> declared_types_;
+    std::set<string> generic_names_;
+  };
+
+  // A utility class for collecting a list of import statements to declare.
+  class TypeImporter : public TypeVisitor {
+   public:
+    explicit TypeImporter(const string& current_package)
+      : current_package_(current_package) {}
+    virtual ~TypeImporter() = default;
+    const std::set<string> imports() {
+      return imports_;
+    }
+   protected:
+    virtual void DoVisit(const Type& type);
+
+   private:
+    string current_package_;
+    std::set<string> imports_;
+  };
+
   string left_margin_;
   string line_prefix_;
   bool newline_ = true;
+  std::stack<GenericNamespace*> generic_namespaces_;
+
+  SourceWriter& WriteModifiers(int modifiers);
+  SourceWriter& WriteDoc(const string& description,
+    const string& return_description = "",
+    const std::list<Variable>* parameters = nullptr);
+  SourceWriter& WriteAnnotations(const std::list<Annotation>& annotations);
+  SourceWriter& WriteGenerics(const std::list<const Type*>& generics);
+  GenericNamespace* PushGenericNamespace(int modifiers);
+  void PopGenericNamespace();
 };
 
 // A writer that outputs source code into a file.
@@ -128,6 +254,7 @@ class SourceBufferWriter : public SourceWriter {
   string* buffer_;
 };
 
+}  // namespace java
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_JAVA_SRC_GEN_CC_SOURCE_WRITER_H_
diff --git a/tensorflow/java/src/gen/cc/source_writer_test.cc b/tensorflow/java/src/gen/cc/source_writer_test.cc
index e973895754..6926a5a411 100644
--- a/tensorflow/java/src/gen/cc/source_writer_test.cc
+++ b/tensorflow/java/src/gen/cc/source_writer_test.cc
@@ -13,11 +13,15 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#include "tensorflow/java/src/gen/cc/source_writer.h"
+#include <list>
+
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/java/src/gen/cc/java_defs.h"
+#include "tensorflow/java/src/gen/cc/source_writer.h"
 
 namespace tensorflow {
+namespace java {
 namespace {
 
 TEST(AppendTest, SingleLineText) {
@@ -211,5 +215,366 @@ TEST(MarginTest, EmptyPrefix) {
   ASSERT_STREQ(expected, writer.str().data());
 }
 
+TEST(StreamTest, BlocksAndLines) {
+  SourceBufferWriter writer;
+
+  writer.Append("int i = 0;").EndLine()
+        .Append("int j = 10;").EndLine()
+        .Append("if (true)")
+        .BeginBlock()
+          .Append("int aLongWayToTen = 0;").EndLine()
+          .Append("while (++i <= j)")
+          .BeginBlock()
+            .Append("++aLongWayToTen;").EndLine()
+          .EndBlock()
+        .EndBlock();
+
+  const char* expected =
+      "int i = 0;\n"
+      "int j = 10;\n"
+      "if (true) {\n"
+      "  int aLongWayToTen = 0;\n"
+      "  while (++i <= j) {\n"
+      "    ++aLongWayToTen;\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(StreamTest, Types) {
+  SourceBufferWriter writer;
+  Type generic = Type::Generic("T").add_supertype(Type::Class("Number"));
+
+  writer.AppendType(Type::Int()).Append(", ")
+        .AppendType(Type::Class("String")).Append(", ")
+        .AppendType(generic).Append(", ")
+        .AppendType(Type::ListOf(generic)).Append(", ")
+        .AppendType(Type::ListOf(Type::IterableOf(generic))).Append(", ")
+        .AppendType(Type::ListOf(Type::Generic()));
+
+  const char* expected =
+      "int, String, T, List<T>, List<Iterable<T>>, List<?>";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(StreamTest, FileSnippet) {
+  SourceBufferWriter writer;
+  const string& fname = "tensorflow/java/src/gen/resources/test.snippet.java";
+
+  writer.WriteFromFile(fname)
+        .BeginBlock()
+        .WriteFromFile(fname)
+        .EndBlock();
+
+  const char* expected =
+      "// Here is a little snippet\n"
+      "System.out.println(\"Hello!\");\n"
+      "{\n"
+      "  // Here is a little snippet\n"
+      "  System.out.println(\"Hello!\");\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, SimpleClass) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+
+  writer.BeginType(clazz, nullptr, PUBLIC).EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test {\n}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, SimpleClassWithDependencies) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  std::list<Type> deps;
+  deps.push_back(Type::Class("TypeA", "org.test.sub"));
+  deps.push_back(Type::Class("TypeA", "org.test.sub"));  // a second time
+  deps.push_back(Type::Class("TypeB", "org.other"));
+  deps.push_back(Type::Class("SamePackageType", "org.tensorflow"));
+  deps.push_back(Type::Class("NoPackageType"));
+
+  writer.BeginType(clazz, &deps, PUBLIC).EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "import org.other.TypeB;\n"
+      "import org.test.sub.TypeA;\n\n"
+      "public class Test {\n}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, AnnotatedAndDocumentedClass) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  clazz.description("This class has a\n<p>\nmultiline description.");
+  clazz.add_annotation(Annotation::Create("Bean"));
+  clazz.add_annotation(Annotation::Create("SuppressWarnings")
+      .attributes("\"rawtypes\""));
+
+  writer.BeginType(clazz, nullptr, PUBLIC).EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "/**\n"
+      " * This class has a\n"
+      " * <p>\n"
+      " * multiline description.\n"
+      " **/\n"
+      "@Bean\n"
+      "@SuppressWarnings(\"rawtypes\")\n"
+      "public class Test {\n}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, ParameterizedClass) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  clazz.add_parameter(Type::Generic("T"));
+  clazz.add_parameter(Type::Generic("U").add_supertype(Type::Class("Number")));
+
+  writer.BeginType(clazz, nullptr, PUBLIC).EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T, U extends Number> {\n}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, ParameterizedClassAndSupertypes) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type type_t = Type::Generic("T");
+  clazz.add_parameter(type_t);
+  Type type_u = Type::Generic("U").add_supertype(Type::Class("Number"));
+  clazz.add_parameter(type_u);
+  clazz.add_supertype(Type::Interface("Parametrizable").add_parameter(type_u));
+  clazz.add_supertype(Type::Interface("Runnable"));
+  clazz.add_supertype(Type::Class("SuperTest").add_parameter(type_t));
+
+  writer.BeginType(clazz, nullptr, PUBLIC).EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T, U extends Number>"
+      " extends SuperTest<T> implements Parametrizable<U>, Runnable {\n}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, ParameterizedClassFields) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
+  clazz.add_parameter(type_t);
+  std::list<Variable> static_fields;
+  static_fields.push_back(Variable::Create("field1", Type::Class("String")));
+  std::list<Variable> member_fields;
+  member_fields.push_back(Variable::Create("field2", Type::Class("String")));
+  member_fields.push_back(Variable::Create("field3", type_t));
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .WriteFields(static_fields, STATIC | PUBLIC | FINAL)
+          .WriteFields(member_fields, PRIVATE)
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T extends Number> {\n"
+      "  \n"
+      "  public static final String field1;\n"
+      "  \n"
+      "  private String field2;\n"
+      "  private T field3;\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, SimpleInnerClass) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type inner_class = Type::Class("InnerTest");
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginInnerType(inner_class, PUBLIC)
+          .EndType()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test {\n"
+      "  \n"
+      "  public class InnerTest {\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteType, StaticParameterizedInnerClass) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
+  clazz.add_parameter(type_t);
+  Type inner_class = Type::Class("InnerTest");
+  inner_class.add_parameter(type_t);
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginInnerType(inner_class, PUBLIC | STATIC)
+          .EndType()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T extends Number> {\n"
+      "  \n"
+      "  public static class InnerTest<T extends Number> {\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteMethod, SimpleMethod) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Method method = Method::Create("doNothing", Type::Void());
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginMethod(method, PUBLIC).EndMethod()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test {\n"
+      "  \n"
+      "  public void doNothing() {\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteMethod, AnnotatedAndDocumentedMethod) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Method method = Method::Create("doNothing", Type::Void());
+  method.description("This method has a\n<p>\nmultiline description.");
+  method.add_annotation(Annotation::Create("Override"));
+  method.add_annotation(Annotation::Create("SuppressWarnings")
+      .attributes("\"rawtypes\""));
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginMethod(method, PUBLIC).EndMethod()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test {\n"
+      "  \n"
+      "  /**\n"
+      "   * This method has a\n"
+      "   * <p>\n"
+      "   * multiline description.\n"
+      "   **/\n"
+      "  @Override\n"
+      "  @SuppressWarnings(\"rawtypes\")\n"
+      "  public void doNothing() {\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteMethod, DocumentedMethodWithArguments) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Method method = Method::Create("boolToInt", Type::Int());
+  method.description("Converts a boolean to an int");
+  method.return_description("int value for this boolean");
+  method.add_argument(Variable::Create("b", Type::Boolean()));
+  Variable reverse = Variable::Create("reverse", Type::Boolean());
+  reverse.description("if true, value is reversed");
+  method.add_argument(reverse);
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginMethod(method, PUBLIC)
+            .Append("if (b && !reverse)")
+            .BeginBlock()
+              .Append("return 1;").EndLine()
+            .EndBlock()
+          .Append("return 0;").EndLine()
+          .EndMethod()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test {\n"
+      "  \n"
+      "  /**\n"
+      "   * Converts a boolean to an int\n"
+      "   * \n"
+      "   * @param b\n"
+      "   * @param reverse if true, value is reversed\n"
+      "   * @return int value for this boolean\n"
+      "   **/\n"
+      "  public int boolToInt(boolean b, boolean reverse) {\n"
+      "    if (b && !reverse) {\n"
+      "      return 1;\n"
+      "    }\n"
+      "    return 0;\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteMethod, ParameterizedMethod) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
+  clazz.add_parameter(type_t);
+  Method method = Method::Create("doNothing", type_t);
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginMethod(method, PUBLIC)
+            .Append("return null;").EndLine()
+          .EndMethod()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T extends Number> {\n"
+      "  \n"
+      "  public T doNothing() {\n"
+      "    return null;\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
+TEST(WriteMethod, StaticParameterizedMethod) {
+  SourceBufferWriter writer;
+  Type clazz = Type::Class("Test", "org.tensorflow");
+  Type type_t = Type::Generic("T").add_supertype(Type::Class("Number"));
+  clazz.add_parameter(type_t);
+  Method method = Method::Create("doNothing", type_t);
+
+  writer.BeginType(clazz, nullptr, PUBLIC)
+          .BeginMethod(method, PUBLIC | STATIC)
+            .Append("return null;").EndLine()
+          .EndMethod()
+        .EndType();
+
+  const char* expected =
+      "package org.tensorflow;\n\n"
+      "public class Test<T extends Number> {\n"
+      "  \n"
+      "  public static <T extends Number> T doNothing() {\n"
+      "    return null;\n"
+      "  }\n"
+      "}\n";
+  ASSERT_STREQ(expected, writer.str().data());
+}
+
 }  // namespace
+}  // namespace java
 }  // namespace tensorflow
diff --git a/tensorflow/java/src/gen/resources/test.snippet.java b/tensorflow/java/src/gen/resources/test.snippet.java
new file mode 100644
index 0000000000..5e412a9aef
--- /dev/null
+++ b/tensorflow/java/src/gen/resources/test.snippet.java
@@ -0,0 +1,2 @@
+// Here is a little snippet
+System.out.println("Hello!");
-- 
GitLab


From 134f4ca0a70ef0373f5436b890be0f8585badb34 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 00:57:37 -0700
Subject: [PATCH 601/960] Internal change

PiperOrigin-RevId: 190735724
---
 tensorflow/compiler/jit/xla_tensor_info.h | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/compiler/jit/xla_tensor_info.h b/tensorflow/compiler/jit/xla_tensor_info.h
index fbd6ad770f..a02f642c37 100644
--- a/tensorflow/compiler/jit/xla_tensor_info.h
+++ b/tensorflow/compiler/jit/xla_tensor_info.h
@@ -71,6 +71,12 @@ class XlaTensorInfoManager : public AllocatorWrapper {
   // Creates a new XlaTensorInfoManager, delegating all DeallocateRaw calls to
   // allocator.
   XlaTensorInfoManager(Allocator* allocator) : AllocatorWrapper(allocator) {}
+  ~XlaTensorInfoManager() {
+    // Destroy the tensor info hashtable under the lock, to ensure all accesses
+    // to the hashtable are properly sequenced.
+    mutex_lock lock(lock_);
+    tensor_infos_.clear();
+  }
 
   // Returns the XlaTensorInfo for the given device memory pointer or nullptr if
   // none exists.
-- 
GitLab


From 8f3f838d2b14c78ae778031231ff5460e4403c80 Mon Sep 17 00:00:00 2001
From: Taehoon Lee <me@taehoonlee.com>
Date: Thu, 29 Mar 2018 00:22:54 +0900
Subject: [PATCH 602/960] Fix typos (#18048)

---
 .../toco/graph_transformations/unpartition_embedding_lookup.cc  | 2 +-
 tensorflow/contrib/verbs/rdma.h                                 | 2 +-
 tensorflow/core/common_runtime/scoped_allocator_mgr.cc          | 2 +-
 tensorflow/core/kernels/mkl_softmax_op.cc                       | 2 +-
 tensorflow/python/kernel_tests/init_ops_test.py                 | 2 +-
 tensorflow/python/ops/custom_gradient.py                        | 2 +-
 tensorflow/python/training/distribute.py                        | 2 +-
 7 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
index 48c326651f..cbea39bcc0 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/unpartition_embedding_lookup.cc
@@ -165,7 +165,7 @@ bool UnpartitionEmbeddingLookup::Run(Model* model, std::size_t op_index) {
   CHECK(mod_op && mod_op->type == OperatorType::kFloorMod)
       << "Unsupported partition strategy";
   CHECK_EQ(mod_op, GetOpWithOutput(*model, indices_partition_op->inputs[1]))
-      << "Indices and data parition ops require the same partition strategy "
+      << "Indices and data partition ops require the same partition strategy "
          "and inputs";
 
   // Glob together all of the gather data. This is not yet in the correct order.
diff --git a/tensorflow/contrib/verbs/rdma.h b/tensorflow/contrib/verbs/rdma.h
index 94203ee2b3..c9df6beb6b 100644
--- a/tensorflow/contrib/verbs/rdma.h
+++ b/tensorflow/contrib/verbs/rdma.h
@@ -262,7 +262,7 @@ class RdmaTensorRequest {
   // Receive tensor content (RDMA write was completed).
   //
   // Decode proto if required and/or move to GPU if the content was not
-  // written to it directly (GPU direct is not avaliable). Afterwards,
+  // written to it directly (GPU direct is not available). Afterwards,
   // invoke Done().
   void RecvTensorContent();
 
diff --git a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
index e1f70404e3..be79cc4507 100644
--- a/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
+++ b/tensorflow/core/common_runtime/scoped_allocator_mgr.cc
@@ -103,7 +103,7 @@ ScopedAllocatorContainer::~ScopedAllocatorContainer() {
   // In normal execution the table should be empty and all of its
   // contents deleted via Drop.  When when a step ends early
   // (e.g. through abnormal termination) we need to clean up
-  // explicitly.  So long as graph exection of the associated step has
+  // explicitly.  So long as graph execution of the associated step has
   // completey terminated this should be safe.
   for (auto& it : allocators_) {
     if (it.second.field_index == ScopedAllocator::kBackingIndex) {
diff --git a/tensorflow/core/kernels/mkl_softmax_op.cc b/tensorflow/core/kernels/mkl_softmax_op.cc
index 170523b5b4..f79e18cff2 100644
--- a/tensorflow/core/kernels/mkl_softmax_op.cc
+++ b/tensorflow/core/kernels/mkl_softmax_op.cc
@@ -102,7 +102,7 @@ class MklSoftmaxOp : public OpKernel {
       // Softmax MklDnn output layout is same as input layout.
       auto dst_pd = src.GetUsrMemPrimDesc();
 
-      // if input is MKL shape, ouput is also MKL shape.
+      // if input is MKL shape, output is also MKL shape.
       // if input is TF shape, output is also TF shape
       if (src_mkl_shape.IsMklTensor()) {
         output_mkl_shape.SetMklTensor(true);
diff --git a/tensorflow/python/kernel_tests/init_ops_test.py b/tensorflow/python/kernel_tests/init_ops_test.py
index c1755985ee..3c4d038ef9 100644
--- a/tensorflow/python/kernel_tests/init_ops_test.py
+++ b/tensorflow/python/kernel_tests/init_ops_test.py
@@ -618,7 +618,7 @@ class ConvolutionDeltaOrthogonalInitializerTest(test.TestCase):
     for dtype in [dtypes.float32]:
       for kernel_size in [[3], [8], [3, 5], [2, 4], [3, 3, 3], [2, 2, 2]]:
         tol = 1e-2
-        # Check orthogonality by computing the 2-norms of the inputs and ouputs.
+        # Check orthogonality by computing the 2-norms of the inputs and outputs.
         if len(kernel_size) == 1:
           shape = [4, 32, 64]
           convolution = convolutional.conv1d
diff --git a/tensorflow/python/ops/custom_gradient.py b/tensorflow/python/ops/custom_gradient.py
index 9eacac1b37..dfa07abfc6 100644
--- a/tensorflow/python/ops/custom_gradient.py
+++ b/tensorflow/python/ops/custom_gradient.py
@@ -95,7 +95,7 @@ def custom_gradient(f):
     if not context.executing_eagerly():
       if kwargs:
         raise ValueError(
-            "The custom_gradient decorator currently suports keywords "
+            "The custom_gradient decorator currently supports keywords "
             "arguments only when eager execution is enabled.")
       name = "CustomGradient-%s" % ops.uid()
       args = [ops.convert_to_tensor(x) for x in args]
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 9261e13230..3571a41d4f 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -519,7 +519,7 @@ class DistributionStrategy(object):
   in the distributed vs. single tower cases.
   """
 
-  # TODO(josh11b): Raise an exception if variable paritioning requested before
+  # TODO(josh11b): Raise an exception if variable partitioning requested before
   #   we add support.
   # TODO(josh11b): Also `parameter_device_index` property?
   # TODO(josh11b): `map()`
-- 
GitLab


From 119ed5aa2acb6df04595835f6dfa99f5422449f2 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 28 Mar 2018 08:25:14 -0700
Subject: [PATCH 603/960] Move ExecuteNode and CopyToDevice_Internal

PiperOrigin-RevId: 190775681
---
 tensorflow/c/eager/BUILD                      |   1 +
 tensorflow/c/eager/c_api.cc                   | 139 ++++--------------
 tensorflow/core/common_runtime/eager/BUILD    |  17 +++
 .../core/common_runtime/eager/execute.cc      |  24 +++
 .../core/common_runtime/eager/execute.h       |   4 +
 .../core/common_runtime/eager/execute_node.h  |  88 +++++++++++
 6 files changed, 161 insertions(+), 112 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/eager/execute_node.h

diff --git a/tensorflow/c/eager/BUILD b/tensorflow/c/eager/BUILD
index e57011a08b..a2d96357ac 100644
--- a/tensorflow/c/eager/BUILD
+++ b/tensorflow/c/eager/BUILD
@@ -31,6 +31,7 @@ tf_cuda_library(
             "//tensorflow/core/common_runtime/eager:context",
             "//tensorflow/core/common_runtime/eager:eager_executor",
             "//tensorflow/core/common_runtime/eager:execute",
+            "//tensorflow/core/common_runtime/eager:execute_node",
             "//tensorflow/core/common_runtime/eager:kernel_and_device",
             "//tensorflow/core/common_runtime/eager:tensor_handle",
             "//tensorflow/core/common_runtime/eager:copy_to_device_node",
diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index ac7114f71e..028865d360 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/device_set.h"
 #include "tensorflow/core/common_runtime/eager/copy_to_device_node.h"
 #include "tensorflow/core/common_runtime/eager/execute.h"
+#include "tensorflow/core/common_runtime/eager/execute_node.h"
 #include "tensorflow/core/common_runtime/function.h"
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/node_def_util.h"
@@ -435,39 +436,8 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 namespace {
 
-// TODO(apassos) move to TensorHandle
-tensorflow::TensorHandle* TFE_TensorHandleCopyToDevice_Internal(
-    tensorflow::TensorHandle* h, TFE_Context* ctx, const char* device_name,
-    TF_Status* status) {
-  status->status = ctx->context.GetStatus();
-  if (!status->status.ok()) {
-    return nullptr;
-  }
-  tensorflow::Device* dstd = ctx->context.HostCPU();
-  if (device_name != nullptr && strlen(device_name) > 0) {
-    status->status =
-        ctx->context.device_mgr()->LookupDevice(device_name, &dstd);
-    if (!status->status.ok()) return nullptr;
-  }
-  if (ctx->context.Async()) {
-    // Note that `h` may not be currently ready. However execution order will
-    // make sure that `h` is ready before the copy is actually done.
-    tensorflow::CopyToDeviceNode* node =
-        new tensorflow::CopyToDeviceNode(h, dstd, &ctx->context);
-    tensorflow::TensorHandle* output = node->dst();
-    // Note that calling Add makes `node` accessible by the EagerExecutor
-    // thread. So further accesses need to be thread-safe.
-    ctx->context.ExecutorAdd(node);
-    return output;
-  } else {
-    tensorflow::TensorHandle* output = nullptr;
-    status->status = h->CopyToDevice(&ctx->context, dstd, &output);
-    return output;
-  }
-}
-
 tensorflow::Status ValidateInputTypeAndPlacement(
-    TFE_Context* ctx, tensorflow::Device* host_device,
+    tensorflow::EagerContext* ctx, tensorflow::Device* host_device,
     tensorflow::Device* op_device, TFE_Op* op,
     const tensorflow::OpKernel* kernel) {
   const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types();
@@ -484,8 +454,8 @@ tensorflow::Status ValidateInputTypeAndPlacement(
     const tensorflow::Device* actual_device =
         handle_device == nullptr ? host_device : handle_device;
     if (expected_device != actual_device) {
-      switch (TFE_ContextGetDevicePlacementPolicy(ctx)) {
-        case TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32:
+      switch (ctx->GetDevicePlacementPolicy()) {
+        case tensorflow::DEVICE_PLACEMENT_SILENT_FOR_INT32:
           // TODO(xpan): See if we could bubble python related error up
           // to python level.
           if (handle->dtype == tensorflow::DT_INT32) {
@@ -494,7 +464,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
             break;
           }
           TF_FALLTHROUGH_INTENDED;
-        case TFE_DEVICE_PLACEMENT_EXPLICIT:
+        case tensorflow::DEVICE_PLACEMENT_EXPLICIT:
           return tensorflow::errors::InvalidArgument(
               "Tensors on conflicting devices:"
               " cannot compute ",
@@ -506,7 +476,7 @@ tensorflow::Status ValidateInputTypeAndPlacement(
               " or transparently copied by using tfe.enable_eager_execution("
               "tfe.DEVICE_PLACEMENT_SILENT). Copying tensors between devices"
               " may slow down your model");
-        case TFE_DEVICE_PLACEMENT_WARN:
+        case tensorflow::DEVICE_PLACEMENT_WARN:
           LOG(WARNING) << "before computing " << op->name << " input #" << i
                        << " was expected to be on " << expected_device->name()
                        << " but is actually on " << actual_device->name()
@@ -514,17 +484,14 @@ tensorflow::Status ValidateInputTypeAndPlacement(
                        << "). This triggers a copy which can be a performance "
                           "bottleneck.";
           break;
-        case TFE_DEVICE_PLACEMENT_SILENT:  // Do nothing.
+        case tensorflow::DEVICE_PLACEMENT_SILENT:  // Do nothing.
           break;
       }
       // We are only here if the policy is warn or silent copies, so we should
       // trigger a copy.
-      TF_Status* s = TF_NewStatus();
-      tensorflow::TensorHandle* copied_tensor =
-          TFE_TensorHandleCopyToDevice_Internal(
-              handle, ctx, expected_device->name().c_str(), s);
-      tensorflow::Status status = s->status;
-      TF_DeleteStatus(s);
+      tensorflow::TensorHandle* copied_tensor = nullptr;
+      tensorflow::Status status = tensorflow::EagerCopyToDevice(
+          handle, ctx, expected_device->name().c_str(), &copied_tensor);
       if (!status.ok()) {
         if (copied_tensor != nullptr) copied_tensor->Unref();
         return tensorflow::errors::Internal(
@@ -576,68 +543,6 @@ tensorflow::Device* SelectDevice(const tensorflow::NodeDef& ndef,
 }
 
 
-// TODO(agarwal): move EagerExecutor and EagerNode related code to a separate
-// file.
-class ExecuteNode : public tensorflow::EagerNode {
- public:
-  ExecuteNode(TFE_Op* op, tensorflow::KernelAndDevice* kernel,
-              tensorflow::NodeExecStats* maybe_stats,
-              const tensorflow::DataTypeVector& output_dtypes,
-              TFE_TensorHandle** retvals, int num_retvals)
-      : tensorflow::EagerNode(op->ctx->context.NextId()),
-        ctx_(op->ctx),
-        op_device_(op->device),
-        inputs_(op->inputs),
-        kernel_(kernel),
-        maybe_stats_(maybe_stats),
-        retvals_(num_retvals) {
-    for (auto handle : inputs_) {
-      handle->Ref();
-    }
-    TFE_Context* ctx = op->ctx;
-    for (int i = 0; i < num_retvals; ++i) {
-      tensorflow::TensorHandle* h =
-          new tensorflow::TensorHandle(id, output_dtypes[i], &ctx->context);
-      h->Ref();
-      retvals[i] = new TFE_TensorHandle(h);
-      retvals_[i] = h;
-    }
-  }
-
-  ~ExecuteNode() override {
-    for (auto handle : inputs_) {
-      handle->Unref();
-    }
-    for (auto handle : retvals_) {
-      handle->Unref();
-    }
-  }
-
-  tensorflow::Status Run() override {
-    const tensorflow::Status status = tensorflow::EagerExecute(
-        &ctx_->context, op_device_, inputs_, kernel_, maybe_stats_.get(),
-        retvals_.begin(), retvals_.size());
-    if (status.ok()) {
-      return status;
-    } else {
-      return tensorflow::Status(
-          status.code(),
-          tensorflow::strings::StrCat("Got error, \"", status.error_message(),
-                                      "\" while executing kernel ",
-                                      kernel_->kernel()->def().DebugString()));
-    }
-  }
-
- private:
-  TFE_Context* ctx_;
-  tensorflow::Device* op_device_;
-  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 4> inputs_;
-  tensorflow::KernelAndDevice* kernel_;
-  std::unique_ptr<tensorflow::NodeExecStats> maybe_stats_;
-  tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> retvals_;
-};
-
-
 #ifdef TENSORFLOW_EAGER_USE_XLA
 // Synthesizes and returns a wrapper function over `op`, which must be a
 // primitive op (e.g. matmul).
@@ -961,8 +866,8 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // device from the one requested above.
     device = kernel->device();
   }
-  status->status = ValidateInputTypeAndPlacement(ctx, ctx->context.HostCPU(),
-                                                 device, op, kernel->kernel());
+  status->status = ValidateInputTypeAndPlacement(
+      &ctx->context, ctx->context.HostCPU(), device, op, kernel->kernel());
   if (!status->status.ok()) return;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats;
   if (ctx->context.ShouldStoreMetadata()) {
@@ -977,9 +882,18 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     // Note that for async mode, execution order will make sure that all
     // input handles are ready before executing them.
     // TODO(agarwal): Consider executing "cheap" kernels inline for performance.
-    tensorflow::EagerNode* node =
-        new ExecuteNode(op, kernel, maybe_stats.release(), output_dtypes,
-                        retvals, *num_retvals);
+    tensorflow::gtl::InlinedVector<tensorflow::TensorHandle*, 2> handle_retvals(
+        *num_retvals);
+    tensorflow::uint64 id = op->ctx->context.NextId();
+    for (int i = 0; i < *num_retvals; ++i) {
+      tensorflow::TensorHandle* h =
+          new tensorflow::TensorHandle(id, output_dtypes[i], &op->ctx->context);
+      retvals[i] = new TFE_TensorHandle(h);
+      handle_retvals[i] = h;
+    }
+    tensorflow::EagerNode* node = new tensorflow::ExecuteNode(
+        id, &op->ctx->context, op->device, op->inputs, kernel,
+        maybe_stats.release(), output_dtypes, handle_retvals);
     ctx->context.ExecutorAdd(node);
   } else {
     // Execute checks if retvals[i] is nullptr or not to figure if it needs to
@@ -999,8 +913,9 @@ TFE_TensorHandle* TFE_TensorHandleCopyToDevice(TFE_TensorHandle* h,
                                                TFE_Context* ctx,
                                                const char* device_name,
                                                TF_Status* status) {
-  tensorflow::TensorHandle* handle = TFE_TensorHandleCopyToDevice_Internal(
-      h->handle, ctx, device_name, status);
+  tensorflow::TensorHandle* handle;
+  status->status = tensorflow::EagerCopyToDevice(h->handle, &ctx->context,
+                                                 device_name, &handle);
   if (status->status.ok()) {
     return new TFE_TensorHandle(handle);
   }
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 56e2e1094f..9e8baab618 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -156,6 +156,23 @@ cc_library(
     hdrs = ["execute.h"],
     deps = [
         ":context",
+        ":copy_to_device_node",
+        ":kernel_and_device",
+        ":tensor_handle",
+        "//tensorflow/core:core_cpu_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:protos_all_cc",
+    ],
+)
+
+cc_library(
+    name = "execute_node",
+    hdrs = ["execute_node.h"],
+    deps = [
+        ":context",
+        ":eager_executor",
+        ":execute",
         ":kernel_and_device",
         ":tensor_handle",
         "//tensorflow/core:core_cpu_lib",
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 9e6dddaa02..4f16e42568 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/copy_to_device_node.h"
 #include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
 #include "tensorflow/core/common_runtime/eager/tensor_handle.h"
 #include "tensorflow/core/framework/step_stats.pb.h"
@@ -107,4 +108,27 @@ Status EagerExecute(EagerContext* ctx, Device* device,
   return Status::OK();
 }
 
+Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx,
+                         const char* device_name, TensorHandle** result) {
+  TF_RETURN_IF_ERROR(ctx->GetStatus());
+  Device* dstd = ctx->HostCPU();
+  if (device_name != nullptr && strlen(device_name) > 0) {
+    TF_RETURN_IF_ERROR(ctx->device_mgr()->LookupDevice(device_name, &dstd));
+  }
+  if (ctx->Async()) {
+    // Note that `h` may not be currently ready. However execution order will
+    // make sure that `h` is ready before the copy is actually done.
+    CopyToDeviceNode* node = new CopyToDeviceNode(h, dstd, ctx);
+    TensorHandle* output = node->dst();
+    // Note that calling Add makes `node` accessible by the EagerExecutor
+    // thread. So further accesses need to be thread-safe.
+    ctx->ExecutorAdd(node);
+    *result = output;
+    return Status::OK();
+  } else {
+    TF_RETURN_IF_ERROR(h->CopyToDevice(ctx, dstd, result));
+    return Status::OK();
+  }
+}
+
 }  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/eager/execute.h b/tensorflow/core/common_runtime/eager/execute.h
index 1c0ea6bcde..0f6ad031e1 100644
--- a/tensorflow/core/common_runtime/eager/execute.h
+++ b/tensorflow/core/common_runtime/eager/execute.h
@@ -32,6 +32,10 @@ Status EagerExecute(EagerContext* ctx, Device* device,
                     KernelAndDevice* kernel, NodeExecStats* maybe_stats,
                     TensorHandle** retvals, int num_retvals);
 
+// Low-level utility to copy a tensor handle from one device to another.
+Status EagerCopyToDevice(TensorHandle* h, EagerContext* ctx,
+                         const char* device_name, TensorHandle** result);
+
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_H_
diff --git a/tensorflow/core/common_runtime/eager/execute_node.h b/tensorflow/core/common_runtime/eager/execute_node.h
new file mode 100644
index 0000000000..93018dd969
--- /dev/null
+++ b/tensorflow/core/common_runtime/eager/execute_node.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_NODE_H_
+#define TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_NODE_H_
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/eager/context.h"
+#include "tensorflow/core/common_runtime/eager/eager_executor.h"
+#include "tensorflow/core/common_runtime/eager/execute.h"
+#include "tensorflow/core/common_runtime/eager/kernel_and_device.h"
+#include "tensorflow/core/common_runtime/eager/tensor_handle.h"
+#include "tensorflow/core/framework/step_stats.pb.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/inlined_vector.h"
+
+namespace tensorflow {
+
+class ExecuteNode : public EagerNode {
+ public:
+  ExecuteNode(uint64 id, EagerContext* ctx, Device* op_device,
+              const tensorflow::gtl::InlinedVector<TensorHandle*, 4>& inputs,
+              KernelAndDevice* kernel, NodeExecStats* maybe_stats,
+              const DataTypeVector& output_dtypes,
+              const tensorflow::gtl::InlinedVector<TensorHandle*, 2>& retvals)
+      : EagerNode(id),
+        ctx_(ctx),
+        op_device_(op_device),
+        inputs_(inputs),
+        kernel_(kernel),
+        maybe_stats_(maybe_stats),
+        retvals_(retvals) {
+    for (auto handle : inputs_) {
+      handle->Ref();
+    }
+    for (auto handle : retvals_) {
+      handle->Ref();
+    }
+  }
+
+  ~ExecuteNode() override {
+    for (auto handle : inputs_) {
+      handle->Unref();
+    }
+    for (auto handle : retvals_) {
+      handle->Unref();
+    }
+  }
+
+  tensorflow::Status Run() override {
+    const Status status =
+        EagerExecute(ctx_, op_device_, inputs_, kernel_, maybe_stats_.get(),
+                     retvals_.begin(), retvals_.size());
+    if (status.ok()) {
+      return status;
+    } else {
+      return Status(status.code(),
+                    strings::StrCat("Got error, \"", status.error_message(),
+                                    "\" while executing kernel ",
+                                    kernel_->kernel()->def().DebugString()));
+    }
+  }
+
+ private:
+  tensorflow::EagerContext* ctx_;
+  tensorflow::Device* op_device_;
+  tensorflow::gtl::InlinedVector<TensorHandle*, 4> inputs_;
+  tensorflow::KernelAndDevice* kernel_;
+  std::unique_ptr<NodeExecStats> maybe_stats_;
+  tensorflow::gtl::InlinedVector<TensorHandle*, 2> retvals_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_COMMON_RUNTIME_EAGER_EXECUTE_NODE_H_
-- 
GitLab


From 0a2a35e210d899d81a7e0478eeb49ea478b05bb8 Mon Sep 17 00:00:00 2001
From: hsm207 <hsm207@users.noreply.github.com>
Date: Wed, 28 Mar 2018 12:05:21 -0400
Subject: [PATCH 604/960] Fix typo (#17947)

---
 tensorflow/python/eager/execution_callbacks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/execution_callbacks.py b/tensorflow/python/eager/execution_callbacks.py
index 535361498a..9a08259653 100644
--- a/tensorflow/python/eager/execution_callbacks.py
+++ b/tensorflow/python/eager/execution_callbacks.py
@@ -253,7 +253,7 @@ def add_execution_callback(callback):
       `f(op_type, op_name, attrs, inputs, outputs)`.
       `op_type` is the type of the operation that was just executed (e.g.,
         `MatMul`).
-      `op_name` is the name of the operation that has was just executed. This
+      `op_name` is the name of the operation that was just executed. This
         name is set by the client who created the operation and can be `None` if
         it is unset.
       `attrs` contains the attributes of the operation as a `tuple` of
-- 
GitLab


From 5a213116df09c19c3ee0eecb5fc79444e5671e80 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Wed, 28 Mar 2018 10:03:06 -0700
Subject: [PATCH 605/960] Allow positional arguments in tf.keras.Model
 subclasses

Makes the tf.keras.Layer.__call__ signature identical to tf.layers.Layer.__call__, but makes passing positional arguments other than "inputs" an error in most cases. The only case it's allowed is subclassed Models which do not have an "inputs" argument to their call() method.

This means subclassed Models no longer need to pass all but the first argument as a keyword argument (or do list packing/unpacking) when call() takes multiple Tensor arguments.

Includes errors for cases where whether an argument indicates an input is ambiguous, but otherwise doesn't do much to support non-"inputs" call() signatures for shape inference or deferred Tensors. The definition of an input/non-input is pretty clear, so that cleanup will mostly be tracking down all of the users of "self.call" and getting them to pass inputs as positional arguments if necessary.

PiperOrigin-RevId: 190787899
---
 .../eager/python/examples/spinn/spinn_test.py |  13 +-
 .../keras/_impl/keras/engine/base_layer.py    |  90 +++++++++++-
 .../keras/_impl/keras/engine/network.py       |   9 +-
 .../keras/_impl/keras/engine/training.py      |   5 +
 .../_impl/keras/model_subclassing_test.py     | 130 +++++++++++++++++-
 tensorflow/python/layers/base.py              |   2 +
 third_party/examples/eager/spinn/spinn.py     |  29 ++--
 7 files changed, 246 insertions(+), 32 deletions(-)

diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 591d99edcd..9261823d77 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -173,7 +173,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
         right_in.append(tf.random_normal((1, size * 2)))
         tracking.append(tf.random_normal((1, tracker_size * 2)))
 
-      out = reducer(left_in, right_in=right_in, tracking=tracking)
+      out = reducer(left_in, right_in, tracking=tracking)
       self.assertEqual(batch_size, len(out))
       self.assertEqual(tf.float32, out[0].dtype)
       self.assertEqual((1, size * 2), out[0].shape)
@@ -227,7 +227,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
       self.assertEqual((batch_size, size * 2), stacks[0][0].shape)
 
       for _ in range(2):
-        out1, out2 = tracker(bufs, stacks=stacks)
+        out1, out2 = tracker(bufs, stacks)
         self.assertIsNone(out2)
         self.assertEqual(batch_size, len(out1))
         self.assertEqual(tf.float32, out1[0].dtype)
@@ -260,7 +260,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
       self.assertEqual(tf.int64, transitions.dtype)
       self.assertEqual((num_transitions, 1), transitions.shape)
 
-      out = s(buffers, transitions=transitions, training=True)
+      out = s(buffers, transitions, training=True)
       self.assertEqual(tf.float32, out.dtype)
       self.assertEqual((1, embedding_dims), out.shape)
 
@@ -286,15 +286,12 @@ class SpinnTest(test_util.TensorFlowTestCase):
                                                          vocab_size)
 
       # Invoke model under non-training mode.
-      logits = model(
-          prem, premise_transition=prem_trans, hypothesis=hypo,
-          hypothesis_transition=hypo_trans, training=False)
+      logits = model(prem, prem_trans, hypo, hypo_trans, training=False)
       self.assertEqual(tf.float32, logits.dtype)
       self.assertEqual((batch_size, d_out), logits.shape)
 
       # Invoke model under training model.
-      logits = model(prem, premise_transition=prem_trans, hypothesis=hypo,
-                     hypothesis_transition=hypo_trans, training=True)
+      logits = model(prem, prem_trans, hypo, hypo_trans, training=True)
       self.assertEqual(tf.float32, logits.dtype)
       self.assertEqual((batch_size, d_out), logits.shape)
 
diff --git a/tensorflow/python/keras/_impl/keras/engine/base_layer.py b/tensorflow/python/keras/_impl/keras/engine/base_layer.py
index 5615241ae3..755607aafb 100644
--- a/tensorflow/python/keras/_impl/keras/engine/base_layer.py
+++ b/tensorflow/python/keras/_impl/keras/engine/base_layer.py
@@ -19,6 +19,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import inspect  # Necessary supplement to tf_inspect to deal with variadic args.
+
 from six.moves import zip  # pylint: disable=redefined-builtin
 
 from tensorflow.python.eager import context
@@ -30,6 +32,8 @@ from tensorflow.python.keras._impl.keras import regularizers
 from tensorflow.python.keras._impl.keras.utils import generic_utils
 from tensorflow.python.layers import base as tf_base_layers
 from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import tf_decorator
+from tensorflow.python.util import tf_inspect
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -143,6 +147,7 @@ class Layer(tf_base_layers.Layer):
     super(Layer, self).__init__(
         name=name, dtype=dtype, trainable=trainable,
         activity_regularizer=kwargs.get('activity_regularizer'))
+    self._uses_inputs_arg = True
 
     # Add properties that are Keras-only for now.
     self.supports_masking = False
@@ -213,7 +218,71 @@ class Layer(tf_base_layers.Layer):
     """
     return inputs
 
-  def __call__(self, inputs, **kwargs):
+  def _inputs_from_call_args(self, call_args, call_kwargs):
+    """Get Layer inputs from __call__ *args and **kwargs.
+
+    Args:
+      call_args: The positional arguments passed to __call__.
+      call_kwargs: The keyword argument dict passed to __call__.
+
+    Returns:
+      A tuple of (inputs, non_input_kwargs). These may be the same objects as
+      were passed in (call_args and call_kwargs).
+    """
+    if getattr(self, '_uses_inputs_arg', True):
+      assert len(call_args) == 1  # TypeError raised earlier in __call__.
+      return call_args[0], call_kwargs
+    else:
+      call_arg_spec = tf_inspect.getargspec(self.call)
+      # There is no explicit "inputs" argument expected or provided to
+      # call(). Arguments which have default values are considered non-inputs,
+      # and arguments without are considered inputs.
+      if call_arg_spec.defaults:
+        if call_arg_spec.varargs is not None:
+          raise TypeError(
+              'Layer.call() may not accept both *args and arguments with '
+              'default values (unable to determine which are inputs to the '
+              'Layer).')
+        keyword_arg_names = set(
+            call_arg_spec.args[-len(call_arg_spec.defaults):])
+      else:
+        keyword_arg_names = set()
+        # Training is never an input argument name, to allow signatures like
+        # call(x, training).
+      keyword_arg_names.add('training')
+      _, unwrapped_call = tf_decorator.unwrap(self.call)
+      bound_args = inspect.getcallargs(
+          unwrapped_call, *call_args, **call_kwargs)
+      if call_arg_spec.keywords is not None:
+        var_kwargs = bound_args.pop(call_arg_spec.keywords)
+        bound_args.update(var_kwargs)
+        keyword_arg_names = keyword_arg_names.union(var_kwargs.keys())
+      all_args = call_arg_spec.args
+      if all_args and bound_args[all_args[0]] is self:
+        # Ignore the 'self' argument of methods
+        bound_args.pop(call_arg_spec.args[0])
+        all_args = all_args[1:]
+      non_input_arg_values = {}
+      input_arg_values = []
+      remaining_args_are_keyword = False
+      for argument_name in all_args:
+        if argument_name in keyword_arg_names:
+          remaining_args_are_keyword = True
+        else:
+          if remaining_args_are_keyword:
+            raise TypeError(
+                'Found a positional argument to call() after a non-input '
+                'argument. All arguments after "training" must be keyword '
+                'arguments, and are not tracked as inputs to the Layer.')
+        if remaining_args_are_keyword:
+          non_input_arg_values[argument_name] = bound_args[argument_name]
+        else:
+          input_arg_values.append(bound_args[argument_name])
+      if call_arg_spec.varargs is not None:
+        input_arg_values.extend(bound_args[call_arg_spec.varargs])
+      return input_arg_values, non_input_arg_values
+
+  def __call__(self, inputs, *args, **kwargs):
     """Wrapper around self.call(), for handling internal references.
 
     If a Keras tensor is passed:
@@ -226,6 +295,10 @@ class Layer(tf_base_layers.Layer):
 
     Arguments:
         inputs: Can be a tensor or list/tuple of tensors.
+        *args: Additional positional arguments to be passed to `call()`. Only
+          allowed in subclassed Models with custom call() signatures. In other
+          cases, `Layer` inputs must be passed using the `inputs` argument and
+          non-inputs must be keyword arguments.
         **kwargs: Additional keyword arguments to be passed to `call()`.
 
     Returns:
@@ -234,12 +307,25 @@ class Layer(tf_base_layers.Layer):
     Raises:
         ValueError: in case the layer is missing shape information
             for its `build` call.
+        TypeError: If positional arguments are passed and this `Layer` is not a
+            subclassed `Model`.
     """
     # Actually call the layer (optionally building it).
-    output = super(Layer, self).__call__(inputs, **kwargs)
+    output = super(Layer, self).__call__(inputs, *args, **kwargs)
+
+    if args and getattr(self, '_uses_inputs_arg', True):
+      raise TypeError(
+          'This Layer takes an `inputs` argument to call(), and only the '
+          '`inputs` argument may be specified as a positional argument. Pass '
+          'everything else as a keyword argument (those arguments will not be '
+          'tracked as inputs to the Layer).')
+
     if context.executing_eagerly():
       return output
 
+    inputs, kwargs = self._inputs_from_call_args(
+        call_args=(inputs,) + args, call_kwargs=kwargs)
+
     if hasattr(self, '_symbolic_set_inputs') and not self.inputs:
       # Subclassed network: explicitly set metadata normally set by a call to
       # self._set_inputs().
diff --git a/tensorflow/python/keras/_impl/keras/engine/network.py b/tensorflow/python/keras/_impl/keras/engine/network.py
index ea4be0d293..9f1c7de115 100644
--- a/tensorflow/python/keras/_impl/keras/engine/network.py
+++ b/tensorflow/python/keras/_impl/keras/engine/network.py
@@ -117,6 +117,7 @@ class Network(base_layer.Layer):
     self._inbound_nodes = []
 
   def _init_graph_network(self, inputs, outputs, name=None):
+    self._uses_inputs_arg = True
     # Normalize and set self.inputs, self.outputs.
     if isinstance(inputs, (list, tuple)):
       self.inputs = list(inputs)  # Tensor or list of tensors.
@@ -274,11 +275,15 @@ class Network(base_layer.Layer):
   def _init_subclassed_network(self, name=None):
     self._base_init(name=name)
     self._is_graph_network = False
-    if 'training' in tf_inspect.getargspec(self.call).args:
+    call_args = tf_inspect.getargspec(self.call).args
+    if 'training' in call_args:
       self._expects_training_arg = True
     else:
       self._expects_training_arg = False
-
+    if 'inputs' in call_args:
+      self._uses_inputs_arg = True
+    else:
+      self._uses_inputs_arg = False
     self.outputs = None
     self.inputs = None
     self.built = False
diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py
index 08288d353e..971245c162 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training.py
@@ -874,6 +874,11 @@ class Model(Network):
         whether to build the model's graph in inference mode (False), training
         mode (True), or using the Keras learning phase (None).
     """
+    if not getattr(self, '_uses_inputs_arg', True):
+      raise NotImplementedError(
+          'Subclassed Models without "inputs" in their call() signatures do '
+          'not yet support shape inference. File a feature request if this '
+          'limitation bothers you.')
     if self.__class__.__name__ == 'Sequential':
       # Note: we can't test whether the model is `Sequential` via `isinstance`
       # since `Sequential` depends on `Model`.
diff --git a/tensorflow/python/keras/_impl/keras/model_subclassing_test.py b/tensorflow/python/keras/_impl/keras/model_subclassing_test.py
index 58b144365b..4445900330 100644
--- a/tensorflow/python/keras/_impl/keras/model_subclassing_test.py
+++ b/tensorflow/python/keras/_impl/keras/model_subclassing_test.py
@@ -22,7 +22,9 @@ import os
 import tempfile
 
 import numpy as np
+import six
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.keras._impl import keras
@@ -36,6 +38,7 @@ except ImportError:
   h5py = None
 
 
+# pylint: disable=not-callable
 class SimpleTestModel(keras.Model):
 
   def __init__(self, use_bn=False, use_dp=False, num_classes=10):
@@ -104,7 +107,7 @@ class NestedTestModel1(keras.Model):
   def call(self, inputs):
     x = self.dense1(inputs)
     x = self.bn(x)
-    x = self.test_net(x)  # pylint: disable=not-callable
+    x = self.test_net(x)
     return self.dense2(x)
 
 
@@ -161,7 +164,7 @@ def get_nested_model_3(input_dim, num_classes):
       return tensor_shape.TensorShape((input_shape[0], 5))
 
   test_model = Inner()
-  x = test_model(x)  # pylint: disable=not-callable
+  x = test_model(x)
   outputs = keras.layers.Dense(num_classes)(x)
   return keras.Model(inputs, outputs, name='nested_model_3')
 
@@ -574,5 +577,128 @@ class ModelSubclassingTest(test.TestCase):
     self.assertGreater(loss, 0.1)
 
 
+class CustomCallModel(keras.Model):
+
+  def __init__(self):
+    super(CustomCallModel, self).__init__()
+    self.dense1 = keras.layers.Dense(1, activation='relu')
+    self.dense2 = keras.layers.Dense(1, activation='softmax')
+
+  def call(self, first, second, fiddle_with_output='no', training=True):
+    combined = self.dense1(first) + self.dense2(second)
+    if fiddle_with_output == 'yes':
+      return 10. * combined
+    else:
+      return combined
+
+
+class CustomCallSignatureTests(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_no_inputs_in_signature(self):
+    model = CustomCallModel()
+    first = array_ops.ones([2, 3])
+    second = array_ops.ones([2, 5])
+    output = model(first, second)
+    self.evaluate([v.initializer for v in model.variables])
+    expected_output = self.evaluate(model.dense1(first) + model.dense2(second))
+    self.assertAllClose(expected_output, self.evaluate(output))
+    output = model(first, second, fiddle_with_output='yes')
+    self.assertAllClose(10. * expected_output, self.evaluate(output))
+    output = model(first, second=second, training=False)
+    self.assertAllClose(expected_output, self.evaluate(output))
+    if not context.executing_eagerly():
+      six.assertCountEqual(self, [first, second], model.inputs)
+    with self.assertRaises(TypeError):
+      # tf.layers.Layer expects an "inputs" argument, so all-keywords doesn't
+      # work at the moment.
+      model(first=first, second=second, fiddle_with_output='yes')
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_inputs_in_signature(self):
+
+    class HasInputsAndOtherPositional(keras.Model):
+
+      def call(self, inputs, some_other_arg, training=False):
+        return inputs
+
+    model = HasInputsAndOtherPositional()
+    with self.assertRaisesRegexp(
+        TypeError, 'everything else as a keyword argument'):
+      model(array_ops.ones([]), array_ops.ones([]))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_kwargs_in_signature(self):
+
+    class HasKwargs(keras.Model):
+
+      def call(self, x, y=3, **key_words):
+        return x
+
+    model = HasKwargs()
+    arg = array_ops.ones([])
+    model(arg, a=3)
+    if not context.executing_eagerly():
+      six.assertCountEqual(self, [arg], model.inputs)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def test_args_in_signature(self):
+
+    class HasArgs(keras.Model):
+
+      def call(self, x, *args, **kwargs):
+        return [x] + list(args)
+
+    model = HasArgs()
+    arg1 = array_ops.ones([])
+    arg2 = array_ops.ones([])
+    arg3 = array_ops.ones([])
+    model(arg1, arg2, arg3, a=3)
+    if not context.executing_eagerly():
+      six.assertCountEqual(self, [arg1, arg2, arg3], model.inputs)
+
+  def test_args_and_keywords_in_signature(self):
+
+    class HasArgs(keras.Model):
+
+      def call(self, x, training=True, *args, **kwargs):
+        return x
+
+    with context.graph_mode():
+      model = HasArgs()
+      arg1 = array_ops.ones([])
+      arg2 = array_ops.ones([])
+      arg3 = array_ops.ones([])
+      with self.assertRaisesRegexp(TypeError, 'args and arguments with'):
+        model(arg1, arg2, arg3, a=3)
+
+  def test_training_no_default(self):
+
+    class TrainingNoDefault(keras.Model):
+
+      def call(self, x, training):
+        return x
+
+    with context.graph_mode():
+      model = TrainingNoDefault()
+      arg = array_ops.ones([])
+      model(arg, True)
+      six.assertCountEqual(self, [arg], model.inputs)
+
+  def test_training_no_default_with_positional(self):
+
+    class TrainingNoDefaultWithPositional(keras.Model):
+
+      def call(self, x, training, positional):
+        return x
+
+    with context.graph_mode():
+      model = TrainingNoDefaultWithPositional()
+      arg1 = array_ops.ones([])
+      arg2 = array_ops.ones([])
+      arg3 = array_ops.ones([])
+      with self.assertRaisesRegexp(TypeError, 'after a non-input'):
+        model(arg1, arg2, arg3)
+
 if __name__ == '__main__':
   test.main()
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 1e5f26a77f..242cdff6f3 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -625,6 +625,8 @@ class Layer(checkpointable.CheckpointableBase):
     input_list = nest.flatten(inputs)
 
     build_graph = not context.executing_eagerly()
+    # TODO(fchollet, allenl): Make deferred mode work with subclassed Models
+    # which don't use an "inputs" argument.
     in_deferred_mode = isinstance(input_list[0], _DeferredTensor)
     # Ensure the Layer, if being reused, is working with inputs from
     # the same graph as where it was created.
diff --git a/third_party/examples/eager/spinn/spinn.py b/third_party/examples/eager/spinn/spinn.py
index f8fb6ecb0c..8a2b24aa4e 100644
--- a/third_party/examples/eager/spinn/spinn.py
+++ b/third_party/examples/eager/spinn/spinn.py
@@ -266,8 +266,7 @@ class SPINN(tf.keras.Model):
           trackings.append(tracking)
 
       if rights:
-        reducer_output = self.reducer(
-            lefts, right_in=rights, tracking=trackings)
+        reducer_output = self.reducer(lefts, rights, trackings)
         reduced = iter(reducer_output)
 
         for transition, stack in zip(trans, stacks):
@@ -388,10 +387,10 @@ class SNLIClassifier(tf.keras.Model):
 
     # Run the batch-normalized and dropout-processed word vectors through the
     # SPINN encoder.
-    premise = self.encoder(
-        premise_embed, transitions=premise_transition, training=training)
-    hypothesis = self.encoder(
-        hypothesis_embed, transitions=hypothesis_transition, training=training)
+    premise = self.encoder(premise_embed, premise_transition,
+                           training=training)
+    hypothesis = self.encoder(hypothesis_embed, hypothesis_transition,
+                              training=training)
 
     # Combine encoder outputs for premises and hypotheses into logits.
     # Then apply batch normalization and dropuout on the logits.
@@ -465,11 +464,10 @@ class SNLIClassifierTrainer(tfe.Checkpointable):
     """
     with tfe.GradientTape() as tape:
       tape.watch(self._model.variables)
-      # TODO(allenl): Allow passing Layer inputs as position arguments.
       logits = self._model(premise,
-                           premise_transition=premise_transition,
-                           hypothesis=hypothesis,
-                           hypothesis_transition=hypothesis_transition,
+                           premise_transition,
+                           hypothesis,
+                           hypothesis_transition,
                            training=True)
       loss = self.loss(labels, logits)
     gradients = tape.gradient(loss, self._model.variables)
@@ -533,9 +531,7 @@ def _evaluate_on_dataset(snli_data, batch_size, trainer, use_gpu):
       snli_data, batch_size):
     if use_gpu:
       label, prem, hypo = label.gpu(), prem.gpu(), hypo.gpu()
-    logits = trainer.model(
-        prem, premise_transition=prem_trans, hypothesis=hypo,
-        hypothesis_transition=hypo_trans, training=False)
+    logits = trainer.model(prem, prem_trans, hypo, hypo_trans, training=False)
     loss_val = trainer.loss(label, logits)
     batch_size = tf.shape(label)[0]
     mean_loss(loss_val, weights=batch_size.gpu() if use_gpu else batch_size)
@@ -639,11 +635,8 @@ def train_or_infer_spinn(embed,
       hypo, hypo_trans = inference_sentence_pair[1]
       hypo_trans = inference_sentence_pair[1][1]
       inference_logits = model(
-          tf.constant(prem),
-          premise_transition=tf.constant(prem_trans),
-          hypothesis=tf.constant(hypo),
-          hypothesis_transition=tf.constant(hypo_trans),
-          training=False)
+          tf.constant(prem), tf.constant(prem_trans),
+          tf.constant(hypo), tf.constant(hypo_trans), training=False)
       inference_logits = inference_logits[0][1:]
       max_index = tf.argmax(inference_logits)
       print("\nInference logits:")
-- 
GitLab


From 23d6c55ba22596f903696e0dba8037ad81470a39 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 10:03:37 -0700
Subject: [PATCH 606/960] Enable the Grappler arithmetic optimizer by default
 in Python tests.

PiperOrigin-RevId: 190787954
---
 tensorflow/python/framework/test_util.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 43106b6e59..4192a27f65 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -968,8 +968,6 @@ class TensorFlowTestCase(googletest.TestCase):
       config.graph_options.optimizer_options.opt_level = -1
       config.graph_options.rewrite_options.constant_folding = (
           rewriter_config_pb2.RewriterConfig.OFF)
-      config.graph_options.rewrite_options.arithmetic_optimization = (
-          rewriter_config_pb2.RewriterConfig.OFF)
       return config
 
     if graph is None:
-- 
GitLab


From 2d2d4529fe3b99f25e531ee2fc0e9211341c160f Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 10:15:42 -0700
Subject: [PATCH 607/960] Avoid overwriting existing namespace items that might
 replace the converted functions.

PiperOrigin-RevId: 190789781
---
 tensorflow/contrib/autograph/impl/api.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py
index 1c4fcaa622..dce994e50d 100644
--- a/tensorflow/contrib/autograph/impl/api.py
+++ b/tensorflow/contrib/autograph/impl/api.py
@@ -247,7 +247,10 @@ def to_graph(e,
   # The compiled code should see everything the entry function saw.
   # TODO(mdan): This might not work well if the call tree spans modules?
   if tf_inspect.isfunction(e):
-    compiled_node.__dict__.update(inspect_utils.getnamespace(e))
+    for key, val in inspect_utils.getnamespace(e).items():
+      # Avoid overwriting entities that have been transformed.
+      if key not in compiled_node.__dict__:
+        compiled_node.__dict__[key] = val
   compiled_fn = getattr(compiled_node, name)
 
   if verbose:
-- 
GitLab


From 1fe68ce0f4f7ef020cc52d1cc9963dd344fccba0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 10:15:46 -0700
Subject: [PATCH 608/960] internal change

PiperOrigin-RevId: 190789794
---
 tensorflow/core/framework/resource_mgr.h | 15 +++++++--------
 1 file changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/core/framework/resource_mgr.h b/tensorflow/core/framework/resource_mgr.h
index 9a458431e7..c84ea3b034 100644
--- a/tensorflow/core/framework/resource_mgr.h
+++ b/tensorflow/core/framework/resource_mgr.h
@@ -319,14 +319,13 @@ class IsResourceInitialized : public OpKernel {
 // specified type. The type will be a part of the generated op name.
 // TODO(apassos): figure out how to get non-cpu-allocated tensors to work
 // through constant folding so this doesn't have to be marked as stateful.
-#define REGISTER_RESOURCE_HANDLE_OP(Type)                   \
-  REGISTER_OP(#Type "HandleOp")                             \
-      .Attr("container: string = ''")                       \
-      .Attr("shared_name: string = ''")                     \
-      .Output("resource: resource")                         \
-      .SetIsStateful()                                      \
-      .SetShapeFn(tensorflow::shape_inference::ScalarShape) \
-      .Doc("Creates a handle to a " #Type)
+#define REGISTER_RESOURCE_HANDLE_OP(Type) \
+  REGISTER_OP(#Type "HandleOp")           \
+      .Attr("container: string = ''")     \
+      .Attr("shared_name: string = ''")   \
+      .Output("resource: resource")       \
+      .SetIsStateful()                    \
+      .SetShapeFn(tensorflow::shape_inference::ScalarShape)
 
 // Utility op kernel to produce a handle to a resource of type T.
 template <typename T>
-- 
GitLab


From c3603d0a0d9a8390fd9e9423cd661717d2702bfd Mon Sep 17 00:00:00 2001
From: Amit Patankar <amitpatankar@google.com>
Date: Wed, 28 Mar 2018 10:20:49 -0700
Subject: [PATCH 609/960] Trying to fix libtensorflow GPU build. (#18056)

CUDNN path error.
Invalid path to cuDNN 7 toolkit. None of the following files can be found:
C:/tools/cuda\lib/x64/cudnn.lib
C:/tools/cuda\lib/x64/cudnn.lib
---
 tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
index 7b2d7e1a56..d654b433e7 100644
--- a/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
+++ b/tensorflow/tools/ci_build/windows/bazel/bazel_test_lib.sh
@@ -120,7 +120,9 @@ function run_configure_for_gpu_build {
   export TF_CUDA_VERSION=9.0
   export CUDA_TOOLKIT_PATH="C:/Program Files/NVIDIA GPU Computing Toolkit/CUDA/v9.0"
   export TF_CUDNN_VERSION=7.0
-  export CUDNN_INSTALL_PATH="C:/tools/cuda"
+  if [ -z "$CUDNN_INSTALL_PATH" ]; then
+    export CUDNN_INSTALL_PATH="C:/tools/cuda"
+  fi
   export TF_CUDA_COMPUTE_CAPABILITIES="3.7"
   if [ -z "$TF_ENABLE_XLA" ]; then
     export TF_ENABLE_XLA=0
-- 
GitLab


From b1f6c4e0dee8732d8e25262052656e9cdf5ca513 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Wed, 28 Mar 2018 10:25:27 -0700
Subject: [PATCH 610/960] Properly serialize ResourceVariable global_step into
 the metagraph.

Prior to this, saving and restoring a graph with a resource variable global_step
would cause the global_step collection of the reimported graph to contain
a resource tensor (the object underlying the ResourceVariable); the actual
metadata associated with it would be serialized.

PiperOrigin-RevId: 190791443
---
 tensorflow/python/ops/resource_variable_ops.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index df873da98e..2f39ea2e7d 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -1087,6 +1087,11 @@ ops.register_proto_function(
     proto_type=variable_pb2.VariableDef,
     to_proto=_to_proto_fn,
     from_proto=_from_proto_fn)
+ops.register_proto_function(
+    ops.GraphKeys.GLOBAL_STEP,
+    proto_type=variable_pb2.VariableDef,
+    to_proto=_to_proto_fn,
+    from_proto=_from_proto_fn)
 
 
 def is_resource_variable(var):
-- 
GitLab


From 0aaa61ab332611e9dcfd3d1cc25115a8972bd5fd Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 28 Mar 2018 10:55:00 -0700
Subject: [PATCH 611/960] Update version strings to 1.7

---
 tensorflow/core/public/version.h              |  2 +-
 tensorflow/docs_src/install/install_c.md      |  2 +-
 tensorflow/docs_src/install/install_go.md     |  2 +-
 tensorflow/docs_src/install/install_java.md   | 22 +++++++++----------
 tensorflow/docs_src/install/install_linux.md  | 22 +++++++++----------
 tensorflow/docs_src/install/install_mac.md    | 10 ++++-----
 .../docs_src/install/install_sources.md       | 14 ++++++------
 tensorflow/tools/pip_package/setup.py         |  2 +-
 8 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 40eebd1db0..706968d347 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -24,7 +24,7 @@ limitations under the License.
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX "-rc1"
+#define TF_VERSION_SUFFIX ""
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 9059b3f3b6..a3eca4bf37 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 2e47a6d212..1a0956634d 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index eff066d200..cdde45a6f4 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.7.0</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.7.0-rc1</version>
+                 <version>1.7.0</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.7.0</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.7.0</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.7.0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 3c5db9bced..fdf9bf81e7 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -199,7 +199,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -304,7 +304,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -490,7 +490,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -657,14 +657,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -676,14 +676,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -695,14 +695,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -714,14 +714,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index fa6951a8f1..6f55e6a650 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -119,7 +119,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      TensorFlow in the active Virtualenv is as follows:
 
      <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -242,7 +242,7 @@ take the following steps:
      issue the following command:
 
      <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b> </pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -523,7 +523,7 @@ This section documents the relevant values for Mac OS installations.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py2-none-any.whl
 </pre>
 
 
@@ -531,5 +531,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 0454c172f8..73446663e9 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -359,10 +359,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.7.0rc1 on Linux:
+for TensorFlow 1.7.0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -459,8 +459,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
+<tr><td>tensorflow-1.7.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -480,7 +480,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow-1.7.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -495,8 +495,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
+<tr><td>tensorflow-1.7.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 8b83257887..a486631621 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.7.0-rc1'
+_VERSION = '1.7.0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
-- 
GitLab


From 95efef3271d67dd63ec2e397012a20d63d088668 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 11:18:45 -0700
Subject: [PATCH 612/960] Make ArithmeticOptimizer robust to failures of shape
 inference and individual stages.

Get rid of graph annotation and use GraphProperties directly.

PiperOrigin-RevId: 190801044
---
 .../optimizers/arithmetic_optimizer.cc        | 49 +++++++++++--------
 .../optimizers/arithmetic_optimizer.h         |  1 +
 .../optimizers/graph_optimizer_stage.cc       |  4 ++
 .../optimizers/graph_optimizer_stage.h        |  3 ++
 4 files changed, 36 insertions(+), 21 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 5dd0b6f4b0..629872bf19 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -196,8 +196,6 @@ void SetSourceDataType(DataType dtype, NodeDef* node) {
 
 bool IsNumberType(DataType dtype) { return kNumberTypes.Contains(dtype); }
 
-const char kOutputShapesAttr[] = "_output_shapes";
-
 // Shape is symbolically defined if it has a known rank, and each dimension is
 // defined, or is an unknown symbol (dim.size <= -2).
 bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
@@ -234,16 +232,19 @@ bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
 // Returns whether `reshape` is an identity op. The tensor that `reshape`
 // reshapes is the `output_pos`-th output of node `input`.
 bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
-                       const int output_pos) {
-  if (!reshape.attr().count(kOutputShapesAttr) ||
-      !input.attr().count(kOutputShapesAttr)) {
+                       const int output_pos,
+                       const GraphProperties& graph_properties) {
+  const std::vector<OpInfo::TensorProperties>& reshape_props =
+      graph_properties.GetOutputProperties(reshape.name());
+  const std::vector<OpInfo::TensorProperties>& input_props =
+      graph_properties.GetOutputProperties(input.name());
+  if (reshape_props.empty() || input_props.empty() ||
+      input_props.size() <= output_pos) {
     return false;
   }
 
-  PartialTensorShape src_shape(
-      input.attr().at(kOutputShapesAttr).list().shape(output_pos));
-  PartialTensorShape dst_shape(
-      reshape.attr().at(kOutputShapesAttr).list().shape(0));
+  const PartialTensorShape& src_shape = input_props[output_pos].shape();
+  const PartialTensorShape& dst_shape = reshape_props[0].shape();
   if (src_shape.unknown_rank() || dst_shape.unknown_rank()) {
     return false;
   }
@@ -1272,7 +1273,8 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     // outputs tensors of shape [M, N] while feeding it with tensors of shape
     // [M*N] (or worse). The reshape nodes are then necessary to update the
     // tensor metadata to the required shape.
-    if (ReshapeIsIdentity(*reshape, *input, output_pos)) {
+    if (can_use_shapes_ &&
+        ReshapeIsIdentity(*reshape, *input, output_pos, *graph_properties_)) {
       return reshape->input(0);
     }
   }
@@ -1586,11 +1588,11 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
 
   std::vector<std::unique_ptr<ArithmeticOptimizerStage>> stages;
 
-  if (options_.combine_add_to_addn) {
+  if (options_.combine_add_to_addn && can_use_shapes_) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new AddOpsRewriteStage(ctx, ctx_ext)));
   }
-  if (options_.hoist_common_factor_out_of_aggregation) {
+  if (options_.hoist_common_factor_out_of_aggregation && can_use_shapes_) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new HoistCommonFactorOutOfAggregation(ctx, ctx_ext)));
   }
@@ -1627,7 +1629,15 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     if (simplified_tensor.empty()) {
       for (auto& stage : stages) {
         if (stage->IsSupported(node)) {
-          TF_RETURN_IF_ERROR(stage->TrySimplify(node, &simplified_tensor));
+          const Status stage_status =
+              stage->TrySimplify(node, &simplified_tensor);
+          // Each stage must be "error safe" (just like exception safe). In
+          // case of any error it must leave optimized graph unmodified.
+          if (!stage_status.ok()) {
+            LOG(WARNING) << "Failed to run arithmetic optimizer stage "
+                         << stage->stage_name()
+                         << ". Error: " << stage_status.error_message();
+          }
           if (!simplified_tensor.empty()) {
             break;
           }
@@ -1694,19 +1704,16 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
                                                &frame_map_, &num_frames));
   // Shapes are only needed in aggressive mode.
   graph_properties_.reset(new GraphProperties(item));
-  TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false));
-  // TODO(ezhulenev): Use GraphProperties to lookup tensor shapes directly
-  TF_RETURN_IF_ERROR(graph_properties_->AnnotateOutputShapes(optimized_graph_));
+  const Status status = graph_properties_->InferStatically(false);
+  can_use_shapes_ = status.ok();
+  if (!can_use_shapes_) {
+    LOG(WARNING) << "Shape inference failed.";
+  }
 
   // Perform the optimizations.
   DedupComputations();
   TF_RETURN_IF_ERROR(SimplifyArithmeticOps());
 
-  // Clear output shapes.
-  for (int i = 0; i < optimized_graph->node_size(); ++i) {
-    optimized_graph_->mutable_node(i)->mutable_attr()->erase(kOutputShapesAttr);
-  }
-
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 965f0e9ea2..cdeed0554e 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -126,6 +126,7 @@ class ArithmeticOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   ArithmeticOptimizerOptions options_;
 
+  bool can_use_shapes_ = false;
   bool fetch_nodes_known_ = false;
   std::unordered_set<string> nodes_to_preserve_;
   std::unique_ptr<NodeMap> node_map_;
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
index 7044705ade..1ea57f7b4f 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
@@ -42,6 +42,10 @@ Status GetInputNode(const GraphOptimizerContext& ctx, const string& input,
 Status GetTensorProperties(const GraphOptimizerContext& ctx,
                            const string& tensor,
                            OpInfo::TensorProperties* properties) {
+  if (ctx.graph_properties == nullptr) {
+    return errors::InvalidArgument("Graph properties are unknown.");
+  }
+
   int port;
   string tensor_node_name = ParseNodeName(tensor, &port);
   if (port < 0) {
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
index be95c00d2d..c7af82abbb 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -117,6 +117,9 @@ class GraphOptimizerStage {
       : optimizer_name_(optimizer_name), stage_name_(stage_name), ctx_(ctx) {}
   virtual ~GraphOptimizerStage() = default;
 
+  const string& stage_name() const { return stage_name_; }
+  const string& optimizer_name() const { return optimizer_name_; }
+
   // Check if we should try to simplify node. Returning true doesn't
   // guarantee that node will be simplified.
   //
-- 
GitLab


From 71b917851b8fcd36481306d225fa478e9e6f7b83 Mon Sep 17 00:00:00 2001
From: Chris Ying <chrisying@google.com>
Date: Wed, 28 Mar 2018 11:22:28 -0700
Subject: [PATCH 613/960] Fix TPUClusterResolver tpu parameter for profiler
 tool.

PiperOrigin-RevId: 190801968
---
 .../contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
index a730d6142d..0b78cf8695 100644
--- a/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
+++ b/tensorflow/contrib/tpu/profiler/pip_package/cloud_tpu_profiler/main.py
@@ -76,7 +76,7 @@ def main(unused_argv=None):
   else:
     tpu_cluster_resolver = (
         tf.contrib.cluster_resolver.TPUClusterResolver(
-            tpu_names=[FLAGS.tpu_name],
+            [FLAGS.tpu_name],
             zone=FLAGS.tpu_zone,
             project=FLAGS.gcp_project))
     service_addr = tpu_cluster_resolver.get_master()
-- 
GitLab


From b8384bbe0325c5b1c20838f9e6fd494e78e299dc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 11:24:01 -0700
Subject: [PATCH 614/960] Updating tests in constant_folding_test.cc so that it
 evaluates the optimized and original graph and checks whether the output
 tensors produced by them are the same.

PiperOrigin-RevId: 190802264
---
 .../optimizers/constant_folding_test.cc       | 62 +++++++++++--------
 .../core/grappler/utils/grappler_test.cc      |  5 ++
 .../core/grappler/utils/grappler_test.h       |  3 +
 3 files changed, 44 insertions(+), 26 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 85f877883c..e0ff9b17b1 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -107,8 +107,8 @@ TEST_F(ConstantFoldingTest, SimpleFolding) {
   EXPECT_EQ("Const", node_d.op());
 
   std::vector<string> fetch = {"d"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(1, tensors_expected.size());
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
@@ -193,10 +193,10 @@ TEST_F(ConstantFoldingTest, AddTree) {
 
   // Check that the result nodes have the expected value.
   std::vector<string> fetch = {"c3", "c20"};
-  auto tensor_expected = EvaluateNodes(item.graph, fetch, {});
+  auto tensor_expected = EvaluateNodes(item.graph, fetch);
   EXPECT_EQ(fetch.size(), tensor_expected.size());
   fetch = {"add_child", "mul_child"};
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
     test::ExpectTensorEqual<float>(tensor_expected[i], tensors[i]);
@@ -436,10 +436,10 @@ TEST_F(ConstantFoldingTest, StrengthReduce_Reciprocal) {
 
   // Check that the reciprocals have the expected value.
   std::vector<string> fetch = {"cf_half"};
-  auto tensor_expected = EvaluateNodes(item.graph, fetch, {});
+  auto tensor_expected = EvaluateNodes(item.graph, fetch);
   EXPECT_EQ(fetch.size(), tensor_expected.size());
   fetch = {"ConstantFolding/div_f_recip", "ConstantFolding/realdiv_recip"};
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
     test::ExpectTensorEqual<float>(tensor_expected[0], tensors[i]);
@@ -647,8 +647,8 @@ TEST_F(ConstantFoldingTest, FoldingNodeWithTwoOutputs) {
   EXPECT_EQ("Const", new_d.op());
 
   std::vector<string> fetch = {"e", "f"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(fetch.size(), tensors_expected.size());
   EXPECT_EQ(fetch.size(), tensors.size());
   for (int i = 0; i < fetch.size(); i++) {
@@ -671,7 +671,7 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
   GrapplerItem item;
   item.fetch.push_back("e");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
-  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {});
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
   EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
@@ -688,8 +688,8 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
     if (node.name() == "e") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"e"}, {});
-      auto expected = EvaluateNodes(item.graph, {"e"}, {});
+      auto folded = EvaluateNodes(output, {"e"});
+      auto expected = EvaluateNodes(item.graph, {"e"});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -699,7 +699,7 @@ TEST_F(ConstantFoldingTest, ControlDependencies) {
     }
   }
   EXPECT_EQ(1, found);
-  auto tensors = EvaluateNodes(output, item.fetch, {});
+  auto tensors = EvaluateNodes(output, item.fetch);
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorEqual<int>(tensors_expected[0], tensors[0]);
 }
@@ -735,8 +735,8 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
     if (node.name() == "i1") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"i1"}, {});
-      auto expected = EvaluateNodes(item.graph, {"i1"}, {});
+      auto folded = EvaluateNodes(output, {"i1"});
+      auto expected = EvaluateNodes(item.graph, {"i1"});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -746,8 +746,8 @@ TEST_F(ConstantFoldingTest, ControlDependenciesEmptyFetch) {
     if (node.name() == "i2") {
       EXPECT_EQ("Const", node.op());
       ++found;
-      auto folded = EvaluateNodes(output, {"i2"}, {});
-      auto expected = EvaluateNodes(item.graph, {"i2"}, {});
+      auto folded = EvaluateNodes(output, {"i2"});
+      auto expected = EvaluateNodes(item.graph, {"i2"});
       EXPECT_EQ(1, expected.size());
       EXPECT_EQ(1, folded.size());
       test::ExpectTensorEqual<int>(folded[0], expected[0]);
@@ -775,7 +775,8 @@ TEST_F(ConstantFoldingTest, ControlDependenciesDeduplicate) {
   GrapplerItem item;
   item.fetch.push_back("i2");
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
-
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -794,6 +795,9 @@ TEST_F(ConstantFoldingTest, ControlDependenciesDeduplicate) {
       EXPECT_EQ("^p2", node.input(1));
     }
   }
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<int>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ConstantFoldingTest, VariableNumberOfOutputs) {
@@ -865,8 +869,8 @@ TEST_F(ConstantFoldingTest, VariableNumberOfOutputs) {
   }
   EXPECT_EQ(8, constant_folded);
 
-  auto expected = EvaluateNodes(item.graph, outputs, {});
-  auto optimized = EvaluateNodes(output, outputs, {});
+  auto expected = EvaluateNodes(item.graph, outputs);
+  auto optimized = EvaluateNodes(output, outputs);
   ASSERT_EQ(expected.size(), optimized.size());
   for (int i = 0; i < expected.size(); ++i) {
     test::ExpectTensorEqual<int>(expected[i], optimized[i]);
@@ -1293,7 +1297,7 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   EXPECT_EQ(6, found_nodes);
 
   std::vector<string> fetch = {"out1", "idx1"};
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(2, tensors.size());
   const Tensor& out_value = tensors[0];
   EXPECT_EQ(3 * 5, out_value.NumElements());
@@ -1803,6 +1807,12 @@ TEST_F(ConstantFoldingTest, LargeConstant) {
   EXPECT_EQ(2, found);
 
   EXPECT_GT(1024 * 1024, output.ByteSizeLong());
+
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ConstantFoldingTest, SwitchIdenticalInputs) {
@@ -1948,8 +1958,8 @@ TEST_F(ConstantFoldingTest, PartialFolding_AssociativeAndCommutative) {
     }
 
     std::vector<string> fetch = {"acc0"};
-    auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
-    auto tensors = EvaluateNodes(output, fetch, {});
+    auto tensors_expected = EvaluateNodes(item.graph, fetch);
+    auto tensors = EvaluateNodes(output, fetch);
     EXPECT_EQ(1, tensors_expected.size());
     EXPECT_EQ(1, tensors.size());
     test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
@@ -1983,7 +1993,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
   item.fetch = {"concat0", "concat1", "concat2", "concat3", "concat4",
                 "concat5", "concat6", "concat7", "concat8", "concat9"};
 
-  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"}, {});
+  auto tensors_expected = EvaluateNodes(item.graph, {"concat0"});
   EXPECT_EQ(1, tensors_expected.size());
   ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
@@ -2034,7 +2044,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
     }
   }
 
-  auto tensors = EvaluateNodes(output, {"concat0"}, {});
+  auto tensors = EvaluateNodes(output, {"concat0"});
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
@@ -2132,8 +2142,8 @@ TEST_F(ConstantFoldingTest, TrivialPack) {
   }
 
   std::vector<string> fetch = {"stack"};
-  auto tensors_expected = EvaluateNodes(item.graph, fetch, {});
-  auto tensors = EvaluateNodes(output, fetch, {});
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  auto tensors = EvaluateNodes(output, fetch);
   EXPECT_EQ(1, tensors_expected.size());
   EXPECT_EQ(1, tensors.size());
   EXPECT_EQ(tensors_expected[0].shape(), tensors[0].shape());
diff --git a/tensorflow/core/grappler/utils/grappler_test.cc b/tensorflow/core/grappler/utils/grappler_test.cc
index 5c96359867..910b0acaef 100644
--- a/tensorflow/core/grappler/utils/grappler_test.cc
+++ b/tensorflow/core/grappler/utils/grappler_test.cc
@@ -39,6 +39,11 @@ GrapplerTest::GrapplerTest() {
   cfg->set_debug_stripper(RewriterConfig::OFF);
 }
 
+std::vector<Tensor> GrapplerTest::EvaluateNodes(
+    const GraphDef& graph, const std::vector<string>& node_names) const {
+  return EvaluateNodes(graph, node_names, {});
+}
+
 std::vector<Tensor> GrapplerTest::EvaluateNodes(
     const GraphDef& graph, const std::vector<string>& node_names,
     const std::vector<std::pair<string, Tensor>>& inputs) const {
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 4b160e7f16..3bc7bea454 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -34,6 +34,9 @@ class GrapplerTest : public ::testing::Test {
   GrapplerTest();
 
  protected:
+  std::vector<Tensor> EvaluateNodes(
+      const GraphDef& graph, const std::vector<string>& node_names) const;
+
   std::vector<Tensor> EvaluateNodes(
       const GraphDef& graph, const std::vector<string>& node_names,
       const std::vector<std::pair<string, Tensor>>& inputs) const;
-- 
GitLab


From cc9944abe196827bae38975d813ee3e428349dcb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 11:34:32 -0700
Subject: [PATCH 615/960] In contrib/all_reduce raise a ValueError if the input
 tensors do not have fully-defined shapes.

PiperOrigin-RevId: 190804146
---
 tensorflow/contrib/all_reduce/python/all_reduce.py      | 7 +++----
 tensorflow/contrib/all_reduce/python/all_reduce_test.py | 6 ++++++
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/all_reduce/python/all_reduce.py b/tensorflow/contrib/all_reduce/python/all_reduce.py
index 6658f0d9c1..8add2aacff 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce.py
@@ -38,16 +38,15 @@ def _flatten_tensors(tensors):
     shape: the original shape of each element of input tensors
 
   Raises:
-    ValueError: tensors are empty or non-isomorphic.
+    ValueError: tensors are empty or non-isomorphic or have unknown shape.
   """
   if not tensors:
     raise ValueError("tensors cannot be empty")
   shape = tensors[0].shape
   for tensor in tensors:
     shape = shape.merge_with(tensor.shape)
-  if shape.ndims is None:
-    raise ValueError("At least one of the tensors in 'tensors' must have "
-                     "statically known rank.")
+  if not shape.is_fully_defined():
+    raise ValueError("Tensors must have statically known shape.")
   if len(shape) != 1:
     reshaped = []
     for t in tensors:
diff --git a/tensorflow/contrib/all_reduce/python/all_reduce_test.py b/tensorflow/contrib/all_reduce/python/all_reduce_test.py
index 47bab0a367..b3f5d92259 100644
--- a/tensorflow/contrib/all_reduce/python/all_reduce_test.py
+++ b/tensorflow/contrib/all_reduce/python/all_reduce_test.py
@@ -36,6 +36,12 @@ from tensorflow.python.platform import tf_logging
 
 class AllReduceTest(test_util.TensorFlowTestCase):
 
+  def testFlattenTensorsShapesDefined(self):
+    x = array_ops.placeholder(types_pb2.DT_FLOAT, [None])
+    with self.assertRaisesRegexp(ValueError,
+                                 "must have statically known shape"):
+      ar._flatten_tensors([x, x])
+
   def testRingPermutations(self):
     # 0 devices
     pred_by_c_d, rank_by_c_d = ar._ring_permutations(1, 0, [])
-- 
GitLab


From 70a51319f1d6e42f0d5eadbf65e941419974aac4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 11:48:28 -0700
Subject: [PATCH 616/960] Fixes to DepthwiseConv kernel

PiperOrigin-RevId: 190806668
---
 .../internal/optimized/depthwiseconv_uint8.h  | 11 +++--
 .../depthwiseconv_uint8_3x3_filter.h          | 43 ++++++++++++++++++-
 2 files changed, 47 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
index c71b070680..0f78e0f728 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8.h
@@ -1694,12 +1694,11 @@ inline void DepthwiseConv(const uint8* input_data, const Dims<4>& input_dims,
   TFLITE_DCHECK(output_depth == input_depth * depth_multiplier);
 
 #ifdef __aarch64__
-  // Call kernel optimized for depthwise convolutions using 3x3 filters,
-  // stride = 1, no padding, depth_multiplier = 1 and depth a multiple of 16.
-  if (filter_width == 3 && filter_height == 3 && depth_multiplier == 1 &&
-      (stride_width == 1 || stride_width == 2) &&
-      (stride_height == 1 || stride_height == 2) && pad_width == 0 &&
-      pad_height == 0 && (input_depth % 16) == 0) {
+  // Call kernel optimized for depthwise convolutions using 3x3 filters if
+  // parameters are supported.
+  if (Fast3by3FilterKernelSupported(input_dims, filter_dims, stride_width,
+                                    stride_height, pad_width, pad_height,
+                                    depth_multiplier, output_dims)) {
     DepthwiseConv3by3FilterDepth16(
         input_data, input_dims, input_offset, filter_data, filter_dims,
         filter_offset, bias_data, bias_dims, stride_width, stride_height,
diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
index 9dc76e7608..a349892076 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/depthwiseconv_uint8_3x3_filter.h
@@ -440,6 +440,47 @@ struct ConvKernel3x3FilterDepth16<1, 1> {
   }
 };
 
+inline bool Fast3by3FilterKernelSupported(const Dims<4>& input_dims,
+                                          const Dims<4>& filter_dims,
+                                          int stride_width, int stride_height,
+                                          int pad_width, int pad_height,
+                                          int depth_multiplier,
+                                          const Dims<4>& output_dims) {
+  const int input_height = ArraySize(input_dims, 2);
+  const int input_width = ArraySize(input_dims, 1);
+  const int input_depth = ArraySize(input_dims, 0);
+  const int filter_height = ArraySize(filter_dims, 2);
+  const int filter_width = ArraySize(filter_dims, 1);
+  const int output_height = ArraySize(output_dims, 2);
+  const int output_width = ArraySize(output_dims, 1);
+
+  bool supported = filter_width == 3 && filter_height == 3 &&
+                   depth_multiplier == 1 &&
+                   (stride_width == 1 || stride_width == 2) &&
+                   (stride_height == 1 || stride_height == 2) &&
+                   pad_width == 0 && pad_height == 0 && (input_depth % 16) == 0;
+
+  if (!supported) {
+    return false;
+  }
+
+  // Handle case where padding is zero but type is not kValid. This would
+  // require special boundary case handling that is not supported yet.
+
+  const int out_x = output_width - 1;
+  const int out_y = output_height - 1;
+
+  const int in_x_origin = (out_x * stride_width) - pad_width;
+  const int in_y_origin = (out_y * stride_height) - pad_height;
+
+  const int in_x_end = in_x_origin + filter_width;
+  const int in_y_end = in_y_origin + filter_height;
+
+  // Supported only if filter on the right and bottom boundary lies completely
+  // within the input.
+  return in_x_end <= input_width && in_y_end <= input_height;
+}
+
 inline void DepthwiseConv3by3FilterDepth16(
     const uint8* input_data, const Dims<4>& input_dims, int32 input_offset,
     const uint8* filter_data, const Dims<4>& filter_dims, int32 filter_offset,
@@ -634,7 +675,7 @@ inline void DepthwiseConv3by3FilterDepth16(
         // Handle the rest of the right side.
         for (; out_x < output_width; out_x++) {
           // This code path can only be reached if we're handling >1 x outputs
-          // at a time or support padding.
+          // at a time or support kSame padding.
         }
       }
 
-- 
GitLab


From f9dbf697535f8262d2513ade20ada85431c323f3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 12:00:18 -0700
Subject: [PATCH 617/960] Reorder element wise operators across reshape
 operators.

This allows batch-norm folding to work across reshape operators.

PiperOrigin-RevId: 190808678
---
 tensorflow/contrib/lite/toco/BUILD            |   1 +
 .../graph_transformations.h                   |   1 +
 .../swap_elementwise_binary.cc                | 175 ++++++++++++++++++
 .../toco/graph_transformations/tests/BUILD    |  11 ++
 .../tests/swap_elementwise_binary_test.cc     |  89 +++++++++
 tensorflow/contrib/lite/toco/toco_tooling.cc  |   1 +
 6 files changed, 278 insertions(+)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 051fa8de3c..8ed3e0e14e 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -280,6 +280,7 @@ cc_library(
         "graph_transformations/resolve_tensorflow_switch.cc",
         "graph_transformations/resolve_tensorflow_tile.cc",
         "graph_transformations/resolve_transpose_attributes.cc",
+        "graph_transformations/swap_elementwise_binary.cc",
         "graph_transformations/unfuse_activation_functions.cc",
         "graph_transformations/unpartition_embedding_lookup.cc",
         "graph_transformations/unroll_batch_matmul.cc",
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 640afc7c74..1291825c8e 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -180,6 +180,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveConstantStridedSlice)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantFill)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantGather)
 DECLARE_GRAPH_TRANSFORMATION(ResolveMultiplyByZero)
+DECLARE_GRAPH_TRANSFORMATION(SwapElementwiseBinary)
 DECLARE_GRAPH_TRANSFORMATION(Dequantize)
 DECLARE_GRAPH_TRANSFORMATION(UnpartitionEmbeddingLookup)
 
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc b/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
new file mode 100644
index 0000000000..ecbce58d16
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
@@ -0,0 +1,175 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/runtime/types.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+namespace {
+
+bool ShapesAllowSwapping(const string& input_array_name,
+                         const string& const_array_name, Model* model) {
+  const Array& input_array = model->GetOrCreateArray(input_array_name);
+  const Array& const_array = model->GetOrCreateArray(const_array_name);
+  // Wait until these shapes have been resolved.
+  if (!input_array.has_shape() || !const_array.has_shape()) {
+    return false;
+  }
+
+  // Currently swapping is not handled for scalar const_array, though that could
+  // be done once there is a test model.
+  if (RequiredBufferSizeForShape(input_array.shape()) !=
+      RequiredBufferSizeForShape(const_array.shape())) {
+    return false;
+  }
+
+  return true;
+}
+
+}  // namespace
+
+// Swaps:
+//                   Input
+//                      \
+//                   (Reshape Op)       Const
+//                         \             /
+//                       (Add/Sub/Mul/Div op)
+//                               |
+//                             Output
+//
+// To:
+//
+//                     Input            Const
+//                        \              /
+//                      (Add/Sub/Mul/Div op)
+//                               |
+//                         (Reshape Op)
+//                               |
+//                             Output
+//
+// This can allow Add/Mul ops from batch normalization to be folded into an
+// Input op from a FullyConnected layer.
+bool SwapElementwiseBinary::Run(Model* model, std::size_t op_index) {
+  const auto element_wise_op_it = model->operators.begin() + op_index;
+  std::unique_ptr<Operator>& element_wise_op = *element_wise_op_it;
+  DCHECK(element_wise_op);
+
+  switch (element_wise_op->type) {
+    case OperatorType::kAdd:
+    case OperatorType::kSub:
+    case OperatorType::kMul:
+    case OperatorType::kDiv:
+      break;
+    default:
+      return false;
+  }
+
+  int reshape_input = -1;
+  Operator* op = GetOpWithOutput(*model, element_wise_op->inputs[0]);
+  if (!op) {
+    return false;
+  }
+
+  if (op->type == OperatorType::kTensorFlowReshape) {
+    reshape_input = 0;
+  } else {
+    op = GetOpWithOutput(*model, element_wise_op->inputs[1]);
+    if (!op || op->type != OperatorType::kTensorFlowReshape) {
+      return false;
+    }
+    reshape_input = 1;
+  }
+
+  int const_input = (reshape_input == 0) ? 1 : 0;
+  const string& const_input_array = element_wise_op->inputs[const_input];
+  if (!IsConstantParameterArray(*model, const_input_array)) {
+    return false;
+  }
+
+  // Do not fold division if denominator is not constant.
+  if (element_wise_op->type != OperatorType::kDiv && const_input != 1) {
+    return false;
+  }
+
+  const auto reshape_it =
+      FindOpWithOutput(*model, element_wise_op->inputs[reshape_input]);
+  // Note: we take copies of the tensor names here, instead of const-refs as we
+  // may overwrite the original names.
+  const string reshape_input_name = (*reshape_it)->inputs[0];
+  const string intermediate_name = (*reshape_it)->outputs[0];
+  const string element_wise_output_name = element_wise_op->outputs[0];
+
+  // Check the reshape op input and const op have their shapes resolved.
+  if (!ShapesAllowSwapping(reshape_input_name, const_input_array, model)) {
+    return false;
+  }
+
+  int count_ops_consuming_output = CountOpsWithInput(*model, intermediate_name);
+  DCHECK_GE(count_ops_consuming_output, 1);
+  if (count_ops_consuming_output > 1) {
+    AddMessageF(
+        "Not exchanging element-wise function with %s because it is "
+        "consumed by more than 1 other operator",
+        LogName(**reshape_it));
+    return false;
+  }
+
+  // If the element_wise_op was originally producing an output_array we can't
+  // swap as otherwise the output array would change. It'd be nice to still be
+  // able to swap but if code is relying on the fetch names instead of array
+  // indices this won't work.
+  for (int i = 0; i < model->flags.output_arrays_size(); ++i) {
+    if (model->flags.output_arrays(i) == element_wise_op->outputs[0]) {
+      AddMessageF(
+          "Not exchanging activation function with %s to preserve output array "
+          "name %s",
+          LogName(**reshape_it), element_wise_op->outputs[0]);
+      return false;
+    }
+  }
+
+  // Rewire by changing inputs, including all consumers.
+  // TODO(b/76086261): Replace with new utility function.
+  Operator* consumer = GetFirstOpWithInput(*model, element_wise_output_name);
+  while (consumer) {
+    for (int i = 0; i < consumer->inputs.size(); ++i) {
+      if (consumer->inputs[i] == element_wise_output_name) {
+        consumer->inputs[i] = intermediate_name;
+      }
+    }
+    consumer = GetFirstOpWithInput(*model, element_wise_output_name);
+  }
+  element_wise_op->inputs[reshape_input] = reshape_input_name;
+  (*reshape_it)->inputs[0] = element_wise_output_name;
+
+  // Clear shapes; this will allow shape propagation to fix the sizes for us.
+  model->GetOrCreateArray(element_wise_output_name).clear_shape();
+
+  // Finally, swap operators.  Note that this only works when there are no other
+  // direct descendents of the reshape operator.
+  element_wise_op.swap(*reshape_it);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
index 2f94f9cd8a..b975cc996b 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
@@ -18,6 +18,17 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "swap_elementwise_binary_test",
+    srcs = ["swap_elementwise_binary_test.cc"],
+    deps = [
+        "//tensorflow/contrib/lite/toco:graph_transformations",
+        "//tensorflow/contrib/lite/toco:model",
+        "//tensorflow/contrib/lite/toco:tooling_util",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 tf_cc_test(
     name = "lstm_utils_test",
     srcs = ["lstm_utils_test.cc"],
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc b/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc
new file mode 100644
index 0000000000..c3778017f3
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc
@@ -0,0 +1,89 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+
+namespace toco {
+
+namespace {
+
+int ShapeCount(const std::vector<int>& size) {
+  CHECK(size.size());
+  int count = 1;
+  for (int dim : size) {
+    count *= dim;
+  }
+  return count;
+}
+
+// Adds a new parameter array to the model.
+void AddConstArray(const string& name, const float* data,
+                   const std::vector<int>& size, Model* model) {
+  Array& array = model->GetOrCreateArray(name);
+  array.data_type = ArrayDataType::kFloat;
+  Shape* shape = array.mutable_shape();
+  *(shape->mutable_dims()) = size;
+
+  auto& buffer = array.GetMutableBuffer<toco::ArrayDataType::kFloat>();
+  buffer.data.resize(ShapeCount(size));
+  std::copy(data, data + ShapeCount(size), buffer.data.data());
+}
+
+}  // namespace
+
+TEST(SwapElementwiseBinaryTest, SwapsReshape) {
+  Model model;
+  const float parameters[2][4] = {{0., 1., 2., 3.}, {10., 11., 12., 13.}};
+
+  AddConstArray("before_reshape", parameters[0], {2, 2}, &model);
+  AddConstArray("add_vector", parameters[1], {1, 4}, &model);
+
+  auto reshape_op = absl::make_unique<TensorFlowReshapeOperator>();
+  reshape_op->shape = {1, 4};
+  reshape_op->inputs = {"before_reshape"};
+  reshape_op->outputs = {"after_reshape"};
+  Array& reshape_array = model.GetOrCreateArray("after_reshape");
+  *(reshape_array.mutable_shape()) = {1, 4};
+
+  auto add_op = absl::make_unique<AddOperator>();
+  add_op->inputs = {"after_reshape", "add_vector"};
+  add_op->outputs = {"add"};
+  Array& add_array = model.GetOrCreateArray("add");
+  *(add_array.mutable_shape()) = {1, 4};
+
+  model.operators.push_back(std::move(reshape_op));
+  model.operators.push_back(std::move(add_op));
+
+  auto transformation = absl::make_unique<toco::SwapElementwiseBinary>();
+  ASSERT_TRUE(transformation->Run(&model, 1));
+
+  Operator* op = GetOpWithOutput(model, "add");
+  ASSERT_NE(nullptr, op);
+  ASSERT_EQ(OperatorType::kAdd, op->type);
+  ASSERT_EQ(2, op->inputs.size());
+  for (const string& input : op->inputs) {
+    EXPECT_TRUE(IsConstantParameterArray(model, input))
+        << input << " is not const input";
+  }
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 30dd6fab9e..41ea1481bc 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -90,6 +90,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveTensorFlowTile);
   transformations->Add(new ResolveTensorFlowConcat);
   transformations->Add(new ResolveMultiplyByZero);
+  transformations->Add(new SwapElementwiseBinary);
   transformations->Add(new IdentifyDilatedConv);
   transformations->Add(new IdentifyL2Normalization);
   transformations->Add(new IdentifyL2Pool);
-- 
GitLab


From 195be47024f5608b284c52239d006b756cbad0d5 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 28 Mar 2018 12:06:33 -0700
Subject: [PATCH 618/960] Add some VLOGs to make it easier to see why things
 don't go through the fast path

PiperOrigin-RevId: 190809906
---
 tensorflow/python/eager/pywrap_tfe_src.cc | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 30ef6781ec..73482792d5 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1406,15 +1406,32 @@ bool CheckInputsOk(PyObject* seq, int start_index,
     if (!op_def.input_arg(i).number_attr().empty() ||
         !op_def.input_arg(i).type_list_attr().empty()) {
       // This item should be a seq input.
-      if (!PySequence_Check(item)) return false;
+      if (!PySequence_Check(item)) {
+        VLOG(1) << "Falling back to slow path for Op \"" << op_def.name()
+                << "\", Input \"" << op_def.input_arg(i).name()
+                << "\" since we expected a sequence, but got "
+                << item->ob_type->tp_name;
+        return false;
+      }
       for (Py_ssize_t j = 0; j < PySequence_Fast_GET_SIZE(item); j++) {
         PyObject* inner_item = PySequence_Fast_GET_ITEM(item, j);
         if (!EagerTensor_CheckExact(inner_item) &&
             !CheckResourceVariable(inner_item)) {
+          VLOG(1)
+              << "Falling back to slow path for Op \"" << op_def.name()
+              << "\", Input \"" << op_def.input_arg(i).name() << "\", Index "
+              << j
+              << " since we expected an EagerTensor/ResourceVariable, but got "
+              << inner_item->ob_type->tp_name;
           return false;
         }
       }
     } else if (!EagerTensor_CheckExact(item) && !CheckResourceVariable(item)) {
+      VLOG(1)
+          << "Falling back to slow path for Op \"" << op_def.name()
+          << "\", Input \"" << op_def.input_arg(i).name()
+          << "\" since we expected an EagerTensor/ResourceVariable, but got "
+          << item->ob_type->tp_name;
       return false;
     }
   }
@@ -1894,6 +1911,9 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
                               py_attr_value, &attr_list_sizes, status);
 
         if (TF_GetCode(status) != TF_OK) {
+          VLOG(1) << "Falling back to slow path for Op \"" << op_def->name()
+                  << "\" since we are unable to set the value for attr \""
+                  << attr.name() << "\" due to: " << TF_Message(status);
           RaiseFallbackException(TF_Message(status));
           return nullptr;
         }
-- 
GitLab


From e0956b390aabaf8882dff600056db805f3fccbf6 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 28 Mar 2018 12:16:51 -0700
Subject: [PATCH 619/960] Don't access properties in case they're not present

PiperOrigin-RevId: 190811935
---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 22ede19493..c3f8a1ce22 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1534,6 +1534,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
 
     // Remove Shuffle or Reverse op over scalar values.
     if (use_shape_info &&
+        !properties->GetInputProperties(node->name()).empty() &&
         (IsShuffle(*node) || IsReverse(*node) || IsTranspose(*node))) {
       const auto& shape =
           properties->GetInputProperties(node->name())[0].shape();
-- 
GitLab


From 4ec02c23174b07540d190cec620347ee6f31a8d8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 12:18:24 -0700
Subject: [PATCH 620/960] [XLA] Redesign: add the rest of the XlaBuilder public
 methods.

PiperOrigin-RevId: 190812260
---
 .../xla/client/xla_client/xla_builder.cc      | 107 +++++++++++++++++-
 .../xla/client/xla_client/xla_builder.h       |  71 ++++++++++++
 .../xla/client/xla_client/xla_computation.h   |   2 +
 3 files changed, 179 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 7d39701b10..1b94f9a4eb 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -128,6 +128,18 @@ StatusOr<ProgramShape> XlaBuilder::GetProgramShape() {
   return GetProgramShape(&root_id);
 }
 
+XlaComputation XlaBuilder::BuildAndNoteError() {
+  DCHECK(parent_builder_ != nullptr);
+  auto build_status = Build();
+  if (!build_status.ok()) {
+    parent_builder_->NoteError(
+        AddStatus(build_status.status(),
+                  tensorflow::strings::StrCat("error from: ", name_)));
+    return {};
+  }
+  return build_status.ConsumeValueOrDie();
+}
+
 StatusOr<XlaComputation> XlaBuilder::Build() {
   if (!first_error_.ok()) {
     string backtrace;
@@ -945,6 +957,99 @@ XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
   return UnimplementedOp();
 }
 
+StatusOr<bool> XlaBuilder::IsConstant(const XlaOp& operand,
+                                      int64 num_parameters) {
+  return Unimplemented("IsConstant is not implemented.");
+}
+
+StatusOr<std::unique_ptr<Literal>> XlaBuilder::ComputeConstant(
+    const XlaOp& operand, const Layout* output_layout,
+    tensorflow::gtl::ArraySlice<Literal> parameters) {
+  return Unimplemented("ComputeConstant is not implemented");
+}
+
+std::unique_ptr<XlaBuilder> XlaBuilder::CreateSubBuilder(
+    const string& computation_name) {
+  auto sub_builder = MakeUnique<XlaBuilder>(computation_name);
+  sub_builder->parent_builder_ = this;
+  sub_builder->die_immediately_on_error_ = this->die_immediately_on_error_;
+  return sub_builder;
+}
+
+Status XlaBuilder::SetReturnValue(const XlaOp& operand) {
+  return Unimplemented("SetReturnValue is not implemented.");
+}
+
+/* static */ ConvolutionDimensionNumbers
+XlaBuilder::CreateDefaultConvDimensionNumbers(int num_spatial_dims) {
+  ConvolutionDimensionNumbers dimension_numbers;
+  dimension_numbers.set_input_batch_dimension(kConvBatchDimension);
+  dimension_numbers.set_input_feature_dimension(kConvFeatureDimension);
+  dimension_numbers.set_output_batch_dimension(kConvBatchDimension);
+  dimension_numbers.set_output_feature_dimension(kConvFeatureDimension);
+  dimension_numbers.set_kernel_output_feature_dimension(
+      kConvKernelOutputDimension);
+  dimension_numbers.set_kernel_input_feature_dimension(
+      kConvKernelInputDimension);
+  for (int i = 0; i < num_spatial_dims; ++i) {
+    dimension_numbers.add_input_spatial_dimensions(i + 2);
+    dimension_numbers.add_kernel_spatial_dimensions(i + 2);
+    dimension_numbers.add_output_spatial_dimensions(i + 2);
+  }
+  return dimension_numbers;
+}
+
+/* static */ Status XlaBuilder::Validate(
+    const ConvolutionDimensionNumbers& dnum) {
+  if (dnum.input_spatial_dimensions_size() < 2) {
+    return FailedPrecondition("input spacial dimension < 2: %d",
+                              dnum.input_spatial_dimensions_size());
+  }
+  if (dnum.kernel_spatial_dimensions_size() < 2) {
+    return FailedPrecondition("kernel spacial dimension < 2: %d",
+                              dnum.kernel_spatial_dimensions_size());
+  }
+  if (dnum.output_spatial_dimensions_size() < 2) {
+    return FailedPrecondition("output spacial dimension < 2: %d",
+                              dnum.output_spatial_dimensions_size());
+  }
+
+  if (std::set<int64>(
+          {dnum.input_batch_dimension(), dnum.input_feature_dimension(),
+           dnum.input_spatial_dimensions(0), dnum.input_spatial_dimensions(1)})
+          .size() != 4) {
+    return FailedPrecondition(
+        "dimension numbers for the input are not unique: (%lld, %lld, %lld, "
+        "%lld)",
+        dnum.input_batch_dimension(), dnum.input_feature_dimension(),
+        dnum.input_spatial_dimensions(0), dnum.input_spatial_dimensions(1));
+  }
+  if (std::set<int64>({dnum.kernel_output_feature_dimension(),
+                       dnum.kernel_input_feature_dimension(),
+                       dnum.kernel_spatial_dimensions(0),
+                       dnum.kernel_spatial_dimensions(1)})
+          .size() != 4) {
+    return FailedPrecondition(
+        "dimension numbers for the weight are not unique: (%lld, %lld, %lld, "
+        "%lld)",
+        dnum.kernel_output_feature_dimension(),
+        dnum.kernel_input_feature_dimension(),
+        dnum.kernel_spatial_dimensions(0), dnum.kernel_spatial_dimensions(1));
+  }
+  if (std::set<int64>({dnum.output_batch_dimension(),
+                       dnum.output_feature_dimension(),
+                       dnum.output_spatial_dimensions(0),
+                       dnum.output_spatial_dimensions(1)})
+          .size() != 4) {
+    return FailedPrecondition(
+        "dimension numbers for the output are not unique: (%lld, %lld, %lld, "
+        "%lld)",
+        dnum.output_batch_dimension(), dnum.output_feature_dimension(),
+        dnum.output_spatial_dimensions(0), dnum.output_spatial_dimensions(1));
+  }
+  return Status::OK();
+}
+
 StatusOr<XlaOp> XlaBuilder::AddInstruction(
     HloInstructionProto&& instr, HloOpcode opcode,
     tensorflow::gtl::ArraySlice<XlaOp> operands) {
@@ -986,7 +1091,7 @@ StatusOr<const HloInstructionProto*> XlaBuilder::LookUpInstruction(
 }
 
 XlaOp XlaBuilder::UnimplementedOp() {
-  NoteError(Unimplemented("Op not yet implemented"));
+  NoteError(Unimplemented("Op not implemented"));
   return {};
 }
 
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index c5c35159e0..f66feb93ce 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -335,6 +335,26 @@ class XlaBuilder {
   XlaOp DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
                    const DotDimensionNumbers& dimension_numbers);
 
+  // Default dimension numbers used for a 2D convolution.
+  static constexpr int64 kConvBatchDimension = 0;
+  static constexpr int64 kConvFeatureDimension = 1;
+  static constexpr int64 kConvFirstSpatialDimension = 2;
+  static constexpr int64 kConvSecondSpatialDimension = 3;
+  static constexpr int64 kConvKernelOutputDimension = 0;
+  static constexpr int64 kConvKernelInputDimension = 1;
+  static constexpr int64 kConvKernelFirstSpatialDimension = 2;
+  static constexpr int64 kConvKernelSecondSpatialDimension = 3;
+
+  // Creates a default ConvolutionDimensionNumbers. For a 2D convolution, for
+  // the input operand {batch, feature, height, width} = {0, 1, 2, 3} and for
+  // the kernel operand
+  // {output_feature, input_feature, height, width} = {0, 1, 2, 3}.
+  static ConvolutionDimensionNumbers CreateDefaultConvDimensionNumbers(
+      int num_spatial_dims = 2);
+
+  // Returns an error if the convolution dimension numbers have conflicts.
+  static Status Validate(const ConvolutionDimensionNumbers& dnum);
+
   // Enqueues a convolution instruction onto the computation, which uses the
   // default convolution dimension numbers.
   XlaOp Conv(const XlaOp& lhs, const XlaOp& rhs,
@@ -711,10 +731,59 @@ class XlaBuilder {
                       const XlaOp& grad_output, float epsilon,
                       int64 feature_index);
 
+  // Computes the value of a constant indicated by a XlaOp using a non-optimized
+  // interpreter on the host.
+  //
+  // The operand must represent a constant value, which in this case
+  // means that it must not statically depend on any parameter of the
+  // computation that is being built other then the ones specified on the
+  // parameter list. The parameters in the list will be indexed by their
+  // parameter id property so the number of parameters specified should be at
+  // least as many as the largest used parameter index.
+  //
+  // `IsConstant` can be used to test whether a computation is a compile-time
+  // constant without evaluation it. `ComputeConstant` only succeeds for
+  // computations where `IsConstant` returns true.
+  //
+  // This functionality can be useful when translating a computation
+  // into XLA where something that looked dynamic is required by
+  // XLA to be specified as a constant. E.g. the source
+  // computation (outside of XLA) may include a dynamic
+  // computation of the shape of something and ComputeConstant lets
+  // you determine what the value of that computation is in the case
+  // where the value can be determined at compile time.
+  //
+  // If output_layout is non-null, then the output of the computation
+  // will be stored using that layout.
+  StatusOr<std::unique_ptr<Literal>> ComputeConstant(
+      const XlaOp& operand, const Layout* output_layout = nullptr,
+      tensorflow::gtl::ArraySlice<Literal> parameters = {});
+
+  // Returns a new XlaBuilder whose resultant Computation is used only by this
+  // XlaBuilder. The sub-XlaBuilder has the same die_immediately_on_error
+  // behavior as the parent.
+  std::unique_ptr<XlaBuilder> CreateSubBuilder(const string& computation_name);
+
+  // Modifies the computation being built so that executions of it will return
+  // the value associated with operand, rather than the last expression enqueued
+  // on the XlaBuilder. Any subsequent operations added to the XlaBuilder will
+  // not have any effect unless SetReturnValue is called again.
+  Status SetReturnValue(const XlaOp& operand);
+
   // Builds the computation with the requested operations, or returns a non-ok
   // status.
   StatusOr<XlaComputation> Build();
 
+  // Builds the computation with the requested operations, or notes an error in
+  // the parent XlaBuilder and returns an empty computation if building failed.
+  // This function is intended to be used where the returned XlaComputation is
+  // only used by the parent XlaBuilder and hence further operation on the
+  // returned XlaComputation will simply be error'ed out if an error occurred
+  // while building this computation. If the built computation is to be used by
+  // a XlaBuilder other than the parent XlaBuilder then Build() should be used
+  // instead.
+  XlaComputation BuildAndNoteError();
+
   // Returns the first error that was encountered while building the
   // computation. When an error is encountered, by default we return a vacuous
   // XlaOp and inform the user of the error that occurred while
@@ -814,6 +883,8 @@ class XlaBuilder {
 
   // Mode bit that indicates whether to die when a first error is encountered.
   bool die_immediately_on_error_ = false;
+
+  XlaBuilder* parent_builder_{nullptr};
 };
 
 template <typename NativeT>
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
index 5b89747fdd..78e1e3c32c 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_computation.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
@@ -29,6 +29,8 @@ namespace xla {
 // TODO(b/74197823): Replace xla::Computation with this one.
 class XlaComputation {
  public:
+  XlaComputation() : unique_id_(-1) {}
+
   XlaComputation(const XlaComputation&) = delete;
   XlaComputation& operator=(const XlaComputation&) = delete;
 
-- 
GitLab


From 7863645e0323d3b2ef034a6499ec6673f0cca761 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 12:19:27 -0700
Subject: [PATCH 621/960] When importing meta graphs under name scopes, the
 names of the created ops are prepended with the scopes. Since the saver_def
 of the meta graph does not contain this information, we need to pass it
 explicitly to Saver.

PiperOrigin-RevId: 190812434
---
 tensorflow/python/training/saver.py      | 20 +++++++++++----
 tensorflow/python/training/saver_test.py | 32 ++++++++++++++++++++++++
 2 files changed, 47 insertions(+), 5 deletions(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index ba0d038475..cec581d997 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -1924,12 +1924,22 @@ def import_meta_graph(meta_graph_or_file, clear_devices=False,
   else:
     meta_graph_def = meta_graph_or_file
 
-  meta_graph.import_scoped_meta_graph(meta_graph_def,
-                                      clear_devices=clear_devices,
-                                      import_scope=import_scope,
-                                      **kwargs)
+  imported_vars = meta_graph.import_scoped_meta_graph(
+      meta_graph_def,
+      clear_devices=clear_devices,
+      import_scope=import_scope,
+      **kwargs)
+
   if meta_graph_def.HasField("saver_def"):
-    return Saver(saver_def=meta_graph_def.saver_def, name=import_scope)
+    # Infer the scope that is prepended by `import_scoped_meta_graph`.
+    scope = import_scope
+    var_names = list(imported_vars.keys())
+    if var_names:
+      sample_key = var_names[0]
+      sample_var = imported_vars[sample_key]
+      scope = sample_var.name[:-len(sample_key)]
+
+    return Saver(saver_def=meta_graph_def.saver_def, name=scope)
   else:
     if variables._all_saveable_objects():  # pylint: disable=protected-access
       # Return the default saver instance for all graph variables.
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index 7de778f298..d1c24b3930 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2341,6 +2341,38 @@ class MetaGraphTest(test.TestCase):
               10, size=[1, 10])
       })
 
+  def testImportIntoImplicitNamescope(self):
+    # Test that we can import a meta graph into an implicit namescope.
+    test_dir = self._get_test_dir("import_into_namescope")
+    filename = os.path.join(test_dir, "ckpt")
+    image = array_ops.placeholder(dtypes.float32, [None, 784], name="image")
+    label = array_ops.placeholder(dtypes.float32, [None, 10], name="label")
+    with session.Session() as sess:
+      weights = variables.Variable(
+          random_ops.random_uniform([784, 10]), name="weights")
+      bias = variables.Variable(array_ops.zeros([10]), name="bias")
+      logit = nn_ops.relu(math_ops.matmul(image, weights) + bias, name="logits")
+      nn_ops.softmax(logit, name="prediction")
+      cost = nn_ops.softmax_cross_entropy_with_logits(labels=label,
+                                                      logits=logit, name="cost")
+      adam.AdamOptimizer().minimize(cost, name="optimize")
+      saver = saver_module.Saver()
+      sess.run(variables.global_variables_initializer())
+      saver.save(sess, filename)
+
+    graph = ops_lib.Graph()
+    with session.Session(graph=graph) as sess:
+      with ops_lib.name_scope("new_model"):
+        new_saver = saver_module.import_meta_graph(
+            filename + ".meta", graph=graph)
+
+      new_saver.restore(sess, filename)
+      sess.run(["new_model/optimize"], {
+          "new_model/image:0": np.random.random([1, 784]),
+          "new_model/label:0": np.random.randint(
+              10, size=[1, 10])
+      })
+
   def testClearDevicesOnImport(self):
     # Test that we import a graph without its devices and run successfully.
     with ops_lib.Graph().as_default():
-- 
GitLab


From f80486324807181614ac71367dbb9cf588aa2804 Mon Sep 17 00:00:00 2001
From: Noah Eisen <ncteisen@google.com>
Date: Wed, 28 Mar 2018 12:28:32 -0700
Subject: [PATCH 622/960] Upgrade gRPC version used in OSS Tensorflow

PiperOrigin-RevId: 190813848
---
 tensorflow/contrib/cmake/external/grpc.cmake | 2 +-
 tensorflow/tools/pip_package/BUILD           | 1 +
 tensorflow/workspace.bzl                     | 9 +++++----
 3 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index cc218e8ab8..abfc69243e 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -17,7 +17,7 @@ include (ExternalProject)
 set(GRPC_INCLUDE_DIRS ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/include)
 set(GRPC_URL https://github.com/grpc/grpc.git)
 set(GRPC_BUILD ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc)
-set(GRPC_TAG 575bda39755b98d1f7099406bb57a6e3b2074874)
+set(GRPC_TAG bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2)
 
 if(WIN32)
   if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 16c47f7555..dd75eda231 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -113,6 +113,7 @@ filegroup(
         "@lmdb//:LICENSE",
         "@local_config_sycl//sycl:LICENSE.text",
         "@grpc//third_party/nanopb:LICENSE.txt",
+        "@grpc//third_party/address_sorting:LICENSE",
         "@nasm//:LICENSE",
         "@nsync//:LICENSE",
         "@pcre//:LICENCE",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 206a5a3d99..9fcbfb664b 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -405,13 +405,14 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "grpc",
       urls = [
-          "https://mirror.bazel.build/github.com/grpc/grpc/archive/575bda39755b98d1f7099406bb57a6e3b2074874.tar.gz",
-          "https://github.com/grpc/grpc/archive/575bda39755b98d1f7099406bb57a6e3b2074874.tar.gz",
+          "https://mirror.bazel.build/github.com/grpc/grpc/archive/bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2.tar.gz",
+          "https://github.com/grpc/grpc/archive/bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2.tar.gz",
       ],
-      sha256 = "f08a5c8e265191b39cc74915b1bc1fd380d86cd0176c92b7cce30b6ac50514ad",
-      strip_prefix = "grpc-575bda39755b98d1f7099406bb57a6e3b2074874",
+      sha256 = "0a05bd355e4571b01d813dddffa38e57e689ac41b264dc9b1bd6ec66463ef5d6",
+      strip_prefix = "grpc-bd6bdf93279a39a8cd92978fd7c9d14eccd98fc2",
   )
 
+
   tf_http_archive(
       name = "linenoise",
       sha256 = "7f51f45887a3d31b4ce4fa5965210a5e64637ceac12720cfce7954d6a2e812f7",
-- 
GitLab


From 560ef036727c871bab57faa9942ccaff977ef88a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 12:29:32 -0700
Subject: [PATCH 623/960] Supports quantized reduce_mean in TF Lite.

PiperOrigin-RevId: 190813997
---
 .../internal/reference/reference_ops.h        | 22 +++--
 tensorflow/contrib/lite/kernels/mean.cc       | 62 +++++++++++---
 tensorflow/contrib/lite/kernels/mean_test.cc  | 81 +++++++++++--------
 .../graph_transformations/hardcode_min_max.cc |  1 +
 .../toco/graph_transformations/quantize.cc    |  2 +-
 5 files changed, 114 insertions(+), 54 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index ce12fad95d..33d60afa26 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -3183,19 +3183,20 @@ inline void Exp(const T* input_data, const size_t num_elements,
   }
 }
 
-template <typename T>
-inline void Mean(T* input_data, const int* input_dims, const int input_num_dims,
+template <typename T, typename U>
+inline bool Mean(T* input_data, const int* input_dims, const int input_num_dims,
                  T* output_data, const int* output_dims,
                  const int output_num_dims, const int* axis,
                  const int num_axis_dimensions, bool keep_dims, int* temp_index,
-                 int* resolved_axis) {
+                 int* resolved_axis, U* temp_sum) {
   // resets output data.
   size_t num_outputs = 1;
   for (int idx = 0; idx < output_num_dims; ++idx) {
     num_outputs *= static_cast<size_t>(output_dims[idx]);
   }
   for (size_t idx = 0; idx < num_outputs; ++idx) {
-    output_data[idx] = 0;
+    output_data[idx] = T();
+    temp_sum[idx] = U();
   }
   // resets temp index.
   for (int idx = 0; idx < input_num_dims; ++idx) {
@@ -3228,19 +3229,24 @@ inline void Mean(T* input_data, const int* input_dims, const int input_num_dims,
     size_t output_offset =
         ReducedOutputOffset(input_num_dims, input_dims, temp_index,
                             num_resolved_axis, resolved_axis);
-    output_data[output_offset] += input_data[input_offset];
+    temp_sum[output_offset] += static_cast<U>(input_data[input_offset]);
   }
   // takes average by num of elements added to get mean.
   size_t num_elements_in_axis = 1;
   for (int idx = 0; idx < num_resolved_axis; ++idx) {
-    num_elements_in_axis *= static_cast<size_t>(input_dims[resolved_axis[idx]]);
+    size_t current = static_cast<size_t>(input_dims[resolved_axis[idx]]);
+    if (current > (std::numeric_limits<U>::max() / num_elements_in_axis)) {
+      return false;
+    }
+    num_elements_in_axis *= current;
   }
   if (num_elements_in_axis > 0) {
     for (size_t idx = 0; idx < num_outputs; ++idx) {
-      output_data[idx] = static_cast<T>(static_cast<float>(output_data[idx]) /
-                                        num_elements_in_axis);
+      output_data[idx] =
+          static_cast<T>(temp_sum[idx] / static_cast<U>(num_elements_in_axis));
     }
   }
+  return true;
 }
 
 template <typename T>
diff --git a/tensorflow/contrib/lite/kernels/mean.cc b/tensorflow/contrib/lite/kernels/mean.cc
index aff19581ea..047bdd1039 100644
--- a/tensorflow/contrib/lite/kernels/mean.cc
+++ b/tensorflow/contrib/lite/kernels/mean.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include <vector>
 #include "tensorflow/contrib/lite/builtin_op_data.h"
 #include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/quantization_util.h"
 #include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
 #include "tensorflow/contrib/lite/kernels/internal/tensor.h"
 #include "tensorflow/contrib/lite/kernels/kernel_util.h"
@@ -48,7 +49,7 @@ void* Init(TfLiteContext* context, const char* buffer, size_t length) {
   // Creates two temp tensors to store index and axis for internal
   // implementation only.
   auto* scratch_tensor_index = new int;
-  context->AddTensors(context, 2, scratch_tensor_index);
+  context->AddTensors(context, 3, scratch_tensor_index);
   return scratch_tensor_index;
 }
 
@@ -64,6 +65,14 @@ TfLiteStatus ResizeTempAxis(TfLiteContext* context, MeanContext* op_context,
   return context->ResizeTensor(context, resolved_axis, axis_size);
 }
 
+// Resizes the temp tensor that stores temp sum of reduced elements.
+TfLiteStatus ResizeTempSum(TfLiteContext* context, MeanContext* op_context,
+                           TfLiteTensor* temp_sum) {
+  TfLiteIntArray* size = TfLiteIntArrayCreate(1);
+  size->data[0] = static_cast<int>(NumElements(op_context->output));
+  return context->ResizeTensor(context, temp_sum, size);
+}
+
 // Resizes output array based on the input size and resolved axis.
 TfLiteStatus ResizeOutputTensor(TfLiteContext* context,
                                 MeanContext* op_context) {
@@ -135,7 +144,7 @@ TfLiteStatus InitializeTemporaries(TfLiteContext* context, TfLiteNode* node,
   // Creates a temp index to iterate through input data.
   int* scratch_tensor_index = reinterpret_cast<int*>(node->user_data);
   TfLiteIntArrayFree(node->temporaries);
-  node->temporaries = TfLiteIntArrayCreate(2);
+  node->temporaries = TfLiteIntArrayCreate(3);
   node->temporaries->data[0] = *scratch_tensor_index;
   TfLiteTensor* scratch_tensor = &context->tensors[node->temporaries->data[0]];
   scratch_tensor->type = kTfLiteInt32;
@@ -149,6 +158,25 @@ TfLiteStatus InitializeTemporaries(TfLiteContext* context, TfLiteNode* node,
   node->temporaries->data[1] = *scratch_tensor_index + 1;
   TfLiteTensor* resolved_axis = &context->tensors[node->temporaries->data[1]];
   resolved_axis->type = kTfLiteInt32;
+  // Creates a temp tensor to store temp sums when calculating mean.
+  node->temporaries->data[2] = *scratch_tensor_index + 2;
+  TfLiteTensor* temp_sum = &context->tensors[node->temporaries->data[2]];
+  switch (op_context->input->type) {
+    case kTfLiteFloat32:
+      temp_sum->type = kTfLiteFloat32;
+      break;
+    case kTfLiteInt32:
+      temp_sum->type = kTfLiteInt64;
+      break;
+    case kTfLiteInt64:
+      temp_sum->type = kTfLiteInt64;
+      break;
+    case kTfLiteUInt8:
+      temp_sum->type = kTfLiteInt32;
+      break;
+    default:
+      return kTfLiteError;
+  }
   return kTfLiteOk;
 }
 
@@ -160,16 +188,20 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_OK(context, InitializeTemporaries(context, node, &op_context));
 
   TfLiteTensor* resolved_axis = &context->tensors[node->temporaries->data[1]];
+  TfLiteTensor* temp_sum = &context->tensors[node->temporaries->data[2]];
   // Leaves work to Eval if axis is not constant; else resizes output.
   if (!IsConstantTensor(op_context.axis)) {
     SetTensorToDynamic(op_context.output);
     SetTensorToDynamic(resolved_axis);
+    SetTensorToDynamic(temp_sum);
     return kTfLiteOk;
   }
   resolved_axis->allocation_type = kTfLiteArenaRw;
   TF_LITE_ENSURE_OK(context,
                     ResizeTempAxis(context, &op_context, resolved_axis));
-  return ResizeOutputTensor(context, &op_context);
+  TF_LITE_ENSURE_OK(context, ResizeOutputTensor(context, &op_context));
+  temp_sum->allocation_type = kTfLiteArenaRw;
+  return ResizeTempSum(context, &op_context, temp_sum);
 }
 
 template <KernelType kernel_type>
@@ -178,14 +210,16 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
   int num_axis = static_cast<int>(NumElements(op_context.axis));
   TfLiteTensor* temp_index = &context->tensors[node->temporaries->data[0]];
   TfLiteTensor* resolved_axis = &context->tensors[node->temporaries->data[1]];
+  TfLiteTensor* temp_sum = &context->tensors[node->temporaries->data[2]];
   // Resize the output tensor if the output tensor is dynamic.
   if (IsDynamicTensor(op_context.output)) {
     TF_LITE_ENSURE_OK(context,
                       ResizeTempAxis(context, &op_context, resolved_axis));
     TF_LITE_ENSURE_OK(context, ResizeOutputTensor(context, &op_context));
+    TF_LITE_ENSURE_OK(context, ResizeTempSum(context, &op_context, temp_sum));
   }
 
-#define TF_LITE_MEAN(kernel_type, data_type)                        \
+#define TF_LITE_MEAN(kernel_type, data_type, temp_data_type)        \
   kernel_type::Mean<>(                                              \
       GetTensorData<data_type>(op_context.input),                   \
       op_context.input->dims->data, op_context.input->dims->size,   \
@@ -193,21 +227,26 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       op_context.output->dims->data, op_context.output->dims->size, \
       GetTensorData<int>(op_context.axis), num_axis,                \
       op_context.params->keep_dims, GetTensorData<int>(temp_index), \
-      GetTensorData<int>(resolved_axis))
+      GetTensorData<int>(resolved_axis),                            \
+      GetTensorData<temp_data_type>(temp_sum))
 
   if (kernel_type == kReference) {
     switch (op_context.input->type) {
       case kTfLiteFloat32:
-        TF_LITE_MEAN(reference_ops, float);
+        TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, float, float));
         break;
       case kTfLiteInt32:
-        TF_LITE_MEAN(reference_ops, int);
-        break;
-      case kTfLiteUInt8:
-        TF_LITE_MEAN(reference_ops, uint8_t);
+        TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, int, int64_t));
         break;
       case kTfLiteInt64:
-        TF_LITE_MEAN(reference_ops, int64_t);
+        TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, int64_t, int64_t));
+        break;
+      case kTfLiteUInt8:
+        TF_LITE_ENSURE_EQ(context, op_context.input->params.scale,
+                          op_context.output->params.scale);
+        TF_LITE_ENSURE_EQ(context, op_context.input->params.zero_point,
+                          op_context.output->params.zero_point);
+        TF_LITE_ENSURE(context, TF_LITE_MEAN(reference_ops, uint8_t, int));
         break;
       default:
         return kTfLiteError;
@@ -216,7 +255,6 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
 #undef TF_LITE_MEAN
   return kTfLiteOk;
 }
-
 }  // namespace mean
 
 TfLiteRegistration* Register_MEAN_REF() {
diff --git a/tensorflow/contrib/lite/kernels/mean_test.cc b/tensorflow/contrib/lite/kernels/mean_test.cc
index 2d6d4bc2da..79c9957f76 100644
--- a/tensorflow/contrib/lite/kernels/mean_test.cc
+++ b/tensorflow/contrib/lite/kernels/mean_test.cc
@@ -37,8 +37,15 @@ class BaseMeanOpModel : public SingleOpModel {
     return ExtractVector<T>(output_);
   }
 
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
+                               GetScale(output_), GetZeroPoint(output_));
+  }
+
   std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
 
+  int Input() { return input_; }
+
  protected:
   int input_;
   int axis_;
@@ -142,56 +149,64 @@ TEST(DynamicFloatMeanOpTest, Scale) {
   EXPECT_THAT(m.GetOutput<float>(), ElementsAreArray(ArrayFloatNear({9.527})));
 }
 
+// for quantized Add, the error shouldn't exceed step
+float GetTolerance(int min, int max) { return (max - min) / 255.0; }
+
 TEST(ConstUint8MeanOpTest, NotKeepDims) {
-  std::initializer_list<uint8_t> data = {1,  2,  3,  4,  5,  6,  7,  8,
-                                         9,  10, 11, 12, 13, 14, 15, 16,
-                                         17, 18, 19, 20, 21, 22, 23, 24};
-  MeanOpConstModel m({TensorType_UINT8, {4, 3, 2}}, {TensorType_UINT8, {2}},
-                     {4}, {1, 0, -3, -3}, false);
-  m.SetInput(data);
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::initializer_list<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MeanOpConstModel m({TensorType_UINT8, {1, 3, 2}, -1.0, 1.0},
+                     {TensorType_UINT8, {2}, -1.0, 1.0}, {1}, {1}, false);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
-  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
-  EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({12, 13}));
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(m.GetDequantizedOutput(), ElementsAreArray(ArrayFloatNear(
+                                            {0.4, 0.4}, kQuantizedTolerance)));
 }
 
 TEST(ConstUint8MeanOpTest, KeepDims) {
-  std::initializer_list<uint8_t> data = {1,  2,  3,  4,  5,  6,  7,  8,
-                                         9,  10, 11, 12, 13, 14, 15, 16,
-                                         17, 18, 19, 20, 21, 22, 23, 24};
-  MeanOpConstModel m({TensorType_UINT8, {4, 3, 2}}, {TensorType_UINT8, {3}},
-                     {2}, {0, 2}, true);
-  m.SetInput(data);
+  float kQuantizedTolerance = GetTolerance(-1.0, 1.0);
+  std::initializer_list<float> data = {0.4, 0.2, 0.3, 0.4, 0.5, 0.6};
+  MeanOpConstModel m({TensorType_UINT8, {3, 2}, -1.0, 1.0},
+                     {TensorType_UINT8, {3}, -1.0, 1.0}, {1}, {1}, true);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
-  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
-  EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({10, 12, 14}));
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear({0.3, 0.35, 0.55}, kQuantizedTolerance)));
 }
 
 TEST(DynamicUint8MeanOpTest, NotKeepDims) {
-  std::initializer_list<uint8_t> data = {1,  2,  3,  4,  5,  6,  7,  8,
-                                         9,  10, 11, 12, 13, 14, 15, 16,
-                                         17, 18, 19, 20, 21, 22, 23, 24};
-  MeanOpDynamicModel m({TensorType_UINT8, {4, 3, 2}}, {TensorType_UINT8, {2}},
-                       {TensorType_INT32, {4}}, false);
-  std::initializer_list<int> axis = {1, 0, -3, -3};
+  float kQuantizedTolerance = GetTolerance(-5.0, 2.0);
+  std::initializer_list<float> data = {1.3, -4.8, -3.6, 0.24};
+  MeanOpDynamicModel m({TensorType_UINT8, {2, 2}, -5.0, 2.0},
+                       {TensorType_UINT8, {2}, -5.0, 2.0},
+                       {TensorType_INT32, {1}}, false);
+  std::initializer_list<int> axis = {1};
   m.SetAxis(axis);
-  m.SetInput(data);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
   EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({2}));
-  EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({12, 13}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear({-1.75, -1.68}, kQuantizedTolerance)));
 }
 
 TEST(DynamicUint8MeanOpTest, KeepDims) {
-  std::initializer_list<uint8_t> data = {1,  2,  3,  4,  5,  6,  7,  8,
-                                         9,  10, 11, 12, 13, 14, 15, 16,
-                                         17, 18, 19, 20, 21, 22, 23, 24};
-  MeanOpDynamicModel m({TensorType_UINT8, {4, 3, 2}}, {TensorType_UINT8, {3}},
-                       {TensorType_INT32, {2}}, true);
-  std::initializer_list<int> axis = {0, 2};
+  float kQuantizedTolerance = GetTolerance(-10.0, 12.0);
+  std::initializer_list<float> data = {11.14, -0.14, 7.423, 0.879};
+  MeanOpDynamicModel m({TensorType_UINT8, {2, 2}, -10.0, 12.0},
+                       {TensorType_UINT8, {2}, -10.0, 12.0},
+                       {TensorType_INT32, {1}}, true);
+  std::initializer_list<int> axis = {0};
   m.SetAxis(axis);
-  m.SetInput(data);
+  m.QuantizeAndPopulate<uint8_t>(m.Input(), data);
   m.Invoke();
-  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 3, 1}));
-  EXPECT_THAT(m.GetOutput<uint8_t>(), ElementsAreArray({10, 12, 14}));
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({1, 2}));
+  EXPECT_THAT(
+      m.GetDequantizedOutput(),
+      ElementsAreArray(ArrayFloatNear({9.2815, 0.3695}, kQuantizedTolerance)));
 }
 
 }  // namespace
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc b/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
index 5cc82da5d5..7c97ef0d31 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
@@ -332,6 +332,7 @@ bool HardcodeMinMax::Run(Model* model, std::size_t op_index) {
     case OperatorType::kPad:
     case OperatorType::kGather:
     case OperatorType::kTranspose:
+    case OperatorType::kMean:
       changed = HardcodeMinMaxFromFirstInput(model, op);
       break;
 
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
index 9679ea0a77..9fcc95e1fe 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
@@ -52,7 +52,7 @@ bool SupportsQuantization(const Operator& op) {
          type == OperatorType::kStridedSlice ||
          type == OperatorType::kDepthToSpace ||
          type == OperatorType::kLstmCell || type == OperatorType::kGather ||
-         type == OperatorType::kTranspose;
+         type == OperatorType::kTranspose || type == OperatorType::kMean;
 }
 
 template <ArrayDataType A>
-- 
GitLab


From dcbc0f007da212ae123efdd9eb86a72208a849da Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Wed, 28 Mar 2018 15:59:04 -0400
Subject: [PATCH 624/960] Update api.py (#18049)

Avoid overwriting existing namespace items that might replace the converted functions.
---
 tensorflow/contrib/autograph/impl/api.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/impl/api.py b/tensorflow/contrib/autograph/impl/api.py
index 1c4fcaa622..dce994e50d 100644
--- a/tensorflow/contrib/autograph/impl/api.py
+++ b/tensorflow/contrib/autograph/impl/api.py
@@ -247,7 +247,10 @@ def to_graph(e,
   # The compiled code should see everything the entry function saw.
   # TODO(mdan): This might not work well if the call tree spans modules?
   if tf_inspect.isfunction(e):
-    compiled_node.__dict__.update(inspect_utils.getnamespace(e))
+    for key, val in inspect_utils.getnamespace(e).items():
+      # Avoid overwriting entities that have been transformed.
+      if key not in compiled_node.__dict__:
+        compiled_node.__dict__[key] = val
   compiled_fn = getattr(compiled_node, name)
 
   if verbose:
-- 
GitLab


From 52aeafdf04af9f95500067dc353fd80728032b63 Mon Sep 17 00:00:00 2001
From: ngc92 <7938269+ngc92@users.noreply.github.com>
Date: Wed, 28 Mar 2018 21:59:25 +0200
Subject: [PATCH 625/960] documenting that init_op will not be run when loading
 from checkpoint (#18051)

---
 tensorflow/python/training/session_manager.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/training/session_manager.py b/tensorflow/python/training/session_manager.py
index 360e02fb44..a00ceb9021 100644
--- a/tensorflow/python/training/session_manager.py
+++ b/tensorflow/python/training/session_manager.py
@@ -229,10 +229,14 @@ class SessionManager(object):
     up to `max_wait_secs`, for recovery to succeed.
 
     If the model cannot be recovered successfully then it is initialized by
-    either running the provided `init_op`, or calling the provided `init_fn`.
-    The local_init_op is also run after init_op and init_fn, regardless of
+    running the `init_op` and calling `init_fn` if they are provided.
+    The `local_init_op` is also run after init_op and init_fn, regardless of
     whether the model was recovered successfully, but only if
-    ready_for_local_init_op passes.
+    `ready_for_local_init_op` passes.
+
+    If the model is recovered from a checkpoint it is assumed that all
+    global variables have been initialized, in particular neither `init_op`
+    nor `init_fn` will be executed.
 
     It is an error if the model cannot be recovered and no `init_op`
     or `init_fn` or `local_init_op` are passed.
-- 
GitLab


From 23c9e506bba637d9528cdf0c3a18a4cb05135a3a Mon Sep 17 00:00:00 2001
From: Loo Rong Jie <loorongjie@gmail.com>
Date: Thu, 29 Mar 2018 04:03:55 +0800
Subject: [PATCH 626/960] Replace PLATFORM_WINDOWS to _MSC_VER as it only
 applies to MSVC (#18047)

---
 tensorflow/core/platform/cpu_info.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/platform/cpu_info.h b/tensorflow/core/platform/cpu_info.h
index 331f3e5251..bb77650e26 100644
--- a/tensorflow/core/platform/cpu_info.h
+++ b/tensorflow/core/platform/cpu_info.h
@@ -18,7 +18,7 @@ limitations under the License.
 
 #include <string>
 
-#if defined(PLATFORM_WINDOWS)
+#if defined(_MSC_VER)
 #include "tensorflow/core/platform/windows/cpu_info.h"
 #endif
 
-- 
GitLab


From 6708117d292e09f259a4c685f8ca4d81cd6a0bd9 Mon Sep 17 00:00:00 2001
From: shengfuintel <sheng.fu@intel.com>
Date: Wed, 28 Mar 2018 13:04:08 -0700
Subject: [PATCH 627/960] Fixed the bug in mkl_input_conversion to compare
 tensorflow shape instead of mkl shape (#18033)

---
 .../core/kernels/mkl_input_conversion_op.cc   | 52 +++++++++----------
 1 file changed, 25 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/kernels/mkl_input_conversion_op.cc b/tensorflow/core/kernels/mkl_input_conversion_op.cc
index d91f7107c5..68d3e1c9ab 100644
--- a/tensorflow/core/kernels/mkl_input_conversion_op.cc
+++ b/tensorflow/core/kernels/mkl_input_conversion_op.cc
@@ -263,21 +263,18 @@ class MklInputConversionOp : public OpKernel {
 
  private:
   void Compute(OpKernelContext* context) override {
-    const Tensor& input_tensor_0 = MklGetInput(context, 0);
+    const int kInputIndex_0 = 0, kInputIndex_1 = 1;
+    const Tensor& input_tensor_0 = MklGetInput(context, kInputIndex_0);
     MklDnnShape input_shape_0;
-    GetMklShape(context, 0, &input_shape_0);
+    GetMklShape(context, kInputIndex_0, &input_shape_0);
 
-    const Tensor& input_tensor_1 = MklGetInput(context, 1);
+    const Tensor& input_tensor_1 = MklGetInput(context, kInputIndex_1);
     MklDnnShape input_shape_1;
-    GetMklShape(context, 1, &input_shape_1);
-
-    bool tf_shapes_are_same =
-        context->input(0).shape() == context->input(1).shape();
+    GetMklShape(context, kInputIndex_1, &input_shape_1);
 
-    VLOG(1) << "MklInputConversionOp: Input shapes are "
-            << (tf_shapes_are_same ? "*same*" : "*different*") << ": "
-            << context->input(0).shape().DebugString() << " and "
-            << context->input(1).shape().DebugString();
+    VLOG(1) << "MklInputConversionOp: Input shapes are: "
+            << context->input(kInputIndex_0).shape().DebugString() << " and "
+            << context->input(kInputIndex_1).shape().DebugString();
 
     // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
     // if both inputs are in TF format, just copy input tensors to output.
@@ -285,15 +282,19 @@ class MklInputConversionOp : public OpKernel {
       VLOG(1) << "MklInputConversionOp: No conversion needed, "
               << "copying TF inputs to output";
 
-      ForwardTfTensorInToOut(context, 0, 0);
-      ForwardTfTensorInToOut(context, 1, 1);
+      ForwardTfTensorInToOut(context, kInputIndex_0, kInputIndex_0);
+      ForwardTfTensorInToOut(context, kInputIndex_1, kInputIndex_1);
       return;
     }
 
     // - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - -
     // If both inputs are in MKL format
     if (input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
-      if (tf_shapes_are_same) {
+      // It is safer to compare the original TensorFlow shapes than to compare
+      // Mkl shapes since element wise ops are forwarded to Eigen implementation.
+      TensorShape tf_shape0 = input_shape_0.GetTfShape();
+      TensorShape tf_shape1 = input_shape_1.GetTfShape();
+      if (tf_shape0 == tf_shape1) {
         auto input0_md = input_shape_0.GetMklLayout();
         auto input1_md = input_shape_1.GetMklLayout();
 
@@ -302,8 +303,8 @@ class MklInputConversionOp : public OpKernel {
           VLOG(1) << "MklInputConversionOp: No conversion needed, "
                   << "copying MKL inputs with identical shapes to output";
 
-          ForwardMklTensorInToOut(context, 0, 0);
-          ForwardMklTensorInToOut(context, 1, 1);
+          ForwardMklTensorInToOut(context, kInputIndex_0, kInputIndex_0);
+          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
           return;
         } else {
           VLOG(1) << "MklInputConversionOp: Shape is same, but format is "
@@ -324,7 +325,7 @@ class MklInputConversionOp : public OpKernel {
           mkl_output_mkl_shape.SetMklLayout(&input1_md);
 
           // Create output Mkl tensor for index 0
-          AllocateOutputSetMklShape(context, 0, &tensor_out,
+          AllocateOutputSetMklShape(context, kInputIndex_0, &tensor_out,
                                     input_tensor_0.shape(),
                                     mkl_output_mkl_shape);
 
@@ -342,7 +343,7 @@ class MklInputConversionOp : public OpKernel {
           stream(stream::kind::eager).submit(net).wait();
 
           // Input1 will be passed through
-          ForwardMklTensorInToOut(context, 1, 1);
+          ForwardMklTensorInToOut(context, kInputIndex_1, kInputIndex_1);
           return;
         }
       }
@@ -361,11 +362,11 @@ class MklInputConversionOp : public OpKernel {
               << "converted MKL inputs to TF format";
 
       MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
-                                           op_data_type, has_avx512f_, 0);
+                                           op_data_type, has_avx512f_, kInputIndex_0);
       MklToTfOp<Device, T>::ConvertMklToTf(this, context, data_format_str,
-                                           op_data_type, has_avx512f_, 1);
-      SetDummyMklShapeOutput(context, 0);
-      SetDummyMklShapeOutput(context, 1);
+                                           op_data_type, has_avx512f_, kInputIndex_1);
+      SetDummyMklShapeOutput(context, kInputIndex_0);
+      SetDummyMklShapeOutput(context, kInputIndex_1);
       return;
     }
 
@@ -377,7 +378,6 @@ class MklInputConversionOp : public OpKernel {
     const Tensor* mkl_tensor;
     const MklDnnShape* mkl_shape;
     const Tensor* tf_tensor;
-    MklDnnShape* tf_mkl_shape;
     uint mkl_tensor_index;
     uint tf_tensor_index;
     if (input_shape_0.IsMklTensor() && !input_shape_1.IsMklTensor()) {
@@ -385,14 +385,12 @@ class MklInputConversionOp : public OpKernel {
       mkl_shape = &input_shape_0;
       mkl_tensor_index = 0;
       tf_tensor = &input_tensor_1;
-      tf_mkl_shape = &input_shape_1;
       tf_tensor_index = 1;
     } else if (!input_shape_0.IsMklTensor() && input_shape_1.IsMklTensor()) {
       mkl_tensor = &input_tensor_1;
       mkl_shape = &input_shape_1;
       mkl_tensor_index = 1;
       tf_tensor = &input_tensor_0;
-      tf_mkl_shape = &input_shape_0;
       tf_tensor_index = 0;
     } else {
       CHECK(false) << "MklInputConversionOp: Unexpected combination of input "
@@ -466,8 +464,8 @@ class MklInputConversionOp : public OpKernel {
     }
 
     VLOG(1) << "MklInputConversionOp: Shapes (output): "
-            << context->mutable_output(0)->shape().DebugString() << " and "
-            << context->mutable_output(1)->shape().DebugString();
+            << context->mutable_output(kInputIndex_0)->shape().DebugString() << " and "
+            << context->mutable_output(kInputIndex_1)->shape().DebugString();
 
     VLOG(1) << "MklInputConversion completed successfully.";
   }
-- 
GitLab


From bb4e724f429ae5c9afad3a343dc1f483ecde1f74 Mon Sep 17 00:00:00 2001
From: George Sterpu <george.sterpu@gmail.com>
Date: Wed, 28 Mar 2018 22:05:28 +0200
Subject: [PATCH 628/960] small update ctc_ops docstring (#18046)

---
 tensorflow/python/ops/ctc_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/ctc_ops.py b/tensorflow/python/ops/ctc_ops.py
index 4b57e2de79..908e793902 100644
--- a/tensorflow/python/ops/ctc_ops.py
+++ b/tensorflow/python/ops/ctc_ops.py
@@ -218,7 +218,7 @@ def ctc_greedy_decoder(inputs, sequence_length, merge_repeated=True):
         The rows store: `[batch, time]`.
       `decoded.values`: Values vector, size `(total_decoded_outputs)`.
         The vector stores the decoded classes.
-      `decoded.shape`: Shape vector, size `(2)`.
+      `decoded.dense_shape`: Shape vector, size `(2)`.
         The shape values are: `[batch_size, max_decoded_length]`
     neg_sum_logits: A `float` matrix `(batch_size x 1)` containing, for the
         sequence found, the negative of the sum of the greatest logit at each
@@ -265,7 +265,7 @@ def ctc_beam_search_decoder(inputs, sequence_length, beam_width=100,
         The rows store: [batch, time].
       `decoded[j].values`: Values vector, size `(total_decoded_outputs[j])`.
         The vector stores the decoded classes for beam j.
-      `decoded[j].shape`: Shape vector, size `(2)`.
+      `decoded[j].dense_shape`: Shape vector, size `(2)`.
         The shape values are: `[batch_size, max_decoded_length[j]]`.
     log_probability: A `float` matrix `(batch_size x top_paths)` containing
         sequence log-probabilities.
-- 
GitLab


From 480ac84aa8390e19a54bd2feef3a6069d959bb4e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 13:11:12 -0700
Subject: [PATCH 629/960] Add op cost model for MaxPool, AvgPool,
 FusedBatchNorm, their grad ops, and ReluGrad.

PiperOrigin-RevId: 190821116
---
 .../grappler/costs/op_level_cost_estimator.cc | 306 +++++++++++++-
 .../grappler/costs/op_level_cost_estimator.h  |  14 +-
 .../costs/op_level_cost_estimator_test.cc     | 391 ++++++++++++++++++
 3 files changed, 709 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 905cc2a215..0f6307cfdf 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -50,6 +50,12 @@ constexpr char kPreventGradient[] = "PreventGradient";
 constexpr char kGather[] = "Gather";
 constexpr char kGatherV2[] = "GatherV2";
 constexpr char kSlice[] = "Slice";
+constexpr char kMaxPool[] = "MaxPool";
+constexpr char kMaxPoolGrad[] = "MaxPoolGrad";
+constexpr char kAvgPool[] = "AvgPool";
+constexpr char kAvgPoolGrad[] = "AvgPoolGrad";
+constexpr char kFusedBatchNorm[] = "FusedBatchNorm";
+constexpr char kFusedBatchNormGrad[] = "FusedBatchNormGrad";
 
 static const Costs::Duration kMinComputeTime(1);
 
@@ -71,14 +77,39 @@ Padding GetPadding(const OpInfo& op_features) {
   return Padding::SAME;  // Default padding.
 }
 
+bool IsTraining(const OpInfo& op_info) {
+  if (op_info.attr().find("is_training") != op_info.attr().end() &&
+      op_info.attr().at("is_training").b()) {
+    return true;
+  }
+  return false;
+}
+
+// TODO(dyoon): support non-4D tensors in the c ost functions of convolution
+// related ops (Conv, Pool, BatchNorm, and their backprops) and the related
+// helper functions.
 std::vector<int64> GetStrides(const OpInfo& op_features) {
   if (op_features.attr().find("strides") != op_features.attr().end()) {
     const auto strides = op_features.attr().at("strides").list().i();
+    CHECK(strides.size() == 4) << "Attr strides is not a length-4 vector: "
+                               << op_features.DebugString();
     return {strides[0], strides[1], strides[2], strides[3]};
   }
   return {1, 1, 1, 1};
 }
 
+std::vector<int64> GetKernelSize(const OpInfo& op_info) {
+  if (op_info.attr().find("ksize") != op_info.attr().end()) {
+    const auto ksize = op_info.attr().at("ksize").list().i();
+    CHECK(ksize.size() == 4)
+        << "Attr ksize is not a length-4 vector: " << op_info.DebugString();
+    return {ksize[0], ksize[1], ksize[2], ksize[3]};
+  }
+  // Note that FusedBatchNorm doesn't have ksize attr, but GetKernelSize returns
+  // {1, 1, 1, 1} in that case.
+  return {1, 1, 1, 1};
+}
+
 int64 GetOutputSize(const int64 input, const int64 filter, const int64 stride,
                     const Padding& padding) {
   // Logic for calculating output shape is from GetWindowedOutputSizeVerbose()
@@ -193,7 +224,15 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kRank, wrap(&OpLevelCostEstimator::PredictMetadata)},
       {kShape, wrap(&OpLevelCostEstimator::PredictMetadata)},
-      {kSize, wrap(&OpLevelCostEstimator::PredictMetadata)}};
+      {kSize, wrap(&OpLevelCostEstimator::PredictMetadata)},
+      {kMaxPool, wrap(&OpLevelCostEstimator::PredictMaxPool)},
+      {kMaxPoolGrad, wrap(&OpLevelCostEstimator::PredictMaxPoolGrad)},
+      {kAvgPool, wrap(&OpLevelCostEstimator::PredictAvgPool)},
+      {kAvgPoolGrad, wrap(&OpLevelCostEstimator::PredictAvgPoolGrad)},
+      {kFusedBatchNorm, wrap(&OpLevelCostEstimator::PredictFusedBatchNorm)},
+      {kFusedBatchNormGrad,
+       wrap(&OpLevelCostEstimator::PredictFusedBatchNormGrad)},
+  };
 
 #define EIGEN_COST(X) Eigen::internal::functor_traits<Eigen::internal::X>::Cost
 
@@ -258,6 +297,7 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
                       {"QuantizedAdd", EIGEN_COST(scalar_sum_op<float>)},
                       {"QuantizedMul", EIGEN_COST(scalar_product_op<float>)},
                       {"RealDiv", EIGEN_COST(scalar_quotient_op<float>)},
+                      {"ReluGrad", EIGEN_COST(scalar_max_op<float>)},
                       {"SquareDifference", 1},
                       {"Sub", EIGEN_COST(scalar_difference_op<float>)},
                       {"TruncateDiv", EIGEN_COST(scalar_quotient_op<float>)},
@@ -1044,5 +1084,269 @@ Costs OpLevelCostEstimator::PredictGatherOrSlice(
   return costs;
 }
 
+/* static */
+OpLevelCostEstimator::ConvolutionDimensions
+OpLevelCostEstimator::OpDimensionsFromInputs(
+    const TensorShapeProto& original_image_shape, const OpInfo& op_info,
+    bool* found_unknown_shapes) {
+  VLOG(2) << "op features: " << op_info.DebugString();
+  VLOG(2) << "Original image shape: " << original_image_shape.DebugString();
+  auto image_shape =
+      MaybeGetMinimumShape(original_image_shape, 4, found_unknown_shapes);
+  VLOG(2) << "Image shape: " << image_shape.DebugString();
+
+  int x_index, y_index, channel_index;
+  const string& data_format = GetDataFormat(op_info);
+  if (data_format == "NCHW") {
+    x_index = 2;
+    y_index = 3;
+    channel_index = 1;
+  } else {
+    x_index = 1;
+    y_index = 2;
+    channel_index = 3;
+  }
+  int64 batch = image_shape.dim(0).size();
+  int64 ix = image_shape.dim(x_index).size();
+  int64 iy = image_shape.dim(y_index).size();
+  int64 iz = image_shape.dim(channel_index).size();
+
+  // Note that FusedBatchNorm doesn't have ksize attr, but GetKernelSize returns
+  // {1, 1, 1, 1} in that case.
+  std::vector<int64> ksize = GetKernelSize(op_info);
+  int64 kx = ksize[x_index];
+  int64 ky = ksize[y_index];
+
+  std::vector<int64> strides = GetStrides(op_info);
+  int64 sx = strides[x_index];
+  int64 sy = strides[y_index];
+  const auto padding = GetPadding(op_info);
+
+  int64 ox = GetOutputSize(ix, kx, sx, padding);
+  int64 oy = GetOutputSize(iy, ky, sy, padding);
+  int64 oz = iz;
+
+  OpLevelCostEstimator::ConvolutionDimensions conv_dims = {
+      batch, ix, iy, iz, kx, ky, oz, ox, oy, sx, sy, padding};
+  return conv_dims;
+}
+
+Costs OpLevelCostEstimator::PredictMaxPool(const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // x: op_info.inputs(0)
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+  // kx * ky - 1 comparisons per output (kx * xy > 1)
+  // or 1 copy per output (kx * k1 = 1).
+  int per_output_ops = dims.kx * dims.ky == 1 ? 1 : dims.kx * dims.ky - 1;
+  int64 ops = dims.batch * dims.ox * dims.oy * dims.oz * per_output_ops;
+
+  double total_input_size = 0;
+  if (dims.ky >= dims.sy) {
+    total_input_size =
+        CalculateTensorSize(op_info.inputs(0), &found_unknown_shapes);
+  } else {  // dims.ky < dims.sy
+    // Vertical stride is larger than vertical kernel; assuming row-major
+    // format, skip unnecessary rows (or read every kx rows per sy rows, as the
+    // others are not used for output).
+    const auto data_size = DataTypeSize(BaseType(op_info.inputs(0).dtype()));
+    total_input_size =
+        data_size * dims.batch * dims.ix * dims.ky * dims.oy * dims.iz;
+  }
+  const double total_output_size =
+      CalculateOutputSize(op_info, &found_unknown_shapes);
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size, op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
+Costs OpLevelCostEstimator::PredictMaxPoolGrad(
+    const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // x: op_info.inputs(0)
+  // y: op_info.inputs(1)
+  // y_grad: op_info.inputs(2)
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+
+  int64 ops = 0;
+  if (dims.kx == 1 && dims.ky == 1) {
+    // 1x1 window. No need to know which input was max.
+    ops = dims.batch * dims.ix * dims.iy * dims.iz;
+  } else if (dims.kx <= dims.sx && dims.ky <= dims.sy) {
+    // Non-overlapping window: re-run maxpool, then assign zero or y_grad.
+    ops = dims.batch * dims.iz *
+          (dims.ox * dims.oy * (dims.kx * dims.ky - 1) + dims.ix * dims.iy);
+  } else {
+    // Overlapping window: initialize with zeros, re-run maxpool, then
+    // accumulate y_gad to proper x_grad locations.
+    ops = dims.batch * dims.iz *
+          (dims.ox * dims.oy * (dims.kx * dims.ky - 1) + dims.ix * dims.iy * 2);
+  }
+
+  // Just read x and y_grad; no need to read y as we assume MaxPoolGrad re-run
+  // MaxPool internally.
+  double total_input_size =
+      CalculateTensorSize(op_info.inputs(0), &found_unknown_shapes);
+  total_input_size +=
+      CalculateTensorSize(op_info.inputs(2), &found_unknown_shapes);
+  // Write x_grad; size equal to x.
+  const double total_output_size =
+      CalculateTensorSize(op_info.inputs(0), &found_unknown_shapes);
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size, op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
+Costs OpLevelCostEstimator::PredictAvgPool(const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // x: op_info.inputs(0)
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+
+  // kx * ky - 1 additions and 1 multiplication per output.
+  int64 ops = dims.batch * dims.ox * dims.oy * dims.oz * dims.kx * dims.ky;
+
+  double total_input_size = 0;
+  if (dims.ky >= dims.sy) {
+    total_input_size =
+        CalculateTensorSize(op_info.inputs(0), &found_unknown_shapes);
+  } else {  // dims.ky < dims.sy
+    // vertical stride is larger than vertical kernel; assuming row-major
+    // format, skip unnecessary rows (or read every kx rows per sy rows, as the
+    // others are not used for output).
+    const auto data_size = DataTypeSize(BaseType(op_info.inputs(0).dtype()));
+    total_input_size =
+        data_size * dims.batch * dims.ix * dims.ky * dims.oy * dims.iz;
+  }
+  const double total_output_size =
+      CalculateOutputSize(op_info, &found_unknown_shapes);
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size, op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
+Costs OpLevelCostEstimator::PredictAvgPoolGrad(
+    const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // x: op_info.inputs(0)
+  // y_grad: op_info.inputs(1)
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+
+  int64 ops = 0;
+  if (dims.kx <= dims.sx && dims.ky <= dims.sy) {
+    // Non-overlapping window.
+    ops = dims.batch * dims.iz * (dims.ix * dims.iy + dims.ox * dims.oy);
+  } else {
+    // Overlapping window.
+    ops = dims.batch * dims.iz *
+          (dims.ix * dims.iy + dims.ox * dims.oy * (dims.kx * dims.ky + 1));
+  }
+
+  const double total_input_size =
+      CalculateInputSize(op_info, &found_unknown_shapes);
+  const double total_output_size =
+      CalculateOutputSize(op_info, &found_unknown_shapes);
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size, op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
+Costs OpLevelCostEstimator::PredictFusedBatchNorm(
+    const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // x: op_info.inputs(0)
+  // scale: op_info.inputs(1)
+  // offset: op_info.inputs(2)
+  // mean: op_info.inputs(3)  --> only for inference
+  // variance: op_info.inputs(4) --> only for inference
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+  const bool is_training = IsTraining(op_info);
+
+  int64 ops = 0;
+  const auto rsqrt_cost = Eigen::internal::functor_traits<
+      Eigen::internal::scalar_rsqrt_op<float>>::Cost;
+  if (is_training) {
+    ops = dims.iz * (dims.batch * dims.ix * dims.iy * 4 + 6 + rsqrt_cost);
+  } else {
+    ops = dims.batch * dims.ix * dims.iy * dims.iz * 2;
+  }
+
+  const double size_nhwc =
+      CalculateTensorSize(op_info.inputs(0), &found_unknown_shapes);
+  const double size_c =
+      CalculateTensorSize(op_info.inputs(1), &found_unknown_shapes);
+  double total_input_size = 0.0;
+  double total_internal_read_size = 0.0;
+  double total_output_size = 0.0;
+  if (is_training) {
+    total_input_size = size_nhwc + size_c * 2;
+    total_output_size = size_nhwc + size_c * 4;
+    total_internal_read_size = size_nhwc;
+  } else {
+    total_input_size = size_nhwc + size_c * 4;
+    total_output_size = size_nhwc;
+  }
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size + total_internal_read_size,
+      op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
+Costs OpLevelCostEstimator::PredictFusedBatchNormGrad(
+    const OpContext& op_context) const {
+  bool found_unknown_shapes = false;
+  const auto& op_info = op_context.op_info;
+  // y_backprop: op_info.inputs(0)
+  // x: op_info.inputs(1)
+  // scale: op_info.inputs(2)
+  // mean: op_info.inputs(3)
+  // variance or inverse of variance: op_info.inputs(4)
+  ConvolutionDimensions dims = OpDimensionsFromInputs(
+      op_info.inputs(1).shape(), op_info, &found_unknown_shapes);
+
+  int64 ops = 0;
+  const auto rsqrt_cost = Eigen::internal::functor_traits<
+      Eigen::internal::scalar_rsqrt_op<float>>::Cost;
+  ops = dims.iz * (dims.batch * dims.ix * dims.iy * 11 + 5 + rsqrt_cost);
+
+  const double size_nhwc =
+      CalculateTensorSize(op_info.inputs(1), &found_unknown_shapes);
+  const double size_c =
+      CalculateTensorSize(op_info.inputs(2), &found_unknown_shapes);
+  double total_input_size = size_nhwc * 2 + size_c * 2;
+  double total_internal_read_size = size_nhwc;
+  double total_output_size = size_nhwc * 1 + size_c * 2;
+
+  Costs costs = PredictOpCountBasedCost(
+      ops, total_input_size + total_output_size + total_internal_read_size,
+      op_info);
+  costs.inaccurate = found_unknown_shapes;
+  costs.max_memory = total_output_size;
+  return costs;
+}
+
 }  // end namespace grappler
 }  // end namespace tensorflow
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.h b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
index 1b3babb206..fcbecbb6dc 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.h
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.h
@@ -145,6 +145,12 @@ class OpLevelCostEstimator {
   Costs PredictBatchMatMul(const OpContext& op_context) const;
   Costs PredictMetadata(const OpContext& op_context) const;
   Costs PredictGatherOrSlice(const OpContext& op_context) const;
+  Costs PredictMaxPool(const OpContext& op_context) const;
+  Costs PredictMaxPoolGrad(const OpContext& op_context) const;
+  Costs PredictAvgPool(const OpContext& op_context) const;
+  Costs PredictAvgPoolGrad(const OpContext& op_context) const;
+  Costs PredictFusedBatchNorm(const OpContext& op_context) const;
+  Costs PredictFusedBatchNormGrad(const OpContext& op_context) const;
 
   // Utility function for safe division. Returns 0
   // if rhs is 0 or negative.
@@ -156,9 +162,15 @@ class OpLevelCostEstimator {
     }
   }
 
+  // For convolution and its grad ops.
   static ConvolutionDimensions ConvolutionDimensionsFromInputs(
       const TensorShapeProto& original_image_shape,
-      const TensorShapeProto& original_filter_shape, const OpInfo& op_features,
+      const TensorShapeProto& original_filter_shape, const OpInfo& op_info,
+      bool* found_unknown_shapes);
+
+  // For Pooling, FusedBatchNorm, and their grad ops.
+  static ConvolutionDimensions OpDimensionsFromInputs(
+      const TensorShapeProto& original_image_shape, const OpInfo& op_info,
       bool* found_unknown_shapes);
 
  protected:
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 99bf28f21b..56915ed821 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -14,6 +14,8 @@ limitations under the License.
 ==============================================================================*/
 
 #include "tensorflow/core/grappler/costs/op_level_cost_estimator.h"
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/attr_value_util.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/framework/tensor_shape.pb.h"
@@ -169,6 +171,130 @@ OpContext DescribeBiasAdd(int size1, int size2) {
   return op_context;
 }
 
+int GetOutputSize(const int x, const int k, const int s,
+                  const string& padding) {
+  if (padding == "SAME") {
+    return (x + s - 1) / s;
+  } else {
+    return (x - k + s) / s;
+  }
+}
+
+std::vector<int> GetPoolingOutputSize(const std::vector<int>& input,
+                                      const std::vector<int>& ksize,
+                                      const std::vector<int>& strides,
+                                      const string& data_format,
+                                      const string& padding) {
+  // h, w, and c indices: default with NHWC.
+  int h_index = 1;
+  int w_index = 2;
+  int c_index = 3;
+  if (data_format == "NCHW") {
+    h_index = 2;
+    w_index = 3;
+    c_index = 1;
+  }
+  // Extract parameters.
+  int n = input[0];
+  int h = input[h_index];
+  int w = input[w_index];
+  int c = input[c_index];
+  int sx = strides[h_index];
+  int sy = strides[w_index];
+  int kx = ksize[h_index];
+  int ky = ksize[w_index];
+
+  // Output activation size: default with VALID padding.
+  int ho = GetOutputSize(h, kx, sx, padding);
+  int wo = GetOutputSize(w, ky, sy, padding);
+
+  std::vector<int> output;
+  if (data_format == "NHWC") {
+    output = {n, ho, wo, c};
+  } else {
+    output = {n, c, ho, wo};
+  }
+  return output;
+}
+
+OpContext DescribePoolingOp(const string& op_name, const std::vector<int>& x,
+                            const std::vector<int>& ksize,
+                            const std::vector<int>& strides,
+                            const string& data_format, const string& padding) {
+  OpContext op_context;
+  auto& op_info = op_context.op_info;
+  SetCpuDevice(&op_info);
+  op_info.set_op(op_name);
+
+  const std::vector<int> y =
+      GetPoolingOutputSize(x, ksize, strides, data_format, padding);
+  if (op_name == "AvgPool" || op_name == "MaxPool") {
+    // input: x, output: y.
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
+    DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_outputs());
+  } else if (op_name == "AvgPoolGrad") {
+    // input: x, y_grad, output: x_grad.
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
+    DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_inputs());
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_outputs());
+  } else if (op_name == "MaxPoolGrad") {
+    // input: x, y, y_grad, output: x_grad.
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
+    DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_inputs());
+    DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_inputs());
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_outputs());
+  }
+  auto* attr = op_info.mutable_attr();
+  SetAttrValue(data_format, &(*attr)["data_format"]);
+  SetAttrValue(padding, &(*attr)["padding"]);
+  SetAttrValue(strides, &(*attr)["strides"]);
+  SetAttrValue(ksize, &(*attr)["ksize"]);
+  return op_context;
+}
+
+OpContext DescribeFusedBatchNorm(const bool is_training, const bool is_grad,
+                                 const std::vector<int>& x,
+                                 const string& data_format) {
+  // First, get MaxPool op info with unit stride and unit window.
+  OpContext op_context = DescribePoolingOp("MaxPool", x, {1, 1, 1, 1},
+                                           {1, 1, 1, 1}, data_format, "SAME");
+  auto& op_info = op_context.op_info;
+  // Override op name.
+  if (is_grad) {
+    op_info.set_op("FusedBatchNormGrad");
+  } else {
+    op_info.set_op("FusedBatchNorm");
+  }
+
+  // Add additional input output tensors.
+  if (is_grad) {
+    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
+  }
+  int num_1d_inputs = is_grad ? 3 : 4;
+  for (int i = 0; i < num_1d_inputs; i++) {
+    auto* tensor = op_info.add_inputs();
+    auto* shape = tensor->mutable_shape();
+    shape->add_dim()->set_size(x[3]);
+    tensor->set_dtype(DT_FLOAT);
+  }
+  for (int i = 0; i < 4; i++) {
+    auto* tensor = op_info.add_outputs();
+    auto* shape = tensor->mutable_shape();
+    shape->add_dim()->set_size(x[3]);
+    tensor->set_dtype(DT_FLOAT);
+  }
+
+  // Delete unnecessary attr.
+  auto* attr = op_context.op_info.mutable_attr();
+  attr->erase("ksize");
+  attr->erase("strides");
+  attr->erase("padding");
+
+  // Additional attrs for FusedBatchNorm.
+  SetAttrValue(is_training, &(*attr)["is_training"]);
+
+  return op_context;
+}
 }  // namespace
 
 class OpLevelCostEstimatorTest : public ::testing::Test {
@@ -192,6 +318,50 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
     estimator_.compute_memory_overlap_ = value;
   }
 
+  void ValidateOpDimensionsFromImputs(const int n, const int h, const int w,
+                                      const int c, const int kx, const int ky,
+                                      const int sx, const int sy,
+                                      const string& data_format,
+                                      const string& padding) {
+    OpContext op_context;
+    int ho;
+    int wo;
+    if (data_format == "NHWC") {
+      op_context = DescribePoolingOp("MaxPool", {n, h, w, c}, {1, kx, ky, 1},
+                                     {1, sx, sy, 1}, "NHWC", padding);
+      ho = op_context.op_info.outputs(0).shape().dim(1).size();
+      wo = op_context.op_info.outputs(0).shape().dim(2).size();
+    } else {
+      op_context = DescribePoolingOp("MaxPool", {n, c, h, w}, {1, 1, kx, ky},
+                                     {1, 1, sx, sy}, "NCHW", padding);
+      ho = op_context.op_info.outputs(0).shape().dim(2).size();
+      wo = op_context.op_info.outputs(0).shape().dim(3).size();
+    }
+
+    bool found_unknown_shapes;
+    auto dims = OpLevelCostEstimator::OpDimensionsFromInputs(
+        op_context.op_info.inputs(0).shape(), op_context.op_info,
+        &found_unknown_shapes);
+    Padding padding_enum;
+    if (padding == "VALID") {
+      padding_enum = Padding::VALID;
+    } else {
+      padding_enum = Padding::SAME;
+    }
+    EXPECT_EQ(n, dims.batch);
+    EXPECT_EQ(h, dims.ix);
+    EXPECT_EQ(w, dims.iy);
+    EXPECT_EQ(c, dims.iz);
+    EXPECT_EQ(kx, dims.kx);
+    EXPECT_EQ(ky, dims.ky);
+    EXPECT_EQ(sx, dims.sx);
+    EXPECT_EQ(sy, dims.sy);
+    EXPECT_EQ(ho, dims.ox);
+    EXPECT_EQ(wo, dims.oy);
+    EXPECT_EQ(c, dims.oz);
+    EXPECT_EQ(padding_enum, dims.padding);
+  }
+
   OpLevelCostEstimator estimator_;
 };
 
@@ -443,5 +613,226 @@ TEST_F(OpLevelCostEstimatorTest, GetTensorShapeProtoFromTensorProto) {
   }
 }
 
+TEST_F(OpLevelCostEstimatorTest, OpDimensionsFromInputs) {
+  std::vector<string> paddings = {"VALID", "SAME"};
+  std::vector<string> formats = {"NHWC", "NCHW"};
+  for (const auto& p : paddings) {
+    for (const auto& f : formats) {
+      // n, h, w, c, kx, ky, sx, sy, data_format, padding.
+      ValidateOpDimensionsFromImputs(10, 20, 20, 100, 3, 3, 2, 2, f, p);
+      ValidateOpDimensionsFromImputs(10, 20, 20, 100, 1, 1, 3, 3, f, p);
+      ValidateOpDimensionsFromImputs(10, 200, 200, 100, 5, 5, 3, 3, f, p);
+      ValidateOpDimensionsFromImputs(10, 14, 14, 3840, 3, 3, 2, 2, f, p);
+    }
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictMaxPool) {
+  auto predict_max_pool = [this](const int n, const int in, const int c,
+                                 const int k, const int s,
+                                 const string& padding) -> Costs {
+    OpContext op_context = DescribePoolingOp(
+        "MaxPool", {n, in, in, c}, {1, k, k, 1}, {1, s, s, 1}, "NHWC", padding);
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    // Typical 3xz3 window with 2x2 stride.
+    auto costs = predict_max_pool(10, 20, 384, 3, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1075200), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(307200), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(768000), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 1x1 window with 2x2 stride: used for shortcut in resnet-50.
+    auto costs = predict_max_pool(10, 20, 384, 1, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(499200), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(38400), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(460800), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 2x2 window with 3x3 stride.
+    auto costs = predict_max_pool(10, 20, 384, 2, 3, "VALID");
+    EXPECT_EQ(Costs::Duration(561792), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(56448), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(505344), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictMaxPoolGrad) {
+  auto predict_max_pool_grad = [this](const int n, const int in, const int c,
+                                      const int k, const int s,
+                                      const string& padding) -> Costs {
+    OpContext op_context =
+        DescribePoolingOp("MaxPoolGrad", {n, in, in, c}, {1, k, k, 1},
+                          {1, s, s, 1}, "NHWC", padding);
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    // Typical 3xz3 window with 2x2 stride.
+    auto costs = predict_max_pool_grad(10, 20, 384, 3, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1996800), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(614400), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 1x1 window with 2x2 stride: used for shortcut in resnet-50.
+    auto costs = predict_max_pool_grad(10, 20, 384, 1, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1536000), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(153600), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 2x2 window with 3x3 stride.
+    auto costs = predict_max_pool_grad(10, 20, 384, 2, 3, "VALID");
+    EXPECT_EQ(Costs::Duration(1514112), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(210048), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1304064), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictAvgPool) {
+  auto predict_avg_pool = [this](const int n, const int in, const int c,
+                                 const int k, const int s,
+                                 const string& padding) -> Costs {
+    OpContext op_context = DescribePoolingOp(
+        "AvgPool", {n, in, in, c}, {1, k, k, 1}, {1, s, s, 1}, "NHWC", padding);
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    // Typical 3xz3 window with 2x2 stride.
+    auto costs = predict_avg_pool(10, 20, 384, 3, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1113600), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(345600), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(768000), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 1x1 window with 2x2 stride: used for shortcut in resnet-50.
+    auto costs = predict_avg_pool(10, 20, 384, 1, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(499200), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(38400), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(460800), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 2x2 window with 3x3 stride.
+    auto costs = predict_avg_pool(10, 20, 384, 2, 3, "VALID");
+    EXPECT_EQ(Costs::Duration(580608), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(75264), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(505344), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictAvgPoolGrad) {
+  auto predict_avg_pool_grad = [this](const int n, const int in, const int c,
+                                      const int k, const int s,
+                                      const string& padding) -> Costs {
+    OpContext op_context =
+        DescribePoolingOp("AvgPoolGrad", {n, in, in, c}, {1, k, k, 1},
+                          {1, s, s, 1}, "NHWC", padding);
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    // Typical 3xz3 window with 2x2 stride.
+    auto costs = predict_avg_pool_grad(10, 20, 384, 3, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1920000), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(537600), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 1x1 window with 2x2 stride: used for shortcut in resnet-50.
+    auto costs = predict_avg_pool_grad(10, 20, 384, 1, 2, "SAME");
+    EXPECT_EQ(Costs::Duration(1574400), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(192000), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+  {
+    // 2x2 window with 3x3 stride.
+    auto costs = predict_avg_pool_grad(10, 20, 384, 2, 3, "VALID");
+    EXPECT_EQ(Costs::Duration(1476480), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(172416), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(1304064), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictFusedBatchNorm) {
+  auto predict_fused_bn = [this](const int n, const int in, const int c,
+                                 const bool is_training) -> Costs {
+    OpContext op_context = DescribeFusedBatchNorm(
+        is_training, /*is_grad=*/false, {n, in, in, c}, "NHWC");
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    auto costs = predict_fused_bn(10, 20, 96, /*is_training=*/true);
+    EXPECT_EQ(Costs::Duration(614737), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(153706), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(461031), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+
+  {
+    auto costs = predict_fused_bn(10, 20, 32, /*is_training=*/true);
+    EXPECT_EQ(Costs::Duration(204913), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(51236), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(153677), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+
+  {
+    auto costs = predict_fused_bn(10, 20, 96, /*is_training=*/false);
+    EXPECT_EQ(Costs::Duration(384154), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(76800), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(307354), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+
+  {
+    auto costs = predict_fused_bn(10, 20, 32, /*is_training=*/false);
+    EXPECT_EQ(Costs::Duration(128052), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(25600), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(102452), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
+
+TEST_F(OpLevelCostEstimatorTest, PredictFusedBatchNormGrad) {
+  auto predict_fused_bn_grad = [this](const int n, const int in,
+                                      const int c) -> Costs {
+    OpContext op_context = DescribeFusedBatchNorm(
+        /*is_training=*/false, /*is_grad=*/true, {n, in, in, c}, "NHWC");
+    return estimator_.PredictCosts(op_context);
+  };
+
+  {
+    auto costs = predict_fused_bn_grad(10, 20, 96);
+    EXPECT_EQ(Costs::Duration(1037050), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(422496), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(614554), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+
+  {
+    auto costs = predict_fused_bn_grad(128, 7, 384);
+    EXPECT_EQ(Costs::Duration(6503809), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(2649677), costs.compute_time);
+    EXPECT_EQ(Costs::Duration(3854132), costs.memory_time);
+    EXPECT_FALSE(costs.inaccurate);
+  }
+}
 }  // end namespace grappler
 }  // end namespace tensorflow
-- 
GitLab


From a477242f91010480ca72b052a6adbb50f00ea43b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 13:13:20 -0700
Subject: [PATCH 630/960] Add comment that explicitly states that
 InitTableIterator is Thread-unsafe.

PiperOrigin-RevId: 190821427
---
 tensorflow/core/kernels/initializable_lookup_table.h | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/kernels/initializable_lookup_table.h b/tensorflow/core/kernels/initializable_lookup_table.h
index b16c76dc7f..edb779540f 100644
--- a/tensorflow/core/kernels/initializable_lookup_table.h
+++ b/tensorflow/core/kernels/initializable_lookup_table.h
@@ -92,6 +92,8 @@ class InitializableLookupTable : public LookupInterface {
   //
   // Then the iterator is exhausted, valid returns false and status returns
   // Status::OutOfRange.
+  //
+  // This class is Thread-unsafe.
   class InitTableIterator {
    public:
     InitTableIterator() {}
-- 
GitLab


From 01583b714c4144dbf11e1f2ae5189f051d130d13 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 13:21:05 -0700
Subject: [PATCH 631/960] [XLA] Redesign: add the rest of client-service
 interfaces.

The basic idea is, on the client side, for each public method that has a Computation parameter, add an overload with XlaCompuation. If such method needs to call the service side, add corresponding service interfaces.

Also make XlaCompuation::GetProgramShape return StatusOr, to be consistent with the Computation class.

PiperOrigin-RevId: 190822601
---
 tensorflow/compiler/xla/client/client.cc      | 19 ++++++++
 tensorflow/compiler/xla/client/client.h       | 44 +++++++++++++++++++
 .../xla/client/xla_client/xla_builder.cc      | 10 +++--
 .../xla/client/xla_client/xla_computation.cc  |  5 ++-
 .../xla/client/xla_client/xla_computation.h   |  3 +-
 tensorflow/compiler/xla/service/service.cc    | 10 +++++
 tensorflow/compiler/xla/service/service.h     | 16 +++++++
 tensorflow/compiler/xla/service_interface.h   |  8 ++++
 tensorflow/compiler/xla/xla.proto             |  9 ++++
 9 files changed, 118 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index 5ce3c45528..a857c4ff0b 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -317,6 +317,12 @@ StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
   return std::move(outputs);
 }
 
+StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
+    tensorflow::gtl::ArraySlice<XlaComputationInstance> computations) {
+  return Unimplemented(
+      "ExecuteParallel is not yet implemented for XlaComputation.");
+}
+
 StatusOr<std::vector<DeviceHandle>> Client::GetDeviceHandles(
     int64 device_count) {
   if (device_count < 1) {
@@ -393,6 +399,13 @@ StatusOr<ComputationStats> Client::GetComputationStats(
   return response.stats();
 }
 
+StatusOr<ComputationStats> Client::GetComputationStats(
+    const XlaComputation& computation,
+    const DebugOptions& debug_options) const {
+  return Unimplemented(
+      "GetComputationStats is not yet implemented for XlaComputation");
+}
+
 StatusOr<std::unique_ptr<ProgramShape>> Client::GetComputationShape(
     const Computation& computation) {
   GetComputationShapeRequest request;
@@ -410,6 +423,12 @@ StatusOr<std::unique_ptr<ProgramShape>> Client::GetComputationShape(
   return WrapUnique(response.release_program_shape());
 }
 
+StatusOr<std::unique_ptr<ProgramShape>> Client::GetComputationShape(
+    const XlaComputation& computation) {
+  TF_ASSIGN_OR_RETURN(const auto& result, computation.GetProgramShape());
+  return MakeUnique<ProgramShape>(result);
+}
+
 StatusOr<Shape> Client::GetShape(const GlobalData& data) {
   GetShapeRequest request;
   *request.mutable_data() = data.handle();
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index ec87646ebf..226b788d54 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -99,6 +99,36 @@ class Client {
   StatusOr<std::vector<std::unique_ptr<GlobalData>>> ExecuteParallel(
       tensorflow::gtl::ArraySlice<ComputationInstance> computations);
 
+  // A struct to represent a computation instance to be executed.
+  // * If execution_options.device_handles is not empty, the computation is
+  //   executed on the devices associated with the handles by partitioning the
+  //   computation based on the attached sharding attributes. Otherwise, a
+  //   device is chosen by the service.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  struct XlaComputationInstance {
+    const XlaComputation& computation;
+    std::vector<GlobalData*> arguments;
+    ExecutionOptions execution_options;
+    ExecutionProfile* execution_profile;
+
+    XlaComputationInstance(const XlaComputation& computation,
+                           std::vector<GlobalData*> arguments,
+                           ExecutionOptions execution_options,
+                           ExecutionProfile* execution_profile)
+        : computation(computation),
+          arguments(std::move(arguments)),
+          execution_options(execution_options),
+          execution_profile(execution_profile) {}
+  };
+
+  // Executes a list XlaComputationInstances and returns global data produced
+  // from each computation.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::vector<std::unique_ptr<GlobalData>>> ExecuteParallel(
+      tensorflow::gtl::ArraySlice<XlaComputationInstance> computations);
+
   // Requests device_count device handles available on the target. The returned
   // device handles are used to specify the devices to execute the computations
   // (see ExecuteParallel) or to transfer data (see TransferToServer or
@@ -175,6 +205,13 @@ class Client {
   StatusOr<ComputationStats> GetComputationStats(
       const Computation& computation, const DebugOptions& debug_options) const;
 
+  // Retrieves the statistics of the given computation.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<ComputationStats> GetComputationStats(
+      const XlaComputation& computation,
+      const DebugOptions& debug_options) const;
+
   // Returns the Shape of the given array specified by 'data'. The shape
   // includes the Layout of the array as it is stored on the service.
   StatusOr<Shape> GetShape(const GlobalData& data);
@@ -184,6 +221,13 @@ class Client {
   StatusOr<std::unique_ptr<ProgramShape>> GetComputationShape(
       const Computation& computation);
 
+  // As above, but returns the shape of the provided computation (parameter
+  // types/names and return type).
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  StatusOr<std::unique_ptr<ProgramShape>> GetComputationShape(
+      const XlaComputation& computation);
+
   // Creates a channel handle that can be used to transfer data between
   // two computations via a pair of Send and Recv instructions.
   StatusOr<ChannelHandle> CreateChannelHandle();
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 1b94f9a4eb..e51a8b14c0 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -369,10 +369,12 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
     }
     c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                 [](const Shape& shape) { return &shape; });
-    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
-                        ShapeInference::InferCallShape(
-                            operand_shape_ptrs,
-                            /*to_apply=*/computation.GetProgramShape()));
+    TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape,
+                        computation.GetProgramShape());
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferCallShape(operand_shape_ptrs,
+                                       /*to_apply=*/called_program_shape));
 
     // Add called computation.
     instr.add_called_computation_ids(
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.cc b/tensorflow/compiler/xla/client/xla_client/xla_computation.cc
index 3681792eee..a6752c6010 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_computation.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.cc
@@ -17,9 +17,12 @@ limitations under the License.
 
 #include <utility>
 
+#include "tensorflow/compiler/xla/status_macros.h"
+
 namespace xla {
 
-const ProgramShape& XlaComputation::GetProgramShape() const {
+StatusOr<ProgramShape> XlaComputation::GetProgramShape() const {
+  TF_RET_CHECK(proto_.has_program_shape());
   return proto_.program_shape();
 }
 
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_computation.h b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
index 78e1e3c32c..2a3c695266 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_computation.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_computation.h
@@ -40,7 +40,8 @@ class XlaComputation {
 
   // Returns the "program shape" (parameter and return shapes) for this
   // computation.
-  const ProgramShape& GetProgramShape() const;
+  StatusOr<ProgramShape> GetProgramShape() const;
+
   const HloModuleProto& proto() const { return proto_; }
 
  private:
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index 1d379f0d03..af05e3f516 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -837,6 +837,11 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status Service::ExecuteGraphParallel(
+    const ExecuteGraphParallelRequest* arg, ExecuteParallelResponse* result) {
+  return Unimplemented("execute-graph-parallel is not yet implemented");
+}
+
 tensorflow::Status Service::GetDeviceHandles(const GetDeviceHandlesRequest* arg,
                                              GetDeviceHandlesResponse* result) {
   const int64 available_device_count = execute_backend_->device_count();
@@ -1445,6 +1450,11 @@ tensorflow::Status Service::GetComputationStats(
   return tensorflow::Status::OK();
 }
 
+tensorflow::Status Service::GetComputationGraphStats(
+    const ComputationGraphStatsRequest* arg, ComputationStatsResponse* result) {
+  return Unimplemented("get-computation-graph-stats is not yet implemented");
+}
+
 template <typename RequestT, typename ResponseT>
 tensorflow::Status Service::AddInstruction(
     const RequestT* arg, ResponseT* result,
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index 773f0a642d..ebe4a2e043 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -126,6 +126,15 @@ class Service : public ServiceInterface {
   tensorflow::Status ExecuteParallel(const ExecuteParallelRequest* arg,
                                      ExecuteParallelResponse* result) override;
 
+  // Executes one or more computations in parallel with the provided global data
+  // passed as immutable arguments. Returns global data output for each
+  // computation.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  tensorflow::Status ExecuteGraphParallel(
+      const ExecuteGraphParallelRequest* arg,
+      ExecuteParallelResponse* result) override;
+
   // Requests one or more device handles from the target.
   //
   // When N device handles are requested and the number of replicas is R, at
@@ -224,6 +233,13 @@ class Service : public ServiceInterface {
       const ComputationStatsRequest* arg,
       ComputationStatsResponse* result) override;
 
+  // Retrieves the statistics of a computation.
+  //
+  // TODO(b/74197823): This is a part of a NOT YET ready refactor.
+  tensorflow::Status GetComputationGraphStats(
+      const ComputationGraphStatsRequest* arg,
+      ComputationStatsResponse* result) override;
+
   // Snapshots the current state of a computation handle into a serializable
   // protocol buffer form, so it can be loaded via
   // LoadComputationSnapshot.
diff --git a/tensorflow/compiler/xla/service_interface.h b/tensorflow/compiler/xla/service_interface.h
index d8235113dd..32aae64973 100644
--- a/tensorflow/compiler/xla/service_interface.h
+++ b/tensorflow/compiler/xla/service_interface.h
@@ -60,6 +60,10 @@ class ServiceInterface {
   virtual tensorflow::Status ExecuteParallel(
       const ExecuteParallelRequest* arg, ExecuteParallelResponse* result) = 0;
 
+  virtual tensorflow::Status ExecuteGraphParallel(
+      const ExecuteGraphParallelRequest* arg,
+      ExecuteParallelResponse* result) = 0;
+
   virtual tensorflow::Status ExecuteAsync(const ExecuteAsyncRequest* arg,
                                           ExecuteAsyncResponse* result) = 0;
 
@@ -72,6 +76,10 @@ class ServiceInterface {
   virtual tensorflow::Status GetComputationStats(
       const ComputationStatsRequest* arg, ComputationStatsResponse* result) = 0;
 
+  virtual tensorflow::Status GetComputationGraphStats(
+      const ComputationGraphStatsRequest* arg,
+      ComputationStatsResponse* result) = 0;
+
   virtual tensorflow::Status GetComputationShape(
       const GetComputationShapeRequest* arg,
       GetComputationShapeResponse* result) = 0;
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index edf1b07af8..5cb18113e5 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -299,6 +299,11 @@ message ComputationStatsRequest {
   DebugOptions debug_options = 2;
 }
 
+message ComputationGraphStatsRequest {
+  HloModuleProto computation = 1;
+  DebugOptions debug_options = 2;
+}
+
 message ComputationStatsResponse {
   ComputationStats stats = 1;
 }
@@ -355,6 +360,10 @@ message ExecuteParallelRequest {
   repeated ExecuteRequest requests = 1;
 }
 
+message ExecuteGraphParallelRequest {
+  repeated ExecuteGraphRequest requests = 1;
+}
+
 message ExecuteResponse {
   GlobalDataHandle output = 1;
   ExecutionProfile profile = 2;
-- 
GitLab


From 70666858800a55585ae2775f97a1731db305388a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 13:27:07 -0700
Subject: [PATCH 632/960] Make sure tensor size match before inspecting their
 content.

PiperOrigin-RevId: 190823557
---
 .../contrib/lite/testing/tflite_driver.cc       | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/lite/testing/tflite_driver.cc b/tensorflow/contrib/lite/testing/tflite_driver.cc
index 613223f3d4..c399f4f2b7 100644
--- a/tensorflow/contrib/lite/testing/tflite_driver.cc
+++ b/tensorflow/contrib/lite/testing/tflite_driver.cc
@@ -56,12 +56,16 @@ void SetTensorData(const std::vector<T>& values, TfLitePtrUnion* data) {
 
 class TfLiteDriver::Expectation {
  public:
-  Expectation() { data_.raw = nullptr; }
+  Expectation() {
+    data_.raw = nullptr;
+    num_elements_ = 0;
+  }
   ~Expectation() { delete[] data_.raw; }
   template <typename T>
   void SetData(const string& csv_values) {
     const auto& values = testing::Split<T>(csv_values, ",");
-    data_.raw = new char[values.size() * sizeof(T)];
+    num_elements_ = values.size();
+    data_.raw = new char[num_elements_ * sizeof(T)];
     SetTensorData(values, &data_);
   }
 
@@ -88,7 +92,13 @@ class TfLiteDriver::Expectation {
     constexpr double kRelativeThreshold = 1e-2f;
     constexpr double kAbsoluteThreshold = 1e-4f;
 
-    int tensor_size = tensor.bytes / sizeof(T);
+    size_t tensor_size = tensor.bytes / sizeof(T);
+
+    if (tensor_size != num_elements_) {
+      std::cerr << "Expected a tensor with " << num_elements_
+                << " elements, got " << tensor_size << std::endl;
+      return false;
+    }
 
     bool good_output = true;
     for (int i = 0; i < tensor_size; ++i) {
@@ -115,6 +125,7 @@ class TfLiteDriver::Expectation {
   }
 
   TfLitePtrUnion data_;
+  size_t num_elements_;
 };
 
 TfLiteDriver::TfLiteDriver(bool use_nnapi) : use_nnapi_(use_nnapi) {}
-- 
GitLab


From d355f4e2644b68ea643f573c564936ec23b93787 Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Wed, 28 Mar 2018 14:04:01 -0700
Subject: [PATCH 633/960] [tf.data] Autotune prefetch buffer sizes

In order to make it easier for tf.data users to achieve
high performance with their input pipelines, this change adds
the ability for the prefetch op to automatically tune its buffer
size.

To use the auto-tuning configuration of the `prefetch` transformation,
simply skip passing in a buffer size. Example:

```python
dataset = # ...
dataset = dataset.prefetch()  # Look ma, no buffer value req'd!
```
PiperOrigin-RevId: 190829736
---
 tensorflow/contrib/data/__init__.py           |  3 +
 tensorflow/core/kernels/data/BUILD            | 21 +++++
 .../core/kernels/data/prefetch_autotuner.cc   | 46 +++++++++++
 .../core/kernels/data/prefetch_autotuner.h    | 71 ++++++++++++++++
 .../kernels/data/prefetch_autotuner_test.cc   | 82 +++++++++++++++++++
 .../core/kernels/data/prefetch_dataset_op.cc  | 13 ++-
 tensorflow/python/data/ops/dataset_ops.py     |  2 +
 7 files changed, 235 insertions(+), 3 deletions(-)
 create mode 100644 tensorflow/core/kernels/data/prefetch_autotuner.cc
 create mode 100644 tensorflow/core/kernels/data/prefetch_autotuner.h
 create mode 100644 tensorflow/core/kernels/data/prefetch_autotuner_test.cc

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 766721d8d2..7c3a9f82ff 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -82,3 +82,6 @@ from tensorflow.python.ops.parsing_ops import parse_single_example_v2 as parse_s
 
 from tensorflow.python.util.all_util import remove_undocumented
 remove_undocumented(__name__)
+
+# A constant that can be used to enable auto-tuning.
+AUTOTUNE = -1
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index 01754ec21a..a8784e3656 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -10,6 +10,7 @@ licenses(["notice"])  # Apache 2.0
 load(
     "//tensorflow:tensorflow.bzl",
     "tf_kernel_library",
+    "tf_cc_test",
 )
 
 filegroup(
@@ -295,11 +296,31 @@ tf_kernel_library(
     ],
 )
 
+cc_library(
+    name = "prefetch_autotuner",
+    srcs = ["prefetch_autotuner.cc"],
+    hdrs = ["prefetch_autotuner.h"],
+    deps = [
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_cc_test(
+    name = "prefetch_autotuner_test",
+    srcs = ["prefetch_autotuner_test.cc"],
+    deps = [
+        ":prefetch_autotuner",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+    ],
+)
+
 tf_kernel_library(
     name = "prefetch_dataset_op",
     srcs = ["prefetch_dataset_op.cc"],
     deps = [
         ":dataset",
+        ":prefetch_autotuner",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:dataset_ops_op_lib",
         "//tensorflow/core:framework",
diff --git a/tensorflow/core/kernels/data/prefetch_autotuner.cc b/tensorflow/core/kernels/data/prefetch_autotuner.cc
new file mode 100644
index 0000000000..b3272f6bcd
--- /dev/null
+++ b/tensorflow/core/kernels/data/prefetch_autotuner.cc
@@ -0,0 +1,46 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/data/prefetch_autotuner.h"
+
+namespace tensorflow {
+
+PrefetchAutotuner::PrefetchAutotuner(int64 initial_buffer_size)
+    : buffer_limit_(initial_buffer_size) {
+  if (initial_buffer_size == kAutoTune) {
+    mode_ = Mode::kUpswing;
+    buffer_limit_ = 1;
+  }
+}
+
+void PrefetchAutotuner::RecordConsumption(size_t current_buffer_size) {
+  switch (mode_) {
+    case Mode::kDisabled:
+      return;
+    case Mode::kUpswing:
+      if (current_buffer_size == buffer_limit_) {
+        mode_ = Mode::kDownswing;
+      }
+      return;
+    case Mode::kDownswing:
+      if (current_buffer_size == 0) {
+        buffer_limit_ *= 2;  // Increase the buffer size.
+        mode_ = Mode::kUpswing;
+      }
+      return;
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/prefetch_autotuner.h b/tensorflow/core/kernels/data/prefetch_autotuner.h
new file mode 100644
index 0000000000..fa8a184072
--- /dev/null
+++ b/tensorflow/core/kernels/data/prefetch_autotuner.h
@@ -0,0 +1,71 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
+#define TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
+
+#include "tensorflow/core/platform/types.h"
+
+namespace tensorflow {
+
+// PrefetchAutotuner dynamically adjusts the buffer size of a prefetch iterator.
+//
+// PrefetchAutotuner attempts to find the minimum buffer size such that there is
+// always at least 1 element in the prefetch queue every time the downstream
+// iterator calls GetNext().
+//
+// One common failure mode of input pipelines is being throughput bound. No
+// amount of prefetching can address that performance mode. In order to guard
+// against this condition, PrefetchAutotuner will only increase the buffer_limit
+// if the prefetching thread is able to successfully fill the buffer at its
+// current size.
+//
+// Note: in the current implementation, we never decrease the buffer_limit().
+// This should change in the future!
+//
+// PrefetchAutotuner is NOT thread safe.
+class PrefetchAutotuner {
+ public:
+  static const int64 kAutoTune = -1;
+
+  explicit PrefetchAutotuner(int64 initial_buffer_size);
+
+  int64 buffer_limit() const { return buffer_limit_; }
+
+  void RecordConsumption(size_t current_buffer_size);
+  void RecordEmpty() { RecordConsumption(0); }
+
+ private:
+  // PrefetchAutotuner operates as a state machine.
+  enum class Mode {
+    // Disables the autotuning.
+    kDisabled,
+
+    // We have increased the size of the buffer, and will transition to
+    // kDownswing if we successfully fill the buffer.
+    kUpswing,
+
+    // We have successfully filled a buffer of this size. If we ever block the
+    // downstream iterator, we should increase the buffer size.
+    kDownswing,
+  };
+
+  int64 buffer_limit_;
+  Mode mode_ = Mode::kDisabled;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_DATA_PREFETCH_AUTOTUNER_H_
diff --git a/tensorflow/core/kernels/data/prefetch_autotuner_test.cc b/tensorflow/core/kernels/data/prefetch_autotuner_test.cc
new file mode 100644
index 0000000000..2f573dfb35
--- /dev/null
+++ b/tensorflow/core/kernels/data/prefetch_autotuner_test.cc
@@ -0,0 +1,82 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/data/prefetch_autotuner.h"
+
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+TEST(PrefetchAutotuner, Disabled) {
+  PrefetchAutotuner t(2);
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(0);
+  t.RecordConsumption(2);
+  t.RecordConsumption(0);
+  t.RecordConsumption(2);
+  EXPECT_EQ(2, t.buffer_limit());
+}
+
+TEST(PrefetchAutotuner, Enabled) {
+  PrefetchAutotuner t(PrefetchAutotuner::kAutoTune);
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(1);
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(2);
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(1);
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(4, t.buffer_limit());
+  t.RecordConsumption(4);
+  EXPECT_EQ(4, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(8, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to stay the same!
+  EXPECT_EQ(8, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to stay the same!
+  EXPECT_EQ(8, t.buffer_limit());
+}
+
+TEST(PrefetchAutotuner, EnabledSteady) {
+  PrefetchAutotuner t(PrefetchAutotuner::kAutoTune);
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(1);
+  EXPECT_EQ(1, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(2);
+  EXPECT_EQ(2, t.buffer_limit());
+  t.RecordConsumption(0);  // Expect buffer limit to increase.
+  EXPECT_EQ(4, t.buffer_limit());
+
+  // Never reach zero again.
+  std::vector<size_t> consumption_values = {2, 3, 1, 4, 1, 2, 3, 1};
+  for (int i = 0; i < consumption_values.size(); ++i) {
+    t.RecordConsumption(consumption_values[i]);
+    EXPECT_EQ(4, t.buffer_limit())
+        << "Failed at index " << i << " with value: " << consumption_values[i];
+  }
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/data/prefetch_dataset_op.cc b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
index 1c548a30d2..536de81fd8 100644
--- a/tensorflow/core/kernels/data/prefetch_dataset_op.cc
+++ b/tensorflow/core/kernels/data/prefetch_dataset_op.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include "tensorflow/core/framework/partial_tensor_shape.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/kernels/data/dataset.h"
+#include "tensorflow/core/kernels/data/prefetch_autotuner.h"
 #include "tensorflow/core/lib/core/error_codes.pb.h"
 
 namespace tensorflow {
@@ -37,7 +38,8 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
     int64 buffer_size;
     OP_REQUIRES_OK(
         ctx, ParseScalarArgument<int64>(ctx, "buffer_size", &buffer_size));
-    OP_REQUIRES(ctx, buffer_size > 0,
+    OP_REQUIRES(ctx,
+                buffer_size > 0 || buffer_size == PrefetchAutotuner::kAutoTune,
                 errors::InvalidArgument("buffer_size must be > 0"));
 
     *output = new Dataset(ctx, input, buffer_size);
@@ -85,7 +87,8 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
      public:
       explicit Iterator(const Params& params)
           : DatasetIterator<Dataset>(params),
-            input_impl_(params.dataset->input_->MakeIterator(params.prefix)) {}
+            input_impl_(params.dataset->input_->MakeIterator(params.prefix)),
+            auto_tuner_(params.dataset->buffer_size_) {}
 
       ~Iterator() override {
         // Signal the prefetch thread to terminate it. We will then
@@ -113,6 +116,7 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
           // Wait until the next element in the buffer has been
           // produced, or we are shutting down.
           while (!cancelled_ && !prefetch_thread_finished_ && buffer_.empty()) {
+            auto_tuner_.RecordEmpty();
             cond_var_.wait(l);
           }
 
@@ -129,6 +133,7 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
             if (s.ok()) {
               *out_tensors = std::move(buffer_.front().value);
             }
+            auto_tuner_.RecordConsumption(buffer_.size());
             buffer_.pop_front();
             *end_of_sequence = false;
 
@@ -242,7 +247,8 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
           // 1. Wait for a slot in the buffer.
           {
             mutex_lock l(mu_);
-            while (!cancelled_ && buffer_.size() == dataset()->buffer_size_) {
+            while (!cancelled_ &&
+                   buffer_.size() == auto_tuner_.buffer_limit()) {
               cond_var_.wait(l);
             }
 
@@ -323,6 +329,7 @@ class PrefetchDatasetOp : public UnaryDatasetOpKernel {
       mutex parent_mu_ ACQUIRED_BEFORE(mu_);
       const std::unique_ptr<IteratorBase> input_impl_ GUARDED_BY(parent_mu_);
       condition_variable cond_var_;
+      PrefetchAutotuner auto_tuner_ GUARDED_BY(mu_);
       std::deque<BufferElement> buffer_ GUARDED_BY(mu_);
       std::unique_ptr<Thread> prefetch_thread_ GUARDED_BY(mu_);
       bool cancelled_ GUARDED_BY(mu_) = false;
diff --git a/tensorflow/python/data/ops/dataset_ops.py b/tensorflow/python/data/ops/dataset_ops.py
index c0a6283be4..8729e085a3 100644
--- a/tensorflow/python/data/ops/dataset_ops.py
+++ b/tensorflow/python/data/ops/dataset_ops.py
@@ -2043,6 +2043,8 @@ class PrefetchDataset(Dataset):
     """See `Dataset.prefetch()` for details."""
     super(PrefetchDataset, self).__init__()
     self._input_dataset = input_dataset
+    if buffer_size is None:
+      buffer_size = -1  # This is the sentinel for auto-tuning.
     self._buffer_size = ops.convert_to_tensor(
         buffer_size, dtype=dtypes.int64, name="buffer_size")
 
-- 
GitLab


From 3c0229c36ad7ade3cf795e3171c3c563e0222ed2 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Thu, 29 Mar 2018 05:28:16 +0800
Subject: [PATCH 634/960] Fix broken wiki link of Positive-definite_matrix in
 linalg api guide (#18057)

* Fix broken wiki link of Positive-definite_matrix in linalg api guide

* Fix minor intent
---
 .../contrib/linalg/python/ops/linear_operator_block_diag.py | 3 +--
 tensorflow/python/ops/linalg/linear_operator.py             | 3 +--
 tensorflow/python/ops/linalg/linear_operator_composition.py | 3 +--
 tensorflow/python/ops/linalg/linear_operator_diag.py        | 3 +--
 tensorflow/python/ops/linalg/linear_operator_full_matrix.py | 3 +--
 tensorflow/python/ops/linalg/linear_operator_identity.py    | 6 ++----
 .../python/ops/linalg/linear_operator_lower_triangular.py   | 3 +--
 7 files changed, 8 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
index 80649bd52d..9d3af66c92 100644
--- a/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
+++ b/tensorflow/contrib/linalg/python/ops/linear_operator_block_diag.py
@@ -138,8 +138,7 @@ class LinearOperatorBlockDiag(linear_operator.LinearOperator):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
         This is true by default, and will raise a `ValueError` otherwise.
       name: A name for this `LinearOperator`.  Default is the individual
diff --git a/tensorflow/python/ops/linalg/linear_operator.py b/tensorflow/python/ops/linalg/linear_operator.py
index c7513d5b40..193c787baa 100644
--- a/tensorflow/python/ops/linalg/linear_operator.py
+++ b/tensorflow/python/ops/linalg/linear_operator.py
@@ -166,8 +166,7 @@ class LinearOperator(object):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       name: A name for this `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_composition.py b/tensorflow/python/ops/linalg/linear_operator_composition.py
index ecd30e4d7e..0292bc51dc 100644
--- a/tensorflow/python/ops/linalg/linear_operator_composition.py
+++ b/tensorflow/python/ops/linalg/linear_operator_composition.py
@@ -134,8 +134,7 @@ class LinearOperatorComposition(linear_operator.LinearOperator):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       name: A name for this `LinearOperator`.  Default is the individual
         operators names joined with `_o_`.
diff --git a/tensorflow/python/ops/linalg/linear_operator_diag.py b/tensorflow/python/ops/linalg/linear_operator_diag.py
index e180e83026..5beaea65a5 100644
--- a/tensorflow/python/ops/linalg/linear_operator_diag.py
+++ b/tensorflow/python/ops/linalg/linear_operator_diag.py
@@ -132,8 +132,7 @@ class LinearOperatorDiag(linear_operator.LinearOperator):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       name: A name for this `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py
index f979fb37d6..5ba3b090ae 100644
--- a/tensorflow/python/ops/linalg/linear_operator_full_matrix.py
+++ b/tensorflow/python/ops/linalg/linear_operator_full_matrix.py
@@ -125,8 +125,7 @@ class LinearOperatorFullMatrix(linear_operator.LinearOperator):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       name: A name for this `LinearOperator`.
 
diff --git a/tensorflow/python/ops/linalg/linear_operator_identity.py b/tensorflow/python/ops/linalg/linear_operator_identity.py
index 50f3d407e8..45929eb4e2 100644
--- a/tensorflow/python/ops/linalg/linear_operator_identity.py
+++ b/tensorflow/python/ops/linalg/linear_operator_identity.py
@@ -236,8 +236,7 @@ class LinearOperatorIdentity(BaseLinearOperatorIdentity):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       assert_proper_shapes:  Python `bool`.  If `False`, only perform static
         checks that initialization and method arguments have proper shape.
@@ -576,8 +575,7 @@ class LinearOperatorScaledIdentity(BaseLinearOperatorIdentity):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       assert_proper_shapes:  Python `bool`.  If `False`, only perform static
         checks that initialization and method arguments have proper shape.
diff --git a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
index a5130188b6..c4d386ccb4 100644
--- a/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
+++ b/tensorflow/python/ops/linalg/linear_operator_lower_triangular.py
@@ -133,8 +133,7 @@ class LinearOperatorLowerTriangular(linear_operator.LinearOperator):
         meaning the quadratic form `x^H A x` has positive real part for all
         nonzero `x`.  Note that we do not require the operator to be
         self-adjoint to be positive-definite.  See:
-        https://en.wikipedia.org/wiki/Positive-definite_matrix\
-            #Extension_for_non_symmetric_matrices
+        https://en.wikipedia.org/wiki/Positive-definite_matrix#Extension_for_non-symmetric_matrices
       is_square:  Expect that this operator acts like square [batch] matrices.
       name: A name for this `LinearOperator`.
 
-- 
GitLab


From ef6552b544b3c3bf6808be807b30dd9bd4f19669 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 28 Mar 2018 14:30:39 -0700
Subject: [PATCH 635/960] [tf.data] Fix reference leak in
 FunctionBufferingResource.

Previously, the FunctionBufferingResource's destructor would never be called, which led to use-after-free (of the underlying Device object) errors in the prefetching function.

PiperOrigin-RevId: 190834415
---
 tensorflow/contrib/cmake/tf_tests.cmake                | 1 +
 tensorflow/contrib/data/kernels/prefetching_kernels.cc | 1 +
 2 files changed, 2 insertions(+)

diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 237f4fe33a..f793877c8b 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -281,6 +281,7 @@ if (tensorflow_BUILD_PYTHON_TESTS)
       "${tensorflow_source_dir}/tensorflow/python/data/kernel_tests/iterator_ops_cluster_test.py"
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/interleave_dataset_op_test.py"  # Deadlocks
       "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/sloppy_transformation_dataset_op_test.py"  # b/65430561
+      "${tensorflow_source_dir}/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py"  # Segfaults on Windows.
       # tensor_forest tests (also note that we exclude the hybrid tests for now)
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/count_extremely_random_stats_op_test.py"  # Results in wrong order.
       "${tensorflow_source_dir}/tensorflow/contrib/tensor_forest/python/kernel_tests/sample_inputs_op_test.py"  # Results in wrong order.
diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 79d1fc3494..f51570db85 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -314,6 +314,7 @@ class FunctionBufferResourceHandleOp : public OpKernel {
                     source_device, target_device, func_args, thread_pool_size_);
                 return Status::OK();
               }));
+      core::ScopedUnref s(buffer);
       OP_REQUIRES_OK(ctx, buffer->Instantiate());
       initialized_ = true;
     }
-- 
GitLab


From e97c9e91e016efd951dc52e82744f607d948bb2a Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Wed, 28 Mar 2018 14:36:18 -0700
Subject: [PATCH 636/960] Merge changes from github.

PiperOrigin-RevId: 190835392
---
 RELEASE.md                                    |  60 +++
 configure.py                                  |   2 +-
 tensorflow/BUILD                              |   7 +
 tensorflow/contrib/BUILD                      |  27 +-
 .../boosted_trees/kernels/quantile_ops.cc     |   2 +-
 .../boosted_trees/lib/utils/batch_features.cc |   2 +-
 .../lib/utils/batch_features_test.cc          |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.cc  |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.h   |   2 +-
 .../lib/utils/sparse_column_iterable_test.cc  |   2 +-
 .../boosted_trees/proto/tree_config.proto     |   2 +-
 .../kernel_tests/prediction_ops_test.py       |  10 +-
 .../python/kernel_tests/quantile_ops_test.py  |   2 +-
 .../boosted_trees/python/ops/quantile_ops.py  |   2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |   3 +
 .../kernel_tests/batch_dataset_op_test.py     |  14 +
 tensorflow/contrib/eager/python/BUILD         |   6 +-
 .../eager/python/examples/spinn/spinn_test.py |   1 -
 .../python/estimator/replicate_model_fn.py    |   2 +-
 .../factorization/kernels/clustering_ops.cc   |   2 +-
 .../python/ops/factorization_ops.py           |  14 +-
 .../python/ops/factorization_ops_test.py      |  12 +-
 .../factorization/python/ops/gmm_ops.py       |   4 +-
 .../factorization/python/ops/gmm_test.py      |   2 +-
 .../factorization/python/ops/kmeans_test.py   |   4 +-
 .../contrib/factorization/python/ops/wals.py  |   2 +-
 tensorflow/contrib/learn/BUILD                |   1 +
 .../learn/python/learn/estimators/linear.py   |   4 +-
 .../linear_optimizer/python/sdca_estimator.py |   4 +-
 tensorflow/contrib/lite/README.md             |   3 +
 tensorflow/contrib/lite/builtin_ops.h         |   1 +
 tensorflow/contrib/lite/g3doc/models.md       |   2 +-
 tensorflow/contrib/lite/kernels/BUILD         |  13 +
 .../internal/reference/reference_ops.h        |  25 ++
 tensorflow/contrib/lite/kernels/maximum.cc    | 106 +++++
 .../contrib/lite/kernels/maximum_test.cc      |  81 ++++
 tensorflow/contrib/lite/kernels/register.cc   |   2 +
 tensorflow/contrib/lite/model.cc              |   3 +
 tensorflow/contrib/lite/nnapi_delegate.cc     |   1 +
 tensorflow/contrib/lite/python/lite.py        |  22 +-
 tensorflow/contrib/lite/schema/schema.fbs     |   5 +
 .../contrib/lite/schema/schema_generated.h    | 124 +++++-
 tensorflow/contrib/lite/testing/BUILD         |   1 +
 .../contrib/lite/testing/generate_examples.py |  36 ++
 .../testing/generated_examples_zip_test.cc    |   1 +
 .../contrib/lite/toco/tflite/operator.cc      |   2 +
 .../contrib/lite/toco/tflite/operator_test.cc |   2 +
 tensorflow/contrib/lookup/lookup_ops.py       |   2 +-
 .../contrib/makefile/download_dependencies.sh |   2 +-
 tensorflow/contrib/makefile/tf_op_files.txt   |   1 +
 .../seq2seq/kernels/beam_search_ops.cc        |   2 +-
 .../seq2seq/python/ops/attention_wrapper.py   |   8 +-
 .../seq2seq/python/ops/beam_search_decoder.py |   6 +-
 .../slim/python/slim/data/parallel_reader.py  |   4 +-
 .../slim/python/slim/data/prefetch_queue.py   |   4 +-
 .../python/slim/data/tfexample_decoder.py     |   2 +-
 tensorflow/contrib/tensorrt/README.md         |  46 ++-
 .../contrib/tensorrt/convert/convert_graph.cc |  20 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 375 ++++++++++--------
 .../contrib/tensorrt/segment/segment.cc       |  55 ++-
 tensorflow/contrib/tensorrt/segment/segment.h |   4 +-
 .../contrib/tensorrt/segment/segment_test.cc  |   8 +-
 .../timeseries/python/timeseries/ar_model.py  |   2 +-
 .../python/timeseries/math_utils.py           |   2 +-
 .../timeseries/state_space_models/varma.py    |   4 +-
 .../base_api/api_def_MatrixSolveLs.pbtxt      |   6 +-
 .../core/common_runtime/mkl_cpu_allocator.cc  |   3 -
 tensorflow/core/framework/common_shape_fns.cc |   4 +-
 tensorflow/core/framework/common_shape_fns.h  |   8 +-
 tensorflow/core/framework/shape_inference.h   |   1 +
 .../core/kernels/mkl_fused_batch_norm_op.cc   |   2 +-
 .../core/kernels/segment_reduction_ops.h      |   7 +
 tensorflow/core/kernels/snapshot_op.cc        |  30 ++
 tensorflow/core/kernels/snapshot_op.h         |  26 +-
 tensorflow/core/kernels/snapshot_op_gpu.cu.cc |   9 +-
 tensorflow/core/kernels/xent_op.cc            |  65 ++-
 tensorflow/core/kernels/xent_op.h             |  35 +-
 tensorflow/core/kernels/xent_op_gpu.cu.cc     |   9 +-
 tensorflow/core/ops/array_ops.cc              |  26 +-
 tensorflow/core/ops/nn_ops.cc                 |  23 +-
 tensorflow/core/ops/nn_ops_test.cc            |  16 +-
 tensorflow/core/public/version.h              |   4 +-
 .../python/contrib.bayesflow.monte_carlo.md   |  36 +-
 .../api_guides/python/contrib.losses.md       |  28 +-
 .../docs_src/community/documentation.md       |  38 +-
 tensorflow/docs_src/install/install_c.md      |   2 +-
 tensorflow/docs_src/install/install_go.md     |   2 +-
 tensorflow/docs_src/install/install_java.md   |  22 +-
 tensorflow/docs_src/install/install_linux.md  |  22 +-
 tensorflow/docs_src/install/install_mac.md    |  14 +-
 .../docs_src/install/install_sources.md       |   9 +-
 tensorflow/docs_src/mobile/optimizing.md      |   2 +
 tensorflow/docs_src/mobile/prepare_models.md  |   2 +-
 tensorflow/python/BUILD                       |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  26 +-
 tensorflow/python/kernel_tests/testdata/BUILD |   2 +-
 .../python/kernel_tests/xent_op_test.py       |  81 +++-
 tensorflow/python/layers/convolutional.py     |   2 +
 .../python/layers/convolutional_test.py       |   6 +
 tensorflow/python/ops/linalg_ops.py           |   2 +-
 .../python/training/monitored_session.py      |  33 +-
 .../python/training/monitored_session_test.py |  36 ++
 tensorflow/tensorflow.bzl                     |   4 +-
 .../tools/api/golden/tensorflow.train.pbtxt   |   2 +-
 .../tools/ci_build/osx/libtensorflow_cpu.sh   |   2 +-
 tensorflow/tools/docker/Dockerfile.devel      |   2 +-
 .../tools/docker/Dockerfile.devel-cpu-mkl     |   2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |   2 +-
 tensorflow/tools/lib_package/BUILD            |   2 -
 tensorflow/tools/pip_package/BUILD            |   1 -
 tensorflow/tools/pip_package/setup.py         |   6 +-
 tensorflow/workspace.bzl                      | 133 ++++---
 third_party/mkl/BUILD                         |  46 ++-
 third_party/mkl/MKL_LICENSE                   | 201 ++++++++++
 third_party/mkl/build_defs.bzl                |  12 +
 third_party/mkl/mkl.BUILD                     |  27 +-
 116 files changed, 1703 insertions(+), 556 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/maximum.cc
 create mode 100644 tensorflow/contrib/lite/kernels/maximum_test.cc
 create mode 100644 third_party/mkl/MKL_LICENSE

diff --git a/RELEASE.md b/RELEASE.md
index 6f54dee58f..c63d9f20c9 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,3 +1,63 @@
+# Release 1.7.0
+
+## Major Features And Improvements
+* Eager mode is moving out of contrib, try `tf.enable_eager_execution()`.
+* Graph rewrites emulating fixed-point quantization compatible with TensorFlow Lite, supported by new `tf.contrib.quantize` package.
+* Easily customize gradient computation with `tf.custom_gradient`.
+* [TensorBoard Debugger Plugin](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md), the graphical user interface (GUI) of TensorFlow Debugger (tfdbg), is now in alpha.
+* Experimental support for reading a sqlite database as a `Dataset` with new `tf.contrib.data.SqlDataset`.
+* Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`.
+* Better text processing with `tf.regex_replace`.
+* Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length`
+
+## Bug Fixes and Other Changes
+* Accelerated Linear Algebra (XLA):
+  * Add `MaxPoolGradGrad` support for XLA
+  * CSE pass from Tensorflow is now disabled in XLA.
+* `tf.data`:
+  * `tf.data.Dataset`
+    * Add support for building C++ Dataset op kernels as external libraries, using the `tf.load_op_library()` mechanism.
+    * `Dataset.list_files()` now shuffles its output by default.
+    * `Dataset.shuffle(..., seed=tf.constant(0, dtype=tf.int64))` now yields the same sequence of elements as `Dataset.shuffle(..., seed=0)`.
+  * Add `num_parallel_reads` argument to `tf.data.TFRecordDataset`.
+* `tf.contrib`:
+  * `tf.contrib.bayesflow.halton_sequence` now supports randomization.
+  * Add support for scalars in `tf.contrib.all_reduce`.
+  * Add `effective_sample_size` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `potential_scale_reduction` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `BatchNormalization`, `Kumaraswamy` bijectors.
+  * Deprecate `tf.contrib.learn`. Please check contrib/learn/README.md for instructions on how to convert existing code.
+  * `tf.contrib.data`
+    * Remove deprecated `tf.contrib.data.Dataset`, `tf.contrib.data.Iterator`, `tf.contrib.data.FixedLengthRecordDataset`, `tf.contrib.data.TextLineDataset`, and `tf.contrib.data.TFRecordDataset` classes.
+    * Added `bucket_by_sequence_length`, `sliding_window_batch`, and `make_batched_features_dataset`
+  * Remove unmaintained `tf.contrib.ndlstm`. You can find it externally at https://github.com/tmbarchive/tfndlstm.
+  * Moved most of `tf.contrib.bayesflow` to its own repo: `tfp`
+* Other:
+  * tf.py_func now reports the full stack trace if an exception occurs.
+  * Integrate `TPUClusterResolver` with GKE's integration for Cloud TPUs.
+  * Add a library for statistical testing of samplers.
+  * Add Helpers to stream data from the GCE VM to a Cloud TPU.
+  * Integrate ClusterResolvers with TPUEstimator.
+  * Unify metropolis_hastings interface with HMC kernel.
+  * Move LIBXSMM convolutions to a separate --define flag so that they are disabled by default.
+  * Fix `MomentumOptimizer` lambda.
+  * Reduce `tfp.layers` boilerplate via programmable docstrings.
+  * Add `auc_with_confidence_intervals`, a method for computing the AUC and confidence interval with linearithmic time complexity.
+  * `regression_head` now accepts customized link function, to satisfy the usage that user can define their own link function if the `array_ops.identity` does not meet the requirement.
+  * Fix `initialized_value` and `initial_value` behaviors for `ResourceVariables` created from `VariableDef` protos.
+  * Add TensorSpec to represent the specification of Tensors.
+  * Constant folding pass is now deterministic.
+  * Support `float16` `dtype` in `tf.linalg.*`.
+  * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abe, Alistair Low, Andy Kernahan, Appledore, Ben, Ben Barsdell, Boris Pfahringer, Brad Wannow, Brett Koonce, Carl Thomé, cclauss, Chengzhi Chen, Chris Drake, Christopher Yeh, Clayne Robison, Codrut Grosu, Daniel Trebbien, Danny Goodman, David Goodwin, David Norman, Deron Eriksson, Donggeon Lim, Donny Viszneki, DosLin, DylanDmitri, Francisco Guerrero, Fred Reiss, gdh1995, Giuseppe, Glenn Weidner, gracehoney, Guozhong Zhuang, Haichen "Hc" Li, Harald Husum, harumitsu.nobuta, Henry Spivey, hsm207, Jekyll Song, Jerome, Jiongyan Zhang, jjsjann123, John Sungjin Park, Johnson145, JoshVarty, Julian Wolff, Jun Wang, June-One, Kamil Sindi, Kb Sriram, Kdavis-Mozilla, Kenji, lazypanda1, Liang-Chi Hsieh, Loo Rong Jie, Mahesh Bhosale, MandarJKulkarni, ManHyuk, Marcus Ong, Marshal Hayes, Martin Pool, matthieudelaro, mdfaijul, mholzel, Michael Zhou, Ming Li, Minmin Sun, Myungjoo Ham, MyungsungKwak, Naman Kamra, Peng Yu, Penghao Cen, Phil, Raghuraman-K, resec, Rohin Mohanadas, Sandeep N Gupta, Scott Tseng, seaotterman, Seo Sanghyeon, Sergei Lebedev, Ted Chang, terrytangyuan, Tim H, tkunic, Tod, vihanjain, Yan Facai (颜发才), Yin Li, Yong Tang, Yukun Chen, Yusuke Yamada
+
+
+
 # Release 1.6.0
 
 ## Breaking Changes
diff --git a/configure.py b/configure.py
index 22b9abedd7..0f52c0ec99 100644
--- a/configure.py
+++ b/configure.py
@@ -1414,7 +1414,7 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform',
-                'with_kafka_support', False, 'kafka')
+                'with_kafka_support', True, 'kafka')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 6ab43638ba..29a01efc84 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -240,6 +240,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "with_kafka_support_windows_override",
+    define_values = {"with_kafka_support": "true"},
+    values = {"cpu": "x64_windows"},
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "with_gcp_support_android_override",
     define_values = {"with_gcp_support": "true"},
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index bdbd738906..fb81b50fe8 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -51,7 +51,6 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
-        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -110,7 +109,13 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -120,7 +125,6 @@ cc_library(
         "//tensorflow/contrib/boosted_trees:boosted_trees_kernels",
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -133,7 +137,13 @@ cc_library(
         "//tensorflow/contrib/text:all_kernels",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_cuda([
         "//tensorflow/contrib/nccl:nccl_kernels",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_kernels",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -146,7 +156,6 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
@@ -157,7 +166,13 @@ cc_library(
         "//tensorflow/contrib/tensor_forest:tensor_forest_ops_op_lib",
         "//tensorflow/contrib/text:all_ops",
         "//tensorflow/contrib/tpu:all_ops",
-    ],
+    ] + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_ops_op_lib",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 filegroup(
diff --git a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
index 0f4c2298f5..0b28f81e7c 100644
--- a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
@@ -253,7 +253,7 @@ class CreateQuantileAccumulatorOp : public OpKernel {
  private:
   float epsilon_;
   int32 num_quantiles_;
-  // An upperbound on the number of enteries that the summaries might have
+  // An upper bound on the number of entries that the summaries might have
   // for a feature.
   int64 max_elements_;
   bool generate_quantiles_;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
index cf4f9a097a..35b059f349 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
@@ -54,7 +54,7 @@ Status BatchFeatures::Initialize(
     TF_CHECK_AND_RETURN_IF_ERROR(
         dense_float_feature.dim_size(1) == 1,
         errors::InvalidArgument(
-            "Dense float features may not be multi-valent: dim_size(1) = ",
+            "Dense float features may not be multivalent: dim_size(1) = ",
             dense_float_feature.dim_size(1)));
     dense_float_feature_columns_.emplace_back(dense_float_feature);
   }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
index 609519e8b1..cfe9101e74 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
@@ -59,7 +59,7 @@ TEST_F(BatchFeaturesTest, DenseFloatFeatures_Multivalent) {
   BatchFeatures batch_features(1);
   auto dense_vec = AsTensor<float>({3.0f, 7.0f}, {1, 2});
   auto expected_error = InvalidArgument(
-      "Dense float features may not be multi-valent: dim_size(1) = 2");
+      "Dense float features may not be multivalent: dim_size(1) = 2");
   EXPECT_EQ(expected_error,
             batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {}));
 }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
index db34db998a..ce67db797d 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
@@ -54,7 +54,7 @@ Status DropoutUtils::DropOutTrees(
   if (probability_of_skipping_dropout < 0 ||
       probability_of_skipping_dropout > 1) {
     return errors::InvalidArgument(
-        "Probability of skiping dropout must be in [0,1] range");
+        "Probability of skipping dropout must be in [0,1] range");
   }
   const auto num_trees = weights.size();
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
index 928bfbfe5c..77c16da541 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
@@ -66,7 +66,7 @@ class DropoutUtils {
       // Current weights and num_updates will be updated as a result of this
       // func
       std::vector<float>* current_weights,
-      // How many weight assignements have been done for each tree already.
+      // How many weight assignments have been done for each tree already.
       std::vector<int32>* num_updates);
 };
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
index 0138aae3db..cc7604745e 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
@@ -34,7 +34,7 @@ TEST_F(SparseColumnIterableTest, Empty) {
 }
 
 TEST_F(SparseColumnIterableTest, Iterate) {
-  // 8 examples having 7 sparse features with the 3rd and 7th multi-valent.
+  // 8 examples having 7 sparse features with the 3rd and 7th multivalent.
   // This can be visualized like the following:
   // Instance | Sparse |
   // 0        |  x     |
diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
index 4407c4d981..81411aa84a 100644
--- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto
+++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
@@ -53,7 +53,7 @@ message DenseFloatBinarySplit {
   // Float feature column and split threshold describing
   // the rule feature <= threshold.
   int32 feature_column = 1;
-  // If feature column is multivalent, this holds the index of the dimensiong
+  // If feature column is multivalent, this holds the index of the dimension
   // for the split. Defaults to 0.
   int32 dimension_id = 5;
   float threshold = 2;
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
index c1acf35160..cf55759aaa 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
@@ -120,8 +120,8 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     """Sets up the prediction tests.
 
     Create a batch of two examples having one dense float, two sparse float
-    single valued, one sparse float multidimensionl and one sparse int features.
-    The data looks like the following:
+    single valued, one sparse float multidimensional and one sparse int
+    features.  The data looks like the following:
     | Instance | Dense0 | SparseF0 | SparseF1 | SparseI0 | SparseM
     | 0        |  7     |    -3    |          |    9,1   | __, 5.0
     | 1        | -2     |          | 4        |          |  3, ___
@@ -810,7 +810,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     # building. This tree should never be dropped.
     num_trees = 10
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 10 trees with some weights.
       for i in range(0, num_trees):
@@ -951,7 +951,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testDropOutZeroProb(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 1000 trees with some weights.
       for i in range(0, 999):
@@ -994,7 +994,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testAveragingAllTrees(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       adjusted_tree_ensemble_config = (
           tree_config_pb2.DecisionTreeEnsembleConfig())
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
index 81f58de28c..074623699d 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
@@ -482,7 +482,7 @@ class QuantilesOpTest(test_util.TensorFlowTestCase):
     """Sets up the quantile op tests.
 
     Create a batch of 4 examples having 2 dense and 4 sparse features.
-    Forth sparse feature is multivalent (3 dimensional)
+    Fourth sparse feature is multivalent (3 dimensional)
     The data looks like this
     | Instance | Dense 0 | Dense 1 | Sparse 0 | Sparse 1 |Sparse 2| SparseM
     | 0        |   -0.1  |  -1     |   -2     |   0.1    |        |_ ,1,_
diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
index 97d57e8b23..1b184d296b 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
@@ -184,7 +184,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
     """Finalizes quantile summary stream and resets it for next iteration.
 
     Args:
-      stamp_token: Exepcted current token.
+      stamp_token: Expected current token.
       next_stamp_token: Next value for the token.
     Returns:
       A list of quantiles or approximate boundaries.
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index f793877c8b..92f2ab6dea 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -210,6 +210,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py"
     # Test is flaky on Windows GPU builds (b/38283730).
     "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/gmm_test.py"
+    # Disable following manual tag in BUILD.
+    "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/layers/convolutional_test.py"
+
   )
   if (WIN32)
     set(tf_test_src_py_exclude
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 5abb38c2d2..75482f67da 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -413,6 +413,20 @@ class BatchDatasetTest(test.TestCase):
   def testMapAndBatchPartialBatchDropRemainder(self):
     return self._testMapAndBatchPartialBatchHelper(drop_remainder=True)
 
+  def testMapAndBatchYieldsPartialBatch(self):
+    iterator = (dataset_ops.Dataset.range(10)
+                .apply(batching.map_and_batch(
+                    lambda x: array_ops.reshape(x * x, [1]), 4))
+                .make_one_shot_iterator())
+    self.assertEqual([None, 1], iterator.output_shapes.as_list())
+    next_element = iterator.get_next()
+    with self.test_session() as sess:
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 4fba014d6f..80176397c0 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -270,7 +270,11 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = ["notsan"],
+    tags = [
+        "no_oss",  # b/74395663
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
+    ],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 9261823d77..9adf47d505 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -418,7 +418,6 @@ class SpinnTest(test_util.TensorFlowTestCase):
                     if event.summary.value
                     and event.summary.value[0].tag == "train/loss"]
     self.assertEqual(config.epochs, len(train_losses))
-    self.assertLess(train_losses[-1], train_losses[0])
 
     # 5. Verify that checkpoints exist and contains all the expected variables.
     self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*")))
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index e0fae2c992..fa2697800e 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replice only on the subset of available GPUs.
+      argument can be used to replicate only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
index dd61f59585..2a6c97e8b9 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
@@ -353,7 +353,7 @@ class NearestNeighborsOp : public OpKernel {
     auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
     const int64 num_threads = worker_threads.num_threads;
     // This kernel might be configured to use fewer than the total number of
-    // available CPUs on the host machine. To avoid descructive interference
+    // available CPUs on the host machine. To avoid destructive interference
     // with other jobs running on the host machine, we must only use a fraction
     // of total available L3 cache. Unfortunately, we cannot query the host
     // machine to get the number of physical CPUs. So, we use a fixed per-CPU
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
index 054888e734..8e0ed1d80e 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
@@ -106,7 +106,7 @@ class WALSModel(object):
       # the prep_gramian_op for row(column) can be run.
       worker_init_op = model.worker_init
 
-      # To be run once per interation sweep before the row(column) update
+      # To be run once per integration sweep before the row(column) update
       # initialize ops can be run. Note that in the distributed training
       # situations, this should only be run by the chief trainer. All other
       # trainers need to block until this is done.
@@ -118,9 +118,9 @@ class WALSModel(object):
       init_row_update_op = model.initialize_row_update_op
       init_col_update_op = model.initialize_col_update_op
 
-      # Ops to upate row(column). This can either take the entire sparse tensor
-      # or slices of sparse tensor. For distributed trainer, each trainer
-      # handles just part of the matrix.
+      # Ops to update row(column). This can either take the entire sparse
+      # tensor or slices of sparse tensor. For distributed trainer, each
+      # trainer handles just part of the matrix.
       _, row_update_op, unreg_row_loss, row_reg, _ = model.update_row_factors(
            sp_input=matrix_slices_from_queue_for_worker_shard)
       row_loss = unreg_row_loss + row_reg
@@ -220,7 +220,7 @@ class WALSModel(object):
         in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of
         inner lists matching the number of row factor shards and the elements in
         each inner list are the weights for the rows of the corresponding row
-        factor shard. In this case,  w_ij = unonbserved_weight +
+        factor shard. In this case,  w_ij = unobserved_weight +
                                             row_weights[i] * col_weights[j].
         - If this is a single non-negative real number, this value is used for
         all row weights and w_ij = unobserved_weight + row_weights *
@@ -435,7 +435,7 @@ class WALSModel(object):
       gramian: Variable storing the gramian calculated from the factors.
 
     Returns:
-      A op that updates the gramian with the calcuated value from the factors.
+      A op that updates the gramian with the calculated value from the factors.
     """
     partial_gramians = []
     for f in factors:
@@ -564,7 +564,7 @@ class WALSModel(object):
 
     Note that specifically this initializes the cache of the row and column
     weights on workers when `use_factors_weights_cache` is True. In this case,
-    if these weights are being calcualted and reset after the object is created,
+    if these weights are being calculated and reset after the object is created,
     it is important to ensure this ops is run afterwards so the cache reflects
     the correct values.
     """
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
index c813733915..bb5140aeb3 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
@@ -210,7 +210,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -283,8 +283,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 3 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
-      # weights and feature vectors are identical to that used in model
+      # This is expected to reproduce the same column factors in the model as
+      # the weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
@@ -385,7 +385,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -462,8 +462,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 2 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
-      # weights and feature vectors are identical to that used in model
+      # This is expected to reproduce the same column factors in the model as
+      # the weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index 98d6434f47..14d4c733e3 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -280,7 +280,7 @@ class GmmAlgorithm(object):
     self._define_score_samples()
 
   def _define_full_covariance_probs(self, shard_id, shard):
-    """Defines the full covariance probabilties per example in a class.
+    """Defines the full covariance probabilities per example in a class.
 
     Updates a matrix with dimension num_examples X num_classes.
 
@@ -344,7 +344,7 @@ class GmmAlgorithm(object):
   def _define_prior_log_prob_operation(self, shard_id):
     """Computes the prior probability of all samples.
 
-    Updates a vector where each item is the prior probabibility of an
+    Updates a vector where each item is the prior probability of an
     input example.
 
     Args:
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_test.py b/tensorflow/contrib/factorization/python/ops/gmm_test.py
index 00a4734eb6..4fc9c96e9d 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_test.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_test.py
@@ -210,7 +210,7 @@ class GMMTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 0103cc4439..88eb9cf692 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -413,7 +413,7 @@ class KMeansCosineDistanceTest(KMeansTestBase):
     self.assertAllClose(score, self.true_score, atol=1e-2)
 
   def test_predict_kmeans_plus_plus(self):
-    # Most points are concetrated near one center. KMeans++ is likely to find
+    # Most points are concentrated near one center. KMeans++ is likely to find
     # the less populated centers.
     points = np.array(
         [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2],
@@ -604,7 +604,7 @@ class KMeansTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 4fe22ea26e..62db3bb4c4 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -235,7 +235,7 @@ def _wals_factorization_model_function(features, labels, mode, params):
         num_items: An integer, the total number of items of this axis.
         update_fn: A function that takes one argument (`sp_input`), and that
         returns a tuple of
-          * new_factors: A flot Tensor of the factor values after update.
+          * new_factors: A float Tensor of the factor values after update.
           * update_op: a TensorFlow op which updates the factors.
           * loss: A float Tensor, the unregularized loss.
           * reg_loss: A float Tensor, the regularization loss.
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 9c59150580..16f80a876f 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -226,6 +226,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/monitors_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_pip_gpu"],  # b/74437598
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index 64d7ecc68e..70b70af98c 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -243,8 +243,8 @@ def sdca_model_fn(features, labels, mode, params):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(
-      features.values(), parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
index 05794a42c5..d4e54c82f9 100644
--- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
+++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
@@ -140,8 +140,8 @@ def sdca_model_fn(features, labels, mode, params, config=None):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(features.values(),
-                                        parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index 2680d515eb..c15ae3f233 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -126,6 +126,9 @@ The above pre-trained models have been trained on the ImageNet data set, which c
 
 The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
 
+# Getting started with RaspberryPi
+
+Using RaspberryPi can be accomplished by following the [Makefile instructions](g3doc/rpi.md). That will give a you a static library (.a) that you can build your app against. Python bindings will be coming soon as well as a demo app.
 
 ### Train a custom model
 A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index d7993e60cc..17b791e4e2 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -79,6 +79,7 @@ typedef enum {
   kTfLiteBuiltinBidirectionalSequenceLstm = 52,
   kTfLiteBuiltinCast = 53,
   kTfLiteBuiltinPrelu = 54,
+  kTfLiteBuiltinMaximum = 55,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 5b393140d6..48f43d4fc4 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,4 +1,4 @@
-#List of Hosted Models
+# List of Hosted Models
 
 *   [Inception V3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
 *   [Inception V3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 1450c1e14b..c423c00bf5 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -156,6 +156,7 @@ cc_library(
         "local_response_norm.cc",
         "lsh_projection.cc",
         "lstm.cc",
+        "maximum.cc",
         "mean.cc",
         "mfcc.cc",
         "mul.cc",
@@ -536,6 +537,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "maximum_test",
+    size = "small",
+    srcs = ["maximum_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "mean_test",
     size = "small",
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 33d60afa26..3575974ae9 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -404,6 +404,7 @@ inline void DepthToSpace(const T* input_data, const Dims<4>& input_dims,
           const int in_d =
               out_d + ((out_h % block_size) * block_size + out_w % block_size) *
                           output_depth;
+
           const int in_w = out_w / block_size;
           const int in_h = out_h / block_size;
           const int in_b = out_b;
@@ -3363,6 +3364,30 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+template <typename T>
+void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
+                       const T* input2_data, const Dims<4>& input2_dims,
+                       T* output_data, const Dims<4>& output_dims) {
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          auto out_idx = Offset(output_dims, c, x, y, b);
+          auto in1_idx = SubscriptToIndex(desc1, c, x, y, b);
+          auto in2_idx = SubscriptToIndex(desc2, c, x, y, b);
+          auto in1_val = input1_data[in1_idx];
+          auto in2_val = input2_data[in2_idx];
+          output_data[out_idx] = in1_val > in2_val ? in1_val : in2_val;
+        }
+      }
+    }
+  }
+}
+
 template <typename T1, typename T2, typename T3>
 void ArgMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims,
             T2* output_data, const Dims<4>& output_dims) {
diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum.cc
new file mode 100644
index 0000000000..9fdf2b47ea
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum.cc
@@ -0,0 +1,106 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <string.h>
+#include <vector>
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+#include "tensorflow/contrib/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace maximum {
+
+// This file has a reference implemenation of TFMaximum.
+enum KernelType {
+  kReference,
+};
+
+constexpr int kInputTensor1 = 0;
+constexpr int kInputTensor2 = 1;
+constexpr int kOutputTensor = 0;
+
+struct MaximumContext {
+  MaximumContext(TfLiteContext* context, TfLiteNode* node) {
+    input1 = GetInput(context, node, kInputTensor1);
+    input2 = GetInput(context, node, kInputTensor2);
+    output = GetOutput(context, node, kOutputTensor);
+  }
+  TfLiteTensor* input1;
+  TfLiteTensor* input2;
+  TfLiteTensor* output;
+};
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  MaximumContext op_context(context, node);
+  TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type);
+  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input2->dims);
+  op_context.output->type = op_context.input2->type;
+  return context->ResizeTensor(context, op_context.output, output_dims);
+}
+
+template <KernelType kernel_type>
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  MaximumContext op_context(context, node);
+
+#define TF_LITE_MAXIMUM(kernel_type, data_type)    \
+  kernel_type::TensorFlowMaximum<data_type>(       \
+      GetTensorData<data_type>(op_context.input1), \
+      GetTensorDims(op_context.input1),            \
+      GetTensorData<data_type>(op_context.input2), \
+      GetTensorDims(op_context.input2),            \
+      GetTensorData<data_type>(op_context.output), \
+      GetTensorDims(op_context.output))
+
+  if (kernel_type == kReference) {
+    switch (op_context.output->type) {
+      case kTfLiteFloat32:
+        TF_LITE_MAXIMUM(reference_ops, float);
+        break;
+      default:
+        context->ReportError(context,
+                             "Type %d is currently not supported by Maximum.",
+                             op_context.output->type);
+        return kTfLiteError;
+    }
+  } else {
+    context->ReportError(context,
+                         "Type %d is currently not supported by Maximum.",
+                         op_context.output->type);
+    return kTfLiteError;
+  }
+#undef TF_LITE_MAXIMUM
+  return kTfLiteOk;
+}
+
+}  // namespace maximum
+
+TfLiteRegistration* Register_MAXIMUM_REF() {
+  static TfLiteRegistration r = {nullptr, nullptr, maximum::Prepare,
+                                 maximum::Eval<maximum::kReference>};
+  return &r;
+}
+
+TfLiteRegistration* Register_MAXIMUM() { return Register_MAXIMUM_REF(); }
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc
new file mode 100644
index 0000000000..b3fd7d4e6f
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum_test.cc
@@ -0,0 +1,81 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class MaximumOpModel : public SingleOpModel {
+ public:
+  MaximumOpModel(const TensorData& input1, const TensorData& input2,
+                 const TensorType& output) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_MAXIMUM, BuiltinOptions_MaximumOptions,
+                 CreateMaximumOptions(builder_).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  template <class T>
+  void SetInput1(std::initializer_list<T> data) {
+    PopulateTensor(input1_, data);
+  }
+
+  template <class T>
+  void SetInput2(std::initializer_list<T> data) {
+    PopulateTensor(input2_, data);
+  }
+
+  template <class T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+TEST(MaximumOpTest, FloatTest) {
+  std::initializer_list<float> data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44};
+  std::initializer_list<float> data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43};
+  MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}},
+                   {TensorType_FLOAT32, {3, 1, 2}}, TensorType_FLOAT32);
+  m.SetInput1<float>(data1);
+  m.SetInput2<float>(data2);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2}));
+  EXPECT_THAT(
+      m.GetOutput<float>(),
+      ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43})));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 62045f0a4d..0f98154b90 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -76,6 +76,7 @@ TfLiteRegistration* Register_LOG_SOFTMAX();
 TfLiteRegistration* Register_CAST();
 TfLiteRegistration* Register_DEQUANTIZE();
 TfLiteRegistration* Register_PRELU();
+TfLiteRegistration* Register_MAXIMUM();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -133,6 +134,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
   AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
+  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index b7ccdf070b..791d1378f3 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -597,6 +597,9 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
       builtin_data = reinterpret_cast<void*>(params);
       break;
     }
+    case BuiltinOperator_MAXIMUM: {
+      break;
+    }
     case BuiltinOperator_DELEGATE: {
       // TODO(ycling): Revisit when supporting saving delegated models.
       error_reporter->Report("DELEGATE op shouldn't exist in model.");
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index e31b7c03a5..decaf9f160 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -350,6 +350,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
       case tflite::BuiltinOperator_DELEGATE:
       case tflite::BuiltinOperator_CAST:
       case tflite::BuiltinOperator_PRELU:
+      case tflite::BuiltinOperator_MAXIMUM:
         FATAL("Op code %d is currently not delegated to NNAPI", builtin);
         nn_op_type = -1;  // set to invalid
         break;
diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index 35d224924e..ed6dd036f9 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -25,9 +25,9 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import os
-import subprocess
-import tempfile
+import os as _os
+import subprocess as _subprocess
+import tempfile as _tempfile
 
 # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import convert_op_hints_to_stubs
@@ -74,7 +74,7 @@ else:
   _toco_from_proto_bin = _resource_loader.get_path_to_datafile(
       "../toco/python/toco_from_protos")
 
-if _toco_from_proto_bin and not os.path.exists(_toco_from_proto_bin):
+if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin):
   _toco_from_proto_bin = "toco_from_protos"
 
 
@@ -102,10 +102,10 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
     return _toco_python.TocoConvert(
         model_flags_str, toco_flags_str, input_data_str)
 
-  with tempfile.NamedTemporaryFile() as fp_toco, \
-           tempfile.NamedTemporaryFile() as fp_model, \
-           tempfile.NamedTemporaryFile() as fp_input, \
-           tempfile.NamedTemporaryFile() as fp_output:
+  with _tempfile.NamedTemporaryFile() as fp_toco, \
+           _tempfile.NamedTemporaryFile() as fp_model, \
+           _tempfile.NamedTemporaryFile() as fp_input, \
+           _tempfile.NamedTemporaryFile() as fp_output:
     fp_model.write(model_flags_str)
     fp_toco.write(toco_flags_str)
     fp_input.write(input_data_str)
@@ -118,11 +118,11 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
         fp_output.name
     ]
     cmdline = " ".join(cmd)
-    proc = subprocess.Popen(
+    proc = _subprocess.Popen(
         cmdline,
         shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
+        stdout=_subprocess.PIPE,
+        stderr=_subprocess.STDOUT,
         close_fds=True)
     stdout, stderr = proc.communicate()
     exitcode = proc.returncode
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index e1075971e9..7d2e00fe32 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -131,6 +131,7 @@ enum BuiltinOperator : byte {
   BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   CAST = 53,
   PRELU = 54,
+  MAXIMUM = 55,
 }
 
 // Options for the builtin operators.
@@ -173,6 +174,7 @@ union BuiltinOptions {
   LogSoftmaxOptions,
   CastOptions,
   DequantizeOptions,
+  MaximumOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -384,6 +386,9 @@ table CastOptions {
 table DequantizeOptions {
 }
 
+table MaximumOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 86daeaf5cc..66a97a1460 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -145,6 +145,9 @@ struct CastOptionsT;
 struct DequantizeOptions;
 struct DequantizeOptionsT;
 
+struct MaximumOptions;
+struct MaximumOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -255,11 +258,12 @@ enum BuiltinOperator {
   BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   BuiltinOperator_CAST = 53,
   BuiltinOperator_PRELU = 54,
+  BuiltinOperator_MAXIMUM = 55,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_PRELU
+  BuiltinOperator_MAX = BuiltinOperator_MAXIMUM
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -313,7 +317,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
     BuiltinOperator_DELEGATE,
     BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
     BuiltinOperator_CAST,
-    BuiltinOperator_PRELU
+    BuiltinOperator_PRELU,
+    BuiltinOperator_MAXIMUM
   };
   return values;
 }
@@ -375,6 +380,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "BIDIRECTIONAL_SEQUENCE_LSTM",
     "CAST",
     "PRELU",
+    "MAXIMUM",
     nullptr
   };
   return names;
@@ -425,11 +431,12 @@ enum BuiltinOptions {
   BuiltinOptions_LogSoftmaxOptions = 36,
   BuiltinOptions_CastOptions = 37,
   BuiltinOptions_DequantizeOptions = 38,
+  BuiltinOptions_MaximumOptions = 39,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_DequantizeOptions
+  BuiltinOptions_MAX = BuiltinOptions_MaximumOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[39] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -469,7 +476,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[39] {
     BuiltinOptions_SplitOptions,
     BuiltinOptions_LogSoftmaxOptions,
     BuiltinOptions_CastOptions,
-    BuiltinOptions_DequantizeOptions
+    BuiltinOptions_DequantizeOptions,
+    BuiltinOptions_MaximumOptions
   };
   return values;
 }
@@ -515,6 +523,7 @@ inline const char **EnumNamesBuiltinOptions() {
     "LogSoftmaxOptions",
     "CastOptions",
     "DequantizeOptions",
+    "MaximumOptions",
     nullptr
   };
   return names;
@@ -681,6 +690,10 @@ template<> struct BuiltinOptionsTraits<DequantizeOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_DequantizeOptions;
 };
 
+template<> struct BuiltinOptionsTraits<MaximumOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1016,6 +1029,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_DequantizeOptions ?
       reinterpret_cast<const DequantizeOptionsT *>(value) : nullptr;
   }
+  MaximumOptionsT *AsMaximumOptions() {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<MaximumOptionsT *>(value) : nullptr;
+  }
+  const MaximumOptionsT *AsMaximumOptions() const {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<const MaximumOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -3759,6 +3780,46 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(
 
 flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffers::FlatBufferBuilder &_fbb, const DequantizeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct MaximumOptionsT : public flatbuffers::NativeTable {
+  typedef MaximumOptions TableType;
+  MaximumOptionsT() {
+  }
+};
+
+struct MaximumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef MaximumOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  MaximumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<MaximumOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct MaximumOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit MaximumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  MaximumOptionsBuilder &operator=(const MaximumOptionsBuilder &);
+  flatbuffers::Offset<MaximumOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<MaximumOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  MaximumOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -3990,6 +4051,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const DequantizeOptions *builtin_options_as_DequantizeOptions() const {
     return builtin_options_type() == BuiltinOptions_DequantizeOptions ? static_cast<const DequantizeOptions *>(builtin_options()) : nullptr;
   }
+  const MaximumOptions *builtin_options_as_MaximumOptions() const {
+    return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast<const MaximumOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -4168,6 +4232,10 @@ template<> inline const DequantizeOptions *Operator::builtin_options_as<Dequanti
   return builtin_options_as_DequantizeOptions();
 }
 
+template<> inline const MaximumOptions *Operator::builtin_options_as<MaximumOptions>() const {
+  return builtin_options_as_MaximumOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -5696,6 +5764,29 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffer
       _fbb);
 }
 
+inline MaximumOptionsT *MaximumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new MaximumOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void MaximumOptions::UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<MaximumOptions> MaximumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateMaximumOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateMaximumOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -6028,6 +6119,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -6198,6 +6293,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -6356,6 +6455,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const DequantizeOptionsT *>(value);
       return CreateDequantizeOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptionsT *>(value);
+      return CreateMaximumOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -6514,6 +6617,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new DequantizeOptionsT(*reinterpret_cast<DequantizeOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      value = new MaximumOptionsT(*reinterpret_cast<MaximumOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -6711,6 +6818,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<MaximumOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 555ea90034..12b7b3c350 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -36,6 +36,7 @@ gen_zipped_test_files(
         "local_response_norm.zip",
         "log_softmax.zip",
         "max_pool.zip",
+        "maximum.zip",
         "mean.zip",
         "mul.zip",
         "pad.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index cb5c500136..8045052452 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -862,6 +862,41 @@ def make_log_softmax_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_maximum_tests(zip_path):
+  """Make a set of tests to do maximum."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the maximum op testing graph."""
+    input_tensor_1 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_1",
+        shape=parameters["input_shape_1"])
+    input_tensor_2 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_2",
+        shape=parameters["input_shape_2"])
+
+    out = tf.maximum(input_tensor_1, input_tensor_2)
+    return [input_tensor_1, input_tensor_2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_1"]),
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_2"])
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_binary_op_tests_func(binary_operator):
   """Return a function that does a test on a binary operator."""
   return lambda zip_path: make_binary_op_tests(zip_path, binary_operator)
@@ -1977,6 +2012,7 @@ def main(unused_args):
         "exp.zip": make_exp_tests,
         "log_softmax.zip": make_log_softmax_tests,
         "lstm.zip": make_lstm_tests,
+        "maximum.zip": make_maximum_tests,
     }
     out = FLAGS.zip_to_output
     bin_path = FLAGS.toco
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index a4a7283508..6697b86e79 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -253,6 +253,7 @@ INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
+INSTANTIATE_TESTS(maximum)
 INSTANTIATE_TESTS(max_pool)
 INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(mul)
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index f23249cfa1..0989bfe5a3 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -863,6 +863,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
   ops.emplace_back(new SimpleOperator<ExpOperator>("EXP", OperatorType::kExp));
   ops.emplace_back(new SimpleOperator<LogSoftmaxOperator>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
+  ops.emplace_back(new SimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum));
 
   return ops;
 }
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index 9c19f8d464..f7a213ecfc 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -109,6 +109,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<ExpOperator>("EXP", OperatorType::kExp);
   CheckSimpleOperator<LogSoftmaxOperator>("LOG_SOFTMAX",
                                           OperatorType::kLogSoftmax);
+  CheckSimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index a57a1e5421..a03e731be3 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -494,7 +494,7 @@ class MutableDenseHashTable(LookupInterface):
                                                   value_dtype=tf.int64,
                                                   default_value=-1,
                                                   empty_key=0)
-  table.insert(keys, values)
+  sess.run(table.insert(keys, values))
   out = table.lookup(query_keys)
   print(out.eval())
   ```
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 4ae18b2cef..8b415e6527 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -34,7 +34,7 @@ PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.
 RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
 FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)"
-CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
+CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 5a812af4e9..7a7683c953 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -258,6 +258,7 @@ tensorflow/core/kernels/requantize.cc
 tensorflow/core/kernels/remote_fused_graph_execute_op.cc
 tensorflow/core/kernels/remote_fused_graph_execute_utils.cc
 tensorflow/core/kernels/batch_matmul_op_real.cc
+tensorflow/core/kernels/random_op.cc
 tensorflow/core/ops/training_ops.cc
 tensorflow/core/ops/string_ops.cc
 tensorflow/core/ops/state_ops.cc
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index dfa12e873a..a9a32b7b25 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -74,7 +74,7 @@ class GatherTreeOp : public OpKernel {
         ctx,
         step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0),
         errors::InvalidArgument("batch size dimensions step_ids.shape[1] and "
-                                "max_seqeuence_lengths.shape[0] must match.  "
+                                "max_sequence_lengths.shape[0] must match.  "
                                 "but shapes are: ",
                                 step_ids_shape.DebugString(), " and ",
                                 max_sequence_lengths.shape().DebugString()));
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 9ff8a343f1..be53779826 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -736,7 +736,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
   """Base attention mechanism for monotonic attention.
 
   Simply overrides the initial_alignments function to provide a dirac
-  distribution,which is needed in order for the monotonic attention
+  distribution, which is needed in order for the monotonic attention
   distributions to have the correct behavior.
   """
 
@@ -763,7 +763,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -867,7 +867,7 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -1133,7 +1133,7 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
       output_attention: Python bool.  If `True` (default), the output at each
         time step is the attention value.  This is the behavior of Luong-style
         attention mechanisms.  If `False`, the output at each time step is
-        the output of `cell`.  This is the beahvior of Bhadanau-style
+        the output of `cell`.  This is the behavior of Bhadanau-style
         attention mechanisms.  In both cases, the `attention` tensor is
         propagated to the next time step via the state and is used there.
         This flag only controls whether the attention mechanism is propagated
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index a26107b0d7..184144f64a 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -821,9 +821,9 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight):
   Returns:
     The scores normalized by the length_penalty.
   """
-  length_penality_ = _length_penalty(
+  length_penalty_ = _length_penalty(
       sequence_lengths=sequence_lengths, penalty_factor=length_penalty_weight)
-  return log_probs / length_penality_
+  return log_probs / length_penalty_
 
 
 def _length_penalty(sequence_lengths, penalty_factor):
@@ -860,7 +860,7 @@ def _mask_probs(probs, eos_token, finished):
   unfinished beams remain unchanged.
 
   Args:
-    probs: Log probabiltiies of shape `[batch_size, beam_width, vocab_size]`
+    probs: Log probabilities of shape `[batch_size, beam_width, vocab_size]`
     eos_token: An int32 id corresponding to the EOS token to allocate
       probability to.
     finished: A boolean tensor of shape `[batch_size, beam_width]` that
diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
index b3343aef47..99ad487630 100644
--- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
+++ b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
@@ -115,8 +115,8 @@ class ParallelReader(io_ops.ReaderBase):
     reader needs to start reading from a new file since it has finished with
     the previous file).
 
-    A queue runner for enqueing in the `common_queue` is automatically added to
-    the TF QueueRunners collection.
+    A queue runner for enqueuing in the `common_queue` is automatically added
+    to the TF QueueRunners collection.
 
     Args:
       queue: A Queue or a mutable string Tensor representing a handle
diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
index 37e9c4754c..62bd200361 100644
--- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
+++ b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
@@ -36,9 +36,9 @@ def prefetch_queue(tensors,
                    dynamic_pad=False,
                    shared_name=None,
                    name=None):
-  """Creates a queue to prefetech tensors from `tensors`.
+  """Creates a queue to prefetch tensors from `tensors`.
 
-  A queue runner for enqueing tensors into the prefetch_queue is automatically
+  A queue runner for enqueuing tensors into the prefetch_queue is automatically
   added to the TF QueueRunners collection.
 
   Example:
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index b3b61e1dfe..f2d31dc8db 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -124,7 +124,7 @@ class BoundingBox(ItemHandler):
     super(BoundingBox, self).__init__(self._full_keys)
 
   def tensors_to_item(self, keys_to_tensors):
-    """Maps the given dictionary of tensors to a contatenated list of bboxes.
+    """Maps the given dictionary of tensors to a concatenated list of bboxes.
 
     Args:
       keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index 461e627e99..6eafc1754c 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -1,15 +1,15 @@
-Using TensorRT in TensorFlow
-============================
+# Using TensorRT in TensorFlow
+
 
 This module provides necessary bindings and introduces TRT_engine_op
 operator that wraps a subgraph in TensorRT. This is still a work in progress
 but should be useable with most common graphs.
 
-Compilation
------------
+## Compilation
+
 
 In order to compile the module, you need to have a local TensorRT
-installation (libnvinfer.so and respective include files). During the
+installation ( libnvinfer.so and respective include files ). During the
 configuration step, TensorRT should be enabled and installation path
 should be set. If installed through package managers (deb,rpm),
 configure script should find the necessary components from the system
@@ -22,4 +22,38 @@ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use can be found in test/test_tftrt.py directory
+will be available. An example use can be found in test/test_tftrt.py script
+
+## Installing TensorRT 3.0.4
+
+In order to make use of TensorRT integration, you will need a local installation of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). Due to compiler compatibility, you will need to download and install the TensorRT 3.0.4 tarball for _Ubuntu 14.04_, i.e., **_TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz_**, even if you are using Ubuntu 16.04 or later.
+
+### Preparing TensorRT installation
+
+Once you have downloaded TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz, you will need to unpack it to an installation directory, which will be referred to as <install_dir>. Please replace <install_dir> with the full path of actual installation directory you choose in commands below.
+
+```shell
+cd <install_dir> && tar -zxf /path/to/TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz
+```
+
+After unpacking the binaries, you have several options to use them:
+
+#### To run TensorFlow as a user without superuser privileges
+
+For a regular user without any sudo rights, you should add TensorRT to your `$LD_LIBRARY_PATH`:
+
+  ```shell
+   export LD_LIBRARY_PATH=<install_dir>/TensorRT-3.0.4/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+  ```
+
+Then you are ready to use TensorFlow-TensorRT integration. `$LD_LIBRARY_PATH` must contain the path to TensorRT installation for TensorFlow-TensorRT integration to work. If you are using a VirtualEnv-like setup, you can add the command above to your `bin/activate` script or to your `.bashrc` script.
+
+#### To run TensorFlow as a superuser
+
+ When running as a superuser, such as in a container or via sudo, the `$LD_LIBRARY_PATH` approach above may not work. The following is preferred when the user has superuser privileges:
+
+  ```shell
+  echo "<install_dir>/TensorRT-3.0.4/lib" | sudo tee /etc/ld.so.conf.d/tensorrt304.conf && sudo ldconfig
+  ```
+
+  Please ensure that any existing deb package installation of TensorRT is removed before following these instructions to avoid package conflicts.
\ No newline at end of file
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index eea8c8efa2..ff8cc6374d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,12 +49,13 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::Node* node) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
       "Identity",
+      "Snapshot",
       "Const",
       "Conv2D",
       "MaxPool",
@@ -74,7 +75,7 @@ bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
       // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
-  return candidate_ops.count(node_def.op());
+  return candidate_ops.count(node->type_string());
 }
 
 void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
@@ -84,10 +85,10 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->in_edges()) {
       if (!subgraph_node_ids.count(edge->src()->id()) &&
-          !edge->src()->IsSource()) {
+          !edge->src()->IsSource() && !edge->IsControlEdge()) {
         incoming_edges->insert(edge);
       } else {
-        VLOG(2) << edge->src()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->src()->name() << " N, ";
       }
     }
   }
@@ -100,11 +101,11 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
-          !edge->dst()->IsSink()) {
-        VLOG(2) << edge->dst()->name() << " Y, ";
+          !edge->dst()->IsSink() && !edge->IsControlEdge()) {
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
       } else {
-        VLOG(2) << edge->dst()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -409,8 +410,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
       if (status != tensorflow::Status::OK()) {
         LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \n"
-                     << status.ToString() << " SKIPPING......";
+                     << " due to: \"" << status.ToString()
+                     << "\" SKIPPING......( " << subgraph_node_names.size()
+                     << " nodes)";
       }
       count++;
     }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 92a692baa7..370911e4d9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -53,8 +53,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
-
 namespace {
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
@@ -430,9 +430,8 @@ class Converter {
   tensorflow::tensorrt::TRTWeightStore* weight_store_;
   bool fp16_;
   void register_op_converters();
-  std::vector<TRT_TensorOrWeights> get_inputs(
-      const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs;
+  tensorflow::Status get_inputs(const tensorflow::NodeDef& node_def,
+                                std::vector<TRT_TensorOrWeights>* inputs) {
     for (auto const& input_name : node_def.input()) {
       /*************************************************************************
        * TODO(jie) handle case 1) here
@@ -453,13 +452,17 @@ class Converter {
 
       VLOG(2) << "retrieve input: " << name;
       if (trt_tensors_.count(name)) {
-        inputs.push_back(trt_tensors_.at(name));
+        inputs->push_back(trt_tensors_.at(name));
       } else {
-        LOG(FATAL) << "input: " << name << " not available for node at, "
-                   << node_def.name();
+        string str("Node ");
+        StrAppend(&str, node_def.name(), " should have an input named '", name,
+                  "' but it is not available");
+        LOG(WARNING) << "input: " << name << " not available for node at "
+                     << node_def.name();
+        return tensorflow::errors::InvalidArgument(str);
       }
     }
-    return inputs;
+    return tensorflow::Status::OK();
   }
 
  public:
@@ -483,7 +486,8 @@ class Converter {
   }
 
   tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs = this->get_inputs(node_def);
+    std::vector<TRT_TensorOrWeights> inputs;
+    TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs));
     string op = node_def.op();
     if (!op_registry_.count(op)) {
       return tensorflow::errors::Unimplemented(
@@ -548,6 +552,19 @@ class Converter {
   }
 };
 
+TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx,
+                                    const TRT_ShapedWeights& weights_src) {
+  auto dtype_new = tensorflow::DataType::DT_HALF;
+  TRT_ShapedWeights weights =
+      ctx.get_temp_weights(dtype_new, weights_src.shape_);
+  const float* src = static_cast<const float*>(weights_src.GetValues());
+  Eigen::half* dst = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(weights.GetValues()));
+  for (int64_t i = 0; i < weights_src.count(); i++) {
+    dst[i] = Eigen::half_impl::float_to_half_rtne(src[i]);
+  }
+  return weights;
+}
 // ****************************************************************************
 // Constant folding functions
 // TODO(jie): once optimizer kicks in, we should have done constant folding
@@ -875,7 +892,7 @@ tensorflow::Status BinaryTensorOpWeight(
 
   // Check type consistency
   nvinfer1::DataType ttype;
-  TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
+  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
 
   // Check scale mode
   auto dims_w = weights.shape_;
@@ -957,6 +974,10 @@ tensorflow::Status BinaryTensorOpWeight(
     }
   }
 
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, weights);
+  }
+
   // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
@@ -998,9 +1019,7 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
     const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs,
-    int group  // group ==0 specifies depthwise conv
-) {
+    std::vector<TRT_TensorOrWeights>* outputs, int group) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   TFAttrs attrs(node_def);
@@ -1025,6 +1044,10 @@ tensorflow::Status ConvertConv2DHelper(
   VLOG(2) << "groups count: " << num_groups;
 
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights_rsck = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
   TRT_ShapedWeights biases(weights.type_);
@@ -1134,9 +1157,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
+    return tensorflow::errors::Unimplemented(
+        "binary op: " + node_def.op() +
+        " not supported at: " + node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1295,8 +1318,11 @@ tensorflow::Status ConvertScale(Converter& ctx,
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
-  // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights empty_weights(weights.type_);
 
   TFAttrs attrs(node_def);
@@ -1376,8 +1402,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.float_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1391,33 +1420,16 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      auto half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      half_tensor.device(defd) =
-          tensor.flat<float>().template cast<Eigen::half>();
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<float> tensor_data(
-          weights_tensor.float_val().begin(),
-          weights_tensor.float_val()
-              .end());  //  make a local copy first to flatten
-      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<float> tensor_data(
+        weights_tensor.float_val().begin(),
+        weights_tensor.float_val()
+            .end());  //  make a local copy first to flatten
+    memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.int_val().empty()) {
     VLOG(2) << "int!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
@@ -1432,8 +1444,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.int_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1447,62 +1462,23 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      switch (dtype) {
-        case (tensorflow::DT_INT32): {
-          half_tensor.device(defd) =
-              tensor.flat<int32>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT16): {
-          half_tensor.device(defd) =
-              tensor.flat<int16>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT8): {
-          half_tensor.device(defd) =
-              tensor.flat<int8>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_UINT8): {
-          half_tensor.device(defd) =
-              tensor.flat<uint8>().template cast<Eigen::half>();
-          break;
-        }
-        default:
-          return tensorflow::errors::InvalidArgument(
-              "Datatype " + tensorflow::DataTypeString(dtype) +
-              " for FP16 conversion");
-          break;
-      };
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
-      len_data = std::max(len_data, len_tensor);
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<int32> tensor_data(
-          weights_tensor.int_val().begin(),
-          weights_tensor.int_val()
-              .end());  //  make a local copy first to flatten
-                        //  doesn't have to be contiguous
-      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    //  we should not have converted //if (ctx.isFP16()) {
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+    len_data = std::max(len_data, len_tensor);
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<int32> tensor_data(
+        weights_tensor.int_val().begin(),
+        weights_tensor.int_val().end());  //  make a local copy first to flatten
+                                          //  doesn't have to be contigous
+    memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.tensor_content().empty()) {
+    //  obsolete method.
+    //  After optimization path, we do not see weights in this format.
+    //  fp16 conversion technically should be needed here.
     VLOG(2) << "TENSOR!!!" << node_def.name();
     const auto& content = weights_tensor.tensor_content();
 
@@ -1784,8 +1760,6 @@ tensorflow::Status ConvertConcat(Converter& ctx,
   TRT_ShapedWeights axis = inputs.at(input_size).weights();
 
   TFAttrs attrs(node_def);
-  // auto attr_size = attrs.at("N")->i();
-  // auto data_type = attrs.get<nvinfer1::DataType>("T");
   auto index_type = attrs.get<tensorflow::DataType>("Tidx");
 
   // TODO(jie): handle data type
@@ -1875,71 +1849,103 @@ tensorflow::Status ConvertFusedBatchNorm(
         "only is_training=false is supported, at " + node_def.name());
   }
   nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
-  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
-  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
-  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
-  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
-  TRT_ShapedWeights combined_scale_weights =
-      ctx.get_temp_weights_like(scale_weights);
-  TRT_ShapedWeights combined_offset_weights =
-      ctx.get_temp_weights_like(offset_weights);
-  size_t nweight = scale_weights.count();
-  if ((scale_weights.type_ == offset_weights.type_) &&
-      (mean_weights.type_ == variance_weights.type_) &&
-      (scale_weights.type_ == variance_weights.type_)) {
-    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
-        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+
+  //  Check parameter types
+  auto parameter_type = inputs.at(1).weights().type_;
+  if ((parameter_type != tensorflow::DataType::DT_FLOAT) &&
+      (parameter_type != tensorflow::DataType::DT_HALF)) {
+    return tensorflow::errors::Unimplemented(
+        "only float32 or float16 weight data type is supported, for node " +
+        node_def.name() + " got " + tensorflow::DataTypeString(parameter_type));
+  }
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().type_ != parameter_type) {
       return tensorflow::errors::Unimplemented(
-          "only float32 or float16 weight data type is supported, for node " +
-          node_def.name() + " got " +
-          tensorflow::DataTypeString(scale_weights.type_));
+          "Inconsistent parameter type for batchnormis not supported, at: " +
+          node_def.name());
     }
-    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
-        float offset =
-            (static_cast<float const*>(offset_weights.GetValues()))[i];
-        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
-        float variance =
-            (static_cast<float const*>(variance_weights.GetValues()))[i];
-        float& combined_scale_ref = const_cast<float*>(
-            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
-        float& combined_offset_ref = const_cast<float*>(
-            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
-        combined_scale_ref = scale / sqrtf(variance + epsilon);
-        combined_offset_ref = offset - mean * combined_scale_ref;
-      }
-    } else {
-      const Eigen::half* scale_vals =
-          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
-      const Eigen::half* off_vals =
-          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
-      const Eigen::half* mean_vals =
-          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
-      const Eigen::half* variance_vals =
-          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
-      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
-      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale(scale_vals[i]);
-        float offset(off_vals[i]);
-        float mean(mean_vals[i]);
-        float variance(variance_vals[i]);
-        float combined_scale_ref = scale / sqrtf(variance + epsilon);
-        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
-        float combined_offset_ref = offset - mean * combined_scale_ref;
-        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+  }
+
+  TRT_ShapedWeights dummy_power_weights(parameter_type);
+  size_t nweight = 0;
+  for (int i = 1; i < 5; i++) {
+    nweight = std::max(nweight, (size_t)inputs.at(i).weights().count());
+  }
+  TRT_ShapedWeights* ptr_shape_weights = nullptr;
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().count() == nweight) {
+      ptr_shape_weights =
+          const_cast<TRT_ShapedWeights*>(&(inputs.at(i).weights()));
+    } else if (inputs.at(i).weights().count() != 1) {
+      return tensorflow::errors::InvalidArgument(
+          "Inconsistent batchnorm parameter count, at: " + node_def.name());
+    }
+  }
+  //  We could technically have two weights with different shape.
+  //  that requires two addScale op, arguably less performant
+  TRT_ShapedWeights combined_scale_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+  TRT_ShapedWeights combined_offset_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+
+  const Eigen::half* cast_vals_array[4];
+  const float* vals_array[4];
+  for (int j = 0; j < 4; j++) {
+    cast_vals_array[j] =
+        static_cast<Eigen::half const*>(inputs.at(j + 1).weights().GetValues());
+    vals_array[j] =
+        static_cast<float const*>(inputs.at(j + 1).weights().GetValues());
+  }
+  Eigen::half* cast_combined_scale_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+  Eigen::half* cast_combined_offset_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+  float* combined_scale_vals = const_cast<float*>(
+      static_cast<float const*>(combined_scale_weights.GetValues()));
+  float* combined_offset_vals = const_cast<float*>(
+      static_cast<float const*>(combined_offset_weights.GetValues()));
+
+  for (size_t i = 0; i < nweight; ++i) {
+    float batchnorm_data[4];
+    for (int j = 0; j < 4; j++) {
+      if (inputs.at(j + 1).weights().count() != 1) {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][i];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][i]);
+        }
+      } else {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][0];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][0]);
+        }
       }
     }
+    float scale = batchnorm_data[0];
+    float offset = batchnorm_data[1];
+    float mean = batchnorm_data[2];
+    float variance = batchnorm_data[3];
+    float combined_scale_val = scale / sqrtf(variance + epsilon);
+    float combined_offset_val = offset - mean * combined_scale_val;
+    if (parameter_type == tensorflow::DT_FLOAT) {
+      combined_scale_vals[i] = combined_scale_val;
+      combined_offset_vals[i] = combined_offset_val;
+    } else if (parameter_type == tensorflow::DT_HALF) {
+      cast_combined_scale_vals[i] = Eigen::half(combined_scale_val);
+      cast_combined_offset_vals[i] = Eigen::half(combined_offset_val);
+    }
   }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      combined_offset_weights.GetWeightsForTRT(),
-      combined_scale_weights.GetWeightsForTRT(),
-      dummy_power_weights.GetWeightsForTRT());
+
+  nvinfer1::ScaleMode mode = nweight == 1 ? nvinfer1::ScaleMode::kUNIFORM
+                                          : nvinfer1::ScaleMode::kCHANNEL;
+  nvinfer1::IScaleLayer* layer =
+      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
+                              combined_offset_weights.GetWeightsForTRT(),
+                              combined_scale_weights.GetWeightsForTRT(),
+                              dummy_power_weights.GetWeightsForTRT());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -2050,6 +2056,7 @@ void Converter::register_op_converters() {
   op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
+  op_registry_["Snapshot"] = ConvertIdentity;  // Snapshot should be removed
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -2143,8 +2150,11 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
   calib_res->thr_->join();
   delete calib_res->thr_;
   if (!calib_res->engine_) {
-    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+    LOG(ERROR) << "Calibration failed!, engine does not exist. Did you run "
                   "calibration graph?";
+    return tensorflow::errors::FailedPrecondition(
+        "Calibration graph needs to be executed on"
+        " calibration data before convertsion to inference graph");
   }
   auto weight_rmgr = trt_rm->getManager("WeightStore");
   TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
@@ -2181,7 +2191,7 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
     return status;
   }
   auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_CHECK_OK(status);
+  TF_RETURN_IF_ERROR(status);
   for (size_t i = 0; i < out_edges.size(); i++) {
     VLOG(1) << "Connecting trt_engine_node output " << i << " with "
             << out_edges.at(i)->dst()->name() << " port "
@@ -2279,6 +2289,12 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Data type conversion for input '" << node_name
+                   << "' failed";
+      return type_status;
+    }
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "accessing output index of: " << output_idx
@@ -2346,8 +2362,8 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node'" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2504,7 +2520,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Type conversion failed for " << node_name;
+      return type_status;
+    }
 
     VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
@@ -2515,8 +2535,12 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
     //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4)
-      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+    if (op_info.shape().dim_size() != 4) {
+      string err_str = "Require 4 dimensional input.";
+      StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ",
+                shape_inference_node_name);
+      return tensorflow::errors::Unimplemented(err_str);
+    }
 
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
@@ -2577,8 +2601,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node '" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2622,7 +2646,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
   TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
       engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name;
+  LOG(INFO) << "finished engine " << engine_name << " containing "
+            << s.subgraph_node_ids.size() << " nodes";
 
   // Build the TRT op
   tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6193f0b0a1..8fc4697c51 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -80,13 +80,20 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> in_edges(dst->in_edges().begin(),
                                                 dst->in_edges().end());
   for (const tensorflow::Edge* in_edge : in_edges) {
-    if (in_edge->src() != src) {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-      if (e->src() == graph->source_node()) {
-        graph->AddEdge(e->src(), e->src_output(), src,
-                       tensorflow::Graph::kControlSlot);
-      } else {
-        graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+    if (in_edge->IsControlEdge()) {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        graph->AddControlEdge(e->src(), src);
+      }
+    } else {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        if (e->src() == graph->source_node()) {
+          graph->AddEdge(e->src(), e->src_output(), src,
+                         tensorflow::Graph::kControlSlot);
+        } else {
+          graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+        }
       }
     }
   }
@@ -94,12 +101,19 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> out_edges(dst->out_edges().begin(),
                                                  dst->out_edges().end());
   for (const tensorflow::Edge* out_edge : out_edges) {
-    tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-    if (e->dst() == graph->sink_node()) {
-      graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
-                     e->dst_input());
+    if (out_edge->IsControlEdge()) {
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      graph->AddControlEdge(src, e->dst());
     } else {
-      graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      if (e->dst() == graph->sink_node()) {
+        VLOG(1) << " edge to sink node " << src->name() << " -> "
+                << e->dst()->name();
+        graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
+                       e->dst_input());
+      } else {
+        graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      }
     }
   }
 
@@ -118,7 +132,7 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
   // Create a Graph representation of the GraphDef.
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
@@ -136,7 +150,7 @@ tensorflow::Status SegmentGraph(
   for (int i = 0; i < graph.num_node_ids(); ++i) {
     tensorflow::Node* node = graph.FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0 ||
-        !candidate_fn(node->def())) {
+        !candidate_fn(node)) {
       node = nullptr;
     }
     node_segments.emplace_back(node);
@@ -155,7 +169,7 @@ tensorflow::Status SegmentGraph(
 
   for (const tensorflow::Node* node : order) {
     // All output nodes of 'node' have been visited...
-    VLOG(2) << "Trying node " << node->name();
+    VLOG(2) << "Trying node " << node->name() << " id=" << node->id();
 
     // 'node' must be a TRT candidate...
     if (node_segments[node->id()].Value() == nullptr) {
@@ -169,8 +183,12 @@ tensorflow::Status SegmentGraph(
     while (true) {
       std::set<const tensorflow::Edge*> contract_edges;
       for (const tensorflow::Edge* out_edge : node->out_edges()) {
-        VLOG(2) << "... out node " << out_edge->dst()->name();
-
+        VLOG(2) << "... out node " << out_edge->dst()->name() << " ( "
+                << out_edge->dst()->id() << " <- " << node->id() << " )";
+        if (out_edge->IsControlEdge()) {
+          VLOG(2) << "... ... Control Edge, Skipping";
+          continue;
+        }
         // Out node must be TRT candidate...
         if (node_segments[out_edge->dst()->id()].Value() == nullptr) {
           VLOG(2) << "... ... not a TRT candidate";
@@ -196,7 +214,8 @@ tensorflow::Status SegmentGraph(
         const tensorflow::Node* src = contract_edge->src();
         const tensorflow::Node* dst = contract_edge->dst();
 
-        VLOG(2) << "Merge " << src->name() << " <- " << dst->name();
+        VLOG(2) << "Merge " << src->name() << " <- " << dst->name() << " ("
+                << src->id() << " <- " << dst->id();
         node_segments[src->id()].Merge(&node_segments[dst->id()]);
 
         // Contracting the edge leaves disconnected graph edges.
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index ee6e2b3ed2..7e8685f44a 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -20,10 +20,12 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+
 namespace tensorrt {
 namespace segment {
 
@@ -46,7 +48,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
 
 }  // namespace segment
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 74cbc5f2b3..7ddabec268 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -35,7 +35,7 @@ class SegmentTest : public ::testing::Test {
   TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph,
                     TF_Status* s, const char* name);
 
-  std::function<bool(const NodeDef&)> MakeCandidateFn(
+  std::function<bool(const Node*)> MakeCandidateFn(
       const std::set<string>& node_names);
 
  protected:
@@ -60,10 +60,10 @@ bool SegmentTest::GetGraphDef(TF_Graph* graph,
   return ret;
 }
 
-std::function<bool(const NodeDef&)> SegmentTest::MakeCandidateFn(
+std::function<bool(const Node*)> SegmentTest::MakeCandidateFn(
     const std::set<string>& node_names) {
-  return [node_names](const NodeDef& node) -> bool {
-    return node_names.find(node.name()) != node_names.end();
+  return [node_names](const Node* node) -> bool {
+    return node_names.find(node->name()) != node_names.end();
   };
 }
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index ff140efd48..4f6527a546 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -70,7 +70,7 @@ class ARModel(model.TimeSeriesModel):
       input_window_size: Number of past time steps of data to look at when doing
         the regression.
       output_window_size: Number of future time steps to predict. Note that
-        setting it to > 1 empiricaly seems to give a better fit.
+        setting it to > 1 empirically seems to give a better fit.
       num_features: number of input features per time step.
       num_time_buckets: Number of buckets into which to divide (time %
         periodicity) for generating time based features.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index 23452a81c3..26793c80bf 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -185,7 +185,7 @@ def batch_matrix_pow(matrices, powers):
                     { matmul(A, power(matmul(A, A), (p - 1) / 2)) for odd p
       power(A, 0) = I
 
-    The power(A, 0) = I case is handeled by starting with accumulator set to the
+    The power(A, 0) = I case is handled by starting with accumulator set to the
     identity matrix; matrices with zero residual powers are passed through
     unchanged.
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
index 1afc58cfb2..6746dd7b43 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
@@ -107,7 +107,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state transition matrix. It has shape
-        [self.state_dimendion, self.state_dimension].
+        [self.state_dimension, self.state_dimension].
     """
     # Pad any unused AR blocks with zeros. The extra state is necessary if
     # ma_order >= ar_order.
@@ -127,7 +127,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state noise transform matrix. It has shape
-        [self.state_dimendion, self.num_features].
+        [self.state_dimension, self.num_features].
     """
     # Noise is broadcast, through the moving average coefficients, to
     # un-observed parts of the latent state.
diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
index 51d91399f8..e667c328ae 100644
--- a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
@@ -49,14 +49,14 @@ in the batch:
 If `fast` is `True`, then the solution is computed by solving the normal
 equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
 \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-\lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). 
+If \\(m \lt n\\) then `output` is computed as
 \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
 minimum-norm solution to the under-determined linear system, i.e.
 \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
 subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
 when \\(A\\) is numerically full rank and has a condition number
-\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is
 sufficiently large.
 
 If `fast` is `False` an algorithm based on the numerically robust complete
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 43a909466e..829c19204a 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-constexpr const char* MklCPUAllocator::kMaxLimitStr;
-constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
-
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 623248b6ce..2fb17c2b02 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1210,7 +1210,7 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
-Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index) {
   ShapeHandle shape_x = c->input(0);
   ShapeHandle shape_y = c->input(1);
   if (!c->RankKnown(shape_x) || !c->RankKnown(shape_y)) {
@@ -1272,7 +1272,7 @@ Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
     }
   }
 
-  c->set_output(0, c->MakeShape(dims));
+  c->set_output(output_index, c->MakeShape(dims));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 293c40e04d..789746b403 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -265,9 +265,15 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
+// Shape function for binary operators that broadcast their inputs
+// and with output to output_index.
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index);
+
 // Shape function for binary operators that broadcast their inputs.
 // Tested by ops/math_ops_test.cc.
-Status BroadcastBinaryOpShapeFn(InferenceContext* c);
+inline Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+  return BroadcastBinaryOpOutputShapeFn(c, 0);
+}
 
 // Shape function for random operations.
 Status RandomShape(shape_inference::InferenceContext* c);
diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h
index e3cc848a16..accc587000 100644
--- a/tensorflow/core/framework/shape_inference.h
+++ b/tensorflow/core/framework/shape_inference.h
@@ -317,6 +317,7 @@ class InferenceContext {
     input_tensors_as_shapes_ = input_tensors_as_shapes;
   }
 
+  ShapeHandle output(int64 idx) const { return outputs_[idx]; }
   void set_output(int idx, ShapeHandle shape) { outputs_[idx] = shape; }
   Status set_output(StringPiece output_name,
                     const std::vector<ShapeHandle>& shapes);
diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 333a6570dc..62aafa7930 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -933,7 +933,7 @@ class MklFusedBatchNormOp : public OpKernel {
   bool is_training_;
   T* mean_values_;
   T* variance_values_;
-  size_t depth_;  // batch normalization is done for per channel.
+  int depth_;  // batch normalization is done for per channel.
 
   void ExtractParams(OpKernelContext* context) {
     const Tensor& input = MklGetInput(context, 0);
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index 4abfbfb1a6..7badc00572 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -23,6 +23,13 @@ limitations under the License.
 // non-GPU targets. This only breaks in clang, because it's more strict for
 // template code and CudaAtomicMax is used in template context.
 
+// This file requires the following include because it uses CudaAtomicMax:
+// #include "tensorflow/core/util/cuda_kernel_helper.h"
+
+// Unfortunately we can't add the #include, since it breaks compilation for
+// non-GPU targets. This only breaks in clang, because it's more strict for
+// template code and CudaAtomicMax is used in template context.
+
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/core/kernels/snapshot_op.cc b/tensorflow/core/kernels/snapshot_op.cc
index 50157d5d48..fe04dcf72e 100644
--- a/tensorflow/core/kernels/snapshot_op.cc
+++ b/tensorflow/core/kernels/snapshot_op.cc
@@ -22,6 +22,26 @@ limitations under the License.
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename Device, typename Scalar>
+class SnapshotOp : public OpKernel {
+ public:
+  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& input = context->input(0);
+    Tensor* output = nullptr;
+    // Try to use buffer forwarding to avoid an explicit copy.
+    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                {0}, 0, input.shape(), &output));
+    if (!output->SharesBufferWith(input)) {
+      functor::Snapshot<Device, Scalar> functor;
+      functor(context->eigen_device<Device>(), input.flat<Scalar>(),
+              output->flat<Scalar>());
+    }
+  }
+};
 
 #define REGISTER_KERNEL(TYPE)                                        \
   REGISTER_KERNEL_BUILDER(                                           \
@@ -31,6 +51,16 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
 
+#if GOOGLE_CUDA
+#define REGISTER_KERNEL(TYPE)                                        \
+  REGISTER_KERNEL_BUILDER(                                           \
+      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
+      SnapshotOp<GPUDevice, TYPE>);
+
+TF_CALL_POD_TYPES(REGISTER_KERNEL);
+#undef REGISTER_KERNEL
+#endif
+
 #if TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SyclDevice;
 #define REGISTER_SYCL_KERNEL(TYPE)                                    \
diff --git a/tensorflow/core/kernels/snapshot_op.h b/tensorflow/core/kernels/snapshot_op.h
index b94834f159..a18065d42b 100644
--- a/tensorflow/core/kernels/snapshot_op.h
+++ b/tensorflow/core/kernels/snapshot_op.h
@@ -26,29 +26,19 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
+namespace functor {
 
+// Functor used by SnapshotOp.
 template <typename Device, typename Scalar>
-class SnapshotOp : public OpKernel {
- public:
-  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input = context->input(0);
-    Tensor* output = nullptr;
-    // Try to use buffer forwarding to avoid an explicit copy.
-    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 0, input.shape(), &output));
-    if (!output->SharesBufferWith(input)) {
-      // We had to allocate a new buffer since the refcount on the input was
-      // greater than 1. Copy the input to the new buffer.
-      const Device& device = context->eigen_device<Device>();
-      device.memcpy(output->template flat<Scalar>().data(),
-                    input.template flat<Scalar>().data(),
-                    input.NumElements() * sizeof(Scalar));
-    }
+struct Snapshot {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar>::ConstTensor input,
+                  typename TTypes<Scalar>::Tensor output) {
+    device.memcpy(output.data(), input.data(), input.size() * sizeof(Scalar));
   }
 };
 
+}  // namespace functor
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_KERNELS_SNAPSHOT_OP_H_
diff --git a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
index 52070be838..e4e3bd5220 100644
--- a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
@@ -24,13 +24,10 @@ limitations under the License.
 namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_KERNEL(TYPE)                                        \
-  REGISTER_KERNEL_BUILDER(                                           \
-      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
-      SnapshotOp<GPUDevice, TYPE>);
+// Definition of the GPU implementations declared in softsign_op.cc.
+#define DEFINE_GPU_KERNELS(T) template struct functor::Snapshot<GPUDevice, T>;
 
-TF_CALL_POD_TYPES(REGISTER_KERNEL);
-#undef REGISTER_KERNEL
+TF_CALL_POD_TYPES(DEFINE_GPU_KERNELS);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/xent_op.cc b/tensorflow/core/kernels/xent_op.cc
index a6a71fdfaf..9a3612bd72 100644
--- a/tensorflow/core/kernels/xent_op.cc
+++ b/tensorflow/core/kernels/xent_op.cc
@@ -17,12 +17,14 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
-#include "tensorflow/core/kernels/xent_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/xent_op.h"
+#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
@@ -41,37 +43,56 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& logits_in = context->input(0);
     const Tensor& labels_in = context->input(1);
-    OP_REQUIRES(context, logits_in.IsSameSize(labels_in),
-                errors::InvalidArgument(
-                    "logits and labels must be same size: logits_size=",
-                    logits_in.shape().DebugString(),
-                    " labels_size=", labels_in.shape().DebugString()));
-    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits_in.shape()),
-                errors::InvalidArgument("logits must be 2-dimensional"));
-    // As we already tested that both inputs have the same shape no need to
-    // check that "labels" is a matrix too.
+
+    TensorShape shape_in = logits_in.shape();
+
+    BCast bcast(BCast::FromShape(logits_in.shape()),
+                BCast::FromShape(labels_in.shape()));
+    if (!logits_in.IsSameSize(labels_in)) {
+      OP_REQUIRES(context, bcast.IsValid(),
+                  errors::InvalidArgument(
+                      "logits and labels must be broadcastable: logits_size=",
+                      logits_in.shape().DebugString(),
+                      " labels_size=", labels_in.shape().DebugString()));
+      shape_in = BCast::ToShape(bcast.output_shape());
+    }
+    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
+                errors::InvalidArgument("logits and labels must be beither "
+                                        "2-dimensional, or roadcasted to "
+                                        "2-dimensional"));
 
     // loss is 1-D (one per example), and size is batch_size.
 
     Tensor scratch;
     OP_REQUIRES_OK(
         context, context->allocate_temp(DataTypeToEnum<T>::value,
-                                        TensorShape({logits_in.dim_size(0), 1}),
+                                        TensorShape({shape_in.dim_size(0), 1}),
                                         &scratch));
 
     Tensor* loss_out = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(
-                       0, TensorShape({logits_in.dim_size(0)}), &loss_out));
+                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
     Tensor* back_out = nullptr;
     // Try to reuse the logits_in buffer for the backprop output.
     OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 1, logits_in.shape(), &back_out));
-    if (logits_in.dim_size(0) > 0) {
+                                {0}, 1, shape_in, &back_out));
+    if (shape_in.dim_size(0) > 0) {
       functor::XentFunctor<Device, T> functor;
-      functor(context->eigen_device<Device>(), logits_in.matrix<T>(),
-              labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
-              back_out->matrix<T>());
+      if (logits_in.IsSameSize(labels_in)) {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1},
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1}, logits_in.matrix<T>(),
+                labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
+                back_out->matrix<T>());
+      } else {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                BCast::ToIndexArray<2>(bcast.x_bcast()),
+                BCast::ToIndexArray<2>(bcast.y_bcast()),
+                logits_in.template shaped<T, 2>(bcast.x_reshape()),
+                labels_in.template shaped<T, 2>(bcast.y_reshape()),
+                scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
+      }
     }
   }
 };
@@ -81,13 +102,17 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename Device, typename T>
 struct XentFunctorBase {
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device& d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2>& shape,
+                  const Eigen::array<Eigen::DenseIndex, 2>& logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2>& labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<Device, T>::Compute(d, logits, labels, scratch, loss,
-                                      backprop);
+    XentEigenImpl<Device, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                      logits, labels, scratch, loss, backprop);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op.h b/tensorflow/core/kernels/xent_op.h
index e689fca7ff..87be17fca9 100644
--- a/tensorflow/core/kernels/xent_op.h
+++ b/tensorflow/core/kernels/xent_op.h
@@ -18,6 +18,7 @@ limitations under the License.
 // Functor definition for XentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor_types.h"
 
 namespace tensorflow {
@@ -33,7 +34,11 @@ struct XentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
@@ -45,7 +50,11 @@ struct XentFunctor {
 // specializations for both device types.
 template <typename Device, typename T>
 struct XentEigenImpl {
-  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  static void Compute(const Device &d,
+                      const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                      const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                      const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                      typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<T>::ConstMatrix labels,
                       typename TTypes<T>::Matrix scratch,
                       typename TTypes<T>::Vec loss,
@@ -57,8 +66,8 @@ struct XentEigenImpl {
     const int kBatchDim = 0;
     const int kClassDim = 1;
 
-    const int batch_size = logits.dimension(kBatchDim);
-    const int num_classes = logits.dimension(kClassDim);
+    const int batch_size = shape[kBatchDim];
+    const int num_classes = shape[kClassDim];
 
 // These arrays are used to reduce along the class dimension, and broadcast
 // the resulting value to all classes.
@@ -84,10 +93,12 @@ struct XentEigenImpl {
 #endif
 
     // max_logits along classes.
-    scratch.reshape(batch_only).device(d) = logits.maximum(along_class);
+    scratch.reshape(batch_only).device(d) =
+        logits.broadcast(logits_bcast).maximum(along_class);
 
     // logits - max_logits.
-    backprop.device(d) = logits - scratch.broadcast(one_by_class);
+    backprop.device(d) =
+        logits.broadcast(logits_bcast) - scratch.broadcast(one_by_class);
 
     // sum(exp(logits - max_logits)) along classes.
     scratch.reshape(batch_only).device(d) = backprop.exp().sum(along_class);
@@ -99,15 +110,15 @@ struct XentEigenImpl {
     //  sum(-labels *
     //     ((logits - max_logits) - log(sum(exp(logits - max_logits)))))
     //  along classes
-    loss.device(d) =
-        (labels * (scratch.log().eval().broadcast(one_by_class) - backprop))
-            .eval()
-            .sum(along_class);
+    loss.device(d) = (labels.broadcast(labels_bcast) *
+                      (scratch.log().eval().broadcast(one_by_class) - backprop))
+                         .eval()
+                         .sum(along_class);
 
     // backprop: prob - labels, where
     //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
-    backprop.device(d) =
-        (backprop.exp() / scratch.broadcast(one_by_class)) - labels;
+    backprop.device(d) = (backprop.exp() / scratch.broadcast(one_by_class)) -
+                         labels.broadcast(labels_bcast);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op_gpu.cu.cc b/tensorflow/core/kernels/xent_op_gpu.cu.cc
index 05ee7da490..2c0c0b3a02 100644
--- a/tensorflow/core/kernels/xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/xent_op_gpu.cu.cc
@@ -31,12 +31,17 @@ typedef Eigen::GpuDevice GPUDevice;
 namespace functor {
 template <typename T>
 struct XentFunctor<GPUDevice, T> {
-  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const GPUDevice &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<GPUDevice, T>::Compute(d, logits, labels, scratch, loss,
+    XentEigenImpl<GPUDevice, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                         logits, labels, scratch, loss,
                                          backprop);
   }
 };
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 88d2aa3f41..111670c361 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -794,11 +794,35 @@ REGISTER_OP("ReverseV2")
       ShapeHandle input = c->input(0);
       ShapeHandle axis;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &axis));
-      // TODO(aselle): if input(0)'s dimension is known we could validate axis
       if (c->Rank(input) > 8) {
         return errors::InvalidArgument(
             "reverse does not work on tensors with more than 8 dimensions");
       }
+      const Tensor* axis_tensor = c->input_tensor(1);
+      if (axis_tensor != nullptr && c->RankKnown(input)) {
+        int32 rank = c->Rank(input);
+        std::vector<int64> axis_value;
+        if (axis_tensor->dtype() == DT_INT32) {
+          axis_value = AsInt64<int32>(axis_tensor, axis_tensor->NumElements());
+        } else {
+          axis_value = AsInt64<int64>(axis_tensor, axis_tensor->NumElements());
+        }
+        std::vector<bool> axes_dense(c->Rank(input), false);
+        for (int i = 0; i < axis_value.size(); i++) {
+          int64 canonical_axis =
+              axis_value[i] < 0 ? rank + axis_value[i] : axis_value[i];
+          if (canonical_axis < 0 || canonical_axis >= rank) {
+            return errors::InvalidArgument("'axis'[", i, "] = ", axis_value[i],
+                                           " is out of valid range [", 0, ", ",
+                                           rank - 1);
+          }
+          if (axes_dense[canonical_axis]) {
+            return errors::InvalidArgument("axis ", canonical_axis,
+                                           " specified more than once.");
+          }
+          axes_dense[canonical_axis] = true;
+        }
+      }
       c->set_output(0, input);
       return Status::OK();
     });
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 1f4e9753c3..6c2fc60bab 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1062,12 +1062,27 @@ REGISTER_OP("SoftmaxCrossEntropyWithLogits")
     .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input));
-      TF_RETURN_IF_ERROR(c->Merge(input, c->input(1), &input));
+      if (c->WithRank(c->input(0), 2, &input) == Status::OK() &&
+          c->Merge(input, c->input(1), &input) == Status::OK()) {
+        DimensionHandle batch_size = c->Dim(input, 0);
+        c->set_output(0, c->Vector(batch_size));
+        c->set_output(1, input);
+        return Status::OK();
+      }
+      TF_RETURN_IF_ERROR(BroadcastBinaryOpOutputShapeFn(c, 1));
 
-      DimensionHandle batch_size = c->Dim(input, 0);
+      if (!c->RankKnown(c->output(1))) {
+        return errors::InvalidArgument(
+            "Shape must be broadcasted with rank 2, but is rank is unknown.");
+      }
+
+      if (c->Rank(c->output(1)) != 2) {
+        return errors::InvalidArgument(
+            "Shape must be broadcasted with rank 2, but is rank ",
+            c->Rank(c->output(1)));
+      }
+      DimensionHandle batch_size = c->Dim(c->output(1), 0);
       c->set_output(0, c->Vector(batch_size));
-      c->set_output(1, input);
       return Status::OK();
     });
 
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 1b17a7cda6..289b953055 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -410,10 +410,18 @@ TEST(NNOpsTest, SoftmaxCrossEntropyWithLogits_ShapeFn) {
   INFER_OK(op, "[1,?];[?,2]", "[d0_0];[d0_0,d0_1|d1_1]");
   INFER_OK(op, "[?,2];[1,2]", "[d1_0];in1");
 
-  INFER_ERROR("Dimension 0 in both shapes must be equal, but are 1 and 2", op,
-              "[1,?];[2,?]");
-  INFER_ERROR("Shape must be rank 2 but is rank 3", op, "[1,2,3];?");
-  INFER_ERROR("Shapes must be equal rank, but are 2 and 3", op, "?;[1,2,3]");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "[1,2,3];?");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "?;[1,2,3]");
+
+  // Broadcast example
+  // [1,4] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,4];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [2,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[2,1]", "[d0_0];[d0_0|d1_0,d0_1]");
+  // [1,?] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,?];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [?,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[?,1]", "[d0_0];[d0_0|d1_0,d0_1]");
 }
 
 TEST(NNOpsTest, SparseSoftmaxCrossEntropyWithLogits_ShapeFn) {
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 22f2c02b78..40eebd1db0 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 6
+#define TF_MINOR_VERSION 7
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc1"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
index 956dccb64f..f3db5857ae 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
@@ -6,42 +6,42 @@ Monte Carlo integration and helpers.
 ## Background
 
 Monte Carlo integration refers to the practice of estimating an expectation with
-a sample mean.  For example, given random variable `Z in R^k` with density `p`,
+a sample mean.  For example, given random variable `Z in \\(R^k\\)` with density `p`,
 the expectation of function `f` can be approximated like:
 
 ```
-E_p[f(Z)] = \int f(z) p(z) dz
-          ~ S_n
-          := n^{-1} \sum_{i=1}^n f(z_i),  z_i iid samples from p.
+$$E_p[f(Z)] = \int f(z) p(z) dz$$
+$$          ~ S_n
+          := n^{-1} \sum_{i=1}^n f(z_i),  z_i\ iid\ samples\ from\ p.$$
 ```
 
-If `E_p[|f(Z)|] < infinity`, then `S_n --> E_p[f(Z)]` by the strong law of large
-numbers.  If `E_p[f(Z)^2] < infinity`, then `S_n` is asymptotically normal with
-variance `Var[f(Z)] / n`.
+If `\\(E_p[|f(Z)|] < infinity\\)`, then `\\(S_n\\) --> \\(E_p[f(Z)]\\)` by the strong law of large
+numbers.  If `\\(E_p[f(Z)^2] < infinity\\)`, then `\\(S_n\\)` is asymptotically normal with
+variance `\\(Var[f(Z)] / n\\)`.
 
 Practitioners of Bayesian statistics often find themselves wanting to estimate
-`E_p[f(Z)]` when the distribution `p` is known only up to a constant.  For
+`\\(E_p[f(Z)]\\)` when the distribution `p` is known only up to a constant.  For
 example, the joint distribution `p(z, x)` may be known, but the evidence
-`p(x) = \int p(z, x) dz` may be intractable.  In that case, a parameterized
-distribution family `q_lambda(z)` may be chosen, and the optimal `lambda` is the
-one minimizing the KL divergence between `q_lambda(z)` and
-`p(z | x)`.  We only know `p(z, x)`, but that is sufficient to find `lambda`.
+`\\(p(x) = \int p(z, x) dz\\)` may be intractable.  In that case, a parameterized
+distribution family `\\(q_\lambda(z)\\)` may be chosen, and the optimal `\\(\lambda\\)` is the
+one minimizing the KL divergence between `\\(q_\lambda(z)\\)` and
+`\\(p(z | x)\\)`.  We only know `p(z, x)`, but that is sufficient to find `\\(\lambda\\)`.
 
 
 ## Log-space evaluation and subtracting the maximum
 
 Care must be taken when the random variable lives in a high dimensional space.
-For example, the naive importance sample estimate `E_q[f(Z) p(Z) / q(Z)]`
-involves the ratio of two terms `p(Z) / q(Z)`, each of which must have tails
-dropping off faster than `O(|z|^{-(k + 1)})` in order to have finite integral.
+For example, the naive importance sample estimate `\\(E_q[f(Z) p(Z) / q(Z)]\\)`
+involves the ratio of two terms `\\(p(Z) / q(Z)\\)`, each of which must have tails
+dropping off faster than `\\(O(|z|^{-(k + 1)})\\)` in order to have finite integral.
 This ratio would often be zero or infinity up to numerical precision.
 
 For that reason, we write
 
 ```
-Log E_q[ f(Z) p(Z) / q(Z) ]
-   = Log E_q[ exp{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C} ] + C,  where
-C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].
+$$Log E_q[ f(Z) p(Z) / q(Z) ]$$
+$$   = Log E_q[ \exp\{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C\} ] + C,$$  where
+$$C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].$$
 ```
 
 The maximum value of the exponentiated term will be 0.0, and the expectation
diff --git a/tensorflow/docs_src/api_guides/python/contrib.losses.md b/tensorflow/docs_src/api_guides/python/contrib.losses.md
index d7f862625e..8b7442216c 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.losses.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.losses.md
@@ -107,19 +107,19 @@ weighted average over the individual prediction errors:
   loss = tf.contrib.losses.mean_squared_error(predictions, depths, weight)
 ```
 
-@{tf.contrib.losses.absolute_difference}
-@{tf.contrib.losses.add_loss}
-@{tf.contrib.losses.hinge_loss}
-@{tf.contrib.losses.compute_weighted_loss}
-@{tf.contrib.losses.cosine_distance}
-@{tf.contrib.losses.get_losses}
-@{tf.contrib.losses.get_regularization_losses}
-@{tf.contrib.losses.get_total_loss}
-@{tf.contrib.losses.log_loss}
-@{tf.contrib.losses.mean_pairwise_squared_error}
-@{tf.contrib.losses.mean_squared_error}
-@{tf.contrib.losses.sigmoid_cross_entropy}
-@{tf.contrib.losses.softmax_cross_entropy}
-@{tf.contrib.losses.sparse_softmax_cross_entropy}
+* @{tf.contrib.losses.absolute_difference}
+* @{tf.contrib.losses.add_loss}
+* @{tf.contrib.losses.hinge_loss}
+* @{tf.contrib.losses.compute_weighted_loss}
+* @{tf.contrib.losses.cosine_distance}
+* @{tf.contrib.losses.get_losses}
+* @{tf.contrib.losses.get_regularization_losses}
+* @{tf.contrib.losses.get_total_loss}
+* @{tf.contrib.losses.log_loss}
+* @{tf.contrib.losses.mean_pairwise_squared_error}
+* @{tf.contrib.losses.mean_squared_error}
+* @{tf.contrib.losses.sigmoid_cross_entropy}
+* @{tf.contrib.losses.softmax_cross_entropy}
+* @{tf.contrib.losses.sparse_softmax_cross_entropy}
 
 
diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md
index 003e0a25ec..6f2107ef40 100644
--- a/tensorflow/docs_src/community/documentation.md
+++ b/tensorflow/docs_src/community/documentation.md
@@ -477,31 +477,29 @@ should use Markdown in the docstring.
 
 Here's a simple example:
 
-```python
-def foo(x, y, name="bar"):
-  """Computes foo.
+    def foo(x, y, name="bar"):
+      """Computes foo.
 
-  Given two 1-D tensors `x` and `y`, this operation computes the foo.
+      Given two 1-D tensors `x` and `y`, this operation computes the foo.
 
-  Example:
+      Example:
 
-  ```
-  # x is [1, 1]
-  # y is [2, 2]
-  tf.foo(x, y) ==> [3, 3]
-  ```
-  Args:
-    x: A `Tensor` of type `int32`.
-    y: A `Tensor` of type `int32`.
-    name: A name for the operation (optional).
+      ```
+      # x is [1, 1]
+      # y is [2, 2]
+      tf.foo(x, y) ==> [3, 3]
+      ```
+      Args:
+        x: A `Tensor` of type `int32`.
+        y: A `Tensor` of type `int32`.
+        name: A name for the operation (optional).
 
-  Returns:
-    A `Tensor` of type `int32` that is the foo of `x` and `y`.
+      Returns:
+        A `Tensor` of type `int32` that is the foo of `x` and `y`.
 
-  Raises:
-    ValueError: If `x` or `y` are not of type `int32`.
-  """
-```
+      Raises:
+        ValueError: If `x` or `y` are not of type `int32`.
+      """
 
 ## Description of the docstring sections
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 0481c97885..9059b3f3b6 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 8f89898c92..2e47a6d212 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 0ee9c849e1..eff066d200 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0</version>
+                 <version>1.7.0-rc1</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 5e9a84bff6..27b696696d 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -165,7 +165,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -270,7 +270,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -456,7 +456,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -630,14 +630,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -649,14 +649,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -668,14 +668,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -687,14 +687,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 55b460e189..7060ef43da 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -118,8 +118,8 @@ Take the following steps to install TensorFlow with Virtualenv:
      Python 2.7, the command to install
      TensorFlow in the active Virtualenv is as follows:
 
-     <pre> $ <b>pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     <pre> $ <b>pip3 install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -241,8 +241,8 @@ take the following steps:
      you are installing TensorFlow for macOS and Python 2.7
      issue the following command:
 
-     <pre> $ <b>sudo pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
+     <pre> $ <b>sudo pip3 install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -524,7 +524,7 @@ The value you specify depends on your Python version.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
 </pre>
 
 
@@ -532,5 +532,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index a7f33819b4..148f80efe2 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -350,10 +350,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0 on Linux:
+for TensorFlow 1.7.0rc1 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -450,6 +450,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -469,6 +471,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -483,6 +486,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md
index ca9cb043e9..778e4d3a62 100644
--- a/tensorflow/docs_src/mobile/optimizing.md
+++ b/tensorflow/docs_src/mobile/optimizing.md
@@ -233,6 +233,8 @@ order by how long they took. From left to right, the columns are:
 - The cumulative total time of this and the previous ops in the table. This is
   handy for understanding what the distribution of work is across the layers, to
   see if just a few of the nodes are taking up most of the time.
+  
+- The amount of memory consumed by outputs of this type of op.
 
 - Name of the node.
 
diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md
index 360ee302aa..8b22c04d87 100644
--- a/tensorflow/docs_src/mobile/prepare_models.md
+++ b/tensorflow/docs_src/mobile/prepare_models.md
@@ -60,7 +60,7 @@ and serialized as protocol buffers:
   the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
   then we only need a single `GraphDef` file to hold the model architecture and
   the weights. Freezing the graph handles the process of loading the
-  checkpoints, and then converts all Consts to Variables. You can then load the
+  checkpoints, and then converts all Variables to Consts. You can then load the
   resulting file in a single call, without having to restore variable values
   from checkpoints. One thing to watch out for with `GraphDef` files is that
   sometimes they’re stored in text format for easy inspection. These versions
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4f61c01f65..a0dd409205 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1065,7 +1065,7 @@ py_test(
 
 py_test(
     name = "framework_importer_test",
-    size = "medium",
+    size = "large",
     srcs = ["framework/importer_test.py"],
     main = "framework/importer_test.py",
     srcs_version = "PY2AND3",
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index d0ba8020c1..64c1760d5e 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -315,21 +315,39 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
             self.assertAllEqual(x_tf_4, np.asarray(x_np)[:, ::-1])
             self.assertAllEqual(x_tf_5, np.asarray(x_np)[::-1, ::-1])
 
+  # This test covers the axis validation in the shape function
+  # (no eval())
+  def testInvalidAxis(self):
+    x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [-30])
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [2])
+    with self.assertRaisesRegexp(ValueError,
+                                 "axis 0 specified more than once"):
+      array_ops.reverse_v2(x_np, [0, -2])
+
   # This is the version of reverse that uses axis indices rather than
   # bool tensors
   # TODO(b/32254538): Change this test to use array_ops.reverse
+  #
+  # Note: this test passes placeholder as constant axis is validated
+  # in shape function (see testInvalidAxis)
   def testInvalid(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    axis = array_ops.placeholder(dtypes.int32)
     with self.test_session():
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [-30]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [-30]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [2]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "axis 0 specified more than once"):
-        array_ops.reverse_v2(x_np, [0, -2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]})
 
   def testReverse1DimAuto(self):
     for dtype in [
@@ -890,7 +908,7 @@ class StridedSliceAssignChecker(object):
         var = resource_variable_ops.ResourceVariable(self.x)
       else:
         var = variables.Variable(self.x)
-      sess.run(variables.initialize_variables([var]))
+      sess.run(variables.variables_initializer([var]))
       val = sess.run(var[index].assign(value))
       # val_copy is used to check that tf.assign works equivalently to the
       # assign method above.
diff --git a/tensorflow/python/kernel_tests/testdata/BUILD b/tensorflow/python/kernel_tests/testdata/BUILD
index a4a0dfc139..45264c773a 100644
--- a/tensorflow/python/kernel_tests/testdata/BUILD
+++ b/tensorflow/python/kernel_tests/testdata/BUILD
@@ -1,7 +1,7 @@
 # Data files for kernel tests.
 
 package(
-    default_visibility = ["//tensorflow:internal"],
+    default_visibility = ["//visibility:public"],
 )
 
 licenses(["notice"])  # Apache 2.0
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index e3e120a4eb..60c726d54c 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -18,10 +18,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import itertools
+import sys
+
 import numpy as np
 
+from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -88,7 +94,7 @@ class XentTest(test.TestCase):
                                                     4.]]]).astype(dtype)
       np_labels = np.array([[[0., 0., 0., 1.]], [[0., .5, .5,
                                                   0.]]]).astype(dtype)
-      self.assertRaisesRegexp(ValueError, "must be rank 2",
+      self.assertRaisesRegexp(ValueError, "rank 2, but is rank 3",
                               gen_nn_ops.softmax_cross_entropy_with_logits,
                               np_features, np_labels)
 
@@ -128,6 +134,24 @@ class XentTest(test.TestCase):
     self.assertAllClose(
         np.array([1.3862, 1.9401]), np_loss, rtol=1.e-3, atol=1.e-3)
 
+  def testShapeBroadcast(self):
+    np_f = np.array([[1., 2., 3., 4.],
+                     [1., 2., 3., 4.]]).astype(np.float32)
+    np_l = np.array([[0., 0., 0., 1.],
+                     [0., .5, .5, 0.]]).astype(np.float32)
+    np_loss, np_backprop = self._npXent(np_f, np_l)
+    tf_f = constant_op.constant(
+        np.array([[1., 2., 3., 4.]]).astype(np.float32))
+    tf_l = constant_op.constant(
+        np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32))
+    for use_gpu in [False, True]:
+      with self.test_session(use_gpu=use_gpu) as sess:
+        loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
+            tf_f, tf_l)
+        tf_loss, tf_backprop = sess.run([loss, backprop])
+      self.assertAllCloseAccordingToType(np_loss, tf_loss)
+      self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
+
   def testShapeMismatch(self):
     with self.test_session():
       with self.assertRaises(ValueError):
@@ -260,5 +284,60 @@ class XentTest(test.TestCase):
     self.assertAllEqual(np_loss, tf_loss)
 
 
+class XentBenchmark(test.Benchmark):
+
+  def benchmarkZeroDimension(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "zero_dimension_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          logits = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+  def benchmarkSingleClass(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "single_class_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = constant_op.constant([[1.], [-1.], [0.]],
+                                        dtype=dtypes.float32)
+          logits = constant_op.constant([[-1.], [0.], [1.]],
+                                        dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 74e7c63fb3..2d99b1688f 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -180,6 +180,8 @@ class _Conv(base.Layer):
           # bias_add when computing gradients. To use bias_add, we collapse Z
           # and Y into a single dimension to obtain a 4D input tensor.
           outputs_shape = outputs.shape.as_list()
+          if outputs_shape[0] is None:
+            outputs_shape[0] = -1
           outputs_4d = array_ops.reshape(outputs,
                                          [outputs_shape[0], outputs_shape[1],
                                           outputs_shape[2] * outputs_shape[3],
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index 160e732b67..cdb42f5bd1 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -325,6 +325,12 @@ class ConvTest(test.TestCase):
     self.assertEqual(conv3d.kernel_constraint, k_constraint)
     self.assertEqual(conv3d.bias_constraint, b_constraint)
 
+  def testConv3DChannelsFirst(self):
+    # Test case for GitHub issue 15655
+    images = array_ops.placeholder(
+        dtype=dtypes.float32, shape=[None, 1, 32, 32, 32])
+    conv_layers.conv3d(images, 32, 9, data_format='channels_first')
+
 
 @test_util.with_c_api
 class SeparableConv1DTest(test.TestCase):
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 5b4fb4f7c8..170861b43f 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -429,7 +429,7 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
   u, s, v_adj = np.linalg.svd(a, full_matrices=False)
   np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj))
   # tf_a_approx and np_a_approx should be numerically close.
-  ````
+  ```
   @end_compatibility
   """
   s, u, v = gen_linalg_ops.svd(
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 6c5c9e01a7..4ce6f6d002 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -281,13 +281,14 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
                              scaffold=None,
                              hooks=None,
                              chief_only_hooks=None,
-                             save_checkpoint_secs=600,
+                             save_checkpoint_secs=USE_DEFAULT,
                              save_summaries_steps=USE_DEFAULT,
                              save_summaries_secs=USE_DEFAULT,
                              config=None,
                              stop_grace_period_secs=120,
                              log_step_count_steps=100,
-                             max_wait_secs=7200):
+                             max_wait_secs=7200,
+                             save_checkpoint_steps=USE_DEFAULT):
   """Creates a `MonitoredSession` for training.
 
   For a chief, this utility sets proper session initializer/restorer. It also
@@ -310,8 +311,10 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
     chief_only_hooks: list of `SessionRunHook` objects. Activate these hooks if
       `is_chief==True`, ignore otherwise.
     save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved
-      using a default checkpoint saver. If `save_checkpoint_secs` is set to
-      `None`, then the default checkpoint saver isn't used.
+      using a default checkpoint saver. If both `save_checkpoint_steps` and
+      `save_checkpoint_secs` are set to `None`, then the default checkpoint
+      saver isn't used. If both are provided, then only `save_checkpoint_secs`
+      is used. Default 600.
     save_summaries_steps: The frequency, in number of global steps, that the
       summaries are written to disk using a default summary saver. If both
       `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
@@ -330,6 +333,11 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       become available. This should be kept relatively short to help detect
       incorrect code, but sometimes may need to be increased if the chief takes
       a while to start up.
+    save_checkpoint_steps: The frequency, in number of global steps, that a
+      checkpoint is saved using a default checkpoint saver. If both
+      `save_checkpoint_steps` and `save_checkpoint_secs` are set to `None`, then
+      the default checkpoint saver isn't used. If both are provided, then only
+      `save_checkpoint_secs` is used. Default not enabled.
 
   Returns:
     A `MonitoredSession` object.
@@ -342,6 +350,15 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
   elif save_summaries_steps == USE_DEFAULT:
     save_summaries_steps = None
 
+  if (save_checkpoint_steps == USE_DEFAULT and
+      save_checkpoint_secs == USE_DEFAULT):
+    save_checkpoint_steps = None
+    save_checkpoint_secs = 600
+  elif save_checkpoint_secs == USE_DEFAULT:
+    save_checkpoint_secs = None
+  elif save_checkpoint_steps == USE_DEFAULT:
+    save_checkpoint_steps = None
+
   scaffold = scaffold or Scaffold()
   if not is_chief:
     session_creator = WorkerSessionCreator(
@@ -374,9 +391,13 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
           save_steps=save_summaries_steps,
           save_secs=save_summaries_secs,
           output_dir=checkpoint_dir))
-    if save_checkpoint_secs and save_checkpoint_secs > 0:
+    if (save_checkpoint_secs and save_checkpoint_secs > 0) or (
+        save_checkpoint_steps and save_checkpoint_steps > 0):
       all_hooks.append(basic_session_run_hooks.CheckpointSaverHook(
-          checkpoint_dir, save_secs=save_checkpoint_secs, scaffold=scaffold))
+          checkpoint_dir,
+          save_steps=save_checkpoint_steps,
+          save_secs=save_checkpoint_secs,
+          scaffold=scaffold))
 
   if hooks:
     all_hooks.extend(hooks)
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 159b2d5c16..3806056f01 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -282,6 +282,42 @@ class MonitoredTrainingSessionTest(test.TestCase):
           is_chief=True, checkpoint_dir=logdir) as session:
         self.assertEqual(2, session.run(gstep))
 
+  def test_save_checkpoint_steps(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_steps')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_steps=100,
+          log_step_count_steps=10) as session:
+        for _ in range(100):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(100, session.run(gstep))
+
+  def test_save_checkpoint_secs(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_secs')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_secs=0.1,
+          log_step_count_steps=10) as session:
+        session.run(new_gstep)
+        time.sleep(0.2)
+        for _ in range(10):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(11, session.run(gstep))
+
   def test_summaries_steps(self):
     logdir = _test_dir(self.get_temp_dir(), 'test_summaries_steps')
     with ops.Graph().as_default():
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 2d3cb415fe..fcc57d506e 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -22,6 +22,7 @@ load(
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
+    "if_mkl_lnx_x64"
 )
 
 def register_extension_info(**kwargs):
@@ -202,7 +203,8 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False):
           "-ftemplate-depth=900"])
       + if_cuda(["-DGOOGLE_CUDA=1"])
       + if_tensorrt(["-DGOOGLE_TENSORRT=1"])
-      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML", "-fopenmp",])
+      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"])
+      + if_mkl_lnx_x64(["-fopenmp"])
       + if_android_arm(["-mfpu=neon"])
       + if_linux_x86_64(["-msse3"])
       + if_ios_x86_64(["-msse4.1"])
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index c75ee474aa..bec72e1e60 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -238,7 +238,7 @@ tf_module {
   }
   member_method {
     name: "MonitoredTrainingSession"
-    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'600\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\'], "
+    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\', \'save_checkpoint_steps\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'<object object instance>\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\', \'<object object instance>\'], "
   }
   member_method {
     name: "NewCheckpointReader"
diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
index e1b56b9a25..7d471b4703 100755
--- a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
+++ b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
@@ -31,5 +31,5 @@ export TF_NEED_OPENCL_SYCL=0
 export TF_NEED_MKL=0
 export COMPUTECPP_PATH="/usr/local"
 
-export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+export PATH="$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
 build_libtensorflow_tarball "-cpu-darwin-$(uname -m)"
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 22c73c3fe1..11f476d12c 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -70,7 +70,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
 # more difficult to experiment with local changes. Instead, just add
diff --git a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
index 3690e7dfe5..037d13116e 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
@@ -3,7 +3,7 @@ FROM tensorflow/tensorflow:latest-devel
 LABEL maintainer="Clayne Robison<clayne.b.robison@intel.com>"
 
 # These arguments are parameterized. Use --build-args to override.
-ARG TF_BRANCH=r1.6
+ARG TF_BRANCH=r1.7
 ARG WHL_DIR=/whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 69ba340f92..1fcb6428b2 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -79,7 +79,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 3fbdb5cacd..0ede8c6370 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -138,7 +138,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/c/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
@@ -176,7 +175,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/jni/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index dd75eda231..62fec2c402 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -127,7 +127,6 @@ filegroup(
         "@org_python_pypi_backports_weakref//:LICENSE",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]) + if_not_windows([
         "@nccl_archive//:LICENSE.txt",
     ]) + tf_additional_license_deps(),
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e0152da4df..365e8d6b08 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0'
+_VERSION = '1.7.0-rc1'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -39,7 +39,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.13.3',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorboard >= 1.6.0, < 1.7.0',
+    'tensorboard >= 1.7.0, < 1.8.0',
     'termcolor >= 1.1.0',
 ]
 
@@ -62,7 +62,7 @@ else:
 if 'tf_nightly' in project_name:
   for i, pkg in enumerate(REQUIRED_PACKAGES):
     if 'tensorboard' in pkg:
-      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.7.0a0, < 1.8.0a0'
+      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.8.0a0, < 1.9.0a0'
       break
 
 # weakref.finalize and enum were introduced in Python 3.4
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 9fcbfb664b..5f6e717532 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -15,6 +15,11 @@ load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
 
 
+# Sanitize a dependency so that it works correctly from code that includes
+# TensorFlow as a submodule.
+def clean_dep(dep):
+  return str(Label(dep))
+
 # If TensorFlow is linked as a submodule.
 # path_prefix is no longer used.
 # tf_repo_name is thought to be under consideration.
@@ -32,17 +37,37 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   arm_compiler_configure(
       name="local_config_arm_compiler",
       remote_config_repo="../arm_compiler",
-      build_file = str(Label("//third_party/toolchains/cpus/arm:BUILD")))
+      build_file = clean_dep("//third_party/toolchains/cpus/arm:BUILD"))
 
   mkl_repository(
-      name = "mkl",
+      name = "mkl_linux",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+      ],
+      sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f",
+      strip_prefix = "mklml_lnx_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_windows",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip"
+      ],
+      sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4",
+      strip_prefix = "mklml_win_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_darwin",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
-          "https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz"
       ],
-      sha256 = "6b07cb7e5451db67c2e31e785ae458b18f7f363c60a61685488f69e9ae7199d4",
-      strip_prefix = "mklml_lnx_2018.0.1.20171007",
-      build_file = str(Label("//third_party/mkl:mkl.BUILD")),
+      sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f",
+      strip_prefix = "mklml_mac_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
   )
 
   if path_prefix:
@@ -52,12 +77,12 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "mkl_dnn",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
-          "https://github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
+          "https://github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
       ],
-      sha256 = "02e244f63dd95402691a361392504c143eede9a89043426f174836638a9cbf09",
-      strip_prefix = "mkl-dnn-e0bfcaa7fcb2b1e1558f5f0676933c1db807a729",
-      build_file = str(Label("//third_party/mkl_dnn:mkldnn.BUILD")),
+      sha256 = "86fa2a8c12a56e3b725945acedeaa82492746be02545aba6d710f097e013e19e",
+      strip_prefix = "mkl-dnn-0.12",
+      build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"),
   )
 
   tf_http_archive(
@@ -68,7 +93,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
      sha256 = "5996380e3e8b981f55d1c8d58e709c00dbb4806ba367be75d0925a68cc2f6478",
      strip_prefix = "abseil-cpp-720c017e30339fd1786ce4aac68bc8559736e53f",
-     build_file = str(Label("//third_party:com_google_absl.BUILD")),
+     build_file = clean_dep("//third_party:com_google_absl.BUILD"),
   )
 
   tf_http_archive(
@@ -79,8 +104,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0cadb31a35b514bf2dfd6b5d38205da94ef326ec6908fc3fd7c269948467214f",
       strip_prefix = "eigen-eigen-2355b229ea4c",
-      build_file = str(Label("//third_party:eigen.BUILD")),
-      patch_file = str(Label("//third_party:eigen_fix_cuda_compilation.patch"))
+      build_file = clean_dep("//third_party:eigen.BUILD"),
+      patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch")
   )
 
   tf_http_archive(
@@ -93,7 +118,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           # remove the whitelist entry in third_party/repo.bzl.
           # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
       ],
-      build_file = str(Label("//:arm_compiler.BUILD")),
+      build_file = clean_dep("//:arm_compiler.BUILD"),
   )
 
   tf_http_archive(
@@ -104,7 +129,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce",
       strip_prefix = "libxsmm-1.8.1",
-      build_file = str(Label("//third_party:libxsmm.BUILD")),
+      build_file = clean_dep("//third_party:libxsmm.BUILD"),
   )
 
   tf_http_archive(
@@ -117,7 +142,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
       strip_prefix = "or-tools-253f7955c6a1fd805408fba2e42ac6d45b312d15/src",
-      build_file = str(Label("//third_party:ortools.BUILD")),
+      build_file = clean_dep("//third_party:ortools.BUILD"),
   )
 
   tf_http_archive(
@@ -149,7 +174,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0",
       strip_prefix = "farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45",
-      build_file = str(Label("//third_party:farmhash.BUILD")),
+      build_file = clean_dep("//third_party:farmhash.BUILD"),
   )
 
   tf_http_archive(
@@ -160,7 +185,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9",
       strip_prefix = "highwayhash-dfcb97ca4fe9277bf9dc1802dd979b071896453b",
-      build_file = str(Label("//third_party:highwayhash.BUILD")),
+      build_file = clean_dep("//third_party:highwayhash.BUILD"),
   )
 
   tf_http_archive(
@@ -171,7 +196,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324",
       strip_prefix = "nasm-2.12.02",
-      build_file = str(Label("//third_party:nasm.BUILD")),
+      build_file = clean_dep("//third_party:nasm.BUILD"),
   )
 
   tf_http_archive(
@@ -182,7 +207,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77",
       strip_prefix = "libjpeg-turbo-1.5.1",
-      build_file = str(Label("//third_party/jpeg:jpeg.BUILD")),
+      build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"),
   )
 
   tf_http_archive(
@@ -193,7 +218,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2",
       strip_prefix = "libpng-1.2.53",
-      build_file = str(Label("//third_party:png.BUILD")),
+      build_file = clean_dep("//third_party:png.BUILD"),
   )
 
   tf_http_archive(
@@ -204,7 +229,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
       strip_prefix = "sqlite-amalgamation-3200000",
-      build_file = str(Label("//third_party:sqlite.BUILD")),
+      build_file = clean_dep("//third_party:sqlite.BUILD"),
   )
 
   tf_http_archive(
@@ -215,7 +240,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1",
       strip_prefix = "giflib-5.1.4",
-      build_file = str(Label("//third_party:gif.BUILD")),
+      build_file = clean_dep("//third_party:gif.BUILD"),
   )
 
   tf_http_archive(
@@ -226,7 +251,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
       strip_prefix = "six-1.10.0",
-      build_file = str(Label("//third_party:six.BUILD")),
+      build_file = clean_dep("//third_party:six.BUILD"),
   )
 
   tf_http_archive(
@@ -237,7 +262,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d",
       strip_prefix = "astor-0.6.2",
-      build_file = str(Label("//third_party:astor.BUILD")),
+      build_file = clean_dep("//third_party:astor.BUILD"),
   )
 
   tf_http_archive(
@@ -248,7 +273,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "7068908321ecd2774f145193c4b34a11305bd104b4551b09273dfd1d6a374930",
       strip_prefix = "gast-0.2.0",
-      build_file = str(Label("//third_party:gast.BUILD")),
+      build_file = clean_dep("//third_party:gast.BUILD"),
   )
 
   tf_http_archive(
@@ -259,7 +284,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b",
       strip_prefix = "termcolor-1.1.0",
-      build_file = str(Label("//third_party:termcolor.BUILD")),
+      build_file = clean_dep("//third_party:termcolor.BUILD"),
   )
 
   tf_http_archive(
@@ -280,7 +305,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892",
       strip_prefix = "backports.weakref-1.0rc1/src",
-      build_file = str(Label("//third_party:backports_weakref.BUILD")),
+      build_file = clean_dep("//third_party:backports_weakref.BUILD"),
   )
 
   tf_http_archive(
@@ -291,7 +316,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
       strip_prefix = "codegen-1.0",
-      build_file = str(Label("//third_party:codegen.BUILD")),
+      build_file = clean_dep("//third_party:codegen.BUILD"),
   )
 
   filegroup_external(
@@ -376,7 +401,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
       ],
       strip_prefix = "pcre-8.39",
-      build_file = str(Label("//third_party:pcre.BUILD")),
+      build_file = clean_dep("//third_party:pcre.BUILD"),
   )
 
   tf_http_archive(
@@ -388,7 +413,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
       ],
       strip_prefix = "swig-3.0.8",
-      build_file = str(Label("//third_party:swig.BUILD")),
+      build_file = clean_dep("//third_party:swig.BUILD"),
   )
 
   tf_http_archive(
@@ -399,7 +424,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://curl.haxx.se/download/curl-7.49.1.tar.gz",
       ],
       strip_prefix = "curl-7.49.1",
-      build_file = str(Label("//third_party:curl.BUILD")),
+      build_file = clean_dep("//third_party:curl.BUILD"),
   )
 
   tf_http_archive(
@@ -421,7 +446,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
       ],
       strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3",
-      build_file = str(Label("//third_party:linenoise.BUILD")),
+      build_file = clean_dep("//third_party:linenoise.BUILD"),
   )
 
   # TODO(phawkins): currently, this rule uses an unofficial LLVM mirror.
@@ -434,7 +459,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
       strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
-      build_file = str(Label("//third_party/llvm:llvm.BUILD")),
+      build_file = clean_dep("//third_party/llvm:llvm.BUILD"),
   )
 
   tf_http_archive(
@@ -445,7 +470,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326",
       strip_prefix = "lmdb-LMDB_0.9.19/libraries/liblmdb",
-      build_file = str(Label("//third_party:lmdb.BUILD")),
+      build_file = clean_dep("//third_party:lmdb.BUILD"),
   )
 
   tf_http_archive(
@@ -456,7 +481,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2",
       strip_prefix = "jsoncpp-11086dd6a7eba04289944367ca82cea71299ed70",
-      build_file = str(Label("//third_party:jsoncpp.BUILD")),
+      build_file = clean_dep("//third_party:jsoncpp.BUILD"),
   )
 
   tf_http_archive(
@@ -477,7 +502,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
       strip_prefix = "zlib-1.2.8",
-      build_file = str(Label("//third_party:zlib.BUILD")),
+      build_file = clean_dep("//third_party:zlib.BUILD"),
   )
 
   tf_http_archive(
@@ -487,7 +512,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
       ],
       sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296",
-      build_file = str(Label("//third_party/fft2d:fft2d.BUILD")),
+      build_file = clean_dep("//third_party/fft2d:fft2d.BUILD"),
   )
 
   tf_http_archive(
@@ -498,7 +523,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
       strip_prefix = "snappy-1.1.4",
-      build_file = str(Label("//third_party:snappy.BUILD")),
+      build_file = clean_dep("//third_party:snappy.BUILD"),
   )
 
   tf_http_archive(
@@ -509,7 +534,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
       strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7",
-      build_file = str(Label("//third_party:nccl.BUILD")),
+      build_file = clean_dep("//third_party:nccl.BUILD"),
   )
 
   tf_http_archive(
@@ -520,8 +545,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "dd035d57c8f19b0b612dd6eefe6e5eebad76f506e302cccb7c2066f25a83585e",
       strip_prefix = "librdkafka-0.11.1",
-      build_file = str(Label("//third_party:kafka/BUILD")),
-      patch_file = str(Label("//third_party/kafka:config.patch")),
+      build_file = clean_dep("//third_party:kafka/BUILD"),
+      patch_file = clean_dep("//third_party/kafka:config.patch"),
   )
 
   tf_http_archive(
@@ -532,7 +557,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
       strip_prefix = "aws-sdk-cpp-1.3.15",
-      build_file = str(Label("//third_party:aws.BUILD")),
+      build_file = clean_dep("//third_party:aws.BUILD"),
   )
 
   java_import_external(
@@ -568,7 +593,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8",
       strip_prefix = "jemalloc-4.4.0",
-      build_file = str(Label("//third_party:jemalloc.BUILD")),
+      build_file = clean_dep("//third_party:jemalloc.BUILD"),
   )
 
   java_import_external(
@@ -613,7 +638,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4",
       strip_prefix = "pprof-c0fb62ec88c411cc91194465e54db2632845b650",
-      build_file = str(Label("//third_party:pprof.BUILD")),
+      build_file = clean_dep("//third_party:pprof.BUILD"),
   )
 
   tf_http_archive(
@@ -624,7 +649,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
       strip_prefix = "cub-1.8.0",
-      build_file = str(Label("//third_party:cub.BUILD")),
+      build_file = clean_dep("//third_party:cub.BUILD"),
   )
 
   tf_http_archive(
@@ -635,7 +660,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
       ],
       strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
-      build_file = str(Label("//third_party:cython.BUILD")),
+      build_file = clean_dep("//third_party:cython.BUILD"),
       delete = ["BUILD.bazel"],
   )
 
@@ -657,7 +682,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
           "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
       ],
-      build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
+      build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
   )
 
   tf_http_archive(
@@ -668,7 +693,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
           "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
       ],
-      build_file = str(Label("//third_party/flatbuffers:flatbuffers.BUILD")),
+      build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
   )
 
   tf_http_archive(
@@ -678,7 +703,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
       ],
-      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
+      build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
   )
 
   tf_http_archive(
@@ -688,7 +713,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip"
       ],
-      build_file = str(Label("//third_party:tflite_smartreply.BUILD")),
+      build_file = clean_dep("//third_party:tflite_smartreply.BUILD"),
   )
 
   ##############################################################################
@@ -752,7 +777,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # Needed by Protobuf
   native.bind(
       name = "python_headers",
-      actual = str(Label("//util/python:python_headers")),
+      actual = clean_dep("//util/python:python_headers"),
   )
 
   # Needed by Protobuf
diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD
index b27d341404..c2adf578c7 100644
--- a/third_party/mkl/BUILD
+++ b/third_party/mkl/BUILD
@@ -1,7 +1,5 @@
 licenses(["notice"])  # 3-Clause BSD
 
-exports_files(["LICENSE"])
-
 config_setting(
     name = "using_mkl",
     values = {
@@ -10,17 +8,51 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "using_mkl_lnx_x64",
+    values = {
+        "cpu": "k8",
+        "define": "using_mkl=true",
+    },
+    visibility = ["//visibility:public"],
+)
+
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
 )
 
+filegroup(
+    name = "LICENSE",
+    srcs = ["MKL_LICENSE"] + select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:LICENSE",
+        ],
+    }),
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "intel_binary_blob",
-    srcs = if_mkl([
-        "@mkl//:libmklml_intel.so",
-        "@mkl//:libiomp5.so",
-    ]),
     visibility = ["//visibility:public"],
-    deps = ["@mkl//:mkl_headers"],
+    deps = select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:mkl_headers",
+            "@mkl_linux//:mkl_libs_linux",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:mkl_headers",
+            "@mkl_darwin//:mkl_libs_darwin",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:mkl_headers",
+            "@mkl_windows//:mkl_libs_windows",
+        ],
+    }),
 )
diff --git a/third_party/mkl/MKL_LICENSE b/third_party/mkl/MKL_LICENSE
new file mode 100644
index 0000000000..9c8f3ea087
--- /dev/null
+++ b/third_party/mkl/MKL_LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file
diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl
index 8b73ddabdd..53e02769da 100644
--- a/third_party/mkl/build_defs.bzl
+++ b/third_party/mkl/build_defs.bzl
@@ -24,6 +24,18 @@ def if_mkl(if_true, if_false = []):
         "//conditions:default": if_false
     })
 
+def if_mkl_lnx_x64(if_true, if_false = []):
+    """Shorthand for select()'ing on whether we're building with MKL.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with MKL enabled.  Otherwise, the select statement evaluates to if_false.
+
+    """
+    return select({
+        str(Label("//third_party/mkl:using_mkl_lnx_x64")): if_true,
+        "//conditions:default": if_false
+    })
+
 
 def _enable_local_mkl(repository_ctx):
   return _TF_MKL_ROOT in repository_ctx.os.environ
diff --git a/third_party/mkl/mkl.BUILD b/third_party/mkl/mkl.BUILD
index 8db97232e1..c3a71e4ff9 100644
--- a/third_party/mkl/mkl.BUILD
+++ b/third_party/mkl/mkl.BUILD
@@ -17,14 +17,29 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libmklml_intel.so",
-    srcs = ["lib/libmklml_intel.so"],
+cc_library(
+    name = "mkl_libs_linux",
+    srcs = [
+        "lib/libiomp5.so",
+        "lib/libmklml_intel.so",
+    ],
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libiomp5.so",
-    srcs = ["lib/libiomp5.so"],
+cc_library(
+    name = "mkl_libs_darwin",
+    srcs = [
+        "lib/libiomp5.dylib",
+        "lib/libmklml.dylib",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "mkl_libs_windows",
+    srcs = [
+        "lib/libiomp5md.lib",
+        "lib/mklml.lib",
+    ],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From 828ebed1fe252339769ddc0acde83a55219b38c0 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 28 Mar 2018 14:42:57 -0700
Subject: [PATCH 637/960] Internal change.

PiperOrigin-RevId: 190836675
---
 tensorflow/python/kernel_tests/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 228d1c2452..05f34db14b 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -1569,7 +1569,7 @@ cuda_py_test(
 
 cuda_py_test(
     name = "init_ops_test",
-    size = "small",
+    size = "medium",
     srcs = ["init_ops_test.py"],
     additional_deps = [
         "//third_party/py/numpy",
-- 
GitLab


From 355c88503a3a998aef3c1dc51045409778afd578 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 14:47:00 -0700
Subject: [PATCH 638/960] Use high precision to compute
 softmax_cross_entropy_with_logits.

PiperOrigin-RevId: 190837379
---
 tensorflow/core/kernels/cwise_op_log.cc |  4 +-
 tensorflow/python/ops/nn_ops.py         |  8 ++--
 tensorflow/python/ops/nn_test.py        | 51 +++++++++++++++++++++++++
 3 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_op_log.cc b/tensorflow/core/kernels/cwise_op_log.cc
index 98936e0f96..5d17c890cf 100644
--- a/tensorflow/core/kernels/cwise_op_log.cc
+++ b/tensorflow/core/kernels/cwise_op_log.cc
@@ -16,8 +16,8 @@ limitations under the License.
 #include "tensorflow/core/kernels/cwise_ops_common.h"
 
 namespace tensorflow {
-REGISTER5(UnaryOp, CPU, "Log", functor::log, float, Eigen::half, double,
-          complex64, complex128);
+REGISTER6(UnaryOp, CPU, "Log", functor::log, float, Eigen::half, double,
+          bfloat16, complex64, complex128);
 
 #if GOOGLE_CUDA
 REGISTER3(UnaryOp, GPU, "Log", functor::log, float, Eigen::half, double);
diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index a74de39eab..0c55386241 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1836,8 +1836,10 @@ def softmax_cross_entropy_with_logits_v2(
                       [logits, labels]) as name:
     logits = ops.convert_to_tensor(logits, name="logits")
     labels = ops.convert_to_tensor(labels, name="labels")
+    convert_to_float32 = (
+        logits.dtype == dtypes.float16 or logits.dtype == dtypes.bfloat16)
     precise_logits = math_ops.cast(
-        logits, dtypes.float32) if (logits.dtype == dtypes.float16) else logits
+        logits, dtypes.float32) if convert_to_float32 else logits
     # labels and logits must be of the same type
     labels = math_ops.cast(labels, precise_logits.dtype)
     input_rank = array_ops.rank(precise_logits)
@@ -1883,8 +1885,8 @@ def softmax_cross_entropy_with_logits_v2(
       del shape[dim]
       cost.set_shape(shape)
 
-    if logits.dtype == dtypes.float16:
-      return math_ops.cast(cost, dtypes.float16)
+    if convert_to_float32:
+      return math_ops.cast(cost, logits.dtype)
     else:
       return cost
 
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index af9dae2aa6..da86d5f6ca 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -852,6 +852,57 @@ class ComputeSampledLogitsTest(test_lib.TestCase):
       self.assertAllClose(exp_sampled_softmax_loss,
                           got_sampled_softmax_loss.eval(), 1e-4)
 
+  def testSampledSoftmaxLossBf16(self):
+    # A simple test to verify the numerics for bfloat16.
+    def _SoftmaxCrossEntropyWithLogits(logits, targets):
+      # logits, targets: float arrays of the same shape.
+      assert logits.shape == targets.shape
+      stable_exp_logits = np.exp(
+          logits - np.amax(logits, axis=1, keepdims=True))
+      pred = stable_exp_logits / np.sum(stable_exp_logits, 1, keepdims=True)
+      return -np.sum(targets * np.log(pred + 1.0e-20), axis=1)
+
+    np.random.seed(0)
+    num_classes = 5
+    batch_size = 3
+    labels = [0, 1, 2]
+    sampled = [1, 0, 2, 3]
+    (weights, biases, hidden_acts, _, exp_logits,
+     exp_labels) = self._GenerateTestData(
+         num_classes=num_classes,
+         dim=10,
+         batch_size=batch_size,
+         num_true=1,
+         labels=labels,
+         sampled=sampled,
+         subtract_log_q=True)
+    exp_sampled_softmax_loss = _SoftmaxCrossEntropyWithLogits(
+        exp_logits, exp_labels)
+
+    with self.test_session():
+      true_exp_bf16 = np.full(
+          [batch_size, 1], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype)
+      sampled_exp_bf16 = np.full(
+          [len(sampled)], fill_value=0.5, dtype=dtypes.bfloat16.as_numpy_dtype)
+      sampled_vals_bf16 = (sampled, true_exp_bf16, sampled_exp_bf16)
+
+      got_sampled_softmax_loss = math_ops.cast(
+          nn_impl.sampled_softmax_loss(
+              weights=constant_op.constant(weights, dtype=dtypes.bfloat16),
+              biases=constant_op.constant(biases, dtype=dtypes.bfloat16),
+              labels=constant_op.constant(
+                  labels, shape=(batch_size, 1), dtype=dtypes.bfloat16),
+              inputs=constant_op.constant(hidden_acts, dtype=dtypes.bfloat16),
+              num_sampled=4,
+              num_classes=num_classes,
+              num_true=1,
+              sampled_values=sampled_vals_bf16,
+              remove_accidental_hits=False,
+              partition_strategy="div"), dtypes.float32)
+
+      self.assertAllClose(exp_sampled_softmax_loss,
+                          got_sampled_softmax_loss.eval(), 1e-1)
+
 
 class CReluTest(test_lib.TestCase):
 
-- 
GitLab


From 9e6f84b6c8f1d052272d75bcde186b7f1012df48 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 14:48:50 -0700
Subject: [PATCH 639/960] Internal change

PiperOrigin-RevId: 190837707
---
 tensorflow/core/BUILD                         | 29 ++++++++++++++-----
 .../core/platform/default/build_config/BUILD  |  5 ++++
 2 files changed, 27 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 1d11410332..4726946277 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -379,13 +379,13 @@ cc_library(
 )
 
 cc_library(
-    name = "session_message",
-    srcs = ["util/session_message.cc"],
-    hdrs = ["util/session_message.h"],
+    name = "stacktrace",
+    srcs = glob(["platform/*/stacktrace.h"]),
+    hdrs = ["platform/stacktrace.h"],
     deps = [
-        ":framework",
-        ":lib",
-        ":protos_all_cc",
+        ":abi",
+        ":lib_platform",
+        "//tensorflow/core/platform/default/build_config:stacktrace",
     ],
 )
 
@@ -394,8 +394,20 @@ cc_library(
     srcs = ["platform/stacktrace_handler.cc"],
     hdrs = ["platform/stacktrace_handler.h"],
     deps = [
-        ":lib",
+        ":abi",
         ":lib_platform",
+        ":stacktrace",
+    ],
+)
+
+cc_library(
+    name = "session_message",
+    srcs = ["util/session_message.cc"],
+    hdrs = ["util/session_message.h"],
+    deps = [
+        ":framework",
+        ":lib",
+        ":protos_all_cc",
     ],
 )
 
@@ -1624,6 +1636,7 @@ cc_library(
             "platform/**/env_time.cc",
             "platform/**/cuda_libdevice_path.cc",
             "platform/**/device_tracer.cc",
+            "platform/abi.cc",
             "platform/variant_coding.cc",
             "platform/**/variant_cord_coding.cc",
         ],
@@ -1635,6 +1648,7 @@ cc_library(
             "platform/**/stream_executor.h",
             "platform/**/env_time.cc",
             "platform/**/device_tracer.cc",
+            "platform/abi.cc",
             "platform/variant_coding.cc",
             "platform/**/variant_cord_coding.cc",
         ] +
@@ -1648,6 +1662,7 @@ cc_library(
     deps = tf_additional_lib_deps() + [
         ":lib_hash_crc32c_accelerate_internal",
         ":lib_proto_parsing",
+        ":abi",
         "//third_party/eigen3",
         "//tensorflow/core/platform/default/build_config:platformlib",
         "@snappy",
diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index 2cd607edbe..afb1d84d14 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -128,6 +128,11 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "stacktrace",
+    srcs = [],
+)
+
 cc_library(
     name = "gif",
     copts = tf_copts(),
-- 
GitLab


From 15908d912ed26f2517207e0a0bea6cd5768476ee Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 14:52:25 -0700
Subject: [PATCH 640/960] Add DistributionStrategy support to Optimizer.

PiperOrigin-RevId: 190838314
---
 tensorflow/python/training/optimizer.py | 174 +++++++++++++++++++++++-
 1 file changed, 172 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/training/optimizer.py b/tensorflow/python/training/optimizer.py
index bf79714f96..75665fc284 100644
--- a/tensorflow/python/training/optimizer.py
+++ b/tensorflow/python/training/optimizer.py
@@ -35,11 +35,28 @@ from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.training import checkpointable
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import slot_creator
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 
+def get_filtered_grad_fn(grad_fn):
+  # `distributed_context.join()` requires that its arguments are parallel
+  # across threads, and in particular that `grads_and_vars` has the same
+  # variables in the same order.
+
+  # When computing gradients in eager mode with multiple threads, you
+  # can get extra variables with a gradient of `None`. This happens when
+  # those variables are accessed in another thread during the gradient
+  # computation. To get a consistent set of variables, we filter out
+  # those with `None` gradients.
+  def filtered_grad_fn(x=None):
+    return [(g, v) for g, v in grad_fn(x) if g is not None]
+
+  return filtered_grad_fn
+
+
 def _deduplicate_indexed_slices(values, indices):
   """Sums `values` associated with any non-unique `indices`.
 
@@ -335,6 +352,13 @@ class Optimizer(
     #   ... }
     self._deferred_slot_restorations = {}
 
+    # TODO(isaprykin): When using a DistributionStrategy, and when an
+    # optimizer is created in each tower, it might be dangerous to
+    # rely on some Optimer methods.  When such methods are called on a
+    # per-tower optimizer, an exception needs to be thrown.  We do
+    # allow creation per-tower optimizers however, because the
+    # compute_gradients()->apply_gradients() sequence is safe.
+
   def get_name(self):
     return self._name
 
@@ -447,14 +471,33 @@ class Optimizer(
         if var_list is not None:
           tape.watch(var_list)
         loss_value = loss()
+
+        # Scale loss if using a "mean" loss reduction and multiple towers.
+        # Have to be careful to call distribute_lib.get_loss_reduction()
+        # *after* loss() is evaluated, so we know what loss reduction it uses.
+        # TODO(josh11b): Test that we handle weight decay in a reasonable way.
+        if distribute_lib.get_loss_reduction() == "mean":
+          num_towers = distribute_lib.get_distribution_strategy().num_towers
+          if num_towers > 1:
+            loss_value *= (1. / num_towers)
+
       if var_list is None:
         var_list = tape.watched_variables()
       grads = tape.gradient(loss_value, var_list, grad_loss)
       return list(zip(grads, var_list))
+
+    # Non-callable/Tensor loss case
     if context.executing_eagerly():
       raise RuntimeError(
           "`loss` passed to Optimizer.compute_gradients should "
           "be a function when eager execution is enabled.")
+
+    # Scale loss if using a "mean" loss reduction and multiple towers.
+    if distribute_lib.get_loss_reduction() == "mean":
+      num_towers = distribute_lib.get_distribution_strategy().num_towers
+      if num_towers > 1:
+        loss *= (1. / num_towers)
+
     if gate_gradients not in [Optimizer.GATE_NONE, Optimizer.GATE_OP,
                               Optimizer.GATE_GRAPH]:
       raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
@@ -510,11 +553,25 @@ class Optimizer(
     Raises:
       TypeError: If `grads_and_vars` is malformed.
       ValueError: If none of the variables have gradients.
+      RuntimeError: If you should use `_distributed_apply()` instead.
     """
     # This is a default implementation of apply_gradients() that can be shared
     # by most optimizers.  It relies on the subclass implementing the following
     # methods: _create_slots(), _prepare(), _apply_dense(), and _apply_sparse().
 
+    # Handle DistributionStrategy case.
+    if distribute_lib.get_cross_tower_context():
+      raise RuntimeError("Use `_distributed_apply()` instead of "
+                         "`apply_gradients()` in a cross-tower context.")
+    # TODO(isaprykin): Get rid of `has_distribution_strategy()` check by
+    # always calling _distributed_apply(), using the default distribution
+    # as needed.
+    if distribute_lib.has_distribution_strategy():
+      grads_and_vars = get_filtered_grad_fn(lambda _: grads_and_vars)()
+      return distribute_lib.get_tower_context().merge_call(
+          self._distributed_apply, grads_and_vars, global_step, name)
+
+    # No DistributionStrategy case.
     grads_and_vars = tuple(grads_and_vars)  # Make sure repeat iteration works.
     if not grads_and_vars:
       raise ValueError("No variables provided.")
@@ -582,6 +639,95 @@ class Optimizer(
 
       return apply_updates
 
+  def _distributed_apply(self,
+                         distribution,
+                         grads_and_vars,
+                         global_step=None,
+                         name=None):
+    """A version of `apply_gradients` for cross-tower context.
+
+    This is a version of `apply_gradients()` for when you are using a
+    `DistributionStrategy` and are in a cross-tower context. If in a
+    tower context, use `apply_gradients()` as normal.
+
+    Args:
+      distribution: A `DistributionStrategy` object.
+      grads_and_vars: List of (gradient, variable) pairs as returned by
+        `compute_gradients()`, and then aggregated across towers.
+      global_step: Optional (mirrored) `Variable` to increment by one
+        after the variables have been updated.
+      name: Optional name for the returned operation.  Default to the
+        name passed to the `Optimizer` constructor.
+
+    Returns:
+      An `Operation` that applies the specified gradients across all
+      towers. If `global_step` was not None, that operation also
+      increments `global_step`.
+    """
+    reduced_grads = distribution.batch_reduce("sum", grads_and_vars)
+    var_list = [v for _, v in grads_and_vars]
+    grads_and_vars = zip(reduced_grads, var_list)
+    # Note that this is called in a cross-tower context.
+    self._create_slots(var_list)
+
+    def update(v, g):
+      """Apply gradients to a replica variable."""
+      assert v is not None
+
+      try:
+        # Convert the grad to Tensor or IndexedSlices if necessary.
+        g = ops.convert_to_tensor_or_indexed_slices(g)
+      except TypeError:
+        raise TypeError("Gradient must be convertible to a Tensor"
+                        " or IndexedSlices, or None: %s" % g)
+      if not isinstance(g, (ops.Tensor, ops.IndexedSlices)):
+        raise TypeError(
+            "Gradient must be a Tensor, IndexedSlices, or None: %s" % g)
+      p = _get_processor(v)
+
+      scope_name = "" if context.executing_eagerly() else v.op.name
+      # device_policy is set because non-mirrored tensors will be read in
+      # `update_op`. `_resource_apply_dense`, `lr_t`, `beta1_t` and `beta2_t`
+      # is an example.
+      with ops.name_scope(
+          "update_" + scope_name), context.context().device_policy(
+              context.DEVICE_PLACEMENT_SILENT):
+        return p.update_op(self, g)
+
+    with ops.name_scope(name, self._name) as name:
+      self._prepare()
+
+      update_ops = [
+          op
+          for grad, var in grads_and_vars
+          for op in distribution.unwrap(distribution.update(var, update, grad))
+      ]
+
+      def finish(self, update_ops):
+        return self._finish(update_ops, "update")
+
+      non_slot_devices = distribution.non_slot_devices(var_list)
+      # Device policy is needed because hyperparameter tensors (such as
+      # AdamOptimizer's beta1_t) need to be copied across devices in Eager.
+      with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+        finish_updates = distribution.update_non_slot(
+            non_slot_devices, finish, self, update_ops)
+      if global_step is None:
+        apply_updates = distribution.group(finish_updates, name=name)
+      else:
+        with ops.control_dependencies(distribution.unwrap(finish_updates)):
+          apply_updates = distribution.group(distribution.update(
+              global_step, state_ops.assign_add, 1, name=name))
+
+      if not context.executing_eagerly():
+        if isinstance(apply_updates, ops.Tensor):
+          apply_updates = apply_updates.op
+        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
+        if apply_updates not in train_op:
+          train_op.append(apply_updates)
+
+      return apply_updates
+
   def get_slot(self, var, name):
     """Return a slot named `name` created for `var` by the Optimizer.
 
@@ -599,9 +745,25 @@ class Optimizer(
     Returns:
       The `Variable` for the slot if it was created, `None` otherwise.
     """
+    # pylint: disable=protected-access
     named_slots = self._slots.get(name, None)
     if not named_slots:
       return None
+
+    if hasattr(var, "_mirrored_container"):
+      # NOTE: If this isn't patched, then there is no `handle` in
+      # `_resource_apply_dense`.
+      mirrored_container = var._mirrored_container()
+      assert mirrored_container is not None
+      if context.executing_eagerly():
+        key = mirrored_container._unique_id
+      else:
+        key = (mirrored_container.graph, mirrored_container._shared_name)
+      # pylint: enable=protected-access
+      mirrored_slot = named_slots.get(key, None)
+      if mirrored_slot is None: return None
+      return mirrored_slot.get(device=var.device)
+
     return named_slots.get(_var_key(var), None)
 
   def get_slot_names(self):
@@ -645,6 +807,7 @@ class Optimizer(
 
   def _create_non_slot_variable(self, initial_value, name, colocate_with):
     """Add an extra variable, not associated with a slot."""
+    # Recommendation: Use OptimizerV2 if your optimizer uses non-slot variables.
     eager = context.executing_eagerly()
     graph = None if eager else colocate_with.graph
 
@@ -652,7 +815,8 @@ class Optimizer(
     v = self._non_slot_dict.get(key, None)
     if v is None:
       self._maybe_initialize_checkpointable()
-      with ops.colocate_with(colocate_with):
+      distribution_strategy = distribute_lib.get_distribution_strategy()
+      with distribution_strategy.colocate_vars_with(colocate_with):
         if eager:
           restored_initial_value = self._preload_simple_restoration(
               name=name, shape=None)
@@ -694,7 +858,13 @@ class Optimizer(
     return self._get_non_slot_variable(name, graph=graph)
 
   def _get_non_slot_variable(self, name, graph=None):
-    return self._non_slot_dict.get((name, graph), None)
+    non_slot = self._non_slot_dict.get((name, graph), None)
+    if hasattr(non_slot, "_mirrored_container"):
+      # This is a mirrored non-slot.  In order to enable code like `_finish`
+      # to assign to a non-slot, return the current context replica.
+      return non_slot.get()
+    else:
+      return non_slot
 
   def _non_slot_variables(self):
     """Additional variables created by the `Optimizer`.
-- 
GitLab


From 82f2f084268d80c242596116f77a4224fc4e3a0e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 14:59:53 -0700
Subject: [PATCH 641/960] Automated g4 rollback of changelist 190801044

PiperOrigin-RevId: 190839672
---
 .../optimizers/arithmetic_optimizer.cc        | 49 ++++++++-----------
 .../optimizers/arithmetic_optimizer.h         |  1 -
 .../optimizers/graph_optimizer_stage.cc       |  4 --
 .../optimizers/graph_optimizer_stage.h        |  3 --
 4 files changed, 21 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 629872bf19..5dd0b6f4b0 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -196,6 +196,8 @@ void SetSourceDataType(DataType dtype, NodeDef* node) {
 
 bool IsNumberType(DataType dtype) { return kNumberTypes.Contains(dtype); }
 
+const char kOutputShapesAttr[] = "_output_shapes";
+
 // Shape is symbolically defined if it has a known rank, and each dimension is
 // defined, or is an unknown symbol (dim.size <= -2).
 bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
@@ -232,19 +234,16 @@ bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
 // Returns whether `reshape` is an identity op. The tensor that `reshape`
 // reshapes is the `output_pos`-th output of node `input`.
 bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
-                       const int output_pos,
-                       const GraphProperties& graph_properties) {
-  const std::vector<OpInfo::TensorProperties>& reshape_props =
-      graph_properties.GetOutputProperties(reshape.name());
-  const std::vector<OpInfo::TensorProperties>& input_props =
-      graph_properties.GetOutputProperties(input.name());
-  if (reshape_props.empty() || input_props.empty() ||
-      input_props.size() <= output_pos) {
+                       const int output_pos) {
+  if (!reshape.attr().count(kOutputShapesAttr) ||
+      !input.attr().count(kOutputShapesAttr)) {
     return false;
   }
 
-  const PartialTensorShape& src_shape = input_props[output_pos].shape();
-  const PartialTensorShape& dst_shape = reshape_props[0].shape();
+  PartialTensorShape src_shape(
+      input.attr().at(kOutputShapesAttr).list().shape(output_pos));
+  PartialTensorShape dst_shape(
+      reshape.attr().at(kOutputShapesAttr).list().shape(0));
   if (src_shape.unknown_rank() || dst_shape.unknown_rank()) {
     return false;
   }
@@ -1273,8 +1272,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     // outputs tensors of shape [M, N] while feeding it with tensors of shape
     // [M*N] (or worse). The reshape nodes are then necessary to update the
     // tensor metadata to the required shape.
-    if (can_use_shapes_ &&
-        ReshapeIsIdentity(*reshape, *input, output_pos, *graph_properties_)) {
+    if (ReshapeIsIdentity(*reshape, *input, output_pos)) {
       return reshape->input(0);
     }
   }
@@ -1588,11 +1586,11 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
 
   std::vector<std::unique_ptr<ArithmeticOptimizerStage>> stages;
 
-  if (options_.combine_add_to_addn && can_use_shapes_) {
+  if (options_.combine_add_to_addn) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new AddOpsRewriteStage(ctx, ctx_ext)));
   }
-  if (options_.hoist_common_factor_out_of_aggregation && can_use_shapes_) {
+  if (options_.hoist_common_factor_out_of_aggregation) {
     stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
         new HoistCommonFactorOutOfAggregation(ctx, ctx_ext)));
   }
@@ -1629,15 +1627,7 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     if (simplified_tensor.empty()) {
       for (auto& stage : stages) {
         if (stage->IsSupported(node)) {
-          const Status stage_status =
-              stage->TrySimplify(node, &simplified_tensor);
-          // Each stage must be "error safe" (just like exception safe). In
-          // case of any error it must leave optimized graph unmodified.
-          if (!stage_status.ok()) {
-            LOG(WARNING) << "Failed to run arithmetic optimizer stage "
-                         << stage->stage_name()
-                         << ". Error: " << stage_status.error_message();
-          }
+          TF_RETURN_IF_ERROR(stage->TrySimplify(node, &simplified_tensor));
           if (!simplified_tensor.empty()) {
             break;
           }
@@ -1704,16 +1694,19 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
                                                &frame_map_, &num_frames));
   // Shapes are only needed in aggressive mode.
   graph_properties_.reset(new GraphProperties(item));
-  const Status status = graph_properties_->InferStatically(false);
-  can_use_shapes_ = status.ok();
-  if (!can_use_shapes_) {
-    LOG(WARNING) << "Shape inference failed.";
-  }
+  TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false));
+  // TODO(ezhulenev): Use GraphProperties to lookup tensor shapes directly
+  TF_RETURN_IF_ERROR(graph_properties_->AnnotateOutputShapes(optimized_graph_));
 
   // Perform the optimizations.
   DedupComputations();
   TF_RETURN_IF_ERROR(SimplifyArithmeticOps());
 
+  // Clear output shapes.
+  for (int i = 0; i < optimized_graph->node_size(); ++i) {
+    optimized_graph_->mutable_node(i)->mutable_attr()->erase(kOutputShapesAttr);
+  }
+
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index cdeed0554e..965f0e9ea2 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -126,7 +126,6 @@ class ArithmeticOptimizer : public GraphOptimizer {
   RewriterConfig::Toggle opt_level_;
   ArithmeticOptimizerOptions options_;
 
-  bool can_use_shapes_ = false;
   bool fetch_nodes_known_ = false;
   std::unordered_set<string> nodes_to_preserve_;
   std::unique_ptr<NodeMap> node_map_;
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
index 1ea57f7b4f..7044705ade 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.cc
@@ -42,10 +42,6 @@ Status GetInputNode(const GraphOptimizerContext& ctx, const string& input,
 Status GetTensorProperties(const GraphOptimizerContext& ctx,
                            const string& tensor,
                            OpInfo::TensorProperties* properties) {
-  if (ctx.graph_properties == nullptr) {
-    return errors::InvalidArgument("Graph properties are unknown.");
-  }
-
   int port;
   string tensor_node_name = ParseNodeName(tensor, &port);
   if (port < 0) {
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
index c7af82abbb..be95c00d2d 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -117,9 +117,6 @@ class GraphOptimizerStage {
       : optimizer_name_(optimizer_name), stage_name_(stage_name), ctx_(ctx) {}
   virtual ~GraphOptimizerStage() = default;
 
-  const string& stage_name() const { return stage_name_; }
-  const string& optimizer_name() const { return optimizer_name_; }
-
   // Check if we should try to simplify node. Returning true doesn't
   // guarantee that node will be simplified.
   //
-- 
GitLab


From b0e79c1c029f8829de8fce18dc16388d89e50318 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 15:31:19 -0700
Subject: [PATCH 642/960] Refresh Community pages to surface new resources,
 SIGs and mailing lists.

PiperOrigin-RevId: 190845545
---
 tensorflow/docs_src/community/contributing.md | 64 +++++++++++++
 tensorflow/docs_src/community/groups.md       | 17 ++++
 tensorflow/docs_src/community/index.md        | 95 +++++++++++++++----
 tensorflow/docs_src/community/leftnav_files   |  5 +-
 tensorflow/docs_src/community/lists.md        | 35 +++++++
 tensorflow/docs_src/community/welcome.md      | 71 --------------
 6 files changed, 198 insertions(+), 89 deletions(-)
 create mode 100644 tensorflow/docs_src/community/contributing.md
 create mode 100644 tensorflow/docs_src/community/groups.md
 create mode 100644 tensorflow/docs_src/community/lists.md
 delete mode 100644 tensorflow/docs_src/community/welcome.md

diff --git a/tensorflow/docs_src/community/contributing.md b/tensorflow/docs_src/community/contributing.md
new file mode 100644
index 0000000000..b0960df435
--- /dev/null
+++ b/tensorflow/docs_src/community/contributing.md
@@ -0,0 +1,64 @@
+# Contributing to TensorFlow
+
+TensorFlow is an open-source project, and we welcome your participation
+and contribution. This page describes how to get involved.
+
+## Repositories
+
+The code for TensorFlow is hosted in the [TensorFlow GitHub
+organization](https://github.com/tensorflow). Multiple projects are located
+inside the organization, including:
+
+* [TensorFlow](https://github.com/tensorflow/tensorflow)
+* [Models](https://github.com/tensorflow/models)
+* [TensorBoard](https://github.com/tensorflow/tensorboard)
+* [TensorFlow.js](https://github.com/tensorflow/tfjs)
+* [TensorFlow Serving](https://github.com/tensorflow/serving)
+* [TensorFlow Documentation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/docs_src)
+
+## Contributor checklist
+
+* Before contributing to TensorFlow source code, please review the [contribution
+guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).
+
+* Join the
+[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers)
+mailing list, to coordinate and discuss with others contributing to TensorFlow.
+
+* For coding style conventions, read the @{$style_guide$TensorFlow Style Guide}.
+
+* Finally, review @{$documentation$Writing TensorFlow Documentation}, which
+  explains documentation conventions.
+
+You may also wish to review our guide to @{$benchmarks$defining and running benchmarks}.
+
+## Special Interest Groups
+
+To enable focused collaboration on particular areas of TensorFlow, we host
+Special Interest Groups (SIGs). SIGs do their work in public: if you want to
+join and contribute, review the work of the group, and get in touch with the
+relevant SIG leader.
+
+* **SIG Build** focuses on issues surrounding building, packaging, and
+  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/build).
+
+* **SIG TensorBoard** furthers the development and direction of TensorBoard and its plugins.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard).
+
+* **SIG Rust** collaborates on the development of TensorFlow's Rust bindings.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust). 
+
+## Projects developed by the TensorFlow community
+
+The TensorFlow community has created many great projects around TensorFlow, including:
+
+* [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com)
+* [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow)
+* [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials)
+* [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow)
+* [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0))
+* [Operator Vectorization Library](https://github.com/opveclib/opveclib)
+* [Swift language bindings](https://github.com/PerfectlySoft/Perfect-TensorFlow)
+* [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
+* [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
+* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
diff --git a/tensorflow/docs_src/community/groups.md b/tensorflow/docs_src/community/groups.md
new file mode 100644
index 0000000000..d92f5775fa
--- /dev/null
+++ b/tensorflow/docs_src/community/groups.md
@@ -0,0 +1,17 @@
+# User Groups
+
+TensorFlow has communities around the world.
+
+## Asia
+
+* [TensorFlow Korea (TF-KR) User Group](https://www.facebook.com/groups/TensorFlowKR/) _(Korean language)_
+* [TensorFlow User Group Tokyo](https://tfug-tokyo.connpass.com/) _(Japanese Language)_
+* [Soleil Data Dojo](https://soleildatadojo.connpass.com/) _(Japanese language)_
+* [TensorFlow User Group Utsunomiya](https://tfug-utsunomiya.connpass.com/)
+
+
+## Europe
+
+* [TensorFlow Barcelona](https://www.meetup.com/Barcelona-Machine-Learning-Meetup/)
+* [TensorFlow Madrid](https://www.meetup.com/TensorFlow-Madrid/)
+
diff --git a/tensorflow/docs_src/community/index.md b/tensorflow/docs_src/community/index.md
index ebeff8493b..c08aeb7a97 100644
--- a/tensorflow/docs_src/community/index.md
+++ b/tensorflow/docs_src/community/index.md
@@ -1,18 +1,81 @@
 # Community
 
-This section contains the following documents:
-
-  * @{$welcome$Welcome to the TensorFlow Community}, which explains how
-    you can get involved, where to report issues, and where to join
-    like-minded TensorFlow enthusiasts online.
-  * @{$roadmap$Roadmap}, which summarizes upcoming additions to TensorFlow.
-  * @{$documentation$Writing TensorFlow Documentation}, which explains
-    TensorFlow's documentation conventions.  If you are modifying
-    TensorFlow source code or documentation, please read this guide.
-  * @{$style_guide$TensorFlow Style Guide}, which identifies coding style
-    conventions that TensorFlow developers and users should follow.
-  * @{$community/benchmarks$Benchmarks}, Benchmarks, a guide for defining and
-    running a TensorFlow benchmark.
-  * @{$security$Using TensorFlow Securely}, which explains TensorFlow's security
-    model, a list of recent security reports, and information on how you can
-    report a security vulnerability to the TensorFlow team.
+Welcome to the TensorFlow community! This page explains where to get help, and
+different ways to be part of the community. We are committed to fostering an
+open and welcoming environment, and request that you review our [code of
+conduct](https://github.com/tensorflow/tensorflow/blob/master/CODE_OF_CONDUCT.md).
+
+## Get Help
+
+### Technical Questions
+
+To ask or answer technical questions about TensorFlow, use [Stack
+Overflow](https://stackoverflow.com/questions/tagged/tensorflow). For example,
+ask or search about a particular error message you encountered during
+installation.
+
+### Bugs and Feature Requests
+
+To report bugs or make feature requests, file an issue on GitHub. Please choose
+the appropriate repository for the project. Major repositories include:
+
+  * [TensorFlow](https://github.com/tensorflow/tensorflow/issues)
+  * [TensorBoard](https://github.com/tensorflow/tensorboard/issues)
+  * [TensorFlow models](https://github.com/tensorflow/models/issues)
+  
+### Security
+
+Before using TensorFlow, please take a look at our security model, list of
+recent security announcements, and ways you can report security issues to the
+TensorFlow team at the
+[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) page on GitHub.
+
+## Stay Informed
+
+### Announcements Mailing List
+
+All major releases and important announcements are sent to
+[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
+We recommend that you join this list if you depend on TensorFlow in any way.
+
+### Development Roadmap
+
+The @{$roadmap$Roadmap} summarizes plans for upcoming additions to TensorFlow.
+
+### Social Media
+
+For news and updates from around the universe of TensorFlow projects, follow
+[@tensorflow](https://twitter.com/tensorflow) on Twitter.
+
+### YouTube
+
+Our [YouTube Channel](http://youtube.com/tensorflow/) focuses on machine learing
+and AI with TensorFlow. On it we have a number of new shows, including:
+
+- TensorFlow Meets: meet with community contributors to learn and share what they're doing
+- Ask TensorFlow: the team answers the best questions tagged #AskTensorFlow from social media 
+- Coding TensorFlow: short bites with tips for success with TensorFlow
+
+
+## Community Support
+
+### Mailing Lists
+
+For general discussion about TensorFlow development and direction, please join
+the [TensorFlow discuss mailing
+list](https://groups.google.com/a/tensorflow.org/d/forum/discuss).
+
+A number of other mailing lists exist, focused on different project areas, which
+can be found at @{$lists$TensorFlow Mailing Lists}.
+
+### User Groups
+
+To meet with like-minded people local to you, check out the many
+@{$groups$TensorFlow user groups} around the world.
+
+
+## Contributing To TensorFlow
+
+We welcome contributions and collaboration on TensorFlow. For more information,
+please read [Contributing to TensorFlow](contributing.md).
+
diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files
index af344506c7..0bd1f14de9 100644
--- a/tensorflow/docs_src/community/leftnav_files
+++ b/tensorflow/docs_src/community/leftnav_files
@@ -1,7 +1,8 @@
 index.md
-welcome.md
 roadmap.md
+contributing.md
+lists.md
+groups.md
 documentation.md
 style_guide.md
 benchmarks.md
-security.md
diff --git a/tensorflow/docs_src/community/lists.md b/tensorflow/docs_src/community/lists.md
new file mode 100644
index 0000000000..dc9240030e
--- /dev/null
+++ b/tensorflow/docs_src/community/lists.md
@@ -0,0 +1,35 @@
+# Mailing Lists
+
+As a community, we do much of our collaboration on public mailing lists.
+Please note that if you're looking for help using TensorFlow, [Stack
+Overflow](https://stackoverflow.com/questions/tagged/tensorflow) and
+[GitHub issues](https://github.com/tensorflow/tensorflow/issues)
+are the best initial places to look. For more information,
+see [how to get help](/community/#get_help).
+
+## General TensorFlow lists
+
+* [announce](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce) - Low-volume announcements of new releases.
+* [discuss](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss) - General community discussion around TensorFlow.
+* [developers](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers) - Discussion for developers contributing to TensorFlow.
+
+## Project-specific lists
+
+These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
+
+* [tensor2tensor](https://groups.google.com/forum/#!forum/tensor2tensor) - User
+  and peer support for Tensor2Tensor.
+
+## Special Interest Groups
+
+TensorFlow's [Special Interest
+Groups](/community/contributing#special_interest_groups) (SIGs) support
+community collaboration on particular project focuses. Members of these groups
+work together to build and support TensorFlow related projects.
+
+* [build](https://groups.google.com/a/tensorflow.org/forum/#!forum/build) -
+  Supporting SIG Build, for build, distribution and packaging of TensorFlow.
+* [tensorboard](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard) -
+  Supporting SIG TensorBoard, for plugin development and other contribution.
+* [rust](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust) -
+  Supporting SIG Rust, for the Rust language bindings.
diff --git a/tensorflow/docs_src/community/welcome.md b/tensorflow/docs_src/community/welcome.md
deleted file mode 100644
index 6d0458e678..0000000000
--- a/tensorflow/docs_src/community/welcome.md
+++ /dev/null
@@ -1,71 +0,0 @@
-# Welcome to the TensorFlow Community
-
-TensorFlow is an open-source project.  This page explains how to contribute,
-where to ask questions, and how to help each other.
-
-
-## Development
-
-The source code for TensorFlow is on
-[GitHub](https://github.com/tensorflow/tensorflow).
-
-Before contributing to TensorFlow source code, please review the
-[Contribution guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).
-
-### Projects developed by the TensorFlow community
-
-The TensorFlow community has created many great projects around TensorFlow, including:
-
-* [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com)
-* [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow)
-* [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials)
-* [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow)
-* [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0))
-* [Rust language bindings](https://github.com/google/tensorflow-rust)
-* [Operator Vectorization Library](https://github.com/opveclib/opveclib)
-* [Swift language bindings](https://github.com/PerfectlySoft/Perfect-TensorFlow)
-* [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
-* [Edward - A library for probabilistic modeling, inference, and criticism](http://edwardlib.org) ([Github](https://github.com/blei-lab/edward), [Forum](https://discourse.edwardlib.org))
-* [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
-* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - Please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
-
-## TensorFlow Communities Around the World
-
-Asia:
-
-* [TensorFlow Korea (TF-KR) User Group](https://www.facebook.com/groups/TensorFlowKR/) _(Korean language)_
-* [TensorFlow User Group Tokyo](https://tfug-tokyo.connpass.com/) _(Japanese Language)_
-* [Soleil Data Dojo](https://soleildatadojo.connpass.com/) _(Japanese language)_
-* [TensorFlow User Group Utsunomiya](https://tfug-utsunomiya.connpass.com/)
-
-
-Europe:
-
-* [TensorFlow Barcelona](https://www.meetup.com/Barcelona-Machine-Learning-Meetup/)
-* [TensorFlow Madrid](https://www.meetup.com/TensorFlow-Madrid/)
-
-
-
-## Support
-
-TensorFlow provides multiple communication paths.  To pick the right path,
-please read the following list carefully:
-
-  * For new release announcements and security updates, subscribe to
-    [announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
-  * To ask or answer technical questions about TensorFlow, use
-    [Stack Overflow](https://stackoverflow.com/questions/tagged/tensorflow).
-    For example, ask or search Stack Overflow about a particular error message
-    you encountered during installation.
-  * To join general discussions about TensorFlow development and directions,
-    please join the
-    [TensorFlow discuss mailing list](https://groups.google.com/a/tensorflow.org/d/forum/discuss).
-    For example, use this mailing list to learn about new features in
-    upcoming releases of TensorFlow.
-  * To report bugs or make feature requests, use the
-    [TensorFlow issues tracker](https://github.com/tensorflow/tensorflow/issues)
-    on GitHub.  For example, use the issue tracker to request a
-    new operation in TensorFlow.
-  * To report vulnerabilities, please follow our
-    [vulnerability disclosure guidelines](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md).
-
-- 
GitLab


From 6cb3e6e0988a7bd123e683c13dae8470c71822af Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 28 Mar 2018 15:54:31 -0700
Subject: [PATCH 643/960] [tf.data] Expose the symbol
 `tf.contrib.data.make_csv_dataset()`.

PiperOrigin-RevId: 190849333
---
 tensorflow/contrib/data/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 7c3a9f82ff..17048314a4 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -32,6 +32,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@group_by_window
 @@ignore_errors
 @@make_batched_features_dataset
+@@make_csv_dataset
 @@make_saveable_from_iterator
 @@map_and_batch
 @@padded_batch_and_drop_remainder
@@ -70,6 +71,7 @@ from tensorflow.contrib.data.python.ops.interleave_ops import sloppy_interleave
 from tensorflow.contrib.data.python.ops.iterator_ops import make_saveable_from_iterator
 from tensorflow.contrib.data.python.ops.prefetching_ops import prefetch_to_device
 from tensorflow.contrib.data.python.ops.readers import make_batched_features_dataset
+from tensorflow.contrib.data.python.ops.readers import make_csv_dataset
 from tensorflow.contrib.data.python.ops.readers import read_batch_features
 from tensorflow.contrib.data.python.ops.readers import SqlDataset
 from tensorflow.contrib.data.python.ops.resampling import rejection_resample
-- 
GitLab


From 830c19c3f20816dcb5e8e9b6cb51f63cf8461442 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 16:12:51 -0700
Subject: [PATCH 644/960] Add IsSquare bool to the grappler op_types.

PiperOrigin-RevId: 190852501
---
 tensorflow/core/grappler/op_types.cc | 2 ++
 tensorflow/core/grappler/op_types.h  | 1 +
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 1a6751befc..c31ac9b59c 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -309,6 +309,8 @@ bool IsSplitV(const NodeDef& node) { return node.op() == "SplitV"; }
 
 bool IsSqrtGrad(const NodeDef& node) { return node.op() == "SqrtGrad"; }
 
+bool IsSquare(const NodeDef& node) { return node.op() == "Square"; }
+
 bool IsSquaredDifference(const NodeDef& node) {
   return node.op() == "SquaredDifference";
 }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 1ec1cd46e3..39affcbc24 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -121,6 +121,7 @@ bool IsSoftsignGrad(const NodeDef& node);
 bool IsSplit(const NodeDef& node);
 bool IsSplitV(const NodeDef& node);
 bool IsSqrtGrad(const NodeDef& node);
+bool IsSquare(const NodeDef& node);
 bool IsSquaredDifference(const NodeDef& node);
 bool IsSqueeze(const NodeDef& node);
 bool IsStackOp(const NodeDef& node);
-- 
GitLab


From 390e19ab990f5656e09d98624c92b3c80e52937d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 16:16:48 -0700
Subject: [PATCH 645/960] Tower-local variable support for
 DistributionStrategy. Each tower has its own variable, but fetch() and
 checkpoint apply a reduction to get a single value.

PiperOrigin-RevId: 190853123
---
 tensorflow/python/training/distribute.py | 59 +++++++++++++++++++++---
 1 file changed, 53 insertions(+), 6 deletions(-)

diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 757ba71c4a..f98872775a 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -126,16 +126,18 @@ class UpdateContext(object):
 
 
 def get_tower_context():
-  """Returns the current TowerContext or None.
+  """Returns the current TowerContext or None if in a cross-tower context.
 
   Note that execution:
-  1. starts in the default (single-tower) tower context;
-  2. switches to cross-tower context when entering a
-     `with DistributionStrategy.scope():` block;
+  1. starts in the default (single-tower) tower context (this function
+     will return the default TowerContext object);
+  2. switches to cross-tower context (in which case this will return
+     None) when entering a `with DistributionStrategy.scope():` block;
   3. switches to a (non-default) tower context inside
      `call_for_each_tower(fn, ...)`;
   4. if `fn` calls `get_tower_context()->merge_call(merge_fn, ...)`, then
-     inside `merge_fn` you are back in the cross-tower context.
+     inside `merge_fn` you are back in the cross-tower context (and again
+     this function will return None).
 
   Note that you can also go directly from step 1 to 4 to switch to a
   cross-tower context for the default `DistributionStrategy`. You may
@@ -188,6 +190,9 @@ def get_cross_tower_context():
 def get_distribution_strategy():
   """Returns the current `DistributionStrategy` object.
 
+  Prefer to use `get_tower_context()` or `get_cross_tower_context()`
+  instead when possible.
+
   Returns:
     A `DistributionStrategy` object. Inside a
     `with distribution_strategy.scope()` block, it returns
@@ -526,7 +531,6 @@ class DistributionStrategy(object):
   # TODO(josh11b): ClusterSpec/ClusterResolver
   # TODO(josh11b): Partitioned computations, state; sharding
   # TODO(josh11b): Model parallelism: "towers" with multiple devices; shuffling
-  # TODO(josh11b): Tower-local variables
   # TODO(josh11b): List of towers with their worker and parameter devices
   #   (where the parameter devices may overlap in the ps case).
 
@@ -556,6 +560,43 @@ class DistributionStrategy(object):
     # Note: should support "colocate_with" argument.
     raise NotImplementedError("must be implemented in descendants")
 
+  def tower_local_var_scope(self, reduce_method):
+    """Inside this scope, new variables will not be mirrored.
+
+    There will still be one component variable per tower, but there is
+    no requirement that they stay in sync. Instead, when saving them
+    or calling `fetch()`, we use the value that results when calling
+    `reduce()` on all the towers' variables.
+
+    Note: tower-local implies not trainable. Instead, it is expected
+    that each tower will directly update (using `assign_add()` or
+    whatever) its local variable instance but only the aggregated
+    value (accessible using `fetch()`) will be exported from the
+    model. When it is acceptable to only aggregate on export, we
+    greatly reduce communication overhead by using tower-local
+    variables.
+
+    Note: All component variables will be initialized to the same
+    value, using the initialization expression from the first tower.
+    The values will match even if the initialization expression uses
+    random numbers.
+
+    Args:
+      reduce_method: String used as a `method_string` to `reduce()`
+        to get the value to save when checkpointing.
+
+    Returns:
+      A context manager.
+    """
+    def create_tower_local_variable(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      kwargs["use_resource"] = True
+      kwargs["tower_local_reduce_method"] = reduce_method
+      return next_creator(*args, **kwargs)
+
+    _require_distribution_strategy_scope(self)
+    return variable_scope.variable_creator_scope(create_tower_local_variable)
+
   def colocate_vars_with(self, colocate_with_variable):
     """Scope that controls which devices variables will be created on.
 
@@ -984,6 +1025,10 @@ class TowerContext(object):
     finally:
       _pop_per_thread_mode()
 
+  def tower_local_var_scope(self, reduce_method):
+    """Alias for distribution_strategy.tower_local_var_scope()."""
+    return self._distribution_strategy.tower_local_var_scope(reduce_method)
+
   @property
   def is_single_tower(self):
     """Returns whether there is a single tower or multiple."""
@@ -1030,6 +1075,8 @@ class _DefaultDistributionStrategy(DistributionStrategy):
 
     def creator(next_creator, *args, **kwargs):
       _require_distribution_strategy_scope(self)
+      if kwargs.pop("tower_local_reduce_method", None) is not None:
+        kwargs["trainable"] = False
       return next_creator(*args, **kwargs)
 
     return _CurrentDistributionContext(
-- 
GitLab


From 108178da2a20ea2d3899417ee932d46ba1a5c652 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Wed, 28 Mar 2018 16:52:39 -0700
Subject: [PATCH 646/960] Automated g4 rollback of changelist 190835392

PiperOrigin-RevId: 190858242
---
 RELEASE.md                                    |  60 ---
 configure.py                                  |   2 +-
 tensorflow/BUILD                              |   7 -
 tensorflow/contrib/BUILD                      |  27 +-
 .../boosted_trees/kernels/quantile_ops.cc     |   2 +-
 .../boosted_trees/lib/utils/batch_features.cc |   2 +-
 .../lib/utils/batch_features_test.cc          |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.cc  |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.h   |   2 +-
 .../lib/utils/sparse_column_iterable_test.cc  |   2 +-
 .../boosted_trees/proto/tree_config.proto     |   2 +-
 .../kernel_tests/prediction_ops_test.py       |  10 +-
 .../python/kernel_tests/quantile_ops_test.py  |   2 +-
 .../boosted_trees/python/ops/quantile_ops.py  |   2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |   3 -
 .../kernel_tests/batch_dataset_op_test.py     |  14 -
 tensorflow/contrib/eager/python/BUILD         |   6 +-
 .../eager/python/examples/spinn/spinn_test.py |   1 +
 .../python/estimator/replicate_model_fn.py    |   2 +-
 .../factorization/kernels/clustering_ops.cc   |   2 +-
 .../python/ops/factorization_ops.py           |  14 +-
 .../python/ops/factorization_ops_test.py      |  12 +-
 .../factorization/python/ops/gmm_ops.py       |   4 +-
 .../factorization/python/ops/gmm_test.py      |   2 +-
 .../factorization/python/ops/kmeans_test.py   |   4 +-
 .../contrib/factorization/python/ops/wals.py  |   2 +-
 tensorflow/contrib/learn/BUILD                |   1 -
 .../learn/python/learn/estimators/linear.py   |   4 +-
 .../linear_optimizer/python/sdca_estimator.py |   4 +-
 tensorflow/contrib/lite/README.md             |   3 -
 tensorflow/contrib/lite/builtin_ops.h         |   1 -
 tensorflow/contrib/lite/g3doc/models.md       |   2 +-
 tensorflow/contrib/lite/kernels/BUILD         |  13 -
 .../internal/reference/reference_ops.h        |  25 --
 tensorflow/contrib/lite/kernels/maximum.cc    | 106 -----
 .../contrib/lite/kernels/maximum_test.cc      |  81 ----
 tensorflow/contrib/lite/kernels/register.cc   |   2 -
 tensorflow/contrib/lite/model.cc              |   3 -
 tensorflow/contrib/lite/nnapi_delegate.cc     |   1 -
 tensorflow/contrib/lite/python/lite.py        |  22 +-
 tensorflow/contrib/lite/schema/schema.fbs     |   5 -
 .../contrib/lite/schema/schema_generated.h    | 124 +-----
 tensorflow/contrib/lite/testing/BUILD         |   1 -
 .../contrib/lite/testing/generate_examples.py |  36 --
 .../testing/generated_examples_zip_test.cc    |   1 -
 .../contrib/lite/toco/tflite/operator.cc      |   2 -
 .../contrib/lite/toco/tflite/operator_test.cc |   2 -
 tensorflow/contrib/lookup/lookup_ops.py       |   2 +-
 .../contrib/makefile/download_dependencies.sh |   2 +-
 tensorflow/contrib/makefile/tf_op_files.txt   |   1 -
 .../seq2seq/kernels/beam_search_ops.cc        |   2 +-
 .../seq2seq/python/ops/attention_wrapper.py   |   8 +-
 .../seq2seq/python/ops/beam_search_decoder.py |   6 +-
 .../slim/python/slim/data/parallel_reader.py  |   4 +-
 .../slim/python/slim/data/prefetch_queue.py   |   4 +-
 .../python/slim/data/tfexample_decoder.py     |   2 +-
 tensorflow/contrib/tensorrt/README.md         |  46 +--
 .../contrib/tensorrt/convert/convert_graph.cc |  20 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 375 ++++++++----------
 .../contrib/tensorrt/segment/segment.cc       |  55 +--
 tensorflow/contrib/tensorrt/segment/segment.h |   4 +-
 .../contrib/tensorrt/segment/segment_test.cc  |   8 +-
 .../timeseries/python/timeseries/ar_model.py  |   2 +-
 .../python/timeseries/math_utils.py           |   2 +-
 .../timeseries/state_space_models/varma.py    |   4 +-
 .../base_api/api_def_MatrixSolveLs.pbtxt      |   6 +-
 .../core/common_runtime/mkl_cpu_allocator.cc  |   3 +
 tensorflow/core/framework/common_shape_fns.cc |   4 +-
 tensorflow/core/framework/common_shape_fns.h  |   8 +-
 tensorflow/core/framework/shape_inference.h   |   1 -
 .../core/kernels/mkl_fused_batch_norm_op.cc   |   2 +-
 .../core/kernels/segment_reduction_ops.h      |   7 -
 tensorflow/core/kernels/snapshot_op.cc        |  30 --
 tensorflow/core/kernels/snapshot_op.h         |  26 +-
 tensorflow/core/kernels/snapshot_op_gpu.cu.cc |   9 +-
 tensorflow/core/kernels/xent_op.cc            |  65 +--
 tensorflow/core/kernels/xent_op.h             |  35 +-
 tensorflow/core/kernels/xent_op_gpu.cu.cc     |   9 +-
 tensorflow/core/ops/array_ops.cc              |  26 +-
 tensorflow/core/ops/nn_ops.cc                 |  23 +-
 tensorflow/core/ops/nn_ops_test.cc            |  16 +-
 tensorflow/core/public/version.h              |   4 +-
 .../python/contrib.bayesflow.monte_carlo.md   |  36 +-
 .../api_guides/python/contrib.losses.md       |  28 +-
 .../docs_src/community/documentation.md       |  38 +-
 tensorflow/docs_src/install/install_c.md      |   2 +-
 tensorflow/docs_src/install/install_go.md     |   2 +-
 tensorflow/docs_src/install/install_java.md   |  22 +-
 tensorflow/docs_src/install/install_linux.md  |  22 +-
 tensorflow/docs_src/install/install_mac.md    |  14 +-
 .../docs_src/install/install_sources.md       |   9 +-
 tensorflow/docs_src/mobile/optimizing.md      |   2 -
 tensorflow/docs_src/mobile/prepare_models.md  |   2 +-
 tensorflow/python/BUILD                       |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  26 +-
 tensorflow/python/kernel_tests/testdata/BUILD |   2 +-
 .../python/kernel_tests/xent_op_test.py       |  81 +---
 tensorflow/python/layers/convolutional.py     |   2 -
 .../python/layers/convolutional_test.py       |   6 -
 tensorflow/python/ops/linalg_ops.py           |   2 +-
 .../python/training/monitored_session.py      |  33 +-
 .../python/training/monitored_session_test.py |  36 --
 tensorflow/tensorflow.bzl                     |   4 +-
 .../tools/api/golden/tensorflow.train.pbtxt   |   2 +-
 .../tools/ci_build/osx/libtensorflow_cpu.sh   |   2 +-
 tensorflow/tools/docker/Dockerfile.devel      |   2 +-
 .../tools/docker/Dockerfile.devel-cpu-mkl     |   2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |   2 +-
 tensorflow/tools/lib_package/BUILD            |   2 +
 tensorflow/tools/pip_package/BUILD            |   1 +
 tensorflow/tools/pip_package/setup.py         |   6 +-
 tensorflow/workspace.bzl                      | 133 +++----
 third_party/mkl/BUILD                         |  46 +--
 third_party/mkl/MKL_LICENSE                   | 201 ----------
 third_party/mkl/build_defs.bzl                |  12 -
 third_party/mkl/mkl.BUILD                     |  27 +-
 116 files changed, 556 insertions(+), 1703 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/kernels/maximum.cc
 delete mode 100644 tensorflow/contrib/lite/kernels/maximum_test.cc
 delete mode 100644 third_party/mkl/MKL_LICENSE

diff --git a/RELEASE.md b/RELEASE.md
index c63d9f20c9..6f54dee58f 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,63 +1,3 @@
-# Release 1.7.0
-
-## Major Features And Improvements
-* Eager mode is moving out of contrib, try `tf.enable_eager_execution()`.
-* Graph rewrites emulating fixed-point quantization compatible with TensorFlow Lite, supported by new `tf.contrib.quantize` package.
-* Easily customize gradient computation with `tf.custom_gradient`.
-* [TensorBoard Debugger Plugin](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md), the graphical user interface (GUI) of TensorFlow Debugger (tfdbg), is now in alpha.
-* Experimental support for reading a sqlite database as a `Dataset` with new `tf.contrib.data.SqlDataset`.
-* Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`.
-* Better text processing with `tf.regex_replace`.
-* Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length`
-
-## Bug Fixes and Other Changes
-* Accelerated Linear Algebra (XLA):
-  * Add `MaxPoolGradGrad` support for XLA
-  * CSE pass from Tensorflow is now disabled in XLA.
-* `tf.data`:
-  * `tf.data.Dataset`
-    * Add support for building C++ Dataset op kernels as external libraries, using the `tf.load_op_library()` mechanism.
-    * `Dataset.list_files()` now shuffles its output by default.
-    * `Dataset.shuffle(..., seed=tf.constant(0, dtype=tf.int64))` now yields the same sequence of elements as `Dataset.shuffle(..., seed=0)`.
-  * Add `num_parallel_reads` argument to `tf.data.TFRecordDataset`.
-* `tf.contrib`:
-  * `tf.contrib.bayesflow.halton_sequence` now supports randomization.
-  * Add support for scalars in `tf.contrib.all_reduce`.
-  * Add `effective_sample_size` to `tf.contrib.bayesflow.mcmc_diagnostics`.
-  * Add `potential_scale_reduction` to `tf.contrib.bayesflow.mcmc_diagnostics`.
-  * Add `BatchNormalization`, `Kumaraswamy` bijectors.
-  * Deprecate `tf.contrib.learn`. Please check contrib/learn/README.md for instructions on how to convert existing code.
-  * `tf.contrib.data`
-    * Remove deprecated `tf.contrib.data.Dataset`, `tf.contrib.data.Iterator`, `tf.contrib.data.FixedLengthRecordDataset`, `tf.contrib.data.TextLineDataset`, and `tf.contrib.data.TFRecordDataset` classes.
-    * Added `bucket_by_sequence_length`, `sliding_window_batch`, and `make_batched_features_dataset`
-  * Remove unmaintained `tf.contrib.ndlstm`. You can find it externally at https://github.com/tmbarchive/tfndlstm.
-  * Moved most of `tf.contrib.bayesflow` to its own repo: `tfp`
-* Other:
-  * tf.py_func now reports the full stack trace if an exception occurs.
-  * Integrate `TPUClusterResolver` with GKE's integration for Cloud TPUs.
-  * Add a library for statistical testing of samplers.
-  * Add Helpers to stream data from the GCE VM to a Cloud TPU.
-  * Integrate ClusterResolvers with TPUEstimator.
-  * Unify metropolis_hastings interface with HMC kernel.
-  * Move LIBXSMM convolutions to a separate --define flag so that they are disabled by default.
-  * Fix `MomentumOptimizer` lambda.
-  * Reduce `tfp.layers` boilerplate via programmable docstrings.
-  * Add `auc_with_confidence_intervals`, a method for computing the AUC and confidence interval with linearithmic time complexity.
-  * `regression_head` now accepts customized link function, to satisfy the usage that user can define their own link function if the `array_ops.identity` does not meet the requirement.
-  * Fix `initialized_value` and `initial_value` behaviors for `ResourceVariables` created from `VariableDef` protos.
-  * Add TensorSpec to represent the specification of Tensors.
-  * Constant folding pass is now deterministic.
-  * Support `float16` `dtype` in `tf.linalg.*`.
-  * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions.
-
-## Thanks to our Contributors
-
-This release contains contributions from many people at Google, as well as:
-
-4d55397500, Abe, Alistair Low, Andy Kernahan, Appledore, Ben, Ben Barsdell, Boris Pfahringer, Brad Wannow, Brett Koonce, Carl Thomé, cclauss, Chengzhi Chen, Chris Drake, Christopher Yeh, Clayne Robison, Codrut Grosu, Daniel Trebbien, Danny Goodman, David Goodwin, David Norman, Deron Eriksson, Donggeon Lim, Donny Viszneki, DosLin, DylanDmitri, Francisco Guerrero, Fred Reiss, gdh1995, Giuseppe, Glenn Weidner, gracehoney, Guozhong Zhuang, Haichen "Hc" Li, Harald Husum, harumitsu.nobuta, Henry Spivey, hsm207, Jekyll Song, Jerome, Jiongyan Zhang, jjsjann123, John Sungjin Park, Johnson145, JoshVarty, Julian Wolff, Jun Wang, June-One, Kamil Sindi, Kb Sriram, Kdavis-Mozilla, Kenji, lazypanda1, Liang-Chi Hsieh, Loo Rong Jie, Mahesh Bhosale, MandarJKulkarni, ManHyuk, Marcus Ong, Marshal Hayes, Martin Pool, matthieudelaro, mdfaijul, mholzel, Michael Zhou, Ming Li, Minmin Sun, Myungjoo Ham, MyungsungKwak, Naman Kamra, Peng Yu, Penghao Cen, Phil, Raghuraman-K, resec, Rohin Mohanadas, Sandeep N Gupta, Scott Tseng, seaotterman, Seo Sanghyeon, Sergei Lebedev, Ted Chang, terrytangyuan, Tim H, tkunic, Tod, vihanjain, Yan Facai (颜发才), Yin Li, Yong Tang, Yukun Chen, Yusuke Yamada
-
-
-
 # Release 1.6.0
 
 ## Breaking Changes
diff --git a/configure.py b/configure.py
index 0f52c0ec99..22b9abedd7 100644
--- a/configure.py
+++ b/configure.py
@@ -1414,7 +1414,7 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform',
-                'with_kafka_support', True, 'kafka')
+                'with_kafka_support', False, 'kafka')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 29a01efc84..6ab43638ba 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -240,13 +240,6 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
-config_setting(
-    name = "with_kafka_support_windows_override",
-    define_values = {"with_kafka_support": "true"},
-    values = {"cpu": "x64_windows"},
-    visibility = ["//visibility:public"],
-)
-
 config_setting(
     name = "with_gcp_support_android_override",
     define_values = {"with_gcp_support": "true"},
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index fb81b50fe8..bdbd738906 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -51,6 +51,7 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
+        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -109,13 +110,7 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]) + select({
-        "//tensorflow:with_kafka_support_windows_override": [],
-        "//tensorflow:with_kafka_support": [
-            "//tensorflow/contrib/kafka",
-        ],
-        "//conditions:default": [],
-    }),
+    ]),
 )
 
 cc_library(
@@ -125,6 +120,7 @@ cc_library(
         "//tensorflow/contrib/boosted_trees:boosted_trees_kernels",
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
+        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -137,13 +133,7 @@ cc_library(
         "//tensorflow/contrib/text:all_kernels",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_cuda([
         "//tensorflow/contrib/nccl:nccl_kernels",
-    ]) + select({
-        "//tensorflow:with_kafka_support_windows_override": [],
-        "//tensorflow:with_kafka_support": [
-            "//tensorflow/contrib/kafka:dataset_kernels",
-        ],
-        "//conditions:default": [],
-    }),
+    ]),
 )
 
 cc_library(
@@ -156,6 +146,7 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
+        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
@@ -166,13 +157,7 @@ cc_library(
         "//tensorflow/contrib/tensor_forest:tensor_forest_ops_op_lib",
         "//tensorflow/contrib/text:all_ops",
         "//tensorflow/contrib/tpu:all_ops",
-    ] + select({
-        "//tensorflow:with_kafka_support_windows_override": [],
-        "//tensorflow:with_kafka_support": [
-            "//tensorflow/contrib/kafka:dataset_ops_op_lib",
-        ],
-        "//conditions:default": [],
-    }),
+    ],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
index 0b28f81e7c..0f4c2298f5 100644
--- a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
@@ -253,7 +253,7 @@ class CreateQuantileAccumulatorOp : public OpKernel {
  private:
   float epsilon_;
   int32 num_quantiles_;
-  // An upper bound on the number of entries that the summaries might have
+  // An upperbound on the number of enteries that the summaries might have
   // for a feature.
   int64 max_elements_;
   bool generate_quantiles_;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
index 35b059f349..cf4f9a097a 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
@@ -54,7 +54,7 @@ Status BatchFeatures::Initialize(
     TF_CHECK_AND_RETURN_IF_ERROR(
         dense_float_feature.dim_size(1) == 1,
         errors::InvalidArgument(
-            "Dense float features may not be multivalent: dim_size(1) = ",
+            "Dense float features may not be multi-valent: dim_size(1) = ",
             dense_float_feature.dim_size(1)));
     dense_float_feature_columns_.emplace_back(dense_float_feature);
   }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
index cfe9101e74..609519e8b1 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
@@ -59,7 +59,7 @@ TEST_F(BatchFeaturesTest, DenseFloatFeatures_Multivalent) {
   BatchFeatures batch_features(1);
   auto dense_vec = AsTensor<float>({3.0f, 7.0f}, {1, 2});
   auto expected_error = InvalidArgument(
-      "Dense float features may not be multivalent: dim_size(1) = 2");
+      "Dense float features may not be multi-valent: dim_size(1) = 2");
   EXPECT_EQ(expected_error,
             batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {}));
 }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
index ce67db797d..db34db998a 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
@@ -54,7 +54,7 @@ Status DropoutUtils::DropOutTrees(
   if (probability_of_skipping_dropout < 0 ||
       probability_of_skipping_dropout > 1) {
     return errors::InvalidArgument(
-        "Probability of skipping dropout must be in [0,1] range");
+        "Probability of skiping dropout must be in [0,1] range");
   }
   const auto num_trees = weights.size();
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
index 77c16da541..928bfbfe5c 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
@@ -66,7 +66,7 @@ class DropoutUtils {
       // Current weights and num_updates will be updated as a result of this
       // func
       std::vector<float>* current_weights,
-      // How many weight assignments have been done for each tree already.
+      // How many weight assignements have been done for each tree already.
       std::vector<int32>* num_updates);
 };
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
index cc7604745e..0138aae3db 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
@@ -34,7 +34,7 @@ TEST_F(SparseColumnIterableTest, Empty) {
 }
 
 TEST_F(SparseColumnIterableTest, Iterate) {
-  // 8 examples having 7 sparse features with the 3rd and 7th multivalent.
+  // 8 examples having 7 sparse features with the 3rd and 7th multi-valent.
   // This can be visualized like the following:
   // Instance | Sparse |
   // 0        |  x     |
diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
index 81411aa84a..4407c4d981 100644
--- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto
+++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
@@ -53,7 +53,7 @@ message DenseFloatBinarySplit {
   // Float feature column and split threshold describing
   // the rule feature <= threshold.
   int32 feature_column = 1;
-  // If feature column is multivalent, this holds the index of the dimension
+  // If feature column is multivalent, this holds the index of the dimensiong
   // for the split. Defaults to 0.
   int32 dimension_id = 5;
   float threshold = 2;
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
index cf55759aaa..c1acf35160 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
@@ -120,8 +120,8 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     """Sets up the prediction tests.
 
     Create a batch of two examples having one dense float, two sparse float
-    single valued, one sparse float multidimensional and one sparse int
-    features.  The data looks like the following:
+    single valued, one sparse float multidimensionl and one sparse int features.
+    The data looks like the following:
     | Instance | Dense0 | SparseF0 | SparseF1 | SparseI0 | SparseM
     | 0        |  7     |    -3    |          |    9,1   | __, 5.0
     | 1        | -2     |          | 4        |          |  3, ___
@@ -810,7 +810,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     # building. This tree should never be dropped.
     num_trees = 10
     with self.test_session():
-      # Empty tree ensemble.
+      # Empty tree ensenble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 10 trees with some weights.
       for i in range(0, num_trees):
@@ -951,7 +951,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testDropOutZeroProb(self):
     with self.test_session():
-      # Empty tree ensemble.
+      # Empty tree ensenble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 1000 trees with some weights.
       for i in range(0, 999):
@@ -994,7 +994,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testAveragingAllTrees(self):
     with self.test_session():
-      # Empty tree ensemble.
+      # Empty tree ensenble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       adjusted_tree_ensemble_config = (
           tree_config_pb2.DecisionTreeEnsembleConfig())
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
index 074623699d..81f58de28c 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
@@ -482,7 +482,7 @@ class QuantilesOpTest(test_util.TensorFlowTestCase):
     """Sets up the quantile op tests.
 
     Create a batch of 4 examples having 2 dense and 4 sparse features.
-    Fourth sparse feature is multivalent (3 dimensional)
+    Forth sparse feature is multivalent (3 dimensional)
     The data looks like this
     | Instance | Dense 0 | Dense 1 | Sparse 0 | Sparse 1 |Sparse 2| SparseM
     | 0        |   -0.1  |  -1     |   -2     |   0.1    |        |_ ,1,_
diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
index 1b184d296b..97d57e8b23 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
@@ -184,7 +184,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
     """Finalizes quantile summary stream and resets it for next iteration.
 
     Args:
-      stamp_token: Expected current token.
+      stamp_token: Exepcted current token.
       next_stamp_token: Next value for the token.
     Returns:
       A list of quantiles or approximate boundaries.
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index 92f2ab6dea..f793877c8b 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -210,9 +210,6 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py"
     # Test is flaky on Windows GPU builds (b/38283730).
     "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/gmm_test.py"
-    # Disable following manual tag in BUILD.
-    "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/layers/convolutional_test.py"
-
   )
   if (WIN32)
     set(tf_test_src_py_exclude
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 75482f67da..5abb38c2d2 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -413,20 +413,6 @@ class BatchDatasetTest(test.TestCase):
   def testMapAndBatchPartialBatchDropRemainder(self):
     return self._testMapAndBatchPartialBatchHelper(drop_remainder=True)
 
-  def testMapAndBatchYieldsPartialBatch(self):
-    iterator = (dataset_ops.Dataset.range(10)
-                .apply(batching.map_and_batch(
-                    lambda x: array_ops.reshape(x * x, [1]), 4))
-                .make_one_shot_iterator())
-    self.assertEqual([None, 1], iterator.output_shapes.as_list())
-    next_element = iterator.get_next()
-    with self.test_session() as sess:
-      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
-      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
-      self.assertAllEqual([[64], [81]], sess.run(next_element))
-      with self.assertRaises(errors.OutOfRangeError):
-        sess.run(next_element)
-
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 80176397c0..4fba014d6f 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -270,11 +270,7 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = [
-        "no_oss",  # b/74395663
-        "no_windows",  # TODO: needs investigation on Windows
-        "notsan",
-    ],
+    tags = ["notsan"],
 )
 
 filegroup(
diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 9adf47d505..9261823d77 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -418,6 +418,7 @@ class SpinnTest(test_util.TensorFlowTestCase):
                     if event.summary.value
                     and event.summary.value[0].tag == "train/loss"]
     self.assertEqual(config.epochs, len(train_losses))
+    self.assertLess(train_losses[-1], train_losses[0])
 
     # 5. Verify that checkpoints exist and contains all the expected variables.
     self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*")))
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index fa2697800e..e0fae2c992 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replicate only on the subset of available GPUs.
+      argument can be used to replice only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
index 2a6c97e8b9..dd61f59585 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
@@ -353,7 +353,7 @@ class NearestNeighborsOp : public OpKernel {
     auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
     const int64 num_threads = worker_threads.num_threads;
     // This kernel might be configured to use fewer than the total number of
-    // available CPUs on the host machine. To avoid destructive interference
+    // available CPUs on the host machine. To avoid descructive interference
     // with other jobs running on the host machine, we must only use a fraction
     // of total available L3 cache. Unfortunately, we cannot query the host
     // machine to get the number of physical CPUs. So, we use a fixed per-CPU
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
index 8e0ed1d80e..054888e734 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
@@ -106,7 +106,7 @@ class WALSModel(object):
       # the prep_gramian_op for row(column) can be run.
       worker_init_op = model.worker_init
 
-      # To be run once per integration sweep before the row(column) update
+      # To be run once per interation sweep before the row(column) update
       # initialize ops can be run. Note that in the distributed training
       # situations, this should only be run by the chief trainer. All other
       # trainers need to block until this is done.
@@ -118,9 +118,9 @@ class WALSModel(object):
       init_row_update_op = model.initialize_row_update_op
       init_col_update_op = model.initialize_col_update_op
 
-      # Ops to update row(column). This can either take the entire sparse
-      # tensor or slices of sparse tensor. For distributed trainer, each
-      # trainer handles just part of the matrix.
+      # Ops to upate row(column). This can either take the entire sparse tensor
+      # or slices of sparse tensor. For distributed trainer, each trainer
+      # handles just part of the matrix.
       _, row_update_op, unreg_row_loss, row_reg, _ = model.update_row_factors(
            sp_input=matrix_slices_from_queue_for_worker_shard)
       row_loss = unreg_row_loss + row_reg
@@ -220,7 +220,7 @@ class WALSModel(object):
         in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of
         inner lists matching the number of row factor shards and the elements in
         each inner list are the weights for the rows of the corresponding row
-        factor shard. In this case,  w_ij = unobserved_weight +
+        factor shard. In this case,  w_ij = unonbserved_weight +
                                             row_weights[i] * col_weights[j].
         - If this is a single non-negative real number, this value is used for
         all row weights and w_ij = unobserved_weight + row_weights *
@@ -435,7 +435,7 @@ class WALSModel(object):
       gramian: Variable storing the gramian calculated from the factors.
 
     Returns:
-      A op that updates the gramian with the calculated value from the factors.
+      A op that updates the gramian with the calcuated value from the factors.
     """
     partial_gramians = []
     for f in factors:
@@ -564,7 +564,7 @@ class WALSModel(object):
 
     Note that specifically this initializes the cache of the row and column
     weights on workers when `use_factors_weights_cache` is True. In this case,
-    if these weights are being calculated and reset after the object is created,
+    if these weights are being calcualted and reset after the object is created,
     it is important to ensure this ops is run afterwards so the cache reflects
     the correct values.
     """
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
index bb5140aeb3..c813733915 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
@@ -210,7 +210,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reproduce the same row factors in the model as the
+      # This is expected to reprodue the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -283,8 +283,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 3 column feature vectors.
-      # This is expected to reproduce the same column factors in the model as
-      # the weights and feature vectors are identical to that used in model
+      # This is expected to reprodue the same column factors in the model as the
+      # weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
@@ -385,7 +385,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reproduce the same row factors in the model as the
+      # This is expected to reprodue the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -462,8 +462,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 2 column feature vectors.
-      # This is expected to reproduce the same column factors in the model as
-      # the weights and feature vectors are identical to that used in model
+      # This is expected to reprodue the same column factors in the model as the
+      # weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index 14d4c733e3..98d6434f47 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -280,7 +280,7 @@ class GmmAlgorithm(object):
     self._define_score_samples()
 
   def _define_full_covariance_probs(self, shard_id, shard):
-    """Defines the full covariance probabilities per example in a class.
+    """Defines the full covariance probabilties per example in a class.
 
     Updates a matrix with dimension num_examples X num_classes.
 
@@ -344,7 +344,7 @@ class GmmAlgorithm(object):
   def _define_prior_log_prob_operation(self, shard_id):
     """Computes the prior probability of all samples.
 
-    Updates a vector where each item is the prior probability of an
+    Updates a vector where each item is the prior probabibility of an
     input example.
 
     Args:
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_test.py b/tensorflow/contrib/factorization/python/ops/gmm_test.py
index 4fc9c96e9d..00a4734eb6 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_test.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_test.py
@@ -210,7 +210,7 @@ class GMMTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependent on inputs, if input
+  # Note that since cluster initialization is dependendent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 88eb9cf692..0103cc4439 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -413,7 +413,7 @@ class KMeansCosineDistanceTest(KMeansTestBase):
     self.assertAllClose(score, self.true_score, atol=1e-2)
 
   def test_predict_kmeans_plus_plus(self):
-    # Most points are concentrated near one center. KMeans++ is likely to find
+    # Most points are concetrated near one center. KMeans++ is likely to find
     # the less populated centers.
     points = np.array(
         [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2],
@@ -604,7 +604,7 @@ class KMeansTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependent on inputs, if input
+  # Note that since cluster initialization is dependendent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 62db3bb4c4..4fe22ea26e 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -235,7 +235,7 @@ def _wals_factorization_model_function(features, labels, mode, params):
         num_items: An integer, the total number of items of this axis.
         update_fn: A function that takes one argument (`sp_input`), and that
         returns a tuple of
-          * new_factors: A float Tensor of the factor values after update.
+          * new_factors: A flot Tensor of the factor values after update.
           * update_op: a TensorFlow op which updates the factors.
           * loss: A float Tensor, the unregularized loss.
           * reg_loss: A float Tensor, the regularization loss.
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 16f80a876f..9c59150580 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -226,7 +226,6 @@ py_test(
     size = "small",
     srcs = ["python/learn/monitors_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_pip_gpu"],  # b/74437598
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index 70b70af98c..64d7ecc68e 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -243,8 +243,8 @@ def sdca_model_fn(features, labels, mode, params):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_scope(
-      values=features.values(), name_or_scope=parent_scope) as scope:
+  with variable_scope.variable_op_scope(
+      features.values(), parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
index d4e54c82f9..05794a42c5 100644
--- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
+++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
@@ -140,8 +140,8 @@ def sdca_model_fn(features, labels, mode, params, config=None):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_scope(
-      values=features.values(), name_or_scope=parent_scope) as scope:
+  with variable_scope.variable_op_scope(features.values(),
+                                        parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index c15ae3f233..2680d515eb 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -126,9 +126,6 @@ The above pre-trained models have been trained on the ImageNet data set, which c
 
 The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
 
-# Getting started with RaspberryPi
-
-Using RaspberryPi can be accomplished by following the [Makefile instructions](g3doc/rpi.md). That will give a you a static library (.a) that you can build your app against. Python bindings will be coming soon as well as a demo app.
 
 ### Train a custom model
 A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index 17b791e4e2..d7993e60cc 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -79,7 +79,6 @@ typedef enum {
   kTfLiteBuiltinBidirectionalSequenceLstm = 52,
   kTfLiteBuiltinCast = 53,
   kTfLiteBuiltinPrelu = 54,
-  kTfLiteBuiltinMaximum = 55,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 48f43d4fc4..5b393140d6 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,4 +1,4 @@
-# List of Hosted Models
+#List of Hosted Models
 
 *   [Inception V3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
 *   [Inception V3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index c423c00bf5..1450c1e14b 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -156,7 +156,6 @@ cc_library(
         "local_response_norm.cc",
         "lsh_projection.cc",
         "lstm.cc",
-        "maximum.cc",
         "mean.cc",
         "mfcc.cc",
         "mul.cc",
@@ -537,18 +536,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "maximum_test",
-    size = "small",
-    srcs = ["maximum_test.cc"],
-    deps = [
-        ":builtin_ops",
-        "//tensorflow/contrib/lite:framework",
-        "//tensorflow/contrib/lite/kernels:test_util",
-        "@com_google_googletest//:gtest",
-    ],
-)
-
 tf_cc_test(
     name = "mean_test",
     size = "small",
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 3575974ae9..33d60afa26 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -404,7 +404,6 @@ inline void DepthToSpace(const T* input_data, const Dims<4>& input_dims,
           const int in_d =
               out_d + ((out_h % block_size) * block_size + out_w % block_size) *
                           output_depth;
-
           const int in_w = out_w / block_size;
           const int in_h = out_h / block_size;
           const int in_b = out_b;
@@ -3364,30 +3363,6 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
-template <typename T>
-void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
-                       const T* input2_data, const Dims<4>& input2_dims,
-                       T* output_data, const Dims<4>& output_dims) {
-  NdArrayDesc<4> desc1;
-  NdArrayDesc<4> desc2;
-  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
-
-  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
-    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
-      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
-        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
-          auto out_idx = Offset(output_dims, c, x, y, b);
-          auto in1_idx = SubscriptToIndex(desc1, c, x, y, b);
-          auto in2_idx = SubscriptToIndex(desc2, c, x, y, b);
-          auto in1_val = input1_data[in1_idx];
-          auto in2_val = input2_data[in2_idx];
-          output_data[out_idx] = in1_val > in2_val ? in1_val : in2_val;
-        }
-      }
-    }
-  }
-}
-
 template <typename T1, typename T2, typename T3>
 void ArgMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims,
             T2* output_data, const Dims<4>& output_dims) {
diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum.cc
deleted file mode 100644
index 9fdf2b47ea..0000000000
--- a/tensorflow/contrib/lite/kernels/maximum.cc
+++ /dev/null
@@ -1,106 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <string.h>
-#include <vector>
-#include "tensorflow/contrib/lite/builtin_op_data.h"
-#include "tensorflow/contrib/lite/context.h"
-#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
-#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
-#include "tensorflow/contrib/lite/kernels/kernel_util.h"
-#include "tensorflow/contrib/lite/kernels/op_macros.h"
-
-namespace tflite {
-namespace ops {
-namespace builtin {
-namespace maximum {
-
-// This file has a reference implemenation of TFMaximum.
-enum KernelType {
-  kReference,
-};
-
-constexpr int kInputTensor1 = 0;
-constexpr int kInputTensor2 = 1;
-constexpr int kOutputTensor = 0;
-
-struct MaximumContext {
-  MaximumContext(TfLiteContext* context, TfLiteNode* node) {
-    input1 = GetInput(context, node, kInputTensor1);
-    input2 = GetInput(context, node, kInputTensor2);
-    output = GetOutput(context, node, kOutputTensor);
-  }
-  TfLiteTensor* input1;
-  TfLiteTensor* input2;
-  TfLiteTensor* output;
-};
-
-TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
-  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
-  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
-
-  MaximumContext op_context(context, node);
-  TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type);
-  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input2->dims);
-  op_context.output->type = op_context.input2->type;
-  return context->ResizeTensor(context, op_context.output, output_dims);
-}
-
-template <KernelType kernel_type>
-TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
-  MaximumContext op_context(context, node);
-
-#define TF_LITE_MAXIMUM(kernel_type, data_type)    \
-  kernel_type::TensorFlowMaximum<data_type>(       \
-      GetTensorData<data_type>(op_context.input1), \
-      GetTensorDims(op_context.input1),            \
-      GetTensorData<data_type>(op_context.input2), \
-      GetTensorDims(op_context.input2),            \
-      GetTensorData<data_type>(op_context.output), \
-      GetTensorDims(op_context.output))
-
-  if (kernel_type == kReference) {
-    switch (op_context.output->type) {
-      case kTfLiteFloat32:
-        TF_LITE_MAXIMUM(reference_ops, float);
-        break;
-      default:
-        context->ReportError(context,
-                             "Type %d is currently not supported by Maximum.",
-                             op_context.output->type);
-        return kTfLiteError;
-    }
-  } else {
-    context->ReportError(context,
-                         "Type %d is currently not supported by Maximum.",
-                         op_context.output->type);
-    return kTfLiteError;
-  }
-#undef TF_LITE_MAXIMUM
-  return kTfLiteOk;
-}
-
-}  // namespace maximum
-
-TfLiteRegistration* Register_MAXIMUM_REF() {
-  static TfLiteRegistration r = {nullptr, nullptr, maximum::Prepare,
-                                 maximum::Eval<maximum::kReference>};
-  return &r;
-}
-
-TfLiteRegistration* Register_MAXIMUM() { return Register_MAXIMUM_REF(); }
-
-}  // namespace builtin
-}  // namespace ops
-}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc
deleted file mode 100644
index b3fd7d4e6f..0000000000
--- a/tensorflow/contrib/lite/kernels/maximum_test.cc
+++ /dev/null
@@ -1,81 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <gtest/gtest.h>
-#include "tensorflow/contrib/lite/interpreter.h"
-#include "tensorflow/contrib/lite/kernels/register.h"
-#include "tensorflow/contrib/lite/kernels/test_util.h"
-#include "tensorflow/contrib/lite/model.h"
-
-namespace tflite {
-namespace {
-
-using ::testing::ElementsAreArray;
-
-class MaximumOpModel : public SingleOpModel {
- public:
-  MaximumOpModel(const TensorData& input1, const TensorData& input2,
-                 const TensorType& output) {
-    input1_ = AddInput(input1);
-    input2_ = AddInput(input2);
-    output_ = AddOutput(output);
-    SetBuiltinOp(BuiltinOperator_MAXIMUM, BuiltinOptions_MaximumOptions,
-                 CreateMaximumOptions(builder_).Union());
-    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
-  }
-
-  template <class T>
-  void SetInput1(std::initializer_list<T> data) {
-    PopulateTensor(input1_, data);
-  }
-
-  template <class T>
-  void SetInput2(std::initializer_list<T> data) {
-    PopulateTensor(input2_, data);
-  }
-
-  template <class T>
-  std::vector<T> GetOutput() {
-    return ExtractVector<T>(output_);
-  }
-  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
-
- protected:
-  int input1_;
-  int input2_;
-  int output_;
-};
-
-TEST(MaximumOpTest, FloatTest) {
-  std::initializer_list<float> data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44};
-  std::initializer_list<float> data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43};
-  MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}},
-                   {TensorType_FLOAT32, {3, 1, 2}}, TensorType_FLOAT32);
-  m.SetInput1<float>(data1);
-  m.SetInput2<float>(data2);
-  m.Invoke();
-  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2}));
-  EXPECT_THAT(
-      m.GetOutput<float>(),
-      ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43})));
-}
-
-}  // namespace
-}  // namespace tflite
-
-int main(int argc, char** argv) {
-  ::tflite::LogToStderr();
-  ::testing::InitGoogleTest(&argc, argv);
-  return RUN_ALL_TESTS();
-}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 0f98154b90..62045f0a4d 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -76,7 +76,6 @@ TfLiteRegistration* Register_LOG_SOFTMAX();
 TfLiteRegistration* Register_CAST();
 TfLiteRegistration* Register_DEQUANTIZE();
 TfLiteRegistration* Register_PRELU();
-TfLiteRegistration* Register_MAXIMUM();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -134,7 +133,6 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
   AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
-  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 791d1378f3..b7ccdf070b 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -597,9 +597,6 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
       builtin_data = reinterpret_cast<void*>(params);
       break;
     }
-    case BuiltinOperator_MAXIMUM: {
-      break;
-    }
     case BuiltinOperator_DELEGATE: {
       // TODO(ycling): Revisit when supporting saving delegated models.
       error_reporter->Report("DELEGATE op shouldn't exist in model.");
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index decaf9f160..e31b7c03a5 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -350,7 +350,6 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
       case tflite::BuiltinOperator_DELEGATE:
       case tflite::BuiltinOperator_CAST:
       case tflite::BuiltinOperator_PRELU:
-      case tflite::BuiltinOperator_MAXIMUM:
         FATAL("Op code %d is currently not delegated to NNAPI", builtin);
         nn_op_type = -1;  // set to invalid
         break;
diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index ed6dd036f9..35d224924e 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -25,9 +25,9 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import os as _os
-import subprocess as _subprocess
-import tempfile as _tempfile
+import os
+import subprocess
+import tempfile
 
 # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import convert_op_hints_to_stubs
@@ -74,7 +74,7 @@ else:
   _toco_from_proto_bin = _resource_loader.get_path_to_datafile(
       "../toco/python/toco_from_protos")
 
-if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin):
+if _toco_from_proto_bin and not os.path.exists(_toco_from_proto_bin):
   _toco_from_proto_bin = "toco_from_protos"
 
 
@@ -102,10 +102,10 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
     return _toco_python.TocoConvert(
         model_flags_str, toco_flags_str, input_data_str)
 
-  with _tempfile.NamedTemporaryFile() as fp_toco, \
-           _tempfile.NamedTemporaryFile() as fp_model, \
-           _tempfile.NamedTemporaryFile() as fp_input, \
-           _tempfile.NamedTemporaryFile() as fp_output:
+  with tempfile.NamedTemporaryFile() as fp_toco, \
+           tempfile.NamedTemporaryFile() as fp_model, \
+           tempfile.NamedTemporaryFile() as fp_input, \
+           tempfile.NamedTemporaryFile() as fp_output:
     fp_model.write(model_flags_str)
     fp_toco.write(toco_flags_str)
     fp_input.write(input_data_str)
@@ -118,11 +118,11 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
         fp_output.name
     ]
     cmdline = " ".join(cmd)
-    proc = _subprocess.Popen(
+    proc = subprocess.Popen(
         cmdline,
         shell=True,
-        stdout=_subprocess.PIPE,
-        stderr=_subprocess.STDOUT,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT,
         close_fds=True)
     stdout, stderr = proc.communicate()
     exitcode = proc.returncode
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 7d2e00fe32..e1075971e9 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -131,7 +131,6 @@ enum BuiltinOperator : byte {
   BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   CAST = 53,
   PRELU = 54,
-  MAXIMUM = 55,
 }
 
 // Options for the builtin operators.
@@ -174,7 +173,6 @@ union BuiltinOptions {
   LogSoftmaxOptions,
   CastOptions,
   DequantizeOptions,
-  MaximumOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -386,9 +384,6 @@ table CastOptions {
 table DequantizeOptions {
 }
 
-table MaximumOptions {
-}
-
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 66a97a1460..86daeaf5cc 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -145,9 +145,6 @@ struct CastOptionsT;
 struct DequantizeOptions;
 struct DequantizeOptionsT;
 
-struct MaximumOptions;
-struct MaximumOptionsT;
-
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -258,12 +255,11 @@ enum BuiltinOperator {
   BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   BuiltinOperator_CAST = 53,
   BuiltinOperator_PRELU = 54,
-  BuiltinOperator_MAXIMUM = 55,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_MAXIMUM
+  BuiltinOperator_MAX = BuiltinOperator_PRELU
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -317,8 +313,7 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] {
     BuiltinOperator_DELEGATE,
     BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
     BuiltinOperator_CAST,
-    BuiltinOperator_PRELU,
-    BuiltinOperator_MAXIMUM
+    BuiltinOperator_PRELU
   };
   return values;
 }
@@ -380,7 +375,6 @@ inline const char **EnumNamesBuiltinOperator() {
     "BIDIRECTIONAL_SEQUENCE_LSTM",
     "CAST",
     "PRELU",
-    "MAXIMUM",
     nullptr
   };
   return names;
@@ -431,12 +425,11 @@ enum BuiltinOptions {
   BuiltinOptions_LogSoftmaxOptions = 36,
   BuiltinOptions_CastOptions = 37,
   BuiltinOptions_DequantizeOptions = 38,
-  BuiltinOptions_MaximumOptions = 39,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_MaximumOptions
+  BuiltinOptions_MAX = BuiltinOptions_DequantizeOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[39] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -476,8 +469,7 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] {
     BuiltinOptions_SplitOptions,
     BuiltinOptions_LogSoftmaxOptions,
     BuiltinOptions_CastOptions,
-    BuiltinOptions_DequantizeOptions,
-    BuiltinOptions_MaximumOptions
+    BuiltinOptions_DequantizeOptions
   };
   return values;
 }
@@ -523,7 +515,6 @@ inline const char **EnumNamesBuiltinOptions() {
     "LogSoftmaxOptions",
     "CastOptions",
     "DequantizeOptions",
-    "MaximumOptions",
     nullptr
   };
   return names;
@@ -690,10 +681,6 @@ template<> struct BuiltinOptionsTraits<DequantizeOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_DequantizeOptions;
 };
 
-template<> struct BuiltinOptionsTraits<MaximumOptions> {
-  static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions;
-};
-
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1029,14 +1016,6 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_DequantizeOptions ?
       reinterpret_cast<const DequantizeOptionsT *>(value) : nullptr;
   }
-  MaximumOptionsT *AsMaximumOptions() {
-    return type == BuiltinOptions_MaximumOptions ?
-      reinterpret_cast<MaximumOptionsT *>(value) : nullptr;
-  }
-  const MaximumOptionsT *AsMaximumOptions() const {
-    return type == BuiltinOptions_MaximumOptions ?
-      reinterpret_cast<const MaximumOptionsT *>(value) : nullptr;
-  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -3780,46 +3759,6 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(
 
 flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffers::FlatBufferBuilder &_fbb, const DequantizeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
-struct MaximumOptionsT : public flatbuffers::NativeTable {
-  typedef MaximumOptions TableType;
-  MaximumOptionsT() {
-  }
-};
-
-struct MaximumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
-  typedef MaximumOptionsT NativeTableType;
-  bool Verify(flatbuffers::Verifier &verifier) const {
-    return VerifyTableStart(verifier) &&
-           verifier.EndTable();
-  }
-  MaximumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
-  void UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
-  static flatbuffers::Offset<MaximumOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
-};
-
-struct MaximumOptionsBuilder {
-  flatbuffers::FlatBufferBuilder &fbb_;
-  flatbuffers::uoffset_t start_;
-  explicit MaximumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
-        : fbb_(_fbb) {
-    start_ = fbb_.StartTable();
-  }
-  MaximumOptionsBuilder &operator=(const MaximumOptionsBuilder &);
-  flatbuffers::Offset<MaximumOptions> Finish() {
-    const auto end = fbb_.EndTable(start_);
-    auto o = flatbuffers::Offset<MaximumOptions>(end);
-    return o;
-  }
-};
-
-inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(
-    flatbuffers::FlatBufferBuilder &_fbb) {
-  MaximumOptionsBuilder builder_(_fbb);
-  return builder_.Finish();
-}
-
-flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
-
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -4051,9 +3990,6 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const DequantizeOptions *builtin_options_as_DequantizeOptions() const {
     return builtin_options_type() == BuiltinOptions_DequantizeOptions ? static_cast<const DequantizeOptions *>(builtin_options()) : nullptr;
   }
-  const MaximumOptions *builtin_options_as_MaximumOptions() const {
-    return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast<const MaximumOptions *>(builtin_options()) : nullptr;
-  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -4232,10 +4168,6 @@ template<> inline const DequantizeOptions *Operator::builtin_options_as<Dequanti
   return builtin_options_as_DequantizeOptions();
 }
 
-template<> inline const MaximumOptions *Operator::builtin_options_as<MaximumOptions>() const {
-  return builtin_options_as_MaximumOptions();
-}
-
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -5764,29 +5696,6 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffer
       _fbb);
 }
 
-inline MaximumOptionsT *MaximumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
-  auto _o = new MaximumOptionsT();
-  UnPackTo(_o, _resolver);
-  return _o;
-}
-
-inline void MaximumOptions::UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
-  (void)_o;
-  (void)_resolver;
-}
-
-inline flatbuffers::Offset<MaximumOptions> MaximumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
-  return CreateMaximumOptions(_fbb, _o, _rehasher);
-}
-
-inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
-  (void)_rehasher;
-  (void)_o;
-  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
-  return tflite::CreateMaximumOptions(
-      _fbb);
-}
-
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -6119,10 +6028,6 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
-    case BuiltinOptions_MaximumOptions: {
-      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
-      return verifier.VerifyTable(ptr);
-    }
     default: return false;
   }
 }
@@ -6293,10 +6198,6 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return ptr->UnPack(resolver);
     }
-    case BuiltinOptions_MaximumOptions: {
-      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
-      return ptr->UnPack(resolver);
-    }
     default: return nullptr;
   }
 }
@@ -6455,10 +6356,6 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const DequantizeOptionsT *>(value);
       return CreateDequantizeOptions(_fbb, ptr, _rehasher).Union();
     }
-    case BuiltinOptions_MaximumOptions: {
-      auto ptr = reinterpret_cast<const MaximumOptionsT *>(value);
-      return CreateMaximumOptions(_fbb, ptr, _rehasher).Union();
-    }
     default: return 0;
   }
 }
@@ -6617,10 +6514,6 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new DequantizeOptionsT(*reinterpret_cast<DequantizeOptionsT *>(u.value));
       break;
     }
-    case BuiltinOptions_MaximumOptions: {
-      value = new MaximumOptionsT(*reinterpret_cast<MaximumOptionsT *>(u.value));
-      break;
-    }
     default:
       break;
   }
@@ -6818,11 +6711,6 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
-    case BuiltinOptions_MaximumOptions: {
-      auto ptr = reinterpret_cast<MaximumOptionsT *>(value);
-      delete ptr;
-      break;
-    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 12b7b3c350..555ea90034 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -36,7 +36,6 @@ gen_zipped_test_files(
         "local_response_norm.zip",
         "log_softmax.zip",
         "max_pool.zip",
-        "maximum.zip",
         "mean.zip",
         "mul.zip",
         "pad.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index 8045052452..cb5c500136 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -862,41 +862,6 @@ def make_log_softmax_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
-def make_maximum_tests(zip_path):
-  """Make a set of tests to do maximum."""
-
-  test_parameters = [{
-      "input_dtype": [tf.float32],
-      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
-      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
-  }]
-
-  def build_graph(parameters):
-    """Build the maximum op testing graph."""
-    input_tensor_1 = tf.placeholder(
-        dtype=parameters["input_dtype"],
-        name="input_1",
-        shape=parameters["input_shape_1"])
-    input_tensor_2 = tf.placeholder(
-        dtype=parameters["input_dtype"],
-        name="input_2",
-        shape=parameters["input_shape_2"])
-
-    out = tf.maximum(input_tensor_1, input_tensor_2)
-    return [input_tensor_1, input_tensor_2], [out]
-
-  def build_inputs(parameters, sess, inputs, outputs):
-    values = [
-        create_tensor_data(parameters["input_dtype"],
-                           parameters["input_shape_1"]),
-        create_tensor_data(parameters["input_dtype"],
-                           parameters["input_shape_2"])
-    ]
-    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
-
-  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
-
-
 def make_binary_op_tests_func(binary_operator):
   """Return a function that does a test on a binary operator."""
   return lambda zip_path: make_binary_op_tests(zip_path, binary_operator)
@@ -2012,7 +1977,6 @@ def main(unused_args):
         "exp.zip": make_exp_tests,
         "log_softmax.zip": make_log_softmax_tests,
         "lstm.zip": make_lstm_tests,
-        "maximum.zip": make_maximum_tests,
     }
     out = FLAGS.zip_to_output
     bin_path = FLAGS.toco
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index 6697b86e79..a4a7283508 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -253,7 +253,6 @@ INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
-INSTANTIATE_TESTS(maximum)
 INSTANTIATE_TESTS(max_pool)
 INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(mul)
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index 0989bfe5a3..f23249cfa1 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -863,8 +863,6 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
   ops.emplace_back(new SimpleOperator<ExpOperator>("EXP", OperatorType::kExp));
   ops.emplace_back(new SimpleOperator<LogSoftmaxOperator>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
-  ops.emplace_back(new SimpleOperator<TensorFlowMaximumOperator>(
-      "MAXIMUM", OperatorType::kTensorFlowMaximum));
 
   return ops;
 }
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index f7a213ecfc..9c19f8d464 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -109,8 +109,6 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<ExpOperator>("EXP", OperatorType::kExp);
   CheckSimpleOperator<LogSoftmaxOperator>("LOG_SOFTMAX",
                                           OperatorType::kLogSoftmax);
-  CheckSimpleOperator<TensorFlowMaximumOperator>(
-      "MAXIMUM", OperatorType::kTensorFlowMaximum);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index a03e731be3..a57a1e5421 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -494,7 +494,7 @@ class MutableDenseHashTable(LookupInterface):
                                                   value_dtype=tf.int64,
                                                   default_value=-1,
                                                   empty_key=0)
-  sess.run(table.insert(keys, values))
+  table.insert(keys, values)
   out = table.lookup(query_keys)
   print(out.eval())
   ```
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 8b415e6527..4ae18b2cef 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -34,7 +34,7 @@ PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.
 RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
 FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)"
-CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
+CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 7a7683c953..5a812af4e9 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -258,7 +258,6 @@ tensorflow/core/kernels/requantize.cc
 tensorflow/core/kernels/remote_fused_graph_execute_op.cc
 tensorflow/core/kernels/remote_fused_graph_execute_utils.cc
 tensorflow/core/kernels/batch_matmul_op_real.cc
-tensorflow/core/kernels/random_op.cc
 tensorflow/core/ops/training_ops.cc
 tensorflow/core/ops/string_ops.cc
 tensorflow/core/ops/state_ops.cc
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index a9a32b7b25..dfa12e873a 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -74,7 +74,7 @@ class GatherTreeOp : public OpKernel {
         ctx,
         step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0),
         errors::InvalidArgument("batch size dimensions step_ids.shape[1] and "
-                                "max_sequence_lengths.shape[0] must match.  "
+                                "max_seqeuence_lengths.shape[0] must match.  "
                                 "but shapes are: ",
                                 step_ids_shape.DebugString(), " and ",
                                 max_sequence_lengths.shape().DebugString()));
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index be53779826..9ff8a343f1 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -736,7 +736,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
   """Base attention mechanism for monotonic attention.
 
   Simply overrides the initial_alignments function to provide a dirac
-  distribution, which is needed in order for the monotonic attention
+  distribution,which is needed in order for the monotonic attention
   distributions to have the correct behavior.
   """
 
@@ -763,7 +763,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
-  This type of attention enforces a monotonic constraint on the attention
+  This type of attention encorces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -867,7 +867,7 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
-  This type of attention enforces a monotonic constraint on the attention
+  This type of attention encorces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -1133,7 +1133,7 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
       output_attention: Python bool.  If `True` (default), the output at each
         time step is the attention value.  This is the behavior of Luong-style
         attention mechanisms.  If `False`, the output at each time step is
-        the output of `cell`.  This is the behavior of Bhadanau-style
+        the output of `cell`.  This is the beahvior of Bhadanau-style
         attention mechanisms.  In both cases, the `attention` tensor is
         propagated to the next time step via the state and is used there.
         This flag only controls whether the attention mechanism is propagated
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index 184144f64a..a26107b0d7 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -821,9 +821,9 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight):
   Returns:
     The scores normalized by the length_penalty.
   """
-  length_penalty_ = _length_penalty(
+  length_penality_ = _length_penalty(
       sequence_lengths=sequence_lengths, penalty_factor=length_penalty_weight)
-  return log_probs / length_penalty_
+  return log_probs / length_penality_
 
 
 def _length_penalty(sequence_lengths, penalty_factor):
@@ -860,7 +860,7 @@ def _mask_probs(probs, eos_token, finished):
   unfinished beams remain unchanged.
 
   Args:
-    probs: Log probabilities of shape `[batch_size, beam_width, vocab_size]`
+    probs: Log probabiltiies of shape `[batch_size, beam_width, vocab_size]`
     eos_token: An int32 id corresponding to the EOS token to allocate
       probability to.
     finished: A boolean tensor of shape `[batch_size, beam_width]` that
diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
index 99ad487630..b3343aef47 100644
--- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
+++ b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
@@ -115,8 +115,8 @@ class ParallelReader(io_ops.ReaderBase):
     reader needs to start reading from a new file since it has finished with
     the previous file).
 
-    A queue runner for enqueuing in the `common_queue` is automatically added
-    to the TF QueueRunners collection.
+    A queue runner for enqueing in the `common_queue` is automatically added to
+    the TF QueueRunners collection.
 
     Args:
       queue: A Queue or a mutable string Tensor representing a handle
diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
index 62bd200361..37e9c4754c 100644
--- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
+++ b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
@@ -36,9 +36,9 @@ def prefetch_queue(tensors,
                    dynamic_pad=False,
                    shared_name=None,
                    name=None):
-  """Creates a queue to prefetch tensors from `tensors`.
+  """Creates a queue to prefetech tensors from `tensors`.
 
-  A queue runner for enqueuing tensors into the prefetch_queue is automatically
+  A queue runner for enqueing tensors into the prefetch_queue is automatically
   added to the TF QueueRunners collection.
 
   Example:
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index f2d31dc8db..b3b61e1dfe 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -124,7 +124,7 @@ class BoundingBox(ItemHandler):
     super(BoundingBox, self).__init__(self._full_keys)
 
   def tensors_to_item(self, keys_to_tensors):
-    """Maps the given dictionary of tensors to a concatenated list of bboxes.
+    """Maps the given dictionary of tensors to a contatenated list of bboxes.
 
     Args:
       keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index 6eafc1754c..461e627e99 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -1,15 +1,15 @@
-# Using TensorRT in TensorFlow
-
+Using TensorRT in TensorFlow
+============================
 
 This module provides necessary bindings and introduces TRT_engine_op
 operator that wraps a subgraph in TensorRT. This is still a work in progress
 but should be useable with most common graphs.
 
-## Compilation
-
+Compilation
+-----------
 
 In order to compile the module, you need to have a local TensorRT
-installation ( libnvinfer.so and respective include files ). During the
+installation (libnvinfer.so and respective include files). During the
 configuration step, TensorRT should be enabled and installation path
 should be set. If installed through package managers (deb,rpm),
 configure script should find the necessary components from the system
@@ -22,38 +22,4 @@ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use can be found in test/test_tftrt.py script
-
-## Installing TensorRT 3.0.4
-
-In order to make use of TensorRT integration, you will need a local installation of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). Due to compiler compatibility, you will need to download and install the TensorRT 3.0.4 tarball for _Ubuntu 14.04_, i.e., **_TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz_**, even if you are using Ubuntu 16.04 or later.
-
-### Preparing TensorRT installation
-
-Once you have downloaded TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz, you will need to unpack it to an installation directory, which will be referred to as <install_dir>. Please replace <install_dir> with the full path of actual installation directory you choose in commands below.
-
-```shell
-cd <install_dir> && tar -zxf /path/to/TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz
-```
-
-After unpacking the binaries, you have several options to use them:
-
-#### To run TensorFlow as a user without superuser privileges
-
-For a regular user without any sudo rights, you should add TensorRT to your `$LD_LIBRARY_PATH`:
-
-  ```shell
-   export LD_LIBRARY_PATH=<install_dir>/TensorRT-3.0.4/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
-  ```
-
-Then you are ready to use TensorFlow-TensorRT integration. `$LD_LIBRARY_PATH` must contain the path to TensorRT installation for TensorFlow-TensorRT integration to work. If you are using a VirtualEnv-like setup, you can add the command above to your `bin/activate` script or to your `.bashrc` script.
-
-#### To run TensorFlow as a superuser
-
- When running as a superuser, such as in a container or via sudo, the `$LD_LIBRARY_PATH` approach above may not work. The following is preferred when the user has superuser privileges:
-
-  ```shell
-  echo "<install_dir>/TensorRT-3.0.4/lib" | sudo tee /etc/ld.so.conf.d/tensorrt304.conf && sudo ldconfig
-  ```
-
-  Please ensure that any existing deb package installation of TensorRT is removed before following these instructions to avoid package conflicts.
\ No newline at end of file
+will be available. An example use can be found in test/test_tftrt.py directory
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index ff8cc6374d..eea8c8efa2 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,13 +49,12 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::Node* node) {
+bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
       "Identity",
-      "Snapshot",
       "Const",
       "Conv2D",
       "MaxPool",
@@ -75,7 +74,7 @@ bool IsTensorRTCandidate(const tensorflow::Node* node) {
       // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
-  return candidate_ops.count(node->type_string());
+  return candidate_ops.count(node_def.op());
 }
 
 void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
@@ -85,10 +84,10 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->in_edges()) {
       if (!subgraph_node_ids.count(edge->src()->id()) &&
-          !edge->src()->IsSource() && !edge->IsControlEdge()) {
+          !edge->src()->IsSource()) {
         incoming_edges->insert(edge);
       } else {
-        VLOG(2) << node->name() << " -> " << edge->src()->name() << " N, ";
+        VLOG(2) << edge->src()->name() << " N, ";
       }
     }
   }
@@ -101,11 +100,11 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
-          !edge->dst()->IsSink() && !edge->IsControlEdge()) {
-        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " Y, ";
+          !edge->dst()->IsSink()) {
+        VLOG(2) << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
       } else {
-        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " N, ";
+        VLOG(2) << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -410,9 +409,8 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
       if (status != tensorflow::Status::OK()) {
         LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \"" << status.ToString()
-                     << "\" SKIPPING......( " << subgraph_node_names.size()
-                     << " nodes)";
+                     << " due to: \n"
+                     << status.ToString() << " SKIPPING......";
       }
       count++;
     }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 370911e4d9..92a692baa7 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -53,8 +53,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
-using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
+
 namespace {
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
@@ -430,8 +430,9 @@ class Converter {
   tensorflow::tensorrt::TRTWeightStore* weight_store_;
   bool fp16_;
   void register_op_converters();
-  tensorflow::Status get_inputs(const tensorflow::NodeDef& node_def,
-                                std::vector<TRT_TensorOrWeights>* inputs) {
+  std::vector<TRT_TensorOrWeights> get_inputs(
+      const tensorflow::NodeDef& node_def) {
+    std::vector<TRT_TensorOrWeights> inputs;
     for (auto const& input_name : node_def.input()) {
       /*************************************************************************
        * TODO(jie) handle case 1) here
@@ -452,17 +453,13 @@ class Converter {
 
       VLOG(2) << "retrieve input: " << name;
       if (trt_tensors_.count(name)) {
-        inputs->push_back(trt_tensors_.at(name));
+        inputs.push_back(trt_tensors_.at(name));
       } else {
-        string str("Node ");
-        StrAppend(&str, node_def.name(), " should have an input named '", name,
-                  "' but it is not available");
-        LOG(WARNING) << "input: " << name << " not available for node at "
-                     << node_def.name();
-        return tensorflow::errors::InvalidArgument(str);
+        LOG(FATAL) << "input: " << name << " not available for node at, "
+                   << node_def.name();
       }
     }
-    return tensorflow::Status::OK();
+    return inputs;
   }
 
  public:
@@ -486,8 +483,7 @@ class Converter {
   }
 
   tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs;
-    TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs));
+    std::vector<TRT_TensorOrWeights> inputs = this->get_inputs(node_def);
     string op = node_def.op();
     if (!op_registry_.count(op)) {
       return tensorflow::errors::Unimplemented(
@@ -552,19 +548,6 @@ class Converter {
   }
 };
 
-TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx,
-                                    const TRT_ShapedWeights& weights_src) {
-  auto dtype_new = tensorflow::DataType::DT_HALF;
-  TRT_ShapedWeights weights =
-      ctx.get_temp_weights(dtype_new, weights_src.shape_);
-  const float* src = static_cast<const float*>(weights_src.GetValues());
-  Eigen::half* dst = const_cast<Eigen::half*>(
-      static_cast<Eigen::half const*>(weights.GetValues()));
-  for (int64_t i = 0; i < weights_src.count(); i++) {
-    dst[i] = Eigen::half_impl::float_to_half_rtne(src[i]);
-  }
-  return weights;
-}
 // ****************************************************************************
 // Constant folding functions
 // TODO(jie): once optimizer kicks in, we should have done constant folding
@@ -892,7 +875,7 @@ tensorflow::Status BinaryTensorOpWeight(
 
   // Check type consistency
   nvinfer1::DataType ttype;
-  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
+  TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
 
   // Check scale mode
   auto dims_w = weights.shape_;
@@ -974,10 +957,6 @@ tensorflow::Status BinaryTensorOpWeight(
     }
   }
 
-  if (ctx.isFP16()) {
-    weights = ConvertFP32ToFP16(ctx, weights);
-  }
-
   // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
@@ -1019,7 +998,9 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
     const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs, int group) {
+    std::vector<TRT_TensorOrWeights>* outputs,
+    int group  // group ==0 specifies depthwise conv
+) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   TFAttrs attrs(node_def);
@@ -1044,10 +1025,6 @@ tensorflow::Status ConvertConv2DHelper(
   VLOG(2) << "groups count: " << num_groups;
 
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
-  if (ctx.isFP16()) {
-    weights_rsck = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
-  }
-
   TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
   TRT_ShapedWeights biases(weights.type_);
@@ -1157,9 +1134,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented(
-        "binary op: " + node_def.op() +
-        " not supported at: " + node_def.name());
+    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
+                                             " not supported at: " +
+                                             node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1318,11 +1295,8 @@ tensorflow::Status ConvertScale(Converter& ctx,
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
+  // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
-  if (ctx.isFP16()) {
-    weights = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
-  }
-
   TRT_ShapedWeights empty_weights(weights.type_);
 
   TFAttrs attrs(node_def);
@@ -1402,11 +1376,8 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.float_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
-                       << " kUNIFORM, at: " << node_def.name();
-          string err_str("Broadcast method is not supported for '");
-          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
-          return tensorflow::errors::InvalidArgument(err_str);
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
         }
       }
     } else {
@@ -1420,16 +1391,33 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    size_t len_data = tensorflow::DataTypeSize(dtype);
-    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
-    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-    std::vector<float> tensor_data(
-        weights_tensor.float_val().begin(),
-        weights_tensor.float_val()
-            .end());  //  make a local copy first to flatten
-    memcpy(dst, tensor_data.data(), len_data);  // store into weight store
-    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      auto half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      half_tensor.device(defd) =
+          tensor.flat<float>().template cast<Eigen::half>();
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<float> tensor_data(
+          weights_tensor.float_val().begin(),
+          weights_tensor.float_val()
+              .end());  //  make a local copy first to flatten
+      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    }
   } else if (!weights_tensor.int_val().empty()) {
     VLOG(2) << "int!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
@@ -1444,11 +1432,8 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.int_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
-                       << " kUNIFORM, at: " << node_def.name();
-          string err_str("Broadcast method is not supported for '");
-          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
-          return tensorflow::errors::InvalidArgument(err_str);
+          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
+                     << " kUNIFORM, at: " << node_def.name();
         }
       }
     } else {
@@ -1462,23 +1447,62 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    //  we should not have converted //if (ctx.isFP16()) {
-    size_t len_data = tensorflow::DataTypeSize(dtype);
-    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
-    size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
-    len_data = std::max(len_data, len_tensor);
-    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-    std::vector<int32> tensor_data(
-        weights_tensor.int_val().begin(),
-        weights_tensor.int_val().end());  //  make a local copy first to flatten
-                                          //  doesn't have to be contigous
-    memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
-    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    if (ctx.isFP16()) {
+      auto dtype_new = tensorflow::DataType::DT_HALF;
+      size_t len_data = tensorflow::DataTypeSize(dtype_new);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
+      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
+      Eigen::DefaultDevice defd;
+      switch (dtype) {
+        case (tensorflow::DT_INT32): {
+          half_tensor.device(defd) =
+              tensor.flat<int32>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT16): {
+          half_tensor.device(defd) =
+              tensor.flat<int16>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_INT8): {
+          half_tensor.device(defd) =
+              tensor.flat<int8>().template cast<Eigen::half>();
+          break;
+        }
+        case (tensorflow::DT_UINT8): {
+          half_tensor.device(defd) =
+              tensor.flat<uint8>().template cast<Eigen::half>();
+          break;
+        }
+        default:
+          return tensorflow::errors::InvalidArgument(
+              "Datatype " + tensorflow::DataTypeString(dtype) +
+              " for FP16 conversion");
+          break;
+      };
+      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
+      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
+    } else {
+      size_t len_data = tensorflow::DataTypeSize(dtype);
+      for (int i = 0; i < scalar_shape.nbDims; i++)
+        len_data *= scalar_shape.d[i];
+      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+      len_data = std::max(len_data, len_tensor);
+      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+      std::vector<int32> tensor_data(
+          weights_tensor.int_val().begin(),
+          weights_tensor.int_val()
+              .end());  //  make a local copy first to flatten
+                        //  doesn't have to be contiguous
+      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
+    }
   } else if (!weights_tensor.tensor_content().empty()) {
-    //  obsolete method.
-    //  After optimization path, we do not see weights in this format.
-    //  fp16 conversion technically should be needed here.
     VLOG(2) << "TENSOR!!!" << node_def.name();
     const auto& content = weights_tensor.tensor_content();
 
@@ -1760,6 +1784,8 @@ tensorflow::Status ConvertConcat(Converter& ctx,
   TRT_ShapedWeights axis = inputs.at(input_size).weights();
 
   TFAttrs attrs(node_def);
+  // auto attr_size = attrs.at("N")->i();
+  // auto data_type = attrs.get<nvinfer1::DataType>("T");
   auto index_type = attrs.get<tensorflow::DataType>("Tidx");
 
   // TODO(jie): handle data type
@@ -1849,103 +1875,71 @@ tensorflow::Status ConvertFusedBatchNorm(
         "only is_training=false is supported, at " + node_def.name());
   }
   nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-
-  //  Check parameter types
-  auto parameter_type = inputs.at(1).weights().type_;
-  if ((parameter_type != tensorflow::DataType::DT_FLOAT) &&
-      (parameter_type != tensorflow::DataType::DT_HALF)) {
-    return tensorflow::errors::Unimplemented(
-        "only float32 or float16 weight data type is supported, for node " +
-        node_def.name() + " got " + tensorflow::DataTypeString(parameter_type));
-  }
-  for (int i = 1; i < 5; i++) {
-    if (inputs.at(i).weights().type_ != parameter_type) {
-      return tensorflow::errors::Unimplemented(
-          "Inconsistent parameter type for batchnormis not supported, at: " +
-          node_def.name());
-    }
-  }
-
-  TRT_ShapedWeights dummy_power_weights(parameter_type);
-  size_t nweight = 0;
-  for (int i = 1; i < 5; i++) {
-    nweight = std::max(nweight, (size_t)inputs.at(i).weights().count());
-  }
-  TRT_ShapedWeights* ptr_shape_weights = nullptr;
-  for (int i = 1; i < 5; i++) {
-    if (inputs.at(i).weights().count() == nweight) {
-      ptr_shape_weights =
-          const_cast<TRT_ShapedWeights*>(&(inputs.at(i).weights()));
-    } else if (inputs.at(i).weights().count() != 1) {
-      return tensorflow::errors::InvalidArgument(
-          "Inconsistent batchnorm parameter count, at: " + node_def.name());
-    }
-  }
-  //  We could technically have two weights with different shape.
-  //  that requires two addScale op, arguably less performant
+  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
+  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
+  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
+  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
+  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
   TRT_ShapedWeights combined_scale_weights =
-      ctx.get_temp_weights_like(*ptr_shape_weights);
+      ctx.get_temp_weights_like(scale_weights);
   TRT_ShapedWeights combined_offset_weights =
-      ctx.get_temp_weights_like(*ptr_shape_weights);
-
-  const Eigen::half* cast_vals_array[4];
-  const float* vals_array[4];
-  for (int j = 0; j < 4; j++) {
-    cast_vals_array[j] =
-        static_cast<Eigen::half const*>(inputs.at(j + 1).weights().GetValues());
-    vals_array[j] =
-        static_cast<float const*>(inputs.at(j + 1).weights().GetValues());
-  }
-  Eigen::half* cast_combined_scale_vals = const_cast<Eigen::half*>(
-      static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
-  Eigen::half* cast_combined_offset_vals = const_cast<Eigen::half*>(
-      static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
-  float* combined_scale_vals = const_cast<float*>(
-      static_cast<float const*>(combined_scale_weights.GetValues()));
-  float* combined_offset_vals = const_cast<float*>(
-      static_cast<float const*>(combined_offset_weights.GetValues()));
-
-  for (size_t i = 0; i < nweight; ++i) {
-    float batchnorm_data[4];
-    for (int j = 0; j < 4; j++) {
-      if (inputs.at(j + 1).weights().count() != 1) {
-        if (parameter_type == tensorflow::DT_FLOAT) {
-          batchnorm_data[j] = vals_array[j][i];
-        } else if (parameter_type == tensorflow::DT_HALF) {
-          batchnorm_data[j] =
-              Eigen::half_impl::half_to_float(cast_vals_array[j][i]);
-        }
-      } else {
-        if (parameter_type == tensorflow::DT_FLOAT) {
-          batchnorm_data[j] = vals_array[j][0];
-        } else if (parameter_type == tensorflow::DT_HALF) {
-          batchnorm_data[j] =
-              Eigen::half_impl::half_to_float(cast_vals_array[j][0]);
-        }
-      }
+      ctx.get_temp_weights_like(offset_weights);
+  size_t nweight = scale_weights.count();
+  if ((scale_weights.type_ == offset_weights.type_) &&
+      (mean_weights.type_ == variance_weights.type_) &&
+      (scale_weights.type_ == variance_weights.type_)) {
+    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
+        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+      return tensorflow::errors::Unimplemented(
+          "only float32 or float16 weight data type is supported, for node " +
+          node_def.name() + " got " +
+          tensorflow::DataTypeString(scale_weights.type_));
     }
-    float scale = batchnorm_data[0];
-    float offset = batchnorm_data[1];
-    float mean = batchnorm_data[2];
-    float variance = batchnorm_data[3];
-    float combined_scale_val = scale / sqrtf(variance + epsilon);
-    float combined_offset_val = offset - mean * combined_scale_val;
-    if (parameter_type == tensorflow::DT_FLOAT) {
-      combined_scale_vals[i] = combined_scale_val;
-      combined_offset_vals[i] = combined_offset_val;
-    } else if (parameter_type == tensorflow::DT_HALF) {
-      cast_combined_scale_vals[i] = Eigen::half(combined_scale_val);
-      cast_combined_offset_vals[i] = Eigen::half(combined_offset_val);
+    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
+        float offset =
+            (static_cast<float const*>(offset_weights.GetValues()))[i];
+        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
+        float variance =
+            (static_cast<float const*>(variance_weights.GetValues()))[i];
+        float& combined_scale_ref = const_cast<float*>(
+            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
+        float& combined_offset_ref = const_cast<float*>(
+            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
+        combined_scale_ref = scale / sqrtf(variance + epsilon);
+        combined_offset_ref = offset - mean * combined_scale_ref;
+      }
+    } else {
+      const Eigen::half* scale_vals =
+          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
+      const Eigen::half* off_vals =
+          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
+      const Eigen::half* mean_vals =
+          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
+      const Eigen::half* variance_vals =
+          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
+      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
+          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+      for (size_t i = 0; i < nweight; ++i) {
+        float scale(scale_vals[i]);
+        float offset(off_vals[i]);
+        float mean(mean_vals[i]);
+        float variance(variance_vals[i]);
+        float combined_scale_ref = scale / sqrtf(variance + epsilon);
+        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
+        float combined_offset_ref = offset - mean * combined_scale_ref;
+        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+      }
     }
   }
-
-  nvinfer1::ScaleMode mode = nweight == 1 ? nvinfer1::ScaleMode::kUNIFORM
-                                          : nvinfer1::ScaleMode::kCHANNEL;
-  nvinfer1::IScaleLayer* layer =
-      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
-                              combined_offset_weights.GetWeightsForTRT(),
-                              combined_scale_weights.GetWeightsForTRT(),
-                              dummy_power_weights.GetWeightsForTRT());
+  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
+      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
+      combined_offset_weights.GetWeightsForTRT(),
+      combined_scale_weights.GetWeightsForTRT(),
+      dummy_power_weights.GetWeightsForTRT());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -2056,7 +2050,6 @@ void Converter::register_op_converters() {
   op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
-  op_registry_["Snapshot"] = ConvertIdentity;  // Snapshot should be removed
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -2150,11 +2143,8 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
   calib_res->thr_->join();
   delete calib_res->thr_;
   if (!calib_res->engine_) {
-    LOG(ERROR) << "Calibration failed!, engine does not exist. Did you run "
+    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
                   "calibration graph?";
-    return tensorflow::errors::FailedPrecondition(
-        "Calibration graph needs to be executed on"
-        " calibration data before convertsion to inference graph");
   }
   auto weight_rmgr = trt_rm->getManager("WeightStore");
   TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
@@ -2191,7 +2181,7 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
     return status;
   }
   auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_RETURN_IF_ERROR(status);
+  TF_CHECK_OK(status);
   for (size_t i = 0; i < out_edges.size(); i++) {
     VLOG(1) << "Connecting trt_engine_node output " << i << " with "
             << out_edges.at(i)->dst()->name() << " port "
@@ -2289,12 +2279,6 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    auto type_status = ConvertDType(tf_dtype, &dtype);
-    if (type_status != tensorflow::Status::OK()) {
-      LOG(WARNING) << "Data type conversion for input '" << node_name
-                   << "' failed";
-      return type_status;
-    }
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "accessing output index of: " << output_idx
@@ -2362,8 +2346,8 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument("Output node'" + tensor_name +
-                                                 "' is weights not tensor");
+      return tensorflow::errors::InvalidArgument(
+          "Output node is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2520,11 +2504,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    auto type_status = ConvertDType(tf_dtype, &dtype);
-    if (type_status != tensorflow::Status::OK()) {
-      LOG(WARNING) << "Type conversion failed for " << node_name;
-      return type_status;
-    }
+    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
@@ -2535,12 +2515,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
     //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4) {
-      string err_str = "Require 4 dimensional input.";
-      StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ",
-                shape_inference_node_name);
-      return tensorflow::errors::Unimplemented(err_str);
-    }
+    if (op_info.shape().dim_size() != 4)
+      return tensorflow::errors::Unimplemented("require 4 dimensional input");
 
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
@@ -2601,8 +2577,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument("Output node '" + tensor_name +
-                                                 "' is weights not tensor");
+      return tensorflow::errors::InvalidArgument(
+          "Output node is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2646,8 +2622,7 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
   TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
       engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name << " containing "
-            << s.subgraph_node_ids.size() << " nodes";
+  LOG(INFO) << "finished engine " << engine_name;
 
   // Build the TRT op
   tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 8fc4697c51..6193f0b0a1 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -80,20 +80,13 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> in_edges(dst->in_edges().begin(),
                                                 dst->in_edges().end());
   for (const tensorflow::Edge* in_edge : in_edges) {
-    if (in_edge->IsControlEdge()) {
-      if (in_edge->src() != src) {
-        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-        graph->AddControlEdge(e->src(), src);
-      }
-    } else {
-      if (in_edge->src() != src) {
-        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-        if (e->src() == graph->source_node()) {
-          graph->AddEdge(e->src(), e->src_output(), src,
-                         tensorflow::Graph::kControlSlot);
-        } else {
-          graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
-        }
+    if (in_edge->src() != src) {
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+      if (e->src() == graph->source_node()) {
+        graph->AddEdge(e->src(), e->src_output(), src,
+                       tensorflow::Graph::kControlSlot);
+      } else {
+        graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
       }
     }
   }
@@ -101,19 +94,12 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> out_edges(dst->out_edges().begin(),
                                                  dst->out_edges().end());
   for (const tensorflow::Edge* out_edge : out_edges) {
-    if (out_edge->IsControlEdge()) {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-      graph->AddControlEdge(src, e->dst());
+    tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+    if (e->dst() == graph->sink_node()) {
+      graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
+                     e->dst_input());
     } else {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-      if (e->dst() == graph->sink_node()) {
-        VLOG(1) << " edge to sink node " << src->name() << " -> "
-                << e->dst()->name();
-        graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
-                       e->dst_input());
-      } else {
-        graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
-      }
+      graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
     }
   }
 
@@ -132,7 +118,7 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
+    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
   // Create a Graph representation of the GraphDef.
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
@@ -150,7 +136,7 @@ tensorflow::Status SegmentGraph(
   for (int i = 0; i < graph.num_node_ids(); ++i) {
     tensorflow::Node* node = graph.FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0 ||
-        !candidate_fn(node)) {
+        !candidate_fn(node->def())) {
       node = nullptr;
     }
     node_segments.emplace_back(node);
@@ -169,7 +155,7 @@ tensorflow::Status SegmentGraph(
 
   for (const tensorflow::Node* node : order) {
     // All output nodes of 'node' have been visited...
-    VLOG(2) << "Trying node " << node->name() << " id=" << node->id();
+    VLOG(2) << "Trying node " << node->name();
 
     // 'node' must be a TRT candidate...
     if (node_segments[node->id()].Value() == nullptr) {
@@ -183,12 +169,8 @@ tensorflow::Status SegmentGraph(
     while (true) {
       std::set<const tensorflow::Edge*> contract_edges;
       for (const tensorflow::Edge* out_edge : node->out_edges()) {
-        VLOG(2) << "... out node " << out_edge->dst()->name() << " ( "
-                << out_edge->dst()->id() << " <- " << node->id() << " )";
-        if (out_edge->IsControlEdge()) {
-          VLOG(2) << "... ... Control Edge, Skipping";
-          continue;
-        }
+        VLOG(2) << "... out node " << out_edge->dst()->name();
+
         // Out node must be TRT candidate...
         if (node_segments[out_edge->dst()->id()].Value() == nullptr) {
           VLOG(2) << "... ... not a TRT candidate";
@@ -214,8 +196,7 @@ tensorflow::Status SegmentGraph(
         const tensorflow::Node* src = contract_edge->src();
         const tensorflow::Node* dst = contract_edge->dst();
 
-        VLOG(2) << "Merge " << src->name() << " <- " << dst->name() << " ("
-                << src->id() << " <- " << dst->id();
+        VLOG(2) << "Merge " << src->name() << " <- " << dst->name();
         node_segments[src->id()].Merge(&node_segments[dst->id()]);
 
         // Contracting the edge leaves disconnected graph edges.
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index 7e8685f44a..ee6e2b3ed2 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -20,12 +20,10 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/framework/graph.pb.h"
-#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
-
 namespace tensorrt {
 namespace segment {
 
@@ -48,7 +46,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
+    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
 
 }  // namespace segment
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 7ddabec268..74cbc5f2b3 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -35,7 +35,7 @@ class SegmentTest : public ::testing::Test {
   TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph,
                     TF_Status* s, const char* name);
 
-  std::function<bool(const Node*)> MakeCandidateFn(
+  std::function<bool(const NodeDef&)> MakeCandidateFn(
       const std::set<string>& node_names);
 
  protected:
@@ -60,10 +60,10 @@ bool SegmentTest::GetGraphDef(TF_Graph* graph,
   return ret;
 }
 
-std::function<bool(const Node*)> SegmentTest::MakeCandidateFn(
+std::function<bool(const NodeDef&)> SegmentTest::MakeCandidateFn(
     const std::set<string>& node_names) {
-  return [node_names](const Node* node) -> bool {
-    return node_names.find(node->name()) != node_names.end();
+  return [node_names](const NodeDef& node) -> bool {
+    return node_names.find(node.name()) != node_names.end();
   };
 }
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index 4f6527a546..ff140efd48 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -70,7 +70,7 @@ class ARModel(model.TimeSeriesModel):
       input_window_size: Number of past time steps of data to look at when doing
         the regression.
       output_window_size: Number of future time steps to predict. Note that
-        setting it to > 1 empirically seems to give a better fit.
+        setting it to > 1 empiricaly seems to give a better fit.
       num_features: number of input features per time step.
       num_time_buckets: Number of buckets into which to divide (time %
         periodicity) for generating time based features.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index 26793c80bf..23452a81c3 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -185,7 +185,7 @@ def batch_matrix_pow(matrices, powers):
                     { matmul(A, power(matmul(A, A), (p - 1) / 2)) for odd p
       power(A, 0) = I
 
-    The power(A, 0) = I case is handled by starting with accumulator set to the
+    The power(A, 0) = I case is handeled by starting with accumulator set to the
     identity matrix; matrices with zero residual powers are passed through
     unchanged.
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
index 6746dd7b43..1afc58cfb2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
@@ -107,7 +107,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state transition matrix. It has shape
-        [self.state_dimension, self.state_dimension].
+        [self.state_dimendion, self.state_dimension].
     """
     # Pad any unused AR blocks with zeros. The extra state is necessary if
     # ma_order >= ar_order.
@@ -127,7 +127,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state noise transform matrix. It has shape
-        [self.state_dimension, self.num_features].
+        [self.state_dimendion, self.num_features].
     """
     # Noise is broadcast, through the moving average coefficients, to
     # un-observed parts of the latent state.
diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
index e667c328ae..51d91399f8 100644
--- a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
@@ -49,14 +49,14 @@ in the batch:
 If `fast` is `True`, then the solution is computed by solving the normal
 equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
 \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). 
-If \\(m \lt n\\) then `output` is computed as
+problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
+\lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
 \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
 minimum-norm solution to the under-determined linear system, i.e.
 \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
 subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
 when \\(A\\) is numerically full rank and has a condition number
-\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is
+\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
 sufficiently large.
 
 If `fast` is `False` an algorithm based on the numerically robust complete
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 829c19204a..43a909466e 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,6 +19,9 @@ limitations under the License.
 
 namespace tensorflow {
 
+constexpr const char* MklCPUAllocator::kMaxLimitStr;
+constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 2fb17c2b02..623248b6ce 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1210,7 +1210,7 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
-Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index) {
+Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
   ShapeHandle shape_x = c->input(0);
   ShapeHandle shape_y = c->input(1);
   if (!c->RankKnown(shape_x) || !c->RankKnown(shape_y)) {
@@ -1272,7 +1272,7 @@ Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index) {
     }
   }
 
-  c->set_output(output_index, c->MakeShape(dims));
+  c->set_output(0, c->MakeShape(dims));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 789746b403..293c40e04d 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -265,15 +265,9 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
-// Shape function for binary operators that broadcast their inputs
-// and with output to output_index.
-Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index);
-
 // Shape function for binary operators that broadcast their inputs.
 // Tested by ops/math_ops_test.cc.
-inline Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
-  return BroadcastBinaryOpOutputShapeFn(c, 0);
-}
+Status BroadcastBinaryOpShapeFn(InferenceContext* c);
 
 // Shape function for random operations.
 Status RandomShape(shape_inference::InferenceContext* c);
diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h
index accc587000..e3cc848a16 100644
--- a/tensorflow/core/framework/shape_inference.h
+++ b/tensorflow/core/framework/shape_inference.h
@@ -317,7 +317,6 @@ class InferenceContext {
     input_tensors_as_shapes_ = input_tensors_as_shapes;
   }
 
-  ShapeHandle output(int64 idx) const { return outputs_[idx]; }
   void set_output(int idx, ShapeHandle shape) { outputs_[idx] = shape; }
   Status set_output(StringPiece output_name,
                     const std::vector<ShapeHandle>& shapes);
diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 62aafa7930..333a6570dc 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -933,7 +933,7 @@ class MklFusedBatchNormOp : public OpKernel {
   bool is_training_;
   T* mean_values_;
   T* variance_values_;
-  int depth_;  // batch normalization is done for per channel.
+  size_t depth_;  // batch normalization is done for per channel.
 
   void ExtractParams(OpKernelContext* context) {
     const Tensor& input = MklGetInput(context, 0);
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index 7badc00572..4abfbfb1a6 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -23,13 +23,6 @@ limitations under the License.
 // non-GPU targets. This only breaks in clang, because it's more strict for
 // template code and CudaAtomicMax is used in template context.
 
-// This file requires the following include because it uses CudaAtomicMax:
-// #include "tensorflow/core/util/cuda_kernel_helper.h"
-
-// Unfortunately we can't add the #include, since it breaks compilation for
-// non-GPU targets. This only breaks in clang, because it's more strict for
-// template code and CudaAtomicMax is used in template context.
-
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/core/kernels/snapshot_op.cc b/tensorflow/core/kernels/snapshot_op.cc
index fe04dcf72e..50157d5d48 100644
--- a/tensorflow/core/kernels/snapshot_op.cc
+++ b/tensorflow/core/kernels/snapshot_op.cc
@@ -22,26 +22,6 @@ limitations under the License.
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
-typedef Eigen::GpuDevice GPUDevice;
-
-template <typename Device, typename Scalar>
-class SnapshotOp : public OpKernel {
- public:
-  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input = context->input(0);
-    Tensor* output = nullptr;
-    // Try to use buffer forwarding to avoid an explicit copy.
-    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 0, input.shape(), &output));
-    if (!output->SharesBufferWith(input)) {
-      functor::Snapshot<Device, Scalar> functor;
-      functor(context->eigen_device<Device>(), input.flat<Scalar>(),
-              output->flat<Scalar>());
-    }
-  }
-};
 
 #define REGISTER_KERNEL(TYPE)                                        \
   REGISTER_KERNEL_BUILDER(                                           \
@@ -51,16 +31,6 @@ class SnapshotOp : public OpKernel {
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
 
-#if GOOGLE_CUDA
-#define REGISTER_KERNEL(TYPE)                                        \
-  REGISTER_KERNEL_BUILDER(                                           \
-      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
-      SnapshotOp<GPUDevice, TYPE>);
-
-TF_CALL_POD_TYPES(REGISTER_KERNEL);
-#undef REGISTER_KERNEL
-#endif
-
 #if TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SyclDevice;
 #define REGISTER_SYCL_KERNEL(TYPE)                                    \
diff --git a/tensorflow/core/kernels/snapshot_op.h b/tensorflow/core/kernels/snapshot_op.h
index a18065d42b..b94834f159 100644
--- a/tensorflow/core/kernels/snapshot_op.h
+++ b/tensorflow/core/kernels/snapshot_op.h
@@ -26,19 +26,29 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
-namespace functor {
 
-// Functor used by SnapshotOp.
 template <typename Device, typename Scalar>
-struct Snapshot {
-  void operator()(const Device& device,
-                  typename TTypes<Scalar>::ConstTensor input,
-                  typename TTypes<Scalar>::Tensor output) {
-    device.memcpy(output.data(), input.data(), input.size() * sizeof(Scalar));
+class SnapshotOp : public OpKernel {
+ public:
+  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& input = context->input(0);
+    Tensor* output = nullptr;
+    // Try to use buffer forwarding to avoid an explicit copy.
+    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                {0}, 0, input.shape(), &output));
+    if (!output->SharesBufferWith(input)) {
+      // We had to allocate a new buffer since the refcount on the input was
+      // greater than 1. Copy the input to the new buffer.
+      const Device& device = context->eigen_device<Device>();
+      device.memcpy(output->template flat<Scalar>().data(),
+                    input.template flat<Scalar>().data(),
+                    input.NumElements() * sizeof(Scalar));
+    }
   }
 };
 
-}  // namespace functor
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_KERNELS_SNAPSHOT_OP_H_
diff --git a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
index e4e3bd5220..52070be838 100644
--- a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
@@ -24,10 +24,13 @@ limitations under the License.
 namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
-// Definition of the GPU implementations declared in softsign_op.cc.
-#define DEFINE_GPU_KERNELS(T) template struct functor::Snapshot<GPUDevice, T>;
+#define REGISTER_KERNEL(TYPE)                                        \
+  REGISTER_KERNEL_BUILDER(                                           \
+      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
+      SnapshotOp<GPUDevice, TYPE>);
 
-TF_CALL_POD_TYPES(DEFINE_GPU_KERNELS);
+TF_CALL_POD_TYPES(REGISTER_KERNEL);
+#undef REGISTER_KERNEL
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/xent_op.cc b/tensorflow/core/kernels/xent_op.cc
index 9a3612bd72..a6a71fdfaf 100644
--- a/tensorflow/core/kernels/xent_op.cc
+++ b/tensorflow/core/kernels/xent_op.cc
@@ -17,14 +17,12 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
+#include "tensorflow/core/kernels/xent_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
-#include "tensorflow/core/kernels/xent_op.h"
-#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
@@ -43,56 +41,37 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& logits_in = context->input(0);
     const Tensor& labels_in = context->input(1);
-
-    TensorShape shape_in = logits_in.shape();
-
-    BCast bcast(BCast::FromShape(logits_in.shape()),
-                BCast::FromShape(labels_in.shape()));
-    if (!logits_in.IsSameSize(labels_in)) {
-      OP_REQUIRES(context, bcast.IsValid(),
-                  errors::InvalidArgument(
-                      "logits and labels must be broadcastable: logits_size=",
-                      logits_in.shape().DebugString(),
-                      " labels_size=", labels_in.shape().DebugString()));
-      shape_in = BCast::ToShape(bcast.output_shape());
-    }
-    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
-                errors::InvalidArgument("logits and labels must be beither "
-                                        "2-dimensional, or roadcasted to "
-                                        "2-dimensional"));
+    OP_REQUIRES(context, logits_in.IsSameSize(labels_in),
+                errors::InvalidArgument(
+                    "logits and labels must be same size: logits_size=",
+                    logits_in.shape().DebugString(),
+                    " labels_size=", labels_in.shape().DebugString()));
+    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits_in.shape()),
+                errors::InvalidArgument("logits must be 2-dimensional"));
+    // As we already tested that both inputs have the same shape no need to
+    // check that "labels" is a matrix too.
 
     // loss is 1-D (one per example), and size is batch_size.
 
     Tensor scratch;
     OP_REQUIRES_OK(
         context, context->allocate_temp(DataTypeToEnum<T>::value,
-                                        TensorShape({shape_in.dim_size(0), 1}),
+                                        TensorShape({logits_in.dim_size(0), 1}),
                                         &scratch));
 
     Tensor* loss_out = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(
-                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
+                       0, TensorShape({logits_in.dim_size(0)}), &loss_out));
     Tensor* back_out = nullptr;
     // Try to reuse the logits_in buffer for the backprop output.
     OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 1, shape_in, &back_out));
-    if (shape_in.dim_size(0) > 0) {
+                                {0}, 1, logits_in.shape(), &back_out));
+    if (logits_in.dim_size(0) > 0) {
       functor::XentFunctor<Device, T> functor;
-      if (logits_in.IsSameSize(labels_in)) {
-        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
-                Eigen::array<Eigen::DenseIndex, 2>{1, 1},
-                Eigen::array<Eigen::DenseIndex, 2>{1, 1}, logits_in.matrix<T>(),
-                labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
-                back_out->matrix<T>());
-      } else {
-        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
-                BCast::ToIndexArray<2>(bcast.x_bcast()),
-                BCast::ToIndexArray<2>(bcast.y_bcast()),
-                logits_in.template shaped<T, 2>(bcast.x_reshape()),
-                labels_in.template shaped<T, 2>(bcast.y_reshape()),
-                scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
-      }
+      functor(context->eigen_device<Device>(), logits_in.matrix<T>(),
+              labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
+              back_out->matrix<T>());
     }
   }
 };
@@ -102,17 +81,13 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename Device, typename T>
 struct XentFunctorBase {
-  void operator()(const Device& d,
-                  const Eigen::DSizes<Eigen::DenseIndex, 2>& shape,
-                  const Eigen::array<Eigen::DenseIndex, 2>& logits_bcast,
-                  const Eigen::array<Eigen::DenseIndex, 2>& labels_bcast,
-                  typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<Device, T>::Compute(d, shape, logits_bcast, labels_bcast,
-                                      logits, labels, scratch, loss, backprop);
+    XentEigenImpl<Device, T>::Compute(d, logits, labels, scratch, loss,
+                                      backprop);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op.h b/tensorflow/core/kernels/xent_op.h
index 87be17fca9..e689fca7ff 100644
--- a/tensorflow/core/kernels/xent_op.h
+++ b/tensorflow/core/kernels/xent_op.h
@@ -18,7 +18,6 @@ limitations under the License.
 // Functor definition for XentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
-
 #include "tensorflow/core/framework/tensor_types.h"
 
 namespace tensorflow {
@@ -34,11 +33,7 @@ struct XentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device &d,
-                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
-                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
-                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
-                  typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
@@ -50,11 +45,7 @@ struct XentFunctor {
 // specializations for both device types.
 template <typename Device, typename T>
 struct XentEigenImpl {
-  static void Compute(const Device &d,
-                      const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
-                      const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
-                      const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
-                      typename TTypes<T>::ConstMatrix logits,
+  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<T>::ConstMatrix labels,
                       typename TTypes<T>::Matrix scratch,
                       typename TTypes<T>::Vec loss,
@@ -66,8 +57,8 @@ struct XentEigenImpl {
     const int kBatchDim = 0;
     const int kClassDim = 1;
 
-    const int batch_size = shape[kBatchDim];
-    const int num_classes = shape[kClassDim];
+    const int batch_size = logits.dimension(kBatchDim);
+    const int num_classes = logits.dimension(kClassDim);
 
 // These arrays are used to reduce along the class dimension, and broadcast
 // the resulting value to all classes.
@@ -93,12 +84,10 @@ struct XentEigenImpl {
 #endif
 
     // max_logits along classes.
-    scratch.reshape(batch_only).device(d) =
-        logits.broadcast(logits_bcast).maximum(along_class);
+    scratch.reshape(batch_only).device(d) = logits.maximum(along_class);
 
     // logits - max_logits.
-    backprop.device(d) =
-        logits.broadcast(logits_bcast) - scratch.broadcast(one_by_class);
+    backprop.device(d) = logits - scratch.broadcast(one_by_class);
 
     // sum(exp(logits - max_logits)) along classes.
     scratch.reshape(batch_only).device(d) = backprop.exp().sum(along_class);
@@ -110,15 +99,15 @@ struct XentEigenImpl {
     //  sum(-labels *
     //     ((logits - max_logits) - log(sum(exp(logits - max_logits)))))
     //  along classes
-    loss.device(d) = (labels.broadcast(labels_bcast) *
-                      (scratch.log().eval().broadcast(one_by_class) - backprop))
-                         .eval()
-                         .sum(along_class);
+    loss.device(d) =
+        (labels * (scratch.log().eval().broadcast(one_by_class) - backprop))
+            .eval()
+            .sum(along_class);
 
     // backprop: prob - labels, where
     //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
-    backprop.device(d) = (backprop.exp() / scratch.broadcast(one_by_class)) -
-                         labels.broadcast(labels_bcast);
+    backprop.device(d) =
+        (backprop.exp() / scratch.broadcast(one_by_class)) - labels;
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op_gpu.cu.cc b/tensorflow/core/kernels/xent_op_gpu.cu.cc
index 2c0c0b3a02..05ee7da490 100644
--- a/tensorflow/core/kernels/xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/xent_op_gpu.cu.cc
@@ -31,17 +31,12 @@ typedef Eigen::GpuDevice GPUDevice;
 namespace functor {
 template <typename T>
 struct XentFunctor<GPUDevice, T> {
-  void operator()(const GPUDevice &d,
-                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
-                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
-                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
-                  typename TTypes<T>::ConstMatrix logits,
+  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<GPUDevice, T>::Compute(d, shape, logits_bcast, labels_bcast,
-                                         logits, labels, scratch, loss,
+    XentEigenImpl<GPUDevice, T>::Compute(d, logits, labels, scratch, loss,
                                          backprop);
   }
 };
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 111670c361..88d2aa3f41 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -794,35 +794,11 @@ REGISTER_OP("ReverseV2")
       ShapeHandle input = c->input(0);
       ShapeHandle axis;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &axis));
+      // TODO(aselle): if input(0)'s dimension is known we could validate axis
       if (c->Rank(input) > 8) {
         return errors::InvalidArgument(
             "reverse does not work on tensors with more than 8 dimensions");
       }
-      const Tensor* axis_tensor = c->input_tensor(1);
-      if (axis_tensor != nullptr && c->RankKnown(input)) {
-        int32 rank = c->Rank(input);
-        std::vector<int64> axis_value;
-        if (axis_tensor->dtype() == DT_INT32) {
-          axis_value = AsInt64<int32>(axis_tensor, axis_tensor->NumElements());
-        } else {
-          axis_value = AsInt64<int64>(axis_tensor, axis_tensor->NumElements());
-        }
-        std::vector<bool> axes_dense(c->Rank(input), false);
-        for (int i = 0; i < axis_value.size(); i++) {
-          int64 canonical_axis =
-              axis_value[i] < 0 ? rank + axis_value[i] : axis_value[i];
-          if (canonical_axis < 0 || canonical_axis >= rank) {
-            return errors::InvalidArgument("'axis'[", i, "] = ", axis_value[i],
-                                           " is out of valid range [", 0, ", ",
-                                           rank - 1);
-          }
-          if (axes_dense[canonical_axis]) {
-            return errors::InvalidArgument("axis ", canonical_axis,
-                                           " specified more than once.");
-          }
-          axes_dense[canonical_axis] = true;
-        }
-      }
       c->set_output(0, input);
       return Status::OK();
     });
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 6c2fc60bab..1f4e9753c3 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1062,27 +1062,12 @@ REGISTER_OP("SoftmaxCrossEntropyWithLogits")
     .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
-      if (c->WithRank(c->input(0), 2, &input) == Status::OK() &&
-          c->Merge(input, c->input(1), &input) == Status::OK()) {
-        DimensionHandle batch_size = c->Dim(input, 0);
-        c->set_output(0, c->Vector(batch_size));
-        c->set_output(1, input);
-        return Status::OK();
-      }
-      TF_RETURN_IF_ERROR(BroadcastBinaryOpOutputShapeFn(c, 1));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input));
+      TF_RETURN_IF_ERROR(c->Merge(input, c->input(1), &input));
 
-      if (!c->RankKnown(c->output(1))) {
-        return errors::InvalidArgument(
-            "Shape must be broadcasted with rank 2, but is rank is unknown.");
-      }
-
-      if (c->Rank(c->output(1)) != 2) {
-        return errors::InvalidArgument(
-            "Shape must be broadcasted with rank 2, but is rank ",
-            c->Rank(c->output(1)));
-      }
-      DimensionHandle batch_size = c->Dim(c->output(1), 0);
+      DimensionHandle batch_size = c->Dim(input, 0);
       c->set_output(0, c->Vector(batch_size));
+      c->set_output(1, input);
       return Status::OK();
     });
 
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 289b953055..1b17a7cda6 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -410,18 +410,10 @@ TEST(NNOpsTest, SoftmaxCrossEntropyWithLogits_ShapeFn) {
   INFER_OK(op, "[1,?];[?,2]", "[d0_0];[d0_0,d0_1|d1_1]");
   INFER_OK(op, "[?,2];[1,2]", "[d1_0];in1");
 
-  INFER_ERROR("Shape must be broadcasted with rank 2", op, "[1,2,3];?");
-  INFER_ERROR("Shape must be broadcasted with rank 2", op, "?;[1,2,3]");
-
-  // Broadcast example
-  // [1,4] and [2,4] are broadcasted to [2,4]
-  INFER_OK(op, "[1,4];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
-  // [2,4] and [2,1] are broadcasted to [2,4]
-  INFER_OK(op, "[2,4];[2,1]", "[d0_0];[d0_0|d1_0,d0_1]");
-  // [1,?] and [2,4] are broadcasted to [2,4]
-  INFER_OK(op, "[1,?];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
-  // [2,4] and [?,1] are broadcasted to [2,4]
-  INFER_OK(op, "[2,4];[?,1]", "[d0_0];[d0_0|d1_0,d0_1]");
+  INFER_ERROR("Dimension 0 in both shapes must be equal, but are 1 and 2", op,
+              "[1,?];[2,?]");
+  INFER_ERROR("Shape must be rank 2 but is rank 3", op, "[1,2,3];?");
+  INFER_ERROR("Shapes must be equal rank, but are 2 and 3", op, "?;[1,2,3]");
 }
 
 TEST(NNOpsTest, SparseSoftmaxCrossEntropyWithLogits_ShapeFn) {
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 40eebd1db0..22f2c02b78 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 7
+#define TF_MINOR_VERSION 6
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX "-rc1"
+#define TF_VERSION_SUFFIX ""
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
index f3db5857ae..956dccb64f 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
@@ -6,42 +6,42 @@ Monte Carlo integration and helpers.
 ## Background
 
 Monte Carlo integration refers to the practice of estimating an expectation with
-a sample mean.  For example, given random variable `Z in \\(R^k\\)` with density `p`,
+a sample mean.  For example, given random variable `Z in R^k` with density `p`,
 the expectation of function `f` can be approximated like:
 
 ```
-$$E_p[f(Z)] = \int f(z) p(z) dz$$
-$$          ~ S_n
-          := n^{-1} \sum_{i=1}^n f(z_i),  z_i\ iid\ samples\ from\ p.$$
+E_p[f(Z)] = \int f(z) p(z) dz
+          ~ S_n
+          := n^{-1} \sum_{i=1}^n f(z_i),  z_i iid samples from p.
 ```
 
-If `\\(E_p[|f(Z)|] < infinity\\)`, then `\\(S_n\\) --> \\(E_p[f(Z)]\\)` by the strong law of large
-numbers.  If `\\(E_p[f(Z)^2] < infinity\\)`, then `\\(S_n\\)` is asymptotically normal with
-variance `\\(Var[f(Z)] / n\\)`.
+If `E_p[|f(Z)|] < infinity`, then `S_n --> E_p[f(Z)]` by the strong law of large
+numbers.  If `E_p[f(Z)^2] < infinity`, then `S_n` is asymptotically normal with
+variance `Var[f(Z)] / n`.
 
 Practitioners of Bayesian statistics often find themselves wanting to estimate
-`\\(E_p[f(Z)]\\)` when the distribution `p` is known only up to a constant.  For
+`E_p[f(Z)]` when the distribution `p` is known only up to a constant.  For
 example, the joint distribution `p(z, x)` may be known, but the evidence
-`\\(p(x) = \int p(z, x) dz\\)` may be intractable.  In that case, a parameterized
-distribution family `\\(q_\lambda(z)\\)` may be chosen, and the optimal `\\(\lambda\\)` is the
-one minimizing the KL divergence between `\\(q_\lambda(z)\\)` and
-`\\(p(z | x)\\)`.  We only know `p(z, x)`, but that is sufficient to find `\\(\lambda\\)`.
+`p(x) = \int p(z, x) dz` may be intractable.  In that case, a parameterized
+distribution family `q_lambda(z)` may be chosen, and the optimal `lambda` is the
+one minimizing the KL divergence between `q_lambda(z)` and
+`p(z | x)`.  We only know `p(z, x)`, but that is sufficient to find `lambda`.
 
 
 ## Log-space evaluation and subtracting the maximum
 
 Care must be taken when the random variable lives in a high dimensional space.
-For example, the naive importance sample estimate `\\(E_q[f(Z) p(Z) / q(Z)]\\)`
-involves the ratio of two terms `\\(p(Z) / q(Z)\\)`, each of which must have tails
-dropping off faster than `\\(O(|z|^{-(k + 1)})\\)` in order to have finite integral.
+For example, the naive importance sample estimate `E_q[f(Z) p(Z) / q(Z)]`
+involves the ratio of two terms `p(Z) / q(Z)`, each of which must have tails
+dropping off faster than `O(|z|^{-(k + 1)})` in order to have finite integral.
 This ratio would often be zero or infinity up to numerical precision.
 
 For that reason, we write
 
 ```
-$$Log E_q[ f(Z) p(Z) / q(Z) ]$$
-$$   = Log E_q[ \exp\{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C\} ] + C,$$  where
-$$C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].$$
+Log E_q[ f(Z) p(Z) / q(Z) ]
+   = Log E_q[ exp{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C} ] + C,  where
+C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].
 ```
 
 The maximum value of the exponentiated term will be 0.0, and the expectation
diff --git a/tensorflow/docs_src/api_guides/python/contrib.losses.md b/tensorflow/docs_src/api_guides/python/contrib.losses.md
index 8b7442216c..d7f862625e 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.losses.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.losses.md
@@ -107,19 +107,19 @@ weighted average over the individual prediction errors:
   loss = tf.contrib.losses.mean_squared_error(predictions, depths, weight)
 ```
 
-* @{tf.contrib.losses.absolute_difference}
-* @{tf.contrib.losses.add_loss}
-* @{tf.contrib.losses.hinge_loss}
-* @{tf.contrib.losses.compute_weighted_loss}
-* @{tf.contrib.losses.cosine_distance}
-* @{tf.contrib.losses.get_losses}
-* @{tf.contrib.losses.get_regularization_losses}
-* @{tf.contrib.losses.get_total_loss}
-* @{tf.contrib.losses.log_loss}
-* @{tf.contrib.losses.mean_pairwise_squared_error}
-* @{tf.contrib.losses.mean_squared_error}
-* @{tf.contrib.losses.sigmoid_cross_entropy}
-* @{tf.contrib.losses.softmax_cross_entropy}
-* @{tf.contrib.losses.sparse_softmax_cross_entropy}
+@{tf.contrib.losses.absolute_difference}
+@{tf.contrib.losses.add_loss}
+@{tf.contrib.losses.hinge_loss}
+@{tf.contrib.losses.compute_weighted_loss}
+@{tf.contrib.losses.cosine_distance}
+@{tf.contrib.losses.get_losses}
+@{tf.contrib.losses.get_regularization_losses}
+@{tf.contrib.losses.get_total_loss}
+@{tf.contrib.losses.log_loss}
+@{tf.contrib.losses.mean_pairwise_squared_error}
+@{tf.contrib.losses.mean_squared_error}
+@{tf.contrib.losses.sigmoid_cross_entropy}
+@{tf.contrib.losses.softmax_cross_entropy}
+@{tf.contrib.losses.sparse_softmax_cross_entropy}
 
 
diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md
index 6f2107ef40..003e0a25ec 100644
--- a/tensorflow/docs_src/community/documentation.md
+++ b/tensorflow/docs_src/community/documentation.md
@@ -477,29 +477,31 @@ should use Markdown in the docstring.
 
 Here's a simple example:
 
-    def foo(x, y, name="bar"):
-      """Computes foo.
+```python
+def foo(x, y, name="bar"):
+  """Computes foo.
 
-      Given two 1-D tensors `x` and `y`, this operation computes the foo.
+  Given two 1-D tensors `x` and `y`, this operation computes the foo.
 
-      Example:
+  Example:
 
-      ```
-      # x is [1, 1]
-      # y is [2, 2]
-      tf.foo(x, y) ==> [3, 3]
-      ```
-      Args:
-        x: A `Tensor` of type `int32`.
-        y: A `Tensor` of type `int32`.
-        name: A name for the operation (optional).
+  ```
+  # x is [1, 1]
+  # y is [2, 2]
+  tf.foo(x, y) ==> [3, 3]
+  ```
+  Args:
+    x: A `Tensor` of type `int32`.
+    y: A `Tensor` of type `int32`.
+    name: A name for the operation (optional).
 
-      Returns:
-        A `Tensor` of type `int32` that is the foo of `x` and `y`.
+  Returns:
+    A `Tensor` of type `int32` that is the foo of `x` and `y`.
 
-      Raises:
-        ValueError: If `x` or `y` are not of type `int32`.
-      """
+  Raises:
+    ValueError: If `x` or `y` are not of type `int32`.
+  """
+```
 
 ## Description of the docstring sections
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 9059b3f3b6..0481c97885 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 2e47a6d212..8f89898c92 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index eff066d200..0ee9c849e1 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.7.0-rc1</version>
+                 <version>1.6.0</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.6.0</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 27b696696d..5e9a84bff6 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -165,7 +165,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -270,7 +270,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -456,7 +456,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -630,14 +630,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -649,14 +649,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -668,14 +668,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -687,14 +687,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 7060ef43da..55b460e189 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -118,8 +118,8 @@ Take the following steps to install TensorFlow with Virtualenv:
      Python 2.7, the command to install
      TensorFlow in the active Virtualenv is as follows:
 
-     <pre> $ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b></pre>
+     <pre> $ <b>pip install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -241,8 +241,8 @@ take the following steps:
      you are installing TensorFlow for macOS and Python 2.7
      issue the following command:
 
-     <pre> $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b> </pre>
+     <pre> $ <b>sudo pip install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -524,7 +524,7 @@ The value you specify depends on your Python version.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
 </pre>
 
 
@@ -532,5 +532,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-a
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index 148f80efe2..a7f33819b4 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -350,10 +350,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.7.0rc1 on Linux:
+for TensorFlow 1.6.0 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -450,8 +450,6 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -471,7 +469,6 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -486,8 +483,6 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
-<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
-<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md
index 778e4d3a62..ca9cb043e9 100644
--- a/tensorflow/docs_src/mobile/optimizing.md
+++ b/tensorflow/docs_src/mobile/optimizing.md
@@ -233,8 +233,6 @@ order by how long they took. From left to right, the columns are:
 - The cumulative total time of this and the previous ops in the table. This is
   handy for understanding what the distribution of work is across the layers, to
   see if just a few of the nodes are taking up most of the time.
-  
-- The amount of memory consumed by outputs of this type of op.
 
 - Name of the node.
 
diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md
index 8b22c04d87..360ee302aa 100644
--- a/tensorflow/docs_src/mobile/prepare_models.md
+++ b/tensorflow/docs_src/mobile/prepare_models.md
@@ -60,7 +60,7 @@ and serialized as protocol buffers:
   the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
   then we only need a single `GraphDef` file to hold the model architecture and
   the weights. Freezing the graph handles the process of loading the
-  checkpoints, and then converts all Variables to Consts. You can then load the
+  checkpoints, and then converts all Consts to Variables. You can then load the
   resulting file in a single call, without having to restore variable values
   from checkpoints. One thing to watch out for with `GraphDef` files is that
   sometimes they’re stored in text format for easy inspection. These versions
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a0dd409205..4f61c01f65 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1065,7 +1065,7 @@ py_test(
 
 py_test(
     name = "framework_importer_test",
-    size = "large",
+    size = "medium",
     srcs = ["framework/importer_test.py"],
     main = "framework/importer_test.py",
     srcs_version = "PY2AND3",
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 64c1760d5e..d0ba8020c1 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -315,39 +315,21 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
             self.assertAllEqual(x_tf_4, np.asarray(x_np)[:, ::-1])
             self.assertAllEqual(x_tf_5, np.asarray(x_np)[::-1, ::-1])
 
-  # This test covers the axis validation in the shape function
-  # (no eval())
-  def testInvalidAxis(self):
-    x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
-    with self.assertRaisesRegexp(ValueError,
-                                 "is out of valid range"):
-      array_ops.reverse_v2(x_np, [-30])
-    with self.assertRaisesRegexp(ValueError,
-                                 "is out of valid range"):
-      array_ops.reverse_v2(x_np, [2])
-    with self.assertRaisesRegexp(ValueError,
-                                 "axis 0 specified more than once"):
-      array_ops.reverse_v2(x_np, [0, -2])
-
   # This is the version of reverse that uses axis indices rather than
   # bool tensors
   # TODO(b/32254538): Change this test to use array_ops.reverse
-  #
-  # Note: this test passes placeholder as constant axis is validated
-  # in shape function (see testInvalidAxis)
   def testInvalid(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
-    axis = array_ops.placeholder(dtypes.int32)
     with self.test_session():
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [-30]})
+        array_ops.reverse_v2(x_np, [-30]).eval()
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [2]})
+        array_ops.reverse_v2(x_np, [2]).eval()
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "axis 0 specified more than once"):
-        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]})
+        array_ops.reverse_v2(x_np, [0, -2]).eval()
 
   def testReverse1DimAuto(self):
     for dtype in [
@@ -908,7 +890,7 @@ class StridedSliceAssignChecker(object):
         var = resource_variable_ops.ResourceVariable(self.x)
       else:
         var = variables.Variable(self.x)
-      sess.run(variables.variables_initializer([var]))
+      sess.run(variables.initialize_variables([var]))
       val = sess.run(var[index].assign(value))
       # val_copy is used to check that tf.assign works equivalently to the
       # assign method above.
diff --git a/tensorflow/python/kernel_tests/testdata/BUILD b/tensorflow/python/kernel_tests/testdata/BUILD
index 45264c773a..a4a0dfc139 100644
--- a/tensorflow/python/kernel_tests/testdata/BUILD
+++ b/tensorflow/python/kernel_tests/testdata/BUILD
@@ -1,7 +1,7 @@
 # Data files for kernel tests.
 
 package(
-    default_visibility = ["//visibility:public"],
+    default_visibility = ["//tensorflow:internal"],
 )
 
 licenses(["notice"])  # Apache 2.0
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index 60c726d54c..e3e120a4eb 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -18,16 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import itertools
-import sys
-
 import numpy as np
 
-from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import ops
-from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -94,7 +88,7 @@ class XentTest(test.TestCase):
                                                     4.]]]).astype(dtype)
       np_labels = np.array([[[0., 0., 0., 1.]], [[0., .5, .5,
                                                   0.]]]).astype(dtype)
-      self.assertRaisesRegexp(ValueError, "rank 2, but is rank 3",
+      self.assertRaisesRegexp(ValueError, "must be rank 2",
                               gen_nn_ops.softmax_cross_entropy_with_logits,
                               np_features, np_labels)
 
@@ -134,24 +128,6 @@ class XentTest(test.TestCase):
     self.assertAllClose(
         np.array([1.3862, 1.9401]), np_loss, rtol=1.e-3, atol=1.e-3)
 
-  def testShapeBroadcast(self):
-    np_f = np.array([[1., 2., 3., 4.],
-                     [1., 2., 3., 4.]]).astype(np.float32)
-    np_l = np.array([[0., 0., 0., 1.],
-                     [0., .5, .5, 0.]]).astype(np.float32)
-    np_loss, np_backprop = self._npXent(np_f, np_l)
-    tf_f = constant_op.constant(
-        np.array([[1., 2., 3., 4.]]).astype(np.float32))
-    tf_l = constant_op.constant(
-        np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32))
-    for use_gpu in [False, True]:
-      with self.test_session(use_gpu=use_gpu) as sess:
-        loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
-            tf_f, tf_l)
-        tf_loss, tf_backprop = sess.run([loss, backprop])
-      self.assertAllCloseAccordingToType(np_loss, tf_loss)
-      self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
-
   def testShapeMismatch(self):
     with self.test_session():
       with self.assertRaises(ValueError):
@@ -284,60 +260,5 @@ class XentTest(test.TestCase):
     self.assertAllEqual(np_loss, tf_loss)
 
 
-class XentBenchmark(test.Benchmark):
-
-  def benchmarkZeroDimension(self):
-    for (m, n, p, use_gpu) in itertools.product(
-        [128],
-        [10, 100, 1000, 10000, 100000],
-        [0.001, 0.01, 0.5, 0.99, 1.0],
-        [False]):
-      k = int(p * n)
-      if k == 0:
-        continue
-      name = "zero_dimension_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
-      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
-      with ops.Graph().as_default():
-        with ops.device(device):
-          labels = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
-          logits = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
-          op = nn_ops.softmax_cross_entropy_with_logits(
-              labels=labels, logits=logits)
-        with session.Session() as sess:
-          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
-          gb_processed_input = m * n / 1.0e9
-          throughput = gb_processed_input / r["wall_time"]
-          print("Benchmark: %s \t wall_time: %0.03g s \t "
-                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
-          sys.stdout.flush()
-
-  def benchmarkSingleClass(self):
-    for (m, n, p, use_gpu) in itertools.product(
-        [128],
-        [10, 100, 1000, 10000, 100000],
-        [0.001, 0.01, 0.5, 0.99, 1.0],
-        [False]):
-      k = int(p * n)
-      if k == 0:
-        continue
-      name = "single_class_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
-      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
-      with ops.Graph().as_default():
-        with ops.device(device):
-          labels = constant_op.constant([[1.], [-1.], [0.]],
-                                        dtype=dtypes.float32)
-          logits = constant_op.constant([[-1.], [0.], [1.]],
-                                        dtype=dtypes.float32)
-          op = nn_ops.softmax_cross_entropy_with_logits(
-              labels=labels, logits=logits)
-        with session.Session() as sess:
-          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
-          gb_processed_input = m * n / 1.0e9
-          throughput = gb_processed_input / r["wall_time"]
-          print("Benchmark: %s \t wall_time: %0.03g s \t "
-                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
-          sys.stdout.flush()
-
-
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 2d99b1688f..74e7c63fb3 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -180,8 +180,6 @@ class _Conv(base.Layer):
           # bias_add when computing gradients. To use bias_add, we collapse Z
           # and Y into a single dimension to obtain a 4D input tensor.
           outputs_shape = outputs.shape.as_list()
-          if outputs_shape[0] is None:
-            outputs_shape[0] = -1
           outputs_4d = array_ops.reshape(outputs,
                                          [outputs_shape[0], outputs_shape[1],
                                           outputs_shape[2] * outputs_shape[3],
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index cdb42f5bd1..160e732b67 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -325,12 +325,6 @@ class ConvTest(test.TestCase):
     self.assertEqual(conv3d.kernel_constraint, k_constraint)
     self.assertEqual(conv3d.bias_constraint, b_constraint)
 
-  def testConv3DChannelsFirst(self):
-    # Test case for GitHub issue 15655
-    images = array_ops.placeholder(
-        dtype=dtypes.float32, shape=[None, 1, 32, 32, 32])
-    conv_layers.conv3d(images, 32, 9, data_format='channels_first')
-
 
 @test_util.with_c_api
 class SeparableConv1DTest(test.TestCase):
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 170861b43f..5b4fb4f7c8 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -429,7 +429,7 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
   u, s, v_adj = np.linalg.svd(a, full_matrices=False)
   np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj))
   # tf_a_approx and np_a_approx should be numerically close.
-  ```
+  ````
   @end_compatibility
   """
   s, u, v = gen_linalg_ops.svd(
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 4ce6f6d002..6c5c9e01a7 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -281,14 +281,13 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
                              scaffold=None,
                              hooks=None,
                              chief_only_hooks=None,
-                             save_checkpoint_secs=USE_DEFAULT,
+                             save_checkpoint_secs=600,
                              save_summaries_steps=USE_DEFAULT,
                              save_summaries_secs=USE_DEFAULT,
                              config=None,
                              stop_grace_period_secs=120,
                              log_step_count_steps=100,
-                             max_wait_secs=7200,
-                             save_checkpoint_steps=USE_DEFAULT):
+                             max_wait_secs=7200):
   """Creates a `MonitoredSession` for training.
 
   For a chief, this utility sets proper session initializer/restorer. It also
@@ -311,10 +310,8 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
     chief_only_hooks: list of `SessionRunHook` objects. Activate these hooks if
       `is_chief==True`, ignore otherwise.
     save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved
-      using a default checkpoint saver. If both `save_checkpoint_steps` and
-      `save_checkpoint_secs` are set to `None`, then the default checkpoint
-      saver isn't used. If both are provided, then only `save_checkpoint_secs`
-      is used. Default 600.
+      using a default checkpoint saver. If `save_checkpoint_secs` is set to
+      `None`, then the default checkpoint saver isn't used.
     save_summaries_steps: The frequency, in number of global steps, that the
       summaries are written to disk using a default summary saver. If both
       `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
@@ -333,11 +330,6 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       become available. This should be kept relatively short to help detect
       incorrect code, but sometimes may need to be increased if the chief takes
       a while to start up.
-    save_checkpoint_steps: The frequency, in number of global steps, that a
-      checkpoint is saved using a default checkpoint saver. If both
-      `save_checkpoint_steps` and `save_checkpoint_secs` are set to `None`, then
-      the default checkpoint saver isn't used. If both are provided, then only
-      `save_checkpoint_secs` is used. Default not enabled.
 
   Returns:
     A `MonitoredSession` object.
@@ -350,15 +342,6 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
   elif save_summaries_steps == USE_DEFAULT:
     save_summaries_steps = None
 
-  if (save_checkpoint_steps == USE_DEFAULT and
-      save_checkpoint_secs == USE_DEFAULT):
-    save_checkpoint_steps = None
-    save_checkpoint_secs = 600
-  elif save_checkpoint_secs == USE_DEFAULT:
-    save_checkpoint_secs = None
-  elif save_checkpoint_steps == USE_DEFAULT:
-    save_checkpoint_steps = None
-
   scaffold = scaffold or Scaffold()
   if not is_chief:
     session_creator = WorkerSessionCreator(
@@ -391,13 +374,9 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
           save_steps=save_summaries_steps,
           save_secs=save_summaries_secs,
           output_dir=checkpoint_dir))
-    if (save_checkpoint_secs and save_checkpoint_secs > 0) or (
-        save_checkpoint_steps and save_checkpoint_steps > 0):
+    if save_checkpoint_secs and save_checkpoint_secs > 0:
       all_hooks.append(basic_session_run_hooks.CheckpointSaverHook(
-          checkpoint_dir,
-          save_steps=save_checkpoint_steps,
-          save_secs=save_checkpoint_secs,
-          scaffold=scaffold))
+          checkpoint_dir, save_secs=save_checkpoint_secs, scaffold=scaffold))
 
   if hooks:
     all_hooks.extend(hooks)
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 3806056f01..159b2d5c16 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -282,42 +282,6 @@ class MonitoredTrainingSessionTest(test.TestCase):
           is_chief=True, checkpoint_dir=logdir) as session:
         self.assertEqual(2, session.run(gstep))
 
-  def test_save_checkpoint_steps(self):
-    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_steps')
-    with ops.Graph().as_default():
-      gstep = variables_lib.get_or_create_global_step()
-      new_gstep = state_ops.assign_add(gstep, 1)
-      with monitored_session.MonitoredTrainingSession(
-          is_chief=True,
-          checkpoint_dir=logdir,
-          save_checkpoint_steps=100,
-          log_step_count_steps=10) as session:
-        for _ in range(100):
-          session.run(new_gstep)
-      # A restart will find the checkpoint and recover automatically.
-      with monitored_session.MonitoredTrainingSession(
-          is_chief=True, checkpoint_dir=logdir) as session:
-        self.assertEqual(100, session.run(gstep))
-
-  def test_save_checkpoint_secs(self):
-    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_secs')
-    with ops.Graph().as_default():
-      gstep = variables_lib.get_or_create_global_step()
-      new_gstep = state_ops.assign_add(gstep, 1)
-      with monitored_session.MonitoredTrainingSession(
-          is_chief=True,
-          checkpoint_dir=logdir,
-          save_checkpoint_secs=0.1,
-          log_step_count_steps=10) as session:
-        session.run(new_gstep)
-        time.sleep(0.2)
-        for _ in range(10):
-          session.run(new_gstep)
-      # A restart will find the checkpoint and recover automatically.
-      with monitored_session.MonitoredTrainingSession(
-          is_chief=True, checkpoint_dir=logdir) as session:
-        self.assertEqual(11, session.run(gstep))
-
   def test_summaries_steps(self):
     logdir = _test_dir(self.get_temp_dir(), 'test_summaries_steps')
     with ops.Graph().as_default():
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index fcc57d506e..2d3cb415fe 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -22,7 +22,6 @@ load(
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
-    "if_mkl_lnx_x64"
 )
 
 def register_extension_info(**kwargs):
@@ -203,8 +202,7 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False):
           "-ftemplate-depth=900"])
       + if_cuda(["-DGOOGLE_CUDA=1"])
       + if_tensorrt(["-DGOOGLE_TENSORRT=1"])
-      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"])
-      + if_mkl_lnx_x64(["-fopenmp"])
+      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML", "-fopenmp",])
       + if_android_arm(["-mfpu=neon"])
       + if_linux_x86_64(["-msse3"])
       + if_ios_x86_64(["-msse4.1"])
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index bec72e1e60..c75ee474aa 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -238,7 +238,7 @@ tf_module {
   }
   member_method {
     name: "MonitoredTrainingSession"
-    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\', \'save_checkpoint_steps\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'<object object instance>\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\', \'<object object instance>\'], "
+    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'600\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\'], "
   }
   member_method {
     name: "NewCheckpointReader"
diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
index 7d471b4703..e1b56b9a25 100755
--- a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
+++ b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
@@ -31,5 +31,5 @@ export TF_NEED_OPENCL_SYCL=0
 export TF_NEED_MKL=0
 export COMPUTECPP_PATH="/usr/local"
 
-export PATH="$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
 build_libtensorflow_tarball "-cpu-darwin-$(uname -m)"
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 11f476d12c..22c73c3fe1 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -70,7 +70,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
 # more difficult to experiment with local changes. Instead, just add
diff --git a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
index 037d13116e..3690e7dfe5 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
@@ -3,7 +3,7 @@ FROM tensorflow/tensorflow:latest-devel
 LABEL maintainer="Clayne Robison<clayne.b.robison@intel.com>"
 
 # These arguments are parameterized. Use --build-args to override.
-ARG TF_BRANCH=r1.7
+ARG TF_BRANCH=r1.6
 ARG WHL_DIR=/whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 1fcb6428b2..69ba340f92 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -79,7 +79,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 0ede8c6370..3fbdb5cacd 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -138,6 +138,7 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
+        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/c/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
@@ -175,6 +176,7 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
+        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/jni/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 62fec2c402..dd75eda231 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -127,6 +127,7 @@ filegroup(
         "@org_python_pypi_backports_weakref//:LICENSE",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
+        "@mkl//:LICENSE",
     ]) + if_not_windows([
         "@nccl_archive//:LICENSE.txt",
     ]) + tf_additional_license_deps(),
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index 365e8d6b08..e0152da4df 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.7.0-rc1'
+_VERSION = '1.6.0'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -39,7 +39,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.13.3',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorboard >= 1.7.0, < 1.8.0',
+    'tensorboard >= 1.6.0, < 1.7.0',
     'termcolor >= 1.1.0',
 ]
 
@@ -62,7 +62,7 @@ else:
 if 'tf_nightly' in project_name:
   for i, pkg in enumerate(REQUIRED_PACKAGES):
     if 'tensorboard' in pkg:
-      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.8.0a0, < 1.9.0a0'
+      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.7.0a0, < 1.8.0a0'
       break
 
 # weakref.finalize and enum were introduced in Python 3.4
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5f6e717532..9fcbfb664b 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -15,11 +15,6 @@ load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
 
 
-# Sanitize a dependency so that it works correctly from code that includes
-# TensorFlow as a submodule.
-def clean_dep(dep):
-  return str(Label(dep))
-
 # If TensorFlow is linked as a submodule.
 # path_prefix is no longer used.
 # tf_repo_name is thought to be under consideration.
@@ -37,37 +32,17 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   arm_compiler_configure(
       name="local_config_arm_compiler",
       remote_config_repo="../arm_compiler",
-      build_file = clean_dep("//third_party/toolchains/cpus/arm:BUILD"))
+      build_file = str(Label("//third_party/toolchains/cpus/arm:BUILD")))
 
   mkl_repository(
-      name = "mkl_linux",
-      urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
-          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
-      ],
-      sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f",
-      strip_prefix = "mklml_lnx_2018.0.1.20171227",
-      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
-  )
-  mkl_repository(
-      name = "mkl_windows",
-      urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
-          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip"
-      ],
-      sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4",
-      strip_prefix = "mklml_win_2018.0.1.20171227",
-      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
-  )
-  mkl_repository(
-      name = "mkl_darwin",
+      name = "mkl",
       urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
-          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz"
+          "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
+          "https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
       ],
-      sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f",
-      strip_prefix = "mklml_mac_2018.0.1.20171227",
-      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+      sha256 = "6b07cb7e5451db67c2e31e785ae458b18f7f363c60a61685488f69e9ae7199d4",
+      strip_prefix = "mklml_lnx_2018.0.1.20171007",
+      build_file = str(Label("//third_party/mkl:mkl.BUILD")),
   )
 
   if path_prefix:
@@ -77,12 +52,12 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "mkl_dnn",
       urls = [
-          "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
-          "https://github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
+          "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
+          "https://github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
       ],
-      sha256 = "86fa2a8c12a56e3b725945acedeaa82492746be02545aba6d710f097e013e19e",
-      strip_prefix = "mkl-dnn-0.12",
-      build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"),
+      sha256 = "02e244f63dd95402691a361392504c143eede9a89043426f174836638a9cbf09",
+      strip_prefix = "mkl-dnn-e0bfcaa7fcb2b1e1558f5f0676933c1db807a729",
+      build_file = str(Label("//third_party/mkl_dnn:mkldnn.BUILD")),
   )
 
   tf_http_archive(
@@ -93,7 +68,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
      sha256 = "5996380e3e8b981f55d1c8d58e709c00dbb4806ba367be75d0925a68cc2f6478",
      strip_prefix = "abseil-cpp-720c017e30339fd1786ce4aac68bc8559736e53f",
-     build_file = clean_dep("//third_party:com_google_absl.BUILD"),
+     build_file = str(Label("//third_party:com_google_absl.BUILD")),
   )
 
   tf_http_archive(
@@ -104,8 +79,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0cadb31a35b514bf2dfd6b5d38205da94ef326ec6908fc3fd7c269948467214f",
       strip_prefix = "eigen-eigen-2355b229ea4c",
-      build_file = clean_dep("//third_party:eigen.BUILD"),
-      patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch")
+      build_file = str(Label("//third_party:eigen.BUILD")),
+      patch_file = str(Label("//third_party:eigen_fix_cuda_compilation.patch"))
   )
 
   tf_http_archive(
@@ -118,7 +93,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           # remove the whitelist entry in third_party/repo.bzl.
           # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
       ],
-      build_file = clean_dep("//:arm_compiler.BUILD"),
+      build_file = str(Label("//:arm_compiler.BUILD")),
   )
 
   tf_http_archive(
@@ -129,7 +104,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce",
       strip_prefix = "libxsmm-1.8.1",
-      build_file = clean_dep("//third_party:libxsmm.BUILD"),
+      build_file = str(Label("//third_party:libxsmm.BUILD")),
   )
 
   tf_http_archive(
@@ -142,7 +117,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
       strip_prefix = "or-tools-253f7955c6a1fd805408fba2e42ac6d45b312d15/src",
-      build_file = clean_dep("//third_party:ortools.BUILD"),
+      build_file = str(Label("//third_party:ortools.BUILD")),
   )
 
   tf_http_archive(
@@ -174,7 +149,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0",
       strip_prefix = "farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45",
-      build_file = clean_dep("//third_party:farmhash.BUILD"),
+      build_file = str(Label("//third_party:farmhash.BUILD")),
   )
 
   tf_http_archive(
@@ -185,7 +160,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9",
       strip_prefix = "highwayhash-dfcb97ca4fe9277bf9dc1802dd979b071896453b",
-      build_file = clean_dep("//third_party:highwayhash.BUILD"),
+      build_file = str(Label("//third_party:highwayhash.BUILD")),
   )
 
   tf_http_archive(
@@ -196,7 +171,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324",
       strip_prefix = "nasm-2.12.02",
-      build_file = clean_dep("//third_party:nasm.BUILD"),
+      build_file = str(Label("//third_party:nasm.BUILD")),
   )
 
   tf_http_archive(
@@ -207,7 +182,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77",
       strip_prefix = "libjpeg-turbo-1.5.1",
-      build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"),
+      build_file = str(Label("//third_party/jpeg:jpeg.BUILD")),
   )
 
   tf_http_archive(
@@ -218,7 +193,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2",
       strip_prefix = "libpng-1.2.53",
-      build_file = clean_dep("//third_party:png.BUILD"),
+      build_file = str(Label("//third_party:png.BUILD")),
   )
 
   tf_http_archive(
@@ -229,7 +204,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
       strip_prefix = "sqlite-amalgamation-3200000",
-      build_file = clean_dep("//third_party:sqlite.BUILD"),
+      build_file = str(Label("//third_party:sqlite.BUILD")),
   )
 
   tf_http_archive(
@@ -240,7 +215,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1",
       strip_prefix = "giflib-5.1.4",
-      build_file = clean_dep("//third_party:gif.BUILD"),
+      build_file = str(Label("//third_party:gif.BUILD")),
   )
 
   tf_http_archive(
@@ -251,7 +226,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
       strip_prefix = "six-1.10.0",
-      build_file = clean_dep("//third_party:six.BUILD"),
+      build_file = str(Label("//third_party:six.BUILD")),
   )
 
   tf_http_archive(
@@ -262,7 +237,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d",
       strip_prefix = "astor-0.6.2",
-      build_file = clean_dep("//third_party:astor.BUILD"),
+      build_file = str(Label("//third_party:astor.BUILD")),
   )
 
   tf_http_archive(
@@ -273,7 +248,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "7068908321ecd2774f145193c4b34a11305bd104b4551b09273dfd1d6a374930",
       strip_prefix = "gast-0.2.0",
-      build_file = clean_dep("//third_party:gast.BUILD"),
+      build_file = str(Label("//third_party:gast.BUILD")),
   )
 
   tf_http_archive(
@@ -284,7 +259,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b",
       strip_prefix = "termcolor-1.1.0",
-      build_file = clean_dep("//third_party:termcolor.BUILD"),
+      build_file = str(Label("//third_party:termcolor.BUILD")),
   )
 
   tf_http_archive(
@@ -305,7 +280,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892",
       strip_prefix = "backports.weakref-1.0rc1/src",
-      build_file = clean_dep("//third_party:backports_weakref.BUILD"),
+      build_file = str(Label("//third_party:backports_weakref.BUILD")),
   )
 
   tf_http_archive(
@@ -316,7 +291,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
       strip_prefix = "codegen-1.0",
-      build_file = clean_dep("//third_party:codegen.BUILD"),
+      build_file = str(Label("//third_party:codegen.BUILD")),
   )
 
   filegroup_external(
@@ -401,7 +376,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
       ],
       strip_prefix = "pcre-8.39",
-      build_file = clean_dep("//third_party:pcre.BUILD"),
+      build_file = str(Label("//third_party:pcre.BUILD")),
   )
 
   tf_http_archive(
@@ -413,7 +388,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
       ],
       strip_prefix = "swig-3.0.8",
-      build_file = clean_dep("//third_party:swig.BUILD"),
+      build_file = str(Label("//third_party:swig.BUILD")),
   )
 
   tf_http_archive(
@@ -424,7 +399,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://curl.haxx.se/download/curl-7.49.1.tar.gz",
       ],
       strip_prefix = "curl-7.49.1",
-      build_file = clean_dep("//third_party:curl.BUILD"),
+      build_file = str(Label("//third_party:curl.BUILD")),
   )
 
   tf_http_archive(
@@ -446,7 +421,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
       ],
       strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3",
-      build_file = clean_dep("//third_party:linenoise.BUILD"),
+      build_file = str(Label("//third_party:linenoise.BUILD")),
   )
 
   # TODO(phawkins): currently, this rule uses an unofficial LLVM mirror.
@@ -459,7 +434,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
       strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
-      build_file = clean_dep("//third_party/llvm:llvm.BUILD"),
+      build_file = str(Label("//third_party/llvm:llvm.BUILD")),
   )
 
   tf_http_archive(
@@ -470,7 +445,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326",
       strip_prefix = "lmdb-LMDB_0.9.19/libraries/liblmdb",
-      build_file = clean_dep("//third_party:lmdb.BUILD"),
+      build_file = str(Label("//third_party:lmdb.BUILD")),
   )
 
   tf_http_archive(
@@ -481,7 +456,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2",
       strip_prefix = "jsoncpp-11086dd6a7eba04289944367ca82cea71299ed70",
-      build_file = clean_dep("//third_party:jsoncpp.BUILD"),
+      build_file = str(Label("//third_party:jsoncpp.BUILD")),
   )
 
   tf_http_archive(
@@ -502,7 +477,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
       strip_prefix = "zlib-1.2.8",
-      build_file = clean_dep("//third_party:zlib.BUILD"),
+      build_file = str(Label("//third_party:zlib.BUILD")),
   )
 
   tf_http_archive(
@@ -512,7 +487,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
       ],
       sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296",
-      build_file = clean_dep("//third_party/fft2d:fft2d.BUILD"),
+      build_file = str(Label("//third_party/fft2d:fft2d.BUILD")),
   )
 
   tf_http_archive(
@@ -523,7 +498,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
       strip_prefix = "snappy-1.1.4",
-      build_file = clean_dep("//third_party:snappy.BUILD"),
+      build_file = str(Label("//third_party:snappy.BUILD")),
   )
 
   tf_http_archive(
@@ -534,7 +509,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
       strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7",
-      build_file = clean_dep("//third_party:nccl.BUILD"),
+      build_file = str(Label("//third_party:nccl.BUILD")),
   )
 
   tf_http_archive(
@@ -545,8 +520,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "dd035d57c8f19b0b612dd6eefe6e5eebad76f506e302cccb7c2066f25a83585e",
       strip_prefix = "librdkafka-0.11.1",
-      build_file = clean_dep("//third_party:kafka/BUILD"),
-      patch_file = clean_dep("//third_party/kafka:config.patch"),
+      build_file = str(Label("//third_party:kafka/BUILD")),
+      patch_file = str(Label("//third_party/kafka:config.patch")),
   )
 
   tf_http_archive(
@@ -557,7 +532,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
       strip_prefix = "aws-sdk-cpp-1.3.15",
-      build_file = clean_dep("//third_party:aws.BUILD"),
+      build_file = str(Label("//third_party:aws.BUILD")),
   )
 
   java_import_external(
@@ -593,7 +568,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8",
       strip_prefix = "jemalloc-4.4.0",
-      build_file = clean_dep("//third_party:jemalloc.BUILD"),
+      build_file = str(Label("//third_party:jemalloc.BUILD")),
   )
 
   java_import_external(
@@ -638,7 +613,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4",
       strip_prefix = "pprof-c0fb62ec88c411cc91194465e54db2632845b650",
-      build_file = clean_dep("//third_party:pprof.BUILD"),
+      build_file = str(Label("//third_party:pprof.BUILD")),
   )
 
   tf_http_archive(
@@ -649,7 +624,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
       strip_prefix = "cub-1.8.0",
-      build_file = clean_dep("//third_party:cub.BUILD"),
+      build_file = str(Label("//third_party:cub.BUILD")),
   )
 
   tf_http_archive(
@@ -660,7 +635,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
       ],
       strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
-      build_file = clean_dep("//third_party:cython.BUILD"),
+      build_file = str(Label("//third_party:cython.BUILD")),
       delete = ["BUILD.bazel"],
   )
 
@@ -682,7 +657,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
           "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
       ],
-      build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
+      build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
   )
 
   tf_http_archive(
@@ -693,7 +668,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
           "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
       ],
-      build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
+      build_file = str(Label("//third_party/flatbuffers:flatbuffers.BUILD")),
   )
 
   tf_http_archive(
@@ -703,7 +678,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
       ],
-      build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
+      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
   )
 
   tf_http_archive(
@@ -713,7 +688,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip"
       ],
-      build_file = clean_dep("//third_party:tflite_smartreply.BUILD"),
+      build_file = str(Label("//third_party:tflite_smartreply.BUILD")),
   )
 
   ##############################################################################
@@ -777,7 +752,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # Needed by Protobuf
   native.bind(
       name = "python_headers",
-      actual = clean_dep("//util/python:python_headers"),
+      actual = str(Label("//util/python:python_headers")),
   )
 
   # Needed by Protobuf
diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD
index c2adf578c7..b27d341404 100644
--- a/third_party/mkl/BUILD
+++ b/third_party/mkl/BUILD
@@ -1,17 +1,10 @@
 licenses(["notice"])  # 3-Clause BSD
 
-config_setting(
-    name = "using_mkl",
-    values = {
-        "define": "using_mkl=true",
-    },
-    visibility = ["//visibility:public"],
-)
+exports_files(["LICENSE"])
 
 config_setting(
-    name = "using_mkl_lnx_x64",
+    name = "using_mkl",
     values = {
-        "cpu": "k8",
         "define": "using_mkl=true",
     },
     visibility = ["//visibility:public"],
@@ -22,37 +15,12 @@ load(
     "if_mkl",
 )
 
-filegroup(
-    name = "LICENSE",
-    srcs = ["MKL_LICENSE"] + select({
-        "@org_tensorflow//tensorflow:linux_x86_64": [
-            "@mkl_linux//:LICENSE",
-        ],
-        "@org_tensorflow//tensorflow:darwin": [
-            "@mkl_darwin//:LICENSE",
-        ],
-        "@org_tensorflow//tensorflow:windows": [
-            "@mkl_windows//:LICENSE",
-        ],
-    }),
-    visibility = ["//visibility:public"],
-)
-
 cc_library(
     name = "intel_binary_blob",
+    srcs = if_mkl([
+        "@mkl//:libmklml_intel.so",
+        "@mkl//:libiomp5.so",
+    ]),
     visibility = ["//visibility:public"],
-    deps = select({
-        "@org_tensorflow//tensorflow:linux_x86_64": [
-            "@mkl_linux//:mkl_headers",
-            "@mkl_linux//:mkl_libs_linux",
-        ],
-        "@org_tensorflow//tensorflow:darwin": [
-            "@mkl_darwin//:mkl_headers",
-            "@mkl_darwin//:mkl_libs_darwin",
-        ],
-        "@org_tensorflow//tensorflow:windows": [
-            "@mkl_windows//:mkl_headers",
-            "@mkl_windows//:mkl_libs_windows",
-        ],
-    }),
+    deps = ["@mkl//:mkl_headers"],
 )
diff --git a/third_party/mkl/MKL_LICENSE b/third_party/mkl/MKL_LICENSE
deleted file mode 100644
index 9c8f3ea087..0000000000
--- a/third_party/mkl/MKL_LICENSE
+++ /dev/null
@@ -1,201 +0,0 @@
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "{}"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright {yyyy} {name of copyright owner}
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.
\ No newline at end of file
diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl
index 53e02769da..8b73ddabdd 100644
--- a/third_party/mkl/build_defs.bzl
+++ b/third_party/mkl/build_defs.bzl
@@ -24,18 +24,6 @@ def if_mkl(if_true, if_false = []):
         "//conditions:default": if_false
     })
 
-def if_mkl_lnx_x64(if_true, if_false = []):
-    """Shorthand for select()'ing on whether we're building with MKL.
-
-    Returns a select statement which evaluates to if_true if we're building
-    with MKL enabled.  Otherwise, the select statement evaluates to if_false.
-
-    """
-    return select({
-        str(Label("//third_party/mkl:using_mkl_lnx_x64")): if_true,
-        "//conditions:default": if_false
-    })
-
 
 def _enable_local_mkl(repository_ctx):
   return _TF_MKL_ROOT in repository_ctx.os.environ
diff --git a/third_party/mkl/mkl.BUILD b/third_party/mkl/mkl.BUILD
index c3a71e4ff9..8db97232e1 100644
--- a/third_party/mkl/mkl.BUILD
+++ b/third_party/mkl/mkl.BUILD
@@ -17,29 +17,14 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-cc_library(
-    name = "mkl_libs_linux",
-    srcs = [
-        "lib/libiomp5.so",
-        "lib/libmklml_intel.so",
-    ],
-    visibility = ["//visibility:public"],
-)
-
-cc_library(
-    name = "mkl_libs_darwin",
-    srcs = [
-        "lib/libiomp5.dylib",
-        "lib/libmklml.dylib",
-    ],
+filegroup(
+    name = "libmklml_intel.so",
+    srcs = ["lib/libmklml_intel.so"],
     visibility = ["//visibility:public"],
 )
 
-cc_library(
-    name = "mkl_libs_windows",
-    srcs = [
-        "lib/libiomp5md.lib",
-        "lib/mklml.lib",
-    ],
+filegroup(
+    name = "libiomp5.so",
+    srcs = ["lib/libiomp5.so"],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From a0e0685ca974e484de9200caf8c414dcb55277bb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 17:06:44 -0700
Subject: [PATCH 647/960] Collective Ops Part 1

The basic interface definitions, local-only versions of remote-access,
param-resolution, device-resolution and mgr.

A collective op is able to execute synchronously across devices
and across separate graphs. Collective ops to be introduced eventually
include broadcast and all-reduce.  This change is part of a series of
changes that will introduce the necessary infrastructure then the
initial op implementations.

PiperOrigin-RevId: 190860248
---
 tensorflow/core/BUILD                         |  16 +
 .../core/common_runtime/buf_rendezvous.cc     | 166 +++++
 .../core/common_runtime/buf_rendezvous.h      | 103 +++
 .../common_runtime/buf_rendezvous_test.cc     | 197 ++++++
 .../common_runtime/collective_executor_mgr.cc | 114 +++
 .../common_runtime/collective_executor_mgr.h  |  70 ++
 .../collective_executor_mgr_test.cc           |  98 +++
 .../collective_param_resolver_local.cc        | 666 ++++++++++++++++++
 .../collective_param_resolver_local.h         | 209 ++++++
 .../collective_param_resolver_local_test.cc   | 151 ++++
 .../common_runtime/collective_rma_local.cc    | 108 +++
 .../common_runtime/collective_rma_local.h     |  88 +++
 .../collective_rma_local_test.cc              | 148 ++++
 .../common_runtime/device_resolver_local.cc   |  49 ++
 .../common_runtime/device_resolver_local.h    |  48 ++
 .../device_resolver_local_test.cc             |  87 +++
 tensorflow/core/framework/collective.cc       | 120 ++++
 tensorflow/core/framework/collective.h        | 308 ++++++++
 tensorflow/core/framework/op_kernel.h         |   1 +
 19 files changed, 2747 insertions(+)
 create mode 100644 tensorflow/core/common_runtime/buf_rendezvous.cc
 create mode 100644 tensorflow/core/common_runtime/buf_rendezvous.h
 create mode 100644 tensorflow/core/common_runtime/buf_rendezvous_test.cc
 create mode 100644 tensorflow/core/common_runtime/collective_executor_mgr.cc
 create mode 100644 tensorflow/core/common_runtime/collective_executor_mgr.h
 create mode 100644 tensorflow/core/common_runtime/collective_executor_mgr_test.cc
 create mode 100644 tensorflow/core/common_runtime/collective_param_resolver_local.cc
 create mode 100644 tensorflow/core/common_runtime/collective_param_resolver_local.h
 create mode 100644 tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
 create mode 100644 tensorflow/core/common_runtime/collective_rma_local.cc
 create mode 100644 tensorflow/core/common_runtime/collective_rma_local.h
 create mode 100644 tensorflow/core/common_runtime/collective_rma_local_test.cc
 create mode 100644 tensorflow/core/common_runtime/device_resolver_local.cc
 create mode 100644 tensorflow/core/common_runtime/device_resolver_local.h
 create mode 100644 tensorflow/core/common_runtime/device_resolver_local_test.cc
 create mode 100644 tensorflow/core/framework/collective.cc
 create mode 100644 tensorflow/core/framework/collective.h

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 4726946277..712106492b 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -455,6 +455,7 @@ tf_cuda_library(
         "framework/attr_value_util.h",
         "framework/bfloat16.h",
         "framework/cancellation.h",
+        "framework/collective.h",
         "framework/common_shape_fns.h",
         "framework/control_flow.h",  # TODO(josh11b): Make internal?
         "framework/dataset.h",
@@ -2172,6 +2173,11 @@ tf_cuda_library(
 CORE_CPU_LIB_HEADERS = CORE_CPU_BASE_HDRS + [
     "common_runtime/allocator_retry.h",
     "common_runtime/bfc_allocator.h",
+    "common_runtime/collective_executor_mgr.h",
+    "common_runtime/collective_param_resolver_local.h",
+    "common_runtime/collective_rma_local.h",
+    "common_runtime/device_resolver_local.h",
+    "common_runtime/buf_rendezvous.h",
     "common_runtime/build_graph_options.h",
     "common_runtime/constant_folding.h",
     "common_runtime/copy_tensor.h",
@@ -2210,7 +2216,11 @@ tf_cuda_library(
         "common_runtime/accumulate_n_optimizer.cc",
         "common_runtime/allocator_retry.cc",
         "common_runtime/bfc_allocator.cc",
+        "common_runtime/buf_rendezvous.cc",
         "common_runtime/build_graph_options.cc",
+        "common_runtime/collective_executor_mgr.cc",
+        "common_runtime/collective_param_resolver_local.cc",
+        "common_runtime/collective_rma_local.cc",
         "common_runtime/constant_folding.cc",
         "common_runtime/copy_tensor.cc",
         "common_runtime/costmodel_manager.cc",
@@ -2218,6 +2228,7 @@ tf_cuda_library(
         "common_runtime/device.cc",
         "common_runtime/device_factory.cc",
         "common_runtime/device_mgr.cc",
+        "common_runtime/device_resolver_local.cc",
         "common_runtime/device_set.cc",
         "common_runtime/executor.cc",
         "common_runtime/function.cc",
@@ -2825,6 +2836,11 @@ tf_cc_tests(
     name = "higher_level_tests",
     size = "small",
     srcs = [
+        "common_runtime/buf_rendezvous_test.cc",
+        "common_runtime/collective_executor_mgr_test.cc",
+        "common_runtime/collective_param_resolver_local_test.cc",
+        "common_runtime/collective_rma_local_test.cc",
+        "common_runtime/device_resolver_local_test.cc",
         "common_runtime/device_set_test.cc",
         "common_runtime/optimization_registry_test.cc",
         "common_runtime/pending_counts_test.cc",
diff --git a/tensorflow/core/common_runtime/buf_rendezvous.cc b/tensorflow/core/common_runtime/buf_rendezvous.cc
new file mode 100644
index 0000000000..b57eb2943a
--- /dev/null
+++ b/tensorflow/core/common_runtime/buf_rendezvous.cc
@@ -0,0 +1,166 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/buf_rendezvous.h"
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/process_util.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/notification.h"
+
+namespace tensorflow {
+
+BufRendezvous::~BufRendezvous() {
+  mutex_lock l(mu_);
+  if (!hook_table_.empty()) {
+    PurgeTable(errors::Internal("Delete called on non-empty BufRendezvous"),
+               &hook_table_);
+  }
+}
+
+void BufRendezvous::StartAbort(const Status& s) {
+  CHECK(!s.ok());
+  HookTable dummy_table;
+  {
+    mutex_lock l(mu_);
+    status_.Update(s);
+    hook_table_.swap(dummy_table);
+  }
+  PurgeTable(s, &dummy_table);
+}
+
+void BufRendezvous::PurgeTable(const Status& s, HookTable* table) {
+  for (auto& it : *table) {
+    Hook* h = it.second;
+    if (h->cons_cb != nullptr) {
+      h->cons_cb(s, nullptr);
+    }
+    if (h->prod_cb != nullptr) {
+      h->prod_cb(s);
+    }
+    delete h;
+  }
+  table->clear();
+}
+
+string BufRendezvous::Hook::DebugString() const {
+  return strings::StrCat("[dev:", (prod_dev ? prod_dev->name() : "none"),
+                         ", ctx:", reinterpret_cast<uint64>(prod_ctx),
+                         ", val:", reinterpret_cast<uint64>(prod_value),
+                         ", pcb:", reinterpret_cast<uint64>(&prod_cb),
+                         ", ccb:", reinterpret_cast<uint64>(&cons_cb), "]");
+}
+
+void BufRendezvous::ProvideBuf(const string& key, Device* dev,
+                               DeviceContext* dev_ctx, const Tensor* v,
+                               const AllocatorAttributes& attr,
+                               const ProducerCallback& done) {
+  Hook* h = nullptr;
+  Status providebuf_status;
+  do {
+    mutex_lock l(mu_);
+    if (!status_.ok()) {
+      providebuf_status = status_;
+      break;
+    } else {
+      auto it = hook_table_.find(key);
+      if (it == hook_table_.end()) {
+        h = new Hook;
+        it = hook_table_.insert(std::make_pair(key, h)).first;
+      } else {
+        if (it->second->prod_cb != nullptr) {
+          providebuf_status = errors::Internal(
+              "BufRendezvous::ProvideBuf already called for key ", key);
+          break;
+        }
+        h = it->second;
+      }
+      // Populate Hook with all of the prod values.
+      h->prod_dev = dev;
+      h->prod_ctx = dev_ctx;
+      h->prod_value = v;
+      h->prod_attr = attr;
+      h->prod_cb = done;
+      // If consumer is waiting, kick off right away, removing Hook from table.
+      if (h->cons_cb != nullptr) {
+        hook_table_.erase(it);
+      } else {
+        h = nullptr;
+      }
+    }
+  } while (false);
+  if (h) {
+    h->cons_cb(Status::OK(), h);
+  }
+  if (!providebuf_status.ok()) {
+    done(providebuf_status);
+  }
+}
+
+void BufRendezvous::ConsumeBuf(const string& key,
+                               const ConsumerCallback& done) {
+  Hook* existing_hook = nullptr;
+  Status consumebuf_status;
+  do {
+    mutex_lock l(mu_);
+    if (!status_.ok()) {
+      consumebuf_status = status_;
+      break;
+    }
+    auto it = hook_table_.find(key);
+    if (it != hook_table_.end()) {
+      // Prepare to consume immediately.
+      if (it->second->cons_cb) {
+        consumebuf_status =
+            errors::Internal("Second consumer arrived for key ", key);
+        break;
+      }
+      existing_hook = it->second;
+      hook_table_.erase(it);
+      existing_hook->cons_cb = done;
+    } else {
+      // Hang consumer callback on the Hook.
+      Hook* h = new Hook;
+      hook_table_[key] = h;
+      h->cons_cb = done;
+      return;
+    }
+  } while (false);
+  if (existing_hook) {
+    existing_hook->cons_cb(Status::OK(), existing_hook);
+    return;
+  }
+  if (!consumebuf_status.ok()) {
+    done(consumebuf_status, nullptr);
+    return;
+  }
+}
+
+/*static*/
+void BufRendezvous::DoneWithHook(Hook* h) {
+  h->prod_cb(Status::OK());
+  delete h;
+}
+
+void BufRendezvous::LogContents() {
+  mutex_lock l(mu_);
+  LOG(INFO) << strings::StrCat("BufRendezvous ",
+                               strings::Hex(reinterpret_cast<uint64>(this)),
+                               " step_id=", step_id_, " current contents:");
+  for (auto it : hook_table_) {
+    LOG(INFO) << it.first << ":" << it.second->DebugString();
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/buf_rendezvous.h b/tensorflow/core/common_runtime/buf_rendezvous.h
new file mode 100644
index 0000000000..e94e88b323
--- /dev/null
+++ b/tensorflow/core/common_runtime/buf_rendezvous.h
@@ -0,0 +1,103 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
+#define TENSORFLOW_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
+
+#include <functional>
+#include <string>
+
+#include "tensorflow/core/framework/allocator.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
+#include "tensorflow/core/platform/mutex.h"
+
+namespace tensorflow {
+class Device;
+class DeviceContext;
+class Tensor;
+
+// EXPERIMENTAL: RDMA oriented producer/consumer rendezvous on a local
+// Tensor value for which DMAHelper::CanUseDMA() is true, i.e. dense
+// numeric types.  Similar to Rendezvous but never owns a Ref on the
+// tensor, instead it uses an explicit callback to the producer when
+// the consumer side is finished with the value.  This allows the
+// producer to perform in-place updates on the source buffer or to take
+// other actions that depend on knowing the consumer has passed a certain
+// execution point.
+class BufRendezvous {
+ public:
+  explicit BufRendezvous(uint64 step_id) : step_id_(step_id) {}
+
+  ~BufRendezvous();
+
+  // Inform all all waiting parties that this BufRendezvous is defunct
+  // because of an error Status interrupting the Step.
+  void StartAbort(const Status& s);
+
+  struct Hook;
+  // Provided by the consumer to be called when access to the buffer
+  // is available.  If the Status arg is not OK, then hook will not
+  // be populated.  Ownership of Hook passes to consumer with the
+  // callback.
+  typedef std::function<void(const Status&, Hook*)> ConsumerCallback;
+  // Provided by the producer to be called when the consumer has finished
+  // reading the buffer and will no longer access it.
+  typedef std::function<void(const Status&)> ProducerCallback;
+
+  struct Hook {
+    Device* prod_dev;
+    DeviceContext* prod_ctx;
+    const Tensor* prod_value;
+    AllocatorAttributes prod_attr;
+    ProducerCallback prod_cb;
+    ConsumerCallback cons_cb;
+    Hook()
+        : prod_dev(nullptr),
+          prod_ctx(nullptr),
+          prod_value(nullptr),
+          prod_cb(nullptr),
+          cons_cb(nullptr) {}
+    string DebugString() const;
+  };
+
+  // Called to advertise availability of a Tensor value corresponding
+  // to key.  That value must stay valid until done is called.
+  void ProvideBuf(const string& key, Device* dev, DeviceContext* dev_ctx,
+                  const Tensor* v, const AllocatorAttributes& attr,
+                  const ProducerCallback& done);
+
+  // Called to request access to a Tensor value corresponding to key.
+  // Consumer is provide with a Hook as soon as availble.
+  void ConsumeBuf(const string& key, const ConsumerCallback& done);
+
+  // Consumer must call this function when it's done reading the Hook provided
+  // by the ConsumerCallback.  This function will invoke the producer callback
+  // and then delete h.
+  static void DoneWithHook(Hook* h);
+
+  // Write the current contents of the table to the INFO log.
+  void LogContents();
+
+ protected:
+  const uint64 step_id_;
+  mutex mu_;
+  Status status_ GUARDED_BY(mu_);
+  typedef gtl::FlatMap<string, Hook*> HookTable;
+  HookTable hook_table_ GUARDED_BY(mu_);
+
+  void PurgeTable(const Status& s, HookTable* table);
+};
+}  // namespace tensorflow
+#endif  // TENSORFLOW_COMMON_RUNTIME_BUF_RENDEZVOUS_H_
diff --git a/tensorflow/core/common_runtime/buf_rendezvous_test.cc b/tensorflow/core/common_runtime/buf_rendezvous_test.cc
new file mode 100644
index 0000000000..0e798235bf
--- /dev/null
+++ b/tensorflow/core/common_runtime/buf_rendezvous_test.cc
@@ -0,0 +1,197 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/buf_rendezvous.h"
+
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/types.pb.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+#define NUM_DEVS 3
+
+class BufRendezvousTest : public ::testing::Test {
+ protected:
+  BufRendezvousTest() {
+    br_.reset(new BufRendezvous(123));
+    fake_dev_ptr_ = reinterpret_cast<Device*>(512LLU);
+    fake_dev_ctx_ = reinterpret_cast<DeviceContext*>(1024LLU);
+    a_ = Tensor(DT_FLOAT, TensorShape({24}));
+    b_ = Tensor(DT_FLOAT, TensorShape({24}));
+  }
+
+  Device* fake_dev_ptr_ = nullptr;
+  DeviceContext* fake_dev_ctx_ = nullptr;
+  Tensor a_;
+  Tensor b_;
+  AllocatorAttributes aa_;
+  std::unique_ptr<BufRendezvous> br_;
+};
+
+TEST_F(BufRendezvousTest, CorrectUseProducerFirst) {
+  Status prod_status;
+  Status cons_status;
+  bool prod_callback_called = false;
+  bool cons_callback_called = false;
+  Notification note;
+  br_->ProvideBuf(
+      "key0", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+      [&note, &prod_status, &prod_callback_called](const Status& s) {
+        prod_status = s;
+        prod_callback_called = true;
+        note.Notify();
+      });
+  EXPECT_FALSE(prod_callback_called);
+  br_->ConsumeBuf("key0", [this, &cons_status, &cons_callback_called](
+                              const Status& s, BufRendezvous::Hook* h) {
+    cons_status = s;
+    cons_callback_called = true;
+    ASSERT_TRUE(h != nullptr);
+    EXPECT_EQ(h->prod_dev, fake_dev_ptr_);
+    EXPECT_EQ(h->prod_ctx, fake_dev_ctx_);
+    EXPECT_EQ(h->prod_value, &a_);
+    br_->DoneWithHook(h);
+  });
+  EXPECT_TRUE(cons_callback_called);
+  note.WaitForNotification();
+  EXPECT_TRUE(prod_callback_called);
+  TF_EXPECT_OK(cons_status);
+  TF_EXPECT_OK(prod_status);
+}
+
+TEST_F(BufRendezvousTest, CorrectUseConsumerFirst) {
+  Status prod_status;
+  Status cons_status;
+  bool prod_callback_called = false;
+  bool cons_callback_called = false;
+  Notification note;
+  br_->ConsumeBuf("key0", [this, &cons_status, &cons_callback_called](
+                              const Status& s, BufRendezvous::Hook* h) {
+    cons_status = s;
+    cons_callback_called = true;
+    ASSERT_TRUE(h != nullptr);
+    EXPECT_EQ(h->prod_dev, fake_dev_ptr_);
+    EXPECT_EQ(h->prod_ctx, fake_dev_ctx_);
+    EXPECT_EQ(h->prod_value, &a_);
+    br_->DoneWithHook(h);
+  });
+  EXPECT_FALSE(cons_callback_called);
+  br_->ProvideBuf(
+      "key0", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+      [&note, &prod_status, &prod_callback_called](const Status& s) {
+        prod_status = s;
+        prod_callback_called = true;
+        note.Notify();
+      });
+  EXPECT_TRUE(cons_callback_called);
+  note.WaitForNotification();
+  EXPECT_TRUE(prod_callback_called);
+  TF_EXPECT_OK(cons_status);
+  TF_EXPECT_OK(prod_status);
+}
+
+TEST_F(BufRendezvousTest, ErrorDuplicatePut) {
+  bool prod_callback_called = false;
+  br_->ProvideBuf("key0", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+                  [this, &prod_callback_called](const Status& s) {
+                    prod_callback_called = true;
+                  });
+  Status bad_status;
+  Notification note;
+  br_->ProvideBuf("key0", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+                  [&bad_status, &note](const Status& s) {
+                    bad_status = s;
+                    note.Notify();
+                  });
+  note.WaitForNotification();
+  EXPECT_FALSE(bad_status.ok());
+  EXPECT_EQ("BufRendezvous::ProvideBuf already called for key key0",
+            bad_status.error_message());
+  EXPECT_FALSE(prod_callback_called);
+  br_.reset();
+}
+
+TEST_F(BufRendezvousTest, ErrorDeleteNonEmpty) {
+  Status cons_status;
+  br_->ConsumeBuf(
+      "key0", [this, &cons_status](const Status& s, BufRendezvous::Hook* h) {
+        cons_status = s;
+        EXPECT_EQ(h, nullptr);
+      });
+  EXPECT_TRUE(cons_status.ok());
+  br_.reset();
+  EXPECT_FALSE(cons_status.ok());
+  EXPECT_EQ("Delete called on non-empty BufRendezvous",
+            cons_status.error_message());
+}
+
+TEST_F(BufRendezvousTest, AbortNonEmpty) {
+  Status cons_status;
+  Status prod_status;
+  Notification prod_note;
+  Notification cons_note;
+  br_->ConsumeBuf("key0", [this, &cons_note, &cons_status](
+                              const Status& s, BufRendezvous::Hook* h) {
+    cons_status = s;
+    cons_note.Notify();
+  });
+  br_->ProvideBuf("key1", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+                  [this, &prod_note, &prod_status](const Status& s) {
+                    prod_status = s;
+                    prod_note.Notify();
+                  });
+  br_->StartAbort(errors::Internal("Falling sky detected"));
+  prod_note.WaitForNotification();
+  cons_note.WaitForNotification();
+  EXPECT_FALSE(prod_status.ok());
+  EXPECT_EQ(prod_status.error_message(), "Falling sky detected");
+  EXPECT_FALSE(cons_status.ok());
+  EXPECT_EQ(cons_status.error_message(), "Falling sky detected");
+}
+
+TEST_F(BufRendezvousTest, AbortEmpty) {
+  br_->StartAbort(errors::Internal("Falling sky detected"));
+}
+
+TEST_F(BufRendezvousTest, UseAfterAbort) {
+  br_->StartAbort(errors::Internal("Falling sky detected"));
+  Status cons_status;
+  Status prod_status;
+  Notification prod_note;
+  Notification cons_note;
+  br_->ConsumeBuf("key0", [this, &cons_note, &cons_status](
+                              const Status& s, BufRendezvous::Hook* h) {
+    cons_status = s;
+    cons_note.Notify();
+  });
+  br_->ProvideBuf("key1", fake_dev_ptr_, fake_dev_ctx_, &a_, aa_,
+                  [this, &prod_note, &prod_status](const Status& s) {
+                    prod_status = s;
+                    prod_note.Notify();
+                  });
+  prod_note.WaitForNotification();
+  cons_note.WaitForNotification();
+  EXPECT_FALSE(prod_status.ok());
+  EXPECT_EQ(prod_status.error_message(), "Falling sky detected");
+  EXPECT_FALSE(cons_status.ok());
+  EXPECT_EQ(cons_status.error_message(), "Falling sky detected");
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr.cc b/tensorflow/core/common_runtime/collective_executor_mgr.cc
new file mode 100644
index 0000000000..a5c4946e58
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_executor_mgr.cc
@@ -0,0 +1,114 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_executor_mgr.h"
+
+#include "tensorflow/core/common_runtime/build_graph_options.h"
+#include "tensorflow/core/common_runtime/collective_rma_local.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/protobuf/config.pb.h"
+
+namespace tensorflow {
+namespace {
+// TODO(tucker): Temporary class just until a real CollectiveExecutor
+// implementation is submitted in a later CL.
+class DummyCollectiveExecutor : public CollectiveExecutor {
+ public:
+  explicit DummyCollectiveExecutor(CollectiveExecutorMgr* ce_mgr)
+      : CollectiveExecutor(ce_mgr) {}
+
+  ~DummyCollectiveExecutor() override {}
+
+  void RecvFromPeer(const string& peer_device, const string& peer_task,
+                    bool peer_is_local, const string& key, Device* to_device,
+                    DeviceContext* to_device_ctx,
+                    const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
+                    const DeviceLocality& client_locality,
+                    const StatusCallback& done) override {
+    done(errors::Internal("Unimplemented"));
+  }
+
+  void PostToPeer(const string& peer_device, const string& peer_task,
+                  const string& key, Device* from_device,
+                  DeviceContext* from_device_ctx,
+                  const AllocatorAttributes& from_alloc_attr,
+                  const Tensor* from_tensor,
+                  const DeviceLocality& client_locality,
+                  const StatusCallback& done) override {
+    done(errors::Internal("Unimplemented"));
+  }
+
+ private:
+  TF_DISALLOW_COPY_AND_ASSIGN(DummyCollectiveExecutor);
+};
+}  // namespace
+
+CollectiveExecutorMgr::CollectiveExecutorMgr(
+    const ConfigProto& config, const DeviceMgr* dev_mgr,
+    DeviceResolverInterface* dev_resolver,
+    ParamResolverInterface* param_resolver)
+    : dev_mgr_(dev_mgr),
+      dev_resolver_(dev_resolver),
+      param_resolver_(param_resolver) {}
+
+CollectiveExecutorMgr::~CollectiveExecutorMgr() {
+  for (auto iter : executor_table_) {
+    iter.second->Unref();
+  }
+}
+
+CollectiveExecutor* CollectiveExecutorMgr::FindOrCreate(int64 step_id) {
+  CollectiveExecutor* ce = nullptr;
+  {
+    mutex_lock l(exec_mu_);
+    auto it = executor_table_.find(step_id);
+    if (it != executor_table_.end()) {
+      ce = it->second;
+    } else {
+      ce = new DummyCollectiveExecutor(this);
+      executor_table_[step_id] = ce;
+    }
+    ce->Ref();
+  }
+  return ce;
+}
+
+void CollectiveExecutorMgr::Cleanup(int64 step_id) {
+  CollectiveExecutor* ce = nullptr;
+  {
+    mutex_lock l(exec_mu_);
+    auto it = executor_table_.find(step_id);
+    if (it != executor_table_.end()) {
+      ce = it->second;
+      executor_table_.erase(it);
+    }
+  }
+  if (ce) ce->Unref();
+}
+
+void CollectiveExecutorMgr::GetStepSequenceAsync(
+    const GetStepSequenceRequest* request, GetStepSequenceResponse* response,
+    const StatusCallback& done) {
+  done(errors::Internal(
+      "CollectiveExecutorMgr does not implement GetStepSequence."));
+}
+
+void CollectiveExecutorMgr::RefreshStepIdSequenceAsync(
+    int64 graph_key, const StatusCallback& done) {
+  done(errors::Internal(
+      "CollectiveExecutorMgr does not implement RefreshStepIdSequence."));
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr.h b/tensorflow/core/common_runtime/collective_executor_mgr.h
new file mode 100644
index 0000000000..4b42e2b4d1
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_executor_mgr.h
@@ -0,0 +1,70 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
+#define TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
+
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
+
+namespace tensorflow {
+class ConfigProto;
+class DeviceMgr;
+
+class CollectiveExecutorMgr : public CollectiveExecutorMgrInterface {
+ public:
+  CollectiveExecutorMgr(const ConfigProto& config, const DeviceMgr* dev_mgr,
+                        DeviceResolverInterface* dev_resolver,
+                        ParamResolverInterface* param_resolver);
+
+  virtual ~CollectiveExecutorMgr();
+
+  CollectiveExecutor* FindOrCreate(int64 step_id) override;
+
+  void Cleanup(int64 step_id) override;
+
+  ParamResolverInterface* GetParamResolver() const override {
+    return param_resolver_.get();
+  }
+
+  DeviceResolverInterface* GetDeviceResolver() const override {
+    return dev_resolver_.get();
+  }
+
+  void GetStepSequenceAsync(const GetStepSequenceRequest* request,
+                            GetStepSequenceResponse* response,
+                            const StatusCallback& done) override;
+
+  void RefreshStepIdSequenceAsync(int64 graph_key,
+                                  const StatusCallback& done) override;
+
+  int64 NextStepId(int64 graph_key) override {
+    return CollectiveExecutor::kInvalidId;
+  }
+
+  void RetireStepId(int64 graph_key, int64 step_id) override {}
+
+ protected:
+  const DeviceMgr* dev_mgr_;
+  std::unique_ptr<DeviceResolverInterface> dev_resolver_;
+  std::unique_ptr<ParamResolverInterface> param_resolver_;
+  CollectiveRemoteAccess* remote_access_;
+  string task_name_;
+  mutex exec_mu_;
+  // Map from step_id to CollectiveExecutor
+  gtl::FlatMap<int64, CollectiveExecutor*> executor_table_ GUARDED_BY(exec_mu_);
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_EXECUTOR_MGR_H_
diff --git a/tensorflow/core/common_runtime/collective_executor_mgr_test.cc b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
new file mode 100644
index 0000000000..34c9163d6a
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_executor_mgr_test.cc
@@ -0,0 +1,98 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_executor_mgr.h"
+
+#include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace {
+
+#define NUM_DEVS 3
+
+class CollectiveExecutorMgrTest : public ::testing::Test {
+ protected:
+  CollectiveExecutorMgrTest() {
+    ConfigProto cp;
+    SessionOptions options;
+    auto* device_count = options.config.mutable_device_count();
+    string task_name = "/job:localhost/replica:0/task:0";
+    device_count->insert({"CPU", NUM_DEVS});
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
+    DeviceResolverLocal* drl = new DeviceResolverLocal(device_mgr_.get());
+    cme_.reset(new CollectiveExecutorMgr(
+        cp, device_mgr_.get(), drl,
+        new CollectiveParamResolverLocal(device_mgr_.get(), drl, task_name)));
+  }
+
+  std::unique_ptr<CollectiveExecutorMgr> cme_;
+  std::vector<Device*> devices_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+};
+
+TEST_F(CollectiveExecutorMgrTest, FindOrCreate) {
+  CollectiveExecutor::Handle* h =
+      new CollectiveExecutor::Handle(cme_->FindOrCreate(1), true);
+  EXPECT_TRUE(h->get());
+  CollectiveExecutor::Handle* h2 =
+      new CollectiveExecutor::Handle(cme_->FindOrCreate(1), true);
+  EXPECT_EQ(h->get(), h2->get());
+  CollectiveExecutor* ce = h->get();
+  delete h;
+  delete h2;
+  CollectiveExecutor::Handle h3(cme_->FindOrCreate(1), true);
+  EXPECT_EQ(ce, h3.get());
+  cme_->Cleanup(1);
+}
+
+TEST_F(CollectiveExecutorMgrTest, StepSequenceRelated) {
+  EXPECT_EQ(CollectiveExecutor::kInvalidId, cme_->NextStepId(123));
+  Notification ss_note;
+  Status ss_status;
+  cme_->RefreshStepIdSequenceAsync(
+      123, [this, &ss_status, &ss_note](const Status& s) {
+        ss_status = s;
+        ss_note.Notify();
+      });
+  ss_note.WaitForNotification();
+  EXPECT_FALSE(ss_status.ok());
+  EXPECT_EQ(ss_status.error_message(),
+            "CollectiveExecutorMgr does not implement RefreshStepIdSequence.");
+  Notification gs_note;
+  Status gs_status;
+  GetStepSequenceRequest* req = nullptr;
+  GetStepSequenceResponse* resp = nullptr;
+  cme_->GetStepSequenceAsync(req, resp,
+                             [this, &gs_status, &gs_note](const Status& s) {
+                               gs_status = s;
+                               gs_note.Notify();
+                             });
+  gs_note.WaitForNotification();
+  EXPECT_FALSE(gs_status.ok());
+  EXPECT_EQ(gs_status.error_message(),
+            "CollectiveExecutorMgr does not implement GetStepSequence.");
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.cc b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
new file mode 100644
index 0000000000..b34950b2f4
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.cc
@@ -0,0 +1,666 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
+
+#include "tensorflow/core/common_runtime/device_mgr.h"
+
+namespace tensorflow {
+
+CollectiveParamResolverLocal::CollectiveParamResolverLocal(
+    const DeviceMgr* dev_mgr, DeviceResolverInterface* dev_resolver,
+    const string& task_name)
+    : dev_mgr_(dev_mgr), dev_resolver_(dev_resolver), task_name_(task_name) {}
+
+void CollectiveParamResolverLocal::CompleteGroupAsync(
+    const CompleteGroupRequest* request, CompleteGroupResponse* response,
+    CancellationManager* cancel_mgr, const StatusCallback& done) {
+  done(
+      errors::Internal("CompleteGroup is not implemented by "
+                       "CollectiveParamResolverLocal which is "
+                       "intended only for non-distributed deployment."));
+}
+
+void CollectiveParamResolverLocal::CompleteGroupLocal(
+    const string& device, CollectiveParams* cp, const GroupRecCallback& done) {
+  VLOG(1) << "CompleteGroupLocal " << cp << ": " << cp->ToString();
+  std::vector<StatusCallback> to_be_called;
+  GroupRec* gr = nullptr;
+  {
+    mutex_lock l(group_mu_);
+    auto it = group_table_.find(cp->group.group_key);
+    if (it == group_table_.end()) {
+      gr = new GroupRec;
+      gr->group.group_key = cp->group.group_key;
+      gr->group.group_size = cp->group.group_size;
+      gr->group.device_type = cp->group.device_type;
+      group_table_[gr->group.group_key].reset(gr);
+      VLOG(2) << "New group_key=" << gr->group.group_key
+              << " group_size=" << gr->group.group_size;
+    } else {
+      gr = it->second.get();
+    }
+  }
+  Status status;
+  {
+    mutex_lock gr_lock(gr->mu);
+    if (!gr->device_set.empty()) {
+      // Check for consistency with existing GroupRec.
+      if (cp->group.device_type != gr->group.device_type) {
+        status = errors::Internal(
+            "Collective Op ", cp->name, " is assigned to device ", device,
+            " with type ", cp->group.device_type.type_string(),
+            " and group_key ", cp->group.group_key, " but that group has type ",
+            gr->group.device_type.type_string());
+      } else if (cp->group.group_size != gr->group.group_size) {
+        status = errors::Internal(
+            "Collective Op ", cp->name, " has group_size ",
+            cp->group.group_size, " and group_key", cp->group.group_key,
+            " but that group has size ", gr->group.group_size);
+      }
+    }
+    if (status.ok()) {
+      // Insert device if not already present.
+      auto it = gr->device_set.find(device);
+      if (it == gr->device_set.end()) {
+        if (gr->device_set.size() == gr->group.group_size) {
+          // The group is already full.
+          status = errors::Internal(
+              "Collective Op ", cp->name, " is assigned to device ", device,
+              " and group_key ", cp->group.group_key,
+              " but that group doesn't contain that device.");
+        } else {
+          // This is a new device that has not yet joined the group.
+          gr->device_set.insert(device);
+          gr->device_list.push_back(device);
+          DeviceNameUtils::ParsedName parsed_device;
+          DeviceNameUtils::ParseFullName(device, &parsed_device);
+          string task_name = strings::StrCat("/job:", parsed_device.job,
+                                             "/replica:", parsed_device.replica,
+                                             "/task:", parsed_device.task);
+          gr->task_set.insert(task_name);
+          gr->task_list.push_back(task_name);
+          gr->group.num_tasks = static_cast<int32>(gr->task_set.size());
+          VLOG(1) << "group_key=" << gr->group.group_key
+                  << " group_size=" << gr->group.group_size
+                  << " dev_set=" << gr->device_set.size();
+        }
+      }
+    }
+
+    if (status.ok()) {
+      // If the group is not yet complete, queue to wait for it.
+      VLOG(2) << "group_size " << gr->group.group_size << " set size "
+              << gr->device_set.size() << " gr " << gr;
+
+      if (gr->device_set.size() < gr->group.group_size) {
+        gr->waiting.push_back(std::bind(done, std::placeholders::_1, gr));
+        return;
+      }
+      CHECK_EQ(gr->device_set.size(), gr->group.group_size);
+      if (!gr->waiting.empty()) {
+        std::swap(to_be_called, gr->waiting);
+      }
+    }
+  }
+  done(status, gr);
+  for (int i = 0; i < to_be_called.size(); ++i) {
+    to_be_called[i](Status::OK());
+  }
+}
+
+namespace {
+
+struct DevRec {
+  string task;
+  string device;
+  int original_rank;
+  int local_rank;
+  int global_rank;
+  const DeviceLocality* locality;
+};
+typedef std::unordered_map<string, DevRec> TaskDeviceMap;
+typedef std::unordered_map<string, TaskDeviceMap> GlobalDeviceMap;
+
+// Create a populated GlobalDeviceMap from CollInstanceParams and localities.
+GlobalDeviceMap BuildDevRecs(const CollInstanceParams& ip,
+                             const std::vector<DeviceLocality>& localities) {
+  GlobalDeviceMap gdm;
+  CHECK_EQ(ip.device_names.size(), ip.task_names.size());
+  CHECK_EQ(ip.device_names.size(), localities.size());
+  for (int i = 0; i < ip.device_names.size(); ++i) {
+    TaskDeviceMap& tdm = gdm[ip.task_names[i]];
+    DevRec* dr = &tdm[ip.device_names[i]];
+    dr->task = ip.task_names[i];
+    dr->device = ip.device_names[i];
+    dr->original_rank = i;
+    dr->local_rank = 0;   // Will be populated later by OrderTaskDeviceMap.
+    dr->global_rank = 0;  // Will be populated later by EstablishGlobalRank.
+    dr->locality = &localities[i];
+  }
+  return gdm;
+}
+
+void OrderTaskDeviceMap(TaskDeviceMap* tdm) {
+  CHECK_GT(tdm->size(), 0);  // Should never be called with 0 devices
+  int least_rank = -1;
+  string next_device;
+  std::set<string> selected;
+  // Starting device is one with the least initial rank.
+  for (const auto& it : *tdm) {
+    if (least_rank < 0 || it.second.original_rank < least_rank) {
+      least_rank = it.second.original_rank;
+      next_device = it.second.device;
+    }
+  }
+  CHECK_GE(least_rank, 0);
+  DeviceNameUtils::ParsedName parsed_name;
+  CHECK(DeviceNameUtils::ParseFullName(next_device, &parsed_name));
+  // NOTE: InterconnectLink has only a device_id, nothing more, so for
+  // the time being if there's more than one device at a task we
+  // assume they're all GPUs.
+
+  int next_rank = 0;
+  while (true) {
+    selected.insert(next_device);
+    DevRec* dr = &(*tdm)[next_device];
+    dr->local_rank = next_rank;
+    ++next_rank;
+    if (selected.size() == tdm->size()) {
+      break;
+    }
+    // For the present time we assume Locality links only cover GPUs.
+    // For multiple CPUs, just take them in order.
+    const InterconnectLink* best_link = nullptr;
+    if (parsed_name.type == "GPU") {
+      for (const InterconnectLink& il : dr->locality->links().link()) {
+        parsed_name.id = il.device_id();
+        string endpoint_device =
+            DeviceNameUtils::ParsedNameToString(parsed_name);
+        if (selected.find(endpoint_device) != selected.end()) {
+          continue;
+        }
+        if (best_link == nullptr || il.strength() > best_link->strength()) {
+          best_link = &il;
+        }
+      }
+    }
+    if (best_link != nullptr) {
+      // Follow the best edge
+      parsed_name.id = best_link->device_id();
+      next_device = DeviceNameUtils::ParsedNameToString(parsed_name);
+    } else {
+      // No good edges, alas. Pick the lowest initial rank among remaining
+      // devices.
+      least_rank = -1;
+      for (const auto& it : *tdm) {
+        if (selected.find(it.second.device) != selected.end()) {
+          continue;
+        }
+        if (least_rank < 0 || it.second.original_rank < least_rank) {
+          least_rank = it.second.original_rank;
+          next_device = it.second.device;
+        }
+      }
+      CHECK_GE(least_rank, 0);
+    }
+  }
+}
+
+// The first time a shared CollectiveParams is established for a
+// shared set of instances we compute a good rank order for all the
+// devices in the group, that is appropriate for a ring algorithm.
+// This order need not be the same across different instance groups
+// sharing the same device group where there is more than one good
+// order.
+GlobalDeviceMap EstablishGlobalRank(
+    CollectiveParams* cp, const std::vector<DeviceLocality>& localities) {
+  VLOG(1) << "EstablishGlobalRank";
+  GlobalDeviceMap gdm = BuildDevRecs(cp->instance, localities);
+  for (auto& iter : gdm) {
+    TaskDeviceMap& tdm = iter.second;
+    OrderTaskDeviceMap(&tdm);
+  }
+  // Connect the global rank order by the order in which tasks first appear.
+  std::set<string> ordered_tasks;
+  int next_rank = 0;
+  for (int i = 0; i < cp->instance.task_names.size(); ++i) {
+    const string& task_name = cp->instance.task_names[i];
+    if (ordered_tasks.find(task_name) != ordered_tasks.end()) {
+      continue;
+    }
+    ordered_tasks.insert(task_name);
+    TaskDeviceMap* tdm = &gdm[task_name];
+    for (auto& it : *tdm) {
+      it.second.global_rank = it.second.local_rank + next_rank;
+    }
+    next_rank += tdm->size();
+  }
+  return gdm;
+}
+
+// Sort cp->instance.device_names lexicographically, but do by first
+// computing a reordering permutation so we can keep cp->instance.task_names
+// in corresponding order.
+void SortDevicesAndTasks(CollectiveParams* cp) {
+  VLOG(1) << "SortDevicesAndTasks " << cp << " instance " << &cp->instance;
+  CHECK(cp);
+  CHECK_EQ(cp->group.group_size, cp->instance.device_names.size());
+  CHECK_EQ(cp->group.group_size, cp->instance.task_names.size());
+  std::vector<int> perm(cp->group.group_size);
+  // TODO(tucker): substitute std::iota when the windows build supports it.
+  // std::iota(perm.begin(), perm.end(), 0);
+  for (int i = 0; i < perm.size(); ++i) {
+    perm[i] = i;
+  }
+  std::sort(perm.begin(), perm.end(), [cp](const int& a, const int& b) {
+    return cp->instance.device_names[a] < cp->instance.device_names[b];
+  });
+  std::vector<string> new_devs;
+  std::vector<string> new_tasks;
+  new_devs.reserve(cp->group.group_size);
+  new_tasks.reserve(cp->group.group_size);
+  for (int pi : perm) {
+    new_devs.push_back(cp->instance.device_names[pi]);
+    new_tasks.push_back(cp->instance.task_names[pi]);
+  }
+  cp->instance.device_names = std::move(new_devs);
+  cp->instance.task_names = std::move(new_tasks);
+  VLOG(1) << "Modified device_names on " << cp;
+}
+
+// Establish the requested number of subdivision permutations based on the
+// ring order implicit in the device order.
+void GenerateSubdivPerms(const string& device, int source_rank,
+                         CollectiveParams* cp) {
+  CHECK_GT(cp->instance.impl_details.subdiv_offsets.size(), 0);
+  cp->instance.impl_details.subdiv_permutations.resize(
+      cp->instance.impl_details.subdiv_offsets.size());
+  // Each subdiv permutation is a ring formed by rotating each
+  // single-task subsequence of devices by an offset.  This makes most
+  // sense when each task has the same number of devices but we can't
+  // depend on that being the case so we'll compute something that
+  // works in any case.
+
+  // Start by counting the devices in each task.
+  // Precondition: device_names must be sorted so that all devices in
+  // the same task are adjacent.
+  VLOG(2) << "Sorted task names: "
+          << str_util::Join(cp->instance.task_names, ", ");
+  std::vector<int> dev_per_task;
+  const string* prior_task_name = &cp->instance.task_names[0];
+  int dev_count = 1;
+  for (int di = 1; di < cp->group.group_size; ++di) {
+    if (cp->instance.task_names[di] != *prior_task_name) {
+      dev_per_task.push_back(dev_count);
+      dev_count = 1;
+      prior_task_name = &cp->instance.task_names[di];
+    } else {
+      ++dev_count;
+    }
+  }
+  dev_per_task.push_back(dev_count);
+  CHECK_EQ(cp->group.num_tasks, dev_per_task.size());
+
+  // Generate a ring permutation for each requested offset.
+  CHECK_GT(cp->instance.impl_details.subdiv_offsets.size(), 0);
+  VLOG(2) << "Setting up perms for cp " << cp << " subdiv_permutations "
+          << &cp->instance.impl_details.subdiv_permutations;
+  cp->instance.impl_details.subdiv_permutations.resize(
+      cp->instance.impl_details.subdiv_offsets.size());
+  cp->subdiv_rank.resize(cp->instance.impl_details.subdiv_offsets.size(), -1);
+  for (int sdi = 0; sdi < cp->instance.impl_details.subdiv_offsets.size();
+       ++sdi) {
+    std::vector<int>& perm = cp->instance.impl_details.subdiv_permutations[sdi];
+    CHECK_EQ(perm.size(), 0);
+    int offset = cp->instance.impl_details.subdiv_offsets[sdi];
+    int prior_dev_count = 0;
+    for (int ti = 0; ti < cp->group.num_tasks; ++ti) {
+      for (int di = 0; di < dev_per_task[ti]; ++di) {
+        int offset_di = (di + offset) % dev_per_task[ti];
+        int permuted_di = prior_dev_count + offset_di;
+        perm.push_back(permuted_di);
+        if (cp->instance.device_names[prior_dev_count + di] == device) {
+          CHECK_EQ(prior_dev_count + di, cp->default_rank);
+          cp->subdiv_rank[sdi] = permuted_di;
+        }
+      }
+      prior_dev_count += dev_per_task[ti];
+    }
+    CHECK_EQ(cp->group.group_size, perm.size());
+  }
+
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    CHECK_GE(source_rank, 0);
+    cp->subdiv_source_rank.resize(
+        cp->instance.impl_details.subdiv_offsets.size(), -1);
+    for (int sdi = 0; sdi < cp->subdiv_source_rank.size(); ++sdi) {
+      for (int j = 0; j < cp->group.group_size; ++j) {
+        if (cp->instance.impl_details.subdiv_permutations[sdi][j] ==
+            source_rank) {
+          cp->subdiv_source_rank[sdi] = j;
+          break;
+        }
+      }
+      CHECK_GE(cp->subdiv_source_rank[sdi], 0);
+    }
+  }
+
+  if (VLOG_IS_ON(1)) {
+    // Log the computed ring order for each subdiv.
+    string buf;
+    for (int sdi = 0;
+         sdi < cp->instance.impl_details.subdiv_permutations.size(); ++sdi) {
+      buf = strings::StrCat("Subdiv ", sdi, " device order:\n");
+      for (int di = 0;
+           di < cp->instance.impl_details.subdiv_permutations[sdi].size();
+           ++di) {
+        int idx = cp->instance.impl_details.subdiv_permutations[sdi][di];
+        strings::StrAppend(&buf, cp->instance.device_names[idx], "\n");
+      }
+      strings::StrAppend(&buf, " subdiv_offsets: ");
+      for (auto o : cp->instance.impl_details.subdiv_offsets)
+        strings::StrAppend(&buf, o, " ");
+      strings::StrAppend(&buf, " SubdivRank: ");
+      for (auto d : cp->subdiv_rank) strings::StrAppend(&buf, d, " ");
+      VLOG(1) << buf;
+    }
+  }
+}
+
+}  // namespace
+
+void CollectiveParamResolverLocal::CompleteTaskIsLocal(const string& task_name,
+                                                       CollectiveParams* cp) {
+  cp->task.is_local.resize(cp->group.group_size, false);
+  for (int i = 0; i < cp->group.group_size; ++i) {
+    cp->task.is_local[i] = (cp->instance.task_names[i] == task_name);
+  }
+}
+
+void CollectiveParamResolverLocal::SetDefaultRank(const string& device,
+                                                  CollectiveParams* cp) {
+  CHECK_EQ(cp->group.group_size, cp->instance.device_names.size()) << cp;
+  for (int i = 0; i < cp->group.group_size; ++i) {
+    if (cp->instance.device_names[i] == device) {
+      cp->default_rank = i;
+      break;
+    }
+  }
+}
+
+Status CollectiveParamResolverLocal::InitInstanceSharedParams(
+    GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir) {
+  VLOG(1) << "InitInstanceSharedParams " << ir;
+  ir->shared.instance = cp->instance;
+  {
+    mutex_lock gl(gr->mu);
+    ir->shared.group = gr->group;
+    ir->shared.instance.device_names.assign(gr->device_list.begin(),
+                                            gr->device_list.end());
+    ir->shared.instance.task_names.assign(gr->task_list.begin(),
+                                          gr->task_list.end());
+    VLOG(2) << "Initialized names for instance: "
+            << ir->shared.instance.ToString();
+  }
+  ir->shared.default_rank = -1;
+
+  // Sort devce_names lexicographcally, keeping task_names in
+  // corresponding order.
+  SortDevicesAndTasks(&ir->shared);
+
+  // Get Locality data for all devices.
+
+  // Set is_local and task_names in *shared prior to invoking
+  // GetDeviceLocalitiesAsync.  In a distributed context this function can be
+  // called by a derived class, some of the devices may be non-local and
+  // GetDeviceLocalitiesAsync will use those fields to launch RPCs.
+  CompleteTaskIsLocal(task_name_, &ir->shared);
+  std::vector<DeviceLocality> localities;
+  Notification note;
+  Status status;
+  dev_resolver_->GetDeviceLocalitiesAsync(ir->shared.instance, &localities,
+                                          [&note, &status](const Status& s) {
+                                            status = s;
+                                            note.Notify();
+                                          });
+  note.WaitForNotification();
+  if (status.ok()) {
+    CompleteDefaultRanking(gr, cp, ir, localities);
+  }
+  return status;
+}
+
+void CollectiveParamResolverLocal::CompleteDefaultRanking(
+    GroupRec* gr, const CollectiveParams* cp, InstanceRec* ir,
+    const std::vector<DeviceLocality>& localities) {
+  // Establish an instance-specific default rank order for devices
+  // based on localities.  This rank order should be a good ring
+  // order, if possible.
+  GlobalDeviceMap gdm = EstablishGlobalRank(&ir->shared, localities);
+  // Reflect the new global ranking on shared
+  size_t num_devices = ir->shared.group.group_size;
+  std::vector<string> new_device_names(num_devices, "");
+  std::vector<string> new_task_names(num_devices, "");
+  for (const auto& git : gdm) {
+    const TaskDeviceMap& tdm = git.second;
+    for (const auto& tit : tdm) {
+      const DevRec& dr = tit.second;
+      new_device_names[dr.global_rank] =
+          ir->shared.instance.device_names[dr.original_rank];
+      new_task_names[dr.global_rank] =
+          ir->shared.instance.task_names[dr.original_rank];
+    }
+  }
+
+  ir->shared.instance.device_names = new_device_names;
+  ir->shared.instance.task_names = new_task_names;
+  if (VLOG_IS_ON(2)) {
+    string buf;
+    for (const auto& d : cp->instance.device_names)
+      strings::StrAppend(&buf, "\n", d);
+    VLOG(2) << "Optimized device order for " << ir->shared.name << ": " << buf;
+  }
+}
+
+void CollectiveParamResolverLocal::CallbackWithStatus(
+    const InstanceRecCallback& done, InstanceRec* irec) {
+  Status s;
+  {
+    mutex_lock l(irec->out_mu);
+    s = irec->status;
+  }
+  done(s, irec);
+}
+
+void CollectiveParamResolverLocal::FindInstanceRec(
+    GroupRec* gr, CollectiveParams* cp, const InstanceRecCallback& done) {
+  InstanceRec* irec = nullptr;
+  bool exit_outside_locks = false;
+  {
+    mutex_lock l(instance_mu_);
+    auto it = instance_table_.find(cp->instance.instance_key);
+    if (it != instance_table_.end()) {
+      irec = it->second.get();
+      {
+        mutex_lock l(irec->in_mu);
+        if (irec->is_init) {
+          exit_outside_locks = true;
+        } else {
+          irec->init_waiters.push_back([this, gr, cp, done](InstanceRec* irec) {
+            CallbackWithStatus(done, irec);
+          });
+          return;
+        }
+      }
+    } else {
+      // Create new InstanceRec.
+      irec = new InstanceRec;
+      instance_table_[cp->instance.instance_key].reset(irec);
+    }
+  }
+  if (exit_outside_locks) {
+    CallbackWithStatus(done, irec);
+    return;
+  }
+  // Initialize the new InstanceRec while holding out_mu.
+  {
+    mutex_lock il(irec->out_mu);
+    irec->known.resize(cp->group.group_size, false);
+    irec->status = InitInstanceSharedParams(gr, cp, irec);
+  }
+  // Prepare to invoke any waiters that accumlated during initialization.
+  std::vector<IRConsumer> init_waiters;
+  {
+    mutex_lock tl(instance_mu_);
+    {
+      mutex_lock l(irec->in_mu);
+      irec->is_init = true;
+      if (!irec->init_waiters.empty()) {
+        std::swap(init_waiters, irec->init_waiters);
+      }
+    }
+  }
+  CallbackWithStatus(done, irec);
+  for (auto& f : init_waiters) {
+    f(irec);
+  }
+}
+
+void CollectiveParamResolverLocal::CompleteParamsAsync(
+    const string& device, CollectiveParams* cp, CancellationManager* cancel_mgr,
+    const StatusCallback& done) {
+  VLOG(1) << "CompleteParams " << device << " for " << cp << ": "
+          << cp->ToString();
+  CompleteGroupLocal(
+      device, cp, [this, device, cp, done](const Status& s, GroupRec* gr) {
+        if (s.ok()) {
+          CompleteInstanceLocal(device, gr, cp, cp->is_source, done);
+        } else {
+          done(s);
+        }
+      });
+}
+
+void CollectiveParamResolverLocal::CompleteInstanceAsync(
+    const CompleteInstanceRequest* request, CompleteInstanceResponse* response,
+    CancellationManager* cancel_mgr, const StatusCallback& done) {
+  done(
+      errors::Internal("CompleteInstance is not implemented by "
+                       "CollectiveParamResolverLocal which is "
+                       "intended only for non-distributed deployment."));
+}
+
+void CollectiveParamResolverLocal::CompleteInstanceLocal(
+    const string& device, GroupRec* gr, CollectiveParams* cp, bool is_source,
+    const StatusCallback& done) {
+  VLOG(1) << "CompleteInstanceLocal " << device
+          << " instance_key: " << cp->instance.instance_key << " gr " << gr;
+
+  // Populate the group portion of *cp from *gr.  Most of it should already
+  // match.
+  DCHECK_EQ(cp->group.group_key, gr->group.group_key);
+  DCHECK_EQ(cp->group.group_size, gr->group.group_size);
+  DCHECK_EQ(cp->group.device_type, gr->group.device_type);
+  cp->group = gr->group;
+
+  // Get the shared InstanceRec for this instance.
+  FindInstanceRec(gr, cp,
+                  [this, device, gr, cp, is_source, done](const Status& s,
+                                                          InstanceRec* ir) {
+                    if (s.ok()) {
+                      CompleteInstanceFromInitializedIRec(device, gr, cp, ir,
+                                                          is_source, done);
+                    } else {
+                      done(s);
+                    }
+                  });
+}
+
+void CollectiveParamResolverLocal::CompleteInstanceFromInitializedIRec(
+    const string& device, GroupRec* gr, CollectiveParams* cp, InstanceRec* ir,
+    bool is_source, const StatusCallback& done) {
+  // Populate the fields common across instance.
+  {
+    mutex_lock l(ir->out_mu);
+    // custom operator= does a deep copy.
+    cp->instance = ir->shared.instance;
+  }
+  // Populate the fields common across task, also default_rank.
+  SetDefaultRank(device, cp);
+  CompleteTaskIsLocal(task_name_, cp);
+  // If broadcast, may need to wait for source discovery.
+  if (cp->instance.type == BROADCAST_COLLECTIVE) {
+    CompleteInstanceSource(ir, cp, is_source,
+                           [this, ir, device, cp, done](InstanceRec* irec) {
+                             CHECK_EQ(ir, irec);
+                             Status s;
+                             int source_rank;
+                             {
+                               mutex_lock l(irec->out_mu);
+                               s = irec->status;
+                               source_rank = ir->source_rank;
+                             }
+                             if (s.ok()) {
+                               GenerateSubdivPerms(device, source_rank, cp);
+                             }
+                             done(s);
+                           });
+    return;
+  } else {
+    GenerateSubdivPerms(device, 0, cp);
+  }
+  done(Status::OK());
+}
+
+void CollectiveParamResolverLocal::CompleteInstanceSource(InstanceRec* ir,
+                                                          CollectiveParams* cp,
+                                                          bool is_source,
+                                                          const IRConsumer& f) {
+  std::vector<IRConsumer> ready_waiters;
+  {
+    mutex_lock l(ir->out_mu);
+    CHECK_EQ(cp->group.group_size, ir->known.size());
+    CHECK_GE(cp->default_rank, 0);
+    if (!ir->known[cp->default_rank]) {
+      ir->known[cp->default_rank] = true;
+      ++ir->known_count;
+      if (is_source) {
+        if (ir->source_rank >= 0) {
+          ir->status = errors::Internal("Instance ", cp->instance.instance_key,
+                                        " already has source ", ir->source_rank,
+                                        ", recevied second claim from ",
+                                        cp->default_rank);
+        } else {
+          ir->source_rank = cp->default_rank;
+        }
+      }
+    }
+    if (ir->known_count < ir->shared.group.group_size) {
+      ir->known_waiters.push_back(f);
+      return;
+    }
+    CHECK_EQ(ir->known_count, ir->shared.group.group_size);
+    CHECK_GE(ir->source_rank, 0);
+    if (!ir->known_waiters.empty()) {
+      ready_waiters = std::move(ir->known_waiters);
+    }
+  }
+  f(ir);
+  for (auto& f : ready_waiters) {
+    f(ir);
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local.h b/tensorflow/core/common_runtime/collective_param_resolver_local.h
new file mode 100644
index 0000000000..ff3415b0a9
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local.h
@@ -0,0 +1,209 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
+#define TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
+
+#include <string>
+
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/lib/gtl/flatmap.h"
+
+namespace tensorflow {
+class CompleteGroupRequest;
+class CompleteGroupResponse;
+class CompleteInstanceRequest;
+class CompleteInstanceResponse;
+class DeviceMgr;
+
+// Implements ParamResolverInterface for a single-task context.
+// It also implements the functionality necessary to serve as the
+// group leader for param resolution in a multi-task context.
+class CollectiveParamResolverLocal : public ParamResolverInterface {
+ public:
+  CollectiveParamResolverLocal(const DeviceMgr* dev_mgr,
+                               DeviceResolverInterface* dev_resolver,
+                               const string& task_name);
+
+  ~CollectiveParamResolverLocal() override {}
+
+  void CompleteParamsAsync(const string& device, CollectiveParams* cp,
+                           CancellationManager* cancel_mgr,
+                           const StatusCallback& done) override;
+
+  void CompleteGroupAsync(const CompleteGroupRequest* request,
+                          CompleteGroupResponse* response,
+                          CancellationManager* cancel_mgr,
+                          const StatusCallback& done) override;
+
+  void CompleteInstanceAsync(const CompleteInstanceRequest* request,
+                             CompleteInstanceResponse* response,
+                             CancellationManager* cancel_mgr,
+                             const StatusCallback& done) override;
+
+ protected:
+  // Used to complete/verify CollGroup.
+  struct GroupRec {
+    CollGroupParams group;
+    mutex mu;
+    Status status GUARDED_BY(mu);
+    std::set<string> device_set GUARDED_BY(mu);
+    std::vector<string> device_list GUARDED_BY(mu);
+    std::set<string> task_set GUARDED_BY(mu);
+    std::vector<string> task_list GUARDED_BY(mu);
+    std::vector<StatusCallback> waiting GUARDED_BY(mu);
+  };
+
+  // Finds the GroupRec that corresponds to cp->group_key.
+  // Also populates cp->group from that group_rec.
+  // Will wait until GroupRec is fully populated or an error arises before
+  // calling done.  Callback GroupRec* arg is only valid if status is ok.
+  // Ownership of GroupRec stays with this object and does not pass to the
+  // callback.
+  typedef std::function<void(const Status& s, GroupRec* gr)> GroupRecCallback;
+  void CompleteGroupLocal(const string& device, CollectiveParams* cp,
+                          const GroupRecCallback& done)
+      LOCKS_EXCLUDED(group_mu_);
+
+  // Used to complete/verify CollInstance.
+  struct InstanceRec;
+  typedef std::function<void(InstanceRec*)> IRConsumer;
+  struct InstanceRec {
+    // This structure has two mutexes so that a possibly long
+    // initialization can be done without holding the instance_mu_
+    // table lock the whole time (which can cause an excessive number
+    // of threads to block on it), and because the compiler may not
+    // permit mutex locks to be taken in more than one order.
+    //
+    // out_mu guards access to most of the fields.
+    // in_mu guards access to a queue of comsumer callbacks wanting to
+    // read the fields guarded by out_mu.
+    //
+    // The in_mu should be locked only while holding instance_mu_; the
+    // out_mu should be locked only while not holding
+    // instance_mu_.
+    //
+    // When is_init is false (the initial value) any potential user
+    // other than the creator should queue a callback on init_waiters.
+    // As soon as the shared member of this structure is fully
+    // initialized is_init will be set true and those callbacks will
+    // be invoked.
+    //
+    // Once inserted in the table this structure will never be replaced
+    // so users can capture the pointer while holding instance_mu_,
+    // drop that lock, then take a lock on out_mu before
+    // reading/modifying its values.
+    mutex in_mu;
+    bool is_init GUARDED_BY(in_mu);
+    std::vector<IRConsumer> init_waiters GUARDED_BY(in_mu);
+
+    // Values to be shared by all instances, constant after initialization.
+    mutex out_mu;
+    CollectiveParams shared GUARDED_BY(out_mu);
+    // If an error occurs during initialization this structure stays in
+    // the table with a non-OK status.  Purging the table and restarting
+    // needs to be done at a higher level.
+    Status status GUARDED_BY(out_mu);
+
+    // These fields are used to count the instances that have called
+    // in and become known while resolving broadcast source identity.
+    int source_rank GUARDED_BY(out_mu);
+    int known_count GUARDED_BY(out_mu);
+    std::vector<bool> known GUARDED_BY(out_mu);
+    std::vector<IRConsumer> known_waiters GUARDED_BY(out_mu);
+
+    InstanceRec() : is_init(false), source_rank(-1), known_count(0) {}
+  };
+
+  // Find the InstanceRec with the same instance_key as cp.  If it doesn't
+  // already exist, create and initialize from gr and cp.
+  //
+  // Precondition: *gr must be a complete GroupRec, i.e. the value set
+  // by CompleteGroupLocal. *cp must be populated with all the fields
+  // required by InitInstanceSharedParams.  Ownership of InstanceRec stays
+  // with this object and does not pass to the callback.
+  typedef std::function<void(const Status& s, InstanceRec* ir)>
+      InstanceRecCallback;
+  void FindInstanceRec(GroupRec* gr, CollectiveParams* cp,
+                       const InstanceRecCallback& done)
+      LOCKS_EXCLUDED(instance_mu_, gr->mu, group_mu_);
+
+  // Populate *ir with device membership from gr, then initialize to be specific
+  // to cp->instance_key, i.e. order the devices and tasks.
+  //
+  // Preconditions:
+  //  cp is populated with all DeviceLocalities
+  Status InitInstanceSharedParams(GroupRec* gr, const CollectiveParams* cp,
+                                  InstanceRec* ir)
+      EXCLUSIVE_LOCKS_REQUIRED(ir->out_mu) LOCKS_EXCLUDED(gr->mu);
+
+  // Establishes the final order of ir->shared.instance.device_names and
+  // ir->shared.instance.task_names by considering localities of all devices.
+  void CompleteDefaultRanking(GroupRec* gr, const CollectiveParams* cp,
+                              InstanceRec* ir,
+                              const std::vector<DeviceLocality>& localities)
+      EXCLUSIVE_LOCKS_REQUIRED(ir->out_mu);
+
+  // Finish populating *cp.
+  // Precondition: *gr has been fully populated by CompleteGroupLocal.
+  void CompleteInstanceLocal(const string& device, GroupRec* gr,
+                             CollectiveParams* cp, bool is_source,
+                             const StatusCallback& done)
+      LOCKS_EXCLUDED(instance_mu_, gr->mu, group_mu_);
+
+  // Finish populating *cp from fully initialized *ir.
+  // Precondition: *gr and *ir are fully populated.
+  void CompleteInstanceFromInitializedIRec(const string& device, GroupRec* gr,
+                                           CollectiveParams* cp,
+                                           InstanceRec* ir, bool is_source,
+                                           const StatusCallback& done)
+      LOCKS_EXCLUDED(ir->out_mu);
+
+  // Complete source data for a broadcast instance.
+  // Precondition: *cp has complete group data and default_rank.
+  void CompleteInstanceSource(InstanceRec* ir, CollectiveParams* cp,
+                              bool is_source, const IRConsumer& f)
+      LOCKS_EXCLUDED(ir->out_mu);
+
+  // If cp.device_names contains only devices local to this process
+  // populates *localities, else returns an error.
+  Status GetLocalDeviceLocalities(const CollectiveParams& cp,
+                                  std::vector<DeviceLocality>* localities);
+
+  // Sets CollTaskParams.is_local and CollectiveParams.default_rank.
+  // Precondition: cp->device_names is fully populated and in final order.
+  void CompleteTaskIsLocal(const string& task_name, CollectiveParams* cp);
+
+  // Sets cp->instance_default_rank according to location of device in
+  // current ordering of cp->instance.device_names.
+  void SetDefaultRank(const string& device, CollectiveParams* cp);
+
+  // Helper to grab status under lock, invoke callback out of lock.
+  void CallbackWithStatus(const InstanceRecCallback& done, InstanceRec* irec)
+      LOCKS_EXCLUDED(irec->out_mu);
+
+  const DeviceMgr* dev_mgr_;
+  DeviceResolverInterface* dev_resolver_;
+  string task_name_;
+  mutex group_mu_;
+  gtl::FlatMap<int32, std::unique_ptr<GroupRec>> group_table_
+      GUARDED_BY(group_mu_);
+  mutex instance_mu_;
+  gtl::FlatMap<int32, std::unique_ptr<InstanceRec>> instance_table_
+      GUARDED_BY(instance_mu_);
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_PARAM_RESOLVER_LOCAL_H_
diff --git a/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
new file mode 100644
index 0000000000..4e3c7125f2
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_param_resolver_local_test.cc
@@ -0,0 +1,151 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_executor_mgr.h"
+
+#include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace {
+
+#define NUM_DEVS 3
+
+class CollectiveParamResolverLocalTest : public ::testing::Test {
+ protected:
+  CollectiveParamResolverLocalTest() {
+    ConfigProto cp;
+    SessionOptions options;
+    string task_name = "/job:localhost/replica:0/task:0";
+    auto* device_count = options.config.mutable_device_count();
+    device_count->insert({"CPU", NUM_DEVS});
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
+    drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
+    prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
+                                                task_name));
+  }
+
+  std::vector<Device*> devices_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<DeviceResolverLocal> drl_;
+  std::unique_ptr<CollectiveParamResolverLocal> prl_;
+};
+
+TEST_F(CollectiveParamResolverLocalTest, CompleteParamsReduction1Task) {
+  CollectiveParams cps[NUM_DEVS];
+  Status statuses[NUM_DEVS];
+  Notification note[NUM_DEVS];
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    CollectiveParams* cp = &cps[i];
+    cp->group.group_key = 1;
+    cp->group.group_size = 3;
+    cp->group.device_type = DeviceType("CPU");
+    cp->group.num_tasks = 1;
+    cp->instance.instance_key = 7;
+    cp->instance.type = REDUCTION_COLLECTIVE;
+    cp->instance.data_type = DataType(DT_FLOAT);
+    cp->instance.shape = TensorShape({5});
+    cp->instance.device_names.push_back(
+        strings::StrCat("/job:localhost/replica:0/task:0/device:CPU:", i));
+    cp->instance.impl_details.subdiv_offsets.push_back(0);
+    cp->is_source = false;
+    Env::Default()->SchedClosure([this, i, cp, &note, &statuses]() {
+      prl_->CompleteParamsAsync(cp->instance.device_names[0], cp,
+                                nullptr /*CancellationManager*/,
+                                [this, &statuses, &note, i](const Status& s) {
+                                  statuses[i] = s;
+                                  note[i].Notify();
+                                });
+    });
+  }
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    note[i].WaitForNotification();
+  }
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    TF_ASSERT_OK(statuses[i]);
+    ASSERT_EQ(cps[i].instance.device_names.size(), 3);
+    for (int j = 0; j < NUM_DEVS; ++j) {
+      EXPECT_EQ(
+          strings::StrCat("/job:localhost/replica:0/task:0/device:CPU:", j),
+          cps[i].instance.device_names[j]);
+      EXPECT_TRUE(cps[i].task.is_local[j]);
+    }
+    EXPECT_EQ(cps[i].subdiv_rank[0], i);
+    EXPECT_EQ(cps[i].subdiv_source_rank.size(), 0);
+    EXPECT_FALSE(cps[i].is_source);
+    EXPECT_EQ(cps[i].default_rank, i);
+  }
+}
+
+TEST_F(CollectiveParamResolverLocalTest, CompleteParamsBroadcast1Task) {
+  CollectiveParams cps[NUM_DEVS];
+  Status statuses[NUM_DEVS];
+  Notification note[NUM_DEVS];
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    CollectiveParams* cp = &cps[i];
+    cp->group.group_key = 1;
+    cp->group.group_size = 3;
+    cp->group.device_type = DeviceType("CPU");
+    cp->group.num_tasks = 1;
+    cp->instance.instance_key = 3;
+    cp->instance.type = BROADCAST_COLLECTIVE;
+    cp->instance.data_type = DataType(DT_FLOAT);
+    cp->instance.shape = TensorShape({5});
+    cp->instance.device_names.push_back(
+        strings::StrCat("/job:localhost/replica:0/task:0/device:CPU:", i));
+    cp->instance.impl_details.subdiv_offsets.push_back(0);
+    cp->is_source = (i == 1);
+    Env::Default()->SchedClosure([this, i, cp, &note, &statuses]() {
+      prl_->CompleteParamsAsync(cp->instance.device_names[0], cp,
+                                nullptr /*CancellationManager*/,
+                                [this, &statuses, &note, i](const Status& s) {
+                                  statuses[i] = s;
+                                  note[i].Notify();
+                                });
+    });
+  }
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    note[i].WaitForNotification();
+  }
+  for (int i = 0; i < NUM_DEVS; ++i) {
+    TF_ASSERT_OK(statuses[i]);
+    ASSERT_EQ(cps[i].instance.device_names.size(), 3);
+    for (int j = 0; j < NUM_DEVS; ++j) {
+      EXPECT_EQ(
+          strings::StrCat("/job:localhost/replica:0/task:0/device:CPU:", j),
+          cps[i].instance.device_names[j]);
+      EXPECT_TRUE(cps[i].task.is_local[j]);
+    }
+    ASSERT_GT(cps[i].subdiv_rank.size(), 0);
+    EXPECT_EQ(cps[i].subdiv_rank[0], i);
+    ASSERT_GT(cps[i].subdiv_source_rank.size(), 0);
+    EXPECT_EQ(cps[i].subdiv_source_rank[0], 1);
+    EXPECT_EQ(cps[i].is_source, (i == 1));
+    EXPECT_EQ(cps[i].default_rank, i);
+  }
+}
+
+// TEST_F(CollectiveParamResolverLocalTest,
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_rma_local.cc b/tensorflow/core/common_runtime/collective_rma_local.cc
new file mode 100644
index 0000000000..ad9b32ce35
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_rma_local.cc
@@ -0,0 +1,108 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_rma_local.h"
+
+#include "tensorflow/core/common_runtime/copy_tensor.h"
+#include "tensorflow/core/common_runtime/dma_helper.h"
+
+namespace tensorflow {
+
+void CollectiveRemoteAccessLocal::StartAbort(const Status& s) {
+  buf_rendezvous_.StartAbort(s);
+}
+
+void CollectiveRemoteAccessLocal::RecvFromPeer(
+    const string& peer_device, const string& peer_task, bool peer_is_local,
+    const string& key, Device* to_device, DeviceContext* to_device_ctx,
+    const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
+    const DeviceLocality& client_locality, const StatusCallback& done) {
+  VLOG(1) << "RecvFromPeer " << this << " from " << peer_device << " key "
+          << key;
+  if (!peer_is_local) {
+    done(
+        errors::Internal("CollectiveRemoteAccessLocal::RecvFromPeer "
+                         "called with peer_is_local=false"));
+    return;
+  }
+  buf_rendezvous_.ConsumeBuf(
+      key, [this, to_tensor, to_device_ctx, to_device, to_alloc_attr, done](
+               const Status& s, BufRendezvous::Hook* hook) {
+        if (!s.ok()) {
+          done(s);
+          delete hook;
+        } else {
+          int64 recv_bytes = to_tensor->TotalBytes();
+          CHECK_EQ(recv_bytes, hook->prod_value->TotalBytes());
+          MemCpyAsync(hook->prod_ctx,    // src DeviceContext
+                      to_device_ctx,     // dst DeviceContext
+                      hook->prod_dev,    // src Device
+                      to_device,         // dst Device
+                      hook->prod_attr,   // src AllocatorAttributes
+                      to_alloc_attr,     // dst AllocatorAttributes
+                      hook->prod_value,  // src Tensor*
+                      to_tensor,         // dst Tensor*
+                      [hook, done](const Status& s) {
+                        done(s);
+                        hook->prod_cb(s);
+                        delete hook;
+                      });
+        }
+      });
+}
+
+void CollectiveRemoteAccessLocal::PostToPeer(
+    const string& peer_device, const string& peer_task, const string& key,
+    Device* from_device, DeviceContext* from_device_ctx,
+    const AllocatorAttributes& from_alloc_attr, const Tensor* from_tensor,
+    const DeviceLocality& client_locality, const StatusCallback& done) {
+  VLOG(1) << "PostToPeer " << this << " key " << key
+          << " step_id_=" << step_id_;
+  buf_rendezvous_.ProvideBuf(key, from_device, from_device_ctx, from_tensor,
+                             from_alloc_attr, done);
+}
+
+/*static*/
+void CollectiveRemoteAccessLocal::MemCpyAsync(
+    DeviceContext* src_dev_ctx, DeviceContext* dst_dev_ctx, Device* src_dev,
+    Device* dst_dev, const AllocatorAttributes& src_attr,
+    const AllocatorAttributes& dst_attr, const Tensor* src, Tensor* dst,
+    const StatusCallback& done) {
+  // We want a real copy to happen, i.e. the bytes inside of src should be
+  // transferred to the buffer backing dst.  If src and dst are on different
+  // devices then CopyTensor::ViaDMA will do just that.  But if they're both
+  // the same CPU, then it will actually just reset dst to point to src.
+  // Since this routine is used for copying between devices and within a
+  // device, we need to detect and bypass the wrong-semantics case.
+  const DeviceType src_device_type(
+      src_attr.on_host() ? DEVICE_CPU : src_dev->attributes().device_type());
+  const DeviceType dst_device_type(
+      dst_attr.on_host() ? DEVICE_CPU : dst_dev->attributes().device_type());
+  const bool non_cpu_src = src_device_type != DeviceType(DEVICE_CPU);
+  const bool non_cpu_dst = dst_device_type != DeviceType(DEVICE_CPU);
+  if (non_cpu_src) CHECK(src_dev_ctx);
+  if (non_cpu_dst) CHECK(dst_dev_ctx);
+  if (non_cpu_src || non_cpu_dst) {
+    CopyTensor::ViaDMA("",  // edge name (non-existent)
+                       src_dev_ctx, dst_dev_ctx, src_dev, dst_dev, src_attr,
+                       dst_attr, src, dst, done);
+  } else {
+    int64 bytes = src->TotalBytes();
+    DCHECK_EQ(dst->TotalBytes(), bytes);
+    memcpy(DMAHelper::base(dst), DMAHelper::base(src), bytes);
+    done(Status::OK());
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/collective_rma_local.h b/tensorflow/core/common_runtime/collective_rma_local.h
new file mode 100644
index 0000000000..d25dd5f04a
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_rma_local.h
@@ -0,0 +1,88 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_ACCESS_H_
+#define TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_ACCESS_H_
+#include "tensorflow/core/common_runtime/buf_rendezvous.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/rendezvous.h"
+
+namespace tensorflow {
+
+// Basic implementation of PerStepCollectiveRemoteAccess.
+class CollectiveRemoteAccessLocal : public PerStepCollectiveRemoteAccess {
+ public:
+  CollectiveRemoteAccessLocal(const DeviceMgr* dev_mgr,
+                              DeviceResolverInterface* dev_resolver,
+                              int64 step_id)
+      : dev_mgr_(dev_mgr),
+        dev_resolver_(dev_resolver),
+        buf_rendezvous_(step_id),
+        step_id_(step_id) {}
+
+  virtual ~CollectiveRemoteAccessLocal() {}
+
+  void StartAbort(const Status& s);
+
+  void RecvFromPeer(const string& peer_device, const string& peer_task,
+                    bool peer_is_local, const string& key, Device* to_device,
+                    DeviceContext* to_device_ctx,
+                    const AllocatorAttributes& to_alloc_attr, Tensor* to_tensor,
+                    const DeviceLocality& client_locality,
+                    const StatusCallback& done) override;
+
+  void PostToPeer(const string& peer_device, const string& peer_task,
+                  const string& key, Device* from_device,
+                  DeviceContext* from_device_ctx,
+                  const AllocatorAttributes& from_alloc_attr,
+                  const Tensor* from_tensor,
+                  const DeviceLocality& client_locality,
+                  const StatusCallback& done) override;
+
+  void GetDeviceLocalitiesAsync(const CollInstanceParams& ci_params,
+                                std::vector<DeviceLocality>* localities,
+                                const StatusCallback& done) override {
+    dev_resolver_->GetDeviceLocalitiesAsync(ci_params, localities, done);
+  }
+
+  void GetLocalityAsync(const string& device, const string& task,
+                        DeviceLocality* locality,
+                        const StatusCallback& done) override {
+    dev_resolver_->GetLocalityAsync(device, task, locality, done);
+  }
+
+  void ClearTask(const string& task) override {
+    dev_resolver_->ClearTask(task);
+  }
+
+  // Copy utility that always copies bytes from src to dst even if
+  // they are on the same device, unlike CopyTensor::ViaDMA which will
+  // just change the dst buffer pointer in that case.
+  static void MemCpyAsync(DeviceContext* src_dev_ctx,
+                          DeviceContext* dst_dev_ctx, Device* src_dev,
+                          Device* dst_dev, const AllocatorAttributes& src_attr,
+                          const AllocatorAttributes& dst_attr,
+                          const Tensor* src, Tensor* dst,
+                          const StatusCallback& done);
+
+ protected:
+  const DeviceMgr* dev_mgr_;               // not owned
+  DeviceResolverInterface* dev_resolver_;  // not owned
+  BufRendezvous buf_rendezvous_;
+  int64 step_id_;
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_COMMON_RUNTIME_COLLECTIVE_RMA_LOCAL_ACCESS_H_
diff --git a/tensorflow/core/common_runtime/collective_rma_local_test.cc b/tensorflow/core/common_runtime/collective_rma_local_test.cc
new file mode 100644
index 0000000000..dcd4272d96
--- /dev/null
+++ b/tensorflow/core/common_runtime/collective_rma_local_test.cc
@@ -0,0 +1,148 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/collective_rma_local.h"
+
+#include "tensorflow/core/common_runtime/buf_rendezvous.h"
+#include "tensorflow/core/common_runtime/collective_param_resolver_local.h"
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+#include "tensorflow/core/common_runtime/dma_helper.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace {
+
+#define NUM_DEVS 3
+static const int kStepId = 123;
+
+class CollectiveRemoteAccessLocalTest : public ::testing::Test {
+ protected:
+  const string kTaskName = "/job:localhost/replica:0/task:0";
+
+  CollectiveRemoteAccessLocalTest() {
+    ConfigProto cp;
+    SessionOptions options;
+    auto* device_count = options.config.mutable_device_count();
+    device_count->insert({"CPU", NUM_DEVS});
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, kTaskName, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
+    drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
+    prl_.reset(new CollectiveParamResolverLocal(device_mgr_.get(), drl_.get(),
+                                                kTaskName));
+    rma_.reset(new CollectiveRemoteAccessLocal(device_mgr_.get(), drl_.get(),
+                                               kStepId));
+  }
+
+  std::vector<Device*> devices_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<DeviceResolverLocal> drl_;
+  std::unique_ptr<CollectiveParamResolverLocal> prl_;
+  std::unique_ptr<CollectiveRemoteAccessLocal> rma_;
+};
+
+TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU0) {
+  Device* cpu0 = nullptr;
+  AllocatorAttributes attr;
+  DeviceLocality dev_locality;
+  TF_ASSERT_OK(device_mgr_->LookupDevice(kTaskName + "/device:CPU:0", &cpu0));
+  Tensor sink_tensor(DT_FLOAT, TensorShape({8}));
+  Notification recv_note;
+  Status recv_status;
+  rma_->RecvFromPeer(kTaskName + "/device:CPU:0", kTaskName, true /*is_local*/,
+                     "key_0", cpu0 /*to_device*/, nullptr /*to_device_ctx*/,
+                     attr /*to_alloc_attr*/, &sink_tensor, dev_locality,
+                     [this, &recv_note, &recv_status](const Status& s) {
+                       recv_status = s;
+                       recv_note.Notify();
+                     });
+  Tensor source_tensor(DT_FLOAT, TensorShape({8}));
+  for (int i = 0; i < 8; ++i) {
+    source_tensor.flat<float>()(i) = i / 2;
+  }
+  // Tensors have distinct storage.
+  EXPECT_NE(DMAHelper::base(&source_tensor), DMAHelper::base(&sink_tensor));
+  Notification send_note;
+  Status send_status;
+  rma_->PostToPeer(kTaskName + "/device:CPU:0", kTaskName, "key_0",
+                   cpu0 /*from_device*/, nullptr /*from_device_ctx*/,
+                   attr /*to_alloc_attr*/, &source_tensor, dev_locality,
+                   [this, &send_note, &send_status](const Status& s) {
+                     send_status = s;
+                     send_note.Notify();
+                   });
+  recv_note.WaitForNotification();
+  send_note.WaitForNotification();
+  TF_EXPECT_OK(recv_status);
+  TF_EXPECT_OK(send_status);
+  // Sink tensor gets the source tensor values.
+  for (int i = 0; i < 8; ++i) {
+    EXPECT_EQ(sink_tensor.flat<float>()(i), i / 2);
+  }
+  // And still has distinct storage.
+  EXPECT_NE(DMAHelper::base(&source_tensor), DMAHelper::base(&sink_tensor));
+}
+
+TEST_F(CollectiveRemoteAccessLocalTest, PostRecvCPU1_2) {
+  Device* cpu2 = nullptr;
+  AllocatorAttributes attr;
+  DeviceLocality dev_locality;
+  TF_ASSERT_OK(device_mgr_->LookupDevice(kTaskName + "/device:CPU:2", &cpu2));
+  Tensor sink_tensor(DT_FLOAT, TensorShape({8}));
+  Notification recv_note;
+  Status recv_status;
+  rma_->RecvFromPeer(kTaskName + "/device:CPU:1", kTaskName, true /*is_local*/,
+                     "key_0", cpu2 /*to_device*/, nullptr /*to_device_ctx*/,
+                     attr /*to_alloc_attr*/, &sink_tensor, dev_locality,
+                     [this, &recv_note, &recv_status](const Status& s) {
+                       recv_status = s;
+                       recv_note.Notify();
+                     });
+  Tensor source_tensor(DT_FLOAT, TensorShape({8}));
+  for (int i = 0; i < 8; ++i) {
+    source_tensor.flat<float>()(i) = i / 2;
+  }
+  // Tensors have distinct storage.
+  EXPECT_NE(DMAHelper::base(&source_tensor), DMAHelper::base(&sink_tensor));
+  Device* cpu1 = nullptr;
+  TF_ASSERT_OK(device_mgr_->LookupDevice(kTaskName + "/device:CPU:1", &cpu1));
+  Notification send_note;
+  Status send_status;
+  rma_->PostToPeer(kTaskName + "/device:CPU:2", kTaskName, "key_0",
+                   cpu1 /*from_device*/, nullptr /*from_device_ctx*/,
+                   attr /*to_alloc_attr*/, &source_tensor, dev_locality,
+                   [this, &send_note, &send_status](const Status& s) {
+                     send_status = s;
+                     send_note.Notify();
+                   });
+  recv_note.WaitForNotification();
+  send_note.WaitForNotification();
+  TF_EXPECT_OK(recv_status);
+  TF_EXPECT_OK(send_status);
+  // Sink tensor gets the source tensor values.
+  for (int i = 0; i < 8; ++i) {
+    EXPECT_EQ(sink_tensor.flat<float>()(i), i / 2);
+  }
+  // And still has distinct storage.
+  EXPECT_NE(DMAHelper::base(&source_tensor), DMAHelper::base(&sink_tensor));
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/device_resolver_local.cc b/tensorflow/core/common_runtime/device_resolver_local.cc
new file mode 100644
index 0000000000..17ef4a2284
--- /dev/null
+++ b/tensorflow/core/common_runtime/device_resolver_local.cc
@@ -0,0 +1,49 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+
+#include "tensorflow/core/common_runtime/device_mgr.h"
+
+namespace tensorflow {
+
+void DeviceResolverLocal::GetDeviceLocalitiesAsync(
+    const CollInstanceParams& ci_params,
+    std::vector<DeviceLocality>* localities, const StatusCallback& done) {
+  localities->clear();
+  for (const string& device_name : ci_params.device_names) {
+    Device* dev;
+    Status s = dev_mgr_->LookupDevice(device_name, &dev);
+    if (!s.ok()) {
+      done(s);
+      return;
+    }
+    localities->push_back(dev->attributes().locality());
+  }
+  done(Status::OK());
+}
+
+void DeviceResolverLocal::GetLocalityAsync(const string& device,
+                                           const string& task,
+                                           DeviceLocality* locality,
+                                           const StatusCallback& done) {
+  Device* dev;
+  Status s = dev_mgr_->LookupDevice(device, &dev);
+  if (s.ok()) {
+    *locality = dev->attributes().locality();
+  }
+  done(s);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/common_runtime/device_resolver_local.h b/tensorflow/core/common_runtime/device_resolver_local.h
new file mode 100644
index 0000000000..098eccdf84
--- /dev/null
+++ b/tensorflow/core/common_runtime/device_resolver_local.h
@@ -0,0 +1,48 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
+#define TENSORFLOW_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
+
+#include <string>
+
+#include "tensorflow/core/framework/collective.h"
+#include "tensorflow/core/framework/device_attributes.pb.h"
+
+namespace tensorflow {
+class DeviceMgr;
+
+// Implements DeviceResolverInterface in a single-task context.
+class DeviceResolverLocal : public DeviceResolverInterface {
+ public:
+  DeviceResolverLocal(const DeviceMgr* dev_mgr) : dev_mgr_(dev_mgr) {}
+
+  virtual ~DeviceResolverLocal() {}
+
+  void GetDeviceLocalitiesAsync(const CollInstanceParams& ci_params,
+                                std::vector<DeviceLocality>* localities,
+                                const StatusCallback& done) override;
+
+  void GetLocalityAsync(const string& device, const string& task,
+                        DeviceLocality* locality,
+                        const StatusCallback& done) override;
+
+  void ClearTask(const string& task) override {}
+
+ protected:
+  const DeviceMgr* dev_mgr_;
+};
+
+}  // namespace tensorflow
+#endif  // TENSORFLOW_COMMON_RUNTIME_DEVICE_RESOLVER_LOCAL_H_
diff --git a/tensorflow/core/common_runtime/device_resolver_local_test.cc b/tensorflow/core/common_runtime/device_resolver_local_test.cc
new file mode 100644
index 0000000000..f5a6471ff7
--- /dev/null
+++ b/tensorflow/core/common_runtime/device_resolver_local_test.cc
@@ -0,0 +1,87 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/device_resolver_local.h"
+
+#include "tensorflow/core/common_runtime/device.h"
+#include "tensorflow/core/common_runtime/device_factory.h"
+#include "tensorflow/core/common_runtime/device_mgr.h"
+#include "tensorflow/core/lib/core/notification.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/public/session_options.h"
+
+namespace tensorflow {
+namespace {
+
+#define NUM_DEVS 3
+
+class DeviceResolverLocalTest : public ::testing::Test {
+ protected:
+  DeviceResolverLocalTest() {
+    ConfigProto cp;
+    SessionOptions options;
+    string task_name = "/job:localhost/replica:0/task:0";
+    auto* device_count = options.config.mutable_device_count();
+    device_count->insert({"CPU", NUM_DEVS});
+    TF_CHECK_OK(DeviceFactory::AddDevices(options, task_name, &devices_));
+    device_mgr_.reset(new DeviceMgr(devices_));
+    drl_.reset(new DeviceResolverLocal(device_mgr_.get()));
+  }
+
+  std::vector<Device*> devices_;
+  std::unique_ptr<DeviceMgr> device_mgr_;
+  std::unique_ptr<DeviceResolverLocal> drl_;
+};
+
+TEST_F(DeviceResolverLocalTest, GetDeviceLocalitiesKnown) {
+  CollectiveParams cp;
+  std::vector<DeviceLocality> localities;
+  cp.instance.device_names.push_back(
+      "/job:localhost/replica:0/task:0/device:CPU:1");
+  cp.instance.device_names.push_back(
+      "/job:localhost/replica:0/task:0/device:CPU:2");
+  Notification note;
+  Status status;
+  drl_->GetDeviceLocalitiesAsync(cp.instance, &localities,
+                                 [this, &note, &status](const Status& s) {
+                                   status = s;
+                                   note.Notify();
+                                 });
+  note.WaitForNotification();
+  TF_EXPECT_OK(status);
+  EXPECT_EQ(2, localities.size());
+}
+
+TEST_F(DeviceResolverLocalTest, GetDeviceLocalitiesUnknown) {
+  CollectiveParams cp;
+  std::vector<DeviceLocality> localities;
+  // In some builds there may be 1 GPU, but there should never be 9.
+  cp.instance.device_names.push_back(
+      "/job:localhost/replica:0/task:0/device:GPU:9");
+  Notification note;
+  Status status;
+  drl_->GetDeviceLocalitiesAsync(cp.instance, &localities,
+                                 [this, &note, &status](const Status& s) {
+                                   status = s;
+                                   note.Notify();
+                                 });
+  note.WaitForNotification();
+  EXPECT_FALSE(status.ok());
+  EXPECT_EQ(0, localities.size());
+}
+
+}  // namespace
+}  // namespace tensorflow
diff --git a/tensorflow/core/framework/collective.cc b/tensorflow/core/framework/collective.cc
new file mode 100644
index 0000000000..a26f2c2f31
--- /dev/null
+++ b/tensorflow/core/framework/collective.cc
@@ -0,0 +1,120 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/framework/collective.h"
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/strcat.h"
+
+namespace tensorflow {
+
+string CollGroupParams::ToString() const {
+  return strings::StrCat("CollGroupParams {group_key=", group_key,
+                         " group_size=", group_size,
+                         " device_type=", device_type.type_string(),
+                         " num_tasks=", num_tasks, "}");
+}
+
+CollInstanceParams& CollInstanceParams::operator=(
+    const CollInstanceParams& other) {
+  if (this != &other) {
+    instance_key = other.instance_key;
+    type = other.type;
+    data_type = other.data_type;
+    shape = other.shape;
+    device_names.clear();
+    device_names.assign(other.device_names.begin(), other.device_names.end());
+    task_names.assign(other.task_names.begin(), other.task_names.end());
+    impl_details.subdiv_offsets.assign(
+        other.impl_details.subdiv_offsets.begin(),
+        other.impl_details.subdiv_offsets.end());
+    impl_details.subdiv_permutations.clear();
+    for (auto p : other.impl_details.subdiv_permutations) {
+      impl_details.subdiv_permutations.push_back(
+          std::vector<int>(p.begin(), p.end()));
+    }
+    impl_details.subdiv_source_rank.assign(
+        other.impl_details.subdiv_source_rank.begin(),
+        other.impl_details.subdiv_source_rank.end());
+  }
+  return *this;
+}
+
+string CollInstanceParams::ToString() const {
+  string v = strings::StrCat("CollInstanceParams { instance_key=", instance_key,
+                             " type=", type, " data_type=", data_type,
+                             " shape=", shape.DebugString(), " devices {");
+  for (const auto& d : device_names) {
+    strings::StrAppend(&v, d, ",");
+  }
+  strings::StrAppend(&v, "} task_names={");
+  for (const auto& n : task_names) {
+    strings::StrAppend(&v, n, ", ");
+  }
+  strings::StrAppend(&v, "}, subdiv_offsets={");
+  for (const auto& d : impl_details.subdiv_offsets) {
+    strings::StrAppend(&v, d, ",");
+  }
+  strings::StrAppend(&v, "}, subdiv_perms={");
+  for (const auto& p : impl_details.subdiv_permutations) {
+    strings::StrAppend(&v, "{");
+    for (const auto& i : p) {
+      strings::StrAppend(&v, i, ",");
+    }
+    strings::StrAppend(&v, "}");  // one subdiv
+  }
+  strings::StrAppend(&v, "}");  // all subdivs
+  return v;
+}
+
+string CollTaskParams::ToString() const {
+  string v = strings::StrCat("CollTaskParams {is_local={");
+  for (const auto& b : is_local) {
+    strings::StrAppend(&v, static_cast<int>(b), ",");
+  }
+  strings::StrAppend(&v, "}}");
+  return v;
+}
+
+string CollectiveParams::ToString() const {
+  string v = strings::StrCat("CollectiveParams ", name, " {", group.ToString());
+  strings::StrAppend(&v, " ", instance.ToString());
+  strings::StrAppend(&v, " ", task.ToString());
+  strings::StrAppend(&v, " default_rank=", default_rank,
+                     " is_source=", is_source, " subdiv_rank={");
+  for (const auto& r : subdiv_rank) {
+    strings::StrAppend(&v, r, ",");
+  }
+  if (!subdiv_source_rank.empty()) {
+    strings::StrAppend(&v, " subdiv_rank={");
+    for (const auto& r : subdiv_source_rank) {
+      strings::StrAppend(&v, r, ",");
+    }
+    strings::StrAppend(&v, "}");
+  }
+  strings::StrAppend(&v, "}}");
+  return v;
+}
+
+/*static*/ OpKernelContext::Params* CollectiveExecutor::CtxParams(
+    OpKernelContext* ctx) {
+  return ctx->params_;
+}
+
+/*static*/
+int64 CollectiveExecutor::kInvalidId = -1;
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/framework/collective.h b/tensorflow/core/framework/collective.h
new file mode 100644
index 0000000000..362d345133
--- /dev/null
+++ b/tensorflow/core/framework/collective.h
@@ -0,0 +1,308 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_FRAMEWORK_COLLECTIVE_EXECUTOR_H_
+#define TENSORFLOW_FRAMEWORK_COLLECTIVE_EXECUTOR_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/framework/device_base.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+class BufRendezvous;
+class CancellationManager;
+class CompleteGroupRequest;
+class CompleteGroupResponse;
+class CompleteInstanceRequest;
+class CompleteInstanceResponse;
+class DeviceLocality;
+class GetStepSequenceRequest;
+class GetStepSequenceResponse;
+class Op;
+class Tensor;
+
+// Types of supported collective operations.
+enum CollectiveType {
+  REDUCTION_COLLECTIVE = 0,
+  BROADCAST_COLLECTIVE,
+  UNDEFINED_COLLECTIVE,
+};
+
+// Data common to all members of a device group.
+// All members share the same device set but its order is
+// particular to an instance so it is stored there.
+struct CollGroupParams {
+  int32 group_key;
+  int32 group_size;
+  DeviceType device_type;
+  int32 num_tasks;  // number of distinct tasks in group
+  string ToString() const;
+  CollGroupParams() : device_type(DEVICE_CPU) {}
+};
+
+// The best implementation of a collective op depends on many factors
+// including the number of devices involved, the topology of
+// interconnects between them and the sizes of inputs.  This structure
+// is used in generating and representing data movement choreography
+// for each specific algorithm, hence it does not have a single, fixed
+// interpretation.  On first execution the runtime will update this
+// structure with decisions that will guide all subsequent executions.
+struct CollImplDetails {
+  std::vector<std::vector<int>> subdiv_permutations;
+  std::vector<int> subdiv_offsets;
+  // broadcast only: rank of source in each subdiv
+  std::vector<int> subdiv_source_rank;
+};
+
+// Data common to all members of a collective instance.
+struct CollInstanceParams {
+  int32 instance_key;  // Identifies all participating graph nodes.
+  CollectiveType type;
+  DataType data_type;
+  TensorShape shape;
+  // Fully qualified name of device for each member, in default rank order.
+  std::vector<string> device_names;
+  // Task name prefix of corresponding device name.
+  std::vector<string> task_names;
+  CollImplDetails impl_details;
+  string ToString() const;
+  CollInstanceParams& operator=(const struct CollInstanceParams& other);
+};
+
+// Data common to all instance members in the same task.
+struct CollTaskParams {
+  // True for devices that are local to the process, i.e. no RPC needed.
+  std::vector<bool> is_local;
+  string ToString() const;
+};
+
+// Unique to a single CollectiveOp node.
+struct CollectiveParams {
+  CollGroupParams group;
+  CollInstanceParams instance;
+  CollTaskParams task;
+
+  string name;       // node name used only for log or error messages
+  int default_rank;  // index of this op within device_names
+  bool is_source;    // broadcast only
+  // Rank of this device in each subdivision permutation.
+  std::vector<int> subdiv_rank;
+  std::vector<int> subdiv_source_rank;
+  const Tensor* in_tensor;             // kernel input
+  Tensor* out_tensor;                  // kernel output
+  std::unique_ptr<OpKernel> merge_op;  // reduction only
+  std::unique_ptr<OpKernel> final_op;  // reduction only
+  OpKernelContext* op_context;
+  string ToString() const;
+};
+
+class CollectiveExecutor;
+
+// Interface that provides resolution of device localities.
+class DeviceResolverInterface {
+ public:
+  virtual ~DeviceResolverInterface() {}
+
+  // Collects DeviceLocality protobufs from all of the devices identified
+  // in 'col_params'.
+  virtual void GetDeviceLocalitiesAsync(const CollInstanceParams& inst_params,
+                                        std::vector<DeviceLocality>* localities,
+                                        const StatusCallback& done) = 0;
+
+  // Populate *locality with the DeviceLocality of the specified
+  // device.
+  virtual void GetLocalityAsync(const string& device, const string& task,
+                                DeviceLocality* locality,
+                                const StatusCallback& done) = 0;
+
+  // Clear the cache of device data belonging
+  // to the specified task.
+  virtual void ClearTask(const string& task) = 0;
+};
+
+// Interface that provides resolution of shared CollectiveParams fields.
+class ParamResolverInterface {
+ public:
+  virtual ~ParamResolverInterface() {}
+
+  // Called by each collective op at first execution in order to fill out
+  // the CollectiveParams structure with data gathered from the full
+  // (maybe distributed) collection of peer nodes.
+  virtual void CompleteParamsAsync(const string& device, CollectiveParams* cp,
+                                   CancellationManager* cancel_mgr,
+                                   const StatusCallback& done) = 0;
+
+  // Used within a distributed implementation to discover/verify
+  // data shared across a device group.
+  virtual void CompleteGroupAsync(const CompleteGroupRequest* request,
+                                  CompleteGroupResponse* response,
+                                  CancellationManager* cancel_mgr,
+                                  const StatusCallback& done) = 0;
+
+  // Used within a distributed implementation to discover/verify data
+  // shared across an instance group.
+  virtual void CompleteInstanceAsync(const CompleteInstanceRequest* request,
+                                     CompleteInstanceResponse* response,
+                                     CancellationManager* cancel_mgr,
+                                     const StatusCallback& done) = 0;
+};
+
+// Graphs which utilize Collective Ops in a common instance must
+// execute with identical step_ids even if they are disjoint graphs
+// run by otherwise independent tasks.  This interface supplies
+// coordinated step_ids to use in such cases.
+class StepSequenceInterface {
+ public:
+  virtual ~StepSequenceInterface() {}
+
+  // Used with a distributed implementation to coordinate step_id
+  // sequences across tasks.
+  virtual void GetStepSequenceAsync(const GetStepSequenceRequest* request,
+                                    GetStepSequenceResponse* response,
+                                    const StatusCallback& done) = 0;
+
+  // Refresh the local per-graph_key step_id sequence from collective
+  // group leader, if applicable.
+  virtual void RefreshStepIdSequenceAsync(int64 graph_key,
+                                          const StatusCallback& done) = 0;
+
+  // Returns the the step_id that should be used for initiating a new execution
+  // on the specified graph. May return the same step_id multiple times if
+  // RetireStepId or RefreshStepIdReservation is not called.
+  virtual int64 NextStepId(int64 graph_key) = 0;
+
+  // Reports that execution of the given step has completed successfully.
+  // Should be called immediately after a step completes with OK status,
+  // prior to calling NextStepId().  If the step fails, don't call.
+  virtual void RetireStepId(int64 graph_key, int64 step_id) = 0;
+};
+
+// Interface that provides access to per-step CollectiveExecutor
+// instances and various distributed resolution capabilities.
+class CollectiveExecutorMgrInterface : public StepSequenceInterface {
+ public:
+  virtual ~CollectiveExecutorMgrInterface() {}
+
+  // Returns the step-specific CollectiveExecutor, creating if one does not
+  // already exist.  The caller assumes ownership of one Ref on the object.
+  virtual CollectiveExecutor* FindOrCreate(int64 step_id) = 0;
+
+  // If there is a CollectiveExecutor for step_id, remove it from the
+  // table.
+  virtual void Cleanup(int64 step_id) = 0;
+
+  virtual ParamResolverInterface* GetParamResolver() const = 0;
+
+  virtual DeviceResolverInterface* GetDeviceResolver() const = 0;
+};
+
+// Interface that a Collective Op implementation uses to exchange data
+// with peers.  Note that data exchange is currently limited to types
+// for which DMAHelper::CanUseDMA() returns true, i.e.  dense numeric
+// types.
+class PeerAccessInterface {
+ public:
+  virtual ~PeerAccessInterface() {}
+
+  virtual void RecvFromPeer(const string& peer_device, const string& peer_task,
+                            bool peer_is_local, const string& key,
+                            Device* to_device, DeviceContext* to_device_ctx,
+                            const AllocatorAttributes& to_alloc_attr,
+                            Tensor* to_tensor,
+                            const DeviceLocality& client_locality,
+                            const StatusCallback& done) = 0;
+
+  virtual void PostToPeer(const string& peer_device, const string& peer_task,
+                          const string& key, Device* from_device,
+                          DeviceContext* from_device_ctx,
+                          const AllocatorAttributes& from_alloc_attr,
+                          const Tensor* from_tensor,
+                          const DeviceLocality& client_locality,
+                          const StatusCallback& done) = 0;
+};
+
+class PerStepCollectiveRemoteAccess;
+
+// A step-specific object that can execute a collective operation completely
+// described by a CollectiveParams object.
+class CollectiveExecutor : public PeerAccessInterface, public core::RefCounted {
+ public:
+  virtual void StartAbort(const Status& s) {}
+
+  virtual void ExecuteAsync(OpKernelContext* ctx,
+                            const CollectiveParams& col_params,
+                            const string& exec_key, StatusCallback done) {
+    done(errors::Internal(
+        "A collective Op has been called in a context in which "
+        "a CollectiveExecutor has not been provided."));
+  }
+
+  virtual void CompleteParamsAsync(const string& device, CollectiveParams* cp,
+                                   CancellationManager* cancel_mgr,
+                                   StatusCallback done) {
+    cem_->GetParamResolver()->CompleteParamsAsync(device, cp, cancel_mgr, done);
+  }
+
+  virtual PerStepCollectiveRemoteAccess* remote_access() { return nullptr; }
+
+  // Used to designate an invalid group or instance key.
+  static int64 kInvalidId;
+
+  // Lexically scoped handle for Ref.
+  class Handle {
+   public:
+    explicit Handle(CollectiveExecutor* ce, bool inherit_ref) : ce_(ce) {
+      if (!inherit_ref) ce->Ref();
+    }
+    ~Handle() { ce_->Unref(); }
+    CollectiveExecutor* get() const { return ce_; }
+
+   private:
+    CollectiveExecutor* ce_;
+  };
+
+ protected:
+  explicit CollectiveExecutor(CollectiveExecutorMgrInterface* cem)
+      : cem_(cem) {}
+
+  // For use only by derived classes
+  static OpKernelContext::Params* CtxParams(OpKernelContext* ctx);
+  CollectiveExecutorMgrInterface* cem_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(CollectiveExecutor);
+};
+
+// Interface of a helper object that provices a CollectiveExecutor with
+// all of the remote access it needs.
+class CollectiveRemoteAccess : public PeerAccessInterface,
+                               public DeviceResolverInterface {
+ public:
+  virtual ~CollectiveRemoteAccess() {}
+};
+
+// A per-step version of CollectiveRemoteAccess that cleans up outstanding
+// communications in case step execution is abandoned.
+class PerStepCollectiveRemoteAccess : public CollectiveRemoteAccess {
+ public:
+  virtual ~PerStepCollectiveRemoteAccess() {}
+  virtual void StartAbort(const Status& s) = 0;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_FRAMEWORK_COLLECTIVE_EXECUTOR_H_
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index 5ccd45efc9..2d97160830 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -1101,6 +1101,7 @@ class OpKernelContext {
   void NotifyUseOfPersistentTensor(const Tensor& tensor);
 
   Status status_;
+  friend class CollectiveExecutor;  // for access to params_
   Params* params_;    // not owned
   mutable mutex mu_;  // mutable so const accessors can acquire the lock
   gtl::InlinedVector<WrappedAllocator, 4> wrapped_allocators_ GUARDED_BY(mu_);
-- 
GitLab


From 4be2f41f30554d71ba48eb03b44d05a424bf41af Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 28 Mar 2018 17:16:10 -0700
Subject: [PATCH 648/960] Missed ScopedUnref in ResourceGather

PiperOrigin-RevId: 190861558
---
 tensorflow/core/kernels/resource_variable_ops.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index e134e476f6..d1675f27dd 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -503,6 +503,7 @@ class ResourceGatherOp : public OpKernel {
   void Compute(OpKernelContext* c) override {
     Var* v = nullptr;
     OP_REQUIRES_OK(c, LookupResource(c, HandleFromInput(c, 0), &v));
+    core::ScopedUnref su(v);
     // NOTE: We hold the lock for the whole gather operation instead
     // of increasing the reference count of v->tensor() to avoid a
     // situation where a write to the same variable will see a
-- 
GitLab


From 74949ee09b0ff48a2ff1ca7a27475ec6c2583d43 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 17:36:30 -0700
Subject: [PATCH 649/960] Further speed up statistical_testing_test by breaking
 up DKWM test.

PiperOrigin-RevId: 190863893
---
 .../kernel_tests/statistical_testing_test.py  | 26 ++++++++++++++++---
 1 file changed, 22 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
index c0e7bdd259..0400c80c29 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/statistical_testing_test.py
@@ -141,16 +141,16 @@ class StatisticalTestingTest(test.TestCase):
 
   def test_dkwm_mean_two_sample_assertion(self):
     rng = np.random.RandomState(seed=0)
-    num_samples = 15000
+    num_samples = 4000
 
-    # 15000 samples is chosen to be enough to find discrepancies of
-    # size 0.1 or more with assurance 1e-6, as confirmed here:
+    # 4000 samples is chosen to be enough to find discrepancies of
+    # size 0.2 or more with assurance 1e-6, as confirmed here:
     with self.test_session() as sess:
       d = st.min_discrepancy_of_true_means_detectable_by_dkwm_two_sample(
           num_samples, 0., 1., num_samples, 0., 1.,
           false_fail_rate=1e-6, false_pass_rate=1e-6)
       d = sess.run(d)
-      self.assertLess(d, 0.1)
+      self.assertLess(d, 0.2)
 
     # Test that the test assertion agrees that the standard
     # uniform distribution has the same mean as itself.
@@ -160,6 +160,15 @@ class StatisticalTestingTest(test.TestCase):
       sess.run(st.assert_true_mean_equal_by_dkwm_two_sample(
           samples1, 0., 1., samples2, 0., 1., false_fail_rate=1e-6))
 
+  def test_dkwm_mean_two_sample_assertion_beta_2_1_false(self):
+    rng = np.random.RandomState(seed=0)
+    num_samples = 4000
+    samples1 = rng.uniform(size=num_samples).astype(np.float32)
+
+    # As established above, 4000 samples is enough to find discrepancies
+    # of size 0.2 or more with assurance 1e-6.
+
+    with self.test_session() as sess:
       # Test that the test assertion confirms that the mean of the
       # standard uniform distribution is different from the mean of beta(2, 1).
       beta_high_samples = rng.beta(2, 1, size=num_samples).astype(np.float32)
@@ -169,6 +178,15 @@ class StatisticalTestingTest(test.TestCase):
             beta_high_samples, 0., 1.,
             false_fail_rate=1e-6))
 
+  def test_dkwm_mean_two_sample_assertion_beta_1_2_false(self):
+    rng = np.random.RandomState(seed=0)
+    num_samples = 4000
+    samples1 = rng.uniform(size=num_samples).astype(np.float32)
+
+    # As established above, 4000 samples is enough to find discrepancies
+    # of size 0.2 or more with assurance 1e-6.
+
+    with self.test_session() as sess:
       # Test that the test assertion confirms that the mean of the
       # standard uniform distribution is different from the mean of beta(1, 2).
       beta_low_samples = rng.beta(1, 2, size=num_samples).astype(np.float32)
-- 
GitLab


From 628552228c76d2ee7f2eef4d56175a89941e3e1d Mon Sep 17 00:00:00 2001
From: Brennan Saeta <saeta@google.com>
Date: Wed, 28 Mar 2018 17:54:01 -0700
Subject: [PATCH 650/960] TPU: Implement 3rd gen input pipeline config.

In this new configuration, we are able to drive a Cloud TPU at full device performance, and achieve over 3k images/sec on ResNet-50. The previous bottleneck was the un-pipeline-able split that occurred after the iterator.get_next() call. This split (when not splitting on the batch-major dimension) caused the training job to be single-threaded-CPU-bottlenecked, resulting in a performance of only ~2650 images/sec on ResNet-50.

This latest input pipeline configuration requires the use of datasets. By requiring datasets, we gain the ability to call get_next() num_replicas times per host, and avoid the expensive split op. (Note: this also opens up potential future avenues for further optimization.) Despite this, we retain a lot of nice usability properties that per_host_v1 (aka input pipeline config v2) gave us.

PiperOrigin-RevId: 190865741
---
 .../contrib/tpu/python/tpu/tpu_config.py      | 32 ++++++--
 .../contrib/tpu/python/tpu/tpu_context.py     | 12 ++-
 .../contrib/tpu/python/tpu/tpu_estimator.py   | 79 +++++++++++++++++--
 3 files changed, 107 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_config.py b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
index 38b5ea2310..cc1a7fd801 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_config.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_config.py
@@ -35,10 +35,16 @@ _TF_CONFIG_ENV = run_config_lib._TF_CONFIG_ENV
 _SERVICE_KEY = run_config_lib._SERVICE_KEY
 _TPU_WORKER_JOB_NAME = 'tpu_worker_job_name'
 _NUM_CORES_PER_HOST = 8
-
 # pylint: enable=protected-access
 
 
+class InputPipelineConfig(object):
+  r"""Please see the definition of these values in TPUConfig."""
+  PER_SHARD_V1 = 1
+  PER_HOST_V1 = 2
+  PER_HOST_V2 = 3
+
+
 # TODO(b/72511246) Provide a simplified api to configure model parallelism.
 class TPUConfig(
     collections.namedtuple('TPUConfig', [
@@ -68,13 +74,16 @@ class TPUConfig(
       partitioned across 4 cores which span two cores in both x and y
       coordinates.  Please refer to @{tf.contrib.tpu.Topology} for the
       geometry of a TPU mesh.
-    per_host_input_for_training: If `True`, `input_fn` is invoked Per-Host
-      rather than Per-Core. With Per-Host input pipeline deployment, `input_fn`
-      is invoked once on each host. With Per-Core input pipeline deployment, it
-      is invoked once for each core. To be precise, with a global batch size
-      `train_batch_size` in `TPUEstimator` constructor, the batch size for each
-      shard is `train_batch_size` // #hosts. With Per-Core input pipeline
-      deployment, the shard batch size is `train_batch_size` // #cores.
+    per_host_input_for_training: If `True`, `PER_HOST_V1`, or `PER_HOST_V2`,
+      `input_fn` is invoked per-host rather than per-core. With per-host input
+      pipeline configuration, `input_fn` is invoked once on each host. With the
+      per-core input pipeline configuration, it is invoked once for each core.
+      With a global batch size `train_batch_size` in `TPUEstimator` constructor,
+      the batch size for each shard is `train_batch_size` // #hosts in the
+      `True` or `PER_HOST_V1` mode. In `PER_HOST_V2` mode, it is
+      `train_batch_size` // #cores. With the per-core input pipeline
+      configuration, the shard batch size is also `train_batch_size` // #cores.
+      Note: per_host_input_for_training==PER_SHARD_V1 only supports mode.TRAIN.
     tpu_job_name: The name of the TPU job. Typically, this name is auto-inferred
       within TPUEstimator, however when using ClusterSpec propagation in more
       esoteric cluster configurations, you may need to specify the job name as a
@@ -117,6 +126,13 @@ class TPUConfig(
         raise ValueError('computation_shape elements can only be 1 or 2; got '
                          'computation_shape={}'.format(computation_shape))
 
+    # per_host_input_for_training may be True, False, or integer in [1..3].
+    # Map legacy values (True, False) to numeric values.
+    if per_host_input_for_training is False:
+      per_host_input_for_training = InputPipelineConfig.PER_SHARD_V1
+    elif per_host_input_for_training is True:
+      per_host_input_for_training = InputPipelineConfig.PER_HOST_V1
+
     # Check initial_infeed_sleep_secs.
     if initial_infeed_sleep_secs:
       util_lib.check_positive_integer(initial_infeed_sleep_secs,
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_context.py b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
index 3bac2db77e..fbc1173e49 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_context.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_context.py
@@ -24,6 +24,7 @@ import copy
 import numpy as np
 
 from tensorflow.contrib.tpu.python.tpu import device_assignment  as tpu_device_assignment
+from tensorflow.contrib.tpu.python.tpu import tpu_config
 from tensorflow.contrib.tpu.python.tpu import tpu_system_metadata as tpu_system_metadata_lib
 from tensorflow.python.estimator import model_fn as model_fn_lib
 from tensorflow.python.platform import tf_logging as logging
@@ -205,7 +206,13 @@ class _TPUContext(object):
     """Return true if input_fn is invoked per-core (other than per-host)."""
     mode = self._assert_mode()
     return (mode == model_fn_lib.ModeKeys.TRAIN and
-            not self._config.tpu_config.per_host_input_for_training)
+            (self._config.tpu_config.per_host_input_for_training is
+             tpu_config.InputPipelineConfig.PER_SHARD_V1))
+
+  def is_input_per_host_with_iterators(self):
+    """Return true if input_fn should be run in the per-host v2 config."""
+    return (self._config.tpu_config.per_host_input_for_training is
+            tpu_config.InputPipelineConfig.PER_HOST_V2)
 
   def is_running_on_cpu(self, is_export_mode=False):
     """Determines whether the input_fn and model_fn should be invoked on CPU.
@@ -271,7 +278,8 @@ class _TPUContext(object):
       return global_batch_size
 
     # On TPU
-    if self.is_input_sharded_per_core():
+    if self.is_input_sharded_per_core() or (
+        self.is_input_per_host_with_iterators()):
       # We prohibit per core input sharding for the model parallelism case,
       # therefore it is safe to use num_cores here.
       return global_batch_size // self.num_cores
diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index 152f8c8c69..fa56708f44 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -740,6 +740,61 @@ def generate_per_host_enqueue_ops_fn_for_host(
   return enqueue_ops_fn, captured_infeed_queue, hooks, is_dataset
 
 
+def generate_per_host_v2_enqueue_ops_fn_for_host(
+    ctx, input_fn, inputs_structure_recorder, device, host_id):
+  """Generates infeed enqueue ops for per-host input_fn on a single host."""
+  del host_id  # unused
+  captured_infeed_queue = _CapturedObject()
+  hooks = []
+
+  with ops.device(device):
+    inputs = _Inputs.from_input_fn(input_fn())
+
+    is_dataset = inputs.is_dataset
+    if not is_dataset:
+      raise TypeError('`input_fn` must return a `Dataset` for the PER_HOST_V2 '
+                      'input pipeline configuration.')
+    if ctx.mode == model_fn_lib.ModeKeys.PREDICT:
+      # TODO(b/XXX): Add predict support for PER_HOST_V2
+      raise TypeError('Most PREDICT not yet supported in PER_HOST_V2 mode.')
+
+    hooks.append(inputs.dataset_initializer_hook())
+
+  def enqueue_ops_fn():
+    """Generates the per_host enqueue ops."""
+    control_deps = []
+    per_host_sharded_inputs = []
+    num_replicas_per_host = ctx.num_of_replicas_per_host
+    with ops.device(device):
+      if not inputs.is_dataset:
+        raise TypeError('`input_fn` must return a `Dataset` for this mode.')
+      for _ in range(num_replicas_per_host):
+        # Use control dependencies to ensure a deterministic ordering.
+        with ops.control_dependencies(control_deps):
+          features, labels = inputs.features_and_labels()  # Calls get_next()
+
+        inputs_structure_recorder.validate_and_record_structure(
+            features, labels)
+        flattened_inputs = (
+            inputs_structure_recorder.flatten_features_and_labels(
+                features, labels))
+
+        control_deps.extend(flattened_inputs)
+        per_host_sharded_inputs.append(flattened_inputs)
+
+    infeed_queue = tpu_feed.InfeedQueue(
+        number_of_tuple_elements=len(per_host_sharded_inputs[0]))
+    captured_infeed_queue.capture(infeed_queue)
+    infeed_queue.set_configuration_from_sharded_input_tensors(
+        per_host_sharded_inputs)
+
+    per_host_enqueue_ops = infeed_queue.generate_enqueue_ops(
+        per_host_sharded_inputs, tpu_ordinal_function=ctx.tpu_ordinal_function)
+    return per_host_enqueue_ops
+
+  return enqueue_ops_fn, captured_infeed_queue, hooks, is_dataset
+
+
 class _InputPipeline(object):
   """`_InputPipeline` handles invoking `input_fn` and piping to infeed queue.
 
@@ -975,10 +1030,17 @@ class _InputPipeline(object):
         host_device = tpu_host_placement_fn(host_id=host_id)
         with ops.device(host_device):
           with ops.name_scope('input_pipeline_task%d' % (host_id)):
-            enqueue_ops_fn, captured_infeed_queue, hooks, is_dataset = (
-                generate_per_host_enqueue_ops_fn_for_host(
-                    self._ctx, self._input_fn, self._inputs_structure_recorder,
-                    self._batch_axis, host_device, host_id))
+            if self._ctx.is_input_per_host_with_iterators():
+              enqueue_ops_fn, captured_infeed_queue, hooks, is_dataset = (
+                  generate_per_host_v2_enqueue_ops_fn_for_host(
+                      self._ctx, self._input_fn,
+                      self._inputs_structure_recorder, host_device, host_id))
+            else:
+              enqueue_ops_fn, captured_infeed_queue, hooks, is_dataset = (
+                  generate_per_host_enqueue_ops_fn_for_host(
+                      self._ctx, self._input_fn,
+                      self._inputs_structure_recorder, self._batch_axis,
+                      host_device, host_id))
             all_hooks.extend(hooks)
 
             # NOTE(xiejw): We dispatch here based on the return type of the
@@ -1724,7 +1786,7 @@ class TPUEstimator(estimator_lib.Estimator):
         labels to match up with the corresponding images. If None is supplied,
         and per_host_input_for_training is True, batches will be sharded based
         on the major dimension. If tpu_config.per_host_input_for_training is
-        False, batch_axis is ignored.
+        False or `PER_HOST_V2`, batch_axis is ignored.
 
     Raises:
       ValueError: `params` has reserved keys already.
@@ -1744,7 +1806,8 @@ class TPUEstimator(estimator_lib.Estimator):
         raise ValueError('`train_batch_size` cannot be `None`')
       util_lib.check_positive_integer(train_batch_size, 'train_batch_size')
 
-      if (not config.tpu_config.per_host_input_for_training and
+      if (config.tpu_config.per_host_input_for_training is
+          tpu_config.InputPipelineConfig.PER_SHARD_V1 and
           config.tpu_config.computation_shape):
         raise ValueError(
             'Model parallelism only supports per host input for training. '
@@ -2362,6 +2425,10 @@ class _Inputs(object):
   def features_and_labels(self):
     """Gets `features` and `labels`."""
     if self.is_dataset:
+      if self._iterator is None:
+        raise RuntimeError('Internal error: Must call dataset_initializer_hook '
+                           'before calling features_and_labels(). Please file '
+                           'a bug!')
       return _Inputs._parse_inputs(self._iterator.get_next())
 
     return (self._features, self._labels)
-- 
GitLab


From 17dfe3ed7db7fb4d41f8933adead4737c30a92c9 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 28 Mar 2018 18:26:30 -0700
Subject: [PATCH 651/960] Implement assert_same_structure in C++

Also implements helper functions nest._is_namedtuple
nest._same_namedtuple.

Also, fix a bug in FlattenHelper where error from recursive
calls were not propagated up immediately.

This change implements a good chunk of machinery that will
allow us to move map_structure to C++.

Before:
entry {
  name: "NestBenchmark.assert_same_structure_6_elem"
  iters: 30000
  wall_time: 4.79532718658e-05
}

entry {
  name: "NestBenchmark.assert_same_structure_60_elem"
  iters: 30000
  wall_time: 0.000403008667628
}

After:
entry {
  name: "NestBenchmark.assert_same_structure_6_elem"
  iters: 30000
  wall_time: 1.65301720301e-05
}

entry {
  name: "NestBenchmark.assert_same_structure_60_elem"
  iters: 30000
  wall_time: 0.000147621099154
}

PiperOrigin-RevId: 190869007
---
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/framework/test_util.py      |   8 +-
 .../kernel_tests/functional_ops_test.py       |   4 +-
 tensorflow/python/util/nest.py                |  90 +----
 tensorflow/python/util/nest_test.py           | 156 +++++---
 tensorflow/python/util/util.cc                | 374 +++++++++++++++++-
 tensorflow/python/util/util.h                 |  51 +++
 tensorflow/python/util/util.i                 |   9 +
 8 files changed, 545 insertions(+), 148 deletions(-)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 4f61c01f65..09c1965d7e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -298,6 +298,7 @@ cc_library(
     srcs = ["util/util.cc"],
     hdrs = ["util/util.h"],
     deps = [
+        ":safe_ptr",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//util/python:python_headers",
diff --git a/tensorflow/python/framework/test_util.py b/tensorflow/python/framework/test_util.py
index 4192a27f65..bf00fa6439 100644
--- a/tensorflow/python/framework/test_util.py
+++ b/tensorflow/python/framework/test_util.py
@@ -487,7 +487,13 @@ def assert_no_new_pyobjects_executing_eagerly(f):
       gc.collect()
       # There should be no new Python objects hanging around.
       new_count = len(gc.get_objects())
-      self.assertEqual(previous_count, new_count)
+      # In some cases (specifacally on MacOS), new_count is somehow
+      # smaller than previous_count.
+      # Using plain assert because not all classes using this decorator
+      # have assertLessEqual
+      assert new_count <= previous_count, (
+          "new_count(%d) is not less than or equal to previous_count(%d)" % (
+              new_count, previous_count))
       gc.enable()
 
   return decorator
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index f5717a5a21..1301ef9d19 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -229,7 +229,7 @@ class FunctionalOpsTest(test.TestCase):
     with self.test_session():
       nums = np.array([1, 2, 3, 4, 5, 6])
       with self.assertRaisesRegexp(
-          TypeError, r"two structures don't have the same sequence type."):
+          TypeError, r"two structures don't have the same nested structure"):
         # lambda emits tuple, but dtype is a list
         functional_ops.map_fn(
             lambda x: ((x + 3) * 2, -(x + 3) * 2),
@@ -316,7 +316,7 @@ class FunctionalOpsTest(test.TestCase):
       initializer = np.array(1.0)
       # Multiply a * 1 each time
       with self.assertRaisesRegexp(
-          ValueError, "two structures don't have the same number of elements"):
+          ValueError, "two structures don't have the same nested structure"):
         functional_ops.scan(lambda a, x: (a, -a), elems, initializer)
 
   def testScan_Scoped(self):
diff --git a/tensorflow/python/util/nest.py b/tensorflow/python/util/nest.py
index 23c2c48f4b..5622431bc9 100644
--- a/tensorflow/python/util/nest.py
+++ b/tensorflow/python/util/nest.py
@@ -60,15 +60,7 @@ def _is_namedtuple(instance, strict=False):
   Returns:
     True if `instance` is a `namedtuple`.
   """
-  # Attemp to limit the test to plain namedtuple (not stuff inheriting from it).
-  if not isinstance(instance, tuple):
-    return False
-  if strict and instance.__class__.__base__ != tuple:
-    return False
-  return (
-      hasattr(instance, "_fields") and
-      isinstance(instance._fields, _collections.Sequence) and
-      all(isinstance(f, _six.string_types) for f in instance._fields))
+  return _pywrap_tensorflow.IsNamedtuple(instance, strict)
 
 
 def _sequence_like(instance, args):
@@ -157,76 +149,7 @@ def flatten(nest):
 
 def _same_namedtuples(nest1, nest2):
   """Returns True if the two namedtuples have the same name and fields."""
-  if nest1._fields != nest2._fields:
-    return False
-  if nest1.__class__.__name__ != nest2.__class__.__name__:
-    return False
-  return True
-
-
-def _recursive_assert_same_structure(nest1, nest2, check_types):
-  """Helper function for `assert_same_structure`.
-
-  See `assert_same_structure` for further information about namedtuples.
-
-  Args:
-    nest1: An arbitrarily nested structure.
-    nest2: An arbitrarily nested structure.
-    check_types: If `True` (default) types of sequences are checked as
-        well, including the keys of dictionaries. If set to `False`, for example
-        a list and a tuple of objects will look the same if they have the same
-        size. Note that namedtuples with identical name and fields are always
-        considered to have the same shallow structure.
-
-  Returns:
-    True if `nest1` and `nest2` have the same structure.
-
-  Raises:
-    ValueError: If the two structure don't have the same nested structre.
-    TypeError: If the two structure don't have the same sequence type.
-    ValueError: If the two dictionaries don't have the same set of keys.
-  """
-  is_sequence_nest1 = is_sequence(nest1)
-  if is_sequence_nest1 != is_sequence(nest2):
-    raise ValueError(
-        "The two structures don't have the same nested structure.\n\n"
-        "First structure: %s\n\nSecond structure: %s." % (nest1, nest2))
-
-  if not is_sequence_nest1:
-    return  # finished checking
-
-  if check_types:
-    type_nest1 = type(nest1)
-    type_nest2 = type(nest2)
-
-    # Duck-typing means that nest should be fine with two different namedtuples
-    # with identical name and fields.
-    if _is_namedtuple(nest1, True) and _is_namedtuple(nest2, True):
-      if not _same_namedtuples(nest1, nest2):
-        raise TypeError(
-            "The two namedtuples don't have the same sequence type. First "
-            "structure has type %s, while second structure has type %s."
-            % (type_nest1, type_nest2))
-    else:
-      if type_nest1 != type_nest2:
-        raise TypeError(
-            "The two structures don't have the same sequence type. First "
-            "structure has type %s, while second structure has type %s."
-            % (type_nest1, type_nest2))
-
-    if isinstance(nest1, dict):
-      keys1 = set(_six.iterkeys(nest1))
-      keys2 = set(_six.iterkeys(nest2))
-      if keys1 != keys2:
-        raise ValueError(
-            "The two dictionaries don't have the same set of keys. First "
-            "structure has keys {}, while second structure has keys {}."
-            .format(keys1, keys2))
-
-  nest1_as_sequence = [n for n in _yield_value(nest1)]
-  nest2_as_sequence = [n for n in _yield_value(nest2)]
-  for n1, n2 in zip(nest1_as_sequence, nest2_as_sequence):
-    _recursive_assert_same_structure(n1, n2, check_types)
+  return _pywrap_tensorflow.SameNamedtuples(nest1, nest2)
 
 
 def assert_same_structure(nest1, nest2, check_types=True):
@@ -257,14 +180,7 @@ def assert_same_structure(nest1, nest2, check_types=True):
     TypeError: If the two structures differ in the type of sequence in any of
       their substructures. Only possible if `check_types` is `True`.
   """
-  len_nest1 = len(flatten(nest1)) if is_sequence(nest1) else 1
-  len_nest2 = len(flatten(nest2)) if is_sequence(nest2) else 1
-  if len_nest1 != len_nest2:
-    raise ValueError("The two structures don't have the same number of "
-                     "elements.\n\nFirst structure (%i elements): %s\n\n"
-                     "Second structure (%i elements): %s"
-                     % (len_nest1, nest1, len_nest2, nest2))
-  _recursive_assert_same_structure(nest1, nest2, check_types)
+  _pywrap_tensorflow.AssertSameStructure(nest1, nest2, check_types)
 
 
 def flatten_dict_items(dictionary):
diff --git a/tensorflow/python/util/nest_test.py b/tensorflow/python/util/nest_test.py
index 4439d6241e..2f12b25354 100644
--- a/tensorflow/python/util/nest_test.py
+++ b/tensorflow/python/util/nest_test.py
@@ -19,11 +19,14 @@ from __future__ import division
 from __future__ import print_function
 
 import collections
+import time
 
 import numpy as np
+from six.moves import xrange  # pylint: disable=redefined-builtin
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
@@ -32,6 +35,9 @@ from tensorflow.python.util import nest
 
 class NestTest(test.TestCase):
 
+  PointXY = collections.namedtuple("Point", ["x", "y"])  # pylint: disable=invalid-name
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testFlattenAndPack(self):
     structure = ((3, 4), 5, (6, 7, (9, 10), 8))
     flat = ["a", "b", "c", "d", "e", "f", "g", "h"]
@@ -39,8 +45,8 @@ class NestTest(test.TestCase):
     self.assertEqual(
         nest.pack_sequence_as(structure, flat), (("a", "b"), "c",
                                                  ("d", "e", ("f", "g"), "h")))
-    point = collections.namedtuple("Point", ["x", "y"])
-    structure = (point(x=4, y=2), ((point(x=1, y=0),),))
+    structure = (NestTest.PointXY(x=4, y=2),
+                 ((NestTest.PointXY(x=1, y=0),),))
     flat = [4, 2, 1, 0]
     self.assertEqual(nest.flatten(structure), flat)
     restructured_from_flat = nest.pack_sequence_as(structure, flat)
@@ -66,6 +72,7 @@ class NestTest(test.TestCase):
     with self.assertRaises(ValueError):
       nest.pack_sequence_as([5, 6, [7, 8]], ["a", "b", "c"])
 
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testFlattenDictOrder(self):
     """`flatten` orders dicts by key, including OrderedDicts."""
     ordered = collections.OrderedDict([("d", 3), ("b", 1), ("a", 0), ("c", 2)])
@@ -87,12 +94,14 @@ class NestTest(test.TestCase):
         ordered_reconstruction)
     self.assertEqual({"d": 3, "b": 1, "a": 0, "c": 2}, plain_reconstruction)
 
+  Abc = collections.namedtuple("A", ("b", "c"))  # pylint: disable=invalid-name
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testFlattenAndPack_withDicts(self):
     # A nice messy mix of tuples, lists, dicts, and `OrderedDict`s.
-    named_tuple = collections.namedtuple("A", ("b", "c"))
     mess = [
         "z",
-        named_tuple(3, 4),
+        NestTest.Abc(3, 4),
         {
             "c": [
                 1,
@@ -111,7 +120,7 @@ class NestTest(test.TestCase):
 
     structure_of_mess = [
         14,
-        named_tuple("a", True),
+        NestTest.Abc("a", True),
         {
             "c": [
                 0,
@@ -157,6 +166,7 @@ class NestTest(test.TestCase):
       nest.pack_sequence_as(["hello", "world"],
                             ["and", "goodbye", "again"])
 
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testIsSequence(self):
     self.assertFalse(nest.is_sequence("1234"))
     self.assertTrue(nest.is_sequence([1, 3, [4, 5]]))
@@ -186,6 +196,23 @@ class NestTest(test.TestCase):
         ValueError, "Key had [0-9]* elements, but value had [0-9]* elements"):
       nest.flatten_dict_items(another_bad_dictionary)
 
+  # pylint does not correctly recognize these as class names and
+  # suggests to use variable style under_score naming.
+  # pylint: disable=invalid-name
+  Named0ab = collections.namedtuple("named_0", ("a", "b"))
+  Named1ab = collections.namedtuple("named_1", ("a", "b"))
+  SameNameab = collections.namedtuple("same_name", ("a", "b"))
+  SameNameab2 = collections.namedtuple("same_name", ("a", "b"))
+  SameNamexy = collections.namedtuple("same_name", ("x", "y"))
+  SameName1xy = collections.namedtuple("same_name_1", ("x", "y"))
+  SameName1xy2 = collections.namedtuple("same_name_1", ("x", "y"))
+  NotSameName = collections.namedtuple("not_same_name", ("a", "b"))
+  # pylint: enable=invalid-name
+
+  class SameNamedType1(SameNameab):
+    pass
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testAssertSameStructure(self):
     structure1 = (((1, 2), 3), 4, (5, 6))
     structure2 = ((("foo1", "foo2"), "foo3"), "foo4", ("foo5", "foo6"))
@@ -198,23 +225,32 @@ class NestTest(test.TestCase):
 
     with self.assertRaisesRegexp(
         ValueError,
-        ("don't have the same number of elements\\.\n\n"
-         "First structure \\(6 elements\\):.*?"
-         "\n\nSecond structure \\(2 elements\\):")):
+        ("The two structures don't have the same nested structure\\.\n\n"
+         "First structure:.*?\n\n"
+         "Second structure:.*\n\n"
+         "More specifically: Substructure "
+         r'"type=tuple str=\(\(1, 2\), 3\)" is a sequence, while '
+         'substructure "type=str str=spam" is not')):
       nest.assert_same_structure(structure1, structure_different_num_elements)
 
     with self.assertRaisesRegexp(
         ValueError,
-        ("don't have the same number of elements\\.\n\n"
-         "First structure \\(2 elements\\):.*?"
-         "\n\nSecond structure \\(1 elements\\):")):
+        ("The two structures don't have the same nested structure\\.\n\n"
+         "First structure:.*?\n\n"
+         "Second structure:.*\n\n"
+         r'More specifically: Substructure "type=list str=\[0, 1\]" '
+         r'is a sequence, while substructure "type=ndarray str=\[0 1\]" '
+         "is not")):
       nest.assert_same_structure([0, 1], np.array([0, 1]))
 
     with self.assertRaisesRegexp(
         ValueError,
-        ("don't have the same number of elements\\.\n\n"
-         "First structure \\(1 elements\\):.*"
-         "\n\nSecond structure \\(2 elements\\):")):
+        ("The two structures don't have the same nested structure\\.\n\n"
+         "First structure:.*?\n\n"
+         "Second structure:.*\n\n"
+         r'More specifically: Substructure "type=list str=\[0, 1\]" '
+         'is a sequence, while substructure "type=int str=0" '
+         "is not")):
       nest.assert_same_structure(0, [0, 1])
 
     self.assertRaises(TypeError, nest.assert_same_structure, (0, 1), [0, 1])
@@ -225,21 +261,21 @@ class NestTest(test.TestCase):
          "First structure: .*?\n\nSecond structure: ")):
       nest.assert_same_structure(structure1, structure_different_nesting)
 
-    named_type_0 = collections.namedtuple("named_0", ("a", "b"))
-    named_type_1 = collections.namedtuple("named_1", ("a", "b"))
     self.assertRaises(TypeError, nest.assert_same_structure, (0, 1),
-                      named_type_0("a", "b"))
+                      NestTest.Named0ab("a", "b"))
 
-    nest.assert_same_structure(named_type_0(3, 4), named_type_0("a", "b"))
+    nest.assert_same_structure(NestTest.Named0ab(3, 4),
+                               NestTest.Named0ab("a", "b"))
 
     self.assertRaises(TypeError, nest.assert_same_structure,
-                      named_type_0(3, 4), named_type_1(3, 4))
+                      NestTest.Named0ab(3, 4), NestTest.Named1ab(3, 4))
 
     with self.assertRaisesRegexp(
         ValueError,
         ("don't have the same nested structure\\.\n\n"
          "First structure: .*?\n\nSecond structure: ")):
-      nest.assert_same_structure(named_type_0(3, 4), named_type_0([3], 4))
+      nest.assert_same_structure(NestTest.Named0ab(3, 4),
+                                 NestTest.Named0ab([3], 4))
 
     with self.assertRaisesRegexp(
         ValueError,
@@ -258,36 +294,33 @@ class NestTest(test.TestCase):
                                  "don't have the same set of keys"):
       nest.assert_same_structure({"a": 1}, {"b": 1})
 
-    same_name_type_0 = collections.namedtuple("same_name", ("a", "b"))
-    same_name_type_1 = collections.namedtuple("same_name", ("a", "b"))
-    nest.assert_same_structure(same_name_type_0(0, 1), same_name_type_1(2, 3))
+    nest.assert_same_structure(NestTest.SameNameab(0, 1),
+                               NestTest.SameNameab2(2, 3))
 
     # This assertion is expected to pass: two namedtuples with the same
     # name and field names are considered to be identical.
-    same_name_type_2 = collections.namedtuple("same_name_1", ("x", "y"))
-    same_name_type_3 = collections.namedtuple("same_name_1", ("x", "y"))
     nest.assert_same_structure(
-        same_name_type_0(same_name_type_2(0, 1), 2),
-        same_name_type_1(same_name_type_3(2, 3), 4))
+        NestTest.SameNameab(NestTest.SameName1xy(0, 1), 2),
+        NestTest.SameNameab2(NestTest.SameName1xy2(2, 3), 4))
 
     expected_message = "The two structures don't have the same.*"
     with self.assertRaisesRegexp(ValueError, expected_message):
-      nest.assert_same_structure(same_name_type_0(0, same_name_type_1(1, 2)),
-                                 same_name_type_1(same_name_type_0(0, 1), 2))
+      nest.assert_same_structure(
+          NestTest.SameNameab(0, NestTest.SameNameab2(1, 2)),
+          NestTest.SameNameab2(NestTest.SameNameab(0, 1), 2))
 
-    same_name_type_1 = collections.namedtuple("not_same_name", ("a", "b"))
     self.assertRaises(TypeError, nest.assert_same_structure,
-                      same_name_type_0(0, 1), same_name_type_1(2, 3))
+                      NestTest.SameNameab(0, 1), NestTest.NotSameName(2, 3))
 
-    same_name_type_1 = collections.namedtuple("same_name", ("x", "y"))
     self.assertRaises(TypeError, nest.assert_same_structure,
-                      same_name_type_0(0, 1), same_name_type_1(2, 3))
+                      NestTest.SameNameab(0, 1), NestTest.SameNamexy(2, 3))
 
-    class SameNamedType1(collections.namedtuple("same_name", ("a", "b"))):
-      pass
     self.assertRaises(TypeError, nest.assert_same_structure,
-                      same_name_type_0(0, 1), SameNamedType1(2, 3))
+                      NestTest.SameNameab(0, 1), NestTest.SameNamedType1(2, 3))
 
+  EmptyNT = collections.namedtuple("empty_nt", "")  # pylint: disable=invalid-name
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testMapStructure(self):
     structure1 = (((1, 2), 3), 4, (5, 6))
     structure2 = (((7, 8), 9), 10, (11, 12))
@@ -310,9 +343,8 @@ class NestTest(test.TestCase):
     self.assertEqual((), nest.map_structure(lambda x: x + 1, ()))
     self.assertEqual([], nest.map_structure(lambda x: x + 1, []))
     self.assertEqual({}, nest.map_structure(lambda x: x + 1, {}))
-    empty_nt = collections.namedtuple("empty_nt", "")
-    self.assertEqual(empty_nt(), nest.map_structure(lambda x: x + 1,
-                                                    empty_nt()))
+    self.assertEqual(NestTest.EmptyNT(), nest.map_structure(lambda x: x + 1,
+                                                            NestTest.EmptyNT()))
 
     # This is checking actual equality of types, empty list != empty tuple
     self.assertNotEqual((), nest.map_structure(lambda x: x + 1, []))
@@ -352,10 +384,12 @@ class NestTest(test.TestCase):
     with self.assertRaisesRegexp(ValueError, "Only valid keyword argument"):
       nest.map_structure(lambda x: None, structure1, check_types=False, foo="a")
 
+  ABTuple = collections.namedtuple("ab_tuple", "a, b")  # pylint: disable=invalid-name
+
+  @test_util.assert_no_new_pyobjects_executing_eagerly
   def testMapStructureWithStrings(self):
-    ab_tuple = collections.namedtuple("ab_tuple", "a, b")
-    inp_a = ab_tuple(a="foo", b=("bar", "baz"))
-    inp_b = ab_tuple(a=2, b=(1, 3))
+    inp_a = NestTest.ABTuple(a="foo", b=("bar", "baz"))
+    inp_b = NestTest.ABTuple(a=2, b=(1, 3))
     out = nest.map_structure(lambda string, repeats: string * repeats,
                              inp_a,
                              inp_b)
@@ -363,8 +397,8 @@ class NestTest(test.TestCase):
     self.assertEqual("bar", out.b[0])
     self.assertEqual("bazbazbaz", out.b[1])
 
-    nt = ab_tuple(a=("something", "something_else"),
-                  b="yet another thing")
+    nt = NestTest.ABTuple(a=("something", "something_else"),
+                          b="yet another thing")
     rev_nt = nest.map_structure(lambda x: x[::-1], nt)
     # Check the output is the correct structure, and all strings are reversed.
     nest.assert_same_structure(nt, rev_nt)
@@ -431,10 +465,8 @@ class NestTest(test.TestCase):
 
     # This assertion is expected to pass: two namedtuples with the same
     # name and field names are considered to be identical.
-    same_name_type_0 = collections.namedtuple("same_name", ("a", "b"))
-    same_name_type_1 = collections.namedtuple("same_name", ("a", "b"))
-    inp_shallow = same_name_type_0(1, 2)
-    inp_deep = same_name_type_1(1, [1, 2, 3])
+    inp_shallow = NestTest.SameNameab(1, 2)
+    inp_deep = NestTest.SameNameab2(1, [1, 2, 3])
     nest.assert_shallow_structure(inp_shallow, inp_deep, check_types=False)
     nest.assert_shallow_structure(inp_shallow, inp_deep, check_types=True)
 
@@ -466,7 +498,7 @@ class NestTest(test.TestCase):
                      [1, {"c": 2}, 3, (4, 5)])
 
     # Namedtuples.
-    ab_tuple = collections.namedtuple("ab_tuple", "a, b")
+    ab_tuple = NestTest.ABTuple
     input_tree = ab_tuple(a=[0, 1], b=2)
     shallow_tree = ab_tuple(a=0, b=1)
     input_tree_flattened_as_shallow_tree = nest.flatten_up_to(shallow_tree,
@@ -681,5 +713,31 @@ class NestTest(test.TestCase):
           list(nest.flatten_with_joined_string_paths(inputs)), expected)
 
 
+class NestBenchmark(test.Benchmark):
+
+  def run_and_report(self, s1, s2, name):
+    burn_iter, test_iter = 100, 30000
+
+    for _ in xrange(burn_iter):
+      nest.assert_same_structure(s1, s2)
+
+    t0 = time.time()
+    for _ in xrange(test_iter):
+      nest.assert_same_structure(s1, s2)
+    t1 = time.time()
+
+    self.report_benchmark(iters=test_iter, wall_time=(t1 - t0) / test_iter,
+                          name=name)
+
+  def benchmark_assert_structure(self):
+    s1 = (((1, 2), 3), 4, (5, 6))
+    s2 = ((("foo1", "foo2"), "foo3"), "foo4", ("foo5", "foo6"))
+    self.run_and_report(s1, s2, "assert_same_structure_6_elem")
+
+    s1 = (((1, 2), 3), 4, (5, 6)) * 10
+    s2 = ((("foo1", "foo2"), "foo3"), "foo4", ("foo5", "foo6")) * 10
+    self.run_and_report(s1, s2, "assert_same_structure_60_elem")
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/util/util.cc b/tensorflow/python/util/util.cc
index a41fa7df25..70aee4a3f6 100644
--- a/tensorflow/python/util/util.cc
+++ b/tensorflow/python/util/util.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/logging.h"
+#include "tensorflow/python/lib/core/safe_ptr.h"
 
 namespace tensorflow {
 namespace swig {
@@ -27,6 +28,113 @@ PyObject* CollectionsSequenceType = nullptr;
 
 bool WarnedThatSetIsNotSequence = false;
 
+bool IsString(PyObject* o) {
+  return PyBytes_Check(o) ||
+#if PY_MAJOR_VERSION < 3
+         PyString_Check(o) ||
+#endif
+         PyUnicode_Check(o);
+}
+
+// Equivalent to Python's 'o.__class__.__name__'
+// Note that '__class__' attribute is set only in new-style classes.
+// A lot of tensorflow code uses __class__ without checks, so it seems like
+// we only support new-style classes.
+StringPiece GetClassName(PyObject* o) {
+  // __class__ is equivalent to type() for new style classes.
+  // type() is equivalent to PyObject_Type()
+  // (https://docs.python.org/3.5/c-api/object.html#c.PyObject_Type)
+  // PyObject_Type() is equivalent to o->ob_type except for Py_INCREF, which
+  // we don't need here.
+  PyTypeObject* type = o->ob_type;
+
+  // __name__ is the value of `tp_name` after the last '.'
+  // (https://docs.python.org/2/c-api/typeobj.html#c.PyTypeObject.tp_name)
+  StringPiece name(type->tp_name);
+  size_t pos = name.rfind('.');
+  if (pos != StringPiece::npos) {
+    name.remove_prefix(pos + 1);
+  }
+  return name;
+}
+
+string PyObjectToString(PyObject* o) {
+  if (o == nullptr) {
+    return "<null object>";
+  }
+  PyObject* str = PyObject_Str(o);
+  if (str) {
+#if PY_MAJOR_VERSION < 3
+    string s(PyString_AS_STRING(str));
+#else
+    string s(PyUnicode_AsUTF8(str));
+#endif
+    Py_DECREF(str);
+    return tensorflow::strings::StrCat("type=", GetClassName(o), " str=", s);
+  } else {
+    return "<failed to execute str() on object>";
+  }
+}
+
+// Implements the same idea as tensorflow.util.nest._yield_value
+// During construction we check if the iterable is a dictionary.
+// If so, we construct a sequence from its sorted keys that will be used
+// for iteration.
+// If not, we construct a sequence directly from the iterable.
+// At each step, we get the next element from the sequence and use it
+// either as a key or return it directly.
+//
+// 'iterable' must not be modified while ValIterator is used.
+class ValIterator {
+ public:
+  explicit ValIterator(PyObject* iterable) : dict_(nullptr), index_(0) {
+    if (PyDict_Check(iterable)) {
+      dict_ = iterable;
+      // PyDict_Keys returns a list, which can be used with
+      // PySequence_Fast_GET_ITEM.
+      seq_ = PyDict_Keys(iterable);
+      // Iterate through dictionaries in a deterministic order by sorting the
+      // keys. Notice this means that we ignore the original order of
+      // `OrderedDict` instances. This is intentional, to avoid potential
+      // bugs caused by mixing ordered and plain dicts (e.g., flattening
+      // a dict but using a corresponding `OrderedDict` to pack it back).
+      PyList_Sort(seq_);
+    } else {
+      seq_ = PySequence_Fast(iterable, "");
+    }
+    size_ = PySequence_Fast_GET_SIZE(seq_);
+  }
+
+  ~ValIterator() { Py_DECREF(seq_); }
+
+  // Return a borrowed reference to the next element from iterable.
+  // Return nullptr when iteration is over.
+  PyObject* next() {
+    PyObject* element = nullptr;
+    if (index_ < size_) {
+      // Both PySequence_Fast_GET_ITEM and PyDict_GetItem return borrowed
+      // references.
+      element = PySequence_Fast_GET_ITEM(seq_, index_);
+      ++index_;
+      if (dict_ != nullptr) {
+        element = PyDict_GetItem(dict_, element);
+        if (element == nullptr) {
+          PyErr_SetString(PyExc_RuntimeError,
+                          "Dictionary was modified during iteration over it");
+          return nullptr;
+        }
+      }
+    }
+    return element;
+  }
+
+ private:
+  PyObject* seq_;
+  PyObject* dict_;
+  Py_ssize_t size_;
+  Py_ssize_t index_;
+};
+
 // Returns 1 if `o` is considered a sequence for the purposes of Flatten().
 // Returns 0 otherwise.
 // Returns -1 if an error occurred.
@@ -38,7 +146,7 @@ int IsSequenceHelper(PyObject* o) {
                     "so consider avoiding using them.";
     WarnedThatSetIsNotSequence = true;
   }
-  if (CollectionsSequenceType == nullptr) {
+  if (TF_PREDICT_FALSE(CollectionsSequenceType == nullptr)) {
     PyErr_SetString(
         PyExc_RuntimeError,
         tensorflow::strings::StrCat(
@@ -49,11 +157,7 @@ int IsSequenceHelper(PyObject* o) {
   }
   int is_instance = PyObject_IsInstance(o, CollectionsSequenceType);
   if (is_instance == -1) return -1;
-  return static_cast<int>(is_instance != 0 && !PyBytes_Check(o) &&
-#if PY_MAJOR_VERSION < 3
-                          !PyString_Check(o) &&
-#endif
-                          !PyUnicode_Check(o));
+  return static_cast<int>(is_instance != 0 && !IsString(o));
 }
 
 bool FlattenHelper(PyObject* nested, PyObject* list) {
@@ -75,12 +179,16 @@ bool FlattenHelper(PyObject* nested, PyObject* list) {
       // while the method is running.
       PyObject* key = PyList_GET_ITEM(keys, i);
       PyObject* val = PyDict_GetItem(nested, key);
-      if (Py_EnterRecursiveCall(" in Flatten")) {
+      if (Py_EnterRecursiveCall(" in flatten")) {
         Py_DECREF(keys);
         return false;
       }
-      FlattenHelper(val, list);
+      const bool success = FlattenHelper(val, list);
       Py_LeaveRecursiveCall();
+      if (!success) {
+        Py_DECREF(keys);
+        return false;
+      }
     }
     Py_DECREF(keys);
     return true;
@@ -90,13 +198,159 @@ bool FlattenHelper(PyObject* nested, PyObject* list) {
   PyObject* item;
   PyObject* iterator = PyObject_GetIter(nested);
   while ((item = PyIter_Next(iterator)) != nullptr) {
-    FlattenHelper(item, list);
+    if (Py_EnterRecursiveCall(" in flatten")) {
+      Py_DECREF(iterator);
+      Py_DECREF(item);
+      return false;
+    }
+    bool success = FlattenHelper(item, list);
+    Py_LeaveRecursiveCall();
+    if (!success) {
+      Py_DECREF(iterator);
+      Py_DECREF(item);
+      return false;
+    }
     Py_DECREF(item);
   }
   Py_DECREF(iterator);
   return true;
 }
 
+// Sets error using keys of 'dict1' and 'dict2'.
+// 'dict1' and 'dict2' are assumed to be Python dictionaries.
+void SetDifferentKeysError(PyObject* dict1, PyObject* dict2, string* error_msg,
+                           bool* is_type_error) {
+  PyObject* k1 = PyDict_Keys(dict1);
+  PyObject* k2 = PyDict_Keys(dict2);
+  *is_type_error = false;
+  *error_msg = tensorflow::strings::StrCat(
+      "The two dictionaries don't have the same set of keys. "
+      "First structure has keys ",
+      PyObjectToString(k1), ", while second structure has keys ",
+      PyObjectToString(k2));
+  Py_DECREF(k1);
+  Py_DECREF(k2);
+}
+
+// Returns true iff there were no "internal" errors. In other words,
+// errors that has nothing to do with structure checking.
+// If an "internal" error occured, the appropriate Python error will be
+// set and the caller can propage it directly to the user.
+//
+// Both `error_msg` and `is_type_error` must be non-null. `error_msg` must
+// be empty.
+// Leaves `error_msg` empty if structures matched. Else, fills `error_msg`
+// with appropriate error and sets `is_type_error` to true iff
+// the error to be raised should be TypeError.
+bool AssertSameStructureHelper(PyObject* o1, PyObject* o2, bool check_types,
+                               string* error_msg, bool* is_type_error) {
+  DCHECK(error_msg);
+  DCHECK(is_type_error);
+  const bool is_seq1 = IsSequence(o1);
+  const bool is_seq2 = IsSequence(o2);
+  if (PyErr_Occurred()) return false;
+  if (is_seq1 != is_seq2) {
+    string seq_str = is_seq1 ? PyObjectToString(o1) : PyObjectToString(o2);
+    string non_seq_str = is_seq1 ? PyObjectToString(o2) : PyObjectToString(o1);
+    *is_type_error = false;
+    *error_msg = tensorflow::strings::StrCat(
+        "Substructure \"", seq_str, "\" is a sequence, while substructure \"",
+        non_seq_str, "\" is not");
+    return true;
+  }
+
+  // Got to scalars, so finished checking. Structures are the same.
+  if (!is_seq1) return true;
+
+  if (check_types) {
+    const PyTypeObject* type1 = o1->ob_type;
+    const PyTypeObject* type2 = o2->ob_type;
+
+    // We treat two different namedtuples with identical name and fields
+    // as having the same type.
+    const PyObject* o1_tuple = IsNamedtuple(o1, true);
+    if (o1_tuple == nullptr) return false;
+    const PyObject* o2_tuple = IsNamedtuple(o2, true);
+    if (o2_tuple == nullptr) {
+      Py_DECREF(o1_tuple);
+      return false;
+    }
+    bool both_tuples = o1_tuple == Py_True && o2_tuple == Py_True;
+    Py_DECREF(o1_tuple);
+    Py_DECREF(o2_tuple);
+
+    if (both_tuples) {
+      const PyObject* same_tuples = SameNamedtuples(o1, o2);
+      if (same_tuples == nullptr) return false;
+      bool not_same_tuples = same_tuples != Py_True;
+      Py_DECREF(same_tuples);
+      if (not_same_tuples) {
+        *is_type_error = true;
+        *error_msg = tensorflow::strings::StrCat(
+            "The two namedtuples don't have the same sequence type. "
+            "First structure ",
+            PyObjectToString(o1), " has type ", type1->tp_name,
+            ", while second structure ", PyObjectToString(o2), " has type ",
+            type2->tp_name);
+        return true;
+      }
+    } else if (type1 != type2) {
+      *is_type_error = true;
+      *error_msg = tensorflow::strings::StrCat(
+          "The two namedtuples don't have the same sequence type. "
+          "First structure ",
+          PyObjectToString(o1), " has type ", type1->tp_name,
+          ", while second structure ", PyObjectToString(o2), " has type ",
+          type2->tp_name);
+      return true;
+    }
+
+    if (PyDict_Check(o1)) {
+      if (PyDict_Size(o1) != PyDict_Size(o2)) {
+        SetDifferentKeysError(o1, o2, error_msg, is_type_error);
+        return true;
+      }
+
+      PyObject* key;
+      Py_ssize_t pos = 0;
+      while (PyDict_Next(o1, &pos, &key, nullptr)) {
+        if (PyDict_GetItem(o2, key) == nullptr) {
+          SetDifferentKeysError(o1, o2, error_msg, is_type_error);
+          return true;
+        }
+      }
+    }
+  }
+
+  ValIterator iter1(o1);
+  ValIterator iter2(o2);
+
+  while (true) {
+    PyObject* v1 = iter1.next();
+    PyObject* v2 = iter2.next();
+    if (v1 != nullptr && v2 != nullptr) {
+      if (Py_EnterRecursiveCall(" in assert_same_structure")) {
+        return false;
+      }
+      bool no_internal_errors = AssertSameStructureHelper(
+          v1, v2, check_types, error_msg, is_type_error);
+      Py_LeaveRecursiveCall();
+      if (!no_internal_errors) return false;
+      if (!error_msg->empty()) return true;
+    } else if (v1 == nullptr && v2 == nullptr) {
+      // Done with all recursive calls. Structure matched.
+      return true;
+    } else {
+      *is_type_error = false;
+      *error_msg = tensorflow::strings::StrCat(
+          "The two structures don't have the same number of elements. ",
+          "First structure: ", PyObjectToString(o1),
+          ". Second structure: ", PyObjectToString(o2));
+      return true;
+    }
+  }
+}
+
 }  // anonymous namespace
 
 void RegisterSequenceClass(PyObject* sequence_class) {
@@ -123,5 +377,107 @@ PyObject* Flatten(PyObject* nested) {
     return nullptr;
   }
 }
+
+PyObject* IsNamedtuple(PyObject* o, bool strict) {
+  // Must be subclass of tuple
+  if (!PyTuple_Check(o)) {
+    Py_RETURN_FALSE;
+  }
+
+  // If strict, o.__class__.__base__ must be tuple
+  if (strict) {
+    PyObject* klass = PyObject_GetAttrString(o, "__class__");
+    if (klass == nullptr) return nullptr;
+    PyObject* base = PyObject_GetAttrString(klass, "__base__");
+    Py_DECREF(klass);
+    if (base == nullptr) return nullptr;
+
+    const PyTypeObject* base_type = reinterpret_cast<PyTypeObject*>(base);
+    // built-in object types are singletons
+    bool tuple_base = base_type == &PyTuple_Type;
+    Py_DECREF(base);
+    if (!tuple_base) {
+      Py_RETURN_FALSE;
+    }
+  }
+
+  if (TF_PREDICT_FALSE(CollectionsSequenceType == nullptr)) {
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        tensorflow::strings::StrCat(
+            "collections.Sequence type has not been set. "
+            "Please call RegisterSequenceClass before using this module")
+            .c_str());
+    return nullptr;
+  }
+
+  // o must have attribute '_fields' and every element in
+  // '_fields' must be a string.
+  int has_fields = PyObject_HasAttrString(o, "_fields");
+  if (!has_fields) {
+    Py_RETURN_FALSE;
+  }
+
+  Safe_PyObjectPtr fields = make_safe(PyObject_GetAttrString(o, "_fields"));
+  int is_instance = PyObject_IsInstance(fields.get(), CollectionsSequenceType);
+  if (is_instance == 0) {
+    Py_RETURN_FALSE;
+  } else if (is_instance == -1) {
+    return nullptr;
+  }
+
+  Safe_PyObjectPtr seq = make_safe(PySequence_Fast(fields.get(), ""));
+  const Py_ssize_t s = PySequence_Fast_GET_SIZE(seq.get());
+  for (Py_ssize_t i = 0; i < s; ++i) {
+    // PySequence_Fast_GET_ITEM returns borrowed ref
+    PyObject* elem = PySequence_Fast_GET_ITEM(seq.get(), i);
+    if (!IsString(elem)) {
+      Py_RETURN_FALSE;
+    }
+  }
+
+  Py_RETURN_TRUE;
+}
+
+PyObject* SameNamedtuples(PyObject* o1, PyObject* o2) {
+  PyObject* f1 = PyObject_GetAttrString(o1, "_fields");
+  PyObject* f2 = PyObject_GetAttrString(o2, "_fields");
+  if (f1 == nullptr || f2 == nullptr) {
+    Py_XDECREF(f1);
+    Py_XDECREF(f2);
+    PyErr_SetString(
+        PyExc_RuntimeError,
+        "Expected namedtuple-like objects (that have _fields attr)");
+    return nullptr;
+  }
+
+  if (PyObject_RichCompareBool(f1, f2, Py_NE)) {
+    Py_RETURN_FALSE;
+  }
+
+  if (GetClassName(o1).compare(GetClassName(o2)) == 0) {
+    Py_RETURN_TRUE;
+  } else {
+    Py_RETURN_FALSE;
+  }
+}
+
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types) {
+  string error_msg;
+  bool is_type_error = false;
+  AssertSameStructureHelper(o1, o2, check_types, &error_msg, &is_type_error);
+  if (!error_msg.empty()) {
+    PyErr_SetString(
+        is_type_error ? PyExc_TypeError : PyExc_ValueError,
+        tensorflow::strings::StrCat(
+            "The two structures don't have the same nested structure.\n\n",
+            "First structure: ", PyObjectToString(o1), "\n\nSecond structure: ",
+            PyObjectToString(o2), "\n\nMore specifically: ", error_msg)
+            .c_str());
+    return nullptr;
+  }
+  Py_RETURN_NONE;
+}
+
 }  // namespace swig
 }  // namespace tensorflow
diff --git a/tensorflow/python/util/util.h b/tensorflow/python/util/util.h
index 2af71dc753..c325baa5f8 100644
--- a/tensorflow/python/util/util.h
+++ b/tensorflow/python/util/util.h
@@ -33,6 +33,57 @@ namespace swig {
 //   dict.
 bool IsSequence(PyObject* o);
 
+// Implements the same interface as tensorflow.util.nest._is_namedtuple
+// Returns Py_True iff `instance` should be considered a `namedtuple`.
+//
+// Args:
+//   instance: An instance of a Python object.
+//   strict: If True, `instance` is considered to be a `namedtuple` only if
+//       it is a "plain" namedtuple. For instance, a class inheriting
+//       from a `namedtuple` will be considered to be a `namedtuple`
+//       iff `strict=False`.
+//
+// Returns:
+//   True if `instance` is a `namedtuple`.
+PyObject* IsNamedtuple(PyObject* o, bool strict);
+
+// Implements the same interface as tensorflow.util.nest._same_namedtuples
+// Returns Py_True iff the two namedtuples have the same name and fields.
+// Raises RuntimeError if `o1` or `o2` don't look like namedtuples (don't have
+// '_fields' attribute).
+PyObject* SameNamedtuples(PyObject* o1, PyObject* o2);
+
+// Asserts that two structures are nested in the same way.
+//
+// Note that namedtuples with identical name and fields are always considered
+// to have the same shallow structure (even with `check_types=True`).
+// For intance, this code will print `True`:
+//
+// ```python
+// def nt(a, b):
+//   return collections.namedtuple('foo', 'a b')(a, b)
+// print(assert_same_structure(nt(0, 1), nt(2, 3)))
+// ```
+//
+// Args:
+//  nest1: an arbitrarily nested structure.
+//  nest2: an arbitrarily nested structure.
+//  check_types: if `true`, types of sequences are checked as
+//      well, including the keys of dictionaries. If set to `false`, for example
+//      a list and a tuple of objects will look the same if they have the same
+//      size. Note that namedtuples with identical name and fields are always
+//      considered to have the same shallow structure.
+//
+// Raises:
+//  ValueError: If the two structures do not have the same number of elements or
+//    if the two structures are not nested in the same way.
+//  TypeError: If the two structures differ in the type of sequence in any of
+//    their substructures. Only possible if `check_types` is `True`.
+//
+// Returns:
+//  Py_None on success, nullptr on error.
+PyObject* AssertSameStructure(PyObject* o1, PyObject* o2, bool check_types);
+
 // Implements the same interface as tensorflow.util.nest.flatten
 //
 // Returns a flat list from a given nested structure.
diff --git a/tensorflow/python/util/util.i b/tensorflow/python/util/util.i
index d69084fc00..b7f201b6fe 100644
--- a/tensorflow/python/util/util.i
+++ b/tensorflow/python/util/util.i
@@ -34,6 +34,15 @@ limitations under the License.
 %unignore tensorflow::swig::IsSequence;
 %noexception tensorflow::swig::IsSequence;
 
+%unignore tensorflow::swig::IsNamedtuple;
+%noexception tensorflow::swig::IsNamedtuple;
+
+%unignore tensorflow::swig::SameNamedtuples;
+%noexception tensorflow::swig::SameNamedtuples;
+
+%unignore tensorflow::swig::AssertSameStructure;
+%noexception tensorflow::swig::AssertSameStructure;
+
 %unignore tensorflow::swig::Flatten;
 %noexception tensorflow::swig::Flatten;
 
-- 
GitLab


From 59a12553545c3d8f957a1a6e618561d4228f7f59 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 28 Mar 2018 18:26:46 -0700
Subject: [PATCH 652/960] Relax CuDNN version requirements because CuDNN is
 backwards compatible within a major release starting with CuDNN 7.0

PiperOrigin-RevId: 190869028
---
 tensorflow/stream_executor/BUILD              |  6 +-
 tensorflow/stream_executor/cuda/cuda_dnn.cc   | 87 +++++++++++++------
 .../stream_executor/cuda/cudnn_version.cc     | 42 +++++++++
 .../stream_executor/cuda/cudnn_version.h      | 51 +++++++++++
 .../cuda/cudnn_version_test.cc                | 75 ++++++++++++++++
 5 files changed, 233 insertions(+), 28 deletions(-)
 create mode 100644 tensorflow/stream_executor/cuda/cudnn_version.cc
 create mode 100644 tensorflow/stream_executor/cuda/cudnn_version.h
 create mode 100644 tensorflow/stream_executor/cuda/cudnn_version_test.cc

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 1865240014..27cdb860fe 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -56,7 +56,10 @@ cc_library(
             [
                 "cuda/*.cc",
             ],
-            exclude = ["cuda/cuda_platform_id.cc"],
+            exclude = [
+                "cuda/*_test.cc",
+                "cuda/cuda_platform_id.cc",
+            ],
         ),
     ),
     copts = select({
@@ -72,6 +75,7 @@ cc_library(
         ":stream_executor",
         "//tensorflow/core:lib",
         "//tensorflow/core/kernels:ops_util",
+        "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
     ] + if_cuda_is_configured([
         "//tensorflow/core:cuda",
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index ab5e6590e0..1aea0485fd 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -18,7 +18,9 @@ limitations under the License.
 #include <functional>
 #include <memory>
 
+#include "absl/strings/str_cat.h"
 #include "third_party/eigen3/Eigen/Core"
+#include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/util/env_var.h"
 #include "tensorflow/stream_executor/cuda/cuda_activation.h"
 #include "tensorflow/stream_executor/cuda/cuda_diagnostics.h"
@@ -27,6 +29,7 @@ limitations under the License.
 #include "tensorflow/stream_executor/cuda/cuda_platform_id.h"
 #include "tensorflow/stream_executor/cuda/cuda_stream.h"
 #include "tensorflow/stream_executor/cuda/cuda_timer.h"
+#include "tensorflow/stream_executor/cuda/cudnn_version.h"
 #include "tensorflow/stream_executor/dnn.h"
 #include "tensorflow/stream_executor/lib/env.h"
 #include "tensorflow/stream_executor/lib/error.h"
@@ -55,15 +58,6 @@ NarrowT CheckedNarrowing(const WideT& wide) {
   return narrow;
 }
 
-// Returns the "Compatibility" version number from the CuDNN version number.
-// This is the number that tries to indicate ABI compatibility.
-//
-// For example, if cudnn_version is 5107, the compatibility version
-// number will be 5100.
-size_t cudnnCompatibilityVersion(size_t cudnn_version) {
-  return (cudnn_version / 100) * 100;
-}
-
 }  // namespace
 
 namespace perftools {
@@ -109,6 +103,22 @@ string ToString(cudnnStatus_t status) {
   }
 }
 
+#if CUDNN_VERSION >= 6000
+string ToString(libraryPropertyType type) {
+  switch (type) {
+    case MAJOR_VERSION:
+      return "MAJOR_VERSION";
+    case MINOR_VERSION:
+      return "MINOR_VERSION";
+    case PATCH_LEVEL:
+      return "PATCH_LEVEL";
+    default:
+      return absl::StrCat(
+          "<unknown libraryPropertyType: ", static_cast<int>(type), ">");
+  }
+}
+#endif
+
 template <typename T>
 cudnnDataType_t GetCudnnDataType();
 
@@ -360,6 +370,34 @@ cudnnConvolutionBwdFilterAlgo_t ToConvBackwardFilterAlgo(
   }
 }
 
+#if CUDNN_VERSION >= 6000
+port::Status GetCudnnProperty(libraryPropertyType type, int* value) {
+  cudnnStatus_t status = cudnnGetProperty(type, value);
+  if (status != CUDNN_STATUS_SUCCESS) {
+    const string error =
+        absl::StrCat("cudnnGetProperty failed for type: ", ToString(type),
+                     " with status: ", ToString(status));
+    LOG(ERROR) << error;
+    return port::Status{port::error::INTERNAL, error};
+  }
+  return port::Status::OK();
+}
+#endif
+
+port::Status GetLoadedCudnnVersion(CudnnVersion* version) {
+#if CUDNN_VERSION >= 6000
+  TF_RETURN_IF_ERROR(GetCudnnProperty(MAJOR_VERSION, &version->major_version));
+  TF_RETURN_IF_ERROR(GetCudnnProperty(MINOR_VERSION, &version->minor_version));
+  TF_RETURN_IF_ERROR(GetCudnnProperty(PATCH_LEVEL, &version->patch_level));
+#else
+  size_t loaded_version = ::cudnnGetVersion();
+  version->major_version = loaded_version / 1000;
+  version->minor_version = (loaded_version / 100) % 10;
+  version->patch_level = loaded_version % 100;
+#endif
+  return port::Status::OK();
+}
+
 }  // namespace
 
 CudnnSupport::CudnnSupport(CUDAExecutor* parent)
@@ -376,24 +414,19 @@ port::Status CudnnSupport::Init() {
   auto status = wrap::cudnnCreate(
       parent_, reinterpret_cast<cudnnHandle_t*>(&dnn_handle_));
   if (status == CUDNN_STATUS_SUCCESS) {
-    // Check whether loaded version of CuDNN matches what the source
-    // was built with.
-    size_t loaded_version = ::cudnnGetVersion();
-    size_t loaded_compat_version = cudnnCompatibilityVersion(loaded_version);
-    size_t compiled_compat_version = cudnnCompatibilityVersion(CUDNN_VERSION);
-    bool library_loaded_matches_source =
-        (loaded_compat_version == compiled_compat_version);
-    if (!library_loaded_matches_source) {
-      const string error =
-          port::StrCat("Loaded runtime CuDNN library: ", loaded_version,
-                       " (compatibility version ", loaded_compat_version,
-                       ") but source was compiled with ", CUDNN_VERSION,
-                       " (compatibility version ", compiled_compat_version,
-                       ").  If using a binary install, upgrade your CuDNN "
-                       "library to match.  If building from sources, "
-                       "make sure the library loaded at runtime matches a "
-                       "compatible version specified during compile "
-                       "configuration.");
+    CudnnVersion source_version(CUDNN_MAJOR, CUDNN_MINOR, CUDNN_PATCHLEVEL);
+
+    CudnnVersion loaded_version;
+    TF_RETURN_IF_ERROR(GetLoadedCudnnVersion(&loaded_version));
+    if (!IsSourceCompatibleWithCudnnLibrary(source_version, loaded_version)) {
+      const tensorflow::string error = absl::StrCat(
+          "Loaded runtime CuDNN library: ", loaded_version.ToString(),
+          " but source was compiled with: ", source_version.ToString(),
+          ".  CuDNN library major and minor version needs to match or have "
+          "higher minor version in case of CuDNN 7.0 or later version. If "
+          "using a binary install, upgrade your CuDNN library.  If building "
+          "from sources, make sure the library loaded at runtime is compatible "
+          "with the version specified during compile configuration.");
       LOG(ERROR) << error;
       return port::Status{port::error::INTERNAL, error};
     }
diff --git a/tensorflow/stream_executor/cuda/cudnn_version.cc b/tensorflow/stream_executor/cuda/cudnn_version.cc
new file mode 100644
index 0000000000..5591801aae
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cudnn_version.cc
@@ -0,0 +1,42 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/cuda/cudnn_version.h"
+
+namespace perftools {
+namespace gputools {
+namespace cuda {
+
+bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
+                                        CudnnVersion loaded_version) {
+  // Major version is neither forward or backward compatible and therefore major
+  // versions needs to match between source and library.
+  //
+  // Minor version is backward-compatible beginning with CuDNN 7 and therefore
+  // minor version of library needs to be same or higher.
+  //
+  // Patch releases are always forward and backward compatible and therefore
+  // need not match.
+  if (loaded_version.major_version != source_version.major_version) {
+    return false;
+  }
+  return ((loaded_version.minor_version == source_version.minor_version) ||
+          (source_version.major_version >= 7 &&
+           loaded_version.minor_version >= source_version.minor_version));
+}
+
+}  // namespace cuda
+}  // namespace gputools
+}  // namespace perftools
diff --git a/tensorflow/stream_executor/cuda/cudnn_version.h b/tensorflow/stream_executor/cuda/cudnn_version.h
new file mode 100644
index 0000000000..058cc87bfa
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cudnn_version.h
@@ -0,0 +1,51 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
+#define TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
+
+#include <string>
+
+#include "absl/strings/str_join.h"
+
+namespace perftools {
+namespace gputools {
+namespace cuda {
+
+struct CudnnVersion {
+  CudnnVersion() = default;
+
+  CudnnVersion(int major, int minor, int patch)
+      : major_version(major), minor_version(minor), patch_level(patch) {}
+
+  std::string ToString() const {
+    return absl::StrJoin({major_version, minor_version, patch_level}, ".");
+  }
+
+  int major_version;
+  int minor_version;
+  int patch_level;
+};
+
+// Returns true if the given source CuDNN version is compatible with the given
+// loaded version.
+bool IsSourceCompatibleWithCudnnLibrary(CudnnVersion source_version,
+                                        CudnnVersion loaded_version);
+
+}  // namespace cuda
+}  // namespace gputools
+}  // namespace perftools
+
+#endif  // TENSORFLOW_STREAM_EXECUTOR_CUDA_CUDNN_VERSION_H_
diff --git a/tensorflow/stream_executor/cuda/cudnn_version_test.cc b/tensorflow/stream_executor/cuda/cudnn_version_test.cc
new file mode 100644
index 0000000000..230adafeb1
--- /dev/null
+++ b/tensorflow/stream_executor/cuda/cudnn_version_test.cc
@@ -0,0 +1,75 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/stream_executor/cuda/cudnn_version.h"
+
+#include "testing/base/public/gunit.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace perftools {
+namespace gputools {
+namespace cuda {
+namespace {
+
+TEST(CuDNNVersion, ToString) {
+  CudnnVersion version(7, 0, 12);
+  EXPECT_EQ(version.ToString(), "7.0.12");
+}
+
+TEST(IsSourceCompatibleWithCudnnLibraryTest, Basic) {
+  // Returns true if both major and minor versions are matching and even if the
+  // patch versions are not matching.
+  EXPECT_TRUE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(7, 0, 12),
+      /*loaded_version=*/CudnnVersion(7, 0, 14)));
+  EXPECT_TRUE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(6, 1, 14),
+      /*loaded_version=*/CudnnVersion(6, 1, 00)));
+
+  // Returns false if major versions are not matching as they are neither
+  // forward or backward compatible.
+  EXPECT_FALSE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(7, 0, 12),
+      /*loaded_version=*/CudnnVersion(6, 1, 14)));
+  EXPECT_FALSE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(8, 1, 15),
+      /*loaded_version=*/CudnnVersion(7, 0, 14)));
+
+  // Returns true if the loaded version is equal or higher because minor version
+  // are backward compatible with CuDNN version 7.
+  EXPECT_TRUE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(7, 0, 14),
+      /*loaded_version=*/CudnnVersion(7, 1, 14)));
+  EXPECT_TRUE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(7, 0, 14),
+      /*loaded_version=*/CudnnVersion(7, 1, 15)));
+  EXPECT_FALSE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(7, 1, 15),
+      /*loaded_version=*/CudnnVersion(7, 0, 14)));
+
+  // Returns false if minor versions are not matching for version 6. Before
+  // version 7, minor versions are also neither forward or backward compatible.
+  EXPECT_FALSE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(6, 0, 14),
+      /*loaded_version=*/CudnnVersion(6, 1, 15)));
+  EXPECT_FALSE(IsSourceCompatibleWithCudnnLibrary(
+      /*source_version=*/CudnnVersion(6, 1, 14),
+      /*loaded_version=*/CudnnVersion(6, 0, 14)));
+}
+
+}  // namespace
+}  // namespace cuda
+}  // namespace gputools
+}  // namespace perftools
-- 
GitLab


From 2b41d75654012f917cda1b54aee090d73086ab84 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 18:54:09 -0700
Subject: [PATCH 653/960] [XLA] Redesign: implement GetComputationStats.

PiperOrigin-RevId: 190871262
---
 tensorflow/compiler/xla/client/client.cc   | 47 ++++++++++++++++++++--
 tensorflow/compiler/xla/client/client.h    |  2 +
 tensorflow/compiler/xla/service/service.cc | 20 ++++++++-
 3 files changed, 65 insertions(+), 4 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index a857c4ff0b..c4c8894374 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -276,7 +276,12 @@ StatusOr<std::unique_ptr<GlobalData>> Client::Execute(
 
   if (execution_profile != nullptr) {
     *execution_profile = response.profile();
-    // TODO(b/74197823): Get execution stats for the graph and VLOG(1) them.
+    if (VLOG_IS_ON(1)) {
+      TF_ASSIGN_OR_RETURN(
+          auto execution_stats,
+          ExecutionStatsAsString(computation, response.profile()));
+      VLOG(1) << execution_stats;
+    }
   }
 
   return MakeUnique<GlobalData>(stub_, response.output());
@@ -402,8 +407,22 @@ StatusOr<ComputationStats> Client::GetComputationStats(
 StatusOr<ComputationStats> Client::GetComputationStats(
     const XlaComputation& computation,
     const DebugOptions& debug_options) const {
-  return Unimplemented(
-      "GetComputationStats is not yet implemented for XlaComputation");
+  ComputationGraphStatsRequest request;
+
+  // TODO(b/74197823): Find a way to avoid the copy of the hlo proto.
+  *request.mutable_computation() = computation.proto();
+  *request.mutable_debug_options() = debug_options;
+  ComputationStatsResponse response;
+
+  VLOG(1) << "making computation graph stats request";
+  Status s = stub_->GetComputationGraphStats(&request, &response);
+  VLOG(1) << "done with request";
+
+  if (!s.ok()) {
+    return s;
+  }
+  CHECK(response.has_stats());
+  return response.stats();
 }
 
 StatusOr<std::unique_ptr<ProgramShape>> Client::GetComputationShape(
@@ -467,6 +486,28 @@ StatusOr<string> Client::ExecutionStatsAsString(
   return string("[Execution Statistics] not available.");
 }
 
+StatusOr<string> Client::ExecutionStatsAsString(
+    const XlaComputation& computation, const ExecutionProfile& profile) {
+  TF_ASSIGN_OR_RETURN(
+      auto computation_stats,
+      GetComputationStats(computation,
+                          legacy_flags::GetDebugOptionsFromFlags()));
+  int64 total_flops =
+      computation_stats.flop_count() + computation_stats.transcendental_count();
+  if (profile.compute_time_ns() > 0) {
+    int64 nanoseconds = profile.compute_time_ns();
+    int64 cycle_count = profile.compute_cycle_count();
+    double gflops = total_flops / nanoseconds;
+    return tensorflow::strings::StrCat(
+        "[Execution Statistics] flop count: ", computation_stats.flop_count(),
+        ", transcendental count: ", computation_stats.transcendental_count(),
+        ", compute execution time: ", nanoseconds, " nsec",
+        ", compute cycles: ", cycle_count, ", performance: ", gflops,
+        "gflop/s");
+  }
+  return string("[Execution Statistics] not available.");
+}
+
 StatusOr<ChannelHandle> Client::CreateChannelHandle() {
   CreateChannelHandleRequest request;
   CreateChannelHandleResponse response;
diff --git a/tensorflow/compiler/xla/client/client.h b/tensorflow/compiler/xla/client/client.h
index 226b788d54..05d707dab1 100644
--- a/tensorflow/compiler/xla/client/client.h
+++ b/tensorflow/compiler/xla/client/client.h
@@ -241,6 +241,8 @@ class Client {
   // ExecutionProfile returned from an execution of the computation.
   StatusOr<string> ExecutionStatsAsString(const Computation& computation,
                                           const ExecutionProfile& profile);
+  StatusOr<string> ExecutionStatsAsString(const XlaComputation& computation,
+                                          const ExecutionProfile& profile);
 
   ServiceInterface* stub_;  // Stub that this client is connected on.
 
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index af05e3f516..ca8071b7bb 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -1452,7 +1452,25 @@ tensorflow::Status Service::GetComputationStats(
 
 tensorflow::Status Service::GetComputationGraphStats(
     const ComputationGraphStatsRequest* arg, ComputationStatsResponse* result) {
-  return Unimplemented("get-computation-graph-stats is not yet implemented");
+  HloModuleConfig config;
+  config.set_debug_options(arg->debug_options());
+  TF_ASSIGN_OR_RETURN(std::unique_ptr<HloModule> module,
+                      HloModule::CreateFromProto(arg->computation(), config));
+
+  hlo_graph_dumper::MaybeDumpHloModule(*module,
+                                       "computation statistics subject");
+
+  // Run HLO analysis to get the computation statistics.
+  HloCostAnalysis analysis(
+      execute_backend_->compiler()->ShapeSizeBytesFunction());
+
+  TF_RETURN_IF_ERROR(module->entry_computation()->Accept(&analysis));
+
+  ComputationStats stats;
+  stats.set_flop_count(analysis.flop_count());
+  stats.set_transcendental_count(analysis.transcendental_count());
+  *result->mutable_stats() = stats;
+  return tensorflow::Status::OK();
 }
 
 template <typename RequestT, typename ResponseT>
-- 
GitLab


From 3e51f9ede54bc61a8d4f7797992ab78140467d08 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Wed, 28 Mar 2018 18:59:13 -0700
Subject: [PATCH 654/960] Default to disable including the coordinator in the
 job

---
 .../cluster_resolver/python/training/tpu_cluster_resolver.py    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
index 300b19733e..95c5c920aa 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
@@ -73,7 +73,7 @@ class TPUClusterResolver(ClusterResolver):
                zone=None,
                project=None,
                job_name='worker',
-               coordinator_name='coordinator',
+               coordinator_name=None,
                coordinator_address=None,
                credentials='default',
                service=None):
-- 
GitLab


From 991e205a78f67ce21b0918613a45cfd7c3e348fd Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Thu, 29 Mar 2018 10:05:43 +0800
Subject: [PATCH 655/960] Fix the incorect format of math equation in
 factorization_ops (#18054)

* Fix the incorect format of math equation in factorization_ops

* Fix minor intent format

* Fix pylint issues

* Fix serveral minor intent
---
 .../python/ops/factorization_ops.py           | 81 ++++++++++---------
 1 file changed, 41 insertions(+), 40 deletions(-)

diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
index 8e0ed1d80e..3f3e3e0f25 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
@@ -51,9 +51,9 @@ class WALSModel(object):
   r"""A model for Weighted Alternating Least Squares matrix factorization.
 
   It minimizes the following loss function over U, V:
-   \\(
-   \|\sqrt W \odot (A - U V^T) \|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2)
-   )\\
+  $$
+   \|\sqrt W \odot (A - U V^T)\|_F^2 + \lambda (\|U\|_F^2 + \|V\|_F^2)
+  $$
     where,
     A: input matrix,
     W: weight matrix. Note that the (element-wise) square root of the weights
@@ -61,12 +61,12 @@ class WALSModel(object):
     U, V: row_factors and column_factors matrices,
     \\(\lambda)\\: regularization.
   Also we assume that W is of the following special form:
-  \\( W_{ij} = W_0 + R_i * C_j )\\  if \\(A_{ij} \ne 0)\\,
-  \\(W_{ij} = W_0)\\ otherwise.
+  \\( W_{ij} = W_0 + R_i * C_j \\)  if \\(A_{ij} \ne 0\\),
+  \\(W_{ij} = W_0\\) otherwise.
   where,
-  \\(W_0)\\: unobserved_weight,
-  \\(R_i)\\: row_weights,
-  \\(C_j)\\: col_weights.
+  \\(W_0\\): unobserved_weight,
+  \\(R_i\\): row_weights,
+  \\(C_j\\): col_weights.
 
   Note that the current implementation supports two operation modes: The default
   mode is for the condition where row_factors and col_factors can individually
@@ -82,14 +82,15 @@ class WALSModel(object):
   normalized as follows:
     _, _, unregularized_loss, regularization, sum_weights =
         update_row_factors(sp_input)
-  if sp_input contains the rows {A_i, i \in I}, and the input matrix A has n
-  total rows, then the minibatch loss = unregularized_loss + regularization is
-   \\(
+  if sp_input contains the rows \\({A_i, i \in I}\\), and the input matrix A
+  has n total rows, then the minibatch loss = unregularized_loss +
+  regularization is
+   $$
    (\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 + \lambda \|U_I\|_F^2) * n / |I| +
    \lambda \|V\|_F^2
-   )\\
+   $$
   The sum_weights tensor contains the normalized sum of weights
-  sum(W_I) * n / |I|.
+  \\(sum(W_I) * n / |I|\\).
 
   A typical usage example (pseudocode):
 
@@ -217,13 +218,13 @@ class WALSModel(object):
         - When set to None, w_ij = unobserved_weight, which simplifies to ALS.
         Note that col_weights must also be set to "None" in this case.
         - If it is a list of lists of non-negative real numbers, it needs to be
-        in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of
-        inner lists matching the number of row factor shards and the elements in
-        each inner list are the weights for the rows of the corresponding row
-        factor shard. In this case,  w_ij = unobserved_weight +
-                                            row_weights[i] * col_weights[j].
+        in the form of \\([[w_0, w_1, ...], [w_k, ... ], [...]]\\), with the
+        number of inner lists matching the number of row factor shards and the
+        elements in each inner list are the weights for the rows of the
+        corresponding row factor shard. In this case,  \\(w_ij\\) =
+        unobserved_weight + row_weights[i] * col_weights[j].
         - If this is a single non-negative real number, this value is used for
-        all row weights and w_ij = unobserved_weight + row_weights *
+        all row weights and \\(w_ij\\) = unobserved_weight + row_weights *
                                    col_weights[j].
         Note that it is allowed to have row_weights as a list while col_weights
         a single number or vice versa.
@@ -665,18 +666,18 @@ class WALSModel(object):
         factors.
       unregularized_loss: A tensor (scalar) that contains the normalized
         minibatch loss corresponding to sp_input, without the regularization
-        term. If sp_input contains the rows {A_{i, :}, i \in I}, and the input
-        matrix A has n total rows, then the unregularized loss is:
-        (\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 * n / |I|
+        term. If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the
+        input matrix A has n total rows, then the unregularized loss is:
+        \\(\|\sqrt W_I \odot (A_I - U_I V^T)\|_F^2 * n / |I|\\)
         The total loss is unregularized_loss + regularization.
       regularization: A tensor (scalar) that contains the normalized
         regularization term for the minibatch loss corresponding to sp_input.
-        If sp_input contains the rows {A_{i, :}, i \in I}, and the input matrix
-        A has n total rows, then the regularization term is:
-        \lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2.
+        If sp_input contains the rows \\({A_{i, :}, i \in I}\\), and the input
+        matrix A has n total rows, then the regularization term is:
+        \\(\lambda \|U_I\|_F^2) * n / |I| + \lambda \|V\|_F^2\\).
       sum_weights: The sum of the weights W_I corresponding to sp_input,
-        normalized by a factor of n / |I|. The root weighted squared error is:
-        \sqrt(unregularized_loss / sum_weights).
+        normalized by a factor of \\(n / |I|\\). The root weighted squared
+        error is: \sqrt(unregularized_loss / sum_weights).
     """
     return self._process_input_helper(
         True, sp_input=sp_input, transpose_input=transpose_input)
@@ -698,18 +699,18 @@ class WALSModel(object):
         factors.
       unregularized_loss: A tensor (scalar) that contains the normalized
         minibatch loss corresponding to sp_input, without the regularization
-        term. If sp_input contains the columns {A_{:, j}, j \in J}, and the
-        input matrix A has m total columns, then the unregularized loss is:
-        (\|\sqrt W_J \odot (A_J - U V_J^T)\|_F^2 * m / |I|
+        term. If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and
+        the input matrix A has m total columns, then the unregularized loss is:
+        \\(\|\sqrt W_J \odot (A_J - U V_J^T)\|_F^2 * m / |I|\\)
         The total loss is unregularized_loss + regularization.
       regularization: A tensor (scalar) that contains the normalized
         regularization term for the minibatch loss corresponding to sp_input.
-        If sp_input contains the columns {A_{:, j}, j \in J}, and the input
-        matrix A has m total columns, then the regularization term is:
-        \lambda \|V_J\|_F^2) * m / |J| + \lambda \|U\|_F^2.
+        If sp_input contains the columns \\({A_{:, j}, j \in J}\\), and the
+        input matrix A has m total columns, then the regularization term is:
+        \\(\lambda \|V_J\|_F^2) * m / |J| + \lambda \|U\|_F^2\\).
       sum_weights: The sum of the weights W_J corresponding to sp_input,
-        normalized by a factor of m / |J|. The root weighted squared error is:
-        \sqrt(unregularized_loss / sum_weights).
+        normalized by a factor of \\(m / |J|\\). The root weighted squared
+        error is: \sqrt(unregularized_loss / sum_weights).
     """
     return self._process_input_helper(
         False, sp_input=sp_input, transpose_input=transpose_input)
@@ -720,8 +721,8 @@ class WALSModel(object):
                           projection_weights=None):
     """Projects the row factors.
 
-    This computes the row embedding u_i for an observed row a_i by solving
-    one iteration of the update equations.
+    This computes the row embedding \\(u_i\\) for an observed row \\(a_i\\) by
+    solving one iteration of the update equations.
 
     Args:
       sp_input: A SparseTensor representing a set of rows. Please note that the
@@ -753,8 +754,8 @@ class WALSModel(object):
                           projection_weights=None):
     """Projects the column factors.
 
-    This computes the column embedding v_j for an observed column a_j by solving
-    one iteration of the update equations.
+    This computes the column embedding \\(v_j\\) for an observed column
+    \\(a_j\\) by solving one iteration of the update equations.
 
     Args:
       sp_input: A SparseTensor representing a set of columns. Please note that
@@ -938,7 +939,7 @@ class WALSModel(object):
     loss_sp_input = (sparse_ops.sparse_transpose(new_sp_input)
                      if transpose_input else new_sp_input)
     # sp_approx is the low rank estimate of the input matrix, formed by
-    # computing the product <u_i, v_j> for (i, j) in loss_sp_input.indices.
+    # computing the product <\\(u_i, v_j\\)> for (i, j) in loss_sp_input.indices.
     sp_approx_vals = gen_factorization_ops.masked_matmul(
         new_left_values,
         right,
-- 
GitLab


From a5a90e6b55c19bd14d5effa5cb1695ddbe31026f Mon Sep 17 00:00:00 2001
From: Suharsh Sivakumar <suharshs@google.com>
Date: Wed, 28 Mar 2018 19:21:08 -0700
Subject: [PATCH 656/960] Relax limitations on rerouting graph outputs.

- Allow multiple outputs of output_tensors in fold_batch_norms.
- Allow duplicate consumers in quantize.
- I also quick a fix issue for matching final layers that have batch norm.

PiperOrigin-RevId: 190873003
---
 .../quantize/python/fold_batch_norms.py        |  6 +++---
 tensorflow/contrib/quantize/python/quantize.py | 18 ++++++++++++------
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/quantize/python/fold_batch_norms.py b/tensorflow/contrib/quantize/python/fold_batch_norms.py
index 5750be6f4c..4a8f8a04cc 100644
--- a/tensorflow/contrib/quantize/python/fold_batch_norms.py
+++ b/tensorflow/contrib/quantize/python/fold_batch_norms.py
@@ -134,9 +134,9 @@ def _FoldFusedBatchNorms(graph, is_training, freeze_batch_norm_delay):
 
       nodes_modified_count = graph_editor.reroute_ts(bias_add_tensor,
                                                      match.output_tensor)
-      if nodes_modified_count != 1:
-        raise ValueError(
-            'Unexpected inputs to op: %s' % match.output_tensor.name)
+      if nodes_modified_count == 0:
+        raise ValueError('Folding batch norms failed, %s had no outputs.' %
+                         match.output_tensor.name)
 
 
 def _FindFusedBatchNorms(graph):
diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 019d123a68..2889016a84 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -305,7 +305,8 @@ def _FindLayersToQuantize(graph):
   # the output of the final BiasAdd must be quantized. So we treat the BiasAdd
   # as the 'activation_op' in the _LayerMatch, to ensure that it's output is
   # quantized.
-  final_layer_matcher = graph_matcher.GraphMatcher(bias_add_pattern)
+  final_layer_matcher = graph_matcher.GraphMatcher(
+      graph_matcher.OneofPattern([bias_add_pattern, folded_bias_add_pattern]))
   for match_result in final_layer_matcher.match_graph(graph):
     layer_op = match_result.get_op(layer_pattern)
     weight_tensor = match_result.get_tensor(weight_identity_pattern)
@@ -463,11 +464,16 @@ def _InsertQuantOp(context,
         lambda: inputs,
         name=name_prefix + '/delayed_quant')
 
-  nodes_modified_count = graph_editor.reroute_ts(
-      [quant], [inputs], can_modify=consumers)
-  if nodes_modified_count != len(consumers):
-    raise ValueError('Some inputs not quantized for ops: [%s]' % ', '.join(
-        [consumer.name for consumer in consumers]))
+  if consumers:
+    tensors_modified_count = graph_editor.reroute_ts(
+        [quant], [inputs], can_modify=consumers)
+    # Some operations can have multiple output tensors going to the same
+    # consumer. Since consumers is a set, we need to ensure that
+    # tensors_modified_count is greater than or equal to the length of the set
+    # of consumers.
+    if tensors_modified_count < len(consumers):
+      raise ValueError('No inputs quantized for ops: [%s]' % ', '.join(
+          [consumer.name for consumer in consumers]))
 
 
 def _GetContextFromOp(op):
-- 
GitLab


From aef7d8b3e877924973e3d8d8e6266ba7b8322a66 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Wed, 28 Mar 2018 19:27:36 -0700
Subject: [PATCH 657/960] Fix the test

---
 .../python/training/tpu_cluster_resolver_test.py          | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
index 48c3f6bb4f..e1e3e6867a 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
@@ -117,7 +117,8 @@ class TPUClusterResolverTest(test.TestCase):
         zone=None,
         tpu=['test-tpu-1'],
         credentials=None,
-        service=self.mock_service_client(tpu_map=tpu_map))
+        service=self.mock_service_client(tpu_map=tpu_map),
+        coordinator_name='coordinator')
 
     actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
     expected_proto = """
@@ -170,6 +171,7 @@ class TPUClusterResolverTest(test.TestCase):
         project='test-project',
         zone='us-central1-c',
         tpu=['test-tpu-1'],
+        coordinator_name='coordinator',
         coordinator_address='10.128.1.5:10203',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
@@ -196,6 +198,7 @@ class TPUClusterResolverTest(test.TestCase):
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
+        coordinator_name='coordinator',
         coordinator_address='10.128.1.5:10203',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
@@ -239,7 +242,8 @@ class TPUClusterResolverTest(test.TestCase):
     tpu_cluster_resolver = TPUClusterResolver(
         tpu='test-tpu-1',
         credentials=None,
-        service=self.mock_service_client(tpu_map=tpu_map))
+        service=self.mock_service_client(tpu_map=tpu_map),
+        coordinator_name='coordinator')
 
     actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
     expected_proto = """
-- 
GitLab


From 789e442513e85ab1caeb1e03997b0aafa3cd76d7 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Wed, 28 Mar 2018 20:44:51 -0700
Subject: [PATCH 658/960] [tf.data] Maintain a reference on the
 FunctionBufferingResource while a get-next operation is active.

Previously, the reference count on a FunctionBufferingResource could drop to 0 and it could be deleted (e.g. by a DestroyResourceOp) while a get-next operation is active on it. This would lead to use-after-free errors.

PiperOrigin-RevId: 190878208
---
 tensorflow/contrib/data/kernels/prefetching_kernels.cc | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index f51570db85..2afb8dbbf4 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -374,25 +374,27 @@ class FunctionBufferingResourceGetNextOp : public AsyncOpKernel {
     OP_REQUIRES_OK_ASYNC(
         ctx, LookupResource<FunctionBufferingResource>(ctx, handle, &buffer),
         done);
-    core::ScopedUnref s(buffer);
 
     if (buffer->Finished()) {
+      buffer->Unref();
       ctx->SetStatus(errors::OutOfRange("end_of_sequence"));
       done();
       return;
     }
 
     FunctionBufferCallback callback =
-        [ctx, done](const BufferElement& buffer_element) {
+        [ctx, buffer, done](const BufferElement& buffer_element) {
           Status s = buffer_element.status;
           if (!s.ok()) {
             ctx->SetStatus(s);
+            buffer->Unref();
             done();
             return;
           }
           for (size_t i = 0; i < buffer_element.value.size(); ++i) {
             ctx->set_output(i, buffer_element.value[i]);
           }
+          buffer->Unref();
           done();
         };
     buffer->MaybeGet(std::move(callback));
-- 
GitLab


From bb582f1b6fad474bc446c78a6683247a8eb6048e Mon Sep 17 00:00:00 2001
From: Martin Wicke <wicke@google.com>
Date: Wed, 28 Mar 2018 20:46:14 -0700
Subject: [PATCH 659/960] Remove all_opensource_files. It's not needed any
 more.

PiperOrigin-RevId: 190878279
---
 tensorflow/BUILD                              | 298 ------------------
 tensorflow/c/BUILD                            |  15 -
 tensorflow/cc/BUILD                           |  12 -
 tensorflow/cc/saved_model/BUILD               |  15 -
 tensorflow/cc/saved_model/python/BUILD        |  12 -
 tensorflow/cc/tools/BUILD                     |  15 -
 tensorflow/compiler/aot/BUILD                 |  14 -
 tensorflow/compiler/aot/tests/BUILD           |  14 -
 tensorflow/compiler/jit/BUILD                 |  14 -
 tensorflow/compiler/jit/graphcycles/BUILD     |  14 -
 tensorflow/compiler/jit/kernels/BUILD         |  14 -
 tensorflow/compiler/jit/legacy_flags/BUILD    |  14 -
 tensorflow/compiler/jit/ops/BUILD             |  14 -
 tensorflow/compiler/plugin/BUILD              |  14 -
 tensorflow/compiler/tests/BUILD               |  14 -
 tensorflow/compiler/tf2xla/BUILD              |  14 -
 tensorflow/compiler/tf2xla/cc/BUILD           |  14 -
 tensorflow/compiler/tf2xla/kernels/BUILD      |  14 -
 tensorflow/compiler/tf2xla/lib/BUILD          |  14 -
 tensorflow/compiler/tf2xla/ops/BUILD          |  14 -
 tensorflow/compiler/xla/BUILD                 |  12 -
 tensorflow/compiler/xla/client/BUILD          |  14 -
 tensorflow/compiler/xla/client/lib/BUILD      |  14 -
 .../compiler/xla/client/xla_client/BUILD      |  14 -
 tensorflow/compiler/xla/legacy_flags/BUILD    |  14 -
 tensorflow/compiler/xla/python/BUILD          |  12 -
 tensorflow/compiler/xla/service/BUILD         |  14 -
 tensorflow/compiler/xla/service/cpu/BUILD     |  14 -
 tensorflow/compiler/xla/service/gpu/BUILD     |  14 -
 .../xla/service/gpu/llvm_gpu_backend/BUILD    |  14 -
 .../compiler/xla/service/interpreter/BUILD    |  11 -
 tensorflow/compiler/xla/service/llvm_ir/BUILD |  14 -
 tensorflow/compiler/xla/tests/BUILD           |  14 -
 tensorflow/compiler/xla/tools/BUILD           |  14 -
 tensorflow/compiler/xla/tools/parser/BUILD    |  14 -
 tensorflow/contrib/BUILD                      |  12 -
 tensorflow/contrib/all_reduce/BUILD           |  13 -
 tensorflow/contrib/android/BUILD              |  14 -
 tensorflow/contrib/batching/BUILD             |  11 -
 tensorflow/contrib/batching/test_util/BUILD   |  11 -
 tensorflow/contrib/batching/util/BUILD        |  12 -
 tensorflow/contrib/bayesflow/BUILD            |  12 -
 tensorflow/contrib/boosted_trees/BUILD        |   9 -
 .../boosted_trees/estimator_batch/BUILD       |   9 -
 tensorflow/contrib/boosted_trees/lib/BUILD    |  11 -
 tensorflow/contrib/boosted_trees/proto/BUILD  |  11 -
 .../contrib/boosted_trees/resources/BUILD     |  11 -
 tensorflow/contrib/cloud/BUILD                |  12 -
 tensorflow/contrib/cloud/kernels/BUILD        |  14 -
 tensorflow/contrib/cluster_resolver/BUILD     |  13 -
 tensorflow/contrib/coder/BUILD                |  11 -
 tensorflow/contrib/compiler/BUILD             |  12 -
 tensorflow/contrib/copy_graph/BUILD           |  12 -
 tensorflow/contrib/crf/BUILD                  |  12 -
 tensorflow/contrib/cudnn_rnn/BUILD            |  12 -
 tensorflow/contrib/data/BUILD                 |  14 -
 tensorflow/contrib/data/kernels/BUILD         |  11 -
 .../contrib/data/python/kernel_tests/BUILD    |  14 -
 tensorflow/contrib/data/python/ops/BUILD      |  12 -
 tensorflow/contrib/decision_trees/proto/BUILD |   8 -
 tensorflow/contrib/deprecated/BUILD           |  12 -
 tensorflow/contrib/distributions/BUILD        |  12 -
 tensorflow/contrib/eager/proto/BUILD          |  11 -
 tensorflow/contrib/eager/python/BUILD         |  13 -
 tensorflow/contrib/estimator/BUILD            |  12 -
 tensorflow/contrib/factorization/BUILD        |  13 -
 .../contrib/factorization/examples/BUILD      |  11 -
 .../contrib/factorization/kernels/BUILD       |  11 -
 tensorflow/contrib/feature_column/BUILD       |  12 -
 tensorflow/contrib/ffmpeg/BUILD               |  12 -
 tensorflow/contrib/ffmpeg/default/BUILD       |  12 -
 tensorflow/contrib/framework/BUILD            |  12 -
 tensorflow/contrib/fused_conv/BUILD           |  12 -
 tensorflow/contrib/gan/BUILD                  |  12 -
 tensorflow/contrib/gdr/BUILD                  |  12 -
 tensorflow/contrib/graph_editor/BUILD         |  12 -
 tensorflow/contrib/grid_rnn/BUILD             |  12 -
 tensorflow/contrib/hooks/BUILD                |  11 -
 .../contrib/hvx/clock_cycle_profiling/BUILD   |  12 -
 .../contrib/hvx/hvx_ops_support_checker/BUILD |  11 -
 tensorflow/contrib/image/BUILD                |  12 -
 tensorflow/contrib/input_pipeline/BUILD       |  11 -
 .../contrib/input_pipeline/kernels/BUILD      |  11 -
 tensorflow/contrib/integrate/BUILD            |  11 -
 tensorflow/contrib/kafka/BUILD                |  14 -
 tensorflow/contrib/keras/BUILD                |  12 -
 tensorflow/contrib/kernel_methods/BUILD       |  12 -
 tensorflow/contrib/kfac/BUILD                 |  12 -
 tensorflow/contrib/kfac/examples/BUILD        |  12 -
 tensorflow/contrib/kfac/examples/tests/BUILD  |  12 -
 .../contrib/kfac/python/kernel_tests/BUILD    |  12 -
 tensorflow/contrib/kfac/python/ops/BUILD      |  12 -
 tensorflow/contrib/labeled_tensor/BUILD       |  11 -
 tensorflow/contrib/layers/BUILD               |  12 -
 tensorflow/contrib/layers/kernels/BUILD       |  11 -
 tensorflow/contrib/learn/BUILD                |  12 -
 .../contrib/learn/python/learn/datasets/BUILD |  12 -
 tensorflow/contrib/legacy_seq2seq/BUILD       |  12 -
 tensorflow/contrib/libsvm/BUILD               |  12 -
 tensorflow/contrib/linalg/BUILD               |  12 -
 tensorflow/contrib/linear_optimizer/BUILD     |  11 -
 tensorflow/contrib/lite/BUILD                 |  15 -
 .../contrib/lite/examples/label_image/BUILD   |  12 -
 tensorflow/contrib/lite/java/BUILD            |  12 -
 .../contrib/lite/java/demo/app/src/main/BUILD |  12 -
 .../lite/java/demo/app/src/main/assets/BUILD  |  12 -
 .../contrib/lite/java/src/main/native/BUILD   |  12 -
 .../testhelper/java/org/tensorflow/lite/BUILD |  12 -
 tensorflow/contrib/lite/kernels/BUILD         |  12 -
 .../contrib/lite/kernels/internal/BUILD       |  12 -
 tensorflow/contrib/lite/models/BUILD          |  12 -
 .../contrib/lite/models/smartreply/BUILD      |  12 -
 tensorflow/contrib/lite/nnapi/BUILD           |  12 -
 tensorflow/contrib/lite/python/BUILD          |  12 -
 tensorflow/contrib/lite/schema/BUILD          |  12 -
 tensorflow/contrib/lite/testing/BUILD         |  12 -
 tensorflow/contrib/lite/toco/BUILD            |  12 -
 .../toco/graph_transformations/tests/BUILD    |  12 -
 tensorflow/contrib/lite/toco/python/BUILD     |  12 -
 .../lite/toco/tensorflow_graph_matching/BUILD |  12 -
 tensorflow/contrib/lite/toco/tflite/BUILD     |  12 -
 tensorflow/contrib/lite/tools/BUILD           |  12 -
 tensorflow/contrib/lookup/BUILD               |  12 -
 tensorflow/contrib/losses/BUILD               |  12 -
 tensorflow/contrib/makefile/BUILD             |   9 -
 tensorflow/contrib/memory_stats/BUILD         |  12 -
 tensorflow/contrib/meta_graph_transform/BUILD |  12 -
 tensorflow/contrib/metrics/BUILD              |  11 -
 tensorflow/contrib/model_pruning/BUILD        |  12 -
 .../model_pruning/examples/cifar10/BUILD      |  12 -
 tensorflow/contrib/mpi_collectives/BUILD      |  12 -
 tensorflow/contrib/nccl/BUILD                 |  12 -
 tensorflow/contrib/nearest_neighbor/BUILD     |  12 -
 tensorflow/contrib/nn/BUILD                   |  11 -
 tensorflow/contrib/opt/BUILD                  |  11 -
 tensorflow/contrib/periodic_resample/BUILD    |  12 -
 tensorflow/contrib/predictor/BUILD            |  12 -
 tensorflow/contrib/quantization/BUILD         |  12 -
 tensorflow/contrib/quantize/BUILD             |  12 -
 tensorflow/contrib/receptive_field/BUILD      |  12 -
 tensorflow/contrib/reduce_slice_ops/BUILD     |  12 -
 .../contrib/remote_fused_graph/pylib/BUILD    |  12 -
 tensorflow/contrib/resampler/BUILD            |  11 -
 tensorflow/contrib/rnn/BUILD                  |  13 -
 tensorflow/contrib/saved_model/BUILD          |  12 -
 .../contrib/saved_model/cc/saved_model/BUILD  |   6 -
 tensorflow/contrib/seq2seq/BUILD              |  12 -
 tensorflow/contrib/session_bundle/BUILD       |  12 -
 .../contrib/session_bundle/example/BUILD      |  13 -
 tensorflow/contrib/signal/BUILD               |  12 -
 tensorflow/contrib/slim/BUILD                 |  12 -
 .../contrib/slim/python/slim/data/BUILD       |  12 -
 .../contrib/slim/python/slim/nets/BUILD       |  12 -
 tensorflow/contrib/solvers/BUILD              |  13 -
 tensorflow/contrib/sparsemax/BUILD            |  12 -
 tensorflow/contrib/specs/BUILD                |  12 -
 tensorflow/contrib/staging/BUILD              |  12 -
 tensorflow/contrib/stat_summarizer/BUILD      |  12 -
 tensorflow/contrib/stateless/BUILD            |  12 -
 tensorflow/contrib/summary/BUILD              |  12 -
 tensorflow/contrib/tensor_forest/BUILD        |  14 -
 tensorflow/contrib/tensor_forest/hybrid/BUILD |  12 -
 .../contrib/tensor_forest/kernels/v4/BUILD    |   5 -
 tensorflow/contrib/tensor_forest/proto/BUILD  |   8 -
 tensorflow/contrib/tensorboard/BUILD          |  12 -
 tensorflow/contrib/tensorboard/db/BUILD       |   6 -
 tensorflow/contrib/tensorrt/BUILD             |  12 -
 tensorflow/contrib/testing/BUILD              |  12 -
 tensorflow/contrib/text/BUILD                 |  11 -
 tensorflow/contrib/tfprof/BUILD               |  12 -
 tensorflow/contrib/timeseries/BUILD           |  12 -
 tensorflow/contrib/timeseries/examples/BUILD  |  12 -
 .../timeseries/python/timeseries/BUILD        |  12 -
 .../timeseries/state_space_models/BUILD       |  12 -
 tensorflow/contrib/tpu/BUILD                  |  13 -
 tensorflow/contrib/tpu/profiler/BUILD         |  12 -
 tensorflow/contrib/tpu/proto/BUILD            |  11 -
 tensorflow/contrib/training/BUILD             |  12 -
 tensorflow/contrib/util/BUILD                 |  12 -
 tensorflow/contrib/verbs/BUILD                |  12 -
 tensorflow/core/BUILD                         |  14 +-
 tensorflow/core/api_def/BUILD                 |  12 -
 tensorflow/core/common_runtime/eager/BUILD    |  15 -
 tensorflow/core/debug/BUILD                   |  15 -
 tensorflow/core/distributed_runtime/BUILD     |  12 -
 tensorflow/core/distributed_runtime/rpc/BUILD |  12 -
 tensorflow/core/grappler/BUILD                |  12 -
 tensorflow/core/grappler/clusters/BUILD       |  12 -
 tensorflow/core/grappler/costs/BUILD          |  12 -
 tensorflow/core/grappler/inputs/BUILD         |  12 -
 tensorflow/core/grappler/optimizers/BUILD     |  12 -
 tensorflow/core/grappler/utils/BUILD          |  12 -
 tensorflow/core/kernels/BUILD                 |  12 -
 tensorflow/core/kernels/batching_util/BUILD   |  12 -
 tensorflow/core/kernels/data/BUILD            |  12 -
 tensorflow/core/kernels/data/sql/BUILD        |  12 -
 tensorflow/core/kernels/fuzzing/BUILD         |  12 -
 tensorflow/core/kernels/hexagon/BUILD         |  12 -
 tensorflow/core/kernels/neon/BUILD            |  12 -
 tensorflow/core/lib/db/BUILD                  |   6 -
 tensorflow/core/ops/compat/BUILD              |  15 -
 tensorflow/core/platform/cloud/BUILD          |  14 -
 .../core/platform/default/build_config/BUILD  |  12 -
 tensorflow/core/platform/hadoop/BUILD         |  12 -
 tensorflow/core/platform/s3/BUILD             |  12 -
 tensorflow/core/profiler/BUILD                |  15 -
 tensorflow/core/profiler/internal/BUILD       |  14 -
 .../core/profiler/internal/advisor/BUILD      |  15 -
 tensorflow/core/util/ctc/BUILD                |  12 -
 tensorflow/core/util/tensor_bundle/BUILD      |  15 -
 tensorflow/examples/adding_an_op/BUILD        |  12 -
 tensorflow/examples/android/BUILD             |  16 -
 tensorflow/examples/benchmark/BUILD           |   6 -
 .../examples/get_started/regression/BUILD     |  12 -
 .../examples/how_tos/reading_data/BUILD       |  12 -
 tensorflow/examples/image_retraining/BUILD    |  12 -
 tensorflow/examples/label_image/BUILD         |  16 +-
 tensorflow/examples/learn/BUILD               |  12 -
 tensorflow/examples/multibox_detector/BUILD   |  14 -
 tensorflow/examples/saved_model/BUILD         |  13 -
 tensorflow/examples/speech_commands/BUILD     |  12 -
 .../examples/tutorials/estimators/BUILD       |  12 -
 tensorflow/examples/tutorials/layers/BUILD    |  12 -
 tensorflow/examples/tutorials/mnist/BUILD     |  12 -
 tensorflow/examples/tutorials/monitors/BUILD  |  12 -
 tensorflow/examples/tutorials/word2vec/BUILD  |  11 -
 tensorflow/examples/wav_to_spectrogram/BUILD  |  14 -
 tensorflow/java/BUILD                         |  12 -
 tensorflow/python/BUILD                       |  12 -
 tensorflow/python/data/BUILD                  |  12 -
 tensorflow/python/data/kernel_tests/BUILD     |  12 -
 tensorflow/python/data/ops/BUILD              |  12 -
 tensorflow/python/data/util/BUILD             |  12 -
 tensorflow/python/debug/BUILD                 |  12 -
 tensorflow/python/eager/BUILD                 |  15 -
 tensorflow/python/estimator/BUILD             |  12 -
 tensorflow/python/feature_column/BUILD        |  12 -
 tensorflow/python/keras/BUILD                 |  12 -
 tensorflow/python/kernel_tests/BUILD          |  12 -
 .../python/kernel_tests/distributions/BUILD   |  12 -
 tensorflow/python/kernel_tests/linalg/BUILD   |  12 -
 tensorflow/python/kernel_tests/random/BUILD   |  12 -
 tensorflow/python/ops/distributions/BUILD     |  12 -
 tensorflow/python/ops/linalg/BUILD            |  12 -
 tensorflow/python/ops/losses/BUILD            |  12 -
 tensorflow/python/profiler/BUILD              |  15 -
 tensorflow/python/profiler/internal/BUILD     |  15 -
 tensorflow/python/saved_model/BUILD           |  12 -
 tensorflow/python/tools/BUILD                 |  14 -
 tensorflow/tools/api/generator/BUILD          |  12 -
 tensorflow/tools/api/golden/BUILD             |  12 -
 tensorflow/tools/api/lib/BUILD                |  12 -
 tensorflow/tools/api/tests/BUILD              |  12 -
 tensorflow/tools/benchmark/BUILD              |   9 -
 tensorflow/tools/build_info/BUILD             |  15 -
 tensorflow/tools/common/BUILD                 |  11 -
 tensorflow/tools/compatibility/BUILD          |  15 -
 tensorflow/tools/dist_test/server/BUILD       |  12 -
 tensorflow/tools/docker/BUILD                 |  12 -
 tensorflow/tools/docker/notebooks/BUILD       |  12 -
 tensorflow/tools/docs/BUILD                   |  11 -
 tensorflow/tools/git/BUILD                    |  15 -
 tensorflow/tools/graph_transforms/BUILD       |  11 -
 tensorflow/tools/mlpbtxt/BUILD                |  12 -
 tensorflow/tools/proto_text/BUILD             |  15 -
 tensorflow/tools/quantization/BUILD           |  12 -
 tensorflow/tools/test/BUILD                   |  12 -
 tensorflow/user_ops/BUILD                     |  12 -
 third_party/hadoop/BUILD                      |  12 -
 third_party/mpi/BUILD                         |  12 -
 third_party/sycl/BUILD                        |  12 -
 third_party/sycl/sycl/BUILD                   |  12 -
 272 files changed, 4 insertions(+), 3610 deletions(-)

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 6ab43638ba..0021b657d8 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -394,304 +394,6 @@ package_group(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
-filegroup(
-    name = "all_opensource_files",
-    data = [
-        ":all_files",
-        "//tensorflow/c:all_files",
-        "//tensorflow/cc:all_files",
-        "//tensorflow/cc/saved_model:all_files",
-        "//tensorflow/cc/saved_model/python:all_files",
-        "//tensorflow/cc/tools:all_files",
-        "//tensorflow/compiler/aot:all_files",
-        "//tensorflow/compiler/aot/tests:all_files",
-        "//tensorflow/compiler/jit:all_files",
-        "//tensorflow/compiler/jit/graphcycles:all_files",
-        "//tensorflow/compiler/jit/kernels:all_files",
-        "//tensorflow/compiler/jit/legacy_flags:all_files",
-        "//tensorflow/compiler/jit/ops:all_files",
-        "//tensorflow/compiler/plugin:all_files",
-        "//tensorflow/compiler/tests:all_files",
-        "//tensorflow/compiler/tf2xla:all_files",
-        "//tensorflow/compiler/tf2xla/cc:all_files",
-        "//tensorflow/compiler/tf2xla/kernels:all_files",
-        "//tensorflow/compiler/tf2xla/lib:all_files",
-        "//tensorflow/compiler/tf2xla/ops:all_files",
-        "//tensorflow/compiler/xla:all_files",
-        "//tensorflow/compiler/xla/client:all_files",
-        "//tensorflow/compiler/xla/client/lib:all_files",
-        "//tensorflow/compiler/xla/client/xla_client:all_files",
-        "//tensorflow/compiler/xla/legacy_flags:all_files",
-        "//tensorflow/compiler/xla/python:all_files",
-        "//tensorflow/compiler/xla/service:all_files",
-        "//tensorflow/compiler/xla/service/cpu:all_files",
-        "//tensorflow/compiler/xla/service/gpu:all_files",
-        "//tensorflow/compiler/xla/service/gpu/llvm_gpu_backend:all_files",
-        "//tensorflow/compiler/xla/service/interpreter:all_files",
-        "//tensorflow/compiler/xla/service/llvm_ir:all_files",
-        "//tensorflow/compiler/xla/tests:all_files",
-        "//tensorflow/compiler/xla/tools:all_files",
-        "//tensorflow/compiler/xla/tools/parser:all_files",
-        "//tensorflow/contrib:all_files",
-        "//tensorflow/contrib/all_reduce:all_files",
-        "//tensorflow/contrib/android:all_files",
-        "//tensorflow/contrib/autograph:all_files",
-        "//tensorflow/contrib/autograph/converters:all_files",
-        "//tensorflow/contrib/autograph/impl:all_files",
-        "//tensorflow/contrib/autograph/pyct:all_files",
-        "//tensorflow/contrib/autograph/pyct/static_analysis:all_files",
-        "//tensorflow/contrib/autograph/utils:all_files",
-        "//tensorflow/contrib/batching:all_files",
-        "//tensorflow/contrib/bayesflow:all_files",
-        "//tensorflow/contrib/boosted_trees:all_files",
-        "//tensorflow/contrib/boosted_trees/estimator_batch:all_files",
-        "//tensorflow/contrib/boosted_trees/lib:all_files",
-        "//tensorflow/contrib/boosted_trees/proto:all_files",
-        "//tensorflow/contrib/boosted_trees/resources:all_files",
-        "//tensorflow/contrib/cloud:all_files",
-        "//tensorflow/contrib/cloud/kernels:all_files",
-        "//tensorflow/contrib/cluster_resolver:all_files",
-        "//tensorflow/contrib/coder:all_files",
-        "//tensorflow/contrib/compiler:all_files",
-        "//tensorflow/contrib/copy_graph:all_files",
-        "//tensorflow/contrib/crf:all_files",
-        "//tensorflow/contrib/cudnn_rnn:all_files",
-        "//tensorflow/contrib/data:all_files",
-        "//tensorflow/contrib/data/kernels:all_files",
-        "//tensorflow/contrib/data/python/kernel_tests:all_files",
-        "//tensorflow/contrib/data/python/ops:all_files",
-        "//tensorflow/contrib/decision_trees/proto:all_files",
-        "//tensorflow/contrib/deprecated:all_files",
-        "//tensorflow/contrib/distributions:all_files",
-        "//tensorflow/contrib/eager/proto:all_files",
-        "//tensorflow/contrib/eager/python:all_files",
-        "//tensorflow/contrib/estimator:all_files",
-        "//tensorflow/contrib/factorization:all_files",
-        "//tensorflow/contrib/factorization/examples:all_files",
-        "//tensorflow/contrib/factorization/kernels:all_files",
-        "//tensorflow/contrib/feature_column:all_files",
-        "//tensorflow/contrib/ffmpeg:all_files",
-        "//tensorflow/contrib/ffmpeg/default:all_files",
-        "//tensorflow/contrib/framework:all_files",
-        "//tensorflow/contrib/fused_conv:all_files",
-        "//tensorflow/contrib/gan:all_files",
-        "//tensorflow/contrib/gdr:all_files",
-        "//tensorflow/contrib/graph_editor:all_files",
-        "//tensorflow/contrib/grid_rnn:all_files",
-        "//tensorflow/contrib/hooks:all_files",
-        "//tensorflow/contrib/hvx/clock_cycle_profiling:all_files",
-        "//tensorflow/contrib/hvx/hvx_ops_support_checker:all_files",
-        "//tensorflow/contrib/image:all_files",
-        "//tensorflow/contrib/input_pipeline:all_files",
-        "//tensorflow/contrib/input_pipeline/kernels:all_files",
-        "//tensorflow/contrib/integrate:all_files",
-        "//tensorflow/contrib/keras:all_files",
-        "//tensorflow/contrib/kernel_methods:all_files",
-        "//tensorflow/contrib/kfac:all_files",
-        "//tensorflow/contrib/kfac/examples:all_files",
-        "//tensorflow/contrib/kfac/examples/tests:all_files",
-        "//tensorflow/contrib/kfac/python/kernel_tests:all_files",
-        "//tensorflow/contrib/kfac/python/ops:all_files",
-        "//tensorflow/contrib/labeled_tensor:all_files",
-        "//tensorflow/contrib/layers:all_files",
-        "//tensorflow/contrib/layers/kernels:all_files",
-        "//tensorflow/contrib/learn:all_files",
-        "//tensorflow/contrib/learn/python/learn/datasets:all_files",
-        "//tensorflow/contrib/legacy_seq2seq:all_files",
-        "//tensorflow/contrib/libsvm:all_files",
-        "//tensorflow/contrib/linalg:all_files",
-        "//tensorflow/contrib/linear_optimizer:all_files",
-        "//tensorflow/contrib/lite:all_files",
-        "//tensorflow/contrib/lite/java:all_files",
-        "//tensorflow/contrib/lite/java/demo/app/src/main:all_files",
-        "//tensorflow/contrib/lite/java/demo/app/src/main/assets:all_files",
-        "//tensorflow/contrib/lite/java/src/main/native:all_files",
-        "//tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite:all_files",
-        "//tensorflow/contrib/lite/kernels:all_files",
-        "//tensorflow/contrib/lite/kernels/internal:all_files",
-        "//tensorflow/contrib/lite/models/smartreply:all_files",
-        "//tensorflow/contrib/lite/nnapi:all_files",
-        "//tensorflow/contrib/lite/python:all_files",
-        "//tensorflow/contrib/lite/schema:all_files",
-        "//tensorflow/contrib/lite/testing:all_files",
-        "//tensorflow/contrib/lite/toco:all_files",
-        "//tensorflow/contrib/lite/toco/graph_transformations/tests:all_files",
-        "//tensorflow/contrib/lite/toco/python:all_files",
-        "//tensorflow/contrib/lite/toco/tensorflow_graph_matching:all_files",
-        "//tensorflow/contrib/lite/toco/tflite:all_files",
-        "//tensorflow/contrib/lite/tools:all_files",
-        "//tensorflow/contrib/lookup:all_files",
-        "//tensorflow/contrib/losses:all_files",
-        "//tensorflow/contrib/makefile:all_files",
-        "//tensorflow/contrib/memory_stats:all_files",
-        "//tensorflow/contrib/meta_graph_transform:all_files",
-        "//tensorflow/contrib/metrics:all_files",
-        "//tensorflow/contrib/model_pruning:all_files",
-        "//tensorflow/contrib/model_pruning/examples/cifar10:all_files",
-        "//tensorflow/contrib/nccl:all_files",
-        "//tensorflow/contrib/nearest_neighbor:all_files",
-        "//tensorflow/contrib/nn:all_files",
-        "//tensorflow/contrib/opt:all_files",
-        "//tensorflow/contrib/periodic_resample:all_files",
-        "//tensorflow/contrib/predictor:all_files",
-        "//tensorflow/contrib/quantize:all_files",
-        "//tensorflow/contrib/receptive_field:all_files",
-        "//tensorflow/contrib/reduce_slice_ops:all_files",
-        "//tensorflow/contrib/remote_fused_graph/pylib:all_files",
-        "//tensorflow/contrib/resampler:all_files",
-        "//tensorflow/contrib/rnn:all_files",
-        "//tensorflow/contrib/saved_model:all_files",
-        "//tensorflow/contrib/saved_model/cc/saved_model:all_files",
-        "//tensorflow/contrib/seq2seq:all_files",
-        "//tensorflow/contrib/session_bundle:all_files",
-        "//tensorflow/contrib/session_bundle/example:all_files",
-        "//tensorflow/contrib/signal:all_files",
-        "//tensorflow/contrib/slim:all_files",
-        "//tensorflow/contrib/slim/python/slim/data:all_files",
-        "//tensorflow/contrib/slim/python/slim/nets:all_files",
-        "//tensorflow/contrib/solvers:all_files",
-        "//tensorflow/contrib/sparsemax:all_files",
-        "//tensorflow/contrib/specs:all_files",
-        "//tensorflow/contrib/staging:all_files",
-        "//tensorflow/contrib/stat_summarizer:all_files",
-        "//tensorflow/contrib/stateless:all_files",
-        "//tensorflow/contrib/summary:all_files",
-        "//tensorflow/contrib/tensor_forest:all_files",
-        "//tensorflow/contrib/tensor_forest/hybrid:all_files",
-        "//tensorflow/contrib/tensor_forest/kernels/v4:all_files",
-        "//tensorflow/contrib/tensor_forest/proto:all_files",
-        "//tensorflow/contrib/tensorboard:all_files",
-        "//tensorflow/contrib/tensorboard/db:all_files",
-        "//tensorflow/contrib/tensorrt:all_files",
-        "//tensorflow/contrib/testing:all_files",
-        "//tensorflow/contrib/text:all_files",
-        "//tensorflow/contrib/tfprof:all_files",
-        "//tensorflow/contrib/timeseries:all_files",
-        "//tensorflow/contrib/timeseries/examples:all_files",
-        "//tensorflow/contrib/timeseries/python/timeseries:all_files",
-        "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:all_files",
-        "//tensorflow/contrib/tpu:all_files",
-        "//tensorflow/contrib/tpu/profiler:all_files",
-        "//tensorflow/contrib/tpu/proto:all_files",
-        "//tensorflow/contrib/training:all_files",
-        "//tensorflow/contrib/util:all_files",
-        "//tensorflow/contrib/verbs:all_files",
-        "//tensorflow/core:all_files",
-        "//tensorflow/core/api_def:all_files",
-        "//tensorflow/core/common_runtime/eager:all_files",
-        "//tensorflow/core/debug:all_files",
-        "//tensorflow/core/distributed_runtime:all_files",
-        "//tensorflow/core/distributed_runtime/rpc:all_files",
-        "//tensorflow/core/grappler:all_files",
-        "//tensorflow/core/grappler/clusters:all_files",
-        "//tensorflow/core/grappler/costs:all_files",
-        "//tensorflow/core/grappler/inputs:all_files",
-        "//tensorflow/core/grappler/optimizers:all_files",
-        "//tensorflow/core/grappler/utils:all_files",
-        "//tensorflow/core/kernels:all_files",
-        "//tensorflow/core/kernels/batching_util:all_files",
-        "//tensorflow/core/kernels/data:all_files",
-        "//tensorflow/core/kernels/data/sql:all_files",
-        "//tensorflow/core/kernels/fuzzing:all_files",
-        "//tensorflow/core/kernels/hexagon:all_files",
-        "//tensorflow/core/kernels/neon:all_files",
-        "//tensorflow/core/lib/db:all_files",
-        "//tensorflow/core/ops/compat:all_files",
-        "//tensorflow/core/platform/cloud:all_files",
-        "//tensorflow/core/platform/default/build_config:all_files",
-        "//tensorflow/core/platform/hadoop:all_files",
-        "//tensorflow/core/platform/s3:all_files",
-        "//tensorflow/core/profiler:all_files",
-        "//tensorflow/core/profiler/internal:all_files",
-        "//tensorflow/core/profiler/internal/advisor:all_files",
-        "//tensorflow/core/util/ctc:all_files",
-        "//tensorflow/core/util/tensor_bundle:all_files",
-        "//tensorflow/examples/adding_an_op:all_files",
-        "//tensorflow/examples/android:all_files",
-        "//tensorflow/examples/benchmark:all_files",
-        "//tensorflow/examples/get_started/regression:all_files",
-        "//tensorflow/examples/how_tos/reading_data:all_files",
-        "//tensorflow/examples/image_retraining:all_files",
-        "//tensorflow/examples/label_image:all_files",
-        "//tensorflow/examples/learn:all_files",
-        "//tensorflow/examples/multibox_detector:all_files",
-        "//tensorflow/examples/saved_model:all_files",
-        "//tensorflow/examples/speech_commands:all_files",
-        "//tensorflow/examples/tutorials/estimators:all_files",
-        "//tensorflow/examples/tutorials/layers:all_files",
-        "//tensorflow/examples/tutorials/mnist:all_files",
-        "//tensorflow/examples/tutorials/monitors:all_files",
-        "//tensorflow/examples/tutorials/word2vec:all_files",
-        "//tensorflow/examples/wav_to_spectrogram:all_files",
-        "//tensorflow/go:all_files",
-        "//tensorflow/java:all_files",
-        "//tensorflow/java/src/main/java/org/tensorflow/examples:all_files",
-        "//tensorflow/java/src/main/native:all_files",
-        "//tensorflow/python:all_files",
-        "//tensorflow/python/data:all_files",
-        "//tensorflow/python/data/kernel_tests:all_files",
-        "//tensorflow/python/data/ops:all_files",
-        "//tensorflow/python/data/util:all_files",
-        "//tensorflow/python/debug:all_files",
-        "//tensorflow/python/eager:all_files",
-        "//tensorflow/python/estimator:all_files",
-        "//tensorflow/python/feature_column:all_files",
-        "//tensorflow/python/keras:all_files",
-        "//tensorflow/python/kernel_tests:all_files",
-        "//tensorflow/python/kernel_tests/distributions:all_files",
-        "//tensorflow/python/kernel_tests/linalg:all_files",
-        "//tensorflow/python/kernel_tests/random:all_files",
-        "//tensorflow/python/kernel_tests/testdata:all_files",
-        "//tensorflow/python/ops/distributions:all_files",
-        "//tensorflow/python/ops/linalg:all_files",
-        "//tensorflow/python/ops/losses:all_files",
-        "//tensorflow/python/profiler:all_files",
-        "//tensorflow/python/profiler/internal:all_files",
-        "//tensorflow/python/saved_model:all_files",
-        "//tensorflow/python/tools:all_files",
-        "//tensorflow/tools/api/generator:all_files",
-        "//tensorflow/tools/api/golden:all_files",
-        "//tensorflow/tools/api/lib:all_files",
-        "//tensorflow/tools/api/tests:all_files",
-        "//tensorflow/tools/benchmark:all_files",
-        "//tensorflow/tools/build_info:all_files",
-        "//tensorflow/tools/ci_build/gpu_build:all_files",
-        "//tensorflow/tools/common:all_files",
-        "//tensorflow/tools/compatibility:all_files",
-        "//tensorflow/tools/dist_test/server:all_files",
-        "//tensorflow/tools/docker:all_files",
-        "//tensorflow/tools/docker/notebooks:all_files",
-        "//tensorflow/tools/docs:all_files",
-        "//tensorflow/tools/git:all_files",
-        "//tensorflow/tools/graph_transforms:all_files",
-        "//tensorflow/tools/mlpbtxt:all_files",
-        "//tensorflow/tools/proto_text:all_files",
-        "//tensorflow/tools/quantization:all_files",
-        "//tensorflow/tools/test:all_files",
-        "//tensorflow/user_ops:all_files",
-        "//third_party/eigen3:all_files",
-        "//third_party/fft2d:all_files",
-        "//third_party/flatbuffers:all_files",
-        "//third_party/hadoop:all_files",
-        "//third_party/sycl:all_files",
-        "//third_party/sycl/sycl:all_files",
-    ],
-    visibility = ["//visibility:public"],
-)
-
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
diff --git a/tensorflow/c/BUILD b/tensorflow/c/BUILD
index 249135f728..2367014cd0 100644
--- a/tensorflow/c/BUILD
+++ b/tensorflow/c/BUILD
@@ -287,18 +287,3 @@ tf_cuda_library(
         "//tensorflow/python:cpp_shape_inference_proto_cc",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/cc/BUILD b/tensorflow/cc/BUILD
index 9060c19e9d..079e063d3e 100644
--- a/tensorflow/cc/BUILD
+++ b/tensorflow/cc/BUILD
@@ -620,18 +620,6 @@ tf_cc_binary(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "queue_runner",
     srcs = ["training/queue_runner.cc"],
diff --git a/tensorflow/cc/saved_model/BUILD b/tensorflow/cc/saved_model/BUILD
index d29ad3ebcb..06a3be18e0 100644
--- a/tensorflow/cc/saved_model/BUILD
+++ b/tensorflow/cc/saved_model/BUILD
@@ -94,18 +94,3 @@ filegroup(
         "testdata/half_plus_two/**",
     ]),
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/cc/saved_model/python/BUILD b/tensorflow/cc/saved_model/python/BUILD
index f5fbc75edc..6f04ebdc55 100644
--- a/tensorflow/cc/saved_model/python/BUILD
+++ b/tensorflow/cc/saved_model/python/BUILD
@@ -7,18 +7,6 @@ package(
     default_visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_py_clif_cc")
 
 tf_py_clif_cc(
diff --git a/tensorflow/cc/tools/BUILD b/tensorflow/cc/tools/BUILD
index f413a5cc52..6f1c873540 100644
--- a/tensorflow/cc/tools/BUILD
+++ b/tensorflow/cc/tools/BUILD
@@ -41,18 +41,3 @@ tf_cc_test(
         "//tensorflow/core:testlib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/aot/BUILD b/tensorflow/compiler/aot/BUILD
index ffa2d08829..fa03b1f3c2 100644
--- a/tensorflow/compiler/aot/BUILD
+++ b/tensorflow/compiler/aot/BUILD
@@ -250,17 +250,3 @@ exports_files([
     "benchmark_main.template",  # used by tf_library(...,gen_benchmark=True)
     "test.cc",  # used by tf_library(...,gen_test=True)
 ])
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/aot/tests/BUILD b/tensorflow/compiler/aot/tests/BUILD
index 28aab6eb61..b053dad1b5 100644
--- a/tensorflow/compiler/aot/tests/BUILD
+++ b/tensorflow/compiler/aot/tests/BUILD
@@ -182,17 +182,3 @@ tf_cc_test(
         "//third_party/eigen3",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 8e505da622..9ea246ffdc 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -365,20 +365,6 @@ tf_cc_test(
     ],
 )
 
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # This target can be used by XLA device plugins to prevent circular dependencies, and provides access to all of the required headers for building a device library.
 cc_header_only_library(
     name = "xla_jit_headers_lib",
diff --git a/tensorflow/compiler/jit/graphcycles/BUILD b/tensorflow/compiler/jit/graphcycles/BUILD
index 15507b3851..676f71a75a 100644
--- a/tensorflow/compiler/jit/graphcycles/BUILD
+++ b/tensorflow/compiler/jit/graphcycles/BUILD
@@ -27,17 +27,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/jit/kernels/BUILD b/tensorflow/compiler/jit/kernels/BUILD
index 616a7f8f15..00a6f4075f 100644
--- a/tensorflow/compiler/jit/kernels/BUILD
+++ b/tensorflow/compiler/jit/kernels/BUILD
@@ -41,17 +41,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/jit/legacy_flags/BUILD b/tensorflow/compiler/jit/legacy_flags/BUILD
index 9cd66fc13c..5d211f4d73 100644
--- a/tensorflow/compiler/jit/legacy_flags/BUILD
+++ b/tensorflow/compiler/jit/legacy_flags/BUILD
@@ -63,17 +63,3 @@ cc_library(
             "//tensorflow/core:lib",
         ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/jit/ops/BUILD b/tensorflow/compiler/jit/ops/BUILD
index e5787ca4c8..c9e46bc147 100644
--- a/tensorflow/compiler/jit/ops/BUILD
+++ b/tensorflow/compiler/jit/ops/BUILD
@@ -17,17 +17,3 @@ cc_library(
     deps = ["//tensorflow/core:framework"],
     alwayslink = 1,
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/plugin/BUILD b/tensorflow/compiler/plugin/BUILD
index da4bc44c7a..238fd15166 100644
--- a/tensorflow/compiler/plugin/BUILD
+++ b/tensorflow/compiler/plugin/BUILD
@@ -49,17 +49,3 @@ cc_library(
         "//tensorflow/compiler/jit:xla_device",
     ],
 )
-
-#-----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 1c5a8f8e69..edabdc218a 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -835,17 +835,3 @@ tf_xla_py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index eb20ca501c..8c33bf179c 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -462,17 +462,3 @@ cc_library(
         "//tensorflow/core:protos_all_cc",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tf2xla/cc/BUILD b/tensorflow/compiler/tf2xla/cc/BUILD
index 311dddca94..c30bb9cacd 100644
--- a/tensorflow/compiler/tf2xla/cc/BUILD
+++ b/tensorflow/compiler/tf2xla/cc/BUILD
@@ -51,17 +51,3 @@ cc_library(
         "//tensorflow/core:protos_all_cc",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tf2xla/kernels/BUILD b/tensorflow/compiler/tf2xla/kernels/BUILD
index 0bbfe86de3..f1bc7d6af4 100644
--- a/tensorflow/compiler/tf2xla/kernels/BUILD
+++ b/tensorflow/compiler/tf2xla/kernels/BUILD
@@ -217,17 +217,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tf2xla/lib/BUILD b/tensorflow/compiler/tf2xla/lib/BUILD
index 488fda74bf..344773c8c5 100644
--- a/tensorflow/compiler/tf2xla/lib/BUILD
+++ b/tensorflow/compiler/tf2xla/lib/BUILD
@@ -140,17 +140,3 @@ cc_library(
         "//tensorflow/core:lib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/tf2xla/ops/BUILD b/tensorflow/compiler/tf2xla/ops/BUILD
index 98f72b3792..aeb743a663 100644
--- a/tensorflow/compiler/tf2xla/ops/BUILD
+++ b/tensorflow/compiler/tf2xla/ops/BUILD
@@ -39,17 +39,3 @@ tf_gen_op_wrapper_py(
         ":sendrecv_ops",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/BUILD b/tensorflow/compiler/xla/BUILD
index cd13db4d30..751777222f 100644
--- a/tensorflow/compiler/xla/BUILD
+++ b/tensorflow/compiler/xla/BUILD
@@ -654,18 +654,6 @@ tf_cc_test(
 
 # -----------------------------------------------------------------------------
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # This is a headers target that extra XLA devices can use to prevent circular dependencies.  Devices that are compiled as separate shared objects can also use it to prevent linking of library code.
 cc_header_only_library(
     name = "xla_headers_lib",
diff --git a/tensorflow/compiler/xla/client/BUILD b/tensorflow/compiler/xla/client/BUILD
index 5094e5ce67..a299c2afd4 100644
--- a/tensorflow/compiler/xla/client/BUILD
+++ b/tensorflow/compiler/xla/client/BUILD
@@ -214,17 +214,3 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index fca2bf2688..d02972f2c0 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -48,17 +48,3 @@ cc_library(
         "//tensorflow/core:lib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/client/xla_client/BUILD b/tensorflow/compiler/xla/client/xla_client/BUILD
index 60f13e04cb..b1dba16856 100644
--- a/tensorflow/compiler/xla/client/xla_client/BUILD
+++ b/tensorflow/compiler/xla/client/xla_client/BUILD
@@ -76,17 +76,3 @@ tf_cc_test(
         "//tensorflow/core:test",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/legacy_flags/BUILD b/tensorflow/compiler/xla/legacy_flags/BUILD
index 0a9725db0a..89353448e2 100644
--- a/tensorflow/compiler/xla/legacy_flags/BUILD
+++ b/tensorflow/compiler/xla/legacy_flags/BUILD
@@ -75,17 +75,3 @@ tf_cc_test(
             "//tensorflow/core:test",
         ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/python/BUILD b/tensorflow/compiler/xla/python/BUILD
index e2972f0601..0517a5502e 100644
--- a/tensorflow/compiler/xla/python/BUILD
+++ b/tensorflow/compiler/xla/python/BUILD
@@ -72,15 +72,3 @@ tf_py_wrap_cc(
         "//tensorflow/compiler/xla/service:cpu_plugin",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index bde749d317..b7d1bf64d0 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -2651,17 +2651,3 @@ cc_library(
         "//tensorflow/core:lib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 0faa9e9c41..966e2d0fc5 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -916,17 +916,3 @@ tf_cc_test(
         "//tensorflow/core:test",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/service/gpu/BUILD b/tensorflow/compiler/xla/service/gpu/BUILD
index 93b2f2a474..f1707442fe 100644
--- a/tensorflow/compiler/xla/service/gpu/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/BUILD
@@ -700,17 +700,3 @@ tf_cc_test(
         "//tensorflow/core:test",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD
index f4c4dcdafd..86c4ac18b0 100644
--- a/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD
+++ b/tensorflow/compiler/xla/service/gpu/llvm_gpu_backend/BUILD
@@ -68,17 +68,3 @@ tf_cc_test(
         "@llvm//:support",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/service/interpreter/BUILD b/tensorflow/compiler/xla/service/interpreter/BUILD
index 0db3863f24..4550548495 100644
--- a/tensorflow/compiler/xla/service/interpreter/BUILD
+++ b/tensorflow/compiler/xla/service/interpreter/BUILD
@@ -120,14 +120,3 @@ cc_library(
         "//tensorflow/core:stream_executor_no_cuda",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/compiler/xla/service/llvm_ir/BUILD b/tensorflow/compiler/xla/service/llvm_ir/BUILD
index 37261ed1e6..f1e7fc2953 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/BUILD
+++ b/tensorflow/compiler/xla/service/llvm_ir/BUILD
@@ -169,17 +169,3 @@ cc_library(
         "@llvm//:core",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 2fd97fa38e..e337669aeb 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1960,17 +1960,3 @@ tf_cc_test(
         "//tensorflow/core:test",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/tools/BUILD b/tensorflow/compiler/xla/tools/BUILD
index 2e55f609d1..0bc4045a54 100644
--- a/tensorflow/compiler/xla/tools/BUILD
+++ b/tensorflow/compiler/xla/tools/BUILD
@@ -223,17 +223,3 @@ tf_cc_binary(
         "//tensorflow/core:lib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/compiler/xla/tools/parser/BUILD b/tensorflow/compiler/xla/tools/parser/BUILD
index 97aacf6b39..0fa4b98d0a 100644
--- a/tensorflow/compiler/xla/tools/parser/BUILD
+++ b/tensorflow/compiler/xla/tools/parser/BUILD
@@ -70,17 +70,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-# -----------------------------------------------------------------------------
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index bdbd738906..1ca70e7122 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -159,15 +159,3 @@ cc_library(
         "//tensorflow/contrib/tpu:all_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/all_reduce/BUILD b/tensorflow/contrib/all_reduce/BUILD
index 8dff93b4f8..62d1b1cf07 100644
--- a/tensorflow/contrib/all_reduce/BUILD
+++ b/tensorflow/contrib/all_reduce/BUILD
@@ -45,16 +45,3 @@ tf_py_test(
         "//tensorflow/python:state_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/android/BUILD b/tensorflow/contrib/android/BUILD
index 4bff3c27d2..60306ebdc6 100644
--- a/tensorflow/contrib/android/BUILD
+++ b/tensorflow/contrib/android/BUILD
@@ -38,20 +38,6 @@ cc_library(
     alwayslink = 1,
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # JAR with Java bindings to TF.
 android_library(
     name = "android_tensorflow_inference_java",
diff --git a/tensorflow/contrib/batching/BUILD b/tensorflow/contrib/batching/BUILD
index ee67909133..d65c990c87 100644
--- a/tensorflow/contrib/batching/BUILD
+++ b/tensorflow/contrib/batching/BUILD
@@ -112,14 +112,3 @@ py_test(
         "//tensorflow/python:script_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/batching/test_util/BUILD b/tensorflow/contrib/batching/test_util/BUILD
index 6db627faad..7cb2d8079b 100644
--- a/tensorflow/contrib/batching/test_util/BUILD
+++ b/tensorflow/contrib/batching/test_util/BUILD
@@ -8,17 +8,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
-
 cc_library(
     name = "fake_clock_env",
     testonly = 1,
diff --git a/tensorflow/contrib/batching/util/BUILD b/tensorflow/contrib/batching/util/BUILD
index 2a84a7712a..8f81b6702f 100644
--- a/tensorflow/contrib/batching/util/BUILD
+++ b/tensorflow/contrib/batching/util/BUILD
@@ -8,18 +8,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "**/google_*",
-        ],
-    ),
-)
-
 cc_library(
     name = "periodic_function_dynamic",
     hdrs = ["periodic_function.h"],
diff --git a/tensorflow/contrib/bayesflow/BUILD b/tensorflow/contrib/bayesflow/BUILD
index a55029b314..5a2d7f6a3c 100644
--- a/tensorflow/contrib/bayesflow/BUILD
+++ b/tensorflow/contrib/bayesflow/BUILD
@@ -57,15 +57,3 @@ cuda_py_test(
         "//tensorflow/python:random_seed",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD
index 6fdcd0f996..ddeda0079c 100644
--- a/tensorflow/contrib/boosted_trees/BUILD
+++ b/tensorflow/contrib/boosted_trees/BUILD
@@ -14,15 +14,6 @@ load("//tensorflow:tensorflow.bzl", "tf_gen_op_wrapper_py")
 load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
 load("//tensorflow:tensorflow.bzl", "tf_custom_op_py_library")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = ["**/OWNERS"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 package_group(name = "friends")
 
 cc_library(
diff --git a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
index dcd235f876..17e20c4b31 100644
--- a/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
+++ b/tensorflow/contrib/boosted_trees/estimator_batch/BUILD
@@ -10,15 +10,6 @@ package(
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = ["**/*"],
-        exclude = ["**/OWNERS"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "init_py",
     srcs = ["__init__.py"],
diff --git a/tensorflow/contrib/boosted_trees/lib/BUILD b/tensorflow/contrib/boosted_trees/lib/BUILD
index 131bd48562..3028c22817 100644
--- a/tensorflow/contrib/boosted_trees/lib/BUILD
+++ b/tensorflow/contrib/boosted_trees/lib/BUILD
@@ -15,17 +15,6 @@ load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # Utils
 
 cc_library(
diff --git a/tensorflow/contrib/boosted_trees/proto/BUILD b/tensorflow/contrib/boosted_trees/proto/BUILD
index 9a61e163eb..b07f0a4314 100644
--- a/tensorflow/contrib/boosted_trees/proto/BUILD
+++ b/tensorflow/contrib/boosted_trees/proto/BUILD
@@ -4,17 +4,6 @@ exports_files(["LICENSE"])
 
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "learner_proto",
     srcs = [
diff --git a/tensorflow/contrib/boosted_trees/resources/BUILD b/tensorflow/contrib/boosted_trees/resources/BUILD
index 9fc101612f..c065186845 100644
--- a/tensorflow/contrib/boosted_trees/resources/BUILD
+++ b/tensorflow/contrib/boosted_trees/resources/BUILD
@@ -9,17 +9,6 @@ package(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "stamped_resource",
     hdrs = ["stamped_resource.h"],
diff --git a/tensorflow/contrib/cloud/BUILD b/tensorflow/contrib/cloud/BUILD
index fe8bd072af..f3a75e8688 100644
--- a/tensorflow/contrib/cloud/BUILD
+++ b/tensorflow/contrib/cloud/BUILD
@@ -14,18 +14,6 @@ load(
     "tf_py_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_gen_op_libs(
     op_lib_names = ["bigquery_reader_ops"],
     deps = [
diff --git a/tensorflow/contrib/cloud/kernels/BUILD b/tensorflow/contrib/cloud/kernels/BUILD
index d5fc604de9..ff46f0daa8 100644
--- a/tensorflow/contrib/cloud/kernels/BUILD
+++ b/tensorflow/contrib/cloud/kernels/BUILD
@@ -20,20 +20,6 @@ load(
     "tf_proto_library",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_kernel_library(
     name = "bigquery_reader_ops",
     srcs = ["bigquery_reader_ops.cc"],
diff --git a/tensorflow/contrib/cluster_resolver/BUILD b/tensorflow/contrib/cluster_resolver/BUILD
index 1a124eca36..c239e6f8f9 100644
--- a/tensorflow/contrib/cluster_resolver/BUILD
+++ b/tensorflow/contrib/cluster_resolver/BUILD
@@ -10,19 +10,6 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
-
 py_library(
     name = "cluster_resolver_pip",
     srcs = [
diff --git a/tensorflow/contrib/coder/BUILD b/tensorflow/contrib/coder/BUILD
index ec3d550b70..ce12e38248 100644
--- a/tensorflow/contrib/coder/BUILD
+++ b/tensorflow/contrib/coder/BUILD
@@ -154,14 +154,3 @@ tf_py_test(
     ],
     main = "python/ops/coder_ops_test.py",
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/compiler/BUILD b/tensorflow/contrib/compiler/BUILD
index 388d8e6ed6..bcee0b04c8 100644
--- a/tensorflow/contrib/compiler/BUILD
+++ b/tensorflow/contrib/compiler/BUILD
@@ -46,15 +46,3 @@ cuda_py_test(
     ],
     xla_enabled = True,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/copy_graph/BUILD b/tensorflow/contrib/copy_graph/BUILD
index 8ec706df74..fa44c4d54e 100644
--- a/tensorflow/contrib/copy_graph/BUILD
+++ b/tensorflow/contrib/copy_graph/BUILD
@@ -41,15 +41,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/crf/BUILD b/tensorflow/contrib/crf/BUILD
index 7aad4abdb9..5c1a17df4f 100644
--- a/tensorflow/contrib/crf/BUILD
+++ b/tensorflow/contrib/crf/BUILD
@@ -40,15 +40,3 @@ cuda_py_tests(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/cudnn_rnn/BUILD b/tensorflow/contrib/cudnn_rnn/BUILD
index fa86ad38c9..8b5d13f725 100644
--- a/tensorflow/contrib/cudnn_rnn/BUILD
+++ b/tensorflow/contrib/cudnn_rnn/BUILD
@@ -123,15 +123,3 @@ cuda_py_test(
         "requires_cudnn5",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 9e25a77d9f..35312f06b3 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -44,17 +44,3 @@ tf_custom_op_library(
 tf_gen_op_libs(
     op_lib_names = ["dataset_ops"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/data/kernels/BUILD b/tensorflow/contrib/data/kernels/BUILD
index c87da7dfaa..83ada6fb67 100644
--- a/tensorflow/contrib/data/kernels/BUILD
+++ b/tensorflow/contrib/data/kernels/BUILD
@@ -61,14 +61,3 @@ cc_library(
         "@protobuf_archive//:protobuf_headers",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 0b3bf63f79..0f4c9e48cf 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -513,17 +513,3 @@ tf_py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 647620eb84..236792bb98 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -183,15 +183,3 @@ py_library(
         "//tensorflow/python/data/util:sparse",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/decision_trees/proto/BUILD b/tensorflow/contrib/decision_trees/proto/BUILD
index ae3847b8b6..3b50a48336 100644
--- a/tensorflow/contrib/decision_trees/proto/BUILD
+++ b/tensorflow/contrib/decision_trees/proto/BUILD
@@ -13,14 +13,6 @@ load(
     "tf_pyclif_proto_library",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "generic_tree_model",
     srcs = ["generic_tree_model.proto"],
diff --git a/tensorflow/contrib/deprecated/BUILD b/tensorflow/contrib/deprecated/BUILD
index 3dfbbf5527..401527f1e7 100644
--- a/tensorflow/contrib/deprecated/BUILD
+++ b/tensorflow/contrib/deprecated/BUILD
@@ -30,15 +30,3 @@ py_test(
         "//tensorflow/python:logging_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 682448b84b..231abaa2f3 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -746,18 +746,6 @@ cuda_py_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # === Bijector Tests ==========================================================
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/proto/BUILD b/tensorflow/contrib/eager/proto/BUILD
index aedfec8924..b016d2dcb5 100644
--- a/tensorflow/contrib/eager/proto/BUILD
+++ b/tensorflow/contrib/eager/proto/BUILD
@@ -4,17 +4,6 @@ exports_files(["LICENSE"])
 
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "checkpointable_object_graph_proto",
     srcs = [
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 4fba014d6f..7a8c11e3bb 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -272,16 +272,3 @@ cuda_py_test(
     ],
     tags = ["notsan"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index c846343d6d..d125e40f6c 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -9,18 +9,6 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "estimator_py",
     srcs = ["__init__.py"],
diff --git a/tensorflow/contrib/factorization/BUILD b/tensorflow/contrib/factorization/BUILD
index ad8568ad44..0a648d5d40 100644
--- a/tensorflow/contrib/factorization/BUILD
+++ b/tensorflow/contrib/factorization/BUILD
@@ -347,16 +347,3 @@ cuda_py_test(
     ],
     main = "python/kernel_tests/masked_matmul_benchmark.py",
 )
-
-# All files
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/factorization/examples/BUILD b/tensorflow/contrib/factorization/examples/BUILD
index bbe842bd5c..363baa121a 100644
--- a/tensorflow/contrib/factorization/examples/BUILD
+++ b/tensorflow/contrib/factorization/examples/BUILD
@@ -21,14 +21,3 @@ tf_py_test(
     ],
     tags = ["notsan"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/factorization/kernels/BUILD b/tensorflow/contrib/factorization/kernels/BUILD
index 44eab56011..ea8b9a17a2 100644
--- a/tensorflow/contrib/factorization/kernels/BUILD
+++ b/tensorflow/contrib/factorization/kernels/BUILD
@@ -67,14 +67,3 @@ tf_cc_test(
         "//tensorflow/core:testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/feature_column/BUILD b/tensorflow/contrib/feature_column/BUILD
index 3614b2b15a..aab7d0c9e8 100644
--- a/tensorflow/contrib/feature_column/BUILD
+++ b/tensorflow/contrib/feature_column/BUILD
@@ -8,18 +8,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "feature_column_py",
     srcs = ["__init__.py"],
diff --git a/tensorflow/contrib/ffmpeg/BUILD b/tensorflow/contrib/ffmpeg/BUILD
index eccce99071..f7b3273a4d 100644
--- a/tensorflow/contrib/ffmpeg/BUILD
+++ b/tensorflow/contrib/ffmpeg/BUILD
@@ -180,15 +180,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/ffmpeg/default/BUILD b/tensorflow/contrib/ffmpeg/default/BUILD
index 6b455567d7..59bad8982d 100644
--- a/tensorflow/contrib/ffmpeg/default/BUILD
+++ b/tensorflow/contrib/ffmpeg/default/BUILD
@@ -74,15 +74,3 @@ tf_cc_test(
         "//tensorflow/core:test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/framework/BUILD b/tensorflow/contrib/framework/BUILD
index ac043fda06..b1c8ad49ea 100644
--- a/tensorflow/contrib/framework/BUILD
+++ b/tensorflow/contrib/framework/BUILD
@@ -321,15 +321,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/fused_conv/BUILD b/tensorflow/contrib/fused_conv/BUILD
index ce37672895..0eb6889db1 100644
--- a/tensorflow/contrib/fused_conv/BUILD
+++ b/tensorflow/contrib/fused_conv/BUILD
@@ -157,15 +157,3 @@ cuda_py_test(
         "requires_cudnn6",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 0eb0e3cbe2..9e56d3c039 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -544,15 +544,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/gdr/BUILD b/tensorflow/contrib/gdr/BUILD
index 707ae25d48..e534fdc177 100644
--- a/tensorflow/contrib/gdr/BUILD
+++ b/tensorflow/contrib/gdr/BUILD
@@ -9,18 +9,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "c_srcs",
     data = glob([
diff --git a/tensorflow/contrib/graph_editor/BUILD b/tensorflow/contrib/graph_editor/BUILD
index 967ad2fc09..1711100e3a 100644
--- a/tensorflow/contrib/graph_editor/BUILD
+++ b/tensorflow/contrib/graph_editor/BUILD
@@ -39,18 +39,6 @@ py_library(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "match",
     srcs = ["tests/match.py"],
diff --git a/tensorflow/contrib/grid_rnn/BUILD b/tensorflow/contrib/grid_rnn/BUILD
index d601a1ec6f..d0b4464066 100644
--- a/tensorflow/contrib/grid_rnn/BUILD
+++ b/tensorflow/contrib/grid_rnn/BUILD
@@ -41,15 +41,3 @@ cuda_py_tests(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/hooks/BUILD b/tensorflow/contrib/hooks/BUILD
index 1b528d7afc..d65b2d6026 100644
--- a/tensorflow/contrib/hooks/BUILD
+++ b/tensorflow/contrib/hooks/BUILD
@@ -23,14 +23,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD
index 324035100d..e39c60b252 100644
--- a/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD
+++ b/tensorflow/contrib/hvx/clock_cycle_profiling/BUILD
@@ -13,18 +13,6 @@ exports_files(["LICENSE"])
 
 package(default_visibility = ["//visibility:public"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_cc_binary(
     name = "clock_cycle_profiling",
     testonly = 1,
diff --git a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
index 909dc396a3..0081fb6177 100644
--- a/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
+++ b/tensorflow/contrib/hvx/hvx_ops_support_checker/BUILD
@@ -10,17 +10,6 @@ exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
-
 tf_cc_binary(
     name = "hvx_ops_support_checker",
     testonly = 1,
diff --git a/tensorflow/contrib/image/BUILD b/tensorflow/contrib/image/BUILD
index 79eb3762ed..da450480b3 100755
--- a/tensorflow/contrib/image/BUILD
+++ b/tensorflow/contrib/image/BUILD
@@ -384,15 +384,3 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/input_pipeline/BUILD b/tensorflow/contrib/input_pipeline/BUILD
index 9d6b4d5d87..0e34315db4 100644
--- a/tensorflow/contrib/input_pipeline/BUILD
+++ b/tensorflow/contrib/input_pipeline/BUILD
@@ -114,14 +114,3 @@ tf_cc_tests(
         "//tensorflow/core:testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/input_pipeline/kernels/BUILD b/tensorflow/contrib/input_pipeline/kernels/BUILD
index f20a6e38d4..797605b8fe 100644
--- a/tensorflow/contrib/input_pipeline/kernels/BUILD
+++ b/tensorflow/contrib/input_pipeline/kernels/BUILD
@@ -17,14 +17,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/integrate/BUILD b/tensorflow/contrib/integrate/BUILD
index 66948c1ea1..0b7d64f4ed 100644
--- a/tensorflow/contrib/integrate/BUILD
+++ b/tensorflow/contrib/integrate/BUILD
@@ -42,14 +42,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/kafka/BUILD b/tensorflow/contrib/kafka/BUILD
index 1c3974871c..3913c9dc7a 100644
--- a/tensorflow/contrib/kafka/BUILD
+++ b/tensorflow/contrib/kafka/BUILD
@@ -119,17 +119,3 @@ tf_py_test(
         "notap",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/keras/BUILD b/tensorflow/contrib/keras/BUILD
index 7e0019ce4a..7a4cab20d1 100644
--- a/tensorflow/contrib/keras/BUILD
+++ b/tensorflow/contrib/keras/BUILD
@@ -52,15 +52,3 @@ py_library(
         "//tensorflow/python/keras",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kernel_methods/BUILD b/tensorflow/contrib/kernel_methods/BUILD
index eff7dfeb4c..87c2dcd89b 100644
--- a/tensorflow/contrib/kernel_methods/BUILD
+++ b/tensorflow/contrib/kernel_methods/BUILD
@@ -90,15 +90,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kfac/BUILD b/tensorflow/contrib/kfac/BUILD
index 9a5759bf14..b719046b37 100644
--- a/tensorflow/contrib/kfac/BUILD
+++ b/tensorflow/contrib/kfac/BUILD
@@ -24,15 +24,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kfac/examples/BUILD b/tensorflow/contrib/kfac/examples/BUILD
index 89965eda37..7dd40c19c5 100644
--- a/tensorflow/contrib/kfac/examples/BUILD
+++ b/tensorflow/contrib/kfac/examples/BUILD
@@ -58,15 +58,3 @@ py_library(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kfac/examples/tests/BUILD b/tensorflow/contrib/kfac/examples/tests/BUILD
index ce7da95c12..ede7f183fe 100644
--- a/tensorflow/contrib/kfac/examples/tests/BUILD
+++ b/tensorflow/contrib/kfac/examples/tests/BUILD
@@ -50,15 +50,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index 146ae8b7e2..f73c24f8fb 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -155,15 +155,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/kfac/python/ops/BUILD b/tensorflow/contrib/kfac/python/ops/BUILD
index d721ad08af..b897fd68a0 100644
--- a/tensorflow/contrib/kfac/python/ops/BUILD
+++ b/tensorflow/contrib/kfac/python/ops/BUILD
@@ -244,15 +244,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 894e6f6946..18b265ae80 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -213,14 +213,3 @@ py_test(
         "//tensorflow/python:math_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index 852d06e1e3..4be55468db 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -390,15 +390,3 @@ py_test(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/layers/kernels/BUILD b/tensorflow/contrib/layers/kernels/BUILD
index e407a9ce01..7aae09ff3e 100644
--- a/tensorflow/contrib/layers/kernels/BUILD
+++ b/tensorflow/contrib/layers/kernels/BUILD
@@ -18,14 +18,3 @@ cc_library(
     ],
     alwayslink = 1,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 9c59150580..924918be4f 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -873,15 +873,3 @@ py_binary(
         "//tensorflow/python:platform",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/learn/python/learn/datasets/BUILD b/tensorflow/contrib/learn/python/learn/datasets/BUILD
index 8bf372841d..2c7215bba3 100644
--- a/tensorflow/contrib/learn/python/learn/datasets/BUILD
+++ b/tensorflow/contrib/learn/python/learn/datasets/BUILD
@@ -44,18 +44,6 @@ py_binary(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_test(
     name = "base_test",
     size = "small",
diff --git a/tensorflow/contrib/legacy_seq2seq/BUILD b/tensorflow/contrib/legacy_seq2seq/BUILD
index 1fa55132b1..8c2c4fd29c 100644
--- a/tensorflow/contrib/legacy_seq2seq/BUILD
+++ b/tensorflow/contrib/legacy_seq2seq/BUILD
@@ -60,15 +60,3 @@ cuda_py_tests(
     ],
     tags = ["noasan"],  # times out b/63678675
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/libsvm/BUILD b/tensorflow/contrib/libsvm/BUILD
index df96402a4f..4dccb9be7c 100644
--- a/tensorflow/contrib/libsvm/BUILD
+++ b/tensorflow/contrib/libsvm/BUILD
@@ -88,15 +88,3 @@ tf_py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/linalg/BUILD b/tensorflow/contrib/linalg/BUILD
index 359255374d..a7812f74d1 100644
--- a/tensorflow/contrib/linalg/BUILD
+++ b/tensorflow/contrib/linalg/BUILD
@@ -61,15 +61,3 @@ cuda_py_test(
     shard_count = 4,
     tags = ["noasan"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/linear_optimizer/BUILD b/tensorflow/contrib/linear_optimizer/BUILD
index cea3627ed5..5b89c6cef9 100644
--- a/tensorflow/contrib/linear_optimizer/BUILD
+++ b/tensorflow/contrib/linear_optimizer/BUILD
@@ -138,14 +138,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/lite/BUILD b/tensorflow/contrib/lite/BUILD
index 18efa64507..ac269d540a 100644
--- a/tensorflow/contrib/lite/BUILD
+++ b/tensorflow/contrib/lite/BUILD
@@ -271,18 +271,3 @@ cc_test(
 #        ],
 #    }),
 #)
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "downloads",
-            "examples",
-            "gen",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/examples/label_image/BUILD b/tensorflow/contrib/lite/examples/label_image/BUILD
index 959347b549..9322e186a2 100644
--- a/tensorflow/contrib/lite/examples/label_image/BUILD
+++ b/tensorflow/contrib/lite/examples/label_image/BUILD
@@ -69,15 +69,3 @@ cc_library(
 #         "//testing/base/public:gunit",
 #     ],
 # )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/java/BUILD b/tensorflow/contrib/lite/java/BUILD
index f52d6ba6c5..7f7a2632dd 100644
--- a/tensorflow/contrib/lite/java/BUILD
+++ b/tensorflow/contrib/lite/java/BUILD
@@ -167,15 +167,3 @@ tflite_jni_binary(
         "//tensorflow/contrib/lite/java/src/main/native",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/BUILD b/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
index 5eb749aae6..d6fbef9cc9 100644
--- a/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
+++ b/tensorflow/contrib/lite/java/demo/app/src/main/BUILD
@@ -27,15 +27,3 @@ android_binary(
         "@androidsdk//com.android.support:support-v4-25.2.0",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/java/demo/app/src/main/assets/BUILD b/tensorflow/contrib/lite/java/demo/app/src/main/assets/BUILD
index dd0cd6c98f..ce68160b68 100644
--- a/tensorflow/contrib/lite/java/demo/app/src/main/assets/BUILD
+++ b/tensorflow/contrib/lite/java/demo/app/src/main/assets/BUILD
@@ -10,15 +10,3 @@ exports_files(
         ],
     ),
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/java/src/main/native/BUILD b/tensorflow/contrib/lite/java/src/main/native/BUILD
index 3571182ca9..4399ed2025 100644
--- a/tensorflow/contrib/lite/java/src/main/native/BUILD
+++ b/tensorflow/contrib/lite/java/src/main/native/BUILD
@@ -95,15 +95,3 @@ exports_files(
         "version_script.lds",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/BUILD b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/BUILD
index 2b4f37bc6c..b524246d43 100644
--- a/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/BUILD
+++ b/tensorflow/contrib/lite/java/src/testhelper/java/org/tensorflow/lite/BUILD
@@ -16,15 +16,3 @@ android_library(
         "//tensorflow/contrib/lite/java:tensorflowlite_java",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 1450c1e14b..058f995d75 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -911,16 +911,4 @@ tf_cc_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tflite_portable_test_suite()
diff --git a/tensorflow/contrib/lite/kernels/internal/BUILD b/tensorflow/contrib/lite/kernels/internal/BUILD
index aa3957bee1..167c0f1fde 100644
--- a/tensorflow/contrib/lite/kernels/internal/BUILD
+++ b/tensorflow/contrib/lite/kernels/internal/BUILD
@@ -431,15 +431,3 @@ cc_library(
 )
 
 exports_files(["optimized/eigen_tensor_reduced_instantiations_oss.h"])
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/models/BUILD b/tensorflow/contrib/lite/models/BUILD
index 6a1255b586..efa47b06fa 100644
--- a/tensorflow/contrib/lite/models/BUILD
+++ b/tensorflow/contrib/lite/models/BUILD
@@ -12,15 +12,3 @@ load("//tensorflow/contrib/lite:build_def.bzl", "tflite_copts")
 exports_files(glob([
     "testdata/*",
 ]))
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/models/smartreply/BUILD b/tensorflow/contrib/lite/models/smartreply/BUILD
index 733c3f4c7f..a82d1f2eb6 100644
--- a/tensorflow/contrib/lite/models/smartreply/BUILD
+++ b/tensorflow/contrib/lite/models/smartreply/BUILD
@@ -86,15 +86,3 @@ cc_test(
         "@com_google_googletest//:gtest",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/nnapi/BUILD b/tensorflow/contrib/lite/nnapi/BUILD
index 402f1e949b..467a2b7a7b 100644
--- a/tensorflow/contrib/lite/nnapi/BUILD
+++ b/tensorflow/contrib/lite/nnapi/BUILD
@@ -11,15 +11,3 @@ cc_library(
     ],
     linkopts = ["-ldl"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index 411d5c0d27..e70aa51298 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -118,15 +118,3 @@ py_library(
         ":convert_saved_model",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/schema/BUILD b/tensorflow/contrib/lite/schema/BUILD
index da65ec659c..246ec85fe4 100644
--- a/tensorflow/contrib/lite/schema/BUILD
+++ b/tensorflow/contrib/lite/schema/BUILD
@@ -70,16 +70,4 @@ cc_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tflite_portable_test_suite()
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 555ea90034..10e810a6e0 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -373,16 +373,4 @@ tf_cc_test(
     }),
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tflite_portable_test_suite()
diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 8ed3e0e14e..bba61627f9 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -420,15 +420,3 @@ tf_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
index b975cc996b..a2008ddbdb 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
@@ -39,15 +39,3 @@ tf_cc_test(
         "@com_google_googletest//:gtest_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/toco/python/BUILD b/tensorflow/contrib/lite/toco/python/BUILD
index 17115047d2..5a40451b3a 100644
--- a/tensorflow/contrib/lite/toco/python/BUILD
+++ b/tensorflow/contrib/lite/toco/python/BUILD
@@ -63,15 +63,3 @@ tf_py_test(
     ],
     tags = ["no_pip"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD b/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
index 0c1a1141fc..336e94de1e 100644
--- a/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
+++ b/tensorflow/contrib/lite/toco/tensorflow_graph_matching/BUILD
@@ -88,15 +88,3 @@ cc_library(
         "//tensorflow/core:protos_all_cc",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/toco/tflite/BUILD b/tensorflow/contrib/lite/toco/tflite/BUILD
index 9d3e1daf12..e0191801a0 100644
--- a/tensorflow/contrib/lite/toco/tflite/BUILD
+++ b/tensorflow/contrib/lite/toco/tflite/BUILD
@@ -137,15 +137,3 @@ tf_cc_test(
         "@flatbuffers",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/lite/tools/BUILD b/tensorflow/contrib/lite/tools/BUILD
index b5abbc0712..44fde69a1e 100644
--- a/tensorflow/contrib/lite/tools/BUILD
+++ b/tensorflow/contrib/lite/tools/BUILD
@@ -91,18 +91,6 @@ cc_library(
     deps = ["//tensorflow/contrib/lite:framework"],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "verifier",
     srcs = ["verifier.cc"],
diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD
index 8ca03f4193..02b4f80252 100644
--- a/tensorflow/contrib/lookup/BUILD
+++ b/tensorflow/contrib/lookup/BUILD
@@ -47,15 +47,3 @@ tf_py_test(
     ],
     grpc_enabled = True,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/losses/BUILD b/tensorflow/contrib/losses/BUILD
index 5694211521..728f75f8ef 100644
--- a/tensorflow/contrib/losses/BUILD
+++ b/tensorflow/contrib/losses/BUILD
@@ -97,15 +97,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/makefile/BUILD b/tensorflow/contrib/makefile/BUILD
index 701eeb44fe..1abb46f4d4 100644
--- a/tensorflow/contrib/makefile/BUILD
+++ b/tensorflow/contrib/makefile/BUILD
@@ -3,12 +3,3 @@
 licenses(["notice"])  # Apache 2.0
 
 package(default_visibility = ["//visibility:private"])
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = ["**/OWNERS"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/memory_stats/BUILD b/tensorflow/contrib/memory_stats/BUILD
index 72424c32e7..63843b993c 100644
--- a/tensorflow/contrib/memory_stats/BUILD
+++ b/tensorflow/contrib/memory_stats/BUILD
@@ -79,15 +79,3 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/meta_graph_transform/BUILD b/tensorflow/contrib/meta_graph_transform/BUILD
index 4b5b1c3e15..24400789f8 100644
--- a/tensorflow/contrib/meta_graph_transform/BUILD
+++ b/tensorflow/contrib/meta_graph_transform/BUILD
@@ -59,15 +59,3 @@ filegroup(
         "**/*.py",
     ]),
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/metrics/BUILD b/tensorflow/contrib/metrics/BUILD
index e90c525113..5ca42f41c1 100644
--- a/tensorflow/contrib/metrics/BUILD
+++ b/tensorflow/contrib/metrics/BUILD
@@ -97,14 +97,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/model_pruning/BUILD b/tensorflow/contrib/model_pruning/BUILD
index ca3f13479e..f50575b2cf 100644
--- a/tensorflow/contrib/model_pruning/BUILD
+++ b/tensorflow/contrib/model_pruning/BUILD
@@ -125,15 +125,3 @@ py_library(
         ":rnn_cells",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/model_pruning/examples/cifar10/BUILD b/tensorflow/contrib/model_pruning/examples/cifar10/BUILD
index e7848adcc5..30ea912222 100644
--- a/tensorflow/contrib/model_pruning/examples/cifar10/BUILD
+++ b/tensorflow/contrib/model_pruning/examples/cifar10/BUILD
@@ -68,15 +68,3 @@ py_binary(
         "//tensorflow/contrib/model_pruning:pruning",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/mpi_collectives/BUILD b/tensorflow/contrib/mpi_collectives/BUILD
index 9f9802b8fe..a7be92a35e 100644
--- a/tensorflow/contrib/mpi_collectives/BUILD
+++ b/tensorflow/contrib/mpi_collectives/BUILD
@@ -126,15 +126,3 @@ tf_py_test(
     ],
     tags = ["manual"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD
index 94d01efee1..6cbfd03881 100644
--- a/tensorflow/contrib/nccl/BUILD
+++ b/tensorflow/contrib/nccl/BUILD
@@ -141,15 +141,3 @@ cuda_py_test(
         "notap",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/nearest_neighbor/BUILD b/tensorflow/contrib/nearest_neighbor/BUILD
index 9500c18b1d..6fa7624467 100644
--- a/tensorflow/contrib/nearest_neighbor/BUILD
+++ b/tensorflow/contrib/nearest_neighbor/BUILD
@@ -111,15 +111,3 @@ tf_py_test(
         "//tensorflow/python:client_testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/nn/BUILD b/tensorflow/contrib/nn/BUILD
index 5543eb6c6e..ef7ab22646 100644
--- a/tensorflow/contrib/nn/BUILD
+++ b/tensorflow/contrib/nn/BUILD
@@ -98,14 +98,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/opt/BUILD b/tensorflow/contrib/opt/BUILD
index bacf15bbd6..c57c5e3f29 100644
--- a/tensorflow/contrib/opt/BUILD
+++ b/tensorflow/contrib/opt/BUILD
@@ -265,14 +265,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/periodic_resample/BUILD b/tensorflow/contrib/periodic_resample/BUILD
index bd9078ae76..6ca7fe8b6e 100644
--- a/tensorflow/contrib/periodic_resample/BUILD
+++ b/tensorflow/contrib/periodic_resample/BUILD
@@ -94,18 +94,6 @@ py_test(
 #     srcs_version = "PY2AND3",
 # )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "custom_op_sources",
     srcs = glob(
diff --git a/tensorflow/contrib/predictor/BUILD b/tensorflow/contrib/predictor/BUILD
index a80f060b91..36e21af618 100644
--- a/tensorflow/contrib/predictor/BUILD
+++ b/tensorflow/contrib/predictor/BUILD
@@ -8,18 +8,6 @@ exports_files(["LICENSE"])
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "predictor",
     srcs = ["__init__.py"],
diff --git a/tensorflow/contrib/quantization/BUILD b/tensorflow/contrib/quantization/BUILD
index c19a31afb2..2de10e8fae 100644
--- a/tensorflow/contrib/quantization/BUILD
+++ b/tensorflow/contrib/quantization/BUILD
@@ -49,15 +49,3 @@ filegroup(
         "**/*.py",
     ]),
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/quantize/BUILD b/tensorflow/contrib/quantize/BUILD
index 0b76296204..b9918fdee1 100644
--- a/tensorflow/contrib/quantize/BUILD
+++ b/tensorflow/contrib/quantize/BUILD
@@ -246,15 +246,3 @@ py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/receptive_field/BUILD b/tensorflow/contrib/receptive_field/BUILD
index e975aeaea7..9325a14745 100644
--- a/tensorflow/contrib/receptive_field/BUILD
+++ b/tensorflow/contrib/receptive_field/BUILD
@@ -106,15 +106,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/reduce_slice_ops/BUILD b/tensorflow/contrib/reduce_slice_ops/BUILD
index b31f4488f5..02b3d66e46 100644
--- a/tensorflow/contrib/reduce_slice_ops/BUILD
+++ b/tensorflow/contrib/reduce_slice_ops/BUILD
@@ -101,15 +101,3 @@ tf_cc_test(
         "//tensorflow/core:testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
index 27f0a7f58f..996b55f9b8 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD
+++ b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
@@ -48,15 +48,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/resampler/BUILD b/tensorflow/contrib/resampler/BUILD
index f0ecc8b85a..48345d7030 100644
--- a/tensorflow/contrib/resampler/BUILD
+++ b/tensorflow/contrib/resampler/BUILD
@@ -85,14 +85,3 @@ cuda_py_test(
         "//tensorflow/python:array_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/rnn/BUILD b/tensorflow/contrib/rnn/BUILD
index 7e5e35d0b5..43c0f75955 100644
--- a/tensorflow/contrib/rnn/BUILD
+++ b/tensorflow/contrib/rnn/BUILD
@@ -321,19 +321,6 @@ tf_cc_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "tools/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_gen_op_libs(
     op_lib_names = [
         "lstm_ops",
diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index 245fe07f2b..faad40d335 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -81,15 +81,3 @@ py_test(
         "//tensorflow/python/saved_model:utils",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/saved_model/cc/saved_model/BUILD b/tensorflow/contrib/saved_model/cc/saved_model/BUILD
index ea4da80ba3..3c616c555b 100644
--- a/tensorflow/contrib/saved_model/cc/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/cc/saved_model/BUILD
@@ -49,9 +49,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(["*"]),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/seq2seq/BUILD b/tensorflow/contrib/seq2seq/BUILD
index ab80c68b1a..a62069a252 100644
--- a/tensorflow/contrib/seq2seq/BUILD
+++ b/tensorflow/contrib/seq2seq/BUILD
@@ -211,15 +211,3 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 75a753ed89..31717305e7 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -17,18 +17,6 @@ load(
     "tf_cc_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-)
-
 # TODO(b/32673259): add a test to continuously validate these files.
 filegroup(
     name = "session_bundle_half_plus_two",
diff --git a/tensorflow/contrib/session_bundle/example/BUILD b/tensorflow/contrib/session_bundle/example/BUILD
index dbbae01f36..9a56eab431 100644
--- a/tensorflow/contrib/session_bundle/example/BUILD
+++ b/tensorflow/contrib/session_bundle/example/BUILD
@@ -10,19 +10,6 @@ exports_files(["LICENSE"])
 
 # vardef("PYTHON_BIN_PATH", "/usr/bin/python")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-    visibility = ["//visibility:public"],
-)
-
 py_binary(
     name = "export_half_plus_two",
     srcs = [
diff --git a/tensorflow/contrib/signal/BUILD b/tensorflow/contrib/signal/BUILD
index a83fc20596..fdecceff52 100644
--- a/tensorflow/contrib/signal/BUILD
+++ b/tensorflow/contrib/signal/BUILD
@@ -130,15 +130,3 @@ cuda_py_tests(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/slim/BUILD b/tensorflow/contrib/slim/BUILD
index c2f106c2b2..516e3ea073 100644
--- a/tensorflow/contrib/slim/BUILD
+++ b/tensorflow/contrib/slim/BUILD
@@ -178,15 +178,3 @@ py_test(
         "//tensorflow/python:summary",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD
index 5daabbd62e..dc12e67fc6 100644
--- a/tensorflow/contrib/slim/python/slim/data/BUILD
+++ b/tensorflow/contrib/slim/python/slim/data/BUILD
@@ -193,15 +193,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/slim/python/slim/nets/BUILD b/tensorflow/contrib/slim/python/slim/nets/BUILD
index 7f03aaf085..8bbdf96384 100644
--- a/tensorflow/contrib/slim/python/slim/nets/BUILD
+++ b/tensorflow/contrib/slim/python/slim/nets/BUILD
@@ -317,15 +317,3 @@ py_test(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/solvers/BUILD b/tensorflow/contrib/solvers/BUILD
index 87b67486ad..5247288d54 100644
--- a/tensorflow/contrib/solvers/BUILD
+++ b/tensorflow/contrib/solvers/BUILD
@@ -93,16 +93,3 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-# All files
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/sparsemax/BUILD b/tensorflow/contrib/sparsemax/BUILD
index fcfaa2aba4..b729fff261 100644
--- a/tensorflow/contrib/sparsemax/BUILD
+++ b/tensorflow/contrib/sparsemax/BUILD
@@ -65,15 +65,3 @@ cuda_py_tests(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/specs/BUILD b/tensorflow/contrib/specs/BUILD
index 084953a0a2..055b04db8a 100644
--- a/tensorflow/contrib/specs/BUILD
+++ b/tensorflow/contrib/specs/BUILD
@@ -60,15 +60,3 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/staging/BUILD b/tensorflow/contrib/staging/BUILD
index bc4a289468..0c86f3db1d 100644
--- a/tensorflow/contrib/staging/BUILD
+++ b/tensorflow/contrib/staging/BUILD
@@ -6,18 +6,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "staging",
     srcs = ["__init__.py"],
diff --git a/tensorflow/contrib/stat_summarizer/BUILD b/tensorflow/contrib/stat_summarizer/BUILD
index 5fd02efbf6..d4096751c4 100644
--- a/tensorflow/contrib/stat_summarizer/BUILD
+++ b/tensorflow/contrib/stat_summarizer/BUILD
@@ -32,15 +32,3 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/stateless/BUILD b/tensorflow/contrib/stateless/BUILD
index 6e259e1d32..dcbef2881d 100644
--- a/tensorflow/contrib/stateless/BUILD
+++ b/tensorflow/contrib/stateless/BUILD
@@ -38,15 +38,3 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/summary/BUILD b/tensorflow/contrib/summary/BUILD
index 80563c5e15..fda1367b15 100644
--- a/tensorflow/contrib/summary/BUILD
+++ b/tensorflow/contrib/summary/BUILD
@@ -83,18 +83,6 @@ py_library(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # NOTE: target cannot be testonly because it needs to be in the pip
 # package. Sigh.
 py_library(
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 1e4cc3f095..11a59ec22b 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -16,20 +16,6 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "kernels/v4/*",
-            "proto/*",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # ---------------------------------- V2 ops ------------------------------------------#
 filegroup(
     name = "v2_op_sources",
diff --git a/tensorflow/contrib/tensor_forest/hybrid/BUILD b/tensorflow/contrib/tensor_forest/hybrid/BUILD
index a2a3b485f6..b7185e09c7 100644
--- a/tensorflow/contrib/tensor_forest/hybrid/BUILD
+++ b/tensorflow/contrib/tensor_forest/hybrid/BUILD
@@ -11,18 +11,6 @@ package(default_visibility = ["//visibility:public"])
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "custom_op_sources",
     srcs = glob(
diff --git a/tensorflow/contrib/tensor_forest/kernels/v4/BUILD b/tensorflow/contrib/tensor_forest/kernels/v4/BUILD
index 794b76d858..b1b1559383 100644
--- a/tensorflow/contrib/tensor_forest/kernels/v4/BUILD
+++ b/tensorflow/contrib/tensor_forest/kernels/v4/BUILD
@@ -11,11 +11,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(["**/*"]),
-)
-
 DECISION_TREE_RESOURCE_DEPS = [
     ":decision_node_evaluator",
     ":input_data",
diff --git a/tensorflow/contrib/tensor_forest/proto/BUILD b/tensorflow/contrib/tensor_forest/proto/BUILD
index 1cfef44af1..04fd6a9839 100644
--- a/tensorflow/contrib/tensor_forest/proto/BUILD
+++ b/tensorflow/contrib/tensor_forest/proto/BUILD
@@ -6,14 +6,6 @@ load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 
 package(default_visibility = ["//visibility:public"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "fertile_stats_proto",
     srcs = ["fertile_stats.proto"],
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index d833744d0c..f4efd9717d 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -88,15 +88,3 @@ py_test(
         "//tensorflow/python:platform",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/tensorboard/db/BUILD b/tensorflow/contrib/tensorboard/db/BUILD
index 4175d8adb5..3f6b4cdc9a 100644
--- a/tensorflow/contrib/tensorboard/db/BUILD
+++ b/tensorflow/contrib/tensorboard/db/BUILD
@@ -135,9 +135,3 @@ tf_cc_binary(
         "//tensorflow/core/lib/db:sqlite",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(["*"]),
-    visibility = ["//tensorflow:__pkg__"],
-)
diff --git a/tensorflow/contrib/tensorrt/BUILD b/tensorflow/contrib/tensorrt/BUILD
index 906cc3f034..2f316767b3 100644
--- a/tensorflow/contrib/tensorrt/BUILD
+++ b/tensorflow/contrib/tensorrt/BUILD
@@ -272,15 +272,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/testing/BUILD b/tensorflow/contrib/testing/BUILD
index 0be6aa755b..8a40e111d7 100644
--- a/tensorflow/contrib/testing/BUILD
+++ b/tensorflow/contrib/testing/BUILD
@@ -22,15 +22,3 @@ py_library(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/text/BUILD b/tensorflow/contrib/text/BUILD
index 698fdd830f..38d91f7e49 100644
--- a/tensorflow/contrib/text/BUILD
+++ b/tensorflow/contrib/text/BUILD
@@ -111,14 +111,3 @@ py_test(
         "//tensorflow/python:training",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/tfprof/BUILD b/tensorflow/contrib/tfprof/BUILD
index 28adce71d4..e7f4ebdd36 100644
--- a/tensorflow/contrib/tfprof/BUILD
+++ b/tensorflow/contrib/tfprof/BUILD
@@ -20,15 +20,3 @@ py_library(
         "//tensorflow/python/profiler:tfprof_logger",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/timeseries/BUILD b/tensorflow/contrib/timeseries/BUILD
index 6ba069778c..f2b8786a52 100644
--- a/tensorflow/contrib/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/BUILD
@@ -31,15 +31,3 @@ py_library(
         "//tensorflow/contrib/timeseries/python/timeseries/state_space_models:test_utils",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index bb86ecb220..40cf9147b3 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -106,15 +106,3 @@ py_test(
         "//tensorflow/python/estimator:estimator_py",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index ed3ed4c0e1..55a25e39fe 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -442,15 +442,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index c86d06e923..ca25ccd2b8 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -268,15 +268,3 @@ py_library(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 95dc6f5ced..3e32a7a85c 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -283,16 +283,3 @@ tf_py_test(
         "//tensorflow/python:framework_test_lib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/contrib/tpu/profiler/BUILD b/tensorflow/contrib/tpu/profiler/BUILD
index 56ddd7eff1..1c32993e8e 100644
--- a/tensorflow/contrib/tpu/profiler/BUILD
+++ b/tensorflow/contrib/tpu/profiler/BUILD
@@ -6,18 +6,6 @@ load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library_cc")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "tpu_profiler_proto",
     srcs = ["tpu_profiler.proto"],
diff --git a/tensorflow/contrib/tpu/proto/BUILD b/tensorflow/contrib/tpu/proto/BUILD
index e166098567..fcfbbe1a21 100644
--- a/tensorflow/contrib/tpu/proto/BUILD
+++ b/tensorflow/contrib/tpu/proto/BUILD
@@ -4,17 +4,6 @@ exports_files(["LICENSE"])
 
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "tpu_embedding_config_proto",
     srcs = [
diff --git a/tensorflow/contrib/training/BUILD b/tensorflow/contrib/training/BUILD
index 6ae2f38252..4d2bfd3e43 100644
--- a/tensorflow/contrib/training/BUILD
+++ b/tensorflow/contrib/training/BUILD
@@ -308,18 +308,6 @@ py_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_proto_library(
     name = "protos_all",
     srcs = glob(["**/*.proto"]),
diff --git a/tensorflow/contrib/util/BUILD b/tensorflow/contrib/util/BUILD
index 6c766e4f1c..d9ccda8e89 100644
--- a/tensorflow/contrib/util/BUILD
+++ b/tensorflow/contrib/util/BUILD
@@ -75,15 +75,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/contrib/verbs/BUILD b/tensorflow/contrib/verbs/BUILD
index 80a5d07ea4..9720fd6e86 100644
--- a/tensorflow/contrib/verbs/BUILD
+++ b/tensorflow/contrib/verbs/BUILD
@@ -11,18 +11,6 @@ load("//tensorflow:tensorflow.bzl", "tf_cuda_library")
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "c_srcs",
     data = glob([
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 712106492b..d46241450c 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -149,6 +149,8 @@ load(
     "if_mkl",
 )
 
+exports_files(["ops/ops.pbtxt"])
+
 # -----------------------------------------------------------------------------
 # Public targets
 
@@ -3851,18 +3853,6 @@ cc_library(
 # -----------------------------------------------------------------------------
 # Google-internal targets go here (must be at the end).
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 alias(
     name = "android_srcs_no_runtime",
     actual = ":mobile_srcs_no_runtime",
diff --git a/tensorflow/core/api_def/BUILD b/tensorflow/core/api_def/BUILD
index 58dbac4e8e..19d6438809 100644
--- a/tensorflow/core/api_def/BUILD
+++ b/tensorflow/core/api_def/BUILD
@@ -17,18 +17,6 @@ load(
     "tf_cc_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "base_api_def",
     srcs = glob(["base_api/*"]),
diff --git a/tensorflow/core/common_runtime/eager/BUILD b/tensorflow/core/common_runtime/eager/BUILD
index 9e8baab618..941a0e61c7 100644
--- a/tensorflow/core/common_runtime/eager/BUILD
+++ b/tensorflow/core/common_runtime/eager/BUILD
@@ -135,21 +135,6 @@ tf_cc_test(
     ],
 )
 
-# -----------------------------------------------------------------------------
-# Google-internal targets.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "execute",
     srcs = ["execute.cc"],
diff --git a/tensorflow/core/debug/BUILD b/tensorflow/core/debug/BUILD
index f6fe9edb02..5fab740e92 100644
--- a/tensorflow/core/debug/BUILD
+++ b/tensorflow/core/debug/BUILD
@@ -339,18 +339,3 @@ cc_library(
 #     ],
 #     visibility = ["//visibility:public"],
 # )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/core/distributed_runtime/BUILD b/tensorflow/core/distributed_runtime/BUILD
index 434626bd2d..b07cb8cdcb 100644
--- a/tensorflow/core/distributed_runtime/BUILD
+++ b/tensorflow/core/distributed_runtime/BUILD
@@ -7,18 +7,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "c_srcs",
     data = glob([
diff --git a/tensorflow/core/distributed_runtime/rpc/BUILD b/tensorflow/core/distributed_runtime/rpc/BUILD
index 9dae1b9859..9c655bfa31 100644
--- a/tensorflow/core/distributed_runtime/rpc/BUILD
+++ b/tensorflow/core/distributed_runtime/rpc/BUILD
@@ -5,18 +5,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "c_srcs",
     data = glob([
diff --git a/tensorflow/core/grappler/BUILD b/tensorflow/core/grappler/BUILD
index 2ca9b720ee..9dcc6765f5 100644
--- a/tensorflow/core/grappler/BUILD
+++ b/tensorflow/core/grappler/BUILD
@@ -3,18 +3,6 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cuda_library")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "op_types",
     srcs = ["op_types.cc"],
diff --git a/tensorflow/core/grappler/clusters/BUILD b/tensorflow/core/grappler/clusters/BUILD
index b653f902e8..9ecf5a6cf7 100644
--- a/tensorflow/core/grappler/clusters/BUILD
+++ b/tensorflow/core/grappler/clusters/BUILD
@@ -8,18 +8,6 @@ load(
     "tf_cuda_tests_tags",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 config_setting(
     name = "xsmm",
     licenses = ["notice"],
diff --git a/tensorflow/core/grappler/costs/BUILD b/tensorflow/core/grappler/costs/BUILD
index df5a26f475..33949319d5 100644
--- a/tensorflow/core/grappler/costs/BUILD
+++ b/tensorflow/core/grappler/costs/BUILD
@@ -6,18 +6,6 @@ load(
     "tf_protos_grappler",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "graph_properties_testdata",
     srcs = glob([
diff --git a/tensorflow/core/grappler/inputs/BUILD b/tensorflow/core/grappler/inputs/BUILD
index b683216590..ffa204028c 100644
--- a/tensorflow/core/grappler/inputs/BUILD
+++ b/tensorflow/core/grappler/inputs/BUILD
@@ -2,18 +2,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "utils",
     srcs = [
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 19ff788aba..0d3a488f85 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -12,18 +12,6 @@ load(
     "tf_protos_grappler",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "static_schedule",
     srcs = ["static_schedule.cc"],
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index 939031c44b..baf24c2505 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -2,18 +2,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "scc",
     srcs = ["scc.cc"],
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index b469c01881..ca54978421 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6147,18 +6147,6 @@ tf_kernel_library(
 # -----------------------------------------------------------------------------
 # Google-internal targets.  These must be at the end for syncrepo.
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 # Library to link with when compiling the cwise_op kernels directly,
 # e.g. for selective registration.
 # should not be linked by projects that also link the cwise_op library.
diff --git a/tensorflow/core/kernels/batching_util/BUILD b/tensorflow/core/kernels/batching_util/BUILD
index 4397410a5c..de05c647d6 100644
--- a/tensorflow/core/kernels/batching_util/BUILD
+++ b/tensorflow/core/kernels/batching_util/BUILD
@@ -8,18 +8,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "**/google_*",
-        ],
-    ),
-)
-
 cc_library(
     name = "periodic_function_dynamic",
     srcs = ["periodic_function.cc"],
diff --git a/tensorflow/core/kernels/data/BUILD b/tensorflow/core/kernels/data/BUILD
index a8784e3656..8c4f0218ee 100644
--- a/tensorflow/core/kernels/data/BUILD
+++ b/tensorflow/core/kernels/data/BUILD
@@ -13,18 +13,6 @@ load(
     "tf_cc_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "stats_aggregator",
     hdrs = ["stats_aggregator.h"],
diff --git a/tensorflow/core/kernels/data/sql/BUILD b/tensorflow/core/kernels/data/sql/BUILD
index f4698bdaf7..dc59120875 100644
--- a/tensorflow/core/kernels/data/sql/BUILD
+++ b/tensorflow/core/kernels/data/sql/BUILD
@@ -7,18 +7,6 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "sql",
     srcs = [
diff --git a/tensorflow/core/kernels/fuzzing/BUILD b/tensorflow/core/kernels/fuzzing/BUILD
index 9a7eca03ce..aab4b009b5 100644
--- a/tensorflow/core/kernels/fuzzing/BUILD
+++ b/tensorflow/core/kernels/fuzzing/BUILD
@@ -17,18 +17,6 @@ cc_library(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 load("//tensorflow/core/kernels/fuzzing:tf_ops_fuzz_target_lib.bzl", "tf_ops_fuzz_target_lib")
 
 tf_ops_fuzz_target_lib("identity")
diff --git a/tensorflow/core/kernels/hexagon/BUILD b/tensorflow/core/kernels/hexagon/BUILD
index 7688305019..4870d9ae20 100644
--- a/tensorflow/core/kernels/hexagon/BUILD
+++ b/tensorflow/core/kernels/hexagon/BUILD
@@ -13,18 +13,6 @@ load(
     "tf_kernel_library",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_cc_test(
     name = "graph_transferer_test",
     size = "small",
diff --git a/tensorflow/core/kernels/neon/BUILD b/tensorflow/core/kernels/neon/BUILD
index c3d24e50ef..313d40c082 100644
--- a/tensorflow/core/kernels/neon/BUILD
+++ b/tensorflow/core/kernels/neon/BUILD
@@ -12,18 +12,6 @@ load(
     "tf_kernel_library",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_kernel_library(
     name = "neon_depthwise_conv_op",
     hdrs = [
diff --git a/tensorflow/core/lib/db/BUILD b/tensorflow/core/lib/db/BUILD
index 9ff87e8d66..ce09c2009a 100644
--- a/tensorflow/core/lib/db/BUILD
+++ b/tensorflow/core/lib/db/BUILD
@@ -42,9 +42,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(["*"]),
-    visibility = ["//tensorflow:__pkg__"],
-)
diff --git a/tensorflow/core/ops/compat/BUILD b/tensorflow/core/ops/compat/BUILD
index 6cdb1586bc..c613ab144f 100644
--- a/tensorflow/core/ops/compat/BUILD
+++ b/tensorflow/core/ops/compat/BUILD
@@ -57,18 +57,3 @@ tf_cc_binary(
         "//tensorflow/core:lib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index 21636641e7..3ee7be3c4e 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -14,20 +14,6 @@ load(
     "if_windows",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        include = [
-            "**/*",
-        ],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "expiring_lru_cache",
     hdrs = ["expiring_lru_cache.h"],
diff --git a/tensorflow/core/platform/default/build_config/BUILD b/tensorflow/core/platform/default/build_config/BUILD
index afb1d84d14..447056eb4b 100644
--- a/tensorflow/core/platform/default/build_config/BUILD
+++ b/tensorflow/core/platform/default/build_config/BUILD
@@ -223,15 +223,3 @@ alias(
     actual = ":mobile_srcs",
     visibility = ["//visibility:public"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/core/platform/hadoop/BUILD b/tensorflow/core/platform/hadoop/BUILD
index 774a439855..7c38c399bd 100644
--- a/tensorflow/core/platform/hadoop/BUILD
+++ b/tensorflow/core/platform/hadoop/BUILD
@@ -12,18 +12,6 @@ load(
     "tf_cc_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "hadoop_file_system",
     srcs = ["hadoop_file_system.cc"],
diff --git a/tensorflow/core/platform/s3/BUILD b/tensorflow/core/platform/s3/BUILD
index 3a0ad2e9bd..21038cfeb1 100644
--- a/tensorflow/core/platform/s3/BUILD
+++ b/tensorflow/core/platform/s3/BUILD
@@ -13,18 +13,6 @@ load(
     "tf_cc_test",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 tf_cc_binary(
     name = "s3_file_system.so",
     srcs = [
diff --git a/tensorflow/core/profiler/BUILD b/tensorflow/core/profiler/BUILD
index 5ce6f1046d..3d3203cdaa 100644
--- a/tensorflow/core/profiler/BUILD
+++ b/tensorflow/core/profiler/BUILD
@@ -4,21 +4,6 @@ package(
 
 licenses(["notice"])  # Apache 2.0
 
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_additional_all_protos")
diff --git a/tensorflow/core/profiler/internal/BUILD b/tensorflow/core/profiler/internal/BUILD
index 05a798bff8..8dcfde9a2a 100644
--- a/tensorflow/core/profiler/internal/BUILD
+++ b/tensorflow/core/profiler/internal/BUILD
@@ -365,17 +365,3 @@ cc_library(
         "//tensorflow/core:regexp_internal",
     ],
 )
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/core/profiler/internal/advisor/BUILD b/tensorflow/core/profiler/internal/advisor/BUILD
index 40cfd1e12e..1fedb05ae3 100644
--- a/tensorflow/core/profiler/internal/advisor/BUILD
+++ b/tensorflow/core/profiler/internal/advisor/BUILD
@@ -73,18 +73,3 @@ tf_cc_test(
         "//tensorflow/core/profiler/internal:tfprof_tf_testlib",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/core/util/ctc/BUILD b/tensorflow/core/util/ctc/BUILD
index 1521349e4d..317420204e 100644
--- a/tensorflow/core/util/ctc/BUILD
+++ b/tensorflow/core/util/ctc/BUILD
@@ -26,18 +26,6 @@ alias(
     actual = ":mobile_srcs",
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "ctc",
     deps = [
diff --git a/tensorflow/core/util/tensor_bundle/BUILD b/tensorflow/core/util/tensor_bundle/BUILD
index 166bd0f659..648358606c 100644
--- a/tensorflow/core/util/tensor_bundle/BUILD
+++ b/tensorflow/core/util/tensor_bundle/BUILD
@@ -75,18 +75,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/adding_an_op/BUILD b/tensorflow/examples/adding_an_op/BUILD
index b3ed6589ed..cf8054be6a 100644
--- a/tensorflow/examples/adding_an_op/BUILD
+++ b/tensorflow/examples/adding_an_op/BUILD
@@ -139,15 +139,3 @@ tf_cc_binary(
         "//tensorflow/core:framework",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/android/BUILD b/tensorflow/examples/android/BUILD
index 1214647797..a088d7cf2f 100644
--- a/tensorflow/examples/android/BUILD
+++ b/tensorflow/examples/android/BUILD
@@ -100,22 +100,6 @@ filegroup(
 )
 # LINT.ThenChange(//tensorflow/examples/android/download-models.gradle)
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-            "gradleBuild/**",
-            "libs/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 filegroup(
     name = "java_files",
     srcs = glob(["src/**/*.java"]),
diff --git a/tensorflow/examples/benchmark/BUILD b/tensorflow/examples/benchmark/BUILD
index c4bb0a5bd9..98611a9aad 100644
--- a/tensorflow/examples/benchmark/BUILD
+++ b/tensorflow/examples/benchmark/BUILD
@@ -23,9 +23,3 @@ tf_py_logged_benchmark(
     name = "sample_logged_benchmark",
     target = "//tensorflow/examples/benchmark:sample_benchmark",
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(["**/*"]),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/get_started/regression/BUILD b/tensorflow/examples/get_started/regression/BUILD
index 577b970c90..bee94d7d90 100644
--- a/tensorflow/examples/get_started/regression/BUILD
+++ b/tensorflow/examples/get_started/regression/BUILD
@@ -2,18 +2,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_test(
     name = "test",
     size = "medium",
diff --git a/tensorflow/examples/how_tos/reading_data/BUILD b/tensorflow/examples/how_tos/reading_data/BUILD
index 4a43585d53..64a054d371 100644
--- a/tensorflow/examples/how_tos/reading_data/BUILD
+++ b/tensorflow/examples/how_tos/reading_data/BUILD
@@ -54,15 +54,3 @@ py_binary(
         "//tensorflow/examples/tutorials/mnist:input_data",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/image_retraining/BUILD b/tensorflow/examples/image_retraining/BUILD
index 9f9244a74c..ecd79a3b00 100644
--- a/tensorflow/examples/image_retraining/BUILD
+++ b/tensorflow/examples/image_retraining/BUILD
@@ -49,15 +49,3 @@ py_test(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/label_image/BUILD b/tensorflow/examples/label_image/BUILD
index 2abbe9dacc..c50fd93d03 100644
--- a/tensorflow/examples/label_image/BUILD
+++ b/tensorflow/examples/label_image/BUILD
@@ -9,6 +9,8 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
+exports_files(["data/grace_hopper.jpg"])
+
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 
 tf_cc_binary(
@@ -60,17 +62,3 @@ py_binary(
         "//tensorflow:tensorflow_py",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/learn/BUILD b/tensorflow/examples/learn/BUILD
index aba7f600b5..bdbcb0b163 100644
--- a/tensorflow/examples/learn/BUILD
+++ b/tensorflow/examples/learn/BUILD
@@ -152,15 +152,3 @@ sh_test(
         "notap",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/multibox_detector/BUILD b/tensorflow/examples/multibox_detector/BUILD
index 91a5bfa51c..4f9908cd52 100644
--- a/tensorflow/examples/multibox_detector/BUILD
+++ b/tensorflow/examples/multibox_detector/BUILD
@@ -27,17 +27,3 @@ tf_cc_binary(
         "//tensorflow/core:tensorflow",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/saved_model/BUILD b/tensorflow/examples/saved_model/BUILD
index 1cdf5ec6e1..ebefc6576d 100644
--- a/tensorflow/examples/saved_model/BUILD
+++ b/tensorflow/examples/saved_model/BUILD
@@ -8,19 +8,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "g3doc/sitemap.md",
-        ],
-    ),
-    visibility = ["//visibility:public"],
-)
-
 py_binary(
     name = "saved_model_half_plus_two",
     srcs = [
diff --git a/tensorflow/examples/speech_commands/BUILD b/tensorflow/examples/speech_commands/BUILD
index 12479211c3..13bca34a86 100644
--- a/tensorflow/examples/speech_commands/BUILD
+++ b/tensorflow/examples/speech_commands/BUILD
@@ -245,15 +245,3 @@ tf_cc_binary(
         "//tensorflow/core:protos_all_cc",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/tutorials/estimators/BUILD b/tensorflow/examples/tutorials/estimators/BUILD
index ecbc1a431d..bab609f208 100644
--- a/tensorflow/examples/tutorials/estimators/BUILD
+++ b/tensorflow/examples/tutorials/estimators/BUILD
@@ -20,15 +20,3 @@ py_binary(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/tutorials/layers/BUILD b/tensorflow/examples/tutorials/layers/BUILD
index f8a29c79c6..aad78b1840 100644
--- a/tensorflow/examples/tutorials/layers/BUILD
+++ b/tensorflow/examples/tutorials/layers/BUILD
@@ -19,15 +19,3 @@ py_binary(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/tutorials/mnist/BUILD b/tensorflow/examples/tutorials/mnist/BUILD
index 6d4e67063d..aa1b2ec2db 100644
--- a/tensorflow/examples/tutorials/mnist/BUILD
+++ b/tensorflow/examples/tutorials/mnist/BUILD
@@ -132,15 +132,3 @@ py_test(
         "//tensorflow:tensorflow_py",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/tutorials/monitors/BUILD b/tensorflow/examples/tutorials/monitors/BUILD
index 4220e8144d..1c49e3fe53 100644
--- a/tensorflow/examples/tutorials/monitors/BUILD
+++ b/tensorflow/examples/tutorials/monitors/BUILD
@@ -23,15 +23,3 @@ py_binary(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/examples/tutorials/word2vec/BUILD b/tensorflow/examples/tutorials/word2vec/BUILD
index bfcf459269..2e19c038bd 100644
--- a/tensorflow/examples/tutorials/word2vec/BUILD
+++ b/tensorflow/examples/tutorials/word2vec/BUILD
@@ -21,14 +21,3 @@ py_binary(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/examples/wav_to_spectrogram/BUILD b/tensorflow/examples/wav_to_spectrogram/BUILD
index c99870c686..cc8835728d 100644
--- a/tensorflow/examples/wav_to_spectrogram/BUILD
+++ b/tensorflow/examples/wav_to_spectrogram/BUILD
@@ -49,17 +49,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/java/BUILD b/tensorflow/java/BUILD
index 5a533e3b60..acaf1a44eb 100644
--- a/tensorflow/java/BUILD
+++ b/tensorflow/java/BUILD
@@ -388,15 +388,3 @@ genrule(
     cmd = "cp $< $@",
     output_to_bindir = 1,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 09c1965d7e..0c3c3c4e06 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4389,18 +4389,6 @@ py_test(
     ],
 )
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cuda_py_test(
     name = "accumulate_n_benchmark",
     size = "large",
diff --git a/tensorflow/python/data/BUILD b/tensorflow/python/data/BUILD
index b5bee36dcd..3e08c1587e 100644
--- a/tensorflow/python/data/BUILD
+++ b/tensorflow/python/data/BUILD
@@ -15,15 +15,3 @@ py_library(
         "//tensorflow/python/data/ops:readers",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/data/kernel_tests/BUILD b/tensorflow/python/data/kernel_tests/BUILD
index 8b8adefa65..ed0c11e6c1 100644
--- a/tensorflow/python/data/kernel_tests/BUILD
+++ b/tensorflow/python/data/kernel_tests/BUILD
@@ -367,15 +367,3 @@ tf_py_test(
         "no_windows",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/data/ops/BUILD b/tensorflow/python/data/ops/BUILD
index 3119ab0037..fa2e86eab1 100644
--- a/tensorflow/python/data/ops/BUILD
+++ b/tensorflow/python/data/ops/BUILD
@@ -59,15 +59,3 @@ py_library(
         "//tensorflow/python/eager:context",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/data/util/BUILD b/tensorflow/python/data/util/BUILD
index b1bdbdab37..0fc32d51b9 100644
--- a/tensorflow/python/data/util/BUILD
+++ b/tensorflow/python/data/util/BUILD
@@ -109,15 +109,3 @@ py_test(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 512d292ee2..4195586313 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -1095,15 +1095,3 @@ sh_test(
         ":offline_analyzer",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/eager/BUILD b/tensorflow/python/eager/BUILD
index 0e089a26eb..8c0d3feece 100644
--- a/tensorflow/python/eager/BUILD
+++ b/tensorflow/python/eager/BUILD
@@ -398,21 +398,6 @@ py_test(
     ],
 )
 
-# -----------------------------------------------------------------------------
-# Google-internal targets.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "imperative_grad",
     srcs = ["imperative_grad.py"],
diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 5afb5a7dd5..1fcff18a3a 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -9,18 +9,6 @@ licenses(["notice"])  # Apache 2.0
 load("//tensorflow:tensorflow.bzl", "py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "estimator_py",
     srcs = ["estimator_lib.py"],
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 238a90b67d..0ae9900a1d 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -6,18 +6,6 @@ licenses(["notice"])  # Apache 2.0
 
 load("//tensorflow:tensorflow.bzl", "py_test")
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_library(
     name = "feature_column_py",
     srcs = ["feature_column_lib.py"],
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 16033e9b8f..2a06907f49 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -868,15 +868,3 @@ py_library(
         "//third_party/py/numpy",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 05f34db14b..ea210346c1 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2945,15 +2945,3 @@ tf_py_test(
         "//tensorflow/python/eager:tape",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/kernel_tests/distributions/BUILD b/tensorflow/python/kernel_tests/distributions/BUILD
index e220d05692..f3cc9636f9 100644
--- a/tensorflow/python/kernel_tests/distributions/BUILD
+++ b/tensorflow/python/kernel_tests/distributions/BUILD
@@ -280,15 +280,3 @@ cuda_py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/kernel_tests/linalg/BUILD b/tensorflow/python/kernel_tests/linalg/BUILD
index fd1b5bab6f..9555e51099 100644
--- a/tensorflow/python/kernel_tests/linalg/BUILD
+++ b/tensorflow/python/kernel_tests/linalg/BUILD
@@ -140,15 +140,3 @@ cuda_py_test(
     ],
     shard_count = 5,
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/kernel_tests/random/BUILD b/tensorflow/python/kernel_tests/random/BUILD
index 88a4ddf7f2..acd7566eec 100644
--- a/tensorflow/python/kernel_tests/random/BUILD
+++ b/tensorflow/python/kernel_tests/random/BUILD
@@ -121,15 +121,3 @@ cuda_py_test(
         "//tensorflow/python:random_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/ops/distributions/BUILD b/tensorflow/python/ops/distributions/BUILD
index 50b956a267..9d9ede7ad7 100644
--- a/tensorflow/python/ops/distributions/BUILD
+++ b/tensorflow/python/ops/distributions/BUILD
@@ -26,15 +26,3 @@ py_library(
         "@six_archive//:six",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/ops/linalg/BUILD b/tensorflow/python/ops/linalg/BUILD
index ce8c1580fe..07659ef44c 100644
--- a/tensorflow/python/ops/linalg/BUILD
+++ b/tensorflow/python/ops/linalg/BUILD
@@ -34,15 +34,3 @@ py_library(
         "//tensorflow/python:special_math_ops",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/ops/losses/BUILD b/tensorflow/python/ops/losses/BUILD
index 07741e0c3c..4aea0265a7 100644
--- a/tensorflow/python/ops/losses/BUILD
+++ b/tensorflow/python/ops/losses/BUILD
@@ -43,15 +43,3 @@ py_test(
         "//tensorflow/python:framework_for_generated_wrappers",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/profiler/BUILD b/tensorflow/python/profiler/BUILD
index c815aad0a0..0654104a34 100644
--- a/tensorflow/python/profiler/BUILD
+++ b/tensorflow/python/profiler/BUILD
@@ -156,18 +156,3 @@ py_test(
         "@com_google_pprof//:pprof_proto_py",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/profiler/internal/BUILD b/tensorflow/python/profiler/internal/BUILD
index 362a1c49e6..994206cd63 100644
--- a/tensorflow/python/profiler/internal/BUILD
+++ b/tensorflow/python/profiler/internal/BUILD
@@ -70,18 +70,3 @@ cuda_py_test(
         "no_pip",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/saved_model/BUILD b/tensorflow/python/saved_model/BUILD
index 30e0a099d8..2609a5d222 100644
--- a/tensorflow/python/saved_model/BUILD
+++ b/tensorflow/python/saved_model/BUILD
@@ -235,15 +235,3 @@ py_test(
 
 # -----------------------------------------------------------------------------
 # Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/python/tools/BUILD b/tensorflow/python/tools/BUILD
index 1de1adcfbc..6e39ce8c80 100644
--- a/tensorflow/python/tools/BUILD
+++ b/tensorflow/python/tools/BUILD
@@ -258,17 +258,3 @@ py_test(
         "//tensorflow/core:protos_all_py",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-            "bin/**",
-            "gen/**",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD
index d9b0260c9f..6722536358 100644
--- a/tensorflow/tools/api/generator/BUILD
+++ b/tensorflow/tools/api/generator/BUILD
@@ -5,18 +5,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 py_binary(
     name = "create_python_api",
     srcs = ["create_python_api.py"],
diff --git a/tensorflow/tools/api/golden/BUILD b/tensorflow/tools/api/golden/BUILD
index 08436396a6..ebdf42df2c 100644
--- a/tensorflow/tools/api/golden/BUILD
+++ b/tensorflow/tools/api/golden/BUILD
@@ -10,15 +10,3 @@ filegroup(
     name = "api_golden",
     srcs = glob(["*.pbtxt"]),
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/api/lib/BUILD b/tensorflow/tools/api/lib/BUILD
index 2d3b838957..3f4fb91042 100644
--- a/tensorflow/tools/api/lib/BUILD
+++ b/tensorflow/tools/api/lib/BUILD
@@ -26,15 +26,3 @@ py_library(
         "//tensorflow/python:util",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD
index 15bf1abb5f..0dc154b6d2 100644
--- a/tensorflow/tools/api/tests/BUILD
+++ b/tensorflow/tools/api/tests/BUILD
@@ -42,15 +42,3 @@ tf_cc_binary(
         "//tensorflow/core:op_gen_lib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/benchmark/BUILD b/tensorflow/tools/benchmark/BUILD
index 6ed2594e6a..566a172ea7 100644
--- a/tensorflow/tools/benchmark/BUILD
+++ b/tensorflow/tools/benchmark/BUILD
@@ -90,12 +90,3 @@ tf_cc_binary(
     visibility = ["//visibility:public"],
     deps = [":benchmark_model_lib"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = ["**/OWNERS"],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/build_info/BUILD b/tensorflow/tools/build_info/BUILD
index cdc47076ce..7307417805 100644
--- a/tensorflow/tools/build_info/BUILD
+++ b/tensorflow/tools/build_info/BUILD
@@ -9,18 +9,3 @@ exports_files(
         "gen_build_info.py",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/common/BUILD b/tensorflow/tools/common/BUILD
index 316e5469e7..b9032c046e 100644
--- a/tensorflow/tools/common/BUILD
+++ b/tensorflow/tools/common/BUILD
@@ -44,14 +44,3 @@ py_test(
         "//tensorflow/python:platform_test",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/tools/compatibility/BUILD b/tensorflow/tools/compatibility/BUILD
index 4f90c4d940..b7bfb29aae 100644
--- a/tensorflow/tools/compatibility/BUILD
+++ b/tensorflow/tools/compatibility/BUILD
@@ -68,18 +68,3 @@ exports_files(
         "testdata/test_file_v0_11.py",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/dist_test/server/BUILD b/tensorflow/tools/dist_test/server/BUILD
index 865af8dd7b..003a19a9ab 100644
--- a/tensorflow/tools/dist_test/server/BUILD
+++ b/tensorflow/tools/dist_test/server/BUILD
@@ -37,15 +37,3 @@ py_test(
         "//tensorflow/python:client_testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/docker/BUILD b/tensorflow/tools/docker/BUILD
index 7d5ae0a94d..849ba49f71 100644
--- a/tensorflow/tools/docker/BUILD
+++ b/tensorflow/tools/docker/BUILD
@@ -13,15 +13,3 @@ py_binary(
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/docker/notebooks/BUILD b/tensorflow/tools/docker/notebooks/BUILD
index 89f473df4b..e9f26899c9 100644
--- a/tensorflow/tools/docker/notebooks/BUILD
+++ b/tensorflow/tools/docker/notebooks/BUILD
@@ -3,15 +3,3 @@ package(default_visibility = ["//visibility:private"])
 licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE"])
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index 8f10bc9e0c..d370fbd246 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -142,14 +142,3 @@ py_test(
         "//tensorflow/python:client_testlib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/tools/git/BUILD b/tensorflow/tools/git/BUILD
index 942ceab85f..daa17fbd50 100644
--- a/tensorflow/tools/git/BUILD
+++ b/tensorflow/tools/git/BUILD
@@ -9,18 +9,3 @@ licenses(["notice"])  # Apache 2.0
 exports_files(
     ["gen_git_source.py"],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/graph_transforms/BUILD b/tensorflow/tools/graph_transforms/BUILD
index 6e21aa2846..1ad1895269 100644
--- a/tensorflow/tools/graph_transforms/BUILD
+++ b/tensorflow/tools/graph_transforms/BUILD
@@ -313,14 +313,3 @@ tf_py_test(
     ],
     main = "python/transform_graph_test.py",
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-)
diff --git a/tensorflow/tools/mlpbtxt/BUILD b/tensorflow/tools/mlpbtxt/BUILD
index f9f48c6500..89c683c8c4 100644
--- a/tensorflow/tools/mlpbtxt/BUILD
+++ b/tensorflow/tools/mlpbtxt/BUILD
@@ -32,15 +32,3 @@ tf_cc_binary(
         "//tensorflow/core:op_gen_lib",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/proto_text/BUILD b/tensorflow/tools/proto_text/BUILD
index 39c4aac1e8..ef7bfdd3c9 100644
--- a/tensorflow/tools/proto_text/BUILD
+++ b/tensorflow/tools/proto_text/BUILD
@@ -96,18 +96,3 @@ tf_cc_test(
         "//tensorflow/core:test_main",
     ],
 )
-
-# -----------------------------------------------------------------------------
-# Google-internal targets.  These must be at the end for syncrepo.
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/quantization/BUILD b/tensorflow/tools/quantization/BUILD
index e99ad06a06..17443a8617 100644
--- a/tensorflow/tools/quantization/BUILD
+++ b/tensorflow/tools/quantization/BUILD
@@ -76,15 +76,3 @@ py_binary(
         "//tensorflow/python:platform",
     ],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/tools/test/BUILD b/tensorflow/tools/test/BUILD
index 159a8c1cfb..4b2026b947 100644
--- a/tensorflow/tools/test/BUILD
+++ b/tensorflow/tools/test/BUILD
@@ -92,15 +92,3 @@ tf_py_logged_benchmark(
     name = "rnn_op_benchmark",
     target = "//tensorflow/python/kernel_tests:rnn_test",
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/tensorflow/user_ops/BUILD b/tensorflow/user_ops/BUILD
index e8198efe2e..71443cc41e 100644
--- a/tensorflow/user_ops/BUILD
+++ b/tensorflow/user_ops/BUILD
@@ -50,15 +50,3 @@ tf_py_test(
     additional_deps = ["//tensorflow:tensorflow_py"],
     data = [":invalid_op.so"],
 )
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/third_party/hadoop/BUILD b/third_party/hadoop/BUILD
index 9e98154400..c3c5e428be 100644
--- a/third_party/hadoop/BUILD
+++ b/third_party/hadoop/BUILD
@@ -4,18 +4,6 @@ licenses(["notice"])  # Apache 2.0
 
 exports_files(["LICENSE.txt"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 cc_library(
     name = "hdfs",
     hdrs = ["hdfs.h"],
diff --git a/third_party/mpi/BUILD b/third_party/mpi/BUILD
index ff3f437e92..1d6ac2fceb 100644
--- a/third_party/mpi/BUILD
+++ b/third_party/mpi/BUILD
@@ -1,17 +1,5 @@
 licenses(["restricted"])
 
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-
 load("//third_party/mpi:mpi.bzl", "mpi_hdr")
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 
diff --git a/third_party/sycl/BUILD b/third_party/sycl/BUILD
index fbdf19f205..f631b6df06 100644
--- a/third_party/sycl/BUILD
+++ b/third_party/sycl/BUILD
@@ -1,15 +1,3 @@
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
diff --git a/third_party/sycl/sycl/BUILD b/third_party/sycl/sycl/BUILD
index bc1d18b7b5..b045609954 100644
--- a/third_party/sycl/sycl/BUILD
+++ b/third_party/sycl/sycl/BUILD
@@ -5,15 +5,3 @@
 package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # Apache 2.0
-
-filegroup(
-    name = "all_files",
-    srcs = glob(
-        ["**/*"],
-        exclude = [
-            "**/METADATA",
-            "**/OWNERS",
-        ],
-    ),
-    visibility = ["//tensorflow:__subpackages__"],
-)
-- 
GitLab


From 3f7adc710495e1160acd956c482779247ef1f101 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Wed, 28 Mar 2018 20:51:01 -0700
Subject: [PATCH 660/960] Support structured source in GradientTape.gradient

Before this change, it was easy to forget [] around the source tensor.
This mistake lead to GradientTape.gradient(), returning a list of Nones.
Nones normally tell to the user that the source and the target are
not connected via differentiable operations, which is not the source
of the error in this case.

Instead of adding a check that `sources` is a list of tensors, this CL
adds ability to handle structured source (which includes a lone tensor),
similarly to many existing TensorFlow APIs.

Also, with Alex's help, it fixes a bug where repeated tensors in
`sources` were not handled correctly.

PiperOrigin-RevId: 190878583
---
 tensorflow/c/eager/tape.h                 | 21 ++++++----
 tensorflow/python/eager/backprop.py       | 33 ++++++++++------
 tensorflow/python/eager/backprop_test.py  | 47 +++++++++++++++++++++++
 tensorflow/python/eager/pywrap_tfe_src.cc |  4 ++
 4 files changed, 85 insertions(+), 20 deletions(-)

diff --git a/tensorflow/c/eager/tape.h b/tensorflow/c/eager/tape.h
index c7bd3bdafd..97c323b872 100644
--- a/tensorflow/c/eager/tape.h
+++ b/tensorflow/c/eager/tape.h
@@ -601,23 +601,28 @@ Status GradientTape<Gradient, BackwardFunction>::ComputeGradient(
   }
   CHECK(state.op_tape.empty());
   result->reserve(source_tensor_ids.size());
+  gtl::FlatSet<int64> used_gradient_ids(source_tensor_ids.size());
   for (auto is : source_tensor_ids) {
     auto grad_it = gradients.find(is);
     if (grad_it == gradients.end()) {
       result->push_back(nullptr);
     } else {
-      if (grad_it->second.size() == 1) {
-        result->push_back(grad_it->second[0]);
-      } else {
-        result->push_back(vspace.AggregateGradients(grad_it->second));
+      if (grad_it->second.size() > 1) {
+        Gradient* grad = vspace.AggregateGradients(grad_it->second);
+        grad_it->second.clear();
+        grad_it->second.push_back(grad);
       }
-      gradients.erase(grad_it);
+      result->push_back(grad_it->second[0]);
+      used_gradient_ids.insert(is);
     }
   }
-  VLOG(1) << "Final gradients size: " << gradients.size();
+  VLOG(1) << "Final gradients size: "
+          << gradients.size() - used_gradient_ids.size();
   for (auto grad_pair : gradients) {
-    for (const auto& g : grad_pair.second) {
-      vspace.DeleteGradient(g);
+    if (used_gradient_ids.find(grad_pair.first) == used_gradient_ids.end()) {
+      for (const auto& g : grad_pair.second) {
+        vspace.DeleteGradient(g);
+      }
     }
   }
   return Status::OK();
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index c54a5a1445..209b012621 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -646,6 +646,13 @@ _default_vspace = imperative_grad.VSpace(
     ones=_ones)
 
 
+def _handle_or_self(x):
+  """If x is ResourceVariable, return its handle, else x."""
+  if isinstance(x, resource_variable_ops.ResourceVariable):
+    x = x.handle
+  return x
+
+
 @tf_export("GradientTape")
 class GradientTape(object):
   """Record operations for automatic differentiation.
@@ -723,9 +730,7 @@ class GradientTape(object):
       tensor: a Tensor or list of Tensors.
     """
     for t in nest.flatten(tensor):
-      if isinstance(t, resource_variable_ops.ResourceVariable):
-        t = t.handle
-      tape.watch(t)
+      tape.watch(_handle_or_self(t))
 
   def watched_variables(self):
     # Sorting variables by id, which is monotonically increasing in construction
@@ -739,14 +744,15 @@ class GradientTape(object):
 
     Args:
       target: Tensor to be differentiated.
-      sources: a list of Tensors or Variables. `target` will be differentiated
-        against elements in `sources`.
+      sources: a list or nested structure of Tensors or Variables. `target`
+        will be differentiated against elements in `sources`.
       output_gradients: a list of gradients, one for each element of
         target. Defaults to None.
 
     Returns:
-      a list of Tensors (or IndexedSlices, or None), one for each element in
-      `sources`.
+      a list or nested structure of Tensors (or IndexedSlices, or None),
+      one for each element in `sources`. Returned structure is the same as
+      the structure of `sources`.
 
     Raises:
       RuntimeError: if called inside the context of the tape, or if called more
@@ -756,12 +762,15 @@ class GradientTape(object):
       raise RuntimeError("GradientTape.gradient can only be called once "
                          "on non-persistent tapes, and "
                          "only when the context manager has exited.")
-    sources = [x.handle if isinstance(x, resource_variable_ops.ResourceVariable)
-               else x
-               for x in sources]
-    grad = imperative_grad.imperative_grad(
-        _default_vspace, self._tape, [target], sources,
+    flat_sources = nest.flatten(sources)
+    flat_sources = [_handle_or_self(x) for x in flat_sources]
+
+    flat_grad = imperative_grad.imperative_grad(
+        _default_vspace, self._tape, [target], flat_sources,
         output_gradients=output_gradients)
+
     if not self._persistent:
       self._tape = None
+
+    grad = nest.pack_sequence_as(sources, flat_grad)
     return grad
diff --git a/tensorflow/python/eager/backprop_test.py b/tensorflow/python/eager/backprop_test.py
index f04d89a6d9..991b4dbe7a 100644
--- a/tensorflow/python/eager/backprop_test.py
+++ b/tensorflow/python/eager/backprop_test.py
@@ -369,6 +369,53 @@ class BackpropTest(test.TestCase):
 
     self.assertEqual(backprop.implicit_grad(f)()[0][0], None)
 
+  @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
+  def testGradientTapeRepeatedSource(self):
+    with backprop.GradientTape(persistent=False) as g:
+      x = constant_op.constant(3.0)
+      g.watch(x)
+      y = 2 * x
+    grad = g.gradient(target=y, sources=[x, x])
+    self.assertEqual(self.evaluate(grad), [2.0, 2.0])
+
+  @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
+  def testPersistentGradientTapeRepeatedSource(self):
+    with backprop.GradientTape(persistent=True) as g:
+      x = constant_op.constant(3.0)
+      y = constant_op.constant(5.0)
+      g.watch(x)
+      g.watch(y)
+      z = x * x + x * y
+    grad = g.gradient(target=z, sources=[x, x])
+    self.assertEqual(self.evaluate(grad), [11.0, 11.0])
+    grad = g.gradient(target=z, sources=[y, x])
+    self.assertEqual(self.evaluate(grad), [3.0, 11.0])
+
+  @test_util.assert_no_new_tensors
+  @test_util.run_in_graph_and_eager_modes()
+  def testGradientTapeStructure(self):
+    with backprop.GradientTape(persistent=True) as g:
+      # Using different constant values because constant tensors are
+      # cached, leading to a different gradient then what one might expect.
+      x1 = constant_op.constant(3.0)
+      x2 = constant_op.constant(3.1)
+      x3 = constant_op.constant(3.2)
+      g.watch(x1)
+      g.watch(x2)
+      g.watch(x3)
+      y = x1  + 2 * x2  + 3 * x3
+    self.assertEqual(self.evaluate(g.gradient(y, x1)), [1.0])
+    self.assertEqual(self.evaluate(g.gradient(y, (x1,))), (1.0,))
+    self.assertEqual(self.evaluate(g.gradient(y, (x1, x2))), (1.0, 2.0))
+    self.assertEqual(self.evaluate(g.gradient(y, [(x1, x2), (x2, x3)])),
+                     [(1.0, 2.0), (2.0, 3.0)])
+    self.assertEqual(self.evaluate(g.gradient(y, (x1, x2, [x1, x3]))),
+                     (1.0, 2.0, [1.0, 3.0]))
+    self.assertEqual(self.evaluate(g.gradient(y, [x1, {'x2': x2, 'x3': x3}])),
+                     [1.0, {'x2': 2.0, 'x3': 3.0}])
+
   @test_util.assert_no_new_tensors
   @test_util.run_in_graph_and_eager_modes()
   def testGradientTape(self):
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 73482792d5..8a398f6447 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1372,11 +1372,15 @@ PyObject* TFE_Py_TapeGradient(PyObject* tape, PyObject* vspace,
   }
   if (!result.empty()) {
     PyObject* py_result = PyList_New(result.size());
+    tensorflow::gtl::FlatSet<PyObject*> seen_results(result.size());
     for (int i = 0; i < result.size(); ++i) {
       if (result[i] == nullptr) {
         Py_INCREF(Py_None);
         result[i] = Py_None;
+      } else if (seen_results.find(result[i]) != seen_results.end()) {
+        Py_INCREF(result[i]);
       }
+      seen_results.insert(result[i]);
       PyList_SET_ITEM(py_result, i, reinterpret_cast<PyObject*>(result[i]));
     }
     return py_result;
-- 
GitLab


From 5bc7c510fd99dd6f887eb2c5834ae8297891dea7 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 28 Mar 2018 21:07:02 -0700
Subject: [PATCH 661/960] Fixed the shape function of the SplitV op that
 incorrectly often assumed that the shape of all the outputs is the same.

PiperOrigin-RevId: 190879600
---
 .../python/kernel_tests/shape_ops_test.py       |  5 +----
 .../contrib/signal/python/ops/shape_ops.py      |  2 --
 tensorflow/core/ops/array_ops.cc                | 17 +++++++++--------
 3 files changed, 10 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py b/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
index bc4663fbb0..64cc8c7ea5 100644
--- a/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
+++ b/tensorflow/contrib/signal/python/kernel_tests/shape_ops_test.py
@@ -338,10 +338,7 @@ class FrameTest(test.TestCase):
 
   def test_constant_folding(self):
     """frame should be constant foldable for constant inputs."""
-    # Padding is incorrectly defined in shape_ops.py (the rank of the padding
-    # tensor should be equal to the rank of the input tensor + 1): only test
-    # with padding set to False to avoid this.
-    for pad_end in [False]:
+    for pad_end in [True, False]:
       g = ops.Graph()
       with g.as_default():
         frame_length, frame_step = 32, 16
diff --git a/tensorflow/contrib/signal/python/ops/shape_ops.py b/tensorflow/contrib/signal/python/ops/shape_ops.py
index 97fe20866b..1ddc2941ec 100644
--- a/tensorflow/contrib/signal/python/ops/shape_ops.py
+++ b/tensorflow/contrib/signal/python/ops/shape_ops.py
@@ -139,8 +139,6 @@ def frame(signal, frame_length, frame_step, pad_end=False, pad_value=0, axis=-1,
            [[0, pad_samples]],
            array_ops.zeros([num_inner_dimensions, 2], dtype=pad_samples.dtype)],
           0)
-      # TODO(rjryan): the paddings tensor must of rank tf.rank(signal) + 1. This
-      # isn't the case here and should be fixed.
       signal = array_ops.pad(signal, paddings, constant_values=pad_value)
 
       signal_shape = array_ops.shape(signal)
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index 88d2aa3f41..af8afc90f5 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -494,9 +494,9 @@ REGISTER_OP("SplitV")
       const Tensor* size_splits = c->input_tensor(1);
       if (rank == InferenceContext::kUnknownRank) {
         // If the rank of input tensor is unknown, then return unknown shapes.
-        output_shape = c->UnknownShape();
+        // Note that the shape of each output can be different.
         for (int i = 0; i < num_outputs; ++i) {
-          c->set_output(i, output_shape);
+          c->set_output(i, c->UnknownShape());
         }
       } else if (rank == 0) {
         // Throw error if input is a scalar.
@@ -505,18 +505,19 @@ REGISTER_OP("SplitV")
         // If split dimension is known, but the sizes are unknown, then
         // only the split dimension is unknown
         output_shape = input;
-        TF_RETURN_IF_ERROR(c->ReplaceDim(output_shape,
-                                         c->Value(split_dimension),
-                                         c->UnknownDim(), &output_shape));
         for (int i = 0; i < num_outputs; ++i) {
+          TF_RETURN_IF_ERROR(c->ReplaceDim(output_shape,
+                                           c->Value(split_dimension),
+                                           c->UnknownDim(), &output_shape));
           c->set_output(i, output_shape);
         }
       } else if (size_splits == nullptr && !c->ValueKnown(split_dimension)) {
         // If split dimension or tensor containing the split sizes is unknown,
-        // then return unknown shapes of same rank as input.
-        output_shape = c->UnknownShapeOfRank(rank);
+        // then return unknown shapes of same rank as input. Note that each
+        // output shape can be different since splitv doesn't always split
+        // tensors evenly.
         for (int i = 0; i < num_outputs; ++i) {
-          c->set_output(i, output_shape);
+          c->set_output(i, c->UnknownShapeOfRank(rank));
         }
       } else {
         // Determine the output shape if split dimension and split sizes are
-- 
GitLab


From aeaec465f2f08e32c524e23fb7b0ac016f3dc6a9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 21:11:16 -0700
Subject: [PATCH 662/960] Fix TensorList decoding bug. Thanks to Alexandre
 Passos for finding this.

PiperOrigin-RevId: 190879840
---
 tensorflow/core/kernels/list_kernels.cc | 1 +
 tensorflow/core/kernels/list_kernels.h  | 3 ++-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/list_kernels.cc b/tensorflow/core/kernels/list_kernels.cc
index baf0a4abe4..9e7786f25e 100644
--- a/tensorflow/core/kernels/list_kernels.cc
+++ b/tensorflow/core/kernels/list_kernels.cc
@@ -112,6 +112,7 @@ bool TensorList::Decode(const VariantTensorData& data) {
       dims.push_back(scratch);
     }
   }
+  element_shape = PartialTensorShape(dims);
   return true;
 }
 
diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 9733883001..8af48f0a67 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -83,7 +83,8 @@ class TensorListStack : public OpKernel {
                                         DataTypeString(l->element_dtype)));
     OP_REQUIRES(c, l->element_shape.IsFullyDefined(),
                 errors::InvalidArgument("Tried to stack elements from a list "
-                                        "with non-fully-defined shape."));
+                                        "with non-fully-defined shape: ",
+                                        l->element_shape.DebugString()));
     if (num_elements_ != -1) {
       OP_REQUIRES(c, l->tensors.size() == num_elements_,
                   errors::InvalidArgument("Operation expected a list with ",
-- 
GitLab


From 163bf8d0620a08d186c1315b0789e898f09759f8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 21:52:30 -0700
Subject: [PATCH 663/960] DistributionStrategy-enable Estimator.

PiperOrigin-RevId: 190882152
---
 .../python/learn/estimators/run_config.py     |  11 +-
 tensorflow/python/estimator/estimator.py      | 341 +++++++++++++-----
 tensorflow/python/estimator/run_config.py     |   3 +-
 3 files changed, 271 insertions(+), 84 deletions(-)

diff --git a/tensorflow/contrib/learn/python/learn/estimators/run_config.py b/tensorflow/contrib/learn/python/learn/estimators/run_config.py
index 1d161093de..f3500bf56f 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/run_config.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/run_config.py
@@ -290,8 +290,15 @@ class RunConfig(ClusterConfig, core_run_config.RunConfig):
         Note - using this argument, it is easy to provide settings which break
         otherwise perfectly good models. Use with care.
     """
-    super(RunConfig, self).__init__(
-        master=master, evaluation_master=evaluation_master)
+    # Neither parent class calls super().__init__(), so here we have to
+    # manually call their __init__() methods.
+    ClusterConfig.__init__(
+        self, master=master, evaluation_master=evaluation_master)
+    # For too long this code didn't call:
+    #   core_run_config.RunConfig.__init__(self)
+    # so instead of breaking compatibility with that assumption, we
+    # just manually initialize this field:
+    self._distribute = None
 
     gpu_options = config_pb2.GPUOptions(
         per_process_gpu_memory_fraction=gpu_memory_fraction)
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 6a4132bca2..2fe521b063 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -41,8 +41,11 @@ from tensorflow.python.estimator.export.export import get_temp_export_dir
 from tensorflow.python.estimator.export.export import get_timestamped_export_dir
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import metrics as metrics_lib
+from tensorflow.python.ops import resources
+from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.saved_model import builder as saved_model_builder
@@ -50,6 +53,7 @@ from tensorflow.python.saved_model import tag_constants
 from tensorflow.python.summary import summary
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import device_setter
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import evaluation
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import saver
@@ -183,6 +187,9 @@ class Estimator(object):
             config)
       self._config = config
 
+    # The distribute field contains an instance of DistributionStrategy.
+    self._distribution = self._config.distribute
+
     # Model directory.
     model_dir = compat_internal.path_to_str(model_dir)
     if (model_dir is not None) and (self._config.model_dir is not None):
@@ -682,11 +689,25 @@ class Estimator(object):
   def _get_features_and_labels_from_input_fn(self, input_fn, mode):
     """Extracts the `features` and labels from return values of `input_fn`."""
     result = self._call_input_fn(input_fn, mode)
+    # TODO(anjalisridhar): What about the default DistributionStrategy? Perhaps
+    # using any input is alright in that case. There is also a
+    # has_dataset_or_queue_runner function that we may want to extend and use.
+    if (self._distribution is not None and
+        not isinstance(result, dataset_ops.Dataset)):
+      raise ValueError('input_fn() must return a tf.data.Dataset when using a '
+                       'DistributionStrategy.')
     input_hooks = []
     if isinstance(result, dataset_ops.Dataset):
-      iterator = result.make_initializable_iterator()
-      input_hooks.append(_DatasetInitializerHook(iterator))
-      result = iterator.get_next()
+      if self._distribution is not None and mode == model_fn_lib.ModeKeys.TRAIN:
+        # TODO(josh11b): This is currently using a one-shot iterator, we
+        # will update this to an initializeable iterator once the
+        # necessory support for creating an initializable iterator is
+        # available.
+        result = self._distribution.distribute_dataset(result).get_next()
+      else:
+        iterator = result.make_initializable_iterator()
+        input_hooks.append(_DatasetInitializerHook(iterator))
+        result = iterator.get_next()
     if isinstance(result, (list, tuple)):
       if len(result) != 2:
         raise ValueError(
@@ -815,6 +836,12 @@ class Estimator(object):
     return model_fn_results
 
   def _train_model(self, input_fn, hooks, saving_listeners):
+    if self._distribution:
+      return self._train_model_distributed(input_fn, hooks, saving_listeners)
+    else:
+      return self._train_model_default(input_fn, hooks, saving_listeners)
+
+  def _train_model_default(self, input_fn, hooks, saving_listeners):
     worker_hooks = []
     with ops.Graph().as_default() as g, g.device(self._device_fn):
       random_seed.set_random_seed(self._config.tf_random_seed)
@@ -826,86 +853,209 @@ class Estimator(object):
       worker_hooks.extend(input_hooks)
       estimator_spec = self._call_model_fn(
           features, labels, model_fn_lib.ModeKeys.TRAIN, self.config)
+      return self._train_with_estimator_spec(estimator_spec, worker_hooks,
+                                             hooks, global_step_tensor,
+                                             saving_listeners)
 
-      if self._warm_start_settings:
-        logging.info('Warm-starting with WarmStartSettings: %s' %
-                     (self._warm_start_settings,))
-        # pylint: disable=protected-access
-        warm_starting_util.warm_start(*self._warm_start_settings)
-        # pylint: enable=protected-access
-      # Check if the user created a loss summary, and add one if they didn't.
-      # We assume here that the summary is called 'loss'. If it is not, we will
-      # make another one with the name 'loss' to ensure it shows up in the right
-      # graph in TensorBoard.
-      if not any([x.op.name == 'loss'
-                  for x in ops.get_collection(ops.GraphKeys.SUMMARIES)]):
-        summary.scalar('loss', estimator_spec.loss)
-      ops.add_to_collection(ops.GraphKeys.LOSSES, estimator_spec.loss)
-      worker_hooks.extend(hooks)
-      worker_hooks.extend([
-          training.NanTensorHook(estimator_spec.loss),
-          training.LoggingTensorHook(
-              {
-                  'loss': estimator_spec.loss,
-                  'step': global_step_tensor
-              },
-              every_n_iter=self._config.log_step_count_steps)
-      ])
-      worker_hooks.extend(estimator_spec.training_hooks)
-
-      if not (estimator_spec.scaffold.saver or
-              ops.get_collection(ops.GraphKeys.SAVERS)):
-        ops.add_to_collection(
-            ops.GraphKeys.SAVERS,
-            training.Saver(
-                sharded=True,
-                max_to_keep=self._config.keep_checkpoint_max,
-                keep_checkpoint_every_n_hours=(
-                    self._config.keep_checkpoint_every_n_hours),
-                defer_build=True,
-                save_relative_paths=True))
-
-      chief_hooks = []
-      all_hooks = worker_hooks + list(estimator_spec.training_chief_hooks)
-      saver_hooks = [
-          h for h in all_hooks if isinstance(h, training.CheckpointSaverHook)]
-      if (self._config.save_checkpoints_secs or
-          self._config.save_checkpoints_steps):
-        if not saver_hooks:
-          chief_hooks = [
-              training.CheckpointSaverHook(
-                  self._model_dir,
-                  save_secs=self._config.save_checkpoints_secs,
-                  save_steps=self._config.save_checkpoints_steps,
-                  scaffold=estimator_spec.scaffold)
-          ]
-          saver_hooks = [chief_hooks[0]]
-      if saving_listeners:
-        if not saver_hooks:
-          raise ValueError(
-              'There should be a CheckpointSaverHook to use saving_listeners. '
-              'Please set one of the RunConfig.save_checkpoints_steps or '
-              'RunConfig.save_checkpoints_secs.')
+  def _train_model_distributed(self, input_fn, hooks, saving_listeners):
+    worker_hooks = []
+    with ops.Graph().as_default() as g:
+      with self._distribution.scope():
+        random_seed.set_random_seed(self._config.tf_random_seed)
+        features, labels, input_hooks = (
+            self._get_features_and_labels_from_input_fn(
+                input_fn, model_fn_lib.ModeKeys.TRAIN))
+        worker_hooks.extend(input_hooks)
+        global_step_tensor = self._create_and_assert_global_step(g)
+        # The default destination for the global_step_tensor fetch call is the
+        # CPU.
+        global_step_read_tensor = self._distribution.fetch(global_step_tensor)
+        # we want to add to the global collection in the main thread not the
+        # tower threads.
+        ops.add_to_collection(training_util.GLOBAL_STEP_READ_KEY,
+                              global_step_read_tensor)
+        grouped_estimator_spec = self._distribution.call_for_each_tower(
+            self._call_model_fn,
+            features,
+            labels,  # although this will be None it seems
+            model_fn_lib.ModeKeys.TRAIN,
+            self.config)
+
+        # TODO(anjalisridhar): Figure out how to resolve the folowing scaffold
+        # parameters: init_feed_dict, init_fn.
+        scaffold_list = self._distribution.unwrap(
+            grouped_estimator_spec.scaffold)
+        init_feed_dict = [
+            s.init_feed_dict
+            for s in scaffold_list
+            if s.init_feed_dict is not None
+        ]
+        if init_feed_dict:
+          init_feed_dict = self._distribution.group(init_feed_dict)
         else:
-          # It is expected to have one CheckpointSaverHook. If multiple, we pick
-          # up the first one to add listener.
-          saver_hooks[0]._listeners.extend(saving_listeners)  # pylint: disable=protected-access
-      with training.MonitoredTrainingSession(
-          master=self._config.master,
-          is_chief=self._config.is_chief,
-          checkpoint_dir=self._model_dir,
-          scaffold=estimator_spec.scaffold,
-          hooks=worker_hooks,
-          chief_only_hooks=(
-              tuple(chief_hooks) + tuple(estimator_spec.training_chief_hooks)),
-          save_checkpoint_secs=0,  # Saving is handled by a hook.
-          save_summaries_steps=self._config.save_summary_steps,
-          config=self._session_config,
-          log_step_count_steps=self._config.log_step_count_steps) as mon_sess:
-        loss = None
-        while not mon_sess.should_stop():
-          _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
-      return loss
+          init_feed_dict = None
+
+        init_fn = [s.init_fn for s in scaffold_list if s.init_fn is not None]
+        if init_fn:
+          init_fn = self._distribution.group(init_fn)
+        else:
+          init_fn = None
+
+        init_op = [s.init_op for s in scaffold_list if s.init_op is not None]
+        if init_op:
+          init_op = self._distribution.group(init_op)
+        else:
+          init_op = None
+
+        ready_op = self._distribution.call_for_each_tower(
+            create_per_tower_ready_op, grouped_estimator_spec.scaffold)
+        if ready_op is not None:
+          ready_op = self._distribution.group(ready_op)
+        else:
+          ready_op = None
+
+        ready_for_local_init_op = self._distribution.call_for_each_tower(
+            create_per_tower_ready_for_local_init_op,
+            grouped_estimator_spec.scaffold)
+        if ready_for_local_init_op is not None:
+          ready_for_local_init_op = self._distribution.group(
+              ready_for_local_init_op)
+        else:
+          ready_for_local_init_op = None
+
+        local_init_op = [
+            s.local_init_op
+            for s in scaffold_list
+            if s.local_init_op is not None
+        ]
+        if local_init_op:
+          local_init_op = self._distribution.group(local_init_op)
+        else:
+          local_init_op = None
+
+        summary_op = [
+            s.summary_op for s in scaffold_list if s.summary_op is not None
+        ]
+        if summary_op:
+          summary_op = self._distribution.group(summary_op)
+        else:
+          summary_op = None
+
+        scaffold = monitored_session.Scaffold(
+            init_op=init_op,
+            ready_op=ready_op,
+            ready_for_local_init_op=ready_for_local_init_op,
+            local_init_op=local_init_op,
+            summary_op=summary_op,
+            init_feed_dict=init_feed_dict,
+            init_fn=init_fn)
+
+        def get_hooks_from_the_first_device(per_device_hooks):
+          hooks_list = self._distribution.unwrap(per_device_hooks)
+          assert hooks_list
+          return hooks_list[0]
+
+        training_hooks = get_hooks_from_the_first_device(
+            grouped_estimator_spec.training_hooks)
+        training_chief_hooks = get_hooks_from_the_first_device(
+            grouped_estimator_spec.training_chief_hooks)
+
+        estimator_spec = model_fn_lib.EstimatorSpec(
+            mode=grouped_estimator_spec.mode,
+            loss=self._distribution.unwrap(
+                self._distribution.reduce(distribute_lib.get_loss_reduction(),
+                                          grouped_estimator_spec.loss,
+                                          destinations='/device:CPU:0'))[0],
+            train_op=self._distribution.group(grouped_estimator_spec.train_op),
+            training_hooks=training_hooks,
+            training_chief_hooks=training_chief_hooks,
+            scaffold=scaffold)
+        return self._train_with_estimator_spec(estimator_spec, worker_hooks,
+                                               hooks, global_step_read_tensor,
+                                               saving_listeners)
+
+  def _train_with_estimator_spec(self, estimator_spec, worker_hooks, hooks,
+                                 global_step_tensor, saving_listeners):
+    """Train a model with the given Estimator Spec."""
+    if self._warm_start_settings:
+      logging.info('Warm-starting with WarmStartSettings: %s' %
+                   (self._warm_start_settings,))
+      # pylint: disable=protected-access
+      warm_starting_util.warm_start(*self._warm_start_settings)
+      # pylint: enable=protected-access
+    # Check if the user created a loss summary, and add one if they didn't.
+    # We assume here that the summary is called 'loss'. If it is not, we will
+    # make another one with the name 'loss' to ensure it shows up in the right
+    # graph in TensorBoard.
+    if not any([x.op.name == 'loss'
+                for x in ops.get_collection(ops.GraphKeys.SUMMARIES)]):
+      summary.scalar('loss', estimator_spec.loss)
+    ops.add_to_collection(ops.GraphKeys.LOSSES, estimator_spec.loss)
+    worker_hooks.extend(hooks)
+    worker_hooks.extend([
+        training.NanTensorHook(estimator_spec.loss),
+        training.LoggingTensorHook(
+            {
+                'loss': estimator_spec.loss,
+                'step': global_step_tensor
+            },
+            every_n_iter=self._config.log_step_count_steps)
+    ])
+    worker_hooks.extend(estimator_spec.training_hooks)
+
+    if not (estimator_spec.scaffold.saver or
+            ops.get_collection(ops.GraphKeys.SAVERS)):
+      ops.add_to_collection(
+          ops.GraphKeys.SAVERS,
+          training.Saver(
+              sharded=True,
+              max_to_keep=self._config.keep_checkpoint_max,
+              keep_checkpoint_every_n_hours=(
+                  self._config.keep_checkpoint_every_n_hours),
+              defer_build=True,
+              save_relative_paths=True))
+
+    chief_hooks = []
+    all_hooks = worker_hooks + list(estimator_spec.training_chief_hooks)
+    saver_hooks = [
+        h for h in all_hooks if isinstance(h, training.CheckpointSaverHook)]
+    if (self._config.save_checkpoints_secs or
+        self._config.save_checkpoints_steps):
+      if not saver_hooks:
+        chief_hooks = [
+            training.CheckpointSaverHook(
+                self._model_dir,
+                save_secs=self._config.save_checkpoints_secs,
+                save_steps=self._config.save_checkpoints_steps,
+                scaffold=estimator_spec.scaffold)
+        ]
+        saver_hooks = [chief_hooks[0]]
+    if saving_listeners:
+      if not saver_hooks:
+        raise ValueError(
+            'There should be a CheckpointSaverHook to use saving_listeners. '
+            'Please set one of the RunConfig.save_checkpoints_steps or '
+            'RunConfig.save_checkpoints_secs.')
+      else:
+        # It is expected to have one CheckpointSaverHook. If multiple, we pick
+        # up the first one to add listener.
+        saver_hooks[0]._listeners.extend(saving_listeners)  # pylint: disable=protected-access
+    with training.MonitoredTrainingSession(
+        master=self._config.master,
+        is_chief=self._config.is_chief,
+        checkpoint_dir=self._model_dir,
+        scaffold=estimator_spec.scaffold,
+        hooks=worker_hooks,
+        chief_only_hooks=(
+            tuple(chief_hooks) + tuple(estimator_spec.training_chief_hooks)),
+        save_checkpoint_secs=0,  # Saving is handled by a hook.
+        save_summaries_steps=self._config.save_summary_steps,
+        config=self._session_config,
+        log_step_count_steps=self._config.log_step_count_steps) as mon_sess:
+      loss = None
+      while not mon_sess.should_stop():
+        _, loss = mon_sess.run([estimator_spec.train_op, estimator_spec.loss])
+    return loss
 
   def _evaluate_model(self,
                       input_fn,
@@ -972,6 +1122,35 @@ class Estimator(object):
     return eval_results
 
 
+def create_per_tower_ready_op(scaffold):
+  """Create a Scaffold.ready_op inside a tower."""
+  if scaffold.ready_op:
+    return scaffold.ready_op
+
+  def default_ready_op():
+    return array_ops.concat([
+        variables.report_uninitialized_variables(),
+        resources.report_uninitialized_resources()
+    ], 0)
+
+  return monitored_session.Scaffold.get_or_default(
+      'ready_op', ops.GraphKeys.READY_OP, default_ready_op)
+
+
+def create_per_tower_ready_for_local_init_op(scaffold):
+  """Create a Scaffold.ready_for_local_init_op inside a tower."""
+  if scaffold.ready_for_local_init_op:
+    return scaffold.ready_for_local_init_op
+
+  def default_ready_for_local_init_op():
+    return variables.report_uninitialized_variables(
+        variables.global_variables())
+
+  return monitored_session.Scaffold.get_or_default(
+      'ready_for_local_init_op', ops.GraphKeys.READY_FOR_LOCAL_INIT_OP,
+      default_ready_for_local_init_op)
+
+
 def _check_checkpoint_available(model_dir):
   latest_path = saver.latest_checkpoint(model_dir)
   if not latest_path:
diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 141eaeff64..41415b89e9 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -688,7 +688,7 @@ class RunConfig(object):
 
     Only the properties in the following list are allowed to be replaced:
 
-      - `model_dir`.
+      - `model_dir`,
       - `tf_random_seed`,
       - `save_summary_steps`,
       - `save_checkpoints_steps`,
@@ -697,6 +697,7 @@ class RunConfig(object):
       - `keep_checkpoint_max`,
       - `keep_checkpoint_every_n_hours`,
       - `log_step_count_steps`,
+      - `distribute`.
 
     In addition, either `save_checkpoints_steps` or `save_checkpoints_secs`
     can be set (should not be both).
-- 
GitLab


From 695aa649da752315596934319dd601854495dec5 Mon Sep 17 00:00:00 2001
From: Michael Case <mikecase@google.com>
Date: Wed, 28 Mar 2018 22:46:25 -0700
Subject: [PATCH 664/960] Add --announce_rc Bazel arg to several of our builds.

This will help to...
 - Refactor the build scripts without accidently adding functional changes.
 - Help debug several issues where some options aren't being added correctly
   by configure script.

PiperOrigin-RevId: 190884531
---
 tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh   | 1 +
 tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh   | 1 +
 tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh | 2 +-
 3 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
index 338066131b..c7cc16e669 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_py2_cc_core.sh
@@ -33,6 +33,7 @@ yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
     --test_timeout 300,450,1200,3600 --config=opt \
+    --announce_rc \
     --test_size_filters=small,medium \
     --jobs=${N_JOBS} --build_tests_only --test_output=errors -k -- \
     //tensorflow/... -//tensorflow/compiler/... -//tensorflow/contrib/...
diff --git a/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh
index 920a261ae3..7e0e81a1eb 100755
--- a/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh
+++ b/tensorflow/tools/ci_build/osx/cpu/run_py3_cc_core.sh
@@ -31,6 +31,7 @@ export PYTHON_BIN_PATH=$(which python3)
 yes "" | $PYTHON_BIN_PATH configure.py
 which bazel
 bazel test --test_tag_filters=-no_oss,-gpu,-benchmark-test,-nomac,-no_mac \
+    --announce_rc \
     --test_timeout 300,450,1200,3600 \
     --test_size_filters=small,medium \
     --jobs=${N_JOBS} --build_tests_only --test_output=errors -k -- \
diff --git a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
index 94276c6c5c..7dfee8f371 100644
--- a/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
+++ b/tensorflow/tools/ci_build/windows/libtensorflow_gpu.sh
@@ -41,7 +41,7 @@ run_configure_for_gpu_build
 # build_libtensorflow_tarball in ../builds/libtensorflow.sh
 # cannot be used on Windows since it relies on pkg_tar rules.
 # So we do something special here
-bazel build -c opt --copt=/arch:AVX \
+bazel build -c opt --copt=/arch:AVX --announce_rc \
   tensorflow:libtensorflow.so \
   tensorflow/tools/lib_package:clicenses_generate \
   tensorflow/java:libtensorflow_jni.so \
-- 
GitLab


From 8df77178a8d41b392928ec17e6ca4867698407ff Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 28 Mar 2018 23:31:26 -0700
Subject: [PATCH 665/960] Move the swapping kernels to the all_kernels library
 to avoid registering them more than once from tensorflow/contrib.

PiperOrigin-RevId: 190887394
---
 tensorflow/core/BUILD                     | 4 ++++
 tensorflow/core/grappler/optimizers/BUILD | 9 +++++----
 2 files changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index d46241450c..b8dbd90ab8 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -144,6 +144,7 @@ load(
     "tf_cuda_tests_tags",
     "if_static",
 )
+load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
@@ -939,6 +940,9 @@ cc_library(
         "//tensorflow/core/kernels:mkl_softmax_op",
         "//tensorflow/core/kernels:mkl_tfconv_op",
         "//tensorflow/core/kernels:mkl_aggregate_ops",
+    ]) + if_cuda([
+        "//tensorflow/core/grappler/optimizers:gpu_swapping_kernels",
+        "//tensorflow/core/grappler/optimizers:gpu_swapping_ops",
     ]),
 )
 
diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 0d3a488f85..2c365c467c 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -361,6 +361,7 @@ tf_kernel_library(
     srcs = [
         "gpu_swapping_kernels.cc",
     ],
+    visibility = ["//tensorflow:__subpackages__"],
     deps = [
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
@@ -373,6 +374,7 @@ cc_library(
     srcs = [
         "gpu_swapping_ops.cc",
     ],
+    visibility = ["//tensorflow:__subpackages__"],
     deps = [
         "//tensorflow/core:core_cpu_base",
         "//tensorflow/core:framework",
@@ -406,10 +408,7 @@ cc_library(
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:topological_sort",
         "//tensorflow/core/grappler/utils:traversal",
-    ] + if_cuda([
-        ":gpu_swapping_kernels",
-        ":gpu_swapping_ops",
-    ]),
+    ],
 )
 
 tf_cuda_only_cc_test(
@@ -417,6 +416,8 @@ tf_cuda_only_cc_test(
     srcs = ["memory_optimizer_test.cc"],
     tags = ["no_cuda_on_cpu_tap"],  # Do not re-enable again without actually testing.
     deps = [
+        ":gpu_swapping_kernels",
+        ":gpu_swapping_ops",
         ":memory_optimizer",
         "//tensorflow/cc:cc_ops",
         "//tensorflow/core:ops",
-- 
GitLab


From fd25620e80d628d77c5e9a03e87d6a4e10eccd27 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 29 Mar 2018 04:34:29 -0700
Subject: [PATCH 666/960] Internal change.

PiperOrigin-RevId: 190913047
---
 tensorflow/contrib/eager/python/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 7a8c11e3bb..48372d7ae0 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -80,6 +80,7 @@ cuda_py_test(
         "//tensorflow/python/data",
         "//tensorflow/python/eager:test",
     ],
+    tags = ["noguitar"],
 )
 
 py_library(
-- 
GitLab


From 93cf42ac3530d24009179c45c88a444383719c9b Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Thu, 29 Mar 2018 22:54:53 +0800
Subject: [PATCH 667/960] Fix math equation format in layers (#18069)

---
 tensorflow/contrib/layers/python/layers/layers.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/tensorflow/contrib/layers/python/layers/layers.py b/tensorflow/contrib/layers/python/layers/layers.py
index 350bcb3bca..10d7f6d076 100644
--- a/tensorflow/contrib/layers/python/layers/layers.py
+++ b/tensorflow/contrib/layers/python/layers/layers.py
@@ -3045,16 +3045,16 @@ def legacy_fully_connected(x,
   `activation_fn` is `None`, the result of `y = w * x + b` is
   returned.
 
-  If `x` has shape [\\\(\\text{dim}_0, \\text{dim}_1, ..., \\text{dim}_n\\\)]
-  with more than 2 dimensions (\\\(n > 1\\\)), then we repeat the matrix
+  If `x` has shape [\\(\text{dim}_0, \text{dim}_1, ..., \text{dim}_n\\)]
+  with more than 2 dimensions (\\(n > 1\\)), then we repeat the matrix
   multiply along the first dimensions. The result r is a tensor of shape
-  [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`],
-  where \\\( r_{i_0, ..., i_{n-1}, k} =
-  \\sum_{0 \\leq j < \\text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\\).
+  [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`],
+  where \\( r_{i_0, ..., i_{n-1}, k} =
+  \sum_{0 \leq j < \text{dim}_n} x_{i_0, ... i_{n-1}, j} \cdot w_{j, k}\\).
   This is accomplished by reshaping `x` to 2-D
-  [\\\(\\text{dim}_0 \\cdot ... \\cdot \\text{dim}_{n-1}, \\text{dim}_n\\\)]
+  [\\(\text{dim}_0 \cdot ... \cdot \text{dim}_{n-1}, \text{dim}_n\\)]
   before the matrix multiply and afterwards reshaping it to
-  [\\\(\\text{dim}_0, ..., \\text{dim}_{n-1},\\\) `num_output_units`].
+  [\\(\text{dim}_0, ..., \text{dim}_{n-1},\\) `num_output_units`].
 
   This op creates `w` and optionally `b`. Bias (`b`) can be disabled by setting
   `bias_init` to `None`.
-- 
GitLab


From 481dca1987e030f9986ce16ae05142617d631641 Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <vomjom@vomjom.net>
Date: Thu, 29 Mar 2018 07:55:23 -0700
Subject: [PATCH 668/960] Default disable including the coordinator in the TPU
 job (#18073)

* Default disable including the coordinator in the TPU job

* Fix the test
---
 .../python/training/tpu_cluster_resolver.py               | 2 +-
 .../python/training/tpu_cluster_resolver_test.py          | 8 ++++++--
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
index 300b19733e..95c5c920aa 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver.py
@@ -73,7 +73,7 @@ class TPUClusterResolver(ClusterResolver):
                zone=None,
                project=None,
                job_name='worker',
-               coordinator_name='coordinator',
+               coordinator_name=None,
                coordinator_address=None,
                credentials='default',
                service=None):
diff --git a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
index 48c3f6bb4f..e1e3e6867a 100644
--- a/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
+++ b/tensorflow/contrib/cluster_resolver/python/training/tpu_cluster_resolver_test.py
@@ -117,7 +117,8 @@ class TPUClusterResolverTest(test.TestCase):
         zone=None,
         tpu=['test-tpu-1'],
         credentials=None,
-        service=self.mock_service_client(tpu_map=tpu_map))
+        service=self.mock_service_client(tpu_map=tpu_map),
+        coordinator_name='coordinator')
 
     actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
     expected_proto = """
@@ -170,6 +171,7 @@ class TPUClusterResolverTest(test.TestCase):
         project='test-project',
         zone='us-central1-c',
         tpu=['test-tpu-1'],
+        coordinator_name='coordinator',
         coordinator_address='10.128.1.5:10203',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
@@ -196,6 +198,7 @@ class TPUClusterResolverTest(test.TestCase):
         project='test-project',
         zone='us-central1-c',
         tpu='test-tpu-1',
+        coordinator_name='coordinator',
         coordinator_address='10.128.1.5:10203',
         credentials=None,
         service=self.mock_service_client(tpu_map=tpu_map))
@@ -239,7 +242,8 @@ class TPUClusterResolverTest(test.TestCase):
     tpu_cluster_resolver = TPUClusterResolver(
         tpu='test-tpu-1',
         credentials=None,
-        service=self.mock_service_client(tpu_map=tpu_map))
+        service=self.mock_service_client(tpu_map=tpu_map),
+        coordinator_name='coordinator')
 
     actual_cluster_spec = tpu_cluster_resolver.cluster_spec()
     expected_proto = """
-- 
GitLab


From 76c569a29ec33d1965757eeed1bdc317f2fb5e87 Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Thu, 29 Mar 2018 07:56:15 -0700
Subject: [PATCH 669/960] Add meta-distribution which reshapes batch dims.

PiperOrigin-RevId: 190930846
---
 tensorflow/contrib/distributions/BUILD        |  14 +
 tensorflow/contrib/distributions/__init__.py  |   4 +-
 .../python/kernel_tests/batch_reshape_test.py | 531 ++++++++++++++++++
 .../distributions/python/ops/batch_reshape.py | 333 +++++++++++
 4 files changed, 881 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
 create mode 100644 tensorflow/contrib/distributions/python/ops/batch_reshape.py

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 231abaa2f3..de08eb491b 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -456,6 +456,20 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "batch_reshape_test",
+    size = "small",
+    srcs = ["python/kernel_tests/batch_reshape_test.py"],
+    additional_deps = [
+        ":distributions_py",
+        "//third_party/py/numpy",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_for_generated_wrappers",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
 cuda_py_test(
     name = "sample_stats_test",
     size = "medium",
diff --git a/tensorflow/contrib/distributions/__init__.py b/tensorflow/contrib/distributions/__init__.py
index 61c411271d..4d4489468d 100644
--- a/tensorflow/contrib/distributions/__init__.py
+++ b/tensorflow/contrib/distributions/__init__.py
@@ -24,6 +24,7 @@ from __future__ import print_function
 
 from tensorflow.contrib.distributions.python.ops import bijectors
 from tensorflow.contrib.distributions.python.ops.autoregressive import *
+from tensorflow.contrib.distributions.python.ops.batch_reshape import *
 from tensorflow.contrib.distributions.python.ops.binomial import *
 from tensorflow.contrib.distributions.python.ops.cauchy import *
 from tensorflow.contrib.distributions.python.ops.chi2 import *
@@ -96,9 +97,10 @@ _allowed_symbols = [
     'ReparameterizationType',
     'Distribution',
     'Autoregressive',
-    'Binomial',
+    'BatchReshape',
     'Bernoulli',
     'Beta',
+    'Binomial',
     'BetaWithSoftplusConcentration',
     'Categorical',
     'Chi2',
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
new file mode 100644
index 0000000000..4d2f40e27f
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
@@ -0,0 +1,531 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for BatchReshape."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.distributions.python.ops import batch_reshape as batch_reshape_lib
+from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_lib
+from tensorflow.contrib.distributions.python.ops import wishart as wishart_lib
+from tensorflow.python.framework import constant_op
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops.distributions import normal as normal_lib
+from tensorflow.python.platform import test
+
+
+class _BatchReshapeTest(object):
+
+  def make_wishart(self, dims, new_batch_shape, old_batch_shape):
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = self.dtype([
+        [[1., 0.5],
+         [0.5, 1.]],
+        [[0.5, 0.25],
+         [0.25, 0.75]],
+    ])
+    scale = np.reshape(np.concatenate([scale, scale], axis=0),
+                       old_batch_shape + [dims, dims])
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    wishart = wishart_lib.WishartFull(df=5, scale=scale_ph)
+    reshape_wishart = batch_reshape_lib.BatchReshape(
+        distribution=wishart,
+        batch_shape=new_batch_shape_ph,
+        validate_args=True)
+
+    return wishart, reshape_wishart
+
+  def test_matrix_variate_sample_and_log_prob(self):
+    dims = 2
+    new_batch_shape = [4]
+    old_batch_shape = [2, 2]
+    wishart, reshape_wishart = self.make_wishart(
+        dims, new_batch_shape, old_batch_shape)
+
+    batch_shape = reshape_wishart.batch_shape_tensor()
+    event_shape = reshape_wishart.event_shape_tensor()
+
+    expected_sample_shape = [3, 1] + new_batch_shape + [dims, dims]
+    x = wishart.sample([3, 1], seed=42)
+    expected_sample = array_ops.reshape(x, expected_sample_shape)
+    actual_sample = reshape_wishart.sample([3, 1], seed=42)
+
+    expected_log_prob_shape = [3, 1] + new_batch_shape
+    expected_log_prob = array_ops.reshape(
+        wishart.log_prob(x), expected_log_prob_shape)
+    actual_log_prob = reshape_wishart.log_prob(expected_sample)
+
+    with self.test_session() as sess:
+      [
+          batch_shape_,
+          event_shape_,
+          expected_sample_, actual_sample_,
+          expected_log_prob_, actual_log_prob_,
+      ] = sess.run([
+          batch_shape,
+          event_shape,
+          expected_sample, actual_sample,
+          expected_log_prob, actual_log_prob,
+      ])
+
+    self.assertAllEqual(new_batch_shape, batch_shape_)
+    self.assertAllEqual([dims, dims], event_shape_)
+    self.assertAllClose(expected_sample_, actual_sample_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_log_prob_, actual_log_prob_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(new_batch_shape, reshape_wishart.batch_shape)
+    self.assertAllEqual([dims, dims], reshape_wishart.event_shape)
+    self.assertAllEqual(expected_sample_shape, actual_sample.shape)
+    self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape)
+
+  def test_matrix_variate_stats(self):
+    dims = 2
+    new_batch_shape = [4]
+    old_batch_shape = [2, 2]
+    wishart, reshape_wishart = self.make_wishart(
+        dims, new_batch_shape, old_batch_shape)
+
+    expected_scalar_stat_shape = new_batch_shape
+    expected_matrix_stat_shape = new_batch_shape + [dims, dims]
+
+    expected_entropy = array_ops.reshape(
+        wishart.entropy(), expected_scalar_stat_shape)
+    actual_entropy = reshape_wishart.entropy()
+
+    expected_mean = array_ops.reshape(
+        wishart.mean(), expected_matrix_stat_shape)
+    actual_mean = reshape_wishart.mean()
+
+    expected_mode = array_ops.reshape(
+        wishart.mode(), expected_matrix_stat_shape)
+    actual_mode = reshape_wishart.mode()
+
+    expected_stddev = array_ops.reshape(
+        wishart.stddev(), expected_matrix_stat_shape)
+    actual_stddev = reshape_wishart.stddev()
+
+    expected_variance = array_ops.reshape(
+        wishart.variance(), expected_matrix_stat_shape)
+    actual_variance = reshape_wishart.variance()
+
+    with self.test_session() as sess:
+      [
+          expected_entropy_, actual_entropy_,
+          expected_mean_, actual_mean_,
+          expected_mode_, actual_mode_,
+          expected_stddev_, actual_stddev_,
+          expected_variance_, actual_variance_,
+      ] = sess.run([
+          expected_entropy, actual_entropy,
+          expected_mean, actual_mean,
+          expected_mode, actual_mode,
+          expected_stddev, actual_stddev,
+          expected_variance, actual_variance,
+      ])
+
+    self.assertAllClose(expected_entropy_, actual_entropy_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mean_, actual_mean_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mode_, actual_mode_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_stddev_, actual_stddev_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_variance_, actual_variance_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape)
+    self.assertAllEqual(expected_matrix_stat_shape, actual_mean.shape)
+    self.assertAllEqual(expected_matrix_stat_shape, actual_mode.shape)
+    self.assertAllEqual(expected_matrix_stat_shape, actual_stddev.shape)
+    self.assertAllEqual(expected_matrix_stat_shape, actual_variance.shape)
+
+  def make_normal(self, new_batch_shape, old_batch_shape):
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = self.dtype(0.5 + np.arange(
+        np.prod(old_batch_shape)).reshape(old_batch_shape))
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    normal = normal_lib.Normal(loc=self.dtype(0), scale=scale_ph)
+    reshape_normal = batch_reshape_lib.BatchReshape(
+        distribution=normal,
+        batch_shape=new_batch_shape_ph,
+        validate_args=True)
+    return normal, reshape_normal
+
+  def test_scalar_variate_sample_and_log_prob(self):
+    new_batch_shape = [2, 2]
+    old_batch_shape = [4]
+
+    normal, reshape_normal = self.make_normal(
+        new_batch_shape, old_batch_shape)
+
+    batch_shape = reshape_normal.batch_shape_tensor()
+    event_shape = reshape_normal.event_shape_tensor()
+
+    expected_sample_shape = new_batch_shape
+    x = normal.sample(seed=52)
+    expected_sample = array_ops.reshape(x, expected_sample_shape)
+    actual_sample = reshape_normal.sample(seed=52)
+
+    expected_log_prob_shape = new_batch_shape
+    expected_log_prob = array_ops.reshape(
+        normal.log_prob(x), expected_log_prob_shape)
+    actual_log_prob = reshape_normal.log_prob(expected_sample)
+
+    with self.test_session() as sess:
+      [
+          batch_shape_,
+          event_shape_,
+          expected_sample_, actual_sample_,
+          expected_log_prob_, actual_log_prob_,
+      ] = sess.run([
+          batch_shape,
+          event_shape,
+          expected_sample, actual_sample,
+          expected_log_prob, actual_log_prob,
+      ])
+    self.assertAllEqual(new_batch_shape, batch_shape_)
+    self.assertAllEqual([], event_shape_)
+    self.assertAllClose(expected_sample_, actual_sample_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_log_prob_, actual_log_prob_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(new_batch_shape, reshape_normal.batch_shape)
+    self.assertAllEqual([], reshape_normal.event_shape)
+    self.assertAllEqual(expected_sample_shape, actual_sample.shape)
+    self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape)
+
+  def test_scalar_variate_stats(self):
+    new_batch_shape = [2, 2]
+    old_batch_shape = [4]
+
+    normal, reshape_normal = self.make_normal(new_batch_shape, old_batch_shape)
+
+    expected_scalar_stat_shape = new_batch_shape
+
+    expected_entropy = array_ops.reshape(
+        normal.entropy(), expected_scalar_stat_shape)
+    actual_entropy = reshape_normal.entropy()
+
+    expected_mean = array_ops.reshape(
+        normal.mean(), expected_scalar_stat_shape)
+    actual_mean = reshape_normal.mean()
+
+    expected_mode = array_ops.reshape(
+        normal.mode(), expected_scalar_stat_shape)
+    actual_mode = reshape_normal.mode()
+
+    expected_stddev = array_ops.reshape(
+        normal.stddev(), expected_scalar_stat_shape)
+    actual_stddev = reshape_normal.stddev()
+
+    expected_variance = array_ops.reshape(
+        normal.variance(), expected_scalar_stat_shape)
+    actual_variance = reshape_normal.variance()
+
+    with self.test_session() as sess:
+      [
+          expected_entropy_, actual_entropy_,
+          expected_mean_, actual_mean_,
+          expected_mode_, actual_mode_,
+          expected_stddev_, actual_stddev_,
+          expected_variance_, actual_variance_,
+      ] = sess.run([
+          expected_entropy, actual_entropy,
+          expected_mean, actual_mean,
+          expected_mode, actual_mode,
+          expected_stddev, actual_stddev,
+          expected_variance, actual_variance,
+      ])
+    self.assertAllClose(expected_entropy_, actual_entropy_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mean_, actual_mean_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mode_, actual_mode_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_stddev_, actual_stddev_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_variance_, actual_variance_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape)
+    self.assertAllEqual(expected_scalar_stat_shape, actual_mean.shape)
+    self.assertAllEqual(expected_scalar_stat_shape, actual_mode.shape)
+    self.assertAllEqual(expected_scalar_stat_shape, actual_stddev.shape)
+    self.assertAllEqual(expected_scalar_stat_shape, actual_variance.shape)
+
+  def make_mvn(self, dims, new_batch_shape, old_batch_shape):
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = np.ones(old_batch_shape + [dims], self.dtype)
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph)
+    reshape_mvn = batch_reshape_lib.BatchReshape(
+        distribution=mvn,
+        batch_shape=new_batch_shape_ph,
+        validate_args=True)
+    return mvn, reshape_mvn
+
+  def test_vector_variate_sample_and_log_prob(self):
+    dims = 3
+    new_batch_shape = [2, 1]
+    old_batch_shape = [2]
+    mvn, reshape_mvn = self.make_mvn(
+        dims, new_batch_shape, old_batch_shape)
+
+    batch_shape = reshape_mvn.batch_shape_tensor()
+    event_shape = reshape_mvn.event_shape_tensor()
+
+    expected_sample_shape = [3] + new_batch_shape + [dims]
+    x = mvn.sample(3, seed=62)
+    expected_sample = array_ops.reshape(x, expected_sample_shape)
+    actual_sample = reshape_mvn.sample(3, seed=62)
+
+    expected_log_prob_shape = [3] + new_batch_shape
+    expected_log_prob = array_ops.reshape(
+        mvn.log_prob(x), expected_log_prob_shape)
+    actual_log_prob = reshape_mvn.log_prob(expected_sample)
+
+    with self.test_session() as sess:
+      [
+          batch_shape_,
+          event_shape_,
+          expected_sample_, actual_sample_,
+          expected_log_prob_, actual_log_prob_,
+      ] = sess.run([
+          batch_shape,
+          event_shape,
+          expected_sample, actual_sample,
+          expected_log_prob, actual_log_prob,
+      ])
+    self.assertAllEqual(new_batch_shape, batch_shape_)
+    self.assertAllEqual([dims], event_shape_)
+    self.assertAllClose(expected_sample_, actual_sample_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_log_prob_, actual_log_prob_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(new_batch_shape, reshape_mvn.batch_shape)
+    self.assertAllEqual([dims], reshape_mvn.event_shape)
+    self.assertAllEqual(expected_sample_shape, actual_sample.shape)
+    self.assertAllEqual(expected_log_prob_shape, actual_log_prob.shape)
+
+  def test_vector_variate_stats(self):
+    dims = 3
+    new_batch_shape = [2, 1]
+    old_batch_shape = [2]
+    mvn, reshape_mvn = self.make_mvn(
+        dims, new_batch_shape, old_batch_shape)
+
+    expected_scalar_stat_shape = new_batch_shape
+
+    expected_entropy = array_ops.reshape(
+        mvn.entropy(), expected_scalar_stat_shape)
+    actual_entropy = reshape_mvn.entropy()
+
+    expected_vector_stat_shape = new_batch_shape + [dims]
+
+    expected_mean = array_ops.reshape(
+        mvn.mean(), expected_vector_stat_shape)
+    actual_mean = reshape_mvn.mean()
+
+    expected_mode = array_ops.reshape(
+        mvn.mode(), expected_vector_stat_shape)
+    actual_mode = reshape_mvn.mode()
+
+    expected_stddev = array_ops.reshape(
+        mvn.stddev(), expected_vector_stat_shape)
+    actual_stddev = reshape_mvn.stddev()
+
+    expected_variance = array_ops.reshape(
+        mvn.variance(), expected_vector_stat_shape)
+    actual_variance = reshape_mvn.variance()
+
+    expected_matrix_stat_shape = new_batch_shape + [dims, dims]
+
+    expected_covariance = array_ops.reshape(
+        mvn.covariance(), expected_matrix_stat_shape)
+    actual_covariance = reshape_mvn.covariance()
+
+    with self.test_session() as sess:
+      [
+          expected_entropy_, actual_entropy_,
+          expected_mean_, actual_mean_,
+          expected_mode_, actual_mode_,
+          expected_stddev_, actual_stddev_,
+          expected_variance_, actual_variance_,
+          expected_covariance_, actual_covariance_,
+      ] = sess.run([
+          expected_entropy, actual_entropy,
+          expected_mean, actual_mean,
+          expected_mode, actual_mode,
+          expected_stddev, actual_stddev,
+          expected_variance, actual_variance,
+          expected_covariance, actual_covariance,
+      ])
+    self.assertAllClose(expected_entropy_, actual_entropy_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mean_, actual_mean_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_mode_, actual_mode_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_stddev_, actual_stddev_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_variance_, actual_variance_,
+                        atol=0., rtol=1e-6)
+    self.assertAllClose(expected_covariance_, actual_covariance_,
+                        atol=0., rtol=1e-6)
+    if not self.is_static_shape:
+      return
+    self.assertAllEqual(expected_scalar_stat_shape, actual_entropy.shape)
+    self.assertAllEqual(expected_vector_stat_shape, actual_mean.shape)
+    self.assertAllEqual(expected_vector_stat_shape, actual_mode.shape)
+    self.assertAllEqual(expected_vector_stat_shape, actual_stddev.shape)
+    self.assertAllEqual(expected_vector_stat_shape, actual_variance.shape)
+    self.assertAllEqual(expected_matrix_stat_shape, actual_covariance.shape)
+
+  def test_bad_reshape_size(self):
+    dims = 2
+    new_batch_shape = [2, 3]
+    old_batch_shape = [2]   # 2 != 2*3
+
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = np.ones(old_batch_shape + [dims], self.dtype)
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph)
+
+    if self.is_static_shape:
+      with self.assertRaisesRegexp(
+          ValueError, (r"`batch_shape` size \(6\) must match "
+                       r"`distribution\.batch_shape` size \(2\)")):
+        batch_reshape_lib.BatchReshape(
+            distribution=mvn,
+            batch_shape=new_batch_shape_ph,
+            validate_args=True)
+
+    else:
+      with self.test_session():
+        with self.assertRaisesOpError(r"`batch_shape` size must match "
+                                      r"`distributions.batch_shape` size"):
+          batch_reshape_lib.BatchReshape(
+              distribution=mvn,
+              batch_shape=new_batch_shape_ph,
+              validate_args=True).sample().eval()
+
+  def test_non_positive_shape(self):
+    dims = 2
+    new_batch_shape = [-1, -2]   # -1*-2=2 so will pass size check.
+    old_batch_shape = [2]
+
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = np.ones(old_batch_shape + [dims], self.dtype)
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph)
+
+    if self.is_static_shape:
+      with self.assertRaisesRegexp(ValueError, r".*must be positive.*"):
+        batch_reshape_lib.BatchReshape(
+            distribution=mvn,
+            batch_shape=new_batch_shape_ph,
+            validate_args=True)
+
+    else:
+      with self.test_session():
+        with self.assertRaisesOpError(r".*must be positive.*"):
+          batch_reshape_lib.BatchReshape(
+              distribution=mvn,
+              batch_shape=new_batch_shape_ph,
+              validate_args=True).sample().eval()
+
+  def test_non_vector_shape(self):
+    dims = 2
+    new_batch_shape = 2
+    old_batch_shape = [2]
+
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+
+    scale = np.ones(old_batch_shape + [dims], self.dtype)
+    scale_ph = array_ops.placeholder_with_default(
+        scale, shape=scale.shape if self.is_static_shape else None)
+    mvn = mvn_lib.MultivariateNormalDiag(scale_diag=scale_ph)
+
+    if self.is_static_shape:
+      with self.assertRaisesRegexp(ValueError, r".*must be a vector.*"):
+        batch_reshape_lib.BatchReshape(
+            distribution=mvn,
+            batch_shape=new_batch_shape_ph,
+            validate_args=True)
+
+    else:
+      with self.test_session():
+        with self.assertRaisesOpError(r".*must be a vector.*"):
+          batch_reshape_lib.BatchReshape(
+              distribution=mvn,
+              batch_shape=new_batch_shape_ph,
+              validate_args=True).sample().eval()
+
+
+class BatchReshapeStaticTest(_BatchReshapeTest, test.TestCase):
+
+  dtype = np.float32
+  is_static_shape = True
+
+
+class BatchReshapeDynamicTest(_BatchReshapeTest, test.TestCase):
+
+  dtype = np.float64
+  is_static_shape = False
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/batch_reshape.py b/tensorflow/contrib/distributions/python/ops/batch_reshape.py
new file mode 100644
index 0000000000..c7ee9b2117
--- /dev/null
+++ b/tensorflow/contrib/distributions/python/ops/batch_reshape.py
@@ -0,0 +1,333 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""The BatchReshape distribution."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.framework import tensor_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops.distributions import distribution as distribution_lib
+
+
+__all__ = [
+    "BatchReshape",
+]
+
+
+class BatchReshape(distribution_lib.Distribution):
+  """The Batch-Reshaping distribution.
+
+  This "meta-distribution" reshapes the batch dimensions of another
+  distribution.
+
+  Note: Unlike `tf.reshape`, the `BatchReshape` distribution does not support
+  `-1` for flattening.
+
+  #### Examples
+
+  ```python
+  tfd = tf.contrib.distributions
+
+  dtype = np.float32
+  dims = 2
+  new_batch_shape = [1, 2, 3]
+  old_batch_shape = [6]
+
+  scale = np.ones(old_batch_shape + [dims], dtype)
+  mvn = tfd.MultivariateNormalDiag(scale_diag=scale)
+  reshape_mvn = tfd.BatchReshape(
+      distribution=mvn,
+      batch_shape=new_batch_shape,
+      validate_args=True)
+
+  reshape_mvn.batch_shape
+  # ==> [1, 2, 3]
+
+  x = reshape_mvn.sample(sample_shape=[4, 5])
+  x.shape
+  # ==> [4, 5, 1, 2, 3, 2] == sample_shape + new_batch_shape + [dims]
+
+  reshape_mvn.log_prob(x).shape
+  # ==> [4, 5, 1, 2, 3] == sample_shape + new_batch_shape
+  ```
+
+  """
+
+  def __init__(self,
+               distribution,
+               batch_shape,
+               validate_args=False,
+               allow_nan_stats=True,
+               name=None):
+    """Construct BatchReshape distribution.
+
+    Args:
+      distribution: The base distribution instance to reshape. Typically an
+        instance of `Distribution`.
+      batch_shape: Positive `int`-like vector-shaped `Tensor` representing the
+        new shape of the batch dimensions.
+      validate_args: Python `bool`, default `False`. When `True` distribution
+        parameters are checked for validity despite possibly degrading runtime
+        performance. When `False` invalid inputs may silently render incorrect
+        outputs.
+      allow_nan_stats: Python `bool`, default `True`. When `True`, statistics
+        (e.g., mean, mode, variance) use the value "`NaN`" to indicate the
+        result is undefined. When `False`, an exception is raised if one or
+        more of the statistic's batch members are undefined.
+      name: The name to give Ops created by the initializer.
+        Default value: `"BatchReshape" + distribution.name`.
+
+    Raises:
+      ValueError: if `batch_shape` is not a vector.
+      ValueError: if `batch_shape` has non-positive elements.
+      ValueError: if `batch_shape` size is not the same as a
+        `distribution.batch_shape` size.
+    """
+    parameters = locals()
+    name = name or "BatchReshape" + distribution.name
+    self._distribution = distribution
+    with ops.name_scope(name, values=[batch_shape]) as name:
+      self._batch_shape_ = ops.convert_to_tensor(
+          batch_shape,
+          dtype=dtypes.int32,
+          name="batch_shape")
+      self._batch_shape_static = tensor_util.constant_value(self._batch_shape_)
+      if self._batch_shape_static is not None:
+        self._batch_shape_static = np.int32(self._batch_shape_static)
+      self._runtime_assertions = make_runtime_assertions(
+          self._distribution,
+          self._batch_shape_,
+          validate_args,
+          self._batch_shape_static)
+      super(BatchReshape, self).__init__(
+          dtype=self._distribution.dtype,
+          reparameterization_type=self._distribution.reparameterization_type,
+          validate_args=validate_args,
+          allow_nan_stats=allow_nan_stats,
+          parameters=parameters,
+          graph_parents=(
+              [self._batch_shape_] +
+              self._distribution._graph_parents),  # pylint: disable=protected-access
+          name=name)
+
+  @property
+  def distribution(self):
+    return self._distribution
+
+  def _batch_shape_tensor(self):
+    with ops.control_dependencies(self._runtime_assertions):
+      return array_ops.identity(self._batch_shape_)
+
+  def _batch_shape(self):
+    return tensor_shape.TensorShape(self._batch_shape_static)
+
+  def _event_shape_tensor(self):
+    with ops.control_dependencies(self._runtime_assertions):
+      return array_ops.identity(self.distribution.event_shape_tensor())
+
+  def _event_shape(self):
+    return self.distribution.event_shape
+
+  def _sample_n(self, n, seed=None):
+    with ops.control_dependencies(self._runtime_assertions):
+      x = self.distribution.sample(sample_shape=n, seed=seed)
+      new_shape = array_ops.concat([
+          [n],
+          self.batch_shape_tensor(),
+          self.event_shape_tensor(),
+      ], axis=0)
+      return array_ops.reshape(x, new_shape)
+
+  def _log_prob(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.log_prob, x)
+
+  def _prob(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.prob, x)
+
+  def _log_cdf(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.log_cdf, x)
+
+  def _cdf(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.cdf, x)
+
+  def _log_survival_function(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.log_survival_function, x)
+
+  def _survival_function(self, x):
+    return self._call_reshape_input_output(
+        self.distribution.survival_function, x)
+
+  def _entropy(self):
+    return self._call_and_reshape_output(
+        self.distribution.entropy,
+        [],
+        [tensor_shape.scalar()])
+
+  def _mean(self):
+    return self._call_and_reshape_output(self.distribution.mean)
+
+  def _mode(self):
+    return self._call_and_reshape_output(self.distribution.mode)
+
+  def _stddev(self):
+    return self._call_and_reshape_output(self.distribution.stddev)
+
+  def _variance(self):
+    return self._call_and_reshape_output(self.distribution.variance)
+
+  def _covariance(self):
+    return self._call_and_reshape_output(
+        self.distribution.covariance,
+        [self.event_shape_tensor()]*2,
+        [self.event_shape]*2)
+
+  def _sample_shape(self, x):
+    """Computes graph and static `sample_shape`."""
+    x_ndims = (array_ops.rank(x) if x.shape.ndims is None else x.shape.ndims)
+    event_ndims = (array_ops.size(self.event_shape_tensor())
+                   if self.event_shape.ndims is None
+                   else self.event_shape.ndims)
+    batch_ndims = (array_ops.size(self.batch_shape_tensor())
+                   if self.batch_shape.ndims is None
+                   else self.batch_shape.ndims)
+    sample_ndims = x_ndims - batch_ndims - event_ndims
+    if isinstance(sample_ndims, int):
+      static_sample_shape = x.shape[:sample_ndims]
+    else:
+      static_sample_shape = tensor_shape.TensorShape(None)
+    if static_sample_shape.is_fully_defined():
+      sample_shape = np.int32(static_sample_shape.as_list())
+    else:
+      sample_shape = array_ops.shape(x)[:sample_ndims]
+    return sample_shape, static_sample_shape
+
+  def _call_reshape_input_output(self, fn, x):
+    """Calls `fn`, appropriately reshaping its input `x` and output."""
+    with ops.control_dependencies(self._runtime_assertions):
+      sample_shape, static_sample_shape = self._sample_shape(x)
+      old_shape = array_ops.concat([
+          sample_shape,
+          self.distribution.batch_shape_tensor(),
+          self.event_shape_tensor(),
+      ], axis=0)
+      result = fn(array_ops.reshape(x, old_shape))
+      new_shape = array_ops.concat([
+          sample_shape,
+          self.batch_shape_tensor(),
+      ], axis=0)
+      result = array_ops.reshape(result, new_shape)
+      if (static_sample_shape.ndims is not None and
+          self.batch_shape.ndims is not None):
+        new_shape = static_sample_shape.concatenate(self.batch_shape)
+        result.set_shape(result.shape.merge_with(new_shape))
+      return result
+
+  def _call_and_reshape_output(
+      self,
+      fn,
+      event_shape_list=None,
+      static_event_shape_list=None):
+    """Calls `fn` and appropriately reshapes its output."""
+    with ops.control_dependencies(self._runtime_assertions):
+      if event_shape_list is None:
+        event_shape_list = [self._event_shape_tensor()]
+      if static_event_shape_list is None:
+        static_event_shape_list = [self.event_shape]
+      new_shape = array_ops.concat(
+          [self.batch_shape_tensor()] + event_shape_list,
+          axis=0)
+      result = array_ops.reshape(fn(), new_shape)
+      if (self.batch_shape.ndims is not None and
+          self.event_shape.ndims is not None):
+        event_shape = tensor_shape.TensorShape([])
+        for rss in static_event_shape_list:
+          event_shape = event_shape.concatenate(rss)
+        static_shape = result.shape.merge_with(
+            self.batch_shape.concatenate(event_shape))
+        result.set_shape(static_shape)
+      return result
+
+
+def make_runtime_assertions(
+    distribution,
+    batch_shape,
+    validate_args,
+    batch_shape_static):
+  """Helper to __init__ which makes or raises assertions."""
+  runtime_assertions = []
+
+  if batch_shape.shape.ndims is not None:
+    if batch_shape.shape.ndims != 1:
+      raise ValueError("`batch_shape` must be a vector "
+                       "(saw rank: {}).".format(
+                           batch_shape.shape.ndims))
+  elif validate_args:
+    runtime_assertions += [
+        check_ops.assert_rank(
+            batch_shape,
+            1,
+            message="`batch_shape` must be a vector.",
+            name="assert_batch_shape_is_vector"),
+    ]
+
+  batch_size_static = np.prod(batch_shape_static)
+  dist_batch_size_static = (
+      None if not distribution.batch_shape.is_fully_defined()
+      else np.prod(distribution.batch_shape).value)
+
+  if batch_size_static is not None and dist_batch_size_static is not None:
+    if batch_size_static != dist_batch_size_static:
+      raise ValueError("`batch_shape` size ({}) must match "
+                       "`distribution.batch_shape` size ({}).".format(
+                           batch_size_static,
+                           dist_batch_size_static))
+  elif validate_args:
+    runtime_assertions += [
+        check_ops.assert_equal(
+            math_ops.reduce_prod(batch_shape),
+            math_ops.reduce_prod(distribution.batch_shape_tensor()),
+            message=("`batch_shape` size must match "
+                     "`distributions.batch_shape` size."),
+            name="assert_batch_size"),
+    ]
+
+  if batch_shape_static is not None:
+    if np.any(batch_shape_static < 1):
+      raise ValueError("`batch_shape` elements must be positive "
+                       "(i.e., larger than zero).")
+  elif validate_args:
+    runtime_assertions += [
+        check_ops.assert_positive(
+            batch_shape,
+            message=("`batch_shape` elements must be positive "
+                     "(i.e., larger than zero)."),
+            name="assert_batch_shape_positive")
+    ]
+
+  return runtime_assertions
-- 
GitLab


From 86868a156860877fc6e8c3393baf4942b6b7dbd4 Mon Sep 17 00:00:00 2001
From: Andrew Selle <aselle@andyselle.com>
Date: Thu, 29 Mar 2018 07:59:46 -0700
Subject: [PATCH 670/960] Disable the toco binary in pip feature until it can
 used shared libs (#18061)

* Disable the toco binary in pip feature until it can used shared libraries.

The binary size was doubled by the saved model change. Since to process
saved models most of the TensorFlow runtime is needed. A workaround to
this is in the works and should be submitted in the next couple weeks.

* Fix linter errors with unused tensorflow libs

* Mollify the linter by removing os.
---
 tensorflow/contrib/lite/toco/python/BUILD           |  3 ---
 tensorflow/contrib/lite/toco/python/toco_wrapper.py | 13 +++++++++----
 tensorflow/tools/pip_package/build_pip_package.sh   |  4 +++-
 3 files changed, 12 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/python/BUILD b/tensorflow/contrib/lite/toco/python/BUILD
index 17115047d2..86d91bd3be 100644
--- a/tensorflow/contrib/lite/toco/python/BUILD
+++ b/tensorflow/contrib/lite/toco/python/BUILD
@@ -45,9 +45,6 @@ py_binary(
     name = "toco_wrapper",
     srcs = ["toco_wrapper.py"],
     srcs_version = "PY2AND3",
-    deps = [
-        "//tensorflow:tensorflow_py",
-    ],
 )
 
 tf_py_test(
diff --git a/tensorflow/contrib/lite/toco/python/toco_wrapper.py b/tensorflow/contrib/lite/toco/python/toco_wrapper.py
index e39b5f22c7..6d6b500d7e 100644
--- a/tensorflow/contrib/lite/toco/python/toco_wrapper.py
+++ b/tensorflow/contrib/lite/toco/python/toco_wrapper.py
@@ -22,14 +22,19 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
 import sys
-import tensorflow as tf
 
 
 def main():
   # Pip installs the binary in aux-bin off of main site-package install.
   # Just find it and exec, passing all arguments in the process.
   # TODO(aselle): it is unfortunate to use all of tensorflow to lookup binary.
-  binary = os.path.join(tf.__path__[0], 'aux-bin/toco')
-  os.execvp(binary, sys.argv)
+  print("""TOCO from pip install is currently not working on command line.
+Please use the python TOCO API or use
+bazel run tensorflow/contrib/lite:toco -- <args> from a TensorFlow source dir.
+""")
+  sys.exit(1)
+  # TODO(aselle): Replace this when we find a way to run toco without
+  # blowing up executable size.
+  # binary = os.path.join(tf.__path__[0], 'aux-bin/toco')
+  # os.execvp(binary, sys.argv)
diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index dc31e4c5f7..feb3114bde 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -139,7 +139,9 @@ function main() {
     fi
     mkdir "${TMPDIR}/tensorflow/aux-bin"
     # Install toco as a binary in aux-bin.
-    cp bazel-bin/tensorflow/contrib/lite/toco/toco ${TMPDIR}/tensorflow/aux-bin/
+    # TODO(aselle): Re-enable this when we find a way to do it without doubling
+    # the whl size (over the limit).
+    # cp bazel-bin/tensorflow/contrib/lite/toco/toco ${TMPDIR}/tensorflow/aux-bin/
   fi
 
   # protobuf pip package doesn't ship with header files. Copy the headers
-- 
GitLab


From 608fb59318ca0a1f2a05fae4d23b06cf6e162300 Mon Sep 17 00:00:00 2001
From: Rachel Lim <rachelim@google.com>
Date: Thu, 29 Mar 2018 08:19:17 -0700
Subject: [PATCH 671/960] [tf.data] Optimizations on make_csv_dataset
 internals.

PiperOrigin-RevId: 190933143
---
 tensorflow/contrib/data/python/ops/readers.py | 61 ++++++++++++++-----
 1 file changed, 46 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/data/python/ops/readers.py b/tensorflow/contrib/data/python/ops/readers.py
index 95edca6cdd..9a48aa02fb 100644
--- a/tensorflow/contrib/data/python/ops/readers.py
+++ b/tensorflow/contrib/data/python/ops/readers.py
@@ -18,9 +18,11 @@ from __future__ import division
 from __future__ import print_function
 
 import csv
+from math import ceil
 
 import numpy as np
 
+from tensorflow.contrib.data.python.ops import batching
 from tensorflow.contrib.data.python.ops import interleave_ops
 from tensorflow.contrib.data.python.ops import shuffle_ops
 from tensorflow.python.data.ops import dataset_ops
@@ -176,6 +178,9 @@ def make_csv_dataset(
     shuffle_buffer_size=10000,
     shuffle_seed=None,
     prefetch_buffer_size=1,
+    num_parallel_reads=1,
+    num_parallel_parser_calls=2,
+    sloppy=False,
     default_float_type=dtypes.float32,
     num_rows_for_inference=100,
 ):
@@ -231,6 +236,15 @@ def make_csv_dataset(
     prefetch_buffer_size: An int specifying the number of feature batches to
       prefetch for performance improvement. Recommended value is the number of
       batches consumed per training step.
+    num_parallel_reads: Number of threads used to read CSV records from files.
+      If >1, the results will be interleaved.
+    num_parallel_parser_calls: Number of parallel invocations of the CSV parsing
+      function on CSV records.
+    sloppy: If `True`, reading performance will be improved at
+      the cost of non-deterministic ordering. If `False`, the order of elements
+      produced is deterministic prior to shuffling (elements are still
+      randomized if `shuffle=True`. Note that if the seed is set, then order
+      of elements after shuffling is deterministic). Defaults to `False`.
     default_float_type: Either `tf.float32` or `tf.float64`. If defaults are
       not provided, float-like strings are interpreted to be this type.
     num_rows_for_inference: Number of rows of a file to use for type inference
@@ -247,11 +261,16 @@ def make_csv_dataset(
   Raises:
     ValueError: If any of the arguments is malformed.
   """
-  filenames = _get_file_names(file_pattern, shuffle)
+  # Create dataset of all matching filenames
+  filenames = _get_file_names(file_pattern, False)
+  dataset = dataset_ops.Dataset.from_tensor_slices(filenames)
+  if shuffle:
+    dataset = dataset.shuffle(len(filenames), shuffle_seed)
+
+  # Clean arguments; figure out column names and defaults
   if comment is not None and len(comment) != 1:
     raise ValueError("`comment` arg must be a single-character string or None")
 
-  # Clean arguments; figure out column names and defaults
   if column_names is None:
     if not header:
       raise ValueError("Cannot infer column names without a header line.")
@@ -272,7 +291,6 @@ def make_csv_dataset(
         filenames, len(column_names), field_delim, use_quote_delim, na_value,
         header, comment, default_float_type, num_rows_for_inference)
 
-  dataset = dataset_ops.Dataset.from_tensor_slices(filenames)
   if label_name is not None and label_name not in column_names:
     raise ValueError("`label_name` provided must be one of the columns.")
 
@@ -311,16 +329,31 @@ def make_csv_dataset(
       return features, label
     return features
 
-  # TODO(rachelim): interleave records from files for better shuffling
-  dataset = dataset.flat_map(filename_to_dataset)
-  # TODO(rachelim): use fused shuffle_and_repeat for perf
-  if shuffle:
+  # Read files sequentially or in parallel
+  dataset = dataset.apply(
+      interleave_ops.parallel_interleave(
+          filename_to_dataset, cycle_length=num_parallel_reads, sloppy=sloppy))
+
+  if num_epochs != 1 and shuffle:
+    # Use shuffle_and_repeat for perf
+    dataset = dataset.apply(
+        shuffle_ops.shuffle_and_repeat(shuffle_buffer_size, num_epochs,
+                                       shuffle_seed))
+  elif shuffle:
     dataset = dataset.shuffle(shuffle_buffer_size, shuffle_seed)
-  if num_epochs != 1:
+  elif num_epochs != 1:
     dataset = dataset.repeat(num_epochs)
 
-  dataset = dataset.batch(batch_size)
-  dataset = dataset.map(decode_csv)
+  # Use map_and_batch for perf
+  # TODO(b/76425672): use num_parallel_calls for better performance tuning when
+  # that is added
+  dataset = dataset.apply(
+      batching.map_and_batch(
+          map_func=decode_csv,
+          batch_size=batch_size,
+          num_parallel_batches=int(
+              ceil(num_parallel_parser_calls / batch_size))))
+
   dataset = dataset.prefetch(prefetch_buffer_size)
   return dataset
 
@@ -416,12 +449,10 @@ def make_batched_features_dataset(file_pattern,
     `Tensor` or `SparseTensor` objects.
   """
   # Create dataset of all matching filenames
+  filenames = _get_file_names(file_pattern, False)
+  dataset = dataset_ops.Dataset.from_tensor_slices(filenames)
   if shuffle:
-    dataset = dataset_ops.Dataset.list_files(file_pattern, shuffle=True)
-  else:
-    # TODO(b/73959787): Use Dataset.list_files() once ordering is deterministic.
-    filenames = _get_file_names(file_pattern, shuffle)
-    dataset = dataset_ops.Dataset.from_tensor_slices(filenames)
+    dataset = dataset.shuffle(len(filenames), shuffle_seed)
 
   # Read `Example` records from files as tensor objects.
   if reader_args is None:
-- 
GitLab


From a98351b9c6c691b6873ef5a5c3e8e48bf42bd14c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 09:41:53 -0700
Subject: [PATCH 672/960] Upgrade Eigen version.

PiperOrigin-RevId: 190942370
---
 tensorflow/core/kernels/cwise_ops.h | 21 ---------------------
 tensorflow/workspace.bzl            |  8 ++++----
 2 files changed, 4 insertions(+), 25 deletions(-)

diff --git a/tensorflow/core/kernels/cwise_ops.h b/tensorflow/core/kernels/cwise_ops.h
index 06918075a4..a80905d145 100644
--- a/tensorflow/core/kernels/cwise_ops.h
+++ b/tensorflow/core/kernels/cwise_ops.h
@@ -27,27 +27,6 @@ limitations under the License.
 #include "tensorflow/core/kernels/bounds_check.h"
 
 namespace Eigen {
-namespace numext {
-#if GOOGLE_CUDA
-template <>
-EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<float> exp(
-    const std::complex<float>& x) {
-  auto com = ::expf(x.real());
-  auto res_real = com * ::cosf(x.imag());
-  auto res_imag = com * ::sinf(x.imag());
-  return std::complex<float>(res_real, res_imag);
-}
-template <>
-EIGEN_DEVICE_FUNC EIGEN_ALWAYS_INLINE std::complex<double> exp(
-    const std::complex<double>& x) {
-  auto com = ::exp(x.real());
-  auto res_real = com * ::cos(x.imag());
-  auto res_imag = com * ::sin(x.imag());
-  return std::complex<double>(res_real, res_imag);
-}
-#endif
-}  // namespace numext
-
 namespace internal {
 
 template <typename T>
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 9fcbfb664b..0e31358236 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -74,11 +74,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "eigen_archive",
       urls = [
-          "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/2355b229ea4c.tar.gz",
-          "https://bitbucket.org/eigen/eigen/get/2355b229ea4c.tar.gz",
+          "https://mirror.bazel.build/bitbucket.org/eigen/eigen/get/6913f0cf7d06.tar.gz",
+          "https://bitbucket.org/eigen/eigen/get/6913f0cf7d06.tar.gz",
       ],
-      sha256 = "0cadb31a35b514bf2dfd6b5d38205da94ef326ec6908fc3fd7c269948467214f",
-      strip_prefix = "eigen-eigen-2355b229ea4c",
+      sha256 = "791b836cacd03e20bae5bdd25f1c4a5505a0a9975ba94a61eb4e2631fbd1d53a",
+      strip_prefix = "eigen-eigen-6913f0cf7d06",
       build_file = str(Label("//third_party:eigen.BUILD")),
       patch_file = str(Label("//third_party:eigen_fix_cuda_compilation.patch"))
   )
-- 
GitLab


From ae3d20f9aef78554f0d0f5eec13982e9802a45d2 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Thu, 29 Mar 2018 09:42:05 -0700
Subject: [PATCH 673/960] Add bitcast for equal bitwidth casts.

Map bitcasts to XLA bitcast HLO if the bitwidth of the elementtype is the same.

PiperOrigin-RevId: 190942398
---
 tensorflow/compiler/tests/unary_ops_test.py   | 14 ++++++
 tensorflow/compiler/tf2xla/kernels/cast_op.cc | 45 +++++++++++++++++++
 2 files changed, 59 insertions(+)

diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index 3d3e112f48..a8ab235378 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -600,6 +600,20 @@ class UnaryOpsTest(XLATestCase):
               src,
               expected=dst)
 
+  def testBitcast(self):
+    self._assertOpOutputMatchesExpected(
+        lambda x: array_ops.bitcast(x, dtypes.int32),
+        np.array([1, 0x3f800000], np.int32),
+        expected=np.array([1, 0x3f800000], np.int32))
+    self._assertOpOutputMatchesExpected(
+        lambda x: array_ops.bitcast(x, dtypes.float32),
+        np.array([1, 0x3f800000], np.int32),
+        expected=np.array([1e-45, 1.0], np.float32))
+    self._assertOpOutputMatchesExpected(
+        lambda x: array_ops.bitcast(x, dtypes.int32),
+        np.array([1e-45, 1.0], np.float32),
+        expected=np.array([1, 0x3f800000], np.int32))
+
   def testInvertPermutation(self):
     self._assertOpOutputMatchesExpected(
         array_ops.invert_permutation,
diff --git a/tensorflow/compiler/tf2xla/kernels/cast_op.cc b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
index 43a6a747c6..c52b2dcb7e 100644
--- a/tensorflow/compiler/tf2xla/kernels/cast_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/cast_op.cc
@@ -62,5 +62,50 @@ class CastOp : public XlaOpKernel {
 
 REGISTER_XLA_OP(Name("Cast"), CastOp);
 
+class BitcastOp : public XlaOpKernel {
+ public:
+  explicit BitcastOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("T", &src_dtype_));
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("type", &dst_dtype_));
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(src_dtype_, &src_type_));
+    OP_REQUIRES_OK(ctx, DataTypeToPrimitiveType(dst_dtype_, &dst_type_));
+  }
+
+  void Compile(XlaOpKernelContext* ctx) override {
+    xla::ComputationBuilder* builder = ctx->builder();
+    xla::ComputationDataHandle input = ctx->Input(0);
+    xla::ComputationDataHandle output;
+
+    if (src_dtype_ == dst_dtype_) {
+      output = input;
+    } else {
+      // The only complex type in XLA is C64, so error out if the bitcast has a
+      // complex source or destination type and the bitcast is not trivial.
+      OP_REQUIRES(ctx,
+                  !xla::primitive_util::IsComplexType(src_type_) &&
+                      !xla::primitive_util::IsComplexType(dst_type_),
+                  errors::Unimplemented("Complex types not supported."));
+      // XLA bitcast requires that the bit-width of the source and destination
+      // matches, and currently only the simple lowering is performed.
+      OP_REQUIRES(ctx,
+                  xla::primitive_util::BitWidth(src_type_) ==
+                      xla::primitive_util::BitWidth(dst_type_),
+                  errors::Unimplemented(
+                      "Only bitcasts between equally sized types supported."));
+      output = builder->BitcastConvertType(input, dst_type_);
+    }
+
+    ctx->SetOutput(0, output);
+  }
+
+ protected:
+  DataType src_dtype_, dst_dtype_;
+  xla::PrimitiveType src_type_, dst_type_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(BitcastOp);
+};
+
+REGISTER_XLA_OP(Name("Bitcast"), BitcastOp);
+
 }  // anonymous namespace
 }  // namespace tensorflow
-- 
GitLab


From a2b6c3c124664d682094a1ecfa9cc00cca8ada85 Mon Sep 17 00:00:00 2001
From: Younghee Kwon <youngheek@google.com>
Date: Thu, 29 Mar 2018 09:43:19 -0700
Subject: [PATCH 674/960] Added kernels and estimators for Gradient Boosting
 Trees algorithm. BoostedTreesClassifier and BoostedTreesRegressor are added
 to tf.estimator. Also some training utility functions are added to
 tf.contrib.estimator.

PiperOrigin-RevId: 190942599
---
 tensorflow/contrib/cmake/python_modules.txt   |    1 +
 tensorflow/contrib/cmake/python_protos.txt    |    1 +
 tensorflow/contrib/cmake/tf_core_ops.cmake    |    7 +-
 tensorflow/contrib/cmake/tf_python.cmake      |    1 +
 tensorflow/contrib/estimator/BUILD            |   31 +
 tensorflow/contrib/estimator/__init__.py      |    3 +
 .../python/estimator/boosted_trees.py         |  323 ++++
 .../python/estimator/boosted_trees_test.py    |  207 +++
 tensorflow/contrib/makefile/tf_op_files.txt   |    6 +
 .../contrib/makefile/tf_proto_files.txt       |    1 +
 tensorflow/core/BUILD                         |    3 +
 ...tedTreesCalculateBestGainsPerFeature.pbtxt |   87 +
 .../api_def_BoostedTreesCreateEnsemble.pbtxt  |   23 +
 ..._def_BoostedTreesDeserializeEnsemble.pbtxt |   26 +
 ...BoostedTreesEnsembleResourceHandleOp.pbtxt |    5 +
 ...pi_def_BoostedTreesGetEnsembleStates.pbtxt |   35 +
 ...api_def_BoostedTreesMakeStatsSummary.pbtxt |   56 +
 .../api_def_BoostedTreesPredict.pbtxt         |   41 +
 ...pi_def_BoostedTreesSerializeEnsemble.pbtxt |   23 +
 .../api_def_BoostedTreesTrainingPredict.pbtxt |   69 +
 .../api_def_BoostedTreesUpdateEnsemble.pbtxt  |   82 +
 ...ef_IsBoostedTreesEnsembleInitialized.pbtxt |   17 +
 tensorflow/core/kernels/BUILD                 |    7 +
 tensorflow/core/kernels/boosted_trees/BUILD   |   89 +
 .../kernels/boosted_trees/boosted_trees.proto |  113 ++
 .../kernels/boosted_trees/prediction_ops.cc   |  263 +++
 .../kernels/boosted_trees/resource_ops.cc     |  189 +++
 .../core/kernels/boosted_trees/resources.cc   |  301 ++++
 .../core/kernels/boosted_trees/resources.h    |  221 +++
 .../core/kernels/boosted_trees/stats_ops.cc   |  296 ++++
 .../kernels/boosted_trees/training_ops.cc     |  219 +++
 tensorflow/core/ops/boosted_trees_ops.cc      |  319 ++++
 tensorflow/python/BUILD                       |   22 +
 tensorflow/python/__init__.py                 |    2 +
 tensorflow/python/estimator/BUILD             |   48 +
 .../python/estimator/canned/boosted_trees.py  |  736 +++++++++
 .../estimator/canned/boosted_trees_test.py    |  799 +++++++++
 tensorflow/python/estimator/estimator_lib.py  |    4 +
 .../python/kernel_tests/boosted_trees/BUILD   |   76 +
 .../kernel_tests/boosted_trees/__init__.py    |    0
 .../boosted_trees/prediction_ops_test.py      |  926 +++++++++++
 .../boosted_trees/resource_ops_test.py        |  228 +++
 .../boosted_trees/stats_ops_test.py           |  289 ++++
 .../boosted_trees/training_ops_test.py        | 1465 +++++++++++++++++
 tensorflow/python/ops/boosted_trees_ops.py    |  160 ++
 tensorflow/python/training/device_setter.py   |   13 +-
 ....estimator.-boosted-trees-classifier.pbtxt |   54 +
 ...w.estimator.-boosted-trees-regressor.pbtxt |   54 +
 .../api/golden/tensorflow.estimator.pbtxt     |    8 +
 49 files changed, 7939 insertions(+), 10 deletions(-)
 create mode 100644 tensorflow/contrib/estimator/python/estimator/boosted_trees.py
 create mode 100644 tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesCreateEnsemble.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesDeserializeEnsemble.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesEnsembleResourceHandleOp.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesGetEnsembleStates.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesMakeStatsSummary.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesPredict.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesSerializeEnsemble.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesTrainingPredict.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_BoostedTreesUpdateEnsemble.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_IsBoostedTreesEnsembleInitialized.pbtxt
 create mode 100644 tensorflow/core/kernels/boosted_trees/BUILD
 create mode 100644 tensorflow/core/kernels/boosted_trees/boosted_trees.proto
 create mode 100644 tensorflow/core/kernels/boosted_trees/prediction_ops.cc
 create mode 100644 tensorflow/core/kernels/boosted_trees/resource_ops.cc
 create mode 100644 tensorflow/core/kernels/boosted_trees/resources.cc
 create mode 100644 tensorflow/core/kernels/boosted_trees/resources.h
 create mode 100644 tensorflow/core/kernels/boosted_trees/stats_ops.cc
 create mode 100644 tensorflow/core/kernels/boosted_trees/training_ops.cc
 create mode 100644 tensorflow/core/ops/boosted_trees_ops.cc
 create mode 100644 tensorflow/python/estimator/canned/boosted_trees.py
 create mode 100644 tensorflow/python/estimator/canned/boosted_trees_test.py
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/BUILD
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/__init__.py
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
 create mode 100644 tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
 create mode 100644 tensorflow/python/ops/boosted_trees_ops.py
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-classifier.pbtxt
 create mode 100644 tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-regressor.pbtxt

diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 112b690511..cc7d791042 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -79,6 +79,7 @@ tensorflow/python/keras/_impl/keras/preprocessing
 tensorflow/python/keras/_impl/keras/utils
 tensorflow/python/keras/_impl/keras/wrappers
 tensorflow/python/kernel_tests
+tensorflow/python/kernel_tests/boosted_trees
 tensorflow/python/kernel_tests/distributions
 tensorflow/python/kernel_tests/linalg
 tensorflow/python/kernel_tests/random
diff --git a/tensorflow/contrib/cmake/python_protos.txt b/tensorflow/contrib/cmake/python_protos.txt
index c03c0c80fe..0c80d529af 100644
--- a/tensorflow/contrib/cmake/python_protos.txt
+++ b/tensorflow/contrib/cmake/python_protos.txt
@@ -1,4 +1,5 @@
 tensorflow/core
+tensorflow/core/kernels/boosted_trees
 tensorflow/core/profiler
 tensorflow/python
 tensorflow/contrib/boosted_trees/proto
diff --git a/tensorflow/contrib/cmake/tf_core_ops.cmake b/tensorflow/contrib/cmake/tf_core_ops.cmake
index d6712aa2b4..092a48bc6b 100644
--- a/tensorflow/contrib/cmake/tf_core_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_core_ops.cmake
@@ -15,8 +15,9 @@
 set(tf_op_lib_names
     "audio_ops"
     "array_ops"
-		"batch_ops"
+    "batch_ops"
     "bitwise_ops"
+    "boosted_trees_ops"
     "candidate_sampling_ops"
     "checkpoint_ops"
     "control_flow_ops"
@@ -28,7 +29,7 @@ set(tf_op_lib_names
     "image_ops"
     "io_ops"
     "linalg_ops"
-		"list_ops"
+    "list_ops"
     "lookup_ops"
     "logging_ops"
     "manip_ops"
@@ -48,7 +49,7 @@ set(tf_op_lib_names
     "state_ops"
     "stateless_random_ops"
     "string_ops"
-		"summary_ops"
+    "summary_ops"
     "training_ops"
 )
 
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 31e715b654..b776307924 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -319,6 +319,7 @@ GENERATE_PYTHON_OP_LIB("audio_ops")
 GENERATE_PYTHON_OP_LIB("array_ops")
 GENERATE_PYTHON_OP_LIB("batch_ops")
 GENERATE_PYTHON_OP_LIB("bitwise_ops")
+GENERATE_PYTHON_OP_LIB("boosted_trees_ops")
 GENERATE_PYTHON_OP_LIB("math_ops")
 GENERATE_PYTHON_OP_LIB("functional_ops")
 GENERATE_PYTHON_OP_LIB("candidate_sampling_ops")
diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index d125e40f6c..2be62c9438 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -14,6 +14,7 @@ py_library(
     srcs = ["__init__.py"],
     srcs_version = "PY2AND3",
     deps = [
+        ":boosted_trees",
         ":dnn",
         ":dnn_linear_combined",
         ":extenders",
@@ -26,6 +27,36 @@ py_library(
     ],
 )
 
+py_library(
+    name = "boosted_trees",
+    srcs = ["python/estimator/boosted_trees.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python/estimator",
+        "//tensorflow/python/estimator:boosted_trees",
+    ],
+)
+
+py_test(
+    name = "boosted_trees_test",
+    size = "medium",
+    srcs = ["python/estimator/boosted_trees_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+        "notsan",
+    ],
+    deps = [
+        ":boosted_trees",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:training",
+        "//tensorflow/python/estimator:numpy_io",
+        "//tensorflow/python/feature_column",
+        "//third_party/py/numpy",
+    ],
+)
+
 py_library(
     name = "dnn",
     srcs = ["python/estimator/dnn.py"],
diff --git a/tensorflow/contrib/estimator/__init__.py b/tensorflow/contrib/estimator/__init__.py
index 6b9f9575b6..d2fc2c4bfa 100644
--- a/tensorflow/contrib/estimator/__init__.py
+++ b/tensorflow/contrib/estimator/__init__.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 # pylint: disable=unused-import,line-too-long,wildcard-import
+from tensorflow.contrib.estimator.python.estimator.boosted_trees import *
 from tensorflow.contrib.estimator.python.estimator.dnn import *
 from tensorflow.contrib.estimator.python.estimator.dnn_linear_combined import *
 from tensorflow.contrib.estimator.python.estimator.extenders import *
@@ -44,6 +45,8 @@ _allowed_symbols = [
     'DNNEstimator',
     'DNNLinearCombinedEstimator',
     'LinearEstimator',
+    'boosted_trees_classifier_train_in_memory',
+    'boosted_trees_regressor_train_in_memory',
     'call_logit_fn',
     'dnn_logit_fn_builder',
     'linear_logit_fn_builder',
diff --git a/tensorflow/contrib/estimator/python/estimator/boosted_trees.py b/tensorflow/contrib/estimator/python/estimator/boosted_trees.py
new file mode 100644
index 0000000000..5880164519
--- /dev/null
+++ b/tensorflow/contrib/estimator/python/estimator/boosted_trees.py
@@ -0,0 +1,323 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Boosted Trees estimators."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator.canned import boosted_trees as canned_boosted_trees
+
+
+class _BoostedTreesEstimator(estimator.Estimator):
+  """An Estimator for Tensorflow Boosted Trees models."""
+
+  def __init__(self,
+               feature_columns,
+               n_batches_per_layer,
+               head,
+               model_dir=None,
+               weight_column=None,
+               n_trees=100,
+               max_depth=6,
+               learning_rate=0.1,
+               l1_regularization=0.,
+               l2_regularization=0.,
+               tree_complexity=0.,
+               config=None):
+    """Initializes a `BoostedTreesEstimator` instance.
+
+    Args:
+      feature_columns: An iterable containing all the feature columns used by
+        the model. All items in the set should be instances of classes derived
+        from `FeatureColumn`.
+      n_batches_per_layer: the number of batches to collect statistics per
+        layer.
+      head: the `Head` instance defined for Estimator.
+      model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+      weight_column: A string or a `_NumericColumn` created by
+        `tf.feature_column.numeric_column` defining feature column representing
+        weights. It is used to downweight or boost examples during training. It
+        will be multiplied by the loss of the example. If it is a string, it is
+        used as a key to fetch weight tensor from the `features`. If it is a
+        `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
+        then weight_column.normalizer_fn is applied on it to get weight tensor.
+      n_trees: number trees to be created.
+      max_depth: maximum depth of the tree to grow.
+      learning_rate: shrinkage parameter to be used when a tree added to the
+        model.
+      l1_regularization: regularization multiplier applied to the absolute
+        weights of the tree leafs.
+      l2_regularization: regularization multiplier applied to the square weights
+        of the tree leafs.
+      tree_complexity: regularization factor to penalize trees with more leaves.
+      config: `RunConfig` object to configure the runtime settings.
+    """
+    # TODO(youngheek): param validations.
+
+    # HParams for the model.
+    tree_hparams = canned_boosted_trees.TreeHParams(
+        n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
+        tree_complexity)
+
+    def _model_fn(features, labels, mode, config):
+      return canned_boosted_trees._bt_model_fn(  # pylint: disable=protected-access
+          features, labels, mode, head, feature_columns, tree_hparams,
+          n_batches_per_layer, config)
+
+    super(_BoostedTreesEstimator, self).__init__(
+        model_fn=_model_fn, model_dir=model_dir, config=config)
+
+
+def boosted_trees_classifier_train_in_memory(
+    train_input_fn,
+    feature_columns,
+    model_dir=None,
+    n_classes=canned_boosted_trees._HOLD_FOR_MULTI_CLASS_SUPPORT,
+    weight_column=None,
+    label_vocabulary=None,
+    n_trees=100,
+    max_depth=6,
+    learning_rate=0.1,
+    l1_regularization=0.,
+    l2_regularization=0.,
+    tree_complexity=0.,
+    config=None,
+    train_hooks=None):
+  """Trains a boosted tree classifier with in memory dataset.
+
+  Example:
+
+  ```python
+  bucketized_feature_1 = bucketized_column(
+    numeric_column('feature_1'), BUCKET_BOUNDARIES_1)
+  bucketized_feature_2 = bucketized_column(
+    numeric_column('feature_2'), BUCKET_BOUNDARIES_2)
+
+  def input_fn_train():
+    dataset = create-dataset-from-training-data
+    # Don't use repeat or cache, since it is assumed to be one epoch
+    # This is either tf.data.Dataset, or a tuple of feature dict and label.
+    return dataset
+
+  classifier = boosted_trees_classifier_train_in_memory(
+      train_input_fn,
+      feature_columns=[bucketized_feature_1, bucketized_feature_2],
+      n_trees=100,
+      ... <some other params>
+  )
+
+  def input_fn_eval():
+    ...
+    return dataset
+
+  metrics = classifier.evaluate(input_fn=input_fn_eval, steps=10)
+  ```
+
+  Args:
+    train_input_fn: the input function returns a dataset containing a single
+      epoch of *unbatched* features and labels.
+    feature_columns: An iterable containing all the feature columns used by
+      the model. All items in the set should be instances of classes derived
+      from `FeatureColumn`.
+    model_dir: Directory to save model parameters, graph and etc. This can
+      also be used to load checkpoints from the directory into a estimator
+      to continue training a previously saved model.
+    n_classes: number of label classes. Default is binary classification.
+      Multiclass support is not yet implemented.
+    weight_column: A string or a `_NumericColumn` created by
+      `tf.feature_column.numeric_column` defining feature column representing
+      weights. It is used to downweight or boost examples during training. It
+      will be multiplied by the loss of the example. If it is a string, it is
+      used as a key to fetch weight tensor from the `features`. If it is a
+      `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
+      then weight_column.normalizer_fn is applied on it to get weight tensor.
+    label_vocabulary: A list of strings represents possible label values. If
+      given, labels must be string type and have any value in
+      `label_vocabulary`. If it is not given, that means labels are
+      already encoded as integer or float within [0, 1] for `n_classes=2` and
+      encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 .
+      Also there will be errors if vocabulary is not provided and labels are
+      string.
+    n_trees: number trees to be created.
+    max_depth: maximum depth of the tree to grow.
+    learning_rate: shrinkage parameter to be used when a tree added to the
+      model.
+    l1_regularization: regularization multiplier applied to the absolute
+      weights of the tree leafs.
+    l2_regularization: regularization multiplier applied to the square weights
+      of the tree leafs.
+    tree_complexity: regularization factor to penalize trees with more leaves.
+    config: `RunConfig` object to configure the runtime settings.
+    train_hooks: a list of Hook instances to be passed to estimator.train().
+
+  Returns:
+    a `BoostedTreesClassifier` instance created with the given arguments and
+      trained with the data loaded up on memory from the input_fn.
+
+  Raises:
+    ValueError: when wrong arguments are given or unsupported functionalities
+       are requested.
+  """
+  # pylint: disable=protected-access
+  # TODO(nponomareva): Support multi-class cases.
+  if n_classes == canned_boosted_trees._HOLD_FOR_MULTI_CLASS_SUPPORT:
+    n_classes = 2
+  head, closed_form = (
+      canned_boosted_trees._create_classification_head_and_closed_form(
+          n_classes, weight_column, label_vocabulary=label_vocabulary))
+
+  # HParams for the model.
+  tree_hparams = canned_boosted_trees.TreeHParams(
+      n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
+      tree_complexity)
+
+  def _model_fn(features, labels, mode, config):
+    return canned_boosted_trees._bt_model_fn(
+        features,
+        labels,
+        mode,
+        head,
+        feature_columns,
+        tree_hparams,
+        n_batches_per_layer=1,
+        config=config,
+        closed_form_grad_and_hess_fn=closed_form,
+        train_in_memory=True)
+
+  in_memory_classifier = estimator.Estimator(
+      model_fn=_model_fn, model_dir=model_dir, config=config)
+
+  in_memory_classifier.train(input_fn=train_input_fn, hooks=train_hooks)
+
+  return in_memory_classifier
+  # pylint: enable=protected-access
+
+
+def boosted_trees_regressor_train_in_memory(
+    train_input_fn,
+    feature_columns,
+    model_dir=None,
+    label_dimension=canned_boosted_trees._HOLD_FOR_MULTI_DIM_SUPPORT,
+    weight_column=None,
+    n_trees=100,
+    max_depth=6,
+    learning_rate=0.1,
+    l1_regularization=0.,
+    l2_regularization=0.,
+    tree_complexity=0.,
+    config=None,
+    train_hooks=None):
+  """Trains a boosted tree regressor with in memory dataset.
+
+  Example:
+
+  ```python
+  bucketized_feature_1 = bucketized_column(
+    numeric_column('feature_1'), BUCKET_BOUNDARIES_1)
+  bucketized_feature_2 = bucketized_column(
+    numeric_column('feature_2'), BUCKET_BOUNDARIES_2)
+
+  def input_fn_train():
+    dataset = create-dataset-from-training-data
+    # Don't use repeat or cache, since it is assumed to be one epoch
+    # This is either tf.data.Dataset, or a tuple of feature dict and label.
+    return dataset
+
+  regressor = boosted_trees_regressor_train_in_memory(
+      train_input_fn,
+      feature_columns=[bucketized_feature_1, bucketized_feature_2],
+      n_trees=100,
+      ... <some other params>
+  )
+
+  def input_fn_eval():
+    ...
+    return dataset
+
+  metrics = regressor.evaluate(input_fn=input_fn_eval, steps=10)
+  ```
+
+  Args:
+    train_input_fn: the input function returns a dataset containing a single
+      epoch of *unbatched* features and labels.
+    feature_columns: An iterable containing all the feature columns used by
+      the model. All items in the set should be instances of classes derived
+      from `FeatureColumn`.
+    model_dir: Directory to save model parameters, graph and etc. This can
+      also be used to load checkpoints from the directory into a estimator
+      to continue training a previously saved model.
+    label_dimension: Number of regression targets per example.
+      Multi-dimensional support is not yet implemented.
+    weight_column: A string or a `_NumericColumn` created by
+      `tf.feature_column.numeric_column` defining feature column representing
+      weights. It is used to downweight or boost examples during training. It
+      will be multiplied by the loss of the example. If it is a string, it is
+      used as a key to fetch weight tensor from the `features`. If it is a
+      `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
+      then weight_column.normalizer_fn is applied on it to get weight tensor.
+    n_trees: number trees to be created.
+    max_depth: maximum depth of the tree to grow.
+    learning_rate: shrinkage parameter to be used when a tree added to the
+      model.
+    l1_regularization: regularization multiplier applied to the absolute
+      weights of the tree leafs.
+    l2_regularization: regularization multiplier applied to the square weights
+      of the tree leafs.
+    tree_complexity: regularization factor to penalize trees with more leaves.
+    config: `RunConfig` object to configure the runtime settings.
+    train_hooks: a list of Hook instances to be passed to estimator.train().
+
+  Returns:
+    a `BoostedTreesClassifier` instance created with the given arguments and
+      trained with the data loaded up on memory from the input_fn.
+
+  Raises:
+    ValueError: when wrong arguments are given or unsupported functionalities
+       are requested.
+  """
+  # pylint: disable=protected-access
+  # TODO(nponomareva): Extend it to multi-dimension cases.
+  if label_dimension == canned_boosted_trees._HOLD_FOR_MULTI_DIM_SUPPORT:
+    label_dimension = 1
+  head = canned_boosted_trees._create_regression_head(label_dimension,
+                                                      weight_column)
+
+  # HParams for the model.
+  tree_hparams = canned_boosted_trees.TreeHParams(
+      n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
+      tree_complexity)
+
+  def _model_fn(features, labels, mode, config):
+    return canned_boosted_trees._bt_model_fn(
+        features,
+        labels,
+        mode,
+        head,
+        feature_columns,
+        tree_hparams,
+        n_batches_per_layer=1,
+        config=config,
+        train_in_memory=True)
+
+  in_memory_regressor = estimator.Estimator(
+      model_fn=_model_fn, model_dir=model_dir, config=config)
+
+  in_memory_regressor.train(input_fn=train_input_fn, hooks=train_hooks)
+
+  return in_memory_regressor
+  # pylint: enable=protected-access
diff --git a/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py b/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py
new file mode 100644
index 0000000000..e99a87f3b3
--- /dev/null
+++ b/tensorflow/contrib/estimator/python/estimator/boosted_trees_test.py
@@ -0,0 +1,207 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests boosted_trees estimators."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.estimator.python.estimator import boosted_trees
+from tensorflow.python.estimator.canned import boosted_trees as canned_boosted_trees
+from tensorflow.python.estimator.inputs import numpy_io
+from tensorflow.python.feature_column import feature_column
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import checkpoint_utils
+
+NUM_FEATURES = 3
+
+BUCKET_BOUNDARIES = [-2., .5, 12.]  # Boundaries for all the features.
+INPUT_FEATURES = np.array(
+    [
+        [12.5, 1.0, -2.001, -2.0001, -1.999],  # feature_0 quantized:[3,2,0,0,1]
+        [2.0, -3.0, 0.5, 0.0, 0.4995],         # feature_1 quantized:[2,0,2,1,1]
+        [3.0, 20.0, 50.0, -100.0, 102.75],     # feature_2 quantized:[2,3,3,0,3]
+    ],
+    dtype=np.float32)
+CLASSIFICATION_LABELS = [[0.], [1.], [1.], [0.], [0.]]
+REGRESSION_LABELS = [[1.5], [0.3], [0.2], [2.], [5.]]
+FEATURES_DICT = {'f_%d' % i: INPUT_FEATURES[i] for i in range(NUM_FEATURES)}
+
+
+def _make_train_input_fn(is_classification):
+  """Makes train input_fn for classification/regression."""
+
+  def _input_fn():
+    features = dict(FEATURES_DICT)
+    if is_classification:
+      labels = CLASSIFICATION_LABELS
+    else:
+      labels = REGRESSION_LABELS
+    return features, labels
+
+  return _input_fn
+
+
+class BoostedTreesEstimatorTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._head = canned_boosted_trees._create_regression_head(label_dimension=1)
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES)
+        for i in range(NUM_FEATURES)
+    }
+
+  def _assert_checkpoint(self, model_dir, expected_global_step):
+    self.assertEqual(expected_global_step,
+                     checkpoint_utils.load_variable(model_dir,
+                                                    ops.GraphKeys.GLOBAL_STEP))
+
+  def testTrainAndEvaluateEstimator(self):
+    input_fn = _make_train_input_fn(is_classification=False)
+
+    est = boosted_trees._BoostedTreesEstimator(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=2,
+        head=self._head,
+        max_depth=5)
+
+    # It will stop after 10 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(input_fn, steps=num_steps)
+    self._assert_checkpoint(est.model_dir, 11)
+    eval_res = est.evaluate(input_fn=input_fn, steps=1)
+    self.assertAllClose(eval_res['average_loss'], 0.913176)
+
+  def testInferEstimator(self):
+    train_input_fn = _make_train_input_fn(is_classification=False)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False)
+
+    est = boosted_trees._BoostedTreesEstimator(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=1,
+        max_depth=5,
+        head=self._head)
+
+    # It will stop after 5 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(train_input_fn, steps=num_steps)
+    self._assert_checkpoint(est.model_dir, 6)
+
+    predictions = list(est.predict(input_fn=predict_input_fn))
+    self.assertEquals(5, len(predictions))
+    self.assertAllClose([0.703549], predictions[0]['predictions'])
+    self.assertAllClose([0.266539], predictions[1]['predictions'])
+    self.assertAllClose([0.256479], predictions[2]['predictions'])
+    self.assertAllClose([1.088732], predictions[3]['predictions'])
+    self.assertAllClose([1.901732], predictions[4]['predictions'])
+
+
+class BoostedTreesClassifierTrainInMemoryTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES)
+        for i in range(NUM_FEATURES)
+    }
+
+  def _assert_checkpoint(self, model_dir, expected_global_step):
+    self.assertEqual(expected_global_step,
+                     checkpoint_utils.load_variable(model_dir,
+                                                    ops.GraphKeys.GLOBAL_STEP))
+
+  def testBinaryClassifierTrainInMemoryAndEvalAndInfer(self):
+    train_input_fn = _make_train_input_fn(is_classification=True)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False)
+
+    est = boosted_trees.boosted_trees_classifier_train_in_memory(
+        train_input_fn=train_input_fn,
+        feature_columns=self._feature_columns,
+        n_trees=1,
+        max_depth=5)
+    # It will stop after 5 steps because of the max depth and num trees.
+    self._assert_checkpoint(est.model_dir, 6)
+
+    # Check eval.
+    eval_res = est.evaluate(input_fn=train_input_fn, steps=1)
+    self.assertAllClose(eval_res['accuracy'], 1.0)
+
+    # Check predict that all labels are correct.
+    predictions = list(est.predict(input_fn=predict_input_fn))
+    self.assertEquals(5, len(predictions))
+    self.assertAllClose([0], predictions[0]['class_ids'])
+    self.assertAllClose([1], predictions[1]['class_ids'])
+    self.assertAllClose([1], predictions[2]['class_ids'])
+    self.assertAllClose([0], predictions[3]['class_ids'])
+    self.assertAllClose([0], predictions[4]['class_ids'])
+
+
+class BoostedTreesRegressorTrainInMemoryTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES)
+        for i in range(NUM_FEATURES)
+    }
+
+  def _assert_checkpoint(self, model_dir, expected_global_step):
+    self.assertEqual(expected_global_step,
+                     checkpoint_utils.load_variable(model_dir,
+                                                    ops.GraphKeys.GLOBAL_STEP))
+
+  def testRegressorTrainInMemoryAndEvalAndInfer(self):
+    train_input_fn = _make_train_input_fn(is_classification=False)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False)
+
+    est = boosted_trees.boosted_trees_regressor_train_in_memory(
+        train_input_fn=train_input_fn,
+        feature_columns=self._feature_columns,
+        n_trees=1,
+        max_depth=5)
+    # It will stop after 5 steps because of the max depth and num trees.
+    self._assert_checkpoint(est.model_dir, 6)
+
+    # Check eval.
+    eval_res = est.evaluate(input_fn=train_input_fn, steps=1)
+    self.assertAllClose(eval_res['average_loss'], 2.2136638)
+
+    # Validate predictions.
+    predictions = list(est.predict(input_fn=predict_input_fn))
+    self.assertEquals(5, len(predictions))
+    self.assertAllClose([0.703549], predictions[0]['predictions'])
+    self.assertAllClose([0.266539], predictions[1]['predictions'])
+    self.assertAllClose([0.256479], predictions[2]['predictions'])
+    self.assertAllClose([1.088732], predictions[3]['predictions'])
+    self.assertAllClose([1.901732], predictions[4]['predictions'])
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 5a812af4e9..15786291ed 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -228,6 +228,11 @@ tensorflow/core/kernels/cast_op_impl_int64.cc
 tensorflow/core/kernels/cast_op_impl_int8.cc
 tensorflow/core/kernels/cast_op_impl_uint16.cc
 tensorflow/core/kernels/cast_op_impl_uint8.cc
+tensorflow/core/kernels/boosted_trees/prediction_ops.cc
+tensorflow/core/kernels/boosted_trees/resource_ops.cc
+tensorflow/core/kernels/boosted_trees/resources.cc
+tensorflow/core/kernels/boosted_trees/stats_ops.cc
+tensorflow/core/kernels/boosted_trees/training_ops.cc
 tensorflow/core/kernels/bias_op.cc
 tensorflow/core/kernels/bcast_ops.cc
 tensorflow/core/kernels/batch_norm_op.cc
@@ -285,6 +290,7 @@ tensorflow/core/ops/data_flow_ops.cc
 tensorflow/core/ops/ctc_ops.cc
 tensorflow/core/ops/control_flow_ops.cc
 tensorflow/core/ops/candidate_sampling_ops.cc
+tensorflow/core/ops/boosted_trees_ops.cc
 tensorflow/core/ops/array_ops.cc
 tensorflow/core/ops/array_grad.cc
 tensorflow/core/kernels/spacetobatch_functor.cc
diff --git a/tensorflow/contrib/makefile/tf_proto_files.txt b/tensorflow/contrib/makefile/tf_proto_files.txt
index d569bde637..1f254692d7 100644
--- a/tensorflow/contrib/makefile/tf_proto_files.txt
+++ b/tensorflow/contrib/makefile/tf_proto_files.txt
@@ -18,6 +18,7 @@ tensorflow/core/protobuf/device_properties.proto
 tensorflow/core/protobuf/rewriter_config.proto
 tensorflow/core/protobuf/tensor_bundle.proto
 tensorflow/core/lib/core/error_codes.proto
+tensorflow/core/kernels/boosted_trees/boosted_trees.proto
 tensorflow/core/framework/versions.proto
 tensorflow/core/framework/variable.proto
 tensorflow/core/framework/types.proto
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index b8dbd90ab8..614e06cf83 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -629,6 +629,7 @@ tf_gen_op_libs(
     op_lib_names = [
         "batch_ops",
         "bitwise_ops",
+        "boosted_trees_ops",
         "candidate_sampling_ops",
         "checkpoint_ops",
         "control_flow_ops",
@@ -741,6 +742,7 @@ cc_library(
         ":audio_ops_op_lib",
         ":batch_ops_op_lib",
         ":bitwise_ops_op_lib",
+        ":boosted_trees_ops_op_lib",
         ":candidate_sampling_ops_op_lib",
         ":checkpoint_ops_op_lib",
         ":control_flow_ops_op_lib",
@@ -882,6 +884,7 @@ cc_library(
         "//tensorflow/core/kernels:audio",
         "//tensorflow/core/kernels:batch_kernels",
         "//tensorflow/core/kernels:bincount_op",
+        "//tensorflow/core/kernels:boosted_trees_ops",
         "//tensorflow/core/kernels:candidate_sampler_ops",
         "//tensorflow/core/kernels:checkpoint_ops",
         "//tensorflow/core/kernels:control_flow_ops",
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt
new file mode 100644
index 0000000000..b1921e3507
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCalculateBestGainsPerFeature.pbtxt
@@ -0,0 +1,87 @@
+op {
+  graph_op_name: "BoostedTreesCalculateBestGainsPerFeature"
+  visibility: HIDDEN
+  in_arg {
+    name: "node_id_range"
+    description: <<END
+A Rank 1 tensor (shape=[2]) to specify the range [first, last] of node ids to process within `stats_summary_list`. The nodes are iterated between the two nodes specified by the tensor, as like `for node_id in range(node_id_range[0], node_id_range[1]+1)` (Note that the last index node_id_range[1] is inclusive).
+END
+  }
+  in_arg {
+    name: "stats_summary_list"
+    description: <<END
+A list of Rank 3 tensor (#shape=[max_splits, bucket, 2]) for accumulated stats summary (gradient/hessian) per node per buckets for each feature. The first dimension of the tensor is the maximum number of splits, and thus not all elements of it will be used, but only the indexes specified by node_ids will be used.
+END
+  }
+  out_arg {
+    name: "node_ids_list"
+    description: <<END
+An output list of Rank 1 tensors indicating possible split node ids for each feature. The length of the list is num_features, but each tensor has different size as each feature provides different possible nodes. See above for details like shapes and sizes.
+END
+  }
+  out_arg {
+    name: "gains_list"
+    description: <<END
+An output list of Rank 1 tensors indicating the best gains for each feature to split for certain nodes. See above for details like shapes and sizes.
+END
+  }
+  out_arg {
+    name: "thresholds_list"
+    description: <<END
+An output list of Rank 1 tensors indicating the bucket id to compare with (as a threshold) for split in each node. See above for details like shapes and sizes.
+END
+  }
+  out_arg {
+    name: "left_node_contribs_list"
+    description: <<END
+A list of Rank 2 tensors indicating the contribution of the left nodes when branching from parent nodes (given by the tensor element in the output node_ids_list) to the left direction by the given threshold for each feature. This value will be used to make the left node value by adding to the parent node value. Second dimension size is 1 for 1-dimensional logits, but would be larger for multi-class problems. See above for details like shapes and sizes.
+END
+  }
+  out_arg {
+    name: "right_node_contribs_list"
+    description: <<END
+A list of Rank 2 tensors, with the same shape/conditions as left_node_contribs_list, but just that the value is for the right node.
+END
+  }
+  attr {
+    name: "l1"
+    description: <<END
+l1 regularization factor on leaf weights, per instance based.
+END
+  }
+  attr {
+    name: "l2"
+    description: <<END
+l2 regularization factor on leaf weights, per instance based.
+END
+  }
+  attr {
+    name: "tree_complexity"
+    description: <<END
+adjustment to the gain, per leaf based.
+END
+  }
+  attr {
+    name: "max_splits"
+    description: <<END
+the number of nodes that can be split in the whole tree. Used as a dimension of output tensors.
+END
+  }
+  attr {
+    name: "num_features"
+    description: <<END
+inferred from the size of `stats_summary_list`; the number of total features.
+END
+  }
+  summary: "Calculates gains for each feature and returns the best possible split information for the feature."
+  description: <<END
+The split information is the best threshold (bucket id), gains and left/right node contributions per node for each feature.
+
+It is possible that not all nodes can be split on each feature. Hence, the list of possible nodes can differ between the features. Therefore, we return `node_ids_list` for each feature, containing the list of nodes that this feature can be used to split.
+
+In this manner, the output is the best split per features and per node, so that it needs to be combined later to produce the best split for each node (among all possible features).
+
+The length of output lists are all of the same length, `num_features`.
+The output shapes are compatible in a way that the first dimension of all tensors of all lists are the same and equal to the number of possible split nodes for each feature.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesCreateEnsemble.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCreateEnsemble.pbtxt
new file mode 100644
index 0000000000..aee73b910f
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesCreateEnsemble.pbtxt
@@ -0,0 +1,23 @@
+op {
+  graph_op_name: "BoostedTreesCreateEnsemble"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the tree ensemble resource to be created.
+END
+  }
+  in_arg {
+    name: "stamp_token"
+    description: <<END
+Token to use as the initial value of the resource stamp.
+END
+  }
+  in_arg {
+    name: "tree_ensemble_serialized"
+    description: <<END
+Serialized proto of the tree ensemble.
+END
+  }
+  summary: "Creates a tree ensemble model and returns a handle to it."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesDeserializeEnsemble.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesDeserializeEnsemble.pbtxt
new file mode 100644
index 0000000000..b1602ba045
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesDeserializeEnsemble.pbtxt
@@ -0,0 +1,26 @@
+op {
+  graph_op_name: "BoostedTreesDeserializeEnsemble"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the tree ensemble.
+END
+  }
+  in_arg {
+    name: "stamp_token"
+    description: <<END
+Token to use as the new value of the resource stamp.
+END
+  }
+  in_arg {
+    name: "tree_ensemble_serialized"
+    description: <<END
+Serialized proto of the ensemble.
+END
+  }
+  summary: "Deserializes a serialized tree ensemble config and replaces current tree"
+  description: <<END
+ensemble.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesEnsembleResourceHandleOp.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesEnsembleResourceHandleOp.pbtxt
new file mode 100644
index 0000000000..1bce5639a2
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesEnsembleResourceHandleOp.pbtxt
@@ -0,0 +1,5 @@
+op {
+  graph_op_name: "BoostedTreesEnsembleResourceHandleOp"
+  visibility: HIDDEN
+  summary: "Creates a handle to a BoostedTreesEnsembleResource"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesGetEnsembleStates.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesGetEnsembleStates.pbtxt
new file mode 100644
index 0000000000..ef45a92498
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesGetEnsembleStates.pbtxt
@@ -0,0 +1,35 @@
+op {
+  graph_op_name: "BoostedTreesGetEnsembleStates"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the tree ensemble.
+END
+  }
+  out_arg {
+    name: "stamp_token"
+    description: <<END
+Stamp token of the tree ensemble resource.
+END
+  }
+  out_arg {
+    name: "num_trees"
+    description: <<END
+The number of trees in the tree ensemble resource.
+END
+  }
+  out_arg {
+    name: "num_finalized_trees"
+    description: <<END
+The number of trees that were finished successfully.
+END
+  }
+  out_arg {
+    name: "num_attempted_layers"
+    description: <<END
+The number of layers we attempted to build (but not necessarily succeeded).
+END
+  }
+  summary: "Retrieves the tree ensemble resource stamp token."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesMakeStatsSummary.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesMakeStatsSummary.pbtxt
new file mode 100644
index 0000000000..dc0856c900
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesMakeStatsSummary.pbtxt
@@ -0,0 +1,56 @@
+op {
+  graph_op_name: "BoostedTreesMakeStatsSummary"
+  visibility: HIDDEN
+  in_arg {
+    name: "node_ids"
+    description: <<END
+int32 Rank 1 Tensor containing node ids, which each example falls into for the requested layer.
+END
+  }
+  in_arg {
+    name: "gradients"
+    description: <<END
+float32; Rank 2 Tensor (shape=[#examples, 1]) for gradients.
+END
+  }
+  in_arg {
+    name: "hessians"
+    description: <<END
+float32; Rank 2 Tensor (shape=[#examples, 1]) for hessians.
+END
+  }
+  in_arg {
+    name: "bucketized_features_list"
+    description: <<END
+int32 list of Rank 1 Tensors, each containing the bucketized feature (for each feature column).
+END
+  }
+  out_arg {
+    name: "stats_summary"
+    description: <<END
+output Rank 4 Tensor (shape=[#features, #splits, #buckets, 2]) containing accumulated stats put into the corresponding node and bucket. The first index of 4th dimension refers to gradients, and the second to hessians.
+END
+  }
+  attr {
+    name: "max_splits"
+    description: <<END
+int; the maximum number of splits possible in the whole tree.
+END
+  }
+  attr {
+    name: "num_buckets"
+    description: <<END
+int; equals to the maximum possible value of bucketized feature.
+END
+  }
+  attr {
+    name: "num_features"
+    description: <<END
+int; inferred from the size of bucketized_features_list; the number of features.
+END
+  }
+  summary: "Makes the summary of accumulated stats for the batch."
+  description: <<END
+The summary stats contains gradients and hessians accumulated into the corresponding node and bucket for each example.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesPredict.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesPredict.pbtxt
new file mode 100644
index 0000000000..b23e77a1fa
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesPredict.pbtxt
@@ -0,0 +1,41 @@
+op {
+  graph_op_name: "BoostedTreesPredict"
+  visibility: HIDDEN
+  in_arg {
+    name: "bucketized_features"
+    description: <<END
+A list of rank 1 Tensors containing bucket id for each
+feature.
+END
+  }
+  out_arg {
+    name: "logits"
+    description: <<END
+Output rank 2 Tensor containing logits for each example.
+END
+  }
+  attr {
+    name: "num_bucketized_features"
+    description: <<END
+Inferred.
+END
+  }
+  attr {
+    name: "logits_dimension"
+    description: <<END
+scalar, dimension of the logits, to be used for partial logits
+shape.
+END
+  }
+  attr {
+    name: "max_depth"
+    description: <<END
+scalar, max depth of trees. To be used for parallelization costs.
+END
+  }
+  summary: "Runs multiple additive regression ensemble predictors on input instances and"
+  description: <<END
+computes the logits. It is designed to be used during prediction.
+It traverses all the trees and calculates the final score for each instance.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesSerializeEnsemble.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesSerializeEnsemble.pbtxt
new file mode 100644
index 0000000000..c0b3688d8a
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesSerializeEnsemble.pbtxt
@@ -0,0 +1,23 @@
+op {
+  graph_op_name: "BoostedTreesSerializeEnsemble"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the tree ensemble.
+END
+  }
+  out_arg {
+    name: "stamp_token"
+    description: <<END
+Stamp token of the tree ensemble resource.
+END
+  }
+  out_arg {
+    name: "tree_ensemble_serialized"
+    description: <<END
+Serialized proto of the ensemble.
+END
+  }
+  summary: "Serializes the tree ensemble to a proto."
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesTrainingPredict.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesTrainingPredict.pbtxt
new file mode 100644
index 0000000000..7203d3cb58
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesTrainingPredict.pbtxt
@@ -0,0 +1,69 @@
+op {
+  graph_op_name: "BoostedTreesTrainingPredict"
+  visibility: HIDDEN
+  in_arg {
+    name: "cached_tree_ids"
+    description: <<END
+Rank 1 Tensor containing cached tree ids which is the starting
+tree of prediction.
+END
+  }
+  in_arg {
+    name: "cached_node_ids"
+    description: <<END
+Rank 1 Tensor containing cached node id which is the starting
+node of prediction.
+END
+  }
+  in_arg {
+    name: "bucketized_features"
+    description: <<END
+A list of rank 1 Tensors containing bucket id for each
+feature.
+END
+  }
+  out_arg {
+    name: "partial_logits"
+    description: <<END
+Rank 2 Tensor containing logits update (with respect to cached
+values stored) for each example.
+END
+  }
+  out_arg {
+    name: "tree_ids"
+    description: <<END
+Rank 1 Tensor containing new tree ids for each example.
+END
+  }
+  out_arg {
+    name: "node_ids"
+    description: <<END
+Rank 1 Tensor containing new node ids in the new tree_ids.
+END
+  }
+  attr {
+    name: "num_bucketized_features"
+    description: <<END
+Inferred.
+END
+  }
+  attr {
+    name: "logits_dimension"
+    description: <<END
+scalar, dimension of the logits, to be used for partial logits
+shape.
+END
+  }
+  attr {
+    name: "max_depth"
+    description: <<END
+scalar, max depth of trees. To be used for parallelization costs.
+END
+  }
+  summary: "Runs multiple additive regression ensemble predictors on input instances and"
+  description: <<END
+computes the update to cached logits. It is designed to be used during training.
+It traverses the trees starting from cached tree id and cached node id and
+calculates the updates to be pushed to the cache.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_BoostedTreesUpdateEnsemble.pbtxt b/tensorflow/core/api_def/base_api/api_def_BoostedTreesUpdateEnsemble.pbtxt
new file mode 100644
index 0000000000..00f8953875
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_BoostedTreesUpdateEnsemble.pbtxt
@@ -0,0 +1,82 @@
+op {
+  graph_op_name: "BoostedTreesUpdateEnsemble"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the ensemble variable.
+END
+  }
+  in_arg {
+    name: "feature_ids"
+    description: <<END
+Rank 1 tensor with ids for each feature. This is the real id of
+the feature that will be used in the split.
+END
+  }
+  in_arg {
+    name: "node_ids"
+    description: <<END
+List of rank 1 tensors representing the nodes for which this feature
+has a split.
+END
+  }
+  in_arg {
+    name: "gains"
+    description: <<END
+List of rank 1 tensors representing the gains for each of the feature's
+split.
+END
+  }
+  in_arg {
+    name: "thresholds"
+    description: <<END
+List of rank 1 tensors representing the thesholds for each of the
+feature's split.
+END
+  }
+  in_arg {
+    name: "left_node_contribs"
+    description: <<END
+List of rank 2 tensors with left leaf contribs for each of
+the feature's splits. Will be added to the previous node values to constitute
+the values of the left nodes.
+END
+  }
+  in_arg {
+    name: "right_node_contribs"
+    description: <<END
+List of rank 2 tensors with right leaf contribs for each
+of the feature's splits. Will be added to the previous node values to constitute
+the values of the right nodes.
+END
+  }
+  attr {
+    name: "max_depth"
+    description: <<END
+Max depth of the tree to build.
+END
+  }
+  attr {
+    name: "learning_rate"
+    description: <<END
+shrinkage const for each new tree.
+END
+  }
+  attr {
+    name: "pruning_mode"
+    description: <<END
+0-No pruning, 1-Pre-pruning, 2-Post-pruning.
+END
+  }
+  attr {
+    name: "num_features"
+    description: <<END
+Number of features that have best splits returned. INFERRED.
+END
+  }
+  summary: "Updates the tree ensemble by either adding a layer to the last tree being grown"
+  description: <<END
+or by starting a new tree.
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_IsBoostedTreesEnsembleInitialized.pbtxt b/tensorflow/core/api_def/base_api/api_def_IsBoostedTreesEnsembleInitialized.pbtxt
new file mode 100644
index 0000000000..d54b7ef32a
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_IsBoostedTreesEnsembleInitialized.pbtxt
@@ -0,0 +1,17 @@
+op {
+  graph_op_name: "IsBoostedTreesEnsembleInitialized"
+  visibility: HIDDEN
+  in_arg {
+    name: "tree_ensemble_handle"
+    description: <<END
+Handle to the tree ensemble resouce.
+END
+  }
+  out_arg {
+    name: "is_initialized"
+    description: <<END
+output boolean on whether it is initialized or not.
+END
+  }
+  summary: "Checks whether a tree ensemble has been initialized."
+}
diff --git a/tensorflow/core/kernels/BUILD b/tensorflow/core/kernels/BUILD
index ca54978421..d2a2cdd13d 100644
--- a/tensorflow/core/kernels/BUILD
+++ b/tensorflow/core/kernels/BUILD
@@ -6096,6 +6096,13 @@ cc_library(
     ],
 )
 
+tf_kernel_library(
+    name = "boosted_trees_ops",
+    deps = [
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_ops",
+    ],
+)
+
 cc_library(
     name = "captured_function",
     hdrs = ["captured_function.h"],
diff --git a/tensorflow/core/kernels/boosted_trees/BUILD b/tensorflow/core/kernels/boosted_trees/BUILD
new file mode 100644
index 0000000000..62327dfe1d
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/BUILD
@@ -0,0 +1,89 @@
+# Description:
+#   OpKernels for boosted trees ops.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_kernel_library")
+load(
+    "//tensorflow/core:platform/default/build_config.bzl",
+    "tf_proto_library",
+)
+
+tf_proto_library(
+    name = "boosted_trees_proto",
+    srcs = ["boosted_trees.proto"],
+    cc_api_version = 2,
+    visibility = ["//visibility:public"],
+)
+
+tf_kernel_library(
+    name = "prediction_ops",
+    srcs = ["prediction_ops.cc"],
+    deps = [
+        ":resource_ops",
+        ":resources",
+        "//tensorflow/core:boosted_trees_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core:lib_internal",
+    ],
+)
+
+cc_library(
+    name = "resources",
+    srcs = ["resources.cc"],
+    hdrs = ["resources.h"],
+    deps = [
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "resource_ops",
+    srcs = ["resource_ops.cc"],
+    deps = [
+        ":resources",
+        "//tensorflow/core:boosted_trees_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "stats_ops",
+    srcs = ["stats_ops.cc"],
+    deps = [
+        "//tensorflow/core:boosted_trees_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+    ],
+)
+
+tf_kernel_library(
+    name = "training_ops",
+    srcs = ["training_ops.cc"],
+    deps = [
+        ":resources",
+        "//tensorflow/core:boosted_trees_ops_op_lib",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_cc",
+    ],
+)
+
+tf_kernel_library(
+    name = "boosted_trees_ops",
+    deps = [
+        ":prediction_ops",
+        ":resource_ops",
+        ":stats_ops",
+        ":training_ops",
+    ],
+)
diff --git a/tensorflow/core/kernels/boosted_trees/boosted_trees.proto b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
new file mode 100644
index 0000000000..106ceedc00
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/boosted_trees.proto
@@ -0,0 +1,113 @@
+syntax = "proto3";
+
+package tensorflow.boosted_trees;
+option cc_enable_arenas = true;
+option java_outer_classname = "BoostedTreesProtos";
+option java_multiple_files = true;
+option java_package = "org.tensorflow.framework";
+
+// Node describes a node in a tree.
+message Node {
+  oneof node {
+    Leaf leaf = 1;
+    BucketizedSplit bucketized_split = 2;
+  }
+  NodeMetadata metadata = 777;
+}
+
+// NodeMetadata encodes metadata associated with each node in a tree.
+message NodeMetadata {
+  // The gain associated with this node.
+  float gain = 1;
+
+  // The original leaf node before this node was split.
+  Leaf original_leaf = 2;
+}
+
+// Leaves can either hold dense or sparse information.
+message Leaf {
+  oneof leaf {
+    // See third_party/tensorflow/contrib/decision_trees/
+    // proto/generic_tree_model.proto
+    // for a description of how vector and sparse_vector might be used.
+    Vector vector = 1;
+    SparseVector sparse_vector = 2;
+  }
+  float scalar = 3;
+}
+
+message Vector {
+  repeated float value = 1;
+}
+
+message SparseVector {
+  repeated int32 index = 1;
+  repeated float value = 2;
+}
+
+message BucketizedSplit {
+  // Float feature column and split threshold describing
+  // the rule feature <= threshold.
+  int32 feature_id = 1;
+  int32 threshold = 2;
+
+  // Node children indexing into a contiguous
+  // vector of nodes starting from the root.
+  int32 left_id = 3;
+  int32 right_id = 4;
+}
+
+// Tree describes a list of connected nodes.
+// Node 0 must be the root and can carry any payload including a leaf
+// in the case of representing the bias.
+// Note that each node id is implicitly its index in the list of nodes.
+message Tree {
+  repeated Node nodes = 1;
+}
+
+message TreeMetadata {
+  // Number of layers grown for this tree.
+  int32 num_layers_grown = 2;
+
+  // Whether the tree is finalized in that no more layers can be grown.
+  bool is_finalized = 3;
+
+  // If tree was finalized and post pruning happened, it is possible that cache
+  // still refers to some nodes that were deleted or that the node ids changed
+  // (e.g. node id 5 became node id 2 due to pruning of the other branch).
+  // The mapping below allows us to understand where the old ids now map to and
+  // how the values should be adjusted due to post-pruning.
+  // The size of the list should be equal to the number of nodes in the tree
+  // before post-pruning happened.
+  // If the node was pruned, it will have new_node_id equal to the id of a node
+  // that this node was collapsed into. For a node that didn't get pruned, it is
+  // possible that its id still changed, so new_node_id will have the
+  // corresponding id in the pruned tree.
+  // If post-pruning didn't happen, or it did and it had no effect (e.g. no
+  // nodes got pruned), this list will be empty.
+  repeated PostPruneNodeUpdate post_pruned_nodes_meta = 4;
+
+  message PostPruneNodeUpdate {
+    int32 new_node_id = 1;
+    float logit_change = 2;
+  }
+}
+
+message GrowingMetadata {
+  // Number of trees that we have attempted to build. After pruning, these
+  // trees might have been removed.
+  int64 num_trees_attempted = 1;
+  // Number of layers that we have attempted to build. After pruning, these
+  // layers might have been removed.
+  int64 num_layers_attempted = 2;
+}
+
+// TreeEnsemble describes an ensemble of decision trees.
+message TreeEnsemble {
+  repeated Tree trees = 1;
+  repeated float tree_weights = 2;
+
+  repeated TreeMetadata tree_metadata = 3;
+  // Metadata that is used during the training.
+  GrowingMetadata growing_metadata = 4;
+}
diff --git a/tensorflow/core/kernels/boosted_trees/prediction_ops.cc b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc
new file mode 100644
index 0000000000..b13a450546
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/prediction_ops.cc
@@ -0,0 +1,263 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <algorithm>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/framework/device_base.h"
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/kernels/boosted_trees/resources.h"
+#include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/core/refcount.h"
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/protobuf.h"
+#include "tensorflow/core/platform/types.h"
+#include "tensorflow/core/util/work_sharder.h"
+
+namespace tensorflow {
+
+// The Op used during training time to get the predictions so far with the
+// current ensemble being built.
+// Expect some logits are cached from the previous step and passed through
+// to be reused.
+class BoostedTreesTrainingPredictOp : public OpKernel {
+ public:
+  explicit BoostedTreesTrainingPredictOp(OpKernelConstruction* const context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("num_bucketized_features",
+                                             &num_bucketized_features_));
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("logits_dimension", &logits_dimension_));
+    OP_REQUIRES(context, logits_dimension_ == 1,
+                errors::InvalidArgument(
+                    "Currently only one dimensional outputs are supported."));
+    OP_REQUIRES_OK(context, context->GetAttr("max_depth", &max_depth_));
+  }
+
+  void Compute(OpKernelContext* const context) override {
+    BoostedTreesEnsembleResource* resource;
+    // Get the resource.
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &resource));
+    // Release the reference to the resource once we're done using it.
+    core::ScopedUnref unref_me(resource);
+
+    // Get the inputs.
+    OpInputList bucketized_features_list;
+    OP_REQUIRES_OK(context, context->input_list("bucketized_features",
+                                                &bucketized_features_list));
+    std::vector<tensorflow::TTypes<int32>::ConstVec> batch_bucketized_features;
+    batch_bucketized_features.reserve(bucketized_features_list.size());
+    for (const Tensor& tensor : bucketized_features_list) {
+      batch_bucketized_features.emplace_back(tensor.vec<int32>());
+    }
+    const int batch_size = batch_bucketized_features[0].size();
+
+    const Tensor* cached_tree_ids_t;
+    OP_REQUIRES_OK(context,
+                   context->input("cached_tree_ids", &cached_tree_ids_t));
+    const auto cached_tree_ids = cached_tree_ids_t->vec<int32>();
+
+    const Tensor* cached_node_ids_t;
+    OP_REQUIRES_OK(context,
+                   context->input("cached_node_ids", &cached_node_ids_t));
+    const auto cached_node_ids = cached_node_ids_t->vec<int32>();
+
+    // Allocate outputs.
+    Tensor* output_partial_logits_t = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output("partial_logits",
+                                            {batch_size, logits_dimension_},
+                                            &output_partial_logits_t));
+    auto output_partial_logits = output_partial_logits_t->matrix<float>();
+
+    Tensor* output_tree_ids_t = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output("tree_ids", {batch_size},
+                                                     &output_tree_ids_t));
+    auto output_tree_ids = output_tree_ids_t->vec<int32>();
+
+    Tensor* output_node_ids_t = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output("node_ids", {batch_size},
+                                                     &output_node_ids_t));
+    auto output_node_ids = output_node_ids_t->vec<int32>();
+
+    // Indicate that the latest tree was used.
+    const int32 latest_tree = resource->num_trees() - 1;
+
+    if (latest_tree < 0) {
+      // Ensemble was empty. Nothing changes.
+      output_node_ids = cached_node_ids;
+      output_tree_ids = cached_tree_ids;
+      // All the predictions are zeros.
+      output_partial_logits.setZero();
+    } else {
+      output_tree_ids.setConstant(latest_tree);
+      auto do_work = [&resource, &batch_bucketized_features, &cached_tree_ids,
+                      &cached_node_ids, &output_partial_logits,
+                      &output_node_ids, batch_size,
+                      latest_tree](int32 start, int32 end) {
+        for (int32 i = start; i < end; ++i) {
+          int32 tree_id = cached_tree_ids(i);
+          int32 node_id = cached_node_ids(i);
+          float partial_tree_logit = 0.0;
+
+          // If the tree was pruned, returns the node id into which the
+          // current_node_id was pruned, as well the correction of the cached
+          // logit prediction.
+          resource->GetPostPruneCorrection(tree_id, node_id, &node_id,
+                                           &partial_tree_logit);
+
+          // Logic in the loop adds the cached node value again if it is a leaf.
+          // If it is not a leaf anymore we need to subtract the old node's
+          // value. The following logic handles both of these cases.
+          partial_tree_logit -= resource->node_value(tree_id, node_id);
+          float partial_all_logit = 0.0;
+          while (true) {
+            if (resource->is_leaf(tree_id, node_id)) {
+              partial_tree_logit += resource->node_value(tree_id, node_id);
+
+              // Tree is done
+              partial_all_logit +=
+                  resource->GetTreeWeight(tree_id) * partial_tree_logit;
+              partial_tree_logit = 0.0;
+              // Stop if it was the latest tree.
+              if (tree_id == latest_tree) {
+                break;
+              }
+              // Move onto other trees.
+              ++tree_id;
+              node_id = 0;
+            } else {
+              node_id = resource->next_node(tree_id, node_id, i,
+                                            batch_bucketized_features);
+            }
+          }
+          output_node_ids(i) = node_id;
+          output_partial_logits(i, 0) = partial_all_logit;
+        }
+      };
+      // Assume we will not go over more than one full tree. 4 is a magic
+      // number.
+      const int64 cost = 4 * max_depth_;
+      thread::ThreadPool* const worker_threads =
+          context->device()->tensorflow_cpu_worker_threads()->workers;
+      Shard(worker_threads->NumThreads(), worker_threads, batch_size,
+            /*cost_per_unit=*/cost, do_work);
+    }
+  }
+
+ private:
+  int32 logits_dimension_;         // the size of the output prediction vector.
+  int32 num_bucketized_features_;  // Indicates the number of features.
+  int32 max_depth_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("BoostedTreesTrainingPredict").Device(DEVICE_CPU),
+                        BoostedTreesTrainingPredictOp);
+
+// The Op to get the predictions at the evaluation/inference time.
+class BoostedTreesPredictOp : public OpKernel {
+ public:
+  explicit BoostedTreesPredictOp(OpKernelConstruction* const context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("num_bucketized_features",
+                                             &num_bucketized_features_));
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("logits_dimension", &logits_dimension_));
+    OP_REQUIRES(context, logits_dimension_ == 1,
+                errors::InvalidArgument(
+                    "Currently only one dimensional outputs are supported."));
+    OP_REQUIRES_OK(context, context->GetAttr("max_depth", &max_depth_));
+  }
+
+  void Compute(OpKernelContext* const context) override {
+    BoostedTreesEnsembleResource* resource;
+    // Get the resource.
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &resource));
+    // Release the reference to the resource once we're done using it.
+    core::ScopedUnref unref_me(resource);
+
+    // Get the inputs.
+    OpInputList bucketized_features_list;
+    OP_REQUIRES_OK(context, context->input_list("bucketized_features",
+                                                &bucketized_features_list));
+    std::vector<tensorflow::TTypes<int32>::ConstVec> batch_bucketized_features;
+    batch_bucketized_features.reserve(bucketized_features_list.size());
+    for (const Tensor& tensor : bucketized_features_list) {
+      batch_bucketized_features.emplace_back(tensor.vec<int32>());
+    }
+    const int batch_size = batch_bucketized_features[0].size();
+
+    // Allocate outputs.
+    Tensor* output_logits_t = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(
+                                "logits", {batch_size, logits_dimension_},
+                                &output_logits_t));
+    auto output_logits = output_logits_t->matrix<float>();
+
+    const int32 latest_tree = resource->num_trees() - 1;
+
+    auto do_work = [&resource, &batch_bucketized_features, &output_logits,
+                    batch_size, latest_tree](int32 start, int32 end) {
+      for (int32 i = start; i < end; ++i) {
+        float tree_logit = 0.0;
+        int32 tree_id = 0;
+        int32 node_id = 0;
+        while (true) {
+          if (resource->is_leaf(tree_id, node_id)) {
+            tree_logit += resource->GetTreeWeight(tree_id) *
+                          resource->node_value(tree_id, node_id);
+
+            // Stop if it was the latest tree.
+            if (tree_id == latest_tree) {
+              break;
+            }
+            // Move onto other trees.
+            ++tree_id;
+            node_id = 0;
+          } else {
+            node_id = resource->next_node(tree_id, node_id, i,
+                                          batch_bucketized_features);
+          }
+        }
+        output_logits(i, 0) = tree_logit;
+      }
+    };
+    const int64 cost = (latest_tree + 1) * max_depth_;
+    thread::ThreadPool* const worker_threads =
+        context->device()->tensorflow_cpu_worker_threads()->workers;
+    Shard(worker_threads->NumThreads(), worker_threads, batch_size,
+          /*cost_per_unit=*/cost, do_work);
+  }
+
+ private:
+  int32
+      logits_dimension_;  // Indicates the size of the output prediction vector.
+  int32 num_bucketized_features_;  // Indicates the number of features.
+  int32 max_depth_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("BoostedTreesPredict").Device(DEVICE_CPU),
+                        BoostedTreesPredictOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/boosted_trees/resource_ops.cc b/tensorflow/core/kernels/boosted_trees/resource_ops.cc
new file mode 100644
index 0000000000..f49242d856
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/resource_ops.cc
@@ -0,0 +1,189 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <memory>
+#include <string>
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/tensor.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/framework/tensor_types.h"
+#include "tensorflow/core/kernels/boosted_trees/resources.h"
+
+namespace tensorflow {
+
+REGISTER_RESOURCE_HANDLE_KERNEL(BoostedTreesEnsembleResource);
+
+REGISTER_KERNEL_BUILDER(
+    Name("IsBoostedTreesEnsembleInitialized").Device(DEVICE_CPU),
+    IsResourceInitialized<BoostedTreesEnsembleResource>);
+
+// Creates a tree ensemble resource.
+class BoostedTreesCreateEnsembleOp : public OpKernel {
+ public:
+  explicit BoostedTreesCreateEnsembleOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    // Get the stamp token.
+    const Tensor* stamp_token_t;
+    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
+    int64 stamp_token = stamp_token_t->scalar<int64>()();
+
+    // Get the tree ensemble proto.
+    const Tensor* tree_ensemble_serialized_t;
+    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
+                                           &tree_ensemble_serialized_t));
+    std::unique_ptr<BoostedTreesEnsembleResource> result(
+        new BoostedTreesEnsembleResource());
+    if (!result->InitFromSerialized(
+            tree_ensemble_serialized_t->scalar<string>()(), stamp_token)) {
+      result->Unref();
+      OP_REQUIRES(
+          context, false,
+          errors::InvalidArgument("Unable to parse tree ensemble proto."));
+    }
+
+    // Only create one, if one does not exist already. Report status for all
+    // other exceptions.
+    auto status =
+        CreateResource(context, HandleFromInput(context, 0), result.release());
+    if (status.code() != tensorflow::error::ALREADY_EXISTS) {
+      OP_REQUIRES_OK(context, status);
+    }
+  }
+};
+
+REGISTER_KERNEL_BUILDER(Name("BoostedTreesCreateEnsemble").Device(DEVICE_CPU),
+                        BoostedTreesCreateEnsembleOp);
+
+// Op for retrieving some model states (needed for training).
+class BoostedTreesGetEnsembleStatesOp : public OpKernel {
+ public:
+  explicit BoostedTreesGetEnsembleStatesOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    // Looks up the resource.
+    BoostedTreesEnsembleResource* tree_ensemble_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &tree_ensemble_resource));
+    tf_shared_lock l(*tree_ensemble_resource->get_mutex());
+    core::ScopedUnref unref_me(tree_ensemble_resource);
+
+    // Sets the outputs.
+    const int num_trees = tree_ensemble_resource->num_trees();
+    const int num_finalized_trees =
+        (num_trees <= 0 ||
+         tree_ensemble_resource->IsTreeFinalized(num_trees - 1))
+            ? num_trees
+            : num_trees - 1;
+    const int num_attempted_layers =
+        tree_ensemble_resource->GetNumLayersAttempted();
+
+    // growing_metadata
+    Tensor* output_stamp_token_t = nullptr;
+    Tensor* output_num_trees_t = nullptr;
+    Tensor* output_num_finalized_trees_t = nullptr;
+    Tensor* output_num_attempted_layers_t = nullptr;
+
+    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
+                                                     &output_stamp_token_t));
+    OP_REQUIRES_OK(context, context->allocate_output(1, TensorShape(),
+                                                     &output_num_trees_t));
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(2, TensorShape(),
+                                            &output_num_finalized_trees_t));
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(3, TensorShape(),
+                                            &output_num_attempted_layers_t));
+
+    output_stamp_token_t->scalar<int64>()() = tree_ensemble_resource->stamp();
+    output_num_trees_t->scalar<int32>()() = num_trees;
+    output_num_finalized_trees_t->scalar<int32>()() = num_finalized_trees;
+    output_num_attempted_layers_t->scalar<int32>()() = num_attempted_layers;
+  }
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("BoostedTreesGetEnsembleStates").Device(DEVICE_CPU),
+    BoostedTreesGetEnsembleStatesOp);
+
+// Op for serializing a model.
+class BoostedTreesSerializeEnsembleOp : public OpKernel {
+ public:
+  explicit BoostedTreesSerializeEnsembleOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    BoostedTreesEnsembleResource* tree_ensemble_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &tree_ensemble_resource));
+    tf_shared_lock l(*tree_ensemble_resource->get_mutex());
+    core::ScopedUnref unref_me(tree_ensemble_resource);
+    Tensor* output_stamp_token_t = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(0, TensorShape(),
+                                                     &output_stamp_token_t));
+    output_stamp_token_t->scalar<int64>()() = tree_ensemble_resource->stamp();
+    Tensor* output_proto_t = nullptr;
+    OP_REQUIRES_OK(context,
+                   context->allocate_output(1, TensorShape(), &output_proto_t));
+    output_proto_t->scalar<string>()() =
+        tree_ensemble_resource->SerializeAsString();
+  }
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("BoostedTreesSerializeEnsemble").Device(DEVICE_CPU),
+    BoostedTreesSerializeEnsembleOp);
+
+// Op for deserializing a tree ensemble variable from a checkpoint.
+class BoostedTreesDeserializeEnsembleOp : public OpKernel {
+ public:
+  explicit BoostedTreesDeserializeEnsembleOp(OpKernelConstruction* context)
+      : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    BoostedTreesEnsembleResource* tree_ensemble_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &tree_ensemble_resource));
+    mutex_lock l(*tree_ensemble_resource->get_mutex());
+    core::ScopedUnref unref_me(tree_ensemble_resource);
+
+    // Get the stamp token.
+    const Tensor* stamp_token_t;
+    OP_REQUIRES_OK(context, context->input("stamp_token", &stamp_token_t));
+    int64 stamp_token = stamp_token_t->scalar<int64>()();
+
+    // Get the tree ensemble proto.
+    const Tensor* tree_ensemble_serialized_t;
+    OP_REQUIRES_OK(context, context->input("tree_ensemble_serialized",
+                                           &tree_ensemble_serialized_t));
+    // Deallocate all the previous objects on the resource.
+    tree_ensemble_resource->Reset();
+    OP_REQUIRES(
+        context,
+        tree_ensemble_resource->InitFromSerialized(
+            tree_ensemble_serialized_t->scalar<string>()(), stamp_token),
+        errors::InvalidArgument("Unable to parse tree ensemble proto."));
+  }
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("BoostedTreesDeserializeEnsemble").Device(DEVICE_CPU),
+    BoostedTreesDeserializeEnsembleOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/boosted_trees/resources.cc b/tensorflow/core/kernels/boosted_trees/resources.cc
new file mode 100644
index 0000000000..2ea12c522c
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/resources.cc
@@ -0,0 +1,301 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/kernels/boosted_trees/resources.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+
+int32 BoostedTreesEnsembleResource::next_node(
+    const int32 tree_id, const int32 node_id, const int32 index_in_batch,
+    const std::vector<TTypes<int32>::ConstVec>& bucketized_features) const {
+  DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+  DCHECK_LT(node_id, tree_ensemble_->trees(tree_id).nodes_size());
+  const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id);
+  DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+  const auto& split = node.bucketized_split();
+  if (bucketized_features[split.feature_id()](index_in_batch) <=
+      split.threshold()) {
+    return split.left_id();
+  } else {
+    return split.right_id();
+  }
+}
+
+float BoostedTreesEnsembleResource::node_value(const int32 tree_id,
+                                               const int32 node_id) const {
+  DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+  DCHECK_LT(node_id, tree_ensemble_->trees(tree_id).nodes_size());
+  const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id);
+  if (node.node_case() == boosted_trees::Node::kLeaf) {
+    return node.leaf().scalar();
+  } else {
+    return node.metadata().original_leaf().scalar();
+  }
+}
+
+void BoostedTreesEnsembleResource::UpdateGrowingMetadata() const {
+  tree_ensemble_->mutable_growing_metadata()->set_num_layers_attempted(
+      tree_ensemble_->growing_metadata().num_layers_attempted() + 1);
+
+  const int n_trees = num_trees();
+
+  if (n_trees <= 0 ||
+      // Checks if we are building the first layer of the dummy empty tree
+      ((n_trees == 1 || IsTreeFinalized(n_trees - 2)) &&
+       (tree_ensemble_->trees(n_trees - 1).nodes_size() == 1))) {
+    tree_ensemble_->mutable_growing_metadata()->set_num_trees_attempted(
+        tree_ensemble_->growing_metadata().num_trees_attempted() + 1);
+  }
+}
+
+// Add a tree to the ensemble and returns a new tree_id.
+int32 BoostedTreesEnsembleResource::AddNewTree(const float weight) {
+  const int32 new_tree_id = tree_ensemble_->trees_size();
+  auto* node = tree_ensemble_->add_trees()->add_nodes();
+  node->mutable_leaf()->set_scalar(0.0);
+  tree_ensemble_->add_tree_weights(weight);
+  tree_ensemble_->add_tree_metadata();
+
+  return new_tree_id;
+}
+
+void BoostedTreesEnsembleResource::AddBucketizedSplitNode(
+    const int32 tree_id, const int32 node_id, const int32 feature_id,
+    const int32 threshold, const float gain, const float left_contrib,
+    const float right_contrib, int32* left_node_id, int32* right_node_id) {
+  auto* tree = tree_ensemble_->mutable_trees(tree_id);
+  auto* node = tree->mutable_nodes(node_id);
+  DCHECK_EQ(node->node_case(), boosted_trees::Node::kLeaf);
+  float prev_node_value = node->leaf().scalar();
+  *left_node_id = tree->nodes_size();
+  *right_node_id = *left_node_id + 1;
+  auto* left_node = tree->add_nodes();
+  auto* right_node = tree->add_nodes();
+  if (node_id != 0) {
+    // Save previous leaf value if it is not the first leaf in the tree.
+    node->mutable_metadata()->mutable_original_leaf()->Swap(
+        node->mutable_leaf());
+  }
+  node->mutable_metadata()->set_gain(gain);
+  auto* new_split = node->mutable_bucketized_split();
+  new_split->set_feature_id(feature_id);
+  new_split->set_threshold(threshold);
+  new_split->set_left_id(*left_node_id);
+  new_split->set_right_id(*right_node_id);
+  // TODO(npononareva): this is LAYER-BY-LAYER boosting; add WHOLE-TREE.
+  left_node->mutable_leaf()->set_scalar(prev_node_value + left_contrib);
+  right_node->mutable_leaf()->set_scalar(prev_node_value + right_contrib);
+}
+
+void BoostedTreesEnsembleResource::Reset() {
+  // Reset stamp.
+  set_stamp(-1);
+
+  // Clear tree ensemle.
+  arena_.Reset();
+  CHECK_EQ(0, arena_.SpaceAllocated());
+  tree_ensemble_ =
+      protobuf::Arena::CreateMessage<boosted_trees::TreeEnsemble>(&arena_);
+}
+
+void BoostedTreesEnsembleResource::PostPruneTree(const int32 current_tree) {
+  // No-op if tree is empty.
+  auto* tree = tree_ensemble_->mutable_trees(current_tree);
+  int32 num_nodes = tree->nodes_size();
+  if (num_nodes == 0) {
+    return;
+  }
+
+  std::vector<int32> nodes_to_delete;
+  // If a node was pruned, we need to save the change of the prediction from
+  // this node to its parent, as well as the parent id.
+  std::vector<std::pair<int32, float>> nodes_changes;
+  nodes_changes.reserve(num_nodes);
+  for (int32 i = 0; i < num_nodes; ++i) {
+    nodes_changes.emplace_back(i, 0.0);
+  }
+  // Prune the tree recursively starting from the root. Each node that has
+  // negative gain and only leaf children will be pruned recursively up from
+  // the bottom of the tree. This method returns the list of nodes pruned, and
+  // updates the nodes in the tree not to refer to those pruned nodes.
+  RecursivelyDoPostPrunePreparation(current_tree, 0, &nodes_to_delete,
+                                    &nodes_changes);
+
+  if (nodes_to_delete.empty()) {
+    // No pruning happened, and no post-processing needed.
+    return;
+  }
+
+  // Sort node ids so they are in asc order.
+  std::sort(nodes_to_delete.begin(), nodes_to_delete.end());
+
+  // We need to
+  // - update split left and right children ids with new indices
+  // - actually remove the nodes that need to be removed
+  // - save the information about pruned node so we could recover the
+  // predictions from cache. Build a map for old node index=>new node index.
+  // nodes_to_delete contains nodes who's indices should be skipped, in
+  // ascending order. Save the information about new indices into meta.
+  std::map<int32, int32> old_to_new_ids;
+  int32 new_index = 0;
+  int32 index_for_deleted = 0;
+  auto* post_prune_meta = tree_ensemble_->mutable_tree_metadata(current_tree)
+                              ->mutable_post_pruned_nodes_meta();
+
+  for (int32 i = 0; i < num_nodes; ++i) {
+    if (index_for_deleted < nodes_to_delete.size() &&
+        i == nodes_to_delete[index_for_deleted]) {
+      // Node i will get removed,
+      ++index_for_deleted;
+      // Update meta info that will allow us to use cached predictions from
+      // those nodes.
+      int32 new_id;
+      float logit_change;
+      CalculateParentAndLogitUpdate(i, nodes_changes, &new_id, &logit_change);
+      auto* meta = post_prune_meta->Add();
+      meta->set_new_node_id(old_to_new_ids[new_id]);
+      meta->set_logit_change(logit_change);
+    } else {
+      old_to_new_ids[i] = new_index++;
+      auto* meta = post_prune_meta->Add();
+      // Update meta info that will allow us to use cached predictions from
+      // those nodes.
+      meta->set_new_node_id(old_to_new_ids[i]);
+      meta->set_logit_change(0.0);
+    }
+  }
+  index_for_deleted = 0;
+  int32 i = 0;
+  protobuf::RepeatedPtrField<boosted_trees::Node> new_nodes;
+  new_nodes.Reserve(old_to_new_ids.size());
+  for (auto node : *(tree->mutable_nodes())) {
+    if (index_for_deleted < nodes_to_delete.size() &&
+        i == nodes_to_delete[index_for_deleted]) {
+      ++index_for_deleted;
+      ++i;
+      continue;
+    } else {
+      if (node.node_case() == boosted_trees::Node::kBucketizedSplit) {
+        node.mutable_bucketized_split()->set_left_id(
+            old_to_new_ids[node.bucketized_split().left_id()]);
+        node.mutable_bucketized_split()->set_right_id(
+            old_to_new_ids[node.bucketized_split().right_id()]);
+      }
+      *new_nodes.Add() = std::move(node);
+    }
+    ++i;
+  }
+  // Replace all the nodes in a tree with the ones we keep.
+  *tree->mutable_nodes() = std::move(new_nodes);
+
+  // Note that if the whole tree got pruned, we will end up with one node.
+  // We can't remove that tree because it will cause problems with cache.
+}
+
+void BoostedTreesEnsembleResource::GetPostPruneCorrection(
+    const int32 tree_id, const int32 initial_node_id, int32* current_node_id,
+    float* logit_update) const {
+  DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+  if (IsTreeFinalized(tree_id) && IsTreePostPruned(tree_id)) {
+    DCHECK_LT(
+        initial_node_id,
+        tree_ensemble_->tree_metadata(tree_id).post_pruned_nodes_meta_size());
+    const auto& meta =
+        tree_ensemble_->tree_metadata(tree_id).post_pruned_nodes_meta(
+            initial_node_id);
+    *current_node_id = meta.new_node_id();
+    *logit_update += meta.logit_change();
+  }
+}
+
+bool BoostedTreesEnsembleResource::IsTerminalSplitNode(
+    const int32 tree_id, const int32 node_id) const {
+  const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id);
+  DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+  const int32 left_id = node.bucketized_split().left_id();
+  const int32 right_id = node.bucketized_split().right_id();
+  return is_leaf(tree_id, left_id) && is_leaf(tree_id, right_id);
+}
+
+// For each pruned node, finds the leaf where it finally ended up and
+// calculates the total update from that pruned node prediction.
+void BoostedTreesEnsembleResource::CalculateParentAndLogitUpdate(
+    const int32 start_node_id,
+    const std::vector<std::pair<int32, float>>& nodes_change, int32* parent_id,
+    float* change) const {
+  *change = 0.0;
+  int32 node_id = start_node_id;
+  int32 parent = nodes_change[node_id].first;
+
+  while (parent != node_id) {
+    (*change) += nodes_change[node_id].second;
+    node_id = parent;
+    parent = nodes_change[node_id].first;
+  }
+  *parent_id = parent;
+}
+
+void BoostedTreesEnsembleResource::RecursivelyDoPostPrunePreparation(
+    const int32 tree_id, const int32 node_id,
+    std::vector<int32>* nodes_to_delete,
+    std::vector<std::pair<int32, float>>* nodes_meta) {
+  auto* node = tree_ensemble_->mutable_trees(tree_id)->mutable_nodes(node_id);
+  DCHECK_NE(node->node_case(), boosted_trees::Node::NODE_NOT_SET);
+  // Base case when we reach a leaf.
+  if (node->node_case() == boosted_trees::Node::kLeaf) {
+    return;
+  }
+
+  // Traverse node children first and recursively prune their sub-trees.
+  RecursivelyDoPostPrunePreparation(tree_id, node->bucketized_split().left_id(),
+                                    nodes_to_delete, nodes_meta);
+  RecursivelyDoPostPrunePreparation(tree_id,
+                                    node->bucketized_split().right_id(),
+                                    nodes_to_delete, nodes_meta);
+
+  // Two conditions must be satisfied to prune the node:
+  // 1- The split gain is negative.
+  // 2- After depth-first pruning, the node only has leaf children.
+  const auto& node_metadata = node->metadata();
+  if (node_metadata.gain() < 0 && IsTerminalSplitNode(tree_id, node_id)) {
+    const int32 left_id = node->bucketized_split().left_id();
+    const int32 right_id = node->bucketized_split().right_id();
+
+    // Save children that need to be deleted.
+    nodes_to_delete->push_back(left_id);
+    nodes_to_delete->push_back(right_id);
+
+    // Change node back into leaf.
+    *node->mutable_leaf() = node_metadata.original_leaf();
+    const float parent_value = node_value(tree_id, node_id);
+
+    // Save the old values of weights of children.
+    (*nodes_meta)[left_id].first = node_id;
+    (*nodes_meta)[left_id].second = parent_value - node_value(tree_id, left_id);
+
+    (*nodes_meta)[right_id].first = node_id;
+    (*nodes_meta)[right_id].second =
+        parent_value - node_value(tree_id, right_id);
+
+    // Clear gain for leaf node.
+    node->clear_metadata();
+  }
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/boosted_trees/resources.h b/tensorflow/core/kernels/boosted_trees/resources.h
new file mode 100644
index 0000000000..c82588b950
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/resources.h
@@ -0,0 +1,221 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
+#define TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
+
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/kernels/boosted_trees/boosted_trees.pb.h"
+#include "tensorflow/core/platform/mutex.h"
+#include "tensorflow/core/platform/protobuf.h"
+
+namespace tensorflow {
+
+// A StampedResource is a resource that has a stamp token associated with it.
+// Before reading from or applying updates to the resource, the stamp should
+// be checked to verify that the update is not stale.
+class StampedResource : public ResourceBase {
+ public:
+  StampedResource() : stamp_(-1) {}
+
+  bool is_stamp_valid(int64 stamp) const { return stamp_ == stamp; }
+
+  int64 stamp() const { return stamp_; }
+  void set_stamp(int64 stamp) { stamp_ = stamp; }
+
+ private:
+  int64 stamp_;
+};
+
+// Keep a tree ensemble in memory for efficient evaluation and mutation.
+class BoostedTreesEnsembleResource : public StampedResource {
+ public:
+  // Constructor.
+  BoostedTreesEnsembleResource()
+      : tree_ensemble_(
+            protobuf::Arena::CreateMessage<boosted_trees::TreeEnsemble>(
+                &arena_)) {}
+
+  string DebugString() override {
+    return strings::StrCat("TreeEnsemble[size=", tree_ensemble_->trees_size(),
+                           "]");
+  }
+
+  bool InitFromSerialized(const string& serialized, const int64 stamp_token) {
+    CHECK_EQ(stamp(), -1) << "Must Reset before Init.";
+    if (ParseProtoUnlimited(tree_ensemble_, serialized)) {
+      set_stamp(stamp_token);
+      return true;
+    }
+    return false;
+  }
+
+  string SerializeAsString() const {
+    return tree_ensemble_->SerializeAsString();
+  }
+
+  int32 num_trees() const { return tree_ensemble_->trees_size(); }
+
+  // Find the next node to which the example (specified by index_in_batch)
+  // traverses down from the current node indicated by tree_id and node_id.
+  // Args:
+  //   tree_id: the index of the tree in the ensemble.
+  //   node_id: the index of the node within the tree.
+  //   index_in_batch: the index of the example within the batch (relevant to
+  //       the index of the row to read in each bucketized_features).
+  //   bucketized_features: vector of feature Vectors.
+  int32 next_node(
+      const int32 tree_id, const int32 node_id, const int32 index_in_batch,
+      const std::vector<TTypes<int32>::ConstVec>& bucketized_features) const;
+
+  float node_value(const int32 tree_id, const int32 node_id) const;
+
+  int32 GetNumLayersGrown(const int32 tree_id) const {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    return tree_ensemble_->tree_metadata(tree_id).num_layers_grown();
+  }
+
+  void SetNumLayersGrown(const int32 tree_id, int32 new_num_layers) const {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    tree_ensemble_->mutable_tree_metadata(tree_id)->set_num_layers_grown(
+        new_num_layers);
+  }
+
+  void UpdateGrowingMetadata() const;
+
+  int32 GetNumLayersAttempted() {
+    return tree_ensemble_->growing_metadata().num_layers_attempted();
+  }
+
+  bool is_leaf(const int32 tree_id, const int32 node_id) const {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    DCHECK_LT(node_id, tree_ensemble_->trees(tree_id).nodes_size());
+    const auto& node = tree_ensemble_->trees(tree_id).nodes(node_id);
+    return node.node_case() == boosted_trees::Node::kLeaf;
+  }
+
+  int32 feature_id(const int32 tree_id, const int32 node_id) const {
+    const auto node = tree_ensemble_->trees(tree_id).nodes(node_id);
+    DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+    return node.bucketized_split().feature_id();
+  }
+
+  int32 bucket_threshold(const int32 tree_id, const int32 node_id) const {
+    const auto node = tree_ensemble_->trees(tree_id).nodes(node_id);
+    DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+    return node.bucketized_split().threshold();
+  }
+
+  int32 left_id(const int32 tree_id, const int32 node_id) const {
+    const auto node = tree_ensemble_->trees(tree_id).nodes(node_id);
+    DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+    return node.bucketized_split().left_id();
+  }
+
+  int32 right_id(const int32 tree_id, const int32 node_id) const {
+    const auto node = tree_ensemble_->trees(tree_id).nodes(node_id);
+    DCHECK_EQ(node.node_case(), boosted_trees::Node::kBucketizedSplit);
+    return node.bucketized_split().right_id();
+  }
+
+  // Add a tree to the ensemble and returns a new tree_id.
+  int32 AddNewTree(const float weight);
+
+  // Grows the tree by adding a split and leaves.
+  void AddBucketizedSplitNode(const int32 tree_id, const int32 node_id,
+                              const int32 feature_id, const int32 threshold,
+                              const float gain, const float left_contrib,
+                              const float right_contrib, int32* left_node_id,
+                              int32* right_node_id);
+
+  // Retrieves tree weights and returns as a vector.
+  // It involves a copy, so should be called only sparingly (like once per
+  // iteration, not per example).
+  std::vector<float> GetTreeWeights() const {
+    return {tree_ensemble_->tree_weights().begin(),
+            tree_ensemble_->tree_weights().end()};
+  }
+
+  float GetTreeWeight(const int32 tree_id) const {
+    return tree_ensemble_->tree_weights(tree_id);
+  }
+
+  float IsTreeFinalized(const int32 tree_id) const {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    return tree_ensemble_->tree_metadata(tree_id).is_finalized();
+  }
+
+  float IsTreePostPruned(const int32 tree_id) const {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    return tree_ensemble_->tree_metadata(tree_id)
+               .post_pruned_nodes_meta_size() > 0;
+  }
+
+  void SetIsFinalized(const int32 tree_id, const bool is_finalized) {
+    DCHECK_LT(tree_id, tree_ensemble_->trees_size());
+    return tree_ensemble_->mutable_tree_metadata(tree_id)->set_is_finalized(
+        is_finalized);
+  }
+
+  // Sets the weight of i'th tree.
+  void SetTreeWeight(const int32 tree_id, const float weight) {
+    DCHECK_GE(tree_id, 0);
+    DCHECK_LT(tree_id, num_trees());
+    tree_ensemble_->set_tree_weights(tree_id, weight);
+  }
+
+  // Resets the resource and frees the protos in arena.
+  // Caller needs to hold the mutex lock while calling this.
+  virtual void Reset();
+
+  void PostPruneTree(const int32 current_tree);
+
+  // For a given node, returns the id in a pruned tree, as well as correction
+  // to the cached prediction that should be applied. If tree was not
+  // post-pruned, current_node_id will be equal to initial_node_id and logit
+  // update will be equal to zero.
+  void GetPostPruneCorrection(const int32 tree_id, const int32 initial_node_id,
+                              int32* current_node_id,
+                              float* logit_update) const;
+  mutex* get_mutex() { return &mu_; }
+
+ private:
+  // Helper method to check whether a node is a terminal node in that it
+  // only has leaf nodes as children.
+  bool IsTerminalSplitNode(const int32 tree_id, const int32 node_id) const;
+
+  // For each pruned node, finds the leaf where it finally ended up and
+  // calculates the total update from that pruned node prediction.
+  void CalculateParentAndLogitUpdate(
+      const int32 start_node_id,
+      const std::vector<std::pair<int32, float>>& nodes_change,
+      int32* parent_id, float* change) const;
+
+  // Helper method to collect the information to be used to prune some nodes in
+  // the tree.
+  void RecursivelyDoPostPrunePreparation(
+      const int32 tree_id, const int32 node_id,
+      std::vector<int32>* nodes_to_delete,
+      std::vector<std::pair<int32, float>>* nodes_meta);
+
+ protected:
+  protobuf::Arena arena_;
+  mutex mu_;
+  boosted_trees::TreeEnsemble* tree_ensemble_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_KERNELS_BOOSTED_TREES_RESOURCES_H_
diff --git a/tensorflow/core/kernels/boosted_trees/stats_ops.cc b/tensorflow/core/kernels/boosted_trees/stats_ops.cc
new file mode 100644
index 0000000000..33fdab6a86
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/stats_ops.cc
@@ -0,0 +1,296 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <vector>
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor.h"
+
+namespace tensorflow {
+
+namespace {
+const float kEps = 1e-15;
+}  // namespace
+
+class BoostedTreesCalculateBestGainsPerFeatureOp : public OpKernel {
+ public:
+  explicit BoostedTreesCalculateBestGainsPerFeatureOp(
+      OpKernelConstruction* const context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("l1", &l1_));
+    OP_REQUIRES_OK(context, context->GetAttr("l2", &l2_));
+    OP_REQUIRES_OK(context,
+                   context->GetAttr("tree_complexity", &tree_complexity_));
+    OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_));
+    OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_));
+  }
+
+  void Compute(OpKernelContext* const context) override {
+    // node_id_range
+    const Tensor* node_id_range_t;
+    OP_REQUIRES_OK(context, context->input("node_id_range", &node_id_range_t));
+    const auto node_id_range = node_id_range_t->vec<int32>();
+    int32 node_id_first = node_id_range(0);
+    int32 node_id_last = node_id_range(1);  // inclusive.
+    // stats_summary_list
+    OpInputList stats_summary_list;
+    OP_REQUIRES_OK(context, context->input_list("stats_summary_list",
+                                                &stats_summary_list));
+    const int64 num_buckets = stats_summary_list[0].dim_size(1);
+    std::vector<TTypes<float, 3>::ConstTensor> stats_summary;
+    stats_summary.reserve(stats_summary_list.size());
+    for (const auto& tensor : stats_summary_list) {
+      stats_summary.emplace_back(tensor.tensor<float, 3>());
+    }
+
+    // Allocate output lists of tensors:
+    OpOutputList output_node_ids_list;
+    OP_REQUIRES_OK(
+        context, context->output_list("node_ids_list", &output_node_ids_list));
+    OpOutputList output_gains_list;
+    OP_REQUIRES_OK(context,
+                   context->output_list("gains_list", &output_gains_list));
+    OpOutputList output_thresholds_list;
+    OP_REQUIRES_OK(context, context->output_list("thresholds_list",
+                                                 &output_thresholds_list));
+    OpOutputList output_left_node_contribs_list;
+    OP_REQUIRES_OK(context,
+                   context->output_list("left_node_contribs_list",
+                                        &output_left_node_contribs_list));
+    OpOutputList output_right_node_contribs_list;
+    OP_REQUIRES_OK(context,
+                   context->output_list("right_node_contribs_list",
+                                        &output_right_node_contribs_list));
+
+    // Get the best split info per node for each feature.
+    for (int feature_idx = 0; feature_idx < num_features_; ++feature_idx) {
+      std::vector<float> cum_grad;
+      std::vector<float> cum_hess;
+      cum_grad.reserve(num_buckets);
+      cum_hess.reserve(num_buckets);
+
+      std::vector<int32> output_node_ids;
+      std::vector<float> output_gains;
+      std::vector<int32> output_thresholds;
+      std::vector<float> output_left_node_contribs;
+      std::vector<float> output_right_node_contribs;
+      for (int node_id = node_id_first; node_id <= node_id_last; ++node_id) {
+        // Calculate gains.
+        cum_grad.clear();
+        cum_hess.clear();
+        float total_grad = 0.0;
+        float total_hess = 0.0;
+        for (int bucket = 0; bucket < num_buckets; ++bucket) {
+          // TODO(nponomareva): Consider multi-dimensional gradients/hessians.
+          total_grad += stats_summary[feature_idx](node_id, bucket, 0);
+          total_hess += stats_summary[feature_idx](node_id, bucket, 1);
+          cum_grad.push_back(total_grad);
+          cum_hess.push_back(total_hess);
+        }
+        float best_gain = std::numeric_limits<float>::lowest();
+        float best_bucket = 0;
+        float best_contrib_for_left = 0.0;
+        float best_contrib_for_right = 0.0;
+        // Parent gain.
+        float parent_gain;
+        float unused;
+        CalculateWeightsAndGains(total_grad, total_hess, &unused, &parent_gain);
+
+        for (int bucket = 0; bucket < num_buckets; ++bucket) {
+          const float cum_grad_bucket = cum_grad[bucket];
+          const float cum_hess_bucket = cum_hess[bucket];
+          // Left child.
+          float contrib_for_left;
+          float gain_for_left;
+          CalculateWeightsAndGains(cum_grad_bucket, cum_hess_bucket,
+                                   &contrib_for_left, &gain_for_left);
+          // Right child.
+          float contrib_for_right;
+          float gain_for_right;
+          CalculateWeightsAndGains(total_grad - cum_grad_bucket,
+                                   total_hess - cum_hess_bucket,
+                                   &contrib_for_right, &gain_for_right);
+
+          if (gain_for_left + gain_for_right > best_gain) {
+            best_gain = gain_for_left + gain_for_right;
+            best_bucket = bucket;
+            best_contrib_for_left = contrib_for_left;
+            best_contrib_for_right = contrib_for_right;
+          }
+        }  // for bucket
+        output_node_ids.push_back(node_id);
+        // Remove the parent gain for the parent node.
+        output_gains.push_back(best_gain - parent_gain);
+        output_thresholds.push_back(best_bucket);
+        output_left_node_contribs.push_back(best_contrib_for_left);
+        output_right_node_contribs.push_back(best_contrib_for_right);
+      }  // for node_id
+      const int num_nodes = output_node_ids.size();
+      // output_node_ids
+      Tensor* output_node_ids_t;
+      OP_REQUIRES_OK(context,
+                     output_node_ids_list.allocate(feature_idx, {num_nodes},
+                                                   &output_node_ids_t));
+      auto output_node_ids_vec = output_node_ids_t->vec<int32>();
+      // output_gains
+      Tensor* output_gains_t;
+      OP_REQUIRES_OK(context, output_gains_list.allocate(
+                                  feature_idx, {num_nodes}, &output_gains_t));
+      auto output_gains_vec = output_gains_t->vec<float>();
+      // output_thresholds
+      Tensor* output_thresholds_t;
+      OP_REQUIRES_OK(context,
+                     output_thresholds_list.allocate(feature_idx, {num_nodes},
+                                                     &output_thresholds_t));
+      auto output_thresholds_vec = output_thresholds_t->vec<int32>();
+      // output_left_node_contribs
+      Tensor* output_left_node_contribs_t;
+      OP_REQUIRES_OK(context, output_left_node_contribs_list.allocate(
+                                  feature_idx, {num_nodes, 1},
+                                  &output_left_node_contribs_t));
+      auto output_left_node_contribs_matrix =
+          output_left_node_contribs_t->matrix<float>();
+      // output_right_node_contribs
+      Tensor* output_right_node_contribs_t;
+      OP_REQUIRES_OK(context, output_right_node_contribs_list.allocate(
+                                  feature_idx, {num_nodes, 1},
+                                  &output_right_node_contribs_t));
+      auto output_right_node_contribs_matrix =
+          output_right_node_contribs_t->matrix<float>();
+      // Sets output tensors from vectors.
+      for (int i = 0; i < num_nodes; ++i) {
+        output_node_ids_vec(i) = output_node_ids[i];
+        // Adjust the gains to penalize by tree complexity.
+        output_gains_vec(i) = output_gains[i] - tree_complexity_;
+        output_thresholds_vec(i) = output_thresholds[i];
+        // Logits are 1-dimensional for now.
+        // TODO(nponomareva): Consider multi-dimensional logits.
+        output_left_node_contribs_matrix(i, 0) = output_left_node_contribs[i];
+        output_right_node_contribs_matrix(i, 0) = output_right_node_contribs[i];
+      }
+    }  // for f
+  }
+
+ private:
+  void CalculateWeightsAndGains(const float g, const float h, float* weight,
+                                float* gain) {
+    //
+    // The formula for weight is -(g+l1*sgn(w))/(H+l2), for gain it is
+    // (g+l1*sgn(w))^2/(h+l2).
+    // This is because for each leaf we optimize
+    // 1/2(h+l2)*w^2+g*w+l1*abs(w)
+    float g_with_l1 = g;
+    // Apply L1 regularization.
+    // 1) Assume w>0 => w=-(g+l1)/(h+l2)=> g+l1 < 0 => g < -l1
+    // 2) Assume w<0 => w=-(g-l1)/(h+l2)=> g-l1 > 0 => g > l1
+    // For g from (-l1, l1), thus there is no solution => set to 0.
+    if (l1_ > 0) {
+      if (g > l1_) {
+        g_with_l1 -= l1_;
+      } else if (g < -l1_) {
+        g_with_l1 += l1_;
+      } else {
+        *weight = 0.0;
+        *gain = 0.0;
+        return;
+      }
+    }
+    // Apply L2 regularization.
+    if (h + l2_ <= kEps) {
+      // Avoid division by 0 or infinitesimal.
+      *weight = 0;
+      *gain = 0;
+    } else {
+      *weight = -g_with_l1 / (h + l2_);
+      *gain = -g_with_l1 * (*weight);
+    }
+  }
+
+  float l1_;
+  float l2_;
+  float tree_complexity_;
+  int max_splits_;
+  int num_features_;
+};
+
+REGISTER_KERNEL_BUILDER(
+    Name("BoostedTreesCalculateBestGainsPerFeature").Device(DEVICE_CPU),
+    BoostedTreesCalculateBestGainsPerFeatureOp);
+
+class BoostedTreesMakeStatsSummaryOp : public OpKernel {
+ public:
+  explicit BoostedTreesMakeStatsSummaryOp(OpKernelConstruction* const context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("max_splits", &max_splits_));
+    OP_REQUIRES_OK(context, context->GetAttr("num_buckets", &num_buckets_));
+    OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_));
+  }
+
+  void Compute(OpKernelContext* const context) override {
+    // node_ids
+    const Tensor* node_ids_t;
+    OP_REQUIRES_OK(context, context->input("node_ids", &node_ids_t));
+    const auto node_ids = node_ids_t->vec<int32>();
+    // gradients
+    const Tensor* gradients_t;
+    OP_REQUIRES_OK(context, context->input("gradients", &gradients_t));
+    const auto gradients = gradients_t->matrix<float>();
+    // hessians
+    const Tensor* hessians_t;
+    OP_REQUIRES_OK(context, context->input("hessians", &hessians_t));
+    const auto hessians = hessians_t->matrix<float>();
+    // bucketized_features
+    OpInputList bucketized_features_list;
+    OP_REQUIRES_OK(context, context->input_list("bucketized_features_list",
+                                                &bucketized_features_list));
+    std::vector<tensorflow::TTypes<int32>::ConstVec> bucketized_features;
+    bucketized_features.reserve(num_features_);
+    for (const Tensor& tensor : bucketized_features_list) {
+      bucketized_features.emplace_back(tensor.vec<int32>());
+    }
+
+    // Infer batch size.
+    const int64 batch_size = node_ids_t->dim_size(0);
+    // Allocate output stats tensor (Rank 4).
+    Tensor* output_stats_summary_t = nullptr;
+    OP_REQUIRES_OK(context, context->allocate_output(
+                                "stats_summary",
+                                {num_features_, max_splits_, num_buckets_, 2},
+                                &output_stats_summary_t));
+    auto output_stats_summary = output_stats_summary_t->tensor<float, 4>();
+    output_stats_summary.setZero();
+
+    // Partition by node, and then bucketize.
+    for (int feature_idx = 0; feature_idx < num_features_; ++feature_idx) {
+      const auto& features = bucketized_features[feature_idx];
+      for (int i = 0; i < batch_size; ++i) {
+        const int32 node = node_ids(i);
+        const int32 bucket = features(i);
+        output_stats_summary(feature_idx, node, bucket, 0) += gradients(i, 0);
+        output_stats_summary(feature_idx, node, bucket, 1) += hessians(i, 0);
+      }
+    }
+  }
+
+ private:
+  int max_splits_;
+  int num_buckets_;
+  int num_features_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("BoostedTreesMakeStatsSummary").Device(DEVICE_CPU),
+                        BoostedTreesMakeStatsSummaryOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/kernels/boosted_trees/training_ops.cc b/tensorflow/core/kernels/boosted_trees/training_ops.cc
new file mode 100644
index 0000000000..b9ded4054a
--- /dev/null
+++ b/tensorflow/core/kernels/boosted_trees/training_ops.cc
@@ -0,0 +1,219 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/framework/op_kernel.h"
+#include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/boosted_trees/resources.h"
+
+namespace tensorflow {
+
+namespace {
+constexpr float kLayerByLayerTreeWeight = 1.0;
+
+// TODO(nponomareva, youngheek): consider using vector.
+struct SplitCandidate {
+  SplitCandidate() {}
+
+  // Index in the list of the feature ids.
+  int64 feature_idx;
+
+  // Index in the tensor of node_ids for the feature with idx feature_idx.
+  int64 candidate_idx;
+
+  float gain;
+};
+
+enum PruningMode { kNoPruning = 0, kPrePruning = 1, kPostPruning = 2 };
+
+}  // namespace
+
+class BoostedTreesUpdateEnsembleOp : public OpKernel {
+ public:
+  explicit BoostedTreesUpdateEnsembleOp(OpKernelConstruction* const context)
+      : OpKernel(context) {
+    OP_REQUIRES_OK(context, context->GetAttr("max_depth", &max_depth_));
+    OP_REQUIRES_OK(context, context->GetAttr("learning_rate", &learning_rate_));
+    OP_REQUIRES_OK(context, context->GetAttr("num_features", &num_features_));
+
+    int32 pruning_index;
+    OP_REQUIRES_OK(context, context->GetAttr("pruning_mode", &pruning_index));
+    pruning_mode_ = static_cast<PruningMode>(pruning_index);
+  }
+
+  void Compute(OpKernelContext* const context) override {
+    // Get decision tree ensemble.
+    BoostedTreesEnsembleResource* ensemble_resource;
+    OP_REQUIRES_OK(context, LookupResource(context, HandleFromInput(context, 0),
+                                           &ensemble_resource));
+    core::ScopedUnref unref_me(ensemble_resource);
+    mutex_lock l(*ensemble_resource->get_mutex());
+    // Increase the ensemble stamp.
+    ensemble_resource->set_stamp(ensemble_resource->stamp() + 1);
+
+    // Read node ids, gains, thresholds and node contribs.
+    OpInputList node_ids_list;
+    OpInputList gains_list;
+    OpInputList thresholds_list;
+    OpInputList left_node_contribs;
+    OpInputList right_node_contribs;
+    OP_REQUIRES_OK(context, context->input_list("node_ids", &node_ids_list));
+    OP_REQUIRES_OK(context, context->input_list("gains", &gains_list));
+    OP_REQUIRES_OK(context,
+                   context->input_list("thresholds", &thresholds_list));
+    OP_REQUIRES_OK(context, context->input_list("left_node_contribs",
+                                                &left_node_contribs));
+    OP_REQUIRES_OK(context, context->input_list("right_node_contribs",
+                                                &right_node_contribs));
+
+    const Tensor* feature_ids_t;
+    OP_REQUIRES_OK(context, context->input("feature_ids", &feature_ids_t));
+
+    auto feature_ids = feature_ids_t->vec<int32>();
+
+    // Find best splits for each active node.
+    std::map<int32, SplitCandidate> best_splits;
+    FindBestSplitsPerNode(context, node_ids_list, gains_list, &best_splits);
+
+    int32 current_tree =
+        UpdateGlobalAttemptsAndRetrieveGrowableTree(ensemble_resource);
+
+    // No-op if no new splits can be considered.
+    if (best_splits.empty()) {
+      LOG(WARNING) << "Not growing tree ensemble as no good splits were found.";
+      return;
+    }
+
+    const int32 new_num_layers =
+        ensemble_resource->GetNumLayersGrown(current_tree) + 1;
+    VLOG(1) << "Adding layer #" << new_num_layers - 1 << " to tree #"
+            << current_tree << " of ensemble of " << current_tree + 1
+            << " trees.";
+    bool split_happened = false;
+    // Add the splits to the tree.
+    for (auto& split_entry : best_splits) {
+      const int32 node_id = split_entry.first;
+      const SplitCandidate& candidate = split_entry.second;
+
+      const int64 feature_idx = candidate.feature_idx;
+      const int64 candidate_idx = candidate.candidate_idx;
+
+      const int32 feature_id = feature_ids(feature_idx);
+      const int32 threshold =
+          thresholds_list[feature_idx].vec<int32>()(candidate_idx);
+      const float gain = gains_list[feature_idx].vec<float>()(candidate_idx);
+
+      if (pruning_mode_ == kPrePruning) {
+        // Don't consider negative splits if we're pre-pruning the tree.
+        // Note that zero-gain splits are acceptable.
+        if (gain < 0) {
+          continue;
+        }
+      }
+      // For now assume that the weights vectors are one dimensional.
+      // TODO(nponomareva): change here for multiclass.
+      const float left_contrib =
+          learning_rate_ *
+          left_node_contribs[feature_idx].matrix<float>()(candidate_idx, 0);
+      const float right_contrib =
+          learning_rate_ *
+          right_node_contribs[feature_idx].matrix<float>()(candidate_idx, 0);
+
+      // unused.
+      int32 left_node_id;
+      int32 right_node_id;
+
+      ensemble_resource->AddBucketizedSplitNode(
+          current_tree, node_id, feature_id, threshold, gain, left_contrib,
+          right_contrib, &left_node_id, &right_node_id);
+      split_happened = true;
+    }
+    if (split_happened) {
+      // Update growable tree metadata.
+      ensemble_resource->SetNumLayersGrown(current_tree, new_num_layers);
+      // Finalize the tree if needed.
+      if (ensemble_resource->GetNumLayersGrown(current_tree) >= max_depth_) {
+        ensemble_resource->SetIsFinalized(current_tree, true);
+        if (pruning_mode_ == kPostPruning) {
+          ensemble_resource->PostPruneTree(current_tree);
+        }
+        if (ensemble_resource->num_trees() > 0) {
+          // Create a dummy new tree with an empty node.
+          ensemble_resource->AddNewTree(kLayerByLayerTreeWeight);
+        }
+      }
+    }
+  }
+
+ private:
+  int32 UpdateGlobalAttemptsAndRetrieveGrowableTree(
+      BoostedTreesEnsembleResource* const ensemble_resource) {
+    int32 num_trees = ensemble_resource->num_trees();
+    int32 current_tree = num_trees - 1;
+
+    // Increment global attempt stats.
+    ensemble_resource->UpdateGrowingMetadata();
+
+    // Note we don't set tree weight to be equal to learning rate, since we
+    // apply learning rate to leaf weights instead, when doing layer-by-layer
+    // boosting.
+    if (num_trees <= 0) {
+      // Create a new tree with a no-op leaf.
+      current_tree = ensemble_resource->AddNewTree(kLayerByLayerTreeWeight);
+    }
+    return current_tree;
+  }
+
+  // Helper method which effectively does a reduce over all split candidates
+  // and finds the best split for each node.
+  void FindBestSplitsPerNode(
+      OpKernelContext* const context, const OpInputList& node_ids_list,
+      const OpInputList& gains_list,
+      std::map<int32, SplitCandidate>* best_split_per_node) {
+    // Find best split per node going through every feature candidate.
+    for (int64 feature_idx = 0; feature_idx < num_features_; ++feature_idx) {
+      const auto& node_ids = node_ids_list[feature_idx].vec<int32>();
+      const auto& gains = gains_list[feature_idx].vec<float>();
+
+      for (size_t candidate_idx = 0; candidate_idx < node_ids.size();
+           ++candidate_idx) {
+        // Get current split candidate.
+        const auto& node_id = node_ids(candidate_idx);
+        const auto& gain = gains(candidate_idx);
+
+        auto best_split_it = best_split_per_node->find(node_id);
+        SplitCandidate candidate;
+        candidate.feature_idx = feature_idx;
+        candidate.candidate_idx = candidate_idx;
+        candidate.gain = gain;
+
+        if (best_split_it == best_split_per_node->end() ||
+            gain > best_split_it->second.gain) {
+          (*best_split_per_node)[node_id] = candidate;
+        }
+      }
+    }
+  }
+
+ private:
+  int32 num_features_;
+  float learning_rate_;
+  int32 max_depth_;
+  PruningMode pruning_mode_;
+};
+
+REGISTER_KERNEL_BUILDER(Name("BoostedTreesUpdateEnsemble").Device(DEVICE_CPU),
+                        BoostedTreesUpdateEnsembleOp);
+
+}  // namespace tensorflow
diff --git a/tensorflow/core/ops/boosted_trees_ops.cc b/tensorflow/core/ops/boosted_trees_ops.cc
new file mode 100644
index 0000000000..297e94655f
--- /dev/null
+++ b/tensorflow/core/ops/boosted_trees_ops.cc
@@ -0,0 +1,319 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include <vector>
+
+#include "tensorflow/core/framework/common_shape_fns.h"
+#include "tensorflow/core/framework/op.h"
+#include "tensorflow/core/framework/resource_mgr.h"
+#include "tensorflow/core/framework/shape_inference.h"
+
+namespace tensorflow {
+
+REGISTER_RESOURCE_HANDLE_OP(BoostedTreesEnsembleResource);
+
+REGISTER_OP("IsBoostedTreesEnsembleInitialized")
+    .Input("tree_ensemble_handle: resource")
+    .Output("is_initialized: bool")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      c->set_output(0, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesCalculateBestGainsPerFeature")
+    .Input("node_id_range: int32")
+    .Input("stats_summary_list: num_features * float32")
+    .Attr("l1: float")
+    .Attr("l2: float")
+    .Attr("tree_complexity: float")
+    .Attr("max_splits: int >= 1")
+    .Attr("num_features: int >= 1")  // not passed but populated automatically.
+    .Output("node_ids_list: num_features * int32")
+    .Output("gains_list: num_features * float32")
+    .Output("thresholds_list: num_features * int32")
+    .Output("left_node_contribs_list: num_features * float32")
+    .Output("right_node_contribs_list: num_features * float32")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      // Confirms the rank of the inputs and sets the shape of the outputs.
+      int max_splits;
+      int num_features;
+      float l1, l2, tree_complexity;
+      TF_RETURN_IF_ERROR(c->GetAttr("l1", &l1));
+      if (l1 < 0) {
+        return errors::InvalidArgument("l1 must be non-negative.");
+      }
+      TF_RETURN_IF_ERROR(c->GetAttr("l2", &l2));
+      if (l2 < 0) {
+        return errors::InvalidArgument("l2 must be non-negative.");
+      }
+      TF_RETURN_IF_ERROR(c->GetAttr("tree_complexity", &tree_complexity));
+      if (tree_complexity < 0) {
+        return errors::InvalidArgument("Tree complexity must be non-negative.");
+      }
+      TF_RETURN_IF_ERROR(c->GetAttr("max_splits", &max_splits));
+      TF_RETURN_IF_ERROR(c->GetAttr("num_features", &num_features));
+      shape_inference::ShapeHandle node_id_range_shape;
+      shape_inference::ShapeHandle unused_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &node_id_range_shape));
+      TF_RETURN_IF_ERROR(
+          c->Merge(node_id_range_shape, c->MakeShape({2}), &unused_shape));
+      // Checks that all stats summary entries are of the same shape.
+      shape_inference::ShapeHandle summary_shape_base;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 3, &summary_shape_base));
+      TF_RETURN_IF_ERROR(c->Merge(summary_shape_base,
+                                  c->MakeShape({max_splits, -1, 2}),
+                                  &unused_shape));
+      for (int i = 1; i < num_features; ++i) {
+        shape_inference::ShapeHandle summary_shape;
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(1 + i), 3, &summary_shape));
+        TF_RETURN_IF_ERROR(
+            c->Merge(summary_shape_base, summary_shape, &unused_shape));
+      }
+      // Sets the output lists.
+      std::vector<shape_inference::ShapeHandle> output_shapes_vec(
+          num_features, c->MakeShape({-1}));
+      TF_RETURN_IF_ERROR(c->set_output("node_ids_list", output_shapes_vec));
+      TF_RETURN_IF_ERROR(c->set_output("gains_list", output_shapes_vec));
+      TF_RETURN_IF_ERROR(c->set_output("thresholds_list", output_shapes_vec));
+      std::vector<shape_inference::ShapeHandle> output_shapes_contribs(
+          num_features, c->MakeShape({-1, 1}));
+      TF_RETURN_IF_ERROR(
+          c->set_output("left_node_contribs_list", output_shapes_contribs));
+      TF_RETURN_IF_ERROR(
+          c->set_output("right_node_contribs_list", output_shapes_contribs));
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesCreateEnsemble")
+    .Input("tree_ensemble_handle: resource")
+    .Input("stamp_token: int64")
+    .Input("tree_ensemble_serialized: string")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input));
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesDeserializeEnsemble")
+    .Input("tree_ensemble_handle: resource")
+    .Input("stamp_token: int64")
+    .Input("tree_ensemble_serialized: string")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &unused_input));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 0, &unused_input));
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesGetEnsembleStates")
+    .Input("tree_ensemble_handle: resource")
+    .Output("stamp_token: int64")
+    .Output("num_trees: int32")
+    .Output("num_finalized_trees: int32")
+    .Output("num_attempted_layers: int32")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      c->set_output(0, c->Scalar());
+      c->set_output(1, c->Scalar());
+      c->set_output(2, c->Scalar());
+      c->set_output(3, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesMakeStatsSummary")
+    .Input("node_ids: int32")
+    .Input("gradients: float")
+    .Input("hessians: float")
+    .Input("bucketized_features_list: num_features * int32")
+    .Attr("max_splits: int >= 1")
+    .Attr("num_buckets: int >= 1")
+    .Attr("num_features: int >= 1")
+    .Output("stats_summary: float")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      // Sets the shape of the output as a Rank 4 Tensor.
+      int max_splits;
+      int num_buckets;
+      int num_features;
+      TF_RETURN_IF_ERROR(c->GetAttr("max_splits", &max_splits));
+      TF_RETURN_IF_ERROR(c->GetAttr("num_buckets", &num_buckets));
+      TF_RETURN_IF_ERROR(c->GetAttr("num_features", &num_features));
+      shape_inference::ShapeHandle node_ids_shape;
+      shape_inference::ShapeHandle gradients_shape;
+      shape_inference::ShapeHandle hessians_shape;
+      shape_inference::ShapeHandle bucketized_feature_shape;
+      shape_inference::ShapeHandle unused_shape;
+      shape_inference::DimensionHandle unused_dim;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 1, &node_ids_shape));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 2, &gradients_shape));
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(2), 2, &hessians_shape));
+      TF_RETURN_IF_ERROR(c->Merge(c->Dim(node_ids_shape, 0),
+                                  c->Dim(gradients_shape, 0), &unused_dim));
+      TF_RETURN_IF_ERROR(
+          c->Merge(gradients_shape, hessians_shape, &unused_shape));
+      for (int f = 0; f < num_features; ++f) {
+        TF_RETURN_IF_ERROR(
+            c->WithRank(c->input(3 + f), 1, &bucketized_feature_shape));
+        TF_RETURN_IF_ERROR(c->Merge(c->Dim(node_ids_shape, 0),
+                                    c->Dim(bucketized_feature_shape, 0),
+                                    &unused_dim));
+      }
+      c->set_output(0,
+                    c->MakeShape({num_features, max_splits, num_buckets, 2}));
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesPredict")
+    .Input("tree_ensemble_handle: resource")
+    .Input("bucketized_features: num_bucketized_features * int32")
+    .Attr("num_bucketized_features: int >= 1")
+    .Attr("logits_dimension: int")
+    .Attr("max_depth: int >= 1")
+    .Output("logits: float")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle feature_shape;
+      int num_bucketized_features;
+      TF_RETURN_IF_ERROR(
+          c->GetAttr("num_bucketized_features", &num_bucketized_features));
+      shape_inference::ShapeHandle unused_input;
+      for (int i = 0; i < num_bucketized_features; ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i + 1), 1, &feature_shape));
+        // Check that the shapes of all bucketized features are the same.
+        TF_RETURN_IF_ERROR(c->Merge(c->input(1), feature_shape, &unused_input));
+      }
+
+      int logits_dimension;
+      TF_RETURN_IF_ERROR(c->GetAttr("logits_dimension", &logits_dimension));
+      auto logits_shape =
+          c->MakeShape({c->Dim(feature_shape, 0), logits_dimension});
+      // Logits.
+      c->set_output(0, logits_shape);
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesSerializeEnsemble")
+    .Input("tree_ensemble_handle: resource")
+    .Output("stamp_token: int64")
+    .Output("tree_ensemble_serialized: string")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle unused_input;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 0, &unused_input));
+      c->set_output(0, c->Scalar());
+      c->set_output(1, c->Scalar());
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesTrainingPredict")
+    .Input("tree_ensemble_handle: resource")
+    .Input("cached_tree_ids: int32")
+    .Input("cached_node_ids: int32")
+    .Input("bucketized_features: num_bucketized_features * int32")
+    .Attr("num_bucketized_features: int >= 1")
+    .Attr("logits_dimension: int")
+    .Attr("max_depth: int >= 1")
+    .Output("partial_logits: float")
+    .Output("tree_ids: int32")
+    .Output("node_ids: int32")
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle feature_shape;
+      int num_bucketized_features;
+      TF_RETURN_IF_ERROR(
+          c->GetAttr("num_bucketized_features", &num_bucketized_features));
+
+      int max_depth;
+      TF_RETURN_IF_ERROR(c->GetAttr("max_depth", &max_depth));
+
+      shape_inference::ShapeHandle unused_input;
+      for (int i = 0; i < num_bucketized_features; ++i) {
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i + 3), 1, &feature_shape));
+        TF_RETURN_IF_ERROR(
+            c->Merge(c->input(i + 3), feature_shape, &unused_input));
+      }
+      // all inputs/outputs except logits should have same shape.
+      TF_RETURN_IF_ERROR(c->Merge(c->input(1), feature_shape, &unused_input));
+      TF_RETURN_IF_ERROR(c->Merge(c->input(2), feature_shape, &unused_input));
+
+      int logits_dimension;
+      TF_RETURN_IF_ERROR(c->GetAttr("logits_dimension", &logits_dimension));
+      auto logits_shape =
+          c->MakeShape({c->Dim(feature_shape, 0), logits_dimension});
+      // Partial logits.
+      c->set_output(0, logits_shape);
+      // Tree ids.
+      c->set_output(1, c->MakeShape({c->Dim(feature_shape, 0)}));
+      // Node ids.
+      c->set_output(2, c->MakeShape({c->Dim(feature_shape, 0)}));
+      return Status::OK();
+    });
+
+REGISTER_OP("BoostedTreesUpdateEnsemble")
+    .Input("tree_ensemble_handle: resource")
+    .Input("feature_ids: int32")
+    .Input("node_ids: num_features * int32")
+    .Input("gains: num_features * float")
+    .Input("thresholds: num_features * int32")
+    .Input("left_node_contribs: num_features * float")
+    .Input("right_node_contribs: num_features * float")
+    .Attr("max_depth: int >= 1")
+    .Attr("learning_rate: float")
+    .Attr("pruning_mode: int >=0")
+    .Attr("num_features: int >= 0")  // Inferred.
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle shape_handle;
+      int num_features;
+      TF_RETURN_IF_ERROR(c->GetAttr("num_features", &num_features));
+
+      // Feature_ids, should be one for each feature.
+      shape_inference::ShapeHandle feature_ids_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &feature_ids_shape));
+      TF_RETURN_IF_ERROR(
+          c->Merge(c->input(1), c->Vector(num_features), &shape_handle));
+
+      for (int i = 0; i < num_features; ++i) {
+        // Node ids.
+        TF_RETURN_IF_ERROR(c->WithRank(c->input(i + 2), 1, &shape_handle));
+        auto shape_rank_1 = c->MakeShape({c->Dim(shape_handle, 0)});
+        auto shape_rank_2 = c->MakeShape({c->Dim(shape_handle, 0), 1});
+
+        // Gains.
+        TF_RETURN_IF_ERROR(
+            c->WithRank(c->input(i + num_features + 2), 1, &shape_handle));
+        // TODO(nponomareva): replace this with input("name",vector of shapes).
+        TF_RETURN_IF_ERROR(c->Merge(c->input(i + num_features + 2),
+                                    shape_rank_1, &shape_handle));
+        // Thresholds.
+        TF_RETURN_IF_ERROR(
+            c->WithRank(c->input(i + num_features * 2 + 2), 1, &shape_handle));
+        TF_RETURN_IF_ERROR(c->Merge(c->input(i + num_features * 2 + 2),
+                                    shape_rank_1, &shape_handle));
+        // Left and right node contribs.
+        TF_RETURN_IF_ERROR(
+            c->WithRank(c->input(i + num_features * 3 + 2), 2, &shape_handle));
+        TF_RETURN_IF_ERROR(c->Merge(c->input(i + num_features * 3 + 2),
+                                    shape_rank_2, &shape_handle));
+        TF_RETURN_IF_ERROR(
+            c->WithRank(c->input(i + num_features * 4 + 2), 2, &shape_handle));
+        TF_RETURN_IF_ERROR(c->Merge(c->input(i + num_features * 4 + 2),
+                                    shape_rank_2, &shape_handle));
+      }
+      return Status::OK();
+    });
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 0c3c3c4e06..2cc3c48c3c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -73,6 +73,7 @@ py_library(
     deps = [
         ":array_ops",
         ":bitwise_ops",
+        ":boosted_trees_ops",
         ":check_ops",
         ":client",
         ":client_testlib",
@@ -1374,6 +1375,14 @@ tf_gen_op_wrapper_private_py(
     ],
 )
 
+tf_gen_op_wrapper_private_py(
+    name = "boosted_trees_ops_gen",
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/core:boosted_trees_ops_op_lib",
+    ],
+)
+
 tf_gen_op_wrapper_private_py(
     name = "summary_ops_gen",
     visibility = ["//tensorflow:__subpackages__"],
@@ -1623,6 +1632,19 @@ py_library(
     ],
 )
 
+py_library(
+    name = "boosted_trees_ops",
+    srcs = ["ops/boosted_trees_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":boosted_trees_ops_gen",
+        ":framework",
+        ":ops",
+        ":training",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+    ],
+)
+
 py_library(
     name = "sets",
     srcs = [
diff --git a/tensorflow/python/__init__.py b/tensorflow/python/__init__.py
index 3346937904..ab1d01a835 100644
--- a/tensorflow/python/__init__.py
+++ b/tensorflow/python/__init__.py
@@ -98,6 +98,8 @@ from tensorflow.python.summary import summary
 from tensorflow.python.user_ops import user_ops
 from tensorflow.python.util import compat
 
+# Import boosted trees ops to make sure the ops are registered (but unused).
+from tensorflow.python.ops import gen_boosted_trees_ops as _gen_boosted_trees_ops
 
 # Import cudnn rnn ops to make sure their ops are registered.
 from tensorflow.python.ops import gen_cudnn_rnn_ops as _
diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index 1fcff18a3a..f93bc221cc 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -15,6 +15,7 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":baseline",
+        ":boosted_trees",
         ":dnn",
         ":dnn_linear_combined",
         ":estimator",
@@ -239,6 +240,53 @@ py_test(
     ],
 )
 
+py_library(
+    name = "boosted_trees",
+    srcs = ["canned/boosted_trees.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":estimator",
+        ":head",
+        ":model_fn",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:boosted_trees_ops",
+        "//tensorflow/python:data_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:lookup_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:summary",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/feature_column",
+        "//tensorflow/python/ops/losses",
+    ],
+)
+
+py_test(
+    name = "boosted_trees_test",
+    size = "medium",
+    srcs = ["canned/boosted_trees_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":boosted_trees",
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:training",
+        "//tensorflow/python/estimator:numpy_io",
+        "//tensorflow/python/feature_column",
+    ],
+)
+
 py_library(
     name = "dnn",
     srcs = ["canned/dnn.py"],
diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py
new file mode 100644
index 0000000000..a9bbabd598
--- /dev/null
+++ b/tensorflow/python/estimator/canned/boosted_trees.py
@@ -0,0 +1,736 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Estimator classes for BoostedTrees."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+
+from tensorflow.python.estimator import estimator
+from tensorflow.python.estimator import model_fn
+from tensorflow.python.estimator.canned import head as head_lib
+from tensorflow.python.feature_column import feature_column as feature_column_lib
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import boosted_trees_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import data_flow_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import lookup_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops.losses import losses
+from tensorflow.python.summary import summary
+from tensorflow.python.training import session_run_hook
+from tensorflow.python.training import training_util
+from tensorflow.python.util.tf_export import tf_export
+
+TreeHParams = collections.namedtuple(
+    'TreeHParams',
+    ['n_trees', 'max_depth', 'learning_rate', 'l1', 'l2', 'tree_complexity'])
+
+_HOLD_FOR_MULTI_CLASS_SUPPORT = object()
+_HOLD_FOR_MULTI_DIM_SUPPORT = object()
+
+
+def _get_transformed_features(features, feature_columns):
+  """Gets the transformed features from features/feature_columns pair.
+
+  Args:
+    features: a dicionary of name to Tensor.
+    feature_columns: a list/set of tf.feature_column.
+
+  Returns:
+    result_features: a list of the transformed features, sorted by the name.
+    num_buckets: the maximum number of buckets across bucketized_columns.
+
+  Raises:
+    ValueError: when unsupported features/columns are tried.
+  """
+  num_buckets = 1
+  # pylint:disable=protected-access
+  for fc in feature_columns:
+    if isinstance(fc, feature_column_lib._BucketizedColumn):
+      # N boundaries creates (N+1) buckets.
+      num_buckets = max(num_buckets, len(fc.boundaries) + 1)
+    else:
+      raise ValueError('For now, only bucketized_column is supported but '
+                       'got: {}'.format(fc))
+  transformed = feature_column_lib._transform_features(features,
+                                                       feature_columns)
+  # pylint:enable=protected-access
+  result_features = []
+  for column in sorted(transformed, key=lambda tc: tc.name):
+    source_name = column.source_column.name
+    squeezed_tensor = array_ops.squeeze(transformed[column], axis=1)
+    if len(squeezed_tensor.shape) > 1:
+      raise ValueError('For now, only supports features equivalent to rank 1 '
+                       'but column `{}` got: {}'.format(
+                           source_name, features[source_name].shape))
+    result_features.append(squeezed_tensor)
+  return result_features, num_buckets
+
+
+def _keep_as_local_variable(tensor, name=None):
+  """Stores a tensor as a local Variable for faster read."""
+  return variable_scope.variable(
+      initial_value=tensor,
+      trainable=False,
+      collections=[ops.GraphKeys.LOCAL_VARIABLES],
+      validate_shape=False,
+      name=name)
+
+
+class _CacheTrainingStatesUsingHashTable(object):
+  """Caching logits, etc. using MutableHashTable."""
+
+  def __init__(self, example_ids, logits_dimension):
+    """Creates a cache with the given configuration.
+
+    It maintains a MutableDenseHashTable for all values.
+    The API lookup() and insert() would have those specs,
+      tree_ids: shape=[batch_size], dtype=int32
+      node_ids: shape=[batch_size], dtype=int32
+      logits: shape=[batch_size, logits_dimension], dtype=float32
+    However in the MutableDenseHashTable, ids are bitcasted into float32 and
+    all values are concatenated as a single tensor (of float32).
+
+    Hence conversion happens internally before inserting to the HashTable and
+    after lookup from it.
+
+    Args:
+      example_ids: a Rank 1 tensor to be used as a key of the cache.
+      logits_dimension: a constant (int) for the dimension of logits.
+
+    Raises:
+      ValueError: if example_ids is other than int64 or string.
+    """
+    if dtypes.as_dtype(dtypes.int64).is_compatible_with(example_ids.dtype):
+      empty_key = -1 << 62
+    elif dtypes.as_dtype(dtypes.string).is_compatible_with(example_ids.dtype):
+      empty_key = ''
+    else:
+      raise ValueError('Unsupported example_id_feature dtype %s.',
+                       example_ids.dtype)
+    # Cache holds latest <tree_id, node_id, logits> for each example.
+    # tree_id and node_id are both int32 but logits is a float32.
+    # To reduce the overhead, we store all of them together as float32 and
+    # bitcast the ids to int32.
+    self._table_ref = lookup_ops.mutable_dense_hash_table_v2(
+        empty_key=empty_key, value_dtype=dtypes.float32, value_shape=[3])
+    self._example_ids = example_ids
+    self._logits_dimension = logits_dimension
+
+  def lookup(self):
+    """Returns cached_tree_ids, cached_node_ids, cached_logits."""
+    cached_tree_ids, cached_node_ids, cached_logits = array_ops.split(
+        lookup_ops.lookup_table_find_v2(
+            self._table_ref, self._example_ids, default_value=[0.0, 0.0, 0.0]),
+        [1, 1, self._logits_dimension],
+        axis=1)
+    cached_tree_ids = array_ops.squeeze(
+        array_ops.bitcast(cached_tree_ids, dtypes.int32))
+    cached_node_ids = array_ops.squeeze(
+        array_ops.bitcast(cached_node_ids, dtypes.int32))
+    return (cached_tree_ids, cached_node_ids, cached_logits)
+
+  def insert(self, tree_ids, node_ids, logits):
+    """Inserts values and returns the op."""
+    insert_op = lookup_ops.lookup_table_insert_v2(
+        self._table_ref, self._example_ids,
+        array_ops.concat(
+            [
+                array_ops.expand_dims(
+                    array_ops.bitcast(tree_ids, dtypes.float32), 1),
+                array_ops.expand_dims(
+                    array_ops.bitcast(node_ids, dtypes.float32), 1),
+                logits,
+            ],
+            axis=1,
+            name='value_concat_for_cache_insert'))
+    return insert_op
+
+
+class _CacheTrainingStatesUsingVariables(object):
+  """Caching logits, etc. using Variables."""
+
+  def __init__(self, batch_size, logits_dimension):
+    """Creates a cache with the given configuration.
+
+    It maintains three variables, tree_ids, node_ids, logits, for caching.
+      tree_ids: shape=[batch_size], dtype=int32
+      node_ids: shape=[batch_size], dtype=int32
+      logits: shape=[batch_size, logits_dimension], dtype=float32
+
+    Note, this can be used only with in-memory data setting.
+
+    Args:
+      batch_size: `int`, the size of the cache.
+      logits_dimension: a constant (int) for the dimension of logits.
+    """
+    self._logits_dimension = logits_dimension
+    self._tree_ids = _keep_as_local_variable(
+        array_ops.zeros([batch_size], dtype=dtypes.int32),
+        name='tree_ids_cache')
+    self._node_ids = _keep_as_local_variable(
+        array_ops.zeros([batch_size], dtype=dtypes.int32),
+        name='node_ids_cache')
+    self._logits = _keep_as_local_variable(
+        array_ops.zeros([batch_size, logits_dimension], dtype=dtypes.float32),
+        name='logits_cache')
+
+  def lookup(self):
+    """Returns cached_tree_ids, cached_node_ids, cached_logits."""
+    return (self._tree_ids, self._node_ids, self._logits)
+
+  def insert(self, tree_ids, node_ids, logits):
+    """Inserts values and returns the op."""
+    return control_flow_ops.group(
+        [
+            self._tree_ids.assign(tree_ids),
+            self._node_ids.assign(node_ids),
+            self._logits.assign(logits)
+        ],
+        name='cache_insert')
+
+
+class StopAtAttemptsHook(session_run_hook.SessionRunHook):
+  """Hook that requests stop at the number of trees."""
+
+  def __init__(self, num_finalized_trees_tensor, num_attempted_layers_tensor,
+               max_trees, max_depth):
+    self._num_finalized_trees_tensor = num_finalized_trees_tensor
+    self._num_attempted_layers_tensor = num_attempted_layers_tensor
+    self._max_trees = max_trees
+    self._max_depth = max_depth
+
+  def before_run(self, run_context):
+    return session_run_hook.SessionRunArgs(
+        [self._num_finalized_trees_tensor, self._num_attempted_layers_tensor])
+
+  def after_run(self, run_context, run_values):
+    num_finalized_trees, num_attempted_layers = run_values.results
+    if (num_finalized_trees >= self._max_trees or
+        1.0 * num_attempted_layers / self._max_depth > 2 * self._max_trees):
+      run_context.request_stop()
+
+
+class StopAtNumTreesHook(session_run_hook.SessionRunHook):
+  """Hook that requests stop at the number of trees."""
+
+  def __init__(self, num_trees_tensor, max_trees):
+    self._num_trees_tensor = num_trees_tensor
+    self._max_trees = max_trees
+
+  def before_run(self, run_context):
+    return session_run_hook.SessionRunArgs(self._num_trees_tensor)
+
+  def after_run(self, run_context, run_values):
+    num_trees = run_values.results
+    if num_trees > self._max_trees:
+      run_context.request_stop()
+
+
+def _bt_model_fn(
+    features,
+    labels,
+    mode,
+    head,
+    feature_columns,
+    tree_hparams,
+    n_batches_per_layer,
+    config,
+    closed_form_grad_and_hess_fn=None,
+    example_id_column_name=None,
+    # TODO(youngheek): replace this later using other options.
+    train_in_memory=False,
+    name='TreeEnsembleModel'):
+  """Gradient Boosted Decision Tree model_fn.
+
+  Args:
+    features: dict of `Tensor`.
+    labels: `Tensor` of shape [batch_size, 1] or [batch_size] labels of
+      dtype `int32` or `int64` in the range `[0, n_classes)`.
+    mode: Defines whether this is training, evaluation or prediction.
+      See `ModeKeys`.
+    head: A `head_lib._Head` instance.
+    feature_columns: Iterable of `feature_column._FeatureColumn` model inputs.
+    tree_hparams: TODO. collections.namedtuple for hyper parameters.
+    n_batches_per_layer: A `Tensor` of `int64`. Each layer is built after at
+      least n_batches_per_layer accumulations.
+    config: `RunConfig` object to configure the runtime settings.
+    closed_form_grad_and_hess_fn: a function that accepts logits and labels
+      and returns gradients and hessians. By default, they are created by
+      tf.gradients() from the loss.
+    example_id_column_name: Name of the feature for a unique ID per example.
+      Currently experimental -- not exposed to public API.
+    train_in_memory: `bool`, when true, it assumes the dataset is in memory,
+      i.e., input_fn should return the entire dataset as a single batch, and
+      also n_batches_per_layer should be set as 1.
+    name: Name to use for the model.
+
+  Returns:
+      An `EstimatorSpec` instance.
+
+  Raises:
+    ValueError: mode or params are invalid, or features has the wrong type.
+  """
+  is_single_machine = (config.num_worker_replicas == 1)
+  if train_in_memory:
+    assert n_batches_per_layer == 1, (
+        'When train_in_memory is enabled, input_fn should return the entire '
+        'dataset as a single batch, and n_batches_per_layer should be set as '
+        '1.')
+  worker_device = control_flow_ops.no_op().device
+  # maximum number of splits possible in the whole tree =2^(D-1)-1
+  # TODO(youngheek): perhaps storage could be optimized by storing stats with
+  # the dimension max_splits_per_layer, instead of max_splits (for the entire
+  # tree).
+  max_splits = (1 << tree_hparams.max_depth) - 1
+  with ops.name_scope(name) as name:
+    # Prepare.
+    global_step = training_util.get_or_create_global_step()
+    input_feature_list, num_buckets = _get_transformed_features(
+        features, feature_columns)
+    if train_in_memory and mode == model_fn.ModeKeys.TRAIN:
+      input_feature_list = [
+          _keep_as_local_variable(feature) for feature in input_feature_list
+      ]
+    num_features = len(input_feature_list)
+
+    cache = None
+    if mode == model_fn.ModeKeys.TRAIN:
+      if train_in_memory and is_single_machine:  # maybe just train_in_memory?
+        batch_size = array_ops.shape(input_feature_list[0])[0]
+        cache = _CacheTrainingStatesUsingVariables(batch_size,
+                                                   head.logits_dimension)
+      elif example_id_column_name:
+        example_ids = features[example_id_column_name]
+        cache = _CacheTrainingStatesUsingHashTable(example_ids,
+                                                   head.logits_dimension)
+
+    # Create Ensemble resources.
+    if is_single_machine:
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(name=name)
+      local_tree_ensemble = tree_ensemble
+      ensemble_reload = control_flow_ops.no_op()
+    else:
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(name=name)
+      with ops.device(worker_device):
+        local_tree_ensemble = boosted_trees_ops.TreeEnsemble(
+            name=name + '_local', is_local=True)
+      # TODO(soroush): Do partial updates if this becomes a bottleneck.
+      ensemble_reload = local_tree_ensemble.deserialize(
+          *tree_ensemble.serialize())
+
+    # Create logits.
+    if mode != model_fn.ModeKeys.TRAIN:
+      logits = boosted_trees_ops.predict(
+          tree_ensemble_handle=local_tree_ensemble.resource_handle,
+          bucketized_features=input_feature_list,
+          logits_dimension=head.logits_dimension,
+          max_depth=tree_hparams.max_depth)
+    else:
+      if cache:
+        cached_tree_ids, cached_node_ids, cached_logits = cache.lookup()
+      else:
+        # Always start from the beginning when no cache is set up.
+        batch_size = array_ops.shape(input_feature_list[0])[0]
+        cached_tree_ids, cached_node_ids, cached_logits = (
+            array_ops.zeros([batch_size], dtype=dtypes.int32),
+            array_ops.zeros([batch_size], dtype=dtypes.int32),
+            array_ops.zeros(
+                [batch_size, head.logits_dimension], dtype=dtypes.float32))
+      with ops.control_dependencies([ensemble_reload]):
+        (stamp_token, num_trees, num_finalized_trees,
+         num_attempted_layers) = local_tree_ensemble.get_states()
+        summary.scalar('ensemble/num_trees', num_trees)
+        summary.scalar('ensemble/num_finalized_trees', num_finalized_trees)
+        summary.scalar('ensemble/num_attempted_layers', num_attempted_layers)
+
+        partial_logits, tree_ids, node_ids = boosted_trees_ops.training_predict(
+            tree_ensemble_handle=local_tree_ensemble.resource_handle,
+            cached_tree_ids=cached_tree_ids,
+            cached_node_ids=cached_node_ids,
+            bucketized_features=input_feature_list,
+            logits_dimension=head.logits_dimension,
+            max_depth=tree_hparams.max_depth)
+      logits = cached_logits + partial_logits
+
+    # Create training graph.
+    def _train_op_fn(loss):
+      """Run one training iteration."""
+      train_op = []
+      if cache:
+        train_op.append(cache.insert(tree_ids, node_ids, logits))
+      if closed_form_grad_and_hess_fn:
+        gradients, hessians = closed_form_grad_and_hess_fn(logits, labels)
+      else:
+        gradients = gradients_impl.gradients(loss, logits, name='Gradients')[0]
+        hessians = gradients_impl.gradients(
+            gradients, logits, name='Hessians')[0]
+      stats_summary_list = [
+          array_ops.squeeze(
+              boosted_trees_ops.make_stats_summary(
+                  node_ids=node_ids,
+                  gradients=gradients,
+                  hessians=hessians,
+                  bucketized_features_list=[input_feature_list[f]],
+                  max_splits=max_splits,
+                  num_buckets=num_buckets),
+              axis=0) for f in range(num_features)
+      ]
+
+      def grow_tree_from_stats_summaries(stats_summary_list):
+        """Updates ensemble based on the best gains from stats summaries."""
+        (node_ids_per_feature, gains_list, thresholds_list,
+         left_node_contribs_list, right_node_contribs_list) = (
+             boosted_trees_ops.calculate_best_gains_per_feature(
+                 node_id_range=array_ops.stack([
+                     math_ops.reduce_min(node_ids),
+                     math_ops.reduce_max(node_ids)
+                 ]),
+                 stats_summary_list=stats_summary_list,
+                 l1=tree_hparams.l1,
+                 l2=tree_hparams.l2,
+                 tree_complexity=tree_hparams.tree_complexity,
+                 max_splits=max_splits))
+        grow_op = boosted_trees_ops.update_ensemble(
+            # Confirm if local_tree_ensemble or tree_ensemble should be used.
+            tree_ensemble.resource_handle,
+            feature_ids=math_ops.range(0, num_features, dtype=dtypes.int32),
+            node_ids=node_ids_per_feature,
+            gains=gains_list,
+            thresholds=thresholds_list,
+            left_node_contribs=left_node_contribs_list,
+            right_node_contribs=right_node_contribs_list,
+            learning_rate=tree_hparams.learning_rate,
+            max_depth=tree_hparams.max_depth,
+            pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING)
+        return grow_op
+
+      if train_in_memory and is_single_machine:
+        train_op.append(state_ops.assign_add(global_step, 1))
+        train_op.append(grow_tree_from_stats_summaries(stats_summary_list))
+      else:
+        summary_accumulator = data_flow_ops.ConditionalAccumulator(
+            dtype=dtypes.float32,
+            # The stats consist of gradients and hessians (the last dimension).
+            shape=[num_features, max_splits, num_buckets, 2],
+            shared_name='stats_summary_accumulator')
+        apply_grad = summary_accumulator.apply_grad(
+            array_ops.stack(stats_summary_list, axis=0), stamp_token)
+
+        def grow_tree_from_accumulated_summaries_fn():
+          """Updates the tree with the best layer from accumulated summaries."""
+          # Take out the accumulated summaries from the accumulator and grow.
+          stats_summary_list = array_ops.unstack(
+              summary_accumulator.take_grad(1), axis=0)
+          grow_op = grow_tree_from_stats_summaries(stats_summary_list)
+          return grow_op
+
+        with ops.control_dependencies([apply_grad]):
+          train_op.append(state_ops.assign_add(global_step, 1))
+          if config.is_chief:
+            train_op.append(
+                control_flow_ops.cond(
+                    math_ops.greater_equal(
+                        summary_accumulator.num_accumulated(),
+                        n_batches_per_layer),
+                    grow_tree_from_accumulated_summaries_fn,
+                    control_flow_ops.no_op,
+                    name='wait_until_n_batches_accumulated'))
+
+      return control_flow_ops.group(train_op, name='train_op')
+
+  estimator_spec = head.create_estimator_spec(
+      features=features,
+      mode=mode,
+      labels=labels,
+      train_op_fn=_train_op_fn,
+      logits=logits)
+  if mode == model_fn.ModeKeys.TRAIN:
+    # Add an early stop hook.
+    estimator_spec = estimator_spec._replace(
+        training_hooks=estimator_spec.training_hooks +
+        (StopAtNumTreesHook(num_trees, tree_hparams.n_trees),))
+  return estimator_spec
+
+
+def _create_classification_head(n_classes,
+                                weight_column=None,
+                                label_vocabulary=None):
+  """Creates a classification head. Refer to canned.head for details on args."""
+  # TODO(nponomareva): Support multi-class cases.
+  if n_classes == 2:
+    # pylint: disable=protected-access
+    return head_lib._binary_logistic_head_with_sigmoid_cross_entropy_loss(
+        weight_column=weight_column,
+        label_vocabulary=label_vocabulary,
+        loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE)
+    # pylint: enable=protected-access
+  else:
+    raise ValueError('For now only binary classification is supported.'
+                     'n_classes given as {}'.format(n_classes))
+
+
+def _create_classification_head_and_closed_form(n_classes, weight_column,
+                                                label_vocabulary):
+  """Creates a head for classifier and the closed form gradients/hessians."""
+  head = _create_classification_head(n_classes, weight_column, label_vocabulary)
+  if n_classes == 2 and weight_column is None and label_vocabulary is None:
+    # Use the closed-form gradients/hessians for 2 class.
+    def _grad_and_hess_for_logloss(logits, labels):
+      # TODO(youngheek): add weights handling.
+      predictions = math_ops.reciprocal(math_ops.exp(-logits) + 1.0)
+      normalizer = math_ops.reciprocal(
+          math_ops.cast(array_ops.size(predictions), dtypes.float32))
+      gradients = (predictions - labels) * normalizer
+      hessians = predictions * (1.0 - predictions) * normalizer
+      return gradients, hessians
+
+    closed_form = _grad_and_hess_for_logloss
+  else:
+    closed_form = None
+  return (head, closed_form)
+
+
+def _create_regression_head(label_dimension, weight_column=None):
+  if label_dimension != 1:
+    raise ValueError('For now only 1 dimension regression is supported.'
+                     'label_dimension given as {}'.format(label_dimension))
+  # pylint: disable=protected-access
+  return head_lib._regression_head_with_mean_squared_error_loss(
+      label_dimension=label_dimension,
+      weight_column=weight_column,
+      loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE)
+  # pylint: enable=protected-access
+
+
+@tf_export('estimator.BoostedTreesClassifier')
+class BoostedTreesClassifier(estimator.Estimator):
+  """A Classifier for Tensorflow Boosted Trees models."""
+
+  def __init__(
+      self,
+      feature_columns,
+      n_batches_per_layer,
+      model_dir=None,
+      n_classes=_HOLD_FOR_MULTI_CLASS_SUPPORT,
+      weight_column=None,
+      label_vocabulary=None,
+      n_trees=100,
+      max_depth=6,
+      learning_rate=0.1,
+      l1_regularization=0.,
+      l2_regularization=0.,
+      tree_complexity=0.,
+      config=None):
+    """Initializes a `BoostedTreesClassifier` instance.
+
+    Example:
+
+    ```python
+    bucketized_feature_1 = bucketized_column(
+      numeric_column('feature_1'), BUCKET_BOUNDARIES_1)
+    bucketized_feature_2 = bucketized_column(
+      numeric_column('feature_2'), BUCKET_BOUNDARIES_2)
+
+    classifier = estimator.BoostedTreesClassifier(
+        feature_columns=[bucketized_feature_1, bucketized_feature_2],
+        n_trees=100,
+        ... <some other params>
+    )
+
+    def input_fn_train():
+      ...
+      return dataset
+
+    classifier.train(input_fn=input_fn_train)
+
+    def input_fn_eval():
+      ...
+      return dataset
+
+    metrics = classifier.evaluate(input_fn=input_fn_eval)
+    ```
+
+    Args:
+      feature_columns: An iterable containing all the feature columns used by
+        the model. All items in the set should be instances of classes derived
+        from `FeatureColumn`.
+      n_batches_per_layer: the number of batches to collect statistics per
+        layer.
+      model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+      n_classes: number of label classes. Default is binary classification.
+        Multiclass support is not yet implemented.
+      weight_column: A string or a `_NumericColumn` created by
+        `tf.feature_column.numeric_column` defining feature column representing
+        weights. It is used to downweight or boost examples during training. It
+        will be multiplied by the loss of the example. If it is a string, it is
+        used as a key to fetch weight tensor from the `features`. If it is a
+        `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
+        then weight_column.normalizer_fn is applied on it to get weight tensor.
+      label_vocabulary: A list of strings represents possible label values. If
+        given, labels must be string type and have any value in
+        `label_vocabulary`. If it is not given, that means labels are
+        already encoded as integer or float within [0, 1] for `n_classes=2` and
+        encoded as integer values in {0, 1,..., n_classes-1} for `n_classes`>2 .
+        Also there will be errors if vocabulary is not provided and labels are
+        string.
+      n_trees: number trees to be created.
+      max_depth: maximum depth of the tree to grow.
+      learning_rate: shrinkage parameter to be used when a tree added to the
+        model.
+      l1_regularization: regularization multiplier applied to the absolute
+        weights of the tree leafs.
+      l2_regularization: regularization multiplier applied to the square weights
+        of the tree leafs.
+      tree_complexity: regularization factor to penalize trees with more leaves.
+      config: `RunConfig` object to configure the runtime settings.
+
+    Raises:
+      ValueError: when wrong arguments are given or unsupported functionalities
+         are requested.
+    """
+    # TODO(nponomareva): Support multi-class cases.
+    if n_classes == _HOLD_FOR_MULTI_CLASS_SUPPORT:
+      n_classes = 2
+    head, closed_form = _create_classification_head_and_closed_form(
+        n_classes, weight_column, label_vocabulary=label_vocabulary)
+
+    # HParams for the model.
+    tree_hparams = TreeHParams(
+        n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
+        tree_complexity)
+
+    def _model_fn(features, labels, mode, config):
+      return _bt_model_fn(  # pylint: disable=protected-access
+          features,
+          labels,
+          mode,
+          head,
+          feature_columns,
+          tree_hparams,
+          n_batches_per_layer,
+          config,
+          closed_form_grad_and_hess_fn=closed_form)
+
+    super(BoostedTreesClassifier, self).__init__(
+        model_fn=_model_fn, model_dir=model_dir, config=config)
+
+
+@tf_export('estimator.BoostedTreesRegressor')
+class BoostedTreesRegressor(estimator.Estimator):
+  """A Regressor for Tensorflow Boosted Trees models."""
+
+  def __init__(
+      self,
+      feature_columns,
+      n_batches_per_layer,
+      model_dir=None,
+      label_dimension=_HOLD_FOR_MULTI_DIM_SUPPORT,
+      weight_column=None,
+      n_trees=100,
+      max_depth=6,
+      learning_rate=0.1,
+      l1_regularization=0.,
+      l2_regularization=0.,
+      tree_complexity=0.,
+      config=None):
+    """Initializes a `BoostedTreesRegressor` instance.
+
+    Example:
+
+    ```python
+    bucketized_feature_1 = bucketized_column(
+      numeric_column('feature_1'), BUCKET_BOUNDARIES_1)
+    bucketized_feature_2 = bucketized_column(
+      numeric_column('feature_2'), BUCKET_BOUNDARIES_2)
+
+    regressor = estimator.BoostedTreesRegressor(
+        feature_columns=[bucketized_feature_1, bucketized_feature_2],
+        n_trees=100,
+        ... <some other params>
+    )
+
+    def input_fn_train():
+      ...
+      return dataset
+
+    regressor.train(input_fn=input_fn_train)
+
+    def input_fn_eval():
+      ...
+      return dataset
+
+    metrics = regressor.evaluate(input_fn=input_fn_eval)
+    ```
+
+    Args:
+      feature_columns: An iterable containing all the feature columns used by
+        the model. All items in the set should be instances of classes derived
+        from `FeatureColumn`.
+      n_batches_per_layer: the number of batches to collect statistics per
+        layer.
+      model_dir: Directory to save model parameters, graph and etc. This can
+        also be used to load checkpoints from the directory into a estimator
+        to continue training a previously saved model.
+      label_dimension: Number of regression targets per example.
+        Multi-dimensional support is not yet implemented.
+      weight_column: A string or a `_NumericColumn` created by
+        `tf.feature_column.numeric_column` defining feature column representing
+        weights. It is used to downweight or boost examples during training. It
+        will be multiplied by the loss of the example. If it is a string, it is
+        used as a key to fetch weight tensor from the `features`. If it is a
+        `_NumericColumn`, raw tensor is fetched by key `weight_column.key`,
+        then weight_column.normalizer_fn is applied on it to get weight tensor.
+      n_trees: number trees to be created.
+      max_depth: maximum depth of the tree to grow.
+      learning_rate: shrinkage parameter to be used when a tree added to the
+        model.
+      l1_regularization: regularization multiplier applied to the absolute
+        weights of the tree leafs.
+      l2_regularization: regularization multiplier applied to the square weights
+        of the tree leafs.
+      tree_complexity: regularization factor to penalize trees with more leaves.
+      config: `RunConfig` object to configure the runtime settings.
+
+    Raises:
+      ValueError: when wrong arguments are given or unsupported functionalities
+         are requested.
+    """
+    # TODO(nponomareva): Extend it to multi-dimension cases.
+    if label_dimension == _HOLD_FOR_MULTI_DIM_SUPPORT:
+      label_dimension = 1
+    head = _create_regression_head(label_dimension, weight_column)
+
+    # HParams for the model.
+    tree_hparams = TreeHParams(
+        n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
+        tree_complexity)
+
+    def _model_fn(features, labels, mode, config):
+      return _bt_model_fn(  # pylint: disable=protected-access
+          features, labels, mode, head, feature_columns, tree_hparams,
+          n_batches_per_layer, config)
+
+    super(BoostedTreesRegressor, self).__init__(
+        model_fn=_model_fn, model_dir=model_dir, config=config)
diff --git a/tensorflow/python/estimator/canned/boosted_trees_test.py b/tensorflow/python/estimator/canned/boosted_trees_test.py
new file mode 100644
index 0000000000..9276fbaaa1
--- /dev/null
+++ b/tensorflow/python/estimator/canned/boosted_trees_test.py
@@ -0,0 +1,799 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests boosted_trees estimators and model_fn."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.core.kernels.boosted_trees import boosted_trees_pb2
+from tensorflow.python.estimator import model_fn
+from tensorflow.python.estimator import run_config
+from tensorflow.python.estimator.canned import boosted_trees
+from tensorflow.python.estimator.inputs import numpy_io
+from tensorflow.python.feature_column import feature_column
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import gen_boosted_trees_ops
+from tensorflow.python.ops import resources
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import googletest
+from tensorflow.python.training import checkpoint_utils
+
+NUM_FEATURES = 3
+
+BUCKET_BOUNDARIES = [-2., .5, 12.]  # Boundaries for all the features.
+INPUT_FEATURES = np.array(
+    [
+        [12.5, 1.0, -2.001, -2.0001, -1.999],  # feature_0 quantized:[3,2,0,0,1]
+        [2.0, -3.0, 0.5, 0.0, 0.4995],         # feature_1 quantized:[2,0,2,1,1]
+        [3.0, 20.0, 50.0, -100.0, 102.75],     # feature_2 quantized:[2,3,3,0,3]
+    ],
+    dtype=np.float32)
+CLASSIFICATION_LABELS = [[0.], [1.], [1.], [0.], [0.]]
+REGRESSION_LABELS = [[1.5], [0.3], [0.2], [2.], [5.]]
+FEATURES_DICT = {'f_%d' % i: INPUT_FEATURES[i] for i in range(NUM_FEATURES)}
+
+# EXAMPLE_ID is not exposed to Estimator yet, but supported at model_fn level.
+EXAMPLE_IDS = np.array([0, 1, 2, 3, 4], dtype=np.int64)
+EXAMPLE_ID_COLUMN = '__example_id__'
+
+
+def _make_train_input_fn(is_classification):
+  """Makes train input_fn for classification/regression."""
+
+  def _input_fn():
+    features = dict(FEATURES_DICT)
+    features[EXAMPLE_ID_COLUMN] = constant_op.constant(EXAMPLE_IDS)
+    if is_classification:
+      labels = CLASSIFICATION_LABELS
+    else:
+      labels = REGRESSION_LABELS
+    return features, labels
+
+  return _input_fn
+
+
+class BoostedTreesClassifierTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES)
+        for i in range(NUM_FEATURES)
+    }
+
+  def _assert_checkpoint(self, model_dir, expected_global_step):
+    self.assertEqual(expected_global_step,
+                     checkpoint_utils.load_variable(model_dir,
+                                                    ops.GraphKeys.GLOBAL_STEP))
+
+  def testTrainAndEvaluateBinaryClassifier(self):
+    input_fn = _make_train_input_fn(is_classification=True)
+
+    est = boosted_trees.BoostedTreesClassifier(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=1,
+        max_depth=5)
+
+    # It will stop after 5 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(input_fn, steps=num_steps)
+    self._assert_checkpoint(est.model_dir, 6)
+    eval_res = est.evaluate(input_fn=input_fn, steps=1)
+    self.assertAllClose(eval_res['accuracy'], 1.0)
+
+  def testInferBinaryClassifier(self):
+    train_input_fn = _make_train_input_fn(is_classification=True)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False)
+
+    est = boosted_trees.BoostedTreesClassifier(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=1,
+        max_depth=5)
+
+    # It will stop after 5 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(train_input_fn, steps=num_steps)
+
+    predictions = list(est.predict(input_fn=predict_input_fn))
+    self.assertEquals(5, len(predictions))
+    # All labels are correct.
+    self.assertAllClose([0], predictions[0]['class_ids'])
+    self.assertAllClose([1], predictions[1]['class_ids'])
+    self.assertAllClose([1], predictions[2]['class_ids'])
+    self.assertAllClose([0], predictions[3]['class_ids'])
+    self.assertAllClose([0], predictions[4]['class_ids'])
+
+
+class BoostedTreesRegressionTest(test_util.TensorFlowTestCase):
+
+  def setUp(self):
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES)
+        for i in range(NUM_FEATURES)
+    }
+
+  def _assert_checkpoint(self, model_dir, expected_global_step):
+    self.assertEqual(expected_global_step,
+                     checkpoint_utils.load_variable(model_dir,
+                                                    ops.GraphKeys.GLOBAL_STEP))
+
+  def testTrainAndEvaluateRegressor(self):
+    input_fn = _make_train_input_fn(is_classification=False)
+
+    est = boosted_trees.BoostedTreesRegressor(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=2,
+        max_depth=5)
+
+    # It will stop after 10 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(input_fn, steps=num_steps)
+    self._assert_checkpoint(est.model_dir, 11)
+    eval_res = est.evaluate(input_fn=input_fn, steps=1)
+    self.assertAllClose(eval_res['average_loss'], 0.913176)
+
+  def testInferRegressor(self):
+    train_input_fn = _make_train_input_fn(is_classification=False)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x=FEATURES_DICT, y=None, batch_size=1, num_epochs=1, shuffle=False)
+
+    est = boosted_trees.BoostedTreesRegressor(
+        feature_columns=self._feature_columns,
+        n_batches_per_layer=1,
+        n_trees=1,
+        max_depth=5)
+
+    # It will stop after 5 steps because of the max depth and num trees.
+    num_steps = 100
+    # Train for a few steps, and validate final checkpoint.
+    est.train(train_input_fn, steps=num_steps)
+    self._assert_checkpoint(est.model_dir, 6)
+
+    predictions = list(est.predict(input_fn=predict_input_fn))
+
+    self.assertEquals(5, len(predictions))
+    self.assertAllClose([0.703549], predictions[0]['predictions'])
+    self.assertAllClose([0.266539], predictions[1]['predictions'])
+    self.assertAllClose([0.256479], predictions[2]['predictions'])
+    self.assertAllClose([1.088732], predictions[3]['predictions'])
+    self.assertAllClose([1.901732], predictions[4]['predictions'])
+
+
+class ModelFnTests(test_util.TensorFlowTestCase):
+  """Tests bt_model_fn including unexposed internal functionalities."""
+
+  def setUp(self):
+    self._feature_columns = {
+        feature_column.bucketized_column(
+            feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
+            BUCKET_BOUNDARIES) for i in range(NUM_FEATURES)
+    }
+    self._tree_hparams = boosted_trees.TreeHParams(
+        n_trees=2,
+        max_depth=2,
+        learning_rate=0.1,
+        l1=0.,
+        l2=0.01,
+        tree_complexity=0.)
+
+  def _get_expected_ensembles_for_classification(self):
+    first_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 2
+              threshold: 2
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 0.387675
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.181818
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0625
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+        """
+    second_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 2
+              threshold: 2
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 0.387675
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 3
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 0.0
+              original_leaf {
+                scalar: -0.181818
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 0
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 0.105518
+              original_leaf {
+                scalar: 0.0625
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.348397
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.181818
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.224091
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.056815
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.0
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 0
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+        """
+    third_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 2
+              threshold: 2
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 0.387675
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 3
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 0.0
+              original_leaf {
+                scalar: -0.181818
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 0
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 0.105518
+              original_leaf {
+                scalar: 0.0625
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.348397
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.181818
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.224091
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.056815
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 0
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 0.287131
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.162042
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.086986
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 2
+          num_layers_attempted: 3
+        }
+        """
+    return (first_round, second_round, third_round)
+
+  def _get_expected_ensembles_for_regression(self):
+    first_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 1
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 1.169714
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.241322
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.083951
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+        """
+    second_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 1
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 1.169714
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 1
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 2.673407
+              original_leaf {
+                scalar: 0.241322
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 0
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 0.324102
+              original_leaf {
+                scalar: 0.083951
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.563167
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.247047
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.095273
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.222102
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.0
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 0
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+        """
+    third_round = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 1
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 1.169714
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 1
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 2.673407
+              original_leaf {
+                scalar: 0.241322
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 0
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 0.324102
+              original_leaf {
+                scalar: 0.083951
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.563167
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.247047
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.095273
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.222102
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 0
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 0.981026
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.005166
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.180281
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 2
+          num_layers_attempted: 3
+        }
+        """
+    return (first_round, second_round, third_round)
+
+  def _get_train_op_and_ensemble(self, head, config, is_classification,
+                                 train_in_memory):
+    """Calls bt_model_fn() and returns the train_op and ensemble_serialzed."""
+    features, labels = _make_train_input_fn(is_classification)()
+    estimator_spec = boosted_trees._bt_model_fn(  # pylint:disable=protected-access
+        features=features,
+        labels=labels,
+        mode=model_fn.ModeKeys.TRAIN,
+        head=head,
+        feature_columns=self._feature_columns,
+        tree_hparams=self._tree_hparams,
+        example_id_column_name=EXAMPLE_ID_COLUMN,
+        n_batches_per_layer=1,
+        config=config,
+        train_in_memory=train_in_memory)
+    resources.initialize_resources(resources.shared_resources()).run()
+    variables.global_variables_initializer().run()
+    variables.local_variables_initializer().run()
+
+    # Gets the train_op and serialized proto of the ensemble.
+    shared_resources = resources.shared_resources()
+    self.assertEqual(1, len(shared_resources))
+    train_op = estimator_spec.train_op
+    with ops.control_dependencies([train_op]):
+      _, ensemble_serialized = (
+          gen_boosted_trees_ops.boosted_trees_serialize_ensemble(
+              shared_resources[0].handle))
+    return train_op, ensemble_serialized
+
+  def testTrainClassifierInMemory(self):
+    ops.reset_default_graph()
+    expected_first, expected_second, expected_third = (
+        self._get_expected_ensembles_for_classification())
+    with self.test_session() as sess:
+      # Train with train_in_memory mode.
+      with sess.graph.as_default():
+        train_op, ensemble_serialized = self._get_train_op_and_ensemble(
+            boosted_trees._create_classification_head(n_classes=2),
+            run_config.RunConfig(),
+            is_classification=True,
+            train_in_memory=True)
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      # Validate the trained ensemble.
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_first, ensemble_proto)
+
+      # Run one more time and validate the trained ensemble.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_second, ensemble_proto)
+
+      # Third round training and validation.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_third, ensemble_proto)
+
+  def testTrainClassifierNonInMemory(self):
+    ops.reset_default_graph()
+    expected_first, expected_second, expected_third = (
+        self._get_expected_ensembles_for_classification())
+    with self.test_session() as sess:
+      # Train without train_in_memory mode.
+      with sess.graph.as_default():
+        train_op, ensemble_serialized = self._get_train_op_and_ensemble(
+            boosted_trees._create_classification_head(n_classes=2),
+            run_config.RunConfig(),
+            is_classification=True,
+            train_in_memory=False)
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      # Validate the trained ensemble.
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_first, ensemble_proto)
+
+      # Run one more time and validate the trained ensemble.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_second, ensemble_proto)
+
+      # Third round training and validation.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_third, ensemble_proto)
+
+  def testTrainRegressorInMemory(self):
+    ops.reset_default_graph()
+    expected_first, expected_second, expected_third = (
+        self._get_expected_ensembles_for_regression())
+    with self.test_session() as sess:
+      # Train with train_in_memory mode.
+      with sess.graph.as_default():
+        train_op, ensemble_serialized = self._get_train_op_and_ensemble(
+            boosted_trees._create_regression_head(label_dimension=1),
+            run_config.RunConfig(),
+            is_classification=False,
+            train_in_memory=True)
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      # Validate the trained ensemble.
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_first, ensemble_proto)
+
+      # Run one more time and validate the trained ensemble.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_second, ensemble_proto)
+
+      # Third round training and validation.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_third, ensemble_proto)
+
+  def testTrainRegressorNonInMemory(self):
+    ops.reset_default_graph()
+    expected_first, expected_second, expected_third = (
+        self._get_expected_ensembles_for_regression())
+    with self.test_session() as sess:
+      # Train without train_in_memory mode.
+      with sess.graph.as_default():
+        train_op, ensemble_serialized = self._get_train_op_and_ensemble(
+            boosted_trees._create_regression_head(label_dimension=1),
+            run_config.RunConfig(),
+            is_classification=False,
+            train_in_memory=False)
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      # Validate the trained ensemble.
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_first, ensemble_proto)
+
+      # Run one more time and validate the trained ensemble.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_second, ensemble_proto)
+
+      # Third round training and validation.
+      _, serialized = sess.run([train_op, ensemble_serialized])
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      ensemble_proto.ParseFromString(serialized)
+      self.assertProtoEquals(expected_third, ensemble_proto)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/estimator/estimator_lib.py b/tensorflow/python/estimator/estimator_lib.py
index be8930b3cb..60c59cbc18 100644
--- a/tensorflow/python/estimator/estimator_lib.py
+++ b/tensorflow/python/estimator/estimator_lib.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 # pylint: disable=unused-import,line-too-long,wildcard-import
 from tensorflow.python.estimator.canned.baseline import BaselineClassifier
 from tensorflow.python.estimator.canned.baseline import BaselineRegressor
+from tensorflow.python.estimator.canned.boosted_trees import BoostedTreesClassifier
+from tensorflow.python.estimator.canned.boosted_trees import BoostedTreesRegressor
 from tensorflow.python.estimator.canned.dnn import DNNClassifier
 from tensorflow.python.estimator.canned.dnn import DNNRegressor
 from tensorflow.python.estimator.canned.dnn_linear_combined import DNNLinearCombinedClassifier
@@ -52,6 +54,8 @@ _allowed_symbols = [
     # Canned Estimators
     'BaselineClassifier',
     'BaselineRegressor',
+    'BoostedTreesClassifier',
+    'BoostedTreesRegressor',
     'DNNClassifier',
     'DNNRegressor',
     'DNNLinearCombinedClassifier',
diff --git a/tensorflow/python/kernel_tests/boosted_trees/BUILD b/tensorflow/python/kernel_tests/boosted_trees/BUILD
new file mode 100644
index 0000000000..30e6289420
--- /dev/null
+++ b/tensorflow/python/kernel_tests/boosted_trees/BUILD
@@ -0,0 +1,76 @@
+# Description:
+#   Kernel tests for Boosted Trees.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+load("//tensorflow:tensorflow.bzl", "tf_py_test")
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+tf_py_test(
+    name = "resource_ops_test",
+    size = "small",
+    srcs = ["resource_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+        "//tensorflow/python:boosted_trees_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variables",
+    ],
+)
+
+tf_py_test(
+    name = "prediction_ops_test",
+    size = "small",
+    srcs = ["prediction_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:boosted_trees_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:resources",
+    ],
+)
+
+tf_py_test(
+    name = "stats_ops_test",
+    size = "small",
+    srcs = ["stats_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/python:boosted_trees_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+    ],
+)
+
+tf_py_test(
+    name = "training_ops_test",
+    size = "small",
+    srcs = ["training_ops_test.py"],
+    additional_deps = [
+        "//tensorflow/core/kernels/boosted_trees:boosted_trees_proto_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:boosted_trees_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:resources",
+    ],
+)
diff --git a/tensorflow/python/kernel_tests/boosted_trees/__init__.py b/tensorflow/python/kernel_tests/boosted_trees/__init__.py
new file mode 100644
index 0000000000..e69de29bb2
diff --git a/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
new file mode 100644
index 0000000000..d132f15e51
--- /dev/null
+++ b/tensorflow/python/kernel_tests/boosted_trees/prediction_ops_test.py
@@ -0,0 +1,926 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests boosted_trees prediction kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from google.protobuf import text_format
+from tensorflow.core.kernels.boosted_trees import boosted_trees_pb2
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import boosted_trees_ops
+from tensorflow.python.ops import resources
+from tensorflow.python.platform import googletest
+
+
+class TrainingPredictionOpsTest(test_util.TensorFlowTestCase):
+  """Tests prediction ops for training."""
+
+  def testCachedPredictionOnEmptyEnsemble(self):
+    """Tests that prediction on a dummy ensemble does not fail."""
+    with self.test_session() as session:
+      # Create a dummy ensemble.
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto='')
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # No previous cached values.
+      cached_tree_ids = [0, 0]
+      cached_node_ids = [0, 0]
+
+      # We have two features: 0 and 1. Values don't matter here on a dummy
+      # ensemble.
+      feature_0_values = [67, 5]
+      feature_1_values = [9, 17]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=2,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # Nothing changed.
+      self.assertAllClose(cached_tree_ids, new_tree_ids)
+      self.assertAllClose(cached_node_ids, new_node_ids)
+      self.assertAllClose([[0], [0]], logits_updates)
+
+  def testNoCachedPredictionButTreeExists(self):
+    """Tests that predictions are updated once trees are added."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 15
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 1.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 8.79
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_metadata {
+          is_finalized: true
+          num_layers_grown: 1
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Two examples, none were cached before.
+      cached_tree_ids = [0, 0]
+      cached_node_ids = [0, 0]
+
+      feature_0_values = [67, 5]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=2,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # We are in the first tree.
+      self.assertAllClose([0, 0], new_tree_ids)
+      self.assertAllClose([2, 1], new_node_ids)
+      self.assertAllClose([[0.1 * 8.79], [0.1 * 1.14]], logits_updates)
+
+  def testCachedPredictionIsCurrent(self):
+    """Tests that prediction based on previous node in the tree works."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 15
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+              original_leaf {
+                scalar: -2
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 1.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 8.79
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_metadata {
+          is_finalized: true
+          num_layers_grown: 2
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Two examples, one was cached in node 1 first, another in node 0.
+      cached_tree_ids = [0, 0]
+      cached_node_ids = [1, 2]
+
+      # We have two features: 0 and 1. Values don't matter because trees didn't
+      # change.
+      feature_0_values = [67, 5]
+      feature_1_values = [9, 17]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=4,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # Nothing changed.
+      self.assertAllClose(cached_tree_ids, new_tree_ids)
+      self.assertAllClose(cached_node_ids, new_node_ids)
+      self.assertAllClose([[0], [0]], logits_updates)
+
+  def testCachedPredictionFromTheSameTree(self):
+    """Tests that prediction based on previous node in the tree works."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 15
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+              original_leaf {
+                scalar: -2
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 7
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 1.4
+              original_leaf {
+                scalar: 7.14
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 7
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 2.7
+              original_leaf {
+                scalar: -4.375
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 1.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 8.79
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -5.875
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -2.075
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_metadata {
+          is_finalized: true
+          num_layers_grown: 2
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Two examples, one was cached in node 1 first, another in node 0.
+      cached_tree_ids = [0, 0]
+      cached_node_ids = [1, 0]
+
+      # We have two features: 0 and 1.
+      feature_0_values = [67, 5]
+      feature_1_values = [9, 17]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=4,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # We are still in the same tree.
+      self.assertAllClose([0, 0], new_tree_ids)
+      # When using the full tree, the first example will end up in node 4,
+      # the second in node 5.
+      self.assertAllClose([4, 5], new_node_ids)
+      # Full predictions for each instance would be 8.79 and -5.875,
+      # so an update from the previous cached values lr*(7.14 and -2) would be
+      # 1.65 and -3.875, and then multiply them by 0.1 (lr)
+      self.assertAllClose([[0.1 * 1.65], [0.1 * -3.875]], logits_updates)
+
+  def testCachedPredictionFromPreviousTree(self):
+    """Tests the predictions work when we have cache from previous trees."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 28
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 1.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 8.79
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 26
+              left_id: 1
+              right_id: 2
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 50
+              left_id: 3
+              right_id: 4
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 5
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 6
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 34
+              left_id: 1
+              right_id: 2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -7.0
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 5.0
+            }
+          }
+        }
+        tree_metadata {
+          is_finalized: true
+        }
+        tree_metadata {
+          is_finalized: true
+        }
+        tree_metadata {
+          is_finalized: false
+        }
+        tree_weights: 0.1
+        tree_weights: 0.1
+        tree_weights: 0.1
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Two examples, one was cached in node 1 first, another in node 2.
+      cached_tree_ids = [0, 0]
+      cached_node_ids = [1, 0]
+
+      # We have two features: 0 and 1.
+      feature_0_values = [36, 32]
+      feature_1_values = [11, 27]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=2,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+      # Example 1 will get to node 3 in tree 1 and node 2 of tree 2
+      # Example 2 will get to node 2 in tree 1 and node 1 of tree 2
+
+      # We are in the last tree.
+      self.assertAllClose([2, 2], new_tree_ids)
+      # When using the full tree, the first example will end up in node 4,
+      # the second in node 5.
+      self.assertAllClose([2, 1], new_node_ids)
+      # Example 1: tree 0: 8.79, tree 1: 5.0, tree 2: 5.0 = >
+      #            change = 0.1*(5.0+5.0)
+      # Example 2: tree 0: 1.14, tree 1: 7.0, tree 2: -7 = >
+      #            change= 0.1(1.14+7.0-7.0)
+      self.assertAllClose([[1], [0.114]], logits_updates)
+
+  def testCachedPredictionFromTheSameTreeWithPostPrunedNodes(self):
+    """Tests that prediction based on previous node in the tree works."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id:0
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.01
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 5
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 0.5
+              original_leaf {
+                scalar: 0.0143
+               }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0553
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0783
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 3
+          is_finalized: true
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 2
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.07
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.083
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 3
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 4
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.22
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.57
+          }
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 3
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble.
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      cached_tree_ids = [0, 0, 0, 0, 0, 0]
+      # Leaves 3,4, 7 and 8 got deleted during post-pruning, leaves 5 and 6
+      # changed the ids to 3 and 4 respectively.
+      cached_node_ids = [3, 4, 5, 6, 7, 8]
+
+      # We have two features: 0 and 1.
+      feature_0_values = [12, 17, 35, 36, 23, 11]
+      feature_1_values = [12, 12, 17, 18, 123, 24]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=3,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # We are still in the same tree.
+      self.assertAllClose([0, 0, 0, 0, 0, 0], new_tree_ids)
+      # Examples from leaves 3,4,7,8 should be in leaf 1, examples from leaf 5
+      # and 6 in leaf 3 and 4.
+      self.assertAllClose([1, 1, 3, 4, 1, 1], new_node_ids)
+
+      cached_values = [[0.08], [0.093], [0.0553], [0.0783], [0.15 + 0.08],
+                       [0.5 + 0.08]]
+      self.assertAllClose([[0.01], [0.01], [0.0553], [0.0783], [0.01], [0.01]],
+                          logits_updates + cached_values)
+
+  def testCachedPredictionFromThePreviousTreeWithPostPrunedNodes(self):
+    """Tests that prediction based on previous node in the tree works."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id:0
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.01
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 5
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 0.5
+              original_leaf {
+                scalar: 0.0143
+               }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0553
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0783
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.55
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 3
+          is_finalized: true
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 2
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.07
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.083
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 3
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 4
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.22
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.57
+          }
+        }
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 2
+          num_layers_attempted: 4
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble.
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      cached_tree_ids = [0, 0, 0, 0, 0, 0]
+      # Leaves 3,4, 7 and 8 got deleted during post-pruning, leaves 5 and 6
+      # changed the ids to 3 and 4 respectively.
+      cached_node_ids = [3, 4, 5, 6, 7, 8]
+
+      # We have two features: 0 and 1.
+      feature_0_values = [12, 17, 35, 36, 23, 11]
+      feature_1_values = [12, 12, 17, 18, 123, 24]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=3,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # We are in the last tree.
+      self.assertAllClose([1, 1, 1, 1, 1, 1], new_tree_ids)
+      # Examples from leaves 3,4,7,8 should be in leaf 1, examples from leaf 5
+      # and 6 in leaf 3 and 4 in tree 0. For tree 1, all of the examples are in
+      # the root node.
+      self.assertAllClose([0, 0, 0, 0, 0, 0], new_node_ids)
+
+      cached_values = [[0.08], [0.093], [0.0553], [0.0783], [0.15 + 0.08],
+                       [0.5 + 0.08]]
+      root = 0.55
+      self.assertAllClose([[root + 0.01], [root + 0.01], [root + 0.0553],
+                           [root + 0.0783], [root + 0.01], [root + 0.01]],
+                          logits_updates + cached_values)
+
+  def testCachedPredictionTheWholeTreeWasPruned(self):
+    """Tests that prediction based on previous node in the tree works."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.00
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: true
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: -6.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: 5.0
+          }
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble.
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      cached_tree_ids = [
+          0,
+          0,
+      ]
+      # The predictions were cached in 1 and 2, both were pruned to the root.
+      cached_node_ids = [1, 2]
+
+      # We have two features: 0 and 1.These are not going to be used anywhere.
+      feature_0_values = [12, 17]
+      feature_1_values = [12, 12]
+
+      # Grow tree ensemble.
+      predict_op = boosted_trees_ops.training_predict(
+          tree_ensemble_handle,
+          max_depth=1,
+          cached_tree_ids=cached_tree_ids,
+          cached_node_ids=cached_node_ids,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1)
+
+      logits_updates, new_tree_ids, new_node_ids = session.run(predict_op)
+
+      # We are in the last tree.
+      self.assertAllClose([0, 0], new_tree_ids)
+      self.assertAllClose([0, 0], new_node_ids)
+
+      self.assertAllClose([[-6.0], [5.0]], logits_updates)
+
+
+class PredictionOpsTest(test_util.TensorFlowTestCase):
+  """Tests prediction ops for inference."""
+
+  def testPredictionMultipleTree(self):
+    """Tests the predictions work when we have multiple trees."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 28
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 1.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 8.79
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 26
+              left_id: 1
+              right_id: 2
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 50
+              left_id: 3
+              right_id: 4
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.0
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 5.0
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 6.0
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 34
+              left_id: 1
+              right_id: 2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -7.0
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 5.0
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_weights: 0.2
+        tree_weights: 1.0
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      feature_0_values = [36, 32]
+      feature_1_values = [11, 27]
+
+      # Example 1: tree 0: 1.14, tree 1: 5.0, tree 2: 5.0 = >
+      #            logit = 0.1*5.0+0.2*5.0+1*5
+      # Example 2: tree 0: 1.14, tree 1: 7.0, tree 2: -7 = >
+      #            logit= 0.1*1.14+0.2*7.0-1*7.0
+      expected_logits = [[6.114], [-5.486]]
+
+      # Do with parallelization, e.g. EVAL
+      predict_op = boosted_trees_ops.predict(
+          tree_ensemble_handle,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1,
+          max_depth=2)
+
+      logits = session.run(predict_op)
+      self.assertAllClose(expected_logits, logits)
+
+      # Do without parallelization, e.g. INFER - the result is the same
+      predict_op = boosted_trees_ops.predict(
+          tree_ensemble_handle,
+          bucketized_features=[feature_0_values, feature_1_values],
+          logits_dimension=1,
+          max_depth=2)
+
+      logits = session.run(predict_op)
+      self.assertAllClose(expected_logits, logits)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
new file mode 100644
index 0000000000..a223241e89
--- /dev/null
+++ b/tensorflow/python/kernel_tests/boosted_trees/resource_ops_test.py
@@ -0,0 +1,228 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for boosted_trees resource kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from google.protobuf import text_format
+
+from tensorflow.core.kernels.boosted_trees import boosted_trees_pb2
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import boosted_trees_ops
+from tensorflow.python.ops import resources
+from tensorflow.python.platform import googletest
+
+
+class ResourceOpsTest(test_util.TensorFlowTestCase):
+  """Tests resource_ops."""
+
+  def testCreate(self):
+    with self.test_session():
+      ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
+      resources.initialize_resources(resources.shared_resources()).run()
+      stamp_token = ensemble.get_stamp_token()
+      self.assertEqual(0, stamp_token.eval())
+      (_, num_trees, num_finalized_trees,
+       num_attempted_layers) = ensemble.get_states()
+      self.assertEqual(0, num_trees.eval())
+      self.assertEqual(0, num_finalized_trees.eval())
+      self.assertEqual(0, num_attempted_layers.eval())
+
+  def testCreateWithProto(self):
+    with self.test_session():
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            bucketized_split {
+              threshold: 21
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 1.4
+              original_leaf {
+                scalar: 7.14
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 7
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 2.7
+              original_leaf {
+                scalar: -4.375
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 6.54
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.305
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.525
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.145
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 75
+              threshold: 21
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -1.4
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.6
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.165
+            }
+          }
+        }
+        tree_weights: 0.15
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 2
+          num_layers_attempted: 6
+        }
+      """, ensemble_proto)
+      ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble',
+          stamp_token=7,
+          serialized_proto=ensemble_proto.SerializeToString())
+      resources.initialize_resources(resources.shared_resources()).run()
+      (stamp_token, num_trees, num_finalized_trees,
+       num_attempted_layers) = ensemble.get_states()
+      self.assertEqual(7, stamp_token.eval())
+      self.assertEqual(2, num_trees.eval())
+      self.assertEqual(1, num_finalized_trees.eval())
+      self.assertEqual(6, num_attempted_layers.eval())
+
+  def testSerializeDeserialize(self):
+    with self.test_session():
+      # Initialize.
+      ensemble = boosted_trees_ops.TreeEnsemble('ensemble', stamp_token=5)
+      resources.initialize_resources(resources.shared_resources()).run()
+      (stamp_token, num_trees, num_finalized_trees,
+       num_attempted_layers) = ensemble.get_states()
+      self.assertEqual(5, stamp_token.eval())
+      self.assertEqual(0, num_trees.eval())
+      self.assertEqual(0, num_finalized_trees.eval())
+      self.assertEqual(0, num_attempted_layers.eval())
+
+      # Deserialize.
+      ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 75
+              threshold: 21
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -1.4
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.6
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.165
+            }
+          }
+        }
+        tree_weights: 0.5
+        tree_metadata {
+          num_layers_grown: 4  # it's fake intentionally.
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 5
+        }
+      """, ensemble_proto)
+      with ops.control_dependencies([
+          ensemble.deserialize(
+              stamp_token=3,
+              serialized_proto=ensemble_proto.SerializeToString())
+      ]):
+        (stamp_token, num_trees, num_finalized_trees,
+         num_attempted_layers) = ensemble.get_states()
+      self.assertEqual(3, stamp_token.eval())
+      self.assertEqual(1, num_trees.eval())
+      # This reads from metadata, not really counting the layers.
+      self.assertEqual(5, num_attempted_layers.eval())
+      self.assertEqual(0, num_finalized_trees.eval())
+
+      # Serialize.
+      new_ensemble_proto = boosted_trees_pb2.TreeEnsemble()
+      new_stamp_token, new_serialized = ensemble.serialize()
+      self.assertEqual(3, new_stamp_token.eval())
+      new_ensemble_proto.ParseFromString(new_serialized.eval())
+      self.assertProtoEquals(ensemble_proto, new_ensemble_proto)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
new file mode 100644
index 0000000000..a54cc43517
--- /dev/null
+++ b/tensorflow/python/kernel_tests/boosted_trees/stats_ops_test.py
@@ -0,0 +1,289 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for boosted_trees stats kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import boosted_trees_ops
+from tensorflow.python.platform import googletest
+
+
+class StatsOpsTest(test_util.TensorFlowTestCase):
+  """Tests stats_ops."""
+
+  def testCalculateBestGainsWithoutRegularization(self):
+    """Testing Gain calculation without any regularization."""
+    with self.test_session() as sess:
+      max_splits = 7
+      node_id_range = [1, 2]  # node 1 through 2 will be processed.
+      stats_summary_list = [
+          [
+              [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+              [[0., 0.], [-.33, .58], [0., 0.], [.3, .4]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 0
+          [
+              [[0., 0.], [0., 0.], [.08, .09], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.3, .5], [-.05, .06], [.06, .07]],  # node 1
+              [[.1, .1], [.2, .3], [-.4, .5], [.07, .08]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 1
+      ]  # num_features * shape=[max_splits, num_buckets, 2]
+
+      (node_ids_list, gains_list, thresholds_list, left_node_contribs_list,
+       right_node_contribs_list
+      ) = boosted_trees_ops.calculate_best_gains_per_feature(
+          node_id_range,
+          stats_summary_list,
+          l1=0.0,
+          l2=0.0,
+          tree_complexity=0.0,
+          max_splits=max_splits)
+
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllClose([[0.004775, 0.41184], [0.02823, 0.41184]],
+                          sess.run(gains_list))
+      self.assertAllEqual([[1, 1], [1, 1]], sess.run(thresholds_list))
+      # The left node contrib will be later added to the previous node value to
+      # make the left node value, and the same for right node contrib.
+      self.assertAllClose([[[-.416667], [.568966]], [[-.6], [-.75]]],
+                          sess.run(left_node_contribs_list))
+      self.assertAllClose([[[-.592593], [-.75]], [[-.076923], [.568966]]],
+                          sess.run(right_node_contribs_list))
+
+  def testCalculateBestGainsWithL2(self):
+    """Testing Gain calculation with L2."""
+    with self.test_session() as sess:
+      max_splits = 7
+      node_id_range = [1, 2]  # node 1 through 2 will be processed.
+      stats_summary_list = [
+          [
+              [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+              [[0., 0.], [-.33, .58], [0., 0.], [.3, .4]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 0
+          [
+              [[0., 0.], [0., 0.], [.08, .09], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.3, .5], [-.05, .06], [.06, .07]],  # node 1
+              [[.1, .1], [.2, .3], [-.4, .5], [.07, .08]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 1
+      ]  # num_features * shape=[max_splits, num_buckets, 2]
+
+      (node_ids_list, gains_list, thresholds_list, left_node_contribs_list,
+       right_node_contribs_list
+      ) = boosted_trees_ops.calculate_best_gains_per_feature(
+          node_id_range,
+          stats_summary_list,
+          l1=0.0,
+          l2=0.1,
+          tree_complexity=0.0,
+          max_splits=max_splits)
+
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllClose([[0., 0.33931375], [0.01879096, 0.33931375]],
+                          sess.run(gains_list))
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      # The left node contrib will be later added to the previous node value to
+      # make the left node value, and the same for right node contrib.
+      self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
+                          sess.run(left_node_contribs_list))
+      self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]],
+                          sess.run(right_node_contribs_list))
+
+  def testCalculateBestGainsWithL1(self):
+    """Testing Gain calculation with L1."""
+    with self.test_session() as sess:
+      max_splits = 7
+      node_id_range = [1, 2]  # node 1 through 2 will be processed.
+      stats_summary_list = [
+          [
+              [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+              [[0., 0.], [-.33, .58], [0., 0.], [.3, .4]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 0
+          [
+              [[0., 0.], [0., 0.], [.08, .09], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.3, .5], [-.05, .06], [.06, .07]],  # node 1
+              [[.1, .1], [.2, .3], [-.4, .5], [.07, .08]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 1
+      ]  # num_features * shape=[max_splits, num_buckets, 2]
+
+      l1 = 0.1
+      (node_ids_list, gains_list, thresholds_list, left_node_contribs_list,
+       right_node_contribs_list
+      ) = boosted_trees_ops.calculate_best_gains_per_feature(
+          node_id_range,
+          stats_summary_list,
+          l1=l1,
+          l2=0.0,
+          tree_complexity=0.0,
+          max_splits=max_splits)
+
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+      self.assertAllClose([[[0.0], [0.3965517]], [[-0.4], [-0.5]]],
+                          sess.run(left_node_contribs_list))
+
+      self.assertAllClose([[[-0.3333333], [-0.5]], [[0.0], [0.396552]]],
+                          sess.run(right_node_contribs_list))
+
+      # Gain should also include an adjustment of the gradient by l1.
+      self.assertAllClose([[0.0, 0.191207], [0.01, 0.191207]],
+                          sess.run(gains_list))
+
+  def testCalculateBestGainsWithTreeComplexity(self):
+    """Testing Gain calculation with L2."""
+    with self.test_session() as sess:
+      max_splits = 7
+      node_id_range = [1, 2]  # node 1 through 2 will be processed.
+      stats_summary_list = [
+          [
+              [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+              [[0., 0.], [-.33, .58], [0., 0.], [.3, .4]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 0
+          [
+              [[0., 0.], [0., 0.], [.08, .09], [0., 0.]],  # node 0; ignored
+              [[0., 0.], [.3, .5], [-.05, .06], [.06, .07]],  # node 1
+              [[.1, .1], [.2, .3], [-.4, .5], [.07, .08]],  # node 2
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 3; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 4; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 5; ignored
+              [[0., 0.], [0., 0.], [0., 0.], [0., 0.]],  # node 6; ignored
+          ],  # feature 1
+      ]  # num_features * shape=[max_splits, num_buckets, 2]
+
+      l2 = 0.1
+      tree_complexity = 3.
+      (node_ids_list, gains_list, thresholds_list, left_node_contribs_list,
+       right_node_contribs_list
+      ) = boosted_trees_ops.calculate_best_gains_per_feature(
+          node_id_range,
+          stats_summary_list,
+          l1=0.0,
+          l2=l2,
+          tree_complexity=tree_complexity,
+          max_splits=max_splits)
+
+      self.assertAllEqual([[1, 2], [1, 2]], sess.run(node_ids_list))
+
+      self.assertAllClose([[-3., -2.66068625], [-2.98120904, -2.66068625]],
+                          sess.run(gains_list))
+
+      self.assertAllEqual([[0, 1], [1, 1]], sess.run(thresholds_list))
+      # The left node contrib will be later added to the previous node value to
+      # make the left node value, and the same for right node contrib.
+      self.assertAllClose([[[0.], [.485294]], [[-.5], [-.6]]],
+                          sess.run(left_node_contribs_list))
+      self.assertAllClose([[[-.424658], [-.6]], [[-.043478], [.485294]]],
+                          sess.run(right_node_contribs_list))
+
+  def testMakeStatsSummarySimple(self):
+    """Simple test for MakeStatsSummary."""
+    with self.test_session():
+      self.assertAllClose([[[[1., 5.], [2., 6.]], [[3., 7.], [4., 8.]]]],
+                          boosted_trees_ops.make_stats_summary(
+                              node_ids=[0, 0, 1, 1],
+                              gradients=[[1.], [2.], [3.], [4.]],
+                              hessians=[[5.], [6.], [7.], [8.]],
+                              bucketized_features_list=[[0, 1, 0, 1]],
+                              max_splits=2,
+                              num_buckets=2).eval())
+
+  def testMakeStatsSummaryAccumulate(self):
+    """Tests that Summary actually accumulates."""
+    with self.test_session():
+      max_splits = 3
+      num_buckets = 4
+      node_ids = [1, 1, 2, 2, 1, 1, 2, 0]
+      gradients = [[.1], [.2], [.3], [-.4], [-.05], [.06], [.07], [.08]]
+      hessians = [[.2], [.3], [.4], [.5], [.06], [.07], [.08], [.09]]
+
+      # Tests a single feature.
+      bucketized_features = [[3, 1, 2, 0, 1, 2, 0, 1]]
+      result = boosted_trees_ops.make_stats_summary(
+          node_ids, gradients, hessians, bucketized_features, max_splits,
+          num_buckets)  # shape=[max_splits, num_buckets, num_features, 2]
+      self.assertAllClose(
+          [[
+              [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0
+              [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+              [[-.33, .58], [0., 0.], [.3, .4], [0., 0.]],  # node 2
+          ]],
+          result.eval())
+
+  def testMakeStatsSummaryMultipleFeatures(self):
+    """Tests that MakeStatsSummary works for multiple features."""
+    with self.test_session():
+      max_splits = 3
+      num_buckets = 4
+      node_ids = [1, 1, 2, 2, 1, 1, 2, 0]
+      gradients = [[.1], [.2], [.3], [-.4], [-.05], [.06], [.07], [.08]]
+      hessians = [[.2], [.3], [.4], [.5], [.06], [.07], [.08], [.09]]
+
+      # Tests multiple features.
+      # The output from another feature will stored be in 3rd dimension.
+      bucketized_features = [[3, 1, 2, 0, 1, 2, 0, 1], [0, 0, 0, 2, 2, 3, 3, 2]]
+      result = boosted_trees_ops.make_stats_summary(
+          node_ids, gradients, hessians, bucketized_features, max_splits,
+          num_buckets)  # shape=[max_splits, num_buckets, num_features, 2]
+      self.assertAllClose(
+          [
+              [
+                  [[0., 0.], [.08, .09], [0., 0.], [0., 0.]],  # node 0
+                  [[0., 0.], [.15, .36], [.06, .07], [.1, .2]],  # node 1
+                  [[-.33, .58], [0., 0.], [.3, .4], [0., 0.]],  # node 2
+              ],  # feature 0
+              [
+                  [[0., 0.], [0., 0.], [.08, .09], [0., 0.]],  # node 0
+                  [[.3, .5], [0., 0.], [-.05, .06], [.06, .07]],  # node 1
+                  [[.3, .4], [0., 0.], [-.4, .5], [.07, .08]],  # node 2
+              ],  # feature 1
+          ],
+          result.eval())
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
new file mode 100644
index 0000000000..4226ff75c2
--- /dev/null
+++ b/tensorflow/python/kernel_tests/boosted_trees/training_ops_test.py
@@ -0,0 +1,1465 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for boosted_trees training kernels."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from google.protobuf import text_format
+from tensorflow.core.kernels.boosted_trees import boosted_trees_pb2
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import boosted_trees_ops
+from tensorflow.python.ops import resources
+from tensorflow.python.platform import googletest
+
+
+class UpdateTreeEnsembleOpTest(test_util.TensorFlowTestCase):
+  """Tests for growing tree ensemble from split candidates."""
+
+  def testGrowWithEmptyEnsemble(self):
+    """Test growing an empty ensemble."""
+    with self.test_session() as session:
+      # Create empty ensemble.
+      tree_ensemble = boosted_trees_ops.TreeEnsemble('ensemble')
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      feature_ids = [0, 2, 6]
+
+      # Prepare feature inputs.
+      # Note that features 1 & 3 have the same gain but different splits.
+      feature1_nodes = np.array([0], dtype=np.int32)
+      feature1_gains = np.array([7.62], dtype=np.float32)
+      feature1_thresholds = np.array([52], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-4.375]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[7.143]], dtype=np.float32)
+
+      feature2_nodes = np.array([0], dtype=np.int32)
+      feature2_gains = np.array([0.63], dtype=np.float32)
+      feature2_thresholds = np.array([23], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[-0.6]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.24]], dtype=np.float32)
+
+      # Feature split with the highest gain.
+      feature3_nodes = np.array([0], dtype=np.int32)
+      feature3_gains = np.array([7.65], dtype=np.float32)
+      feature3_thresholds = np.array([7], dtype=np.int32)
+      feature3_left_node_contribs = np.array([[-4.89]], dtype=np.float32)
+      feature3_right_node_contribs = np.array([[5.3]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=0.1,
+          pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING,
+          # Tree will be finalized now, since we will reach depth 1.
+          max_depth=1,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes, feature3_nodes],
+          gains=[feature1_gains, feature2_gains, feature3_gains],
+          thresholds=[
+              feature1_thresholds, feature2_thresholds, feature3_thresholds
+          ],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs,
+              feature3_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs,
+              feature3_right_node_contribs
+          ])
+      session.run(grow_op)
+
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      # Note that since the tree is finalized, we added a new dummy tree.
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 6
+              threshold: 7
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.65
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.489
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.53
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.0
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: true
+        }
+        tree_metadata {
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testGrowExistingEnsembleTreeNotFinalized(self):
+    """Test growing an existing ensemble with the last tree not finalized."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.714
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.4375
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare feature inputs.
+      # feature 1 only has a candidate for node 1, feature 2 has candidates
+      # for both nodes and feature 3 only has a candidate for node 2.
+
+      feature_ids = [0, 1, 0]
+
+      feature1_nodes = np.array([1], dtype=np.int32)
+      feature1_gains = np.array([1.4], dtype=np.float32)
+      feature1_thresholds = np.array([21], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-6.0]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[1.65]], dtype=np.float32)
+
+      feature2_nodes = np.array([1, 2], dtype=np.int32)
+      feature2_gains = np.array([0.63, 2.7], dtype=np.float32)
+      feature2_thresholds = np.array([23, 7], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[-0.6], [-1.5]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.24], [2.3]], dtype=np.float32)
+
+      feature3_nodes = np.array([2], dtype=np.int32)
+      feature3_gains = np.array([1.7], dtype=np.float32)
+      feature3_thresholds = np.array([3], dtype=np.int32)
+      feature3_left_node_contribs = np.array([[-0.75]], dtype=np.float32)
+      feature3_right_node_contribs = np.array([[1.93]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=0.1,
+          pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING,
+          # tree is going to be finalized now, since we reach depth 2.
+          max_depth=2,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes, feature3_nodes],
+          gains=[feature1_gains, feature2_gains, feature3_gains],
+          thresholds=[
+              feature1_thresholds, feature2_thresholds, feature3_thresholds
+          ],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs,
+              feature3_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs,
+              feature3_right_node_contribs
+          ])
+      session.run(grow_op)
+
+      # Expect the split for node 1 to be chosen from feature 1 and
+      # the split for node 2 to be chosen from feature 2.
+      # The grown tree should be finalized as max tree depth is 2 and we have
+      # grown 2 layers.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            bucketized_split {
+              threshold: 21
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 1.4
+              original_leaf {
+                scalar: 0.714
+              }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 7
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 2.7
+              original_leaf {
+                scalar: -0.4375
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.114
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.879
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.5875
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.2075
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.0
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          is_finalized: true
+          num_layers_grown: 2
+        }
+        tree_metadata {
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testGrowExistingEnsembleTreeFinalized(self):
+    """Test growing an existing ensemble with the last tree finalized."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+              scalar: 0.0
+            }
+          }
+        }
+        tree_weights: 0.15
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: true
+        }
+        tree_metadata {
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare feature inputs.
+
+      feature_ids = [75]
+
+      feature1_nodes = np.array([0], dtype=np.int32)
+      feature1_gains = np.array([-1.4], dtype=np.float32)
+      feature1_thresholds = np.array([21], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-6.0]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[1.65]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING,
+          learning_rate=0.1,
+          max_depth=2,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes],
+          gains=[feature1_gains],
+          thresholds=[feature1_thresholds],
+          left_node_contribs=[feature1_left_node_contribs],
+          right_node_contribs=[feature1_right_node_contribs])
+      session.run(grow_op)
+
+      # Expect a new tree added, with a split on feature 75
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+       trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+        }
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 75
+              threshold: 21
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -1.4
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.6
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.165
+            }
+          }
+        }
+        tree_weights: 0.15
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: true
+        }
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 2
+          num_layers_attempted: 2
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testPrePruning(self):
+    """Test growing an existing ensemble with pre-pruning."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare feature inputs.
+      # For node 1, the best split is on feature 2 (gain -0.63), but the gain
+      # is negative so node 1 will not be split.
+      # For node 2, the best split is on feature 3, gain is positive.
+
+      feature_ids = [0, 1, 0]
+
+      feature1_nodes = np.array([1], dtype=np.int32)
+      feature1_gains = np.array([-1.4], dtype=np.float32)
+      feature1_thresholds = np.array([21], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-6.0]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[1.65]], dtype=np.float32)
+
+      feature2_nodes = np.array([1, 2], dtype=np.int32)
+      feature2_gains = np.array([-0.63, 2.7], dtype=np.float32)
+      feature2_thresholds = np.array([23, 7], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[-0.6], [-1.5]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.24], [2.3]], dtype=np.float32)
+
+      feature3_nodes = np.array([2], dtype=np.int32)
+      feature3_gains = np.array([2.8], dtype=np.float32)
+      feature3_thresholds = np.array([3], dtype=np.int32)
+      feature3_left_node_contribs = np.array([[-0.75]], dtype=np.float32)
+      feature3_right_node_contribs = np.array([[1.93]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=0.1,
+          pruning_mode=boosted_trees_ops.PruningMode.PRE_PRUNING,
+          max_depth=3,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes, feature3_nodes],
+          gains=[feature1_gains, feature2_gains, feature3_gains],
+          thresholds=[
+              feature1_thresholds, feature2_thresholds, feature3_thresholds
+          ],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs,
+              feature3_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs,
+              feature3_right_node_contribs
+          ])
+      session.run(grow_op)
+
+      # Expect the split for node 1 to be chosen from feature 1 and
+      # the split for node 2 to be chosen from feature 2.
+      # The grown tree should not be finalized as max tree depth is 3 and
+      # it's only grown 2 layers.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 0
+              threshold: 3
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 2.8
+              original_leaf {
+                scalar: -4.375
+              }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.45
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.182
+            }
+          }
+        }
+        tree_weights: 0.1
+        tree_metadata {
+          is_finalized: false
+          num_layers_grown: 2
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testMetadataWhenCantSplitDueToEmptySplits(self):
+    """Test that the metadata is updated even though we can't split."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.714
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.4375
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare feature inputs.
+      # feature 1 only has a candidate for node 1, feature 2 has candidates
+      # for both nodes and feature 3 only has a candidate for node 2.
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=0.1,
+          pruning_mode=boosted_trees_ops.PruningMode.NO_PRUNING,
+          max_depth=2,
+          # No splits are available.
+          feature_ids=[],
+          node_ids=[],
+          gains=[],
+          thresholds=[],
+          left_node_contribs=[],
+          right_node_contribs=[])
+      session.run(grow_op)
+
+      # Expect no new splits created, but attempted (global) stats updated. Meta
+      # data for this tree should not be updated (we didn't succeed building a
+      # layer.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.714
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -0.4375
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testMetadataWhenCantSplitDuePrePruning(self):
+    """Test metadata is updated correctly when no split due to prepruning."""
+    with self.test_session() as session:
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      text_format.Merge("""
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """, tree_ensemble_config)
+
+      # Create existing ensemble with one root split
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare feature inputs.
+      feature_ids = [0, 1, 0]
+
+      # All the gains are negative.
+      feature1_nodes = np.array([1], dtype=np.int32)
+      feature1_gains = np.array([-1.4], dtype=np.float32)
+      feature1_thresholds = np.array([21], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-6.0]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[1.65]], dtype=np.float32)
+
+      feature2_nodes = np.array([1, 2], dtype=np.int32)
+      feature2_gains = np.array([-0.63, -2.7], dtype=np.float32)
+      feature2_thresholds = np.array([23, 7], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[-0.6], [-1.5]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.24], [2.3]], dtype=np.float32)
+
+      feature3_nodes = np.array([2], dtype=np.int32)
+      feature3_gains = np.array([-2.8], dtype=np.float32)
+      feature3_thresholds = np.array([3], dtype=np.int32)
+      feature3_left_node_contribs = np.array([[-0.75]], dtype=np.float32)
+      feature3_right_node_contribs = np.array([[1.93]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=0.1,
+          pruning_mode=boosted_trees_ops.PruningMode.PRE_PRUNING,
+          max_depth=3,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes, feature3_nodes],
+          gains=[feature1_gains, feature2_gains, feature3_gains],
+          thresholds=[
+              feature1_thresholds, feature2_thresholds, feature3_thresholds
+          ],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs,
+              feature3_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs,
+              feature3_right_node_contribs
+          ])
+      session.run(grow_op)
+
+      # Expect that no new split was created because all the gains were negative
+      # Global metadata should be updated, tree metadata should not be updated.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      tree_ensemble = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 4
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.14
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, tree_ensemble)
+
+  def testPostPruningOfSomeNodes(self):
+    """Test growing an ensemble with post-pruning."""
+    with self.test_session() as session:
+      # Create empty ensemble.
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare inputs.
+      # Second feature has larger (but still negative gain).
+      feature_ids = [0, 1]
+
+      feature1_nodes = np.array([0], dtype=np.int32)
+      feature1_gains = np.array([-1.3], dtype=np.float32)
+      feature1_thresholds = np.array([7], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[0.013]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[0.0143]], dtype=np.float32)
+
+      feature2_nodes = np.array([0], dtype=np.int32)
+      feature2_gains = np.array([-0.2], dtype=np.float32)
+      feature2_thresholds = np.array([33], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[0.01]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.0143]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=3,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes],
+          gains=[feature1_gains, feature2_gains],
+          thresholds=[feature1_thresholds, feature2_thresholds],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs
+          ])
+
+      session.run(grow_op)
+
+      # Expect the split from second features to be chosen despite the negative
+      # gain.
+      # No pruning happened just yet.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.01
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0143
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, res_ensemble)
+
+      # Prepare the second layer.
+      # Note that node 1 gain is negative and node 2 gain is positive.
+      feature_ids = [3]
+      feature1_nodes = np.array([1, 2], dtype=np.int32)
+      feature1_gains = np.array([-0.2, 0.5], dtype=np.float32)
+      feature1_thresholds = np.array([7, 5], dtype=np.int32)
+      feature1_left_node_contribs = np.array(
+          [[0.07], [0.041]], dtype=np.float32)
+      feature1_right_node_contribs = np.array(
+          [[0.083], [0.064]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=3,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes],
+          gains=[feature1_gains],
+          thresholds=[feature1_thresholds],
+          left_node_contribs=[feature1_left_node_contribs],
+          right_node_contribs=[feature1_right_node_contribs])
+
+      session.run(grow_op)
+
+      # After adding this layer, the tree will not be finalized
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id:1
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.2
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 3
+              threshold: 7
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: -0.2
+              original_leaf {
+                scalar: 0.01
+               }
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 3
+              threshold: 5
+              left_id: 5
+              right_id: 6
+            }
+            metadata {
+              gain: 0.5
+              original_leaf {
+                scalar: 0.0143
+               }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.08
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.093
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0553
+            }
+          }
+          nodes {
+            leaf {
+                scalar: 0.0783
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 2
+          is_finalized: false
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 2
+        }
+       """
+      self.assertEqual(new_stamp, 2)
+
+      self.assertProtoEquals(expected_result, res_ensemble)
+      # Now split the leaf 3, again with negative gain. After this layer, the
+      # tree will be finalized, and post-pruning happens. The leafs 3,4,7,8 will
+      # be pruned out.
+
+      # Prepare the third layer.
+      feature_ids = [92]
+      feature1_nodes = np.array([3], dtype=np.int32)
+      feature1_gains = np.array([-0.45], dtype=np.float32)
+      feature1_thresholds = np.array([11], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[0.15]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[0.5]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=3,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes],
+          gains=[feature1_gains],
+          thresholds=[feature1_thresholds],
+          left_node_contribs=[feature1_left_node_contribs],
+          right_node_contribs=[feature1_right_node_contribs])
+
+      session.run(grow_op)
+      # After adding this layer, the tree will be finalized
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+      # Node that nodes 3, 4, 7 and 8 got deleted, so metadata stores has ids
+      # mapped to their parent node 1, with the respective change in logits.
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id:1
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.2
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.01
+            }
+          }
+          nodes {
+            bucketized_split {
+              feature_id: 3
+              threshold: 5
+              left_id: 3
+              right_id: 4
+            }
+            metadata {
+              gain: 0.5
+              original_leaf {
+                scalar: 0.0143
+               }
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0553
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0783
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 3
+          is_finalized: true
+          post_pruned_nodes_meta {
+            new_node_id: 0
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 2
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.07
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.083
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 3
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 4
+            logit_change: 0.0
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.22
+          }
+          post_pruned_nodes_meta {
+            new_node_id: 1
+            logit_change: -0.57
+          }
+        }
+        tree_metadata {
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 3
+        }
+       """
+      self.assertEqual(new_stamp, 3)
+      self.assertProtoEquals(expected_result, res_ensemble)
+
+  def testPostPruningOfAllNodes(self):
+    """Test growing an ensemble with post-pruning, with all nodes are pruned."""
+    with self.test_session() as session:
+      # Create empty ensemble.
+      # Create empty ensemble.
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare inputs. All have negative gains.
+      feature_ids = [0, 1]
+
+      feature1_nodes = np.array([0], dtype=np.int32)
+      feature1_gains = np.array([-1.3], dtype=np.float32)
+      feature1_thresholds = np.array([7], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[0.013]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[0.0143]], dtype=np.float32)
+
+      feature2_nodes = np.array([0], dtype=np.int32)
+      feature2_gains = np.array([-0.62], dtype=np.float32)
+      feature2_thresholds = np.array([33], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[0.01]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.0143]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=2,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes],
+          gains=[feature1_gains, feature2_gains],
+          thresholds=[feature1_thresholds, feature2_thresholds],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs
+          ])
+
+      session.run(grow_op)
+
+      # Expect the split from feature 2 to be chosen despite the negative gain.
+      # The grown tree should not be finalized as max tree depth is 2 so no
+      # pruning occurs.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 1
+              threshold: 33
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: -0.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.01
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 0.0143
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, res_ensemble)
+
+      # Prepare inputs.
+      # All have negative gain.
+      feature_ids = [3]
+      feature1_nodes = np.array([1, 2], dtype=np.int32)
+      feature1_gains = np.array([-0.2, -0.5], dtype=np.float32)
+      feature1_thresholds = np.array([77, 79], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[0.023], [0.3]], dtype=np.float32)
+      feature1_right_node_contribs = np.array(
+          [[0.012343], [24]], dtype=np.float32)
+
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=2,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes],
+          gains=[feature1_gains],
+          thresholds=[feature1_thresholds],
+          left_node_contribs=[feature1_left_node_contribs],
+          right_node_contribs=[feature1_right_node_contribs])
+
+      session.run(grow_op)
+
+      # Expect the split from feature 1 to be chosen despite the negative gain.
+      # The grown tree should be finalized. Since all nodes have negative gain,
+      # the whole tree is pruned.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+
+      # Expect the ensemble to be empty as post-pruning will prune
+      # the entire finalized tree.
+      self.assertEqual(new_stamp, 2)
+      self.assertProtoEquals("""
+      trees {
+        nodes {
+          leaf {
+          }
+        }
+      }
+      trees {
+        nodes {
+          leaf {
+          }
+        }
+      }
+      tree_weights: 1.0
+      tree_weights: 1.0
+      tree_metadata{
+        num_layers_grown: 2
+        is_finalized: true
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: 0.0
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -0.01
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -0.0143
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -0.033
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -0.022343
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -0.3143
+        }
+        post_pruned_nodes_meta {
+          new_node_id: 0
+          logit_change: -24.0143
+        }
+      }
+      tree_metadata {
+      }
+      growing_metadata {
+        num_trees_attempted: 1
+        num_layers_attempted: 2
+      }
+      """, res_ensemble)
+
+  def testPostPruningChangesNothing(self):
+    """Test growing an ensemble with post-pruning with all gains >0."""
+    with self.test_session() as session:
+      # Create empty ensemble.
+      tree_ensemble_config = boosted_trees_pb2.TreeEnsemble()
+      tree_ensemble = boosted_trees_ops.TreeEnsemble(
+          'ensemble', serialized_proto=tree_ensemble_config.SerializeToString())
+      tree_ensemble_handle = tree_ensemble.resource_handle
+
+      resources.initialize_resources(resources.shared_resources()).run()
+
+      # Prepare inputs.
+      # Second feature has larger (but still negative gain).
+      feature_ids = [3, 4]
+
+      feature1_nodes = np.array([0], dtype=np.int32)
+      feature1_gains = np.array([7.62], dtype=np.float32)
+      feature1_thresholds = np.array([52], dtype=np.int32)
+      feature1_left_node_contribs = np.array([[-4.375]], dtype=np.float32)
+      feature1_right_node_contribs = np.array([[7.143]], dtype=np.float32)
+
+      feature2_nodes = np.array([0], dtype=np.int32)
+      feature2_gains = np.array([0.63], dtype=np.float32)
+      feature2_thresholds = np.array([23], dtype=np.int32)
+      feature2_left_node_contribs = np.array([[-0.6]], dtype=np.float32)
+      feature2_right_node_contribs = np.array([[0.24]], dtype=np.float32)
+
+      # Grow tree ensemble.
+      grow_op = boosted_trees_ops.update_ensemble(
+          tree_ensemble_handle,
+          learning_rate=1.0,
+          pruning_mode=boosted_trees_ops.PruningMode.POST_PRUNING,
+          max_depth=1,
+          feature_ids=feature_ids,
+          node_ids=[feature1_nodes, feature2_nodes],
+          gains=[feature1_gains, feature2_gains],
+          thresholds=[feature1_thresholds, feature2_thresholds],
+          left_node_contribs=[
+              feature1_left_node_contribs, feature2_left_node_contribs
+          ],
+          right_node_contribs=[
+              feature1_right_node_contribs, feature2_right_node_contribs
+          ])
+
+      session.run(grow_op)
+
+      # Expect the split from the first feature to be chosen.
+      # Pruning got triggered but changed nothing.
+      new_stamp, serialized = session.run(tree_ensemble.serialize())
+      res_ensemble = boosted_trees_pb2.TreeEnsemble()
+      res_ensemble.ParseFromString(serialized)
+
+      expected_result = """
+        trees {
+          nodes {
+            bucketized_split {
+              feature_id: 3
+              threshold: 52
+              left_id: 1
+              right_id: 2
+            }
+            metadata {
+              gain: 7.62
+            }
+          }
+          nodes {
+            leaf {
+              scalar: -4.375
+            }
+          }
+          nodes {
+            leaf {
+              scalar: 7.143
+            }
+          }
+        }
+        trees {
+          nodes {
+            leaf {
+            }
+          }
+        }
+        tree_weights: 1.0
+        tree_weights: 1.0
+        tree_metadata {
+          num_layers_grown: 1
+          is_finalized: true
+        }
+        tree_metadata {
+        }
+        growing_metadata {
+          num_trees_attempted: 1
+          num_layers_attempted: 1
+        }
+      """
+      self.assertEqual(new_stamp, 1)
+      self.assertProtoEquals(expected_result, res_ensemble)
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/python/ops/boosted_trees_ops.py b/tensorflow/python/ops/boosted_trees_ops.py
new file mode 100644
index 0000000000..174d00987f
--- /dev/null
+++ b/tensorflow/python/ops/boosted_trees_ops.py
@@ -0,0 +1,160 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Ops for boosted_trees."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_boosted_trees_ops
+from tensorflow.python.ops import resources
+
+# Re-exporting ops used by other modules.
+# pylint: disable=unused-import
+from tensorflow.python.ops.gen_boosted_trees_ops import boosted_trees_calculate_best_gains_per_feature as calculate_best_gains_per_feature
+from tensorflow.python.ops.gen_boosted_trees_ops import boosted_trees_make_stats_summary as make_stats_summary
+from tensorflow.python.ops.gen_boosted_trees_ops import boosted_trees_predict as predict
+from tensorflow.python.ops.gen_boosted_trees_ops import boosted_trees_training_predict as training_predict
+from tensorflow.python.ops.gen_boosted_trees_ops import boosted_trees_update_ensemble as update_ensemble
+# pylint: enable=unused-import
+
+from tensorflow.python.training import saver
+
+
+class PruningMode(object):
+  NO_PRUNING, PRE_PRUNING, POST_PRUNING = range(0, 3)
+
+
+class _TreeEnsembleSavable(saver.BaseSaverBuilder.SaveableObject):
+  """SaveableObject implementation for TreeEnsemble."""
+
+  def __init__(self, resource_handle, create_op, name):
+    """Creates a _TreeEnsembleSavable object.
+
+    Args:
+      resource_handle: handle to the decision tree ensemble variable.
+      create_op: the op to initialize the variable.
+      name: the name to save the tree ensemble variable under.
+    """
+    stamp_token, serialized = (
+        gen_boosted_trees_ops.boosted_trees_serialize_ensemble(resource_handle))
+    # slice_spec is useful for saving a slice from a variable.
+    # It's not meaningful the tree ensemble variable. So we just pass an empty
+    # value.
+    slice_spec = ''
+    specs = [
+        saver.BaseSaverBuilder.SaveSpec(stamp_token, slice_spec,
+                                        name + '_stamp'),
+        saver.BaseSaverBuilder.SaveSpec(serialized, slice_spec,
+                                        name + '_serialized'),
+    ]
+    super(_TreeEnsembleSavable, self).__init__(resource_handle, specs, name)
+    self._resource_handle = resource_handle
+    self._create_op = create_op
+
+  def restore(self, restored_tensors, unused_restored_shapes):
+    """Restores the associated tree ensemble from 'restored_tensors'.
+
+    Args:
+      restored_tensors: the tensors that were loaded from a checkpoint.
+      unused_restored_shapes: the shapes this object should conform to after
+        restore. Not meaningful for trees.
+
+    Returns:
+      The operation that restores the state of the tree ensemble variable.
+    """
+    with ops.control_dependencies([self._create_op]):
+      return gen_boosted_trees_ops.boosted_trees_deserialize_ensemble(
+          self._resource_handle,
+          stamp_token=restored_tensors[0],
+          tree_ensemble_serialized=restored_tensors[1])
+
+
+class TreeEnsemble(object):
+  """Creates TreeEnsemble resource."""
+
+  def __init__(self, name, stamp_token=0, is_local=False, serialized_proto=''):
+    with ops.name_scope(name, 'TreeEnsemble') as name:
+      self._resource_handle = (
+          gen_boosted_trees_ops.boosted_trees_ensemble_resource_handle_op(
+              container='', shared_name=name, name=name))
+      create_op = gen_boosted_trees_ops.boosted_trees_create_ensemble(
+          self.resource_handle,
+          stamp_token,
+          tree_ensemble_serialized=serialized_proto)
+      is_initialized_op = (
+          gen_boosted_trees_ops.is_boosted_trees_ensemble_initialized(
+              self._resource_handle))
+      # Adds the variable to the savable list.
+      if not is_local:
+        saveable = _TreeEnsembleSavable(self.resource_handle, create_op,
+                                        self.resource_handle.name)
+        ops.add_to_collection(ops.GraphKeys.SAVEABLE_OBJECTS, saveable)
+      resources.register_resource(
+          self.resource_handle,
+          create_op,
+          is_initialized_op,
+          is_shared=not is_local)
+
+  @property
+  def resource_handle(self):
+    return self._resource_handle
+
+  def get_stamp_token(self):
+    """Returns the current stamp token of the resource."""
+    stamp_token, _, _, _ = (
+        gen_boosted_trees_ops.boosted_trees_get_ensemble_states(
+            self.resource_handle))
+    return stamp_token
+
+  def get_states(self):
+    """Returns states of the tree ensemble.
+
+    Returns:
+      stamp_token, num_trees, num_finalized_trees, num_attempted_layers.
+    """
+    stamp_token, num_trees, num_finalized_trees, num_attempted_layers = (
+        gen_boosted_trees_ops.boosted_trees_get_ensemble_states(
+            self.resource_handle))
+    # Use identity to give names.
+    return (array_ops.identity(stamp_token, name='stamp_token'),
+            array_ops.identity(num_trees, name='num_trees'),
+            array_ops.identity(num_finalized_trees, name='num_finalized_trees'),
+            array_ops.identity(
+                num_attempted_layers, name='num_attempted_layers'))
+
+  def serialize(self):
+    """Serializes the ensemble into proto and returns the serialized proto.
+
+    Returns:
+      stamp_token: int64 scalar Tensor to denote the stamp of the resource.
+      serialized_proto: string scalar Tensor of the serialized proto.
+    """
+    return gen_boosted_trees_ops.boosted_trees_serialize_ensemble(
+        self.resource_handle)
+
+  def deserialize(self, stamp_token, serialized_proto):
+    """Deserialize the input proto and resets the ensemble from it.
+
+    Args:
+      stamp_token: int64 scalar Tensor to denote the stamp of the resource.
+      serialized_proto: string scalar Tensor of the serialized proto.
+
+    Returns:
+      Operation (for dependencies).
+    """
+    return gen_boosted_trees_ops.boosted_trees_deserialize_ensemble(
+        self.resource_handle, stamp_token, serialized_proto)
diff --git a/tensorflow/python/training/device_setter.py b/tensorflow/python/training/device_setter.py
index d31c375b4c..be80c36571 100644
--- a/tensorflow/python/training/device_setter.py
+++ b/tensorflow/python/training/device_setter.py
@@ -25,14 +25,13 @@ from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.training import server_lib
 from tensorflow.python.util.tf_export import tf_export
 
-# This is a tuple of PS ops used by tf.estimator.Esitmator which should work in
+# This is a tuple of PS ops used by tf.estimator.Estimator which should work in
 # almost all of cases.
-STANDARD_PS_OPS = (
-    "Variable", "VariableV2", "AutoReloadVariable", "MutableHashTable",
-    "MutableHashTableV2", "MutableHashTableOfTensors",
-    "MutableHashTableOfTensorsV2", "MutableDenseHashTable",
-    "MutableDenseHashTableV2", "VarHandleOp"
-)
+STANDARD_PS_OPS = ("Variable", "VariableV2", "AutoReloadVariable",
+                   "MutableHashTable", "MutableHashTableV2",
+                   "MutableHashTableOfTensors", "MutableHashTableOfTensorsV2",
+                   "MutableDenseHashTable", "MutableDenseHashTableV2",
+                   "VarHandleOp", "BoostedTreesEnsembleResourceHandleOp")
 
 
 class _RoundRobinStrategy(object):
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-classifier.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-classifier.pbtxt
new file mode 100644
index 0000000000..fd9be8c759
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-classifier.pbtxt
@@ -0,0 +1,54 @@
+path: "tensorflow.estimator.BoostedTreesClassifier"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.canned.boosted_trees.BoostedTreesClassifier\'>"
+  is_instance: "<class \'tensorflow.python.estimator.estimator.Estimator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "config"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "model_dir"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "model_fn"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "params"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feature_columns\', \'n_batches_per_layer\', \'model_dir\', \'n_classes\', \'weight_column\', \'label_vocabulary\', \'n_trees\', \'max_depth\', \'learning_rate\', \'l1_regularization\', \'l2_regularization\', \'tree_complexity\', \'config\'], varargs=None, keywords=None, defaults=[\'None\', \'<object object instance>\', \'None\', \'None\', \'100\', \'6\', \'0.1\', \'0.0\', \'0.0\', \'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "evaluate"
+    argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "export_savedmodel"
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
+  }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "latest_checkpoint"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "predict"
+    argspec: "args=[\'self\', \'input_fn\', \'predict_keys\', \'hooks\', \'checkpoint_path\', \'yield_single_examples\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "train"
+    argspec: "args=[\'self\', \'input_fn\', \'hooks\', \'steps\', \'max_steps\', \'saving_listeners\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-regressor.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-regressor.pbtxt
new file mode 100644
index 0000000000..6b305be43f
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-boosted-trees-regressor.pbtxt
@@ -0,0 +1,54 @@
+path: "tensorflow.estimator.BoostedTreesRegressor"
+tf_class {
+  is_instance: "<class \'tensorflow.python.estimator.canned.boosted_trees.BoostedTreesRegressor\'>"
+  is_instance: "<class \'tensorflow.python.estimator.estimator.Estimator\'>"
+  is_instance: "<type \'object\'>"
+  member {
+    name: "config"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "model_dir"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "model_fn"
+    mtype: "<type \'property\'>"
+  }
+  member {
+    name: "params"
+    mtype: "<type \'property\'>"
+  }
+  member_method {
+    name: "__init__"
+    argspec: "args=[\'self\', \'feature_columns\', \'n_batches_per_layer\', \'model_dir\', \'label_dimension\', \'weight_column\', \'n_trees\', \'max_depth\', \'learning_rate\', \'l1_regularization\', \'l2_regularization\', \'tree_complexity\', \'config\'], varargs=None, keywords=None, defaults=[\'None\', \'<object object instance>\', \'None\', \'100\', \'6\', \'0.1\', \'0.0\', \'0.0\', \'0.0\', \'None\'], "
+  }
+  member_method {
+    name: "evaluate"
+    argspec: "args=[\'self\', \'input_fn\', \'steps\', \'hooks\', \'checkpoint_path\', \'name\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+  member_method {
+    name: "export_savedmodel"
+    argspec: "args=[\'self\', \'export_dir_base\', \'serving_input_receiver_fn\', \'assets_extra\', \'as_text\', \'checkpoint_path\', \'strip_default_attrs\'], varargs=None, keywords=None, defaults=[\'None\', \'False\', \'None\', \'False\'], "
+  }
+  member_method {
+    name: "get_variable_names"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "get_variable_value"
+    argspec: "args=[\'self\', \'name\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "latest_checkpoint"
+    argspec: "args=[\'self\'], varargs=None, keywords=None, defaults=None"
+  }
+  member_method {
+    name: "predict"
+    argspec: "args=[\'self\', \'input_fn\', \'predict_keys\', \'hooks\', \'checkpoint_path\', \'yield_single_examples\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'True\'], "
+  }
+  member_method {
+    name: "train"
+    argspec: "args=[\'self\', \'input_fn\', \'hooks\', \'steps\', \'max_steps\', \'saving_listeners\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'None\', \'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
index a7a6cc1e49..4946f2c51a 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.pbtxt
@@ -8,6 +8,14 @@ tf_module {
     name: "BaselineRegressor"
     mtype: "<type \'type\'>"
   }
+  member {
+    name: "BoostedTreesClassifier"
+    mtype: "<type \'type\'>"
+  }
+  member {
+    name: "BoostedTreesRegressor"
+    mtype: "<type \'type\'>"
+  }
   member {
     name: "DNNClassifier"
     mtype: "<type \'type\'>"
-- 
GitLab


From 33603b1983e8497601d2a256d6347684cf7d0bdf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 09:46:06 -0700
Subject: [PATCH 675/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 190942952

---
 tensorflow/go/op/wrappers.go | 26768 ++++++++++++++++-----------------
 1 file changed, 13384 insertions(+), 13384 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index 838f4f2301..a33703ad6f 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1320,201 +1320,134 @@ func PadV2(scope *Scope, input tf.Output, paddings tf.Output, constant_values tf
 	return op.Output(0)
 }
 
-// Returns the complex conjugate of a complex number.
-//
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// complex numbers that are the complex conjugate of each element in `input`. The
-// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
-// real part and *b* is the imaginary part.
-//
-// The complex conjugate returned by this operation is of the form \\(a - bj\\).
-//
-// For example:
+// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
-// ```
-func Conj(scope *Scope, input tf.Output) (output tf.Output) {
+// This is typically used by gradient computations for a broadcasting operation.
+func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Conj",
+		Type: "BroadcastGradientArgs",
 		Input: []tf.Input{
-			input,
+			s0, s1,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
-type ResourceSparseApplyMomentumAttr func(optionalAttr)
-
-// ResourceSparseApplyMomentumUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyMomentumUseLocking(value bool) ResourceSparseApplyMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// ResourceSparseApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
-//
-// value: If `True`, the tensor passed to compute grad will be
-// var - lr * momentum * accum, so in the end, the var you get is actually
-// var - lr * momentum * accum.
-// If not specified, defaults to false
-func ResourceSparseApplyMomentumUseNesterov(value bool) ResourceSparseApplyMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
+	return op.Output(0), op.Output(1)
 }
 
-// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
-//
-// Set use_nesterov = True if you want to use Nesterov momentum.
-//
-// That is for rows we have grad for, we update var and accum as follows:
-//
-// accum = accum * momentum + grad
-// var -= lr * accum
+// Returns the gradient of `Tile`.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	momentum: Momentum. Must be a scalar.
+// DEPRECATED at GraphDef version 3: TileGrad has been replaced with reduce_sum
 //
-// Returns the created operation.
-func ResourceSparseApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyMomentumAttr) (o *tf.Operation) {
+// Since `Tile` takes an input and repeats the input `multiples` times
+// along each dimension, `TileGrad` takes in `multiples` and aggregates
+// each repeated tile of `input` into `output`.
+func TileGrad(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyMomentum",
+		Type: "TileGrad",
 		Input: []tf.Input{
-			var_, accum, lr, grad, indices, momentum,
+			input, multiples,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Creates a sequence of numbers.
-//
-// This operation creates a sequence of numbers that begins at `start` and
-// extends by increments of `delta` up to but not including `limit`.
-//
-// For example:
+// Constructs a tensor by tiling a given tensor.
 //
-// ```
-// # 'start' is 3
-// # 'limit' is 18
-// # 'delta' is 3
-// tf.range(start, limit, delta) ==> [3, 6, 9, 12, 15]
-// ```
+// This operation creates a new tensor by replicating `input` `multiples` times.
+// The output tensor's i'th dimension has `input.dims(i) * multiples[i]` elements,
+// and the values of `input` are replicated `multiples[i]` times along the 'i'th
+// dimension. For example, tiling `[a b c d]` by `[2]` produces
+// `[a b c d a b c d]`.
 //
 // Arguments:
-//	start: 0-D (scalar). First entry in the sequence.
-//	limit: 0-D (scalar). Upper limit of sequence, exclusive.
-//	delta: 0-D (scalar). Optional. Default is 1. Number that increments `start`.
-//
-// Returns 1-D.
-func Range(scope *Scope, start tf.Output, limit tf.Output, delta tf.Output) (output tf.Output) {
+//	input: 1-D or higher.
+//	multiples: 1-D. Length must be the same as the number of dimensions in `input`
+func Tile(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Range",
+		Type: "Tile",
 		Input: []tf.Input{
-			start, limit, delta,
+			input, multiples,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes gradients for SparseSegmentSqrtN.
-//
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
-//
-// Arguments:
-//	grad: gradient propagated to the SparseSegmentSqrtN op.
-//	indices: indices passed to the corresponding SparseSegmentSqrtN op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentSqrtN op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentSqrtN op.
-func SparseSegmentSqrtNGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// StridedSliceGradAttr is an optional argument to StridedSliceGrad.
+type StridedSliceGradAttr func(optionalAttr)
+
+// StridedSliceGradBeginMask sets the optional begin_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradBeginMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["begin_mask"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtNGrad",
-		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
-		},
+}
+
+// StridedSliceGradEndMask sets the optional end_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradEndMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["end_mask"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the mean along sparse segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
-// dimension, selecting a subset of dimension 0, specified by `indices`.
-//
-// Arguments:
-//
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentMean(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// StridedSliceGradEllipsisMask sets the optional ellipsis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradEllipsisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["ellipsis_mask"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentMean",
-		Input: []tf.Input{
-			data, indices, segment_ids,
-		},
+}
+
+// StridedSliceGradNewAxisMask sets the optional new_axis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradNewAxisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["new_axis_mask"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Pop the element at the top of the stack.
+// StridedSliceGradShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
+// If not specified, defaults to 0
+func StridedSliceGradShrinkAxisMask(value int64) StridedSliceGradAttr {
+	return func(m optionalAttr) {
+		m["shrink_axis_mask"] = value
+	}
+}
+
+// Returns the gradient of `StridedSlice`.
 //
-// Arguments:
-//	handle: The handle to a stack.
-//	elem_type: The type of the elem that is popped.
+// Since `StridedSlice` cuts out pieces of its `input` which is size
+// `shape`, its gradient will have the same shape (which is passed here
+// as `shape`). The gradient will be zero in any element that the slice
+// does not select.
 //
-// Returns The tensor that is popped from the top of the stack.
-func StackPopV2(scope *Scope, handle tf.Output, elem_type tf.DataType) (elem tf.Output) {
+// Arguments are the same as StridedSliceGrad with the exception that
+// `dy` is the input gradient to be propagated and `shape` is the
+// shape of `StridedSlice`'s `input`.
+func StridedSliceGrad(scope *Scope, shape tf.Output, begin tf.Output, end tf.Output, strides tf.Output, dy tf.Output, optional ...StridedSliceGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"elem_type": elem_type}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "StackPopV2",
+		Type: "StridedSliceGrad",
 		Input: []tf.Input{
-			handle,
+			shape, begin, end, strides, dy,
 		},
 		Attrs: attrs,
 	}
@@ -1522,104 +1455,181 @@ func StackPopV2(scope *Scope, handle tf.Output, elem_type tf.DataType) (elem tf.
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor.
-//
-// Like `SparseSegmentSum`, but allows missing ids in `segment_ids`. If an id is
-// misisng, the `output` tensor at that position will be zeroed.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// StridedSliceAttr is an optional argument to StridedSlice.
+type StridedSliceAttr func(optionalAttr)
+
+// StridedSliceBeginMask sets the optional begin_mask attribute to value.
 //
-// For example:
+// value: a bitmask where a bit i being 1 means to ignore the begin
+// value and instead use the largest interval possible. At runtime
+// begin[i] will be replaced with `[0, n-1) if `stride[i] > 0` or
+// `[-1, n-1]` if `stride[i] < 0`
+// If not specified, defaults to 0
+func StridedSliceBeginMask(value int64) StridedSliceAttr {
+	return func(m optionalAttr) {
+		m["begin_mask"] = value
+	}
+}
+
+// StridedSliceEndMask sets the optional end_mask attribute to value.
 //
-// ```python
-// c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
+// value: analogous to `begin_mask`
+// If not specified, defaults to 0
+func StridedSliceEndMask(value int64) StridedSliceAttr {
+	return func(m optionalAttr) {
+		m["end_mask"] = value
+	}
+}
+
+// StridedSliceEllipsisMask sets the optional ellipsis_mask attribute to value.
 //
-// tf.sparse_segment_sum_with_num_segments(
-//     c, tf.constant([0, 1]), tf.constant([0, 0]), num_segments=3)
-// # => [[0 0 0 0]
-// #     [0 0 0 0]
-// #     [0 0 0 0]]
-//
-// tf.sparse_segment_sum_with_num_segments(c,
-//                                         tf.constant([0, 1]),
-//                                         tf.constant([0, 2],
-//                                         num_segments=4))
-// # => [[ 1  2  3  4]
-// #     [ 0  0  0  0]
-// #     [-1 -2 -3 -4]
-// #     [ 0  0  0  0]]
-// ```
-//
-// Arguments:
-//
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//	num_segments: Should equal the number of distinct segment IDs.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `num_segments`.
-func SparseSegmentSumWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentSumWithNumSegments",
-		Input: []tf.Input{
-			data, indices, segment_ids, num_segments,
-		},
+// value: a bitmask where bit `i` being 1 means the `i`th
+// position is actually an ellipsis. One bit at most can be 1.
+// If `ellipsis_mask == 0`, then an implicit ellipsis mask of `1 << (m+1)`
+// is provided. This means that `foo[3:5] == foo[3:5, ...]`. An ellipsis
+// implicitly creates as many range specifications as necessary to fully
+// specify the sliced range for every dimension. For example for a 4-dimensional
+// tensor `foo` the slice `foo[2, ..., 5:8]` implies `foo[2, :, :, 5:8]`.
+// If not specified, defaults to 0
+func StridedSliceEllipsisMask(value int64) StridedSliceAttr {
+	return func(m optionalAttr) {
+		m["ellipsis_mask"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// SparseToDenseAttr is an optional argument to SparseToDense.
-type SparseToDenseAttr func(optionalAttr)
+// StridedSliceNewAxisMask sets the optional new_axis_mask attribute to value.
+//
+// value: a bitmask where bit `i` being 1 means the `i`th
+// specification creates a new shape 1 dimension. For example
+// `foo[:4, tf.newaxis, :2]` would produce a shape `(4, 1, 2)` tensor.
+// If not specified, defaults to 0
+func StridedSliceNewAxisMask(value int64) StridedSliceAttr {
+	return func(m optionalAttr) {
+		m["new_axis_mask"] = value
+	}
+}
 
-// SparseToDenseValidateIndices sets the optional validate_indices attribute to value.
+// StridedSliceShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
 //
-// value: If true, indices are checked to make sure they are sorted in
-// lexicographic order and that there are no repeats.
-// If not specified, defaults to true
-func SparseToDenseValidateIndices(value bool) SparseToDenseAttr {
+// value: a bitmask where bit `i` implies that the `i`th
+// specification should shrink the dimensionality. begin and end
+// must imply a slice of size 1 in the dimension. For example in
+// python one might do `foo[:, 3, :]` which would result in
+// `shrink_axis_mask` being 2.
+// If not specified, defaults to 0
+func StridedSliceShrinkAxisMask(value int64) StridedSliceAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["shrink_axis_mask"] = value
 	}
 }
 
-// Converts a sparse representation into a dense tensor.
+// Return a strided slice from `input`.
 //
-// Builds an array `dense` with shape `output_shape` such that
+// Note, most python users will want to use the Python `Tensor.__getitem__`
+// or `Variable.__getitem__` rather than this op directly.
 //
-// ```
-// # If sparse_indices is scalar
-// dense[i] = (i == sparse_indices ? sparse_values : default_value)
+// The goal of this op is to produce a new tensor with a subset of
+// the elements from the `n` dimensional `input` tensor. The subset is chosen using
+// a sequence of `m` sparse range specifications encoded into the arguments
+// of this function. Note, in some cases
+// `m` could be equal to `n`, but this need not be the case. Each
+// range specification entry can be one of the following:
 //
-// # If sparse_indices is a vector, then for each i
-// dense[sparse_indices[i]] = sparse_values[i]
+// - An ellipsis (...). Ellipses are used to imply zero or more
+//   dimensions of full-dimension selection and are produced using
+//   `ellipsis_mask`. For example, `foo[...]` is the identity slice.
 //
-// # If sparse_indices is an n by d matrix, then for each i in [0, n)
-// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i]
+// - A new axis. This is used to insert a new shape=1 dimension and is
+//   produced using `new_axis_mask`. For example, `foo[:, ...]` where
+//   `foo` is shape `(3, 4)` produces a `(1, 3, 4)` tensor.
+//
+//
+// - A range `begin:end:stride`. This is used to specify how much to choose from
+//   a given dimension. `stride` can be any integer but 0.  `begin` is an integer
+//   which represents the index of the first value to select while `end` represents
+//   the index of the last value to select. The number of values selected in each
+//   dimension is `end - begin` if `stride > 0` and `begin - end` if `stride < 0`.
+//   `begin` and `end` can be negative where `-1` is the last element, `-2` is
+//   the second to last. `begin_mask` controls whether to replace the explicitly
+//   given `begin` with an implicit effective value of `0` if `stride > 0` and
+//   `-1` if `stride < 0`. `end_mask` is analogous but produces the number
+//   required to create the largest open interval. For example, given a shape
+//   `(3,)` tensor `foo[:]`, the effective `begin` and `end` are `0` and `3`. Do
+//   not assume this is equivalent to `foo[0:-1]` which has an effective `begin`
+//   and `end` of `0` and `2`. Another example is `foo[-2::-1]` which reverses the
+//   first dimension of a tensor while dropping the last two (in the original
+//   order elements). For example `foo = [1,2,3,4]; foo[-2::-1]` is `[4,3]`.
+//
+// - A single index. This is used to keep only elements that have a given
+//   index. For example (`foo[2, :]` on a shape `(5,6)` tensor produces a
+//   shape `(6,)` tensor. This is encoded in `begin` and `end` and
+//   `shrink_axis_mask`.
+//
+// Each conceptual range specification is encoded in the op's argument. This
+// encoding is best understand by considering a non-trivial example. In
+// particular,
+// `foo[1, 2:4, None, ..., :-3:-1, :]` will be encoded as
+//
+// ```
+// begin = [1, 2, x, x, 0, x] # x denotes don't care (usually 0)
+// end = [2, 4, x, x, -3, x]
+// strides = [1, 1, x, x, -1, 1]
+// begin_mask = 1<<4 | 1 << 5 = 48
+// end_mask = 1<<5 = 32
+// ellipsis_mask = 1<<3 = 8
+// new_axis_mask = 1<<2 4
+// shrink_axis_mask = 1<<0
 // ```
 //
-// All other values in `dense` are set to `default_value`.  If `sparse_values` is a
-// scalar, all sparse indices are set to this single value.
+// In this case if `foo.shape` is (5, 5, 5, 5, 5, 5) the final shape of
+// the slice becomes (2, 1, 5, 5, 2, 5).
+// Let us walk step by step through each argument specification.
 //
-// Indices should be sorted in lexicographic order, and indices must not
-// contain any repeats. If `validate_indices` is true, these properties
-// are checked during execution.
+// 1.  The first argument in the example slice is turned into `begin = 1` and
+// `end = begin + 1 = 2`. To disambiguate from the original spec `2:4` we
+// also set the appropriate bit in `shrink_axis_mask`.
+//
+// 2. `2:4` is contributes 2, 4, 1 to begin, end, and stride. All masks have
+// zero bits contributed.
+//
+// 3. None is a synonym for `tf.newaxis`. This means insert a dimension of size 1
+// dimension in the final shape. Dummy values are contributed to begin,
+// end and stride, while the new_axis_mask bit is set.
+//
+// 4. `...` grab the full ranges from as many dimensions as needed to
+// fully specify a slice for every dimension of the input shape.
+//
+// 5. `:-3:-1` shows the use of negative indices. A negative index `i` associated
+// with a dimension that has shape `s` is converted to a positive index
+// `s + i`. So `-1` becomes `s-1` (i.e. the last element). This conversion
+// is done internally so begin, end and strides receive x, -3, and -1.
+// The appropriate begin_mask bit is set to indicate the start range is the
+// full range (ignoring the x).
+//
+// 6. `:` indicates that the entire contents of the corresponding dimension
+// is selected. This is equivalent to `::` or `0::1`. begin, end, and strides
+// receive 0, 0, and 1, respectively. The appropriate bits in `begin_mask` and
+// `end_mask` are also set.
+//
+// *Requirements*:
+//   `0 != strides[i] for i in [0, m)`
+//   `ellipsis_mask must be a power of two (only one ellipsis)`
 //
 // Arguments:
-//	sparse_indices: 0-D, 1-D, or 2-D.  `sparse_indices[i]` contains the complete
-// index where `sparse_values[i]` will be placed.
-//	output_shape: 1-D.  Shape of the dense output tensor.
-//	sparse_values: 1-D.  Values corresponding to each row of `sparse_indices`,
-// or a scalar value to be used for all sparse indices.
-//	default_value: Scalar value to set for indices not specified in
-// `sparse_indices`.
 //
-// Returns Dense output tensor of shape `output_shape`.
-func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) {
+//	begin: `begin[k]` specifies the offset into the `k`th range specification.
+// The exact dimension this corresponds to will be determined by context.
+// Out-of-bounds values will be silently clamped. If the `k`th bit of
+// `begin_mask` then `begin[k]` is ignored and the full range of the
+// appropriate dimension is used instead. Negative values causes indexing
+// to start from the highest element e.g. If `foo==[1,2,3]` then `foo[-1]==3`.
+//	end: `end[i]` is like `begin` with the exception that `end_mask` is
+// used to determine full ranges.
+//	strides: `strides[i]` specifies the increment in the `i`th specification
+// after extracting a given element. Negative indices will reverse
+// the original order. Out or range values are
+// clamped to `[0,dim[i]) if slice[i]>0` or `[-1,dim[i]-1] if slice[i] < 0`
+func StridedSlice(scope *Scope, input tf.Output, begin tf.Output, end tf.Output, strides tf.Output, optional ...StridedSliceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -1628,9 +1638,9 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseToDense",
+		Type: "StridedSlice",
 		Input: []tf.Input{
-			sparse_indices, output_shape, sparse_values, default_value,
+			input, begin, end, strides,
 		},
 		Attrs: attrs,
 	}
@@ -1638,227 +1648,273 @@ func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Outpu
 	return op.Output(0)
 }
 
-// Counts the number of occurrences of each value in an integer array.
+// Return a slice from 'input'.
 //
-// Outputs a vector with length `size` and the same dtype as `weights`. If
-// `weights` are empty, then index `i` stores the number of times the value `i` is
-// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of
-// the value in `weights` at each index where the corresponding value in `arr` is
-// `i`.
+// The output tensor is a tensor with dimensions described by 'size'
+// whose values are extracted from 'input' starting at the offsets in
+// 'begin'.
 //
-// Values in `arr` outside of the range [0, size) are ignored.
+// *Requirements*:
+//   0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)
 //
 // Arguments:
-//	arr: int32 `Tensor`.
-//	size: non-negative int32 scalar `Tensor`.
-//	weights: is an int32, int64, float32, or float64 `Tensor` with the same
-// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights
-// equal to 1.
 //
-// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for
-// each value in the range [0, size).
-func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) {
+//	begin: begin[i] specifies the offset into the 'i'th dimension of
+// 'input' to slice from.
+//	size: size[i] specifies the number of elements of the 'i'th dimension
+// of 'input' to slice. If size[i] is -1, all remaining elements in dimension
+// i are included in the slice (i.e. this is equivalent to setting
+// size[i] = input.dim_size(i) - begin[i]).
+func Slice(scope *Scope, input tf.Output, begin tf.Output, size tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Bincount",
+		Type: "Slice",
 		Input: []tf.Input{
-			arr, size, weights,
+			input, begin, size,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// SizeAttr is an optional argument to Size.
+type SizeAttr func(optionalAttr)
+
+// SizeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func SizeOutType(value tf.DataType) SizeAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Returns the size of a tensor.
 //
-// Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
-// dimension, selecting a subset of dimension 0, specified by `indices`.
+// This operation returns an integer representing the number of elements in
+// `input`.
 //
 // For example:
 //
-// ```python
-// c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
-//
-// # Select two rows, one segment.
-// tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0]))
-// # => [[0 0 0 0]]
-//
-// # Select two rows, two segment.
-// tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 1]))
-// # => [[ 1  2  3  4]
-// #     [-1 -2 -3 -4]]
-//
-// # Select all rows, two segments.
-// tf.sparse_segment_sum(c, tf.constant([0, 1, 2]), tf.constant([0, 0, 1]))
-// # => [[0 0 0 0]
-// #     [5 6 7 8]]
-//
-// # Which is equivalent to:
-// tf.segment_sum(c, tf.constant([0, 0, 1]))
 // ```
-//
-// Arguments:
-//
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentSum(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+// # 't' is [[[1, 1,, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]]
+// size(t) ==> 12
+// ```
+func Size(scope *Scope, input tf.Output, optional ...SizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSum",
+		Type: "Size",
 		Input: []tf.Input{
-			data, indices, segment_ids,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes hyperbolic sine of x element-wise.
-func Sinh(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns the complex conjugate of a complex number.
+//
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// complex numbers that are the complex conjugate of each element in `input`. The
+// complex numbers in `input` must be of the form \\(a + bj\\), where *a* is the
+// real part and *b* is the imaginary part.
+//
+// The complex conjugate returned by this operation is of the form \\(a - bj\\).
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.conj(input) ==> [-2.25 - 4.75j, 3.25 - 5.75j]
+// ```
+func Conj(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Sinh",
+		Type: "Conj",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes rectified linear 6: `min(max(features, 0), 6)`.
-func Relu6(scope *Scope, features tf.Output) (activations tf.Output) {
+// ResourceSparseApplyMomentumAttr is an optional argument to ResourceSparseApplyMomentum.
+type ResourceSparseApplyMomentumAttr func(optionalAttr)
+
+// ResourceSparseApplyMomentumUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyMomentumUseLocking(value bool) ResourceSparseApplyMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceSparseApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, the tensor passed to compute grad will be
+// var - lr * momentum * accum, so in the end, the var you get is actually
+// var - lr * momentum * accum.
+// If not specified, defaults to false
+func ResourceSparseApplyMomentumUseNesterov(value bool) ResourceSparseApplyMomentumAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update relevant entries in '*var' and '*accum' according to the momentum scheme.
+//
+// Set use_nesterov = True if you want to use Nesterov momentum.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+//
+// accum = accum * momentum + grad
+// var -= lr * accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, momentum tf.Output, optional ...ResourceSparseApplyMomentumAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Relu6",
+		Type: "ResourceSparseApplyMomentum",
 		Input: []tf.Input{
-			features,
+			var_, accum, lr, grad, indices, momentum,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Computes the sum along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// `(output[i] = sum_{j...} data[j...]` where the sum is over tuples `j...` such
-// that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
-// need not be sorted and need not cover all values in the full
-// range of valid values.
+// Creates a sequence of numbers.
 //
-// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
-// If the given segment ID `i` is negative, the value is dropped and will not be
-// added to the sum of the segment.
+// This operation creates a sequence of numbers that begins at `start` and
+// extends by increments of `delta` up to but not including `limit`.
 //
-// `num_segments` should equal the number of distinct segment IDs.
+// For example:
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
-// </div>
+// ```
+// # 'start' is 3
+// # 'limit' is 18
+// # 'delta' is 3
+// tf.range(start, limit, delta) ==> [3, 6, 9, 12, 15]
+// ```
 //
 // Arguments:
+//	start: 0-D (scalar). First entry in the sequence.
+//	limit: 0-D (scalar). Upper limit of sequence, exclusive.
+//	delta: 0-D (scalar). Optional. Default is 1. Number that increments `start`.
 //
-//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
-//
-//
-// Returns Has same shape as data, except for the first `segment_ids.rank`
-// dimensions, which are replaced with a single dimension which has size
-// `num_segments`.
-func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+// Returns 1-D.
+func Range(scope *Scope, start tf.Output, limit tf.Output, delta tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentSum",
+		Type: "Range",
 		Input: []tf.Input{
-			data, segment_ids, num_segments,
+			start, limit, delta,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns which elements of x are finite.
+// Computes gradients for SparseSegmentSqrtN.
 //
-// @compatibility(numpy)
-// Equivalent to np.isfinite
-// @end_compatibility
-func IsFinite(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
+//
+// Arguments:
+//	grad: gradient propagated to the SparseSegmentSqrtN op.
+//	indices: indices passed to the corresponding SparseSegmentSqrtN op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentSqrtN op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentSqrtN op.
+func SparseSegmentSqrtNGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IsFinite",
+		Type: "SparseSegmentSqrtNGrad",
 		Input: []tf.Input{
-			x,
+			grad, indices, segment_ids, output_dim0,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatMulAttr is an optional argument to MatMul.
-type MatMulAttr func(optionalAttr)
-
-// MatMulTransposeA sets the optional transpose_a attribute to value.
+// Computes the mean along sparse segments of a tensor.
 //
-// value: If true, "a" is transposed before multiplication.
-// If not specified, defaults to false
-func MatMulTransposeA(value bool) MatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
-	}
-}
-
-// MatMulTransposeB sets the optional transpose_b attribute to value.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// value: If true, "b" is transposed before multiplication.
-// If not specified, defaults to false
-func MatMulTransposeB(value bool) MatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_b"] = value
+// Like `SegmentMean`, but `segment_ids` can have rank less than `data`'s first
+// dimension, selecting a subset of dimension 0, specified by `indices`.
+//
+// Arguments:
+//
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SparseSegmentMean(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseSegmentMean",
+		Input: []tf.Input{
+			data, indices, segment_ids,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Multiply the matrix "a" by the matrix "b".
+// Pop the element at the top of the stack.
 //
-// The inputs must be two-dimensional matrices and the inner dimension of
-// "a" (after being transposed if transpose_a is true) must match the
-// outer dimension of "b" (after being transposed if transposed_b is
-// true).
+// Arguments:
+//	handle: The handle to a stack.
+//	elem_type: The type of the elem that is popped.
 //
-// *Note*: The default kernel implementation for MatMul on GPUs uses
-// cublas.
-func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (product tf.Output) {
+// Returns The tensor that is popped from the top of the stack.
+func StackPopV2(scope *Scope, handle tf.Output, elem_type tf.DataType) (elem tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"elem_type": elem_type}
 	opspec := tf.OpSpec{
-		Type: "MatMul",
+		Type: "StackPopV2",
 		Input: []tf.Input{
-			a, b,
+			handle,
 		},
 		Attrs: attrs,
 	}
@@ -1866,303 +1922,372 @@ func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (pro
 	return op.Output(0)
 }
 
-// Selects elements from `x` or `y`, depending on `condition`.
+// Computes the sum along sparse segments of a tensor.
 //
-// The `x`, and `y` tensors must all have the same shape, and the
-// output will also have that shape.
+// Like `SparseSegmentSum`, but allows missing ids in `segment_ids`. If an id is
+// misisng, the `output` tensor at that position will be zeroed.
 //
-// The `condition` tensor must be a scalar if `x` and `y` are scalars.
-// If `x` and `y` are vectors or higher rank, then `condition` must be either a
-// scalar, a vector with size matching the first dimension of `x`, or must have
-// the same shape as `x`.
-//
-// The `condition` tensor acts as a mask that chooses, based on the value at each
-// element, whether the corresponding element / row in the output should be
-// taken from `x` (if true) or `y` (if false).
-//
-// If `condition` is a vector and `x` and `y` are higher rank matrices, then
-// it chooses which row (outer dimension) to copy from `x` and `y`.
-// If `condition` has the same shape as `x` and `y`, then it chooses which
-// element to copy from `x` and `y`.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
 // For example:
 //
 // ```python
-// # 'condition' tensor is [[True,  False]
-// #                        [False, True]]
-// # 't' is [[1, 2],
-// #         [3, 4]]
-// # 'e' is [[5, 6],
-// #         [7, 8]]
-// select(condition, t, e)  # => [[1, 6], [7, 4]]
-//
+// c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
 //
-// # 'condition' tensor is [True, False]
-// # 't' is [[1, 2],
-// #         [3, 4]]
-// # 'e' is [[5, 6],
-// #         [7, 8]]
-// select(condition, t, e) ==> [[1, 2],
-//                              [7, 8]]
+// tf.sparse_segment_sum_with_num_segments(
+//     c, tf.constant([0, 1]), tf.constant([0, 0]), num_segments=3)
+// # => [[0 0 0 0]
+// #     [0 0 0 0]
+// #     [0 0 0 0]]
 //
+// tf.sparse_segment_sum_with_num_segments(c,
+//                                         tf.constant([0, 1]),
+//                                         tf.constant([0, 2],
+//                                         num_segments=4))
+// # => [[ 1  2  3  4]
+// #     [ 0  0  0  0]
+// #     [-1 -2 -3 -4]
+// #     [ 0  0  0  0]]
 // ```
 //
 // Arguments:
 //
-//	x: = A `Tensor` which may have the same shape as `condition`.
-// If `condition` is rank 1, `x` may have higher rank,
-// but its first dimension must match the size of `condition`.
-//	y: = A `Tensor` with the same type and shape as `x`.
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	num_segments: Should equal the number of distinct segment IDs.
 //
-// Returns = A `Tensor` with the same type and shape as `x` and `y`.
-func Select(scope *Scope, condition tf.Output, x tf.Output, y tf.Output) (output tf.Output) {
+// Returns Has same shape as data, except for dimension 0 which
+// has size `num_segments`.
+func SparseSegmentSumWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Select",
+		Type: "SparseSegmentSumWithNumSegments",
 		Input: []tf.Input{
-			condition, x, y,
+			data, indices, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the truth value of x OR y element-wise.
+// SparseToDenseAttr is an optional argument to SparseToDense.
+type SparseToDenseAttr func(optionalAttr)
+
+// SparseToDenseValidateIndices sets the optional validate_indices attribute to value.
 //
-// *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func LogicalOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// value: If true, indices are checked to make sure they are sorted in
+// lexicographic order and that there are no repeats.
+// If not specified, defaults to true
+func SparseToDenseValidateIndices(value bool) SparseToDenseAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Converts a sparse representation into a dense tensor.
+//
+// Builds an array `dense` with shape `output_shape` such that
+//
+// ```
+// # If sparse_indices is scalar
+// dense[i] = (i == sparse_indices ? sparse_values : default_value)
+//
+// # If sparse_indices is a vector, then for each i
+// dense[sparse_indices[i]] = sparse_values[i]
+//
+// # If sparse_indices is an n by d matrix, then for each i in [0, n)
+// dense[sparse_indices[i][0], ..., sparse_indices[i][d-1]] = sparse_values[i]
+// ```
+//
+// All other values in `dense` are set to `default_value`.  If `sparse_values` is a
+// scalar, all sparse indices are set to this single value.
+//
+// Indices should be sorted in lexicographic order, and indices must not
+// contain any repeats. If `validate_indices` is true, these properties
+// are checked during execution.
+//
+// Arguments:
+//	sparse_indices: 0-D, 1-D, or 2-D.  `sparse_indices[i]` contains the complete
+// index where `sparse_values[i]` will be placed.
+//	output_shape: 1-D.  Shape of the dense output tensor.
+//	sparse_values: 1-D.  Values corresponding to each row of `sparse_indices`,
+// or a scalar value to be used for all sparse indices.
+//	default_value: Scalar value to set for indices not specified in
+// `sparse_indices`.
+//
+// Returns Dense output tensor of shape `output_shape`.
+func SparseToDense(scope *Scope, sparse_indices tf.Output, output_shape tf.Output, sparse_values tf.Output, default_value tf.Output, optional ...SparseToDenseAttr) (dense tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LogicalOr",
+		Type: "SparseToDense",
 		Input: []tf.Input{
-			x, y,
+			sparse_indices, output_shape, sparse_values, default_value,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
-//
-// The regularized incomplete beta integral is defined as:
-//
-//
-// \\(I_x(a, b) = \frac{B(x; a, b)}{B(a, b)}\\)
-//
-// where
+// Counts the number of occurrences of each value in an integer array.
 //
+// Outputs a vector with length `size` and the same dtype as `weights`. If
+// `weights` are empty, then index `i` stores the number of times the value `i` is
+// counted in `arr`. If `weights` are non-empty, then index `i` stores the sum of
+// the value in `weights` at each index where the corresponding value in `arr` is
+// `i`.
 //
-// \\(B(x; a, b) = \int_0^x t^{a-1} (1 - t)^{b-1} dt\\)
+// Values in `arr` outside of the range [0, size) are ignored.
 //
+// Arguments:
+//	arr: int32 `Tensor`.
+//	size: non-negative int32 scalar `Tensor`.
+//	weights: is an int32, int64, float32, or float64 `Tensor` with the same
+// shape as `arr`, or a length-0 `Tensor`, in which case it acts as all weights
+// equal to 1.
 //
-// is the incomplete beta function and \\(B(a, b)\\) is the *complete*
-// beta function.
-func Betainc(scope *Scope, a tf.Output, b tf.Output, x tf.Output) (z tf.Output) {
+// Returns 1D `Tensor` with length equal to `size`. The counts or summed weights for
+// each value in the range [0, size).
+func Bincount(scope *Scope, arr tf.Output, size tf.Output, weights tf.Output) (bins tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Betainc",
+		Type: "Bincount",
 		Input: []tf.Input{
-			a, b, x,
+			arr, size, weights,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
-//
-// N is the size of the segment being reduced.
-//
-// Like `SparseSegmentSqrtN`, but allows missing ids in `segment_ids`. If an id is
-// misisng, the `output` tensor at that position will be zeroed.
+// Computes the sum along sparse segments of a tensor.
 //
 // Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
 // segments.
 //
+// Like `SegmentSum`, but `segment_ids` can have rank less than `data`'s first
+// dimension, selecting a subset of dimension 0, specified by `indices`.
+//
+// For example:
+//
+// ```python
+// c = tf.constant([[1,2,3,4], [-1,-2,-3,-4], [5,6,7,8]])
+//
+// # Select two rows, one segment.
+// tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 0]))
+// # => [[0 0 0 0]]
+//
+// # Select two rows, two segment.
+// tf.sparse_segment_sum(c, tf.constant([0, 1]), tf.constant([0, 1]))
+// # => [[ 1  2  3  4]
+// #     [-1 -2 -3 -4]]
+//
+// # Select all rows, two segments.
+// tf.sparse_segment_sum(c, tf.constant([0, 1, 2]), tf.constant([0, 0, 1]))
+// # => [[0 0 0 0]
+// #     [5 6 7 8]]
+//
+// # Which is equivalent to:
+// tf.segment_sum(c, tf.constant([0, 0, 1]))
+// ```
+//
 // Arguments:
 //
 //	indices: A 1-D tensor. Has same rank as `segment_ids`.
 //	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//	num_segments: Should equal the number of distinct segment IDs.
 //
 // Returns Has same shape as data, except for dimension 0 which
 // has size `k`, the number of segments.
-func SparseSegmentSqrtNWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+func SparseSegmentSum(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtNWithNumSegments",
+		Type: "SparseSegmentSum",
 		Input: []tf.Input{
-			data, indices, segment_ids, num_segments,
+			data, indices, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the upper regularized incomplete Gamma function `Q(a, x)`.
-//
-// The upper regularized incomplete Gamma function is defined as:
-//
-// \\(Q(a, x) = Gamma(a, x) / Gamma(a) = 1 - P(a, x)\\)
-//
-// where
-//
-// \\(Gamma(a, x) = int_{x}^{\infty} t^{a-1} exp(-t) dt\\)
-//
-// is the upper incomplete Gama function.
-//
-// Note, above `P(a, x)` (`Igamma`) is the lower regularized complete
-// Gamma function.
-func Igammac(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// Computes hyperbolic sine of x element-wise.
+func Sinh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Igammac",
+		Type: "Sinh",
 		Input: []tf.Input{
-			a, x,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// LogUniformCandidateSamplerAttr is an optional argument to LogUniformCandidateSampler.
-type LogUniformCandidateSamplerAttr func(optionalAttr)
-
-// LogUniformCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func LogUniformCandidateSamplerSeed(value int64) LogUniformCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// Computes rectified linear 6: `min(max(features, 0), 6)`.
+func Relu6(scope *Scope, features tf.Output) (activations tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// LogUniformCandidateSamplerSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func LogUniformCandidateSamplerSeed2(value int64) LogUniformCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+	opspec := tf.OpSpec{
+		Type: "Relu6",
+		Input: []tf.Input{
+			features,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Generates labels for candidate sampling with a log-uniform distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
+// Computes the sum along segments of a tensor.
 //
-// For each batch, this op picks a single set of sampled candidate labels.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+// Computes a tensor such that
+// `(output[i] = sum_{j...} data[j...]` where the sum is over tuples `j...` such
+// that `segment_ids[j...] == i`.  Unlike `SegmentSum`, `segment_ids`
+// need not be sorted and need not cover all values in the full
+// range of valid values.
 //
-// Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
+// If the given segment ID `i` is negative, the value is dropped and will not be
+// added to the sum of the segment.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func LogUniformCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...LogUniformCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// `num_segments` should equal the number of distinct segment IDs.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentSum.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A tensor whose shape is a prefix of `data.shape`.
+//
+//
+// Returns Has same shape as data, except for the first `segment_ids.rank`
+// dimensions, which are replaced with a single dimension which has size
+// `num_segments`.
+func UnsortedSegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "LogUniformCandidateSampler",
+		Type: "UnsortedSegmentSum",
 		Input: []tf.Input{
-			true_classes,
+			data, segment_ids, num_segments,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Returns (x - y)(x - y) element-wise.
-//
-// *NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func SquaredDifference(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// ResourceStridedSliceAssignAttr is an optional argument to ResourceStridedSliceAssign.
+type ResourceStridedSliceAssignAttr func(optionalAttr)
+
+// ResourceStridedSliceAssignBeginMask sets the optional begin_mask attribute to value.
+// If not specified, defaults to 0
+func ResourceStridedSliceAssignBeginMask(value int64) ResourceStridedSliceAssignAttr {
+	return func(m optionalAttr) {
+		m["begin_mask"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SquaredDifference",
-		Input: []tf.Input{
-			x, y,
-		},
+}
+
+// ResourceStridedSliceAssignEndMask sets the optional end_mask attribute to value.
+// If not specified, defaults to 0
+func ResourceStridedSliceAssignEndMask(value int64) ResourceStridedSliceAssignAttr {
+	return func(m optionalAttr) {
+		m["end_mask"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Forwards the input to the output.
+// ResourceStridedSliceAssignEllipsisMask sets the optional ellipsis_mask attribute to value.
+// If not specified, defaults to 0
+func ResourceStridedSliceAssignEllipsisMask(value int64) ResourceStridedSliceAssignAttr {
+	return func(m optionalAttr) {
+		m["ellipsis_mask"] = value
+	}
+}
+
+// ResourceStridedSliceAssignNewAxisMask sets the optional new_axis_mask attribute to value.
+// If not specified, defaults to 0
+func ResourceStridedSliceAssignNewAxisMask(value int64) ResourceStridedSliceAssignAttr {
+	return func(m optionalAttr) {
+		m["new_axis_mask"] = value
+	}
+}
+
+// ResourceStridedSliceAssignShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
+// If not specified, defaults to 0
+func ResourceStridedSliceAssignShrinkAxisMask(value int64) ResourceStridedSliceAssignAttr {
+	return func(m optionalAttr) {
+		m["shrink_axis_mask"] = value
+	}
+}
+
+// Assign `value` to the sliced l-value reference of `ref`.
 //
-// This operator represents the loop termination condition used by the
-// "pivot" switches of a loop.
+// The values of `value` are assigned to the positions in the variable
+// `ref` that are selected by the slice parameters. The slice parameters
+// `begin, `end`, `strides`, etc. work exactly as in `StridedSlice`.
 //
-// Arguments:
-//	input: A boolean scalar, representing the branch predicate of the Switch op.
+// NOTE this op currently does not support broadcasting and so `value`'s
+// shape must be exactly the shape produced by the slice of `ref`.
 //
-// Returns The same tensor as `input`.
-func LoopCond(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns the created operation.
+func ResourceStridedSliceAssign(scope *Scope, ref tf.Output, begin tf.Output, end tf.Output, strides tf.Output, value tf.Output, optional ...ResourceStridedSliceAssignAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LoopCond",
+		Type: "ResourceStridedSliceAssign",
 		Input: []tf.Input{
-			input,
+			ref, begin, end, strides, value,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ApproximateEqualAttr is an optional argument to ApproximateEqual.
-type ApproximateEqualAttr func(optionalAttr)
+// ArgMaxAttr is an optional argument to ArgMax.
+type ArgMaxAttr func(optionalAttr)
 
-// ApproximateEqualTolerance sets the optional tolerance attribute to value.
-// If not specified, defaults to 1e-05
-func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
+// ArgMaxOutputType sets the optional output_type attribute to value.
+// If not specified, defaults to DT_INT64
+func ArgMaxOutputType(value tf.DataType) ArgMaxAttr {
 	return func(m optionalAttr) {
-		m["tolerance"] = value
+		m["output_type"] = value
 	}
 }
 
-// Returns the truth value of abs(x-y) < tolerance element-wise.
-func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
+// Returns the index with the largest value across dimensions of a tensor.
+//
+// Note that in case of ties the identity of the return value is not guaranteed.
+//
+// Arguments:
+//
+//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
+// Describes which dimension of the input Tensor to reduce across. For vectors,
+// use dimension = 0.
+func ArgMax(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMaxAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -2171,9 +2296,9 @@ func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...Approx
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ApproximateEqual",
+		Type: "ArgMax",
 		Input: []tf.Input{
-			x, y,
+			input, dimension,
 		},
 		Attrs: attrs,
 	}
@@ -2181,72 +2306,58 @@ func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...Approx
 	return op.Output(0)
 }
 
-// Returns x / y element-wise.
+// Returns which elements of x are finite.
 //
-// *NOTE*: `Div` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Div(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.isfinite
+// @end_compatibility
+func IsFinite(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Div",
+		Type: "IsFinite",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns x * y element-wise.
+// MatMulAttr is an optional argument to MatMul.
+type MatMulAttr func(optionalAttr)
+
+// MatMulTransposeA sets the optional transpose_a attribute to value.
 //
-// *NOTE*: `Multiply` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Mul(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Mul",
-		Input: []tf.Input{
-			x, y,
-		},
+// value: If true, "a" is transposed before multiplication.
+// If not specified, defaults to false
+func MatMulTransposeA(value bool) MatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_a"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// BiasAddAttr is an optional argument to BiasAdd.
-type BiasAddAttr func(optionalAttr)
-
-// BiasAddDataFormat sets the optional data_format attribute to value.
+// MatMulTransposeB sets the optional transpose_b attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the bias tensor will be added to the last dimension
-// of the value tensor.
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// The tensor will be added to "in_channels", the third-to-the-last
-//     dimension.
-// If not specified, defaults to "NHWC"
-func BiasAddDataFormat(value string) BiasAddAttr {
+// value: If true, "b" is transposed before multiplication.
+// If not specified, defaults to false
+func MatMulTransposeB(value bool) MatMulAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["transpose_b"] = value
 	}
 }
 
-// Adds `bias` to `value`.
-//
-// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
-// Broadcasting is supported, so `value` may have any number of dimensions.
+// Multiply the matrix "a" by the matrix "b".
 //
-// Arguments:
-//	value: Any number of dimensions.
-//	bias: 1-D with size the last dimension of `value`.
+// The inputs must be two-dimensional matrices and the inner dimension of
+// "a" (after being transposed if transpose_a is true) must match the
+// outer dimension of "b" (after being transposed if transposed_b is
+// true).
 //
-// Returns Broadcasted sum of `value` and `bias`.
-func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddAttr) (output tf.Output) {
+// *Note*: The default kernel implementation for MatMul on GPUs uses
+// cublas.
+func MatMul(scope *Scope, a tf.Output, b tf.Output, optional ...MatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -2255,9 +2366,9 @@ func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddA
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "BiasAdd",
+		Type: "MatMul",
 		Input: []tf.Input{
-			value, bias,
+			a, b,
 		},
 		Attrs: attrs,
 	}
@@ -2265,87 +2376,79 @@ func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddA
 	return op.Output(0)
 }
 
-// SparseReduceSumSparseAttr is an optional argument to SparseReduceSumSparse.
-type SparseReduceSumSparseAttr func(optionalAttr)
-
-// SparseReduceSumSparseKeepDims sets the optional keep_dims attribute to value.
+// Selects elements from `x` or `y`, depending on `condition`.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SparseReduceSumSparseKeepDims(value bool) SparseReduceSumSparseAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the sum of elements across dimensions of a SparseTensor.
+// The `x`, and `y` tensors must all have the same shape, and the
+// output will also have that shape.
 //
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_sum()`.  In contrast to SparseReduceSum, this Op returns a
-// SparseTensor.
+// The `condition` tensor must be a scalar if `x` and `y` are scalars.
+// If `x` and `y` are vectors or higher rank, then `condition` must be either a
+// scalar, a vector with size matching the first dimension of `x`, or must have
+// the same shape as `x`.
 //
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
+// The `condition` tensor acts as a mask that chooses, based on the value at each
+// element, whether the corresponding element / row in the output should be
+// taken from `x` (if true) or `y` (if false).
 //
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// If `condition` is a vector and `x` and `y` are higher rank matrices, then
+// it chooses which row (outer dimension) to copy from `x` and `y`.
+// If `condition` has the same shape as `x` and `y`, then it chooses which
+// element to copy from `x` and `y`.
+//
+// For example:
+//
+// ```python
+// # 'condition' tensor is [[True,  False]
+// #                        [False, True]]
+// # 't' is [[1, 2],
+// #         [3, 4]]
+// # 'e' is [[5, 6],
+// #         [7, 8]]
+// select(condition, t, e)  # => [[1, 6], [7, 4]]
+//
+//
+// # 'condition' tensor is [True, False]
+// # 't' is [[1, 2],
+// #         [3, 4]]
+// # 'e' is [[5, 6],
+// #         [7, 8]]
+// select(condition, t, e) ==> [[1, 2],
+//                              [7, 8]]
+//
+// ```
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
-func SparseReduceSumSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "SparseReduceSumSparse",
-		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// Returns x + y element-wise.
 //
-// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+//	x: = A `Tensor` which may have the same shape as `condition`.
+// If `condition` is rank 1, `x` may have higher rank,
+// but its first dimension must match the size of `condition`.
+//	y: = A `Tensor` with the same type and shape as `x`.
+//
+// Returns = A `Tensor` with the same type and shape as `x` and `y`.
+func Select(scope *Scope, condition tf.Output, x tf.Output, y tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AddV2",
+		Type: "Select",
 		Input: []tf.Input{
-			x, y,
+			condition, x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns x + y element-wise.
+// Returns the truth value of x OR y element-wise.
 //
-// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+// *NOTE*: `LogicalOr` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+func LogicalOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Add",
+		Type: "LogicalOr",
 		Input: []tf.Input{
 			x, y,
 		},
@@ -2354,219 +2457,201 @@ func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// NthElementAttr is an optional argument to NthElement.
-type NthElementAttr func(optionalAttr)
-
-// NthElementReverse sets the optional reverse attribute to value.
+// Compute the regularized incomplete beta integral \\(I_x(a, b)\\).
 //
-// value: When set to True, find the nth-largest value in the vector and vice
-// versa.
-// If not specified, defaults to false
-func NthElementReverse(value bool) NthElementAttr {
-	return func(m optionalAttr) {
-		m["reverse"] = value
-	}
-}
-
-// Finds values of the `n`-th order statistic for the last dimension.
+// The regularized incomplete beta integral is defined as:
 //
-// If the input is a vector (rank-1), finds the entries which is the nth-smallest
-// value in the vector and outputs their values as scalar tensor.
 //
-// For matrices (resp. higher rank input), computes the entries which is the
-// nth-smallest value in each row (resp. vector along the last dimension). Thus,
+// \\(I_x(a, b) = \frac{B(x; a, b)}{B(a, b)}\\)
 //
-//     values.shape = input.shape[:-1]
+// where
 //
-// Arguments:
-//	input: 1-D or higher with last dimension at least `n+1`.
-//	n: 0-D. Position of sorted vector to select along the last dimension (along
-// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
 //
-// Returns The `n`-th order statistic along each last dimensional slice.
-func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
+// \\(B(x; a, b) = \int_0^x t^{a-1} (1 - t)^{b-1} dt\\)
+//
+//
+// is the incomplete beta function and \\(B(a, b)\\) is the *complete*
+// beta function.
+func Betainc(scope *Scope, a tf.Output, b tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "NthElement",
+		Type: "Betainc",
 		Input: []tf.Input{
-			input, n,
+			a, b, x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the Max along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// This operator is similar to the [unsorted segment sum operator](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
-// Instead of computing the sum over segments, it computes the maximum
-// such that:
+// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
 //
-// \\(output_i = \max_j data_j\\) where max is over `j` such
-// that `segment_ids[j] == i`.
+// N is the size of the segment being reduced.
 //
-// If the maximum is empty for a given segment ID `i`, it outputs the smallest possible value for specific numeric type,
-//  `output[i] = numeric_limits<T>::min()`.
+// Like `SparseSegmentSqrtN`, but allows missing ids in `segment_ids`. If an id is
+// misisng, the `output` tensor at that position will be zeroed.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
-// </div>
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
 // Arguments:
 //
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.
-//
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	num_segments: Should equal the number of distinct segment IDs.
 //
 // Returns Has same shape as data, except for dimension 0 which
-// has size `num_segments`.
-func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+// has size `k`, the number of segments.
+func SparseSegmentSqrtNWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "UnsortedSegmentMax",
+		Type: "SparseSegmentSqrtNWithNumSegments",
 		Input: []tf.Input{
-			data, segment_ids, num_segments,
+			data, indices, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes exponential of x element-wise.  \\(y = e^x\\).
-func Exp(scope *Scope, x tf.Output) (y tf.Output) {
+// Compute the upper regularized incomplete Gamma function `Q(a, x)`.
+//
+// The upper regularized incomplete Gamma function is defined as:
+//
+// \\(Q(a, x) = Gamma(a, x) / Gamma(a) = 1 - P(a, x)\\)
+//
+// where
+//
+// \\(Gamma(a, x) = int_{x}^{\infty} t^{a-1} exp(-t) dt\\)
+//
+// is the upper incomplete Gama function.
+//
+// Note, above `P(a, x)` (`Igamma`) is the lower regularized complete
+// Gamma function.
+func Igammac(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Exp",
+		Type: "Igammac",
 		Input: []tf.Input{
-			x,
+			a, x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns an element-wise indication of the sign of a number.
-//
-// `y = sign(x) = -1` if `x < 0`; 0 if `x == 0`; 1 if `x > 0`.
+// LogUniformCandidateSamplerAttr is an optional argument to LogUniformCandidateSampler.
+type LogUniformCandidateSamplerAttr func(optionalAttr)
+
+// LogUniformCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`.
-func Sign(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Sign",
-		Input: []tf.Input{
-			x,
-		},
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func LogUniformCandidateSamplerSeed(value int64) LogUniformCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// ArgMinAttr is an optional argument to ArgMin.
-type ArgMinAttr func(optionalAttr)
-
-// ArgMinOutputType sets the optional output_type attribute to value.
-// If not specified, defaults to DT_INT64
-func ArgMinOutputType(value tf.DataType) ArgMinAttr {
+// LogUniformCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func LogUniformCandidateSamplerSeed2(value int64) LogUniformCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["output_type"] = value
+		m["seed2"] = value
 	}
 }
 
-// Returns the index with the smallest value across dimensions of a tensor.
+// Generates labels for candidate sampling with a log-uniform distribution.
 //
-// Note that in case of ties the identity of the return value is not guaranteed.
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
+//
+// For each batch, this op picks a single set of sampled candidate labels.
+//
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
 // Arguments:
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
 //
-//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
-// Describes which dimension of the input Tensor to reduce across. For vectors,
-// use dimension = 0.
-func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMinAttr) (output tf.Output) {
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func LogUniformCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...LogUniformCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ArgMin",
+		Type: "LogUniformCandidateSampler",
 		Input: []tf.Input{
-			input, dimension,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Convert the quantized 'input' tensor into a lower-precision 'output', using the
-//
-// output range specified with 'requested_output_min' and 'requested_output_max'.
-//
-// [input_min, input_max] are scalar floats that specify the range for the float
-// interpretation of the 'input' data. For example, if input_min is -1.0f and
-// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
-// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
-//
-// Arguments:
-//
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//	requested_output_min: The float value that the minimum quantized output value represents.
-//	requested_output_max: The float value that the maximum quantized output value represents.
-//	out_type: The type of the output. Should be a lower bit depth than Tinput.
+// Returns (x - y)(x - y) element-wise.
 //
-// Returns The requested_output_min value is copied into this output.The requested_output_max value is copied into this output.
-func Requantize(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, requested_output_min tf.Output, requested_output_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// *NOTE*: `SquaredDifference` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func SquaredDifference(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "Requantize",
+		Type: "SquaredDifference",
 		Input: []tf.Input{
-			input, input_min, input_max, requested_output_min, requested_output_max,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Computes the determinant of one or more square matrices.
+// Forwards the input to the output.
 //
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. The output is a tensor containing the determinants
-// for all input submatrices `[..., :, :]`.
+// This operator represents the loop termination condition used by the
+// "pivot" switches of a loop.
 //
 // Arguments:
-//	input: Shape is `[..., M, M]`.
+//	input: A boolean scalar, representing the branch predicate of the Switch op.
 //
-// Returns Shape is `[...]`.
-func MatrixDeterminant(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns The same tensor as `input`.
+func LoopCond(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixDeterminant",
+		Type: "LoopCond",
 		Input: []tf.Input{
 			input,
 		},
@@ -2575,121 +2660,114 @@ func MatrixDeterminant(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Computes sin of x element-wise.
-func Sin(scope *Scope, x tf.Output) (y tf.Output) {
+// ApproximateEqualAttr is an optional argument to ApproximateEqual.
+type ApproximateEqualAttr func(optionalAttr)
+
+// ApproximateEqualTolerance sets the optional tolerance attribute to value.
+// If not specified, defaults to 1e-05
+func ApproximateEqualTolerance(value float32) ApproximateEqualAttr {
+	return func(m optionalAttr) {
+		m["tolerance"] = value
+	}
+}
+
+// Returns the truth value of abs(x-y) < tolerance element-wise.
+func ApproximateEqual(scope *Scope, x tf.Output, y tf.Output, optional ...ApproximateEqualAttr) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Sin",
+		Type: "ApproximateEqual",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the complementary error function of `x` element-wise.
-func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns x / y element-wise.
+//
+// *NOTE*: `Div` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Div(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Erfc",
+		Type: "Div",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes Psi, the derivative of Lgamma (the log of the absolute value of
+// Returns x * y element-wise.
 //
-// `Gamma(x)`), element-wise.
-func Digamma(scope *Scope, x tf.Output) (y tf.Output) {
+// *NOTE*: `Multiply` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Mul(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Digamma",
+		Type: "Mul",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv2DBackpropFilterAttr is an optional argument to Conv2DBackpropFilter.
-type Conv2DBackpropFilterAttr func(optionalAttr)
-
-// Conv2DBackpropFilterUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
-// If not specified, defaults to true
-func Conv2DBackpropFilterUseCudnnOnGpu(value bool) Conv2DBackpropFilterAttr {
-	return func(m optionalAttr) {
-		m["use_cudnn_on_gpu"] = value
-	}
-}
+// BiasAddAttr is an optional argument to BiasAdd.
+type BiasAddAttr func(optionalAttr)
 
-// Conv2DBackpropFilterDataFormat sets the optional data_format attribute to value.
+// BiasAddDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
+// default format "NHWC", the bias tensor will be added to the last dimension
+// of the value tensor.
 // Alternatively, the format could be "NCHW", the data storage order of:
 //     [batch, in_channels, in_height, in_width].
+// The tensor will be added to "in_channels", the third-to-the-last
+//     dimension.
 // If not specified, defaults to "NHWC"
-func Conv2DBackpropFilterDataFormat(value string) Conv2DBackpropFilterAttr {
+func BiasAddDataFormat(value string) BiasAddAttr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// Conv2DBackpropFilterDilations sets the optional dilations attribute to value.
+// Adds `bias` to `value`.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func Conv2DBackpropFilterDilations(value []int64) Conv2DBackpropFilterAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of convolution with respect to the filter.
+// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
+// Broadcasting is supported, so `value` may have any number of dimensions.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
-//	filter_sizes: An integer vector representing the tensor shape of `filter`,
-// where `filter` is a 4-D
-// `[filter_height, filter_width, in_channels, out_channels]` tensor.
-//	out_backprop: 4-D with shape `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution. Must be in the same order as the dimension specified with
-// format.
-//	padding: The type of padding algorithm to use.
+//	value: Any number of dimensions.
+//	bias: 1-D with size the last dimension of `value`.
 //
-// Returns 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
-// the `filter` input of the convolution.
-func Conv2DBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv2DBackpropFilterAttr) (output tf.Output) {
+// Returns Broadcasted sum of `value` and `bias`.
+func BiasAdd(scope *Scope, value tf.Output, bias tf.Output, optional ...BiasAddAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv2DBackpropFilter",
+		Type: "BiasAdd",
 		Input: []tf.Input{
-			input, filter_sizes, out_backprop,
+			value, bias,
 		},
 		Attrs: attrs,
 	}
@@ -2697,138 +2775,211 @@ func Conv2DBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output,
 	return op.Output(0)
 }
 
-// Returns the number of work units this Reader has finished processing.
+// SparseReduceSumSparseAttr is an optional argument to SparseReduceSumSparse.
+type SparseReduceSumSparseAttr func(optionalAttr)
+
+// SparseReduceSumSparseKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SparseReduceSumSparseKeepDims(value bool) SparseReduceSumSparseAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the sum of elements across dimensions of a SparseTensor.
+//
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_sum()`.  In contrast to SparseReduceSum, this Op returns a
+// SparseTensor.
+//
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
 //
 // Arguments:
-//	reader_handle: Handle to a Reader.
-func ReaderNumWorkUnitsCompletedV2(scope *Scope, reader_handle tf.Output) (units_completed tf.Output) {
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+func SparseReduceSumSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ReaderNumWorkUnitsCompletedV2",
+		Type: "SparseReduceSumSparse",
 		Input: []tf.Input{
-			reader_handle,
+			input_indices, input_values, input_shape, reduction_axes,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the log of the absolute value of `Gamma(x)` element-wise.
-func Lgamma(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns x + y element-wise.
+//
+// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func AddV2(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Lgamma",
+		Type: "AddV2",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the reverse mode backpropagated gradient of the Cholesky algorithm.
-//
-// For an explanation see "Differentiation of the Cholesky algorithm" by
-// Iain Murray http://arxiv.org/abs/1602.07527.
-//
-// Arguments:
-//	l: Output of batch Cholesky algorithm l = cholesky(A). Shape is `[..., M, M]`.
-// Algorithm depends only on lower triangular part of the innermost matrices of
-// this tensor.
-//	grad: df/dl where f is some scalar function. Shape is `[..., M, M]`.
-// Algorithm depends only on lower triangular part of the innermost matrices of
-// this tensor.
+// Returns x + y element-wise.
 //
-// Returns Symmetrized version of df/dA . Shape is `[..., M, M]`
-func CholeskyGrad(scope *Scope, l tf.Output, grad tf.Output) (output tf.Output) {
+// *NOTE*: `Add` supports broadcasting. `AddN` does not. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Add(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "CholeskyGrad",
+		Type: "Add",
 		Input: []tf.Input{
-			l, grad,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the mean along sparse segments of a tensor.
-//
-// Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
-// misisng, the `output` tensor at that position will be zeroed.
+// NthElementAttr is an optional argument to NthElement.
+type NthElementAttr func(optionalAttr)
+
+// NthElementReverse sets the optional reverse attribute to value.
 //
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// value: When set to True, find the nth-largest value in the vector and vice
+// versa.
+// If not specified, defaults to false
+func NthElementReverse(value bool) NthElementAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Finds values of the `n`-th order statistic for the last dimension.
 //
-// Arguments:
+// If the input is a vector (rank-1), finds the entries which is the nth-smallest
+// value in the vector and outputs their values as scalar tensor.
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
-//	num_segments: Should equal the number of distinct segment IDs.
+// For matrices (resp. higher rank input), computes the entries which is the
+// nth-smallest value in each row (resp. vector along the last dimension). Thus,
 //
-// Returns Has same shape as data, except for dimension 0 which has size
-// `num_segments`.
-func SparseSegmentMeanWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
+//     values.shape = input.shape[:-1]
+//
+// Arguments:
+//	input: 1-D or higher with last dimension at least `n+1`.
+//	n: 0-D. Position of sorted vector to select along the last dimension (along
+// each row for matrices). Valid range of n is `[0, input.shape[:-1])`
+//
+// Returns The `n`-th order statistic along each last dimensional slice.
+func NthElement(scope *Scope, input tf.Output, n tf.Output, optional ...NthElementAttr) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanWithNumSegments",
+		Type: "NthElement",
 		Input: []tf.Input{
-			data, indices, segment_ids, num_segments,
+			input, n,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes hyperbolic cosine of x element-wise.
-func Cosh(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes the Max along segments of a tensor.
+//
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
+//
+// This operator is similar to the [unsorted segment sum operator](../../../api_docs/python/math_ops.md#UnsortedSegmentSum).
+// Instead of computing the sum over segments, it computes the maximum
+// such that:
+//
+// \\(output_i = \max_j data_j\\) where max is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the maximum is empty for a given segment ID `i`, it outputs the smallest possible value for specific numeric type,
+//  `output[i] = numeric_limits<T>::min()`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/UnsortedSegmentMax.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.
+//
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `num_segments`.
+func UnsortedSegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Cosh",
+		Type: "UnsortedSegmentMax",
 		Input: []tf.Input{
-			x,
+			data, segment_ids, num_segments,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a dataset that emits each dim-0 slice of `components` once.
-func TensorSliceDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+// Computes exponential of x element-wise.  \\(y = e^x\\).
+func Exp(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorSliceDataset",
+		Type: "Exp",
 		Input: []tf.Input{
-			tf.OutputList(components),
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes natural logarithm of (1 + x) element-wise.
+// Returns an element-wise indication of the sign of a number.
 //
-// I.e., \\(y = \log_e (1 + x)\\).
-func Log1p(scope *Scope, x tf.Output) (y tf.Output) {
+// `y = sign(x) = -1` if `x < 0`; 0 if `x == 0`; 1 if `x > 0`.
+//
+// For complex numbers, `y = sign(x) = x / |x|` if `x != 0`, otherwise `y = 0`.
+func Sign(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Log1p",
+		Type: "Sign",
 		Input: []tf.Input{
 			x,
 		},
@@ -2837,83 +2988,125 @@ func Log1p(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes rectified linear 6 gradients for a Relu6 operation.
+// ArgMinAttr is an optional argument to ArgMin.
+type ArgMinAttr func(optionalAttr)
+
+// ArgMinOutputType sets the optional output_type attribute to value.
+// If not specified, defaults to DT_INT64
+func ArgMinOutputType(value tf.DataType) ArgMinAttr {
+	return func(m optionalAttr) {
+		m["output_type"] = value
+	}
+}
+
+// Returns the index with the smallest value across dimensions of a tensor.
+//
+// Note that in case of ties the identity of the return value is not guaranteed.
 //
 // Arguments:
-//	gradients: The backpropagated gradients to the corresponding Relu6 operation.
-//	features: The features passed as input to the corresponding Relu6 operation, or
-// its output; using either one produces the same result.
 //
-// Returns The gradients:
-// `gradients * (features > 0) * (features < 6)`.
-func Relu6Grad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
+// Describes which dimension of the input Tensor to reduce across. For vectors,
+// use dimension = 0.
+func ArgMin(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Relu6Grad",
+		Type: "ArgMin",
 		Input: []tf.Input{
-			gradients, features,
+			input, dimension,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBicubicAttr is an optional argument to ResizeBicubic.
-type ResizeBicubicAttr func(optionalAttr)
-
-// ResizeBicubicAlignCorners sets the optional align_corners attribute to value.
+// Convert the quantized 'input' tensor into a lower-precision 'output', using the
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeBicubicAlignCorners(value bool) ResizeBicubicAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
+// output range specified with 'requested_output_min' and 'requested_output_max'.
+//
+// [input_min, input_max] are scalar floats that specify the range for the float
+// interpretation of the 'input' data. For example, if input_min is -1.0f and
+// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
+//
+// Arguments:
+//
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//	requested_output_min: The float value that the minimum quantized output value represents.
+//	requested_output_max: The float value that the maximum quantized output value represents.
+//	out_type: The type of the output. Should be a lower bit depth than Tinput.
+//
+// Returns The requested_output_min value is copied into this output.The requested_output_max value is copied into this output.
+func Requantize(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, requested_output_min tf.Output, requested_output_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"out_type": out_type}
+	opspec := tf.OpSpec{
+		Type: "Requantize",
+		Input: []tf.Input{
+			input, input_min, input_max, requested_output_min, requested_output_max,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Resize `images` to `size` using bicubic interpolation.
+// Computes the determinant of one or more square matrices.
 //
-// Input images can be of different types but output images are always float.
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. The output is a tensor containing the determinants
+// for all input submatrices `[..., :, :]`.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	input: Shape is `[..., M, M]`.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBicubicAttr) (resized_images tf.Output) {
+// Returns Shape is `[...]`.
+func MatrixDeterminant(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+	opspec := tf.OpSpec{
+		Type: "MatrixDeterminant",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes sin of x element-wise.
+func Sin(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBicubic",
+		Type: "Sin",
 		Input: []tf.Input{
-			images, size,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes natural logarithm of x element-wise.
-//
-// I.e., \\(y = \log_e x\\).
-func Log(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes the complementary error function of `x` element-wise.
+func Erfc(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Log",
+		Type: "Erfc",
 		Input: []tf.Input{
 			x,
 		},
@@ -2922,16 +3115,15 @@ func Log(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Rounds the values of a tensor to the nearest integer, element-wise.
+// Computes Psi, the derivative of Lgamma (the log of the absolute value of
 //
-// Rounds half to even.  Also known as bankers rounding. If you want to round
-// according to the current system rounding mode use std::cint.
-func Round(scope *Scope, x tf.Output) (y tf.Output) {
+// `Gamma(x)`), element-wise.
+func Digamma(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Round",
+		Type: "Digamma",
 		Input: []tf.Input{
 			x,
 		},
@@ -2940,172 +3132,179 @@ func Round(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// RecordInputAttr is an optional argument to RecordInput.
-type RecordInputAttr func(optionalAttr)
+// Conv2DBackpropFilterAttr is an optional argument to Conv2DBackpropFilter.
+type Conv2DBackpropFilterAttr func(optionalAttr)
 
-// RecordInputFileRandomSeed sets the optional file_random_seed attribute to value.
-//
-// value: Random seeds used to produce randomized records.
-// If not specified, defaults to 301
-func RecordInputFileRandomSeed(value int64) RecordInputAttr {
+// Conv2DBackpropFilterUseCudnnOnGpu sets the optional use_cudnn_on_gpu attribute to value.
+// If not specified, defaults to true
+func Conv2DBackpropFilterUseCudnnOnGpu(value bool) Conv2DBackpropFilterAttr {
 	return func(m optionalAttr) {
-		m["file_random_seed"] = value
+		m["use_cudnn_on_gpu"] = value
 	}
 }
 
-// RecordInputFileShuffleShiftRatio sets the optional file_shuffle_shift_ratio attribute to value.
+// Conv2DBackpropFilterDataFormat sets the optional data_format attribute to value.
 //
-// value: Shifts the list of files after the list is randomly
-// shuffled.
-// If not specified, defaults to 0
-func RecordInputFileShuffleShiftRatio(value float32) RecordInputAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func Conv2DBackpropFilterDataFormat(value string) Conv2DBackpropFilterAttr {
 	return func(m optionalAttr) {
-		m["file_shuffle_shift_ratio"] = value
+		m["data_format"] = value
 	}
 }
 
-// RecordInputFileBufferSize sets the optional file_buffer_size attribute to value.
+// Conv2DBackpropFilterDilations sets the optional dilations attribute to value.
 //
-// value: The randomization shuffling buffer.
-// If not specified, defaults to 10000
-func RecordInputFileBufferSize(value int64) RecordInputAttr {
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func Conv2DBackpropFilterDilations(value []int64) Conv2DBackpropFilterAttr {
 	return func(m optionalAttr) {
-		m["file_buffer_size"] = value
+		m["dilations"] = value
 	}
 }
 
-// RecordInputFileParallelism sets the optional file_parallelism attribute to value.
-//
-// value: How many sstables are opened and concurrently iterated over.
-// If not specified, defaults to 16
-func RecordInputFileParallelism(value int64) RecordInputAttr {
-	return func(m optionalAttr) {
-		m["file_parallelism"] = value
-	}
-}
-
-// RecordInputBatchSize sets the optional batch_size attribute to value.
-//
-// value: The batch size.
-// If not specified, defaults to 32
-func RecordInputBatchSize(value int64) RecordInputAttr {
-	return func(m optionalAttr) {
-		m["batch_size"] = value
-	}
-}
-
-// RecordInputCompressionType sets the optional compression_type attribute to value.
-//
-// value: The type of compression for the file. Currently ZLIB and
-// GZIP are supported. Defaults to none.
-// If not specified, defaults to ""
-func RecordInputCompressionType(value string) RecordInputAttr {
-	return func(m optionalAttr) {
-		m["compression_type"] = value
-	}
-}
-
-// Emits randomized records.
+// Computes the gradients of convolution with respect to the filter.
 //
 // Arguments:
-//	file_pattern: Glob pattern for the data files.
+//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
+//	filter_sizes: An integer vector representing the tensor shape of `filter`,
+// where `filter` is a 4-D
+// `[filter_height, filter_width, in_channels, out_channels]` tensor.
+//	out_backprop: 4-D with shape `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution. Must be in the same order as the dimension specified with
+// format.
+//	padding: The type of padding algorithm to use.
 //
-// Returns A tensor of shape [batch_size].
-func RecordInput(scope *Scope, file_pattern string, optional ...RecordInputAttr) (records tf.Output) {
+// Returns 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
+// the `filter` input of the convolution.
+func Conv2DBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv2DBackpropFilterAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"file_pattern": file_pattern}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RecordInput",
-
+		Type: "Conv2DBackpropFilter",
+		Input: []tf.Input{
+			input, filter_sizes, out_backprop,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes reciprocal of square root of x element-wise.
+// Returns the number of work units this Reader has finished processing.
 //
-// I.e., \\(y = 1 / \sqrt{x}\\).
-func Rsqrt(scope *Scope, x tf.Output) (y tf.Output) {
+// Arguments:
+//	reader_handle: Handle to a Reader.
+func ReaderNumWorkUnitsCompletedV2(scope *Scope, reader_handle tf.Output) (units_completed tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Rsqrt",
+		Type: "ReaderNumWorkUnitsCompletedV2",
 		Input: []tf.Input{
-			x,
+			reader_handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixInverseAttr is an optional argument to MatrixInverse.
-type MatrixInverseAttr func(optionalAttr)
-
-// MatrixInverseAdjoint sets the optional adjoint attribute to value.
-// If not specified, defaults to false
-func MatrixInverseAdjoint(value bool) MatrixInverseAttr {
-	return func(m optionalAttr) {
-		m["adjoint"] = value
+// Computes the log of the absolute value of `Gamma(x)` element-wise.
+func Lgamma(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
+	opspec := tf.OpSpec{
+		Type: "Lgamma",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the inverse of one or more square invertible matrices or their
+// Computes the reverse mode backpropagated gradient of the Cholesky algorithm.
 //
-// adjoints (conjugate transposes).
+// For an explanation see "Differentiation of the Cholesky algorithm" by
+// Iain Murray http://arxiv.org/abs/1602.07527.
 //
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. The output is a tensor of the same shape as the input
-// containing the inverse for all input submatrices `[..., :, :]`.
+// Arguments:
+//	l: Output of batch Cholesky algorithm l = cholesky(A). Shape is `[..., M, M]`.
+// Algorithm depends only on lower triangular part of the innermost matrices of
+// this tensor.
+//	grad: df/dl where f is some scalar function. Shape is `[..., M, M]`.
+// Algorithm depends only on lower triangular part of the innermost matrices of
+// this tensor.
 //
-// The op uses LU decomposition with partial pivoting to compute the inverses.
+// Returns Symmetrized version of df/dA . Shape is `[..., M, M]`
+func CholeskyGrad(scope *Scope, l tf.Output, grad tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "CholeskyGrad",
+		Input: []tf.Input{
+			l, grad,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the mean along sparse segments of a tensor.
 //
-// If a matrix is not invertible there is no guarantee what the op does. It
-// may detect the condition and raise an exception or it may simply return a
-// garbage result.
+// Like `SparseSegmentMean`, but allows missing ids in `segment_ids`. If an id is
+// misisng, the `output` tensor at that position will be zeroed.
+//
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
 // Arguments:
-//	input: Shape is `[..., M, M]`.
 //
-// Returns Shape is `[..., M, M]`.
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//	num_segments: Should equal the number of distinct segment IDs.
 //
-// @compatibility(numpy)
-// Equivalent to np.linalg.inv
-// @end_compatibility
-func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) {
+// Returns Has same shape as data, except for dimension 0 which has size
+// `num_segments`.
+func SparseSegmentMeanWithNumSegments(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output, num_segments tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MatrixInverse",
+		Type: "SparseSegmentMeanWithNumSegments",
 		Input: []tf.Input{
-			input,
+			data, indices, segment_ids, num_segments,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes square of x element-wise.
-//
-// I.e., \\(y = x * x = x^2\\).
-func Square(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes hyperbolic cosine of x element-wise.
+func Cosh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Square",
+		Type: "Cosh",
 		Input: []tf.Input{
 			x,
 		},
@@ -3114,33 +3313,32 @@ func Square(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes exponential linear: `exp(features) - 1` if < 0, `features` otherwise.
-//
-// See [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
-// ](http://arxiv.org/abs/1511.07289)
-func Elu(scope *Scope, features tf.Output) (activations tf.Output) {
+// Creates a dataset that emits each dim-0 slice of `components` once.
+func TensorSliceDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "Elu",
+		Type: "TensorSliceDataset",
 		Input: []tf.Input{
-			features,
+			tf.OutputList(components),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the reciprocal of x element-wise.
+// Computes natural logarithm of (1 + x) element-wise.
 //
-// I.e., \\(y = 1 / x\\).
-func Reciprocal(scope *Scope, x tf.Output) (y tf.Output) {
+// I.e., \\(y = \log_e (1 + x)\\).
+func Log1p(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Reciprocal",
+		Type: "Log1p",
 		Input: []tf.Input{
 			x,
 		},
@@ -3149,283 +3347,248 @@ func Reciprocal(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// OrderedMapClearAttr is an optional argument to OrderedMapClear.
-type OrderedMapClearAttr func(optionalAttr)
-
-// OrderedMapClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// Computes rectified linear 6 gradients for a Relu6 operation.
 //
-// REQUIRES: value >= 0
-func OrderedMapClearCapacity(value int64) OrderedMapClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// OrderedMapClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// Arguments:
+//	gradients: The backpropagated gradients to the corresponding Relu6 operation.
+//	features: The features passed as input to the corresponding Relu6 operation, or
+// its output; using either one produces the same result.
 //
-// REQUIRES: value >= 0
-func OrderedMapClearMemoryLimit(value int64) OrderedMapClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
+// Returns The gradients:
+// `gradients * (features > 0) * (features < 6)`.
+func Relu6Grad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// OrderedMapClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapClearContainer(value string) OrderedMapClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
+	opspec := tf.OpSpec{
+		Type: "Relu6Grad",
+		Input: []tf.Input{
+			gradients, features,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// OrderedMapClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapClearSharedName(value string) OrderedMapClearAttr {
+// ResizeBicubicAttr is an optional argument to ResizeBicubic.
+type ResizeBicubicAttr func(optionalAttr)
+
+// ResizeBicubicAlignCorners sets the optional align_corners attribute to value.
+//
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeBicubicAlignCorners(value bool) ResizeBicubicAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Op removes all elements in the underlying container.
+// Resize `images` to `size` using bicubic interpolation.
 //
-// Returns the created operation.
-func OrderedMapClear(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapClearAttr) (o *tf.Operation) {
+// Input images can be of different types but output images are always float.
+//
+// Arguments:
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
+//
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeBicubic(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBicubicAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapClear",
-
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Computes the reciprocal of x element-wise.
-//
-// I.e., \\(y = 1 / x\\).
-func Inv(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Inv",
+		Type: "ResizeBicubic",
 		Input: []tf.Input{
-			x,
+			images, size,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ComplexAbsAttr is an optional argument to ComplexAbs.
-type ComplexAbsAttr func(optionalAttr)
-
-// ComplexAbsTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ComplexAbsTout(value tf.DataType) ComplexAbsAttr {
-	return func(m optionalAttr) {
-		m["Tout"] = value
-	}
-}
-
-// Computes the complex absolute value of a tensor.
+// Computes natural logarithm of x element-wise.
 //
-// Given a tensor `x` of complex numbers, this operation returns a tensor of type
-// `float` or `double` that is the absolute value of each element in `x`. All
-// elements in `x` must be complex numbers of the form \\(a + bj\\). The absolute
-// value is computed as \\( \sqrt{a^2 + b^2}\\).
-func ComplexAbs(scope *Scope, x tf.Output, optional ...ComplexAbsAttr) (y tf.Output) {
+// I.e., \\(y = \log_e x\\).
+func Log(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ComplexAbs",
+		Type: "Log",
 		Input: []tf.Input{
 			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the truth value of x AND y element-wise.
+// Rounds the values of a tensor to the nearest integer, element-wise.
 //
-// *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func LogicalAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Rounds half to even.  Also known as bankers rounding. If you want to round
+// according to the current system rounding mode use std::cint.
+func Round(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LogicalAnd",
+		Type: "Round",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Cast x of type SrcT to y of DstT.
-func Cast(scope *Scope, x tf.Output, DstT tf.DataType) (y tf.Output) {
-	if scope.Err() != nil {
-		return
+// RecordInputAttr is an optional argument to RecordInput.
+type RecordInputAttr func(optionalAttr)
+
+// RecordInputFileRandomSeed sets the optional file_random_seed attribute to value.
+//
+// value: Random seeds used to produce randomized records.
+// If not specified, defaults to 301
+func RecordInputFileRandomSeed(value int64) RecordInputAttr {
+	return func(m optionalAttr) {
+		m["file_random_seed"] = value
 	}
-	attrs := map[string]interface{}{"DstT": DstT}
-	opspec := tf.OpSpec{
-		Type: "Cast",
-		Input: []tf.Input{
-			x,
-		},
-		Attrs: attrs,
+}
+
+// RecordInputFileShuffleShiftRatio sets the optional file_shuffle_shift_ratio attribute to value.
+//
+// value: Shifts the list of files after the list is randomly
+// shuffled.
+// If not specified, defaults to 0
+func RecordInputFileShuffleShiftRatio(value float32) RecordInputAttr {
+	return func(m optionalAttr) {
+		m["file_shuffle_shift_ratio"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// MaxAttr is an optional argument to Max.
-type MaxAttr func(optionalAttr)
+// RecordInputFileBufferSize sets the optional file_buffer_size attribute to value.
+//
+// value: The randomization shuffling buffer.
+// If not specified, defaults to 10000
+func RecordInputFileBufferSize(value int64) RecordInputAttr {
+	return func(m optionalAttr) {
+		m["file_buffer_size"] = value
+	}
+}
 
-// MaxKeepDims sets the optional keep_dims attribute to value.
+// RecordInputFileParallelism sets the optional file_parallelism attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func MaxKeepDims(value bool) MaxAttr {
+// value: How many sstables are opened and concurrently iterated over.
+// If not specified, defaults to 16
+func RecordInputFileParallelism(value int64) RecordInputAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["file_parallelism"] = value
 	}
 }
 
-// Computes the maximum of elements across dimensions of a tensor.
+// RecordInputBatchSize sets the optional batch_size attribute to value.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// value: The batch size.
+// If not specified, defaults to 32
+func RecordInputBatchSize(value int64) RecordInputAttr {
+	return func(m optionalAttr) {
+		m["batch_size"] = value
+	}
+}
+
+// RecordInputCompressionType sets the optional compression_type attribute to value.
+//
+// value: The type of compression for the file. Currently ZLIB and
+// GZIP are supported. Defaults to none.
+// If not specified, defaults to ""
+func RecordInputCompressionType(value string) RecordInputAttr {
+	return func(m optionalAttr) {
+		m["compression_type"] = value
+	}
+}
+
+// Emits randomized records.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	file_pattern: Glob pattern for the data files.
 //
-// Returns The reduced tensor.
-func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (output tf.Output) {
+// Returns A tensor of shape [batch_size].
+func RecordInput(scope *Scope, file_pattern string, optional ...RecordInputAttr) (records tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"file_pattern": file_pattern}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Max",
-		Input: []tf.Input{
-			input, axis,
-		},
+		Type: "RecordInput",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Quantized Batch normalization.
-//
-// This op is deprecated and will be removed in the future. Prefer
-// `tf.nn.batch_normalization`.
-//
-// Arguments:
-//	t: A 4D input Tensor.
-//	t_min: The value represented by the lowest quantized input.
-//	t_max: The value represented by the highest quantized input.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	m_min: The value represented by the lowest quantized mean.
-//	m_max: The value represented by the highest quantized mean.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v_min: The value represented by the lowest quantized variance.
-//	v_max: The value represented by the highest quantized variance.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	beta_min: The value represented by the lowest quantized offset.
-//	beta_max: The value represented by the highest quantized offset.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	gamma_min: The value represented by the lowest quantized gamma.
-//	gamma_max: The value represented by the highest quantized gamma.
+// Computes reciprocal of square root of x element-wise.
 //
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func QuantizedBatchNormWithGlobalNormalization(scope *Scope, t tf.Output, t_min tf.Output, t_max tf.Output, m tf.Output, m_min tf.Output, m_max tf.Output, v tf.Output, v_min tf.Output, v_max tf.Output, beta tf.Output, beta_min tf.Output, beta_max tf.Output, gamma tf.Output, gamma_min tf.Output, gamma_max tf.Output, out_type tf.DataType, variance_epsilon float32, scale_after_normalization bool) (result tf.Output, result_min tf.Output, result_max tf.Output) {
+// I.e., \\(y = 1 / \sqrt{x}\\).
+func Rsqrt(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type, "variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
 	opspec := tf.OpSpec{
-		Type: "QuantizedBatchNormWithGlobalNormalization",
+		Type: "Rsqrt",
 		Input: []tf.Input{
-			t, t_min, t_max, m, m_min, m_max, v, v_min, v_max, beta, beta_min, beta_max, gamma, gamma_min, gamma_max,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// HistogramFixedWidthAttr is an optional argument to HistogramFixedWidth.
-type HistogramFixedWidthAttr func(optionalAttr)
+// MatrixInverseAttr is an optional argument to MatrixInverse.
+type MatrixInverseAttr func(optionalAttr)
 
-// HistogramFixedWidthDtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_INT32
-func HistogramFixedWidthDtype(value tf.DataType) HistogramFixedWidthAttr {
+// MatrixInverseAdjoint sets the optional adjoint attribute to value.
+// If not specified, defaults to false
+func MatrixInverseAdjoint(value bool) MatrixInverseAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Return histogram of values.
+// Computes the inverse of one or more square invertible matrices or their
 //
-// Given the tensor `values`, this operation returns a rank 1 histogram counting
-// the number of entries in `values` that fall into every bin.  The bins are
-// equal width and determined by the arguments `value_range` and `nbins`.
+// adjoints (conjugate transposes).
 //
-// ```python
-// # Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
-// nbins = 5
-// value_range = [0.0, 5.0]
-// new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. The output is a tensor of the same shape as the input
+// containing the inverse for all input submatrices `[..., :, :]`.
 //
-// with tf.get_default_session() as sess:
-//   hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
-//   variables.global_variables_initializer().run()
-//   sess.run(hist) => [2, 1, 1, 0, 2]
-// ```
+// The op uses LU decomposition with partial pivoting to compute the inverses.
+//
+// If a matrix is not invertible there is no guarantee what the op does. It
+// may detect the condition and raise an exception or it may simply return a
+// garbage result.
 //
 // Arguments:
-//	values: Numeric `Tensor`.
-//	value_range: Shape [2] `Tensor` of same `dtype` as `values`.
-// values <= value_range[0] will be mapped to hist[0],
-// values >= value_range[1] will be mapped to hist[-1].
-//	nbins: Scalar `int32 Tensor`.  Number of histogram bins.
+//	input: Shape is `[..., M, M]`.
 //
-// Returns A 1-D `Tensor` holding histogram of values.
-func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output, nbins tf.Output, optional ...HistogramFixedWidthAttr) (out tf.Output) {
+// Returns Shape is `[..., M, M]`.
+//
+// @compatibility(numpy)
+// Equivalent to np.linalg.inv
+// @end_compatibility
+func MatrixInverse(scope *Scope, input tf.Output, optional ...MatrixInverseAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -3434,9 +3597,9 @@ func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "HistogramFixedWidth",
+		Type: "MatrixInverse",
 		Input: []tf.Input{
-			values, value_range, nbins,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -3444,263 +3607,151 @@ func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output,
 	return op.Output(0)
 }
 
-// Adds Tensor 'bias' to Tensor 'input' for Quantized types.
-//
-// Broadcasts the values of bias on dimensions 0..N-2 of 'input'.
-//
-// Arguments:
+// Computes square of x element-wise.
 //
-//	bias: A 1D bias Tensor with size matching the last dimension of 'input'.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	min_bias: The float value that the lowest quantized bias value represents.
-//	max_bias: The float value that the highest quantized bias value represents.
-//
-//
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input tf.Output, max_input tf.Output, min_bias tf.Output, max_bias tf.Output, out_type tf.DataType) (output tf.Output, min_out tf.Output, max_out tf.Output) {
+// I.e., \\(y = x * x = x^2\\).
+func Square(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "QuantizedBiasAdd",
+		Type: "Square",
 		Input: []tf.Input{
-			input, bias, min_input, max_input, min_bias, max_bias,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Produces the average pool of the input tensor for quantized types.
-//
-// Arguments:
-//	input: 4-D with shape `[batch, height, width, channels]`.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	ksize: The size of the window for each dimension of the input tensor.
-// The length must be 4 to match the number of dimensions of the input.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor.  The length must be 4 to match the number of dimensions of the input.
-//	padding: The type of padding algorithm to use.
+// Computes exponential linear: `exp(features) - 1` if < 0, `features` otherwise.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedAvgPool(scope *Scope, input tf.Output, min_input tf.Output, max_input tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output, min_output tf.Output, max_output tf.Output) {
+// See [Fast and Accurate Deep Network Learning by Exponential Linear Units (ELUs)
+// ](http://arxiv.org/abs/1511.07289)
+func Elu(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "QuantizedAvgPool",
+		Type: "Elu",
 		Input: []tf.Input{
-			input, min_input, max_input,
+			features,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// FractionalAvgPoolAttr is an optional argument to FractionalAvgPool.
-type FractionalAvgPoolAttr func(optionalAttr)
-
-// FractionalAvgPoolPseudoRandom sets the optional pseudo_random attribute to value.
+// Computes the reciprocal of x element-wise.
 //
-// value: When set to True, generates the pooling sequence in a
-// pseudorandom fashion, otherwise, in a random fashion. Check paper [Benjamin
-// Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) for
-// difference between pseudorandom and random.
-// If not specified, defaults to false
-func FractionalAvgPoolPseudoRandom(value bool) FractionalAvgPoolAttr {
-	return func(m optionalAttr) {
-		m["pseudo_random"] = value
+// I.e., \\(y = 1 / x\\).
+func Reciprocal(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Reciprocal",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FractionalAvgPoolOverlapping sets the optional overlapping attribute to value.
-//
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
-//
-// `index  0  1  2  3  4`
-//
-// `value  20 5  16 3  7`
+// OrderedMapClearAttr is an optional argument to OrderedMapClear.
+type OrderedMapClearAttr func(optionalAttr)
+
+// OrderedMapClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [41/3, 26/3] for fractional avg pooling.
-// If not specified, defaults to false
-func FractionalAvgPoolOverlapping(value bool) FractionalAvgPoolAttr {
+// REQUIRES: value >= 0
+func OrderedMapClearCapacity(value int64) OrderedMapClearAttr {
 	return func(m optionalAttr) {
-		m["overlapping"] = value
+		m["capacity"] = value
 	}
 }
 
-// FractionalAvgPoolDeterministic sets the optional deterministic attribute to value.
+// OrderedMapClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// value: When set to True, a fixed pooling region will be used when
-// iterating over a FractionalAvgPool node in the computation graph. Mainly used
-// in unit test to make FractionalAvgPool deterministic.
-// If not specified, defaults to false
-func FractionalAvgPoolDeterministic(value bool) FractionalAvgPoolAttr {
+// REQUIRES: value >= 0
+func OrderedMapClearMemoryLimit(value int64) OrderedMapClearAttr {
 	return func(m optionalAttr) {
-		m["deterministic"] = value
+		m["memory_limit"] = value
 	}
 }
 
-// FractionalAvgPoolSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func FractionalAvgPoolSeed(value int64) FractionalAvgPoolAttr {
+// OrderedMapClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapClearContainer(value string) OrderedMapClearAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["container"] = value
 	}
 }
 
-// FractionalAvgPoolSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func FractionalAvgPoolSeed2(value int64) FractionalAvgPoolAttr {
+// OrderedMapClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapClearSharedName(value string) OrderedMapClearAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Performs fractional average pooling on the input.
-//
-// Fractional average pooling is similar to Fractional max pooling in the pooling
-// region generation step. The only difference is that after pooling regions are
-// generated, a mean operation is performed instead of a max operation in each
-// pooling region.
-//
-// Arguments:
-//	value: 4-D with shape `[batch, height, width, channels]`.
-//	pooling_ratio: Pooling ratio for each dimension of `value`, currently only
-// supports row and col dimension and should be >= 1.0. For example, a valid
-// pooling ratio looks like [1.0, 1.44, 1.73, 1.0]. The first and last elements
-// must be 1.0 because we don't allow pooling on batch and channels
-// dimensions. 1.44 and 1.73 are pooling ratio on height and width dimensions
-// respectively.
+// Op removes all elements in the underlying container.
 //
-// Returns output tensor after fractional avg pooling.row pooling sequence, needed to calculate gradient.column pooling sequence, needed to calculate gradient.
-func FractionalAvgPool(scope *Scope, value tf.Output, pooling_ratio []float32, optional ...FractionalAvgPoolAttr) (output tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output) {
+// Returns the created operation.
+func OrderedMapClear(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapClearAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"pooling_ratio": pooling_ratio}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FractionalAvgPool",
-		Input: []tf.Input{
-			value,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// RandomCropAttr is an optional argument to RandomCrop.
-type RandomCropAttr func(optionalAttr)
-
-// RandomCropSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomCropSeed(value int64) RandomCropAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+		Type: "OrderedMapClear",
 
-// RandomCropSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomCropSeed2(value int64) RandomCropAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+		Attrs: attrs,
 	}
+	return scope.AddOperation(opspec)
 }
 
-// Randomly crop `image`.
-//
-// DEPRECATED at GraphDef version 8: Random crop is now pure Python
-//
-// `size` is a 1-D int64 tensor with 2 elements representing the crop height and
-// width.  The values must be non negative.
-//
-// This Op picks a random location in `image` and crops a `height` by `width`
-// rectangle from that location.  The random location is picked so the cropped
-// area will fit inside the original image.
-//
-// Arguments:
-//	image: 3-D of shape `[height, width, channels]`.
-//	size: 1-D of length 2 containing: `crop_height`, `crop_width`..
+// Computes the reciprocal of x element-wise.
 //
-// Returns 3-D of shape `[crop_height, crop_width, channels].`
-func RandomCrop(scope *Scope, image tf.Output, size tf.Output, optional ...RandomCropAttr) (output tf.Output) {
+// I.e., \\(y = 1 / x\\).
+func Inv(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomCrop",
+		Type: "Inv",
 		Input: []tf.Input{
-			image, size,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TopKV2Attr is an optional argument to TopKV2.
-type TopKV2Attr func(optionalAttr)
+// ComplexAbsAttr is an optional argument to ComplexAbs.
+type ComplexAbsAttr func(optionalAttr)
 
-// TopKV2Sorted sets the optional sorted attribute to value.
-//
-// value: If true the resulting `k` elements will be sorted by the values in
-// descending order.
-// If not specified, defaults to true
-func TopKV2Sorted(value bool) TopKV2Attr {
+// ComplexAbsTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ComplexAbsTout(value tf.DataType) ComplexAbsAttr {
 	return func(m optionalAttr) {
-		m["sorted"] = value
+		m["Tout"] = value
 	}
 }
 
-// Finds values and indices of the `k` largest elements for the last dimension.
-//
-// If the input is a vector (rank-1), finds the `k` largest entries in the vector
-// and outputs their values and indices as vectors.  Thus `values[j]` is the
-// `j`-th largest entry in `input`, and its index is `indices[j]`.
-//
-// For matrices (resp. higher rank input), computes the top `k` entries in each
-// row (resp. vector along the last dimension).  Thus,
-//
-//     values.shape = indices.shape = input.shape[:-1] + [k]
-//
-// If two elements are equal, the lower-index element appears first.
-//
-// Arguments:
-//	input: 1-D or higher with last dimension at least `k`.
-//	k: 0-D.  Number of top elements to look for along the last dimension (along each
-// row for matrices).
+// Computes the complex absolute value of a tensor.
 //
-// Returns The `k` largest elements along each last dimensional slice.The indices of `values` within the last dimension of `input`.
-func TopKV2(scope *Scope, input tf.Output, k tf.Output, optional ...TopKV2Attr) (values tf.Output, indices tf.Output) {
+// Given a tensor `x` of complex numbers, this operation returns a tensor of type
+// `float` or `double` that is the absolute value of each element in `x`. All
+// elements in `x` must be complex numbers of the form \\(a + bj\\). The absolute
+// value is computed as \\( \sqrt{a^2 + b^2}\\).
+func ComplexAbs(scope *Scope, x tf.Output, optional ...ComplexAbsAttr) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -3709,26 +3760,26 @@ func TopKV2(scope *Scope, input tf.Output, k tf.Output, optional ...TopKV2Attr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TopKV2",
+		Type: "ComplexAbs",
 		Input: []tf.Input{
-			input, k,
+			x,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Returns x // y element-wise.
+// Returns the truth value of x AND y element-wise.
 //
-// *NOTE*: `FloorDiv` supports broadcasting. More about broadcasting
+// *NOTE*: `LogicalAnd` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func FloorDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+func LogicalAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "FloorDiv",
+		Type: "LogicalAnd",
 		Input: []tf.Input{
 			x, y,
 		},
@@ -3737,122 +3788,154 @@ func FloorDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// Returns a batched diagonal tensor with a given batched diagonal values.
-//
-// Given a `diagonal`, this operation returns a tensor with the `diagonal` and
-// everything else padded with zeros. The diagonal is computed as follows:
-//
-// Assume `diagonal` has `k` dimensions `[I, J, K, ..., N]`, then the output is a
-// tensor of rank `k+1` with dimensions [I, J, K, ..., N, N]` where:
-//
-// `output[i, j, k, ..., m, n] = 1{m=n} * diagonal[i, j, k, ..., n]`.
-//
-// For example:
-//
-// ```
-// # 'diagonal' is [[1, 2, 3, 4], [5, 6, 7, 8]]
-//
-// and diagonal.shape = (2, 4)
-//
-// tf.matrix_diag(diagonal) ==> [[[1, 0, 0, 0]
-//                                      [0, 2, 0, 0]
-//                                      [0, 0, 3, 0]
-//                                      [0, 0, 0, 4]],
-//                                     [[5, 0, 0, 0]
-//                                      [0, 6, 0, 0]
-//                                      [0, 0, 7, 0]
-//                                      [0, 0, 0, 8]]]
-//
-// which has shape (2, 4, 4)
-// ```
-//
-// Arguments:
-//	diagonal: Rank `k`, where `k >= 1`.
-//
-// Returns Rank `k+1`, with `output.shape = diagonal.shape + [diagonal.shape[-1]]`.
-func MatrixDiag(scope *Scope, diagonal tf.Output) (output tf.Output) {
+// Cast x of type SrcT to y of DstT.
+func Cast(scope *Scope, x tf.Output, DstT tf.DataType) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"DstT": DstT}
 	opspec := tf.OpSpec{
-		Type: "MatrixDiag",
+		Type: "Cast",
 		Input: []tf.Input{
-			diagonal,
+			x,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the truth value of (x <= y) element-wise.
+// MaxAttr is an optional argument to Max.
+type MaxAttr func(optionalAttr)
+
+// MaxKeepDims sets the optional keep_dims attribute to value.
 //
-// *NOTE*: `LessEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func LessEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func MaxKeepDims(value bool) MaxAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the maximum of elements across dimensions of a tensor.
+//
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
+//
+// Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Max(scope *Scope, input tf.Output, axis tf.Output, optional ...MaxAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LessEqual",
+		Type: "Max",
 		Input: []tf.Input{
-			x, y,
+			input, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes softmax activations.
-//
-// For each batch `i` and class `j` we have
+// Quantized Batch normalization.
 //
-//     softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j]))
+// This op is deprecated and will be removed in the future. Prefer
+// `tf.nn.batch_normalization`.
 //
 // Arguments:
-//	logits: 2-D with shape `[batch_size, num_classes]`.
+//	t: A 4D input Tensor.
+//	t_min: The value represented by the lowest quantized input.
+//	t_max: The value represented by the highest quantized input.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	m_min: The value represented by the lowest quantized mean.
+//	m_max: The value represented by the highest quantized mean.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v_min: The value represented by the lowest quantized variance.
+//	v_max: The value represented by the highest quantized variance.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	beta_min: The value represented by the lowest quantized offset.
+//	beta_max: The value represented by the highest quantized offset.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	gamma_min: The value represented by the lowest quantized gamma.
+//	gamma_max: The value represented by the highest quantized gamma.
 //
-// Returns Same shape as `logits`.
-func Softmax(scope *Scope, logits tf.Output) (softmax tf.Output) {
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func QuantizedBatchNormWithGlobalNormalization(scope *Scope, t tf.Output, t_min tf.Output, t_max tf.Output, m tf.Output, m_min tf.Output, m_max tf.Output, v tf.Output, v_min tf.Output, v_max tf.Output, beta tf.Output, beta_min tf.Output, beta_max tf.Output, gamma tf.Output, gamma_min tf.Output, gamma_max tf.Output, out_type tf.DataType, variance_epsilon float32, scale_after_normalization bool) (result tf.Output, result_min tf.Output, result_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"out_type": out_type, "variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
 	opspec := tf.OpSpec{
-		Type: "Softmax",
+		Type: "QuantizedBatchNormWithGlobalNormalization",
 		Input: []tf.Input{
-			logits,
+			t, t_min, t_max, m, m_min, m_max, v, v_min, v_max, beta, beta_min, beta_max, gamma, gamma_min, gamma_max,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DecodeBmpAttr is an optional argument to DecodeBmp.
-type DecodeBmpAttr func(optionalAttr)
+// HistogramFixedWidthAttr is an optional argument to HistogramFixedWidth.
+type HistogramFixedWidthAttr func(optionalAttr)
 
-// DecodeBmpChannels sets the optional channels attribute to value.
-// If not specified, defaults to 0
-func DecodeBmpChannels(value int64) DecodeBmpAttr {
+// HistogramFixedWidthDtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT32
+func HistogramFixedWidthDtype(value tf.DataType) HistogramFixedWidthAttr {
 	return func(m optionalAttr) {
-		m["channels"] = value
+		m["dtype"] = value
 	}
 }
 
-// Decode the first frame of a BMP-encoded image to a uint8 tensor.
+// Return histogram of values.
 //
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
+// Given the tensor `values`, this operation returns a rank 1 histogram counting
+// the number of entries in `values` that fall into every bin.  The bins are
+// equal width and determined by the arguments `value_range` and `nbins`.
 //
-// Accepted values are:
+// ```python
+// # Bins will be:  (-inf, 1), [1, 2), [2, 3), [3, 4), [4, inf)
+// nbins = 5
+// value_range = [0.0, 5.0]
+// new_values = [-1.0, 0.0, 1.5, 2.0, 5.0, 15]
 //
-// *   0: Use the number of channels in the BMP-encoded image.
-// *   3: output an RGB image.
-// *   4: output an RGBA image.
+// with tf.get_default_session() as sess:
+//   hist = tf.histogram_fixed_width(new_values, value_range, nbins=5)
+//   variables.global_variables_initializer().run()
+//   sess.run(hist) => [2, 1, 1, 0, 2]
+// ```
 //
 // Arguments:
-//	contents: 0-D.  The BMP-encoded image.
+//	values: Numeric `Tensor`.
+//	value_range: Shape [2] `Tensor` of same `dtype` as `values`.
+// values <= value_range[0] will be mapped to hist[0],
+// values >= value_range[1] will be mapped to hist[-1].
+//	nbins: Scalar `int32 Tensor`.  Number of histogram bins.
 //
-// Returns 3-D with shape `[height, width, channels]`. RGB order
-func DecodeBmp(scope *Scope, contents tf.Output, optional ...DecodeBmpAttr) (image tf.Output) {
+// Returns A 1-D `Tensor` holding histogram of values.
+func HistogramFixedWidth(scope *Scope, values tf.Output, value_range tf.Output, nbins tf.Output, optional ...HistogramFixedWidthAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -3861,9 +3944,9 @@ func DecodeBmp(scope *Scope, contents tf.Output, optional ...DecodeBmpAttr) (ima
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeBmp",
+		Type: "HistogramFixedWidth",
 		Input: []tf.Input{
-			contents,
+			values, value_range, nbins,
 		},
 		Attrs: attrs,
 	}
@@ -3871,158 +3954,222 @@ func DecodeBmp(scope *Scope, contents tf.Output, optional ...DecodeBmpAttr) (ima
 	return op.Output(0)
 }
 
-// Computes softsign gradients for a softsign operation.
+// Adds Tensor 'bias' to Tensor 'input' for Quantized types.
+//
+// Broadcasts the values of bias on dimensions 0..N-2 of 'input'.
 //
 // Arguments:
-//	gradients: The backpropagated gradients to the corresponding softsign operation.
-//	features: The features passed as input to the corresponding softsign operation.
 //
-// Returns The gradients: `gradients / (1 + abs(features)) ** 2`.
-func SoftsignGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+//	bias: A 1D bias Tensor with size matching the last dimension of 'input'.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	min_bias: The float value that the lowest quantized bias value represents.
+//	max_bias: The float value that the highest quantized bias value represents.
+//
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedBiasAdd(scope *Scope, input tf.Output, bias tf.Output, min_input tf.Output, max_input tf.Output, min_bias tf.Output, max_bias tf.Output, out_type tf.DataType) (output tf.Output, min_out tf.Output, max_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "SoftsignGrad",
+		Type: "QuantizedBiasAdd",
 		Input: []tf.Input{
-			gradients, features,
+			input, bias, min_input, max_input, min_bias, max_bias,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// BatchMatMulAttr is an optional argument to BatchMatMul.
-type BatchMatMulAttr func(optionalAttr)
-
-// BatchMatMulAdjX sets the optional adj_x attribute to value.
+// Produces the average pool of the input tensor for quantized types.
 //
-// value: If `True`, adjoint the slices of `x`. Defaults to `False`.
-// If not specified, defaults to false
-func BatchMatMulAdjX(value bool) BatchMatMulAttr {
-	return func(m optionalAttr) {
-		m["adj_x"] = value
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, channels]`.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	ksize: The size of the window for each dimension of the input tensor.
+// The length must be 4 to match the number of dimensions of the input.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor.  The length must be 4 to match the number of dimensions of the input.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedAvgPool(scope *Scope, input tf.Output, min_input tf.Output, max_input tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output, min_output tf.Output, max_output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	opspec := tf.OpSpec{
+		Type: "QuantizedAvgPool",
+		Input: []tf.Input{
+			input, min_input, max_input,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// BatchMatMulAdjY sets the optional adj_y attribute to value.
+// FractionalAvgPoolAttr is an optional argument to FractionalAvgPool.
+type FractionalAvgPoolAttr func(optionalAttr)
+
+// FractionalAvgPoolPseudoRandom sets the optional pseudo_random attribute to value.
 //
-// value: If `True`, adjoint the slices of `y`. Defaults to `False`.
+// value: When set to True, generates the pooling sequence in a
+// pseudorandom fashion, otherwise, in a random fashion. Check paper [Benjamin
+// Graham, Fractional Max-Pooling](http://arxiv.org/abs/1412.6071) for
+// difference between pseudorandom and random.
 // If not specified, defaults to false
-func BatchMatMulAdjY(value bool) BatchMatMulAttr {
+func FractionalAvgPoolPseudoRandom(value bool) FractionalAvgPoolAttr {
 	return func(m optionalAttr) {
-		m["adj_y"] = value
+		m["pseudo_random"] = value
 	}
 }
 
-// Multiplies slices of two tensors in batches.
+// FractionalAvgPoolOverlapping sets the optional overlapping attribute to value.
 //
-// Multiplies all slices of `Tensor` `x` and `y` (each slice can be
-// viewed as an element of a batch), and arranges the individual results
-// in a single output tensor of the same batch size. Each of the
-// individual slices can optionally be adjointed (to adjoint a matrix
-// means to transpose and conjugate it) before multiplication by setting
-// the `adj_x` or `adj_y` flag to `True`, which are by default `False`.
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
 //
-// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
-// and `[..., r_y, c_y]`.
+// `index  0  1  2  3  4`
 //
-// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where:
+// `value  20 5  16 3  7`
 //
-//     r_o = c_x if adj_x else r_x
-//     c_o = r_y if adj_y else c_y
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [41/3, 26/3] for fractional avg pooling.
+// If not specified, defaults to false
+func FractionalAvgPoolOverlapping(value bool) FractionalAvgPoolAttr {
+	return func(m optionalAttr) {
+		m["overlapping"] = value
+	}
+}
+
+// FractionalAvgPoolDeterministic sets the optional deterministic attribute to value.
 //
-// It is computed as:
+// value: When set to True, a fixed pooling region will be used when
+// iterating over a FractionalAvgPool node in the computation graph. Mainly used
+// in unit test to make FractionalAvgPool deterministic.
+// If not specified, defaults to false
+func FractionalAvgPoolDeterministic(value bool) FractionalAvgPoolAttr {
+	return func(m optionalAttr) {
+		m["deterministic"] = value
+	}
+}
+
+// FractionalAvgPoolSeed sets the optional seed attribute to value.
 //
-//     output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func FractionalAvgPoolSeed(value int64) FractionalAvgPoolAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// FractionalAvgPoolSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func FractionalAvgPoolSeed2(value int64) FractionalAvgPoolAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Performs fractional average pooling on the input.
+//
+// Fractional average pooling is similar to Fractional max pooling in the pooling
+// region generation step. The only difference is that after pooling regions are
+// generated, a mean operation is performed instead of a max operation in each
+// pooling region.
 //
 // Arguments:
-//	x: 2-D or higher with shape `[..., r_x, c_x]`.
-//	y: 2-D or higher with shape `[..., r_y, c_y]`.
+//	value: 4-D with shape `[batch, height, width, channels]`.
+//	pooling_ratio: Pooling ratio for each dimension of `value`, currently only
+// supports row and col dimension and should be >= 1.0. For example, a valid
+// pooling ratio looks like [1.0, 1.44, 1.73, 1.0]. The first and last elements
+// must be 1.0 because we don't allow pooling on batch and channels
+// dimensions. 1.44 and 1.73 are pooling ratio on height and width dimensions
+// respectively.
 //
-// Returns 3-D or higher with shape `[..., r_o, c_o]`
-func BatchMatMul(scope *Scope, x tf.Output, y tf.Output, optional ...BatchMatMulAttr) (output tf.Output) {
+// Returns output tensor after fractional avg pooling.row pooling sequence, needed to calculate gradient.column pooling sequence, needed to calculate gradient.
+func FractionalAvgPool(scope *Scope, value tf.Output, pooling_ratio []float32, optional ...FractionalAvgPoolAttr) (output tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"pooling_ratio": pooling_ratio}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "BatchMatMul",
+		Type: "FractionalAvgPool",
 		Input: []tf.Input{
-			x, y,
+			value,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns which elements of x are NaN.
+// RandomCropAttr is an optional argument to RandomCrop.
+type RandomCropAttr func(optionalAttr)
+
+// RandomCropSeed sets the optional seed attribute to value.
 //
-// @compatibility(numpy)
-// Equivalent to np.isnan
-// @end_compatibility
-func IsNan(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IsNan",
-		Input: []tf.Input{
-			x,
-		},
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomCropSeed(value int64) RandomCropAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes rectified linear gradients for a Relu operation.
-//
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Relu operation.
-//	features: The features passed as input to the corresponding Relu operation, OR
-// the outputs of that operation (both work equivalently).
+// RandomCropSeed2 sets the optional seed2 attribute to value.
 //
-// Returns `gradients * (features > 0)`.
-func ReluGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReluGrad",
-		Input: []tf.Input{
-			gradients, features,
-		},
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomCropSeed2(value int64) RandomCropAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the gradient of morphological 2-D dilation with respect to the input.
+// Randomly crop `image`.
+//
+// DEPRECATED at GraphDef version 8: Random crop is now pure Python
+//
+// `size` is a 1-D int64 tensor with 2 elements representing the crop height and
+// width.  The values must be non negative.
+//
+// This Op picks a random location in `image` and crops a `height` by `width`
+// rectangle from that location.  The random location is picked so the cropped
+// area will fit inside the original image.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
-//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
-//	out_backprop: 4-D with shape `[batch, out_height, out_width, depth]`.
-//	strides: 1-D of length 4. The stride of the sliding window for each dimension of
-// the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
-//	rates: 1-D of length 4. The input stride for atrous morphological dilation.
-// Must be: `[1, rate_height, rate_width, 1]`.
-//	padding: The type of padding algorithm to use.
+//	image: 3-D of shape `[height, width, channels]`.
+//	size: 1-D of length 2 containing: `crop_height`, `crop_width`..
 //
-// Returns 4-D with shape `[batch, in_height, in_width, depth]`.
-func Dilation2DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, rates []int64, padding string) (in_backprop tf.Output) {
+// Returns 3-D of shape `[crop_height, crop_width, channels].`
+func RandomCrop(scope *Scope, image tf.Output, size tf.Output, optional ...RandomCropAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Dilation2DBackpropInput",
+		Type: "RandomCrop",
 		Input: []tf.Input{
-			input, filter, out_backprop,
+			image, size,
 		},
 		Attrs: attrs,
 	}
@@ -4030,238 +4177,203 @@ func Dilation2DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, ou
 	return op.Output(0)
 }
 
-// CTCBeamSearchDecoderAttr is an optional argument to CTCBeamSearchDecoder.
-type CTCBeamSearchDecoderAttr func(optionalAttr)
+// TopKV2Attr is an optional argument to TopKV2.
+type TopKV2Attr func(optionalAttr)
 
-// CTCBeamSearchDecoderMergeRepeated sets the optional merge_repeated attribute to value.
+// TopKV2Sorted sets the optional sorted attribute to value.
 //
-// value: If true, merge repeated classes in output.
+// value: If true the resulting `k` elements will be sorted by the values in
+// descending order.
 // If not specified, defaults to true
-func CTCBeamSearchDecoderMergeRepeated(value bool) CTCBeamSearchDecoderAttr {
+func TopKV2Sorted(value bool) TopKV2Attr {
 	return func(m optionalAttr) {
-		m["merge_repeated"] = value
+		m["sorted"] = value
 	}
 }
 
-// Performs beam search decoding on the logits given in input.
+// Finds values and indices of the `k` largest elements for the last dimension.
 //
-// A note about the attribute merge_repeated: For the beam search decoder,
-// this means that if consecutive entries in a beam are the same, only
-// the first of these is emitted.  That is, when the top path is "A B B B B",
-// "A B" is returned if merge_repeated = True but "A B B B B" is
-// returned if merge_repeated = False.
+// If the input is a vector (rank-1), finds the `k` largest entries in the vector
+// and outputs their values and indices as vectors.  Thus `values[j]` is the
+// `j`-th largest entry in `input`, and its index is `indices[j]`.
+//
+// For matrices (resp. higher rank input), computes the top `k` entries in each
+// row (resp. vector along the last dimension).  Thus,
+//
+//     values.shape = indices.shape = input.shape[:-1] + [k]
+//
+// If two elements are equal, the lower-index element appears first.
 //
 // Arguments:
-//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
-//	sequence_length: A vector containing sequence lengths, size `(batch)`.
-//	beam_width: A scalar >= 0 (beam search beam width).
-//	top_paths: A scalar >= 0, <= beam_width (controls output size).
+//	input: 1-D or higher with last dimension at least `k`.
+//	k: 0-D.  Number of top elements to look for along the last dimension (along each
+// row for matrices).
 //
-// Returns A list (length: top_paths) of indices matrices.  Matrix j,
-// size `(total_decoded_outputs[j] x 2)`, has indices of a
-// `SparseTensor<int64, 2>`.  The rows store: [batch, time].A list (length: top_paths) of values vectors.  Vector j,
-// size `(length total_decoded_outputs[j])`, has the values of a
-// `SparseTensor<int64, 2>`.  The vector stores the decoded classes for beam j.A list (length: top_paths) of shape vector.  Vector j,
-// size `(2)`, stores the shape of the decoded `SparseTensor[j]`.
-// Its values are: `[batch_size, max_decoded_length[j]]`.A matrix, shaped: `(batch_size x top_paths)`.  The
-// sequence log-probabilities.
-func CTCBeamSearchDecoder(scope *Scope, inputs tf.Output, sequence_length tf.Output, beam_width int64, top_paths int64, optional ...CTCBeamSearchDecoderAttr) (decoded_indices []tf.Output, decoded_values []tf.Output, decoded_shape []tf.Output, log_probability tf.Output) {
+// Returns The `k` largest elements along each last dimensional slice.The indices of `values` within the last dimension of `input`.
+func TopKV2(scope *Scope, input tf.Output, k tf.Output, optional ...TopKV2Attr) (values tf.Output, indices tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"beam_width": beam_width, "top_paths": top_paths}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "CTCBeamSearchDecoder",
+		Type: "TopKV2",
 		Input: []tf.Input{
-			inputs, sequence_length,
+			input, k,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Returns x // y element-wise.
+//
+// *NOTE*: `FloorDiv` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func FloorDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if decoded_indices, idx, err = makeOutputList(op, idx, "decoded_indices"); err != nil {
-		scope.UpdateErr("CTCBeamSearchDecoder", err)
-		return
-	}
-	if decoded_values, idx, err = makeOutputList(op, idx, "decoded_values"); err != nil {
-		scope.UpdateErr("CTCBeamSearchDecoder", err)
-		return
-	}
-	if decoded_shape, idx, err = makeOutputList(op, idx, "decoded_shape"); err != nil {
-		scope.UpdateErr("CTCBeamSearchDecoder", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "FloorDiv",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
-	log_probability = op.Output(idx)
-	return decoded_indices, decoded_values, decoded_shape, log_probability
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// AudioSpectrogramAttr is an optional argument to AudioSpectrogram.
-type AudioSpectrogramAttr func(optionalAttr)
-
-// AudioSpectrogramMagnitudeSquared sets the optional magnitude_squared attribute to value.
+// Returns a batched diagonal tensor with a given batched diagonal values.
 //
-// value: Whether to return the squared magnitude or just the
-// magnitude. Using squared magnitude can avoid extra calculations.
-// If not specified, defaults to false
-func AudioSpectrogramMagnitudeSquared(value bool) AudioSpectrogramAttr {
-	return func(m optionalAttr) {
-		m["magnitude_squared"] = value
-	}
-}
-
-// Produces a visualization of audio data over time.
+// Given a `diagonal`, this operation returns a tensor with the `diagonal` and
+// everything else padded with zeros. The diagonal is computed as follows:
 //
-// Spectrograms are a standard way of representing audio information as a series of
-// slices of frequency information, one slice for each window of time. By joining
-// these together into a sequence, they form a distinctive fingerprint of the sound
-// over time.
+// Assume `diagonal` has `k` dimensions `[I, J, K, ..., N]`, then the output is a
+// tensor of rank `k+1` with dimensions [I, J, K, ..., N, N]` where:
 //
-// This op expects to receive audio data as an input, stored as floats in the range
-// -1 to 1, together with a window width in samples, and a stride specifying how
-// far to move the window between slices. From this it generates a three
-// dimensional output. The lowest dimension has an amplitude value for each
-// frequency during that time slice. The next dimension is time, with successive
-// frequency slices. The final dimension is for the channels in the input, so a
-// stereo audio input would have two here for example.
+// `output[i, j, k, ..., m, n] = 1{m=n} * diagonal[i, j, k, ..., n]`.
 //
-// This means the layout when converted and saved as an image is rotated 90 degrees
-// clockwise from a typical spectrogram. Time is descending down the Y axis, and
-// the frequency decreases from left to right.
+// For example:
 //
-// Each value in the result represents the square root of the sum of the real and
-// imaginary parts of an FFT on the current window of samples. In this way, the
-// lowest dimension represents the power of each frequency in the current window,
-// and adjacent windows are concatenated in the next dimension.
+// ```
+// # 'diagonal' is [[1, 2, 3, 4], [5, 6, 7, 8]]
 //
-// To get a more intuitive and visual look at what this operation does, you can run
-// tensorflow/examples/wav_to_spectrogram to read in an audio file and save out the
-// resulting spectrogram as a PNG image.
+// and diagonal.shape = (2, 4)
+//
+// tf.matrix_diag(diagonal) ==> [[[1, 0, 0, 0]
+//                                      [0, 2, 0, 0]
+//                                      [0, 0, 3, 0]
+//                                      [0, 0, 0, 4]],
+//                                     [[5, 0, 0, 0]
+//                                      [0, 6, 0, 0]
+//                                      [0, 0, 7, 0]
+//                                      [0, 0, 0, 8]]]
+//
+// which has shape (2, 4, 4)
+// ```
 //
 // Arguments:
-//	input: Float representation of audio data.
-//	window_size: How wide the input window is in samples. For the highest efficiency
-// this should be a power of two, but other values are accepted.
-//	stride: How widely apart the center of adjacent sample windows should be.
+//	diagonal: Rank `k`, where `k >= 1`.
 //
-// Returns 3D representation of the audio frequencies as an image.
-func AudioSpectrogram(scope *Scope, input tf.Output, window_size int64, stride int64, optional ...AudioSpectrogramAttr) (spectrogram tf.Output) {
+// Returns Rank `k+1`, with `output.shape = diagonal.shape + [diagonal.shape[-1]]`.
+func MatrixDiag(scope *Scope, diagonal tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"window_size": window_size, "stride": stride}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AudioSpectrogram",
+		Type: "MatrixDiag",
 		Input: []tf.Input{
-			input,
+			diagonal,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the polygamma function \\(\psi^{(n)}(x)\\).
-//
-// The polygamma function is defined as:
-//
-//
-// \\(\psi^{(n)}(x) = \frac{d^n}{dx^n} \psi(x)\\)
+// Returns the truth value of (x <= y) element-wise.
 //
-// where \\(\psi(x)\\) is the digamma function.
-func Polygamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// *NOTE*: `LessEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func LessEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Polygamma",
+		Type: "LessEqual",
 		Input: []tf.Input{
-			a, x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Computes softmax activations.
+//
+// For each batch `i` and class `j` we have
+//
+//     softmax[i, j] = exp(logits[i, j]) / sum_j(exp(logits[i, j]))
 //
 // Arguments:
-//	input: The original input.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
-// input of `max_pool`.
-//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	logits: 2-D with shape `[batch_size, num_classes]`.
 //
-// Returns Gradients of gradients w.r.t. the input of `max_pool`.
-func MaxPoolGradGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
+// Returns Same shape as `logits`.
+func Softmax(scope *Scope, logits tf.Output) (softmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGradWithArgmax",
+		Type: "Softmax",
 		Input: []tf.Input{
-			input, grad, argmax,
+			logits,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2.
-type MaxPoolGradGradV2Attr func(optionalAttr)
+// DecodeBmpAttr is an optional argument to DecodeBmp.
+type DecodeBmpAttr func(optionalAttr)
 
-// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr {
+// DecodeBmpChannels sets the optional channels attribute to value.
+// If not specified, defaults to 0
+func DecodeBmpChannels(value int64) DecodeBmpAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["channels"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Decode the first frame of a BMP-encoded image to a uint8 tensor.
+//
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the BMP-encoded image.
+// *   3: output an RGB image.
+// *   4: output an RGBA image.
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	contents: 0-D.  The BMP-encoded image.
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) {
+// Returns 3-D with shape `[height, width, channels]`. RGB order
+func DecodeBmp(scope *Scope, contents tf.Output, optional ...DecodeBmpAttr) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGradV2",
+		Type: "DecodeBmp",
 		Input: []tf.Input{
-			orig_input, orig_output, grad, ksize, strides,
+			contents,
 		},
 		Attrs: attrs,
 	}
@@ -4269,75 +4381,88 @@ func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output
 	return op.Output(0)
 }
 
-// Computes gradients of the maxpooling function.
+// Computes softsign gradients for a softsign operation.
 //
 // Arguments:
-//	input: The original input.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
-// output of `max_pool`.
-//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	gradients: The backpropagated gradients to the corresponding softsign operation.
+//	features: The features passed as input to the corresponding softsign operation.
 //
-// Returns Gradients w.r.t. the input of `max_pool`.
-func MaxPoolGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
+// Returns The gradients: `gradients / (1 + abs(features)) ** 2`.
+func SoftsignGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradWithArgmax",
+		Type: "SoftsignGrad",
 		Input: []tf.Input{
-			input, grad, argmax,
+			gradients, features,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AvgPool3DAttr is an optional argument to AvgPool3D.
-type AvgPool3DAttr func(optionalAttr)
+// BatchMatMulAttr is an optional argument to BatchMatMul.
+type BatchMatMulAttr func(optionalAttr)
 
-// AvgPool3DDataFormat sets the optional data_format attribute to value.
+// BatchMatMulAdjX sets the optional adj_x attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func AvgPool3DDataFormat(value string) AvgPool3DAttr {
+// value: If `True`, adjoint the slices of `x`. Defaults to `False`.
+// If not specified, defaults to false
+func BatchMatMulAdjX(value bool) BatchMatMulAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["adj_x"] = value
 	}
 }
 
-// Performs 3D average pooling on the input.
+// BatchMatMulAdjY sets the optional adj_y attribute to value.
+//
+// value: If `True`, adjoint the slices of `y`. Defaults to `False`.
+// If not specified, defaults to false
+func BatchMatMulAdjY(value bool) BatchMatMulAttr {
+	return func(m optionalAttr) {
+		m["adj_y"] = value
+	}
+}
+
+// Multiplies slices of two tensors in batches.
+//
+// Multiplies all slices of `Tensor` `x` and `y` (each slice can be
+// viewed as an element of a batch), and arranges the individual results
+// in a single output tensor of the same batch size. Each of the
+// individual slices can optionally be adjointed (to adjoint a matrix
+// means to transpose and conjugate it) before multiplication by setting
+// the `adj_x` or `adj_y` flag to `True`, which are by default `False`.
+//
+// The input tensors `x` and `y` are 2-D or higher with shape `[..., r_x, c_x]`
+// and `[..., r_y, c_y]`.
+//
+// The output tensor is 2-D or higher with shape `[..., r_o, c_o]`, where:
+//
+//     r_o = c_x if adj_x else r_x
+//     c_o = r_y if adj_y else c_y
+//
+// It is computed as:
+//
+//     output[..., :, :] = matrix(x[..., :, :]) * matrix(y[..., :, :])
 //
 // Arguments:
-//	input: Shape `[batch, depth, rows, cols, channels]` tensor to pool over.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+//	x: 2-D or higher with shape `[..., r_x, c_x]`.
+//	y: 2-D or higher with shape `[..., r_y, c_y]`.
 //
-// Returns The average pooled output tensor.
-func AvgPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DAttr) (output tf.Output) {
+// Returns 3-D or higher with shape `[..., r_o, c_o]`
+func BatchMatMul(scope *Scope, x tf.Output, y tf.Output, optional ...BatchMatMulAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AvgPool3D",
+		Type: "BatchMatMul",
 		Input: []tf.Input{
-			input,
+			x, y,
 		},
 		Attrs: attrs,
 	}
@@ -4345,208 +4470,69 @@ func AvgPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, pa
 	return op.Output(0)
 }
 
-// Returns element-wise remainder of division. This emulates C semantics in that
-//
-// the result here is consistent with a truncating divide. E.g.
-// `tf.truncatediv(x, y) * y + truncate_mod(x, y) = x`.
+// Returns which elements of x are NaN.
 //
-// *NOTE*: `Mod` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Mod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.isnan
+// @end_compatibility
+func IsNan(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Mod",
+		Type: "IsNan",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DepthToSpaceAttr is an optional argument to DepthToSpace.
-type DepthToSpaceAttr func(optionalAttr)
-
-// DepthToSpaceDataFormat sets the optional data_format attribute to value.
-// If not specified, defaults to "NHWC"
-func DepthToSpaceDataFormat(value string) DepthToSpaceAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// DepthToSpace for tensors of type T.
-//
-// Rearranges data from depth into blocks of spatial data.
-// This is the reverse transformation of SpaceToDepth. More specifically,
-// this op outputs a copy of the input tensor where values from the `depth`
-// dimension are moved in spatial blocks to the `height` and `width` dimensions.
-// The attr `block_size` indicates the input block size and how the data is moved.
-//
-//   * Chunks of data of size `block_size * block_size` from depth are rearranged
-//     into non-overlapping blocks of size `block_size x block_size`
-//   * The width the output tensor is `input_depth * block_size`, whereas the
-//     height is `input_height * block_size`.
-//   * The Y, X coordinates within each block of the output image are determined
-//     by the high order component of the input channel index.
-//   * The depth of the input tensor must be divisible by
-//     `block_size * block_size`.
-//
-// The `data_format` attr specifies the layout of the input and output tensors
-// with the following options:
-//   "NHWC": `[ batch, height, width, channels ]`
-//   "NCHW": `[ batch, channels, height, width ]`
-//   "NCHW_VECT_C":
-//       `qint8 [ batch, channels / 4, height, width, 4 ]`
-//
-// It is useful to consider the operation as transforming a 6-D Tensor.
-// e.g. for data_format = NHWC,
-//      Each element in the input tensor can be specified via 6 coordinates,
-//      ordered by decreasing memory layout significance as:
-//      n,iY,iX,bY,bX,oC  (where n=batch index, iX, iY means X or Y coordinates
-//                         within the input image, bX, bY means coordinates
-//                         within the output block, oC means output channels).
-//      The output would be the input transposed to the following layout:
-//      n,iY,bY,iX,bX,oC
-//
-// This operation is useful for resizing the activations between convolutions
-// (but keeping all data), e.g. instead of pooling. It is also useful for training
-// purely convolutional models.
-//
-// For example, given an input of shape `[1, 1, 1, 4]`, data_format = "NHWC" and
-// block_size = 2:
-//
-// ```
-// x = [[[[1, 2, 3, 4]]]]
-//
-// ```
-//
-// This operation will output a tensor of shape `[1, 2, 2, 1]`:
-//
-// ```
-//    [[[[1], [2]],
-//      [[3], [4]]]]
-// ```
-//
-// Here, the input has a batch of 1 and each batch element has shape `[1, 1, 4]`,
-// the corresponding output will have 2x2 elements and will have a depth of
-// 1 channel (1 = `4 / (block_size * block_size)`).
-// The output element shape is `[2, 2, 1]`.
-//
-// For an input tensor with larger depth, here of shape `[1, 1, 1, 12]`, e.g.
-//
-// ```
-// x = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
-// ```
-//
-// This operation, for block size of 2, will return the following tensor of shape
-// `[1, 2, 2, 3]`
-//
-// ```
-//    [[[[1, 2, 3], [4, 5, 6]],
-//      [[7, 8, 9], [10, 11, 12]]]]
-//
-// ```
-//
-// Similarly, for the following input of shape `[1 2 2 4]`, and a block size of 2:
-//
-// ```
-// x =  [[[[1, 2, 3, 4],
-//        [5, 6, 7, 8]],
-//       [[9, 10, 11, 12],
-//        [13, 14, 15, 16]]]]
-// ```
-//
-// the operator will return the following tensor of shape `[1 4 4 1]`:
-//
-// ```
-// x = [[[ [1],   [2],  [5],  [6]],
-//       [ [3],   [4],  [7],  [8]],
-//       [ [9],  [10], [13],  [14]],
-//       [ [11], [12], [15],  [16]]]]
-//
-// ```
+// Computes rectified linear gradients for a Relu operation.
 //
 // Arguments:
+//	gradients: The backpropagated gradients to the corresponding Relu operation.
+//	features: The features passed as input to the corresponding Relu operation, OR
+// the outputs of that operation (both work equivalently).
 //
-//	block_size: The size of the spatial block, same as in Space2Depth.
-func DepthToSpace(scope *Scope, input tf.Output, block_size int64, optional ...DepthToSpaceAttr) (output tf.Output) {
+// Returns `gradients * (features > 0)`.
+func ReluGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"block_size": block_size}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DepthToSpace",
+		Type: "ReluGrad",
 		Input: []tf.Input{
-			input,
+			gradients, features,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv3DBackpropInputV2Attr is an optional argument to Conv3DBackpropInputV2.
-type Conv3DBackpropInputV2Attr func(optionalAttr)
-
-// Conv3DBackpropInputV2DataFormat sets the optional data_format attribute to value.
-//
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func Conv3DBackpropInputV2DataFormat(value string) Conv3DBackpropInputV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Conv3DBackpropInputV2Dilations sets the optional dilations attribute to value.
-//
-// value: 1-D tensor of length 5.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DBackpropInputV2Dilations(value []int64) Conv3DBackpropInputV2Attr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of 3-D convolution with respect to the input.
+// Computes the gradient of morphological 2-D dilation with respect to the input.
 //
 // Arguments:
-//	input_sizes: An integer vector representing the tensor shape of `input`,
-// where `input` is a 5-D
-// `[batch, depth, rows, cols, in_channels]` tensor.
-//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
-// `in_channels` must match between `input` and `filter`.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
+//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
+//	out_backprop: 4-D with shape `[batch, out_height, out_width, depth]`.
+//	strides: 1-D of length 4. The stride of the sliding window for each dimension of
+// the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
+//	rates: 1-D of length 4. The input stride for atrous morphological dilation.
+// Must be: `[1, rate_height, rate_width, 1]`.
 //	padding: The type of padding algorithm to use.
-func Conv3DBackpropInputV2(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropInputV2Attr) (output tf.Output) {
+//
+// Returns 4-D with shape `[batch, in_height, in_width, depth]`.
+func Dilation2DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, rates []int64, padding string) (in_backprop tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropInputV2",
+		Type: "Dilation2DBackpropInput",
 		Input: []tf.Input{
-			input_sizes, filter, out_backprop,
+			input, filter, out_backprop,
 		},
 		Attrs: attrs,
 	}
@@ -4554,159 +4540,139 @@ func Conv3DBackpropInputV2(scope *Scope, input_sizes tf.Output, filter tf.Output
 	return op.Output(0)
 }
 
-// Computes square root of x element-wise.
+// CTCBeamSearchDecoderAttr is an optional argument to CTCBeamSearchDecoder.
+type CTCBeamSearchDecoderAttr func(optionalAttr)
+
+// CTCBeamSearchDecoderMergeRepeated sets the optional merge_repeated attribute to value.
 //
-// I.e., \\(y = \sqrt{x} = x^{1/2}\\).
-func Sqrt(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Sqrt",
-		Input: []tf.Input{
-			x,
-		},
+// value: If true, merge repeated classes in output.
+// If not specified, defaults to true
+func CTCBeamSearchDecoderMergeRepeated(value bool) CTCBeamSearchDecoderAttr {
+	return func(m optionalAttr) {
+		m["merge_repeated"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes the gradients of 3-D convolution with respect to the filter.
+// Performs beam search decoding on the logits given in input.
 //
-// DEPRECATED at GraphDef version 10: Use Conv3DBackpropFilterV2
+// A note about the attribute merge_repeated: For the beam search decoder,
+// this means that if consecutive entries in a beam are the same, only
+// the first of these is emitted.  That is, when the top path is "A B B B B",
+// "A B" is returned if merge_repeated = True but "A B B B B" is
+// returned if merge_repeated = False.
 //
 // Arguments:
-//	input: Shape `[batch, depth, rows, cols, in_channels]`.
-//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
-// `in_channels` must match between `input` and `filter`.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) {
+//	inputs: 3-D, shape: `(max_time x batch_size x num_classes)`, the logits.
+//	sequence_length: A vector containing sequence lengths, size `(batch)`.
+//	beam_width: A scalar >= 0 (beam search beam width).
+//	top_paths: A scalar >= 0, <= beam_width (controls output size).
+//
+// Returns A list (length: top_paths) of indices matrices.  Matrix j,
+// size `(total_decoded_outputs[j] x 2)`, has indices of a
+// `SparseTensor<int64, 2>`.  The rows store: [batch, time].A list (length: top_paths) of values vectors.  Vector j,
+// size `(length total_decoded_outputs[j])`, has the values of a
+// `SparseTensor<int64, 2>`.  The vector stores the decoded classes for beam j.A list (length: top_paths) of shape vector.  Vector j,
+// size `(2)`, stores the shape of the decoded `SparseTensor[j]`.
+// Its values are: `[batch_size, max_decoded_length[j]]`.A matrix, shaped: `(batch_size x top_paths)`.  The
+// sequence log-probabilities.
+func CTCBeamSearchDecoder(scope *Scope, inputs tf.Output, sequence_length tf.Output, beam_width int64, top_paths int64, optional ...CTCBeamSearchDecoderAttr) (decoded_indices []tf.Output, decoded_values []tf.Output, decoded_shape []tf.Output, log_probability tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"beam_width": beam_width, "top_paths": top_paths}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropFilter",
+		Type: "CTCBeamSearchDecoder",
 		Input: []tf.Input{
-			input, filter, out_backprop,
+			inputs, sequence_length,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the gradient for the rsqrt of `x` wrt its input.
-//
-// Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy`
-// is the corresponding input gradient.
-func RsqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "RsqrtGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
+	var idx int
+	var err error
+	if decoded_indices, idx, err = makeOutputList(op, idx, "decoded_indices"); err != nil {
+		scope.UpdateErr("CTCBeamSearchDecoder", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if decoded_values, idx, err = makeOutputList(op, idx, "decoded_values"); err != nil {
+		scope.UpdateErr("CTCBeamSearchDecoder", err)
+		return
+	}
+	if decoded_shape, idx, err = makeOutputList(op, idx, "decoded_shape"); err != nil {
+		scope.UpdateErr("CTCBeamSearchDecoder", err)
+		return
+	}
+	log_probability = op.Output(idx)
+	return decoded_indices, decoded_values, decoded_shape, log_probability
 }
 
-// ReverseSequenceAttr is an optional argument to ReverseSequence.
-type ReverseSequenceAttr func(optionalAttr)
+// AudioSpectrogramAttr is an optional argument to AudioSpectrogram.
+type AudioSpectrogramAttr func(optionalAttr)
 
-// ReverseSequenceBatchDim sets the optional batch_dim attribute to value.
+// AudioSpectrogramMagnitudeSquared sets the optional magnitude_squared attribute to value.
 //
-// value: The dimension along which reversal is performed.
-// If not specified, defaults to 0
-func ReverseSequenceBatchDim(value int64) ReverseSequenceAttr {
+// value: Whether to return the squared magnitude or just the
+// magnitude. Using squared magnitude can avoid extra calculations.
+// If not specified, defaults to false
+func AudioSpectrogramMagnitudeSquared(value bool) AudioSpectrogramAttr {
 	return func(m optionalAttr) {
-		m["batch_dim"] = value
+		m["magnitude_squared"] = value
 	}
 }
 
-// Reverses variable length slices.
-//
-// This op first slices `input` along the dimension `batch_dim`, and for each
-// slice `i`, reverses the first `seq_lengths[i]` elements along
-// the dimension `seq_dim`.
-//
-// The elements of `seq_lengths` must obey `seq_lengths[i] <= input.dims[seq_dim]`,
-// and `seq_lengths` must be a vector of length `input.dims[batch_dim]`.
-//
-// The output slice `i` along dimension `batch_dim` is then given by input
-// slice `i`, with the first `seq_lengths[i]` slices along dimension
-// `seq_dim` reversed.
-//
-// For example:
-//
-// ```
-// # Given this:
-// batch_dim = 0
-// seq_dim = 1
-// input.dims = (4, 8, ...)
-// seq_lengths = [7, 2, 3, 5]
-//
-// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
-// output[0, 0:7, :, ...] = input[0, 7:0:-1, :, ...]
-// output[1, 0:2, :, ...] = input[1, 2:0:-1, :, ...]
-// output[2, 0:3, :, ...] = input[2, 3:0:-1, :, ...]
-// output[3, 0:5, :, ...] = input[3, 5:0:-1, :, ...]
+// Produces a visualization of audio data over time.
 //
-// # while entries past seq_lens are copied through:
-// output[0, 7:, :, ...] = input[0, 7:, :, ...]
-// output[1, 2:, :, ...] = input[1, 2:, :, ...]
-// output[2, 3:, :, ...] = input[2, 3:, :, ...]
-// output[3, 2:, :, ...] = input[3, 2:, :, ...]
-// ```
+// Spectrograms are a standard way of representing audio information as a series of
+// slices of frequency information, one slice for each window of time. By joining
+// these together into a sequence, they form a distinctive fingerprint of the sound
+// over time.
 //
-// In contrast, if:
+// This op expects to receive audio data as an input, stored as floats in the range
+// -1 to 1, together with a window width in samples, and a stride specifying how
+// far to move the window between slices. From this it generates a three
+// dimensional output. The lowest dimension has an amplitude value for each
+// frequency during that time slice. The next dimension is time, with successive
+// frequency slices. The final dimension is for the channels in the input, so a
+// stereo audio input would have two here for example.
 //
-// ```
-// # Given this:
-// batch_dim = 2
-// seq_dim = 0
-// input.dims = (8, ?, 4, ...)
-// seq_lengths = [7, 2, 3, 5]
+// This means the layout when converted and saved as an image is rotated 90 degrees
+// clockwise from a typical spectrogram. Time is descending down the Y axis, and
+// the frequency decreases from left to right.
 //
-// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
-// output[0:7, :, 0, :, ...] = input[7:0:-1, :, 0, :, ...]
-// output[0:2, :, 1, :, ...] = input[2:0:-1, :, 1, :, ...]
-// output[0:3, :, 2, :, ...] = input[3:0:-1, :, 2, :, ...]
-// output[0:5, :, 3, :, ...] = input[5:0:-1, :, 3, :, ...]
+// Each value in the result represents the square root of the sum of the real and
+// imaginary parts of an FFT on the current window of samples. In this way, the
+// lowest dimension represents the power of each frequency in the current window,
+// and adjacent windows are concatenated in the next dimension.
 //
-// # while entries past seq_lens are copied through:
-// output[7:, :, 0, :, ...] = input[7:, :, 0, :, ...]
-// output[2:, :, 1, :, ...] = input[2:, :, 1, :, ...]
-// output[3:, :, 2, :, ...] = input[3:, :, 2, :, ...]
-// output[2:, :, 3, :, ...] = input[2:, :, 3, :, ...]
-// ```
+// To get a more intuitive and visual look at what this operation does, you can run
+// tensorflow/examples/wav_to_spectrogram to read in an audio file and save out the
+// resulting spectrogram as a PNG image.
 //
 // Arguments:
-//	input: The input to reverse.
-//	seq_lengths: 1-D with length `input.dims(batch_dim)` and
-// `max(seq_lengths) <= input.dims(seq_dim)`
-//	seq_dim: The dimension which is partially reversed.
+//	input: Float representation of audio data.
+//	window_size: How wide the input window is in samples. For the highest efficiency
+// this should be a power of two, but other values are accepted.
+//	stride: How widely apart the center of adjacent sample windows should be.
 //
-// Returns The partially reversed input. It has the same shape as `input`.
-func ReverseSequence(scope *Scope, input tf.Output, seq_lengths tf.Output, seq_dim int64, optional ...ReverseSequenceAttr) (output tf.Output) {
+// Returns 3D representation of the audio frequencies as an image.
+func AudioSpectrogram(scope *Scope, input tf.Output, window_size int64, stride int64, optional ...AudioSpectrogramAttr) (spectrogram tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"seq_dim": seq_dim}
+	attrs := map[string]interface{}{"window_size": window_size, "stride": stride}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ReverseSequence",
+		Type: "AudioSpectrogram",
 		Input: []tf.Input{
-			input, seq_lengths,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -4714,76 +4680,50 @@ func ReverseSequence(scope *Scope, input tf.Output, seq_lengths tf.Output, seq_d
 	return op.Output(0)
 }
 
-// DepthwiseConv2dNativeAttr is an optional argument to DepthwiseConv2dNative.
-type DepthwiseConv2dNativeAttr func(optionalAttr)
-
-// DepthwiseConv2dNativeDataFormat sets the optional data_format attribute to value.
+// Compute the polygamma function \\(\psi^{(n)}(x)\\).
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
-// If not specified, defaults to "NHWC"
-func DepthwiseConv2dNativeDataFormat(value string) DepthwiseConv2dNativeAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// DepthwiseConv2dNativeDilations sets the optional dilations attribute to value.
+// The polygamma function is defined as:
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func DepthwiseConv2dNativeDilations(value []int64) DepthwiseConv2dNativeAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
+//
+// \\(\psi^{(n)}(x) = \frac{d^n}{dx^n} \psi(x)\\)
+//
+// where \\(\psi(x)\\) is the digamma function.
+func Polygamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Polygamma",
+		Input: []tf.Input{
+			a, x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes a 2-D depthwise convolution given 4-D `input` and `filter` tensors.
-//
-// Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
-// and a filter / kernel tensor of shape
-// `[filter_height, filter_width, in_channels, channel_multiplier]`, containing
-// `in_channels` convolutional filters of depth 1, `depthwise_conv2d` applies
-// a different filter to each input channel (expanding from 1 channel to
-// `channel_multiplier` channels for each), then concatenates the results
-// together. Thus, the output has `in_channels * channel_multiplier` channels.
-//
-// ```
-// for k in 0..in_channels-1
-//   for q in 0..channel_multiplier-1
-//     output[b, i, j, k * channel_multiplier + q] =
-//       sum_{di, dj} input[b, strides[1] * i + di, strides[2] * j + dj, k] *
-//                         filter[di, dj, k, q]
-// ```
-//
-// Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
-// horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
+// Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
-//
-//
-//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
-// of `input`.
+//	input: The original input.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
+// input of `max_pool`.
+//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
 //	padding: The type of padding algorithm to use.
-func DepthwiseConv2dNative(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeAttr) (output tf.Output) {
+//
+// Returns Gradients of gradients w.r.t. the input of `max_pool`.
+func MaxPoolGradGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "DepthwiseConv2dNative",
+		Type: "MaxPoolGradGradWithArgmax",
 		Input: []tf.Input{
-			input, filter,
+			input, grad, argmax,
 		},
 		Attrs: attrs,
 	}
@@ -4791,10 +4731,10 @@ func DepthwiseConv2dNative(scope *Scope, input tf.Output, filter tf.Output, stri
 	return op.Output(0)
 }
 
-// MaxPoolGradV2Attr is an optional argument to MaxPoolGradV2.
-type MaxPoolGradV2Attr func(optionalAttr)
+// MaxPoolGradGradV2Attr is an optional argument to MaxPoolGradGradV2.
+type MaxPoolGradGradV2Attr func(optionalAttr)
 
-// MaxPoolGradV2DataFormat sets the optional data_format attribute to value.
+// MaxPoolGradGradV2DataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
 // default format "NHWC", the data is stored in the order of:
@@ -4802,25 +4742,25 @@ type MaxPoolGradV2Attr func(optionalAttr)
 // Alternatively, the format could be "NCHW", the data storage order of:
 //     [batch, in_channels, in_height, in_width].
 // If not specified, defaults to "NHWC"
-func MaxPoolGradV2DataFormat(value string) MaxPoolGradV2Attr {
+func MaxPoolGradGradV2DataFormat(value string) MaxPoolGradGradV2Attr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// Computes gradients of the maxpooling function.
+// Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
 //	orig_input: The original input tensor.
 //	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
+//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
 //	ksize: The size of the window for each dimension of the input tensor.
 //	strides: The stride of the sliding window for each dimension of the
 // input tensor.
 //	padding: The type of padding algorithm to use.
 //
-// Returns Gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradV2Attr) (output tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradGradV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -4829,7 +4769,7 @@ func MaxPoolGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, gr
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradV2",
+		Type: "MaxPoolGradGradV2",
 		Input: []tf.Input{
 			orig_input, orig_output, grad, ksize, strides,
 		},
@@ -4839,60 +4779,64 @@ func MaxPoolGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, gr
 	return op.Output(0)
 }
 
-// Restore a reader to a previously saved state.
-//
-// Not all Readers support being restored, so this can produce an
-// Unimplemented error.
+// Computes gradients of the maxpooling function.
 //
 // Arguments:
-//	reader_handle: Handle to a Reader.
-//	state: Result of a ReaderSerializeState of a Reader with type
-// matching reader_handle.
+//	input: The original input.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t. the
+// output of `max_pool`.
+//	argmax: The indices of the maximum values chosen for each output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns the created operation.
-func ReaderRestoreStateV2(scope *Scope, reader_handle tf.Output, state tf.Output) (o *tf.Operation) {
+// Returns Gradients w.r.t. the input of `max_pool`.
+func MaxPoolGradWithArgmax(scope *Scope, input tf.Output, grad tf.Output, argmax tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "ReaderRestoreStateV2",
+		Type: "MaxPoolGradWithArgmax",
 		Input: []tf.Input{
-			reader_handle, state,
+			input, grad, argmax,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MaxPoolGradAttr is an optional argument to MaxPoolGrad.
-type MaxPoolGradAttr func(optionalAttr)
+// AvgPool3DAttr is an optional argument to AvgPool3D.
+type AvgPool3DAttr func(optionalAttr)
 
-// MaxPoolGradDataFormat sets the optional data_format attribute to value.
+// AvgPool3DDataFormat sets the optional data_format attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradDataFormat(value string) MaxPoolGradAttr {
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func AvgPool3DDataFormat(value string) AvgPool3DAttr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// Computes gradients of the maxpooling function.
+// Performs 3D average pooling on the input.
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
+//	input: Shape `[batch, depth, rows, cols, channels]` tensor to pool over.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
 //	padding: The type of padding algorithm to use.
 //
-// Returns Gradients w.r.t. the input to `max_pool`.
-func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradAttr) (output tf.Output) {
+// Returns The average pooled output tensor.
+func AvgPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -4901,9 +4845,9 @@ func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGrad",
+		Type: "AvgPool3D",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -4911,482 +4855,493 @@ func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad
 	return op.Output(0)
 }
 
-// CropAndResizeAttr is an optional argument to CropAndResize.
-type CropAndResizeAttr func(optionalAttr)
-
-// CropAndResizeMethod sets the optional method attribute to value.
-//
-// value: A string specifying the interpolation method. Only 'bilinear' is
-// supported for now.
-// If not specified, defaults to "bilinear"
-func CropAndResizeMethod(value string) CropAndResizeAttr {
-	return func(m optionalAttr) {
-		m["method"] = value
-	}
-}
-
-// CropAndResizeExtrapolationValue sets the optional extrapolation_value attribute to value.
-//
-// value: Value used for extrapolation, when applicable.
-// If not specified, defaults to 0
-func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
-	return func(m optionalAttr) {
-		m["extrapolation_value"] = value
-	}
-}
-
-// Extracts crops from the input image tensor and bilinearly resizes them (possibly
-//
-// with aspect ratio change) to a common output size specified by `crop_size`. This
-// is more general than the `crop_to_bounding_box` op which extracts a fixed size
-// slice from the input image and does not allow resizing or aspect ratio change.
-//
-// Returns a tensor with `crops` from the input `image` at positions defined at the
-// bounding box locations in `boxes`. The cropped boxes are all resized (with
-// bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
-// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
-// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
-// method will give identical results to using `tf.image.resize_bilinear()`
-// with `align_corners=True`.
+// Returns element-wise remainder of division. This emulates C semantics in that
 //
-// Arguments:
-//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
-// Both `image_height` and `image_width` need to be positive.
-//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
-// specifies the coordinates of a box in the `box_ind[i]` image and is specified
-// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
-// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
-// `[0, 1]` interval of normalized image height is mapped to
-// `[0, image_height - 1]` in image height coordinates. We do allow `y1` > `y2`, in
-// which case the sampled crop is an up-down flipped version of the original
-// image. The width dimension is treated similarly. Normalized coordinates
-// outside the `[0, 1]` range are allowed, in which case we use
-// `extrapolation_value` to extrapolate the input image values.
-//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
-// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
-//	crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`. All
-// cropped image patches are resized to this size. The aspect ratio of the image
-// content is not preserved. Both `crop_height` and `crop_width` need to be
-// positive.
+// the result here is consistent with a truncating divide. E.g.
+// `tf.truncatediv(x, y) * y + truncate_mod(x, y) = x`.
 //
-// Returns A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
-func CropAndResize(scope *Scope, image tf.Output, boxes tf.Output, box_ind tf.Output, crop_size tf.Output, optional ...CropAndResizeAttr) (crops tf.Output) {
+// *NOTE*: `Mod` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Mod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "CropAndResize",
+		Type: "Mod",
 		Input: []tf.Input{
-			image, boxes, box_ind, crop_size,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Fills empty rows in the input 2-D `SparseTensor` with a default value.
+// DepthToSpaceAttr is an optional argument to DepthToSpace.
+type DepthToSpaceAttr func(optionalAttr)
+
+// DepthToSpaceDataFormat sets the optional data_format attribute to value.
+// If not specified, defaults to "NHWC"
+func DepthToSpaceDataFormat(value string) DepthToSpaceAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// DepthToSpace for tensors of type T.
 //
-// The input `SparseTensor` is represented via the tuple of inputs
-// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
-// same `dense_shape` but with indices `output_indices` and values
-// `output_values`.
+// Rearranges data from depth into blocks of spatial data.
+// This is the reverse transformation of SpaceToDepth. More specifically,
+// this op outputs a copy of the input tensor where values from the `depth`
+// dimension are moved in spatial blocks to the `height` and `width` dimensions.
+// The attr `block_size` indicates the input block size and how the data is moved.
 //
-// This op inserts a single entry for every row that doesn't have any values.
-// The index is created as `[row, 0, ..., 0]` and the inserted value
-// is `default_value`.
+//   * Chunks of data of size `block_size * block_size` from depth are rearranged
+//     into non-overlapping blocks of size `block_size x block_size`
+//   * The width the output tensor is `input_depth * block_size`, whereas the
+//     height is `input_height * block_size`.
+//   * The Y, X coordinates within each block of the output image are determined
+//     by the high order component of the input channel index.
+//   * The depth of the input tensor must be divisible by
+//     `block_size * block_size`.
 //
-// For example, suppose `sp_input` has shape `[5, 6]` and non-empty values:
+// The `data_format` attr specifies the layout of the input and output tensors
+// with the following options:
+//   "NHWC": `[ batch, height, width, channels ]`
+//   "NCHW": `[ batch, channels, height, width ]`
+//   "NCHW_VECT_C":
+//       `qint8 [ batch, channels / 4, height, width, 4 ]`
 //
-//     [0, 1]: a
-//     [0, 3]: b
-//     [2, 0]: c
-//     [3, 1]: d
+// It is useful to consider the operation as transforming a 6-D Tensor.
+// e.g. for data_format = NHWC,
+//      Each element in the input tensor can be specified via 6 coordinates,
+//      ordered by decreasing memory layout significance as:
+//      n,iY,iX,bY,bX,oC  (where n=batch index, iX, iY means X or Y coordinates
+//                         within the input image, bX, bY means coordinates
+//                         within the output block, oC means output channels).
+//      The output would be the input transposed to the following layout:
+//      n,iY,bY,iX,bX,oC
 //
-// Rows 1 and 4 are empty, so the output will be of shape `[5, 6]` with values:
+// This operation is useful for resizing the activations between convolutions
+// (but keeping all data), e.g. instead of pooling. It is also useful for training
+// purely convolutional models.
 //
-//     [0, 1]: a
-//     [0, 3]: b
-//     [1, 0]: default_value
-//     [2, 0]: c
-//     [3, 1]: d
-//     [4, 0]: default_value
+// For example, given an input of shape `[1, 1, 1, 4]`, data_format = "NHWC" and
+// block_size = 2:
 //
-// The output `SparseTensor` will be in row-major order and will have the
-// same shape as the input.
+// ```
+// x = [[[[1, 2, 3, 4]]]]
 //
-// This op also returns an indicator vector shaped `[dense_shape[0]]` such that
+// ```
 //
-//     empty_row_indicator[i] = True iff row i was an empty row.
+// This operation will output a tensor of shape `[1, 2, 2, 1]`:
 //
-// And a reverse index map vector shaped `[indices.shape[0]]` that is used during
-// backpropagation,
+// ```
+//    [[[[1], [2]],
+//      [[3], [4]]]]
+// ```
 //
-//     reverse_index_map[j] = out_j s.t. indices[j, :] == output_indices[out_j, :]
+// Here, the input has a batch of 1 and each batch element has shape `[1, 1, 4]`,
+// the corresponding output will have 2x2 elements and will have a depth of
+// 1 channel (1 = `4 / (block_size * block_size)`).
+// The output element shape is `[2, 2, 1]`.
+//
+// For an input tensor with larger depth, here of shape `[1, 1, 1, 12]`, e.g.
+//
+// ```
+// x = [[[[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]]]]
+// ```
+//
+// This operation, for block size of 2, will return the following tensor of shape
+// `[1, 2, 2, 3]`
+//
+// ```
+//    [[[[1, 2, 3], [4, 5, 6]],
+//      [[7, 8, 9], [10, 11, 12]]]]
+//
+// ```
+//
+// Similarly, for the following input of shape `[1 2 2 4]`, and a block size of 2:
+//
+// ```
+// x =  [[[[1, 2, 3, 4],
+//        [5, 6, 7, 8]],
+//       [[9, 10, 11, 12],
+//        [13, 14, 15, 16]]]]
+// ```
+//
+// the operator will return the following tensor of shape `[1 4 4 1]`:
+//
+// ```
+// x = [[[ [1],   [2],  [5],  [6]],
+//       [ [3],   [4],  [7],  [8]],
+//       [ [9],  [10], [13],  [14]],
+//       [ [11], [12], [15],  [16]]]]
+//
+// ```
 //
 // Arguments:
-//	indices: 2-D. the indices of the sparse tensor.
-//	values: 1-D. the values of the sparse tensor.
-//	dense_shape: 1-D. the shape of the sparse tensor.
-//	default_value: 0-D. default value to insert into location `[row, 0, ..., 0]`
-//   for rows missing from the input sparse tensor.
-// output indices: 2-D. the indices of the filled sparse tensor.
 //
-// Returns 1-D. the values of the filled sparse tensor.1-D. whether the dense row was missing in the
-// input sparse tensor.1-D. a map from the input indices to the output indices.
-func SparseFillEmptyRows(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output, default_value tf.Output) (output_indices tf.Output, output_values tf.Output, empty_row_indicator tf.Output, reverse_index_map tf.Output) {
+//	block_size: The size of the spatial block, same as in Space2Depth.
+func DepthToSpace(scope *Scope, input tf.Output, block_size int64, optional ...DepthToSpaceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"block_size": block_size}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseFillEmptyRows",
+		Type: "DepthToSpace",
 		Input: []tf.Input{
-			indices, values, dense_shape, default_value,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
+	return op.Output(0)
 }
 
-// Reverses specific dimensions of a tensor.
-//
-// Given a `tensor`, and a `bool` tensor `dims` representing the dimensions
-// of `tensor`, this operation reverses each dimension i of `tensor` where
-// `dims[i]` is `True`.
-//
-// `tensor` can have up to 8 dimensions. The number of dimensions
-// of `tensor` must equal the number of elements in `dims`. In other words:
-//
-// `rank(tensor) = size(dims)`
-//
-// For example:
-//
-// ```
-// # tensor 't' is [[[[ 0,  1,  2,  3],
-// #                  [ 4,  5,  6,  7],
-// #                  [ 8,  9, 10, 11]],
-// #                 [[12, 13, 14, 15],
-// #                  [16, 17, 18, 19],
-// #                  [20, 21, 22, 23]]]]
-// # tensor 't' shape is [1, 2, 3, 4]
-//
-// # 'dims' is [False, False, False, True]
-// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
-//                         [ 7,  6,  5,  4],
-//                         [ 11, 10, 9, 8]],
-//                        [[15, 14, 13, 12],
-//                         [19, 18, 17, 16],
-//                         [23, 22, 21, 20]]]]
+// Conv3DBackpropInputV2Attr is an optional argument to Conv3DBackpropInputV2.
+type Conv3DBackpropInputV2Attr func(optionalAttr)
+
+// Conv3DBackpropInputV2DataFormat sets the optional data_format attribute to value.
 //
-// # 'dims' is [False, True, False, False]
-// reverse(t, dims) ==> [[[[12, 13, 14, 15],
-//                         [16, 17, 18, 19],
-//                         [20, 21, 22, 23]
-//                        [[ 0,  1,  2,  3],
-//                         [ 4,  5,  6,  7],
-//                         [ 8,  9, 10, 11]]]]
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func Conv3DBackpropInputV2DataFormat(value string) Conv3DBackpropInputV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Conv3DBackpropInputV2Dilations sets the optional dilations attribute to value.
 //
-// # 'dims' is [False, False, True, False]
-// reverse(t, dims) ==> [[[[8, 9, 10, 11],
-//                         [4, 5, 6, 7],
-//                         [0, 1, 2, 3]]
-//                        [[20, 21, 22, 23],
-//                         [16, 17, 18, 19],
-//                         [12, 13, 14, 15]]]]
-// ```
+// value: 1-D tensor of length 5.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
+func Conv3DBackpropInputV2Dilations(value []int64) Conv3DBackpropInputV2Attr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of 3-D convolution with respect to the input.
 //
 // Arguments:
-//	tensor: Up to 8-D.
-//	dims: 1-D. The dimensions to reverse.
-//
-// Returns The same shape as `tensor`.
-func Reverse(scope *Scope, tensor tf.Output, dims tf.Output) (output tf.Output) {
+//	input_sizes: An integer vector representing the tensor shape of `input`,
+// where `input` is a 5-D
+// `[batch, depth, rows, cols, in_channels]` tensor.
+//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
+// `in_channels` must match between `input` and `filter`.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3DBackpropInputV2(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropInputV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Reverse",
+		Type: "Conv3DBackpropInputV2",
 		Input: []tf.Input{
-			tensor, dims,
+			input_sizes, filter, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes log softmax activations.
-//
-// For each batch `i` and class `j` we have
-//
-//     logsoftmax[i, j] = logits[i, j] - log(sum(exp(logits[i])))
-//
-// Arguments:
-//	logits: 2-D with shape `[batch_size, num_classes]`.
+// Computes square root of x element-wise.
 //
-// Returns Same shape as `logits`.
-func LogSoftmax(scope *Scope, logits tf.Output) (logsoftmax tf.Output) {
+// I.e., \\(y = \sqrt{x} = x^{1/2}\\).
+func Sqrt(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LogSoftmax",
+		Type: "Sqrt",
 		Input: []tf.Input{
-			logits,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the inverse permutation of a tensor.
-//
-// This operation computes the inverse of an index permutation. It takes a 1-D
-// integer tensor `x`, which represents the indices of a zero-based array, and
-// swaps each value with its index position. In other words, for an output tensor
-// `y` and an input tensor `x`, this operation computes the following:
-//
-// `y[x[i]] = i for i in [0, 1, ..., len(x) - 1]`
-//
-// The values must include 0. There can be no duplicate values or negative values.
-//
-// For example:
+// Computes the gradients of 3-D convolution with respect to the filter.
 //
-// ```
-// # tensor `x` is [3, 4, 0, 2, 1]
-// invert_permutation(x) ==> [2, 4, 3, 0, 1]
-// ```
+// DEPRECATED at GraphDef version 10: Use Conv3DBackpropFilterV2
 //
 // Arguments:
-//	x: 1-D.
-//
-// Returns 1-D.
-func InvertPermutation(scope *Scope, x tf.Output) (y tf.Output) {
+//	input: Shape `[batch, depth, rows, cols, in_channels]`.
+//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
+// `in_channels` must match between `input` and `filter`.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "InvertPermutation",
+		Type: "Conv3DBackpropFilter",
 		Input: []tf.Input{
-			x,
+			input, filter, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// BiasAddGradAttr is an optional argument to BiasAddGrad.
-type BiasAddGradAttr func(optionalAttr)
-
-// BiasAddGradDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the bias tensor will be added to the last dimension
-// of the value tensor.
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// The tensor will be added to "in_channels", the third-to-the-last
-//     dimension.
-// If not specified, defaults to "NHWC"
-func BiasAddGradDataFormat(value string) BiasAddGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// The backward operation for "BiasAdd" on the "bias" tensor.
-//
-// It accumulates all the values from out_backprop into the feature dimension.
-// For NHWC data format, the feature dimension is the last. For NCHW data format,
-// the feature dimension is the third-to-last.
-//
-// Arguments:
-//	out_backprop: Any number of dimensions.
+// Computes the gradient for the rsqrt of `x` wrt its input.
 //
-// Returns 1-D with size the feature dimension of `out_backprop`.
-func BiasAddGrad(scope *Scope, out_backprop tf.Output, optional ...BiasAddGradAttr) (output tf.Output) {
+// Specifically, `grad = dy * -0.5 * y^3`, where `y = rsqrt(x)`, and `dy`
+// is the corresponding input gradient.
+func RsqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "BiasAddGrad",
+		Type: "RsqrtGrad",
 		Input: []tf.Input{
-			out_backprop,
+			y, dy,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FusedBatchNormV2Attr is an optional argument to FusedBatchNormV2.
-type FusedBatchNormV2Attr func(optionalAttr)
+// ReverseSequenceAttr is an optional argument to ReverseSequence.
+type ReverseSequenceAttr func(optionalAttr)
 
-// FusedBatchNormV2Epsilon sets the optional epsilon attribute to value.
+// ReverseSequenceBatchDim sets the optional batch_dim attribute to value.
 //
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormV2Epsilon(value float32) FusedBatchNormV2Attr {
+// value: The dimension along which reversal is performed.
+// If not specified, defaults to 0
+func ReverseSequenceBatchDim(value int64) ReverseSequenceAttr {
 	return func(m optionalAttr) {
-		m["epsilon"] = value
+		m["batch_dim"] = value
 	}
 }
 
-// FusedBatchNormV2DataFormat sets the optional data_format attribute to value.
+// Reverses variable length slices.
 //
-// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormV2DataFormat(value string) FusedBatchNormV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// FusedBatchNormV2IsTraining sets the optional is_training attribute to value.
+// This op first slices `input` along the dimension `batch_dim`, and for each
+// slice `i`, reverses the first `seq_lengths[i]` elements along
+// the dimension `seq_dim`.
 //
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormV2IsTraining(value bool) FusedBatchNormV2Attr {
-	return func(m optionalAttr) {
-		m["is_training"] = value
-	}
-}
-
-// Batch normalization.
+// The elements of `seq_lengths` must obey `seq_lengths[i] <= input.dims[seq_dim]`,
+// and `seq_lengths` must be a vector of length `input.dims[batch_dim]`.
 //
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// The output slice `i` along dimension `batch_dim` is then given by input
+// slice `i`, with the first `seq_lengths[i]` slices along dimension
+// `seq_dim` reversed.
 //
-// Arguments:
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	offset: A 1D Tensor for offset, to shift to the normalized x.
-//	mean: A 1D Tensor for population mean. Used for inference only;
-// must be empty for training.
-//	variance: A 1D Tensor for population variance. Used for inference only;
-// must be empty for training.
+// For example:
 //
-// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
-// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
-// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
-// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
-// in the cuDNN case), to be reused in the gradient computation.
-func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormV2Attr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
+// ```
+// # Given this:
+// batch_dim = 0
+// seq_dim = 1
+// input.dims = (4, 8, ...)
+// seq_lengths = [7, 2, 3, 5]
+//
+// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
+// output[0, 0:7, :, ...] = input[0, 7:0:-1, :, ...]
+// output[1, 0:2, :, ...] = input[1, 2:0:-1, :, ...]
+// output[2, 0:3, :, ...] = input[2, 3:0:-1, :, ...]
+// output[3, 0:5, :, ...] = input[3, 5:0:-1, :, ...]
+//
+// # while entries past seq_lens are copied through:
+// output[0, 7:, :, ...] = input[0, 7:, :, ...]
+// output[1, 2:, :, ...] = input[1, 2:, :, ...]
+// output[2, 3:, :, ...] = input[2, 3:, :, ...]
+// output[3, 2:, :, ...] = input[3, 2:, :, ...]
+// ```
+//
+// In contrast, if:
+//
+// ```
+// # Given this:
+// batch_dim = 2
+// seq_dim = 0
+// input.dims = (8, ?, 4, ...)
+// seq_lengths = [7, 2, 3, 5]
+//
+// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
+// output[0:7, :, 0, :, ...] = input[7:0:-1, :, 0, :, ...]
+// output[0:2, :, 1, :, ...] = input[2:0:-1, :, 1, :, ...]
+// output[0:3, :, 2, :, ...] = input[3:0:-1, :, 2, :, ...]
+// output[0:5, :, 3, :, ...] = input[5:0:-1, :, 3, :, ...]
+//
+// # while entries past seq_lens are copied through:
+// output[7:, :, 0, :, ...] = input[7:, :, 0, :, ...]
+// output[2:, :, 1, :, ...] = input[2:, :, 1, :, ...]
+// output[3:, :, 2, :, ...] = input[3:, :, 2, :, ...]
+// output[2:, :, 3, :, ...] = input[2:, :, 3, :, ...]
+// ```
+//
+// Arguments:
+//	input: The input to reverse.
+//	seq_lengths: 1-D with length `input.dims(batch_dim)` and
+// `max(seq_lengths) <= input.dims(seq_dim)`
+//	seq_dim: The dimension which is partially reversed.
+//
+// Returns The partially reversed input. It has the same shape as `input`.
+func ReverseSequence(scope *Scope, input tf.Output, seq_lengths tf.Output, seq_dim int64, optional ...ReverseSequenceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"seq_dim": seq_dim}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNormV2",
+		Type: "ReverseSequence",
 		Input: []tf.Input{
-			x, scale, offset, mean, variance,
+			input, seq_lengths,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0)
 }
 
-// Returns the rank of a tensor.
+// DepthwiseConv2dNativeAttr is an optional argument to DepthwiseConv2dNative.
+type DepthwiseConv2dNativeAttr func(optionalAttr)
+
+// DepthwiseConv2dNativeDataFormat sets the optional data_format attribute to value.
 //
-// This operation returns an integer representing the rank of `input`.
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func DepthwiseConv2dNativeDataFormat(value string) DepthwiseConv2dNativeAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// DepthwiseConv2dNativeDilations sets the optional dilations attribute to value.
 //
-// For example:
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func DepthwiseConv2dNativeDilations(value []int64) DepthwiseConv2dNativeAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes a 2-D depthwise convolution given 4-D `input` and `filter` tensors.
+//
+// Given an input tensor of shape `[batch, in_height, in_width, in_channels]`
+// and a filter / kernel tensor of shape
+// `[filter_height, filter_width, in_channels, channel_multiplier]`, containing
+// `in_channels` convolutional filters of depth 1, `depthwise_conv2d` applies
+// a different filter to each input channel (expanding from 1 channel to
+// `channel_multiplier` channels for each), then concatenates the results
+// together. Thus, the output has `in_channels * channel_multiplier` channels.
 //
 // ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// # shape of tensor 't' is [2, 2, 3]
-// rank(t) ==> 3
+// for k in 0..in_channels-1
+//   for q in 0..channel_multiplier-1
+//     output[b, i, j, k * channel_multiplier + q] =
+//       sum_{di, dj} input[b, strides[1] * i + di, strides[2] * j + dj, k] *
+//                         filter[di, dj, k, q]
 // ```
 //
-// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
-// of a tensor is the number of indices required to uniquely select each element
-// of the tensor. Rank is also known as "order", "degree", or "ndims."
-func Rank(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Rank",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Transforms a Tensor into a serialized TensorProto proto.
+// Must have `strides[0] = strides[3] = 1`.  For the most common case of the same
+// horizontal and vertices strides, `strides = [1, stride, stride, 1]`.
 //
 // Arguments:
-//	tensor: A Tensor of type `T`.
 //
-// Returns A serialized TensorProto proto of the input tensor.
-func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
+//
+//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
+// of `input`.
+//	padding: The type of padding algorithm to use.
+func DepthwiseConv2dNative(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SerializeTensor",
+		Type: "DepthwiseConv2dNative",
 		Input: []tf.Input{
-			tensor,
+			input, filter,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixSolveAttr is an optional argument to MatrixSolve.
-type MatrixSolveAttr func(optionalAttr)
+// MaxPoolGradV2Attr is an optional argument to MaxPoolGradV2.
+type MaxPoolGradV2Attr func(optionalAttr)
 
-// MatrixSolveAdjoint sets the optional adjoint attribute to value.
+// MaxPoolGradV2DataFormat sets the optional data_format attribute to value.
 //
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-// adjoint.
-// If not specified, defaults to false
-func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradV2DataFormat(value string) MaxPoolGradV2Attr {
 	return func(m optionalAttr) {
-		m["adjoint"] = value
+		m["data_format"] = value
 	}
 }
 
-// Solves systems of linear equations.
-//
-// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
-// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
-// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `True` then each output matrix satisfies
-// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
+// Computes gradients of the maxpooling function.
 //
 // Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
+// Returns Gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradV2(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolGradV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixSolve",
+		Type: "MaxPoolGradV2",
 		Input: []tf.Input{
-			matrix, rhs,
+			orig_input, orig_output, grad, ksize, strides,
 		},
 		Attrs: attrs,
 	}
@@ -5394,122 +5349,71 @@ func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...Matr
 	return op.Output(0)
 }
 
-// Computes acos of x element-wise.
-func Acos(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Acos",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Real-valued fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most dimension of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
-// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
-// followed by the `fft_length / 2` positive-frequency terms.
+// Restore a reader to a previously saved state.
 //
-// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
+// Not all Readers support being restored, so this can produce an
+// Unimplemented error.
 //
 // Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
-//   frequency components of its 1D Fourier transform.
+//	reader_handle: Handle to a Reader.
+//	state: Result of a ReaderSerializeState of a Reader with type
+// matching reader_handle.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft
-// @end_compatibility
-func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Returns the created operation.
+func ReaderRestoreStateV2(scope *Scope, reader_handle tf.Output, state tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RFFT",
+		Type: "ReaderRestoreStateV2",
 		Input: []tf.Input{
-			input, fft_length,
+			reader_handle, state,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// DepthwiseConv2dNativeBackpropFilterAttr is an optional argument to DepthwiseConv2dNativeBackpropFilter.
-type DepthwiseConv2dNativeBackpropFilterAttr func(optionalAttr)
+// MaxPoolGradAttr is an optional argument to MaxPoolGrad.
+type MaxPoolGradAttr func(optionalAttr)
 
-// DepthwiseConv2dNativeBackpropFilterDataFormat sets the optional data_format attribute to value.
+// MaxPoolGradDataFormat sets the optional data_format attribute to value.
 //
 // value: Specify the data format of the input and output data. With the
 // default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
+//     [batch, in_height, in_width, in_channels].
 // Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
+//     [batch, in_channels, in_height, in_width].
 // If not specified, defaults to "NHWC"
-func DepthwiseConv2dNativeBackpropFilterDataFormat(value string) DepthwiseConv2dNativeBackpropFilterAttr {
+func MaxPoolGradDataFormat(value string) MaxPoolGradAttr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// DepthwiseConv2dNativeBackpropFilterDilations sets the optional dilations attribute to value.
-//
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func DepthwiseConv2dNativeBackpropFilterDilations(value []int64) DepthwiseConv2dNativeBackpropFilterAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of depthwise convolution with respect to the filter.
+// Computes gradients of the maxpooling function.
 //
 // Arguments:
-//	input: 4-D with shape based on `data_format`.  For example, if
-// `data_format` is 'NHWC' then `input` is a 4-D `[batch, in_height,
-// in_width, in_channels]` tensor.
-//	filter_sizes: An integer vector representing the tensor shape of `filter`,
-// where `filter` is a 4-D
-// `[filter_height, filter_width, in_channels, depthwise_multiplier]` tensor.
-//	out_backprop: 4-D with shape  based on `data_format`.
-// For example, if `data_format` is 'NHWC' then
-// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients w.r.t. the output of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
 //	padding: The type of padding algorithm to use.
 //
-// Returns 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
-// the `filter` input of the convolution.
-func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropFilterAttr) (output tf.Output) {
+// Returns Gradients w.r.t. the input to `max_pool`.
+func MaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DepthwiseConv2dNativeBackpropFilter",
+		Type: "MaxPoolGrad",
 		Input: []tf.Input{
-			input, filter_sizes, out_backprop,
+			orig_input, orig_output, grad,
 		},
 		Attrs: attrs,
 	}
@@ -5517,58 +5421,66 @@ func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_s
 	return op.Output(0)
 }
 
-// LRNGradAttr is an optional argument to LRNGrad.
-type LRNGradAttr func(optionalAttr)
+// CropAndResizeAttr is an optional argument to CropAndResize.
+type CropAndResizeAttr func(optionalAttr)
 
-// LRNGradDepthRadius sets the optional depth_radius attribute to value.
+// CropAndResizeMethod sets the optional method attribute to value.
 //
-// value: A depth radius.
-// If not specified, defaults to 5
-func LRNGradDepthRadius(value int64) LRNGradAttr {
+// value: A string specifying the interpolation method. Only 'bilinear' is
+// supported for now.
+// If not specified, defaults to "bilinear"
+func CropAndResizeMethod(value string) CropAndResizeAttr {
 	return func(m optionalAttr) {
-		m["depth_radius"] = value
+		m["method"] = value
 	}
 }
 
-// LRNGradBias sets the optional bias attribute to value.
+// CropAndResizeExtrapolationValue sets the optional extrapolation_value attribute to value.
 //
-// value: An offset (usually > 0 to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNGradBias(value float32) LRNGradAttr {
+// value: Value used for extrapolation, when applicable.
+// If not specified, defaults to 0
+func CropAndResizeExtrapolationValue(value float32) CropAndResizeAttr {
 	return func(m optionalAttr) {
-		m["bias"] = value
+		m["extrapolation_value"] = value
 	}
 }
 
-// LRNGradAlpha sets the optional alpha attribute to value.
+// Extracts crops from the input image tensor and bilinearly resizes them (possibly
 //
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNGradAlpha(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["alpha"] = value
-	}
-}
-
-// LRNGradBeta sets the optional beta attribute to value.
+// with aspect ratio change) to a common output size specified by `crop_size`. This
+// is more general than the `crop_to_bounding_box` op which extracts a fixed size
+// slice from the input image and does not allow resizing or aspect ratio change.
 //
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNGradBeta(value float32) LRNGradAttr {
-	return func(m optionalAttr) {
-		m["beta"] = value
-	}
-}
-
-// Gradients for Local Response Normalization.
+// Returns a tensor with `crops` from the input `image` at positions defined at the
+// bounding box locations in `boxes`. The cropped boxes are all resized (with
+// bilinear interpolation) to a fixed `size = [crop_height, crop_width]`. The
+// result is a 4-D tensor `[num_boxes, crop_height, crop_width, depth]`. The
+// resizing is corner aligned. In particular, if `boxes = [[0, 0, 1, 1]]`, the
+// method will give identical results to using `tf.image.resize_bilinear()`
+// with `align_corners=True`.
 //
 // Arguments:
-//	input_grads: 4-D with shape `[batch, height, width, channels]`.
-//	input_image: 4-D with shape `[batch, height, width, channels]`.
-//	output_image: 4-D with shape `[batch, height, width, channels]`.
+//	image: A 4-D tensor of shape `[batch, image_height, image_width, depth]`.
+// Both `image_height` and `image_width` need to be positive.
+//	boxes: A 2-D tensor of shape `[num_boxes, 4]`. The `i`-th row of the tensor
+// specifies the coordinates of a box in the `box_ind[i]` image and is specified
+// in normalized coordinates `[y1, x1, y2, x2]`. A normalized coordinate value of
+// `y` is mapped to the image coordinate at `y * (image_height - 1)`, so as the
+// `[0, 1]` interval of normalized image height is mapped to
+// `[0, image_height - 1]` in image height coordinates. We do allow `y1` > `y2`, in
+// which case the sampled crop is an up-down flipped version of the original
+// image. The width dimension is treated similarly. Normalized coordinates
+// outside the `[0, 1]` range are allowed, in which case we use
+// `extrapolation_value` to extrapolate the input image values.
+//	box_ind: A 1-D tensor of shape `[num_boxes]` with int32 values in `[0, batch)`.
+// The value of `box_ind[i]` specifies the image that the `i`-th box refers to.
+//	crop_size: A 1-D tensor of 2 elements, `size = [crop_height, crop_width]`. All
+// cropped image patches are resized to this size. The aspect ratio of the image
+// content is not preserved. Both `crop_height` and `crop_width` need to be
+// positive.
 //
-// Returns The gradients for LRN.
-func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
+// Returns A 4-D tensor of shape `[num_boxes, crop_height, crop_width, depth]`.
+func CropAndResize(scope *Scope, image tf.Output, boxes tf.Output, box_ind tf.Output, crop_size tf.Output, optional ...CropAndResizeAttr) (crops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5577,9 +5489,9 @@ func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "LRNGrad",
+		Type: "CropAndResize",
 		Input: []tf.Input{
-			input_grads, input_image, output_image,
+			image, boxes, box_ind, crop_size,
 		},
 		Attrs: attrs,
 	}
@@ -5587,309 +5499,225 @@ func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_
 	return op.Output(0)
 }
 
-// AnyAttr is an optional argument to Any.
-type AnyAttr func(optionalAttr)
-
-// AnyKeepDims sets the optional keep_dims attribute to value.
+// Fills empty rows in the input 2-D `SparseTensor` with a default value.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func AnyKeepDims(value bool) AnyAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
-}
-
-// Computes the "logical or" of elements across dimensions of a tensor.
+// The input `SparseTensor` is represented via the tuple of inputs
+// (`indices`, `values`, `dense_shape`).  The output `SparseTensor` has the
+// same `dense_shape` but with indices `output_indices` and values
+// `output_values`.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// This op inserts a single entry for every row that doesn't have any values.
+// The index is created as `[row, 0, ..., 0]` and the inserted value
+// is `default_value`.
+//
+// For example, suppose `sp_input` has shape `[5, 6]` and non-empty values:
+//
+//     [0, 1]: a
+//     [0, 3]: b
+//     [2, 0]: c
+//     [3, 1]: d
+//
+// Rows 1 and 4 are empty, so the output will be of shape `[5, 6]` with values:
+//
+//     [0, 1]: a
+//     [0, 3]: b
+//     [1, 0]: default_value
+//     [2, 0]: c
+//     [3, 1]: d
+//     [4, 0]: default_value
+//
+// The output `SparseTensor` will be in row-major order and will have the
+// same shape as the input.
+//
+// This op also returns an indicator vector shaped `[dense_shape[0]]` such that
+//
+//     empty_row_indicator[i] = True iff row i was an empty row.
+//
+// And a reverse index map vector shaped `[indices.shape[0]]` that is used during
+// backpropagation,
+//
+//     reverse_index_map[j] = out_j s.t. indices[j, :] == output_indices[out_j, :]
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	indices: 2-D. the indices of the sparse tensor.
+//	values: 1-D. the values of the sparse tensor.
+//	dense_shape: 1-D. the shape of the sparse tensor.
+//	default_value: 0-D. default value to insert into location `[row, 0, ..., 0]`
+//   for rows missing from the input sparse tensor.
+// output indices: 2-D. the indices of the filled sparse tensor.
 //
-// Returns The reduced tensor.
-func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
+// Returns 1-D. the values of the filled sparse tensor.1-D. whether the dense row was missing in the
+// input sparse tensor.1-D. a map from the input indices to the output indices.
+func SparseFillEmptyRows(scope *Scope, indices tf.Output, values tf.Output, dense_shape tf.Output, default_value tf.Output) (output_indices tf.Output, output_values tf.Output, empty_row_indicator tf.Output, reverse_index_map tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Any",
+		Type: "SparseFillEmptyRows",
 		Input: []tf.Input{
-			input, axis,
+			indices, values, dense_shape, default_value,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3)
 }
 
-// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
-type ResourceApplyFtrlAttr func(optionalAttr)
-
-// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
+// Reverses specific dimensions of a tensor.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the Ftrl-proximal scheme.
+// Given a `tensor`, and a `bool` tensor `dims` representing the dimensions
+// of `tensor`, this operation reverses each dimension i of `tensor` where
+// `dims[i]` is `True`.
 //
-// accum_new = accum + grad * grad
-// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// `tensor` can have up to 8 dimensions. The number of dimensions
+// of `tensor` must equal the number of elements in `dims`. In other words:
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 regulariation. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
+// `rank(tensor) = size(dims)`
 //
-// Returns the created operation.
-func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrl",
-		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, lr_power,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// RandomUniformAttr is an optional argument to RandomUniform.
-type RandomUniformAttr func(optionalAttr)
-
-// RandomUniformSeed sets the optional seed attribute to value.
+// For example:
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformSeed(value int64) RandomUniformAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomUniformSeed2 sets the optional seed2 attribute to value.
+// ```
+// # tensor 't' is [[[[ 0,  1,  2,  3],
+// #                  [ 4,  5,  6,  7],
+// #                  [ 8,  9, 10, 11]],
+// #                 [[12, 13, 14, 15],
+// #                  [16, 17, 18, 19],
+// #                  [20, 21, 22, 23]]]]
+// # tensor 't' shape is [1, 2, 3, 4]
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformSeed2(value int64) RandomUniformAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from a uniform distribution.
+// # 'dims' is [False, False, False, True]
+// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
+//                         [ 7,  6,  5,  4],
+//                         [ 11, 10, 9, 8]],
+//                        [[15, 14, 13, 12],
+//                         [19, 18, 17, 16],
+//                         [23, 22, 21, 20]]]]
 //
-// The generated values follow a uniform distribution in the range `[0, 1)`. The
-// lower bound 0 is included in the range, while the upper bound 1 is excluded.
+// # 'dims' is [False, True, False, False]
+// reverse(t, dims) ==> [[[[12, 13, 14, 15],
+//                         [16, 17, 18, 19],
+//                         [20, 21, 22, 23]
+//                        [[ 0,  1,  2,  3],
+//                         [ 4,  5,  6,  7],
+//                         [ 8,  9, 10, 11]]]]
+//
+// # 'dims' is [False, False, True, False]
+// reverse(t, dims) ==> [[[[8, 9, 10, 11],
+//                         [4, 5, 6, 7],
+//                         [0, 1, 2, 3]]
+//                        [[20, 21, 22, 23],
+//                         [16, 17, 18, 19],
+//                         [12, 13, 14, 15]]]]
+// ```
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	tensor: Up to 8-D.
+//	dims: 1-D. The dimensions to reverse.
 //
-// Returns A tensor of the specified shape filled with uniform random values.
-func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
+// Returns The same shape as `tensor`.
+func Reverse(scope *Scope, tensor tf.Output, dims tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniform",
+		Type: "Reverse",
 		Input: []tf.Input{
-			shape,
+			tensor, dims,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AssertAttr is an optional argument to Assert.
-type AssertAttr func(optionalAttr)
-
-// AssertSummarize sets the optional summarize attribute to value.
+// Computes log softmax activations.
 //
-// value: Print this many entries of each tensor.
-// If not specified, defaults to 3
-func AssertSummarize(value int64) AssertAttr {
-	return func(m optionalAttr) {
-		m["summarize"] = value
-	}
-}
-
-// Asserts that the given condition is true.
+// For each batch `i` and class `j` we have
 //
-// If `condition` evaluates to false, print the list of tensors in `data`.
-// `summarize` determines how many entries of the tensors to print.
+//     logsoftmax[i, j] = logits[i, j] - log(sum(exp(logits[i])))
 //
 // Arguments:
-//	condition: The condition to evaluate.
-//	data: The tensors to print out when condition is false.
-//
-// Returns the created operation.
-func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "Assert",
-		Input: []tf.Input{
-			condition, tf.OutputList(data),
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
-//
-// For each entry in `x`, calculates the number of `1` (on) bits in the binary
-// representation of that entry.
+//	logits: 2-D with shape `[batch_size, num_classes]`.
 //
-// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
-// `int32` or `int64` and perform the bitcount on the result, than to feed in
-// 8- or 16-bit inputs and then aggregate the resulting counts.
-func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns Same shape as `logits`.
+func LogSoftmax(scope *Scope, logits tf.Output) (logsoftmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "PopulationCount",
+		Type: "LogSoftmax",
 		Input: []tf.Input{
-			x,
+			logits,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Split a `SparseTensor` into `num_split` tensors along one dimension.
+// Computes the inverse permutation of a tensor.
 //
-// If the `shape[split_dim]` is not an integer multiple of `num_split`. Slices
-// `[0 : shape[split_dim] % num_split]` gets one extra dimension.
-// For example, if `split_dim = 1` and `num_split = 2` and the input is
+// This operation computes the inverse of an index permutation. It takes a 1-D
+// integer tensor `x`, which represents the indices of a zero-based array, and
+// swaps each value with its index position. In other words, for an output tensor
+// `y` and an input tensor `x`, this operation computes the following:
 //
-//     input_tensor = shape = [2, 7]
-//     [    a   d e  ]
-//     [b c          ]
+// `y[x[i]] = i for i in [0, 1, ..., len(x) - 1]`
 //
-// Graphically the output tensors are:
+// The values must include 0. There can be no duplicate values or negative values.
 //
-//     output_tensor[0] = shape = [2, 4]
-//     [    a  ]
-//     [b c    ]
+// For example:
 //
-//     output_tensor[1] = shape = [2, 3]
-//     [ d e  ]
-//     [      ]
+// ```
+// # tensor `x` is [3, 4, 0, 2, 1]
+// invert_permutation(x) ==> [2, 4, 3, 0, 1]
+// ```
 //
 // Arguments:
-//	split_dim: 0-D.  The dimension along which to split.  Must be in the range
-// `[0, rank(shape))`.
-//	indices: 2-D tensor represents the indices of the sparse tensor.
-//	values: 1-D tensor represents the values of the sparse tensor.
-//	shape: 1-D. tensor represents the shape of the sparse tensor.
-// output indices: A list of 1-D tensors represents the indices of the output
-// sparse tensors.
-//	num_split: The number of ways to split.
+//	x: 1-D.
 //
-// Returns A list of 1-D tensors represents the values of the output sparse
-// tensors.A list of 1-D tensors represents the shape of the output sparse
-// tensors.
-func SparseSplit(scope *Scope, split_dim tf.Output, indices tf.Output, values tf.Output, shape tf.Output, num_split int64) (output_indices []tf.Output, output_values []tf.Output, output_shape []tf.Output) {
+// Returns 1-D.
+func InvertPermutation(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "SparseSplit",
+		Type: "InvertPermutation",
 		Input: []tf.Input{
-			split_dim, indices, values, shape,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output_indices, idx, err = makeOutputList(op, idx, "output_indices"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	if output_values, idx, err = makeOutputList(op, idx, "output_values"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	if output_shape, idx, err = makeOutputList(op, idx, "output_shape"); err != nil {
-		scope.UpdateErr("SparseSplit", err)
-		return
-	}
-	return output_indices, output_values, output_shape
+	return op.Output(0)
 }
 
-// RandomPoissonAttr is an optional argument to RandomPoisson.
-type RandomPoissonAttr func(optionalAttr)
-
-// RandomPoissonSeed sets the optional seed attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed(value int64) RandomPoissonAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+// BiasAddGradAttr is an optional argument to BiasAddGrad.
+type BiasAddGradAttr func(optionalAttr)
 
-// RandomPoissonSeed2 sets the optional seed2 attribute to value.
-// If not specified, defaults to 0
-func RandomPoissonSeed2(value int64) RandomPoissonAttr {
+// BiasAddGradDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the bias tensor will be added to the last dimension
+// of the value tensor.
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// The tensor will be added to "in_channels", the third-to-the-last
+//     dimension.
+// If not specified, defaults to "NHWC"
+func BiasAddGradDataFormat(value string) BiasAddGradAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["data_format"] = value
 	}
 }
 
-// Use RandomPoissonV2 instead.
+// The backward operation for "BiasAdd" on the "bias" tensor.
 //
-// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
-func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
+// It accumulates all the values from out_backprop into the feature dimension.
+// For NHWC data format, the feature dimension is the last. For NCHW data format,
+// the feature dimension is the third-to-last.
+//
+// Arguments:
+//	out_backprop: Any number of dimensions.
+//
+// Returns 1-D with size the feature dimension of `out_backprop`.
+func BiasAddGrad(scope *Scope, out_backprop tf.Output, optional ...BiasAddGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5898,9 +5726,9 @@ func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...Ra
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomPoisson",
+		Type: "BiasAddGrad",
 		Input: []tf.Input{
-			shape, rate,
+			out_backprop,
 		},
 		Attrs: attrs,
 	}
@@ -5908,46 +5736,60 @@ func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...Ra
 	return op.Output(0)
 }
 
-// ResourceSparseApplyFtrlV2Attr is an optional argument to ResourceSparseApplyFtrlV2.
-type ResourceSparseApplyFtrlV2Attr func(optionalAttr)
+// FusedBatchNormV2Attr is an optional argument to FusedBatchNormV2.
+type FusedBatchNormV2Attr func(optionalAttr)
 
-// ResourceSparseApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
+// FusedBatchNormV2Epsilon sets the optional epsilon attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyFtrlV2UseLocking(value bool) ResourceSparseApplyFtrlV2Attr {
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormV2Epsilon(value float32) FusedBatchNormV2Attr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["epsilon"] = value
 	}
 }
 
-// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
+// FusedBatchNormV2DataFormat sets the optional data_format attribute to value.
 //
-// That is for rows we have grad for, we update var, accum and linear as follows:
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormV2DataFormat(value string) FusedBatchNormV2Attr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// FusedBatchNormV2IsTraining sets the optional is_training attribute to value.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormV2IsTraining(value bool) FusedBatchNormV2Attr {
+	return func(m optionalAttr) {
+		m["is_training"] = value
+	}
+}
+
+// Batch normalization.
 //
-//	lr_power: Scaling factor. Must be a scalar.
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
 //
-// Returns the created operation.
-func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlV2Attr) (o *tf.Operation) {
+// Arguments:
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	offset: A 1D Tensor for offset, to shift to the normalized x.
+//	mean: A 1D Tensor for population mean. Used for inference only;
+// must be empty for training.
+//	variance: A 1D Tensor for population variance. Used for inference only;
+// must be empty for training.
+//
+// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
+// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
+// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
+// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
+// in the cuDNN case), to be reused in the gradient computation.
+func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormV2Attr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -5956,133 +5798,94 @@ func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, li
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyFtrlV2",
+		Type: "FusedBatchNormV2",
 		Input: []tf.Input{
-			var_, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power,
+			x, scale, offset, mean, variance,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// Associates the given iterator with the given statistics aggregator.
-//
-// Returns the created operation.
-func IteratorSetStatsAggregator(scope *Scope, iterator_handle tf.Output, stats_aggregator_handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IteratorSetStatsAggregator",
-		Input: []tf.Input{
-			iterator_handle, stats_aggregator_handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// DataFormatVecPermuteAttr is an optional argument to DataFormatVecPermute.
-type DataFormatVecPermuteAttr func(optionalAttr)
-
-// DataFormatVecPermuteSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatVecPermuteSrcFormat(value string) DataFormatVecPermuteAttr {
-	return func(m optionalAttr) {
-		m["src_format"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// DataFormatVecPermuteDstFormat sets the optional dst_format attribute to value.
+// Returns the rank of a tensor.
 //
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatVecPermuteDstFormat(value string) DataFormatVecPermuteAttr {
-	return func(m optionalAttr) {
-		m["dst_format"] = value
-	}
-}
-
-// Returns the permuted vector/tensor in the destination data format given the
+// This operation returns an integer representing the rank of `input`.
 //
-// one in the source data format.
+// For example:
 //
-// Arguments:
-//	x: Vector of size 4 or Tensor of shape (4, 2) in source data format.
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// # shape of tensor 't' is [2, 2, 3]
+// rank(t) ==> 3
+// ```
 //
-// Returns Vector of size 4 or Tensor of shape (4, 2) in destination data format.
-func DataFormatVecPermute(scope *Scope, x tf.Output, optional ...DataFormatVecPermuteAttr) (y tf.Output) {
+// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
+// of a tensor is the number of indices required to uniquely select each element
+// of the tensor. Rank is also known as "order", "degree", or "ndims."
+func Rank(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DataFormatVecPermute",
+		Type: "Rank",
 		Input: []tf.Input{
-			x,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes tan of x element-wise.
-func Tan(scope *Scope, x tf.Output) (y tf.Output) {
+// Transforms a Tensor into a serialized TensorProto proto.
+//
+// Arguments:
+//	tensor: A Tensor of type `T`.
+//
+// Returns A serialized TensorProto proto of the input tensor.
+func SerializeTensor(scope *Scope, tensor tf.Output) (serialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Tan",
+		Type: "SerializeTensor",
 		Input: []tf.Input{
-			x,
+			tensor,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
-type ResourceSparseApplyFtrlAttr func(optionalAttr)
+// MatrixSolveAttr is an optional argument to MatrixSolve.
+type MatrixSolveAttr func(optionalAttr)
 
-// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
+// MatrixSolveAdjoint sets the optional adjoint attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+// adjoint.
 // If not specified, defaults to false
-func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
+func MatrixSolveAdjoint(value bool) MatrixSolveAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
+// Solves systems of linear equations.
 //
-// That is for rows we have grad for, we update var, accum and linear as follows:
-// accum_new = accum + grad * grad
-// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
+// `Matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. `Rhs` is a tensor of shape `[..., M, K]`. The `output` is
+// a tensor shape `[..., M, K]`.  If `adjoint` is `False` then each output matrix
+// satisfies `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `True` then each output matrix satisfies
+// `adjoint(matrix[..., :, :]) * output[..., :, :] = rhs[..., :, :]`.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	lr_power: Scaling factor. Must be a scalar.
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
 //
-// Returns the created operation.
-func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
+// Returns Shape is `[..., M, K]`.
+func MatrixSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixSolveAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6091,26 +5894,23 @@ func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, line
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyFtrl",
+		Type: "MatrixSolve",
 		Input: []tf.Input{
-			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
+			matrix, rhs,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns which elements of x are Inf.
-//
-// @compatibility(numpy)
-// Equivalent to np.isinf
-// @end_compatibility
-func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes acos of x element-wise.
+func Acos(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IsInf",
+		Type: "Acos",
 		Input: []tf.Input{
 			x,
 		},
@@ -6119,72 +5919,107 @@ func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
+// Real-valued fast Fourier transform.
 //
-// N is the size of the segment being reduced.
+// Computes the 1-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most dimension of `input`.
 //
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT` only returns the
+// `fft_length / 2 + 1` unique components of the FFT: the zero-frequency term,
+// followed by the `fft_length / 2` positive-frequency terms.
+//
+// Along the axis `RFFT` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-//	indices: A 1-D tensor. Has same rank as `segment_ids`.
-//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+// Returns A complex64 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length / 2 + 1` unique
+//   frequency components of its 1D Fourier transform.
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft
+// @end_compatibility
+func RFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSegmentSqrtN",
+		Type: "RFFT",
 		Input: []tf.Input{
-			data, indices, segment_ids,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
-type StatelessTruncatedNormalAttr func(optionalAttr)
+// DepthwiseConv2dNativeBackpropFilterAttr is an optional argument to DepthwiseConv2dNativeBackpropFilter.
+type DepthwiseConv2dNativeBackpropFilterAttr func(optionalAttr)
 
-// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
+// DepthwiseConv2dNativeBackpropFilterDataFormat sets the optional data_format attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
-	}
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func DepthwiseConv2dNativeBackpropFilterDataFormat(value string) DepthwiseConv2dNativeBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
 }
 
-// Outputs deterministic pseudorandom values from a truncated normal distribution.
-//
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
+// DepthwiseConv2dNativeBackpropFilterDilations sets the optional dilations attribute to value.
 //
-// The outputs are a deterministic function of `shape` and `seed`.
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func DepthwiseConv2dNativeBackpropFilterDilations(value []int64) DepthwiseConv2dNativeBackpropFilterAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes the gradients of depthwise convolution with respect to the filter.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+//	input: 4-D with shape based on `data_format`.  For example, if
+// `data_format` is 'NHWC' then `input` is a 4-D `[batch, in_height,
+// in_width, in_channels]` tensor.
+//	filter_sizes: An integer vector representing the tensor shape of `filter`,
+// where `filter` is a 4-D
+// `[filter_height, filter_width, in_channels, depthwise_multiplier]` tensor.
+//	out_backprop: 4-D with shape  based on `data_format`.
+// For example, if `data_format` is 'NHWC' then
+// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution.
+//	padding: The type of padding algorithm to use.
 //
-// Returns Random values with specified shape.
-func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
+// Returns 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.  Gradient w.r.t.
+// the `filter` input of the convolution.
+func DepthwiseConv2dNativeBackpropFilter(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropFilterAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessTruncatedNormal",
+		Type: "DepthwiseConv2dNativeBackpropFilter",
 		Input: []tf.Input{
-			shape, seed,
+			input, filter_sizes, out_backprop,
 		},
 		Attrs: attrs,
 	}
@@ -6192,51 +6027,69 @@ func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, opt
 	return op.Output(0)
 }
 
-// RestoreSliceAttr is an optional argument to RestoreSlice.
-type RestoreSliceAttr func(optionalAttr)
+// LRNGradAttr is an optional argument to LRNGrad.
+type LRNGradAttr func(optionalAttr)
 
-// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
+// LRNGradDepthRadius sets the optional depth_radius attribute to value.
 //
-// value: Index of file to open first if multiple files match
-// `file_pattern`. See the documentation for `Restore`.
-// If not specified, defaults to -1
-func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
+// value: A depth radius.
+// If not specified, defaults to 5
+func LRNGradDepthRadius(value int64) LRNGradAttr {
 	return func(m optionalAttr) {
-		m["preferred_shard"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// Restores a tensor from checkpoint files.
+// LRNGradBias sets the optional bias attribute to value.
 //
-// This is like `Restore` except that restored tensor can be listed as filling
-// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
-// larger tensor and the slice that the restored tensor covers.
+// value: An offset (usually > 0 to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNGradBias(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["bias"] = value
+	}
+}
+
+// LRNGradAlpha sets the optional alpha attribute to value.
 //
-// The `shape_and_slice` input has the same format as the
-// elements of the `shapes_and_slices` input of the `SaveSlices` op.
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNGradAlpha(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNGradBeta sets the optional beta attribute to value.
+//
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNGradBeta(value float32) LRNGradAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Gradients for Local Response Normalization.
 //
 // Arguments:
-//	file_pattern: Must have a single element. The pattern of the files from
-// which we read the tensor.
-//	tensor_name: Must have a single element. The name of the tensor to be
-// restored.
-//	shape_and_slice: Scalar. The shapes and slice specifications to use when
-// restoring a tensors.
-//	dt: The type of the tensor to be restored.
+//	input_grads: 4-D with shape `[batch, height, width, channels]`.
+//	input_image: 4-D with shape `[batch, height, width, channels]`.
+//	output_image: 4-D with shape `[batch, height, width, channels]`.
 //
-// Returns The restored tensor.
-func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
+// Returns The gradients for LRN.
+func LRNGrad(scope *Scope, input_grads tf.Output, input_image tf.Output, output_image tf.Output, optional ...LRNGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dt": dt}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RestoreSlice",
+		Type: "LRNGrad",
 		Input: []tf.Input{
-			file_pattern, tensor_name, shape_and_slice,
+			input_grads, input_image, output_image,
 		},
 		Attrs: attrs,
 	}
@@ -6244,31 +6097,33 @@ func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, s
 	return op.Output(0)
 }
 
-// ImagAttr is an optional argument to Imag.
-type ImagAttr func(optionalAttr)
+// AnyAttr is an optional argument to Any.
+type AnyAttr func(optionalAttr)
 
-// ImagTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func ImagTout(value tf.DataType) ImagAttr {
+// AnyKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AnyKeepDims(value bool) AnyAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Returns the imaginary part of a complex number.
+// Computes the "logical or" of elements across dimensions of a tensor.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the imaginary part of each element in `input`. All
-// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part returned by this operation.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
-// For example:
+// Arguments:
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.imag(input) ==> [4.75, 5.75]
-// ```
-func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
+// Returns The reduced tensor.
+func Any(scope *Scope, input tf.Output, axis tf.Output, optional ...AnyAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6277,9 +6132,9 @@ func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Imag",
+		Type: "Any",
 		Input: []tf.Input{
-			input,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -6287,34 +6142,41 @@ func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output
 	return op.Output(0)
 }
 
-// ComplexAttr is an optional argument to Complex.
-type ComplexAttr func(optionalAttr)
+// ResourceApplyFtrlAttr is an optional argument to ResourceApplyFtrl.
+type ResourceApplyFtrlAttr func(optionalAttr)
 
-// ComplexTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_COMPLEX64
-func ComplexTout(value tf.DataType) ComplexAttr {
+// ResourceApplyFtrlUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyFtrlUseLocking(value bool) ResourceApplyFtrlAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Converts two real numbers to a complex number.
-//
-// Given a tensor `real` representing the real part of a complex number, and a
-// tensor `imag` representing the imaginary part of a complex number, this
-// operation returns complex numbers elementwise of the form \\(a + bj\\), where
-// *a* represents the `real` part and *b* represents the `imag` part.
+// Update '*var' according to the Ftrl-proximal scheme.
 //
-// The input tensors `real` and `imag` must have the same shape.
+// accum_new = accum + grad * grad
+// linear += grad - (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
-// For example:
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 regulariation. Must be a scalar.
+//	lr_power: Scaling factor. Must be a scalar.
 //
-// ```
-// # tensor 'real' is [2.25, 3.25]
-// # tensor `imag` is [4.75, 5.75]
-// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
-// ```
-func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
+// Returns the created operation.
+func ResourceApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6323,95 +6185,93 @@ func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Complex",
+		Type: "ResourceApplyFtrl",
 		Input: []tf.Input{
-			real, imag,
+			var_, accum, linear, grad, lr, l1, l2, lr_power,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
-type UniqueWithCountsAttr func(optionalAttr)
+// RandomUniformAttr is an optional argument to RandomUniform.
+type RandomUniformAttr func(optionalAttr)
 
-// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
+// RandomUniformSeed sets the optional seed attribute to value.
+//
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformSeed(value int64) RandomUniformAttr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["seed"] = value
 	}
 }
 
-// Finds unique elements in a 1-D tensor.
+// RandomUniformSeed2 sets the optional seed2 attribute to value.
 //
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. Finally, it returns a third tensor `count` that
-// contains the count of each element of `y` in `x`. In other words:
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformSeed2(value int64) RandomUniformAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from a uniform distribution.
 //
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
-//
-// For example:
-//
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx, count = unique_with_counts(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// count ==> [2, 1, 3, 1, 2]
-// ```
+// The generated values follow a uniform distribution in the range `[0, 1)`. The
+// lower bound 0 is included in the range, while the upper bound 1 is excluded.
 //
 // Arguments:
-//	x: 1-D.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns 1-D.1-D.1-D.
-func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
+// Returns A tensor of the specified shape filled with uniform random values.
+func RandomUniform(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomUniformAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UniqueWithCounts",
+		Type: "RandomUniform",
 		Input: []tf.Input{
-			x,
+			shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// StatelessRandomNormalAttr is an optional argument to StatelessRandomNormal.
-type StatelessRandomNormalAttr func(optionalAttr)
+// AssertAttr is an optional argument to Assert.
+type AssertAttr func(optionalAttr)
 
-// StatelessRandomNormalDtype sets the optional dtype attribute to value.
+// AssertSummarize sets the optional summarize attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessRandomNormalDtype(value tf.DataType) StatelessRandomNormalAttr {
+// value: Print this many entries of each tensor.
+// If not specified, defaults to 3
+func AssertSummarize(value int64) AssertAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["summarize"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom values from a normal distribution.
-//
-// The generated values will have mean 0 and standard deviation 1.
+// Asserts that the given condition is true.
 //
-// The outputs are a deterministic function of `shape` and `seed`.
+// If `condition` evaluates to false, print the list of tensors in `data`.
+// `summarize` determines how many entries of the tensors to print.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+//	condition: The condition to evaluate.
+//	data: The tensors to print out when condition is false.
 //
-// Returns Random values with specified shape.
-func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomNormalAttr) (output tf.Output) {
+// Returns the created operation.
+func Assert(scope *Scope, condition tf.Output, data []tf.Output, optional ...AssertAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6420,202 +6280,184 @@ func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, option
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessRandomNormal",
+		Type: "Assert",
 		Input: []tf.Input{
-			shape, seed,
+			condition, tf.OutputList(data),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Reshapes a quantized tensor as per the Reshape op.
-//
-// ```
-//
-// Arguments:
+// Computes element-wise population count (a.k.a. popcount, bitsum, bitcount).
 //
-//	shape: Defines the shape of the output tensor.
-//	input_min: The minimum value of the input.
-//	input_max: The maximum value of the input.
+// For each entry in `x`, calculates the number of `1` (on) bits in the binary
+// representation of that entry.
 //
-// Returns This value is copied from input_min.This value is copied from input_max.
-func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// **NOTE**: It is more efficient to first `tf.bitcast` your tensors into
+// `int32` or `int64` and perform the bitcount on the result, than to feed in
+// 8- or 16-bit inputs and then aggregate the resulting counts.
+func PopulationCount(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedReshape",
+		Type: "PopulationCount",
 		Input: []tf.Input{
-			tensor, shape, input_min, input_max,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// GatherAttr is an optional argument to Gather.
-type GatherAttr func(optionalAttr)
-
-// GatherValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func GatherValidateIndices(value bool) GatherAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
+	return op.Output(0)
 }
 
-// Gather slices from `params` according to `indices`.
+// Split a `SparseTensor` into `num_split` tensors along one dimension.
 //
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
+// If the `shape[split_dim]` is not an integer multiple of `num_split`. Slices
+// `[0 : shape[split_dim] % num_split]` gets one extra dimension.
+// For example, if `split_dim = 1` and `num_split = 2` and the input is
 //
-// ```python
-//     # Scalar indices
-//     output[:, ..., :] = params[indices, :, ... :]
+//     input_tensor = shape = [2, 7]
+//     [    a   d e  ]
+//     [b c          ]
 //
-//     # Vector indices
-//     output[i, :, ..., :] = params[indices[i], :, ... :]
+// Graphically the output tensors are:
 //
-//     # Higher rank indices
-//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
-// ```
+//     output_tensor[0] = shape = [2, 4]
+//     [    a  ]
+//     [b c    ]
 //
-// If `indices` is a permutation and `len(indices) == params.shape[0]` then
-// this operation will permute `params` accordingly.
+//     output_tensor[1] = shape = [2, 3]
+//     [ d e  ]
+//     [      ]
 //
-// `validate_indices`: DEPRECATED. If this operation is assigned to CPU, values in
-// `indices` are always validated to be within range. If assigned to GPU,
-// out-of-bound indices result in safe but unspecified behavior, which may include
-// raising an error.
+// Arguments:
+//	split_dim: 0-D.  The dimension along which to split.  Must be in the range
+// `[0, rank(shape))`.
+//	indices: 2-D tensor represents the indices of the sparse tensor.
+//	values: 1-D tensor represents the values of the sparse tensor.
+//	shape: 1-D. tensor represents the shape of the sparse tensor.
+// output indices: A list of 1-D tensors represents the indices of the output
+// sparse tensors.
+//	num_split: The number of ways to split.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
-// </div>
-func Gather(scope *Scope, params tf.Output, indices tf.Output, optional ...GatherAttr) (output tf.Output) {
+// Returns A list of 1-D tensors represents the values of the output sparse
+// tensors.A list of 1-D tensors represents the shape of the output sparse
+// tensors.
+func SparseSplit(scope *Scope, split_dim tf.Output, indices tf.Output, values tf.Output, shape tf.Output, num_split int64) (output_indices []tf.Output, output_values []tf.Output, output_shape []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "Gather",
+		Type: "SparseSplit",
 		Input: []tf.Input{
-			params, indices,
+			split_dim, indices, values, shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the truth value of (x != y) element-wise.
-//
-// *NOTE*: `NotEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func NotEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "NotEqual",
-		Input: []tf.Input{
-			x, y,
-		},
+	var idx int
+	var err error
+	if output_indices, idx, err = makeOutputList(op, idx, "output_indices"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if output_values, idx, err = makeOutputList(op, idx, "output_values"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
+	}
+	if output_shape, idx, err = makeOutputList(op, idx, "output_shape"); err != nil {
+		scope.UpdateErr("SparseSplit", err)
+		return
+	}
+	return output_indices, output_values, output_shape
 }
 
-// Inverse 3D real-valued fast Fourier transform.
-//
-// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most 3 dimensions of `input`.
-//
-// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
-// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
-// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
-// from the size of the inner-most 3 dimensions of `input`. If the FFT length used
-// to compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
-//
-// Along each axis `IRFFT3D` is computed on, if `fft_length` (or
-// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
-//
-// Returns A float32 tensor of the same rank as `input`. The inner-most 3
-//   dimensions of `input` are replaced with the `fft_length` samples of their
-//   inverse 3D real Fourier transform.
+// RandomPoissonAttr is an optional argument to RandomPoisson.
+type RandomPoissonAttr func(optionalAttr)
+
+// RandomPoissonSeed sets the optional seed attribute to value.
+// If not specified, defaults to 0
+func RandomPoissonSeed(value int64) RandomPoissonAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomPoissonSeed2 sets the optional seed2 attribute to value.
+// If not specified, defaults to 0
+func RandomPoissonSeed2(value int64) RandomPoissonAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Use RandomPoissonV2 instead.
 //
-// @compatibility(numpy)
-// Equivalent to np.irfftn with 3 dimensions.
-// @end_compatibility
-func IRFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// DEPRECATED at GraphDef version 25: Replaced by RandomPoissonV2
+func RandomPoisson(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IRFFT3D",
+		Type: "RandomPoisson",
 		Input: []tf.Input{
-			input, fft_length,
+			shape, rate,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StringSplitAttr is an optional argument to StringSplit.
-type StringSplitAttr func(optionalAttr)
+// ResourceSparseApplyFtrlV2Attr is an optional argument to ResourceSparseApplyFtrlV2.
+type ResourceSparseApplyFtrlV2Attr func(optionalAttr)
 
-// StringSplitSkipEmpty sets the optional skip_empty attribute to value.
+// ResourceSparseApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
 //
-// value: A `bool`. If `True`, skip the empty strings from the result.
-// If not specified, defaults to true
-func StringSplitSkipEmpty(value bool) StringSplitAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyFtrlV2UseLocking(value bool) ResourceSparseApplyFtrlV2Attr {
 	return func(m optionalAttr) {
-		m["skip_empty"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Split elements of `input` based on `delimiter` into a `SparseTensor`.
-//
-// Let N be the size of source (typically N will be the batch size). Split each
-// element of `input` based on `delimiter` and return a `SparseTensor`
-// containing the splitted tokens. Empty tokens are ignored.
-//
-// `delimiter` can be empty, or a string of split characters. If `delimiter` is an
-//  empty string, each element of `input` is split into individual single-byte
-//  character strings, including splitting of UTF-8 multibyte sequences. Otherwise
-//  every character of `delimiter` is a potential split point.
-//
-// For example:
-//   N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output
-//   will be
+// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
 //
-//   indices = [0, 0;
-//              0, 1;
-//              1, 0;
-//              1, 1;
-//              1, 2]
-//   shape = [2, 3]
-//   values = ['hello', 'world', 'a', 'b', 'c']
+// That is for rows we have grad for, we update var, accum and linear as follows:
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
-//	input: 1-D. Strings to split.
-//	delimiter: 0-D. Delimiter characters (bytes), or empty string.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
 //
-// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse
-// tensor, where the first value is N and the second value is the maximum number
-// of tokens in a single input entry.
-func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) {
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceSparseApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6624,43 +6466,63 @@ func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ..
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringSplit",
+		Type: "ResourceSparseApplyFtrlV2",
 		Input: []tf.Input{
-			input, delimiter,
+			var_, accum, linear, grad, indices, lr, l1, l2, l2_shrinkage, lr_power,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
-// ResizeBilinearAttr is an optional argument to ResizeBilinear.
-type ResizeBilinearAttr func(optionalAttr)
+// Associates the given iterator with the given statistics aggregator.
+//
+// Returns the created operation.
+func IteratorSetStatsAggregator(scope *Scope, iterator_handle tf.Output, stats_aggregator_handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IteratorSetStatsAggregator",
+		Input: []tf.Input{
+			iterator_handle, stats_aggregator_handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
 
-// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
+// DataFormatVecPermuteAttr is an optional argument to DataFormatVecPermute.
+type DataFormatVecPermuteAttr func(optionalAttr)
+
+// DataFormatVecPermuteSrcFormat sets the optional src_format attribute to value.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatVecPermuteSrcFormat(value string) DataFormatVecPermuteAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["src_format"] = value
 	}
 }
 
-// Resize `images` to `size` using bilinear interpolation.
+// DataFormatVecPermuteDstFormat sets the optional dst_format attribute to value.
 //
-// Input images can be of different types but output images are always float.
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatVecPermuteDstFormat(value string) DataFormatVecPermuteAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the permuted vector/tensor in the destination data format given the
+//
+// one in the source data format.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	x: Vector of size 4 or Tensor of shape (4, 2) in source data format.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
+// Returns Vector of size 4 or Tensor of shape (4, 2) in destination data format.
+func DataFormatVecPermute(scope *Scope, x tf.Output, optional ...DataFormatVecPermuteAttr) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -6669,9 +6531,9 @@ func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBilinear",
+		Type: "DataFormatVecPermute",
 		Input: []tf.Input{
-			images, size,
+			x,
 		},
 		Attrs: attrs,
 	}
@@ -6679,168 +6541,160 @@ func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...
 	return op.Output(0)
 }
 
-// Computes softsign: `features / (abs(features) + 1)`.
-func Softsign(scope *Scope, features tf.Output) (activations tf.Output) {
+// Computes tan of x element-wise.
+func Tan(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Softsign",
+		Type: "Tan",
 		Input: []tf.Input{
-			features,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// GenerateVocabRemappingAttr is an optional argument to GenerateVocabRemapping.
-type GenerateVocabRemappingAttr func(optionalAttr)
+// ResourceSparseApplyFtrlAttr is an optional argument to ResourceSparseApplyFtrl.
+type ResourceSparseApplyFtrlAttr func(optionalAttr)
 
-// GenerateVocabRemappingOldVocabSize sets the optional old_vocab_size attribute to value.
-//
-// value: Number of entries in the old vocab file to consider.  If -1,
-// use the entire old vocabulary.
-// If not specified, defaults to -1
+// ResourceSparseApplyFtrlUseLocking sets the optional use_locking attribute to value.
 //
-// REQUIRES: value >= -1
-func GenerateVocabRemappingOldVocabSize(value int64) GenerateVocabRemappingAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyFtrlUseLocking(value bool) ResourceSparseApplyFtrlAttr {
 	return func(m optionalAttr) {
-		m["old_vocab_size"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Given a path to new and old vocabulary files, returns a remapping Tensor of
-//
-// length `num_new_vocab`, where `remapping[i]` contains the row number in the old
-// vocabulary that corresponds to row `i` in the new vocabulary (starting at line
-// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i`
-// in the new vocabulary is not in the old vocabulary.  The old vocabulary is
-// constrained to the first `old_vocab_size` entries if `old_vocab_size` is not the
-// default value of -1.
-//
-// `num_vocab_offset` enables
-// use in the partitioned variable case, and should generally be set through
-// examining partitioning info.  The format of the files should be a text file,
-// with each line containing a single entity within the vocabulary.
-//
-// For example, with `new_vocab_file` a text file containing each of the following
-// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3],
-// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be
-// `[0, -1, 2]`.
-//
-// The op also returns a count of how many entries in the new vocabulary
-// were present in the old vocabulary, which is used to calculate the number of
-// values to initialize in a weight matrix remapping
+// Update relevant entries in '*var' according to the Ftrl-proximal scheme.
 //
-// This functionality can be used to remap both row vocabularies (typically,
-// features) and column vocabularies (typically, classes) from TensorFlow
-// checkpoints.  Note that the partitioning logic relies on contiguous vocabularies
-// corresponding to div-partitioned variables.  Moreover, the underlying remapping
-// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should
-// use the corresponding index_table_from_file() as the FeatureColumn framework
-// does (as opposed to tf.feature_to_id(), which uses a CuckooTable).
+// That is for rows we have grad for, we update var, accum and linear as follows:
+// accum_new = accum + grad * grad
+// linear += grad + (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
-//	new_vocab_file: Path to the new vocab file.
-//	old_vocab_file: Path to the old vocab file.
-//	new_vocab_offset: How many entries into the new vocab file to start reading.
-//	num_new_vocab: Number of entries in the new vocab file to remap.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	lr_power: Scaling factor. Must be a scalar.
 //
-// Returns A Tensor of length num_new_vocab where the element at index i
-// is equal to the old ID that maps to the new ID i.  This element is -1 for any
-// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab.
-func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64, optional ...GenerateVocabRemappingAttr) (remapping tf.Output, num_present tf.Output) {
+// Returns the created operation.
+func ResourceSparseApplyFtrl(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, lr_power tf.Output, optional ...ResourceSparseApplyFtrlAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "GenerateVocabRemapping",
+		Type: "ResourceSparseApplyFtrl",
 		Input: []tf.Input{
-			new_vocab_file, old_vocab_file,
+			var_, accum, linear, grad, indices, lr, l1, l2, lr_power,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return scope.AddOperation(opspec)
 }
 
-// Assigns sparse updates to the variable referenced by `resource`.
-//
-// This operation computes
+// Returns which elements of x are Inf.
 //
-//     # Scalar indices
-//     ref[indices, ...] = updates[...]
+// @compatibility(numpy)
+// Equivalent to np.isinf
+// @end_compatibility
+func IsInf(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IsInf",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the sum along sparse segments of a tensor divided by the sqrt of N.
 //
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] = updates[i, ...]
+// N is the size of the segment being reduced.
 //
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] = updates[i, ..., j, ...]
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
 // Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
 //
-// Returns the created operation.
-func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+//	indices: A 1-D tensor. Has same rank as `segment_ids`.
+//	segment_ids: A 1-D tensor. Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SparseSegmentSqrtN(scope *Scope, data tf.Output, indices tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterUpdate",
+		Type: "SparseSegmentSqrtN",
 		Input: []tf.Input{
-			resource, indices, updates,
+			data, indices, segment_ids,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
-type AvgPoolGradAttr func(optionalAttr)
+// StatelessTruncatedNormalAttr is an optional argument to StatelessTruncatedNormal.
+type StatelessTruncatedNormalAttr func(optionalAttr)
 
-// AvgPoolGradDataFormat sets the optional data_format attribute to value.
+// StatelessTruncatedNormalDtype sets the optional dtype attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessTruncatedNormalDtype(value tf.DataType) StatelessTruncatedNormalAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["dtype"] = value
 	}
 }
 
-// Computes gradients of the average pooling function.
+// Outputs deterministic pseudorandom values from a truncated normal distribution.
+//
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
+//
+// The outputs are a deterministic function of `shape` and `seed`.
 //
 // Arguments:
-//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
-//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
-// the output of `avg_pool`.
-//	ksize: The size of the sliding window for each dimension of the input.
-//	strides: The stride of the sliding window for each dimension of the input.
-//	padding: The type of padding algorithm to use.
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
 //
-// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
-func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
+// Returns Random values with specified shape.
+func StatelessTruncatedNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessTruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AvgPoolGrad",
+		Type: "StatelessTruncatedNormal",
 		Input: []tf.Input{
-			orig_input_shape, grad,
+			shape, seed,
 		},
 		Attrs: attrs,
 	}
@@ -6848,184 +6702,129 @@ func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize
 	return op.Output(0)
 }
 
-// StageClearAttr is an optional argument to StageClear.
-type StageClearAttr func(optionalAttr)
+// RestoreSliceAttr is an optional argument to RestoreSlice.
+type RestoreSliceAttr func(optionalAttr)
 
-// StageClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// RestoreSlicePreferredShard sets the optional preferred_shard attribute to value.
 //
-// REQUIRES: value >= 0
-func StageClearCapacity(value int64) StageClearAttr {
+// value: Index of file to open first if multiple files match
+// `file_pattern`. See the documentation for `Restore`.
+// If not specified, defaults to -1
+func RestoreSlicePreferredShard(value int64) RestoreSliceAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["preferred_shard"] = value
 	}
 }
 
-// StageClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// Restores a tensor from checkpoint files.
 //
-// REQUIRES: value >= 0
-func StageClearMemoryLimit(value int64) StageClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// StageClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func StageClearContainer(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// StageClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func StageClearSharedName(value string) StageClearAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes all elements in the underlying container.
+// This is like `Restore` except that restored tensor can be listed as filling
+// only a slice of a larger tensor.  `shape_and_slice` specifies the shape of the
+// larger tensor and the slice that the restored tensor covers.
 //
-// Returns the created operation.
-func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+// The `shape_and_slice` input has the same format as the
+// elements of the `shapes_and_slices` input of the `SaveSlices` op.
+//
+// Arguments:
+//	file_pattern: Must have a single element. The pattern of the files from
+// which we read the tensor.
+//	tensor_name: Must have a single element. The name of the tensor to be
+// restored.
+//	shape_and_slice: Scalar. The shapes and slice specifications to use when
+// restoring a tensors.
+//	dt: The type of the tensor to be restored.
+//
+// Returns The restored tensor.
+func RestoreSlice(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, shape_and_slice tf.Output, dt tf.DataType, optional ...RestoreSliceAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{"dt": dt}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StageClear",
-
+		Type: "RestoreSlice",
+		Input: []tf.Input{
+			file_pattern, tensor_name, shape_and_slice,
+		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
-type ComputeAccidentalHitsAttr func(optionalAttr)
-
-// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
+// ImagAttr is an optional argument to Imag.
+type ImagAttr func(optionalAttr)
 
-// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+// ImagTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func ImagTout(value tf.DataType) ImagAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["Tout"] = value
 	}
 }
 
-// Computes the ids of the positions in sampled_candidates that match true_labels.
+// Returns the imaginary part of a complex number.
 //
-// When doing log-odds NCE, the result of this op should be passed through a
-// SparseToDense op, then added to the logits of the sampled candidates. This has
-// the effect of 'removing' the sampled labels that match the true labels by
-// making the classifier sure that they are sampled labels.
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the imaginary part of each element in `input`. All
+// elements in `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part returned by this operation.
 //
-// Arguments:
-//	true_classes: The true_classes output of UnpackSparseLabels.
-//	sampled_candidates: The sampled_candidates output of CandidateSampler.
-//	num_true: Number of true labels per context.
+// For example:
 //
-// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
-// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
-// is -FLOAT_MAX.
-func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.imag(input) ==> [4.75, 5.75]
+// ```
+func Imag(scope *Scope, input tf.Output, optional ...ImagAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ComputeAccidentalHits",
+		Type: "Imag",
 		Input: []tf.Input{
-			true_classes, sampled_candidates,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// CumsumAttr is an optional argument to Cumsum.
-type CumsumAttr func(optionalAttr)
-
-// CumsumExclusive sets the optional exclusive attribute to value.
-//
-// value: If `True`, perform exclusive cumsum.
-// If not specified, defaults to false
-func CumsumExclusive(value bool) CumsumAttr {
-	return func(m optionalAttr) {
-		m["exclusive"] = value
-	}
-}
+// ComplexAttr is an optional argument to Complex.
+type ComplexAttr func(optionalAttr)
 
-// CumsumReverse sets the optional reverse attribute to value.
-//
-// value: A `bool` (default: False).
-// If not specified, defaults to false
-func CumsumReverse(value bool) CumsumAttr {
+// ComplexTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_COMPLEX64
+func ComplexTout(value tf.DataType) ComplexAttr {
 	return func(m optionalAttr) {
-		m["reverse"] = value
+		m["Tout"] = value
 	}
 }
 
-// Compute the cumulative sum of the tensor `x` along `axis`.
-//
-// By default, this op performs an inclusive cumsum, which means that the first
-// element of the input is identical to the first element of the output:
-//
-// ```python
-// tf.cumsum([a, b, c])  # => [a, a + b, a + b + c]
-// ```
+// Converts two real numbers to a complex number.
 //
-// By setting the `exclusive` kwarg to `True`, an exclusive cumsum is
-// performed instead:
+// Given a tensor `real` representing the real part of a complex number, and a
+// tensor `imag` representing the imaginary part of a complex number, this
+// operation returns complex numbers elementwise of the form \\(a + bj\\), where
+// *a* represents the `real` part and *b* represents the `imag` part.
 //
-// ```python
-// tf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]
-// ```
+// The input tensors `real` and `imag` must have the same shape.
 //
-// By setting the `reverse` kwarg to `True`, the cumsum is performed in the
-// opposite direction:
+// For example:
 //
-// ```python
-// tf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]
 // ```
-//
-// This is more efficient than using separate `tf.reverse` ops.
-//
-// The `reverse` and `exclusive` kwargs can also be combined:
-//
-// ```python
-// tf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]
+// # tensor 'real' is [2.25, 3.25]
+// # tensor `imag` is [4.75, 5.75]
+// tf.complex(real, imag) ==> [[2.25 + 4.75j], [3.25 + 5.75j]]
 // ```
-//
-// Arguments:
-//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
-// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
-// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
-//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
-// `[-rank(x), rank(x))`.
-func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (out tf.Output) {
+func Complex(scope *Scope, real tf.Output, imag tf.Output, optional ...ComplexAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -7034,9 +6833,9 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Cumsum",
+		Type: "Complex",
 		Input: []tf.Input{
-			x, axis,
+			real, imag,
 		},
 		Attrs: attrs,
 	}
@@ -7044,26 +6843,42 @@ func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (
 	return op.Output(0)
 }
 
-// QuantizedRelu6Attr is an optional argument to QuantizedRelu6.
-type QuantizedRelu6Attr func(optionalAttr)
+// UniqueWithCountsAttr is an optional argument to UniqueWithCounts.
+type UniqueWithCountsAttr func(optionalAttr)
 
-// QuantizedRelu6OutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedRelu6OutType(value tf.DataType) QuantizedRelu6Attr {
+// UniqueWithCountsOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueWithCountsOutIdx(value tf.DataType) UniqueWithCountsAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["out_idx"] = value
 	}
 }
 
-// Computes Quantized Rectified Linear 6: `min(max(features, 0), 6)`
+// Finds unique elements in a 1-D tensor.
 //
-// Arguments:
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. Finally, it returns a third tensor `count` that
+// contains the count of each element of `y` in `x`. In other words:
 //
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
 //
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedRelu6Attr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+// For example:
+//
+// ```
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx, count = unique_with_counts(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+// count ==> [2, 1, 3, 1, 2]
+// ```
+//
+// Arguments:
+//	x: 1-D.
+//
+// Returns 1-D.1-D.1-D.
+func UniqueWithCounts(scope *Scope, x tf.Output, optional ...UniqueWithCountsAttr) (y tf.Output, idx tf.Output, count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -7072,9 +6887,9 @@ func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, ma
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedRelu6",
+		Type: "UniqueWithCounts",
 		Input: []tf.Input{
-			features, min_features, max_features,
+			x,
 		},
 		Attrs: attrs,
 	}
@@ -7082,278 +6897,291 @@ func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, ma
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2.
-type FixedLengthRecordReaderV2Attr func(optionalAttr)
-
-// FixedLengthRecordReaderV2HeaderBytes sets the optional header_bytes attribute to value.
-//
-// value: Number of bytes in the header, defaults to 0.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2HeaderBytes(value int64) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["header_bytes"] = value
-	}
-}
-
-// FixedLengthRecordReaderV2FooterBytes sets the optional footer_bytes attribute to value.
-//
-// value: Number of bytes in the footer, defaults to 0.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2FooterBytes(value int64) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["footer_bytes"] = value
-	}
-}
-
-// FixedLengthRecordReaderV2HopBytes sets the optional hop_bytes attribute to value.
-//
-// value: Number of bytes to hop before each read. Default of 0 means using
-// record_bytes.
-// If not specified, defaults to 0
-func FixedLengthRecordReaderV2HopBytes(value int64) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["hop_bytes"] = value
-	}
-}
+// StatelessRandomNormalAttr is an optional argument to StatelessRandomNormal.
+type StatelessRandomNormalAttr func(optionalAttr)
 
-// FixedLengthRecordReaderV2Container sets the optional container attribute to value.
+// StatelessRandomNormalDtype sets the optional dtype attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2Container(value string) FixedLengthRecordReaderV2Attr {
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessRandomNormalDtype(value tf.DataType) StatelessRandomNormalAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["dtype"] = value
 	}
 }
 
-// FixedLengthRecordReaderV2SharedName sets the optional shared_name attribute to value.
+// Outputs deterministic pseudorandom values from a normal distribution.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2SharedName(value string) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// FixedLengthRecordReaderV2Encoding sets the optional encoding attribute to value.
+// The generated values will have mean 0 and standard deviation 1.
 //
-// value: The type of encoding for the file. Currently ZLIB and GZIP
-// are supported. Defaults to none.
-// If not specified, defaults to ""
-func FixedLengthRecordReaderV2Encoding(value string) FixedLengthRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["encoding"] = value
-	}
-}
-
-// A Reader that outputs fixed-length records from a file.
+// The outputs are a deterministic function of `shape` and `seed`.
 //
 // Arguments:
-//	record_bytes: Number of bytes in the record.
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
 //
-// Returns The handle to reference the Reader.
-func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...FixedLengthRecordReaderV2Attr) (reader_handle tf.Output) {
+// Returns Random values with specified shape.
+func StatelessRandomNormal(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"record_bytes": record_bytes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FixedLengthRecordReaderV2",
-
+		Type: "StatelessRandomNormal",
+		Input: []tf.Input{
+			shape, seed,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process.
+// Reshapes a quantized tensor as per the Reshape op.
 //
-// Note that the hash function may change from time to time.
-// This functionality will be deprecated and it's recommended to use
-// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
+// ```
 //
 // Arguments:
 //
-//	num_buckets: The number of buckets.
+//	shape: Defines the shape of the output tensor.
+//	input_min: The minimum value of the input.
+//	input_max: The maximum value of the input.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
+// Returns This value is copied from input_min.This value is copied from input_max.
+func QuantizedReshape(scope *Scope, tensor tf.Output, shape tf.Output, input_min tf.Output, input_max tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "StringToHashBucket",
+		Type: "QuantizedReshape",
 		Input: []tf.Input{
-			string_tensor,
+			tensor, shape, input_min, input_max,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes gradients for the exponential linear (Elu) operation.
+// GatherAttr is an optional argument to Gather.
+type GatherAttr func(optionalAttr)
+
+// GatherValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func GatherValidateIndices(value bool) GatherAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Gather slices from `params` according to `indices`.
 //
-// Arguments:
-//	gradients: The backpropagated gradients to the corresponding Elu operation.
-//	outputs: The outputs of the corresponding Elu operation.
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
 //
-// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
-// `gradients` otherwise.
-func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
+// ```python
+//     # Scalar indices
+//     output[:, ..., :] = params[indices, :, ... :]
+//
+//     # Vector indices
+//     output[i, :, ..., :] = params[indices[i], :, ... :]
+//
+//     # Higher rank indices
+//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
+// ```
+//
+// If `indices` is a permutation and `len(indices) == params.shape[0]` then
+// this operation will permute `params` accordingly.
+//
+// `validate_indices`: DEPRECATED. If this operation is assigned to CPU, values in
+// `indices` are always validated to be within range. If assigned to GPU,
+// out-of-bound indices result in safe but unspecified behavior, which may include
+// raising an error.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/Gather.png" alt>
+// </div>
+func Gather(scope *Scope, params tf.Output, indices tf.Output, optional ...GatherAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "EluGrad",
+		Type: "Gather",
 		Input: []tf.Input{
-			gradients, outputs,
+			params, indices,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a dataset that contains `count` elements from the `input_dataset`.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be taken. A value of `-1` indicates that all of `input_dataset`
-// is taken.
-//
+// Returns the truth value of (x != y) element-wise.
 //
-func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// *NOTE*: `NotEqual` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func NotEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TakeDataset",
+		Type: "NotEqual",
 		Input: []tf.Input{
-			input_dataset, count,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// The gradient operator for the SparseAdd op.
+// Inverse 3D real-valued fast Fourier transform.
 //
-// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
-// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
-// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
-// values of A and B.
+// Computes the inverse 3-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most 3 dimensions of `input`.
+//
+// The inner-most 3 dimensions of `input` are assumed to be the result of `RFFT3D`:
+// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
+// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
+// from the size of the inner-most 3 dimensions of `input`. If the FFT length used
+// to compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
+//
+// Along each axis `IRFFT3D` is computed on, if `fft_length` (or
+// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
-//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
-// the non-empty values of the sum.
-//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
-//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
-//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
-// `[nnz(sum), ndims]`.
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
 //
-// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
-// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
-// non-empty values of B.
-func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
+// Returns A float32 tensor of the same rank as `input`. The inner-most 3
+//   dimensions of `input` are replaced with the `fft_length` samples of their
+//   inverse 3D real Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.irfftn with 3 dimensions.
+// @end_compatibility
+func IRFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseAddGrad",
+		Type: "IRFFT3D",
 		Input: []tf.Input{
-			backprop_val_grad, a_indices, b_indices, sum_indices,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Computes atan of x element-wise.
-func Atan(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Atan",
-		Input: []tf.Input{
-			x,
-		},
+// StringSplitAttr is an optional argument to StringSplit.
+type StringSplitAttr func(optionalAttr)
+
+// StringSplitSkipEmpty sets the optional skip_empty attribute to value.
+//
+// value: A `bool`. If `True`, skip the empty strings from the result.
+// If not specified, defaults to true
+func StringSplitSkipEmpty(value bool) StringSplitAttr {
+	return func(m optionalAttr) {
+		m["skip_empty"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Encode audio data using the WAV file format.
+// Split elements of `input` based on `delimiter` into a `SparseTensor`.
 //
-// This operation will generate a string suitable to be saved out to create a .wav
-// audio file. It will be encoded in the 16-bit PCM format. It takes in float
-// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
-// that range.
+// Let N be the size of source (typically N will be the batch size). Split each
+// element of `input` based on `delimiter` and return a `SparseTensor`
+// containing the splitted tokens. Empty tokens are ignored.
 //
-// `audio` is a 2-D float Tensor of shape `[length, channels]`.
-// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
+// `delimiter` can be empty, or a string of split characters. If `delimiter` is an
+//  empty string, each element of `input` is split into individual single-byte
+//  character strings, including splitting of UTF-8 multibyte sequences. Otherwise
+//  every character of `delimiter` is a potential split point.
+//
+// For example:
+//   N = 2, input[0] is 'hello world' and input[1] is 'a b c', then the output
+//   will be
+//
+//   indices = [0, 0;
+//              0, 1;
+//              1, 0;
+//              1, 1;
+//              1, 2]
+//   shape = [2, 3]
+//   values = ['hello', 'world', 'a', 'b', 'c']
 //
 // Arguments:
-//	audio: 2-D with shape `[length, channels]`.
-//	sample_rate: Scalar containing the sample frequency.
+//	input: 1-D. Strings to split.
+//	delimiter: 0-D. Delimiter characters (bytes), or empty string.
 //
-// Returns 0-D. WAV-encoded file contents.
-func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
+// Returns A dense matrix of int64 representing the indices of the sparse tensor.A vector of strings corresponding to the splited values.a length-2 vector of int64 representing the shape of the sparse
+// tensor, where the first value is N and the second value is the maximum number
+// of tokens in a single input entry.
+func StringSplit(scope *Scope, input tf.Output, delimiter tf.Output, optional ...StringSplitAttr) (indices tf.Output, values tf.Output, shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "EncodeWav",
+		Type: "StringSplit",
 		Input: []tf.Input{
-			audio, sample_rate,
+			input, delimiter,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
+// ResizeBilinearAttr is an optional argument to ResizeBilinear.
+type ResizeBilinearAttr func(optionalAttr)
+
+// ResizeBilinearAlignCorners sets the optional align_corners attribute to value.
 //
-// The hash function is deterministic on the content of the string within the
-// process. The hash function is a keyed hash function, where attribute `key`
-// defines the key of the hash function. `key` is an array of 2 elements.
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeBilinearAlignCorners(value bool) ResizeBilinearAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Resize `images` to `size` using bilinear interpolation.
 //
-// A strong hash is important when inputs may be malicious, e.g. URLs with
-// additional components. Adversaries could try to make their inputs hash to the
-// same bucket for a denial-of-service attack or to skew the results. A strong
-// hash prevents this by making it difficult, if not infeasible, to compute inputs
-// that hash to the same bucket. This comes at a cost of roughly 4x higher compute
-// time than `tf.string_to_hash_bucket_fast`.
+// Input images can be of different types but output images are always float.
 //
 // Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
-//	key: The key for the keyed hash function passed as a list of two uint64
-// elements.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64, key []int64) (output tf.Output) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeBilinear(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeBilinearAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets, "key": key}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "StringToHashBucketStrong",
+		Type: "ResizeBilinear",
 		Input: []tf.Input{
-			input,
+			images, size,
 		},
 		Attrs: attrs,
 	}
@@ -7361,252 +7189,336 @@ func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64,
 	return op.Output(0)
 }
 
-// Applies softmax to a batched N-D `SparseTensor`.
-//
-// The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
-// (where `N >= 2`), and with indices sorted in the canonical lexicographic order.
-//
-// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost
-// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly
-// zero elements do not participate*.  Specifically, the algorithm is equivalent
-// to the following:
-//
-//   (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix
-//       with shape `[B, C]`, along the size-C dimension;
-//   (2) Masks out the original implicitly-zero locations;
-//   (3) Renormalizes the remaining elements.
-//
-// Hence, the `SparseTensor` result has exactly the same non-zero indices and
-// shape.
-//
-// Arguments:
-//	sp_indices: 2-D.  `NNZ x R` matrix with the indices of non-empty values in a
-// SparseTensor, in canonical ordering.
-//	sp_values: 1-D.  `NNZ` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//
-// Returns 1-D.  The `NNZ` values for the result `SparseTensor`.
-func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) {
+// Computes softsign: `features / (abs(features) + 1)`.
+func Softsign(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSoftmax",
+		Type: "Softsign",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape,
+			features,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
-//
-// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
-// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
-// are placed in `outputs[i]` in lexicographic order of `js`, and the first
-// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
-// In detail,
-//
-// ```python
-//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
+// GenerateVocabRemappingAttr is an optional argument to GenerateVocabRemapping.
+type GenerateVocabRemappingAttr func(optionalAttr)
+
+// GenerateVocabRemappingOldVocabSize sets the optional old_vocab_size attribute to value.
 //
-//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
-// ```
+// value: Number of entries in the old vocab file to consider.  If -1,
+// use the entire old vocabulary.
+// If not specified, defaults to -1
 //
-// `data.shape` must start with `partitions.shape`.
+// REQUIRES: value >= -1
+func GenerateVocabRemappingOldVocabSize(value int64) GenerateVocabRemappingAttr {
+	return func(m optionalAttr) {
+		m["old_vocab_size"] = value
+	}
+}
+
+// Given a path to new and old vocabulary files, returns a remapping Tensor of
 //
-// For example:
+// length `num_new_vocab`, where `remapping[i]` contains the row number in the old
+// vocabulary that corresponds to row `i` in the new vocabulary (starting at line
+// `new_vocab_offset` and up to `num_new_vocab` entities), or `-1` if entry `i`
+// in the new vocabulary is not in the old vocabulary.  The old vocabulary is
+// constrained to the first `old_vocab_size` entries if `old_vocab_size` is not the
+// default value of -1.
 //
-// ```python
-//     # Scalar partitions.
-//     partitions = 1
-//     num_partitions = 2
-//     data = [10, 20]
-//     outputs[0] = []  # Empty with shape [0, 2]
-//     outputs[1] = [[10, 20]]
+// `num_vocab_offset` enables
+// use in the partitioned variable case, and should generally be set through
+// examining partitioning info.  The format of the files should be a text file,
+// with each line containing a single entity within the vocabulary.
 //
-//     # Vector partitions.
-//     partitions = [0, 0, 1, 1, 0]
-//     num_partitions = 2
-//     data = [10, 20, 30, 40, 50]
-//     outputs[0] = [10, 20, 50]
-//     outputs[1] = [30, 40]
-// ```
+// For example, with `new_vocab_file` a text file containing each of the following
+// elements on a single line: `[f0, f1, f2, f3]`, old_vocab_file = [f1, f0, f3],
+// `num_new_vocab = 3, new_vocab_offset = 1`, the returned remapping would be
+// `[0, -1, 2]`.
 //
-// See `dynamic_stitch` for an example on how to merge partitions back.
+// The op also returns a count of how many entries in the new vocabulary
+// were present in the old vocabulary, which is used to calculate the number of
+// values to initialize in a weight matrix remapping
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
-// </div>
+// This functionality can be used to remap both row vocabularies (typically,
+// features) and column vocabularies (typically, classes) from TensorFlow
+// checkpoints.  Note that the partitioning logic relies on contiguous vocabularies
+// corresponding to div-partitioned variables.  Moreover, the underlying remapping
+// uses an IndexTable (as opposed to an inexact CuckooTable), so client code should
+// use the corresponding index_table_from_file() as the FeatureColumn framework
+// does (as opposed to tf.feature_to_id(), which uses a CuckooTable).
 //
 // Arguments:
+//	new_vocab_file: Path to the new vocab file.
+//	old_vocab_file: Path to the old vocab file.
+//	new_vocab_offset: How many entries into the new vocab file to start reading.
+//	num_new_vocab: Number of entries in the new vocab file to remap.
 //
-//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
-//	num_partitions: The number of partitions to output.
-func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
+// Returns A Tensor of length num_new_vocab where the element at index i
+// is equal to the old ID that maps to the new ID i.  This element is -1 for any
+// new ID that is not found in the old vocabulary.Number of new vocab entries found in old vocab.
+func GenerateVocabRemapping(scope *Scope, new_vocab_file tf.Output, old_vocab_file tf.Output, new_vocab_offset int64, num_new_vocab int64, optional ...GenerateVocabRemappingAttr) (remapping tf.Output, num_present tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_partitions": num_partitions}
+	attrs := map[string]interface{}{"new_vocab_offset": new_vocab_offset, "num_new_vocab": num_new_vocab}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "DynamicPartition",
+		Type: "GenerateVocabRemapping",
 		Input: []tf.Input{
-			data, partitions,
+			new_vocab_file, old_vocab_file,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
-		scope.UpdateErr("DynamicPartition", err)
-		return
-	}
-	return outputs
+	return op.Output(0), op.Output(1)
 }
 
-// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad.
-type ResourceApplyAdagradAttr func(optionalAttr)
+// Assigns sparse updates to the variable referenced by `resource`.
+//
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] = updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] = updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] = updates[i, ..., j, ...]
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
+//
+// Returns the created operation.
+func ResourceScatterUpdate(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceScatterUpdate",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
 
-// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value.
+// AvgPoolGradAttr is an optional argument to AvgPoolGrad.
+type AvgPoolGradAttr func(optionalAttr)
+
+// AvgPoolGradDataFormat sets the optional data_format attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolGradDataFormat(value string) AvgPoolGradAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["data_format"] = value
 	}
 }
 
-// Update '*var' according to the adagrad scheme.
-//
-// accum += grad * grad
-// var -= lr * grad * (1 / sqrt(accum))
+// Computes gradients of the average pooling function.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	grad: The gradient.
+//	orig_input_shape: 1-D.  Shape of the original input to `avg_pool`.
+//	grad: 4-D with shape `[batch, height, width, channels]`.  Gradients w.r.t.
+// the output of `avg_pool`.
+//	ksize: The size of the sliding window for each dimension of the input.
+//	strides: The stride of the sliding window for each dimension of the input.
+//	padding: The type of padding algorithm to use.
 //
-// Returns the created operation.
-func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) {
+// Returns 4-D.  Gradients w.r.t. the input of `avg_pool`.
+func AvgPoolGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdagrad",
+		Type: "AvgPoolGrad",
 		Input: []tf.Input{
-			var_, accum, lr, grad,
+			orig_input_shape, grad,
 		},
 		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// StageClearAttr is an optional argument to StageClear.
+type StageClearAttr func(optionalAttr)
+
+// StageClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearCapacity(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// StageClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func StageClearMemoryLimit(value int64) StageClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// StageClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func StageClearContainer(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// StageClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func StageClearSharedName(value string) StageClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func StageClear(scope *Scope, dtypes []tf.DataType, optional ...StageClearAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "StageClear",
+
+		Attrs: attrs,
+	}
 	return scope.AddOperation(opspec)
 }
 
-// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
-type ResourceApplyPowerSignAttr func(optionalAttr)
+// ComputeAccidentalHitsAttr is an optional argument to ComputeAccidentalHits.
+type ComputeAccidentalHitsAttr func(optionalAttr)
 
-// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
+// ComputeAccidentalHitsSeed sets the optional seed attribute to value.
 //
-// value: If `True`, updating of the var and m tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed(value int64) ComputeAccidentalHitsAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["seed"] = value
 	}
 }
 
-// Update '*var' according to the AddSign update.
+// ComputeAccidentalHitsSeed2 sets the optional seed2 attribute to value.
 //
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
-// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
-// variable <- variable - lr_t * update
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ComputeAccidentalHitsSeed2(value int64) ComputeAccidentalHitsAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Computes the ids of the positions in sampled_candidates that match true_labels.
+//
+// When doing log-odds NCE, the result of this op should be passed through a
+// SparseToDense op, then added to the logits of the sampled candidates. This has
+// the effect of 'removing' the sampled labels that match the true labels by
+// making the classifier sure that they are sampled labels.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	logbase: Must be a scalar.
-//	sign_decay: Must be a scalar.
-//	beta: Must be a scalar.
-//	grad: The gradient.
+//	true_classes: The true_classes output of UnpackSparseLabels.
+//	sampled_candidates: The sampled_candidates output of CandidateSampler.
+//	num_true: Number of true labels per context.
 //
-// Returns the created operation.
-func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
+// Returns A vector of indices corresponding to rows of true_candidates.A vector of IDs of positions in sampled_candidates that match a true_label
+// for the row with the corresponding index in indices.A vector of the same length as indices and ids, in which each element
+// is -FLOAT_MAX.
+func ComputeAccidentalHits(scope *Scope, true_classes tf.Output, sampled_candidates tf.Output, num_true int64, optional ...ComputeAccidentalHitsAttr) (indices tf.Output, ids tf.Output, weights tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"num_true": num_true}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyPowerSign",
+		Type: "ComputeAccidentalHits",
 		Input: []tf.Input{
-			var_, m, lr, logbase, sign_decay, beta, grad,
+			true_classes, sampled_candidates,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// CumprodAttr is an optional argument to Cumprod.
-type CumprodAttr func(optionalAttr)
+// CumsumAttr is an optional argument to Cumsum.
+type CumsumAttr func(optionalAttr)
 
-// CumprodExclusive sets the optional exclusive attribute to value.
+// CumsumExclusive sets the optional exclusive attribute to value.
 //
-// value: If `True`, perform exclusive cumprod.
+// value: If `True`, perform exclusive cumsum.
 // If not specified, defaults to false
-func CumprodExclusive(value bool) CumprodAttr {
+func CumsumExclusive(value bool) CumsumAttr {
 	return func(m optionalAttr) {
 		m["exclusive"] = value
 	}
 }
 
-// CumprodReverse sets the optional reverse attribute to value.
+// CumsumReverse sets the optional reverse attribute to value.
 //
 // value: A `bool` (default: False).
 // If not specified, defaults to false
-func CumprodReverse(value bool) CumprodAttr {
+func CumsumReverse(value bool) CumsumAttr {
 	return func(m optionalAttr) {
 		m["reverse"] = value
 	}
 }
 
-// Compute the cumulative product of the tensor `x` along `axis`.
+// Compute the cumulative sum of the tensor `x` along `axis`.
 //
-// By default, this op performs an inclusive cumprod, which means that the first
+// By default, this op performs an inclusive cumsum, which means that the first
 // element of the input is identical to the first element of the output:
 //
 // ```python
-// tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
+// tf.cumsum([a, b, c])  # => [a, a + b, a + b + c]
 // ```
 //
-// By setting the `exclusive` kwarg to `True`, an exclusive cumprod is
+// By setting the `exclusive` kwarg to `True`, an exclusive cumsum is
 // performed instead:
 //
 // ```python
-// tf.cumprod([a, b, c], exclusive=True)  # => [1, a, a * b]
+// tf.cumsum([a, b, c], exclusive=True)  # => [0, a, a + b]
 // ```
 //
-// By setting the `reverse` kwarg to `True`, the cumprod is performed in the
+// By setting the `reverse` kwarg to `True`, the cumsum is performed in the
 // opposite direction:
 //
 // ```python
-// tf.cumprod([a, b, c], reverse=True)  # => [a * b * c, b * c, c]
+// tf.cumsum([a, b, c], reverse=True)  # => [a + b + c, b + c, c]
 // ```
 //
 // This is more efficient than using separate `tf.reverse` ops.
@@ -7614,7 +7526,7 @@ func CumprodReverse(value bool) CumprodAttr {
 // The `reverse` and `exclusive` kwargs can also be combined:
 //
 // ```python
-// tf.cumprod([a, b, c], exclusive=True, reverse=True)  # => [b * c, c, 1]
+// tf.cumsum([a, b, c], exclusive=True, reverse=True)  # => [b + c, c, 0]
 // ```
 //
 // Arguments:
@@ -7623,7 +7535,7 @@ func CumprodReverse(value bool) CumprodAttr {
 // `complex128`, `qint8`, `quint8`, `qint32`, `half`.
 //	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
 // `[-rank(x), rank(x))`.
-func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr) (out tf.Output) {
+func Cumsum(scope *Scope, x tf.Output, axis tf.Output, optional ...CumsumAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -7632,7 +7544,7 @@ func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Cumprod",
+		Type: "Cumsum",
 		Input: []tf.Input{
 			x, axis,
 		},
@@ -7642,125 +7554,157 @@ func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr)
 	return op.Output(0)
 }
 
-// Computes the mean along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// QuantizedRelu6Attr is an optional argument to QuantizedRelu6.
+type QuantizedRelu6Attr func(optionalAttr)
+
+// QuantizedRelu6OutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedRelu6OutType(value tf.DataType) QuantizedRelu6Attr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Computes Quantized Rectified Linear 6: `min(max(features, 0), 6)`
 //
-// Computes a tensor such that
-// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is
-// over `j` such that `segment_ids[j] == i` and `N` is the total number of
-// values summed.
+// Arguments:
 //
-// If the mean is empty for a given segment ID `i`, `output[i] = 0`.
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedRelu6(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedRelu6Attr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SegmentMean",
+		Type: "QuantizedRelu6",
 		Input: []tf.Input{
-			data, segment_ids,
+			features, min_features, max_features,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp.
-type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr)
+// FixedLengthRecordReaderV2Attr is an optional argument to FixedLengthRecordReaderV2.
+type FixedLengthRecordReaderV2Attr func(optionalAttr)
 
-// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
+// FixedLengthRecordReaderV2HeaderBytes sets the optional header_bytes attribute to value.
 //
-// value: If `True`, updating of the var, mg, ms, and mom tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr {
+// value: Number of bytes in the header, defaults to 0.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2HeaderBytes(value int64) FixedLengthRecordReaderV2Attr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["header_bytes"] = value
 	}
 }
 
-// Update '*var' according to the centered RMSProp algorithm.
+// FixedLengthRecordReaderV2FooterBytes sets the optional footer_bytes attribute to value.
 //
-// The centered RMSProp algorithm uses an estimate of the centered second moment
-// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
-// uses the (uncentered) second moment. This often helps with training, but is
-// slightly more expensive in terms of computation and memory.
+// value: Number of bytes in the footer, defaults to 0.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2FooterBytes(value int64) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["footer_bytes"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2HopBytes sets the optional hop_bytes attribute to value.
 //
-// Note that in dense implementation of this algorithm, mg, ms, and mom will
-// update even if the grad is zero, but in this sparse implementation, mg, ms,
-// and mom will not update in iterations during which the grad is zero.
+// value: Number of bytes to hop before each read. Default of 0 means using
+// record_bytes.
+// If not specified, defaults to 0
+func FixedLengthRecordReaderV2HopBytes(value int64) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["hop_bytes"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2Container sets the optional container attribute to value.
 //
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// mean_grad = decay * mean_grad + (1-decay) * gradient
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2Container(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2SharedName(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// FixedLengthRecordReaderV2Encoding sets the optional encoding attribute to value.
 //
-// Arguments:
-//	var_: Should be from a Variable().
-//	mg: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
+// value: The type of encoding for the file. Currently ZLIB and GZIP
+// are supported. Defaults to none.
+// If not specified, defaults to ""
+func FixedLengthRecordReaderV2Encoding(value string) FixedLengthRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["encoding"] = value
+	}
+}
+
+// A Reader that outputs fixed-length records from a file.
 //
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var, ms and mom.
+// Arguments:
+//	record_bytes: Number of bytes in the record.
 //
-// Returns the created operation.
-func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) {
+// Returns The handle to reference the Reader.
+func FixedLengthRecordReaderV2(scope *Scope, record_bytes int64, optional ...FixedLengthRecordReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"record_bytes": record_bytes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyCenteredRMSProp",
-		Input: []tf.Input{
-			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices,
-		},
+		Type: "FixedLengthRecordReaderV2",
+
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Creates a dataset that batches `batch_size` elements from `input_dataset`.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// Arguments:
+// The hash function is deterministic on the content of the string within the
+// process.
 //
-//	batch_size: A scalar representing the number of elements to accumulate in a
-// batch.
+// Note that the hash function may change from time to time.
+// This functionality will be deprecated and it's recommended to use
+// `tf.string_to_hash_bucket_fast()` or `tf.string_to_hash_bucket_strong()`.
 //
+// Arguments:
 //
-func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucket(scope *Scope, string_tensor tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "BatchDataset",
+		Type: "StringToHashBucket",
 		Input: []tf.Input{
-			input_dataset, batch_size,
+			string_tensor,
 		},
 		Attrs: attrs,
 	}
@@ -7768,232 +7712,301 @@ func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, o
 	return op.Output(0)
 }
 
-// Inverse fast Fourier transform.
-//
-// Computes the inverse 1-dimensional discrete Fourier transform over the
-// inner-most dimension of `input`.
+// Computes gradients for the exponential linear (Elu) operation.
 //
 // Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its inverse 1D Fourier transform.
+//	gradients: The backpropagated gradients to the corresponding Elu operation.
+//	outputs: The outputs of the corresponding Elu operation.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.ifft
-// @end_compatibility
-func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns The gradients: `gradients * (outputs + 1)` if outputs < 0,
+// `gradients` otherwise.
+func EluGrad(scope *Scope, gradients tf.Output, outputs tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IFFT",
+		Type: "EluGrad",
 		Input: []tf.Input{
-			input,
+			gradients, outputs,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Generates values in an interval.
-//
-// A sequence of `num` evenly-spaced values are generated beginning at `start`.
-// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
-// so that the last one is exactly `stop`.
+// Creates a dataset that contains `count` elements from the `input_dataset`.
 //
-// For example:
+// Arguments:
 //
-// ```
-// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
-// ```
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be taken. A value of `-1` indicates that all of `input_dataset`
+// is taken.
 //
-// Arguments:
-//	start: First entry in the range.
-//	stop: Last entry in the range.
-//	num: Number of values to generate.
 //
-// Returns 1-D. The generated values.
-func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
+func TakeDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "LinSpace",
+		Type: "TakeDataset",
 		Input: []tf.Input{
-			start, stop, num,
+			input_dataset, count,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
-type DestroyResourceOpAttr func(optionalAttr)
-
-// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
-//
-// value: whether to ignore the error when the resource
-// doesn't exist.
-// If not specified, defaults to true
-func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
-	return func(m optionalAttr) {
-		m["ignore_lookup_error"] = value
-	}
-}
-
-// Deletes the resource specified by the handle.
+// The gradient operator for the SparseAdd op.
 //
-// All subsequent operations using the resource will result in a NotFound
-// error status.
+// The SparseAdd op calculates A + B, where A, B, and the sum are all represented
+// as `SparseTensor` objects.  This op takes in the upstream gradient w.r.t.
+// non-empty values of the sum, and outputs the gradients w.r.t. the non-empty
+// values of A and B.
 //
 // Arguments:
-//	resource: handle to the resource to delete.
+//	backprop_val_grad: 1-D with shape `[nnz(sum)]`.  The gradient with respect to
+// the non-empty values of the sum.
+//	a_indices: 2-D.  The `indices` of the `SparseTensor` A, size `[nnz(A), ndims]`.
+//	b_indices: 2-D.  The `indices` of the `SparseTensor` B, size `[nnz(B), ndims]`.
+//	sum_indices: 2-D.  The `indices` of the sum `SparseTensor`, size
+// `[nnz(sum), ndims]`.
 //
-// Returns the created operation.
-func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
+// Returns 1-D with shape `[nnz(A)]`. The gradient with respect to the
+// non-empty values of A.1-D with shape `[nnz(B)]`. The gradient with respect to the
+// non-empty values of B.
+func SparseAddGrad(scope *Scope, backprop_val_grad tf.Output, a_indices tf.Output, b_indices tf.Output, sum_indices tf.Output) (a_val_grad tf.Output, b_val_grad tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DestroyResourceOp",
+		Type: "SparseAddGrad",
 		Input: []tf.Input{
-			resource,
+			backprop_val_grad, a_indices, b_indices, sum_indices,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// LRNAttr is an optional argument to LRN.
-type LRNAttr func(optionalAttr)
-
-// LRNDepthRadius sets the optional depth_radius attribute to value.
-//
-// value: 0-D.  Half-width of the 1-D normalization window.
-// If not specified, defaults to 5
-func LRNDepthRadius(value int64) LRNAttr {
-	return func(m optionalAttr) {
-		m["depth_radius"] = value
+// Computes atan of x element-wise.
+func Atan(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// LRNBias sets the optional bias attribute to value.
-//
-// value: An offset (usually positive to avoid dividing by 0).
-// If not specified, defaults to 1
-func LRNBias(value float32) LRNAttr {
-	return func(m optionalAttr) {
-		m["bias"] = value
+	opspec := tf.OpSpec{
+		Type: "Atan",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// LRNAlpha sets the optional alpha attribute to value.
+// Encode audio data using the WAV file format.
 //
-// value: A scale factor, usually positive.
-// If not specified, defaults to 1
-func LRNAlpha(value float32) LRNAttr {
-	return func(m optionalAttr) {
-		m["alpha"] = value
+// This operation will generate a string suitable to be saved out to create a .wav
+// audio file. It will be encoded in the 16-bit PCM format. It takes in float
+// values in the range -1.0f to 1.0f, and any outside that value will be clamped to
+// that range.
+//
+// `audio` is a 2-D float Tensor of shape `[length, channels]`.
+// `sample_rate` is a scalar Tensor holding the rate to use (e.g. 44100).
+//
+// Arguments:
+//	audio: 2-D with shape `[length, channels]`.
+//	sample_rate: Scalar containing the sample frequency.
+//
+// Returns 0-D. WAV-encoded file contents.
+func EncodeWav(scope *Scope, audio tf.Output, sample_rate tf.Output) (contents tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "EncodeWav",
+		Input: []tf.Input{
+			audio, sample_rate,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// LRNBeta sets the optional beta attribute to value.
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// value: An exponent.
-// If not specified, defaults to 0.5
-func LRNBeta(value float32) LRNAttr {
-	return func(m optionalAttr) {
-		m["beta"] = value
+// The hash function is deterministic on the content of the string within the
+// process. The hash function is a keyed hash function, where attribute `key`
+// defines the key of the hash function. `key` is an array of 2 elements.
+//
+// A strong hash is important when inputs may be malicious, e.g. URLs with
+// additional components. Adversaries could try to make their inputs hash to the
+// same bucket for a denial-of-service attack or to skew the results. A strong
+// hash prevents this by making it difficult, if not infeasible, to compute inputs
+// that hash to the same bucket. This comes at a cost of roughly 4x higher compute
+// time than `tf.string_to_hash_bucket_fast`.
+//
+// Arguments:
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
+//	key: The key for the keyed hash function passed as a list of two uint64
+// elements.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketStrong(scope *Scope, input tf.Output, num_buckets int64, key []int64) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets, "key": key}
+	opspec := tf.OpSpec{
+		Type: "StringToHashBucketStrong",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Local Response Normalization.
+// Applies softmax to a batched N-D `SparseTensor`.
 //
-// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last
-// dimension), and each vector is normalized independently.  Within a given vector,
-// each component is divided by the weighted, squared sum of inputs within
-// `depth_radius`.  In detail,
+// The inputs represent an N-D SparseTensor  with logical shape `[..., B, C]`
+// (where `N >= 2`), and with indices sorted in the canonical lexicographic order.
 //
-//     sqr_sum[a, b, c, d] =
-//         sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2)
-//     output = input / (bias + alpha * sqr_sum) ** beta
+// This op is equivalent to applying the normal `tf.nn.softmax()` to each innermost
+// logical submatrix with shape `[B, C]`, but with the catch that *the implicitly
+// zero elements do not participate*.  Specifically, the algorithm is equivalent
+// to the following:
 //
-// For details, see [Krizhevsky et al., ImageNet classification with deep
-// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks).
+//   (1) Applies `tf.nn.softmax()` to a densified view of each innermost submatrix
+//       with shape `[B, C]`, along the size-C dimension;
+//   (2) Masks out the original implicitly-zero locations;
+//   (3) Renormalizes the remaining elements.
+//
+// Hence, the `SparseTensor` result has exactly the same non-zero indices and
+// shape.
 //
 // Arguments:
-//	input: 4-D.
-func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) {
+//	sp_indices: 2-D.  `NNZ x R` matrix with the indices of non-empty values in a
+// SparseTensor, in canonical ordering.
+//	sp_values: 1-D.  `NNZ` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//
+// Returns 1-D.  The `NNZ` values for the result `SparseTensor`.
+func SparseSoftmax(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "LRN",
+		Type: "SparseSoftmax",
 		Input: []tf.Input{
-			input,
+			sp_indices, sp_values, sp_shape,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Creates a dataset that zips together `input_datasets`.
-func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Partitions `data` into `num_partitions` tensors using indices from `partitions`.
+//
+// For each index tuple `js` of size `partitions.ndim`, the slice `data[js, ...]`
+// becomes part of `outputs[partitions[js]]`.  The slices with `partitions[js] = i`
+// are placed in `outputs[i]` in lexicographic order of `js`, and the first
+// dimension of `outputs[i]` is the number of entries in `partitions` equal to `i`.
+// In detail,
+//
+// ```python
+//     outputs[i].shape = [sum(partitions == i)] + data.shape[partitions.ndim:]
+//
+//     outputs[i] = pack([data[js, ...] for js if partitions[js] == i])
+// ```
+//
+// `data.shape` must start with `partitions.shape`.
+//
+// For example:
+//
+// ```python
+//     # Scalar partitions.
+//     partitions = 1
+//     num_partitions = 2
+//     data = [10, 20]
+//     outputs[0] = []  # Empty with shape [0, 2]
+//     outputs[1] = [[10, 20]]
+//
+//     # Vector partitions.
+//     partitions = [0, 0, 1, 1, 0]
+//     num_partitions = 2
+//     data = [10, 20, 30, 40, 50]
+//     outputs[0] = [10, 20, 50]
+//     outputs[1] = [30, 40]
+// ```
+//
+// See `dynamic_stitch` for an example on how to merge partitions back.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicPartition.png" alt>
+// </div>
+//
+// Arguments:
+//
+//	partitions: Any shape.  Indices in the range `[0, num_partitions)`.
+//	num_partitions: The number of partitions to output.
+func DynamicPartition(scope *Scope, data tf.Output, partitions tf.Output, num_partitions int64) (outputs []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{"num_partitions": num_partitions}
 	opspec := tf.OpSpec{
-		Type: "ZipDataset",
+		Type: "DynamicPartition",
 		Input: []tf.Input{
-			tf.OutputList(input_datasets),
+			data, partitions,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
+		scope.UpdateErr("DynamicPartition", err)
+		return
+	}
+	return outputs
 }
 
-// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
-type ResourceSparseApplyAdagradAttr func(optionalAttr)
+// ResourceApplyAdagradAttr is an optional argument to ResourceApplyAdagrad.
+type ResourceApplyAdagradAttr func(optionalAttr)
 
-// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
+// ResourceApplyAdagradUseLocking sets the optional use_locking attribute to value.
 //
 // value: If `True`, updating of the var and accum tensors will be protected
 // by a lock; otherwise the behavior is undefined, but may exhibit less
 // contention.
 // If not specified, defaults to false
-func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
+func ResourceApplyAdagradUseLocking(value bool) ResourceApplyAdagradAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
+// Update '*var' according to the adagrad scheme.
 //
-// That is for rows we have grad for, we update var and accum as follows:
 // accum += grad * grad
 // var -= lr * grad * (1 / sqrt(accum))
 //
 // Arguments:
 //	var_: Should be from a Variable().
 //	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
 //	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
+func ResourceApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, optional ...ResourceApplyAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8002,87 +8015,66 @@ func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, l
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagrad",
+		Type: "ResourceApplyAdagrad",
 		Input: []tf.Input{
-			var_, accum, lr, grad, indices,
+			var_, accum, lr, grad,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// 2D real-valued fast Fourier transform.
-//
-// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most 2 dimensions of `input`.
-//
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
-// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
-// of `output`: the zero-frequency term, followed by the `fft_length / 2`
-// positive-frequency terms.
-//
-// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
-//
-// Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
-//
-// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their 2D Fourier transform. The
-//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
-//   components.
+// Return the shape of s0 op s1 with broadcast.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.rfft2
-// @end_compatibility
-func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
+// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
+func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RFFT2D",
+		Type: "BroadcastArgs",
 		Input: []tf.Input{
-			input, fft_length,
+			s0, s1,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeAreaAttr is an optional argument to ResizeArea.
-type ResizeAreaAttr func(optionalAttr)
+// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
+type DataFormatDimMapAttr func(optionalAttr)
 
-// ResizeAreaAlignCorners sets the optional align_corners attribute to value.
+// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeAreaAlignCorners(value bool) ResizeAreaAttr {
+// value: source data format.
+// If not specified, defaults to "NHWC"
+func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["src_format"] = value
 	}
 }
 
-// Resize `images` to `size` using area interpolation.
+// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
 //
-// Input images can be of different types but output images are always float.
+// value: destination data format.
+// If not specified, defaults to "NCHW"
+func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+	return func(m optionalAttr) {
+		m["dst_format"] = value
+	}
+}
+
+// Returns the dimension index in the destination data format given the one in
 //
-// Each output pixel is computed by first transforming the pixel's footprint into
-// the input tensor and then averaging the pixels that intersect the footprint. An
-// input pixel's contribution to the average is weighted by the fraction of its
-// area that intersects the footprint.  This is the same as OpenCV's INTER_AREA.
+// the source data format.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	x: A Tensor with each element as a dimension index in source data format.
+// Must be in the range [-4, 4).
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) {
+// Returns A Tensor with each element as a dimension index in destination data format.
+func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8091,9 +8083,9 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeArea",
+		Type: "DataFormatDimMap",
 		Input: []tf.Input{
-			images, size,
+			x,
 		},
 		Attrs: attrs,
 	}
@@ -8101,91 +8093,116 @@ func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...Resi
 	return op.Output(0)
 }
 
-// Pads a tensor with zeros.
-//
-// This operation pads a `input` with zeros according to the `paddings` you
-// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
-// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
-// how many zeros to add before the contents of `input` in that dimension, and
-// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
-// in that dimension.
-//
-// The padded size of each dimension D of the output is:
-//
-// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
-//
-// For example:
+// ResourceApplyPowerSignAttr is an optional argument to ResourceApplyPowerSign.
+type ResourceApplyPowerSignAttr func(optionalAttr)
+
+// ResourceApplyPowerSignUseLocking sets the optional use_locking attribute to value.
 //
-// ```
-// # 't' is [[1, 1], [2, 2]]
-// # 'paddings' is [[1, 1], [2, 2]]
-// # rank of 't' is 2
-// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
-//                       [0, 0, 1, 1, 0, 0]
-//                       [0, 0, 2, 2, 0, 0]
-//                       [0, 0, 0, 0, 0, 0]]
-// ```
-func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Pad",
-		Input: []tf.Input{
-			input, paddings,
-		},
+// value: If `True`, updating of the var and m tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyPowerSignUseLocking(value bool) ResourceApplyPowerSignAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Checks whether a resource handle-based variable has been initialized.
+// Update '*var' according to the AddSign update.
+//
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// update <- exp(logbase * sign_decay * sign(g) * sign(m_t)) * g
+// variable <- variable - lr_t * update
 //
 // Arguments:
-//	resource: the input resource handle.
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	logbase: Must be a scalar.
+//	sign_decay: Must be a scalar.
+//	beta: Must be a scalar.
+//	grad: The gradient.
 //
-// Returns a scalar boolean which is true if the variable has been
-// initialized.
-func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
+// Returns the created operation.
+func ResourceApplyPowerSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, logbase tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyPowerSignAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "VarIsInitializedOp",
+		Type: "ResourceApplyPowerSign",
 		Input: []tf.Input{
-			resource,
+			var_, m, lr, logbase, sign_decay, beta, grad,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
-type StatelessRandomUniformAttr func(optionalAttr)
+// CumprodAttr is an optional argument to Cumprod.
+type CumprodAttr func(optionalAttr)
 
-// StatelessRandomUniformDtype sets the optional dtype attribute to value.
+// CumprodExclusive sets the optional exclusive attribute to value.
 //
-// value: The type of the output.
-// If not specified, defaults to DT_FLOAT
-func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
+// value: If `True`, perform exclusive cumprod.
+// If not specified, defaults to false
+func CumprodExclusive(value bool) CumprodAttr {
 	return func(m optionalAttr) {
-		m["dtype"] = value
+		m["exclusive"] = value
 	}
 }
 
-// Outputs deterministic pseudorandom random values from a uniform distribution.
+// CumprodReverse sets the optional reverse attribute to value.
 //
-// The generated values follow a uniform distribution in the range `[0, 1)`. The
-// lower bound 0 is included in the range, while the upper bound 1 is excluded.
+// value: A `bool` (default: False).
+// If not specified, defaults to false
+func CumprodReverse(value bool) CumprodAttr {
+	return func(m optionalAttr) {
+		m["reverse"] = value
+	}
+}
+
+// Compute the cumulative product of the tensor `x` along `axis`.
 //
-// The outputs are a deterministic function of `shape` and `seed`.
+// By default, this op performs an inclusive cumprod, which means that the first
+// element of the input is identical to the first element of the output:
 //
-// Arguments:
-//	shape: The shape of the output tensor.
-//	seed: 2 seeds (shape [2]).
+// ```python
+// tf.cumprod([a, b, c])  # => [a, a * b, a * b * c]
+// ```
 //
-// Returns Random values with specified shape.
-func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) {
+// By setting the `exclusive` kwarg to `True`, an exclusive cumprod is
+// performed instead:
+//
+// ```python
+// tf.cumprod([a, b, c], exclusive=True)  # => [1, a, a * b]
+// ```
+//
+// By setting the `reverse` kwarg to `True`, the cumprod is performed in the
+// opposite direction:
+//
+// ```python
+// tf.cumprod([a, b, c], reverse=True)  # => [a * b * c, b * c, c]
+// ```
+//
+// This is more efficient than using separate `tf.reverse` ops.
+//
+// The `reverse` and `exclusive` kwargs can also be combined:
+//
+// ```python
+// tf.cumprod([a, b, c], exclusive=True, reverse=True)  # => [b * c, c, 1]
+// ```
+//
+// Arguments:
+//	x: A `Tensor`. Must be one of the following types: `float32`, `float64`,
+// `int64`, `int32`, `uint8`, `uint16`, `int16`, `int8`, `complex64`,
+// `complex128`, `qint8`, `quint8`, `qint32`, `half`.
+//	axis: A `Tensor` of type `int32` (default: 0). Must be in the range
+// `[-rank(x), rank(x))`.
+func Cumprod(scope *Scope, x tf.Output, axis tf.Output, optional ...CumprodAttr) (out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8194,9 +8211,9 @@ func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optio
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StatelessRandomUniform",
+		Type: "Cumprod",
 		Input: []tf.Input{
-			shape, seed,
+			x, axis,
 		},
 		Attrs: attrs,
 	}
@@ -8204,69 +8221,91 @@ func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optio
 	return op.Output(0)
 }
 
-// Makes its input available to the next iteration.
+// Computes the mean along segments of a tensor.
+//
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
+//
+// Computes a tensor such that
+// \\(output_i = \frac{\sum_j data_j}{N}\\) where `mean` is
+// over `j` such that `segment_ids[j] == i` and `N` is the total number of
+// values summed.
+//
+// If the mean is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMean.png" alt>
+// </div>
 //
 // Arguments:
-//	data: The tensor to be made available to the next iteration.
 //
-// Returns The same tensor as `data`.
-func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMean(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "NextIteration",
+		Type: "SegmentMean",
 		Input: []tf.Input{
-			data,
+			data, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Output a fact about factorials.
-func Fact(scope *Scope) (fact tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Fact",
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// AngleAttr is an optional argument to Angle.
-type AngleAttr func(optionalAttr)
+// ResourceSparseApplyCenteredRMSPropAttr is an optional argument to ResourceSparseApplyCenteredRMSProp.
+type ResourceSparseApplyCenteredRMSPropAttr func(optionalAttr)
 
-// AngleTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func AngleTout(value tf.DataType) AngleAttr {
+// ResourceSparseApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, mg, ms, and mom tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyCenteredRMSPropUseLocking(value bool) ResourceSparseApplyCenteredRMSPropAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Returns the argument of a complex number.
+// Update '*var' according to the centered RMSProp algorithm.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the argument of each element in `input`. All elements in
-// `input` must be complex numbers of the form \\(a + bj\\), where *a*
-// is the real part and *b* is the imaginary part.
+// The centered RMSProp algorithm uses an estimate of the centered second moment
+// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
+// uses the (uncentered) second moment. This often helps with training, but is
+// slightly more expensive in terms of computation and memory.
 //
-// The argument returned by this operation is of the form \\(atan2(b, a)\\).
+// Note that in dense implementation of this algorithm, mg, ms, and mom will
+// update even if the grad is zero, but in this sparse implementation, mg, ms,
+// and mom will not update in iterations during which the grad is zero.
 //
-// For example:
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// mean_grad = decay * mean_grad + (1-decay) * gradient
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.angle(input) ==> [2.0132, 1.056]
-// ```
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
 //
-// @compatibility(numpy)
-// Equivalent to np.angle.
-// @end_compatibility
-func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) {
+// Arguments:
+//	var_: Should be from a Variable().
+//	mg: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
+//
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var, ms and mom.
+//
+// Returns the created operation.
+func ResourceSparseApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyCenteredRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8275,184 +8314,123 @@ func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Angle",
+		Type: "ResourceSparseApplyCenteredRMSProp",
 		Input: []tf.Input{
-			input,
+			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad, indices,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// VarHandleOpAttr is an optional argument to VarHandleOp.
-type VarHandleOpAttr func(optionalAttr)
-
-// VarHandleOpContainer sets the optional container attribute to value.
+// Creates a dataset that batches `batch_size` elements from `input_dataset`.
 //
-// value: the container this variable is placed in.
-// If not specified, defaults to ""
-func VarHandleOpContainer(value string) VarHandleOpAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// VarHandleOpSharedName sets the optional shared_name attribute to value.
+// Arguments:
 //
-// value: the name by which this variable is referred to.
-// If not specified, defaults to ""
-func VarHandleOpSharedName(value string) VarHandleOpAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Creates a handle to a Variable resource.
+//	batch_size: A scalar representing the number of elements to accumulate in a
+// batch.
 //
-// Arguments:
-//	dtype: the type of this variable. Must agree with the dtypes
-// of all ops using this variable.
-//	shape: The (possibly partially specified) shape of this variable.
-func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
+//
+func BatchDataset(scope *Scope, input_dataset tf.Output, batch_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "VarHandleOp",
-
+		Type: "BatchDataset",
+		Input: []tf.Input{
+			input_dataset, batch_size,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Elementwise computes the bitwise XOR of `x` and `y`.
+// Inverse fast Fourier transform.
 //
-// The result will have those bits set, that are different in `x` and `y`. The
-// computation is performed on the underlying representations of `x` and `y`.
-func BitwiseXor(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Computes the inverse 1-dimensional discrete Fourier transform over the
+// inner-most dimension of `input`.
+//
+// Arguments:
+//	input: A complex64 tensor.
+//
+// Returns A complex64 tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its inverse 1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifft
+// @end_compatibility
+func IFFT(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BitwiseXor",
+		Type: "IFFT",
 		Input: []tf.Input{
-			x, y,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Deserialize `SparseTensor` objects.
-//
-// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where
-// the last dimension stores serialized `SparseTensor` objects and the other N
-// dimensions (N >= 0) correspond to a batch. The ranks of the original
-// `SparseTensor` objects must all match. When the final `SparseTensor` is
-// created, its rank is the rank of the incoming `SparseTensor` objects plus N;
-// the sparse tensors have been concatenated along new dimensions, one for each
-// batch.
-//
-// The output `SparseTensor` object's shape values for the original dimensions
-// are the max across the input `SparseTensor` objects' shape values for the
-// corresponding dimensions. The new dimensions match the size of the batch.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in
-// standard lexicographic order.  If this is not the case, after this
-// step run `SparseReorder` to restore index ordering.
-//
-// For example, if the serialized input is a `[2 x 3]` matrix representing two
-// original `SparseTensor` objects:
-//
-//     index = [ 0]
-//             [10]
-//             [20]
-//     values = [1, 2, 3]
-//     shape = [50]
-//
-// and
+// Generates values in an interval.
 //
-//     index = [ 2]
-//             [10]
-//     values = [4, 5]
-//     shape = [30]
+// A sequence of `num` evenly-spaced values are generated beginning at `start`.
+// If `num > 1`, the values in the sequence increase by `stop - start / num - 1`,
+// so that the last one is exactly `stop`.
 //
-// then the final deserialized `SparseTensor` will be:
+// For example:
 //
-//     index = [0  0]
-//             [0 10]
-//             [0 20]
-//             [1  2]
-//             [1 10]
-//     values = [1, 2, 3, 4, 5]
-//     shape = [2 50]
+// ```
+// tf.linspace(10.0, 12.0, 3, name="linspace") => [ 10.0  11.0  12.0]
+// ```
 //
 // Arguments:
-//	serialized_sparse: The serialized `SparseTensor` objects. The last dimension
-// must have 3 columns.
-//	dtype: The `dtype` of the serialized `SparseTensor` objects.
-func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+//	start: First entry in the range.
+//	stop: Last entry in the range.
+//	num: Number of values to generate.
+//
+// Returns 1-D. The generated values.
+func LinSpace(scope *Scope, start tf.Output, stop tf.Output, num tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "DeserializeSparse",
+		Type: "LinSpace",
 		Input: []tf.Input{
-			serialized_sparse,
+			start, stop, num,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
-type ResourceApplyRMSPropAttr func(optionalAttr)
+// DestroyResourceOpAttr is an optional argument to DestroyResourceOp.
+type DestroyResourceOpAttr func(optionalAttr)
 
-// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
+// DestroyResourceOpIgnoreLookupError sets the optional ignore_lookup_error attribute to value.
 //
-// value: If `True`, updating of the var, ms, and mom tensors is protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
+// value: whether to ignore the error when the resource
+// doesn't exist.
+// If not specified, defaults to true
+func DestroyResourceOpIgnoreLookupError(value bool) DestroyResourceOpAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["ignore_lookup_error"] = value
 	}
 }
 
-// Update '*var' according to the RMSProp algorithm.
-//
-// Note that in dense implementation of this algorithm, ms and mom will
-// update even if the grad is zero, but in this sparse implementation, ms
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+// Deletes the resource specified by the handle.
 //
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
+// All subsequent operations using the resource will result in a NotFound
+// error status.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
-//
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
+//	resource: handle to the resource to delete.
 //
 // Returns the created operation.
-func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
+func DestroyResourceOp(scope *Scope, resource tf.Output, optional ...DestroyResourceOpAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8461,38 +8439,75 @@ func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Out
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyRMSProp",
+		Type: "DestroyResourceOp",
 		Input: []tf.Input{
-			var_, ms, mom, lr, rho, momentum, epsilon, grad,
+			resource,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// SizeAttr is an optional argument to Size.
-type SizeAttr func(optionalAttr)
+// LRNAttr is an optional argument to LRN.
+type LRNAttr func(optionalAttr)
 
-// SizeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func SizeOutType(value tf.DataType) SizeAttr {
+// LRNDepthRadius sets the optional depth_radius attribute to value.
+//
+// value: 0-D.  Half-width of the 1-D normalization window.
+// If not specified, defaults to 5
+func LRNDepthRadius(value int64) LRNAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["depth_radius"] = value
 	}
 }
 
-// Returns the size of a tensor.
+// LRNBias sets the optional bias attribute to value.
 //
-// This operation returns an integer representing the number of elements in
-// `input`.
+// value: An offset (usually positive to avoid dividing by 0).
+// If not specified, defaults to 1
+func LRNBias(value float32) LRNAttr {
+	return func(m optionalAttr) {
+		m["bias"] = value
+	}
+}
+
+// LRNAlpha sets the optional alpha attribute to value.
 //
-// For example:
+// value: A scale factor, usually positive.
+// If not specified, defaults to 1
+func LRNAlpha(value float32) LRNAttr {
+	return func(m optionalAttr) {
+		m["alpha"] = value
+	}
+}
+
+// LRNBeta sets the optional beta attribute to value.
 //
-// ```
-// # 't' is [[[1, 1,, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]]
-// size(t) ==> 12
-// ```
-func Size(scope *Scope, input tf.Output, optional ...SizeAttr) (output tf.Output) {
+// value: An exponent.
+// If not specified, defaults to 0.5
+func LRNBeta(value float32) LRNAttr {
+	return func(m optionalAttr) {
+		m["beta"] = value
+	}
+}
+
+// Local Response Normalization.
+//
+// The 4-D `input` tensor is treated as a 3-D array of 1-D vectors (along the last
+// dimension), and each vector is normalized independently.  Within a given vector,
+// each component is divided by the weighted, squared sum of inputs within
+// `depth_radius`.  In detail,
+//
+//     sqr_sum[a, b, c, d] =
+//         sum(input[a, b, c, d - depth_radius : d + depth_radius + 1] ** 2)
+//     output = input / (bias + alpha * sqr_sum) ** beta
+//
+// For details, see [Krizhevsky et al., ImageNet classification with deep
+// convolutional neural networks (NIPS 2012)](http://papers.nips.cc/paper/4824-imagenet-classification-with-deep-convolutional-neural-networks).
+//
+// Arguments:
+//	input: 4-D.
+func LRN(scope *Scope, input tf.Output, optional ...LRNAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8501,7 +8516,7 @@ func Size(scope *Scope, input tf.Output, optional ...SizeAttr) (output tf.Output
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Size",
+		Type: "LRN",
 		Input: []tf.Input{
 			input,
 		},
@@ -8511,68 +8526,53 @@ func Size(scope *Scope, input tf.Output, optional ...SizeAttr) (output tf.Output
 	return op.Output(0)
 }
 
-// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate.
-type ResourceScatterNdUpdateAttr func(optionalAttr)
+// Creates a dataset that zips together `input_datasets`.
+func ZipDataset(scope *Scope, input_datasets []tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "ZipDataset",
+		Input: []tf.Input{
+			tf.OutputList(input_datasets),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyAdagradAttr is an optional argument to ResourceSparseApplyAdagrad.
+type ResourceSparseApplyAdagradAttr func(optionalAttr)
+
+// ResourceSparseApplyAdagradUseLocking sets the optional use_locking attribute to value.
 //
-// value: An optional bool. Defaults to True. If True, the assignment will
-// be protected by a lock; otherwise the behavior is undefined,
-// but may exhibit less contention.
-// If not specified, defaults to true
-func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyAdagradUseLocking(value bool) ResourceSparseApplyAdagradAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Applies sparse `updates` to individual values or slices within a given
-//
-// variable according to `indices`.
-//
-// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
-//
-// `indices` must be integer tensor, containing indices into `ref`.
-// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
-//
-// The innermost dimension of `indices` (with length `K`) corresponds to
-// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
-// dimension of `ref`.
-//
-// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
-//
-// ```
-// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
-// ```
-//
-// For example, say we want to update 4 scattered elements to a rank-1 tensor to
-// 8 elements. In Python, that update would look like this:
-//
-// ```python
-//     ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8])
-//     indices = tf.constant([[4], [3], [1] ,[7]])
-//     updates = tf.constant([9, 10, 11, 12])
-//     update = tf.scatter_nd_update(ref, indices, updates)
-//     with tf.Session() as sess:
-//       print sess.run(update)
-// ```
-//
-// The resulting update to ref would look like this:
-//
-//     [1, 11, 3, 10, 9, 6, 7, 12]
+// Update relevant entries in '*var' and '*accum' according to the adagrad scheme.
 //
-// See @{tf.scatter_nd} for more details about how to make updates to
-// slices.
+// That is for rows we have grad for, we update var and accum as follows:
+// accum += grad * grad
+// var -= lr * grad * (1 / sqrt(accum))
 //
 // Arguments:
-//	ref: A resource handle. Must be from a VarHandleOp.
-//	indices: A Tensor. Must be one of the following types: int32, int64.
-// A tensor of indices into ref.
-//	updates: A Tensor. Must have the same type as ref. A tensor of updated
-// values to add to ref.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Learning rate. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
+func ResourceSparseApplyAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8581,109 +8581,87 @@ func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, upd
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterNdUpdate",
+		Type: "ResourceSparseApplyAdagrad",
 		Input: []tf.Input{
-			ref, indices, updates,
+			var_, accum, lr, grad, indices,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// SqueezeAttr is an optional argument to Squeeze.
-type SqueezeAttr func(optionalAttr)
-
-// SqueezeAxis sets the optional axis attribute to value.
-//
-// value: If specified, only squeezes the dimensions listed. The dimension
-// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
-// be in the range `[-rank(input), rank(input))`.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func SqueezeAxis(value []int64) SqueezeAttr {
-	return func(m optionalAttr) {
-		m["squeeze_dims"] = value
-	}
-}
-
-// Removes dimensions of size 1 from the shape of a tensor.
-//
-// Given a tensor `input`, this operation returns a tensor of the same type with
-// all dimensions of size 1 removed. If you don't want to remove all size 1
-// dimensions, you can remove specific size 1 dimensions by specifying
-// `axis`.
-//
-// For example:
+// 2D real-valued fast Fourier transform.
 //
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t)) ==> [2, 3]
-// ```
+// Computes the 2-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most 2 dimensions of `input`.
 //
-// Or, to remove specific size 1 dimensions:
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT2D` only returns the
+// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+// of `output`: the zero-frequency term, followed by the `fft_length / 2`
+// positive-frequency terms.
 //
-// ```
-// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
-// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
-// ```
+// Along each axis `RFFT2D` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
 //
 // Arguments:
-//	input: The `input` to squeeze.
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
 //
-// Returns Contains the same data as `input`, but has one or more dimensions of
-// size 1 removed.
-func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
+// Returns A complex64 tensor of the same rank as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their 2D Fourier transform. The
+//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
+//   components.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfft2
+// @end_compatibility
+func RFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Squeeze",
+		Type: "RFFT2D",
 		Input: []tf.Input{
-			input,
+			input, fft_length,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
-type ResourceApplyAdadeltaAttr func(optionalAttr)
+// ResizeAreaAttr is an optional argument to ResizeArea.
+type ResizeAreaAttr func(optionalAttr)
 
-// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
+// ResizeAreaAlignCorners sets the optional align_corners attribute to value.
 //
-// value: If True, updating of the var, accum and update_accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
 // If not specified, defaults to false
-func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
+func ResizeAreaAlignCorners(value bool) ResizeAreaAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Update '*var' according to the adadelta scheme.
+// Resize `images` to `size` using area interpolation.
 //
-// accum = rho() * accum + (1 - rho()) * grad.square();
-// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
-// update_accum = rho() * update_accum + (1 - rho()) * update.square();
-// var -= update;
+// Input images can be of different types but output images are always float.
+//
+// Each output pixel is computed by first transforming the pixel's footprint into
+// the input tensor and then averaging the pixels that intersect the footprint. An
+// input pixel's contribution to the average is weighted by the fraction of its
+// area that intersects the footprint.  This is the same as OpenCV's INTER_AREA.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	accum_update: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay factor. Must be a scalar.
-//	epsilon: Constant factor. Must be a scalar.
-//	grad: The gradient.
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// Returns the created operation.
-func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeArea(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeAreaAttr) (resized_images tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8692,165 +8670,101 @@ func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdadelta",
+		Type: "ResizeArea",
 		Input: []tf.Input{
-			var_, accum, accum_update, lr, rho, epsilon, grad,
+			images, size,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression.
-type NonMaxSuppressionAttr func(optionalAttr)
-
-// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value.
-//
-// value: A float representing the threshold for deciding whether boxes
-// overlap too much with respect to IOU.
-// If not specified, defaults to 0.5
-func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr {
-	return func(m optionalAttr) {
-		m["iou_threshold"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Greedily selects a subset of bounding boxes in descending order of score,
+// Pads a tensor with zeros.
 //
-// pruning away boxes that have high intersection-over-union (IOU) overlap
-// with previously selected boxes.  Bounding boxes are supplied as
-// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
-// diagonal pair of box corners and the coordinates can be provided as normalized
-// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
-// is agnostic to where the origin is in the coordinate system.  Note that this
-// algorithm is invariant to orthogonal transformations and translations
-// of the coordinate system; thus translating or reflections of the coordinate
-// system result in the same boxes being selected by the algorithm.
-// The output of this operation is a set of integers indexing into the input
-// collection of bounding boxes representing the selected boxes.  The bounding
-// box coordinates corresponding to the selected indices can then be obtained
-// using the `tf.gather operation`.  For example:
-//   selected_indices = tf.image.non_max_suppression(
-//       boxes, scores, max_output_size, iou_threshold)
-//   selected_boxes = tf.gather(boxes, selected_indices)
+// This operation pads a `input` with zeros according to the `paddings` you
+// specify. `paddings` is an integer tensor with shape `[Dn, 2]`, where n is the
+// rank of `input`. For each dimension D of `input`, `paddings[D, 0]` indicates
+// how many zeros to add before the contents of `input` in that dimension, and
+// `paddings[D, 1]` indicates how many zeros to add after the contents of `input`
+// in that dimension.
 //
-// Arguments:
-//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
-//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
-// score corresponding to each box (each row of boxes).
-//	max_output_size: A scalar integer tensor representing the maximum number of
-// boxes to be selected by non max suppression.
+// The padded size of each dimension D of the output is:
 //
-// Returns A 1-D integer tensor of shape `[M]` representing the selected
-// indices from the boxes tensor, where `M <= max_output_size`.
-func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "NonMaxSuppression",
-		Input: []tf.Input{
-			boxes, scores, max_output_size,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset that emits `components` as a tuple of tensors once.
-func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
+// `paddings(D, 0) + input.dim_size(D) + paddings(D, 1)`
+//
+// For example:
+//
+// ```
+// # 't' is [[1, 1], [2, 2]]
+// # 'paddings' is [[1, 1], [2, 2]]
+// # rank of 't' is 2
+// pad(t, paddings) ==> [[0, 0, 0, 0, 0, 0]
+//                       [0, 0, 1, 1, 0, 0]
+//                       [0, 0, 2, 2, 0, 0]
+//                       [0, 0, 0, 0, 0, 0]]
+// ```
+func Pad(scope *Scope, input tf.Output, paddings tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "TensorDataset",
+		Type: "Pad",
 		Input: []tf.Input{
-			tf.OutputList(components),
+			input, paddings,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Component-wise multiplies a SparseTensor by a dense Tensor.
-//
-// The output locations corresponding to the implicitly zero elements in the sparse
-// tensor will be zero (i.e., will not take up storage space), regardless of the
-// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN).
-//
-// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
-// the other direction.
+// Checks whether a resource handle-based variable has been initialized.
 //
 // Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//	dense: `R`-D.  The dense Tensor operand.
+//	resource: the input resource handle.
 //
-// Returns 1-D.  The `N` values that are operated on.
-func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+// Returns a scalar boolean which is true if the variable has been
+// initialized.
+func VarIsInitializedOp(scope *Scope, resource tf.Output) (is_initialized tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseDenseCwiseMul",
+		Type: "VarIsInitializedOp",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape, dense,
+			resource,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
-type ResourceSparseApplyRMSPropAttr func(optionalAttr)
+// StatelessRandomUniformAttr is an optional argument to StatelessRandomUniform.
+type StatelessRandomUniformAttr func(optionalAttr)
 
-// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value.
+// StatelessRandomUniformDtype sets the optional dtype attribute to value.
 //
-// value: If `True`, updating of the var, ms, and mom tensors is protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr {
+// value: The type of the output.
+// If not specified, defaults to DT_FLOAT
+func StatelessRandomUniformDtype(value tf.DataType) StatelessRandomUniformAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["dtype"] = value
 	}
 }
 
-// Update '*var' according to the RMSProp algorithm.
-//
-// Note that in dense implementation of this algorithm, ms and mom will
-// update even if the grad is zero, but in this sparse implementation, ms
-// and mom will not update in iterations during which the grad is zero.
+// Outputs deterministic pseudorandom random values from a uniform distribution.
 //
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+// The generated values follow a uniform distribution in the range `[0, 1)`. The
+// lower bound 0 is included in the range, while the upper bound 1 is excluded.
 //
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
-// var <- var - mom
+// The outputs are a deterministic function of `shape` and `seed`.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
-//
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var, ms and mom.
+//	shape: The shape of the output tensor.
+//	seed: 2 seeds (shape [2]).
 //
-// Returns the created operation.
-func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) {
+// Returns Random values with specified shape.
+func StatelessRandomUniform(scope *Scope, shape tf.Output, seed tf.Output, optional ...StatelessRandomUniformAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -8859,506 +8773,455 @@ func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyRMSProp",
+		Type: "StatelessRandomUniform",
 		Input: []tf.Input{
-			var_, ms, mom, lr, rho, momentum, epsilon, grad, indices,
+			shape, seed,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns the truth value of (x > y) element-wise.
+// Makes its input available to the next iteration.
 //
-// *NOTE*: `Greater` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Arguments:
+//	data: The tensor to be made available to the next iteration.
+//
+// Returns The same tensor as `data`.
+func NextIteration(scope *Scope, data tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Greater",
+		Type: "NextIteration",
 		Input: []tf.Input{
-			x, y,
+			data,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox.
-type SampleDistortedBoundingBoxAttr func(optionalAttr)
+// Output a fact about factorials.
+func Fact(scope *Scope) (fact tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Fact",
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// SampleDistortedBoundingBoxSeed sets the optional seed attribute to value.
-//
-// value: If either `seed` or `seed2` are set to non-zero, the random number
-// generator is seeded by the given `seed`.  Otherwise, it is seeded by a random
-// seed.
-// If not specified, defaults to 0
-func SampleDistortedBoundingBoxSeed(value int64) SampleDistortedBoundingBoxAttr {
+// AngleAttr is an optional argument to Angle.
+type AngleAttr func(optionalAttr)
+
+// AngleTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func AngleTout(value tf.DataType) AngleAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["Tout"] = value
 	}
 }
 
-// SampleDistortedBoundingBoxSeed2 sets the optional seed2 attribute to value.
+// Returns the argument of a complex number.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func SampleDistortedBoundingBoxSeed2(value int64) SampleDistortedBoundingBoxAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the argument of each element in `input`. All elements in
+// `input` must be complex numbers of the form \\(a + bj\\), where *a*
+// is the real part and *b* is the imaginary part.
+//
+// The argument returned by this operation is of the form \\(atan2(b, a)\\).
+//
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.angle(input) ==> [2.0132, 1.056]
+// ```
+//
+// @compatibility(numpy)
+// Equivalent to np.angle.
+// @end_compatibility
+func Angle(scope *Scope, input tf.Output, optional ...AngleAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Angle",
+		Input: []tf.Input{
+			input,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SampleDistortedBoundingBoxMinObjectCovered sets the optional min_object_covered attribute to value.
+// VarHandleOpAttr is an optional argument to VarHandleOp.
+type VarHandleOpAttr func(optionalAttr)
+
+// VarHandleOpContainer sets the optional container attribute to value.
 //
-// value: The cropped area of the image must contain at least this
-// fraction of any bounding box supplied. The value of this parameter should be
-// non-negative. In the case of 0, the cropped area does not need to overlap
-// any of the bounding boxes supplied.
-// If not specified, defaults to 0.1
-func SampleDistortedBoundingBoxMinObjectCovered(value float32) SampleDistortedBoundingBoxAttr {
+// value: the container this variable is placed in.
+// If not specified, defaults to ""
+func VarHandleOpContainer(value string) VarHandleOpAttr {
 	return func(m optionalAttr) {
-		m["min_object_covered"] = value
+		m["container"] = value
 	}
 }
 
-// SampleDistortedBoundingBoxAspectRatioRange sets the optional aspect_ratio_range attribute to value.
+// VarHandleOpSharedName sets the optional shared_name attribute to value.
 //
-// value: The cropped area of the image must have an aspect ratio =
-// width / height within this range.
-// If not specified, defaults to <f:0.75 f:1.33 >
-func SampleDistortedBoundingBoxAspectRatioRange(value []float32) SampleDistortedBoundingBoxAttr {
+// value: the name by which this variable is referred to.
+// If not specified, defaults to ""
+func VarHandleOpSharedName(value string) VarHandleOpAttr {
 	return func(m optionalAttr) {
-		m["aspect_ratio_range"] = value
+		m["shared_name"] = value
 	}
 }
 
-// SampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value.
-//
-// value: The cropped area of the image must contain a fraction of the
-// supplied image within in this range.
-// If not specified, defaults to <f:0.05 f:1 >
-func SampleDistortedBoundingBoxAreaRange(value []float32) SampleDistortedBoundingBoxAttr {
-	return func(m optionalAttr) {
-		m["area_range"] = value
-	}
-}
-
-// SampleDistortedBoundingBoxMaxAttempts sets the optional max_attempts attribute to value.
-//
-// value: Number of attempts at generating a cropped region of the image
-// of the specified constraints. After `max_attempts` failures, return the entire
-// image.
-// If not specified, defaults to 100
-func SampleDistortedBoundingBoxMaxAttempts(value int64) SampleDistortedBoundingBoxAttr {
-	return func(m optionalAttr) {
-		m["max_attempts"] = value
-	}
-}
-
-// SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value.
-//
-// value: Controls behavior if no bounding boxes supplied.
-// If true, assume an implicit bounding box covering the whole input. If false,
-// raise an error.
-// If not specified, defaults to false
-func SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxAttr {
-	return func(m optionalAttr) {
-		m["use_image_if_no_bounding_boxes"] = value
-	}
-}
-
-// Generate a single randomly distorted bounding box for an image.
-//
-// Bounding box annotations are often supplied in addition to ground-truth labels
-// in image recognition or object localization tasks. A common technique for
-// training such a system is to randomly distort an image while preserving
-// its content, i.e. *data augmentation*. This Op outputs a randomly distorted
-// localization of an object, i.e. bounding box, given an `image_size`,
-// `bounding_boxes` and a series of constraints.
-//
-// The output of this Op is a single bounding box that may be used to crop the
-// original image. The output is returned as 3 tensors: `begin`, `size` and
-// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
-// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
-// what the bounding box looks like.
-//
-// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
-// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
-// height of the underlying image.
-//
-// For example,
-//
-// ```python
-//     # Generate a single distorted bounding box.
-//     begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
-//         tf.shape(image),
-//         bounding_boxes=bounding_boxes)
-//
-//     # Draw the bounding box in an image summary.
-//     image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
-//                                                   bbox_for_draw)
-//     tf.summary.image('images_with_box', image_with_box)
-//
-//     # Employ the bounding box to distort the image.
-//     distorted_image = tf.slice(image, begin, size)
-// ```
-//
-// Note that if no bounding box information is available, setting
-// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit
-// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
-// false and no bounding boxes are supplied, an error is raised.
+// Creates a handle to a Variable resource.
 //
 // Arguments:
-//	image_size: 1-D, containing `[height, width, channels]`.
-//	bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes
-// associated with the image.
-//
-// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to
-// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to
-// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box.
-// Provide as input to `tf.image.draw_bounding_boxes`.
-func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, optional ...SampleDistortedBoundingBoxAttr) (begin tf.Output, size tf.Output, bboxes tf.Output) {
+//	dtype: the type of this variable. Must agree with the dtypes
+// of all ops using this variable.
+//	shape: The (possibly partially specified) shape of this variable.
+func VarHandleOp(scope *Scope, dtype tf.DataType, shape tf.Shape, optional ...VarHandleOpAttr) (resource tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SampleDistortedBoundingBox",
-		Input: []tf.Input{
-			image_size, bounding_boxes,
-		},
+		Type: "VarHandleOp",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Converts each string in the input Tensor to its hash mod by a number of buckets.
-//
-// The hash function is deterministic on the content of the string within the
-// process and will never change. However, it is not suitable for cryptography.
-// This function may be used when CPU time is scarce and inputs are trusted or
-// unimportant. There is a risk of adversaries constructing inputs that all hash
-// to the same bucket. To prevent this problem, use a strong hash function with
-// `tf.string_to_hash_bucket_strong`.
-//
-// Arguments:
-//	input: The strings to assign a hash bucket.
-//	num_buckets: The number of buckets.
+// Elementwise computes the bitwise XOR of `x` and `y`.
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
+// The result will have those bits set, that are different in `x` and `y`. The
+// computation is performed on the underlying representations of `x` and `y`.
+func BitwiseXor(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "StringToHashBucketFast",
+		Type: "BitwiseXor",
 		Input: []tf.Input{
-			input,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
+// Deserialize `SparseTensor` objects.
 //
-// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// The input `serialized_sparse` must have the shape `[?, ?, ..., ?, 3]` where
+// the last dimension stores serialized `SparseTensor` objects and the other N
+// dimensions (N >= 0) correspond to a batch. The ranks of the original
+// `SparseTensor` objects must all match. When the final `SparseTensor` is
+// created, its rank is the rank of the incoming `SparseTensor` objects plus N;
+// the sparse tensors have been concatenated along new dimensions, one for each
+// batch.
+//
+// The output `SparseTensor` object's shape values for the original dimensions
+// are the max across the input `SparseTensor` objects' shape values for the
+// corresponding dimensions. The new dimensions match the size of the batch.
+//
+// The input `SparseTensor` objects' indices are assumed ordered in
+// standard lexicographic order.  If this is not the case, after this
+// step run `SparseReorder` to restore index ordering.
+//
+// For example, if the serialized input is a `[2 x 3]` matrix representing two
+// original `SparseTensor` objects:
+//
+//     index = [ 0]
+//             [10]
+//             [20]
+//     values = [1, 2, 3]
+//     shape = [50]
+//
+// and
+//
+//     index = [ 2]
+//             [10]
+//     values = [4, 5]
+//     shape = [30]
+//
+// then the final deserialized `SparseTensor` will be:
+//
+//     index = [0  0]
+//             [0 10]
+//             [0 20]
+//             [1  2]
+//             [1 10]
+//     values = [1, 2, 3, 4, 5]
+//     shape = [2 50]
+//
+// Arguments:
+//	serialized_sparse: The serialized `SparseTensor` objects. The last dimension
+// must have 3 columns.
+//	dtype: The `dtype` of the serialized `SparseTensor` objects.
+func DeserializeSparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "Maximum",
+		Type: "DeserializeSparse",
 		Input: []tf.Input{
-			x, y,
+			serialized_sparse,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
-type TensorArrayGatherV3Attr func(optionalAttr)
+// ResourceApplyRMSPropAttr is an optional argument to ResourceApplyRMSProp.
+type ResourceApplyRMSPropAttr func(optionalAttr)
 
-// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
+// ResourceApplyRMSPropUseLocking sets the optional use_locking attribute to value.
 //
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
+// value: If `True`, updating of the var, ms, and mom tensors is protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyRMSPropUseLocking(value bool) ResourceApplyRMSPropAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Gather specific elements from the TensorArray into output `value`.
+// Update '*var' according to the RMSProp algorithm.
 //
-// All elements selected by `indices` must have the same shape.
+// Note that in dense implementation of this algorithm, ms and mom will
+// update even if the grad is zero, but in this sparse implementation, ms
+// and mom will not update in iterations during which the grad is zero.
+//
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
+//
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
 //
 // Arguments:
-//	handle: The handle to a TensorArray.
-//	indices: The locations in the TensorArray from which to read tensor elements.
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
+//	var_: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
 //
-// Returns All of the elements in the TensorArray, concatenated along a new
-// axis (the new dimension 0).
-func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV3",
+		Type: "ResourceApplyRMSProp",
 		Input: []tf.Input{
-			handle, indices, flow_in,
+			var_, ms, mom, lr, rho, momentum, epsilon, grad,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Returns x / y element-wise for integer types.
-//
-// Truncation designates that negative numbers will round fractional quantities
-// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
-// than Python semantics. See `FloorDiv` for a division function that matches
-// Python Semantics.
+// ResourceScatterNdUpdateAttr is an optional argument to ResourceScatterNdUpdate.
+type ResourceScatterNdUpdateAttr func(optionalAttr)
+
+// ResourceScatterNdUpdateUseLocking sets the optional use_locking attribute to value.
 //
-// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TruncateDiv",
-		Input: []tf.Input{
-			x, y,
-		},
+// value: An optional bool. Defaults to True. If True, the assignment will
+// be protected by a lock; otherwise the behavior is undefined,
+// but may exhibit less contention.
+// If not specified, defaults to true
+func ResourceScatterNdUpdateUseLocking(value bool) ResourceScatterNdUpdateAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Restores tensors from a V2 checkpoint.
-//
-// For backward compatibility with the V1 format, this Op currently allows
-// restoring from a V1 checkpoint as well:
-//   - This Op first attempts to find the V2 index file pointed to by "prefix", and
-//     if found proceed to read it as a V2 checkpoint;
-//   - Otherwise the V1 read path is invoked.
-// Relying on this behavior is not recommended, as the ability to fall back to read
-// V1 might be deprecated and eventually removed.
-//
-// By default, restores the named tensors in full.  If the caller wishes to restore
-// specific slices of stored tensors, "shape_and_slices" should be non-empty
-// strings and correspondingly well-formed.
+// Applies sparse `updates` to individual values or slices within a given
 //
-// Callers must ensure all the named tensors are indeed stored in the checkpoint.
+// variable according to `indices`.
 //
-// Arguments:
-//	prefix: Must have a single element.  The prefix of a V2 checkpoint.
-//	tensor_names: shape {N}.  The names of the tensors to be restored.
-//	shape_and_slices: shape {N}.  The slice specs of the tensors to be restored.
-// Empty strings indicate that they are non-partitioned tensors.
-//	dtypes: shape {N}.  The list of expected dtype for the tensors.  Must match
-// those stored in the checkpoint.
+// `ref` is a `Tensor` with rank `P` and `indices` is a `Tensor` of rank `Q`.
 //
-// Returns shape {N}.  The restored tensors, whose shapes are read from the
-// checkpoint directly.
-func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	opspec := tf.OpSpec{
-		Type: "RestoreV2",
-		Input: []tf.Input{
-			prefix, tensor_names, shape_and_slices,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil {
-		scope.UpdateErr("RestoreV2", err)
-		return
-	}
-	return tensors
-}
-
-// Creates a dataset that skips `count` elements from the `input_dataset`.
+// `indices` must be integer tensor, containing indices into `ref`.
+// It must be shape `[d_0, ..., d_{Q-2}, K]` where `0 < K <= P`.
 //
-// Arguments:
+// The innermost dimension of `indices` (with length `K`) corresponds to
+// indices into elements (if `K = P`) or slices (if `K < P`) along the `K`th
+// dimension of `ref`.
 //
-//	count: A scalar representing the number of elements from the `input_dataset`
-// that should be skipped.  If count is -1, skips everything.
+// `updates` is `Tensor` of rank `Q-1+P-K` with shape:
 //
+// ```
+// [d_0, ..., d_{Q-2}, ref.shape[K], ..., ref.shape[P-1]].
+// ```
 //
-func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "SkipDataset",
-		Input: []tf.Input{
-			input_dataset, count,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the maximum along segments of a tensor.
+// For example, say we want to update 4 scattered elements to a rank-1 tensor to
+// 8 elements. In Python, that update would look like this:
 //
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
+// ```python
+//     ref = tfe.Variable([1, 2, 3, 4, 5, 6, 7, 8])
+//     indices = tf.constant([[4], [3], [1] ,[7]])
+//     updates = tf.constant([9, 10, 11, 12])
+//     update = tf.scatter_nd_update(ref, indices, updates)
+//     with tf.Session() as sess:
+//       print sess.run(update)
+// ```
 //
-// Computes a tensor such that
-// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
-// that `segment_ids[j] == i`.
+// The resulting update to ref would look like this:
 //
-// If the max is empty for a given segment ID `i`, `output[i] = 0`.
+//     [1, 11, 3, 10, 9, 6, 7, 12]
 //
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
-// </div>
+// See @{tf.scatter_nd} for more details about how to make updates to
+// slices.
 //
 // Arguments:
+//	ref: A resource handle. Must be from a VarHandleOp.
+//	indices: A Tensor. Must be one of the following types: int32, int64.
+// A tensor of indices into ref.
+//	updates: A Tensor. Must have the same type as ref. A tensor of updated
+// values to add to ref.
 //
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+// Returns the created operation.
+func ResourceScatterNdUpdate(scope *Scope, ref tf.Output, indices tf.Output, updates tf.Output, optional ...ResourceScatterNdUpdateAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "SegmentMax",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes hyperbolic tangent of `x` element-wise.
-func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Tanh",
+		Type: "ResourceScatterNdUpdate",
 		Input: []tf.Input{
-			x,
+			ref, indices, updates,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Decode web-safe base64-encoded strings.
-//
-// Input may or may not have padding at the end. See EncodeBase64 for padding.
-// Web-safe means that input must use - and _ instead of + and /.
+// SqueezeAttr is an optional argument to Squeeze.
+type SqueezeAttr func(optionalAttr)
+
+// SqueezeAxis sets the optional axis attribute to value.
 //
-// Arguments:
-//	input: Base64 strings to decode.
+// value: If specified, only squeezes the dimensions listed. The dimension
+// index starts at 0. It is an error to squeeze a dimension that is not 1. Must
+// be in the range `[-rank(input), rank(input))`.
+// If not specified, defaults to <>
 //
-// Returns Decoded strings.
-func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "DecodeBase64",
-		Input: []tf.Input{
-			input,
-		},
+// REQUIRES: len(value) >= 0
+func SqueezeAxis(value []int64) SqueezeAttr {
+	return func(m optionalAttr) {
+		m["squeeze_dims"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Store the input tensor in the state of the current session.
+// Removes dimensions of size 1 from the shape of a tensor.
+//
+// Given a tensor `input`, this operation returns a tensor of the same type with
+// all dimensions of size 1 removed. If you don't want to remove all size 1
+// dimensions, you can remove specific size 1 dimensions by specifying
+// `axis`.
+//
+// For example:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t)) ==> [2, 3]
+// ```
+//
+// Or, to remove specific size 1 dimensions:
+//
+// ```
+// # 't' is a tensor of shape [1, 2, 1, 3, 1, 1]
+// shape(squeeze(t, [2, 4])) ==> [1, 2, 3, 1]
+// ```
 //
 // Arguments:
-//	value: The tensor to be stored.
+//	input: The `input` to squeeze.
 //
-// Returns The handle for the tensor stored in the session state, represented
-// as a string.
-func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
+// Returns Contains the same data as `input`, but has one or more dimensions of
+// size 1 removed.
+func Squeeze(scope *Scope, input tf.Output, optional ...SqueezeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "GetSessionHandle",
+		Type: "Squeeze",
 		Input: []tf.Input{
-			value,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
-type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
+// ResourceApplyAdadeltaAttr is an optional argument to ResourceApplyAdadelta.
+type ResourceApplyAdadeltaAttr func(optionalAttr)
 
-// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
+// ResourceApplyAdadeltaUseLocking sets the optional use_locking attribute to value.
 //
-// value: If True, updating of the var and accum tensors will be protected by
+// value: If True, updating of the var, accum and update_accum tensors will be protected by
 // a lock; otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr {
+func ResourceApplyAdadeltaUseLocking(value bool) ResourceApplyAdadeltaAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm.
+// Update '*var' according to the adadelta scheme.
 //
-// That is for rows we have grad for, we update var and accum as follows:
-// accum += grad * grad
-// prox_v = var
-// prox_v -= lr * grad * (1 / sqrt(accum))
-// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
+// accum = rho() * accum + (1 - rho()) * grad.square();
+// update = (update_accum + epsilon).sqrt() * (accum + epsilon()).rsqrt() * grad;
+// update_accum = rho() * update_accum + (1 - rho()) * update.square();
+// var -= update;
 //
 // Arguments:
 //	var_: Should be from a Variable().
 //	accum: Should be from a Variable().
-//	lr: Learning rate. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
+//	accum_update: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay factor. Must be a scalar.
+//	epsilon: Constant factor. Must be a scalar.
 //	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) {
+func ResourceApplyAdadelta(scope *Scope, var_ tf.Output, accum tf.Output, accum_update tf.Output, lr tf.Output, rho tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdadeltaAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9367,157 +9230,165 @@ func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.O
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyProximalAdagrad",
+		Type: "ResourceApplyAdadelta",
 		Input: []tf.Input{
-			var_, accum, lr, l1, l2, grad, indices,
+			var_, accum, accum_update, lr, rho, epsilon, grad,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Returns element-wise largest integer not greater than x.
-func Floor(scope *Scope, x tf.Output) (y tf.Output) {
+// NonMaxSuppressionAttr is an optional argument to NonMaxSuppression.
+type NonMaxSuppressionAttr func(optionalAttr)
+
+// NonMaxSuppressionIouThreshold sets the optional iou_threshold attribute to value.
+//
+// value: A float representing the threshold for deciding whether boxes
+// overlap too much with respect to IOU.
+// If not specified, defaults to 0.5
+func NonMaxSuppressionIouThreshold(value float32) NonMaxSuppressionAttr {
+	return func(m optionalAttr) {
+		m["iou_threshold"] = value
+	}
+}
+
+// Greedily selects a subset of bounding boxes in descending order of score,
+//
+// pruning away boxes that have high intersection-over-union (IOU) overlap
+// with previously selected boxes.  Bounding boxes are supplied as
+// [y1, x1, y2, x2], where (y1, x1) and (y2, x2) are the coordinates of any
+// diagonal pair of box corners and the coordinates can be provided as normalized
+// (i.e., lying in the interval [0, 1]) or absolute.  Note that this algorithm
+// is agnostic to where the origin is in the coordinate system.  Note that this
+// algorithm is invariant to orthogonal transformations and translations
+// of the coordinate system; thus translating or reflections of the coordinate
+// system result in the same boxes being selected by the algorithm.
+// The output of this operation is a set of integers indexing into the input
+// collection of bounding boxes representing the selected boxes.  The bounding
+// box coordinates corresponding to the selected indices can then be obtained
+// using the `tf.gather operation`.  For example:
+//   selected_indices = tf.image.non_max_suppression(
+//       boxes, scores, max_output_size, iou_threshold)
+//   selected_boxes = tf.gather(boxes, selected_indices)
+//
+// Arguments:
+//	boxes: A 2-D float tensor of shape `[num_boxes, 4]`.
+//	scores: A 1-D float tensor of shape `[num_boxes]` representing a single
+// score corresponding to each box (each row of boxes).
+//	max_output_size: A scalar integer tensor representing the maximum number of
+// boxes to be selected by non max suppression.
+//
+// Returns A 1-D integer tensor of shape `[M]` representing the selected
+// indices from the boxes tensor, where `M <= max_output_size`.
+func NonMaxSuppression(scope *Scope, boxes tf.Output, scores tf.Output, max_output_size tf.Output, optional ...NonMaxSuppressionAttr) (selected_indices tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Floor",
+		Type: "NonMaxSuppression",
 		Input: []tf.Input{
-			x,
+			boxes, scores, max_output_size,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the Gauss error function of `x` element-wise.
-func Erf(scope *Scope, x tf.Output) (y tf.Output) {
+// Creates a dataset that emits `components` as a tuple of tensors once.
+func TensorDataset(scope *Scope, components []tf.Output, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "Erf",
+		Type: "TensorDataset",
 		Input: []tf.Input{
-			x,
+			tf.OutputList(components),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// OneHotAttr is an optional argument to OneHot.
-type OneHotAttr func(optionalAttr)
+// Component-wise multiplies a SparseTensor by a dense Tensor.
+//
+// The output locations corresponding to the implicitly zero elements in the sparse
+// tensor will be zero (i.e., will not take up storage space), regardless of the
+// contents of the dense tensor (even if it's +/-INF and that INF*0 == NaN).
+//
+// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+// the other direction.
+//
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//	dense: `R`-D.  The dense Tensor operand.
+//
+// Returns 1-D.  The `N` values that are operated on.
+func SparseDenseCwiseMul(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseDenseCwiseMul",
+		Input: []tf.Input{
+			sp_indices, sp_values, sp_shape, dense,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// OneHotAxis sets the optional axis attribute to value.
+// ResourceSparseApplyRMSPropAttr is an optional argument to ResourceSparseApplyRMSProp.
+type ResourceSparseApplyRMSPropAttr func(optionalAttr)
+
+// ResourceSparseApplyRMSPropUseLocking sets the optional use_locking attribute to value.
 //
-// value: The axis to fill (default: -1, a new inner-most axis).
-// If not specified, defaults to -1
-func OneHotAxis(value int64) OneHotAttr {
+// value: If `True`, updating of the var, ms, and mom tensors is protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceSparseApplyRMSPropUseLocking(value bool) ResourceSparseApplyRMSPropAttr {
 	return func(m optionalAttr) {
-		m["axis"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Returns a one-hot tensor.
-//
-// The locations represented by indices in `indices` take value `on_value`,
-// while all other locations take value `off_value`.
-//
-// If the input `indices` is rank `N`, the output will have rank `N+1`,
-// The new axis is created at dimension `axis` (default: the new axis is
-// appended at the end).
-//
-// If `indices` is a scalar the output shape will be a vector of length `depth`.
-//
-// If `indices` is a vector of length `features`, the output shape will be:
-// ```
-//   features x depth if axis == -1
-//   depth x features if axis == 0
-// ```
-//
-// If `indices` is a matrix (batch) with shape `[batch, features]`,
-// the output shape will be:
-// ```
-//   batch x features x depth if axis == -1
-//   batch x depth x features if axis == 1
-//   depth x batch x features if axis == 0
-// ```
-//
-//
-// Examples
-// =========
-//
-// Suppose that
-//
-// ```
-//   indices = [0, 2, -1, 1]
-//   depth = 3
-//   on_value = 5.0
-//   off_value = 0.0
-//   axis = -1
-// ```
-//
-// Then output is `[4 x 3]`:
-//
-//     ```output =
-//       [5.0 0.0 0.0]  // one_hot(0)
-//       [0.0 0.0 5.0]  // one_hot(2)
-//       [0.0 0.0 0.0]  // one_hot(-1)
-//       [0.0 5.0 0.0]  // one_hot(1)
-//     ```
-//
-// Suppose that
-//
-// ```
-//   indices = [0, 2, -1, 1]
-//   depth = 3
-//   on_value = 0.0
-//   off_value = 3.0
-//   axis = 0
-// ```
-//
-// Then output is `[3 x 4]`:
-//
-//     ```output =
-//       [0.0 3.0 3.0 3.0]
-//       [3.0 3.0 3.0 0.0]
-//       [3.0 3.0 3.0 3.0]
-//       [3.0 0.0 3.0 3.0]
-//     //  ^                one_hot(0)
-//     //      ^            one_hot(2)
-//     //          ^        one_hot(-1)
-//     //              ^    one_hot(1)
-//     ```
-// Suppose that
+// Update '*var' according to the RMSProp algorithm.
 //
-// ```
-//   indices = [[0, 2], [1, -1]]
-//   depth = 3
-//   on_value = 1.0
-//   off_value = 0.0
-//   axis = -1
-// ```
+// Note that in dense implementation of this algorithm, ms and mom will
+// update even if the grad is zero, but in this sparse implementation, ms
+// and mom will not update in iterations during which the grad is zero.
 //
-// Then output is `[2 x 2 x 3]`:
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon)
 //
-//     ```output =
-//       [
-//         [1.0, 0.0, 0.0]  // one_hot(0)
-//         [0.0, 0.0, 1.0]  // one_hot(2)
-//       ][
-//         [0.0, 1.0, 0.0]  // one_hot(1)
-//         [0.0, 0.0, 0.0]  // one_hot(-1)
-//       ]```
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms + epsilon)
+// var <- var - mom
 //
 // Arguments:
-//	indices: A tensor of indices.
-//	depth: A scalar defining the depth of the one hot dimension.
-//	on_value: A scalar defining the value to fill in output when `indices[j] = i`.
-//	off_value: A scalar defining the value to fill in output when `indices[j] != i`.
+//	var_: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
 //
-// Returns The one-hot tensor.
-func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output, off_value tf.Output, optional ...OneHotAttr) (output tf.Output) {
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var, ms and mom.
+//
+// Returns the created operation.
+func ResourceSparseApplyRMSProp(scope *Scope, var_ tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyRMSPropAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9526,128 +9397,168 @@ func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OneHot",
+		Type: "ResourceSparseApplyRMSProp",
 		Input: []tf.Input{
-			indices, depth, on_value, off_value,
+			var_, ms, mom, lr, rho, momentum, epsilon, grad, indices,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Reads the value of a variable.
-//
-// The tensor returned by this operation is immutable.
-//
-// The value returned by this operation is guaranteed to be influenced by all the
-// writes on which this operation depends directly or indirectly, and to not be
-// influenced by any of the writes which depend directly or indirectly on this
-// operation.
+// Returns the truth value of (x > y) element-wise.
 //
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	dtype: the dtype of the value.
-func ReadVariableOp(scope *Scope, resource tf.Output, dtype tf.DataType) (value tf.Output) {
+// *NOTE*: `Greater` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Greater(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "ReadVariableOp",
+		Type: "Greater",
 		Input: []tf.Input{
-			resource,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad.
-type MaxPool3DGradAttr func(optionalAttr)
+// SampleDistortedBoundingBoxAttr is an optional argument to SampleDistortedBoundingBox.
+type SampleDistortedBoundingBoxAttr func(optionalAttr)
 
-// MaxPool3DGradDataFormat sets the optional data_format attribute to value.
+// SampleDistortedBoundingBoxSeed sets the optional seed attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr {
+// value: If either `seed` or `seed2` are set to non-zero, the random number
+// generator is seeded by the given `seed`.  Otherwise, it is seeded by a random
+// seed.
+// If not specified, defaults to 0
+func SampleDistortedBoundingBoxSeed(value int64) SampleDistortedBoundingBoxAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["seed"] = value
 	}
 }
 
-// Computes gradients of max pooling function.
+// SampleDistortedBoundingBoxSeed2 sets the optional seed2 attribute to value.
 //
-// Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func SampleDistortedBoundingBoxSeed2(value int64) SampleDistortedBoundingBoxAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
+}
+
+// SampleDistortedBoundingBoxMinObjectCovered sets the optional min_object_covered attribute to value.
+//
+// value: The cropped area of the image must contain at least this
+// fraction of any bounding box supplied. The value of this parameter should be
+// non-negative. In the case of 0, the cropped area does not need to overlap
+// any of the bounding boxes supplied.
+// If not specified, defaults to 0.1
+func SampleDistortedBoundingBoxMinObjectCovered(value float32) SampleDistortedBoundingBoxAttr {
+	return func(m optionalAttr) {
+		m["min_object_covered"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "MaxPool3DGrad",
-		Input: []tf.Input{
-			orig_input, orig_output, grad,
-		},
-		Attrs: attrs,
+}
+
+// SampleDistortedBoundingBoxAspectRatioRange sets the optional aspect_ratio_range attribute to value.
+//
+// value: The cropped area of the image must have an aspect ratio =
+// width / height within this range.
+// If not specified, defaults to <f:0.75 f:1.33 >
+func SampleDistortedBoundingBoxAspectRatioRange(value []float32) SampleDistortedBoundingBoxAttr {
+	return func(m optionalAttr) {
+		m["aspect_ratio_range"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// SparseReduceSumAttr is an optional argument to SparseReduceSum.
-type SparseReduceSumAttr func(optionalAttr)
+// SampleDistortedBoundingBoxAreaRange sets the optional area_range attribute to value.
+//
+// value: The cropped area of the image must contain a fraction of the
+// supplied image within in this range.
+// If not specified, defaults to <f:0.05 f:1 >
+func SampleDistortedBoundingBoxAreaRange(value []float32) SampleDistortedBoundingBoxAttr {
+	return func(m optionalAttr) {
+		m["area_range"] = value
+	}
+}
 
-// SparseReduceSumKeepDims sets the optional keep_dims attribute to value.
+// SampleDistortedBoundingBoxMaxAttempts sets the optional max_attempts attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
+// value: Number of attempts at generating a cropped region of the image
+// of the specified constraints. After `max_attempts` failures, return the entire
+// image.
+// If not specified, defaults to 100
+func SampleDistortedBoundingBoxMaxAttempts(value int64) SampleDistortedBoundingBoxAttr {
+	return func(m optionalAttr) {
+		m["max_attempts"] = value
+	}
+}
+
+// SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes sets the optional use_image_if_no_bounding_boxes attribute to value.
+//
+// value: Controls behavior if no bounding boxes supplied.
+// If true, assume an implicit bounding box covering the whole input. If false,
+// raise an error.
 // If not specified, defaults to false
-func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr {
+func SampleDistortedBoundingBoxUseImageIfNoBoundingBoxes(value bool) SampleDistortedBoundingBoxAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["use_image_if_no_bounding_boxes"] = value
 	}
 }
 
-// Computes the sum of elements across dimensions of a SparseTensor.
+// Generate a single randomly distorted bounding box for an image.
 //
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
-// instead of a sparse one.
+// Bounding box annotations are often supplied in addition to ground-truth labels
+// in image recognition or object localization tasks. A common technique for
+// training such a system is to randomly distort an image while preserving
+// its content, i.e. *data augmentation*. This Op outputs a randomly distorted
+// localization of an object, i.e. bounding box, given an `image_size`,
+// `bounding_boxes` and a series of constraints.
 //
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
+// The output of this Op is a single bounding box that may be used to crop the
+// original image. The output is returned as 3 tensors: `begin`, `size` and
+// `bboxes`. The first 2 tensors can be fed directly into `tf.slice` to crop the
+// image. The latter may be supplied to `tf.image.draw_bounding_boxes` to visualize
+// what the bounding box looks like.
 //
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// Bounding boxes are supplied and returned as `[y_min, x_min, y_max, x_max]`. The
+// bounding box coordinates are floats in `[0.0, 1.0]` relative to the width and
+// height of the underlying image.
+//
+// For example,
+//
+// ```python
+//     # Generate a single distorted bounding box.
+//     begin, size, bbox_for_draw = tf.image.sample_distorted_bounding_box(
+//         tf.shape(image),
+//         bounding_boxes=bounding_boxes)
+//
+//     # Draw the bounding box in an image summary.
+//     image_with_box = tf.image.draw_bounding_boxes(tf.expand_dims(image, 0),
+//                                                   bbox_for_draw)
+//     tf.summary.image('images_with_box', image_with_box)
+//
+//     # Employ the bounding box to distort the image.
+//     distorted_image = tf.slice(image, begin, size)
+// ```
+//
+// Note that if no bounding box information is available, setting
+// `use_image_if_no_bounding_boxes = true` will assume there is a single implicit
+// bounding box covering the whole image. If `use_image_if_no_bounding_boxes` is
+// false and no bounding boxes are supplied, an error is raised.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+//	image_size: 1-D, containing `[height, width, channels]`.
+//	bounding_boxes: 3-D with shape `[batch, N, 4]` describing the N bounding boxes
+// associated with the image.
 //
-// Returns `R-K`-D.  The reduced Tensor.
-func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) {
+// Returns 1-D, containing `[offset_height, offset_width, 0]`. Provide as input to
+// `tf.slice`.1-D, containing `[target_height, target_width, -1]`. Provide as input to
+// `tf.slice`.3-D with shape `[1, 1, 4]` containing the distorted bounding box.
+// Provide as input to `tf.image.draw_bounding_boxes`.
+func SampleDistortedBoundingBox(scope *Scope, image_size tf.Output, bounding_boxes tf.Output, optional ...SampleDistortedBoundingBoxAttr) (begin tf.Output, size tf.Output, bboxes tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -9656,245 +9567,168 @@ func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Outp
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseReduceSum",
+		Type: "SampleDistortedBoundingBox",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
+			image_size, bounding_boxes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns element-wise remainder of division. This emulates C semantics in that
+// Converts each string in the input Tensor to its hash mod by a number of buckets.
 //
-// the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
-// y + truncate_mod(x, y) = x`.
+// The hash function is deterministic on the content of the string within the
+// process and will never change. However, it is not suitable for cryptography.
+// This function may be used when CPU time is scarce and inputs are trusted or
+// unimportant. There is a risk of adversaries constructing inputs that all hash
+// to the same bucket. To prevent this problem, use a strong hash function with
+// `tf.string_to_hash_bucket_strong`.
 //
-// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Arguments:
+//	input: The strings to assign a hash bucket.
+//	num_buckets: The number of buckets.
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToHashBucketFast(scope *Scope, input tf.Output, num_buckets int64) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_buckets": num_buckets}
 	opspec := tf.OpSpec{
-		Type: "TruncateMod",
+		Type: "StringToHashBucketFast",
 		Input: []tf.Input{
-			x, y,
+			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse 2D real-valued fast Fourier transform.
-//
-// Computes the inverse 2-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most 2 dimensions of `input`.
-//
-// The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`:
-// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
-// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
-// from the size of the inner-most 2 dimensions of `input`. If the FFT length used
-// to compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
-//
-// Along each axis `IRFFT2D` is computed on, if `fft_length` (or
-// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
-// corresponding dimension of `input`, the dimension is cropped. If it is larger,
-// the dimension is padded with zeros.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
-//
-// Returns A float32 tensor of the same rank as `input`. The inner-most 2
-//   dimensions of `input` are replaced with the `fft_length` samples of their
-//   inverse 2D Fourier transform.
+// Returns the max of x and y (i.e. x > y ? x : y) element-wise.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.irfft2
-// @end_compatibility
-func IRFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// *NOTE*: `Maximum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Maximum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IRFFT2D",
+		Type: "Maximum",
 		Input: []tf.Input{
-			input, fft_length,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DecodeJpegAttr is an optional argument to DecodeJpeg.
-type DecodeJpegAttr func(optionalAttr)
+// TensorArrayGatherV3Attr is an optional argument to TensorArrayGatherV3.
+type TensorArrayGatherV3Attr func(optionalAttr)
 
-// DecodeJpegChannels sets the optional channels attribute to value.
+// TensorArrayGatherV3ElementShape sets the optional element_shape attribute to value.
 //
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodeJpegChannels(value int64) DecodeJpegAttr {
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayGatherV3ElementShape(value tf.Shape) TensorArrayGatherV3Attr {
 	return func(m optionalAttr) {
-		m["channels"] = value
+		m["element_shape"] = value
 	}
 }
 
-// DecodeJpegRatio sets the optional ratio attribute to value.
+// Gather specific elements from the TensorArray into output `value`.
 //
-// value: Downscaling ratio.
-// If not specified, defaults to 1
-func DecodeJpegRatio(value int64) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["ratio"] = value
-	}
-}
-
-// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
+// All elements selected by `indices` must have the same shape.
 //
-// value: If true use a slower but nicer upscaling of the
-// chroma planes (yuv420/422 only).
-// If not specified, defaults to true
-func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["fancy_upscaling"] = value
-	}
-}
-
-// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
+// Arguments:
+//	handle: The handle to a TensorArray.
+//	indices: The locations in the TensorArray from which to read tensor elements.
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// value: If true try to recover an image from truncated input.
-// If not specified, defaults to false
-func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["try_recover_truncated"] = value
+// Returns All of the elements in the TensorArray, concatenated along a new
+// axis (the new dimension 0).
+func TensorArrayGatherV3(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV3Attr) (value tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
-//
-// value: The minimum required fraction of lines before a truncated
-// input is accepted.
-// If not specified, defaults to 1
-func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["acceptable_fraction"] = value
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
 	}
-}
-
-// DecodeJpegDctMethod sets the optional dct_method attribute to value.
-//
-// value: string specifying a hint about the algorithm used for
-// decompression.  Defaults to "" which maps to a system-specific
-// default.  Currently valid values are ["INTEGER_FAST",
-// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
-// jpeg library changes to a version that does not have that specific
-// option.)
-// If not specified, defaults to ""
-func DecodeJpegDctMethod(value string) DecodeJpegAttr {
-	return func(m optionalAttr) {
-		m["dct_method"] = value
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGatherV3",
+		Input: []tf.Input{
+			handle, indices, flow_in,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Decode a JPEG-encoded image to a uint8 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the JPEG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-//
-// If needed, the JPEG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// The attr `ratio` allows downscaling the image by an integer factor during
-// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
-// downscaling the image later.
-//
-//
-// This op also supports decoding PNGs and non-animated GIFs since the interface is
-// the same, though it is cleaner to use `tf.image.decode_image`.
+// Returns x / y element-wise for integer types.
 //
-// Arguments:
-//	contents: 0-D.  The JPEG-encoded image.
+// Truncation designates that negative numbers will round fractional quantities
+// toward zero. I.e. -7 / 5 = -1. This matches C semantics but it is different
+// than Python semantics. See `FloorDiv` for a division function that matches
+// Python Semantics.
 //
-// Returns 3-D with shape `[height, width, channels]`..
-func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) {
+// *NOTE*: `TruncateDiv` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func TruncateDiv(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DecodeJpeg",
+		Type: "TruncateDiv",
 		Input: []tf.Input{
-			contents,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Transforms a vector of brain.Example protos (as strings) into typed tensors.
+// Restores tensors from a V2 checkpoint.
+//
+// For backward compatibility with the V1 format, this Op currently allows
+// restoring from a V1 checkpoint as well:
+//   - This Op first attempts to find the V2 index file pointed to by "prefix", and
+//     if found proceed to read it as a V2 checkpoint;
+//   - Otherwise the V1 read path is invoked.
+// Relying on this behavior is not recommended, as the ability to fall back to read
+// V1 might be deprecated and eventually removed.
+//
+// By default, restores the named tensors in full.  If the caller wishes to restore
+// specific slices of stored tensors, "shape_and_slices" should be non-empty
+// strings and correspondingly well-formed.
+//
+// Callers must ensure all the named tensors are indeed stored in the checkpoint.
 //
 // Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	names: A vector containing the names of the serialized protos.
-// May contain, for example, table key (descriptive) names for the
-// corresponding serialized protos.  These are purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty vector if no names are available.
-// If non-empty, this vector must be the same length as "serialized".
-//	sparse_keys: A list of Nsparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: A list of Ndense string Tensors (scalars).
-// The keys expected in the Examples' features associated with dense values.
-//	dense_defaults: A list of Ndense Tensors (some may be empty).
-// dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	sparse_types: A list of Nsparse types; the data types of data in each Feature
-// given in sparse_keys.
-// Currently the ParseExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
-// given in dense_keys.
-// The number of elements in the Feature corresponding to dense_key[j]
-// must always equal dense_shapes[j].NumEntries().
-// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
-// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
-// The dense outputs are just the inputs row-stacked by batch.
-// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
-// the shape of the output Tensor dense_values[j] will be
-// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
-// of elements of length D1 * .... * DN, across all minibatch entries
-// in the input.  Any minibatch entry with less than M blocks of elements of
-// length D1 * ... * DN will be padded with the corresponding default_value
-// scalar element along the second dimension.
-func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+//	prefix: Must have a single element.  The prefix of a V2 checkpoint.
+//	tensor_names: shape {N}.  The names of the tensors to be restored.
+//	shape_and_slices: shape {N}.  The slice specs of the tensors to be restored.
+// Empty strings indicate that they are non-partitioned tensors.
+//	dtypes: shape {N}.  The list of expected dtype for the tensors.  Must match
+// those stored in the checkpoint.
+//
+// Returns shape {N}.  The restored tensors, whose shapes are read from the
+// checkpoint directly.
+func RestoreV2(scope *Scope, prefix tf.Output, tensor_names tf.Output, shape_and_slices tf.Output, dtypes []tf.DataType) (tensors []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	opspec := tf.OpSpec{
-		Type: "ParseExample",
+		Type: "RestoreV2",
 		Input: []tf.Input{
-			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
+			prefix, tensor_names, shape_and_slices,
 		},
 		Attrs: attrs,
 	}
@@ -9904,58 +9738,30 @@ func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_ke
 	}
 	var idx int
 	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseExample", err)
-		return
-	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseExample", err)
+	if tensors, idx, err = makeOutputList(op, idx, "tensors"); err != nil {
+		scope.UpdateErr("RestoreV2", err)
 		return
 	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
-}
-
-// VariableShapeAttr is an optional argument to VariableShape.
-type VariableShapeAttr func(optionalAttr)
-
-// VariableShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func VariableShapeOutType(value tf.DataType) VariableShapeAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
+	return tensors
 }
 
-// Returns the shape of the variable pointed to by `resource`.
+// Creates a dataset that skips `count` elements from the `input_dataset`.
 //
-// This operation returns a 1-D integer tensor representing the shape of `input`.
+// Arguments:
 //
-// For example:
+//	count: A scalar representing the number of elements from the `input_dataset`
+// that should be skipped.  If count is -1, skips everything.
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr) (output tf.Output) {
+//
+func SkipDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "VariableShape",
+		Type: "SkipDataset",
 		Input: []tf.Input{
-			input,
+			input_dataset, count,
 		},
 		Attrs: attrs,
 	}
@@ -9963,165 +9769,134 @@ func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr)
 	return op.Output(0)
 }
 
-// Computes softmax cross entropy cost and gradients to backpropagate.
+// Computes the maximum along segments of a tensor.
 //
-// Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
-// a matrix of label probabilities, but rather a single label per row
-// of features.  This label is considered to have probability 1.0 for the
-// given row.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// Inputs are the logits, not probabilities.
+// Computes a tensor such that
+// \\(output_i = \max_j(data_j)\\) where `max` is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the max is empty for a given segment ID `i`, `output[i] = 0`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMax.png" alt>
+// </div>
 //
 // Arguments:
-//	features: batch_size x num_classes matrix
-//	labels: batch_size vector with values in [0, num_classes).
-// This is the label for the given minibatch entry.
 //
-// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
-func SparseSoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
-	if scope.Err() != nil {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMax(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSoftmaxCrossEntropyWithLogits",
+		Type: "SegmentMax",
 		Input: []tf.Input{
-			features, labels,
+			data, segment_ids,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Fast Fourier transform.
-//
-// Computes the 1-dimensional discrete Fourier transform over the inner-most
-// dimension of `input`.
-//
-// Arguments:
-//	input: A complex64 tensor.
-//
-// Returns A complex64 tensor of the same shape as `input`. The inner-most
-//   dimension of `input` is replaced with its 1D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.fft
-// @end_compatibility
-func FFT(scope *Scope, input tf.Output) (output tf.Output) {
+// Computes hyperbolic tangent of `x` element-wise.
+func Tanh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "FFT",
+		Type: "Tanh",
 		Input: []tf.Input{
-			input,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
+// Decode web-safe base64-encoded strings.
+//
+// Input may or may not have padding at the end. See EncodeBase64 for padding.
+// Web-safe means that input must use - and _ instead of + and /.
 //
 // Arguments:
-//	serialized: A scalar string containing a serialized TensorProto proto.
-//	out_type: The type of the serialized tensor.  The provided type must match the
-// type of the serialized tensor and no implicit conversion will take place.
+//	input: Base64 strings to decode.
 //
-// Returns A Tensor of type `out_type`.
-func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
+// Returns Decoded strings.
+func DecodeBase64(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "ParseTensor",
+		Type: "DecodeBase64",
 		Input: []tf.Input{
-			serialized,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
-type MaxPoolWithArgmaxAttr func(optionalAttr)
-
-// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
-// If not specified, defaults to DT_INT64
-func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
-	return func(m optionalAttr) {
-		m["Targmax"] = value
-	}
-}
-
-// Performs max pooling on the input and outputs both max values and indices.
-//
-// The indices in `argmax` are flattened, so that a maximum value at position
-// `[b, y, x, c]` becomes flattened index
-// `((b * height + y) * width + x) * channels + c`.
-//
-// The indices returned are always in `[0, height) x [0, width)` before flattening,
-// even if padding is involved and the mathematically correct answer is outside
-// (either negative or too large).  This is a bug, but fixing it is difficult to do
-// in a safe backwards compatible way, especially due to flattening.
+// Store the input tensor in the state of the current session.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	value: The tensor to be stored.
 //
-// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
-func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
+// Returns The handle for the tensor stored in the session state, represented
+// as a string.
+func GetSessionHandle(scope *Scope, value tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolWithArgmax",
+		Type: "GetSessionHandle",
 		Input: []tf.Input{
-			input,
+			value,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
-type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
+// ResourceSparseApplyProximalAdagradAttr is an optional argument to ResourceSparseApplyProximalAdagrad.
+type ResourceSparseApplyProximalAdagradAttr func(optionalAttr)
 
-// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
 //
 // value: If True, updating of the var and accum tensors will be protected by
 // a lock; otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
+func ResourceSparseApplyProximalAdagradUseLocking(value bool) ResourceSparseApplyProximalAdagradAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
+// Sparse update entries in '*var' and '*accum' according to FOBOS algorithm.
+//
+// That is for rows we have grad for, we update var and accum as follows:
+// accum += grad * grad
+// prox_v = var
+// prox_v -= lr * grad * (1 / sqrt(accum))
+// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	gradient_accumulator: Should be from a Variable().
-//	gradient_squared_accumulator: Should be from a Variable().
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
+//	accum: Should be from a Variable().
 //	lr: Learning rate. Must be a scalar.
 //	l1: L1 regularization. Must be a scalar.
 //	l2: L2 regularization. Must be a scalar.
-//	global_step: Training step number. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
 // Returns the created operation.
-func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
+func ResourceSparseApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10130,133 +9905,157 @@ func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumul
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyAdagradDA",
+		Type: "ResourceSparseApplyProximalAdagrad",
 		Input: []tf.Input{
-			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
+			var_, accum, lr, l1, l2, grad, indices,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// EncodeJpegAttr is an optional argument to EncodeJpeg.
-type EncodeJpegAttr func(optionalAttr)
-
-// EncodeJpegFormat sets the optional format attribute to value.
-//
-// value: Per pixel image format.
-// If not specified, defaults to ""
-func EncodeJpegFormat(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["format"] = value
+// Returns element-wise largest integer not greater than x.
+func Floor(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// EncodeJpegQuality sets the optional quality attribute to value.
-//
-// value: Quality of the compression from 0 to 100 (higher is better and slower).
-// If not specified, defaults to 95
-func EncodeJpegQuality(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["quality"] = value
+	opspec := tf.OpSpec{
+		Type: "Floor",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegProgressive sets the optional progressive attribute to value.
-//
-// value: If True, create a JPEG that loads progressively (coarse to fine).
-// If not specified, defaults to false
-func EncodeJpegProgressive(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["progressive"] = value
+// Computes the Gauss error function of `x` element-wise.
+func Erf(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
-//
-// value: If True, spend CPU/RAM to reduce size with no quality change.
-// If not specified, defaults to false
-func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["optimize_size"] = value
+	opspec := tf.OpSpec{
+		Type: "Erf",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
-//
-// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
-// If not specified, defaults to true
-func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["chroma_downsampling"] = value
-	}
-}
+// OneHotAttr is an optional argument to OneHot.
+type OneHotAttr func(optionalAttr)
 
-// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
+// OneHotAxis sets the optional axis attribute to value.
 //
-// value: Unit used to specify `x_density` and `y_density`:
-// pixels per inch (`'in'`) or centimeter (`'cm'`).
-// If not specified, defaults to "in"
-func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+// value: The axis to fill (default: -1, a new inner-most axis).
+// If not specified, defaults to -1
+func OneHotAxis(value int64) OneHotAttr {
 	return func(m optionalAttr) {
-		m["density_unit"] = value
+		m["axis"] = value
 	}
 }
 
-// EncodeJpegXDensity sets the optional x_density attribute to value.
+// Returns a one-hot tensor.
 //
-// value: Horizontal pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegXDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["x_density"] = value
-	}
-}
-
-// EncodeJpegYDensity sets the optional y_density attribute to value.
+// The locations represented by indices in `indices` take value `on_value`,
+// while all other locations take value `off_value`.
 //
-// value: Vertical pixels per density unit.
-// If not specified, defaults to 300
-func EncodeJpegYDensity(value int64) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["y_density"] = value
-	}
-}
-
-// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
+// If the input `indices` is rank `N`, the output will have rank `N+1`,
+// The new axis is created at dimension `axis` (default: the new axis is
+// appended at the end).
 //
-// value: If not empty, embed this XMP metadata in the image header.
-// If not specified, defaults to ""
-func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
-	return func(m optionalAttr) {
-		m["xmp_metadata"] = value
-	}
-}
-
-// JPEG-encode an image.
+// If `indices` is a scalar the output shape will be a vector of length `depth`.
 //
-// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
+// If `indices` is a vector of length `features`, the output shape will be:
+// ```
+//   features x depth if axis == -1
+//   depth x features if axis == 0
+// ```
 //
-// The attr `format` can be used to override the color format of the encoded
-// output.  Values can be:
+// If `indices` is a matrix (batch) with shape `[batch, features]`,
+// the output shape will be:
+// ```
+//   batch x features x depth if axis == -1
+//   batch x depth x features if axis == 1
+//   depth x batch x features if axis == 0
+// ```
 //
-// *   `''`: Use a default format based on the number of channels in the image.
-// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
-//     of `image` must be 1.
-// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
-//     of `image` must be 3.
 //
-// If `format` is not specified or is the empty string, a default format is picked
-// in function of the number of channels in `image`:
+// Examples
+// =========
 //
-// *   1: Output a grayscale image.
-// *   3: Output an RGB image.
+// Suppose that
+//
+// ```
+//   indices = [0, 2, -1, 1]
+//   depth = 3
+//   on_value = 5.0
+//   off_value = 0.0
+//   axis = -1
+// ```
+//
+// Then output is `[4 x 3]`:
+//
+//     ```output =
+//       [5.0 0.0 0.0]  // one_hot(0)
+//       [0.0 0.0 5.0]  // one_hot(2)
+//       [0.0 0.0 0.0]  // one_hot(-1)
+//       [0.0 5.0 0.0]  // one_hot(1)
+//     ```
+//
+// Suppose that
+//
+// ```
+//   indices = [0, 2, -1, 1]
+//   depth = 3
+//   on_value = 0.0
+//   off_value = 3.0
+//   axis = 0
+// ```
+//
+// Then output is `[3 x 4]`:
+//
+//     ```output =
+//       [0.0 3.0 3.0 3.0]
+//       [3.0 3.0 3.0 0.0]
+//       [3.0 3.0 3.0 3.0]
+//       [3.0 0.0 3.0 3.0]
+//     //  ^                one_hot(0)
+//     //      ^            one_hot(2)
+//     //          ^        one_hot(-1)
+//     //              ^    one_hot(1)
+//     ```
+// Suppose that
+//
+// ```
+//   indices = [[0, 2], [1, -1]]
+//   depth = 3
+//   on_value = 1.0
+//   off_value = 0.0
+//   axis = -1
+// ```
+//
+// Then output is `[2 x 2 x 3]`:
+//
+//     ```output =
+//       [
+//         [1.0, 0.0, 0.0]  // one_hot(0)
+//         [0.0, 0.0, 1.0]  // one_hot(2)
+//       ][
+//         [0.0, 1.0, 0.0]  // one_hot(1)
+//         [0.0, 0.0, 0.0]  // one_hot(-1)
+//       ]```
 //
 // Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
+//	indices: A tensor of indices.
+//	depth: A scalar defining the depth of the one hot dimension.
+//	on_value: A scalar defining the value to fill in output when `indices[j] = i`.
+//	off_value: A scalar defining the value to fill in output when `indices[j] != i`.
 //
-// Returns 0-D. JPEG-encoded image.
-func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
+// Returns The one-hot tensor.
+func OneHot(scope *Scope, indices tf.Output, depth tf.Output, on_value tf.Output, off_value tf.Output, optional ...OneHotAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10265,9 +10064,9 @@ func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (cont
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodeJpeg",
+		Type: "OneHot",
 		Input: []tf.Input{
-			image,
+			indices, depth, on_value, off_value,
 		},
 		Attrs: attrs,
 	}
@@ -10275,48 +10074,118 @@ func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (cont
 	return op.Output(0)
 }
 
-// MultinomialAttr is an optional argument to Multinomial.
-type MultinomialAttr func(optionalAttr)
+// Reads the value of a variable.
+//
+// The tensor returned by this operation is immutable.
+//
+// The value returned by this operation is guaranteed to be influenced by all the
+// writes on which this operation depends directly or indirectly, and to not be
+// influenced by any of the writes which depend directly or indirectly on this
+// operation.
+//
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	dtype: the dtype of the value.
+func ReadVariableOp(scope *Scope, resource tf.Output, dtype tf.DataType) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	opspec := tf.OpSpec{
+		Type: "ReadVariableOp",
+		Input: []tf.Input{
+			resource,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// MultinomialSeed sets the optional seed attribute to value.
+// MaxPool3DGradAttr is an optional argument to MaxPool3DGrad.
+type MaxPool3DGradAttr func(optionalAttr)
+
+// MaxPool3DGradDataFormat sets the optional data_format attribute to value.
 //
-// value: If either seed or seed2 is set to be non-zero, the internal random number
-// generator is seeded by the given seed.  Otherwise, a random seed is used.
-// If not specified, defaults to 0
-func MultinomialSeed(value int64) MultinomialAttr {
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func MaxPool3DGradDataFormat(value string) MaxPool3DGradAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["data_format"] = value
 	}
 }
 
-// MultinomialSeed2 sets the optional seed2 attribute to value.
+// Computes gradients of max pooling function.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func MultinomialSeed2(value int64) MultinomialAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+// Arguments:
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func MaxPool3DGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MaxPool3DGrad",
+		Input: []tf.Input{
+			orig_input, orig_output, grad,
+		},
+		Attrs: attrs,
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MultinomialOutputDtype sets the optional output_dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
+// SparseReduceSumAttr is an optional argument to SparseReduceSum.
+type SparseReduceSumAttr func(optionalAttr)
+
+// SparseReduceSumKeepDims sets the optional keep_dims attribute to value.
+//
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SparseReduceSumKeepDims(value bool) SparseReduceSumAttr {
 	return func(m optionalAttr) {
-		m["output_dtype"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Draws samples from a multinomial distribution.
+// Computes the sum of elements across dimensions of a SparseTensor.
+//
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_sum()`.  In particular, this Op also returns a dense `Tensor`
+// instead of a sparse one.
+//
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
 //
 // Arguments:
-//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
-// represents the unnormalized log probabilities for all classes.
-//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
 //
-// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
-// contains the drawn class labels with range `[0, num_classes)`.
-func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
+// Returns `R-K`-D.  The reduced Tensor.
+func SparseReduceSum(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceSumAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10325,9 +10194,9 @@ func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Multinomial",
+		Type: "SparseReduceSum",
 		Input: []tf.Input{
-			logits, num_samples,
+			input_indices, input_values, input_shape, reduction_axes,
 		},
 		Attrs: attrs,
 	}
@@ -10335,53 +10204,61 @@ func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional
 	return op.Output(0)
 }
 
-// Returns the truth value of NOT x element-wise.
-func LogicalNot(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns element-wise remainder of division. This emulates C semantics in that
+//
+// the result here is consistent with a truncating divide. E.g. `truncate(x / y) *
+// y + truncate_mod(x, y) = x`.
+//
+// *NOTE*: `TruncateMod` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func TruncateMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LogicalNot",
+		Type: "TruncateMod",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// 3D real-valued fast Fourier transform.
+// Inverse 2D real-valued fast Fourier transform.
 //
-// Computes the 3-dimensional discrete Fourier transform of a real-valued signal
-// over the inner-most 3 dimensions of `input`.
+// Computes the inverse 2-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most 2 dimensions of `input`.
 //
-// Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
-// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
-// of `output`: the zero-frequency term, followed by the `fft_length / 2`
-// positive-frequency terms.
+// The inner-most 2 dimensions of `input` are assumed to be the result of `RFFT2D`:
+// The inner-most dimension contains the `fft_length / 2 + 1` unique components of
+// the DFT of a real-valued signal. If `fft_length` is not provided, it is computed
+// from the size of the inner-most 2 dimensions of `input`. If the FFT length used
+// to compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
 //
-// Along each axis `RFFT3D` is computed on, if `fft_length` is smaller than the
+// Along each axis `IRFFT2D` is computed on, if `fft_length` (or
+// `fft_length / 2 + 1` for the inner-most dimension) is smaller than the
 // corresponding dimension of `input`, the dimension is cropped. If it is larger,
 // the dimension is padded with zeros.
 //
 // Arguments:
-//	input: A float32 tensor.
-//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [2]. The FFT length for each dimension.
 //
-// Returns A complex64 tensor of the same rank as `input`. The inner-most 3
-//   dimensions of `input` are replaced with the their 3D Fourier transform. The
-//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
-//   components.
+// Returns A float32 tensor of the same rank as `input`. The inner-most 2
+//   dimensions of `input` are replaced with the `fft_length` samples of their
+//   inverse 2D Fourier transform.
 //
 // @compatibility(numpy)
-// Equivalent to np.fft.rfftn with 3 dimensions.
+// Equivalent to np.fft.irfft2
 // @end_compatibility
-func RFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+func IRFFT2D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "RFFT3D",
+		Type: "IRFFT2D",
 		Input: []tf.Input{
 			input, fft_length,
 		},
@@ -10390,137 +10267,231 @@ func RFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Outp
 	return op.Output(0)
 }
 
-// TensorArrayV3Attr is an optional argument to TensorArrayV3.
-type TensorArrayV3Attr func(optionalAttr)
+// DecodeJpegAttr is an optional argument to DecodeJpeg.
+type DecodeJpegAttr func(optionalAttr)
 
-// TensorArrayV3ElementShape sets the optional element_shape attribute to value.
+// DecodeJpegChannels sets the optional channels attribute to value.
 //
-// value: The expected shape of an element, if known. Used to
-// validate the shapes of TensorArray elements. If this shape is not
-// fully specified, gathering zero-size TensorArrays is an error.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayV3ElementShape(value tf.Shape) TensorArrayV3Attr {
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodeJpegChannels(value int64) DecodeJpegAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["channels"] = value
 	}
 }
 
-// TensorArrayV3DynamicSize sets the optional dynamic_size attribute to value.
+// DecodeJpegRatio sets the optional ratio attribute to value.
 //
-// value: A boolean that determines whether writes to the TensorArray
-// are allowed to grow the size.  By default, this is not allowed.
-// If not specified, defaults to false
-func TensorArrayV3DynamicSize(value bool) TensorArrayV3Attr {
+// value: Downscaling ratio.
+// If not specified, defaults to 1
+func DecodeJpegRatio(value int64) DecodeJpegAttr {
 	return func(m optionalAttr) {
-		m["dynamic_size"] = value
+		m["ratio"] = value
 	}
 }
 
-// TensorArrayV3ClearAfterRead sets the optional clear_after_read attribute to value.
+// DecodeJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
 //
-// value: If true (default), Tensors in the TensorArray are cleared
-// after being read.  This disables multiple read semantics but allows early
-// release of memory.
+// value: If true use a slower but nicer upscaling of the
+// chroma planes (yuv420/422 only).
 // If not specified, defaults to true
-func TensorArrayV3ClearAfterRead(value bool) TensorArrayV3Attr {
+func DecodeJpegFancyUpscaling(value bool) DecodeJpegAttr {
 	return func(m optionalAttr) {
-		m["clear_after_read"] = value
+		m["fancy_upscaling"] = value
 	}
 }
 
-// TensorArrayV3IdenticalElementShapes sets the optional identical_element_shapes attribute to value.
+// DecodeJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
 //
-// value: If true (default is false), then all
-// elements in the TensorArray will be expected to have have identical shapes.
-// This allows certain behaviors, like dynamically checking for
-// consistent shapes on write, and being able to fill in properly
-// shaped zero tensors on stack -- even if the element_shape attribute
-// is not fully defined.
+// value: If true try to recover an image from truncated input.
 // If not specified, defaults to false
-func TensorArrayV3IdenticalElementShapes(value bool) TensorArrayV3Attr {
+func DecodeJpegTryRecoverTruncated(value bool) DecodeJpegAttr {
 	return func(m optionalAttr) {
-		m["identical_element_shapes"] = value
+		m["try_recover_truncated"] = value
 	}
 }
 
-// TensorArrayV3TensorArrayName sets the optional tensor_array_name attribute to value.
+// DecodeJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
 //
-// value: Overrides the name used for the temporary tensor_array
-// resource. Default value is the name of the 'TensorArray' op (which
-// is guaranteed unique).
+// value: The minimum required fraction of lines before a truncated
+// input is accepted.
+// If not specified, defaults to 1
+func DecodeJpegAcceptableFraction(value float32) DecodeJpegAttr {
+	return func(m optionalAttr) {
+		m["acceptable_fraction"] = value
+	}
+}
+
+// DecodeJpegDctMethod sets the optional dct_method attribute to value.
+//
+// value: string specifying a hint about the algorithm used for
+// decompression.  Defaults to "" which maps to a system-specific
+// default.  Currently valid values are ["INTEGER_FAST",
+// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
+// jpeg library changes to a version that does not have that specific
+// option.)
 // If not specified, defaults to ""
-func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr {
+func DecodeJpegDctMethod(value string) DecodeJpegAttr {
 	return func(m optionalAttr) {
-		m["tensor_array_name"] = value
+		m["dct_method"] = value
 	}
 }
 
-// An array of Tensors of given size.
+// Decode a JPEG-encoded image to a uint8 tensor.
 //
-// Write data via Write and read via Read or Pack.
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the JPEG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
+//
+// If needed, the JPEG-encoded image is transformed to match the requested number
+// of color channels.
+//
+// The attr `ratio` allows downscaling the image by an integer factor during
+// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+// downscaling the image later.
+//
+//
+// This op also supports decoding PNGs and non-animated GIFs since the interface is
+// the same, though it is cleaner to use `tf.image.decode_image`.
 //
 // Arguments:
-//	size: The size of the array.
-//	dtype: The type of the elements on the tensor_array.
+//	contents: 0-D.  The JPEG-encoded image.
 //
-// Returns The handle to the TensorArray.A scalar used to control gradient flow.
-func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) {
+// Returns 3-D with shape `[height, width, channels]`..
+func DecodeJpeg(scope *Scope, contents tf.Output, optional ...DecodeJpegAttr) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayV3",
+		Type: "DecodeJpeg",
 		Input: []tf.Input{
-			size,
+			contents,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// MaxPool3DAttr is an optional argument to MaxPool3D.
-type MaxPool3DAttr func(optionalAttr)
-
-// MaxPool3DDataFormat sets the optional data_format attribute to value.
+// Transforms a vector of brain.Example protos (as strings) into typed tensors.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func MaxPool3DDataFormat(value string) MaxPool3DAttr {
+// Arguments:
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	names: A vector containing the names of the serialized protos.
+// May contain, for example, table key (descriptive) names for the
+// corresponding serialized protos.  These are purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty vector if no names are available.
+// If non-empty, this vector must be the same length as "serialized".
+//	sparse_keys: A list of Nsparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: A list of Ndense string Tensors (scalars).
+// The keys expected in the Examples' features associated with dense values.
+//	dense_defaults: A list of Ndense Tensors (some may be empty).
+// dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	sparse_types: A list of Nsparse types; the data types of data in each Feature
+// given in sparse_keys.
+// Currently the ParseExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: A list of Ndense shapes; the shapes of data in each Feature
+// given in dense_keys.
+// The number of elements in the Feature corresponding to dense_key[j]
+// must always equal dense_shapes[j].NumEntries().
+// If dense_shapes[j] == (D0, D1, ..., DN) then the shape of output
+// Tensor dense_values[j] will be (|serialized|, D0, D1, ..., DN):
+// The dense outputs are just the inputs row-stacked by batch.
+// This works for dense_shapes[j] = (-1, D1, ..., DN).  In this case
+// the shape of the output Tensor dense_values[j] will be
+// (|serialized|, M, D1, .., DN), where M is the maximum number of blocks
+// of elements of length D1 * .... * DN, across all minibatch entries
+// in the input.  Any minibatch entry with less than M blocks of elements of
+// length D1 * ... * DN will be padded with the corresponding default_value
+// scalar element along the second dimension.
+func ParseExample(scope *Scope, serialized tf.Output, names tf.Output, sparse_keys []tf.Output, dense_keys []tf.Output, dense_defaults []tf.Output, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"sparse_types": sparse_types, "dense_shapes": dense_shapes}
+	opspec := tf.OpSpec{
+		Type: "ParseExample",
+		Input: []tf.Input{
+			serialized, names, tf.OutputList(sparse_keys), tf.OutputList(dense_keys), tf.OutputList(dense_defaults),
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseExample", err)
+		return
+	}
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
+}
+
+// VariableShapeAttr is an optional argument to VariableShape.
+type VariableShapeAttr func(optionalAttr)
+
+// VariableShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func VariableShapeOutType(value tf.DataType) VariableShapeAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["out_type"] = value
 	}
 }
 
-// Performs 3D max pooling on the input.
+// Returns the shape of the variable pointed to by `resource`.
 //
-// Arguments:
-//	input: Shape `[batch, depth, rows, cols, channels]` tensor to pool over.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// This operation returns a 1-D integer tensor representing the shape of `input`.
 //
-// Returns The max pooled output tensor.
-func MaxPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DAttr) (output tf.Output) {
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func VariableShape(scope *Scope, input tf.Output, optional ...VariableShapeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPool3D",
+		Type: "VariableShape",
 		Input: []tf.Input{
 			input,
 		},
@@ -10530,171 +10501,55 @@ func MaxPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, pa
 	return op.Output(0)
 }
 
-// Computes the gradients of 3-D convolution with respect to the input.
+// Computes softmax cross entropy cost and gradients to backpropagate.
 //
-// DEPRECATED at GraphDef version 10: Use Conv3DBackpropInputV2
+// Unlike `SoftmaxCrossEntropyWithLogits`, this operation does not accept
+// a matrix of label probabilities, but rather a single label per row
+// of features.  This label is considered to have probability 1.0 for the
+// given row.
+//
+// Inputs are the logits, not probabilities.
 //
 // Arguments:
-//	input: Shape `[batch, depth, rows, cols, in_channels]`.
-//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
-// `in_channels` must match between `input` and `filter`.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
-	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropInput",
-		Input: []tf.Input{
-			input, filter, out_backprop,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceApplyProximalAdagradAttr is an optional argument to ResourceApplyProximalAdagrad.
-type ResourceApplyProximalAdagradAttr func(optionalAttr)
-
-// ResourceApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
-//
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyProximalAdagradUseLocking(value bool) ResourceApplyProximalAdagradAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' and '*accum' according to FOBOS with Adagrad learning rate.
-//
-// accum += grad * grad
-// prox_v = var - lr * grad * (1 / sqrt(accum))
-// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	grad: The gradient.
+//	features: batch_size x num_classes matrix
+//	labels: batch_size vector with values in [0, num_classes).
+// This is the label for the given minibatch entry.
 //
-// Returns the created operation.
-func ResourceApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, optional ...ResourceApplyProximalAdagradAttr) (o *tf.Operation) {
+// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
+func SparseSoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyProximalAdagrad",
+		Type: "SparseSoftmaxCrossEntropyWithLogits",
 		Input: []tf.Input{
-			var_, accum, lr, l1, l2, grad,
+			features, labels,
 		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
-}
-
-// MutableHashTableOfTensorsV2Attr is an optional argument to MutableHashTableOfTensorsV2.
-type MutableHashTableOfTensorsV2Attr func(optionalAttr)
-
-// MutableHashTableOfTensorsV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableHashTableOfTensorsV2Container(value string) MutableHashTableOfTensorsV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MutableHashTableOfTensorsV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableHashTableOfTensorsV2SharedName(value string) MutableHashTableOfTensorsV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// MutableHashTableOfTensorsV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
-// If not specified, defaults to false
-func MutableHashTableOfTensorsV2UseNodeNameSharing(value bool) MutableHashTableOfTensorsV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
-	}
-}
-
-// MutableHashTableOfTensorsV2ValueShape sets the optional value_shape attribute to value.
-// If not specified, defaults to <>
-func MutableHashTableOfTensorsV2ValueShape(value tf.Shape) MutableHashTableOfTensorsV2Attr {
-	return func(m optionalAttr) {
-		m["value_shape"] = value
-	}
-}
-
-// Creates an empty hash table.
-//
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a vector. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
-//
-// Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
-//
-// Returns Handle to a table.
-func MutableHashTableOfTensorsV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableOfTensorsV2Attr) (table_handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "MutableHashTableOfTensorsV2",
-
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Inverse 2D fast Fourier transform.
+// Fast Fourier transform.
 //
-// Computes the inverse 2-dimensional discrete Fourier transform over the
-// inner-most 2 dimensions of `input`.
+// Computes the 1-dimensional discrete Fourier transform over the inner-most
+// dimension of `input`.
 //
 // Arguments:
 //	input: A complex64 tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
+// Returns A complex64 tensor of the same shape as `input`. The inner-most
+//   dimension of `input` is replaced with its 1D Fourier transform.
 //
 // @compatibility(numpy)
-// Equivalent to np.fft.ifft2
+// Equivalent to np.fft.fft
 // @end_compatibility
-func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
+func FFT(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IFFT2D",
+		Type: "FFT",
 		Input: []tf.Input{
 			input,
 		},
@@ -10703,95 +10558,108 @@ func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Creates a tensor filled with a scalar value.
-//
-// This operation creates a tensor of shape `dims` and fills it with `value`.
-//
-// For example:
-//
-// ```
-// # Output tensor has shape [2, 3].
-// fill([2, 3], 9) ==> [[9, 9, 9]
-//                      [9, 9, 9]]
-// ```
+// Transforms a serialized tensorflow.TensorProto proto into a Tensor.
 //
 // Arguments:
-//	dims: 1-D. Represents the shape of the output tensor.
-//	value: 0-D (scalar). Value to fill the returned tensor.
+//	serialized: A scalar string containing a serialized TensorProto proto.
+//	out_type: The type of the serialized tensor.  The provided type must match the
+// type of the serialized tensor and no implicit conversion will take place.
 //
-// @compatibility(numpy)
-// Equivalent to np.full
-// @end_compatibility
-func Fill(scope *Scope, dims tf.Output, value tf.Output) (output tf.Output) {
+// Returns A Tensor of type `out_type`.
+func ParseTensor(scope *Scope, serialized tf.Output, out_type tf.DataType) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "Fill",
+		Type: "ParseTensor",
 		Input: []tf.Input{
-			dims, value,
+			serialized,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// 2D fast Fourier transform.
+// MaxPoolWithArgmaxAttr is an optional argument to MaxPoolWithArgmax.
+type MaxPoolWithArgmaxAttr func(optionalAttr)
+
+// MaxPoolWithArgmaxTargmax sets the optional Targmax attribute to value.
+// If not specified, defaults to DT_INT64
+func MaxPoolWithArgmaxTargmax(value tf.DataType) MaxPoolWithArgmaxAttr {
+	return func(m optionalAttr) {
+		m["Targmax"] = value
+	}
+}
+
+// Performs max pooling on the input and outputs both max values and indices.
 //
-// Computes the 2-dimensional discrete Fourier transform over the inner-most
-// 2 dimensions of `input`.
+// The indices in `argmax` are flattened, so that a maximum value at position
+// `[b, y, x, c]` becomes flattened index
+// `((b * height + y) * width + x) * channels + c`.
 //
-// Arguments:
-//	input: A complex64 tensor.
+// The indices returned are always in `[0, height) x [0, width)` before flattening,
+// even if padding is involved and the mathematically correct answer is outside
+// (either negative or too large).  This is a bug, but fixing it is difficult to do
+// in a safe backwards compatible way, especially due to flattening.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
-//   dimensions of `input` are replaced with their 2D Fourier transform.
+// Arguments:
+//	input: 4-D with shape `[batch, height, width, channels]`.  Input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// @compatibility(numpy)
-// Equivalent to np.fft.fft2
-// @end_compatibility
-func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns The max pooled output tensor.4-D.  The flattened indices of the max values chosen for each output.
+func MaxPoolWithArgmax(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolWithArgmaxAttr) (output tf.Output, argmax tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "FFT2D",
+		Type: "MaxPoolWithArgmax",
 		Input: []tf.Input{
 			input,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent.
-type ResourceApplyProximalGradientDescentAttr func(optionalAttr)
+// ResourceSparseApplyAdagradDAAttr is an optional argument to ResourceSparseApplyAdagradDA.
+type ResourceSparseApplyAdagradDAAttr func(optionalAttr)
 
-// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
+// ResourceSparseApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
 //
-// value: If True, the subtraction will be protected by a lock;
-// otherwise the behavior is undefined, but may exhibit less contention.
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr {
+func ResourceSparseApplyAdagradDAUseLocking(value bool) ResourceSparseApplyAdagradDAAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// Update '*var' as FOBOS algorithm with fixed learning rate.
-//
-// prox_v = var - alpha * delta
-// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
+// Update entries in '*var' and '*accum' according to the proximal adagrad scheme.
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	alpha: Scaling factor. Must be a scalar.
+//	gradient_accumulator: Should be from a Variable().
+//	gradient_squared_accumulator: Should be from a Variable().
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
+//	lr: Learning rate. Must be a scalar.
 //	l1: L1 regularization. Must be a scalar.
 //	l2: L2 regularization. Must be a scalar.
-//	delta: The change.
+//	global_step: Training step number. Must be a scalar.
 //
 // Returns the created operation.
-func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) {
+func ResourceSparseApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, indices tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceSparseApplyAdagradDAAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10800,154 +10668,193 @@ func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyProximalGradientDescent",
+		Type: "ResourceSparseApplyAdagradDA",
 		Input: []tf.Input{
-			var_, alpha, l1, l2, delta,
+			var_, gradient_accumulator, gradient_squared_accumulator, grad, indices, lr, l1, l2, global_step,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Computes the gradient for the sqrt of `x` wrt its input.
+// EncodeJpegAttr is an optional argument to EncodeJpeg.
+type EncodeJpegAttr func(optionalAttr)
+
+// EncodeJpegFormat sets the optional format attribute to value.
 //
-// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy`
-// is the corresponding input gradient.
-func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: Per pixel image format.
+// If not specified, defaults to ""
+func EncodeJpegFormat(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["format"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SqrtGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
+}
+
+// EncodeJpegQuality sets the optional quality attribute to value.
+//
+// value: Quality of the compression from 0 to 100 (higher is better and slower).
+// If not specified, defaults to 95
+func EncodeJpegQuality(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["quality"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Get the value of the tensor specified by its handle.
+// EncodeJpegProgressive sets the optional progressive attribute to value.
 //
-// Arguments:
-//	handle: The handle for a tensor stored in the session state.
-//	dtype: The type of the output value.
+// value: If True, create a JPEG that loads progressively (coarse to fine).
+// If not specified, defaults to false
+func EncodeJpegProgressive(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["progressive"] = value
+	}
+}
+
+// EncodeJpegOptimizeSize sets the optional optimize_size attribute to value.
 //
-// Returns The tensor for the given handle.
-func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: If True, spend CPU/RAM to reduce size with no quality change.
+// If not specified, defaults to false
+func EncodeJpegOptimizeSize(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["optimize_size"] = value
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	opspec := tf.OpSpec{
-		Type: "GetSessionTensor",
-		Input: []tf.Input{
-			handle,
-		},
-		Attrs: attrs,
+}
+
+// EncodeJpegChromaDownsampling sets the optional chroma_downsampling attribute to value.
+//
+// value: See http://en.wikipedia.org/wiki/Chroma_subsampling.
+// If not specified, defaults to true
+func EncodeJpegChromaDownsampling(value bool) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["chroma_downsampling"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns x - y element-wise.
+// EncodeJpegDensityUnit sets the optional density_unit attribute to value.
 //
-// *NOTE*: `Subtract` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: Unit used to specify `x_density` and `y_density`:
+// pixels per inch (`'in'`) or centimeter (`'cm'`).
+// If not specified, defaults to "in"
+func EncodeJpegDensityUnit(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["density_unit"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Sub",
-		Input: []tf.Input{
-			x, y,
-		},
+}
+
+// EncodeJpegXDensity sets the optional x_density attribute to value.
+//
+// value: Horizontal pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegXDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["x_density"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes softmax cross entropy cost and gradients to backpropagate.
+// EncodeJpegYDensity sets the optional y_density attribute to value.
 //
-// Inputs are the logits, not probabilities.
+// value: Vertical pixels per density unit.
+// If not specified, defaults to 300
+func EncodeJpegYDensity(value int64) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["y_density"] = value
+	}
+}
+
+// EncodeJpegXmpMetadata sets the optional xmp_metadata attribute to value.
+//
+// value: If not empty, embed this XMP metadata in the image header.
+// If not specified, defaults to ""
+func EncodeJpegXmpMetadata(value string) EncodeJpegAttr {
+	return func(m optionalAttr) {
+		m["xmp_metadata"] = value
+	}
+}
+
+// JPEG-encode an image.
+//
+// `image` is a 3-D uint8 Tensor of shape `[height, width, channels]`.
+//
+// The attr `format` can be used to override the color format of the encoded
+// output.  Values can be:
+//
+// *   `''`: Use a default format based on the number of channels in the image.
+// *   `grayscale`: Output a grayscale JPEG image.  The `channels` dimension
+//     of `image` must be 1.
+// *   `rgb`: Output an RGB JPEG image. The `channels` dimension
+//     of `image` must be 3.
+//
+// If `format` is not specified or is the empty string, a default format is picked
+// in function of the number of channels in `image`:
+//
+// *   1: Output a grayscale image.
+// *   3: Output an RGB image.
 //
 // Arguments:
-//	features: batch_size x num_classes matrix
-//	labels: batch_size x num_classes matrix
-// The caller must ensure that each batch of labels represents a valid
-// probability distribution.
+//	image: 3-D with shape `[height, width, channels]`.
 //
-// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
-func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
+// Returns 0-D. JPEG-encoded image.
+func EncodeJpeg(scope *Scope, image tf.Output, optional ...EncodeJpegAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SoftmaxCrossEntropyWithLogits",
+		Type: "EncodeJpeg",
 		Input: []tf.Input{
-			features, labels,
+			image,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// ReduceJoinAttr is an optional argument to ReduceJoin.
-type ReduceJoinAttr func(optionalAttr)
+// MultinomialAttr is an optional argument to Multinomial.
+type MultinomialAttr func(optionalAttr)
 
-// ReduceJoinKeepDims sets the optional keep_dims attribute to value.
+// MultinomialSeed sets the optional seed attribute to value.
 //
-// value: If `True`, retain reduced dimensions with length `1`.
-// If not specified, defaults to false
-func ReduceJoinKeepDims(value bool) ReduceJoinAttr {
+// value: If either seed or seed2 is set to be non-zero, the internal random number
+// generator is seeded by the given seed.  Otherwise, a random seed is used.
+// If not specified, defaults to 0
+func MultinomialSeed(value int64) MultinomialAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["seed"] = value
 	}
 }
 
-// ReduceJoinSeparator sets the optional separator attribute to value.
+// MultinomialSeed2 sets the optional seed2 attribute to value.
 //
-// value: The separator to use when joining.
-// If not specified, defaults to ""
-func ReduceJoinSeparator(value string) ReduceJoinAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func MultinomialSeed2(value int64) MultinomialAttr {
 	return func(m optionalAttr) {
-		m["separator"] = value
+		m["seed2"] = value
 	}
 }
 
-// Joins a string Tensor across the given dimensions.
-//
-// Computes the string join across dimensions in the given string Tensor of shape
-// `[d_0, d_1, ..., d_n-1]`.  Returns a new Tensor created by joining the input
-// strings with the given separator (default: empty string).  Negative indices are
-// counted backwards from the end, with `-1` being equivalent to `n - 1`.
-//
-// For example:
-//
-// ```python
-// # tensor `a` is [["a", "b"], ["c", "d"]]
-// tf.reduce_join(a, 0) ==> ["ac", "bd"]
-// tf.reduce_join(a, 1) ==> ["ab", "cd"]
-// tf.reduce_join(a, -2) = tf.reduce_join(a, 0) ==> ["ac", "bd"]
-// tf.reduce_join(a, -1) = tf.reduce_join(a, 1) ==> ["ab", "cd"]
-// tf.reduce_join(a, 0, keep_dims=True) ==> [["ac", "bd"]]
-// tf.reduce_join(a, 1, keep_dims=True) ==> [["ab"], ["cd"]]
-// tf.reduce_join(a, 0, separator=".") ==> ["a.c", "b.d"]
-// tf.reduce_join(a, [0, 1]) ==> ["acbd"]
-// tf.reduce_join(a, [1, 0]) ==> ["abcd"]
-// tf.reduce_join(a, []) ==> ["abcd"]
-// ```
+// MultinomialOutputDtype sets the optional output_dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func MultinomialOutputDtype(value tf.DataType) MultinomialAttr {
+	return func(m optionalAttr) {
+		m["output_dtype"] = value
+	}
+}
+
+// Draws samples from a multinomial distribution.
 //
 // Arguments:
-//	inputs: The input to be joined.  All reduced indices must have non-zero size.
-//	reduction_indices: The dimensions to reduce over.  Dimensions are reduced in the
-// order specified.  Omitting `reduction_indices` is equivalent to passing
-// `[n-1, n-2, ..., 0]`.  Negative indices from `-n` to `-1` are supported.
+//	logits: 2-D Tensor with shape `[batch_size, num_classes]`.  Each slice `[i, :]`
+// represents the unnormalized log probabilities for all classes.
+//	num_samples: 0-D.  Number of independent samples to draw for each row slice.
 //
-// Returns Has shape equal to that of the input with reduced dimensions removed or
-// set to `1` depending on `keep_dims`.
-func ReduceJoin(scope *Scope, inputs tf.Output, reduction_indices tf.Output, optional ...ReduceJoinAttr) (output tf.Output) {
+// Returns 2-D Tensor with shape `[batch_size, num_samples]`.  Each slice `[i, :]`
+// contains the drawn class labels with range `[0, num_classes)`.
+func Multinomial(scope *Scope, logits tf.Output, num_samples tf.Output, optional ...MultinomialAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -10956,9 +10863,9 @@ func ReduceJoin(scope *Scope, inputs tf.Output, reduction_indices tf.Output, opt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ReduceJoin",
+		Type: "Multinomial",
 		Input: []tf.Input{
-			inputs, reduction_indices,
+			logits, num_samples,
 		},
 		Attrs: attrs,
 	}
@@ -10966,13 +10873,13 @@ func ReduceJoin(scope *Scope, inputs tf.Output, reduction_indices tf.Output, opt
 	return op.Output(0)
 }
 
-// Computes cos of x element-wise.
-func Cos(scope *Scope, x tf.Output) (y tf.Output) {
+// Returns the truth value of NOT x element-wise.
+func LogicalNot(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Cos",
+		Type: "LogicalNot",
 		Input: []tf.Input{
 			x,
 		},
@@ -10981,184 +10888,245 @@ func Cos(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad.
-type FusedBatchNormGradAttr func(optionalAttr)
-
-// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value.
+// 3D real-valued fast Fourier transform.
 //
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr {
-	return func(m optionalAttr) {
-		m["epsilon"] = value
-	}
-}
-
-// FusedBatchNormGradDataFormat sets the optional data_format attribute to value.
+// Computes the 3-dimensional discrete Fourier transform of a real-valued signal
+// over the inner-most 3 dimensions of `input`.
 //
-// value: The data format for y_backprop, x, x_backprop.
-// Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
+// Since the DFT of a real signal is Hermitian-symmetric, `RFFT3D` only returns the
+// `fft_length / 2 + 1` unique components of the FFT for the inner-most dimension
+// of `output`: the zero-frequency term, followed by the `fft_length / 2`
+// positive-frequency terms.
+//
+// Along each axis `RFFT3D` is computed on, if `fft_length` is smaller than the
+// corresponding dimension of `input`, the dimension is cropped. If it is larger,
+// the dimension is padded with zeros.
+//
+// Arguments:
+//	input: A float32 tensor.
+//	fft_length: An int32 tensor of shape [3]. The FFT length for each dimension.
+//
+// Returns A complex64 tensor of the same rank as `input`. The inner-most 3
+//   dimensions of `input` are replaced with the their 3D Fourier transform. The
+//   inner-most dimension contains `fft_length / 2 + 1` unique frequency
+//   components.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.rfftn with 3 dimensions.
+// @end_compatibility
+func RFFT3D(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "RFFT3D",
+		Input: []tf.Input{
+			input, fft_length,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FusedBatchNormGradIsTraining sets the optional is_training attribute to value.
+// TensorArrayV3Attr is an optional argument to TensorArrayV3.
+type TensorArrayV3Attr func(optionalAttr)
+
+// TensorArrayV3ElementShape sets the optional element_shape attribute to value.
 //
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
+// value: The expected shape of an element, if known. Used to
+// validate the shapes of TensorArray elements. If this shape is not
+// fully specified, gathering zero-size TensorArrays is an error.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayV3ElementShape(value tf.Shape) TensorArrayV3Attr {
+	return func(m optionalAttr) {
+		m["element_shape"] = value
+	}
+}
+
+// TensorArrayV3DynamicSize sets the optional dynamic_size attribute to value.
+//
+// value: A boolean that determines whether writes to the TensorArray
+// are allowed to grow the size.  By default, this is not allowed.
+// If not specified, defaults to false
+func TensorArrayV3DynamicSize(value bool) TensorArrayV3Attr {
+	return func(m optionalAttr) {
+		m["dynamic_size"] = value
+	}
+}
+
+// TensorArrayV3ClearAfterRead sets the optional clear_after_read attribute to value.
+//
+// value: If true (default), Tensors in the TensorArray are cleared
+// after being read.  This disables multiple read semantics but allows early
+// release of memory.
 // If not specified, defaults to true
-func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr {
+func TensorArrayV3ClearAfterRead(value bool) TensorArrayV3Attr {
 	return func(m optionalAttr) {
-		m["is_training"] = value
+		m["clear_after_read"] = value
 	}
 }
 
-// Gradient for batch normalization.
+// TensorArrayV3IdenticalElementShapes sets the optional identical_element_shapes attribute to value.
 //
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// value: If true (default is false), then all
+// elements in the TensorArray will be expected to have have identical shapes.
+// This allows certain behaviors, like dynamically checking for
+// consistent shapes on write, and being able to fill in properly
+// shaped zero tensors on stack -- even if the element_shape attribute
+// is not fully defined.
+// If not specified, defaults to false
+func TensorArrayV3IdenticalElementShapes(value bool) TensorArrayV3Attr {
+	return func(m optionalAttr) {
+		m["identical_element_shapes"] = value
+	}
+}
+
+// TensorArrayV3TensorArrayName sets the optional tensor_array_name attribute to value.
+//
+// value: Overrides the name used for the temporary tensor_array
+// resource. Default value is the name of the 'TensorArray' op (which
+// is guaranteed unique).
+// If not specified, defaults to ""
+func TensorArrayV3TensorArrayName(value string) TensorArrayV3Attr {
+	return func(m optionalAttr) {
+		m["tensor_array_name"] = value
+	}
+}
+
+// An array of Tensors of given size.
+//
+// Write data via Write and read via Read or Pack.
 //
 // Arguments:
-//	y_backprop: A 4D Tensor for the gradient with respect to y.
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
-// mean to be reused in gradient computation. When is_training is
-// False, a 1D Tensor for the population mean to be reused in both
-// 1st and 2nd order gradient computation.
-//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
-// variance (inverted variance in the cuDNN case) to be reused in
-// gradient computation. When is_training is False, a 1D Tensor
-// for the population variance to be reused in both 1st and 2nd
-// order gradient computation.
+//	size: The size of the array.
+//	dtype: The type of the elements on the tensor_array.
 //
-// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
-// in FusedBatchNorm.
-func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
+// Returns The handle to the TensorArray.A scalar used to control gradient flow.
+func TensorArrayV3(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV3Attr) (handle tf.Output, flow tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNormGrad",
+		Type: "TensorArrayV3",
 		Input: []tf.Input{
-			y_backprop, x, scale, reserve_space_1, reserve_space_2,
+			size,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0), op.Output(1)
 }
 
-// TopKAttr is an optional argument to TopK.
-type TopKAttr func(optionalAttr)
+// MaxPool3DAttr is an optional argument to MaxPool3D.
+type MaxPool3DAttr func(optionalAttr)
 
-// TopKSorted sets the optional sorted attribute to value.
+// MaxPool3DDataFormat sets the optional data_format attribute to value.
 //
-// value: If true the resulting `k` elements will be sorted by the values in
-// descending order.
-// If not specified, defaults to true
-func TopKSorted(value bool) TopKAttr {
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func MaxPool3DDataFormat(value string) MaxPool3DAttr {
 	return func(m optionalAttr) {
-		m["sorted"] = value
+		m["data_format"] = value
 	}
 }
 
-// Finds values and indices of the `k` largest elements for the last dimension.
-//
-// DEPRECATED at GraphDef version 7: Use TopKV2 instead
-//
-// If the input is a vector (rank-1), finds the `k` largest entries in the vector
-// and outputs their values and indices as vectors.  Thus `values[j]` is the
-// `j`-th largest entry in `input`, and its index is `indices[j]`.
-//
-// For matrices (resp. higher rank input), computes the top `k` entries in each
-// row (resp. vector along the last dimension).  Thus,
-//
-//     values.shape = indices.shape = input.shape[:-1] + [k]
-//
-// If two elements are equal, the lower-index element appears first.
-//
-// If `k` varies dynamically, use `TopKV2` below.
+// Performs 3D max pooling on the input.
 //
 // Arguments:
-//	input: 1-D or higher with last dimension at least `k`.
-//	k: Number of top elements to look for along the last dimension (along each
-// row for matrices).
+//	input: Shape `[batch, depth, rows, cols, channels]` tensor to pool over.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
 //
-// Returns The `k` largest elements along each last dimensional slice.The indices of `values` within the last dimension of `input`.
-func TopK(scope *Scope, input tf.Output, k int64, optional ...TopKAttr) (values tf.Output, indices tf.Output) {
+// Returns The max pooled output tensor.
+func MaxPool3D(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"k": k}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TopK",
+		Type: "MaxPool3D",
 		Input: []tf.Input{
 			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
-//
-// The Hurwitz zeta function is defined as:
+// Computes the gradients of 3-D convolution with respect to the input.
 //
+// DEPRECATED at GraphDef version 10: Use Conv3DBackpropInputV2
 //
-// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
-func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
+// Arguments:
+//	input: Shape `[batch, depth, rows, cols, in_channels]`.
+//	filter: Shape `[depth, rows, cols, in_channels, out_channels]`.
+// `in_channels` must match between `input` and `filter`.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3DBackpropInput(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "Zeta",
+		Type: "Conv3DBackpropInput",
 		Input: []tf.Input{
-			x, q,
+			input, filter, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ProdAttr is an optional argument to Prod.
-type ProdAttr func(optionalAttr)
+// ResourceApplyProximalAdagradAttr is an optional argument to ResourceApplyProximalAdagrad.
+type ResourceApplyProximalAdagradAttr func(optionalAttr)
 
-// ProdKeepDims sets the optional keep_dims attribute to value.
+// ResourceApplyProximalAdagradUseLocking sets the optional use_locking attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
 // If not specified, defaults to false
-func ProdKeepDims(value bool) ProdAttr {
+func ResourceApplyProximalAdagradUseLocking(value bool) ResourceApplyProximalAdagradAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Computes the product of elements across dimensions of a tensor.
+// Update '*var' and '*accum' according to FOBOS with Adagrad learning rate.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// accum += grad * grad
+// prox_v = var - lr * grad * (1 / sqrt(accum))
+// var = sign(prox_v)/(1+lr*l2) * max{|prox_v|-lr*l1,0}
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	grad: The gradient.
 //
-// Returns The reduced tensor.
-func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
+// Returns the created operation.
+func ResourceApplyProximalAdagrad(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, optional ...ResourceApplyProximalAdagradAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -11167,95 +11135,104 @@ func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Prod",
+		Type: "ResourceApplyProximalAdagrad",
 		Input: []tf.Input{
-			input, axis,
+			var_, accum, lr, l1, l2, grad,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
-type FusedResizeAndPadConv2DAttr func(optionalAttr)
+// MutableHashTableOfTensorsV2Attr is an optional argument to MutableHashTableOfTensorsV2.
+type MutableHashTableOfTensorsV2Attr func(optionalAttr)
 
-// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
+// MutableHashTableOfTensorsV2Container sets the optional container attribute to value.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1),
-// which exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func MutableHashTableOfTensorsV2Container(value string) MutableHashTableOfTensorsV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MutableHashTableOfTensorsV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableHashTableOfTensorsV2SharedName(value string) MutableHashTableOfTensorsV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// MutableHashTableOfTensorsV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 // If not specified, defaults to false
-func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
+func MutableHashTableOfTensorsV2UseNodeNameSharing(value bool) MutableHashTableOfTensorsV2Attr {
 	return func(m optionalAttr) {
-		m["resize_align_corners"] = value
+		m["use_node_name_sharing"] = value
 	}
 }
 
-// Performs a resize and padding as a preprocess during a convolution.
+// MutableHashTableOfTensorsV2ValueShape sets the optional value_shape attribute to value.
+// If not specified, defaults to <>
+func MutableHashTableOfTensorsV2ValueShape(value tf.Shape) MutableHashTableOfTensorsV2Attr {
+	return func(m optionalAttr) {
+		m["value_shape"] = value
+	}
+}
+
+// Creates an empty hash table.
 //
-// It's often possible to do spatial transformations more efficiently as part of
-// the packing stage of a convolution, so this op allows for an optimized
-// implementation where these stages are fused together. This prevents the need to
-// write out the intermediate results as whole tensors, reducing memory pressure,
-// and we can get some latency gains by merging the transformation calculations.
-// The data_format attribute for Conv2D isn't supported by this op, and defaults to
-// 'NHWC' order.
-// Internally this op uses a single per-graph scratch buffer, which means that it
-// will block if multiple versions are being run in parallel. This is because this
-// operator is primarily an optimization to minimize memory usage.
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a vector. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
-//	size: A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
-//	paddings: A two-column matrix specifying the padding sizes. The number of
-// rows must be the same as the rank of `input`.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, out_channels]`.
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
 //
-//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
-// of `input`. Must be in the same order as the dimension specified with format.
-//	padding: The type of padding algorithm to use.
-func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string, optional ...FusedResizeAndPadConv2DAttr) (output tf.Output) {
+// Returns Handle to a table.
+func MutableHashTableOfTensorsV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableOfTensorsV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedResizeAndPadConv2D",
-		Input: []tf.Input{
-			input, size, paddings, filter,
-		},
+		Type: "MutableHashTableOfTensorsV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse 3D fast Fourier transform.
+// Inverse 2D fast Fourier transform.
 //
-// Computes the inverse 3-dimensional discrete Fourier transform over the
-// inner-most 3 dimensions of `input`.
+// Computes the inverse 2-dimensional discrete Fourier transform over the
+// inner-most 2 dimensions of `input`.
 //
 // Arguments:
 //	input: A complex64 tensor.
 //
-// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
-//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their inverse 2D Fourier transform.
 //
 // @compatibility(numpy)
-// Equivalent to np.fft.ifftn with 3 dimensions.
+// Equivalent to np.fft.ifft2
 // @end_compatibility
-func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+func IFFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "IFFT3D",
+		Type: "IFFT2D",
 		Input: []tf.Input{
 			input,
 		},
@@ -11264,126 +11241,95 @@ func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
 	return op.Output(0)
 }
 
-// Adds `bias` to `value`.
+// Creates a tensor filled with a scalar value.
 //
-// This is a deprecated version of BiasAdd and will be soon removed.
+// This operation creates a tensor of shape `dims` and fills it with `value`.
 //
-// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
-// Broadcasting is supported, so `value` may have any number of dimensions.
+// For example:
+//
+// ```
+// # Output tensor has shape [2, 3].
+// fill([2, 3], 9) ==> [[9, 9, 9]
+//                      [9, 9, 9]]
+// ```
 //
 // Arguments:
-//	value: Any number of dimensions.
-//	bias: 1-D with size the last dimension of `value`.
+//	dims: 1-D. Represents the shape of the output tensor.
+//	value: 0-D (scalar). Value to fill the returned tensor.
 //
-// Returns Broadcasted sum of `value` and `bias`.
-func BiasAddV1(scope *Scope, value tf.Output, bias tf.Output) (output tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.full
+// @end_compatibility
+func Fill(scope *Scope, dims tf.Output, value tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BiasAddV1",
+		Type: "Fill",
 		Input: []tf.Input{
-			value, bias,
+			dims, value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Reverses specific dimensions of a tensor.
-//
-// NOTE `tf.reverse` has now changed behavior in preparation for 1.0.
-// `tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.
-//
-// Given a `tensor`, and a `int32` tensor `axis` representing the set of
-// dimensions of `tensor` to reverse. This operation reverses each dimension
-// `i` for which there exists `j` s.t. `axis[j] == i`.
-//
-// `tensor` can have up to 8 dimensions. The number of dimensions specified
-// in `axis` may be 0 or more entries. If an index is specified more than
-// once, a InvalidArgument error is raised.
-//
-// For example:
-//
-// ```
-// # tensor 't' is [[[[ 0,  1,  2,  3],
-// #                  [ 4,  5,  6,  7],
-// #                  [ 8,  9, 10, 11]],
-// #                 [[12, 13, 14, 15],
-// #                  [16, 17, 18, 19],
-// #                  [20, 21, 22, 23]]]]
-// # tensor 't' shape is [1, 2, 3, 4]
-//
-// # 'dims' is [3] or 'dims' is [-1]
-// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
-//                         [ 7,  6,  5,  4],
-//                         [ 11, 10, 9, 8]],
-//                        [[15, 14, 13, 12],
-//                         [19, 18, 17, 16],
-//                         [23, 22, 21, 20]]]]
-//
-// # 'dims' is '[1]' (or 'dims' is '[-3]')
-// reverse(t, dims) ==> [[[[12, 13, 14, 15],
-//                         [16, 17, 18, 19],
-//                         [20, 21, 22, 23]
-//                        [[ 0,  1,  2,  3],
-//                         [ 4,  5,  6,  7],
-//                         [ 8,  9, 10, 11]]]]
+// 2D fast Fourier transform.
 //
-// # 'dims' is '[2]' (or 'dims' is '[-2]')
-// reverse(t, dims) ==> [[[[8, 9, 10, 11],
-//                         [4, 5, 6, 7],
-//                         [0, 1, 2, 3]]
-//                        [[20, 21, 22, 23],
-//                         [16, 17, 18, 19],
-//                         [12, 13, 14, 15]]]]
-// ```
+// Computes the 2-dimensional discrete Fourier transform over the inner-most
+// 2 dimensions of `input`.
 //
 // Arguments:
-//	tensor: Up to 8-D.
-//	axis: 1-D. The indices of the dimensions to reverse. Must be in the range
-// `[-rank(tensor), rank(tensor))`.
+//	input: A complex64 tensor.
 //
-// Returns The same shape as `tensor`.
-func ReverseV2(scope *Scope, tensor tf.Output, axis tf.Output) (output tf.Output) {
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 2
+//   dimensions of `input` are replaced with their 2D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.fft2
+// @end_compatibility
+func FFT2D(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReverseV2",
+		Type: "FFT2D",
 		Input: []tf.Input{
-			tensor, axis,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RealAttr is an optional argument to Real.
-type RealAttr func(optionalAttr)
+// ResourceApplyProximalGradientDescentAttr is an optional argument to ResourceApplyProximalGradientDescent.
+type ResourceApplyProximalGradientDescentAttr func(optionalAttr)
 
-// RealTout sets the optional Tout attribute to value.
-// If not specified, defaults to DT_FLOAT
-func RealTout(value tf.DataType) RealAttr {
+// ResourceApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
+//
+// value: If True, the subtraction will be protected by a lock;
+// otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyProximalGradientDescentUseLocking(value bool) ResourceApplyProximalGradientDescentAttr {
 	return func(m optionalAttr) {
-		m["Tout"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Returns the real part of a complex number.
+// Update '*var' as FOBOS algorithm with fixed learning rate.
 //
-// Given a tensor `input` of complex numbers, this operation returns a tensor of
-// type `float` that is the real part of each element in `input`. All elements in
-// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
-//  part returned by this operation and *b* is the imaginary part.
+// prox_v = var - alpha * delta
+// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
 //
-// For example:
+// Arguments:
+//	var_: Should be from a Variable().
+//	alpha: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	delta: The change.
 //
-// ```
-// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
-// tf.real(input) ==> [-2.25, 3.25]
-// ```
-func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
+// Returns the created operation.
+func ResourceApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, delta tf.Output, optional ...ResourceApplyProximalGradientDescentAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -11392,108 +11338,154 @@ func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Real",
+		Type: "ResourceApplyProximalGradientDescent",
 		Input: []tf.Input{
-			input,
+			var_, alpha, l1, l2, delta,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// AudioSummaryAttr is an optional argument to AudioSummary.
-type AudioSummaryAttr func(optionalAttr)
-
-// AudioSummaryMaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
+// Computes the gradient for the sqrt of `x` wrt its input.
 //
-// REQUIRES: value >= 1
-func AudioSummaryMaxOutputs(value int64) AudioSummaryAttr {
-	return func(m optionalAttr) {
-		m["max_outputs"] = value
+// Specifically, `grad = dy * 0.5 / y`, where `y = sqrt(x)`, and `dy`
+// is the corresponding input gradient.
+func SqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SqrtGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Outputs a `Summary` protocol buffer with audio.
-//
-// DEPRECATED at GraphDef version 15: Use AudioSummaryV2.
+// Get the value of the tensor specified by its handle.
 //
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+// Arguments:
+//	handle: The handle for a tensor stored in the session state.
+//	dtype: The type of the output value.
 //
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// Returns The tensor for the given handle.
+func GetSessionTensor(scope *Scope, handle tf.Output, dtype tf.DataType) (value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	opspec := tf.OpSpec{
+		Type: "GetSessionTensor",
+		Input: []tf.Input{
+			handle,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Returns x - y element-wise.
 //
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+// *NOTE*: `Subtract` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Sub(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Sub",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes softmax cross entropy cost and gradients to backpropagate.
+//
+// Inputs are the logits, not probabilities.
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
+//	features: batch_size x num_classes matrix
+//	labels: batch_size x num_classes matrix
+// The caller must ensure that each batch of labels represents a valid
+// probability distribution.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func AudioSummary(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate float32, optional ...AudioSummaryAttr) (summary tf.Output) {
+// Returns Per example loss (batch_size vector).backpropagated gradients (batch_size x num_classes matrix).
+func SoftmaxCrossEntropyWithLogits(scope *Scope, features tf.Output, labels tf.Output) (loss tf.Output, backprop tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"sample_rate": sample_rate}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AudioSummary",
+		Type: "SoftmaxCrossEntropyWithLogits",
 		Input: []tf.Input{
-			tag, tensor,
+			features, labels,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// QrAttr is an optional argument to Qr.
-type QrAttr func(optionalAttr)
+// ReduceJoinAttr is an optional argument to ReduceJoin.
+type ReduceJoinAttr func(optionalAttr)
 
-// QrFullMatrices sets the optional full_matrices attribute to value.
+// ReduceJoinKeepDims sets the optional keep_dims attribute to value.
 //
-// value: If true, compute full-sized `q` and `r`. If false
-// (the default), compute only the leading `P` columns of `q`.
+// value: If `True`, retain reduced dimensions with length `1`.
 // If not specified, defaults to false
-func QrFullMatrices(value bool) QrAttr {
+func ReduceJoinKeepDims(value bool) ReduceJoinAttr {
 	return func(m optionalAttr) {
-		m["full_matrices"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Computes the QR decompositions of one or more matrices.
+// ReduceJoinSeparator sets the optional separator attribute to value.
 //
-// Computes the QR decomposition of each inner matrix in `tensor` such that
-// `tensor[..., :, :] = q[..., :, :] * r[..., :,:])`
+// value: The separator to use when joining.
+// If not specified, defaults to ""
+func ReduceJoinSeparator(value string) ReduceJoinAttr {
+	return func(m optionalAttr) {
+		m["separator"] = value
+	}
+}
+
+// Joins a string Tensor across the given dimensions.
+//
+// Computes the string join across dimensions in the given string Tensor of shape
+// `[d_0, d_1, ..., d_n-1]`.  Returns a new Tensor created by joining the input
+// strings with the given separator (default: empty string).  Negative indices are
+// counted backwards from the end, with `-1` being equivalent to `n - 1`.
+//
+// For example:
 //
 // ```python
-// # a is a tensor.
-// # q is a tensor of orthonormal matrices.
-// # r is a tensor of upper triangular matrices.
-// q, r = qr(a)
-// q_full, r_full = qr(a, full_matrices=True)
+// # tensor `a` is [["a", "b"], ["c", "d"]]
+// tf.reduce_join(a, 0) ==> ["ac", "bd"]
+// tf.reduce_join(a, 1) ==> ["ab", "cd"]
+// tf.reduce_join(a, -2) = tf.reduce_join(a, 0) ==> ["ac", "bd"]
+// tf.reduce_join(a, -1) = tf.reduce_join(a, 1) ==> ["ab", "cd"]
+// tf.reduce_join(a, 0, keep_dims=True) ==> [["ac", "bd"]]
+// tf.reduce_join(a, 1, keep_dims=True) ==> [["ab"], ["cd"]]
+// tf.reduce_join(a, 0, separator=".") ==> ["a.c", "b.d"]
+// tf.reduce_join(a, [0, 1]) ==> ["acbd"]
+// tf.reduce_join(a, [1, 0]) ==> ["abcd"]
+// tf.reduce_join(a, []) ==> ["abcd"]
 // ```
 //
 // Arguments:
-//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//	inputs: The input to be joined.  All reduced indices must have non-zero size.
+//	reduction_indices: The dimensions to reduce over.  Dimensions are reduced in the
+// order specified.  Omitting `reduction_indices` is equivalent to passing
+// `[n-1, n-2, ..., 0]`.  Negative indices from `-n` to `-1` are supported.
 //
-// Returns Orthonormal basis for range of `a`. If `full_matrices` is `False` then
-// shape is `[..., M, P]`; if `full_matrices` is `True` then shape is
-// `[..., M, M]`.Triangular factor. If `full_matrices` is `False` then shape is
-// `[..., P, N]`. If `full_matrices` is `True` then shape is `[..., M, N]`.
-func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Output) {
+// Returns Has shape equal to that of the input with reduced dimensions removed or
+// set to `1` depending on `keep_dims`.
+func ReduceJoin(scope *Scope, inputs tf.Output, reduction_indices tf.Output, optional ...ReduceJoinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -11502,107 +11494,88 @@ func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Qr",
+		Type: "ReduceJoin",
 		Input: []tf.Input{
-			input,
+			inputs, reduction_indices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
-func BytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Computes cos of x element-wise.
+func Cos(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "BytesProducedStatsDataset",
+		Type: "Cos",
 		Input: []tf.Input{
-			input_dataset, tag,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceSparseApplyProximalGradientDescentAttr is an optional argument to ResourceSparseApplyProximalGradientDescent.
-type ResourceSparseApplyProximalGradientDescentAttr func(optionalAttr)
+// FusedBatchNormGradAttr is an optional argument to FusedBatchNormGrad.
+type FusedBatchNormGradAttr func(optionalAttr)
 
-// ResourceSparseApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
+// FusedBatchNormGradEpsilon sets the optional epsilon attribute to value.
 //
-// value: If True, the subtraction will be protected by a lock;
-// otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceSparseApplyProximalGradientDescentUseLocking(value bool) ResourceSparseApplyProximalGradientDescentAttr {
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormGradEpsilon(value float32) FusedBatchNormGradAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["epsilon"] = value
 	}
 }
 
-// Sparse update '*var' as FOBOS algorithm with fixed learning rate.
-//
-// That is for rows we have grad for, we update var as follows:
-// prox_v = var - alpha * grad
-// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	alpha: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	grad: The gradient.
-//	indices: A vector of indices into the first dimension of var and accum.
+// FusedBatchNormGradDataFormat sets the optional data_format attribute to value.
 //
-// Returns the created operation.
-func ResourceSparseApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalGradientDescentAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceSparseApplyProximalGradientDescent",
-		Input: []tf.Input{
-			var_, alpha, l1, l2, grad, indices,
-		},
-		Attrs: attrs,
+// value: The data format for y_backprop, x, x_backprop.
+// Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormGradDataFormat(value string) FusedBatchNormGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// MeanAttr is an optional argument to Mean.
-type MeanAttr func(optionalAttr)
-
-// MeanKeepDims sets the optional keep_dims attribute to value.
+// FusedBatchNormGradIsTraining sets the optional is_training attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func MeanKeepDims(value bool) MeanAttr {
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
+// If not specified, defaults to true
+func FusedBatchNormGradIsTraining(value bool) FusedBatchNormGradAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["is_training"] = value
 	}
 }
 
-// Computes the mean of elements across dimensions of a tensor.
+// Gradient for batch normalization.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	y_backprop: A 4D Tensor for the gradient with respect to y.
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	reserve_space_1: When is_training is True, a 1D Tensor for the computed batch
+// mean to be reused in gradient computation. When is_training is
+// False, a 1D Tensor for the population mean to be reused in both
+// 1st and 2nd order gradient computation.
+//	reserve_space_2: When is_training is True, a 1D Tensor for the computed batch
+// variance (inverted variance in the cuDNN case) to be reused in
+// gradient computation. When is_training is False, a 1D Tensor
+// for the population variance to be reused in both 1st and 2nd
+// order gradient computation.
 //
-// Returns The reduced tensor.
-func Mean(scope *Scope, input tf.Output, axis tf.Output, optional ...MeanAttr) (output tf.Output) {
+// Returns A 4D Tensor for the gradient with respect to x.A 1D Tensor for the gradient with respect to scale.A 1D Tensor for the gradient with respect to offset.Unused placeholder to match the mean input in FusedBatchNorm.Unused placeholder to match the variance input
+// in FusedBatchNorm.
+func FusedBatchNormGrad(scope *Scope, y_backprop tf.Output, x tf.Output, scale tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output, optional ...FusedBatchNormGradAttr) (x_backprop tf.Output, scale_backprop tf.Output, offset_backprop tf.Output, reserve_space_3 tf.Output, reserve_space_4 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -11611,229 +11584,189 @@ func Mean(scope *Scope, input tf.Output, axis tf.Output, optional ...MeanAttr) (
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Mean",
+		Type: "FusedBatchNormGrad",
 		Input: []tf.Input{
-			input, axis,
+			y_backprop, x, scale, reserve_space_1, reserve_space_2,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// InitializeTableFromTextFileV2Attr is an optional argument to InitializeTableFromTextFileV2.
-type InitializeTableFromTextFileV2Attr func(optionalAttr)
+// TopKAttr is an optional argument to TopK.
+type TopKAttr func(optionalAttr)
 
-// InitializeTableFromTextFileV2VocabSize sets the optional vocab_size attribute to value.
+// TopKSorted sets the optional sorted attribute to value.
 //
-// value: Number of elements of the file, use -1 if unknown.
-// If not specified, defaults to -1
-//
-// REQUIRES: value >= -1
-func InitializeTableFromTextFileV2VocabSize(value int64) InitializeTableFromTextFileV2Attr {
+// value: If true the resulting `k` elements will be sorted by the values in
+// descending order.
+// If not specified, defaults to true
+func TopKSorted(value bool) TopKAttr {
 	return func(m optionalAttr) {
-		m["vocab_size"] = value
+		m["sorted"] = value
 	}
 }
 
-// InitializeTableFromTextFileV2Delimiter sets the optional delimiter attribute to value.
+// Finds values and indices of the `k` largest elements for the last dimension.
 //
-// value: Delimiter to separate fields in a line.
-// If not specified, defaults to "\t"
-func InitializeTableFromTextFileV2Delimiter(value string) InitializeTableFromTextFileV2Attr {
-	return func(m optionalAttr) {
-		m["delimiter"] = value
-	}
-}
-
-// Initializes a table from a text file.
+// DEPRECATED at GraphDef version 7: Use TopKV2 instead
 //
-// It inserts one key-value pair into the table for each line of the file.
-// The key and value is extracted from the whole line content, elements from the
-// split line based on `delimiter` or the line number (starting from zero).
-// Where to extract the key and value from a line is specified by `key_index` and
-// `value_index`.
+// If the input is a vector (rank-1), finds the `k` largest entries in the vector
+// and outputs their values and indices as vectors.  Thus `values[j]` is the
+// `j`-th largest entry in `input`, and its index is `indices[j]`.
 //
-// - A value of -1 means use the line number(starting from zero), expects `int64`.
-// - A value of -2 means use the whole line content, expects `string`.
-// - A value >= 0 means use the index (starting at zero) of the split line based
-//   on `delimiter`.
+// For matrices (resp. higher rank input), computes the top `k` entries in each
+// row (resp. vector along the last dimension).  Thus,
+//
+//     values.shape = indices.shape = input.shape[:-1] + [k]
+//
+// If two elements are equal, the lower-index element appears first.
+//
+// If `k` varies dynamically, use `TopKV2` below.
 //
 // Arguments:
-//	table_handle: Handle to a table which will be initialized.
-//	filename: Filename of a vocabulary text file.
-//	key_index: Column index in a line to get the table `key` values from.
-//	value_index: Column index that represents information of a line to get the table
-// `value` values from.
+//	input: 1-D or higher with last dimension at least `k`.
+//	k: Number of top elements to look for along the last dimension (along each
+// row for matrices).
 //
-// Returns the created operation.
-func InitializeTableFromTextFileV2(scope *Scope, table_handle tf.Output, filename tf.Output, key_index int64, value_index int64, optional ...InitializeTableFromTextFileV2Attr) (o *tf.Operation) {
+// Returns The `k` largest elements along each last dimensional slice.The indices of `values` within the last dimension of `input`.
+func TopK(scope *Scope, input tf.Output, k int64, optional ...TopKAttr) (values tf.Output, indices tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_index": key_index, "value_index": value_index}
+	attrs := map[string]interface{}{"k": k}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "InitializeTableFromTextFileV2",
+		Type: "TopK",
 		Input: []tf.Input{
-			table_handle, filename,
+			input,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// QuantizedReluAttr is an optional argument to QuantizedRelu.
-type QuantizedReluAttr func(optionalAttr)
-
-// QuantizedReluOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedReluOutType(value tf.DataType) QuantizedReluAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Computes Quantized Rectified Linear: `max(features, 0)`
+// Compute the Hurwitz zeta function \\(\zeta(x, q)\\).
 //
-// Arguments:
+// The Hurwitz zeta function is defined as:
 //
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
 //
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+// \\(\zeta(x, q) = \sum_{n=0}^{\infty} (q + n)^{-x}\\)
+func Zeta(scope *Scope, x tf.Output, q tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedRelu",
+		Type: "Zeta",
 		Input: []tf.Input{
-			features, min_features, max_features,
+			x, q,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Reshapes a SparseTensor to represent values in a new dense shape.
-//
-// This operation has the same semantics as reshape on the represented dense
-// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
-//
-// If one component of `new_shape` is the special value -1, the size of that
-// dimension is computed so that the total dense size remains constant.  At
-// most one component of `new_shape` can be -1.  The number of dense elements
-// implied by `new_shape` must be the same as the number of dense elements
-// originally implied by `input_shape`.
+// ProdAttr is an optional argument to Prod.
+type ProdAttr func(optionalAttr)
+
+// ProdKeepDims sets the optional keep_dims attribute to value.
 //
-// Reshaping does not affect the order of values in the SparseTensor.
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func ProdKeepDims(value bool) ProdAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the product of elements across dimensions of a tensor.
 //
-// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
-// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
-// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
-// `output_shape` has length `R_out`.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
-// SparseTensor.
-//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
-//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
-// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
-// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
-// filled in.
-func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
+// Returns The reduced tensor.
+func Prod(scope *Scope, input tf.Output, axis tf.Output, optional ...ProdAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "SparseReshape",
-		Input: []tf.Input{
-			input_indices, input_shape, new_shape,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// Deprecated. Use TensorArraySplitV3
-//
-// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
-func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArraySplitV2",
+		Type: "Prod",
 		Input: []tf.Input{
-			handle, value, lengths, flow_in,
+			input, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// PackAttr is an optional argument to Pack.
-type PackAttr func(optionalAttr)
+// FusedResizeAndPadConv2DAttr is an optional argument to FusedResizeAndPadConv2D.
+type FusedResizeAndPadConv2DAttr func(optionalAttr)
 
-// PackAxis sets the optional axis attribute to value.
+// FusedResizeAndPadConv2DResizeAlignCorners sets the optional resize_align_corners attribute to value.
 //
-// value: Dimension along which to pack.  Negative values wrap around, so the
-// valid range is `[-(R+1), R+1)`.
-// If not specified, defaults to 0
-func PackAxis(value int64) PackAttr {
+// value: If true, rescale input by (new_height - 1) / (height - 1),
+// which exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func FusedResizeAndPadConv2DResizeAlignCorners(value bool) FusedResizeAndPadConv2DAttr {
 	return func(m optionalAttr) {
-		m["axis"] = value
+		m["resize_align_corners"] = value
 	}
 }
 
-// Packs a list of `N` rank-`R` tensors into one rank-`(R+1)` tensor.
-//
-// Packs the `N` tensors in `values` into a tensor with rank one higher than each
-// tensor in `values`, by packing them along the `axis` dimension.
-// Given a list of tensors of shape `(A, B, C)`;
-//
-// if `axis == 0` then the `output` tensor will have the shape `(N, A, B, C)`.
-// if `axis == 1` then the `output` tensor will have the shape `(A, N, B, C)`.
-// Etc.
-//
-// For example:
-//
-// ```
-// # 'x' is [1, 4]
-// # 'y' is [2, 5]
-// # 'z' is [3, 6]
-// pack([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
-// pack([x, y, z], axis=1) => [[1, 2, 3], [4, 5, 6]]
-// ```
+// Performs a resize and padding as a preprocess during a convolution.
 //
-// This is the opposite of `unpack`.
+// It's often possible to do spatial transformations more efficiently as part of
+// the packing stage of a convolution, so this op allows for an optimized
+// implementation where these stages are fused together. This prevents the need to
+// write out the intermediate results as whole tensors, reducing memory pressure,
+// and we can get some latency gains by merging the transformation calculations.
+// The data_format attribute for Conv2D isn't supported by this op, and defaults to
+// 'NHWC' order.
+// Internally this op uses a single per-graph scratch buffer, which means that it
+// will block if multiple versions are being run in parallel. This is because this
+// operator is primarily an optimization to minimize memory usage.
 //
 // Arguments:
-//	values: Must be of same shape and type.
+//	input: 4-D with shape `[batch, in_height, in_width, in_channels]`.
+//	size: A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
+//	paddings: A two-column matrix specifying the padding sizes. The number of
+// rows must be the same as the rank of `input`.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, out_channels]`.
 //
-// Returns The packed tensor.
-func Pack(scope *Scope, values []tf.Output, optional ...PackAttr) (output tf.Output) {
+//	strides: 1-D of length 4.  The stride of the sliding window for each dimension
+// of `input`. Must be in the same order as the dimension specified with format.
+//	padding: The type of padding algorithm to use.
+func FusedResizeAndPadConv2D(scope *Scope, input tf.Output, size tf.Output, paddings tf.Output, filter tf.Output, mode string, strides []int64, padding string, optional ...FusedResizeAndPadConv2DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"mode": mode, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Pack",
+		Type: "FusedResizeAndPadConv2D",
 		Input: []tf.Input{
-			tf.OutputList(values),
+			input, size, paddings, filter,
 		},
 		Attrs: attrs,
 	}
@@ -11841,138 +11774,154 @@ func Pack(scope *Scope, values []tf.Output, optional ...PackAttr) (output tf.Out
 	return op.Output(0)
 }
 
-// Reorders a SparseTensor into the canonical, row-major ordering.
-//
-// Note that by convention, all sparse ops preserve the canonical ordering along
-// increasing dimension number. The only time ordering can be violated is during
-// manual manipulation of the indices and values vectors to add entries.
-//
-// Reordering does not affect the shape of the SparseTensor.
+// Inverse 3D fast Fourier transform.
 //
-// If the tensor has rank `R` and `N` non-empty values, `input_indices` has
-// shape `[N, R]`, input_values has length `N`, and input_shape has length `R`.
+// Computes the inverse 3-dimensional discrete Fourier transform over the
+// inner-most 3 dimensions of `input`.
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	input: A complex64 tensor.
 //
-// Returns 2-D.  `N x R` matrix with the same indices as input_indices, but
-// in canonical row-major ordering.1-D.  `N` non-empty values corresponding to `output_indices`.
-func SparseReorder(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
+// Returns A complex64 tensor of the same shape as `input`. The inner-most 3
+//   dimensions of `input` are replaced with their inverse 3D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.ifftn with 3 dimensions.
+// @end_compatibility
+func IFFT3D(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseReorder",
+		Type: "IFFT3D",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Computes rectified linear: `max(features, 0)`.
-func Relu(scope *Scope, features tf.Output) (activations tf.Output) {
+// Adds `bias` to `value`.
+//
+// This is a deprecated version of BiasAdd and will be soon removed.
+//
+// This is a special case of `tf.add` where `bias` is restricted to be 1-D.
+// Broadcasting is supported, so `value` may have any number of dimensions.
+//
+// Arguments:
+//	value: Any number of dimensions.
+//	bias: 1-D with size the last dimension of `value`.
+//
+// Returns Broadcasted sum of `value` and `bias`.
+func BiasAddV1(scope *Scope, value tf.Output, bias tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Relu",
+		Type: "BiasAddV1",
 		Input: []tf.Input{
-			features,
+			value, bias,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyAddSignAttr is an optional argument to ResourceApplyAddSign.
-type ResourceApplyAddSignAttr func(optionalAttr)
-
-// ResourceApplyAddSignUseLocking sets the optional use_locking attribute to value.
+// Reverses specific dimensions of a tensor.
 //
-// value: If `True`, updating of the var and m tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAddSignUseLocking(value bool) ResourceApplyAddSignAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' according to the AddSign update.
+// NOTE `tf.reverse` has now changed behavior in preparation for 1.0.
+// `tf.reverse_v2` is currently an alias that will be deprecated before TF 1.0.
 //
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
-// update <- (alpha + sign_decay * sign(g) *sign(m)) * g
-// variable <- variable - lr_t * update
+// Given a `tensor`, and a `int32` tensor `axis` representing the set of
+// dimensions of `tensor` to reverse. This operation reverses each dimension
+// `i` for which there exists `j` s.t. `axis[j] == i`.
+//
+// `tensor` can have up to 8 dimensions. The number of dimensions specified
+// in `axis` may be 0 or more entries. If an index is specified more than
+// once, a InvalidArgument error is raised.
+//
+// For example:
+//
+// ```
+// # tensor 't' is [[[[ 0,  1,  2,  3],
+// #                  [ 4,  5,  6,  7],
+// #                  [ 8,  9, 10, 11]],
+// #                 [[12, 13, 14, 15],
+// #                  [16, 17, 18, 19],
+// #                  [20, 21, 22, 23]]]]
+// # tensor 't' shape is [1, 2, 3, 4]
+//
+// # 'dims' is [3] or 'dims' is [-1]
+// reverse(t, dims) ==> [[[[ 3,  2,  1,  0],
+//                         [ 7,  6,  5,  4],
+//                         [ 11, 10, 9, 8]],
+//                        [[15, 14, 13, 12],
+//                         [19, 18, 17, 16],
+//                         [23, 22, 21, 20]]]]
+//
+// # 'dims' is '[1]' (or 'dims' is '[-3]')
+// reverse(t, dims) ==> [[[[12, 13, 14, 15],
+//                         [16, 17, 18, 19],
+//                         [20, 21, 22, 23]
+//                        [[ 0,  1,  2,  3],
+//                         [ 4,  5,  6,  7],
+//                         [ 8,  9, 10, 11]]]]
+//
+// # 'dims' is '[2]' (or 'dims' is '[-2]')
+// reverse(t, dims) ==> [[[[8, 9, 10, 11],
+//                         [4, 5, 6, 7],
+//                         [0, 1, 2, 3]]
+//                        [[20, 21, 22, 23],
+//                         [16, 17, 18, 19],
+//                         [12, 13, 14, 15]]]]
+// ```
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	alpha: Must be a scalar.
-//	sign_decay: Must be a scalar.
-//	beta: Must be a scalar.
-//	grad: The gradient.
+//	tensor: Up to 8-D.
+//	axis: 1-D. The indices of the dimensions to reverse. Must be in the range
+// `[-rank(tensor), rank(tensor))`.
 //
-// Returns the created operation.
-func ResourceApplyAddSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, alpha tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyAddSignAttr) (o *tf.Operation) {
+// Returns The same shape as `tensor`.
+func ReverseV2(scope *Scope, tensor tf.Output, axis tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAddSign",
+		Type: "ReverseV2",
 		Input: []tf.Input{
-			var_, m, lr, alpha, sign_decay, beta, grad,
+			tensor, axis,
 		},
-		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// FractionalMaxPoolGradAttr is an optional argument to FractionalMaxPoolGrad.
-type FractionalMaxPoolGradAttr func(optionalAttr)
+// RealAttr is an optional argument to Real.
+type RealAttr func(optionalAttr)
 
-// FractionalMaxPoolGradOverlapping sets the optional overlapping attribute to value.
-//
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
-//
-// `index  0  1  2  3  4`
-//
-// `value  20 5  16 3  7`
-//
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [20, 16] for fractional max pooling.
-// If not specified, defaults to false
-func FractionalMaxPoolGradOverlapping(value bool) FractionalMaxPoolGradAttr {
+// RealTout sets the optional Tout attribute to value.
+// If not specified, defaults to DT_FLOAT
+func RealTout(value tf.DataType) RealAttr {
 	return func(m optionalAttr) {
-		m["overlapping"] = value
+		m["Tout"] = value
 	}
 }
 
-// Computes gradient of the FractionalMaxPool function.
+// Returns the real part of a complex number.
 //
-// Arguments:
-//	orig_input: Original input for `fractional_max_pool`
-//	orig_output: Original output for `fractional_max_pool`
-//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
-// w.r.t. the output of `fractional_max_pool`.
-//	row_pooling_sequence: row pooling sequence, form pooling region with
-// col_pooling_sequence.
-//	col_pooling_sequence: column pooling sequence, form pooling region with
-// row_pooling sequence.
+// Given a tensor `input` of complex numbers, this operation returns a tensor of
+// type `float` that is the real part of each element in `input`. All elements in
+// `input` must be complex numbers of the form \\(a + bj\\), where *a* is the real
+//  part returned by this operation and *b* is the imaginary part.
 //
-// Returns 4-D.  Gradients w.r.t. the input of `fractional_max_pool`.
-func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalMaxPoolGradAttr) (output tf.Output) {
+// For example:
+//
+// ```
+// # tensor 'input' is [-2.25 + 4.75j, 3.25 + 5.75j]
+// tf.real(input) ==> [-2.25, 3.25]
+// ```
+func Real(scope *Scope, input tf.Output, optional ...RealAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -11981,9 +11930,9 @@ func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FractionalMaxPoolGrad",
+		Type: "Real",
 		Input: []tf.Input{
-			orig_input, orig_output, out_backprop, row_pooling_sequence, col_pooling_sequence,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -11991,86 +11940,98 @@ func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Ou
 	return op.Output(0)
 }
 
-// ResourceApplyAdagradDAAttr is an optional argument to ResourceApplyAdagradDA.
-type ResourceApplyAdagradDAAttr func(optionalAttr)
+// AudioSummaryAttr is an optional argument to AudioSummary.
+type AudioSummaryAttr func(optionalAttr)
 
-// ResourceApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
+// AudioSummaryMaxOutputs sets the optional max_outputs attribute to value.
 //
-// value: If True, updating of the var and accum tensors will be protected by
-// a lock; otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyAdagradDAUseLocking(value bool) ResourceApplyAdagradDAAttr {
+// value: Max number of batch elements to generate audio for.
+// If not specified, defaults to 3
+//
+// REQUIRES: value >= 1
+func AudioSummaryMaxOutputs(value int64) AudioSummaryAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["max_outputs"] = value
 	}
 }
 
-// Update '*var' according to the proximal adagrad scheme.
+// Outputs a `Summary` protocol buffer with audio.
+//
+// DEPRECATED at GraphDef version 15: Use AudioSummaryV2.
+//
+// The summary has up to `max_outputs` summary values containing audio. The
+// audio is built from `tensor` which must be 3-D with shape `[batch_size,
+// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+//
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
+//
+// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+// *  If `max_outputs` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	gradient_accumulator: Should be from a Variable().
-//	gradient_squared_accumulator: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regularization. Must be a scalar.
-//	l2: L2 regularization. Must be a scalar.
-//	global_step: Training step number. Must be a scalar.
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 2-D of shape `[batch_size, frames]`.
+//	sample_rate: The sample rate of the signal in hertz.
 //
-// Returns the created operation.
-func ResourceApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceApplyAdagradDAAttr) (o *tf.Operation) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func AudioSummary(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate float32, optional ...AudioSummaryAttr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"sample_rate": sample_rate}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdagradDA",
+		Type: "AudioSummary",
 		Input: []tf.Input{
-			var_, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step,
+			tag, tensor,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SparseReduceMaxSparseAttr is an optional argument to SparseReduceMaxSparse.
-type SparseReduceMaxSparseAttr func(optionalAttr)
+// QrAttr is an optional argument to Qr.
+type QrAttr func(optionalAttr)
 
-// SparseReduceMaxSparseKeepDims sets the optional keep_dims attribute to value.
+// QrFullMatrices sets the optional full_matrices attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
+// value: If true, compute full-sized `q` and `r`. If false
+// (the default), compute only the leading `P` columns of `q`.
 // If not specified, defaults to false
-func SparseReduceMaxSparseKeepDims(value bool) SparseReduceMaxSparseAttr {
+func QrFullMatrices(value bool) QrAttr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["full_matrices"] = value
 	}
 }
 
-// Computes the max of elements across dimensions of a SparseTensor.
+// Computes the QR decompositions of one or more matrices.
 //
-// This Op takes a SparseTensor and is the sparse counterpart to
-// `tf.reduce_max()`.  In contrast to SparseReduceMax, this Op returns a
-// SparseTensor.
+// Computes the QR decomposition of each inner matrix in `tensor` such that
+// `tensor[..., :, :] = q[..., :, :] * r[..., :,:])`
 //
-// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
-// with length 1.
-//
-// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
-// with a single element is returned.  Additionally, the axes can be negative,
-// which are interpreted according to the indexing rules in Python.
+// ```python
+// # a is a tensor.
+// # q is a tensor of orthonormal matrices.
+// # r is a tensor of upper triangular matrices.
+// q, r = qr(a)
+// q_full, r_full = qr(a, full_matrices=True)
+// ```
 //
 // Arguments:
-//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
-//	input_shape: 1-D.  Shape of the input SparseTensor.
-//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
-func SparseReduceMaxSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//
+// Returns Orthonormal basis for range of `a`. If `full_matrices` is `False` then
+// shape is `[..., M, P]`; if `full_matrices` is `True` then shape is
+// `[..., M, M]`.Triangular factor. If `full_matrices` is `False` then shape is
+// `[..., P, N]`. If `full_matrices` is `True` then shape is `[..., M, N]`.
+func Qr(scope *Scope, input tf.Output, optional ...QrAttr) (q tf.Output, r tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12079,33 +12040,26 @@ func SparseReduceMaxSparse(scope *Scope, input_indices tf.Output, input_values t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseReduceMaxSparse",
+		Type: "Qr",
 		Input: []tf.Input{
-			input_indices, input_values, input_shape, reduction_axes,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1)
 }
 
-// Creates a dataset that emits the outputs of `input_dataset` `count` times.
-//
-// Arguments:
-//
-//	count: A scalar representing the number of times that `input_dataset` should
-// be repeated. A value of `-1` indicates that it should be repeated infinitely.
-//
-//
-func RepeatDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Records the bytes size of each element of `input_dataset` in a StatsAggregator.
+func BytesProducedStatsDataset(scope *Scope, input_dataset tf.Output, tag tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "RepeatDataset",
+		Type: "BytesProducedStatsDataset",
 		Input: []tf.Input{
-			input_dataset, count,
+			input_dataset, tag,
 		},
 		Attrs: attrs,
 	}
@@ -12113,65 +12067,36 @@ func RepeatDataset(scope *Scope, input_dataset tf.Output, count tf.Output, outpu
 	return op.Output(0)
 }
 
-// AddManySparseToTensorsMapAttr is an optional argument to AddManySparseToTensorsMap.
-type AddManySparseToTensorsMapAttr func(optionalAttr)
-
-// AddManySparseToTensorsMapContainer sets the optional container attribute to value.
-//
-// value: The container name for the `SparseTensorsMap` created by this op.
-// If not specified, defaults to ""
-func AddManySparseToTensorsMapContainer(value string) AddManySparseToTensorsMapAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// ResourceSparseApplyProximalGradientDescentAttr is an optional argument to ResourceSparseApplyProximalGradientDescent.
+type ResourceSparseApplyProximalGradientDescentAttr func(optionalAttr)
 
-// AddManySparseToTensorsMapSharedName sets the optional shared_name attribute to value.
+// ResourceSparseApplyProximalGradientDescentUseLocking sets the optional use_locking attribute to value.
 //
-// value: The shared name for the `SparseTensorsMap` created by this op.
-// If blank, the new Operation's unique name is used.
-// If not specified, defaults to ""
-func AddManySparseToTensorsMapSharedName(value string) AddManySparseToTensorsMapAttr {
+// value: If True, the subtraction will be protected by a lock;
+// otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceSparseApplyProximalGradientDescentUseLocking(value bool) ResourceSparseApplyProximalGradientDescentAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Add an `N`-minibatch `SparseTensor` to a `SparseTensorsMap`, return `N` handles.
-//
-// A `SparseTensor` of rank `R` is represented by three tensors: `sparse_indices`,
-// `sparse_values`, and `sparse_shape`, where
-//
-// ```sparse_indices.shape[1] == sparse_shape.shape[0] == R```
-//
-// An `N`-minibatch of `SparseTensor` objects is represented as a `SparseTensor`
-// having a first `sparse_indices` column taking values between `[0, N)`, where
-// the minibatch size `N == sparse_shape[0]`.
-//
-// The input `SparseTensor` must have rank `R` greater than 1, and the first
-// dimension is treated as the minibatch dimension.  Elements of the `SparseTensor`
-// must be sorted in increasing order of this first dimension.  The stored
-// `SparseTensor` objects pointed to by each row of the output `sparse_handles`
-// will have rank `R-1`.
+// Sparse update '*var' as FOBOS algorithm with fixed learning rate.
 //
-// The `SparseTensor` values can then be read out as part of a minibatch by passing
-// the given keys as vector elements to `TakeManySparseFromTensorsMap`.  To ensure
-// the correct `SparseTensorsMap` is accessed, ensure that the same
-// `container` and `shared_name` are passed to that Op.  If no `shared_name`
-// is provided here, instead use the *name* of the Operation created by calling
-// `AddManySparseToTensorsMap` as the `shared_name` passed to
-// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
+// That is for rows we have grad for, we update var as follows:
+// prox_v = var - alpha * grad
+// var = sign(prox_v)/(1+alpha*l2) * max{|prox_v|-alpha*l1,0}
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
-// `sparse_indices[:, 0]` must be ordered values in `[0, N)`.
-//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
-// The minibatch size `N == sparse_shape[0]`.
+//	var_: Should be from a Variable().
+//	alpha: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	grad: The gradient.
+//	indices: A vector of indices into the first dimension of var and accum.
 //
-// Returns 1-D.  The handles of the `SparseTensor` now stored in the
-// `SparseTensorsMap`.  Shape: `[N]`.
-func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddManySparseToTensorsMapAttr) (sparse_handles tf.Output) {
+// Returns the created operation.
+func ResourceSparseApplyProximalGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, l1 tf.Output, l2 tf.Output, grad tf.Output, indices tf.Output, optional ...ResourceSparseApplyProximalGradientDescentAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12180,70 +12105,29 @@ func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_va
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AddManySparseToTensorsMap",
+		Type: "ResourceSparseApplyProximalGradientDescent",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			var_, alpha, l1, l2, grad, indices,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Concatenates tensors along one dimension.
-//
-// Arguments:
-//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [-rank(values), rank(values)).
-//
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.
-func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ConcatV2",
-		Input: []tf.Input{
-			tf.OutputList(values), axis,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Reads and outputs the entire contents of the input filename.
-func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReadFile",
-		Input: []tf.Input{
-			filename,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// MinAttr is an optional argument to Min.
-type MinAttr func(optionalAttr)
+// MeanAttr is an optional argument to Mean.
+type MeanAttr func(optionalAttr)
 
-// MinKeepDims sets the optional keep_dims attribute to value.
+// MeanKeepDims sets the optional keep_dims attribute to value.
 //
 // value: If true, retain reduced dimensions with length 1.
 // If not specified, defaults to false
-func MinKeepDims(value bool) MinAttr {
+func MeanKeepDims(value bool) MeanAttr {
 	return func(m optionalAttr) {
 		m["keep_dims"] = value
 	}
 }
 
-// Computes the minimum of elements across dimensions of a tensor.
+// Computes the mean of elements across dimensions of a tensor.
 //
 // Reduces `input` along the dimensions given in `axis`. Unless
 // `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
@@ -12256,7 +12140,7 @@ func MinKeepDims(value bool) MinAttr {
 // `[-rank(input), rank(input))`.
 //
 // Returns The reduced tensor.
-func Min(scope *Scope, input tf.Output, axis tf.Output, optional ...MinAttr) (output tf.Output) {
+func Mean(scope *Scope, input tf.Output, axis tf.Output, optional ...MeanAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12265,7 +12149,7 @@ func Min(scope *Scope, input tf.Output, axis tf.Output, optional ...MinAttr) (ou
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Min",
+		Type: "Mean",
 		Input: []tf.Input{
 			input, axis,
 		},
@@ -12275,95 +12159,90 @@ func Min(scope *Scope, input tf.Output, axis tf.Output, optional ...MinAttr) (ou
 	return op.Output(0)
 }
 
-// Shuffle dimensions of x according to a permutation.
+// InitializeTableFromTextFileV2Attr is an optional argument to InitializeTableFromTextFileV2.
+type InitializeTableFromTextFileV2Attr func(optionalAttr)
+
+// InitializeTableFromTextFileV2VocabSize sets the optional vocab_size attribute to value.
 //
-// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
-//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
-func Transpose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: Number of elements of the file, use -1 if unknown.
+// If not specified, defaults to -1
+//
+// REQUIRES: value >= -1
+func InitializeTableFromTextFileV2VocabSize(value int64) InitializeTableFromTextFileV2Attr {
+	return func(m optionalAttr) {
+		m["vocab_size"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "Transpose",
-		Input: []tf.Input{
-			x, perm,
-		},
+}
+
+// InitializeTableFromTextFileV2Delimiter sets the optional delimiter attribute to value.
+//
+// value: Delimiter to separate fields in a line.
+// If not specified, defaults to "\t"
+func InitializeTableFromTextFileV2Delimiter(value string) InitializeTableFromTextFileV2Attr {
+	return func(m optionalAttr) {
+		m["delimiter"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes sigmoid of `x` element-wise.
+// Initializes a table from a text file.
 //
-// Specifically, `y = 1 / (1 + exp(-x))`.
-func Sigmoid(scope *Scope, x tf.Output) (y tf.Output) {
+// It inserts one key-value pair into the table for each line of the file.
+// The key and value is extracted from the whole line content, elements from the
+// split line based on `delimiter` or the line number (starting from zero).
+// Where to extract the key and value from a line is specified by `key_index` and
+// `value_index`.
+//
+// - A value of -1 means use the line number(starting from zero), expects `int64`.
+// - A value of -2 means use the whole line content, expects `string`.
+// - A value >= 0 means use the index (starting at zero) of the split line based
+//   on `delimiter`.
+//
+// Arguments:
+//	table_handle: Handle to a table which will be initialized.
+//	filename: Filename of a vocabulary text file.
+//	key_index: Column index in a line to get the table `key` values from.
+//	value_index: Column index that represents information of a line to get the table
+// `value` values from.
+//
+// Returns the created operation.
+func InitializeTableFromTextFileV2(scope *Scope, table_handle tf.Output, filename tf.Output, key_index int64, value_index int64, optional ...InitializeTableFromTextFileV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"key_index": key_index, "value_index": value_index}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Sigmoid",
+		Type: "InitializeTableFromTextFileV2",
 		Input: []tf.Input{
-			x,
+			table_handle, filename,
 		},
+		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// FusedBatchNormAttr is an optional argument to FusedBatchNorm.
-type FusedBatchNormAttr func(optionalAttr)
+// QuantizedReluAttr is an optional argument to QuantizedRelu.
+type QuantizedReluAttr func(optionalAttr)
 
-// FusedBatchNormEpsilon sets the optional epsilon attribute to value.
-//
-// value: A small float number added to the variance of x.
-// If not specified, defaults to 0.0001
-func FusedBatchNormEpsilon(value float32) FusedBatchNormAttr {
-	return func(m optionalAttr) {
-		m["epsilon"] = value
-	}
-}
-
-// FusedBatchNormDataFormat sets the optional data_format attribute to value.
-//
-// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
-// If not specified, defaults to "NHWC"
-func FusedBatchNormDataFormat(value string) FusedBatchNormAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// FusedBatchNormIsTraining sets the optional is_training attribute to value.
-//
-// value: A bool value to indicate the operation is for training (default)
-// or inference.
-// If not specified, defaults to true
-func FusedBatchNormIsTraining(value bool) FusedBatchNormAttr {
+// QuantizedReluOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedReluOutType(value tf.DataType) QuantizedReluAttr {
 	return func(m optionalAttr) {
-		m["is_training"] = value
+		m["out_type"] = value
 	}
 }
 
-// Batch normalization.
-//
-// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
-// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+// Computes Quantized Rectified Linear: `max(features, 0)`
 //
 // Arguments:
-//	x: A 4D Tensor for input data.
-//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
-//	offset: A 1D Tensor for offset, to shift to the normalized x.
-//	mean: A 1D Tensor for population mean. Used for inference only;
-// must be empty for training.
-//	variance: A 1D Tensor for population variance. Used for inference only;
-// must be empty for training.
 //
-// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
-// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
-// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
-// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
-// in the cuDNN case), to be reused in the gradient computation.
-func FusedBatchNorm(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormAttr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
+//
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedRelu(scope *Scope, features tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12372,136 +12251,116 @@ func FusedBatchNorm(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FusedBatchNorm",
+		Type: "QuantizedRelu",
 		Input: []tf.Input{
-			x, scale, offset, mean, variance,
+			features, min_features, max_features,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// RandomStandardNormalAttr is an optional argument to RandomStandardNormal.
-type RandomStandardNormalAttr func(optionalAttr)
-
-// RandomStandardNormalSeed sets the optional seed attribute to value.
+// Reshapes a SparseTensor to represent values in a new dense shape.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomStandardNormalSeed(value int64) RandomStandardNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomStandardNormalSeed2 sets the optional seed2 attribute to value.
+// This operation has the same semantics as reshape on the represented dense
+// tensor.  The `input_indices` are recomputed based on the requested `new_shape`.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomStandardNormalSeed2(value int64) RandomStandardNormalAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from a normal distribution.
+// If one component of `new_shape` is the special value -1, the size of that
+// dimension is computed so that the total dense size remains constant.  At
+// most one component of `new_shape` can be -1.  The number of dense elements
+// implied by `new_shape` must be the same as the number of dense elements
+// originally implied by `input_shape`.
 //
-// The generated values will have mean 0 and standard deviation 1.
+// Reshaping does not affect the order of values in the SparseTensor.
+//
+// If the input tensor has rank `R_in` and `N` non-empty values, and `new_shape`
+// has length `R_out`, then `input_indices` has shape `[N, R_in]`,
+// `input_shape` has length `R_in`, `output_indices` has shape `[N, R_out]`, and
+// `output_shape` has length `R_out`.
 //
 // Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+//	input_indices: 2-D.  `N x R_in` matrix with the indices of non-empty values in a
+// SparseTensor.
+//	input_shape: 1-D.  `R_in` vector with the input SparseTensor's dense shape.
+//	new_shape: 1-D.  `R_out` vector with the requested new dense shape.
 //
-// Returns A tensor of the specified shape filled with random normal values.
-func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomStandardNormalAttr) (output tf.Output) {
+// Returns 2-D.  `N x R_out` matrix with the updated indices of non-empty
+// values in the output SparseTensor.1-D.  `R_out` vector with the full dense shape of the output
+// SparseTensor.  This is the same as `new_shape` but with any -1 dimensions
+// filled in.
+func SparseReshape(scope *Scope, input_indices tf.Output, input_shape tf.Output, new_shape tf.Output) (output_indices tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomStandardNormal",
+		Type: "SparseReshape",
 		Input: []tf.Input{
-			shape,
+			input_indices, input_shape, new_shape,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Component-wise divides a SparseTensor by a dense Tensor.
-//
-// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
-// the other direction.
-//
-// Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//	dense: `R`-D.  The dense Tensor operand.
+// Deprecated. Use TensorArraySplitV3
 //
-// Returns 1-D.  The `N` values that are operated on.
-func SparseDenseCwiseDiv(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArraySplitV3
+func TensorArraySplitV2(scope *Scope, handle tf.Output, value tf.Output, lengths tf.Output, flow_in tf.Output) (flow_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseDenseCwiseDiv",
+		Type: "TensorArraySplitV2",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape, dense,
+			handle, value, lengths, flow_in,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FractionalAvgPoolGradAttr is an optional argument to FractionalAvgPoolGrad.
-type FractionalAvgPoolGradAttr func(optionalAttr)
+// PackAttr is an optional argument to Pack.
+type PackAttr func(optionalAttr)
 
-// FractionalAvgPoolGradOverlapping sets the optional overlapping attribute to value.
-//
-// value: When set to True, it means when pooling, the values at the boundary
-// of adjacent pooling cells are used by both cells. For example:
-//
-// `index  0  1  2  3  4`
-//
-// `value  20 5  16 3  7`
+// PackAxis sets the optional axis attribute to value.
 //
-// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
-// The result would be [41/3, 26/3] for fractional avg pooling.
-// If not specified, defaults to false
-func FractionalAvgPoolGradOverlapping(value bool) FractionalAvgPoolGradAttr {
+// value: Dimension along which to pack.  Negative values wrap around, so the
+// valid range is `[-(R+1), R+1)`.
+// If not specified, defaults to 0
+func PackAxis(value int64) PackAttr {
 	return func(m optionalAttr) {
-		m["overlapping"] = value
+		m["axis"] = value
 	}
 }
 
-// Computes gradient of the FractionalAvgPool function.
+// Packs a list of `N` rank-`R` tensors into one rank-`(R+1)` tensor.
 //
-// Unlike FractionalMaxPoolGrad, we don't need to find arg_max for
-// FractionalAvgPoolGrad, we just need to evenly back-propagate each element of
-// out_backprop to those indices that form the same pooling cell. Therefore, we
-// just need to know the shape of original input tensor, instead of the whole
-// tensor.
+// Packs the `N` tensors in `values` into a tensor with rank one higher than each
+// tensor in `values`, by packing them along the `axis` dimension.
+// Given a list of tensors of shape `(A, B, C)`;
+//
+// if `axis == 0` then the `output` tensor will have the shape `(N, A, B, C)`.
+// if `axis == 1` then the `output` tensor will have the shape `(A, N, B, C)`.
+// Etc.
+//
+// For example:
+//
+// ```
+// # 'x' is [1, 4]
+// # 'y' is [2, 5]
+// # 'z' is [3, 6]
+// pack([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
+// pack([x, y, z], axis=1) => [[1, 2, 3], [4, 5, 6]]
+// ```
+//
+// This is the opposite of `unpack`.
 //
 // Arguments:
-//	orig_input_tensor_shape: Original input tensor shape for `fractional_avg_pool`
-//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
-// w.r.t. the output of `fractional_avg_pool`.
-//	row_pooling_sequence: row pooling sequence, form pooling region with
-// col_pooling_sequence.
-//	col_pooling_sequence: column pooling sequence, form pooling region with
-// row_pooling sequence.
+//	values: Must be of same shape and type.
 //
-// Returns 4-D.  Gradients w.r.t. the input of `fractional_avg_pool`.
-func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalAvgPoolGradAttr) (output tf.Output) {
+// Returns The packed tensor.
+func Pack(scope *Scope, values []tf.Output, optional ...PackAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12510,9 +12369,9 @@ func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FractionalAvgPoolGrad",
+		Type: "Pack",
 		Input: []tf.Input{
-			orig_input_tensor_shape, out_backprop, row_pooling_sequence, col_pooling_sequence,
+			tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
@@ -12520,74 +12379,86 @@ func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_
 	return op.Output(0)
 }
 
-// Concatenates tensors along one dimension.
+// Reorders a SparseTensor into the canonical, row-major ordering.
+//
+// Note that by convention, all sparse ops preserve the canonical ordering along
+// increasing dimension number. The only time ordering can be violated is during
+// manual manipulation of the indices and values vectors to add entries.
+//
+// Reordering does not affect the shape of the SparseTensor.
+//
+// If the tensor has rank `R` and `N` non-empty values, `input_indices` has
+// shape `[N, R]`, input_values has length `N`, and input_shape has length `R`.
 //
 // Arguments:
-//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [0, rank(values)).
-//	values: The `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
 //
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.
-func Concat(scope *Scope, concat_dim tf.Output, values []tf.Output) (output tf.Output) {
+// Returns 2-D.  `N x R` matrix with the same indices as input_indices, but
+// in canonical row-major ordering.1-D.  `N` non-empty values corresponding to `output_indices`.
+func SparseReorder(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Concat",
+		Type: "SparseReorder",
 		Input: []tf.Input{
-			concat_dim, tf.OutputList(values),
+			input_indices, input_values, input_shape,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Computes rectified linear: `max(features, 0)`.
+func Relu(scope *Scope, features tf.Output) (activations tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Relu",
+		Input: []tf.Input{
+			features,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyMomentumAttr is an optional argument to ResourceApplyMomentum.
-type ResourceApplyMomentumAttr func(optionalAttr)
+// ResourceApplyAddSignAttr is an optional argument to ResourceApplyAddSign.
+type ResourceApplyAddSignAttr func(optionalAttr)
 
-// ResourceApplyMomentumUseLocking sets the optional use_locking attribute to value.
+// ResourceApplyAddSignUseLocking sets the optional use_locking attribute to value.
 //
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
+// value: If `True`, updating of the var and m tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
 // contention.
 // If not specified, defaults to false
-func ResourceApplyMomentumUseLocking(value bool) ResourceApplyMomentumAttr {
+func ResourceApplyAddSignUseLocking(value bool) ResourceApplyAddSignAttr {
 	return func(m optionalAttr) {
 		m["use_locking"] = value
 	}
 }
 
-// ResourceApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
-//
-// value: If `True`, the tensor passed to compute grad will be
-// var - lr * momentum * accum, so in the end, the var you get is actually
-// var - lr * momentum * accum.
-// If not specified, defaults to false
-func ResourceApplyMomentumUseNesterov(value bool) ResourceApplyMomentumAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
-}
-
-// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
-//
-// want to use Nesterov momentum.
+// Update '*var' according to the AddSign update.
 //
-// accum = accum * momentum + grad
-// var -= lr * accum
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+// update <- (alpha + sign_decay * sign(g) *sign(m)) * g
+// variable <- variable - lr_t * update
 //
 // Arguments:
 //	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
+//	m: Should be from a Variable().
 //	lr: Scaling factor. Must be a scalar.
+//	alpha: Must be a scalar.
+//	sign_decay: Must be a scalar.
+//	beta: Must be a scalar.
 //	grad: The gradient.
-//	momentum: Momentum. Must be a scalar.
 //
 // Returns the created operation.
-func ResourceApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyMomentumAttr) (o *tf.Operation) {
+func ResourceApplyAddSign(scope *Scope, var_ tf.Output, m tf.Output, lr tf.Output, alpha tf.Output, sign_decay tf.Output, beta tf.Output, grad tf.Output, optional ...ResourceApplyAddSignAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12596,56 +12467,61 @@ func ResourceApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyMomentum",
+		Type: "ResourceApplyAddSign",
 		Input: []tf.Input{
-			var_, accum, lr, grad, momentum,
+			var_, m, lr, alpha, sign_decay, beta, grad,
 		},
 		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
-type MaxPoolGradGradAttr func(optionalAttr)
+// FractionalMaxPoolGradAttr is an optional argument to FractionalMaxPoolGrad.
+type FractionalMaxPoolGradAttr func(optionalAttr)
 
-// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
+// FractionalMaxPoolGradOverlapping sets the optional overlapping attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
+//
+// `index  0  1  2  3  4`
+//
+// `value  20 5  16 3  7`
+//
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [20, 16] for fractional max pooling.
+// If not specified, defaults to false
+func FractionalMaxPoolGradOverlapping(value bool) FractionalMaxPoolGradAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["overlapping"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Computes gradient of the FractionalMaxPool function.
 //
 // Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	orig_input: Original input for `fractional_max_pool`
+//	orig_output: Original output for `fractional_max_pool`
+//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
+// w.r.t. the output of `fractional_max_pool`.
+//	row_pooling_sequence: row pooling sequence, form pooling region with
+// col_pooling_sequence.
+//	col_pooling_sequence: column pooling sequence, form pooling region with
+// row_pooling sequence.
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
+// Returns 4-D.  Gradients w.r.t. the input of `fractional_max_pool`.
+func FractionalMaxPoolGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalMaxPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolGradGrad",
+		Type: "FractionalMaxPoolGrad",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			orig_input, orig_output, out_backprop, row_pooling_sequence, col_pooling_sequence,
 		},
 		Attrs: attrs,
 	}
@@ -12653,208 +12529,198 @@ func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output,
 	return op.Output(0)
 }
 
-// Returns element-wise integer closest to x.
-//
-// If the result is midway between two representable values,
-// the even representable is chosen.
-// For example:
-//
-// ```
-// rint(-1.5) ==> -2.0
-// rint(0.5000001) ==> 1.0
-// rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.]
-// ```
-func Rint(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Rint",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey.
-type OrderedMapUnstageNoKeyAttr func(optionalAttr)
+// ResourceApplyAdagradDAAttr is an optional argument to ResourceApplyAdagradDA.
+type ResourceApplyAdagradDAAttr func(optionalAttr)
 
-// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// ResourceApplyAdagradDAUseLocking sets the optional use_locking attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr {
+// value: If True, updating of the var and accum tensors will be protected by
+// a lock; otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyAdagradDAUseLocking(value bool) ResourceApplyAdagradDAAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["use_locking"] = value
 	}
 }
 
-// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// Update '*var' according to the proximal adagrad scheme.
 //
-// REQUIRES: value >= 0
-func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes and returns the (key, value) element with the smallest
+// Arguments:
+//	var_: Should be from a Variable().
+//	gradient_accumulator: Should be from a Variable().
+//	gradient_squared_accumulator: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regularization. Must be a scalar.
+//	l2: L2 regularization. Must be a scalar.
+//	global_step: Training step number. Must be a scalar.
 //
-// key from the underlying container.   If the underlying container
-// does not contain elements, the op will block until it does.
-func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
+// Returns the created operation.
+func ResourceApplyAdagradDA(scope *Scope, var_ tf.Output, gradient_accumulator tf.Output, gradient_squared_accumulator tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, global_step tf.Output, optional ...ResourceApplyAdagradDAAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapUnstageNoKey",
+		Type: "ResourceApplyAdagradDA",
 		Input: []tf.Input{
-			indices,
+			var_, gradient_accumulator, gradient_squared_accumulator, grad, lr, l1, l2, global_step,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	key = op.Output(idx)
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("OrderedMapUnstageNoKey", err)
-		return
-	}
-	return key, values
+	return scope.AddOperation(opspec)
 }
 
-// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad.
-type MaxPool3DGradGradAttr func(optionalAttr)
+// SparseReduceMaxSparseAttr is an optional argument to SparseReduceMaxSparse.
+type SparseReduceMaxSparseAttr func(optionalAttr)
 
-// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value.
+// SparseReduceMaxSparseKeepDims sets the optional keep_dims attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SparseReduceMaxSparseKeepDims(value bool) SparseReduceMaxSparseAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Computes second-order gradients of the maxpooling function.
+// Computes the max of elements across dimensions of a SparseTensor.
 //
-// Arguments:
-//	orig_input: The original input tensor.
-//	orig_output: The original output tensor.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// This Op takes a SparseTensor and is the sparse counterpart to
+// `tf.reduce_max()`.  In contrast to SparseReduceMax, this Op returns a
+// SparseTensor.
 //
-// Returns Gradients of gradients w.r.t. the input to `max_pool`.
-func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) {
+// Reduces `sp_input` along the dimensions given in `reduction_axes`.  Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `reduction_axes`. If `keep_dims` is true, the reduced dimensions are retained
+// with length 1.
+//
+// If `reduction_axes` has no entries, all dimensions are reduced, and a tensor
+// with a single element is returned.  Additionally, the axes can be negative,
+// which are interpreted according to the indexing rules in Python.
+//
+// Arguments:
+//	input_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	input_values: 1-D.  `N` non-empty values corresponding to `input_indices`.
+//	input_shape: 1-D.  Shape of the input SparseTensor.
+//	reduction_axes: 1-D.  Length-`K` vector containing the reduction axes.
+func SparseReduceMaxSparse(scope *Scope, input_indices tf.Output, input_values tf.Output, input_shape tf.Output, reduction_axes tf.Output, optional ...SparseReduceMaxSparseAttr) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MaxPool3DGradGrad",
+		Type: "SparseReduceMaxSparse",
 		Input: []tf.Input{
-			orig_input, orig_output, grad,
+			input_indices, input_values, input_shape, reduction_axes,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2.
-type Conv3DBackpropFilterV2Attr func(optionalAttr)
-
-// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value.
+// Creates a dataset that emits the outputs of `input_dataset` `count` times.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr {
+// Arguments:
+//
+//	count: A scalar representing the number of times that `input_dataset` should
+// be repeated. A value of `-1` indicates that it should be repeated infinitely.
+//
+//
+func RepeatDataset(scope *Scope, input_dataset tf.Output, count tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	opspec := tf.OpSpec{
+		Type: "RepeatDataset",
+		Input: []tf.Input{
+			input_dataset, count,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// AddManySparseToTensorsMapAttr is an optional argument to AddManySparseToTensorsMap.
+type AddManySparseToTensorsMapAttr func(optionalAttr)
+
+// AddManySparseToTensorsMapContainer sets the optional container attribute to value.
+//
+// value: The container name for the `SparseTensorsMap` created by this op.
+// If not specified, defaults to ""
+func AddManySparseToTensorsMapContainer(value string) AddManySparseToTensorsMapAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["container"] = value
 	}
 }
 
-// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value.
+// AddManySparseToTensorsMapSharedName sets the optional shared_name attribute to value.
 //
-// value: 1-D tensor of length 5.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr {
+// value: The shared name for the `SparseTensorsMap` created by this op.
+// If blank, the new Operation's unique name is used.
+// If not specified, defaults to ""
+func AddManySparseToTensorsMapSharedName(value string) AddManySparseToTensorsMapAttr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Computes the gradients of 3-D convolution with respect to the filter.
+// Add an `N`-minibatch `SparseTensor` to a `SparseTensorsMap`, return `N` handles.
+//
+// A `SparseTensor` of rank `R` is represented by three tensors: `sparse_indices`,
+// `sparse_values`, and `sparse_shape`, where
+//
+// ```sparse_indices.shape[1] == sparse_shape.shape[0] == R```
+//
+// An `N`-minibatch of `SparseTensor` objects is represented as a `SparseTensor`
+// having a first `sparse_indices` column taking values between `[0, N)`, where
+// the minibatch size `N == sparse_shape[0]`.
+//
+// The input `SparseTensor` must have rank `R` greater than 1, and the first
+// dimension is treated as the minibatch dimension.  Elements of the `SparseTensor`
+// must be sorted in increasing order of this first dimension.  The stored
+// `SparseTensor` objects pointed to by each row of the output `sparse_handles`
+// will have rank `R-1`.
+//
+// The `SparseTensor` values can then be read out as part of a minibatch by passing
+// the given keys as vector elements to `TakeManySparseFromTensorsMap`.  To ensure
+// the correct `SparseTensorsMap` is accessed, ensure that the same
+// `container` and `shared_name` are passed to that Op.  If no `shared_name`
+// is provided here, instead use the *name* of the Operation created by calling
+// `AddManySparseToTensorsMap` as the `shared_name` passed to
+// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
 //
 // Arguments:
-//	input: Shape `[batch, depth, rows, cols, in_channels]`.
-//	filter_sizes: An integer vector representing the tensor shape of `filter`,
-// where `filter` is a 5-D
-// `[filter_depth, filter_height, filter_width, in_channels, out_channels]`
-// tensor.
-//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
-// out_channels]`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
-func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) {
+//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
+// `sparse_indices[:, 0]` must be ordered values in `[0, N)`.
+//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
+// The minibatch size `N == sparse_shape[0]`.
+//
+// Returns 1-D.  The handles of the `SparseTensor` now stored in the
+// `SparseTensorsMap`.  Shape: `[N]`.
+func AddManySparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddManySparseToTensorsMapAttr) (sparse_handles tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv3DBackpropFilterV2",
+		Type: "AddManySparseToTensorsMap",
 		Input: []tf.Input{
-			input, filter_sizes, out_backprop,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
@@ -12862,77 +12728,73 @@ func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Outpu
 	return op.Output(0)
 }
 
-// Execute a sub graph on a remote processor.
-//
-// The graph specifications(such as graph itself, input tensors and output names)
-// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo
-// as serialized_remote_fused_graph_execute_info.
-// The specifications will be passed to a dedicated registered
-// remote fused graph executor.  The executor will send the graph specifications
-// to a remote processor and execute that graph.  The execution results
-// will be passed to consumer nodes as outputs of this node.
+// Concatenates tensors along one dimension.
 //
 // Arguments:
-//	inputs: Arbitrary number of tensors with arbitrary data types
-//
-//	serialized_remote_fused_graph_execute_info: Serialized protocol buffer
-// of RemoteFusedGraphExecuteInfo which contains graph specifications.
+//	values: List of `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//	axis: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [-rank(values), rank(values)).
 //
-// Returns Arbitrary number of tensors with arbitrary data types
-func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) {
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.
+func ConcatV2(scope *Scope, values []tf.Output, axis tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info}
 	opspec := tf.OpSpec{
-		Type: "RemoteFusedGraphExecute",
+		Type: "ConcatV2",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			tf.OutputList(values), axis,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Reads and outputs the entire contents of the input filename.
+func ReadFile(scope *Scope, filename tf.Output) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
-		scope.UpdateErr("RemoteFusedGraphExecute", err)
-		return
+	opspec := tf.OpSpec{
+		Type: "ReadFile",
+		Input: []tf.Input{
+			filename,
+		},
 	}
-	return outputs
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// SerializeManySparseAttr is an optional argument to SerializeManySparse.
-type SerializeManySparseAttr func(optionalAttr)
+// MinAttr is an optional argument to Min.
+type MinAttr func(optionalAttr)
 
-// SerializeManySparseOutType sets the optional out_type attribute to value.
+// MinKeepDims sets the optional keep_dims attribute to value.
 //
-// value: The `dtype` to use for serialization; the supported types are `string`
-// (default) and `variant`.
-// If not specified, defaults to DT_STRING
-func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func MinKeepDims(value bool) MinAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
-//
-// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
-// is treated as the minibatch dimension.  Elements of the `SparseTensor`
-// must be sorted in increasing order of this first dimension.  The serialized
-// `SparseTensor` objects going into each row of `serialized_sparse` will have
-// rank `R-1`.
+// Computes the minimum of elements across dimensions of a tensor.
 //
-// The minibatch size `N` is extracted from `sparse_shape[0]`.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
-func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
+//
+// Returns The reduced tensor.
+func Min(scope *Scope, input tf.Output, axis tf.Output, optional ...MinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -12941,9 +12803,9 @@ func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SerializeManySparse",
+		Type: "Min",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -12951,13 +12813,33 @@ func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values t
 	return op.Output(0)
 }
 
-// Computes inverse hyperbolic cosine of x element-wise.
-func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
+// Shuffle dimensions of x according to a permutation.
+//
+// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+func Transpose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Acosh",
+		Type: "Transpose",
+		Input: []tf.Input{
+			x, perm,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes sigmoid of `x` element-wise.
+//
+// Specifically, `y = 1 / (1 + exp(-x))`.
+func Sigmoid(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Sigmoid",
 		Input: []tf.Input{
 			x,
 		},
@@ -12966,408 +12848,342 @@ func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
 	return op.Output(0)
 }
 
-// TensorArrayV2Attr is an optional argument to TensorArrayV2.
-type TensorArrayV2Attr func(optionalAttr)
+// FusedBatchNormAttr is an optional argument to FusedBatchNorm.
+type FusedBatchNormAttr func(optionalAttr)
 
-// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
-// If not specified, defaults to <unknown_rank:true >
-func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
+// FusedBatchNormEpsilon sets the optional epsilon attribute to value.
+//
+// value: A small float number added to the variance of x.
+// If not specified, defaults to 0.0001
+func FusedBatchNormEpsilon(value float32) FusedBatchNormAttr {
 	return func(m optionalAttr) {
-		m["element_shape"] = value
+		m["epsilon"] = value
 	}
 }
 
-// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
-// If not specified, defaults to false
-func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
+// FusedBatchNormDataFormat sets the optional data_format attribute to value.
+//
+// value: The data format for x and y. Either "NHWC" (default) or "NCHW".
+// If not specified, defaults to "NHWC"
+func FusedBatchNormDataFormat(value string) FusedBatchNormAttr {
 	return func(m optionalAttr) {
-		m["dynamic_size"] = value
+		m["data_format"] = value
 	}
 }
 
-// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
+// FusedBatchNormIsTraining sets the optional is_training attribute to value.
+//
+// value: A bool value to indicate the operation is for training (default)
+// or inference.
 // If not specified, defaults to true
-func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
+func FusedBatchNormIsTraining(value bool) FusedBatchNormAttr {
 	return func(m optionalAttr) {
-		m["clear_after_read"] = value
+		m["is_training"] = value
 	}
 }
 
-// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
-// If not specified, defaults to ""
-func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
-	return func(m optionalAttr) {
-		m["tensor_array_name"] = value
-	}
-}
-
-// Deprecated. Use TensorArrayV3
+// Batch normalization.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayV3
-func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
+// Note that the size of 4D Tensors are defined by either "NHWC" or "NCHW".
+// The size of 1D Tensors matches the dimension C of the 4D Tensors.
+//
+// Arguments:
+//	x: A 4D Tensor for input data.
+//	scale: A 1D Tensor for scaling factor, to scale the normalized x.
+//	offset: A 1D Tensor for offset, to shift to the normalized x.
+//	mean: A 1D Tensor for population mean. Used for inference only;
+// must be empty for training.
+//	variance: A 1D Tensor for population variance. Used for inference only;
+// must be empty for training.
+//
+// Returns A 4D Tensor for output data.A 1D Tensor for the computed batch mean, to be used by TensorFlow
+// to compute the running mean.A 1D Tensor for the computed batch variance, to be used by
+// TensorFlow to compute the running variance.A 1D Tensor for the computed batch mean, to be reused
+// in the gradient computation.A 1D Tensor for the computed batch variance (inverted variance
+// in the cuDNN case), to be reused in the gradient computation.
+func FusedBatchNorm(scope *Scope, x tf.Output, scale tf.Output, offset tf.Output, mean tf.Output, variance tf.Output, optional ...FusedBatchNormAttr) (y tf.Output, batch_mean tf.Output, batch_variance tf.Output, reserve_space_1 tf.Output, reserve_space_2 tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayV2",
+		Type: "FusedBatchNorm",
 		Input: []tf.Input{
-			size,
+			x, scale, offset, mean, variance,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// DecodeCSVAttr is an optional argument to DecodeCSV.
-type DecodeCSVAttr func(optionalAttr)
-
-// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
-//
-// value: char delimiter to separate fields in a record.
-// If not specified, defaults to ","
-func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
-	return func(m optionalAttr) {
-		m["field_delim"] = value
-	}
-}
+// RandomStandardNormalAttr is an optional argument to RandomStandardNormal.
+type RandomStandardNormalAttr func(optionalAttr)
 
-// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
+// RandomStandardNormalSeed sets the optional seed attribute to value.
 //
-// value: If false, treats double quotation marks as regular
-// characters inside of the string fields (ignoring RFC 4180, Section 2,
-// Bullet 5).
-// If not specified, defaults to true
-func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomStandardNormalSeed(value int64) RandomStandardNormalAttr {
 	return func(m optionalAttr) {
-		m["use_quote_delim"] = value
+		m["seed"] = value
 	}
 }
 
-// DecodeCSVNaValue sets the optional na_value attribute to value.
+// RandomStandardNormalSeed2 sets the optional seed2 attribute to value.
 //
-// value: Additional string to recognize as NA/NaN.
-// If not specified, defaults to ""
-func DecodeCSVNaValue(value string) DecodeCSVAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomStandardNormalSeed2(value int64) RandomStandardNormalAttr {
 	return func(m optionalAttr) {
-		m["na_value"] = value
+		m["seed2"] = value
 	}
 }
 
-// Convert CSV records to tensors. Each column maps to one tensor.
+// Outputs random values from a normal distribution.
 //
-// RFC 4180 format is expected for the CSV records.
-// (https://tools.ietf.org/html/rfc4180)
-// Note that we allow leading and trailing spaces with int or float field.
+// The generated values will have mean 0 and standard deviation 1.
 //
 // Arguments:
-//	records: Each string is a record/row in the csv and all records should have
-// the same format.
-//	record_defaults: One tensor per column of the input record, with either a
-// scalar default value for that column or empty if the column is required.
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns Each tensor will have the same shape as records.
-func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
+// Returns A tensor of the specified shape filled with random normal values.
+func RandomStandardNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...RandomStandardNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeCSV",
+		Type: "RandomStandardNormal",
 		Input: []tf.Input{
-			records, tf.OutputList(record_defaults),
+			shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("DecodeCSV", err)
-		return
-	}
-	return output
+	return op.Output(0)
 }
 
-// MapClearAttr is an optional argument to MapClear.
-type MapClearAttr func(optionalAttr)
-
-// MapClearCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// Component-wise divides a SparseTensor by a dense Tensor.
 //
-// REQUIRES: value >= 0
-func MapClearCapacity(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
-	}
-}
-
-// MapClearMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// *Limitation*: this Op only broadcasts the dense side to the sparse side, but not
+// the other direction.
 //
-// REQUIRES: value >= 0
-func MapClearMemoryLimit(value int64) MapClearAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// MapClearContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapClearContainer(value string) MapClearAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MapClearSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapClearSharedName(value string) MapClearAttr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// Op removes all elements in the underlying container.
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//	dense: `R`-D.  The dense Tensor operand.
 //
-// Returns the created operation.
-func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
+// Returns 1-D.  The `N` values that are operated on.
+func SparseDenseCwiseDiv(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MapClear",
-
-		Attrs: attrs,
+		Type: "SparseDenseCwiseDiv",
+		Input: []tf.Input{
+			sp_indices, sp_values, sp_shape, dense,
+		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
-type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
+// FractionalAvgPoolGradAttr is an optional argument to FractionalAvgPoolGrad.
+type FractionalAvgPoolGradAttr func(optionalAttr)
 
-// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+// FractionalAvgPoolGradOverlapping sets the optional overlapping attribute to value.
 //
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+// value: When set to True, it means when pooling, the values at the boundary
+// of adjacent pooling cells are used by both cells. For example:
 //
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
+// `index  0  1  2  3  4`
+//
+// `value  20 5  16 3  7`
+//
+// If the pooling sequence is [0, 2, 4], then 16, at index 2 will be used twice.
+// The result would be [41/3, 26/3] for fractional avg pooling.
+// If not specified, defaults to false
+func FractionalAvgPoolGradOverlapping(value bool) FractionalAvgPoolGradAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["overlapping"] = value
 	}
 }
 
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
-//
-// For each batch, this op picks a single set of sampled candidate labels.
+// Computes gradient of the FractionalAvgPool function.
 //
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+// Unlike FractionalMaxPoolGrad, we don't need to find arg_max for
+// FractionalAvgPoolGrad, we just need to evenly back-propagate each element of
+// out_backprop to those indices that form the same pooling cell. Therefore, we
+// just need to know the shape of original input tensor, instead of the whole
+// tensor.
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to randomly sample.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//	range_max: The sampler will sample integers from the interval [0, range_max).
+//	orig_input_tensor_shape: Original input tensor shape for `fractional_avg_pool`
+//	out_backprop: 4-D with shape `[batch, height, width, channels]`.  Gradients
+// w.r.t. the output of `fractional_avg_pool`.
+//	row_pooling_sequence: row pooling sequence, form pooling region with
+// col_pooling_sequence.
+//	col_pooling_sequence: column pooling sequence, form pooling region with
+// row_pooling sequence.
 //
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+// Returns 4-D.  Gradients w.r.t. the input of `fractional_avg_pool`.
+func FractionalAvgPoolGrad(scope *Scope, orig_input_tensor_shape tf.Output, out_backprop tf.Output, row_pooling_sequence tf.Output, col_pooling_sequence tf.Output, optional ...FractionalAvgPoolGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ThreadUnsafeUnigramCandidateSampler",
+		Type: "FractionalAvgPoolGrad",
 		Input: []tf.Input{
-			true_classes,
+			orig_input_tensor_shape, out_backprop, row_pooling_sequence, col_pooling_sequence,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// MaxPoolV2Attr is an optional argument to MaxPoolV2.
-type MaxPoolV2Attr func(optionalAttr)
-
-// MaxPoolV2DataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
+// Concatenates tensors along one dimension.
 //
 // Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
+//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [0, rank(values)).
+//	values: The `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
 //
-// Returns The max pooled output tensor.
-func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.
+func Concat(scope *Scope, concat_dim tf.Output, values []tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MaxPoolV2",
+		Type: "Concat",
 		Input: []tf.Input{
-			input, ksize, strides,
+			concat_dim, tf.OutputList(values),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MutableDenseHashTableV2Attr is an optional argument to MutableDenseHashTableV2.
-type MutableDenseHashTableV2Attr func(optionalAttr)
+// ResourceApplyMomentumAttr is an optional argument to ResourceApplyMomentum.
+type ResourceApplyMomentumAttr func(optionalAttr)
 
-// MutableDenseHashTableV2Container sets the optional container attribute to value.
+// ResourceApplyMomentumUseLocking sets the optional use_locking attribute to value.
 //
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableDenseHashTableV2Container(value string) MutableDenseHashTableV2Attr {
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyMomentumUseLocking(value bool) ResourceApplyMomentumAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["use_locking"] = value
 	}
 }
 
-// MutableDenseHashTableV2SharedName sets the optional shared_name attribute to value.
+// ResourceApplyMomentumUseNesterov sets the optional use_nesterov attribute to value.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableDenseHashTableV2SharedName(value string) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// MutableDenseHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// value: If `True`, the tensor passed to compute grad will be
+// var - lr * momentum * accum, so in the end, the var you get is actually
+// var - lr * momentum * accum.
 // If not specified, defaults to false
-func MutableDenseHashTableV2UseNodeNameSharing(value bool) MutableDenseHashTableV2Attr {
+func ResourceApplyMomentumUseNesterov(value bool) ResourceApplyMomentumAttr {
 	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
+		m["use_nesterov"] = value
 	}
 }
 
-// MutableDenseHashTableV2ValueShape sets the optional value_shape attribute to value.
+// Update '*var' according to the momentum scheme. Set use_nesterov = True if you
 //
-// value: The shape of each value.
-// If not specified, defaults to <>
-func MutableDenseHashTableV2ValueShape(value tf.Shape) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["value_shape"] = value
-	}
-}
-
-// MutableDenseHashTableV2InitialNumBuckets sets the optional initial_num_buckets attribute to value.
+// want to use Nesterov momentum.
 //
-// value: The initial number of hash table buckets. Must be a power
-// to 2.
-// If not specified, defaults to 131072
-func MutableDenseHashTableV2InitialNumBuckets(value int64) MutableDenseHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["initial_num_buckets"] = value
+// accum = accum * momentum + grad
+// var -= lr * accum
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	grad: The gradient.
+//	momentum: Momentum. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyMomentum(scope *Scope, var_ tf.Output, accum tf.Output, lr tf.Output, grad tf.Output, momentum tf.Output, optional ...ResourceApplyMomentumAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyMomentum",
+		Input: []tf.Input{
+			var_, accum, lr, grad, momentum,
+		},
+		Attrs: attrs,
 	}
+	return scope.AddOperation(opspec)
 }
 
-// MutableDenseHashTableV2MaxLoadFactor sets the optional max_load_factor attribute to value.
+// MaxPoolGradGradAttr is an optional argument to MaxPoolGradGrad.
+type MaxPoolGradGradAttr func(optionalAttr)
+
+// MaxPoolGradGradDataFormat sets the optional data_format attribute to value.
 //
-// value: The maximum ratio between number of entries and number of
-// buckets before growing the table. Must be between 0 and 1.
-// If not specified, defaults to 0.8
-func MutableDenseHashTableV2MaxLoadFactor(value float32) MutableDenseHashTableV2Attr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolGradGradDataFormat(value string) MaxPoolGradGradAttr {
 	return func(m optionalAttr) {
-		m["max_load_factor"] = value
+		m["data_format"] = value
 	}
 }
 
-// Creates an empty hash table that uses tensors as the backing store.
-//
-// It uses "open addressing" with quadratic reprobing to resolve
-// collisions.
-//
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a scalar. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
+// Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
-//	empty_key: The key used to represent empty key buckets internally. Must not
-// be used in insert or lookup operations.
-//	value_dtype: Type of the table values.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: 4-D.  Gradients of gradients w.r.t. the input of `max_pool`.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// Returns Handle to a table.
-func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, value_dtype tf.DataType, optional ...MutableDenseHashTableV2Attr) (table_handle tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPoolGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolGradGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"value_dtype": value_dtype}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MutableDenseHashTableV2",
+		Type: "MaxPoolGradGrad",
 		Input: []tf.Input{
-			empty_key,
+			orig_input, orig_output, grad,
 		},
 		Attrs: attrs,
 	}
@@ -13375,47 +13191,75 @@ func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, value_dtype tf.D
 	return op.Output(0)
 }
 
-// StageSizeAttr is an optional argument to StageSize.
-type StageSizeAttr func(optionalAttr)
+// Returns element-wise integer closest to x.
+//
+// If the result is midway between two representable values,
+// the even representable is chosen.
+// For example:
+//
+// ```
+// rint(-1.5) ==> -2.0
+// rint(0.5000001) ==> 1.0
+// rint([-1.7, -1.5, -0.2, 0.2, 1.5, 1.7, 2.0]) ==> [-2., -2., -0., 0., 2., 2., 2.]
+// ```
+func Rint(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Rint",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// StageSizeCapacity sets the optional capacity attribute to value.
+// OrderedMapUnstageNoKeyAttr is an optional argument to OrderedMapUnstageNoKey.
+type OrderedMapUnstageNoKeyAttr func(optionalAttr)
+
+// OrderedMapUnstageNoKeyCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageSizeCapacity(value int64) StageSizeAttr {
+func OrderedMapUnstageNoKeyCapacity(value int64) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// StageSizeMemoryLimit sets the optional memory_limit attribute to value.
+// OrderedMapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func StageSizeMemoryLimit(value int64) StageSizeAttr {
+func OrderedMapUnstageNoKeyMemoryLimit(value int64) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// StageSizeContainer sets the optional container attribute to value.
+// OrderedMapUnstageNoKeyContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func StageSizeContainer(value string) StageSizeAttr {
+func OrderedMapUnstageNoKeyContainer(value string) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// StageSizeSharedName sets the optional shared_name attribute to value.
+// OrderedMapUnstageNoKeySharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func StageSizeSharedName(value string) StageSizeAttr {
+func OrderedMapUnstageNoKeySharedName(value string) OrderedMapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op returns the number of elements in the underlying container.
-func StageSize(scope *Scope, dtypes []tf.DataType, optional ...StageSizeAttr) (size tf.Output) {
+// Op removes and returns the (key, value) element with the smallest
+//
+// key from the underlying container.   If the underlying container
+// does not contain elements, the op will block until it does.
+func OrderedMapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -13424,200 +13268,79 @@ func StageSize(scope *Scope, dtypes []tf.DataType, optional ...StageSizeAttr) (s
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StageSize",
-
+		Type: "OrderedMapUnstageNoKey",
+		Input: []tf.Input{
+			indices,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Produces the max pool of the input tensor for quantized types.
-//
-// Arguments:
-//	input: The 4D (batch x rows x cols x depth) Tensor to MaxReduce over.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	ksize: The size of the window for each dimension of the input tensor.
-// The length must be 4 to match the number of dimensions of the input.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor. The length must be 4 to match the number of dimensions of the input.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedMaxPool(scope *Scope, input tf.Output, min_input tf.Output, max_input tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output, min_output tf.Output, max_output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	opspec := tf.OpSpec{
-		Type: "QuantizedMaxPool",
-		Input: []tf.Input{
-			input, min_input, max_input,
-		},
-		Attrs: attrs,
+	var idx int
+	var err error
+	key = op.Output(idx)
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("OrderedMapUnstageNoKey", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return key, values
 }
 
-// Computes softplus: `log(exp(features) + 1)`.
-func Softplus(scope *Scope, features tf.Output) (activations tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Softplus",
-		Input: []tf.Input{
-			features,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes exponential of x - 1 element-wise.
-//
-// I.e., \\(y = (\exp x) - 1\\).
-func Expm1(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Expm1",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the number of records this Reader has produced.
-//
-// This is the same as the number of ReaderRead executions that have
-// succeeded.
-//
-// Arguments:
-//	reader_handle: Handle to a Reader.
-func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_produced tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReaderNumRecordsProducedV2",
-		Input: []tf.Input{
-			reader_handle,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
+// MaxPool3DGradGradAttr is an optional argument to MaxPool3DGradGrad.
+type MaxPool3DGradGradAttr func(optionalAttr)
 
-// Computes the sum along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// \\(output_i = \sum_j data_j\\) where sum is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
+// MaxPool3DGradGradDataFormat sets the optional data_format attribute to value.
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentSum",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func MaxPool3DGradGradDataFormat(value string) MaxPool3DGradGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Creates a dataset that emits the lines of one or more text files.
+// Computes second-order gradients of the maxpooling function.
 //
 // Arguments:
-//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
-// read.
-//	compression_type: A scalar containing either (i) the empty string (no
-// compression), (ii) "ZLIB", or (iii) "GZIP".
-//	buffer_size: A scalar containing the number of bytes to buffer.
-func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TextLineDataset",
-		Input: []tf.Input{
-			filenames, compression_type, buffer_size,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes gradients for SparseSegmentMean.
-//
-// Returns tensor "output" with same shape as grad, except for dimension 0 whose
-// value is output_dim0.
+//	orig_input: The original input tensor.
+//	orig_output: The original output tensor.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
 //
-// Arguments:
-//	grad: gradient propagated to the SparseSegmentMean op.
-//	indices: indices passed to the corresponding SparseSegmentMean op.
-//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
-//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
-func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
+// Returns Gradients of gradients w.r.t. the input to `max_pool`.
+func MaxPool3DGradGrad(scope *Scope, orig_input tf.Output, orig_output tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPool3DGradGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "SparseSegmentMeanGrad",
-		Input: []tf.Input{
-			grad, indices, segment_ids, output_dim0,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the truth value of (x >= y) element-wise.
-//
-// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "GreaterEqual",
+		Type: "MaxPool3DGradGrad",
 		Input: []tf.Input{
-			x, y,
+			orig_input, orig_output, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Conv3DAttr is an optional argument to Conv3D.
-type Conv3DAttr func(optionalAttr)
+// Conv3DBackpropFilterV2Attr is an optional argument to Conv3DBackpropFilterV2.
+type Conv3DBackpropFilterV2Attr func(optionalAttr)
 
-// Conv3DDataFormat sets the optional data_format attribute to value.
+// Conv3DBackpropFilterV2DataFormat sets the optional data_format attribute to value.
 //
 // value: The data format of the input and output data. With the
 // default format "NDHWC", the data is stored in the order of:
@@ -13625,13 +13348,13 @@ type Conv3DAttr func(optionalAttr)
 // Alternatively, the format could be "NCDHW", the data storage order is:
 //     [batch, in_channels, in_depth, in_height, in_width].
 // If not specified, defaults to "NDHWC"
-func Conv3DDataFormat(value string) Conv3DAttr {
+func Conv3DBackpropFilterV2DataFormat(value string) Conv3DBackpropFilterV2Attr {
 	return func(m optionalAttr) {
 		m["data_format"] = value
 	}
 }
 
-// Conv3DDilations sets the optional dilations attribute to value.
+// Conv3DBackpropFilterV2Dilations sets the optional dilations attribute to value.
 //
 // value: 1-D tensor of length 5.  The dilation factor for each dimension of
 // `input`. If set to k > 1, there will be k-1 skipped cells between each
@@ -13639,28 +13362,26 @@ func Conv3DDataFormat(value string) Conv3DAttr {
 // value of `data_format`, see above for details. Dilations in the batch and
 // depth dimensions must be 1.
 // If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
-func Conv3DDilations(value []int64) Conv3DAttr {
+func Conv3DBackpropFilterV2Dilations(value []int64) Conv3DBackpropFilterV2Attr {
 	return func(m optionalAttr) {
 		m["dilations"] = value
 	}
 }
 
-// Computes a 3-D convolution given 5-D `input` and `filter` tensors.
-//
-// In signal processing, cross-correlation is a measure of similarity of
-// two waveforms as a function of a time-lag applied to one of them. This
-// is also known as a sliding dot product or sliding inner-product.
-//
-// Our Conv3D implements a form of cross-correlation.
+// Computes the gradients of 3-D convolution with respect to the filter.
 //
 // Arguments:
-//	input: Shape `[batch, in_depth, in_height, in_width, in_channels]`.
-//	filter: Shape `[filter_depth, filter_height, filter_width, in_channels,
-// out_channels]`. `in_channels` must match between `input` and `filter`.
+//	input: Shape `[batch, depth, rows, cols, in_channels]`.
+//	filter_sizes: An integer vector representing the tensor shape of `filter`,
+// where `filter` is a 5-D
+// `[filter_depth, filter_height, filter_width, in_channels, out_channels]`
+// tensor.
+//	out_backprop: Backprop signal of shape `[batch, out_depth, out_rows, out_cols,
+// out_channels]`.
 //	strides: 1-D tensor of length 5. The stride of the sliding window for each
 // dimension of `input`. Must have `strides[0] = strides[4] = 1`.
 //	padding: The type of padding algorithm to use.
-func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv3DAttr) (output tf.Output) {
+func Conv3DBackpropFilterV2(scope *Scope, input tf.Output, filter_sizes tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...Conv3DBackpropFilterV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -13669,9 +13390,9 @@ func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, pa
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Conv3D",
+		Type: "Conv3DBackpropFilterV2",
 		Input: []tf.Input{
-			input, filter,
+			input, filter_sizes, out_backprop,
 		},
 		Attrs: attrs,
 	}
@@ -13679,293 +13400,216 @@ func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, pa
 	return op.Output(0)
 }
 
-// Adds up a SparseTensor and a dense Tensor, using these special rules:
-//
-// (1) Broadcasts the dense side to have the same shape as the sparse side, if
-//     eligible;
-// (2) Then, only the dense values pointed to by the indices of the SparseTensor
-//     participate in the cwise addition.
+// Execute a sub graph on a remote processor.
 //
-// By these rules, the result is a logical SparseTensor with exactly the same
-// indices and shape, but possibly with different non-zero values.  The output of
-// this Op is the resultant non-zero values.
+// The graph specifications(such as graph itself, input tensors and output names)
+// are stored as a serialized protocol buffer of RemoteFusedGraphExecuteInfo
+// as serialized_remote_fused_graph_execute_info.
+// The specifications will be passed to a dedicated registered
+// remote fused graph executor.  The executor will send the graph specifications
+// to a remote processor and execute that graph.  The execution results
+// will be passed to consumer nodes as outputs of this node.
 //
 // Arguments:
-//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, possibly not in canonical ordering.
-//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
-//	sp_shape: 1-D.  Shape of the input SparseTensor.
-//	dense: `R`-D.  The dense Tensor operand.
+//	inputs: Arbitrary number of tensors with arbitrary data types
 //
-// Returns 1-D.  The `N` values that are operated on.
-func SparseDenseCwiseAdd(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+//	serialized_remote_fused_graph_execute_info: Serialized protocol buffer
+// of RemoteFusedGraphExecuteInfo which contains graph specifications.
+//
+// Returns Arbitrary number of tensors with arbitrary data types
+func RemoteFusedGraphExecute(scope *Scope, inputs []tf.Output, Toutputs []tf.DataType, serialized_remote_fused_graph_execute_info string) (outputs []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"Toutputs": Toutputs, "serialized_remote_fused_graph_execute_info": serialized_remote_fused_graph_execute_info}
 	opspec := tf.OpSpec{
-		Type: "SparseDenseCwiseAdd",
+		Type: "RemoteFusedGraphExecute",
 		Input: []tf.Input{
-			sp_indices, sp_values, sp_shape, dense,
+			tf.OutputList(inputs),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Read an element from the TensorArray into output `value`.
-//
-// Arguments:
-//	handle: The handle to a TensorArray.
-//
-//	flow_in: A float scalar that enforces proper chaining of operations.
-//	dtype: The type of the elem that is returned.
-//
-// Returns The tensor that is read from the TensorArray.
-func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV3",
-		Input: []tf.Input{
-			handle, index, flow_in,
-		},
-		Attrs: attrs,
+	var idx int
+	var err error
+	if outputs, idx, err = makeOutputList(op, idx, "outputs"); err != nil {
+		scope.UpdateErr("RemoteFusedGraphExecute", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return outputs
 }
 
-// QuantizeV2Attr is an optional argument to QuantizeV2.
-type QuantizeV2Attr func(optionalAttr)
-
-// QuantizeV2Mode sets the optional mode attribute to value.
-// If not specified, defaults to "MIN_COMBINED"
-func QuantizeV2Mode(value string) QuantizeV2Attr {
-	return func(m optionalAttr) {
-		m["mode"] = value
-	}
-}
+// SerializeManySparseAttr is an optional argument to SerializeManySparse.
+type SerializeManySparseAttr func(optionalAttr)
 
-// QuantizeV2RoundMode sets the optional round_mode attribute to value.
-// If not specified, defaults to "HALF_AWAY_FROM_ZERO"
-func QuantizeV2RoundMode(value string) QuantizeV2Attr {
+// SerializeManySparseOutType sets the optional out_type attribute to value.
+//
+// value: The `dtype` to use for serialization; the supported types are `string`
+// (default) and `variant`.
+// If not specified, defaults to DT_STRING
+func SerializeManySparseOutType(value tf.DataType) SerializeManySparseAttr {
 	return func(m optionalAttr) {
-		m["round_mode"] = value
+		m["out_type"] = value
 	}
 }
 
-// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
-//
-// [min_range, max_range] are scalar floats that specify the range for
-// the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.  The
-// 'round_mode' attribute controls which rounding tie-breaking algorithm is used
-// when rounding float values to their quantized equivalents.
-//
-// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
-//
-// ```
-// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
-// if T == qint8, out[i] -= (range(T) + 1) / 2.0
-// ```
-// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
-//
-// *MIN_COMBINED Mode Example*
-//
-// Assume the input is type float and has a possible range of [0.0, 6.0] and the
-// output type is quint8 ([0, 255]). The min_range and max_range values should be
-// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each
-// value of the input by 255/6 and cast to quint8.
-//
-// If the output type was qint8 ([-128, 127]), the operation will additionally
-// subtract each value by 128 prior to casting, so that the range of values aligns
-// with the range of qint8.
-//
-// If the mode is 'MIN_FIRST', then this approach is used:
-//
-// ```
-// num_discrete_values = 1 << (# of bits in T)
-// range_adjust = num_discrete_values / (num_discrete_values - 1)
-// range = (range_max - range_min) * range_adjust
-// range_scale = num_discrete_values / range
-// quantized = round(input * range_scale) - round(range_min * range_scale) +
-//   numeric_limits<T>::min()
-// quantized = max(quantized, numeric_limits<T>::min())
-// quantized = min(quantized, numeric_limits<T>::max())
-// ```
-//
-// The biggest difference between this and MIN_COMBINED is that the minimum range
-// is rounded first, before it's subtracted from the rounded value. With
-// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing
-// and dequantizing will introduce a larger and larger error.
-//
-// *SCALED mode Example*
-//
-// `SCALED` mode matches the quantization approach used in
-// `QuantizeAndDequantize{V2|V3}`.
-//
-// If the mode is `SCALED`, we do not use the full range of the output type,
-// choosing to elide the lowest possible value for symmetry (e.g., output range is
-// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
-// 0.
-//
-// We first find the range of values in our tensor. The
-// range we use is always centered on 0, so we find m such that
-// ```c++
-//   m = max(abs(input_min), abs(input_max))
-// ```
-//
-// Our input tensor range is then `[-m, m]`.
-//
-// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
-// If T is signed, this is
-// ```
-//   num_bits = sizeof(T) * 8
-//   [min_fixed, max_fixed] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
-// ```
-//
-// Otherwise, if T is unsigned, the fixed-point range is
-// ```
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
-// ```
-//
-// From this we compute our scaling factor, s:
-// ```c++
-//   s = (max_fixed - min_fixed) / (2 * m)
-// ```
+// Serialize an `N`-minibatch `SparseTensor` into an `[N, 3]` `Tensor` object.
 //
-// Now we can quantize the elements of our tensor:
-// ```c++
-// result = round(input * s)
-// ```
+// The `SparseTensor` must have rank `R` greater than 1, and the first dimension
+// is treated as the minibatch dimension.  Elements of the `SparseTensor`
+// must be sorted in increasing order of this first dimension.  The serialized
+// `SparseTensor` objects going into each row of `serialized_sparse` will have
+// rank `R-1`.
 //
-// One thing to watch out for is that the operator may choose to adjust the
-// requested minimum and maximum values slightly during the quantization process,
-// so you should always use the output ports as the range for further calculations.
-// For example, if the requested minimum and maximum values are close to equal,
-// they will be separated by a small epsilon value to prevent ill-formed quantized
-// buffers from being created. Otherwise, you can end up with buffers where all the
-// quantized values map to the same float value, which causes problems for
-// operations that have to perform further calculations on them.
+// The minibatch size `N` is extracted from `sparse_shape[0]`.
 //
 // Arguments:
-//
-//	min_range: The minimum scalar value possibly produced for the input.
-//	max_range: The maximum scalar value possibly produced for the input.
-//
-//
-// Returns The quantized data produced from the float input.The actual minimum scalar value used for the output.The actual maximum scalar value used for the output.
-func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+//	sparse_indices: 2-D.  The `indices` of the minibatch `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the minibatch `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the minibatch `SparseTensor`.
+func SerializeManySparse(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...SerializeManySparseAttr) (serialized_sparse tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"T": T}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeV2",
+		Type: "SerializeManySparse",
 		Input: []tf.Input{
-			input, min_range, max_range,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Returns the truth value of (x < y) element-wise.
-//
-// *NOTE*: `Less` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// Computes inverse hyperbolic cosine of x element-wise.
+func Acosh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Less",
+		Type: "Acosh",
 		Input: []tf.Input{
-			x, y,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedReluXAttr is an optional argument to QuantizedReluX.
-type QuantizedReluXAttr func(optionalAttr)
+// TensorArrayV2Attr is an optional argument to TensorArrayV2.
+type TensorArrayV2Attr func(optionalAttr)
 
-// QuantizedReluXOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QUINT8
-func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
+// TensorArrayV2ElementShape sets the optional element_shape attribute to value.
+// If not specified, defaults to <unknown_rank:true >
+func TensorArrayV2ElementShape(value tf.Shape) TensorArrayV2Attr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["element_shape"] = value
 	}
 }
 
-// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
-//
-// Arguments:
-//
-//
-//	min_features: The float value that the lowest quantized value represents.
-//	max_features: The float value that the highest quantized value represents.
+// TensorArrayV2DynamicSize sets the optional dynamic_size attribute to value.
+// If not specified, defaults to false
+func TensorArrayV2DynamicSize(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["dynamic_size"] = value
+	}
+}
+
+// TensorArrayV2ClearAfterRead sets the optional clear_after_read attribute to value.
+// If not specified, defaults to true
+func TensorArrayV2ClearAfterRead(value bool) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["clear_after_read"] = value
+	}
+}
+
+// TensorArrayV2TensorArrayName sets the optional tensor_array_name attribute to value.
+// If not specified, defaults to ""
+func TensorArrayV2TensorArrayName(value string) TensorArrayV2Attr {
+	return func(m optionalAttr) {
+		m["tensor_array_name"] = value
+	}
+}
+
+// Deprecated. Use TensorArrayV3
 //
-// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
-func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayV3
+func TensorArrayV2(scope *Scope, size tf.Output, dtype tf.DataType, optional ...TensorArrayV2Attr) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedReluX",
+		Type: "TensorArrayV2",
 		Input: []tf.Input{
-			features, max_value, min_features, max_features,
+			size,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
-type WholeFileReaderV2Attr func(optionalAttr)
+// DecodeCSVAttr is an optional argument to DecodeCSV.
+type DecodeCSVAttr func(optionalAttr)
 
-// WholeFileReaderV2Container sets the optional container attribute to value.
+// DecodeCSVFieldDelim sets the optional field_delim attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
+// value: char delimiter to separate fields in a record.
+// If not specified, defaults to ","
+func DecodeCSVFieldDelim(value string) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["field_delim"] = value
 	}
 }
 
-// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
+// DecodeCSVUseQuoteDelim sets the optional use_quote_delim attribute to value.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
+// value: If false, treats double quotation marks as regular
+// characters inside of the string fields (ignoring RFC 4180, Section 2,
+// Bullet 5).
+// If not specified, defaults to true
+func DecodeCSVUseQuoteDelim(value bool) DecodeCSVAttr {
+	return func(m optionalAttr) {
+		m["use_quote_delim"] = value
+	}
+}
+
+// DecodeCSVNaValue sets the optional na_value attribute to value.
+//
+// value: Additional string to recognize as NA/NaN.
 // If not specified, defaults to ""
-func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
+func DecodeCSVNaValue(value string) DecodeCSVAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["na_value"] = value
 	}
 }
 
-// A Reader that outputs the entire contents of a file as a value.
+// Convert CSV records to tensors. Each column maps to one tensor.
 //
-// To use, enqueue filenames in a Queue.  The output of ReaderRead will
-// be a filename (key) and the contents of that file (value).
+// RFC 4180 format is expected for the CSV records.
+// (https://tools.ietf.org/html/rfc4180)
+// Note that we allow leading and trailing spaces with int or float field.
 //
-// Returns The handle to reference the Reader.
-func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
+// Arguments:
+//	records: Each string is a record/row in the csv and all records should have
+// the same format.
+//	record_defaults: One tensor per column of the input record, with either a
+// scalar default value for that column or empty if the column is required.
+//
+// Returns Each tensor will have the same shape as records.
+func DecodeCSV(scope *Scope, records tf.Output, record_defaults []tf.Output, optional ...DecodeCSVAttr) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -13974,56 +13618,9 @@ func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "WholeFileReaderV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Transforms a tf.Example proto (as a string) into typed tensors.
-//
-// Arguments:
-//	serialized: A vector containing a batch of binary serialized Example protos.
-//	dense_defaults: A list of Tensors (some may be empty), whose length matches
-// the length of `dense_keys`. dense_defaults[j] provides default values
-// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
-// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
-// The input type is inferred from dense_defaults[j], even when it's empty.
-// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
-// then the shape of dense_defaults[j] must match that of dense_shapes[j].
-// If dense_shapes[j] has an undefined major dimension (variable strides dense
-// feature), dense_defaults[j] must contain a single element:
-// the padding element.
-//	num_sparse: The number of sparse features to be parsed from the example. This
-// must match the lengths of `sparse_keys` and `sparse_types`.
-//	sparse_keys: A list of `num_sparse` strings.
-// The keys expected in the Examples' features associated with sparse values.
-//	dense_keys: The keys expected in the Examples' features associated with dense
-// values.
-//	sparse_types: A list of `num_sparse` types; the data types of data in each
-// Feature given in sparse_keys.
-// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-//	dense_shapes: The shapes of data in each Feature given in dense_keys.
-// The length of this list must match the length of `dense_keys`.  The
-// number of elements in the Feature corresponding to dense_key[j] must
-// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
-// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
-// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
-// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
-// D1, .., DN), where M is the number of blocks of elements of length
-// D1 * .... * DN, in the input.
-func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
-	opspec := tf.OpSpec{
-		Type: "ParseSingleExample",
+		Type: "DecodeCSV",
 		Input: []tf.Input{
-			serialized, tf.OutputList(dense_defaults),
+			records, tf.OutputList(record_defaults),
 		},
 		Attrs: attrs,
 	}
@@ -14033,81 +13630,137 @@ func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.
 	}
 	var idx int
 	var err error
-	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("DecodeCSV", err)
 		return
 	}
-	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+	return output
+}
+
+// MapClearAttr is an optional argument to MapClear.
+type MapClearAttr func(optionalAttr)
+
+// MapClearCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapClearCapacity(value int64) MapClearAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
 	}
-	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
-		return
+}
+
+// MapClearMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
+//
+// REQUIRES: value >= 0
+func MapClearMemoryLimit(value int64) MapClearAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
 	}
-	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleExample", err)
+}
+
+// MapClearContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapClearContainer(value string) MapClearAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapClearSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapClearSharedName(value string) MapClearAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes all elements in the underlying container.
+//
+// Returns the created operation.
+func MapClear(scope *Scope, dtypes []tf.DataType, optional ...MapClearAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
 		return
 	}
-	return sparse_indices, sparse_values, sparse_shapes, dense_values
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "MapClear",
+
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
 }
 
-// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
-type QuantizedConv2DAttr func(optionalAttr)
+// ThreadUnsafeUnigramCandidateSamplerAttr is an optional argument to ThreadUnsafeUnigramCandidateSampler.
+type ThreadUnsafeUnigramCandidateSamplerAttr func(optionalAttr)
 
-// QuantizedConv2DOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
+// ThreadUnsafeUnigramCandidateSamplerSeed sets the optional seed attribute to value.
+//
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["seed"] = value
 	}
 }
 
-// QuantizedConv2DDilations sets the optional dilations attribute to value.
+// ThreadUnsafeUnigramCandidateSamplerSeed2 sets the optional seed2 attribute to value.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each
-// filter element on that dimension. The dimension order is determined by the
-// value of `data_format`, see above for details. Dilations in the batch and
-// depth dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func QuantizedConv2DDilations(value []int64) QuantizedConv2DAttr {
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func ThreadUnsafeUnigramCandidateSamplerSeed2(value int64) ThreadUnsafeUnigramCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["dilations"] = value
+		m["seed2"] = value
 	}
 }
 
-// Computes a 2D convolution given quantized 4D input and filter tensors.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// The inputs are quantized tensors where the lowest value represents the real
-// number of the associated minimum, and the highest represents the maximum.
-// This means that you can only interpret the quantized output in the same way, by
-// taking the returned minimum and maximum values into account.
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
 //
-// Arguments:
+// For each batch, this op picks a single set of sampled candidate labels.
 //
-//	filter: filter's input_depth dimension must match input's depth dimensions.
-//	min_input: The float value that the lowest quantized input value represents.
-//	max_input: The float value that the highest quantized input value represents.
-//	min_filter: The float value that the lowest quantized filter value represents.
-//	max_filter: The float value that the highest quantized filter value represents.
-//	strides: The stride of the sliding window for each dimension of the input
-// tensor.
-//	padding: The type of padding algorithm to use.
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
+// Arguments:
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to randomly sample.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
+//	range_max: The sampler will sample integers from the interval [0, range_max).
+//
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func ThreadUnsafeUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, range_max int64, optional ...ThreadUnsafeUnigramCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique, "range_max": range_max}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedConv2D",
+		Type: "ThreadUnsafeUnigramCandidateSampler",
 		Input: []tf.Input{
-			input, filter, min_input, max_input, min_filter, max_filter,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
@@ -14115,44 +13768,45 @@ func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ResourceGatherAttr is an optional argument to ResourceGather.
-type ResourceGatherAttr func(optionalAttr)
+// MaxPoolV2Attr is an optional argument to MaxPoolV2.
+type MaxPoolV2Attr func(optionalAttr)
 
-// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
+// MaxPoolV2DataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolV2DataFormat(value string) MaxPoolV2Attr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["data_format"] = value
 	}
 }
 
-// Gather slices from the variable pointed to by `resource` according to `indices`.
-//
-// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
-// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
-//
-// ```python
-//     # Scalar indices
-//     output[:, ..., :] = params[indices, :, ... :]
+// Performs max pooling on the input.
 //
-//     # Vector indices
-//     output[i, :, ..., :] = params[indices[i], :, ... :]
+// Arguments:
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-//     # Higher rank indices
-//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
-// ```
-func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
+// Returns The max pooled output tensor.
+func MaxPoolV2(scope *Scope, input tf.Output, ksize tf.Output, strides tf.Output, padding string, optional ...MaxPoolV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{"padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceGather",
+		Type: "MaxPoolV2",
 		Input: []tf.Input{
-			resource, indices,
+			input, ksize, strides,
 		},
 		Attrs: attrs,
 	}
@@ -14160,112 +13814,146 @@ func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype t
 	return op.Output(0)
 }
 
-// Delete the TensorArray from its resource container.
-//
-// This enables the user to close and release the resource in the middle
-// of a step/run.
+// MutableDenseHashTableV2Attr is an optional argument to MutableDenseHashTableV2.
+type MutableDenseHashTableV2Attr func(optionalAttr)
+
+// MutableDenseHashTableV2Container sets the optional container attribute to value.
 //
-// Arguments:
-//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func MutableDenseHashTableV2Container(value string) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MutableDenseHashTableV2SharedName sets the optional shared_name attribute to value.
 //
-// Returns the created operation.
-func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableDenseHashTableV2SharedName(value string) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayCloseV3",
-		Input: []tf.Input{
-			handle,
-		},
+}
+
+// MutableDenseHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// If not specified, defaults to false
+func MutableDenseHashTableV2UseNodeNameSharing(value bool) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["use_node_name_sharing"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Adds two `SparseTensor` objects to produce another `SparseTensor`.
+// MutableDenseHashTableV2ValueShape sets the optional value_shape attribute to value.
 //
-// The input `SparseTensor` objects' indices are assumed ordered in standard
-// lexicographic order.  If this is not the case, before this step run
-// `SparseReorder` to restore index ordering.
+// value: The shape of each value.
+// If not specified, defaults to <>
+func MutableDenseHashTableV2ValueShape(value tf.Shape) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["value_shape"] = value
+	}
+}
+
+// MutableDenseHashTableV2InitialNumBuckets sets the optional initial_num_buckets attribute to value.
 //
-// By default, if two values sum to zero at some index, the output `SparseTensor`
-// would still include that particular location in its index, storing a zero in the
-// corresponding value slot.  To override this, callers can specify `thresh`,
-// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
-// corresponding value and index would then not be included.  In particular,
-// `thresh == 0` (default) means everything is kept and actual thresholding happens
-// only for a positive value.
+// value: The initial number of hash table buckets. Must be a power
+// to 2.
+// If not specified, defaults to 131072
+func MutableDenseHashTableV2InitialNumBuckets(value int64) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["initial_num_buckets"] = value
+	}
+}
+
+// MutableDenseHashTableV2MaxLoadFactor sets the optional max_load_factor attribute to value.
 //
-// In the following shapes, `nnz` is the count after taking `thresh` into account.
+// value: The maximum ratio between number of entries and number of
+// buckets before growing the table. Must be between 0 and 1.
+// If not specified, defaults to 0.8
+func MutableDenseHashTableV2MaxLoadFactor(value float32) MutableDenseHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["max_load_factor"] = value
+	}
+}
+
+// Creates an empty hash table that uses tensors as the backing store.
+//
+// It uses "open addressing" with quadratic reprobing to resolve
+// collisions.
+//
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a scalar. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
-//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
-//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
-//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
-//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
-// pair takes space.
-func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
+//	empty_key: The key used to represent empty key buckets internally. Must not
+// be used in insert or lookup operations.
+//	value_dtype: Type of the table values.
+//
+// Returns Handle to a table.
+func MutableDenseHashTableV2(scope *Scope, empty_key tf.Output, value_dtype tf.DataType, optional ...MutableDenseHashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"value_dtype": value_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseAdd",
+		Type: "MutableDenseHashTableV2",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
+			empty_key,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// OrderedMapPeekAttr is an optional argument to OrderedMapPeek.
-type OrderedMapPeekAttr func(optionalAttr)
+// StageSizeAttr is an optional argument to StageSize.
+type StageSizeAttr func(optionalAttr)
 
-// OrderedMapPeekCapacity sets the optional capacity attribute to value.
+// StageSizeCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapPeekCapacity(value int64) OrderedMapPeekAttr {
+func StageSizeCapacity(value int64) StageSizeAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// OrderedMapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// StageSizeMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func OrderedMapPeekMemoryLimit(value int64) OrderedMapPeekAttr {
+func StageSizeMemoryLimit(value int64) StageSizeAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// OrderedMapPeekContainer sets the optional container attribute to value.
+// StageSizeContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func OrderedMapPeekContainer(value string) OrderedMapPeekAttr {
+func StageSizeContainer(value string) StageSizeAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// OrderedMapPeekSharedName sets the optional shared_name attribute to value.
+// StageSizeSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func OrderedMapPeekSharedName(value string) OrderedMapPeekAttr {
+func StageSizeSharedName(value string) StageSizeAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op peeks at the values at the specified key.  If the
-//
-// underlying container does not contain this key
-// this op will block until it does.   This Op is optimized for
-// performance.
-func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapPeekAttr) (values []tf.Output) {
+// Op returns the number of elements in the underlying container.
+func StageSize(scope *Scope, dtypes []tf.DataType, optional ...StageSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -14274,253 +13962,188 @@ func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapPeek",
-		Input: []tf.Input{
-			key, indices,
-		},
+		Type: "StageSize",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Produces the max pool of the input tensor for quantized types.
+//
+// Arguments:
+//	input: The 4D (batch x rows x cols x depth) Tensor to MaxReduce over.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	ksize: The size of the window for each dimension of the input tensor.
+// The length must be 4 to match the number of dimensions of the input.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor. The length must be 4 to match the number of dimensions of the input.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedMaxPool(scope *Scope, input tf.Output, min_input tf.Output, max_input tf.Output, ksize []int64, strides []int64, padding string) (output tf.Output, min_output tf.Output, max_output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("OrderedMapPeek", err)
-		return
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	opspec := tf.OpSpec{
+		Type: "QuantizedMaxPool",
+		Input: []tf.Input{
+			input, min_input, max_input,
+		},
+		Attrs: attrs,
 	}
-	return values
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
-type DecodeAndCropJpegAttr func(optionalAttr)
-
-// DecodeAndCropJpegChannels sets the optional channels attribute to value.
-//
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["channels"] = value
+// Computes softplus: `log(exp(features) + 1)`.
+func Softplus(scope *Scope, features tf.Output) (activations tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
-//
-// value: Downscaling ratio.
-// If not specified, defaults to 1
-func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["ratio"] = value
+	opspec := tf.OpSpec{
+		Type: "Softplus",
+		Input: []tf.Input{
+			features,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
+// Computes exponential of x - 1 element-wise.
 //
-// value: If true use a slower but nicer upscaling of the
-// chroma planes (yuv420/422 only).
-// If not specified, defaults to true
-func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["fancy_upscaling"] = value
+// I.e., \\(y = (\exp x) - 1\\).
+func Expm1(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Expm1",
+		Input: []tf.Input{
+			x,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
+// Returns the number of records this Reader has produced.
 //
-// value: If true try to recover an image from truncated input.
-// If not specified, defaults to false
-func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["try_recover_truncated"] = value
+// This is the same as the number of ReaderRead executions that have
+// succeeded.
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+func ReaderNumRecordsProducedV2(scope *Scope, reader_handle tf.Output) (records_produced tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderNumRecordsProducedV2",
+		Input: []tf.Input{
+			reader_handle,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
+// Computes the sum along segments of a tensor.
 //
-// value: The minimum required fraction of lines before a truncated
-// input is accepted.
-// If not specified, defaults to 1
-func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["acceptable_fraction"] = value
-	}
-}
-
-// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
-//
-// value: string specifying a hint about the algorithm used for
-// decompression.  Defaults to "" which maps to a system-specific
-// default.  Currently valid values are ["INTEGER_FAST",
-// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
-// jpeg library changes to a version that does not have that specific
-// option.)
-// If not specified, defaults to ""
-func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
-	return func(m optionalAttr) {
-		m["dct_method"] = value
-	}
-}
-
-// Decode and Crop a JPEG-encoded image to a uint8 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the JPEG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-//
-// If needed, the JPEG-encoded image is transformed to match the requested number
-// of color channels.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// The attr `ratio` allows downscaling the image by an integer factor during
-// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
-// downscaling the image later.
+// Computes a tensor such that
+// \\(output_i = \sum_j data_j\\) where sum is over `j` such
+// that `segment_ids[j] == i`.
 //
+// If the sum is empty for a given segment ID `i`, `output[i] = 0`.
 //
-// It is equivalent to a combination of decode and crop, but much faster by only
-// decoding partial jpeg image.
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentSum.png" alt>
+// </div>
 //
 // Arguments:
-//	contents: 0-D.  The JPEG-encoded image.
-//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
 //
-// Returns 3-D with shape `[height, width, channels]`..
-func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentSum(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DecodeAndCropJpeg",
+		Type: "SegmentSum",
 		Input: []tf.Input{
-			contents, crop_window,
+			data, segment_ids,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AllCandidateSamplerAttr is an optional argument to AllCandidateSampler.
-type AllCandidateSamplerAttr func(optionalAttr)
-
-// AllCandidateSamplerSeed sets the optional seed attribute to value.
-//
-// value: If either seed or seed2 are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func AllCandidateSamplerSeed(value int64) AllCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// AllCandidateSamplerSeed2 sets the optional seed2 attribute to value.
-//
-// value: An second seed to avoid seed collision.
-// If not specified, defaults to 0
-func AllCandidateSamplerSeed2(value int64) AllCandidateSamplerAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Generates labels for candidate sampling with a learned unigram distribution.
-//
-// See explanations of candidate sampling and the data formats at
-// go/candidate-sampling.
-//
-// For each batch, this op picks a single set of sampled candidate labels.
-//
-// The advantages of sampling candidates per-batch are simplicity and the
-// possibility of efficient dense matrix multiplication. The disadvantage is that
-// the sampled candidates must be chosen independently of the context and of the
-// true labels.
+// Creates a dataset that emits the lines of one or more text files.
 //
 // Arguments:
-//	true_classes: A batch_size * num_true matrix, in which each row contains the
-// IDs of the num_true target_classes in the corresponding original label.
-//	num_true: Number of true labels per context.
-//	num_sampled: Number of candidates to produce.
-//	unique: If unique is true, we sample with rejection, so that all sampled
-// candidates in a batch are unique. This requires some approximation to
-// estimate the post-rejection sampling probabilities.
-//
-// Returns A vector of length num_sampled, in which each element is
-// the ID of a sampled candidate.A batch_size * num_true matrix, representing
-// the number of times each candidate is expected to occur in a batch
-// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
-// candidate representing the number of times the candidate is expected
-// to occur in a batch of sampled candidates.  If unique=true, then this is a
-// probability.
-func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
+//	filenames: A scalar or a vector containing the name(s) of the file(s) to be
+// read.
+//	compression_type: A scalar containing either (i) the empty string (no
+// compression), (ii) "ZLIB", or (iii) "GZIP".
+//	buffer_size: A scalar containing the number of bytes to buffer.
+func TextLineDataset(scope *Scope, filenames tf.Output, compression_type tf.Output, buffer_size tf.Output) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AllCandidateSampler",
+		Type: "TextLineDataset",
 		Input: []tf.Input{
-			true_classes,
+			filenames, compression_type, buffer_size,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Saves the input tensors to disk.
-//
-// The size of `tensor_names` must match the number of tensors in `data`. `data[i]`
-// is written to `filename` with name `tensor_names[i]`.
+// Computes gradients for SparseSegmentMean.
 //
-// See also `SaveSlices`.
+// Returns tensor "output" with same shape as grad, except for dimension 0 whose
+// value is output_dim0.
 //
 // Arguments:
-//	filename: Must have a single element. The name of the file to which we write
-// the tensor.
-//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
-//	data: `N` tensors to save.
-//
-// Returns the created operation.
-func Save(scope *Scope, filename tf.Output, tensor_names tf.Output, data []tf.Output) (o *tf.Operation) {
+//	grad: gradient propagated to the SparseSegmentMean op.
+//	indices: indices passed to the corresponding SparseSegmentMean op.
+//	segment_ids: segment_ids passed to the corresponding SparseSegmentMean op.
+//	output_dim0: dimension 0 of "data" passed to SparseSegmentMean op.
+func SparseSegmentMeanGrad(scope *Scope, grad tf.Output, indices tf.Output, segment_ids tf.Output, output_dim0 tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Save",
+		Type: "SparseSegmentMeanGrad",
 		Input: []tf.Input{
-			filename, tensor_names, tf.OutputList(data),
+			grad, indices, segment_ids, output_dim0,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns element-wise remainder of division. When `x < 0` xor `y < 0` is
-//
-// true, this follows Python semantics in that the result here is consistent
-// with a flooring divide. E.g. `floor(x / y) * y + mod(x, y) = x`.
+// Returns the truth value of (x >= y) element-wise.
 //
-// *NOTE*: `FloorMod` supports broadcasting. More about broadcasting
+// *NOTE*: `GreaterEqual` supports broadcasting. More about broadcasting
 // [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+func GreaterEqual(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "FloorMod",
+		Type: "GreaterEqual",
 		Input: []tf.Input{
 			x, y,
 		},
@@ -14529,60 +14152,64 @@ func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
-type SparseTensorDenseMatMulAttr func(optionalAttr)
+// Conv3DAttr is an optional argument to Conv3D.
+type Conv3DAttr func(optionalAttr)
 
-// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
+// Conv3DDataFormat sets the optional data_format attribute to value.
 //
-// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
-// is transpose(conj(A)).  Otherwise it's transpose(A).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func Conv3DDataFormat(value string) Conv3DAttr {
 	return func(m optionalAttr) {
-		m["adjoint_a"] = value
+		m["data_format"] = value
 	}
 }
 
-// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
+// Conv3DDilations sets the optional dilations attribute to value.
 //
-// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
-// is transpose(conj(B)).  Otherwise it's transpose(B).
-// If not specified, defaults to false
-func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
+// value: 1-D tensor of length 5.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 i:1 >
+func Conv3DDilations(value []int64) Conv3DAttr {
 	return func(m optionalAttr) {
-		m["adjoint_b"] = value
+		m["dilations"] = value
 	}
 }
 
-// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
+// Computes a 3-D convolution given 5-D `input` and `filter` tensors.
 //
-// No validity checking is performed on the indices of A.  However, the following
-// input format is recommended for optimal behavior:
+// In signal processing, cross-correlation is a measure of similarity of
+// two waveforms as a function of a time-lag applied to one of them. This
+// is also known as a sliding dot product or sliding inner-product.
 //
-// if adjoint_a == false:
-//   A should be sorted in lexicographically increasing order.  Use SparseReorder
-//   if you're not sure.
-// if adjoint_a == true:
-//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
-//   order instead of "row major" order).
+// Our Conv3D implements a form of cross-correlation.
 //
 // Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
-//	b: 2-D.  A dense Matrix.
-func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
+//	input: Shape `[batch, in_depth, in_height, in_width, in_channels]`.
+//	filter: Shape `[filter_depth, filter_height, filter_width, in_channels,
+// out_channels]`. `in_channels` must match between `input` and `filter`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
+func Conv3D(scope *Scope, input tf.Output, filter tf.Output, strides []int64, padding string, optional ...Conv3DAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseMatMul",
+		Type: "Conv3D",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
+			input, filter,
 		},
 		Attrs: attrs,
 	}
@@ -14590,235 +14217,317 @@ func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Outp
 	return op.Output(0)
 }
 
-// Deserialize and concatenate `SparseTensors` from a serialized minibatch.
+// Adds up a SparseTensor and a dense Tensor, using these special rules:
 //
-// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
-// `N` is the minibatch size and the rows correspond to packed outputs of
-// `SerializeSparse`.  The ranks of the original `SparseTensor` objects
-// must all match.  When the final `SparseTensor` is created, it has rank one
-// higher than the ranks of the incoming `SparseTensor` objects
-// (they have been concatenated along a new row dimension).
+// (1) Broadcasts the dense side to have the same shape as the sparse side, if
+//     eligible;
+// (2) Then, only the dense values pointed to by the indices of the SparseTensor
+//     participate in the cwise addition.
 //
-// The output `SparseTensor` object's shape values for all dimensions but the
-// first are the max across the input `SparseTensor` objects' shape values
-// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
-// size.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in
-// standard lexicographic order.  If this is not the case, after this
-// step run `SparseReorder` to restore index ordering.
-//
-// For example, if the serialized input is a `[2 x 3]` matrix representing two
-// original `SparseTensor` objects:
-//
-//     index = [ 0]
-//             [10]
-//             [20]
-//     values = [1, 2, 3]
-//     shape = [50]
+// By these rules, the result is a logical SparseTensor with exactly the same
+// indices and shape, but possibly with different non-zero values.  The output of
+// this Op is the resultant non-zero values.
 //
-// and
+// Arguments:
+//	sp_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, possibly not in canonical ordering.
+//	sp_values: 1-D.  `N` non-empty values corresponding to `sp_indices`.
+//	sp_shape: 1-D.  Shape of the input SparseTensor.
+//	dense: `R`-D.  The dense Tensor operand.
 //
-//     index = [ 2]
-//             [10]
-//     values = [4, 5]
-//     shape = [30]
+// Returns 1-D.  The `N` values that are operated on.
+func SparseDenseCwiseAdd(scope *Scope, sp_indices tf.Output, sp_values tf.Output, sp_shape tf.Output, dense tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SparseDenseCwiseAdd",
+		Input: []tf.Input{
+			sp_indices, sp_values, sp_shape, dense,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Read an element from the TensorArray into output `value`.
 //
-// then the final deserialized `SparseTensor` will be:
+// Arguments:
+//	handle: The handle to a TensorArray.
 //
-//     index = [0  0]
-//             [0 10]
-//             [0 20]
-//             [1  2]
-//             [1 10]
-//     values = [1, 2, 3, 4, 5]
-//     shape = [2 50]
+//	flow_in: A float scalar that enforces proper chaining of operations.
+//	dtype: The type of the elem that is returned.
 //
-// Arguments:
-//	serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects.
-// Must have 3 columns.
-//	dtype: The `dtype` of the serialized `SparseTensor` objects.
-func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+// Returns The tensor that is read from the TensorArray.
+func TensorArrayReadV3(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "DeserializeManySparse",
+		Type: "TensorArrayReadV3",
 		Input: []tf.Input{
-			serialized_sparse,
+			handle, index, flow_in,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// StringJoinAttr is an optional argument to StringJoin.
-type StringJoinAttr func(optionalAttr)
+// QuantizeV2Attr is an optional argument to QuantizeV2.
+type QuantizeV2Attr func(optionalAttr)
 
-// StringJoinSeparator sets the optional separator attribute to value.
-//
-// value: string, an optional join separator.
-// If not specified, defaults to ""
-func StringJoinSeparator(value string) StringJoinAttr {
+// QuantizeV2Mode sets the optional mode attribute to value.
+// If not specified, defaults to "MIN_COMBINED"
+func QuantizeV2Mode(value string) QuantizeV2Attr {
 	return func(m optionalAttr) {
-		m["separator"] = value
+		m["mode"] = value
 	}
 }
 
-// Joins the strings in the given list of string tensors into one tensor;
+// QuantizeV2RoundMode sets the optional round_mode attribute to value.
+// If not specified, defaults to "HALF_AWAY_FROM_ZERO"
+func QuantizeV2RoundMode(value string) QuantizeV2Attr {
+	return func(m optionalAttr) {
+		m["round_mode"] = value
+	}
+}
+
+// Quantize the 'input' tensor of type float to 'output' tensor of type 'T'.
 //
-// with the given separator (default is an empty separator).
+// [min_range, max_range] are scalar floats that specify the range for
+// the 'input' data. The 'mode' attribute controls exactly which calculations are
+// used to convert the float values to their quantized equivalents.  The
+// 'round_mode' attribute controls which rounding tie-breaking algorithm is used
+// when rounding float values to their quantized equivalents.
+//
+// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+//
+// ```
+// out[i] = (in[i] - min_range) * range(T) / (max_range - min_range)
+// if T == qint8, out[i] -= (range(T) + 1) / 2.0
+// ```
+// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+//
+// *MIN_COMBINED Mode Example*
+//
+// Assume the input is type float and has a possible range of [0.0, 6.0] and the
+// output type is quint8 ([0, 255]). The min_range and max_range values should be
+// specified as 0.0 and 6.0. Quantizing from float to quint8 will multiply each
+// value of the input by 255/6 and cast to quint8.
+//
+// If the output type was qint8 ([-128, 127]), the operation will additionally
+// subtract each value by 128 prior to casting, so that the range of values aligns
+// with the range of qint8.
+//
+// If the mode is 'MIN_FIRST', then this approach is used:
+//
+// ```
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
+// range = (range_max - range_min) * range_adjust
+// range_scale = num_discrete_values / range
+// quantized = round(input * range_scale) - round(range_min * range_scale) +
+//   numeric_limits<T>::min()
+// quantized = max(quantized, numeric_limits<T>::min())
+// quantized = min(quantized, numeric_limits<T>::max())
+// ```
+//
+// The biggest difference between this and MIN_COMBINED is that the minimum range
+// is rounded first, before it's subtracted from the rounded value. With
+// MIN_COMBINED, a small bias is introduced where repeated iterations of quantizing
+// and dequantizing will introduce a larger and larger error.
+//
+// *SCALED mode Example*
+//
+// `SCALED` mode matches the quantization approach used in
+// `QuantizeAndDequantize{V2|V3}`.
+//
+// If the mode is `SCALED`, we do not use the full range of the output type,
+// choosing to elide the lowest possible value for symmetry (e.g., output range is
+// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+// 0.
+//
+// We first find the range of values in our tensor. The
+// range we use is always centered on 0, so we find m such that
+// ```c++
+//   m = max(abs(input_min), abs(input_max))
+// ```
+//
+// Our input tensor range is then `[-m, m]`.
+//
+// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+// If T is signed, this is
+// ```
+//   num_bits = sizeof(T) * 8
+//   [min_fixed, max_fixed] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+// ```
+//
+// Otherwise, if T is unsigned, the fixed-point range is
+// ```
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+// ```
+//
+// From this we compute our scaling factor, s:
+// ```c++
+//   s = (max_fixed - min_fixed) / (2 * m)
+// ```
+//
+// Now we can quantize the elements of our tensor:
+// ```c++
+// result = round(input * s)
+// ```
+//
+// One thing to watch out for is that the operator may choose to adjust the
+// requested minimum and maximum values slightly during the quantization process,
+// so you should always use the output ports as the range for further calculations.
+// For example, if the requested minimum and maximum values are close to equal,
+// they will be separated by a small epsilon value to prevent ill-formed quantized
+// buffers from being created. Otherwise, you can end up with buffers where all the
+// quantized values map to the same float value, which causes problems for
+// operations that have to perform further calculations on them.
 //
 // Arguments:
-//	inputs: A list of string tensors.  The tensors must all have the same shape,
-// or be scalars.  Scalars may be mixed in; these will be broadcast to the shape
-// of non-scalar inputs.
-func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) {
+//
+//	min_range: The minimum scalar value possibly produced for the input.
+//	max_range: The maximum scalar value possibly produced for the input.
+//
+//
+// Returns The quantized data produced from the float input.The actual minimum scalar value used for the output.The actual maximum scalar value used for the output.
+func QuantizeV2(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, T tf.DataType, optional ...QuantizeV2Attr) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"T": T}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringJoin",
+		Type: "QuantizeV2",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			input, min_range, max_range,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns immutable tensor from memory region.
-//
-// The current implementation memmaps the tensor from a file.
+// Returns the truth value of (x < y) element-wise.
 //
-// Arguments:
-//	dtype: Type of the returned tensor.
-//	shape: Shape of the returned tensor.
-//	memory_region_name: Name of readonly memory region used by the tensor, see
-// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
-func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
+// *NOTE*: `Less` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Less(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
 	opspec := tf.OpSpec{
-		Type: "ImmutableConst",
-
-		Attrs: attrs,
+		Type: "Less",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Inverse real-valued fast Fourier transform.
+// QuantizedReluXAttr is an optional argument to QuantizedReluX.
+type QuantizedReluXAttr func(optionalAttr)
+
+// QuantizedReluXOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QUINT8
+func QuantizedReluXOutType(value tf.DataType) QuantizedReluXAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Computes Quantized Rectified Linear X: `min(max(features, 0), max_value)`
 //
-// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
-// signal over the inner-most dimension of `input`.
+// Arguments:
 //
-// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
-// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
-// `fft_length` is not provided, it is computed from the size of the inner-most
-// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
-// compute `input` is odd, it should be provided since it cannot be inferred
-// properly.
 //
-// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
-// than the corresponding dimension of `input`, the dimension is cropped. If it is
-// larger, the dimension is padded with zeros.
+//	min_features: The float value that the lowest quantized value represents.
+//	max_features: The float value that the highest quantized value represents.
 //
-// Arguments:
-//	input: A complex64 tensor.
-//	fft_length: An int32 tensor of shape [1]. The FFT length.
-//
-// Returns A float32 tensor of the same rank as `input`. The inner-most
-//   dimension of `input` is replaced with the `fft_length` samples of its inverse
-//   1D Fourier transform.
-//
-// @compatibility(numpy)
-// Equivalent to np.fft.irfft
-// @end_compatibility
-func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
+// Returns Has the same output shape as "features".The float value that the lowest quantized value represents.The float value that the highest quantized value represents.
+func QuantizedReluX(scope *Scope, features tf.Output, max_value tf.Output, min_features tf.Output, max_features tf.Output, optional ...QuantizedReluXAttr) (activations tf.Output, min_activations tf.Output, max_activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "IRFFT",
+		Type: "QuantizedReluX",
 		Input: []tf.Input{
-			input, fft_length,
+			features, max_value, min_features, max_features,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Concatenates a list of `SparseTensor` along the specified dimension.
-//
-// Concatenation is with respect to the dense versions of these sparse tensors.
-// It is assumed that each input is a `SparseTensor` whose elements are ordered
-// along increasing dimension number.
-//
-// All inputs' shapes must match, except for the concat dimension.  The
-// `indices`, `values`, and `shapes` lists must have the same length.
-//
-// The output shape is identical to the inputs', except along the concat
-// dimension, where it is the sum of the inputs' sizes along that dimension.
-//
-// The output elements will be resorted to preserve the sort order along
-// increasing dimension number.
-//
-// This op runs in `O(M log M)` time, where `M` is the total number of non-empty
-// values across all inputs. This is due to the need for an internal sort in
-// order to concatenate efficiently across an arbitrary dimension.
-//
-// For example, if `concat_dim = 1` and the inputs are
-//
-//     sp_inputs[0]: shape = [2, 3]
-//     [0, 2]: "a"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
-//
-//     sp_inputs[1]: shape = [2, 4]
-//     [0, 1]: "d"
-//     [0, 2]: "e"
-//
-// then the output will be
-//
-//     shape = [2, 7]
-//     [0, 2]: "a"
-//     [0, 4]: "d"
-//     [0, 5]: "e"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
+// QuantizedConv2DAttr is an optional argument to QuantizedConv2D.
+type QuantizedConv2DAttr func(optionalAttr)
+
+// QuantizedConv2DOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedConv2DOutType(value tf.DataType) QuantizedConv2DAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// QuantizedConv2DDilations sets the optional dilations attribute to value.
 //
-// Graphically this is equivalent to doing
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each
+// filter element on that dimension. The dimension order is determined by the
+// value of `data_format`, see above for details. Dilations in the batch and
+// depth dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func QuantizedConv2DDilations(value []int64) QuantizedConv2DAttr {
+	return func(m optionalAttr) {
+		m["dilations"] = value
+	}
+}
+
+// Computes a 2D convolution given quantized 4D input and filter tensors.
 //
-//     [    a] concat [  d e  ] = [    a   d e  ]
-//     [b c  ]        [       ]   [b c          ]
+// The inputs are quantized tensors where the lowest value represents the real
+// number of the associated minimum, and the highest represents the maximum.
+// This means that you can only interpret the quantized output in the same way, by
+// taking the returned minimum and maximum values into account.
 //
 // Arguments:
-//	indices: 2-D.  Indices of each input `SparseTensor`.
-//	values: 1-D.  Non-empty values of each `SparseTensor`.
-//	shapes: 1-D.  Shapes of each `SparseTensor`.
-//	concat_dim: Dimension to concatenate along. Must be in range [-rank, rank),
-// where rank is the number of dimensions in each input `SparseTensor`.
 //
-// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
-func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+//	filter: filter's input_depth dimension must match input's depth dimensions.
+//	min_input: The float value that the lowest quantized input value represents.
+//	max_input: The float value that the highest quantized input value represents.
+//	min_filter: The float value that the lowest quantized filter value represents.
+//	max_filter: The float value that the highest quantized filter value represents.
+//	strides: The stride of the sliding window for each dimension of the input
+// tensor.
+//	padding: The type of padding algorithm to use.
+//
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedConv2D(scope *Scope, input tf.Output, filter tf.Output, min_input tf.Output, max_input tf.Output, min_filter tf.Output, max_filter tf.Output, strides []int64, padding string, optional ...QuantizedConv2DAttr) (output tf.Output, min_output tf.Output, max_output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"concat_dim": concat_dim}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseConcat",
+		Type: "QuantizedConv2D",
 		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes),
+			input, filter, min_input, max_input, min_filter, max_filter,
 		},
 		Attrs: attrs,
 	}
@@ -14826,253 +14535,282 @@ func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Generates sparse cross from a list of sparse and dense tensors.
-//
-// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
-// representing features of one feature column. It outputs a 2D `SparseTensor` with
-// the batchwise crosses of these features.
-//
-// For example, if the inputs are
-//
-//     inputs[0]: SparseTensor with shape = [2, 2]
-//     [0, 0]: "a"
-//     [1, 0]: "b"
-//     [1, 1]: "c"
-//
-//     inputs[1]: SparseTensor with shape = [2, 1]
-//     [0, 0]: "d"
-//     [1, 0]: "e"
-//
-//     inputs[2]: Tensor [["f"], ["g"]]
-//
-// then the output will be
-//
-//     shape = [2, 2]
-//     [0, 0]: "a_X_d_X_f"
-//     [1, 0]: "b_X_e_X_g"
-//     [1, 1]: "c_X_e_X_g"
-//
-// if hashed_output=true then the output will be
-//
-//     shape = [2, 2]
-//     [0, 0]: FingerprintCat64(
-//                 Fingerprint64("f"), FingerprintCat64(
-//                     Fingerprint64("d"), Fingerprint64("a")))
-//     [1, 0]: FingerprintCat64(
-//                 Fingerprint64("g"), FingerprintCat64(
-//                     Fingerprint64("e"), Fingerprint64("b")))
-//     [1, 1]: FingerprintCat64(
-//                 Fingerprint64("g"), FingerprintCat64(
-//                     Fingerprint64("e"), Fingerprint64("c")))
+// ResourceGatherAttr is an optional argument to ResourceGather.
+type ResourceGatherAttr func(optionalAttr)
+
+// ResourceGatherValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func ResourceGatherValidateIndices(value bool) ResourceGatherAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
+	}
+}
+
+// Gather slices from the variable pointed to by `resource` according to `indices`.
 //
-// Arguments:
-//	indices: 2-D.  Indices of each input `SparseTensor`.
-//	values: 1-D.   values of each `SparseTensor`.
-//	shapes: 1-D.   Shapes of each `SparseTensor`.
-//	dense_inputs: 2-D.    Columns represented by dense `Tensor`.
-//	hashed_output: If true, returns the hash of the cross instead of the string.
-// This will allow us avoiding string manipulations.
-//	num_buckets: It is used if hashed_output is true.
-// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value.
-//	hash_key: Specify the hash_key that will be used by the `FingerprintCat64`
-// function to combine the crosses fingerprints.
+// `indices` must be an integer tensor of any dimension (usually 0-D or 1-D).
+// Produces an output tensor with shape `indices.shape + params.shape[1:]` where:
 //
+// ```python
+//     # Scalar indices
+//     output[:, ..., :] = params[indices, :, ... :]
 //
+//     # Vector indices
+//     output[i, :, ..., :] = params[indices[i], :, ... :]
 //
-// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated or hashed
-// `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
-func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+//     # Higher rank indices
+//     output[i, ..., j, :, ... :] = params[indices[i, ..., j], :, ..., :]
+// ```
+func ResourceGather(scope *Scope, resource tf.Output, indices tf.Output, dtype tf.DataType, optional ...ResourceGatherAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseCross",
+		Type: "ResourceGather",
 		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs),
+			resource, indices,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Concatenates quantized tensors along one dimension.
+// Delete the TensorArray from its resource container.
+//
+// This enables the user to close and release the resource in the middle
+// of a step/run.
 //
 // Arguments:
-//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
-// range [0, rank(values)).
-//	values: The `N` Tensors to concatenate. Their ranks and types must match,
-// and their sizes must match in all dimensions except `concat_dim`.
-//	input_mins: The minimum scalar values for each of the input tensors.
-//	input_maxes: The maximum scalar values for each of the input tensors.
+//	handle: The handle to a TensorArray (output of TensorArray or TensorArrayGrad).
 //
-// Returns A `Tensor` with the concatenation of values stacked along the
-// `concat_dim` dimension.  This tensor's shape matches that of `values` except
-// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
-func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
+// Returns the created operation.
+func TensorArrayCloseV3(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedConcat",
+		Type: "TensorArrayCloseV3",
 		Input: []tf.Input{
-			concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes),
+			handle,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
-// Slice a `SparseTensor` based on the `start` and `size`.
-//
-// For example, if the input is
+// Adds two `SparseTensor` objects to produce another `SparseTensor`.
 //
-//     input_tensor = shape = [2, 7]
-//     [    a   d e  ]
-//     [b c          ]
+// The input `SparseTensor` objects' indices are assumed ordered in standard
+// lexicographic order.  If this is not the case, before this step run
+// `SparseReorder` to restore index ordering.
 //
-// Graphically the output tensors are:
+// By default, if two values sum to zero at some index, the output `SparseTensor`
+// would still include that particular location in its index, storing a zero in the
+// corresponding value slot.  To override this, callers can specify `thresh`,
+// indicating that if the sum has a magnitude strictly smaller than `thresh`, its
+// corresponding value and index would then not be included.  In particular,
+// `thresh == 0` (default) means everything is kept and actual thresholding happens
+// only for a positive value.
 //
-//     sparse_slice([0, 0], [2, 4]) = shape = [2, 4]
-//     [    a  ]
-//     [b c    ]
-//
-//     sparse_slice([0, 4], [2, 3]) = shape = [2, 3]
-//     [ d e  ]
-//     [      ]
+// In the following shapes, `nnz` is the count after taking `thresh` into account.
 //
 // Arguments:
-//	indices: 2-D tensor represents the indices of the sparse tensor.
-//	values: 1-D tensor represents the values of the sparse tensor.
-//	shape: 1-D. tensor represents the shape of the sparse tensor.
-//	start: 1-D. tensor represents the start of the slice.
-//	size: 1-D. tensor represents the size of the slice.
-// output indices: A list of 1-D tensors represents the indices of the output
-// sparse tensors.
-//
-// Returns A list of 1-D tensors represents the values of the output sparse
-// tensors.A list of 1-D tensors represents the shape of the output sparse
-// tensors.
-func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
+//	a_indices: 2-D.  The `indices` of the first `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	a_values: 1-D.  The `values` of the first `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the first `SparseTensor`, size `[ndims]` Vector.
+//	b_indices: 2-D.  The `indices` of the second `SparseTensor`, size `[nnz, ndims]` Matrix.
+//	b_values: 1-D.  The `values` of the second `SparseTensor`, size `[nnz]` Vector.
+//	b_shape: 1-D.  The `shape` of the second `SparseTensor`, size `[ndims]` Vector.
+//	thresh: 0-D.  The magnitude threshold that determines if an output value/index
+// pair takes space.
+func SparseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output, thresh tf.Output) (sum_indices tf.Output, sum_values tf.Output, sum_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSlice",
+		Type: "SparseAdd",
 		Input: []tf.Input{
-			indices, values, shape, start, size,
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape, thresh,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
+// OrderedMapPeekAttr is an optional argument to OrderedMapPeek.
+type OrderedMapPeekAttr func(optionalAttr)
+
+// OrderedMapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// This Op does not require `a_indices` be sorted in standard lexicographic order.
+// REQUIRES: value >= 0
+func OrderedMapPeekCapacity(value int64) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// OrderedMapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
-//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
-//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
-//	b: `ndims`-D Tensor.  With shape `a_shape`.
-func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
+// REQUIRES: value >= 0
+func OrderedMapPeekMemoryLimit(value int64) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// OrderedMapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func OrderedMapPeekContainer(value string) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// OrderedMapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func OrderedMapPeekSharedName(value string) OrderedMapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
+//
+// underlying container does not contain this key
+// this op will block until it does.   This Op is optimized for
+// performance.
+func OrderedMapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...OrderedMapPeekAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SparseTensorDenseAdd",
+		Type: "OrderedMapPeek",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b,
+			key, indices,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("OrderedMapPeek", err)
+		return
+	}
+	return values
 }
 
-// Returns the set of files matching one or more glob patterns.
+// DecodeAndCropJpegAttr is an optional argument to DecodeAndCropJpeg.
+type DecodeAndCropJpegAttr func(optionalAttr)
+
+// DecodeAndCropJpegChannels sets the optional channels attribute to value.
 //
-// Note that this routine only supports wildcard characters in the
-// basename portion of the pattern, not in the directory portion.
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodeAndCropJpegChannels(value int64) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["channels"] = value
+	}
+}
+
+// DecodeAndCropJpegRatio sets the optional ratio attribute to value.
 //
-// Arguments:
-//	pattern: Shell wildcard pattern(s). Scalar or vector of type string.
+// value: Downscaling ratio.
+// If not specified, defaults to 1
+func DecodeAndCropJpegRatio(value int64) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["ratio"] = value
+	}
+}
+
+// DecodeAndCropJpegFancyUpscaling sets the optional fancy_upscaling attribute to value.
 //
-// Returns A vector of matching filenames.
-func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: If true use a slower but nicer upscaling of the
+// chroma planes (yuv420/422 only).
+// If not specified, defaults to true
+func DecodeAndCropJpegFancyUpscaling(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["fancy_upscaling"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "MatchingFiles",
-		Input: []tf.Input{
-			pattern,
-		},
+}
+
+// DecodeAndCropJpegTryRecoverTruncated sets the optional try_recover_truncated attribute to value.
+//
+// value: If true try to recover an image from truncated input.
+// If not specified, defaults to false
+func DecodeAndCropJpegTryRecoverTruncated(value bool) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["try_recover_truncated"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
-type MatrixSolveLsAttr func(optionalAttr)
+// DecodeAndCropJpegAcceptableFraction sets the optional acceptable_fraction attribute to value.
+//
+// value: The minimum required fraction of lines before a truncated
+// input is accepted.
+// If not specified, defaults to 1
+func DecodeAndCropJpegAcceptableFraction(value float32) DecodeAndCropJpegAttr {
+	return func(m optionalAttr) {
+		m["acceptable_fraction"] = value
+	}
+}
 
-// MatrixSolveLsFast sets the optional fast attribute to value.
-// If not specified, defaults to true
-func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
+// DecodeAndCropJpegDctMethod sets the optional dct_method attribute to value.
+//
+// value: string specifying a hint about the algorithm used for
+// decompression.  Defaults to "" which maps to a system-specific
+// default.  Currently valid values are ["INTEGER_FAST",
+// "INTEGER_ACCURATE"].  The hint may be ignored (e.g., the internal
+// jpeg library changes to a version that does not have that specific
+// option.)
+// If not specified, defaults to ""
+func DecodeAndCropJpegDctMethod(value string) DecodeAndCropJpegAttr {
 	return func(m optionalAttr) {
-		m["fast"] = value
+		m["dct_method"] = value
 	}
 }
 
-// Solves one or more linear least-squares problems.
+// Decode and Crop a JPEG-encoded image to a uint8 tensor.
 //
-// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
-// type as `matrix` and shape `[..., M, K]`.
-// The output is a tensor shape `[..., N, K]` where each output matrix solves
-// each of the equations
-// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
-// in the least squares sense.
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
 //
-// We use the following notation for (complex) matrix and right-hand sides
-// in the batch:
+// Accepted values are:
 //
-// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
-// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
-// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
-// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
+// *   0: Use the number of channels in the JPEG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
 //
-// If `fast` is `True`, then the solution is computed by solving the normal
-// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
-// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
-// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
-// minimum-norm solution to the under-determined linear system, i.e.
-// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
-// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
-// when \\(A\\) is numerically full rank and has a condition number
-// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
-// sufficiently large.
+// If needed, the JPEG-encoded image is transformed to match the requested number
+// of color channels.
 //
-// If `fast` is `False` an algorithm based on the numerically robust complete
-// orthogonal decomposition is used. This computes the minimum-norm
-// least-squares solution, even when \\(A\\) is rank deficient. This path is
-// typically 6-7 times slower than the fast path. If `fast` is `False` then
-// `l2_regularizer` is ignored.
+// The attr `ratio` allows downscaling the image by an integer factor during
+// decoding.  Allowed values are: 1, 2, 4, and 8.  This is much faster than
+// downscaling the image later.
 //
-// Arguments:
-//	matrix: Shape is `[..., M, N]`.
-//	rhs: Shape is `[..., M, K]`.
-//	l2_regularizer: Scalar tensor.
 //
-// @compatibility(numpy)
-// Equivalent to np.linalg.lstsq
-// @end_compatibility
+// It is equivalent to a combination of decode and crop, but much faster by only
+// decoding partial jpeg image.
 //
-// Returns Shape is `[..., N, K]`.
-func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
+// Arguments:
+//	contents: 0-D.  The JPEG-encoded image.
+//	crop_window: 1-D.  The crop window: [crop_y, crop_x, crop_height, crop_width].
+//
+// Returns 3-D with shape `[height, width, channels]`..
+func DecodeAndCropJpeg(scope *Scope, contents tf.Output, crop_window tf.Output, optional ...DecodeAndCropJpegAttr) (image tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15081,9 +14819,9 @@ func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixSolveLs",
+		Type: "DecodeAndCropJpeg",
 		Input: []tf.Input{
-			matrix, rhs, l2_regularizer,
+			contents, crop_window,
 		},
 		Attrs: attrs,
 	}
@@ -15091,93 +14829,71 @@ func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer
 	return op.Output(0)
 }
 
-// Elementwise computes the bitwise OR of `x` and `y`.
+// AllCandidateSamplerAttr is an optional argument to AllCandidateSampler.
+type AllCandidateSamplerAttr func(optionalAttr)
+
+// AllCandidateSamplerSeed sets the optional seed attribute to value.
 //
-// The result will have those bits set, that are set in `x`, `y` or both. The
-// computation is performed on the underlying representations of `x` and `y`.
-func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BitwiseOr",
-		Input: []tf.Input{
-			x, y,
-		},
+// value: If either seed or seed2 are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func AllCandidateSamplerSeed(value int64) AllCandidateSamplerAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
-type SparseToSparseSetOperationAttr func(optionalAttr)
-
-// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
+// AllCandidateSamplerSeed2 sets the optional seed2 attribute to value.
+//
+// value: An second seed to avoid seed collision.
+// If not specified, defaults to 0
+func AllCandidateSamplerSeed2(value int64) AllCandidateSamplerAttr {
 	return func(m optionalAttr) {
-		m["validate_indices"] = value
+		m["seed2"] = value
 	}
 }
 
-// Applies set operation along last dimension of 2 `SparseTensor` inputs.
-//
-// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
-//
-// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
-// order and range of `set1` and `set2` indices.
-//
-// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
-// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
+// Generates labels for candidate sampling with a learned unigram distribution.
 //
-// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
-// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
-// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
-// ignored.
+// See explanations of candidate sampling and the data formats at
+// go/candidate-sampling.
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set1`
-// and `set2` indices.
+// For each batch, this op picks a single set of sampled candidate labels.
 //
-// Output `result` is a `SparseTensor` represented by `result_indices`,
-// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
-// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
-// dimension contains the result of `set_operation` applied to the corresponding
-// `[0...n-1]` dimension of `set`.
+// The advantages of sampling candidates per-batch are simplicity and the
+// possibility of efficient dense matrix multiplication. The disadvantage is that
+// the sampled candidates must be chosen independently of the context and of the
+// true labels.
 //
 // Arguments:
-//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
-// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
-// max set size across `0...n-1` dimensions.
-//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
-// order.
-//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
-// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
-// max set size across `0...n-1` dimensions.
-//
+//	true_classes: A batch_size * num_true matrix, in which each row contains the
+// IDs of the num_true target_classes in the corresponding original label.
+//	num_true: Number of true labels per context.
+//	num_sampled: Number of candidates to produce.
+//	unique: If unique is true, we sample with rejection, so that all sampled
+// candidates in a batch are unique. This requires some approximation to
+// estimate the post-rejection sampling probabilities.
 //
-// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
-// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
-// is the max result set size across all `0...n-1` dimensions.
-func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+// Returns A vector of length num_sampled, in which each element is
+// the ID of a sampled candidate.A batch_size * num_true matrix, representing
+// the number of times each candidate is expected to occur in a batch
+// of sampled candidates. If unique=true, then this is a probability.A vector of length num_sampled, for each sampled
+// candidate representing the number of times the candidate is expected
+// to occur in a batch of sampled candidates.  If unique=true, then this is a
+// probability.
+func AllCandidateSampler(scope *Scope, true_classes tf.Output, num_true int64, num_sampled int64, unique bool, optional ...AllCandidateSamplerAttr) (sampled_candidates tf.Output, true_expected_count tf.Output, sampled_expected_count tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"set_operation": set_operation}
+	attrs := map[string]interface{}{"num_true": num_true, "num_sampled": num_sampled, "unique": unique}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseToSparseSetOperation",
+		Type: "AllCandidateSampler",
 		Input: []tf.Input{
-			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
+			true_classes,
 		},
 		Attrs: attrs,
 	}
@@ -15185,55 +14901,97 @@ func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_value
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes numerical negative value element-wise.
+// Saves the input tensors to disk.
 //
-// I.e., \\(y = -x\\).
-func Neg(scope *Scope, x tf.Output) (y tf.Output) {
+// The size of `tensor_names` must match the number of tensors in `data`. `data[i]`
+// is written to `filename` with name `tensor_names[i]`.
+//
+// See also `SaveSlices`.
+//
+// Arguments:
+//	filename: Must have a single element. The name of the file to which we write
+// the tensor.
+//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
+//	data: `N` tensors to save.
+//
+// Returns the created operation.
+func Save(scope *Scope, filename tf.Output, tensor_names tf.Output, data []tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Neg",
+		Type: "Save",
 		Input: []tf.Input{
-			x,
+			filename, tensor_names, tf.OutputList(data),
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns element-wise remainder of division. When `x < 0` xor `y < 0` is
+//
+// true, this follows Python semantics in that the result here is consistent
+// with a flooring divide. E.g. `floor(x / y) * y + mod(x, y) = x`.
+//
+// *NOTE*: `FloorMod` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func FloorMod(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "FloorMod",
+		Input: []tf.Input{
+			x, y,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
-type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
+// SparseTensorDenseMatMulAttr is an optional argument to SparseTensorDenseMatMul.
+type SparseTensorDenseMatMulAttr func(optionalAttr)
 
-// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
+// SparseTensorDenseMatMulAdjointA sets the optional adjoint_a attribute to value.
+//
+// value: Use the adjoint of A in the matrix multiply.  If A is complex, this
+// is transpose(conj(A)).  Otherwise it's transpose(A).
+// If not specified, defaults to false
+func SparseTensorDenseMatMulAdjointA(value bool) SparseTensorDenseMatMulAttr {
 	return func(m optionalAttr) {
-		m["num_bits"] = value
+		m["adjoint_a"] = value
 	}
 }
 
-// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
+// SparseTensorDenseMatMulAdjointB sets the optional adjoint_b attribute to value.
+//
+// value: Use the adjoint of B in the matrix multiply.  If B is complex, this
+// is transpose(conj(B)).  Otherwise it's transpose(B).
 // If not specified, defaults to false
-func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
+func SparseTensorDenseMatMulAdjointB(value bool) SparseTensorDenseMatMulAttr {
 	return func(m optionalAttr) {
-		m["narrow_range"] = value
+		m["adjoint_b"] = value
 	}
 }
 
-// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
+// Multiply SparseTensor (of rank 2) "A" by dense matrix "B".
 //
-// and `max` to 'outputs' tensor of same shape as `inputs`.
+// No validity checking is performed on the indices of A.  However, the following
+// input format is recommended for optimal behavior:
 //
-// `[min; max]` define the clamping range for the `inputs` data.
-// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
-// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
-// then de-quantized and output as floats in `[min; max]` interval.
-// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
+// if adjoint_a == false:
+//   A should be sorted in lexicographically increasing order.  Use SparseReorder
+//   if you're not sure.
+// if adjoint_a == true:
+//   A should be sorted in order of increasing dimension 1 (i.e., "column major"
+//   order instead of "row major" order).
 //
-// This operation has a gradient and thus allows for training `min` and `max`
-// values.
-func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
+// Arguments:
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, size `[nnz, 2]` Matrix.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, size `[nnz]` Vector.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, size `[2]` Vector.
+//	b: 2-D.  A dense Matrix.
+func SparseTensorDenseMatMul(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output, optional ...SparseTensorDenseMatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15242,9 +15000,9 @@ func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "FakeQuantWithMinMaxVars",
+		Type: "SparseTensorDenseMatMul",
 		Input: []tf.Input{
-			inputs, min, max,
+			a_indices, a_values, a_shape, b,
 		},
 		Attrs: attrs,
 	}
@@ -15252,524 +15010,555 @@ func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max
 	return op.Output(0)
 }
 
-// Returns the element-wise min of two SparseTensors.
+// Deserialize and concatenate `SparseTensors` from a serialized minibatch.
 //
-// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
+// The input `serialized_sparse` must be a string matrix of shape `[N x 3]` where
+// `N` is the minibatch size and the rows correspond to packed outputs of
+// `SerializeSparse`.  The ranks of the original `SparseTensor` objects
+// must all match.  When the final `SparseTensor` is created, it has rank one
+// higher than the ranks of the incoming `SparseTensor` objects
+// (they have been concatenated along a new row dimension).
 //
-// Arguments:
-//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, in the canonical lexicographic ordering.
-//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
-//	a_shape: 1-D.  Shape of the input SparseTensor.
-//	b_indices: counterpart to `a_indices` for the other operand.
-//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
-//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
+// The output `SparseTensor` object's shape values for all dimensions but the
+// first are the max across the input `SparseTensor` objects' shape values
+// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
+// size.
 //
-// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
-func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
+// The input `SparseTensor` objects' indices are assumed ordered in
+// standard lexicographic order.  If this is not the case, after this
+// step run `SparseReorder` to restore index ordering.
+//
+// For example, if the serialized input is a `[2 x 3]` matrix representing two
+// original `SparseTensor` objects:
+//
+//     index = [ 0]
+//             [10]
+//             [20]
+//     values = [1, 2, 3]
+//     shape = [50]
+//
+// and
+//
+//     index = [ 2]
+//             [10]
+//     values = [4, 5]
+//     shape = [30]
+//
+// then the final deserialized `SparseTensor` will be:
+//
+//     index = [0  0]
+//             [0 10]
+//             [0 20]
+//             [1  2]
+//             [1 10]
+//     values = [1, 2, 3, 4, 5]
+//     shape = [2 50]
+//
+// Arguments:
+//	serialized_sparse: 2-D, The `N` serialized `SparseTensor` objects.
+// Must have 3 columns.
+//	dtype: The `dtype` of the serialized `SparseTensor` objects.
+func DeserializeManySparse(scope *Scope, serialized_sparse tf.Output, dtype tf.DataType) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "SparseSparseMinimum",
+		Type: "DeserializeManySparse",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
+			serialized_sparse,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Constructs a tensor by tiling a given tensor.
-//
-// This operation creates a new tensor by replicating `input` `multiples` times.
-// The output tensor's i'th dimension has `input.dims(i) * multiples[i]` elements,
-// and the values of `input` are replicated `multiples[i]` times along the 'i'th
-// dimension. For example, tiling `[a b c d]` by `[2]` produces
-// `[a b c d a b c d]`.
-//
-// Arguments:
-//	input: 1-D or higher.
-//	multiples: 1-D. Length must be the same as the number of dimensions in `input`
-func Tile(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Tile",
-		Input: []tf.Input{
-			input, multiples,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// TakeManySparseFromTensorsMapAttr is an optional argument to TakeManySparseFromTensorsMap.
-type TakeManySparseFromTensorsMapAttr func(optionalAttr)
-
-// TakeManySparseFromTensorsMapContainer sets the optional container attribute to value.
-//
-// value: The container name for the `SparseTensorsMap` read by this op.
-// If not specified, defaults to ""
-func TakeManySparseFromTensorsMapContainer(value string) TakeManySparseFromTensorsMapAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// StringJoinAttr is an optional argument to StringJoin.
+type StringJoinAttr func(optionalAttr)
 
-// TakeManySparseFromTensorsMapSharedName sets the optional shared_name attribute to value.
+// StringJoinSeparator sets the optional separator attribute to value.
 //
-// value: The shared name for the `SparseTensorsMap` read by this op.
-// It should not be blank; rather the `shared_name` or unique Operation name
-// of the Op that created the original `SparseTensorsMap` should be used.
+// value: string, an optional join separator.
 // If not specified, defaults to ""
-func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTensorsMapAttr {
+func StringJoinSeparator(value string) StringJoinAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["separator"] = value
 	}
 }
 
-// Read `SparseTensors` from a `SparseTensorsMap` and concatenate them.
-//
-// The input `sparse_handles` must be an `int64` matrix of shape `[N, 1]` where
-// `N` is the minibatch size and the rows correspond to the output handles of
-// `AddSparseToTensorsMap` or `AddManySparseToTensorsMap`.  The ranks of the
-// original `SparseTensor` objects that went into the given input ops must all
-// match.  When the final `SparseTensor` is created, it has rank one
-// higher than the ranks of the incoming `SparseTensor` objects
-// (they have been concatenated along a new row dimension on the left).
-//
-// The output `SparseTensor` object's shape values for all dimensions but the
-// first are the max across the input `SparseTensor` objects' shape values
-// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
-// size.
-//
-// The input `SparseTensor` objects' indices are assumed ordered in
-// standard lexicographic order.  If this is not the case, after this
-// step run `SparseReorder` to restore index ordering.
-//
-// For example, if the handles represent an input, which is a `[2, 3]` matrix
-// representing two original `SparseTensor` objects:
-//
-// ```
-//     index = [ 0]
-//             [10]
-//             [20]
-//     values = [1, 2, 3]
-//     shape = [50]
-// ```
-//
-// and
-//
-// ```
-//     index = [ 2]
-//             [10]
-//     values = [4, 5]
-//     shape = [30]
-// ```
-//
-// then the final `SparseTensor` will be:
+// Joins the strings in the given list of string tensors into one tensor;
 //
-// ```
-//     index = [0  0]
-//             [0 10]
-//             [0 20]
-//             [1  2]
-//             [1 10]
-//     values = [1, 2, 3, 4, 5]
-//     shape = [2 50]
-// ```
+// with the given separator (default is an empty separator).
 //
 // Arguments:
-//	sparse_handles: 1-D, The `N` serialized `SparseTensor` objects.
-// Shape: `[N]`.
-//	dtype: The `dtype` of the `SparseTensor` objects stored in the
-// `SparseTensorsMap`.
-//
-// Returns 2-D.  The `indices` of the minibatch `SparseTensor`.1-D.  The `values` of the minibatch `SparseTensor`.1-D.  The `shape` of the minibatch `SparseTensor`.
-func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype tf.DataType, optional ...TakeManySparseFromTensorsMapAttr) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
+//	inputs: A list of string tensors.  The tensors must all have the same shape,
+// or be scalars.  Scalars may be mixed in; these will be broadcast to the shape
+// of non-scalar inputs.
+func StringJoin(scope *Scope, inputs []tf.Output, optional ...StringJoinAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TakeManySparseFromTensorsMap",
+		Type: "StringJoin",
 		Input: []tf.Input{
-			sparse_handles,
+			tf.OutputList(inputs),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// MaxPoolAttr is an optional argument to MaxPool.
-type MaxPoolAttr func(optionalAttr)
-
-// MaxPoolDataFormat sets the optional data_format attribute to value.
+// Returns immutable tensor from memory region.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func MaxPoolDataFormat(value string) MaxPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs max pooling on the input.
+// The current implementation memmaps the tensor from a file.
 //
 // Arguments:
-//	input: 4-D input to pool over.
-//	ksize: The size of the window for each dimension of the input tensor.
-//	strides: The stride of the sliding window for each dimension of the
-// input tensor.
-//	padding: The type of padding algorithm to use.
-//
-// Returns The max pooled output tensor.
-func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
+//	dtype: Type of the returned tensor.
+//	shape: Shape of the returned tensor.
+//	memory_region_name: Name of readonly memory region used by the tensor, see
+// NewReadOnlyMemoryRegionFromFile in tensorflow::Env.
+func ImmutableConst(scope *Scope, dtype tf.DataType, shape tf.Shape, memory_region_name string) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"dtype": dtype, "shape": shape, "memory_region_name": memory_region_name}
 	opspec := tf.OpSpec{
-		Type: "MaxPool",
-		Input: []tf.Input{
-			input,
-		},
+		Type: "ImmutableConst",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Says whether the targets are in the top `K` predictions.
-//
-// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
-// prediction for the target class is among the top `k` predictions among
-// all predictions for example `i`. Note that the behavior of `InTopK` differs
-// from the `TopK` op in its handling of ties; if multiple classes have the
-// same prediction value and straddle the top-`k` boundary, all of those
-// classes are considered to be in the top `k`.
+// Inverse real-valued fast Fourier transform.
 //
-// More formally, let
+// Computes the inverse 1-dimensional discrete Fourier transform of a real-valued
+// signal over the inner-most dimension of `input`.
 //
-//   \\(predictions_i\\) be the predictions for all classes for example `i`,
-//   \\(targets_i\\) be the target class for example `i`,
-//   \\(out_i\\) be the output for example `i`,
+// The inner-most dimension of `input` is assumed to be the result of `RFFT`: the
+// `fft_length / 2 + 1` unique components of the DFT of a real-valued signal. If
+// `fft_length` is not provided, it is computed from the size of the inner-most
+// dimension of `input` (`fft_length = 2 * (inner - 1)`). If the FFT length used to
+// compute `input` is odd, it should be provided since it cannot be inferred
+// properly.
 //
-// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+// Along the axis `IRFFT` is computed on, if `fft_length / 2 + 1` is smaller
+// than the corresponding dimension of `input`, the dimension is cropped. If it is
+// larger, the dimension is padded with zeros.
 //
 // Arguments:
-//	predictions: A `batch_size` x `classes` tensor.
-//	targets: A `batch_size` vector of class ids.
-//	k: Number of top elements to look at for computing precision.
+//	input: A complex64 tensor.
+//	fft_length: An int32 tensor of shape [1]. The FFT length.
 //
-// Returns Computed precision at `k` as a `bool Tensor`.
-func InTopKV2(scope *Scope, predictions tf.Output, targets tf.Output, k tf.Output) (precision tf.Output) {
+// Returns A float32 tensor of the same rank as `input`. The inner-most
+//   dimension of `input` is replaced with the `fft_length` samples of its inverse
+//   1D Fourier transform.
+//
+// @compatibility(numpy)
+// Equivalent to np.fft.irfft
+// @end_compatibility
+func IRFFT(scope *Scope, input tf.Output, fft_length tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "InTopKV2",
+		Type: "IRFFT",
 		Input: []tf.Input{
-			predictions, targets, k,
+			input, fft_length,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Assigns a new value to a variable.
+// Concatenates a list of `SparseTensor` along the specified dimension.
 //
-// Any ReadVariableOp with a control dependency on this op is guaranteed to return
-// this value or a subsequent newer value of the variable.
+// Concatenation is with respect to the dense versions of these sparse tensors.
+// It is assumed that each input is a `SparseTensor` whose elements are ordered
+// along increasing dimension number.
 //
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value to set the new tensor to use.
+// All inputs' shapes must match, except for the concat dimension.  The
+// `indices`, `values`, and `shapes` lists must have the same length.
 //
-// Returns the created operation.
-func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AssignVariableOp",
-		Input: []tf.Input{
-			resource, value,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Returns a tensor of ones with the same shape and type as x.
+// The output shape is identical to the inputs', except along the concat
+// dimension, where it is the sum of the inputs' sizes along that dimension.
 //
-// Arguments:
-//	x: a tensor of type T.
+// The output elements will be resorted to preserve the sort order along
+// increasing dimension number.
 //
-// Returns a tensor of the same shape and type as x but filled with ones.
-func OnesLike(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "OnesLike",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// The gradient of SparseFillEmptyRows.
+// This op runs in `O(M log M)` time, where `M` is the total number of non-empty
+// values across all inputs. This is due to the need for an internal sort in
+// order to concatenate efficiently across an arbitrary dimension.
 //
-// Takes vectors reverse_index_map, shaped `[N]`, and grad_values,
-// shaped `[N_full]`, where `N_full >= N` and copies data into either
-// `d_values` or `d_default_value`.  Here `d_values` is shaped `[N]` and
-// `d_default_value` is a scalar.
+// For example, if `concat_dim = 1` and the inputs are
 //
-//   d_values[j] = grad_values[reverse_index_map[j]]
-//   d_default_value = sum_{k : 0 .. N_full - 1} (
-//      grad_values[k] * 1{k not in reverse_index_map})
+//     sp_inputs[0]: shape = [2, 3]
+//     [0, 2]: "a"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+//     sp_inputs[1]: shape = [2, 4]
+//     [0, 1]: "d"
+//     [0, 2]: "e"
+//
+// then the output will be
+//
+//     shape = [2, 7]
+//     [0, 2]: "a"
+//     [0, 4]: "d"
+//     [0, 5]: "e"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+// Graphically this is equivalent to doing
+//
+//     [    a] concat [  d e  ] = [    a   d e  ]
+//     [b c  ]        [       ]   [b c          ]
 //
 // Arguments:
-//	reverse_index_map: 1-D.  The reverse index map from SparseFillEmptyRows.
-//	grad_values: 1-D.  The gradients from backprop.
+//	indices: 2-D.  Indices of each input `SparseTensor`.
+//	values: 1-D.  Non-empty values of each `SparseTensor`.
+//	shapes: 1-D.  Shapes of each `SparseTensor`.
+//	concat_dim: Dimension to concatenate along. Must be in range [-rank, rank),
+// where rank is the number of dimensions in each input `SparseTensor`.
 //
-// Returns 1-D.  The backprop into values.0-D.  The backprop into default_value.
-func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_values tf.Output) (d_values tf.Output, d_default_value tf.Output) {
+// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
+func SparseConcat(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, concat_dim int64) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"concat_dim": concat_dim}
 	opspec := tf.OpSpec{
-		Type: "SparseFillEmptyRowsGrad",
+		Type: "SparseConcat",
 		Input: []tf.Input{
-			reverse_index_map, grad_values,
+			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)`
+// Generates sparse cross from a list of sparse and dense tensors.
 //
-// if < 0, `scale * features` otherwise.
+// The op takes two lists, one of 2D `SparseTensor` and one of 2D `Tensor`, each
+// representing features of one feature column. It outputs a 2D `SparseTensor` with
+// the batchwise crosses of these features.
 //
-// See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
-func Selu(scope *Scope, features tf.Output) (activations tf.Output) {
+// For example, if the inputs are
+//
+//     inputs[0]: SparseTensor with shape = [2, 2]
+//     [0, 0]: "a"
+//     [1, 0]: "b"
+//     [1, 1]: "c"
+//
+//     inputs[1]: SparseTensor with shape = [2, 1]
+//     [0, 0]: "d"
+//     [1, 0]: "e"
+//
+//     inputs[2]: Tensor [["f"], ["g"]]
+//
+// then the output will be
+//
+//     shape = [2, 2]
+//     [0, 0]: "a_X_d_X_f"
+//     [1, 0]: "b_X_e_X_g"
+//     [1, 1]: "c_X_e_X_g"
+//
+// if hashed_output=true then the output will be
+//
+//     shape = [2, 2]
+//     [0, 0]: FingerprintCat64(
+//                 Fingerprint64("f"), FingerprintCat64(
+//                     Fingerprint64("d"), Fingerprint64("a")))
+//     [1, 0]: FingerprintCat64(
+//                 Fingerprint64("g"), FingerprintCat64(
+//                     Fingerprint64("e"), Fingerprint64("b")))
+//     [1, 1]: FingerprintCat64(
+//                 Fingerprint64("g"), FingerprintCat64(
+//                     Fingerprint64("e"), Fingerprint64("c")))
+//
+// Arguments:
+//	indices: 2-D.  Indices of each input `SparseTensor`.
+//	values: 1-D.   values of each `SparseTensor`.
+//	shapes: 1-D.   Shapes of each `SparseTensor`.
+//	dense_inputs: 2-D.    Columns represented by dense `Tensor`.
+//	hashed_output: If true, returns the hash of the cross instead of the string.
+// This will allow us avoiding string manipulations.
+//	num_buckets: It is used if hashed_output is true.
+// output = hashed_value%num_buckets if num_buckets > 0 else hashed_value.
+//	hash_key: Specify the hash_key that will be used by the `FingerprintCat64`
+// function to combine the crosses fingerprints.
+//
+//
+//
+// Returns 2-D.  Indices of the concatenated `SparseTensor`.1-D.  Non-empty values of the concatenated or hashed
+// `SparseTensor`.1-D.  Shape of the concatenated `SparseTensor`.
+func SparseCross(scope *Scope, indices []tf.Output, values []tf.Output, shapes []tf.Output, dense_inputs []tf.Output, hashed_output bool, num_buckets int64, hash_key int64, out_type tf.DataType, internal_type tf.DataType) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"hashed_output": hashed_output, "num_buckets": num_buckets, "hash_key": hash_key, "out_type": out_type, "internal_type": internal_type}
 	opspec := tf.OpSpec{
-		Type: "Selu",
+		Type: "SparseCross",
 		Input: []tf.Input{
-			features,
+			tf.OutputList(indices), tf.OutputList(values), tf.OutputList(shapes), tf.OutputList(dense_inputs),
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// SetSizeAttr is an optional argument to SetSize.
-type SetSizeAttr func(optionalAttr)
-
-// SetSizeValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func SetSizeValidateIndices(value bool) SetSizeAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Number of unique elements along last dimension of input `set`.
-//
-// Input `set` is a `SparseTensor` represented by `set_indices`, `set_values`,
-// and `set_shape`. The last dimension contains values in a set, duplicates are
-// allowed but ignored.
-//
-// If `validate_indices` is `True`, this op validates the order and range of `set`
-// indices.
+// Concatenates quantized tensors along one dimension.
 //
 // Arguments:
-//	set_indices: 2D `Tensor`, indices of a `SparseTensor`.
-//	set_values: 1D `Tensor`, values of a `SparseTensor`.
-//	set_shape: 1D `Tensor`, shape of a `SparseTensor`.
+//	concat_dim: 0-D.  The dimension along which to concatenate.  Must be in the
+// range [0, rank(values)).
+//	values: The `N` Tensors to concatenate. Their ranks and types must match,
+// and their sizes must match in all dimensions except `concat_dim`.
+//	input_mins: The minimum scalar values for each of the input tensors.
+//	input_maxes: The maximum scalar values for each of the input tensors.
 //
-// Returns For `set` ranked `n`, this is a `Tensor` with rank `n-1`, and the same 1st
-// `n-1` dimensions as `set`. Each value is the number of unique elements in
-// the corresponding `[0...n-1]` dimension of `set`.
-func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shape tf.Output, optional ...SetSizeAttr) (size tf.Output) {
+// Returns A `Tensor` with the concatenation of values stacked along the
+// `concat_dim` dimension.  This tensor's shape matches that of `values` except
+// in `concat_dim` where it has the sum of the sizes.The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
+func QuantizedConcat(scope *Scope, concat_dim tf.Output, values []tf.Output, input_mins []tf.Output, input_maxes []tf.Output) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "SetSize",
+		Type: "QuantizedConcat",
 		Input: []tf.Input{
-			set_indices, set_values, set_shape,
+			concat_dim, tf.OutputList(values), tf.OutputList(input_mins), tf.OutputList(input_maxes),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the sign and the log of the absolute value of the determinant of
+// Slice a `SparseTensor` based on the `start` and `size`.
 //
-// one or more square matrices.
+// For example, if the input is
 //
-// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
-// form square matrices. The outputs are two tensors containing the signs and
-// absolute values of the log determinants for all N input submatrices
-// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
-// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
-// is the LU decomposition of the input and P is the corresponding
-// permutation matrix.
+//     input_tensor = shape = [2, 7]
+//     [    a   d e  ]
+//     [b c          ]
+//
+// Graphically the output tensors are:
+//
+//     sparse_slice([0, 0], [2, 4]) = shape = [2, 4]
+//     [    a  ]
+//     [b c    ]
+//
+//     sparse_slice([0, 4], [2, 3]) = shape = [2, 3]
+//     [ d e  ]
+//     [      ]
 //
 // Arguments:
-//	input: Shape is `[N, M, M]`.
+//	indices: 2-D tensor represents the indices of the sparse tensor.
+//	values: 1-D tensor represents the values of the sparse tensor.
+//	shape: 1-D. tensor represents the shape of the sparse tensor.
+//	start: 1-D. tensor represents the start of the slice.
+//	size: 1-D. tensor represents the size of the slice.
+// output indices: A list of 1-D tensors represents the indices of the output
+// sparse tensors.
 //
-// Returns The signs of the log determinants of the inputs. Shape is `[N]`.The logs of the absolute values of the determinants
-// of the N input matrices.  Shape is `[N]`.
-func LogMatrixDeterminant(scope *Scope, input tf.Output) (sign tf.Output, log_abs_determinant tf.Output) {
+// Returns A list of 1-D tensors represents the values of the output sparse
+// tensors.A list of 1-D tensors represents the shape of the output sparse
+// tensors.
+func SparseSlice(scope *Scope, indices tf.Output, values tf.Output, shape tf.Output, start tf.Output, size tf.Output) (output_indices tf.Output, output_values tf.Output, output_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LogMatrixDeterminant",
+		Type: "SparseSlice",
 		Input: []tf.Input{
-			input,
+			indices, values, shape, start, size,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// SumAttr is an optional argument to Sum.
-type SumAttr func(optionalAttr)
-
-// SumKeepDims sets the optional keep_dims attribute to value.
-//
-// value: If true, retain reduced dimensions with length 1.
-// If not specified, defaults to false
-func SumKeepDims(value bool) SumAttr {
-	return func(m optionalAttr) {
-		m["keep_dims"] = value
-	}
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the sum of elements across dimensions of a tensor.
+// Adds up a `SparseTensor` and a dense `Tensor`, producing a dense `Tensor`.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// This Op does not require `a_indices` be sorted in standard lexicographic order.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
-//
-// Returns The reduced tensor.
-func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
+//	a_indices: 2-D.  The `indices` of the `SparseTensor`, with shape `[nnz, ndims]`.
+//	a_values: 1-D.  The `values` of the `SparseTensor`, with shape `[nnz]`.
+//	a_shape: 1-D.  The `shape` of the `SparseTensor`, with shape `[ndims]`.
+//	b: `ndims`-D Tensor.  With shape `a_shape`.
+func SparseTensorDenseAdd(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Sum",
+		Type: "SparseTensorDenseAdd",
 		Input: []tf.Input{
-			input, axis,
+			a_indices, a_values, a_shape, b,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Delete the tensor specified by its handle in the session.
+// Returns the set of files matching one or more glob patterns.
+//
+// Note that this routine only supports wildcard characters in the
+// basename portion of the pattern, not in the directory portion.
 //
 // Arguments:
-//	handle: The handle for a tensor stored in the session state.
+//	pattern: Shell wildcard pattern(s). Scalar or vector of type string.
 //
-// Returns the created operation.
-func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
+// Returns A vector of matching filenames.
+func MatchingFiles(scope *Scope, pattern tf.Output) (filenames tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DeleteSessionTensor",
+		Type: "MatchingFiles",
 		Input: []tf.Input{
-			handle,
+			pattern,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// L2 Loss.
+// MatrixSolveLsAttr is an optional argument to MatrixSolveLs.
+type MatrixSolveLsAttr func(optionalAttr)
+
+// MatrixSolveLsFast sets the optional fast attribute to value.
+// If not specified, defaults to true
+func MatrixSolveLsFast(value bool) MatrixSolveLsAttr {
+	return func(m optionalAttr) {
+		m["fast"] = value
+	}
+}
+
+// Solves one or more linear least-squares problems.
 //
-// Computes half the L2 norm of a tensor without the `sqrt`:
+// `matrix` is a tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form real or complex matrices of size `[M, N]`. `Rhs` is a tensor of the same
+// type as `matrix` and shape `[..., M, K]`.
+// The output is a tensor shape `[..., N, K]` where each output matrix solves
+// each of the equations
+// `matrix[..., :, :]` * `output[..., :, :]` = `rhs[..., :, :]`
+// in the least squares sense.
 //
-//     output = sum(t ** 2) / 2
+// We use the following notation for (complex) matrix and right-hand sides
+// in the batch:
+//
+// `matrix`=\\(A \in \mathbb{C}^{m \times n}\\),
+// `rhs`=\\(B  \in \mathbb{C}^{m \times k}\\),
+// `output`=\\(X  \in \mathbb{C}^{n \times k}\\),
+// `l2_regularizer`=\\(\lambda \in \mathbb{R}\\).
+//
+// If `fast` is `True`, then the solution is computed by solving the normal
+// equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
+// \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
+// problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
+// \lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+// \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
+// minimum-norm solution to the under-determined linear system, i.e.
+// \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
+// subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
+// when \\(A\\) is numerically full rank and has a condition number
+// \\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+// sufficiently large.
+//
+// If `fast` is `False` an algorithm based on the numerically robust complete
+// orthogonal decomposition is used. This computes the minimum-norm
+// least-squares solution, even when \\(A\\) is rank deficient. This path is
+// typically 6-7 times slower than the fast path. If `fast` is `False` then
+// `l2_regularizer` is ignored.
 //
 // Arguments:
-//	t: Typically 2-D, but may have any dimensions.
+//	matrix: Shape is `[..., M, N]`.
+//	rhs: Shape is `[..., M, K]`.
+//	l2_regularizer: Scalar tensor.
 //
-// Returns 0-D.
-func L2Loss(scope *Scope, t tf.Output) (output tf.Output) {
+// @compatibility(numpy)
+// Equivalent to np.linalg.lstsq
+// @end_compatibility
+//
+// Returns Shape is `[..., N, K]`.
+func MatrixSolveLs(scope *Scope, matrix tf.Output, rhs tf.Output, l2_regularizer tf.Output, optional ...MatrixSolveLsAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "L2Loss",
+		Type: "MatrixSolveLs",
 		Input: []tf.Input{
-			t,
+			matrix, rhs, l2_regularizer,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DenseToSparseSetOperationAttr is an optional argument to DenseToSparseSetOperation.
-type DenseToSparseSetOperationAttr func(optionalAttr)
+// Elementwise computes the bitwise OR of `x` and `y`.
+//
+// The result will have those bits set, that are set in `x`, `y` or both. The
+// computation is performed on the underlying representations of `x` and `y`.
+func BitwiseOr(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BitwiseOr",
+		Input: []tf.Input{
+			x, y,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// DenseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// SparseToSparseSetOperationAttr is an optional argument to SparseToSparseSetOperation.
+type SparseToSparseSetOperationAttr func(optionalAttr)
+
+// SparseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
 // If not specified, defaults to true
-func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperationAttr {
+func SparseToSparseSetOperationValidateIndices(value bool) SparseToSparseSetOperationAttr {
 	return func(m optionalAttr) {
 		m["validate_indices"] = value
 	}
 }
 
-// Applies set operation along last dimension of `Tensor` and `SparseTensor`.
+// Applies set operation along last dimension of 2 `SparseTensor` inputs.
 //
 // See SetOperationOp::SetOperationFromContext for values of `set_operation`.
 //
+// If `validate_indices` is `True`, `SparseToSparseSetOperation` validates the
+// order and range of `set1` and `set2` indices.
+//
+// Input `set1` is a `SparseTensor` represented by `set1_indices`, `set1_values`,
+// and `set1_shape`. For `set1` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set2`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
+//
 // Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
 // and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
 // as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
 // ignored.
 //
-// If `validate_indices` is `True`, this op validates the order and range of `set2`
-// indices.
+// If `validate_indices` is `True`, this op validates the order and range of `set1`
+// and `set2` indices.
 //
 // Output `result` is a `SparseTensor` represented by `result_indices`,
 // `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
@@ -15778,21 +15567,26 @@ func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperat
 // `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set1_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set1_shape: 1D `Tensor`, shape of a `SparseTensor`. `set1_shape[0...n-1]` must
+// be the same as `set2_shape[0...n-1]`, `set1_shape[n]` is the
+// max set size across `0...n-1` dimensions.
 //	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
 // order.
 //	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
 // order.
 //	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
-// be the same as the 1st `n-1` dimensions of `set1`, `result_shape[n]` is the
-// max set size across `n-1` dimensions.
+// be the same as `set1_shape[0...n-1]`, `set2_shape[n]` is the
+// max set size across `0...n-1` dimensions.
 //
 //
 // Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
 // the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
 // is the max result set size across all `0...n-1` dimensions.
-func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...DenseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+func SparseToSparseSetOperation(scope *Scope, set1_indices tf.Output, set1_values tf.Output, set1_shape tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...SparseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -15801,9 +15595,9 @@ func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Out
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DenseToSparseSetOperation",
+		Type: "SparseToSparseSetOperation",
 		Input: []tf.Input{
-			set1, set2_indices, set2_values, set2_shape,
+			set1_indices, set1_values, set1_shape, set2_indices, set2_values, set2_shape,
 		},
 		Attrs: attrs,
 	}
@@ -15811,85 +15605,66 @@ func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Out
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Subtracts a value from the current value of a variable.
-//
-// Any ReadVariableOp which depends directly or indirectly on this assign is
-// guaranteed to see the incremented value or a subsequent newer one.
-//
-// Outputs the incremented value, which can be used to totally order the
-// increments to this variable.
-//
-// Arguments:
-//	resource: handle to the resource in which to store the variable.
-//	value: the value by which the variable will be incremented.
+// Computes numerical negative value element-wise.
 //
-// Returns the created operation.
-func AssignSubVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
+// I.e., \\(y = -x\\).
+func Neg(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AssignSubVariableOp",
+		Type: "Neg",
 		Input: []tf.Input{
-			resource, value,
+			x,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// RestoreAttr is an optional argument to Restore.
-type RestoreAttr func(optionalAttr)
+// FakeQuantWithMinMaxVarsAttr is an optional argument to FakeQuantWithMinMaxVars.
+type FakeQuantWithMinMaxVarsAttr func(optionalAttr)
 
-// RestorePreferredShard sets the optional preferred_shard attribute to value.
-//
-// value: Index of file to open first if multiple files match
-// `file_pattern`.
-// If not specified, defaults to -1
-func RestorePreferredShard(value int64) RestoreAttr {
+// FakeQuantWithMinMaxVarsNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func FakeQuantWithMinMaxVarsNumBits(value int64) FakeQuantWithMinMaxVarsAttr {
 	return func(m optionalAttr) {
-		m["preferred_shard"] = value
+		m["num_bits"] = value
 	}
 }
 
-// Restores a tensor from checkpoint files.
-//
-// Reads a tensor stored in one or several files. If there are several files (for
-// instance because a tensor was saved as slices), `file_pattern` may contain
-// wildcard symbols (`*` and `?`) in the filename portion only, not in the
-// directory portion.
-//
-// If a `file_pattern` matches several files, `preferred_shard` can be used to hint
-// in which file the requested tensor is likely to be found. This op will first
-// open the file at index `preferred_shard` in the list of matching files and try
-// to restore tensors from that file.  Only if some tensors or tensor slices are
-// not found in that first file, then the Op opens all the files. Setting
-// `preferred_shard` to match the value passed as the `shard` input
-// of a matching `Save` Op may speed up Restore.  This attribute only affects
-// performance, not correctness.  The default value -1 means files are processed in
-// order.
+// FakeQuantWithMinMaxVarsNarrowRange sets the optional narrow_range attribute to value.
+// If not specified, defaults to false
+func FakeQuantWithMinMaxVarsNarrowRange(value bool) FakeQuantWithMinMaxVarsAttr {
+	return func(m optionalAttr) {
+		m["narrow_range"] = value
+	}
+}
+
+// Fake-quantize the 'inputs' tensor of type float via global float scalars `min`
 //
-// See also `RestoreSlice`.
+// and `max` to 'outputs' tensor of same shape as `inputs`.
 //
-// Arguments:
-//	file_pattern: Must have a single element. The pattern of the files from
-// which we read the tensor.
-//	tensor_name: Must have a single element. The name of the tensor to be
-// restored.
-//	dt: The type of the tensor to be restored.
+// `[min; max]` define the clamping range for the `inputs` data.
+// `inputs` values are quantized into the quantization range (`[0; 2^num_bits - 1]`
+// when `narrow_range` is false and `[1; 2^num_bits - 1]` when it is true) and
+// then de-quantized and output as floats in `[min; max]` interval.
+// `num_bits` is the bitwidth of the quantization; between 2 and 8, inclusive.
 //
-// Returns The restored tensor.
-func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf.DataType, optional ...RestoreAttr) (tensor tf.Output) {
+// This operation has a gradient and thus allows for training `min` and `max`
+// values.
+func FakeQuantWithMinMaxVars(scope *Scope, inputs tf.Output, min tf.Output, max tf.Output, optional ...FakeQuantWithMinMaxVarsAttr) (outputs tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dt": dt}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Restore",
+		Type: "FakeQuantWithMinMaxVars",
 		Input: []tf.Input{
-			file_pattern, tensor_name,
+			inputs, min, max,
 		},
 		Attrs: attrs,
 	}
@@ -15897,240 +15672,175 @@ func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf.
 	return op.Output(0)
 }
 
-// QuantizedResizeBilinearAttr is an optional argument to QuantizedResizeBilinear.
-type QuantizedResizeBilinearAttr func(optionalAttr)
-
-// QuantizedResizeBilinearAlignCorners sets the optional align_corners attribute to value.
-//
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func QuantizedResizeBilinearAlignCorners(value bool) QuantizedResizeBilinearAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
-	}
-}
-
-// Resize quantized `images` to `size` using quantized bilinear interpolation.
+// Returns the element-wise min of two SparseTensors.
 //
-// Input images and output images must be quantized types.
+// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
-//
-//
+//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, in the canonical lexicographic ordering.
+//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
+//	a_shape: 1-D.  Shape of the input SparseTensor.
+//	b_indices: counterpart to `a_indices` for the other operand.
+//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
+//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min tf.Output, max tf.Output, optional ...QuantizedResizeBilinearAttr) (resized_images tf.Output, out_min tf.Output, out_max tf.Output) {
+// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
+func SparseSparseMinimum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedResizeBilinear",
+		Type: "SparseSparseMinimum",
 		Input: []tf.Input{
-			images, size, min, max,
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1)
 }
 
-// Computes the minimum along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the min is empty for a given segment ID `i`, `output[i] = 0`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
-// </div>
-//
-// Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
+// TakeManySparseFromTensorsMapAttr is an optional argument to TakeManySparseFromTensorsMap.
+type TakeManySparseFromTensorsMapAttr func(optionalAttr)
+
+// TakeManySparseFromTensorsMapContainer sets the optional container attribute to value.
 //
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "SegmentMin",
-		Input: []tf.Input{
-			data, segment_ids,
-		},
+// value: The container name for the `SparseTensorsMap` read by this op.
+// If not specified, defaults to ""
+func TakeManySparseFromTensorsMapContainer(value string) TakeManySparseFromTensorsMapAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
-type SdcaOptimizerAttr func(optionalAttr)
-
-// SdcaOptimizerAdaptative sets the optional adaptative attribute to value.
+// TakeManySparseFromTensorsMapSharedName sets the optional shared_name attribute to value.
 //
-// value: Whether to use Adapative SDCA for the inner loop.
-// If not specified, defaults to false
-func SdcaOptimizerAdaptative(value bool) SdcaOptimizerAttr {
+// value: The shared name for the `SparseTensorsMap` read by this op.
+// It should not be blank; rather the `shared_name` or unique Operation name
+// of the Op that created the original `SparseTensorsMap` should be used.
+// If not specified, defaults to ""
+func TakeManySparseFromTensorsMapSharedName(value string) TakeManySparseFromTensorsMapAttr {
 	return func(m optionalAttr) {
-		m["adaptative"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Distributed version of Stochastic Dual Coordinate Ascent (SDCA) optimizer for
+// Read `SparseTensors` from a `SparseTensorsMap` and concatenate them.
 //
-// linear models with L1 + L2 regularization. As global optimization objective is
-// strongly-convex, the optimizer optimizes the dual objective at each step. The
-// optimizer applies each update one example at a time. Examples are sampled
-// uniformly, and the optimizer is learning rate free and enjoys linear convergence
-// rate.
+// The input `sparse_handles` must be an `int64` matrix of shape `[N, 1]` where
+// `N` is the minibatch size and the rows correspond to the output handles of
+// `AddSparseToTensorsMap` or `AddManySparseToTensorsMap`.  The ranks of the
+// original `SparseTensor` objects that went into the given input ops must all
+// match.  When the final `SparseTensor` is created, it has rank one
+// higher than the ranks of the incoming `SparseTensor` objects
+// (they have been concatenated along a new row dimension on the left).
 //
-// [Proximal Stochastic Dual Coordinate Ascent](http://arxiv.org/pdf/1211.2717v1.pdf).<br>
-// Shai Shalev-Shwartz, Tong Zhang. 2012
+// The output `SparseTensor` object's shape values for all dimensions but the
+// first are the max across the input `SparseTensor` objects' shape values
+// for the corresponding dimensions.  Its first shape value is `N`, the minibatch
+// size.
 //
-// $$Loss Objective = \sum f_{i} (wx_{i}) + (l2 / 2) * |w|^2 + l1 * |w|$$
+// The input `SparseTensor` objects' indices are assumed ordered in
+// standard lexicographic order.  If this is not the case, after this
+// step run `SparseReorder` to restore index ordering.
 //
-// [Adding vs. Averaging in Distributed Primal-Dual Optimization](http://arxiv.org/abs/1502.03508).<br>
-// Chenxin Ma, Virginia Smith, Martin Jaggi, Michael I. Jordan,
-// Peter Richtarik, Martin Takac. 2015
+// For example, if the handles represent an input, which is a `[2, 3]` matrix
+// representing two original `SparseTensor` objects:
 //
-// [Stochastic Dual Coordinate Ascent with Adaptive Probabilities](https://arxiv.org/abs/1502.08053).<br>
-// Dominik Csiba, Zheng Qu, Peter Richtarik. 2015
+// ```
+//     index = [ 0]
+//             [10]
+//             [20]
+//     values = [1, 2, 3]
+//     shape = [50]
+// ```
+//
+// and
+//
+// ```
+//     index = [ 2]
+//             [10]
+//     values = [4, 5]
+//     shape = [30]
+// ```
+//
+// then the final `SparseTensor` will be:
+//
+// ```
+//     index = [0  0]
+//             [0 10]
+//             [0 20]
+//             [1  2]
+//             [1 10]
+//     values = [1, 2, 3, 4, 5]
+//     shape = [2 50]
+// ```
 //
 // Arguments:
-//	sparse_example_indices: a list of vectors which contain example indices.
-//	sparse_feature_indices: a list of vectors which contain feature indices.
-//	sparse_feature_values: a list of vectors which contains feature value
-// associated with each feature group.
-//	dense_features: a list of matrices which contains the dense feature values.
-//	example_weights: a vector which contains the weight associated with each
-// example.
-//	example_labels: a vector which contains the label/target associated with each
-// example.
-//	sparse_indices: a list of vectors where each value is the indices which has
-// corresponding weights in sparse_weights. This field maybe omitted for the
-// dense approach.
-//	sparse_weights: a list of vectors where each value is the weight associated with
-// a sparse feature group.
-//	dense_weights: a list of vectors where the values are the weights associated
-// with a dense feature group.
-//	example_state_data: a list of vectors containing the example state data.
-//	loss_type: Type of the primal loss. Currently SdcaSolver supports logistic,
-// squared and hinge losses.
-//	l1: Symmetric l1 regularization strength.
-//	l2: Symmetric l2 regularization strength.
-//	num_loss_partitions: Number of partitions of the global loss function.
-//	num_inner_iterations: Number of iterations per mini-batch.
+//	sparse_handles: 1-D, The `N` serialized `SparseTensor` objects.
+// Shape: `[N]`.
+//	dtype: The `dtype` of the `SparseTensor` objects stored in the
+// `SparseTensorsMap`.
 //
-// Returns a list of vectors containing the updated example state
-// data.a list of vectors where each value is the delta
-// weights associated with a sparse feature group.a list of vectors where the values are the delta
-// weights associated with a dense feature group.
-func SdcaOptimizer(scope *Scope, sparse_example_indices []tf.Output, sparse_feature_indices []tf.Output, sparse_feature_values []tf.Output, dense_features []tf.Output, example_weights tf.Output, example_labels tf.Output, sparse_indices []tf.Output, sparse_weights []tf.Output, dense_weights []tf.Output, example_state_data tf.Output, loss_type string, l1 float32, l2 float32, num_loss_partitions int64, num_inner_iterations int64, optional ...SdcaOptimizerAttr) (out_example_state_data tf.Output, out_delta_sparse_weights []tf.Output, out_delta_dense_weights []tf.Output) {
+// Returns 2-D.  The `indices` of the minibatch `SparseTensor`.1-D.  The `values` of the minibatch `SparseTensor`.1-D.  The `shape` of the minibatch `SparseTensor`.
+func TakeManySparseFromTensorsMap(scope *Scope, sparse_handles tf.Output, dtype tf.DataType, optional ...TakeManySparseFromTensorsMapAttr) (sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"loss_type": loss_type, "l1": l1, "l2": l2, "num_loss_partitions": num_loss_partitions, "num_inner_iterations": num_inner_iterations}
+	attrs := map[string]interface{}{"dtype": dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SdcaOptimizer",
+		Type: "TakeManySparseFromTensorsMap",
 		Input: []tf.Input{
-			tf.OutputList(sparse_example_indices), tf.OutputList(sparse_feature_indices), tf.OutputList(sparse_feature_values), tf.OutputList(dense_features), example_weights, example_labels, tf.OutputList(sparse_indices), tf.OutputList(sparse_weights), tf.OutputList(dense_weights), example_state_data,
+			sparse_handles,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	out_example_state_data = op.Output(idx)
-	if out_delta_sparse_weights, idx, err = makeOutputList(op, idx, "out_delta_sparse_weights"); err != nil {
-		scope.UpdateErr("SdcaOptimizer", err)
-		return
-	}
-	if out_delta_dense_weights, idx, err = makeOutputList(op, idx, "out_delta_dense_weights"); err != nil {
-		scope.UpdateErr("SdcaOptimizer", err)
-		return
-	}
-	return out_example_state_data, out_delta_sparse_weights, out_delta_dense_weights
-}
-
-// SparseMatMulAttr is an optional argument to SparseMatMul.
-type SparseMatMulAttr func(optionalAttr)
-
-// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_a"] = value
-	}
-}
-
-// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
-// If not specified, defaults to false
-func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["transpose_b"] = value
-	}
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
-	return func(m optionalAttr) {
-		m["a_is_sparse"] = value
-	}
-}
+// MaxPoolAttr is an optional argument to MaxPool.
+type MaxPoolAttr func(optionalAttr)
 
-// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
-// If not specified, defaults to false
-func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
+// MaxPoolDataFormat sets the optional data_format attribute to value.
+//
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func MaxPoolDataFormat(value string) MaxPoolAttr {
 	return func(m optionalAttr) {
-		m["b_is_sparse"] = value
+		m["data_format"] = value
 	}
 }
 
-// Multiply matrix "a" by matrix "b".
+// Performs max pooling on the input.
 //
-// The inputs must be two-dimensional matrices and the inner dimension of "a" must
-// match the outer dimension of "b". This op is optimized for the case where at
-// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
-// matrix multiply on one platform was 30% zero values in the sparse matrix.
+// Arguments:
+//	input: 4-D input to pool over.
+//	ksize: The size of the window for each dimension of the input tensor.
+//	strides: The stride of the sliding window for each dimension of the
+// input tensor.
+//	padding: The type of padding algorithm to use.
 //
-// The gradient computation of this operation will only take advantage of sparsity
-// in the input gradient when that gradient comes from a Relu.
-func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
+// Returns The max pooled output tensor.
+func MaxPool(scope *Scope, input tf.Output, ksize []int64, strides []int64, padding string, optional ...MaxPoolAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseMatMul",
+		Type: "MaxPool",
 		Input: []tf.Input{
-			a, b,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -16138,217 +15848,164 @@ func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatM
 	return op.Output(0)
 }
 
-// Computes the power of one value to another.
+// Says whether the targets are in the top `K` predictions.
 //
-// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
-// corresponding elements in `x` and `y`. For example:
+// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+// prediction for the target class is among the top `k` predictions among
+// all predictions for example `i`. Note that the behavior of `InTopK` differs
+// from the `TopK` op in its handling of ties; if multiple classes have the
+// same prediction value and straddle the top-`k` boundary, all of those
+// classes are considered to be in the top `k`.
 //
-// ```
-// # tensor 'x' is [[2, 2]], [3, 3]]
-// # tensor 'y' is [[8, 16], [2, 3]]
-// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
-// ```
-func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// More formally, let
+//
+//   \\(predictions_i\\) be the predictions for all classes for example `i`,
+//   \\(targets_i\\) be the target class for example `i`,
+//   \\(out_i\\) be the output for example `i`,
+//
+// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+//
+// Arguments:
+//	predictions: A `batch_size` x `classes` tensor.
+//	targets: A `batch_size` vector of class ids.
+//	k: Number of top elements to look at for computing precision.
+//
+// Returns Computed precision at `k` as a `bool Tensor`.
+func InTopKV2(scope *Scope, predictions tf.Output, targets tf.Output, k tf.Output) (precision tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Pow",
+		Type: "InTopKV2",
 		Input: []tf.Input{
-			x, y,
+			predictions, targets, k,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ShapeAttr is an optional argument to Shape.
-type ShapeAttr func(optionalAttr)
-
-// ShapeOutType sets the optional out_type attribute to value.
-// If not specified, defaults to DT_INT32
-func ShapeOutType(value tf.DataType) ShapeAttr {
-	return func(m optionalAttr) {
-		m["out_type"] = value
-	}
-}
-
-// Returns the shape of a tensor.
+// Assigns a new value to a variable.
 //
-// This operation returns a 1-D integer tensor representing the shape of `input`.
+// Any ReadVariableOp with a control dependency on this op is guaranteed to return
+// this value or a subsequent newer value of the variable.
 //
-// For example:
+// Arguments:
+//	resource: handle to the resource in which to store the variable.
+//	value: the value to set the new tensor to use.
 //
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// shape(t) ==> [2, 2, 3]
-// ```
-func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
+// Returns the created operation.
+func AssignVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Shape",
+		Type: "AssignVariableOp",
 		Input: []tf.Input{
-			input,
+			resource, value,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Computes fingerprints of the input strings.
+// Returns a tensor of ones with the same shape and type as x.
 //
 // Arguments:
-//	input: vector of strings to compute fingerprints on.
+//	x: a tensor of type T.
 //
-// Returns a (N,2) shaped matrix where N is the number of elements in the input
-// vector. Each row contains the low and high parts of the fingerprint.
-func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns a tensor of the same shape and type as x but filled with ones.
+func OnesLike(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SdcaFprint",
+		Type: "OnesLike",
 		Input: []tf.Input{
-			input,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
-type RandomPoissonV2Attr func(optionalAttr)
-
-// RandomPoissonV2Seed sets the optional seed attribute to value.
+// The gradient of SparseFillEmptyRows.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["seed"] = value
-	}
-}
-
-// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
+// Takes vectors reverse_index_map, shaped `[N]`, and grad_values,
+// shaped `[N_full]`, where `N_full >= N` and copies data into either
+// `d_values` or `d_default_value`.  Here `d_values` is shaped `[N]` and
+// `d_default_value` is a scalar.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
+//   d_values[j] = grad_values[reverse_index_map[j]]
+//   d_default_value = sum_{k : 0 .. N_full - 1} (
+//      grad_values[k] * 1{k not in reverse_index_map})
+//
+// Arguments:
+//	reverse_index_map: 1-D.  The reverse index map from SparseFillEmptyRows.
+//	grad_values: 1-D.  The gradients from backprop.
+//
+// Returns 1-D.  The backprop into values.0-D.  The backprop into default_value.
+func SparseFillEmptyRowsGrad(scope *Scope, reverse_index_map tf.Output, grad_values tf.Output) (d_values tf.Output, d_default_value tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// RandomPoissonV2Dtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_INT64
-func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
+	opspec := tf.OpSpec{
+		Type: "SparseFillEmptyRowsGrad",
+		Input: []tf.Input{
+			reverse_index_map, grad_values,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Outputs random values from the Poisson distribution(s) described by rate.
-//
-// This op uses two algorithms, depending on rate. If rate >= 10, then
-// the algorithm by Hormann is used to acquire samples via
-// transformation-rejection.
-// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
-//
-// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
-// random variables.
-// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
-// Programming, Volume 2. Addison Wesley
+// Computes scaled exponential linear: `scale * alpha * (exp(features) - 1)`
 //
-// Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in rate.
-//	rate: A tensor in which each scalar is a "rate" parameter describing the
-// associated poisson distribution.
+// if < 0, `scale * features` otherwise.
 //
-// Returns A tensor with shape `shape + shape(rate)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `rate[i0, i1, ...iN]`.
-func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
+// See [Self-Normalizing Neural Networks](https://arxiv.org/abs/1706.02515)
+func Selu(scope *Scope, features tf.Output) (activations tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "RandomPoissonV2",
+		Type: "Selu",
 		Input: []tf.Input{
-			shape, rate,
+			features,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
-type MatrixTriangularSolveAttr func(optionalAttr)
+// SetSizeAttr is an optional argument to SetSize.
+type SetSizeAttr func(optionalAttr)
 
-// MatrixTriangularSolveLower sets the optional lower attribute to value.
-//
-// value: Boolean indicating whether the innermost matrices in `matrix` are
-// lower or upper triangular.
+// SetSizeValidateIndices sets the optional validate_indices attribute to value.
 // If not specified, defaults to true
-func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
-	return func(m optionalAttr) {
-		m["lower"] = value
-	}
-}
-
-// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
-//
-// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
-//          adjoint.
-//
-// @compatibility(numpy)
-// Equivalent to np.linalg.triangular_solve
-// @end_compatibility
-// If not specified, defaults to false
-func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
+func SetSizeValidateIndices(value bool) SetSizeAttr {
 	return func(m optionalAttr) {
-		m["adjoint"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Solves systems of linear equations with upper or lower triangular matrices by
-//
-// backsubstitution.
+// Number of unique elements along last dimension of input `set`.
 //
-// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
-// square matrices. If `lower` is `True` then the strictly upper triangular part
-// of each inner-most matrix is assumed to be zero and not accessed.
-// If `lower` is False then the strictly lower triangular part of each inner-most
-// matrix is assumed to be zero and not accessed.
-// `rhs` is a tensor of shape `[..., M, K]`.
+// Input `set` is a `SparseTensor` represented by `set_indices`, `set_values`,
+// and `set_shape`. The last dimension contains values in a set, duplicates are
+// allowed but ignored.
 //
-// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
-// `True` then the innermost matrices in `output` satisfy matrix equations
-// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
-// If `adjoint` is `False` then the strictly then the  innermost matrices in
-// `output` satisfy matrix equations
-// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+// If `validate_indices` is `True`, this op validates the order and range of `set`
+// indices.
 //
 // Arguments:
-//	matrix: Shape is `[..., M, M]`.
-//	rhs: Shape is `[..., M, K]`.
+//	set_indices: 2D `Tensor`, indices of a `SparseTensor`.
+//	set_values: 1D `Tensor`, values of a `SparseTensor`.
+//	set_shape: 1D `Tensor`, shape of a `SparseTensor`.
 //
-// Returns Shape is `[..., M, K]`.
-func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
+// Returns For `set` ranked `n`, this is a `Tensor` with rank `n-1`, and the same 1st
+// `n-1` dimensions as `set`. Each value is the number of unique elements in
+// the corresponding `[0...n-1]` dimension of `set`.
+func SetSize(scope *Scope, set_indices tf.Output, set_values tf.Output, set_shape tf.Output, optional ...SetSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16357,9 +16014,9 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixTriangularSolve",
+		Type: "SetSize",
 		Input: []tf.Input{
-			matrix, rhs,
+			set_indices, set_values, set_shape,
 		},
 		Attrs: attrs,
 	}
@@ -16367,108 +16024,75 @@ func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, option
 	return op.Output(0)
 }
 
-// Computes inverse hyperbolic sine of x element-wise.
-func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
+// Computes the sign and the log of the absolute value of the determinant of
+//
+// one or more square matrices.
+//
+// The input is a tensor of shape `[N, M, M]` whose inner-most 2 dimensions
+// form square matrices. The outputs are two tensors containing the signs and
+// absolute values of the log determinants for all N input submatrices
+// `[..., :, :]` such that the determinant = sign*exp(log_abs_determinant).
+// The log_abs_determinant is computed as det(P)*sum(log(diag(LU))) where LU
+// is the LU decomposition of the input and P is the corresponding
+// permutation matrix.
+//
+// Arguments:
+//	input: Shape is `[N, M, M]`.
+//
+// Returns The signs of the log determinants of the inputs. Shape is `[N]`.The logs of the absolute values of the determinants
+// of the N input matrices.  Shape is `[N]`.
+func LogMatrixDeterminant(scope *Scope, input tf.Output) (sign tf.Output, log_abs_determinant tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Asinh",
+		Type: "LogMatrixDeterminant",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Creates a dataset with a range of values. Corresponds to python's xrange.
-//
-// Arguments:
-//	start: corresponds to start in python's xrange().
-//	stop: corresponds to stop in python's xrange().
-//	step: corresponds to step in python's xrange().
-//
-//
-func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
-	opspec := tf.OpSpec{
-		Type: "RangeDataset",
-		Input: []tf.Input{
-			start, stop, step,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// DepthwiseConv2dNativeBackpropInputAttr is an optional argument to DepthwiseConv2dNativeBackpropInput.
-type DepthwiseConv2dNativeBackpropInputAttr func(optionalAttr)
+// SumAttr is an optional argument to Sum.
+type SumAttr func(optionalAttr)
 
-// DepthwiseConv2dNativeBackpropInputDataFormat sets the optional data_format attribute to value.
+// SumKeepDims sets the optional keep_dims attribute to value.
 //
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, height, width, channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, channels, height, width].
-// If not specified, defaults to "NHWC"
-func DepthwiseConv2dNativeBackpropInputDataFormat(value string) DepthwiseConv2dNativeBackpropInputAttr {
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func SumKeepDims(value bool) SumAttr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["keep_dims"] = value
 	}
 }
 
-// DepthwiseConv2dNativeBackpropInputDilations sets the optional dilations attribute to value.
+// Computes the sum of elements across dimensions of a tensor.
 //
-// value: 1-D tensor of length 4.  The dilation factor for each dimension of
-// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
-// element on that dimension. The dimension order is determined by the value of
-// `data_format`, see above for details. Dilations in the batch and depth
-// dimensions must be 1.
-// If not specified, defaults to <i:1 i:1 i:1 i:1 >
-func DepthwiseConv2dNativeBackpropInputDilations(value []int64) DepthwiseConv2dNativeBackpropInputAttr {
-	return func(m optionalAttr) {
-		m["dilations"] = value
-	}
-}
-
-// Computes the gradients of depthwise convolution with respect to the input.
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	input_sizes: An integer vector representing the shape of `input`, based
-// on `data_format`.  For example, if `data_format` is 'NHWC' then
-//  `input` is a 4-D `[batch, height, width, channels]` tensor.
-//	filter: 4-D with shape
-// `[filter_height, filter_width, in_channels, depthwise_multiplier]`.
-//	out_backprop: 4-D with shape  based on `data_format`.
-// For example, if `data_format` is 'NHWC' then
-// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
-// Gradients w.r.t. the output of the convolution.
-//	strides: The stride of the sliding window for each dimension of the input
-// of the convolution.
-//	padding: The type of padding algorithm to use.
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// Returns 4-D with shape according to `data_format`.  For example, if
-// `data_format` is 'NHWC', output shape is `[batch, in_height,
-// in_width, in_channels]`.  Gradient w.r.t. the input of the
-// convolution.
-func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropInputAttr) (output tf.Output) {
+// Returns The reduced tensor.
+func Sum(scope *Scope, input tf.Output, axis tf.Output, optional ...SumAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DepthwiseConv2dNativeBackpropInput",
+		Type: "Sum",
 		Input: []tf.Input{
-			input_sizes, filter, out_backprop,
+			input, axis,
 		},
 		Attrs: attrs,
 	}
@@ -16476,252 +16100,191 @@ func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, fil
 	return op.Output(0)
 }
 
-// Adds sparse updates to the variable referenced by `resource`.
-//
-// This operation computes
-//
-//     # Scalar indices
-//     ref[indices, ...] += updates[...]
-//
-//     # Vector indices (for each i)
-//     ref[indices[i], ...] += updates[i, ...]
-//
-//     # High rank indices (for each i, ..., j)
-//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
-//
-// Duplicate entries are handled correctly: if multiple `indices` reference
-// the same location, their contributions add.
-//
-// Requires `updates.shape = indices.shape + ref.shape[1:]`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
-// </div>
+// Delete the tensor specified by its handle in the session.
 //
 // Arguments:
-//	resource: Should be from a `Variable` node.
-//	indices: A tensor of indices into the first dimension of `ref`.
-//	updates: A tensor of updated values to add to `ref`.
+//	handle: The handle for a tensor stored in the session state.
 //
 // Returns the created operation.
-func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
+func DeleteSessionTensor(scope *Scope, handle tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceScatterAdd",
+		Type: "DeleteSessionTensor",
 		Input: []tf.Input{
-			resource, indices, updates,
+			handle,
 		},
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Says whether the targets are in the top `K` predictions.
-//
-// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
-// prediction for the target class is among the top `k` predictions among
-// all predictions for example `i`. Note that the behavior of `InTopK` differs
-// from the `TopK` op in its handling of ties; if multiple classes have the
-// same prediction value and straddle the top-`k` boundary, all of those
-// classes are considered to be in the top `k`.
-//
-// More formally, let
+// L2 Loss.
 //
-//   \\(predictions_i\\) be the predictions for all classes for example `i`,
-//   \\(targets_i\\) be the target class for example `i`,
-//   \\(out_i\\) be the output for example `i`,
+// Computes half the L2 norm of a tensor without the `sqrt`:
 //
-// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
+//     output = sum(t ** 2) / 2
 //
 // Arguments:
-//	predictions: A `batch_size` x `classes` tensor.
-//	targets: A `batch_size` vector of class ids.
-//	k: Number of top elements to look at for computing precision.
+//	t: Typically 2-D, but may have any dimensions.
 //
-// Returns Computed Precision at `k` as a `bool Tensor`.
-func InTopK(scope *Scope, predictions tf.Output, targets tf.Output, k int64) (precision tf.Output) {
+// Returns 0-D.
+func L2Loss(scope *Scope, t tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"k": k}
 	opspec := tf.OpSpec{
-		Type: "InTopK",
+		Type: "L2Loss",
 		Input: []tf.Input{
-			predictions, targets,
+			t,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the gradient for the inverse of `x` wrt its input.
-//
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ReciprocalGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
+// DenseToSparseSetOperationAttr is an optional argument to DenseToSparseSetOperation.
+type DenseToSparseSetOperationAttr func(optionalAttr)
+
+// DenseToSparseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func DenseToSparseSetOperationValidateIndices(value bool) DenseToSparseSetOperationAttr {
+	return func(m optionalAttr) {
+		m["validate_indices"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+// Applies set operation along last dimension of `Tensor` and `SparseTensor`.
 //
-// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
-// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+//
+// Input `set2` is a `SparseTensor` represented by `set2_indices`, `set2_values`,
+// and `set2_shape`. For `set2` ranked `n`, 1st `n-1` dimensions must be the same
+// as `set1`. Dimension `n` contains values in a set, duplicates are allowed but
+// ignored.
+//
+// If `validate_indices` is `True`, this op validates the order and range of `set2`
+// indices.
+//
+// Output `result` is a `SparseTensor` represented by `result_indices`,
+// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+// dimension contains the result of `set_operation` applied to the corresponding
+// `[0...n-1]` dimension of `set`.
+//
+// Arguments:
+//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set2_indices: 2D `Tensor`, indices of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_values: 1D `Tensor`, values of a `SparseTensor`. Must be in row-major
+// order.
+//	set2_shape: 1D `Tensor`, shape of a `SparseTensor`. `set2_shape[0...n-1]` must
+// be the same as the 1st `n-1` dimensions of `set1`, `result_shape[n]` is the
+// max set size across `n-1` dimensions.
+//
+//
+// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
+// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
+// is the max result set size across all `0...n-1` dimensions.
+func DenseToSparseSetOperation(scope *Scope, set1 tf.Output, set2_indices tf.Output, set2_values tf.Output, set2_shape tf.Output, set_operation string, optional ...DenseToSparseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"set_operation": set_operation}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Minimum",
+		Type: "DenseToSparseSetOperation",
 		Input: []tf.Input{
-			x, y,
+			set1, set2_indices, set2_values, set2_shape,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Returns the element-wise sum of a list of tensors.
-//
-// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
-// wait for all of its inputs to be ready before beginning to sum. This can
-// save memory if inputs are ready at different times, since minimum temporary
-// storage is proportional to the output size rather than the inputs size.
+// Subtracts a value from the current value of a variable.
 //
-// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
+// Any ReadVariableOp which depends directly or indirectly on this assign is
+// guaranteed to see the incremented value or a subsequent newer one.
 //
-// Returns a `Tensor` of same shape and type as the elements of `inputs`.
+// Outputs the incremented value, which can be used to totally order the
+// increments to this variable.
 //
 // Arguments:
-//	inputs: A list of `Tensor` objects, each with same shape and type.
-//	shape: Shape of elements of `inputs`.
-func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
+//	resource: handle to the resource in which to store the variable.
+//	value: the value by which the variable will be incremented.
+//
+// Returns the created operation.
+func AssignSubVariableOp(scope *Scope, resource tf.Output, value tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "AccumulateNV2",
+		Type: "AssignSubVariableOp",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			resource, value,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Convert the quantized 'input' tensor into a lower-precision 'output', using the
-//
-// actual distribution of the values to maximize the usage of the lower bit depth
-// and adjusting the output min and max ranges accordingly.
-//
-// [input_min, input_max] are scalar floats that specify the range for the float
-// interpretation of the 'input' data. For example, if input_min is -1.0f and
-// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
-// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
-//
-// This operator tries to squeeze as much precision as possible into an output with
-// a lower bit depth by calculating the actual min and max values found in the
-// data. For example, maybe that quint16 input has no values lower than 16,384 and
-// none higher than 49,152. That means only half the range is actually needed, all
-// the float interpretations are between -0.5f and 0.5f, so if we want to compress
-// the data into a quint8 output, we can use that range rather than the theoretical
-// -1.0f to 1.0f that is suggested by the input min and max.
-//
-// In practice, this is most useful for taking output from operations like
-// QuantizedMatMul that can produce higher bit-depth outputs than their inputs and
-// may have large potential output ranges, but in practice have a distribution of
-// input values that only uses a small fraction of the possible range. By feeding
-// that output into this operator, we can reduce it from 32 bits down to 8 with
-// minimal loss of accuracy.
-//
-// Arguments:
-//
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//	out_type: The type of the output. Should be a lower bit depth than Tinput.
-//
-// Returns The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
-func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"out_type": out_type}
-	opspec := tf.OpSpec{
-		Type: "QuantizeDownAndShrinkRange",
-		Input: []tf.Input{
-			input, input_min, input_max,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// RandomGammaAttr is an optional argument to RandomGamma.
-type RandomGammaAttr func(optionalAttr)
+// RestoreAttr is an optional argument to Restore.
+type RestoreAttr func(optionalAttr)
 
-// RandomGammaSeed sets the optional seed attribute to value.
+// RestorePreferredShard sets the optional preferred_shard attribute to value.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomGammaSeed(value int64) RandomGammaAttr {
+// value: Index of file to open first if multiple files match
+// `file_pattern`.
+// If not specified, defaults to -1
+func RestorePreferredShard(value int64) RestoreAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["preferred_shard"] = value
 	}
 }
 
-// RandomGammaSeed2 sets the optional seed2 attribute to value.
+// Restores a tensor from checkpoint files.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomGammaSeed2(value int64) RandomGammaAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random values from the Gamma distribution(s) described by alpha.
+// Reads a tensor stored in one or several files. If there are several files (for
+// instance because a tensor was saved as slices), `file_pattern` may contain
+// wildcard symbols (`*` and `?`) in the filename portion only, not in the
+// directory portion.
 //
-// This op uses the algorithm by Marsaglia et al. to acquire samples via
-// transformation-rejection from pairs of uniform and normal random variables.
-// See http://dl.acm.org/citation.cfm?id=358414
+// If a `file_pattern` matches several files, `preferred_shard` can be used to hint
+// in which file the requested tensor is likely to be found. This op will first
+// open the file at index `preferred_shard` in the list of matching files and try
+// to restore tensors from that file.  Only if some tensors or tensor slices are
+// not found in that first file, then the Op opens all the files. Setting
+// `preferred_shard` to match the value passed as the `shard` input
+// of a matching `Save` Op may speed up Restore.  This attribute only affects
+// performance, not correctness.  The default value -1 means files are processed in
+// order.
+//
+// See also `RestoreSlice`.
 //
 // Arguments:
-//	shape: 1-D integer tensor. Shape of independent samples to draw from each
-// distribution described by the shape parameters given in alpha.
-//	alpha: A tensor in which each scalar is a "shape" parameter describing the
-// associated gamma distribution.
+//	file_pattern: Must have a single element. The pattern of the files from
+// which we read the tensor.
+//	tensor_name: Must have a single element. The name of the tensor to be
+// restored.
+//	dt: The type of the tensor to be restored.
 //
-// Returns A tensor with shape `shape + shape(alpha)`. Each slice
-// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
-// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
-func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
+// Returns The restored tensor.
+func Restore(scope *Scope, file_pattern tf.Output, tensor_name tf.Output, dt tf.DataType, optional ...RestoreAttr) (tensor tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"dt": dt}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomGamma",
+		Type: "Restore",
 		Input: []tf.Input{
-			shape, alpha,
+			file_pattern, tensor_name,
 		},
 		Attrs: attrs,
 	}
@@ -16729,48 +16292,35 @@ func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...Ran
 	return op.Output(0)
 }
 
-// RandomUniformIntAttr is an optional argument to RandomUniformInt.
-type RandomUniformIntAttr func(optionalAttr)
+// QuantizedResizeBilinearAttr is an optional argument to QuantizedResizeBilinear.
+type QuantizedResizeBilinearAttr func(optionalAttr)
 
-// RandomUniformIntSeed sets the optional seed attribute to value.
+// QuantizedResizeBilinearAlignCorners sets the optional align_corners attribute to value.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func QuantizedResizeBilinearAlignCorners(value bool) QuantizedResizeBilinearAttr {
 	return func(m optionalAttr) {
-		m["seed"] = value
+		m["align_corners"] = value
 	}
 }
 
-// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
+// Resize quantized `images` to `size` using quantized bilinear interpolation.
 //
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
-	return func(m optionalAttr) {
-		m["seed2"] = value
-	}
-}
-
-// Outputs random integers from a uniform distribution.
+// Input images and output images must be quantized types.
 //
-// The generated values are uniform integers in the range `[minval, maxval)`.
-// The lower bound `minval` is included in the range, while the upper bound
-// `maxval` is excluded.
+// Arguments:
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
 //
-// The random integers are slightly biased unless `maxval - minval` is an exact
-// power of two.  The bias is small for values of `maxval - minval` significantly
-// smaller than the range of the output (either `2^32` or `2^64`).
 //
-// Arguments:
-//	shape: The shape of the output tensor.
-//	minval: 0-D.  Inclusive lower bound on the generated integers.
-//	maxval: 0-D.  Exclusive upper bound on the generated integers.
 //
-// Returns A tensor of the specified shape filled with uniform random integers.
-func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func QuantizedResizeBilinear(scope *Scope, images tf.Output, size tf.Output, min tf.Output, max tf.Output, optional ...QuantizedResizeBilinearAttr) (resized_images tf.Output, out_min tf.Output, out_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16779,153 +16329,192 @@ func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomUniformInt",
+		Type: "QuantizedResizeBilinear",
 		Input: []tf.Input{
-			shape, minval, maxval,
+			images, size, min, max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// SkipgramAttr is an optional argument to Skipgram.
-type SkipgramAttr func(optionalAttr)
-
-// SkipgramWindowSize sets the optional window_size attribute to value.
+// Computes the minimum along segments of a tensor.
 //
-// value: The number of words to predict to the left and right of the target.
-// If not specified, defaults to 5
-func SkipgramWindowSize(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["window_size"] = value
-	}
-}
-
-// SkipgramMinCount sets the optional min_count attribute to value.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
 //
-// value: The minimum number of word occurrences for it to be included in the
-// vocabulary.
-// If not specified, defaults to 5
-func SkipgramMinCount(value int64) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["min_count"] = value
-	}
-}
-
-// SkipgramSubsample sets the optional subsample attribute to value.
+// Computes a tensor such that
+// \\(output_i = \min_j(data_j)\\) where `min` is over `j` such
+// that `segment_ids[j] == i`.
 //
-// value: Threshold for word occurrence. Words that appear with higher
-// frequency will be randomly down-sampled. Set to 0 to disable.
-// If not specified, defaults to 0.001
-func SkipgramSubsample(value float32) SkipgramAttr {
-	return func(m optionalAttr) {
-		m["subsample"] = value
-	}
-}
-
-// Parses a text file and creates a batch of examples.
+// If the min is empty for a given segment ID `i`, `output[i] = 0`.
 //
-// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentMin.png" alt>
+// </div>
 //
 // Arguments:
-//	filename: The corpus's text file name.
-//	batch_size: The size of produced batch.
 //
-// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
-func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentMin(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Skipgram",
-
-		Attrs: attrs,
+		Type: "SegmentMin",
+		Input: []tf.Input{
+			data, segment_ids,
+		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
+	return op.Output(0)
 }
 
-// StringToNumberAttr is an optional argument to StringToNumber.
-type StringToNumberAttr func(optionalAttr)
+// SdcaOptimizerAttr is an optional argument to SdcaOptimizer.
+type SdcaOptimizerAttr func(optionalAttr)
 
-// StringToNumberOutType sets the optional out_type attribute to value.
+// SdcaOptimizerAdaptative sets the optional adaptative attribute to value.
 //
-// value: The numeric type to interpret each string in `string_tensor` as.
-// If not specified, defaults to DT_FLOAT
-func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
+// value: Whether to use Adapative SDCA for the inner loop.
+// If not specified, defaults to false
+func SdcaOptimizerAdaptative(value bool) SdcaOptimizerAttr {
 	return func(m optionalAttr) {
-		m["out_type"] = value
+		m["adaptative"] = value
 	}
 }
 
-// Converts each string in the input Tensor to the specified numeric type.
-//
-// (Note that int32 overflow results in an error while float overflow
-// results in a rounded value.)
+// Distributed version of Stochastic Dual Coordinate Ascent (SDCA) optimizer for
 //
-// Returns A Tensor of the same shape as the input `string_tensor`.
-func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
+// linear models with L1 + L2 regularization. As global optimization objective is
+// strongly-convex, the optimizer optimizes the dual objective at each step. The
+// optimizer applies each update one example at a time. Examples are sampled
+// uniformly, and the optimizer is learning rate free and enjoys linear convergence
+// rate.
+//
+// [Proximal Stochastic Dual Coordinate Ascent](http://arxiv.org/pdf/1211.2717v1.pdf).<br>
+// Shai Shalev-Shwartz, Tong Zhang. 2012
+//
+// $$Loss Objective = \sum f_{i} (wx_{i}) + (l2 / 2) * |w|^2 + l1 * |w|$$
+//
+// [Adding vs. Averaging in Distributed Primal-Dual Optimization](http://arxiv.org/abs/1502.03508).<br>
+// Chenxin Ma, Virginia Smith, Martin Jaggi, Michael I. Jordan,
+// Peter Richtarik, Martin Takac. 2015
+//
+// [Stochastic Dual Coordinate Ascent with Adaptive Probabilities](https://arxiv.org/abs/1502.08053).<br>
+// Dominik Csiba, Zheng Qu, Peter Richtarik. 2015
+//
+// Arguments:
+//	sparse_example_indices: a list of vectors which contain example indices.
+//	sparse_feature_indices: a list of vectors which contain feature indices.
+//	sparse_feature_values: a list of vectors which contains feature value
+// associated with each feature group.
+//	dense_features: a list of matrices which contains the dense feature values.
+//	example_weights: a vector which contains the weight associated with each
+// example.
+//	example_labels: a vector which contains the label/target associated with each
+// example.
+//	sparse_indices: a list of vectors where each value is the indices which has
+// corresponding weights in sparse_weights. This field maybe omitted for the
+// dense approach.
+//	sparse_weights: a list of vectors where each value is the weight associated with
+// a sparse feature group.
+//	dense_weights: a list of vectors where the values are the weights associated
+// with a dense feature group.
+//	example_state_data: a list of vectors containing the example state data.
+//	loss_type: Type of the primal loss. Currently SdcaSolver supports logistic,
+// squared and hinge losses.
+//	l1: Symmetric l1 regularization strength.
+//	l2: Symmetric l2 regularization strength.
+//	num_loss_partitions: Number of partitions of the global loss function.
+//	num_inner_iterations: Number of iterations per mini-batch.
+//
+// Returns a list of vectors containing the updated example state
+// data.a list of vectors where each value is the delta
+// weights associated with a sparse feature group.a list of vectors where the values are the delta
+// weights associated with a dense feature group.
+func SdcaOptimizer(scope *Scope, sparse_example_indices []tf.Output, sparse_feature_indices []tf.Output, sparse_feature_values []tf.Output, dense_features []tf.Output, example_weights tf.Output, example_labels tf.Output, sparse_indices []tf.Output, sparse_weights []tf.Output, dense_weights []tf.Output, example_state_data tf.Output, loss_type string, l1 float32, l2 float32, num_loss_partitions int64, num_inner_iterations int64, optional ...SdcaOptimizerAttr) (out_example_state_data tf.Output, out_delta_sparse_weights []tf.Output, out_delta_dense_weights []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"loss_type": loss_type, "l1": l1, "l2": l2, "num_loss_partitions": num_loss_partitions, "num_inner_iterations": num_inner_iterations}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "StringToNumber",
+		Type: "SdcaOptimizer",
 		Input: []tf.Input{
-			string_tensor,
+			tf.OutputList(sparse_example_indices), tf.OutputList(sparse_feature_indices), tf.OutputList(sparse_feature_values), tf.OutputList(dense_features), example_weights, example_labels, tf.OutputList(sparse_indices), tf.OutputList(sparse_weights), tf.OutputList(dense_weights), example_state_data,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	out_example_state_data = op.Output(idx)
+	if out_delta_sparse_weights, idx, err = makeOutputList(op, idx, "out_delta_sparse_weights"); err != nil {
+		scope.UpdateErr("SdcaOptimizer", err)
+		return
+	}
+	if out_delta_dense_weights, idx, err = makeOutputList(op, idx, "out_delta_dense_weights"); err != nil {
+		scope.UpdateErr("SdcaOptimizer", err)
+		return
+	}
+	return out_example_state_data, out_delta_sparse_weights, out_delta_dense_weights
 }
 
-// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
-type ResourceApplyFtrlV2Attr func(optionalAttr)
+// SparseMatMulAttr is an optional argument to SparseMatMul.
+type SparseMatMulAttr func(optionalAttr)
 
-// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var and accum tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
+// SparseMatMulTransposeA sets the optional transpose_a attribute to value.
 // If not specified, defaults to false
-func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
+func SparseMatMulTransposeA(value bool) SparseMatMulAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["transpose_a"] = value
 	}
 }
 
-// Update '*var' according to the Ftrl-proximal scheme.
-//
-// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
-// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
-// linear += grad_with_shrinkage +
-//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
-// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
-// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
-// accum = accum_new
-//
-// Arguments:
-//	var_: Should be from a Variable().
-//	accum: Should be from a Variable().
-//	linear: Should be from a Variable().
-//	grad: The gradient.
-//	lr: Scaling factor. Must be a scalar.
-//	l1: L1 regulariation. Must be a scalar.
-//	l2: L2 shrinkage regulariation. Must be a scalar.
+// SparseMatMulTransposeB sets the optional transpose_b attribute to value.
+// If not specified, defaults to false
+func SparseMatMulTransposeB(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_b"] = value
+	}
+}
+
+// SparseMatMulAIsSparse sets the optional a_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulAIsSparse(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["a_is_sparse"] = value
+	}
+}
+
+// SparseMatMulBIsSparse sets the optional b_is_sparse attribute to value.
+// If not specified, defaults to false
+func SparseMatMulBIsSparse(value bool) SparseMatMulAttr {
+	return func(m optionalAttr) {
+		m["b_is_sparse"] = value
+	}
+}
+
+// Multiply matrix "a" by matrix "b".
 //
-//	lr_power: Scaling factor. Must be a scalar.
+// The inputs must be two-dimensional matrices and the inner dimension of "a" must
+// match the outer dimension of "b". This op is optimized for the case where at
+// least one of "a" or "b" is sparse. The breakeven for using this versus a dense
+// matrix multiply on one platform was 30% zero values in the sparse matrix.
 //
-// Returns the created operation.
-func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
+// The gradient computation of this operation will only take advantage of sparsity
+// in the input gradient when that gradient comes from a Relu.
+func SparseMatMul(scope *Scope, a tf.Output, b tf.Output, optional ...SparseMatMulAttr) (product tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -16934,64 +16523,73 @@ func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyFtrlV2",
+		Type: "SparseMatMul",
 		Input: []tf.Input{
-			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
+			a, b,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// TruncatedNormalAttr is an optional argument to TruncatedNormal.
-type TruncatedNormalAttr func(optionalAttr)
-
-// TruncatedNormalSeed sets the optional seed attribute to value.
+// Computes the power of one value to another.
 //
-// value: If either `seed` or `seed2` are set to be non-zero, the random number
-// generator is seeded by the given seed.  Otherwise, it is seeded by a
-// random seed.
-// If not specified, defaults to 0
-func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
-	return func(m optionalAttr) {
-		m["seed"] = value
+// Given a tensor `x` and a tensor `y`, this operation computes \\(x^y\\) for
+// corresponding elements in `x` and `y`. For example:
+//
+// ```
+// # tensor 'x' is [[2, 2]], [3, 3]]
+// # tensor 'y' is [[8, 16], [2, 3]]
+// tf.pow(x, y) ==> [[256, 65536], [9, 27]]
+// ```
+func Pow(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Pow",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
-//
-// value: A second seed to avoid seed collision.
-// If not specified, defaults to 0
-func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
+// ShapeAttr is an optional argument to Shape.
+type ShapeAttr func(optionalAttr)
+
+// ShapeOutType sets the optional out_type attribute to value.
+// If not specified, defaults to DT_INT32
+func ShapeOutType(value tf.DataType) ShapeAttr {
 	return func(m optionalAttr) {
-		m["seed2"] = value
+		m["out_type"] = value
 	}
 }
 
-// Outputs random values from a truncated normal distribution.
+// Returns the shape of a tensor.
 //
-// The generated values follow a normal distribution with mean 0 and standard
-// deviation 1, except that values whose magnitude is more than 2 standard
-// deviations from the mean are dropped and re-picked.
+// This operation returns a 1-D integer tensor representing the shape of `input`.
 //
-// Arguments:
-//	shape: The shape of the output tensor.
-//	dtype: The type of the output.
+// For example:
 //
-// Returns A tensor of the specified shape filled with random truncated normal
-// values.
-func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// shape(t) ==> [2, 2, 3]
+// ```
+func Shape(scope *Scope, input tf.Output, optional ...ShapeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TruncatedNormal",
+		Type: "Shape",
 		Input: []tf.Input{
-			shape,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -16999,49 +16597,82 @@ func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional
 	return op.Output(0)
 }
 
-// RandomShuffleAttr is an optional argument to RandomShuffle.
-type RandomShuffleAttr func(optionalAttr)
+// Computes fingerprints of the input strings.
+//
+// Arguments:
+//	input: vector of strings to compute fingerprints on.
+//
+// Returns a (N,2) shaped matrix where N is the number of elements in the input
+// vector. Each row contains the low and high parts of the fingerprint.
+func SdcaFprint(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SdcaFprint",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
 
-// RandomShuffleSeed sets the optional seed attribute to value.
+// RandomPoissonV2Attr is an optional argument to RandomPoissonV2.
+type RandomPoissonV2Attr func(optionalAttr)
+
+// RandomPoissonV2Seed sets the optional seed attribute to value.
 //
 // value: If either `seed` or `seed2` are set to be non-zero, the random number
 // generator is seeded by the given seed.  Otherwise, it is seeded by a
 // random seed.
 // If not specified, defaults to 0
-func RandomShuffleSeed(value int64) RandomShuffleAttr {
+func RandomPoissonV2Seed(value int64) RandomPoissonV2Attr {
 	return func(m optionalAttr) {
 		m["seed"] = value
 	}
 }
 
-// RandomShuffleSeed2 sets the optional seed2 attribute to value.
+// RandomPoissonV2Seed2 sets the optional seed2 attribute to value.
 //
 // value: A second seed to avoid seed collision.
 // If not specified, defaults to 0
-func RandomShuffleSeed2(value int64) RandomShuffleAttr {
+func RandomPoissonV2Seed2(value int64) RandomPoissonV2Attr {
 	return func(m optionalAttr) {
 		m["seed2"] = value
 	}
 }
 
-// Randomly shuffles a tensor along its first dimension.
+// RandomPoissonV2Dtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_INT64
+func RandomPoissonV2Dtype(value tf.DataType) RandomPoissonV2Attr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Outputs random values from the Poisson distribution(s) described by rate.
 //
-//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
-//   to one and only one `output[i]`. For example, a mapping that might occur for a
-//   3x2 tensor is:
+// This op uses two algorithms, depending on rate. If rate >= 10, then
+// the algorithm by Hormann is used to acquire samples via
+// transformation-rejection.
+// See http://www.sciencedirect.com/science/article/pii/0167668793909974.
 //
-// ```
-// [[1, 2],       [[5, 6],
-//  [3, 4],  ==>   [1, 2],
-//  [5, 6]]        [3, 4]]
-// ```
+// Otherwise, Knuth's algorithm is used to acquire samples via multiplying uniform
+// random variables.
+// See Donald E. Knuth (1969). Seminumerical Algorithms. The Art of Computer
+// Programming, Volume 2. Addison Wesley
 //
 // Arguments:
-//	value: The tensor to be shuffled.
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in rate.
+//	rate: A tensor in which each scalar is a "rate" parameter describing the
+// associated poisson distribution.
 //
-// Returns A tensor of same shape and type as `value`, shuffled along its first
-// dimension.
-func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
+// Returns A tensor with shape `shape + shape(rate)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `rate[i0, i1, ...iN]`.
+func RandomPoissonV2(scope *Scope, shape tf.Output, rate tf.Output, optional ...RandomPoissonV2Attr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17050,9 +16681,9 @@ func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr)
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "RandomShuffle",
+		Type: "RandomPoissonV2",
 		Input: []tf.Input{
-			value,
+			shape, rate,
 		},
 		Attrs: attrs,
 	}
@@ -17060,293 +16691,259 @@ func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr)
 	return op.Output(0)
 }
 
-// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
-type OrderedMapIncompleteSizeAttr func(optionalAttr)
+// MatrixTriangularSolveAttr is an optional argument to MatrixTriangularSolve.
+type MatrixTriangularSolveAttr func(optionalAttr)
 
-// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
+// MatrixTriangularSolveLower sets the optional lower attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
+// value: Boolean indicating whether the innermost matrices in `matrix` are
+// lower or upper triangular.
+// If not specified, defaults to true
+func MatrixTriangularSolveLower(value bool) MatrixTriangularSolveAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["lower"] = value
 	}
 }
 
-// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
+// MatrixTriangularSolveAdjoint sets the optional adjoint attribute to value.
 //
-// REQUIRES: value >= 0
-func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["memory_limit"] = value
-	}
-}
-
-// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
+// value: Boolean indicating whether to solve with `matrix` or its (block-wise)
+//          adjoint.
+//
+// @compatibility(numpy)
+// Equivalent to np.linalg.triangular_solve
+// @end_compatibility
+// If not specified, defaults to false
+func MatrixTriangularSolveAdjoint(value bool) MatrixTriangularSolveAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["adjoint"] = value
 	}
 }
 
-// Op returns the number of incomplete elements in the underlying container.
-func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
+// Solves systems of linear equations with upper or lower triangular matrices by
+//
+// backsubstitution.
+//
+// `matrix` is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions form
+// square matrices. If `lower` is `True` then the strictly upper triangular part
+// of each inner-most matrix is assumed to be zero and not accessed.
+// If `lower` is False then the strictly lower triangular part of each inner-most
+// matrix is assumed to be zero and not accessed.
+// `rhs` is a tensor of shape `[..., M, K]`.
+//
+// The output is a tensor of shape `[..., M, K]`. If `adjoint` is
+// `True` then the innermost matrices in `output` satisfy matrix equations
+// `matrix[..., :, :] * output[..., :, :] = rhs[..., :, :]`.
+// If `adjoint` is `False` then the strictly then the  innermost matrices in
+// `output` satisfy matrix equations
+// `adjoint(matrix[..., i, k]) * output[..., k, j] = rhs[..., i, j]`.
+//
+// Arguments:
+//	matrix: Shape is `[..., M, M]`.
+//	rhs: Shape is `[..., M, K]`.
+//
+// Returns Shape is `[..., M, K]`.
+func MatrixTriangularSolve(scope *Scope, matrix tf.Output, rhs tf.Output, optional ...MatrixTriangularSolveAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "OrderedMapIncompleteSize",
-
+		Type: "MatrixTriangularSolve",
+		Input: []tf.Input{
+			matrix, rhs,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DecodeRawAttr is an optional argument to DecodeRaw.
-type DecodeRawAttr func(optionalAttr)
-
-// DecodeRawLittleEndian sets the optional little_endian attribute to value.
-//
-// value: Whether the input `bytes` are in little-endian order.
-// Ignored for `out_type` values that are stored in a single byte like
-// `uint8`.
-// If not specified, defaults to true
-func DecodeRawLittleEndian(value bool) DecodeRawAttr {
-	return func(m optionalAttr) {
-		m["little_endian"] = value
-	}
-}
-
-// Reinterpret the bytes of a string as a vector of numbers.
-//
-// Arguments:
-//	bytes: All the elements must have the same length.
-//
-//
-// Returns A Tensor with one more dimension than the input `bytes`.  The
-// added dimension will have size equal to the length of the elements
-// of `bytes` divided by the number of bytes to represent `out_type`.
-func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...DecodeRawAttr) (output tf.Output) {
+// Computes inverse hyperbolic sine of x element-wise.
+func Asinh(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"out_type": out_type}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DecodeRaw",
+		Type: "Asinh",
 		Input: []tf.Input{
-			bytes,
+			x,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Copy a tensor setting everything outside a central band in each innermost matrix
-//
-// to zero.
-//
-// The `band` part is computed as follows:
-// Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
-// tensor with the same shape where
-//
-// `band[i, j, k, ..., m, n] = in_band(m, n) * input[i, j, k, ..., m, n]`.
-//
-// The indicator function
-//
-// `in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) &&
-//                  (num_upper < 0 || (n-m) <= num_upper)`.
-//
-// For example:
-//
-// ```
-// # if 'input' is [[ 0,  1,  2, 3]
-//                  [-1,  0,  1, 2]
-//                  [-2, -1,  0, 1]
-//                  [-3, -2, -1, 0]],
-//
-// tf.matrix_band_part(input, 1, -1) ==> [[ 0,  1,  2, 3]
-//                                        [-1,  0,  1, 2]
-//                                        [ 0, -1,  0, 1]
-//                                        [ 0,  0, -1, 0]],
-//
-// tf.matrix_band_part(input, 2, 1) ==> [[ 0,  1,  0, 0]
-//                                       [-1,  0,  1, 0]
-//                                       [-2, -1,  0, 1]
-//                                       [ 0, -2, -1, 0]]
-// ```
-//
-// Useful special cases:
-//
-// ```
-//  tf.matrix_band_part(input, 0, -1) ==> Upper triangular part.
-//  tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.
-//  tf.matrix_band_part(input, 0, 0) ==> Diagonal.
-// ```
+// Creates a dataset with a range of values. Corresponds to python's xrange.
 //
 // Arguments:
-//	input: Rank `k` tensor.
-//	num_lower: 0-D tensor. Number of subdiagonals to keep. If negative, keep entire
-// lower triangle.
-//	num_upper: 0-D tensor. Number of superdiagonals to keep. If negative, keep
-// entire upper triangle.
+//	start: corresponds to start in python's xrange().
+//	stop: corresponds to stop in python's xrange().
+//	step: corresponds to step in python's xrange().
 //
-// Returns Rank `k` tensor of the same shape as input. The extracted banded tensor.
-func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_upper tf.Output) (band tf.Output) {
+//
+func RangeDataset(scope *Scope, start tf.Output, stop tf.Output, step tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "MatrixBandPart",
+		Type: "RangeDataset",
 		Input: []tf.Input{
-			input, num_lower, num_upper,
+			start, stop, step,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
-type QuantizedMatMulAttr func(optionalAttr)
-
-// QuantizedMatMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
-}
+// DepthwiseConv2dNativeBackpropInputAttr is an optional argument to DepthwiseConv2dNativeBackpropInput.
+type DepthwiseConv2dNativeBackpropInputAttr func(optionalAttr)
 
-// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
+// DepthwiseConv2dNativeBackpropInputDataFormat sets the optional data_format attribute to value.
 //
-// value: If true, `a` is transposed before multiplication.
-// If not specified, defaults to false
-func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, height, width, channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, channels, height, width].
+// If not specified, defaults to "NHWC"
+func DepthwiseConv2dNativeBackpropInputDataFormat(value string) DepthwiseConv2dNativeBackpropInputAttr {
 	return func(m optionalAttr) {
-		m["transpose_a"] = value
+		m["data_format"] = value
 	}
 }
 
-// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
+// DepthwiseConv2dNativeBackpropInputDilations sets the optional dilations attribute to value.
 //
-// value: If true, `b` is transposed before multiplication.
-// If not specified, defaults to false
-func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
+// value: 1-D tensor of length 4.  The dilation factor for each dimension of
+// `input`. If set to k > 1, there will be k-1 skipped cells between each filter
+// element on that dimension. The dimension order is determined by the value of
+// `data_format`, see above for details. Dilations in the batch and depth
+// dimensions must be 1.
+// If not specified, defaults to <i:1 i:1 i:1 i:1 >
+func DepthwiseConv2dNativeBackpropInputDilations(value []int64) DepthwiseConv2dNativeBackpropInputAttr {
 	return func(m optionalAttr) {
-		m["transpose_b"] = value
+		m["dilations"] = value
 	}
 }
 
-// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
-//
-// value: The type of output produced by activation function
-// following this operation.
-// If not specified, defaults to DT_QUINT8
-func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
-	return func(m optionalAttr) {
-		m["Tactivation"] = value
-	}
-}
-
-// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
-//
-// The inputs must be two-dimensional matrices and the inner dimension of
-// `a` (after being transposed if `transpose_a` is non-zero) must match the
-// outer dimension of `b` (after being transposed if `transposed_b` is
-// non-zero).
+// Computes the gradients of depthwise convolution with respect to the input.
 //
 // Arguments:
-//	a: Must be a two-dimensional tensor.
-//	b: Must be a two-dimensional tensor.
-//	min_a: The float value that the lowest quantized `a` value represents.
-//	max_a: The float value that the highest quantized `a` value represents.
-//	min_b: The float value that the lowest quantized `b` value represents.
-//	max_b: The float value that the highest quantized `b` value represents.
+//	input_sizes: An integer vector representing the shape of `input`, based
+// on `data_format`.  For example, if `data_format` is 'NHWC' then
+//  `input` is a 4-D `[batch, height, width, channels]` tensor.
+//	filter: 4-D with shape
+// `[filter_height, filter_width, in_channels, depthwise_multiplier]`.
+//	out_backprop: 4-D with shape  based on `data_format`.
+// For example, if `data_format` is 'NHWC' then
+// out_backprop shape is `[batch, out_height, out_width, out_channels]`.
+// Gradients w.r.t. the output of the convolution.
+//	strides: The stride of the sliding window for each dimension of the input
+// of the convolution.
+//	padding: The type of padding algorithm to use.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
-func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
+// Returns 4-D with shape according to `data_format`.  For example, if
+// `data_format` is 'NHWC', output shape is `[batch, in_height,
+// in_width, in_channels]`.  Gradient w.r.t. the input of the
+// convolution.
+func DepthwiseConv2dNativeBackpropInput(scope *Scope, input_sizes tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, padding string, optional ...DepthwiseConv2dNativeBackpropInputAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMatMul",
+		Type: "DepthwiseConv2dNativeBackpropInput",
 		Input: []tf.Input{
-			a, b, min_a, max_a, min_b, max_b,
+			input_sizes, filter, out_backprop,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Does nothing. Serves as a control trigger for scheduling.
+// Adds sparse updates to the variable referenced by `resource`.
 //
-// Only useful as a placeholder for control edges.
+// This operation computes
+//
+//     # Scalar indices
+//     ref[indices, ...] += updates[...]
+//
+//     # Vector indices (for each i)
+//     ref[indices[i], ...] += updates[i, ...]
+//
+//     # High rank indices (for each i, ..., j)
+//     ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+//
+// Duplicate entries are handled correctly: if multiple `indices` reference
+// the same location, their contributions add.
+//
+// Requires `updates.shape = indices.shape + ref.shape[1:]`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+// </div>
+//
+// Arguments:
+//	resource: Should be from a `Variable` node.
+//	indices: A tensor of indices into the first dimension of `ref`.
+//	updates: A tensor of updated values to add to `ref`.
 //
 // Returns the created operation.
-func ControlTrigger(scope *Scope) (o *tf.Operation) {
+func ResourceScatterAdd(scope *Scope, resource tf.Output, indices tf.Output, updates tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ControlTrigger",
+		Type: "ResourceScatterAdd",
+		Input: []tf.Input{
+			resource, indices, updates,
+		},
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Batch normalization.
+// Says whether the targets are in the top `K` predictions.
 //
-// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+// This outputs a `batch_size` bool array, an entry `out[i]` is `true` if the
+// prediction for the target class is among the top `k` predictions among
+// all predictions for example `i`. Note that the behavior of `InTopK` differs
+// from the `TopK` op in its handling of ties; if multiple classes have the
+// same prediction value and straddle the top-`k` boundary, all of those
+// classes are considered to be in the top `k`.
 //
-// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+// More formally, let
+//
+//   \\(predictions_i\\) be the predictions for all classes for example `i`,
+//   \\(targets_i\\) be the target class for example `i`,
+//   \\(out_i\\) be the output for example `i`,
+//
+// $$out_i = predictions_{i, targets_i} \in TopKIncludingTies(predictions_i)$$
 //
 // Arguments:
-//	t: A 4D input Tensor.
-//	m: A 1D mean Tensor with size matching the last dimension of t.
-// This is the first output from tf.nn.moments,
-// or a saved moving average thereof.
-//	v: A 1D variance Tensor with size matching the last dimension of t.
-// This is the second output from tf.nn.moments,
-// or a saved moving average thereof.
-//	beta: A 1D beta Tensor with size matching the last dimension of t.
-// An offset to be added to the normalized tensor.
-//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
-// If "scale_after_normalization" is true, this tensor will be multiplied
-// with the normalized tensor.
-//	variance_epsilon: A small float number to avoid dividing by 0.
-//	scale_after_normalization: A bool indicating whether the resulted tensor
-// needs to be multiplied with gamma.
-func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
+//	predictions: A `batch_size` x `classes` tensor.
+//	targets: A `batch_size` vector of class ids.
+//	k: Number of top elements to look at for computing precision.
+//
+// Returns Computed Precision at `k` as a `bool Tensor`.
+func InTopK(scope *Scope, predictions tf.Output, targets tf.Output, k int64) (precision tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
+	attrs := map[string]interface{}{"k": k}
 	opspec := tf.OpSpec{
-		Type: "BatchNormWithGlobalNormalization",
+		Type: "InTopK",
 		Input: []tf.Input{
-			t, m, v, beta, gamma,
+			predictions, targets,
 		},
 		Attrs: attrs,
 	}
@@ -17354,106 +16951,113 @@ func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v
 	return op.Output(0)
 }
 
-// Deprecated. Use TensorArrayReadV3
+// Computes the gradient for the inverse of `x` wrt its input.
 //
-// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
-func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func ReciprocalGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "TensorArrayReadV2",
+		Type: "ReciprocalGrad",
 		Input: []tf.Input{
-			handle, index, flow_in,
+			y, dy,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizedMulAttr is an optional argument to QuantizedMul.
-type QuantizedMulAttr func(optionalAttr)
-
-// QuantizedMulToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
+// Returns the min of x and y (i.e. x < y ? x : y) element-wise.
+//
+// *NOTE*: `Minimum` supports broadcasting. More about broadcasting
+// [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func Minimum(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Minimum",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Returns x * y element-wise, working on quantized buffers.
-//
-// Arguments:
+// Returns the element-wise sum of a list of tensors.
 //
+// `tf.accumulate_n_v2` performs the same operation as `tf.add_n`, but does not
+// wait for all of its inputs to be ready before beginning to sum. This can
+// save memory if inputs are ready at different times, since minimum temporary
+// storage is proportional to the output size rather than the inputs size.
 //
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
+// Unlike the original `accumulate_n`, `accumulate_n_v2` is differentiable.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+// Returns a `Tensor` of same shape and type as the elements of `inputs`.
 //
-// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
+// Arguments:
+//	inputs: A list of `Tensor` objects, each with same shape and type.
+//	shape: Shape of elements of `inputs`.
+func AccumulateNV2(scope *Scope, inputs []tf.Output, shape tf.Shape) (sum tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "QuantizedMul",
+		Type: "AccumulateNV2",
 		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
+			tf.OutputList(inputs),
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
-}
-
-// QuantizedAddAttr is an optional argument to QuantizedAdd.
-type QuantizedAddAttr func(optionalAttr)
-
-// QuantizedAddToutput sets the optional Toutput attribute to value.
-// If not specified, defaults to DT_QINT32
-func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
-	return func(m optionalAttr) {
-		m["Toutput"] = value
-	}
+	return op.Output(0)
 }
 
-// Returns x + y element-wise, working on quantized buffers.
+// Convert the quantized 'input' tensor into a lower-precision 'output', using the
 //
-// Arguments:
+// actual distribution of the values to maximize the usage of the lower bit depth
+// and adjusting the output min and max ranges accordingly.
 //
+// [input_min, input_max] are scalar floats that specify the range for the float
+// interpretation of the 'input' data. For example, if input_min is -1.0f and
+// input_max is 1.0f, and we are dealing with quint16 quantized data, then a 0
+// value in the 16-bit data should be interpreted as -1.0f, and a 65535 means 1.0f.
 //
-//	min_x: The float value that the lowest quantized `x` value represents.
-//	max_x: The float value that the highest quantized `x` value represents.
-//	min_y: The float value that the lowest quantized `y` value represents.
-//	max_y: The float value that the highest quantized `y` value represents.
+// This operator tries to squeeze as much precision as possible into an output with
+// a lower bit depth by calculating the actual min and max values found in the
+// data. For example, maybe that quint16 input has no values lower than 16,384 and
+// none higher than 49,152. That means only half the range is actually needed, all
+// the float interpretations are between -0.5f and 0.5f, so if we want to compress
+// the data into a quint8 output, we can use that range rather than the theoretical
+// -1.0f to 1.0f that is suggested by the input min and max.
 //
-// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+// In practice, this is most useful for taking output from operations like
+// QuantizedMatMul that can produce higher bit-depth outputs than their inputs and
+// may have large potential output ranges, but in practice have a distribution of
+// input values that only uses a small fraction of the possible range. By feeding
+// that output into this operator, we can reduce it from 32 bits down to 8 with
+// minimal loss of accuracy.
 //
-// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
-// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
-func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
+// Arguments:
+//
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//	out_type: The type of the output. Should be a lower bit depth than Tinput.
+//
+// Returns The float value that the minimum quantized output value represents.The float value that the maximum quantized output value represents.
+func QuantizeDownAndShrinkRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, out_type tf.DataType) (output tf.Output, output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"out_type": out_type}
 	opspec := tf.OpSpec{
-		Type: "QuantizedAdd",
+		Type: "QuantizeDownAndShrinkRange",
 		Input: []tf.Input{
-			x, y, min_x, max_x, min_y, max_y,
+			input, input_min, input_max,
 		},
 		Attrs: attrs,
 	}
@@ -17461,65 +17065,47 @@ func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// MfccAttr is an optional argument to Mfcc.
-type MfccAttr func(optionalAttr)
+// RandomGammaAttr is an optional argument to RandomGamma.
+type RandomGammaAttr func(optionalAttr)
 
-// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
+// RandomGammaSeed sets the optional seed attribute to value.
 //
-// value: The highest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 4000
-func MfccUpperFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["upper_frequency_limit"] = value
-	}
-}
-
-// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
-//
-// value: The lowest frequency to use when calculating the
-// ceptstrum.
-// If not specified, defaults to 20
-func MfccLowerFrequencyLimit(value float32) MfccAttr {
-	return func(m optionalAttr) {
-		m["lower_frequency_limit"] = value
-	}
-}
-
-// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
-//
-// value: Resolution of the Mel bank used internally.
-// If not specified, defaults to 40
-func MfccFilterbankChannelCount(value int64) MfccAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomGammaSeed(value int64) RandomGammaAttr {
 	return func(m optionalAttr) {
-		m["filterbank_channel_count"] = value
+		m["seed"] = value
 	}
 }
 
-// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
+// RandomGammaSeed2 sets the optional seed2 attribute to value.
 //
-// value: How many output channels to produce per time slice.
-// If not specified, defaults to 13
-func MfccDctCoefficientCount(value int64) MfccAttr {
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomGammaSeed2(value int64) RandomGammaAttr {
 	return func(m optionalAttr) {
-		m["dct_coefficient_count"] = value
+		m["seed2"] = value
 	}
 }
 
-// Transforms a spectrogram into a form that's useful for speech recognition.
+// Outputs random values from the Gamma distribution(s) described by alpha.
 //
-// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
-// been effective as an input feature for machine learning. They are created by
-// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
-// higher frequencies that are less significant to the human ear. They have a long
-// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
-// is a good resource to learn more.
+// This op uses the algorithm by Marsaglia et al. to acquire samples via
+// transformation-rejection from pairs of uniform and normal random variables.
+// See http://dl.acm.org/citation.cfm?id=358414
 //
 // Arguments:
-//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
-// set to true.
-//	sample_rate: How many samples per second the source audio used.
-func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
+//	shape: 1-D integer tensor. Shape of independent samples to draw from each
+// distribution described by the shape parameters given in alpha.
+//	alpha: A tensor in which each scalar is a "shape" parameter describing the
+// associated gamma distribution.
+//
+// Returns A tensor with shape `shape + shape(alpha)`. Each slice
+// `[:, ..., :, i0, i1, ...iN]` contains the samples drawn for
+// `alpha[i0, i1, ...iN]`. The dtype of the output matches the dtype of alpha.
+func RandomGamma(scope *Scope, shape tf.Output, alpha tf.Output, optional ...RandomGammaAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17528,9 +17114,9 @@ func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional .
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Mfcc",
+		Type: "RandomGamma",
 		Input: []tf.Input{
-			spectrogram, sample_rate,
+			shape, alpha,
 		},
 		Attrs: attrs,
 	}
@@ -17538,202 +17124,245 @@ func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional .
 	return op.Output(0)
 }
 
-// Given a quantized tensor described by (input, input_min, input_max), outputs a
-//
-// range that covers the actual values present in that tensor.  This op is
-// typically used to produce the requested_output_min and requested_output_max for
-// Requantize.
-//
-// Arguments:
-//
-//	input_min: The float value that the minimum quantized input value represents.
-//	input_max: The float value that the maximum quantized input value represents.
-//
-// Returns The computed min output.the computed max output.
-func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RequantizationRange",
-		Input: []tf.Input{
-			input, input_min, input_max,
-		},
+// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
+type QuantizeAndDequantizeAttr func(optionalAttr)
+
+// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
+	return func(m optionalAttr) {
+		m["signed_input"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
 }
 
-// MapPeekAttr is an optional argument to MapPeek.
-type MapPeekAttr func(optionalAttr)
-
-// MapPeekCapacity sets the optional capacity attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekCapacity(value int64) MapPeekAttr {
+// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
+// If not specified, defaults to 8
+func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["capacity"] = value
+		m["num_bits"] = value
 	}
 }
 
-// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
-// If not specified, defaults to 0
-//
-// REQUIRES: value >= 0
-func MapPeekMemoryLimit(value int64) MapPeekAttr {
+// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
+// If not specified, defaults to false
+func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["memory_limit"] = value
+		m["range_given"] = value
 	}
 }
 
-// MapPeekContainer sets the optional container attribute to value.
-// If not specified, defaults to ""
-func MapPeekContainer(value string) MapPeekAttr {
+// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["container"] = value
+		m["input_min"] = value
 	}
 }
 
-// MapPeekSharedName sets the optional shared_name attribute to value.
-// If not specified, defaults to ""
-func MapPeekSharedName(value string) MapPeekAttr {
+// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
+// If not specified, defaults to 0
+func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["input_max"] = value
 	}
 }
 
-// Op peeks at the values at the specified key.  If the
+// Use QuantizeAndDequantizeV2 instead.
 //
-// underlying container does not contain this key
-// this op will block until it does.
-func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
+// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
+func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"dtypes": dtypes}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapPeek",
+		Type: "QuantizeAndDequantize",
 		Input: []tf.Input{
-			key, indices,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapPeek", err)
-		return
-	}
-	return values
+	return op.Output(0)
 }
 
-// Looks up keys in a table, outputs the corresponding values.
+// Returns locations of nonzero / true values in a tensor.
 //
-// The tensor `keys` must of the same type as the keys of the table.
-// The output `values` is of the type of the table values.
+// This operation returns the coordinates of true elements in `condition`. The
+// coordinates are returned in a 2-D tensor where the first dimension (rows)
+// represents the number of true elements, and the second dimension (columns)
+// represents the coordinates of the true elements. Keep in mind, the shape of
+// the output tensor can vary depending on how many true values there are in
+// `condition`. Indices are output in row-major order.
 //
-// The scalar `default_value` is the value output for keys not present in the
-// table. It must also be of the same type as the table values.
+// For example:
 //
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
+// ```
+// # 'input' tensor is [[True, False]
+// #                    [True, False]]
+// # 'input' has two true values, so output has two coordinates.
+// # 'input' has rank of 2, so coordinates have two indices.
+// where(input) ==> [[0, 0],
+//                   [1, 0]]
+//
+// # `condition` tensor is [[[True, False]
+// #                     [True, False]]
+// #                    [[False, True]
+// #                     [False, True]]
+// #                    [[False, False]
+// #                     [False, True]]]
+// # 'input' has 5 true values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 //
+// # `condition` tensor is [[[1.5,  0.0]
+// #                     [-0.5, 0.0]]
+// #                    [[0.0,  0.25]
+// #                     [0.0,  0.75]]
+// #                    [[0.0,  0.0]
+// #                     [0.0,  0.01]]]
+// # 'input' has 5 nonzero values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
 //
-// Returns Same shape as `keys`.  Values found in the table, or `default_values`
-// for missing keys.
-func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
+// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
+// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
+// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
+// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
+// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
+// # 'input' has rank of 3, so coordinates have three indices.
+// where(input) ==> [[0, 0, 0],
+//                   [0, 1, 0],
+//                   [1, 0, 1],
+//                   [1, 1, 1],
+//                   [2, 1, 1]]
+// ```
+func Where(scope *Scope, condition tf.Output) (index tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableFindV2",
+		Type: "Where",
 		Input: []tf.Input{
-			table_handle, keys, default_value,
+			condition,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Bucketizes 'input' based on 'boundaries'.
+// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
+type QueueDequeueV2Attr func(optionalAttr)
+
+// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// For example, if the inputs are
-//     boundaries = [0, 10, 100]
-//     input = [[-5, 10000]
-//              [150,   10]
-//              [5,    100]]
+// value: If the queue is empty, this operation will block for up to
+// timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
+	return func(m optionalAttr) {
+		m["timeout_ms"] = value
+	}
+}
+
+// Dequeues a tuple of one or more tensors from the given queue.
 //
-// then the output will be
-//     output = [[0, 3]
-//               [3, 2]
-//               [1, 3]]
+// This operation has k outputs, where k is the number of components
+// in the tuples stored in the given queue, and output i is the ith
+// component of the dequeued tuple.
 //
-// Arguments:
-//	input: Any shape of Tensor contains with int or float type.
-//	boundaries: A sorted list of floats gives the boundary of the buckets.
+// N.B. If the queue is empty, this operation will block until an element
+// has been dequeued (or 'timeout_ms' elapses, if specified).
 //
-// Returns Same shape with 'input', each value of input replaced with bucket index.
+// Arguments:
+//	handle: The handle to a queue.
+//	component_types: The type of each component in a tuple.
 //
-// @compatibility(numpy)
-// Equivalent to np.digitize.
-// @end_compatibility
-func Bucketize(scope *Scope, input tf.Output, boundaries []float32) (output tf.Output) {
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"boundaries": boundaries}
+	attrs := map[string]interface{}{"component_types": component_types}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Bucketize",
+		Type: "QueueDequeueV2",
 		Input: []tf.Input{
-			input,
+			handle,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueV2", err)
+		return
+	}
+	return components
 }
 
-// EncodePngAttr is an optional argument to EncodePng.
-type EncodePngAttr func(optionalAttr)
+// RandomUniformIntAttr is an optional argument to RandomUniformInt.
+type RandomUniformIntAttr func(optionalAttr)
 
-// EncodePngCompression sets the optional compression attribute to value.
+// RandomUniformIntSeed sets the optional seed attribute to value.
 //
-// value: Compression level.
-// If not specified, defaults to -1
-func EncodePngCompression(value int64) EncodePngAttr {
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomUniformIntSeed(value int64) RandomUniformIntAttr {
 	return func(m optionalAttr) {
-		m["compression"] = value
+		m["seed"] = value
 	}
 }
 
-// PNG-encode an image.
+// RandomUniformIntSeed2 sets the optional seed2 attribute to value.
 //
-// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
-// where `channels` is:
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomUniformIntSeed2(value int64) RandomUniformIntAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random integers from a uniform distribution.
 //
-// *   1: for grayscale.
-// *   2: for grayscale + alpha.
-// *   3: for RGB.
-// *   4: for RGBA.
+// The generated values are uniform integers in the range `[minval, maxval)`.
+// The lower bound `minval` is included in the range, while the upper bound
+// `maxval` is excluded.
 //
-// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
-// default or a value from 0 to 9.  9 is the highest compression level, generating
-// the smallest output, but is slower.
+// The random integers are slightly biased unless `maxval - minval` is an exact
+// power of two.  The bias is small for values of `maxval - minval` significantly
+// smaller than the range of the output (either `2^32` or `2^64`).
 //
 // Arguments:
-//	image: 3-D with shape `[height, width, channels]`.
+//	shape: The shape of the output tensor.
+//	minval: 0-D.  Inclusive lower bound on the generated integers.
+//	maxval: 0-D.  Exclusive upper bound on the generated integers.
 //
-// Returns 0-D. PNG-encoded image.
-func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
+// Returns A tensor of the specified shape filled with uniform random integers.
+func RandomUniformInt(scope *Scope, shape tf.Output, minval tf.Output, maxval tf.Output, optional ...RandomUniformIntAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17742,9 +17371,9 @@ func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (conten
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "EncodePng",
+		Type: "RandomUniformInt",
 		Input: []tf.Input{
-			image,
+			shape, minval, maxval,
 		},
 		Attrs: attrs,
 	}
@@ -17752,91 +17381,143 @@ func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (conten
 	return op.Output(0)
 }
 
-// Updates the table to associates keys with values.
-//
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
+// SkipgramAttr is an optional argument to Skipgram.
+type SkipgramAttr func(optionalAttr)
+
+// SkipgramWindowSize sets the optional window_size attribute to value.
 //
-// Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
+// value: The number of words to predict to the left and right of the target.
+// If not specified, defaults to 5
+func SkipgramWindowSize(value int64) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["window_size"] = value
+	}
+}
+
+// SkipgramMinCount sets the optional min_count attribute to value.
 //
-// Returns the created operation.
-func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
+// value: The minimum number of word occurrences for it to be included in the
+// vocabulary.
+// If not specified, defaults to 5
+func SkipgramMinCount(value int64) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["min_count"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "LookupTableInsertV2",
-		Input: []tf.Input{
-			table_handle, keys, values,
-		},
+}
+
+// SkipgramSubsample sets the optional subsample attribute to value.
+//
+// value: Threshold for word occurrence. Words that appear with higher
+// frequency will be randomly down-sampled. Set to 0 to disable.
+// If not specified, defaults to 0.001
+func SkipgramSubsample(value float32) SkipgramAttr {
+	return func(m optionalAttr) {
+		m["subsample"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// Returns element-wise smallest integer in not less than x.
-func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
+// Parses a text file and creates a batch of examples.
+//
+// DEPRECATED at GraphDef version 19: Moving word2vec into tensorflow_models/tutorials and deprecating its ops here as a result
+//
+// Arguments:
+//	filename: The corpus's text file name.
+//	batch_size: The size of produced batch.
+//
+// Returns A vector of words in the corpus.Frequencies of words. Sorted in the non-ascending order.Number of words per epoch in the data file.The current epoch number.The total number of words processed so far.A vector of word ids.A vector of word ids.
+func Skipgram(scope *Scope, filename string, batch_size int64, optional ...SkipgramAttr) (vocab_word tf.Output, vocab_freq tf.Output, words_per_epoch tf.Output, current_epoch tf.Output, total_words_processed tf.Output, examples tf.Output, labels tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"filename": filename, "batch_size": batch_size}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Ceil",
-		Input: []tf.Input{
-			x,
-		},
+		Type: "Skipgram",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4), op.Output(5), op.Output(6)
 }
 
-// Computes the number of elements in the given table.
+// StringToNumberAttr is an optional argument to StringToNumber.
+type StringToNumberAttr func(optionalAttr)
+
+// StringToNumberOutType sets the optional out_type attribute to value.
 //
-// Arguments:
-//	table_handle: Handle to the table.
+// value: The numeric type to interpret each string in `string_tensor` as.
+// If not specified, defaults to DT_FLOAT
+func StringToNumberOutType(value tf.DataType) StringToNumberAttr {
+	return func(m optionalAttr) {
+		m["out_type"] = value
+	}
+}
+
+// Converts each string in the input Tensor to the specified numeric type.
 //
-// Returns Scalar that contains number of elements in the table.
-func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
+// (Note that int32 overflow results in an error while float overflow
+// results in a rounded value.)
+//
+// Returns A Tensor of the same shape as the input `string_tensor`.
+func StringToNumber(scope *Scope, string_tensor tf.Output, optional ...StringToNumberAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableSizeV2",
+		Type: "StringToNumber",
 		Input: []tf.Input{
-			table_handle,
+			string_tensor,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
-type ResizeBilinearGradAttr func(optionalAttr)
+// ResourceApplyFtrlV2Attr is an optional argument to ResourceApplyFtrlV2.
+type ResourceApplyFtrlV2Attr func(optionalAttr)
 
-// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
+// ResourceApplyFtrlV2UseLocking sets the optional use_locking attribute to value.
 //
-// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of grads and original_image. If false, rescale by
-// orig_height / height. Treat similarly the width dimension.
+// value: If `True`, updating of the var and accum tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
 // If not specified, defaults to false
-func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
+func ResourceApplyFtrlV2UseLocking(value bool) ResourceApplyFtrlV2Attr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["use_locking"] = value
 	}
 }
 
-// Computes the gradient of bilinear interpolation.
+// Update '*var' according to the Ftrl-proximal scheme.
+//
+// grad_with_shrinkage = grad + 2 * l2_shrinkage * var
+// accum_new = accum + grad_with_shrinkage * grad_with_shrinkage
+// linear += grad_with_shrinkage +
+//     (accum_new^(-lr_power) - accum^(-lr_power)) / lr * var
+// quadratic = 1.0 / (accum_new^(lr_power) * lr) + 2 * l2
+// var = (sign(linear) * l1 - linear) / quadratic if |linear| > l1 else 0.0
+// accum = accum_new
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
+//	var_: Should be from a Variable().
+//	accum: Should be from a Variable().
+//	linear: Should be from a Variable().
+//	grad: The gradient.
+//	lr: Scaling factor. Must be a scalar.
+//	l1: L1 regulariation. Must be a scalar.
+//	l2: L2 shrinkage regulariation. Must be a scalar.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
+//	lr_power: Scaling factor. Must be a scalar.
+//
+// Returns the created operation.
+func ResourceApplyFtrlV2(scope *Scope, var_ tf.Output, accum tf.Output, linear tf.Output, grad tf.Output, lr tf.Output, l1 tf.Output, l2 tf.Output, l2_shrinkage tf.Output, lr_power tf.Output, optional ...ResourceApplyFtrlV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17845,108 +17526,173 @@ func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBilinearGrad",
+		Type: "ResourceApplyFtrlV2",
 		Input: []tf.Input{
-			grads, original_image,
+			var_, accum, linear, grad, lr, l1, l2, l2_shrinkage, lr_power,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Outputs all keys and values in the table.
+// TruncatedNormalAttr is an optional argument to TruncatedNormal.
+type TruncatedNormalAttr func(optionalAttr)
+
+// TruncatedNormalSeed sets the optional seed attribute to value.
 //
-// Arguments:
-//	table_handle: Handle to the table.
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func TruncatedNormalSeed(value int64) TruncatedNormalAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// TruncatedNormalSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func TruncatedNormalSeed2(value int64) TruncatedNormalAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Outputs random values from a truncated normal distribution.
 //
+// The generated values follow a normal distribution with mean 0 and standard
+// deviation 1, except that values whose magnitude is more than 2 standard
+// deviations from the mean are dropped and re-picked.
 //
+// Arguments:
+//	shape: The shape of the output tensor.
+//	dtype: The type of the output.
 //
-// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
-func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+// Returns A tensor of the specified shape filled with random truncated normal
+// values.
+func TruncatedNormal(scope *Scope, shape tf.Output, dtype tf.DataType, optional ...TruncatedNormalAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableExportV2",
+		Type: "TruncatedNormal",
 		Input: []tf.Input{
-			table_handle,
+			shape,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Replaces the contents of the table with the specified keys and values.
+// RandomShuffleAttr is an optional argument to RandomShuffle.
+type RandomShuffleAttr func(optionalAttr)
+
+// RandomShuffleSeed sets the optional seed attribute to value.
 //
-// The tensor `keys` must be of the same type as the keys of the table.
-// The tensor `values` must be of the type of the table values.
+// value: If either `seed` or `seed2` are set to be non-zero, the random number
+// generator is seeded by the given seed.  Otherwise, it is seeded by a
+// random seed.
+// If not specified, defaults to 0
+func RandomShuffleSeed(value int64) RandomShuffleAttr {
+	return func(m optionalAttr) {
+		m["seed"] = value
+	}
+}
+
+// RandomShuffleSeed2 sets the optional seed2 attribute to value.
+//
+// value: A second seed to avoid seed collision.
+// If not specified, defaults to 0
+func RandomShuffleSeed2(value int64) RandomShuffleAttr {
+	return func(m optionalAttr) {
+		m["seed2"] = value
+	}
+}
+
+// Randomly shuffles a tensor along its first dimension.
+//
+//   The tensor is shuffled along dimension 0, such that each `value[j]` is mapped
+//   to one and only one `output[i]`. For example, a mapping that might occur for a
+//   3x2 tensor is:
+//
+// ```
+// [[1, 2],       [[5, 6],
+//  [3, 4],  ==>   [1, 2],
+//  [5, 6]]        [3, 4]]
+// ```
 //
 // Arguments:
-//	table_handle: Handle to the table.
-//	keys: Any shape.  Keys to look up.
-//	values: Values to associate with keys.
+//	value: The tensor to be shuffled.
 //
-// Returns the created operation.
-func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+// Returns A tensor of same shape and type as `value`, shuffled along its first
+// dimension.
+func RandomShuffle(scope *Scope, value tf.Output, optional ...RandomShuffleAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "LookupTableImportV2",
+		Type: "RandomShuffle",
 		Input: []tf.Input{
-			table_handle, keys, values,
+			value,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
-type MapUnstageNoKeyAttr func(optionalAttr)
+// OrderedMapIncompleteSizeAttr is an optional argument to OrderedMapIncompleteSize.
+type OrderedMapIncompleteSizeAttr func(optionalAttr)
 
-// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
+// OrderedMapIncompleteSizeCapacity sets the optional capacity attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
+func OrderedMapIncompleteSizeCapacity(value int64) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["capacity"] = value
 	}
 }
 
-// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
+// OrderedMapIncompleteSizeMemoryLimit sets the optional memory_limit attribute to value.
 // If not specified, defaults to 0
 //
 // REQUIRES: value >= 0
-func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
+func OrderedMapIncompleteSizeMemoryLimit(value int64) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["memory_limit"] = value
 	}
 }
 
-// MapUnstageNoKeyContainer sets the optional container attribute to value.
+// OrderedMapIncompleteSizeContainer sets the optional container attribute to value.
 // If not specified, defaults to ""
-func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
+func OrderedMapIncompleteSizeContainer(value string) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
+// OrderedMapIncompleteSizeSharedName sets the optional shared_name attribute to value.
 // If not specified, defaults to ""
-func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
+func OrderedMapIncompleteSizeSharedName(value string) OrderedMapIncompleteSizeAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// Op removes and returns a random (key, value)
-//
-// from the underlying container.   If the underlying container
-// does not contain elements, the op will block until it does.
-func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
+// Op returns the number of incomplete elements in the underlying container.
+func OrderedMapIncompleteSize(scope *Scope, dtypes []tf.DataType, optional ...OrderedMapIncompleteSizeAttr) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -17955,246 +17701,180 @@ func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, opti
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MapUnstageNoKey",
-		Input: []tf.Input{
-			indices,
-		},
+		Type: "OrderedMapIncompleteSize",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	key = op.Output(idx)
-	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
-		scope.UpdateErr("MapUnstageNoKey", err)
-		return
-	}
-	return key, values
-}
-
-// HashTableV2Attr is an optional argument to HashTableV2.
-type HashTableV2Attr func(optionalAttr)
-
-// HashTableV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func HashTableV2Container(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
+	return op.Output(0)
 }
 
-// HashTableV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func HashTableV2SharedName(value string) HashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
+// DecodeRawAttr is an optional argument to DecodeRaw.
+type DecodeRawAttr func(optionalAttr)
 
-// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// DecodeRawLittleEndian sets the optional little_endian attribute to value.
 //
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
+// value: Whether the input `bytes` are in little-endian order.
+// Ignored for `out_type` values that are stored in a single byte like
+// `uint8`.
+// If not specified, defaults to true
+func DecodeRawLittleEndian(value bool) DecodeRawAttr {
 	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
+		m["little_endian"] = value
 	}
 }
 
-// Creates a non-initialized hash table.
-//
-// This op creates a hash table, specifying the type of its keys and values.
-// Before using the table you will have to initialize it.  After initialization the
-// table will be immutable.
+// Reinterpret the bytes of a string as a vector of numbers.
 //
 // Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
+//	bytes: All the elements must have the same length.
 //
-// Returns Handle to a table.
-func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
+//
+// Returns A Tensor with one more dimension than the input `bytes`.  The
+// added dimension will have size equal to the length of the elements
+// of `bytes` divided by the number of bytes to represent `out_type`.
+func DecodeRaw(scope *Scope, bytes tf.Output, out_type tf.DataType, optional ...DecodeRawAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	attrs := map[string]interface{}{"out_type": out_type}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "HashTableV2",
-
+		Type: "DecodeRaw",
+		Input: []tf.Input{
+			bytes,
+		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
-type MutableHashTableV2Attr func(optionalAttr)
-
-// MutableHashTableV2Container sets the optional container attribute to value.
+// Copy a tensor setting everything outside a central band in each innermost matrix
 //
-// value: If non-empty, this table is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
+// to zero.
 //
-// value: If non-empty, this table is shared under the given name across
-// multiple sessions.
-// If not specified, defaults to ""
-func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+// The `band` part is computed as follows:
+// Assume `input` has `k` dimensions `[I, J, K, ..., M, N]`, then the output is a
+// tensor with the same shape where
 //
-// value: If true and shared_name is empty, the table is shared
-// using the node name.
-// If not specified, defaults to false
-func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
-	return func(m optionalAttr) {
-		m["use_node_name_sharing"] = value
-	}
-}
-
-// Creates an empty hash table.
+// `band[i, j, k, ..., m, n] = in_band(m, n) * input[i, j, k, ..., m, n]`.
 //
-// This op creates a mutable hash table, specifying the type of its keys and
-// values. Each value must be a scalar. Data can be inserted into the table using
-// the insert operations. It does not support the initialization operation.
+// The indicator function
 //
-// Arguments:
-//	key_dtype: Type of the table keys.
-//	value_dtype: Type of the table values.
+// `in_band(m, n) = (num_lower < 0 || (m-n) <= num_lower)) &&
+//                  (num_upper < 0 || (n-m) <= num_upper)`.
 //
-// Returns Handle to a table.
-func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
+// For example:
+//
+// ```
+// # if 'input' is [[ 0,  1,  2, 3]
+//                  [-1,  0,  1, 2]
+//                  [-2, -1,  0, 1]
+//                  [-3, -2, -1, 0]],
+//
+// tf.matrix_band_part(input, 1, -1) ==> [[ 0,  1,  2, 3]
+//                                        [-1,  0,  1, 2]
+//                                        [ 0, -1,  0, 1]
+//                                        [ 0,  0, -1, 0]],
+//
+// tf.matrix_band_part(input, 2, 1) ==> [[ 0,  1,  0, 0]
+//                                       [-1,  0,  1, 0]
+//                                       [-2, -1,  0, 1]
+//                                       [ 0, -2, -1, 0]]
+// ```
+//
+// Useful special cases:
+//
+// ```
+//  tf.matrix_band_part(input, 0, -1) ==> Upper triangular part.
+//  tf.matrix_band_part(input, -1, 0) ==> Lower triangular part.
+//  tf.matrix_band_part(input, 0, 0) ==> Diagonal.
+// ```
+//
+// Arguments:
+//	input: Rank `k` tensor.
+//	num_lower: 0-D tensor. Number of subdiagonals to keep. If negative, keep entire
+// lower triangle.
+//	num_upper: 0-D tensor. Number of superdiagonals to keep. If negative, keep
+// entire upper triangle.
+//
+// Returns Rank `k` tensor of the same shape as input. The extracted banded tensor.
+func MatrixBandPart(scope *Scope, input tf.Output, num_lower tf.Output, num_upper tf.Output) (band tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "MutableHashTableV2",
-
-		Attrs: attrs,
+		Type: "MatrixBandPart",
+		Input: []tf.Input{
+			input, num_lower, num_upper,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DequantizeAttr is an optional argument to Dequantize.
-type DequantizeAttr func(optionalAttr)
+// QuantizedMatMulAttr is an optional argument to QuantizedMatMul.
+type QuantizedMatMulAttr func(optionalAttr)
 
-// DequantizeMode sets the optional mode attribute to value.
-// If not specified, defaults to "MIN_COMBINED"
-func DequantizeMode(value string) DequantizeAttr {
+// QuantizedMatMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMatMulToutput(value tf.DataType) QuantizedMatMulAttr {
 	return func(m optionalAttr) {
-		m["mode"] = value
+		m["Toutput"] = value
 	}
 }
 
-// Dequantize the 'input' tensor into a float Tensor.
-//
-// [min_range, max_range] are scalar floats that specify the range for
-// the 'input' data. The 'mode' attribute controls exactly which calculations are
-// used to convert the float values to their quantized equivalents.
-//
-// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
-//
-// ```
-// if T == qint8, in[i] += (range(T) + 1)/ 2.0
-// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
-// ```
-// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
-//
-// *MIN_COMBINED Mode Example*
-//
-// If the input comes from a QuantizedRelu6, the output type is
-// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
-// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
-// Dequantize on quint8 will take each value, cast to float, and multiply
-// by 6 / 255.
-// Note that if quantizedtype is qint8, the operation will additionally add
-// each value by 128 prior to casting.
-//
-// If the mode is 'MIN_FIRST', then this approach is used:
-//
-// ```c++
-// num_discrete_values = 1 << (# of bits in T)
-// range_adjust = num_discrete_values / (num_discrete_values - 1)
-// range = (range_max - range_min) * range_adjust
-// range_scale = range / num_discrete_values
-// const double offset_input = static_cast<double>(input) - lowest_quantized;
-// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
-// ```
-//
-// *SCALED mode Example*
-//
-// `SCALED` mode matches the quantization approach used in
-// `QuantizeAndDequantize{V2|V3}`.
-//
-// If the mode is `SCALED`, we do not use the full range of the output type,
-// choosing to elide the lowest possible value for symmetry (e.g., output range is
-// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
-// 0.
-//
-// We first find the range of values in our tensor. The
-// range we use is always centered on 0, so we find m such that
-// ```c++
-//   m = max(abs(input_min), abs(input_max))
-// ```
-//
-// Our input tensor range is then `[-m, m]`.
+// QuantizedMatMulTransposeA sets the optional transpose_a attribute to value.
 //
-// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
-// If T is signed, this is
-// ```
-//   num_bits = sizeof(T) * 8
-//   [min_fixed, max_fixed] =
-//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
-// ```
+// value: If true, `a` is transposed before multiplication.
+// If not specified, defaults to false
+func QuantizedMatMulTransposeA(value bool) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_a"] = value
+	}
+}
+
+// QuantizedMatMulTransposeB sets the optional transpose_b attribute to value.
 //
-// Otherwise, if T is unsigned, the fixed-point range is
-// ```
-//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
-// ```
+// value: If true, `b` is transposed before multiplication.
+// If not specified, defaults to false
+func QuantizedMatMulTransposeB(value bool) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["transpose_b"] = value
+	}
+}
+
+// QuantizedMatMulTactivation sets the optional Tactivation attribute to value.
 //
-// From this we compute our scaling factor, s:
-// ```c++
-//   s = (2 * m) / (max_fixed - min_fixed)
-// ```
+// value: The type of output produced by activation function
+// following this operation.
+// If not specified, defaults to DT_QUINT8
+func QuantizedMatMulTactivation(value tf.DataType) QuantizedMatMulAttr {
+	return func(m optionalAttr) {
+		m["Tactivation"] = value
+	}
+}
+
+// Perform a quantized matrix multiplication of  `a` by the matrix `b`.
 //
-// Now we can dequantize the elements of our tensor:
-// ```c++
-// result = input * s
-// ```
+// The inputs must be two-dimensional matrices and the inner dimension of
+// `a` (after being transposed if `transpose_a` is non-zero) must match the
+// outer dimension of `b` (after being transposed if `transposed_b` is
+// non-zero).
 //
 // Arguments:
+//	a: Must be a two-dimensional tensor.
+//	b: Must be a two-dimensional tensor.
+//	min_a: The float value that the lowest quantized `a` value represents.
+//	max_a: The float value that the highest quantized `a` value represents.
+//	min_b: The float value that the lowest quantized `b` value represents.
+//	max_b: The float value that the highest quantized `b` value represents.
 //
-//	min_range: The minimum scalar value possibly produced for the input.
-//	max_range: The maximum scalar value possibly produced for the input.
-func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+func QuantizedMatMul(scope *Scope, a tf.Output, b tf.Output, min_a tf.Output, max_a tf.Output, min_b tf.Output, max_b tf.Output, optional ...QuantizedMatMulAttr) (out tf.Output, min_out tf.Output, max_out tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18203,115 +17883,114 @@ func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Dequantize",
+		Type: "QuantizedMatMul",
 		Input: []tf.Input{
-			input, min_range, max_range,
+			a, b, min_a, max_a, min_b, max_b,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Flips all bits elementwise.
+// Does nothing. Serves as a control trigger for scheduling.
 //
-// The result will have exactly those bits set, that are not set in `x`. The
-// computation is performed on the underlying representation of x.
-func Invert(scope *Scope, x tf.Output) (y tf.Output) {
+// Only useful as a placeholder for control edges.
+//
+// Returns the created operation.
+func ControlTrigger(scope *Scope) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Invert",
-		Input: []tf.Input{
-			x,
-		},
+		Type: "ControlTrigger",
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Deprecated. Disallowed in GraphDef version >= 2.
+// Batch normalization.
 //
-// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
-func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
+// DEPRECATED at GraphDef version 9: Use tf.nn.batch_normalization()
+//
+// This op is deprecated. Prefer `tf.nn.batch_normalization`.
+//
+// Arguments:
+//	t: A 4D input Tensor.
+//	m: A 1D mean Tensor with size matching the last dimension of t.
+// This is the first output from tf.nn.moments,
+// or a saved moving average thereof.
+//	v: A 1D variance Tensor with size matching the last dimension of t.
+// This is the second output from tf.nn.moments,
+// or a saved moving average thereof.
+//	beta: A 1D beta Tensor with size matching the last dimension of t.
+// An offset to be added to the normalized tensor.
+//	gamma: A 1D gamma Tensor with size matching the last dimension of t.
+// If "scale_after_normalization" is true, this tensor will be multiplied
+// with the normalized tensor.
+//	variance_epsilon: A small float number to avoid dividing by 0.
+//	scale_after_normalization: A bool indicating whether the resulted tensor
+// needs to be multiplied with gamma.
+func BatchNormWithGlobalNormalization(scope *Scope, t tf.Output, m tf.Output, v tf.Output, beta tf.Output, gamma tf.Output, variance_epsilon float32, scale_after_normalization bool) (result tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"variance_epsilon": variance_epsilon, "scale_after_normalization": scale_after_normalization}
 	opspec := tf.OpSpec{
-		Type: "AdjustContrast",
+		Type: "BatchNormWithGlobalNormalization",
 		Input: []tf.Input{
-			images, contrast_factor, min_value, max_value,
+			t, m, v, beta, gamma,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Table initializer that takes two tensors for keys and values respectively.
-//
-// Arguments:
-//	table_handle: Handle to a table which will be initialized.
-//	keys: Keys of type Tkey.
-//	values: Values of type Tval.
+// Deprecated. Use TensorArrayReadV3
 //
-// Returns the created operation.
-func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
+// DEPRECATED at GraphDef version 26: Use TensorArrayReadV3
+func TensorArrayReadV2(scope *Scope, handle tf.Output, index tf.Output, flow_in tf.Output, dtype tf.DataType) (value tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtype": dtype}
 	opspec := tf.OpSpec{
-		Type: "InitializeTableV2",
+		Type: "TensorArrayReadV2",
 		Input: []tf.Input{
-			table_handle, keys, values,
+			handle, index, flow_in,
 		},
+		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// PrintAttr is an optional argument to Print.
-type PrintAttr func(optionalAttr)
+// QuantizedMulAttr is an optional argument to QuantizedMul.
+type QuantizedMulAttr func(optionalAttr)
 
-// PrintMessage sets the optional message attribute to value.
-//
-// value: A string, prefix of the error message.
-// If not specified, defaults to ""
-func PrintMessage(value string) PrintAttr {
+// QuantizedMulToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedMulToutput(value tf.DataType) QuantizedMulAttr {
 	return func(m optionalAttr) {
-		m["message"] = value
+		m["Toutput"] = value
 	}
 }
 
-// PrintFirstN sets the optional first_n attribute to value.
+// Returns x * y element-wise, working on quantized buffers.
 //
-// value: Only log `first_n` number of times. -1 disables logging.
-// If not specified, defaults to -1
-func PrintFirstN(value int64) PrintAttr {
-	return func(m optionalAttr) {
-		m["first_n"] = value
-	}
-}
-
-// PrintSummarize sets the optional summarize attribute to value.
+// Arguments:
 //
-// value: Only print this many entries of each tensor.
-// If not specified, defaults to 3
-func PrintSummarize(value int64) PrintAttr {
-	return func(m optionalAttr) {
-		m["summarize"] = value
-	}
-}
-
-// Prints a list of tensors.
 //
-// Passes `input` through to `output` and prints `data` when evaluating.
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
 //
-// Arguments:
-//	input: The tensor passed to `output`
-//	data: A list of tensors to print out when op is evaluated.
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
 //
-// Returns = The unmodified `input` tensor
-func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
+// *NOTE*: `QuantizedMul` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedMul(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedMulAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18320,103 +17999,119 @@ func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Print",
+		Type: "QuantizedMul",
 		Input: []tf.Input{
-			input, tf.OutputList(data),
+			x, y, min_x, max_x, min_y, max_y,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
-//
-// Arguments:
-//	tag: A string attached to this summary. Used for organization in TensorBoard.
-//	tensor: A tensor to serialize.
-//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
-// data.
-func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorSummaryV2",
-		Input: []tf.Input{
-			tag, tensor, serialized_summary_metadata,
-		},
+// QuantizedAddAttr is an optional argument to QuantizedAdd.
+type QuantizedAddAttr func(optionalAttr)
+
+// QuantizedAddToutput sets the optional Toutput attribute to value.
+// If not specified, defaults to DT_QINT32
+func QuantizedAddToutput(value tf.DataType) QuantizedAddAttr {
+	return func(m optionalAttr) {
+		m["Toutput"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
+// Returns x + y element-wise, working on quantized buffers.
 //
 // Arguments:
 //
-//	buffer_size: The maximum number of elements to buffer in an iterator over
-// this dataset.
 //
+//	min_x: The float value that the lowest quantized `x` value represents.
+//	max_x: The float value that the highest quantized `x` value represents.
+//	min_y: The float value that the lowest quantized `y` value represents.
+//	max_y: The float value that the highest quantized `y` value represents.
 //
-func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
+// Returns The float value that the lowest quantized output value represents.The float value that the highest quantized output value represents.
+//
+// *NOTE*: `QuantizedAdd` supports limited forms of broadcasting. More about
+// broadcasting [here](http://docs.scipy.org/doc/numpy/user/basics.broadcasting.html)
+func QuantizedAdd(scope *Scope, x tf.Output, y tf.Output, min_x tf.Output, max_x tf.Output, min_y tf.Output, max_y tf.Output, optional ...QuantizedAddAttr) (z tf.Output, min_z tf.Output, max_z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "PrefetchDataset",
+		Type: "QuantizedAdd",
 		Input: []tf.Input{
-			input_dataset, buffer_size,
+			x, y, min_x, max_x, min_y, max_y,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// TensorSummaryAttr is an optional argument to TensorSummary.
-type TensorSummaryAttr func(optionalAttr)
+// MfccAttr is an optional argument to Mfcc.
+type MfccAttr func(optionalAttr)
 
-// TensorSummaryDescription sets the optional description attribute to value.
+// MfccUpperFrequencyLimit sets the optional upper_frequency_limit attribute to value.
 //
-// value: A json-encoded SummaryDescription proto.
-// If not specified, defaults to ""
-func TensorSummaryDescription(value string) TensorSummaryAttr {
+// value: The highest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 4000
+func MfccUpperFrequencyLimit(value float32) MfccAttr {
 	return func(m optionalAttr) {
-		m["description"] = value
+		m["upper_frequency_limit"] = value
 	}
 }
 
-// TensorSummaryLabels sets the optional labels attribute to value.
+// MfccLowerFrequencyLimit sets the optional lower_frequency_limit attribute to value.
 //
-// value: An unused list of strings.
-// If not specified, defaults to <>
-func TensorSummaryLabels(value []string) TensorSummaryAttr {
+// value: The lowest frequency to use when calculating the
+// ceptstrum.
+// If not specified, defaults to 20
+func MfccLowerFrequencyLimit(value float32) MfccAttr {
 	return func(m optionalAttr) {
-		m["labels"] = value
+		m["lower_frequency_limit"] = value
 	}
 }
 
-// TensorSummaryDisplayName sets the optional display_name attribute to value.
+// MfccFilterbankChannelCount sets the optional filterbank_channel_count attribute to value.
 //
-// value: An unused string.
-// If not specified, defaults to ""
-func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+// value: Resolution of the Mel bank used internally.
+// If not specified, defaults to 40
+func MfccFilterbankChannelCount(value int64) MfccAttr {
 	return func(m optionalAttr) {
-		m["display_name"] = value
+		m["filterbank_channel_count"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with a tensor.
+// MfccDctCoefficientCount sets the optional dct_coefficient_count attribute to value.
 //
-// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
-// a tag as well as a serialized SummaryMetadata proto string that contains
-// plugin-specific data. We will keep this op to maintain backwards compatibility.
+// value: How many output channels to produce per time slice.
+// If not specified, defaults to 13
+func MfccDctCoefficientCount(value int64) MfccAttr {
+	return func(m optionalAttr) {
+		m["dct_coefficient_count"] = value
+	}
+}
+
+// Transforms a spectrogram into a form that's useful for speech recognition.
+//
+// Mel Frequency Cepstral Coefficients are a way of representing audio data that's
+// been effective as an input feature for machine learning. They are created by
+// taking the spectrum of a spectrogram (a 'cepstrum'), and discarding some of the
+// higher frequencies that are less significant to the human ear. They have a long
+// history in the speech recognition world, and https://en.wikipedia.org/wiki/Mel-frequency_cepstrum
+// is a good resource to learn more.
 //
 // Arguments:
-//	tensor: A tensor to serialize.
-func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
+//	spectrogram: Typically produced by the Spectrogram op, with magnitude_squared
+// set to true.
+//	sample_rate: How many samples per second the source audio used.
+func Mfcc(scope *Scope, spectrogram tf.Output, sample_rate tf.Output, optional ...MfccAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18425,9 +18120,9 @@ func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "TensorSummary",
+		Type: "Mfcc",
 		Input: []tf.Input{
-			tensor,
+			spectrogram, sample_rate,
 		},
 		Attrs: attrs,
 	}
@@ -18435,174 +18130,163 @@ func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr
 	return op.Output(0)
 }
 
-// Computes the gradient for the tanh of `x` wrt its input.
+// Given a quantized tensor described by (input, input_min, input_max), outputs a
 //
-// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
-// is the corresponding input gradient.
-func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+// range that covers the actual values present in that tensor.  This op is
+// typically used to produce the requested_output_min and requested_output_max for
+// Requantize.
+//
+// Arguments:
+//
+//	input_min: The float value that the minimum quantized input value represents.
+//	input_max: The float value that the maximum quantized input value represents.
+//
+// Returns The computed min output.the computed max output.
+func RequantizationRange(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output) (output_min tf.Output, output_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TanhGrad",
+		Type: "RequantizationRange",
 		Input: []tf.Input{
-			y, dy,
+			input, input_min, input_max,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Outputs a `Summary` protocol buffer with scalar values.
+// MapPeekAttr is an optional argument to MapPeek.
+type MapPeekAttr func(optionalAttr)
+
+// MapPeekCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// The input `tags` and `values` must have the same shape.  The generated summary
-// has a summary value for each tag-value pair in `tags` and `values`.
+// REQUIRES: value >= 0
+func MapPeekCapacity(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// MapPeekMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	tags: Tags for the summary.
-//	values: Same shape as `tags.  Values for the summary.
+// REQUIRES: value >= 0
+func MapPeekMemoryLimit(value int64) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapPeekContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapPeekContainer(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapPeekSharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapPeekSharedName(value string) MapPeekAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op peeks at the values at the specified key.  If the
 //
-// Returns Scalar.  Serialized `Summary` protocol buffer.
-func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
+// underlying container does not contain this key
+// this op will block until it does.
+func MapPeek(scope *Scope, key tf.Output, indices tf.Output, dtypes []tf.DataType, optional ...MapPeekAttr) (values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"dtypes": dtypes}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ScalarSummary",
+		Type: "MapPeek",
 		Input: []tf.Input{
-			tags, values,
+			key, indices,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Outputs a `Summary` protocol buffer with a histogram.
-//
-// The generated
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// has one summary value containing a histogram for `values`.
-//
-// This op reports an `InvalidArgument` error if any value is not finite.
-//
-// Arguments:
-//	tag: Scalar.  Tag to use for the `Summary.Value`.
-//	values: Any shape. Values to use to build the histogram.
-//
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "HistogramSummary",
-		Input: []tf.Input{
-			tag, values,
-		},
+	var idx int
+	var err error
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapPeek", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return values
 }
 
-// Computes the number of elements in the given queue.
+// Looks up keys in a table, outputs the corresponding values.
+//
+// The tensor `keys` must of the same type as the keys of the table.
+// The output `values` is of the type of the table values.
+//
+// The scalar `default_value` is the value output for keys not present in the
+// table. It must also be of the same type as the table values.
 //
 // Arguments:
-//	handle: The handle to a queue.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
 //
-// Returns The number of elements in the given queue.
-func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
+//
+// Returns Same shape as `keys`.  Values found in the table, or `default_values`
+// for missing keys.
+func LookupTableFindV2(scope *Scope, table_handle tf.Output, keys tf.Output, default_value tf.Output) (values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueSizeV2",
+		Type: "LookupTableFindV2",
 		Input: []tf.Input{
-			handle,
+			table_handle, keys, default_value,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ImageSummaryAttr is an optional argument to ImageSummary.
-type ImageSummaryAttr func(optionalAttr)
-
-// ImageSummaryMaxImages sets the optional max_images attribute to value.
-//
-// value: Max number of batch elements to generate images for.
-// If not specified, defaults to 3
-//
-// REQUIRES: value >= 1
-func ImageSummaryMaxImages(value int64) ImageSummaryAttr {
-	return func(m optionalAttr) {
-		m["max_images"] = value
-	}
-}
-
-// ImageSummaryBadColor sets the optional bad_color attribute to value.
-//
-// value: Color to use for pixels with non-finite values.
-// If not specified, defaults to <dtype:DT_UINT8 tensor_shape:<dim:<size:4 > > int_val:255 int_val:0 int_val:0 int_val:255 >
-func ImageSummaryBadColor(value tf.Tensor) ImageSummaryAttr {
-	return func(m optionalAttr) {
-		m["bad_color"] = value
-	}
-}
-
-// Outputs a `Summary` protocol buffer with images.
-//
-// The summary has up to `max_images` summary values containing images. The
-// images are built from `tensor` which must be 4-D with shape `[batch_size,
-// height, width, channels]` and where `channels` can be:
-//
-// *  1: `tensor` is interpreted as Grayscale.
-// *  3: `tensor` is interpreted as RGB.
-// *  4: `tensor` is interpreted as RGBA.
-//
-// The images have the same number of channels as the input tensor. For float
-// input, the values are normalized one image at a time to fit in the range
-// `[0, 255]`.  `uint8` values are unchanged.  The op uses two different
-// normalization algorithms:
-//
-// *  If the input values are all positive, they are rescaled so the largest one
-//    is 255.
-//
-// *  If any input value is negative, the values are shifted so input value 0.0
-//    is at 127.  They are then rescaled so that either the smallest value is 0,
-//    or the largest one is 255.
-//
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// Bucketizes 'input' based on 'boundaries'.
 //
-// *  If `max_images` is 1, the summary value tag is '*tag*/image'.
-// *  If `max_images` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
+// For example, if the inputs are
+//     boundaries = [0, 10, 100]
+//     input = [[-5, 10000]
+//              [150,   10]
+//              [5,    100]]
 //
-// The `bad_color` argument is the color to use in the generated images for
-// non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
-// Each element must be in the range `[0, 255]` (It represents the value of a
-// pixel in the output image).  Non-finite values in the input tensor are
-// replaced by this tensor in the output image.  The default value is the color
-// red.
+// then the output will be
+//     output = [[0, 3]
+//               [3, 2]
+//               [1, 3]]
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 4-D of shape `[batch_size, height, width, channels]` where
-// `channels` is 1, 3, or 4.
+//	input: Any shape of Tensor contains with int or float type.
+//	boundaries: A sorted list of floats gives the boundary of the buckets.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...ImageSummaryAttr) (summary tf.Output) {
+// Returns Same shape with 'input', each value of input replaced with bucket index.
+//
+// @compatibility(numpy)
+// Equivalent to np.digitize.
+// @end_compatibility
+func Bucketize(scope *Scope, input tf.Output, boundaries []float32) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"boundaries": boundaries}
 	opspec := tf.OpSpec{
-		Type: "ImageSummary",
+		Type: "Bucketize",
 		Input: []tf.Input{
-			tag, tensor,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -18610,42 +18294,38 @@ func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...Ima
 	return op.Output(0)
 }
 
-// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
-type AudioSummaryV2Attr func(optionalAttr)
+// EncodePngAttr is an optional argument to EncodePng.
+type EncodePngAttr func(optionalAttr)
 
-// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
-//
-// value: Max number of batch elements to generate audio for.
-// If not specified, defaults to 3
+// EncodePngCompression sets the optional compression attribute to value.
 //
-// REQUIRES: value >= 1
-func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
+// value: Compression level.
+// If not specified, defaults to -1
+func EncodePngCompression(value int64) EncodePngAttr {
 	return func(m optionalAttr) {
-		m["max_outputs"] = value
+		m["compression"] = value
 	}
 }
 
-// Outputs a `Summary` protocol buffer with audio.
+// PNG-encode an image.
 //
-// The summary has up to `max_outputs` summary values containing audio. The
-// audio is built from `tensor` which must be 3-D with shape `[batch_size,
-// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
-// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
+// `image` is a 3-D uint8 or uint16 Tensor of shape `[height, width, channels]`
+// where `channels` is:
 //
-// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
-// build the `tag` of the summary values:
+// *   1: for grayscale.
+// *   2: for grayscale + alpha.
+// *   3: for RGB.
+// *   4: for RGBA.
 //
-// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
-// *  If `max_outputs` is greater than 1, the summary value tags are
-//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
+// The ZLIB compression level, `compression`, can be -1 for the PNG-encoder
+// default or a value from 0 to 9.  9 is the highest compression level, generating
+// the smallest output, but is slower.
 //
 // Arguments:
-//	tag: Scalar. Used to build the `tag` attribute of the summary values.
-//	tensor: 2-D of shape `[batch_size, frames]`.
-//	sample_rate: The sample rate of the signal in hertz.
+//	image: 3-D with shape `[height, width, channels]`.
 //
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
+// Returns 0-D. PNG-encoded image.
+func EncodePng(scope *Scope, image tf.Output, optional ...EncodePngAttr) (contents tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18654,9 +18334,9 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AudioSummaryV2",
+		Type: "EncodePng",
 		Input: []tf.Input{
-			tag, tensor, sample_rate,
+			image,
 		},
 		Attrs: attrs,
 	}
@@ -18664,161 +18344,91 @@ func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate t
 	return op.Output(0)
 }
 
-// AvgPoolAttr is an optional argument to AvgPool.
-type AvgPoolAttr func(optionalAttr)
-
-// AvgPoolDataFormat sets the optional data_format attribute to value.
-//
-// value: Specify the data format of the input and output data. With the
-// default format "NHWC", the data is stored in the order of:
-//     [batch, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCHW", the data storage order of:
-//     [batch, in_channels, in_height, in_width].
-// If not specified, defaults to "NHWC"
-func AvgPoolDataFormat(value string) AvgPoolAttr {
-	return func(m optionalAttr) {
-		m["data_format"] = value
-	}
-}
-
-// Performs average pooling on the input.
+// Updates the table to associates keys with values.
 //
-// Each entry in `output` is the mean of the corresponding size `ksize`
-// window in `value`.
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
 // Arguments:
-//	value: 4-D with shape `[batch, height, width, channels]`.
-//	ksize: The size of the sliding window for each dimension of `value`.
-//	strides: The stride of the sliding window for each dimension of `value`.
-//	padding: The type of padding algorithm to use.
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
 //
-// Returns The average pooled output tensor.
-func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolAttr) (output tf.Output) {
+// Returns the created operation.
+func LookupTableInsertV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "AvgPool",
+		Type: "LookupTableInsertV2",
 		Input: []tf.Input{
-			value,
+			table_handle, keys, values,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Merges summaries.
-//
-// This op creates a
-// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
-// protocol buffer that contains the union of all the values in the input
-// summaries.
-//
-// When the Op is run, it reports an `InvalidArgument` error if multiple values
-// in the summaries to merge use the same tag.
-//
-// Arguments:
-//	inputs: Can be of any shape.  Each must contain serialized `Summary` protocol
-// buffers.
-//
-// Returns Scalar. Serialized `Summary` protocol buffer.
-func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
+// Returns element-wise smallest integer in not less than x.
+func Ceil(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MergeSummary",
+		Type: "Ceil",
 		Input: []tf.Input{
-			tf.OutputList(inputs),
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the gradient of morphological 2-D dilation with respect to the filter.
+// Computes the number of elements in the given table.
 //
 // Arguments:
-//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
-//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
-//	out_backprop: 4-D with shape `[batch, out_height, out_width, depth]`.
-//	strides: 1-D of length 4. The stride of the sliding window for each dimension of
-// the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
-//	rates: 1-D of length 4. The input stride for atrous morphological dilation.
-// Must be: `[1, rate_height, rate_width, 1]`.
-//	padding: The type of padding algorithm to use.
+//	table_handle: Handle to the table.
 //
-// Returns 3-D with shape `[filter_height, filter_width, depth]`.
-func Dilation2DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, rates []int64, padding string) (filter_backprop tf.Output) {
+// Returns Scalar that contains number of elements in the table.
+func LookupTableSizeV2(scope *Scope, table_handle tf.Output) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "Dilation2DBackpropFilter",
+		Type: "LookupTableSizeV2",
 		Input: []tf.Input{
-			input, filter, out_backprop,
+			table_handle,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// AddSparseToTensorsMapAttr is an optional argument to AddSparseToTensorsMap.
-type AddSparseToTensorsMapAttr func(optionalAttr)
-
-// AddSparseToTensorsMapContainer sets the optional container attribute to value.
-//
-// value: The container name for the `SparseTensorsMap` created by this op.
-// If not specified, defaults to ""
-func AddSparseToTensorsMapContainer(value string) AddSparseToTensorsMapAttr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// ResizeBilinearGradAttr is an optional argument to ResizeBilinearGrad.
+type ResizeBilinearGradAttr func(optionalAttr)
 
-// AddSparseToTensorsMapSharedName sets the optional shared_name attribute to value.
+// ResizeBilinearGradAlignCorners sets the optional align_corners attribute to value.
 //
-// value: The shared name for the `SparseTensorsMap` created by this op.
-// If blank, the new Operation's unique name is used.
-// If not specified, defaults to ""
-func AddSparseToTensorsMapSharedName(value string) AddSparseToTensorsMapAttr {
+// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of grads and original_image. If false, rescale by
+// orig_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeBilinearGradAlignCorners(value bool) ResizeBilinearGradAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["align_corners"] = value
 	}
 }
 
-// Add a `SparseTensor` to a `SparseTensorsMap` return its handle.
-//
-// A `SparseTensor` is represented by three tensors: `sparse_indices`,
-// `sparse_values`, and `sparse_shape`.
-//
-// This operator takes the given `SparseTensor` and adds it to a container
-// object (a `SparseTensorsMap`).  A unique key within this container is generated
-// in the form of an `int64`, and this is the value that is returned.
-//
-// The `SparseTensor` can then be read out as part of a minibatch by passing
-// the key as a vector element to `TakeManySparseFromTensorsMap`.  To ensure
-// the correct `SparseTensorsMap` is accessed, ensure that the same
-// `container` and `shared_name` are passed to that Op.  If no `shared_name`
-// is provided here, instead use the *name* of the Operation created by calling
-// `AddSparseToTensorsMap` as the `shared_name` passed to
-// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
+// Computes the gradient of bilinear interpolation.
 //
 // Arguments:
-//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
-//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
-//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
 //
-// Returns 0-D.  The handle of the `SparseTensor` now stored in the
-// `SparseTensorsMap`.
-func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddSparseToTensorsMapAttr) (sparse_handle tf.Output) {
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBilinearGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBilinearGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -18827,9 +18437,9 @@ func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AddSparseToTensorsMap",
+		Type: "ResizeBilinearGrad",
 		Input: []tf.Input{
-			sparse_indices, sparse_values, sparse_shape,
+			grads, original_image,
 		},
 		Attrs: attrs,
 	}
@@ -18837,94 +18447,109 @@ func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values
 	return op.Output(0)
 }
 
-// Computes the matrix exponential of one or more square matrices:
+// Outputs all keys and values in the table.
 //
-// exp(A) = \sum_{n=0}^\infty A^n/n!
+// Arguments:
+//	table_handle: Handle to the table.
 //
-// The exponential is computed using a combination of the scaling and squaring
-// method and the Pade approximation. Details can be founds in:
-// Nicholas J. Higham, "The scaling and squaring method for the matrix exponential
-// revisited," SIAM J. Matrix Anal. Applic., 26:1179-1193, 2005.
 //
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices. The output is a tensor of the same shape as the input
-// containing the exponential for all input submatrices `[..., :, :]`.
 //
-// Arguments:
-//	input: Shape is `[..., M, M]`.
+// Returns Vector of all keys present in the table.Tensor of all values in the table. Indexed in parallel with `keys`.
+func LookupTableExportV2(scope *Scope, table_handle tf.Output, Tkeys tf.DataType, Tvalues tf.DataType) (keys tf.Output, values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"Tkeys": Tkeys, "Tvalues": Tvalues}
+	opspec := tf.OpSpec{
+		Type: "LookupTableExportV2",
+		Input: []tf.Input{
+			table_handle,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Replaces the contents of the table with the specified keys and values.
 //
-// Returns Shape is `[..., M, M]`.
+// The tensor `keys` must be of the same type as the keys of the table.
+// The tensor `values` must be of the type of the table values.
 //
-// @compatibility(scipy)
-// Equivalent to scipy.linalg.expm
-// @end_compatibility
-func MatrixExponential(scope *Scope, input tf.Output) (output tf.Output) {
+// Arguments:
+//	table_handle: Handle to the table.
+//	keys: Any shape.  Keys to look up.
+//	values: Values to associate with keys.
+//
+// Returns the created operation.
+func LookupTableImportV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "MatrixExponential",
+		Type: "LookupTableImportV2",
 		Input: []tf.Input{
-			input,
+			table_handle, keys, values,
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// QueueDequeueUpToV2Attr is an optional argument to QueueDequeueUpToV2.
-type QueueDequeueUpToV2Attr func(optionalAttr)
+// MapUnstageNoKeyAttr is an optional argument to MapUnstageNoKey.
+type MapUnstageNoKeyAttr func(optionalAttr)
 
-// QueueDequeueUpToV2TimeoutMs sets the optional timeout_ms attribute to value.
+// MapUnstageNoKeyCapacity sets the optional capacity attribute to value.
+// If not specified, defaults to 0
 //
-// value: If the queue has fewer than n elements, this operation
-// will block for up to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueUpToV2TimeoutMs(value int64) QueueDequeueUpToV2Attr {
+// REQUIRES: value >= 0
+func MapUnstageNoKeyCapacity(value int64) MapUnstageNoKeyAttr {
 	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+		m["capacity"] = value
 	}
 }
 
-// Dequeues `n` tuples of one or more tensors from the given queue.
-//
-// This operation is not supported by all queues.  If a queue does not support
-// DequeueUpTo, then an Unimplemented error is returned.
-//
-// If the queue is closed and there are more than 0 but less than `n`
-// elements remaining, then instead of returning an OutOfRange error like
-// QueueDequeueMany, less than `n` elements are returned immediately.  If
-// the queue is closed and there are 0 elements left in the queue, then
-// an OutOfRange error is returned just like in QueueDequeueMany.
-// Otherwise the behavior is identical to QueueDequeueMany:
-//
-// This operation concatenates queue-element component tensors along the
-// 0th dimension to make a single component tensor.  All of the components
-// in the dequeued tuple will have size n in the 0th dimension.
-//
-// This operation has `k` outputs, where `k` is the number of components in
-// the tuples stored in the given queue, and output `i` is the ith
-// component of the dequeued tuple.
+// MapUnstageNoKeyMemoryLimit sets the optional memory_limit attribute to value.
+// If not specified, defaults to 0
 //
-// Arguments:
-//	handle: The handle to a queue.
-//	n: The number of tuples to dequeue.
-//	component_types: The type of each component in a tuple.
+// REQUIRES: value >= 0
+func MapUnstageNoKeyMemoryLimit(value int64) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["memory_limit"] = value
+	}
+}
+
+// MapUnstageNoKeyContainer sets the optional container attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeyContainer(value string) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MapUnstageNoKeySharedName sets the optional shared_name attribute to value.
+// If not specified, defaults to ""
+func MapUnstageNoKeySharedName(value string) MapUnstageNoKeyAttr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// Op removes and returns a random (key, value)
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueUpToV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueUpToV2Attr) (components []tf.Output) {
+// from the underlying container.   If the underlying container
+// does not contain elements, the op will block until it does.
+func MapUnstageNoKey(scope *Scope, indices tf.Output, dtypes []tf.DataType, optional ...MapUnstageNoKeyAttr) (key tf.Output, values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
+	attrs := map[string]interface{}{"dtypes": dtypes}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueUpToV2",
+		Type: "MapUnstageNoKey",
 		Input: []tf.Input{
-			handle, n,
+			indices,
 		},
 		Attrs: attrs,
 	}
@@ -18934,196 +18559,234 @@ func QueueDequeueUpToV2(scope *Scope, handle tf.Output, n tf.Output, component_t
 	}
 	var idx int
 	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueUpToV2", err)
+	key = op.Output(idx)
+	if values, idx, err = makeOutputList(op, idx, "values"); err != nil {
+		scope.UpdateErr("MapUnstageNoKey", err)
 		return
 	}
-	return components
+	return key, values
 }
 
-// Computes the Cholesky decomposition of one or more square matrices.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices.
-//
-// The input has to be symmetric and positive definite. Only the lower-triangular
-// part of the input will be used for this operation. The upper-triangular part
-// will not be read.
-//
-// The output is a tensor of the same shape as the input
-// containing the Cholesky decompositions for all input submatrices `[..., :, :]`.
-//
-// **Note**: The gradient computation on GPU is faster for large matrices but
-// not for large batch dimensions when the submatrices are small. In this
-// case it might be faster to use the CPU.
-//
-// Arguments:
-//	input: Shape is `[..., M, M]`.
+// HashTableV2Attr is an optional argument to HashTableV2.
+type HashTableV2Attr func(optionalAttr)
+
+// HashTableV2Container sets the optional container attribute to value.
 //
-// Returns Shape is `[..., M, M]`.
-func Cholesky(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Cholesky",
-		Input: []tf.Input{
-			input,
-		},
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func HashTableV2Container(value string) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Writes contents to the file at input filename. Creates file and recursively
+// HashTableV2SharedName sets the optional shared_name attribute to value.
 //
-// creates directory if not existing.
-//
-// Arguments:
-//	filename: scalar. The name of the file to which we write the contents.
-//	contents: scalar. The content to be written to the output file.
-//
-// Returns the created operation.
-func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "WriteFile",
-		Input: []tf.Input{
-			filename, contents,
-		},
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func HashTableV2SharedName(value string) HashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// AllAttr is an optional argument to All.
-type AllAttr func(optionalAttr)
-
-// AllKeepDims sets the optional keep_dims attribute to value.
+// HashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
 //
-// value: If true, retain reduced dimensions with length 1.
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
 // If not specified, defaults to false
-func AllKeepDims(value bool) AllAttr {
+func HashTableV2UseNodeNameSharing(value bool) HashTableV2Attr {
 	return func(m optionalAttr) {
-		m["keep_dims"] = value
+		m["use_node_name_sharing"] = value
 	}
 }
 
-// Computes the "logical and" of elements across dimensions of a tensor.
+// Creates a non-initialized hash table.
 //
-// Reduces `input` along the dimensions given in `axis`. Unless
-// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
-// `axis`. If `keep_dims` is true, the reduced dimensions are
-// retained with length 1.
+// This op creates a hash table, specifying the type of its keys and values.
+// Before using the table you will have to initialize it.  After initialization the
+// table will be immutable.
 //
 // Arguments:
-//	input: The tensor to reduce.
-//	axis: The dimensions to reduce. Must be in the range
-// `[-rank(input), rank(input))`.
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
 //
-// Returns The reduced tensor.
-func All(scope *Scope, input tf.Output, axis tf.Output, optional ...AllAttr) (output tf.Output) {
+// Returns Handle to a table.
+func HashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...HashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "All",
-		Input: []tf.Input{
-			input, axis,
-		},
+		Type: "HashTableV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
-//
-// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
-//
-// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
-// form square matrices, with the same constraints as the single matrix
-// SelfAdjointEig.
-//
-// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
-// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
+// MutableHashTableV2Attr is an optional argument to MutableHashTableV2.
+type MutableHashTableV2Attr func(optionalAttr)
+
+// MutableHashTableV2Container sets the optional container attribute to value.
 //
-// Arguments:
-//	input: Shape is `[..., M, M]`.
+// value: If non-empty, this table is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func MutableHashTableV2Container(value string) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// MutableHashTableV2SharedName sets the optional shared_name attribute to value.
 //
-// Returns Shape is `[..., M+1, M]`.
-func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// value: If non-empty, this table is shared under the given name across
+// multiple sessions.
+// If not specified, defaults to ""
+func MutableHashTableV2SharedName(value string) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
 	}
-	opspec := tf.OpSpec{
-		Type: "SelfAdjointEig",
-		Input: []tf.Input{
-			input,
-		},
+}
+
+// MutableHashTableV2UseNodeNameSharing sets the optional use_node_name_sharing attribute to value.
+//
+// value: If true and shared_name is empty, the table is shared
+// using the node name.
+// If not specified, defaults to false
+func MutableHashTableV2UseNodeNameSharing(value bool) MutableHashTableV2Attr {
+	return func(m optionalAttr) {
+		m["use_node_name_sharing"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// Computes softplus gradients for a softplus operation.
+// Creates an empty hash table.
+//
+// This op creates a mutable hash table, specifying the type of its keys and
+// values. Each value must be a scalar. Data can be inserted into the table using
+// the insert operations. It does not support the initialization operation.
 //
 // Arguments:
-//	gradients: The backpropagated gradients to the corresponding softplus operation.
-//	features: The features passed as input to the corresponding softplus operation.
+//	key_dtype: Type of the table keys.
+//	value_dtype: Type of the table values.
 //
-// Returns The gradients: `gradients / (1 + exp(-features))`.
-func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
+// Returns Handle to a table.
+func MutableHashTableV2(scope *Scope, key_dtype tf.DataType, value_dtype tf.DataType, optional ...MutableHashTableV2Attr) (table_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"key_dtype": key_dtype, "value_dtype": value_dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "SoftplusGrad",
-		Input: []tf.Input{
-			gradients, features,
-		},
+		Type: "MutableHashTableV2",
+
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
-type SelfAdjointEigV2Attr func(optionalAttr)
+// DequantizeAttr is an optional argument to Dequantize.
+type DequantizeAttr func(optionalAttr)
 
-// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
-//
-// value: If `True` then eigenvectors will be computed and returned in `v`.
-// Otherwise, only the eigenvalues will be computed.
-// If not specified, defaults to true
-func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
+// DequantizeMode sets the optional mode attribute to value.
+// If not specified, defaults to "MIN_COMBINED"
+func DequantizeMode(value string) DequantizeAttr {
 	return func(m optionalAttr) {
-		m["compute_v"] = value
+		m["mode"] = value
 	}
 }
 
-// Computes the eigen decomposition of one or more square self-adjoint matrices.
+// Dequantize the 'input' tensor into a float Tensor.
 //
-// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
-// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
+// [min_range, max_range] are scalar floats that specify the range for
+// the 'input' data. The 'mode' attribute controls exactly which calculations are
+// used to convert the float values to their quantized equivalents.
 //
-// ```python
-// # a is a tensor.
-// # e is a tensor of eigenvalues.
-// # v is a tensor of eigenvectors.
-// e, v = self_adjoint_eig(a)
-// e = self_adjoint_eig(a, compute_v=False)
+// In 'MIN_COMBINED' mode, each value of the tensor will undergo the following:
+//
+// ```
+// if T == qint8, in[i] += (range(T) + 1)/ 2.0
+// out[i] = min_range + (in[i]* (max_range - min_range) / range(T))
+// ```
+// here `range(T) = numeric_limits<T>::max() - numeric_limits<T>::min()`
+//
+// *MIN_COMBINED Mode Example*
+//
+// If the input comes from a QuantizedRelu6, the output type is
+// quint8 (range of 0-255) but the possible range of QuantizedRelu6 is
+// 0-6.  The min_range and max_range values are therefore 0.0 and 6.0.
+// Dequantize on quint8 will take each value, cast to float, and multiply
+// by 6 / 255.
+// Note that if quantizedtype is qint8, the operation will additionally add
+// each value by 128 prior to casting.
+//
+// If the mode is 'MIN_FIRST', then this approach is used:
+//
+// ```c++
+// num_discrete_values = 1 << (# of bits in T)
+// range_adjust = num_discrete_values / (num_discrete_values - 1)
+// range = (range_max - range_min) * range_adjust
+// range_scale = range / num_discrete_values
+// const double offset_input = static_cast<double>(input) - lowest_quantized;
+// result = range_min + ((input - numeric_limits<T>::min()) * range_scale)
+// ```
+//
+// *SCALED mode Example*
+//
+// `SCALED` mode matches the quantization approach used in
+// `QuantizeAndDequantize{V2|V3}`.
+//
+// If the mode is `SCALED`, we do not use the full range of the output type,
+// choosing to elide the lowest possible value for symmetry (e.g., output range is
+// -127 to 127, not -128 to 127 for signed 8 bit quantization), so that 0.0 maps to
+// 0.
+//
+// We first find the range of values in our tensor. The
+// range we use is always centered on 0, so we find m such that
+// ```c++
+//   m = max(abs(input_min), abs(input_max))
+// ```
+//
+// Our input tensor range is then `[-m, m]`.
+//
+// Next, we choose our fixed-point quantization buckets, `[min_fixed, max_fixed]`.
+// If T is signed, this is
+// ```
+//   num_bits = sizeof(T) * 8
+//   [min_fixed, max_fixed] =
+//       [-(1 << (num_bits - 1) - 1), (1 << (num_bits - 1)) - 1]
+// ```
+//
+// Otherwise, if T is unsigned, the fixed-point range is
+// ```
+//   [min_fixed, max_fixed] = [0, (1 << num_bits) - 1]
+// ```
+//
+// From this we compute our scaling factor, s:
+// ```c++
+//   s = (2 * m) / (max_fixed - min_fixed)
+// ```
+//
+// Now we can dequantize the elements of our tensor:
+// ```c++
+// result = input * s
 // ```
 //
 // Arguments:
-//	input: `Tensor` input of shape `[N, N]`.
 //
-// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
-func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
+//	min_range: The minimum scalar value possibly produced for the input.
+//	max_range: The maximum scalar value possibly produced for the input.
+func Dequantize(scope *Scope, input tf.Output, min_range tf.Output, max_range tf.Output, optional ...DequantizeAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -19132,147 +18795,115 @@ func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SelfAdjointEigV2",
+		Type: "Dequantize",
 		Input: []tf.Input{
-			input,
+			input, min_range, max_range,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Adjust the saturation of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A scale is then applied all the saturation
-// values, and then remapped back to RGB colorspace.
+// Flips all bits elementwise.
 //
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	scale: A float scale to add to the saturation.
-//
-// Returns The hue-adjusted image or images.
-func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
+// The result will have exactly those bits set, that are not set in `x`. The
+// computation is performed on the underlying representation of x.
+func Invert(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "AdjustSaturation",
+		Type: "Invert",
 		Input: []tf.Input{
-			images, scale,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// SvdAttr is an optional argument to Svd.
-type SvdAttr func(optionalAttr)
-
-// SvdComputeUv sets the optional compute_uv attribute to value.
+// Deprecated. Disallowed in GraphDef version >= 2.
 //
-// value: If true, left and right singular vectors will be
-// computed and returned in `u` and `v`, respectively.
-// If false, `u` and `v` are not set and should never referenced.
-// If not specified, defaults to true
-func SvdComputeUv(value bool) SvdAttr {
-	return func(m optionalAttr) {
-		m["compute_uv"] = value
+// DEPRECATED at GraphDef version 2: Use AdjustContrastv2 instead
+func AdjustContrast(scope *Scope, images tf.Output, contrast_factor tf.Output, min_value tf.Output, max_value tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// SvdFullMatrices sets the optional full_matrices attribute to value.
-//
-// value: If true, compute full-sized `u` and `v`. If false
-// (the default), compute only the leading `P` singular vectors.
-// Ignored if `compute_uv` is `False`.
-// If not specified, defaults to false
-func SvdFullMatrices(value bool) SvdAttr {
-	return func(m optionalAttr) {
-		m["full_matrices"] = value
+	opspec := tf.OpSpec{
+		Type: "AdjustContrast",
+		Input: []tf.Input{
+			images, contrast_factor, min_value, max_value,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the singular value decompositions of one or more matrices.
-//
-// Computes the SVD of each inner matrix in `input` such that
-// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
-//
-// ```python
-// # a is a tensor containing a batch of matrices.
-// # s is a tensor of singular values for each matrix.
-// # u is the tensor containing of left singular vectors for each matrix.
-// # v is the tensor containing of right singular vectors for each matrix.
-// s, u, v = svd(a)
-// s, _, _ = svd(a, compute_uv=False)
-// ```
+// Table initializer that takes two tensors for keys and values respectively.
 //
 // Arguments:
-//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
-// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//	table_handle: Handle to a table which will be initialized.
+//	keys: Keys of type Tkey.
+//	values: Values of type Tval.
 //
-// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., M, P]`; if `full_matrices` is `True` then shape is
-// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
-// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
-// Undefined if `compute_uv` is false.
-func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
+// Returns the created operation.
+func InitializeTableV2(scope *Scope, table_handle tf.Output, keys tf.Output, values tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Svd",
+		Type: "InitializeTableV2",
 		Input: []tf.Input{
-			input,
+			table_handle, keys, values,
 		},
-		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return scope.AddOperation(opspec)
 }
 
-// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
-type QueueEnqueueManyV2Attr func(optionalAttr)
+// PrintAttr is an optional argument to Print.
+type PrintAttr func(optionalAttr)
 
-// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
+// PrintMessage sets the optional message attribute to value.
 //
-// value: If the queue is too full, this operation will block for up
-// to timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
+// value: A string, prefix of the error message.
+// If not specified, defaults to ""
+func PrintMessage(value string) PrintAttr {
 	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+		m["message"] = value
 	}
 }
 
-// Enqueues zero or more tuples of one or more tensors in the given queue.
+// PrintFirstN sets the optional first_n attribute to value.
 //
-// This operation slices each component tensor along the 0th dimension to
-// make multiple queue elements. All of the tuple components must have the
-// same size in the 0th dimension.
+// value: Only log `first_n` number of times. -1 disables logging.
+// If not specified, defaults to -1
+func PrintFirstN(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["first_n"] = value
+	}
+}
+
+// PrintSummarize sets the optional summarize attribute to value.
 //
-// The components input has k elements, which correspond to the components of
-// tuples stored in the given queue.
+// value: Only print this many entries of each tensor.
+// If not specified, defaults to 3
+func PrintSummarize(value int64) PrintAttr {
+	return func(m optionalAttr) {
+		m["summarize"] = value
+	}
+}
+
+// Prints a list of tensors.
 //
-// N.B. If the queue is full, this operation will block until the given
-// elements have been enqueued (or 'timeout_ms' elapses, if specified).
+// Passes `input` through to `output` and prints `data` when evaluating.
 //
 // Arguments:
-//	handle: The handle to a queue.
-//	components: One or more tensors from which the enqueued tensors should
-// be taken.
+//	input: The tensor passed to `output`
+//	data: A list of tensors to print out when op is evaluated.
 //
-// Returns the created operation.
-func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
+// Returns = The unmodified `input` tensor
+func Print(scope *Scope, input tf.Output, data []tf.Output, optional ...PrintAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -19281,123 +18912,103 @@ func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QueueEnqueueManyV2",
+		Type: "Print",
 		Input: []tf.Input{
-			handle, tf.OutputList(components),
+			input, tf.OutputList(data),
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Computes the product along segments of a tensor.
-//
-// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
-// segments.
-//
-// Computes a tensor such that
-// \\(output_i = \prod_j data_j\\) where the product is over `j` such
-// that `segment_ids[j] == i`.
-//
-// If the product is empty for a given segment ID `i`, `output[i] = 1`.
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
-// </div>
+// Outputs a `Summary` protocol buffer with a tensor and per-plugin data.
 //
 // Arguments:
-//
-//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
-// first dimension.  Values should be sorted and can be repeated.
-//
-// Returns Has same shape as data, except for dimension 0 which
-// has size `k`, the number of segments.
-func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
+//	tag: A string attached to this summary. Used for organization in TensorBoard.
+//	tensor: A tensor to serialize.
+//	serialized_summary_metadata: A serialized SummaryMetadata proto. Contains plugin
+// data.
+func TensorSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, serialized_summary_metadata tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SegmentProd",
+		Type: "TensorSummaryV2",
 		Input: []tf.Input{
-			data, segment_ids,
+			tag, tensor, serialized_summary_metadata,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Converts one or more images from RGB to HSV.
+// Creates a dataset that asynchronously prefetches elements from `input_dataset`.
 //
-// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
-// value of the pixels. The output is only well defined if the value in `images`
-// are in `[0,1]`.
+// Arguments:
 //
-// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
-// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
-// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
+//	buffer_size: The maximum number of elements to buffer in an iterator over
+// this dataset.
 //
-// Arguments:
-//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
 //
-// Returns `images` converted to HSV.
-func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
+func PrefetchDataset(scope *Scope, input_dataset tf.Output, buffer_size tf.Output, output_types []tf.DataType, output_shapes []tf.Shape) (handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"output_types": output_types, "output_shapes": output_shapes}
 	opspec := tf.OpSpec{
-		Type: "RGBToHSV",
+		Type: "PrefetchDataset",
 		Input: []tf.Input{
-			images,
+			input_dataset, buffer_size,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Does nothing. Only useful as a placeholder for control edges.
+// TensorSummaryAttr is an optional argument to TensorSummary.
+type TensorSummaryAttr func(optionalAttr)
+
+// TensorSummaryDescription sets the optional description attribute to value.
 //
-// Returns the created operation.
-func NoOp(scope *Scope) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "NoOp",
+// value: A json-encoded SummaryDescription proto.
+// If not specified, defaults to ""
+func TensorSummaryDescription(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["description"] = value
 	}
-	return scope.AddOperation(opspec)
 }
 
-// MergeV2CheckpointsAttr is an optional argument to MergeV2Checkpoints.
-type MergeV2CheckpointsAttr func(optionalAttr)
-
-// MergeV2CheckpointsDeleteOldDirs sets the optional delete_old_dirs attribute to value.
+// TensorSummaryLabels sets the optional labels attribute to value.
 //
-// value: see above.
-// If not specified, defaults to true
-func MergeV2CheckpointsDeleteOldDirs(value bool) MergeV2CheckpointsAttr {
+// value: An unused list of strings.
+// If not specified, defaults to <>
+func TensorSummaryLabels(value []string) TensorSummaryAttr {
 	return func(m optionalAttr) {
-		m["delete_old_dirs"] = value
+		m["labels"] = value
 	}
 }
 
-// V2 format specific: merges the metadata files of sharded checkpoints.  The
-//
-// result is one logical checkpoint, with one physical metadata file and renamed
-// data files.
+// TensorSummaryDisplayName sets the optional display_name attribute to value.
 //
-// Intended for "grouping" multiple checkpoints in a sharded checkpoint setup.
+// value: An unused string.
+// If not specified, defaults to ""
+func TensorSummaryDisplayName(value string) TensorSummaryAttr {
+	return func(m optionalAttr) {
+		m["display_name"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with a tensor.
 //
-// If delete_old_dirs is true, attempts to delete recursively the dirname of each
-// path in the input checkpoint_prefixes.  This is useful when those paths are non
-// user-facing temporary locations.
+// This op is being phased out in favor of TensorSummaryV2, which lets callers pass
+// a tag as well as a serialized SummaryMetadata proto string that contains
+// plugin-specific data. We will keep this op to maintain backwards compatibility.
 //
 // Arguments:
-//	checkpoint_prefixes: prefixes of V2 checkpoints to merge.
-//	destination_prefix: scalar.  The desired final prefix.  Allowed to be the same
-// as one of the checkpoint_prefixes.
-//
-// Returns the created operation.
-func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination_prefix tf.Output, optional ...MergeV2CheckpointsAttr) (o *tf.Operation) {
+//	tensor: A tensor to serialize.
+func TensorSummary(scope *Scope, tensor tf.Output, optional ...TensorSummaryAttr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -19406,524 +19017,411 @@ func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "MergeV2Checkpoints",
+		Type: "TensorSummary",
 		Input: []tf.Input{
-			checkpoint_prefixes, destination_prefix,
+			tensor,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Saves input tensors slices to disk.
-//
-// This is like `Save` except that tensors can be listed in the saved file as being
-// a slice of a larger tensor.  `shapes_and_slices` specifies the shape of the
-// larger tensor and the slice that this tensor covers. `shapes_and_slices` must
-// have as many elements as `tensor_names`.
-//
-// Elements of the `shapes_and_slices` input must either be:
-//
-// *  The empty string, in which case the corresponding tensor is
-//    saved normally.
-// *  A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the
-//    `dimI` are the dimensions of the larger tensor and `slice-spec`
-//    specifies what part is covered by the tensor to save.
-//
-// `slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1`
-// where each `sliceI` is either:
+// Computes the gradient for the tanh of `x` wrt its input.
 //
-// *  The string `-` meaning that the slice covers all indices of this dimension
-// *  `start,length` where `start` and `length` are integers.  In that
-//    case the slice covers `length` indices starting at `start`.
+// Specifically, `grad = dy * (1 - y*y)`, where `y = tanh(x)`, and `dy`
+// is the corresponding input gradient.
+func TanhGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "TanhGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Outputs a `Summary` protocol buffer with scalar values.
 //
-// See also `Save`.
+// The input `tags` and `values` must have the same shape.  The generated summary
+// has a summary value for each tag-value pair in `tags` and `values`.
 //
 // Arguments:
-//	filename: Must have a single element. The name of the file to which we write the
-// tensor.
-//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
-//	shapes_and_slices: Shape `[N]`.  The shapes and slice specifications to use when
-// saving the tensors.
-//	data: `N` tensors to save.
+//	tags: Tags for the summary.
+//	values: Same shape as `tags.  Values for the summary.
 //
-// Returns the created operation.
-func SaveSlices(scope *Scope, filename tf.Output, tensor_names tf.Output, shapes_and_slices tf.Output, data []tf.Output) (o *tf.Operation) {
+// Returns Scalar.  Serialized `Summary` protocol buffer.
+func ScalarSummary(scope *Scope, tags tf.Output, values tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SaveSlices",
+		Type: "ScalarSummary",
 		Input: []tf.Input{
-			filename, tensor_names, shapes_and_slices, tf.OutputList(data),
+			tags, values,
 		},
 	}
-	return scope.AddOperation(opspec)
-}
-
-// DenseToDenseSetOperationAttr is an optional argument to DenseToDenseSetOperation.
-type DenseToDenseSetOperationAttr func(optionalAttr)
-
-// DenseToDenseSetOperationValidateIndices sets the optional validate_indices attribute to value.
-// If not specified, defaults to true
-func DenseToDenseSetOperationValidateIndices(value bool) DenseToDenseSetOperationAttr {
-	return func(m optionalAttr) {
-		m["validate_indices"] = value
-	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Applies set operation along last dimension of 2 `Tensor` inputs.
+// Outputs a `Summary` protocol buffer with a histogram.
 //
-// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
+// The generated
+// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+// has one summary value containing a histogram for `values`.
 //
-// Output `result` is a `SparseTensor` represented by `result_indices`,
-// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
-// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
-// dimension contains the result of `set_operation` applied to the corresponding
-// `[0...n-1]` dimension of `set`.
+// This op reports an `InvalidArgument` error if any value is not finite.
 //
 // Arguments:
-//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
-//	set2: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set1`.
-// Dimension `n` contains values in a set, duplicates are allowed but ignored.
-//
+//	tag: Scalar.  Tag to use for the `Summary.Value`.
+//	values: Any shape. Values to use to build the histogram.
 //
-// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
-// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
-// is the max result set size across all `0...n-1` dimensions.
-func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_operation string, optional ...DenseToDenseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func HistogramSummary(scope *Scope, tag tf.Output, values tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"set_operation": set_operation}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DenseToDenseSetOperation",
+		Type: "HistogramSummary",
 		Input: []tf.Input{
-			set1, set2,
+			tag, values,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0)
 }
 
-// Generate a sharded filename. The filename is printf formatted as
+// Computes the number of elements in the given queue.
 //
-//    %s-%05d-of-%05d, basename, shard, num_shards.
-func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
+// Arguments:
+//	handle: The handle to a queue.
+//
+// Returns The number of elements in the given queue.
+func QueueSizeV2(scope *Scope, handle tf.Output) (size tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ShardedFilename",
+		Type: "QueueSizeV2",
 		Input: []tf.Input{
-			basename, shard, num_shards,
+			handle,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// BatchToSpace for N-D tensors of type T.
-//
-// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
-// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
-// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
-// the input.  The spatial dimensions of this intermediate result are then
-// optionally cropped according to `crops` to produce the output.  This is the
-// reverse of SpaceToBatch.  See below for a precise description.
+// ImageSummaryAttr is an optional argument to ImageSummary.
+type ImageSummaryAttr func(optionalAttr)
+
+// ImageSummaryMaxImages sets the optional max_images attribute to value.
 //
-// Arguments:
-//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
-// where spatial_shape has M dimensions.
-//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
-//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
-//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
-//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
-//   required that
-//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
+// value: Max number of batch elements to generate images for.
+// If not specified, defaults to 3
 //
-// This operation is equivalent to the following steps:
+// REQUIRES: value >= 1
+func ImageSummaryMaxImages(value int64) ImageSummaryAttr {
+	return func(m optionalAttr) {
+		m["max_images"] = value
+	}
+}
+
+// ImageSummaryBadColor sets the optional bad_color attribute to value.
 //
-// 1. Reshape `input` to `reshaped` of shape:
-//      [block_shape[0], ..., block_shape[M-1],
-//       batch / prod(block_shape),
-//       input_shape[1], ..., input_shape[N-1]]
+// value: Color to use for pixels with non-finite values.
+// If not specified, defaults to <dtype:DT_UINT8 tensor_shape:<dim:<size:4 > > int_val:255 int_val:0 int_val:0 int_val:255 >
+func ImageSummaryBadColor(value tf.Tensor) ImageSummaryAttr {
+	return func(m optionalAttr) {
+		m["bad_color"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with images.
 //
-// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
-//      [batch / prod(block_shape),
+// The summary has up to `max_images` summary values containing images. The
+// images are built from `tensor` which must be 4-D with shape `[batch_size,
+// height, width, channels]` and where `channels` can be:
 //
-//       input_shape[1], block_shape[0],
-//       ...,
-//       input_shape[M], block_shape[M-1],
+// *  1: `tensor` is interpreted as Grayscale.
+// *  3: `tensor` is interpreted as RGB.
+// *  4: `tensor` is interpreted as RGBA.
 //
-//       input_shape[M+1], ..., input_shape[N-1]]
+// The images have the same number of channels as the input tensor. For float
+// input, the values are normalized one image at a time to fit in the range
+// `[0, 255]`.  `uint8` values are unchanged.  The op uses two different
+// normalization algorithms:
 //
-// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
-//      [batch / prod(block_shape),
+// *  If the input values are all positive, they are rescaled so the largest one
+//    is 255.
 //
-//       input_shape[1] * block_shape[0],
-//       ...,
-//       input_shape[M] * block_shape[M-1],
+// *  If any input value is negative, the values are shifted so input value 0.0
+//    is at 127.  They are then rescaled so that either the smallest value is 0,
+//    or the largest one is 255.
 //
-//       input_shape[M+1],
-//       ...,
-//       input_shape[N-1]]
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
 //
-// 4. Crop the start and end of dimensions `[1, ..., M]` of
-//    `reshaped_permuted` according to `crops` to produce the output of shape:
-//      [batch / prod(block_shape),
+// *  If `max_images` is 1, the summary value tag is '*tag*/image'.
+// *  If `max_images` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/image/0', '*tag*/image/1', etc.
 //
-//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
-//       ...,
-//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+// The `bad_color` argument is the color to use in the generated images for
+// non-finite input values.  It is a `unit8` 1-D tensor of length `channels`.
+// Each element must be in the range `[0, 255]` (It represents the value of a
+// pixel in the output image).  Non-finite values in the input tensor are
+// replaced by this tensor in the output image.  The default value is the color
+// red.
 //
-//       input_shape[M+1], ..., input_shape[N-1]]
+// Arguments:
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 4-D of shape `[batch_size, height, width, channels]` where
+// `channels` is 1, 3, or 4.
 //
-// Some examples:
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func ImageSummary(scope *Scope, tag tf.Output, tensor tf.Output, optional ...ImageSummaryAttr) (summary tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ImageSummary",
+		Input: []tf.Input{
+			tag, tensor,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// AudioSummaryV2Attr is an optional argument to AudioSummaryV2.
+type AudioSummaryV2Attr func(optionalAttr)
+
+// AudioSummaryV2MaxOutputs sets the optional max_outputs attribute to value.
 //
-// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
+// value: Max number of batch elements to generate audio for.
+// If not specified, defaults to 3
 //
-// ```
-// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 1]` and value:
-//
-// ```
-// x = [[[[1], [2]], [[3], [4]]]]
-// ```
-//
-// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
-// ```
-//
-// The output tensor has shape `[1, 2, 2, 3]` and value:
-//
-// ```
-// x = [[[[1, 2, 3], [4, 5, 6]],
-//       [[7, 8, 9], [10, 11, 12]]]]
-// ```
-//
-// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [0, 0]]`:
-//
-// ```
-// x = [[[[1], [3]], [[9], [11]]],
-//      [[[2], [4]], [[10], [12]]],
-//      [[[5], [7]], [[13], [15]]],
-//      [[[6], [8]], [[14], [16]]]]
-// ```
-//
-// The output tensor has shape `[1, 4, 4, 1]` and value:
+// REQUIRES: value >= 1
+func AudioSummaryV2MaxOutputs(value int64) AudioSummaryV2Attr {
+	return func(m optionalAttr) {
+		m["max_outputs"] = value
+	}
+}
+
+// Outputs a `Summary` protocol buffer with audio.
 //
-// ```
-// x = [[[1],   [2],  [3],  [4]],
-//      [[5],   [6],  [7],  [8]],
-//      [[9],  [10], [11],  [12]],
-//      [[13], [14], [15],  [16]]]
-// ```
+// The summary has up to `max_outputs` summary values containing audio. The
+// audio is built from `tensor` which must be 3-D with shape `[batch_size,
+// frames, channels]` or 2-D with shape `[batch_size, frames]`. The values are
+// assumed to be in the range of `[-1.0, 1.0]` with a sample rate of `sample_rate`.
 //
-// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
-//     `crops = [[0, 0], [2, 0]]`:
+// The `tag` argument is a scalar `Tensor` of type `string`.  It is used to
+// build the `tag` of the summary values:
 //
-// ```
-// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
-//      [[[0], [2], [4]]], [[[0], [10], [12]]],
-//      [[[0], [5], [7]]], [[[0], [13], [15]]],
-//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
-// ```
+// *  If `max_outputs` is 1, the summary value tag is '*tag*/audio'.
+// *  If `max_outputs` is greater than 1, the summary value tags are
+//    generated sequentially as '*tag*/audio/0', '*tag*/audio/1', etc.
 //
-// The output tensor has shape `[2, 2, 4, 1]` and value:
+// Arguments:
+//	tag: Scalar. Used to build the `tag` attribute of the summary values.
+//	tensor: 2-D of shape `[batch_size, frames]`.
+//	sample_rate: The sample rate of the signal in hertz.
 //
-// ```
-// x = [[[[1],   [2],  [3],  [4]],
-//       [[5],   [6],  [7],  [8]]],
-//      [[[9],  [10], [11],  [12]],
-//       [[13], [14], [15],  [16]]]]
-// ```
-func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func AudioSummaryV2(scope *Scope, tag tf.Output, tensor tf.Output, sample_rate tf.Output, optional ...AudioSummaryV2Attr) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "BatchToSpaceND",
+		Type: "AudioSummaryV2",
 		Input: []tf.Input{
-			input, block_shape, crops,
+			tag, tensor, sample_rate,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// UnpackAttr is an optional argument to Unpack.
-type UnpackAttr func(optionalAttr)
+// AvgPoolAttr is an optional argument to AvgPool.
+type AvgPoolAttr func(optionalAttr)
 
-// UnpackAxis sets the optional axis attribute to value.
+// AvgPoolDataFormat sets the optional data_format attribute to value.
 //
-// value: Dimension along which to unpack.  Negative values wrap around, so the
-// valid range is `[-R, R)`.
-// If not specified, defaults to 0
-func UnpackAxis(value int64) UnpackAttr {
+// value: Specify the data format of the input and output data. With the
+// default format "NHWC", the data is stored in the order of:
+//     [batch, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCHW", the data storage order of:
+//     [batch, in_channels, in_height, in_width].
+// If not specified, defaults to "NHWC"
+func AvgPoolDataFormat(value string) AvgPoolAttr {
 	return func(m optionalAttr) {
-		m["axis"] = value
+		m["data_format"] = value
 	}
 }
 
-// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
-//
-// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
-// For example, given a tensor of shape `(A, B, C, D)`;
-//
-// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
-//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
-//   dimension unpacked along is gone, unlike `split`).
-//
-// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
-//   and each tensor in `output` will have shape `(A, C, D)`.
-// Etc.
+// Performs average pooling on the input.
 //
-// This is the opposite of `pack`.
+// Each entry in `output` is the mean of the corresponding size `ksize`
+// window in `value`.
 //
 // Arguments:
-//	value: 1-D or higher, with `axis` dimension size equal to `num`.
-//
+//	value: 4-D with shape `[batch, height, width, channels]`.
+//	ksize: The size of the sliding window for each dimension of `value`.
+//	strides: The stride of the sliding window for each dimension of `value`.
+//	padding: The type of padding algorithm to use.
 //
-// Returns The list of tensors unpacked from `value`.
-func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
+// Returns The average pooled output tensor.
+func AvgPool(scope *Scope, value tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPoolAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num": num}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "Unpack",
+		Type: "AvgPool",
 		Input: []tf.Input{
 			value,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Unpack", err)
-		return
-	}
-	return output
+	return op.Output(0)
 }
 
-// Increments variable pointed to by 'resource' until it reaches 'limit'.
+// Merges summaries.
 //
-// Arguments:
-//	resource: Should be from a scalar `Variable` node.
-//	limit: If incrementing ref would bring it above limit, instead generates an
-// 'OutOfRange' error.
+// This op creates a
+// [`Summary`](https://www.tensorflow.org/code/tensorflow/core/framework/summary.proto)
+// protocol buffer that contains the union of all the values in the input
+// summaries.
+//
+// When the Op is run, it reports an `InvalidArgument` error if multiple values
+// in the summaries to merge use the same tag.
 //
+// Arguments:
+//	inputs: Can be of any shape.  Each must contain serialized `Summary` protocol
+// buffers.
 //
-// Returns A copy of the input before increment. If nothing else modifies the
-// input, the values produced will all be distinct.
-func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
+// Returns Scalar. Serialized `Summary` protocol buffer.
+func MergeSummary(scope *Scope, inputs []tf.Output) (summary tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"limit": limit, "T": T}
 	opspec := tf.OpSpec{
-		Type: "ResourceCountUpTo",
+		Type: "MergeSummary",
 		Input: []tf.Input{
-			resource,
+			tf.OutputList(inputs),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Delete the stack from its resource container.
+// Computes the gradient of morphological 2-D dilation with respect to the filter.
 //
 // Arguments:
-//	handle: The handle to a stack.
+//	input: 4-D with shape `[batch, in_height, in_width, depth]`.
+//	filter: 3-D with shape `[filter_height, filter_width, depth]`.
+//	out_backprop: 4-D with shape `[batch, out_height, out_width, depth]`.
+//	strides: 1-D of length 4. The stride of the sliding window for each dimension of
+// the input tensor. Must be: `[1, stride_height, stride_width, 1]`.
+//	rates: 1-D of length 4. The input stride for atrous morphological dilation.
+// Must be: `[1, rate_height, rate_width, 1]`.
+//	padding: The type of padding algorithm to use.
 //
-// Returns the created operation.
-func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "StackCloseV2",
-		Input: []tf.Input{
-			handle,
-		},
-	}
-	return scope.AddOperation(opspec)
-}
-
-// Generate a glob pattern matching all sharded file names.
-func ShardedFilespec(scope *Scope, basename tf.Output, num_shards tf.Output) (filename tf.Output) {
+// Returns 3-D with shape `[filter_height, filter_width, depth]`.
+func Dilation2DBackpropFilter(scope *Scope, input tf.Output, filter tf.Output, out_backprop tf.Output, strides []int64, rates []int64, padding string) (filter_backprop tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"strides": strides, "rates": rates, "padding": padding}
 	opspec := tf.OpSpec{
-		Type: "ShardedFilespec",
+		Type: "Dilation2DBackpropFilter",
 		Input: []tf.Input{
-			basename, num_shards,
+			input, filter, out_backprop,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// TextLineReaderV2Attr is an optional argument to TextLineReaderV2.
-type TextLineReaderV2Attr func(optionalAttr)
-
-// TextLineReaderV2SkipHeaderLines sets the optional skip_header_lines attribute to value.
-//
-// value: Number of lines to skip from the beginning of every file.
-// If not specified, defaults to 0
-func TextLineReaderV2SkipHeaderLines(value int64) TextLineReaderV2Attr {
-	return func(m optionalAttr) {
-		m["skip_header_lines"] = value
-	}
-}
+// AddSparseToTensorsMapAttr is an optional argument to AddSparseToTensorsMap.
+type AddSparseToTensorsMapAttr func(optionalAttr)
 
-// TextLineReaderV2Container sets the optional container attribute to value.
+// AddSparseToTensorsMapContainer sets the optional container attribute to value.
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
+// value: The container name for the `SparseTensorsMap` created by this op.
 // If not specified, defaults to ""
-func TextLineReaderV2Container(value string) TextLineReaderV2Attr {
+func AddSparseToTensorsMapContainer(value string) AddSparseToTensorsMapAttr {
 	return func(m optionalAttr) {
 		m["container"] = value
 	}
 }
 
-// TextLineReaderV2SharedName sets the optional shared_name attribute to value.
+// AddSparseToTensorsMapSharedName sets the optional shared_name attribute to value.
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
+// value: The shared name for the `SparseTensorsMap` created by this op.
+// If blank, the new Operation's unique name is used.
 // If not specified, defaults to ""
-func TextLineReaderV2SharedName(value string) TextLineReaderV2Attr {
+func AddSparseToTensorsMapSharedName(value string) AddSparseToTensorsMapAttr {
 	return func(m optionalAttr) {
 		m["shared_name"] = value
 	}
 }
 
-// A Reader that outputs the lines of a file delimited by '\n'.
+// Add a `SparseTensor` to a `SparseTensorsMap` return its handle.
 //
-// Returns The handle to reference the Reader.
-func TextLineReaderV2(scope *Scope, optional ...TextLineReaderV2Attr) (reader_handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TextLineReaderV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// LoadAndRemapMatrixAttr is an optional argument to LoadAndRemapMatrix.
-type LoadAndRemapMatrixAttr func(optionalAttr)
-
-// LoadAndRemapMatrixMaxRowsInMemory sets the optional max_rows_in_memory attribute to value.
-//
-// value: The maximum number of rows to load from the checkpoint at
-// once. If less than or equal to 0, the entire matrix will be loaded into
-// memory. Setting this arg trades increased disk reads for lower memory usage.
-// If not specified, defaults to -1
-func LoadAndRemapMatrixMaxRowsInMemory(value int64) LoadAndRemapMatrixAttr {
-	return func(m optionalAttr) {
-		m["max_rows_in_memory"] = value
-	}
-}
-
-// Loads a 2-D (matrix) `Tensor` with name `old_tensor_name` from the checkpoint
-//
-// at `ckpt_path` and potentially reorders its rows and columns using the
-// specified remappings.
-//
-// Most users should use one of the wrapper initializers (such as
-// `tf.contrib.framework.load_and_remap_matrix_initializer`) instead of this
-// function directly.
-//
-// The remappings are 1-D tensors with the following properties:
-//
-// * `row_remapping` must have exactly `num_rows` entries. Row `i` of the output
-//   matrix will be initialized from the row corresponding to index
-//   `row_remapping[i]` in the old `Tensor` from the checkpoint.
-// * `col_remapping` must have either 0 entries (indicating that no column
-//   reordering is needed) or `num_cols` entries. If specified, column `j` of the
-//   output matrix will be initialized from the column corresponding to index
-//   `col_remapping[j]` in the old `Tensor` from the checkpoint.
-// * A value of -1 in either of the remappings signifies a "missing" entry. In that
-//   case, values from the `initializing_values` tensor will be used to fill that
-//   missing row or column. If `row_remapping` has `r` missing entries and
-//   `col_remapping` has `c` missing entries, then the following condition must be
-//   true:
-//
-// `(r * num_cols) + (c * num_rows) - (r * c) == len(initializing_values)`
-//
-// The remapping tensors can be generated using the GenerateVocabRemapping op.
+// A `SparseTensor` is represented by three tensors: `sparse_indices`,
+// `sparse_values`, and `sparse_shape`.
 //
-// As an example, with row_remapping = [1, 0, -1], col_remapping = [0, 2, -1],
-// initializing_values = [0.5, -0.5, 0.25, -0.25, 42], and w(i, j) representing
-// the value from row i, column j of the old tensor in the checkpoint, the output
-// matrix will look like the following:
+// This operator takes the given `SparseTensor` and adds it to a container
+// object (a `SparseTensorsMap`).  A unique key within this container is generated
+// in the form of an `int64`, and this is the value that is returned.
 //
-// [[w(1, 0),  w(1, 2),  0.5],
-//  [w(0, 0),  w(0, 2), -0.5],
-//  [0.25,    -0.25,      42]]
+// The `SparseTensor` can then be read out as part of a minibatch by passing
+// the key as a vector element to `TakeManySparseFromTensorsMap`.  To ensure
+// the correct `SparseTensorsMap` is accessed, ensure that the same
+// `container` and `shared_name` are passed to that Op.  If no `shared_name`
+// is provided here, instead use the *name* of the Operation created by calling
+// `AddSparseToTensorsMap` as the `shared_name` passed to
+// `TakeManySparseFromTensorsMap`.  Ensure the Operations are colocated.
 //
 // Arguments:
-//	ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) from
-// which the old matrix `Tensor` will be loaded.
-//	old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint.
-//	row_remapping: An int `Tensor` of row remappings (generally created by
-// `generate_vocab_remapping`).  Even if no row remapping is needed, this must
-// still be an index-valued Tensor (e.g. [0, 1, 2, ...]), or a shifted
-// index-valued `Tensor` (e.g. [8, 9, 10, ...], for partitioned `Variables`).
-//	col_remapping: An int `Tensor` of column remappings (generally created by
-// `generate_vocab_remapping`).  May be a size-0 `Tensor` if only row remapping
-// is to be done (e.g. column ordering is the same).
-//	initializing_values: A float `Tensor` containing  values to fill in for cells
-// in the output matrix that are not loaded from the checkpoint. Length must be
-// exactly the same as the number of missing / new cells.
-//	num_rows: Number of rows (length of the 1st dimension) in the output matrix.
-//	num_cols: Number of columns (length of the 2nd dimension) in the output matrix.
+//	sparse_indices: 2-D.  The `indices` of the `SparseTensor`.
+//	sparse_values: 1-D.  The `values` of the `SparseTensor`.
+//	sparse_shape: 1-D.  The `shape` of the `SparseTensor`.
 //
-// Returns Output matrix containing existing values loaded from the
-// checkpoint, and with any missing values filled in from initializing_values.
-func LoadAndRemapMatrix(scope *Scope, ckpt_path tf.Output, old_tensor_name tf.Output, row_remapping tf.Output, col_remapping tf.Output, initializing_values tf.Output, num_rows int64, num_cols int64, optional ...LoadAndRemapMatrixAttr) (output_matrix tf.Output) {
+// Returns 0-D.  The handle of the `SparseTensor` now stored in the
+// `SparseTensorsMap`.
+func AddSparseToTensorsMap(scope *Scope, sparse_indices tf.Output, sparse_values tf.Output, sparse_shape tf.Output, optional ...AddSparseToTensorsMapAttr) (sparse_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_rows": num_rows, "num_cols": num_cols}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "LoadAndRemapMatrix",
+		Type: "AddSparseToTensorsMap",
 		Input: []tf.Input{
-			ckpt_path, old_tensor_name, row_remapping, col_remapping, initializing_values,
+			sparse_indices, sparse_values, sparse_shape,
 		},
 		Attrs: attrs,
 	}
@@ -19931,306 +19429,293 @@ func LoadAndRemapMatrix(scope *Scope, ckpt_path tf.Output, old_tensor_name tf.Ou
 	return op.Output(0)
 }
 
-// TFRecordReaderV2Attr is an optional argument to TFRecordReaderV2.
-type TFRecordReaderV2Attr func(optionalAttr)
-
-// TFRecordReaderV2Container sets the optional container attribute to value.
+// Computes the matrix exponential of one or more square matrices:
 //
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func TFRecordReaderV2Container(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
-
-// TFRecordReaderV2SharedName sets the optional shared_name attribute to value.
+// exp(A) = \sum_{n=0}^\infty A^n/n!
 //
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func TFRecordReaderV2SharedName(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
-	}
-}
-
-// TFRecordReaderV2CompressionType sets the optional compression_type attribute to value.
-// If not specified, defaults to ""
-func TFRecordReaderV2CompressionType(value string) TFRecordReaderV2Attr {
-	return func(m optionalAttr) {
-		m["compression_type"] = value
-	}
-}
-
-// A Reader that outputs the records from a TensorFlow Records file.
+// The exponential is computed using a combination of the scaling and squaring
+// method and the Pade approximation. Details can be founds in:
+// Nicholas J. Higham, "The scaling and squaring method for the matrix exponential
+// revisited," SIAM J. Matrix Anal. Applic., 26:1179-1193, 2005.
 //
-// Returns The handle to reference the Reader.
-func TFRecordReaderV2(scope *Scope, optional ...TFRecordReaderV2Attr) (reader_handle tf.Output) {
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices. The output is a tensor of the same shape as the input
+// containing the exponential for all input submatrices `[..., :, :]`.
+//
+// Arguments:
+//	input: Shape is `[..., M, M]`.
+//
+// Returns Shape is `[..., M, M]`.
+//
+// @compatibility(scipy)
+// Equivalent to scipy.linalg.expm
+// @end_compatibility
+func MatrixExponential(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "TFRecordReaderV2",
-
-		Attrs: attrs,
+		Type: "MatrixExponential",
+		Input: []tf.Input{
+			input,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizeAndDequantizeV3Attr is an optional argument to QuantizeAndDequantizeV3.
-type QuantizeAndDequantizeV3Attr func(optionalAttr)
-
-// QuantizeAndDequantizeV3SignedInput sets the optional signed_input attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeV3SignedInput(value bool) QuantizeAndDequantizeV3Attr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
+// QueueDequeueUpToV2Attr is an optional argument to QueueDequeueUpToV2.
+type QueueDequeueUpToV2Attr func(optionalAttr)
 
-// QuantizeAndDequantizeV3RangeGiven sets the optional range_given attribute to value.
-// If not specified, defaults to true
-func QuantizeAndDequantizeV3RangeGiven(value bool) QuantizeAndDequantizeV3Attr {
+// QueueDequeueUpToV2TimeoutMs sets the optional timeout_ms attribute to value.
+//
+// value: If the queue has fewer than n elements, this operation
+// will block for up to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueDequeueUpToV2TimeoutMs(value int64) QueueDequeueUpToV2Attr {
 	return func(m optionalAttr) {
-		m["range_given"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Quantizes then dequantizes a tensor.
+// Dequeues `n` tuples of one or more tensors from the given queue.
 //
-// This is almost identical to QuantizeAndDequantizeV2, except that num_bits is a
-// tensor, so its value can change during training.
-func QuantizeAndDequantizeV3(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, num_bits tf.Output, optional ...QuantizeAndDequantizeV3Attr) (output tf.Output) {
+// This operation is not supported by all queues.  If a queue does not support
+// DequeueUpTo, then an Unimplemented error is returned.
+//
+// If the queue is closed and there are more than 0 but less than `n`
+// elements remaining, then instead of returning an OutOfRange error like
+// QueueDequeueMany, less than `n` elements are returned immediately.  If
+// the queue is closed and there are 0 elements left in the queue, then
+// an OutOfRange error is returned just like in QueueDequeueMany.
+// Otherwise the behavior is identical to QueueDequeueMany:
+//
+// This operation concatenates queue-element component tensors along the
+// 0th dimension to make a single component tensor.  All of the components
+// in the dequeued tuple will have size n in the 0th dimension.
+//
+// This operation has `k` outputs, where `k` is the number of components in
+// the tuples stored in the given queue, and output `i` is the ith
+// component of the dequeued tuple.
+//
+// Arguments:
+//	handle: The handle to a queue.
+//	n: The number of tuples to dequeue.
+//	component_types: The type of each component in a tuple.
+//
+// Returns One or more tensors that were dequeued as a tuple.
+func QueueDequeueUpToV2(scope *Scope, handle tf.Output, n tf.Output, component_types []tf.DataType, optional ...QueueDequeueUpToV2Attr) (components []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"component_types": component_types}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantizeV3",
+		Type: "QueueDequeueUpToV2",
 		Input: []tf.Input{
-			input, input_min, input_max, num_bits,
+			handle, n,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// IdentityReaderV2Attr is an optional argument to IdentityReaderV2.
-type IdentityReaderV2Attr func(optionalAttr)
-
-// IdentityReaderV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this reader is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func IdentityReaderV2Container(value string) IdentityReaderV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// IdentityReaderV2SharedName sets the optional shared_name attribute to value.
-//
-// value: If non-empty, this reader is named in the given bucket
-// with this shared_name. Otherwise, the node name is used instead.
-// If not specified, defaults to ""
-func IdentityReaderV2SharedName(value string) IdentityReaderV2Attr {
-	return func(m optionalAttr) {
-		m["shared_name"] = value
+	var idx int
+	var err error
+	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
+		scope.UpdateErr("QueueDequeueUpToV2", err)
+		return
 	}
+	return components
 }
 
-// A Reader that outputs the queued work as both the key and value.
+// Computes the Cholesky decomposition of one or more square matrices.
 //
-// To use, enqueue strings in a Queue.  ReaderRead will take the front
-// work string and output (work, work).
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices.
 //
-// Returns The handle to reference the Reader.
-func IdentityReaderV2(scope *Scope, optional ...IdentityReaderV2Attr) (reader_handle tf.Output) {
+// The input has to be symmetric and positive definite. Only the lower-triangular
+// part of the input will be used for this operation. The upper-triangular part
+// will not be read.
+//
+// The output is a tensor of the same shape as the input
+// containing the Cholesky decompositions for all input submatrices `[..., :, :]`.
+//
+// **Note**: The gradient computation on GPU is faster for large matrices but
+// not for large batch dimensions when the submatrices are small. In this
+// case it might be faster to use the CPU.
+//
+// Arguments:
+//	input: Shape is `[..., M, M]`.
+//
+// Returns Shape is `[..., M, M]`.
+func Cholesky(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "IdentityReaderV2",
-
-		Attrs: attrs,
+		Type: "Cholesky",
+		Input: []tf.Input{
+			input,
+		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResourceApplyGradientDescentAttr is an optional argument to ResourceApplyGradientDescent.
-type ResourceApplyGradientDescentAttr func(optionalAttr)
-
-// ResourceApplyGradientDescentUseLocking sets the optional use_locking attribute to value.
+// Writes contents to the file at input filename. Creates file and recursively
 //
-// value: If `True`, the subtraction will be protected by a lock;
-// otherwise the behavior is undefined, but may exhibit less contention.
-// If not specified, defaults to false
-func ResourceApplyGradientDescentUseLocking(value bool) ResourceApplyGradientDescentAttr {
-	return func(m optionalAttr) {
-		m["use_locking"] = value
-	}
-}
-
-// Update '*var' by subtracting 'alpha' * 'delta' from it.
+// creates directory if not existing.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	alpha: Scaling factor. Must be a scalar.
-//	delta: The change.
+//	filename: scalar. The name of the file to which we write the contents.
+//	contents: scalar. The content to be written to the output file.
 //
 // Returns the created operation.
-func ResourceApplyGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, delta tf.Output, optional ...ResourceApplyGradientDescentAttr) (o *tf.Operation) {
+func WriteFile(scope *Scope, filename tf.Output, contents tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyGradientDescent",
+		Type: "WriteFile",
 		Input: []tf.Input{
-			var_, alpha, delta,
+			filename, contents,
 		},
-		Attrs: attrs,
 	}
 	return scope.AddOperation(opspec)
 }
 
-// Returns the next record (key, value pair) produced by a Reader.
+// AllAttr is an optional argument to All.
+type AllAttr func(optionalAttr)
+
+// AllKeepDims sets the optional keep_dims attribute to value.
 //
-// Will dequeue from the input queue if necessary (e.g. when the
-// Reader needs to start reading from a new file since it has finished
-// with the previous file).
+// value: If true, retain reduced dimensions with length 1.
+// If not specified, defaults to false
+func AllKeepDims(value bool) AllAttr {
+	return func(m optionalAttr) {
+		m["keep_dims"] = value
+	}
+}
+
+// Computes the "logical and" of elements across dimensions of a tensor.
+//
+// Reduces `input` along the dimensions given in `axis`. Unless
+// `keep_dims` is true, the rank of the tensor is reduced by 1 for each entry in
+// `axis`. If `keep_dims` is true, the reduced dimensions are
+// retained with length 1.
 //
 // Arguments:
-//	reader_handle: Handle to a Reader.
-//	queue_handle: Handle to a Queue, with string work items.
+//	input: The tensor to reduce.
+//	axis: The dimensions to reduce. Must be in the range
+// `[-rank(input), rank(input))`.
 //
-// Returns A scalar.A scalar.
-func ReaderReadV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output) (key tf.Output, value tf.Output) {
+// Returns The reduced tensor.
+func All(scope *Scope, input tf.Output, axis tf.Output, optional ...AllAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "ReaderReadV2",
+		Type: "All",
 		Input: []tf.Input{
-			reader_handle, queue_handle,
+			input, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Returns up to `num_records` (key, value) pairs produced by a Reader.
+// Computes the Eigen Decomposition of a batch of square self-adjoint matrices.
 //
-// Will dequeue from the input queue if necessary (e.g. when the
-// Reader needs to start reading from a new file since it has finished
-// with the previous file).
-// It may return less than `num_records` even before the last batch.
+// DEPRECATED at GraphDef version 11: Use SelfAdjointEigV2 instead.
+//
+// The input is a tensor of shape `[..., M, M]` whose inner-most 2 dimensions
+// form square matrices, with the same constraints as the single matrix
+// SelfAdjointEig.
+//
+// The result is a [..., M+1, M] matrix with [..., 0,:] containing the
+// eigenvalues, and subsequent [...,1:, :] containing the eigenvectors.
 //
 // Arguments:
-//	reader_handle: Handle to a `Reader`.
-//	queue_handle: Handle to a `Queue`, with string work items.
-//	num_records: number of records to read from `Reader`.
+//	input: Shape is `[..., M, M]`.
 //
-// Returns A 1-D tensor.A 1-D tensor.
-func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output, num_records tf.Output) (keys tf.Output, values tf.Output) {
+// Returns Shape is `[..., M+1, M]`.
+func SelfAdjointEig(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReaderReadUpToV2",
+		Type: "SelfAdjointEig",
 		Input: []tf.Input{
-			reader_handle, queue_handle, num_records,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Restore a Reader to its initial clean state.
+// Computes softplus gradients for a softplus operation.
 //
 // Arguments:
-//	reader_handle: Handle to a Reader.
+//	gradients: The backpropagated gradients to the corresponding softplus operation.
+//	features: The features passed as input to the corresponding softplus operation.
 //
-// Returns the created operation.
-func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
+// Returns The gradients: `gradients / (1 + exp(-features))`.
+func SoftplusGrad(scope *Scope, gradients tf.Output, features tf.Output) (backprops tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ReaderResetV2",
+		Type: "SoftplusGrad",
 		Input: []tf.Input{
-			reader_handle,
+			gradients, features,
 		},
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// ResourceApplyAdamAttr is an optional argument to ResourceApplyAdam.
-type ResourceApplyAdamAttr func(optionalAttr)
+// SelfAdjointEigV2Attr is an optional argument to SelfAdjointEigV2.
+type SelfAdjointEigV2Attr func(optionalAttr)
 
-// ResourceApplyAdamUseLocking sets the optional use_locking attribute to value.
+// SelfAdjointEigV2ComputeV sets the optional compute_v attribute to value.
 //
-// value: If `True`, updating of the var, m, and v tensors will be protected
-// by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyAdamUseLocking(value bool) ResourceApplyAdamAttr {
+// value: If `True` then eigenvectors will be computed and returned in `v`.
+// Otherwise, only the eigenvalues will be computed.
+// If not specified, defaults to true
+func SelfAdjointEigV2ComputeV(value bool) SelfAdjointEigV2Attr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["compute_v"] = value
 	}
 }
 
-// ResourceApplyAdamUseNesterov sets the optional use_nesterov attribute to value.
+// Computes the eigen decomposition of one or more square self-adjoint matrices.
 //
-// value: If `True`, uses the nesterov update.
-// If not specified, defaults to false
-func ResourceApplyAdamUseNesterov(value bool) ResourceApplyAdamAttr {
-	return func(m optionalAttr) {
-		m["use_nesterov"] = value
-	}
-}
-
-// Update '*var' according to the Adam algorithm.
+// Computes the eigenvalues and (optionally) eigenvectors of each inner matrix in
+// `input` such that `input[..., :, :] = v[..., :, :] * diag(e[..., :])`.
 //
-// lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
-// m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t
-// v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t
-// variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+// ```python
+// # a is a tensor.
+// # e is a tensor of eigenvalues.
+// # v is a tensor of eigenvectors.
+// e, v = self_adjoint_eig(a)
+// e = self_adjoint_eig(a, compute_v=False)
+// ```
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	m: Should be from a Variable().
-//	v: Should be from a Variable().
-//	beta1_power: Must be a scalar.
-//	beta2_power: Must be a scalar.
-//	lr: Scaling factor. Must be a scalar.
-//	beta1: Momentum factor. Must be a scalar.
-//	beta2: Momentum factor. Must be a scalar.
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
+//	input: `Tensor` input of shape `[N, N]`.
 //
-// Returns the created operation.
-func ResourceApplyAdam(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamAttr) (o *tf.Operation) {
+// Returns Eigenvalues. Shape is `[N]`.Eigenvectors. Shape is `[N, N]`.
+func SelfAdjointEigV2(scope *Scope, input tf.Output, optional ...SelfAdjointEigV2Attr) (e tf.Output, v tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -20239,62 +19724,95 @@ func ResourceApplyAdam(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, b
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyAdam",
+		Type: "SelfAdjointEigV2",
 		Input: []tf.Input{
-			var_, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
+			input,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
 }
 
-// Store the input tensor in the state of the current session.
+// Adjust the saturation of one or more images.
+//
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
+//
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A scale is then applied all the saturation
+// values, and then remapped back to RGB colorspace.
 //
 // Arguments:
-//	value: The tensor to be stored.
+//	images: Images to adjust.  At least 3-D.
+//	scale: A float scale to add to the saturation.
 //
-// Returns The handle for the tensor stored in the session state, represented
-// as a ResourceHandle object.
-func GetSessionHandleV2(scope *Scope, value tf.Output) (handle tf.Output) {
+// Returns The hue-adjusted image or images.
+func AdjustSaturation(scope *Scope, images tf.Output, scale tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "GetSessionHandleV2",
+		Type: "AdjustSaturation",
 		Input: []tf.Input{
-			value,
+			images, scale,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
-type ResizeBicubicGradAttr func(optionalAttr)
+// SvdAttr is an optional argument to Svd.
+type SvdAttr func(optionalAttr)
 
-// ResizeBicubicGradAlignCorners sets the optional align_corners attribute to value.
+// SvdComputeUv sets the optional compute_uv attribute to value.
 //
-// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of grads and original_image. If false, rescale by
-// orig_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeBicubicGradAlignCorners(value bool) ResizeBicubicGradAttr {
+// value: If true, left and right singular vectors will be
+// computed and returned in `u` and `v`, respectively.
+// If false, `u` and `v` are not set and should never referenced.
+// If not specified, defaults to true
+func SvdComputeUv(value bool) SvdAttr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["compute_uv"] = value
 	}
 }
 
-// Computes the gradient of bicubic interpolation.
-//
-// Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
-// The image tensor that was resized.
+// SvdFullMatrices sets the optional full_matrices attribute to value.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
-// Gradients with respect to the input image. Input image must have been
-// float or double.
-func ResizeBicubicGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBicubicGradAttr) (output tf.Output) {
+// value: If true, compute full-sized `u` and `v`. If false
+// (the default), compute only the leading `P` singular vectors.
+// Ignored if `compute_uv` is `False`.
+// If not specified, defaults to false
+func SvdFullMatrices(value bool) SvdAttr {
+	return func(m optionalAttr) {
+		m["full_matrices"] = value
+	}
+}
+
+// Computes the singular value decompositions of one or more matrices.
+//
+// Computes the SVD of each inner matrix in `input` such that
+// `input[..., :, :] = u[..., :, :] * diag(s[..., :, :]) * transpose(v[..., :, :])`
+//
+// ```python
+// # a is a tensor containing a batch of matrices.
+// # s is a tensor of singular values for each matrix.
+// # u is the tensor containing of left singular vectors for each matrix.
+// # v is the tensor containing of right singular vectors for each matrix.
+// s, u, v = svd(a)
+// s, _, _ = svd(a, compute_uv=False)
+// ```
+//
+// Arguments:
+//	input: A tensor of shape `[..., M, N]` whose inner-most 2 dimensions
+// form matrices of size `[M, N]`. Let `P` be the minimum of `M` and `N`.
+//
+// Returns Singular values. Shape is `[..., P]`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., M, P]`; if `full_matrices` is `True` then shape is
+// `[..., M, M]`. Undefined if `compute_uv` is `False`.Left singular vectors. If `full_matrices` is `False` then shape is
+// `[..., N, P]`. If `full_matrices` is `True` then shape is `[..., N, N]`.
+// Undefined if `compute_uv` is false.
+func Svd(scope *Scope, input tf.Output, optional ...SvdAttr) (s tf.Output, u tf.Output, v tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -20303,41 +19821,50 @@ func ResizeBicubicGrad(scope *Scope, grads tf.Output, original_image tf.Output,
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeBicubicGrad",
+		Type: "Svd",
 		Input: []tf.Input{
-			grads, original_image,
+			input,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// ResizeNearestNeighborAttr is an optional argument to ResizeNearestNeighbor.
-type ResizeNearestNeighborAttr func(optionalAttr)
+// QueueEnqueueManyV2Attr is an optional argument to QueueEnqueueManyV2.
+type QueueEnqueueManyV2Attr func(optionalAttr)
 
-// ResizeNearestNeighborAlignCorners sets the optional align_corners attribute to value.
+// QueueEnqueueManyV2TimeoutMs sets the optional timeout_ms attribute to value.
 //
-// value: If true, rescale input by (new_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of images and resized images. If false, rescale
-// by new_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeNearestNeighborAlignCorners(value bool) ResizeNearestNeighborAttr {
+// value: If the queue is too full, this operation will block for up
+// to timeout_ms milliseconds.
+// Note: This option is not supported yet.
+// If not specified, defaults to -1
+func QueueEnqueueManyV2TimeoutMs(value int64) QueueEnqueueManyV2Attr {
 	return func(m optionalAttr) {
-		m["align_corners"] = value
+		m["timeout_ms"] = value
 	}
 }
 
-// Resize `images` to `size` using nearest neighbor interpolation.
+// Enqueues zero or more tuples of one or more tensors in the given queue.
+//
+// This operation slices each component tensor along the 0th dimension to
+// make multiple queue elements. All of the tuple components must have the
+// same size in the 0th dimension.
+//
+// The components input has k elements, which correspond to the components of
+// tuples stored in the given queue.
+//
+// N.B. If the queue is full, this operation will block until the given
+// elements have been enqueued (or 'timeout_ms' elapses, if specified).
 //
 // Arguments:
-//	images: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
-// new size for the images.
+//	handle: The handle to a queue.
+//	components: One or more tensors from which the enqueued tensors should
+// be taken.
 //
-// Returns 4-D with shape
-// `[batch, new_height, new_width, channels]`.
-func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeNearestNeighborAttr) (resized_images tf.Output) {
+// Returns the created operation.
+func QueueEnqueueManyV2(scope *Scope, handle tf.Output, components []tf.Output, optional ...QueueEnqueueManyV2Attr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -20346,226 +19873,123 @@ func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optio
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResizeNearestNeighbor",
+		Type: "QueueEnqueueManyV2",
 		Input: []tf.Input{
-			images, size,
+			handle, tf.OutputList(components),
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ResizeNearestNeighborGradAttr is an optional argument to ResizeNearestNeighborGrad.
-type ResizeNearestNeighborGradAttr func(optionalAttr)
-
-// ResizeNearestNeighborGradAlignCorners sets the optional align_corners attribute to value.
+// Computes the product along segments of a tensor.
 //
-// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
-// exactly aligns the 4 corners of grads and original_image. If false, rescale by
-// orig_height / height. Treat similarly the width dimension.
-// If not specified, defaults to false
-func ResizeNearestNeighborGradAlignCorners(value bool) ResizeNearestNeighborGradAttr {
-	return func(m optionalAttr) {
-		m["align_corners"] = value
-	}
-}
-
-// Computes the gradient of nearest neighbor interpolation.
+// Read @{$math_ops#segmentation$the section on segmentation} for an explanation of
+// segments.
+//
+// Computes a tensor such that
+// \\(output_i = \prod_j data_j\\) where the product is over `j` such
+// that `segment_ids[j] == i`.
+//
+// If the product is empty for a given segment ID `i`, `output[i] = 1`.
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/SegmentProd.png" alt>
+// </div>
 //
 // Arguments:
-//	grads: 4-D with shape `[batch, height, width, channels]`.
-//	size: = A 1-D int32 Tensor of 2 elements: `orig_height, orig_width`. The
-// original input size.
 //
-// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`. Gradients
-// with respect to the input image.
-func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, optional ...ResizeNearestNeighborGradAttr) (output tf.Output) {
+//	segment_ids: A 1-D tensor whose rank is equal to the rank of `data`'s
+// first dimension.  Values should be sorted and can be repeated.
+//
+// Returns Has same shape as data, except for dimension 0 which
+// has size `k`, the number of segments.
+func SegmentProd(scope *Scope, data tf.Output, segment_ids tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ResizeNearestNeighborGrad",
+		Type: "SegmentProd",
 		Input: []tf.Input{
-			grads, size,
+			data, segment_ids,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ExtractJpegShapeAttr is an optional argument to ExtractJpegShape.
-type ExtractJpegShapeAttr func(optionalAttr)
-
-// ExtractJpegShapeOutputType sets the optional output_type attribute to value.
+// Converts one or more images from RGB to HSV.
 //
-// value: (Optional) The output type of the operation (int32 or int64).
-// Defaults to int32.
-// If not specified, defaults to DT_INT32
-func ExtractJpegShapeOutputType(value tf.DataType) ExtractJpegShapeAttr {
-	return func(m optionalAttr) {
-		m["output_type"] = value
-	}
-}
-
-// Extract the shape information of a JPEG-encoded image.
+// Outputs a tensor of the same shape as the `images` tensor, containing the HSV
+// value of the pixels. The output is only well defined if the value in `images`
+// are in `[0,1]`.
 //
-// This op only parses the image header, so it is much faster than DecodeJpeg.
+// `output[..., 0]` contains hue, `output[..., 1]` contains saturation, and
+// `output[..., 2]` contains value. All HSV values are in `[0,1]`. A hue of 0
+// corresponds to pure red, hue 1/3 is pure green, and 2/3 is pure blue.
 //
 // Arguments:
-//	contents: 0-D. The JPEG-encoded image.
+//	images: 1-D or higher rank. RGB data to convert. Last dimension must be size 3.
 //
-// Returns 1-D. The image shape with format [height, width, channels].
-func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegShapeAttr) (image_shape tf.Output) {
+// Returns `images` converted to HSV.
+func RGBToHSV(scope *Scope, images tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "ExtractJpegShape",
+		Type: "RGBToHSV",
 		Input: []tf.Input{
-			contents,
+			images,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// PaddingFIFOQueueV2Attr is an optional argument to PaddingFIFOQueueV2.
-type PaddingFIFOQueueV2Attr func(optionalAttr)
-
-// PaddingFIFOQueueV2Shapes sets the optional shapes attribute to value.
-//
-// value: The shape of each component in a value. The length of this attr must
-// be either 0 or the same as the length of component_types.
-// Shapes of fixed rank but variable size are allowed by setting
-// any shape dimension to -1.  In this case, the inputs' shape may vary along
-// the given dimension, and DequeueMany will pad the given dimension with
-// zeros up to the maximum shape of all elements in the given batch.
-// If the length of this attr is 0, different queue elements may have
-// different ranks and shapes, but only one element may be dequeued at a time.
-// If not specified, defaults to <>
+// Does nothing. Only useful as a placeholder for control edges.
 //
-// REQUIRES: len(value) >= 0
-func PaddingFIFOQueueV2Shapes(value []tf.Shape) PaddingFIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["shapes"] = value
+// Returns the created operation.
+func NoOp(scope *Scope) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// PaddingFIFOQueueV2Capacity sets the optional capacity attribute to value.
-//
-// value: The upper bound on the number of elements in this queue.
-// Negative numbers mean no limit.
-// If not specified, defaults to -1
-func PaddingFIFOQueueV2Capacity(value int64) PaddingFIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["capacity"] = value
+	opspec := tf.OpSpec{
+		Type: "NoOp",
 	}
+	return scope.AddOperation(opspec)
 }
 
-// PaddingFIFOQueueV2Container sets the optional container attribute to value.
-//
-// value: If non-empty, this queue is placed in the given container.
-// Otherwise, a default container is used.
-// If not specified, defaults to ""
-func PaddingFIFOQueueV2Container(value string) PaddingFIFOQueueV2Attr {
-	return func(m optionalAttr) {
-		m["container"] = value
-	}
-}
+// MergeV2CheckpointsAttr is an optional argument to MergeV2Checkpoints.
+type MergeV2CheckpointsAttr func(optionalAttr)
 
-// PaddingFIFOQueueV2SharedName sets the optional shared_name attribute to value.
+// MergeV2CheckpointsDeleteOldDirs sets the optional delete_old_dirs attribute to value.
 //
-// value: If non-empty, this queue will be shared under the given name
-// across multiple sessions.
-// If not specified, defaults to ""
-func PaddingFIFOQueueV2SharedName(value string) PaddingFIFOQueueV2Attr {
+// value: see above.
+// If not specified, defaults to true
+func MergeV2CheckpointsDeleteOldDirs(value bool) MergeV2CheckpointsAttr {
 	return func(m optionalAttr) {
-		m["shared_name"] = value
+		m["delete_old_dirs"] = value
 	}
 }
 
-// A queue that produces elements in first-in first-out order.
+// V2 format specific: merges the metadata files of sharded checkpoints.  The
 //
-// Variable-size shapes are allowed by setting the corresponding shape dimensions
-// to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
-// size of any given element in the minibatch.  See below for details.
+// result is one logical checkpoint, with one physical metadata file and renamed
+// data files.
+//
+// Intended for "grouping" multiple checkpoints in a sharded checkpoint setup.
+//
+// If delete_old_dirs is true, attempts to delete recursively the dirname of each
+// path in the input checkpoint_prefixes.  This is useful when those paths are non
+// user-facing temporary locations.
 //
 // Arguments:
-//	component_types: The type of each component in a value.
+//	checkpoint_prefixes: prefixes of V2 checkpoints to merge.
+//	destination_prefix: scalar.  The desired final prefix.  Allowed to be the same
+// as one of the checkpoint_prefixes.
 //
-// Returns The handle to the queue.
-func PaddingFIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...PaddingFIFOQueueV2Attr) (handle tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "PaddingFIFOQueueV2",
-
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// DecodePngAttr is an optional argument to DecodePng.
-type DecodePngAttr func(optionalAttr)
-
-// DecodePngChannels sets the optional channels attribute to value.
-//
-// value: Number of color channels for the decoded image.
-// If not specified, defaults to 0
-func DecodePngChannels(value int64) DecodePngAttr {
-	return func(m optionalAttr) {
-		m["channels"] = value
-	}
-}
-
-// DecodePngDtype sets the optional dtype attribute to value.
-// If not specified, defaults to DT_UINT8
-func DecodePngDtype(value tf.DataType) DecodePngAttr {
-	return func(m optionalAttr) {
-		m["dtype"] = value
-	}
-}
-
-// Decode a PNG-encoded image to a uint8 or uint16 tensor.
-//
-// The attr `channels` indicates the desired number of color channels for the
-// decoded image.
-//
-// Accepted values are:
-//
-// *   0: Use the number of channels in the PNG-encoded image.
-// *   1: output a grayscale image.
-// *   3: output an RGB image.
-// *   4: output an RGBA image.
-//
-// If needed, the PNG-encoded image is transformed to match the requested number
-// of color channels.
-//
-// This op also supports decoding JPEGs and non-animated GIFs since the interface
-// is the same, though it is cleaner to use `tf.image.decode_image`.
-//
-// Arguments:
-//	contents: 0-D.  The PNG-encoded image.
-//
-// Returns 3-D with shape `[height, width, channels]`.
-func DecodePng(scope *Scope, contents tf.Output, optional ...DecodePngAttr) (image tf.Output) {
+// Returns the created operation.
+func MergeV2Checkpoints(scope *Scope, checkpoint_prefixes tf.Output, destination_prefix tf.Output, optional ...MergeV2CheckpointsAttr) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
@@ -20574,183 +19998,1351 @@ func DecodePng(scope *Scope, contents tf.Output, optional ...DecodePngAttr) (ima
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodePng",
+		Type: "MergeV2Checkpoints",
 		Input: []tf.Input{
-			contents,
+			checkpoint_prefixes, destination_prefix,
 		},
 		Attrs: attrs,
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// Decode the first frame of a GIF-encoded image to a uint8 tensor.
+// Saves input tensors slices to disk.
 //
-// GIF with frame or transparency compression are not supported
-// convert animated GIF from compressed to uncompressed by:
+// This is like `Save` except that tensors can be listed in the saved file as being
+// a slice of a larger tensor.  `shapes_and_slices` specifies the shape of the
+// larger tensor and the slice that this tensor covers. `shapes_and_slices` must
+// have as many elements as `tensor_names`.
 //
-//     convert $src.gif -coalesce $dst.gif
+// Elements of the `shapes_and_slices` input must either be:
 //
-// This op also supports decoding JPEGs and PNGs, though it is cleaner to use
-// `tf.image.decode_image`.
+// *  The empty string, in which case the corresponding tensor is
+//    saved normally.
+// *  A string of the form `dim0 dim1 ... dimN-1 slice-spec` where the
+//    `dimI` are the dimensions of the larger tensor and `slice-spec`
+//    specifies what part is covered by the tensor to save.
+//
+// `slice-spec` itself is a `:`-separated list: `slice0:slice1:...:sliceN-1`
+// where each `sliceI` is either:
+//
+// *  The string `-` meaning that the slice covers all indices of this dimension
+// *  `start,length` where `start` and `length` are integers.  In that
+//    case the slice covers `length` indices starting at `start`.
+//
+// See also `Save`.
 //
 // Arguments:
-//	contents: 0-D.  The GIF-encoded image.
+//	filename: Must have a single element. The name of the file to which we write the
+// tensor.
+//	tensor_names: Shape `[N]`. The names of the tensors to be saved.
+//	shapes_and_slices: Shape `[N]`.  The shapes and slice specifications to use when
+// saving the tensors.
+//	data: `N` tensors to save.
 //
-// Returns 4-D with shape `[num_frames, height, width, 3]`. RGB order
-func DecodeGif(scope *Scope, contents tf.Output) (image tf.Output) {
+// Returns the created operation.
+func SaveSlices(scope *Scope, filename tf.Output, tensor_names tf.Output, shapes_and_slices tf.Output, data []tf.Output) (o *tf.Operation) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "DecodeGif",
+		Type: "SaveSlices",
 		Input: []tf.Input{
-			contents,
+			filename, tensor_names, shapes_and_slices, tf.OutputList(data),
 		},
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return scope.AddOperation(opspec)
 }
 
-// ResourceApplyCenteredRMSPropAttr is an optional argument to ResourceApplyCenteredRMSProp.
-type ResourceApplyCenteredRMSPropAttr func(optionalAttr)
+// DenseToDenseSetOperationAttr is an optional argument to DenseToDenseSetOperation.
+type DenseToDenseSetOperationAttr func(optionalAttr)
 
-// ResourceApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
-//
-// value: If `True`, updating of the var, mg, ms, and mom tensors is
-// protected by a lock; otherwise the behavior is undefined, but may exhibit less
-// contention.
-// If not specified, defaults to false
-func ResourceApplyCenteredRMSPropUseLocking(value bool) ResourceApplyCenteredRMSPropAttr {
+// DenseToDenseSetOperationValidateIndices sets the optional validate_indices attribute to value.
+// If not specified, defaults to true
+func DenseToDenseSetOperationValidateIndices(value bool) DenseToDenseSetOperationAttr {
 	return func(m optionalAttr) {
-		m["use_locking"] = value
+		m["validate_indices"] = value
 	}
 }
 
-// Update '*var' according to the centered RMSProp algorithm.
-//
-// The centered RMSProp algorithm uses an estimate of the centered second moment
-// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
-// uses the (uncentered) second moment. This often helps with training, but is
-// slightly more expensive in terms of computation and memory.
-//
-// Note that in dense implementation of this algorithm, mg, ms, and mom will
-// update even if the grad is zero, but in this sparse implementation, mg, ms,
-// and mom will not update in iterations during which the grad is zero.
-//
-// mean_square = decay * mean_square + (1-decay) * gradient ** 2
-// mean_grad = decay * mean_grad + (1-decay) * gradient
+// Applies set operation along last dimension of 2 `Tensor` inputs.
 //
-// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+// See SetOperationOp::SetOperationFromContext for values of `set_operation`.
 //
-// mg <- rho * mg_{t-1} + (1-rho) * grad
-// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
-// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon)
-// var <- var - mom
+// Output `result` is a `SparseTensor` represented by `result_indices`,
+// `result_values`, and `result_shape`. For `set1` and `set2` ranked `n`, this
+// has rank `n` and the same 1st `n-1` dimensions as `set1` and `set2`. The `nth`
+// dimension contains the result of `set_operation` applied to the corresponding
+// `[0...n-1]` dimension of `set`.
 //
 // Arguments:
-//	var_: Should be from a Variable().
-//	mg: Should be from a Variable().
-//	ms: Should be from a Variable().
-//	mom: Should be from a Variable().
-//	lr: Scaling factor. Must be a scalar.
-//	rho: Decay rate. Must be a scalar.
+//	set1: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set2`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
+//	set2: `Tensor` with rank `n`. 1st `n-1` dimensions must be the same as `set1`.
+// Dimension `n` contains values in a set, duplicates are allowed but ignored.
 //
-//	epsilon: Ridge term. Must be a scalar.
-//	grad: The gradient.
 //
-// Returns the created operation.
-func ResourceApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyCenteredRMSPropAttr) (o *tf.Operation) {
+// Returns 2D indices of a `SparseTensor`.1D values of a `SparseTensor`.1D `Tensor` shape of a `SparseTensor`. `result_shape[0...n-1]` is
+// the same as the 1st `n-1` dimensions of `set1` and `set2`, `result_shape[n]`
+// is the max result set size across all `0...n-1` dimensions.
+func DenseToDenseSetOperation(scope *Scope, set1 tf.Output, set2 tf.Output, set_operation string, optional ...DenseToDenseSetOperationAttr) (result_indices tf.Output, result_values tf.Output, result_shape tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
+	attrs := map[string]interface{}{"set_operation": set_operation}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "ResourceApplyCenteredRMSProp",
+		Type: "DenseToDenseSetOperation",
 		Input: []tf.Input{
-			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad,
+			set1, set2,
 		},
 		Attrs: attrs,
 	}
-	return scope.AddOperation(opspec)
-}
-
-// Returns a list of tensors with the same shapes and contents as the input
-//
-// tensors.
-//
-// This op can be used to override the gradient for complicated functions. For
-// example, suppose y = f(x) and we wish to apply a custom function g for backprop
-// such that dx = g(dy). In Python,
-//
-// ```python
-// with tf.get_default_graph().gradient_override_map(
-//     {'IdentityN': 'OverrideGradientWithG'}):
-//   y, _ = identity_n([f(x), x])
-//
-// @tf.RegisterGradient('OverrideGradientWithG')
-// def ApplyG(op, dy, _):
-//   return [None, g(dy)]  # Do not backprop to f(x).
-// ```
-func IdentityN(scope *Scope, input []tf.Output) (output []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "IdentityN",
-		Input: []tf.Input{
-			tf.OutputList(input),
-		},
-	}
 	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
-	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("IdentityN", err)
-		return
-	}
-	return output
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Computes the gradient of the sigmoid of `x` wrt its input.
+// Generate a sharded filename. The filename is printf formatted as
 //
-// Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
-// `dy` is the corresponding input gradient.
-func SigmoidGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+//    %s-%05d-of-%05d, basename, shard, num_shards.
+func ShardedFilename(scope *Scope, basename tf.Output, shard tf.Output, num_shards tf.Output) (filename tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "SigmoidGrad",
+		Type: "ShardedFilename",
 		Input: []tf.Input{
-			y, dy,
+			basename, shard, num_shards,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Convert one or more images from HSV to RGB.
-//
-// Outputs a tensor of the same shape as the `images` tensor, containing the RGB
-// value of the pixels. The output is only well defined if the value in `images`
-// are in `[0,1]`.
+// BatchToSpace for N-D tensors of type T.
 //
-// See `rgb_to_hsv` for a description of the HSV encoding.
+// This operation reshapes the "batch" dimension 0 into `M + 1` dimensions of shape
+// `block_shape + [batch]`, interleaves these blocks back into the grid defined by
+// the spatial dimensions `[1, ..., M]`, to obtain a result with the same rank as
+// the input.  The spatial dimensions of this intermediate result are then
+// optionally cropped according to `crops` to produce the output.  This is the
+// reverse of SpaceToBatch.  See below for a precise description.
 //
 // Arguments:
-//	images: 1-D or higher rank. HSV data to convert. Last dimension must be size 3.
+//	input: N-D with shape `input_shape = [batch] + spatial_shape + remaining_shape`,
+// where spatial_shape has M dimensions.
+//	block_shape: 1-D with shape `[M]`, all values must be >= 1.
+//	crops: 2-D with shape `[M, 2]`, all values must be >= 0.
+//   `crops[i] = [crop_start, crop_end]` specifies the amount to crop from input
+//   dimension `i + 1`, which corresponds to spatial dimension `i`.  It is
+//   required that
+//   `crop_start[i] + crop_end[i] <= block_shape[i] * input_shape[i + 1]`.
 //
-// Returns `images` converted to RGB.
-func HSVToRGB(scope *Scope, images tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
+// This operation is equivalent to the following steps:
+//
+// 1. Reshape `input` to `reshaped` of shape:
+//      [block_shape[0], ..., block_shape[M-1],
+//       batch / prod(block_shape),
+//       input_shape[1], ..., input_shape[N-1]]
+//
+// 2. Permute dimensions of `reshaped` to produce `permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1], block_shape[0],
+//       ...,
+//       input_shape[M], block_shape[M-1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// 3. Reshape `permuted` to produce `reshaped_permuted` of shape
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0],
+//       ...,
+//       input_shape[M] * block_shape[M-1],
+//
+//       input_shape[M+1],
+//       ...,
+//       input_shape[N-1]]
+//
+// 4. Crop the start and end of dimensions `[1, ..., M]` of
+//    `reshaped_permuted` according to `crops` to produce the output of shape:
+//      [batch / prod(block_shape),
+//
+//       input_shape[1] * block_shape[0] - crops[0,0] - crops[0,1],
+//       ...,
+//       input_shape[M] * block_shape[M-1] - crops[M-1,0] - crops[M-1,1],
+//
+//       input_shape[M+1], ..., input_shape[N-1]]
+//
+// Some examples:
+//
+// (1) For the following input of shape `[4, 1, 1, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[[1]]], [[[2]]], [[[3]]], [[[4]]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 1]` and value:
+//
+// ```
+// x = [[[[1], [2]], [[3], [4]]]]
+// ```
+//
+// (2) For the following input of shape `[4, 1, 1, 3]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// [[[1, 2, 3]], [[4, 5, 6]], [[7, 8, 9]], [[10, 11, 12]]]
+// ```
+//
+// The output tensor has shape `[1, 2, 2, 3]` and value:
+//
+// ```
+// x = [[[[1, 2, 3], [4, 5, 6]],
+//       [[7, 8, 9], [10, 11, 12]]]]
+// ```
+//
+// (3) For the following input of shape `[4, 2, 2, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [0, 0]]`:
+//
+// ```
+// x = [[[[1], [3]], [[9], [11]]],
+//      [[[2], [4]], [[10], [12]]],
+//      [[[5], [7]], [[13], [15]]],
+//      [[[6], [8]], [[14], [16]]]]
+// ```
+//
+// The output tensor has shape `[1, 4, 4, 1]` and value:
+//
+// ```
+// x = [[[1],   [2],  [3],  [4]],
+//      [[5],   [6],  [7],  [8]],
+//      [[9],  [10], [11],  [12]],
+//      [[13], [14], [15],  [16]]]
+// ```
+//
+// (4) For the following input of shape `[8, 1, 3, 1]`, `block_shape = [2, 2]`, and
+//     `crops = [[0, 0], [2, 0]]`:
+//
+// ```
+// x = [[[[0], [1], [3]]], [[[0], [9], [11]]],
+//      [[[0], [2], [4]]], [[[0], [10], [12]]],
+//      [[[0], [5], [7]]], [[[0], [13], [15]]],
+//      [[[0], [6], [8]]], [[[0], [14], [16]]]]
+// ```
+//
+// The output tensor has shape `[2, 2, 4, 1]` and value:
+//
+// ```
+// x = [[[[1],   [2],  [3],  [4]],
+//       [[5],   [6],  [7],  [8]]],
+//      [[[9],  [10], [11],  [12]],
+//       [[13], [14], [15],  [16]]]]
+// ```
+func BatchToSpaceND(scope *Scope, input tf.Output, block_shape tf.Output, crops tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "BatchToSpaceND",
+		Input: []tf.Input{
+			input, block_shape, crops,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// UnpackAttr is an optional argument to Unpack.
+type UnpackAttr func(optionalAttr)
+
+// UnpackAxis sets the optional axis attribute to value.
+//
+// value: Dimension along which to unpack.  Negative values wrap around, so the
+// valid range is `[-R, R)`.
+// If not specified, defaults to 0
+func UnpackAxis(value int64) UnpackAttr {
+	return func(m optionalAttr) {
+		m["axis"] = value
+	}
+}
+
+// Unpacks a given dimension of a rank-`R` tensor into `num` rank-`(R-1)` tensors.
+//
+// Unpacks `num` tensors from `value` by chipping it along the `axis` dimension.
+// For example, given a tensor of shape `(A, B, C, D)`;
+//
+// If `axis == 0` then the i'th tensor in `output` is the slice `value[i, :, :, :]`
+//   and each tensor in `output` will have shape `(B, C, D)`. (Note that the
+//   dimension unpacked along is gone, unlike `split`).
+//
+// If `axis == 1` then the i'th tensor in `output` is the slice `value[:, i, :, :]`
+//   and each tensor in `output` will have shape `(A, C, D)`.
+// Etc.
+//
+// This is the opposite of `pack`.
+//
+// Arguments:
+//	value: 1-D or higher, with `axis` dimension size equal to `num`.
+//
+//
+// Returns The list of tensors unpacked from `value`.
+func Unpack(scope *Scope, value tf.Output, num int64, optional ...UnpackAttr) (output []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num": num}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "Unpack",
+		Input: []tf.Input{
+			value,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Unpack", err)
+		return
+	}
+	return output
+}
+
+// Increments variable pointed to by 'resource' until it reaches 'limit'.
+//
+// Arguments:
+//	resource: Should be from a scalar `Variable` node.
+//	limit: If incrementing ref would bring it above limit, instead generates an
+// 'OutOfRange' error.
+//
+//
+// Returns A copy of the input before increment. If nothing else modifies the
+// input, the values produced will all be distinct.
+func ResourceCountUpTo(scope *Scope, resource tf.Output, limit int64, T tf.DataType) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"limit": limit, "T": T}
+	opspec := tf.OpSpec{
+		Type: "ResourceCountUpTo",
+		Input: []tf.Input{
+			resource,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Delete the stack from its resource container.
+//
+// Arguments:
+//	handle: The handle to a stack.
+//
+// Returns the created operation.
+func StackCloseV2(scope *Scope, handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "StackCloseV2",
+		Input: []tf.Input{
+			handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Generate a glob pattern matching all sharded file names.
+func ShardedFilespec(scope *Scope, basename tf.Output, num_shards tf.Output) (filename tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ShardedFilespec",
+		Input: []tf.Input{
+			basename, num_shards,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// TextLineReaderV2Attr is an optional argument to TextLineReaderV2.
+type TextLineReaderV2Attr func(optionalAttr)
+
+// TextLineReaderV2SkipHeaderLines sets the optional skip_header_lines attribute to value.
+//
+// value: Number of lines to skip from the beginning of every file.
+// If not specified, defaults to 0
+func TextLineReaderV2SkipHeaderLines(value int64) TextLineReaderV2Attr {
+	return func(m optionalAttr) {
+		m["skip_header_lines"] = value
+	}
+}
+
+// TextLineReaderV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func TextLineReaderV2Container(value string) TextLineReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// TextLineReaderV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func TextLineReaderV2SharedName(value string) TextLineReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A Reader that outputs the lines of a file delimited by '\n'.
+//
+// Returns The handle to reference the Reader.
+func TextLineReaderV2(scope *Scope, optional ...TextLineReaderV2Attr) (reader_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TextLineReaderV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// LoadAndRemapMatrixAttr is an optional argument to LoadAndRemapMatrix.
+type LoadAndRemapMatrixAttr func(optionalAttr)
+
+// LoadAndRemapMatrixMaxRowsInMemory sets the optional max_rows_in_memory attribute to value.
+//
+// value: The maximum number of rows to load from the checkpoint at
+// once. If less than or equal to 0, the entire matrix will be loaded into
+// memory. Setting this arg trades increased disk reads for lower memory usage.
+// If not specified, defaults to -1
+func LoadAndRemapMatrixMaxRowsInMemory(value int64) LoadAndRemapMatrixAttr {
+	return func(m optionalAttr) {
+		m["max_rows_in_memory"] = value
+	}
+}
+
+// Loads a 2-D (matrix) `Tensor` with name `old_tensor_name` from the checkpoint
+//
+// at `ckpt_path` and potentially reorders its rows and columns using the
+// specified remappings.
+//
+// Most users should use one of the wrapper initializers (such as
+// `tf.contrib.framework.load_and_remap_matrix_initializer`) instead of this
+// function directly.
+//
+// The remappings are 1-D tensors with the following properties:
+//
+// * `row_remapping` must have exactly `num_rows` entries. Row `i` of the output
+//   matrix will be initialized from the row corresponding to index
+//   `row_remapping[i]` in the old `Tensor` from the checkpoint.
+// * `col_remapping` must have either 0 entries (indicating that no column
+//   reordering is needed) or `num_cols` entries. If specified, column `j` of the
+//   output matrix will be initialized from the column corresponding to index
+//   `col_remapping[j]` in the old `Tensor` from the checkpoint.
+// * A value of -1 in either of the remappings signifies a "missing" entry. In that
+//   case, values from the `initializing_values` tensor will be used to fill that
+//   missing row or column. If `row_remapping` has `r` missing entries and
+//   `col_remapping` has `c` missing entries, then the following condition must be
+//   true:
+//
+// `(r * num_cols) + (c * num_rows) - (r * c) == len(initializing_values)`
+//
+// The remapping tensors can be generated using the GenerateVocabRemapping op.
+//
+// As an example, with row_remapping = [1, 0, -1], col_remapping = [0, 2, -1],
+// initializing_values = [0.5, -0.5, 0.25, -0.25, 42], and w(i, j) representing
+// the value from row i, column j of the old tensor in the checkpoint, the output
+// matrix will look like the following:
+//
+// [[w(1, 0),  w(1, 2),  0.5],
+//  [w(0, 0),  w(0, 2), -0.5],
+//  [0.25,    -0.25,      42]]
+//
+// Arguments:
+//	ckpt_path: Path to the TensorFlow checkpoint (version 2, `TensorBundle`) from
+// which the old matrix `Tensor` will be loaded.
+//	old_tensor_name: Name of the 2-D `Tensor` to load from checkpoint.
+//	row_remapping: An int `Tensor` of row remappings (generally created by
+// `generate_vocab_remapping`).  Even if no row remapping is needed, this must
+// still be an index-valued Tensor (e.g. [0, 1, 2, ...]), or a shifted
+// index-valued `Tensor` (e.g. [8, 9, 10, ...], for partitioned `Variables`).
+//	col_remapping: An int `Tensor` of column remappings (generally created by
+// `generate_vocab_remapping`).  May be a size-0 `Tensor` if only row remapping
+// is to be done (e.g. column ordering is the same).
+//	initializing_values: A float `Tensor` containing  values to fill in for cells
+// in the output matrix that are not loaded from the checkpoint. Length must be
+// exactly the same as the number of missing / new cells.
+//	num_rows: Number of rows (length of the 1st dimension) in the output matrix.
+//	num_cols: Number of columns (length of the 2nd dimension) in the output matrix.
+//
+// Returns Output matrix containing existing values loaded from the
+// checkpoint, and with any missing values filled in from initializing_values.
+func LoadAndRemapMatrix(scope *Scope, ckpt_path tf.Output, old_tensor_name tf.Output, row_remapping tf.Output, col_remapping tf.Output, initializing_values tf.Output, num_rows int64, num_cols int64, optional ...LoadAndRemapMatrixAttr) (output_matrix tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"num_rows": num_rows, "num_cols": num_cols}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "LoadAndRemapMatrix",
+		Input: []tf.Input{
+			ckpt_path, old_tensor_name, row_remapping, col_remapping, initializing_values,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// TFRecordReaderV2Attr is an optional argument to TFRecordReaderV2.
+type TFRecordReaderV2Attr func(optionalAttr)
+
+// TFRecordReaderV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func TFRecordReaderV2Container(value string) TFRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// TFRecordReaderV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func TFRecordReaderV2SharedName(value string) TFRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// TFRecordReaderV2CompressionType sets the optional compression_type attribute to value.
+// If not specified, defaults to ""
+func TFRecordReaderV2CompressionType(value string) TFRecordReaderV2Attr {
+	return func(m optionalAttr) {
+		m["compression_type"] = value
+	}
+}
+
+// A Reader that outputs the records from a TensorFlow Records file.
+//
+// Returns The handle to reference the Reader.
+func TFRecordReaderV2(scope *Scope, optional ...TFRecordReaderV2Attr) (reader_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TFRecordReaderV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// QuantizeAndDequantizeV3Attr is an optional argument to QuantizeAndDequantizeV3.
+type QuantizeAndDequantizeV3Attr func(optionalAttr)
+
+// QuantizeAndDequantizeV3SignedInput sets the optional signed_input attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeV3SignedInput(value bool) QuantizeAndDequantizeV3Attr {
+	return func(m optionalAttr) {
+		m["signed_input"] = value
+	}
+}
+
+// QuantizeAndDequantizeV3RangeGiven sets the optional range_given attribute to value.
+// If not specified, defaults to true
+func QuantizeAndDequantizeV3RangeGiven(value bool) QuantizeAndDequantizeV3Attr {
+	return func(m optionalAttr) {
+		m["range_given"] = value
+	}
+}
+
+// Quantizes then dequantizes a tensor.
+//
+// This is almost identical to QuantizeAndDequantizeV2, except that num_bits is a
+// tensor, so its value can change during training.
+func QuantizeAndDequantizeV3(scope *Scope, input tf.Output, input_min tf.Output, input_max tf.Output, num_bits tf.Output, optional ...QuantizeAndDequantizeV3Attr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "QuantizeAndDequantizeV3",
+		Input: []tf.Input{
+			input, input_min, input_max, num_bits,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// IdentityReaderV2Attr is an optional argument to IdentityReaderV2.
+type IdentityReaderV2Attr func(optionalAttr)
+
+// IdentityReaderV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func IdentityReaderV2Container(value string) IdentityReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// IdentityReaderV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func IdentityReaderV2SharedName(value string) IdentityReaderV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A Reader that outputs the queued work as both the key and value.
+//
+// To use, enqueue strings in a Queue.  ReaderRead will take the front
+// work string and output (work, work).
+//
+// Returns The handle to reference the Reader.
+func IdentityReaderV2(scope *Scope, optional ...IdentityReaderV2Attr) (reader_handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "IdentityReaderV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyGradientDescentAttr is an optional argument to ResourceApplyGradientDescent.
+type ResourceApplyGradientDescentAttr func(optionalAttr)
+
+// ResourceApplyGradientDescentUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, the subtraction will be protected by a lock;
+// otherwise the behavior is undefined, but may exhibit less contention.
+// If not specified, defaults to false
+func ResourceApplyGradientDescentUseLocking(value bool) ResourceApplyGradientDescentAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' by subtracting 'alpha' * 'delta' from it.
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	alpha: Scaling factor. Must be a scalar.
+//	delta: The change.
+//
+// Returns the created operation.
+func ResourceApplyGradientDescent(scope *Scope, var_ tf.Output, alpha tf.Output, delta tf.Output, optional ...ResourceApplyGradientDescentAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyGradientDescent",
+		Input: []tf.Input{
+			var_, alpha, delta,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns the next record (key, value pair) produced by a Reader.
+//
+// Will dequeue from the input queue if necessary (e.g. when the
+// Reader needs to start reading from a new file since it has finished
+// with the previous file).
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+//	queue_handle: Handle to a Queue, with string work items.
+//
+// Returns A scalar.A scalar.
+func ReaderReadV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output) (key tf.Output, value tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderReadV2",
+		Input: []tf.Input{
+			reader_handle, queue_handle,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Returns up to `num_records` (key, value) pairs produced by a Reader.
+//
+// Will dequeue from the input queue if necessary (e.g. when the
+// Reader needs to start reading from a new file since it has finished
+// with the previous file).
+// It may return less than `num_records` even before the last batch.
+//
+// Arguments:
+//	reader_handle: Handle to a `Reader`.
+//	queue_handle: Handle to a `Queue`, with string work items.
+//	num_records: number of records to read from `Reader`.
+//
+// Returns A 1-D tensor.A 1-D tensor.
+func ReaderReadUpToV2(scope *Scope, reader_handle tf.Output, queue_handle tf.Output, num_records tf.Output) (keys tf.Output, values tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderReadUpToV2",
+		Input: []tf.Input{
+			reader_handle, queue_handle, num_records,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0), op.Output(1)
+}
+
+// Restore a Reader to its initial clean state.
+//
+// Arguments:
+//	reader_handle: Handle to a Reader.
+//
+// Returns the created operation.
+func ReaderResetV2(scope *Scope, reader_handle tf.Output) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "ReaderResetV2",
+		Input: []tf.Input{
+			reader_handle,
+		},
+	}
+	return scope.AddOperation(opspec)
+}
+
+// ResourceApplyAdamAttr is an optional argument to ResourceApplyAdam.
+type ResourceApplyAdamAttr func(optionalAttr)
+
+// ResourceApplyAdamUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, m, and v tensors will be protected
+// by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyAdamUseLocking(value bool) ResourceApplyAdamAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// ResourceApplyAdamUseNesterov sets the optional use_nesterov attribute to value.
+//
+// value: If `True`, uses the nesterov update.
+// If not specified, defaults to false
+func ResourceApplyAdamUseNesterov(value bool) ResourceApplyAdamAttr {
+	return func(m optionalAttr) {
+		m["use_nesterov"] = value
+	}
+}
+
+// Update '*var' according to the Adam algorithm.
+//
+// lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
+// m_t <- beta1 * m_{t-1} + (1 - beta1) * g_t
+// v_t <- beta2 * v_{t-1} + (1 - beta2) * g_t * g_t
+// variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	m: Should be from a Variable().
+//	v: Should be from a Variable().
+//	beta1_power: Must be a scalar.
+//	beta2_power: Must be a scalar.
+//	lr: Scaling factor. Must be a scalar.
+//	beta1: Momentum factor. Must be a scalar.
+//	beta2: Momentum factor. Must be a scalar.
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyAdam(scope *Scope, var_ tf.Output, m tf.Output, v tf.Output, beta1_power tf.Output, beta2_power tf.Output, lr tf.Output, beta1 tf.Output, beta2 tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyAdamAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyAdam",
+		Input: []tf.Input{
+			var_, m, v, beta1_power, beta2_power, lr, beta1, beta2, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Store the input tensor in the state of the current session.
+//
+// Arguments:
+//	value: The tensor to be stored.
+//
+// Returns The handle for the tensor stored in the session state, represented
+// as a ResourceHandle object.
+func GetSessionHandleV2(scope *Scope, value tf.Output) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GetSessionHandleV2",
+		Input: []tf.Input{
+			value,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResizeBicubicGradAttr is an optional argument to ResizeBicubicGrad.
+type ResizeBicubicGradAttr func(optionalAttr)
+
+// ResizeBicubicGradAlignCorners sets the optional align_corners attribute to value.
+//
+// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of grads and original_image. If false, rescale by
+// orig_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeBicubicGradAlignCorners(value bool) ResizeBicubicGradAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Computes the gradient of bicubic interpolation.
+//
+// Arguments:
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	original_image: 4-D with shape `[batch, orig_height, orig_width, channels]`,
+// The image tensor that was resized.
+//
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`.
+// Gradients with respect to the input image. Input image must have been
+// float or double.
+func ResizeBicubicGrad(scope *Scope, grads tf.Output, original_image tf.Output, optional ...ResizeBicubicGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResizeBicubicGrad",
+		Input: []tf.Input{
+			grads, original_image,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResizeNearestNeighborAttr is an optional argument to ResizeNearestNeighbor.
+type ResizeNearestNeighborAttr func(optionalAttr)
+
+// ResizeNearestNeighborAlignCorners sets the optional align_corners attribute to value.
+//
+// value: If true, rescale input by (new_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of images and resized images. If false, rescale
+// by new_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeNearestNeighborAlignCorners(value bool) ResizeNearestNeighborAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Resize `images` to `size` using nearest neighbor interpolation.
+//
+// Arguments:
+//	images: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `new_height, new_width`.  The
+// new size for the images.
+//
+// Returns 4-D with shape
+// `[batch, new_height, new_width, channels]`.
+func ResizeNearestNeighbor(scope *Scope, images tf.Output, size tf.Output, optional ...ResizeNearestNeighborAttr) (resized_images tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResizeNearestNeighbor",
+		Input: []tf.Input{
+			images, size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResizeNearestNeighborGradAttr is an optional argument to ResizeNearestNeighborGrad.
+type ResizeNearestNeighborGradAttr func(optionalAttr)
+
+// ResizeNearestNeighborGradAlignCorners sets the optional align_corners attribute to value.
+//
+// value: If true, rescale grads by (orig_height - 1) / (height - 1), which
+// exactly aligns the 4 corners of grads and original_image. If false, rescale by
+// orig_height / height. Treat similarly the width dimension.
+// If not specified, defaults to false
+func ResizeNearestNeighborGradAlignCorners(value bool) ResizeNearestNeighborGradAttr {
+	return func(m optionalAttr) {
+		m["align_corners"] = value
+	}
+}
+
+// Computes the gradient of nearest neighbor interpolation.
+//
+// Arguments:
+//	grads: 4-D with shape `[batch, height, width, channels]`.
+//	size: = A 1-D int32 Tensor of 2 elements: `orig_height, orig_width`. The
+// original input size.
+//
+// Returns 4-D with shape `[batch, orig_height, orig_width, channels]`. Gradients
+// with respect to the input image.
+func ResizeNearestNeighborGrad(scope *Scope, grads tf.Output, size tf.Output, optional ...ResizeNearestNeighborGradAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResizeNearestNeighborGrad",
+		Input: []tf.Input{
+			grads, size,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ExtractJpegShapeAttr is an optional argument to ExtractJpegShape.
+type ExtractJpegShapeAttr func(optionalAttr)
+
+// ExtractJpegShapeOutputType sets the optional output_type attribute to value.
+//
+// value: (Optional) The output type of the operation (int32 or int64).
+// Defaults to int32.
+// If not specified, defaults to DT_INT32
+func ExtractJpegShapeOutputType(value tf.DataType) ExtractJpegShapeAttr {
+	return func(m optionalAttr) {
+		m["output_type"] = value
+	}
+}
+
+// Extract the shape information of a JPEG-encoded image.
+//
+// This op only parses the image header, so it is much faster than DecodeJpeg.
+//
+// Arguments:
+//	contents: 0-D. The JPEG-encoded image.
+//
+// Returns 1-D. The image shape with format [height, width, channels].
+func ExtractJpegShape(scope *Scope, contents tf.Output, optional ...ExtractJpegShapeAttr) (image_shape tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ExtractJpegShape",
+		Input: []tf.Input{
+			contents,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// PaddingFIFOQueueV2Attr is an optional argument to PaddingFIFOQueueV2.
+type PaddingFIFOQueueV2Attr func(optionalAttr)
+
+// PaddingFIFOQueueV2Shapes sets the optional shapes attribute to value.
+//
+// value: The shape of each component in a value. The length of this attr must
+// be either 0 or the same as the length of component_types.
+// Shapes of fixed rank but variable size are allowed by setting
+// any shape dimension to -1.  In this case, the inputs' shape may vary along
+// the given dimension, and DequeueMany will pad the given dimension with
+// zeros up to the maximum shape of all elements in the given batch.
+// If the length of this attr is 0, different queue elements may have
+// different ranks and shapes, but only one element may be dequeued at a time.
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func PaddingFIFOQueueV2Shapes(value []tf.Shape) PaddingFIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shapes"] = value
+	}
+}
+
+// PaddingFIFOQueueV2Capacity sets the optional capacity attribute to value.
+//
+// value: The upper bound on the number of elements in this queue.
+// Negative numbers mean no limit.
+// If not specified, defaults to -1
+func PaddingFIFOQueueV2Capacity(value int64) PaddingFIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["capacity"] = value
+	}
+}
+
+// PaddingFIFOQueueV2Container sets the optional container attribute to value.
+//
+// value: If non-empty, this queue is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func PaddingFIFOQueueV2Container(value string) PaddingFIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
+	}
+}
+
+// PaddingFIFOQueueV2SharedName sets the optional shared_name attribute to value.
+//
+// value: If non-empty, this queue will be shared under the given name
+// across multiple sessions.
+// If not specified, defaults to ""
+func PaddingFIFOQueueV2SharedName(value string) PaddingFIFOQueueV2Attr {
+	return func(m optionalAttr) {
+		m["shared_name"] = value
+	}
+}
+
+// A queue that produces elements in first-in first-out order.
+//
+// Variable-size shapes are allowed by setting the corresponding shape dimensions
+// to 0 in the shape attr.  In this case DequeueMany will pad up to the maximum
+// size of any given element in the minibatch.  See below for details.
+//
+// Arguments:
+//	component_types: The type of each component in a value.
+//
+// Returns The handle to the queue.
+func PaddingFIFOQueueV2(scope *Scope, component_types []tf.DataType, optional ...PaddingFIFOQueueV2Attr) (handle tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"component_types": component_types}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "PaddingFIFOQueueV2",
+
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// DecodePngAttr is an optional argument to DecodePng.
+type DecodePngAttr func(optionalAttr)
+
+// DecodePngChannels sets the optional channels attribute to value.
+//
+// value: Number of color channels for the decoded image.
+// If not specified, defaults to 0
+func DecodePngChannels(value int64) DecodePngAttr {
+	return func(m optionalAttr) {
+		m["channels"] = value
+	}
+}
+
+// DecodePngDtype sets the optional dtype attribute to value.
+// If not specified, defaults to DT_UINT8
+func DecodePngDtype(value tf.DataType) DecodePngAttr {
+	return func(m optionalAttr) {
+		m["dtype"] = value
+	}
+}
+
+// Decode a PNG-encoded image to a uint8 or uint16 tensor.
+//
+// The attr `channels` indicates the desired number of color channels for the
+// decoded image.
+//
+// Accepted values are:
+//
+// *   0: Use the number of channels in the PNG-encoded image.
+// *   1: output a grayscale image.
+// *   3: output an RGB image.
+// *   4: output an RGBA image.
+//
+// If needed, the PNG-encoded image is transformed to match the requested number
+// of color channels.
+//
+// This op also supports decoding JPEGs and non-animated GIFs since the interface
+// is the same, though it is cleaner to use `tf.image.decode_image`.
+//
+// Arguments:
+//	contents: 0-D.  The PNG-encoded image.
+//
+// Returns 3-D with shape `[height, width, channels]`.
+func DecodePng(scope *Scope, contents tf.Output, optional ...DecodePngAttr) (image tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodePng",
+		Input: []tf.Input{
+			contents,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Decode the first frame of a GIF-encoded image to a uint8 tensor.
+//
+// GIF with frame or transparency compression are not supported
+// convert animated GIF from compressed to uncompressed by:
+//
+//     convert $src.gif -coalesce $dst.gif
+//
+// This op also supports decoding JPEGs and PNGs, though it is cleaner to use
+// `tf.image.decode_image`.
+//
+// Arguments:
+//	contents: 0-D.  The GIF-encoded image.
+//
+// Returns 4-D with shape `[num_frames, height, width, 3]`. RGB order
+func DecodeGif(scope *Scope, contents tf.Output) (image tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "DecodeGif",
+		Input: []tf.Input{
+			contents,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ResourceApplyCenteredRMSPropAttr is an optional argument to ResourceApplyCenteredRMSProp.
+type ResourceApplyCenteredRMSPropAttr func(optionalAttr)
+
+// ResourceApplyCenteredRMSPropUseLocking sets the optional use_locking attribute to value.
+//
+// value: If `True`, updating of the var, mg, ms, and mom tensors is
+// protected by a lock; otherwise the behavior is undefined, but may exhibit less
+// contention.
+// If not specified, defaults to false
+func ResourceApplyCenteredRMSPropUseLocking(value bool) ResourceApplyCenteredRMSPropAttr {
+	return func(m optionalAttr) {
+		m["use_locking"] = value
+	}
+}
+
+// Update '*var' according to the centered RMSProp algorithm.
+//
+// The centered RMSProp algorithm uses an estimate of the centered second moment
+// (i.e., the variance) for normalization, as opposed to regular RMSProp, which
+// uses the (uncentered) second moment. This often helps with training, but is
+// slightly more expensive in terms of computation and memory.
+//
+// Note that in dense implementation of this algorithm, mg, ms, and mom will
+// update even if the grad is zero, but in this sparse implementation, mg, ms,
+// and mom will not update in iterations during which the grad is zero.
+//
+// mean_square = decay * mean_square + (1-decay) * gradient ** 2
+// mean_grad = decay * mean_grad + (1-decay) * gradient
+//
+// Delta = learning_rate * gradient / sqrt(mean_square + epsilon - mean_grad ** 2)
+//
+// mg <- rho * mg_{t-1} + (1-rho) * grad
+// ms <- rho * ms_{t-1} + (1-rho) * grad * grad
+// mom <- momentum * mom_{t-1} + lr * grad / sqrt(ms - mg * mg + epsilon)
+// var <- var - mom
+//
+// Arguments:
+//	var_: Should be from a Variable().
+//	mg: Should be from a Variable().
+//	ms: Should be from a Variable().
+//	mom: Should be from a Variable().
+//	lr: Scaling factor. Must be a scalar.
+//	rho: Decay rate. Must be a scalar.
+//
+//	epsilon: Ridge term. Must be a scalar.
+//	grad: The gradient.
+//
+// Returns the created operation.
+func ResourceApplyCenteredRMSProp(scope *Scope, var_ tf.Output, mg tf.Output, ms tf.Output, mom tf.Output, lr tf.Output, rho tf.Output, momentum tf.Output, epsilon tf.Output, grad tf.Output, optional ...ResourceApplyCenteredRMSPropAttr) (o *tf.Operation) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ResourceApplyCenteredRMSProp",
+		Input: []tf.Input{
+			var_, mg, ms, mom, lr, rho, momentum, epsilon, grad,
+		},
+		Attrs: attrs,
+	}
+	return scope.AddOperation(opspec)
+}
+
+// Returns a list of tensors with the same shapes and contents as the input
+//
+// tensors.
+//
+// This op can be used to override the gradient for complicated functions. For
+// example, suppose y = f(x) and we wish to apply a custom function g for backprop
+// such that dx = g(dy). In Python,
+//
+// ```python
+// with tf.get_default_graph().gradient_override_map(
+//     {'IdentityN': 'OverrideGradientWithG'}):
+//   y, _ = identity_n([f(x), x])
+//
+// @tf.RegisterGradient('OverrideGradientWithG')
+// def ApplyG(op, dy, _):
+//   return [None, g(dy)]  # Do not backprop to f(x).
+// ```
+func IdentityN(scope *Scope, input []tf.Output) (output []tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "IdentityN",
+		Input: []tf.Input{
+			tf.OutputList(input),
+		},
+	}
+	op := scope.AddOperation(opspec)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("IdentityN", err)
+		return
+	}
+	return output
+}
+
+// Computes the gradient of the sigmoid of `x` wrt its input.
+//
+// Specifically, `grad = dy * y * (1 - y)`, where `y = sigmoid(x)`, and
+// `dy` is the corresponding input gradient.
+func SigmoidGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "SigmoidGrad",
+		Input: []tf.Input{
+			y, dy,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Convert one or more images from HSV to RGB.
+//
+// Outputs a tensor of the same shape as the `images` tensor, containing the RGB
+// value of the pixels. The output is only well defined if the value in `images`
+// are in `[0,1]`.
+//
+// See `rgb_to_hsv` for a description of the HSV encoding.
+//
+// Arguments:
+//	images: 1-D or higher rank. HSV data to convert. Last dimension must be size 3.
+//
+// Returns `images` converted to RGB.
+func HSVToRGB(scope *Scope, images tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
 		Type: "HSVToRGB",
@@ -22780,305 +23372,112 @@ func TensorArrayGatherV2ElementShape(value tf.Shape) TensorArrayGatherV2Attr {
 func TensorArrayGatherV2(scope *Scope, handle tf.Output, indices tf.Output, flow_in tf.Output, dtype tf.DataType, optional ...TensorArrayGatherV2Attr) (value tf.Output) {
 	if scope.Err() != nil {
 		return
-	}
-	attrs := map[string]interface{}{"dtype": dtype}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "TensorArrayGatherV2",
-		Input: []tf.Input{
-			handle, indices, flow_in,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Interleave the values from the `data` tensors into a single tensor.
-//
-// Builds a merged tensor such that
-//
-// ```python
-//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
-// ```
-//
-// For example, if each `indices[m]` is scalar or vector, we have
-//
-// ```python
-//     # Scalar indices:
-//     merged[indices[m], ...] = data[m][...]
-//
-//     # Vector indices:
-//     merged[indices[m][i], ...] = data[m][i, ...]
-// ```
-//
-// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
-// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
-// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
-// `constant`, the output shape is
-//
-//     merged.shape = [max(indices)] + constant
-//
-// Values may be merged in parallel, so if an index appears in both `indices[m][i]`
-// and `indices[n][j]`, the result may be invalid. This differs from the normal
-// DynamicStitch operator that defines the behavior in that case.
-//
-// For example:
-//
-// ```python
-//     indices[0] = 6
-//     indices[1] = [4, 1]
-//     indices[2] = [[5, 2], [0, 3]]
-//     data[0] = [61, 62]
-//     data[1] = [[41, 42], [11, 12]]
-//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
-//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
-//               [51, 52], [61, 62]]
-// ```
-//
-// This method can be used to merge partitions created by `dynamic_partition`
-// as illustrated on the following example:
-//
-// ```python
-//     # Apply function (increments x_i) on elements for which a certain condition
-//     # apply (x_i != -1 in this example).
-//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
-//     condition_mask=tf.not_equal(x,tf.constant(-1.))
-//     partitioned_data = tf.dynamic_partition(
-//         x, tf.cast(condition_mask, tf.int32) , 2)
-//     partitioned_data[1] = partitioned_data[1] + 1.0
-//     condition_indices = tf.dynamic_partition(
-//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
-//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
-//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
-//     # unchanged.
-// ```
-//
-// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
-// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
-// </div>
-func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ParallelDynamicStitch",
-		Input: []tf.Input{
-			tf.OutputList(indices), tf.OutputList(data),
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes the gradient for the inverse of `x` wrt its input.
-//
-// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
-// is the corresponding input gradient.
-func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "InvGrad",
-		Input: []tf.Input{
-			y, dy,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// StridedSliceAttr is an optional argument to StridedSlice.
-type StridedSliceAttr func(optionalAttr)
-
-// StridedSliceBeginMask sets the optional begin_mask attribute to value.
-//
-// value: a bitmask where a bit i being 1 means to ignore the begin
-// value and instead use the largest interval possible. At runtime
-// begin[i] will be replaced with `[0, n-1) if `stride[i] > 0` or
-// `[-1, n-1]` if `stride[i] < 0`
-// If not specified, defaults to 0
-func StridedSliceBeginMask(value int64) StridedSliceAttr {
-	return func(m optionalAttr) {
-		m["begin_mask"] = value
-	}
-}
-
-// StridedSliceEndMask sets the optional end_mask attribute to value.
-//
-// value: analogous to `begin_mask`
-// If not specified, defaults to 0
-func StridedSliceEndMask(value int64) StridedSliceAttr {
-	return func(m optionalAttr) {
-		m["end_mask"] = value
-	}
-}
-
-// StridedSliceEllipsisMask sets the optional ellipsis_mask attribute to value.
-//
-// value: a bitmask where bit `i` being 1 means the `i`th
-// position is actually an ellipsis. One bit at most can be 1.
-// If `ellipsis_mask == 0`, then an implicit ellipsis mask of `1 << (m+1)`
-// is provided. This means that `foo[3:5] == foo[3:5, ...]`. An ellipsis
-// implicitly creates as many range specifications as necessary to fully
-// specify the sliced range for every dimension. For example for a 4-dimensional
-// tensor `foo` the slice `foo[2, ..., 5:8]` implies `foo[2, :, :, 5:8]`.
-// If not specified, defaults to 0
-func StridedSliceEllipsisMask(value int64) StridedSliceAttr {
-	return func(m optionalAttr) {
-		m["ellipsis_mask"] = value
-	}
-}
-
-// StridedSliceNewAxisMask sets the optional new_axis_mask attribute to value.
-//
-// value: a bitmask where bit `i` being 1 means the `i`th
-// specification creates a new shape 1 dimension. For example
-// `foo[:4, tf.newaxis, :2]` would produce a shape `(4, 1, 2)` tensor.
-// If not specified, defaults to 0
-func StridedSliceNewAxisMask(value int64) StridedSliceAttr {
-	return func(m optionalAttr) {
-		m["new_axis_mask"] = value
-	}
-}
-
-// StridedSliceShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
-//
-// value: a bitmask where bit `i` implies that the `i`th
-// specification should shrink the dimensionality. begin and end
-// must imply a slice of size 1 in the dimension. For example in
-// python one might do `foo[:, 3, :]` which would result in
-// `shrink_axis_mask` being 2.
-// If not specified, defaults to 0
-func StridedSliceShrinkAxisMask(value int64) StridedSliceAttr {
-	return func(m optionalAttr) {
-		m["shrink_axis_mask"] = value
-	}
-}
-
-// Return a strided slice from `input`.
-//
-// Note, most python users will want to use the Python `Tensor.__getitem__`
-// or `Variable.__getitem__` rather than this op directly.
-//
-// The goal of this op is to produce a new tensor with a subset of
-// the elements from the `n` dimensional `input` tensor. The subset is chosen using
-// a sequence of `m` sparse range specifications encoded into the arguments
-// of this function. Note, in some cases
-// `m` could be equal to `n`, but this need not be the case. Each
-// range specification entry can be one of the following:
-//
-// - An ellipsis (...). Ellipses are used to imply zero or more
-//   dimensions of full-dimension selection and are produced using
-//   `ellipsis_mask`. For example, `foo[...]` is the identity slice.
-//
-// - A new axis. This is used to insert a new shape=1 dimension and is
-//   produced using `new_axis_mask`. For example, `foo[:, ...]` where
-//   `foo` is shape `(3, 4)` produces a `(1, 3, 4)` tensor.
-//
-//
-// - A range `begin:end:stride`. This is used to specify how much to choose from
-//   a given dimension. `stride` can be any integer but 0.  `begin` is an integer
-//   which represents the index of the first value to select while `end` represents
-//   the index of the last value to select. The number of values selected in each
-//   dimension is `end - begin` if `stride > 0` and `begin - end` if `stride < 0`.
-//   `begin` and `end` can be negative where `-1` is the last element, `-2` is
-//   the second to last. `begin_mask` controls whether to replace the explicitly
-//   given `begin` with an implicit effective value of `0` if `stride > 0` and
-//   `-1` if `stride < 0`. `end_mask` is analogous but produces the number
-//   required to create the largest open interval. For example, given a shape
-//   `(3,)` tensor `foo[:]`, the effective `begin` and `end` are `0` and `3`. Do
-//   not assume this is equivalent to `foo[0:-1]` which has an effective `begin`
-//   and `end` of `0` and `2`. Another example is `foo[-2::-1]` which reverses the
-//   first dimension of a tensor while dropping the last two (in the original
-//   order elements). For example `foo = [1,2,3,4]; foo[-2::-1]` is `[4,3]`.
-//
-// - A single index. This is used to keep only elements that have a given
-//   index. For example (`foo[2, :]` on a shape `(5,6)` tensor produces a
-//   shape `(6,)` tensor. This is encoded in `begin` and `end` and
-//   `shrink_axis_mask`.
+	}
+	attrs := map[string]interface{}{"dtype": dtype}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "TensorArrayGatherV2",
+		Input: []tf.Input{
+			handle, indices, flow_in,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Interleave the values from the `data` tensors into a single tensor.
 //
-// Each conceptual range specification is encoded in the op's argument. This
-// encoding is best understand by considering a non-trivial example. In
-// particular,
-// `foo[1, 2:4, None, ..., :-3:-1, :]` will be encoded as
+// Builds a merged tensor such that
 //
-// ```
-// begin = [1, 2, x, x, 0, x] # x denotes don't care (usually 0)
-// end = [2, 4, x, x, -3, x]
-// strides = [1, 1, x, x, -1, 1]
-// begin_mask = 1<<4 | 1 << 5 = 48
-// end_mask = 1<<5 = 32
-// ellipsis_mask = 1<<3 = 8
-// new_axis_mask = 1<<2 4
-// shrink_axis_mask = 1<<0
+// ```python
+//     merged[indices[m][i, ..., j], ...] = data[m][i, ..., j, ...]
 // ```
 //
-// In this case if `foo.shape` is (5, 5, 5, 5, 5, 5) the final shape of
-// the slice becomes (2, 1, 5, 5, 2, 5).
-// Let us walk step by step through each argument specification.
+// For example, if each `indices[m]` is scalar or vector, we have
 //
-// 1.  The first argument in the example slice is turned into `begin = 1` and
-// `end = begin + 1 = 2`. To disambiguate from the original spec `2:4` we
-// also set the appropriate bit in `shrink_axis_mask`.
+// ```python
+//     # Scalar indices:
+//     merged[indices[m], ...] = data[m][...]
 //
-// 2. `2:4` is contributes 2, 4, 1 to begin, end, and stride. All masks have
-// zero bits contributed.
+//     # Vector indices:
+//     merged[indices[m][i], ...] = data[m][i, ...]
+// ```
 //
-// 3. None is a synonym for `tf.newaxis`. This means insert a dimension of size 1
-// dimension in the final shape. Dummy values are contributed to begin,
-// end and stride, while the new_axis_mask bit is set.
+// Each `data[i].shape` must start with the corresponding `indices[i].shape`,
+// and the rest of `data[i].shape` must be constant w.r.t. `i`.  That is, we
+// must have `data[i].shape = indices[i].shape + constant`.  In terms of this
+// `constant`, the output shape is
 //
-// 4. `...` grab the full ranges from as many dimensions as needed to
-// fully specify a slice for every dimension of the input shape.
+//     merged.shape = [max(indices)] + constant
 //
-// 5. `:-3:-1` shows the use of negative indices. A negative index `i` associated
-// with a dimension that has shape `s` is converted to a positive index
-// `s + i`. So `-1` becomes `s-1` (i.e. the last element). This conversion
-// is done internally so begin, end and strides receive x, -3, and -1.
-// The appropriate begin_mask bit is set to indicate the start range is the
-// full range (ignoring the x).
+// Values may be merged in parallel, so if an index appears in both `indices[m][i]`
+// and `indices[n][j]`, the result may be invalid. This differs from the normal
+// DynamicStitch operator that defines the behavior in that case.
 //
-// 6. `:` indicates that the entire contents of the corresponding dimension
-// is selected. This is equivalent to `::` or `0::1`. begin, end, and strides
-// receive 0, 0, and 1, respectively. The appropriate bits in `begin_mask` and
-// `end_mask` are also set.
+// For example:
 //
-// *Requirements*:
-//   `0 != strides[i] for i in [0, m)`
-//   `ellipsis_mask must be a power of two (only one ellipsis)`
+// ```python
+//     indices[0] = 6
+//     indices[1] = [4, 1]
+//     indices[2] = [[5, 2], [0, 3]]
+//     data[0] = [61, 62]
+//     data[1] = [[41, 42], [11, 12]]
+//     data[2] = [[[51, 52], [21, 22]], [[1, 2], [31, 32]]]
+//     merged = [[1, 2], [11, 12], [21, 22], [31, 32], [41, 42],
+//               [51, 52], [61, 62]]
+// ```
 //
-// Arguments:
+// This method can be used to merge partitions created by `dynamic_partition`
+// as illustrated on the following example:
 //
-//	begin: `begin[k]` specifies the offset into the `k`th range specification.
-// The exact dimension this corresponds to will be determined by context.
-// Out-of-bounds values will be silently clamped. If the `k`th bit of
-// `begin_mask` then `begin[k]` is ignored and the full range of the
-// appropriate dimension is used instead. Negative values causes indexing
-// to start from the highest element e.g. If `foo==[1,2,3]` then `foo[-1]==3`.
-//	end: `end[i]` is like `begin` with the exception that `end_mask` is
-// used to determine full ranges.
-//	strides: `strides[i]` specifies the increment in the `i`th specification
-// after extracting a given element. Negative indices will reverse
-// the original order. Out or range values are
-// clamped to `[0,dim[i]) if slice[i]>0` or `[-1,dim[i]-1] if slice[i] < 0`
-func StridedSlice(scope *Scope, input tf.Output, begin tf.Output, end tf.Output, strides tf.Output, optional ...StridedSliceAttr) (output tf.Output) {
+// ```python
+//     # Apply function (increments x_i) on elements for which a certain condition
+//     # apply (x_i != -1 in this example).
+//     x=tf.constant([0.1, -1., 5.2, 4.3, -1., 7.4])
+//     condition_mask=tf.not_equal(x,tf.constant(-1.))
+//     partitioned_data = tf.dynamic_partition(
+//         x, tf.cast(condition_mask, tf.int32) , 2)
+//     partitioned_data[1] = partitioned_data[1] + 1.0
+//     condition_indices = tf.dynamic_partition(
+//         tf.range(tf.shape(x)[0]), tf.cast(condition_mask, tf.int32) , 2)
+//     x = tf.dynamic_stitch(condition_indices, partitioned_data)
+//     # Here x=[1.1, -1., 6.2, 5.3, -1, 8.4], the -1. values remain
+//     # unchanged.
+// ```
+//
+// <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+// <img style="width:100%" src="https://www.tensorflow.org/images/DynamicStitch.png" alt>
+// </div>
+func ParallelDynamicStitch(scope *Scope, indices []tf.Output, data []tf.Output) (merged tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
+	opspec := tf.OpSpec{
+		Type: "ParallelDynamicStitch",
+		Input: []tf.Input{
+			tf.OutputList(indices), tf.OutputList(data),
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// Computes the gradient for the inverse of `x` wrt its input.
+//
+// Specifically, `grad = -dy * y*y`, where `y = 1/x`, and `dy`
+// is the corresponding input gradient.
+func InvGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
 	opspec := tf.OpSpec{
-		Type: "StridedSlice",
+		Type: "InvGrad",
 		Input: []tf.Input{
-			input, begin, end, strides,
+			y, dy,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
@@ -23230,117 +23629,7 @@ func Unstage(scope *Scope, dtypes []tf.DataType, optional ...UnstageAttr) (value
 		scope.UpdateErr("Unstage", err)
 		return
 	}
-	return values
-}
-
-// ArgMaxAttr is an optional argument to ArgMax.
-type ArgMaxAttr func(optionalAttr)
-
-// ArgMaxOutputType sets the optional output_type attribute to value.
-// If not specified, defaults to DT_INT64
-func ArgMaxOutputType(value tf.DataType) ArgMaxAttr {
-	return func(m optionalAttr) {
-		m["output_type"] = value
-	}
-}
-
-// Returns the index with the largest value across dimensions of a tensor.
-//
-// Note that in case of ties the identity of the return value is not guaranteed.
-//
-// Arguments:
-//
-//	dimension: int32 or int64, must be in the range `[-rank(input), rank(input))`.
-// Describes which dimension of the input Tensor to reduce across. For vectors,
-// use dimension = 0.
-func ArgMax(scope *Scope, input tf.Output, dimension tf.Output, optional ...ArgMaxAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ArgMax",
-		Input: []tf.Input{
-			input, dimension,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// ResourceStridedSliceAssignAttr is an optional argument to ResourceStridedSliceAssign.
-type ResourceStridedSliceAssignAttr func(optionalAttr)
-
-// ResourceStridedSliceAssignBeginMask sets the optional begin_mask attribute to value.
-// If not specified, defaults to 0
-func ResourceStridedSliceAssignBeginMask(value int64) ResourceStridedSliceAssignAttr {
-	return func(m optionalAttr) {
-		m["begin_mask"] = value
-	}
-}
-
-// ResourceStridedSliceAssignEndMask sets the optional end_mask attribute to value.
-// If not specified, defaults to 0
-func ResourceStridedSliceAssignEndMask(value int64) ResourceStridedSliceAssignAttr {
-	return func(m optionalAttr) {
-		m["end_mask"] = value
-	}
-}
-
-// ResourceStridedSliceAssignEllipsisMask sets the optional ellipsis_mask attribute to value.
-// If not specified, defaults to 0
-func ResourceStridedSliceAssignEllipsisMask(value int64) ResourceStridedSliceAssignAttr {
-	return func(m optionalAttr) {
-		m["ellipsis_mask"] = value
-	}
-}
-
-// ResourceStridedSliceAssignNewAxisMask sets the optional new_axis_mask attribute to value.
-// If not specified, defaults to 0
-func ResourceStridedSliceAssignNewAxisMask(value int64) ResourceStridedSliceAssignAttr {
-	return func(m optionalAttr) {
-		m["new_axis_mask"] = value
-	}
-}
-
-// ResourceStridedSliceAssignShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
-// If not specified, defaults to 0
-func ResourceStridedSliceAssignShrinkAxisMask(value int64) ResourceStridedSliceAssignAttr {
-	return func(m optionalAttr) {
-		m["shrink_axis_mask"] = value
-	}
-}
-
-// Assign `value` to the sliced l-value reference of `ref`.
-//
-// The values of `value` are assigned to the positions in the variable
-// `ref` that are selected by the slice parameters. The slice parameters
-// `begin, `end`, `strides`, etc. work exactly as in `StridedSlice`.
-//
-// NOTE this op currently does not support broadcasting and so `value`'s
-// shape must be exactly the shape produced by the slice of `ref`.
-//
-// Returns the created operation.
-func ResourceStridedSliceAssign(scope *Scope, ref tf.Output, begin tf.Output, end tf.Output, strides tf.Output, value tf.Output, optional ...ResourceStridedSliceAssignAttr) (o *tf.Operation) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ResourceStridedSliceAssign",
-		Input: []tf.Input{
-			ref, begin, end, strides, value,
-		},
-		Attrs: attrs,
-	}
-	return scope.AddOperation(opspec)
+	return values
 }
 
 // QueueEnqueueV2Attr is an optional argument to QueueEnqueueV2.
@@ -26224,259 +26513,96 @@ func FixedUnigramCandidateSampler(scope *Scope, true_classes tf.Output, num_true
 	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Elementwise computes the bitwise AND of `x` and `y`.
-//
-// The result will have those bits set, that are set in both `x` and `y`. The
-// computation is performed on the underlying representations of `x` and `y`.
-func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "BitwiseAnd",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Elementwise computes the bitwise left-shift of `x` and `y`.
-//
-// If `y` is negative, or greater than or equal to the width of `x` in bits the
-// result is implementation defined.
-func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "LeftShift",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Elementwise computes the bitwise right-shift of `x` and `y`.
-//
-// Performs a logical shift for unsigned integer types, and an arithmetic shift
-// for signed integer types.
-//
-// If `y` is negative, or greater than or equal to than the width of `x` in bits
-// the result is implementation defined.
-func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "RightShift",
-		Input: []tf.Input{
-			x, y,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
+// WholeFileReaderV2Attr is an optional argument to WholeFileReaderV2.
+type WholeFileReaderV2Attr func(optionalAttr)
 
-// Adjust the hue of one or more images.
-//
-// `images` is a tensor of at least 3 dimensions.  The last dimension is
-// interpretted as channels, and must be three.
-//
-// The input image is considered in the RGB colorspace. Conceptually, the RGB
-// colors are first mapped into HSV. A delta is then applied all the hue values,
-// and then remapped back to RGB colorspace.
-//
-// Arguments:
-//	images: Images to adjust.  At least 3-D.
-//	delta: A float delta to add to the hue.
+// WholeFileReaderV2Container sets the optional container attribute to value.
 //
-// Returns The hue-adjusted image or images.
-func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "AdjustHue",
-		Input: []tf.Input{
-			images, delta,
-		},
+// value: If non-empty, this reader is placed in the given container.
+// Otherwise, a default container is used.
+// If not specified, defaults to ""
+func WholeFileReaderV2Container(value string) WholeFileReaderV2Attr {
+	return func(m optionalAttr) {
+		m["container"] = value
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
 }
 
-// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
-type AvgPool3DGradAttr func(optionalAttr)
-
-// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
+// WholeFileReaderV2SharedName sets the optional shared_name attribute to value.
 //
-// value: The data format of the input and output data. With the
-// default format "NDHWC", the data is stored in the order of:
-//     [batch, in_depth, in_height, in_width, in_channels].
-// Alternatively, the format could be "NCDHW", the data storage order is:
-//     [batch, in_channels, in_depth, in_height, in_width].
-// If not specified, defaults to "NDHWC"
-func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+// value: If non-empty, this reader is named in the given bucket
+// with this shared_name. Otherwise, the node name is used instead.
+// If not specified, defaults to ""
+func WholeFileReaderV2SharedName(value string) WholeFileReaderV2Attr {
 	return func(m optionalAttr) {
-		m["data_format"] = value
+		m["shared_name"] = value
 	}
 }
 
-// Computes gradients of average pooling function.
+// A Reader that outputs the entire contents of a file as a value.
 //
-// Arguments:
-//	orig_input_shape: The original input dimensions.
-//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
-//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
-// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
-//	strides: 1-D tensor of length 5. The stride of the sliding window for each
-// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
-//	padding: The type of padding algorithm to use.
+// To use, enqueue filenames in a Queue.  The output of ReaderRead will
+// be a filename (key) and the contents of that file (value).
 //
-// Returns The backprop for input.
-func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
+// Returns The handle to reference the Reader.
+func WholeFileReaderV2(scope *Scope, optional ...WholeFileReaderV2Attr) (reader_handle tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	attrs := map[string]interface{}{}
 	for _, a := range optional {
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "AvgPool3DGrad",
-		Input: []tf.Input{
-			orig_input_shape, grad,
-		},
+		Type: "WholeFileReaderV2",
+
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
-type ParseSingleSequenceExampleAttr func(optionalAttr)
-
-// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
-//
-// value: A list of Ncontext_sparse types; the data types of data in
-// each context Feature given in context_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["context_sparse_types"] = value
-	}
-}
-
-// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["feature_list_dense_types"] = value
-	}
-}
-
-// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
-//
-// value: A list of Ncontext_dense shapes; the shapes of data in
-// each context Feature given in context_dense_keys.
-// The number of elements in the Feature corresponding to context_dense_key[j]
-// must always equal context_dense_shapes[j].NumEntries().
-// The shape of context_dense_values[j] will match context_dense_shapes[j].
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["context_dense_shapes"] = value
-	}
-}
-
-// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
-//
-// value: A list of Nfeature_list_sparse types; the data types
-// of data in each FeatureList given in feature_list_sparse_keys.
-// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
-// DT_INT64 (Int64List), and DT_STRING (BytesList).
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["feature_list_sparse_types"] = value
-	}
-}
-
-// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
-//
-// value: A list of Nfeature_list_dense shapes; the shapes of
-// data in each FeatureList given in feature_list_dense_keys.
-// The shape of each Feature in the FeatureList corresponding to
-// feature_list_dense_key[j] must always equal
-// feature_list_dense_shapes[j].NumEntries().
-// If not specified, defaults to <>
-//
-// REQUIRES: len(value) >= 0
-func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
-	return func(m optionalAttr) {
-		m["feature_list_dense_shapes"] = value
-	}
-}
-
-// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
+// Transforms a tf.Example proto (as a string) into typed tensors.
 //
 // Arguments:
-//	serialized: A scalar containing a binary serialized SequenceExample proto.
-//	feature_list_dense_missing_assumed_empty: A vector listing the
-// FeatureList keys which may be missing from the SequenceExample.  If the
-// associated FeatureList is missing, it is treated as empty.  By default,
-// any FeatureList not listed in this vector must exist in the SequenceExample.
-//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
-// The keys expected in the Examples' features associated with context_sparse
-// values.
-//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' context features associated with
-// dense values.
-//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
-// (scalars).  The keys expected in the FeatureLists associated with sparse
+//	serialized: A vector containing a batch of binary serialized Example protos.
+//	dense_defaults: A list of Tensors (some may be empty), whose length matches
+// the length of `dense_keys`. dense_defaults[j] provides default values
+// when the example's feature_map lacks dense_key[j].  If an empty Tensor is
+// provided for dense_defaults[j], then the Feature dense_keys[j] is required.
+// The input type is inferred from dense_defaults[j], even when it's empty.
+// If dense_defaults[j] is not empty, and dense_shapes[j] is fully defined,
+// then the shape of dense_defaults[j] must match that of dense_shapes[j].
+// If dense_shapes[j] has an undefined major dimension (variable strides dense
+// feature), dense_defaults[j] must contain a single element:
+// the padding element.
+//	num_sparse: The number of sparse features to be parsed from the example. This
+// must match the lengths of `sparse_keys` and `sparse_types`.
+//	sparse_keys: A list of `num_sparse` strings.
+// The keys expected in the Examples' features associated with sparse values.
+//	dense_keys: The keys expected in the Examples' features associated with dense
 // values.
-//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
-// The keys expected in the SequenceExamples' feature_lists associated
-// with lists of dense values.
-//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
-// context_dense_defaults[j] provides default values
-// when the SequenceExample's context map lacks context_dense_key[j].
-// If an empty Tensor is provided for context_dense_defaults[j],
-// then the Feature context_dense_keys[j] is required.
-// The input type is inferred from context_dense_defaults[j], even when it's
-// empty.  If context_dense_defaults[j] is not empty, its shape must match
-// context_dense_shapes[j].
-//	debug_name: A scalar containing the name of the serialized proto.
-// May contain, for example, table key (descriptive) name for the
-// corresponding serialized proto.  This is purely useful for debugging
-// purposes, and the presence of values here has no effect on the output.
-// May also be an empty scalar if no name is available.
-func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
+//	sparse_types: A list of `num_sparse` types; the data types of data in each
+// Feature given in sparse_keys.
+// Currently the ParseSingleExample op supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+//	dense_shapes: The shapes of data in each Feature given in dense_keys.
+// The length of this list must match the length of `dense_keys`.  The
+// number of elements in the Feature corresponding to dense_key[j] must
+// always equal dense_shapes[j].NumEntries().  If dense_shapes[j] ==
+// (D0, D1, ..., DN) then the shape of output Tensor dense_values[j]
+// will be (D0, D1, ..., DN): In the case dense_shapes[j] = (-1, D1,
+// ..., DN), the shape of the output Tensor dense_values[j] will be (M,
+// D1, .., DN), where M is the number of blocks of elements of length
+// D1 * .... * DN, in the input.
+func ParseSingleExample(scope *Scope, serialized tf.Output, dense_defaults []tf.Output, num_sparse int64, sparse_keys []string, dense_keys []string, sparse_types []tf.DataType, dense_shapes []tf.Shape) (sparse_indices []tf.Output, sparse_values []tf.Output, sparse_shapes []tf.Output, dense_values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"num_sparse": num_sparse, "sparse_keys": sparse_keys, "dense_keys": dense_keys, "sparse_types": sparse_types, "dense_shapes": dense_shapes}
 	opspec := tf.OpSpec{
-		Type: "ParseSingleSequenceExample",
+		Type: "ParseSingleExample",
 		Input: []tf.Input{
-			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
+			serialized, tf.OutputList(dense_defaults),
 		},
 		Attrs: attrs,
 	}
@@ -26486,290 +26612,278 @@ func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list
 	}
 	var idx int
 	var err error
-	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
-		return
-	}
-	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
+	if sparse_indices, idx, err = makeOutputList(op, idx, "sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
 		return
 	}
-	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
+	if sparse_values, idx, err = makeOutputList(op, idx, "sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
 		return
 	}
-	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
+	if sparse_shapes, idx, err = makeOutputList(op, idx, "sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
 		return
 	}
-	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
-		scope.UpdateErr("ParseSingleSequenceExample", err)
+	if dense_values, idx, err = makeOutputList(op, idx, "dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleExample", err)
 		return
 	}
-	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
+	return sparse_indices, sparse_values, sparse_shapes, dense_values
 }
 
-// DecodeWavAttr is an optional argument to DecodeWav.
-type DecodeWavAttr func(optionalAttr)
-
-// DecodeWavDesiredChannels sets the optional desired_channels attribute to value.
+// Elementwise computes the bitwise AND of `x` and `y`.
 //
-// value: Number of sample channels wanted.
-// If not specified, defaults to -1
-func DecodeWavDesiredChannels(value int64) DecodeWavAttr {
-	return func(m optionalAttr) {
-		m["desired_channels"] = value
+// The result will have those bits set, that are set in both `x` and `y`. The
+// computation is performed on the underlying representations of `x` and `y`.
+func BitwiseAnd(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
-}
-
-// DecodeWavDesiredSamples sets the optional desired_samples attribute to value.
-//
-// value: Length of audio requested.
-// If not specified, defaults to -1
-func DecodeWavDesiredSamples(value int64) DecodeWavAttr {
-	return func(m optionalAttr) {
-		m["desired_samples"] = value
+	opspec := tf.OpSpec{
+		Type: "BitwiseAnd",
+		Input: []tf.Input{
+			x, y,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Decode a 16-bit PCM WAV file to a float tensor.
-//
-// The -32768 to 32767 signed 16-bit values will be scaled to -1.0 to 1.0 in float.
-//
-// When desired_channels is set, if the input contains fewer channels than this
-// then the last channel will be duplicated to give the requested number, else if
-// the input has more channels than requested then the additional channels will be
-// ignored.
-//
-// If desired_samples is set, then the audio will be cropped or padded with zeroes
-// to the requested length.
-//
-// The first output contains a Tensor with the content of the audio samples. The
-// lowest dimension will be the number of channels, and the second will be the
-// number of samples. For example, a ten-sample-long stereo WAV file should give an
-// output shape of [10, 2].
-//
-// Arguments:
-//	contents: The WAV-encoded audio, usually from a file.
+// Elementwise computes the bitwise left-shift of `x` and `y`.
 //
-// Returns 2-D with shape `[length, channels]`.Scalar holding the sample rate found in the WAV header.
-func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (audio tf.Output, sample_rate tf.Output) {
+// If `y` is negative, or greater than or equal to the width of `x` in bits the
+// result is implementation defined.
+func LeftShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "DecodeWav",
+		Type: "LeftShift",
 		Input: []tf.Input{
-			contents,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
-}
-
-// UniqueAttr is an optional argument to Unique.
-type UniqueAttr func(optionalAttr)
-
-// UniqueOutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueOutIdx(value tf.DataType) UniqueAttr {
-	return func(m optionalAttr) {
-		m["out_idx"] = value
-	}
+	return op.Output(0)
 }
 
-// Finds unique elements in a 1-D tensor.
-//
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. In other words:
-//
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
-//
-// For example:
-//
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx = unique(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// ```
+// Elementwise computes the bitwise right-shift of `x` and `y`.
 //
-// Arguments:
-//	x: 1-D.
+// Performs a logical shift for unsigned integer types, and an arithmetic shift
+// for signed integer types.
 //
-// Returns 1-D.1-D.
-func Unique(scope *Scope, x tf.Output, optional ...UniqueAttr) (y tf.Output, idx tf.Output) {
+// If `y` is negative, or greater than or equal to than the width of `x` in bits
+// the result is implementation defined.
+func RightShift(scope *Scope, x tf.Output, y tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "Unique",
+		Type: "RightShift",
 		Input: []tf.Input{
-			x,
+			x, y,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0)
 }
 
-// Concatenates a list of `N` tensors along the first dimension.
-//
-// The input tensors are all required to have size 1 in the first dimension.
-//
-// For example:
+// Adjust the hue of one or more images.
 //
-// ```
-// # 'x' is [[1, 4]]
-// # 'y' is [[2, 5]]
-// # 'z' is [[3, 6]]
-// parallel_concat([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
-// ```
+// `images` is a tensor of at least 3 dimensions.  The last dimension is
+// interpretted as channels, and must be three.
 //
-// The difference between concat and parallel_concat is that concat requires all
-// of the inputs be computed before the operation will begin but doesn't require
-// that the input shapes be known during graph construction.  Parallel concat
-// will copy pieces of the input into the output as they become available, in
-// some situations this can provide a performance benefit.
+// The input image is considered in the RGB colorspace. Conceptually, the RGB
+// colors are first mapped into HSV. A delta is then applied all the hue values,
+// and then remapped back to RGB colorspace.
 //
 // Arguments:
-//	values: Tensors to be concatenated. All must have size 1 in the first dimension
-// and same shape.
-//	shape: the final shape of the result; should be equal to the shapes of any input
-// but with the number of input values in the first dimension.
+//	images: Images to adjust.  At least 3-D.
+//	delta: A float delta to add to the hue.
 //
-// Returns The concatenated tensor.
-func ParallelConcat(scope *Scope, values []tf.Output, shape tf.Shape) (output tf.Output) {
+// Returns The hue-adjusted image or images.
+func AdjustHue(scope *Scope, images tf.Output, delta tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "ParallelConcat",
+		Type: "AdjustHue",
 		Input: []tf.Input{
-			tf.OutputList(values),
+			images, delta,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Compute the lower regularized incomplete Gamma function `Q(a, x)`.
-//
-// The lower regularized incomplete Gamma function is defined as:
-//
-//
-// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
-//
-// where
+// AvgPool3DGradAttr is an optional argument to AvgPool3DGrad.
+type AvgPool3DGradAttr func(optionalAttr)
+
+// AvgPool3DGradDataFormat sets the optional data_format attribute to value.
 //
-// \\(gamma(a, x) = int_{0}^{x} t^{a-1} exp(-t) dt\\)
+// value: The data format of the input and output data. With the
+// default format "NDHWC", the data is stored in the order of:
+//     [batch, in_depth, in_height, in_width, in_channels].
+// Alternatively, the format could be "NCDHW", the data storage order is:
+//     [batch, in_channels, in_depth, in_height, in_width].
+// If not specified, defaults to "NDHWC"
+func AvgPool3DGradDataFormat(value string) AvgPool3DGradAttr {
+	return func(m optionalAttr) {
+		m["data_format"] = value
+	}
+}
+
+// Computes gradients of average pooling function.
 //
-// is the lower incomplete Gamma function.
+// Arguments:
+//	orig_input_shape: The original input dimensions.
+//	grad: Output backprop of shape `[batch, depth, rows, cols, channels]`.
+//	ksize: 1-D tensor of length 5. The size of the window for each dimension of
+// the input tensor. Must have `ksize[0] = ksize[4] = 1`.
+//	strides: 1-D tensor of length 5. The stride of the sliding window for each
+// dimension of `input`. Must have `strides[0] = strides[4] = 1`.
+//	padding: The type of padding algorithm to use.
 //
-// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
-// Gamma function.
-func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
+// Returns The backprop for input.
+func AvgPool3DGrad(scope *Scope, orig_input_shape tf.Output, grad tf.Output, ksize []int64, strides []int64, padding string, optional ...AvgPool3DGradAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"ksize": ksize, "strides": strides, "padding": padding}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Igamma",
+		Type: "AvgPool3DGrad",
 		Input: []tf.Input{
-			a, x,
+			orig_input_shape, grad,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Computes offsets of concat inputs within its output.
+// ParseSingleSequenceExampleAttr is an optional argument to ParseSingleSequenceExample.
+type ParseSingleSequenceExampleAttr func(optionalAttr)
+
+// ParseSingleSequenceExampleContextSparseTypes sets the optional context_sparse_types attribute to value.
 //
-// For example:
+// value: A list of Ncontext_sparse types; the data types of data in
+// each context Feature given in context_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
 //
-// ```
-// # 'x' is [2, 2, 7]
-// # 'y' is [2, 3, 7]
-// # 'z' is [2, 5, 7]
-// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
-// ```
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["context_sparse_types"] = value
+	}
+}
+
+// ParseSingleSequenceExampleFeatureListDenseTypes sets the optional feature_list_dense_types attribute to value.
+// If not specified, defaults to <>
 //
-// This is typically used by gradient computations for a concat operation.
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_types"] = value
+	}
+}
+
+// ParseSingleSequenceExampleContextDenseShapes sets the optional context_dense_shapes attribute to value.
 //
-// Arguments:
-//	concat_dim: The dimension along which to concatenate.
-//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
+// value: A list of Ncontext_dense shapes; the shapes of data in
+// each context Feature given in context_dense_keys.
+// The number of elements in the Feature corresponding to context_dense_key[j]
+// must always equal context_dense_shapes[j].NumEntries().
+// The shape of context_dense_values[j] will match context_dense_shapes[j].
+// If not specified, defaults to <>
 //
-// Returns The `N` int32 vectors representing the starting offset
-// of input tensors within the concatenated output.
-func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "ConcatOffset",
-		Input: []tf.Input{
-			concat_dim, tf.OutputList(shape),
-		},
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleContextDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["context_dense_shapes"] = value
 	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
-		return
+}
+
+// ParseSingleSequenceExampleFeatureListSparseTypes sets the optional feature_list_sparse_types attribute to value.
+//
+// value: A list of Nfeature_list_sparse types; the data types
+// of data in each FeatureList given in feature_list_sparse_keys.
+// Currently the ParseSingleSequenceExample supports DT_FLOAT (FloatList),
+// DT_INT64 (Int64List), and DT_STRING (BytesList).
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListSparseTypes(value []tf.DataType) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_sparse_types"] = value
 	}
-	var idx int
-	var err error
-	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
-		scope.UpdateErr("ConcatOffset", err)
-		return
+}
+
+// ParseSingleSequenceExampleFeatureListDenseShapes sets the optional feature_list_dense_shapes attribute to value.
+//
+// value: A list of Nfeature_list_dense shapes; the shapes of
+// data in each FeatureList given in feature_list_dense_keys.
+// The shape of each Feature in the FeatureList corresponding to
+// feature_list_dense_key[j] must always equal
+// feature_list_dense_shapes[j].NumEntries().
+// If not specified, defaults to <>
+//
+// REQUIRES: len(value) >= 0
+func ParseSingleSequenceExampleFeatureListDenseShapes(value []tf.Shape) ParseSingleSequenceExampleAttr {
+	return func(m optionalAttr) {
+		m["feature_list_dense_shapes"] = value
 	}
-	return offset
 }
 
-// Splits a tensor into `num_split` tensors along one dimension.
+// Transforms a scalar brain.SequenceExample proto (as strings) into typed tensors.
 //
 // Arguments:
-//	axis: 0-D.  The dimension along which to split.  Must be in the range
-// `[-rank(value), rank(value))`.
-//	value: The tensor to split.
-//	num_split: The number of ways to split.  Must evenly divide
-// `value.shape[split_dim]`.
-//
-// Returns They are identically shaped tensors, whose shape matches that of `value`
-// except along `axis`, where their sizes are
-// `values.shape[split_dim] / num_split`.
-func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (output []tf.Output) {
+//	serialized: A scalar containing a binary serialized SequenceExample proto.
+//	feature_list_dense_missing_assumed_empty: A vector listing the
+// FeatureList keys which may be missing from the SequenceExample.  If the
+// associated FeatureList is missing, it is treated as empty.  By default,
+// any FeatureList not listed in this vector must exist in the SequenceExample.
+//	context_sparse_keys: A list of Ncontext_sparse string Tensors (scalars).
+// The keys expected in the Examples' features associated with context_sparse
+// values.
+//	context_dense_keys: A list of Ncontext_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' context features associated with
+// dense values.
+//	feature_list_sparse_keys: A list of Nfeature_list_sparse string Tensors
+// (scalars).  The keys expected in the FeatureLists associated with sparse
+// values.
+//	feature_list_dense_keys: A list of Nfeature_list_dense string Tensors (scalars).
+// The keys expected in the SequenceExamples' feature_lists associated
+// with lists of dense values.
+//	context_dense_defaults: A list of Ncontext_dense Tensors (some may be empty).
+// context_dense_defaults[j] provides default values
+// when the SequenceExample's context map lacks context_dense_key[j].
+// If an empty Tensor is provided for context_dense_defaults[j],
+// then the Feature context_dense_keys[j] is required.
+// The input type is inferred from context_dense_defaults[j], even when it's
+// empty.  If context_dense_defaults[j] is not empty, its shape must match
+// context_dense_shapes[j].
+//	debug_name: A scalar containing the name of the serialized proto.
+// May contain, for example, table key (descriptive) name for the
+// corresponding serialized proto.  This is purely useful for debugging
+// purposes, and the presence of values here has no effect on the output.
+// May also be an empty scalar if no name is available.
+func ParseSingleSequenceExample(scope *Scope, serialized tf.Output, feature_list_dense_missing_assumed_empty tf.Output, context_sparse_keys []tf.Output, context_dense_keys []tf.Output, feature_list_sparse_keys []tf.Output, feature_list_dense_keys []tf.Output, context_dense_defaults []tf.Output, debug_name tf.Output, optional ...ParseSingleSequenceExampleAttr) (context_sparse_indices []tf.Output, context_sparse_values []tf.Output, context_sparse_shapes []tf.Output, context_dense_values []tf.Output, feature_list_sparse_indices []tf.Output, feature_list_sparse_values []tf.Output, feature_list_sparse_shapes []tf.Output, feature_list_dense_values []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
 	opspec := tf.OpSpec{
-		Type: "Split",
+		Type: "ParseSingleSequenceExample",
 		Input: []tf.Input{
-			axis, value,
+			serialized, feature_list_dense_missing_assumed_empty, tf.OutputList(context_sparse_keys), tf.OutputList(context_dense_keys), tf.OutputList(feature_list_sparse_keys), tf.OutputList(feature_list_dense_keys), tf.OutputList(context_dense_defaults), debug_name,
 		},
 		Attrs: attrs,
 	}
@@ -26779,158 +26893,86 @@ func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (outp
 	}
 	var idx int
 	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("Split", err)
+	if context_sparse_indices, idx, err = makeOutputList(op, idx, "context_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	return output
-}
-
-// Splits a tensor into `num_split` tensors along one dimension.
-//
-// Arguments:
-//	value: The tensor to split.
-//	size_splits: list containing the sizes of each output tensor along the split
-// dimension. Must sum to the dimension of value along split_dim.
-// Can contain one -1 indicating that dimension is to be inferred.
-//	axis: 0-D.  The dimension along which to split.  Must be in the range
-// `[-rank(value), rank(value))`.
-//
-//
-// Returns Tensors whose shape matches that of `value`
-// except along `axis`, where their sizes are
-// `size_splits[i]`.
-func SplitV(scope *Scope, value tf.Output, size_splits tf.Output, axis tf.Output, num_split int64) (output []tf.Output) {
-	if scope.Err() != nil {
+	if context_sparse_values, idx, err = makeOutputList(op, idx, "context_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	attrs := map[string]interface{}{"num_split": num_split}
-	opspec := tf.OpSpec{
-		Type: "SplitV",
-		Input: []tf.Input{
-			value, size_splits, axis,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	if scope.Err() != nil {
+	if context_sparse_shapes, idx, err = makeOutputList(op, idx, "context_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	var idx int
-	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("SplitV", err)
+	if context_dense_values, idx, err = makeOutputList(op, idx, "context_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	return output
-}
-
-// Gives a guarantee to the TF runtime that the input tensor is a constant.
-//
-// The runtime is then free to make optimizations based on this.
-//
-// Only accepts value typed tensors as inputs and rejects resource variable handles
-// as input.
-//
-// Returns the input tensor without modification.
-func GuaranteeConst(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
+	if feature_list_sparse_indices, idx, err = makeOutputList(op, idx, "feature_list_sparse_indices"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "GuaranteeConst",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns a tensor of zeros with the same shape and type as x.
-//
-// Arguments:
-//	x: a tensor of type T.
-//
-// Returns a tensor of the same shape and type as x but filled with zeros.
-func ZerosLike(scope *Scope, x tf.Output) (y tf.Output) {
-	if scope.Err() != nil {
+	if feature_list_sparse_values, idx, err = makeOutputList(op, idx, "feature_list_sparse_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "ZerosLike",
-		Input: []tf.Input{
-			x,
-		},
+	if feature_list_sparse_shapes, idx, err = makeOutputList(op, idx, "feature_list_sparse_shapes"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
 	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
-type QuantizedInstanceNormAttr func(optionalAttr)
-
-// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
-//
-// value: If True, `given_y_min` and `given_y_min`
-// and `given_y_max` are used as the output range. Otherwise,
-// the implementation computes the output range.
-// If not specified, defaults to false
-func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["output_range_given"] = value
+	if feature_list_dense_values, idx, err = makeOutputList(op, idx, "feature_list_dense_values"); err != nil {
+		scope.UpdateErr("ParseSingleSequenceExample", err)
+		return
 	}
+	return context_sparse_indices, context_sparse_values, context_sparse_shapes, context_dense_values, feature_list_sparse_indices, feature_list_sparse_values, feature_list_sparse_shapes, feature_list_dense_values
 }
 
-// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
-//
-// value: Output in `y_min` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["given_y_min"] = value
-	}
-}
+// DecodeWavAttr is an optional argument to DecodeWav.
+type DecodeWavAttr func(optionalAttr)
 
-// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
+// DecodeWavDesiredChannels sets the optional desired_channels attribute to value.
 //
-// value: Output in `y_max` if `output_range_given` is True.
-// If not specified, defaults to 0
-func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
+// value: Number of sample channels wanted.
+// If not specified, defaults to -1
+func DecodeWavDesiredChannels(value int64) DecodeWavAttr {
 	return func(m optionalAttr) {
-		m["given_y_max"] = value
+		m["desired_channels"] = value
 	}
 }
 
-// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
+// DecodeWavDesiredSamples sets the optional desired_samples attribute to value.
 //
-// value: A small float number to avoid dividing by 0.
-// If not specified, defaults to 1e-05
-func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
+// value: Length of audio requested.
+// If not specified, defaults to -1
+func DecodeWavDesiredSamples(value int64) DecodeWavAttr {
 	return func(m optionalAttr) {
-		m["variance_epsilon"] = value
+		m["desired_samples"] = value
 	}
 }
 
-// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
+// Decode a 16-bit PCM WAV file to a float tensor.
 //
-// value: Minimum value of `y_max - y_min`
-// If not specified, defaults to 0.001
-func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
-	return func(m optionalAttr) {
-		m["min_separation"] = value
-	}
-}
-
-// Quantized Instance normalization.
+// The -32768 to 32767 signed 16-bit values will be scaled to -1.0 to 1.0 in float.
+//
+// When desired_channels is set, if the input contains fewer channels than this
+// then the last channel will be duplicated to give the requested number, else if
+// the input has more channels than requested then the additional channels will be
+// ignored.
+//
+// If desired_samples is set, then the audio will be cropped or padded with zeroes
+// to the requested length.
+//
+// The first output contains a Tensor with the content of the audio samples. The
+// lowest dimension will be the number of channels, and the second will be the
+// number of samples. For example, a ten-sample-long stereo WAV file should give an
+// output shape of [10, 2].
 //
 // Arguments:
-//	x: A 4D input Tensor.
-//	x_min: The value represented by the lowest quantized input.
-//	x_max: The value represented by the highest quantized input.
+//	contents: The WAV-encoded audio, usually from a file.
 //
-// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
-func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
+// Returns 2-D with shape `[length, channels]`.Scalar holding the sample rate found in the WAV header.
+func DecodeWav(scope *Scope, contents tf.Output, optional ...DecodeWavAttr) (audio tf.Output, sample_rate tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -26939,196 +26981,103 @@ func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizedInstanceNorm",
+		Type: "DecodeWav",
 		Input: []tf.Input{
-			x, x_min, x_max,
+			contents,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1), op.Output(2)
+	return op.Output(0), op.Output(1)
 }
 
-// Returns the diagonal part of the tensor.
-//
-// This operation returns a tensor with the `diagonal` part
-// of the `input`. The `diagonal` part is computed as follows:
+// UniqueAttr is an optional argument to Unique.
+type UniqueAttr func(optionalAttr)
+
+// UniqueOutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueOutIdx(value tf.DataType) UniqueAttr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Finds unique elements in a 1-D tensor.
 //
-// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
-// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. In other words:
 //
-// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
 //
 // For example:
 //
 // ```
-// # 'input' is [[1, 0, 0, 0]
-//               [0, 2, 0, 0]
-//               [0, 0, 3, 0]
-//               [0, 0, 0, 4]]
-//
-// tf.diag_part(input) ==> [1, 2, 3, 4]
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx = unique(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
 // ```
 //
 // Arguments:
-//	input: Rank k tensor where k is even and not zero.
+//	x: 1-D.
 //
-// Returns The extracted diagonal.
-func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
+// Returns 1-D.1-D.
+func Unique(scope *Scope, x tf.Output, optional ...UniqueAttr) (y tf.Output, idx tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "DiagPart",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Returns the element-wise max of two SparseTensors.
-//
-// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
-//
-// Arguments:
-//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
-// SparseTensor, in the canonical lexicographic ordering.
-//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
-//	a_shape: 1-D.  Shape of the input SparseTensor.
-//	b_indices: counterpart to `a_indices` for the other operand.
-//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
-//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
-//
-// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
-func SparseSparseMaximum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
-	if scope.Err() != nil {
-		return
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "SparseSparseMaximum",
+		Type: "Unique",
 		Input: []tf.Input{
-			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
+			x,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1)
 }
 
-// Returns a batched matrix tensor with new batched diagonal values.
-//
-// Given `input` and `diagonal`, this operation returns a tensor with the
-// same shape and values as `input`, except for the main diagonal of the
-// innermost matrices.  These will be overwritten by the values in `diagonal`.
-//
-// The output is computed as follows:
-//
-// Assume `input` has `k+1` dimensions `[I, J, K, ..., M, N]` and `diagonal` has
-// `k` dimensions `[I, J, K, ..., min(M, N)]`.  Then the output is a
-// tensor of rank `k+1` with dimensions `[I, J, K, ..., M, N]` where:
-//
-//   * `output[i, j, k, ..., m, n] = diagonal[i, j, k, ..., n]` for `m == n`.
-//   * `output[i, j, k, ..., m, n] = input[i, j, k, ..., m, n]` for `m != n`.
-//
-// Arguments:
-//	input: Rank `k+1`, where `k >= 1`.
-//	diagonal: Rank `k`, where `k >= 1`.
-//
-// Returns Rank `k+1`, with `output.shape = input.shape`.
-func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "MatrixSetDiag",
-		Input: []tf.Input{
-			input, diagonal,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// EditDistanceAttr is an optional argument to EditDistance.
-type EditDistanceAttr func(optionalAttr)
-
-// EditDistanceNormalize sets the optional normalize attribute to value.
+// Concatenates a list of `N` tensors along the first dimension.
 //
-// value: boolean (if true, edit distances are normalized by length of truth).
+// The input tensors are all required to have size 1 in the first dimension.
 //
-// The output is:
-// If not specified, defaults to true
-func EditDistanceNormalize(value bool) EditDistanceAttr {
-	return func(m optionalAttr) {
-		m["normalize"] = value
-	}
-}
-
-// Computes the (possibly normalized) Levenshtein Edit Distance.
+// For example:
 //
-// The inputs are variable-length sequences provided by SparseTensors
-//   (hypothesis_indices, hypothesis_values, hypothesis_shape)
-// and
-//   (truth_indices, truth_values, truth_shape).
+// ```
+// # 'x' is [[1, 4]]
+// # 'y' is [[2, 5]]
+// # 'z' is [[3, 6]]
+// parallel_concat([x, y, z]) => [[1, 4], [2, 5], [3, 6]]  # Pack along first dim.
+// ```
 //
-// The inputs are:
+// The difference between concat and parallel_concat is that concat requires all
+// of the inputs be computed before the operation will begin but doesn't require
+// that the input shapes be known during graph construction.  Parallel concat
+// will copy pieces of the input into the output as they become available, in
+// some situations this can provide a performance benefit.
 //
 // Arguments:
-//	hypothesis_indices: The indices of the hypothesis list SparseTensor.
-// This is an N x R int64 matrix.
-//	hypothesis_values: The values of the hypothesis list SparseTensor.
-// This is an N-length vector.
-//	hypothesis_shape: The shape of the hypothesis list SparseTensor.
-// This is an R-length vector.
-//	truth_indices: The indices of the truth list SparseTensor.
-// This is an M x R int64 matrix.
-//	truth_values: The values of the truth list SparseTensor.
-// This is an M-length vector.
-//	truth_shape: truth indices, vector.
-//
-// Returns A dense float tensor with rank R - 1.
-//
-// For the example input:
-//
-//     // hypothesis represents a 2x1 matrix with variable-length values:
-//     //   (0,0) = ["a"]
-//     //   (1,0) = ["b"]
-//     hypothesis_indices = [[0, 0, 0],
-//                           [1, 0, 0]]
-//     hypothesis_values = ["a", "b"]
-//     hypothesis_shape = [2, 1, 1]
-//
-//     // truth represents a 2x2 matrix with variable-length values:
-//     //   (0,0) = []
-//     //   (0,1) = ["a"]
-//     //   (1,0) = ["b", "c"]
-//     //   (1,1) = ["a"]
-//     truth_indices = [[0, 1, 0],
-//                      [1, 0, 0],
-//                      [1, 0, 1],
-//                      [1, 1, 0]]
-//     truth_values = ["a", "b", "c", "a"]
-//     truth_shape = [2, 2, 2]
-//     normalize = true
-//
-// The output will be:
+//	values: Tensors to be concatenated. All must have size 1 in the first dimension
+// and same shape.
+//	shape: the final shape of the result; should be equal to the shapes of any input
+// but with the number of input values in the first dimension.
 //
-//     // output is a 2x2 matrix with edit distances normalized by truth lengths.
-//     output = [[inf, 1.0],  // (0,0): no truth, (0,1): no hypothesis
-//               [0.5, 1.0]]  // (1,0): addition, (1,1): no hypothesis
-func EditDistance(scope *Scope, hypothesis_indices tf.Output, hypothesis_values tf.Output, hypothesis_shape tf.Output, truth_indices tf.Output, truth_values tf.Output, truth_shape tf.Output, optional ...EditDistanceAttr) (output tf.Output) {
+// Returns The concatenated tensor.
+func ParallelConcat(scope *Scope, values []tf.Output, shape tf.Shape) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"shape": shape}
 	opspec := tf.OpSpec{
-		Type: "EditDistance",
+		Type: "ParallelConcat",
 		Input: []tf.Input{
-			hypothesis_indices, hypothesis_values, hypothesis_shape, truth_indices, truth_values, truth_shape,
+			tf.OutputList(values),
 		},
 		Attrs: attrs,
 	}
@@ -27136,142 +27085,63 @@ func EditDistance(scope *Scope, hypothesis_indices tf.Output, hypothesis_values
 	return op.Output(0)
 }
 
-// Gather slices from `params` into a Tensor with shape specified by `indices`.
-//
-// `indices` is an K-dimensional integer tensor, best thought of as a
-// (K-1)-dimensional tensor of indices into `params`, where each element defines a
-// slice of `params`:
-//
-//     output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
-//
-// Whereas in @{tf.gather} `indices` defines slices into the first
-// dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
-// first `N` dimensions of `params`, where `N = indices.shape[-1]`.
-//
-// The last dimension of `indices` can be at most the rank of
-// `params`:
-//
-//     indices.shape[-1] <= params.rank
-//
-// The last dimension of `indices` corresponds to elements
-// (if `indices.shape[-1] == params.rank`) or slices
-// (if `indices.shape[-1] < params.rank`) along dimension `indices.shape[-1]`
-// of `params`.  The output tensor has shape
-//
-//     indices.shape[:-1] + params.shape[indices.shape[-1]:]
-//
-// Some examples below.
-//
-// Simple indexing into a matrix:
-//
-// ```python
-//     indices = [[0, 0], [1, 1]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = ['a', 'd']
-// ```
-//
-// Slice indexing into a matrix:
-//
-// ```python
-//     indices = [[1], [0]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [['c', 'd'], ['a', 'b']]
-// ```
-//
-// Indexing into a 3-tensor:
-//
-// ```python
-//     indices = [[1]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[['a1', 'b1'], ['c1', 'd1']]]
-//
-//
-//     indices = [[0, 1], [1, 0]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [['c0', 'd0'], ['a1', 'b1']]
-//
-//
-//     indices = [[0, 0, 1], [1, 0, 1]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = ['b0', 'b1']
-// ```
-//
-// Batched indexing into a matrix:
-//
-// ```python
-//     indices = [[[0, 0]], [[0, 1]]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [['a'], ['b']]
-// ```
-//
-// Batched slice indexing into a matrix:
-//
-// ```python
-//     indices = [[[1]], [[0]]]
-//     params = [['a', 'b'], ['c', 'd']]
-//     output = [[['c', 'd']], [['a', 'b']]]
-// ```
+// Compute the lower regularized incomplete Gamma function `Q(a, x)`.
 //
-// Batched indexing into a 3-tensor:
+// The lower regularized incomplete Gamma function is defined as:
 //
-// ```python
-//     indices = [[[1]], [[0]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[[['a1', 'b1'], ['c1', 'd1']]],
-//               [[['a0', 'b0'], ['c0', 'd0']]]]
 //
-//     indices = [[[0, 1], [1, 0]], [[0, 0], [1, 1]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [[['c0', 'd0'], ['a1', 'b1']],
-//               [['a0', 'b0'], ['c1', 'd1']]]
+// \\(P(a, x) = gamma(a, x) / Gamma(a) = 1 - Q(a, x)\\)
 //
+// where
 //
-//     indices = [[[0, 0, 1], [1, 0, 1]], [[0, 1, 1], [1, 1, 0]]]
-//     params = [[['a0', 'b0'], ['c0', 'd0']],
-//               [['a1', 'b1'], ['c1', 'd1']]]
-//     output = [['b0', 'b1'], ['d0', 'c1']]
-// ```
+// \\(gamma(a, x) = int_{0}^{x} t^{a-1} exp(-t) dt\\)
 //
-// Arguments:
-//	params: The tensor from which to gather values.
-//	indices: Index tensor.
+// is the lower incomplete Gamma function.
 //
-// Returns Values from `params` gathered from indices given by `indices`, with
-// shape `indices.shape[:-1] + params.shape[indices.shape[-1]:]`.
-func GatherNd(scope *Scope, params tf.Output, indices tf.Output) (output tf.Output) {
+// Note, above `Q(a, x)` (`Igammac`) is the upper regularized complete
+// Gamma function.
+func Igamma(scope *Scope, a tf.Output, x tf.Output) (z tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "GatherNd",
+		Type: "Igamma",
 		Input: []tf.Input{
-			params, indices,
+			a, x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Eagerly executes a python function to compute func(input)->output. The
+// Computes offsets of concat inputs within its output.
 //
-// semantics of the input, output, and attributes are the same as those for
-// PyFunc.
-func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataType) (output []tf.Output) {
+// For example:
+//
+// ```
+// # 'x' is [2, 2, 7]
+// # 'y' is [2, 3, 7]
+// # 'z' is [2, 5, 7]
+// concat_offset(2, [x, y, z]) => [0, 0, 0], [0, 2, 0], [0, 5, 0]
+// ```
+//
+// This is typically used by gradient computations for a concat operation.
+//
+// Arguments:
+//	concat_dim: The dimension along which to concatenate.
+//	shape: The `N` int32 vectors representing shape of tensors being concatenated.
+//
+// Returns The `N` int32 vectors representing the starting offset
+// of input tensors within the concatenated output.
+func ConcatOffset(scope *Scope, concat_dim tf.Output, shape []tf.Output) (offset []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"token": token, "Tout": Tout}
 	opspec := tf.OpSpec{
-		Type: "EagerPyFunc",
+		Type: "ConcatOffset",
 		Input: []tf.Input{
-			tf.OutputList(input),
+			concat_dim, tf.OutputList(shape),
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	if scope.Err() != nil {
@@ -27279,190 +27149,195 @@ func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataTy
 	}
 	var idx int
 	var err error
-	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
-		scope.UpdateErr("EagerPyFunc", err)
+	if offset, idx, err = makeOutputList(op, idx, "offset"); err != nil {
+		scope.UpdateErr("ConcatOffset", err)
 		return
 	}
-	return output
+	return offset
 }
 
-// Stops gradient computation.
-//
-// When executed in a graph, this op outputs its input tensor as-is.
-//
-// When building ops to compute gradients, this op prevents the contribution of
-// its inputs to be taken into account.  Normally, the gradient generator adds ops
-// to a graph to compute the derivatives of a specified 'loss' by recursively
-// finding out inputs that contributed to its computation.  If you insert this op
-// in the graph it inputs are masked from the gradient generator.  They are not
-// taken into account for computing gradients.
+// Splits a tensor into `num_split` tensors along one dimension.
 //
-// This is useful any time you want to compute a value with TensorFlow but need
-// to pretend that the value was a constant. Some examples include:
+// Arguments:
+//	axis: 0-D.  The dimension along which to split.  Must be in the range
+// `[-rank(value), rank(value))`.
+//	value: The tensor to split.
+//	num_split: The number of ways to split.  Must evenly divide
+// `value.shape[split_dim]`.
 //
-// *  The *EM* algorithm where the *M-step* should not involve backpropagation
-//    through the output of the *E-step*.
-// *  Contrastive divergence training of Boltzmann machines where, when
-//    differentiating the energy function, the training must not backpropagate
-//    through the graph that generated the samples from the model.
-// *  Adversarial training, where no backprop should happen through the adversarial
-//    example generation process.
-func StopGradient(scope *Scope, input tf.Output) (output tf.Output) {
+// Returns They are identically shaped tensors, whose shape matches that of `value`
+// except along `axis`, where their sizes are
+// `values.shape[split_dim] / num_split`.
+func Split(scope *Scope, axis tf.Output, value tf.Output, num_split int64) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "StopGradient",
+		Type: "Split",
 		Input: []tf.Input{
-			input,
+			axis, value,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// Computes asin of x element-wise.
-func Asin(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	opspec := tf.OpSpec{
-		Type: "Asin",
-		Input: []tf.Input{
-			x,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
-// PreventGradientAttr is an optional argument to PreventGradient.
-type PreventGradientAttr func(optionalAttr)
-
-// PreventGradientMessage sets the optional message attribute to value.
-//
-// value: Will be printed in the error when anyone tries to differentiate
-// this operation.
-// If not specified, defaults to ""
-func PreventGradientMessage(value string) PreventGradientAttr {
-	return func(m optionalAttr) {
-		m["message"] = value
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("Split", err)
+		return
 	}
+	return output
 }
 
-// An identity op that triggers an error if a gradient is requested.
-//
-// When executed in a graph, this op outputs its input tensor as-is.
-//
-// When building ops to compute gradients, the TensorFlow gradient system
-// will return an error when trying to lookup the gradient of this op,
-// because no gradient must ever be registered for this function.  This
-// op exists to prevent subtle bugs from silently returning unimplemented
-// gradients in some corner cases.
+// Splits a tensor into `num_split` tensors along one dimension.
 //
 // Arguments:
-//	input: any tensor.
+//	value: The tensor to split.
+//	size_splits: list containing the sizes of each output tensor along the split
+// dimension. Must sum to the dimension of value along split_dim.
+// Can contain one -1 indicating that dimension is to be inferred.
+//	axis: 0-D.  The dimension along which to split.  Must be in the range
+// `[-rank(value), rank(value))`.
 //
-// Returns the same input tensor.
-func PreventGradient(scope *Scope, input tf.Output, optional ...PreventGradientAttr) (output tf.Output) {
+//
+// Returns Tensors whose shape matches that of `value`
+// except along `axis`, where their sizes are
+// `size_splits[i]`.
+func SplitV(scope *Scope, value tf.Output, size_splits tf.Output, axis tf.Output, num_split int64) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"num_split": num_split}
 	opspec := tf.OpSpec{
-		Type: "PreventGradient",
+		Type: "SplitV",
 		Input: []tf.Input{
-			input,
+			value, size_splits, axis,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	if scope.Err() != nil {
+		return
+	}
+	var idx int
+	var err error
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("SplitV", err)
+		return
+	}
+	return output
 }
 
-// Checks a tensor for NaN and Inf values.
+// Gives a guarantee to the TF runtime that the input tensor is a constant.
 //
-// When run, reports an `InvalidArgument` error if `tensor` has any values
-// that are not a number (NaN) or infinity (Inf). Otherwise, passes `tensor` as-is.
+// The runtime is then free to make optimizations based on this.
 //
-// Arguments:
+// Only accepts value typed tensors as inputs and rejects resource variable handles
+// as input.
 //
-//	message: Prefix of the error message.
-func CheckNumerics(scope *Scope, tensor tf.Output, message string) (output tf.Output) {
+// Returns the input tensor without modification.
+func GuaranteeConst(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"message": message}
 	opspec := tf.OpSpec{
-		Type: "CheckNumerics",
+		Type: "GuaranteeConst",
 		Input: []tf.Input{
-			tensor,
+			input,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Shuffle dimensions of x according to a permutation and conjugate the result.
+// Returns a tensor of zeros with the same shape and type as x.
 //
-// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
-//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
-//   `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
-func ConjugateTranspose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
+// Arguments:
+//	x: a tensor of type T.
+//
+// Returns a tensor of the same shape and type as x but filled with zeros.
+func ZerosLike(scope *Scope, x tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "ConjugateTranspose",
+		Type: "ZerosLike",
 		Input: []tf.Input{
-			x, perm,
+			x,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// UniqueV2Attr is an optional argument to UniqueV2.
-type UniqueV2Attr func(optionalAttr)
+// QuantizedInstanceNormAttr is an optional argument to QuantizedInstanceNorm.
+type QuantizedInstanceNormAttr func(optionalAttr)
 
-// UniqueV2OutIdx sets the optional out_idx attribute to value.
-// If not specified, defaults to DT_INT32
-func UniqueV2OutIdx(value tf.DataType) UniqueV2Attr {
+// QuantizedInstanceNormOutputRangeGiven sets the optional output_range_given attribute to value.
+//
+// value: If True, `given_y_min` and `given_y_min`
+// and `given_y_max` are used as the output range. Otherwise,
+// the implementation computes the output range.
+// If not specified, defaults to false
+func QuantizedInstanceNormOutputRangeGiven(value bool) QuantizedInstanceNormAttr {
 	return func(m optionalAttr) {
-		m["out_idx"] = value
+		m["output_range_given"] = value
 	}
 }
 
-// Finds unique elements in a 1-D tensor.
+// QuantizedInstanceNormGivenYMin sets the optional given_y_min attribute to value.
 //
-// This operation returns a tensor `y` containing all of the unique elements of `x`
-// sorted in the same order that they occur in `x`. This operation also returns a
-// tensor `idx` the same size as `x` that contains the index of each value of `x`
-// in the unique output `y`. In other words:
+// value: Output in `y_min` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMin(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["given_y_min"] = value
+	}
+}
+
+// QuantizedInstanceNormGivenYMax sets the optional given_y_max attribute to value.
 //
-// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+// value: Output in `y_max` if `output_range_given` is True.
+// If not specified, defaults to 0
+func QuantizedInstanceNormGivenYMax(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["given_y_max"] = value
+	}
+}
+
+// QuantizedInstanceNormVarianceEpsilon sets the optional variance_epsilon attribute to value.
 //
-// For example:
+// value: A small float number to avoid dividing by 0.
+// If not specified, defaults to 1e-05
+func QuantizedInstanceNormVarianceEpsilon(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["variance_epsilon"] = value
+	}
+}
+
+// QuantizedInstanceNormMinSeparation sets the optional min_separation attribute to value.
 //
-// ```
-// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
-// y, idx = unique(x)
-// y ==> [1, 2, 4, 7, 8]
-// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
-// ```
+// value: Minimum value of `y_max - y_min`
+// If not specified, defaults to 0.001
+func QuantizedInstanceNormMinSeparation(value float32) QuantizedInstanceNormAttr {
+	return func(m optionalAttr) {
+		m["min_separation"] = value
+	}
+}
+
+// Quantized Instance normalization.
 //
 // Arguments:
-//	x: A `Tensor`.
-//	axis: A `Tensor` of type `int64` (default: 0). The axis of the Tensor to
-// find the unique elements.
+//	x: A 4D input Tensor.
+//	x_min: The value represented by the lowest quantized input.
+//	x_max: The value represented by the highest quantized input.
 //
-// Returns A `Tensor`. Unique elements along the `axis` of `Tensor` x.A 1-D Tensor. Has the same type as x that contains the index of each
-// value of x in the output y.
-func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Attr) (y tf.Output, idx tf.Output) {
+// Returns A 4D Tensor.The value represented by the lowest quantized output.The value represented by the highest quantized output.
+func QuantizedInstanceNorm(scope *Scope, x tf.Output, x_min tf.Output, x_max tf.Output, optional ...QuantizedInstanceNormAttr) (y tf.Output, y_min tf.Output, y_max tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27471,187 +27346,185 @@ func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Att
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "UniqueV2",
+		Type: "QuantizedInstanceNorm",
 		Input: []tf.Input{
-			x, axis,
+			x, x_min, x_max,
 		},
 		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0), op.Output(1)
+	return op.Output(0), op.Output(1), op.Output(2)
 }
 
-// Return a slice from 'input'.
+// Returns the diagonal part of the tensor.
 //
-// The output tensor is a tensor with dimensions described by 'size'
-// whose values are extracted from 'input' starting at the offsets in
-// 'begin'.
+// This operation returns a tensor with the `diagonal` part
+// of the `input`. The `diagonal` part is computed as follows:
 //
-// *Requirements*:
-//   0 <= begin[i] <= begin[i] + size[i] <= Di  for i in [0, n)
+// Assume `input` has dimensions `[D1,..., Dk, D1,..., Dk]`, then the output is a
+// tensor of rank `k` with dimensions `[D1,..., Dk]` where:
+//
+// `diagonal[i1,..., ik] = input[i1, ..., ik, i1,..., ik]`.
+//
+// For example:
+//
+// ```
+// # 'input' is [[1, 0, 0, 0]
+//               [0, 2, 0, 0]
+//               [0, 0, 3, 0]
+//               [0, 0, 0, 4]]
+//
+// tf.diag_part(input) ==> [1, 2, 3, 4]
+// ```
 //
 // Arguments:
+//	input: Rank k tensor where k is even and not zero.
 //
-//	begin: begin[i] specifies the offset into the 'i'th dimension of
-// 'input' to slice from.
-//	size: size[i] specifies the number of elements of the 'i'th dimension
-// of 'input' to slice. If size[i] is -1, all remaining elements in dimension
-// i are included in the slice (i.e. this is equivalent to setting
-// size[i] = input.dim_size(i) - begin[i]).
-func Slice(scope *Scope, input tf.Output, begin tf.Output, size tf.Output) (output tf.Output) {
+// Returns The extracted diagonal.
+func DiagPart(scope *Scope, input tf.Output) (diagonal tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Slice",
+		Type: "DiagPart",
 		Input: []tf.Input{
-			input, begin, size,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// StridedSliceGradAttr is an optional argument to StridedSliceGrad.
-type StridedSliceGradAttr func(optionalAttr)
-
-// StridedSliceGradBeginMask sets the optional begin_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradBeginMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["begin_mask"] = value
-	}
-}
-
-// StridedSliceGradEndMask sets the optional end_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradEndMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["end_mask"] = value
-	}
-}
-
-// StridedSliceGradEllipsisMask sets the optional ellipsis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradEllipsisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["ellipsis_mask"] = value
-	}
-}
-
-// StridedSliceGradNewAxisMask sets the optional new_axis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradNewAxisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["new_axis_mask"] = value
-	}
-}
-
-// StridedSliceGradShrinkAxisMask sets the optional shrink_axis_mask attribute to value.
-// If not specified, defaults to 0
-func StridedSliceGradShrinkAxisMask(value int64) StridedSliceGradAttr {
-	return func(m optionalAttr) {
-		m["shrink_axis_mask"] = value
-	}
-}
-
-// Returns the gradient of `StridedSlice`.
+// Returns the element-wise max of two SparseTensors.
 //
-// Since `StridedSlice` cuts out pieces of its `input` which is size
-// `shape`, its gradient will have the same shape (which is passed here
-// as `shape`). The gradient will be zero in any element that the slice
-// does not select.
+// Assumes the two SparseTensors have the same shape, i.e., no broadcasting.
 //
-// Arguments are the same as StridedSliceGrad with the exception that
-// `dy` is the input gradient to be propagated and `shape` is the
-// shape of `StridedSlice`'s `input`.
-func StridedSliceGrad(scope *Scope, shape tf.Output, begin tf.Output, end tf.Output, strides tf.Output, dy tf.Output, optional ...StridedSliceGradAttr) (output tf.Output) {
+// Arguments:
+//	a_indices: 2-D.  `N x R` matrix with the indices of non-empty values in a
+// SparseTensor, in the canonical lexicographic ordering.
+//	a_values: 1-D.  `N` non-empty values corresponding to `a_indices`.
+//	a_shape: 1-D.  Shape of the input SparseTensor.
+//	b_indices: counterpart to `a_indices` for the other operand.
+//	b_values: counterpart to `a_values` for the other operand; must be of the same dtype.
+//	b_shape: counterpart to `a_shape` for the other operand; the two shapes must be equal.
+//
+// Returns 2-D.  The indices of the output SparseTensor.1-D.  The values of the output SparseTensor.
+func SparseSparseMaximum(scope *Scope, a_indices tf.Output, a_values tf.Output, a_shape tf.Output, b_indices tf.Output, b_values tf.Output, b_shape tf.Output) (output_indices tf.Output, output_values tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{}
-	for _, a := range optional {
-		a(attrs)
-	}
 	opspec := tf.OpSpec{
-		Type: "StridedSliceGrad",
+		Type: "SparseSparseMaximum",
 		Input: []tf.Input{
-			shape, begin, end, strides, dy,
+			a_indices, a_values, a_shape, b_indices, b_values, b_shape,
 		},
-		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
-	return op.Output(0)
+	return op.Output(0), op.Output(1)
 }
 
-// Returns the gradient of `Tile`.
+// Returns a batched matrix tensor with new batched diagonal values.
+//
+// Given `input` and `diagonal`, this operation returns a tensor with the
+// same shape and values as `input`, except for the main diagonal of the
+// innermost matrices.  These will be overwritten by the values in `diagonal`.
 //
-// DEPRECATED at GraphDef version 3: TileGrad has been replaced with reduce_sum
+// The output is computed as follows:
 //
-// Since `Tile` takes an input and repeats the input `multiples` times
-// along each dimension, `TileGrad` takes in `multiples` and aggregates
-// each repeated tile of `input` into `output`.
-func TileGrad(scope *Scope, input tf.Output, multiples tf.Output) (output tf.Output) {
+// Assume `input` has `k+1` dimensions `[I, J, K, ..., M, N]` and `diagonal` has
+// `k` dimensions `[I, J, K, ..., min(M, N)]`.  Then the output is a
+// tensor of rank `k+1` with dimensions `[I, J, K, ..., M, N]` where:
+//
+//   * `output[i, j, k, ..., m, n] = diagonal[i, j, k, ..., n]` for `m == n`.
+//   * `output[i, j, k, ..., m, n] = input[i, j, k, ..., m, n]` for `m != n`.
+//
+// Arguments:
+//	input: Rank `k+1`, where `k >= 1`.
+//	diagonal: Rank `k`, where `k >= 1`.
+//
+// Returns Rank `k+1`, with `output.shape = input.shape`.
+func MatrixSetDiag(scope *Scope, input tf.Output, diagonal tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "TileGrad",
+		Type: "MatrixSetDiag",
 		Input: []tf.Input{
-			input, multiples,
+			input, diagonal,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// QuantizeAndDequantizeAttr is an optional argument to QuantizeAndDequantize.
-type QuantizeAndDequantizeAttr func(optionalAttr)
+// EditDistanceAttr is an optional argument to EditDistance.
+type EditDistanceAttr func(optionalAttr)
 
-// QuantizeAndDequantizeSignedInput sets the optional signed_input attribute to value.
+// EditDistanceNormalize sets the optional normalize attribute to value.
+//
+// value: boolean (if true, edit distances are normalized by length of truth).
+//
+// The output is:
 // If not specified, defaults to true
-func QuantizeAndDequantizeSignedInput(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["signed_input"] = value
-	}
-}
-
-// QuantizeAndDequantizeNumBits sets the optional num_bits attribute to value.
-// If not specified, defaults to 8
-func QuantizeAndDequantizeNumBits(value int64) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["num_bits"] = value
-	}
-}
-
-// QuantizeAndDequantizeRangeGiven sets the optional range_given attribute to value.
-// If not specified, defaults to false
-func QuantizeAndDequantizeRangeGiven(value bool) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["range_given"] = value
-	}
-}
-
-// QuantizeAndDequantizeInputMin sets the optional input_min attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMin(value float32) QuantizeAndDequantizeAttr {
-	return func(m optionalAttr) {
-		m["input_min"] = value
-	}
-}
-
-// QuantizeAndDequantizeInputMax sets the optional input_max attribute to value.
-// If not specified, defaults to 0
-func QuantizeAndDequantizeInputMax(value float32) QuantizeAndDequantizeAttr {
+func EditDistanceNormalize(value bool) EditDistanceAttr {
 	return func(m optionalAttr) {
-		m["input_max"] = value
+		m["normalize"] = value
 	}
 }
 
-// Use QuantizeAndDequantizeV2 instead.
+// Computes the (possibly normalized) Levenshtein Edit Distance.
 //
-// DEPRECATED at GraphDef version 22: Replaced by QuantizeAndDequantizeV2
-func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAndDequantizeAttr) (output tf.Output) {
+// The inputs are variable-length sequences provided by SparseTensors
+//   (hypothesis_indices, hypothesis_values, hypothesis_shape)
+// and
+//   (truth_indices, truth_values, truth_shape).
+//
+// The inputs are:
+//
+// Arguments:
+//	hypothesis_indices: The indices of the hypothesis list SparseTensor.
+// This is an N x R int64 matrix.
+//	hypothesis_values: The values of the hypothesis list SparseTensor.
+// This is an N-length vector.
+//	hypothesis_shape: The shape of the hypothesis list SparseTensor.
+// This is an R-length vector.
+//	truth_indices: The indices of the truth list SparseTensor.
+// This is an M x R int64 matrix.
+//	truth_values: The values of the truth list SparseTensor.
+// This is an M-length vector.
+//	truth_shape: truth indices, vector.
+//
+// Returns A dense float tensor with rank R - 1.
+//
+// For the example input:
+//
+//     // hypothesis represents a 2x1 matrix with variable-length values:
+//     //   (0,0) = ["a"]
+//     //   (1,0) = ["b"]
+//     hypothesis_indices = [[0, 0, 0],
+//                           [1, 0, 0]]
+//     hypothesis_values = ["a", "b"]
+//     hypothesis_shape = [2, 1, 1]
+//
+//     // truth represents a 2x2 matrix with variable-length values:
+//     //   (0,0) = []
+//     //   (0,1) = ["a"]
+//     //   (1,0) = ["b", "c"]
+//     //   (1,1) = ["a"]
+//     truth_indices = [[0, 1, 0],
+//                      [1, 0, 0],
+//                      [1, 0, 1],
+//                      [1, 1, 0]]
+//     truth_values = ["a", "b", "c", "a"]
+//     truth_shape = [2, 2, 2]
+//     normalize = true
+//
+// The output will be:
+//
+//     // output is a 2x2 matrix with edit distances normalized by truth lengths.
+//     output = [[inf, 1.0],  // (0,0): no truth, (0,1): no hypothesis
+//               [0.5, 1.0]]  // (1,0): addition, (1,1): no hypothesis
+func EditDistance(scope *Scope, hypothesis_indices tf.Output, hypothesis_values tf.Output, hypothesis_shape tf.Output, truth_indices tf.Output, truth_values tf.Output, truth_shape tf.Output, optional ...EditDistanceAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27660,9 +27533,9 @@ func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAn
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "QuantizeAndDequantize",
+		Type: "EditDistance",
 		Input: []tf.Input{
-			input,
+			hypothesis_indices, hypothesis_values, hypothesis_shape, truth_indices, truth_values, truth_shape,
 		},
 		Attrs: attrs,
 	}
@@ -27670,47 +27543,140 @@ func QuantizeAndDequantize(scope *Scope, input tf.Output, optional ...QuantizeAn
 	return op.Output(0)
 }
 
-// QueueDequeueV2Attr is an optional argument to QueueDequeueV2.
-type QueueDequeueV2Attr func(optionalAttr)
-
-// QueueDequeueV2TimeoutMs sets the optional timeout_ms attribute to value.
+// Gather slices from `params` into a Tensor with shape specified by `indices`.
 //
-// value: If the queue is empty, this operation will block for up to
-// timeout_ms milliseconds.
-// Note: This option is not supported yet.
-// If not specified, defaults to -1
-func QueueDequeueV2TimeoutMs(value int64) QueueDequeueV2Attr {
-	return func(m optionalAttr) {
-		m["timeout_ms"] = value
+// `indices` is an K-dimensional integer tensor, best thought of as a
+// (K-1)-dimensional tensor of indices into `params`, where each element defines a
+// slice of `params`:
+//
+//     output[i_0, ..., i_{K-2}] = params[indices[i0, ..., i_{K-2}]]
+//
+// Whereas in @{tf.gather} `indices` defines slices into the first
+// dimension of `params`, in `tf.gather_nd`, `indices` defines slices into the
+// first `N` dimensions of `params`, where `N = indices.shape[-1]`.
+//
+// The last dimension of `indices` can be at most the rank of
+// `params`:
+//
+//     indices.shape[-1] <= params.rank
+//
+// The last dimension of `indices` corresponds to elements
+// (if `indices.shape[-1] == params.rank`) or slices
+// (if `indices.shape[-1] < params.rank`) along dimension `indices.shape[-1]`
+// of `params`.  The output tensor has shape
+//
+//     indices.shape[:-1] + params.shape[indices.shape[-1]:]
+//
+// Some examples below.
+//
+// Simple indexing into a matrix:
+//
+// ```python
+//     indices = [[0, 0], [1, 1]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = ['a', 'd']
+// ```
+//
+// Slice indexing into a matrix:
+//
+// ```python
+//     indices = [[1], [0]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [['c', 'd'], ['a', 'b']]
+// ```
+//
+// Indexing into a 3-tensor:
+//
+// ```python
+//     indices = [[1]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[['a1', 'b1'], ['c1', 'd1']]]
+//
+//
+//     indices = [[0, 1], [1, 0]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [['c0', 'd0'], ['a1', 'b1']]
+//
+//
+//     indices = [[0, 0, 1], [1, 0, 1]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = ['b0', 'b1']
+// ```
+//
+// Batched indexing into a matrix:
+//
+// ```python
+//     indices = [[[0, 0]], [[0, 1]]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [['a'], ['b']]
+// ```
+//
+// Batched slice indexing into a matrix:
+//
+// ```python
+//     indices = [[[1]], [[0]]]
+//     params = [['a', 'b'], ['c', 'd']]
+//     output = [[['c', 'd']], [['a', 'b']]]
+// ```
+//
+// Batched indexing into a 3-tensor:
+//
+// ```python
+//     indices = [[[1]], [[0]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[[['a1', 'b1'], ['c1', 'd1']]],
+//               [[['a0', 'b0'], ['c0', 'd0']]]]
+//
+//     indices = [[[0, 1], [1, 0]], [[0, 0], [1, 1]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [[['c0', 'd0'], ['a1', 'b1']],
+//               [['a0', 'b0'], ['c1', 'd1']]]
+//
+//
+//     indices = [[[0, 0, 1], [1, 0, 1]], [[0, 1, 1], [1, 1, 0]]]
+//     params = [[['a0', 'b0'], ['c0', 'd0']],
+//               [['a1', 'b1'], ['c1', 'd1']]]
+//     output = [['b0', 'b1'], ['d0', 'c1']]
+// ```
+//
+// Arguments:
+//	params: The tensor from which to gather values.
+//	indices: Index tensor.
+//
+// Returns Values from `params` gathered from indices given by `indices`, with
+// shape `indices.shape[:-1] + params.shape[indices.shape[-1]:]`.
+func GatherNd(scope *Scope, params tf.Output, indices tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "GatherNd",
+		Input: []tf.Input{
+			params, indices,
+		},
 	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// Dequeues a tuple of one or more tensors from the given queue.
-//
-// This operation has k outputs, where k is the number of components
-// in the tuples stored in the given queue, and output i is the ith
-// component of the dequeued tuple.
-//
-// N.B. If the queue is empty, this operation will block until an element
-// has been dequeued (or 'timeout_ms' elapses, if specified).
-//
-// Arguments:
-//	handle: The handle to a queue.
-//	component_types: The type of each component in a tuple.
+// Eagerly executes a python function to compute func(input)->output. The
 //
-// Returns One or more tensors that were dequeued as a tuple.
-func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataType, optional ...QueueDequeueV2Attr) (components []tf.Output) {
+// semantics of the input, output, and attributes are the same as those for
+// PyFunc.
+func EagerPyFunc(scope *Scope, input []tf.Output, token string, Tout []tf.DataType) (output []tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
-	attrs := map[string]interface{}{"component_types": component_types}
-	for _, a := range optional {
-		a(attrs)
-	}
+	attrs := map[string]interface{}{"token": token, "Tout": Tout}
 	opspec := tf.OpSpec{
-		Type: "QueueDequeueV2",
+		Type: "EagerPyFunc",
 		Input: []tf.Input{
-			handle,
+			tf.OutputList(input),
 		},
 		Attrs: attrs,
 	}
@@ -27720,121 +27686,92 @@ func QueueDequeueV2(scope *Scope, handle tf.Output, component_types []tf.DataTyp
 	}
 	var idx int
 	var err error
-	if components, idx, err = makeOutputList(op, idx, "components"); err != nil {
-		scope.UpdateErr("QueueDequeueV2", err)
+	if output, idx, err = makeOutputList(op, idx, "output"); err != nil {
+		scope.UpdateErr("EagerPyFunc", err)
 		return
 	}
-	return components
+	return output
 }
 
-// Returns locations of nonzero / true values in a tensor.
-//
-// This operation returns the coordinates of true elements in `condition`. The
-// coordinates are returned in a 2-D tensor where the first dimension (rows)
-// represents the number of true elements, and the second dimension (columns)
-// represents the coordinates of the true elements. Keep in mind, the shape of
-// the output tensor can vary depending on how many true values there are in
-// `condition`. Indices are output in row-major order.
-//
-// For example:
+// Stops gradient computation.
 //
-// ```
-// # 'input' tensor is [[True, False]
-// #                    [True, False]]
-// # 'input' has two true values, so output has two coordinates.
-// # 'input' has rank of 2, so coordinates have two indices.
-// where(input) ==> [[0, 0],
-//                   [1, 0]]
+// When executed in a graph, this op outputs its input tensor as-is.
 //
-// # `condition` tensor is [[[True, False]
-// #                     [True, False]]
-// #                    [[False, True]
-// #                     [False, True]]
-// #                    [[False, False]
-// #                     [False, True]]]
-// # 'input' has 5 true values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
+// When building ops to compute gradients, this op prevents the contribution of
+// its inputs to be taken into account.  Normally, the gradient generator adds ops
+// to a graph to compute the derivatives of a specified 'loss' by recursively
+// finding out inputs that contributed to its computation.  If you insert this op
+// in the graph it inputs are masked from the gradient generator.  They are not
+// taken into account for computing gradients.
 //
-// # `condition` tensor is [[[1.5,  0.0]
-// #                     [-0.5, 0.0]]
-// #                    [[0.0,  0.25]
-// #                     [0.0,  0.75]]
-// #                    [[0.0,  0.0]
-// #                     [0.0,  0.01]]]
-// # 'input' has 5 nonzero values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
+// This is useful any time you want to compute a value with TensorFlow but need
+// to pretend that the value was a constant. Some examples include:
 //
-// # `condition` tensor is [[[1.5 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.5j, 0.0  + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.25 + 1.5j]
-// #                     [0.0 + 0.0j, 0.75 + 0.0j]]
-// #                    [[0.0 + 0.0j, 0.0  + 0.0j]
-// #                     [0.0 + 0.0j, 0.01 + 0.0j]]]
-// # 'input' has 5 nonzero magnitude values, so output has 5 coordinates.
-// # 'input' has rank of 3, so coordinates have three indices.
-// where(input) ==> [[0, 0, 0],
-//                   [0, 1, 0],
-//                   [1, 0, 1],
-//                   [1, 1, 1],
-//                   [2, 1, 1]]
-// ```
-func Where(scope *Scope, condition tf.Output) (index tf.Output) {
+// *  The *EM* algorithm where the *M-step* should not involve backpropagation
+//    through the output of the *E-step*.
+// *  Contrastive divergence training of Boltzmann machines where, when
+//    differentiating the energy function, the training must not backpropagate
+//    through the graph that generated the samples from the model.
+// *  Adversarial training, where no backprop should happen through the adversarial
+//    example generation process.
+func StopGradient(scope *Scope, input tf.Output) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "Where",
+		Type: "StopGradient",
 		Input: []tf.Input{
-			condition,
+			input,
 		},
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// DataFormatDimMapAttr is an optional argument to DataFormatDimMap.
-type DataFormatDimMapAttr func(optionalAttr)
-
-// DataFormatDimMapSrcFormat sets the optional src_format attribute to value.
-//
-// value: source data format.
-// If not specified, defaults to "NHWC"
-func DataFormatDimMapSrcFormat(value string) DataFormatDimMapAttr {
-	return func(m optionalAttr) {
-		m["src_format"] = value
+// Computes asin of x element-wise.
+func Asin(scope *Scope, x tf.Output) (y tf.Output) {
+	if scope.Err() != nil {
+		return
 	}
+	opspec := tf.OpSpec{
+		Type: "Asin",
+		Input: []tf.Input{
+			x,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
 }
 
-// DataFormatDimMapDstFormat sets the optional dst_format attribute to value.
+// PreventGradientAttr is an optional argument to PreventGradient.
+type PreventGradientAttr func(optionalAttr)
+
+// PreventGradientMessage sets the optional message attribute to value.
 //
-// value: destination data format.
-// If not specified, defaults to "NCHW"
-func DataFormatDimMapDstFormat(value string) DataFormatDimMapAttr {
+// value: Will be printed in the error when anyone tries to differentiate
+// this operation.
+// If not specified, defaults to ""
+func PreventGradientMessage(value string) PreventGradientAttr {
 	return func(m optionalAttr) {
-		m["dst_format"] = value
+		m["message"] = value
 	}
 }
 
-// Returns the dimension index in the destination data format given the one in
+// An identity op that triggers an error if a gradient is requested.
 //
-// the source data format.
+// When executed in a graph, this op outputs its input tensor as-is.
+//
+// When building ops to compute gradients, the TensorFlow gradient system
+// will return an error when trying to lookup the gradient of this op,
+// because no gradient must ever be registered for this function.  This
+// op exists to prevent subtle bugs from silently returning unimplemented
+// gradients in some corner cases.
 //
 // Arguments:
-//	x: A Tensor with each element as a dimension index in source data format.
-// Must be in the range [-4, 4).
+//	input: any tensor.
 //
-// Returns A Tensor with each element as a dimension index in destination data format.
-func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAttr) (y tf.Output) {
+// Returns the same input tensor.
+func PreventGradient(scope *Scope, input tf.Output, optional ...PreventGradientAttr) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
@@ -27843,9 +27780,9 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt
 		a(attrs)
 	}
 	opspec := tf.OpSpec{
-		Type: "DataFormatDimMap",
+		Type: "PreventGradient",
 		Input: []tf.Input{
-			x,
+			input,
 		},
 		Attrs: attrs,
 	}
@@ -27853,36 +27790,99 @@ func DataFormatDimMap(scope *Scope, x tf.Output, optional ...DataFormatDimMapAtt
 	return op.Output(0)
 }
 
-// Return the shape of s0 op s1 with broadcast.
+// Checks a tensor for NaN and Inf values.
 //
-// Given `s0` and `s1`, tensors that represent shapes, compute `r0`, the
-// broadcasted shape. `s0`, `s1` and `r0` are all integer vectors.
-func BroadcastArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output) {
+// When run, reports an `InvalidArgument` error if `tensor` has any values
+// that are not a number (NaN) or infinity (Inf). Otherwise, passes `tensor` as-is.
+//
+// Arguments:
+//
+//	message: Prefix of the error message.
+func CheckNumerics(scope *Scope, tensor tf.Output, message string) (output tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
+	attrs := map[string]interface{}{"message": message}
 	opspec := tf.OpSpec{
-		Type: "BroadcastArgs",
+		Type: "CheckNumerics",
 		Input: []tf.Input{
-			s0, s1,
+			tensor,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0)
 }
 
-// Return the reduction indices for computing gradients of s0 op s1 with broadcast.
+// Shuffle dimensions of x according to a permutation and conjugate the result.
 //
-// This is typically used by gradient computations for a broadcasting operation.
-func BroadcastGradientArgs(scope *Scope, s0 tf.Output, s1 tf.Output) (r0 tf.Output, r1 tf.Output) {
+// The output `y` has the same rank as `x`. The shapes of `x` and `y` satisfy:
+//   `y.shape[i] == x.shape[perm[i]] for i in [0, 1, ..., rank(x) - 1]`
+//   `y[i,j,k,...,s,t,u] == conj(x[perm[i], perm[j], perm[k],...,perm[s], perm[t], perm[u]])`
+func ConjugateTranspose(scope *Scope, x tf.Output, perm tf.Output) (y tf.Output) {
 	if scope.Err() != nil {
 		return
 	}
 	opspec := tf.OpSpec{
-		Type: "BroadcastGradientArgs",
+		Type: "ConjugateTranspose",
 		Input: []tf.Input{
-			s0, s1,
+			x, perm,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// UniqueV2Attr is an optional argument to UniqueV2.
+type UniqueV2Attr func(optionalAttr)
+
+// UniqueV2OutIdx sets the optional out_idx attribute to value.
+// If not specified, defaults to DT_INT32
+func UniqueV2OutIdx(value tf.DataType) UniqueV2Attr {
+	return func(m optionalAttr) {
+		m["out_idx"] = value
+	}
+}
+
+// Finds unique elements in a 1-D tensor.
+//
+// This operation returns a tensor `y` containing all of the unique elements of `x`
+// sorted in the same order that they occur in `x`. This operation also returns a
+// tensor `idx` the same size as `x` that contains the index of each value of `x`
+// in the unique output `y`. In other words:
+//
+// `y[idx[i]] = x[i] for i in [0, 1,...,rank(x) - 1]`
+//
+// For example:
+//
+// ```
+// # tensor 'x' is [1, 1, 2, 4, 4, 4, 7, 8, 8]
+// y, idx = unique(x)
+// y ==> [1, 2, 4, 7, 8]
+// idx ==> [0, 0, 1, 2, 2, 2, 3, 4, 4]
+// ```
+//
+// Arguments:
+//	x: A `Tensor`.
+//	axis: A `Tensor` of type `int64` (default: 0). The axis of the Tensor to
+// find the unique elements.
+//
+// Returns A `Tensor`. Unique elements along the `axis` of `Tensor` x.A 1-D Tensor. Has the same type as x that contains the index of each
+// value of x in the output y.
+func UniqueV2(scope *Scope, x tf.Output, axis tf.Output, optional ...UniqueV2Attr) (y tf.Output, idx tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "UniqueV2",
+		Input: []tf.Input{
+			x, axis,
 		},
+		Attrs: attrs,
 	}
 	op := scope.AddOperation(opspec)
 	return op.Output(0), op.Output(1)
-- 
GitLab


From 950b4e59f2dbd1836a17f3a3f83e4b829c10db77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 10:06:52 -0700
Subject: [PATCH 676/960] Automated g4 rollback of changelist 190728742

PiperOrigin-RevId: 190946066
---
 .../python/kernel_tests/distribution_test.py     | 16 ----------------
 .../distributions/python/ops/autoregressive.py   |  2 +-
 .../contrib/distributions/python/ops/binomial.py |  2 +-
 .../contrib/distributions/python/ops/cauchy.py   |  2 +-
 .../contrib/distributions/python/ops/chi2.py     |  4 ++--
 .../distributions/python/ops/deterministic.py    |  2 +-
 .../distributions/python/ops/geometric.py        |  2 +-
 .../contrib/distributions/python/ops/gumbel.py   |  2 +-
 .../distributions/python/ops/half_normal.py      |  2 +-
 .../distributions/python/ops/independent.py      |  2 +-
 .../distributions/python/ops/inverse_gamma.py    |  4 ++--
 .../distributions/python/ops/kumaraswamy.py      |  9 ++++-----
 .../contrib/distributions/python/ops/logistic.py |  2 +-
 .../contrib/distributions/python/ops/mixture.py  |  2 +-
 .../python/ops/mixture_same_family.py            |  2 +-
 .../contrib/distributions/python/ops/mvn_diag.py |  4 ++--
 .../python/ops/mvn_diag_plus_low_rank.py         |  2 +-
 .../python/ops/mvn_full_covariance.py            |  2 +-
 .../python/ops/mvn_linear_operator.py            |  2 +-
 .../contrib/distributions/python/ops/mvn_tril.py |  2 +-
 .../python/ops/negative_binomial.py              |  2 +-
 .../python/ops/onehot_categorical.py             |  2 +-
 .../contrib/distributions/python/ops/poisson.py  |  2 +-
 .../python/ops/poisson_lognormal.py              |  2 +-
 .../python/ops/quantized_distribution.py         |  2 +-
 .../python/ops/relaxed_bernoulli.py              |  2 +-
 .../python/ops/relaxed_onehot_categorical.py     |  2 +-
 .../distributions/python/ops/sinh_arcsinh.py     |  3 +--
 .../python/ops/vector_diffeomixture.py           |  2 +-
 .../python/ops/vector_exponential_diag.py        |  2 +-
 .../ops/vector_exponential_linear_operator.py    |  2 +-
 .../python/ops/vector_sinh_arcsinh_diag.py       |  2 +-
 .../distributions/python/ops/vector_student_t.py |  2 +-
 .../contrib/distributions/python/ops/wishart.py  | 10 +++++-----
 tensorflow/python/ops/distributions/bernoulli.py |  2 +-
 tensorflow/python/ops/distributions/beta.py      |  6 +++---
 .../python/ops/distributions/categorical.py      |  2 +-
 tensorflow/python/ops/distributions/dirichlet.py |  2 +-
 .../ops/distributions/dirichlet_multinomial.py   |  2 +-
 .../python/ops/distributions/distribution.py     | 10 ++--------
 .../python/ops/distributions/exponential.py      |  4 ++--
 tensorflow/python/ops/distributions/gamma.py     |  4 ++--
 tensorflow/python/ops/distributions/laplace.py   |  4 ++--
 .../python/ops/distributions/multinomial.py      |  2 +-
 tensorflow/python/ops/distributions/normal.py    |  4 ++--
 tensorflow/python/ops/distributions/student_t.py |  4 ++--
 .../distributions/transformed_distribution.py    |  2 +-
 tensorflow/python/ops/distributions/uniform.py   |  2 +-
 48 files changed, 65 insertions(+), 89 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
index 750fe00d63..68e0d9cb82 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/distribution_test.py
@@ -190,22 +190,6 @@ class DistributionTest(test.TestCase):
       y = dist._set_sample_static_shape(x, sample_shape)
       self.assertTrue(y.get_shape().ndims is None)
 
-  def testNameScopeWorksCorrectly(self):
-    x = tfd.Normal(loc=0., scale=1., name="x")
-    x_duplicate = tfd.Normal(loc=0., scale=1., name="x")
-    with ops.name_scope("y") as name:
-      y = tfd.Bernoulli(logits=0., name=name)
-    x_sample = x.sample(name="custom_sample")
-    x_sample_duplicate = x.sample(name="custom_sample")
-    x_log_prob = x.log_prob(0., name="custom_log_prob")
-
-    self.assertEqual(x.name, "x")
-    self.assertEqual(x_duplicate.name, "x_1")
-    self.assertEqual(y.name, "y")
-    self.assertTrue(x_sample.name.startswith("x/custom_sample"))
-    self.assertTrue(x_sample_duplicate.name.startswith("x/custom_sample_1"))
-    self.assertTrue(x_log_prob.name.startswith("x/custom_log_prob"))
-
   def testStrWorksCorrectlyScalar(self):
     normal = tfd.Normal(loc=np.float16(0), scale=np.float16(1))
     self.assertEqual(
diff --git a/tensorflow/contrib/distributions/python/ops/autoregressive.py b/tensorflow/contrib/distributions/python/ops/autoregressive.py
index 88ed012784..69f3d57ff0 100644
--- a/tensorflow/contrib/distributions/python/ops/autoregressive.py
+++ b/tensorflow/contrib/distributions/python/ops/autoregressive.py
@@ -145,7 +145,7 @@ class Autoregressive(distribution_lib.Distribution):
       ValueError: if `num_steps < 1`.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       self._distribution_fn = distribution_fn
       self._sample0 = sample0
       self._distribution0 = (distribution_fn() if sample0 is None
diff --git a/tensorflow/contrib/distributions/python/ops/binomial.py b/tensorflow/contrib/distributions/python/ops/binomial.py
index 12d1603178..6a1bb39ab2 100644
--- a/tensorflow/contrib/distributions/python/ops/binomial.py
+++ b/tensorflow/contrib/distributions/python/ops/binomial.py
@@ -164,7 +164,7 @@ class Binomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
+    with ops.name_scope(name, values=[total_count, logits, probs]):
       self._total_count = self._maybe_assert_valid_total_count(
           ops.convert_to_tensor(total_count, name="total_count"),
           validate_args)
diff --git a/tensorflow/contrib/distributions/python/ops/cauchy.py b/tensorflow/contrib/distributions/python/ops/cauchy.py
index daacfe657f..6f5d724a2a 100644
--- a/tensorflow/contrib/distributions/python/ops/cauchy.py
+++ b/tensorflow/contrib/distributions/python/ops/cauchy.py
@@ -121,7 +121,7 @@ class Cauchy(distribution.Distribution):
       TypeError: if `loc` and `scale` have different `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)]
                                     if validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/chi2.py b/tensorflow/contrib/distributions/python/ops/chi2.py
index c77c5fd208..e610f469e5 100644
--- a/tensorflow/contrib/distributions/python/ops/chi2.py
+++ b/tensorflow/contrib/distributions/python/ops/chi2.py
@@ -88,7 +88,7 @@ class Chi2(gamma.Gamma):
     # not true in the parent class "gamma."  therefore, passing
     # allow_nan_stats=True
     # through to the parent class results in unnecessary asserts.
-    with ops.name_scope(name, values=[df]) as name:
+    with ops.name_scope(name, values=[df]):
       with ops.control_dependencies([
           check_ops.assert_positive(df),
       ] if validate_args else []):
@@ -120,7 +120,7 @@ class Chi2WithAbsDf(Chi2):
                allow_nan_stats=True,
                name="Chi2WithAbsDf"):
     parameters = locals()
-    with ops.name_scope(name, values=[df]) as name:
+    with ops.name_scope(name, values=[df]):
       super(Chi2WithAbsDf, self).__init__(
           df=math_ops.floor(
               math_ops.abs(df, name="abs_df"),
diff --git a/tensorflow/contrib/distributions/python/ops/deterministic.py b/tensorflow/contrib/distributions/python/ops/deterministic.py
index a42350430e..8049522e9f 100644
--- a/tensorflow/contrib/distributions/python/ops/deterministic.py
+++ b/tensorflow/contrib/distributions/python/ops/deterministic.py
@@ -87,7 +87,7 @@ class _BaseDeterministic(distribution.Distribution):
       ValueError:  If `loc` is a scalar.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, atol, rtol]) as name:
+    with ops.name_scope(name, values=[loc, atol, rtol]):
       loc = ops.convert_to_tensor(loc, name="loc")
       if is_vector and validate_args:
         msg = "Argument loc must be at least rank 1."
diff --git a/tensorflow/contrib/distributions/python/ops/geometric.py b/tensorflow/contrib/distributions/python/ops/geometric.py
index 53dd42f4c8..8f190e48a7 100644
--- a/tensorflow/contrib/distributions/python/ops/geometric.py
+++ b/tensorflow/contrib/distributions/python/ops/geometric.py
@@ -86,7 +86,7 @@ class Geometric(distribution.Distribution):
     """
 
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]) as name:
+    with ops.name_scope(name, values=[logits, probs]):
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits, probs, validate_args=validate_args, name=name)
 
diff --git a/tensorflow/contrib/distributions/python/ops/gumbel.py b/tensorflow/contrib/distributions/python/ops/gumbel.py
index 2c261073ee..8d05ad6b80 100644
--- a/tensorflow/contrib/distributions/python/ops/gumbel.py
+++ b/tensorflow/contrib/distributions/python/ops/gumbel.py
@@ -125,7 +125,7 @@ class _Gumbel(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/half_normal.py b/tensorflow/contrib/distributions/python/ops/half_normal.py
index d0df2befd6..fc0751a6e0 100644
--- a/tensorflow/contrib/distributions/python/ops/half_normal.py
+++ b/tensorflow/contrib/distributions/python/ops/half_normal.py
@@ -106,7 +106,7 @@ class HalfNormal(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[scale]) as name:
+    with ops.name_scope(name, values=[scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._scale = array_ops.identity(scale, name="scale")
diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py
index d7d0b0c78b..7dcb3e3ac4 100644
--- a/tensorflow/contrib/distributions/python/ops/independent.py
+++ b/tensorflow/contrib/distributions/python/ops/independent.py
@@ -119,7 +119,7 @@ class Independent(distribution_lib.Distribution):
     parameters = locals()
     name = name or "Independent" + distribution.name
     self._distribution = distribution
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       if reinterpreted_batch_ndims is None:
         reinterpreted_batch_ndims = self._get_default_reinterpreted_batch_ndims(
             distribution)
diff --git a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
index 502bd4f493..51ac61dcf6 100644
--- a/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
+++ b/tensorflow/contrib/distributions/python/ops/inverse_gamma.py
@@ -126,7 +126,7 @@ class InverseGamma(distribution.Distribution):
       TypeError: if `concentration` and `rate` are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]) as name:
+    with ops.name_scope(name, values=[concentration, rate]):
       with ops.control_dependencies([
           check_ops.assert_positive(concentration),
           check_ops.assert_positive(rate),
@@ -281,7 +281,7 @@ class InverseGammaWithSoftplusConcentrationRate(InverseGamma):
                allow_nan_stats=True,
                name="InverseGammaWithSoftplusConcentrationRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]) as name:
+    with ops.name_scope(name, values=[concentration, rate]):
       super(InverseGammaWithSoftplusConcentrationRate, self).__init__(
           concentration=nn.softplus(concentration,
                                     name="softplus_concentration"),
diff --git a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
index 66682b2ff5..192dede6ff 100644
--- a/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
+++ b/tensorflow/contrib/distributions/python/ops/kumaraswamy.py
@@ -151,11 +151,10 @@ class Kumaraswamy(transformed_distribution.TransformedDistribution):
         more of the statistic's batch members are undefined.
       name: Python `str` name prefixed to Ops created by this class.
     """
-    with ops.name_scope(name, values=[concentration1, concentration0]) as name:
-      concentration1 = ops.convert_to_tensor(
-          concentration1, name="concentration1")
-      concentration0 = ops.convert_to_tensor(
-          concentration0, name="concentration0")
+    concentration1 = ops.convert_to_tensor(
+        concentration1, name="concentration1")
+    concentration0 = ops.convert_to_tensor(
+        concentration0, name="concentration0")
     super(Kumaraswamy, self).__init__(
         distribution=uniform.Uniform(
             low=array_ops.zeros([], dtype=concentration1.dtype),
diff --git a/tensorflow/contrib/distributions/python/ops/logistic.py b/tensorflow/contrib/distributions/python/ops/logistic.py
index c83b5bc2e3..68e6bca5a5 100644
--- a/tensorflow/contrib/distributions/python/ops/logistic.py
+++ b/tensorflow/contrib/distributions/python/ops/logistic.py
@@ -120,7 +120,7 @@ class Logistic(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
diff --git a/tensorflow/contrib/distributions/python/ops/mixture.py b/tensorflow/contrib/distributions/python/ops/mixture.py
index 2ef294af2e..cef6a143fc 100644
--- a/tensorflow/contrib/distributions/python/ops/mixture.py
+++ b/tensorflow/contrib/distributions/python/ops/mixture.py
@@ -145,7 +145,7 @@ class Mixture(distribution.Distribution):
           "none of the components provide a static number of ndims")
 
     # Ensure that all batch and event ndims are consistent.
-    with ops.name_scope(name, values=[cat.logits]) as name:
+    with ops.name_scope(name, values=[cat.logits]):
       num_components = cat.event_size
       static_num_components = tensor_util.constant_value(num_components)
       if static_num_components is None:
diff --git a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
index 0b1301e551..b93bdc5ab4 100644
--- a/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
+++ b/tensorflow/contrib/distributions/python/ops/mixture_same_family.py
@@ -131,7 +131,7 @@ class MixtureSameFamily(distribution.Distribution):
         `components_distribution` rightmost batch shape.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       self._mixture_distribution = mixture_distribution
       self._components_distribution = components_distribution
       self._runtime_assertions = []
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag.py b/tensorflow/contrib/distributions/python/ops/mvn_diag.py
index e3236c2db9..e862552880 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_diag.py
@@ -194,7 +194,7 @@ class MultivariateNormalDiag(
       ValueError: if at most `scale_identity_multiplier` is specified.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier]):
         # No need to validate_args while making diag_scale.  The returned
@@ -225,7 +225,7 @@ class MultivariateNormalDiagWithSoftplusScale(MultivariateNormalDiag):
                allow_nan_stats=True,
                name="MultivariateNormalDiagWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[scale_diag]) as name:
+    with ops.name_scope(name, values=[scale_diag]):
       super(MultivariateNormalDiagWithSoftplusScale, self).__init__(
           loc=loc,
           scale_diag=nn.softplus(scale_diag),
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
index 2f6a6f198c..413e88f03a 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_diag_plus_low_rank.py
@@ -218,7 +218,7 @@ class MultivariateNormalDiagPlusLowRank(
     parameters = locals()
     def _convert_to_tensor(x, name):
       return None if x is None else ops.convert_to_tensor(x, name=name)
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier, scale_perturb_factor,
           scale_perturb_diag]):
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
index 86fcd4db54..4bea99fbb7 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_full_covariance.py
@@ -159,7 +159,7 @@ class MultivariateNormalFullCovariance(mvn_tril.MultivariateNormalTriL):
     parameters = locals()
 
     # Convert the covariance_matrix up to a scale_tril and call MVNTriL.
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=[loc, covariance_matrix]):
         if covariance_matrix is None:
           scale_tril = None
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
index 44c92312c7..a739979289 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_linear_operator.py
@@ -176,7 +176,7 @@ class MultivariateNormalLinearOperator(
     if not scale.dtype.is_floating:
       raise TypeError("`scale` parameter must have floating-point dtype.")
 
-    with ops.name_scope(name, values=[loc] + scale.graph_parents) as name:
+    with ops.name_scope(name, values=[loc] + scale.graph_parents):
       # Since expand_dims doesn't preserve constant-ness, we obtain the
       # non-dynamic value if possible.
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
diff --git a/tensorflow/contrib/distributions/python/ops/mvn_tril.py b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
index d6f8b731cb..6c7dc4ca7a 100644
--- a/tensorflow/contrib/distributions/python/ops/mvn_tril.py
+++ b/tensorflow/contrib/distributions/python/ops/mvn_tril.py
@@ -184,7 +184,7 @@ class MultivariateNormalTriL(
       return None if x is None else ops.convert_to_tensor(x, name=name)
     if loc is None and scale_tril is None:
       raise ValueError("Must specify one or both of `loc`, `scale_tril`.")
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=[loc, scale_tril]):
         loc = _convert_to_tensor(loc, name="loc")
         scale_tril = _convert_to_tensor(scale_tril, name="scale_tril")
diff --git a/tensorflow/contrib/distributions/python/ops/negative_binomial.py b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
index eeaf9c0a5e..3a58df80da 100644
--- a/tensorflow/contrib/distributions/python/ops/negative_binomial.py
+++ b/tensorflow/contrib/distributions/python/ops/negative_binomial.py
@@ -91,7 +91,7 @@ class NegativeBinomial(distribution.Distribution):
     """
 
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
+    with ops.name_scope(name, values=[total_count, logits, probs]):
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits, probs, validate_args=validate_args, name=name)
       with ops.control_dependencies(
diff --git a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
index fd5e8b10b0..46c2cc8b7a 100644
--- a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
@@ -116,7 +116,7 @@ class OneHotCategorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]) as name:
+    with ops.name_scope(name, values=[logits, probs]):
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           name=name, logits=logits, probs=probs, validate_args=validate_args,
           multidimensional=True)
diff --git a/tensorflow/contrib/distributions/python/ops/poisson.py b/tensorflow/contrib/distributions/python/ops/poisson.py
index a84aad6fc9..02e97c0a2f 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson.py
@@ -94,7 +94,7 @@ class Poisson(distribution.Distribution):
       TypeError: if `log_rate` is not a float-type.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[rate]) as name:
+    with ops.name_scope(name, values=[rate]):
       if (rate is None) == (log_rate is None):
         raise ValueError("Must specify exactly one of `rate` and `log_rate`.")
       elif log_rate is None:
diff --git a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
index ea5514d345..92f2bba182 100644
--- a/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
+++ b/tensorflow/contrib/distributions/python/ops/poisson_lognormal.py
@@ -256,7 +256,7 @@ class PoissonLogNormalQuadratureCompound(distribution_lib.Distribution):
         `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       if loc is not None:
         loc = ops.convert_to_tensor(loc, name="loc")
       if scale is not None:
diff --git a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py b/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
index 1ef7651d03..8aebb79b91 100644
--- a/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
+++ b/tensorflow/contrib/distributions/python/ops/quantized_distribution.py
@@ -217,7 +217,7 @@ class QuantizedDistribution(distributions.Distribution):
     values = (
         list(distribution.parameters.values()) +
         [low, high])
-    with ops.name_scope(name, values=values) as name:
+    with ops.name_scope(name, values=values):
       self._dist = distribution
 
       if low is not None:
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
index 00c1ede5c1..b525809015 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
@@ -166,7 +166,7 @@ class RelaxedBernoulli(transformed_distribution.TransformedDistribution):
       ValueError: If both `probs` and `logits` are passed, or if neither.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs, temperature]) as name:
+    with ops.name_scope(name, values=[logits, probs, temperature]):
       with ops.control_dependencies([check_ops.assert_positive(temperature)]
                                     if validate_args else []):
         self._temperature = array_ops.identity(temperature, name="temperature")
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
index b60a4872aa..ff33f327c7 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
@@ -163,7 +163,7 @@ class ExpRelaxedOneHotCategorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs, temperature]) as name:
+    with ops.name_scope(name, values=[logits, probs, temperature]):
 
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           name=name, logits=logits, probs=probs, validate_args=validate_args,
diff --git a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
index 7a8f5bbfbe..0d8a192691 100644
--- a/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
+++ b/tensorflow/contrib/distributions/python/ops/sinh_arcsinh.py
@@ -134,8 +134,7 @@ class SinhArcsinh(transformed_distribution.TransformedDistribution):
     """
     parameters = locals()
 
-    with ops.name_scope(name,
-                        values=[loc, scale, skewness, tailweight]) as name:
+    with ops.name_scope(name, values=[loc, scale, skewness, tailweight]):
       loc = ops.convert_to_tensor(loc, name="loc")
       dtype = loc.dtype
       scale = ops.convert_to_tensor(scale, name="scale", dtype=dtype)
diff --git a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
index 026de97be7..971d65c4a6 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_diffeomixture.py
@@ -396,7 +396,7 @@ class VectorDiffeomixture(distribution_lib.Distribution):
       ValueError: if `not distribution.is_scalar_event`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[mix_loc, temperature]) as name:
+    with ops.name_scope(name, values=[mix_loc, temperature]):
       if not scale or len(scale) < 2:
         raise ValueError("Must specify list (or list-like object) of scale "
                          "LinearOperators, one for each component with "
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
index e265b5d0f7..526fe2d39a 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_diag.py
@@ -176,7 +176,7 @@ class VectorExponentialDiag(
       ValueError: if at most `scale_identity_multiplier` is specified.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=[
           loc, scale_diag, scale_identity_multiplier]):
         # No need to validate_args while making diag_scale.  The returned
diff --git a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
index 89136d6760..9d5fd9ac41 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_exponential_linear_operator.py
@@ -181,7 +181,7 @@ class VectorExponentialLinearOperator(
     if not scale.dtype.is_floating:
       raise TypeError("`scale` parameter must have floating-point dtype.")
 
-    with ops.name_scope(name, values=[loc] + scale.graph_parents) as name:
+    with ops.name_scope(name, values=[loc] + scale.graph_parents):
       # Since expand_dims doesn't preserve constant-ness, we obtain the
       # non-dynamic value if possible.
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
diff --git a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
index b8ac7c10ca..003c66b941 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_sinh_arcsinh_diag.py
@@ -169,7 +169,7 @@ class VectorSinhArcsinhDiag(transformed_distribution.TransformedDistribution):
         name,
         values=[
             loc, scale_diag, scale_identity_multiplier, skewness, tailweight
-        ]) as name:
+        ]):
       loc = ops.convert_to_tensor(loc, name="loc") if loc is not None else loc
       tailweight = 1. if tailweight is None else tailweight
       has_default_skewness = skewness is None
diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
index 23a01cc95c..8c67647a61 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
@@ -178,7 +178,7 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution):
     parameters = locals()
     graph_parents = [df, loc, scale_identity_multiplier, scale_diag,
                      scale_tril, scale_perturb_factor, scale_perturb_diag]
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       with ops.name_scope("init", values=graph_parents):
         # The shape of the _VectorStudentT distribution is governed by the
         # relationship between df.batch_shape and affine.batch_shape. In
diff --git a/tensorflow/contrib/distributions/python/ops/wishart.py b/tensorflow/contrib/distributions/python/ops/wishart.py
index 91453fed5d..5a8c94dabf 100644
--- a/tensorflow/contrib/distributions/python/ops/wishart.py
+++ b/tensorflow/contrib/distributions/python/ops/wishart.py
@@ -109,7 +109,7 @@ class _WishartLinearOperator(distribution.Distribution):
     """
     parameters = locals()
     self._cholesky_input_output_matrices = cholesky_input_output_matrices
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name) as ns:
       with ops.name_scope("init", values=[df, scale_operator]):
         if not scale_operator.dtype.is_floating:
           raise TypeError(
@@ -163,7 +163,7 @@ class _WishartLinearOperator(distribution.Distribution):
         parameters=parameters,
         graph_parents=([self._df, self._dimension] +
                        self._scale_operator.graph_parents),
-        name=name)
+        name=ns)
 
   @property
   def df(self):
@@ -531,7 +531,7 @@ class WishartCholesky(_WishartLinearOperator):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[scale]) as name:
+    with ops.name_scope(name, values=[scale]):
       with ops.name_scope("init", values=[scale]):
         scale = ops.convert_to_tensor(scale)
         if validate_args:
@@ -647,7 +647,7 @@ class WishartFull(_WishartLinearOperator):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name) as ns:
       with ops.name_scope("init", values=[scale]):
         scale = ops.convert_to_tensor(scale)
         if validate_args:
@@ -666,5 +666,5 @@ class WishartFull(_WishartLinearOperator):
         cholesky_input_output_matrices=cholesky_input_output_matrices,
         validate_args=validate_args,
         allow_nan_stats=allow_nan_stats,
-        name=name)
+        name=ns)
     self._parameters = parameters
diff --git a/tensorflow/python/ops/distributions/bernoulli.py b/tensorflow/python/ops/distributions/bernoulli.py
index 2c9f0e9a32..68aaf3815e 100644
--- a/tensorflow/python/ops/distributions/bernoulli.py
+++ b/tensorflow/python/ops/distributions/bernoulli.py
@@ -72,7 +72,7 @@ class Bernoulli(distribution.Distribution):
       ValueError: If p and logits are passed, or if neither are passed.
     """
     parameters = locals()
-    with ops.name_scope(name) as name:
+    with ops.name_scope(name):
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits=logits,
           probs=probs,
diff --git a/tensorflow/python/ops/distributions/beta.py b/tensorflow/python/ops/distributions/beta.py
index 8beab99bf8..469bcadb8e 100644
--- a/tensorflow/python/ops/distributions/beta.py
+++ b/tensorflow/python/ops/distributions/beta.py
@@ -151,7 +151,7 @@ class Beta(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration1, concentration0]) as name:
+    with ops.name_scope(name, values=[concentration1, concentration0]):
       self._concentration1 = self._maybe_assert_valid_concentration(
           ops.convert_to_tensor(concentration1, name="concentration1"),
           validate_args)
@@ -323,7 +323,7 @@ class BetaWithSoftplusConcentration(Beta):
                name="BetaWithSoftplusConcentration"):
     parameters = locals()
     with ops.name_scope(name, values=[concentration1,
-                                      concentration0]) as name:
+                                      concentration0]) as ns:
       super(BetaWithSoftplusConcentration, self).__init__(
           concentration1=nn.softplus(concentration1,
                                      name="softplus_concentration1"),
@@ -331,7 +331,7 @@ class BetaWithSoftplusConcentration(Beta):
                                      name="softplus_concentration0"),
           validate_args=validate_args,
           allow_nan_stats=allow_nan_stats,
-          name=name)
+          name=ns)
     self._parameters = parameters
 
 
diff --git a/tensorflow/python/ops/distributions/categorical.py b/tensorflow/python/ops/distributions/categorical.py
index 66fa9e110c..9161e3fa9f 100644
--- a/tensorflow/python/ops/distributions/categorical.py
+++ b/tensorflow/python/ops/distributions/categorical.py
@@ -183,7 +183,7 @@ class Categorical(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[logits, probs]) as name:
+    with ops.name_scope(name, values=[logits, probs]):
       self._logits, self._probs = distribution_util.get_logits_and_probs(
           logits=logits,
           probs=probs,
diff --git a/tensorflow/python/ops/distributions/dirichlet.py b/tensorflow/python/ops/distributions/dirichlet.py
index eafcd5c78f..25afeec936 100644
--- a/tensorflow/python/ops/distributions/dirichlet.py
+++ b/tensorflow/python/ops/distributions/dirichlet.py
@@ -155,7 +155,7 @@ class Dirichlet(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration]) as name:
+    with ops.name_scope(name, values=[concentration]):
       self._concentration = self._maybe_assert_valid_concentration(
           ops.convert_to_tensor(concentration, name="concentration"),
           validate_args)
diff --git a/tensorflow/python/ops/distributions/dirichlet_multinomial.py b/tensorflow/python/ops/distributions/dirichlet_multinomial.py
index fe0ed7e07d..03a98c56ba 100644
--- a/tensorflow/python/ops/distributions/dirichlet_multinomial.py
+++ b/tensorflow/python/ops/distributions/dirichlet_multinomial.py
@@ -192,7 +192,7 @@ class DirichletMultinomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, concentration]) as name:
+    with ops.name_scope(name, values=[total_count, concentration]):
       # Broadcasting works because:
       # * The broadcasting convention is to prepend dimensions of size [1], and
       #   we use the last dimension for the distribution, whereas
diff --git a/tensorflow/python/ops/distributions/distribution.py b/tensorflow/python/ops/distributions/distribution.py
index b6e6a01fc8..7c43bf54fc 100644
--- a/tensorflow/python/ops/distributions/distribution.py
+++ b/tensorflow/python/ops/distributions/distribution.py
@@ -434,19 +434,13 @@ class Distribution(_BaseDistribution):
     for i, t in enumerate(graph_parents):
       if t is None or not tensor_util.is_tensor(t):
         raise ValueError("Graph parent item %d is not a Tensor; %s." % (i, t))
-    if name is None:
-      with ops.name_scope(type(self).__name__) as name:
-        pass
-    # If a name ends with a '/' it is a "name scope" and we use it as-is, after
-    # removing the trailing '/'.
-    name = name[:-1] if (name and name[-1] == "/") else name
     self._dtype = dtype
     self._reparameterization_type = reparameterization_type
     self._allow_nan_stats = allow_nan_stats
     self._validate_args = validate_args
     self._parameters = parameters or {}
     self._graph_parents = graph_parents
-    self._name = name
+    self._name = name or type(self).__name__
 
   @classmethod
   def param_shapes(cls, sample_shape, name="DistributionParamShapes"):
@@ -1163,7 +1157,7 @@ class Distribution(_BaseDistribution):
   @contextlib.contextmanager
   def _name_scope(self, name=None, values=None):
     """Helper function to standardize op scope."""
-    with ops.name_scope(self.name + "/"):  # use absolute name scope
+    with ops.name_scope(self.name):
       with ops.name_scope(name, values=(
           ([] if values is None else values) + self._graph_parents)) as scope:
         yield scope
diff --git a/tensorflow/python/ops/distributions/exponential.py b/tensorflow/python/ops/distributions/exponential.py
index cf0e729e1a..6345a76d48 100644
--- a/tensorflow/python/ops/distributions/exponential.py
+++ b/tensorflow/python/ops/distributions/exponential.py
@@ -95,7 +95,7 @@ class Exponential(gamma.Gamma):
     # true in the parent class "Gamma."  Therefore, passing
     # allow_nan_stats=True
     # through to the parent class results in unnecessary asserts.
-    with ops.name_scope(name, values=[rate]) as name:
+    with ops.name_scope(name, values=[rate]):
       self._rate = ops.convert_to_tensor(rate, name="rate")
     super(Exponential, self).__init__(
         concentration=array_ops.ones([], dtype=self._rate.dtype),
@@ -144,7 +144,7 @@ class ExponentialWithSoftplusRate(Exponential):
                allow_nan_stats=True,
                name="ExponentialWithSoftplusRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[rate]) as name:
+    with ops.name_scope(name, values=[rate]):
       super(ExponentialWithSoftplusRate, self).__init__(
           rate=nn.softplus(rate, name="softplus_rate"),
           validate_args=validate_args,
diff --git a/tensorflow/python/ops/distributions/gamma.py b/tensorflow/python/ops/distributions/gamma.py
index d39f7c56d3..adb1f4f9a8 100644
--- a/tensorflow/python/ops/distributions/gamma.py
+++ b/tensorflow/python/ops/distributions/gamma.py
@@ -127,7 +127,7 @@ class Gamma(distribution.Distribution):
       TypeError: if `concentration` and `rate` are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]) as name:
+    with ops.name_scope(name, values=[concentration, rate]):
       with ops.control_dependencies([
           check_ops.assert_positive(concentration),
           check_ops.assert_positive(rate),
@@ -262,7 +262,7 @@ class GammaWithSoftplusConcentrationRate(Gamma):
                allow_nan_stats=True,
                name="GammaWithSoftplusConcentrationRate"):
     parameters = locals()
-    with ops.name_scope(name, values=[concentration, rate]) as name:
+    with ops.name_scope(name, values=[concentration, rate]):
       super(GammaWithSoftplusConcentrationRate, self).__init__(
           concentration=nn.softplus(concentration,
                                     name="softplus_concentration"),
diff --git a/tensorflow/python/ops/distributions/laplace.py b/tensorflow/python/ops/distributions/laplace.py
index 3ccfc618d1..e98ac855c5 100644
--- a/tensorflow/python/ops/distributions/laplace.py
+++ b/tensorflow/python/ops/distributions/laplace.py
@@ -101,7 +101,7 @@ class Laplace(distribution.Distribution):
       TypeError: if `loc` and `scale` are of different dtype.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
@@ -218,7 +218,7 @@ class LaplaceWithSoftplusScale(Laplace):
                allow_nan_stats=True,
                name="LaplaceWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       super(LaplaceWithSoftplusScale, self).__init__(
           loc=loc,
           scale=nn.softplus(scale, name="softplus_scale"),
diff --git a/tensorflow/python/ops/distributions/multinomial.py b/tensorflow/python/ops/distributions/multinomial.py
index ab77f5c1f8..4ae67a009b 100644
--- a/tensorflow/python/ops/distributions/multinomial.py
+++ b/tensorflow/python/ops/distributions/multinomial.py
@@ -183,7 +183,7 @@ class Multinomial(distribution.Distribution):
       name: Python `str` name prefixed to Ops created by this class.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[total_count, logits, probs]) as name:
+    with ops.name_scope(name, values=[total_count, logits, probs]):
       self._total_count = ops.convert_to_tensor(total_count, name="total_count")
       if validate_args:
         self._total_count = (
diff --git a/tensorflow/python/ops/distributions/normal.py b/tensorflow/python/ops/distributions/normal.py
index 20d4420e91..32e8a49c81 100644
--- a/tensorflow/python/ops/distributions/normal.py
+++ b/tensorflow/python/ops/distributions/normal.py
@@ -132,7 +132,7 @@ class Normal(distribution.Distribution):
       TypeError: if `loc` and `scale` have different `dtype`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[loc, scale]) as name:
+    with ops.name_scope(name, values=[loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(scale)] if
                                     validate_args else []):
         self._loc = array_ops.identity(loc, name="loc")
@@ -244,7 +244,7 @@ class NormalWithSoftplusScale(Normal):
                allow_nan_stats=True,
                name="NormalWithSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[scale]) as name:
+    with ops.name_scope(name, values=[scale]):
       super(NormalWithSoftplusScale, self).__init__(
           loc=loc,
           scale=nn.softplus(scale, name="softplus_scale"),
diff --git a/tensorflow/python/ops/distributions/student_t.py b/tensorflow/python/ops/distributions/student_t.py
index 961b07a7bd..9d9e65b4e8 100644
--- a/tensorflow/python/ops/distributions/student_t.py
+++ b/tensorflow/python/ops/distributions/student_t.py
@@ -158,7 +158,7 @@ class StudentT(distribution.Distribution):
       TypeError: if loc and scale are different dtypes.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[df, loc, scale]) as name:
+    with ops.name_scope(name, values=[df, loc, scale]):
       with ops.control_dependencies([check_ops.assert_positive(df)]
                                     if validate_args else []):
         self._df = array_ops.identity(df, name="df")
@@ -350,7 +350,7 @@ class StudentTWithAbsDfSoftplusScale(StudentT):
                allow_nan_stats=True,
                name="StudentTWithAbsDfSoftplusScale"):
     parameters = locals()
-    with ops.name_scope(name, values=[df, scale]) as name:
+    with ops.name_scope(name, values=[df, scale]):
       super(StudentTWithAbsDfSoftplusScale, self).__init__(
           df=math_ops.floor(math_ops.abs(df)),
           loc=loc,
diff --git a/tensorflow/python/ops/distributions/transformed_distribution.py b/tensorflow/python/ops/distributions/transformed_distribution.py
index 6290297153..1efcf9d32e 100644
--- a/tensorflow/python/ops/distributions/transformed_distribution.py
+++ b/tensorflow/python/ops/distributions/transformed_distribution.py
@@ -258,7 +258,7 @@ class TransformedDistribution(distribution_lib.Distribution):
     parameters = locals()
     name = name or (("" if bijector is None else bijector.name) +
                     distribution.name)
-    with ops.name_scope(name, values=[event_shape, batch_shape]) as name:
+    with ops.name_scope(name, values=[event_shape, batch_shape]):
       # For convenience we define some handy constants.
       self._zero = constant_op.constant(0, dtype=dtypes.int32, name="zero")
       self._empty = constant_op.constant([], dtype=dtypes.int32, name="empty")
diff --git a/tensorflow/python/ops/distributions/uniform.py b/tensorflow/python/ops/distributions/uniform.py
index b8acd0ded6..ec623b55eb 100644
--- a/tensorflow/python/ops/distributions/uniform.py
+++ b/tensorflow/python/ops/distributions/uniform.py
@@ -103,7 +103,7 @@ class Uniform(distribution.Distribution):
       InvalidArgumentError: if `low >= high` and `validate_args=False`.
     """
     parameters = locals()
-    with ops.name_scope(name, values=[low, high]) as name:
+    with ops.name_scope(name, values=[low, high]):
       with ops.control_dependencies([
           check_ops.assert_less(
               low, high, message="uniform not defined when low >= high.")
-- 
GitLab


From fb8c7b1d4f230995de9411370cdbdd7b2174adc3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 10:12:42 -0700
Subject: [PATCH 677/960] LSTM support: Further quantized Fully-Connected op
 optimization.

PiperOrigin-RevId: 190946885
---
 .../internal/optimized/optimized_ops.h        | 169 ++++++++++++++++--
 1 file changed, 152 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index e079ff3f4c..4661004d09 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -324,6 +324,22 @@ void Gemm(const Eigen::MatrixBase<Lhs>& lhs, const Eigen::MatrixBase<Rhs>& rhs,
   }
 }
 
+inline void optimized_ops_preload_l1_stream(const uint8* ptr) {
+#ifdef GEMMLOWP_ARM_64
+  asm volatile("prfm pldl1strm, [%[ptr]]\n" ::[ptr] "r"(ptr) :);
+#else
+  gemmlowp::Prefetch(ptr);
+#endif
+}
+
+inline void optimized_ops_preload_l1_keep(const uint8* ptr) {
+#ifdef GEMMLOWP_ARM_64
+  asm volatile("prfm pldl1keep, [%[ptr]]\n" ::[ptr] "r"(ptr) :);
+#else
+  gemmlowp::Prefetch(ptr);
+#endif
+}
+
 #ifdef GEMMLOWP_NEON
 // In the common case of batch size 1, a fully-connected node degenerates
 // to a matrix*vector product. LSTM cells contain a fully-connected node;
@@ -516,6 +532,124 @@ inline void GEMVForLstmCell(const uint8* input_data, const Dims<4>& input_dims,
 }
 #endif
 
+#ifdef GEMMLOWP_NEON
+inline void GEMVForLstmCellWithSymmetricRange(
+    const uint8* input_data, const Dims<4>& input_dims,
+    const uint8* weights_data, const Dims<4>& weights_dims,
+    const int32* bias_data, const Dims<4>& bias_dims, int32 accum_multiplier,
+    int accum_shift, int16* output_data, const Dims<4>& output_dims) {
+  gemmlowp::ScopedProfilingLabel label("GEMVForLstmCellWithSymmetricRange");
+  TFLITE_DCHECK(IsPackedWithoutStrides(input_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(weights_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(bias_dims));
+  TFLITE_DCHECK(IsPackedWithoutStrides(output_dims));
+  TFLITE_DCHECK_EQ(ArraySize(output_dims, 1) * ArraySize(output_dims, 2) *
+                       ArraySize(output_dims, 3),
+                   1);
+  const int input_size = input_dims.strides[3];
+  const int output_size = MatchingArraySize(weights_dims, 1, output_dims, 0);
+  // This special fast path for quantized LSTM cells does not try to support
+  // odd sizes that we haven't encountered in any LSTM cell, that would
+  // require special code (that would go untested until any LSTM cell
+  // exercises it). We just guard our assumptions about size evenness with
+  // the following assertions.
+  TFLITE_DCHECK(!(output_size % 4));
+  TFLITE_DCHECK(!(input_size % 8));
+  const int32* bias_ptr = bias_data;
+  int16* output_ptr = output_data;
+  const uint8x16_t signbit = vdupq_n_u8(0x80);
+  for (int in = 0; in < input_size; in += 32) {
+    optimized_ops_preload_l1_keep(input_data + in);
+  }
+  for (int out = 0; out < output_size; out += 4) {
+    const uint8* weights_ptr_0 = weights_data + out * input_size;
+    const uint8* weights_ptr_1 = weights_ptr_0 + 1 * input_size;
+    const uint8* weights_ptr_2 = weights_ptr_0 + 2 * input_size;
+    const uint8* weights_ptr_3 = weights_ptr_0 + 3 * input_size;
+
+    int32x4_t acc_0 = vdupq_n_s32(0);
+    int32x4_t acc_1 = vdupq_n_s32(0);
+    int32x4_t acc_2 = vdupq_n_s32(0);
+    int32x4_t acc_3 = vdupq_n_s32(0);
+    int in = 0;
+    const int kReadAhead = 256;
+    // Handle 16 levels of depth at a time.
+    for (; in < input_size; in += 16) {
+      int8x16_t weights_val_0 =
+          vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(weights_ptr_0)));
+      int8x16_t weights_val_1 =
+          vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(weights_ptr_1)));
+      int8x16_t weights_val_2 =
+          vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(weights_ptr_2)));
+      int8x16_t weights_val_3 =
+          vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(weights_ptr_3)));
+      int8x16_t input_val =
+          vreinterpretq_s8_u8(veorq_u8(signbit, vld1q_u8(input_data + in)));
+      int16x8_t acc16_0 =
+          vmull_s8(vget_low_s8(weights_val_0), vget_low_s8(input_val));
+      int16x8_t acc16_1 =
+          vmull_s8(vget_low_s8(weights_val_1), vget_low_s8(input_val));
+      int16x8_t acc16_2 =
+          vmull_s8(vget_low_s8(weights_val_2), vget_low_s8(input_val));
+      int16x8_t acc16_3 =
+          vmull_s8(vget_low_s8(weights_val_3), vget_low_s8(input_val));
+      acc16_0 = vmlal_s8(acc16_0, vget_high_s8(weights_val_0),
+                         vget_high_s8(input_val));
+      acc16_1 = vmlal_s8(acc16_1, vget_high_s8(weights_val_1),
+                         vget_high_s8(input_val));
+      acc16_2 = vmlal_s8(acc16_2, vget_high_s8(weights_val_2),
+                         vget_high_s8(input_val));
+      acc16_3 = vmlal_s8(acc16_3, vget_high_s8(weights_val_3),
+                         vget_high_s8(input_val));
+      acc_0 = vpadalq_s16(acc_0, acc16_0);
+      acc_1 = vpadalq_s16(acc_1, acc16_1);
+      acc_2 = vpadalq_s16(acc_2, acc16_2);
+      acc_3 = vpadalq_s16(acc_3, acc16_3);
+      weights_ptr_0 += 16;
+      weights_ptr_1 += 16;
+      weights_ptr_2 += 16;
+      weights_ptr_3 += 16;
+      optimized_ops_preload_l1_stream(weights_ptr_0 + kReadAhead);
+      optimized_ops_preload_l1_stream(weights_ptr_1 + kReadAhead);
+      optimized_ops_preload_l1_stream(weights_ptr_2 + kReadAhead);
+      optimized_ops_preload_l1_stream(weights_ptr_3 + kReadAhead);
+    }
+    // Horizontally reduce accumulators
+    int32x2_t pairwise_reduced_acc_0, pairwise_reduced_acc_1,
+        pairwise_reduced_acc_2, pairwise_reduced_acc_3;
+    pairwise_reduced_acc_0 =
+        vpadd_s32(vget_low_s32(acc_0), vget_high_s32(acc_0));
+    pairwise_reduced_acc_1 =
+        vpadd_s32(vget_low_s32(acc_1), vget_high_s32(acc_1));
+    pairwise_reduced_acc_2 =
+        vpadd_s32(vget_low_s32(acc_2), vget_high_s32(acc_2));
+    pairwise_reduced_acc_3 =
+        vpadd_s32(vget_low_s32(acc_3), vget_high_s32(acc_3));
+    const int32x2_t reduced_lo =
+        vpadd_s32(pairwise_reduced_acc_0, pairwise_reduced_acc_1);
+    const int32x2_t reduced_hi =
+        vpadd_s32(pairwise_reduced_acc_2, pairwise_reduced_acc_3);
+    int32x4_t reduced = vcombine_s32(reduced_lo, reduced_hi);
+    // Add bias values.
+    int32x4_t bias_vec = vld1q_s32(bias_ptr);
+    bias_ptr += 4;
+    reduced = vaddq_s32(reduced, bias_vec);
+    int left_shift = accum_shift > 0 ? accum_shift : 0;
+    int right_shift = accum_shift > 0 ? 0 : -accum_shift;
+    reduced = vshlq_s32(reduced, vdupq_n_s32(left_shift));
+    // Multiply by the fixed-point multiplier.
+    reduced = vqrdmulhq_n_s32(reduced, accum_multiplier);
+    // Rounding-shift-right.
+    using gemmlowp::RoundingDivideByPOT;
+    reduced = RoundingDivideByPOT(reduced, right_shift);
+    // Narrow values down to 16 bit signed.
+    const int16x4_t res16 = vqmovn_s32(reduced);
+    vst1_s16(output_ptr, res16);
+    output_ptr += 4;
+  }
+}
+#endif
+
 inline void FullyConnected(const float* input_data, const Dims<4>& input_dims,
                            const float* weights_data,
                            const Dims<4>& weights_dims, const float* bias_data,
@@ -559,14 +693,6 @@ void FullyConnected(const float* input_data, const Dims<4>& input_dims,
                  output_data, output_dims);
 }
 
-inline void preload_l1_stream(const uint8* ptr) {
-#ifdef GEMMLOWP_ARM_64
-  asm volatile("prfm pldl1strm, [%[ptr]]\n" ::[ptr] "r"(ptr) :);
-#else
-  gemmlowp::Prefetch(ptr);
-#endif
-}
-
 #ifdef USE_NEON
 inline void FullyConnectedAsGEMV(
     const uint8* input_data, const Dims<4>& input_dims, int32 input_offset,
@@ -587,10 +713,10 @@ inline void FullyConnectedAsGEMV(
   const int output_size = MatchingArraySize(filter_dims, 1, output_dims, 0);
   static constexpr int kPeel = 4;
   for (int k = 0; k < input_size; k += 64) {
-    preload_l1_stream(input_data + k);
+    optimized_ops_preload_l1_stream(input_data + k);
   }
   for (int k = 0; k < kPeel * input_size; k += 64) {
-    preload_l1_stream(filter_data + k);
+    optimized_ops_preload_l1_stream(filter_data + k);
   }
   TFLITE_DCHECK(!(output_size % kPeel));
   const int32* bias_ptr = bias_data;
@@ -609,7 +735,7 @@ inline void FullyConnectedAsGEMV(
       for (int k = 0; k < kPeel; k++) {
         const uint8* filter_ptr = filter_data + in + (out + k) * input_size;
         filter_val_u8[k] = vld1q_u8(filter_ptr);
-        preload_l1_stream(filter_ptr + 64);
+        optimized_ops_preload_l1_stream(filter_ptr + 64);
       }
       int16x8_t input_val[2];
       const uint8x8_t low = vget_low_u8(input_val_u8);
@@ -834,13 +960,22 @@ inline void FullyConnected(
   // the output range is [-2^3, 2^3] == [-8, 8]. The rationale for that
   // is explained in the function comment above.
 #ifdef GEMMLOWP_NEON
-  if (batches == 1 && !(output_depth % 4) && !(accum_depth % 8) &&
-      input_offset == -128 && output_activation_min == -32768 &&
+  if (batches == 1 && input_offset == -128 && output_activation_min == -32768 &&
       output_activation_max == 32767) {
-    GEMVForLstmCell(input_data, input_dims, filter_data, filter_dims,
-                    filter_offset, bias_data_int32, bias_dims,
-                    output_multiplier, -output_shift, output_data, output_dims);
-    return;
+    if (filter_offset == -128 && !(output_depth % 4) && !(accum_depth % 16)) {
+      GEMVForLstmCellWithSymmetricRange(input_data, input_dims, filter_data,
+                                        filter_dims, bias_data_int32, bias_dims,
+                                        output_multiplier, -output_shift,
+                                        output_data, output_dims);
+      return;
+    }
+    if (!(output_depth % 4) && !(accum_depth % 8)) {
+      GEMVForLstmCell(input_data, input_dims, filter_data, filter_dims,
+                      filter_offset, bias_data_int32, bias_dims,
+                      output_multiplier, -output_shift, output_data,
+                      output_dims);
+      return;
+    }
   }
 #endif
   gemmlowp::MatrixMap<const uint8, gemmlowp::MapOrder::RowMajor> weights_matrix(
-- 
GitLab


From 9a7a63aff142658db6d54027815a54a267be808a Mon Sep 17 00:00:00 2001
From: Justin Lebar <jlebar@google.com>
Date: Thu, 29 Mar 2018 10:30:31 -0700
Subject: [PATCH 678/960] [XLA:GPU] Assume that tuple sub-buffers are available
 at runtime.

Previously we assumed this was not the case, and allowed front-ends to
pass in a pointer to tuple without also passing in pointers to
sub-buffers.

This mostly worked: Whenever we wanted a tuple sub-buffer, we'd just
chase the tuple's pointers in our emitted kernel.

But this doesn't work if we ever need a pointer to that sub-buffer on
the host.  Which we do if e.g. the sub-buffer is an input to a cudnn
call.

There are various ways to make this work, but by far the simplest and
most efficient is simply to specify away this problem, and say that the
front-end *must* give us all the pointers we want.  This is what the
earlier change, "Assert that all buffers and sub-buffers passed to XLA
have an explicit pointer" did.

This change adds a testcase and lets us skip some pointer chasing when
we have a tuple whose sub-buffers are known statically.

PiperOrigin-RevId: 190949743
---
 .../xla/service/gpu/ir_emitter_unnested.cc    | 108 +++++++-----------
 .../compiler/xla/tests/dot_operation_test.cc  |  19 +++
 2 files changed, 60 insertions(+), 67 deletions(-)

diff --git a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
index 199e6b7874..d29cc21ab1 100644
--- a/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
+++ b/tensorflow/compiler/xla/service/gpu/ir_emitter_unnested.cc
@@ -145,37 +145,6 @@ void UpdateLaunchDimensions(const LaunchDimensions& launch_dims, Thunk* thunk,
        llvm::ConstantAsMetadata::get(threads_per_block_ir_value)}));
 }
 
-// Tries to get a Slice for the given instruction at the given index, but
-// returns nullopt if we might not know the slice's address at runtime without
-// dereferencing a containing tuple.
-//
-// In particular, when XLA accepts a parameter of tuple type, the caller has the
-// option of telling XLA what are the values inside of the tuple, or just giving
-// XLA a pointer to the top-level tuple and letting us chase the pointers on the
-// GPU.  We therefore cannot rely having these pointers to parameter sub-buffers
-// being present when we run the program.
-optional<BufferAllocation::Slice> GetKnownAtRuntimeSlice(
-    const HloInstruction* instr, const ShapeIndex& index,
-    const BufferAssignment& buffer_assn) {
-  auto maybe_slice = buffer_assn.GetUniqueSlice(instr, index);
-  if (!maybe_slice.ok()) {
-    return nullopt;
-  }
-  // BufferAllocation gives a slice and alloc to every buffer accessed by XLA,
-  // but we don't necessarily know the runtime address of sub-buffers of input
-  // parameters.
-  const BufferAllocation::Slice& slice = maybe_slice.ValueOrDie();
-  const BufferAllocation* alloc = slice.allocation();
-  if (alloc->IsInputOrOutput() && !alloc->maybe_live_out() &&
-      !alloc->param_shape_index().empty()) {
-    return nullopt;
-  }
-
-  // Otherwise, we will know the address of this slice at runtime without having
-  // to dereference a tuple.
-  return slice;
-}
-
 }  // namespace
 
 IrEmitterUnnested::IrEmitterUnnested(const HloModuleConfig& hlo_module_config,
@@ -206,7 +175,7 @@ bool ImplementedAsHostToDeviceMemcpy(const BufferAssignment& buffer_assignment,
   return hlo.opcode() == HloOpcode::kCopy &&
          hlo.operand(0)->opcode() == HloOpcode::kConstant &&
          ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()) &&
-         GetKnownAtRuntimeSlice(&hlo, {}, buffer_assignment).has_value();
+         buffer_assignment.GetUniqueTopLevelSlice(&hlo).ok();
 }
 
 bool ImplementedAsDeviceToDeviceMemcpy(
@@ -216,13 +185,13 @@ bool ImplementedAsDeviceToDeviceMemcpy(
   //
   // 1. `hlo` is a kCopy instruction.
   // 2. `hlo` and its operand have the same shape (thus the same layout too).
-  // 3. The operand to `hlo` has a buffer assignment (constants do not, for
-  //    instance) which means the source buffer also resides on the device.
+  // 3. `hlo` and its operand have a statically-known buffer assignment
+  //     (constants do not, for instance), which means the source buffer also
+  //     resides on the device.
   return hlo.opcode() == HloOpcode::kCopy &&
          ShapeUtil::Equal(hlo.operand(0)->shape(), hlo.shape()) &&
-         GetKnownAtRuntimeSlice(&hlo, {}, buffer_assignment).has_value() &&
-         GetKnownAtRuntimeSlice(hlo.operand(0), {}, buffer_assignment)
-             .has_value();
+         buffer_assignment.GetUniqueTopLevelSlice(&hlo).ok() &&
+         buffer_assignment.GetUniqueTopLevelSlice(hlo.operand(0)).ok();
 }
 }  // namespace
 
@@ -1959,49 +1928,54 @@ GetHloBufferSlices(const HloInstruction* hlo,
       -> optional<std::pair<BufferAllocation::Slice, ShapeIndex>> {
     // Simple, common case: Is the buffer for instr known at runtime?  If so,
     // we're done.
-    auto slice = GetKnownAtRuntimeSlice(instr, index, buffer_assn);
-    if (slice.has_value()) {
-      return {{*slice, ShapeIndex()}};
+    auto slice = buffer_assn.GetUniqueSlice(instr, index);
+    if (slice.ok()) {
+      return {{slice.ValueOrDie(), ShapeIndex()}};
     }
 
-    // If we don't know the buffer for instr at index, see if we know the buffer
-    // for instr at index without its last element.  If so, we can dynamically
-    // find the buffer for instr by dereferencing a pointer in that buffer.
-    // Continue looking this way until we run out of elements in 'index'.
-    ShapeIndex new_index = index;
-    ShapeIndex gte_indices;
-    while (!new_index.empty()) {
-      gte_indices.push_front(new_index.back());
-      new_index.pop_back();
-      auto slice = GetKnownAtRuntimeSlice(instr, new_index, buffer_assn);
-      if (slice.has_value()) {
-        return {{*slice, gte_indices}};
-      }
-    }
-
-    // If *that* didn't work, walk up any bitcasts that we might see.  These
-    // must appear before any GTE instructions, because it's illegal to bitcast
-    // to a tuple type.
+    // If that didn't work, walk up any bitcasts that we might see.  These must
+    // appear before any GTE instructions, because it's illegal to bitcast to a
+    // tuple type.
     const HloInstruction* parent = instr;
     while (parent->opcode() == HloOpcode::kBitcast) {
       parent = parent->operand(0);
 
-      auto slice = GetKnownAtRuntimeSlice(parent, {}, buffer_assn);
-      if (slice.has_value()) {
-        return {{*slice, gte_indices}};
+      auto slice = buffer_assn.GetUniqueSlice(parent, {});
+      if (slice.ok()) {
+        return {{slice.ValueOrDie(), ShapeIndex()}};
       }
     }
 
-    // Finally, check whether instr is a GTE instruction.  If it is, see if we
-    // can get a buffer for its parent, and continue walking up parents until we
-    // find a defined buffer or we hit something that's not a GTE.
+    // Check whether instr is a GTE instruction.  If it is, see if we can get a
+    // buffer for its parent, and continue walking up parents until we find a
+    // defined buffer or we hit something that's not a GTE.
+    ShapeIndex gte_indices;
     while (parent->opcode() == HloOpcode::kGetTupleElement) {
       gte_indices.push_front(parent->tuple_index());
       parent = parent->operand(0);
 
-      auto slice = GetKnownAtRuntimeSlice(parent, {}, buffer_assn);
-      if (slice.has_value()) {
-        return {{*slice, gte_indices}};
+      auto slice = buffer_assn.GetUniqueSlice(parent, {});
+      if (slice.ok()) {
+        return {{slice.ValueOrDie(), gte_indices}};
+      }
+    }
+
+    // Finally, if we don't know the buffer for instr at index, see if we know
+    // the buffer for instr at index without its last element.  If so, we can
+    // dynamically find the buffer for instr by dereferencing a pointer in that
+    // buffer.  Continue looking this way until we run out of elements in
+    // 'index'.
+    //
+    // We can almost always get a buffer without resorting to this.  The only
+    // exception is for cases where the relevant sub-buffer is truly unknowable,
+    // for example the sub-buffer of a tuple-shaped select.
+    ShapeIndex new_index = index;
+    while (!new_index.empty()) {
+      gte_indices.push_front(new_index.back());
+      new_index.pop_back();
+      auto slice = buffer_assn.GetUniqueSlice(instr, new_index);
+      if (slice.ok()) {
+        return {{slice.ValueOrDie(), gte_indices}};
       }
     }
 
diff --git a/tensorflow/compiler/xla/tests/dot_operation_test.cc b/tensorflow/compiler/xla/tests/dot_operation_test.cc
index 09b1dd283e..7b994a4c17 100644
--- a/tensorflow/compiler/xla/tests/dot_operation_test.cc
+++ b/tensorflow/compiler/xla/tests/dot_operation_test.cc
@@ -54,6 +54,25 @@ using TypesF16F32F64CF64 =
 #error "Situation not handled yet"
 #endif
 
+// Check that we can safely pass an input tuple's elements to a dot operation.
+TEST_F(DotOperationTest, DotOfInputTupleElem) {
+  ComputationBuilder builder(client_, TestName());
+
+  ComputationDataHandle param;
+  auto param_data = CreateParameterAndTransferLiteral(
+      0,
+      *Literal::MakeTuple({Literal::CreateR2<float>({{1, 2}, {3, 4}}).get(),
+                           Literal::CreateR2<float>({{5, 6}, {7, 8}}).get()}),
+      "arg0", &builder, &param);
+  auto lhs = builder.GetTupleElement(param, 0);
+  auto rhs = builder.GetTupleElement(param, 1);
+  builder.Dot(lhs, rhs);
+
+  ComputeAndCompareLiteral(&builder,
+                           *Literal::CreateR2<float>({{19, 22}, {43, 50}}),
+                           {param_data.get()});
+}
+
 template <typename T>
 class DotOperationTest_F16F32F64CF64 : public DotOperationTest {};
 TYPED_TEST_CASE(DotOperationTest_F16F32F64CF64, TypesF16F32F64CF64);
-- 
GitLab


From ab1766893951335cd6d3e9b5b51d67d46af889a1 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Thu, 29 Mar 2018 10:40:55 -0700
Subject: [PATCH 679/960] Cache op input's fetched from the C API.

PiperOrigin-RevId: 190951499
---
 tensorflow/python/framework/ops.py | 26 +++++++++++++++++---------
 1 file changed, 17 insertions(+), 9 deletions(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 4b0f3f3a20..6930737a0c 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -1665,6 +1665,9 @@ class Operation(object):
       self._control_inputs_val = control_input_ops
       self._node_def_val = copy.deepcopy(node_def)
       self._op_def_val = op_def
+    else:
+      # This will be set by self.inputs.
+      self._inputs_val = None
 
     self._id_value = self._graph._next_id()  # pylint: disable=protected-access
     self._original_op = original_op
@@ -1938,6 +1941,8 @@ class Operation(object):
       raise TypeError("tensor must be a Tensor: %s" % tensor)
     _assert_same_graph(self, tensor)
     if self._c_op:
+      # Reset cached inputs.
+      self._inputs_val = None
       with errors.raise_exception_on_not_ok_status() as status:
         c_api.UpdateEdge(
             self._graph._c_graph,  # pylint: disable=protected-access
@@ -2054,15 +2059,18 @@ class Operation(object):
   def inputs(self):
     """The list of `Tensor` objects representing the data inputs of this op."""
     if self._c_op:
-      tf_outputs = c_api.GetOperationInputs(self._c_op)
-      # pylint: disable=protected-access
-      retval = [
-          self.graph._get_tensor_by_tf_output(tf_output)
-          for tf_output in tf_outputs
-      ]
-      # pylint: enable=protected-access
-      return Operation._InputList(retval)
-    return Operation._InputList(self._inputs_val)
+      if self._inputs_val is None:
+        tf_outputs = c_api.GetOperationInputs(self._c_op)
+        # pylint: disable=protected-access
+        retval = [
+            self.graph._get_tensor_by_tf_output(tf_output)
+            for tf_output in tf_outputs
+        ]
+        # pylint: enable=protected-access
+        self._inputs_val = Operation._InputList(retval)
+      return self._inputs_val
+    else:
+      return Operation._InputList(self._inputs_val)
 
   @property
   def _inputs(self):
-- 
GitLab


From 9fbb5b3b8fef1caa2ee2ca4a0f8dde900d1f2aa5 Mon Sep 17 00:00:00 2001
From: Patrick Nguyen <drpng@google.com>
Date: Thu, 29 Mar 2018 10:41:36 -0700
Subject: [PATCH 680/960] Allow experimental string attrs for functions.

PiperOrigin-RevId: 190951605
---
 tensorflow/python/framework/function.py      | 6 ++++++
 tensorflow/python/framework/function_test.py | 9 +++++++++
 2 files changed, 15 insertions(+)

diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 14d72d8a3d..82dd2a3356 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -934,6 +934,12 @@ def _parse_kwargs_as_attrs(func_name, **kwargs):
           s=("function_%s" % func_name).encode())
     # pylint: enable=protected-access
 
+  kwargs_keys = list(kwargs.keys())
+  for key in kwargs_keys:
+    if key.startswith("experimental_"):
+      attrs[key] = attr_value_pb2.AttrValue(s=compat.as_bytes(kwargs[key]))
+      del kwargs[key]
+
   if kwargs:
     raise ValueError("Unknown keyword arguments: %s" % kwargs.keys())
   return attrs
diff --git a/tensorflow/python/framework/function_test.py b/tensorflow/python/framework/function_test.py
index 65ca801cbe..83d256fab6 100644
--- a/tensorflow/python/framework/function_test.py
+++ b/tensorflow/python/framework/function_test.py
@@ -1227,6 +1227,15 @@ class FunctionsFromProtos(test.TestCase):
         ValueError, "FunctionDefLibrary contains cyclic gradient functions!"):
       function._from_library(library)
 
+  def testExperimentalAttrs(self):
+
+    @function.Defun(dtypes.int32, experimental_tag="tag_value")
+    def FunctionWithAttr(i):
+      return array_ops.identity(i)
+    self.assertTrue("experimental_tag" in FunctionWithAttr.definition.attr)
+    self.assertEqual(
+        FunctionWithAttr.definition.attr["experimental_tag"].s, b"tag_value")
+
 
 @test_util.with_c_api
 class FunctionOverloadTest(test.TestCase):
-- 
GitLab


From 63dffd5a3bc4e94e74cb140cbf7a68e0e5644ad6 Mon Sep 17 00:00:00 2001
From: Jianwei Xie <xiejw@google.com>
Date: Thu, 29 Mar 2018 10:50:46 -0700
Subject: [PATCH 681/960] Automated g4 rollback of changelist 190858242

PiperOrigin-RevId: 190953197
---
 RELEASE.md                                    |  60 +++
 configure.py                                  |   2 +-
 tensorflow/BUILD                              |   7 +
 tensorflow/contrib/BUILD                      |  27 +-
 .../boosted_trees/kernels/quantile_ops.cc     |   2 +-
 .../boosted_trees/lib/utils/batch_features.cc |   2 +-
 .../lib/utils/batch_features_test.cc          |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.cc  |   2 +-
 .../boosted_trees/lib/utils/dropout_utils.h   |   2 +-
 .../lib/utils/sparse_column_iterable_test.cc  |   2 +-
 .../boosted_trees/proto/tree_config.proto     |   2 +-
 .../kernel_tests/prediction_ops_test.py       |  10 +-
 .../python/kernel_tests/quantile_ops_test.py  |   2 +-
 .../boosted_trees/python/ops/quantile_ops.py  |   2 +-
 tensorflow/contrib/cmake/tf_tests.cmake       |   3 +
 .../kernel_tests/batch_dataset_op_test.py     |  14 +
 tensorflow/contrib/eager/python/BUILD         |   6 +-
 .../eager/python/examples/spinn/spinn_test.py |   1 -
 .../python/estimator/replicate_model_fn.py    |   2 +-
 .../factorization/kernels/clustering_ops.cc   |   2 +-
 .../python/ops/factorization_ops.py           |  14 +-
 .../python/ops/factorization_ops_test.py      |  12 +-
 .../factorization/python/ops/gmm_ops.py       |   4 +-
 .../factorization/python/ops/gmm_test.py      |   2 +-
 .../factorization/python/ops/kmeans_test.py   |   4 +-
 .../contrib/factorization/python/ops/wals.py  |   2 +-
 tensorflow/contrib/learn/BUILD                |   1 +
 .../learn/python/learn/estimators/linear.py   |   4 +-
 .../linear_optimizer/python/sdca_estimator.py |   4 +-
 tensorflow/contrib/lite/README.md             |   3 +
 tensorflow/contrib/lite/builtin_ops.h         |   1 +
 tensorflow/contrib/lite/g3doc/models.md       |   2 +-
 tensorflow/contrib/lite/kernels/BUILD         |  13 +
 .../internal/reference/reference_ops.h        |  25 ++
 tensorflow/contrib/lite/kernels/maximum.cc    | 106 +++++
 .../contrib/lite/kernels/maximum_test.cc      |  81 ++++
 tensorflow/contrib/lite/kernels/register.cc   |   2 +
 tensorflow/contrib/lite/model.cc              |   3 +
 tensorflow/contrib/lite/nnapi_delegate.cc     |   1 +
 tensorflow/contrib/lite/python/lite.py        |  22 +-
 tensorflow/contrib/lite/schema/schema.fbs     |   5 +
 .../contrib/lite/schema/schema_generated.h    | 124 +++++-
 tensorflow/contrib/lite/testing/BUILD         |   1 +
 .../contrib/lite/testing/generate_examples.py |  36 ++
 .../testing/generated_examples_zip_test.cc    |   1 +
 .../contrib/lite/toco/tflite/operator.cc      |   2 +
 .../contrib/lite/toco/tflite/operator_test.cc |   2 +
 tensorflow/contrib/lookup/lookup_ops.py       |   2 +-
 .../contrib/makefile/download_dependencies.sh |   2 +-
 tensorflow/contrib/makefile/tf_op_files.txt   |   1 +
 .../seq2seq/kernels/beam_search_ops.cc        |   2 +-
 .../seq2seq/python/ops/attention_wrapper.py   |   8 +-
 .../seq2seq/python/ops/beam_search_decoder.py |   6 +-
 .../slim/python/slim/data/parallel_reader.py  |   4 +-
 .../slim/python/slim/data/prefetch_queue.py   |   4 +-
 .../python/slim/data/tfexample_decoder.py     |   2 +-
 tensorflow/contrib/tensorrt/README.md         |  46 ++-
 .../contrib/tensorrt/convert/convert_graph.cc |  20 +-
 .../contrib/tensorrt/convert/convert_nodes.cc | 375 ++++++++++--------
 .../contrib/tensorrt/segment/segment.cc       |  55 ++-
 tensorflow/contrib/tensorrt/segment/segment.h |   4 +-
 .../contrib/tensorrt/segment/segment_test.cc  |   8 +-
 .../timeseries/python/timeseries/ar_model.py  |   2 +-
 .../python/timeseries/math_utils.py           |   2 +-
 .../timeseries/state_space_models/varma.py    |   4 +-
 .../base_api/api_def_MatrixSolveLs.pbtxt      |   6 +-
 .../core/common_runtime/mkl_cpu_allocator.cc  |   3 -
 tensorflow/core/framework/common_shape_fns.cc |   4 +-
 tensorflow/core/framework/common_shape_fns.h  |   8 +-
 tensorflow/core/framework/shape_inference.h   |   1 +
 .../core/kernels/mkl_fused_batch_norm_op.cc   |   2 +-
 .../core/kernels/segment_reduction_ops.h      |   7 +
 tensorflow/core/kernels/snapshot_op.cc        |  30 ++
 tensorflow/core/kernels/snapshot_op.h         |  26 +-
 tensorflow/core/kernels/snapshot_op_gpu.cu.cc |   9 +-
 tensorflow/core/kernels/xent_op.cc            |  65 ++-
 tensorflow/core/kernels/xent_op.h             |  35 +-
 tensorflow/core/kernels/xent_op_gpu.cu.cc     |   9 +-
 tensorflow/core/ops/array_ops.cc              |  26 +-
 tensorflow/core/ops/nn_ops.cc                 |  23 +-
 tensorflow/core/ops/nn_ops_test.cc            |  16 +-
 tensorflow/core/public/version.h              |   4 +-
 .../python/contrib.bayesflow.monte_carlo.md   |  36 +-
 .../api_guides/python/contrib.losses.md       |  28 +-
 .../docs_src/community/documentation.md       |  38 +-
 tensorflow/docs_src/install/install_c.md      |   2 +-
 tensorflow/docs_src/install/install_go.md     |   2 +-
 tensorflow/docs_src/install/install_java.md   |  22 +-
 tensorflow/docs_src/install/install_linux.md  |  22 +-
 tensorflow/docs_src/install/install_mac.md    |  14 +-
 .../docs_src/install/install_sources.md       |   9 +-
 tensorflow/docs_src/mobile/optimizing.md      |   2 +
 tensorflow/docs_src/mobile/prepare_models.md  |   2 +-
 tensorflow/python/BUILD                       |   2 +-
 .../python/kernel_tests/array_ops_test.py     |  26 +-
 tensorflow/python/kernel_tests/testdata/BUILD |   2 +-
 .../python/kernel_tests/xent_op_test.py       |  81 +++-
 tensorflow/python/layers/convolutional.py     |   2 +
 .../python/layers/convolutional_test.py       |   6 +
 tensorflow/python/ops/linalg_ops.py           |   2 +-
 .../python/training/monitored_session.py      |  33 +-
 .../python/training/monitored_session_test.py |  36 ++
 tensorflow/tensorflow.bzl                     |   4 +-
 .../tools/api/golden/tensorflow.train.pbtxt   |   2 +-
 .../tools/ci_build/osx/libtensorflow_cpu.sh   |   2 +-
 tensorflow/tools/docker/Dockerfile.devel      |   2 +-
 .../tools/docker/Dockerfile.devel-cpu-mkl     |   2 +-
 tensorflow/tools/docker/Dockerfile.devel-gpu  |   2 +-
 tensorflow/tools/lib_package/BUILD            |   2 -
 tensorflow/tools/pip_package/BUILD            |   1 -
 tensorflow/tools/pip_package/setup.py         |   6 +-
 tensorflow/workspace.bzl                      | 133 ++++---
 third_party/mkl/BUILD                         |  46 ++-
 third_party/mkl/MKL_LICENSE                   | 201 ++++++++++
 third_party/mkl/build_defs.bzl                |  12 +
 third_party/mkl/mkl.BUILD                     |  27 +-
 116 files changed, 1703 insertions(+), 556 deletions(-)
 create mode 100644 tensorflow/contrib/lite/kernels/maximum.cc
 create mode 100644 tensorflow/contrib/lite/kernels/maximum_test.cc
 create mode 100644 third_party/mkl/MKL_LICENSE

diff --git a/RELEASE.md b/RELEASE.md
index 6f54dee58f..c63d9f20c9 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,3 +1,63 @@
+# Release 1.7.0
+
+## Major Features And Improvements
+* Eager mode is moving out of contrib, try `tf.enable_eager_execution()`.
+* Graph rewrites emulating fixed-point quantization compatible with TensorFlow Lite, supported by new `tf.contrib.quantize` package.
+* Easily customize gradient computation with `tf.custom_gradient`.
+* [TensorBoard Debugger Plugin](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md), the graphical user interface (GUI) of TensorFlow Debugger (tfdbg), is now in alpha.
+* Experimental support for reading a sqlite database as a `Dataset` with new `tf.contrib.data.SqlDataset`.
+* Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`.
+* Better text processing with `tf.regex_replace`.
+* Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length`
+
+## Bug Fixes and Other Changes
+* Accelerated Linear Algebra (XLA):
+  * Add `MaxPoolGradGrad` support for XLA
+  * CSE pass from Tensorflow is now disabled in XLA.
+* `tf.data`:
+  * `tf.data.Dataset`
+    * Add support for building C++ Dataset op kernels as external libraries, using the `tf.load_op_library()` mechanism.
+    * `Dataset.list_files()` now shuffles its output by default.
+    * `Dataset.shuffle(..., seed=tf.constant(0, dtype=tf.int64))` now yields the same sequence of elements as `Dataset.shuffle(..., seed=0)`.
+  * Add `num_parallel_reads` argument to `tf.data.TFRecordDataset`.
+* `tf.contrib`:
+  * `tf.contrib.bayesflow.halton_sequence` now supports randomization.
+  * Add support for scalars in `tf.contrib.all_reduce`.
+  * Add `effective_sample_size` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `potential_scale_reduction` to `tf.contrib.bayesflow.mcmc_diagnostics`.
+  * Add `BatchNormalization`, `Kumaraswamy` bijectors.
+  * Deprecate `tf.contrib.learn`. Please check contrib/learn/README.md for instructions on how to convert existing code.
+  * `tf.contrib.data`
+    * Remove deprecated `tf.contrib.data.Dataset`, `tf.contrib.data.Iterator`, `tf.contrib.data.FixedLengthRecordDataset`, `tf.contrib.data.TextLineDataset`, and `tf.contrib.data.TFRecordDataset` classes.
+    * Added `bucket_by_sequence_length`, `sliding_window_batch`, and `make_batched_features_dataset`
+  * Remove unmaintained `tf.contrib.ndlstm`. You can find it externally at https://github.com/tmbarchive/tfndlstm.
+  * Moved most of `tf.contrib.bayesflow` to its own repo: `tfp`
+* Other:
+  * tf.py_func now reports the full stack trace if an exception occurs.
+  * Integrate `TPUClusterResolver` with GKE's integration for Cloud TPUs.
+  * Add a library for statistical testing of samplers.
+  * Add Helpers to stream data from the GCE VM to a Cloud TPU.
+  * Integrate ClusterResolvers with TPUEstimator.
+  * Unify metropolis_hastings interface with HMC kernel.
+  * Move LIBXSMM convolutions to a separate --define flag so that they are disabled by default.
+  * Fix `MomentumOptimizer` lambda.
+  * Reduce `tfp.layers` boilerplate via programmable docstrings.
+  * Add `auc_with_confidence_intervals`, a method for computing the AUC and confidence interval with linearithmic time complexity.
+  * `regression_head` now accepts customized link function, to satisfy the usage that user can define their own link function if the `array_ops.identity` does not meet the requirement.
+  * Fix `initialized_value` and `initial_value` behaviors for `ResourceVariables` created from `VariableDef` protos.
+  * Add TensorSpec to represent the specification of Tensors.
+  * Constant folding pass is now deterministic.
+  * Support `float16` `dtype` in `tf.linalg.*`.
+  * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions.
+
+## Thanks to our Contributors
+
+This release contains contributions from many people at Google, as well as:
+
+4d55397500, Abe, Alistair Low, Andy Kernahan, Appledore, Ben, Ben Barsdell, Boris Pfahringer, Brad Wannow, Brett Koonce, Carl Thomé, cclauss, Chengzhi Chen, Chris Drake, Christopher Yeh, Clayne Robison, Codrut Grosu, Daniel Trebbien, Danny Goodman, David Goodwin, David Norman, Deron Eriksson, Donggeon Lim, Donny Viszneki, DosLin, DylanDmitri, Francisco Guerrero, Fred Reiss, gdh1995, Giuseppe, Glenn Weidner, gracehoney, Guozhong Zhuang, Haichen "Hc" Li, Harald Husum, harumitsu.nobuta, Henry Spivey, hsm207, Jekyll Song, Jerome, Jiongyan Zhang, jjsjann123, John Sungjin Park, Johnson145, JoshVarty, Julian Wolff, Jun Wang, June-One, Kamil Sindi, Kb Sriram, Kdavis-Mozilla, Kenji, lazypanda1, Liang-Chi Hsieh, Loo Rong Jie, Mahesh Bhosale, MandarJKulkarni, ManHyuk, Marcus Ong, Marshal Hayes, Martin Pool, matthieudelaro, mdfaijul, mholzel, Michael Zhou, Ming Li, Minmin Sun, Myungjoo Ham, MyungsungKwak, Naman Kamra, Peng Yu, Penghao Cen, Phil, Raghuraman-K, resec, Rohin Mohanadas, Sandeep N Gupta, Scott Tseng, seaotterman, Seo Sanghyeon, Sergei Lebedev, Ted Chang, terrytangyuan, Tim H, tkunic, Tod, vihanjain, Yan Facai (颜发才), Yin Li, Yong Tang, Yukun Chen, Yusuke Yamada
+
+
+
 # Release 1.6.0
 
 ## Breaking Changes
diff --git a/configure.py b/configure.py
index 22b9abedd7..0f52c0ec99 100644
--- a/configure.py
+++ b/configure.py
@@ -1414,7 +1414,7 @@ def main():
   set_build_var(environ_cp, 'TF_NEED_S3', 'Amazon S3 File System',
                 'with_s3_support', True, 's3')
   set_build_var(environ_cp, 'TF_NEED_KAFKA', 'Apache Kafka Platform',
-                'with_kafka_support', False, 'kafka')
+                'with_kafka_support', True, 'kafka')
   set_build_var(environ_cp, 'TF_ENABLE_XLA', 'XLA JIT', 'with_xla_support',
                 False, 'xla')
   set_build_var(environ_cp, 'TF_NEED_GDR', 'GDR', 'with_gdr_support',
diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 0021b657d8..3d5737a9d7 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -240,6 +240,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "with_kafka_support_windows_override",
+    define_values = {"with_kafka_support": "true"},
+    values = {"cpu": "x64_windows"},
+    visibility = ["//visibility:public"],
+)
+
 config_setting(
     name = "with_gcp_support_android_override",
     define_values = {"with_gcp_support": "true"},
diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 1ca70e7122..c211ad8b9b 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -51,7 +51,6 @@ py_library(
         "//tensorflow/contrib/image:single_image_random_dot_stereograms_py",
         "//tensorflow/contrib/input_pipeline:input_pipeline_py",
         "//tensorflow/contrib/integrate:integrate_py",
-        "//tensorflow/contrib/kafka",
         "//tensorflow/contrib/keras",
         "//tensorflow/contrib/kernel_methods",
         "//tensorflow/contrib/kfac",
@@ -110,7 +109,13 @@ py_library(
         "//tensorflow/python:util",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -120,7 +125,6 @@ cc_library(
         "//tensorflow/contrib/boosted_trees:boosted_trees_kernels",
         "//tensorflow/contrib/coder:all_kernels",
         "//tensorflow/contrib/data/kernels:dataset_kernels",
-        "//tensorflow/contrib/kafka:dataset_kernels",
         "//tensorflow/contrib/factorization/kernels:all_kernels",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_kernels",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_kernel",
@@ -133,7 +137,13 @@ cc_library(
         "//tensorflow/contrib/text:all_kernels",
     ] + if_mpi(["//tensorflow/contrib/mpi_collectives:mpi_collectives_py"]) + if_cuda([
         "//tensorflow/contrib/nccl:nccl_kernels",
-    ]),
+    ]) + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_kernels",
+        ],
+        "//conditions:default": [],
+    }),
 )
 
 cc_library(
@@ -146,7 +156,6 @@ cc_library(
         "//tensorflow/contrib/factorization:all_ops",
         "//tensorflow/contrib/framework:all_ops",
         "//tensorflow/contrib/input_pipeline:input_pipeline_ops_op_lib",
-        "//tensorflow/contrib/kafka:dataset_ops_op_lib",
         "//tensorflow/contrib/layers:sparse_feature_cross_op_op_lib",
         "//tensorflow/contrib/nccl:nccl_ops_op_lib",
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_ops_op_lib",
@@ -157,5 +166,11 @@ cc_library(
         "//tensorflow/contrib/tensor_forest:tensor_forest_ops_op_lib",
         "//tensorflow/contrib/text:all_ops",
         "//tensorflow/contrib/tpu:all_ops",
-    ],
+    ] + select({
+        "//tensorflow:with_kafka_support_windows_override": [],
+        "//tensorflow:with_kafka_support": [
+            "//tensorflow/contrib/kafka:dataset_ops_op_lib",
+        ],
+        "//conditions:default": [],
+    }),
 )
diff --git a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
index 0f4c2298f5..0b28f81e7c 100644
--- a/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
+++ b/tensorflow/contrib/boosted_trees/kernels/quantile_ops.cc
@@ -253,7 +253,7 @@ class CreateQuantileAccumulatorOp : public OpKernel {
  private:
   float epsilon_;
   int32 num_quantiles_;
-  // An upperbound on the number of enteries that the summaries might have
+  // An upper bound on the number of entries that the summaries might have
   // for a feature.
   int64 max_elements_;
   bool generate_quantiles_;
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
index cf4f9a097a..35b059f349 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.cc
@@ -54,7 +54,7 @@ Status BatchFeatures::Initialize(
     TF_CHECK_AND_RETURN_IF_ERROR(
         dense_float_feature.dim_size(1) == 1,
         errors::InvalidArgument(
-            "Dense float features may not be multi-valent: dim_size(1) = ",
+            "Dense float features may not be multivalent: dim_size(1) = ",
             dense_float_feature.dim_size(1)));
     dense_float_feature_columns_.emplace_back(dense_float_feature);
   }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
index 609519e8b1..cfe9101e74 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features_test.cc
@@ -59,7 +59,7 @@ TEST_F(BatchFeaturesTest, DenseFloatFeatures_Multivalent) {
   BatchFeatures batch_features(1);
   auto dense_vec = AsTensor<float>({3.0f, 7.0f}, {1, 2});
   auto expected_error = InvalidArgument(
-      "Dense float features may not be multi-valent: dim_size(1) = 2");
+      "Dense float features may not be multivalent: dim_size(1) = 2");
   EXPECT_EQ(expected_error,
             batch_features.Initialize({dense_vec}, {}, {}, {}, {}, {}, {}));
 }
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
index db34db998a..ce67db797d 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.cc
@@ -54,7 +54,7 @@ Status DropoutUtils::DropOutTrees(
   if (probability_of_skipping_dropout < 0 ||
       probability_of_skipping_dropout > 1) {
     return errors::InvalidArgument(
-        "Probability of skiping dropout must be in [0,1] range");
+        "Probability of skipping dropout must be in [0,1] range");
   }
   const auto num_trees = weights.size();
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
index 928bfbfe5c..77c16da541 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/dropout_utils.h
@@ -66,7 +66,7 @@ class DropoutUtils {
       // Current weights and num_updates will be updated as a result of this
       // func
       std::vector<float>* current_weights,
-      // How many weight assignements have been done for each tree already.
+      // How many weight assignments have been done for each tree already.
       std::vector<int32>* num_updates);
 };
 
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
index 0138aae3db..cc7604745e 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
+++ b/tensorflow/contrib/boosted_trees/lib/utils/sparse_column_iterable_test.cc
@@ -34,7 +34,7 @@ TEST_F(SparseColumnIterableTest, Empty) {
 }
 
 TEST_F(SparseColumnIterableTest, Iterate) {
-  // 8 examples having 7 sparse features with the 3rd and 7th multi-valent.
+  // 8 examples having 7 sparse features with the 3rd and 7th multivalent.
   // This can be visualized like the following:
   // Instance | Sparse |
   // 0        |  x     |
diff --git a/tensorflow/contrib/boosted_trees/proto/tree_config.proto b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
index 4407c4d981..81411aa84a 100644
--- a/tensorflow/contrib/boosted_trees/proto/tree_config.proto
+++ b/tensorflow/contrib/boosted_trees/proto/tree_config.proto
@@ -53,7 +53,7 @@ message DenseFloatBinarySplit {
   // Float feature column and split threshold describing
   // the rule feature <= threshold.
   int32 feature_column = 1;
-  // If feature column is multivalent, this holds the index of the dimensiong
+  // If feature column is multivalent, this holds the index of the dimension
   // for the split. Defaults to 0.
   int32 dimension_id = 5;
   float threshold = 2;
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
index c1acf35160..cf55759aaa 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/prediction_ops_test.py
@@ -120,8 +120,8 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     """Sets up the prediction tests.
 
     Create a batch of two examples having one dense float, two sparse float
-    single valued, one sparse float multidimensionl and one sparse int features.
-    The data looks like the following:
+    single valued, one sparse float multidimensional and one sparse int
+    features.  The data looks like the following:
     | Instance | Dense0 | SparseF0 | SparseF1 | SparseI0 | SparseM
     | 0        |  7     |    -3    |          |    9,1   | __, 5.0
     | 1        | -2     |          | 4        |          |  3, ___
@@ -810,7 +810,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
     # building. This tree should never be dropped.
     num_trees = 10
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 10 trees with some weights.
       for i in range(0, num_trees):
@@ -951,7 +951,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testDropOutZeroProb(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       # Add 1000 trees with some weights.
       for i in range(0, 999):
@@ -994,7 +994,7 @@ class PredictionOpsTest(test_util.TensorFlowTestCase):
 
   def testAveragingAllTrees(self):
     with self.test_session():
-      # Empty tree ensenble.
+      # Empty tree ensemble.
       tree_ensemble_config = tree_config_pb2.DecisionTreeEnsembleConfig()
       adjusted_tree_ensemble_config = (
           tree_config_pb2.DecisionTreeEnsembleConfig())
diff --git a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
index 81f58de28c..074623699d 100644
--- a/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
+++ b/tensorflow/contrib/boosted_trees/python/kernel_tests/quantile_ops_test.py
@@ -482,7 +482,7 @@ class QuantilesOpTest(test_util.TensorFlowTestCase):
     """Sets up the quantile op tests.
 
     Create a batch of 4 examples having 2 dense and 4 sparse features.
-    Forth sparse feature is multivalent (3 dimensional)
+    Fourth sparse feature is multivalent (3 dimensional)
     The data looks like this
     | Instance | Dense 0 | Dense 1 | Sparse 0 | Sparse 1 |Sparse 2| SparseM
     | 0        |   -0.1  |  -1     |   -2     |   0.1    |        |_ ,1,_
diff --git a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
index 97d57e8b23..1b184d296b 100644
--- a/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
+++ b/tensorflow/contrib/boosted_trees/python/ops/quantile_ops.py
@@ -184,7 +184,7 @@ class QuantileAccumulator(saver.BaseSaverBuilder.SaveableObject):
     """Finalizes quantile summary stream and resets it for next iteration.
 
     Args:
-      stamp_token: Exepcted current token.
+      stamp_token: Expected current token.
       next_stamp_token: Next value for the token.
     Returns:
       A list of quantiles or approximate boundaries.
diff --git a/tensorflow/contrib/cmake/tf_tests.cmake b/tensorflow/contrib/cmake/tf_tests.cmake
index f793877c8b..92f2ab6dea 100644
--- a/tensorflow/contrib/cmake/tf_tests.cmake
+++ b/tensorflow/contrib/cmake/tf_tests.cmake
@@ -210,6 +210,9 @@ if (tensorflow_BUILD_PYTHON_TESTS)
     "${tensorflow_source_dir}/tensorflow/contrib/learn/python/learn/learn_io/graph_io_test.py"
     # Test is flaky on Windows GPU builds (b/38283730).
     "${tensorflow_source_dir}/tensorflow/contrib/factorization/python/ops/gmm_test.py"
+    # Disable following manual tag in BUILD.
+    "${tensorflow_source_dir}/tensorflow/python/keras/_impl/keras/layers/convolutional_test.py"
+
   )
   if (WIN32)
     set(tf_test_src_py_exclude
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 5abb38c2d2..75482f67da 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -413,6 +413,20 @@ class BatchDatasetTest(test.TestCase):
   def testMapAndBatchPartialBatchDropRemainder(self):
     return self._testMapAndBatchPartialBatchHelper(drop_remainder=True)
 
+  def testMapAndBatchYieldsPartialBatch(self):
+    iterator = (dataset_ops.Dataset.range(10)
+                .apply(batching.map_and_batch(
+                    lambda x: array_ops.reshape(x * x, [1]), 4))
+                .make_one_shot_iterator())
+    self.assertEqual([None, 1], iterator.output_shapes.as_list())
+    next_element = iterator.get_next()
+    with self.test_session() as sess:
+      self.assertAllEqual([[0], [1], [4], [9]], sess.run(next_element))
+      self.assertAllEqual([[16], [25], [36], [49]], sess.run(next_element))
+      self.assertAllEqual([[64], [81]], sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testMapAndBatchSparse(self):
 
     def _sparse(i):
diff --git a/tensorflow/contrib/eager/python/BUILD b/tensorflow/contrib/eager/python/BUILD
index 48372d7ae0..edb9130266 100644
--- a/tensorflow/contrib/eager/python/BUILD
+++ b/tensorflow/contrib/eager/python/BUILD
@@ -271,5 +271,9 @@ cuda_py_test(
         "//tensorflow/python/eager:test",
         "//tensorflow/python/keras",
     ],
-    tags = ["notsan"],
+    tags = [
+        "no_oss",  # b/74395663
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
+    ],
 )
diff --git a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
index 9261823d77..9adf47d505 100644
--- a/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
+++ b/tensorflow/contrib/eager/python/examples/spinn/spinn_test.py
@@ -418,7 +418,6 @@ class SpinnTest(test_util.TensorFlowTestCase):
                     if event.summary.value
                     and event.summary.value[0].tag == "train/loss"]
     self.assertEqual(config.epochs, len(train_losses))
-    self.assertLess(train_losses[-1], train_losses[0])
 
     # 5. Verify that checkpoints exist and contains all the expected variables.
     self.assertTrue(glob.glob(os.path.join(config.logdir, "ckpt*")))
diff --git a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
index e0fae2c992..fa2697800e 100644
--- a/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
+++ b/tensorflow/contrib/estimator/python/estimator/replicate_model_fn.py
@@ -136,7 +136,7 @@ def replicate_model_fn(model_fn,
       the train_op argument of `EstimatorSpec`.
     loss_reduction: controls whether losses are summed or averaged.
     devices: Optional list of devices to replicate the model across.  This
-      argument can be used to replice only on the subset of available GPUs.
+      argument can be used to replicate only on the subset of available GPUs.
       If `None`, then all available GPUs are going to be used for replication.
       If no GPUs are available, then the model is going to be placed on the CPU.
 
diff --git a/tensorflow/contrib/factorization/kernels/clustering_ops.cc b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
index dd61f59585..2a6c97e8b9 100644
--- a/tensorflow/contrib/factorization/kernels/clustering_ops.cc
+++ b/tensorflow/contrib/factorization/kernels/clustering_ops.cc
@@ -353,7 +353,7 @@ class NearestNeighborsOp : public OpKernel {
     auto worker_threads = *(context->device()->tensorflow_cpu_worker_threads());
     const int64 num_threads = worker_threads.num_threads;
     // This kernel might be configured to use fewer than the total number of
-    // available CPUs on the host machine. To avoid descructive interference
+    // available CPUs on the host machine. To avoid destructive interference
     // with other jobs running on the host machine, we must only use a fraction
     // of total available L3 cache. Unfortunately, we cannot query the host
     // machine to get the number of physical CPUs. So, we use a fixed per-CPU
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops.py b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
index 054888e734..8e0ed1d80e 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops.py
@@ -106,7 +106,7 @@ class WALSModel(object):
       # the prep_gramian_op for row(column) can be run.
       worker_init_op = model.worker_init
 
-      # To be run once per interation sweep before the row(column) update
+      # To be run once per integration sweep before the row(column) update
       # initialize ops can be run. Note that in the distributed training
       # situations, this should only be run by the chief trainer. All other
       # trainers need to block until this is done.
@@ -118,9 +118,9 @@ class WALSModel(object):
       init_row_update_op = model.initialize_row_update_op
       init_col_update_op = model.initialize_col_update_op
 
-      # Ops to upate row(column). This can either take the entire sparse tensor
-      # or slices of sparse tensor. For distributed trainer, each trainer
-      # handles just part of the matrix.
+      # Ops to update row(column). This can either take the entire sparse
+      # tensor or slices of sparse tensor. For distributed trainer, each
+      # trainer handles just part of the matrix.
       _, row_update_op, unreg_row_loss, row_reg, _ = model.update_row_factors(
            sp_input=matrix_slices_from_queue_for_worker_shard)
       row_loss = unreg_row_loss + row_reg
@@ -220,7 +220,7 @@ class WALSModel(object):
         in the form of [[w_0, w_1, ...], [w_k, ... ], [...]], with the number of
         inner lists matching the number of row factor shards and the elements in
         each inner list are the weights for the rows of the corresponding row
-        factor shard. In this case,  w_ij = unonbserved_weight +
+        factor shard. In this case,  w_ij = unobserved_weight +
                                             row_weights[i] * col_weights[j].
         - If this is a single non-negative real number, this value is used for
         all row weights and w_ij = unobserved_weight + row_weights *
@@ -435,7 +435,7 @@ class WALSModel(object):
       gramian: Variable storing the gramian calculated from the factors.
 
     Returns:
-      A op that updates the gramian with the calcuated value from the factors.
+      A op that updates the gramian with the calculated value from the factors.
     """
     partial_gramians = []
     for f in factors:
@@ -564,7 +564,7 @@ class WALSModel(object):
 
     Note that specifically this initializes the cache of the row and column
     weights on workers when `use_factors_weights_cache` is True. In this case,
-    if these weights are being calcualted and reset after the object is created,
+    if these weights are being calculated and reset after the object is created,
     it is important to ensure this ops is run afterwards so the cache reflects
     the correct values.
     """
diff --git a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
index c813733915..bb5140aeb3 100644
--- a/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
+++ b/tensorflow/contrib/factorization/python/ops/factorization_ops_test.py
@@ -210,7 +210,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -283,8 +283,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 3 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
-      # weights and feature vectors are identical to that used in model
+      # This is expected to reproduce the same column factors in the model as
+      # the weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
@@ -385,7 +385,7 @@ class WalsModelTest(test.TestCase):
 
       # Test row projection.
       # Using the specified projection weights for the 2 row feature vectors.
-      # This is expected to reprodue the same row factors in the model as the
+      # This is expected to reproduce the same row factors in the model as the
       # weights and feature vectors are identical to that used in model
       # training.
       projected_rows = wals_model.project_row_factors(
@@ -462,8 +462,8 @@ class WalsModelTest(test.TestCase):
 
       # Test column projection.
       # Using the specified projection weights for the 2 column feature vectors.
-      # This is expected to reprodue the same column factors in the model as the
-      # weights and feature vectors are identical to that used in model
+      # This is expected to reproduce the same column factors in the model as
+      # the weights and feature vectors are identical to that used in model
       # training.
       projected_cols = wals_model.project_col_factors(
           sp_input=sp_feeder,
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index 98d6434f47..14d4c733e3 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -280,7 +280,7 @@ class GmmAlgorithm(object):
     self._define_score_samples()
 
   def _define_full_covariance_probs(self, shard_id, shard):
-    """Defines the full covariance probabilties per example in a class.
+    """Defines the full covariance probabilities per example in a class.
 
     Updates a matrix with dimension num_examples X num_classes.
 
@@ -344,7 +344,7 @@ class GmmAlgorithm(object):
   def _define_prior_log_prob_operation(self, shard_id):
     """Computes the prior probability of all samples.
 
-    Updates a vector where each item is the prior probabibility of an
+    Updates a vector where each item is the prior probability of an
     input example.
 
     Args:
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_test.py b/tensorflow/contrib/factorization/python/ops/gmm_test.py
index 00a4734eb6..4fc9c96e9d 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_test.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_test.py
@@ -210,7 +210,7 @@ class GMMTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans_test.py b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
index 0103cc4439..88eb9cf692 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans_test.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans_test.py
@@ -413,7 +413,7 @@ class KMeansCosineDistanceTest(KMeansTestBase):
     self.assertAllClose(score, self.true_score, atol=1e-2)
 
   def test_predict_kmeans_plus_plus(self):
-    # Most points are concetrated near one center. KMeans++ is likely to find
+    # Most points are concentrated near one center. KMeans++ is likely to find
     # the less populated centers.
     points = np.array(
         [[2.5, 3.5], [2.5, 3.5], [-2, 3], [-2, 3], [-3, -3], [-3.1, -3.2],
@@ -604,7 +604,7 @@ class KMeansTestQueues(test.TestCase):
     return _fn
 
   # This test makes sure that there are no deadlocks when using a QueueRunner.
-  # Note that since cluster initialization is dependendent on inputs, if input
+  # Note that since cluster initialization is dependent on inputs, if input
   # is generated using a QueueRunner, one has to make sure that these runners
   # are started before the initialization.
   def test_queues(self):
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 4fe22ea26e..62db3bb4c4 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -235,7 +235,7 @@ def _wals_factorization_model_function(features, labels, mode, params):
         num_items: An integer, the total number of items of this axis.
         update_fn: A function that takes one argument (`sp_input`), and that
         returns a tuple of
-          * new_factors: A flot Tensor of the factor values after update.
+          * new_factors: A float Tensor of the factor values after update.
           * update_op: a TensorFlow op which updates the factors.
           * loss: A float Tensor, the unregularized loss.
           * reg_loss: A float Tensor, the regularization loss.
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index 924918be4f..ba55365c14 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -226,6 +226,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/monitors_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_pip_gpu"],  # b/74437598
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
diff --git a/tensorflow/contrib/learn/python/learn/estimators/linear.py b/tensorflow/contrib/learn/python/learn/estimators/linear.py
index 64d7ecc68e..70b70af98c 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/linear.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/linear.py
@@ -243,8 +243,8 @@ def sdca_model_fn(features, labels, mode, params):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(
-      features.values(), parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
index 05794a42c5..d4e54c82f9 100644
--- a/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
+++ b/tensorflow/contrib/linear_optimizer/python/sdca_estimator.py
@@ -140,8 +140,8 @@ def sdca_model_fn(features, labels, mode, params, config=None):
 
   parent_scope = "linear"
 
-  with variable_scope.variable_op_scope(features.values(),
-                                        parent_scope) as scope:
+  with variable_scope.variable_scope(
+      values=features.values(), name_or_scope=parent_scope) as scope:
     features = features.copy()
     features.update(layers.transform_features(features, feature_columns))
     logits, columns_to_variables, bias = (
diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index 2680d515eb..c15ae3f233 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -126,6 +126,9 @@ The above pre-trained models have been trained on the ImageNet data set, which c
 
 The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
 
+# Getting started with RaspberryPi
+
+Using RaspberryPi can be accomplished by following the [Makefile instructions](g3doc/rpi.md). That will give a you a static library (.a) that you can build your app against. Python bindings will be coming soon as well as a demo app.
 
 ### Train a custom model
 A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
diff --git a/tensorflow/contrib/lite/builtin_ops.h b/tensorflow/contrib/lite/builtin_ops.h
index d7993e60cc..17b791e4e2 100644
--- a/tensorflow/contrib/lite/builtin_ops.h
+++ b/tensorflow/contrib/lite/builtin_ops.h
@@ -79,6 +79,7 @@ typedef enum {
   kTfLiteBuiltinBidirectionalSequenceLstm = 52,
   kTfLiteBuiltinCast = 53,
   kTfLiteBuiltinPrelu = 54,
+  kTfLiteBuiltinMaximum = 55,
 } TfLiteBuiltinOperator;
 
 #ifdef __cplusplus
diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 5b393140d6..48f43d4fc4 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,4 +1,4 @@
-#List of Hosted Models
+# List of Hosted Models
 
 *   [Inception V3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
 *   [Inception V3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
diff --git a/tensorflow/contrib/lite/kernels/BUILD b/tensorflow/contrib/lite/kernels/BUILD
index 058f995d75..df0f3cbeb0 100644
--- a/tensorflow/contrib/lite/kernels/BUILD
+++ b/tensorflow/contrib/lite/kernels/BUILD
@@ -156,6 +156,7 @@ cc_library(
         "local_response_norm.cc",
         "lsh_projection.cc",
         "lstm.cc",
+        "maximum.cc",
         "mean.cc",
         "mfcc.cc",
         "mul.cc",
@@ -536,6 +537,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "maximum_test",
+    size = "small",
+    srcs = ["maximum_test.cc"],
+    deps = [
+        ":builtin_ops",
+        "//tensorflow/contrib/lite:framework",
+        "//tensorflow/contrib/lite/kernels:test_util",
+        "@com_google_googletest//:gtest",
+    ],
+)
+
 tf_cc_test(
     name = "mean_test",
     size = "small",
diff --git a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
index 33d60afa26..3575974ae9 100644
--- a/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h
@@ -404,6 +404,7 @@ inline void DepthToSpace(const T* input_data, const Dims<4>& input_dims,
           const int in_d =
               out_d + ((out_h % block_size) * block_size + out_w % block_size) *
                           output_depth;
+
           const int in_w = out_w / block_size;
           const int in_h = out_h / block_size;
           const int in_b = out_b;
@@ -3363,6 +3364,30 @@ void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
   }
 }
 
+template <typename T>
+void TensorFlowMaximum(const T* input1_data, const Dims<4>& input1_dims,
+                       const T* input2_data, const Dims<4>& input2_dims,
+                       T* output_data, const Dims<4>& output_dims) {
+  NdArrayDesc<4> desc1;
+  NdArrayDesc<4> desc2;
+  NdArrayDescsForElementwiseBroadcast(input1_dims, input2_dims, &desc1, &desc2);
+
+  for (int b = 0; b < ArraySize(output_dims, 3); ++b) {
+    for (int y = 0; y < ArraySize(output_dims, 2); ++y) {
+      for (int x = 0; x < ArraySize(output_dims, 1); ++x) {
+        for (int c = 0; c < ArraySize(output_dims, 0); ++c) {
+          auto out_idx = Offset(output_dims, c, x, y, b);
+          auto in1_idx = SubscriptToIndex(desc1, c, x, y, b);
+          auto in2_idx = SubscriptToIndex(desc2, c, x, y, b);
+          auto in1_val = input1_data[in1_idx];
+          auto in2_val = input2_data[in2_idx];
+          output_data[out_idx] = in1_val > in2_val ? in1_val : in2_val;
+        }
+      }
+    }
+  }
+}
+
 template <typename T1, typename T2, typename T3>
 void ArgMax(const T3* axis, const T1* input_data, const Dims<4>& input_dims,
             T2* output_data, const Dims<4>& output_dims) {
diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum.cc
new file mode 100644
index 0000000000..9fdf2b47ea
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum.cc
@@ -0,0 +1,106 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <string.h>
+#include <vector>
+#include "tensorflow/contrib/lite/builtin_op_data.h"
+#include "tensorflow/contrib/lite/context.h"
+#include "tensorflow/contrib/lite/kernels/internal/reference/reference_ops.h"
+#include "tensorflow/contrib/lite/kernels/internal/tensor.h"
+#include "tensorflow/contrib/lite/kernels/kernel_util.h"
+#include "tensorflow/contrib/lite/kernels/op_macros.h"
+
+namespace tflite {
+namespace ops {
+namespace builtin {
+namespace maximum {
+
+// This file has a reference implemenation of TFMaximum.
+enum KernelType {
+  kReference,
+};
+
+constexpr int kInputTensor1 = 0;
+constexpr int kInputTensor2 = 1;
+constexpr int kOutputTensor = 0;
+
+struct MaximumContext {
+  MaximumContext(TfLiteContext* context, TfLiteNode* node) {
+    input1 = GetInput(context, node, kInputTensor1);
+    input2 = GetInput(context, node, kInputTensor2);
+    output = GetOutput(context, node, kOutputTensor);
+  }
+  TfLiteTensor* input1;
+  TfLiteTensor* input2;
+  TfLiteTensor* output;
+};
+
+TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
+  TF_LITE_ENSURE_EQ(context, NumInputs(node), 2);
+  TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
+
+  MaximumContext op_context(context, node);
+  TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type);
+  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input2->dims);
+  op_context.output->type = op_context.input2->type;
+  return context->ResizeTensor(context, op_context.output, output_dims);
+}
+
+template <KernelType kernel_type>
+TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
+  MaximumContext op_context(context, node);
+
+#define TF_LITE_MAXIMUM(kernel_type, data_type)    \
+  kernel_type::TensorFlowMaximum<data_type>(       \
+      GetTensorData<data_type>(op_context.input1), \
+      GetTensorDims(op_context.input1),            \
+      GetTensorData<data_type>(op_context.input2), \
+      GetTensorDims(op_context.input2),            \
+      GetTensorData<data_type>(op_context.output), \
+      GetTensorDims(op_context.output))
+
+  if (kernel_type == kReference) {
+    switch (op_context.output->type) {
+      case kTfLiteFloat32:
+        TF_LITE_MAXIMUM(reference_ops, float);
+        break;
+      default:
+        context->ReportError(context,
+                             "Type %d is currently not supported by Maximum.",
+                             op_context.output->type);
+        return kTfLiteError;
+    }
+  } else {
+    context->ReportError(context,
+                         "Type %d is currently not supported by Maximum.",
+                         op_context.output->type);
+    return kTfLiteError;
+  }
+#undef TF_LITE_MAXIMUM
+  return kTfLiteOk;
+}
+
+}  // namespace maximum
+
+TfLiteRegistration* Register_MAXIMUM_REF() {
+  static TfLiteRegistration r = {nullptr, nullptr, maximum::Prepare,
+                                 maximum::Eval<maximum::kReference>};
+  return &r;
+}
+
+TfLiteRegistration* Register_MAXIMUM() { return Register_MAXIMUM_REF(); }
+
+}  // namespace builtin
+}  // namespace ops
+}  // namespace tflite
diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc
new file mode 100644
index 0000000000..b3fd7d4e6f
--- /dev/null
+++ b/tensorflow/contrib/lite/kernels/maximum_test.cc
@@ -0,0 +1,81 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <gtest/gtest.h>
+#include "tensorflow/contrib/lite/interpreter.h"
+#include "tensorflow/contrib/lite/kernels/register.h"
+#include "tensorflow/contrib/lite/kernels/test_util.h"
+#include "tensorflow/contrib/lite/model.h"
+
+namespace tflite {
+namespace {
+
+using ::testing::ElementsAreArray;
+
+class MaximumOpModel : public SingleOpModel {
+ public:
+  MaximumOpModel(const TensorData& input1, const TensorData& input2,
+                 const TensorType& output) {
+    input1_ = AddInput(input1);
+    input2_ = AddInput(input2);
+    output_ = AddOutput(output);
+    SetBuiltinOp(BuiltinOperator_MAXIMUM, BuiltinOptions_MaximumOptions,
+                 CreateMaximumOptions(builder_).Union());
+    BuildInterpreter({GetShape(input1_), GetShape(input2_)});
+  }
+
+  template <class T>
+  void SetInput1(std::initializer_list<T> data) {
+    PopulateTensor(input1_, data);
+  }
+
+  template <class T>
+  void SetInput2(std::initializer_list<T> data) {
+    PopulateTensor(input2_, data);
+  }
+
+  template <class T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
+  std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
+
+ protected:
+  int input1_;
+  int input2_;
+  int output_;
+};
+
+TEST(MaximumOpTest, FloatTest) {
+  std::initializer_list<float> data1 = {1.0, 0.0, -1.0, 11.0, -2.0, -1.44};
+  std::initializer_list<float> data2 = {-1.0, 0.0, 1.0, 12.0, -3.0, -1.43};
+  MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}},
+                   {TensorType_FLOAT32, {3, 1, 2}}, TensorType_FLOAT32);
+  m.SetInput1<float>(data1);
+  m.SetInput2<float>(data2);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2}));
+  EXPECT_THAT(
+      m.GetOutput<float>(),
+      ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43})));
+}
+
+}  // namespace
+}  // namespace tflite
+
+int main(int argc, char** argv) {
+  ::tflite::LogToStderr();
+  ::testing::InitGoogleTest(&argc, argv);
+  return RUN_ALL_TESTS();
+}
diff --git a/tensorflow/contrib/lite/kernels/register.cc b/tensorflow/contrib/lite/kernels/register.cc
index 62045f0a4d..0f98154b90 100644
--- a/tensorflow/contrib/lite/kernels/register.cc
+++ b/tensorflow/contrib/lite/kernels/register.cc
@@ -76,6 +76,7 @@ TfLiteRegistration* Register_LOG_SOFTMAX();
 TfLiteRegistration* Register_CAST();
 TfLiteRegistration* Register_DEQUANTIZE();
 TfLiteRegistration* Register_PRELU();
+TfLiteRegistration* Register_MAXIMUM();
 
 BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_RELU, Register_RELU());
@@ -133,6 +134,7 @@ BuiltinOpResolver::BuiltinOpResolver() {
   AddBuiltin(BuiltinOperator_CAST, Register_CAST());
   AddBuiltin(BuiltinOperator_DEQUANTIZE, Register_DEQUANTIZE());
   AddBuiltin(BuiltinOperator_PRELU, Register_PRELU());
+  AddBuiltin(BuiltinOperator_MAXIMUM, Register_MAXIMUM());
 
   // TODO(andrewharp, ahentz): Move these somewhere more appropriate so that
   // custom ops aren't always included by default.
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index b7ccdf070b..791d1378f3 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -597,6 +597,9 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
       builtin_data = reinterpret_cast<void*>(params);
       break;
     }
+    case BuiltinOperator_MAXIMUM: {
+      break;
+    }
     case BuiltinOperator_DELEGATE: {
       // TODO(ycling): Revisit when supporting saving delegated models.
       error_reporter->Report("DELEGATE op shouldn't exist in model.");
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index e31b7c03a5..decaf9f160 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -350,6 +350,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
       case tflite::BuiltinOperator_DELEGATE:
       case tflite::BuiltinOperator_CAST:
       case tflite::BuiltinOperator_PRELU:
+      case tflite::BuiltinOperator_MAXIMUM:
         FATAL("Op code %d is currently not delegated to NNAPI", builtin);
         nn_op_type = -1;  // set to invalid
         break;
diff --git a/tensorflow/contrib/lite/python/lite.py b/tensorflow/contrib/lite/python/lite.py
index 35d224924e..ed6dd036f9 100644
--- a/tensorflow/contrib/lite/python/lite.py
+++ b/tensorflow/contrib/lite/python/lite.py
@@ -25,9 +25,9 @@ EXPERIMENTAL: APIs here are unstable and likely to change without notice.
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-import os
-import subprocess
-import tempfile
+import os as _os
+import subprocess as _subprocess
+import tempfile as _tempfile
 
 # pylint: disable=unused-import
 from tensorflow.contrib.lite.python.op_hint import convert_op_hints_to_stubs
@@ -74,7 +74,7 @@ else:
   _toco_from_proto_bin = _resource_loader.get_path_to_datafile(
       "../toco/python/toco_from_protos")
 
-if _toco_from_proto_bin and not os.path.exists(_toco_from_proto_bin):
+if _toco_from_proto_bin and not _os.path.exists(_toco_from_proto_bin):
   _toco_from_proto_bin = "toco_from_protos"
 
 
@@ -102,10 +102,10 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
     return _toco_python.TocoConvert(
         model_flags_str, toco_flags_str, input_data_str)
 
-  with tempfile.NamedTemporaryFile() as fp_toco, \
-           tempfile.NamedTemporaryFile() as fp_model, \
-           tempfile.NamedTemporaryFile() as fp_input, \
-           tempfile.NamedTemporaryFile() as fp_output:
+  with _tempfile.NamedTemporaryFile() as fp_toco, \
+           _tempfile.NamedTemporaryFile() as fp_model, \
+           _tempfile.NamedTemporaryFile() as fp_input, \
+           _tempfile.NamedTemporaryFile() as fp_output:
     fp_model.write(model_flags_str)
     fp_toco.write(toco_flags_str)
     fp_input.write(input_data_str)
@@ -118,11 +118,11 @@ def toco_convert_protos(model_flags_str, toco_flags_str, input_data_str):
         fp_output.name
     ]
     cmdline = " ".join(cmd)
-    proc = subprocess.Popen(
+    proc = _subprocess.Popen(
         cmdline,
         shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
+        stdout=_subprocess.PIPE,
+        stderr=_subprocess.STDOUT,
         close_fds=True)
     stdout, stderr = proc.communicate()
     exitcode = proc.returncode
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index e1075971e9..7d2e00fe32 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -131,6 +131,7 @@ enum BuiltinOperator : byte {
   BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   CAST = 53,
   PRELU = 54,
+  MAXIMUM = 55,
 }
 
 // Options for the builtin operators.
@@ -173,6 +174,7 @@ union BuiltinOptions {
   LogSoftmaxOptions,
   CastOptions,
   DequantizeOptions,
+  MaximumOptions,
 }
 
 enum Padding : byte { SAME, VALID }
@@ -384,6 +386,9 @@ table CastOptions {
 table DequantizeOptions {
 }
 
+table MaximumOptions {
+}
+
 // An OperatorCode can be an enum value (BuiltinOperator) if the operator is a
 // builtin, or a string if the operator is custom.
 table OperatorCode {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 86daeaf5cc..66a97a1460 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -145,6 +145,9 @@ struct CastOptionsT;
 struct DequantizeOptions;
 struct DequantizeOptionsT;
 
+struct MaximumOptions;
+struct MaximumOptionsT;
+
 struct OperatorCode;
 struct OperatorCodeT;
 
@@ -255,11 +258,12 @@ enum BuiltinOperator {
   BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM = 52,
   BuiltinOperator_CAST = 53,
   BuiltinOperator_PRELU = 54,
+  BuiltinOperator_MAXIMUM = 55,
   BuiltinOperator_MIN = BuiltinOperator_ADD,
-  BuiltinOperator_MAX = BuiltinOperator_PRELU
+  BuiltinOperator_MAX = BuiltinOperator_MAXIMUM
 };
 
-inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
+inline BuiltinOperator (&EnumValuesBuiltinOperator())[54] {
   static BuiltinOperator values[] = {
     BuiltinOperator_ADD,
     BuiltinOperator_AVERAGE_POOL_2D,
@@ -313,7 +317,8 @@ inline BuiltinOperator (&EnumValuesBuiltinOperator())[53] {
     BuiltinOperator_DELEGATE,
     BuiltinOperator_BIDIRECTIONAL_SEQUENCE_LSTM,
     BuiltinOperator_CAST,
-    BuiltinOperator_PRELU
+    BuiltinOperator_PRELU,
+    BuiltinOperator_MAXIMUM
   };
   return values;
 }
@@ -375,6 +380,7 @@ inline const char **EnumNamesBuiltinOperator() {
     "BIDIRECTIONAL_SEQUENCE_LSTM",
     "CAST",
     "PRELU",
+    "MAXIMUM",
     nullptr
   };
   return names;
@@ -425,11 +431,12 @@ enum BuiltinOptions {
   BuiltinOptions_LogSoftmaxOptions = 36,
   BuiltinOptions_CastOptions = 37,
   BuiltinOptions_DequantizeOptions = 38,
+  BuiltinOptions_MaximumOptions = 39,
   BuiltinOptions_MIN = BuiltinOptions_NONE,
-  BuiltinOptions_MAX = BuiltinOptions_DequantizeOptions
+  BuiltinOptions_MAX = BuiltinOptions_MaximumOptions
 };
 
-inline BuiltinOptions (&EnumValuesBuiltinOptions())[39] {
+inline BuiltinOptions (&EnumValuesBuiltinOptions())[40] {
   static BuiltinOptions values[] = {
     BuiltinOptions_NONE,
     BuiltinOptions_Conv2DOptions,
@@ -469,7 +476,8 @@ inline BuiltinOptions (&EnumValuesBuiltinOptions())[39] {
     BuiltinOptions_SplitOptions,
     BuiltinOptions_LogSoftmaxOptions,
     BuiltinOptions_CastOptions,
-    BuiltinOptions_DequantizeOptions
+    BuiltinOptions_DequantizeOptions,
+    BuiltinOptions_MaximumOptions
   };
   return values;
 }
@@ -515,6 +523,7 @@ inline const char **EnumNamesBuiltinOptions() {
     "LogSoftmaxOptions",
     "CastOptions",
     "DequantizeOptions",
+    "MaximumOptions",
     nullptr
   };
   return names;
@@ -681,6 +690,10 @@ template<> struct BuiltinOptionsTraits<DequantizeOptions> {
   static const BuiltinOptions enum_value = BuiltinOptions_DequantizeOptions;
 };
 
+template<> struct BuiltinOptionsTraits<MaximumOptions> {
+  static const BuiltinOptions enum_value = BuiltinOptions_MaximumOptions;
+};
+
 struct BuiltinOptionsUnion {
   BuiltinOptions type;
   void *value;
@@ -1016,6 +1029,14 @@ struct BuiltinOptionsUnion {
     return type == BuiltinOptions_DequantizeOptions ?
       reinterpret_cast<const DequantizeOptionsT *>(value) : nullptr;
   }
+  MaximumOptionsT *AsMaximumOptions() {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<MaximumOptionsT *>(value) : nullptr;
+  }
+  const MaximumOptionsT *AsMaximumOptions() const {
+    return type == BuiltinOptions_MaximumOptions ?
+      reinterpret_cast<const MaximumOptionsT *>(value) : nullptr;
+  }
 };
 
 bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *obj, BuiltinOptions type);
@@ -3759,6 +3780,46 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(
 
 flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffers::FlatBufferBuilder &_fbb, const DequantizeOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
 
+struct MaximumOptionsT : public flatbuffers::NativeTable {
+  typedef MaximumOptions TableType;
+  MaximumOptionsT() {
+  }
+};
+
+struct MaximumOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
+  typedef MaximumOptionsT NativeTableType;
+  bool Verify(flatbuffers::Verifier &verifier) const {
+    return VerifyTableStart(verifier) &&
+           verifier.EndTable();
+  }
+  MaximumOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  void UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver = nullptr) const;
+  static flatbuffers::Offset<MaximumOptions> Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+};
+
+struct MaximumOptionsBuilder {
+  flatbuffers::FlatBufferBuilder &fbb_;
+  flatbuffers::uoffset_t start_;
+  explicit MaximumOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
+        : fbb_(_fbb) {
+    start_ = fbb_.StartTable();
+  }
+  MaximumOptionsBuilder &operator=(const MaximumOptionsBuilder &);
+  flatbuffers::Offset<MaximumOptions> Finish() {
+    const auto end = fbb_.EndTable(start_);
+    auto o = flatbuffers::Offset<MaximumOptions>(end);
+    return o;
+  }
+};
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(
+    flatbuffers::FlatBufferBuilder &_fbb) {
+  MaximumOptionsBuilder builder_(_fbb);
+  return builder_.Finish();
+}
+
+flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher = nullptr);
+
 struct OperatorCodeT : public flatbuffers::NativeTable {
   typedef OperatorCode TableType;
   BuiltinOperator builtin_code;
@@ -3990,6 +4051,9 @@ struct Operator FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   const DequantizeOptions *builtin_options_as_DequantizeOptions() const {
     return builtin_options_type() == BuiltinOptions_DequantizeOptions ? static_cast<const DequantizeOptions *>(builtin_options()) : nullptr;
   }
+  const MaximumOptions *builtin_options_as_MaximumOptions() const {
+    return builtin_options_type() == BuiltinOptions_MaximumOptions ? static_cast<const MaximumOptions *>(builtin_options()) : nullptr;
+  }
   const flatbuffers::Vector<uint8_t> *custom_options() const {
     return GetPointer<const flatbuffers::Vector<uint8_t> *>(VT_CUSTOM_OPTIONS);
   }
@@ -4168,6 +4232,10 @@ template<> inline const DequantizeOptions *Operator::builtin_options_as<Dequanti
   return builtin_options_as_DequantizeOptions();
 }
 
+template<> inline const MaximumOptions *Operator::builtin_options_as<MaximumOptions>() const {
+  return builtin_options_as_MaximumOptions();
+}
+
 struct OperatorBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
@@ -5696,6 +5764,29 @@ inline flatbuffers::Offset<DequantizeOptions> CreateDequantizeOptions(flatbuffer
       _fbb);
 }
 
+inline MaximumOptionsT *MaximumOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
+  auto _o = new MaximumOptionsT();
+  UnPackTo(_o, _resolver);
+  return _o;
+}
+
+inline void MaximumOptions::UnPackTo(MaximumOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
+  (void)_o;
+  (void)_resolver;
+}
+
+inline flatbuffers::Offset<MaximumOptions> MaximumOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
+  return CreateMaximumOptions(_fbb, _o, _rehasher);
+}
+
+inline flatbuffers::Offset<MaximumOptions> CreateMaximumOptions(flatbuffers::FlatBufferBuilder &_fbb, const MaximumOptionsT *_o, const flatbuffers::rehasher_function_t *_rehasher) {
+  (void)_rehasher;
+  (void)_o;
+  struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const MaximumOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  return tflite::CreateMaximumOptions(
+      _fbb);
+}
+
 inline OperatorCodeT *OperatorCode::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
   auto _o = new OperatorCodeT();
   UnPackTo(_o, _resolver);
@@ -6028,6 +6119,10 @@ inline bool VerifyBuiltinOptions(flatbuffers::Verifier &verifier, const void *ob
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return verifier.VerifyTable(ptr);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return verifier.VerifyTable(ptr);
+    }
     default: return false;
   }
 }
@@ -6198,6 +6293,10 @@ inline void *BuiltinOptionsUnion::UnPack(const void *obj, BuiltinOptions type, c
       auto ptr = reinterpret_cast<const DequantizeOptions *>(obj);
       return ptr->UnPack(resolver);
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptions *>(obj);
+      return ptr->UnPack(resolver);
+    }
     default: return nullptr;
   }
 }
@@ -6356,6 +6455,10 @@ inline flatbuffers::Offset<void> BuiltinOptionsUnion::Pack(flatbuffers::FlatBuff
       auto ptr = reinterpret_cast<const DequantizeOptionsT *>(value);
       return CreateDequantizeOptions(_fbb, ptr, _rehasher).Union();
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<const MaximumOptionsT *>(value);
+      return CreateMaximumOptions(_fbb, ptr, _rehasher).Union();
+    }
     default: return 0;
   }
 }
@@ -6514,6 +6617,10 @@ inline BuiltinOptionsUnion::BuiltinOptionsUnion(const BuiltinOptionsUnion &u) FL
       value = new DequantizeOptionsT(*reinterpret_cast<DequantizeOptionsT *>(u.value));
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      value = new MaximumOptionsT(*reinterpret_cast<MaximumOptionsT *>(u.value));
+      break;
+    }
     default:
       break;
   }
@@ -6711,6 +6818,11 @@ inline void BuiltinOptionsUnion::Reset() {
       delete ptr;
       break;
     }
+    case BuiltinOptions_MaximumOptions: {
+      auto ptr = reinterpret_cast<MaximumOptionsT *>(value);
+      delete ptr;
+      break;
+    }
     default: break;
   }
   value = nullptr;
diff --git a/tensorflow/contrib/lite/testing/BUILD b/tensorflow/contrib/lite/testing/BUILD
index 10e810a6e0..62f20638ba 100644
--- a/tensorflow/contrib/lite/testing/BUILD
+++ b/tensorflow/contrib/lite/testing/BUILD
@@ -36,6 +36,7 @@ gen_zipped_test_files(
         "local_response_norm.zip",
         "log_softmax.zip",
         "max_pool.zip",
+        "maximum.zip",
         "mean.zip",
         "mul.zip",
         "pad.zip",
diff --git a/tensorflow/contrib/lite/testing/generate_examples.py b/tensorflow/contrib/lite/testing/generate_examples.py
index cb5c500136..8045052452 100644
--- a/tensorflow/contrib/lite/testing/generate_examples.py
+++ b/tensorflow/contrib/lite/testing/generate_examples.py
@@ -862,6 +862,41 @@ def make_log_softmax_tests(zip_path):
   make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
 
 
+def make_maximum_tests(zip_path):
+  """Make a set of tests to do maximum."""
+
+  test_parameters = [{
+      "input_dtype": [tf.float32],
+      "input_shape_1": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+      "input_shape_2": [[3], [1, 100], [4, 2, 3], [5, 224, 224, 3]],
+  }]
+
+  def build_graph(parameters):
+    """Build the maximum op testing graph."""
+    input_tensor_1 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_1",
+        shape=parameters["input_shape_1"])
+    input_tensor_2 = tf.placeholder(
+        dtype=parameters["input_dtype"],
+        name="input_2",
+        shape=parameters["input_shape_2"])
+
+    out = tf.maximum(input_tensor_1, input_tensor_2)
+    return [input_tensor_1, input_tensor_2], [out]
+
+  def build_inputs(parameters, sess, inputs, outputs):
+    values = [
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_1"]),
+        create_tensor_data(parameters["input_dtype"],
+                           parameters["input_shape_2"])
+    ]
+    return values, sess.run(outputs, feed_dict=dict(zip(inputs, values)))
+
+  make_zip_of_tests(zip_path, test_parameters, build_graph, build_inputs)
+
+
 def make_binary_op_tests_func(binary_operator):
   """Return a function that does a test on a binary operator."""
   return lambda zip_path: make_binary_op_tests(zip_path, binary_operator)
@@ -1977,6 +2012,7 @@ def main(unused_args):
         "exp.zip": make_exp_tests,
         "log_softmax.zip": make_log_softmax_tests,
         "lstm.zip": make_lstm_tests,
+        "maximum.zip": make_maximum_tests,
     }
     out = FLAGS.zip_to_output
     bin_path = FLAGS.toco
diff --git a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
index a4a7283508..6697b86e79 100644
--- a/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
+++ b/tensorflow/contrib/lite/testing/generated_examples_zip_test.cc
@@ -253,6 +253,7 @@ INSTANTIATE_TESTS(l2_pool)
 INSTANTIATE_TESTS(l2norm)
 INSTANTIATE_TESTS(local_response_norm)
 INSTANTIATE_TESTS(log_softmax)
+INSTANTIATE_TESTS(maximum)
 INSTANTIATE_TESTS(max_pool)
 INSTANTIATE_TESTS(mean)
 INSTANTIATE_TESTS(mul)
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index f23249cfa1..0989bfe5a3 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -863,6 +863,8 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
   ops.emplace_back(new SimpleOperator<ExpOperator>("EXP", OperatorType::kExp));
   ops.emplace_back(new SimpleOperator<LogSoftmaxOperator>(
       "LOG_SOFTMAX", OperatorType::kLogSoftmax));
+  ops.emplace_back(new SimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum));
 
   return ops;
 }
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index 9c19f8d464..f7a213ecfc 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -109,6 +109,8 @@ TEST_F(OperatorTest, SimpleOperators) {
   CheckSimpleOperator<ExpOperator>("EXP", OperatorType::kExp);
   CheckSimpleOperator<LogSoftmaxOperator>("LOG_SOFTMAX",
                                           OperatorType::kLogSoftmax);
+  CheckSimpleOperator<TensorFlowMaximumOperator>(
+      "MAXIMUM", OperatorType::kTensorFlowMaximum);
 }
 
 TEST_F(OperatorTest, BuiltinAdd) {
diff --git a/tensorflow/contrib/lookup/lookup_ops.py b/tensorflow/contrib/lookup/lookup_ops.py
index a57a1e5421..a03e731be3 100644
--- a/tensorflow/contrib/lookup/lookup_ops.py
+++ b/tensorflow/contrib/lookup/lookup_ops.py
@@ -494,7 +494,7 @@ class MutableDenseHashTable(LookupInterface):
                                                   value_dtype=tf.int64,
                                                   default_value=-1,
                                                   empty_key=0)
-  table.insert(keys, values)
+  sess.run(table.insert(keys, values))
   out = table.lookup(query_keys)
   print(out.eval())
   ```
diff --git a/tensorflow/contrib/makefile/download_dependencies.sh b/tensorflow/contrib/makefile/download_dependencies.sh
index 4ae18b2cef..8b415e6527 100755
--- a/tensorflow/contrib/makefile/download_dependencies.sh
+++ b/tensorflow/contrib/makefile/download_dependencies.sh
@@ -34,7 +34,7 @@ PROTOBUF_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/protobuf/.
 RE2_URL="$(grep -o 'https://mirror.bazel.build/github.com/google/re2/.*tar\.gz' "${BZL_FILE_PATH}" | head -n1)"
 FFT2D_URL="$(grep -o 'http.*fft\.tgz' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 ABSL_URL="$(grep -o 'https://github.com/abseil/abseil-cpp/.*tar.gz' "${BZL_FILE_PATH}" | head -n1)"
-CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v bazel-mirror | head -n1)"
+CUB_URL="$(grep -o 'https.*cub/archive.*zip' "${BZL_FILE_PATH}" | grep -v mirror.bazel | head -n1)"
 
 # TODO(petewarden): Some new code in Eigen triggers a clang bug with iOS arm64,
 #                   so work around it by patching the source.
diff --git a/tensorflow/contrib/makefile/tf_op_files.txt b/tensorflow/contrib/makefile/tf_op_files.txt
index 15786291ed..b6acf71b9d 100644
--- a/tensorflow/contrib/makefile/tf_op_files.txt
+++ b/tensorflow/contrib/makefile/tf_op_files.txt
@@ -263,6 +263,7 @@ tensorflow/core/kernels/requantize.cc
 tensorflow/core/kernels/remote_fused_graph_execute_op.cc
 tensorflow/core/kernels/remote_fused_graph_execute_utils.cc
 tensorflow/core/kernels/batch_matmul_op_real.cc
+tensorflow/core/kernels/random_op.cc
 tensorflow/core/ops/training_ops.cc
 tensorflow/core/ops/string_ops.cc
 tensorflow/core/ops/state_ops.cc
diff --git a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
index dfa12e873a..a9a32b7b25 100644
--- a/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
+++ b/tensorflow/contrib/seq2seq/kernels/beam_search_ops.cc
@@ -74,7 +74,7 @@ class GatherTreeOp : public OpKernel {
         ctx,
         step_ids_shape.dim_size(1) == max_sequence_lengths.shape().dim_size(0),
         errors::InvalidArgument("batch size dimensions step_ids.shape[1] and "
-                                "max_seqeuence_lengths.shape[0] must match.  "
+                                "max_sequence_lengths.shape[0] must match.  "
                                 "but shapes are: ",
                                 step_ids_shape.DebugString(), " and ",
                                 max_sequence_lengths.shape().DebugString()));
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 9ff8a343f1..be53779826 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -736,7 +736,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
   """Base attention mechanism for monotonic attention.
 
   Simply overrides the initial_alignments function to provide a dirac
-  distribution,which is needed in order for the monotonic attention
+  distribution, which is needed in order for the monotonic attention
   distributions to have the correct behavior.
   """
 
@@ -763,7 +763,7 @@ class _BaseMonotonicAttentionMechanism(_BaseAttentionMechanism):
 class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Bahadanau-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -867,7 +867,7 @@ class BahdanauMonotonicAttention(_BaseMonotonicAttentionMechanism):
 class LuongMonotonicAttention(_BaseMonotonicAttentionMechanism):
   """Monotonic attention mechanism with Luong-style energy function.
 
-  This type of attention encorces a monotonic constraint on the attention
+  This type of attention enforces a monotonic constraint on the attention
   distributions; that is once the model attends to a given point in the memory
   it can't attend to any prior points at subsequence output timesteps.  It
   achieves this by using the _monotonic_probability_fn instead of softmax to
@@ -1133,7 +1133,7 @@ class AttentionWrapper(rnn_cell_impl.RNNCell):
       output_attention: Python bool.  If `True` (default), the output at each
         time step is the attention value.  This is the behavior of Luong-style
         attention mechanisms.  If `False`, the output at each time step is
-        the output of `cell`.  This is the beahvior of Bhadanau-style
+        the output of `cell`.  This is the behavior of Bhadanau-style
         attention mechanisms.  In both cases, the `attention` tensor is
         propagated to the next time step via the state and is used there.
         This flag only controls whether the attention mechanism is propagated
diff --git a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
index a26107b0d7..184144f64a 100644
--- a/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
+++ b/tensorflow/contrib/seq2seq/python/ops/beam_search_decoder.py
@@ -821,9 +821,9 @@ def _get_scores(log_probs, sequence_lengths, length_penalty_weight):
   Returns:
     The scores normalized by the length_penalty.
   """
-  length_penality_ = _length_penalty(
+  length_penalty_ = _length_penalty(
       sequence_lengths=sequence_lengths, penalty_factor=length_penalty_weight)
-  return log_probs / length_penality_
+  return log_probs / length_penalty_
 
 
 def _length_penalty(sequence_lengths, penalty_factor):
@@ -860,7 +860,7 @@ def _mask_probs(probs, eos_token, finished):
   unfinished beams remain unchanged.
 
   Args:
-    probs: Log probabiltiies of shape `[batch_size, beam_width, vocab_size]`
+    probs: Log probabilities of shape `[batch_size, beam_width, vocab_size]`
     eos_token: An int32 id corresponding to the EOS token to allocate
       probability to.
     finished: A boolean tensor of shape `[batch_size, beam_width]` that
diff --git a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
index b3343aef47..99ad487630 100644
--- a/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
+++ b/tensorflow/contrib/slim/python/slim/data/parallel_reader.py
@@ -115,8 +115,8 @@ class ParallelReader(io_ops.ReaderBase):
     reader needs to start reading from a new file since it has finished with
     the previous file).
 
-    A queue runner for enqueing in the `common_queue` is automatically added to
-    the TF QueueRunners collection.
+    A queue runner for enqueuing in the `common_queue` is automatically added
+    to the TF QueueRunners collection.
 
     Args:
       queue: A Queue or a mutable string Tensor representing a handle
diff --git a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
index 37e9c4754c..62bd200361 100644
--- a/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
+++ b/tensorflow/contrib/slim/python/slim/data/prefetch_queue.py
@@ -36,9 +36,9 @@ def prefetch_queue(tensors,
                    dynamic_pad=False,
                    shared_name=None,
                    name=None):
-  """Creates a queue to prefetech tensors from `tensors`.
+  """Creates a queue to prefetch tensors from `tensors`.
 
-  A queue runner for enqueing tensors into the prefetch_queue is automatically
+  A queue runner for enqueuing tensors into the prefetch_queue is automatically
   added to the TF QueueRunners collection.
 
   Example:
diff --git a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
index b3b61e1dfe..f2d31dc8db 100644
--- a/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
+++ b/tensorflow/contrib/slim/python/slim/data/tfexample_decoder.py
@@ -124,7 +124,7 @@ class BoundingBox(ItemHandler):
     super(BoundingBox, self).__init__(self._full_keys)
 
   def tensors_to_item(self, keys_to_tensors):
-    """Maps the given dictionary of tensors to a contatenated list of bboxes.
+    """Maps the given dictionary of tensors to a concatenated list of bboxes.
 
     Args:
       keys_to_tensors: a mapping of TF-Example keys to parsed tensors.
diff --git a/tensorflow/contrib/tensorrt/README.md b/tensorflow/contrib/tensorrt/README.md
index 461e627e99..6eafc1754c 100644
--- a/tensorflow/contrib/tensorrt/README.md
+++ b/tensorflow/contrib/tensorrt/README.md
@@ -1,15 +1,15 @@
-Using TensorRT in TensorFlow
-============================
+# Using TensorRT in TensorFlow
+
 
 This module provides necessary bindings and introduces TRT_engine_op
 operator that wraps a subgraph in TensorRT. This is still a work in progress
 but should be useable with most common graphs.
 
-Compilation
------------
+## Compilation
+
 
 In order to compile the module, you need to have a local TensorRT
-installation (libnvinfer.so and respective include files). During the
+installation ( libnvinfer.so and respective include files ). During the
 configuration step, TensorRT should be enabled and installation path
 should be set. If installed through package managers (deb,rpm),
 configure script should find the necessary components from the system
@@ -22,4 +22,38 @@ bazel-bin/tensorflow/tools/pip_package/build_pip_package /tmp/
 ```
 
 After the installation of tensorflow package, TensorRT transformation
-will be available. An example use can be found in test/test_tftrt.py directory
+will be available. An example use can be found in test/test_tftrt.py script
+
+## Installing TensorRT 3.0.4
+
+In order to make use of TensorRT integration, you will need a local installation of TensorRT 3.0.4 from the [NVIDIA Developer website](https://developer.nvidia.com/tensorrt). Due to compiler compatibility, you will need to download and install the TensorRT 3.0.4 tarball for _Ubuntu 14.04_, i.e., **_TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz_**, even if you are using Ubuntu 16.04 or later.
+
+### Preparing TensorRT installation
+
+Once you have downloaded TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz, you will need to unpack it to an installation directory, which will be referred to as <install_dir>. Please replace <install_dir> with the full path of actual installation directory you choose in commands below.
+
+```shell
+cd <install_dir> && tar -zxf /path/to/TensorRT-3.0.4.Ubuntu-14.04.5.x86_64.cuda-9.0.cudnn7.0-tar.gz
+```
+
+After unpacking the binaries, you have several options to use them:
+
+#### To run TensorFlow as a user without superuser privileges
+
+For a regular user without any sudo rights, you should add TensorRT to your `$LD_LIBRARY_PATH`:
+
+  ```shell
+   export LD_LIBRARY_PATH=<install_dir>/TensorRT-3.0.4/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}
+  ```
+
+Then you are ready to use TensorFlow-TensorRT integration. `$LD_LIBRARY_PATH` must contain the path to TensorRT installation for TensorFlow-TensorRT integration to work. If you are using a VirtualEnv-like setup, you can add the command above to your `bin/activate` script or to your `.bashrc` script.
+
+#### To run TensorFlow as a superuser
+
+ When running as a superuser, such as in a container or via sudo, the `$LD_LIBRARY_PATH` approach above may not work. The following is preferred when the user has superuser privileges:
+
+  ```shell
+  echo "<install_dir>/TensorRT-3.0.4/lib" | sudo tee /etc/ld.so.conf.d/tensorrt304.conf && sudo ldconfig
+  ```
+
+  Please ensure that any existing deb package installation of TensorRT is removed before following these instructions to avoid package conflicts.
\ No newline at end of file
diff --git a/tensorflow/contrib/tensorrt/convert/convert_graph.cc b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
index eea8c8efa2..ff8cc6374d 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_graph.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_graph.cc
@@ -49,12 +49,13 @@ namespace tensorrt {
 namespace convert {
 namespace {
 
-bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
+bool IsTensorRTCandidate(const tensorflow::Node* node) {
   // LINT.IfChange
   // TODO(jie): Segmentation shouldn't associated with op name.
   //            Split it into a registration for each kernel.
   static const std::set<string> candidate_ops = {
       "Identity",
+      "Snapshot",
       "Const",
       "Conv2D",
       "MaxPool",
@@ -74,7 +75,7 @@ bool IsTensorRTCandidate(const tensorflow::NodeDef& node_def) {
       // TODO(ben,jie): ...
   };
   // LINT.ThenChange(//tensorflow/contrib/tensorrt/convert/convert_nodes.h)
-  return candidate_ops.count(node_def.op());
+  return candidate_ops.count(node->type_string());
 }
 
 void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
@@ -84,10 +85,10 @@ void GetSubGraphIncomingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->in_edges()) {
       if (!subgraph_node_ids.count(edge->src()->id()) &&
-          !edge->src()->IsSource()) {
+          !edge->src()->IsSource() && !edge->IsControlEdge()) {
         incoming_edges->insert(edge);
       } else {
-        VLOG(2) << edge->src()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->src()->name() << " N, ";
       }
     }
   }
@@ -100,11 +101,11 @@ void GetSubGraphOutgoingEdges(const tensorflow::Graph& graph,
     const tensorflow::Node* node = graph.FindNodeId(node_id);
     for (const tensorflow::Edge* edge : node->out_edges()) {
       if (!subgraph_node_ids.count(edge->dst()->id()) &&
-          !edge->dst()->IsSink()) {
-        VLOG(2) << edge->dst()->name() << " Y, ";
+          !edge->dst()->IsSink() && !edge->IsControlEdge()) {
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " Y, ";
         outgoing_edges->insert(edge);
       } else {
-        VLOG(2) << edge->dst()->name() << " N, ";
+        VLOG(2) << node->name() << " -> " << edge->dst()->name() << " N, ";
       }
     }
   }
@@ -409,8 +410,9 @@ tensorflow::Status ConvertGraphDefToTensorRT(
       tensorflow::Status status = ConvertSubGraphToTensorRT(&p);
       if (status != tensorflow::Status::OK()) {
         LOG(WARNING) << "subgraph conversion error for subgraph_index:" << count
-                     << " due to: \n"
-                     << status.ToString() << " SKIPPING......";
+                     << " due to: \"" << status.ToString()
+                     << "\" SKIPPING......( " << subgraph_node_names.size()
+                     << " nodes)";
       }
       count++;
     }
diff --git a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
index 92a692baa7..370911e4d9 100644
--- a/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
+++ b/tensorflow/contrib/tensorrt/convert/convert_nodes.cc
@@ -53,8 +53,8 @@ limitations under the License.
 namespace tensorflow {
 namespace tensorrt {
 namespace convert {
+using ::tensorflow::strings::StrAppend;
 using ::tensorflow::strings::StrCat;
-
 namespace {
 
 inline tensorflow::Status ConvertDType(tensorflow::DataType tf_dtype,
@@ -430,9 +430,8 @@ class Converter {
   tensorflow::tensorrt::TRTWeightStore* weight_store_;
   bool fp16_;
   void register_op_converters();
-  std::vector<TRT_TensorOrWeights> get_inputs(
-      const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs;
+  tensorflow::Status get_inputs(const tensorflow::NodeDef& node_def,
+                                std::vector<TRT_TensorOrWeights>* inputs) {
     for (auto const& input_name : node_def.input()) {
       /*************************************************************************
        * TODO(jie) handle case 1) here
@@ -453,13 +452,17 @@ class Converter {
 
       VLOG(2) << "retrieve input: " << name;
       if (trt_tensors_.count(name)) {
-        inputs.push_back(trt_tensors_.at(name));
+        inputs->push_back(trt_tensors_.at(name));
       } else {
-        LOG(FATAL) << "input: " << name << " not available for node at, "
-                   << node_def.name();
+        string str("Node ");
+        StrAppend(&str, node_def.name(), " should have an input named '", name,
+                  "' but it is not available");
+        LOG(WARNING) << "input: " << name << " not available for node at "
+                     << node_def.name();
+        return tensorflow::errors::InvalidArgument(str);
       }
     }
-    return inputs;
+    return tensorflow::Status::OK();
   }
 
  public:
@@ -483,7 +486,8 @@ class Converter {
   }
 
   tensorflow::Status convert_node(const tensorflow::NodeDef& node_def) {
-    std::vector<TRT_TensorOrWeights> inputs = this->get_inputs(node_def);
+    std::vector<TRT_TensorOrWeights> inputs;
+    TF_RETURN_IF_ERROR(this->get_inputs(node_def, &inputs));
     string op = node_def.op();
     if (!op_registry_.count(op)) {
       return tensorflow::errors::Unimplemented(
@@ -548,6 +552,19 @@ class Converter {
   }
 };
 
+TRT_ShapedWeights ConvertFP32ToFP16(Converter& ctx,
+                                    const TRT_ShapedWeights& weights_src) {
+  auto dtype_new = tensorflow::DataType::DT_HALF;
+  TRT_ShapedWeights weights =
+      ctx.get_temp_weights(dtype_new, weights_src.shape_);
+  const float* src = static_cast<const float*>(weights_src.GetValues());
+  Eigen::half* dst = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(weights.GetValues()));
+  for (int64_t i = 0; i < weights_src.count(); i++) {
+    dst[i] = Eigen::half_impl::float_to_half_rtne(src[i]);
+  }
+  return weights;
+}
 // ****************************************************************************
 // Constant folding functions
 // TODO(jie): once optimizer kicks in, we should have done constant folding
@@ -875,7 +892,7 @@ tensorflow::Status BinaryTensorOpWeight(
 
   // Check type consistency
   nvinfer1::DataType ttype;
-  TF_CHECK_OK(ConvertDType(weights.type_, &ttype));
+  TF_RETURN_IF_ERROR(ConvertDType(weights.type_, &ttype));
 
   // Check scale mode
   auto dims_w = weights.shape_;
@@ -957,6 +974,10 @@ tensorflow::Status BinaryTensorOpWeight(
     }
   }
 
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, weights);
+  }
+
   // prepare weights
   TRT_ShapedWeights shift_weights(weights.type_);
   TRT_ShapedWeights scale_weights(weights.type_);
@@ -998,9 +1019,7 @@ enum class ConvolutionType { DEFAULT, DEPTHWISE_CONV };
 tensorflow::Status ConvertConv2DHelper(
     Converter& ctx, const tensorflow::NodeDef& node_def,
     const std::vector<TRT_TensorOrWeights>& inputs,
-    std::vector<TRT_TensorOrWeights>* outputs,
-    int group  // group ==0 specifies depthwise conv
-) {
+    std::vector<TRT_TensorOrWeights>* outputs, int group) {
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
   TFAttrs attrs(node_def);
@@ -1025,6 +1044,10 @@ tensorflow::Status ConvertConv2DHelper(
   VLOG(2) << "groups count: " << num_groups;
 
   TRT_ShapedWeights weights_rsck = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights_rsck = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights weights = ctx.get_temp_weights_like(weights_rsck);
   ReorderRSCKToKCRS(weights_rsck, &weights, num_groups);
   TRT_ShapedWeights biases(weights.type_);
@@ -1134,9 +1157,9 @@ tensorflow::Status BinaryTensorOpTensor(
   CHECK_EQ_TYPE(tensor_r->getType(), dtype);
   auto op_pair = ops.find(node_def.op());
   if (op_pair == ops.end())
-    return tensorflow::errors::Unimplemented("binary op: " + node_def.op() +
-                                             " not supported at: " +
-                                             node_def.name());
+    return tensorflow::errors::Unimplemented(
+        "binary op: " + node_def.op() +
+        " not supported at: " + node_def.name());
 
   nvinfer1::IElementWiseLayer* layer = ctx.network()->addElementWise(
       *const_cast<nvinfer1::ITensor*>(tensor_l),
@@ -1295,8 +1318,11 @@ tensorflow::Status ConvertScale(Converter& ctx,
   // Implement tensor binaryOp weight [channel wise] for now;
   const nvinfer1::ITensor* tensor = inputs.at(0).tensor();
 
-  // TODO(jie): handle NHWC/NCHW transpose;
   TRT_ShapedWeights weights = inputs.at(1).weights();
+  if (ctx.isFP16()) {
+    weights = ConvertFP32ToFP16(ctx, inputs.at(1).weights());
+  }
+
   TRT_ShapedWeights empty_weights(weights.type_);
 
   TFAttrs attrs(node_def);
@@ -1376,8 +1402,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.float_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1391,33 +1420,16 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      auto half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      half_tensor.device(defd) =
-          tensor.flat<float>().template cast<Eigen::half>();
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<float> tensor_data(
-          weights_tensor.float_val().begin(),
-          weights_tensor.float_val()
-              .end());  //  make a local copy first to flatten
-      memcpy(dst, tensor_data.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<float> tensor_data(
+        weights_tensor.float_val().begin(),
+        weights_tensor.float_val()
+            .end());  //  make a local copy first to flatten
+    memcpy(dst, tensor_data.data(), len_data);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.int_val().empty()) {
     VLOG(2) << "int!!!" << node_def.name();
     nvinfer1::Dims scalar_shape;
@@ -1432,8 +1444,11 @@ tensorflow::Status ConvertConst(Converter& ctx,
           scalar_shape.d[0] = weights_tensor.int_val_size();
           scalar_shape.type[0] = nvinfer1::DimensionType::kSPATIAL;
         } else {
-          LOG(FATAL) << "Broadcast on weights only supports kCHANNEL and"
-                     << " kUNIFORM, at: " << node_def.name();
+          LOG(WARNING) << "Broadcast on weights only supports kCHANNEL and"
+                       << " kUNIFORM, at: " << node_def.name();
+          string err_str("Broadcast method is not supported for '");
+          StrAppend(&err_str, node_def.name(), "' of type ", node_def.op());
+          return tensorflow::errors::InvalidArgument(err_str);
         }
       }
     } else {
@@ -1447,62 +1462,23 @@ tensorflow::Status ConvertConst(Converter& ctx,
         scalar_shape.type[i] = nvinfer1::DimensionType::kSPATIAL;
       }
     }
-    if (ctx.isFP16()) {
-      auto dtype_new = tensorflow::DataType::DT_HALF;
-      size_t len_data = tensorflow::DataTypeSize(dtype_new);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      tensorflow::Tensor temp_tensor(tensorflow::DT_HALF, tensor.shape());
-      TTypes<Eigen::half>::Flat half_tensor = temp_tensor.flat<Eigen::half>();
-      Eigen::DefaultDevice defd;
-      switch (dtype) {
-        case (tensorflow::DT_INT32): {
-          half_tensor.device(defd) =
-              tensor.flat<int32>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT16): {
-          half_tensor.device(defd) =
-              tensor.flat<int16>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_INT8): {
-          half_tensor.device(defd) =
-              tensor.flat<int8>().template cast<Eigen::half>();
-          break;
-        }
-        case (tensorflow::DT_UINT8): {
-          half_tensor.device(defd) =
-              tensor.flat<uint8>().template cast<Eigen::half>();
-          break;
-        }
-        default:
-          return tensorflow::errors::InvalidArgument(
-              "Datatype " + tensorflow::DataTypeString(dtype) +
-              " for FP16 conversion");
-          break;
-      };
-      memcpy(dst, half_tensor.data(), len_data);  // store into weight store
-      weights = TRT_ShapedWeights(dtype_new, dst, scalar_shape);
-    } else {
-      size_t len_data = tensorflow::DataTypeSize(dtype);
-      for (int i = 0; i < scalar_shape.nbDims; i++)
-        len_data *= scalar_shape.d[i];
-      size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
-      len_data = std::max(len_data, len_tensor);
-      ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
-      void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
-      std::vector<int32> tensor_data(
-          weights_tensor.int_val().begin(),
-          weights_tensor.int_val()
-              .end());  //  make a local copy first to flatten
-                        //  doesn't have to be contiguous
-      memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
-      weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
-    }
+    //  we should not have converted //if (ctx.isFP16()) {
+    size_t len_data = tensorflow::DataTypeSize(dtype);
+    for (int i = 0; i < scalar_shape.nbDims; i++) len_data *= scalar_shape.d[i];
+    size_t len_tensor = weights_tensor.int_val_size() * sizeof(int32);
+    len_data = std::max(len_data, len_tensor);
+    ctx.weight_store()->store_.push_back(std::vector<uint8_t>(len_data));
+    void* dst = static_cast<void*>(&(ctx.weight_store()->store_.back()[0]));
+    std::vector<int32> tensor_data(
+        weights_tensor.int_val().begin(),
+        weights_tensor.int_val().end());  //  make a local copy first to flatten
+                                          //  doesn't have to be contigous
+    memcpy(dst, tensor_data.data(), len_tensor);  // store into weight store
+    weights = TRT_ShapedWeights(dtype, dst, scalar_shape);
   } else if (!weights_tensor.tensor_content().empty()) {
+    //  obsolete method.
+    //  After optimization path, we do not see weights in this format.
+    //  fp16 conversion technically should be needed here.
     VLOG(2) << "TENSOR!!!" << node_def.name();
     const auto& content = weights_tensor.tensor_content();
 
@@ -1784,8 +1760,6 @@ tensorflow::Status ConvertConcat(Converter& ctx,
   TRT_ShapedWeights axis = inputs.at(input_size).weights();
 
   TFAttrs attrs(node_def);
-  // auto attr_size = attrs.at("N")->i();
-  // auto data_type = attrs.get<nvinfer1::DataType>("T");
   auto index_type = attrs.get<tensorflow::DataType>("Tidx");
 
   // TODO(jie): handle data type
@@ -1875,71 +1849,103 @@ tensorflow::Status ConvertFusedBatchNorm(
         "only is_training=false is supported, at " + node_def.name());
   }
   nvinfer1::ITensor const* tensor = inputs.at(0).tensor();
-  TRT_ShapedWeights scale_weights = inputs.at(1).weights();
-  TRT_ShapedWeights offset_weights = inputs.at(2).weights();
-  TRT_ShapedWeights mean_weights = inputs.at(3).weights();
-  TRT_ShapedWeights variance_weights = inputs.at(4).weights();
-  TRT_ShapedWeights dummy_power_weights(scale_weights.type_);
-  TRT_ShapedWeights combined_scale_weights =
-      ctx.get_temp_weights_like(scale_weights);
-  TRT_ShapedWeights combined_offset_weights =
-      ctx.get_temp_weights_like(offset_weights);
-  size_t nweight = scale_weights.count();
-  if ((scale_weights.type_ == offset_weights.type_) &&
-      (mean_weights.type_ == variance_weights.type_) &&
-      (scale_weights.type_ == variance_weights.type_)) {
-    if ((scale_weights.type_ != tensorflow::DataType::DT_FLOAT) &&
-        (scale_weights.type_ != tensorflow::DataType::DT_HALF)) {
+
+  //  Check parameter types
+  auto parameter_type = inputs.at(1).weights().type_;
+  if ((parameter_type != tensorflow::DataType::DT_FLOAT) &&
+      (parameter_type != tensorflow::DataType::DT_HALF)) {
+    return tensorflow::errors::Unimplemented(
+        "only float32 or float16 weight data type is supported, for node " +
+        node_def.name() + " got " + tensorflow::DataTypeString(parameter_type));
+  }
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().type_ != parameter_type) {
       return tensorflow::errors::Unimplemented(
-          "only float32 or float16 weight data type is supported, for node " +
-          node_def.name() + " got " +
-          tensorflow::DataTypeString(scale_weights.type_));
+          "Inconsistent parameter type for batchnormis not supported, at: " +
+          node_def.name());
     }
-    if (scale_weights.type_ == tensorflow::DT_FLOAT) {
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale = (static_cast<float const*>(scale_weights.GetValues()))[i];
-        float offset =
-            (static_cast<float const*>(offset_weights.GetValues()))[i];
-        float mean = (static_cast<float const*>(mean_weights.GetValues()))[i];
-        float variance =
-            (static_cast<float const*>(variance_weights.GetValues()))[i];
-        float& combined_scale_ref = const_cast<float*>(
-            static_cast<float const*>(combined_scale_weights.GetValues()))[i];
-        float& combined_offset_ref = const_cast<float*>(
-            static_cast<float const*>(combined_offset_weights.GetValues()))[i];
-        combined_scale_ref = scale / sqrtf(variance + epsilon);
-        combined_offset_ref = offset - mean * combined_scale_ref;
-      }
-    } else {
-      const Eigen::half* scale_vals =
-          (static_cast<Eigen::half const*>(scale_weights.GetValues()));
-      const Eigen::half* off_vals =
-          (static_cast<Eigen::half const*>(offset_weights.GetValues()));
-      const Eigen::half* mean_vals =
-          (static_cast<Eigen::half const*>(mean_weights.GetValues()));
-      const Eigen::half* variance_vals =
-          (static_cast<Eigen::half const*>(variance_weights.GetValues()));
-      Eigen::half* comb_scale_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
-      Eigen::half* comb_off_vals = const_cast<Eigen::half*>(
-          static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
-      for (size_t i = 0; i < nweight; ++i) {
-        float scale(scale_vals[i]);
-        float offset(off_vals[i]);
-        float mean(mean_vals[i]);
-        float variance(variance_vals[i]);
-        float combined_scale_ref = scale / sqrtf(variance + epsilon);
-        comb_scale_vals[i] = Eigen::half(combined_scale_ref);
-        float combined_offset_ref = offset - mean * combined_scale_ref;
-        comb_off_vals[i] = Eigen::half(combined_offset_ref);
+  }
+
+  TRT_ShapedWeights dummy_power_weights(parameter_type);
+  size_t nweight = 0;
+  for (int i = 1; i < 5; i++) {
+    nweight = std::max(nweight, (size_t)inputs.at(i).weights().count());
+  }
+  TRT_ShapedWeights* ptr_shape_weights = nullptr;
+  for (int i = 1; i < 5; i++) {
+    if (inputs.at(i).weights().count() == nweight) {
+      ptr_shape_weights =
+          const_cast<TRT_ShapedWeights*>(&(inputs.at(i).weights()));
+    } else if (inputs.at(i).weights().count() != 1) {
+      return tensorflow::errors::InvalidArgument(
+          "Inconsistent batchnorm parameter count, at: " + node_def.name());
+    }
+  }
+  //  We could technically have two weights with different shape.
+  //  that requires two addScale op, arguably less performant
+  TRT_ShapedWeights combined_scale_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+  TRT_ShapedWeights combined_offset_weights =
+      ctx.get_temp_weights_like(*ptr_shape_weights);
+
+  const Eigen::half* cast_vals_array[4];
+  const float* vals_array[4];
+  for (int j = 0; j < 4; j++) {
+    cast_vals_array[j] =
+        static_cast<Eigen::half const*>(inputs.at(j + 1).weights().GetValues());
+    vals_array[j] =
+        static_cast<float const*>(inputs.at(j + 1).weights().GetValues());
+  }
+  Eigen::half* cast_combined_scale_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_scale_weights.GetValues()));
+  Eigen::half* cast_combined_offset_vals = const_cast<Eigen::half*>(
+      static_cast<Eigen::half const*>(combined_offset_weights.GetValues()));
+  float* combined_scale_vals = const_cast<float*>(
+      static_cast<float const*>(combined_scale_weights.GetValues()));
+  float* combined_offset_vals = const_cast<float*>(
+      static_cast<float const*>(combined_offset_weights.GetValues()));
+
+  for (size_t i = 0; i < nweight; ++i) {
+    float batchnorm_data[4];
+    for (int j = 0; j < 4; j++) {
+      if (inputs.at(j + 1).weights().count() != 1) {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][i];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][i]);
+        }
+      } else {
+        if (parameter_type == tensorflow::DT_FLOAT) {
+          batchnorm_data[j] = vals_array[j][0];
+        } else if (parameter_type == tensorflow::DT_HALF) {
+          batchnorm_data[j] =
+              Eigen::half_impl::half_to_float(cast_vals_array[j][0]);
+        }
       }
     }
+    float scale = batchnorm_data[0];
+    float offset = batchnorm_data[1];
+    float mean = batchnorm_data[2];
+    float variance = batchnorm_data[3];
+    float combined_scale_val = scale / sqrtf(variance + epsilon);
+    float combined_offset_val = offset - mean * combined_scale_val;
+    if (parameter_type == tensorflow::DT_FLOAT) {
+      combined_scale_vals[i] = combined_scale_val;
+      combined_offset_vals[i] = combined_offset_val;
+    } else if (parameter_type == tensorflow::DT_HALF) {
+      cast_combined_scale_vals[i] = Eigen::half(combined_scale_val);
+      cast_combined_offset_vals[i] = Eigen::half(combined_offset_val);
+    }
   }
-  nvinfer1::IScaleLayer* layer = ctx.network()->addScale(
-      *const_cast<nvinfer1::ITensor*>(tensor), nvinfer1::ScaleMode::kCHANNEL,
-      combined_offset_weights.GetWeightsForTRT(),
-      combined_scale_weights.GetWeightsForTRT(),
-      dummy_power_weights.GetWeightsForTRT());
+
+  nvinfer1::ScaleMode mode = nweight == 1 ? nvinfer1::ScaleMode::kUNIFORM
+                                          : nvinfer1::ScaleMode::kCHANNEL;
+  nvinfer1::IScaleLayer* layer =
+      ctx.network()->addScale(*const_cast<nvinfer1::ITensor*>(tensor), mode,
+                              combined_offset_weights.GetWeightsForTRT(),
+                              combined_scale_weights.GetWeightsForTRT(),
+                              dummy_power_weights.GetWeightsForTRT());
   nvinfer1::ITensor* output_tensor = layer->getOutput(0);
   outputs->push_back(TRT_TensorOrWeights(output_tensor));
   return tensorflow::Status::OK();
@@ -2050,6 +2056,7 @@ void Converter::register_op_converters() {
   op_registry_["Const"] = ConvertConst;
   // TODO(ben,jie): this is a temp hack.
   op_registry_["Identity"] = ConvertIdentity;  // Identity should be removed
+  op_registry_["Snapshot"] = ConvertIdentity;  // Snapshot should be removed
 
   // resnet_50_v1 slim implementation
   op_registry_["Add"] = ConvertBinary;
@@ -2143,8 +2150,11 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
   calib_res->thr_->join();
   delete calib_res->thr_;
   if (!calib_res->engine_) {
-    LOG(FATAL) << "Calibration failed!, engine is nullptr. Did you run "
+    LOG(ERROR) << "Calibration failed!, engine does not exist. Did you run "
                   "calibration graph?";
+    return tensorflow::errors::FailedPrecondition(
+        "Calibration graph needs to be executed on"
+        " calibration data before convertsion to inference graph");
   }
   auto weight_rmgr = trt_rm->getManager("WeightStore");
   TF_CHECK_OK(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
@@ -2181,7 +2191,7 @@ tensorflow::Status ConvertCalibrationNodeToEngineNode(
     return status;
   }
   auto trt_engine_node = graph.AddNode(engine_node, &status);
-  TF_CHECK_OK(status);
+  TF_RETURN_IF_ERROR(status);
   for (size_t i = 0; i < out_edges.size(); i++) {
     VLOG(1) << "Connecting trt_engine_node output " << i << " with "
             << out_edges.at(i)->dst()->name() << " port "
@@ -2279,6 +2289,12 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Data type conversion for input '" << node_name
+                   << "' failed";
+      return type_status;
+    }
     TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
 
     VLOG(2) << "accessing output index of: " << output_idx
@@ -2346,8 +2362,8 @@ tensorflow::Status InjectCalibrationNode(tensorrt::convert::SubGraphParams& s) {
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node'" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2504,7 +2520,11 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     input_dtypes.push_back(tf_dtype);
 
     nvinfer1::DataType dtype(nvinfer1::DataType::kFLOAT);
-    TF_CHECK_OK(ConvertDType(tf_dtype, &dtype));
+    auto type_status = ConvertDType(tf_dtype, &dtype);
+    if (type_status != tensorflow::Status::OK()) {
+      LOG(WARNING) << "Type conversion failed for " << node_name;
+      return type_status;
+    }
 
     VLOG(2) << "Accessing output index of: " << output_idx
             << ", at node: " << node_name
@@ -2515,8 +2535,12 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
 
     // TODO(jie): TRT 3.x only support 4 dimensional input tensor.
     //            update the code once TRT 4.0 comes out.
-    if (op_info.shape().dim_size() != 4)
-      return tensorflow::errors::Unimplemented("require 4 dimensional input");
+    if (op_info.shape().dim_size() != 4) {
+      string err_str = "Require 4 dimensional input.";
+      StrAppend(&err_str, " Got ", op_info.shape().dim_size(), " ",
+                shape_inference_node_name);
+      return tensorflow::errors::Unimplemented(err_str);
+    }
 
     for (int i = 1; i < op_info.shape().dim_size(); i++) {
       VLOG(2) << "dimension: " << i
@@ -2577,8 +2601,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
     output_names.push_back(tensor_name);
     auto tensor_or_weights = converter.get_tensor(tensor_name);
     if (!tensor_or_weights.is_tensor()) {
-      return tensorflow::errors::InvalidArgument(
-          "Output node is weights not tensor");
+      return tensorflow::errors::InvalidArgument("Output node '" + tensor_name +
+                                                 "' is weights not tensor");
     }
     nvinfer1::ITensor* tensor = tensor_or_weights.tensor();
     if (!tensor) {
@@ -2622,7 +2646,8 @@ tensorflow::Status ConvertSubGraphToTensorRTNodeDef(
   }
   TF_RETURN_IF_ERROR(weight_rmgr->Delete<tensorflow::tensorrt::TRTWeightStore>(
       engine_name, engine_name));
-  LOG(INFO) << "finished engine " << engine_name;
+  LOG(INFO) << "finished engine " << engine_name << " containing "
+            << s.subgraph_node_ids.size() << " nodes";
 
   // Build the TRT op
   tensorflow::NodeDefBuilder op_builder(engine_name, "TRTEngineOp");
diff --git a/tensorflow/contrib/tensorrt/segment/segment.cc b/tensorflow/contrib/tensorrt/segment/segment.cc
index 6193f0b0a1..8fc4697c51 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment.cc
@@ -80,13 +80,20 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> in_edges(dst->in_edges().begin(),
                                                 dst->in_edges().end());
   for (const tensorflow::Edge* in_edge : in_edges) {
-    if (in_edge->src() != src) {
-      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
-      if (e->src() == graph->source_node()) {
-        graph->AddEdge(e->src(), e->src_output(), src,
-                       tensorflow::Graph::kControlSlot);
-      } else {
-        graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+    if (in_edge->IsControlEdge()) {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        graph->AddControlEdge(e->src(), src);
+      }
+    } else {
+      if (in_edge->src() != src) {
+        tensorflow::Edge* e = const_cast<tensorflow::Edge*>(in_edge);
+        if (e->src() == graph->source_node()) {
+          graph->AddEdge(e->src(), e->src_output(), src,
+                         tensorflow::Graph::kControlSlot);
+        } else {
+          graph->AddEdge(e->src(), e->src_output(), src, 0 /* input index */);
+        }
       }
     }
   }
@@ -94,12 +101,19 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
   std::vector<const tensorflow::Edge*> out_edges(dst->out_edges().begin(),
                                                  dst->out_edges().end());
   for (const tensorflow::Edge* out_edge : out_edges) {
-    tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
-    if (e->dst() == graph->sink_node()) {
-      graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
-                     e->dst_input());
+    if (out_edge->IsControlEdge()) {
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      graph->AddControlEdge(src, e->dst());
     } else {
-      graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      tensorflow::Edge* e = const_cast<tensorflow::Edge*>(out_edge);
+      if (e->dst() == graph->sink_node()) {
+        VLOG(1) << " edge to sink node " << src->name() << " -> "
+                << e->dst()->name();
+        graph->AddEdge(src, tensorflow::Graph::kControlSlot, e->dst(),
+                       e->dst_input());
+      } else {
+        graph->AddEdge(src, 0 /* output index */, e->dst(), e->dst_input());
+      }
     }
   }
 
@@ -118,7 +132,7 @@ void ContractEdge(tensorflow::Edge* edge, tensorflow::Graph* graph,
 
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments) {
   // Create a Graph representation of the GraphDef.
   tensorflow::FunctionLibraryDefinition flib(tensorflow::OpRegistry::Global(),
@@ -136,7 +150,7 @@ tensorflow::Status SegmentGraph(
   for (int i = 0; i < graph.num_node_ids(); ++i) {
     tensorflow::Node* node = graph.FindNodeId(i);
     if (options.exclude_node_list.count(node->name()) != 0 ||
-        !candidate_fn(node->def())) {
+        !candidate_fn(node)) {
       node = nullptr;
     }
     node_segments.emplace_back(node);
@@ -155,7 +169,7 @@ tensorflow::Status SegmentGraph(
 
   for (const tensorflow::Node* node : order) {
     // All output nodes of 'node' have been visited...
-    VLOG(2) << "Trying node " << node->name();
+    VLOG(2) << "Trying node " << node->name() << " id=" << node->id();
 
     // 'node' must be a TRT candidate...
     if (node_segments[node->id()].Value() == nullptr) {
@@ -169,8 +183,12 @@ tensorflow::Status SegmentGraph(
     while (true) {
       std::set<const tensorflow::Edge*> contract_edges;
       for (const tensorflow::Edge* out_edge : node->out_edges()) {
-        VLOG(2) << "... out node " << out_edge->dst()->name();
-
+        VLOG(2) << "... out node " << out_edge->dst()->name() << " ( "
+                << out_edge->dst()->id() << " <- " << node->id() << " )";
+        if (out_edge->IsControlEdge()) {
+          VLOG(2) << "... ... Control Edge, Skipping";
+          continue;
+        }
         // Out node must be TRT candidate...
         if (node_segments[out_edge->dst()->id()].Value() == nullptr) {
           VLOG(2) << "... ... not a TRT candidate";
@@ -196,7 +214,8 @@ tensorflow::Status SegmentGraph(
         const tensorflow::Node* src = contract_edge->src();
         const tensorflow::Node* dst = contract_edge->dst();
 
-        VLOG(2) << "Merge " << src->name() << " <- " << dst->name();
+        VLOG(2) << "Merge " << src->name() << " <- " << dst->name() << " ("
+                << src->id() << " <- " << dst->id();
         node_segments[src->id()].Merge(&node_segments[dst->id()]);
 
         // Contracting the edge leaves disconnected graph edges.
diff --git a/tensorflow/contrib/tensorrt/segment/segment.h b/tensorflow/contrib/tensorrt/segment/segment.h
index ee6e2b3ed2..7e8685f44a 100644
--- a/tensorflow/contrib/tensorrt/segment/segment.h
+++ b/tensorflow/contrib/tensorrt/segment/segment.h
@@ -20,10 +20,12 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/framework/graph.pb.h"
+#include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace tensorflow {
+
 namespace tensorrt {
 namespace segment {
 
@@ -46,7 +48,7 @@ struct SegmentOptions {
 // @return the status.
 tensorflow::Status SegmentGraph(
     const tensorflow::GraphDef& gdef,
-    const std::function<bool(const tensorflow::NodeDef&)>& candidate_fn,
+    const std::function<bool(const tensorflow::Node*)>& candidate_fn,
     const SegmentOptions& options, SegmentNodesVector* segments);
 
 }  // namespace segment
diff --git a/tensorflow/contrib/tensorrt/segment/segment_test.cc b/tensorflow/contrib/tensorrt/segment/segment_test.cc
index 74cbc5f2b3..7ddabec268 100644
--- a/tensorflow/contrib/tensorrt/segment/segment_test.cc
+++ b/tensorflow/contrib/tensorrt/segment/segment_test.cc
@@ -35,7 +35,7 @@ class SegmentTest : public ::testing::Test {
   TF_Operation* Add(TF_Operation* l, TF_Operation* r, TF_Graph* graph,
                     TF_Status* s, const char* name);
 
-  std::function<bool(const NodeDef&)> MakeCandidateFn(
+  std::function<bool(const Node*)> MakeCandidateFn(
       const std::set<string>& node_names);
 
  protected:
@@ -60,10 +60,10 @@ bool SegmentTest::GetGraphDef(TF_Graph* graph,
   return ret;
 }
 
-std::function<bool(const NodeDef&)> SegmentTest::MakeCandidateFn(
+std::function<bool(const Node*)> SegmentTest::MakeCandidateFn(
     const std::set<string>& node_names) {
-  return [node_names](const NodeDef& node) -> bool {
-    return node_names.find(node.name()) != node_names.end();
+  return [node_names](const Node* node) -> bool {
+    return node_names.find(node->name()) != node_names.end();
   };
 }
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
index ff140efd48..4f6527a546 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/ar_model.py
@@ -70,7 +70,7 @@ class ARModel(model.TimeSeriesModel):
       input_window_size: Number of past time steps of data to look at when doing
         the regression.
       output_window_size: Number of future time steps to predict. Note that
-        setting it to > 1 empiricaly seems to give a better fit.
+        setting it to > 1 empirically seems to give a better fit.
       num_features: number of input features per time step.
       num_time_buckets: Number of buckets into which to divide (time %
         periodicity) for generating time based features.
diff --git a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
index 23452a81c3..26793c80bf 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/math_utils.py
@@ -185,7 +185,7 @@ def batch_matrix_pow(matrices, powers):
                     { matmul(A, power(matmul(A, A), (p - 1) / 2)) for odd p
       power(A, 0) = I
 
-    The power(A, 0) = I case is handeled by starting with accumulator set to the
+    The power(A, 0) = I case is handled by starting with accumulator set to the
     identity matrix; matrices with zero residual powers are passed through
     unchanged.
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
index 1afc58cfb2..6746dd7b43 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/varma.py
@@ -107,7 +107,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state transition matrix. It has shape
-        [self.state_dimendion, self.state_dimension].
+        [self.state_dimension, self.state_dimension].
     """
     # Pad any unused AR blocks with zeros. The extra state is necessary if
     # ma_order >= ar_order.
@@ -127,7 +127,7 @@ class VARMA(state_space_model.StateSpaceModel):
 
     Returns:
       the state noise transform matrix. It has shape
-        [self.state_dimendion, self.num_features].
+        [self.state_dimension, self.num_features].
     """
     # Noise is broadcast, through the moving average coefficients, to
     # un-observed parts of the latent state.
diff --git a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
index 51d91399f8..e667c328ae 100644
--- a/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
+++ b/tensorflow/core/api_def/base_api/api_def_MatrixSolveLs.pbtxt
@@ -49,14 +49,14 @@ in the batch:
 If `fast` is `True`, then the solution is computed by solving the normal
 equations using Cholesky decomposition. Specifically, if \\(m \ge n\\) then
 \\(X = (A^H A + \lambda I)^{-1} A^H B\\), which solves the least-squares
-problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 +
-\lambda ||Z||_F^2\\). If \\(m \lt n\\) then `output` is computed as
+problem \\(X = \mathrm{argmin}_{Z \in \Re^{n \times k} } ||A Z - B||_F^2 + \lambda ||Z||_F^2\\). 
+If \\(m \lt n\\) then `output` is computed as
 \\(X = A^H (A A^H + \lambda I)^{-1} B\\), which (for \\(\lambda = 0\\)) is the
 minimum-norm solution to the under-determined linear system, i.e.
 \\(X = \mathrm{argmin}_{Z \in \mathbb{C}^{n \times k} } ||Z||_F^2 \\),
 subject to \\(A Z = B\\). Notice that the fast path is only numerically stable
 when \\(A\\) is numerically full rank and has a condition number
-\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or\\(\lambda\\) is
+\\(\mathrm{cond}(A) \lt \frac{1}{\sqrt{\epsilon_{mach} } }\\) or \\(\lambda\\) is
 sufficiently large.
 
 If `fast` is `False` an algorithm based on the numerically robust complete
diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 43a909466e..829c19204a 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-constexpr const char* MklCPUAllocator::kMaxLimitStr;
-constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
-
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 623248b6ce..2fb17c2b02 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -1210,7 +1210,7 @@ Status ConcatV2Shape(InferenceContext* c) {
                            c->num_inputs() - 1 /* dim_index */);
 }
 
-Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index) {
   ShapeHandle shape_x = c->input(0);
   ShapeHandle shape_y = c->input(1);
   if (!c->RankKnown(shape_x) || !c->RankKnown(shape_y)) {
@@ -1272,7 +1272,7 @@ Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
     }
   }
 
-  c->set_output(0, c->MakeShape(dims));
+  c->set_output(output_index, c->MakeShape(dims));
   return Status::OK();
 }
 
diff --git a/tensorflow/core/framework/common_shape_fns.h b/tensorflow/core/framework/common_shape_fns.h
index 293c40e04d..789746b403 100644
--- a/tensorflow/core/framework/common_shape_fns.h
+++ b/tensorflow/core/framework/common_shape_fns.h
@@ -265,9 +265,15 @@ Status ConcatShape(shape_inference::InferenceContext* c,
 // Shape function for concat operations.
 Status ConcatV2Shape(shape_inference::InferenceContext* c);
 
+// Shape function for binary operators that broadcast their inputs
+// and with output to output_index.
+Status BroadcastBinaryOpOutputShapeFn(InferenceContext* c, int output_index);
+
 // Shape function for binary operators that broadcast their inputs.
 // Tested by ops/math_ops_test.cc.
-Status BroadcastBinaryOpShapeFn(InferenceContext* c);
+inline Status BroadcastBinaryOpShapeFn(InferenceContext* c) {
+  return BroadcastBinaryOpOutputShapeFn(c, 0);
+}
 
 // Shape function for random operations.
 Status RandomShape(shape_inference::InferenceContext* c);
diff --git a/tensorflow/core/framework/shape_inference.h b/tensorflow/core/framework/shape_inference.h
index e3cc848a16..accc587000 100644
--- a/tensorflow/core/framework/shape_inference.h
+++ b/tensorflow/core/framework/shape_inference.h
@@ -317,6 +317,7 @@ class InferenceContext {
     input_tensors_as_shapes_ = input_tensors_as_shapes;
   }
 
+  ShapeHandle output(int64 idx) const { return outputs_[idx]; }
   void set_output(int idx, ShapeHandle shape) { outputs_[idx] = shape; }
   Status set_output(StringPiece output_name,
                     const std::vector<ShapeHandle>& shapes);
diff --git a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
index 333a6570dc..62aafa7930 100644
--- a/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
+++ b/tensorflow/core/kernels/mkl_fused_batch_norm_op.cc
@@ -933,7 +933,7 @@ class MklFusedBatchNormOp : public OpKernel {
   bool is_training_;
   T* mean_values_;
   T* variance_values_;
-  size_t depth_;  // batch normalization is done for per channel.
+  int depth_;  // batch normalization is done for per channel.
 
   void ExtractParams(OpKernelContext* context) {
     const Tensor& input = MklGetInput(context, 0);
diff --git a/tensorflow/core/kernels/segment_reduction_ops.h b/tensorflow/core/kernels/segment_reduction_ops.h
index 4abfbfb1a6..7badc00572 100644
--- a/tensorflow/core/kernels/segment_reduction_ops.h
+++ b/tensorflow/core/kernels/segment_reduction_ops.h
@@ -23,6 +23,13 @@ limitations under the License.
 // non-GPU targets. This only breaks in clang, because it's more strict for
 // template code and CudaAtomicMax is used in template context.
 
+// This file requires the following include because it uses CudaAtomicMax:
+// #include "tensorflow/core/util/cuda_kernel_helper.h"
+
+// Unfortunately we can't add the #include, since it breaks compilation for
+// non-GPU targets. This only breaks in clang, because it's more strict for
+// template code and CudaAtomicMax is used in template context.
+
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
diff --git a/tensorflow/core/kernels/snapshot_op.cc b/tensorflow/core/kernels/snapshot_op.cc
index 50157d5d48..fe04dcf72e 100644
--- a/tensorflow/core/kernels/snapshot_op.cc
+++ b/tensorflow/core/kernels/snapshot_op.cc
@@ -22,6 +22,26 @@ limitations under the License.
 
 namespace tensorflow {
 typedef Eigen::ThreadPoolDevice CPUDevice;
+typedef Eigen::GpuDevice GPUDevice;
+
+template <typename Device, typename Scalar>
+class SnapshotOp : public OpKernel {
+ public:
+  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
+
+  void Compute(OpKernelContext* context) override {
+    const Tensor& input = context->input(0);
+    Tensor* output = nullptr;
+    // Try to use buffer forwarding to avoid an explicit copy.
+    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
+                                {0}, 0, input.shape(), &output));
+    if (!output->SharesBufferWith(input)) {
+      functor::Snapshot<Device, Scalar> functor;
+      functor(context->eigen_device<Device>(), input.flat<Scalar>(),
+              output->flat<Scalar>());
+    }
+  }
+};
 
 #define REGISTER_KERNEL(TYPE)                                        \
   REGISTER_KERNEL_BUILDER(                                           \
@@ -31,6 +51,16 @@ typedef Eigen::ThreadPoolDevice CPUDevice;
 TF_CALL_POD_TYPES(REGISTER_KERNEL);
 #undef REGISTER_KERNEL
 
+#if GOOGLE_CUDA
+#define REGISTER_KERNEL(TYPE)                                        \
+  REGISTER_KERNEL_BUILDER(                                           \
+      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
+      SnapshotOp<GPUDevice, TYPE>);
+
+TF_CALL_POD_TYPES(REGISTER_KERNEL);
+#undef REGISTER_KERNEL
+#endif
+
 #if TENSORFLOW_USE_SYCL
 typedef Eigen::SyclDevice SyclDevice;
 #define REGISTER_SYCL_KERNEL(TYPE)                                    \
diff --git a/tensorflow/core/kernels/snapshot_op.h b/tensorflow/core/kernels/snapshot_op.h
index b94834f159..a18065d42b 100644
--- a/tensorflow/core/kernels/snapshot_op.h
+++ b/tensorflow/core/kernels/snapshot_op.h
@@ -26,29 +26,19 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 
 namespace tensorflow {
+namespace functor {
 
+// Functor used by SnapshotOp.
 template <typename Device, typename Scalar>
-class SnapshotOp : public OpKernel {
- public:
-  explicit SnapshotOp(OpKernelConstruction* context) : OpKernel(context) {}
-
-  void Compute(OpKernelContext* context) override {
-    const Tensor& input = context->input(0);
-    Tensor* output = nullptr;
-    // Try to use buffer forwarding to avoid an explicit copy.
-    OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 0, input.shape(), &output));
-    if (!output->SharesBufferWith(input)) {
-      // We had to allocate a new buffer since the refcount on the input was
-      // greater than 1. Copy the input to the new buffer.
-      const Device& device = context->eigen_device<Device>();
-      device.memcpy(output->template flat<Scalar>().data(),
-                    input.template flat<Scalar>().data(),
-                    input.NumElements() * sizeof(Scalar));
-    }
+struct Snapshot {
+  void operator()(const Device& device,
+                  typename TTypes<Scalar>::ConstTensor input,
+                  typename TTypes<Scalar>::Tensor output) {
+    device.memcpy(output.data(), input.data(), input.size() * sizeof(Scalar));
   }
 };
 
+}  // namespace functor
 }  // namespace tensorflow
 
 #endif  // TENSORFLOW_KERNELS_SNAPSHOT_OP_H_
diff --git a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
index 52070be838..e4e3bd5220 100644
--- a/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/snapshot_op_gpu.cu.cc
@@ -24,13 +24,10 @@ limitations under the License.
 namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 
-#define REGISTER_KERNEL(TYPE)                                        \
-  REGISTER_KERNEL_BUILDER(                                           \
-      Name("Snapshot").Device(DEVICE_GPU).TypeConstraint<TYPE>("T"), \
-      SnapshotOp<GPUDevice, TYPE>);
+// Definition of the GPU implementations declared in softsign_op.cc.
+#define DEFINE_GPU_KERNELS(T) template struct functor::Snapshot<GPUDevice, T>;
 
-TF_CALL_POD_TYPES(REGISTER_KERNEL);
-#undef REGISTER_KERNEL
+TF_CALL_POD_TYPES(DEFINE_GPU_KERNELS);
 
 }  // namespace tensorflow
 
diff --git a/tensorflow/core/kernels/xent_op.cc b/tensorflow/core/kernels/xent_op.cc
index a6a71fdfaf..9a3612bd72 100644
--- a/tensorflow/core/kernels/xent_op.cc
+++ b/tensorflow/core/kernels/xent_op.cc
@@ -17,12 +17,14 @@ limitations under the License.
 
 #define EIGEN_USE_THREADS
 
-#include "tensorflow/core/kernels/xent_op.h"
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/kernels/xent_op.h"
+#include "tensorflow/core/util/bcast.h"
 
 namespace tensorflow {
 
@@ -41,37 +43,56 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
   void Compute(OpKernelContext* context) override {
     const Tensor& logits_in = context->input(0);
     const Tensor& labels_in = context->input(1);
-    OP_REQUIRES(context, logits_in.IsSameSize(labels_in),
-                errors::InvalidArgument(
-                    "logits and labels must be same size: logits_size=",
-                    logits_in.shape().DebugString(),
-                    " labels_size=", labels_in.shape().DebugString()));
-    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(logits_in.shape()),
-                errors::InvalidArgument("logits must be 2-dimensional"));
-    // As we already tested that both inputs have the same shape no need to
-    // check that "labels" is a matrix too.
+
+    TensorShape shape_in = logits_in.shape();
+
+    BCast bcast(BCast::FromShape(logits_in.shape()),
+                BCast::FromShape(labels_in.shape()));
+    if (!logits_in.IsSameSize(labels_in)) {
+      OP_REQUIRES(context, bcast.IsValid(),
+                  errors::InvalidArgument(
+                      "logits and labels must be broadcastable: logits_size=",
+                      logits_in.shape().DebugString(),
+                      " labels_size=", labels_in.shape().DebugString()));
+      shape_in = BCast::ToShape(bcast.output_shape());
+    }
+    OP_REQUIRES(context, TensorShapeUtils::IsMatrix(shape_in),
+                errors::InvalidArgument("logits and labels must be beither "
+                                        "2-dimensional, or roadcasted to "
+                                        "2-dimensional"));
 
     // loss is 1-D (one per example), and size is batch_size.
 
     Tensor scratch;
     OP_REQUIRES_OK(
         context, context->allocate_temp(DataTypeToEnum<T>::value,
-                                        TensorShape({logits_in.dim_size(0), 1}),
+                                        TensorShape({shape_in.dim_size(0), 1}),
                                         &scratch));
 
     Tensor* loss_out = nullptr;
     OP_REQUIRES_OK(context,
                    context->allocate_output(
-                       0, TensorShape({logits_in.dim_size(0)}), &loss_out));
+                       0, TensorShape({shape_in.dim_size(0)}), &loss_out));
     Tensor* back_out = nullptr;
     // Try to reuse the logits_in buffer for the backprop output.
     OP_REQUIRES_OK(context, context->forward_input_or_allocate_output(
-                                {0}, 1, logits_in.shape(), &back_out));
-    if (logits_in.dim_size(0) > 0) {
+                                {0}, 1, shape_in, &back_out));
+    if (shape_in.dim_size(0) > 0) {
       functor::XentFunctor<Device, T> functor;
-      functor(context->eigen_device<Device>(), logits_in.matrix<T>(),
-              labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
-              back_out->matrix<T>());
+      if (logits_in.IsSameSize(labels_in)) {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1},
+                Eigen::array<Eigen::DenseIndex, 2>{1, 1}, logits_in.matrix<T>(),
+                labels_in.matrix<T>(), scratch.matrix<T>(), loss_out->vec<T>(),
+                back_out->matrix<T>());
+      } else {
+        functor(context->eigen_device<Device>(), shape_in.AsEigenDSizes<2>(),
+                BCast::ToIndexArray<2>(bcast.x_bcast()),
+                BCast::ToIndexArray<2>(bcast.y_bcast()),
+                logits_in.template shaped<T, 2>(bcast.x_reshape()),
+                labels_in.template shaped<T, 2>(bcast.y_reshape()),
+                scratch.matrix<T>(), loss_out->vec<T>(), back_out->matrix<T>());
+      }
     }
   }
 };
@@ -81,13 +102,17 @@ class SoftmaxXentWithLogitsOp : public OpKernel {
 namespace functor {
 template <typename Device, typename T>
 struct XentFunctorBase {
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device& d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2>& shape,
+                  const Eigen::array<Eigen::DenseIndex, 2>& logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2>& labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<Device, T>::Compute(d, logits, labels, scratch, loss,
-                                      backprop);
+    XentEigenImpl<Device, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                      logits, labels, scratch, loss, backprop);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op.h b/tensorflow/core/kernels/xent_op.h
index e689fca7ff..87be17fca9 100644
--- a/tensorflow/core/kernels/xent_op.h
+++ b/tensorflow/core/kernels/xent_op.h
@@ -18,6 +18,7 @@ limitations under the License.
 // Functor definition for XentOp, must be compilable by nvcc.
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
+
 #include "tensorflow/core/framework/tensor_types.h"
 
 namespace tensorflow {
@@ -33,7 +34,11 @@ struct XentFunctor {
   // scratch: temporary tensor, dims: batch_size, 1
   // loss: output tensor for the loss, dims: batch_size.
   // backprop: output tensor for the backprop, dims: batch_size, num_classes.
-  void operator()(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const Device &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
@@ -45,7 +50,11 @@ struct XentFunctor {
 // specializations for both device types.
 template <typename Device, typename T>
 struct XentEigenImpl {
-  static void Compute(const Device& d, typename TTypes<T>::ConstMatrix logits,
+  static void Compute(const Device &d,
+                      const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                      const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                      const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                      typename TTypes<T>::ConstMatrix logits,
                       typename TTypes<T>::ConstMatrix labels,
                       typename TTypes<T>::Matrix scratch,
                       typename TTypes<T>::Vec loss,
@@ -57,8 +66,8 @@ struct XentEigenImpl {
     const int kBatchDim = 0;
     const int kClassDim = 1;
 
-    const int batch_size = logits.dimension(kBatchDim);
-    const int num_classes = logits.dimension(kClassDim);
+    const int batch_size = shape[kBatchDim];
+    const int num_classes = shape[kClassDim];
 
 // These arrays are used to reduce along the class dimension, and broadcast
 // the resulting value to all classes.
@@ -84,10 +93,12 @@ struct XentEigenImpl {
 #endif
 
     // max_logits along classes.
-    scratch.reshape(batch_only).device(d) = logits.maximum(along_class);
+    scratch.reshape(batch_only).device(d) =
+        logits.broadcast(logits_bcast).maximum(along_class);
 
     // logits - max_logits.
-    backprop.device(d) = logits - scratch.broadcast(one_by_class);
+    backprop.device(d) =
+        logits.broadcast(logits_bcast) - scratch.broadcast(one_by_class);
 
     // sum(exp(logits - max_logits)) along classes.
     scratch.reshape(batch_only).device(d) = backprop.exp().sum(along_class);
@@ -99,15 +110,15 @@ struct XentEigenImpl {
     //  sum(-labels *
     //     ((logits - max_logits) - log(sum(exp(logits - max_logits)))))
     //  along classes
-    loss.device(d) =
-        (labels * (scratch.log().eval().broadcast(one_by_class) - backprop))
-            .eval()
-            .sum(along_class);
+    loss.device(d) = (labels.broadcast(labels_bcast) *
+                      (scratch.log().eval().broadcast(one_by_class) - backprop))
+                         .eval()
+                         .sum(along_class);
 
     // backprop: prob - labels, where
     //   prob = exp(logits - max_logits) / sum(exp(logits - max_logits))
-    backprop.device(d) =
-        (backprop.exp() / scratch.broadcast(one_by_class)) - labels;
+    backprop.device(d) = (backprop.exp() / scratch.broadcast(one_by_class)) -
+                         labels.broadcast(labels_bcast);
   }
 };
 
diff --git a/tensorflow/core/kernels/xent_op_gpu.cu.cc b/tensorflow/core/kernels/xent_op_gpu.cu.cc
index 05ee7da490..2c0c0b3a02 100644
--- a/tensorflow/core/kernels/xent_op_gpu.cu.cc
+++ b/tensorflow/core/kernels/xent_op_gpu.cu.cc
@@ -31,12 +31,17 @@ typedef Eigen::GpuDevice GPUDevice;
 namespace functor {
 template <typename T>
 struct XentFunctor<GPUDevice, T> {
-  void operator()(const GPUDevice& d, typename TTypes<T>::ConstMatrix logits,
+  void operator()(const GPUDevice &d,
+                  const Eigen::DSizes<Eigen::DenseIndex, 2> &shape,
+                  const Eigen::array<Eigen::DenseIndex, 2> &logits_bcast,
+                  const Eigen::array<Eigen::DenseIndex, 2> &labels_bcast,
+                  typename TTypes<T>::ConstMatrix logits,
                   typename TTypes<T>::ConstMatrix labels,
                   typename TTypes<T>::Matrix scratch,
                   typename TTypes<T>::Vec loss,
                   typename TTypes<T>::Matrix backprop) {
-    XentEigenImpl<GPUDevice, T>::Compute(d, logits, labels, scratch, loss,
+    XentEigenImpl<GPUDevice, T>::Compute(d, shape, logits_bcast, labels_bcast,
+                                         logits, labels, scratch, loss,
                                          backprop);
   }
 };
diff --git a/tensorflow/core/ops/array_ops.cc b/tensorflow/core/ops/array_ops.cc
index af8afc90f5..62ce70eb6b 100644
--- a/tensorflow/core/ops/array_ops.cc
+++ b/tensorflow/core/ops/array_ops.cc
@@ -795,11 +795,35 @@ REGISTER_OP("ReverseV2")
       ShapeHandle input = c->input(0);
       ShapeHandle axis;
       TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 1, &axis));
-      // TODO(aselle): if input(0)'s dimension is known we could validate axis
       if (c->Rank(input) > 8) {
         return errors::InvalidArgument(
             "reverse does not work on tensors with more than 8 dimensions");
       }
+      const Tensor* axis_tensor = c->input_tensor(1);
+      if (axis_tensor != nullptr && c->RankKnown(input)) {
+        int32 rank = c->Rank(input);
+        std::vector<int64> axis_value;
+        if (axis_tensor->dtype() == DT_INT32) {
+          axis_value = AsInt64<int32>(axis_tensor, axis_tensor->NumElements());
+        } else {
+          axis_value = AsInt64<int64>(axis_tensor, axis_tensor->NumElements());
+        }
+        std::vector<bool> axes_dense(c->Rank(input), false);
+        for (int i = 0; i < axis_value.size(); i++) {
+          int64 canonical_axis =
+              axis_value[i] < 0 ? rank + axis_value[i] : axis_value[i];
+          if (canonical_axis < 0 || canonical_axis >= rank) {
+            return errors::InvalidArgument("'axis'[", i, "] = ", axis_value[i],
+                                           " is out of valid range [", 0, ", ",
+                                           rank - 1);
+          }
+          if (axes_dense[canonical_axis]) {
+            return errors::InvalidArgument("axis ", canonical_axis,
+                                           " specified more than once.");
+          }
+          axes_dense[canonical_axis] = true;
+        }
+      }
       c->set_output(0, input);
       return Status::OK();
     });
diff --git a/tensorflow/core/ops/nn_ops.cc b/tensorflow/core/ops/nn_ops.cc
index 1f4e9753c3..6c2fc60bab 100644
--- a/tensorflow/core/ops/nn_ops.cc
+++ b/tensorflow/core/ops/nn_ops.cc
@@ -1062,12 +1062,27 @@ REGISTER_OP("SoftmaxCrossEntropyWithLogits")
     .Attr("T: {half, bfloat16, float, double}")
     .SetShapeFn([](InferenceContext* c) {
       ShapeHandle input;
-      TF_RETURN_IF_ERROR(c->WithRank(c->input(0), 2, &input));
-      TF_RETURN_IF_ERROR(c->Merge(input, c->input(1), &input));
+      if (c->WithRank(c->input(0), 2, &input) == Status::OK() &&
+          c->Merge(input, c->input(1), &input) == Status::OK()) {
+        DimensionHandle batch_size = c->Dim(input, 0);
+        c->set_output(0, c->Vector(batch_size));
+        c->set_output(1, input);
+        return Status::OK();
+      }
+      TF_RETURN_IF_ERROR(BroadcastBinaryOpOutputShapeFn(c, 1));
 
-      DimensionHandle batch_size = c->Dim(input, 0);
+      if (!c->RankKnown(c->output(1))) {
+        return errors::InvalidArgument(
+            "Shape must be broadcasted with rank 2, but is rank is unknown.");
+      }
+
+      if (c->Rank(c->output(1)) != 2) {
+        return errors::InvalidArgument(
+            "Shape must be broadcasted with rank 2, but is rank ",
+            c->Rank(c->output(1)));
+      }
+      DimensionHandle batch_size = c->Dim(c->output(1), 0);
       c->set_output(0, c->Vector(batch_size));
-      c->set_output(1, input);
       return Status::OK();
     });
 
diff --git a/tensorflow/core/ops/nn_ops_test.cc b/tensorflow/core/ops/nn_ops_test.cc
index 1b17a7cda6..289b953055 100644
--- a/tensorflow/core/ops/nn_ops_test.cc
+++ b/tensorflow/core/ops/nn_ops_test.cc
@@ -410,10 +410,18 @@ TEST(NNOpsTest, SoftmaxCrossEntropyWithLogits_ShapeFn) {
   INFER_OK(op, "[1,?];[?,2]", "[d0_0];[d0_0,d0_1|d1_1]");
   INFER_OK(op, "[?,2];[1,2]", "[d1_0];in1");
 
-  INFER_ERROR("Dimension 0 in both shapes must be equal, but are 1 and 2", op,
-              "[1,?];[2,?]");
-  INFER_ERROR("Shape must be rank 2 but is rank 3", op, "[1,2,3];?");
-  INFER_ERROR("Shapes must be equal rank, but are 2 and 3", op, "?;[1,2,3]");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "[1,2,3];?");
+  INFER_ERROR("Shape must be broadcasted with rank 2", op, "?;[1,2,3]");
+
+  // Broadcast example
+  // [1,4] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,4];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [2,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[2,1]", "[d0_0];[d0_0|d1_0,d0_1]");
+  // [1,?] and [2,4] are broadcasted to [2,4]
+  INFER_OK(op, "[1,?];[2,4]", "[d1_0];[d1_0,d0_1|d1_1]");
+  // [2,4] and [?,1] are broadcasted to [2,4]
+  INFER_OK(op, "[2,4];[?,1]", "[d0_0];[d0_0|d1_0,d0_1]");
 }
 
 TEST(NNOpsTest, SparseSoftmaxCrossEntropyWithLogits_ShapeFn) {
diff --git a/tensorflow/core/public/version.h b/tensorflow/core/public/version.h
index 22f2c02b78..40eebd1db0 100644
--- a/tensorflow/core/public/version.h
+++ b/tensorflow/core/public/version.h
@@ -19,12 +19,12 @@ limitations under the License.
 // TensorFlow uses semantic versioning, see http://semver.org/.
 
 #define TF_MAJOR_VERSION 1
-#define TF_MINOR_VERSION 6
+#define TF_MINOR_VERSION 7
 #define TF_PATCH_VERSION 0
 
 // TF_VERSION_SUFFIX is non-empty for pre-releases (e.g. "-alpha", "-alpha.1",
 // "-beta", "-rc", "-rc.1")
-#define TF_VERSION_SUFFIX ""
+#define TF_VERSION_SUFFIX "-rc1"
 
 #define TF_STR_HELPER(x) #x
 #define TF_STR(x) TF_STR_HELPER(x)
diff --git a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
index 956dccb64f..f3db5857ae 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.bayesflow.monte_carlo.md
@@ -6,42 +6,42 @@ Monte Carlo integration and helpers.
 ## Background
 
 Monte Carlo integration refers to the practice of estimating an expectation with
-a sample mean.  For example, given random variable `Z in R^k` with density `p`,
+a sample mean.  For example, given random variable `Z in \\(R^k\\)` with density `p`,
 the expectation of function `f` can be approximated like:
 
 ```
-E_p[f(Z)] = \int f(z) p(z) dz
-          ~ S_n
-          := n^{-1} \sum_{i=1}^n f(z_i),  z_i iid samples from p.
+$$E_p[f(Z)] = \int f(z) p(z) dz$$
+$$          ~ S_n
+          := n^{-1} \sum_{i=1}^n f(z_i),  z_i\ iid\ samples\ from\ p.$$
 ```
 
-If `E_p[|f(Z)|] < infinity`, then `S_n --> E_p[f(Z)]` by the strong law of large
-numbers.  If `E_p[f(Z)^2] < infinity`, then `S_n` is asymptotically normal with
-variance `Var[f(Z)] / n`.
+If `\\(E_p[|f(Z)|] < infinity\\)`, then `\\(S_n\\) --> \\(E_p[f(Z)]\\)` by the strong law of large
+numbers.  If `\\(E_p[f(Z)^2] < infinity\\)`, then `\\(S_n\\)` is asymptotically normal with
+variance `\\(Var[f(Z)] / n\\)`.
 
 Practitioners of Bayesian statistics often find themselves wanting to estimate
-`E_p[f(Z)]` when the distribution `p` is known only up to a constant.  For
+`\\(E_p[f(Z)]\\)` when the distribution `p` is known only up to a constant.  For
 example, the joint distribution `p(z, x)` may be known, but the evidence
-`p(x) = \int p(z, x) dz` may be intractable.  In that case, a parameterized
-distribution family `q_lambda(z)` may be chosen, and the optimal `lambda` is the
-one minimizing the KL divergence between `q_lambda(z)` and
-`p(z | x)`.  We only know `p(z, x)`, but that is sufficient to find `lambda`.
+`\\(p(x) = \int p(z, x) dz\\)` may be intractable.  In that case, a parameterized
+distribution family `\\(q_\lambda(z)\\)` may be chosen, and the optimal `\\(\lambda\\)` is the
+one minimizing the KL divergence between `\\(q_\lambda(z)\\)` and
+`\\(p(z | x)\\)`.  We only know `p(z, x)`, but that is sufficient to find `\\(\lambda\\)`.
 
 
 ## Log-space evaluation and subtracting the maximum
 
 Care must be taken when the random variable lives in a high dimensional space.
-For example, the naive importance sample estimate `E_q[f(Z) p(Z) / q(Z)]`
-involves the ratio of two terms `p(Z) / q(Z)`, each of which must have tails
-dropping off faster than `O(|z|^{-(k + 1)})` in order to have finite integral.
+For example, the naive importance sample estimate `\\(E_q[f(Z) p(Z) / q(Z)]\\)`
+involves the ratio of two terms `\\(p(Z) / q(Z)\\)`, each of which must have tails
+dropping off faster than `\\(O(|z|^{-(k + 1)})\\)` in order to have finite integral.
 This ratio would often be zero or infinity up to numerical precision.
 
 For that reason, we write
 
 ```
-Log E_q[ f(Z) p(Z) / q(Z) ]
-   = Log E_q[ exp{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C} ] + C,  where
-C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].
+$$Log E_q[ f(Z) p(Z) / q(Z) ]$$
+$$   = Log E_q[ \exp\{Log[f(Z)] + Log[p(Z)] - Log[q(Z)] - C\} ] + C,$$  where
+$$C := Max[ Log[f(Z)] + Log[p(Z)] - Log[q(Z)] ].$$
 ```
 
 The maximum value of the exponentiated term will be 0.0, and the expectation
diff --git a/tensorflow/docs_src/api_guides/python/contrib.losses.md b/tensorflow/docs_src/api_guides/python/contrib.losses.md
index d7f862625e..8b7442216c 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.losses.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.losses.md
@@ -107,19 +107,19 @@ weighted average over the individual prediction errors:
   loss = tf.contrib.losses.mean_squared_error(predictions, depths, weight)
 ```
 
-@{tf.contrib.losses.absolute_difference}
-@{tf.contrib.losses.add_loss}
-@{tf.contrib.losses.hinge_loss}
-@{tf.contrib.losses.compute_weighted_loss}
-@{tf.contrib.losses.cosine_distance}
-@{tf.contrib.losses.get_losses}
-@{tf.contrib.losses.get_regularization_losses}
-@{tf.contrib.losses.get_total_loss}
-@{tf.contrib.losses.log_loss}
-@{tf.contrib.losses.mean_pairwise_squared_error}
-@{tf.contrib.losses.mean_squared_error}
-@{tf.contrib.losses.sigmoid_cross_entropy}
-@{tf.contrib.losses.softmax_cross_entropy}
-@{tf.contrib.losses.sparse_softmax_cross_entropy}
+* @{tf.contrib.losses.absolute_difference}
+* @{tf.contrib.losses.add_loss}
+* @{tf.contrib.losses.hinge_loss}
+* @{tf.contrib.losses.compute_weighted_loss}
+* @{tf.contrib.losses.cosine_distance}
+* @{tf.contrib.losses.get_losses}
+* @{tf.contrib.losses.get_regularization_losses}
+* @{tf.contrib.losses.get_total_loss}
+* @{tf.contrib.losses.log_loss}
+* @{tf.contrib.losses.mean_pairwise_squared_error}
+* @{tf.contrib.losses.mean_squared_error}
+* @{tf.contrib.losses.sigmoid_cross_entropy}
+* @{tf.contrib.losses.softmax_cross_entropy}
+* @{tf.contrib.losses.sparse_softmax_cross_entropy}
 
 
diff --git a/tensorflow/docs_src/community/documentation.md b/tensorflow/docs_src/community/documentation.md
index 003e0a25ec..6f2107ef40 100644
--- a/tensorflow/docs_src/community/documentation.md
+++ b/tensorflow/docs_src/community/documentation.md
@@ -477,31 +477,29 @@ should use Markdown in the docstring.
 
 Here's a simple example:
 
-```python
-def foo(x, y, name="bar"):
-  """Computes foo.
+    def foo(x, y, name="bar"):
+      """Computes foo.
 
-  Given two 1-D tensors `x` and `y`, this operation computes the foo.
+      Given two 1-D tensors `x` and `y`, this operation computes the foo.
 
-  Example:
+      Example:
 
-  ```
-  # x is [1, 1]
-  # y is [2, 2]
-  tf.foo(x, y) ==> [3, 3]
-  ```
-  Args:
-    x: A `Tensor` of type `int32`.
-    y: A `Tensor` of type `int32`.
-    name: A name for the operation (optional).
+      ```
+      # x is [1, 1]
+      # y is [2, 2]
+      tf.foo(x, y) ==> [3, 3]
+      ```
+      Args:
+        x: A `Tensor` of type `int32`.
+        y: A `Tensor` of type `int32`.
+        name: A name for the operation (optional).
 
-  Returns:
-    A `Tensor` of type `int32` that is the foo of `x` and `y`.
+      Returns:
+        A `Tensor` of type `int32` that is the foo of `x` and `y`.
 
-  Raises:
-    ValueError: If `x` or `y` are not of type `int32`.
-  """
-```
+      Raises:
+        ValueError: If `x` or `y` are not of type `int32`.
+      """
 
 ## Description of the docstring sections
 
diff --git a/tensorflow/docs_src/install/install_c.md b/tensorflow/docs_src/install/install_c.md
index 0481c97885..9059b3f3b6 100644
--- a/tensorflow/docs_src/install/install_c.md
+++ b/tensorflow/docs_src/install/install_c.md
@@ -38,7 +38,7 @@ enable TensorFlow for C:
          OS="linux" # Change to "darwin" for macOS
          TARGET_DIRECTORY="/usr/local"
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_go.md b/tensorflow/docs_src/install/install_go.md
index 8f89898c92..2e47a6d212 100644
--- a/tensorflow/docs_src/install/install_go.md
+++ b/tensorflow/docs_src/install/install_go.md
@@ -38,7 +38,7 @@ steps to install this library and enable TensorFlow for Go:
          TF_TYPE="cpu" # Change to "gpu" for GPU support
          TARGET_DIRECTORY='/usr/local'
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-${TF_TYPE}-$(go env GOOS)-x86_64-1.7.0-rc1.tar.gz" |
          sudo tar -C $TARGET_DIRECTORY -xz
 
      The `tar` command extracts the TensorFlow C library into the `lib`
diff --git a/tensorflow/docs_src/install/install_java.md b/tensorflow/docs_src/install/install_java.md
index 0ee9c849e1..eff066d200 100644
--- a/tensorflow/docs_src/install/install_java.md
+++ b/tensorflow/docs_src/install/install_java.md
@@ -36,7 +36,7 @@ following to the project's `pom.xml` to use the TensorFlow Java APIs:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>tensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -65,7 +65,7 @@ As an example, these steps will create a Maven project that uses TensorFlow:
                <dependency>
                  <groupId>org.tensorflow</groupId>
                  <artifactId>tensorflow</artifactId>
-                 <version>1.6.0</version>
+                 <version>1.7.0-rc1</version>
                </dependency>
              </dependencies>
          </project>
@@ -123,12 +123,12 @@ instead:
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 <dependency>
   <groupId>org.tensorflow</groupId>
   <artifactId>libtensorflow_jni_gpu</artifactId>
-  <version>1.6.0</version>
+  <version>1.7.0-rc1</version>
 </dependency>
 ```
 
@@ -147,7 +147,7 @@ refer to the simpler instructions above instead.
 Take the following steps to install TensorFlow for Java on Linux or macOS:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
 
   2. Decide whether you will run TensorFlow for Java on CPU(s) only or with
@@ -166,7 +166,7 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
          OS=$(uname -s | tr '[:upper:]' '[:lower:]')
          mkdir -p ./jni
          curl -L \
-           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.6.0.tar.gz" |
+           "https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-${TF_TYPE}-${OS}-x86_64-1.7.0-rc1.tar.gz" |
            tar -xz -C ./jni
 
 ### Install on Windows
@@ -174,10 +174,10 @@ Take the following steps to install TensorFlow for Java on Linux or macOS:
 Take the following steps to install TensorFlow for Java on Windows:
 
   1. Download
-     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.6.0.jar),
+     [libtensorflow.jar](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow-1.7.0-rc1.jar),
      which is the TensorFlow Java Archive (JAR).
   2. Download the following Java Native Interface (JNI) file appropriate for
-     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.6.0.zip).
+     [TensorFlow for Java on Windows](https://storage.googleapis.com/tensorflow/libtensorflow/libtensorflow_jni-cpu-windows-x86_64-1.7.0-rc1.zip).
   3. Extract this .zip file.
 
 
@@ -225,7 +225,7 @@ must be part of your `classpath`. For example, you can include the
 downloaded `.jar` in your `classpath` by using the `-cp` compilation flag
 as follows:
 
-<pre><b>javac -cp libtensorflow-1.6.0.jar HelloTF.java</b></pre>
+<pre><b>javac -cp libtensorflow-1.7.0-rc1.jar HelloTF.java</b></pre>
 
 
 ### Running
@@ -239,11 +239,11 @@ two files are available to the JVM:
 For example, the following command line executes the `HelloTF` program on Linux
 and macOS X:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar:. -Djava.library.path=./jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar:. -Djava.library.path=./jni HelloTF</b></pre>
 
 And the following command line executes the `HelloTF` program on Windows:
 
-<pre><b>java -cp libtensorflow-1.6.0.jar;. -Djava.library.path=jni HelloTF</b></pre>
+<pre><b>java -cp libtensorflow-1.7.0-rc1.jar;. -Djava.library.path=jni HelloTF</b></pre>
 
 If the program prints <tt>Hello from <i>version</i></tt>, you've successfully
 installed TensorFlow for Java and are ready to use the API.  If the program
diff --git a/tensorflow/docs_src/install/install_linux.md b/tensorflow/docs_src/install/install_linux.md
index 5e9a84bff6..27b696696d 100644
--- a/tensorflow/docs_src/install/install_linux.md
+++ b/tensorflow/docs_src/install/install_linux.md
@@ -165,7 +165,7 @@ Take the following steps to install TensorFlow with Virtualenv:
      Virtualenv environment:
 
      <pre>(tensorflow)$ <b>pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common_installation_problems).
@@ -270,7 +270,7 @@ take the following steps:
 
      <pre>
      $ <b>sudo pip3 install --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b>
      </pre>
 
      If this step fails, see
@@ -456,7 +456,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
 
      <pre>
      (tensorflow)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl</b></pre>
 
 <a name="ValidateYourInstallation"></a>
 ## Validate your installation
@@ -630,14 +630,14 @@ This section documents the relevant values for Linux installations.
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp27-none-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp27-none-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -649,14 +649,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp34-cp34m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp34-cp34m-linux_x86_64.whl
 </pre>
 
 Note that GPU support requires the NVIDIA hardware and software described in
@@ -668,14 +668,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp35-cp35m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp35-cp35m-linux_x86_64.whl
 </pre>
 
 
@@ -687,14 +687,14 @@ Note that GPU support requires the NVIDIA hardware and software described in
 CPU only:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/cpu/tensorflow-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
 GPU support:
 
 <pre>
-https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.6.0-cp36-cp36m-linux_x86_64.whl
+https://storage.googleapis.com/tensorflow/linux/gpu/tensorflow_gpu-1.7.0rc1-cp36-cp36m-linux_x86_64.whl
 </pre>
 
 
diff --git a/tensorflow/docs_src/install/install_mac.md b/tensorflow/docs_src/install/install_mac.md
index 55b460e189..7060ef43da 100644
--- a/tensorflow/docs_src/install/install_mac.md
+++ b/tensorflow/docs_src/install/install_mac.md
@@ -118,8 +118,8 @@ Take the following steps to install TensorFlow with Virtualenv:
      Python 2.7, the command to install
      TensorFlow in the active Virtualenv is as follows:
 
-     <pre> $ <b>pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     <pre> $ <b>pip3 install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b></pre>
 
 If you encounter installation problems, see
 [Common Installation Problems](#common-installation-problems).
@@ -241,8 +241,8 @@ take the following steps:
      you are installing TensorFlow for macOS and Python 2.7
      issue the following command:
 
-     <pre> $ <b>sudo pip install --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b> </pre>
+     <pre> $ <b>sudo pip3 install --upgrade \
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl</b> </pre>
 
      If the preceding command fails, see
      [installation problems](#common-installation-problems).
@@ -350,7 +350,7 @@ Take the following steps to install TensorFlow in an Anaconda environment:
      TensorFlow for Python 2.7:
 
      <pre> (<i>targetDirectory</i>)$ <b>pip install --ignore-installed --upgrade \
-     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl</b></pre>
+     https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl</b></pre>
 
 
 <a name="ValidateYourInstallation"></a>
@@ -524,7 +524,7 @@ The value you specify depends on your Python version.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py2-none-any.whl
 </pre>
 
 
@@ -532,5 +532,5 @@ https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py2-none-any.
 
 
 <pre>
-https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.6.0-py3-none-any.whl
+https://storage.googleapis.com/tensorflow/mac/cpu/tensorflow-1.7.0rc1-py3-none-any.whl
 </pre>
diff --git a/tensorflow/docs_src/install/install_sources.md b/tensorflow/docs_src/install/install_sources.md
index a7f33819b4..148f80efe2 100644
--- a/tensorflow/docs_src/install/install_sources.md
+++ b/tensorflow/docs_src/install/install_sources.md
@@ -350,10 +350,10 @@ Invoke `pip install` to install that pip package.
 The filename of the `.whl` file depends on your platform.
 For example, the following command will install the pip package
 
-for TensorFlow 1.6.0 on Linux:
+for TensorFlow 1.7.0rc1 on Linux:
 
 <pre>
-$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.6.0-py2-none-any.whl</b>
+$ <b>sudo pip install /tmp/tensorflow_pkg/tensorflow-1.7.0rc1-py2-none-any.whl</b>
 </pre>
 
 ## Validate your installation
@@ -450,6 +450,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Linux**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.10.0</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.9.0</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>GCC 4.8</td><td>Bazel 0.8.0</td><td>N/A</td><td>N/A</td></tr>
@@ -469,6 +471,7 @@ Stack Overflow and specify the `tensorflow` tag.
 **Mac**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.10.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.8.1</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow-1.4.0</td><td>CPU</td><td>2.7, 3.3-3.6</td><td>Clang from xcode</td><td>Bazel 0.5.4</td><td>N/A</td><td>N/A</td></tr>
@@ -483,6 +486,8 @@ Stack Overflow and specify the `tensorflow` tag.
 **Windows**
 <table>
 <tr><th>Version:</th><th>CPU/GPU:</th><th>Python Version:</th><th>Compiler:</th><th>Build Tools:</th><th>cuDNN:</th><th>CUDA:</th></tr>
+<tr><td>tensorflow-1.7.0rc1</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
+<tr><td>tensorflow_gpu-1.7.0rc1</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.6.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
 <tr><td>tensorflow_gpu-1.6.0</td><td>GPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>7</td><td>9</td></tr>
 <tr><td>tensorflow-1.5.0</td><td>CPU</td><td>3.5-3.6</td><td>MSVC 2015 update 3</td><td>Cmake v3.6.3</td><td>N/A</td><td>N/A</td></tr>
diff --git a/tensorflow/docs_src/mobile/optimizing.md b/tensorflow/docs_src/mobile/optimizing.md
index ca9cb043e9..778e4d3a62 100644
--- a/tensorflow/docs_src/mobile/optimizing.md
+++ b/tensorflow/docs_src/mobile/optimizing.md
@@ -233,6 +233,8 @@ order by how long they took. From left to right, the columns are:
 - The cumulative total time of this and the previous ops in the table. This is
   handy for understanding what the distribution of work is across the layers, to
   see if just a few of the nodes are taking up most of the time.
+  
+- The amount of memory consumed by outputs of this type of op.
 
 - Name of the node.
 
diff --git a/tensorflow/docs_src/mobile/prepare_models.md b/tensorflow/docs_src/mobile/prepare_models.md
index 360ee302aa..8b22c04d87 100644
--- a/tensorflow/docs_src/mobile/prepare_models.md
+++ b/tensorflow/docs_src/mobile/prepare_models.md
@@ -60,7 +60,7 @@ and serialized as protocol buffers:
   the `NodeDef`, so if all the `Variable` weights are converted to `Const` nodes,
   then we only need a single `GraphDef` file to hold the model architecture and
   the weights. Freezing the graph handles the process of loading the
-  checkpoints, and then converts all Consts to Variables. You can then load the
+  checkpoints, and then converts all Variables to Consts. You can then load the
   resulting file in a single call, without having to restore variable values
   from checkpoints. One thing to watch out for with `GraphDef` files is that
   sometimes they’re stored in text format for easy inspection. These versions
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 2cc3c48c3c..b05d87635f 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -1067,7 +1067,7 @@ py_test(
 
 py_test(
     name = "framework_importer_test",
-    size = "medium",
+    size = "large",
     srcs = ["framework/importer_test.py"],
     main = "framework/importer_test.py",
     srcs_version = "PY2AND3",
diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index d0ba8020c1..64c1760d5e 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -315,21 +315,39 @@ class ReverseV2Test(test_util.TensorFlowTestCase):
             self.assertAllEqual(x_tf_4, np.asarray(x_np)[:, ::-1])
             self.assertAllEqual(x_tf_5, np.asarray(x_np)[::-1, ::-1])
 
+  # This test covers the axis validation in the shape function
+  # (no eval())
+  def testInvalidAxis(self):
+    x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [-30])
+    with self.assertRaisesRegexp(ValueError,
+                                 "is out of valid range"):
+      array_ops.reverse_v2(x_np, [2])
+    with self.assertRaisesRegexp(ValueError,
+                                 "axis 0 specified more than once"):
+      array_ops.reverse_v2(x_np, [0, -2])
+
   # This is the version of reverse that uses axis indices rather than
   # bool tensors
   # TODO(b/32254538): Change this test to use array_ops.reverse
+  #
+  # Note: this test passes placeholder as constant axis is validated
+  # in shape function (see testInvalidAxis)
   def testInvalid(self):
     x_np = np.array([[1, 2, 3], [4, 5, 6]], dtype=np.float32)
+    axis = array_ops.placeholder(dtypes.int32)
     with self.test_session():
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [-30]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [-30]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "is out of valid range"):
-        array_ops.reverse_v2(x_np, [2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [2]})
       with self.assertRaisesRegexp(errors_impl.InvalidArgumentError,
                                    "axis 0 specified more than once"):
-        array_ops.reverse_v2(x_np, [0, -2]).eval()
+        array_ops.reverse_v2(x_np, axis).eval(feed_dict={axis: [0, -2]})
 
   def testReverse1DimAuto(self):
     for dtype in [
@@ -890,7 +908,7 @@ class StridedSliceAssignChecker(object):
         var = resource_variable_ops.ResourceVariable(self.x)
       else:
         var = variables.Variable(self.x)
-      sess.run(variables.initialize_variables([var]))
+      sess.run(variables.variables_initializer([var]))
       val = sess.run(var[index].assign(value))
       # val_copy is used to check that tf.assign works equivalently to the
       # assign method above.
diff --git a/tensorflow/python/kernel_tests/testdata/BUILD b/tensorflow/python/kernel_tests/testdata/BUILD
index a4a0dfc139..45264c773a 100644
--- a/tensorflow/python/kernel_tests/testdata/BUILD
+++ b/tensorflow/python/kernel_tests/testdata/BUILD
@@ -1,7 +1,7 @@
 # Data files for kernel tests.
 
 package(
-    default_visibility = ["//tensorflow:internal"],
+    default_visibility = ["//visibility:public"],
 )
 
 licenses(["notice"])  # Apache 2.0
diff --git a/tensorflow/python/kernel_tests/xent_op_test.py b/tensorflow/python/kernel_tests/xent_op_test.py
index e3e120a4eb..60c726d54c 100644
--- a/tensorflow/python/kernel_tests/xent_op_test.py
+++ b/tensorflow/python/kernel_tests/xent_op_test.py
@@ -18,10 +18,16 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import itertools
+import sys
+
 import numpy as np
 
+from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_nn_ops
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import gradients_impl
@@ -88,7 +94,7 @@ class XentTest(test.TestCase):
                                                     4.]]]).astype(dtype)
       np_labels = np.array([[[0., 0., 0., 1.]], [[0., .5, .5,
                                                   0.]]]).astype(dtype)
-      self.assertRaisesRegexp(ValueError, "must be rank 2",
+      self.assertRaisesRegexp(ValueError, "rank 2, but is rank 3",
                               gen_nn_ops.softmax_cross_entropy_with_logits,
                               np_features, np_labels)
 
@@ -128,6 +134,24 @@ class XentTest(test.TestCase):
     self.assertAllClose(
         np.array([1.3862, 1.9401]), np_loss, rtol=1.e-3, atol=1.e-3)
 
+  def testShapeBroadcast(self):
+    np_f = np.array([[1., 2., 3., 4.],
+                     [1., 2., 3., 4.]]).astype(np.float32)
+    np_l = np.array([[0., 0., 0., 1.],
+                     [0., .5, .5, 0.]]).astype(np.float32)
+    np_loss, np_backprop = self._npXent(np_f, np_l)
+    tf_f = constant_op.constant(
+        np.array([[1., 2., 3., 4.]]).astype(np.float32))
+    tf_l = constant_op.constant(
+        np.array([[0., 0., 0., 1.], [0., .5, .5, 0.]]).astype(np.float32))
+    for use_gpu in [False, True]:
+      with self.test_session(use_gpu=use_gpu) as sess:
+        loss, backprop = gen_nn_ops.softmax_cross_entropy_with_logits(
+            tf_f, tf_l)
+        tf_loss, tf_backprop = sess.run([loss, backprop])
+      self.assertAllCloseAccordingToType(np_loss, tf_loss)
+      self.assertAllCloseAccordingToType(np_backprop, tf_backprop)
+
   def testShapeMismatch(self):
     with self.test_session():
       with self.assertRaises(ValueError):
@@ -260,5 +284,60 @@ class XentTest(test.TestCase):
     self.assertAllEqual(np_loss, tf_loss)
 
 
+class XentBenchmark(test.Benchmark):
+
+  def benchmarkZeroDimension(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "zero_dimension_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          logits = array_ops.zeros([0, 2, 4], dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+  def benchmarkSingleClass(self):
+    for (m, n, p, use_gpu) in itertools.product(
+        [128],
+        [10, 100, 1000, 10000, 100000],
+        [0.001, 0.01, 0.5, 0.99, 1.0],
+        [False]):
+      k = int(p * n)
+      if k == 0:
+        continue
+      name = "single_class_m_%d_n_%d_k_%g_use_gpu_%s" % (m, n, k, use_gpu)
+      device = "/%s:0" % ("gpu" if use_gpu else "cpu")
+      with ops.Graph().as_default():
+        with ops.device(device):
+          labels = constant_op.constant([[1.], [-1.], [0.]],
+                                        dtype=dtypes.float32)
+          logits = constant_op.constant([[-1.], [0.], [1.]],
+                                        dtype=dtypes.float32)
+          op = nn_ops.softmax_cross_entropy_with_logits(
+              labels=labels, logits=logits)
+        with session.Session() as sess:
+          r = self.run_op_benchmark(sess, op, min_iters=100, name=name)
+          gb_processed_input = m * n / 1.0e9
+          throughput = gb_processed_input / r["wall_time"]
+          print("Benchmark: %s \t wall_time: %0.03g s \t "
+                "Throughput: %0.03g GB/s" % (name, r["wall_time"], throughput))
+          sys.stdout.flush()
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/layers/convolutional.py b/tensorflow/python/layers/convolutional.py
index 74e7c63fb3..2d99b1688f 100644
--- a/tensorflow/python/layers/convolutional.py
+++ b/tensorflow/python/layers/convolutional.py
@@ -180,6 +180,8 @@ class _Conv(base.Layer):
           # bias_add when computing gradients. To use bias_add, we collapse Z
           # and Y into a single dimension to obtain a 4D input tensor.
           outputs_shape = outputs.shape.as_list()
+          if outputs_shape[0] is None:
+            outputs_shape[0] = -1
           outputs_4d = array_ops.reshape(outputs,
                                          [outputs_shape[0], outputs_shape[1],
                                           outputs_shape[2] * outputs_shape[3],
diff --git a/tensorflow/python/layers/convolutional_test.py b/tensorflow/python/layers/convolutional_test.py
index 160e732b67..cdb42f5bd1 100644
--- a/tensorflow/python/layers/convolutional_test.py
+++ b/tensorflow/python/layers/convolutional_test.py
@@ -325,6 +325,12 @@ class ConvTest(test.TestCase):
     self.assertEqual(conv3d.kernel_constraint, k_constraint)
     self.assertEqual(conv3d.bias_constraint, b_constraint)
 
+  def testConv3DChannelsFirst(self):
+    # Test case for GitHub issue 15655
+    images = array_ops.placeholder(
+        dtype=dtypes.float32, shape=[None, 1, 32, 32, 32])
+    conv_layers.conv3d(images, 32, 9, data_format='channels_first')
+
 
 @test_util.with_c_api
 class SeparableConv1DTest(test.TestCase):
diff --git a/tensorflow/python/ops/linalg_ops.py b/tensorflow/python/ops/linalg_ops.py
index 5b4fb4f7c8..170861b43f 100644
--- a/tensorflow/python/ops/linalg_ops.py
+++ b/tensorflow/python/ops/linalg_ops.py
@@ -429,7 +429,7 @@ def svd(tensor, full_matrices=False, compute_uv=True, name=None):
   u, s, v_adj = np.linalg.svd(a, full_matrices=False)
   np_a_approx = np.dot(u, np.dot(np.diag(s), v_adj))
   # tf_a_approx and np_a_approx should be numerically close.
-  ````
+  ```
   @end_compatibility
   """
   s, u, v = gen_linalg_ops.svd(
diff --git a/tensorflow/python/training/monitored_session.py b/tensorflow/python/training/monitored_session.py
index 6c5c9e01a7..4ce6f6d002 100644
--- a/tensorflow/python/training/monitored_session.py
+++ b/tensorflow/python/training/monitored_session.py
@@ -281,13 +281,14 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
                              scaffold=None,
                              hooks=None,
                              chief_only_hooks=None,
-                             save_checkpoint_secs=600,
+                             save_checkpoint_secs=USE_DEFAULT,
                              save_summaries_steps=USE_DEFAULT,
                              save_summaries_secs=USE_DEFAULT,
                              config=None,
                              stop_grace_period_secs=120,
                              log_step_count_steps=100,
-                             max_wait_secs=7200):
+                             max_wait_secs=7200,
+                             save_checkpoint_steps=USE_DEFAULT):
   """Creates a `MonitoredSession` for training.
 
   For a chief, this utility sets proper session initializer/restorer. It also
@@ -310,8 +311,10 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
     chief_only_hooks: list of `SessionRunHook` objects. Activate these hooks if
       `is_chief==True`, ignore otherwise.
     save_checkpoint_secs: The frequency, in seconds, that a checkpoint is saved
-      using a default checkpoint saver. If `save_checkpoint_secs` is set to
-      `None`, then the default checkpoint saver isn't used.
+      using a default checkpoint saver. If both `save_checkpoint_steps` and
+      `save_checkpoint_secs` are set to `None`, then the default checkpoint
+      saver isn't used. If both are provided, then only `save_checkpoint_secs`
+      is used. Default 600.
     save_summaries_steps: The frequency, in number of global steps, that the
       summaries are written to disk using a default summary saver. If both
       `save_summaries_steps` and `save_summaries_secs` are set to `None`, then
@@ -330,6 +333,11 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
       become available. This should be kept relatively short to help detect
       incorrect code, but sometimes may need to be increased if the chief takes
       a while to start up.
+    save_checkpoint_steps: The frequency, in number of global steps, that a
+      checkpoint is saved using a default checkpoint saver. If both
+      `save_checkpoint_steps` and `save_checkpoint_secs` are set to `None`, then
+      the default checkpoint saver isn't used. If both are provided, then only
+      `save_checkpoint_secs` is used. Default not enabled.
 
   Returns:
     A `MonitoredSession` object.
@@ -342,6 +350,15 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
   elif save_summaries_steps == USE_DEFAULT:
     save_summaries_steps = None
 
+  if (save_checkpoint_steps == USE_DEFAULT and
+      save_checkpoint_secs == USE_DEFAULT):
+    save_checkpoint_steps = None
+    save_checkpoint_secs = 600
+  elif save_checkpoint_secs == USE_DEFAULT:
+    save_checkpoint_secs = None
+  elif save_checkpoint_steps == USE_DEFAULT:
+    save_checkpoint_steps = None
+
   scaffold = scaffold or Scaffold()
   if not is_chief:
     session_creator = WorkerSessionCreator(
@@ -374,9 +391,13 @@ def MonitoredTrainingSession(master='',  # pylint: disable=invalid-name
           save_steps=save_summaries_steps,
           save_secs=save_summaries_secs,
           output_dir=checkpoint_dir))
-    if save_checkpoint_secs and save_checkpoint_secs > 0:
+    if (save_checkpoint_secs and save_checkpoint_secs > 0) or (
+        save_checkpoint_steps and save_checkpoint_steps > 0):
       all_hooks.append(basic_session_run_hooks.CheckpointSaverHook(
-          checkpoint_dir, save_secs=save_checkpoint_secs, scaffold=scaffold))
+          checkpoint_dir,
+          save_steps=save_checkpoint_steps,
+          save_secs=save_checkpoint_secs,
+          scaffold=scaffold))
 
   if hooks:
     all_hooks.extend(hooks)
diff --git a/tensorflow/python/training/monitored_session_test.py b/tensorflow/python/training/monitored_session_test.py
index 159b2d5c16..3806056f01 100644
--- a/tensorflow/python/training/monitored_session_test.py
+++ b/tensorflow/python/training/monitored_session_test.py
@@ -282,6 +282,42 @@ class MonitoredTrainingSessionTest(test.TestCase):
           is_chief=True, checkpoint_dir=logdir) as session:
         self.assertEqual(2, session.run(gstep))
 
+  def test_save_checkpoint_steps(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_steps')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_steps=100,
+          log_step_count_steps=10) as session:
+        for _ in range(100):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(100, session.run(gstep))
+
+  def test_save_checkpoint_secs(self):
+    logdir = _test_dir(self.get_temp_dir(), 'test_save_checkpoint_secs')
+    with ops.Graph().as_default():
+      gstep = variables_lib.get_or_create_global_step()
+      new_gstep = state_ops.assign_add(gstep, 1)
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True,
+          checkpoint_dir=logdir,
+          save_checkpoint_secs=0.1,
+          log_step_count_steps=10) as session:
+        session.run(new_gstep)
+        time.sleep(0.2)
+        for _ in range(10):
+          session.run(new_gstep)
+      # A restart will find the checkpoint and recover automatically.
+      with monitored_session.MonitoredTrainingSession(
+          is_chief=True, checkpoint_dir=logdir) as session:
+        self.assertEqual(11, session.run(gstep))
+
   def test_summaries_steps(self):
     logdir = _test_dir(self.get_temp_dir(), 'test_summaries_steps')
     with ops.Graph().as_default():
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 2d3cb415fe..fcc57d506e 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -22,6 +22,7 @@ load(
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
+    "if_mkl_lnx_x64"
 )
 
 def register_extension_info(**kwargs):
@@ -202,7 +203,8 @@ def tf_copts(android_optimization_level_override="-O2", is_external=False):
           "-ftemplate-depth=900"])
       + if_cuda(["-DGOOGLE_CUDA=1"])
       + if_tensorrt(["-DGOOGLE_TENSORRT=1"])
-      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML", "-fopenmp",])
+      + if_mkl(["-DINTEL_MKL=1", "-DEIGEN_USE_VML"])
+      + if_mkl_lnx_x64(["-fopenmp"])
       + if_android_arm(["-mfpu=neon"])
       + if_linux_x86_64(["-msse3"])
       + if_ios_x86_64(["-msse4.1"])
diff --git a/tensorflow/tools/api/golden/tensorflow.train.pbtxt b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
index c75ee474aa..bec72e1e60 100644
--- a/tensorflow/tools/api/golden/tensorflow.train.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.train.pbtxt
@@ -238,7 +238,7 @@ tf_module {
   }
   member_method {
     name: "MonitoredTrainingSession"
-    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'600\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\'], "
+    argspec: "args=[\'master\', \'is_chief\', \'checkpoint_dir\', \'scaffold\', \'hooks\', \'chief_only_hooks\', \'save_checkpoint_secs\', \'save_summaries_steps\', \'save_summaries_secs\', \'config\', \'stop_grace_period_secs\', \'log_step_count_steps\', \'max_wait_secs\', \'save_checkpoint_steps\'], varargs=None, keywords=None, defaults=[\'\', \'True\', \'None\', \'None\', \'None\', \'None\', \'<object object instance>\', \'<object object instance>\', \'<object object instance>\', \'None\', \'120\', \'100\', \'7200\', \'<object object instance>\'], "
   }
   member_method {
     name: "NewCheckpointReader"
diff --git a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
index e1b56b9a25..7d471b4703 100755
--- a/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
+++ b/tensorflow/tools/ci_build/osx/libtensorflow_cpu.sh
@@ -31,5 +31,5 @@ export TF_NEED_OPENCL_SYCL=0
 export TF_NEED_MKL=0
 export COMPUTECPP_PATH="/usr/local"
 
-export PATH="/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
+export PATH="$PATH:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin"
 build_libtensorflow_tarball "-cpu-darwin-$(uname -m)"
diff --git a/tensorflow/tools/docker/Dockerfile.devel b/tensorflow/tools/docker/Dockerfile.devel
index 22c73c3fe1..11f476d12c 100644
--- a/tensorflow/tools/docker/Dockerfile.devel
+++ b/tensorflow/tools/docker/Dockerfile.devel
@@ -70,7 +70,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # TODO(craigcitro): Don't install the pip package, since it makes it
 # more difficult to experiment with local changes. Instead, just add
diff --git a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
index 3690e7dfe5..037d13116e 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
+++ b/tensorflow/tools/docker/Dockerfile.devel-cpu-mkl
@@ -3,7 +3,7 @@ FROM tensorflow/tensorflow:latest-devel
 LABEL maintainer="Clayne Robison<clayne.b.robison@intel.com>"
 
 # These arguments are parameterized. Use --build-args to override.
-ARG TF_BRANCH=r1.6
+ARG TF_BRANCH=r1.7
 ARG WHL_DIR=/whl
 
 RUN apt-get update && apt-get install -y --no-install-recommends \
diff --git a/tensorflow/tools/docker/Dockerfile.devel-gpu b/tensorflow/tools/docker/Dockerfile.devel-gpu
index 69ba340f92..1fcb6428b2 100644
--- a/tensorflow/tools/docker/Dockerfile.devel-gpu
+++ b/tensorflow/tools/docker/Dockerfile.devel-gpu
@@ -79,7 +79,7 @@ RUN mkdir /bazel && \
 
 # Download and build TensorFlow.
 WORKDIR /tensorflow
-RUN git clone --branch=r1.6 --depth=1 https://github.com/tensorflow/tensorflow.git .
+RUN git clone --branch=r1.7 --depth=1 https://github.com/tensorflow/tensorflow.git .
 
 # Configure the build for our CUDA configuration.
 ENV CI_BUILD_PYTHON python
diff --git a/tensorflow/tools/lib_package/BUILD b/tensorflow/tools/lib_package/BUILD
index 3fbdb5cacd..0ede8c6370 100644
--- a/tensorflow/tools/lib_package/BUILD
+++ b/tensorflow/tools/lib_package/BUILD
@@ -138,7 +138,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/c/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
@@ -176,7 +175,6 @@ genrule(
         "@zlib_archive//:zlib.h",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]),
     outs = ["include/tensorflow/jni/LICENSE"],
     cmd = "$(location :concat_licenses.sh) $(SRCS) >$@",
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index dd75eda231..62fec2c402 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -127,7 +127,6 @@ filegroup(
         "@org_python_pypi_backports_weakref//:LICENSE",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-        "@mkl//:LICENSE",
     ]) + if_not_windows([
         "@nccl_archive//:LICENSE.txt",
     ]) + tf_additional_license_deps(),
diff --git a/tensorflow/tools/pip_package/setup.py b/tensorflow/tools/pip_package/setup.py
index e0152da4df..365e8d6b08 100644
--- a/tensorflow/tools/pip_package/setup.py
+++ b/tensorflow/tools/pip_package/setup.py
@@ -29,7 +29,7 @@ from setuptools.dist import Distribution
 # This version string is semver compatible, but incompatible with pip.
 # For pip, we will remove all '-' characters from this string, and use the
 # result for pip.
-_VERSION = '1.6.0'
+_VERSION = '1.7.0-rc1'
 
 REQUIRED_PACKAGES = [
     'absl-py >= 0.1.6',
@@ -39,7 +39,7 @@ REQUIRED_PACKAGES = [
     'numpy >= 1.13.3',
     'six >= 1.10.0',
     'protobuf >= 3.4.0',
-    'tensorboard >= 1.6.0, < 1.7.0',
+    'tensorboard >= 1.7.0, < 1.8.0',
     'termcolor >= 1.1.0',
 ]
 
@@ -62,7 +62,7 @@ else:
 if 'tf_nightly' in project_name:
   for i, pkg in enumerate(REQUIRED_PACKAGES):
     if 'tensorboard' in pkg:
-      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.7.0a0, < 1.8.0a0'
+      REQUIRED_PACKAGES[i] = 'tb-nightly >= 1.8.0a0, < 1.9.0a0'
       break
 
 # weakref.finalize and enum were introduced in Python 3.4
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0e31358236..ac6380dd3e 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -15,6 +15,11 @@ load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
 
 
+# Sanitize a dependency so that it works correctly from code that includes
+# TensorFlow as a submodule.
+def clean_dep(dep):
+  return str(Label(dep))
+
 # If TensorFlow is linked as a submodule.
 # path_prefix is no longer used.
 # tf_repo_name is thought to be under consideration.
@@ -32,17 +37,37 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   arm_compiler_configure(
       name="local_config_arm_compiler",
       remote_config_repo="../arm_compiler",
-      build_file = str(Label("//third_party/toolchains/cpus/arm:BUILD")))
+      build_file = clean_dep("//third_party/toolchains/cpus/arm:BUILD"))
 
   mkl_repository(
-      name = "mkl",
+      name = "mkl_linux",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+      ],
+      sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f",
+      strip_prefix = "mklml_lnx_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_windows",
+      urls = [
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip"
+      ],
+      sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4",
+      strip_prefix = "mklml_win_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
+  )
+  mkl_repository(
+      name = "mkl_darwin",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
-          "https://github.com/01org/mkl-dnn/releases/download/v0.11/mklml_lnx_2018.0.1.20171007.tgz",
+          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
+          "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz"
       ],
-      sha256 = "6b07cb7e5451db67c2e31e785ae458b18f7f363c60a61685488f69e9ae7199d4",
-      strip_prefix = "mklml_lnx_2018.0.1.20171007",
-      build_file = str(Label("//third_party/mkl:mkl.BUILD")),
+      sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f",
+      strip_prefix = "mklml_mac_2018.0.1.20171227",
+      build_file = clean_dep("//third_party/mkl:mkl.BUILD")
   )
 
   if path_prefix:
@@ -52,12 +77,12 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "mkl_dnn",
       urls = [
-          "https://mirror.bazel.build/github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
-          "https://github.com/01org/mkl-dnn/archive/e0bfcaa7fcb2b1e1558f5f0676933c1db807a729.tar.gz",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
+          "https://github.com/intel/mkl-dnn/archive/v0.12.tar.gz",
       ],
-      sha256 = "02e244f63dd95402691a361392504c143eede9a89043426f174836638a9cbf09",
-      strip_prefix = "mkl-dnn-e0bfcaa7fcb2b1e1558f5f0676933c1db807a729",
-      build_file = str(Label("//third_party/mkl_dnn:mkldnn.BUILD")),
+      sha256 = "86fa2a8c12a56e3b725945acedeaa82492746be02545aba6d710f097e013e19e",
+      strip_prefix = "mkl-dnn-0.12",
+      build_file = clean_dep("//third_party/mkl_dnn:mkldnn.BUILD"),
   )
 
   tf_http_archive(
@@ -68,7 +93,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
      sha256 = "5996380e3e8b981f55d1c8d58e709c00dbb4806ba367be75d0925a68cc2f6478",
      strip_prefix = "abseil-cpp-720c017e30339fd1786ce4aac68bc8559736e53f",
-     build_file = str(Label("//third_party:com_google_absl.BUILD")),
+     build_file = clean_dep("//third_party:com_google_absl.BUILD"),
   )
 
   tf_http_archive(
@@ -79,8 +104,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "791b836cacd03e20bae5bdd25f1c4a5505a0a9975ba94a61eb4e2631fbd1d53a",
       strip_prefix = "eigen-eigen-6913f0cf7d06",
-      build_file = str(Label("//third_party:eigen.BUILD")),
-      patch_file = str(Label("//third_party:eigen_fix_cuda_compilation.patch"))
+      build_file = clean_dep("//third_party:eigen.BUILD"),
+      patch_file = clean_dep("//third_party:eigen_fix_cuda_compilation.patch")
   )
 
   tf_http_archive(
@@ -93,7 +118,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           # remove the whitelist entry in third_party/repo.bzl.
           # "https://github.com/raspberrypi/tools/archive/0e906ebc527eab1cdbf7adabff5b474da9562e9f.tar.gz",
       ],
-      build_file = str(Label("//:arm_compiler.BUILD")),
+      build_file = clean_dep("//:arm_compiler.BUILD"),
   )
 
   tf_http_archive(
@@ -104,7 +129,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ade869c3f42f23b5263c7d594aa3c7e5e61ac6a3afcaf5d6e42899d2a7986ce",
       strip_prefix = "libxsmm-1.8.1",
-      build_file = str(Label("//third_party:libxsmm.BUILD")),
+      build_file = clean_dep("//third_party:libxsmm.BUILD"),
   )
 
   tf_http_archive(
@@ -117,7 +142,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "932075525642b04ac6f1b50589f1df5cd72ec2f448b721fd32234cf183f0e755",
       strip_prefix = "or-tools-253f7955c6a1fd805408fba2e42ac6d45b312d15/src",
-      build_file = str(Label("//third_party:ortools.BUILD")),
+      build_file = clean_dep("//third_party:ortools.BUILD"),
   )
 
   tf_http_archive(
@@ -149,7 +174,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6560547c63e4af82b0f202cb710ceabb3f21347a4b996db565a411da5b17aba0",
       strip_prefix = "farmhash-816a4ae622e964763ca0862d9dbd19324a1eaf45",
-      build_file = str(Label("//third_party:farmhash.BUILD")),
+      build_file = clean_dep("//third_party:farmhash.BUILD"),
   )
 
   tf_http_archive(
@@ -160,7 +185,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "0f30a15b1566d93f146c8d149878a06e91d9bb7ec2cfd76906df62a82be4aac9",
       strip_prefix = "highwayhash-dfcb97ca4fe9277bf9dc1802dd979b071896453b",
-      build_file = str(Label("//third_party:highwayhash.BUILD")),
+      build_file = clean_dep("//third_party:highwayhash.BUILD"),
   )
 
   tf_http_archive(
@@ -171,7 +196,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "00b0891c678c065446ca59bcee64719d0096d54d6886e6e472aeee2e170ae324",
       strip_prefix = "nasm-2.12.02",
-      build_file = str(Label("//third_party:nasm.BUILD")),
+      build_file = clean_dep("//third_party:nasm.BUILD"),
   )
 
   tf_http_archive(
@@ -182,7 +207,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "c15a9607892113946379ccea3ca8b85018301b200754f209453ab21674268e77",
       strip_prefix = "libjpeg-turbo-1.5.1",
-      build_file = str(Label("//third_party/jpeg:jpeg.BUILD")),
+      build_file = clean_dep("//third_party/jpeg:jpeg.BUILD"),
   )
 
   tf_http_archive(
@@ -193,7 +218,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "716c59c7dfc808a4c368f8ada526932be72b2fcea11dd85dc9d88b1df1dfe9c2",
       strip_prefix = "libpng-1.2.53",
-      build_file = str(Label("//third_party:png.BUILD")),
+      build_file = clean_dep("//third_party:png.BUILD"),
   )
 
   tf_http_archive(
@@ -204,7 +229,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "208780b3616f9de0aeb50822b7a8f5482f6515193859e91ed61637be6ad74fd4",
       strip_prefix = "sqlite-amalgamation-3200000",
-      build_file = str(Label("//third_party:sqlite.BUILD")),
+      build_file = clean_dep("//third_party:sqlite.BUILD"),
   )
 
   tf_http_archive(
@@ -215,7 +240,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "34a7377ba834397db019e8eb122e551a49c98f49df75ec3fcc92b9a794a4f6d1",
       strip_prefix = "giflib-5.1.4",
-      build_file = str(Label("//third_party:gif.BUILD")),
+      build_file = clean_dep("//third_party:gif.BUILD"),
   )
 
   tf_http_archive(
@@ -226,7 +251,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
       strip_prefix = "six-1.10.0",
-      build_file = str(Label("//third_party:six.BUILD")),
+      build_file = clean_dep("//third_party:six.BUILD"),
   )
 
   tf_http_archive(
@@ -237,7 +262,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "ff6d2e2962d834acb125cc4dcc80c54a8c17c253f4cc9d9c43b5102a560bb75d",
       strip_prefix = "astor-0.6.2",
-      build_file = str(Label("//third_party:astor.BUILD")),
+      build_file = clean_dep("//third_party:astor.BUILD"),
   )
 
   tf_http_archive(
@@ -248,7 +273,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "7068908321ecd2774f145193c4b34a11305bd104b4551b09273dfd1d6a374930",
       strip_prefix = "gast-0.2.0",
-      build_file = str(Label("//third_party:gast.BUILD")),
+      build_file = clean_dep("//third_party:gast.BUILD"),
   )
 
   tf_http_archive(
@@ -259,7 +284,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1d6d69ce66211143803fbc56652b41d73b4a400a2891d7bf7a1cdf4c02de613b",
       strip_prefix = "termcolor-1.1.0",
-      build_file = str(Label("//third_party:termcolor.BUILD")),
+      build_file = clean_dep("//third_party:termcolor.BUILD"),
   )
 
   tf_http_archive(
@@ -280,7 +305,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "8813bf712a66b3d8b85dc289e1104ed220f1878cf981e2fe756dfaabe9a82892",
       strip_prefix = "backports.weakref-1.0rc1/src",
-      build_file = str(Label("//third_party:backports_weakref.BUILD")),
+      build_file = clean_dep("//third_party:backports_weakref.BUILD"),
   )
 
   tf_http_archive(
@@ -291,7 +316,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2dadd04a2802de27e0fe5a19b76538f6da9d39ff244036afa00c1bba754de5ee",
       strip_prefix = "codegen-1.0",
-      build_file = str(Label("//third_party:codegen.BUILD")),
+      build_file = clean_dep("//third_party:codegen.BUILD"),
   )
 
   filegroup_external(
@@ -376,7 +401,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://ftp.exim.org/pub/pcre/pcre-8.39.tar.gz",
       ],
       strip_prefix = "pcre-8.39",
-      build_file = str(Label("//third_party:pcre.BUILD")),
+      build_file = clean_dep("//third_party:pcre.BUILD"),
   )
 
   tf_http_archive(
@@ -388,7 +413,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://pilotfiber.dl.sourceforge.net/project/swig/swig/swig-3.0.8/swig-3.0.8.tar.gz",
       ],
       strip_prefix = "swig-3.0.8",
-      build_file = str(Label("//third_party:swig.BUILD")),
+      build_file = clean_dep("//third_party:swig.BUILD"),
   )
 
   tf_http_archive(
@@ -399,7 +424,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://curl.haxx.se/download/curl-7.49.1.tar.gz",
       ],
       strip_prefix = "curl-7.49.1",
-      build_file = str(Label("//third_party:curl.BUILD")),
+      build_file = clean_dep("//third_party:curl.BUILD"),
   )
 
   tf_http_archive(
@@ -421,7 +446,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/antirez/linenoise/archive/c894b9e59f02203dbe4e2be657572cf88c4230c3.tar.gz",
       ],
       strip_prefix = "linenoise-c894b9e59f02203dbe4e2be657572cf88c4230c3",
-      build_file = str(Label("//third_party:linenoise.BUILD")),
+      build_file = clean_dep("//third_party:linenoise.BUILD"),
   )
 
   # TODO(phawkins): currently, this rule uses an unofficial LLVM mirror.
@@ -434,7 +459,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
       strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
-      build_file = str(Label("//third_party/llvm:llvm.BUILD")),
+      build_file = clean_dep("//third_party/llvm:llvm.BUILD"),
   )
 
   tf_http_archive(
@@ -445,7 +470,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "108532fb94c6f227558d45be3f3347b52539f0f58290a7bb31ec06c462d05326",
       strip_prefix = "lmdb-LMDB_0.9.19/libraries/liblmdb",
-      build_file = str(Label("//third_party:lmdb.BUILD")),
+      build_file = clean_dep("//third_party:lmdb.BUILD"),
   )
 
   tf_http_archive(
@@ -456,7 +481,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "07d34db40593d257324ec5fb9debc4dc33f29f8fb44e33a2eeb35503e61d0fe2",
       strip_prefix = "jsoncpp-11086dd6a7eba04289944367ca82cea71299ed70",
-      build_file = str(Label("//third_party:jsoncpp.BUILD")),
+      build_file = clean_dep("//third_party:jsoncpp.BUILD"),
   )
 
   tf_http_archive(
@@ -477,7 +502,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
       strip_prefix = "zlib-1.2.8",
-      build_file = str(Label("//third_party:zlib.BUILD")),
+      build_file = clean_dep("//third_party:zlib.BUILD"),
   )
 
   tf_http_archive(
@@ -487,7 +512,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "http://www.kurims.kyoto-u.ac.jp/~ooura/fft.tgz",
       ],
       sha256 = "52bb637c70b971958ec79c9c8752b1df5ff0218a4db4510e60826e0cb79b5296",
-      build_file = str(Label("//third_party/fft2d:fft2d.BUILD")),
+      build_file = clean_dep("//third_party/fft2d:fft2d.BUILD"),
   )
 
   tf_http_archive(
@@ -498,7 +523,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
       strip_prefix = "snappy-1.1.4",
-      build_file = str(Label("//third_party:snappy.BUILD")),
+      build_file = clean_dep("//third_party:snappy.BUILD"),
   )
 
   tf_http_archive(
@@ -509,7 +534,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
       strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7",
-      build_file = str(Label("//third_party:nccl.BUILD")),
+      build_file = clean_dep("//third_party:nccl.BUILD"),
   )
 
   tf_http_archive(
@@ -520,8 +545,8 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "dd035d57c8f19b0b612dd6eefe6e5eebad76f506e302cccb7c2066f25a83585e",
       strip_prefix = "librdkafka-0.11.1",
-      build_file = str(Label("//third_party:kafka/BUILD")),
-      patch_file = str(Label("//third_party/kafka:config.patch")),
+      build_file = clean_dep("//third_party:kafka/BUILD"),
+      patch_file = clean_dep("//third_party/kafka:config.patch"),
   )
 
   tf_http_archive(
@@ -532,7 +557,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "b888d8ce5fc10254c3dd6c9020c7764dd53cf39cf011249d0b4deda895de1b7c",
       strip_prefix = "aws-sdk-cpp-1.3.15",
-      build_file = str(Label("//third_party:aws.BUILD")),
+      build_file = clean_dep("//third_party:aws.BUILD"),
   )
 
   java_import_external(
@@ -568,7 +593,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "3c8f25c02e806c3ce0ab5fb7da1817f89fc9732709024e2a81b6b82f7cc792a8",
       strip_prefix = "jemalloc-4.4.0",
-      build_file = str(Label("//third_party:jemalloc.BUILD")),
+      build_file = clean_dep("//third_party:jemalloc.BUILD"),
   )
 
   java_import_external(
@@ -613,7 +638,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "e0928ca4aa10ea1e0551e2d7ce4d1d7ea2d84b2abbdef082b0da84268791d0c4",
       strip_prefix = "pprof-c0fb62ec88c411cc91194465e54db2632845b650",
-      build_file = str(Label("//third_party:pprof.BUILD")),
+      build_file = clean_dep("//third_party:pprof.BUILD"),
   )
 
   tf_http_archive(
@@ -624,7 +649,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "6bfa06ab52a650ae7ee6963143a0bbc667d6504822cbd9670369b598f18c58c3",
       strip_prefix = "cub-1.8.0",
-      build_file = str(Label("//third_party:cub.BUILD")),
+      build_file = clean_dep("//third_party:cub.BUILD"),
   )
 
   tf_http_archive(
@@ -635,7 +660,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://github.com/cython/cython/archive/3732784c45cfb040a5b0936951d196f83a12ea17.tar.gz",
       ],
       strip_prefix = "cython-3732784c45cfb040a5b0936951d196f83a12ea17",
-      build_file = str(Label("//third_party:cython.BUILD")),
+      build_file = clean_dep("//third_party:cython.BUILD"),
       delete = ["BUILD.bazel"],
   )
 
@@ -657,7 +682,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
           "https://github.com/intel/ARM_NEON_2_x86_SSE/archive/0f77d9d182265259b135dad949230ecbf1a2633d.tar.gz",
       ],
-      build_file = str(Label("//third_party:arm_neon_2_x86_sse.BUILD")),
+      build_file = clean_dep("//third_party:arm_neon_2_x86_sse.BUILD"),
   )
 
   tf_http_archive(
@@ -668,7 +693,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
           "https://github.com/google/flatbuffers/archive/971a68110e4fc1bace10fcb6deeb189e7e1a34ce.tar.gz",
       ],
-      build_file = str(Label("//third_party/flatbuffers:flatbuffers.BUILD")),
+      build_file = clean_dep("//third_party/flatbuffers:flatbuffers.BUILD"),
   )
 
   tf_http_archive(
@@ -678,7 +703,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip",
       ],
-      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
+      build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
   )
 
   tf_http_archive(
@@ -688,7 +713,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
           "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip",
           "https://storage.googleapis.com/download.tensorflow.org/models/tflite/smartreply_1.0_2017_11_01.zip"
       ],
-      build_file = str(Label("//third_party:tflite_smartreply.BUILD")),
+      build_file = clean_dep("//third_party:tflite_smartreply.BUILD"),
   )
 
   ##############################################################################
@@ -752,7 +777,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   # Needed by Protobuf
   native.bind(
       name = "python_headers",
-      actual = str(Label("//util/python:python_headers")),
+      actual = clean_dep("//util/python:python_headers"),
   )
 
   # Needed by Protobuf
diff --git a/third_party/mkl/BUILD b/third_party/mkl/BUILD
index b27d341404..c2adf578c7 100644
--- a/third_party/mkl/BUILD
+++ b/third_party/mkl/BUILD
@@ -1,7 +1,5 @@
 licenses(["notice"])  # 3-Clause BSD
 
-exports_files(["LICENSE"])
-
 config_setting(
     name = "using_mkl",
     values = {
@@ -10,17 +8,51 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+config_setting(
+    name = "using_mkl_lnx_x64",
+    values = {
+        "cpu": "k8",
+        "define": "using_mkl=true",
+    },
+    visibility = ["//visibility:public"],
+)
+
 load(
     "//third_party/mkl:build_defs.bzl",
     "if_mkl",
 )
 
+filegroup(
+    name = "LICENSE",
+    srcs = ["MKL_LICENSE"] + select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:LICENSE",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:LICENSE",
+        ],
+    }),
+    visibility = ["//visibility:public"],
+)
+
 cc_library(
     name = "intel_binary_blob",
-    srcs = if_mkl([
-        "@mkl//:libmklml_intel.so",
-        "@mkl//:libiomp5.so",
-    ]),
     visibility = ["//visibility:public"],
-    deps = ["@mkl//:mkl_headers"],
+    deps = select({
+        "@org_tensorflow//tensorflow:linux_x86_64": [
+            "@mkl_linux//:mkl_headers",
+            "@mkl_linux//:mkl_libs_linux",
+        ],
+        "@org_tensorflow//tensorflow:darwin": [
+            "@mkl_darwin//:mkl_headers",
+            "@mkl_darwin//:mkl_libs_darwin",
+        ],
+        "@org_tensorflow//tensorflow:windows": [
+            "@mkl_windows//:mkl_headers",
+            "@mkl_windows//:mkl_libs_windows",
+        ],
+    }),
 )
diff --git a/third_party/mkl/MKL_LICENSE b/third_party/mkl/MKL_LICENSE
new file mode 100644
index 0000000000..9c8f3ea087
--- /dev/null
+++ b/third_party/mkl/MKL_LICENSE
@@ -0,0 +1,201 @@
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "{}"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright {yyyy} {name of copyright owner}
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
\ No newline at end of file
diff --git a/third_party/mkl/build_defs.bzl b/third_party/mkl/build_defs.bzl
index 8b73ddabdd..53e02769da 100644
--- a/third_party/mkl/build_defs.bzl
+++ b/third_party/mkl/build_defs.bzl
@@ -24,6 +24,18 @@ def if_mkl(if_true, if_false = []):
         "//conditions:default": if_false
     })
 
+def if_mkl_lnx_x64(if_true, if_false = []):
+    """Shorthand for select()'ing on whether we're building with MKL.
+
+    Returns a select statement which evaluates to if_true if we're building
+    with MKL enabled.  Otherwise, the select statement evaluates to if_false.
+
+    """
+    return select({
+        str(Label("//third_party/mkl:using_mkl_lnx_x64")): if_true,
+        "//conditions:default": if_false
+    })
+
 
 def _enable_local_mkl(repository_ctx):
   return _TF_MKL_ROOT in repository_ctx.os.environ
diff --git a/third_party/mkl/mkl.BUILD b/third_party/mkl/mkl.BUILD
index 8db97232e1..c3a71e4ff9 100644
--- a/third_party/mkl/mkl.BUILD
+++ b/third_party/mkl/mkl.BUILD
@@ -17,14 +17,29 @@ cc_library(
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libmklml_intel.so",
-    srcs = ["lib/libmklml_intel.so"],
+cc_library(
+    name = "mkl_libs_linux",
+    srcs = [
+        "lib/libiomp5.so",
+        "lib/libmklml_intel.so",
+    ],
     visibility = ["//visibility:public"],
 )
 
-filegroup(
-    name = "libiomp5.so",
-    srcs = ["lib/libiomp5.so"],
+cc_library(
+    name = "mkl_libs_darwin",
+    srcs = [
+        "lib/libiomp5.dylib",
+        "lib/libmklml.dylib",
+    ],
+    visibility = ["//visibility:public"],
+)
+
+cc_library(
+    name = "mkl_libs_windows",
+    srcs = [
+        "lib/libiomp5md.lib",
+        "lib/mklml.lib",
+    ],
     visibility = ["//visibility:public"],
 )
-- 
GitLab


From c194a0ea67a6fb61bd23b39fbb2a49b664e2dba1 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 11:02:56 -0700
Subject: [PATCH 682/960] Automated g4 rollback of changelist 190808678

PiperOrigin-RevId: 190955400
---
 tensorflow/contrib/lite/toco/BUILD            |   1 -
 .../graph_transformations.h                   |   1 -
 .../swap_elementwise_binary.cc                | 175 ------------------
 .../toco/graph_transformations/tests/BUILD    |  11 --
 .../tests/swap_elementwise_binary_test.cc     |  89 ---------
 tensorflow/contrib/lite/toco/toco_tooling.cc  |   1 -
 6 files changed, 278 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index bba61627f9..d552de313c 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -280,7 +280,6 @@ cc_library(
         "graph_transformations/resolve_tensorflow_switch.cc",
         "graph_transformations/resolve_tensorflow_tile.cc",
         "graph_transformations/resolve_transpose_attributes.cc",
-        "graph_transformations/swap_elementwise_binary.cc",
         "graph_transformations/unfuse_activation_functions.cc",
         "graph_transformations/unpartition_embedding_lookup.cc",
         "graph_transformations/unroll_batch_matmul.cc",
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 1291825c8e..640afc7c74 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -180,7 +180,6 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveConstantStridedSlice)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantFill)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantGather)
 DECLARE_GRAPH_TRANSFORMATION(ResolveMultiplyByZero)
-DECLARE_GRAPH_TRANSFORMATION(SwapElementwiseBinary)
 DECLARE_GRAPH_TRANSFORMATION(Dequantize)
 DECLARE_GRAPH_TRANSFORMATION(UnpartitionEmbeddingLookup)
 
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc b/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
deleted file mode 100644
index ecbce58d16..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/swap_elementwise_binary.cc
+++ /dev/null
@@ -1,175 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/runtime/types.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-
-namespace {
-
-bool ShapesAllowSwapping(const string& input_array_name,
-                         const string& const_array_name, Model* model) {
-  const Array& input_array = model->GetOrCreateArray(input_array_name);
-  const Array& const_array = model->GetOrCreateArray(const_array_name);
-  // Wait until these shapes have been resolved.
-  if (!input_array.has_shape() || !const_array.has_shape()) {
-    return false;
-  }
-
-  // Currently swapping is not handled for scalar const_array, though that could
-  // be done once there is a test model.
-  if (RequiredBufferSizeForShape(input_array.shape()) !=
-      RequiredBufferSizeForShape(const_array.shape())) {
-    return false;
-  }
-
-  return true;
-}
-
-}  // namespace
-
-// Swaps:
-//                   Input
-//                      \
-//                   (Reshape Op)       Const
-//                         \             /
-//                       (Add/Sub/Mul/Div op)
-//                               |
-//                             Output
-//
-// To:
-//
-//                     Input            Const
-//                        \              /
-//                      (Add/Sub/Mul/Div op)
-//                               |
-//                         (Reshape Op)
-//                               |
-//                             Output
-//
-// This can allow Add/Mul ops from batch normalization to be folded into an
-// Input op from a FullyConnected layer.
-bool SwapElementwiseBinary::Run(Model* model, std::size_t op_index) {
-  const auto element_wise_op_it = model->operators.begin() + op_index;
-  std::unique_ptr<Operator>& element_wise_op = *element_wise_op_it;
-  DCHECK(element_wise_op);
-
-  switch (element_wise_op->type) {
-    case OperatorType::kAdd:
-    case OperatorType::kSub:
-    case OperatorType::kMul:
-    case OperatorType::kDiv:
-      break;
-    default:
-      return false;
-  }
-
-  int reshape_input = -1;
-  Operator* op = GetOpWithOutput(*model, element_wise_op->inputs[0]);
-  if (!op) {
-    return false;
-  }
-
-  if (op->type == OperatorType::kTensorFlowReshape) {
-    reshape_input = 0;
-  } else {
-    op = GetOpWithOutput(*model, element_wise_op->inputs[1]);
-    if (!op || op->type != OperatorType::kTensorFlowReshape) {
-      return false;
-    }
-    reshape_input = 1;
-  }
-
-  int const_input = (reshape_input == 0) ? 1 : 0;
-  const string& const_input_array = element_wise_op->inputs[const_input];
-  if (!IsConstantParameterArray(*model, const_input_array)) {
-    return false;
-  }
-
-  // Do not fold division if denominator is not constant.
-  if (element_wise_op->type != OperatorType::kDiv && const_input != 1) {
-    return false;
-  }
-
-  const auto reshape_it =
-      FindOpWithOutput(*model, element_wise_op->inputs[reshape_input]);
-  // Note: we take copies of the tensor names here, instead of const-refs as we
-  // may overwrite the original names.
-  const string reshape_input_name = (*reshape_it)->inputs[0];
-  const string intermediate_name = (*reshape_it)->outputs[0];
-  const string element_wise_output_name = element_wise_op->outputs[0];
-
-  // Check the reshape op input and const op have their shapes resolved.
-  if (!ShapesAllowSwapping(reshape_input_name, const_input_array, model)) {
-    return false;
-  }
-
-  int count_ops_consuming_output = CountOpsWithInput(*model, intermediate_name);
-  DCHECK_GE(count_ops_consuming_output, 1);
-  if (count_ops_consuming_output > 1) {
-    AddMessageF(
-        "Not exchanging element-wise function with %s because it is "
-        "consumed by more than 1 other operator",
-        LogName(**reshape_it));
-    return false;
-  }
-
-  // If the element_wise_op was originally producing an output_array we can't
-  // swap as otherwise the output array would change. It'd be nice to still be
-  // able to swap but if code is relying on the fetch names instead of array
-  // indices this won't work.
-  for (int i = 0; i < model->flags.output_arrays_size(); ++i) {
-    if (model->flags.output_arrays(i) == element_wise_op->outputs[0]) {
-      AddMessageF(
-          "Not exchanging activation function with %s to preserve output array "
-          "name %s",
-          LogName(**reshape_it), element_wise_op->outputs[0]);
-      return false;
-    }
-  }
-
-  // Rewire by changing inputs, including all consumers.
-  // TODO(b/76086261): Replace with new utility function.
-  Operator* consumer = GetFirstOpWithInput(*model, element_wise_output_name);
-  while (consumer) {
-    for (int i = 0; i < consumer->inputs.size(); ++i) {
-      if (consumer->inputs[i] == element_wise_output_name) {
-        consumer->inputs[i] = intermediate_name;
-      }
-    }
-    consumer = GetFirstOpWithInput(*model, element_wise_output_name);
-  }
-  element_wise_op->inputs[reshape_input] = reshape_input_name;
-  (*reshape_it)->inputs[0] = element_wise_output_name;
-
-  // Clear shapes; this will allow shape propagation to fix the sizes for us.
-  model->GetOrCreateArray(element_wise_output_name).clear_shape();
-
-  // Finally, swap operators.  Note that this only works when there are no other
-  // direct descendents of the reshape operator.
-  element_wise_op.swap(*reshape_it);
-
-  return true;
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
index a2008ddbdb..8dcd4adc90 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
+++ b/tensorflow/contrib/lite/toco/graph_transformations/tests/BUILD
@@ -18,17 +18,6 @@ tf_cc_test(
     ],
 )
 
-tf_cc_test(
-    name = "swap_elementwise_binary_test",
-    srcs = ["swap_elementwise_binary_test.cc"],
-    deps = [
-        "//tensorflow/contrib/lite/toco:graph_transformations",
-        "//tensorflow/contrib/lite/toco:model",
-        "//tensorflow/contrib/lite/toco:tooling_util",
-        "@com_google_googletest//:gtest_main",
-    ],
-)
-
 tf_cc_test(
     name = "lstm_utils_test",
     srcs = ["lstm_utils_test.cc"],
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc b/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc
deleted file mode 100644
index c3778017f3..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/tests/swap_elementwise_binary_test.cc
+++ /dev/null
@@ -1,89 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include <gtest/gtest.h>
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-
-namespace toco {
-
-namespace {
-
-int ShapeCount(const std::vector<int>& size) {
-  CHECK(size.size());
-  int count = 1;
-  for (int dim : size) {
-    count *= dim;
-  }
-  return count;
-}
-
-// Adds a new parameter array to the model.
-void AddConstArray(const string& name, const float* data,
-                   const std::vector<int>& size, Model* model) {
-  Array& array = model->GetOrCreateArray(name);
-  array.data_type = ArrayDataType::kFloat;
-  Shape* shape = array.mutable_shape();
-  *(shape->mutable_dims()) = size;
-
-  auto& buffer = array.GetMutableBuffer<toco::ArrayDataType::kFloat>();
-  buffer.data.resize(ShapeCount(size));
-  std::copy(data, data + ShapeCount(size), buffer.data.data());
-}
-
-}  // namespace
-
-TEST(SwapElementwiseBinaryTest, SwapsReshape) {
-  Model model;
-  const float parameters[2][4] = {{0., 1., 2., 3.}, {10., 11., 12., 13.}};
-
-  AddConstArray("before_reshape", parameters[0], {2, 2}, &model);
-  AddConstArray("add_vector", parameters[1], {1, 4}, &model);
-
-  auto reshape_op = absl::make_unique<TensorFlowReshapeOperator>();
-  reshape_op->shape = {1, 4};
-  reshape_op->inputs = {"before_reshape"};
-  reshape_op->outputs = {"after_reshape"};
-  Array& reshape_array = model.GetOrCreateArray("after_reshape");
-  *(reshape_array.mutable_shape()) = {1, 4};
-
-  auto add_op = absl::make_unique<AddOperator>();
-  add_op->inputs = {"after_reshape", "add_vector"};
-  add_op->outputs = {"add"};
-  Array& add_array = model.GetOrCreateArray("add");
-  *(add_array.mutable_shape()) = {1, 4};
-
-  model.operators.push_back(std::move(reshape_op));
-  model.operators.push_back(std::move(add_op));
-
-  auto transformation = absl::make_unique<toco::SwapElementwiseBinary>();
-  ASSERT_TRUE(transformation->Run(&model, 1));
-
-  Operator* op = GetOpWithOutput(model, "add");
-  ASSERT_NE(nullptr, op);
-  ASSERT_EQ(OperatorType::kAdd, op->type);
-  ASSERT_EQ(2, op->inputs.size());
-  for (const string& input : op->inputs) {
-    EXPECT_TRUE(IsConstantParameterArray(model, input))
-        << input << " is not const input";
-  }
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 41ea1481bc..30dd6fab9e 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -90,7 +90,6 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveTensorFlowTile);
   transformations->Add(new ResolveTensorFlowConcat);
   transformations->Add(new ResolveMultiplyByZero);
-  transformations->Add(new SwapElementwiseBinary);
   transformations->Add(new IdentifyDilatedConv);
   transformations->Add(new IdentifyL2Normalization);
   transformations->Add(new IdentifyL2Pool);
-- 
GitLab


From 1d7c2fa60f717dea7239970d96f7d4bf96842039 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Fri, 30 Mar 2018 02:11:55 +0800
Subject: [PATCH 683/960] Raise a nicer error message when trying to call
 gradients with while loop (#18052)

* Produce a nicer error message when trying to call gradients on a while loop without properly serializing graph via MetaGraphDef

* Fix syntax and lint error

* Fix minor intent: Wrong continued indentation (add 2 spaces)
---
 tensorflow/python/ops/control_flow_ops.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/python/ops/control_flow_ops.py b/tensorflow/python/ops/control_flow_ops.py
index 1278768d8b..710287012e 100644
--- a/tensorflow/python/ops/control_flow_ops.py
+++ b/tensorflow/python/ops/control_flow_ops.py
@@ -833,6 +833,9 @@ class GradLoopState(object):
     if outer_grad_state:
       outer_forward_ctxt = outer_grad_state.forward_context
     else:
+      if not hasattr(forward_ctxt, 'outer_context'):
+        raise ValueError("Failed to call gradients on a while loop without"
+                         "properly serializing graph via MetaGraphDef")
       outer_forward_ctxt = forward_ctxt.outer_context
 
     # Add the forward loop counter.
-- 
GitLab


From d044a1ffa87e772076a14ace7a16bb97886a0804 Mon Sep 17 00:00:00 2001
From: Alan Yee <alyee@ucsd.edu>
Date: Thu, 29 Mar 2018 11:12:07 -0700
Subject: [PATCH 684/960] Update README.md (#18076)

Add YouTube channel
---
 README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/README.md b/README.md
index 0a309ebe2d..c66f7e3f3f 100644
--- a/README.md
+++ b/README.md
@@ -86,6 +86,7 @@ The TensorFlow project strives to abide by generally accepted best practices in
 
 * [TensorFlow Website](https://www.tensorflow.org)
 * [TensorFlow White Papers](https://www.tensorflow.org/about/bib)
+* [TensorFlow YouTube Channel](https://www.youtube.com/channel/UC0rqucBdTuFTjJiefW5t-IQ)
 * [TensorFlow Model Zoo](https://github.com/tensorflow/models)
 * [TensorFlow MOOC on Udacity](https://www.udacity.com/course/deep-learning--ud730)
 * [TensorFlow Course at Stanford](https://web.stanford.edu/class/cs20si)
-- 
GitLab


From f2c3b869d354c05e497a79118a13a599dfc256bc Mon Sep 17 00:00:00 2001
From: "Xiaoming (Jason) Cui" <xiaoming.cui@intel.com>
Date: Thu, 29 Mar 2018 11:12:36 -0700
Subject: [PATCH 685/960] [INTEL MKL] utilize test_util.IsMklEnabled() to check
 if the MKL support is turned on or not (#18062)

* Fixed issue #92, timeline_test unit test fails, changed the test so that it can take cpu name changed with MKLDNN naming conversion

* [INTEL MKL] utilize test_util.IsMklEnabled() to check if the MKL support is turned on or not
---
 tensorflow/python/client/timeline_test.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/client/timeline_test.py b/tensorflow/python/client/timeline_test.py
index 5e6b5acdb0..c046e9cfd4 100644
--- a/tensorflow/python/client/timeline_test.py
+++ b/tensorflow/python/client/timeline_test.py
@@ -24,6 +24,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.client import timeline
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import test_util
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
@@ -155,9 +156,7 @@ class TimelineTest(test.TestCase):
     ctf = step_analysis.chrome_trace.format_to_string()
     self._validateTrace(ctf)
     maximums = step_analysis.allocator_maximums
-    cpuname = 'cpu'
-    if 'mklcpu' in maximums:
-      cpuname = 'mkl' + cpuname
+    cpuname = 'mklcpu' if test_util.IsMklEnabled() else 'cpu'
     self.assertTrue(cpuname in maximums)
     cpu_max = maximums[
         'cuda_host_bfc'] if 'cuda_host_bfc' in maximums else maximums[cpuname]
-- 
GitLab


From 7c7350dfb35276eff2b8039bfa2def13bb736a4b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 11:24:44 -0700
Subject: [PATCH 686/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 190959179
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 348 ++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 348 ++++++++++++++++++
 2 files changed, 696 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 05d6e02281..7cdf36f423 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -10340,6 +10340,342 @@ op {
   }
   is_commutative: true
 }
+op {
+  name: "BoostedTreesCalculateBestGainsPerFeature"
+  input_arg {
+    name: "node_id_range"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "stats_summary_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "node_ids_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "gains_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "thresholds_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "left_node_contribs_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "right_node_contribs_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  attr {
+    name: "l1"
+    type: "float"
+  }
+  attr {
+    name: "l2"
+    type: "float"
+  }
+  attr {
+    name: "tree_complexity"
+    type: "float"
+  }
+  attr {
+    name: "max_splits"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "BoostedTreesCreateEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesDeserializeEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesEnsembleResourceHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesGetEnsembleStates"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "num_trees"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "num_finalized_trees"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "num_attempted_layers"
+    type: DT_INT32
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesMakeStatsSummary"
+  input_arg {
+    name: "node_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "gradients"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "hessians"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "bucketized_features_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "stats_summary"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "max_splits"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_buckets"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "BoostedTreesPredict"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "bucketized_features"
+    type: DT_INT32
+    number_attr: "num_bucketized_features"
+  }
+  output_arg {
+    name: "logits"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "num_bucketized_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesSerializeEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesTrainingPredict"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "cached_tree_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "cached_node_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "bucketized_features"
+    type: DT_INT32
+    number_attr: "num_bucketized_features"
+  }
+  output_arg {
+    name: "partial_logits"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "tree_ids"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "node_ids"
+    type: DT_INT32
+  }
+  attr {
+    name: "num_bucketized_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesUpdateEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "feature_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "node_ids"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "gains"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "thresholds"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "left_node_contribs"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "right_node_contribs"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "learning_rate"
+    type: "float"
+  }
+  attr {
+    name: "pruning_mode"
+    type: "int"
+    has_minimum: true
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+  }
+  is_stateful: true
+}
 op {
   name: "BroadcastArgs"
   input_arg {
@@ -23333,6 +23669,18 @@ op {
     }
   }
 }
+op {
+  name: "IsBoostedTreesEnsembleInitialized"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
 op {
   name: "IsFinite"
   input_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 274a7fbf75..42a68cb712 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -3995,6 +3995,342 @@ op {
   }
   is_commutative: true
 }
+op {
+  name: "BoostedTreesCalculateBestGainsPerFeature"
+  input_arg {
+    name: "node_id_range"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "stats_summary_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "node_ids_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "gains_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "thresholds_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "left_node_contribs_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "right_node_contribs_list"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  attr {
+    name: "l1"
+    type: "float"
+  }
+  attr {
+    name: "l2"
+    type: "float"
+  }
+  attr {
+    name: "tree_complexity"
+    type: "float"
+  }
+  attr {
+    name: "max_splits"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "BoostedTreesCreateEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesDeserializeEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  input_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesEnsembleResourceHandleOp"
+  output_arg {
+    name: "resource"
+    type: DT_RESOURCE
+  }
+  attr {
+    name: "container"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  attr {
+    name: "shared_name"
+    type: "string"
+    default_value {
+      s: ""
+    }
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesGetEnsembleStates"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "num_trees"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "num_finalized_trees"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "num_attempted_layers"
+    type: DT_INT32
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesMakeStatsSummary"
+  input_arg {
+    name: "node_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "gradients"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "hessians"
+    type: DT_FLOAT
+  }
+  input_arg {
+    name: "bucketized_features_list"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  output_arg {
+    name: "stats_summary"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "max_splits"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_buckets"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+}
+op {
+  name: "BoostedTreesPredict"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "bucketized_features"
+    type: DT_INT32
+    number_attr: "num_bucketized_features"
+  }
+  output_arg {
+    name: "logits"
+    type: DT_FLOAT
+  }
+  attr {
+    name: "num_bucketized_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesSerializeEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "stamp_token"
+    type: DT_INT64
+  }
+  output_arg {
+    name: "tree_ensemble_serialized"
+    type: DT_STRING
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesTrainingPredict"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "cached_tree_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "cached_node_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "bucketized_features"
+    type: DT_INT32
+    number_attr: "num_bucketized_features"
+  }
+  output_arg {
+    name: "partial_logits"
+    type: DT_FLOAT
+  }
+  output_arg {
+    name: "tree_ids"
+    type: DT_INT32
+  }
+  output_arg {
+    name: "node_ids"
+    type: DT_INT32
+  }
+  attr {
+    name: "num_bucketized_features"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "logits_dimension"
+    type: "int"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  is_stateful: true
+}
+op {
+  name: "BoostedTreesUpdateEnsemble"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  input_arg {
+    name: "feature_ids"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "node_ids"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "gains"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "thresholds"
+    type: DT_INT32
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "left_node_contribs"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  input_arg {
+    name: "right_node_contribs"
+    type: DT_FLOAT
+    number_attr: "num_features"
+  }
+  attr {
+    name: "max_depth"
+    type: "int"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "learning_rate"
+    type: "float"
+  }
+  attr {
+    name: "pruning_mode"
+    type: "int"
+    has_minimum: true
+  }
+  attr {
+    name: "num_features"
+    type: "int"
+    has_minimum: true
+  }
+  is_stateful: true
+}
 op {
   name: "BroadcastArgs"
   input_arg {
@@ -11365,6 +11701,18 @@ op {
     }
   }
 }
+op {
+  name: "IsBoostedTreesEnsembleInitialized"
+  input_arg {
+    name: "tree_ensemble_handle"
+    type: DT_RESOURCE
+  }
+  output_arg {
+    name: "is_initialized"
+    type: DT_BOOL
+  }
+  is_stateful: true
+}
 op {
   name: "IsFinite"
   input_arg {
-- 
GitLab


From 1c6e292e7cc348218db2048b241a7330cacbbef6 Mon Sep 17 00:00:00 2001
From: Ayush Dubey <ayushd@google.com>
Date: Thu, 29 Mar 2018 11:54:55 -0700
Subject: [PATCH 687/960] Initialize pointer to ScopedAllocatorMgr in
 BaseGPUDevice.

PiperOrigin-RevId: 190964008
---
 tensorflow/core/common_runtime/gpu/gpu_device.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/common_runtime/gpu/gpu_device.cc b/tensorflow/core/common_runtime/gpu/gpu_device.cc
index 52fd20e479..0b9e8f9cc2 100644
--- a/tensorflow/core/common_runtime/gpu/gpu_device.cc
+++ b/tensorflow/core/common_runtime/gpu/gpu_device.cc
@@ -257,6 +257,7 @@ BaseGPUDevice::BaseGPUDevice(const SessionOptions& options, const string& name,
                                                          physical_device_desc)),
       gpu_allocator_(gpu_allocator),
       cpu_allocator_(cpu_allocator),
+      scoped_allocator_mgr_(new ScopedAllocatorMgr(name)),
       tf_gpu_id_(tf_gpu_id),
       sync_every_op_(sync_every_op),
       max_streams_(max_streams) {
-- 
GitLab


From 4ebb2eac303a22b06597facd07793595e105169b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 12:02:50 -0700
Subject: [PATCH 688/960] Leaves attributes on outside_compilation nodes so
 they can be replicated in a later pass.

PiperOrigin-RevId: 190965218
---
 .../jit/encapsulate_subgraphs_pass.cc         |  51 ++--
 .../jit/encapsulate_subgraphs_pass_test.cc    | 279 ++++++++++--------
 tensorflow/contrib/tpu/ops/replication_ops.cc |   2 +
 3 files changed, 187 insertions(+), 145 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 7fc43fb263..53ec6c1e60 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -254,7 +254,8 @@ class Encapsulator {
 
     // Adds _RecvAtHost and _SendFromHost nodes, where needed, to graph_out.
     Status AddOutsideCompilationHostIONodes(
-        const string& subgraph_name,
+        const string& group_attribute, const string& subgraph_name,
+        const string& outside_compilation_attribute,
         const std::unordered_map<const Node*, Node*>& node_images,
         Graph* graph_out);
 
@@ -405,7 +406,9 @@ class Encapsulator {
 
     // Builds a _RecvAtHost node producing all the inputs of an
     // outside_compilation subgraph and stores it in oc_subgraph.recv_at_host.
-    Status AddRecvAtHostNode(const string& subgraph_name,
+    Status AddRecvAtHostNode(const string& group_attribute,
+                             const string& subgraph_name,
+                             const string& outside_compilation_attribute,
                              const string& oc_subgraph_name,
                              OutsideCompilationSubgraph* oc_subgraph,
                              Graph* graph_out);
@@ -414,8 +417,10 @@ class Encapsulator {
     // outside_compilation subgraph and stores it in oc_subgraph.send_from_host.
     Status AddSendFromHostNode(
         const std::unordered_map<const Node*, Node*>& node_images,
-        const string& subgraph_name, const string& oc_subgraph_name,
-        OutsideCompilationSubgraph* oc_subgraph, Graph* graph_out);
+        const string& group_attribute, const string& subgraph_name,
+        const string& outside_compilation_attribute,
+        const string& oc_subgraph_name, OutsideCompilationSubgraph* oc_subgraph,
+        Graph* graph_out);
 
     // The subgraph extracted from the input graph, suitable for being turned
     // into a FunctionDef. Inputs are fed by _Arg nodes, and outputs are
@@ -1114,7 +1119,8 @@ Status Encapsulator::Subgraph::AddHostComputeKeyPlaceholder(
 }
 
 Status Encapsulator::Subgraph::AddRecvAtHostNode(
-    const string& subgraph_name, const string& oc_subgraph_name,
+    const string& group_attribute, const string& subgraph_name,
+    const string& outside_compilation_attribute, const string& oc_subgraph_name,
     OutsideCompilationSubgraph* oc_subgraph, Graph* graph_out) {
   if (host_compute_key_placeholder_ == nullptr) {
     TF_RETURN_IF_ERROR(AddHostComputeKeyPlaceholder(oc_subgraph, graph_out));
@@ -1135,14 +1141,15 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
   NodeDefBuilder builder(strings::StrCat("outside_compilation_", subgraph_name,
                                          "_", oc_subgraph_name, "_recv"),
                          kRecvAtHostOp);
-  // TODO(misard) When we add replication the device placement will have to be
-  // redone.
   builder.Device(device_);
   builder.Attr("Toutputs", dtypes);
-  // TODO(misard) For now we only support TPU device 0.
+  // The correct device_ordinal will be inserted during replication in a
+  // subsequent rewrite.
   builder.Attr("device_ordinal", 0);
   builder.Attr("key", strings::StrCat("host_compute_channel_", subgraph_name,
                                       "_", oc_subgraph_name));
+  builder.Attr(group_attribute, subgraph_name);
+  builder.Attr(outside_compilation_attribute, oc_subgraph_name);
   builder.Input(host_compute_key_placeholder_->name(), 0, DT_STRING);
   Status s = builder.Finalize(&recv_def);
   if (!s.ok()) return s;
@@ -1163,7 +1170,8 @@ Status Encapsulator::Subgraph::AddRecvAtHostNode(
 
 Status Encapsulator::Subgraph::AddSendFromHostNode(
     const std::unordered_map<const Node*, Node*>& node_images,
-    const string& subgraph_name, const string& oc_subgraph_name,
+    const string& group_attribute, const string& subgraph_name,
+    const string& outside_compilation_attribute, const string& oc_subgraph_name,
     OutsideCompilationSubgraph* oc_subgraph, Graph* graph_out) {
   if (host_compute_key_placeholder_ == nullptr) {
     TF_RETURN_IF_ERROR(AddHostComputeKeyPlaceholder(oc_subgraph, graph_out));
@@ -1188,14 +1196,15 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
   NodeDefBuilder builder(strings::StrCat("outside_compilation_", subgraph_name,
                                          "_", oc_subgraph_name, "_send"),
                          kSendFromHostOp);
-  // TODO(misard) When we add replication the device placement will have to be
-  // redone.
   builder.Device(device_);
   builder.Attr("Tinputs", dtypes);
   builder.Attr("key", strings::StrCat("host_compute_channel_", subgraph_name,
                                       "_", oc_subgraph_name));
-  // TODO(misard) For now we only support TPU device 0.
+  // The correct device_ordinal will be inserted during replication in a
+  // subsequent rewrite.
   builder.Attr("device_ordinal", 0);
+  builder.Attr(group_attribute, subgraph_name);
+  builder.Attr(outside_compilation_attribute, oc_subgraph_name);
   builder.Input(inputs);
   builder.Input(host_compute_key_placeholder_->name(), 0, DT_STRING);
   Status s = builder.Finalize(&send_def);
@@ -1216,7 +1225,8 @@ Status Encapsulator::Subgraph::AddSendFromHostNode(
 }
 
 Status Encapsulator::Subgraph::AddOutsideCompilationHostIONodes(
-    const string& subgraph_name,
+    const string& group_attribute, const string& subgraph_name,
+    const string& outside_compilation_attribute,
     const std::unordered_map<const Node*, Node*>& node_images,
     Graph* graph_out) {
   for (auto& outside_compilation_subgraph_entry :
@@ -1226,14 +1236,16 @@ Status Encapsulator::Subgraph::AddOutsideCompilationHostIONodes(
         outside_compilation_subgraph_entry.second;
 
     if (!oc_subgraph.inputs.empty() || !oc_subgraph.control_inputs.empty()) {
-      TF_RETURN_IF_ERROR(
-          AddRecvAtHostNode(subgraph_name, oc_name, &oc_subgraph, graph_out));
+      TF_RETURN_IF_ERROR(AddRecvAtHostNode(group_attribute, subgraph_name,
+                                           outside_compilation_attribute,
+                                           oc_name, &oc_subgraph, graph_out));
     }
 
     if (!oc_subgraph.outputs_by_src.empty() ||
         !oc_subgraph.control_outputs.empty()) {
-      TF_RETURN_IF_ERROR(AddSendFromHostNode(node_images, subgraph_name,
-                                             oc_name, &oc_subgraph, graph_out));
+      TF_RETURN_IF_ERROR(AddSendFromHostNode(
+          node_images, group_attribute, subgraph_name,
+          outside_compilation_attribute, oc_name, &oc_subgraph, graph_out));
     }
   }
   return Status::OK();
@@ -1450,8 +1462,6 @@ Status Encapsulator::CopyNodesToOutputGraph(
             "Parallel checking is not supported when outside_compilation "
             "clusters are present.");
       }
-      image->ClearAttr(group_attribute_);
-      image->ClearAttr(outside_compilation_attribute_);
     }
     (*node_images)[node] = image;
   }
@@ -1477,7 +1487,8 @@ Status Encapsulator::AddOutsideCompilationHostIONodes(
     const string& subgraph_name = subgraph_entry.first;
     Subgraph& subgraph = subgraph_entry.second;
     TF_RETURN_IF_ERROR(subgraph.AddOutsideCompilationHostIONodes(
-        subgraph_name, node_images, graph_out));
+        group_attribute_, subgraph_name, outside_compilation_attribute_,
+        node_images, graph_out));
   }
   return Status::OK();
 }
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 94481a1fde..7899b5d72d 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -382,24 +382,36 @@ Node* KeyPlaceholder(const string& call_node,
       .FinalizeBuilder(&node_builder);
 }
 
-Node* RecvAtHost(ops::NodeOut key_input, const string& key,
+Node* RecvAtHost(ops::NodeOut key_input, const string& cluster,
+                 const string& oc_cluster,
                  const gtl::ArraySlice<DataType>& dtypes,
                  const GraphDefBuilder::Options& opts) {
   if (opts.HaveError()) return nullptr;
-  NodeBuilder node_builder(opts.GetNameForOp("_XlaRecvAtHost"),
+  string key =
+      strings::StrCat("host_compute_channel_", cluster, "_", oc_cluster);
+  string name = strings::StrCat("outside_compilation_", cluster, "_",
+                                oc_cluster, "_recv");
+  NodeBuilder node_builder(opts.WithName(name).GetNameForOp("_XlaRecvAtHost"),
                            "_XlaRecvAtHost", opts.op_registry());
   node_builder.Input(std::move(key_input));
   return opts.WithAttr("Toutputs", dtypes)
       .WithAttr("key", key)
       .WithAttr("device_ordinal", 0)
+      .WithAttr("_encapsulate", cluster)
+      .WithAttr("_outside", oc_cluster)
       .FinalizeBuilder(&node_builder);
 }
 
-Node* SendFromHost(ops::NodeOut key_input, const string& key,
+Node* SendFromHost(ops::NodeOut key_input, const string& cluster,
+                   const string& oc_cluster,
                    const std::vector<ops::NodeOut>& inputs,
                    const GraphDefBuilder::Options& opts) {
   if (opts.HaveError()) return nullptr;
-  NodeBuilder node_builder(opts.GetNameForOp("_XlaSendFromHost"),
+  string key =
+      strings::StrCat("host_compute_channel_", cluster, "_", oc_cluster);
+  string name = strings::StrCat("outside_compilation_", cluster, "_",
+                                oc_cluster, "_send");
+  NodeBuilder node_builder(opts.WithName(name).GetNameForOp("_XlaSendFromHost"),
                            "_XlaSendFromHost", opts.op_registry());
   node_builder.Input(inputs);
   node_builder.Input(std::move(key_input));
@@ -410,6 +422,8 @@ Node* SendFromHost(ops::NodeOut key_input, const string& key,
   return opts.WithAttr("Tinputs", dtypes)
       .WithAttr("key", key)
       .WithAttr("device_ordinal", 0)
+      .WithAttr("_encapsulate", cluster)
+      .WithAttr("_outside", oc_cluster)
       .FinalizeBuilder(&node_builder);
 }
 
@@ -856,14 +870,14 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
     GraphDefBuilder shape(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
         KeyPlaceholderShape(shape.opts().WithName("KnownShape/_0"));
-    Node* recv =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   shape.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT, DT_FLOAT}, shape.opts());
     Node* e = Binary(ops::NodeOut(recv, 0), ops::NodeOut(recv, 1),
-                     shape.opts().WithName("E"));
-    SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                 {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
+                     shape.opts()
+                         .WithName("E")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape.opts());
     TF_EXPECT_OK(
         AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
@@ -901,17 +915,16 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
 
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   b2.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT, DT_FLOAT}, b2.opts());
     Node* e = Binary(ops::NodeOut(recv, 0), ops::NodeOut(recv, 1),
-                     b2.opts().WithName("E").WithControlInputs({recv, b}));
-    Node* send = SendFromHost(ops::NodeOut(key_constant, 0),
-                              "host_compute_channel_F1_O1", {e},
-                              b2.opts()
-                                  .WithName("outside_compilation_F1_O1_send")
-                                  .WithControlInput(e));
+                     b2.opts()
+                         .WithName("E")
+                         .WithControlInputs({recv, b})
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    Node* send = SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e},
+                              b2.opts().WithControlInput(e));
 
     Node* s = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}),
@@ -976,14 +989,14 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
     GraphDefBuilder shape1(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
         KeyPlaceholderShape(shape1.opts().WithName("KnownShape/_0"));
-    Node* recv =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   shape1.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT, DT_FLOAT}, shape1.opts());
     Node* e = Binary(ops::NodeOut(recv, 0), ops::NodeOut(recv, 1),
-                     shape1.opts().WithName("E"));
-    SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                 {e}, shape1.opts().WithName("outside_compilation_F1_O1_send"));
+                     shape1.opts()
+                         .WithName("E")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape1.opts());
     TF_EXPECT_OK(
         AddGraphDefToFunctionLibrary(shape1, "F1_O1", &library_expected));
   }
@@ -992,19 +1005,21 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
     GraphDefBuilder shape2(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
         KeyPlaceholderShape(shape2.opts().WithName("KnownShape/_0"));
-    Node* recv1 =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   shape2.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv1 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                             {DT_FLOAT, DT_FLOAT}, shape2.opts());
     Node* e = Binary(ops::NodeOut(recv1, 0), ops::NodeOut(recv1, 1),
-                     shape2.opts().WithName("E"));
-    Node* recv2 =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2",
-                   {DT_FLOAT, DT_FLOAT},
-                   shape2.opts().WithName("outside_compilation_F1_O2_recv"));
-    Node* h = Binary(ops::NodeOut(recv2, 0), e, shape2.opts().WithName("H"));
-    SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2",
-                 {h}, shape2.opts().WithName("outside_compilation_F1_O2_send"));
+                     shape2.opts()
+                         .WithName("E")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O2",
+                             {DT_FLOAT, DT_FLOAT}, shape2.opts());
+    Node* h = Binary(ops::NodeOut(recv2, 0), e,
+                     shape2.opts()
+                         .WithName("H")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O2"));
+    SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O2", {h}, shape2.opts());
     TF_EXPECT_OK(
         AddGraphDefToFunctionLibrary(shape2, "F1_O2", &library_expected));
   }
@@ -1054,28 +1069,32 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
 
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv1 =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   b2.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv1 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                             {DT_FLOAT, DT_FLOAT}, b2.opts());
     Node* e = Binary(ops::NodeOut(recv1, 0), ops::NodeOut(recv1, 1),
-                     b2.opts().WithName("E").WithControlInputs({recv1, b}));
-    Node* send1 = SendFromHost(ops::NodeOut(key_constant, 0),
-                               "host_compute_channel_F1_O1", {e},
-                               b2.opts()
-                                   .WithName("outside_compilation_F1_O1_send")
-                                   .WithControlInput(e));
-
-    Node* recv2 =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2",
-                   {DT_FLOAT, DT_FLOAT},
-                   b2.opts().WithName("outside_compilation_F1_O2_recv"));
+                     b2.opts()
+                         .WithName("E")
+                         .WithControlInputs({recv1, b})
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    Node* send1 = SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e},
+                               b2.opts().WithControlInput(e));
+
+    Node* recv2 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O2",
+                             {DT_FLOAT, DT_FLOAT}, b2.opts());
     Node* g = Binary(e, ops::NodeOut(recv2, 1),
-                     b2.opts().WithName("G").WithControlInputs({recv2, e}));
-    Node* h = Binary(ops::NodeOut(recv2, 0), e, b2.opts().WithName("H"));
-    Node* send2 = SendFromHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O2", {h},
-        b2.opts().WithName("outside_compilation_F1_O2_send"));
+                     b2.opts()
+                         .WithName("G")
+                         .WithControlInputs({recv2, e})
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O2"));
+    Node* h = Binary(ops::NodeOut(recv2, 0), e,
+                     b2.opts()
+                         .WithName("H")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O2"));
+    Node* send2 =
+        SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O2", {h}, b2.opts());
 
     Node* s = Sequencer(b2.opts()
                             .WithName("F1_sequencer")
@@ -1139,14 +1158,14 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
     GraphDefBuilder shape(GraphDefBuilder::kFailImmediately);
     Node* key_constant =
         KeyPlaceholderShape(shape.opts().WithName("KnownShape/_0"));
-    Node* recv =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   shape.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT, DT_FLOAT}, shape.opts());
     Node* e = Binary(ops::NodeOut(recv, 0), ops::NodeOut(recv, 1),
-                     shape.opts().WithName("E"));
-    SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                 {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
+                     shape.opts()
+                         .WithName("E")
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape.opts());
     TF_EXPECT_OK(
         AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
@@ -1207,17 +1226,16 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
 
     Node* key_constant1 =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv1 =
-        RecvAtHost(ops::NodeOut(key_constant1, 0), "host_compute_channel_F1_O1",
-                   {DT_FLOAT, DT_FLOAT},
-                   b2.opts().WithName("outside_compilation_F1_O1_recv"));
+    Node* recv1 = RecvAtHost(ops::NodeOut(key_constant1, 0), "F1", "O1",
+                             {DT_FLOAT, DT_FLOAT}, b2.opts());
     Node* e = Binary(ops::NodeOut(recv1, 0), ops::NodeOut(recv1, 1),
-                     b2.opts().WithName("E").WithControlInputs({recv1, b}));
-    Node* send1 = SendFromHost(ops::NodeOut(key_constant1, 0),
-                               "host_compute_channel_F1_O1", {e},
-                               b2.opts()
-                                   .WithName("outside_compilation_F1_O1_send")
-                                   .WithControlInput(e));
+                     b2.opts()
+                         .WithName("E")
+                         .WithControlInputs({recv1, b})
+                         .WithAttr("_encapsulate", "F1")
+                         .WithAttr("_outside", "O1"));
+    Node* send1 = SendFromHost(ops::NodeOut(key_constant1, 0), "F1", "O1", {e},
+                               b2.opts().WithControlInput(e));
     Node* s1 = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
         "F1");
@@ -1229,13 +1247,15 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
 
     Node* key_constant2 =
         KeyPlaceholder("F2", b2.opts().WithName("F2_key_placeholder"));
-    Node* recv2 = RecvAtHost(
-        ops::NodeOut(key_constant2, 0), "host_compute_channel_F2_O1",
-        {DT_FLOAT}, b2.opts().WithName("outside_compilation_F2_O1_recv"));
-    Node* h = Binary(ops::NodeOut(call1, 1), recv2, b2.opts().WithName("H"));
-    Node* send2 = SendFromHost(
-        ops::NodeOut(key_constant2, 0), "host_compute_channel_F2_O1", {h},
-        b2.opts().WithName("outside_compilation_F2_O1_send"));
+    Node* recv2 = RecvAtHost(ops::NodeOut(key_constant2, 0), "F2", "O1",
+                             {DT_FLOAT}, b2.opts());
+    Node* h = Binary(ops::NodeOut(call1, 1), recv2,
+                     b2.opts()
+                         .WithName("H")
+                         .WithAttr("_encapsulate", "F2")
+                         .WithAttr("_outside", "O1"));
+    Node* send2 = SendFromHost(ops::NodeOut(key_constant2, 0), "F2", "O1", {h},
+                               b2.opts());
 
     Node* s2 = Sequencer(
         b2.opts().WithName("F2_sequencer").WithControlInputs({recv2, send2}),
@@ -1311,12 +1331,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
     Node* a = InputShaped(b2.opts().WithName("A"));
     Node* b = Input(b2.opts().WithName("B"));
 
-    Node* e = Unary(a, b2.opts().WithName("E"));
+    Node* e = Unary(a, b2.opts()
+                           .WithName("E")
+                           .WithAttr("_encapsulate", "F1")
+                           .WithAttr("_outside", "O1"));
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* send1 = SendFromHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {e},
-        b2.opts().WithName("outside_compilation_F1_O1_send"));
+    Node* send1 =
+        SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, b2.opts());
     Node* s1 = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInput(send1), "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
@@ -1395,12 +1417,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
     Node* recv1 =
-        RecvAtHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                   {}, b2.opts().WithName("outside_compilation_F1_O1_recv"));
-    Node* e = Unary(a, b2.opts().WithName("E").WithControlInput(recv1));
-    Node* send1 = SendFromHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {e},
-        b2.opts().WithName("outside_compilation_F1_O1_send"));
+        RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1", {}, b2.opts());
+    Node* e = Unary(a, b2.opts()
+                           .WithName("E")
+                           .WithControlInput(recv1)
+                           .WithAttr("_encapsulate", "F1")
+                           .WithAttr("_outside", "O1"));
+    Node* send1 =
+        SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, b2.opts());
     Node* s1 = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
         "F1");
@@ -1470,10 +1494,12 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
 
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv1 = RecvAtHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {DT_FLOAT},
-        b2.opts().WithName("outside_compilation_F1_O1_recv"));
-    Node* e = Unary(recv1, b2.opts().WithName("E"));
+    Node* recv1 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                             {DT_FLOAT}, b2.opts());
+    Node* e = Unary(recv1, b2.opts()
+                               .WithName("E")
+                               .WithAttr("_encapsulate", "F1")
+                               .WithAttr("_outside", "O1"));
     Node* s1 = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInput(recv1), "F1");
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
@@ -1547,15 +1573,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
 
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv1 = RecvAtHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {DT_FLOAT},
-        b2.opts().WithName("outside_compilation_F1_O1_recv"));
-    Node* e = Unary(recv1, b2.opts().WithName("E"));
-    Node* send1 = SendFromHost(ops::NodeOut(key_constant, 0),
-                               "host_compute_channel_F1_O1", {},
-                               b2.opts()
-                                   .WithName("outside_compilation_F1_O1_send")
-                                   .WithControlInput(e));
+    Node* recv1 = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                             {DT_FLOAT}, b2.opts());
+    Node* e = Unary(recv1, b2.opts()
+                               .WithName("E")
+                               .WithAttr("_encapsulate", "F1")
+                               .WithAttr("_outside", "O1"));
+    Node* send1 = SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {},
+                               b2.opts().WithControlInput(e));
     Node* s1 = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInputs({recv1, send1}),
         "F1");
@@ -1615,7 +1640,10 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputsOrOutputs) {
     Node* a = Input(b2.opts().WithName("A"));
     Node* b = Input(b2.opts().WithName("B"));
 
-    Node* e = Unary(a, b2.opts().WithName("E"));
+    Node* e = Unary(a, b2.opts()
+                           .WithName("E")
+                           .WithAttr("_encapsulate", "F1")
+                           .WithAttr("_outside", "O1"));
     NodeBuilder node_builder1("F1", "F1", lib_def.get());
     node_builder1.Input(a).Input(b);
     Node* call1 = b2.opts().FinalizeBuilder(&node_builder1);
@@ -1666,12 +1694,14 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
     Node* key_constant =
         KeyPlaceholderShape(shape.opts().WithName("KnownShape/_0"));
     Node* known = KnownShape({2}, shape.opts().WithName("KnownShape/_1"));
-    Node* recv = RecvAtHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {DT_FLOAT},
-        shape.opts().WithName("outside_compilation_F1_O1_recv"));
-    Node* e = BinaryUnknownShape(known, recv, shape.opts().WithName("E"));
-    SendFromHost(ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1",
-                 {e}, shape.opts().WithName("outside_compilation_F1_O1_send"));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT}, shape.opts());
+    Node* e = BinaryUnknownShape(known, recv,
+                                 shape.opts()
+                                     .WithName("E")
+                                     .WithAttr("_encapsulate", "F1")
+                                     .WithAttr("_outside", "O1"));
+    SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e}, shape.opts());
     TF_EXPECT_OK(
         AddGraphDefToFunctionLibrary(shape, "F1_O1", &library_expected));
   }
@@ -1709,17 +1739,16 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
 
     Node* key_constant =
         KeyPlaceholder("F1", b2.opts().WithName("F1_key_placeholder"));
-    Node* recv = RecvAtHost(
-        ops::NodeOut(key_constant, 0), "host_compute_channel_F1_O1", {DT_FLOAT},
-        b2.opts().WithName("outside_compilation_F1_O1_recv"));
-    Node* e = BinaryUnknownShape(
-        c, ops::NodeOut(recv, 0),
-        b2.opts().WithName("E").WithControlInputs({recv, b}));
-    Node* send = SendFromHost(ops::NodeOut(key_constant, 0),
-                              "host_compute_channel_F1_O1", {e},
-                              b2.opts()
-                                  .WithName("outside_compilation_F1_O1_send")
-                                  .WithControlInput(e));
+    Node* recv = RecvAtHost(ops::NodeOut(key_constant, 0), "F1", "O1",
+                            {DT_FLOAT}, b2.opts());
+    Node* e = BinaryUnknownShape(c, ops::NodeOut(recv, 0),
+                                 b2.opts()
+                                     .WithName("E")
+                                     .WithControlInputs({recv, b})
+                                     .WithAttr("_encapsulate", "F1")
+                                     .WithAttr("_outside", "O1"));
+    Node* send = SendFromHost(ops::NodeOut(key_constant, 0), "F1", "O1", {e},
+                              b2.opts().WithControlInput(e));
 
     Node* s = Sequencer(
         b2.opts().WithName("F1_sequencer").WithControlInputs({recv, send}),
diff --git a/tensorflow/contrib/tpu/ops/replication_ops.cc b/tensorflow/contrib/tpu/ops/replication_ops.cc
index cba71c6b98..3bdf7c2f83 100644
--- a/tensorflow/contrib/tpu/ops/replication_ops.cc
+++ b/tensorflow/contrib/tpu/ops/replication_ops.cc
@@ -27,6 +27,7 @@ REGISTER_OP("TPUReplicateMetadata")
     .Attr("topology: string = \"\"")
     .Attr("device_assignment: list(int) = []")
     .Attr("computation_shape: list(int) = []")
+    .Attr("host_compute_core: list(string) = []")
     .SetShapeFn(shape_inference::UnknownShape);
 
 REGISTER_OP("TPUReplicatedInput")
@@ -68,6 +69,7 @@ REGISTER_OP("TPUReplicate")
     .Attr("num_replicas: int >= 1")
     .Attr("topology: string = \"\"")
     .Attr("device_assignment: list(int) = []")
+    .Attr("host_compute_core: list(string) = []")
     .Attr("computation_shape: list(int) = []")
     .Attr("Tinputs: list(type) >= 0")
     .Attr("Tbroadcast_inputs: list(type) >= 0")
-- 
GitLab


From b80960d8b7c87a3cf221cdbbb9c68c5970bfd3c7 Mon Sep 17 00:00:00 2001
From: Zhixian Yan <zhixianyan@google.com>
Date: Thu, 29 Mar 2018 12:39:33 -0700
Subject: [PATCH 689/960] Add more tflite hosted models like resnet,
 inception-v4, nasnet.

PiperOrigin-RevId: 190970367
---
 tensorflow/contrib/lite/g3doc/models.md | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/g3doc/models.md b/tensorflow/contrib/lite/g3doc/models.md
index 48f43d4fc4..d8134d5a00 100644
--- a/tensorflow/contrib/lite/g3doc/models.md
+++ b/tensorflow/contrib/lite/g3doc/models.md
@@ -1,7 +1,13 @@
 # List of Hosted Models
 
-*   [Inception V3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
-*   [Inception V3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
+*   [NASNet large](https://storage.googleapis.com/download.tensorflow.org/models/tflite/nasnet_large_2018_03_27.zip)
+*   [NASNet mobile](https://storage.googleapis.com/download.tensorflow.org/models/tflite/nasnet_mobile_2018_03_27.zip)
+*   [ResNet v2 101](https://storage.googleapis.com/download.tensorflow.org/models/tflite/resnet_v2_101_2018_03_27.zip)
+*   [ResNet v2 50](https://storage.googleapis.com/download.tensorflow.org/models/tflite/resnet_v2_50_2018_03_27.zip)
+*   [Inception ResNet v2](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_resnet_v2_2018_03_27.zip)
+*   [Inception v4](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v4_2018_03_27.zip)
+*   [Inception v3 2015](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_2015_2017_11_10.zip)
+*   [Inception v3 Slim 2016](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
 *   [Mobilenet 0.25 128 Float](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_0.25_128_float_2017_11_08.zip)
 *   [Mobilenet 0.25 160 Float](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_0.25_160_float_2017_11_08.zip)
 *   [Mobilenet 0.25 192 Float](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_0.25_192_float_2017_11_08.zip)
-- 
GitLab


From e58e4c754fa6145af2a411b940d8f7347a071b6f Mon Sep 17 00:00:00 2001
From: Jonathan Hseu <jhseu@google.com>
Date: Thu, 29 Mar 2018 12:54:59 -0700
Subject: [PATCH 690/960] Minor adjustments to an error message.

PiperOrigin-RevId: 190972253
---
 tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
index 493d1848c0..eea57ed336 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
@@ -72,9 +72,9 @@ def _query_tpu_system_metadata(master_address, run_config,
               tpu_core_count += 1
           break
     except errors.DeadlineExceededError:
-      msg = ('Fail to connect Tensorflow master. It could be the TPU worker is '
-             'not ready (still under scheduling) or Tensorflow '
-             'master address is correct: got (%s).' %
+      msg = ('Failed to connect to the Tensorflow master. The TPU worker may '
+             'not be ready (still scheduling) or the Tensorflow master address '
+             'is incorrect: got (%s).' %
              (master_address))
 
       # TODO(xiejw): For local or grpc master we might not need retry logic
-- 
GitLab


From d9e5f2754cabd9680d5481464a4085e79856eb78 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Thu, 29 Mar 2018 12:58:43 -0700
Subject: [PATCH 691/960] Avoid evaluating SaveSpec Tensors multiple times when
 executing eagerly

The Saver now calls a SaveSpec callable once when saving and not at all when restoring. Previously saving evaluated the callable twice and restoring once (copying a variable's value each time).

Requires a dtype be passed to a SaveSpec if its tensor is callable.

PiperOrigin-RevId: 190972754
---
 tensorflow/python/training/saver.py      | 40 +++++++++++++++++++-----
 tensorflow/python/training/saver_test.py | 31 ++++++++++++++++++
 2 files changed, 63 insertions(+), 8 deletions(-)

diff --git a/tensorflow/python/training/saver.py b/tensorflow/python/training/saver.py
index cec581d997..e40b8d22ed 100644
--- a/tensorflow/python/training/saver.py
+++ b/tensorflow/python/training/saver.py
@@ -91,17 +91,27 @@ class BaseSaverBuilder(object):
   class SaveSpec(object):
     """Class used to describe tensor slices that need to be saved."""
 
-    def __init__(self, tensor, slice_spec, name):
+    def __init__(self, tensor, slice_spec, name, dtype=None):
       """Creates a `SaveSpec` object.
 
       Args:
         tensor: the tensor to save or callable that produces a tensor to save.
         slice_spec: the slice to be saved. See `Variable.SaveSliceInfo`.
         name: the name to save the tensor under.
+        dtype: The data type of the Tensor. Required if `tensor` is callable.
+          Used for error checking in the restore op.
       """
       self._tensor = tensor
       self.slice_spec = slice_spec
       self.name = name
+      if callable(self._tensor):
+        if dtype is None:
+          raise AssertionError(
+              "When passing a callable `tensor` to a SaveSpec, an explicit "
+              "dtype must be provided.")
+        self.dtype = dtype
+      else:
+        self.dtype = tensor.dtype
 
     @property
     def tensor(self):
@@ -117,14 +127,27 @@ class BaseSaverBuilder(object):
         op: the "producer" object that this class wraps; it produces a list of
           tensors to save.  E.g., a "Variable" object saving its backing tensor.
         specs: a list of SaveSpec, each element of which describes one tensor to
-          save under this object.
+          save under this object. All Tensors must be on the same device.
         name: the name to save the object under.
       """
       self.op = op
       self.specs = specs
       self.name = name
-      # The device of this saveable. All tensors must be on the same device.
-      self.device = specs[0].tensor.device
+      self._device = None
+
+    @property
+    def device(self):
+      """The device for SaveSpec Tensors."""
+      # Note that SaveSpec.tensor runs Tensor-gathering ops when executing
+      # eagerly, making this call potentially very expensive.
+      #
+      # TODO(allenl): Consider another way to gather device information. Lower
+      # priority since this property isn't part of the normal save()/restore()
+      # workflow, but does come up when some alternative builders are passed to
+      # the Saver.
+      if self._device is None:
+        self._device = self.specs[0].tensor.device
+      return self._device
 
     def restore(self, restored_tensors, restored_shapes):
       """Restores this object from 'restored_tensors'.
@@ -148,7 +171,7 @@ class BaseSaverBuilder(object):
     """SaveableObject implementation that handles Variables."""
 
     def __init__(self, var, slice_spec, name):
-      spec = BaseSaverBuilder.SaveSpec(var, slice_spec, name)
+      spec = BaseSaverBuilder.SaveSpec(var, slice_spec, name, dtype=var.dtype)
       super(BaseSaverBuilder.VariableSaveable, self).__init__(var, [spec], name)
 
     def restore(self, restored_tensors, restored_shapes):
@@ -186,7 +209,8 @@ class BaseSaverBuilder(object):
         raise ValueError(
             "Saveable is neither a resource variable nor a read operation."
             " Got: %s" % repr(var))
-      spec = BaseSaverBuilder.SaveSpec(tensor, slice_spec, name)
+      spec = BaseSaverBuilder.SaveSpec(tensor, slice_spec, name,
+                                       dtype=var.dtype)
       super(BaseSaverBuilder.ResourceVariableSaveable, self).__init__(
           var, [spec], name)
 
@@ -295,7 +319,7 @@ class BaseSaverBuilder(object):
               filename_tensor,
               [spec.name],
               [spec.slice_spec],
-              [spec.tensor.dtype])[0])
+              [spec.dtype])[0])
 
     return tensors
   # pylint: enable=unused-argument
@@ -854,7 +878,7 @@ class BulkSaverBuilder(BaseSaverBuilder):
     restore_specs = []
     for saveable in saveables:
       for spec in saveable.specs:
-        restore_specs.append((spec.name, spec.slice_spec, spec.tensor.dtype))
+        restore_specs.append((spec.name, spec.slice_spec, spec.dtype))
 
     names, slices, dtypes = zip(*restore_specs)
     # Load all tensors onto CPU 0 for compatibility with existing code.
diff --git a/tensorflow/python/training/saver_test.py b/tensorflow/python/training/saver_test.py
index d1c24b3930..14dda79979 100644
--- a/tensorflow/python/training/saver_test.py
+++ b/tensorflow/python/training/saver_test.py
@@ -2980,6 +2980,37 @@ class CheckpointableCompatibilityTests(test.TestCase):
       self.assertEqual(42., self.evaluate(v.non_dep_variable))
       self.assertEqual(42., self.evaluate(v.mirrored))
 
+  def testSingleTensorEvaluation(self):
+
+    class _CountingSaveable(saver_module.BaseSaverBuilder.SaveableObject):
+
+      def __init__(self, name):
+        self.eval_count = 0
+        def _tensor():
+          self.eval_count += 1
+          return constant_op.constant([1.])
+        dummy_op = constant_op.constant([2.])
+        super(_CountingSaveable, self).__init__(
+            dummy_op,
+            [saver_module.BaseSaverBuilder.SaveSpec(
+                _tensor, "", name, dtype=dummy_op.dtype)],
+            name)
+
+      def restore(self, restored_tensors, restored_shapes):
+        """Restore the same value into both variables."""
+        pass
+
+    with context.eager_mode():
+      v = _CountingSaveable("foo")
+      saver = saver_module.Saver(var_list=[v])
+      test_dir = self.get_temp_dir()
+      prefix = os.path.join(test_dir, "ckpt")
+      with self.test_session() as sess:
+        save_path = saver.save(sess, prefix)
+        self.assertEqual(1, v.eval_count)
+        saver.restore(sess, save_path)
+        self.assertEqual(1, v.eval_count)
+
 
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From a259ba951d3af9f62a0f95a881abf9ebaa45782b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 13:18:54 -0700
Subject: [PATCH 692/960] Fix docstring.

PiperOrigin-RevId: 190975767
---
 tensorflow/contrib/autograph/converters/break_statements.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/autograph/converters/break_statements.py b/tensorflow/contrib/autograph/converters/break_statements.py
index 721bc0ccd0..48026bccab 100644
--- a/tensorflow/contrib/autograph/converters/break_statements.py
+++ b/tensorflow/contrib/autograph/converters/break_statements.py
@@ -27,7 +27,7 @@ from tensorflow.contrib.autograph.pyct.static_analysis.annos import NodeAnno
 
 
 class BreakCanonicalizationTransformer(transformer.Base):
-  """Canonicalizes continue statements into additional conditionals."""
+  """Canonicalizes break statements into additional conditionals."""
 
   def __init__(self, context):
     super(BreakCanonicalizationTransformer, self).__init__(context)
-- 
GitLab


From eb2be37c12ae2b6c996f3f4c064e3d10f9565eab Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 29 Mar 2018 13:22:45 -0700
Subject: [PATCH 693/960] Internal change.

PiperOrigin-RevId: 190976338
---
 tensorflow/python/layers/normalization.py | 76 +++++++++++------------
 tensorflow/python/training/distribute.py  | 10 +++
 2 files changed, 46 insertions(+), 40 deletions(-)

diff --git a/tensorflow/python/layers/normalization.py b/tensorflow/python/layers/normalization.py
index 29fb92ccb5..83b201e642 100644
--- a/tensorflow/python/layers/normalization.py
+++ b/tensorflow/python/layers/normalization.py
@@ -32,12 +32,12 @@ from tensorflow.python.framework import tensor_shape
 from tensorflow.python.layers import base
 from tensorflow.python.layers import utils
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
-from tensorflow.python.ops import gen_resource_variable_ops
 from tensorflow.python.ops import resource_variable_ops
-from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import state_ops
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import moving_averages
 from tensorflow.python.util.tf_export import tf_export
 
@@ -178,6 +178,11 @@ class BatchNormalization(base.Layer):
       self.renorm_clipping = renorm_clipping
       self.renorm_momentum = renorm_momentum
 
+  def _add_tower_local_variable(self, *args, **kwargs):
+    tower_context = distribute_lib.get_tower_context()
+    with tower_context.tower_local_var_scope('mean'):
+      return self.add_variable(*args, **kwargs)
+
   def build(self, input_shape):
     input_shape = tensor_shape.TensorShape(input_shape)
     if not input_shape.ndims:
@@ -305,14 +310,14 @@ class BatchNormalization(base.Layer):
         self._scope.set_partitioner(None)
       else:
         partitioner = None
-      self.moving_mean = self.add_variable(
+      self.moving_mean = self._add_tower_local_variable(
           name='moving_mean',
           shape=param_shape,
           dtype=param_dtype,
           initializer=self.moving_mean_initializer,
           trainable=False)
 
-      self.moving_variance = self.add_variable(
+      self.moving_variance = self._add_tower_local_variable(
           name='moving_variance',
           shape=param_shape,
           dtype=param_dtype,
@@ -328,7 +333,7 @@ class BatchNormalization(base.Layer):
         # stack to be cleared. The nested ones use a `lambda` to set the desired
         # device and ignore any devices that may be set by the custom getter.
         def _renorm_variable(name, shape):
-          var = self.add_variable(
+          var = self._add_tower_local_variable(
               name=name,
               shape=shape,
               dtype=param_dtype,
@@ -336,24 +341,19 @@ class BatchNormalization(base.Layer):
               trainable=False)
           return var
 
-        with ops.device(None):
-          device = (
-              self.moving_mean.device if context.executing_eagerly() else
-              (lambda _: self.moving_mean.device))
-          with ops.device(device):
-            self.renorm_mean = _renorm_variable('renorm_mean', param_shape)
-            self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ())
-          # We initialize renorm_stddev to 0, and maintain the (0-initialized)
-          # renorm_stddev_weight. This allows us to (1) mix the average
-          # stddev with the minibatch stddev early in training, and (2) compute
-          # the unbiased average stddev by dividing renorm_stddev by the weight.
-          device = (
-              self.moving_variance.device if context.executing_eagerly() else
-              (lambda _: self.moving_variance.device))
-          with ops.device(device):
-            self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape)
-            self.renorm_stddev_weight = _renorm_variable(
-                'renorm_stddev_weight', ())
+        with distribute_lib.get_distribution_strategy().colocate_vars_with(
+            self.moving_mean):
+          self.renorm_mean = _renorm_variable('renorm_mean', param_shape)
+          self.renorm_mean_weight = _renorm_variable('renorm_mean_weight', ())
+        # We initialize renorm_stddev to 0, and maintain the (0-initialized)
+        # renorm_stddev_weight. This allows us to (1) mix the average
+        # stddev with the minibatch stddev early in training, and (2) compute
+        # the unbiased average stddev by dividing renorm_stddev by the weight.
+        with distribute_lib.get_distribution_strategy().colocate_vars_with(
+            self.moving_variance):
+          self.renorm_stddev = _renorm_variable('renorm_stddev', param_shape)
+          self.renorm_stddev_weight = _renorm_variable('renorm_stddev_weight',
+                                                       ())
     finally:
       if partitioner:
         self._scope.set_partitioner(partitioner)
@@ -362,12 +362,11 @@ class BatchNormalization(base.Layer):
   def _assign_moving_average(self, variable, value, momentum):
     with ops.name_scope(None, 'AssignMovingAvg',
                         [variable, value, momentum]) as scope:
-      with ops.colocate_with(variable):
-        decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
-        if decay.dtype != variable.dtype.base_dtype:
-          decay = math_ops.cast(decay, variable.dtype.base_dtype)
-        update_delta = (variable - value) * decay
-        return state_ops.assign_sub(variable, update_delta, name=scope)
+      decay = ops.convert_to_tensor(1.0 - momentum, name='decay')
+      if decay.dtype != variable.dtype.base_dtype:
+        decay = math_ops.cast(decay, variable.dtype.base_dtype)
+      update_delta = (variable - value) * decay
+      return state_ops.assign_sub(variable, update_delta, name=scope)
 
   def _fused_batch_norm(self, inputs, training):
     """Returns the output of fused batch norm."""
@@ -473,16 +472,13 @@ class BatchNormalization(base.Layer):
         return array_ops.identity(var)
       return utils.smart_cond(training, _do_update, _fake_update)
 
-    with ops.colocate_with(self.moving_mean):
-      new_mean = _update_renorm_variable(self.renorm_mean,
-                                         self.renorm_mean_weight,
-                                         mean)
-    with ops.colocate_with(self.moving_variance):
-      new_stddev = _update_renorm_variable(self.renorm_stddev,
-                                           self.renorm_stddev_weight,
-                                           stddev)
-      # Make sqrt(moving_variance + epsilon) = new_stddev.
-      new_variance = math_ops.square(new_stddev) - self.epsilon
+    # TODO(yuefengz): colocate the operations
+    new_mean = _update_renorm_variable(self.renorm_mean,
+                                       self.renorm_mean_weight, mean)
+    new_stddev = _update_renorm_variable(self.renorm_stddev,
+                                         self.renorm_stddev_weight, stddev)
+    # Make sqrt(moving_variance + epsilon) = new_stddev.
+    new_variance = math_ops.square(new_stddev) - self.epsilon
 
     return (r, d, new_mean, new_variance)
 
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index f98872775a..d5106752dd 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1082,6 +1082,16 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     return _CurrentDistributionContext(
         self, variable_scope.variable_creator_scope(creator))
 
+  def tower_local_var_scope(self, reduce_method):
+    """Does not set to resource variables."""
+    def create_tower_local_variable(next_creator, *args, **kwargs):
+      _require_distribution_strategy_scope(self)
+      kwargs["tower_local_reduce_method"] = reduce_method
+      return next_creator(*args, **kwargs)
+
+    _require_distribution_strategy_scope(self)
+    return variable_scope.variable_creator_scope(create_tower_local_variable)
+
   def colocate_vars_with(self, colocate_with_variable):
     """Does not require `self.scope`."""
     _require_distribution_strategy_scope(self)
-- 
GitLab


From ae94d2caaa713393d9c046f46e1ed7303ecf308c Mon Sep 17 00:00:00 2001
From: Nathan Burnham <nburn42@gmail.com>
Date: Thu, 29 Mar 2018 16:26:07 -0400
Subject: [PATCH 694/960] Fixed a spelling error that broke the GANEstimator
 documentation example (#18097)

Fixed a spelling error that broke the tfgan.estimator.GANEstimator documentation example
---
 .../contrib/gan/python/estimator/python/gan_estimator_impl.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
index 082c42eba1..e3fc6bf0f0 100644
--- a/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
+++ b/tensorflow/contrib/gan/python/estimator/python/gan_estimator_impl.py
@@ -88,8 +88,8 @@ class GANEstimator(estimator.Estimator):
           discriminator_fn=discriminator_fn,
           generator_loss_fn=tfgan.losses.wasserstein_generator_loss,
           discriminator_loss_fn=tfgan.losses.wasserstein_discriminator_loss,
-          generator_optimizer=tf.train.AdamOptimizier(0.1, 0.5),
-          discriminator_optimizer=tf.train.AdamOptimizier(0.1, 0.5))
+          generator_optimizer=tf.train.AdamOptimizer(0.1, 0.5),
+          discriminator_optimizer=tf.train.AdamOptimizer(0.1, 0.5))
 
       # Train estimator.
       gan_estimator.train(train_input_fn, steps)
-- 
GitLab


From 690ecae1f2519ed54693d51af0d28372a02ff31e Mon Sep 17 00:00:00 2001
From: Dan Moldovan <mdan@google.com>
Date: Thu, 29 Mar 2018 16:26:19 -0400
Subject: [PATCH 695/960] Initial commit for the demo notebook (#18093)

* Create touch.txt

Dummy file to create the branch and directory structure

* Add files via upload

Initial commit

* Delete touch.txt
---
 .../notebooks/dev_summit_2018_demo.ipynb      | 1970 +++++++++++++++++
 1 file changed, 1970 insertions(+)
 create mode 100644 tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb

diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
new file mode 100644
index 0000000000..3129a39a4b
--- /dev/null
+++ b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
@@ -0,0 +1,1970 @@
+{
+  "nbformat": 4,
+  "nbformat_minor": 0,
+  "metadata": {
+    "colab": {
+      "name": "Dev Summit 2018 - Autograph",
+      "version": "0.3.2",
+      "views": {},
+      "default_view": {},
+      "provenance": [
+        {
+          "file_id": "1wCZUh73zTNs1jzzYjqoxMIdaBWCdKJ2K",
+          "timestamp": 1522238054357
+        },
+        {
+          "file_id": "1_HpC-RrmIv4lNaqeoslUeWaX8zH5IXaJ",
+          "timestamp": 1521743157199
+        },
+        {
+          "file_id": "1mjO2fQ2F9hxpAzw2mnrrUkcgfb7xSGW-",
+          "timestamp": 1520522344607
+        }
+      ],
+      "collapsed_sections": []
+    },
+    "kernelspec": {
+      "name": "python2",
+      "display_name": "Python 2"
+    }
+  },
+  "cells": [
+    {
+      "metadata": {
+        "id": "g7nGs4mzVUHP",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# Experimental: TF Autograph\n",
+        "**TensorFlow Dev Summit, 2018.**\n",
+        "\n",
+        "This interactive notebook demonstrates **autograph**, an experimental source-code transformation library to automatically convert TF.Eager and Python code to TensorFlow graphs.\n",
+        "\n",
+        "**Note: this is pre-alpha software!** The notebook works best with Python 2, for now.\n",
+        "\n",
+        "> ![alt text](https://lh3.googleusercontent.com/QOvy0clmg7siaVKzwmSPAjicWWNQ0OeyaB16plDjSJMf35WD3vLjF6mz4CGrhSHw60HnlZPJjkyDCBzw5XOI0oBGSewyYw=s688)\n",
+        "\n",
+        "### Table of Contents\n",
+        "1. _Write Eager code that is fast and scalable._\n",
+        "2. _Case study: complex control flow._\n",
+        "3. _Case study: training MNIST with Keras._\n",
+        "4. _Case study: building an RNN._"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "uFcgBENZqkB2",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "# Install TensorFlow; note that Colab notebooks run remotely, on virtual\n",
+        "# instances provided by Google.\n",
+        "!pip install -U -q tf-nightly"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "Pa2qpEmoVOGe",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "import os\n",
+        "import time\n",
+        "\n",
+        "import tensorflow as tf\n",
+        "from tensorflow.contrib import autograph\n",
+        "\n",
+        "import matplotlib.pyplot as plt\n",
+        "import numpy as np\n",
+        "import six\n",
+        "\n",
+        "from google.colab import widgets"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "ZVKfj5ttVkqz",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# 1. Write Eager code that is fast and scalable\n",
+        "\n",
+        "TF.Eager gives you more flexibility while coding, but at the cost of losing the benefits of TensorFlow graphs. For example, Eager does not currently support distributed training, exporting models, and a variety of memory and computation optimizations.\n",
+        "\n",
+        "Autograph gives you the best of both worlds: write your code in an Eager style, and we will automatically transform it into the equivalent TF graph code. The graph code can be executed eagerly (as a single op), included as part of a larger graph, or exported."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "snaZRFdWd9ym",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "For example, autograph can convert a function like this:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "9__n8cSIeDnD",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def g(x):\n",
+        "  if x > 0:\n",
+        "    x = x * x\n",
+        "  else:\n",
+        "    x = 0\n",
+        "  return x"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "gq0eQcuReHET",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "... into a TF graph-building function:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "sELSn599ePUF",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 413
+        },
+        "outputId": "bb0c7216-1ca3-4da1-d1fb-589902cdcd1a",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345737505,
+          "user_tz": 240,
+          "elapsed": 243,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "print(autograph.to_code(g))"
+      ],
+      "execution_count": 4,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "from __future__ import print_function\n",
+            "import tensorflow as tf\n",
+            "from tensorflow.contrib.autograph.impl import api as autograph_api\n",
+            "from tensorflow.contrib.autograph import utils as autograph_utils\n",
+            "\n",
+            "def tf__g(x):\n",
+            "  with tf.name_scope('g'):\n",
+            "\n",
+            "    def if_true():\n",
+            "      with tf.name_scope('if_true'):\n",
+            "        x_1, = x,\n",
+            "        x_1 = x_1 * x_1\n",
+            "        return x_1,\n",
+            "\n",
+            "    def if_false():\n",
+            "      with tf.name_scope('if_false'):\n",
+            "        x_1, = x,\n",
+            "        x_1 = 0\n",
+            "        return x_1,\n",
+            "    x = autograph_utils.run_cond(tf.greater(x, 0), if_true, if_false)\n",
+            "    return x\n",
+            "\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "j74n-8hEe6dk",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "You can then use the converted function as you would any regular TF op -- you can pass `Tensor` arguments and it will return `Tensor`s:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "AkVaY0-dfEbH",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 53
+        },
+        "outputId": "4ffe3757-c44d-424c-c2a8-7ddc973bfcce",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345737841,
+          "user_tz": 240,
+          "elapsed": 257,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "tf_g = autograph.to_graph(g)\n",
+        "\n",
+        "with tf.Graph().as_default():  \n",
+        "\n",
+        "  g_ops = tf_g(tf.constant(9))\n",
+        "\n",
+        "  with tf.Session() as sess:\n",
+        "    tf_g_result = sess.run(g_ops)\n",
+        "\n",
+        "  print('g(9) = %s' % g(9))\n",
+        "  print('tf_g(9) = %s' % tf_g_result)"
+      ],
+      "execution_count": 5,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "g(9) = 81\n",
+            "tf_g(9) = 81\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "trrHQBM1VnD0",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# 2. Case study: complex control flow\n",
+        "\n",
+        "Autograph can convert a large chunk of the Python language into graph-equivalent code, and we're adding new supported language features all the time. In this section, we'll give you a taste of some of the functionality in autograph.\n",
+        "Autograph will automatically convert most Python control flow statements into their correct graph equivalent.\n",
+        "  "
+      ]
+    },
+    {
+      "metadata": {
+        "id": "u0YG3DPgZxoW",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "We support common statements like `while`, `for`, `if`, `break`, `return` and more. You can even nest them as much as you like. Imagine trying to write the graph version of this code by hand:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "xJYDzOcrZ8pI",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 35
+        },
+        "outputId": "6c244ee4-b141-4ad6-eefa-cfffa71f33c6",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345738402,
+          "user_tz": 240,
+          "elapsed": 483,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def sum_even(numbers):\n",
+        "  s = 0\n",
+        "  for n in numbers:\n",
+        "    if n % 2 > 0:\n",
+        "      continue\n",
+        "    s += n\n",
+        "  return s\n",
+        "\n",
+        "\n",
+        "tf_sum_even = autograph.to_graph(sum_even)\n",
+        "\n",
+        "with tf.Graph().as_default():  \n",
+        "  with tf.Session() as sess:\n",
+        "    result = sess.run(tf_sum_even(tf.constant([10, 12, 15, 20])))\n",
+        "\n",
+        "  print('Sum of even numbers: %s' % result)"
+      ],
+      "execution_count": 6,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Sum of even numbers: 42\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "_YXo4KOcbKrn",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Try replacing the `continue` in the above code with `break` -- Autograph supports that as well!"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "xHmC0rBIavW_",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "The Python code above is much more readable than the matching graph code. Autograph takes care of tediously converting every piece of Python code into the matching TensorFlow graph version for you, so that you can quickly write maintainable code, but still benefit from the optimizations and deployment benefits of graphs."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "UEHWGpBXbS7g",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Let's try some other useful Python constructs, like `print` and `assert`. We automatically convert Python `assert` statements into the equivalent `tf.Assert` code.  "
+      ]
+    },
+    {
+      "metadata": {
+        "id": "qUU57xlEbauI",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 53
+        },
+        "outputId": "add3db4a-2077-4dd5-f7a7-a5b5a4529c26",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345738697,
+          "user_tz": 240,
+          "elapsed": 253,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def f(x):\n",
+        "  assert x != 0, 'Do not pass zero!'\n",
+        "  return x * x\n",
+        "\n",
+        "tf_f = autograph.to_graph(f)\n",
+        "with tf.Graph().as_default():  \n",
+        "  with tf.Session() as sess:\n",
+        "    try:\n",
+        "      print(sess.run(tf_f(tf.constant(0))))\n",
+        "    except tf.errors.InvalidArgumentError as e:\n",
+        "      print('Got error message: %s' % e.message)"
+      ],
+      "execution_count": 7,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Got error message: assertion failed: [Do not pass zero!]\n",
+            "\t [[Node: f/Assert/Assert = Assert[T=[DT_STRING], summarize=3, _device=\"/job:localhost/replica:0/task:0/device:CPU:0\"](f/NotEqual, f/Assert/Assert/data_0)]]\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "w5hBZaVJbck4",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "You can also use `print` functions in-graph:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "6NdzRKLEboRv",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 35
+        },
+        "outputId": "fb82dfc3-790f-4127-87f6-361805be9e9b",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345739013,
+          "user_tz": 240,
+          "elapsed": 247,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def print_sign(n):\n",
+        "  if n >= 0:\n",
+        "    print(n, 'is positive!')\n",
+        "  else:\n",
+        "    print(n, 'is negative!')\n",
+        "  return n\n",
+        "\n",
+        "\n",
+        "tf_print_sign = autograph.to_graph(print_sign)\n",
+        "with tf.Graph().as_default():\n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(tf_print_sign(tf.constant(1)))"
+      ],
+      "execution_count": 8,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "1 is positive!\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "9u_Z3i3AivLA",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "We can convert lists to TensorArray, so appending to lists also works, with a few modifications:"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "MjhCQJVuiTNR",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 35
+        },
+        "outputId": "dc320b87-595b-4392-d29c-994486fd8a0a",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345744470,
+          "user_tz": 240,
+          "elapsed": 5391,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def f(n):\n",
+        "  numbers = []\n",
+        "  # We ask you to tell us about the element dtype.\n",
+        "  autograph.utils.set_element_type(numbers, tf.int32)\n",
+        "  for i in range(n):\n",
+        "    numbers.append(i)\n",
+        "  return numbers.stack() # Stack the list so that it can be used as a Tensor\n",
+        "\n",
+        "\n",
+        "tf_f = autograph.to_graph(f)\n",
+        "with tf.Graph().as_default():\n",
+        "  with tf.Session() as sess:\n",
+        "    print(sess.run(tf_f(tf.constant(5))))"
+      ],
+      "execution_count": 9,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "[0 1 2 3 4]\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "UdG8ZFrkTAF2",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "And all of these functionalities, and more, can be composed into more complicated code:\n"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "DVs6wt8NKaGQ",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 1
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 53
+        },
+        "cellView": "code",
+        "outputId": "0a4b8d08-8f65-4bbc-85ba-dc4c60563519",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345745186,
+          "user_tz": 240,
+          "elapsed": 658,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def print_primes(n):\n",
+        "  \"\"\"Returns all the prime numbers less than n.\"\"\"\n",
+        "  assert n > 0\n",
+        "  \n",
+        "  primes = []\n",
+        "  autograph.utils.set_element_type(primes, tf.int32)\n",
+        "  for i in range(2, n):\n",
+        "    is_prime = True\n",
+        "    for k in range(2, i):\n",
+        "      if i % k == 0:\n",
+        "        is_prime = False\n",
+        "        break\n",
+        "    if not is_prime:\n",
+        "      continue\n",
+        "    primes.append(i)\n",
+        "  all_primes = primes.stack()\n",
+        "\n",
+        "  print('The prime numbers less than', n, 'are:')\n",
+        "  print(all_primes)\n",
+        "  return tf.no_op()\n",
+        "\n",
+        "    \n",
+        "tf_print_primes = autograph.to_graph(print_primes)\n",
+        "with tf.Graph().as_default():  \n",
+        "  with tf.Session() as sess:\n",
+        "    n = tf.constant(50)\n",
+        "    sess.run(tf_print_primes(n))"
+      ],
+      "execution_count": 10,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "The prime numbers less than 50 are:\n",
+            "[ 2  3  5  7 11 13 17 19 23 29 31 37 41 43 47]\n"
+          ],
+          "name": "stdout"
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "JQ8kQT99VqDk",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# 3. Case study: training MNIST with Keras\n",
+        "\n",
+        "As we've seen, writing control flow in Autograph is easy. So running a training loop in graph should be easy as well!\n",
+        "\n",
+        "Here, we show an example of such a training loop for a simple Keras model that trains on MNIST."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "0CrtGWgwuLJr",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "import gzip\n",
+        "import shutil\n",
+        "\n",
+        "from six.moves import urllib\n",
+        "\n",
+        "\n",
+        "def download(directory, filename):\n",
+        "  filepath = os.path.join(directory, filename)\n",
+        "  if tf.gfile.Exists(filepath):\n",
+        "    return filepath\n",
+        "  if not tf.gfile.Exists(directory):\n",
+        "    tf.gfile.MakeDirs(directory)\n",
+        "  url = 'https://storage.googleapis.com/cvdf-datasets/mnist/' + filename + '.gz'\n",
+        "  zipped_filepath = filepath + '.gz'\n",
+        "  print('Downloading %s to %s' % (url, zipped_filepath))\n",
+        "  urllib.request.urlretrieve(url, zipped_filepath)\n",
+        "  with gzip.open(zipped_filepath, 'rb') as f_in, open(filepath, 'wb') as f_out:\n",
+        "    shutil.copyfileobj(f_in, f_out)\n",
+        "  os.remove(zipped_filepath)\n",
+        "  return filepath\n",
+        "\n",
+        "\n",
+        "def dataset(directory, images_file, labels_file):\n",
+        "  images_file = download(directory, images_file)\n",
+        "  labels_file = download(directory, labels_file)\n",
+        "\n",
+        "  def decode_image(image):\n",
+        "    # Normalize from [0, 255] to [0.0, 1.0]\n",
+        "    image = tf.decode_raw(image, tf.uint8)\n",
+        "    image = tf.cast(image, tf.float32)\n",
+        "    image = tf.reshape(image, [784])\n",
+        "    return image / 255.0\n",
+        "\n",
+        "  def decode_label(label):\n",
+        "    label = tf.decode_raw(label, tf.uint8)\n",
+        "    label = tf.reshape(label, [])\n",
+        "    return tf.to_int32(label)\n",
+        "\n",
+        "  images = tf.data.FixedLengthRecordDataset(\n",
+        "      images_file, 28 * 28, header_bytes=16).map(decode_image)\n",
+        "  labels = tf.data.FixedLengthRecordDataset(\n",
+        "      labels_file, 1, header_bytes=8).map(decode_label)\n",
+        "  return tf.data.Dataset.zip((images, labels))\n",
+        "\n",
+        "\n",
+        "def mnist_train(directory):\n",
+        "  return dataset(directory, 'train-images-idx3-ubyte',\n",
+        "                 'train-labels-idx1-ubyte')\n",
+        "\n",
+        "def mnist_test(directory):\n",
+        "  return dataset(directory, 't10k-images-idx3-ubyte', 't10k-labels-idx1-ubyte')"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "2zu1U9Nqir6L",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "First, we'll define a small three-layer neural network using the Keras API"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "x_MU13boiok2",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def mlp_model(input_shape):\n",
+        "  model = tf.keras.Sequential([\n",
+        "      tf.keras.layers.Dense(100, activation='relu', input_shape=input_shape),\n",
+        "      tf.keras.layers.Dense(100, activation='relu'),\n",
+        "      tf.keras.layers.Dense(10, activation='softmax')])\n",
+        "  model.build()\n",
+        "  return model"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "Wuqg3H8mi0Xj",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Let's connect the model definition (here abbreviated as `m`) to a loss function, so that we can train our model."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "W51sfbONiz_5",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def predict(m, x, y):\n",
+        "  y_p = m(x)\n",
+        "  losses = tf.keras.losses.categorical_crossentropy(y, y_p)\n",
+        "  l = tf.reduce_mean(losses)\n",
+        "  accuracies = tf.keras.metrics.categorical_accuracy(y, y_p)\n",
+        "  accuracy = tf.reduce_mean(accuracies)\n",
+        "  return l, accuracy"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "035tNWQki9tr",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Now the final piece of the problem specification (before loading data, and clicking everything together) is backpropagating the loss through the model, and optimizing the weights using the gradient."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "CsAD0ajbi9iZ",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def fit(m, x, y, opt):\n",
+        "  l, accuracy = predict(m, x, y)\n",
+        "  opt.minimize(l)\n",
+        "  return l, accuracy"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "PcVRIacKjSwb",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "These are some utility functions to download data and generate batches for training"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "RVw57HdTjPzi",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def setup_mnist_data(is_training, hp, batch_size):\n",
+        "  if is_training:\n",
+        "    ds = mnist_train('/tmp/autograph_mnist_data')\n",
+        "    ds = ds.shuffle(batch_size * 10)\n",
+        "  else:\n",
+        "    ds = mnist_test('/tmp/autograph_mnist_data')\n",
+        "  ds = ds.repeat()\n",
+        "  ds = ds.batch(batch_size)\n",
+        "  return ds\n",
+        "\n",
+        "def get_next_batch(ds):\n",
+        "  itr = ds.make_one_shot_iterator()\n",
+        "  image, label = itr.get_next()\n",
+        "  x = tf.to_float(tf.reshape(image, (-1, 28 * 28)))\n",
+        "  y = tf.one_hot(tf.squeeze(label), 10)\n",
+        "  return x, y"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "2zEJH5XNjgFz",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "This function specifies the main training loop. We instantiate the model (using the code above), instantiate an optimizer (here we'll use SGD with momentum, nothing too fancy), and we'll instantiate some lists to keep track of training and test loss and accuracy over time.\n",
+        "\n",
+        "In the loop inside this function, we'll grab a batch of data, apply an update to the weights of our model to improve its performance, and then record its current training loss and accuracy. Every so often, we'll log some information about training as well."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "UUI0566FjZPx",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def train(train_ds, test_ds, hp):\n",
+        "  m = mlp_model((28 * 28,))\n",
+        "  opt = tf.train.MomentumOptimizer(hp.learning_rate, 0.9)\n",
+        "  train_losses = []\n",
+        "  train_losses = autograph.utils.set_element_type(train_losses, tf.float32)\n",
+        "  test_losses = []\n",
+        "  test_losses = autograph.utils.set_element_type(test_losses, tf.float32)\n",
+        "  train_accuracies = []\n",
+        "  train_accuracies = autograph.utils.set_element_type(train_accuracies,\n",
+        "                                                      tf.float32)\n",
+        "  test_accuracies = []\n",
+        "  test_accuracies = autograph.utils.set_element_type(test_accuracies,\n",
+        "                                                     tf.float32)\n",
+        "  i = tf.constant(0)\n",
+        "  while i < hp.max_steps:\n",
+        "    train_x, train_y = get_next_batch(train_ds)\n",
+        "    test_x, test_y = get_next_batch(test_ds)\n",
+        "    step_train_loss, step_train_accuracy = fit(m, train_x, train_y, opt)\n",
+        "    step_test_loss, step_test_accuracy = predict(m, test_x, test_y)\n",
+        "    if i % (hp.max_steps // 10) == 0:\n",
+        "      print('Step', i, 'train loss:', step_train_loss, 'test loss:',\n",
+        "            step_test_loss, 'train accuracy:', step_train_accuracy,\n",
+        "            'test accuracy:', step_test_accuracy)\n",
+        "    train_losses.append(step_train_loss)\n",
+        "    test_losses.append(step_test_loss)\n",
+        "    train_accuracies.append(step_train_accuracy)\n",
+        "    test_accuracies.append(step_test_accuracy)\n",
+        "    i += 1\n",
+        "  return (train_losses.stack(), test_losses.stack(),  train_accuracies.stack(),\n",
+        "          test_accuracies.stack())"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "cYiUQ1ppkHzk",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Everything is ready to go, let's train the model and plot its performance!"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "K1m8TwOKjdNd",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 13
+            },
+            {
+              "item_id": 14
+            },
+            {
+              "item_id": 15
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 988
+        },
+        "outputId": "f9d3eef3-5bea-45c1-ddf9-4edee73e4436",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345800262,
+          "user_tz": 240,
+          "elapsed": 52391,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "with tf.Graph().as_default():\n",
+        "  hp = tf.contrib.training.HParams(\n",
+        "      learning_rate=0.05,\n",
+        "      max_steps=500,\n",
+        "  )\n",
+        "  train_ds = setup_mnist_data(True, hp, 50)\n",
+        "  test_ds = setup_mnist_data(False, hp, 1000)\n",
+        "  tf_train = autograph.to_graph(train)\n",
+        "  (train_losses, test_losses, train_accuracies,\n",
+        "   test_accuracies) = tf_train(train_ds, test_ds, hp)\n",
+        "\n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(tf.global_variables_initializer())\n",
+        "    (train_losses, test_losses, train_accuracies,\n",
+        "     test_accuracies) = sess.run([train_losses, test_losses, train_accuracies,\n",
+        "                                  test_accuracies])\n",
+        "    plt.title('MNIST train/test losses')\n",
+        "    plt.plot(train_losses, label='train loss')\n",
+        "    plt.plot(test_losses, label='test loss')\n",
+        "    plt.legend()\n",
+        "    plt.xlabel('Training step')\n",
+        "    plt.ylabel('Loss')\n",
+        "    plt.show()\n",
+        "    plt.title('MNIST train/test accuracies')\n",
+        "    plt.plot(train_accuracies, label='train accuracy')\n",
+        "    plt.plot(test_accuracies, label='test accuracy')\n",
+        "    plt.legend(loc='lower right')\n",
+        "    plt.xlabel('Training step')\n",
+        "    plt.ylabel('Accuracy')\n",
+        "    plt.show()"
+      ],
+      "execution_count": 17,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/train-images-idx3-ubyte.gz\n",
+            "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/train-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/train-labels-idx1-ubyte.gz\n",
+            "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-images-idx3-ubyte.gz to /tmp/autograph_mnist_data/t10k-images-idx3-ubyte.gz\n",
+            "Downloading https://storage.googleapis.com/cvdf-datasets/mnist/t10k-labels-idx1-ubyte.gz to /tmp/autograph_mnist_data/t10k-labels-idx1-ubyte.gz\n",
+            "Step 0 train loss: 2.244329 test loss: 2.2499208 train accuracy: 0.12 test accuracy: 0.161\n",
+            "Step 50 train loss: 0.64771986 test loss: 0.56013924 train accuracy: 0.82 test accuracy: 0.836\n",
+            "Step 100 train loss: 0.49011207 test loss: 0.42143965 train accuracy: 0.84 test accuracy: 0.879\n",
+            "Step 150 train loss: 0.3768609 test loss: 0.39319593 train accuracy: 0.88 test accuracy: 0.883\n",
+            "Step 200 train loss: 0.36007702 test loss: 0.37089333 train accuracy: 0.9 test accuracy: 0.881\n",
+            "Step 250 train loss: 0.182115 test loss: 0.28543878 train accuracy: 0.94 test accuracy: 0.915\n",
+            "Step 300 train loss: 0.2119576 test loss: 0.22305593 train accuracy: 0.92 test accuracy: 0.93\n",
+            "Step 350 train loss: 0.12932214 test loss: 0.29057172 train accuracy: 0.96 test accuracy: 0.906\n",
+            "Step 400 train loss: 0.22937602 test loss: 0.2200287 train accuracy: 0.92 test accuracy: 0.925\n",
+            "Step 450 train loss: 0.23444137 test loss: 0.19857481 train accuracy: 0.94 test accuracy: 0.94\n"
+          ],
+          "name": "stdout"
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzs3XmAFNW9Pvynlt5mYdhmQMHggnGN\nS9zCD0ElKug1edUY9ZoQTYze3GuiRk1uYjRqRHNj4n5NrhKjiUYlbihGQFRUFDSoKIvgICAO6+xL\n711V5/2jlq7qZaZnpnumZ3g+/zjTXV1dXSP91PecU+dIQggBIiIiGjLkwT4AIiIi6h2GNxER0RDD\n8CYiIhpiGN5ERERDDMObiIhoiGF4ExERDTEMb6JeOOigg3DllVdmPf6rX/0KBx10kGe766+/3rPN\ne++9h9mzZwMAtm3bhkMPPdR57osvvsCPfvQjzJw5EzNnzsTZZ5+NV199FQBw0003YdasWZg1axYO\nO+wwnHLKKc7v4XDY8x7JZBLz58/v9edavXo1Lr300oK2XbBgAebMmdPn97J19/rZs2fjhRde6PO+\niYY7hjdRL3366aee0Ewmk1izZk3WditXrsQnn3xS0D6vu+46TJs2DYsXL8bixYtxyy234LrrrsPO\nnTtxyy23YNGiRVi0aBHGjRuH3//+987vVVVVnv188sknfQrUI444Ag8//HBB2y5fvhxTpkzp83vZ\n+vt6oj0Zw5uol0444QQsWbLE+f3tt9/GV77ylaztrrnmGtx+++0F7bO+vh5HHnmk8/uRRx6JxYsX\nY/z48QUfV3NzM3784x/jo48+wkUXXQTAbAF48MEHMXPmTOi6jlWrVuHcc8/FrFmzcOaZZ2L58uUA\nzFaB0047DQBw//334ze/+Q2uuOIKfP3rX8d5552HxsZG533ee+89HHzwwVnv9cEHH+Bb3/oWTjvt\nNJx//vloaGgAAOzevRsXX3wxzjzzTJx66qm4++67cx5rPu+99x7OOecczJo1C9/+9redC6Vc++3u\ncSEE/vd//xczZ87EKaecgjlz5kDXdQDAwoULcdZZZ+GMM87AN77xDbz33nsFn3eiwcDwJuqlM844\nAy+99JLz+z//+U/MmjUr53ZCCCxatKjHfU6fPh1XXnkl/va3v2HTpk0AgHHjxkGSpIKPa+zYsbjm\nmmtw1FFH4YknnnAeF0Jg8eLFUBQFv/71r3HppZdi0aJFuPzyy3HTTTfl3NeiRYtw/fXX49VXX8WY\nMWPw7LPPAgA2bdqE2tpaTJgwwfNe4XAY//mf/4lrrrkGS5Yswfe+9z1cddVVAIBHH30Uxx13HF5+\n+WUsWLAADQ0NMAwj57FmikQiuOqqq3DDDTdg0aJF+OEPf4jrrrsOhmHk3G9jY2Pex1944QUsWrQI\nzzzzDJYsWYKGhgY8+eSTAIBbbrkFDz74IBYuXIibbroJr7/+esHnnWgwMLyJeun444/Hxo0b0dLS\nglgshlWrVmHKlCk5t73++uvxhz/8AYlEott9/v73v8d3vvMdLFiwAGeddRZmzJjhBEt/nXzyyc7P\n8+fPxxlnnAEAOOaYY5zqONOxxx6LCRMmQJIkHHLIIdi5cycAYMWKFTk/6wcffIBx48Zh6tSpAICz\nzjoLX3zxBXbs2IExY8bg7bffxvvvvw+/34+77roLdXV1BR376tWrMX78eBxzzDEAgJkzZ6KtrQ3b\nt2/Pu998jy9duhTf+ta3UF1dDVVV8e1vfxuvvPIKAGDMmDF46qmnsH37dhx77LH45S9/WdjJJRok\n6mAfANFQoygKTj/9dCxcuBCjR4/GiSeeCFXN/U/psMMOw3HHHYdHHnkERx99dN59BgIBXHrppbj0\n0kvR2dmJRYsW4fbbb8fEiRMxbdq0fh3vyJEjnZ8XLFiAv/3tb4hEIjAMA/mWNqiurnZ+VhTFaV5+\n5513cMkll2Rt39nZiYaGBk8LhN/vR2trKy655BIYhoFbbrkFjY2N+M53voOf/OQnBR17a2srRowY\nkXVsLS0tefeb7/Guri48/PDDmDdvHgBA13WMHj0aAPCnP/0Jf/rTn3Duuedir732wvXXX4/jjz++\noGMkGgwMb6I+OPPMM3H33Xdj1KhRPfbZ/vSnP8W5556LiRMn5ny+tbUV69evd6rWESNG4Pzzz8ey\nZctQX1/f7/C27d69GzfccAOefvppHHLIIfj8888xc+bMgl+vaRrWrFmT8yKkrq4O+++/P5577rmc\nr7388stx+eWXY8uWLbjsssucSronY8aMQXt7u/O7EAIdHR0YM2YMVFXNud+pU6fmfLyurg4zZszA\nd7/73az3+dKXvoTf/va3MAwD8+fPx7XXXotly5YVeGaIBh6bzYn64Oijj0ZjYyM2btzYY4VWV1eH\n73znO7j//vtzPh+Px3HllVd6wmLr1q34+OOPceyxx/bquFRVRTgczllRt7a2oqKiAvvvvz80TXMq\n0EgkUtC+V69ejYMOOgh+vz/rvY488kg0NTXh448/BgA0NDTgZz/7GYQQ+PWvf4133nkHgBmSY8eO\nhSRJ3R6r7YgjjkBzczNWrVoFwBxfMH78eEycODHvfvM9/vWvfx0vvPACYrEYAOCpp57C888/j9bW\nVnz/+99HOByGLMs48sgjezXWgGgwsPIm6gNJknDaaachFotBlnu+Bv7BD36Ap59+Oudze++9N/70\npz/hvvvuw5w5cyCEQFVVFX75y196RqAX4phjjsEf/vAHTJs2DW+++abnuYMPPhjTp0/HzJkzMWbM\nGPziF7/Ahx9+iNmzZ+O///u/e9y3fYtYvve67777cOuttyISicDn8+Gqq66CJEm48MIL8etf/xq3\n3norhBCYMWMGpkyZgh07dnheryhK1ntWVFTgnnvuwa233opoNIrRo0fjrrvu6na/I0eOzPk4AGzc\nuBHnnHMOADPYb7vtNowePRrTpk3Dt771LSiKAp/Ph9tuu61X551ooElcz5uIiGhoYbM5ERHREMPw\nJiIiGmIY3kREREMMw5uIiGiIYXgTERENMUPmVrGmpq6i7m/UqAq0tUWLus89Ec9j//Ec9h/PYXHw\nPPZfsc9hbW11zsf32MpbVbPvKaXe43nsP57D/uM5LA6ex/4bqHO4x4Y3ERHRUMXwJiIiGmIY3kRE\nREMMw5uIiGiIYXgTERENMQxvIiKiIYbhTURENMQwvImIaNh6443XCt723nvvxI4d23vc7sMP38cN\nN/y8P4fVbwxvIiIalnbu3IFXX11c8PZXXXUt9t57QgmPqHiGzPSoREREvXHXXb/D+vXr8Mgjc2EY\nBnbs2I6dO3fgnnv+iN/+9jdoampELBbDD35wOaZOnYYf//hyXHPNz7F06WuIRML44out2L59G668\n8lpMmTI153u89toSzJv3dyiKgoMOOgS33XYL6us34M47fwefzwe/349bbvktdu7cnvVYdXXuqU8L\nsceGd0c4gfc3NOLYg+sG+1CIiIa9f7z+GVZuaCzqPo87uA7nz5ic9/l///fZeO65f+D7378MDz/8\nIDQthT/+8c9oa2vF8cd/DWeccRa2b9+GG2/8BaZOneZ5bWPjbvzhD/fh3XeX44UXns0Z3tFoFA89\n9AAeeeQJVFRU4Oc//yneffddvPzyyzjnnPMwa9a/4YMPVqK1tQUvv7wg6zGGdx9ceecbaO2M46ZL\njsOk8X0/gURENDQccshhAIDq6hFYv34dXnzxOUiSjM7OjqxtjzjiKABAXV0dwuFwzv01NHyBiRO/\nhIqKCgDA0Ucfg/Xr1+PEE0/CH/7wP2ho+AJf//ppmDRp35yP9cceGd5b23YiPOFNSMnD0dwRZ3gT\nEZXY+TMmd1slDwSfzwcAWLJkETo7O/HAA39GZ2cnfvjD2VnbKkp6gREhRM79SZL3OU1LQZJCOPbY\n4/HnP/8Ny5cvw5w5N+PHP74652Nf/eqxff4se2R4f7ztCyjVbTBG70RLZ3ywD4eIiEpAlmXoup71\neHt7O/baa2/Isow333wdqVSqT/vfZ59J2LbtC0SjEVRUVGLVqg9x1VU/xrPPzsOUKSfi9NPPgBAC\n9fUbsGXLpqzHGN69dPykA7G4CZArO9DSwfAmIhqOJk3aD59+ugH33XcnKiurnMdPPnkGfvGLa/DJ\nJ2vxb//2TdTV1eGRR+b2ev+hUAhXXHEVrr32J5AkGUcccRSOPfZY7NzZghtv/AWqqqrg8/lw/fU3\nob7+06zH+kMS+doDykxTU1dR93fjit+ipTOCQyLn4yfnHlHUfe9Jamuri/632dPwHPYfz2Fx8Dz2\nX7HPYW1t7m7dPfY+7y+P2Q+SL4mmcOtgHwoREVGv7LHhPbFmPACgLdk2yEdCRETUO3tseI8JjQIA\nxBFGPKkN8tEQEREVbs8N74rRAADJH+egNSIiGlL22PAeW2FW3pI/xtvFiIhoSNljw3uME96svImI\naGjZY8M75AvCLwcg+eNoZuVNRDQs9WZJUNtHH32ItjbvnUjlsAyo2x4b3gAwMlDDypuIaJjq7ZKg\ntn/+88Ws8C43e+QMa7a6ijFojDWiqSt7UnoiIhra3EuCXnDBRbj99lvQ1dUFXddx9dU/w+TJB+Lx\nxx/Fm28uhSzLmDp1Gg455FAsW/YGtmzZjDlz7sD48eOz9pu5DOjVV1/nLANaWRkCIJdkGVC3PTy8\nxwItQKfWPtiHQkQ0rD332UtY1bimqPs8uu4rOHfyWXmfdy8J+uijf8YJJ/w/fOMbZ2PLls24994/\n4J57/oinnnoc8+cvgqIomD//WRx33NcwefKXcc01P88Z3LmWAf3ww/fx1ltLcc4552H27AuxaNHr\nJVkG1G2PDu/a0FgAQAysvImIhrM1a1ajvb0Nixe/DABIJMzu0pNP/jquvvq/cNpps3D66bN63E+u\nZUDr6zc4S362tOzClCknlWQZULc9OrzrKszwTildMISALEmDfERERMPTuZPP6rZKLjWfT8VPf/oz\nHH64dy2L6677JbZu/Ryvv74EP/nJf+Chh/7a7X5yLQMaCAScJT/XrFlZsmVA3fboAWt25Y1gFNE4\nZ1kjIhpO3EuCHnro4XjrrTcAAFu2bMZTTz2OcDiMRx6Zi0mT9sX3v38ZqqtrEI1G8i4lCniXAQWA\nVas+xEEHHYpnn52Hzs4OfPOb38QFF1yE+voNzmOnn36G81ix7NGV96hgDSQhQw5EEYmnUBXyDfYh\nERFRkbiXBP3hD3+E2267Gf/1Xz+EYRi4+urrUFVVhfb2Nlx22fcQClXg8MOPwIgRNTjqqK/ihhv+\nG7/97Z3Yf/8DPPvMtQzokUcehVgsihtv/AVGjaoBIJdkGVC3PXZJUHvZtp++fgvicQM/O+pa7L/3\niKK+x56ASwj2H89h//EcFgfPY/9xSdABEpCCkNQUIvHUYB8KERFRQfb48A4qIUiqhs4oJ2ohIqKh\nYY8P7wo1BABoj4YH+UiIiIgKs8eHd6XPvFevIxEZ5CMhIiIqzB4f3iMClQCAjjjDm4iIhoY9PrxH\nVZgj+Xa1tw3ykRARERVmjw/v0RXm7WE7OjrQHk4M8tEQERH1bI8P70qfOWBNUpNYvallkI+GiIio\nZwxvn9nnDSWFpvbY4B4MERFRAUo6Peodd9yBDz74AJqm4T/+4z9w+umnO88tX74cd911FxRFwfTp\n03HFFVeU8lDysm8Vk9QUWjvZbE5EROWvZOH97rvvYuPGjZg3bx7a2tpwzjnneMJ7zpw5ePjhhzFu\n3Dh897vfxcyZMzF58uRSHU5eITVo/qBoaOviRC1ERFT+Shbexx13HI44wlx6bcSIEYjFYtB1HYqi\noKGhATU1Ndhrr70AACeddBJWrFgxKOHtV/wAAJ9foK2NlTcREZW/koW3oijOYuXPPPMMpk+fDkVR\nAABNTU0YPXq0s+3o0aPR0NDQ7f5GjaqAqipFPcba2mqM1M3K2+8XaI8kMXZsFSSu690r+SbOp8Lx\nHPYfz2Fx8Dz230Ccw5IvCfrqq6/imWeewV/+8pd+7aetLVqkIzLZK78IISBLMiTFQCKpY+u2NlQG\nuTRoobgKUf/xHPYfz2Fx8Dz237BYVWzZsmX4v//7P8ydOxfV1ekDqKurQ3Nzs/P77t27UVdXV8pD\nyUuSJPhlP2TVXHi9jYPWiIiozJUsvLu6unDHHXfgwQcfxMiRIz3PTZw4EeFwGNu2bYOmaVi6dCmm\nTp1aqkPpkV/xAbIZ3h2R5KAdBxERUSFK1mz+8ssvo62tDVdffbXz2AknnICDDjoIp512Gm6++WZc\ne+21AIAzzzwT++23X6kOpUd+xY9kyhxpHo5xXW8iIipvJQvvCy64ABdccEHe54877jjMmzevVG/f\nKwHFjw6YS4IyvImIqNzt8TOsAYBf9kMXZmhHGN5ERFTmGN4w+7wNGIBksPImIqKyx/BGeqIWyDrC\ncYY3ERGVN4Y3zD5vAGZ4s/ImIqIyx/CG2ecNAKrPYJ83ERGVPYY3rPu8AYRCEitvIiIqewxvpPu8\nQyEgHNMG+WiIiIi6x/BGus87GABiCQ26YQzyEREREeXH8Ea68g5YS3tH46y+iYiofDG8Afhls89b\nVc2KO57UB/NwiIiIusXwRrryllUBwGw6JyIiKlcMbwABJQAAzrKgrLyJiKicMbwBhFQzvCXVrLjj\nSVbeRERUvhjeAIKKNVJNNu/xjiVYeRMRUflieAMIWpW3IZkVd4yVNxERlTGGN4CQGgIAGJJZecdZ\neRMRURljeAMIWgPWdCQBsM+biIjKG8MbgCqrUCQFmhXe7PMmIqJyxvAGIEkSgmoAKWGFNytvIiIq\nYwxvS1AJImkkAABxTtJCRERljOFtCaoBJHQrvDlJCxERlTGGtyWkBpHQk1BkNpsTEVF5Y3hbgkoQ\nAgKBoOCtYkREVNYY3hZ7opZgCIiyz5uIiMoYw9sSVM0pUitCQCSWGuSjISIiyo/hbQlZ85sHQwJJ\nzUAixaZzIiIqTwxvi115B4IGAFbfRERUvhjeFrvP2+c3wzvM8CYiojLF8LbYzeYqw5uIiMocw9ti\nV96yz+zrZngTEVG5YnhbglblLavmbWIMbyIiKlcMb4tdeUNheBMRUXljeFtC1mhzQzJDm+FNRETl\niuFtCTK8iYhoiGB4W+w+b3tNb85vTkRE5YrhbfHJKmRJdtb0TunGIB8RERFRbgxviyRJCClBZ01v\nneFNRERliuHtElQDiGlxKLLEypuIiMoWw9slqAYR1xJQFRmaJgb7cIiIiHJieLsErWZzRQE0Vt5E\nRFSmGN4uITUAAQHVLxjeRERUthjeLj7Fb/5XNRjeRERUthjeLn7ZBwCQVYGUzj5vIiIqTwxvF5+s\nAgAU1YCm9b/ybutK4MEX16G5I9bvfREREdkY3i4+xay8FaU4fd5PvFqP9z7Zjb8u3NDvfREREdkY\n3i4+u9ncZ0ArQrN5PKl7/ktERFQMDG8Xu89bUQwYQsAw2O9NRETlh+HtYjebS4rZZM5Z1oiIqBwx\nvF2c0eayGdq8XYyIiMoRw9vF7vO2K+9i9HsTEREVG8PbxWk2tyvvItwuRkREVGwlDe/6+nqceuqp\nePzxx7OemzFjBi666CLMnj0bs2fPxu7du0t5KAWxK2/I5ujwfjebC1buRERUfGqpdhyNRnHrrbdi\nypQpebeZO3cuKisrS3UIvebPCG8OWCMionJUssrb7/dj7ty5qKurK9VbFF1Ws3mxwlsqzm6IiIiA\nElbeqqpCVbvf/U033YTt27fjmGOOwbXXXgtJGtyUs6dHFZLdbM5mbyIiKj8lC++eXHnllZg2bRpq\nampwxRVXYPHixZg1a1be7UeNqoCqKkU9htraas/vcf9IAIBqLi6Gqqpg1ja94fObp9enKv3aT7kb\nzp9toPAc9h/PYXHwPPbfQJzDQQvvs88+2/l5+vTpqK+v7za829qiRX3/2tpqNDV1eR4Lx1IAgJSW\nBAA0t4TRVBPo83ukkpq1Pz3rvYaLXOeReofnsP94DouD57H/in0O810IDMqtYl1dXbj00kuRTJoh\nuXLlShx44IGDcSge9mhzQ+KANSIiKl8lq7zXrl2L3/3ud9i+fTtUVcXixYsxY8YMTJw4Eaeddhqm\nT5+OCy64AIFAAIceemi3VfdA8St2n7dZMevs8yYiojJUsvA+/PDD8dhjj+V9/uKLL8bFF19cqrfv\nE6fyBitvIiIqX5xhzUWRFEiQYMCsvDnDGhERlSOGt4skSfApPqfy7uk+7x3hXXjsk38grsUH4vCI\niIgADOJo83Lll33QhTVKvIc+7/s+eghdyTDGVdTi9H1PGYjDIyIiYuWdKagEkDQSAAC9m8p7W2MY\nXckwACBpJAfk2IiIiACGd5bairGIGREEv/oqtic3593ulfcbnJ8lzn9KREQDiOGdYXyFORe7pGpY\nrb2af0N3i/ogT+tKRER7FoZ3hnGV6YVUVPjzbifAe8CJiGhwMLwzjK+oTf8iCquoZTabExHRAGJ4\nZxhfOc75OYEINEPLvaGn8GZ4ExHRwGF4Z6j2V+EHX/4h9I4xgCTQGm/r8TXs8iYiooHE8M5h/5pJ\nMLpGAQCaYq05t/GMV8tTebNXnIiISoHhnYOqSBApc7BaLJV7KVLhSmbeKkZERAOJ4Z2DqsiAYU4+\nl8g7AYsnvYmIiAYMwzsHVZEhDAUAkNBzh3chzeZERESlwPDOQVUkQDfDO5knvHuD4U5ERMXE8M5B\nkiQo1pot21o6cm8kvNsTERENFIZ3Hgp8AICV9TuwsyWS9TxHkhMR0WBheOdhhzdkHZ2R7pvO2SxO\nREQDieGdhyqlwzsX4bpXzBD5lw4lIiIqNoZ3HnZ4S0qe6VFd3EFORERUagzvPHyKz5yIRdaR1Lqv\nrA2w8iYiooHD8M5DlRXAUCApOpKp7KZzd7HNZnMiIhpIDO88fKp1r7esI5nqofJmszkREQ0ghnce\n5ixrKiRFQ0LLUXm7f2blTUREA4jhnYeqyAVX3nqe8GZBTkREpcDwzkOWJXN+c1lHIpljxLn7VjEO\nWCMiogHE8M7DMIQ5YE0WSGiprOe9zeY9lNicw4WIiIqI4Z2HYQhAN+c3j2mJ7rdlnzcREQ0ghnce\nuiGcZUFjqXj2Bp5bxdi5TUREA6eg8F67di2WLl0KALj77rtx8cUX4/333y/pgQ023RAQyQAAIKKH\nu92Wfd5ERDSQCgrvOXPmYL/99sP777+PNWvW4MYbb8R9991X6mMbVIYhIBIhAEBMdGU9z1vFiIho\nsBQU3oFAAPvuuy9ee+01nH/++Zg8eTJkeXi3uJuVdzfh7VmYhM3mREQ0cApK4FgshoULF+LVV1/F\niSeeiPb2dnR2dpb62AaVIQREMggASCLXet7pwK7f1pZzxDkXLCEiolIoKLyvueYaLFiwAD/96U9R\nVVWFxx57DJdcckmJD21w6a5m86Scq8873VS+uy2CpvZY9hZ2djPDiYioiNRCNvra176Gww8/HFVV\nVWhubsaUKVPw1a9+tdTHNqgMwwAMFUJToSvRrOfdlTckkQ5q9zZW5c0KnIiIiqmgyvvWW2/FwoUL\n0d7ejgsvvBCPP/44br755hIf2uD60rhqAIBIhKCrkawAzry3O3ezub1taY6RiIj2TAWF9yeffIJv\nf/vbWLhwIc455xzcc8892Lp1a6mPbVBdcsbB+N7Mg+DTqwFZR0cyo49fSieyxMqbiIgGUEHhbYfP\nG2+8gRkzZgAAkslk6Y6qDFQGfTj56AkIiBEAgMZok+d54b63WxI5VyGxA53ZTURExVRQeO+33344\n88wzEYlEcMghh2D+/Pmoqakp9bGVhZAwP+fOsDe8vROziJwBzcqbiIhKoaABa3PmzEF9fT0OOOAA\nAMDkyZNxxx13lPTAykW1MgotALZ37fY8nll557rXO+lrARSJfd5ERFRUBYV3PB7H66+/jnvvvReS\nJOGoo47C5MmTS31sZWGkbzSAXM3mmaPNvQm9qf1ztO31OvwVYyBaTy71YRIR0R6koGbzG2+8EeFw\nGBdeeCHOP/98NDc344Ybbij1sZWFmmAVhACi1uIkH3zaiBfe3gJkNJvruje869s2AQCUmhb2eRMR\nUVEVVHk3Nzfjrrvucn4/5ZRTMHv27JIdVDmpCKpAVIJm6ACAB55fCwA4cLLrukcS6EqGcdt7D+Oc\nyf+GQ8cchNZ4KwBApHzs8yYioqIqeHrUWCw9g1g0GkUi0f0a18NFZVAFhAxN1z2Pp9y/S8DqjlXY\nEdmFBz5+GADQEm8DAIhkiH3eRERUVAVV3hdccAHOOOMMHH744QCAdevW4aqrrirpgZWLiqAPEBI0\n4Q3vpK65fhMQGQndaoe3prLyJiKioioovM877zxMnToV69atgyRJuPHGG/HYY4+V+tjKgll5S9AN\n74xqKS0d3lLGgDUhhFN5QzYY3kREVFQFhTcA7LXXXthrr72c31evXl2SAyo3duWti+6azYUnoBN6\n0pk+VZJ1DlgjIqKi6vOi3HtKNVkZVCGEbC5U4pIy3GEunAFtABDX4+mnFH2POVdERDQw+hzekiQV\n8zjKVkVQBSBBhze8tYzKO2GkB/DFtHR4S7LOAWtERFRU3Tabn3TSSTlDWgiBtra2kh1UOamw+ryF\n8PZdp3QNAdd2yTzhzT5vIiIqtm7D+4knnhio4yhbiixDEjIMpKC5J2KR3TOsGXkrb7DPm4iIiqzb\n8J4wYcJAHUdZkyUJAgZSWrqpXJJdt4pJQMod3qmoazsDBpjeRERUPH3u8y5EfX09Tj31VDz++ONZ\nzy1fvhznnXceLrjgAjzwwAOlPIx+kyADEEhprn5v1R3eAkmRDu+2RIfn9ULSQEREVCwlC+9oNIpb\nb70VU6ZMyfn8nDlzcP/99+PJJ5/EO++8g88++6xUh9JviiRDSAaSrvD2VN4QSBnp9c2d8DbM0ysk\n721mRERE/VGy8Pb7/Zg7dy7q6uqynmtoaEBNTQ322msvyLKMk046CStWrCjVofSbLCkABOJJVwgr\n3klaUsIV3vF28wctCAAQYOVNRETFU7LwVlUVwWAw53NNTU0YPXq08/vo0aPR1NSUc9tyoMgyJFmg\nI5JuGpdUb+Wtwd1sboV3yhwf+2w3AAAgAElEQVSPzsqbiIiKqeAZ1gbbqFEVUFWlqPusra0uaDtV\nMU+TUFzXOlblLTQVkj/puQu8PWk1m1vhDVkv+L2GouH82QYKz2H/8RwWB89j/w3EORyU8K6rq0Nz\nc7Pz++7du3M2r7u1tUW7fb63amur0dTUVdC2spABCdi2M31vu2SHt+5zqvB9qvZGQ3gHuhJh87lU\nABIAQ9IKfq+hpjfnkXLjOew/nsPi4Hnsv2Kfw3wXAiUdbZ7PxIkTEQ6HsW3bNmiahqVLl2Lq1KmD\ncSgFUWTzNHVE0/3akDXz/m3NvP6pwlhMm+AdnCecZnP2eRMRUfGUrPJeu3Ytfve732H79u1QVRWL\nFy/GjBkzMHHiRJx22mm4+eabce211wIAzjzzTOy3336lOpR+UxUF0IHOqGvaU1UDdBWQzHu4fQhA\nldOn0y/7ENet5nb2eRMRURGVLLwPP/zwbpcNPe644zBv3rxSvX1RqbIV3rH0oDQoGoSuOn3fighA\nkdN98j7Fh6iumE0bDG8iIiqiQWk2H2pUK5S7oq7R5opZeUtOePuhSunwViU1fZ+3zGZzIiIqHoZ3\nAXyKGcrhuN3nLZzK2x6spghvs7kqqxCaFeYyK28iIioehncB/NatYl0xK7xlA5IkzD5v2A/5PM3m\nqqxA6NbvisaVxYiIqGgY3gXwWfeX68KqoJ3bxNzh7Tebyi2qpMIwrN9lnUuTEBFR0TC8C1Dh91k/\nmRHszGvuCm9J+Jy+cQBmFW5V3pJVeXdGk7j/2dXY1hgekOMmIqLhieFdAL/PCm/JmkdNsSpwwzXj\nm65mNJur6cpcMdf0/ufyrVi1sRn3Pbt6AI6aiIiGK4Z3ARTJOk2SgCSlK293szkMNavZ3A53STYr\nb3s98GSKA9iIiKjvGN4FUOxbwCSByqAPvoDVg+2qvCXd22yuuprNoegw2OlNRERFMmQWJhlMslV5\nS5JAZciHsGrAACB0BYmNR0EZ2QRZqvbcKqZIKgAZQpedypuIiKgYWHkXIN1sbuDEr4yHL2D1fRsq\njLbxSG35CoRhB7b9GsXZxu7zdkjSwBw4ERENSwzvAshWEP/7qZNx5tcmweczk9i5jxuArouMZnPV\n2UbKvM+bVTgREfUDw7sAduU9fmwIkiRB8dmVtyu8hchoNndV3jL7vImIqHgY3gWQrSVBDWGGtqxa\no8Vdo811XXjmNreb0IWuAIoGwzDSO2SzORER9QPDuwB2Fa1b4S1Z93m7m80NQzgD2wCkg1xXIUlA\nyuDiJEREVBwcbV4AO5S/6NyGz9o3A0rKfMJwVd6GAclVUctOs7n537iWXguceuetj3dgQm0lDti7\nZrAPhYioLDC8C2D3eS/e+joAQLZWGfMMWDPSk7CYr/Fuc+fqe3AELhqQ4x1OYgkNjy7cAAD4yy9m\nDPLREBGVBzabF0B29WUDgJCyJ2nRDYHHXql3frfDW1LNKj2hJyBghntnJIkHnlsDg6POe6TpRs8b\nERHtYRjeBVAk72kSMMy7vQxvn/fGhnbXa8zntN2T0tsgXZl/UN+Enc2REh0xERENZwzvAmSGNwAr\nuNN93PGk7unztkebG51jobWMN3+Gd05znfeP9YhniIgoG8O7ALKsZD+oe4cLRGIpz+8KXK+xKnQD\n3hHnDO+esWeBiCgbw7sAco7KWxgZ/eAwB1c5r5HdK45Z94lL3srbYHj3iOdoePvX+t247I6l2N0a\nHexDIRpSGN4FyN9s7hV2Vd+K69TaQS/YbN5rDO/h7c8vfQLdEFi2eudgHwrRkMLwLkDmaHMATjXt\n5g7j5vZk+glhbtssbYLkT1cYKY6k7hFH5A9v/PMS9Q3DuwC5Ku9RVaFuX7P4vW3pX6yg362uQ/Co\nt5yHUymGd08Y3kRE2RjeBcjV5z1+VBWqQj4AQCjQw1w3OZrYAVbehWCzORFRNoZ3AZQczeaqrDrL\nfFZX+Lp9vcjRxA4AyZSe83FKY3jvGbhWD1HvMLwLIOf4ZnEv/1kdyg7vQyeNxrUXHoWvHTouu/KW\nNQACb3e8jHe2vwcAWPDOFsxd8ElRj3s4YHYTEWVjeBcgksq+jcW9/Gd1hT/r+ZHVfhy272jzOeE9\nzZI/DskfxxfJT/HEp88CAJ5ftgUr1u0q8pEPnLWbW7BibfGPn5U3EVE2hncBJo3YBwDw5VGTncfM\nZnPz5xGV6fAWmlmFj6kYCQCQ5exmcykQg+RPrzJmrxMOwGmKz2f+ss34+LPmPnyK0rrrHx9j7kvF\nbznggDUiomwM7wJU+6vwwIw7cOa+pzqPqa5Z1/yq7Axei6+ZisTGo3DUhP0BABUBNavZXArEIAVi\nzu+NkXQYdxdWndEkXnznc9z7zOr+faAS6unio7fKObxffGeLs+IZEdFAYnj3guIKbFVWPfNu71NX\nBQCoCYzAdbPOwKTx1QCAiqAv655wSUlB8qfD+4GP/+KsEa7p3YR3JJn3uXKR1Io7gr6cm83nL9uC\ntz7eMdiHMaSV8bUZUVljePeCu59blVQ4y2ZIwMGTRgEAamtCzs8AUBlSs/q8IQlP5d2aaIW692YA\ngN7N7WPhaCrvc+Wi2CPoyzm8iYgGC8O7F7Iqbye7JZxxwpfwzan74rJvHOp5TWXQlzUPOmTDCe+v\n7zPd3F9tAyBrWZV3Uk9i4ZZX0Z7oQGe0/CvvRLHD23U6/ufvH6KxPZZ/40HCC4y+4y1iRH3D8O4F\n9/3e7j5vSQJURcbZ0/ZH7UjvzGuVOZrNIRmQ/HEoIoBzDzwLB4a+AknVIPnj0DIq73/Uv4CXtryC\nFzctQke4/MM7WeRZ49x93vUN7Zj32sai7r8YONlO37HZnKhvGN69oHbT551PZUjN7vOWDEi+JFQj\nCAAwdOt5SUBzVXHtiQ6s2LnS+b0jo897xY6VWLBpUS8/RWkVu/IWGVVtOS7mknnBRURUagzvXvBU\n3pKCQtK7MuiDENnN5lBSkI0AAEDT0o+7+7zvWzU3/RJJdgashQLm/h7f8DQWbX0dulE+M7UVu887\nM6zLsYk6VeRBekREPWF490L2aHMzSLrrtzNvFcucpCUBSQIk3QzvlDUOTZIM6Fafd2ckieZYi/Oa\nqBZzKu+qjBndolr59AMnSthsnuv3cqAxvPuNfd9EvcPw7gXPaHO5h8VILLIsZd/n7TMnaJE0c3IX\n3S5WJQOaYQbBtX98C7rQceDIAwAAsVQMHZEEAMCvKp77qbuS4d5/mCJyH0vxR5tn/l4e4a27Dox9\n3n0nCup8IqJMDO9eUFyBrcqq606xHsoG4X1e8pshbM/GJgzreUkgkdTxxqrt0GWzyq5UK+BX/Ihq\nMcQTZjDqhkBcT8/QFklF+vyZisHdtF30Pu/MyrtMwlvT0sfR3b35ROXglZUNWL+1bbAPg4qI4d0L\n7nW9PQPWemzyywhvnxnMwqq8Dd16Xjbw7Fub8bfFn0JSzI5wvxxEhRpCTIs5FZ4hBLqS6cAO55h7\nfSC5w9tdeacMDXd/+Ces2LEy18sKkt1s3uddFZW72uaANSpniZSOp17biN8/uWqwD4WKiOHdC1kD\n1iw9ZXfdqFDOx42kWXk74S0Z2N5kNoFLqtkRHrDCO6rFnYFRhiEQTrnDe5Arb1d4ufu8t3Y24LP2\nLXh8w9N933eZjjZ3BzYHrFE5K5fWKiouhncvSK5RNYprkpae3Pz94zC+60QkPj3G83gqYTbD233e\nkiTgXApY06X65QBCaghxLY6UZm5oCOFpKh/sZnMtT+WtGVquzXsl84unXAasuQepsc+7H6w/Z+bY\nBiLqHsO7j3yyAvf0qN0J+lVMCnwZRkcthKv/OxaVYRgCuqvytkmqGXw+KYAKXxACAklh9pUbRmaz\n+WBX3q4+by0d3rmWUu2tzLDOvO97sLgvWDjavP/K5aJsOOK5HZ4Y3n2UOT1qTxTFOtVGeluR8iMS\nT0HX0n3e6RdYlbcUQIVaYT1vPmYIIJxKjzAPJ7NDMvMfbCKl4911u5zqvZjczebJZPrnLtcxLnrv\niz7tO/N7Rx/gLyJNN7BkZQOice+88u7AZp93/7Fpt3TKpauJiovh3UfmwiSmQu5R9dnh7VqkROgq\nWjsTcPLUGm0OpPu8VRFASA1ab2pW45l93pnN5l/s7sIPf7cUb3603XnsuTc346EFn2D+si0FfT63\ndVta8doH2/I+7xlt7ro4CLtuYVv4Xu/fF8jRbD7AX0TPv7UZT762EX9fUu953N1Uzmbz/mN4l065\ntFZRcTG8+6jQ+7xtimIlvHuFMV3BLY+uRDhid3obTsVsh7cCPypUc8CbZFXjhiE8TdI7Irs8s6wt\nX7sLAPDU6585jzU0dgEANu3o7NVxA8Cd8z7C35fU521+y9fn7b7/3JD7tiJa1gxrA/w9tHFbBwCg\ntTPheZwD1orD/nOyabd0mN3DE8O7j1RZ6dWiCnblLQz7vxKc028FuiS5m83NKlsRfgTUgPVYesBa\nXDPv8z669itoT3Rgbct656V2FSP7EqhvMwPcp5qj4/vTbJ6vOvKMNk+6wtvVIiAUb/gV/J5Z93kP\nbFC2dZnHPbI64Hnc22wuEI2nsO7z1gE9tuGEAVM6bNUYnhjefaRIhU2PalNVO6itjQ1X5S7Sk7TY\n7CpbNgLpJnopfatYzArv0yedAgB4a9sK57VOv/A+a3DvqofwcdNa+K33T/ajSszXt5tvkhZ35d3X\n8M5s8hvoUcntYfO4R1T4PY+nXIP0NM3AnfM+xp1PfYT6hvYBPb6hzv6nM9AXZW5CCCz9cBt2tQ7u\nfAmlwlaN4Ynh3UeqrOLkoycAAA760qgCtvc2m8vCHd7Wn8E1YE3yJyAMCbLwwWc10UtyepKWmBaD\nLBQ89sIuHDhyf2xo24hdkd3m7uzAC5lBMn/Ty/D5zPdI9WPu8XwDX9yjzd39v+5BdYbct+VMM9/S\nEAKabmTNvFYq9mfOfD8tY5KWLTvN7oimMlxvvJw5zeaD2POweWcnHnulHr+a++7gHUQJsfIenhje\nfeSTFXzntC/jjh9NwWH7ju5xe6fytprNZeSqvN3hHYNIhqDrrv512Wo2N4CYFofQfdi8vRMnjDfv\nH/+0bZP5vN1vnjJHqTdGmyGpZgWZ7EezuZ5jGlAhhGeeb/e0oRHXKHhD6Wt4Zw9Yu/z3b2DO3z7o\n0/56w90FkDkoTcszYI0LbPRNb6vD9zc04sEX1xWlqozEzC6q4VqgsvIenhjefaTKKmRJwtiRuWdP\ny9o+Y7S5e7S63Q9uN5srqg7Jn4RIhKDpBnyKtYqY5K6844BuTtEaUioBAM+/vRHN7bF0FaOkB4nF\nVXOFMvfgqriW7hMvRGbl3RZvx/ee+ylWtb0PqGY4u0MtYbgCW+1bs3n2gDXzd7vSLaXWrvT88cmM\nFgv3eXT/LGekdyyhIZbo/2Q1A03TDWzd1TVg79fb6vCP89fivU92Y3cBTd2vf7itV5/FMATunPeR\n526NoYyV9/DE8O4j91SphVCt0eb2JC2q5FrW0x6wZjWLT5xo7lskQkhpRlazOWCGt6GZj8swt4+m\n4nhx+efpK20lHRqblXcANenp835iwzO4d9VD+KhpbUGfQc+oPjd1fI6ElsCyliUIHvkG4Is74W0I\nA5qhQYHVV9zHyjuzz3sgFwGx108Huq+8NU/l7Q3vK+5+C1fc/VaJjrB0HlrwCW55dOWA9eH3tTp0\n5k/Io7E9hsdfqcctj+afXz+ztaSxPYZ1W1rx10Wf9umYyg2ze3gqaXjffvvtuOCCC3DhhRdi9erV\nnudmzJiBiy66CLNnz8bs2bOxe/fuUh5K0Vz+le/hrP1metb2LoRdeUtWda1KKn71PWu61IxmcyVo\n9puKRAU03Ug3m9vN6rIBXegwUnZ4p5/fvKPTudIWcgp1FWMBAElEoY773FMl2iPUN1rN7T3pbrIH\nSTGgjGx0giypm8EXRJV1Avp2q1jSSHpaEIq95Gh3uqLp982cRU3zDFhzDTTsY7N5S6wVN7xzOza0\nbuzbDors/Q2NAFBQZVsMfa0OMy8oMxXy/0vmn2y4dX2w8h6eShbe//rXv7B161bMmzcPt912G267\n7basbebOnYvHHnsMjz32GMaNG1eqQymqI2sPxxn7fb3Xr3Oaza3wliDjgL1roMiS0w/ujDb3m1+Y\nduWdsjPErrytMBO6VZFb64VLio4dzRGzipEMQNYxOjAKFx30LfP5jACt9JnN7YVOr6plfAnYI95t\nysgmZxR2QrdmiDMqrffuW+W9UnseoWNegz20qbezRdW3fYZH1j2BVB/mWe+KuirvjLECqTxzm/e1\ngnz1izfRlmjH3DWPFbS9EAJPLKnHui2lvT2tKuTreaMi6Gu+9HSPfUE5LHX765DHPu/hqWThvWLF\nCpx66qkAgAMOOAAdHR0Ih8M9vGr4UuzR5s7tZVbftyJD2KPN7T5t1aq8k0GkdAML3ramFrUGrNnL\nhcIKbwjF83wkrjkBH1KDOGj0gZ7nbVU+c0BbOJk7vNvi7Xh03ZOQrIuJzCrHvtf8lJFnw4hXQK5u\ncyrUlNXfrYgghC47y6D2VhhmOMk1zX16/b2rHsL7uz/CxwV2Dbh1uirvzJDwDNJzN6FrfWz+tVpy\nNFHYRcb2pghe/WAb7pz3UZ/erzvukfWZF2yl4q4OOyNJrN3ckndbz/H11I3ShzJ6uGXdnlh5G0Lg\nd3//EP9c8flgH0rJ9G6asF5obm7GYYcd5vw+evRoNDU1oaqqynnspptuwvbt23HMMcfg2muvzeov\ndBs1qgKq2rum6p7U1lYXdX/dGdlsNT/azeaKitraavhUGYmUt887FJKBlFlZ+/wqWtpTwCjXJC5W\neAvdrIpGjrA+hxXOmiGchU1GVY/AXnXWrWzW/u3PXREIAl1AzIjmPBfLPnkbK3evQuAIGfH3T0f1\niJBnu+QXZrhVh6ogkgHIwSg0w0BtbTVi7eaAMkX2QST9gJrM+R7vbVuFUcEafHns/p7Ho/EUKoLp\nqk8Zux1GR61nm978/Xyh3v+9U64vPUOSPK/3B9LHJrv6XYMVfmc7dytBT+8dCJj/FDVDK+g4O+Lp\nC7Fi/3/c1pluUQm5Pk8pqT7FeZ9fPLQEja1R3Hftydhv75qsbSOx9EVVVXXQeV2u44y7rrnyfY6a\n1phnm5he+N9tKGgKpy+cC/k8w+Ezh6NJfNrQjk8b2nHJN78y4O8/IP9mSv4Olsz7ZK+88kpMmzYN\nNTU1uOKKK7B48WLMmjUr7+vb2orb91ZbW42mpoEbTdvVZX1BWAEqdKCpqQuyLOWYpMX6YhYyOrvi\nUCUFCddrneZva8BaW6s5ktsO/65Iup9Y0hR0tVnPWzO0NTZ2QpIkdMbMintXuAlf7GyCX/Z5+vI7\nwzFnv1IwjJaWCJpC5nsmkjpefOdTqOOARFQ4rQApI4mmpi7s6jAHOukpCdD8kIKRrPNtCAN3vvMQ\nAOCBGXc4j2/Y2oY7nlyF807e32yokAClphkpyfBML9ubv19LR5dn+22NYUgSMKG2yrPdZ9s7cO/T\nH+Pq849EY0u6RSIWT3le3+EKuIireb2tPepsF0+mq+jujvWtxmVY9Nkbzu/23ydTMqXj3U9247iD\n69DWnv73UKz/j1es24VdLVEctl/61sfWtuiA/DuJu85vo9XPvnFLC6p82Y2Dja576Ztawmiq9uf9\n99zSkm7ty/c5OjLOZVNzz68ZSlpb0/8f9/R5Bvp7sVQiroWEBvrzFPsc5rsQKFmzeV1dHZqb002d\njY2NqK1NV05nn302xowZA1VVMX36dNTX1+fazbDh3ELk6vMGkNHnbQ1YU+1FjmVougG/Yo3Ylg0E\nfIrTbG73ecPwNpvHk5qzTYUagk/2eZ5/7q3NeGfNTqfPOqEncd1bv8b1Cx/CZ9s7nGN2z58uBSOe\npuKuWNJpAZAMn3MsQtagG4bTbA5dgdB8kBQdCc3bdJ7Qc98+ttIaLLVw5RanA1JSNchVfR/5HE15\nJ0/59V/+hRsf/lfWds8s/QyRuIZnlm5yms1HVPg8zea6YeCL3el/nO4+b/e98O6R/d01Xc5bu8Dz\ne0TLfaG6YPnneHThBjz56saSNO3OXfAJFiz/HM0d6XM1UPO25+qXzddk7668e1qOtZAxEpnvM9xW\n4ervx+mIJPHBp03FOZgBMtz+hrmULLynTp2KxYsXAwDWrVuHuro6p8m8q6sLl156KZJJ88t85cqV\nOPDAA0t1KGUhff+vNe847D5vKV1NWuEtK1Z1bshIaQYCavo+74BPTt8CZjWb6xrM6t0K51hCd6rz\nkBqCIiuQhAzJev6fK7bi4X+uzxpwFg5twd3/SPehulcFU0Y1oiWRHhxlGMK5QJCFz6m8JUWDpgkk\nrNHmwlAgUubFR1vcezUaTaXf372wiv1FLqvmY0I3L07kEd5+UPMiQcsK5lw6k7mvhA1hYHc0/cVk\nT6aj6Qa6oklUhXwI+BVPiK1YuxtrXQPFtDyD19yz2el5phAzRPbjLbHcg9B2tZih/vmu0t7j3tKR\n/ruUMrzdrXG5Lm7yjST3hHcPo80LGayV+d7D7YvffQ76MjPh//z9Qzzw/JohNfVvrgmlhpuShfdX\nv/pVHHbYYbjwwgsxZ84c3HTTTXjuueewZMkSVFdXY/r06c5tZKNHj+62yXw4kK0Ba3a/tV15T6yt\nAmA1nctWVW6FNwwFKV0gqKbv8w74FSek7cldNN2AJBSn2Tye0JyAt5cTlaB41wuHQEJPoNJeKxyA\n0FTPVbq78lZrt+PvDQ8imdJR39BuTlpih7er8oaiIaUbSOr2iHgZ0Mzwrm/x3pIW1dKh61772/4y\ntS8OjC6zGVcOeQc8aprAw2sfw8+W3YRIKuoJccMQePHtLc5kOJ3J3IG3cMur+M27v3fudbfvCtB0\nga5oCtUVPvhUb3hv3tHh2UfKM2DNtba5PUJd0pHQcg9Ei2vZrQ/5Rv/b/w/phijpl1OLq0uglMud\nunMkZ3jnCdGwK7x7Or5CgjgrvHvY519eXo///r/lPe63XLg/X19Gntu3Cw6lqX/zXSwPJyXt877u\nuus8vx988MHOzxdffDEuvvjiUr59WXEqbzugrdHml5xxMPbfewdeiSsw7AFp9n+FDE0zYOj23Oe6\n2Wwup8MdsKoj4Qp1pG/NspcTlYXqHW2uaBAQ2K9mknO/t4hXpudgR+4Q+cvL6/Gv9Y046/9NgqRo\nELoC3YCn8tZ1A0nDWr5UV2AkzGOYt+kZHD/hSAStVdJirvDuSHRiZMAcnGR/v8jWoDsRr4DQFUhB\n7/GkdANrms1j//mymwEAPzjsIhwz7ij8a/1uzH97C0LHCEABOhPp4HdXa+/sMCfvWLV7DY6qPdwJ\n70RKRziWwoSxlYgndU94B/2utdxlAy0j/gUlPgJSMIIW3Q/AHHyXTBmAZCB45FuYv6kT3z3sW1nn\nM7P1AzAHreVi37FgGKLHirM/8lXeiZSOrmgSY2sKm1WwJ+4gyZWx+YI3Ek+fn54uYgoZaZ35Pj0F\n/turd1rbGVDk8p/nyhPehkAP89rkNZRaJIbSsfZV+f+fN0yMqQlaP3mbzasr/Pi3KftClc1wkkc2\nIpwKQ7Kq8ZRuIJGy/keUzD5vJ9ytyjulGea93q5wloLm1fLY0Bjzd6E4zeZAetWyoBLElV/5sfmg\nrHtmrAqnIhgd9C668q/1Zn/0xoYOc1CcrkI3BISRWXlbzea6DL3xSxBJM7Cjrv5cd3gv3rrUqdad\nudntixFdhYhXWp/JfZtQdoDtipjHZ37BC2cZ1Q5X5e2e6tReS317s9msbs+E12atJmZW3rInxGLW\nQLTbLjsBoTFtSFR/Dv8Bq+GbsAmrxItOU3hS0yH545D8CWzsyD0NbVw3gzKoBHDKPicCQM570qOp\nmPP31Q0BrciVhbs5tdm1drm7JeGOJ1bh539a4Zl5rj/0HirCfBcoUdd0sz1V3oWFd+ZtgIV98Wd2\nKQgh8PFnzWU3Ha773PYn1IbSLWdsNqeiGVUdwG2XneBUywq8k18okgJJ1RD48ofYEdllzaomIaUZ\nSKYEhCFDkg34fa6QFq5lPo2McA5EASFhTMhscpZyVN4AsGZjJ3738GdQjRAgG051J4RAOBVBlTWR\nS6akpluVt2p++en2RDEaNF044W1oMiBk6O3mYEU7oAHvILKPm9bipS3mGIms6V11FUas0hz17k+/\nRtMMyJL3f2FP8Lmmh+1IdDrv51621J4AJ2m9zl533V6UpLrSD58qw3AtwBK3ngv61Zwz7dmfMakZ\nkHxmOLfEWz2f3WZX3idNnIqJVXtnfwaYs9XNee9ObAq+CkBYK6sV98vJHUSeytsVjvZ88u5m6/5w\nh0GuUMn3GfU83RQ5t+1L5a27jyv//jOPb/naXbj3mdX466INPb7nQHJ/hP5c8w2lanYoXWj0FcN7\nAO01phIHpk6D3joOU+r+n+e5zBHGftkHVTFHmyeSulllywZURXaazYWr2VwYCiRfylkARA5GIeuh\n9LzoIqMyt5qk7XlzzIsD3ak8E3oSmqGhUq1AcrN5n6Q9hzoAJDTdDEddhaYJT5/3X15ej664GZS6\nZq+mZr7WDnXAW3kDwEeNa8xNnT5vb+UNAFIo3XSe0DSnYjyq1jzGlGEHp56ezAaAgMD7uz92nks/\nYU9ba430z2hTHFFhhjeQDji7sgr6FShq9rdh0khCNwxsbwxD8iec998VzZ4C2J7oJqQGnb9VKiPk\nP2hcjY5kJ8LyLsjVbVafd3Er77hrBTXPimnWZ3avsFasL3F3tZ85h735Prk/o2dq2j40my98dytW\nb2rJu02+VfIyZVbe9iDGTdtLv2hOb3gGBvbjNoVi/z9XSkPpQqOvGN4D7IpZU3H1cT/AtMO+1O12\nqqzCp5qVdyJlhndNtYq6kaH0wDO72Vw3zIFhAEJfXQqoCUj+BJRUFYQQ+OPzaxCNCUiygNPsbM8X\n7p5iVU734dn93RVqJTsCVIYAACAASURBVPTmCdA7R8OADsBuEk5Bks3Qjic1T5/3Z9s6sGqTGVS6\nZlXyVmVu94UDQDSjv7cl3obGaHO6/9OpvBWIlNns7p5mNZyMQkDgyNrD8c39Z5rnwqpaw7GU83q9\nrQ5CAG82LEdjW8QTRPY99kKyBt9l3F49wmo2B9Jf1vGEBglAwK84I+LdknoKTy/dhKde/wzwpZug\nd4R3ZW1rV95BNQjVuqVPM7zhvXLXh87PytjtVp934V9OWzq+wJ/XPo5wMpJ3tHE8zxzg9mduaDSv\n8tQJG/HytpcKfu/u9NRsnm+kuztce2w2z9hvNK7h6Tc24Z6nP05v002fd+b+3ecvc8rcZmtA11in\ni6w86D20cBS8nyE09Vyxu5XKEcN7gPl9Cg6eNKrb2eQAwCer8Cmy1WyuQ5FVCDkJQ04BkvWl4fR5\n604VDgDKSPP+eilZiVhCw/ufNmXdCy75zdCwQ1FYfeZ25b0zYgbNCL81QYDzeiu8EXFeH4lrnsob\nSFfYuqZ4Xp9wVd6fN5mVynXH/Bhn7GtOpdueaE9/Qcqu+9l17/EDQJc1rWuVr8IJPrvyjsRS6dHq\nsSroLXthV2wXfvXss3jh7S3pE23tLwnzizfzy7raVXl/vH0zVu/YjFhSRzCgQJYkyEqu8E5imTWo\nya68ge7DO6QE0pV3RrN5S7wN1b4qyEKFXNmRNWBtV2Q33tuZf33zf9TPx6rG1bjx5b/i9sdzbxfP\n009rn4+GRnNMgG/CJqxu/xDhWApPLKnvVxO6O0dyZUrmMqw276IwvWs2D8ey++u7azbPvIBwH1Pm\nc01Wd8OoEYFuj2mgGT3cklfwfoZQNbsn9HkP2Axr1DuqYt5fHEtqSGoGAlAQTnXhXflvgGwu4iKE\nq9lcl5wFFeRqs0lQSoUQtkfmWkHv+9IGpD4/DFLAXrnMHDls6GZzvH070spdq8ztIxMAtDmVM2Qd\nMFTEpU4oMEeoR42Uq/K2mrqtCwwtZVW2OZrN12zdBXUsMMJf5dzSFtPi6S8J2T52NT0JnSss7TnZ\nK32V8Cne4AvHNE+fubbjAKhjd0Ie0YpVG9OTB9n3w8eMMO54/37ElNEA9nKe19ROrK94BsrY/fBk\nwyLz/RJnOyPOJVflbUSrIFeEkdCTqAgoiCU0p88bAHZEssM77qq884V3OBnBmNAoSMlKdIR2Q0fK\nEzi3vncnAKAmMAL71UxCwJ7Uxz4uawBdomIbNm34ctYxAN5mczc7HKMJLf33ADD3pbVYs6kNmiHw\nvZkH5XxtT/L1ecuSBEOIvCuCefq8ezlgzT1ffa73BrxVW+b+3bPmuS/0hBDOQL5yC7nM0eZ9NZQC\nkc3mNGh8soqAT0VXxPyySfc3Cydw7C/7ZMqAcC2bKVeYVZIwZIStLys7PNW6bZBrmiFb4W3Ezfu8\nDatvWlENGMLA6uZPEMIIvPCKNWGIYc+/bo149pnNqCJe4am81boGyKN2Oc3Qeko2mxFzhLd7xLs7\nvJ2Kxj2TnD0Lnavytu9Dr/RVOLPI2f3F4VjK2b/QVAgtPdGNh7WNhhS2djag0f+x5+k2YxeSUgT+\n/dMLm8STOkKBdDcBACTWHwe9dbzzGYP281blXaFU5q68dbvPO2Teiw/vrWIpQ0Ncj6PaV4VqqRaS\nBBiBzpxNyvd/NBd3rLwPQgi8/uE2ayY2gdZ4m3ksqubchZApka/Z3AooTRee127aaf5/YfSjedId\nJO4+b7v1J6nluaDoplk7U+aXeFeOkfLdNptrmeGt53zOfftavhYDIQSefXOTZxbDgeAZbd6Ppu+B\nWqSmGPaE+7wZ3mXi9Emn4OBRBzr3OvtkFUG/4vzDE3L6S8eerGREyAy8aELzNM/KlWZ4G7qcbtbU\nXaOiJQEpEDOraWsCFfteclk2oBk6UkYKWjQEZ35Su9ncqnxl64vciFeiPZxIr3AGwDfhMwgrZFMp\nCaOqA1AlMzyT1rSpumFO8iKEOUNb0BXedsUl5HSfd2azPwBENKvyViucCxk7+CKxFKCmK3e4lk1N\nnwcjPSFOPkp2c3IslUDQnx5dDwBC8zvvsaO1AxV2ePviEJoPtYFx6Eh2eia+AbwD1pZ+YIb79tb0\ngCd7lrsqfyWqYN72JwKdeQcP7Yo2YmP7Jjz+Sj2WvN+AjmSXZzIcuTJ7MFVcS2Bly/KsVeeAdEBp\nugHZdZ99NGVdlAT7vmSonmcglWKHd54Q9FbehQ9YE0KgM9q7ZvPsyts1sM89IY/r4iffRcfnu7rw\nzxVbcftj+bs4SsGd17kGBvbEPb/AUFGs1oZyxvAuE//fAWfgJ0dfZt0iBqiy2WxuS0npL2A7qGsq\nrfCOa5B82TN1CUN2+vjcfeKSrEMKRK0mc8nZFgBk1XACUNcl1768fd72hCkiXoH2cDIdrjCb0u3K\nW+gKVEXGiKDZPG/fLhWOpszPofmh6cKpvONaHAnNACCQ8rUDwgxGu9nefTucHUqVvgrzVjtIzoC4\ncCzlDG4TKX+Oyl3Af9BKz2fPJQVr/vdPj4HeYYYnanYh4LcvqlyD6qxz8MTrG8zKXElBCkZhRKsw\n2mfeKpdZfcdc4b16o1khN7anJ5SxZ56r9lVBMsygFJLebRW0rvlT5+ftHebAwUlV5gBJKZQ9Teyz\nG1/EB13L4Nvn06zn7PBKaYZ5+6HFvmjpzz3NIk+zuT1oMpmnP7uvfd66Yc6alykz4LuvvHM3m7tb\nLqJGFz5sXJ31Pok8XROl1t8Ba3Z4r9ncUvKpeYuluwuw7vx10QZnEp5yx/AuM4p137JPVhH0eatl\nN2HIGGmFdyyhQW8dl7Uvs/K2vmxEOoilQBSSqkEkXTNluSpbu8/VHinuft4OTykQMydesSZnCfpV\nnDzCmkFMNiAk3ZqaVIJPlVFTYb5XJGmGVWs4BikQgxGvQDJleJrNkykdck0z9EA7KhP7mHO4Z1T+\nABC1Ku8qfyUkSYJPVhFJxLHgnS1WeNvN5n4AMoQhpcPfl4AywgxLvWW8s09ZT48U3m+vaucWPpEM\nQsTMufn9B6xG20irerIH1bmqe8jmjGxyVbvZzN01GmN85t9nc8fnnr+RHd6bGiLOHPbuPm+7X7/K\nX+lZtz39hZT9ZRxOpPvZd0fN/v0vjzBnN5QrO7NGnO+KmhPb5Ap2O7xSuuFtclfSa8c/99YmvPbB\ntqzX9iTfaPN05e0Nu2ff3ISXln+OsNTsdH/0ps9b13NX3lpGuOVbqx0AYnmazd2tBLvHvYSH1z6O\n19au97z2b1/8H/wHZy+GUyhDCM/FQ8Gv62cVav89GhrD+M2j72PTjg7c98xqRON9v3ArplxjI/py\nwZJM6Xjzox34y8vre964DDC8y4w9baoqKZ7KO4sho6bKbPKOJjSkPj8MX459A0YsPamK0CWn8naW\nEQUgWc3u9qxn9v4As9nZvlXJXXlnjVZXXCPMAVQEVewd2MfaRoMOzemHVhUZFX4zFCNWsOzobIYk\nCYh4BZKajpCSEd6VZr9gZXQ/81hzNJt3psyFEsYEzYrYp/iwszWM55dtMf/B+lyVN2BeaNjH75rn\nXa4I43jpAnNbpB8/cOJIp5lbaH7rIsDU5WswH5fTg+Ls1gF17834rKUBcrV5cWCER2Kczzw3G9q8\nM63t6mqF0GX88dkNCPnM/bv7vO1b9qp9Va7WAyNdfarZYbR5d5vzc7u1GEyNOhpGIgg51JX1ZaZI\n9oWZAXWfDVDqvjB/l1zN5prhad2RrM8djafw0vKt+PuSeuyM7Ma8T5/POV97Lkae+7ztqYTXbmnF\nR5+lBxf+c8VWvLBuGT6vfhnq3uY8+T32eXtmFzN6rLx1XXQ72txTeWu5K2+7p+mJ1z51LpQMYaAj\n1QZlRO5FZwpx3zOr8V93vdVta4emG57lMM337t993plTwP7+yVX46LNmvPnR9l7vq9iefXMTfnTn\nm9jZ4p06ubtBh24bt7Wjrcv8/zVfyLd0xEs6HXFfMbzLjF15GxCe8E6PJbcYCmoqrfCOpwChoEau\n9TRf64bkDFjzfPFat4l5mrqtn9uqV6fvv3Y1J2eFp6x7Xl8RUOFTFXMOckWH4QlvCRX/P3vfGW9H\nVa/9TN/19H5OzknvIR0SEjpEulIFiShYLyI2BEQR9PpD5aJX5d5XQbHAtYAIypULWABpIXRIg5De\nc0pO3XXKej+sMmv2npOQkJAE5vlAOHvKXrNm9jzr356/wfTMWax0xyDt5EUKSdiOhxjTYM+5eRSY\nJjgdgxEYq+w2H3D7YGkmKkxqERuqESB33eR9z/k51OD4GZztI2F5FehIjxDu/mRMxylzRvgxascA\nsf34rgGLndIG8VhnODZGNZaFPvlpkZvgZSphKnG0pVqwrm+DiGl7xENPoQskT5vT8LwDuXXqoBTz\nlhdQPO6rsAXKmMqRmBc/EwCwTYqZ9xeY7CuJg2QroJhF9GT7AxKnQo42MQijeQN0Rt5xU4fteMg5\nebylPwa10idSYXnnfCL58Su348mtS/D0tucwHAghIg8j2DDD30f+/Cf3Bd3PWgO18DU2Fsfx8Pra\nnmFL1uRzOR4JlXYtzXrfXZ33cAlroXFuhYhEtqCG/b4RAReWkRvHlOJbv34Bn//RUwGyGS488Xah\nlogfcC/Du9Uudnd4aMlGAMCK9cFFUVAlL/yaO3uz+O7/vIyb734RQDjJ9/Tn8dWfPov/vPe1sm0H\nGxF5H2Lgcp8e8QJu85OSl+CyKR8RWeeEqEjFDWiqItxXhq4G4rfE8RPWvLxvkYsabznWy+uwY9vx\n5zUPBT4D4Mufqi4AAqjB2vKERevS4eo0EU3xyds0NCTMIHl35ujLl1reXiDmXXRcEVsnpCRhTvXd\nxUNuH+pitejpz+P7v30Zjh20qDXTptYwczcTT2rqwv51drbD3dXC+qYbgOohldBx2xePRW1lDBk7\nA0MxAaIGLG9DYeSt+AI1gfkC/GQ3x4DjemhPt8EhDr551xMAgK5cD4jiwcumEDM1DGR4jbwjXLfC\n8jZTUtzft7x5XH989RjUaC1iO8cga4X6qwfXw8vSmv2/L1+BL972NJ5bSePv/YX+wHGinaylwXE9\nPLVlCfr1jZClCbjl3ZcpAIoL64h/iYXGa10rUIqubA8Gi0P4zSNv4KofP4Wt3ZlhNbdLX7YD2SJW\n71oLrWETVOba91gI47W1PfjRH1/Dd38d7o4W51IdvLTzFQxk/UUst4qD3+1hlf009DYa/y+zvAsS\nebthbnP/M0X1hFUnt9flHqF9xe6M561d9HmRPQHDLYzeLrRS5aJDEKW6GYFF2zBW86ad9J70MC3/\nsORH3tt+1cbesm0HGxF5H2KQyduSyLs53YA5jTNgKiwm66mIWzoqkqZI7DE0FfUVPkm7riLI29ky\nDsUNkwFIwiEy2Uj//0bvWwDoAsHfTv/fHLNMxHJlyzwRM6DrKrW8VQdEcaEpPB6uIcXc5isHX8fD\n6/+B3iJzKRcSKNgutnXmoCoqNnX3omh7Qq5UfAdhMWtOiEYBLhxk+k1c87MleHNzHwaGXMhtTxW9\n6LvMAboA4W5zVodOHJal7hIYGvMU6P6POGNnYaksN0Amb7Yw8VAU4QNSQt6KXqTKdVDgekR0U4Pq\nghCC7Sx5jeTSKBRdZLJ+jTx/6QvL20j6fd+lmDe3vFNGCiopDy2IVquOKch7xY4NAIA7HlwJQgj6\nCiWlSxqXf6WWt0tCrErNRUXSxMBQEUosCzVGX3KqomJd/4aApekRD//x4m345Yrf4cnXaDLQ+m0D\nw8a8S8l73dYB/PjV22GOXClkfVHiiXpdcq/L4C9xo/0N3Lf+TxhM+fFM/rIujXFvJstgtKwXf3O8\n2rUcAwU/L6DUba5YGcSP/Jv/5ap/H4ekKgO5MmRf8HZ6cpcuSDj25DbfuGMA/3X/soDrfTjyPpRy\nuEuH+HZi3p0lLU7DKjhMQyv77FBBRN6HGHi3MZd4Abd5OkGJQybvmKkFpBh1XUVrbYX4m3gaeofY\nKp9ocDtHiAYn/Bz+viGPQpjbHIA1eSn9n5KYNz1GBzQXRHVE85WYqSNp+eP86/q/+brmtonnV+7E\nt3/zIlxbw2CBveTYGF23xDvAPuelal2d0o+LaDSBTKUdxVy1ECBcriInn58vWlzPg6XSfXVDJu8M\nYiodu6gVB+ApLC9AsUXSXqAcDzSWbjHCdlxPiKcomgvXI9gyRInMy6XYi1AR96efuXeHbE7eKSGB\nC9XzrT5meadNSu6EoKScboiGDYgKkqXPhpyYtq2/F45EzqpnsnI6D6ahwnY9IfIiQzc8NFbHA+1n\nTxt5MmY3TAfgl8ABtAFNxsliTd86keCn6wrk05JhyAYA1m4rr4tWNV8NcHcQOvkshGEnt4ltnHxl\nK01uHSvvs7p3LX6+7C68YD8ItXon9Kb1cFwPHgsDFG0XetPGkkF6tIwS/n0EEBDuKRsv8fZIzm8n\nbC1n4e+N5X3lfzyOl1d34bkV5Tr85eMoP9ef1/zfbtX+9hV7Gnep5e0GLO/wY3mf8mSMl5mW73co\nl5lF5H2IgVvepJS848wFzcmbqIgZQfI2NEVYhAAATxUPKIXix39RalmHrDBD3OoyFATd5lTpTYei\nuVAU1gwFQNzUUBEP9oC2XZ6lreI1Fssjju5b1oxcbVt+80iWM3P9J9W0fz2uCkUliM/5BxQzD6J4\nAVc37bxGaDy9pDOb6xKYjFw1g373uv4NsD0HMS3BxufPnY0C8k4BLmxhvYfNkaHQc7oegcoFDVUX\nRdtFX44nDkpa2GyBMsAWXZt6ekA8BU7RD4koiivkTLmLO2kkqTEqhwYA5NwMVI/OPfdCFIlv+e0c\npLHCZHYU8svno4KwzHvdEfK8A5LL18vTcyUTCpK8xpuPQU+IOZRlcLn17xFPJPFpqor/emCZf97d\nSHgOZotlOR+q7ore67sDf4nzygoS8/MBuFXtegSKlYU17Um83hOMsfMXf3+BHpdVemGNewVG+5so\n2g4efHo9rvrxU1i5cRfUip7AsUrA8vYTqoazvPNOAf/+3K349crf7/aa9qTbrVZ24U3mPQPefsxb\nnvdk3F+YD7eYKP246Nr4+6YncNeqe3Y7vr2F63n42h1LcO9j4W11AZQ6YgJW9HCaCNt76LuxtoL+\n/sLc6283Uc0jBKs29r6jxi97i4i8DzHwhDW3JObNLW9D5a5XD5apo7bSJ0VdV8vIuxQyAQXIhoQ8\nCnsgd/m7EjFK3rL1yWO0MVNDMhaU7LSJLb6DJxEpngHD8jCiIeWXAknhQSK3PWX/5vNAR2MaJ8xs\nDYxXYdnqPMnMMjV/PjSnrDOb43rCbc47hf34lTsAUPUzOugY7E1UCtT2CljeQ12w3mBN4FwyTGbN\nP7NsOx54YpMYe9HxROa9fJ9UaIDioj9TxKadg+jNDQKOibVbB0RCG1RPlCxx8t64Nc+6z0neBcVD\nkRRgEDZ+fq3En1SejY5CEqZdA5PF8hXNgc403Tlx0fmk280YEd4WbnnHtLjwLshKev1539LnBNfd\nnwsmj9VtwD82/QuATzAXnzyOnstxxaLW3jyOnch5W+QtkvGYkp6iEtF5T7a89Za1UONZPLL1kcDx\nolQupMd63inikefpPX3hra1Q48GMZyhyzFsi72Es78c2P4nOXDde3PkqVvS8OSxp7l6m1IM14SX8\nz9q7sXGAVkS83WzzjTv9+/R2Er5K8wGyUmfEMG/NviKTc9DVl8fGnYPIOTlsGiwvS1R3Y3kPN/4u\nFs/mW12XAEYe8SMfwWObnsTTr2/H7//xVuixpXjxjU6ahf9WePjmQCAi70MMgZh3wG1OicVQfOvN\nKnWba6qwfADfsm6o9gl+WPIOURIjw7jNOUbUVfqHqwomtlcFysc42cRMXciJcnDxEz6GuKVjYmsD\nHGLjcxeNRnUFHWdBTiJ2Zbc3V3BTkU4YaKyOB65HJOUxy7syYYpriM96TKjQnbuQkoHjEphsMdLX\n9Dh6cr2iZGt2zZHivM6OUXD7a2ATGy/upPrvXBa11G0OACZbbK3fPuhnzGsOirYrVMrkudVVHVA9\nPL9qJ2761QtQDBq394h0P1TXLxdi5L1lexH5okv34d4JVmGQGeT3UQFxNbjwJ7UvT4nZKRiIW5oY\nLzQbpk7H1S/FefkCSTdcn7wNej5L8cm74PrWZU/Od3trjLxL1dOMjlV4YM1DcDwXhAAT26tw1GRa\nG19winCJC7evDs72MaKigTeMCUPfUAGPv7JVkJDcjc4cuRJqRXfA8pZDQNLFis5hgYQzBqphH1zA\naH3t0gLDldzmEnlb4eQtJ/r9v9fuxLLulaH7DVceV3RtaHV+WODxzc8AKPdqEELws9d/jf9dG1yo\n9PTnWRc8EiDm4cgvb7t4+LmNWL2ZlmxmbT+GvCvfF3rMvoCX5xUdF//96p34/gs/KRM7KvXWBGr3\nh1ns8NACfw4c1xNVDH9a81f86onnsKnTv++7s8K3sERBfr/fDUTkfYhBVcOzzbmVoTGZUUV1ETM0\n1ErkPba1UsiE0pPQ40c2paXPJHeYbJmXan5Lx7O9yzaPb6kVC4yBrI3KlIXjjmgX23lTkpipiZcc\nh6uxlxnLJm+pTeDoFkqSd6+6V4xHdpvLMWthgbsa0gkDDdWJwPWUlsNVJM3A9ei1NN5cnaIuccfz\nhCeBqDbuXf0AvcaqMWhPjQheuEv3W9e3EaYSA8nR+f33y+eXzRG3vAH4Cxtmeedt+sKvTib8/TUD\niurhjU19gOJC0VwQx4TrefA8iJg4J29uUaowaRmT7DYXjVmkBZur0wx5Bp4QZ+cNxC1dkLeiOeLe\n9hcly7tAnzdV84TbnBOXqcZD3ea9WXo8IUy6Vy+KEkYK/9nryXK3ugKTkXPe4wsxQ1wDNCcQ81ZT\nvbjnzT8LBb8f3PMq7n70TSxdxWK3hpSAVbMT1sQXBQm6HvGrGmRIuQWDdjl5F11byMNyD4ilxv3K\nDtUT3gWZvLmGQSl6MoMwSAIN8ToAwxPgcKpyj21+MqDBz/NKZPL9xV9XYe2urVjWvRKPbHxMfH7/\nk2tx9zPPIT7zceitawLqdjIxquke4bnY3p3BH59Yi+/9lraslWV4d2a7Qse4L8ixDP9C0cP6Aerp\nKG3y44sJ2VjbtwHLi0/AGE1DII7r4anXt+FP/1obOIaHRGQJYPkdEZu6BErCf/YzuxGl6WFW/HCS\nvgcCEXkfYjihbSEA4JSO42GZ5daAcFUzy7u1jr4oJnVUY1RzhbAeAQh3LHe5A74rm26XasI7R8Dp\nbIPT2eZvl9zQJJeCN1SJRNHvuGVqJs4/kVoZU0ZS17HIqAZQLPjkHbf0gIAMjAIjW7pPU20Ccxpn\nYFrdJKzr34AhbTs7h3TxcsyaK615GtIJk1qBsopcSUb9rPH1qEun/HMxxboYUzVzXSL01wFfxtXQ\nDJhG8GfCSSTjZGEp/uKptT6FUpiaLITj66sXbQ95FhNorvGTDGO6IVnOvshMNu/QlzBbwMiWNyGA\n6rG+6l65d0K27ImniVp2wE+kKuboPbKYWA50h1U7EAzZQ1DsONA5GvYW1pVMc0SiD0+aMxCDxa5X\nJm/umvcGqJiOVrsNg3JrTql0atsQJVtVVYVlXfAYKTAvCl3EObAMSU9/1HI8ufVZPLLhn3A9V5RM\n9bA2nUqImI3sNlfCyrcUV7yMB/dgeXMPSEz1PUCK6olqD368O1gFNZ4RLm0ZWTuHQk7DB0efAcDv\nA5ArOPjt31eL/Rw3PKntjV1BFy8PXcge7KGcjd89/1TZsX99diNyBiVEo3VtoFc5J38lNgRr0guw\nJtM6/lK1uqyUUf+/ax8WvyEZL+54Bc9L/elLUXSL5fr/kuXN8asVv8OPX75dhMe4Vfzwhn/ihy//\nP2zxVkKv2waoDhzPw6/+7w08tGRjwAshpH+55e2RMg+kKpP3btrfdrPnbLgGPwcCEXkfYphcOwE/\nOf67mNVwBCyj/PYYnGBUDzFTQ1XKwq1XHI0vf5hm+fK4LQBBvgH3ouwelC1vosHeMBVeVs5WD24v\nrJyPKUnfhWxqBj588nh89zPzMGMctRZiElnlmfEbs6jbvLBsIYobJ4rtinR+njRycvvx9OsUjyXE\nlMfdrSnPCsubeNTytgzNT3aDH1fk1xC3NMwd7y88eDcxUzOggCa1aFKHXMI8Dbyvugw5NGCqscC2\ns0afiqq+2eLvQHvOgHyqi6JXBCEKWup8z0jcNH0vCCccx0Qmb9MsbE+lMe+Cr3QH1wAhCLjNdU3y\nTngl91+aJy7/6hQMxE0NMS3OzmvT5iu6DZe4UPKViO+a5hOo6kiWN53LJ1/y+8bLMW+e8ObsGAli\nGzDa30Rfwbcqq6sly3Dlb2BNfRquPgRNVaEqCoqk3PImqitCSfLcPrrxMVz1xNegN9FSL9cjwoPh\nZYOLq0DCWgi5Q/WEBSqTN/GYfKtr+6ED9jwljARkHf3BnA3Xc7FhYBNMLyUWMLe8eBs2D/ou7oJt\nU8liR0c2xyxCRn6PLN0kyc8SbM9tx5cfvwlPrH8hMNwEy81wulqhQRM6/4E4t+KhR6WJX2rp619a\n5PXYdBHVP1TwNQXYgpiXBZaSmWx5bx7ahqU7yrPOf7Xy9/jNyj+Iv4u2G9B8v/Wl/8Y1T92EXfle\n/HrFH7Az0yme9VIZ1NV9a6HX00UQJ+A1fesD+6iJwYDbnH+XJ2nYc0+G63riPnLIev6lynUyIvKO\nAADQVPYjUspdedzyVlRPuNJrKmJCwjBgeTOrNpDMEaKqxlGZNANxW0teCDA0VfrkbmoGFEVBY7Xv\n9pXJm7+EYqbGXKBKILNazlavTtPjWpK+znhpkhx3hauJId8t7lLL2zTUACnpsWLgHKauBRc2jNhM\nzYSmqXC8oOXNyVtX9fLYqpQ3YKlWYNOpI09ERW6cv13zr5d7PbTa7XijawOKjg14KtobZcvbpN4F\nkDLL2/OISNrjpMFnvQAAIABJREFULwlFt0EcAx4jb3gaFIVlC3P3eWkSIRfaAc1GB2huQNzS/fun\nOYiZmp87UIhTS5yoILaBIjJSzLsI4mpYsqwbdz9MXZM524/r8nixl6mE09kORSEYcCh5X/Ghqaiu\nCU6vmhhCwaRuV8NQy8ibXoODdFJ+PoOWqNEuNVlhiwsu7AJQFz63vF3J8k5qKcQd2kRGUT3YbJ4H\n7SHEtBgaN58HZ/toAIBDbL8Gmn1Hykj4vyvFQ6Ho4q3e9cg5eaSdVnj9dWIMXUO+KtiWHuZKdw0M\nZei4RJMdRhp6y1rE5vwdD+/6LYrI4c8rngxc85Cdpde1fip01RALKNntrbe+BcegnhAufyyseOn3\ns9T5E17dvAFf+q9nxCLHigWJqbQ3Oq9l93rpb3ht34bAdtlb4HgObNfG13/+HP7th/8Sn29l5ZM3\nPPtdvLDzZfxr6xLkmOVdCHNJMw8cH2NNrDqwWUkMBkrF1vZuxuceuwbLu94Qn8ltb6EHr4mrJAI0\ncY4QgnvefACvdvnhCdvx0McSE4frQX8gEJH3IYzm2gROmNWKL15whPgskE0eAqOEcDVVCcgbkuEs\nbwC1lbEAoafiQWICgNYaP0lNjudyWJLb3M821/06TEk0hYu4AEBVih4XsFRLM+Cl97OaYGVWsuWt\ny+QddJsbuio0vGWYqgFNU6h2t7SY4Mk3pmqUkbec9CeTsxibNN+xEMtbjWXxt/7f04Q1TxPudtNQ\nYeq+Z0V0RXNMZPIOtSCY5e1fqA04BmzH893mAGJxlLnNP3bqBEHufFvOy9JnytMRs3RhvampPjqn\njLy9giU8EKQYR8YblFzGRX9O2Hf15XwrLONkaEzZMcR+3EqLWRo8zd83zix/T6X3z9BUOGD3UnwH\n/d5kXAqTlHTVcwerpG1sIWeb8DIVYpz8he95HqAX4RXi+FjHlTAddqzqsg531PJOm0nYDoSHpugV\nxQKAex8qrKT4XXHPx7Iu6vKOF5vhDVXD3joGAPDcm742+NZeupghjo6BQWZpMsubX6Wa3hUoAyxk\ng7+/jJ1hc6RAgyFCF778bT/05vVAMQEvkxZiQaVqfRxLNvtlc+NHVOH8U/zcDzW1C/IP8ub/eQld\ng9TFbO8YgYQex/qSJjy25xPj7ct+gy8/eQN2ubQpztbuTKjVammmkKQNI0au9Oc4dCxyxjtA3d6y\nbsCDq/8BAPjjW38Wn/FjHdcLvEMAQElI5J230Vvow5Nbl+Dny+4Sn+8azIuZiCzvCACoxfzRRRNw\nxBh/tT4hdQTcwSoU3pwdekwpucdMLaiQJJM3CZKZkDdlkBOpONpqq6T9yxcSlaZvRfKXWMyULT//\n/LpE3tzy1lTNJ9mSxUVx3RHwWMIUb0nKY96moQlXOOCX9nDi0jU1IBwiX4OuKtjUOYTfPbpOfM7L\no/RQ8vYXKLy0SoYWIG95MVOSw6C6UIkuLNiKhCnlNDjCCiCOQd3mhJM3LwVzoai0tj5XcFDgbnMA\nlim7zTUoACxDCyTNAUCB5JDQaC5CwtIRN+j86rU70KmsgWKypKd8TMwDKcThEgdEp5nJil70xXDY\ngi3LuscRQtDv7PLbz7Lvz7t0e6ezCTuTVNr0kxMvw1ktF9Ahs/71pqHCUYKVA2JRKB5PAhhFjEx3\n4OjmuXRqpC588iKosHIezGKNyDsAaLKiYhQB2/QXSACgUMvbIx6G7AzSZgq27Sc2Op4jkTf9jsp4\nWlLCo9v6cixbv0jnmeTpwAekBc6OPhZGcA08vIS6yGWyo99B5X7zrx1L50hx8M+XttA+5ZkiuocG\nxBzpii5i5tzw1iq7oShAYeN4EMeEogDZYkHEkkvj/tttP8FLU5WA0Iw1+XmYE18AJ/A1W/rx6rrt\nbJ4NjKrsQHd+FwaKfqWCHMte2fMmrftnv+MbfrEUP/uLb81yFN2i0DRwPSJCFv7AWNWJ68sJK0RF\n7oVF9JqsbMBtvnEbnfNdhV6YE16A3voWc6F71I3O3iFpvQLENpnbnC0M8g5eWeu3C/WIhy2dQ3h+\nVac/3ihhLcJwiGtxFFfNg9dfH7q9lLwtUyuxvOWENXr7501uxPc/O5+2/pMs7/GtJf5MMMuCn1sr\nt7wbEv5Cg1tIPGv5e5+Zh7PmjQ0dK7e8AYiM5dIMYJJP4SOTzg1+oWR5u92tZePxX8R+9q0MQzWk\nemH/+3gs2ND0snri0fX+NRoh5C3PtxJI6C8JA6gudMUQZXQVSRNtKRqX16q6AuSbZZa3r89OAuSe\nLTjCbQ4Ahkl8kvc0aBpLAJOS5gACm+TAeSNmakjr/uIrhz5R1uQWLDEPXoH1ZlcGac9yzRPhEL5Y\n4q1fu3O7YKMgLF5ueXsKJYo3B/3yqE1bbdz10Dq2ncdXM7Br3gKgCNLjC4BYjL2U9SIUBUjoSVw0\n4TwoTgyKbqOphu4vXP/FGEBUmAodP7f+O6uepIsgT4Preb4YDot5d2W74REP9fE62K4nFp02sSWl\nO2Z5mwkpt4Fu4yEEl1Vf8DnK2QUUbRe/fGgVVm1hjXpcXWyX8wb4dRLHFGI7ikoT2VZv7sMdf10B\nWymI+VWhS25ztsBgdegkmxZz2DkwhKLtQW9eC60mqKrW7/o1y6qqBKRhAdAOaZK1nnPZ78s10Jyg\nZX49OT80kA35/cmu+tfWlau6FdyicJtD8VhIyQfPc+GLqEwxA89mioKuCkVzgqV10m9Qq+wRTXgc\nhzDLm97Hj7Z/Bl6mIuClsl0Pv3/CL9+7c/n/4DtLbsMDT/niMZHlHWFYqHu4Y2aJNWwZWlD3N1Aq\nRh/kptoE6qvi0FQ1QO6TO2rF/08fU4svXzjdj8ejNL5OURvzCZ94Kl08MJd5Q3UCR030CVa4iAGk\nErIrmrfwLL/YMXV+TJx386pImNA1Bc7WsdQqIeUxfuIRVFoVpaeDqRmi5EgJqXU3VCMgvfjZD07B\n5R+YIf7Ww8hb2t+TpEcntNUFd9RtqIqOuGR5z2+eCxAFesNmP6Pe1ZHJ2zR2yaw6a+ozfptXx0Au\nHyTvbGIjI2h6nzVNoeTLXtqJlEvdpooHz6afxS0dST2Jwlv0+lzFFpa3V4gL0RauVpbxBkUSk8hl\nYN+/dscu/Owvy/HEm5ScSYaFW3jZGrdwTN+789yrg4J8XOY2R7IXUF2MVuaCFBOB73hsgJYUcosx\nriawoycLt6hDt1xRiREgb0BkxOftIjziIR9jFmM+IfIKAFoOt2pjL/64lNbzt6aaqQyqwlu32uVu\n81hSkD/XyOdVBbativtJPy/gsZe34ull27F5Fy2Rq02m/aQ/j7vNFYgcCFsqeWT3dzBrY8122mKX\ne0Dyeep2J4SAe43VWAbEU0AKcXGN37l7KdZvH4AxgvUzcHTkXliEpNMIBzZ4GZ+mKgErmkP+zdhM\nuY84hig5zEreroxdImID3+1N57A8abDgFkTCGkqSyVJGUhCrIyzvrK+q6OqA7gT7juvhSWe268F1\nCfNuqMgXJE8Zu8ai7QbG8GrXcmjpXvEbEfu8S4jI+z2GcLe5VPIVEvMOWJYSudek/Bfr/KlNmDra\nJ3MAAUEYDpnc4WkBlzk9p2+5VyV88RiZ8MQChBGVSGarS6IuLnkDPA26ptDEKkUBoIAUEtAhuarZ\nS8ojBIs6TkBV/wwa72OQyZk37pARqJsHVZKrkhYBCb085i27zT2pfj5VojKnKABcFQ3VCUwbXYu5\nExtQHauC5VZBiQ8FMuppqZhfh6omhkQmLHENDOVtFGzfbd6XXI7p09l99VToqsK6zrH5GPMMVNZb\nmj8TBduFoijwhmhoxEYOipmHQhTAlmLezPLut/tgxIPkzc/Vn83h+VWd+PsKSt6lljePLTpMMCb/\n+kLs7CmIuHavthHfff5HUHU6B5br51qIlynJQzFzIt5tKQlk8g6Ia8BVCjBYtUYpefNcjbybD1iD\nzrYxrByPC9FQ8n19K81gbk01U8ubu80JJe+EpQuXdtKK+d4PI0jeXPdAdPDzCsiKen3672lzx4rf\nnS2XWqksROKYoGI7qng+sgUnQJwAkMl6ICBwPMePeceyIIUEAFXyDri49wnfclR0ByAqFM8MzLWm\nKqHlcmD3UYkPQq3sogtqT4NC6Dhkb1fGLre8lVjWJz+jnFjzbkGUivE5cnc14gOpy2CqlvjMcT04\nnoO8mxeqisTVoaiOmGN6fSXfwd3ujkcXAJoNuAZts+zySgJ/n7LjpTkChkmqO0CIyPsww5566IZa\n3oGYd3mdNycbz/MCCWvyQqBUfrB0eygUglhJrbpsrVt6+PG8QQh/iR49tQkfP20irr5oBkzNpCtu\nNv50wixrSiCTN38RVqUs6KqO6sKkQHtUUzVEOdCYmjZ888jrMK5q9LDXaGhqoJZ9XHMdPnDkCNx0\n2Vz/slUFhVVzoearsKDZF27hgh4yKhMJ6JqKL104HfOnUq+CiQR9YRh+0h0tFSOB8h7enAWOgf6h\nYHY9APSxzm10kaMGLG8A0OtYwhT7bOG0ZurZYQRAyTEPnSQAKNA1BQumNWFEFQ3Z7Mr3wUqweHKJ\n5a0YBVimImWrJ6Brip/YptlQFF+qlQghGVVoxW8Z2ibmwHN8qV23zw8ZKUZRWN4WErSch32HbrJY\nrsVkMJnHIGXRf9/Y0oVfPkL7NDudbSDFOAtNcPJmI0pQi7Ml2URj3sxtTsnbRdzSka70UBmjrV1L\nyZ+7r4u8RxD7DRb1XRhwugHFhcZEgyqsBGrScRAiuc0VOW4vJe0x0ti4Y1C4r0lJ7kHRsyl560W6\nwGChB+Fh01ykkiE04AYXWSqzvHUvjuKGSbC3jaLbmSWqN9MFDsnR3vSKS8chk3e2pH4bAPSGLYjN\noNnmhuWTYFKpggIFBafot2FlY/EKcShOnC7CJLc5j6kHLG/NoUTMUGrdK6oHKJS4HZewcj0ahvIX\nOPQ7MnlbkLfT3QJnRzubA7o9bmmR5R1heBT3QN56iaVoGcGENeLJ2xXpv1wmskSqkyGsLWCY5Q0A\nY6voD5vYFsa0BF3VMtEamoFvf+JI3HrF0SXnpeTI5V1jpoZjp7eIuHiVVcmuRRMNW2QYEnlfcfZ0\nfP7caRjTSo/RNZW9YPh1aSJO1VSTQGOqBhWmbJkH57M0/p00E/jwiePQ3ugfo6kKvMFaJDedgOqY\n/7nc7IGjpabc2rcU+oJVOem4GnIFF7miCy3mZ1VziyWQxyBZ+l051vCFuc0NPRgWEYlVnobPnD0F\nNRUxen+IBuJqKHg5KLotXsS6ruITZ0zGNefT+9WT3wU9zsnbEucCAK1yF5qmv+W77l0dLbVJsVDQ\n67Yj1rLJF3MJkZYFAM9gCnBF+twkYjq83iZUDbIKDL0o9NKTajWyeceP+zJLTjGZNeZpuOXf5iPN\nyHv11h68vtFPsgJo8ppX4vZWrBwsNYZ7/rYJBP4C1CU0YU2zCsg4Q+iobGGeJhWEKFA1Rt6eDVM1\nYLOsZrHAqejBC7gPWsNmaJXsGow4aiuo0EvOLhey4fFuIkkFr9vZi9gUKpzCFy5y3NzziEgMEwtX\nISTjoChJ2Y6qYhnlJeENVSXoLw4grqThdnb4izUtaBUX3jgyMA5ZMjUjZYIHcmMAQPEEeRc3TMIM\n71xYmomiWxAxb1GD7eooFF0YqinKHh2X+Cp2IrFRh6J5yBSkeWTkrQ42wB1gZWWqC9vxqKdDs0Fc\ng3lwuOVNv38wa4v5cLvaUB1jybvsGY9behTzjjA89mR5lwovWKaGie30ITthVmvoS5KngHgEQQlR\nibiUEPIu7fTEccX0T6Bi2wkgmUqcefTIYcdqqDra6lOoqYiVfS6j1Hr33dYkKNTBj5dUz2pSScwc\n71tqhqYG6n0BX7iBZ30nDD9cUGZ5l2Sex8JKxbgbnhDELR1nzO/AvMmNWDituWxfSyuPmXPyVrhl\nzd2sRRfElPpCM3KXQx2Vdf6LWGQrexp0VaVubzlhx2KuVtfXntfZfSaOgSFniMqzshc5d5vH9TgS\nehy7cr2iJI+/zD+4YIw4f6eyRri947qJz35oqug0BwBoXYmCW4ACJZDMJ5frOBq1evM5Rt5snKpD\nCXj2tCTMup3wCnGkvUYqYcleuq/iL4DqUPJm46urjPueE83P6OcvfM8jovWqxvu6aw5yWQVLWJtM\ngy1aHTh0MZ2gNdod6XamSgfAU6FoLJud2DA1U2RNBxfQ/iKNz21N2gI8Ddtz23HHsrsgMvqlccLT\nxMKoM+tnO8ulcIBP3rwlqli4Sm5z3oa3Wm3CN46/KuCh4ffC1bLwiIdxDS04YVYrxrXUse22P5eA\nOG57Fx1vgLyZZXzVjE/jqhmfDswBvRd8gRLDUMaFpZk0YU3EvNn8cfJWTJFQ5rgeduWpp8kX86H/\nDhX8MSi6DS+TRmbVLH8Bwo7f5qyHogDeUGXAbS5yC3K2mA/iGJg7voWek2kimHpkeUfYDfbU67fU\nhZyMGWitT+G2Lx6DxaeMLy9XAkRrPyrmIFnGEonK33v6yJNRH68NxH5lWJqJq886ATd8bI7I+JXB\nm6+UeglKt/NyH8sILjgqmeWt6DbSyZBac0XOXA9+h6YpActbBpf7TOp+LH5PlndY9yQeo+eqcecd\nNwafPnsKmmuTqDQqA/uGldvFWekWfzEeN82vr1WkF7/O483SgqxdnVZ2PuL6lrdcIy7I3/W158e3\nV2H+lEY0pqtEaRBPaNOlhUtNrBo9+V7U1zOyZy/CU+YEdeCJ4oB4KmaOa0RTTQLfuuyowPa8W4Cl\nmaiuKF8EAUBRoyV7OUbeosENK9dzk53wFAfurkbs7M0hm7eF29RGHlrtdroAkcSBYixPQdHcMnf0\n8nW70DfAXMUa70jmBBZIwuOkuFTVLk5Jo6OizV9oen5M2iU2DCk8M5yXAQDqE3WoqYgJ1/1rXcuR\nJf2wJlBJUd/y1oU7l3sv7O0j4Q0wi5aR8zPbn4dLiBAb4QtX/syYY19FhtDx1+ktSFspGLoKj7e5\nZZamrdHjm5J1+OiiCWirpgaBYmVhjnsZWsUudk56n555lWaqb+rpFdfG3ea5IR1JI/heUHQbnsEs\nZ9tAf6YIS7NQCIl5wzGQLzq+qJLqoug6uH/NX+k1MhU7fo0i1q7QOm6xuJcaBdmOh60OFW5xu1tD\nLe+hrB2o8EjH6Hvig8eOwHc/PQ+WoUUx7wjDY97kJswaX4+vLZ4Vup1nVNdZ9Zg2uhZnL6Qu7GSM\nJWaFkTezvUvbBcqiJvKmM0Yvwk3zrw0mp5WgtjKGUc3h5M7JebiYOd8u9MdLkt7SJn0BKbqDdLyc\nvGXLu1RIxtBUv+SoBJwYZMtbLyFXTmAfnXQhxlaNwsiKkqYlAM6Y34HT5rXjU2dNLtv2hWlXBWr0\nwzL2E5rk1lc0jGzyCb+m+xhfI54n+kj3tEFvxw+O/ffgCT1VinlL99RgbnfPz3jXVBWfOmsK6lL+\nvXOKLAFLWrjUxqphezZ67R7qvuS92y3//GkjhYq0CsXTce6xNI9A04KLy135PliaiY+dOhFnzO8o\nmwvCwgCdPVRHnN8jTmI7MszqtC1s685Qy1tWA+WhBdvCSbOobn/CYG1Nx7wu+otzwn9pdZcIJ2ga\nK8nTnMACSSgPcnI26QJjRLoVlsmS+jwNHmhfe9uz0dMnJToNoxxYvXURLM2kSoeyVCk2+vMhW95C\n598RcyD2Y8f/c9OT6NPXQIlTD0ap5a1oHvQxNJuee5Fk8halWCo9vi5OibE6Sc9jtKyHVs3ugfQc\ncm8N11bo7suhM0sJ/Sf3vFn221fTPSC1G0CKFrxsBdZvH0BPn4OcUxAiLVzbgdgx5G0XGgwxxgIZ\nQme2G9PrpooWvdzyzhTZ74Qt1OIaj/v7mgeO6yGDXpCiBZJL0wx1fs/ZImkoZ0uuewOVTGggFgcq\nUxZMQ0XRdvdoYO0vhJs+EQ5ZWKaGK88tt644UkYS35p/HSrMVHhMOqS1J3/Zlbb+k6340pZ77wS+\n5R1O/pogb/riLiXvuJThHeY214gpvqd0gUHJV8XE+Cx0NFQFtnELf3duc+5Wntc8B/Oa54SOP27p\nuOD4saHbUrFYwPIPu0dJPQk4/vbqtH+9llcJe/0UWJOfB1GZFSBZhZahIaZbSOoJP8bo6dBUBTFL\ng9vTgqJuw+zw5SHh6mVd32TLyCkyYpeItyZO44V9hX5U6JXg7RsURUFh9UxY41+B7dlIWAZqrKQI\njWglnouck0M6UYdpo2sxbXQtHlqyEYWVR6FuyhoMEhazJ4Bjq+hoTQjPByfvXqaRbqoxbO/JImbq\nouc44MtbLpzSjounUNnadExanNWzpL1Aq1z6Ha45CKj1rCpAmmON11mzeD57oSeNhL/wJCo8uEhY\nOlzVLbG2gwsYTt5xRp7phBH4neaJn+XtDXBi8suYeLlVIJ9B+v+COgjVGoKXjwsPguyB4z9z/rsy\nNBW9/QRWo+/9Kap0DPUsVl2TKM/VCHw/I/8iyeP1tT348f8+g9gR6+EO1ACuIQiZQ6vugqIAxS3j\nAU9HvujCLChQzaLoC6ym+kEIdWsXii7i4HF5FzZT4RvoKxeEGsxnAZjQ0rS6okKvQhcQ0DywHQ8O\n8ZUCs3lb/K54eCJrboNVxWrfPRWVcbqIzjssVBUbgDbiDeSKp5XNzYFAZHm/B1EXrxk2mexblx9V\n9hmnZbIbgi61yt8JRFx+mFOqSnncXkZCcmvL7U55rJlnm4dZtfzlf3TNSTh7zKmh35PYjdv8ncLU\ntYALN8z7kNb9a7JUU7jhAZoMN29C0NqX3eomW4BUxZi1ThSAKNBUBcmYgfqqONydI6ES2UrSAhYz\nAD+jHxAvYpng5Xr+ilgSs8bXi0XloglzUUmakXcLyDm5gJiPripCHpQjVhL394aqMdM8xf/A1QEo\naK5Jipp8txict7pUBTp7cxjMFuF2t+KktuMB+PKWDekKUXWRNMt/G7LkbYJtH4qvg9FOFzky2QkJ\nYJ6Mp9iIaVbwuXV05L0cYpVDVMSmpLJDBifvGHvuUnED8o8jD2r1FlbPFORbEaf7ajXbpQ575RoO\nAPWsKUYRhCWr3fjxuThnQfnikqvrmYYq7jl3mxdUujyrZ5Z3Q0U5eZeqNxJPRc7JY9naHiEA43bS\nZ3XZup7gHHDPgOwV83QoCi+1I1CT/SC5FFRCyZ9b3takpbAVapWv3iDVkrt8AcF6rTdSQZZRFvOI\nSZZ10XHhoCg8BnLuBPds8GeBjRhxg3fQo+SdSa6F0bwBm3qD7UoPFCLyfp9hREMKN867BjcvuEF8\nxt08Lvt3XP+5uOGoqwPH7U/y5pa1S8KTO9QSy7s05t2UpOpN9bHaQO05fzlzyzuMGPnLP6xjG/+e\nZIjlffHJ4zB+RFVACW5fQL9fETHN/kJ5b2ceFgAASw+St6oquPC4oDu+QrIkeQ9s/pKloQefMK5f\nPBvzpjQibUrk7Oplcyxn3E8f2YxPnDEJx83wBXZ4xj9A5+vKc6dhFksMvPCEsRjZQL8/5+QDSXma\npsLZOg7FdVP9awxJ2otp/vg4cTbX+Za36ypI6v51N1VVwiME67cPQFVUnNi+AIBfTiff0zE1I/zs\neAauugbQOefQG1g3L4mYDFWjiyJmeXuqLeLoHPa2MSDwUGxaRj/gmvNmubdJMQsgRKEd5cDIW/N/\nG0VlqGwMHfXU82GOXAU11ReYJ/n7AKDIiJfY1LXb0ZTG+NagZgPgaxYQ4ru9eYJWERkYqiEWdfXp\n8pBYaSIeHAMFL49ETIfCeoDzkM+ytUHyVpmSn+w14Za8Yuap7oHmwstUwjI1arm7vuVcTG1mx0jN\nhQIxawIt3Y8RqTbUWLWB8Sqai6ydp78VtmjpzxQDx1tSCaCzg4Z3+D1/bPNTeHzz037mPQn3KO5v\nROT9PkRDog6VVvnKmbvGLSWFpmRDYFtIXtY+QxXkHX5SlcfaWcy71FoZXdmBzx7xcXxp9hXB47ik\nNCPv0pp3gFs1VIq0FMJtHmJ5nzJnBK67ZFawZn4foCgKrr5oBmbWURWz0sQdAEiYMVHrbGkmkjFd\nkLKmBUkLAJKmFONn19DMFjgcfOlVmbLw6bOmoDImddjytLJEx3qplOeoiW1YMK1ZzB2AwPOTCLsG\naQ5ly5vfS/klHUbeimv6nhNO3jVJcbzreUhLY2itoZ6ATN5BIqajwkoHqiHkMVZYaehvLkL+9YXi\nM/k+xIzdh5scj8BQLKiJQSjxAXhKOXl7fY1I6Wm4Zl/g+NLKCQFXF2JKybgRVC5TWaxXImdu9QF+\nXH+4RLiiTscwsq4ON1xKQz1hiZIJk96zwWxRsjqZ2xw5VJgp8ZzIz5x8DYCfsEk8DXllANvct0SN\nNo9Db9hRrtYG0JJD/qyLOTviaRH+IPkkYqaGfNHBUM6fI0/3NQ9Kx2N0rGJzRFATr0KMe5mE5e0K\nsR5O/rmCFPPWHDom1QOxTdibJtFxSc/tfW89KMJYils+twcCEXm/j/GlC6ejrT6JY46gJQ+cvGWC\nGsvqoxtq4uUn2EfwOHRYpjbgW+Ya++2kQmq5p9VNLluAcLe5RuiPKszyPml2G7568UyMaPDJ6+On\nTUR7QwpjWqk1EbS8939ayOSRNbjsiPNxycTzsajjxLLtluRaNzUqQiMatygKNFULvDiC5E3nroy8\nSxwnSSNoeZeiPu6Tt0zEHBVSA5rQBUiAvP2xcs+HHDoI08h3XCI8LPwl3taQFDK6iZiBasn676j3\n3fiJmA5VUZHQ/WssXfAYugqST6Hw5mwU101FXaU/3ngIecvEWSi6mKgfDUVzoTdtgIci4iELkLRR\nCaIEO7uVhif880slmpoaIG9P9ZOkOJRA1QDvXS/FsaUua45OiW/e+A7RwS6szDNlMNlbqVaeyt8S\n2MghLXljShd79BqY5rzJa8jpta8k//TVANl5t3aXS6USxwCIhoZqdi8kmeOWDno9Zx45HjGTajP0\nZf2ySRKKQnq/AAAbb0lEQVQbCJxfHo+i29CbN7BrTIjx8Xtijl6OdblV9CBXFzkAckKbodN7Egif\nlNxzy6I/suaaYEXJgUJE3u9jTBtdi29/4ihhhXLXuKym9qULp+Nri2dhTMv+eyC55T0cefPt6YSO\nb19+JCrfpqv6iDHUHdZWzVyKIdZFzNQxqaM68PI5dnoLbrr8SBh6iOUdco79AV3VcXTLkaFjNAzV\n713NSJeTN19Y8aoCUzUQtyTVOmF5S33R6ZkCf6UMyfIOJW/frZowysm7UnqRlxJj6THyNXLrkkus\nAggo1vE6esfxfCEPRkS1FTGcMa8DC6Y14XPnTEW15Sccjm32a/m5cE9ausbSaxC1+P31cLvbxLMD\nhJO3vMAp2C7aY+MB0HI7opAyyxsAKqQmL9yKa2sIL1NEqbWmlv825PvUW/Sbhvglf/52N6RxkRyO\naU+34YNjTgsQZMqS480avEwaWsUuaPVbQBQvcDwAjKroQEqpFp2+xjbV4rR57SK0YW8eL/bV0n1C\nOnU48FBGQzUTKUr6IaW8QRu3jKqvQ8LSkc07yG0ZAQyy6xS96/05mDLC9x7yTPWkkfS9H9K+y7LP\n0jE4BkawBQ4kt7mha9QL4egY1ZzGly+cXrbodPUcTM1EOvHOQmtvFxF5RxDglrfspo5bOsa1VQ13\nyD5B3UPMW1jm8IZ/2YXg46dNxFXnHYGFU6hs4R7lW4eBoRnCZbuv53gnMHVNvMy5vCTPOOfZ2jUx\npg6lKIhJ8WruNi9VsCq1vNvSkmBMyAtVJtQwy1te1ISRe3wYy1tkrEtWolySyMvRbMcT18jjoYqi\nIBEz8IkzJqO5NonqmL+gbK3zn9EjxtJrr5LIPVGywCgNxUwf689XmEuYuJqwVfNFF0nTAnE1EVMP\nI++URN7cyjv32NE4//gxZfsSV99ziZFENos6ThL/z/UQZOudZCqRe/HkQGy/QiJfRVGwqOMEJGzf\nQ5OWyRsKiutoAqLesLnseAC4es7nME+7UCwARjdX4YLjx4pyPrenFflXj5PGT3uNDwfujeGWt73N\nn6ch0Bh52kxhVEsFXI+gp9dD88CxwXOwRe8lp4zHF844DpW5CQAANUkt85SRRJznHYQtJFzd98rx\nksGqbrg1a2jioWvgklMmYOro2jLvQ09uV6gH5kAhIu8IAsfPpAlJs8aHtxvdXxhTORJAmHVIMb1u\nCh1P24K9Om/M1DFjXB3SZhKWZgaSrvYWPEZ6INzme4IpWd5claqmgvc7py8MTmxFtwhLiqNazHug\nqzqumvFptPfRspVSWhhd6ddUf2NxeQWCjHgIecsItbyHiXnLXp20Qq1dXv8L+Ja37bpoSNDnkBBg\nzsRgDgYAVPMFDILiOVzJri3V4o+x1PKWyPu4GS1oqfWvIWUl8K3510IfkhY4ro6PnEItyUVzR9De\n6LYpuqrFJaW9tnrqrm9IBpvoANSDcvq8Dnz9yC/jxBHHBM4vo3YoqONAXE2QCQBMrh+Lr5TkfJSF\nPzxdlNQBCP09aNLzLSc+AsBXPngs4GqC+NIhxxMoYlw8h+Wy0yYKWWRSjAnLXHZpA0DlllNwztgz\n/HOxPIiKBPME9jYF8hIA6k2Z2O7f95baCmiuf2/5d+SLDlRFRYfH+ruzkreUZHmHClY5BkYIqWMF\nbj99RovJbfQjVy+rfvHnItwDc6AQ1XlHEDh9XgcWTmt+227qfcWF4z+ICdVjMatxeuj2SbXj8b2F\n3wyWK+0FNFXDNXM+H3AN7y0Sehx9hf6DYnkbuubXm7JabRHzZqRTKxGX/DIxpSz6CTVjYbkZAD1l\n7D0i5WeOj24O96x8ceZnsHFwS5m7tBRhZYmyNR6WkAYA7eZErCg8g+aUb/0J8nY8zG2cgXV9GxDL\njMTZx00qO77GCo77psvmYiBbFHM1qmoEsCV8DHweZ42vx8dOnRgcu6WjLl6LOSNH47lupn3u6Zg/\npQknzaZCL6+s7mJSpdQzInsqvrZ4Nrr6cuhVN4nvT5oWvvrJo0TYoCXVhIUtR+GxzU/R87s6IHHC\nCGUatrxYg/icf9APSohGVYKJi6RE2lh87vj3Jox8PdZ61elpQoKFX266bC5Wb+7D5JE1UFdUwovv\nYseXPwfHTm/GP5dyOWBK3o01CXz90jlYuWEX/vPe10DsGHXtl1xDtVGHk9tnYkzlKNz94t+xsZMu\ndqaMqsH0TbUY0ZjCX5esByGK8C6kzBTGj/AXXifPbsPmt6rQyYVYuDgMlzw2LZCiJRZZKTOJmDK8\n5U1cHUdPbcIf/klbpBbfnIPYnL/BNQbE9pgxvOt/uGf9QCCyvCMIKIpywIkboC7Go5pn79aqTUuZ\nrfuCpmQjUua+kT9A3c5pI1VWc/5uwNJV1roRqGSJYaUxb9EUAcGyt9KSLz6DpIS9Dc3AhOqxGF9V\n7sLlGFc9Bie3Hzfsdo6w8IdcBx6WkAYAU5Nz8NkjPo6zR/v19kdNpkQ+sb0auqrjkknn47w5c0Q+\nggzZbQ4A7Y1pTB3lx65HVUqysiXPkio66ZW7qrkVP76+zf/Q1f0sZQCmqQUy5mXhoLilo70xHcgb\nOG56G1rqgs9jXPYGuDpOnOV/XzpBVdZ4YlmYlRhIOvR8adLW+iTOOYY1B5LqpsPCHzWDM+H2NqCt\nOE/McXtjGiczmVut6Lv+wyz3uso4KhL02r2S52DyyBpceMJYv1lKyTV091PCHVXZjqnG8aLne1NN\nAl+4YDqV2iWqyI8wVB2WZiIVN/CJMybhyxdOR3tjGpMa/C6AHz91Mlrrk2KR1TdUCCRHUstbCx0P\nAMAxkIobuP6js3Hy7DYACohtwVN5A52g5X3JxPOFpxAID58cKESWd4QIIfjY5ItQcIvvaAGxr9B1\nFW5nG2yjgCvPOx8AUJP21a+AYDKWXH5klpK3SJ0t/56rZn66/MO9wHnjzsIDax7CxJpxZdsaE37o\nZTjytkwd0+qCNevnHDMacyc2BKoBhgOPaZdm1nPwpD7e31lGW30SmzuHUFc1/MtWvi7iagGXv6Vr\nAZd02Eu7OdmI5mQjtmd2oqmy3LshW84nzhyB8SP8fToa0wAUmEocBZIVVutpR7ULyeOEEYcChS7M\nJCI6YWYrTpzVhhNnt+Hz/1WEmuqDSozQZ/mCo+bi0aWNWPyhCaFzoBerwIVd08N4ssZVjcZLna/5\n1QESqtIWyFZekkUt81HNVP50guT+lhdn3PuSZImH3kAN1FgWtudn4C+QmvzMbZyJf215BgBNPj12\nuh8uaapJYHl3HGqKJr8ljSRivN+BY+L85o/h3rcegJryLWuAVtmMba3EkZMace+W5diapS4U4hgB\nsaKjW47ElNqJeK2b9q1/N2PeEXlHiBACUzOHVak70DCYhKuzdRzqE9R6a29M4ZxjRokOaTweXB+v\nDVjbZoj4DDCsmN07wokjjgnGbSXIRCFaNZYgbKyqqgTaq+4Opmbg5gXfGHZxAAC5l06iL9sPBD//\n6AcmoK0hxayr0rHTfyulxjtfuXB2YB/TUAPkHQ/pLqcoCq6Z83m81rUC0+unlm3XVA03zbsWf1n3\nMOY1B88/aSQlNiPXgEJsA5QEJRdNU8X9VhUVcT2GrJMrkTv1O7DpJIbCiqNRlQ4nlTEtlbjinOHl\nluOZDgwZW2FU9pZpP3AsnnQBptVNxuyQMFgypkvtR6llfvyMFpxz7KhABYvrlmfX88WS21/ni+WE\nYGTFCDQmGtCSKs+h+dAxo5B5eRJeGKKqZykjAVPKjxhd3Q4vlxbkXZr1P7atEvW91YK8DcUs03pI\nmynoqg7HcyLLO0KE9zOSMQOXfmBCwPpUFAVnLRgl/q6NV+PauVehxqoWtdMAy1QPwbvUKyGARR0n\n4G8bH0d7upwggXIvwb6gcpjOdhy3f+kUhDlPYqaO044qb4RSio9Nvggv7HwFExpaA5/HTC0QTx7u\npW1qJuY2zRz2/PWJWnxy6uKyz6tSFlrrkujcUgN97Aa/R3XJjeTiIh111Th/8Ww8+vwmHD2Fkpii\nKHBcD4CCUU27n6fhoCkGiqtnY+KY6mFzH3Z3jRPaq1D7Vgp96BYKZJapBcIbANA7SGPSFSG9Cnij\nkXFVo8u2AfQ6bzjqK6GeBUPXcMmcU/DCE4/T79aswH6WqQXaKB83pTyMJHdPTJrhXRJrrCp05roD\nuQ8HGhF5R4hwCIJn/u8OYaSol3Tt8t9T7z57nz36VBzVNCvUnQr4mfEHEqX913eHay6eiQeeWheY\n+yObZuHIpvIOfg3VCZw4bQyeHqB61/EDYHGdd/wY/PTPWRQ3TII3SC3x0kXYpJrxWLVrNU4ffRLG\n1ldibFu4FT1hxL6Ve/L5U/YxPUpTVZw//Rj8YvkGuF10XsMWmJUpujiZPKqmbBscE1+c/CWMqKsu\n38awu/CWoer4ztHXI+fky/bTVQWktxVesg+jnWNw6YfKEyPlBWKlVU7eAK3+6Mx1v6sJaxF5R4jw\nHkLpy4n/fRAMbyiKMixxA8O7+A8WJnZU42sds/e8I8MHpx8JrO2GSzyMZuWP+xMzxtZhbGsVVm30\nPQSliYeXT/kICm4xkMAYhpHDtOfdEy47bRLufvRNXHRSeV7D28XMhmn45lFfxdeefx1AeF+Bs44e\nicqkhWOOaA58fvnpk/Dy6i6MaWh6R9LE1bEqhFG/rqtQMrUoLF+A6qnhYYFKSU2wKhHufeClm2Hh\nkwOFiLwjRHg/4GCw9x5Qmhl/uCFhxHHxxPMO6Hc01iSwamOv+LvU8k4YiVBteY5vXDoHb27uxbi2\nfVNIbKlL4tpLyj0Pe4vGZD14NnxYuMTQNZEhLmPhEc1YWELo+xO6poqSjPgwuvNT6yahVZuADVuK\n+MAJ4eWtnLwjt3mECBH2Cj/43AK4XnnSz26SzQ869kfM+72OxupgedecCeHW4XAY3VKB0S37ZnUf\nKBxKizZFAbhBP5znPWkkcN2xl9Me4lY4ZY6pot6RltSBW2iUIiLvCBHeA6geJpt4yqgavPRmF2aN\nrwvdfjBxqLnND0U0VvtW9e1XH79XMfxDFeYwCmXvJj555iS8sbGPlX3tObSkKsqwxA0A46vH4gfH\n/ntkeUeIEGH/4NjpLRjVVPG26qbfbZjvASI60JgyqhqTOqpx9NSm9wRxA76lezBx9NRmHD2VWsn7\nK6fz3SRuICLvCBHe01AVBR1N+67xfiDwqbMmY/32gVDVtAhBGLqGr148fKnZ4YTT53Xg2eXbUfUu\nqDjuDfzQ0qEYXBoeB3Qpd/PNN+PDH/4wLrroIrz++uuBbc8++yzOP/98fPjDH8Z///d/H8hhRIgQ\n4RDC/ClN+MjJ4/e8Y4T3FM4/fgx+eOXCQBOZQwEXn0wz6WVltsMBB8zyfv7557Fx40bcc889WLt2\nLa6//nrcc889Yvt3vvMd3HnnnWhsbMTixYvxgQ98AGPHjj1Qw4kQIUKECBHKILvQDyccsCXQkiVL\ncPLJJwMAxowZg/7+fgwNDQEANm/ejMrKSjQ3N0NVVRx33HFYsmTJgRpKhAgRIkSI8J7CAbO8u7u7\nMWWK322lpqYGXV1dSKVS6OrqQk1NTWDb5s2bd3u+6uoE9P0cI6uvP7RigYcronl854jm8J0jmsP9\ng2ge3znejTl81xLWSjV59xa9vdn9NBKK+vo0uroG9+s534+I5vGdI5rDd45oDvcPonl859jfczjc\nQuCAuc0bGhrQ3d0t/u7s7ER9fX3otp07d6KhYe/EByJEiBAhQoT3Kw4YeS9YsACPPvooAGDFihVo\naGhAKkVrTdva2jA0NIQtW7bAcRw8/vjjWLBgwYEaSoQIESJEiPCewgFzm8+aNQtTpkzBRRddBEVR\ncOONN+L+++9HOp3GKaecgptuuglf+cpXAACnn346Ro0atYczRogQIUKECBEAQCHvNBj9LmF/x2Gi\n2M7+QTSP7xzRHL5zRHO4fxDN4zvHYR/zjhAhQoQIESIcGETkHSFChAgRIhxmiMg7QoQIESJEOMwQ\nkXeECBEiRIhwmCEi7wgRIkSIEOEww2GTbR4hQoQIESJEoIgs7wgRIkSIEOEwQ0TeESJEiBAhwmGG\niLwjRIgQIUKEwwwReUeIECFChAiHGSLyjhAhQoQIEQ4zROQdIUKECBEiHGY4YF3FDmXcfPPNeO21\n16AoCq6//nocccQRB3tIhzRWr16NK664Ah//+MexePFibN++Hddccw1c10V9fT3+4z/+A6Zp4sEH\nH8RvfvMbqKqKCy+8EBdccMHBHvohg1tuuQUvvfQSHMfBZz7zGUybNi2aw71ALpfDddddh56eHhQK\nBVxxxRWYOHFiNIf7iHw+jzPPPBNXXHEF5s+fH83jXmDp0qX4whe+gHHjxgEAxo8fj09+8pPv/hyS\n9xmWLl1KPv3pTxNCCFmzZg258MILD/KIDm1kMhmyePFi8o1vfIPcfffdhBBCrrvuOvJ///d/hBBC\nfvCDH5Df/va3JJPJkEWLFpGBgQGSy+XIGWecQXp7ew/m0A8ZLFmyhHzyk58khBCya9cuctxxx0Vz\nuJd46KGHyB133EEIIWTLli1k0aJF0Ry+A/zwhz8k5557LvnTn/4UzeNe4rnnniOf//znA58djDl8\n37nNlyxZgpNPPhkAMGbMGPT392NoaOggj+rQhWma+PnPf46Ghgbx2dKlS3HSSScBAE444QQsWbIE\nr732GqZNm4Z0Oo1YLIZZs2bh5ZdfPljDPqQwd+5c/PjHPwYAVFRUIJfLRXO4lzj99NPxqU99CgCw\nfft2NDY2RnO4j1i7di3WrFmD448/HkD0e94fOBhz+L4j7+7ublRXV4u/a2pq0NXVdRBHdGhD13XE\nYrHAZ7lcDqZpAgBqa2vR1dWF7u5u1NTUiH2iefWhaRoSiQQA4L777sOxxx4bzeE+4qKLLsLVV1+N\n66+/PprDfcT3v/99XHfddeLvaB73HmvWrMFnP/tZXHzxxXjmmWcOyhy+L2PeMkikDvuOMNz8RfNa\njn/84x+477778Mtf/hKLFi0Sn0dz+Pbxhz/8AatWrcJXv/rVwPxEc/j28Oc//xkzZszAiBEjQrdH\n87hnjBw5EldeeSVOO+00bN68GZdeeilc1xXb3605fN+Rd0NDA7q7u8XfnZ2dqK+vP4gjOvyQSCSQ\nz+cRi8Wwc+dONDQ0hM7rjBkzDuIoDy089dRT+NnPfoZf/OIXSKfT0RzuJZYvX47a2lo0Nzdj0qRJ\ncF0XyWQymsO9xBNPPIHNmzfjiSeewI4dO2CaZvQs7iUaGxtx+umnAwDa29tRV1eHZcuWvetz+L5z\nmy9YsACPPvooAGDFihVoaGhAKpU6yKM6vHD00UeLOfzb3/6GY445BtOnT8eyZcswMDCATCaDl19+\nGXPmzDnIIz00MDg4iFtuuQW33347qqqqAERzuLd48cUX8ctf/hIADX1ls9loDvcBP/rRj/CnP/0J\n9957Ly644AJcccUV0TzuJR588EHceeedAICuri709PTg3HPPfdfn8H3ZVezWW2/Fiy++CEVRcOON\nN2LixIkHe0iHLJYvX47vf//72Lp1K3RdR2NjI2699VZcd911KBQKaGlpwXe/+10YhoFHHnkEd955\nJxRFweLFi3H22Wcf7OEfErjnnntw2223YdSoUeKz733ve/jGN74RzeHbRD6fx9e//nVs374d+Xwe\nV155JaZOnYprr702msN9xG233YbW1lYsXLgwmse9wNDQEK6++moMDAzAtm1ceeWVmDRp0rs+h+9L\n8o4QIUKECBEOZ7zv3OYRIkSIECHC4Y6IvCNEiBAhQoTDDBF5R4gQIUKECIcZIvKOECFChAgRDjNE\n5B0hQoQIESIcZnjfibREiHC44ZZbbsGyZctQKBSwcuVKzJw5EwBw3nnn4UMf+tDbOscdd9yB8ePH\nCz3rMHz0ox/Fr3/9a2iatj+GHcDOnTuxbt06zJ8/f7+fO0KE9yOiUrEIEQ4TbNmyBR/5yEfw5JNP\nHuyh7DUefPBBrF27Fl/60pcO9lAiRHhPILK8I0Q4jHHbbbdhy5Yt2LZtG6699lrk83nceuutME0T\n+XweN954I6ZMmYLrrrsOs2fPxvz58/Fv//ZvWLhwIV5//XVkMhncfvvtaGxsxIQJE7BixQr89Kc/\nRV9fH3bs2IGNGzfiqKOOwg033IBCoYBrr70WW7duRVNTEzRNw4IFCwI9ijOZDL7yla9gYGAAjuPg\nhBNOwJlnnokf/ehHIISgqqoKl1xyCb797W9j48aNyGQyOPPMM3H55Zfj/vvvx9///ncoioKdO3di\n9OjRuPnmm2EYxkGc4QgRDk1EMe8IEQ5zbNmyBXfddRemTp2Kvr4+3HTTTbjrrrtw6aWX4vbbby/b\nf+3atTj33HPx29/+FpMmTcLDDz9cts/KlSvxk5/8BPfddx/uv/9+9Pf348EHH4TjOPjjH/+Ib37z\nm3jmmWfKjnv22WfhOA5+97vf4Q9/+AMSiQRaW1txzjnn4Oyzz8Zll12Gu+66Cw0NDbj77rvxxz/+\nEQ899BDeeOMNAMCyZctw66234r777sO2bdsOSy9DhAjvBiLLO0KEwxzTp0+HoigAgLq6Otxyyy0o\nFAoYHBxEZWVl2f7V1dUYN24cAKClpQV9fX1l+8yePRuapkHTNFRXV6O/vx+rVq3CkUceCQCor6/H\n7Nmzy46bNWsWfvKTn+ALX/gCjjvuOFxwwQVQ1aCNsHTpUuzYsQMvvPACAKBYLGLTpk3ieN4+debM\nmVi7dq3okxwhQgQfEXlHiHCYQ3YrX3PNNfjWt76F+fPn4/HHHxfNPGSUJqSFpb2E7eN5XoCIS0kZ\noL2M//KXv+CVV17BP//5T5x33nl44IEHAvuYponPfe5zOPXUUwOf33///fA8b7fjihAhAkXkNo8Q\n4T2E7u5ujBs3Dq7r4pFHHkGxWNxv5x49ejReeeUVAEBPTw9eeun/t3eHOAoDYRTHHyGYJlwAMAjg\nAFROSC0STCWCIJCYBhwOwxEqegIkuqLBbRN0LQaBxkBZsdkaDJutmeb/05PJ517eZCbz9bYmSRLF\ncazhcKggCOQ4jm63m2q1mh6Ph6SfVv97VJ/nuXa7XdH+z+ez7ve7Xq+X0jTVYDAobX6gSmjeQIUs\nFgvNZjO1Wi3N53MFQaAoikrZezqdKo5j+b6vTqcj13XfGnq329V6vVYYhqrX6zLGqN1uy3VdrVYr\nNRoNLZdLZVkm3/f1fD7leV7xVWq/39dms9HlclGv15MxppTZgarhqRiAj1yvV6VpqvF4rDzPNZlM\ntN1ui3fn/3U4HHQ6nbTf70vZD6gymjeAjzSbTR2Px+J/4tFoVFpwA/gbmjcAAJbhwhoAAJYhvAEA\nsAzhDQCAZQhvAAAsQ3gDAGAZwhsAAMt8AxJ5C+54P8QOAAAAAElFTkSuQmCC\n",
+            "text/plain": [
+              "<matplotlib.figure.Figure at 0x7f72fab5e290>"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAe8AAAFnCAYAAACPasF4AAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAIABJREFUeJzsvXe8XVWZ///e5dTba3pCQiAJCSWE\nIJGmoSSgjsg4gmCb4Tf+dCwURUdEQXGs41gYFQvDiIyIiKIIJIAgEBJCgJBKertpt59z76m7fv9Y\nu55zboiQBCL783rllXt2WXvttfden6et55Fs27aJECFChAgRIhw1kF/vDkSIECFChAgR/jZE5B0h\nQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8IESJEiBDhKENE3hEiRIgQIcJRhoi8I7yp\nMW3aND796U9Xbf/iF7/ItGnTQsfdcMMNoWOWL1/OBz/4QQB2797NCSec4O3btWsXH/vYx1iwYAEL\nFizgkksu4bHHHgPgpptuYuHChSxcuJCZM2fy9re/3fudy+VC19A0jfvvv/9vvq/Vq1dz1VVXHdSx\nDzzwAF/72tde9bVcvNbz3wi46667+P73v/96dyNChFeE+np3IEKE1xsbN24kl8tRX18PCBJas2ZN\n1XErVqxg/fr1IZIeCZ/97Gd597vfzW233QbAqlWr+PCHP8zDDz/MV77yFe+4+fPn8+1vf5vTTjut\nZjvr16/n/vvv55JLLvmb7umkk07i9ttvP6hjly5dyvnnn/+qr+XitZ7/RsAHPvCB17sLESIcFCLN\nO8KbHm95y1t49NFHvd9LlizhxBNPrDruuuuu4+tf//pBtblp0yZOPvlk7/fJJ5/M4sWLGT169EH3\nq6+vj09+8pO89NJLXHHFFYCwAPz0pz9lwYIFmKbJypUrufTSS1m4cCEXX3wxS5cuBYRV4IILLgDg\n1ltv5atf/Sqf+MQnOO+883jve99LT0+Pd53ly5czffr0qmu98MIL/OM//iMXXHAB73vf++jq6gKg\nu7ubD3/4w1x88cWcf/75fO9736vZ18p7ueqqq1i4cCHz58/njjvu8PatXbuWSy+9lAULFvCBD3zA\nu85I26dNm8b+/fu9893fy5cv5/LLL+fqq6/mM5/5DAD33nsvF110ERdeeCFXXnkle/bsAcC2bb7x\njW8wf/58FixYwC9+8QtvrL74xS8CsH///pD15MknnwTAMAy++MUvsmDBAi644AI++clPVllMIkQ4\n3IjIO8KbHhdddBF//vOfvd8PPvggCxcurHmcbdssWrToFds855xz+PSnP82dd97J1q1bARg1ahSS\nJB10v9rb27nuuus45ZRT+PWvf+1tt22bxYsXoygKX/7yl7nqqqtYtGgRH/3oR7nppptqtrVo0SJu\nuOEGHnvsMdra2rjvvvsA2Lp1Kx0dHYwbNy50rVwux8c//nGuu+46Hn30UT70oQ9x9dVXA/C///u/\nzJ07l4ceeogHHniArq4uLMuq2VcXP/nJTxg/fjyLFi3il7/8Jd/97nfZt28fIISiq6++msWLF3P+\n+edzyy23HHD7gbB+/Xouv/xyvvvd79Lf389Xv/pV7rjjDh555BEmTpzIj3/8YwD+9Kc/sXr1ahYv\nXsx9993HXXfdxerVq0Ntff7zn2f69OksXryYn/3sZ3zuc59jcHCQJUuWsHv3bhYtWsQjjzzC1KlT\nWbly5Sv2LUKEQ4mIvCO86XH66aezefNm+vv7KRaLrFy5knnz5tU89oYbbuA///M/KZfLB2zzO9/5\nDldeeSUPPPAA73znO5k/fz533333Ienv2972Nu/v+++/n4suugiAOXPmeNppJU477TTGjRuHJEnM\nmDHDI85ly5bVvNcXXniBUaNGceaZZwLwzne+k127drF3717a2tpYsmQJzz//PPF4nP/6r/+is7Pz\ngH2+8cYb+dKXvgTAhAkT6OjoYPfu3Wzfvp3BwUHOPfdcQJitb7311hG3vxKSyaR3P21tbbzwwgue\nteO0007zxuepp55iwYIFxGIx6uvreeihh0LWlkKhwPLly/nIRz4CwKRJk5gzZw5PPvkkra2tbN26\nlUcffZRiscg111zD2Wef/Yp9ixDhUCLyeUd400NRFC688EIefvhhWltbOeuss1DV2p/GzJkzmTt3\nLnfccQezZ88esc1EIsFVV13FVVddxdDQEIsWLeLrX/8648ePf80TfXNzs/f3Aw88wJ133kk+n8ey\nLEYqVdDQ0OD9rSgKpmkC8Mwzz3gEFcTQ0BBdXV0hC0Q8HmdgYICPfOQjWJbFV77yFXp6erjyyiv5\n1Kc+dcA+r1mzxtO2ZVmmt7cXy7IYHBwM9U1VVVRVHXH7K6Gpqcn72zRNfvjDH/L4449jmib5fJ7J\nkycDMDg4SGNjo3dsOp0OtTM8PIxt21x++eXetkKhwBlnnMFJJ53EjTfeyK9+9Ss+//nPM3/+fG66\n6aZQexEiHG5E5B0hAnDxxRfzve99j5aWlpo+2yCuvfZaLr30UsaPH19z/8DAAC+//LKntTY2NvK+\n972Pp59+mk2bNh0yLa27u5sbb7yRe++9lxkzZrBjxw4WLFhw0OcbhsGaNWtqCiGdnZ1MmTKF3//+\n9zXP/ehHP8pHP/pRtm/fzr/+678yZ86cA17r+uuv58Mf/jDvf//7kSTJG4OWlhYymQyWZSHLMrqu\n093dPeL28ePHI8uyJ3xks9kRr/nQQw/x+OOPc9ddd9Ha2spvf/tbHnjgAe+6g4OD3rF9fX0kk0nv\nd1tbG4qicN9991FXV1fVtrs6IJPJcMMNN3D77bdz7bXXHnAMIkQ4lIjM5hEiALNnz6anp4fNmzdz\n+umnH/DYzs5OrrzyyhHNuKVSiU9/+tM8/fTT3radO3eyatWqEaPKR4KqquRyuZoa9cDAAOl0milT\npmAYBvfccw8A+Xz+oNpevXo106ZNIx6PV13r5JNPpre3l1WrVgHQ1dXF9ddfj23bfPnLX+aZZ54B\nYOLEibS3tyNJ0gH72t/fz6xZs5AkiT/84Q8Ui0UKhQLHHHMMo0eP5pFHHgHgd7/7HV/+8pdH3A7Q\n0dHBhg0bALjvvvuQ5drTWH9/P+PGjaO1tZXBwUEefvhhb2zmz5/Pgw8+iKZpFAoFrrjiCjZt2hQa\n93PPPZff/OY3ABSLRb7whS+wb98+7rvvPn70ox8BwgoyZcqUgxrvCBEOJSLyjhABkCSJCy64gLe+\n9a0jkkEQ//Iv/4Ku6zX3jR07lp/85CdeVPiFF17Itddeyxe+8IVQBPrBYM6cOfT09HD22Wd72qaL\n6dOnc84557BgwQIuu+wy5s+fzymnnOKtPX8lLF26NOTvDl4rFovxwx/+kFtuuYWLLrqIT3ziEyxc\nuBBJkrj88sv53ve+50W4z549m3nz5h2wr1dffTWf+MQneNe73kWhUOCyyy7jS1/6El1dXfzgBz/g\ntttu48ILL+TPf/4zN998M5Ik1dwOwvJx88038+53v5tUKuUt8avEO9/5TjKZDBdccAGf+cxnuOaa\na9i/fz/f/OY3ufjiiznrrLO48MILec973sN73/teTj311ND5N998MytWrGDhwoW85z3vYcKECYwZ\nM4bzzjuPdevWceGFF3LRRRexZcsW/vmf//mgxjxChEMFKarnHSFChAgRIhxdiDTvCBEiRIgQ4ShD\nRN4RIkSIECHCUYaIvCNEiBAhQoSjDBF5R4gQIUKECEcZIvKOECFChAgRjjIcNUlaenuHD2l7LS1p\nBgcLh7TNNyOicXztiMbwtSMaw0ODaBxfOw71GHZ0NNTc/qbVvFVVeb278HeBaBxfO6IxfO2IxvDQ\nIBrH144jNYZvWvKOECFChAgRjlZE5B0hQoQIESIcZYjIO0KECBEiRDjKEJF3hAgRIkSIcJQhIu8I\nESJEiBDhKENE3hEiRIgQIcJRhoi8I0SIECFChKMMEXlHiBAhQoQIRxkOK3lv2rSJ888/n7vuuqtq\n39KlS3nve9/LZZddxo9+9KPD2Y0IESJEiBDh7wqHjbwLhQK33HIL8+bNq7n/a1/7Grfeeit33303\nzzzzDFu2bDlcXYkQIUKECBH+rnDYyDsej/Pzn/+czs7Oqn1dXV00NTUxZswYZFnm3HPPZdmyZYer\nKxEivGmhGxZL1+6jWDZe76542NuXZ822/te7G0cNXtjYy879wyxduw/Lsl/v7rxq9GWKrN8x8Hp3\nA4D9AwVWbekDoKyZPPdyN7Y98tjmSzovbOw54DFHGoetMImqqqhq7eZ7e3tpbW31fre2ttLV1XXA\n9lpa0oc8Z+xICd8j/G2IxvG143CN4d2PbOTXizdw3twc11x+6mG5xt+Kf/nm4wDc/+13oSiHTn/4\ne3wP9/Tm+NEf1ni/48k4F8075rBe83CNo/vcf3XzQpobEoflGn9rX+79+jv4+d0vsmzNPmRV4aK3\nTq55/I9/8SzPv9zNdVecytvnTHjF9o/Eu3jUVBU71JVuOjoaDnmlsjcjonF87TicY7hhu9BwN+wY\neMM9p737syTjh2YK+nt9D7dWaKobt/dz2tS2w3a9IzGOXXsz6K3pw3qNg0V3zzArN/YAsGnnAKcd\n117zuA3Oc3hh/X5mTWw+YJuHegzfUFXFOjs76evr8353d3fXNK9HiBDhtcE180lIr3NPqqEZ1uvd\nhTc8SroZ+m2aR/+YvZFcOJZtY5jiG1EPYAVqrheWgsHh8hHp18HgdSHv8ePHk8vl2L17N4Zh8MQT\nT3DmmWe+Hl2JEOHvGq6LTnrjcTdGRN6viHIFeRtHsc/bRb6kv95d8GBaticQqcrIH0mLY+bP5N44\n5H3YzOZr167lW9/6Fnv27EFVVRYvXsz8+fMZP348F1xwATfffDOf+cxnALj44ouZPLm2ryFChAiv\nHW9E8tYj8n5FaHp4jEzz6CfvQun11byDQWeWZeP+UuSRddn6VAyAzAE072x5iKZE4yHp48HgsJH3\nrFmz+NWvfjXi/rlz53LPPfccrstHiPCGwf6BAo3pOOmk+Nx6MkXSCdWbEGqhe6BAQzpGOukf0z1Y\noLk+QSJWHbiZzZUxLZvWxmRou+Wazd+A7H0kzOYDQyUUWaKp/rUHSFm2TVd3jgmj6pEliZ7BAk11\nCRLx8PMoayZ9QyXGtde9pusVSjq7e3OhbYPDJbJ5jaa6uLetN1MkGVdoSMcrm6BYNtiyJ8u49rqq\ndwOEANWXLTKmrbqvA0Ml4jGF/mzJu+dK2LZNV0+Ose11ntnZtm329OUZ116H5IxTXeBdz5cM9vbl\n6WxJeedYts2W3VniMZljRjfSkynSmI6FYiJ2dQ8zpq2OmFqbZGudUwslzbdmmJZV9bdl2WzenSGV\nUEknVOJxxfuOhgo6fZkidakYqYR/naV7V/B/G+7lIye8n4s7zjng9Q8VjpqAtQgRjkaUNZMbfvYs\njXVxvv+pswD499uWIQG3//v8mufohsXNd6xg9nHtfPQfZgLQny1x48+X8455k7jk7ClV51z7388A\n8D8VbbpKhvw6cLdpmWzL7mBq85SawsOR0Lw/++OlQPW4vBosfm4X9z6xlcvPO45Tj2vn33/6LBOm\nZ7A7N3LdnH+jOdEEwDf/70V2dg/z7Y/No7059aqvd/MdK+jLlkLbNuzKcO2tS0L3c8Mf7sYup/nF\nxy6vauOexzezZNdKmuoVvnvlZVX7n1i5h3v+spmvXHU64zvqve2mZXljB3DlBcdz3pzxVeev2TbA\n9+9dxZknjuaqd5wAwOMv7uH/Ht3E+88/jtOmdfLvP32W9iZfcFi5uZdfLd7I208dxwcvnAbAqi19\n3HqfiKq//v2z+c7dK5k+sZnPXSFWSGzcNci3fr2SudM7+fgls6r6kc2V+ffbljF1XBM3fHBOjdH0\nEdT8g0vvXFJ/YVMvP7l/rbddSuZomLUSuXkaVqaTz922jHHtddzy/73FO+avu5cAsKJ7JRefeGTI\nO0qPGiHCYYRmiAlhKK8BYDj+tQMZP4tlg7Juhvxre/pymJb9N/vcfBPhkWfvezf/ie+v/CkrulfW\n3K8bZs3tbyT8cevDfGHJLWimxsrNIsh21ZY+9vYXIFair/FZ+kuDdA3v8c7Z2S0ijQcOMrhJt2qb\nkSuJuxZ2De0hPmkDieNfrLm/qzdH4riXKI15ofY1MkVsoKsnrOFXmrbXjrAuf9veLADPrNnvbXtx\nUy8AK17uYSivIaWHyE+7H7mpx2lLRG4/8aI/Zv2Be3VzAGzYlfHb3LsRKV5gxYaemv3oHiwCsGVP\ntuZ+0zIxLfG+BX3uZoC8NSe+oC9bDJ0rN/Wjy3kxxoo4d09fPnRM2RDPOqkcuSVwEXlHiHAYURlf\ndDDapghSssnGtzGsiUm1NyMmt6DPc1XvWnoLB0524pL366F5P71HJF7al+/2tlkBf+PR4PN+ZOcT\nDGnD9BbD45zJlVEa/WVcRaOaaJUDBEC52DW8m2v+egNL9jxb+wDJIjZlNXJzd2iz+1yf6FpywPZ7\nC/6qHsuuHm83IK43EyasSvIeyVRdy6Li3rdhWpiWTWzsVtHGpA0j9jN4vf4KoaW/OMCSwu+Jz3hu\nxPMrz6nEf734E7723HeBcLR7Tisi1WWIH/8Cw8ZwVV8AJFXz/pbragsHZVMck1CqXReHCxF5R4hw\nGFG5tEc/iKU+Zd1Ead9LpvU5/mfdrwF/cnU1hf35Hn625k7+47n/8jQGoCoDl6d315hkNw5sYU3f\n+oO+l1eLxri/TjVI2G/0pWLuhAygW+EI6d5MESnuE0ZBD5MfgHIQEtOjO/8KwIPbH625X2ndj9q+\nl8TxYeuFu7xpW3YnALZR7QEtaQZFxSfvWn0cibzzAQKT6rIMpNbXJH+5xj2qTuCXYdqifcVpq0Yf\na12vp6Ivq3qFCVtOjEzQwf6XtDD5dg3vYcfQLnoKfWim7l9L0fn+y98mOfNZlOZetiUfreoLgBQr\nB/7WqIWyeeSj0CPyjvC64I2UZvBwwqwgU10/OPKWG4RWtze3D/AnJ3epUG9RTMq6pYcmm0rh4EBW\n8x++9DNuW/2/r9ifkfBiz2q+8dz3KRrVpBCc6IPEFyTv16p5W7bFrSt/7hFg9X4bpW0vcsv+mvtf\nCbuGdnt/l4zw5NybKSLFAuRtVCeRMi2bIW2Y32/+MzktX7U/eI2JDeOq+g4gN4nnbNvhB1jWTWzb\nJlN2TMuKUUWufZkScr2vKeZr9NGNZu/LhImx4JiWpbosyZnL2Bd/gd3De6vOryWfuEuuTMuirJtI\nqmjLNqsDNF3BsxAwZe9zTNKphAgEfMkhb9saWRjakFvtPefKe1m+z3cZ5PScZzavJGJNyZLT86G+\nACEhDTV8zu83/5lfrL0LzXnHa1lgDhci8n6TwPUvWrZdMYGaNY97pW2vBat61/LJJz7PtuyOmvst\n2+Kl3rUU9dIBr23VEAAOVV9/uf433LT0m6/6fLcfQfI2TCtEriMJMJpmIsUFIbYmW4Cg2VycP1jy\n/YHByaaSED2zeeC3bdvkdJ9MKv3ouiGIwbKtmtqWi9vX3sXu3F5W9673zgHxXILm/JAGG+hfppzh\nzvX30F3oxbQsBofL2LZd9QxFX6rHqrfYz4bBzdy/9aGq4yzbxjBMYsesIzaxtrm2ss18SQ+ZTHcN\n++RdOSn3ZkpIcX/cCjUEGNO0+c7z/81fup7imb3Lvevphskftz7MZ578Mn0lIaTJkh+xXiwb7O4f\nRG7sQ2l0yLscDnzTdJMhLYdhi/5KEhR1v49lzWT7/iGkhE/Y+RoCxLDdR+yYtfQMD4W254o6iZlL\nSc70a04MlAZDxzyzdzm7zZeRm3tInb6IPbl92LbtRZAXk7tZMfCMr3lXQB2zleuXfImCXgwJoK5F\nJp2IYds2e/Ou8CXh2pJ0w2JgqESuqDNYyrIz/gyJ414C/JgDwzK4e+PveaFntX9fWt5/xnJ1v7Kl\noWqzeby25m1ZNn/peoqVgfaPJHlH0eZvAnQPFvjCT5/l4jMm8fLOQbbvG+J//n0+Dy7bwX1PbuPm\nf57LxFEN/PWlPdy5aCPXv382MyYJ0vjNXzbzyIouvvWxeXS8hsjZIO7fIibbv3Y9w5SmY6r2L937\nHHdv/D2N+jF0r5zOj649J7QsA2BTV4Zv/t+LfPySWcydLrLzPfp8F3c/tpkbPjgHPdVNS6KZ0XWv\nLnPfc/tFAJBhGajy3/aZvLxzkO/cvZIPLZzGceP9VIr5khEycRumTUyt1iZKuomUEGSwfVeZNW39\nXhCNaxbvCfgyQ5p3gBwf2LaY4eQQ0IYkSZR1k49/90nOOGEU557lB9Zc96On+MAFM5h/6niG8hrX\n3LqEc04eiz3xRV7u38R/nHUjsQOMgaZb/P//+SRnnTiGf3nHDD73k6VkpN0kRCBxyKToE7PFnwZv\nB6A50cSG5Z1s2JUhEVcoayafuewUZk5u5cWe1WzdrPDw091V0dvd+XDw0n/d8xLb9g3xb+85ke/+\n5iWuuGgikmKCbGFaJorsE+SqLX384Herue59JzNrShvb9w1xyy+fB+CbH5tHZ3OKVV27vON/8dBq\nxqnT/WsnXkJp9f3QtUzSmfKgR3j3P72de34Dn79iNt/69UpSpz8ROvalbfvZ2TlMc32cz922DOnY\n5SSm+wKQVKHxLVu3n98/v5LkTH9btpynLp6mpBlc/+Ol5EsGiVk+mQxr1Zp3X/ol1NR+8oqBbvhR\n0kPlAnJdmNAHy2F/76833AdAfLLQqH+y5EFSPafQ0SKeUXncc7yUAzlZ+x5iEzZj2LBpcAu5cnXf\nDNPi9kWrKDrmckm2QBZC4pduX06PE6TWcEwXeJ+5ze0PvsykUQ3stzZXxRIM6znyJdFfqYZQ8Z3f\nPUeb7FtB5MZ+5PQw2BJIdugeilr1+bWEuMOFSPN+E2CjE7X50LM72b5PfJCWbXPfk9sAPzr0waXC\nf7Z07T7v3EdWiIIxm7p8Te+1wtXmRlp77EbuZhFmuv6hamn2ryvFMfc+4ZeSve+vIjBm+cbd/PdL\nv+CW5f/5mvsa1BoPFktWi34/tGxnyOddKOkhzbsye5aLkmYguf492eLhZ3d6y1hcTb7HMZvHlXhI\nU3DJ0bAMFu34CwPNKwDxvIediPdn13fTlfMjfVEML3p2l6O1PLVqN893v0TeKJAp1Q7ScbEvI/Yv\nWbMPy7YZGCqHTI1lI0jezrNP+pO1bulelHDZuc+la/ezYWAzt6+9i8cHBUm8vCus+e2vIO91OwYp\nlk1+/dgG1NHbWbRGmFslSZivg1i0XBDzn5buAMS6eq/d/gLL973Attxmb5tml0NLBOzOTeJ/UwgE\ntczmPaVe729TEtaRxc+NUIBJ0dm0O0NXTw7dsFCawgFykmqA5L87f1ixKqQVA2RLeedehCY7arRN\nLOW/vzm9uo+u8UFp2093xo84HypWa+lBzTtoNZJi4t4yWYOd3TnvGVYimbb454unM2/mKN4+2yfI\nn6/9FXvG/L7KvVHWTZZt3hHaJsU0hgs6PYNFL9+BlvYtJA2NYoy27xtClqvzIeS0PANDzvtYg7yF\nZu5bshLTxfcjmXHn+v67nCtWZ4orRWbzCIcSlZGiUrzAQCHrmbfcCdUNsKn000LtwJRXCzenkSzV\nfv08U63j56t15ZST8CS47MM1t9nqa5N+g6biVxOIogdyJQfHslAyQj5vbQTyHtZySJK7QNsMCS8e\neRd8Yqg1BtlymKzcyF8Xe3P+RCkpJprmulWcyzb4wlqmXE3ewTEKEpebgSpE3gEBSDMsUDXkQKT2\nYDHnBWC5SMQVT4iT04JUKpPT7A1EsQ+X/D70JlcTm7iR4lh/nfJAKSx8uglz+lIr+dnqX4aEqoHC\nEHe+fA92zH+PJMW3mtiYge2Oz7aW5q0NBI4TRDE40lI/xaA3U/RiG1yhIISA1hcb5QsBVkEEBA47\nhNubKSI39DM0cTGmFCDvcjUhm5Lfn64B35ozXK6+n6DmXWt5m6aJ96w0AnnbisbZJ43lX981k7NO\nGlO1X2kJC2NlKUd8imOSdueCWJk9TuKaGcc0ITf1hPz6l54nhILebBHd9L8LN2ZgWM95Yyyp1fcg\nqToZR8gNCktSsQXbkpDrhjyXVqYQHs/GeEOkeUc4tIhVJNxPnvIUNy3/ukfq7oTvEnStmsG1siu9\nWnjBOCO8fqZDDF6QTsW1C3qRVervUNp3UyxXTxSGUjs46GAR/AC1V6F5uzm7K8k7XzIOSvMeChCv\npBj0Z/0J1jQtbNv2JlLN1MgXq33Kg+UwWelGONZhMKhNK4bXF89H3uATT7YGebtL2AAKZoA43Ykx\noKGEzeYWyROXED/Gj3LPFKsrMCVisqfpuYFKlQUt9hd88t7W7U/87uQaxEDRHw/TMulveB65foBy\n0xZW9a1DM/yJfqhUYwJWDIYdTasQ8C3bloRqJ8g774zhPV+bQS2gPTvrg0dapy8pOn2ZkhfbYBuB\n4C6XuFS/j5Ll77cKIrmKaxbvzRRDhKZaooJXvkLz3jS4FSvhH7c34z/zXMDEbmbbkGyZTDDOooal\nwRVkhgvV38yY5Hh0S/e+p2DSFu/8eFhrjU3Y4AluFB33k6qxs1tsax6TJTFNuLdkW4yHkhTj25sp\nUTQDz0lLOPeV9yPTlWrNWVI133IQ0MylvTPBjCHFyyROehqAgYL/DZzVeiGtyRaKRumIBeNG5P1m\nQ0CajKsVmrdyZDXvkczmXiCRM2lVLrfaNbybItmQ9haEJudqbg9ix9CuUDRxELkAMb0as7k7gcdU\nudpsHiDQkTSUIT3Qf8UMBVaZlk1eL2AENJ+hgJbktl+pLZtWOFguUwoICLJBWQ8njwlOpBkt7PuE\nsHBQNH1hyU0sEgzyqQxYq4zyHa6hESZiCrudSHsMYbKsDCQKmnF39PmWCKxqrTWoea8f2EivuoHE\nCc+BLO47GImdr0HekmJ4VoVcYLy1DXNRiFN0iNHVzmPHrmJDYZV/vqPlDeVqvE+2BIpBT7ZAr5sg\nJBCZHdNF/Elw3EzE3+ZQC9aQKBE6rPmad5D8k5YgviB59xT6+MHKn3r3D9CdC0SmOwKKOdSKtuUU\nVCsVGsOagVmOcDGU10LzjLZsY2voAAAgAElEQVRtFi0J0Qc3ULJWamC5gryDEfbGUKM3Bjv2i/dR\nSfnHj5VEPIIuFVBkib5MMWzCdsZzqJwj4zyDWj5vggKSs9/oHYdeSHrjLzlj1p0V42V0T6TdmEZK\nTWLaZkjjP5yIyPvvEAOlQTQzaEoNkETg5Yx55C32u2ZzzSpXSemHMsmHa3IdSZu3bLe/Yn/l8ifX\nZOx+XJU+tqLtE9NI0dI/X/Mrfvly7dz6w4Go3FdjNvfIW5FCVox8yXCehY0yagd7ctVLbwAKhk/e\nUkVErGnZVcQ8FCC/2uQttO6gmX446ANWzCrNO6g51zKbBzX3vOFf39e8S9imgm0qlMxqn3cQtZaa\nxWOyPz6KDtgVS+L00Du6dzAgyFnV01qwv7XKoxYD1oOc5vfX1WpRDE+wcTXvuvxUrFwrshX3rDWu\nEKS2+W4J28bT8mwIERtA0m5Ckm36snl6B4uOUO2/N0nTIe9gwJfzHWtbT8Z2hJu8JvrQmy2FiClh\nC7N60KKU06sF3IGCP0Yll7wHRoMZQzHTDGnDXpayWm4C1zIwVNA9rdUcGIXZN576mMidviWzXRxb\n49sXAl8wsMA/xsoJ8pcSBU/zjiXEOOp7jmVS8ngAslqWtqYkvZli2IK2XUT2DRQDgmhgjLRts5x7\nEGOcSqj+flMV30dIKLTodSL0bSNGb6ZIWhWBevkaY3M4EEWbH4XY1T3ML/68nk9eeiKdLeGi9nm9\nwJeWfoPx9WP5wunX8H+PbOIvL/oaZnACiFVq3g5Db2m+l889bdO89VLv2B/9YS3nzxnPFRccf1B9\nfOCZ7by8c5Dr3z/b+1DveXwz2ZxGLqWBAiDxg3tXsdkpSPCpfzyJyWMasQhr3pWlI7tdf6/zcXUP\nFkKBa3nTn4SeXtPFI8v3ceOHTvMi1otGkUw5S4MzET350h4ef3EPLQ0Jpk9soXNyteb98PKdrHi5\nhxs/dNorWiFcYUOpMJs//uJu9vUXkFI54pM2cHfXBo4ZfQ0dHdNC5xfMvC9WKz7hqopMMbGXb6y4\nN3R8vkLzfu7lbh7ZvAncVNWySV+2xH/+5iVQyySmr6Bo+WSlxk1PAHIzuAXJ+4k1W2nMdHHh3An8\n/qmtDA6VGTfL13SDxPenZ3aI8+MlbC2JpOrsGxxC003iMSUsSOL4CZ3+j++oZ3dvDqV1L08Ul1G2\nXQ1JRBkPFzRu+eXzDAyVuPyiseLWJBnLtugeGgScSHTJH3NbjyPFNJ7esI1ZiX5mTWmrGVQUFCCW\nb9hLYgZItoK2+VSSJz8VIkPN0kgACScVpmzF0S2DsqlVuUJsSwJLDWt5FRpfPpNAaRVBcXv6ZEa1\npukPHJM0WxlmK/Gpqyi91Iytpfz2jJj4B+zoHeCWX66gN1MiMcrC/WpiOLWoy4M8u24/v35sM23j\nstDqdlJEUvfkMlzzvb/S2ZRkz2CWeDNgim9GNtPY2GTKQ7SlWnjmZT8S37+voNbqru0W53em2wG4\nc/09nNwxq3a8i2yKNtzgMMdaUVpzJraewDZU1NE76Fk3FmjwrmFmOmlKNEFJBLt2NI9l3fYBVm7d\nBzEorT4Lu1SHjEJ/UVhrmuvj5J0xHNf/DrYMlGHKWi/4rqkuTlmvuIf++fR0OMl0VIP+vAZpMUZ9\nmRId44UrIK8XSODniT9ciDTvoxD//fs17O7Nc/+S7VX7smUhDe52tJYgcYP/UUE1eXuk5Ex++/rD\n2vdjL4i2Hti6iJuWfjNkuq3EH57ezoZdmdBktvi5Lp5d3+1V7zEsg1Vb+ymUDTI5jXXbhfZkOaTq\nkXeFGd9dJuVOYNv2DbFuh29CzZm+VnnnY2vZ11/w2ga8NchlS5DDLxdtpKsnx+qt/fz2iS0hs7mr\nzdz7xFZ27B9mYFhM/I/tepIvLf1GTbO6YYj+xlQ51Hd3PIPEuLXGWveiJTRZ24gJE51kkUooJOMK\nQx3LveMkXZBVIUBGmmFy2x/XMRQ0dct+pLrascf3IzqIxy3vOXmacUwTpk5bwlSKvLBR+JT/vHQn\nz6zd7yWPUWWVklXh/5QNpJiOrSWxTRXd0ti0W5hcS3p4vNpTbRhogM2oVnE/8amrKdhhbV+KldnX\nX2D7viGyeY2N+4VmO8FJbtJfEM9/zrQOkulAycfhZmxLwlIK/OB3Ivgp6Au1ikIjdCO1g+M1Sj8Z\nu5wS5tsa5OvmsVY1oRWu69/gCUFuxrPyunnYhhr2V1eQt2vilhQD07LpaEqScuSQs8fOo8mY6B3b\n0C76Kak6tiWDrXiad1e+i+37hsgVdU8rbUk0M8Y6EXO4ha58F89u3k6uqLN70DeB1xtCECrbBbbu\nzrJsXbfXx6ljBOm675rrLtm63/fne/0P3CMO8br7zh53BhMbxmNjM1QerhKgZEMoIak5j3uBeZKq\nYVsSdrEejDj67uORZBu5LksqoVK2xHc0a2In582ayrFNk9kwuJlp08XzH8yL91wkh5EYk5jAgN6L\nlMgzujXtPceGRJpPvWc2tiV5yk1bU9IXnB3yTlvtTFBO8PqWLfrfaaGkM7vzJE5sn0FnXTtHAhF5\nH4UYcgJC6pPVfiP7gCUvCJnN3UxIru9VqTRlSbVNzot2Pk5faaAqorkWatbudYSDsiHuo9NZu+ul\nAK2INh9Z8xb34i6BclG2AxOxc7/BW3Ozk2mmVm1Wl02yAZPycCk8ybhc/IctDzJQGqyZdcowLZAN\nCsmuqr7Hj3+exPTnvd+1AuLKtiBDu5T2+pSIKSiyhGwEAn1KQrovB8hINyykZA65MbBGOKC9h9Jo\n6qItJRYgb9MCbKRYmeZEI7aeQIqXqtJn7sntJ6HEmdQwAc0uhd6Vjsni2tZwC5gKyCb5ongPKrN8\n1cXS4n1QjFCZSxcJSbwbUkwjmw8kRTHFxDyrbTqqpNDPDuIxiX+7ZBbHT/K1HqtUJywA8ZJnBXH9\ntbGhiRyfOA2A/ny1sCNZCiCBWaE5O/uTagJFlogPC3J9dt/zvrAqmyT0Nuxio/C31iB/M9sqtErX\nv+1sb29OolllpjQdw+XT30NCrqO8+RQAzjhFVC5D1T2N2y6lMTPtKI0DyM3i25Ad8rzm1I+RUJKY\nvULI2Ws6Firn2zH6RzPJnOe0GXgXnb5ceubxpBMqaOJdcZMDlZx3Ttt6EqUX345VrAuR97hRTh4B\nh/iS8RjHNYtqeHkjXxWVPaV9lD+8DQNCaFUMMGOeddF2+iCpGsm44rXxrxefQjKhcsGkc0UDDb1M\n7KzHkp3+OO/8xLiwcClt+xjVmvb6m5QTzD6uQ5i9nW1j2tJV1gNVkZkxfpTTB92L1bDNGLppM731\nOD520j8TV0Yu9XsoEZH3UQg3pWFDuka6wVcIsAp+YO5yD9eXqCgyECAb+cDZygyrOjBDMzVRLEEO\ntx3KmuWQd8kh77FO3WOXICqXihkBn7duGV6gkrf8JlS9yaZsB5f4iD4G/es9gexfwSUvUqJA6rRH\nWbTjL962XLmCvKtyh9fI8mZaxI9byZ66p9haeDm0T2nuC/2u5VPXKGDbYDlZtSTFEOStSEiaX3fZ\ndLRGzfKfeX+5j8TMZUiq4S83CvrNAwFKdllMikrM94drugmqjiTbNMQbsEsppHiJTD6Q7U6y6Cn2\nMrZuNC1JQSZBa4LWuB3bkjB6JmBbigjGGhSknXeimG1TYczwOYK8Ee9lMmVWvXMtsrOkKFYmGxDS\nipYg77H1Yzix/QSM2BAtHWUkSfKIxb1HW0tCrOwJGG4msobisXTUif5nQwF8jvbs+DhtUw2Rr/ve\nJeQE8ZiCVaynM93OjuwuQd6ShSTbWIZ/vhCgrND5Vq4Fu9jgkYvSIN7rlqYYNjZJ1dHsFckjLtcq\nIyl6IChNwth/DBAonOFcI6UmUWQJa0jYyPP0e+MNYPaOp0FtAFsKPUM1bnrnx2Iylkve5QyWbVGS\nRTu2HgdkQdJObAKAGjP8sUMQn/usl+xZ7uVkd3FSu59tJnHcS6RmPYuk6khmjNaGROBaQEwnHlMo\nODEPKUX0rTMlNN5sOYuqytiyLtwWtqC5MbFjkWwFdew28vWbkRQD25KIqWIcG2PNSIkCclOvqG8e\n8Hm7z8H1a8tNfdhjnRUThloVVHskEJH3UYxaUeFBM26tJQth8hbHFsoOwcmSZ+6CEaIxAyjVIJ4l\ne5fzu81/Iu6UKHQTHlQm+we8oLr6VIzm+rgXqeznwq4OWOst9PmEqRiA7ZVelOoyJE56GjsogDj3\nIAX81K7mDWHylOurE9G4ZnMXlZp0Lf+pYfpJNoaMAye3qWV216WiiLB2J+eA5m1LgQxteYe8bf8e\n9pZ3ICkmetfxGN2TgLDmHXz+linGRFZNz/pSMjSSJz0FQL1aj1VOIUli+ZUXSZ7MY9kWY+pGe+lb\ng9HphprHLtWBkRBaqwQ9Q4Jsi6azpGr/MSQK47wJXU6UeEL/XxLTnwtZB1plx7edyntCK0DRsa40\nJxqZ2ijiMFIteeceAuRddDRvyRcw3ICidCzFqAZB3iUr8Jwd8rZ0550xYk6kcfC9EwlyknGFsm7S\nnmwjbxTIlYre+YYhuwMi/ne/rQpSsDVBCLGJGyFWoqlRXNclJUWWQBcEVrByoh+q7hEj+OlTvWVy\nAdO+KsvYWgoZGSvmm91BmHx1A5JyGjk9jNK2l8TMZ5AdQSKlJokpMmbJ1byz/HHrwxjNwuftWg2E\ni8dGaXfW5scCPnkH7rNetm8Fd738W4KYN2Yu7516iX8/ySFQdFRJCEiiLdcXrpGIyRSMIkkl4WXO\na3LqqWfKQyKHhaO5e5kiTJVYYTSSbLFOe1ospzNVYoo4/8KxFyFJoI7ewdi2tDf/ueMcU2XqnMC7\n2Litfl/N2EEVHDrUiMj7KEN/IUPylCeQW/bXXCccJCOj1gtVg7xdYrVtO+QTr6V5u/5qqC7WAHjL\nJJTGAZANr22fvG1PA3LN5om4Qkdziv6hEoZp+ZHyjoYeLIPZEyBeSRZtuZp3/NhVyE7mLjcgxtMw\nAm0EyTtoqQhOhi4KevgeNcMK+fprJWUIErxsB9s8sLAFIg7AUHJYpTS25ZyrGCTiCrIsYztadHnT\nqZ42plt+H12t08o3CpO1c76LEHk7yT0kxcS0bAzTIqsNeoFCti152rmUKLK/wmffnGyixZkw3XSu\nSBaWpHt+WDdCtzebc8bL0byNGLph0eFoS7HxIpuZXJ8N9bfDnoptg9wUWAoGDNvid0uiBVsT10qk\nxHlFowSWjLZtltBuXXOrI2C4qTjr4knGNAvhQx29E6VjlzceAGXNyXtQSiPJlne+uz+pCGIp6xYt\nSeH3HigP+uTtnO8SnEsGleZYs38Mck6YY+Vkgfp6cZ6vecvYegJsWJ9ZizJqJ5IEiuW7GWwthW0H\nnoOsE1fiKLLiLAGVSMuNSIkCx09o8oPLzBjDeY3ZTWcgqQbxY1cj1w1jJ7NOH5IidqMk+rJjaCeP\n7XrSfxCGK4CIMY5PWRsao+A35Uac10JKTXJK58zQNkm2ScpJLzmPa2lQO/YwNGoJBb1ISvXT5SbV\nBEklSaacJaZIQrM2VRJxcb5hWpT2jQ9dwzZjnvvw2JaJIj4hVqalIeG9h+51Fdm3HoTa0OO159rD\njIi8jzI8vnOZSBRw3Es10xAGyeBHf1hbtT84eWu2+LusmTy8fKfIchUMOlGq28+X/PaD5kkXwQpS\nUsqv4OOlHJRsQbrgVeJJxAR52zbc9sd1dGcdE6ZD8oWSwc/+tI7t+4ZYvlVIvHaAmAbcDGSBicI1\nobkfYFk3uOuRDdz57OPszPjpX3/1WKAkZg0ff7DYA8AdD7/MMxt2+PsDWt7zG3q4/+ltXoY1gBUv\nB7JG1bBkDBULWJbN/zz0Mt/9zUrW7u0CyRZBOs49SorQvFVZwpZ1UnIdVqbTm1SCfmQ3iMc2Yz75\nB4Uwl0D2noDpaeZim6abFAMa6JT6qb5Glyiw28ls5b5DxbzM48sdM6yreTt+U9fE6fZxy0AXS9fu\n8zVcI0ZXT47lS2JYxXqkej/gUJIgZTfz2TmfADOOlWsRVhGnbbmhnyFpPzNaj6cp0YBWEtey4jl+\nt/lPDJYzKHoDZt94QAqR992PbWbzfiG8NSTqGN/a4l03Ptl5FxzXgusxsYvChy4lnVgKR4BKqkkS\nMZmybnoWiKyWDWhsYdLxfMoBUhjVmgYkFIe8pXiJtMNHSVX0W5Yc06/zPONOXeykEqg1YMvCwuCQ\ntyXrYc0dyA6oSDGd3WN+i5zyg62GCjpnj52HbVbTQVJJEFcV8jlQibNreE9ovyuY6Hum+hslv5JY\ncL16LeKb0Xo8lxx7MZIk0ZRoqNqfjqX9zHqm6iXsKSf3M1jOkI6F6y00JxrZm9/PQMOLYi4zVS+W\n4p7Ht1AeaGFy9p3e8dZwi5dpsi6pYhtxJFVHVWTf8uhp3lLoHqxcE+WNp4KexDBtNnVl+PkD6w95\nIaeREJH3UQYvwtGSR9C8fXJdvbWvan9wOYcZINpHV3RhmFY4KrZm1R0/GKyW5h3URCXVr9Lk19AN\nrNX1yFtm1mThk3txU6+fwcohnadW7eXZ9d3c8svneXHnDsDRLBHLSTzLgeFrIp3pDq8PAC9s7OXJ\nrmdZXlgUShm5bmcgM1egb66/2fXLu9jTm+dXj/vJN4JLjH58/1r+9MyOUCYwSxJ/nzdnfDga10Hf\ncJ6+TJElq/exbscgK7YJ4cQq1vtai2z4ZnPZICY596m7+Zb9PnqWF1MNkH+15l3YO9Zr31svr1ue\nyVnfNY2J6SkhzXt3r/PsHRLq2qvRtdsJ7nK1UvcenWdh9ApNRx23lb+8sMeL5G9K1lPWTdZvz2L2\nj64al3pzFJObJqEbFtZwC5IEckoIdW5ynrdPOFvcS05MY7uNjSLeApDxScMn7yKPPt/FcKmIbcP0\n8e00pfzJ2DYVLjhtgvfeubGK9YrQqpV0nqb6uDdeKVVohZpmehaIgVLGF5Zcn7kTeCincsSOXYXa\nIVZttNbVMXl0g9NH8b6NHS15edBd8vVQIfx1NjaGfttaCjlRIjZlFSYaKZf8HfK2HdO7jS0sHDbU\nxVNccf5xjG2vJ2ZWL29SZMVZlSJR6vOjqMubT0HvOs57zu3pZibEnWWkqk5RFXPP5I4OLxVqXQ3N\n+x+mLOSCSW8T/ayxfGx0U6OnOYPvv3aRVivJ2zGdpzYiyRa2odJQkRBmzqQp3t9mpsPLQJlMqCTk\nJHJcFwKPa4Gq4bcHMHomYGU7aUzHMEyLp1ftZdm6/fRnj0x+84i8jzJ4ZGHEvIQQQYQCoCo0ye99\n8kzGjvJfZMM2mDymkUmjGiiUDQzDqliPWi0cZEv+MqNaPu9g6khJMTzy9uoDy7XIW+GMmaOZM80h\nXOe6biajYPUeOT0szLmOyTc4oU0d3eH9PaqCvLsHi6GsX36DZu2/HXPgLv1lnt+/MnxOILCndi7j\ngHnc6d+0Cc186rJpVUdqZjkkhA3oTgnIYr0n8UuqCNBRFAkUHQXXzxj0AYoJzrUE2IbqTToN9YHP\nXNHF0idLEe3bEpYsyLikGb7mbsQo66YnxMjJgle8xBUWTC0WIka3L+75AP/90X9gUuMElPpBhu0e\n9sbEWH78nbO5/v2zAbDyTVXjIluOS8AwPdLxVg441291TNXZrF1V79pSfWuEbz1wBQwDhRjzZo5B\nkiTU3aeKZUKKiT12LRPGiOuVyqKm9LX/cBYA8+c1M3/2uEAwWIJETMEGGmMueQ/6AW+u5u1o7kpr\nN2rbPuQ6IYTc8E9v83IPWI5PefrxKe+7cjVvL6eMHo7GnzVugvf3tz8+jwmdghzV9n0YUtkjb1fz\ndp+Vi3Qsxa1Xn8Ox45qIqTIzx4r2bEOl05jBW8ecDvhLSs0+EX9QrzZgDY5G6fdzPnz742+lNS2+\nydiY7QzYuzm+ZSpffN+5/MvFM4BqzbshVs/ExrAZ28qFBZJxLS2hnPZSxZyUrqHNB2FrqVBFwpnH\ntPD2U8czPjlZXG+oDdW5P1mSOG5MBzYW31/zQz/4L0DeDTFfwDH7x5KMKzTVJzBMS9R4l6Ct6dBU\nX3wlROR9lMElC3dyrUTIh1rxosdUORSJbdg6MVUmnVTRdKeggHJgzXsoQN7lGpp3KFtWgLx9zdvv\nk0fejmTtfaTudR3ydgPzpGQOuW4IK9vmE1egv2ogAdKYOmfpiUMm/dlSyKzuJVEIEHZQcLED2ZTu\nWH936B6DwVmVZnWlfTdKu798zG1TUYTJOwjbktCscMrUYVMEuNmlOo+0pFiZZFxBloVA4+ZxxlKw\nLRkppnnpJstWwIXg3INhB56pE8ErGEEiIdWhSeKZarqFZrmFMWLifdATyCjIyYDP2yFRvaSCGcM2\nFc9c6xKrazaPqTJtyRaQID9KVGhS+o9lcvNEjxRcK0pojB0TsW5YHmkpDYNI8aInILgTaX8mXPEL\nwFQC5K255F1EqsuKwCzbJ8J0cRJG9zEAPLN/GT3SJm98VUVmVLodWZJZ2buGPnmrFxOQiiW9dzet\nOMVB9Kz/jjvjbzlL+jwyQBBZc6LJS0lslcWzzpQy3mqKSh+xvP2tnNJxovd7Zsdx3t8xVWFWy6zQ\n8UmPvMU4G/smc0LsLG99eqXw3ZRwnoNkc4w1jytnvFcc5xatGWrj7JaFvHvM+wHoqCCphrgjPIze\nSVxOcOnUd4b2B8n7golv4wunX0MlyhtOp7R2nve7M91BIjYyTbkCigvTDs95drE+RN5u8NtFoy6l\n+OLbwYyhBoJZ61RxDz3FXuRkwUmyI85RFZn6eB0fPuFyJmbeAbZMQzqGqsjohk1vtkRrQ7KqENTh\nQkTebzDolsEL3atGXPJVMv3JtbbPO1A4voJ8Y6rirSEGQLZEBKVTYSlb0MKm3Rqad1+gwENNzTto\nNlcM8k4ke9Eh76CJ17CdJTfOB5WIK0h1Wc8n7loO3OVZSpvwVZt943yTslJtogY/o5N7P2U9LJgk\nqHP6GPQHB9ZDl/2JqXISDd5DZWrP+JS1xKesCbTpkLcsoVNhTrNUdFsLZR3TLFdzjnvFFKR4mURM\n8QOeLH+JkK3HQfXJW7c1Z3mM4pnNTcIJQkLEJTWiSQWQRIpUL3LdUB3BSyItNSKlh4jPWYSUzHkC\nUbkozKlWoQEplUNp24OccM+Pe/fdFHdIIZHHzHQwpnwasiT7BXMMv7b4jBZhnVBNJ5LesDxBQB29\nk+QpTwrLhy15/s7eTMl7Z+aOOhWAMaXT/HE2Y9iGitLc65fRDFilEjEFK+dr/xa+5qwqMnElzsJj\nzmNYy/F88REUZy11OuYHU6WoR5UUitKQJxDalkIqoYAR9zK9uVAlX5sDMHSFhBJnsJxlq5NCdErT\nJIKQyg28a8qF3u+JjX5lrrgqc+boed56cPDJ0hUQsFSmpWbzDqeNyY1+8hcARXIEViksCfnr6yVa\n9anETfE8O5rDxOmSN8C5o89hQsPY0H41UBP+H45d6AsLQVgqdqGJy6dcwTWzP8bcUbOrqskFMXfU\n7NDv9x1/ifftg1jn71aQA0g6wlZSjXvvnRog2/p4WJMXFj4xfm5g2+mjT0XVxfuSTsaIKRKGKQJn\nK8fkcCIi7zcYFu94nP9Z93/cv/Xhmvu9IDFbqql5B3OaV5KvLNuUrTC5xlWZdDIG2BhNO/xoVaiK\nNpcb+1jc/cfqvgQQIrMKn7c6ertXHxfAQiz18j5OtRSuUSyHydsNGDKHW3yTskNoqiJj4pN3a7KV\nhJIIpYN1NSZzqIVOyzFhBwQc19xp5ZrQd87wtlea+4Jthgs01Fiap7h542VPsDGHm/nQ8R/ANhVM\nWw/lHNesssiFbckhzTsekz1BwF0/DIARR1I16p01/yaaFyTkBqwFyRtVJyb5ZFkvi0lIbhhk7eAa\nj7xtM0beqaLVoDoR5bKN2tnlkVCx4JiF841IEsSPXYM6QQRTeZYRSQpN0ma2jQ4nKU88oFG17lnI\nl97yWT4y/QOUN84hVRJJRXTDCsUyiPHQUOwEsiRjWlaoZOqxzZP477d/i3GcGDonKIyBsxzPQTyu\nYA2OQt80h45Um3+Q5QsYFx9zPh+Y8T5/V66JZEz1a0obNu2pdqz4MI0Nzn2ZKumEeBZuJjcXHzxB\ntOUSgmHatCRb6C8OsGlwK82JJi8ILvhadaTamdQwgYXHnIcs++MXU2WSCVUkxnFwaufJ4hoBzTKm\nysyfcDafnfMJPjDjn0J9mtYqgs7M3rApOzPsv++9mZIXhNpWURmsMemblMc3d3IgjFQO2MVJ7TM4\nrmUKkiQRj/vvu7Z9JlY5yWzlHXz5LZ9leutxofPG1o/m+jmf8n7bxbqQ5u0+r2CKYzVQddHVvF2Y\nw63e30GN2nUDphNqiPzbm4+MyRwi8j6ieOz5Lrp6wqkphwoaDyz1g5y2O8kLdgyJZSuPrOjinsc3\ne8uhvCQNstCUlq3bz12PbOS3T2xhqKCFfd4V5Js3CuGkIgHNW27qJT55HWpnIA96KEDGJhYo4wjV\nAWuWbYfK5EmKEYo2V8eJ7E5mthUz60ySkuV9nHvl1aH2RE1rvw61p7kYcZ/YHD92XVL1lr5JG9/G\njq4yacXPmAR45KdvO9ExHVdq3k7U9daTwIxTeukcZDPBkBZ+ZkENasPgZm596o/c+9ctxBM1stsF\nNG+3kIax5zjmjj0RLAXN0vnLCr82s47uCCaSuE9bglhZWCVc4UMPrO/V40iK5UUoIxu+VcJdR+ya\n6yUTSbZIyP6k26AKv3HsmHX8pe8BhmNOX4yY9+wanWPASTiiashWjGLJoqUhUdNnbet+bEWQvO1S\n2iPvYKnaJI2MruskEVOxsh1YTlSxHtC8XcjJAoolnv/9T2/HtGxithCwOlLtSJJUpa25goxVrMPM\ntnGccoa3TxwrYWY7mBYkA0vxtFZJkpg35jRa4+K9NbonElMV7zovbupF1uqRFJN0Y9k737VqeX57\nQNtyMjNahb/YNWnbNqT0jnwAACAASURBVExrOZaSWaZgFJnaPLlm8Q5FVvjc3E/xrikLKrZLwrxs\nJLC1BAoqJ7YLAVRRwiQPMLlpkhfU6eLE9hOIbTsXfdf00PZgVbvebJGCM1e11CdCxzUHyLsj3Uot\n/MeZX+Rrb72h5r4g4oHnlwz8bfZOoLzqbYxPTmZUXW0BIRiBbpdTNc3mSlCgCZJ3haBuF/0I+CDJ\nu0pJXSoW2t4RkfffH/b05vj1Y5u56X+eC23/5cMb+MNT2/ijk6fc9dmokkJfpshv/rKZxc+JZTa6\nqfumV4e871y0kcdf3MOi5bt4fkNPyOddGdzh1om2yk6QkWx6Pu9ahelD/uBE0VtD7aLSbL5++wAl\no+RPtorOcN5J0lLWQRITsbb5VH8Nsmx6H+cwvdiWRGnVOZhZ5+OXA+StaiKBhy37EbxOn+rTMXRL\nx9bjFLJJfvC71aTUdCi5hr+EJ4ahy1X36Js7nWIMRh2y1iisCZIFWMSnP4fS0uMtWQHYYDzDw8/u\n8pa+ubBtKeTzdjNC2UYMWZbEGnDZ4PHnffK2EMk3Jo6qByTQ48LnHTCbG5r/2bpaaSJtihSwiu6T\ntvMcOjqcNe+uoBPQLprjzc44OlYBxe+jm9K0PuaTr6Tqjt88TqFk0NaYrE3eAW3ZM5sjfPluLedY\nYFJWHRLz/LPOulndsJDNMEkAyGaSkmbw4DIh7F7UcQVXTv8nprUI7TFoKhX37rgjSmm0jXOZnvTN\n6u77Z9swJu2n6cSWQxM7wPunXIm2YwZm/1hiqkzKuc79T29n5y4np3Z6nTjdUkgnVc49ZayXZAXC\na59PmSpMvP9w5jGcMdrv07njzwx0vur2PbQ42cckSfJIpLT2TK4c93FPuw1mF4yrI5ugAS47ay7Y\nMmec4I/DhXP9wLjeTNEjrvGdgqynTxTvUFOAvNuStcm7OdHkrYmvhfEd4t0MCl+1zOYH8oMDTJFP\nQ983GZBFeteKtoKat+dWIOxDNzPtmAP+OAQ17wWnC5fD204ZGyJvNxvckUBUVewIoViuvfZv/4CY\nLF3Tn0veiqzSE8gnPVzQ6Q/UL0a2KGtmyHReLBuUpaDmHSbkYUeDtMtpSJRANompCnXJcO7lifHj\n2aVtCpO/G6S07xhGG7Pon/DnqoC1fFlDUkyxbjemISkG/UMlLMumZJSRZJvpbZP5+GfO56tPbKef\nHpAt74PSEZnF7HLaXx8qW77ZPKb564fLKSRkSAhLREdTim5LCwWaqXZSRKzLplgj6yWmUCmXbVER\nyCHsKWMbSU2qZ1sOT7CoS6pCY0oBqoYkWSL5DAifbkX0uhtjEJfjDL94BvETlgc0b9kb/1s+IqKX\nZTuGpYhc4u4MLSkGthHnuHHNfPby2Vz/2FKkZE5MHE77Wjkwm7sJJGI6LQ0xioqF5Wb0slQScoJU\nyuCmj8zllj8+AAjTopsfqjXRChUp6t1o9JyjeadUn4ileAlUDb2QwrJt0kmVn3ziHfz48TgbSi+i\nNGRoUBspBsgqpHlrqZqatxfxK0tIkh+kqBsWsRqEI5kJT7iYPKaBK+efSl+fbyFprwim0ndNI3Hc\nS+h7BbknAqbYoJY3OqTNSSGTKMC4pk7MHuGLjqkyHQHTsV1hGscU39aHFkyjdetuFu0Sgkaw1vak\n0Q386NpzhGUFeNv4M2lPtYX93QcoV/Ctj83zAh49Td2Ih4g0SE6V91OJd501hVMmt4a01ffNn8q7\nz5rMt3+9kr39ec+d0tqQ4NZrziYVF8cGY0Nqrek+GHz5I3PRdCtErkGzubftAH5wgOPUuazrEgpR\nkLxdn/dImncwT4W2KRA3QVggPPeUsZw+YxTppMpTq/wA1WT8yFFqpHkfIVg1UpUCVaYxw3KLhMih\nYhCFkkFf0c/JLSt+SktX8ivr1oE1b6fghuf/U0zH5+1XParPT+WczvnO/mCqVD/pQn3MCc6p8Hkv\nyQo/va0lhd9WFVWSBoZLXtnIpkQ9qiLTXu8kvohp3sdZtovexGa7NZklV/O2QdWwveAmmTq5Ednx\ng7c3J0WQn+l/1JYernYkMi4JE2nZ4V3h47dJxhUM1zfsCACphIrlraUuh0zwthFnsiYKIXjJLZzx\nPnPs6SiWWKftEroiS2TKQ0hIjKpvce6gVhIVE0yVeFymPhUThUEUC1s2sJVgoJjTDyeoTZcLtLW4\n5nKfHBpiDWS1Idqakshp8fyntvqaVGstDckQZnt3kp7deiqzW+eIcUgNI8m2l9WsLqkSjym0SZPR\nd8yEgQl8aMpVBNXFUGCSLdf0eYeIXJFFwiBElbRa0buSmfDM+lPGNFV9R5WBQ9bgaN7ffg22YyUI\nam5BIvdWKQT6EkRdYAKPq3LIxxn0N4MTsJZUkSSJ9nTAOmGE1x2nEiqyJCFJEv90/Lt5+4Szqu53\nJKiKHCLaWvcUJKr4K5C3JElV7cnOto7mJLphsddZMphOxqhLxjyiDa7jrmXyPxioilxlNamteR+Y\nvINCyiuZzYPHnth+AnVqmium/2NVm3WBQlCSJHn9DL67ifiRo9SIvI8QauUZr7Xd07wlhd6MT475\nUrXm7aIuJV4iTTdDRSrCmrfN5sw28Zdjcg6bzZ3lN6UpXuIKKWg293Ihq6QSKkk1GdK8dctgW2GD\nc7AFZswj/L5Myat85Urnk5pEQJJclyURU9AtQ0RKuyZ3JxmDu9YbVRdm4YD/s0FpEfV3FZ2OppQg\n74DmrZXcQLhAZivHZFly5CK1bT9Kx26x3MPSPHJXFYlEXMEoOwJATAv5usHGGhjDhORkp9604Res\nUBNiQgsUtFBkiaw2RH28zsvFbLqme1dIkiyRWMJUfFOuQ85lCuiKIF+9GCAMxyeXNftobnL8pwGz\nbGO8kbxeYG+xy8vHPGOUr9U1JeqIyeIeZdstpOFkbnPMo+lEgg/NfC9WOemZ122nIlk66QpbNnax\nAXXvKbSkwlHESSXBuPix6LunoioyTfV+JLoLNaAdKrLkFXoQmncN8jYSnvm2crKH2r7HukCyjrBZ\n1m+/MR7O8hXsl/gd9h8Hr2NraYrPBXzRAZ93Q9zXhG0zTN6viFfBg0HNVKkIbHu1cO91pxO3U1dJ\nskqck9pnVvnjXytqEXWlUHWg/alEtQl+pIC1hng93z7nZs4c+5aqNmu9ZxAm/1cSKg4lIvI+QhiB\nu6vglsNUZYW+rK9590gb+O2m+/0DA8TqlgYtaWaIUIOat9zcy7J9ItLbKjnLpGJlYorsmM2dNddy\niua0Y/JSqoO9MEVO6sZ4AwOlQTQnA1le9zOvGb0TsE0VJSau35sp8v/Yu/P4qMqzf/yfs81MJpls\nkAAJ+yabICgo4i5Qt69WWxUXcKlaRVu1daFUpbUPuFT9Wbva1trqQ12hllddeLpp1YLWlcUVtAjI\nkkD2zHaW3x9nmXMmM5mQZCYZ5vP+h8xkZnLmJMx1rvu+7uuOWevL7eA9uXqMeVyl+/CrDx7Gqzv+\nbZ6npJ7Y/knrzKBmNUZxFy/ZhVRCoB0DyvxQDc0zbG533rKDrjkkbZ6rcLsrWJTvRWNwM3a173Iy\nd0kU4VckaBFX8PZUrsdR3xSBz96yUo45vxO/5IMkCOb6Z8mcKxdFc7cjuwMUAGiqfYFiPq+4OLGk\nx+nnbL3fiNaGmGiNnESCieYeVrOaFr0egZB1fK7gXR4wA+nKj58xHx8NYGBxYs4x4JfNoXMAJdoQ\nCLGg01TEzmz9PslsEqO5A5V5UWF/gNt75IiC0GGeWBAEnFJ9DtQvx6KqPODMwbqzMzkp8/YOm4uY\nV3YhYlumIbLpaMS3j4PSNNK5uEgOIgBQXtJx7tGdOaWbUxUEAQtGXYzohzM7HFcyRRZR2mFnP8HZ\nIcuI+Z2LG89FQYoe+r3NfUHiHjbvjeAdjWnmErqkQCUIAr459RKcMvLkbv+MVFLOb2e4oHFfdLmH\nsv0phs2TL9DSCaYY4QAS9RoAg/dByT1s/t6n9dANsxfuftd2llu/bEJMtTJcw0BdY9jJAPeXJ/aA\n1ttLrAIq8zXNDy8De/WtqI/sT/xQV/C1h5dlQYbeWG0uMSpuhqJ4h819QgClwSLo4SDEUKPzGu7M\ne2d9G0q1oYjpcTyx/jWs/2C3s7etumeY9fqJIri6pjBiVqFdsbWOcnRlrbmOdsBufNGyHau2/MU8\nUCt428PmghL3NOZwF0KVyFaLVCWC0lJ7eU7iP09Ts13oZm1VKCUqsdvaBGCH+SErVdShrug980nW\n/2NZMiuW7QsdqWq7N/OW4tjXHIEMa3jWmuMHzEzTzLytD3YljrgRQVxXUe4aQraXfCnDzKYgpSF7\nIwvZmUqwq5TbjVZExCYYutnD2mn5GPfDiPuwH19ik/oP8xQ0JqqI7S077SmX2JbEOmDAXPs/sMgM\n3pE2H9o3HO08xh42d9bhC4lhUfu47OBk/32bc9YdPwztAJuuGtcdJCVR8BSs+WQRgwODoe0fAqO9\nFOquMdBVn7MbXjDFvvbuzMo5Blfm7Q48/qQ51YmV46C3mFXlyRciboospXyvV0y5GPH35gGaL2Xm\nndziMxv8nmJAd/DufnAZ6JqKSHXBlC2ZsuxMz3FPz9gXAuky784Up/g7AwBZTrwWg/dByJ15P7Rq\nA155dyfuXvmOp9HK8sfeRn2zOTcc0+PY1xxFZWkAJQEFQsRd9GP9J7IztiIFUtUObCsyd/s5acDp\nAOBtB2oF4UvGLwIMEXpbGUR/GIZoNfiQzbaZPsmHoF+GVl8LQdQhVe72PB+agoaWKN57y/xD/vN7\nr+PXaz7Axm3m4+zgamgydMEMmvubo4gLVvC2Mm9REJ0Mz3OeUvTrdg9ZuzPvErnEeZ8lQSvwuTJv\nLe7q2Caaeyy7s57wl0M9VePun2suvZGgt1ZgQukkSKFGSFWJZXSxz6bCMIC4nZl7Mm+/uYey9f7E\nYDPaNHsLy0TWO7bYWspTuQcQVZSUJC5A7A+BgUHz8a/sfx5hcb815SGgusIOgmaTFA0xRIw2xHeM\nxfTBiTXqdvAGzPXtRpv5evaccNAvozpoBqq2Fsks7LOCS0u7+Tu3i3wG+BJroO3gbVfXjq01f85h\nYwc6owLuoFhZGoAAYGhVx9854B16lCUhkXlrZuadnDFqmp5YrpNuODPpQzlV4RLQ8QPXPUfaWYGX\nnbFVliay/AGl5haVMsy/02CK4J343XXN6CHm//3Dxg3M8MhERul+T6LYu5k3kH4IORvsn+WTRQyz\nKtyryjpvhuK+6PLMSSuJkbVU3+/KcSSTPXPeuQverDbPESczKWmAPPQTfLjTu7ymTdgLsbzOmeON\najFEYqq5jlY30KaJEACE6meiWbaWFllV1MGA7GzacP74ryKyx9zooXaIjMtPnAXdMPBKXQPW7/3M\nyXy11lKIZXUIi/sQ9I81s1PV3NtWFAV8e958/PKjTzF1qoh3/55ocFLiC6IZifWPdr/o3U1WW0+7\nGMfOOiUVMVWHjihEeCtSJ9YOxsdNiZaR5vPNDz17pACAOd/tt+daXQ1GrOB95IwifN76mfVzXWug\nnUYuGgTr/BiajOKAbA25CmZBmL9jsxnJNSw4o2w2Pmr+wNmJKbLhWHO/agDtLQJQYgV915y3JEWc\nJVRicTNaVfPnuzPvb596PJa/vBX75E8hKDFUlgWxwzpGe8574UmH4uebXI1rrO5XQwYU47yTxiIU\n9OGdLwdgXcM/UOoL4ZRDvo5h1SFcEDYb5OwT/us89fCRI3HWCWbryTsunYn9zebWh9VNZlCwh8JP\nnjEUf39nBzTdQHFAdoYd506agj98bG7KcsnJhyEkDMCU0WbWfszUIRhUUYTRNWaf7GWXzkSFK6hV\nlRfh9kuPwODK1FXInjlvSUQsrsEwDKfaPDljVDU9MSef5kP1vmuPxusbduGZl825fk+Rmiu4JQc0\nd5CXU2Tw9187B63huJN1L7t0JprazIs+ewcrO4ja2Zq7u9hti7xVzJnMnjIYA8sCGF2ToiNZkvsW\nH43G1ljSnHfXC9Y6M6DU3BfdMHIbvAM+GcsunYnykB+KJODLfe2oTXMRaHNfdCkpRlnSFay53X/t\nHLz18V488Tdzu9p0GXqqi4NcYPDOEbswTSzdB6m0Ac2tewAkrh63la6Fuyg3psUQi5vLqEQB2Cuo\nKJaLIDQMhVi1y1xcJOowAAQUGYJsZn2TB0zA3z7ZZ3bv8oedtZhavVWQ5jOvnu250jZhPwRBgCDH\nYaiJhgMTBtdC/FhE3JpntTPvimAJmvfHzbXWmuQUpe1vbzH/muxqcSdwxs0GNIpVze5aQlJRFAK8\nsdvJrGPbJiIweb35GnIMorVlpN6ayFxDivke3t3/Nt7d/7Z5pyvzdobQ5Rj8483vG9EihII+TB0z\nEOs273aCfak+BOMGV2P9f8zzJEuCk50FjUrobaUQi5s9xwgATc0wg7ccd4oI/ZIPoiA4PbvF4ia0\nqOZzy1xz3n6fhOpQGfaFzfqD8jLRXLalJ4bNq0PeavD4DrOJSHFAdrLYE8dNw4mY5nmcX5FQWQpU\nxkc79w0vH4RqK3sqtiqFAWDW4MPx7KsfQ9s/BJNHVngyQ3e2NW5AotBt+qihngsxURBwyPBEtfWI\nwR23dxw5OH3w6ThsbjhD58mZt98nQdONRJerNMOZpUGf50PeHdB8nmHljnP0smQeQ6oP9oqQ31lf\nDQChoA+hoLeRjP1+3BcCK+bcDkkUUaIcWMFa8rntTFmJH2VJ8/2pmrR0hyyJqAz5sa85mnYIOVvc\nf0/2KE9n5DRLwVIWrKW4QAPM3/PIFH/Hydw1BRw2Pwg5w+ZWT+WInmKHKxd7yZdfkcwPJ1GDIvoQ\njWuQkpYYybIASUlsU1jXGIERC6BdS6x7tVtzhvzmB669WUMM7TAMwwrePkSsHbxkUUaFvxx14X0Q\ny/dCHmAPi7u2WlQVZ/mUvduYMyft7GGsojUchVhsZuYhV+FOyrWg9rB7WzmiH1vLk5QoxFAj9EgR\nEHd1B/OluPp27fhlX0BIFXsgKHFIrYOh7jSDn7OUyPp9+EQ/Lp9yEcT95m5DdsEaYFZdq9aOSoYu\neLL7BmsBgFi6z5mXtzd+QDwAI+aHEGzGjlZzyL0maSlSib1LkRxDaYld7Z0YNncXOk0UToTeYI6q\ndDXzce+6lG7tbUD2I/blKECXMLC8KG27R3exXbHcvXW86aQqWItZ65cVyRu8Az4JqmZkHDYHktY4\np8mQUs2P2wVv7u1dD4Q9kuD+PZX5Qx365OdCb2XeQOJiLpeZd3d4Mu8U1eBdybzNx2U+X+6Lg1R/\nS9nC4J0jTsGatYGCs/uTLWlLQ7tHud9ndmkSJA2KYG4Dam9q4ARvSXSGtQNyAPWNYQhqAO1qO1Td\nvD+shiEKIoKKGbTsIdKI0WbuYiQYgKZ4CuiqigagOdYC//h3nPvawq41yZriFGm1WNXmRofMW0Wj\n8hnE4hZUxMd4AkiqYOLeYcoZQg81QJDjHdbRFrn28lWsYUnPPLp1DGKRtca8bZIzn+tklFa2LMIq\nHrP+I8qS4BS6tEfi0PbVmIFb9cFd6mq0l0JvLYNUXg+p2mxp65f8zsWa3h6C6I9g475NKJKLMCxU\n63kPIcVV+e/XneO2P2R8kmvNtpgYdTiQzOeiCeeiSC7C5AET0j4mZm0vW1bs82Qi7vXSgiDg4gnn\n4uyxp3d7HW86SoqlYnbzEZ8ieoJOQJGg6ZmHzYH0WZV7Pa6U4jF2Zt3Y0vlFdjr2h36uM9RUpG4U\nZ6VjX8wV+/v+fXXGezHoyox9nS8VS9aF2J2x8U22MHhnQTiqoqXduyuY0yXMyvTiRlJ3LtVbgOEE\nb8Xa9UtUAV1GLK5Bttbl2vPjih28NRkCBNQ1heEXzMBoN2Zpj4dRJAcgSaK5VCfuh2EIaFWb8ceP\nVgEAtP2DPB9WA4OuTRosEVenOMOqKPcrIiL2Bh3OnLcVOBUVMdnMumvh3bIwOXifNOxYs2DKZjVZ\nEcvMPa6T23C650EXTVqAG2dcA3XXqMTx6e75bxHlYiLrtT+c7WAfMMzXtocYJUl0/qO3RVRA9SH+\n3ymIb0/sya3IImCIiG01h6ztna38kh/2SgB7HXZYi2B8+egOGzKU+q3aASXmbCBiaHLK5TElUuLi\npegAMp+ja2bivuN+2GlbSltxkeL5MEquDp9dMxNzhx/f5Z/dZUnLxlQtfebt90nQNHPY3C4sTCdd\n5uS+v7Pg3dDazeCdIvPuK+5h855edOVL5q2kec+pMu/OCtbc2/Wm09MLou5i8M6C6x96Ddc/9Jrn\nPrt6Fk7w9gZ3Q9CgR4LmGlZRcTbZ8CsSSopkCJKOPfti5iYMgt061GroIgnOhhRtERXhqIZia3/h\npqg51xpWwwhamar5QWj2zd7eth0fNXyKQfIIaPW1GDwgEVA9OyxZhg8yX3dAqd8pShs62Oc0QnEy\nb6dtpwpVtIbsFe/8kXsI8VuHXdlh/99xQ8xWlfb/PfcmAYD3P+DQkhqMLR8FGCnmvGF1RLOqdodV\nlzgZROyzqYhvH4ehhrk8ys4AZVFwisbs4VmtvhbaPnP4fNSQUqdHtxEtcvrFA4Bf9jkdLY32xEhA\nqsy3LJAI3s7fhC55AtL/G30Kjhx8OAJS9pbq2PPcQyqDnkrsQTnaaMHdrEiWBOiG4azEUBTJO2yu\nmHPebRHVHJXqJCBJXVjDW2o1jXFn92NqzIu5IQO6N8ztVyQU+eU++2B3S3Vx0l2DrL+T0mJfhkf2\nrXS/d/v3ka63eTK7F7zYyd9YV9eJ97b+ffmUp+xCG8MwnA+WRPA2/9XQMXhDC0DdNQbV46LYGfkC\nsLbLnDV5IF54C04wcipXreCtSCIgxWFEfE5L1XJ/KRoANESbMArmnLddLFUe8mPP/nZz/tgXRZEc\nwHdnX463yxow3bUcZXz5mMTxqTK+MuJknHTUZLz7aT3iqo6nt5hrzysrJWyPxc1kU1NwzNQhEMsF\n/CeyCZKswfBFYBhCh0zbfXt8xRgIgoAbzp0Gnyxi9/52zJo4CDe/9ifnfert3jluWRKwaOL52NL4\neYcLjbISH5paDRiGGfyrS8px+lEjURr0YfaUwSgOKLj27EPx8z9thLprDIQae7g8kXnbRU32nuTj\nhpbhzGNGYV9TBDPGV+FnqzbA3GFcgN40EKK1I5tn2Nx1wTFj0FQkq7CaqMiDvsCGfebPOe+YyZ6i\no1NGmu1qX3rjC+e+AaW9u2/wLRdMx0dfNGDK6AGIxTV8/YQxB1Qo1VPupZR2sLH/litCfs8oix3I\nW9oT+5ink/yhe+uF0xGNe7OpMTVluPTUCc4GGwBw8hFD4fdJmDHeu/NWV10wd5wzrN/XejN4H35I\nFRbOH4+jJg/utdfMhuRs+vuLDkdTa+Iz1/130dn5GTE4hEtPnYBDhqcfteqrCzQG7yxStcSmCppm\nz3mbHxya4A3eEPREYxIkdtzy+yRnq0l7DbOdeTt7RUsCdCEOQwtiZ521UUdxJT4PA/sjDeZuZLrq\nZN5V5QEzeFsXEjXFQ1CsFOG4ad4sa3jpUAwOVmN3+15EP5yFY4+ag1DQh+Om1WD9B4lK7WDQgICw\ntbm9gLOPHY09cRn/ec8cNheUKBD3IVDi/aAt9lQrm+996hgzCE8YYfX/1v3QxXarUKxjRe+RVYfj\nyCGHdzj35SV+8z+rIQCCgepQGfyKhLlHJPp6H35IFYJ+Ge1R1cmU7Yst93CsnXkfOnoAJo9MVH+7\nq5zj28cDhoihlRVQRNmpcTDCJdAjRThhzHTPHL1znEWJC5KdrealwNETRnV4HODNEFJ1EOuJytIA\njp4yBIBZiX3aUSMyPKN3uTNve5jX3rSnqjzgyYrt77dFVFRXdF44l5xVpbsYOW5ajee2KAgd7jsQ\n44ZmnqLIld4M3pIo4sQZQzM/sI8lz0PbIympZJpKyPR30JWitmzo+zGdg5j7Cl/Tra+tYXNnj2Xz\nljlfavfztpc7WTtuOXt0W8HSZ+/HbDdOkTSn4GxHnVn1XVtqZtANkUa0W/PRRdY+t1X2jkuy+bo1\nJemvom8+4jpEP5wJI1zq3bQ+oCSGyJUwRH8EmpUZK7LobK0nyCoEXxRGzO/Zlxfo2s5DJa3mHLPe\n2DED6uxDKRS06wLMoFDiSz386QzJG4bntuyZ844797l55v00H+LbJmGYPsN6Qet+Q0R0w3E4b/xZ\nKX9+kc/nFA8CZuFdukpud/FVLqtac8Gdedvnede+xI5x7mFz9+890/RBbwaufCX1g6H7XMvlUHa6\nTaeyrfB+qzlkL7sCEsPmguBu2WmxN9+wMm87wxZE1QreiblQAAiIAc9rGFYWb2gydlrBe0SlOV+8\nP9Jo7kcNIGgFVLvNYeyzQzEiNAynjpyb9j0E5IDTKtL9HyIYkJ3g/Z+ItZtYuGPwhq/dXI8eD3To\nhdyV4F0ePgSRjXMQ+++UDt/r7EMped/idEt07Kvu5P9/dntUIJF5JweCVEU79lW49+VStwwFzGVP\n0Q+OcgJ4mb/jDlm2rhTP5Bv7nRquM2af50TmXeQ59+7fe6bCqT76XO1XCvECJpdD2U5ilmMcNu9F\numF45lI8mbfmLVhzb7cJwargtoKz0SHztrqLWXPefsneDMMM3ppgXQioMnZY2/UNG1AJn6hgQ/1m\nJxjYa4ZDRebws948ELfMPK/L7y8580bS7kh24xdFFlFkWM1gfFZns5i/Q1WwLMo4YegcDAqmn1eU\nZbFDoZqts0KT5Cvv9MHb/Dd52FwUEsHbnitLfs1Uy4DsD8p0u8h1PE4RRqzIXG5WuRdiJzsu2Mv4\netJoo78RBAGGYSRl3lbw3tcOvyIhFFSSNjFxZ96dz3l39fdwMGPwzi7nsz3HDp5PgT62e387rrjn\nn/j7267+1/HEsqrkgjVB6ph5G9awuW7vNuWLwO9zZ97mtZZTdWy9hu7KvJvaYigt9iHgk+GTzCD9\nft0mAMCospEA9OeBHQAAIABJREFUgPJQ9ypFk5tcOHtuW/RwCLIkmPv/Wpm37rOat8T9Kfv+njv+\nLBw39Oi0P7OzZRzJnbHcyoq9c8IBOXWBlz13XGQdWyITTPS/brcadSRn+ikzbyl1Jp+JfcEW19MX\nOdmvOXxQ560h84m9JMtd4S675rQHlgc6jES4f++ZMu+DbXqhOwrxHOTygqWvLqaZefeStz7aCwBY\n+ddPnPvcm444QyuiO/M2AAiJPautYFgSrwX8m6AM+wQ++WRnztvOvINyEIh3zLxrKspR7h/gVMi2\nurbpBIDRZWYR0qSRlTh99ghMH9e1Stprzz4UX9a3ej4EKkJ+nHroNLypbcWYkrF4+8P9MNpKofgT\nFfHmkjd7NzJft1oHdjZ3lep7t1wwHe9vrcfXjh8DUQT+ZT9WTP2zrz17Cl5Y/wVOn20VaLlesqqi\nCKOGhPD5LnP0IPkDIdV8qzNsbkVaWRJxwdxxad8DAFx51hSsa9iBrZFdnuHjZGcdMwqqpuOsY1IX\ntOWj75w/Df/3n+04+fBEEdSx02rQ0h6Hbhg4ekqiHuPCueMQ8MnY+mWip26m4D24MohTjhyOKaMq\nO33cwcyvSDhzzshO29MebIr8svmeh6R/zxfPH9/lTUk6M2N8FU6cXotjpw3p8WsdCAbvA7S18b+o\nC9fjqCHezQXswCYEWiGW1UPbMwLRlJm3GagF0XA2FnGG0q3g7YsOQoV/KBqKdwBSvMOcd5EcgBAX\nnNakLaq5DehXpo/F7JpEj+sLJ3wNL3z+NzRGm5znAeaQ8NeOTywDy+TwQ6pw+CEdA/3Xjp6Mq6uO\nwsdb67B+rbkft/sqNCgH0BSzh/SVbgZvMem22XMaSD1sPmFEhVOpfv5J4/Avc5dMKGLq4dXqiiAu\nPTWx/lpAYthbFARcd85UfPfnr6c8lmCKLlPJAf74aTU4cXpth8e5nXncGEzYfiZ+uWEfFhxydtrH\nBQMyFn7lkLTfz0dDBhTjklO869/H1pbh21/vuKzOXimwbXeLc1+mYXNBEHDeiWN74Ujz21ePHZ35\nQQeZTO/5pF6qmpclsU/+XzJ4H6AH3vkFAGDW4Bmebln2XHdgqtmcJdJa4Q3emrdgDYCZfetyIhu3\nhs1jcQ2KHgREQBOirszbqjZXJBTFi9BqBe9P2z+CKIiYPND7ITin5kjMqTkSb+95DwNTNFzpLe4P\nUPeSnqASRFMssZtXd7bLSw6YPlmCqplDy501TrAdWzsbr+5ch9HWlEEmiepz89+yksQUQ3Kmn7pg\nzTts3tWGVhWBciyddWPXHlzg3Bdt7o0/iAoJ//K7SdVVZ04ZAJKnWARRSxo2TypYgznsbcQDEKwm\nJPYcciSmQdB9ZvAWI4iq1lIxe523LCIoF6FNaoXgb8OeyJeYWDnes4mF2+GDDuvRe83Ep4hmP2rd\n8GTe7mpyo7uZd9J8kt8nOXPQXWn1eN74s/DVMachIHdvXbS3mYP3WFIOm9tz3vbwd+FNN2ad5ClY\n40cYFSYWrHWTmlRY1CGQGELSsLm9zjuRedubeiSGzc3gFo1rEKyGJHEjirC1TttemuWTRQSVIkCO\nQRpgNvaYOWh6z99UNwmC4HyIuueQPOuVXZttHIjkbPdAd0USBfGAAnfyum+3mKp5bqfaijIx523/\nfEbv3uYtWOvfG2QQZQuDdzfFda3zBwg6/vi3T7Hps30AXJm36FoTaFecJw2bb9vdgi92mvPcUSOM\ntri53tVu0qLIEkqUIATRgFS1A7IgY2rV5J6/qR6wP0QVxTtsbjNUxbOTU1clD5tne79cZ847xfda\n2uOe26kL1rpXbU5dx8ybiMG725Izb7ufucMKyA88/T6AdMPmKgZVFGFghbWHtWvplb0dZtQIO3tx\n25m3IotOYBT9EQwtHppoitJHjpo0CANKAzh8fLVzX1BJtAOVoXSrjaA7eI8cHMKF88b37EAzSZEo\nf+/iGZgwvByzJ3v34lZkEbMmVnsKopKHzZl4976JIyowqDKISSMrUFHau21iifIFL1u7SdW9WVgs\nufuVNY9tf3inK1i7Y9FMPP3uK3izHYAuosgvIRzVnK0129VE8LaboiiyiGI90XQk5Ov7db9nHjMK\nZyYtYXIPm/vl7q0td+/zfPslR2R9swdnnbfr1zRuaDluuXBGx8cKAq4+y+z89vQ/twBwVZsbicdQ\n7xo3tBx3XXVUXx8GUZ9i5t1NquEdNjdbV7rms63MuzJkZsTJvc0Bs1GLTxFRVGT9GgzRqdy2s+zW\nWBva4+3wiT5nWN0niyj3JdYvhtIUqvU1d8FadyrNgUTBmiSaLUaz3Xwh0XGte+Pe9pJBnfVqRJRF\nWc28V6xYgffffx+CIGDp0qWYOjWxdnPlypVYs2YNRFHElClT8P3vfz+bh9LrkofN46ruZNsAnK8H\nWMN6qea8BVmFJIoIBqzgrZutIOubIs6weVu8De1qGAEpALs1hSKLKJMSwbs0zaYbfc09593duWq7\nOMkO2tnecEBI7pd6gOSkJi3MvIkoG7KWeb/55pvYtm0bnnrqKSxfvhzLly93vtfa2opHHnkEK1eu\nxBNPPIGtW7fivffey9ahZEVyG8u4qnn7lVvBu9Rqv6m72qMaqnnNJPniePHzv6MdjQDMOe9ia39i\nSfdBgIDWeBva42FnO0/ALFgr8yeCd1mgf3ZOch9z8qYkXWVn3nZGm+3t9xLD5t2M3slLBhm7iSgL\nsvZJuG7dOsyda+5WNWbMGDQ1NaG11exzrSgKFEVBe3s7VFVFOBxGWVn6/Vb7Ql1jGI+t/djZDjJZ\nqszbvVOY0/LUCgLujUnsrFoYsAN/+XwtXtn5uvVY0Wk6IUkiipUgGqNNiGgRTxaryKI3ePeDOe9U\nZDExsNPtzFtK7K8N5K5Pc7eLxa0n9tU2gURUGLI2bF5fX4/JkxPLlyorK1FXV4eSkhL4/X5ce+21\nmDt3Lvx+P04//XSMGtV5v+aKiiBkuXeXCVVVpZ8rXrHyHWzZ3oiyUABXnNVxO8rikOJ5viCJiXXb\ngJN5y4qEqqoQJFkCYEAQAD3uAwLtHY+nrBhl1hy5IokYXl6DD+o+BQBUliSC9eDqEAJFiYA9fNAg\nVA3su3nvdOfRCNYC7wB6OIjSEn+n5zudygpzSkCWRef5AZ+E8cMruvV6mVx46kT86JE3cN68Q7r1\n+iWhAKqqQrj6nKn45aoNOHXO6C69TjbeS6HhOewdPI89l4tzmLNqc/cwZGtrKx5++GG89NJLKCkp\nwSWXXIKPPvoIEyZMSPv8hoaOwa4nqqpCqKtrSfv9fY1h69/2lI/b19CCOiVxf2tbzDNsPn54CB/s\nBMLhOOrqWtAeiSWK1TQZhi4msnPLVacfin+tM3+uKAoY5B+ED2AGb9lINKNoaQ5DiyZ+dVq72Ol7\nyabOzqMAH04oPh8vvl0HjDO6dYzhtqj1WnCe/7MbjoMgICvveVRVMX57y4kQRaFbr9/cHEZdXQtm\njhuIw7v4Opn+FikznsPewfPYc719DtNdCGRt2Ly6uhr19fXO7b1796KqytzcYuvWrRg2bBgqKyvh\n8/lwxBFHYNOmTdk6lG6xLzbSjdJ2GDbX9KRtPs3MW7NeJ2q0QvBHrBcXALXjdZMsys7wuiyJqA3V\nON9zL7tSJNFTCFWi9M9hcwCo8g0GNB/8Svf+1Ox13u4qc9GqPM+W3hqaL8StGIkoN7IWvOfMmYO1\na9cCADZv3ozq6mqUlJhBpra2Flu3bkUkYgazTZs2YeTIkdk6lG4xUqzTdY8edChYi2uA7B42N7Nq\nOxjvqFqDwNRXrRcSYcQ7NpdQRBmqtaRMEgUMK0kEb/dabrt/+IiQuctSd/t254IdfANK9wZ5ZDm3\nc91ERPkga8PmM2bMwOTJk7FgwQIIgoBly5Zh9erVCIVCmDdvHr7xjW9g0aJFkCQJ06dPxxFHHJH5\nRXPIvdRH1VX88aNVmO3aBlQ1OmbeYlFiqMQQzO87Veae1xZgtJVBLDYff+GEr+HThs9RVTQQmlYH\nwCxYqykZjONqZ0MWZcypmYUnsN45JgD47uGLu70eOVfsgjNfN1qjAole6WKWq8x7C+vUiCgXsjrn\nfdNNN3luu+e0FyxYgAULFmTzx/eIu8nGxvoP8cbut/HG7red76tJvc1jqg6xsinxfGgQBQGaYeCL\nvc3eFzdE6K3lQPUOAImtO4FEm1VZFCAKIs7vZH9nScxun+/eYGfe3a02l6zny3mSeff3iykiOjjk\nRzrTBxKZd+pGG8lz3jEtBiHYAr3NrArXoEIUBei6gR/8YZ33yboIvS310rhZE83+2ccfVpPy+/mm\nImQO6Q8o7V7v9UTm3b+D9xGHmPUcIwf3zzX3RHRwYW/zDARBgF/s2Jc7ntzbXG6EIBjQWiogBJuh\nG6qzx7Wn8xoAASLu/8ZXsOaLCMaVj/Z878hJgzC2tgyVKTZc+NkNxyY6teWJMbVluPvq2RhY1r3g\nbQ+79/fg/c2zJuO8ligGlhVlfjARUQ8xeKfhDJsLqbt6JQ+bq4K5lE2PBiHpkpN5R2Kad/03zD2m\ny0sCWDTp/JQ/e0CaQJevexdXl3c/oLl7m/dnkigycBNRznDYPI3EUjEBmqF3+H6HLUFFa9vOmB/Q\nRWiGBkkUsLehvUPmDZ2nvavsXuH9PfMmIsolRpE03FXDWlKWDXirzQ3DgC5Za7jjPhi6BNWIQxIF\nGAY6ZN727mCUmZ1550vBGhFRLjCKpJEp845riYC8e387oJidwIy4HzBEqIaayBalpODP4N1lSp7M\neRMR5RKjSBruOW/N6Dzz/vmfNkFQYgCAUn8I0BKZNwAIHQrWGIi6yqdIkCUBRT6WZxAR2Ri803A3\nadFTDZu75rwjMRWCEoUiKvjhJbNRO6AUcT0OwT67ycPmev9fn91fyJKI755/GM4/aWxfHwoRUb/B\ndCYDM/NOVbCWCOiabkDyx1DmC6G02I/yYDF2RXRIkpW+JxesaflZNd5XDhle0deHQETUrzDzTkN3\nNWlJOeftWuet6ToMKWoOmQPwS+YabdGa6xaS57w1XjMREVH3MXin47RHFVLPebuGzXUhBggGQtbu\nXgEreAv2RiVi0rC51rHpCxERUVcxeKdhrxRLV7AW01QnOzdEs9K8WDG37fRbu3wJaTNvDpsTEVH3\nMXhnIKYpWPt8dyN+9Zy5B7kmmpXmQTt4S1ZmbQftDpk3h82JiKj7GLy7INWcN0QNb31sbt9pWMG7\nWDaDtzNszsybiIiygME7A90wUg6bu7umGZJZvNZh2FxUARgQ/OGkF2XmTURE3cfgnYFupMm8reBt\nGAYMKXnY3NoRTFQhDdoGsbjZ05iFTVqIiKgnGLwzMAwj5Zy3ORRuQDcMCLKdeZu7StnD5pBUSKX7\nAQCXTlqQk+MlIqKDH4N3BuaweYrMGwAkFf/e+SaU2q0AgGDSnLchqBCKWmDEFVQFB+bkeImI6ODH\n4J2BYaRYKmavAZdUPPnpaufu5DlvTYpADIRhREKQRc5zExFR72DwzkDXOxasybDntL33FyctFYvI\n9eY3wqEO+38TERF1F4N3BobRcT9vUU/MaeuRIud+RTSXgNnD5mFxHwBAiIYwpHgQJMOH+M4xEFiv\nRkREPcDgnYFhGNCT5rwF3cysBUmFEU0Eb8GKyvawuV1ULmh++CQfZukLoe4cl/2DJiKigxqDdwYp\nC9bsJiuSCgjmBPjo8Hzn285SMYtgPV4wmHITEVHPMXin8NH+TyH4zMYqqQrWjLgVjK3gbRgCSvUa\n5/uKKENxFagJujeYExER9QSDd5KWWCt++t5v4J/2CgAz8/7vnibPY1S79kxSIQgGYAiQRG9WHfKF\nnK9FnbuIERFR72HwTtIWbwcAp6hM1XTsaWjzPMauX7MzbxgCxKQzGfKVOF+LOnuZExFR72HwThLT\nY57bcVV35rVtumadNsnsXW4Gb++pLHUFbwHmELoB7+sQERF1B4N3koga8dyOxXVAMAvWoh8dgaJw\nLbR6c37bnXlLSeu/Qkpi2Dz5e0RERD3B4J2kPSl4x1XNybz15gEI7T0aajQAABCUqBO8haQzWepP\nBG8hKXgn3yYiIjoQ7NmZJBz3bt8Zs4bNDQMABLRH44Dqg6EqEAJt1lruFAVrimvOW2SwJiKi3sPM\nO0lY6zhsLgg6YJinqj1ilprr4WIIgTAEUYNhCB0CtGepGDNtIiLqRQzeSbyZt4G4pjtD4wAQjpql\n5kakGIJgQPBFAUPskHlLouR8bX+L5WpERNQbGLyTeDJvwUAsrgGCDlmUMLqmFLo5fg4jXJx4nCFA\nTMquJw+YABgCYtsmdPgeERFRTzB4JwnHXcFbVJ05b1EQ4VcS2bQeDSYel2LYPOQrwYTGi6DtGclh\ncyIi6lUM3knCqmvYXNSdanMR3uAN3fV1ig5rAKwit8SwORERUW9g8E4Sdi0VEyTVWectQoLf5w7Y\n7lPXMfMG4AyxC4zeRETUixi8k3gzbw2abkBwhs1dp8u9Q1iKOW8gEbyd2M2KNSIi6gUM3kncTVoE\n0W5ibgZvn2vY3NATpy7VUjEAmDKyEgBw2NiBnvs5BU5ERD3BJi1JYpqrt7lkB28doiBBkdJn3qnm\nvOfOHIZDhldgWHVJh+8RERF1F4N3kqh7Y5KkzFv2BG9vIE+VeYuCgBGDQx3uJyIi6gkGbxfDMBDX\n4s7txLC5DkmQkoK3O1innvMmIiLKBs55u8R11bttp5TIvCVBhCy5h8q9WXiqYfNkrFcjIqLewODt\nYs93S4JZmCaIGiCqEATAJ/o9mbe7YC3dsHk6zNGJiKgnGLxdolbwDohW9zRRg+Azq89LlNABF6wl\nG1xpvu7omrLeOWAiIipInPN2iVvFakVSEG1aCyCp5sYjAEqVEGQxdcGakWadd7KTZtSiOCBj+riB\nGR9LRESUDoO3i5N5C2aGLEgaBMXMvEt9pZCTsm33110ZNpclEXMOHdJ7B0xERAWJw+YuMavS3O8M\nm6vOsHmZrzT9UrE07VGJiIiyIWPw3rp1ay6Oo1+IWcPmPqMIgJV5W8Pm5f4yyHLP5ryJiIh6Q8bg\n/e1vfxsXXHABVq1ahXA4nOnhec0eNldgBm9zztvMvCuKyrwFa8jc25yIiCgbMs55P//88/jkk0/w\n4osvYuHChZg4cSLOPfdcTJ06NRfHl1N2gxZBV2BoIgRJBXwRGLqIkFKMqBRJ/URD5LA5ERHlTJfm\nvMePH4/rr78eS5YswdatW7F48WJcdNFF+O9//5vlw8stO/OGIQG6DLG4GWKgHdq+wVCUpA5rbhw2\nJyKiHMqYee/cuRN/+tOf8Je//AVjx47F1VdfjWOPPRYbN27EzTffjGeeeSYXx5kT9py3oUkwNAmC\nYt6v7hwHRfL2Nvd0WwOYeRMRUc5kDN4LFy7E17/+dfzhD3/AoEGDnPunTp2aceh8xYoVeP/99yEI\nApYuXep5/K5du/Cd73wH8XgckyZNwp133tmDt9E77A5rhi4CmnlqDEOAEQtAlrztURXZvc5b5Jw3\nERHlTMZh8zVr1mDkyJFO4H7iiSfQ1tYGALj99tvTPu/NN9/Etm3b8NRTT2H58uVYvny55/t33303\nLr/8cjz77LOQJAlffvllT95Hr7CXihmqBEO39u6OKwAESJLgqTZP7rbGYXMiIsqVjMH7e9/7Hurr\n653bkUgEt9xyS8YXXrduHebOnQsAGDNmDJqamtDa2goA0HUdb7/9Nk466SQAwLJly1BTU9OtN9Cb\n7DlvXXNl3roMSTSryd0BO3nZGIfNiYgoVzIG78bGRixatMi5fdlll6G5uTnjC9fX16OiosK5XVlZ\nibq6OgDA/v37UVxcjLvuugsXXHAB7r///u4ce6+z57x1VYSzFEyTnEDtnvNOzrwZvImIKFcyznnH\n43Fs3boVY8aMAQBs2rQJ8Xg8w7M6MgzD8/WePXuwaNEi1NbW4qqrrsLLL7+ME044Ie3zKyqCkGXp\ngH9uZ6qqQp7bwqfmMUqiD7D28jZ0CQFFQlVVCEUlifddFFDgXMIYAgYOKO7weoWiUN93b+I57Dme\nw97B89hzuTiHGYP39773PSxevBgtLS3QNA2VlZW49957M75wdXW1Z7h97969qKqqAgBUVFSgpqYG\nw4cPBwDMnj0bn376aafBu6GhPePPPBBVVSHU1bV47mtpN39GuN2A4Lf28tYlSKKAuroWxOJa4sGu\nixEYApoa2+EvwOQ71XmkA8Nz2HM8h72D57HnevscprsQyDhsPm3aNKxduxbPP/881q5dixdffLFL\nmfecOXOwdu1aAMDmzZtRXV2NkpISAIAsyxg2bJizTnzz5s0YNWpUV99L1tjV5mocTuYNXXIqy93z\n3JJnqRg7rBERUe5kzLxbW1vx5z//GQ0NDQDMYfRVq1bhtdde6/R5M2bMwOTJk7FgwQIIgoBly5Zh\n9erVCIVCmDdvHpYuXYolS5bAMAyMHz/eKV7rS1E9BlmUoaqAPedtqDJ8VtB2B2jJ9bVhCDBARESU\nGxmD9w033ICamhq89tpr+MpXvoLXX38dP/jBD7r04jfddJPn9oQJE5yvR4wYgSeeeOLAjjbL4loc\nPlFBXNUhfjEdyvCPEd5+CJSqjgMU7gI1QTAYvImIKGcyDptHo1HceeedqK2txa233orHHnsML774\nYi6OLeeiWgw+yYeYqkNRy1C291hA9UNJUSgnJbdKNRi+iYgoNzIG73g8jvb2dui6joaGBpSXl2P7\n9u25OLaci2kx+CQz81ZkCbpuBmR3NzWbuymLJAuoCAVydpxERFTYMg6bn3XWWXj66adx7rnn4rTT\nTkNlZSVGjBiRi2PLuZgeQ7lYigZVQzCgQNV0AHDmvN3cwfvsY0alDPBERETZkDF42wVngLmka9++\nfZg4cWLWDyzXDMNATIvDJ/kQ13Qosoj2iAogc+bNGW8iIsqljOmiu7vaoEGDMGnSJCeYH0xUXYUB\nwwzeqg6fLELVzcxbSbEVqOgJ3kRERLmTMfOeOHEifvKTn2D69OlQFMW5f/bs2Vk9sFyLWq1RFVGB\nqhlQZBGaZs15KykK1kT3rmIM30RElDsZg/eHH34IAHjrrbec+wRBOOiCt92gRbY28VZkyZnzTpV5\ne4fN9RwcIRERkSlj8H788cdzcRx9zt4ONBG8RSd4y3IiUFeVB1DXGPEOmzPzJiKiHMoYvC+88MKU\nc9wrV67MygH1leTM2yeLUK1hc9k1RL78yqMQi2t4+p9bnftYsEZERLnUpQ5rtng8jvXr1yMYDGb1\noPqCvZe3CHN+293HXHb1MZcl0dkaVI8UQQyEUSQX5fBIiYio0GUM3rNmzfLcnjNnDq688sqsHVBf\nienmsLmExLC5rUM3Nfs5Hx+B4NCdOPb4g2v+n4iI+reMwTu5m9quXbvw+eefZ+2A+krMybzNU+Ju\nzCKLqZbGGTCixZD3TIFPUlJ8n4iIKDsyBu9LLrnE+VoQBJSUlOC6667L6kH1BSd4GzIA3ZN5y510\nTzv4VrwTEVF/lzF4/+Mf/4Cu6xCtoq14PO5Z732wiOnu4B3zbEYipxk2JyIi6gsZo9LatWuxePFi\n5/ZFF12El156KasH1RfsgjUY5ilxr+2WUgybc3UYERH1lYzB+9FHH8WPf/xj5/bvfvc7PProo1k9\nqL4Qt9Z5Q7fmvBV3wVr6wfGDsVUsERH1bxmDt2EYCIVCzu2SkpKDMmBFtCgAQLCCtzvzdq/ztjHx\nJiKivpJxznvKlCm44YYbMGvWLBiGgVdffRVTpkzJxbHllB287cxbUdzrvDnnTURE/UfG4H3bbbdh\nzZo12LBhAwRBwJlnnolTTjklF8eWU1HVCt6anXm7C9YOvpEGIiLKXxmDdzgchqIouP322wEATzzx\nBMLhMIqLi7N+cLlkZ96GZgZt91KxUNDX4fFVZQEAQG3VwXUeiIio/8s4Hnzrrbeivr7euR2JRHDL\nLbdk9aD6gp1566oZvH2yiOVXHolLTjkEIwaHOjz+lCOH44KTx+HKMybl9DiJiIgyBu/GxkYsWrTI\nuX3ZZZehubk5qwfVFyJaBIqoQNPM24osYsiAYhx/WG3KxyuyhHkzh6XMyomIiLIpY/COx+PYujWx\ng9bGjRsRj8ezelB9IaJFEZD8iKvWHt6ddFUjIiLqSxnnvL/3ve9h8eLFaGlpga7rqKiowL333puL\nY8upqBpFQPYjxuBNRET9XMYINW3aNKxduxarVq3CkiVLUF1djWuuuSYXx5ZTyZm3z9UelYiIqD/J\nmHm/9957WL16NV544QXouo4f/ehHmD9/fi6OLWd0Q0dUi8Ev+xHXmHkTEVH/ljZC/eY3v8Fpp52G\nG2+8EZWVlVi1ahWGDx+O008//aDbmMTuax6Q/IjHzYo1Bm8iIuqv0mbeDz74IMaOHYs77rgDRx11\nFICDt4931FrjHZADaGPmTURE/Vza4P3yyy/jT3/6E5YtWwZd13H22WcflFXmABCx1nj7JbNgTRBS\n7yRGRETUH6RNL6uqqnDVVVdh7dq1WLFiBb744gvs3LkTV199NV555ZVcHmPWOZm3VbDmk6WDdpSB\niIjyX5fGhmfOnIm7774br776Kk444QT8/Oc/z/Zx5VRYjQCAWbCm6hwyJyKifu2AolRJSQkWLFiA\np59+OlvH0ye8mbfG4E1ERP0aoxSA9ngYAFAkB9DcHkdx4OCqpiciooMLgzeAdtUM3qLuRzSmoao8\n0MdHRERElB6DN4D2eDsAIBoxT0dVeVFfHg4REVGnGLwBtFmZd6SNwZuIiPo/Bm8kMu+WVvM2gzcR\nEfVnDN5IzHk3NZnd1TjnTURE/RmDN4C2eDsUUUFLmxm8K0L+Pj4iIiKi9Bi8YQ6bFytBRGPmpiQ+\nhduBEhFR/8XgDXPYPCgXIRLX4FNEiGyNSkRE/VjBB2/d0BFWIwgqRYjFNfiZdRMRUT9X8ME7rEZg\nwECxHEQ6NoGoAAAYmElEQVSUwZuIiPIAg7ddad6so6E5Cr+PwZuIiPq3gg/eMc3co3zL9jYYADNv\nIiLq9wo+eMd1M3gbunkqGLyJiKi/Y/DWVfMLBm8iIsoTDN5W5g3dDNo+peBPCRER9XMFH6ni1pw3\nDPNUBFiwRkRE/RyDtzPnbWfeDN5ERNS/MXhzzpuIiPIMg7cz583gTURE+YHBW/MOmzN4ExFRf5fV\n4L1ixQqcf/75WLBgATZs2JDyMffffz8WLlyYzcPolDNsbhWsiSI3JSEiov4ta8H7zTffxLZt2/DU\nU09h+fLlWL58eYfHbNmyBf/5z3+ydQhdkrxUTNP0PjwaIiKizLIWvNetW4e5c+cCAMaMGYOmpia0\ntrZ6HnP33XfjxhtvzNYhdEksqcOapht9eThEREQZZS1419fXo6KiwrldWVmJuro65/bq1asxa9Ys\n1NbWZusQukR1qs3NzLu4SOnDoyEiIspMztUPMoxERtvY2IjVq1fj0UcfxZ49e7r0/IqKIGS5d4vJ\nqqpCkD63bugi5h85Al89aTwkznsfkKqqUF8fQt7jOew5nsPewfPYc7k4h1kL3tXV1aivr3du7927\nF1VVVQCA9evXY//+/bjooosQi8XwxRdfYMWKFVi6dGna12toaO/V46uqCqGurgXN7ebrGrqEuTNq\nsH9fa4Znkpt9Hqn7eA57juewd/A89lxvn8N0FwJZGzafM2cO1q5dCwDYvHkzqqurUVJSAgA45ZRT\n8MILL+Dpp5/Gz372M0yePLnTwJ1NqqvaXBILfuUcERHlgaxl3jNmzMDkyZOxYMECCIKAZcuWYfXq\n1QiFQpg3b162fuwBi7matMgSh8uJiKj/y+qc90033eS5PWHChA6PGTp0KB5//PFsHkannI1JdAmy\nxMybiIj6v4KPVqquWg1aBBaqERFRXij44B3T4xAMs4qdmTcREeWDgo9WcSt4CwJboxIRUX5g8NZU\nCKw0JyKiPFLwESuuxwFDYqU5ERHlDQZvPW4tEyv4U0FERHmioCOWYRiIaXFAl1lpTkREeaOgg7dq\naDBgwGCDFiIiyiMFHbxjWsz8QpcgcdiciIjyREFHLDt4G5rEYXMiIsobhR28rb7mhsaCNSIiyh8F\nHbHszFtn5k1ERHmkwIM3M28iIso/BR2xYnoi82a1ORER5YvCDt4sWCMiojxU4MHb3stb5FIxIiLK\nGwUdsRLrvGXOeRMRUd4o6IjlLBXTRQ6bExFR3ijs4O3qsMaCNSIiyhcM3gCgSdzPm4iI8kZBR6zE\nsLkERSnoU0FERHmkoCOWe9hcYcEaERHliYKOWFFnqZgEHzNvIiLKEwUdseJWhzWDmTcREeWRgo5Y\nMVfmrchS3x4MERFRFxV08I46c94iFLmgTwUREeWRgo5Yqq5CggRAgI/Bm4iI8kRBRyzVUCEKMgAw\n8yYiorxR0BErrschwpzrZvAmIqJ8UdARK66pruDNgjUiIsoPBR28VUOFYJingJk3ERHli4KOWKqu\nQrAybxasERFRvijoiBXXVQgG57yJiCi/FGzEMgzDzLw5bE5ERHmmYCOWqqvmF8y8iYgozxRsxIpr\nVvDW7cyb1eZERJQfCjd423t5W8PmLFgjIqJ8UbARy868DZ1z3kRElF8KNmLFrMwbmghBACRR6NsD\nIiIi6qKCDd6qlXnrugBFFiEIDN5ERJQfCjZ423t5G5oIRSrY00BERHmoYKOWXbCmaQJ8CivNiYgo\nfxRu8LaHzTWBmTcREeWVgo1acd0O3iIUpWBPAxER5aGCjVpxa85bUwXIYsGeBiIiykMFG7Xcw+ay\nzEpzIiLKH4UbvK2CNV0TITHzJiKiPFKwUcvpbW6IkCVm3kRElD8KN3jbvc11ETKrzYmIKI8UbNSy\nm7TAENkalYiI8krBBm9nP29dhMTMm4iI8kjBRq2Ys6uYBJmZNxER5ZGCDd5x97A5C9aIiCiPyNl8\n8RUrVuD999+HIAhYunQppk6d6nxv/fr1eOCBByCKIkaNGoXly5dDzOGSrYgaNb/QJBasERFRXsla\n1HrzzTexbds2PPXUU1i+fDmWL1/u+f4dd9yBhx56CE8++STa2trw6quvZutQUgqrEQCAocssWCMi\norySteC9bt06zJ07FwAwZswYNDU1obW11fn+6tWrMXjwYABAZWUlGhoasnUoKUXiZvCGJjPzJiKi\nvJK1qFVfX4+KigrndmVlJerq6pzbJSUlAIC9e/fi9ddfx/HHH5+tQ0kprEYhQLCqzZl5ExFR/sjq\nnLebYRgd7tu3bx+uvvpqLFu2zBPoU6moCEKWe2/f7XA8Ap/kRzsElJYEUFUV6rXXLjQ8dz3Hc9hz\nPIe9g+ex53JxDrMWvKurq1FfX+/c3rt3L6qqqpzbra2tuPLKK3HDDTfgmGOOyfh6DQ3tvXp8YTUC\nBQoAIBqNo66upVdfv1BUVYV47nqI57DneA57B89jz/X2OUx3IZC1YfM5c+Zg7dq1AIDNmzejurra\nGSoHgLvvvhuXXHIJjjvuuGwdQqci8QgU0QcALFgjIqK8krXMe8aMGZg8eTIWLFgAQRCwbNkyrF69\nGqFQCMcccwyee+45bNu2Dc8++ywA4IwzzsD555+frcPpIKxGUSGXAgAL1oiIKK9kdc77pptu8tye\nMGGC8/WmTZuy+aM7FddVqLoKQTffPoM3EVHfevnlv+OEE07u0mN/8pP7ce65C1BTU5vlo+q/CjJq\nRa0GLbv2xgBw2JyIqC/t2vUl/va3tV1+/PXXf7egAzeQw2rz/iSimcHb0MzqdS4VIyLqOw88cA8+\n/HAzHn30N9B1HV9+uRO7dn2JBx/8Be66607U1e1FOBzG5ZdfhTlzjsV1112F73znFvzzn39HW1sr\nvvhiG3bu3IFvf/u7mD17jvO6qqpi+fIfdHj+J598hPvvvweiKGDKlGm49trrU95n/5zRo8di1aqn\n0NjYiOnTD8eTT/4v2tvbcd11N+Ldd9/Gyy//HbquY/bsObj11u+ipaUFd955G9ra2lBSUoI77vgf\nXH75Rfj9759AMBjEhg3v4cknV2LFih93+5wVZPCOWsEb9rB5DtuyEhH1Z0//Ywv+89HeXn3NmROq\ncd5JY9N+/4ILFmL16qdx2WVX4pFHHoaqxvGLX/wWDQ37MWvWUTj11DOwc+cO3H77EsyZc6znuXv3\n7sF99z2E9ev/jT//eZUneLe0NKd8/oMP3oebb16KsWPH4Uc/ugO7d+9KeV86W7duwRNPrIbP58O7\n776NX/zitxBFEeeddxauvfabeOKJxzFr1myce+4CPPXUSrzzzls47rgT8dpr/8L8+afgtddewbx5\nX+nROS3I4G33NTc08+0z8yYi6j8mTpwMAAiFSvHhh5uxZs1qCIKI5uamDo+dOvUwAObyZHcXz86e\n/8UX2zB27DgAwO2335n2vnTGjh0Hn89crRQIBHDddVdBkiQ0NjaisbERn3zyEa644hoAwPnnXwQA\nqKmpxW9/+0vMn38K3n33bXzjG1cf+IlxKczgrSU2JQFYsEZEZDvvpLGdZsm5oChmD46//vUlNDc3\n4+c//y2am5txxRULOzxWkhLNu5KbgaV7fqpNsFLdJwiJxE5V1Q7Ht3v3Ljz11Er87ncrEQwGsXDh\nedZrSTAM3fNaY8eOw759+/Dhh5sxatQY+P3+zk9CBgUZtSL2piR25s2CNSKiPiOKIjRN63B/Y2Mj\nhgypgSiKeOWVfyAejx/Q66Z7/siRo7B5s7ni6a677sR///t5yvuKi4uxb5/ZbGzjxvdTvn5FRQWC\nwSA+/vgj7N69G/F4HBMnTsLbb/8HAPDcc6vw4ot/AQCcdNI8PPDAPZg375QDeh+pFGTwthlx88qH\nmTcRUd8ZMWIUPv74Izz00P2e+0844ST8+9+v4vrrr0FRURGqq6vx6KO/6fLrpnv+9dffhJ/97P/D\nNdd8A6FQKUaOHJXyvjPPPAf3338vbr75egwcWNXh9ceNG4+ioiCuueZy/P3v/4ezzjoHP/zhD3Hu\nuRdg06YNuO66q/Dvf7+G448/EQBw8snzsHfvXhx++MyenTAAgpGq6Xg/1Jvt5uJaHNf89hnojdWA\nIeKWC6ZjwojOe6tTamyn2HM8hz3Hc9g7eB57rrNz+Pzza7B79y584xvfPKDXS6Ug57wVSYHeMNi5\nzcybiIiy6Z57/gdffrkTd911X6+8XkEG72SsNiciomy69dbbevX1CjLl1HXvTAEL1oiIKJ8UZPCO\nxr1VjRw2JyKifFKQUSvWIXgz8yYiovxRkME7OfOW2B6ViIjySEFGrWjc2/mGmTcRUd96+eW/H/Bz\n3nvvHTQ07M/C0fR/hRm8Y0mZN+e8iYj6zIFuCWp7/vk1BRu8C3KpWMdhc2beRER9xb0l6PnnX4gV\nK36IlpYWaJqGG264GWPHjsP//u/v8cor/4Qoipgz51hMnDgJr776Mj7//DP8z//ci8GDzd4dfbEN\n6OWXX+VsAxqLReD3F2VlG1A3Bm+w2pyIyLZ6y1/w7t6Nvfqa06sPxTljz0j7ffeWoL///W9x5JFH\n4//9v6/i888/w09+ch8efPAXePLJ/8Vzz70ESZLw3HOrMHPmURg7djy+851bnMAN9M02oOeff6Gz\nDejixVfiZz/7VVa2AXVj8AabtBAR9RcbN25AY2MD1q59AQAQjZobSZ1wwsm44YbFmDfvFMyfn35j\nj77YBrS5uTkn24C6FWTwrgz54ZNF6IYBVTMgCgzeREQAcM7YMzrNkrNNUWTceOPNmDJlquf+m276\nHrZt+y/+8Y+/4lvf+iZ+/es/pHz+wbwNqOfYe+2V8sghwyvw1IrT8fBNJ+DXN5/Q14dDRFTQ3FuC\nTpo0Bf/618sAgM8//wxPPvm/aG1txaOP/gYjRozEZZddiVCoDO3tbSm3Ej2YtwH1nLNefbU8Iksi\nBEHgfDcRUR9zbwn69a+fj507t2Px4itwzz3/g8MOm4GSkhI0NjbgyisX4dvfvhqTJ09BaWkZDjts\nBm677VZ89tlW57X6YhvQ+++/x9kGdOHChVnbBtStILcEBbj1XW/heew5nsOe4znsHTyPPZd8Druz\nDWjy66VSkHPeRERE2dbb24C6MXgTERFlQW9vA+rGCV8iIqI8w+BNRESUZxi8iYiI8gyDNxERUZ5h\n8CYiIsozDN5ERER5hsGbiIgozzB4ExER5Zm8aY9KREREJmbeREREeYbBm4iIKM8weBMREeUZBm8i\nIqI8w+BNRESUZxi8iYiI8kxB7ue9YsUKvP/++xAEAUuXLsXUqVP7+pD6tU8++QSLFy/GpZdeiosv\nvhi7du3CLbfcAk3TUFVVhR//+Mfw+XxYs2YN/vCHP0AURZx33nk499xz+/rQ+417770Xb7/9NlRV\nxTe/+U0ceuihPIcHIBwOY8mSJdi3bx+i0SgWL16MCRMm8Bx2UyQSwRlnnIHFixdj9uzZPI8H4I03\n3sD111+PcePGAQDGjx+PK664Ivfn0Cgwb7zxhnHVVVcZhmEYW7ZsMc4777w+PqL+ra2tzbj44ouN\n2267zXj88ccNwzCMJUuWGC+88IJhGIZx//33GytXrjTa2tqM+fPnG83NzUY4HDZOP/10o6GhoS8P\nvd9Yt26dccUVVxiGYRj79+83jj/+eJ7DA/T8888bv/71rw3DMIwdO3YY8+fP5znsgQceeMA455xz\njFWrVvE8HqD169cb3/rWtzz39cU5LLhh83Xr1mHu3LkAgDFjxqCpqQmtra19fFT9l8/nw29+8xtU\nV1c7973xxhs4+eSTAQAnnngi1q1bh/fffx+HHnooQqEQAoEAZsyYgXfeeaevDrtfmTlzJn7yk58A\nAEpLSxEOh3kOD9Bpp52GK6+8EgCwa9cuDBo0iOewm7Zu3YotW7bghBNOAMD/z72hL85hwQXv+vp6\nVFRUOLcrKytRV1fXh0fUv8myjEAg4LkvHA7D5/MBAAYMGIC6ujrU19ejsrLSeQzPa4IkSQgGgwCA\nZ599FscddxzPYTctWLAAN910E5YuXcpz2E333HMPlixZ4tzmeTxwW7ZswdVXX40LLrgAr7/+ep+c\nw4Kc83Yz2B22R9KdP57Xjv72t7/h2Wefxe9+9zvMnz/fuZ/nsOuefPJJfPjhh7j55ps954fnsGue\ne+45HHbYYRg2bFjK7/M8ZjZy5Ehcd911OPXUU7F9+3YsWrQImqY538/VOSy44F1dXY36+nrn9t69\ne1FVVdWHR5R/gsEgIpEIAoEA9uzZg+rq6pTn9bDDDuvDo+xfXn31VfzqV7/Cb3/7W4RCIZ7DA7Rp\n0yYMGDAAQ4YMwcSJE6FpGoqLi3kOD9DLL7+M7du34+WXX8bu3bvh8/n4t3iABg0ahNNOOw0AMHz4\ncAwcOBAbN27M+TksuGHzOXPmYO3atQCAzZs3o7q6GiUlJX18VPnl6KOPds7h//3f/+HYY4/FtGnT\nsHHjRjQ3N6OtrQ3vvPMOjjjiiD4+0v6hpaUF9957Lx5++GGUl5cD4Dk8UG+99RZ+97vfATCnvtrb\n23kOu+HBBx/EqlWr8PTTT+Pcc8/F4sWLeR4P0Jo1a/DII48AAOrq6rBv3z6cc845OT+HBbmr2H33\n3Ye33noLgiBg2bJlmDBhQl8fUr+1adMm3HPPPdi5cydkWcagQYNw3333YcmSJYhGo6ipqcFdd90F\nRVHw0ksv4ZFHHoEgCLj44otx5pln9vXh9wtPPfUUfvrTn2LUqFHOfXfffTduu+02nsMuikQi+P73\nv49du3YhEonguuuuw5QpU3DrrbfyHHbTT3/6U9TW1uKYY47heTwAra2tuOmmm9Dc3Ix4PI7rrrsO\nEydOzPk5LMjgTURElM8KbticiIgo3zF4ExER5RkGbyIiojzD4E1ERJRnGLyJiIjyTME1aSHKN/fe\ney82btyIaDSKDz74ANOnTwcAfO1rX8NXv/rVLr3Gr3/9a4wfP97pZ53KwoUL8fvf/x6SJPXGYXvs\n2bMHn332GWbPnt3rr01UiLhUjChP7NixAxdeeCH+9a9/9fWhHLA1a9Zg69atuPHGG/v6UIgOCsy8\nifLYT3/6U+zYsQNffvklbr31VkQiEdx3333w+XyIRCJYtmwZJk+ejCVLluDwww/H7Nmzcc011+CY\nY47Bhg0b0NbWhocffhiDBg3CIYccgs2bN+OXv/wlGhsbsXv3bmzbtg1HHnkkbr/9dkSjUdx6663Y\nuXMnBg8eDEmSMGfOHM8exW1tbfjud7+L5uZmqKqKE088EWeccQYefPBBGIaB8vJyXHTRRbjzzjux\nbds2tLW14YwzzsDll1+O1atX469//SsEQcCePXswevRorFixAoqi9OEZJuqfOOdNlOd27NiBxx57\nDFOmTEFjYyN+8IMf4LHHHsOiRYvw8MMPd3j81q1bcc4552DlypWYOHEiXnzxxQ6P+eCDD/DQQw/h\n2WefxerVq9HU1IQ1a9ZAVVU888wzuOOOO/D66693eN6///1vqKqKP/7xj3jyyScRDAZRW1uLs88+\nG2eeeSYuu+wyPPbYY6iursbjjz+OZ555Bs8//zw++ugjAMDGjRv///bu2CW1MIzj+NcONQQRQi3W\nYnBsjDoSBFKNOVaEo0M4REO4HGyrKQin5ob+gDBaoiVyECEipakhWkKkQKFoiERPd5DOzYxLlysX\njvw+4+F5X97tx/PyHh7S6TSHh4eUy2VP3jKI/A/qvEU8bmJiAp/PB8DQ0BC7u7u8vb3x8vLC4OBg\nW73f78c0TQACgQBPT09tNZZlYRgGhmHg9/t5fn7m5uaG6elpAIaHh7Esq23d1NQUe3t7bGxsMDc3\nx8rKCj09rT3CxcUFDw8PXF5eAlCr1bi/v3fXf4xPnZyc5O7uzp2TLCK/KbxFPO7ztbJt22xvbzMz\nM8P5+bk7zOOzrw/Svnv28l2N4zgtQfw1lKE5y/j4+JhiscjZ2RnLy8scHR211PT19bG+vs7CwkLL\n90wmg+M4fzyXiDTp2lyki1QqFUzTpNFocHp6Sq1W69jeY2NjFItFAKrVKldXV201uVyObDaLZVnY\ntk1/fz/VahWfz0e9XgeaXf3HVb3jOOzs7Ljd//X1Na+vr7y/v1MoFBgfH+/Y+UW6iTpvkS6SSCSI\nx+MEAgFWV1exbZuDg4OO7L20tEQ2myUWizE6Oko4HG7r0IPBIKlUiv39fQzDIBKJMDIyQjgcJplM\n0tvby9raGre3t8RiMRqNBvPz8+6o1FAoxObmJqVSCdM0iUQiHTm7SLfRr2Ii8iOPj48UCgWi0SiO\n47C4uMjW1pb73/m/ymQy5PN50ul0R/YT6WbqvEXkRwYGBjg5OXHnE8/OznYsuEXk76jzFhER8Rg9\nWBMREfEYhbeIiIjHKLxFREQ8RuEtIiLiMQpvERERj1F4i4iIeMwvRph4T/csGFUAAAAASUVORK5C\nYII=\n",
+            "text/plain": [
+              "<matplotlib.figure.Figure at 0x7f72f867ef90>"
+            ]
+          },
+          "metadata": {
+            "tags": []
+          }
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "HNqUFL4deCsL",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# 4. Case study: building an RNN\n"
+      ]
+    },
+    {
+      "metadata": {
+        "id": "YkC1k4HEQ7rw",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "In this exercise we build and train a model similar to the RNNColorbot model that was used in the main Eager notebook. The model is adapted for converting and training in graph mode."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "7nkPDl5CTCNb",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "To get started, we load the colorbot dataset. The code is identical to that used in the other exercise and its details are unimportant."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "A0uREmVXCQEw",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def parse(line):\n",
+        "  \"\"\"Parses a line from the colors dataset.\n",
+        "  \n",
+        "  Args:\n",
+        "    line: A comma-separated string containing four items:\n",
+        "        color_name, red, green, and blue, representing the name and\n",
+        "        respectively the RGB value of the color, as an integer\n",
+        "        between 0 and 255.\n",
+        "\n",
+        "  Returns:\n",
+        "    A tuple of three tensors (rgb, chars, length), of shapes: (batch_size, 3),\n",
+        "    (batch_size, max_sequence_length, 256) and respectively (batch_size).\n",
+        "  \"\"\"\n",
+        "  items = tf.string_split([line], \",\").values\n",
+        "  rgb = tf.string_to_number(items[1:], out_type=tf.float32) / 255.0\n",
+        "  color_name = items[0]\n",
+        "  chars = tf.one_hot(tf.decode_raw(color_name, tf.uint8), depth=256)\n",
+        "  length = tf.cast(tf.shape(chars)[0], dtype=tf.int64)\n",
+        "  return rgb, chars, length\n",
+        "\n",
+        "\n",
+        "def maybe_download(filename, work_directory, source_url):\n",
+        "  \"\"\"Downloads the data from source url.\"\"\"\n",
+        "  if not tf.gfile.Exists(work_directory):\n",
+        "    tf.gfile.MakeDirs(work_directory)\n",
+        "  filepath = os.path.join(work_directory, filename)\n",
+        "  if not tf.gfile.Exists(filepath):\n",
+        "    temp_file_name, _ = six.moves.urllib.request.urlretrieve(source_url)\n",
+        "    tf.gfile.Copy(temp_file_name, filepath)\n",
+        "    with tf.gfile.GFile(filepath) as f:\n",
+        "      size = f.size()\n",
+        "    print('Successfully downloaded', filename, size, 'bytes.')\n",
+        "  return filepath\n",
+        "\n",
+        "\n",
+        "def load_dataset(data_dir, url, batch_size, training=True):\n",
+        "  \"\"\"Loads the colors data at path into a tf.PaddedDataset.\"\"\"\n",
+        "  path = maybe_download(os.path.basename(url), data_dir, url)\n",
+        "  dataset = tf.data.TextLineDataset(path)\n",
+        "  dataset = dataset.skip(1)\n",
+        "  dataset = dataset.map(parse)\n",
+        "  dataset = dataset.cache()\n",
+        "  dataset = dataset.repeat()\n",
+        "  if training:\n",
+        "    dataset = dataset.shuffle(buffer_size=3000)\n",
+        "  dataset = dataset.padded_batch(batch_size, padded_shapes=([None], [None, None], []))\n",
+        "  return dataset\n",
+        "\n",
+        "\n",
+        "train_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/train.csv\"\n",
+        "test_url = \"https://raw.githubusercontent.com/random-forests/tensorflow-workshop/master/extras/colorbot/data/test.csv\"\n",
+        "data_dir = \"tmp/rnn/data\""
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "waZ89t3DTUla",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Next, we set up the RNNColobot model, which is very similar to the one we used in the main exercise.\n",
+        "\n",
+        "Autograph doesn't fully support classes yet (but it will soon!), so we'll write the model using simple functions."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "9v8AJouiC44V",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def model_components():\n",
+        "  lower_cell = tf.contrib.rnn.LSTMBlockCell(256)\n",
+        "  lower_cell.build(tf.TensorShape((None, 256)))\n",
+        "  upper_cell = tf.contrib.rnn.LSTMBlockCell(128)\n",
+        "  upper_cell.build(tf.TensorShape((None, 256)))\n",
+        "  relu_layer = tf.layers.Dense(3, activation=tf.nn.relu)\n",
+        "  relu_layer.build(tf.TensorShape((None, 128)))\n",
+        "  return lower_cell, upper_cell, relu_layer\n",
+        "\n",
+        "\n",
+        "def rnn_layer(chars, cell, batch_size, training):\n",
+        "  \"\"\"A simple RNN layer.\n",
+        "  \n",
+        "  Args:\n",
+        "    chars: A Tensor of shape (max_sequence_length, batch_size, input_size)\n",
+        "    cell: An object of type tf.contrib.rnn.LSTMBlockCell\n",
+        "    batch_size: Int, the batch size to use\n",
+        "    training: Boolean, whether the layer is used for training\n",
+        "\n",
+        "  Returns:\n",
+        "    A Tensor of shape (max_sequence_length, batch_size, output_size).\n",
+        "  \"\"\"\n",
+        "  hidden_outputs = []\n",
+        "  autograph.utils.set_element_type(hidden_outputs, tf.float32)\n",
+        "  state, output = cell.zero_state(batch_size, tf.float32)\n",
+        "  n = tf.shape(chars)[0]\n",
+        "  i = 0\n",
+        "  while i < n:\n",
+        "    ch = chars[i]\n",
+        "    cell_output, (state, output) = cell.call(ch, (state, output))\n",
+        "    hidden_outputs.append(cell_output)\n",
+        "    i += 1\n",
+        "  hidden_outputs = hidden_outputs.stack()\n",
+        "  if training:\n",
+        "    hidden_outputs = tf.nn.dropout(hidden_outputs, 0.5)\n",
+        "  return hidden_outputs\n",
+        "\n",
+        "\n",
+        "def model(inputs, lower_cell, upper_cell, relu_layer, batch_size, training):\n",
+        "  \"\"\"RNNColorbot model.\n",
+        "  \n",
+        "  The model consists of two RNN layers (made by lower_cell and upper_cell),\n",
+        "  followed by a fully connected layer with ReLU activation.\n",
+        "  \n",
+        "  Args:\n",
+        "    inputs: A tuple (chars, length)\n",
+        "    lower_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n",
+        "    upper_cell: An object of type tf.contrib.rnn.LSTMBlockCell\n",
+        "    relu_layer: An object of type tf.layers.Dense\n",
+        "    batch_size: Int, the batch size to use\n",
+        "    training: Boolean, whether the layer is used for training\n",
+        "    \n",
+        "  Returns:\n",
+        "    A Tensor of shape (batch_size, 3) - the model predictions.\n",
+        "  \"\"\"\n",
+        "  (chars, length) = inputs\n",
+        "  chars_time_major = tf.transpose(chars, [1, 0, 2])\n",
+        "  chars_time_major.set_shape((None, batch_size, 256))\n",
+        "\n",
+        "  hidden_outputs = rnn_layer(chars_time_major, lower_cell, batch_size, training)\n",
+        "  final_outputs = rnn_layer(hidden_outputs, upper_cell, batch_size, training)\n",
+        "\n",
+        "  # Grab just the end-of-sequence from each output.\n",
+        "  indices = tf.stack([length - 1, range(batch_size)], axis=1)\n",
+        "  sequence_ends = tf.gather_nd(final_outputs, indices)\n",
+        "  return relu_layer(sequence_ends)\n",
+        "\n",
+        "def loss_fn(labels, predictions):\n",
+        "  return tf.reduce_mean((predictions - labels) ** 2)"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "JjK4gXFvFsf4",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "The train and test functions are also similar to the ones used in the Eager notebook. Since the network requires a fixed batch size, we'll train in a single shot, rather than by epoch."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "ZWQMExk0S6X6",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n",
+        "  iterator = train_data.make_one_shot_iterator()\n",
+        "  step = 0\n",
+        "  while step < num_steps:\n",
+        "    labels, chars, sequence_length = iterator.get_next()\n",
+        "    predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=True)\n",
+        "    loss = loss_fn(labels, predictions)\n",
+        "    optimizer.minimize(loss)\n",
+        "    if step % (num_steps // 10) == 0:\n",
+        "      print('Step', step, 'train loss', loss)\n",
+        "    step += 1\n",
+        "  return step\n",
+        "\n",
+        "\n",
+        "def test(eval_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps):\n",
+        "  total_loss = 0.0\n",
+        "  iterator = eval_data.make_one_shot_iterator()\n",
+        "  step = 0\n",
+        "  while step < num_steps:\n",
+        "    labels, chars, sequence_length = iterator.get_next()\n",
+        "    predictions = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, batch_size, training=False)\n",
+        "    total_loss += loss_fn(labels, predictions)\n",
+        "    step += 1\n",
+        "  print('Test loss', total_loss)\n",
+        "  return total_loss\n",
+        "\n",
+        "\n",
+        "def train_model(train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps):\n",
+        "  optimizer = tf.train.AdamOptimizer(learning_rate=0.01)\n",
+        "\n",
+        "  train(optimizer, train_data, lower_cell, upper_cell, relu_layer, batch_size, num_steps=tf.constant(train_steps))\n",
+        "  test(eval_data, lower_cell, upper_cell, relu_layer, 50, num_steps=tf.constant(2))\n",
+        "\n",
+        "  print('Colorbot is ready to generate colors!\\n\\n')\n",
+        "  \n",
+        "  # In graph mode, every op needs to be a dependent of another op.\n",
+        "  # Here, we create a no_op that will drive the execution of all other code in\n",
+        "  # this function. Autograph will add the necessary control dependencies.\n",
+        "  return tf.no_op()"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "iopcs5hXG2od",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Finally, we add code to run inference on a single input, which we'll read from the input.\n",
+        "\n",
+        "Note the `do_not_convert` annotation that lets us disable conversion for certain functions and run them as a `py_func` instead, so you can still call them from compiled code."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "DyU0wnnAFEYj",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "@autograph.do_not_convert(run_as=autograph.RunMode.PY_FUNC)\n",
+        "def draw_prediction(color_name, pred):\n",
+        "  pred = pred * 255\n",
+        "  pred = pred.astype(np.uint8)\n",
+        "  plt.axis('off')\n",
+        "  plt.imshow(pred)\n",
+        "  plt.title(color_name)\n",
+        "  plt.show()\n",
+        "\n",
+        "\n",
+        "def inference(color_name, lower_cell, upper_cell, relu_layer):\n",
+        "  _, chars, sequence_length = parse(color_name)\n",
+        "  chars = tf.expand_dims(chars, 0)\n",
+        "  sequence_length = tf.expand_dims(sequence_length, 0)\n",
+        "  pred = model((chars, sequence_length), lower_cell, upper_cell, relu_layer, 1, training=False)\n",
+        "  pred = tf.minimum(pred, 1.0)\n",
+        "  pred = tf.expand_dims(pred, 0)\n",
+        "  draw_prediction(color_name, pred)\n",
+        "  # Create an op that will drive the entire function.\n",
+        "  return tf.no_op()"
+      ],
+      "execution_count": 0,
+      "outputs": []
+    },
+    {
+      "metadata": {
+        "id": "Nt0Kv5OCHip0",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "Finally, we put everything together.\n",
+        "\n",
+        "Note that the entire training and testing code is all compiled into a single op (`tf_train_model`) that you only execute once! We also still use a `sess.run` loop for the inference part, because that requires keyboard input."
+      ]
+    },
+    {
+      "metadata": {
+        "id": "-GmWa0GtYWdh",
+        "colab_type": "code",
+        "colab": {
+          "autoexec": {
+            "startup": false,
+            "wait_interval": 0
+          },
+          "output_extras": [
+            {
+              "item_id": 12
+            },
+            {
+              "item_id": 13
+            },
+            {
+              "item_id": 14
+            },
+            {
+              "item_id": 15
+            },
+            {
+              "item_id": 16
+            },
+            {
+              "item_id": 17
+            },
+            {
+              "item_id": 18
+            },
+            {
+              "item_id": 19
+            },
+            {
+              "item_id": 20
+            },
+            {
+              "item_id": 21
+            },
+            {
+              "item_id": 23
+            },
+            {
+              "item_id": 24
+            },
+            {
+              "item_id": 25
+            },
+            {
+              "item_id": 26
+            },
+            {
+              "item_id": 27
+            },
+            {
+              "item_id": 28
+            },
+            {
+              "item_id": 29
+            },
+            {
+              "item_id": 30
+            },
+            {
+              "item_id": 31
+            },
+            {
+              "item_id": 32
+            },
+            {
+              "item_id": 33
+            },
+            {
+              "item_id": 34
+            },
+            {
+              "item_id": 35
+            }
+          ],
+          "base_uri": "https://localhost:8080/",
+          "height": 668
+        },
+        "outputId": "61f4af1d-c81e-44db-9079-1a7b8ed8ce58",
+        "executionInfo": {
+          "status": "ok",
+          "timestamp": 1522345877153,
+          "user_tz": 240,
+          "elapsed": 75500,
+          "user": {
+            "displayName": "Dan Moldovan",
+            "photoUrl": "//lh5.googleusercontent.com/-Rneh8xjecyk/AAAAAAAAAAI/AAAAAAAACB4/c5vwsJpbktY/s50-c-k-no/photo.jpg",
+            "userId": "112023154726779574577"
+          }
+        }
+      },
+      "cell_type": "code",
+      "source": [
+        "def run_input_loop(sess, inference_ops, color_name_placeholder):\n",
+        "  \"\"\"Helper function that reads from input and calls the inference ops in a loop.\"\"\"\n",
+        "\n",
+        "  tb = widgets.TabBar([\"RNN Colorbot\"])\n",
+        "  while True:\n",
+        "    with tb.output_to(0):\n",
+        "      try:\n",
+        "        color_name = six.moves.input(\"Give me a color name (or press 'enter' to exit): \")\n",
+        "      except (EOFError, KeyboardInterrupt):\n",
+        "        break\n",
+        "    if not color_name:\n",
+        "      break\n",
+        "    with tb.output_to(0):\n",
+        "      tb.clear_tab()\n",
+        "      sess.run(inference_ops, {color_name_placeholder: color_name})\n",
+        "      plt.show()\n",
+        "\n",
+        "with tf.Graph().as_default():\n",
+        "  # Read the data.\n",
+        "  batch_size = 64\n",
+        "  train_data = load_dataset(data_dir, train_url, batch_size)\n",
+        "  eval_data = load_dataset(data_dir, test_url, 50, training=False)\n",
+        "  \n",
+        "  # Create the model components.\n",
+        "  lower_cell, upper_cell, relu_layer = model_components()\n",
+        "  # Create the helper placeholder for inference.\n",
+        "  color_name_placeholder = tf.placeholder(tf.string, shape=())\n",
+        "  \n",
+        "  # Compile the train / test code.\n",
+        "  tf_train_model = autograph.to_graph(train_model)\n",
+        "  train_model_ops = tf_train_model(\n",
+        "      train_data, eval_data, batch_size, lower_cell, upper_cell, relu_layer, train_steps=100)\n",
+        "  \n",
+        "  # Compile the inference code.\n",
+        "  tf_inference = autograph.to_graph(inference)\n",
+        "  inference_ops = tf_inference(color_name_placeholder, lower_cell, upper_cell, relu_layer)\n",
+        "  \n",
+        "  with tf.Session() as sess:\n",
+        "    sess.run(tf.global_variables_initializer())\n",
+        "    \n",
+        "    # Run training and testing.\n",
+        "    sess.run(train_model_ops)\n",
+        "     \n",
+        "    # Run the inference loop.\n",
+        "    run_input_loop(sess, inference_ops, color_name_placeholder)"
+      ],
+      "execution_count": 22,
+      "outputs": [
+        {
+          "output_type": "stream",
+          "text": [
+            "('Successfully downloaded', 'train.csv', 28010L, 'bytes.')\n",
+            "('Successfully downloaded', 'test.csv', 2414L, 'bytes.')\n",
+            "Step 0 train loss 0.37890616\n",
+            "Step 10 train loss 0.18515904\n",
+            "Step 20 train loss 0.0892782\n",
+            "Step 30 train loss 0.07883155\n",
+            "Step 40 train loss 0.08585831\n",
+            "Step 50 train loss 0.09302989\n",
+            "Step 60 train loss 0.089012615\n",
+            "Step 70 train loss 0.07275697\n",
+            "Step 80 train loss 0.06644974\n",
+            "Step 90 train loss 0.0854013\n",
+            "Test loss 0.13216865Colorbot is ready to generate colors!\n",
+            "\n",
+            "\n",
+            "\n"
+          ],
+          "name": "stdout"
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "<link rel=stylesheet type=text/css href='/nbextensions/google.colab/tabbar.css'></link>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "<script src='/nbextensions/google.colab/tabbar_main.min.js'></script>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "text/plain": [
+              "<IPython.core.display.HTML object>"
+            ],
+            "text/html": [
+              "<div id=\"id1\"></div>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b102d936-3379-11e8-ac70-0242ac110002\"] = colab_lib.createTabBar({\"contentBorder\": [\"0px\"], \"borderColor\": [\"#a7a7a7\"], \"tabNames\": [\"RNN Colorbot\"], \"initialSelection\": 0, \"location\": \"top\", \"contentHeight\": [\"initial\"], \"elementId\": \"id1\"});\n",
+              "//# sourceURL=js_e223a56194"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b103532a-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n",
+              "//# sourceURL=js_b8c6a821fb"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b105b28c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n",
+              "//# sourceURL=js_44805e254b"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b106197a-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n",
+              "//# sourceURL=js_a63d3c6c47"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b1069f44-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b106197a-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_7e203b8bce"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"b1070f38-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n",
+              "//# sourceURL=js_d53293d4a7"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c6d90d5c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"b105b28c-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_3000dc2c05"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c6da872c-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n",
+              "//# sourceURL=js_4136f669a3"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c6dac868-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n",
+              "//# sourceURL=js_2f70dd9aee"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c6db07d8-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6dac868-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_7226726048"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c6dcc6fe-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n",
+              "//# sourceURL=js_72e7709865"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAVQAAAFZCAYAAADHDNdrAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDIuMS4yLCBo\ndHRwOi8vbWF0cGxvdGxpYi5vcmcvNQv5yAAAB9JJREFUeJzt3E1Lle0ax+HTF4jeEAyMBhE0DawI\nwsCH0AIlaGBWNJBo0CDoA0TQhmDXuKAGDioiCA2KlEAlnl05FD9Co8BeaGCQoBDa2jPZsXt4Bvu/\n0+o4Rmvd1zW4rsmP84bFamo0Go0C4H/WvNYHAPhVCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKDy\nUxgeHq5Dhw7V4OBgPXz4sHp7e+vWrVt15cqVOnnyZN2/f78ajUbdvn27+vr6qqenp65du1YrKytV\nVfXhw4e6cOFC9fX1VV9fX01PT1dV1dzcXHV3d9eDBw/q+PHj9ccff9TExMRaXpWfWOtaHwD+zuvX\nr+vOnTs1MTFRbW1tdf78+dW16enpGh8fr/b29hobG6upqal6/Phxbdy4sS5evFgjIyM1NDRUly5d\nqv3799fw8HC9efOmTp8+XVNTU1VV9enTp2pubq5nz57V5ORk3bhxo44dO7ZW1+UnZkJl3Zudna2D\nBw9WR0dHbdiwoQYHB1fX9u7dW+3t7VVV9fLlyxocHKytW7dWa2trnTp1qp4/f16Li4s1MzNT586d\nq6qqXbt21YEDB1an1OXl5Tpx4kRVVe3Zs6fevXv3Yy/IL8OEyrr3+fPnamtrW/2+ffv21c//+Xxh\nYaHu3r1bjx49qqqqlZWVam9vr4WFhWo0GnXmzJnVvYuLi9XV1VVVVS0tLbVp06aqqmpubq6vX7/+\nX+/Dr0tQWfe2bNlSi4uLq98/fvz43X0dHR3V29tbQ0ND3zxfXl6ulpaWevLkSW3evPmbtbm5ufyB\n+W155Wfd6+zsrJmZmZqfn68vX77U2NjYd/cdOXKkxsfHa2lpqaqqRkdH6+nTp9Xa2lqHDx+u0dHR\nqqpaWlqqy5cv1/v373/YHfg9CCrrXmdnZw0MDNTAwECdPXu2enp6vrvv6NGj1dPTUwMDA9Xf318v\nXryo7u7uqqq6evVqzc7OVn9/fw0MDNTOnTtrx44dP/Ia/Aaa/B8qP4NGo1FNTU1VVfXq1au6efPm\nX06qsFZMqKx78/Pz1dXVVW/fvq1Go1GTk5O1b9++tT4W/BcTKj+FkZGRunfvXjU1NdXu3bvr+vXr\ntW3btrU+FnxDUAFCvPIDhAgqQMi6+WH/kX8eXesjAPytf/3jz79cM6EChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCI\noAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIig\nAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAC\nhAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKE\nCCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQI\nKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgq\nQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpA\niKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkCIoAKECCpAiKAChAgqQIigAoQIKkBI\nU6PRaKz1IQB+BSZUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAEEEFCBFUgBBB\nBQgRVIAQQQUIEVSAEEEFCBFUgBBBBQgRVIAQQQUIEVSAkH8D1Aj8lNhhe7QAAAAASUVORK5CYII=\n",
+            "text/plain": [
+              "<matplotlib.figure.Figure at 0x7f72f402e850>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1",
+              "user_output"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c70592aa-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c6da872c-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_25c3aaf79a"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c70842c0-3379-11e8-ac70-0242ac110002\"] = google.colab.output.getActiveOutputArea();\n",
+              "//# sourceURL=js_984c56b816"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c708dec4-3379-11e8-ac70-0242ac110002\"] = document.querySelector(\"#id1_content_0\");\n",
+              "//# sourceURL=js_e0451a1217"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c7092726-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c708dec4-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_7aa23d7385"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c7099044-3379-11e8-ac70-0242ac110002\"] = window[\"id1\"].setSelectedTabIndex(0);\n",
+              "//# sourceURL=js_5722756ddb"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        },
+        {
+          "output_type": "stream",
+          "text": [
+            "Give me a color name (or press 'enter' to exit): \n"
+          ],
+          "name": "stdout"
+        },
+        {
+          "output_type": "display_data",
+          "data": {
+            "application/javascript": [
+              "window[\"c7baac12-3379-11e8-ac70-0242ac110002\"] = google.colab.output.setActiveOutputArea(window[\"c70842c0-3379-11e8-ac70-0242ac110002\"]);\n",
+              "//# sourceURL=js_cdd622e58f"
+            ],
+            "text/plain": [
+              "<IPython.core.display.Javascript object>"
+            ]
+          },
+          "metadata": {
+            "tags": [
+              "id1_content_0",
+              "outputarea_id1"
+            ]
+          }
+        }
+      ]
+    },
+    {
+      "metadata": {
+        "id": "AHJ2c47U-A5W",
+        "colab_type": "text"
+      },
+      "cell_type": "markdown",
+      "source": [
+        "# Where do we go next?\n",
+        "\n",
+        "Autograph is available in tensorflow.contrib, but it's still in its early stages. We're excited about the possibilities it brings — write your machine learning code in the flexible Eager style, but still enjoy all the benefits that come with running in graph mode. A beta version will be available soon -- stay tuned!"
+      ]
+    }
+  ]
+}
\ No newline at end of file
-- 
GitLab


From ebba4f0dfdd4a1a42eba4a59d32222532beec031 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Fri, 30 Mar 2018 04:26:58 +0800
Subject: [PATCH 696/960] Fix math equation format in
 tf.contrib.bayesflow.monte_carlo (#18089)

* Fix math equation format in contrib\bayesflow

* Fix minor pylint error
---
 .../bayesflow/python/ops/monte_carlo_impl.py  | 39 ++++++++++---------
 1 file changed, 20 insertions(+), 19 deletions(-)

diff --git a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
index 985177e897..d193a8459d 100644
--- a/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
+++ b/tensorflow/contrib/bayesflow/python/ops/monte_carlo_impl.py
@@ -44,14 +44,14 @@ def expectation_importance_sampler(f,
                                    n=None,
                                    seed=None,
                                    name='expectation_importance_sampler'):
-  r"""Monte Carlo estimate of `E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]`.
+  r"""Monte Carlo estimate of `\\(E_p[f(Z)] = E_q[f(Z) p(Z) / q(Z)]\\)`.
 
-  With `p(z) := exp{log_p(z)}`, this `Op` returns
+  With `\\(p(z) := exp^{log_p(z)}\\)`, this `Op` returns
 
   ```
-  n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ],  z_i ~ q,
-  \approx E_q[ f(Z) p(Z) / q(Z) ]
-  =       E_p[f(Z)]
+  \\(n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ],  z_i ~ q,\\)
+  \\(\approx E_q[ f(Z) p(Z) / q(Z) ]\\)
+  \\(=       E_p[f(Z)]\\)
   ```
 
   This integral is done in log-space with max-subtraction to better handle the
@@ -95,9 +95,9 @@ def expectation_importance_sampler(f,
       log_values = log_f_z + log_p_z - q_log_prob_z
       return _logspace_mean(log_values)
 
-    # With f_plus(z) = max(0, f(z)), f_minus(z) = max(0, -f(z)),
-    # E_p[f(Z)] = E_p[f_plus(Z)] - E_p[f_minus(Z)]
-    #           = E_p[f_plus(Z) + 1] - E_p[f_minus(Z) + 1]
+    # With \\(f_{plus}(z) = max(0, f(z)), f_{minus}(z) = max(0, -f(z))\\),
+    # \\(E_p[f(Z)] = E_p[f_{plus}(Z)] - E_p[f_{minus}(Z)]\\)
+    # \\(          = E_p[f_{plus}(Z) + 1] - E_p[f_{minus}(Z) + 1]\\)
     # Without incurring bias, 1 is added to each to prevent zeros in logspace.
     # The logarithm is approximately linear around 1 + epsilon, so this is good
     # for small values of 'z' as well.
@@ -121,13 +121,13 @@ def expectation_importance_sampler_logspace(
     name='expectation_importance_sampler_logspace'):
   r"""Importance sampling with a positive function, in log-space.
 
-  With `p(z) := exp{log_p(z)}`, and `f(z) = exp{log_f(z)}`, this `Op`
-  returns
+  With `\\(p(z) := exp^{log_p(z)}\\)`, and `\\(f(z) = exp{log_f(z)}\\)`,
+  this `Op` returns
 
   ```
-  Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ],  z_i ~ q,
-  \approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ]
-  =       Log[E_p[f(Z)]]
+  \\(Log[ n^{-1} sum_{i=1}^n [ f(z_i) p(z_i) / q(z_i) ] ],  z_i ~ q,\\)
+  \\(\approx Log[ E_q[ f(Z) p(Z) / q(Z) ] ]\\)
+  \\(=       Log[E_p[f(Z)]]\\)
   ```
 
   This integral is done in log-space with max-subtraction to better handle the
@@ -196,12 +196,12 @@ def _logspace_mean(log_values):
 
 def expectation(f, samples, log_prob=None, use_reparametrization=True,
                 axis=0, keep_dims=False, name=None):
-  """Computes the Monte-Carlo approximation of `E_p[f(X)]`.
+  """Computes the Monte-Carlo approximation of `\\(E_p[f(X)]\\)`.
 
   This function computes the Monte-Carlo approximation of an expectation, i.e.,
 
   ```none
-  E_p[f(X)] approx= m**-1 sum_i^m f(x_j),  x_j ~iid p(X)
+  \\(E_p[f(X)] \approx= m^{-1} sum_i^m f(x_j),  x_j\  ~iid\ p(X)\\)
   ```
 
   where:
@@ -216,8 +216,8 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True,
   parameterless distribution (e.g.,
   `Normal(Y; m, s) <=> Y = sX + m, X ~ Normal(0,1)`), we can swap gradient and
   expectation, i.e.,
-  `grad[ Avg{ s_i : i=1...n } ] = Avg{ grad[s_i] : i=1...n }` where
-  `S_n = Avg{s_i}` and `s_i = f(x_i), x_i ~ p`.
+  `grad[ Avg{ \\(s_i : i=1...n\\) } ] = Avg{ grad[\\(s_i\\)] : i=1...n }` where
+  `S_n = Avg{\\(s_i\\)}` and `\\(s_i = f(x_i), x_i ~ p\\)`.
 
   However, if p is not reparameterized, TensorFlow's gradient will be incorrect
   since the chain-rule stops at samples of non-reparameterized distributions.
@@ -296,7 +296,8 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True,
   Args:
     f: Python callable which can return `f(samples)`.
     samples: `Tensor` of samples used to form the Monte-Carlo approximation of
-      `E_p[f(X)]`.  A batch of samples should be indexed by `axis` dimensions.
+      `\\(E_p[f(X)]\\)`.  A batch of samples should be indexed by `axis`
+      dimensions.
     log_prob: Python callable which can return `log_prob(samples)`. Must
       correspond to the natural-logarithm of the pdf/pmf of each sample. Only
       required/used if `use_reparametrization=False`.
@@ -316,7 +317,7 @@ def expectation(f, samples, log_prob=None, use_reparametrization=True,
 
   Returns:
     approx_expectation: `Tensor` corresponding to the Monte-Carlo approximation
-      of `E_p[f(X)]`.
+      of `\\(E_p[f(X)]\\)`.
 
   Raises:
     ValueError: if `f` is not a Python `callable`.
-- 
GitLab


From 622c0416bd6a00f9baf53e54a65ad5e5d3b87e30 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 13:24:38 -0700
Subject: [PATCH 697/960] Updating a test in constant_folding_test.cc that uses
 a graph with placeholder nodes by providing input to those nodes. This will
 allow evaluation of the fetch nodes in the optimized and original graph and
 check whether the output tensors produced by them are the same.

PiperOrigin-RevId: 190976595
---
 .../optimizers/constant_folding_test.cc       | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index e0ff9b17b1..16a19ba8ce 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -82,6 +82,14 @@ class ConstantFoldingTest : public GrapplerTest {
   }
 };
 
+template <DataType DTYPE>
+Tensor GetRandomTensor(const TensorShape& shape) {
+  typedef typename EnumToDataType<DTYPE>::Type T;
+  Tensor tensor(DTYPE, shape);
+  tensor.flat<T>() = tensor.flat<T>().random();
+  return tensor;
+}
+
 TEST_F(ConstantFoldingTest, SimpleFolding) {
   // Build a simple graph with a few trivially prunable ops.
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -371,6 +379,23 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
         EXPECT_EQ(2, t.tensor_shape().dim(1).size());
       }
     }
+    auto a_t = GetRandomTensor<DT_FLOAT>(TensorShape({3, 2}));
+    auto b_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 3}));
+    auto x_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
+    auto y_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
+    auto bias_t = GetRandomTensor<DT_FLOAT>(TensorShape({2}));
+
+    auto tensors_expected = EvaluateNodes(
+        item.graph, item.fetch,
+        {{"x", x_t}, {"y", y_t}, {"a", a_t}, {"b", b_t}, {"bias", bias_t}});
+    EXPECT_EQ(item.fetch.size(), tensors_expected.size());
+    auto tensors = EvaluateNodes(
+        output, item.fetch,
+        {{"x", x_t}, {"y", y_t}, {"a", a_t}, {"b", b_t}, {"bias", bias_t}});
+    EXPECT_EQ(item.fetch.size(), tensors.size());
+    for (int i = 0; i < item.fetch.size(); ++i) {
+      test::ExpectTensorNear<float>(tensors_expected[i], tensors[i], 1e-6);
+    }
   }
 }
 
-- 
GitLab


From 405efdd47c20919e5a05c86b0ae2e6c8c150e534 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 13:27:01 -0700
Subject: [PATCH 698/960] Use GraphProperties directly in ArithmeticOptimizer.

PiperOrigin-RevId: 190976918
---
 .../optimizers/arithmetic_optimizer.cc        | 32 ++++++++-----------
 1 file changed, 14 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 5dd0b6f4b0..36b26c18f9 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -196,8 +196,6 @@ void SetSourceDataType(DataType dtype, NodeDef* node) {
 
 bool IsNumberType(DataType dtype) { return kNumberTypes.Contains(dtype); }
 
-const char kOutputShapesAttr[] = "_output_shapes";
-
 // Shape is symbolically defined if it has a known rank, and each dimension is
 // defined, or is an unknown symbol (dim.size <= -2).
 bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
@@ -234,16 +232,20 @@ bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
 // Returns whether `reshape` is an identity op. The tensor that `reshape`
 // reshapes is the `output_pos`-th output of node `input`.
 bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
-                       const int output_pos) {
-  if (!reshape.attr().count(kOutputShapesAttr) ||
-      !input.attr().count(kOutputShapesAttr)) {
+                       const int output_pos,
+                       const GraphProperties& graph_properties) {
+  const std::vector<OpInfo::TensorProperties>& reshape_props =
+      graph_properties.GetOutputProperties(reshape.name());
+  const std::vector<OpInfo::TensorProperties>& input_props =
+      graph_properties.GetOutputProperties(input.name());
+  if (reshape_props.empty() || input_props.empty() ||
+      input_props.size() <= output_pos) {
     return false;
   }
 
-  PartialTensorShape src_shape(
-      input.attr().at(kOutputShapesAttr).list().shape(output_pos));
-  PartialTensorShape dst_shape(
-      reshape.attr().at(kOutputShapesAttr).list().shape(0));
+  const PartialTensorShape& src_shape = input_props[output_pos].shape();
+  const PartialTensorShape& dst_shape = reshape_props[0].shape();
+
   if (src_shape.unknown_rank() || dst_shape.unknown_rank()) {
     return false;
   }
@@ -256,7 +258,8 @@ bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
   // sizes.
   auto num_unknown_dim_sizes = [](const PartialTensorShape& partial_shape) {
     auto dim_sizes = partial_shape.dim_sizes();
-    return std::count(dim_sizes.begin(), dim_sizes.end(), -1);
+    return std::count_if(dim_sizes.begin(), dim_sizes.end(),
+                         [](int dim) { return dim < 0; });
   };
   int src_num_unknown_dim_sizes = num_unknown_dim_sizes(src_shape);
   int dst_num_unknown_dim_sizes = num_unknown_dim_sizes(dst_shape);
@@ -1272,7 +1275,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     // outputs tensors of shape [M, N] while feeding it with tensors of shape
     // [M*N] (or worse). The reshape nodes are then necessary to update the
     // tensor metadata to the required shape.
-    if (ReshapeIsIdentity(*reshape, *input, output_pos)) {
+    if (ReshapeIsIdentity(*reshape, *input, output_pos, *graph_properties_)) {
       return reshape->input(0);
     }
   }
@@ -1695,18 +1698,11 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
   // Shapes are only needed in aggressive mode.
   graph_properties_.reset(new GraphProperties(item));
   TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false));
-  // TODO(ezhulenev): Use GraphProperties to lookup tensor shapes directly
-  TF_RETURN_IF_ERROR(graph_properties_->AnnotateOutputShapes(optimized_graph_));
 
   // Perform the optimizations.
   DedupComputations();
   TF_RETURN_IF_ERROR(SimplifyArithmeticOps());
 
-  // Clear output shapes.
-  for (int i = 0; i < optimized_graph->node_size(); ++i) {
-    optimized_graph_->mutable_node(i)->mutable_attr()->erase(kOutputShapesAttr);
-  }
-
   return Status::OK();
 }
 
-- 
GitLab


From 0390fbec15f3d99c3badce3d666893ff124f7846 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 13:28:05 -0700
Subject: [PATCH 699/960] Docs: Move TFLite docs into tensorflow.org

PiperOrigin-RevId: 190977057
---
 tensorflow/contrib/lite/README.md             | 240 +-----------------
 .../lite/g3doc/TFLite-Architecture.jpg        | Bin 48710 -> 0 bytes
 tensorflow/docs_src/mobile/leftnav_files      |   1 +
 .../docs_src/mobile/tflite/demo_android.md    | 156 ++++++++++--
 tensorflow/docs_src/mobile/tflite/demo_ios.md |   2 +-
 tensorflow/docs_src/mobile/tflite/devguide.md | 224 ++++++++++++++++
 tensorflow/docs_src/mobile/tflite/index.md    |   4 +-
 7 files changed, 363 insertions(+), 264 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg
 create mode 100644 tensorflow/docs_src/mobile/tflite/devguide.md

diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index c15ae3f233..a676b705f1 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -1,238 +1,8 @@
 # TensorFlow Lite
-TensorFlow Lite is TensorFlow's lightweight solution for mobile and embedded devices. It enables low-latency inference of on-device machine learning models with a small binary size and fast performance supporting hardware acceleration.
 
-TensorFlow Lite uses many techniques for achieving low latency like optimizing the kernels for specific mobile apps, pre-fused activations, quantized kernels that allow smaller and faster (fixed-point math) models, and in the future, leverage specialized machine learning hardware to get the best possible performance for a particular model on a particular device.
+TensorFlow Lite is TensorFlow's lightweight solution for mobile and embedded
+devices. It enables low-latency inference of on-device machine learning models
+with a small binary size and fast performance supporting hardware acceleration.
 
-![image](g3doc/TFLite-Architecture.jpg)
-# Getting Started with an Android Demo App
-
-This section contains an example application using TensorFlow Lite for Android devices. The demo is a sample camera app that classifies images continuously using either a quantized Mobilenet model or a floating point Inception-v3 model. A device running Android 5.0 ( API 21) or higher is required to run the demo.
-
-There are 3 ways to get the demo app to your device
- - Download the prebuilt binary or
- - Use Android Studio to build the application or
- - Download the source code for TensorFlow Lite and the demo and build it using bazel
-
-## Description
-In the demo app, inference is done using the TensorFlow Lite Java API. The demo app classifies frames in real-time, displaying the top most probable classifications. It also displays the time taken to detect the object.
-
-## Downloading the pre-built binary
-The fastest path to trying the demo, is to download the pre-built binary
-[TfLiteCameraDemo.apk](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
-
-Once the apk is installed, click the app icon to start the app. The first-time the app is opened, the app asks for runtime permissions to access the device camera. The demo app opens the back-camera of the device and recognizes the objects in the camera's field of view. At the bottom of the image (or at the left of the image if the device is in landscape mode), it shows the latency of classification and the top three objects classified.
-
-## Building in Android Studio using TensorFlow Lite AAR from JCenter
-The simplest way to compile the demo app, and try out changes to the project code is to use AndroidStudio.
-
- - Install the latest version of Android Studio 3 as specified [here](https://developer.android.com/studio/index.html).
- - Make sure the Android SDK version is greater than 26 and NDK version is greater than 14 (in the Android Studio Settings).
- - Import the `tensorflow/contrib/lite/java/demo` directory as a new Android Studio project.
- - Click through installing all the Gradle extensions it requests.
- - Either
-     - Download the quantized Mobilenet TensorFlow Lite model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip)
-         - unzip and copy mobilenet_quant_v1_224.tflite to the assets directory:
-           `tensorflow/contrib/lite/java/demo/app/src/main/assets/`
-     - Or download the floating point Inception-v3 model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
-         - unzip and copy inceptionv3_non_slim_2015.tflite to the assets directory
-         - change the chosen classifier in [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java) from
-         `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`
-         to
-         `classifier = new ImageClassifierFloatInception(getActivity());`
- - Build and run the demo app
-
-## Building TensorFlow Lite and the demo app from source
-
-### Clone the TensorFlow repo
-- git clone
-  [https://github.com/tensorflow/tensorflow](https://github.com/tensorflow/tensorflow)
-
-### Install Bazel
-If bazel is not installed on your system, install it now by following [these directions](https://bazel.build/versions/master/docs/install.html)
-
-NOTE: Bazel does not fully support building Android on Windows yet. Full support for Gradle/CMake builds is coming soon, but in the meantime Windows users should download the [prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk) instead.
-
-### Install Android NDK and SDK
-Bazel is the primary build system for TensorFlow. Bazel and the Android NDK and SDK must be installed on your system.
- - Install the latest version of Bazel as per the instructions on the [Bazel website](https://bazel.build/versions/master/docs/install.html)
- - The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The current recommended version is 14b, which can be found [here](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads).
- - The Android SDK and build tools may be obtained [here](https://developer.android.com/tools/revisions/build-tools.html), or alternatively as part of [Android Studio](https://developer.android.com/studio/index.html). Build tools API >= 23 is required to build the TF Android demo (though it will run on API >= 21 devices).
- - In the root of the TensorFlow repository update the `WORKSPACE` file with the `api_level` and location of the SDK and NDK. If you installed it with AndroidStudio the SDK path can be found in the SDK manager, and the default NDK path is:`{SDK path}/ndk-bundle.`
-
-```
-android_sdk_repository (
-    name = "androidsdk",
-    api_level = 23,
-    build_tools_version = "23.0.2",
-    path = "/home/xxxx/android-sdk-linux/",
-)
-
-android_ndk_repository(
-    name = "androidndk",
-    path = "/home/xxxx/android-ndk-r10e/",
-    api_level = 19,
-)
-```
-
-Additional details on building with Android can be found [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
-
-### Build the source code
-Run bazel with the following command to build the demo.
-
-Build the demo app:
-
-```
-bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
-```
-
-### Note
-
-Currently, we only support building the Android demo app within a Python 2
-environment (due to a Bazel bug).
-
-### More about the demo
-The demo is resizing each camera image frame to (224 width * 224 height) to match the quantized Mobilenet model being used (299 * 299 for Inception-v3). The resized image is converted into a ByteBuffer row by row of size 1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch. 224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents three colors of a pixel. This demo uses the TensorFlow Lite Java inference API for models which take a single input and provide a single output. This outputs a two-dimensional array, with the first dimension being the category index and the second dimension being the confidence of classification. Both models have 1001 unique categories and the app sorts the probabilities of all the categories and displays the top three. The model file must be downloaded and bundled within the assets directory of the app.
-
-# iOS Demo App
-
-Similar to the Android demo app, there's an iOS camera app that uses exactly the same model (224 * 224 quantized Mobilenet).
-
-This demo app requires a camera so it doesn't work with simulators. It need to be executed on a real iOS device. Follow the instructions to build and run the demo app:
-
-1.   Run `tensorflow/contrib/lite/examples/ios/download_models.sh` to download the model files used by the demo app.
-1.   Install [CocoaPods](https://cocoapods.org/) if it wasn't installed yet: `sudo gem install cocoapods`.
-1.   Run `pod install` in `tensorflow/contrib/lite/examples/ios/camera` to generate the workspace file.
-1.   Open the project by running `open tflite_camera_example.xcworkspace`, and build the app in XCode.
-
-# TensorFlow Lite Quick Start
-
-## Step 1. Decide which GraphDef to use
- Depending on the use case, the developer may choose to use one of the popular
- open-sourced models such as InceptionV3 or MobileNets, re-train these models
- with their own custom data set or even build their own custom model.
-
-### Using a pre-trained model
-
-[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html) is a family of mobile-first computer vision models for [TensorFlow](https://www.tensorflow.org/) designed to effectively maximize accuracy while being mindful of the restricted resources for an on-device or embedded application. MobileNets are small, low-latency, low-power models parameterized to meet the resource constraints of a variety of use cases. They can be built upon for classification, detection, embeddings and segmentation similar to how other popular large scale models, such as [Inception](https://arxiv.org/pdf/1602.07261.pdf), are used. Google provides 16 pre-trained [ImageNet](http://www.image-net.org/challenges/LSVRC/)  classification checkpoints for MobileNets for use in mobile projects of all sizes.
-
-[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model which achieves fairly high accuracy in recognizing general objects with 1000 classes, like "Zebra", "Dalmatian", and "Dishwasher". The model extracts general features from input images using a convolutional neural network and classifies them based on those features with fully-connected and softmax layers.
-
-[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)  is an on-device model which provides one-touch replies for an incoming text message by suggesting contextually relevant messages. The model is built specifically for memory constrained devices such as watches & phones and it has been successfully used to surface [Smart Replies on Android Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html). Note that this model only works on Android as of now.
-
-These pre-trained models can be downloaded from [here](g3doc/models.md).
-
-### Retrain Inception-V3 or MobileNet for a custom data set
-The above pre-trained models have been trained on the ImageNet data set, which consists of 1000 predefined classes. A model will need to be re-trained if these classes are not relevant or useful for a given use case. This technique is called transfer learning, which starts with a model that has been already trained on a problem and will then be retrained on a similar problem. Deep learning from scratch can take days, but transfer learning can be done fairly quickly. In order to do this, a developer will need to generate their custom data set labeled with the relevant classes.
-
-The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
-
-# Getting started with RaspberryPi
-
-Using RaspberryPi can be accomplished by following the [Makefile instructions](g3doc/rpi.md). That will give a you a static library (.a) that you can build your app against. Python bindings will be coming soon as well as a demo app.
-
-### Train a custom model
-A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
-
-TensorFlow Lite currently supports a subset of TensorFlow operators. Please refer to [this document](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for details of supported operators and their usage. This
-set will continue to expand in future releases of Tensorflow Lite.
-
-
-## Step 2. Model format conversion
-
-The model generated in Step 1 is a standard Tensorflow model. After the completion of Step 1 a user should have a standard .pb or .pbtxt GraphDef file. If the application developer is using a pre-trained model (as defined in Step 1 above), they can download a ready to use, already converted model for use from [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/models.md). Models generated using retraining (aka transfer learning) or custom models will need to be converted using the steps mentioned below.
-
-A prerequisite to converting the model to the Tensorflow Lite format is to freeze the graph.
-
-Since we employ several formats, the following definitions may be useful:
- - GraphDef (.pb) - a protobuf that represents the TensorFlow training and or computation graph. This contains operators, tensors, and variables definitions.
-
- - CheckPoint (.ckpt) - Serialized variables from a TensorFlow graph. Note, this does not contain the graph structure, so alone it cannot typically be interpreted.
-
- - FrozenGraphDef - a subclass of GraphDef that contains no variables. A GraphDef can be converted to a frozen graphdef by taking a checkpoint and a graphdef and converting every variable into a constant with the value looked up in the checkpoint.
-
- - SavedModel - A collection of GraphDef and CheckPoint together with a signature that labels input and output arguments to a model. A GraphDef and Checkpoint can be extracted from a saved model.
-
- - TensorFlow lite model (.tflite) - a serialized flatbuffer, containing TensorFlow lite operators and Tensors for the TensorFlow lite interpreter. This is most analogous to TensorFlow frozen GraphDefs.
-
-### Freeze Graph
-To use this .pb GraphDef file within TensorFlow Lite, the application developer will need checkpoints containing trained weight parameters. The .pb contains only the structure of the graph. The process of merging the checkpoint values with the graph structure is known as "freezing" the graph.
-
-The developer should know where the checkpoints folder is present or checkpoints can also be downloaded for a pre-trained model (Example: Here is a link to the [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
-
-Graph freezing can be done using the command below (and modifying the arguments appropriately)
-
-```
-bazel build tensorflow/python/tools:freeze_graph
-
-bazel-bin/tensorflow/python/tools/freeze_graph\
-    --input_graph=/tmp/mobilenet_v1_224.pb \
-    --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
-    --input_binary=true --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
-    --output_node_names=MobilenetV1/Predictions/Reshape_1
-```
-
-The user has to first build the freeze_graph script using bazel and then run the script.  The input_binary flag has to be enabled to ensure that the protobuf is read and written in binary format.  The user has to input the .pb and the .ckpt files to freeze the graph The output_node_names may not be obvious outside of the code that built the model. The easiest way to find them is to visualize the graph, either with
-graphviz, or [in tensorboard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3).
-
-This frozen Graphdef is now ready to be converted to flatbuffer format (.tflite) for use on Android or iOS.  On Android users have the flexibility to use either the float or quantized versions of the frozen graphdef, if available, using the Tensorflow Optimizing Converter tool.
-
-Here is a sample command line to convert the frozen Graphdef to '.tflite' format for  The Tensorflow Optimizing Converter supports both float and quantized models, however, different configuration parameters are needed depending on whether a FLOAT or QUANTIZED mode is being used.
-(Here is a link to the pb [file](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)).
-
-```
-bazel build tensorflow/contrib/lite/toco:toco
-
-bazel-bin/tensorflow/contrib/lite/toco/toco \
-  --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
-  --input_format=TENSORFLOW_GRAPHDEF  --output_format=TFLITE \
-  --output_file=/tmp/mobilenet_v1_1.0_224.tflite --inference_type=FLOAT \
-  --input_type=FLOAT --input_arrays=input \
-  --output_arrays=MobilenetV1/Predictions/Reshape_1 --input_shapes=1,224,224,3
-```
-
-- The input_file argument should point to the frozen GraphDef file that holds the model architecture.
-- The output_file argument should point to where the TensorFlow Lite model file should be generated.
-- The input_type and inference_type arguments should be set to FLOAT, unless converted a [quantized](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/) model.
-- Setting the input_array, output_array and input_shape arguments are a bit trickier. The easiest way to find these values is to explore the graph in tensorboard .  The user should reuse the arguments that were used for specifying the output nodes for inference in the `freeze_graph`step.
-
-Note, it is also possible to use the Tensorflow Optimizing Converter through protos either from Python or from the command line see the
-documentation [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py). A developer can then integrate the conversion step into their model design workflow to ensure that a model will be easily convertible to a mobile inference graph. For example,
-
-```python
-import tensorflow as tf
-
-img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
-val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
-out = tf.identity(val, name="out")
-with tf.Session() as sess:
-  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
-  open("converteds_model.tflite", "wb").write(tflite_model)
-
-```
-For detailed instructions on how to use the Tensorflow Optimizing Converter, please see [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
-
-You may refer to the [Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for troubleshooting help. If that doesn't help, please file an [issue](https://github.com/tensorflow/tensorflow/issues).
-
-If you would like to see a visual description of your TensorFlow Lite model after conversion, you can use tensorflow/contrib/lite/tools/visualize.py by running
-```sh
-bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
-```
-and then visualize the resulting HTML file in a browser.
-
-## Step 3. Use the TensorFlow Lite model for inference in a mobile app
-
-After completion of Step 2 the developer should have a .tflite model.
-
-### For Android
-Because Android apps need to be written in Java, and core TensorFlow is in C++, a JNI library is provided to interface between the two. Its interface is aimed only at inference, so it provides the ability to load a graph, set up inputs, and run the model to calculate particular outputs. The full documentation for the set of methods can be seen [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/). The demo app is also open sourced on [github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
-
-The [demo app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app)  uses this interface, so it's a good place to look for example usage. You can also download the prebuilt binary [here](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
-
-Note that you'd need to follow instructions for installing TensorFlow on Android, setting up bazel and Android Studio outlined [here](https://www.tensorflow.org/mobile/android_build).
-
-### For iOS
-Follow the documentation [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md) to get integrate a TFLite model into your app.
-
-## Core ML support
-
-Core ML is a machine learning framework used across Apple products. In addition to using Tensorflow Lite models directly in their applications, developers have the option to convert their trained Tensorflow models to the [CoreML](https://developer.apple.com/machine-learning/) format for use on Apple devices. For information on how to use the converter please refer to the [Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
+See the documentation: https://www.tensorflow.org/mobile/tflite/
+Documentation edits can be made here: [tensorflow/docs_src/mobile/tflite](../../docs_src/mobile/tflite)
diff --git a/tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg b/tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg
deleted file mode 100644
index bc83946647c6a923a8a0bd3a041b42e4febe6a31..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 48710
zcmex=<NpH&0WUXCHwH!~1_nk3Mh1rew;7xnIM~?O*;qN)+1WWcIk<R4czL+Fc_f8|
z`9)-<<mF_gWMmXn^wbrUbd+UeG|V-13=B<7Oyt!qZ7qy!^o&i6K!z}Ka&q!;^GNXW
zN*F21C>oIr{vTiv<Y2nMe1MrziGfLwky()O{}Bdx1_nk}MlfK20!Aig7FITP4o)ua
z|3?_M3NSD+GBY!=Ftf6<urM$%)-p0NGq4D<3Mm>ovIz$!vMUve7&T5@$f4}C@t|nX
z#SbdRNkvVZTw>x9l2WQ_>Kd9_CZ=ZQ7M51dF0O9w9-dyoA)#U65s^{JDXD4c8JStd
zC8cHM6_r)ZEv;?s9i3g1CQq3<ZTgIvvlcC0vUJ(<6)RV5+Pr1!w(UE1?mBe%$kAiR
zPn<k;>GGAU*RJ2VdF$b$$4{O<d;a3(tB;>PfBE|D`;VW$K>lK6U<UgNL_j=-<}X18
zCPpR}7G@T9kiQt2${84$m<3r_6%E;h90S=C3x$=88aYIqCNA7~kW<+>=!0ld(M2vX
z6_bamA3<IN`;0h`HId~rxW^Fwy2Zf5%*enb$SlZU&+z@1#-au;5PrC)*!03xlhe`J
zJ9S&0XPeYkUoVc)@2!85bkY9)e+DUrm46#o)L%GU{gcangDL+niTrxS=q~|m?BCcH
z{%3IJeg8z`-+?RkFB13uY0barAR&K4s{JpI`Thxue=`UCXIR+w{gdc^SB8Z8ge>_l
z2KW9s^}peD{?{gb|3uhd5yr+*!$*T<G_{OoqJbz|G&f~LHMyPr&%nw1o?%|oE~&oy
zuR`^^>ZdX|KG2@sy;|tI(UwiTr}yrfW3cYnj`{8NpZM~>&;QThBzil5(NzQ$BbS-)
zR=0j!|EDKP{r7){#|~0c0$mwIsD?vIyT4vGS>3((s`yj6wYM7d1LMr)wyro{u(@{E
zgQ>YUjJDZpeLc;+W&ehzSN6M=*#1;^F>D08R+4?;q6RJ}rzTGZz9knpZ27XZ*Z=r3
z|KHB}PcoP+zG_P@5j<L%ue{wgl2QG`*{JoZAJ+fr`4<0v{*&x$MT`d(0+=pGUgY38
zwe#iVp3AE{Kbp6d#`=f<_T9W@qEQ30-Q5}g8EjWDZ(Fn~^04l_=^{ZVpPd$oNdKx3
zuDImK>41ILa^I?UG0IHJwGMgy^8$M**i8xwUzDabaA_=Ik!uNHy69T)>wZuEm%B-i
zE@><<Ja&NTRPlxC3+#oi?#wfF9~WPpA{+z@b|Lm4RtAwkM}<<01x#HU)WM7A|8<%F
zZvNAtkmCOgD<=QF{AUHz{KNZyvYY=t|DRzY*W1;dKg$0zC_MYmQ2C$Xi(>3njF=TY
z^yB(JH(lYs{~126d$l88>OaGYql@bI|1)7+3HCBt!XZ~IM!x^6?j^_NZg=$Gs~lgS
z_~-S`Ss_jAvz(ZerX^J~#+j~&+xl<0>64ju$<O8Pwm)@{-YzF^_(M<n?fm`spKX_U
zV^iO_ww~ctT<V@%HNBtR%8!4st&e{mS@h#kWX`OMTYlGTF8=g!$CqQ;!p>Ltrmwzz
zm-*A`#N(^x@iJ8Tcgs#bT<+5o+91>1d?kGD%k5Ho4|`@Ns=WKmV|LAwq59i;$B)VE
zANF3>JT*0lf5nM)$q&E(Y1ONL-TzgDarML1()amq$KQF?5oFNVeoorrT$R+br&mhr
zMPAKMZ*vPx@;b66G_=-#ZF!tFPmO8ycFS9w$CtSl*)eFD&TKsPnDh3}E{2sWUi>=W
zE~>ZhLfN!qKVm-AzFz;~-?m@pnhP#Jek-ujUHQk32iNXCzU7|h`?AG(;(TlA;vZKd
zo?L#iao4}|*-1>Y|J-c(jQfu@*sS;b9sg|&)A4_H-GcHPmDlEE{gb)2CU<tg&Yi-!
z2a7fdoqiwkNBcj6vhDQW&womN@fWT?vAy;`L(Kj|4F)y5Yh&*2xh8I8GA}B>cgB;(
zIfC<#ZQPStXj7*=H7HZGOrPrr-&FO6`X{#Pe+%-TY3=X*&+sI5`ro7fLKrW_XPVDC
z-d=Jzw@As*dFN@vpPSg17R;($vVGPRvyC1?k@8vF{Mq(Kd;MpaSakj0-u6%HO+P%h
zQ{VKTVcq@54zk~7^X!xQ@a1j2`JCjm$qJJvI2?F>Z^!;^*>>7j&KP<fD00cV9VmWp
zx4cxloz(1n-&fHuXFaU3&tG*X#%q@Efwq7J+s|*A=peTK%iQ$CYSs&joP>KnS{|$A
zFuh$-94obL!gb-dk%>Q3kNeF$TWlUD|1tKBn)K?a({!F^pFLKuwR=fMRNLf5uU;1j
z8y%j)aq|WLj|*&Si`QkvDP3h<JA0SQ+ieyT-k$iH{OeMdob0s{@BD2)|1)SkF6?ri
zy6lI}ijO+h8&7Ss+p*qvXNK6_TQ80&GC%)pxc%c<hQeERODiAca=rJ<Y`UiqyJ?=e
zUiIVs@@ors&7HGNYRb0FCsZaGF<YoS@A|r0vSg=9V#2kh2Y&A9-FbYU_F0=yP+YzL
z{rD#r!y6mph4)^zMR(6$I?XbXF~rc~{k<K3rJj|2)LpgZlc@27rsuNRS91R|Jdqdw
zTO@yK&vMuQ3=?Nv{<r%-Llnc8zd2JcyziRC(Rf1nMRwDN*dv!Jx=bsVq<*^HmXy8s
zy8RR8`u_}Z@Bj2xes8jWlG6Wg=T?S@k4KMOeEmBp-1XD+)+F1kC1r9-*K;4u6bf$@
zQAwSnp}=?3=5JQLET~-g&yX|y*Y)Cu?|+J~sej#dfh}zBI`0K9u1)*+`0XY)cAGZ`
zB<<%GeUsAL`bgu}hdXmy9@t!xpYfY(^Wz8q8Eywm{_dlk|18s_n_t*}*^yth54>|~
zR;eDd=LulVFx8DdbZh<1cPizs$~8H+r?#(H@7t8Jqsel@Q(0%`H`(Xj{ZW6U{nBN7
zkh{)|1L@P}gqJUg_D)}NJNkUTk;S2Hd{qIgw?I7vN-?Zf^$a}!Z;}6Z|4$3J>iPaN
zJdwHo-u_>U_O^BE5AXkJp8fave})UKWmgyiT}4oEIQQfDpAuXAzyD`AwtLmXn(Y4!
z%3_`W{xkfzz_wH*(1DdoSi(;Ccg`-ytsA#b3ZJ=c-@!ThRYyLQx2HF~s-Ej~W!Y7(
zZmqBCE7v{iVrYI-w*T{6{=dIP5C7PGc)C$mcJcCCCY?$(k4%3rssFk$_OA%ziAt-f
z*Bh5?o^>g)T!KSjy$+*<(&`Frul*aYq*t$wObqL1*m2A+d&i;%<yj&-8Xrz9S6&$=
zcWlqI&xfUsb=H5IYv8GqRW!-<d&3u{y;@xipS5S$1g{rqdc--C@tiD^hTZa|WmbAE
zCb>e<H9sexTDnAep~VaBqArHd@q8Jx=C1G1RhxG#<58IE%;}8>*F8QW;9b1no->#6
ziH{FMt0o`K40MnYZ#J3hbUboW^lHz<o<kwV>)494<i%dx>V3Cg<=VAkZ%J3isYO=d
z3okl|Y*`U!D|JmWWc3=YHOp5#+VkrB;akThuaaDw`*dR9cUK+f@;Zz4L5ms;OhWcu
zHSz8!-EqHU$vVlhxjX(dNIiM?M}N(W>-RG*?>QOj7Zl-thJB&@lklY%9pnnW%Nbl*
z{M)eaVQ3pChw|e)A7iIACYC2IS-SVjqfGX^-QmA?zK_}xz&<zYp3oJO*Z&!|zVy1}
zB3+eZZ<Buc-hv%=+SioDCLb=i_<TZuf_?DOs#Foi#)tL%bA_(nO5-gyGk^PHL5b*j
zP&8e7)nA**?JKiDjPt7hrQN?Ym?wRhdu*rbVdoV~PUp%*_#4^oX)-*2X@lv0o~$sb
z#djhOt8AElU;60D;5m<FvoC8b;I}sXu|0BYg__WDhTz>rEqC9(%Dg9ZajUd$rP|KT
zvox9v?wmeaB`U&rpyIg475%N}t*)JmO#XBwFZ9HD)9Wt^PUbI)JgOx!QHgQNSGFx0
z3wYuMUvRFA(g@=$l=I1}s^u*#`?gQxg6!h$Z3Ro7v;@4U{e69Tpo3g>e(IyWao;4<
zCuNuRZfCDih+ovV^#zAya=`Io#*@dRN>MWmdIlnv4{!6V@`B}AhG4YjPN6+=*2Qde
zA+>i*?;_{i22+hiT?|yk>&lNswO*UM#8r36F-^II*7lf$`7dLBc|Ouq4wpK}*j9gD
zv^U39OVg_F*p?L`t3PPZYjv&sR^$4g!Ey4xdajrs-T#`^|NIj@|95HYzvBEAkL;&>
z;nF_;`H}uFp821D1kM+%zmW2uVg8~1w>tWt+piSLpQ>UNeg5;K_+K{vpMNyYckF**
zRDb@V`?qQ0KM%jM=znU<)b;s4!;cmJ86XmWLh8Rr+CTrW__sUvpNC&%+&|50bgldc
zcY-m@s=td{{}ue|Q~v4iuxQVJ20Nwy3=2;FXRvRTgjx3Q;-UWx7Panzf0i!@jH`#c
ztq1B0k^0QU`Z;!sJN^Y<&{%H|bDJC7F8eJ7`_IV-9;w%SAre0S`H^^t%Mez{-?E5*
z&QGOK_yc#=QFmx~qdGwNkpll8KmMP8w9^k(O?|g8w@hE%*rbSINncgimqL@(@i%wh
zIJTB+%0AQN6^pvGixn1iX)sU&uXz5SAyEDI@}D77W&Sg~Qu)tN@}EJAVflym{~XW$
z`~08bh12e=M|Gg;*>eBA`oA3ExtLW<!9LUf3=_Lu{xj77$=<b2{Mi0ah8gqU|7YlB
z$Rako5vh*xRK<UWe-0DsuOIl&a3N}S^nZrqTjJmGKMkv?|IeV{VgLF;{R`dDwf`9&
z-_rli{%Jk4&5wj?OSexdi@LO*^B2IfoVL5)hAq20wtPR${O*90@%(F>SAMKt=l{Jm
z*YRjb%&8aWwxm4x8Qc4NK3`OnfZCO-+&m2@d;V?h%E>Di*!F4DSN6mwRoC8k|7W-!
zUB4si-sTnOm%qH0oflHs*>>uA57*c1#1%WwMo)FSon1XIR>w6}!uqpi`6o+{w|Q#5
zA03rfyj$h5dRJ`g-q&Zg3wcU9P7tl1TN*esci&Nw&7wMmPY;Tps_lH<f2zC0<+^@X
zgR#)$<8@gNrykPN-TmvApv|EoU)6m7;12J|yUy-B>y|taSn_PPj?sbNs=X8UN+j*-
z$V=P&p{Cg3b_$P!#W}m=aFvMneiKa(wWa^;tDd{_U+MI3DXL3S*Y&?Q{dZwb>{D6O
z_`>7T-c$R}WLL7(-_t7e?c!X=DRAhM;PtG@UGG(Pdc0rK<v#Pu`d<tA!sT`z4fxEv
zynX3x-x$jYb6QU_moLAxDl$arpL%Nb_X{(g1y9<p_icN!bl|1L&6jfX!s};iGB%ez
zn00*J>LdOtvqWc~F;q;5?kV1KT{|YS*CKfBY1Y(~vV?TaOM7;2Sk$H6@&-H>tFfp-
zaNo;c!FBTg8CLB7&#<of*X7yo<}WpW_@Cid_kV_8FYMRW^0|Kd=iWX0t?=eWUD_}F
z1z_<@DXnGspW%w<znA|)Ro(4>b=u#xKh>c5$NE3RgyQ-={~3M>th+U6#z0&dissq>
zXPBTO|Mi3Y7qL~>{xf`hEB?FrXXt$S{|qOD_<#M7|HU1eUH{`-_uqqmRv$QTCn0w(
za8IYkYB8rPcN#0sDIT}By6#}oEAsN$?=w3;u?Nqq*9q@;X16-IW9#y=s?v>ti{7f7
zi|e{L*+2U9q+KCJr#a#xj~Uc$W!aVcM4Xdlg3*cRT(<R6=OU_BhpgMZw!2Sh_T8J`
zs;6eGm474a&r^5mzW$CCx?66aK6HpfcH12D94UYPQ0t=oJL>LDU4N!)%Yvy2eT&w_
zM8rm&+UU!?dOMfloWEM-7fmG|evZ2Ot#O`#|5K;MVZU5o@NL!BKAGiMUSC{u>bg+A
zmVpIR_gbfCUzPhR%hx1x8L(-7`xW~3j#7j|x>MoAu4|hvtaY1jdTc>z@1rCcMH$9e
zlNCwX@ujt;maoDN2YILG9p(7rxYCuQH_t+rGr-p%dsf5DrNV#SER1*)=l(Q2dHY4x
zT}{@R>SdK8qJ@n;p98XX<=stvo3l{;DT~(UsB;o44$t)ab=$pI!=LBrsfT&H7EQdO
z*W0n#=-&gGx3_GIuTAsydKZz{xy)Me`1A(GnCi>#N@l-y;Wyo_HP84;klplZnTswL
zWIz2qIPq+;X=jx6x7VMn!mrL-b-yv;M|-egW!mFy=DCt9a=upI7C0o?C-J0BQu6CE
zz3Ek-y@d|ke)RjXa#wEat~;*;4yrlrPI$aFXrAgh^St*i-ffbuH?{MQ$G)0A{o<N(
z<NQ!d<_wvCp+VbA-}332Ni5&47ch72j^}byvko6F`<$6wTyTj~(wl*gL3Xug!bVZ8
zQ$kmlUO8v6$JWZLFwA$(QPI51K701D+&*w_@qWj^)2YR*2bw%$Q*6a$>!u!iam9P~
z)ElouS8mUK<>*=xW!I><Gk^PSdF?be|N29lj?FGCt+XlIV60^+cX)o(tq7})U6Z4~
zyh@dc)$~2~yHw+1)U;VK8()3<c721<>d6ORRD2bc+nOu7Y^9FDx+@X~{%EPT&B^@j
zzHFP8MTgg-<+3F^=iD#(tGS>5Kf{&y{|wh1|7M>3?tf|ahyM(JUH>!uEs?*rcTV@)
zKmEq$x75>JMZz!4Qy{X2LFsK_(=)^xhEHv<R*b9dkc*6MPip7s)i=~@eJDP1Co%cP
zDcL!l*~|O?xy-cN|J$_ar>MfN8O~2S@1L?ixa0As_0qi2H;dJ_huQl0>SrxZ-@NnM
z3u$SG(%#4=cWzHJyY>9zuaL{9{J&M%ZaMk;;nt-I{*TXP_cpzXPQTByR=R3+(p=TA
zXD;bXiaBv^f5|jUd9N>;&%8_?%Y`pIY{OhCvvW&&jGo$mhL)`hKgS$>`uFUek}r$y
zoLX5kKi%j+T3%xPoZ^=;4SRM7SKiN<eD;|6d&@sBllJdya;tck5|%to@w|PM@{A>4
zyYKGb6Lk7^l<B5Fe5YI@^54jL-Z<vTDYN6wa+zY+JylKF--Ue6f79KQx-UM5>(R5F
zvmW^yMb5obqI2`8*R83Kk3O}%vu4II?pf>KYTWy?HoHDPu+}MTyV0%l{}``u_bWT>
zn|1Zuqp%z9zpu=SeZ0Kwo!%0rbe<#a|Mcc&PZBp;qb6wF(I0a6;2+JPXDJ;=HhU=B
z23Ruu6P^F9JIL>>cJ#B%udJsYmwerS;#PCm@4BU_@;4LPa}I85o*aE)%D=n&0~}@?
zn_c|H^x%><bH2XN>k}LgimKmwX3W|1Nz(n}bjv@lG}44xa!t#NRwpEzzq8hsP3?+x
zUJ@@GIy;k{$?fFJu$(-_>7LuKs6KtWIPu40_MhR>wtvrFYMuA+3G-%iQK`E=8zsWh
z1?5&qytQx5mN`B3rP$S7?@~T}S6wsp{pF3f<InR%xgJ|6&RZ<ie_TTT+)iKdf6ik2
z-*>IbOUrQ4HJ;J#qEps?XvQRyJV}|=d_8;|>#Ba7&yEtcEQ#golJHOW3)r{jy}Dui
zz1WwVOAhV&&+v8Shr2qL?mM-AFMA=t__}%i=eIHHxAkn^Tfa5>Ipf&h=k{}zV$Vw^
zO&8Mp7tdFhzqa&0L)PoM@az8=Pye0uHZTA6%BzBf`fa+?eRWh=@&)I}*I!#dy;uEm
z(Az0@a}_4P-ah}@jt>2Gy2VfYJ~m&^4p!MQDYNFLYFhn!v1bd@0;5*!e!1M&TI2)o
z?qcn%KVPl0HNLW5*SlBrpCL{8$;qF!8#Hsy<S&cS@p^YC{LV(RYv*1)z8$(RfpO35
z#>RG~MQxudWKN2_>#Le@w{C0UnMbp{bk~;O5472pn|0m)z?-GBOOMU!&(}%u{IsP-
z_`=t9<wxp2NzeZ9FVyk<x|FG@)iZgzFK(CVU+SuIUi<wtw=!FAnORSdJ{HX=Qu%IW
zu*2Og%Xe+tcIPk4{@DL#2(16l5X=5`sg6OVtfmd)zh(a!{_Wy_9lKL%^ZXM}&)%w8
z4k_3)uoY6|hz+rV?cB936{Wisb4~gs!ucc(=RD|t9bx~WA)0Si>-i_z>laP-d}ng&
z#uNL8jNAS*ENDIdMB_h03)Ag?P2%<w*6r(x+4x50{LT~ghZuAJGdOb3KN0w!frEAX
zUlzT3#n@VrBKHkHm)?0||BwNzZcF7GF<{-yNV<9P>t=)74bgq<<b!C7#rkT?tu$Ww
zF|4qED0IpGh1q|G@0YIoKH~q$6jRIoc^%gd!-)CGdZ8j!KX{mqTrMukxc;A^#oOaQ
z!-CuY8SZ-rAAOLmIk_fk&b#Hi9&{aC_xP~<CszOOw|`zb^r&{r&zK-NYyF6<m{-0t
z<v%psuD{^){F6xiA(*Sf<6Wy{HomEpx%93GhrG<hH+t!K1EL0^IU0-MaLw4`z%u!U
z<^19ah<I7Mmfx}2sV(B+)F(z}&-A;lE}G?;!@GS)psUCcYE2r7B6V~p!KSzzME)W7
zabx_Er@2;ObhVNDxaFZpb6g8}y(k?aCQARR>(~9$^U@-uQsegsd}3d#X|26!vAFuF
zu6>IC878P6zff-&BU(~5z2o7&4b$!_O!R&{$zA5UWt^DR5-nc~>-phR16I0C?sd&|
z+TG7ne?rv$e*W_q+Y%}N4#mrB#O^+r*pXDkaol7>sIuSECvVzk1<uWq*>d&nxkFvc
zm)|%2V^@Cjo{D|R&foXFcE-I)S+cHw@*Ek})4GScG*+ol+V{eze`xi~L_(`?|Bkid
zw|bY_W%HJG_k6E&{~21=yZmRE|K&f!KK~y9Y-dG#|1)sLb^d3tfBB!GX8wo8|Ah7m
z|Es_J@2CAk2WgWq^FOLp>VN(-EdTqd{!sr9uP^>T|1<die)6B8l_BS1#QhIbU(Emf
zZ~pg_{~5UKA8KE;fBx70{p5cljGJAP>kox5sz3j${=WJjP~k_s#olhLw)uw-evi4S
zCcXNI_3!=<>q46NI1b9L-iGRdFtM$b3+}P)*tV}X{goRB|F6>Z=HkuIzQ24|y|XU!
zbXMuSxsu(ndYum+tKFV`^-!rte|>S}FVl1Fd9Fv_zDW!Go_tWT_H0Q!vM=VSalgIt
zyZel>_r1B|pUoH;7}yw=uE|I7x;(OmuO)UhHy3T(kK}t~pEIX*9Yb|>AZm~=;GLx{
zjTG$e$U)v<mVwW122_hh>y8I)zOpz|=XqqPpR{Z*LliYe>0o9Iz0tZFhyM%{WFP|*
zYgU-77HcvyJK-7NoAm?xFkYNoX;49rWo&=6R!iE`GwZ_to!s!BVSdQs-)e{d)h4f=
z65jZA*8Ow;zE#H^n*M41j-`JmC;n%!Tj~DIxBbs=_t>!R>YPi_H%~5iR7^WKaf<Mt
zRrl5xX6IM@s<DdQZM}l)p@XG;Yx%Lcp26R`Hov`IcbsRHo2l*Wr9X2D<8PYDe|{}1
z51~q%4_$i8alN3s$w=vaK-7<_?2cvSGZhUj!}x4%U&+dpzAgVgZS{j&`({<j+y`0y
zYbjXgtEm_7Rn$dSJe;IC>7i=Eg7q@<rdobD+<*7)Hu){Ncb_IrTHLogyuRY^HldR*
zzssyg)f$;0d}qldZ>8uXEi2Q%%4X)(*qiRTz^LUZaz->IuG!S?y!7d_<=^{nC0gz*
z`Q4Wf2^C1-1nv6I@Wbuj@j~IV#lroj?9;cuXF3ozZ9!*tfzW;Dwi4}02ifCK?MiGt
zTC_cAaoE@Ti7g*3>QWZ6ynnIZ`22r{O*$UuZwmA;TmlL5h}XLsr{(eo7w1fn+5Yap
z@BJEUPUrKU*4<O=Uo5+S=6BIwvpO#v&3yCh?$z%ZE5Z-IJ}<I%dGSn$W@vl`?TYwg
zS#bS))+Z&SWnV2$B)$#k<$L_w%=^*-7L}aiD$G6C;?7Io{rha+b<U^XB_IJfr9S0G
z{rOd4&z3H=n>As}q}3*itsS0(O`Dy~GEZ{%%!j>8f^H-&nE2zOpT$hSkUO_B<I6P9
zwInQCG%N46OglI{kAsuLucg^9yP|`)2Z&61+$Je{S#NTF`P%c}=BqyqVDUUwA#?TN
z%17CUXBk)DE(~Ak*0bipwoexJZ)WrV{OV=DDaZc#RqbVV^3}aw8M#mUPx*O2Z~gY`
zO8G63Y18&+c2&GnlrdJ%>I^=%<Y{^8GmEQ-woRRSf8x3S3}D9xLHrPUOMCij)BSV*
zJ`dlp^v~*d8gtbSv;P&f3f<rS$FYvx-=JREFMn6vl>6=f8D8oCXL!Bf-`00ewyN9J
zrPuZ=n=Y4F@oBZ}T#>-IkB_M}ugeM>&Rha7OafT#kymqt3!@DFW6V-%BQ58Oy^1{c
zpTQhNt))j$bNGUf|7ZBb()@29`@g2EQTPAZF8a?<P_MQ5@qdP&OpgB<;+g+5IA*QQ
z|IZ+Iv3`gB)C$Sxhna7lGM}g_5*Tk$-FjE?nr+z4xlJqTJS~{Elv<w%KJxL->Q3dL
zB~vZeNNKN|rxmdL#PsMnR}D^;THbz{yl2we)R}p??{bb<JiU|J{yK8mH_eh-|C*1I
zvbPyz@?Kc5zl>Qcdei1w&~MxS48oHvllpI2RlG}(liucBwBoha<mPEH<}qOlCaFuW
zd#Cek`|TrhrFMk&KCy63vpa0kB2eIFc;h0^xx(<m7t`+upMI>qWY?Wn4<=m>ytwT0
z!yOTKtaCnF_dE%HvTN(cZI|EJU6^F$v2=!<2=lBb87m@Zdq2GXMf<mH{G-do3$CtH
zntbHjr8TQAHp@krIXu6%^ds9woA49c!s~Z`I#wp&R=F<w<DyN|>~ekG{nCpg9xRws
zx%Fw7m9O0=<A-k@e2pqEiCH8W9T!b|Wnp^T+r8_ZMNfQf#Do6NT~Aj03Y&N)s;T91
zY5s}M1aadzv2)LDdmYhG(R9I8X-fu|b>E!dSC3y^D<15v@KHLwx7*Ck$p1gXx*b_s
zk7tMYzTCb450hM^?~?VGf2D2jcz5k}XufW<yHslspKHd8^(?bkzO647@q6QN??AYr
zYw_h{7T2amYJZkK&oNij)3*18$%&HRr>mP++&&@ZIAM*w-PVP&cX$2R=E-w%<0CE!
zbIC_LrL;YguUcyrY}$Kb9#`<SqJ7h>wK*Sb*t2`<v`4y^&1dw^S#R6xyy}i8-&7O-
z!qm@mI35L``19+!x@*X0^EY=ya{n{1v`*J6waY)Cv8e0Qc@MYW%U}BbsQ<SJ(#gii
zm0v8yZ~v6j%irR6u1jP6LK_EIl}srO&xONw0?rnB0<J^@GVh&TXxq0vV5gn@hOJ%a
zO!)pYRNhE8b$a)Xw={QApV0I(4r$zt@55C0T;94{BIw<N+vmk=N>@C&vgNE@bm*oI
zkBjE#HDZJeyS}eF>3R2TwRx`ny5*gIdiJTmU2EO{GaRx_NH<IRx$ow7>jJ}n^-@ij
z4}bLJy&UUzQ9@>=LgDeM{omFqWTl6Sw``368u8rXXt;0Mt!Q2s>y}C7s=_i88`wXs
znyRA`dn-0~rPealwbCzK?OM~eeTY3+svPa{FtXzbQ{k-OhfDumkB&F|(=7M*?<?i{
z-2V(5n{Ia6*GXF&)bE{jF<#&{uY|DruY~Rd3Cr{T|Mohy@dfSn%$Wa&>8VagS?;Ur
ze}tyU$!#y4RhcK7JF#me_p}_D^v{dp`IPQOttmE+_MafpBX}&URz7mORK(_K#kpF~
zb7q`)?Oz*qQ~%B3EM6mJv58#_>@0lE_tpj-+*O#Vo8!E)yXRK!M7!Rlk#~>FY(JBF
z&(4SW3ro_HYhQL(_qyHVP7*4Q3(ynKw>M463NJXl-t>y^KHXP0zaG<0`y8?%*6DUo
zSH0oXnr%INh38ETrb!jfk@AeKJ?drp=%MwtH~&7VeK@;1zfF53?<t-M;ZOg?Ut6^8
z!{5!Bk6yicy6~XluY`n0Tb5m2>uvISwSkjGXF!VlsZH-ntt<Cf<(}ESPV&28-+r^Y
zsiBz<9+pii^S{s$a@6bO_Wj9150(3#xpaO%#wWVh)!kL}^QZFMd2d1|Il77jNq~m3
zdeH{=(MSKYUli$Y|6~($|IOUFB7yNv6^-OBBt_Q;Tc))V-t!D$^haKzb@d49v>;?7
z334YMd6CxjQskx43|AIWX&xVH4)c_cKh&LD8V~G{{Lip2YUhhR?r9U1B%YsI!OmrW
zY_Cp(8wX2JW#-?be^$>HZdu;x8S<he;K;ewpH}k<|1)svPXF_H{=c8{kKO;MY>WSV
zuKxFv`ra!0Y4h*IpX%4O-?4vc;eUqYhr8|fy!>~0{a@vC_77Dz{b!h0{GZ{z@}G?V
z3@y`6|EoOzpP{Dz<9hc`mw$Kv41Bu$yZEQYzvX#v{(bm&dHgT;IrWFUQ~xvAJpa$I
z&;7?mWNYL4A*xUNztex}{>@H$^Y6pI{q}$9w?Hv^@jszx{h!b6|9-MRwpSja7~;k^
zHRkEx+rQ1Ne<??tf8=lc$UFa@|E<-3hD$b&k<9J?;hD4l`JDd@-%tE!kb3tu^6a#^
zx<@=_Uza%b@!HP%AD-*$pHGqhezf@7fd?zCJ&m{X21|GJo9HGRV;C|8YRKjP3@x+1
z|EqNT&rlQYx#-T;e{(jUnfPAE{nVa9uW8(@VPRe6Jgz=@ax$N{_9<f+19QuldT>n0
zMb#J9YrXt;{BT))<xw(2w*J6pRF9%WX!gZFTmC)z7XnT&E5J5Y)MsA*H}Sdrhe<c<
z&s+Xyct7!<2}*okU;Go4yjDOW|8IZWn|~k3Pb;(M3*8iOpAL@ct^edxZpWMP@ieo4
zSa%#8>s}0j4Gh1+!=%~0dF#`!JYV0nGVab_)<s>~Yo5W@5Kv6RR(}ol6`S~a2vN_{
zJL;cs7JiXG6V4j@zQZ2Ij#Eqi<5KJ9y;S^_=$(q>e-11E{jRrq_<F~O<9`~1|9<;d
zvgtp=qd#5e!hhSJxv{8A`}WRH3tdHm8K4CYP3WQ4Cc-*p9Gd-F<New*L1Mvwbe6OF
zMlPB%Y0_pbqr&p&dDSZ;@80l9+L4xOU%L4)*W$Rz<(a&1cRO3<Isa^arQLqDq;t#3
zeK~t{>O2|#^6#yfu}5d+zKmD>D^GvsxE3?%QvA%hVQ+HRtxpq(e)s+0f%sy-%Oxkj
zYJPJuXnw!*#iHG_r8!J~7m8^A`z*Jn_Eo`-$dH=zE@6rNtsI|pQ;hvCFTVWXhQvw5
z-{<c7#xCmmt}H5=QnTdQbG<Z?K-WM;^cs+w{2^AG&i$y~@%&<*?t<;xBIhOepYiyk
zZTwH@ZvMZ9mp^Bv_trl#U;4NHTVj~@$uotY51rD|`fe-3$7Rv6U45hYAJtv=|1w_w
zY*K$HdhLIP3$pgcK^No7zdenA8sB;M?_}@YO8*&JwnzVGSYTC`*8ju1bpJ2r<<GLB
zH2&RL{b%(@{x`E?Z(=hgJm=lM<3*b*J=GY0#-xOY^S&v{FnT4Qw4HC}&$5Ebb$85C
ztzPb4a&5QqWWmN>u{E>(FFgFu@KE>Ke})%)^JgxGd*H>Q&9#;DJ~{Hv-IJWDW$}1c
z&ObSgYwvkaon<*E;<0zL^RY*_p0x-&Y}k?VGhJ=w`Tq=XpSZ((XL_ankiPh>DE{s1
z=WZ`rcCYaL*|4o2exJ~jdz9Ly5wY6Lcux7^*A-VTJ&CW-vEyC+c{4NI{=R(StnUI_
zf?iDAWpI$)^upZ?BX=F9TPbtZEy84y?_8byaK>p9!~1U6y>=Qtntt7I)2?qB_2Gw(
zo%NkFcYDXzYzr|76OXIx3vI64|1j-!{TFusGuIuHA3oc)by8-!z>GVq*<=l`&$@A0
z=h3#y?_4-VZ!ezPvch_Y=vp2(tuQ|=t^3zn!k4y(XZiU&Jl<`3sK6ve{Z(f4(W+N-
z%z765lIeOKSADrSYr0g5*{jP7?kz}PYaV*($yL+ueMW*o29f<)sy*vZS7`6Poz~eW
zH{EpEo3;0ztN&-<%-jB-!J*{OS$VjlobB7Ezq#a_UN)Uc=RHG_p~pSBe5;Gevx{PQ
z-W|XD*mUx-<orCBtPMB(pQqbnHNk10O#Rkam(RH#A_leltCcpKknUh#njCI&<kF7^
z6W8jn$*ujVf;F`SU5xn8kQe$d^vA6K3@Ml1o!E~hB~0tG->zM+^~d?Q+VnC<^&Zn1
zi_>e>OhYAACLb-iCAaL>zNw;*k140Ah_Wj;cRelD|H*czen)ox!TL|Sj{iQ_TYdfX
zCuL(m(WI^Y=F)9_a-ObtUT7@p`ogC`t!iSZRSArTe_a3PTor%)SJ%eSKiU5o6!+@C
z{_7gCe%5*WPI>3Mr|dXcMFQgl->`}V`sj&XxPABb`j)5LGbi{Ix2VtCdcGws=+u(N
zD~*>XZC@A_l6UED#G$Pk*EZ#+7cc+tZ8=L-&9q5A8%|FOoKsL7eCL&u+P1FfA1Z%e
zP3Ks^z<93s)=nwo*n%_9rLx3sJewEGd8K&Y^MLEu_J5eBdtjcu`srhTKF$f*c71~M
z>dkD6HlDa7v|5eJeE(0=6OX&%wmhEc$M<J%^zW0?;yJa8OEzAQeHB%^_V1DdM?N!m
z&Qm<b@t;Aesy_Ypt4Xi!=l4#%-f{4`{gZH6_p16^Mf)~HzkG7|x0Tr?|IJp(E2DiE
z#y2aine1_9g}=q4nM>mj=Rf>C<KKmi=R7sc6L*!ZE1It&bcAt|dB9zBp0vfkofc2u
zvfaD>p#H0nE0y}Yca~ib*fBj|n|Zk2#U-nbSAVd49m{b|XL8gW$;qc?nKC&PF-oqp
zUS3t%7TV*LoY1~Fx&Pd<HmkK3H?_ZBDcw-=%KF}2zIp5$FMRpERJ_yr^xUrbeww$^
zwz61>zgP9&;<Pl-^U1Gpb=ORpQ-$TrrB&9X>##^|dAjyp>GY~ALO&|IPR-An%4F1h
ze&IU5`A<D=JM{Lw6Yty7FMZ^8iqjfvgQLZdHi{bZ`eo<x^_X%5?CQRgD-sx!9uVj%
z!WbU(pJ77o`M<@cTOaKIDYoE0L*@RtNw4BR@$UT3ur6)e9`XMSN>}9Hn}1%v?8JYD
zR{h`uXZLr?J1<z&rG3X{!f3uBkZ*)<{AW1fbpP_%_~%jov_pN}6c#W(<(nTI@vU#;
zJ*RgCtCc3zvSly6$~SRU%#yP!I=(Ht!+N%=B13CN<U!Nwwdd^x^?Y1XE^nR5*3(xa
z{Y<FUfzRzgb?EH3_P)MM=8+=xWdVD39`{<GTb!7_^p)7kGWiO-NabliGtAAx>!iEx
zuA8R0@Oh@pB<t`R+eu5`+@1aOU7`QnE5*H+_U9`qez>E_<M!#rQJwvzqQ%XTQq$(E
z%`QtQ<2@GIUKs0fZS8yAMJfBE-Q)MIYk2;7U2)Xz=-r=ATFw-8YSUsZPF|U~(ABDX
zu7uT&&d{yCeT&`NBuzMi0(MLgeSY6`=7N{=+!mLr9+1vG={T=t-hYPapO?9>a^!Dc
z&}G?vE4koy^@p3QtJ-#E>Xf}??(tOGa;Jp(+U}hn-X03d`sn?<xcAtm;-0WcIcaN`
zZd<%=iQz;k9mlyFFHa7YKm6^pqj5ys&r41PCtTxm%TC$m?3iF9<RY*`zND%C*rKNA
zeixH>PLtkPsB*tleEZL{sa6%4r-G)=eP3JlHtAuoAjgW;U$)KuzSHZzQ)7<uuKl||
z*&P1VVEkpRZrQ>2%PzlNq&(H$%;Lat)A+Ymb*mTMK3G_$`=nWtrHUuyyVcz(=GTH;
z)n`g6D^IXwcs&2otTri=J0^iOdrOmw*K2h}eb)Y^wD?CFo6zz6invFHlh*7I`F(Nw
z!D|nC1P;ibx%zUe2jiratJ6;>PVG%*dA4bN{@Z6C?ig@7&M}KDJRj_}rQpj$`}EY3
zUqAb%tiCd(^6O^j8jXcpEtwhIr}*0kMor%?b;PbR+QO#wj<Za9{`#n6AD%uDi)ubu
zsNiBaaq{blq5ApDr@eceduFY$b*}U|!}(s%mUg!4sD$2q_svS{;nQGYU6W0BUd&Y$
z%sBdlXPN5(?!K$h0hg9s`QvJmbx<(W+swF6^HNq>8m|WDb%R|q?QcF>(jT*>J$A>0
zlQU97nH;PHV>gv(=|!IQUVq|A*IM?otZU~#3*2=<QLHE?@~&jf`h9D5tdV-Hzg4C0
zRPC-+XU-XPOkDpYd-8g9gH{dSYWw+L%c?3Qug;0zQh)u9NT6%w^TN?$NCa!|3^b^S
z2a{5L`#q3Yyj3KKVewMYa*yrNYoqRo|6aPO_T$-1FTN#L#Dlavo%H+{id-`n`}Xm?
zmWZd_yQ-@NOZyq-FIa0Y5Mz35z0styqn|Xs9L@aU6TSKDr;YO^EnjZi^{}-y#(iV+
zruc=YJ_;>w|F$VS-8Mhl_Rb*-<xJ@dWvh!bs(SbM>P&kY=;nFC;&1lSi?jVA*GFAU
z4U69w{qbd>Yr%9Wu_YI6)=H@Gn|9i=mO1M#ee+&3_SKh9mZ|-#&u1Jv?fd6kP_%bP
zlIN1vYv#Y#xJUIgEoVvieLY^qVCk|+*M+=Z#1`y4Z}DjM+MCh$Uw!*_a;JLgX)_DX
z^RFXAPD}MZ_?`JGljrcgE!S&n`P|m{CKl%BiyoOcUn_8G;)!*0eiR*YR^6p~$YZfm
z*zYA*W2MY?Y`S7{@|;Vh_M7^@NlQ~Nbw8aS@;2}Ff_rvGF&AF$Uq10{LZIhc@3`bJ
zTh(<7A^pxE&<F$5x~2aa6mHM|^(;FZ-sU`dW&acD;{OaWXLBp`|GBQPznA|!-gn}E
z2Clf}jkD{8?Aje&MZ$MfQmsQf#5%9-AJ_kB`?7xhSJ9-^;Kpb;s4*IEIu8^yd#BV0
zacL~tqkJ7cKDKjd%-VU&S>Duz=GC=#EqBv;dG?T(_d?^QIWh6U)j`{~Z@<3h_13g4
zGv4!TIjbME>GIdiyDm*PuV+m@DIaxm(zD7$kGQam%Tt3YL`6&Yr0e=sWv~5u|DVK@
z2=B{JR{V<o_1Q|=*57}_u04iZ){2}mo3sZsylS;Jc~!LIqPw^Dy*CO|EwFf@E!Pzv
zxALOi{*Hf3{n!4gV|)8-bFS{Rc@x|9^G==NIA9@~^gU|B?;NStofDVGPo2=iyXW85
zv%#@f*<zdeg9@W`4(+Z|t+gsseUPy__HEtXbK7$BeakZcU9R5I|JCaV&lRa)X<4)H
zfwL7(A4%-jjVYP6wI{ynDgUNF_fIXFe>Ct^r&#*TfJ6~K*3!$XJ6HX@Bm3s<t6y_A
zFLjzz8S<riR!CS-mekbaf2uQuw^fM+{}Z)}tXTB;-HoFhU(@6t$9y<d7<4z{#kAD+
zzRo*)UBlvee@u$sUz;z!&i>b-rSWW8LG|md-KbWY|8ec(iPx%6O6<8c-{qvHIpdwW
zl7|l*?!PXLZu`amZ?8=Sv!DHIy?3`?oU&h<wMqS2ip|}UPR3cPua)>$bLHR4{m<}x
zPyP2Bt5-kJm95%)xi&@qLdg6N%^CYYpUeOKBkP*0{PRB?|E|}cO3E@9*fzI-<>I7(
z2W+DI5v@^^3T0RMPsf7)GvxZ_+_-W3<+je;oYth62g0+yY=|&z?d~eyd9drKjmwl-
zTf0pHUCZV`^Caxh5%TGwl$93sJ>y(b@px+F<@Xx4T2Y?MH#`*=Pb<2oeNq3b_G<6h
zqV5@tr#2_gX^wcaPo?>9-Xxi-<L5uA?PcpawoPhk{v+*KmyU!i{a`IuaBJ%w`;Q-&
z#w=fX@IM1r){nmHi;o-0m&!kmo9t7ucuJDT#pA!7O^@p==GANWI<t{`<@3{zcy0GT
zUCh5~*3{Cuw!321KUz}n_2$0U<t4wrnLYn^v#ars_2T{aF0cL1z;gW4nt7hii_=z{
zt=aDJ^gqK^9sfUjUH>!G2K9QK__C|}-!IV%^EqGSR9?RPp-leTj2gv-@;?iiUn<x4
zx_^_^f0kOGedS+B!d$;2r69#3msj23|5?@l!Sq5uM~3si7{*^G{xdud(r^9Z@^ZOL
z-A?;67dBbNpE>+HZuzIhQ)`R;<A3Jwez9zUozj+b%6*yqIr{O}cCYx7Yb5`*OziZ3
zhMc1@9*=vHOhV<l45QrD4zInPR&+Ld@0!Y2@mb;WAKxy%V+P4jH!9{mU%JHX!S^!x
zN7;K9?Mj^XaPc=D)xIO%Z)HO21h1(-dmkPmf44*UpJ<*u<I4waJ9qA!yY~e1pZcg*
zE7q;JxSYRLrTEm6r@IXweAw2V-@DClr|gr{^;VbKuHCznB%$0>zhObhm9Q4<1Lh&Y
zv5t8sHl0`z^U%@OSM5@Ga$r%J%a%Y_L+t8FJaS>sFTY`!z~=Aens7?eL!qb1`bv2J
z)0;m38SXmOdp+4wFJRh|%X=VG=~%FA$ggnQ7?(_6w^+YZ5|-DOJ>_dpR$0lDtI{Vb
zKIzB@?MYYvxP5-tbY@ET!n(I#zN~rc9lqnrlAq^pwBPM1k$)Wd_1UAsb#-sdY}fFG
zN|nvcjn6ysZTjTMJ8M2{laZ_a+gm;>Z>iYQ9F_id8LM)(J)gFgXyniOnX*}etFA16
zcfWJwgO5TjKg)e?8y&p7+JEkD(<|>@Eb98+KN()3A<w}QU~ApKe|Y(A$)$Fu_TIS8
zT3c)S_q4C(>x<PIU)I_S9<I^n+0I;ItE~Oq;Xi}a;r0B|aYfVBPRfT)eW-bP`CIe+
zm3x;|bVd7axbw`TitXM0$FU{G?uA09h1=a;IIcHdy{m3z;j+`a)J_S1R(z59aO0)+
z-L?T6cgAGhEBP|<MM~qfiIK6+nk;gTqQ_s|Q#PM=ns?Kdti#`1zWxgLTw?cE?Z)$e
zk&SUvZG5zRkJ<8b?c-W1<+apTTJNH(=%vC|N)G%O^r?|c>h#Yl*VFYITr0np+o{yf
zGTHxDz&`5X>Z51k-wWBzR}EWqe@38d_K*5EYJYTZH68yO_kQ>Ms)x(@DmCsMUm3N0
z)3WE;H>P`j__Dn<Z6}+BRpZ_Lhjw!)blno|65;5+z%{As7nejuVBEXsCz3X9H8lFZ
z?TtvFYl%!Er1?ow700a7tJb@CqO)=-OSt;#irfcxZ71l92g{fKs=6NUni$Q`b?VZc
zSBD<GN_<*jYb_Uld9CQ)N5M&YO$Qdl@PBQss{i$xZC~9QOSQ+<H-2-8)~+?Rj(Qk-
zzdgRLn}4J7*L6a>?YVa5YEJ*XN$_v|+OpL5-<Qvwv*g2n2II0WjrBP{6DS!q8WJO{
zJ*yt`ENovWxgzS=#eaX-=zi^yn%Cv0`K~K?N?Do4w%+CT(H8p|yIS{HFEc!U^Mu_w
z_pfWcv}}9LzXh&3axL2P?_;GNSE&U$PCZP=)#kg3F1m1Z+ua+1t|A=Jbx9bj9r1~4
z%9byTnK<vkgG(13)RT28s(pDP4n)jbI%odtV7=bu>fRRpSBw9=`YJkWv2X2^oR6mS
zZ`D`E=Hv>$i#o91<G{+RqHLm0{SJoSu0dVV%I22dWfV`q(W_y#FYia=imdPTLhDO@
zm|AyCuUe9&ni?u0+Zyoiz(@VR>`VVM9O%?we#NEz^gqKJexd&i7pmg_a7^k)VmH_=
zShO!5Sv@j)RPV3|E5H4-<1gsvZhvjA_oiIa^L~$f)vsm8+-Cc4n|LnvQCRXT4-v*B
z=xSf$4=!A}%HzXJ^G(N&#wwHrSqm)N7k*jg`q!J!KYg7&CB-u7JO8=5DU4s|ihlaf
z@QuH-{)MgnKNe5-Z|pFpgM2_>y#9ZNgo^3^nlJrlc;GUn9*NEPl@+WNu6|VOPz|Q3
z%Rk%>y83pX^6kveufl~DbtY+homt!XB7^w__&hgytnQIFt=Rp~TlV_j^7y|j;jS_7
zyZ1lYc(?v~>3@a`t|cLO1}N<`Qs#yBN!;?3ZdhgH|ImSF(UMu5YnK}O3m;W@wQh<0
zi7O=xfv&a>rztG+xXaA5?y0f~!zr!oYVYptnz<~S4J4IJbR7O@sRgYGZDf%VsFq^D
zw>{BfzI6Qwx8r{=|2bM~zwAH53BAkzcBubr(tgH|T|$HDDv7O6k*)K8GT#5saDB`C
zU!3b_+Hn78kT>7&`fux(dz=3=C|Lhz`1Nh_e+Ea>&+3i!kDq;ivHx0wbrjCl`|^`K
z8<WrUURG8vnmEycXVDzxqSixZ7ym>&n(e?U@~ro&vynmYg^353UBEb^7QKBJ^Piz{
z|C0@l{~2QCe}+4va0A#cl?1vn&}rwWUDn6)Khvz&|9ZFo7stA;o%hA-PbTmB&#><G
ze})BJFEDqbuHNM~@7J=mxv3H-ryiQ0cyDER>I3P$>fXC{UNMzk@pz(ej>gTC>MLJ5
z81J~gWw&nd;qO(p3*xs#x-E5(X=0Z@wZoqGhtu`Qi|f|ETYv9rLXz|o76x{e>yAtR
zxNn*rp<TG__AOUKCGVw{N!IConW<N|NZSg>Z`pb!v3_e_ML_zb!^{1n7>-{3!=EcQ
zm2L6j`&*v`omklxVB~qs{c-dz`{pfYxuY+i`E{&R=e<Dg!@aHPkw3cqHh+*^yZx5n
zdZoE42E~(@oBym%di0;+u=%g2{T+VWCYpG$PdNQq__powr)~!vSN`zg7y2<jbXxYY
z?Vp$uT3KY8uYX!IqlR_)y*Q6=vrgH~<w}cg2;G<WAi42q-r+~JqNTEB&MRDmN*o2B
zKRYyUPPp}hx#fP_FXxxk`sHsvUi>}Pmd%aBZc0Pi@&~6wF8QrqlvSU*y?0;1DI;5t
zwflu%Ouu#J7H|KpiGNO<7W>rnhJj}uziCvhzsrZ~t?8F?OhvD#?@01q?s|@`){1Ly
z<)go9_d_l`?|PZmr!mX2i^->wA#H|ujsAknD7B5Hvh!Bt<xP^^>~tWtO6Fr+@T&R~
zQJ??qw*S|XeMM*8ALFI3ukYIQdZWkn?js2)`hJgJzKzRNXnG%C`&ldcbV0G^3z6+n
zR{aZK++*3i^<s^csq}|c*Eg+FOuR9N@yD#^ruAIw|EQl{Ha%nQy8Bf|JrCseug%Yn
zv$|CBWy{758@F9~`^d5BuRtH05x;4fZMNq}*?;YoG9Qnxa94}sD`J#TUVh>7E8U5`
z{~5TxyJz1ldGYnQRYUA+LB|8%WnV3u{Xjdj<o4>d?t$6P*~adQ99#{C=c=v;mYL{k
zAN?+M_1?P9f{Aa<judf9ocQ`^_VSph*ZYL7KHo9>&9>{tb}7o%;tWi#9y0kId*t(%
zy%fy4mHKh_rX)rk24$P{&+FXwG9Q_>ZsPJER<g`~d!ui1vM%v!NLTsK@OAb17YvKK
zG#ij6X&8_Exc$$`%>Lc{XWB3Sn*VbwoB#Eo{TIP_g9@zT4q_`7k#@-5(l!4XPVoI_
z__ue-e})UbH76g$|M<E0m;TrF%f4bSx3)0ex`=i{C(Ok!K0njr<8!ULw{o#;@R2Ir
zt$$A2aC3VExNo;OE}MN);E}8PmGGsn;|w-zyKd5bTf22<pT?QumRF3&*V;9y?*6zs
z^2eHtixIbbZB+$~-ah$u;_daVI<xFLm;P~TIXB@r%ZHy6l^<xGo_}iLtow(=XFa%n
zZcB$bXWX40q5J1Q91Ym<(XV}V#m1#q-m=Up=y4U^VXw*lUhw@aXRF=6_a>ct^!n{B
zAM;<1!5g9(&av;X%|4zl#lCmu+Qk_U(p3d_t@yG&@KXL3fBw_9mwqg`6S?8<-Ff9X
zzj&U15q&Ohbo*7*@6$*3Zu$IV@<ow(`==_rsFH73^L9%v!>jV!q3cybd=GE5ke{!%
z^KJg27yGRH{dtnM?Wuk7%8JRT@W-`pn;-Y8`*vMA?!S54m9>u@Pk&h9?vqlwI&*EN
zo${`WdXp+1T~V5@c7INT<B6A{yIE4^x6d-%H`y<?@a&J+)1Y+ubyZY;!`i1C_UyQ_
zRcwpdW+M-$rX8}cSM81E_x)IN>+HRC6YsdoUCwd0wd#-4vNd~kJ@&)4tu~*fv!lD)
z&$Of`oH%GQ?Lgr_g%71$+q&h}FO8pl+DySP!N~Z2*AwjvrGLXcF55`TH7>ue`=-b3
zrSJmDllFhJKEHAh33N5U7+-q+hxb2&Y7_sj7ylVf?cze=HrP!Db;{}0xn}$vj?%UW
zJ@*5t<q^H?mhQiWyqgUJl@lgdg`D#eEUH#qa(k0jVC?ok3fKNKeEzNe_XB9BxYYfB
z^(y}v_OVvpzb)ba><{1kRiSSRO~OHYTuk@>`RDncp@z%;A=~!<4D(Ogzi)K^c1-x^
ze#Y9M^~s%!cIW?aT>GEl`6vJ14?u@%AT(?<n1A*M`&F%P7PGXs{by)_TO<FWIr=}t
z`6u(gA6Wc5ne)&62Es<|C)TKcE5kP)xd`*(PjoNJ-<F7f_J{SW*WNz0uDAP<JOv38
zu!es(4*h2^e;^;S`o^P-b+`WsWY_=vGx^^Sk@`ct`TrU0C)M9)>VMm#|GAF&*V4G-
z!mefak-UVYJSVY!_6Kor5Nut3|NPSXM=Nted<E|;ZF4Q1k4uaF><|2_Lf;haTDQD#
zP5j=sH=T_iTZc3EXKl@WWb1U@WwuA};@>Ja?{L-J`6cxNIbIN6(5%mX2sH;!MEVmO
z`c#Za>_)jJRy4czyG;}EjPf@S^5#hvUCVQvKdNRz)h}TiTMxZV<uf+9{}~je<-hxX
z7X9*X_J0P2-TuEG%>Tu)eg-a?b&u)PV%1*pA$RH4JJXvU-P3exy}g$?dHveb`8R&$
z|7Ct#|Nhc{hSo}pHMb<H*3IsHQ+nypsU?+@ZPs;t^?y@q{;%=d{_ijA5A`Flcm8MC
zzVbhVw#l!a-Sfg-tCoMWoBf}`;dlJ+FZ&<5qjJ95KTZ59S&q-R{<p8{PaS?`^sX{i
z^c9M$<dK~yfBVXRhN<mWQty1)rG3?Y<Cput*l+!3xW5#1)C4LI>^lAxr*}Rr313y8
z{Br*n{;mHR_Iv*mLFMEJ{|nP6PMJM%%H#v57oU1{w)M-c+%M+Sp4QyxdwR#nIBxpi
z{$>9eF4$fF_gDW%AS!3C|Idh8x7hO*SGPvbP5b*^@0;WdhB7^N(X{JF%D>H0`Q_tg
zzv+x;L9p)6>$Rv_Q4HPd|5E~Fj`pIx!N28w|1&J8UH|u2{D;M;oT~YsHVlb)`_F(J
zar1B7%K!5t>OaH%h5tnAK@kTI>eur>nE#o^VgGvZe+I4BUsqg`O?qW<F;L8jX~hB2
z6BW|`yjkSGUi{B6wYuc_c~R9X4<~Kj_+_q0V3@)HRf1*vWdAcv_`3f0*Q3bY@a%{8
zfAUBDXV@Q+RNrjxmg)W2efdRK(GRx%6&h=`S8F`p)Z}qxdQJMAE9zdm1>b-A&%o1j
zJ>)~^Vr|vBT@vnsZ0oB<UaY)#DW$>Z^d~vvS;Y}C>!<(TGyV4hsq?R{=I4dQZ*4Ey
zDw-E-_DbUWo*KR#mPbQYtn@0myKIfcIiqVj8f$0UYH8*iJ~`j8QTv?M+}1_jH3>6h
zc%I9Ihini{dNBXijgv_V@1F!M+UB}$)_1oXI!%5v&x_nJIpG|$W@fVdUCX1He`kij
z+47&^0r%^`Z7WVJYBDHvtvvG*qk)30`~0Y;_&<Yu*!*|qpG9wn3qPL!<JXq|42An=
ztoPUW&rlt2JYD>cTE+X`{ZEvW|GoT|v~T|9`X}O<{~2P0|1&s>erCrd^Ub}F)H6?*
zrs3-Q@0)$C{s}wiR)I_PFVyy>e9-?>bMLSCuM6y(D#ZWhe*eAlKf~tm^qBO<`fK;=
zcg_E8T9!Py>&Km|@#3m4swKj+G`m)v{~ogaLfQn-@ipJqo<7+(GoPjG#exk_CWhvz
z_XkB>ve;OBr;l;P>ZTUkbM>_)+n)%oSAQSvY}R$XR&n2f9sPgRZT?&poj27uhTqQq
zF3<C4OFrFtm1TA(TiEaH{baMQ-L;9Td-@(4^1Wf6)pm}#xMbqDbN-ihK5bhmweRFw
z-_VF1pO-IW<9MY3x>ZN}$K#z_rdj;^qqgu;d9PKl)VqS?8Fo9?eV=-3`{rktuJ1hm
zb$#)yV~Q-LEP>5E53W0Qrf-=2_M+AVzFX@n*uSZ;RCR?K-HX_{GGppDnZoeQ=(Rmr
z8jJQUUW0F*0$J|U+v<PL)8~J^a9#F6{!bl&`s?ohLT|jj|4C-ge};8!JKi(bpG;`}
zx6A(1`u&IhGjQ3pHwHgW2MvGZIMC^UxSy+!-v1QwZvX3D{~2B=?T+XB&v2sZ`rluV
z_Qd{YQ0;sDZ}<9tP1;*Oy#Ld_w*LMaL;HjC7n>~oxcKs`KUUc>QI!`xB>yw)4ADHP
ze5|PNRBfO3BMrYfDSwyb^=B<fxxVX7evCrIn+NGP9`w!KvAS^C?Y6(Slk<Na`g#6g
z%xY`nyfl?xll_g)IIx9V{AbA88<BRQJAcZP3QN<STT@OK<moPXAv5vIS^b&cdZSNA
za;?d3V6QsF_pqBS!N2AbTf*&T*?T+JOv`+COwZc3mh0ED^})+Rl%#u(1X~`9w%+@C
z-j;?tu`0<spUq02x$D&Pf2=-XT188mU-71P9Xqt=p<b-=kLg}5E0?X`dGTyvrs$ch
z%SKyjSeg5ecV#rFZP>K_PO1Ie;w958RVKg6d=$xMUva1U+I!QUxleiLUYvhbD#|$Q
zaj@9;AeQa@6&ClZE`PZ>D<f~ogg<BhiFSFat30{1deh%)$}h4do&|rKbLn=ht#?Jl
zr^q~$uZz00&s5D#L?0Ey#*gvutAG6J^50|sl6Geymm@Eq%D>NkR`lpUL(6&DkK6yW
z<ka7_KNEiWSN@--xAw1_>R%}BOTZz1;^XO%Kv#_WgYX^t7KFWQxPH0n{GW#R{~2E2
z^8d?n{meYp{|x=J?=Sk#5XJD#pXoot+I{~SO6|X8z4^(KwBtOo!0X4x={Yg8KH1Ju
z|1fdG=bNv@KCbyRV~N-L%EDYJm9pe#ahn*MBY!N~JkPGUJ7(4U^?}FvMZRC&xZu`o
zY1S5#rhv+tD`nF^Ia}?S<nh94UDZ`9EjhEzlV(p_vD8T{>-IFgck>>FTi-sduITwP
ztYmk0wd<tF=<hQQt<$#ZnpLs#(E4pv8>UL^C@L|_{W>MHW15Of-Wqlj*Ye!_+=7Xx
zPaA3;T$81}Fm3AEwX?6s2b;IFJ4NcH&q#b)Cf9QH_UsiYmln0g8|fJtWSez8Z_`>5
zGWq&!=g1)6^$Lr+v}asFE(Y+ihhP!GUw@(uTtdVnl@OaQ3NF2-dr>9h#&*6PC;q&a
z@&Aze;`H*5i?6I$bN|ih^}o1d|1;dz{LgUY;@_F2{}~*!_n)-?G3!TtI{$OqbLlrO
zE!teaS#SRf-SVI6{~3hczolOP%ewAAgUWw~N14?Y_4dzXF1^VNjNE^7di)o$-G3(k
zXXsq~+vxSbrt9^S>OY!5l$_tV)HUrtLt1S8g{b$R{Qs%-L#zh%p^sdA^YG7qh8e})
z#3-^mHdS;svP<oeUAMXY^S{KidW6-S`7IR^mXGY2jDKf#{byLP_5PFje|+W6$)Eq(
zZnB#QMaBetZ$Tx04L;X|9{KV7pWRyfJNeJFmq0>qFF5py>h!lyy7%acXQ}S7&<!bE
zhL<mJeJNVum=qke@41=g{<JSP(~sT%B)AcL(Sl*T|I7WKxVQXgSjPz(+yB{&TkhB6
zle(~>MIvcN@5uiQ6J-B0{H^x<&#++D&&db&fBaniCH{51?`!){%=`Z{tk1OnA{=j2
zA^y)Uzpndl2J=lD?tj~U|Gn~`;pXbi>x>WVf1SVouK(XzyRCklx&8N@`{tH^;Dy7N
zm*6Wx2+0k>B8KQz$csf=b#fD<-FNSZeNYnhqsLAz^!ceR8{dYkSg`KQF71H5t#@}%
z=w|8uUims8^5xqinT;v8j4WP9on|#xmCE({+Yx$o_N1*{AtCd8W`BE=xBu>H^Ig?Z
zZ*E0?=b2{}rp+IC^sKI6-Wt8B-!%i>x8~h?xuLexWtGi!mgaj~4n0)w%ynOAnyr;-
zmKYc%mAlFI_fexfE`GMPi*EXsUAv#G<>vM<I`H)KAT2-7*xS!sK5FS*Gt^kSLPsHf
ztF}h&{E~Th!gLcm4xMZAi}lsK8Y;crf#}LZyYu7uKebWyJMNze&qZn_?6q0^_ifF&
znD`GfAMZ2!&oH64|M&8rUBBvm|1(U8UH-Sg|1V4U897`sH{~p|NL(Sn6@{w_@GBoz
zo^X&m`5^x5_WLeKNq=AGpIak7<-;pi)aLj1Kkd1o=6C$T>5%64>!hQPcV3=%VTY%6
z+Qt{_yACJGa&FIFzVOY}<!?oMotGWX&ifSWW&WRmBRDwX)$(MWcU__dbHX%LzN%b|
z6+Hf1Q{BxhcK5HDd+e>Y+`YN@@5bV1?|$0az0cZQzv|Db4ORw;p9?Q-%vcfGr#vYj
zzwmF^e50}(8&d`5y%pUdc~Wg#`u;86TX|%*)G+ajo)7fr@0-d|bTI4gJ5hsIOV-`I
zxTbK~3iYtqb?r9aSD9Pwd1=zo$i~N<JnPh=Evskx9-lVr>9zk1l>&#Byw~-eYZQC!
zy3gBbc}4zF#};)xUEfif$g%aqKea6{i#+!|bQK9V+(1krjNGvX-54VCpW*n``M-Pr
z9IdK!|9$D_oE!Te8XvDAc^q^7CYvOBOugGq22}u%T<{oNGrR!xu}eSB-2VGYe>&$M
z70@=b<sQU@1r9ZgPj7wy)2?s-df__j1NT2gDEwzw*Z*hr#_Ri^SXTaLh~+K3$M&CL
z(n0pWCG}6^>)Zc{)bcYeKX!h}i$z^G8U|`iDN5&L{_5MO)!ug9w02bgBzo+}e#8oo
z_$B`tPWZ{MP43vfY3gsO!0;6lU&w!$7*T)u_}?G-pc8(j`F~aXXV}mCPt>XF$2~9m
z(?9+*9I%`G>e9cY{|tXr%;dj({LfG)4L-HE?eM>O_U{kZw+1ZggR10T?frBA`aSzW
zm-V*)wW+_~1{#rlsJP)j!}9*$AMQU4(O5jG|I<JF56rJF{mXy-UjIMCA?L*Z4F2-p
zAMQWY|HC84{^jF;f6RY`a%ntS{AYjvABJCB_Rs$H`}BW?4-;-+cc`Vte+K=>{}~wU
z*6Kg|x9`)Sb?MgYhp+1_Y}xRj=d1n0$k3IoOY<|A{`t@FK+wudUZe}B7t{X<o?id+
zHUG2vL;XM8_4YqsV?Xmh1K07_Gvxj=^#5V9J^%a<*L@t}gQR&nk~!-VszK3k=U@GS
z)i|RoZ2v=<UH=*8ZTOcCj`E_{|EdoEod03z*E9U}$Nw=@&p-b|v>qvb-I3CW*7ZNi
z)8jv1lRxvHp;i7vUu^yPh<f9HBL8+az6T{M{=5Gf>>q^wXJ~<^Bzc4%R>vcB)E}}e
z{m)>V_~-nOKuT<`EoQ#|@jpX@-1E=>SmP0i4Rq`>FFd_&{m;Ph^!lH#?Vs%*dLpsC
zp~21Y-G0tmlpuRr6J5{x=i6DfKS~qI7smTXy-E4-_e`Kon@7Q_hzIlTw-($}`Onbb
zVSm^D+}@md>*UrcTE*J@XLum<W5@cWJE_H|_mE!FuCocEOG%r3$f`w*zbyT;UBQeS
zt-I1MgoA=h6dqhpYD}*^|Hu3{yKY~&QLlODbfZ81)$ER^x-zH#XLzUt&-y4yqgC|q
zN%x=i@;}(auP*&_|N6c7{|qf%umS}s8LScsgeYYEnptoD^*hK+L~(*@rbtoXzxd<-
z7;2;9&;G4_`k&#4#~NsH0&+4Uh8McZOsSs^N(u5|(?8e8SJwX%f|q<E^++Ysia>W&
z`KN#CAFv~wi7mBab0#jYV{s;|)I)P-SWeE8{WFeAYfad_>Bz7Ck1OYIXo*>}C9eJh
zs}`TuI~UX}uMID|k+M8;dGYMmGxq(Ua`M=Pe?{B>sDvSx=J3Fe{Ljz=%jb(R3Jmsl
z_4W_KP$LUof#ArqK`?v4<<x$Z3J2lnY$Uso3c3221K&a6zw^(3hQ{!JLU0Wt^@m*H
zHOE@yA_6JTx?eZ2{m&r(gAwFxP*DT-CQ`kJ!`Wy>4N7#vgAP|AhpVUwLQY+4|1%&;
zwovW2R<Sk<CQjY{T-tN>@{WU7rt6s2?f<~GSkl@poptq@=~7$IY<!+=sjSSlQy)GW
zzHW6OsN-_{_p_bxaKZJv7ff0%U1nFeYRgT~99PtY0a+rr-}67iggNr>!xbOB|EW_R
z|MhPD7s1%N$^RJ?E$r{DDU?4neX-fXKOVmpZ(Kbwb9(TNy^k~umX*u=3blIm<@`_K
zb^jS2gnqgaWw=^x>CMeAK87y!(mcM)(rc20>W(R{T;jzSxL&7c>)IVU<ebQ}R)Jv>
z`|?X`)`VS)Q|L_G=KA@vrTn4V)vGLvcKOBb7oHW_ynVOae}={*-ol5q-L<BcS}1&5
z7M?L}+pT$f3(~gxM1SmmKl#q(_c=Q6bW%SjY-<ZSKP@R|{_&;i(@rke*lWdfCMeU(
zYL2Vu%~$$~%Wg(*+|YY3^jKM+#nRo=PWyEonQJG>bAYLU&Ei*3WmS~rTKm8?)n!}c
zc4rlqgcr^7of36XiN)}wU~Wqa^V`rh4=;bsb!@u1rtsF2+z0HI@4M=6ZrbJN^``!o
z;npj$6OUi*)LECb-MdpUwe#O=xwhThv!yN9sv6yna#yK2{&as#)d$U8tw)L~>hAxQ
zH8c~^PdQ)c`}CCNXYXY#^%*-&Ps=SAb}c`>EGlwIz}mH&7xUk<S=;u<;@RrRm)iv5
z;*%tvwuVMJK9?<G<=39IeNWqFi!Fbz6^4Yq4SL6YdcwDFk5~eG7w-*RZ&esL+4Ig9
zMRkp3S7(Hmi~pMPpP?k2GxkpWtKh#E<M@iV*wrsQ+ILI&#i9j!_H4WSR(1ED9ewxi
zDeinawP{s<QT8Fv?J1Q{zJwbt`{VLq+WNX}W>HV~OxG)0WRP;FmN9|R-s|OSuPs^^
zYqlmm`g3)^q1wEIt|DR1;JHFAvEAFew`|FLEUvfon$qEFjaLq@``k@U&1T(U>e5&O
zS~|;OUjOmA{omsJr*WQ-{D1st|9kn*>uc}+Gdw<5|LfWPQ+p;Kng8)4|L^5LuV4Q#
z|KpOF{|tARf3n$)dprx@Sl3^kPu_Hx4fyKMTe|Z<L&JZDsQYX95$yNz+B5z$?Abr@
z%g*}WyYyl+r<)bHo19|u{O*@EFMH>nF3ag|eXcxfD%tm+cG7LmE_%8D=7-i#-^Gsy
zAOCUf^6ABAGfv&LD>G<IRF$tXuljpjns@HeBmOFe#*1nnS%-+;dD$<zciBRovKL}^
zXV%~Ub+uO8e_J*C{@*6NCOwg1&MkrS=4#fbYZk3IeWm^uZ?UDjU%RYp*`=4;{OeN<
zx<t#8RC_tJ>VgCJ?5fW8T&~)Zoc(0m#pdHny-&}&=UpwfS>#*yt>@CNWiS8EH(P5r
zt>i^h>$wkBv%2nGJ9XG&$E@s-*(awqoRGBn=>Dzxu5WZ~=?ne0qT99i&F?&Rd7_l3
zs6bwt1w*;%jhENTri)F=^3LR}<UciOj@Gv?%g?gT-FkNErRamttvl`tZepwMFYH?J
zbloYxsJf<UNs|i$uYa|8-JQ8M_0#j7#g8paMay>e7`&2GY&#bbp0(~<;nnt~5!3xQ
zPB@)!x^+z<V@qm*wEgpYvrkLsPm9uR*{m|@S>KumE7qM@+ctCI;bPTUeutm3XPo@L
z&M7@oy*%{Rnj@`W*4em8-?(XBc5aQG`fc;nlwCb*>-p+WIQjp2Qa@u&*Yyw2|Fo$5
zXISU{v+L8x`|mD)F*#Dt<?in)5`JQyMqotL`<^}44_6B5#)mfD3YTt4*ZJ}F)`AJP
zub%a;&x}*tVjkuD$Go7q&2V1GFTpGFYjZw)oob);;Ai#C+wVoMzR9@gD%x=fJRZf#
zAmqoeV(q%M?{lwJM?Rh<rlK=(L#l;^h4LM#va%gQ7hOdfKqH~r+(+iWTKjMLudug?
zKPLYT-T(Xeukf<yf1ibO%s(bo`Yry?peWt{{cO=bL{B$1Y$+De`1Ke5$<*>|ZT~30
z5?tZTY^BQ<PX1XabFb?!)jv^Z|9$lY@`4ITub2P(Y8m$rOJAH_Q7>e-;!SqY?a1s}
zrC1vW3D5cS%YA+)ZF;eO!t$VyyN}MVT<K>jx}?|pZ0fqVxe1?~eo8zqEABtEeD;wd
z?-kK^yGw%wZf`Fu=RdVkV{P5`u<tvkU)7Ad_HX8%dn%gs-&R+L^cNgi<yU^CJnHG5
z4W2SmC4naMYx_;RUCn#fTwSNyylBnajVo7Xvj=tEQ|9PD`Ph2%{7buJy@R)H+OpB;
z*3#prI8Odv!u4pSr}|2@brQEG2Gz#DaQP50T`Ni@Z++*pe;2BkH5h1a3@&5pTAaMv
zH?iis;0nc)ciI<9uKA^{-nHgd?whO)TP9`deldG*db;xMQ;(7tiz@4W3(pdd4_>M}
z=UTU-<0rd!?Hkv4Y0Wd*oO@?#^P;J1O1K}nOcRN|E%8ppf7g>M1{JLfC%rwos(uNB
z!P<uXOOr18KF;2IV;Wc2zJ^N6MP1qxUU9v9^q(PU{=2S++Pk~|de!gJK60+C{rndT
z!`m*kmE}j@e>v-a@R#Y~O)t0Knh?E!kD1{FgZhiNaf>7DbU)0S|5#T3x809DsWP5C
zA7yO(tOWmwK7YESBk^Ov>fqG+sh7il=2cGiJ$B;hKFuH5&g~D+e)*XBZQ7=rTDDsz
z99eOqe8mCfjqDQt8MuyhAGxP@W$S+i?u;_2C1v%uta!|WPwVbF6?*RPC5B3sRvF>G
z?8txOuXb(RzV)Mi&w5i|JvJU)7A9pitNh#}^Pj)dwm&K*`}F;>UMXFpSGk4@FaBpJ
zEah>WrnFX1WaWdN!hYwtXJX4_FMku7wc^s8b(b?U)-IWPZ`FJMJ?A{{ecQ<W$LabX
z-P}ie^&@7<dIw$-Q`@@hQ%jPgVu1OokH@}R{Ob9>VvqIXvq}1b83i#Ft)(|i%>-ke
zOdd0CE<1L2?Up=Ai$DBpOTNW(zm4|3u6XUM=2q*!>G`+vZ8l2AF4x>x$;{4vxxuh&
z<Dv@vm<?Cfs9pJ!SXrC7e4&SKQu~Q;i36X#Hs0^y+c!Vv%*M$4&9gi$tD^qBihIB7
z?fcmG^1B_L&b`?BP&fQooW@7CWzS>U?)zNY>fX8Nw~^R9gZr;eCpR#>zW&UvY3JJF
z<NuT{B~K1)K4w<cdZau$-hb2O@3+=eKB#=^{`vggs`j=&l46Z~JGt*|J(YDm$nQsg
zVvn6;KvD7a^`fWD4@tkQF}f!+`RJto3@OJor`;3X<Ph4O{yg_)-rD0bbG{w5k~p*X
zk;?N^3w8#@>|R}cKXv#0i{GjuWYTlFlrD!)|JJB0wr=C(l+AN)Yo6Wr!N)$xzJINE
z;<}34$HJuccWlYcyYI7M(=3m-Ilik-K4)S0^QJPU$*TN~)%C#o2h}(C<R(AL<}J8+
zR)jOZSp24Q=f!9hCjZmBmlxS)o7K0OfBW>x>=%F6I+MNXK@*qk+$45djl1Bs#Ds9=
zKI`PQUdIyEB|dxEoHC3$_A>JKsXOb#)~2tI4Lv-~<M7@e?T>Vm{Xh83(zy^B^ExOd
z`p)VjvdpWM*d$Lfq-<mSX1VlZe1}c=!|1izzE?|scj)x+e12_qE6eklSL}vXe=erC
zFVA1!9`T=nPbYcyq5X-ws;leyt#`R4-%HmEnG~(euI2CdwfKmg_|;O~C|!S#kE+LJ
zYaL0PbkF18#eAiC|Dpw$tABINcx=G6M498L`@Ox7*B{l3-WORrZI|i&TzQ=}<`4gE
zw%hvfvhBI8sjE*++SENe<4k(K1YZm9+8<T5*{jzjAFSi+J|tG&{cYRs-FxSpFSehi
z6qbB5pkVRj<NF`R+E&cC?iJCwOYh^pS<xRh{h7YVr1$Mg-ur(*ho;Be{uw^0V)8#b
zMeyXH)z{toe|(<*Z|C+;>s3Cg|EY2Q_wrxqmpiI|GNsP{y;ZMF&S=JkewD4*)^y+z
zW7SRn=TQEi;p<!Ve@)rXKr1MYpM8I^er>HeuI&;F_(MKikDmX7|LfXc2jHxqF{l4N
zbHAMWpW)q$JCl#(JUjDu2LEQ6{`v0<`~5UK*Kd05zvAy4p_ALw{>`<1d~4&1HE~9p
zukJVYUlkb3yIWsCI=g%Oq#K^Hr^PgPFV$V&;li!Mo7JE3WTIHzi!Z;zWJRu-ZQ45T
zP=%HL!fm&{WS8gahzTB>ke=0i^2G6&pqJYheLnKC=#}z4&tt`X+I)Ret{E+!RCHCn
z&_>o&Z1LVTrMKGli}o!nP~7ac<M+O@s=wLtzl)rgdZ&D3Gcrt8d-vq|(v->bwgj*4
zU2XPt(;KY{tpKlAZ?0W7J-%^!cILw9?y`Suk40i%Yii$~U2U--BD$#Y-lmKdzwYLR
za=+;QwN_oPIxFw)kDjxIyl2c-iykVid0y~k`PoMnD*Wo-a-1GsXFYlT%euO*`q;JW
zc2EAQWtF*cW$~J=$0mz5HfT8UeYjiCb$U_Pv8A3N<rDW>1uk1FfV0{2;CTJ+^B3m0
ze^^rTCD65|^U*x-57+%SUAGB-^fx}VNqeuWDO+pJ7yhRTIt&b7UiL4#BD|zd;>YDm
zzK+g(-fR6T-vwP5TqlNhSXc3d`v2khb=`Y)jn$5P!I<oQQXg4<$-ce+r&h~FwJ-Sf
zmWk)<{`^Y0xbz2~J%8lus<qD++O1uCFY3gkdp<j?%l=-CnZw6kcUWTm@`^h)%dPfZ
zdQ})0u=(PP{fAStSKCi)*S>XJG5wLe{M|p6a>i1BS40OLxj9?q=ppS1-rJLJGw`v;
z%$gthV{-M%*HyKx%Qo$k+fr+#ee+%2vAj8Zx+L7h*&8H^kJnXkt$r|{G3(ZUhQqt9
zBW+yElb6PNye(foJ5^=sgumA<v^W9^Pn@@Xz5U8Rd8=37s<yY4mR-!xeJgfizG9lE
zVgOUql0G+fyTg}uX<t44GD($l#a+u=7dQ1?^S_(<_1dKklV(TB#NI!$U!cG3(knr~
z?p0A)-5J-p^O)`&3z={*qHs<X-`ri{u4~ReI?rEG8Xx*`-puU6;xg`-(^<wm6_Oqk
zH1FL!A9tYSZ0waf>9eaU%xt+n9|@ms+`X|}_35FRoC<<lL$;sV^W|*DkJAS)Rm`hZ
z`*u0ls&3)ZGpXCB98!%ddm^fm{;vPpPC0#sS8fJ@fzKxAPx}>F^j>!MeqXC~twpDf
z&%60!e%!S!`=v7c*GZpvTXMOO^~<e_CO03ZSIG(qMw1rIb1Tx^rQT7xpQ}ds;;c*W
zv}dmkdlH@bQbjTBidfE*{{1J;+wJ`ryKe4>chR-ckylD(%c6glc8hJ<rFJ*jQi1n$
za|43}vwUyxwjU91A6$NT)<162D-G4ilAnvV3z{zGJ>KW{W}2=4<!x4~eW!0dxjcOq
z@4DE(x2>yJuUntJ`exQeo6U;8@ki_UKen|m%l>u7>gu#jXN7ibi-_Z7v&wyCbo%=J
z<yCP4*K7(OW%s)M$rkwA?Xz#1VV3UX(sOJssT)7_?>-+t<MV-a&0ZgPKYSFOxAl>~
zpzqo{5t`8^X0N&2{67kCp8Wnzwm{qc&^=y%-ps1C{kNaaE#+RR$J4P)t-q?VRbleP
zK7P@Y;o>d5mPIR0EuFNhuRpw6ZR+0J+BYl9HQmBL*_?L$(Y*HT%4ogikFH$)%3QeZ
zxzCl7gCdGXb)gd-6AJsH<>M^BM}78Q-+F7>%3QXYt1Gruzm=2Rb5LemS02YnYo8;N
z?|l9hzLY`KL59IVd$!r?)>RL6Y<JyNO?~<;`NzGxH$$#Wdvr~{)BbAoe}+BvOY_1I
z?4qap-y2$exPP?ssa5}<mOT6Wrl)=&R#cZpVG*p`@BE|ii_NV2N9vY?mT6|C4fZjp
zMP1sD`PYI+9<Z%U28khByvzIrBG+UsRXf2@d-0#?lV@A9LIa<KZ#}1zYU!&NTAb8%
z5;RD9{`cg1=nW)YP<Hn5Uwz6yC69is+J7oLFw*|R#K`{)Mfty<)c49GvjdORD~oSf
zdE?rmP4y3zHvfBaAAIA7Jrdh)amPP5`S8J~=ECm(4E>h>8D7t>*ZSlyRDb-4{k!?k
z<NEFPW~%O)xXO2$6I)2ReAMTDm{ff5p)XlMYp%Lxw`LW!NyVJHwV0e%#6FuMy3C0L
zACAqde_(%Y`)g)6>uIt6_nGp()BiK<2A!4M@D^u6LLLY>vHF+qn=o@<orPUfv_JKH
zp7>AnY){K_r@*e=8>gl`^<7f>CCvYiN?84!+0grMV9a>quQKkRj8FdB8~@aF#hUp)
zJl6bY*lGUvll`%NB=-C(h4Lp3-w1g_D6R1)25(&Jn)09FkZbzillvhTsKA-E?t*`M
zY+L<~Z55sTpMg_!{&zL_H8}_l$VCR<M6IQ+;Bd4%Nv?^gQ~A#jn*Z<Pzp3VkFo>+$
zw^Z~$!xSYxrgZBz?e&bUYk6E!HEv1=EDj9uU3~Vd{!flO_4~4?BJBrxwN)Nd@NZoE
zhsQ4}w(jrv>kpc>zj8*)@;}2B&wnrfg{s1aFB&vauTk1DzrFqwU;g*`{~4S_Z(|<A
zgnW(CyVXcn5;{mtfvzL2xr#g-o;PFNtsW<{(3%esDdDQfGHt#885ASzw)Uys>X|=r
zEo;@rxOD#~S@NGh=Kp7?>;G{PiGB5z_M?9r^-jODTYqES`}Z0@rP}}eiT=-UfAK$)
zN(9@qs>=?fia%-h{ymG4^rM=k@$1NL{T;Sl@BV%2pSYs_{L%e?f7w5FM=--zEPgHY
zX5WKv-n)OFvU6MbpTS=2Kg0JI{~3DaP<U~R`qy}G`rUj}51*r^hR%5yvhfSEiH^&?
zPW>IWQ^Lc)PqFEB{#U2_pW*$*{|r(noIMp5%IrH#0>5^a#TQho-u?SjUOC`D!~E|0
z-(TvF^&_#@2e|QZfSvjfzq(b{O`#Hv*O?~G-#$O}JNu^o^6yh>a5`YGtNt4QO~2dU
zOedgBK4Ln^wAlOi0zaAZ|NOE2Kf`|aKN(2utgj-qNwL%K$ek>&uXI0g#s2xD`~UvR
ze_V`UuD-HcI_>ZB{B;%!!)M>r-w}7t_Wr#9Br~v?XP-h$3{N65hW(IZIN|k4D}Ro8
zF2!3OCr$jDy=Gndk^3J%v;S`Yd41v3Yeh$vT>9N7b7)pC+q?5RI9Ao;T5Z4L=cOS~
zfP-2Bc=6(`OzGk)>rc2n{`)y*edB)k@g1Sq1aAd`#!e@${^S0tR^3%3Y*`|GPRn9`
zwjSjKk2fayKe-P6`&#%<??1z&<E8%@Vy}btRNct9|4Fp@@2kV2e<Xhy=h%NtepR+5
z(6vVM>qA!&@Hxl~t7pZ|+<PUdU0$!~zSr_t{`PmeCci7+J)3HCCo%c%XTjJCt-`sf
z=e|VExv{HHWWL^er*FH{a|@U4nYN<w?QPyI$*&{Ls<>h@Pgm;RI@I!cN6A`m-l}8e
z(<F5tZ}Vi-Il3eHedxPoTe}ZmigGU&-ch`TO;XwP_0=}1)umgq?#-GPYI%Lh-f1iE
ztSo<cQ)$|r=rgLGeWJVSOxG{X`0(=Abn#A=Q}azKo`htty5*R;T;5XRc*6sksgFfX
zoDS(Ger3C&c%=K#JJFDg#D%67QU7*l_(olQ$}YJ*w<<4vUF+Y4rgsnS5Y#$zpvhuI
z(uWu2Q_tSgT~kxJd~fvc((T{E7R%3<d#ReVxLij|*67)_&n~rQnb$atC!9TC+P~?r
z<%*lJtKVPWxM^iaK;AF4X?LwBoL*!*=h}4xzsU~o7;;LUt<9Phdu;LAHP(+M`c<bE
zHu}$ccj?%vTL<-o!i6-`WZ5lEQ@3WDUiYp%l{`sz_Q4bEri&K0c^>)|n^|n19lO;!
zcFEfXf3^MFPH`XYdYb;2xAD}hrzS^vS4C?xI8G?2+j=?jZ?kXoo>zjFvsHMyo|Rnp
z*s7gg#<f~OW!|0ZcKXwd7Ok1O=lzxYyk{PV?#<ipyyE(oiskL+w)KWZ>`V>p=~}%u
z;^(znC1D@6PO7mp%kI8(_jTdz*RM^2I(2U{?Ylhj-kPkaqqE{7&dRJ*QkFf)SHQ9E
z*8H#)%ic{}X}{-b`}TIV>37$EGQNG7%dt~`;<~1HhL2XBUSwnCxVO?HHEEem$#dE4
zrPBj@cFwc)x|yiF>_XHE*W%ojFFxJcd#BJVW>R!R<+;L$D~A$euas}zw*P~7o4W9x
zZM*i(|G@M3;inBZckcMl(BH&<_vOS{(;gr8<`i7}CRbB|#isD}wJpo{Zt698G3lMT
zzz!Z~J_d=iX|XYp-j7;mg}ZNmtb3(o#me?W2cu4hO?!H1pHlAYP5Ft>4;yYMY5OZz
z60<33>5cG%>o|TkKlsmZKXl#nZx?+PtAFRKPXC^<L*h@iSoZ1ShbtclyQLZliI`j5
zjrQy1GMwBt%h=-T+R&x8rS6ulBH?G}{eUen3^#sM|Kn%#-@|`iFI$&><Ua$n&|E&{
zKf_-AnFnXq@7{bG*A8*qd&k$Ww@VomV`mzRx*k|NfoGsJ77%}C8dPoHmB3q##@$!;
zKT$6H_j6r1(wfUv*aY8d{Ab{^Yd>-IpYm7GVHVoK1_QO|Qp)^KYr=nqsvZ9suokF^
zUH^FgN^X<?vGFVD%&2`LzdkJL(u5zc^-y}sJ;80i9(ReE7nCWR*|KGCzOZNh5BK)D
zTNINxO)_U?XI{F$P_lQr#=Wa=+_j5(Qs(t$KdCG_QyDWkYWeb+M+0L;=e06NN$);3
zqupN1Wy!4V`HOBvyqd7tdXwkBwUO1oq>f3Zrl)=Jxfa>0TCz3c$%=KGp8H&0Uz)j)
zSLu{^VW@<tPes9hh7^<k4EDDe?e({1dh4z&Kb_Vtb7<;x)615>v=>i4JW=|F?&66D
zHTMNobJ=e+dA(((PF{+!^YwCP@84$oC)8&IezrU{^R1q8V|v95m%9>uy!oz+GB3WI
z7#ruWQoXC8`dZJff-5E7(G~H6?<AA|-nup~Jjiopey-qLpJSZsLYIC`7j?~F*PmRt
zs<!V|xsriTk6)#1tN1Us`Mg{DMK-%?rLfLOu)nmSvNkSU*W|P5udmuGUaxHQ6Fd=K
z;dcA)n?jGS*hA}8<|OUjp}2kf)^5)l@kvVl*Y|&3-h6P6>)kigpWmq16Te}T*V{Qt
zi+3K<OnzhRmys5Fvh%=Kt?Y##-tLaho-SbURHXk(*xt=qx%Ec(+~sflJg89eHu~b}
zkOxcEE|<UD+N1UOj@p%=tm|Jb68b_mxM^8U(A;_>T1VOYYKuZ;;{80ksmHJDY4g4j
zDcyUn$4vEF(WPr#8jE(E+W<Om1%#2azHiL=&i@RLpYea^|GeH`^FKpXym7P9e}?k(
zY1q#W!gZ!ld|b^*`fNs6`EcEK`w#N3qrW!7S)X=J|9z%^+3EibcP|9GiXJE<`|w}c
z6+^x4q>rWT^cuXE-x+JuX;GJ<gY<+~(@s9zzDdvJxrK!Nxpgn^DK}&;lirr7wT_Wt
zPS?{F>wct(F1Wf&TPFCx<yqyg-j#}3J>C5B(XGrU+e=<5@jMAJ&iZzGWz;Tzn~N?*
zXa4P5wDXqMywu2}>KCR?zvR1l(T*!QiPx3(Z=XBIT!^PcVs-kFZT}f=9ca(r^yc7C
z|M;avA;EpAQTDvkwsWR_Su*wAmyH*e?U%~lbZ5fZyw{r*irCls&1##xtb69$lu40k
z@+Ey!PhSj{IkxJq+Slz<zsqb8xp#Ky&)Ma-mhL^)%DZW&v36?P<@e!AA9>7n#cm1M
zmg>a3GWha-(=&mdX`)l^Tzjkfo%cbe#R}W?#fz)&O1#Q0lXBcs^V?iA?CYatd#~y2
z-@LT>t$x`#j)2p>TTcgu{#_hA^{A`%<iGR%UT*uMpD7e|aD~ctA18HXE<fLF-$%>E
zm!EDk=Ez}Fc&D^;m$p{Ktt|@{@|x~k>!WJ5`T4x%VY`man6-0nqy&3+HgEg7Y^Kiv
zJLXlE-8q$*roww%;#a7Z=Cxy2SIdPjU$8Ra<)!lbSMFRc)y~cpdUY#IDbjdLuFc1Y
zlE8?o>$N8Kq<-_w4D7Y^@?H_>D%!XKN97Ld5RKODp!PuDl|YAi$A?Cb=syF}p@uIP
z&hoAd^O&8z-Z`?)`a^$L{3F(PE0?_#7Z=>U`lw3gPJf25@^9rw>VGlD)<3XSn;e$x
zyS${#{`Rc)Ki~f|9Cy|K_H5yQ2HXD(A1}t_BNWQtyf65l;RS2ye};yy?N>GGm45fu
zZI1pq^JaZsmi*`Y{~2UOz^sq|8Tz^&*KgjZ{GZ_h*X#cbj4%G3`_ccO!9jcdANIws
zrr1xcJ@|Fj{WIrI|1J*r&oJ*l!;cq>5zKYwzpWqH|7BRW|ASoroA(|68NM*>`p<Bn
z%5J6ePw_kQk!$~)d9#1JDgV#+Ae-#rEE4RT-~Xp1@bAtQ_2=vVl!QzFo%>P#Uqg8O
z4}Nzf4~H!L)BTQr&9#4NW%2OPkq5E<)IVOd<36(6?2*0uYbpPe_=4KB?VpV|{kwC;
z{`q5g;DCZj^w@uf^cwg73=2fB|6{(0WH~rgn_rvmpE-9rA{N}iOndnsFNhDCAE1~7
zyA>R#h5M0W?@K+B?O(m*Pwac}>umnB^i)`YKtksDKP&BC<d{Q>mQ?}&T)(ql%dR)J
z)_;4}`Cs*ahR2Kg;Ve?ZX8%7c4R|1t5Hz6hpfJ^d1Bry7ISzB`eiDMF&i>;?NDSi5
zQeSF=+CS;<umk0&8~+(VSqhwy))rUVKGyd8eX57K>*;?6<Ya~vH>=v@rpP`PnQ62(
z$m`PNj8izqurhXk=<h47ULp0~arY_N1wXAFSiY}cwq{X=PVU3Pc8{siuh_JcjjHet
z6fJ5%pC!=DM%@4{{m)LnuKREH%eP+QJ1)I@(3AS4^`iSqg~oW}8$Y~{N4Z?Co$k2j
z^c+KjcQOI&xw)0L%ib$~d-wWS*70v=|1&^Jp~C+RIFjtOpoM;ZpSpIg$qwn-G<oBV
z`ked!89@0s@bBh&Six0~l#h|j`D!1w_|J;^h}D12ys_Vw|DWMGIQiY&4=sk)*&`)#
z<os>7*8OKdZPNPB#vA^@RWJUX_MZWg_ILbeKvs@g9Bh?;>iD&5|Cw`#5gPj6&i>B;
zDZw7&jMl}kuGCLuzozw{!PuG~p#ko4?S7;*uo%hds{{Xq$dCFGM?Eqcmkf^=(LBXk
zS*m2FWXe39@nqKX8l+=AC;zs7WDcp`x7Nx1@IET=<KOMzD_^?1E2eBdsnWT%QgoMR
zMo;v<HQ#@MN+K#BE54_g_u^HP)2p*R^<3DCb!w|Gu+0^1{JVce{e`{NKfCO=f93xr
zzP{ejLHbJ=`?r4!|1&s!eg913-~B80FXHz8Im(cKF+%?KulB#(*Z0p@{JTEjKf}VR
z@1I2&_q!(4=f9Hwq96A!ssHU?=YOrQ@1F@^`#Wm*Xt0c?meEW^zp~}4%zuVgD&Uo`
zT1Vgomzi&QK*i1ahw^)$2wn+b>p#5zC%gIY^ZywZa=l&M_d#^)(Y2l4GiGV>J-cz&
PEh0SG0d`57{QsK(<sPdF

diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files
index 4cf134cc3c..585470d5f0 100644
--- a/tensorflow/docs_src/mobile/leftnav_files
+++ b/tensorflow/docs_src/mobile/leftnav_files
@@ -1,6 +1,7 @@
 index.md
 ### TensorFlow Lite
 tflite/index.md
+tflite/devguide.md
 tflite/demo_android.md
 tflite/demo_ios.md
 >>>
diff --git a/tensorflow/docs_src/mobile/tflite/demo_android.md b/tensorflow/docs_src/mobile/tflite/demo_android.md
index c94b5597a6..7f2f8882a2 100644
--- a/tensorflow/docs_src/mobile/tflite/demo_android.md
+++ b/tensorflow/docs_src/mobile/tflite/demo_android.md
@@ -1,42 +1,144 @@
-# TensorFlow Lite Demo for Android
+# Android Demo App
 
-The TensorFlow Lite demo is a camera app that continuously classifies whatever
-it sees from your device's back camera, using a quantized MobileNet model.
+An example Android application using TensorFLow Lite is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
+The demo is a sample camera app that classifies images continuously
+using either a quantized Mobilenet model or a floating point Inception-v3 model.
+To run the demo, a device running Android 5.0 ( API 21) or higher is required.
 
-You'll need an Android device running Android 5.0 or higher to run the demo.
+In the demo app, inference is done using the TensorFlow Lite Java API. The demo
+app classifies frames in real-time, displaying the top most probable
+classifications. It also displays the time taken to detect the object.
 
-To get you started working with TensorFlow Lite on Android, we'll walk you
-through building and deploying our TensorFlow demo app in Android Studio.
+There are three ways to get the demo app to your device:
 
-Note: For a more detailed guide see the
-[TFLite Codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2-tflite/index.html#0)
+* Download the [prebuilt binary APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+* Use Android Studio to build the application.
+* Download the source code for TensorFlow Lite and the demo and build it using
+  bazel.
 
-It's also possible to build the demo app with Bazel, but we only recommend
-this for advanced users who are very familiar with the Bazel build
-environment. For more information on that, see our page [on Github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite#building-tensorflow-lite-and-the-demo-app-from-source).
 
-## Build and deploy with Android Studio
+## Download the pre-built binary
 
-1. Clone the TensorFlow repository from GitHub if you haven't already:
+The easiest way to try the demo is to download the
+[pre-built binary APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
 
-        git clone https://github.com/tensorflow/tensorflow
+Once the APK is installed, click the app icon to start the program. The first
+time the app is opened, it asks for runtime permissions to access the device
+camera. The demo app opens the back-camera of the device and recognizes objects
+in the camera's field of view. At the bottom of the image (or at the left
+of the image if the device is in landscape mode), it displays top three objects
+classified and the classification latency.
 
-2. Install the latest version of Android Studio from [here](https://developer.android.com/studio/index.html).
 
-3. From the **Welcome to Android Studio** screen, use the **Import Project
-   (Gradle, Eclipse ADT, etc)** option to import the
-   `tensorflow/contrib/lite/java/demo` directory as an existing Android Studio
-   Project.
+## Build in Android Studio with TensorFlow Lite AAR from JCenter
 
-    Android Studio may prompt you to install Gradle upgrades and other tool
-    versions; you should accept these upgrades.
+Use Android Studio to try out changes in the project code and compile the demo
+app:
 
-4. Download the TensorFlow Lite MobileNet model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip).
+* Install the latest version of
+  [Android Studio](https://developer.android.com/studio/index.html).
+* Make sure the Android SDK version is greater than 26 and NDK version is greater
+  than 14 (in the Android Studio settings).
+* Import the `tensorflow/contrib/lite/java/demo` directory as a new
+  Android Studio project.
+* Install all the Gradle extensions it requests.
 
-    Unzip this and copy the `mobilenet_quant_v1_224.tflite` file to the assets
-    directory: `tensorflow/contrib/lite/java/demo/app/src/main/assets/`
+To get a model, either:
 
-5. Build and run the app in Android Studio.
+* Download the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip)
+  and unzip and copy `mobilenet_quant_v1_224.tflite` to the assets directory:
+  `tensorflow/contrib/lite/java/demo/app/src/main/assets/`.
+* Or, download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
+  and unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets
+  directory. Change the chosen classifier in
+  [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)<br>
+  from: `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`<br>
+  to: `classifier = new ImageClassifierFloatInception(getActivity());`.
 
-You'll have to grant permissions for the app to use the device's camera. Point
-the camera at various objects and enjoy seeing how the model classifies things!
+Now you can build and run the demo app.
+
+
+## Build TensorFlow Lite and the demo app from source
+
+### Clone the TensorFlow repo
+
+```sh
+git clone https://github.com/tensorflow/tensorflow
+```
+
+### Install Bazel
+
+If `bazel` is not installed on your system, see
+[Installing Bazel](https://bazel.build/versions/master/docs/install.html).
+
+Note: Bazel does not currently support Android builds on Windows. Windows users
+should download the
+[prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+
+### Install Android NDK and SDK
+
+The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The
+current recommended version is *14b* and can be found on the
+[NDK Archives](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads)
+page.
+
+The Android SDK and build tools can be
+[downloaded separately](https://developer.android.com/tools/revisions/build-tools.html)
+or used as part of
+[Android Studio](https://developer.android.com/studio/index.html). To build the
+TensorFlow Lite Android demo, build tools require API >= 23 (but it will run on
+devices with API >= 21).
+
+In the root of the TensorFlow repository, update the `WORKSPACE` file with the
+`api_level` and location of the SDK and NDK. If you installed it with
+Android Studio, the SDK path can be found in the SDK manager. The default NDK
+path is:`{SDK path}/ndk-bundle.` For example:
+
+```
+android_sdk_repository (
+    name = "androidsdk",
+    api_level = 23,
+    build_tools_version = "23.0.2",
+    path = "/home/xxxx/android-sdk-linux/",
+)
+
+android_ndk_repository(
+    name = "androidndk",
+    path = "/home/xxxx/android-ndk-r10e/",
+    api_level = 19,
+)
+```
+
+Some additional details are available on the
+[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
+
+### Build the source code
+
+To build the demo app, run `bazel`:
+
+```
+bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
+```
+
+Caution: Because of an bazel bug, we only support building the Android demo app
+within a Python 2 environment.
+
+
+## About the demo
+
+The demo app is resizing each camera image frame (224 width * 224 height) to
+match the quantized MobileNets model (299 * 299 for Inception-v3). The resized
+image is converted—row by row—into a
+[ByteBuffer](https://developer.android.com/reference/java/nio/ByteBuffer.html).
+Its size is  1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch.
+224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents
+the 3 colors of a pixel.
+
+This demo uses the TensorFlow Lite Java inference API
+for models which take a single input and provide a single output. This outputs a
+two-dimensional array, with the first dimension being the category index and the
+second dimension being the confidence of classification. Both models have 1001
+unique categories and the app sorts the probabilities of all the categories and
+displays the top three. The model file must be downloaded and bundled within the
+assets directory of the app.
diff --git a/tensorflow/docs_src/mobile/tflite/demo_ios.md b/tensorflow/docs_src/mobile/tflite/demo_ios.md
index 3ee9b1cbca..3be21da89f 100644
--- a/tensorflow/docs_src/mobile/tflite/demo_ios.md
+++ b/tensorflow/docs_src/mobile/tflite/demo_ios.md
@@ -1,4 +1,4 @@
-# TensorFlow Lite Demo for iOS
+# iOS Demo App
 
 The TensorFlow Lite demo is a camera app that continuously classifies whatever
 it sees from your device's back camera, using a quantized MobileNet model. These
diff --git a/tensorflow/docs_src/mobile/tflite/devguide.md b/tensorflow/docs_src/mobile/tflite/devguide.md
new file mode 100644
index 0000000000..5b521dca7b
--- /dev/null
+++ b/tensorflow/docs_src/mobile/tflite/devguide.md
@@ -0,0 +1,224 @@
+# Developer Guide
+
+Using a TensorFlow Lite model in your mobile app requires multiple
+considerations: you must choose a pre-trained or custom model, convert the model
+to a TensorFLow Lite format, and finally, integrate the model in your app.
+
+## 1. Choose a model
+
+Depending on the use case, you can choose one of the popular open-sourced models,
+such as *InceptionV3* or *MobileNets*, and re-train these models with a custom
+data set or even build your own custom model.
+
+### Use a pre-trained model
+
+[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
+is a family of mobile-first computer vision models for TensorFlow designed to
+effectively maximize accuracy, while taking into consideration the restricted
+resources for on-device or embedded applications. MobileNets are small,
+low-latency, low-power models parameterized to meet the resource constraints for
+a variety of uses. They can be used for classification, detection, embeddings, and
+segmentation—similar to other popular large scale models, such as
+[Inception](https://arxiv.org/pdf/1602.07261.pdf). Google provides 16 pre-trained
+[ImageNet](http://www.image-net.org/challenges/LSVRC/) classification checkpoints
+for MobileNets that can be used in mobile projects of all sizes.
+
+[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model
+that achieves fairly high accuracy recognizing general objects with 1000 classes,
+for example, "Zebra", "Dalmatian", and "Dishwasher". The model extracts general
+features from input images using a convolutional neural network and classifies
+them based on those features with fully-connected and softmax layers.
+
+[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
+is an on-device model that provides one-touch replies for incoming text messages
+by suggesting contextually relevant messages. The model is built specifically for
+memory constrained devices, such as watches and phones, and has been successfully
+used in Smart Replies on Android Wear. Currently, this model is Android-specific.
+
+These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md)
+
+### Re-train Inception-V3 or MobileNet for a custom data set
+
+These pre-trained models were trained on the *ImageNet* data set which contains
+1000 predefined classes. If these classes are not sufficient for your use case,
+the model will need to be re-trained. This technique is called
+*transfer learning* and starts with a model that has been already trained on a
+problem, then retrains the model on a similar problem. Deep learning from
+scratch can take days, but transfer learning is fairly quick. In order to do
+this, you need to generate a custom data set labeled with the relevant classes.
+
+The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
+codelab walks through the re-training process step-by-step. The code supports
+both floating point and quantized inference.
+
+### Train a custom model
+
+A developer may choose to train a custom model using Tensorflow (see the
+@{$tutorials} for examples of building and training models). If you have already
+written a model, the first step is to export this to a @{tf.GraphDef} file. This
+is required because some formats do not store the model structure outside the
+code, and we must communicate with other parts of the framework. See
+[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md)
+to create .pb file for the custom model.
+
+TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the
+[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for supported operators and their usage. This set of operators will continue to
+grow in future Tensorflow Lite releases.
+
+
+## 2. Convert the model format
+
+The model generated (or downloaded) in the previous step is a *standard*
+Tensorflow model and you should now have a .pb or .pbtxt @{tf.GraphDef} file.
+Models generated with transfer learning (re-training) or custom models must be
+converted—but, we must first freeze the graph to convert the model to the
+Tensorflow Lite format. This process uses several model formats:
+
+* @{tf.GraphDef} (.pb) —A protobuf that represents the TensorFlow training or
+  computation graph. It contains operators, tensors, and variables definitions.
+* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this
+  does not contain a graph structure, it cannot be interpreted by itself.
+* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain
+  variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a
+  CheckPoint and a `GraphDef`, and converting each variable into a constant
+  using the value retrieved from the CheckPoint.
+* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels
+  input and output arguments to a model. A `GraphDef` and CheckPoint can be
+  extracted from a `SavedModel`.
+* *TensorFlow Lite model* (.tflite) —A serialized
+  [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
+  Lite operators and tensors for the TensorFlow Lite interpreter, similiar to a
+  `FrozenGraphDef`.
+
+### Freeze Graph
+
+To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints
+that contain trained weight parameters. The .pb file only contains the structure
+of the graph. The process of merging the checkpoint values with the graph
+structure is called *freezing the graph*.
+
+You should have a checkpoints folder or download them for a pre-trained model
+(for example,
+[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
+
+To freeze the graph, use the following command (changing the arguments):
+
+```
+freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \
+  --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
+  --input_binary=true \
+  --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
+  --output_node_names=MobileNetV1/Predictions/Reshape_1
+```
+
+The `input_binary` flag must be enabled so the protobuf is read and written in
+a binary format. Set the `input_graph` and `input_checkpoint` files.
+
+The `output_node_names` may not be obvious outside of the code that built the
+model. The easiest way to find them is to visualize the graph, either with
+[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3)
+or `graphviz`.
+
+The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format
+(.tflite) for use on Android or iOS devices. For Android, the Tensorflow
+Optimizing Converter tool supports both float and quantized models. To convert
+the frozen `GraphDef` to the .tflite format:
+
+```
+toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
+  --input_format=TENSORFLOW_GRAPHDEF \
+  --output_format=TFLITE \
+  --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
+  --inference_type=FLOAT \
+  --input_type=FLOAT \
+  --input_arrays=input \
+  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+  --input_shapes=1,224,224,3
+```
+
+The `input_file` argument should reference the frozen `GraphDef` file
+containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
+file used here is available for download. `output_file` is where the TensorFlow
+Lite model will get generated. The `input_type` and `inference_type`
+arguments should be set to `FLOAT`, unless converting a
+@{$performance/quantization$quantized model}. Setting the `input_array`,
+`output_array`, and `input_shape` arguments are not as straightforward. The
+easiest way to find these values is to explore the graph using Tensorboard. Reuse
+the arguments for specifying the output nodes for inference in the
+`freeze_graph` step.
+
+It is also possible to use the Tensorflow Optimizing Converter with protobufs
+from either Python or from the command line (see the 
+[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py)
+example). This allows you to integrate the conversion step into the model design
+workflow, ensuring the model is easily convertible to a mobile inference graph.
+For example:
+
+```python
+import tensorflow as tf
+
+img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
+val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
+out = tf.identity(val, name="out")
+
+with tf.Session() as sess:
+  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
+  open("converteds_model.tflite", "wb").write(tflite_model)
+```
+
+For usage, see the Tensorflow Optimizing Converter
+[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
+
+Refer to the
+[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for troubleshooting help, and if that doesn't help, please
+[file an issue](https://github.com/tensorflow/tensorflow/issues).
+
+The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
+to visualize TensorFlow Lite models after conversion. To build the
+[visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/tools/visualize.py)
+tool:
+
+```sh
+bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
+```
+
+This generates an interactive HTML page listing subgraphs, operations, and a
+graph visualization.
+
+
+## 3. Use the TensorFlow Lite model for inference in a mobile app
+
+After completing the prior steps, you should now have a .tflite model file.
+
+### Android
+
+Since Android apps are written in Java and the core TensorFlow library is in C++,
+a JNI library is provided as an interface. This is only meant for inference—it
+provides the ability to load a graph, set up inputs, and run the model to
+calculate outputs.
+
+The open source Android demo app uses the JNI interface and is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
+You can also download a
+[prebuilt APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+See the @{$tflite/demo_android} guide for details.
+
+The @{$mobile/android_build} guide has instructions for installing TensorFlow on
+Android and setting up `bazel` and Android Studio.
+
+### iOS
+
+To integrate a TensorFlow model in an iOS app, see the
+[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md)
+guide and @{$tflite/demo_ios} guide.
+
+#### Core ML support
+
+Core ML is a machine learning framework used in Apple products. In addition to
+using Tensorflow Lite models directly in your applications, you can convert
+trained Tensorflow models to the
+[CoreML](https://developer.apple.com/machine-learning/) format for use on Apple
+devices. To use the converter, refer to the
+[Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
diff --git a/tensorflow/docs_src/mobile/tflite/index.md b/tensorflow/docs_src/mobile/tflite/index.md
index beb24794fc..11f11ea4dc 100644
--- a/tensorflow/docs_src/mobile/tflite/index.md
+++ b/tensorflow/docs_src/mobile/tflite/index.md
@@ -155,7 +155,9 @@ retraining for both floating point and quantized inference.
 
 The following diagram shows the architectural design of TensorFlow Lite:
 
-![tensorflow lite architecture](https://www.tensorflow.org/images/tflite-architecture.jpg)
+<img src="/images/tflite-architecture.jpg"
+     alt="TensorFlow Lite architecture diagram"
+     style="max-width:600px;">
 
 Starting with a trained TensorFlow model on disk, you'll convert that model to
 the TensorFlow Lite file format (`.tflite`) using the TensorFlow Lite
-- 
GitLab


From 9d1d379bcdd19d496fd8d2659c21a5510e045c5a Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 13:31:23 -0700
Subject: [PATCH 700/960] Docs: Add Eager Execution guide to Programmer's
 Guide.

PiperOrigin-RevId: 190977505
---
 tensorflow/contrib/eager/README.md            |  20 +-
 .../contrib/eager/python/g3doc/guide.md       | 898 +---------------
 .../docs_src/programmers_guide/eager.md       | 992 ++++++++++++++++++
 .../docs_src/programmers_guide/leftnav_files  |   3 +-
 4 files changed, 1015 insertions(+), 898 deletions(-)
 create mode 100644 tensorflow/docs_src/programmers_guide/eager.md

diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md
index 9d2ca07c3a..9a3b780af8 100644
--- a/tensorflow/contrib/eager/README.md
+++ b/tensorflow/contrib/eager/README.md
@@ -1,12 +1,8 @@
 # Eager Execution
 
-> *WARNING*: This is a preview/pre-alpha version. The API and performance
-> characteristics are subject to change.
-
-Eager execution is an experimental interface to TensorFlow that provides an
-imperative programming style (à la [NumPy](http://www.numpy.org)). When you
-enable eager execution, TensorFlow operations execute immediately; you do not
-execute a pre-constructed graph with
+Eager execution provides an imperative interface to TensorFlow (similiar to
+[NumPy](http://www.numpy.org)). When you enable eager execution, TensorFlow
+operations execute immediately; you do not execute a pre-constructed graph with
 [`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session).
 
 For example, consider a simple computation in TensorFlow:
@@ -33,7 +29,7 @@ print(m)
 ## Caveats
 
 This feature is in early stages and work remains to be done in terms of smooth
-support for distributed and multi-GPU training and CPU performance.
+support for distributed and multi-GPU training and performance.
 
 - [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Acomp%3Aeager)
 - Feedback is welcome, please consider
@@ -41,21 +37,23 @@ support for distributed and multi-GPU training and CPU performance.
 
 ## Installation
 
-Eager execution is included in TensorFlow versions 1.5 and above.
+Eager execution is included in TensorFlow versions 1.7 and above.
 Installation instructions at https://www.tensorflow.org/install/
 
 ## Documentation
 
 For an introduction to eager execution in TensorFlow, see:
 
-- [User Guide](python/g3doc/guide.md)
+- [User Guide](https://www.tensorflow.org/programmers_guide/eager) ([source](../../docs_src/programmers_guide/eager.md))
 - Notebook: [Basic Usage](python/examples/notebooks/1_basics.ipynb)
 - Notebook: [Gradients](python/examples/notebooks/2_gradients.ipynb)
 - Notebook: [Importing Data](python/examples/notebooks/3_datasets.ipynb)
 
 ## Changelog
 
-- 2017/10/31: Initial preview release.
+- 2017/10/31: Initial preview release (in TensorFlow 1.5)
 - 2017/12/01: Example of dynamic neural network:
   [SPINN: Stack-augmented Parser-Interpreter Neural Network](https://arxiv.org/abs/1603.06021).
   See [README.md](python/examples/spinn/README.md) for details.
+- 2017/03: Core functionality moved out of the experimental tf.contrib namespace
+  in TensorFlow 1.7.
diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md
index 11064981c6..2d2aba6908 100644
--- a/tensorflow/contrib/eager/python/g3doc/guide.md
+++ b/tensorflow/contrib/eager/python/g3doc/guide.md
@@ -1,892 +1,18 @@
-# TensorFlow Eager Execution
-
-## What is this?
+# Eager execution
 
 Eager execution is a feature that makes TensorFlow execute operations
-immediately: concrete values are returned, instead of a computational graph to
-be executed later.
-
-As a result, enabling eager execution provides:
-
--   A [NumPy](http://www.numpy.org/)-like library for numerical computation with
-    support for GPU acceleration and automatic differentiation.
--   A flexible platform for machine learning research and experimentation.
-
-Eager execution is under active development. This guide walks through an
-alpha/preview release. In particular, not all TensorFlow APIs currently work
-with eager execution enabled, and some models may be slow to execute, compared
-to models defined without using eager execution.
-
-## Installation
-
-Eager execution is included in TensorFlow versions 1.5 and above.
-Installation instructions at https://www.tensorflow.org/install/
-
-The contents of this guide are compatible with TensorFlow 1.5. However, if you
-run into bugs that are fixed in source but not the release, you may want to
-either [build from source](https://www.tensorflow.org/install/install_sources)
-or try a nightly build. The nightly builds are available as:
-
-- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and
-
-- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images.
-
-For example, to run the latest nightly docker image:
-
-```sh
-# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker
-docker pull tensorflow/tensorflow:nightly-gpu
-docker run --runtime=nvidia -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu
-
-# If you do not have a GPU, use the CPU-only image
-docker pull tensorflow/tensorflow:nightly
-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly
-```
-
-And then visit http://localhost:8888 in your browser for a Jupyter notebook
-environment.
-
-## Getting Started
-
-With TensorFlow installed, eager execution is enabled via a single call:
-
-```python
-import tensorflow as tf
-
-import tensorflow.contrib.eager as tfe
-
-tfe.enable_eager_execution()
-```
-
-Enabling eager execution changes how TensorFlow functions behave (in particular,
-`Tensor` objects will reference concrete values instead of being symbolic
-handles to nodes in a computational graph). As a result, eager execution should
-be enabled at the beginning of a program and cannot be disabled afterwards in
-the same program.
-
-Code examples in the rest of this guide assume that eager execution has been
-enabled.
-
-## A library for numerical computation
-
-A significant fraction of the [TensorFlow
-API](https://www.tensorflow.org/api_docs/python/) consists of numerical
-operations:
-[arithmetic operations](https://www.tensorflow.org/api_guides/python/math_ops#Arithmetic_Operators),
-[matrix operations](https://www.tensorflow.org/api_guides/python/math_ops#Matrix_Math_Functions),
-[linear algebra operations](https://www.tensorflow.org/versions/master/api_docs/python/tf/linalg),
-etc.
-
-With eager execution enabled, these operations consume and return
-multi-dimensional arrays as `Tensor` objects, similar to NumPy
-[`ndarray`s](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.html).
-For example:
-
-```python
-# Multiply two 2x2 matrices
-x = tf.matmul([[1, 2],
-               [3, 4]],
-              [[4, 5],
-               [6, 7]])
-# Add one to each element
-# (tf.add supports broadcasting)
-y = tf.add(x, 1)
-
-# Create a random random 5x3 matrix
-z = tf.random_uniform([5, 3])
-
-print(x)
-print(y)
-print(z)
-```
-
-Output:
-
-```
-tf.Tensor(
-[[16 19]
- [36 43]], shape=(2, 2), dtype=int32)
-tf.Tensor(
-[[17 20]
- [37 44]], shape=(2, 2), dtype=int32)
-tf.Tensor(
-[[ 0.25058532  0.0929395   0.54113817]
- [ 0.3108716   0.93350542  0.84909797]
- [ 0.53081679  0.12788558  0.01767385]
- [ 0.29725885  0.33540785  0.83588314]
- [ 0.38877153  0.39720535  0.78914213]], shape=(5, 3), dtype=float32)
-```
-
-For convenience, these operations can also be triggered via operator overloading
-of the `Tensor` object. For example, the `+` operator is equivalent to `tf.add`,
-`-` to `tf.subtract`, `*` to `tf.multiply`, etc.:
-
-```python
-x = (tf.ones([1], dtype=tf.float32) + 1) * 2 - 1
-print(x)
-```
-
-Output:
-
-```
-tf.Tensor([ 3.], shape=(1,), dtype=float32)
-```
-
-### Converting to and from NumPy
-
-The operations above automatically convert Python objects (like lists of
-numbers) and NumPy arrays to `Tensor` objects. `Tensor` objects can also be used
-as NumPy arrays by numpy operations.
-
-```python
-import numpy as np
-
-x = tf.add(1, 1)                     # tf.Tensor with a value of 2
-y = tf.add(np.array(1), np.array(1)) # tf.Tensor with a value of 2
-z = np.multiply(x, y)                # numpy.int64 with a value of 4
-```
-
-Alternatively, they can be explicitly converted using
-[`tf.constant`](https://www.tensorflow.org/api_docs/python/tf/constant), as
-shown in the next example.
-
-Conversely, you can call the `numpy()` method of a `Tensor` object' to obtain
-its NumPy `ndarray` value. For example:
-
-```python
-import numpy as np
-
-np_x = np.array(2., dtype=np.float32)
-x = tf.constant(np_x)
-
-py_y = 3.
-y = tf.constant(py_y)
-
-z = x + y + 1
-
-print(z)
-print(z.numpy())
-```
-
-Output:
-
-```
-tf.Tensor(6.0, shape=(), dtype=float32)
-6.0
-```
-
-### GPU acceleration
-
-Many TensorFlow operations support GPU acceleration. With eager execution
-enabled, [computation is *not* automatically
-offloaded](https://www.tensorflow.org/tutorials/using_gpu) to GPUs. Instead, you
-must explicitly specify when GPUs should be used.
-
-The simplest way to do this is to enclose your computation in a `with
-tf.device('/gpu:0')` block. Also of interest is the `tfe.num_gpus()` function,
-which returns the number of available GPUs.
-
-For example, consider this snippet to measure the time to multiply two 1000x1000
-matrices on CPU:
-
-```python
-import time
-
-def measure(x):
-  # The very first time a GPU is used by TensorFlow, it is initialized.
-  # So exclude the first run from timing.
-  tf.matmul(x, x)
-
-  start = time.time()
-  for i in range(10):
-    tf.matmul(x, x)
-  end = time.time()
-
-  return "Took %s seconds to multiply a %s matrix by itself 10 times" % (end - start, x.shape)
-
-# Run on CPU:
-with tf.device("/cpu:0"):
-  print("CPU: %s" % measure(tf.random_normal([1000, 1000])))
-
-# If a GPU is available, run on GPU:
-if tfe.num_gpus() > 0:
-  with tf.device("/gpu:0"):
-    print("GPU: %s" % measure(tf.random_normal([1000, 1000])))
-```
-
-Output (exact numbers will depend on the characteristics of the hardware):
-
-```python
-CPU: Took 0.145531892776 seconds to multiply a (1000, 1000) matrix by itself 10 times
-GPU: Took 0.000458955764771 seconds to multiply a (1000, 1000) matrix by itself 10 times
-```
-
-Alternatively, methods on the `Tensor` object can be used to explicitly copy the
-`Tensor` to a different device. Operations are typically executed on the device
-on which the inputs are placed. For example:
-
-```python
-x = tf.random_normal([10, 10])
-
-x_gpu0 = x.gpu()
-x_cpu = x.cpu()
-
-_ = tf.matmul(x_cpu, x_cpu)  # Runs on CPU
-_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
-
-if tfe.num_gpus() > 1:
-  x_gpu1 = x.gpu(1)
-  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
-```
-
-### Automatic Differentiation
-
-[Automatic
-differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) is
-very useful when implementing many machine learning algorithms (e.g.,
-[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training
-neural networks). For this purpose, TensorFlow eager execution provides an
-[autograd](https://github.com/HIPS/autograd)-style API for automatic
-differentiation. Specifically, the functions:
-
--   `tfe.gradients_function(f)`: Returns a Python function that computes the
-    derivatives of the Python function `f` with respect to its arguments. `f`
-    must return a scalar value. When the returned function is invoked, it
-    returns a list of `Tensor` objects (one element for each argument of `f`).
--   `tfe.value_and_gradients_function(f)`: Similar to `tfe.gradients_function`,
-    except that when the returned function is invoked, it returns the value of
-    `f` in addition to the list of derivatives of `f` with respect to its
-    arguments.
-
-These functions naturally apply to higher order differentiation as well. For
-example:
-
-```python
-def f(x):
-  return tf.multiply(x, x)  # Or x * x
-assert 9 == f(3.).numpy()
-
-df = tfe.gradients_function(f)
-assert 6 == df(3.)[0].numpy()
-
-# Second order deriviative.
-d2f = tfe.gradients_function(lambda x: df(x)[0])
-assert 2 == d2f(3.)[0].numpy()
-
-# Third order derivative: Will be None
-d3f = tfe.gradients_function(lambda x : d2f(x)[0])
-assert None == d3f(3.)[0]
-```
-
-These functions can be used to train models. For example, consider the following
-simple linear regression model:
-
-```python
-def prediction(input, weight, bias):
-  return input * weight + bias
-
-# A toy dataset of points around 3 * x + 2
-NUM_EXAMPLES = 1000
-training_inputs = tf.random_normal([NUM_EXAMPLES])
-noise = tf.random_normal([NUM_EXAMPLES])
-training_outputs = training_inputs * 3 + 2 + noise
-
-# A loss function: Mean-squared error
-def loss(weight, bias):
-  error = prediction(training_inputs, weight, bias) - training_outputs
-  return tf.reduce_mean(tf.square(error))
-
-# Function that returns the derivative of loss with respect to
-# weight and bias
-grad = tfe.gradients_function(loss)
-
-# Train for 200 steps (starting from some random choice for W and B, on the same
-# batch of data).
-W = 5.
-B = 10.
-learning_rate = 0.01
-print("Initial loss: %f" % loss(W, B).numpy())
-for i in range(200):
-  (dW, dB) = grad(W, B)
-  W -= dW * learning_rate
-  B -= dB * learning_rate
-  if i % 20 == 0:
-    print("Loss at step %d: %f" % (i, loss(W, B).numpy()))
-print("Final loss: %f" % loss(W, B).numpy())
-print("W, B = %f, %f" % (W.numpy(), B.numpy()))
-```
-
-Output: (the exact numbers may vary depending on the randomness in noise)
-
-```
-Initial loss: 66.730003
-Loss at step 0: 64.200096
-Loss at step 20: 29.872814
-Loss at step 40: 14.233772
-Loss at step 60: 7.090570
-Loss at step 80: 3.819887
-Loss at step 100: 2.318821
-Loss at step 120: 1.628385
-Loss at step 140: 1.310142
-Loss at step 160: 1.163167
-Loss at step 180: 1.095162
-Final loss: 1.064711
-W, B = 3.094944, 2.161383
-```
-
-To utilize the GPU, place the code above within a `with tf.device("/gpu:0"):`
-block. (However, this particular model, with only two floating point parameters,
-is unlikely to benefit from GPU acceleration.)
-
-### Customizing gradients
-
-One may want to define custom gradients for an operation, or for a function.
-This may be useful for multiple reasons, including providing a more efficient
-or more [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability)
-gradient for a sequence of operations.
-
-For example, consider the function `log(1 + e^x)`, which commonly occurs in the
-computation of cross entropy and log likelihoods.
-
-```python
-def log1pexp(x):
-  return tf.log(1 + tf.exp(x))
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# Works fine at x = 0.
-assert 0.5 == float(grad_log1pexp(0.)[0])
-
-# Returns a `nan` at x = 100 due to numerical instability.
-import math
-assert math.isnan(float(grad_log1pexp(100.)[0]))
-```
-
-We can define a custom gradient for the above function that analytically
-simplifies the gradient expression.
-
-```python
-@tfe.custom_gradient
-def log1pexp(x):
-  e = tf.exp(x)
-  def grad(dy):
-    return dy * (1 - 1 / (1 + e))
-  return tf.log(1 + e), grad
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# Works as before at x = 0.
-assert 0.5 == float(grad_log1pexp(0.)[0])
-
-# But now works at x = 100 as well.
-assert 1.0 == float(grad_log1pexp(100.)[0])
-```
-Also notice how the gradient function implementation reuses an expression
-(`tf.exp(x)`) computed during the forward pass, hence making the gradient
-computation more efficient by avoiding redundant computation.
-
-## Building and training models
-
-In practice, your computation may have many parameters to be optimized (by
-computing derivatives). Encapsulating them into re-usable classes/objects
-makes the code easier to follow than writing a single top-level function with
-many arguments.
-
-In fact, eager execution encourages use of the [Keras](https://keras.io)-style
-"Layer" classes in the
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers)
-module.
-
-Furthermore, you may want to apply more sophisticated techniques to compute
-parameter updates, such as those in
-[`tf.train.Optimizer`](https://www.tensorflow.org/api_guides/python/train#Optimizers)
-implementations.
-
-This next section walks through using the same `Optimizer` and `Layer` APIs used
-to build trainable TensorFlow graphs in an environment where eager execution is
-enabled.
-
-### Variables and Optimizers
-
-`tfe.Variable` objects store mutable `Tensor` values that can be accessed during
-training, making automatic differentiation easier. In particular, parameters of
-a model can be encapsulated in Python classes as variables.
-
-`tfe.gradients_function(f)` introduced earlier computes the derivatives of `f`
-with respect to its arguments. However, it requires all parameters of interest
-to be arguments of `f`, which becomes cumbersome when `f` depends on a large
-number of trainable parameters.
-
-`tfe.implicit_gradients` is an alternative function with some useful properties:
-
--   It computes the derivatives of `f` with respect to all the `tfe.Variable`s
-    used by `f`.
--   When the returned function is invoked, it returns a list of
-    (gradient value, Variable object) tuples.
-
-Representing model parameters as `Variable` objects, along with the use of
-`tfe.implicit_gradients`, typically results in better encapsulation. For
-example, the linear regression model described above can be written into a
-class:
-
-```python
-class Model(object):
-  def __init__(self):
-    self.W = tfe.Variable(5., name='weight')
-    self.B = tfe.Variable(10., name='bias')
-
-  def predict(self, inputs):
-    return inputs * self.W + self.B
-
-
-# The loss function to be optimized
-def loss(model, inputs, targets):
-  error = model.predict(inputs) - targets
-  return tf.reduce_mean(tf.square(error))
-
-# A toy dataset of points around 3 * x + 2
-NUM_EXAMPLES = 1000
-training_inputs = tf.random_normal([NUM_EXAMPLES])
-noise = tf.random_normal([NUM_EXAMPLES])
-training_outputs = training_inputs * 3 + 2 + noise
-
-# Define:
-# 1. A model
-# 2. Derivatives of a loss function with respect to model parameters
-# 3. A strategy for updating the variables based on the derivatives
-model = Model()
-grad = tfe.implicit_gradients(loss)
-optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
-
-# The training loop
-print("Initial loss: %f" %
-      loss(model, training_inputs, training_outputs).numpy())
-for i in range(201):
-  optimizer.apply_gradients(grad(model, training_inputs, training_outputs))
-  if i % 20 == 0:
-    print("Loss at step %d: %f" %
-          (i, loss(model, training_inputs, training_outputs).numpy()))
-print("Final loss: %f" % loss(model, training_inputs, training_outputs).numpy())
-print("W, B = %s, %s" % (model.W.numpy(), model.B.numpy()))
-```
-
-Output:
-
-```
-Initial loss: 69.693184
-Loss at step 0: 66.987854
-Loss at step 20: 30.553387
-Loss at step 40: 14.250237
-Loss at step 60: 6.955020
-Loss at step 80: 3.690550
-Loss at step 100: 2.229739
-Loss at step 120: 1.576032
-Loss at step 140: 1.283496
-Loss at step 160: 1.152584
-Loss at step 180: 1.093999
-Final loss: 1.067780
-W, B = 3.0114281, 2.0865183
-```
-
-Using `implicit_gradients` avoids the need to provide all the trainable
-parameters of the model as arguments to the `loss` function.
-
-### Using Keras and the Layers API
-
-[Keras](https://keras.io) is a popular API for defining model structures. The
-[`tf.keras.layers`](https://www.tensorflow.org/api_docs/python/tf/keras/layers)
-module provides a set of building blocks for models and is implemented using the
-`tf.layers.Layer` subclasses in the
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers)
-module. We encourage the use of these same building blocks when using
-TensorFlow's eager execution feature. For example, the very same linear
-regression model can be built using `tf.layers.Dense`:
-
-```python
-class Model(object):
-  def __init__(self):
-    self.layer = tf.layers.Dense(1)
-
-  def predict(self, inputs):
-    return self.layer(inputs)
-```
-
-The `tf.layers` API makes it more convenient to define more sophisticated
-models. For example, the following will train an MNIST model:
-
-```python
-class MNISTModel(object):
-  def __init__(self, data_format):
-    # 'channels_first' is typically faster on GPUs
-    # while 'channels_last' is typically faster on CPUs.
-    # See: https://www.tensorflow.org/performance/performance_guide#data_formats
-    if data_format == 'channels_first':
-      self._input_shape = [-1, 1, 28, 28]
-    else:
-      self._input_shape = [-1, 28, 28, 1]
-    self.conv1 = tf.layers.Conv2D(32, 5,
-                                  padding='same',
-                                  activation=tf.nn.relu,
-                                  data_format=data_format)
-    self.max_pool2d = tf.layers.MaxPooling2D(
-        (2, 2), (2, 2), padding='same', data_format=data_format)
-    self.conv2 = tf.layers.Conv2D(64, 5,
-                                  padding='same',
-                                  activation=tf.nn.relu,
-                                  data_format=data_format)
-    self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu)
-    self.dropout = tf.layers.Dropout(0.5)
-    self.dense2 = tf.layers.Dense(10)
-
-  def predict(self, inputs):
-    x = tf.reshape(inputs, self._input_shape)
-    x = self.max_pool2d(self.conv1(x))
-    x = self.max_pool2d(self.conv2(x))
-    x = tf.layers.flatten(x)
-    x = self.dropout(self.dense1(x))
-    return self.dense2(x)
-
-def loss(model, inputs, targets):
-  return tf.reduce_mean(
-      tf.nn.softmax_cross_entropy_with_logits(
-          logits=model.predict(inputs), labels=targets))
-
-
-# Load the training and validation data
-from tensorflow.examples.tutorials.mnist import input_data
-data = input_data.read_data_sets("./mnist_data", one_hot=True)
-
-# Train
-device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last')
-optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
-grad = tfe.implicit_gradients(loss)
-for i in range(20001):
-  with tf.device(device):
-    (inputs, targets) = data.train.next_batch(50)
-    optimizer.apply_gradients(grad(model, inputs, targets))
-    if i % 100 == 0:
-      print("Step %d: Loss on training set : %f" %
-            (i, loss(model, inputs, targets).numpy()))
-print("Loss on test set: %f" % loss(model, data.test.images, data.test.labels).numpy())
-```
-
-For a more complete example, see [the example in the tensorflow/models
-repository](https://github.com/tensorflow/models/tree/master/official/mnist/mnist_eager.py).
-
-### Checkpointing trained variables
-
-TensorFlow Variables (`tfe.Variable`) provide a way to represent shared,
-persistent state of your model. The `tfe.Checkpoint` class provides a means to
-save and restore variables to and from _checkpoints_.
-
-For example:
-
-```python
-# Create variables.
-x = tfe.Variable(10.)
-y = tfe.Variable(5.)
-
-# Indicate that the variables should be saved as "x" and "y".
-checkpoint = tfe.Checkpoint(x=x, y=y)
-
-# Assign new values to the variables and save.
-x.assign(2.)
-save_path = checkpoint.save('/tmp/ckpt')
-
-# Change the variable after saving.
-x.assign(11.)
-assert 16. == (x + y).numpy()  # 11 + 5
-
-# Restore the values in the checkpoint.
-checkpoint.restore(save_path)  # save_path='/tmp/ckpt-1'
-
-assert 7. == (x + y).numpy()  # 2 + 5
-```
-
-### `tf.keras.Model`
-
-You may often want to organize your models using classes, like the `MNISTModel`
-class described above. We recommend inheriting from the `tf.keras.Model` class
-as it provides conveniences like keeping track of all model variables.
-
-Sub-classes of `tf.keras.Model` may register `Layer`s (like classes in
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers), or [Keras
-layers](https://www.tensorflow.org/api_docs/python/tf/keras/layers)) by
-assigning them to attributes (`self.name = layer_object`) and define the
-computation in an implementation of `call()`.
-
-Note that `tf.layers.Layer` objects (like `tf.layers.Dense`) create variables
-lazily, when the first input is encountered.
-
-For example, consider the following two-layer neural network:
-
-```python
-class TwoLayerNet(tf.keras.Model):
-  def __init__(self):
-    super(TwoLayerNet, self).__init__()
-    self.layer1 = tf.layers.Dense(2, activation=tf.nn.relu, use_bias=False)
-    self.layer2 = tf.layers.Dense(3, use_bias=False)
-
-  def call(self, x):
-    return self.layer2(self.layer1(x))
-
-net = TwoLayerNet()
-
-# No variables created yet
-assert 0 == len(net.variables)
-
-# They are created on first input:
-inp = tf.constant([[1.]])
-
-# Since input is a 1x1 matrix, net.l1 has 2 units and net.l2 has 3 units,
-# the output is the product of a 1x1 matrix with a 1x2 matrix with a 2x3
-# matrix.
-assert [1, 3] == net(inp).shape.as_list()  # Invoke net; get output shape.
-assert 1 == len(net.layer1.variables)
-assert 1 == len(net.layer2.variables)
-assert 2 == len(net.variables)  # weights for each layer.
-assert [1, 2] == net.variables[0].shape.as_list()  # weights of layer1.
-assert [2, 3] == net.variables[1].shape.as_list()  # weights of layer2.
-```
-
-The `tf.keras.Model` class is itself a sub-class of `tf.layers.Layer`. This
-allows instances of `tf.keras.Model` to be embedded in other models. For
-example:
-
-```python
-class ThreeLayerNet(tf.keras.Model):
-  def __init__(self):
-    super(ThreeLayerNet, self).__init__()
-    self.a = TwoLayerNet()
-    self.b = tf.layers.Dense(4, use_bias=False)
-
-  def call(self, x):
-    return self.b(self.a(x))
-
-net = ThreeLayerNet()
-
-assert [1, 4] == net(inp).shape.as_list()
-assert 3 == len(net.variables)
-assert [1, 2] == net.variables[0].shape.as_list()
-assert [2, 3] == net.variables[1].shape.as_list()
-assert [3, 4] == net.variables[2].shape.as_list()
-```
-
-See more examples in
-[`tensorflow/contrib/eager/python/examples`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples).
-
-`tfe.Checkpoint` provides a convenient way to save and load training
-checkpoints. Let's define something simple to train. We set an objective for the
-output of our network, choose an optimizer, and a location for the checkpoint:
-
-```python
-objective = tf.constant([[2., 3., 4., 5.]])
-optimizer = tf.train.AdamOptimizer(0.01)
-checkpoint_directory = '/tmp/tfe_example'
-checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
-net = ThreeLayerNet()
-```
-
-We group them in a `tfe.Checkpoint` and request that it be restored. This
-ensures that variables created by these objects are restored before their values
-are used. Our training loop is the same whether starting training or resuming
-from a previous checkpoint:
-
-```python
-global_step = tf.train.get_or_create_global_step()
-checkpoint = tfe.Checkpoint(
-    global_step=global_step, optimizer=optimizer, network=net)
-checkpoint.restore(tf.train.latest_checkpoint(checkpoint_directory))
-for _ in range(100):
-  loss_fn = lambda: tf.norm(net(inp) - objective)
-  optimizer.minimize(loss_fn, global_step=global_step)
-  if tf.equal(global_step % 20, 0):
-    print("Step %d, output %s" % (global_step.numpy(),
-                                  net(inp).numpy()))
-    # Save the checkpoint.
-    checkpoint.save(checkpoint_prefix)
-```
-
-The first time it runs, `Model` variables are initialized randomly. Then the
-output is trained to match the objective we've set:
-
-```
-Step 20, output [[ 0.03575622  0.29863232  0.03474367  0.24735749]]
-Step 40, output [[ 0.40646029  0.9856872   0.46851286  0.95358551]]
-Step 60, output [[ 1.74541104  2.800704    1.79055595  2.74783421]]
-Step 80, output [[ 2.14977384  3.44340849  3.96120024  5.16242075]]
-Step 100, output [[ 1.99943113  3.02364397  3.93500996  4.9610076 ]]
-```
-
-In subsequent iterations, variables are initialized with the values read from
-the latest checkpoint. Running the same code again, we continue from where we
-left off:
-
-```
-Step 120, output [[ 1.99234128  3.0271616   3.98732996  4.96401167]]
-Step 140, output [[ 2.00133467  3.01270437  4.00616646  5.00406504]]
-Step 160, output [[ 1.99647415  2.9956708   3.99064088  4.99632359]]
-Step 180, output [[ 2.00699997  3.00904822  4.00706148  5.01193142]]
-Step 200, output [[ 1.98334622  2.98249531  3.97375059  4.97123432]]
-```
-
-
-### Summaries, metrics and TensorBoard
-
-[TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard)
-is a popular tool for understanding, debugging and optimizing the model training
-process. To benefit from the visualizations offered by TensorBoard, summary
-events need to be written during the course of execution of your program. You
-might find many Tensorflow programs that include the
-[`tf.summary`](https://www.tensorflow.org/api_guides/python/summary) operations
-during graph construction.
-
-`tf.summary` operations are *not* compatible with eager execution, but an
-equivalent alternative exists in
-[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/summary)
-that is compatible with both eager execution and graph construction.
-
-During model construction simply insert summary operations like
-`tf.contrib.summary.scalar`. These operations do nothing by default, unless a
-summary writer is currently active and a writing policy is set.
-
-For example, to record summaries once every 100 global steps, use:
-
-```python
-tf.train.get_or_create_global_step()  # Ensuring the global step variable exists
-writer = tf.contrib.summary.create_file_writer(logdir)
-
-for _ in range(iterations):
-  with writer.as_default():
-    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
-      # your model code goes here
-      tf.contrib.summary.scalar('loss', loss)
-      # ...
-```
-
-See the full mnist example in
-[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
-for a full model using `tf.contrib.summary`.
-
-Similarly to summaries, the metrics in `tf.metrics` are currently not compatible
-with eager execution. We instead provide object-oriented metrics in the
-`tfe.metrics` package, which are compatible with graph construction as well.
-
-Metrics in the `tfe.metrics`, such as `tfe.metrics.Mean` and
-`tfe.Metrics.Accuracy`, all implement an intuitive object-oriented
-interface. Here's an example of how to use the `tfe.metrics.Mean` metric:
-
-```python
-# Metrics are objects, which can be created and destroyed.
-my_mean = tfe.metrics.Mean(name='my_mean')
-# While a metric is active, you can call it as a function to accumulate into its
-# internal state.
-my_mean(0.0)
-my_mean(10.0)
-# Once you've finished updating the metric, you can get its result. In this case
-# a simple average over all the calls to it. If a summary writer is active the
-# metric will write the appropriate summaries using the metric name.
-assert 5.0 == my_mean.result().numpy()
-```
-
-For a full example of a model using metrics for evaluation, see the mnist
-example in
-[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist).
-
-### Input Pipelines
-
-The discussion above has been centered around the computation executed by your
-model. The
-[`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data)
-module provides APIs to build complex input pipelines from simple, reusable
-pieces.
-
-If you're familiar with constructing `tf.data.Dataset` objects when building
-TensorFlow graphs, the same API calls are used when eager execution is enabled.
-However, the process of iterating over elements of the dataset differs between
-eager execution and graph construction. When eager execution is enabled, the
-discussion on iterator creation using `make_one_shot_iterator()` and
-`get_next()` in the
-[Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is
-*not* applicable. Instead, a more Pythonic `Iterator` class is available.
-
-For example:
-
-```python
-# Create a source Dataset from in-memory numpy arrays.
-# For reading from files on disk, you may want to use other Dataset classes
-# like the TextLineDataset or the TFRecordDataset.
-dataset = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])
-
-# Apply transformations, shuffling, batching etc.
-dataset = dataset.map(tf.square).shuffle(2).batch(2)
-
-# Use tfe.Iterator to iterate over the dataset.
-for x in tfe.Iterator(dataset):
-  print(x)
-```
-
-Output:
-
-```
-tf.Tensor([4 9], shape=(2,), dtype=int32)
-tf.Tensor([16 25], shape=(2,), dtype=int32)
-tf.Tensor([36  1], shape=(2,), dtype=int32)
-```
-
-## Interoperating with Graphs
-
-Eager execution improves the process of model development in Python; however,
-because it is in its earliest stages, it does not yet support some features
-available to [TensorFlow
-graphs](https://www.tensorflow.org/get_started/get_started#the_computational_graph)
-that are desirable when deploying models in production. In particular, eager
-execution does not yet support distributed training, exporting models (to other
-[programming languages](https://www.tensorflow.org/api_docs/), [TensorFlow
-serving](https://www.tensorflow.org/serving/), and mobile applications), and
-various memory and computation optimizations that are applied to TensorFlow's
-dataflow graphs.
-
-That said, the APIs used to build modes are exactly the same whether executing
-eagerly or constructing graphs. This means that you can iteratively develop your
-model with eager execution enabled and later, if needed, use the same code to
-reap the benefits of representing models as computational graphs.
-
-For example, the same model definition used to construct a graph in
-[mnist.py`](https://github.com/tensorflow/models/tree/master/official/mnist/mnist.py)
-can be trained with eager execution enabled as in [`mnist_eager.py`](https://github.com/tensorflow/models/tree/master/official/mnist/mnist_eager.py).
-
-Other models in the [examples
-directory](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/)
-demonstrate this as well.
-
-Some differences worth noting:
-
--   There is no notion of a `tf.placeholder` or a `tf.Session` when eager
-    execution is enabled.
--   Many properties on the `tf.Tensor` object, like `tf.Tensor.name`,
-    `tf.Tensor.op`, `tf.Tensor.inputs` are not meaningful when eager execution
-    is enabled and their use will raise an `AttributeError`.
--   To use `tfe.implicit_gradients` in graph construction, variables must be
-    created with [`use_resource=True`] provided to
-    [`tf.get_variable()`](https://www.tensorflow.org/api_docs/python/tf/get_variable)
-    or
-    [`tf.variable_scope()`](https://www.tensorflow.org/api_docs/python/tf/variable_scope).
--   Some API calls (such as the functional-style `tf.layers.dense`,
-    `tf.layers.conv2d`) are not compatible with eager execution. Use of such
-    methods should raise an error indicating the alternative (e.g., the
-    `tf.layers.Dense` and `tf.layers.Conv2D` classes).
-
-## What next?
+immediately: concrete values are returned, instead of creating a computational
+graph that is executed later.
 
-Please give eager execution a spin. This feature is in early stages and is
-evolving, so we welcome your feedback via issues on GitHub (see [known
-issues](https://github.com/tensorflow/tensorflow/labels/comp:eager)).
+A user guide is available: https://www.tensorflow.org/programmers_guide/eager
+([source file](../../../../docs_src/programmers_guide/eager.md))
 
-You may want to browse through some sample code, including benchmarks for some:
+We welcome feedback through [GitHub issues](https://github.com/tensorflow/tensorflow/labels/comp:eager).
 
--   [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression)
--   [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
--   [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50)
--   [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot)
--   [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb)
+Sample code is available, including benchmarks for some:
 
+- [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression)
+- [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
+- [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50)
+- [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot)
+- [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb)
diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
new file mode 100644
index 0000000000..9ae1e602f4
--- /dev/null
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -0,0 +1,992 @@
+# Eager Execution
+
+TensorFlow's eager execution is an imperative programming environment that
+evaluates operations immediately, without an extra graph-building step.
+Operations return concrete values instead of constructing a computational graph
+to run later. This makes it easy to get started with TensorFlow, debug models,
+reduce boilerplate code, and is fun! To follow along with this guide, run the
+code samples below in an interactive `python` interpreter.
+
+Eager execution supports most TensorFlow operations and GPU acceleration.
+Automatic differentiation uses a dynamically-constructed tape instead of a static
+graph to compute gradients. Eager execution is a flexible machine learning
+platform for research and experimentation that provides:
+
+* *An intuitive interface* —Structure your code naturally and use Python data
+  structures. Quickly iterate on small models and small data.
+* *Easier debugging* —Call ops directly to inspect running models and test
+  changes. Use standard Python debugging tools for immediate error reporting.
+* *Natural control flow* —Use Python control flow instead of graph control flow,
+  including support for dynamic models.
+
+For a collection of examples running in eager execution, see:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+Note: Some models may experience increased overhead with eager execution enabled.
+Performance improvements are ongoing, but please
+[file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a
+problem and share your benchmarks.
+
+## Setup and basic usage
+
+Install TensorFlow 1.7 to include the updates for eager execution:
+
+```
+$ pip install --pre --upgrade tensorflow
+```
+
+To start eager execution, add `tf.enable_eager_execution()` to the beginning of
+the program or console session. Do not add this operation to other modules that
+the program calls.
+
+```py
+from __future__ import absolute_import, division, print_function
+
+import tensorflow as tf
+
+tf.enable_eager_execution()
+```
+
+Now you can run TensorFlow operations and the results will return immediately:
+
+```py
+tf.executing_eagerly()        # => True
+
+x = [[2.]]
+m = tf.matmul(x, x)
+print("hello, {}".format(m))  # => "hello, [[4.]]"
+```
+
+Enabling eager execution changes how TensorFlow operations behave—now they
+immediately evaluate and return their values to Python. `tf.Tensor` objects
+reference concrete values instead of symbolic handles to nodes in a computational
+graph. Since there isn't a computational graph to build and run later in a
+session, it's easy to inspect results using `print()` or a debugger. Evaluating,
+printing, and checking tensor values does not break the flow for computing
+gradients.
+
+Eager execution works nicely with [NumPy](http://www.numpy.org/). NumPy
+operations accept `tf.Tensor` arguments. TensorFlow
+[math operations](https://www.tensorflow.org/api_guides/python/math_ops) convert
+Python objects and NumPy arrays to `tf.Tensor` objects. The
+`tf.Tensor.numpy` method returns the object's value as a NumPy `ndarray`.
+
+```py
+a = tf.constant([[1, 2],
+                 [3, 4]])
+print(a)
+# => tf.Tensor([[1 2]
+#               [3 4]], shape=(2, 2), dtype=int32)
+
+# Broadcasting support
+b = tf.add(a, 1)
+print(b)
+# => tf.Tensor([[2 3]
+#               [4 5]], shape=(2, 2), dtype=int32)
+
+# Operator overloading is supported
+print(a * b)
+# => tf.Tensor([[ 2  6]
+#               [12 20]], shape=(2, 2), dtype=int32)
+
+# Use NumPy values
+import numpy as np
+
+c = np.multiply(a, b)
+print(c)
+# => [[ 2  6]
+#     [12 20]]
+
+# Obtain numpy value from a tensor:
+print(a.numpy())
+# => [[1 2]
+#     [3 4]]
+```
+
+The `tfe` module contains symbols available to both eager and graph execution
+environments and is useful for writing code to [work with graphs](#work_with_graphs):
+
+```py
+import tensorflow.contrib.eager as tfe
+```
+
+## Eager training
+
+### Automatic differentiation
+
+[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)
+is useful for implementing machine learning algorithms such as
+[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training
+neural networks. During eager execution, use `tfe.GradientTape` to trace
+operations for computing gradients later.
+
+`tfe.GradientTape` is an opt-in feature to provide maximal performance when
+not tracing. Since different operations can occur during each call, all
+forward-pass operations get recorded to a "tape". To compute the gradient, play
+the tape backwards and then discard. A particular `tfe.GradientTape` can only
+be computed once, subsequent calls throw a runtime error.
+
+```py
+w = tfe.Variable([[1.0]])
+with tfe.GradientTape() as tape:
+  loss = w * w
+
+grad = tape.gradient(loss, [w])
+print(grad)  # => [tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)]
+```
+
+Here's an example of `tfe.GradientTape` that records forward-pass operations
+to train a simple model:
+
+```py
+# A toy dataset of points around 3 * x + 2
+NUM_EXAMPLES = 1000
+training_inputs = tf.random_normal([NUM_EXAMPLES])
+noise = tf.random_normal([NUM_EXAMPLES])
+training_outputs = training_inputs * 3 + 2 + noise
+
+def prediction(input, weight, bias):
+  return input * weight + bias
+
+# A loss function using mean-squared error
+def loss(weights, biases):
+  error = prediction(training_inputs, weights, biases) - training_outputs
+  return tf.reduce_mean(tf.square(error))
+
+# Return the derivative of loss with respect to weight and bias
+def grad(weights, biases):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(weights, biases) 
+  return tape.gradient(loss_value, [weights, biases])
+
+train_steps = 200
+learning_rate = 0.01
+# Start with arbitrary values for W and B on the same batch of data
+W = tfe.Variable(5.)
+B = tfe.Variable(10.)
+
+print("Initial loss: {:.3f}".format(loss(W, B)))
+
+for i in range(train_steps):
+  dW, dB = grad(W, B)
+  W.assign_sub(dW * learning_rate)
+  B.assign_sub(dB * learning_rate)
+  if i % 20 == 0:
+    print("Loss at step {:03d}: {:.3f}".format(i, loss(W, B)))
+
+print("Final loss: {:.3f}".format(loss(W, B)))
+print("W = {}, B = {}".format(W.numpy(), B.numpy()))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 71.204
+Loss at step 000: 68.333
+Loss at step 020: 30.222
+Loss at step 040: 13.691
+Loss at step 060: 6.508
+Loss at step 080: 3.382
+Loss at step 100: 2.018
+Loss at step 120: 1.422
+Loss at step 140: 1.161
+Loss at step 160: 1.046
+Loss at step 180: 0.996
+Final loss: 0.974
+W = 3.01582956314, B = 2.1191945076
+```
+
+Replay the `tfe.GradientTape` to compute the gradients and apply them in a
+training loop. This is demonstrated in an excerpt from the
+[mnist_eager.py](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_eager.py)
+example:
+
+```py
+dataset = tf.data.Dataset.from_tensor_slices((data.train.images,
+                                              data.train.labels))
+...
+for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)):
+  ...
+  with tfe.GradientTape() as tape:
+    logits = model(images, training=True)
+    loss_value = loss(logits, labels)
+  ...
+  grads = tape.gradient(loss_value, model.variables)
+  optimizer.apply_gradients(zip(grads, model.variables),
+                            global_step=tf.train.get_or_create_global_step())
+```
+
+#### Dynamic models
+
+`tfe.GradientTape` can also be used in dynamic models. This example for a
+[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search)
+algorithm looks like normal NumPy code, except there are gradients and is
+differentiable, despite the complex control flow:
+
+```py
+def line_search_step(fn, init_x, rate=1.0):
+  with tfe.GradientTape() as tape:
+    # Variables are automatically recorded, but manually watch a tensor
+    tape.watch(init_x)
+    value = fn(init_x)
+  grad, = tape.gradient(value, [init_x])
+  grad_norm = tf.reduce_sum(grad * grad)
+  init_value = value
+  while value > init_value - rate * grad_norm:
+    x = init_x - rate * grad
+    value = fn(x)
+    rate /= 2.0
+  return x, value
+```
+
+#### Additional functions to compute gradients
+
+`tfe.GradientTape` is a powerful interface for computing gradients, but there
+is another [Autograd](https://github.com/HIPS/autograd)-style API available for
+automatic differentiation. These functions are useful if writing math code with
+only tensors and gradient functions, and without `tfe.Variables`:
+
+* `tfe.gradients_function` —Returns a function that computes the derivatives
+  of its input function parameter with respect to its arguments. The input
+  function parameter must return a scalar value. When the returned function is
+  invoked, it returns a list of `tf.Tensor` objects: one element for each
+  argument of the input function. Since anything of interest must be passed as a
+  function parameter, this becomes unwieldy if there's a dependency on many
+  trainable parameters.
+* `tfe.value_and_gradients_function` —Similar to
+  `tfe.gradients_function`, but when the returned function is invoked, it
+  returns the value from the input function in addition to the list of
+  derivatives of the input function with respect to its arguments.
+
+In the following example, `tfe.gradients_function` takes the `square`
+function as an argument and returns a function that computes the partial
+derivatives of `square` with respect to its inputs. To calculate the derivative
+of `square` at `3`, `grad(3.0)` returns `6`.
+
+```py
+def square(x):
+  return tf.multiply(x, x)
+
+grad = tfe.gradients_function(square)
+
+square(3.)  # => 9.0
+grad(3.)    # => [6.0]
+
+# The second-order derivative of square:
+gradgrad = tfe.gradients_function(lambda x: grad(x)[0])
+gradgrad(3.)  # => [2.0]
+
+# The third-order derivative is None:
+gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])
+gradgradgrad(3.)  # => [None]
+
+
+# With flow control:
+def abs(x):
+  return x if x > 0. else -x
+
+grad = tfe.gradients_function(abs)
+
+grad(3.)   # => [1.0]
+grad(-3.)  # => [-1.0]
+```
+
+### Custom gradients
+
+Custom gradients are an easy way to override gradients in eager and graph
+execution. Within the forward function, define the gradient with respect to the
+inputs, outputs, or intermediate results. For example, here's an easy way to clip
+the norm of the gradients in the backward pass:
+
+```py
+@tf.custom_gradient
+def clip_gradient_by_norm(x, norm):
+  y = tf.identity(x)
+  def grad_fn(dresult):
+    return [tf.clip_by_norm(dresult, norm), None]
+  return y, grad_fn
+```
+
+Custom gradients are commonly used to provide a numerically stable gradient for a
+sequence of operations:
+
+```py
+def log1pexp(x):
+  return tf.log(1 + tf.exp(x))
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# The gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# However, x = 100 fails because of numerical instability.
+grad_log1pexp(100.)  # => [nan]
+```
+
+
+Here, the `log1pexp` function can be analytically simplified with a custom
+gradient. The implementation below reuses the value for `tf.exp(x)` that is
+computed during the forward pass—making it more efficient by eliminating
+redundant calculations:
+
+```py
+@tfe.custom_gradient
+def log1pexp(x):
+  e = tf.exp(x)
+  def grad(dy):
+    return dy * (1 - 1 / (1 + e))
+  return tf.log(1 + e), grad
+
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# As before, the gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# And the gradient computation also works at x = 100.
+grad_log1pexp(100.)  # => [1.0]
+```
+
+
+## Build and train models
+
+There are many parameters to optimize when calculating derivatives. TensorFlow
+code is easier to read when structured into reusable classes and objects instead
+of a single top-level function. Eager execution encourages the use of the
+Keras-style layer classes in the `tf.keras.layers` module. Additionally, the
+`tf.train.Optimizer` classes provide sophisticated techniques to calculate
+parameter updates.
+
+The following example creates a multi-layer model that classifies the standard
+[MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It
+demonstrates the optimizer and layer APIs to build trainable graphs in an eager
+execution environment.
+
+### Build a model
+
+The `tf.keras.Sequential` model is a linear stack of layers. It is easy to
+use for basic models:
+
+```py
+model = tf.keras.Sequential([
+  tf.keras.layers.Dense(10, input_shape=(784,)),  # must declare input shape
+  tf.keras.layers.Dense(10)
+])
+```
+
+Alternatively, organize models in classes by inheriting from `tf.keras.Model`.
+This is a container for layers that is a layer itself, allowing `tf.keras.Model`
+objects to contain other `tf.keras.Model` objects.
+
+```py
+class MNISTModel(tf.keras.Model):
+  def __init__(self):
+    super(MNISTModel, self).__init__()
+    self.dense1 = tf.keras.layers.Dense(units=10)
+    self.dense2 = tf.keras.layers.Dense(units=10)
+
+  def call(self, input):
+    """Run the model."""
+    result = self.dense1(input)
+    result = self.dense2(result)
+    result = self.dense2(result)  # reuse variables from dense2 layer
+    return result
+
+model = MNISTModel()
+```
+
+It's not required to set an input shape for the `tf.keras.Model` class since
+the parameters are set the first time input is passed to the layer.
+
+`tf.keras.layers` classes create and contain their own model variables that
+are tied to the lifetime of their layer objects. To share layer variables, share
+their objects.
+
+### Train a model
+
+Even without training, call the model and inspect the output in eager execution:
+
+```py
+# Create a tensor representing a blank image
+batch = tf.zeros([1, 1, 784])
+print(batch.shape)  # => (1, 1, 784)
+
+result = model(batch)
+# => tf.Tensor([[[ 0.  0., ..., 0.]]], shape=(1, 1, 10), dtype=float32)
+```
+
+This example uses the
+[dataset.py module](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py)
+from the
+[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist),
+download this file to your local directory. Run the following to download the
+MNIST data files to your working directory and prepare a `tf.data.Dataset`
+for training:
+
+```py
+import dataset  # download dataset.py file
+dataset_train = dataset.train('./datasets').shuffle(60000).repeat(4).batch(32)
+```
+
+To train a model, define a loss function to optimize and then calculate
+gradients. Use an optimizer to update the variables:
+
+```py
+def loss(model, x, y):
+  prediction = model(x)
+  return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=prediction)
+
+def grad(model, inputs, targets):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(model, inputs, targets)
+  return tape.gradient(loss_value, model.variables)
+
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
+
+x, y = tfe.Iterator(dataset_train).next()
+print("Initial loss: {:.3f}".format(loss(model, x, y)))
+
+# Training loop
+for (i, (x, y)) in enumerate(tfe.Iterator(dataset_train)):
+  # Calculate derivatives of the input function with respect to its parameters.
+  grads = grad(model, x, y)
+  # Apply the gradient to the model
+  optimizer.apply_gradients(zip(grads, model.variables),
+                            global_step=tf.train.get_or_create_global_step())
+  if i % 200 == 0:
+    print("Loss at step {:04d}: {:.3f}".format(i, loss(model, x, y)))
+
+print("Final loss: {:.3f}".format(loss(model, x, y)))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 2.674
+Loss at step 0000: 2.593
+Loss at step 0200: 2.143
+Loss at step 0400: 2.009
+Loss at step 0600: 2.103
+Loss at step 0800: 1.621
+Loss at step 1000: 1.695
+...
+Loss at step 6600: 0.602
+Loss at step 6800: 0.557
+Loss at step 7000: 0.499
+Loss at step 7200: 0.744
+Loss at step 7400: 0.681
+Final loss: 0.670
+```
+
+And for faster training, move the computation to a GPU:
+
+```py
+with tf.device("/gpu:0"):
+  for (i, (x, y)) in enumerate(tfe.Iterator(dataset_train)):
+    # minimize() is equivalent to the grad() and apply_gradients() calls.
+    optimizer.minimize(lambda: loss(model, x, y),
+                       global_step=tf.train.get_or_create_global_step())
+```
+
+### Variables and optimizers
+
+`tfe.Variable` objects store mutable `tf.Tensor` values accessed during
+training to make automatic differentiation easier. The parameters of a model can
+be encapsulated in classes as variables.
+
+Better encapsulate model parameters by using `tfe.Variable` with
+`tfe.GradientTape`. For example, the automatic differentiation example above
+can be rewritten:
+
+```py
+class Model(tf.keras.Model):
+  def __init__(self):
+    super(Model, self).__init__()
+    self.W = tfe.Variable(5., name='weight')
+    self.B = tfe.Variable(10., name='bias')
+  def predict(self, inputs):
+    return inputs * self.W + self.B
+
+# A toy dataset of points around 3 * x + 2
+NUM_EXAMPLES = 2000
+training_inputs = tf.random_normal([NUM_EXAMPLES])
+noise = tf.random_normal([NUM_EXAMPLES])
+training_outputs = training_inputs * 3 + 2 + noise
+
+# The loss function to be optimized
+def loss(model, inputs, targets):
+  error = model.predict(inputs) - targets
+  return tf.reduce_mean(tf.square(error))
+
+def grad(model, inputs, targets):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(model, inputs, targets)
+  return tape.gradient(loss_value, [model.W, model.B])
+
+# Define:
+# 1. A model.
+# 2. Derivatives of a loss function with respect to model parameters.
+# 3. A strategy for updating the variables based on the derivatives.
+model = Model()
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
+
+print("Initial loss: {:.3f}".format(loss(model, training_inputs, training_outputs)))
+
+# Training loop
+for i in range(300):
+  grads = grad(model, training_inputs, training_outputs)
+  optimizer.apply_gradients(zip(grads, [model.W, model.B]),
+                            global_step=tf.train.get_or_create_global_step())
+  if i % 20 == 0:
+    print("Loss at step {:03d}: {:.3f}".format(i, loss(model, training_inputs, training_outputs)))
+
+print("Final loss: {:.3f}".format(loss(model, training_inputs, training_outputs)))
+print("W = {}, B = {}".format(model.W.numpy(), model.B.numpy()))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 69.066
+Loss at step 000: 66.368
+Loss at step 020: 30.107
+Loss at step 040: 13.959
+Loss at step 060: 6.769
+Loss at step 080: 3.567
+Loss at step 100: 2.141
+Loss at step 120: 1.506
+Loss at step 140: 1.223
+Loss at step 160: 1.097
+Loss at step 180: 1.041
+Loss at step 200: 1.016
+Loss at step 220: 1.005
+Loss at step 240: 1.000
+Loss at step 260: 0.998
+Loss at step 280: 0.997
+Final loss: 0.996
+W = 2.99431324005, B = 2.02129220963
+```
+
+## Use objects for state during eager execution
+
+With graph execution, program state (such as the variables) is stored in global
+collections and their lifetime is managed by the `tf.Session` object. In
+contrast, during eager execution the lifetime of state objects is determined by
+the lifetime of their corresponding Python object.
+
+### Variables are objects
+
+During eager execution, variables persist until the last reference to the object
+is removed, and is then deleted.
+
+```py
+with tf.device("gpu:0"):
+  v = tfe.Variable(tf.random_normal([1000, 1000]))
+  v = None  # v no longer takes up GPU memory
+```
+
+### Object-based saving
+
+`tfe.Checkpoint` can save and restore `tfe.Variable`s to and from
+checkpoints:
+
+```py
+x = tfe.Variable(10.)
+
+checkpoint = tfe.Checkpoint(x=x)  # save as "x"
+
+x.assign(2.)   # Assign a new value to the variables and save.
+save_path = checkpoint.save('./ckpt/')
+
+x.assign(11.)  # Change the variable after saving.
+
+# Restore values from the checkpoint
+checkpoint.restore(save_path)
+
+print(x)  # => 2.0
+```
+
+To save and load models, `tfe.Checkpoint` stores the internal state of objects,
+without requiring hiiden variables. To record the state of a `model`,
+an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`:
+
+```py
+model = MyModel()
+optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
+checkpoint_dir = ‘/path/to/model_dir’
+checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
+root = tfe.Checkpoint(optimizer=optimizer,
+                      model=model,
+                      optimizer_step=tf.train.get_or_create_global_step())
+
+root.save(file_prefix=checkpoint_prefix)
+# or
+root.restore(tf.train.latest_checkpoint(checkpoint_dir))
+```
+
+### Object-oriented metrics
+
+`tfe.metrics` are stored as objects. Update a metric by passing the new data to
+the callable, and retrieve the result using the `tfe.metrics.result` method,
+for example:
+
+```py
+m = tfe.metrics.Mean("loss")
+m(0)
+m(5)
+m.result()  # => 2.5
+m([8, 9])
+m.result()  # => 5.5
+```
+
+#### Summaries and TensorBoard
+
+@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
+understanding, debugging and optimizing the model training process. It uses
+summary events that are written while executing the program.
+
+`tf.contrib.summary` is compatible with both eager and graph execution
+environments. Summary operations, such as `tf.contrib.summary.scalar`, are
+inserted during model construction. For example, to record summaries once every
+100 global steps:
+
+```py
+tf.train.get_or_create_global_step()  # return global step var
+writer = tf.contrib.summary.create_file_writer(logdir)
+global_step=tf.train.get_or_create_global_step()
+
+writer.set_as_default()
+
+for _ in range(iterations):
+  global_step.assign_add(1)
+  # Must include a record_summaries method
+  with tf.contrib.summary.record_summaries_every_n_global_steps(100):
+    # your model code goes here
+    tf.contrib.summary.scalar('loss', loss)
+     ...
+```
+
+## Performance
+
+Computation is not automatically offloaded to GPUs during eager execution. To
+explicitly direct a computation to a GPU, enclose it in a
+`tf.device('/gpu:0')` block:
+
+```py
+import time
+
+def measure(x, steps):
+  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
+  tf.matmul(x, x)
+  start = time.time()
+  for i in range(steps):
+    x = tf.matmul(x, x)
+    _ = x.numpy()  # Make sure to execute op and not just enqueue it
+  end = time.time()
+  return end - start
+
+shape = (1000, 1000)
+steps = 200
+print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
+
+# Run on CPU:
+with tf.device("/cpu:0"):
+  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+
+# Run on GPU, if available:
+if tfe.num_gpus() > 0:
+  with tf.device("/gpu:0"):
+    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+else:
+  print("GPU: not found")
+```
+
+Output (exact numbers depend on hardware):
+
+```
+Time to multiply a (1000, 1000) matrix by itself 200 times:
+CPU: 4.614904403686523 secs
+GPU: 0.5581181049346924 secs
+```
+
+A `tf.Tensor` object can be copied to a different device to execute its
+operations:
+
+```py
+x = tf.random_normal([10, 10])
+
+x_gpu0 = x.gpu()
+x_cpu = x.cpu()
+
+_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
+_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
+
+if tfe.num_gpus() > 1:
+  x_gpu1 = x.gpu(1)
+  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
+```
+
+### Benchmarks
+
+For compute-heavy models, such as
+[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
+training on a GPU, eager execution performance is comparable to graph execution.
+But this gap grows larger for models with less computation and there is work to
+be done for optimizing hot code paths for models with lots of small operations.
+
+
+## Work with graphs
+
+While eager execution makes development and debugging more interactive,
+TensorFlow graph execution has advantages for distributed training, performance
+optimizations, and production deployment. However, writing graph code can feel
+different than writing regular Python code and more difficult to debug.
+
+For building and training graph-constructed models, the Python program first
+builds a graph representing the computation, then invokes `Session.run` to send
+the graph for execution on the C++-based runtime.  This provides:
+
+* Automatic differentiation using static autodiff.
+* Simple deployment to a platform independent server.
+* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
+* Compilation and kernel fusion.
+* Automatic distribution and replication (placing nodes on the distributed system).
+
+Deploying code written for eager execution is more difficult: either generate a
+graph from the model, or run the Python runtime and code directly on the server.
+
+### Write compatible code
+
+The same code written for eager execution will also build a graph during graph
+execution. Do this by simply running the same code in a new Python session where
+eager execution is not enabled.
+
+Most TensorFlow operations work during eager execution, but there are some things
+to keep in mind:
+
+* Use `tf.data` for input processing instead of queues. It's faster and easier.
+* Use object-oriented layer APIs—like `tf.keras.layers` and
+  `tf.keras.Model`—since they have explicit storage for variables.
+* Most model code works the same during eager and graph execution, but there are
+  exceptions. (For example, dynamic models using Python control flow to change the
+  computation based on inputs.)
+* Once eager execution is enabled with `tf.enable_eager_execution`, it
+  cannot be turned off. Start a new Python session to return to graph execution.
+
+It's best to write code for both eager execution *and* graph execution. This
+gives you eager's interactive experimentation and debuggability with the
+distributed performance benefits of graph execution.
+
+Write, debug, and iterate in eager execution, then import the model graph for
+production deployment. Use `tfe.Checkpoint` to save and restore model
+variables, this allows movement between eager and graph execution environments.
+See the examples in:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+### Use eager execution in a graph environment
+
+Selectively enable eager execution in a TensorFlow graph environment using
+`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
+been called.
+
+```py
+def my_py_func(x):
+  x = tf.matmul(x, x)  # You can use tf ops
+  print(x)  # but it's eager!
+  return x
+
+with tf.Session() as sess:
+  x = tf.placeholder(dtype=tf.float32)
+  # Call eager function in graph!
+  pf = tfe.py_func(my_py_func, [x], tf.float32)
+  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
+```
+
+
+A `tfe.Checkpoint` stores the complete internal state of the objects passed to it. Nothing else is implicitly included. To record the state of a `model`, an `optimizer`, and a global step pass each one to the checkpoint's constructor:
+
+```py
+model = MyModel()
+optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
+checkpoint_dir = ‘/path/to/model_dir’
+checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
+root = tfe.Checkpoint(optimizer=optimizer,
+                      model=model,
+                      optimizer_step=tf.train.get_or_create_global_step())
+
+root.save(file_prefix=checkpoint_prefix)
+# or
+root.restore(tf.train.latest_checkpoint(checkpoint_dir))
+```
+
+### Object-oriented metrics
+
+`tfe.metrics` are stored as objects. Update a metric by passing the new data to
+the callable, and retrieve the result using the `tfe.metrics.result` method,
+for example:
+
+```py
+m = tfe.metrics.Mean("loss")
+m(0)
+m(5)
+m.result()  # => 2.5
+m([8, 9])
+m.result()  # => 5.5
+```
+
+#### Summaries and TensorBoard
+
+@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
+understanding, debugging and optimizing the model training process. It uses
+summary events that are written while executing the program.
+
+`tf.contrib.summary` is compatible with both eager and graph execution
+environments. Summary operations, such as `tf.contrib.summary.scalar`, are
+inserted during model construction. For example, to record summaries once every
+100 global steps:
+
+```py
+tf.train.get_or_create_global_step()  # return global step var
+writer = tf.contrib.summary.create_file_writer(logdir)
+
+for _ in range(iterations):
+  with writer.as_default():
+    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
+      # your model code goes here
+      tf.contrib.summary.scalar('loss', loss)
+      ...
+```
+
+## Performance
+
+Computation is not automatically offloaded to GPUs during eager execution. To
+explicitly direct a computation to a GPU, enclose it in a
+`tf.device('/gpu:0')` block:
+
+```py
+import time
+
+def measure(x, steps):
+  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
+  tf.matmul(x, x)
+  start = time.time()
+  for i in range(steps):
+    x = tf.matmul(x, x)
+    _ = x.numpy()  # Make sure to execute op and not just enqueue it
+  end = time.time()
+  return end - start
+
+shape = (1000, 1000)
+steps = 200
+print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
+
+# Run on CPU:
+with tf.device("/cpu:0"):
+  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+
+# Run on GPU, if available:
+if tfe.num_gpus() > 0:
+  with tf.device("/gpu:0"):
+    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+else:
+  print("GPU: not found")
+```
+
+Output (exact numbers depend on hardware):
+
+```
+Time to multiply a (1000, 1000) matrix by itself 200 times:
+CPU: 4.614904403686523 secs
+GPU: 0.5581181049346924 secs
+```
+
+A `tf.Tensor` object can be copied to a different device to execute its
+operations:
+
+```py
+x = tf.random_normal([10, 10])
+
+x_gpu0 = x.gpu()
+x_cpu = x.cpu()
+
+_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
+_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
+
+if tfe.num_gpus() > 1:
+  x_gpu1 = x.gpu(1)
+  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
+```
+
+### Benchmarks
+
+For compute-heavy models, such as
+[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
+training on a GPU, eager execution performance is comparable to graph execution.
+But this gap grows larger for models with less computation and there is work to
+be done for optimizing hot code paths for models with lots of small operations.
+
+
+## Work with graphs
+
+While eager execution makes development and debugging more interactive,
+TensorFlow graph execution has advantages for distributed training, performance
+optimizations, and production deployment. However, writing graph code can feel
+different than writing regular Python code and more difficult to debug.
+
+For building and training graph-constructed models, the Python program first
+builds a graph representing the computation, then invokes `Session.run` to send
+the graph for execution on the C++-based runtime.  This provides:
+
+* Automatic differentiation using static autodiff.
+* Simple deployment to a platform independent server.
+* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
+* Compilation and kernel fusion.
+* Automatic distribution and replication (placing nodes on the distributed system).
+
+Deploying code written for eager execution is more difficult: either generate a
+graph from the model, or run the Python runtime and code directly on the server.
+
+### Write compatible code
+
+The same code written for eager execution will also build a graph during graph
+execution. Do this by simply running the same code in a new Python session where
+eager execution is not enabled.
+
+Most TensorFlow operations work during eager execution, but there are some things
+to keep in mind:
+
+* Use `tf.data` for input processing instead of queues. It's faster and easier.
+* Use object-oriented layer APIs—like `tf.keras.layers` and
+  `tf.keras.Model`—since they have explicit storage for variables.
+* Most model code works the same during eager and graph execution, but there are
+  exceptions. (For example, dynamic models using Python control flow to change the
+  computation based on inputs.)
+* Once eager execution is enabled with `tf.enable_eager_execution`, it
+  cannot be turned off. Start a new Python session to return to graph execution.
+
+It's best to write code for both eager execution *and* graph execution. This
+gives you eager's interactive experimentation and debuggability with the
+distributed performance benefits of graph execution.
+
+Write, debug, and iterate in eager execution, then import the model graph for
+production deployment. Use `tfe.Checkpoint` to save and restore model
+variables, this allows movement between eager and graph execution environments.
+See the examples in:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+### Use eager execution in a graph environment
+
+Selectively enable eager execution in a TensorFlow graph environment using
+`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
+been called.
+
+```py
+def my_py_func(x):
+  x = tf.matmul(x, x)  # You can use tf ops
+  print(x)  # but it's eager!
+  return x
+
+with tf.Session() as sess:
+  x = tf.placeholder(dtype=tf.float32)
+  # Call eager function in graph!
+  pf = tfe.py_func(my_py_func, [x], tf.float32)
+  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
+```
diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files
index 3fe4cb2dda..7ac63bf2e0 100644
--- a/tensorflow/docs_src/programmers_guide/leftnav_files
+++ b/tensorflow/docs_src/programmers_guide/leftnav_files
@@ -1,8 +1,9 @@
 index.md
 
 ### High Level APIs
-estimators.md
+eager.md
 datasets.md
+estimators.md
 
 ### Low Level APIs
 low_level_intro.md
-- 
GitLab


From c6911faaf4702096064542790d8c9e8e6f938d52 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 29 Mar 2018 13:35:34 -0700
Subject: [PATCH 701/960] Turns eager device placement on by default.

Change the device policy to have silent copies, which are logged when
RunMetadata tracking is enabled.

In the process, changed TensorHandle to always keep its context around if
it gets one. Changed TFE_TensorHandleResolve to, if necessary, copy to the
CPU (since the user has no control as to whether this copy is needed by
default).

PiperOrigin-RevId: 190978086
---
 tensorflow/c/eager/c_api.cc                   | 100 ++++++++++++------
 tensorflow/c/eager/c_api.h                    |  18 ++--
 tensorflow/c/eager/c_api_internal.h           |   5 +-
 tensorflow/c/eager/c_api_test.cc              |  10 +-
 .../core/common_runtime/eager/context.cc      |  15 +--
 .../core/common_runtime/eager/context.h       |  16 +--
 .../core/common_runtime/eager/execute.cc      |   6 +-
 .../common_runtime/eager/tensor_handle.cc     |  17 +--
 .../core/common_runtime/eager/tensor_handle.h |  15 ++-
 tensorflow/core/kernels/function_ops.cc       |   5 +
 tensorflow/python/eager/core_test.py          |  12 +--
 tensorflow/python/eager/function_test.py      |  33 +++---
 tensorflow/python/kernel_tests/BUILD          |   4 +
 .../resource_variable_ops_test.py             |   1 +
 14 files changed, 148 insertions(+), 109 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index 028865d360..bb1492fca2 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -201,18 +201,24 @@ TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h, TF_Status* status) {
   const tensorflow::Tensor* t = nullptr;
   status->status = h->handle->TensorAndDevice(&t, &d, &op_device);
   if (!status->status.ok()) return nullptr;
+  tensorflow::TensorHandle* h_cpu = nullptr;
   if (!IsCPU(d)) {
-    TF_SetStatus(status, TF_UNIMPLEMENTED,
-                 tensorflow::strings::StrCat(
-                     "TFE_TensorHandle can be resolved iff it is on CPU (this "
-                     "handle is on ",
-                     d->name(),
-                     "). Consider using TFE_TensorHandleCopyToDevice to get a "
-                     "copy of the tensor on CPU")
-                     .c_str());
-    return nullptr;
+    status->status = h->handle->CopyToDevice(
+        h->handle->Context(), h->handle->Context()->HostCPU(), &h_cpu);
+    if (!status->status.ok()) {
+      return nullptr;
+    }
+    status->status = h_cpu->TensorAndDevice(&t, &d, &op_device);
+    if (!status->status.ok()) {
+      h_cpu->Unref();
+      return nullptr;
+    }
   }
-  return tensorflow::TF_TensorFromTensor(*t, status);
+  TF_Tensor* retval = tensorflow::TF_TensorFromTensor(*t, status);
+  if (h_cpu != nullptr) {
+    h_cpu->Unref();
+  }
+  return retval;
 }
 }  // extern "C"
 
@@ -258,17 +264,6 @@ void TFE_OpSetXLACompilation(TFE_Op* op, unsigned char enable) {
 }
 
 void TFE_OpAddInput(TFE_Op* op, TFE_TensorHandle* h, TF_Status* status) {
-  if (op->device == nullptr) {
-    // Questionable heuristic ...
-    // - If a device was explicitly set on the op, always use that.
-    // - If not, place on the first non-host device seen.
-    tensorflow::Device* d = nullptr;
-    // TODO(agarwal): This call may block if h is not ready. Avoid this if
-    // possible.
-    status->status = h->handle->Device(&d);
-    if (!status->status.ok()) return;
-    if (!IsCPU(d)) op->device = d;
-  }
   h->handle->Ref();
   op->inputs.push_back(h->handle);
   op->attrs.NumInputs(op->inputs.size());
@@ -436,10 +431,39 @@ void TFE_OpSetAttrFunctionList(TFE_Op* op, const char* attr_name,
 
 namespace {
 
+// Initializes the step stats if needed.
+void MaybeInitializeStepStats(tensorflow::StepStats* step_stats,
+                              tensorflow::EagerContext* ctx) {
+  // Lazily initialize the RunMetadata with information about all devices if
+  // this is the first call.
+  while (step_stats->dev_stats_size() < ctx->devices()->size()) {
+    int device_idx = step_stats->dev_stats_size();
+    auto* dev_stats = step_stats->add_dev_stats();
+    dev_stats->set_device(ctx->devices()->at(device_idx)->name());
+  }
+}
+
+int StepStatsDeviceIndex(tensorflow::StepStats* step_stats,
+                         tensorflow::EagerContext* ctx,
+                         tensorflow::Device* device) {
+  // Find the current device's index.
+  if (device == nullptr) {
+    device = ctx->HostCPU();
+  }
+  for (int i = 0; i < ctx->devices()->size(); ++i) {
+    if (ctx->devices()->at(i) == device ||
+        ctx->devices()->at(i)->name() == device->name()) {
+      return i;
+    }
+  }
+  // TODO(apassos) do not fall back to host CPU if device is unknown.
+  return 0;
+}
+
 tensorflow::Status ValidateInputTypeAndPlacement(
-    tensorflow::EagerContext* ctx, tensorflow::Device* host_device,
-    tensorflow::Device* op_device, TFE_Op* op,
-    const tensorflow::OpKernel* kernel) {
+    tensorflow::EagerContext* ctx, tensorflow::Device* op_device, TFE_Op* op,
+    const tensorflow::OpKernel* kernel, tensorflow::RunMetadata* run_metadata) {
+  tensorflow::Device* host_device = ctx->HostCPU();
   const tensorflow::MemoryTypeVector& memtypes = kernel->input_memory_types();
   if (memtypes.size() != op->inputs.size()) {
     return tensorflow::errors::InvalidArgument(
@@ -489,9 +513,22 @@ tensorflow::Status ValidateInputTypeAndPlacement(
       }
       // We are only here if the policy is warn or silent copies, so we should
       // trigger a copy.
+      auto pre_time = tensorflow::Env::Default()->NowMicros();
       tensorflow::TensorHandle* copied_tensor = nullptr;
       tensorflow::Status status = tensorflow::EagerCopyToDevice(
           handle, ctx, expected_device->name().c_str(), &copied_tensor);
+      if (run_metadata != nullptr) {
+        auto* step_stats = run_metadata->mutable_step_stats();
+        MaybeInitializeStepStats(step_stats, ctx);
+        // Record the sending on the source device for now.
+        int device_idx = StepStatsDeviceIndex(step_stats, ctx, handle_device);
+        auto* dev_stats = step_stats->mutable_dev_stats(device_idx);
+        auto* node_stats = dev_stats->add_node_stats();
+        node_stats->set_node_name("_Send");
+        node_stats->set_all_start_micros(pre_time);
+        node_stats->set_op_end_rel_micros(
+            tensorflow::Env::Default()->NowMicros() - pre_time);
+      }
       if (!status.ok()) {
         if (copied_tensor != nullptr) copied_tensor->Unref();
         return tensorflow::errors::Internal(
@@ -785,8 +822,12 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     tensorflow::Device* input_op_device = nullptr;
     status->status = op->inputs[i]->OpDevice(&input_op_device);
     if (!status->status.ok()) return;
+    VLOG(2) << "for op " << op->name << " input " << i << " "
+            << tensorflow::DataTypeString(op->inputs[i]->dtype) << " "
+            << (input_op_device == nullptr ? "cpu" : input_op_device->name())
+            << " " << (op->device == nullptr ? "cpu" : op->device->name());
     if (op->inputs[i]->dtype == tensorflow::DT_RESOURCE &&
-        input_op_device != op->device) {
+        (input_op_device != op->device || input_op_device == nullptr)) {
       tensorflow::Device* d =
           input_op_device == nullptr ? ctx->context.HostCPU() : input_op_device;
       VLOG(1) << "Changing device of operation " << op->name << " to "
@@ -796,16 +837,13 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     }
   }
   tensorflow::Device* device = op->device;
-  if (!ctx->context.SoftPlacement() && device == nullptr) {
-    device = ctx->context.HostCPU();
-  }
 
   tensorflow::Fprint128 cache_key =
       op->attrs.CacheKey(device == nullptr ? "unspecified" : device->name());
   tensorflow::KernelAndDevice* kernel = ctx->context.GetCachedKernel(cache_key);
   if (kernel == nullptr) {
     const tensorflow::NodeDef& ndef = op->attrs.BuildNodeDef();
-    if (ctx->context.SoftPlacement() && device == nullptr) {
+    if (device == nullptr) {
       device = SelectDevice(ndef, ctx, status);
       if (!status->status.ok()) {
         return;
@@ -867,7 +905,9 @@ void TFE_Execute(TFE_Op* op, TFE_TensorHandle** retvals, int* num_retvals,
     device = kernel->device();
   }
   status->status = ValidateInputTypeAndPlacement(
-      &ctx->context, ctx->context.HostCPU(), device, op, kernel->kernel());
+      &ctx->context, device, op, kernel->kernel(),
+      ctx->context.ShouldStoreMetadata() ? ctx->context.RunMetadataProto()
+                                         : nullptr);
   if (!status->status.ok()) return;
   std::unique_ptr<tensorflow::NodeExecStats> maybe_stats;
   if (ctx->context.ShouldStoreMetadata()) {
diff --git a/tensorflow/c/eager/c_api.h b/tensorflow/c/eager/c_api.h
index a5029bf211..3926c22ce1 100644
--- a/tensorflow/c/eager/c_api.h
+++ b/tensorflow/c/eager/c_api.h
@@ -61,17 +61,15 @@ TF_CAPI_EXPORT extern void TFE_ContextOptionsSetConfig(
 // Controls how to act when we try to run an operation on a given device but
 // some input tensors are not on that device.
 typedef enum TFE_ContextDevicePlacementPolicy {
-  // Running operations with input tensors on the wrong device will fail. When
-  // soft placement is enabled acts like TFE_DEVICE_PLACEMENT_SILENT.
+  // Running operations with input tensors on the wrong device will fail.
   TFE_DEVICE_PLACEMENT_EXPLICIT = 0,
   // Copy the tensor to the right device but log a warning.
   TFE_DEVICE_PLACEMENT_WARN = 1,
-  // Silently copy the tensor, which has a performance cost since the
-  // operation will be blocked till the copy completes.
+  // Silently copy the tensor, which has a performance cost since the operation
+  // will be blocked till the copy completes. This is the default placement
+  // policy.
   TFE_DEVICE_PLACEMENT_SILENT = 2,
-  // Default placement policy which silently copies int32 tensors but not other
-  // dtypes.  When soft placement is enabled acts like
-  // TFE_DEVICE_PLACEMENT_SILENT.
+  // Placement policy which silently copies int32 tensors but not other dtypes.
   TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
 } TFE_ContextDevicePlacementPolicy;
 
@@ -162,7 +160,11 @@ TF_CAPI_EXPORT extern int64_t TFE_TensorHandleDim(TFE_TensorHandle* h,
 TF_CAPI_EXPORT extern const char* TFE_TensorHandleDeviceName(
     TFE_TensorHandle* h, TF_Status* status);
 
-// This function will block till the operation that produces `h` has completed.
+// This function will block till the operation that produces `h` has
+// completed. The memory returned might alias the internal memory used by
+// TensorFlow. Hence, callers should not mutate this memory (for example by
+// modifying the memory region pointed to by TF_TensorData() on the returned
+// TF_Tensor).
 TF_CAPI_EXPORT extern TF_Tensor* TFE_TensorHandleResolve(TFE_TensorHandle* h,
                                                          TF_Status* status);
 
diff --git a/tensorflow/c/eager/c_api_internal.h b/tensorflow/c/eager/c_api_internal.h
index e6d2ab75ff..05dc64f521 100644
--- a/tensorflow/c/eager/c_api_internal.h
+++ b/tensorflow/c/eager/c_api_internal.h
@@ -50,8 +50,7 @@ struct TFE_ContextOptions {
   TF_SessionOptions session_options;
   // true if async execution is enabled.
   bool async = false;
-  TFE_ContextDevicePlacementPolicy policy{
-      TFE_DEVICE_PLACEMENT_SILENT_FOR_INT32};
+  TFE_ContextDevicePlacementPolicy policy{TFE_DEVICE_PLACEMENT_SILENT};
 };
 
 struct TFE_Context {
@@ -71,7 +70,7 @@ struct TFE_Context {
 struct TFE_TensorHandle {
   TFE_TensorHandle(const tensorflow::Tensor& t, tensorflow::Device* d,
                    tensorflow::Device* op_device)
-      : handle(new tensorflow::TensorHandle(t, d, op_device)) {}
+      : handle(new tensorflow::TensorHandle(t, d, op_device, nullptr)) {}
 
   TFE_TensorHandle(tensorflow::uint64 node_id, tensorflow::DataType dtype,
                    tensorflow::EagerContext* ctx)
diff --git a/tensorflow/c/eager/c_api_test.cc b/tensorflow/c/eager/c_api_test.cc
index d88a6c1dda..701175e494 100644
--- a/tensorflow/c/eager/c_api_test.cc
+++ b/tensorflow/c/eager/c_api_test.cc
@@ -590,7 +590,13 @@ void Execute_MatMul_CPU_Runtime_Error(bool async) {
   TFE_TensorHandle* m1 = TestMatrixTensorHandle();
   TFE_TensorHandle* m2 = TestMatrixTensorHandle3X2();
   TFE_Op* matmul = MatMulOp(ctx, m1, m2);
+  TFE_OpSetDevice(matmul, "/job:localhost/replica:0/task:0/device:CPU:0",
+                  status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_Op* matmul2 = MatMulOp(ctx, m1, m1);
+  TFE_OpSetDevice(matmul2, "/job:localhost/replica:0/task:0/device:CPU:0",
+                  status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_TensorHandle* retvals[1] = {nullptr};
   int num_retvals = 1;
   TFE_Execute(matmul, &retvals[0], &num_retvals, status);
@@ -693,14 +699,14 @@ TEST(CAPI, Execute_Min_CPU) {
   TF_Tensor* t = TFE_TensorHandleResolve(retvals[0], status);
   ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TFE_DeleteTensorHandle(retvals[0]);
-  TFE_DeleteContext(ctx, status);
-  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   float output[2] = {0};
   EXPECT_EQ(sizeof(output), TF_TensorByteSize(t));
   memcpy(&output[0], TF_TensorData(t), TF_TensorByteSize(t));
   TF_DeleteTensor(t);
   EXPECT_EQ(1, output[0]);
   EXPECT_EQ(3, output[1]);
+  TFE_DeleteContext(ctx, status);
+  ASSERT_EQ(TF_OK, TF_GetCode(status)) << TF_Message(status);
   TF_DeleteStatus(status);
 }
 
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 0566329f18..9c47ad6187 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -17,24 +17,11 @@ limitations under the License.
 
 namespace tensorflow {
 
-ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, ContextDevicePlacementPolicy original_policy) {
-  if (!soft_placement) {
-    return original_policy;
-  }
-  if (original_policy == DEVICE_PLACEMENT_EXPLICIT ||
-      original_policy == DEVICE_PLACEMENT_SILENT_FOR_INT32) {
-    return DEVICE_PLACEMENT_SILENT;
-  }
-  return original_policy;
-}
-
 EagerContext::EagerContext(const SessionOptions& opts,
                            ContextDevicePlacementPolicy default_policy,
                            bool async, std::unique_ptr<DeviceMgr> device_mgr,
                            Rendezvous* rendezvous)
-    : soft_placement_(opts.config.allow_soft_placement()),
-      policy_(PlacementPolicy(soft_placement_, default_policy)),
+    : policy_(default_policy),
       device_manager_(std::move(device_mgr)),
       devices_(device_manager_->ListDevices()),
       rendezvous_(rendezvous),
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index bc97219dae..a88fa5eaa4 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -43,23 +43,18 @@ namespace tensorflow {
 
 // Note: there's a copy enum in eager/c_api.h. It should be kept in sync.
 enum ContextDevicePlacementPolicy {
-  // Running operations with input tensors on the wrong device will fail. When
-  // soft placement is enabled acts like TFE_DEVICE_PLACEMENT_SILENT.
+  // Running operations with input tensors on the wrong device will fail.
   DEVICE_PLACEMENT_EXPLICIT = 0,
   // Copy the tensor to the right device but log a warning.
   DEVICE_PLACEMENT_WARN = 1,
-  // Silently copy the tensor, which has a performance cost since the
-  // operation will be blocked till the copy completes.
+  // Silently copy the tensor, which has a performance cost since the operation
+  // will be blocked till the copy completes. This is the default policy.
   DEVICE_PLACEMENT_SILENT = 2,
   // Default placement policy which silently copies int32 tensors but not other
-  // dtypes.  When soft placement is enabled acts like
-  // TFE_DEVICE_PLACEMENT_SILENT.
+  // dtypes.
   DEVICE_PLACEMENT_SILENT_FOR_INT32 = 3,
 };
 
-ContextDevicePlacementPolicy PlacementPolicy(
-    bool soft_placement, ContextDevicePlacementPolicy original_policy);
-
 class EagerContext {
  public:
   explicit EagerContext(const SessionOptions& opts,
@@ -116,8 +111,6 @@ class EagerContext {
 
   Device* HostCPU() { return devices_[0]; }
 
-  bool SoftPlacement() { return soft_placement_; }
-
   uint64 NextId() { return executor_.NextId(); }
 
   void ExecutorAdd(EagerNode* node) { executor_.Add(node); }
@@ -148,7 +141,6 @@ class EagerContext {
   FunctionLibraryDefinition* FuncLibDef() { return &func_lib_def_; }
 
  private:
-  const bool soft_placement_;
   const ContextDevicePlacementPolicy policy_;
 
   // Note: we cannot use C++11 thread_local here as there is no concept of a
diff --git a/tensorflow/core/common_runtime/eager/execute.cc b/tensorflow/core/common_runtime/eager/execute.cc
index 4f16e42568..98e8471102 100644
--- a/tensorflow/core/common_runtime/eager/execute.cc
+++ b/tensorflow/core/common_runtime/eager/execute.cc
@@ -36,10 +36,6 @@ Status EagerExecute(EagerContext* ctx, Device* device,
                     const gtl::InlinedVector<TensorHandle*, 4>& op_inputs,
                     KernelAndDevice* kernel, NodeExecStats* maybe_stats,
                     TensorHandle** retvals, int num_retvals) {
-  if (!ctx->SoftPlacement() && device == nullptr) {
-    device = ctx->HostCPU();
-  }
-
   if (device == nullptr) {
     // TODO(apassos) debug how the assignment below might return a different
     // device from the one requested above.
@@ -100,7 +96,7 @@ Status EagerExecute(EagerContext* ctx, Device* device,
       d = nullptr;
     }
     if (retvals[i] == nullptr) {
-      retvals[i] = new TensorHandle(outputs[i], d, op_device);
+      retvals[i] = new TensorHandle(outputs[i], d, op_device, ctx);
     } else {
       retvals[i]->SetTensorAndDevice(outputs[i], d, op_device);
     }
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.cc b/tensorflow/core/common_runtime/eager/tensor_handle.cc
index 328cd5dd5c..8e11f7b710 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.cc
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.cc
@@ -47,7 +47,7 @@ namespace tensorflow {
 bool TensorHandle::IsReady() {
   if (node_id == 0) return true;
   mutex_lock l(ctx_mutex_);
-  return ctx_ == nullptr;
+  return is_ready_;
 }
 
 Status TensorHandle::WaitReady() {
@@ -55,7 +55,7 @@ Status TensorHandle::WaitReady() {
   EagerExecutor* executor = nullptr;
   {
     mutex_lock l(ctx_mutex_);
-    if (ctx_ == nullptr) return Status::OK();
+    if (is_ready_) return Status::OK();
     executor = ctx_->Executor();
   }
   return executor->WaitFor(node_id);
@@ -97,9 +97,10 @@ void TensorHandle::SetTensorAndDevice(const tensorflow::Tensor& tensor,
                                       tensorflow::Device* device,
                                       tensorflow::Device* op_device) {
   mutex_lock l(ctx_mutex_);
-  DCHECK(node_id > 0 && ctx_) << "SetTensorAndDevice should be only called  "
-                              << "on non-ready handles.";
-  ctx_ = nullptr;
+  DCHECK(node_id > 0 && !is_ready_)
+      << "SetTensorAndDevice should be only called  "
+      << "on non-ready handles.";
+  is_ready_ = true;
   tensor_ = tensor;
   device_ = device;
   op_device_ = op_device;
@@ -122,7 +123,7 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   const bool both_on_cpu = src_cpu && dst_cpu;
   if (is_same_device || both_on_cpu) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(*src, dstd, dstd);
+    *output = new tensorflow::TensorHandle(*src, dstd, dstd, ctx);
     return tensorflow::Status::OK();
   }
   if (!dst_cpu && (src->dtype() != tensorflow::DT_VARIANT &&
@@ -139,7 +140,7 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   tensorflow::Tensor dst(dstd->GetAllocator(attr), src->dtype(), src->shape());
   if (src->shape().num_elements() == 0) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd, ctx);
     return tensorflow::Status::OK();
   }
   tensorflow::DeviceContext* src_device_context = nullptr;
@@ -170,7 +171,7 @@ Status TensorHandle::CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
   n.WaitForNotification();
   if (status.ok()) {
     dstd = dst_cpu ? nullptr : dstd;
-    *output = new tensorflow::TensorHandle(dst, dstd, dstd);
+    *output = new tensorflow::TensorHandle(dst, dstd, dstd, ctx);
   }
   return status;
 }
diff --git a/tensorflow/core/common_runtime/eager/tensor_handle.h b/tensorflow/core/common_runtime/eager/tensor_handle.h
index eb69a13c06..d66c4d95e2 100644
--- a/tensorflow/core/common_runtime/eager/tensor_handle.h
+++ b/tensorflow/core/common_runtime/eager/tensor_handle.h
@@ -49,13 +49,14 @@ namespace tensorflow {
 // (unrelated to python TensorHandle).
 class TensorHandle : public core::RefCounted {
  public:
-  TensorHandle(const Tensor& t, Device* d, Device* op_device)
+  TensorHandle(const Tensor& t, Device* d, Device* op_device, EagerContext* ctx)
       : dtype(t.dtype()),
         node_id(0),
         tensor_(t),
         device_(d),
         op_device_(op_device),
-        ctx_(nullptr) {}
+        ctx_(ctx),
+        is_ready_(true) {}
 
   TensorHandle(uint64 node_id, DataType dtype, EagerContext* ctx)
       : dtype(dtype),
@@ -63,7 +64,8 @@ class TensorHandle : public core::RefCounted {
         tensor_(dtype),
         device_(nullptr),
         op_device_(nullptr),
-        ctx_(ctx) {
+        ctx_(ctx),
+        is_ready_(ctx == nullptr) {
     DCHECK_GT(node_id, 0);
   }
 
@@ -88,6 +90,12 @@ class TensorHandle : public core::RefCounted {
   Status CopyToDevice(EagerContext* ctx, tensorflow::Device* dstd,
                       TensorHandle** output);
 
+  // Warning: can return nullptr for CPU tensors.
+  EagerContext* Context() {
+    mutex_lock ml(ctx_mutex_);
+    return ctx_;
+  }
+
   // dtype for the handle. It must be the same as t.dtype() once the handle is
   // ready.
   const DataType dtype;
@@ -126,6 +134,7 @@ class TensorHandle : public core::RefCounted {
   // typically true when the handle was produced during async execution.
   // `ctx` object is not owned and should outlive this handle.
   EagerContext* ctx_ GUARDED_BY(ctx_mutex_);
+  bool is_ready_ GUARDED_BY(ctx_mutex_);
 };
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/kernels/function_ops.cc b/tensorflow/core/kernels/function_ops.cc
index 351aad7213..f8e0267578 100644
--- a/tensorflow/core/kernels/function_ops.cc
+++ b/tensorflow/core/kernels/function_ops.cc
@@ -144,6 +144,11 @@ TF_CALL_bool(REGISTER) REGISTER_KERNEL_BUILDER(Name(kRetOp)
                                                    .HostMemory("input")
                                                    .TypeConstraint<int32>("T"),
                                                RetvalOp);
+REGISTER_KERNEL_BUILDER(Name(kRetOp)
+                            .Device(DEVICE_GPU)
+                            .TypeConstraint<ResourceHandle>("T")
+                            .HostMemory("input"),
+                        RetvalOp);
 #undef REGISTER
 
 class PassOn : public OpKernel {
diff --git a/tensorflow/python/eager/core_test.py b/tensorflow/python/eager/core_test.py
index 5f19f64846..3fabe7060e 100644
--- a/tensorflow/python/eager/core_test.py
+++ b/tensorflow/python/eager/core_test.py
@@ -116,8 +116,7 @@ class TFETest(test_util.TensorFlowTestCase):
     cpu_stats = step_stats.dev_stats[0]
     self.assertEqual('/job:localhost/replica:0/task:0/device:CPU:0',
                      cpu_stats.device)
-    self.assertEqual(len(cpu_stats.node_stats), 1)
-    self.assertEqual(cpu_stats.node_stats[0].node_name, 'Add')
+    self.assertGreaterEqual(len(cpu_stats.node_stats), 1)
 
   def testShouldCopy(self):
     if not context.context().num_gpus():
@@ -658,10 +657,11 @@ class SendRecvTest(test_util.TensorFlowTestCase):
     with ops.device('GPU:0'):
       t0 = constant_op.constant(1.0)
       self._send(t0, 't0', self.cpu_device)
-    self.assertAllEqual(
-        self._recv(dtypes.float32, 't0', gpu_device_name),
-        1.0)
-    self._send(constant_op.constant(2.0), 't1', gpu_device_name)
+    with ops.device('cpu:0'):
+      self.assertAllEqual(
+          self._recv(dtypes.float32, 't0', gpu_device_name),
+          1.0)
+      self._send(constant_op.constant(2.0), 't1', gpu_device_name)
     with ops.device('GPU:0'):
       self.assertAllEqual(
           self._recv(dtypes.float32, 't1', self.cpu_device),
diff --git a/tensorflow/python/eager/function_test.py b/tensorflow/python/eager/function_test.py
index fd1d2c25ff..9af197981b 100644
--- a/tensorflow/python/eager/function_test.py
+++ b/tensorflow/python/eager/function_test.py
@@ -26,7 +26,6 @@ from tensorflow.python.eager import tape
 from tensorflow.python.eager import test
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import function as tf_function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
@@ -377,23 +376,23 @@ class FunctionTest(test.TestCase):
     self.assertAllEqual(f(constant_op.constant(1.0)), 2.0)
 
   def testGradientOfGatherWithDefun(self):
+    with ops.device('cpu:0'):
+      v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0])
 
-    v = resource_variable_ops.ResourceVariable([0.0, 1.0, 2.0])
+      def sum_gather():
+        return math_ops.reduce_sum(array_ops.gather(v, [1, 2]))
 
-    def sum_gather():
-      return math_ops.reduce_sum(array_ops.gather(v, [1, 2]))
+      grad_fn = backprop.implicit_grad(sum_gather)
+      gradient = grad_fn()
+      defun_grad_fn = backprop.implicit_grad(function.defun(sum_gather))
+      defun_gradient = defun_grad_fn()
+      self.assertEqual(len(gradient), len(defun_gradient))
 
-    grad_fn = backprop.implicit_grad(sum_gather)
-    gradient = grad_fn()
-    defun_grad_fn = backprop.implicit_grad(function.defun(sum_gather))
-    defun_gradient = defun_grad_fn()
-    self.assertEqual(len(gradient), len(defun_gradient))
-
-    gradient = gradient[0][0]
-    defun_gradient = defun_gradient[0][0]
-    self.assertAllEqual(gradient.values, defun_gradient.values)
-    self.assertAllEqual(gradient.indices, defun_gradient.indices)
-    self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape)
+      gradient = gradient[0][0]
+      defun_gradient = defun_gradient[0][0]
+      self.assertAllEqual(gradient.values, defun_gradient.values)
+      self.assertAllEqual(gradient.indices, defun_gradient.indices)
+      self.assertAllEqual(gradient.dense_shape, defun_gradient.dense_shape)
 
   def testReturningIndexedSlicesWithDefun(self):
 
@@ -476,9 +475,7 @@ class FunctionTest(test.TestCase):
     reshape = function.defun(array_ops.reshape)
     value = constant_op.constant([1., 2.])
     shape = constant_op.constant([2, 1]).gpu()
-    with self.assertRaises(errors.InvalidArgumentError):
-      with ops.device('gpu:0'):
-        reshape(value, shape)
+    reshape(value, shape)  # No error is raised
 
   def testDifferentiableFunctionNoneOutputs(self):
 
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index ea210346c1..5eceb9f768 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -96,6 +96,10 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
     ],
     grpc_enabled = True,
+    tags = [
+        "no_gpu",
+        "nogpu",
+    ],
 )
 
 cuda_py_test(
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index 742564f9bf..c31d5a1f91 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -87,6 +87,7 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
     with context.eager_mode():
       handle = resource_variable_ops.var_handle_op(
           dtype=dtypes.int32, shape=[1], name="foo")
+      resource_variable_ops.assign_variable_op(handle, 1)
       with self.assertRaisesRegexp(errors.InvalidArgumentError,
                                    "Trying to read variable with wrong dtype. "
                                    "Expected float got int32."):
-- 
GitLab


From 1d5069d9f01d509ecd42614b056d3df4d4ba74ac Mon Sep 17 00:00:00 2001
From: Chris Tava <chris1tava@gmail.com>
Date: Thu, 29 Mar 2018 16:41:02 -0400
Subject: [PATCH 702/960] Updating install_golang.sh - bumping to 1.10 (#17989)

---
 tensorflow/tools/ci_build/install/install_golang.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/tools/ci_build/install/install_golang.sh b/tensorflow/tools/ci_build/install/install_golang.sh
index e1edd62cc5..124ad82e91 100755
--- a/tensorflow/tools/ci_build/install/install_golang.sh
+++ b/tensorflow/tools/ci_build/install/install_golang.sh
@@ -16,7 +16,7 @@
 
 set -ex
 
-GOLANG_URL="https://storage.googleapis.com/golang/go1.9.2.linux-amd64.tar.gz"
+GOLANG_URL="https://storage.googleapis.com/golang/go1.10.linux-amd64.tar.gz"
 
 sudo mkdir -p /usr/local
 wget -q -O - "${GOLANG_URL}" | sudo tar -C /usr/local -xz
-- 
GitLab


From 26fdbe7e8b3ec7fe799654cb72e849a6bfb3c5bf Mon Sep 17 00:00:00 2001
From: Michael Kuperstein <mkuper@google.com>
Date: Thu, 29 Mar 2018 13:59:36 -0700
Subject: [PATCH 703/960] [XLA] Remove note about what implementations do for
 DynamicSlice and DynamicUpdateSlice.

It is impossible to commit to a particular "implementation-defined behavior" for all implementations.

PiperOrigin-RevId: 190981804
---
 .../docs_src/performance/xla/operation_semantics.md       | 8 ++------
 1 file changed, 2 insertions(+), 6 deletions(-)

diff --git a/tensorflow/docs_src/performance/xla/operation_semantics.md b/tensorflow/docs_src/performance/xla/operation_semantics.md
index 32f249cf10..217ab596b7 100644
--- a/tensorflow/docs_src/performance/xla/operation_semantics.md
+++ b/tensorflow/docs_src/performance/xla/operation_semantics.md
@@ -788,9 +788,7 @@ DynamicSlice extracts a sub-array from the input array at dynamic
 dimension: [start, start + size). The shape of `start_indices` must be rank ==
 1, with dimension size equal to the rank of `operand`.
 Note: handling of out-of-bounds slice indices (generated by incorrect runtime
-calculation of 'start_indices') is currently implementation-defined. Currently,
-slice indices are computed modulo input dimension sizes to prevent out-of-bound
-array accesses, but this behavior may change in future implementations.
+calculation of 'start_indices') is currently implementation-defined.
 
 <b> `DynamicSlice(operand, start_indices, size_indices)` </b>
 
@@ -847,9 +845,7 @@ is updated.
 The shape of `start_indices` must be rank == 1, with dimension size equal to
 the rank of `operand`.
 Note: handling of out-of-bounds slice indices (generated by incorrect runtime
-calculation of 'start_indices') is currently implementation-defined. Currently,
-slice indices are computed modulo update dimension sizes to prevent out-of-bound
-array accesses, but this behavior may change in future implementations.
+calculation of 'start_indices') is currently implementation-defined.
 
 <b> `DynamicUpdateSlice(operand, update, start_indices)` </b>
 
-- 
GitLab


From 1a9663e9e06075c5b5f8984bb95b36f3458edccf Mon Sep 17 00:00:00 2001
From: Younghee Kwon <youngheek@google.com>
Date: Thu, 29 Mar 2018 14:17:16 -0700
Subject: [PATCH 704/960] boosted_trees: post-submit clean up  - non-public
 objects are renamed.  - is_single_machine is set properly when run_config is
 not populated properly (i.e. empty). PiperOrigin-RevId: 190984693

---
 .../estimator/python/estimator/boosted_trees.py      | 12 ++++++------
 tensorflow/python/estimator/canned/boosted_trees.py  | 12 ++++++------
 .../python/estimator/canned/boosted_trees_test.py    |  2 +-
 3 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/estimator/python/estimator/boosted_trees.py b/tensorflow/contrib/estimator/python/estimator/boosted_trees.py
index 5880164519..314c54ed00 100644
--- a/tensorflow/contrib/estimator/python/estimator/boosted_trees.py
+++ b/tensorflow/contrib/estimator/python/estimator/boosted_trees.py
@@ -67,20 +67,20 @@ class _BoostedTreesEstimator(estimator.Estimator):
       tree_complexity: regularization factor to penalize trees with more leaves.
       config: `RunConfig` object to configure the runtime settings.
     """
-    # TODO(youngheek): param validations.
-
+    # pylint:disable=protected-access
     # HParams for the model.
-    tree_hparams = canned_boosted_trees.TreeHParams(
+    tree_hparams = canned_boosted_trees._TreeHParams(
         n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
         tree_complexity)
 
     def _model_fn(features, labels, mode, config):
-      return canned_boosted_trees._bt_model_fn(  # pylint: disable=protected-access
+      return canned_boosted_trees._bt_model_fn(
           features, labels, mode, head, feature_columns, tree_hparams,
           n_batches_per_layer, config)
 
     super(_BoostedTreesEstimator, self).__init__(
         model_fn=_model_fn, model_dir=model_dir, config=config)
+    # pylint:enable=protected-access
 
 
 def boosted_trees_classifier_train_in_memory(
@@ -182,7 +182,7 @@ def boosted_trees_classifier_train_in_memory(
           n_classes, weight_column, label_vocabulary=label_vocabulary))
 
   # HParams for the model.
-  tree_hparams = canned_boosted_trees.TreeHParams(
+  tree_hparams = canned_boosted_trees._TreeHParams(
       n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
       tree_complexity)
 
@@ -298,7 +298,7 @@ def boosted_trees_regressor_train_in_memory(
                                                       weight_column)
 
   # HParams for the model.
-  tree_hparams = canned_boosted_trees.TreeHParams(
+  tree_hparams = canned_boosted_trees._TreeHParams(
       n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
       tree_complexity)
 
diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py
index a9bbabd598..7f1bcc31f2 100644
--- a/tensorflow/python/estimator/canned/boosted_trees.py
+++ b/tensorflow/python/estimator/canned/boosted_trees.py
@@ -40,7 +40,7 @@ from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
 
-TreeHParams = collections.namedtuple(
+_TreeHParams = collections.namedtuple(
     'TreeHParams',
     ['n_trees', 'max_depth', 'learning_rate', 'l1', 'l2', 'tree_complexity'])
 
@@ -259,8 +259,8 @@ def _bt_model_fn(
     example_id_column_name=None,
     # TODO(youngheek): replace this later using other options.
     train_in_memory=False,
-    name='TreeEnsembleModel'):
-  """Gradient Boosted Decision Tree model_fn.
+    name='boosted_trees'):
+  """Gradient Boosted Trees model_fn.
 
   Args:
     features: dict of `Tensor`.
@@ -290,7 +290,7 @@ def _bt_model_fn(
   Raises:
     ValueError: mode or params are invalid, or features has the wrong type.
   """
-  is_single_machine = (config.num_worker_replicas == 1)
+  is_single_machine = (config.num_worker_replicas <= 1)
   if train_in_memory:
     assert n_batches_per_layer == 1, (
         'When train_in_memory is enabled, input_fn should return the entire '
@@ -617,7 +617,7 @@ class BoostedTreesClassifier(estimator.Estimator):
         n_classes, weight_column, label_vocabulary=label_vocabulary)
 
     # HParams for the model.
-    tree_hparams = TreeHParams(
+    tree_hparams = _TreeHParams(
         n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
         tree_complexity)
 
@@ -723,7 +723,7 @@ class BoostedTreesRegressor(estimator.Estimator):
     head = _create_regression_head(label_dimension, weight_column)
 
     # HParams for the model.
-    tree_hparams = TreeHParams(
+    tree_hparams = _TreeHParams(
         n_trees, max_depth, learning_rate, l1_regularization, l2_regularization,
         tree_complexity)
 
diff --git a/tensorflow/python/estimator/canned/boosted_trees_test.py b/tensorflow/python/estimator/canned/boosted_trees_test.py
index 9276fbaaa1..01e5cc7a5d 100644
--- a/tensorflow/python/estimator/canned/boosted_trees_test.py
+++ b/tensorflow/python/estimator/canned/boosted_trees_test.py
@@ -195,7 +195,7 @@ class ModelFnTests(test_util.TensorFlowTestCase):
             feature_column.numeric_column('f_%d' % i, dtype=dtypes.float32),
             BUCKET_BOUNDARIES) for i in range(NUM_FEATURES)
     }
-    self._tree_hparams = boosted_trees.TreeHParams(
+    self._tree_hparams = boosted_trees._TreeHParams(  # pylint:disable=protected-access
         n_trees=2,
         max_depth=2,
         learning_rate=0.1,
-- 
GitLab


From 489389822636b1229c2e92b717c3e947ccfa23b4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 14:25:10 -0700
Subject: [PATCH 705/960] LSTM support: Add non-uint8 quantized elementwise
 unary operators.

PiperOrigin-RevId: 190986046
---
 .../internal/optimized/optimized_ops.h        | 154 +++++++++++++++---
 1 file changed, 128 insertions(+), 26 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
index 4661004d09..3642da311c 100644
--- a/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
+++ b/tensorflow/contrib/lite/kernels/internal/optimized/optimized_ops.h
@@ -4092,12 +4092,46 @@ inline void Logistic(const uint8* input_data, const Dims<4>& input_dims,
 inline void Logistic(const int16* input_data, const Dims<4>& input_dims,
                      int16* output_data, const Dims<4>& output_dims) {
   gemmlowp::ScopedProfilingLabel label("Logistic/Int16");
-  // This is a copy of the reference implementation. We do not currently have a
-  // properly optimized version.
   const int flat_size = RequiredBufferSizeForDims(output_dims);
   TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input_dims), flat_size);
 
   for (int i = 0; i < flat_size; i++) {
+  }
+
+  int c = 0;
+  const int16* input_data_ptr = input_data;
+  int16* output_data_ptr = output_data;
+#ifdef GEMMLOWP_NEON
+  {
+    // F0 uses 0 integer bits, range [-1, 1].
+    // This is the return type of math functions such as tanh, logistic,
+    // whose range is in [-1, 1].
+    using F0 = gemmlowp::FixedPoint<int16x8_t, 0>;
+    // F3 uses 3 integer bits, range [-8, 8], the input range expected here.
+    using F3 = gemmlowp::FixedPoint<int16x8_t, 3>;
+
+    for (; c <= flat_size - 16; c += 16) {
+      F3 input0 = F3::FromRaw(vld1q_s16(input_data_ptr));
+      F3 input1 = F3::FromRaw(vld1q_s16(input_data_ptr + 8));
+      F0 output0 = gemmlowp::logistic(input0);
+      F0 output1 = gemmlowp::logistic(input1);
+      vst1q_s16(output_data_ptr, output0.raw());
+      vst1q_s16(output_data_ptr + 8, output1.raw());
+
+      input_data_ptr += 16;
+      output_data_ptr += 16;
+    }
+    for (; c <= flat_size - 8; c += 8) {
+      F3 input = F3::FromRaw(vld1q_s16(input_data_ptr));
+      F0 output = gemmlowp::logistic(input);
+      vst1q_s16(output_data_ptr, output.raw());
+
+      input_data_ptr += 8;
+      output_data_ptr += 8;
+    }
+  }
+#endif
+  {
     // F0 uses 0 integer bits, range [-1, 1].
     // This is the return type of math functions such as tanh, logistic,
     // whose range is in [-1, 1].
@@ -4105,9 +4139,14 @@ inline void Logistic(const int16* input_data, const Dims<4>& input_dims,
     // F3 uses 3 integer bits, range [-8, 8], the input range expected here.
     using F3 = gemmlowp::FixedPoint<std::int16_t, 3>;
 
-    const F3 input = F3::FromRaw(input_data[i]);
-    F0 output = gemmlowp::logistic(input);
-    output_data[i] = output.raw();
+    for (; c < flat_size; ++c) {
+      F3 input = F3::FromRaw(*input_data_ptr);
+      F0 output = gemmlowp::logistic(input);
+      *output_data_ptr = output.raw();
+
+      ++input_data_ptr;
+      ++output_data_ptr;
+    }
   }
 }
 
@@ -4274,9 +4313,6 @@ inline void Tanh(const int16* input_data, const Dims<4>& input_dims,
                  int input_left_shift, int16* output_data,
                  const Dims<4>& output_dims) {
   gemmlowp::ScopedProfilingLabel label("Tanh/Int16");
-  // This is a copy of the reference implementation. We do not currently have a
-  // properly optimized version.
-
   // Support for shifts is limited until we have a parameterized version of
   // SaturatingRoundingMultiplyByPOT().
   TFLITE_DCHECK_GE(input_left_shift, 0);
@@ -4285,25 +4321,91 @@ inline void Tanh(const int16* input_data, const Dims<4>& input_dims,
   const int flat_size = RequiredBufferSizeForDims(output_dims);
   TFLITE_DCHECK_EQ(RequiredBufferSizeForDims(input_dims), flat_size);
 
-  // F0 uses 0 integer bits, range [-1, 1].
-  // This is the return type of math functions such as tanh, logistic,
-  // whose range is in [-1, 1].
-  using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
-  // F3 uses 3 integer bits, range [-8, 8], the input range expected here.
-  using F3 = gemmlowp::FixedPoint<std::int16_t, 3>;
-
-  if (input_left_shift == 0) {
-    for (int i = 0; i < flat_size; i++) {
-      F3 input = F3::FromRaw(input_data[i]);
-      F0 output = gemmlowp::tanh(input);
-      output_data[i] = output.raw();
+  int c = 0;
+  const int16* input_data_ptr = input_data;
+  int16* output_data_ptr = output_data;
+#ifdef GEMMLOWP_NEON
+  {
+    // F0 uses 0 integer bits, range [-1, 1].
+    // This is the return type of math functions such as tanh, logistic,
+    // whose range is in [-1, 1].
+    using F0 = gemmlowp::FixedPoint<int16x8_t, 0>;
+    // F3 uses 3 integer bits, range [-8, 8], the input range expected here.
+    using F3 = gemmlowp::FixedPoint<int16x8_t, 3>;
+
+    if (input_left_shift == 0) {
+      for (; c <= flat_size - 16; c += 16) {
+        F3 input0 = F3::FromRaw(vld1q_s16(input_data_ptr));
+        F3 input1 = F3::FromRaw(vld1q_s16(input_data_ptr + 8));
+        F0 output0 = gemmlowp::tanh(input0);
+        F0 output1 = gemmlowp::tanh(input1);
+        vst1q_s16(output_data_ptr, output0.raw());
+        vst1q_s16(output_data_ptr + 8, output1.raw());
+
+        input_data_ptr += 16;
+        output_data_ptr += 16;
+      }
+      for (; c <= flat_size - 8; c += 8) {
+        F3 input = F3::FromRaw(vld1q_s16(input_data_ptr));
+        F0 output = gemmlowp::tanh(input);
+        vst1q_s16(output_data_ptr, output.raw());
+
+        input_data_ptr += 8;
+        output_data_ptr += 8;
+      }
+    } else {
+      for (; c <= flat_size - 16; c += 16) {
+        F3 input0 = F3::FromRaw(gemmlowp::SaturatingRoundingMultiplyByPOT<1>(
+            vld1q_s16(input_data_ptr)));
+        F3 input1 = F3::FromRaw(gemmlowp::SaturatingRoundingMultiplyByPOT<1>(
+            vld1q_s16(input_data_ptr + 8)));
+        F0 output0 = gemmlowp::tanh(input0);
+        F0 output1 = gemmlowp::tanh(input1);
+        vst1q_s16(output_data_ptr, output0.raw());
+        vst1q_s16(output_data_ptr + 8, output1.raw());
+
+        input_data_ptr += 16;
+        output_data_ptr += 16;
+      }
+      for (; c <= flat_size - 8; c += 8) {
+        F3 input = F3::FromRaw(gemmlowp::SaturatingRoundingMultiplyByPOT<1>(
+            vld1q_s16(input_data_ptr)));
+        F0 output = gemmlowp::tanh(input);
+        vst1q_s16(output_data_ptr, output.raw());
+
+        input_data_ptr += 8;
+        output_data_ptr += 8;
+      }
     }
-  } else {
-    for (int i = 0; i < flat_size; i++) {
-      F3 input = F3::FromRaw(
-          gemmlowp::SaturatingRoundingMultiplyByPOT<1>(input_data[i]));
-      F0 output = gemmlowp::tanh(input);
-      output_data[i] = output.raw();
+  }
+#endif
+  {
+    // F0 uses 0 integer bits, range [-1, 1].
+    // This is the return type of math functions such as tanh, logistic,
+    // whose range is in [-1, 1].
+    using F0 = gemmlowp::FixedPoint<std::int16_t, 0>;
+    // F3 uses 3 integer bits, range [-8, 8], the input range expected here.
+    using F3 = gemmlowp::FixedPoint<std::int16_t, 3>;
+
+    if (input_left_shift == 0) {
+      for (; c < flat_size; ++c) {
+        F3 input = F3::FromRaw(*input_data_ptr);
+        F0 output = gemmlowp::tanh(input);
+        *output_data_ptr = output.raw();
+
+        ++input_data_ptr;
+        ++output_data_ptr;
+      }
+    } else {
+      for (; c < flat_size; ++c) {
+        F3 input = F3::FromRaw(
+            gemmlowp::SaturatingRoundingMultiplyByPOT<1>(*input_data_ptr));
+        F0 output = gemmlowp::tanh(input);
+        *output_data_ptr = output.raw();
+
+        ++input_data_ptr;
+        ++output_data_ptr;
+      }
     }
   }
 }
-- 
GitLab


From 79e4a49f7bb458176cbfa5ba1e492b39dada023d Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Thu, 29 Mar 2018 15:05:31 -0700
Subject: [PATCH 706/960] TFLite logs to stderr.

PiperOrigin-RevId: 190992629
---
 tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
index bd49d327c9..85aca36874 100644
--- a/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
+++ b/tensorflow/contrib/lite/nnapi/NeuralNetworksShim.h
@@ -22,7 +22,7 @@ limitations under the License.
 
 // helpers
 
-#define NNAPI_LOG(format, ...) printf(format "\n", __VA_ARGS__);
+#define NNAPI_LOG(format, ...) fprintf(stderr, format "\n", __VA_ARGS__);
 #define LOAD_FUNCTION(name) \
   static name##_fn fn = reinterpret_cast<name##_fn>(loadFunction(#name));
 #define EXECUTE_FUNCTION(...) \
-- 
GitLab


From 4b8f6dc1efec882c3fb0e2c8fc3de74586c800ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 15:18:59 -0700
Subject: [PATCH 707/960] [XLA] Remove some dead code from Executable.

PiperOrigin-RevId: 190994733
---
 tensorflow/compiler/xla/service/BUILD                    | 1 -
 tensorflow/compiler/xla/service/cpu/cpu_executable.h     | 5 -----
 .../compiler/xla/service/cpu/parallel_cpu_executable.h   | 6 ------
 tensorflow/compiler/xla/service/executable.h             | 9 ---------
 tensorflow/compiler/xla/service/gpu/gpu_executable.h     | 5 -----
 5 files changed, 26 deletions(-)

diff --git a/tensorflow/compiler/xla/service/BUILD b/tensorflow/compiler/xla/service/BUILD
index b7d1bf64d0..3a99d84bea 100644
--- a/tensorflow/compiler/xla/service/BUILD
+++ b/tensorflow/compiler/xla/service/BUILD
@@ -730,7 +730,6 @@ cc_library(
         ":computation_layout",
         ":device_memory_allocator",
         ":hlo",
-        ":hlo_cost_analysis",
         ":hlo_execution_profile",
         ":hlo_graph_dumper",
         ":pool",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_executable.h b/tensorflow/compiler/xla/service/cpu/cpu_executable.h
index 267b89a10b..d3502b3a03 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_executable.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_executable.h
@@ -71,11 +71,6 @@ class CpuExecutable : public Executable {
     ir_module_string_ = ir_module_string;
   }
 
-  const Status EqualOrFail(const Executable& executable) {
-    // TODO(b/62952745) Implement equality test on CPU executable.
-    return Unimplemented("Equality test on CPU executable is not implemented.");
-  }
-
   static int64 ShapeSizeBytes(const Shape& shape);
 
   // Type of the computation function we expect in the JIT.
diff --git a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h
index c393e9b8ea..87c0a3df45 100644
--- a/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h
+++ b/tensorflow/compiler/xla/service/cpu/parallel_cpu_executable.h
@@ -83,12 +83,6 @@ class ParallelCpuExecutable : public Executable {
     return ShapeUtil::ByteSizeOf(shape, sizeof(void*));
   }
 
-  const Status EqualOrFail(const Executable& executable) {
-    // TODO(b/62952745) Implement equality test on CPU parallel executable.
-    return Unimplemented(
-        "Equality test on CPU parallel executable is not implemented.");
-  }
-
  private:
   // Allocate buffers required for execution and assign them to the elements of
   // "buffers". "buffers" should be sized to the number of buffers in buffer
diff --git a/tensorflow/compiler/xla/service/executable.h b/tensorflow/compiler/xla/service/executable.h
index 0aee535ee7..a157235f8a 100644
--- a/tensorflow/compiler/xla/service/executable.h
+++ b/tensorflow/compiler/xla/service/executable.h
@@ -22,7 +22,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
 #include "tensorflow/compiler/xla/service/computation_layout.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
-#include "tensorflow/compiler/xla/service/hlo_cost_analysis.h"
 #include "tensorflow/compiler/xla/service/hlo_execution_profile.h"
 #include "tensorflow/compiler/xla/service/hlo_graph_dumper.h"
 #include "tensorflow/compiler/xla/service/hlo_module.h"
@@ -109,14 +108,6 @@ class Executable {
     return execution_profile_;
   }
 
-  // Returns Status::ok() if the two executables are equal to each other.
-  //
-  // An error status is returned otherwise.
-  virtual const Status EqualOrFail(const Executable& executable) {
-    return Unimplemented(
-        "Equality test on this executable is not implemented.");
-  }
-
   const HloProfilePrinterData& hlo_profile_printer_data() const {
     CHECK(hlo_profiling_enabled());
     return *hlo_profile_printer_data_;
diff --git a/tensorflow/compiler/xla/service/gpu/gpu_executable.h b/tensorflow/compiler/xla/service/gpu/gpu_executable.h
index b19cfd43de..dcb3991f41 100644
--- a/tensorflow/compiler/xla/service/gpu/gpu_executable.h
+++ b/tensorflow/compiler/xla/service/gpu/gpu_executable.h
@@ -83,11 +83,6 @@ class GpuExecutable : public Executable {
       const ServiceExecutableRunOptions* run_options,
       tensorflow::gtl::ArraySlice<const ShapedBuffer*> arguments) override;
 
-  const Status EqualOrFail(const Executable& executable) {
-    // TODO(b/62952745) Implement equality test on GPU executable.
-    return Unimplemented("Equality test on GPU executable is not implemented.");
-  }
-
  private:
   // If `block_host_until_done` is false, execution will not block the host
   // until the kernels have completed. This is used as an optimization for
-- 
GitLab


From 40f8291db5c0b05b31d7bbe23b847cdbb2408718 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 29 Mar 2018 15:20:38 -0700
Subject: [PATCH 708/960] Internal change.

PiperOrigin-RevId: 190995029
---
 tensorflow/contrib/boosted_trees/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/boosted_trees/BUILD b/tensorflow/contrib/boosted_trees/BUILD
index ddeda0079c..8eac1243ef 100644
--- a/tensorflow/contrib/boosted_trees/BUILD
+++ b/tensorflow/contrib/boosted_trees/BUILD
@@ -119,7 +119,7 @@ py_library(
 
 py_test(
     name = "gbdt_batch_test",
-    size = "small",
+    size = "medium",
     srcs = ["python/training/functions/gbdt_batch_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-- 
GitLab


From 6f5d7a97cd2c0741ddfa756853ce5321377b5d53 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 15:28:24 -0700
Subject: [PATCH 709/960] Add tf.contrib.distribute, which defines classes
 DistributionStrategy and MirroredStrategy, and related functionality.

Also add tf.contrib.optimizer_v2, an update to the Optimizer API.

RELNOTES: Can now pass tf.contrib.distribute.MirroredStrategy() to
tf.estimator.RunConfig() to run an Estimator model on multiple GPUs
on one machine.
PiperOrigin-RevId: 190996247
---
 tensorflow/contrib/BUILD                      |    2 +
 tensorflow/contrib/__init__.py                |    2 +
 tensorflow/contrib/cmake/python_modules.txt   |    3 +
 tensorflow/contrib/distribute/BUILD           |   36 +
 tensorflow/contrib/distribute/__init__.py     |   52 +
 tensorflow/contrib/distribute/python/BUILD    |  431 ++++++
 .../contrib/distribute/python/combinations.py |  293 ++++
 .../distribute/python/combinations_test.py    |  115 ++
 .../distribute/python/cross_tower_ops.py      |  410 +++++
 .../distribute/python/cross_tower_ops_test.py |  185 +++
 .../distribute/python/cross_tower_utils.py    |  153 ++
 .../distribute/python/minimize_loss_test.py   |  279 ++++
 .../distribute/python/mirrored_strategy.py    |  486 ++++++
 .../python/mirrored_strategy_multigpu_test.py |  435 ++++++
 .../python/mirrored_strategy_test.py          |   91 ++
 .../contrib/distribute/python/monitor.py      |   61 +
 .../contrib/distribute/python/monitor_test.py |   84 +
 .../distribute/python/one_device_strategy.py  |  148 ++
 .../python/one_device_strategy_test.py        |   54 +
 .../distribute/python/optimizer_v2_test.py    |   70 +
 .../distribute/python/prefetching_ops_v2.py   |  167 ++
 .../python/prefetching_ops_v2_test.py         |   68 +
 .../python/shared_variable_creator.py         |   97 ++
 .../python/shared_variable_creator_test.py    |   75 +
 .../python/simple_estimator_example.py        |   97 ++
 .../distribute/python/single_loss_example.py  |  102 ++
 .../contrib/distribute/python/step_fn.py      |  103 ++
 .../contrib/distribute/python/step_fn_test.py |   62 +
 .../distribute/python/strategy_test_lib.py    |  225 +++
 .../contrib/distribute/python/values.py       |  575 +++++++
 .../contrib/distribute/python/values_test.py  |  807 ++++++++++
 tensorflow/contrib/optimizer_v2/BUILD         |  205 +++
 tensorflow/contrib/optimizer_v2/adadelta.py   |  113 ++
 .../contrib/optimizer_v2/adadelta_test.py     |  167 ++
 tensorflow/contrib/optimizer_v2/adagrad.py    |  118 ++
 .../contrib/optimizer_v2/adagrad_test.py      |  282 ++++
 tensorflow/contrib/optimizer_v2/adam.py       |  202 +++
 tensorflow/contrib/optimizer_v2/adam_test.py  |  333 ++++
 .../optimizer_v2/checkpointable_utils_test.py |  686 +++++++++
 .../contrib/optimizer_v2/gradient_descent.py  |   69 +
 .../optimizer_v2/gradient_descent_test.py     |  223 +++
 tensorflow/contrib/optimizer_v2/momentum.py   |  124 ++
 .../contrib/optimizer_v2/momentum_test.py     |  562 +++++++
 .../contrib/optimizer_v2/optimizer_v2.py      | 1352 +++++++++++++++++
 .../optimizer_v2/optimizer_v2_symbols.py      |   42 +
 .../contrib/optimizer_v2/optimizer_v2_test.py |  294 ++++
 tensorflow/contrib/optimizer_v2/rmsprop.py    |  233 +++
 .../contrib/optimizer_v2/rmsprop_test.py      |  449 ++++++
 tensorflow/python/training/distribute.py      |    7 +-
 tensorflow/tools/docs/generate_lib.py         |    1 -
 50 files changed, 11226 insertions(+), 4 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/BUILD
 create mode 100644 tensorflow/contrib/distribute/__init__.py
 create mode 100644 tensorflow/contrib/distribute/python/BUILD
 create mode 100644 tensorflow/contrib/distribute/python/combinations.py
 create mode 100644 tensorflow/contrib/distribute/python/combinations_test.py
 create mode 100644 tensorflow/contrib/distribute/python/cross_tower_ops.py
 create mode 100644 tensorflow/contrib/distribute/python/cross_tower_ops_test.py
 create mode 100644 tensorflow/contrib/distribute/python/cross_tower_utils.py
 create mode 100644 tensorflow/contrib/distribute/python/minimize_loss_test.py
 create mode 100644 tensorflow/contrib/distribute/python/mirrored_strategy.py
 create mode 100644 tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
 create mode 100644 tensorflow/contrib/distribute/python/mirrored_strategy_test.py
 create mode 100644 tensorflow/contrib/distribute/python/monitor.py
 create mode 100644 tensorflow/contrib/distribute/python/monitor_test.py
 create mode 100644 tensorflow/contrib/distribute/python/one_device_strategy.py
 create mode 100644 tensorflow/contrib/distribute/python/one_device_strategy_test.py
 create mode 100644 tensorflow/contrib/distribute/python/optimizer_v2_test.py
 create mode 100644 tensorflow/contrib/distribute/python/prefetching_ops_v2.py
 create mode 100644 tensorflow/contrib/distribute/python/prefetching_ops_v2_test.py
 create mode 100644 tensorflow/contrib/distribute/python/shared_variable_creator.py
 create mode 100644 tensorflow/contrib/distribute/python/shared_variable_creator_test.py
 create mode 100644 tensorflow/contrib/distribute/python/simple_estimator_example.py
 create mode 100644 tensorflow/contrib/distribute/python/single_loss_example.py
 create mode 100644 tensorflow/contrib/distribute/python/step_fn.py
 create mode 100644 tensorflow/contrib/distribute/python/step_fn_test.py
 create mode 100644 tensorflow/contrib/distribute/python/strategy_test_lib.py
 create mode 100644 tensorflow/contrib/distribute/python/values.py
 create mode 100644 tensorflow/contrib/distribute/python/values_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/BUILD
 create mode 100644 tensorflow/contrib/optimizer_v2/adadelta.py
 create mode 100644 tensorflow/contrib/optimizer_v2/adadelta_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/adagrad.py
 create mode 100644 tensorflow/contrib/optimizer_v2/adagrad_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/adam.py
 create mode 100644 tensorflow/contrib/optimizer_v2/adam_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/gradient_descent.py
 create mode 100644 tensorflow/contrib/optimizer_v2/gradient_descent_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/momentum.py
 create mode 100644 tensorflow/contrib/optimizer_v2/momentum_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/optimizer_v2.py
 create mode 100644 tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py
 create mode 100644 tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
 create mode 100644 tensorflow/contrib/optimizer_v2/rmsprop.py
 create mode 100644 tensorflow/contrib/optimizer_v2/rmsprop_test.py

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index c211ad8b9b..0cebb49afb 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -33,6 +33,7 @@ py_library(
         "//tensorflow/contrib/crf:crf_py",
         "//tensorflow/contrib/cudnn_rnn:cudnn_rnn_py",
         "//tensorflow/contrib/data",
+        "//tensorflow/contrib/distribute:distribute",
         "//tensorflow/contrib/deprecated:deprecated_py",
         "//tensorflow/contrib/distributions:distributions_py",
         "//tensorflow/contrib/eager/python:tfe",
@@ -74,6 +75,7 @@ py_library(
         "//tensorflow/contrib/nearest_neighbor:nearest_neighbor_py",
         "//tensorflow/contrib/nn:nn_py",
         "//tensorflow/contrib/opt:opt_py",
+        "//tensorflow/contrib/optimizer_v2:optimizer_v2_py",
         "//tensorflow/contrib/periodic_resample:init_py",
         "//tensorflow/contrib/predictor",
         "//tensorflow/contrib/quantization:quantization_py",
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index 4f6f539027..a8e05df708 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -30,6 +30,7 @@ from tensorflow.contrib import crf
 from tensorflow.contrib import cudnn_rnn
 from tensorflow.contrib import data
 from tensorflow.contrib import deprecated
+from tensorflow.contrib import distribute
 from tensorflow.contrib import distributions
 from tensorflow.contrib import estimator
 from tensorflow.contrib import factorization
@@ -84,6 +85,7 @@ from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
 from tensorflow.contrib.lite.python import lite
+from tensorflow.contrib.optimizer_v2 import optimizer_v2_symbols as optimizer_v2
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
 from tensorflow.contrib.specs import python as specs
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index cc7d791042..b10538d6d6 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -160,6 +160,8 @@ tensorflow/contrib/data/python/ops
 tensorflow/contrib/decision_trees
 tensorflow/contrib/decision_trees/proto
 tensorflow/contrib/deprecated
+tensorflow/contrib/distribute
+tensorflow/contrib/distribute/python
 tensorflow/contrib/distributions
 tensorflow/contrib/distributions/python
 tensorflow/contrib/distributions/python/ops
@@ -342,6 +344,7 @@ tensorflow/contrib/nn/python/ops
 tensorflow/contrib/opt
 tensorflow/contrib/opt/python
 tensorflow/contrib/opt/python/training
+tensorflow/contrib/optimizer_v2
 tensorflow/contrib/pi_examples
 tensorflow/contrib/pi_examples/camera
 tensorflow/contrib/pi_examples/label_image
diff --git a/tensorflow/contrib/distribute/BUILD b/tensorflow/contrib/distribute/BUILD
new file mode 100644
index 0000000000..74b2cd90a1
--- /dev/null
+++ b/tensorflow/contrib/distribute/BUILD
@@ -0,0 +1,36 @@
+# Implementation of a prototype TF distributed computation library.
+
+package(
+    default_visibility = ["//visibility:public"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+py_library(
+    name = "distribute",
+    srcs = ["__init__.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/contrib/distribute/python:cross_tower_ops",
+        "//tensorflow/contrib/distribute/python:mirrored_strategy",
+        "//tensorflow/contrib/distribute/python:monitor",
+        "//tensorflow/contrib/distribute/python:one_device_strategy",
+        "//tensorflow/contrib/distribute/python:step_fn",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/__init__.py b/tensorflow/contrib/distribute/__init__.py
new file mode 100644
index 0000000000..76711baf3a
--- /dev/null
+++ b/tensorflow/contrib/distribute/__init__.py
@@ -0,0 +1,52 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Prototype of a distributed computation library for TF."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import,wildcard-import
+from tensorflow.contrib.distribute.python.cross_tower_ops import *
+from tensorflow.contrib.distribute.python.mirrored_strategy import MirroredStrategy
+from tensorflow.contrib.distribute.python.monitor import Monitor
+from tensorflow.contrib.distribute.python.one_device_strategy import OneDeviceStrategy
+from tensorflow.contrib.distribute.python.step_fn import *
+from tensorflow.python.training.distribute import *
+
+from tensorflow.python.util.all_util import remove_undocumented
+
+
+_allowed_symbols = [
+    'AllReduceCrossTowerOps',
+    'CrossTowerOps',
+    'DistributionStrategy',
+    'MirroredStrategy',
+    'Monitor',
+    'OneDeviceStrategy',
+    'ReductionToOneDeviceCrossTowerOps',
+    'Step',
+    'StandardInputStep',
+    'StandardSingleLossStep',
+    'TowerContext',
+    'get_cross_tower_context',
+    'get_distribution_strategy',
+    'get_loss_reduction',
+    'get_tower_context',
+    'has_distribution_strategy',
+    'require_tower_context',
+]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
new file mode 100644
index 0000000000..4dfd3f7228
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -0,0 +1,431 @@
+# Implementation of a prototype TF distributed computation library.
+
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+# TODO(priyag): Figure out testonly issues that are preventing us from
+# including our tests in pip for now.
+
+py_library(
+    name = "values",
+    srcs = ["values.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":prefetching_ops_v2",
+        "//tensorflow/contrib/data/python/ops:transformation_ops",
+        "//tensorflow/contrib/eager/python:datasets",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:checkpointable",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/eager:context",
+        "@six_archive//:six",
+    ],
+)
+
+cuda_py_test(
+    name = "values_test",
+    srcs = ["values_test.py"],
+    additional_deps = [
+        ":mirrored_strategy",
+        ":values",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python:errors",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/estimator:model_fn",
+    ],
+)
+
+py_library(
+    name = "mirrored_strategy",
+    srcs = ["mirrored_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":cross_tower_ops",
+        ":shared_variable_creator",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:device",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:pywrap_tensorflow",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:tape",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "one_device_strategy",
+    srcs = ["one_device_strategy.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        ":values",
+        "//tensorflow/contrib/eager/python:datasets",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python/eager:context",
+        "@six_archive//:six",
+    ],
+)
+
+py_library(
+    name = "strategy_test_lib",
+    testonly = 1,
+    srcs = ["strategy_test_lib.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:backprop",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_library(
+    name = "combinations",
+    testonly = 1,
+    srcs = ["combinations.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        ":mirrored_strategy",
+        ":one_device_strategy",
+        "//tensorflow/contrib/optimizer_v2:training",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:util",
+        "//tensorflow/python/eager:context",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
+py_test(
+    name = "combinations_test",
+    srcs = ["combinations_test.py"],
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        ":combinations",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_test(
+    name = "mirrored_strategy_test",
+    srcs = ["mirrored_strategy_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        ":mirrored_strategy",
+        ":strategy_test_lib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_test(
+    name = "one_device_strategy_test",
+    srcs = ["one_device_strategy_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        ":one_device_strategy",
+        ":strategy_test_lib",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+cuda_py_test(
+    name = "mirrored_strategy_multigpu_test",
+    srcs = ["mirrored_strategy_multigpu_test.py"],
+    additional_deps = [
+        ":mirrored_strategy",
+        ":values",
+        ":strategy_test_lib",
+        "//tensorflow/core:protos_all_py",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+    tags = [
+        "guitar",
+        "no_pip",
+        "multi_and_single_gpu",
+        # Do not perform the extra analysis on this test, because it is already
+        # performed for the `:mirrored_strategy_test` target.
+        "no_oss",
+        "noasan",
+        "notap",
+        "notsan",
+    ],
+)
+
+py_library(
+    name = "step_fn",
+    srcs = ["step_fn.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/python:training",
+        "//tensorflow/python/eager:backprop",
+    ],
+)
+
+cuda_py_test(
+    name = "minimize_loss_test",
+    srcs = ["minimize_loss_test.py"],
+    additional_deps = [
+        ":combinations",
+        ":single_loss_example",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/ops/losses",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
+cuda_py_test(
+    name = "optimizer_v2_test",
+    srcs = ["optimizer_v2_test.py"],
+    additional_deps = [
+        ":combinations",
+        ":single_loss_example",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
+py_library(
+    name = "single_loss_example",
+    srcs = ["single_loss_example.py"],
+    deps = [
+        ":step_fn",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+    ],
+)
+
+cuda_py_test(
+    name = "step_fn_test",
+    srcs = ["step_fn_test.py"],
+    additional_deps = [
+        ":single_loss_example",
+        ":combinations",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
+py_library(
+    name = "monitor",
+    srcs = ["monitor.py"],
+    visibility = ["//tensorflow:internal"],
+    deps = [
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
+cuda_py_test(
+    name = "monitor_test",
+    srcs = ["monitor_test.py"],
+    additional_deps = [
+        ":combinations",
+        ":monitor",
+        ":one_device_strategy",
+        ":single_loss_example",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:training",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
+py_library(
+    name = "shared_variable_creator",
+    srcs = ["shared_variable_creator.py"],
+    visibility = ["//tensorflow:internal"],
+)
+
+py_test(
+    name = "shared_variable_creator_test",
+    srcs = ["shared_variable_creator_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":shared_variable_creator",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python/eager:test",
+    ],
+)
+
+py_binary(
+    name = "simple_estimator_example",
+    srcs = ["simple_estimator_example.py"],
+    deps = [
+        ":mirrored_strategy",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:training",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/estimator:estimator_py",
+        "//tensorflow/python/estimator:model_fn",
+    ],
+)
+
+py_library(
+    name = "cross_tower_utils",
+    srcs = ["cross_tower_utils.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/contrib/nccl:nccl_py",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+    ],
+)
+
+py_library(
+    name = "cross_tower_ops",
+    srcs = ["cross_tower_ops.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":cross_tower_utils",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python/eager:context",
+        "@six_archive//:six",
+    ],
+)
+
+py_test(
+    name = "cross_tower_ops_test",
+    srcs = ["cross_tower_ops_test.py"],
+    srcs_version = "PY2AND3",
+    tags = [
+        "no_pip",
+    ],
+    deps = [
+        ":combinations",
+        ":cross_tower_ops",
+        ":values",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+        "@absl_py//absl/testing:parameterized",
+    ],
+)
+
+py_library(
+    name = "prefetching_ops_v2",
+    srcs = ["prefetching_ops_v2.py"],
+    deps = [
+        "//tensorflow/contrib/data/python/ops:contrib_op_loader",
+        "//tensorflow/contrib/data/python/ops:prefetching_ops",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/util:nest",
+        "//tensorflow/python/data/util:sparse",
+    ],
+)
+
+cuda_py_test(
+    name = "prefetching_ops_v2_test",
+    srcs = ["prefetching_ops_v2_test.py"],
+    additional_deps = [
+        ":prefetching_ops_v2",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/data/ops:iterator_ops",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
new file mode 100644
index 0000000000..dd8e7c4376
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -0,0 +1,293 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Facilities for creating multiple test combinations.
+
+Here is an example of testing various optimizers in Eager and Graph mode:
+
+class AdditionExample(test.TestCase, parameterized.TestCase):
+  @combinations.generate(
+     combinations.combine(mode=["graph", "eager"],
+                          optimizer=[AdamOptimizer(),
+                                     GradientDescentOptimizer()]))
+  def testOptimizer(self, optimizer):
+    ... f(optimizer)...
+
+This will run `testOptimizer` 4 times with the specified optimizers: 2 in
+Eager and 2 in Graph mode.
+The test will be provided with arguments that match the arguments of combine
+by name.  It is necessary to request all arguments, except for `mode`, which is
+optional.
+
+`combine()` function is available for creating a cross product of various
+options.  `times()` function exists for creating a product of N `combine()`-ed
+results.  See below.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from collections import OrderedDict
+import sys
+from absl.testing import parameterized
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.contrib.distribute.python import one_device_strategy
+from tensorflow.contrib.optimizer_v2 import adam as adam_v2
+from tensorflow.contrib.optimizer_v2 import gradient_descent as gradient_descent_v2
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.training import adam
+from tensorflow.python.training import gradient_descent
+from tensorflow.python.util import tf_inspect
+
+
+GPU_TEST = "test_gpu" in sys.argv[0]
+
+
+def generate(combinations):
+  """A decorator for generating test cases of a test method or a test class.
+
+  Args:
+    combinations: a list of dictionaries created using combine() and times().
+
+  Restrictions:
+   -- there should always be a "mode" argument.  Accepted values are "eager"
+      and "graph".
+   -- arguments of the test method must match by name to get the corresponding
+      value of the combination.  Tests must accept all arguments (except "mode",
+      which is optional).
+   -- distribution argument is special.  It is meant for passing instances of
+      DistributionStrategy.  Each instance is to be passed as `(<int>,
+      <DistributionStrategy>)` tuple, where <int> is the number of required
+      GPUs.  If the required number of GPUs for the DistributionStrategy isn't
+      available then the test case is going to be skipped.
+
+  Returns:
+    a decorator that will cause the test method to be run under the specified
+    conditions.
+
+  Raises:
+    ValueError - if "mode" argument wasn't either "eager" or "graph.
+  """
+
+  def decorator(test_function):
+    """The decorator to be returned."""
+
+    # Generate good test names that can be used with --test_filter.
+    for combination in combinations:
+      # We use OrderedDicts in `combine()` and `times()` to ensure stable
+      # order of keys in each dictionary.
+      assert isinstance(combination, OrderedDict)
+      name = "".join([
+          "_{}_{}".format(
+              "".join(filter(str.isalnum, key)),
+              "".join(filter(str.isalnum, str(value))))
+          for key, value in combination.items()
+      ])
+      combination.update({"testcase_name": "_test{}".format(name)})
+
+    @parameterized.named_parameters(*combinations)
+    def decorated(self, **kwargs):
+      """A wrapped test method that sets up `test_function`."""
+      assert "mode" in kwargs
+      mode = kwargs["mode"]
+
+      if "distribution" in kwargs:
+        distribution = kwargs["distribution"]
+        kwargs["distribution"] = distribution.strategy
+        if not distribution.required_gpus:
+          if GPU_TEST:
+            self.skipTest("Test that doesn't require GPUs.")
+        elif context.num_gpus() < distribution.required_gpus:
+          self.skipTest(
+              "{} GPUs are not available for this test. {} GPUs are available".
+              format(distribution.required_gpus, context.num_gpus()))
+
+      requested_arguments = tf_inspect.getfullargspec(test_function).args
+      missing_arguments = set(list(kwargs.keys()) + ["self"]).difference(
+          set(requested_arguments + ["mode"]))
+      if missing_arguments:
+        raise ValueError("The test is missing arguments {} .".format(
+            missing_arguments))
+
+      kwargs_to_pass = {}
+      for arg in requested_arguments:
+        if arg == "self":
+          kwargs_to_pass[arg] = self
+        else:
+          kwargs_to_pass[arg] = kwargs[arg]
+
+      if mode == "eager":
+        with context.eager_mode(), ops.Graph().as_default():
+          test_function(**kwargs_to_pass)
+      elif mode == "graph":
+        with context.graph_mode(), ops.Graph().as_default():
+          test_function(**kwargs_to_pass)
+      else:
+        raise ValueError(
+            "'mode' has to be either 'eager' or 'graph' and not {}".format(
+                mode))
+
+    return decorated
+  return decorator
+
+
+def combine(**kwargs):
+  """Generate combinations based on its keyword arguments.
+
+  Two sets of returned combinations can be concatenated using +.  Their product
+  can be computed using `times()`.
+
+  Args:
+    **kwargs: keyword arguments of form `option=[possibilities, ...]`.
+
+  Returns:
+    a list of dictionaries for each combination. Keys in the dictionaries are
+    the keyword argument names.  Each key has one value - one of the
+    corresponding keyword argument values.
+  """
+  if not kwargs:
+    return [OrderedDict()]
+
+  sort_by_key = lambda k: k[0][0]
+  kwargs = OrderedDict(sorted(kwargs.items(), key=sort_by_key))
+  first = list(kwargs.items())[0]
+
+  rest = dict(list(kwargs.items())[1:])
+  rest_combined = combine(**rest)
+
+  key = first[0]
+  values = first[1]
+
+  return [
+      OrderedDict(sorted(list(combined.items()) + [(key, v)], key=sort_by_key))
+      for v in values
+      for combined in rest_combined
+  ]
+
+
+def times(*combined):
+  """Generate a product of N sets of combinations.
+
+  times(combine(a=[1,2]), combine(b=[3,4])) == combine(a=[1,2], b=[3,4])
+
+  Args:
+    *combined: N lists of dictionaries that specify combinations.
+
+  Returns:
+    a list of dictionaries for each combination.
+
+  Raises:
+    ValueError: if some of the inputs have overlapping keys.
+  """
+  assert combined
+
+  if len(combined) == 1:
+    return combined[0]
+
+  first = combined[0]
+  rest_combined = times(*combined[1:])
+
+  combined_results = []
+  for a in first:
+    for b in rest_combined:
+      if set(a.keys()).intersection(set(b.keys())):
+        raise ValueError("Keys need to not overlap: {} vs {}".format(
+            a.keys(), b.keys()))
+
+      combined_results.append(OrderedDict(list(a.items()) + list(b.items())))
+  return combined_results
+
+
+class NamedObject(object):
+  """A class that translates an object into a good test name."""
+
+  def __init__(self, name, obj):
+    self._name = name
+    self._obj = obj
+
+  def __getattr__(self, name):
+    return getattr(self._obj, name)
+
+  def __call__(self, *args, **kwargs):
+    return self._obj(*args, **kwargs)
+
+  def __repr__(self):
+    return self._name
+
+
+class NamedDistribution(object):
+  """Translates DistributionStrategy and its data into a good name."""
+
+  def __init__(self, name, distribution, required_gpus):
+    self._distribution = distribution
+    self._name = name
+    self._required_gpus = required_gpus
+
+  def __repr__(self):
+    return self._name
+
+  @property
+  def strategy(self):
+    return self._distribution
+
+  @property
+  def required_gpus(self):
+    return self._required_gpus
+
+
+one_device_strategy = NamedDistribution(
+    "OneDeviceCPU", one_device_strategy.OneDeviceStrategy("/cpu:0"),
+    None)
+mirrored_strategy_with_gpu_and_cpu = NamedDistribution(
+    "MirroredCPUAndGPU",
+    mirrored_strategy.MirroredStrategy(["/gpu:0", "/cpu:0"]), 1)
+mirrored_strategy_with_two_gpus = NamedDistribution(
+    "Mirrored2GPUs",
+    mirrored_strategy.MirroredStrategy(["/gpu:0", "/gpu:1"]), 2)
+
+adam_optimizer_v1_fn = NamedObject(
+    "AdamV1", lambda: adam.AdamOptimizer(0.2, epsilon=1))
+gradient_descent_optimizer_v1_fn = NamedObject(
+    "GradientDescentV1", lambda: gradient_descent.GradientDescentOptimizer(0.2))
+
+adam_optimizer_v2_fn = NamedObject(
+    "AdamV2", lambda: adam_v2.AdamOptimizer(0.2, epsilon=1))
+gradient_descent_optimizer_v2_fn = NamedObject(
+    "GradientDescentV2",
+    lambda: gradient_descent_v2.GradientDescentOptimizer(0.2))
+
+graph_and_eager_modes = ["graph", "eager"]
+
+
+def distributions_and_v1_optimizers():
+  """A common set of combination with DistributionStrategies and Optimizers."""
+  return combine(
+      distribution=[
+          one_device_strategy, mirrored_strategy_with_gpu_and_cpu,
+          mirrored_strategy_with_two_gpus
+      ],
+      optimizer_fn=[adam_optimizer_v1_fn, gradient_descent_optimizer_v1_fn])
+
+
+def distributions_and_v2_optimizers():
+  """DistributionStrategies and V2 Optimizers."""
+  return combine(
+      distribution=[
+          one_device_strategy, mirrored_strategy_with_gpu_and_cpu,
+          mirrored_strategy_with_two_gpus
+      ],
+      optimizer_fn=[adam_optimizer_v2_fn, gradient_descent_optimizer_v2_fn])
diff --git a/tensorflow/contrib/distribute/python/combinations_test.py b/tensorflow/contrib/distribute/python/combinations_test.py
new file mode 100644
index 0000000000..219b24160f
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/combinations_test.py
@@ -0,0 +1,115 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for some testing utils from strategy_test_lib."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from collections import OrderedDict
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.python.eager import test
+
+
+class TestingCombinationsTest(test.TestCase):
+
+  def test_combine(self):
+    self.assertEqual([{
+        "a": 1,
+        "b": 2
+    }, {
+        "a": 1,
+        "b": 3
+    }, {
+        "a": 2,
+        "b": 2
+    }, {
+        "a": 2,
+        "b": 3
+    }], combinations.combine(a=[1, 2], b=[2, 3]))
+
+  def test_add(self):
+    self.assertEqual(
+        [{
+            "a": 1
+        }, {
+            "a": 2
+        }, {
+            "b": 2
+        }, {
+            "b": 3
+        }],
+        combinations.combine(a=[1, 2]) +
+        combinations.combine(b=[2, 3]))
+
+  def test_times(self):
+    c1 = combinations.combine(mode=["graph"], loss=["callable", "tensor"])
+    c2 = combinations.combine(mode=["eager"], loss=["callable"])
+    c3 = combinations.combine(distribution=["d1", "d2"])
+    c4 = combinations.times(c3, c1 + c2)
+    self.assertEqual([
+        OrderedDict([("distribution", "d1"), ("loss", "callable"),
+                     ("mode", "graph")]),
+        OrderedDict([("distribution", "d1"), ("loss", "tensor"),
+                     ("mode", "graph")]),
+        OrderedDict([("distribution", "d1"), ("loss", "callable"),
+                     ("mode", "eager")]),
+        OrderedDict([("distribution", "d2"), ("loss", "callable"),
+                     ("mode", "graph")]),
+        OrderedDict([("distribution", "d2"), ("loss", "tensor"),
+                     ("mode", "graph")]),
+        OrderedDict([("distribution", "d2"), ("loss", "callable"),
+                     ("mode", "eager")])
+    ], c4)
+
+  def test_times_variable_arguments(self):
+    c1 = combinations.combine(mode=["graph", "eager"])
+    c2 = combinations.combine(optimizer=["adam", "gd"])
+    c3 = combinations.combine(distribution=["d1", "d2"])
+    c4 = combinations.times(c3, c1, c2)
+    self.assertEqual([
+        OrderedDict([("distribution", "d1"), ("mode", "graph"),
+                     ("optimizer", "adam")]),
+        OrderedDict([("distribution", "d1"), ("mode", "graph"),
+                     ("optimizer", "gd")]),
+        OrderedDict([("distribution", "d1"), ("mode", "eager"),
+                     ("optimizer", "adam")]),
+        OrderedDict([("distribution", "d1"), ("mode", "eager"),
+                     ("optimizer", "gd")]),
+        OrderedDict([("distribution", "d2"), ("mode", "graph"),
+                     ("optimizer", "adam")]),
+        OrderedDict([("distribution", "d2"), ("mode", "graph"),
+                     ("optimizer", "gd")]),
+        OrderedDict([("distribution", "d2"), ("mode", "eager"),
+                     ("optimizer", "adam")]),
+        OrderedDict([("distribution", "d2"), ("mode", "eager"),
+                     ("optimizer", "gd")])
+    ], c4)
+    self.assertEqual(
+        combinations.combine(
+            mode=["graph", "eager"],
+            optimizer=["adam", "gd"],
+            distribution=["d1", "d2"]), c4)
+
+  def test_overlapping_keys(self):
+    c1 = combinations.combine(mode=["graph"], loss=["callable", "tensor"])
+    c2 = combinations.combine(mode=["eager"], loss=["callable"])
+    with self.assertRaisesRegexp(ValueError, ".*Keys.+overlap.+"):
+      _ = combinations.times(c1, c2)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py
new file mode 100644
index 0000000000..cb98351735
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py
@@ -0,0 +1,410 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Classes for different algortihms of reduction and broadcasting."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+from tensorflow.contrib.distribute.python import cross_tower_utils
+from tensorflow.contrib.distribute.python import values as value_lib
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.training import device_util
+
+
+def _validate_destinations(destinations):
+  if not isinstance(destinations,
+                    (value_lib.DistributedValues, six.string_types, list)):
+    raise ValueError("destinations must be one of a `DistributedValues` object,"
+                     " a device string, a list of device strings or None")
+
+  if not destinations:
+    raise ValueError("destinations can not be empty")
+
+
+def _validate_value_destination_pairs(value_destination_pairs):
+  # pylint: disable=g-missing-docstring
+  if not value_destination_pairs: return False
+  if not isinstance(value_destination_pairs, (list, tuple)): return False
+  if not all([isinstance(pair, tuple) for pair in value_destination_pairs]):
+    return False
+  if not all([isinstance(v[0], value_lib.PerDevice)
+              for v in value_destination_pairs]):
+    return False
+  return True
+
+
+def _get_devices_from(destinations):
+  if isinstance(destinations, value_lib.DistributedValues):
+    return list(destinations.devices)
+  elif isinstance(destinations, six.string_types):
+    return [device_util.canonicalize(destinations)]
+  else:
+    return [
+        device_util.canonicalize(destination) for destination in destinations
+    ]
+
+
+def _devices_match(left, right):
+  return set(_get_devices_from(left)) == set(_get_devices_from(right))
+
+
+def _all_devices_match(value_destination_pairs):
+  if not all([d is None or _devices_match(v, d)
+              for v, d in value_destination_pairs]):
+    return False
+  if not all([_devices_match(v, value_destination_pairs[0][0])
+              for v, _ in value_destination_pairs[1:]]):
+    return False
+  return True
+
+
+def _simple_broadcast(tensor, destinations):
+  index = {}
+  devices = _get_devices_from(destinations)
+  for d in devices:
+    with ops.device(d):
+      index[d] = array_ops.identity(tensor)
+  return value_lib.Mirrored(index)
+
+
+def _simple_reduce(per_device_value, reduce_to_device, accumulation_fn,
+                   method_string):
+  # pylint: disable=g-missing-docstring
+  all_values = []
+  count = 0
+  for v in per_device_value._index.values():  # pylint: disable=protected-access
+    if isinstance(v, value_lib.MapOutput):
+      v_list = v.get()
+      if not v_list:
+        continue
+      count += len(v_list)
+      # Sum within each device before aggregating across devices.
+      v = math_ops.add_n(v_list)
+    else:
+      count += 1
+    all_values.append(v)
+  if not all_values:
+    raise ValueError("`per_device_value` must be non-empty")
+
+  with ops.device(reduce_to_device):
+    with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+      if method_string == "sum":
+        reduced = accumulation_fn(all_values)
+      elif method_string == "mean":
+        reduced = accumulation_fn(all_values) / count
+      else:
+        raise ValueError("`method_string` must be 'sum' or 'mean'")
+  return reduced
+
+
+class CrossTowerOps(object):
+  """Base class for cross-tower reduction and broadcasting algorithms."""
+
+  def __init__(self):
+    pass
+
+  def reduce(self, method_string, per_device_value, destinations=None):
+    """Reduce `per_device_value` to `destinations`.
+
+    It runs the reduction operation defined by `method_string` and put the
+    result on `destinations`.
+
+    Args:
+      method_string: either 'sum' or 'mean' specifying the reduction method.
+      per_device_value: a PerDevice object.
+      destinations: the reduction destinations.
+
+    Returns:
+      a Mirrored object.
+
+    Raises:
+      ValueError: if per_device_value is not a PerDevice object.
+    """
+    if not isinstance(per_device_value, value_lib.PerDevice):
+      raise ValueError("`per_device_value` must be a `PerDevice` object.")
+    if destinations is not None:
+      _validate_destinations(destinations)
+    return self._reduce(method_string, per_device_value, destinations)
+
+  def batch_reduce(self, method_string, value_destination_pairs):
+    """Reduce PerDevice objects in a batch.
+
+    Reduce each first element in `value_destination_pairs` to each second
+    element which indicates the destinations.
+
+    Args:
+      method_string: either 'sum' or 'mean' specifying the reduction method.
+      value_destination_pairs: a list or a tuple of tuples of PerDevice objects
+        and destinations. If a destionation is None, then the destinations
+        are set to match the devices of the input PerDevice object.
+
+    Returns:
+      a list of Mirrored objects.
+
+    Raises:
+      ValueError: if `value_destination_pairs` is not a list or a tuple of
+        tuples of PerDevice objects and destinations
+    """
+    if not _validate_value_destination_pairs(value_destination_pairs):
+      raise ValueError("`value_destination_pairs` must be a list or a tuple of "
+                       "tuples of PerDevice objects and destinations")
+    for _, d in value_destination_pairs:
+      if d is not None:
+        _validate_destinations(d)
+
+    return self._batch_reduce(method_string, value_destination_pairs)
+
+  def broadcast(self, tensor, destinations):
+    """Broadcast the `tensor` to destinations.
+
+    Args:
+      tensor: the tensor to broadcast.
+      destinations: the broadcast destinations.
+
+    Returns:
+      a Mirrored object.
+    """
+    _validate_destinations(destinations)
+    return self._broadcast(tensor, destinations)
+
+  def _reduce(self, method_string, per_device_value, destinations):
+    raise NotImplementedError(
+        "_reduce method must be implemented in descendants.")
+
+  def _batch_reduce(self, method_string, value_destination_pairs):
+    raise NotImplementedError(
+        "_batch_reduce method must be implemented in descendants.")
+
+  def _broadcast(self, tensor, destinations):
+    return _simple_broadcast(tensor, destinations)
+
+
+class ReductionToOneDeviceCrossTowerOps(CrossTowerOps):
+  """Always do reduction to one device first and then do broadcasting.
+
+    Batch reduction is done by reduction on each element one by one.
+  """
+
+  def __init__(self, reduce_to_device=None, accumulation_fn=math_ops.add_n):
+    """Constructor.
+
+    Args:
+      reduce_to_device: the intermediate device to reduce to. If None, reduce
+        to the first device in `destinations` of the reduce() method.
+      accumulation_fn: a function that does accumulation.
+    """
+    self.reduce_to_device = reduce_to_device
+    self.accumulation_fn = accumulation_fn
+    super(ReductionToOneDeviceCrossTowerOps, self).__init__()
+
+  def _reduce(self, method_string, per_device_value, destinations):
+    devices = _get_devices_from(destinations or per_device_value)
+    reduce_to_device = self.reduce_to_device or devices[0]
+    reduced = _simple_reduce(per_device_value, reduce_to_device,
+                             self.accumulation_fn, method_string)
+    return self.broadcast(reduced, devices)
+
+  def _batch_reduce(self, method_string, value_destination_pairs):
+    return [self._reduce(method_string, t, destinations=v)
+            for t, v in value_destination_pairs]
+
+
+def _group_value_by_device(per_device_values):
+  """Group values into sublists by their devices.
+
+  This grouping is needed to call the allreduce library.
+
+  Args:
+    per_device_values: a list of PerDevice obejcts.
+
+  Returns:
+    a list of lists, each sublist has components for its corresponding device of
+      PerDevice objects, paired with a None.
+  """
+  destinations = per_device_values[0].devices
+  grouped = [[] for _ in range(len(destinations))]
+  for per_device_value in per_device_values:
+    # pylint: disable=protected-access
+    for i, v in enumerate(per_device_value._index.values()):
+      assert per_device_value.devices == destinations
+      grouped[i].append((v, None))
+  return grouped
+
+
+def _ungroup_and_make_mirrored(grouped_reduced, destinations, method_string):
+  """Ungroup results from allreduce and make Mirrored objects.
+
+  Each allreduce result would be divided by the number of destinations before
+  Mirrored objects are created if method_string is "mean".
+  """
+  index = [{} for _ in range(len(grouped_reduced[0]))]
+  for d, per_device_reduced in enumerate(grouped_reduced):
+    for i, (v, _) in enumerate(per_device_reduced):
+      if method_string == "mean":
+        index[i][destinations[d]] = v / len(destinations)
+      else:
+        index[i][destinations[d]] = v
+  return [value_lib.Mirrored(v) for v in index]
+
+
+class AllReduceCrossTowerOps(CrossTowerOps):
+  """Reduction using all reduce."""
+
+  def __init__(self, all_reduce_alg="nccl", gradient_repacking=1):
+    """Initialize this subclass of CrossTowerOps with allreduce.
+
+    Gradients would be repacked for more efficient cross-device transportation.
+
+    Args:
+      all_reduce_alg: the allreduce algorithm to use, currently only "nccl" or
+        "hierarchical_copy" are supported.
+      gradient_repacking: If zero, no gradient repacking would be done. If
+        non-zero value it specifies the number of split packs that will be
+        formed.
+    """
+    self.all_reduce_alg = all_reduce_alg
+    self.gradient_repacking = gradient_repacking
+    super(AllReduceCrossTowerOps, self).__init__()
+
+  def _reduce(self, method_string, per_device_value, destinations):
+    if ((destinations is None or _devices_match(per_device_value, destinations))
+        and not context.executing_eagerly()):
+      return self._batch_all_reduce(method_string, [per_device_value])[0]
+    else:
+      devices = _get_devices_from(destinations or per_device_value)
+      reduce_to_device = devices[0]
+      reduced = _simple_reduce(per_device_value, reduce_to_device,
+                               math_ops.add_n, method_string)
+      return self.broadcast(reduced, devices)
+
+  def _batch_reduce(self, method_string, value_destination_pairs):
+    if (_all_devices_match(value_destination_pairs) and
+        not context.executing_eagerly()):
+      return self._batch_all_reduce(method_string,
+                                    [v[0] for v in value_destination_pairs])
+    else:
+      if not context.executing_eagerly():
+        logging.warning("Efficient batch_reduce is not supported if "
+                        "destinations are different.")
+      return [
+          self._reduce(method_string, t, destinations=v)
+          for t, v in value_destination_pairs
+      ]
+
+  def _batch_all_reduce(self, method_string, per_device_values):
+    """All reduce algorithm in a batch."""
+    logging.info("batch_all_reduce invoked for batches size = %d with algorithm"
+                 " = %s and gradient repacking = %d", len(per_device_values),
+                 self.all_reduce_alg, self.gradient_repacking)
+    destinations = per_device_values[0].devices
+    grouped = _group_value_by_device(per_device_values)
+    if self.gradient_repacking == 0:
+      if self.all_reduce_alg == "nccl":
+        reduced = cross_tower_utils.aggregate_gradients_using_nccl(grouped)
+      else:
+        # TODO(yuefengz): check that gpu ids in `destinations` are in ascending
+        # order.
+        reduced = (
+            cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
+                destinations, grouped))
+    else:
+      device_grad_packs = []
+      all_tower_shapes = []
+      all_tower_sizes = []
+      for tower_grads_and_vars in grouped:
+        with ops.colocate_with(tower_grads_and_vars[0][0]):
+          # Flatten all the grads.
+          flat_grads = [
+              array_ops.reshape(g, [-1]) for g, _ in tower_grads_and_vars
+          ]
+          # Remember the original shape of all the grads.
+          tower_shapes = [array_ops.shape(g) for g, _ in tower_grads_and_vars]
+          # Remember the original sizes of all the grads.
+          tower_sizes = [array_ops.size(g) for g, _ in tower_grads_and_vars]
+          # Concat all the flat grads into a big flat tensor.
+          concat_grads = array_ops.concat(flat_grads, 0)
+
+          # Split the big tensor into num_splits packs. In cases where the
+          # total size is not divisible num_splits, the last pack gets
+          # more elements.
+          # TODO(zhengxq): it is possible to optimize away the additional
+          # data movement by copying along the original variable boundary.
+          # TODO(zhengxq): it is also possible to optimize away all the concat
+          # as well.
+          num_splits = self.gradient_repacking
+          total_grad_size = array_ops.size(concat_grads)
+          split_size = total_grad_size // num_splits
+          split_size_last = total_grad_size - split_size * (num_splits - 1)
+          split_sizes = [split_size] * (num_splits - 1) + [split_size_last]
+          grad_packs = array_ops.split(concat_grads, split_sizes)
+
+          # Ready to aggregate the repacked gradients, with fake variables.
+          # TODO(zhengxq): It is hacky to have to use fake variables.
+          # We should remove the need for variables in
+          # aggregate_gradients_using*.
+          device_grad_packs.append(zip(grad_packs, [None] * num_splits))
+          all_tower_shapes.append(tower_shapes)
+          all_tower_sizes.append(tower_sizes)
+
+      # The actual aggregation of the repacked gradients. Note that they are
+      # sharded among different aggregation trees. So it is important to
+      # strike the balance on num_splits.
+      if self.all_reduce_alg == "nccl":
+        summed_device_grad_packs = (
+            cross_tower_utils.aggregate_gradients_using_nccl(device_grad_packs))
+      else:
+        summed_device_grad_packs = (
+            cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
+                destinations, device_grad_packs))
+
+      aggregated_device_grads = []
+      for (summed_tower_grad_packs, tower_grads_and_vars, tower_shapes,
+           tower_sizes) in zip(summed_device_grad_packs, grouped,
+                               all_tower_shapes, all_tower_sizes):
+        # pylint: enable=line-too-long
+        # Reverse the packing operations in the previous steps. Form the
+        # summed gradients back into their original shapes.
+        with ops.colocate_with(summed_tower_grad_packs[0][0]):
+          # Form a list of the summed grad packs.
+          device_grad_packs = [g for g, _ in summed_tower_grad_packs]
+
+          # Concat them back into a big flat tensor.
+          device_grads_concat = array_ops.concat(device_grad_packs, 0)
+
+          # Split the tensors back into their original sizes.
+          grads_with_sizes = array_ops.split(device_grads_concat, tower_sizes)
+
+          # Reshape the tensors back into their original shapes.
+          grads_with_shapes = [
+              array_ops.reshape(grad, shape)
+              for shape, grad in zip(tower_shapes, grads_with_sizes)
+          ]
+
+          # Form the list with the original list of variables.
+          summed_tower_grads = [
+              (g, v)
+              for g, (_, v) in zip(grads_with_shapes, tower_grads_and_vars)
+          ]
+          aggregated_device_grads.append(summed_tower_grads)
+      reduced = aggregated_device_grads
+    return _ungroup_and_make_mirrored(reduced, per_device_values[0].devices,
+                                      method_string)
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
new file mode 100644
index 0000000000..bb43147f5e
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
@@ -0,0 +1,185 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for CrossTowerOps."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import itertools
+
+from absl.testing import parameterized
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
+from tensorflow.contrib.distribute.python import values as value_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def _make_per_device(values, devices):
+  devices = cross_tower_ops_lib._get_devices_from(devices)
+  assert len(values) == len(devices)
+  index = {}
+  for d, v in zip(devices, values):
+    with ops.device(d):
+      placed_v = array_ops.identity(v)
+    index[d] = placed_v
+  return value_lib.PerDevice(index)
+
+
+# pylint: disable=g-doc-args,g-doc-return-or-yield
+def _fake_mirrored(value, devices):
+  """Create a faked Mirrored object for testing.
+
+  All components of the returned Mirrored have the same objects, which is not
+  true in reality.
+  """
+  devices = cross_tower_ops_lib._get_devices_from(devices)
+  return value_lib.Mirrored(
+      {d: v for d, v in zip(devices, [value] * len(devices))})
+
+
+_cpu_device = "/device:CPU:0"
+
+
+class CrossTowerOpsTest(test.TestCase, parameterized.TestCase):
+
+  def _assert_value_equal(self, left, right):
+    if isinstance(left, list):
+      for l, r in zip(left, right):
+        self._assert_value_equal(l, r)
+    else:
+      self.assertEqual(type(left), type(right))
+      self.assertEqual(left.devices, right.devices)
+      if context.executing_eagerly():
+        self.assertEqual([v.numpy() for v in left._index.values()],
+                         list(right._index.values()))
+      else:
+        with self.test_session() as sess:
+          self.assertEqual(
+              sess.run(list(left._index.values())), list(right._index.values()))
+
+  # TODO(yuefengz): decouple the num_gpus check from distribution in
+  # combinations module so that we can pass in devices instead of a distribution
+  # strategy.
+  reduction_to_one_combinations = combinations.combine(
+      cross_tower_ops=[
+          combinations.NamedObject(
+              "DefaultReductionToOneDeviceCrossTowerOps",
+              cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps()),
+          combinations.NamedObject(
+              "ReductionToCPUDeviceCrossTowerOps",
+              cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps(
+                  reduce_to_device=_cpu_device)),
+          combinations.NamedObject(
+              "AccumulateNCrossTowerOp",
+              cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps(
+                  accumulation_fn=math_ops.accumulate_n)),
+      ],
+      distribution=[
+          combinations.one_device_strategy,
+          combinations.mirrored_strategy_with_gpu_and_cpu,
+          combinations.mirrored_strategy_with_two_gpus
+      ],
+      mode=["graph", "eager"])
+  allreduce_combinations = combinations.combine(
+      cross_tower_ops=[
+          combinations.NamedObject("AllReduce",
+                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
+                                       "nccl", 1)),
+          combinations.NamedObject("HierarchicalCopy",
+                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
+                                       "hierarchical_copy", 8)),
+          combinations.NamedObject("AllReduceNoGradientRepacking",
+                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
+                                       "nccl", 0)),
+          combinations.NamedObject("HierarchicalCopyNoGradientRepacking",
+                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
+                                       "hierarchical_copy", 0))
+      ],
+      distribution=[
+          combinations.mirrored_strategy_with_two_gpus
+      ],
+      mode=["graph", "eager"])
+
+  @combinations.generate(reduction_to_one_combinations + allreduce_combinations)
+  def testReductionAndBroadcast(self, cross_tower_ops, distribution):
+    devices = distribution.worker_devices
+
+    values = [constant_op.constant(float(d)) for d in range(len(devices))]
+    per_device = _make_per_device(values, devices)
+    mean = (len(devices) - 1.) / 2.
+
+    values_2 = [constant_op.constant(d + 1.0) for d in range(len(devices))]
+    per_device_2 = _make_per_device(values_2, devices)
+    mean_2 = mean + 1.
+
+    destination_mirrored = _fake_mirrored(1., devices)
+    destination_different = _fake_mirrored(1., _cpu_device)
+    destination_str = _cpu_device
+    destination_list = devices
+
+    all_destinations = [
+        None, destination_mirrored, destination_different, destination_str,
+        destination_list
+    ]
+
+    # test reduce()
+    for destinations in all_destinations:
+      self._assert_value_equal(
+          cross_tower_ops.reduce("mean", per_device, destinations=destinations),
+          _fake_mirrored(mean, destinations or per_device))
+      self._assert_value_equal(
+          cross_tower_ops.reduce(
+              "mean", per_device_2, destinations=destinations),
+          _fake_mirrored(mean_2, destinations or per_device))
+      self._assert_value_equal(
+          cross_tower_ops.reduce("sum", per_device, destinations=destinations),
+          _fake_mirrored(mean * len(devices), destinations or per_device))
+      self._assert_value_equal(
+          cross_tower_ops.reduce(
+              "sum", per_device_2, destinations=destinations),
+          _fake_mirrored(mean_2 * len(devices), destinations or per_device))
+
+    # test batch_reduce()
+    for d1, d2 in itertools.product(all_destinations, all_destinations):
+      self._assert_value_equal(
+          cross_tower_ops.batch_reduce(
+              "mean", [(per_device, d1), (per_device_2, d2)]),
+          [_fake_mirrored(mean, d1 or per_device),
+           _fake_mirrored(mean_2, d2 or per_device_2)])
+      self._assert_value_equal(
+          cross_tower_ops.batch_reduce(
+              "sum", [(per_device, d1), (per_device_2, d2)]),
+          [_fake_mirrored(mean * len(devices), d1 or per_device),
+           _fake_mirrored(mean_2 * len(devices), d2 or per_device_2)])
+
+    # test broadcast()
+    for destinations in all_destinations:
+      if destinations is None:
+        continue
+      else:
+        self._assert_value_equal(
+            cross_tower_ops.broadcast(constant_op.constant(1.), destinations),
+            _fake_mirrored(1., destinations))
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py
new file mode 100644
index 0000000000..93acd835d7
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py
@@ -0,0 +1,153 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utilities for cross_tower_ops."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib import nccl
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def aggregate_gradients_using_nccl(tower_grads):
+  """Aggregate gradients using nccl allreduce."""
+  agg_all_g_and_v = []
+  for single_g_and_v in zip(*tower_grads):
+    single_grads = [g for g, _ in single_g_and_v]
+    agg_grads = nccl.all_sum(single_grads)
+    agg_all_g_and_v.append(
+        [(g, v) for g, (_, v) in zip(agg_grads, single_g_and_v)])
+
+  agg_all_g_and_v = list(zip(*agg_all_g_and_v))
+
+  return agg_all_g_and_v
+
+
+def aggregate_gradients_using_hierarchical_copy(avail_devices, tower_grads):
+  """Aggregate gradients using hierarchical copies.
+
+  Args:
+    avail_devices: available GPU devices.
+    tower_grads: List of lists of (gradient, variable) tuples. The outer list
+      is over towers. The inner list is over individual gradients.
+
+  Returns:
+    The list of (aggregated_gradient, variable), where the gradient has been
+      summed across all towers and the variable is chosen from the first tower.
+  """
+  # This only works for DGX-1 type of machine topology
+  # Device peer to peer matrix
+  # DMA: 0 1 2 3 4 5 6 7
+  # 0:   Y Y Y Y Y N N N
+  # 1:   Y Y Y Y N Y N N
+  # 2:   Y Y Y Y N N Y N
+  # 3:   Y Y Y Y N N N Y
+  # 4:   Y N N N Y Y Y Y
+  # 5:   N Y N N Y Y Y Y
+  # 6:   N N Y N Y Y Y Y
+  # 7:   N N N Y Y Y Y Y
+  agg_grads = []
+  num_devices = len(avail_devices)
+  # In the special case of DGX-1 machine topology, the two groups have equal
+  # size.
+  group_size = num_devices // 2
+  for i, single_grads in enumerate(zip(*tower_grads)):
+    group_0_main_device = i % num_devices
+    group_1_main_device = (group_0_main_device + group_size) % num_devices
+    if group_0_main_device < group_size:
+      group_0_begin = 0
+      group_1_begin = group_size
+    else:
+      group_0_begin = group_size
+      group_1_begin = 0
+
+    # Aggregate the first group.
+    group_0_device_grads = single_grads[group_0_begin:
+                                        group_0_begin + group_size]
+    with ops.device(avail_devices[group_0_main_device]):
+      group_0_agg_grads, _ = aggregate_single_gradient_using_copy(
+          group_0_device_grads, False, False)
+
+    # Aggregate the second group.
+    group_1_device_grads = single_grads[group_1_begin:
+                                        group_1_begin + group_size]
+    with ops.device(avail_devices[group_1_main_device]):
+      group_1_agg_grads, _ = aggregate_single_gradient_using_copy(
+          group_1_device_grads, False, False)
+
+    # Aggregate between the groups.
+    with ops.device(avail_devices[group_0_main_device]):
+      (agg_total_grads, _), _ = aggregate_single_gradient_using_copy(
+          [group_0_agg_grads, group_1_agg_grads], False, False)
+
+    # Broadcast the result back into the root of each group.
+    with ops.device(avail_devices[group_0_main_device]):
+      group_0_agg_grads_bcast = array_ops.identity(agg_total_grads)
+    with ops.device(avail_devices[group_1_main_device]):
+      group_1_agg_grads_bcast = array_ops.identity(agg_total_grads)
+
+    agg_grads_bcast = []
+    for j in range(len(single_grads)):
+      with ops.device(avail_devices[j]):
+        # Broadcast the result back to each member in the group from the root.
+        if (group_0_main_device < group_size) == (j < group_size):
+          src_device_grad = group_0_agg_grads_bcast
+        else:
+          src_device_grad = group_1_agg_grads_bcast
+        agg_grads_bcast.append(array_ops.identity(src_device_grad))
+
+    agg_grads.append(
+        [(g, v) for g, (_, v) in zip(agg_grads_bcast, single_grads)])
+
+  agg_grads = list(zip(*agg_grads))
+
+  return agg_grads
+
+
+def aggregate_single_gradient_using_copy(grad_and_vars, use_mean,
+                                         check_inf_nan):
+  """Calculate the average gradient for a shared variable across all towers.
+
+  Note that this function provides a synchronization point across all towers.
+
+  Args:
+    grad_and_vars: A list or tuple of (gradient, variable) tuples. Each
+      (gradient, variable) pair within the outer list represents the gradient
+      of the variable calculated for a single tower, and the number of pairs
+      equals the number of towers.
+    use_mean: if True, mean is taken, else sum of gradients is taken.
+    check_inf_nan: check grads for nans and infs.
+
+  Returns:
+    The tuple ([(average_gradient, variable),], has_nan_or_inf) where the
+      gradient has been averaged across all towers. The variable is chosen from
+      the first tower. The has_nan_or_inf indicates the grads has nan or inf.
+  """
+  grads = [g for g, _ in grad_and_vars]
+  grad = math_ops.add_n(grads)
+
+  if use_mean and len(grads) > 1:
+    grad = array_ops.multiply(grad, 1.0 / len(grads))
+
+  v = grad_and_vars[0][1]
+  if check_inf_nan:
+    has_nan_or_inf = array_ops.logical_not(
+        array_ops.reduce_all(array_ops.is_finite(grads)))
+    return (grad, v), has_nan_or_inf
+  else:
+    return (grad, v), None
diff --git a/tensorflow/contrib/distribute/python/minimize_loss_test.py b/tensorflow/contrib/distribute/python/minimize_loss_test.py
new file mode 100644
index 0000000000..0fa90df79b
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/minimize_loss_test.py
@@ -0,0 +1,279 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for running legacy optimizer code with DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.contrib.distribute.python.single_loss_example import batchnorm_example
+from tensorflow.contrib.distribute.python.single_loss_example import minimize_loss_example
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables as variables_lib
+from tensorflow.python.ops.losses import losses_impl
+
+
+class MinimizeLossStepTest(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.times(
+          combinations.distributions_and_v1_optimizers(),
+          combinations.combine(mode=["graph"], use_callable_loss=[True, False])
+          + combinations.combine(mode=["eager"], use_callable_loss=[True])))
+  def testTrainNetwork(self, distribution, optimizer_fn,
+                       use_callable_loss=True):
+    with distribution.scope():
+      model_fn, dataset, layer = minimize_loss_example(
+          optimizer_fn,
+          use_bias=True,
+          use_callable_loss=use_callable_loss)
+
+      iterator = distribution.distribute_dataset(dataset)
+
+      def run_step():
+        return distribution.group(
+            distribution.call_for_each_tower(
+                model_fn, iterator.get_next(), run_concurrently=layer.built))
+
+      if not context.executing_eagerly():
+        with self.test_session() as sess:
+          run_step = sess.make_callable(run_step())
+        self.evaluate(variables_lib.global_variables_initializer())
+
+      weights, biases = [], []
+      for _ in range(10):
+        run_step()
+
+        weights.append(self.evaluate(distribution.fetch(layer.kernel)))
+        biases.append(self.evaluate(distribution.fetch(layer.bias)))
+
+      error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
+      is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
+      self.assertTrue(is_not_increasing)
+
+  @combinations.generate(
+      combinations.times(
+          combinations.distributions_and_v1_optimizers() +
+          combinations.distributions_and_v2_optimizers(),
+          combinations.combine(mode=["graph", "eager"])))
+  def testOptimizerInsideModelFn(self, distribution, optimizer_fn):
+    created_variables = []
+    trainable_variables = []
+
+    def appending_creator(next_creator, *args, **kwargs):
+      v = next_creator(*args, **kwargs)
+      created_variables.append(v.name)
+      if "trainable" in kwargs and kwargs["trainable"]:
+        trainable_variables.append(v.name)
+      return v
+
+    # Creator scope needs to be set before it's used inside
+    # `distribution.scope`.
+    with variable_scope.variable_creator_scope(
+        appending_creator), distribution.scope():
+      model_fn, dataset, layer = minimize_loss_example(
+          optimizer_fn,
+          use_bias=True,
+          use_callable_loss=True,
+          create_optimizer_inside_model_fn=True)
+
+      iterator = distribution.distribute_dataset(dataset)
+
+      def run_step():
+        return distribution.group(
+            distribution.call_for_each_tower(
+                model_fn, iterator.get_next(), run_concurrently=layer.built))
+
+      if not context.executing_eagerly():
+        with self.test_session() as sess:
+          run_step = sess.make_callable(run_step())
+        self.evaluate(variables_lib.global_variables_initializer())
+
+      run_step()
+
+      def get_expected_variables(optimizer_fn, num_parameter_devices):
+        variables_map = {
+            "GradientDescent": ["dense/kernel", "dense/bias"],
+            "Adam": [
+                "dense/kernel", "dense/bias", "beta1_power", "beta2_power",
+                "dense/kernel/Adam", "dense/kernel/Adam_1", "dense/bias/Adam",
+                "dense/bias/Adam_1"
+            ]
+        }
+        variables = variables_map[optimizer_fn().get_name()]
+        variables.extend([
+            v + "/replica_{}".format(replica)
+            for v in variables
+            for replica in range(1, num_parameter_devices)
+        ])
+        return set([v + ":0" for v in variables])
+
+      self.assertEqual(
+          get_expected_variables(optimizer_fn,
+                                 len(distribution.parameter_devices)),
+          set(created_variables))
+
+  @combinations.generate(
+      combinations.times(combinations.distributions_and_v1_optimizers(),
+                         combinations.combine(
+                             mode=["graph", "eager"],
+                             momentum=[0.8, 0.9, 0.99],
+                             renorm=[False, True])))
+  def testTrainNetworkWithBatchNorm(self, distribution, optimizer_fn, momentum,
+                                    renorm):
+    """Verifies that moving mean updates are reduced across towers."""
+    with distribution.scope():
+      num_towers = len(distribution.worker_devices)
+      model_fn, dataset, batchnorm = batchnorm_example(
+          optimizer_fn,
+          batch_per_epoch=num_towers,
+          momentum=momentum,
+          renorm=renorm)
+
+      # Disable prefetching since that makes the specific input on each device
+      # to be non deterministic, and this test relies on specific input being
+      # on each device.
+      if isinstance(distribution, mirrored_strategy.MirroredStrategy):
+        distribution._prefetch_on_device = False
+      iterator = distribution.distribute_dataset(dataset)
+
+      def run_step():
+        return control_flow_ops.group(
+            distribution.unwrap(
+                distribution.call_for_each_tower(
+                    model_fn,
+                    iterator.get_next(),
+                    run_concurrently=batchnorm.built)) +
+            ops.get_collection(ops.GraphKeys.UPDATE_OPS))
+
+      if not context.executing_eagerly():
+        with self.test_session() as sess:
+          run_step = sess.make_callable(run_step())
+        self.evaluate(variables_lib.global_variables_initializer())
+
+      expected_moving_means = [0.] * 8
+
+      def averaged_batch_mean(i):
+        # Each batch has shape [16, 8] where the ith element in jth list is
+        # (8 * j + i + tower_id * 100). So the batch mean in each tower is
+        # (60 + i + tower_id * 100). So here comes its batch mean over all
+        # towers:
+        return 60. + i + (num_towers - 1.) / 2. * 100.
+
+      for _ in range(10):
+        run_step()
+        moving_means = self.evaluate(distribution.fetch(batchnorm.moving_mean))
+
+        # We make sure that the moving_mean is updated as if the sample mean is
+        # calculated over all towers.
+        for i, expected_moving_mean in enumerate(expected_moving_means):
+          expected_moving_means[i] -= ((
+              expected_moving_mean - averaged_batch_mean(i)) * (1.0 - momentum))
+          self.assertNear(expected_moving_means[i], moving_means[i], 0.0001)
+
+  @combinations.generate(
+      combinations.times(
+          combinations.combine(
+              distribution=[combinations.one_device_strategy,
+                            combinations.mirrored_strategy_with_gpu_and_cpu,
+                            combinations.mirrored_strategy_with_two_gpus],
+              optimizer_fn=[combinations.gradient_descent_optimizer_v1_fn,
+                            combinations.gradient_descent_optimizer_v2_fn],
+              loss_reduction=[losses_impl.Reduction.SUM,
+                              losses_impl.Reduction.MEAN,
+                              losses_impl.Reduction.SUM_OVER_BATCH_SIZE,
+                              losses_impl.Reduction.SUM_OVER_NONZERO_WEIGHTS]),
+          combinations.combine(mode=["graph"], use_callable_loss=[True, False])
+          + combinations.combine(mode=["eager"], use_callable_loss=[True])))
+  def testMeanVsSum(self, distribution, optimizer_fn, loss_reduction,
+                    use_callable_loss):
+    with distribution.scope():
+      all_vars = []
+
+      def model_fn(x, y):
+
+        def loss_fn():
+          # Use fixed initialization to make the steps deterministic.
+          w = variable_scope.get_variable("w", initializer=[[2.]])
+          all_vars.append(w)
+          predict = math_ops.matmul(x, w)
+          return losses_impl.mean_squared_error(
+              y, predict, reduction=loss_reduction)
+
+        optimizer = optimizer_fn()  # GradientDescent with 0.2 learning rate
+
+        if use_callable_loss:
+          return optimizer.minimize(loss_fn)
+        else:
+          return optimizer.minimize(loss_fn())
+
+      features = dataset_ops.Dataset.from_tensors([[2.], [7.]])
+      labels = dataset_ops.Dataset.from_tensors([[6.], [21.]])
+      dataset = dataset_ops.Dataset.zip((features, labels)).repeat()
+      iterator = distribution.distribute_dataset(dataset)
+
+      def run_step():
+        return distribution.group(
+            distribution.call_for_each_tower(
+                model_fn, *iterator.get_next(), run_concurrently=False))
+
+      if not context.executing_eagerly():
+        with self.test_session() as sess:
+          run_step = sess.make_callable(run_step())
+        self.evaluate(variables_lib.global_variables_initializer())
+
+      run_step()
+
+      self.assertEqual(distribution.num_towers, len(all_vars))
+      v = all_vars[0]
+      self.assertTrue(all([v is vi for vi in all_vars[1:]]))
+      weight = numpy.squeeze(self.evaluate(distribution.fetch(v)))
+      # Our model is:
+      #   predict = x * w
+      #   loss = (predict - y)^2
+      #   dloss/dpredict = 2*(predict - y)
+      #   dloss/dw = 2 * x^T @ (predict - y)
+      # For our batch size of 2, assuming sum loss reduction:
+      #   x = [2, 7]
+      #   y = [6, 21]
+      #   w_initial = 2
+      #   predict = [4, 14]
+      #   predict - y = [-2, -7]
+      #   dloss/dw = 2 <[2, 7], [-2, -7]> = - 2(4 + 49) = -106
+      # So unreplicated the update to w with lr=0.2 is -0.2 * -106 = 21.2
+      # with sum loss reduction, or 10.6 with mean.
+      if loss_reduction == losses_impl.Reduction.SUM:
+        # Note that the "distribution.num_towers" factor will go away once
+        # we split the input across towers, instead of pulling a complete
+        # batch of input per tower.
+        self.assertNear(weight, 2 + 21.2 * distribution.num_towers, 0.0001)
+      else:
+        # One of the mean loss reductions.
+        self.assertNear(weight, 2 + 10.6, 0.0001)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
new file mode 100644
index 0000000000..8cf83c52d8
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -0,0 +1,486 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class MirroredStrategy implementing DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import threading
+import six
+
+from tensorflow.contrib.distribute.python import cross_tower_ops as cross_tower_ops_lib
+from tensorflow.contrib.distribute.python import shared_variable_creator
+from tensorflow.contrib.distribute.python import values
+from tensorflow.python import pywrap_tensorflow
+from tensorflow.python.eager import context
+from tensorflow.python.eager import tape
+from tensorflow.python.framework import device as tf_device
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import coordinator
+from tensorflow.python.training import device_util
+from tensorflow.python.training import distribute as distribute_lib
+
+
+# TODO(josh11b): Replace asserts in this file with if ...: raise ...
+
+
+def _cpu_device(device):
+  cpu_device = tf_device.DeviceSpec.from_string(device)
+  cpu_device.merge_from(tf_device.DeviceSpec(device_type="CPU", device_index=0))
+  return cpu_device.to_string()
+
+
+class _RequestedStop(Exception):
+  pass
+
+
+class MirroredStrategy(distribute_lib.DistributionStrategy):
+  """Mirrors vars to distribute across multiple devices on a single machine.
+
+  This strategy uses one tower per device and sync replication.
+  """
+
+  def __init__(self,
+               devices=None,
+               num_gpus=None,
+               cross_tower_ops=None,
+               prefetch_on_device=None):
+    super(MirroredStrategy, self).__init__()
+    # Convert `num_gpus` into `devices`, shouldn't specify both.
+    if devices is None:
+      if num_gpus is None:
+        num_gpus = context.num_gpus()
+      devices = ["/device:GPU:%d" % d for d in range(num_gpus)]
+    elif num_gpus is not None:
+      raise ValueError("Must only specify one of `devices` and `num_gpus`.")
+
+    assert devices, "Must specify at least one device."
+    assert len(set(devices)) == len(devices), (
+        "No duplicates allowed in `devices` argument.")
+    # TODO(josh11b): Require at least 2 devices?
+    self._devices = devices
+    self._canonical_device_set = set(
+        [device_util.canonicalize(d) for d in devices])
+    self._device_index = values.PerDevice(
+        dict((d, i) for i, d in enumerate(devices)))
+    self.cross_tower_ops = (
+        cross_tower_ops or
+        cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps())
+    self._prefetch_on_device = prefetch_on_device
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    """Create a mirrored variable. See `DistributionStrategy.scope`."""
+    # Figure out what collections this variable should be added to.
+    # We'll add the MirroredVariable to those collections instead.
+    collections = kwargs.pop("collections", None)
+    if collections is None:
+      collections = [ops.GraphKeys.GLOBAL_VARIABLES]
+    kwargs["collections"] = []
+
+    colocate_with = kwargs.pop("colocate_with", None)
+    devices = self._get_devices_from(colocate_with)
+
+    tower_local = kwargs.pop("tower_local_reduce_method", None)
+    if tower_local is not None:
+      kwargs["trainable"] = False
+
+    # TODO(josh11b,apassos): It would be better if variable initialization
+    # was never recorded on the tape instead of having to do this manually
+    # here.
+    with tape.stop_recording():
+      index = {}
+      for i, d in enumerate(devices):
+        with ops.device(d):
+          if i > 0:
+            # Give replicas meaningful distinct names:
+            var0name = index[devices[0]].name.split(":")[0]
+            kwargs["name"] = "%s/replica_%d" % (var0name, i)
+            # Initialize replicas with the same value:
+            if context.executing_eagerly():
+              initial_value = index[devices[0]].value()
+            else:
+              initial_value = index[devices[0]].initial_value
+            kwargs["initial_value"] = array_ops.identity(initial_value)
+          with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+            v = next_creator(*args, **kwargs)
+          assert not isinstance(v, values.DistributedVariable)
+          index[d] = v
+
+      if tower_local is None:
+        result = values.MirroredVariable(index, index[devices[0]])
+      else:
+        result = values.TowerLocalVariable(
+            index, index[devices[0]], tower_local)
+
+    if not context.executing_eagerly():
+      g = ops.get_default_graph()
+      # If "trainable" is True, next_creator() will add the member variables
+      # to the TRAINABLE_VARIABLES collection, so we manually remove
+      # them and replace with the MirroredVariable. We can't set
+      # "trainable" to False for next_creator() since that causes functions
+      # like implicit_gradients to skip those variables.
+      if kwargs.get("trainable", True):
+        collections.append(ops.GraphKeys.TRAINABLE_VARIABLES)
+        l = g.get_collection_ref(ops.GraphKeys.TRAINABLE_VARIABLES)
+        for v in index.values():
+          l.remove(v)
+      g.add_to_collections(collections, result)
+    return result
+
+  def distribute_dataset(self, dataset):
+    per_device_dataset = values.PerDeviceDataset(
+        dataset, self._devices, self._prefetch_on_device)
+    return per_device_dataset.make_one_shot_iterator()
+
+  def _broadcast(self, tensor, destinations):
+    # TODO(josh11b): In eager mode, use one thread per device, or async mode.
+    return self.cross_tower_ops.broadcast(tensor, destinations or self._devices)
+
+  def _call_for_each_tower(self, fn, *args, **kwargs):
+    """Run `fn` in separate threads, once per tower/worker device.
+
+    Args:
+      fn: function to run (will be run once per device, each in its own thread).
+      *args: positional arguments for `fn`
+      **kwargs: keyword arguments for `fn`.
+          `"run_concurrently"`: Boolean indicating whether executions of `fn`
+             can be run concurrently (under eager execution only), defaults to
+             `True`.
+
+    Returns:
+      Merged return value of `fn` across all towers.
+
+    Raises:
+      RuntimeError: If fn() calls get_tower_context().merge_call() a different
+          number of times for when called for different devices.
+    """
+    run_concurrently = kwargs.pop("run_concurrently", True)
+    if not context.executing_eagerly():
+      # Lots of TF library code isn't thread-safe in graph mode, and
+      # there is little to be gained by turning on multithreading when
+      # constructing a graph.
+      run_concurrently = False
+      # Needed for per-thread device, etc. contexts in graph mode.
+      ops.get_default_graph().switch_to_thread_local()
+    elif run_concurrently is None:
+      run_concurrently = True
+
+    coord = coordinator.Coordinator(
+        clean_stop_exception_types=(_RequestedStop,))
+
+    shared_variable_store = {}
+
+    # TODO(isaprykin): Create these threads once instead of during every run()
+    # call.
+    threads = []
+    for index, d in enumerate(self._devices):
+      variable_creator_fn = shared_variable_creator.make_fn(
+          shared_variable_store, index)
+      t = MirroredStrategy._MirroredTowerThread(
+          self, coord, d, variable_creator_fn, fn,
+          *values.select_device(d, args), **values.select_device(d, kwargs))
+      threads.append(t)
+
+    for t in threads:
+      t.start()
+
+    # When `fn` starts `should_run` event is set on _MirroredTowerThread
+    # (`MTT`) threads. The execution waits until
+    # `MTT.has_paused` is set, which indicates that either `fn` is
+    # complete or a `get_tower_context().merge_call()` is called.  If `fn` is
+    # complete, then `MTT.done` is set to True.  Otherwise, arguments
+    # of `get_tower_context().merge_call` from all paused threads are grouped
+    # and the `merge_fn` is performed.  Results of the
+    # `get_tower_context().merge_call` are then set to `MTT.merge_result`.
+    # Each such `get_tower_context().merge_call` call returns the
+    # `MTT.merge_result` for that thread when `MTT.should_run` event
+    # is reset again. Execution of `fn` resumes.
+
+    try:
+      with coord.stop_on_exception():
+        all_done = False
+        while not all_done and not coord.should_stop():
+          done = []
+          if run_concurrently:
+            for t in threads:
+              t.should_run.set()
+            for t in threads:
+              t.has_paused.wait()
+              t.has_paused.clear()
+              if coord.should_stop():
+                return None
+              done.append(t.done)
+          else:
+            for t in threads:
+              t.should_run.set()
+              t.has_paused.wait()
+              t.has_paused.clear()
+              if coord.should_stop():
+                return None
+              done.append(t.done)
+          if coord.should_stop():
+            return None
+          all_done = all(done)
+          if not all_done:
+            if any(done):
+              raise RuntimeError("Some towers made a different number of "
+                                 "tower_context().merge_call() calls.")
+            # get_tower_context().merge_call() case
+            merge_args = values.regroup(
+                {t.device: t.merge_args for t in threads})
+            merge_kwargs = values.regroup(
+                {t.device: t.merge_kwargs for t in threads})
+            merge_result = threads[0].merge_fn(
+                self, *merge_args, **merge_kwargs)
+            for t in threads:
+              t.merge_result = values.select_device(t.device, merge_result)
+    finally:
+      for t in threads:
+        t.should_run.set()
+      coord.join(threads)
+
+    return values.regroup({t.device: t.main_result for t in threads})
+
+  def map(self, map_over, fn, *args, **kwargs):
+    # TODO(josh11b): In eager mode, use one thread per device.
+    index = {}
+    i = 0
+    for m in map_over:
+      d = self._devices[i % len(self._devices)]
+      with ops.device(d):
+        l = index.get(d, [])
+        l.append(fn(m,
+                    *values.select_device_mirrored(d, args),
+                    **values.select_device_mirrored(d, kwargs)))
+        index[d] = l
+    # TODO(josh11b): Need a values.regroup equivalent that handles MapOutput
+    # in addition to PerDevice data.
+    return values.PerDevice({k: values.MapOutput(v) for k, v in index.items()})
+
+  def _reduce(self, method_string, value, destinations):
+    if len(self._devices) == 1 and not isinstance(value, values.PerDevice):
+      value = values.PerDevice({self._devices[0]: value})
+    assert isinstance(value, values.PerDevice)
+    return self.cross_tower_ops.reduce(
+        method_string, value, destinations=destinations)
+
+  def _batch_reduce(self, method_string, value_destination_pairs):
+    return self.cross_tower_ops.batch_reduce(method_string,
+                                             value_destination_pairs)
+
+  def _update(self, var, fn, *args, **kwargs):
+    # TODO(josh11b): Also support TowerLocalVariables here? If so, args and
+    # kwargs don't need to be mirrored.
+    assert isinstance(var, values.MirroredVariable)
+    # TODO(josh11b): In eager mode, use one thread per device.
+    updates = {}
+    for d, v in var._index.items():  # pylint: disable=protected-access
+      name = "update_%d" % self._device_index.get(d)
+      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
+        updates[d] = fn(v,
+                        *values.select_device_mirrored(d, args),
+                        **values.select_device_mirrored(d, kwargs))
+    return values.regroup(updates, values.Mirrored)
+
+  def _update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    assert isinstance(colocate_with, list)
+    # TODO(josh11b): In eager mode, use one thread per device.
+    updates = {}
+    for d in colocate_with:
+      name = "update_%d" % self._device_index.get(d)
+      with ops.device(d), distribute_lib.UpdateContext(d), ops.name_scope(name):
+        updates[d] = fn(*values.select_device_mirrored(d, args),
+                        **values.select_device_mirrored(d, kwargs))
+    return values.regroup(updates, values.Mirrored)
+
+  def _fetch(self, val, destination, fn):
+    """Return a copy of `val` or `fn(val)` on `destination`."""
+    assert isinstance(destination, six.string_types)
+    if isinstance(val, values.TowerLocalVariable):
+      val = self.reduce(val.reduce_method, val, destinations=destination)
+      with ops.device(destination):
+        return fn(self.unwrap(val)[0])
+
+    assert isinstance(val, values.Mirrored), (
+        "val = %s (type %s)" % (val, val.__class__.__name__))
+    if val.on_device(destination):
+      with ops.device(destination):
+        # Use an identity here to make sure we are returning a tensor
+        # instead of e.g. a variable object.
+        return array_ops.identity(fn(val.get(destination)))
+    device = None
+    for d in self._devices:
+      if val.on_device(d):
+        device = d
+        break
+    assert device is not None, (
+        "Could not find destination %s in list of devices %s." %
+        (destination, val.devices))
+    with ops.device(device):
+      v = fn(val.get(device))
+    with ops.device(destination):
+      return array_ops.identity(v)
+
+  def _unwrap(self, val):
+    if isinstance(val, values.DistributedValues):
+      # Return in a deterministic order.
+      if set(val.devices) == self._canonical_device_set:
+        return [val.get(device=d) for d in self._devices]
+      return [val.get(device=d) for d in sorted(val.devices)]
+    return [val]
+
+  @property
+  def is_single_tower(self):
+    return len(self._devices) == 1
+
+  @property
+  def num_towers(self):
+    return len(self._devices)
+
+  def _worker_device_index(self):
+    return self._device_index
+
+  @property
+  def worker_devices(self):
+    # Make a copy to prevent users from accidentally mutating our copy.
+    return list(self._devices)
+
+  @property
+  def parameter_devices(self):
+    return list(self._devices)
+
+  def non_slot_devices(self, var_list):
+    del var_list
+    return list(self._devices)
+
+  def _get_devices_from(self, colocate_with=None):
+    if colocate_with is None:
+      return self._devices
+    elif isinstance(colocate_with, values.DistributedValues):
+      # pylint: disable=protected-access
+      return list(colocate_with._index.keys())
+    elif isinstance(colocate_with, six.string_types):
+      return [colocate_with]
+    else:
+      return colocate_with
+
+  class _MirroredTowerThread(threading.Thread):
+    """A thread that runs() a function on a device."""
+
+    def __init__(self, dist, coord, device, variable_creator_fn, fn, *args,
+                 **kwargs):
+      super(MirroredStrategy._MirroredTowerThread, self).__init__()  # pylint: disable=protected-access
+      self.coord = coord
+      self.distribution = dist
+      self.device = device
+      self.tower_id = dist.worker_devices.index(device)
+      self.variable_creator_fn = variable_creator_fn
+      # State needed to run and return the results of `fn`.
+      self.main_fn = fn
+      self.main_args = args
+      self.main_kwargs = kwargs
+      self.main_result = None
+      self.done = False
+      # State needed to run the next merge_call() (if any) requested via
+      # TowerContext.
+      self.merge_fn = None
+      self.merge_args = None
+      self.merge_kwargs = None
+      self.merge_result = None
+      # We use a thread.Event for the main thread to signal when this
+      # thread should start running (`should_run`), and another for
+      # this thread to transfer control back to the main thread
+      # (`has_paused`, either when it gets to a
+      # `get_tower_context().merge_call` or when `fn` returns). In
+      # either case the event starts cleared, is signaled by calling
+      # set(). The receiving thread waits for the signal by calling
+      # wait() and then immediately clearing the event using clear().
+      self.should_run = threading.Event()
+      self.has_paused = threading.Event()
+      # These fields have to do with inheriting various contexts from the
+      # parent thread:
+      # pylint: disable=protected-access
+      self.context_mode = context.context()._eager_context.mode
+      if not context.context()._context_handle:
+        context.context()._initialize_handle_and_devices()
+      self.context_device_policy = (
+          pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy(
+              context.context()._context_handle))
+      self.graph = ops.get_default_graph()
+      self._variable_creator_stack = self.graph._variable_creator_stack[:]
+      self._captured_var_scope = variable_scope.get_variable_scope()
+      # Adding a "/" at end lets us re-enter this scope later.
+      self._captured_name_scope = self.graph.get_name_scope()
+      if self._captured_name_scope:
+        self._captured_name_scope += "/"
+      if self.tower_id > 0:
+        if not self._captured_name_scope:
+          self._captured_name_scope = ""
+        self._captured_name_scope += "tower_%d/" % self.tower_id
+
+    def run(self):
+      # pylint: disable=protected-access
+      self.graph._variable_creator_stack = self._variable_creator_stack
+      self.should_run.wait()
+      self.should_run.clear()
+      try:
+        if self.coord.should_stop():
+          return
+        with self.coord.stop_on_exception(), \
+            context.context()._mode(self.context_mode), \
+            context.context().device_policy(self.context_device_policy), \
+            self.graph.as_default(), \
+            MirroredTowerContext(self.distribution, self.tower_id), \
+            ops.device(self.device), \
+            ops.name_scope(self._captured_name_scope), \
+            variable_scope.variable_scope(
+                self._captured_var_scope, reuse=self.tower_id > 0), \
+            variable_scope.variable_creator_scope(self.variable_creator_fn):
+          self.main_result = self.main_fn(*self.main_args, **self.main_kwargs)
+          self.done = True
+      finally:
+        self.has_paused.set()
+
+
+class MirroredTowerContext(distribute_lib.TowerContext):
+  """TowerContext used in MirroredStrategy.call_for_each_tower().
+
+  Opened in `_MirroredTowerThread`, to allow the user to invoke
+  `MirroredStrategy`'s specific implementation of `merge_call()`,
+  which works by delegating the function and its arguments to
+  the main thread (the one that invoked
+  `MirroredStrategy.call_for_each_tower()`).
+  """
+
+  def _merge_call(self, fn, *args, **kwargs):
+    """Delegate to the main thread to actually perform merge_call()."""
+    t = threading.current_thread()  # a _MirroredTowerThread
+    t.merge_fn = fn
+    t.merge_args = args
+    t.merge_kwargs = kwargs
+    t.has_paused.set()
+    t.should_run.wait()
+    t.should_run.clear()
+    if t.coord.should_stop():
+      raise _RequestedStop()
+    return t.merge_result
+
+  @property
+  def device(self):
+    distribute_lib.require_tower_context(self)
+    return self._distribution_strategy.worker_devices[self._tower_id]
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
new file mode 100644
index 0000000000..9e9f06da8e
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_multigpu_test.py
@@ -0,0 +1,435 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Multi-GPU tests for MirroredStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import sys
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.contrib.distribute.python import strategy_test_lib
+from tensorflow.contrib.distribute.python import values
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.layers import core
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.training import distribute as distribute_lib
+
+GPU_TEST = "test_gpu" in sys.argv[0]
+
+
+class MirroredTwoDeviceDistributionTest(strategy_test_lib.DistributionTestBase):
+
+  def _get_distribution_strategy(self):
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    if GPU_TEST:
+      self.assertGreater(context.num_gpus(), 0)
+      if context.num_gpus() > 1:
+        devices = ["/device:GPU:0", "/device:GPU:1"]
+    print(self.id().split(".")[-1], "devices:", ", ".join(devices))
+    return mirrored_strategy.MirroredStrategy(devices)
+
+  def testMinimizeLossEager(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_minimize_loss_eager(self._get_distribution_strategy())
+
+  def testMinimizeLossGraph(self):
+    soft_placement = not GPU_TEST
+    print("testMinimizeLossGraph soft_placement:", soft_placement)
+    self._test_minimize_loss_graph(
+        self._get_distribution_strategy(), soft_placement=soft_placement)
+
+  def testMapReduce(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_map_reduce(self._get_distribution_strategy())
+
+  def testDeviceIndex(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_device_index(self._get_distribution_strategy())
+
+  def testTowerId(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_tower_id(self._get_distribution_strategy())
+
+  def testNumTowers(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self.assertEqual(2, self._get_distribution_strategy().num_towers)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testCallAndMergeExceptions(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testRunRegroupError(self):
+
+    def run_fn(device_id):
+      # Generates a list with different lengths on different devices.
+      # Will fail in _regroup() (if more than one device).
+      return list(range(device_id))
+
+    dist = self._get_distribution_strategy()
+    with dist.scope(), self.assertRaises(AssertionError):
+      dist.call_for_each_tower(run_fn, dist.worker_device_index)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testReduceToCpu(self):
+    if not GPU_TEST:
+      self.skipTest("Not GPU test")
+
+    def run_fn(device_id):
+      return device_id
+
+    dist = self._get_distribution_strategy()
+    with dist.scope():
+      result = dist.call_for_each_tower(run_fn, dist.worker_device_index)
+      reduced = dist.reduce("sum", result, destinations="/device:CPU:0")
+      unwrapped = dist.unwrap(reduced)
+      self.assertEqual(1, len(unwrapped))
+      expected = sum(range(len(dist.worker_devices)))
+      self.assertEqual(expected, self.evaluate(unwrapped[0]))
+
+
+@test_util.with_c_api
+class MirroredStrategyVariableCreationTest(test.TestCase):
+
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  def _skip_eager_if_gpus_less_than(self, num_gpus):
+    if context.num_gpus() < num_gpus and context.executing_eagerly():
+      self.skipTest("Enough GPUs not available for this test in eager mode.")
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSingleVariable(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn():
+      # This variable should be created only once across the threads because of
+      # special variable_creator functions used by `dist.call_for_each_tower`.
+      v = variable_scope.variable(1.0, name="foo")
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      self.assertIsInstance(result, values.MirroredVariable)
+      self.assertEquals("foo:0", result.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testUnnamedVariable(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn():
+      v = variable_scope.variable(1.0)
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      self.assertIsInstance(result, values.MirroredVariable)
+      # Default name of "Variable" will be used.
+      self.assertEquals("Variable:0", result.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testMultipleVariables(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn():
+      vs = []
+      for i in range(5):
+        vs.append(variable_scope.variable(1.0, name="foo" + str(i)))
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return vs
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      for i, v in enumerate(result):
+        self.assertIsInstance(v, values.MirroredVariable)
+        self.assertEquals("foo" + str(i) + ":0", v.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testMultipleVariablesWithSameCanonicalName(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn():
+      vs = []
+      vs.append(variable_scope.variable(1.0, name="foo/bar"))
+      vs.append(variable_scope.variable(1.0, name="foo_1/bar"))
+      vs.append(variable_scope.variable(1.0, name="foo_1/bar_1"))
+      vs.append(variable_scope.variable(1.0, name="foo/bar_1"))
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return vs
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      for v in result:
+        self.assertIsInstance(v, values.MirroredVariable)
+      self.assertEquals(4, len(result))
+      self.assertEquals("foo/bar:0", result[0].name)
+      self.assertEquals("foo_1/bar:0", result[1].name)
+      self.assertEquals("foo_1/bar_1:0", result[2].name)
+      self.assertEquals("foo/bar_1:0", result[3].name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testVariableWithSameCanonicalNameAcrossThreads(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn(device_id):
+      v = variable_scope.variable(1.0, name="foo_" + str(device_id))
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(
+          model_fn, dist.worker_device_index, run_concurrently=False)
+      self.assertIsInstance(result, values.MirroredVariable)
+      # The resulting mirrored variable will use the name from the first device.
+      self.assertEquals("foo_0:0", result.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testWithLayers(self):
+    self._skip_eager_if_gpus_less_than(1)
+    def model_fn(features):
+      with variable_scope.variable_scope("common"):
+        layer1 = core.Dense(1)
+        layer1(features)
+        layer2 = core.Dense(1)
+        layer2(features)
+        # This will pause the current thread, and execute the other thread.
+        distribute_lib.get_tower_context().merge_call(lambda _: _)
+        layer3 = core.Dense(1)
+        layer3(features)
+        return [(layer1.kernel, layer1.bias),
+                (layer2.kernel, layer2.bias),
+                (layer3.kernel, layer3.bias)]
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+    features = dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
+    features = dist.distribute_dataset(features).get_next()
+
+    with dist.scope():
+      result = dist.call_for_each_tower(
+          model_fn, features, run_concurrently=False)
+      suffixes = ["", "_1", "_2"]
+      for (kernel, bias), suffix in zip(result, suffixes):
+        self.assertIsInstance(kernel, values.MirroredVariable)
+        self.assertEquals("common/dense" + suffix + "/kernel:0", kernel.name)
+        self.assertIsInstance(bias, values.MirroredVariable)
+        self.assertEquals("common/dense" + suffix + "/bias:0", bias.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testWithGetVariableAndVariableScope(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn():
+      v0 = variable_scope.get_variable("var-thread0", [1])
+      with variable_scope.variable_scope("common"):
+        v1 = variable_scope.get_variable("var-thread1", [1])
+        # This will pause the current thread, and execute the other thread.
+        distribute_lib.get_tower_context().merge_call(lambda _: _)
+        v2 = variable_scope.get_variable("var-thread2", [1])
+
+      return v0, v1, v2
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dist = mirrored_strategy.MirroredStrategy(devices)
+    with dist.scope():
+      with variable_scope.variable_scope("main"):
+        v = variable_scope.get_variable("var-main0", [1])
+        self.assertEquals("main/var-main0:0", v.name)
+
+        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+        self.assertEquals(3, len(result))
+        v0, v1, v2 = result
+        self.assertIsInstance(v0, values.MirroredVariable)
+        self.assertEquals("main/var-thread0:0", v0.name)
+        self.assertIsInstance(v1, values.MirroredVariable)
+        self.assertEquals("main/common/var-thread1:0", v1.name)
+        self.assertIsInstance(v2, values.MirroredVariable)
+        self.assertEquals("main/common/var-thread2:0", v2.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testThreeDevices(self):
+    self._skip_eager_if_gpus_less_than(2)
+
+    def model_fn():
+      v = variable_scope.variable(1.0, name="foo")
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:GPU:1", "/device:CPU:0"])
+
+    with dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      self.assertIsInstance(result, values.MirroredVariable)
+      self.assertEquals("foo:0", result.name)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testNonMatchingVariableCreation(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    def model_fn(name):
+      v = variable_scope.variable(1.0, name=name)
+      distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      names = values.DistributedValues({
+          "/device:CPU:0": "foo",
+          "/device:GPU:0": "bar"
+      })
+      with self.assertRaises(RuntimeError):
+        _ = dist.call_for_each_tower(model_fn, names, run_concurrently=False)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testTowerLocalVariable(self):
+    self._skip_eager_if_gpus_less_than(1)
+
+    all_v_sum = {}
+    all_v_mean = {}
+
+    def model_fn(device_id):
+      tower_context = distribute_lib.get_tower_context()
+      with tower_context.tower_local_var_scope("sum"):
+        v_sum = variable_scope.variable(1.0)
+      with tower_context.tower_local_var_scope("mean"):
+        v_mean = variable_scope.variable(4.0)
+      self.assertTrue(isinstance(v_sum, values.TowerLocalVariable))
+      self.assertTrue(isinstance(v_mean, values.TowerLocalVariable))
+      updates = [v_sum.assign_add(2.0 + device_id),
+                 v_mean.assign(6.0 * device_id)]
+      all_v_sum[device_id] = v_sum
+      all_v_mean[device_id] = v_mean
+      return updates, v_sum, v_mean
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with dist.scope():
+      # Create "sum" and "mean" versions of TowerLocalVariables.
+      ret_ops, ret_v_sum, ret_v_mean = dist.call_for_each_tower(
+          model_fn, dist.worker_device_index, run_concurrently=False)
+      # Should see the same wrapping instance in all towers.
+      self.assertIs(all_v_sum[0], ret_v_sum)
+      self.assertIs(all_v_mean[0], ret_v_mean)
+      for i in range(1, dist.num_towers):
+        self.assertIs(all_v_sum[0], all_v_sum[1])
+        self.assertIs(all_v_mean[0], all_v_mean[1])
+
+      # Apply updates
+      self.evaluate(variables.global_variables_initializer())
+      self.evaluate([y for x in ret_ops for y in dist.unwrap(x)])
+      expected_sum = 0.0
+      expected_mean = 0.0
+      for i, d in enumerate(dist.worker_devices):
+        # Test access within a device scope, should see different values.
+        with ops.device(d):
+          v_sum_value = self.evaluate(ret_v_sum.read_value())
+          v_mean_value = self.evaluate(ret_v_mean.read_value())
+          expected = i + 3.0
+          self.assertEqual(expected, v_sum_value)
+          expected_sum += expected
+          expected = i * 6.0
+          self.assertEqual(expected, v_mean_value)
+          expected_mean += expected
+
+      # fetch() should return the value you get by applying the
+      # reduction across all towers.
+      self.assertEqual(expected_sum, self.evaluate(dist.fetch(ret_v_sum)))
+      expected_mean /= len(dist.worker_devices)
+      self.assertEqual(expected_mean, self.evaluate(dist.fetch(ret_v_mean)))
+
+  # NOTE(priyag): Names and name scopes are ignored in eager, hence we are not
+  # testing this in eager mode.
+
+  def testNameScope(self):
+    def model_fn():
+      with ops.name_scope("foo"):
+        a = constant_op.constant(1.0, name="a")
+        distribute_lib.get_tower_context().merge_call(lambda _: _)
+        b = constant_op.constant(1.0, name="b")
+      return a, b
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with context.graph_mode(), dist.scope():
+      with ops.name_scope("main"):
+        result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+        self.assertEquals(2, len(result))
+        for v, name in zip(result, ["a", "b"]):
+          self.assertIsInstance(v, values.DistributedValues)
+          v0, v1 = dist.unwrap(v)
+          self.assertEquals("main/foo/" + name + ":0", v0.name)
+          self.assertEquals("main/tower_1/foo/" + name + ":0", v1.name)
+
+  def testWithDefaultName(self):
+    def model_fn():
+      with ops.name_scope(None, "foo"):
+        a = constant_op.constant(1.0, name="a")
+        distribute_lib.get_tower_context().merge_call(lambda _: _)
+        b = constant_op.constant(2.0, name="b")
+      return a, b
+
+    dist = mirrored_strategy.MirroredStrategy(
+        ["/device:GPU:0", "/device:CPU:0"])
+
+    with context.graph_mode(), dist.scope():
+      result = dist.call_for_each_tower(model_fn, run_concurrently=False)
+      self.assertEquals(2, len(result))
+      for v, name in zip(result, ["a", "b"]):
+        self.assertIsInstance(v, values.DistributedValues)
+        v0, v1 = dist.unwrap(v)
+        self.assertEquals("foo/" + name + ":0", v0.name)
+        self.assertEquals("tower_1/foo/" + name + ":0", v1.name)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy_test.py b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
new file mode 100644
index 0000000000..a1ef0ecc77
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy_test.py
@@ -0,0 +1,91 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for class MirroredStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.contrib.distribute.python import strategy_test_lib
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import distribute as distribute_lib
+
+
+@test_util.with_c_api
+class MirroredOneCPUDistributionTest(strategy_test_lib.DistributionTestBase):
+
+  def _get_distribution_strategy(self):
+    return mirrored_strategy.MirroredStrategy(["/device:CPU:0"])
+
+  def testMinimizeLossEager(self):
+    self._test_minimize_loss_eager(self._get_distribution_strategy())
+
+  def testMinimizeLossGraph(self):
+    self._test_minimize_loss_graph(self._get_distribution_strategy())
+
+  def testMapReduce(self):
+    self._test_map_reduce(self._get_distribution_strategy())
+
+  def testDeviceIndex(self):
+    self._test_device_index(self._get_distribution_strategy())
+
+  def testTowerId(self):
+    self._test_tower_id(self._get_distribution_strategy())
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testCallAndMergeExceptions(self):
+    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
+
+
+@test_util.with_c_api
+class VariableCreatorStackTest(test.TestCase):
+
+  def testCreatorStacksAreThreadLocal(self):
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dist = mirrored_strategy.MirroredStrategy(devices)
+
+    def model_fn(device_id):
+      assert isinstance(device_id, int)
+      def thread_creator_fn(next_creator, *args, **kwargs):
+        return next_creator(*args, **kwargs) + ":thread_" + str(device_id)
+
+      with variable_scope.variable_creator_scope(thread_creator_fn):
+        # Create a variable in this scope.
+        v = variable_scope.variable(1.0)
+
+        # This will pause the current thread, and execute the other thread.
+        distribute_lib.get_tower_context().merge_call(lambda _: _)
+      return v
+
+    def main_thread_creator(next_creator, *args, **kwargs):
+      # We are not using the underlying next_creator for test purposes.
+      del next_creator, args, kwargs
+      return "main_thread"
+
+    with context.graph_mode(), \
+        dist.scope(), \
+        variable_scope.variable_creator_scope(main_thread_creator):
+      result = dist.call_for_each_tower(model_fn, dist.worker_device_index)
+      result = dist.unwrap(result)
+      expected = ["main_thread:thread_0", "main_thread:thread_1"]
+      self.assertEquals(expected, result)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/monitor.py b/tensorflow/contrib/distribute/python/monitor.py
new file mode 100644
index 0000000000..fe80bb4df5
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/monitor.py
@@ -0,0 +1,61 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Monitor is responsible for training, checkpointing and recovery."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import context
+from tensorflow.python.ops import variables
+
+
+class Monitor(object):
+  """Executes training steps, recovers and checkpoints.
+
+  Note that this class is particularly preliminary, experimental, and
+  expected to change.
+  """
+  # TODO(isaprykin): Support step functions that need multiple session calls.
+  # TODO(isaprykin): Support extra arguments to the step function.
+  # TODO(isaprykin): Support recovery, checkpointing and summaries.
+
+  def __init__(self, step_callable, session=None):
+    """Initialize the Monitor with components for executing training steps.
+
+    Args:
+      step_callable: a training `Step` that's capable of signaling when done.
+      session: a `Session` instance that's needed for graph mode.
+
+    Raises:
+      ValueError: if `session` was provided for eager mode or not provided for
+        graph mode.
+    """
+    if context.executing_eagerly():
+      if session is not None:
+        raise ValueError("Should not provide a `session` in Eager mode.")
+      self._run_step = step_callable
+    else:
+      if session is None:
+        raise ValueError("Should provide a `session` in Graph mode.")
+      self._run_step = session.make_callable(step_callable())
+      session.run(variables.global_variables_initializer())
+
+  def run_steps(self, num_steps=None):
+    step = 0
+    done = False
+    while done is not None and (num_steps is None or step < num_steps):
+      done = self._run_step()
+      step += 1
diff --git a/tensorflow/contrib/distribute/python/monitor_test.py b/tensorflow/contrib/distribute/python/monitor_test.py
new file mode 100644
index 0000000000..8277e1e791
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/monitor_test.py
@@ -0,0 +1,84 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for class Monitor."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python import monitor as monitor_lib
+from tensorflow.contrib.distribute.python import one_device_strategy
+from tensorflow.contrib.distribute.python.single_loss_example import single_loss_example
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import ops
+from tensorflow.python.training import gradient_descent
+
+
+class MonitorTest(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.times(
+          combinations.distributions_and_v1_optimizers(),
+          combinations.combine(mode=combinations.graph_and_eager_modes)))
+  def testTrainNetwork(self, distribution, optimizer_fn):
+    with distribution.scope():
+      single_loss_step, layer = single_loss_example(optimizer_fn, distribution)
+
+      if context.executing_eagerly():
+        monitor = monitor_lib.Monitor(single_loss_step, None)
+      else:
+        with self.test_session() as sess:
+          monitor = monitor_lib.Monitor(single_loss_step, sess)
+
+      monitor.run_steps(1)
+
+      self.assertEqual(1, len(layer.trainable_variables))
+      mirrored_weight_variable = layer.trainable_variables[0]
+      start_error = self.evaluate(distribution.fetch(mirrored_weight_variable))
+      start_error = abs(numpy.array(start_error) - 1)
+
+      monitor.run_steps(9)
+      end_error = self.evaluate(distribution.fetch(mirrored_weight_variable))
+      end_error = abs(numpy.array(end_error) - 1)
+      self.assertGreaterEqual(start_error, end_error)
+
+  def testPassingASessionInEager(self):
+    distribution = one_device_strategy.OneDeviceStrategy(
+        "/device:CPU:0")
+    step_function, _ = single_loss_example(
+        lambda: gradient_descent.GradientDescentOptimizer(0.2), distribution)
+
+    with self.test_session() as sess:
+      with self.assertRaisesRegexp(ValueError, "Should not provide"):
+        _ = monitor_lib.Monitor(step_function, sess)
+
+  def testNotPassingASessionInGraph(self):
+    distribution = one_device_strategy.OneDeviceStrategy(
+        "/device:CPU:0")
+    step_function, _ = single_loss_example(
+        lambda: gradient_descent.GradientDescentOptimizer(0.2), distribution)
+
+    with context.graph_mode(), ops.Graph().as_default():
+      with self.assertRaisesRegexp(ValueError, "Should provide"):
+        _ = monitor_lib.Monitor(step_function, session=None)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy.py b/tensorflow/contrib/distribute/python/one_device_strategy.py
new file mode 100644
index 0000000000..39c49442b9
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/one_device_strategy.py
@@ -0,0 +1,148 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Class OneDeviceStrategy implementing DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import six
+
+from tensorflow.contrib.distribute.python import values
+from tensorflow.contrib.eager.python import datasets
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import distribute as distribute_lib
+
+
+# TODO(josh11b): Replace asserts in this file with if ...: raise ...
+
+
+class OneDeviceStrategy(distribute_lib.DistributionStrategy):
+  """A distribution strategy for running on a single device."""
+  # TODO(josh11b): Do we wrap values in types to generate errors if you are
+  # doing something that won't work with other DistributionStrategy
+  # implementations?
+
+  def __init__(self, device):
+    super(OneDeviceStrategy, self).__init__()
+    self._device = device
+
+  def _create_variable(self, next_creator, *args, **kwargs):
+    # No need to distinguish tower-local variables when not mirroring,
+    # we just enforce that they are not trainable.
+    if kwargs.pop("tower_local_reduce_method", None) is not None:
+      kwargs["trainable"] = False
+
+    colocate_with = kwargs.pop("colocate_with", None)
+    if colocate_with is None:
+      with ops.device(self._device):
+        return next_creator(*args, **kwargs)
+    if isinstance(colocate_with, six.string_types):
+      with ops.device(colocate_with):
+        return next_creator(*args, **kwargs)
+    if (isinstance(colocate_with, list) and len(colocate_with) == 1 and
+        isinstance(colocate_with[0], six.string_types)):
+      with ops.device(colocate_with[0]):
+        return next_creator(*args, **kwargs)
+    with ops.colocate_with(colocate_with):
+      return next_creator(*args, **kwargs)
+
+  def distribute_dataset(self, dataset):
+    if context.executing_eagerly():
+      return datasets.Iterator(dataset)
+    else:
+      return dataset.make_one_shot_iterator()
+
+  def _broadcast(self, tensor, destinations):
+    return tensor
+
+  def _call_for_each_tower(self, fn, *args, **kwargs):
+    # We don't run `fn` in multiple threads in OneDeviceStrategy.
+    kwargs.pop("run_concurrently", None)
+    with ops.device(self._device), _OneDeviceTowerContext(self):
+      return fn(*args, **kwargs)
+
+  def map(self, map_over, fn, *args, **kwargs):
+    with ops.device(self._device):
+      return values.MapOutput([fn(m, *args, **kwargs) for m in map_over])
+
+  def _reduce(self, method_string, value, destinations):
+    if not isinstance(value, values.MapOutput):
+      return value
+    l = value.get()
+    assert l
+    with ops.device(self._device):
+      if method_string == "sum":
+        return math_ops.add_n(l)
+      elif method_string == "mean":
+        return math_ops.add_n(l) / len(l)
+      else:
+        assert False
+
+  def _update(self, var, fn, *args, **kwargs):
+    with ops.device(self._device), distribute_lib.UpdateContext(self._device):
+      return fn(var, *args, **kwargs)
+
+  def _update_non_slot(self, colocate_with, fn, *args, **kwargs):
+    del colocate_with
+    with ops.device(self._device), distribute_lib.UpdateContext(self._device):
+      return fn(*args, **kwargs)
+
+  def _fetch(self, val, destination, fn):
+    """Return a copy of `val` or `fn(val)` on `destination`."""
+    with ops.device(self._device):
+      v = fn(val)
+    with ops.device(destination):
+      return array_ops.identity(v)
+
+  def _unwrap(self, value):
+    return [value]
+
+  @property
+  def is_single_tower(self):
+    return True
+
+  @property
+  def num_towers(self):
+    return 1
+
+  @property
+  def worker_devices(self):
+    return [self._device]
+
+  @property
+  def parameter_devices(self):
+    return [self._device]
+
+  def non_slot_devices(self, var_list):
+    del var_list
+    return [self._device]
+
+  def _worker_device_index(self):
+    return 0
+
+
+class _OneDeviceTowerContext(distribute_lib.TowerContext):
+
+  def __init__(self, distribution_strategy):
+    distribute_lib.TowerContext.__init__(
+        self, distribution_strategy, tower_id=0)
+
+  @property
+  def device(self):
+    return self._distribution_strategy.worker_devices[0]
diff --git a/tensorflow/contrib/distribute/python/one_device_strategy_test.py b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
new file mode 100644
index 0000000000..7101ed0756
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/one_device_strategy_test.py
@@ -0,0 +1,54 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for class OneDeviceStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import one_device_strategy
+from tensorflow.contrib.distribute.python import strategy_test_lib
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+
+
+@test_util.with_c_api
+class OneDeviceStrategyTest(strategy_test_lib.DistributionTestBase):
+
+  def _get_distribution_strategy(self):
+    return one_device_strategy.OneDeviceStrategy("/device:CPU:0")
+
+  def testMinimizeLossEager(self):
+    self._test_minimize_loss_eager(self._get_distribution_strategy())
+
+  def testMinimizeLossGraph(self):
+    self._test_minimize_loss_graph(self._get_distribution_strategy())
+
+  def testMapReduce(self):
+    self._test_map_reduce(self._get_distribution_strategy())
+
+  def testDeviceIndex(self):
+    self._test_device_index(self._get_distribution_strategy())
+
+  def testTowerId(self):
+    self._test_tower_id(self._get_distribution_strategy())
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testCallAndMergeExceptions(self):
+    self._test_call_and_merge_exceptions(self._get_distribution_strategy())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/optimizer_v2_test.py b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
new file mode 100644
index 0000000000..a0912b625f
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/optimizer_v2_test.py
@@ -0,0 +1,70 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for running legacy optimizer code with DistributionStrategy."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python.single_loss_example import minimize_loss_example
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import variables
+
+
+class MinimizeLossOptimizerV2Test(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.times(
+          combinations.distributions_and_v2_optimizers(),
+          combinations.combine(mode=["graph"], use_callable_loss=[True, False])
+          + combinations.combine(mode=["eager"], use_callable_loss=[True])))
+  def testTrainNetwork(self, distribution, optimizer_fn,
+                       use_callable_loss=True):
+    with distribution.scope():
+      model_fn, dataset, layer = minimize_loss_example(
+          optimizer_fn, use_bias=True, use_callable_loss=use_callable_loss)
+
+      iterator = distribution.distribute_dataset(dataset)
+
+      def run_step():
+        return control_flow_ops.group(distribution.unwrap(
+            distribution.call_for_each_tower(
+                model_fn, iterator.get_next(), run_concurrently=layer.built)))
+
+      if not context.executing_eagerly():
+        with self.test_session() as sess:
+          run_step = sess.make_callable(run_step())
+        self.evaluate(variables.global_variables_initializer())
+
+      weights, biases = [], []
+      for _ in range(10):
+        run_step()
+
+        weights.append(self.evaluate(distribution.fetch(layer.kernel)))
+        biases.append(self.evaluate(distribution.fetch(layer.bias)))
+
+      error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
+      is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
+      self.assertTrue(is_not_increasing)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/prefetching_ops_v2.py b/tensorflow/contrib/distribute/python/prefetching_ops_v2.py
new file mode 100644
index 0000000000..b9ffd2f266
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/prefetching_ops_v2.py
@@ -0,0 +1,167 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Extension of prefetching_ops to support more than one device."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import warnings
+
+from tensorflow.contrib.data.python.ops import contrib_op_loader  # pylint: disable=unused-import
+from tensorflow.contrib.data.python.ops import gen_dataset_ops
+from tensorflow.contrib.data.python.ops import prefetching_ops
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.data.ops import iterator_ops
+from tensorflow.python.data.util import nest as data_nest
+from tensorflow.python.data.util import sparse
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
+from tensorflow.python.framework import ops
+from tensorflow.python.util import nest
+
+
+# pylint: disable=protected-access
+class _PrefetchToDeviceIterator(object):
+  """A replacement for @{tf.data.Iterator} that prefetches to another device."""
+
+  def __init__(self, input_dataset, devices, buffer_size):
+    self._input_dataset = input_dataset
+    self._get_next_call_count = 0
+    self._devices = devices
+    input_iterator = input_dataset.make_one_shot_iterator()
+    input_iterator_handle = input_iterator.string_handle()
+
+    @function.Defun(dtypes.string)
+    def _prefetch_fn(handle):
+      remote_iterator = iterator_ops.Iterator.from_string_handle(
+          handle, input_iterator.output_types, input_iterator.output_shapes,
+          input_iterator.output_classes)
+      return remote_iterator.get_next()
+
+    target_device = gen_dataset_ops.iterator_get_device(
+        input_iterator._iterator_resource)
+    self._buffering_resources = []
+    for device in nest.flatten(self._devices):
+      with ops.device(device):
+        buffer_resource_handle = prefetching_ops.function_buffering_resource(
+            f=_prefetch_fn,
+            target_device=target_device,
+            string_arg=input_iterator_handle,
+            buffer_size=buffer_size,
+            thread_pool_size=0)
+        self._buffering_resources.append(buffer_resource_handle)
+
+  def get_next(self, name=None):
+    """See @{tf.data.Iterator.get_next}."""
+    self._get_next_call_count += 1
+    if self._get_next_call_count > iterator_ops.GET_NEXT_CALL_WARNING_THRESHOLD:
+      warnings.warn(iterator_ops.GET_NEXT_CALL_WARNING_MESSAGE)
+
+    flat_result = []
+    # TODO(priyag): This will fail if the input size (typically number of
+    # batches) is not divisible by number of devices.
+    # How do we handle that more gracefully / let the user know?
+    for buffer_resource in self._buffering_resources:
+      flat_ret = gen_dataset_ops.function_buffering_resource_get_next(
+          buffer_resource,
+          output_types=data_nest.flatten(sparse.as_dense_types(
+              self.output_types, self.output_classes)), name=name)
+
+      ret = sparse.deserialize_sparse_tensors(
+          data_nest.pack_sequence_as(self.output_types, flat_ret),
+          self.output_types, self.output_shapes, self.output_classes)
+
+      for tensor, shape in zip(
+          data_nest.flatten(ret), data_nest.flatten(self.output_shapes)):
+        if isinstance(tensor, ops.Tensor):
+          tensor.set_shape(shape)
+      flat_result.append(ret)
+
+    return nest.pack_sequence_as(self._devices, flat_result)
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes
+
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+# pylint: enable=protected-access
+
+
+class _PrefetchToDeviceDataset(dataset_ops.Dataset):
+  """A `Dataset` whose iterator prefetches elements to other device(s)."""
+
+  def __init__(self, input_dataset, devices, buffer_size):
+    self._input_dataset = input_dataset
+    self._devices = devices
+    self._buffer_size = buffer_size if buffer_size is not None else 1
+
+  def make_one_shot_iterator(self):
+    return _PrefetchToDeviceIterator(self._input_dataset, self._devices,
+                                     self._buffer_size)
+
+  def make_initializable_iterator(self, shared_name=None):
+    raise NotImplementedError("`prefetch_to_devices()` is not currently "
+                              "compatible with initializable iterators. Use "
+                              "`make_one_shot_iterator()` instead.")
+
+  def _as_variant_tensor(self):
+    # TODO(mrry): Raise this error earlier (e.g. when one of the Dataset
+    # transformation methods is called.
+    # TODO(mrry): Investigate support for chaining further transformations after
+    # the prefetch, including GPU support.
+    raise NotImplementedError("`prefetch_to_devices()` must be the last "
+                              "transformation in a dataset pipeline.")
+
+  # TODO(priyag): Fix the output types, shapes and classes to match the result
+  # of get_next (which has the additional nesting layer of devices now).
+  @property
+  def output_types(self):
+    return self._input_dataset.output_types
+
+  @property
+  def output_shapes(self):
+    return self._input_dataset.output_shapes
+
+  @property
+  def output_classes(self):
+    return self._input_dataset.output_classes
+
+
+def prefetch_to_devices(devices, buffer_size=None):
+  """A transformation that prefetches dataset values to the given `devices`.
+
+  NOTE: Although the transformation creates a @{tf.data.Dataset}, the
+  transformation must be the final `Dataset` in the input pipeline.
+
+  Args:
+    devices: A nested structure of devices on which to prefetch the data. It can
+      be a single device name, or a tuple or list of device names.
+    buffer_size: (Optional.) The number of elements to buffer on each device.
+      Defaults to an automatically chosen value.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}.
+  """
+  def _apply_fn(dataset):
+    return _PrefetchToDeviceDataset(dataset, devices, buffer_size)
+
+  return _apply_fn
diff --git a/tensorflow/contrib/distribute/python/prefetching_ops_v2_test.py b/tensorflow/contrib/distribute/python/prefetching_ops_v2_test.py
new file mode 100644
index 0000000000..8ed16f4607
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/prefetching_ops_v2_test.py
@@ -0,0 +1,68 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for prefetching_ops_v2."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import prefetching_ops_v2
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import test_util
+from tensorflow.python.platform import test
+
+
+class PrefetchingOpsV2Test(test.TestCase):
+
+  def testPrefetchToOneDevice(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops_v2.prefetch_to_devices("/gpu:0"))
+
+    iterator = device_dataset.make_one_shot_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      for i in range(10):
+        self.assertEqual(i, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+  def testPrefetchToTwoDevicesInAList(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops_v2.prefetch_to_devices(["/cpu:0", "/gpu:0"]))
+
+    iterator = device_dataset.make_one_shot_iterator()
+    next_element = iterator.get_next()
+
+    output = []
+    with self.test_session() as sess:
+      for _ in range(5):
+        result = sess.run(next_element)
+        self.assertEqual(2, len(result))
+        output.extend(result)
+      self.assertEquals(set(range(10)), set(output))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator.py b/tensorflow/contrib/distribute/python/shared_variable_creator.py
new file mode 100644
index 0000000000..aca9c7af05
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/shared_variable_creator.py
@@ -0,0 +1,97 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Utility to re-use variables created on first device on subsequent devices."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import re
+
+_VARIABLE_UNIQUIFYING_REGEX = re.compile(r"_\d/")
+_VARIABLE_UNIQUIFYING_REGEX_AT_END = re.compile(r"_\d$")
+
+
+def _canonicalize_variable_name(name):
+  # If no name is specified, uses default name "Variable".
+  if name is None:
+    return "Variable"
+  # Replace all instances of "_<num>/" with "/"
+  name = _VARIABLE_UNIQUIFYING_REGEX.sub("/", name)
+  # Replace any instances of "_<num>" at the end of the string with ""
+  name = _VARIABLE_UNIQUIFYING_REGEX_AT_END.sub("", name)
+  return name
+
+
+def make_fn(shared_variable_store, device_id):
+  """Construct the variable creator function for device `device_id`.
+
+  Constructs custom variable creator functions for the given device.
+  On first device (device_id == 0), it creates the variable using the
+  `next_creator`, and stores it in the provided `shared_variable_store`.
+  On all other devices (device_id > 0), it tries to re-use the variable
+  already created with the same name. If no such variable exists, it throws an
+  error.
+  Additionally, we de-uniquify variable names before checking for matches. This
+  helps re-use variables which are intended to be the same but have different
+  names due to variable uniquificaton happening upstream. Since this might
+  mean we may have multiple variables with the same canonical name, we store
+  them in a list per canonical name and return them in the same order as well.
+
+  Args:
+    shared_variable_store: A dictionary that we will use to store variables
+      created on the first device, and re-used by creators for other devices.
+    device_id: Integer index of the device whose creator should be
+      constructed.
+
+  Returns:
+    An appropriate creator function based on device_id.
+
+  """
+  variable_scope_access_index = {}
+  assert isinstance(device_id, int)
+
+  def create_new_variable(next_creator, *args, **kwargs):
+    """Create the variable using `next_creator` and store it."""
+    canonical_name = _canonicalize_variable_name(kwargs.get("name"))
+    v = next_creator(*args, **kwargs)
+
+    if canonical_name not in shared_variable_store:
+      shared_variable_store[canonical_name] = []
+    shared_variable_store[canonical_name].append(v)
+    return v
+
+  def reuse_variable(next_creator, *args, **kwargs):
+    """Re-use existing variable from store with same name (in order)."""
+    del next_creator, args
+    name = kwargs.get("name")
+    canonical_name = _canonicalize_variable_name(name)
+
+    try:
+      variable_index = variable_scope_access_index.get(canonical_name, 0)
+      v = shared_variable_store[canonical_name][variable_index]
+      # TODO(priyag): Make this variable re-use more robust by adding checks
+      # that the requested shape and dtype match the existing variable.
+      variable_scope_access_index[canonical_name] = variable_index + 1
+      return v
+    except (KeyError, IndexError):
+      raise RuntimeError(
+          "Tried to create variable {} with mismatching name on device {}".
+          format(name, device_id))
+
+  if device_id == 0:
+    return create_new_variable
+  else:
+    return reuse_variable
diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator_test.py b/tensorflow/contrib/distribute/python/shared_variable_creator_test.py
new file mode 100644
index 0000000000..713494d603
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/shared_variable_creator_test.py
@@ -0,0 +1,75 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for SharedVariableCreator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import shared_variable_creator
+from tensorflow.python.eager import test
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import variable_scope
+
+
+class CanonicalizeVariableNameTest(test.TestCase):
+
+  def _canonicalize(self, name):
+    return shared_variable_creator._canonicalize_variable_name(name)
+
+  def testNoName(self):
+    self.assertEquals("Variable", self._canonicalize(None))
+
+  def testPatternInMiddle(self):
+    self.assertEquals("foo/bar/baz", self._canonicalize("foo_1/bar_1/baz"))
+
+  def testPatternAtEnd(self):
+    self.assertEquals("foo", self._canonicalize("foo_1"))
+
+  def testWrongPatterns(self):
+    self.assertEquals("foo_1:0", self._canonicalize("foo_1:0"))
+    self.assertEquals("foo1", self._canonicalize("foo1"))
+    self.assertEquals("foo_a", self._canonicalize("foo_a"))
+
+
+@test_util.with_c_api
+class SharedVariableCreatorTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testSharedVariable(self):
+
+    shared_variable_store = {}
+    num_devices = 3
+    creator_fns = []
+    for i in range(num_devices):
+      creator_fn = shared_variable_creator.make_fn(shared_variable_store, i)
+      creator_fns.append(creator_fn)
+
+    with variable_scope.variable_creator_scope(creator_fns[0]):
+      v0 = variable_scope.variable(1.0, name="foo")
+
+    with variable_scope.variable_creator_scope(creator_fns[1]):
+      v1 = variable_scope.variable(1.0, name="foo")
+
+    with variable_scope.variable_creator_scope(creator_fns[2]):
+      v2 = variable_scope.variable(1.0, name="foo")
+
+    # v1 and v2 should be same as v0
+    self.assertIs(v1, v0)
+    self.assertIs(v2, v0)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/simple_estimator_example.py b/tensorflow/contrib/distribute/python/simple_estimator_example.py
new file mode 100644
index 0000000000..7095d801ad
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/simple_estimator_example.py
@@ -0,0 +1,97 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A simple example to test the a DistributionStrategy with Estimators.
+
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.estimator import estimator as estimator_lib
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.estimator import run_config
+from tensorflow.python.framework import constant_op
+from tensorflow.python.layers import core
+from tensorflow.python.ops import array_ops
+from tensorflow.python.platform import app
+from tensorflow.python.training import gradient_descent
+from tensorflow.python.training import training_util
+
+
+def build_model_fn_optimizer():
+  """Simple model_fn with optimizer."""
+  # TODO(anjalisridhar): Move this inside the model_fn once OptimizerV2 is
+  # done?
+  optimizer = gradient_descent.GradientDescentOptimizer(0.2)
+
+  def model_fn(features, labels, mode):  # pylint: disable=unused-argument
+    """model_fn which uses a single unit Dense layer."""
+    # You can also use the Flatten layer if you want to test a model without any
+    # weights.
+    layer = core.Dense(1, use_bias=True)
+    logits = layer(features)
+
+    if mode == model_fn_lib.ModeKeys.PREDICT:
+      predictions = {"logits": logits}
+      return model_fn_lib.EstimatorSpec(mode, predictions=predictions)
+
+    def loss_fn():
+      y = array_ops.reshape(logits, []) - constant_op.constant(1.)
+      return y * y
+
+    if mode == model_fn_lib.ModeKeys.EVAL:
+      return model_fn_lib.EstimatorSpec(mode, loss=loss_fn())
+
+    assert mode == model_fn_lib.ModeKeys.TRAIN
+
+    global_step = training_util.get_global_step()
+    train_op = optimizer.minimize(loss_fn(), global_step=global_step)
+    return model_fn_lib.EstimatorSpec(mode, loss=loss_fn(), train_op=train_op)
+
+  return model_fn
+
+
+def main(_):
+  distribution = mirrored_strategy.MirroredStrategy(
+      ["/device:GPU:0", "/device:GPU:1"])
+  config = run_config.RunConfig(distribute=distribution)
+
+  def input_fn():
+    features = dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
+    labels = dataset_ops.Dataset.from_tensors([1.]).repeat(10)
+    return dataset_ops.Dataset.zip((features, labels))
+
+  estimator = estimator_lib.Estimator(
+      model_fn=build_model_fn_optimizer(), config=config)
+  estimator.train(input_fn=input_fn, steps=10)
+
+  eval_result = estimator.evaluate(input_fn=input_fn)
+  print("Eval result: {}".format(eval_result))
+
+  def predict_input_fn():
+    predict_features = dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
+    return predict_features
+
+  predictions = estimator.predict(input_fn=predict_input_fn)
+  # TODO(anjalsridhar): This returns a generator object, figure out how to get
+  # meaningful results here.
+  print("Prediction results: {}".format(predictions))
+
+
+if __name__ == "__main__":
+  app.run(main)
diff --git a/tensorflow/contrib/distribute/python/single_loss_example.py b/tensorflow/contrib/distribute/python/single_loss_example.py
new file mode 100644
index 0000000000..cef5fd2f89
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/single_loss_example.py
@@ -0,0 +1,102 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""A simple network to use in tests and examples."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.distribute.python import step_fn
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.framework import constant_op
+from tensorflow.python.layers import core
+from tensorflow.python.layers import normalization
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+
+
+def single_loss_example(optimizer_fn, distribution, use_bias=False):
+  """Build a very simple network to use in tests and examples."""
+  dataset = dataset_ops.Dataset.from_tensors([[1.]]).repeat()
+  optimizer = optimizer_fn()
+  layer = core.Dense(1, use_bias=use_bias)
+
+  def loss_fn(x):
+    y = array_ops.reshape(layer(x), []) - constant_op.constant(1.)
+    return y * y
+
+  single_loss_step = step_fn.StandardSingleLossStep(dataset, loss_fn, optimizer,
+                                                    distribution)
+
+  # Layer is returned for inspecting the kernels in tests.
+  return single_loss_step, layer
+
+
+def minimize_loss_example(optimizer_fn,
+                          use_bias=False,
+                          use_callable_loss=True,
+                          create_optimizer_inside_model_fn=False):
+  """Example of non-distribution-aware legacy code."""
+  dataset = dataset_ops.Dataset.from_tensors([[1.]]).repeat()
+  # An Optimizer instance is created either outside or inside model_fn.
+  outer_optimizer = None
+  if not create_optimizer_inside_model_fn:
+    outer_optimizer = optimizer_fn()
+
+  layer = core.Dense(1, use_bias=use_bias)
+
+  def model_fn(x):
+    """A very simple model written by the user."""
+
+    def loss_fn():
+      y = array_ops.reshape(layer(x), []) - constant_op.constant(1.)
+      return y * y
+
+    optimizer = outer_optimizer or optimizer_fn()
+
+    if use_callable_loss:
+      return optimizer.minimize(loss_fn)
+    else:
+      return optimizer.minimize(loss_fn())
+
+  return model_fn, dataset, layer
+
+
+def batchnorm_example(optimizer_fn,
+                      batch_per_epoch=1,
+                      momentum=0.9,
+                      renorm=False):
+  """Example of non-distribution-aware legacy code with batch normalization."""
+  # input shape is [16, 8], input values are increasing in both dimensions.
+  dataset = dataset_ops.Dataset.from_tensor_slices(
+      [[[float(x * 8 + y + z * 100)
+         for y in range(8)]
+        for x in range(16)]
+       for z in range(batch_per_epoch)]).repeat()
+  optimizer = optimizer_fn()
+  batchnorm = normalization.BatchNormalization(
+      renorm=renorm, momentum=momentum, fused=False)
+
+  def model_fn(x):
+
+    def loss_fn():
+      y = math_ops.reduce_sum(batchnorm(x, training=True), axis=1)
+      loss = math_ops.reduce_mean(y - constant_op.constant(1.))
+      return loss
+
+    # Callable loss.
+    return optimizer.minimize(loss_fn)
+
+  return model_fn, dataset, batchnorm
diff --git a/tensorflow/contrib/distribute/python/step_fn.py b/tensorflow/contrib/distribute/python/step_fn.py
new file mode 100644
index 0000000000..82514c64be
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/step_fn.py
@@ -0,0 +1,103 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""The step function abstraction represents a single training step."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.training import optimizer as optimizer_lib
+
+
+class Step(object):
+  """Interface for performing each step of a training algorithm."""
+
+  def __init__(self, distribution):
+    self._distribution = distribution
+
+  @property
+  def distribution(self):
+    return self._distribution
+
+  def __call__(self):
+    """Perform one step of this training algorithm."""
+    return self.step(self.inputs())
+
+  def inputs(self):
+    """For the generating the input to be passed to `step()`."""
+    raise NotImplementedError("must be implemented in descendants")
+
+  def step(self, inputs):
+    """Perform the main computation of this training algorithm."""
+    raise NotImplementedError("must be implemented in descendants")
+
+
+class StandardInputStep(Step):
+  """Step with a standard implementation of input handling.
+
+  Args:
+    input_dataset: a tf.data Dataset that provides input.
+  """
+
+  def __init__(self, input_dataset, distribution):
+    Step.__init__(self, distribution)
+    self._distributed_input = distribution.distribute_dataset(input_dataset)
+
+  def inputs(self):
+    return self._distributed_input.get_next()
+
+
+class StandardSingleLossStep(StandardInputStep):
+  """A step function that implements a training step for a feed forward network.
+
+  An instance of this class is intended to be used as a callable:
+
+  ```python
+  ...
+  step = step_fn.StandardSingleLossStep(dataset, loss_fn, optimizer)
+  step.initialize(distribution)
+
+  # Run a single training step on a given DistributionStrategy:
+  step(distribution)
+  ...
+  ```
+
+  Args:
+    input_dataset: a tf.data Dataset that provides input.
+    loss_fn: a function that returns loss.
+    optimizer: an optimizer that implements an update rule.
+    distribution: a `DistributionStrategy` object.
+  """
+
+  def __init__(self, input_dataset, loss_fn, optimizer, distribution):
+    StandardInputStep.__init__(self, input_dataset, distribution)
+    self._loss_fn = loss_fn
+    self._optimizer = optimizer
+    self._is_run_concurrently = False
+
+  def step(self, inputs):
+    with self._distribution.scope():
+      gradients_fn = backprop.implicit_grad(self._loss_fn)
+      gradients_fn = optimizer_lib.get_filtered_grad_fn(gradients_fn)
+
+      grads_and_vars = self.distribution.call_for_each_tower(
+          gradients_fn, inputs, run_concurrently=self._is_run_concurrently)
+      # If threads use layers, then we need to run the first step sequentially,
+      # so that layers.build() is not executed in parallel.  Otherwise, multiple
+      # sets of mirrored variables are going to be created.
+      self._is_run_concurrently = True
+      return self._optimizer._distributed_apply(  # pylint: disable=protected-access
+          self.distribution, grads_and_vars)
diff --git a/tensorflow/contrib/distribute/python/step_fn_test.py b/tensorflow/contrib/distribute/python/step_fn_test.py
new file mode 100644
index 0000000000..75c5ec9659
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/step_fn_test.py
@@ -0,0 +1,62 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for class Step."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from absl.testing import parameterized
+import numpy
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.distribute.python.single_loss_example import single_loss_example
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.ops import variables
+
+
+class SingleLossStepTest(test.TestCase, parameterized.TestCase):
+
+  @combinations.generate(
+      combinations.times(
+          combinations.distributions_and_v1_optimizers(),
+          combinations.combine(mode=combinations.graph_and_eager_modes)))
+  def testTrainNetwork(self, distribution, optimizer_fn):
+    with distribution.scope():
+      single_loss_step, layer = single_loss_example(
+          optimizer_fn, distribution, use_bias=True)
+
+      if context.executing_eagerly():
+        run_step = single_loss_step
+      else:
+        with self.test_session() as sess:
+          run_step = sess.make_callable(single_loss_step())
+      self.evaluate(variables.global_variables_initializer())
+
+      weights, biases = [], []
+      for _ in range(10):
+        run_step()
+
+        weights.append(self.evaluate(distribution.fetch(layer.kernel)))
+        biases.append(self.evaluate(distribution.fetch(layer.bias)))
+
+      error = abs(numpy.add(numpy.squeeze(weights), numpy.squeeze(biases)) - 1)
+      is_not_increasing = all(y <= x for x, y in zip(error, error[1:]))
+      self.assertTrue(is_not_increasing)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/distribute/python/strategy_test_lib.py b/tensorflow/contrib/distribute/python/strategy_test_lib.py
new file mode 100644
index 0000000000..2b4ad9f146
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/strategy_test_lib.py
@@ -0,0 +1,225 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Library for testing DistributionStrategy descendants."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import ops
+from tensorflow.python.layers import core
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.training import optimizer
+
+
+class _TestException(Exception):
+  pass
+
+
+# May be the argument to either distribution.call_for_each_tower() or
+# get_tower_context().merge_call()
+def _raise_exception_fn(_=None):
+  raise _TestException()
+
+
+# Must be the argument to a distribution.call_for_each_tower() call, calls a
+# get_tower_context().merge_call() that raises an exception.
+def _merge_raises_fn():
+  distribute_lib.get_tower_context().merge_call(_raise_exception_fn)
+
+
+# Must be the argument to a get_tower_context().merge_call() call, calls
+# dist.call_for_each_tower() with a function that raises an exception.
+def _call_raises_fn(dist):
+  dist.call_for_each_tower(_raise_exception_fn)
+
+
+# Must be the argument to a distribution.call_for_each_tower() call,
+# calls a get_tower_context().merge_call() that calls a
+# call_for_each_tower() that raises an exception.
+def _merge_call_raises_fn():
+  distribute_lib.get_tower_context().merge_call(_call_raises_fn)
+
+
+# Must be the argument to a get_tower_context().merge_call() call, calls
+# dist.call_for_each_tower() with a function that calls a
+# get_tower_context().merge_call() that raises an exception.
+def _call_merge_raises_fn(dist):
+  dist.call_for_each_tower(_merge_raises_fn)
+
+
+# Must be the argument to a distribution.call_for_each_tower() call, calls a
+# get_tower_context().merge_call() that calls a call_for_each_tower() that
+# calls a get_tower_context().merge_call() that raises an exception.
+def _merge_call_merge_raises_fn():
+  distribute_lib.get_tower_context().merge_call(_call_merge_raises_fn)
+
+
+class DistributionTestBase(test.TestCase):
+  """Some tests that should work with any DistributionStrategy."""
+
+  def _test_minimize_loss_eager(self, d):
+    with d.scope():
+      l = core.Dense(1, use_bias=False)
+
+      def loss(x):
+        # TODO(josh11b): What if this constant was instead a captured
+        # value?  Would it need to be a value that has been passed
+        # through d.broadcast()?
+        y = array_ops.reshape(l(x), []) - constant_op.constant(1.)
+        return y * y
+      # TODO(isaprykin): Extract implicit_grad+get_filtered_grad_fn into a
+      # common `implicit_grad` function and put it in DistributionStrategy.
+      grad_fn = backprop.implicit_grad(loss)
+      grad_fn = optimizer.get_filtered_grad_fn(grad_fn)
+
+      def update(v, g):
+        return v.assign_sub(0.2 * g)
+
+      one = d.broadcast(constant_op.constant([[1.]]))
+
+      def step():
+        """Perform one optimization step."""
+        # Run forward & backward to get gradients, variables list.
+        g_v = d.call_for_each_tower(grad_fn, one, run_concurrently=l.built)
+
+        # Update the variables using the gradients and the update() function.
+        before_list = []
+        after_list = []
+        for g, v in g_v:
+          fetched = d.fetch(v)
+          before_list.append(fetched)
+          # control_dependencies irrelevant but harmless in eager execution
+          with ops.control_dependencies([fetched]):
+            g = d.reduce("sum", g, destinations=v)
+            with ops.control_dependencies(d.unwrap(d.update(v, update, g))):
+              after_list.append(d.fetch(v))
+        return before_list, after_list
+
+      for i in range(10):
+        b, a = step()
+        if i == 0:
+          before, = b  # pylint: disable=unbalanced-tuple-unpacking
+        after, = a  # pylint: disable=unbalanced-tuple-unpacking
+
+      error_before = abs(before.numpy() - 1)
+      error_after = abs(after.numpy() - 1)
+      # Error should go down
+      self.assertLess(error_after, error_before)
+
+  def _test_minimize_loss_graph(self, d, soft_placement=False):
+    config = config_pb2.ConfigProto()
+    config.allow_soft_placement = soft_placement
+    config.gpu_options.per_process_gpu_memory_fraction = 0.3
+    with context.graph_mode(), \
+         ops.Graph().as_default(), \
+         self.test_session(config=config) as sess, \
+         d.scope():
+      l = core.Dense(1, use_bias=False)
+
+      def loss(x):
+        # TODO(josh11b): What if this constant was instead a captured
+        # value?  Would it need to be a value that has been passed
+        # through d.broadcast()?
+        y = array_ops.reshape(l(x), []) - constant_op.constant(1.)
+        return y * y
+
+      grad_fn = backprop.implicit_grad(loss)
+
+      def update(v, g):
+        return v.assign_sub(0.2 * g)
+
+      one = d.broadcast(constant_op.constant([[1.]]))
+
+      def step():
+        """Perform one optimization step."""
+        # Run forward & backward to get gradients, variables list.
+        g_v = d.call_for_each_tower(grad_fn, one)
+
+        # Update the variables using the gradients and the update() function.
+        before_list = []
+        after_list = []
+        for g, v in g_v:
+          fetched = d.fetch(v)
+          before_list.append(fetched)
+          with ops.control_dependencies([fetched]):
+            g = d.reduce("sum", g, destinations=v)
+            with ops.control_dependencies(d.unwrap(d.update(v, update, g))):
+              after_list.append(d.fetch(v))
+        return before_list, after_list
+
+      before_out, after_out = step()
+      variables.global_variables_initializer().run()
+      for i in range(10):
+        b, a = sess.run((before_out, after_out))
+        if i == 0:
+          before, = b
+        after, = a
+
+      error_before = abs(before - 1)
+      error_after = abs(after - 1)
+      # Error should go down
+      self.assertLess(error_after, error_before)
+
+  def _test_map_reduce(self, d, in_graph=None):
+    with d.scope():
+      map_in = [constant_op.constant(i) for i in range(10)]
+      map_out = d.map(map_in, lambda x, y: x * y, 2)
+      observed = d.fetch(d.reduce("sum", map_out))
+      expected = 90  # 2 * (0 + 1 + ... + 9)
+      self.assertEqual(expected, observed.numpy())
+
+  def _test_device_index(self, d):
+    with d.scope():
+      expected_devices = [False] * len(d.worker_devices)
+
+      def mark_devices_fn(device_id):
+        self.assertLess(device_id, len(d.worker_devices))
+        self.assertFalse(expected_devices[device_id])
+        expected_devices[device_id] = True
+
+      d.call_for_each_tower(mark_devices_fn, d.worker_device_index)
+      self.assertAllEqual(expected_devices, [True] * len(d.worker_devices))
+
+  def _test_tower_id(self, d):
+    with d.scope():
+      expected_devices = [False] * len(d.worker_devices)
+
+      def mark_devices_fn():
+        tower_id = distribute_lib.get_tower_context().tower_id
+        self.assertLess(tower_id, len(d.worker_devices))
+        self.assertFalse(expected_devices[tower_id])
+        expected_devices[tower_id] = True
+
+      d.call_for_each_tower(mark_devices_fn)
+      self.assertAllEqual(expected_devices, [True] * len(d.worker_devices))
+
+  def _test_call_and_merge_exceptions(self, dist):
+    with dist.scope():
+      with self.assertRaises(_TestException):
+        dist.call_for_each_tower(_raise_exception_fn)
+      with self.assertRaises(_TestException):
+        dist.call_for_each_tower(_merge_raises_fn)
+      with self.assertRaises(_TestException):
+        dist.call_for_each_tower(_merge_call_raises_fn)
+      with self.assertRaises(_TestException):
+        dist.call_for_each_tower(_merge_call_merge_raises_fn)
diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
new file mode 100644
index 0000000000..c1ba22ed5a
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -0,0 +1,575 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Various classes representing distributed values.
+
+See go/tf-distribution-strategy.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import collections
+import weakref
+
+import six
+
+from tensorflow.contrib.data.python.ops import batching
+from tensorflow.contrib.distribute.python import prefetching_ops_v2
+from tensorflow.contrib.eager.python import datasets
+from tensorflow.python.eager import context
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.training import checkpointable
+from tensorflow.python.training import device_util
+from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.training import saver
+from tensorflow.python.util import nest
+
+
+# pylint: disable=line-too-long
+# TODO(josh11b): Should device values be strings or DeviceSpec objects
+# Not sure DeviceSpec objects are usable as a dict key.
+class DistributedValues(object):
+  """Holds a map from device to values. Either PerDevice or Mirrored."""
+
+  def __init__(self, index):
+    self._index = {device_util.canonicalize(key): value
+                   for key, value in six.iteritems(index)}
+
+  def get(self, device=None):
+    """Returns the value for the current device or raises a ValueError."""
+    if device is None:
+      tower_context = distribute_lib.get_tower_context()
+      if tower_context:
+        device = tower_context.device
+      else:
+        device = distribute_lib.get_update_device()
+        if device is None:
+          device = device_util.current()
+    device = device_util.canonicalize(device)
+    try:
+      return self._index[device]
+    except KeyError:
+      raise ValueError("Device %s not found in %s (current device %s)" %
+                       (device, self._index.keys(), device_util.current()))
+
+  def on_device(self, device):
+    device = device_util.canonicalize(device)
+    return device in self._index
+
+  @property
+  def devices(self):
+    return self._index.keys()
+
+  def __str__(self):
+    return "%s:%s" % (self.__class__.__name__, self._index)
+
+  def __repr__(self):
+    return "%s(%r)" % (self.__class__.__name__, self._index)
+
+  # TODO(josh11b): Possibly make an accessor for _index for use by
+  # DistributionStrategy implementations.
+
+
+class DistributedDelegate(DistributedValues):
+  """A map from device to values; acts as the same type as the values."""
+
+  def __init__(self, index):
+    super(DistributedDelegate, self).__init__(index)
+
+  def __getattr__(self, name):
+    return getattr(self.get(), name)
+
+  # pylint: disable=multiple-statements
+  def __add__(self, o): return self.get() + o
+  def __radd__(self, o): return o + self.get()
+  def __sub__(self, o): return self.get() - o
+  def __rsub__(self, o): return o - self.get()
+  def __mul__(self, o): return self.get() * o
+  def __rmul__(self, o): return o * self.get()
+  def __truediv__(self, o): return self.get() / o
+  def __rtruediv__(self, o): return o / self.get()
+  def __floordiv__(self, o): return self.get() // o
+  def __rfloordiv__(self, o): return o // self.get()
+  def __mod__(self, o): return self.get() % o
+  def __rmod__(self, o): return o % self.get()
+  def __lt__(self, o): return self.get() < o
+  def __le__(self, o): return self.get() <= o
+  def __gt__(self, o): return self.get() > o
+  def __ge__(self, o): return self.get() >= o
+  def __and__(self, o): return self.get() & o
+  def __rand__(self, o): return o & self.get()
+  def __or__(self, o): return self.get() | o
+  def __ror__(self, o): return o | self.get()
+  def __xor__(self, o): return self.get() ^ o
+  def __rxor__(self, o): return o ^ self.get()
+  def __getitem__(self, o): return self.get()[o]
+  def __pow__(self, o, modulo=None): return pow(self.get(), o, modulo)
+  def __rpow__(self, o): return pow(o, self.get())
+  def __invert__(self): return ~self.get()
+  def __neg__(self): return -self.get()
+  def __abs__(self): return abs(self.get())
+
+  def __div__(self, o):
+    try:
+      return self.get().__div__(o)
+    except AttributeError:
+      # See https://docs.python.org/3/library/constants.html#NotImplemented
+      return NotImplemented
+
+  def __rdiv__(self, o):
+    try:
+      return self.get().__rdiv__(o)
+    except AttributeError:
+      # See https://docs.python.org/3/library/constants.html#NotImplemented
+      return NotImplemented
+
+  def __matmul__(self, o):
+    try:
+      return self.get().__matmul__(o)
+    except AttributeError:
+      # See https://docs.python.org/3/library/constants.html#NotImplemented
+      return NotImplemented
+
+  def __rmatmul__(self, o):
+    try:
+      return self.get().__rmatmul__(o)
+    except AttributeError:
+      # See https://docs.python.org/3/library/constants.html#NotImplemented
+      return NotImplemented
+
+  # TODO(josh11b): Even more operator overloads.
+
+
+class PerDevice(DistributedValues):
+  """Holds a map from device to unsynchronized values."""
+  pass
+
+
+class Mirrored(DistributedValues):
+  """Holds a map from device to values which are kept in sync."""
+  pass
+
+
+def _assign_on_device(device, variable, tensor):
+  with ops.device(device):
+    return variable.assign(array_ops.identity(tensor))
+
+
+DistributedVarOp = collections.namedtuple(
+    "DistributedVarOp", ["name", "graph", "type"])
+
+
+class DistributedVariable(DistributedDelegate):
+  """Holds a map from device to variables."""
+  # TODO(josh11b): Support changing the set of variables if e.g. if new
+  # devices are joining or a device is to leave.
+
+  def __init__(self, index):
+    # Child class must set self._primary_var before calling
+    # super(...).__init__(index).
+    self._common_name = self._primary_var.name.split(":")[0]
+    super(DistributedVariable, self).__init__(index)
+
+  @property
+  def initializer(self):
+    return control_flow_ops.group([v.initializer for v in self._index.values()])
+
+  @property
+  def graph(self):
+    return self._primary_var.graph
+
+  @property
+  def _shared_name(self):
+    return self._common_name
+
+  @property
+  def _unique_id(self):
+    return self._primary_var._unique_id   # pylint: disable=protected-access
+
+  @property
+  def name(self):
+    return self._primary_var.name
+
+  @property
+  def dtype(self):
+    return self._primary_var.dtype
+
+  @property
+  def shape(self):
+    return self._primary_var.shape
+
+  def get_shape(self):
+    return self._primary_var.get_shape()
+
+  @property
+  def op(self):
+    # We want cross-tower code that does some var.op.X calls
+    # to work (even if the current device isn't in self.devices), but
+    # other uses of var.op in a cross-tower context to fail.
+    if distribute_lib.get_cross_tower_context():
+      return DistributedVarOp(self._primary_var.op.name,
+                              self._primary_var.op.graph,
+                              self._primary_var.op.type)
+    return self.get().op
+
+  def _should_act_as_resource_variable(self):
+    """Pass resource_variable_ops.is_resource_variable check."""
+    pass
+
+
+# Register a conversion function which reads the value of the variable,
+# allowing instances of the class to be used as tensors.
+def _tensor_conversion(var, dtype=None, name=None, as_ref=False):
+  # Try to avoid assignments to and other mutations of MirroredVariable
+  # state except through a DistributionStrategy.update() call.
+  assert not as_ref
+  return ops.internal_convert_to_tensor(
+      var.get(), dtype=dtype, name=name, as_ref=as_ref)
+
+
+ops.register_tensor_conversion_function(DistributedVariable, _tensor_conversion)
+# TODO(josh11b): ops.register_dense_tensor_like_type(DistributedVariable)?
+
+
+class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
+  """Class for defining how to restore a MirroredVariable."""
+
+  def __init__(self, mirrored_variable, primary_variable, name):
+    self._mirrored_variable = mirrored_variable
+    super(_MirroredSaveable, self).__init__(primary_variable, "", name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    """Restore the same value into all variables."""
+    tensor, = restored_tensors
+    return control_flow_ops.group([
+        _assign_on_device(d, v, tensor)
+        for d, v in six.iteritems(self._mirrored_variable._index)])  # pylint: disable=protected-access
+
+
+def _get_update_device():
+  """Validate we are in update/update_non_slot() and return current device.
+
+  This is used in MirroredVariable.assign* members, to make sure they
+  are only called via an update method, to make sure all components of the
+  variable are being updated in a consistent way.
+
+  Returns:
+    A string device.
+
+  Raises:
+    RuntimeError: If not in distribution.update()/.update_non_slot().
+  """
+  device = distribute_lib.get_update_device()
+  if device is None:
+    raise RuntimeError(
+        "Use DistributionStrategy.update() to modify a MirroredVariable.")
+  return device
+
+
+class MirroredVariable(DistributedVariable, Mirrored,
+                       checkpointable.CheckpointableBase):
+  """Holds a map from device to variables whose values are kept in sync."""
+
+  def __init__(self, index, primary_var):
+    # Use a weakref to make it easy to map from the contained values
+    # to the container without introducing a reference cycle.
+    for v in six.itervalues(index):
+      v._mirrored_container = weakref.ref(self)  # pylint: disable=protected-access
+    self._primary_var = primary_var
+    super(MirroredVariable, self).__init__(index)
+
+  # We use _get_update_device() for the assign* methods to enforce
+  # that we are in an update() function. The arguments to update() are
+  # automatically unwrapped so the update() function would normally
+  # see regular variables, not MirroredVariables. However, the update
+  # function can still operate on wrapped MirroredVariables through
+  # object members, captured arguments, etc. This is more likely in an
+  # update_non_slot() function (like OptimizerV2._finish), which can
+  # update several non-slot variables in one call.
+  def assign_sub(self, *args, **kwargs):
+    return self.get(device=_get_update_device()).assign_sub(*args, **kwargs)
+
+  def assign_add(self, *args, **kwargs):
+    return self.get(device=_get_update_device()).assign_add(*args, **kwargs)
+
+  def assign(self, *args, **kwargs):
+    return self.get(device=_get_update_device()).assign(*args, **kwargs)
+
+  def _gather_saveables_for_checkpoint(self):
+    """Overrides CheckpointableBase method.
+
+    This allows both name-based and object-based save and restore of
+    MirroredVariables.
+
+    Returns:
+      A dictionary mapping attribute names to `SaveableObject` factories.
+    """
+    def _saveable_factory(name=self._common_name):
+      return _MirroredSaveable(self, self._primary_var, name)
+    return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory}
+
+
+class _TowerLocalSaveable(saver.BaseSaverBuilder.SaveableObject):
+  """Class for defining how to restore a TowerLocalVariable."""
+
+  def __init__(self, tower_local_variable, name):
+    self._tower_local_variable = tower_local_variable
+    # We use a callable so that we don't have to evaluate this expression
+    # in the case where we are trying to restore instead of save.
+    def tensor():
+      return distribute_lib.get_distribution_strategy().fetch(
+          tower_local_variable)
+    spec = saver.BaseSaverBuilder.SaveSpec(
+        tensor=tensor,
+        slice_spec="",
+        name=name,
+        dtype=tower_local_variable.dtype)
+    super(_TowerLocalSaveable, self).__init__(tensor, [spec], name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    """Restore the same value into all variables."""
+    tensor, = restored_tensors
+    # To preserve the sum across save and restore, we have to divide the
+    # total across all devices when restoring a variable that was summed
+    # when saving.
+    if self._tower_local_variable.reduce_method == "sum":
+      tensor *= 1. / len(self._tower_local_variable.devices)
+    return control_flow_ops.group([
+        _assign_on_device(d, v, tensor)
+        for d, v in six.iteritems(self._tower_local_variable._index)])  # pylint: disable=protected-access
+
+
+class TowerLocalVariable(DistributedVariable, PerDevice,
+                         checkpointable.CheckpointableBase):
+  """Holds a map from device to variables whose values are reduced on save."""
+
+  def __init__(self, index, primary_var, reduce_method):
+    self._primary_var = primary_var
+    self._reduce_method = reduce_method
+    super(TowerLocalVariable, self).__init__(index)
+
+  def assign_sub(self, *args, **kwargs):
+    return self.get().assign_sub(*args, **kwargs)
+
+  def assign_add(self, *args, **kwargs):
+    return self.get().assign_add(*args, **kwargs)
+
+  def assign(self, *args, **kwargs):
+    return self.get().assign(*args, **kwargs)
+
+  @property
+  def reduce_method(self):
+    return self._reduce_method
+
+  def _gather_saveables_for_checkpoint(self):
+    """Overrides CheckpointableBase method.
+
+    This allows both name-based and object-based save and restore of
+    TowerLocalVariables.
+
+    Returns:
+      A dictionary mapping attribute names to `SaveableObject` factories.
+    """
+    def _saveable_factory(name=self._common_name):
+      return _TowerLocalSaveable(self, name)
+    return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory}
+
+
+def _devices_match(d1, d2):
+  return device_util.canonicalize(d1) == device_util.canonicalize(d2)
+
+
+def regroup(per_device, wrap_class=PerDevice):
+  """Makes device->nest map into a nest of PerDevice/Mirrored values."""
+  items = list(per_device.items())
+  assert items
+  v0 = items[0][1]  # First value
+
+  if isinstance(v0, list):
+    for _, v in items[1:]:
+      assert isinstance(v, list)
+      assert len(v) == len(v0), ("len(v) == %d, len(v0) == %d, v: %s, v0: %s" %
+                                 (len(v), len(v0), v, v0))
+    return [regroup({k: v[i] for k, v in items}, wrap_class)
+            for i in range(len(v0))]
+
+  if isinstance(v0, tuple):
+    for _, v in items[1:]:
+      assert isinstance(v, tuple)
+      assert len(v) == len(v0)
+    regrouped_tuple = tuple(regroup({k: v[i] for k, v in items}, wrap_class)
+                            for i in range(len(v0)))
+    if hasattr(v0, "_fields"):
+      # This tuple is in fact a namedtuple! Create a new namedtuple instance
+      # and initialize it with the regrouped values:
+      assert hasattr(type(v0), "_make")
+      return type(v0)._make(regrouped_tuple)
+    else:
+      return regrouped_tuple
+
+  if isinstance(v0, dict):
+    v0keys = set(v0.keys())
+    for _, v in items[1:]:
+      assert isinstance(v, dict)
+      assert set(v.keys()) == v0keys
+    return {key: regroup({k: v[key] for k, v in items}, wrap_class)
+            for key in v0keys}
+
+  # If exactly the same object across all devices, return it unwrapped.
+  same_id = True
+  for _, v in items[1:]:
+    if v is not v0:
+      same_id = False
+      break
+  # Consider three cases where same_id is true:
+  # * If v0 is a MirroredVariable (and same_id means it is the same
+  #   across all devices), we want to return it. We check
+  #   MirroredVariable specifically since it can look like it
+  #   has a _mirrored_container member since its members do.
+  # * If v0 is a member of a mirrored variable, in which case
+  #   hasattr(v0, "_mirrored_container") is true, we want to
+  #   return the MirroredVariable that contains it using the
+  #   _mirrored_container logic below. This case can trigger
+  #   same_id when there is only one device.
+  # * In any other situation, same_id means we return v0.
+  if same_id and (isinstance(v0, MirroredVariable) or
+                  not hasattr(v0, "_mirrored_container")):
+    return v0
+
+  # Detect the case where each device has a parallel component of the
+  # same MirroredVariable. In this case we want to return the
+  # containing MirroredVariable, after a bunch of sanity checking.
+  # In particular, each component should have the same container,
+  # and the devices of the variables should match the keys of the
+  # per-device dictionary.
+  # TODO(josh11b): Do we need similar logic for TowerLocalVariables?
+  if hasattr(v0, "_mirrored_container"):
+    # pylint: disable=protected-access
+    assert not isinstance(v0, MirroredVariable), (
+        "ids = %s, items = %s" % ([id(v[1]) for v in items], items))
+    assert _devices_match(v0.device, items[0][0]), (
+        "v0.device = %s, items = %s" % (v0.device, items))
+    mirrored_container = v0._mirrored_container()
+    assert mirrored_container is not None
+    for d, v in items[1:]:
+      assert _devices_match(v.device, d), (
+          "v.device = %s, d = %s, items = %s" % (v.device, d, items))
+      assert mirrored_container is v._mirrored_container()
+    return mirrored_container
+  # pylint: enable=protected-access
+
+  return wrap_class(per_device)
+
+
+def select_device(device, structured):
+  """Specialize a nest of regular & per-device values for one device."""
+  def _get(x):
+    return x.get(device) if isinstance(x, DistributedValues) else x
+
+  return nest.map_structure(_get, structured)
+
+
+def select_device_mirrored(device, structured):
+  """Specialize a nest of regular & mirrored values for one device."""
+  def _get_mirrored(x):
+    if isinstance(x, DistributedValues):
+      if not isinstance(x, Mirrored):
+        raise TypeError(
+            "Expected value to be mirrored across towers: %s in %s." %
+            (x, structured))
+      return x.get(device)
+    else:
+      return x
+
+  return nest.map_structure(_get_mirrored, structured)
+
+
+class PerDeviceDataIterator(object):
+  """An iterator (like `tf.data.Iterator`) into a `PerDeviceDataset`."""
+
+  def __init__(self, iterator, devices, prefetch_on_device=None):
+    self._iterator = iterator
+    self._devices = devices
+    self._prefetch_on_device = prefetch_on_device
+
+  def get_next(self, name=None):
+    """Scatter the input across devices."""
+    if self._prefetch_on_device:
+      data_list = self._iterator.get_next(name=name)
+      index = dict(zip(self._devices, data_list))
+    else:
+      batch = self._iterator.get_next(name=name)
+      index = {}
+      def get_ith(i):
+        return lambda x: x[i]
+
+      for i, d in enumerate(self._devices):
+        index[d] = nest.map_structure(get_ith(i), batch)
+        if context.executing_eagerly():
+          with ops.device(d):
+            index[d] = nest.map_structure(array_ops.identity, index[d])
+
+    return regroup(index)
+
+
+class PerDeviceDataset(object):
+  """Like `tf.data.Dataset` split devices, producing `PerDevice` data."""
+
+  def __init__(self, dataset, devices, prefetch_on_device=None):
+    self._devices = devices
+
+    # Default to using prefetching in graph mode, unless specified.
+    # TODO(priyag): Enable prefetching in eager mode.
+    self._prefetch_on_device = prefetch_on_device
+    if self._prefetch_on_device is None:
+      self._prefetch_on_device = not context.executing_eagerly()
+    assert not (self._prefetch_on_device and context.executing_eagerly()), (
+        "Prefetching is only supported in graph mode currently")
+
+    if self._prefetch_on_device:
+      self._dataset = dataset
+    else:
+      # TODO(priyag): If dropping remainder is not appropriate, find another
+      # approach to distributing the dataset when not possible to divide evenly.
+      # Possibly not an issue when we start using PartitionedDataset.
+      self._dataset = dataset.apply(
+          batching.batch_and_drop_remainder(len(devices)))
+
+  def make_one_shot_iterator(self):
+    """Get a one time use iterator for the distributed PerDeviceDataset."""
+    if self._prefetch_on_device:
+      on_device_dataset = self._dataset.apply(
+          prefetching_ops_v2.prefetch_to_devices(self._devices))
+      dataset_iterator = on_device_dataset.make_one_shot_iterator()
+    elif context.executing_eagerly():
+      dataset_iterator = datasets.Iterator(self._dataset)
+    else:
+      dataset_iterator = self._dataset.make_one_shot_iterator()
+
+    return PerDeviceDataIterator(
+        dataset_iterator, self._devices, self._prefetch_on_device)
+
+
+class MapOutput(object):
+  """Map can result in multiple outputs per device."""
+
+  def __init__(self, l):
+    self._l = l
+
+  def get(self):
+    return self._l
diff --git a/tensorflow/contrib/distribute/python/values_test.py b/tensorflow/contrib/distribute/python/values_test.py
new file mode 100644
index 0000000000..5c0d4b7d6c
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/values_test.py
@@ -0,0 +1,807 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for the distributed values library."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import os
+
+from tensorflow.contrib.distribute.python import mirrored_strategy
+from tensorflow.contrib.distribute.python import values
+from tensorflow.core.protobuf import config_pb2
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.estimator import model_fn as model_fn_lib
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import errors
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import device_util
+from tensorflow.python.training import saver as saver_lib
+
+
+@test_util.with_c_api
+class DistributedValuesTest(test.TestCase):
+
+  def testGetEager(self):
+    with ops.device("/device:CPU:0"):
+      one = constant_op.constant(1)
+      two = constant_op.constant(2)
+      v = values.DistributedValues({"/device:CPU:0": one, "/device:GPU:0": two})
+      self.assertEqual(two, v.get("/device:GPU:0"))
+      self.assertEqual(one, v.get())
+      with self.assertRaises(ValueError):
+        self.assertIsNone(v.get("/device:GPU:2"))
+
+  def testGetGraph(self):
+    with context.graph_mode(), \
+        ops.Graph().as_default(), \
+        ops.device("/device:CPU:0"):
+      one = constant_op.constant(1)
+      two = constant_op.constant(2)
+      v = values.DistributedValues({"/device:CPU:0": one, "/device:GPU:0": two})
+      self.assertEqual(two, v.get("/device:GPU:0"))
+      self.assertEqual(one, v.get())
+      with self.assertRaises(ValueError):
+        self.assertIsNone(v.get("/device:GPU:2"))
+
+  def testCanonicalization(self):
+    canonical_cpu = ["/job:localhost/replica:0/task:0/device:CPU:0"]
+    v = values.DistributedValues({"": 42})
+    self.assertEqual(canonical_cpu, list(v._index.keys()))
+    v = values.DistributedValues({"/device:CPU:0": 42})
+    self.assertEqual(canonical_cpu, list(v._index.keys()))
+    v = values.DistributedValues({"/cpu:0": 42})
+    self.assertEqual(canonical_cpu, list(v._index.keys()))
+    v = values.DistributedValues({"/CPU:0": 42})
+    self.assertEqual(canonical_cpu, list(v._index.keys()))
+    with self.assertRaises(AssertionError):
+      v = values.DistributedValues({"/device:cpu:0": 42})
+
+
+@test_util.with_c_api
+class DistributedDelegateTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testGetAttr(self):
+    with ops.device("/device:CPU:0"):
+
+      class Foo(object):
+
+        def __init__(self, x):
+          self.x = x
+
+      v = values.DistributedDelegate(
+          {"/device:CPU:0": Foo(7), "/device:GPU:0": Foo(8)})
+      self.assertEqual(7, v.x)
+      with self.assertRaises(AttributeError):
+        _ = v.y
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testOperatorOverride(self):
+    with ops.device("/device:CPU:0"):
+      v = values.DistributedDelegate({"/device:CPU:0": 7, "/device:GPU:0": 8})
+      # v should act like int(7).
+      self.assertEqual(8, v + 1)
+      self.assertEqual(10, 3 + v)
+      self.assertEqual(14, v + v)
+      self.assertEqual(5, v - 2)
+      self.assertEqual(6, 13 - v)
+      self.assertEqual(0, v - v)
+      self.assertEqual(14, v * 2)
+      self.assertEqual(21, 3 * v)
+      self.assertEqual(49, v * v)
+      self.assertEqual(3.5, v / 2)
+      self.assertEqual(1.5, 10.5 / v)
+      self.assertEqual(3, v // 2)
+      self.assertEqual(2, 15 // v)
+      self.assertEqual(1, v % 2)
+      self.assertEqual(2, 16 % v)
+      self.assertTrue(v < 12)
+      self.assertTrue(v <= 12)
+      self.assertFalse(v > 12)
+      self.assertFalse(v >= 12)
+      self.assertFalse(12 < v)
+      self.assertFalse(12 <= v)
+      self.assertTrue(12 > v)
+      self.assertTrue(12 >= v)
+      self.assertEqual(3, v & 3)
+      self.assertEqual(3, 11 & v)
+      self.assertEqual(15, v | 8)
+      self.assertEqual(23, 16 | v)
+      self.assertEqual(4, v ^ 3)
+      self.assertEqual(12, 11 ^ v)
+      self.assertEqual(343, pow(v, 3))
+      self.assertEqual(3, pow(v, 3, 10))
+      self.assertEqual(128, pow(2, v))
+      self.assertEqual(-7, -v)
+      self.assertEqual(~7, ~v)
+      self.assertEqual(7, abs(v))
+      with self.assertRaises(TypeError):
+        _ = v[2]
+
+
+def _device_str(d):
+  return "/device:GPU:" + str(d)
+
+
+def _nested_value(d):
+  return ("a" + d, ["b" + d, {"c": "d" + d, "e": "f" + d}, "g" + d], "h" + d)
+
+
+def _make_mirrored():
+  v = []
+  index = {}
+  devices = ["/device:GPU:0", "/device:CPU:0"]
+  for d, n, init in zip(devices, ["v", "v/replica"], [1., 2.]):
+    with ops.device(d):
+      v.append(variable_scope.get_variable(
+          name=n, initializer=init, use_resource=True))
+      index[d] = v[-1]
+  mirrored = values.MirroredVariable(index, v[0])
+  return v, devices, mirrored
+
+
+@test_util.with_c_api
+class RegroupAndSelectDeviceTest(test.TestCase):
+
+  def _is_per_device(self, result, expected, klass=values.PerDevice):
+    self.assertIsInstance(result, klass)
+    # We canonicalize the devices to match the device strings returned
+    # by PerDevice, which also does device string canonicalization.
+    devices = [device_util.canonicalize(_device_str(i))
+               for i in range(len(expected))]
+    self.assertEqual(set(devices), set(result.devices))
+    for i, d in enumerate(devices):
+      self.assertEqual(expected[i], result.get(d))
+      self.assertEqual(expected[i], result.get(_device_str(i)))
+
+  def testNested(self):
+    result = values.regroup({_device_str(0): _nested_value("1"),
+                             _device_str(1): _nested_value("2")})
+    self.assertIsInstance(result, tuple)
+    self.assertEqual(3, len(result))
+    self._is_per_device(result[0], ["a1", "a2"])
+    self._is_per_device(result[2], ["h1", "h2"])
+
+    self.assertIsInstance(result[1], list)
+    self.assertEqual(3, len(result[1]))
+    self._is_per_device(result[1][0], ["b1", "b2"])
+    self._is_per_device(result[1][2], ["g1", "g2"])
+
+    self.assertIsInstance(result[1][1], dict)
+    self.assertEqual(set(["c", "e"]), set(result[1][1].keys()))
+    self._is_per_device(result[1][1]["c"], ["d1", "d2"])
+    self._is_per_device(result[1][1]["e"], ["f1", "f2"])
+
+    # Also test that we can undo the merge using select_device()
+    self.assertEqual(_nested_value("1"),
+                     values.select_device(_device_str(0), result))
+    self.assertEqual(_nested_value("2"),
+                     values.select_device(_device_str(1), result))
+    # select_device_mirrored() should fail due to non-mirrored values
+    with self.assertRaises(TypeError):
+      values.select_device_mirrored(_device_str(0), result)
+    with self.assertRaises(TypeError):
+      values.select_device_mirrored(_device_str(1), result)
+
+  def testWrapClass(self):
+    # Normally a mirrored value would be the same across devices, but
+    # for a test it is convenient to be able to tell the values apart.
+    result = values.regroup({_device_str(0): _nested_value("1"),
+                             _device_str(1): _nested_value("2")},
+                            values.Mirrored)
+    self.assertIsInstance(result, tuple)
+    self.assertEqual(3, len(result))
+    self._is_per_device(result[0], ["a1", "a2"], values.Mirrored)
+    self._is_per_device(result[2], ["h1", "h2"], values.Mirrored)
+
+    self.assertIsInstance(result[1], list)
+    self.assertEqual(3, len(result[1]))
+    self._is_per_device(result[1][0], ["b1", "b2"], values.Mirrored)
+    self._is_per_device(result[1][2], ["g1", "g2"], values.Mirrored)
+
+    self.assertIsInstance(result[1][1], dict)
+    self.assertEqual(set(["c", "e"]), set(result[1][1].keys()))
+    self._is_per_device(result[1][1]["c"], ["d1", "d2"], values.Mirrored)
+    self._is_per_device(result[1][1]["e"], ["f1", "f2"], values.Mirrored)
+
+    # Also test that we can undo the merge using select_device()
+    self.assertEqual(_nested_value("1"),
+                     values.select_device(_device_str(0), result))
+    self.assertEqual(_nested_value("2"),
+                     values.select_device(_device_str(1), result))
+    # Values are marked as mirrored, so select_device_mirrored() is allowed.
+    self.assertEqual(_nested_value("1"),
+                     values.select_device_mirrored(_device_str(0), result))
+    self.assertEqual(_nested_value("2"),
+                     values.select_device_mirrored(_device_str(1), result))
+
+  def testMirroredContainer(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+    v, devices, mirrored = _make_mirrored()
+    result = values.regroup(dict(zip(devices, v)))
+    self.assertIs(mirrored, result)
+
+  def testSameId(self):
+    foo = object()
+    result = values.regroup({_device_str(0): ("a", foo),
+                             _device_str(1): ("b", foo)})
+    self.assertIsInstance(result, tuple)
+    self.assertEqual(2, len(result))
+    self._is_per_device(result[0], ["a", "b"])
+    self.assertIs(foo, result[1])
+
+    # Test select_device(), should undo the merge done by regroup().
+    result_0 = values.select_device(_device_str(0), result)
+    self.assertIsInstance(result_0, tuple)
+    self.assertEqual(2, len(result_0))
+    self.assertEqual("a", result_0[0])
+    self.assertIs(foo, result_0[1])
+    result_1 = values.select_device(_device_str(1), result)
+    self.assertIsInstance(result_1, tuple)
+    self.assertEqual(2, len(result_1))
+    self.assertEqual("b", result_1[0])
+    self.assertIs(foo, result_1[1])
+
+  def testOneDevice(self):
+    result = values.regroup({_device_str(0): _nested_value("1")})
+    # On one device regroup() and select_device() are basically identity.
+    self.assertEqual(_nested_value("1"), result)
+    self.assertEqual(_nested_value("1"),
+                     values.select_device(_device_str(0), result))
+
+    # The one exception has to do with MirroredVariables.
+    d = "/device:CPU:0"
+    with ops.device(d):
+      v = variable_scope.get_variable(
+          name="v", initializer=1., use_resource=True)
+      index = {d: v}
+    mirrored = values.MirroredVariable(index, v)
+    result = values.regroup(index)
+    self.assertIs(mirrored, result)
+
+  def testNamedTupleEstimatorSpec(self):
+    with context.graph_mode(), ops.Graph().as_default():
+      created_estimator_specs = {}
+      to_regroup = {}
+
+      for device_id in range(3):
+        spec = model_fn_lib.EstimatorSpec(
+            mode=model_fn_lib.ModeKeys.TRAIN,
+            loss=constant_op.constant(device_id / 2),
+            train_op=array_ops.identity(constant_op.constant(device_id)))
+        created_estimator_specs[device_id] = spec
+        to_regroup[_device_str(device_id)] = spec
+
+      merged_estimator_spec = values.regroup(to_regroup)
+
+      self.assertTrue(
+          isinstance(merged_estimator_spec, model_fn_lib.EstimatorSpec))
+      self.assertEquals(model_fn_lib.ModeKeys.TRAIN, merged_estimator_spec.mode)
+      for device_id in range(3):
+        d = _device_str(device_id)
+        self.assertEquals(created_estimator_specs[device_id].loss,
+                          merged_estimator_spec.loss.get(d))
+        self.assertEquals(created_estimator_specs[device_id].train_op,
+                          merged_estimator_spec.train_op.get(d))
+        # Scaffold is populated by `EstimatorSpec.__new__`.
+        self.assertEquals(created_estimator_specs[device_id].scaffold,
+                          merged_estimator_spec.scaffold.get(d))
+        # Also test that we can undo the merge using select_device()
+        self.assertEquals(created_estimator_specs[device_id],
+                          values.select_device(_device_str(device_id),
+                                               merged_estimator_spec))
+
+
+@test_util.with_c_api
+class PerDeviceDatasetTest(test.TestCase):
+
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  def _test_iterator_no_prefetch(self, devices, dataset, expected_values):
+    per_device_dataset = values.PerDeviceDataset(
+        dataset, devices, prefetch_on_device=False)
+    iterator = per_device_dataset.make_one_shot_iterator()
+
+    for expected_value in expected_values:
+      next_element = iterator.get_next()
+      actual = self.evaluate([
+          values.select_device(d, next_element) for d in devices])
+      self.assertEqual(expected_value, actual)
+
+    with self.assertRaises(errors.OutOfRangeError):
+      next_element = iterator.get_next()
+      self.evaluate([
+          values.select_device(d, next_element) for d in devices])
+
+  def _test_iterator_with_prefetch(self, devices, dataset, expected_values):
+    if not context.executing_eagerly():
+      per_device_dataset = values.PerDeviceDataset(
+          dataset, devices, prefetch_on_device=True)
+      iterator = per_device_dataset.make_one_shot_iterator()
+
+      # With prefetching, we cannot guarantee which input ends up on which
+      # device, so we verify that the complete set seen on all devices is
+      # correct, and equal numbers are distributed to each device.
+      combined_actual = []
+      combined_expected = []
+      for expected_value in expected_values:
+        next_element = iterator.get_next()
+        combined_actual.extend(self.evaluate([
+            values.select_device(d, next_element) for d in devices]))
+        combined_expected.extend(expected_value)
+
+      self.assertEqual(set(combined_expected), set(combined_actual))
+
+      with self.assertRaises(errors.OutOfRangeError):
+        next_element = iterator.get_next()
+        self.evaluate([
+            values.select_device(d, next_element) for d in devices])
+
+  def _test_iterator(self, devices, dataset, expected_values):
+    self._test_iterator_no_prefetch(devices, dataset, expected_values)
+    self._test_iterator_with_prefetch(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testOneDevice(self):
+    devices = ["/device:CPU:0"]
+    dataset = dataset_ops.Dataset.range(10)
+
+    expected_values = [[i] for i in range(10)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testMultipleDevices(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset = dataset_ops.Dataset.range(10)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testTupleDataset(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset1 = dataset_ops.Dataset.range(10)
+    dataset2 = dataset_ops.Dataset.range(10).map(lambda x: x**2)
+    dataset = dataset_ops.Dataset.zip((dataset1, dataset2))
+
+    expected_values = [[(i, i**2), (i+1, (i+1)**2)] for i in range(0, 10, 2)]
+
+    self._test_iterator(devices, dataset, expected_values)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testUnevenDatasetBatches(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    devices = ["/device:CPU:0", "/device:GPU:0"]
+    dataset = dataset_ops.Dataset.range(11)
+
+    expected_values = [[i, i+1] for i in range(0, 10, 2)]
+    self._test_iterator(devices, dataset, expected_values)
+
+
+@test_util.with_c_api
+class MirroredVariableTest(test.TestCase):
+
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testProperties(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    v, _, mirrored = _make_mirrored()
+
+    self.assertEquals(v[0].name, mirrored.name)
+    self.assertEquals(v[0].dtype, mirrored.dtype)
+    self.assertEquals(v[0].shape, mirrored.shape)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testVariableOnAnotherDevice(self):
+    v = variable_scope.get_variable(
+        name="v", initializer=[1.], use_resource=True)
+    index = {"/job:foo/device:CPU:0": v}
+    mirrored = values.MirroredVariable(index, v)
+
+    self.assertEquals(v.name, mirrored.name)
+    self.assertEquals(v.dtype, mirrored.dtype)
+    self.assertEquals(v.shape, mirrored.shape)
+
+  def _assign_mirrored(self, devices, v, new):
+    for d, var, n in zip(devices, v, new):
+      with ops.device(d):
+        self.evaluate(var.assign(n))
+
+  def _save_return_saver(self, sess, var):
+    saver = saver_lib.Saver(var_list=[var])
+    test_dir = self.get_temp_dir()
+    prefix = os.path.join(test_dir, "ckpt")
+    return saver.save(sess, prefix), saver
+
+  def _save(self, sess, var):
+    save_path, _ = self._save_return_saver(sess, var)
+    return save_path
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveAndRestoreMirroredOneGraph(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    with self.test_session() as sess:
+      v, devices, mirrored = _make_mirrored()
+
+      # Overwrite the initial values.
+      self._assign_mirrored(devices, v, [3., 4.])
+
+      # Saves the current value of v[0], 3.
+      save_path, saver = self._save_return_saver(sess, mirrored)
+
+      # Change the values between save and restore.
+      self._assign_mirrored(devices, v, [5., 6.])
+
+      # Restores the saved value of 3. to both variables.
+      saver.restore(sess, save_path)
+      self.assertEqual([3., 3.], self.evaluate([v[0], v[1]]))
+
+  def _save_mirrored(self):
+    """Save variables with mirroring, returns save_path."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, devices, mirrored = _make_mirrored()
+
+      # Overwrite the initial values.
+      self._assign_mirrored(devices, v, [3., 4.])
+
+      # Saves the current value of v[0], 3.
+      save_path = self._save(sess, mirrored)
+
+      # Change the values between save and restore.
+      self._assign_mirrored(devices, v, [5., 6.])
+    return save_path
+
+  def _save_normal(self):
+    """Save variables without mirroring, returns save_path."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      var = variable_scope.get_variable(
+          name="v", initializer=1., use_resource=True)
+
+      # Overwrite the initial value.
+      self.evaluate(var.assign(3.))
+
+      # Saves the current value of var, 3.
+      save_path = self._save(sess, var)
+
+      # Change the values between save and restore.
+      self.evaluate(var.assign(5.))
+    return save_path
+
+  def _restore_normal(self, save_path):
+    """Restore to variables without mirroring in a fresh graph."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      var = variable_scope.get_variable(
+          name="v", initializer=7., use_resource=True)
+
+      # Overwrite the initial value.
+      self.evaluate(var.assign(8.))
+
+      # Restores the saved value of 3. to `var`.
+      saver = saver_lib.Saver(var_list=[var])
+      saver.restore(sess, save_path)
+      self.assertEqual(3., self.evaluate(var))
+
+  def _restore_mirrored(self, save_path):
+    """Restore to variables with mirroring in a fresh graph."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, devices, mirrored = _make_mirrored()
+
+      # Overwrite the initial values.
+      self._assign_mirrored(devices, v, [7., 8.])
+
+      # Restores the saved value of 3. to both variables.
+      saver = saver_lib.Saver(var_list=[mirrored])
+      saver.restore(sess, save_path)
+      self.assertEqual([3., 3.], self.evaluate([v[0], v[1]]))
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveMirroredRestoreMirrored(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_mirrored()
+    self._restore_mirrored(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveMirroredRestoreNormal(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_mirrored()
+    self._restore_normal(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveNormalRestoreMirrored(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_normal()
+    self._restore_mirrored(save_path)
+
+
+_devices = ["/device:GPU:0", "/device:CPU:0"]
+
+
+def _make_tower_local(method):
+  v = []
+  index = {}
+  for d, n, init in zip(_devices, ["v", "v/replica"], [1., 2.]):
+    with ops.device(d):
+      v.append(variable_scope.get_variable(
+          name=n, initializer=init, use_resource=True))
+      index[d] = v[-1]
+  tower_local = values.TowerLocalVariable(index, v[0], method)
+  return v, tower_local
+
+
+@test_util.with_c_api
+class TowerLocalVariableTest(test.TestCase):
+
+  config = config_pb2.ConfigProto()
+  config.allow_soft_placement = True
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testProperties(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    v, tower_local = _make_tower_local("sum")
+
+    self.assertEquals(v[0].name, tower_local.name)
+    self.assertEquals(v[0].dtype, tower_local.dtype)
+    self.assertEquals(v[0].shape, tower_local.shape)
+    self.assertEquals("sum", tower_local.reduce_method)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testVariableOnAnotherDevice(self):
+    v = variable_scope.get_variable(
+        name="v", initializer=[1.], use_resource=True)
+    index = {"/job:foo/device:CPU:0": v}
+    tower_local = values.TowerLocalVariable(index, v, "mean")
+
+    self.assertEquals(v.name, tower_local.name)
+    self.assertEquals(v.dtype, tower_local.dtype)
+    self.assertEquals(v.shape, tower_local.shape)
+    self.assertEquals("mean", tower_local.reduce_method)
+
+  def _assign_tower_local(self, devices, v, new):
+    for d, var, n in zip(devices, v, new):
+      with ops.device(d):
+        self.evaluate(var.assign(n))
+
+  def _save_return_saver(self, sess, var):
+    saver = saver_lib.Saver(var_list=[var])
+    test_dir = self.get_temp_dir()
+    prefix = os.path.join(test_dir, "ckpt")
+    return saver.save(sess, prefix), saver
+
+  def _save(self, sess, var):
+    save_path, _ = self._save_return_saver(sess, var)
+    return save_path
+
+  def _dist_scope(self):
+    return mirrored_strategy.MirroredStrategy(_devices).scope()
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveAndRestoreTowerLocalSumOneGraph(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    with self.test_session() as sess:
+      v, tower_local = _make_tower_local("sum")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [3., 4.])
+
+      with self._dist_scope():
+        # Saves the current value of v[0] + v[1], 7.
+        save_path, saver = self._save_return_saver(sess, tower_local)
+
+        # Change the values between save and restore.
+        self._assign_tower_local(_devices, v, [5., 6.])
+
+        # Restores the saved value of 7. which gets divided equally
+        # between the variables.
+        saver.restore(sess, save_path)
+        self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveAndRestoreTowerLocalMeanOneGraph(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    with self.test_session() as sess:
+      v, tower_local = _make_tower_local("mean")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [3., 4.])
+
+      with self._dist_scope():
+        # Saves the current value of (v[0] + v[1])/2, 3.5.
+        save_path, saver = self._save_return_saver(sess, tower_local)
+
+        # Change the values between save and restore.
+        self._assign_tower_local(_devices, v, [5., 6.])
+
+        # Restores the saved value of 3.5 to both variables.
+        saver.restore(sess, save_path)
+        self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
+
+  def _save_tower_local_mean(self):
+    """Save variables with mirroring, returns save_path."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, tower_local = _make_tower_local("mean")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [3., 4.])
+
+      with self._dist_scope():
+        # Saves the current value of (v[0] + v[1])/2, 3.5
+        save_path = self._save(sess, tower_local)
+
+        # Change the values between save and restore.
+        self._assign_tower_local(_devices, v, [5., 6.])
+    return save_path
+
+  def _save_tower_local_sum(self):
+    """Save variables with mirroring, returns save_path."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, tower_local = _make_tower_local("sum")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [1.5, 2.])
+
+      with self._dist_scope():
+        # Saves the current value of v[0] + v[1], 3.5
+        save_path = self._save(sess, tower_local)
+
+        # Change the values between save and restore.
+        self._assign_tower_local(_devices, v, [5., 6.])
+    return save_path
+
+  def _save_normal(self):
+    """Save variables without mirroring, returns save_path."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      var = variable_scope.get_variable(
+          name="v", initializer=1., use_resource=True)
+
+      # Overwrite the initial value.
+      self.evaluate(var.assign(3.5))
+
+      # Saves the current value of var, 3.5.
+      save_path = self._save(sess, var)
+
+      # Change the values between save and restore.
+      self.evaluate(var.assign(5.))
+    return save_path
+
+  def _restore_normal(self, save_path):
+    """Restore to variables without mirroring in a fresh graph."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      var = variable_scope.get_variable(
+          name="v", initializer=7., use_resource=True)
+
+      # Overwrite the initial value.
+      self.evaluate(var.assign(8.))
+
+      # Restores the saved value of 3.5 to `var`.
+      saver = saver_lib.Saver(var_list=[var])
+      saver.restore(sess, save_path)
+      self.assertEqual(3.5, self.evaluate(var))
+
+  def _restore_tower_local_mean(self, save_path):
+    """Restore to variables with mirroring in a fresh graph."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, tower_local = _make_tower_local("mean")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [7., 8.])
+
+      with self._dist_scope():
+        # Restores the saved value of 3.5 to both variables.
+        saver = saver_lib.Saver(var_list=[tower_local])
+        saver.restore(sess, save_path)
+        self.assertEqual([3.5, 3.5], self.evaluate([v[0], v[1]]))
+
+  def _restore_tower_local_sum(self, save_path):
+    """Restore to variables with mirroring in a fresh graph."""
+    with self.test_session(graph=ops.Graph()) as sess:
+      v, tower_local = _make_tower_local("sum")
+
+      # Overwrite the initial values.
+      self._assign_tower_local(_devices, v, [7., 8.])
+
+      with self._dist_scope():
+        # Restores the saved value of 3.5 to both variables.
+        saver = saver_lib.Saver(var_list=[tower_local])
+        saver.restore(sess, save_path)
+        self.assertEqual([1.75, 1.75], self.evaluate([v[0], v[1]]))
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveTowerLocalRestoreTowerLocalMean(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_tower_local_mean()
+    self._restore_tower_local_mean(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveTowerLocalRestoreTowerLocalSum(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_tower_local_sum()
+    self._restore_tower_local_sum(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveTowerLocalMeanRestoreNormal(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_tower_local_mean()
+    self._restore_normal(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveTowerLocalSumRestoreNormal(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_tower_local_sum()
+    self._restore_normal(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveNormalRestoreTowerLocalMean(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_normal()
+    self._restore_tower_local_mean(save_path)
+
+  @test_util.run_in_graph_and_eager_modes(config=config)
+  def testSaveNormalRestoreTowerLocalSum(self):
+    if context.num_gpus() < 1 and context.executing_eagerly():
+      self.skipTest("A GPU is not available for this test in eager mode.")
+
+    save_path = self._save_normal()
+    self._restore_tower_local_sum(save_path)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/BUILD b/tensorflow/contrib/optimizer_v2/BUILD
new file mode 100644
index 0000000000..26ea9135f5
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/BUILD
@@ -0,0 +1,205 @@
+# Prototype of OptimizerV2.
+
+package(
+    default_visibility = ["//tensorflow:internal"],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+load("//tensorflow:tensorflow.bzl", "py_test")
+load("//tensorflow:tensorflow.bzl", "cuda_py_test")
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+py_library(
+    name = "optimizer_v2_py",
+    srcs = ["optimizer_v2_symbols.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//visibility:public"],
+    deps = [
+        ":training",
+        "//tensorflow/python:util",
+    ],
+)
+
+py_library(
+    name = "training",
+    srcs = [
+        "adadelta.py",
+        "adagrad.py",
+        "adam.py",
+        "gradient_descent.py",
+        "momentum.py",
+        "optimizer_v2.py",
+        "rmsprop.py",
+    ],
+    srcs_version = "PY2AND3",
+    deps = [
+        "//tensorflow/python:control_flow_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+    ],
+)
+
+cuda_py_test(
+    name = "adadelta_test",
+    size = "medium",
+    srcs = ["adadelta_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:variables",
+        "//third_party/py/numpy",
+    ],
+)
+
+cuda_py_test(
+    name = "adagrad_test",
+    size = "small",
+    srcs = ["adagrad_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:client_testlib",
+        "//third_party/py/numpy",
+    ],
+)
+
+cuda_py_test(
+    name = "adam_test",
+    size = "small",
+    srcs = ["adam_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:client_testlib",
+        "//third_party/py/numpy",
+    ],
+)
+
+cuda_py_test(
+    name = "checkpointable_utils_test",
+    srcs = ["checkpointable_utils_test.py"],
+    additional_deps = [
+        ":training",
+        "@six_archive//:six",
+        "//tensorflow/contrib/eager/python:checkpointable_utils",
+        "//tensorflow/python:constant_op",
+        "//tensorflow/python:dtypes",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:init_ops",
+        "//tensorflow/python:layers",
+        "//tensorflow/python:layers_base",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:training",
+        "//tensorflow/python:variable_scope",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/keras",
+    ],
+    tags = ["notsan"],
+)
+
+cuda_py_test(
+    name = "gradient_descent_test",
+    size = "medium",
+    srcs = ["gradient_descent_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+    ],
+)
+
+cuda_py_test(
+    name = "momentum_test",
+    size = "medium",
+    srcs = ["momentum_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:resources",
+        "//tensorflow/python:variables",
+        "//tensorflow/python/eager:context",
+    ],
+)
+
+cuda_py_test(
+    name = "optimizer_v2_test",
+    size = "medium",
+    srcs = ["optimizer_v2_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:framework_test_lib",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:clip_ops",
+        "//tensorflow/python:gradients",
+        "//tensorflow/python:resource_variable_ops",
+        "//tensorflow/python:state_ops",
+        "//tensorflow/python:variables",
+    ],
+)
+
+cuda_py_test(
+    name = "rmsprop_test",
+    size = "small",
+    srcs = ["rmsprop_test.py"],
+    additional_deps = [
+        ":training",
+        "//tensorflow/python:array_ops",
+        "//tensorflow/python:embedding_ops",
+        "//tensorflow/python:framework",
+        "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:platform_test",
+        "//tensorflow/python:client_testlib",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/optimizer_v2/adadelta.py b/tensorflow/contrib/optimizer_v2/adadelta.py
new file mode 100644
index 0000000000..b206f9f61b
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adadelta.py
@@ -0,0 +1,113 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Adadelta for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.training import training_ops
+
+
+class AdadeltaOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the Adadelta algorithm.
+
+  See [M. D. Zeiler](http://arxiv.org/abs/1212.5701)
+  ([pdf](http://arxiv.org/pdf/1212.5701v1.pdf))
+  """
+
+  def __init__(self, learning_rate=0.001, rho=0.95, epsilon=1e-8,
+               use_locking=False, name="Adadelta"):
+    """Construct a new Adadelta optimizer.
+
+    Some of the args below are hyperparameters, where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate.
+        To match the exact form in the original paper use 1.0.
+      rho: A float hyperparameter. The decay rate.
+      epsilon: A float hyperparameter. A constant epsilon used to better
+        condition the grad update.
+      use_locking: If `True` use locks for update operations.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Adadelta".
+    """
+    super(AdadeltaOptimizer, self).__init__(use_locking, name)
+    self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("rho", rho)
+    self._set_hyper("epsilon", epsilon)
+
+  def _create_vars(self, var_list, state):
+    for v in var_list:
+      state.zeros_slot(v, "accum")
+      state.zeros_slot(v, "accum_update")
+
+  def _apply_dense(self, grad, var, state):
+    accum = state.get_slot(var, "accum")
+    accum_update = state.get_slot(var, "accum_update")
+    return training_ops.apply_adadelta(
+        var,
+        accum,
+        accum_update,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        state.get_hyper("rho", var.dtype.base_dtype),
+        state.get_hyper("epsilon", var.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _resource_apply_dense(self, grad, var, state):
+    accum = state.get_slot(var, "accum")
+    accum_update = state.get_slot(var, "accum_update")
+    return training_ops.resource_apply_adadelta(
+        var.handle,
+        accum.handle,
+        accum_update.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        state.get_hyper("rho", var.dtype.base_dtype),
+        state.get_hyper("epsilon", var.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _apply_sparse(self, grad, var, state):
+    accum = state.get_slot(var, "accum")
+    accum_update = state.get_slot(var, "accum_update")
+    return training_ops.sparse_apply_adadelta(
+        var,
+        accum,
+        accum_update,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        state.get_hyper("rho", var.dtype.base_dtype),
+        state.get_hyper("epsilon", var.dtype.base_dtype),
+        grad.values,
+        grad.indices,
+        use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices, state):
+    accum = state.get_slot(var, "accum")
+    accum_update = state.get_slot(var, "accum_update")
+    return training_ops.resource_sparse_apply_adadelta(
+        var.handle,
+        accum.handle,
+        accum_update.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        state.get_hyper("rho", var.dtype.base_dtype),
+        state.get_hyper("epsilon", var.dtype.base_dtype),
+        grad,
+        indices,
+        use_locking=self._use_locking)
diff --git a/tensorflow/contrib/optimizer_v2/adadelta_test.py b/tensorflow/contrib/optimizer_v2/adadelta_test.py
new file mode 100644
index 0000000000..31cfec0d50
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adadelta_test.py
@@ -0,0 +1,167 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Adadelta Optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.optimizer_v2 import adadelta
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class AdadeltaOptimizerTest(test.TestCase):
+
+  def doTestBasic(self, use_resource=False):
+    num_updates = 4  # number of ADADELTA steps to perform
+    for dtype in [dtypes.half, dtypes.float32]:
+      for grad in [0.2, 0.1, 0.01]:
+        for lr in [1.0, 0.5, 0.1]:
+          with self.test_session():
+            var0_init = [1.0, 2.0]
+            var1_init = [3.0, 4.0]
+            if use_resource:
+              var0 = resource_variable_ops.ResourceVariable(
+                  var0_init, dtype=dtype)
+              var1 = resource_variable_ops.ResourceVariable(
+                  var1_init, dtype=dtype)
+            else:
+              var0 = variables.Variable(var0_init, dtype=dtype)
+              var1 = variables.Variable(var1_init, dtype=dtype)
+
+            grads = constant_op.constant([grad, grad], dtype=dtype)
+
+            accum = 0.0
+            accum_update = 0.0
+
+            # ADADELTA gradient optimizer
+            rho = 0.95
+            epsilon = 1e-8
+            adadelta_opt = adadelta.AdadeltaOptimizer(lr, rho, epsilon)
+            adadelta_update = adadelta_opt.apply_gradients(
+                zip([grads, grads], [var0, var1]))
+
+            opt_vars = adadelta_opt.variables()
+            self.assertStartsWith(opt_vars[0].name, var0._shared_name)
+            self.assertStartsWith(opt_vars[1].name, var0._shared_name)
+            self.assertStartsWith(opt_vars[2].name, var1._shared_name)
+            self.assertStartsWith(opt_vars[3].name, var1._shared_name)
+            self.assertEqual(4, len(opt_vars))
+
+            variables.global_variables_initializer().run()
+
+            # Assign slots
+            slot = [None] * 2
+            slot_update = [None] * 2
+            self.assertEqual(["accum", "accum_update"],
+                             adadelta_opt.get_slot_names())
+            slot[0] = adadelta_opt.get_slot(var0, "accum")
+            self.assertEquals(slot[0].get_shape(), var0.get_shape())
+            self.assertFalse(slot[0] in variables.trainable_variables())
+
+            slot_update[0] = adadelta_opt.get_slot(var0, "accum_update")
+            self.assertEquals(slot_update[0].get_shape(), var0.get_shape())
+            self.assertFalse(slot_update[0] in variables.trainable_variables())
+
+            slot[1] = adadelta_opt.get_slot(var1, "accum")
+            self.assertEquals(slot[1].get_shape(), var1.get_shape())
+            self.assertFalse(slot[1] in variables.trainable_variables())
+
+            slot_update[1] = adadelta_opt.get_slot(var1, "accum_update")
+            self.assertEquals(slot_update[1].get_shape(), var1.get_shape())
+            self.assertFalse(slot_update[1] in variables.trainable_variables())
+
+            # Fetch params to validate initial values
+            self.assertAllClose(var0_init, var0.eval())
+            self.assertAllClose(var1_init, var1.eval())
+
+            update = [None] * num_updates
+            tot_update = 0
+            for step in range(num_updates):
+              # Run adadelta update for comparison
+              adadelta_update.run()
+
+              # Perform initial update without previous accum values
+              accum = accum * rho + (grad**2) * (1 - rho)
+              update[step] = (np.sqrt(accum_update + epsilon) *
+                              (1. / np.sqrt(accum + epsilon)) * grad)
+              accum_update = (accum_update * rho + (update[step]**2) *
+                              (1.0 - rho))
+              tot_update += update[step] * lr
+
+              # Check that the accumulators have been updated
+              for slot_idx in range(2):
+                self.assertAllCloseAccordingToType(
+                    np.array([accum, accum], dtype=dtype.as_numpy_dtype()),
+                    slot[slot_idx].eval(),
+                    rtol=1e-5)
+
+                self.assertAllCloseAccordingToType(
+                    np.array(
+                        [accum_update, accum_update],
+                        dtype=dtype.as_numpy_dtype()),
+                    slot_update[slot_idx].eval(),
+                    rtol=1e-5)
+
+              # Check that the parameters have been updated
+              self.assertAllCloseAccordingToType(
+                  np.array(
+                      [var0_init[0] - tot_update, var0_init[1] - tot_update],
+                      dtype=dtype.as_numpy_dtype()),
+                  var0.eval(),
+                  rtol=1e-5)
+
+              self.assertAllCloseAccordingToType(
+                  np.array(
+                      [var1_init[0] - tot_update, var1_init[1] - tot_update],
+                      dtype=dtype.as_numpy_dtype()),
+                  var1.eval(),
+                  rtol=1e-5)
+
+  def testBasic(self):
+    self.doTestBasic(use_resource=False)
+
+  def testResourceBasic(self):
+    self.doTestBasic(use_resource=True)
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = adadelta.AdadeltaOptimizer(
+            1.0, 1.0, 1.0).minimize(loss)
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[-111, -138]], var0.eval())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/adagrad.py b/tensorflow/contrib/optimizer_v2/adagrad.py
new file mode 100644
index 0000000000..e54f990cca
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adagrad.py
@@ -0,0 +1,118 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Adagrad optimizer for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import gen_array_ops
+from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.training import training_ops
+
+
+class AdagradOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the Adagrad algorithm.
+
+  See this [paper](http://www.jmlr.org/papers/volume12/duchi11a/duchi11a.pdf)
+  or this
+  [intro](http://cs.stanford.edu/~ppasupat/a9online/uploads/proximal_notes.pdf).
+  """
+
+  def __init__(self, learning_rate, initial_accumulator_value=0.1,
+               use_locking=False, name="Adagrad"):
+    """Construct a new Adagrad optimizer.
+
+    The learning_rate arg below is a hyperparameter, where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate.
+      initial_accumulator_value: A floating point value.
+        Starting value for the accumulators, must be positive.
+      use_locking: If `True` use locks for update operations.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Adagrad".
+
+    Raises:
+      ValueError: If the `initial_accumulator_value` is invalid.
+    """
+    if initial_accumulator_value <= 0.0:
+      raise ValueError("initial_accumulator_value must be positive: %s" %
+                       initial_accumulator_value)
+    super(AdagradOptimizer, self).__init__(use_locking, name)
+    self._set_hyper("learning_rate", learning_rate)
+
+    self._initial_accumulator_value = initial_accumulator_value
+
+  def _create_vars(self, var_list, state):
+    for v in var_list:
+      with ops.colocate_with(v):
+        dtype = v.dtype.base_dtype
+        if v.get_shape().is_fully_defined():
+          init = init_ops.constant_initializer(self._initial_accumulator_value,
+                                               dtype=dtype)
+        else:
+          # Use a Tensor instead of initializer if variable does not have static
+          # shape.
+          init_constant = gen_array_ops.fill(
+              array_ops.shape(v), self._initial_accumulator_value)
+          init = math_ops.cast(init_constant, dtype)
+      state.create_slot_with_initializer(v, init, v.get_shape(), dtype,
+                                         "accumulator")
+
+  def _apply_dense(self, grad, var, state):
+    acc = state.get_slot(var, "accumulator")
+    return training_ops.apply_adagrad(
+        var,
+        acc,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _resource_apply_dense(self, grad, var, state):
+    acc = state.get_slot(var, "accumulator")
+    return training_ops.resource_apply_adagrad(
+        var.handle,
+        acc.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking)
+
+  def _apply_sparse(self, grad, var, state):
+    acc = state.get_slot(var, "accumulator")
+    return training_ops.sparse_apply_adagrad(
+        var,
+        acc,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad.values,
+        grad.indices,
+        use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices, state):
+    acc = state.get_slot(var, "accumulator")
+    return training_ops.resource_sparse_apply_adagrad(
+        var.handle,
+        acc.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        indices,
+        use_locking=self._use_locking)
diff --git a/tensorflow/contrib/optimizer_v2/adagrad_test.py b/tensorflow/contrib/optimizer_v2/adagrad_test.py
new file mode 100644
index 0000000000..18191c3ef2
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adagrad_test.py
@@ -0,0 +1,282 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional tests for aggregate operations."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.optimizer_v2 import adagrad
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class AdagradOptimizerTest(test.TestCase):
+
+  def doTestBasic(self, use_locking=False, use_resource=False):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
+          var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
+        else:
+          var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+          var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        ada_opt = adagrad.AdagradOptimizer(
+            3.0, initial_accumulator_value=0.1, use_locking=use_locking)
+        ada_update = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Run 3 steps of adagrad
+        for _ in range(3):
+          ada_update.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([2.715679168701172, 3.715679168701172]), var1.eval())
+
+  def testBasic(self):
+    self.doTestBasic(use_locking=False)
+
+  def testBasicResource(self):
+    self.doTestBasic(use_locking=False, use_resource=True)
+
+  def testBasicLocked(self):
+    self.doTestBasic(use_locking=True)
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable(
+            [[1.0, 2.0], [3.0, 4.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = adagrad.AdagradOptimizer(1.0).minimize(loss)
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType(
+            [[1.0, 2.0], [3.0, 4.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[0, 1], [3, 4]], var0.eval(), atol=0.01)
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        ada_opt = adagrad.AdagradOptimizer(
+            constant_op.constant(3.0), initial_accumulator_value=0.1)
+        ada_update = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Run 3 steps of adagrad
+        for _ in range(3):
+          ada_update.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([2.715679168701172, 3.715679168701172]), var1.eval())
+
+  def testSparseBasic(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([[1.0], [2.0]], dtype=dtype)
+        var1 = variables.Variable([[3.0], [4.0]], dtype=dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1], shape=[1, 1], dtype=dtype),
+            constant_op.constant([0]),
+            constant_op.constant([2, 1]))
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(
+                [0.01], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        ada_opt = adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1)
+        ada_update = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([[1.0], [2.0]], var0.eval())
+        self.assertAllClose([[3.0], [4.0]], var1.eval())
+        # Run 3 step of sgd
+        for _ in range(3):
+          ada_update.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            np.array([[-1.6026098728179932], [2.0]]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([[3.0], [3.715679168701172]]), var1.eval())
+
+  def testSparseRepeatedIndices(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        repeated_index_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        aggregated_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]),
+            constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(
+            constant_op.constant(
+                [0.2], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        repeated_update = adagrad.AdagradOptimizer(3.0).apply_gradients(
+            [(grad_repeated_index, repeated_index_update_var)])
+        aggregated_update = adagrad.AdagradOptimizer(3.0).apply_gradients(
+            [(grad_aggregated, aggregated_update_var)])
+        variables.global_variables_initializer().run()
+        self.assertAllClose(aggregated_update_var.eval(),
+                            repeated_index_update_var.eval())
+        for _ in range(3):
+          repeated_update.run()
+          aggregated_update.run()
+          self.assertAllClose(aggregated_update_var.eval(),
+                              repeated_index_update_var.eval())
+
+  def testSparseRepeatedIndicesResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var_repeated = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype)
+        loss_repeated = math_ops.reduce_sum(
+            embedding_ops.embedding_lookup(var_repeated, [0, 0]))
+        var_aggregated = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype)
+        loss_aggregated = 2 * math_ops.reduce_sum(
+            embedding_ops.embedding_lookup(var_aggregated, [0]))
+        update_op_repeated = adagrad.AdagradOptimizer(
+            2.0).minimize(loss_repeated)
+        update_op_aggregated = adagrad.AdagradOptimizer(
+            2.0).minimize(loss_aggregated)
+        variables.global_variables_initializer().run()
+        self.assertAllCloseAccordingToType(
+            var_repeated.eval(), var_aggregated.eval())
+        for _ in range(3):
+          update_op_repeated.run()
+          update_op_aggregated.run()
+          self.assertAllCloseAccordingToType(
+              var_repeated.eval(), var_aggregated.eval())
+
+  def testSparseStability(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        shape = [1, 6]
+        var0 = variables.Variable(
+            [[
+                0.00872496, -0.106952, 0.110467, 0.226505, -0.0147257,
+                -0.0105945
+            ]],
+            dtype=dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(
+                [[
+                    -5.91278e-05, 5.31673e-05, -2.5779e-06, 4.29153e-05,
+                    -8.4877e-05, -9.48906e-05
+                ]],
+                shape=shape,
+                dtype=dtype),
+            constant_op.constant([0]),
+            constant_op.constant(shape))
+        ada_opt = adagrad.AdagradOptimizer(1.0, initial_accumulator_value=0.1)
+        ada_update = ada_opt.apply_gradients(zip([grads0], [var0]))
+        self.assertEqual(["accumulator"], ada_opt.get_slot_names())
+        slot0 = ada_opt.get_slot(var0, "accumulator")
+        init = variables.global_variables_initializer()
+        for _ in range(100):
+          init.run()
+          ada_update.run()
+          self.assertAllCloseAccordingToType(
+              np.array([[0.1, 0.1, 0.1, 0.1, 0.1, 0.1]]), slot0.eval())
+          self.assertAllCloseAccordingToType(
+              np.array([[
+                  0.00891194, -0.10712013, 0.11047515, 0.22636929, -0.0144573,
+                  -0.01029443
+              ]]), var0.eval())
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        ada_opt = adagrad.AdagradOptimizer(3.0)
+        # Apply the optimizer twice.  Both applications will use
+        # the same accums.
+        ada_update1 = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        ada_update2 = ada_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        self.assertEqual(["accumulator"], ada_opt.get_slot_names())
+        slot0 = ada_opt.get_slot(var0, "accumulator")
+        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        slot1 = ada_opt.get_slot(var1, "accumulator")
+        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values.
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Mix the first and the second adagrad for 3 steps.
+        ada_update1.run()
+        ada_update2.run()
+        ada_update1.run()
+        # Validate updated params (the same as with only 1 Adagrad).
+        self.assertAllCloseAccordingToType(
+            np.array([-1.6026098728179932, -0.6026098728179932]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([2.715679168701172, 3.715679168701172]), var1.eval())
+
+  def testDynamicShapeVariable_Ok(self):
+    with self.test_session():
+      v = variable_scope.get_variable("v", initializer=constant_op.constant(1.),
+                                      validate_shape=False)
+      self.assertFalse(v.shape.is_fully_defined())
+      # Creating optimizer should cause no exception.
+      adagrad.AdagradOptimizer(3.0, initial_accumulator_value=0.1)
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/adam.py b/tensorflow/contrib/optimizer_v2/adam.py
new file mode 100644
index 0000000000..42b7f92a76
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adam.py
@@ -0,0 +1,202 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Adam optimizer for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.training import training_ops
+
+
+class AdamOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the Adam algorithm.
+
+  See [Kingma et al., 2014](http://arxiv.org/abs/1412.6980)
+  ([pdf](http://arxiv.org/pdf/1412.6980.pdf)).
+  """
+
+  def __init__(self, learning_rate=0.001, beta1=0.9, beta2=0.999, epsilon=1e-8,
+               use_locking=False, name="Adam"):
+    """Construct a new Adam optimizer.
+
+    Initialization:
+
+    ```
+    m_0 <- 0 (Initialize initial 1st moment vector)
+    v_0 <- 0 (Initialize initial 2nd moment vector)
+    t <- 0 (Initialize timestep)
+    ```
+
+    The update rule for `variable` with gradient `g` uses an optimization
+    described at the end of section2 of the paper:
+
+    ```
+    t <- t + 1
+    lr_t <- learning_rate * sqrt(1 - beta2^t) / (1 - beta1^t)
+
+    m_t <- beta1 * m_{t-1} + (1 - beta1) * g
+    v_t <- beta2 * v_{t-1} + (1 - beta2) * g * g
+    variable <- variable - lr_t * m_t / (sqrt(v_t) + epsilon)
+    ```
+
+    The default value of 1e-8 for epsilon might not be a good default in
+    general. For example, when training an Inception network on ImageNet a
+    current good choice is 1.0 or 0.1. Note that since AdamOptimizer uses the
+    formulation just before Section 2.1 of the Kingma and Ba paper rather than
+    the formulation in Algorithm 1, the "epsilon" referred to here is "epsilon
+    hat" in the paper.
+
+    The sparse implementation of this algorithm (used when the gradient is an
+    IndexedSlices object, typically because of `tf.gather` or an embedding
+    lookup in the forward pass) does apply momentum to variable slices even if
+    they were not used in the forward pass (meaning they have a gradient equal
+    to zero). Momentum decay (beta1) is also applied to the entire momentum
+    accumulator. This means that the sparse behavior is equivalent to the dense
+    behavior (in contrast to some momentum implementations which ignore momentum
+    unless a variable slice was actually used).
+
+    Some of the args below are hyperparameters where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate.
+      beta1: A float hyperparameter. The exponential decay rate for the 1st
+        moment estimates.
+      beta2: A float hyperparameter. The exponential decay rate for the 2nd
+        moment estimates.
+      epsilon: A float hyperparameter. This epsilon is "epsilon hat" in the
+        Kingma and Ba paper (in the formula just before Section 2.1), not the
+        epsilon in Algorithm 1 of the paper.
+      use_locking: If True use locks for update operations.
+      name: Optional name for the operations created when applying gradients.
+        Defaults to "Adam".
+    """
+    super(AdamOptimizer, self).__init__(use_locking, name)
+
+    self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("beta1", beta1)
+    self._set_hyper("beta2", beta2)
+    self._set_hyper("epsilon", epsilon)
+
+  def _get_beta_accumulators(self, state=None):
+    if state is None:
+      state = self._get_per_graph_state()
+    return (state.get_non_slot("beta1_power"),
+            state.get_non_slot("beta2_power"))
+
+  def _create_vars(self, var_list, state):
+    # Non-slot variables end up on the same device(s).
+    state.create_non_slot(initial_value=state.get_hyper("beta1"),
+                          name="beta1_power")
+    state.create_non_slot(initial_value=state.get_hyper("beta2"),
+                          name="beta2_power")
+
+    # Create slots for the first and second moments.
+    for v in var_list:
+      state.zeros_slot(v, "m")
+      state.zeros_slot(v, "v")
+
+  def _apply_dense(self, grad, var, state):
+    m = state.get_slot(var, "m")
+    v = state.get_slot(var, "v")
+    beta1_power, beta2_power = self._get_beta_accumulators(state)
+    return training_ops.apply_adam(
+        var, m, v,
+        math_ops.cast(beta1_power, var.dtype.base_dtype),
+        math_ops.cast(beta2_power, var.dtype.base_dtype),
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        state.get_hyper("beta1", var.dtype.base_dtype),
+        state.get_hyper("beta2", var.dtype.base_dtype),
+        state.get_hyper("epsilon", var.dtype.base_dtype),
+        grad, use_locking=self._use_locking).op
+
+  def _resource_apply_dense(self, grad, var, state):
+    m = state.get_slot(var, "m")
+    v = state.get_slot(var, "v")
+    beta1_power, beta2_power = self._get_beta_accumulators(state)
+    return training_ops.resource_apply_adam(
+        var.handle, m.handle, v.handle,
+        math_ops.cast(beta1_power, grad.dtype.base_dtype),
+        math_ops.cast(beta2_power, grad.dtype.base_dtype),
+        state.get_hyper("learning_rate", grad.dtype.base_dtype),
+        state.get_hyper("beta1", grad.dtype.base_dtype),
+        state.get_hyper("beta2", grad.dtype.base_dtype),
+        state.get_hyper("epsilon", grad.dtype.base_dtype),
+        grad, use_locking=self._use_locking)
+
+  def _apply_sparse_shared(self, grad, var, indices, scatter_add, state):
+    beta1_power, beta2_power = self._get_beta_accumulators(state)
+    beta1_power = math_ops.cast(beta1_power, var.dtype.base_dtype)
+    beta2_power = math_ops.cast(beta2_power, var.dtype.base_dtype)
+    lr_t = state.get_hyper("learning_rate", var.dtype.base_dtype)
+    beta1_t = state.get_hyper("beta1", var.dtype.base_dtype)
+    beta2_t = state.get_hyper("beta2", var.dtype.base_dtype)
+    epsilon_t = state.get_hyper("epsilon", var.dtype.base_dtype)
+    lr = (lr_t * math_ops.sqrt(1 - beta2_power) / (1 - beta1_power))
+    # m_t = beta1 * m + (1 - beta1) * g_t
+    m = state.get_slot(var, "m")
+    m_scaled_g_values = grad * (1 - beta1_t)
+    m_t = state_ops.assign(m, m * beta1_t,
+                           use_locking=self._use_locking)
+    with ops.control_dependencies([m_t]):
+      m_t = scatter_add(m, indices, m_scaled_g_values)
+    # v_t = beta2 * v + (1 - beta2) * (g_t * g_t)
+    v = state.get_slot(var, "v")
+    v_scaled_g_values = (grad * grad) * (1 - beta2_t)
+    v_t = state_ops.assign(v, v * beta2_t, use_locking=self._use_locking)
+    with ops.control_dependencies([v_t]):
+      v_t = scatter_add(v, indices, v_scaled_g_values)
+    v_sqrt = math_ops.sqrt(v_t)
+    var_update = state_ops.assign_sub(var,
+                                      lr * m_t / (v_sqrt + epsilon_t),
+                                      use_locking=self._use_locking)
+    return control_flow_ops.group(*[var_update, m_t, v_t])
+
+  def _apply_sparse(self, grad, var, state):
+    return self._apply_sparse_shared(
+        grad.values, var, grad.indices,
+        lambda x, i, v: state_ops.scatter_add(  # pylint: disable=g-long-lambda
+            x, i, v, use_locking=self._use_locking),
+        state)
+
+  def _resource_scatter_add(self, x, i, v):
+    with ops.control_dependencies(
+        [resource_variable_ops.resource_scatter_add(
+            x.handle, i, v)]):
+      return x.value()
+
+  def _resource_apply_sparse(self, grad, var, indices, state):
+    return self._apply_sparse_shared(
+        grad, var, indices, self._resource_scatter_add, state)
+
+  def _finish(self, state):
+    # Update the power accumulators.
+    beta1_power, beta2_power = self._get_beta_accumulators(state)
+    update_beta1 = beta1_power.assign(
+        beta1_power * state.get_hyper("beta1"),
+        use_locking=self._use_locking)
+    update_beta2 = beta2_power.assign(
+        beta2_power * state.get_hyper("beta2"),
+        use_locking=self._use_locking)
+    return control_flow_ops.group(update_beta1, update_beta2)
diff --git a/tensorflow/contrib/optimizer_v2/adam_test.py b/tensorflow/contrib/optimizer_v2/adam_test.py
new file mode 100644
index 0000000000..d9ad58b0a6
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/adam_test.py
@@ -0,0 +1,333 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Adam optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.contrib.optimizer_v2 import adam
+from tensorflow.python.client import session
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+def adam_update_numpy(param,
+                      g_t,
+                      t,
+                      m,
+                      v,
+                      alpha=0.001,
+                      beta1=0.9,
+                      beta2=0.999,
+                      epsilon=1e-8):
+  alpha_t = alpha * np.sqrt(1 - beta2**t) / (1 - beta1**t)
+
+  m_t = beta1 * m + (1 - beta1) * g_t
+  v_t = beta2 * v + (1 - beta2) * g_t * g_t
+
+  param_t = param - alpha_t * m_t / (np.sqrt(v_t) + epsilon)
+  return param_t, m_t, v_t
+
+
+class AdamOptimizerTest(test.TestCase):
+
+  def doTestSparse(self, use_resource=False):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(var0_np)
+          var1 = resource_variable_ops.ResourceVariable(var1_np)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0_np_indices = np.array([0, 1], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np),
+            constant_op.constant(grads0_np_indices), constant_op.constant([2]))
+        grads1_np_indices = np.array([0, 1], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np),
+            constant_op.constant(grads1_np_indices), constant_op.constant([2]))
+        opt = adam.AdamOptimizer()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = opt._get_beta_accumulators()
+
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSparse(self):
+    self.doTestSparse(use_resource=False)
+
+  def testResourceSparse(self):
+    self.doTestSparse(use_resource=True)
+
+  def testSparseDevicePlacement(self):
+    for index_dtype in [dtypes.int32, dtypes.int64]:
+      with self.test_session(force_gpu=test.is_gpu_available()):
+        # If a GPU is available, tests that all optimizer ops can be placed on
+        # it (i.e. they have GPU kernels).
+        var = variables.Variable([[1.0], [2.0]])
+        indices = constant_op.constant([0, 1], dtype=index_dtype)
+        gathered_sum = math_ops.reduce_sum(array_ops.gather(var, indices))
+        optimizer = adam.AdamOptimizer(3.0)
+        minimize_op = optimizer.minimize(gathered_sum)
+        variables.global_variables_initializer().run()
+        minimize_op.run()
+
+  def testSparseRepeatedIndices(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        repeated_index_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        aggregated_update_var = variables.Variable(
+            [[1.0], [2.0]], dtype=dtype)
+        grad_repeated_index = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1, 0.1], shape=[2, 1], dtype=dtype),
+            constant_op.constant([1, 1]),
+            constant_op.constant([2, 1]))
+        grad_aggregated = ops.IndexedSlices(
+            constant_op.constant(
+                [0.2], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        repeated_update = adam.AdamOptimizer().apply_gradients(
+            [(grad_repeated_index, repeated_index_update_var)])
+        aggregated_update = adam.AdamOptimizer().apply_gradients(
+            [(grad_aggregated, aggregated_update_var)])
+        variables.global_variables_initializer().run()
+        self.assertAllClose(aggregated_update_var.eval(),
+                            repeated_index_update_var.eval())
+        for _ in range(3):
+          repeated_update.run()
+          aggregated_update.run()
+          self.assertAllClose(aggregated_update_var.eval(),
+                              repeated_index_update_var.eval())
+
+  def doTestBasic(self, use_resource=False):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      with self.test_session(graph=ops.Graph()):
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(
+              var0_np, name="var0_%d" % i)
+          var1 = resource_variable_ops.ResourceVariable(
+              var1_np, name="var1_%d" % i)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+
+        opt = adam.AdamOptimizer()
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        opt_variables = opt.variables()
+        beta1_power, beta2_power = opt._get_beta_accumulators()
+        self.assertTrue(beta1_power is not None)
+        self.assertTrue(beta2_power is not None)
+        self.assertIn(beta1_power, opt_variables)
+        self.assertIn(beta2_power, opt_variables)
+
+        with ops.Graph().as_default():
+          # Shouldn't return non-slot variables from other graphs.
+          self.assertEqual(0, len(opt.variables()))
+
+        if not context.executing_eagerly():
+          self.evaluate(variables.global_variables_initializer())
+          # Fetch params to validate initial values
+          self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+          self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+        beta1_power, beta2_power = opt._get_beta_accumulators()
+
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          if not context.executing_eagerly():
+            self.evaluate(update)
+          elif t > 1:
+            opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+
+          self.assertAllCloseAccordingToType(0.9**(t + 1),
+                                             self.evaluate(beta1_power))
+          self.assertAllCloseAccordingToType(0.999**(t + 1),
+                                             self.evaluate(beta2_power))
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, self.evaluate(var0))
+          self.assertAllCloseAccordingToType(var1_np, self.evaluate(var1))
+          if use_resource:
+            self.assertEqual("var0_%d/Adam:0" % (i,),
+                             opt.get_slot(var=var0, name="m").name)
+
+  def testBasic(self):
+    with self.test_session():
+      self.doTestBasic(use_resource=False)
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testResourceBasic(self):
+    self.doTestBasic(use_resource=True)
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adam.AdamOptimizer(constant_op.constant(0.001))
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        beta1_power, beta2_power = opt._get_beta_accumulators()
+
+        # Run 3 steps of Adam
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          update.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        # Initialize variables for numpy implementation.
+        m0, v0, m1, v1 = 0.0, 0.0, 0.0, 0.0
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = adam.AdamOptimizer()
+        update1 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        update2 = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        beta1_power, beta2_power = opt._get_beta_accumulators()
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 3 steps of intertwined Adam1 and Adam2.
+        for t in range(1, 4):
+          self.assertAllCloseAccordingToType(0.9**t, beta1_power.eval())
+          self.assertAllCloseAccordingToType(0.999**t, beta2_power.eval())
+          if t % 2 == 0:
+            update1.run()
+          else:
+            update2.run()
+
+          var0_np, m0, v0 = adam_update_numpy(var0_np, grads0_np, t, m0, v0)
+          var1_np, m1, v1 = adam_update_numpy(var1_np, grads1_np, t, m1, v1)
+
+          # Validate updated params
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testTwoSessions(self):
+    optimizer = adam.AdamOptimizer()
+    g = ops.Graph()
+    with g.as_default():
+      with session.Session():
+        var0 = variables.Variable(np.array([1.0, 2.0]), name="v0")
+        grads0 = constant_op.constant(np.array([0.1, 0.1]))
+        optimizer.apply_gradients([(grads0, var0)])
+
+    gg = ops.Graph()
+    with gg.as_default():
+      with session.Session():
+        var0 = variables.Variable(np.array([1.0, 2.0]), name="v0")
+        grads0 = constant_op.constant(np.array([0.1, 0.1]))
+
+        # If the optimizer saves any state not keyed by graph the following line
+        # fails.
+        optimizer.apply_gradients([(grads0, var0)])
+
+  def testSlotsUniqueEager(self):
+    with context.eager_mode():
+      v1 = resource_variable_ops.ResourceVariable(1.)
+      v2 = resource_variable_ops.ResourceVariable(1.)
+      opt = adam.AdamOptimizer(1.)
+      opt.minimize(lambda: v1 + v2)
+      # There should be two non-slot variables, and two unique slot variables
+      # for v1 and v2 respectively.
+      self.assertEqual(6, len(set(opt.variables())))
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
new file mode 100644
index 0000000000..08f9699e85
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/checkpointable_utils_test.py
@@ -0,0 +1,686 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# TODO(josh11b): Forked from contrib/eager/python to test OptimizerV2 the same way
+# OptimizerV1 is tested. This file should be removed once the fork is resolved.
+
+import functools
+import os
+
+import six
+
+from tensorflow.contrib.eager.python import checkpointable_utils
+from tensorflow.contrib.optimizer_v2 import adam
+from tensorflow.python.client import session as session_lib
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.eager import test
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.keras._impl.keras.engine import training
+from tensorflow.python.layers import core
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.training import checkpointable
+from tensorflow.python.training import saver as core_saver
+from tensorflow.python.training import training_util
+
+
+class NonLayerCheckpointable(checkpointable.Checkpointable):
+
+  def __init__(self):
+    super(NonLayerCheckpointable, self).__init__()
+    self.a_variable = checkpointable_utils.add_variable(
+        self, name="a_variable", shape=[])
+
+
+# pylint: disable=not-callable
+class MyModel(training.Model):
+  """A concrete Model for testing."""
+
+  def __init__(self):
+    super(MyModel, self).__init__()
+    self._named_dense = core.Dense(1, use_bias=True)
+    self._second = core.Dense(1, use_bias=False)
+    # We can still track Checkpointables which aren't Layers.
+    self._non_layer = NonLayerCheckpointable()
+
+  def call(self, values):
+    ret = self._second(self._named_dense(values))
+    return ret
+
+
+class _MirroringSaveable(
+    core_saver.BaseSaverBuilder.ResourceVariableSaveable):
+
+  def __init__(self, primary_variable, mirrored_variable, name):
+    self._primary_variable = primary_variable
+    self._mirrored_variable = mirrored_variable
+    super(_MirroringSaveable, self).__init__(
+        self._primary_variable, "", name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    """Restore the same value into both variables."""
+    tensor, = restored_tensors
+    return control_flow_ops.group(
+        self._primary_variable.assign(tensor),
+        self._mirrored_variable.assign(tensor))
+
+
+class _OwnsMirroredVariables(checkpointable.CheckpointableBase):
+  """A Checkpointable object which returns a more complex SaveableObject."""
+
+  def __init__(self):
+    self.non_dep_variable = variable_scope.get_variable(
+        name="non_dep_variable", initializer=6., use_resource=True)
+    self.mirrored = variable_scope.get_variable(
+        name="mirrored", initializer=15., use_resource=True)
+
+  def _gather_saveables_for_checkpoint(self):
+    def _saveable_factory(name=self.non_dep_variable.name):
+      return _MirroringSaveable(
+          primary_variable=self.non_dep_variable,
+          mirrored_variable=self.mirrored,
+          name=name)
+    return {checkpointable.VARIABLE_VALUE_KEY: _saveable_factory}
+
+  # The Saver sorts by name before parsing, so we need a name property.
+  @property
+  def name(self):
+    return self.non_dep_variable.name
+
+
+class CheckpointingTests(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testNamingWithOptimizer(self):
+    input_value = constant_op.constant([[3.]])
+    model = MyModel()
+    # A nuisance Model using the same optimizer. Its slot variables should not
+    # go in the checkpoint, since it is never depended on.
+    other_model = MyModel()
+    optimizer = adam.AdamOptimizer(0.001)
+    optimizer_step = training_util.get_or_create_global_step()
+    root_checkpointable = checkpointable_utils.Checkpoint(
+        optimizer=optimizer, model=model, optimizer_step=optimizer_step)
+    if context.executing_eagerly():
+      optimizer.minimize(
+          lambda: model(input_value),
+          global_step=optimizer_step)
+      optimizer.minimize(
+          lambda: other_model(input_value),
+          global_step=optimizer_step)
+    else:
+      train_op = optimizer.minimize(
+          model(input_value), global_step=optimizer_step)
+      optimizer.minimize(
+          other_model(input_value),
+          global_step=optimizer_step)
+      self.evaluate(checkpointable_utils.gather_initializers(
+          root_checkpointable))
+      self.evaluate(train_op)
+    named_variables, serialized_graph = (
+        checkpointable_utils._serialize_object_graph(root_checkpointable))
+    expected_checkpoint_names = (
+        # Created in the root node, so no prefix.
+        "optimizer_step",
+        "model/_second/kernel",
+        "model/_named_dense/kernel",
+        "model/_named_dense/bias",
+        # non-Layer dependency of the model
+        "model/_non_layer/a_variable",
+        # The optimizer creates two non-slot variables
+        "optimizer/beta1_power",
+        "optimizer/beta2_power",
+        # Slot variables
+        "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/m",
+        "model/_second/kernel/.OPTIMIZER_SLOT/optimizer/v",
+        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m",
+        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/v",
+        "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/m",
+        "model/_named_dense/bias/.OPTIMIZER_SLOT/optimizer/v",
+    )
+    suffix = "/.ATTRIBUTES/VARIABLE_VALUE"
+    expected_checkpoint_names = [
+        name + suffix for name in expected_checkpoint_names]
+    six.assertCountEqual(self, expected_checkpoint_names,
+                         named_variables.keys())
+    # Check that we've mapped to the right variable objects (not exhaustive)
+    self.assertEqual(
+        "global_step:0",
+        named_variables["optimizer_step" + suffix].name)
+    self.assertEqual(
+        "my_model/dense_1/kernel:0",
+        named_variables["model/_second/kernel" + suffix].name)
+    self.assertEqual(
+        "my_model/dense/kernel:0",
+        named_variables["model/_named_dense/kernel" + suffix].name)
+    self.assertEqual(
+        "beta1_power:0",
+        named_variables["optimizer/beta1_power" + suffix].name)
+    self.assertEqual(
+        "beta2_power:0",
+        named_variables["optimizer/beta2_power" + suffix].name)
+    # Spot check the generated protocol buffers.
+    self.assertEqual("optimizer",
+                     serialized_graph.nodes[0].children[1].local_name)
+    optimizer_node = serialized_graph.nodes[serialized_graph.nodes[0].children[
+        1].node_id]
+    self.assertEqual("beta1_power",
+                     optimizer_node.children[0].local_name)
+    self.assertEqual("beta1_power",
+                     serialized_graph.nodes[optimizer_node.children[0].node_id]
+                     .attributes[0].full_name)
+    self.assertEqual(
+        "my_model/dense/kernel",
+        serialized_graph.nodes[optimizer_node.slot_variables[0]
+                               .original_variable_node_id]
+        .attributes[0].full_name)
+    # We strip off the :0 suffix, as variable.name-based saving does.
+    self.assertEqual(
+        "my_model/dense/kernel/Adam",
+        serialized_graph.nodes[optimizer_node.slot_variables[0]
+                               .slot_variable_node_id]
+        .attributes[0].full_name)
+    self.assertEqual(
+        "my_model/dense/kernel/Adam:0",
+        optimizer.get_slot(
+            var=named_variables["model/_named_dense/kernel" + suffix],
+            name="m").name)
+    self.assertEqual(
+        "model/_named_dense/kernel" + suffix,
+        serialized_graph.nodes[
+            optimizer_node.slot_variables[0]
+            .original_variable_node_id].attributes[0].checkpoint_key)
+    self.assertEqual("m", optimizer_node.slot_variables[0].slot_name)
+    self.assertEqual(
+        "model/_named_dense/kernel/.OPTIMIZER_SLOT/optimizer/m" + suffix,
+        serialized_graph.nodes[
+            optimizer_node.slot_variables[0]
+            .slot_variable_node_id].attributes[0].checkpoint_key)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testSaveRestore(self):
+    model = MyModel()
+    optimizer = adam.AdamOptimizer(0.001)
+    root_checkpointable = checkpointable_utils.Checkpoint(
+        optimizer=optimizer, model=model)
+    input_value = constant_op.constant([[3.]])
+    if context.executing_eagerly():
+      optimizer.minimize(
+          lambda: model(input_value))
+    else:
+      train_op = optimizer.minimize(model(input_value))
+      # TODO(allenl): Make initialization more pleasant when graph building.
+      root_checkpointable.save_counter  # pylint: disable=pointless-statement
+      self.evaluate(checkpointable_utils.gather_initializers(
+          root_checkpointable))
+      self.evaluate(train_op)
+    prefix = os.path.join(self.get_temp_dir(), "ckpt")
+    self.evaluate(state_ops.assign(model._named_dense.variables[1], [42.]))
+    m_bias_slot = optimizer.get_slot(model._named_dense.variables[1], "m")
+    self.evaluate(state_ops.assign(m_bias_slot, [1.5]))
+    save_path = root_checkpointable.save(file_prefix=prefix)
+    self.evaluate(state_ops.assign(model._named_dense.variables[1], [43.]))
+    self.evaluate(state_ops.assign(root_checkpointable.save_counter, 3))
+    optimizer_variables = self.evaluate(optimizer.variables())
+    self.evaluate(state_ops.assign(m_bias_slot, [-2.]))
+    # Immediate restoration
+    status = root_checkpointable.restore(save_path=save_path).assert_consumed()
+    status.run_restore_ops()
+    self.assertAllEqual([42.], self.evaluate(model._named_dense.variables[1]))
+    self.assertAllEqual(1, self.evaluate(root_checkpointable.save_counter))
+    self.assertAllEqual([1.5], self.evaluate(m_bias_slot))
+    if not context.executing_eagerly():
+      return  # Restore-on-create is only supported when executing eagerly
+    on_create_model = MyModel()
+    on_create_optimizer = adam.AdamOptimizer(
+        0.001,
+        # Preserve beta1_power and beta2_power when appying gradients so we can
+        # test that they've been restored correctly.
+        beta1=1.0, beta2=1.0)
+    on_create_root = checkpointable_utils.Checkpoint(
+        optimizer=on_create_optimizer, model=on_create_model)
+    # Deferred restoration
+    status = on_create_root.restore(save_path=save_path)
+    on_create_model(constant_op.constant([[3.]]))  # create variables
+    self.assertAllEqual(1, self.evaluate(on_create_root.save_counter))
+    self.assertAllEqual([42.],
+                        self.evaluate(
+                            on_create_model._named_dense.variables[1]))
+    on_create_m_bias_slot = on_create_optimizer.get_slot(
+        on_create_model._named_dense.variables[1], "m")
+    # Optimizer slot variables are created when the original variable is
+    # restored.
+    self.assertAllEqual([1.5], self.evaluate(on_create_m_bias_slot))
+    self.assertAllEqual(optimizer_variables[2:],
+                        self.evaluate(on_create_optimizer.variables()))
+    dummy_var = resource_variable_ops.ResourceVariable([1.])
+    on_create_optimizer.minimize(loss=dummy_var.read_value)
+    status.assert_consumed()
+    beta1_power, beta2_power = on_create_optimizer._get_beta_accumulators()
+    self.assertAllEqual(optimizer_variables[0], self.evaluate(beta1_power))
+    self.assertAllEqual(optimizer_variables[1], self.evaluate(beta2_power))
+
+  # TODO(allenl): Debug garbage created by this test in python3.
+  def testDeferredRestorationUsageEager(self):
+    """An idiomatic eager execution example."""
+    num_training_steps = 10
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    for training_continuation in range(3):
+      model = MyModel()
+      optimizer = adam.AdamOptimizer(0.001)
+      root = checkpointable_utils.Checkpoint(
+          optimizer=optimizer, model=model,
+          optimizer_step=training_util.get_or_create_global_step())
+      root.restore(core_saver.latest_checkpoint(checkpoint_directory))
+      for _ in range(num_training_steps):
+        # TODO(allenl): Use a Dataset and serialize/checkpoint it.
+        input_value = constant_op.constant([[3.]])
+        optimizer.minimize(
+            lambda: model(input_value),  # pylint: disable=cell-var-from-loop
+            global_step=root.optimizer_step)
+      root.save(file_prefix=checkpoint_prefix)
+      self.assertEqual((training_continuation + 1) * num_training_steps,
+                       root.optimizer_step.numpy())
+
+  def testUsageGraph(self):
+    """Expected usage when graph building."""
+    with context.graph_mode():
+      num_training_steps = 10
+      checkpoint_directory = self.get_temp_dir()
+      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+      for training_continuation in range(3):
+        with ops.Graph().as_default():
+          model = MyModel()
+          optimizer = adam.AdamOptimizer(0.001)
+          root = checkpointable_utils.Checkpoint(
+              optimizer=optimizer, model=model,
+              global_step=training_util.get_or_create_global_step())
+          input_value = constant_op.constant([[3.]])
+          train_op = optimizer.minimize(
+              model(input_value),
+              global_step=root.global_step)
+          checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory)
+          with self.test_session(graph=ops.get_default_graph()) as session:
+            status = root.restore(save_path=checkpoint_path)
+            status.initialize_or_restore(session=session)
+            if checkpoint_path is None:
+              self.assertEqual(0, training_continuation)
+              with self.assertRaises(AssertionError):
+                status.assert_consumed()
+            else:
+              status.assert_consumed()
+            for _ in range(num_training_steps):
+              session.run(train_op)
+            root.save(file_prefix=checkpoint_prefix, session=session)
+            self.assertEqual((training_continuation + 1) * num_training_steps,
+                             session.run(root.global_step))
+            self.assertEqual(training_continuation + 1,
+                             session.run(root.save_counter))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testAgnosticUsage(self):
+    """Graph/eager agnostic usage."""
+    # Does create garbage when executing eagerly due to ops.Graph() creation.
+    num_training_steps = 10
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    for training_continuation in range(3):
+      with ops.Graph().as_default(), self.test_session(
+          graph=ops.get_default_graph()), test_util.device(use_gpu=True):
+        model = MyModel()
+        optimizer = adam.AdamOptimizer(0.001)
+        root = checkpointable_utils.Checkpoint(
+            optimizer=optimizer, model=model,
+            global_step=training_util.get_or_create_global_step())
+        checkpoint_path = core_saver.latest_checkpoint(checkpoint_directory)
+        status = root.restore(save_path=checkpoint_path)
+        input_value = constant_op.constant([[3.]])
+        train_fn = functools.partial(
+            optimizer.minimize,
+            functools.partial(model, input_value),
+            global_step=root.global_step)
+        if not context.executing_eagerly():
+          train_fn = functools.partial(self.evaluate, train_fn())
+        status.initialize_or_restore()
+        for _ in range(num_training_steps):
+          train_fn()
+        root.save(file_prefix=checkpoint_prefix)
+        self.assertEqual((training_continuation + 1) * num_training_steps,
+                         self.evaluate(root.global_step))
+        self.assertEqual(training_continuation + 1,
+                         self.evaluate(root.save_counter))
+
+  def _get_checkpoint_name(self, name):
+    root = checkpointable.Checkpointable()
+    checkpointable_utils.add_variable(
+        root, name=name, shape=[1, 2], dtype=dtypes.float64)
+    named_variables, _ = checkpointable_utils._serialize_object_graph(root)
+    checkpoint_name, = named_variables.keys()
+    with ops.name_scope("root/" + checkpoint_name):
+      pass  # Make sure we can use this as an op name if we prefix it.
+    return checkpoint_name
+
+  def testAnonymousVarsInInit(self):
+
+    class Model(training.Model):
+
+      def __init__(self):
+        super(Model, self).__init__()
+        self.w = resource_variable_ops.ResourceVariable(0.0)
+        self.b = resource_variable_ops.ResourceVariable(0.0)
+        self.vars = [self.w, self.b]
+
+      def call(self, x):
+        return x * self.w + self.b
+
+    with context.eager_mode():
+      model = Model()
+      optimizer = adam.AdamOptimizer(learning_rate=0.05)
+      checkpoint_directory = self.get_temp_dir()
+      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+      checkpoint = checkpointable_utils.Checkpoint(
+          model=model, optimizer=optimizer)
+      for _ in range(2):
+        checkpoint.save(checkpoint_prefix)
+        with backprop.GradientTape() as tape:
+          loss = (constant_op.constant(1.)
+                  - model(constant_op.constant(1.))) ** 2
+        grad = tape.gradient(loss, model.vars)
+        optimizer.apply_gradients(
+            [(g, v) for g, v in zip(grad, model.vars)])
+
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testDeferredSlotRestoration(self):
+    checkpoint_directory = self.get_temp_dir()
+
+    root = checkpointable.Checkpointable()
+    root.var = checkpointable_utils.add_variable(
+        root, name="var", initializer=0.)
+    optimizer = adam.AdamOptimizer(0.1)
+    if context.executing_eagerly():
+      optimizer.minimize(root.var.read_value)
+    else:
+      train_op = optimizer.minimize(root.var)
+      # Note that `optimizer` has not been added as a dependency of
+      # `root`. Create a one-off grouping so that slot variables for `root.var`
+      # get initialized too.
+      self.evaluate(checkpointable_utils.gather_initializers(
+          checkpointable_utils.Checkpoint(root=root, optimizer=optimizer)))
+      self.evaluate(train_op)
+    self.evaluate(state_ops.assign(root.var, 12.))
+    no_slots_path = checkpointable_utils.CheckpointableSaver(root).save(
+        os.path.join(checkpoint_directory, "no_slots"))
+    root.optimizer = optimizer
+    self.evaluate(state_ops.assign(root.var, 13.))
+    self.evaluate(state_ops.assign(optimizer.get_slot(name="m", var=root.var),
+                                   14.))
+    slots_path = checkpointable_utils.CheckpointableSaver(root).save(
+        os.path.join(checkpoint_directory, "with_slots"))
+    new_root = checkpointable.Checkpointable()
+    # Load the slot-containing checkpoint (deferred), then immediately overwrite
+    # the non-slot variable (also deferred).
+    slot_status = checkpointable_utils.CheckpointableSaver(
+        new_root).restore(slots_path)
+    no_slot_status = checkpointable_utils.CheckpointableSaver(
+        new_root).restore(no_slots_path)
+    with self.assertRaises(AssertionError):
+      no_slot_status.assert_consumed()
+    new_root.var = checkpointable_utils.add_variable(
+        new_root, name="var", shape=[])
+    no_slot_status.assert_consumed()
+    no_slot_status.run_restore_ops()
+    self.assertEqual(12., self.evaluate(new_root.var))
+    new_root.optimizer = adam.AdamOptimizer(0.1)
+    with self.assertRaisesRegexp(AssertionError, "beta1_power"):
+      slot_status.assert_consumed()
+    self.assertEqual(12., self.evaluate(new_root.var))
+    if context.executing_eagerly():
+      # Slot variables are only created with restoring initializers when
+      # executing eagerly.
+      self.assertEqual(14., self.evaluate(
+          new_root.optimizer.get_slot(name="m", var=new_root.var)))
+    else:
+      self.assertIs(new_root.optimizer.get_slot(name="m", var=new_root.var),
+                    None)
+    if context.executing_eagerly():
+      new_root.optimizer.minimize(new_root.var.read_value)
+    else:
+      train_op = new_root.optimizer.minimize(new_root.var)
+      # The slot variable now exists; restore() didn't create it, but we should
+      # now have a restore op for it.
+      slot_status.run_restore_ops()
+      self.assertEqual(14., self.evaluate(
+          new_root.optimizer.get_slot(name="m", var=new_root.var)))
+      self.evaluate(train_op)
+    slot_status.assert_consumed()
+
+  def testManySavesGraph(self):
+    """Saves after the first should not modify the graph."""
+    with context.graph_mode():
+      graph = ops.Graph()
+      with graph.as_default(), self.test_session(graph):
+        checkpoint_directory = self.get_temp_dir()
+        checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+        obj = checkpointable.Checkpointable()
+        obj.var = variable_scope.get_variable(name="v", initializer=0.)
+        obj.opt = adam.AdamOptimizer(0.1)
+        obj.opt.minimize(obj.var.read_value())
+        self.evaluate(checkpointable_utils.gather_initializers(obj))
+        saver = checkpointable_utils.CheckpointableSaver(obj)
+        saver.save(checkpoint_prefix)
+        before_ops = graph.get_operations()
+        saver.save(checkpoint_prefix)
+        self.assertEqual(before_ops, graph.get_operations())
+
+  def testManyRestoresGraph(self):
+    """Restores after the first should not modify the graph."""
+    with context.graph_mode():
+      graph = ops.Graph()
+      with graph.as_default(), self.test_session(graph):
+        checkpoint_directory = self.get_temp_dir()
+        checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+        obj = checkpointable.Checkpointable()
+        obj.var = variable_scope.get_variable(name="v", initializer=0.)
+        obj.opt = adam.AdamOptimizer(0.1)
+        obj.opt.minimize(obj.var.read_value())
+        self.evaluate(checkpointable_utils.gather_initializers(obj))
+        saver = checkpointable_utils.CheckpointableSaver(obj)
+        save_path = saver.save(checkpoint_prefix)
+        saver.restore(save_path)
+        before_ops = graph.get_operations()
+        saver.restore(save_path)
+        self.assertEqual(before_ops, graph.get_operations())
+
+  def testMultipleGraphsNonSlotVariables(self):
+    with context.graph_mode():
+      checkpoint_directory = self.get_temp_dir()
+      checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+      optimizer = adam.AdamOptimizer(0.001)
+      # Construct a model in one graph
+      first_graph = ops.Graph()
+      first_session = session_lib.Session(graph=first_graph)
+      with first_graph.as_default(), first_session.as_default():
+        first_variable = resource_variable_ops.ResourceVariable([1.])
+        first_root_checkpointable = checkpointable_utils.Checkpoint(
+            optimizer=optimizer, variable=first_variable)
+        train_op = optimizer.minimize(first_variable.read_value)
+        self.evaluate(checkpointable_utils.gather_initializers(
+            first_root_checkpointable))
+        self.evaluate(train_op)
+        self.evaluate(first_variable.assign([1.]))
+        self.evaluate(optimizer.get_slot(
+            var=first_variable, name="m").assign([2.]))
+        beta1_power, _ = optimizer._get_beta_accumulators()
+        self.evaluate(beta1_power.assign(3.))
+
+      # Save and load in a second graph
+      second_graph = ops.Graph()
+      with second_graph.as_default(), session_lib.Session(graph=second_graph):
+        second_variable = resource_variable_ops.ResourceVariable([1.])
+        second_root_checkpointable = checkpointable_utils.Checkpoint(
+            optimizer=optimizer, variable=second_variable)
+        train_op = optimizer.minimize(second_variable.read_value)
+        second_root_checkpointable.restore(None).initialize_or_restore()
+        self.evaluate(train_op)
+        self.evaluate(second_variable.assign([4.]))
+        self.evaluate(optimizer.get_slot(
+            var=second_variable, name="m").assign([5.]))
+        beta1_power, _ = optimizer._get_beta_accumulators()
+        self.evaluate(beta1_power.assign(6.))
+        save_path = second_root_checkpointable.save(checkpoint_prefix)
+        self.evaluate(second_variable.assign([7.]))
+        self.evaluate(optimizer.get_slot(
+            var=second_variable, name="m").assign([8.]))
+        beta1_power, _ = optimizer._get_beta_accumulators()
+        self.assertAllEqual(6., self.evaluate(beta1_power))
+        status = second_root_checkpointable.restore(save_path)
+        status.assert_consumed().run_restore_ops()
+        self.assertAllEqual([4.], self.evaluate(second_variable))
+        self.assertAllEqual([5.], self.evaluate(optimizer.get_slot(
+            var=second_variable, name="m")))
+        beta1_power, _ = optimizer._get_beta_accumulators()
+        self.assertAllEqual(6., self.evaluate(beta1_power))
+
+      # Check that the first graph is unmolested
+      with first_graph.as_default(), first_session.as_default():
+        self.assertAllEqual([1.], self.evaluate(first_variable))
+        self.assertAllEqual([2.], self.evaluate(optimizer.get_slot(
+            var=first_variable, name="m")))
+        beta1_power, _ = optimizer._get_beta_accumulators()
+        self.assertAllEqual(3., self.evaluate(beta1_power))
+
+
+class CheckpointCompatibilityTests(test.TestCase):
+
+  def _initialized_model(self):
+    input_value = constant_op.constant([[3.]])
+    model = MyModel()
+    optimizer = adam.AdamOptimizer(0.001)
+    optimizer_step = training_util.get_or_create_global_step()
+    root_checkpointable = checkpointable_utils.Checkpoint(
+        optimizer=optimizer, model=model, optimizer_step=optimizer_step)
+    train_op = optimizer.minimize(
+        functools.partial(model, input_value),
+        global_step=optimizer_step)
+    self.evaluate(checkpointable_utils.gather_initializers(
+        root_checkpointable))
+    self.evaluate(train_op)
+    # A regular variable, a slot variable, and a non-slot Optimizer variable
+    # with known values to check when loading.
+    self.evaluate(model._named_dense.bias.assign([1.]))
+    self.evaluate(optimizer.get_slot(
+        var=model._named_dense.bias, name="m").assign([2.]))
+    beta1_power, _ = optimizer._get_beta_accumulators()
+    self.evaluate(beta1_power.assign(3.))
+    return root_checkpointable
+
+  def _set_sentinels(self, root_checkpointable):
+    self.evaluate(root_checkpointable.model._named_dense.bias.assign([101.]))
+    self.evaluate(
+        root_checkpointable.optimizer.get_slot(
+            var=root_checkpointable.model._named_dense.bias, name="m")
+        .assign([102.]))
+    beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators()
+    self.evaluate(beta1_power.assign(103.))
+
+  def _check_sentinels(self, root_checkpointable):
+    self.assertAllEqual(
+        [1.], self.evaluate(root_checkpointable.model._named_dense.bias))
+    self.assertAllEqual([2.], self.evaluate(
+        root_checkpointable.optimizer.get_slot(
+            var=root_checkpointable.model._named_dense.bias, name="m")))
+    beta1_power, _ = root_checkpointable.optimizer._get_beta_accumulators()
+    self.assertAllEqual(3., self.evaluate(beta1_power))
+
+  def _write_name_based_checkpoint(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    with context.graph_mode():
+      save_graph = ops.Graph()
+      with save_graph.as_default(), self.test_session(
+          graph=save_graph) as session:
+        root = self._initialized_model()
+        name_saver = core_saver.Saver()
+        return name_saver.save(
+            sess=session, save_path=checkpoint_prefix,
+            global_step=root.optimizer_step)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testLoadFromNameBasedSaver(self):
+    """Save a name-based checkpoint, load it using the object-based API."""
+    with test_util.device(use_gpu=True):
+      save_path = self._write_name_based_checkpoint()
+      root = self._initialized_model()
+      self._set_sentinels(root)
+      with self.assertRaises(AssertionError):
+        self._check_sentinels(root)
+      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      status = object_saver.restore(save_path)
+      with self.assertRaises(AssertionError):
+        status.assert_consumed()
+      status.run_restore_ops()
+      self._check_sentinels(root)
+      self._set_sentinels(root)
+      status.initialize_or_restore()
+      self._check_sentinels(root)
+
+  # TODO(allenl): Test for the core name-based saver loading object-based
+  # checkpoints once object-based checkpointing is in core.
+
+  def testSaveGraphLoadEager(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    with context.graph_mode():
+      save_graph = ops.Graph()
+      with save_graph.as_default(), self.test_session(
+          graph=save_graph) as session:
+        root = self._initialized_model()
+        object_saver = checkpointable_utils.CheckpointableSaver(root)
+        save_path = object_saver.save(
+            session=session, file_prefix=checkpoint_prefix)
+    with context.eager_mode():
+      root = self._initialized_model()
+      self._set_sentinels(root)
+      root.restore(save_path).assert_consumed()
+      self._check_sentinels(root)
+
+  def testSaveEagerLoadGraph(self):
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    with context.eager_mode():
+      root = self._initialized_model()
+      object_saver = checkpointable_utils.CheckpointableSaver(root)
+      save_path = object_saver.save(file_prefix=checkpoint_prefix)
+    with context.graph_mode():
+      save_graph = ops.Graph()
+      with save_graph.as_default(), self.test_session(
+          graph=save_graph):
+        root = self._initialized_model()
+        self._set_sentinels(root)
+        root.restore(save_path).assert_consumed().run_restore_ops()
+        self._check_sentinels(root)
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/gradient_descent.py b/tensorflow/contrib/optimizer_v2/gradient_descent.py
new file mode 100644
index 0000000000..945c8de559
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/gradient_descent.py
@@ -0,0 +1,69 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""GradientDescent optimizer for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.training import training_ops
+
+
+class GradientDescentOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the gradient descent algorithm."""
+
+  def __init__(self, learning_rate, use_locking=False, name="GradientDescent"):
+    """Construct a new gradient descent optimizer.
+
+    The learning rate arg below is a hyperparameter where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate to use.
+      use_locking: If True use locks for update operations.
+      name: Optional name prefix for the operations created when applying
+        gradients. Defaults to "GradientDescent".
+    """
+    super(GradientDescentOptimizer, self).__init__(use_locking, name)
+    self._set_hyper("learning_rate", learning_rate)
+
+  def _apply_dense(self, grad, var, state):
+    return training_ops.apply_gradient_descent(
+        var,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        use_locking=self._use_locking).op
+
+  def _resource_apply_dense(self, grad, handle, state):
+    lr = state.get_hyper("learning_rate", grad.dtype.base_dtype)
+    return training_ops.resource_apply_gradient_descent(
+        handle.handle, lr, grad, use_locking=self._use_locking)
+
+  def _resource_apply_sparse_duplicate_indices(
+      self, grad, handle, indices, state):
+    lr = state.get_hyper("learning_rate", grad.dtype.base_dtype)
+    return resource_variable_ops.resource_scatter_add(
+        handle.handle, indices, -grad * lr)
+
+  def _apply_sparse_duplicate_indices(self, grad, var, state):
+    delta = ops.IndexedSlices(
+        grad.values * state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad.indices, grad.dense_shape)
+    return var.scatter_sub(delta, use_locking=self._use_locking)
diff --git a/tensorflow/contrib/optimizer_v2/gradient_descent_test.py b/tensorflow/contrib/optimizer_v2/gradient_descent_test.py
new file mode 100644
index 0000000000..ad9aef804f
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/gradient_descent_test.py
@@ -0,0 +1,223 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional test for GradientDescent optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import gradient_descent
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import resources
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class GradientDescentOptimizerTest(test.TestCase):
+
+  def testBasic(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        optimizer = gradient_descent.GradientDescentOptimizer(3.0)
+        sgd_op = optimizer.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
+        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                           var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                           var1.eval())
+        self.assertEqual(0, len(optimizer.variables()))
+
+  def testBasicResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        # TODO(apassos) calling initialize_resources on all resources here
+        # doesn't work because the sessions and graph are reused across unit
+        # tests and this would mean trying to reinitialize variables. Figure out
+        # a long-term solution for this.
+        resources.initialize_resources([var0, var1]).run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
+        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                           var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                           var1.eval())
+
+  def testMinimizeResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(var0, x) + var1
+        loss = pred * pred
+        sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss)
+        # TODO(apassos) calling initialize_resources on all resources here
+        # doesn't work because the sessions and graph are reused across unit
+        # tests and this would mean trying to reinitialize variables. Figure out
+        # a long-term solution for this.
+        resources.initialize_resources([var0, var1]).run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([3.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0
+        np_grad = 2 * np_pred
+        self.assertAllCloseAccordingToType(
+            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval())
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        var1 = resource_variable_ops.ResourceVariable([3.0], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        pred += var1
+        loss = pred * pred
+        sgd_op = gradient_descent.GradientDescentOptimizer(1.0).minimize(loss)
+        # TODO(apassos) calling initialize_resources on all resources here
+        # doesn't work because the sessions and graph are reused across unit
+        # tests and this would mean trying to reinitialize variables. Figure out
+        # a long-term solution for this.
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([3.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        np_pred = 1.0 * 4.0 + 2.0 * 5.0 + 3.0
+        np_grad = 2 * np_pred
+        self.assertAllCloseAccordingToType(
+            [[1.0 - np_grad * 4.0, 2.0 - np_grad * 5.0]], var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - np_grad], var1.eval())
+
+  def testTensorLearningRate(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        lrate = constant_op.constant(3.0)
+        sgd_op = gradient_descent.GradientDescentOptimizer(
+            lrate).apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
+        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                           var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                           var1.eval())
+
+  def testGradWrtRef(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        opt = gradient_descent.GradientDescentOptimizer(3.0)
+        values = [1.0, 3.0]
+        vars_ = [variables.Variable([v], dtype=dtype) for v in values]
+        grads_and_vars = opt.compute_gradients(vars_[0] + vars_[1], vars_)
+        variables.global_variables_initializer().run()
+        for grad, _ in grads_and_vars:
+          self.assertAllCloseAccordingToType([1.0], grad.eval())
+
+  def testWithGlobalStep(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        global_step = variables.Variable(0, trainable=False)
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients(
+            zip([grads0, grads1], [var0, var1]), global_step=global_step)
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([1.0, 2.0], var0.eval())
+        self.assertAllCloseAccordingToType([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params and global_step
+        self.assertAllCloseAccordingToType([1.0 - 3.0 * 0.1, 2.0 - 3.0 * 0.1],
+                                           var0.eval())
+        self.assertAllCloseAccordingToType([3.0 - 3.0 * 0.01, 4.0 - 3.0 * 0.01],
+                                           var1.eval())
+        self.assertAllCloseAccordingToType(1, global_step.eval())
+
+  def testSparseBasic(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([[1.0], [2.0]], dtype=dtype)
+        var1 = variables.Variable([[3.0], [4.0]], dtype=dtype)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(
+                [0.1], shape=[1, 1], dtype=dtype),
+            constant_op.constant([0]),
+            constant_op.constant([2, 1]))
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(
+                [0.01], shape=[1, 1], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([2, 1]))
+        sgd_op = gradient_descent.GradientDescentOptimizer(3.0).apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0], [2.0]], var0.eval())
+        self.assertAllCloseAccordingToType([[3.0], [4.0]], var1.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType([[1.0 - 3.0 * 0.1], [2.0]],
+                                           var0.eval())
+        self.assertAllCloseAccordingToType([[3.0], [4.0 - 3.0 * 0.01]],
+                                           var1.eval())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/momentum.py b/tensorflow/contrib/optimizer_v2/momentum.py
new file mode 100644
index 0000000000..0a5aadc2d1
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/momentum.py
@@ -0,0 +1,124 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Momentum for TensorFlow."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.training import training_ops
+
+
+class MomentumOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the Momentum algorithm.
+
+  Computes (if `use_nesterov = False`):
+
+  ```
+  accumulation = momentum * accumulation + gradient
+  variable -= learning_rate * accumulation
+  ```
+
+  Note that in the dense version of this algorithm, `accumulation` is updated
+  and applied regardless of a gradient's value, whereas the sparse version (when
+  the gradient is an `IndexedSlices`, typically because of `tf.gather` or an
+  embedding) only updates variable slices and corresponding `accumulation` terms
+  when that part of the variable was used in the forward pass.
+  """
+
+  def __init__(self, learning_rate, momentum,
+               use_locking=False, name="Momentum", use_nesterov=False):
+    """Construct a new Momentum optimizer.
+
+    Some of the args below are hyperparameters, where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate.
+      momentum: A float hyperparameter. The momentum.
+      use_locking: If `True` use locks for update operations.
+      name: Optional name prefix for the operations created when applying
+        gradients.  Defaults to "Momentum".
+      use_nesterov: If `True` use Nesterov Momentum.
+        See [Sutskever et al., 2013](
+        http://jmlr.org/proceedings/papers/v28/sutskever13.pdf).
+        This implementation always computes gradients at the value of the
+        variable(s) passed to the optimizer. Using Nesterov Momentum makes the
+        variable(s) track the values called `theta_t + mu*v_t` in the paper.
+
+    @compatibility(eager)
+    When eager execution is enabled, learning_rate and momentum can each be a
+    callable that takes no arguments and returns the actual value to use. This
+    can be useful for changing these values across different invocations of
+    optimizer functions.
+    @end_compatibility
+    """
+    super(MomentumOptimizer, self).__init__(use_locking, name)
+    self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("momentum", momentum)
+    self._use_nesterov = use_nesterov
+
+  def _create_vars(self, var_list, state):
+    for v in var_list:
+      state.zeros_slot(v, "momentum")
+
+  def _apply_dense(self, grad, var, state):
+    mom = state.get_slot(var, "momentum")
+    return training_ops.apply_momentum(
+        var,
+        mom,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        state.get_hyper("momentum", var.dtype.base_dtype),
+        use_locking=self._use_locking,
+        use_nesterov=self._use_nesterov).op
+
+  def _resource_apply_dense(self, grad, var, state):
+    mom = state.get_slot(var, "momentum")
+    return training_ops.resource_apply_momentum(
+        var.handle,
+        mom.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        state.get_hyper("momentum", var.dtype.base_dtype),
+        use_locking=self._use_locking,
+        use_nesterov=self._use_nesterov)
+
+  def _apply_sparse(self, grad, var, state):
+    mom = state.get_slot(var, "momentum")
+    return training_ops.sparse_apply_momentum(
+        var,
+        mom,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad.values,
+        grad.indices,
+        state.get_hyper("momentum", var.dtype.base_dtype),
+        use_locking=self._use_locking,
+        use_nesterov=self._use_nesterov).op
+
+  def _resource_apply_sparse(self, grad, var, indices, state):
+    mom = state.get_slot(var, "momentum")
+    return training_ops.resource_sparse_apply_momentum(
+        var.handle,
+        mom.handle,
+        state.get_hyper("learning_rate", var.dtype.base_dtype),
+        grad,
+        indices,
+        state.get_hyper("momentum", var.dtype.base_dtype),
+        use_locking=self._use_locking,
+        use_nesterov=self._use_nesterov)
diff --git a/tensorflow/contrib/optimizer_v2/momentum_test.py b/tensorflow/contrib/optimizer_v2/momentum_test.py
new file mode 100644
index 0000000000..f37eb48181
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/momentum_test.py
@@ -0,0 +1,562 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Momentum."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+from six.moves import xrange  # pylint: disable=redefined-builtin
+
+from tensorflow.contrib.optimizer_v2 import momentum as momentum_lib
+from tensorflow.python.eager import context
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class MomentumOptimizerTest(test.TestCase):
+
+  def _update_nesterov_momentum_numpy(self, var, accum, g, lr, momentum):
+    var = var + accum * lr * momentum
+    accum = accum * momentum + g
+    var = var - lr * accum
+    var = var - accum * lr * momentum
+    return var, accum
+
+  def doTestBasic(self, use_resource=False, use_callable_params=False):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      if use_resource:
+        var0 = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype, name="var0_%d" % i)
+        var1 = resource_variable_ops.ResourceVariable(
+            [3.0, 4.0], dtype=dtype, name="var1_%d" % i)
+      else:
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+      grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+      grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+      learning_rate = lambda: 2.0
+      momentum = lambda: 0.9
+      if not use_callable_params:
+        learning_rate = learning_rate()
+        momentum = momentum()
+      mom_opt = momentum_lib.MomentumOptimizer(
+          learning_rate=learning_rate, momentum=momentum)
+      mom_update = mom_opt.apply_gradients(
+          zip([grads0, grads1], [var0, var1]))
+
+      if not context.executing_eagerly():
+        self.evaluate(variables.global_variables_initializer())
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+        self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+      # Check we have slots
+      self.assertEqual(["momentum"], mom_opt.get_slot_names())
+      slot0 = mom_opt.get_slot(var0, "momentum")
+      self.assertEquals(slot0.get_shape(), var0.get_shape())
+      slot1 = mom_opt.get_slot(var1, "momentum")
+      self.assertEquals(slot1.get_shape(), var1.get_shape())
+      if not context.executing_eagerly():
+        self.assertFalse(slot0 in variables.trainable_variables())
+        self.assertFalse(slot1 in variables.trainable_variables())
+
+      # Step 1: the momentum accumulators where 0. So we should see a normal
+      # update: v -= grad * learning_rate
+      if not context.executing_eagerly():
+        self.evaluate(mom_update)
+      # Check that the momentum accumulators have been updated.
+      self.assertAllCloseAccordingToType(np.array([0.1, 0.1]),
+                                         self.evaluate(slot0))
+      self.assertAllCloseAccordingToType(np.array([0.01, 0.01]),
+                                         self.evaluate(slot1))
+      # Check that the parameters have been updated.
+      self.assertAllCloseAccordingToType(
+          np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]),
+          self.evaluate(var0))
+      self.assertAllCloseAccordingToType(
+          np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]),
+          self.evaluate(var1))
+      # Step 2: the momentum accumulators contain the previous update.
+      if context.executing_eagerly():
+        mom_opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+      else:
+        self.evaluate(mom_update)
+      # Check that the momentum accumulators have been updated.
+      self.assertAllCloseAccordingToType(
+          np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]),
+          self.evaluate(slot0))
+      self.assertAllCloseAccordingToType(
+          np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+          self.evaluate(slot1))
+      # Check that the parameters have been updated.
+      self.assertAllCloseAccordingToType(
+          np.array([
+              1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
+              2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
+          ]), self.evaluate(var0))
+      self.assertAllCloseAccordingToType(
+          np.array([
+              2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
+                  (0.9 * 0.01 + 0.01) * 2.0)
+          ]), self.evaluate(var1))
+
+  def testBasic(self):
+    with self.test_session():
+      self.doTestBasic(use_resource=False)
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testResourceBasic(self):
+    self.doTestBasic(use_resource=True)
+
+  def testBasicCallableParams(self):
+    with context.eager_mode():
+      self.doTestBasic(use_resource=True, use_callable_params=True)
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testVariablesAcrossGraphs(self):
+    optimizer = momentum_lib.MomentumOptimizer(0.01, 0.5)
+    with ops.Graph().as_default():
+      var0 = resource_variable_ops.ResourceVariable(
+          [1.0, 2.0], dtype=dtypes.float32, name="var0")
+      var1 = resource_variable_ops.ResourceVariable(
+          [3.0, 4.0], dtype=dtypes.float32, name="var1")
+      if context.executing_eagerly():
+        loss = lambda: math_ops.reduce_sum(var0 + var1)
+      else:
+        loss = math_ops.reduce_sum(var0 + var1)
+      optimizer.minimize(loss)
+      optimizer_variables = optimizer.variables()
+      self.assertStartsWith(optimizer_variables[0].name, "var0")
+      self.assertStartsWith(optimizer_variables[1].name, "var1")
+      self.assertEquals(2, len(optimizer_variables))
+
+    with ops.Graph().as_default():
+      var2 = resource_variable_ops.ResourceVariable(
+          [1.0, 2.0], dtype=dtypes.float32, name="var2")
+      var3 = resource_variable_ops.ResourceVariable(
+          [3.0, 4.0], dtype=dtypes.float32, name="var3")
+      if context.executing_eagerly():
+        loss = lambda: math_ops.reduce_sum(var2 + var3)
+      else:
+        loss = math_ops.reduce_sum(var2 + var3)
+      optimizer.minimize(loss)
+      optimizer_variables = optimizer.variables()
+      self.assertStartsWith(optimizer_variables[0].name, "var2")
+      self.assertStartsWith(optimizer_variables[1].name, "var3")
+      self.assertEquals(2, len(optimizer_variables))
+
+  def testNesterovMomentum(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        cost = 5 * var0 * var0 + 3 * var1
+        global_step = variables.Variable(
+            array_ops.zeros([], dtypes.int64), name="global_step")
+        mom_op = momentum_lib.MomentumOptimizer(
+            learning_rate=2.0, momentum=0.9, use_nesterov=True)
+        opt_op = mom_op.minimize(cost, global_step, [var0, var1])
+        variables.global_variables_initializer().run()
+        for t in range(1, 5):
+          opt_op.run()
+          var0_np, accum0_np = self._update_nesterov_momentum_numpy(
+              var0_np, accum0_np, var0_np * 10, 2.0, 0.9)
+          var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np,
+                                                                    accum1_np,
+                                                                    3, 2.0, 0.9)
+          self.assertAllClose(var0_np, var0.eval())
+          self.assertAllClose(var1_np, var1.eval())
+
+  def testSparseNesterovMomentum(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        grads = []
+        for t in range(1, 5):
+          grads.append(var0_np * 10)
+          var0_np, accum0_np = self._update_nesterov_momentum_numpy(
+              var0_np, accum0_np, var0_np * 10, 2.0, 0.9)
+          var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np,
+                                                                    accum1_np,
+                                                                    3, 2.0, 0.9)
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        accum0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        accum1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        loss = 5 * var0 * var0 + 3 * var1
+        mom_op = momentum_lib.MomentumOptimizer(
+            learning_rate=2.0, momentum=0.9, use_nesterov=True)
+        x_feed = array_ops.placeholder(dtype)
+        y_feed = ops.IndexedSlices(
+            x_feed, constant_op.constant([0, 1]), constant_op.constant([2]))
+        grads_and_vars = [(y_feed, var0), (constant_op.constant(
+            [3.0, 3.0], dtype=dtype), var1)]
+        opt_update = mom_op.apply_gradients(grads_and_vars)
+        variables.global_variables_initializer().run()
+        for t in range(1, 5):
+          opt_update.run(feed_dict={x_feed: grads[t - 1]})
+          var0_np, accum0_np = self._update_nesterov_momentum_numpy(
+              var0_np, accum0_np, var0_np * 10, 2.0, 0.9)
+          var1_np, accum1_np = self._update_nesterov_momentum_numpy(var1_np,
+                                                                    accum1_np,
+                                                                    3, 2.0, 0.9)
+          self.assertAllClose(var0_np, var0.eval())
+          self.assertAllClose(var1_np, var1.eval())
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+
+      # pylint: disable=cell-var-from-loop
+      def loss():
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        return pred * pred
+      # pylint: enable=cell-var-from-loop
+
+      opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0)
+      sgd_op = opt.minimize(loss)
+      self.evaluate(variables.global_variables_initializer())
+      # Run 1 step of sgd
+      self.evaluate(sgd_op)
+      # Validate updated params
+      self.assertAllCloseAccordingToType([[-111, -138]], self.evaluate(var0))
+
+  @test_util.run_in_graph_and_eager_modes(reset_test=True)
+  def testMinimizeWith2DIndiciesForEmbeddingLookup(self):
+    var0 = resource_variable_ops.ResourceVariable(array_ops.ones([2, 2]))
+
+    def loss():
+      return math_ops.reduce_sum(embedding_ops.embedding_lookup(var0, [[1]]))
+
+    opt = momentum_lib.MomentumOptimizer(learning_rate=1.0, momentum=0.0)
+    sgd_op = opt.minimize(loss)
+    self.evaluate(variables.global_variables_initializer())
+    self.evaluate(sgd_op)
+    self.assertAllCloseAccordingToType([[1, 1], [0, 0]], self.evaluate(var0))
+
+  def testTensorLearningRateAndMomentum(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        mom_opt = momentum_lib.MomentumOptimizer(
+            learning_rate=constant_op.constant(2.0),
+            momentum=constant_op.constant(0.9))
+        mom_update = mom_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+        # Check we have slots
+        self.assertEqual(["momentum"], mom_opt.get_slot_names())
+        slot0 = mom_opt.get_slot(var0, "momentum")
+        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        self.assertFalse(slot0 in variables.trainable_variables())
+        slot1 = mom_opt.get_slot(var1, "momentum")
+        self.assertEquals(slot1.get_shape(), var1.get_shape())
+        self.assertFalse(slot1 in variables.trainable_variables())
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Step 1: the momentum accumulators where 0. So we should see a normal
+        # update: v -= grad * learning_rate
+        mom_update.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
+        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        # Check that the parameters have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+        # Step 2: the momentum accumulators contain the previous update.
+        mom_update.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+        # Check that the parameters have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
+                2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
+            ]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
+                    (0.9 * 0.01 + 0.01) * 2.0)
+            ]), var1.eval())
+
+  def _dbParamsMom01(self):
+    """Return dist-belief momentum values.
+
+    Return values been generated from the dist-belief momentum unittest,
+    running with a learning rate of 0.1 and a momentum of 0.1.
+
+    These values record how a parameter vector of size 10, initialized with 0.0,
+    gets updated with 10 consecutive momentum steps.  It uses random gradients.
+
+    Returns:
+      db_grad: The gradients to apply
+      db_out: The parameters after the momentum update.
+    """
+    db_grad = [[]] * 10
+    db_out = [[]] * 10
+    # pylint: disable=line-too-long
+    db_grad[0] = [
+        0.00096264342, 0.17914793, 0.93945462, 0.41396621, 0.53037018,
+        0.93197989, 0.78648776, 0.50036013, 0.55345792, 0.96722615
+    ]
+    db_out[0] = [
+        -9.6264346e-05, -0.017914793, -0.093945466, -0.041396622, -0.053037018,
+        -0.093197994, -0.078648776, -0.050036013, -0.055345792, -0.096722618
+    ]
+    db_grad[1] = [
+        0.17075552, 0.88821375, 0.20873757, 0.25236958, 0.57578111, 0.15312378,
+        0.5513742, 0.94687688, 0.16012503, 0.22159521
+    ]
+    db_out[1] = [
+        -0.017181443, -0.10852765, -0.12421377, -0.070773244, -0.11591884,
+        -0.11783017, -0.14165108, -0.14972731, -0.076892875, -0.1285544
+    ]
+    db_grad[2] = [
+        0.35077485, 0.47304362, 0.44412705, 0.44368884, 0.078527533, 0.81223965,
+        0.31168157, 0.43203235, 0.16792089, 0.24644311
+    ]
+    db_out[2] = [
+        -0.053967446, -0.1648933, -0.1716533, -0.1180798, -0.13005978,
+        -0.20151734, -0.17911947, -0.20289968, -0.095839672, -0.15638189
+    ]
+    db_grad[3] = [
+        0.9694621, 0.75035888, 0.28171822, 0.83813518, 0.53807181, 0.3728098,
+        0.81454384, 0.03848977, 0.89759839, 0.93665648
+    ]
+    db_out[3] = [
+        -0.15459226, -0.24556576, -0.20456907, -0.20662397, -0.18528105,
+        -0.24716705, -0.2643207, -0.21206589, -0.18749419, -0.2528303
+    ]
+    db_grad[4] = [
+        0.38578293, 0.8536852, 0.88722926, 0.66276771, 0.13678469, 0.94036359,
+        0.69107032, 0.81897682, 0.5433259, 0.67860287
+    ]
+    db_out[4] = [
+        -0.20323303, -0.33900154, -0.29658359, -0.28175515, -0.20448165,
+        -0.34576839, -0.34194785, -0.29488021, -0.25099224, -0.33033544
+    ]
+    db_grad[5] = [
+        0.27885768, 0.76100707, 0.24625534, 0.81354135, 0.18959245, 0.48038563,
+        0.84163809, 0.41172323, 0.83259648, 0.44941229
+    ]
+    db_out[5] = [
+        -0.23598288, -0.42444581, -0.33041057, -0.3706224, -0.22536094,
+        -0.40366709, -0.43387437, -0.34433398, -0.34060168, -0.38302717
+    ]
+    db_grad[6] = [
+        0.27233034, 0.056316052, 0.5039115, 0.24105175, 0.35697976, 0.75913221,
+        0.73577434, 0.16014607, 0.57500273, 0.071136251
+    ]
+    db_out[6] = [
+        -0.26649091, -0.43862185, -0.38418442, -0.40361428, -0.26314685,
+        -0.48537019, -0.51664448, -0.36529395, -0.40706289, -0.39540997
+    ]
+    db_grad[7] = [
+        0.58697265, 0.2494842, 0.08106143, 0.39954534, 0.15892942, 0.12683646,
+        0.74053431, 0.16033, 0.66625422, 0.73515922
+    ]
+    db_out[7] = [
+        -0.32823896, -0.46498787, -0.39766794, -0.446868, -0.28281838,
+        -0.50622416, -0.59897494, -0.38342294, -0.48033443, -0.47016418
+    ]
+    db_grad[8] = [
+        0.8215279, 0.41994119, 0.95172721, 0.68000203, 0.79439718, 0.43384039,
+        0.55561525, 0.22567581, 0.93331909, 0.29438227
+    ]
+    db_out[8] = [
+        -0.41656655, -0.50961858, -0.49418902, -0.51919359, -0.36422527,
+        -0.55169362, -0.6627695, -0.40780342, -0.58099347, -0.50707781
+    ]
+    db_grad[9] = [
+        0.68297005, 0.67758518, 0.1748755, 0.13266537, 0.70697063, 0.055731893,
+        0.68593478, 0.50580865, 0.12602448, 0.093537711
+    ]
+    db_out[9] = [
+        -0.49369633, -0.58184016, -0.52132869, -0.5396927, -0.44306302,
+        -0.56181377, -0.73774242, -0.46082234, -0.60366184, -0.52012295
+    ]
+    # pylint: enable=line-too-long
+    return db_grad, db_out
+
+  def testLikeDistBeliefMom01(self):
+    with self.test_session():
+      db_grad, db_out = self._dbParamsMom01()
+      num_samples = len(db_grad)
+      var0 = variables.Variable([0.0] * num_samples)
+      grads0 = constant_op.constant([0.0] * num_samples)
+      mom_opt = momentum_lib.MomentumOptimizer(learning_rate=0.1, momentum=0.1)
+      mom_update = mom_opt.apply_gradients(zip([grads0], [var0]))
+      variables.global_variables_initializer().run()
+      for i in xrange(num_samples):
+        mom_update.run(feed_dict={grads0: db_grad[i]})
+        self.assertAllClose(np.array(db_out[i]), var0.eval())
+
+  def testSparse(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable(array_ops.zeros([4, 2], dtype=dtype))
+        var1 = variables.Variable(constant_op.constant(1.0, dtype, [4, 2]))
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(
+                [[.1, .1]], dtype=dtype),
+            constant_op.constant([1]),
+            constant_op.constant([4, 2]))
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(
+                [[.01, .01], [.01, .01]], dtype=dtype),
+            constant_op.constant([2, 3]),
+            constant_op.constant([4, 2]))
+        mom_opt = momentum_lib.MomentumOptimizer(
+            learning_rate=2.0, momentum=0.9)
+        mom_update = mom_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        # Check we have slots
+        self.assertEqual(["momentum"], mom_opt.get_slot_names())
+        slot0 = mom_opt.get_slot(var0, "momentum")
+        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        slot1 = mom_opt.get_slot(var1, "momentum")
+        self.assertEquals(slot1.get_shape(), var1.get_shape())
+
+        # Fetch params to validate initial values
+        self.assertAllClose([0, 0], var0.eval()[0])
+        self.assertAllClose([0, 0], var0.eval()[1])
+        self.assertAllClose([1, 1], var1.eval()[2])
+
+        # Step 1: the momentum accumulators are 0. So we should see a normal
+        # update: v -= grad * learning_rate
+        mom_update.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllCloseAccordingToType(np.array([0, 0]), slot0.eval()[0])
+        self.assertAllCloseAccordingToType(np.array([.1, .1]), slot0.eval()[1])
+        self.assertAllCloseAccordingToType(
+            np.array([.01, .01]), slot1.eval()[2])
+        # Check that the parameters have been updated.
+        self.assertAllCloseAccordingToType(np.array([0, 0]), var0.eval()[0])
+        self.assertAllCloseAccordingToType(
+            np.array([-(0.1 * 2.0), -(0.1 * 2.0)]), var0.eval()[1])
+        self.assertAllCloseAccordingToType(
+            np.array([1.0 - (0.01 * 2.0), 1.0 - (0.01 * 2.0)]), var1.eval()[2])
+        # Step 2: the momentum accumulators contain the previous update.
+        mom_update.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllClose(np.array([0, 0]), slot0.eval()[0])
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval()[1])
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]),
+            slot1.eval()[2])
+        # Check that the parameters have been updated.
+        self.assertAllClose(np.array([0, 0]), var0.eval()[0])
+        self.assertAllCloseAccordingToType(
+            np.array([
+                -(0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0), -(0.1 * 2.0) - (
+                    (0.9 * 0.1 + 0.1) * 2.0)
+            ]), var0.eval()[1])
+        self.assertAllCloseAccordingToType(
+            np.array([
+                0.98 - ((0.9 * 0.01 + 0.01) * 2.0), 0.98 - (
+                    (0.9 * 0.01 + 0.01) * 2.0)
+            ]), var1.eval()[2])
+
+  def testSharing(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        mom_opt = momentum_lib.MomentumOptimizer(
+            learning_rate=2.0, momentum=0.9)
+        mom_update1 = mom_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        mom_update2 = mom_opt.apply_gradients(
+            zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        self.assertEqual(["momentum"], mom_opt.get_slot_names())
+        slot0 = mom_opt.get_slot(var0, "momentum")
+        self.assertEquals(slot0.get_shape(), var0.get_shape())
+        slot1 = mom_opt.get_slot(var1, "momentum")
+        self.assertEquals(slot1.get_shape(), var1.get_shape())
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Step 1: the momentum accumulators where 0. So we should see a normal
+        # update: v -= grad * learning_rate
+        mom_update1.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllCloseAccordingToType(np.array([0.1, 0.1]), slot0.eval())
+        self.assertAllCloseAccordingToType(np.array([0.01, 0.01]), slot1.eval())
+        # Check that the parameters have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([1.0 - (0.1 * 2.0), 2.0 - (0.1 * 2.0)]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([3.0 - (0.01 * 2.0), 4.0 - (0.01 * 2.0)]), var1.eval())
+        # Step 2: the second momentum accumulators contain the previous update.
+        mom_update2.run()
+        # Check that the momentum accumulators have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.1 + 0.1), (0.9 * 0.1 + 0.1)]), slot0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([(0.9 * 0.01 + 0.01), (0.9 * 0.01 + 0.01)]), slot1.eval())
+        # Check that the parameters have been updated.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0),
+                2.0 - (0.1 * 2.0) - ((0.9 * 0.1 + 0.1) * 2.0)
+            ]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                2.98 - ((0.9 * 0.01 + 0.01) * 2.0), 3.98 - (
+                    (0.9 * 0.01 + 0.01) * 2.0)
+            ]), var1.eval())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
new file mode 100644
index 0000000000..471992fdac
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -0,0 +1,1352 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""Version 2 of class Optimizer."""
+# pylint: disable=g-bad-name
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import abc
+
+from tensorflow.python.eager import backprop
+from tensorflow.python.eager import context
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gradients
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.ops import variables
+from tensorflow.python.training import checkpointable
+from tensorflow.python.training import distribute as distribute_lib
+from tensorflow.python.training import optimizer as optimizer_v1
+from tensorflow.python.training import slot_creator
+from tensorflow.python.util import nest
+
+
+class _OptimizableVariable(object):
+  """Interface for abstracting over variables in the optimizers."""
+
+  @abc.abstractmethod
+  def target(self):
+    """Returns the optimization target for this variable."""
+    raise NotImplementedError("Calling an abstract method.")
+
+  @abc.abstractmethod
+  def update_op(self, optimizer, g, *args):
+    """Returns the update ops for updating the variable."""
+    raise NotImplementedError("Calling an abstract method.")
+
+
+class _RefVariableProcessor(_OptimizableVariable):
+  """Processor for Variable."""
+
+  def __init__(self, v):
+    self._v = v
+
+  def target(self):
+    return self._v._ref()  # pylint: disable=protected-access
+
+  def update_op(self, optimizer, g, *args):
+    if isinstance(g, ops.Tensor):
+      update_op = optimizer._apply_dense(g, self._v, *args)  # pylint: disable=protected-access
+      if self._v.constraint is not None:
+        with ops.control_dependencies([update_op]):
+          return self._v.assign(self._v.constraint(self._v))
+      else:
+        return update_op
+    else:
+      assert isinstance(g, ops.IndexedSlices), ("Gradient ", g, " is neither a "
+                                                "tensor nor IndexedSlices.")
+      if self._v.constraint is not None:
+        raise RuntimeError(
+            "Cannot use a constraint function on a sparse variable.")
+      # pylint: disable=protected-access
+      return optimizer._apply_sparse_duplicate_indices(g, self._v, *args)
+
+
+class _DenseReadResourceVariableProcessor(_OptimizableVariable):
+  """Processor for dense ResourceVariables."""
+
+  def __init__(self, v):
+    self._v = v
+
+  def target(self):
+    return self._v
+
+  def update_op(self, optimizer, g, *args):
+    # pylint: disable=protected-access
+    update_op = optimizer._resource_apply_dense(g, self._v.op.inputs[0], *args)
+    if self._v.constraint is not None:
+      with ops.control_dependencies([update_op]):
+        return self._v.assign(self._v.constraint(self._v))
+    else:
+      return update_op
+
+
+class _DenseResourceVariableProcessor(_OptimizableVariable):
+  """Processor for dense ResourceVariables."""
+
+  def __init__(self, v):
+    self._v = v
+
+  def target(self):
+    return self._v
+
+  def update_op(self, optimizer, g, *args):
+    # pylint: disable=protected-access
+    if isinstance(g, ops.IndexedSlices):
+      if self._v.constraint is not None:
+        raise RuntimeError(
+            "Cannot use a constraint function on a sparse variable.")
+      return optimizer._resource_apply_sparse_duplicate_indices(
+          g.values, self._v, g.indices, *args)
+    update_op = optimizer._resource_apply_dense(g, self._v, *args)
+    if self._v.constraint is not None:
+      with ops.control_dependencies([update_op]):
+        return self._v.assign(self._v.constraint(self._v))
+    else:
+      return update_op
+
+
+class _StreamingModelPortProcessor(_OptimizableVariable):
+  """Processor for streaming ModelPorts."""
+
+  def __init__(self, v):
+    self._v = v
+
+  def target(self):
+    return self._v
+
+  def update_op(self, optimizer, g, *args):
+    return g
+
+
+class _TensorProcessor(_OptimizableVariable):
+  """Processor for ordinary Tensors.
+
+  Even though a Tensor can't really be updated, sometimes it is useful to
+  compute the gradients with respect to a Tensor using the optimizer. Updating
+  the Tensor is, of course, unsupported.
+  """
+
+  def __init__(self, v):
+    self._v = v
+
+  def target(self):
+    return self._v
+
+  def update_op(self, optimizer, g, *args):
+    raise NotImplementedError("Trying to update a Tensor ", self._v)
+
+
+def _get_processor(v):
+  """The processor of v."""
+  if context.executing_eagerly():
+    if isinstance(v, ops.Tensor):
+      return _TensorProcessor(v)
+    else:
+      return _DenseResourceVariableProcessor(v)
+  if v.op.type == "VarHandleOp":
+    return _DenseResourceVariableProcessor(v)
+  if isinstance(v, variables.Variable):
+    return _RefVariableProcessor(v)
+  if v.op.type == "SubmodelPort":
+    return _StreamingModelPortProcessor(v)
+  if isinstance(v, ops.Tensor):
+    return _TensorProcessor(v)
+  raise NotImplementedError("Trying to optimize unsupported type ", v)
+
+
+def _var_key_v2(var):
+  """Key for representing a primary variable, for looking up slots."""
+  # pylint: disable=protected-access
+  if hasattr(var, "_mirrored_container"):
+    mirrored_container = var._mirrored_container()
+    assert mirrored_container is not None
+    if context.executing_eagerly():
+      return mirrored_container._unique_id
+    return mirrored_container._shared_name
+  if context.executing_eagerly():
+    return var._unique_id
+  return var.op.name
+
+
+def _resolve(value, name):
+  if callable(value):
+    value = value()
+  return ops.convert_to_tensor(value, name=name)
+
+
+def _is_dynamic(value):
+  """Returns true if __init__ arg `value` should be re-evaluated each step."""
+  if callable(value): return True
+  # Don't need to do anything special in graph mode, since dynamic values
+  # will propagate correctly automatically.
+  # TODO(josh11b): Add per-device caching across steps using variables for
+  # truly static values once we add distributed support.
+  if context.executing_eagerly() and isinstance(
+      value, resource_variable_ops.ResourceVariable):
+    return True
+  return False
+
+
+class _OptimizerV2State(object):
+  """Holds per-graph and per-step optimizer state.
+
+  Use _init_with_static_hyper() to create the state for a graph, and then
+  _copy_with_dynamic_hyper() to convert that to state for a particular step.
+  The difference between the two is that the former only has hyper
+  parameter values that are static and the latter also has values that
+  can change every step (according to _is_dynamic()).
+  """
+
+  def __init__(self, op_name):
+    self._op_name = op_name
+
+  def _init_with_static_hyper(self, hyper):
+    """Initialize a fresh state object from hyper dict."""
+    # self._hyper contains a dict from name to a dict with the Tensor values.
+    # This dict starts with a single item with key "None" with the hyper
+    # parameter value converted to a Tensor. Other items have dtype keys
+    # with that Tensor cast to that dtype.
+    self._hyper = {name: {None: ops.convert_to_tensor(value, name=name)}
+                   for name, (dynamic, value) in hyper.items() if not dynamic}
+    self._slots = {}
+    self._non_slot_dict = {}
+    # Extra state to help Optimizers implement Checkpointable. Holds information
+    # about variables which will be restored as soon as they're created.
+    self._deferred_dependencies = {}  # Non-slot variables
+    self._deferred_slot_restorations = {}  # Slot variables
+
+  def _copy_with_dynamic_hyper(self, hyper, distribution, non_slot_devices):
+    """Create a new state object for a particular step."""
+    ret = _OptimizerV2State(self._op_name)
+    # pylint: disable=protected-access
+    ret._slots = self._slots
+    ret._non_slot_dict = self._non_slot_dict
+    ret._deferred_dependencies = self._deferred_dependencies
+    ret._deferred_slot_restorations = self._deferred_slot_restorations
+    ret._hyper = {name: {None: _resolve(value, name)}
+                  for name, (dynamic, value) in hyper.items() if dynamic}
+    ret._hyper.update(self._hyper)
+    ret._non_slot_devices = non_slot_devices
+    ret._distribution = distribution
+    return ret
+
+  def _variables(self):
+    """Returns a list of all variables held by self."""
+    optimizer_variables = list(self._non_slot_dict.values())
+    for variable_dict in self._slots.values():
+      for slot_for_variable in variable_dict.values():
+        optimizer_variables.append(slot_for_variable)
+    # Sort variables by name so that the return is deterministic.
+    return sorted(optimizer_variables, key=lambda v: v.name)
+
+  def _slot_dict(self, slot_name):
+    """Returns a dict for caching slots created under the given name.
+
+    Args:
+      slot_name: Name for the slot.
+
+    Returns:
+      A dict that maps primary `Variable` objects to the slot created
+      for that variable, under the given slot name.
+    """
+    named_slots = self._slots.get(slot_name, None)
+    if named_slots is None:
+      named_slots = {}
+      self._slots[slot_name] = named_slots
+    return named_slots
+
+  def create_slot(self, var, val, slot_name, optional_op_name=None):
+    """Find or create a slot for a variable.
+
+    Args:
+      var: A `Variable` object.
+      val: A `Tensor`.  The initial value of the slot.
+      slot_name: Name for the slot.
+      optional_op_name: Name to use when scoping the Variable that
+        needs to be created for the slot.
+
+    Returns:
+      A `Variable` object.
+    """
+    named_slots = self._slot_dict(slot_name)
+    var_key = _var_key_v2(var)
+    if var_key not in named_slots:
+      new_slot_variable = slot_creator.create_slot(
+          var, val, optional_op_name or self._op_name)
+      self._restore_slot_variable(
+          slot_name=slot_name, variable=var,
+          slot_variable=new_slot_variable)
+      named_slots[var_key] = new_slot_variable
+    return named_slots[var_key]
+
+  def create_slot_with_initializer(self, var, initializer, shape, dtype,
+                                   slot_name, optional_op_name=None):
+    """Find or create a slot for a variable, using an Initializer.
+
+    Args:
+      var: A `Variable` object.
+      initializer: An `Initializer`.  The initial value of the slot.
+      shape: Shape of the initial value of the slot.
+      dtype: Type of the value of the slot.
+      slot_name: Name for the slot.
+      optional_op_name: Name to use when scoping the Variable that
+        needs to be created for the slot.
+
+    Returns:
+      A `Variable` object.
+    """
+    named_slots = self._slot_dict(slot_name)
+    var_key = _var_key_v2(var)
+    if var_key not in named_slots:
+      new_slot_variable = slot_creator.create_slot_with_initializer(
+          var, initializer, shape, dtype, optional_op_name or self._op_name)
+      self._restore_slot_variable(
+          slot_name=slot_name, variable=var,
+          slot_variable=new_slot_variable)
+      named_slots[var_key] = new_slot_variable
+    return named_slots[var_key]
+
+  def zeros_slot(self, var, slot_name, optional_op_name=None):
+    """Find or create a slot initialized with 0.0.
+
+    Args:
+      var: A `Variable` object.
+      slot_name: Name for the slot.
+      optional_op_name: Name to use when scoping the Variable that
+        needs to be created for the slot.
+
+    Returns:
+      A `Variable` object.
+    """
+    named_slots = self._slot_dict(slot_name)
+    var_key = _var_key_v2(var)
+    if var_key not in named_slots:
+      new_slot_variable = slot_creator.create_zeros_slot(
+          var, optional_op_name or self._op_name)
+      self._restore_slot_variable(
+          slot_name=slot_name, variable=var,
+          slot_variable=new_slot_variable)
+      named_slots[var_key] = new_slot_variable
+    return named_slots[var_key]
+
+  def _create_or_restore_slot_variable(
+      self, slot_variable_position, slot_name, variable,
+      optional_op_name=None):
+    """Restore a slot variable's value, possibly creating it.
+
+    Called when a variable which has an associated slot variable is created or
+    restored. When executing eagerly, we create the slot variable with a
+    restoring initializer.
+
+    No new variables are created when graph building. Instead,
+    _restore_slot_variable catches these after normal creation and adds restore
+    ops to the graph. This method is nonetheless important when graph building
+    for the case when a slot variable has already been created but `variable`
+    has just been added to a dependency graph (causing us to realize that the
+    slot variable needs to be restored).
+
+    Args:
+      slot_variable_position: A `checkpointable._CheckpointPosition` object
+        indicating the slot variable `Checkpointable` object to be restored.
+      slot_name: The name of this `Optimizer`'s slot to restore into.
+      variable: The variable object this slot is being created for.
+      optional_op_name: Name to use when scoping the Variable that
+        needs to be created for the slot.
+    """
+    slot_variable = self.get_slot(var=variable, name=slot_name)
+    if (slot_variable is None and context.executing_eagerly() and
+        slot_variable_position.is_simple_variable()):
+      initializer = checkpointable.CheckpointInitialValue(
+          checkpoint_position=slot_variable_position)
+      slot_variable = self.create_slot(
+          var=variable,
+          val=initializer,
+          slot_name=slot_name,
+          optional_op_name=optional_op_name)
+      # Optimizers do not have unconditional dependencies on their slot
+      # variables (nor do any other objects). They are only saved if the
+      # variables they were created for are also saved.
+    if slot_variable is not None:
+      # If we've either made this slot variable, or if we've pulled out an
+      # existing slot variable, we should restore it.
+      slot_variable_position.restore(slot_variable)
+    else:
+      # We didn't make the slot variable. Defer restoring until it gets created
+      # normally. We keep a list rather than the one with the highest restore
+      # UID in case slot variables have their own dependencies, in which case
+      # those could differ between restores.
+      variable_key = _var_key_v2(variable)
+      self._deferred_slot_restorations.setdefault(
+          slot_name, {}).setdefault(variable_key, []).append(
+              slot_variable_position)
+
+  def get_slot(self, var, name):
+    """Return a slot named `name` created for `var` by the Optimizer.
+
+    Some `Optimizer` subclasses use additional variables.  For example
+    `Momentum` and `Adagrad` use variables to accumulate updates.  This method
+    gives access to these `Variable` objects if for some reason you need them.
+
+    Use `get_slot_names()` to get the list of slot names created by the
+    `Optimizer`.
+
+    Args:
+      var: A variable passed to `minimize()` or `apply_gradients()`.
+      name: A string.
+
+    Returns:
+      The `Variable` for the slot if it was created, `None` otherwise.
+    """
+    named_slots = self._slots.get(name, None)
+    if not named_slots:
+      return None
+    return named_slots.get(_var_key_v2(var), None)
+
+  def get_slot_names(self):
+    """Return a list of the names of slots created by the `Optimizer`.
+
+    See `get_slot()`.
+
+    Returns:
+      A list of strings.
+    """
+    return sorted(self._slots.keys())
+
+  def create_non_slot(self, initial_value, name, colocate_with=None):
+    """Add an extra variable, not associated with a slot."""
+    v = self._non_slot_dict.get(name, None)
+    if v is None:
+      if colocate_with is None: colocate_with = self._non_slot_devices
+      with self._distribution.colocate_vars_with(colocate_with):
+        # TODO(josh11b): Use get_variable() except for the legacy Adam use case.
+        v = variable_scope.variable(initial_value, name=name, trainable=False)
+      self._non_slot_dict[name] = v
+      deferred_dependencies_list = self._deferred_dependencies.pop(name, ())
+      for checkpoint_position in sorted(
+          deferred_dependencies_list,
+          key=lambda restore: restore.checkpoint.restore_uid,
+          reverse=True):
+        checkpoint_position.restore(v)
+    return v
+
+  def _restore_slot_variable(self, slot_name, variable, slot_variable):
+    """Restore a newly created slot variable's value."""
+    variable_key = _var_key_v2(variable)
+    deferred_restorations = self._deferred_slot_restorations.get(
+        slot_name, {}).pop(variable_key, [])
+    # Iterate over restores, highest restore UID first to minimize the number
+    # of assignments.
+    deferred_restorations.sort(key=lambda position: position.restore_uid,
+                               reverse=True)
+    for checkpoint_position in deferred_restorations:
+      checkpoint_position.restore(slot_variable)
+
+  def get_non_slot(self, name):
+    """Returns the non-slot variable identified by `name`."""
+    return self._non_slot_dict.get(name, None)
+
+  def get_hyper(self, name, dtype=None):
+    """Returns the `name` hyper parameter, optionally cast to `dtype`."""
+    dtype_dict = self._hyper[name]
+    # Do we have the value cast to dtype already cached? This should always
+    # succeed when dtype is None.
+    if dtype in dtype_dict:
+      return dtype_dict[dtype]
+    # Not cached, cast to dtype and save the result in the cache.
+    result = math_ops.cast(dtype_dict[None], dtype)
+    dtype_dict[dtype] = result
+    return result
+
+
+class OptimizerV2(optimizer_v1.Optimizer):
+  """Updated base class for optimizers.
+
+  This class defines the API to add Ops to train a model.  You never use this
+  class directly, but instead instantiate one of its subclasses such as
+  `GradientDescentOptimizer`, `AdagradOptimizer`, or `MomentumOptimizer`.
+
+  ### Usage
+
+  ```python
+  # Create an optimizer with the desired parameters.
+  opt = GradientDescentOptimizer(learning_rate=0.1)
+  # Add Ops to the graph to minimize a cost by updating a list of variables.
+  # "cost" is a Tensor, and the list of variables contains tf.Variable
+  # objects.
+  opt_op = opt.minimize(cost, var_list=<list of variables>)
+  ```
+
+  In the training program you will just have to run the returned Op.
+
+  ```python
+  # Execute opt_op to do one step of training:
+  opt_op.run()
+  ```
+
+  ### Processing gradients before applying them.
+
+  Calling `minimize()` takes care of both computing the gradients and
+  applying them to the variables.  If you want to process the gradients
+  before applying them you can instead use the optimizer in three steps:
+
+  1.  Compute the gradients with `compute_gradients()`.
+  2.  Process the gradients as you wish.
+  3.  Apply the processed gradients with `apply_gradients()`.
+
+  Example:
+
+  ```python
+  # Create an optimizer.
+  opt = GradientDescentOptimizer(learning_rate=0.1)
+
+  # Compute the gradients for a list of variables.
+  grads_and_vars = opt.compute_gradients(loss, <list of variables>)
+
+  # grads_and_vars is a list of tuples (gradient, variable).  Do whatever you
+  # need to the 'gradient' part, for example cap them, etc.
+  capped_grads_and_vars = [(MyCapper(gv[0]), gv[1]) for gv in grads_and_vars]
+
+  # Ask the optimizer to apply the capped gradients.
+  opt.apply_gradients(capped_grads_and_vars)
+  ```
+
+  ### Gating Gradients
+
+  Both `minimize()` and `compute_gradients()` accept a `gate_gradients`
+  argument that controls the degree of parallelism during the application of
+  the gradients.
+
+  The possible values are: `GATE_NONE`, `GATE_OP`, and `GATE_GRAPH`.
+
+  <b>`GATE_NONE`</b>: Compute and apply gradients in parallel.  This provides
+  the maximum parallelism in execution, at the cost of some non-reproducibility
+  in the results.  For example the two gradients of `matmul` depend on the input
+  values: With `GATE_NONE` one of the gradients could be applied to one of the
+  inputs _before_ the other gradient is computed resulting in non-reproducible
+  results.
+
+  <b>`GATE_OP`</b>: For each Op, make sure all gradients are computed before
+  they are used.  This prevents race conditions for Ops that generate gradients
+  for multiple inputs where the gradients depend on the inputs.
+
+  <b>`GATE_GRAPH`</b>: Make sure all gradients for all variables are computed
+  before any one of them is used.  This provides the least parallelism but can
+  be useful if you want to process all gradients before applying any of them.
+
+  ### Slots
+
+  Some optimizer subclasses, such as `MomentumOptimizer` and `AdagradOptimizer`
+  allocate and manage additional variables associated with the variables to
+  train.  These are called <i>Slots</i>.  Slots have names and you can ask the
+  optimizer for the names of the slots that it uses.  Once you have a slot name
+  you can ask the optimizer for the variable it created to hold the slot value.
+
+  This can be useful if you want to log debug a training algorithm, report stats
+  about the slots, etc.
+
+  ### Non-slot variables
+
+  Some optimizer subclasses, such as `AdamOptimizer` have variables that
+  are not associated with the variables to train, just the step itself.
+
+  ### Hyper parameters
+
+  These are arguments passed to the optimizer subclass constructor
+  (the `__init__` method), and then passed to `self._set_hyper()`.
+  They can be either regular Python values (like 1.0), tensors, or
+  callables. If they are callable, the callable will be called during
+  `apply_gradients()` to get the value for the hyper parameter.
+
+  ### State
+
+  Internal methods apre passed a `state` argument with the correct
+  values to use for the slot and non-slot variables, and the hyper
+  parameters.
+  """
+
+  # Values for gate_gradients.
+  GATE_NONE = 0
+  GATE_OP = 1
+  GATE_GRAPH = 2
+
+  def __init__(self, use_locking, name):
+    """Create a new Optimizer.
+
+    This must be called by the constructors of subclasses.
+    Note that Optimizer instances should not bind to a single graph,
+    and so shouldn't keep Tensors as member variables. Generally
+    you should be able to use the _set_hyper()/state.get_hyper()
+    facility instead.
+
+    Args:
+      use_locking: Bool. If True apply use locks to prevent concurrent updates
+        to variables.
+      name: A non-empty string.  The name to use for accumulators created
+        for the optimizer.
+
+    Raises:
+      ValueError: If name is malformed.
+      RuntimeError: If _create_slots has been overridden instead of
+          _create_vars.
+    """
+    # Note: We intentionally don't call parent __init__.
+
+    # Optimizer._create_slots was replaced by _create_vars in OptimizerV2.
+    if (self.__class__._create_slots.__code__ is not  # pylint: disable=protected-access
+        OptimizerV2._create_slots.__code__):
+      raise RuntimeError("Override _create_vars instead of _create_slots when "
+                         "descending from OptimizerV2 (class %s)" %
+                         self.__class__.__name__)
+    if not name:
+      raise ValueError("Must specify the optimizer name")
+
+    self._use_locking = use_locking
+    self._name = name
+    # Map from graph_key to state for that graph. We use the graph_key
+    # since it works in both eager and graph mode, and gives the outer
+    # graph inside functions.
+    tower_context = distribute_lib.get_tower_context()
+    if tower_context is None:
+      # In a cross-tower context for a DistributionStrategy, which means
+      # only one Optimizer will be created, not one per tower.
+      self._per_graph_state = {}
+    else:
+      # We use get_tower_context().merge_call() to get a single dict
+      # shared across all model replicas when running with a
+      # DistributionStrategy.
+      self._per_graph_state = tower_context.merge_call(lambda _: {})
+
+    # Hyper parameters, and whether they should be re-evaluated every step.
+    self._hyper = {}
+
+  def _set_hyper(self, name, value):
+    self._hyper[name] = (_is_dynamic(value), value)
+
+  def minimize(self, loss, global_step=None, var_list=None,
+               gate_gradients=GATE_OP, aggregation_method=None,
+               colocate_gradients_with_ops=False, name=None,
+               grad_loss=None, stop_gradients=None,
+               scale_loss_by_num_towers=None):
+    """Add operations to minimize `loss` by updating `var_list`.
+
+    This method simply combines calls `compute_gradients()` and
+    `apply_gradients()`. If you want to process the gradient before applying
+    them call `compute_gradients()` and `apply_gradients()` explicitly instead
+    of using this function.
+
+    Args:
+      loss: A `Tensor` containing the value to minimize.
+      global_step: Optional `Variable` to increment by one after the
+        variables have been updated.
+      var_list: Optional list or tuple of `Variable` objects to update to
+        minimize `loss`.  Defaults to the list of variables collected in
+        the graph under the key `GraphKeys.TRAINABLE_VARIABLES`.
+      gate_gradients: How to gate the computation of gradients.  Can be
+        `GATE_NONE`, `GATE_OP`, or  `GATE_GRAPH`.
+      aggregation_method: Specifies the method used to combine gradient terms.
+        Valid values are defined in the class `AggregationMethod`.
+      colocate_gradients_with_ops: If True, try colocating gradients with
+        the corresponding op.
+      name: Optional name for the returned operation.
+      grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
+      stop_gradients: Optional. A Tensor or list of tensors not to differentiate
+        through.
+      scale_loss_by_num_towers: Optional boolean. If true, scale the loss
+        down by the number of towers. By default, auto-detects whether this
+        is needed.
+
+    Returns:
+      An Operation that updates the variables in `var_list`.  If `global_step`
+      was not `None`, that operation also increments `global_step`.
+
+    Raises:
+      ValueError: If some of the variables are not `Variable` objects.
+
+    @compatibility(eager)
+    When eager execution is enabled, `loss` should be a Python function that
+    takes elements of `var_list` as arguments and computes the value to be
+    minimized. If `var_list` is None, `loss` should take no arguments.
+    Minimization (and gradient computation) is done with respect to the
+    elements of `var_list` if not None, else with respect to any trainable
+    variables created during the execution of the `loss` function.
+    `gate_gradients`, `aggregation_method`, `colocate_gradients_with_ops` and
+    `grad_loss` are ignored when eager execution is enabled.
+    @end_compatibility
+    """
+    grads_and_vars = self.compute_gradients(
+        loss, var_list=var_list, gate_gradients=gate_gradients,
+        aggregation_method=aggregation_method,
+        colocate_gradients_with_ops=colocate_gradients_with_ops,
+        grad_loss=grad_loss, stop_gradients=stop_gradients,
+        scale_loss_by_num_towers=scale_loss_by_num_towers)
+
+    vars_with_grad = [v for g, v in grads_and_vars if g is not None]
+    if not vars_with_grad:
+      raise ValueError(
+          "No gradients provided for any variable, check your graph for ops"
+          " that do not support gradients, between variables %s and loss %s." %
+          ([str(v) for _, v in grads_and_vars], loss))
+
+    return self.apply_gradients(grads_and_vars, global_step=global_step,
+                                name=name)
+
+  def compute_gradients(self, loss, var_list=None,
+                        gate_gradients=GATE_OP,
+                        aggregation_method=None,
+                        colocate_gradients_with_ops=False,
+                        grad_loss=None, stop_gradients=None,
+                        scale_loss_by_num_towers=None):
+    """Compute gradients of `loss` for the variables in `var_list`.
+
+    This is the first part of `minimize()`.  It returns a list
+    of (gradient, variable) pairs where "gradient" is the gradient
+    for "variable".  Note that "gradient" can be a `Tensor`, an
+    `IndexedSlices`, or `None` if there is no gradient for the
+    given variable.
+
+    Args:
+      loss: A Tensor containing the value to minimize or a callable taking
+        no arguments which returns the value to minimize. When eager execution
+        is enabled it must be a callable.
+      var_list: Optional list or tuple of `tf.Variable` to update to minimize
+        `loss`.  Defaults to the list of variables collected in the graph
+        under the key `GraphKeys.TRAINABLE_VARIABLES`.
+      gate_gradients: How to gate the computation of gradients.  Can be
+        `GATE_NONE`, `GATE_OP`, or `GATE_GRAPH`.
+      aggregation_method: Specifies the method used to combine gradient terms.
+        Valid values are defined in the class `AggregationMethod`.
+      colocate_gradients_with_ops: If True, try colocating gradients with
+        the corresponding op.
+      grad_loss: Optional. A `Tensor` holding the gradient computed for `loss`.
+      stop_gradients: Optional. A Tensor or list of tensors not to differentiate
+        through.
+      scale_loss_by_num_towers: Optional boolean. If true, scale the loss
+        down by the number of towers. By default, auto-detects whether this
+        is needed.
+
+    Returns:
+      A list of (gradient, variable) pairs. Variable is always present, but
+      gradient can be `None`.
+
+    Raises:
+      TypeError: If `var_list` contains anything else than `Variable` objects.
+      ValueError: If some arguments are invalid.
+      RuntimeError: If called with eager execution enabled and `loss` is
+        not callable.
+
+    @compatibility(eager)
+    When eager execution is enabled, `gate_gradients`, `aggregation_method`,
+    and `colocate_gradients_with_ops` are ignored.
+    @end_compatibility
+    """
+    # TODO(josh11b): Test that we handle weight decay in a reasonable way.
+    if callable(loss):
+      with backprop.GradientTape() as tape:
+        if var_list is not None:
+          tape.watch(var_list)
+        loss_value = loss()
+
+        # Scale loss for number of towers (callable-loss case). In this case,
+        # we have to be careful to call distribute_lib.get_loss_reduction()
+        # *after* loss() is evaluated, so we know what loss reduction it uses.
+        if scale_loss_by_num_towers is None:
+          scale_loss_by_num_towers = (
+              distribute_lib.get_loss_reduction() == "mean")
+        if scale_loss_by_num_towers:
+          num_towers = distribute_lib.get_distribution_strategy().num_towers
+          if num_towers > 1:
+            loss_value *= 1. / num_towers
+
+      if var_list is None:
+        var_list = tape.watched_variables()
+      grads = tape.gradient(loss_value, var_list, grad_loss)
+      return list(zip(grads, var_list))
+    if context.executing_eagerly():
+      raise RuntimeError(
+          "`loss` passed to Optimizer.compute_gradients should "
+          "be a function when eager execution is enabled.")
+
+    # Scale loss for number of towers (non-callable-loss case).
+    if scale_loss_by_num_towers is None:
+      scale_loss_by_num_towers = (
+          distribute_lib.get_loss_reduction() == "mean")
+    if scale_loss_by_num_towers:
+      num_towers = distribute_lib.get_distribution_strategy().num_towers
+      if num_towers > 1:
+        loss *= 1. / num_towers
+
+    if gate_gradients not in [optimizer_v1.Optimizer.GATE_NONE,
+                              optimizer_v1.Optimizer.GATE_OP,
+                              optimizer_v1.Optimizer.GATE_GRAPH]:
+      raise ValueError("gate_gradients must be one of: Optimizer.GATE_NONE, "
+                       "Optimizer.GATE_OP, Optimizer.GATE_GRAPH.  Not %s" %
+                       gate_gradients)
+    self._assert_valid_dtypes([loss])
+    if grad_loss is not None:
+      self._assert_valid_dtypes([grad_loss])
+    if var_list is None:
+      var_list = (
+          variables.trainable_variables() +
+          ops.get_collection(ops.GraphKeys.TRAINABLE_RESOURCE_VARIABLES))
+    else:
+      var_list = nest.flatten(var_list)
+    # pylint: disable=protected-access
+    var_list += ops.get_collection(ops.GraphKeys._STREAMING_MODEL_PORTS)
+    # pylint: enable=protected-access
+    processors = [_get_processor(v) for v in var_list]
+    if not var_list:
+      raise ValueError("No variables to optimize.")
+    var_refs = [p.target() for p in processors]
+    grads = gradients.gradients(
+        loss, var_refs, grad_ys=grad_loss,
+        gate_gradients=(gate_gradients == optimizer_v1.Optimizer.GATE_OP),
+        aggregation_method=aggregation_method,
+        colocate_gradients_with_ops=colocate_gradients_with_ops,
+        stop_gradients=stop_gradients)
+    if gate_gradients == optimizer_v1.Optimizer.GATE_GRAPH:
+      grads = control_flow_ops.tuple(grads)
+    grads_and_vars = list(zip(grads, var_list))
+    self._assert_valid_dtypes(
+        [v for g, v in grads_and_vars
+         if g is not None and v.dtype != dtypes.resource])
+    return grads_and_vars
+
+  def apply_gradients(self, grads_and_vars, global_step=None, name=None):
+    """Apply gradients to variables.
+
+    This is the second part of `minimize()`. It returns an `Operation` that
+    applies gradients.
+
+    Args:
+      grads_and_vars: List of (gradient, variable) pairs as returned by
+        `compute_gradients()`.
+      global_step: Optional `Variable` to increment by one after the
+        variables have been updated.
+      name: Optional name for the returned operation.  Default to the
+        name passed to the `Optimizer` constructor.
+
+    Returns:
+      An `Operation` that applies the specified gradients. If `global_step`
+      was not None, that operation also increments `global_step`.
+
+    Raises:
+      TypeError: If `grads_and_vars` is malformed.
+      ValueError: If none of the variables have gradients.
+    """
+    # This is a default implementation of apply_gradients() that can be shared
+    # by most optimizers.  It relies on the subclass implementing the following
+    # methods: _create_vars(), _prepare(), _apply_dense(), and _apply_sparse().
+
+    # Filter out variables with gradients of `None`.
+    grads_and_vars = tuple(grads_and_vars)  # Make sure repeat iteration works.
+    if not grads_and_vars:
+      raise ValueError("No variables provided.")
+    filtered = tuple((g, v) for (g, v) in grads_and_vars if g is not None)
+    if not filtered:
+      raise ValueError("No gradients provided for any variable: %s." %
+                       ([str(v) for _, v in grads_and_vars],))
+    return distribute_lib.get_tower_context().merge_call(
+        self.distributed_apply, filtered, global_step=global_step, name=name)
+
+  def _get_or_create_state(self, var_list=None):
+    """Either looks up or creates `_OptimizerV2State`.
+
+    If any variables are available, they should be passed via the `var_list`
+    argument, and these will be used to determine the graph to create/retrieve
+    state for. Otherwise the returned state is for the current default graph.
+
+    Args:
+      var_list: A list of variables to extract a graph from.
+
+    Returns:
+      An `_OptimizerV2State` object.
+    """
+    # Determine the graph_key from the current graph.
+    eager_execution = context.executing_eagerly()
+    if eager_execution or var_list is None:
+      graph = ops.get_default_graph()
+    else:
+      graph = ops._get_graph_from_inputs(var_list)  # pylint: disable=protected-access
+    assert graph is not None
+    graph_key = graph._graph_key  # pylint: disable=protected-access
+
+    # Get the per graph state by looking up the graph_key.
+    if graph_key in self._per_graph_state:
+      per_graph_state = self._per_graph_state[graph_key]
+    else:
+      per_graph_state = _OptimizerV2State(self._name)
+      per_graph_state._init_with_static_hyper(self._hyper)  # pylint: disable=protected-access
+      self._per_graph_state[graph_key] = per_graph_state
+    return per_graph_state
+
+  def distributed_apply(self, distribution, grads_and_vars, global_step, name):
+    """`apply_gradients` for use with a `DistributionStrategy`."""
+    reduced_grads = distribution.batch_reduce("sum", grads_and_vars)
+    var_list = [v for _, v in grads_and_vars]
+    grads_and_vars = zip(reduced_grads, var_list)
+
+    unwrapped_var_list = [x for v in var_list for x in distribution.unwrap(v)]
+    eager_execution = context.executing_eagerly()
+    if eager_execution:
+      # Give a clear error in this case instead of "name not supported
+      # for Eager Tensors" when we compute non_slot_devices.
+      for v in unwrapped_var_list:
+        if isinstance(v, ops.Tensor):
+          raise NotImplementedError("Trying to update a Tensor ", v)
+
+    with ops.name_scope(name, self._name) as name:
+      per_graph_state = self._get_or_create_state(var_list=unwrapped_var_list)
+      # Include the current value of any dynamic hyper parameters in `state`.
+      non_slot_devices = distribution.non_slot_devices(var_list)
+      state = per_graph_state._copy_with_dynamic_hyper(  # pylint: disable=protected-access
+          self._hyper, distribution, non_slot_devices)
+
+    # Create any slot and non-slot variables we need in `state`.
+    with ops.init_scope():
+      self._create_vars(var_list, state)
+
+    with ops.name_scope(name):  # Re-enter name_scope created above
+      # Give the child class a chance to do something before we start
+      # applying gradients.
+      self._prepare(state)
+
+      def update(v, g):
+        """Update variable `v` using gradient `g`."""
+        assert v is not None
+
+        # Convert the grad to Tensor or IndexedSlices if necessary, and
+        # look up a processor for each variable's type.
+        try:
+          g = ops.convert_to_tensor_or_indexed_slices(g)
+        except TypeError:
+          raise TypeError(
+              "Gradient must be convertible to a Tensor"
+              " or IndexedSlices, or None: %s" % g)
+        if not isinstance(g, (ops.Tensor, ops.IndexedSlices)):
+          raise TypeError(
+              "Gradient must be a Tensor, IndexedSlices, or None: %s" % g)
+        processor = _get_processor(v)
+
+        # We colocate all ops created in _apply_dense or _apply_sparse
+        # on the same device as the variable.
+        # TODO(apassos): figure out how to get the variable name here.
+        scope_name = "" if eager_execution else v.op.name
+        # device_policy is set because non-mirrored tensors will be read in
+        # `update_op`.
+        # TODO(josh11b): Make different state objects for each device to
+        # avoid needing to set the device_policy.
+        with ops.name_scope("update_" + scope_name), \
+            context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+          return processor.update_op(self, g, state)
+
+      # Use the processors to update the variables.
+      update_ops = []
+      for grad, var in grads_and_vars:
+        update_ops.extend(distribution.unwrap(distribution.update(
+            var, update, grad)))
+
+      # Give the child class a chance to do something after applying
+      # gradients
+      def finish():
+        # TODO(josh11b): Make different state objects for each device to
+        # avoid needing to set the device_policy.
+        with context.context().device_policy(context.DEVICE_PLACEMENT_SILENT):
+          return self._finish(state)
+
+      update_ops = control_flow_ops.group(update_ops)
+      with ops.control_dependencies([update_ops]):
+        finish_updates = distribution.update_non_slot(non_slot_devices, finish)
+      if finish_updates is None:
+        finish_updates = update_ops
+
+      # Update `global_step` (if any).
+      if global_step is None:
+        apply_updates = distribution.group(finish_updates, name=name)
+      else:
+        with ops.control_dependencies(distribution.unwrap(finish_updates)):
+
+          def update_global_step(global_step):
+            if isinstance(global_step, resource_variable_ops.ResourceVariable):
+              return global_step.assign_add(
+                  ops.convert_to_tensor(1, dtype=global_step.dtype),
+                  read_value=False)
+            else:
+              return state_ops.assign_add(global_step, 1)
+
+          apply_updates = distribution.group(
+              distribution.update(global_step, update_global_step), name=name)
+
+      # Add the training op to the TRAIN_OP graph collection in graph mode.
+      if not eager_execution:
+        if isinstance(apply_updates, ops.Tensor):
+          apply_updates = apply_updates.op
+        train_op = ops.get_collection_ref(ops.GraphKeys.TRAIN_OP)
+        if apply_updates not in train_op:
+          train_op.append(apply_updates)
+
+      return apply_updates
+
+  def get_slot(self, var, name):
+    """Return a slot named `name` created for `var` by the Optimizer.
+
+    Some `Optimizer` subclasses use additional variables.  For example
+    `Momentum` and `Adagrad` use variables to accumulate updates.  This method
+    gives access to these `Variable` objects if for some reason you need them.
+
+    Use `get_slot_names()` to get the list of slot names created by the
+    `Optimizer`.
+
+    Args:
+      var: A variable passed to `minimize()` or `apply_gradients()`.
+      name: A string.
+
+    Returns:
+      The `Variable` for the slot if it was created, `None` otherwise.
+    """
+    state = self._get_state_for_var(var)
+    return state.get_slot(var, name) if state is not None else None
+
+  def get_slot_names(self):
+    """Return a list of the names of slots created by the `Optimizer`.
+
+    See `get_slot()`.
+
+    Returns:
+      A list of strings.
+    """
+    state = self._get_per_graph_state()
+    return state.get_slot_names() if state is not None else []
+
+  def variables(self):
+    """A list of variables which encode the current state of `Optimizer`.
+
+    Includes slot variables and additional global variables created by the
+    optimizer in the current default graph.
+
+    Returns:
+      A list of variables.
+    """
+    state = self._get_per_graph_state()
+    return state._variables() if state is not None else []  # pylint: disable=protected-access
+
+  # --------------
+  # Methods to be implemented by subclasses if they want to use the
+  # inherited implementation of apply_gradients() or compute_gradients().
+  # --------------
+  def _create_vars(self, var_list, state):
+    """Create all slots needed by the variables and any non-slot variables.
+
+    Args:
+      var_list: A list of `Variable` objects.
+      state: An object with these methods:
+        `create_slot(var, val, slot_name, optional_op_name)`,
+        `create_slot_with_initializer(`
+            `var, initializer, shape, dtype, slot_name, optional_op_name)`,
+        `zeros_slot(var, slot_name, optional_op_name)`,
+        `create_non_slot_variable(initial_value, name, colocate_with)`,
+        `get_hyper(name)`
+    """
+    # No slots needed by default
+    pass
+
+  def _prepare(self, state):
+    """Code to execute before applying gradients.
+
+    Note that most uses of _prepare() in Optimizer have been subsumed
+    by explicit support for hyper parameters in OptimizerV2
+
+    Args:
+      state: An object with a `get_hyper(name)` method.
+
+    Returns:
+      Return value will be ignored.
+    """
+    pass
+
+  def _apply_dense(self, grad, var, state):
+    """Add ops to apply dense gradients to `var`.
+
+    Args:
+      grad: A `Tensor`.
+      var: A `Variable` object.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation`.
+    """
+    raise NotImplementedError()
+
+  def _resource_apply_dense(self, grad, handle, state):
+    """Add ops to apply dense gradients to the variable `handle`.
+
+    Args:
+      grad: a `Tensor` representing the gradient.
+      handle: a `Tensor` of dtype `resource` which points to the variable
+       to be updated.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation` which updates the value of the variable.
+    """
+    raise NotImplementedError()
+
+  def _resource_apply_sparse_duplicate_indices(
+      self, grad, handle, indices, state):
+    """Add ops to apply sparse gradients to `handle`, with repeated indices.
+
+    Optimizers which override this method must deal with repeated indices. See
+    the docstring of `_apply_sparse_duplicate_indices` for details. By default
+    the correct behavior, to sum non-unique indices and their associated
+    gradients, is enforced by first pre-processing `grad` and `indices` and
+    passing them on to `_resource_apply_sparse`. Optimizers which deal correctly
+    with duplicate indices may instead override this method to avoid the
+    overhead of summing.
+
+    Args:
+      grad: a `Tensor` representing the gradient for the affected indices.
+      handle: a `Tensor` of dtype `resource` which points to the variable
+       to be updated.
+      indices: a `Tensor` of integral type representing the indices for
+       which the gradient is nonzero. Indices may be repeated.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation` which updates the value of the variable.
+    """
+    # pylint: disable=protected-access
+    summed_grad, unique_indices = optimizer_v1._deduplicate_indexed_slices(
+        values=grad, indices=indices)
+    # pylint: enable=protected-access
+    return self._resource_apply_sparse(
+        summed_grad, handle, unique_indices, state)
+
+  def _resource_apply_sparse(self, grad, handle, indices, state):
+    """Add ops to apply sparse gradients to the variable `handle`.
+
+    Similar to `_apply_sparse`, the `indices` argument to this method has been
+    de-duplicated. Optimizers which deal correctly with non-unique indices may
+    instead override `_resource_apply_sparse_duplicate_indices` to avoid this
+    overhead.
+
+    Args:
+      grad: a `Tensor` representing the gradient for the affected indices.
+      handle: a `Tensor` of dtype `resource` which points to the variable
+       to be updated.
+      indices: a `Tensor` of integral type representing the indices for
+       which the gradient is nonzero. Indices are unique.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation` which updates the value of the variable.
+    """
+    raise NotImplementedError()
+
+  def _apply_sparse_duplicate_indices(self, grad, var, state):
+    """Add ops to apply sparse gradients to `var`, with repeated sparse indices.
+
+    Optimizers which override this method must deal with IndexedSlices objects
+    such as the following:
+
+      IndexedSlicesValue(values=[1, 1], indices=[0, 0], dense_shape=[1])
+
+    The correct interpretation is:
+
+      IndexedSlicesValue(values=[2], indices=[0], dense_shape=[1])
+
+    Many optimizers deal incorrectly with repeated indices when updating based
+    on sparse gradients (e.g. summing squares rather than squaring the sum, or
+    applying momentum terms multiple times). Adding first is always the correct
+    behavior, so this is enforced here by reconstructing the IndexedSlices to
+    have only unique indices, then calling _apply_sparse.
+
+    Optimizers which deal correctly with repeated indices may instead override
+    this method to avoid the overhead of summing indices.
+
+    Args:
+      grad: `IndexedSlices`.
+      var: A `Variable` object.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation`.
+    """
+    # pylint: disable=protected-access
+    summed_values, unique_indices = optimizer_v1._deduplicate_indexed_slices(
+        values=grad.values, indices=grad.indices)
+    # pylint: enable=protected-access
+    gradient_no_duplicate_indices = ops.IndexedSlices(
+        indices=unique_indices,
+        values=summed_values,
+        dense_shape=grad.dense_shape)
+    return self._apply_sparse(gradient_no_duplicate_indices, var, state)
+
+  def _apply_sparse(self, grad, var, state):
+    """Add ops to apply sparse gradients to `var`.
+
+    The IndexedSlices object passed to `grad` in this function is by default
+    pre-processed in `_apply_sparse_duplicate_indices` to remove duplicate
+    indices (see its docstring for details). Optimizers which can tolerate or
+    have correct special cases for duplicate sparse indices may override
+    `_apply_sparse_duplicate_indices` instead of this function, avoiding that
+    overhead.
+
+    Args:
+      grad: `IndexedSlices`, with no repeated indices.
+      var: A `Variable` object.
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      An `Operation`.
+    """
+    raise NotImplementedError()
+
+  def _finish(self, state):
+    """Do what is needed to finish the update.
+
+    This is called inside a scope colocated with any non-slot variables.
+
+    Args:
+      state: An object with `get_slot(var, name)`, `get_non_slot(self, name)`,
+        and `get_hyper(name)` methods.
+
+    Returns:
+      The operation to apply updates, or None if no updates.
+    """
+    return None
+
+  # --------------
+  # Utility methods for subclasses.
+  # --------------
+  def _get_per_graph_state(self):
+    # pylint: disable=protected-access
+    return self._per_graph_state.get(ops.get_default_graph()._graph_key, None)
+
+  def _get_state_for_var(self, var):
+    # pylint: disable=protected-access
+    return self._per_graph_state.get(var._graph_key, None)
+
+  # --------------
+  # Overridden methods from Checkpointable.
+  # --------------
+
+  def _track_checkpointable(self, *args, **kwargs):
+    """Optimizers may not track dependencies. Raises an error."""
+    raise NotImplementedError(
+        "Optimizers may not have dependencies. File a feature request if this "
+        "limitation bothers you.")
+
+  @property
+  def _checkpoint_dependencies(self):
+    """From Checkpointable. Gather graph-specific non-slot variables to save."""
+    current_graph_non_slot_variables = []
+    state = self._get_per_graph_state()
+    if state is not None:
+      for name, variable_object in sorted(
+          state._non_slot_dict.items(),  # pylint: disable=protected-access
+          # Avoid comparing variables
+          key=lambda item: item[0]):
+        current_graph_non_slot_variables.append(
+            checkpointable.CheckpointableReference(
+                name=name, ref=variable_object))
+    # Note: ignores super(); Optimizers may not have any dependencies outside of
+    # state objects.
+    return current_graph_non_slot_variables
+
+  def _lookup_dependency(self, name):
+    """From Checkpointable. Find a non-slot variable in the current graph."""
+    state = self._get_per_graph_state()
+    if state is None:
+      return None
+    else:
+      return state.get_non_slot(name)
+
+  @property
+  def _deferred_dependencies(self):
+    """Lets Checkpointable know where non-slot variables are created.
+
+    If necessary, creates a new state object for the current default graph.
+    Checkpointable will then add entries to that state's deferred dependency
+    dictionary. The state object will check that dictionary when creating
+    non-slot variables, restoring their value if an entry is found.
+
+    Returns:
+      A dictionary which holds deferred dependencies for the current default
+      graph.
+    """
+    state = self._get_or_create_state()
+    return state._deferred_dependencies  # pylint: disable=protected-access
+
+  def _create_or_restore_slot_variable(
+      self, slot_variable_position, slot_name, variable):
+    """Checkpointable: Restore a slot variable's value, possibly creating it.
+
+    Called when a variable which has an associated slot variable is created or
+    restored.
+
+    Args:
+      slot_variable_position: A `checkpointable._CheckpointPosition` object
+        indicating the slot variable `Checkpointable` object to be restored.
+      slot_name: The name of this `Optimizer`'s slot to restore into.
+      variable: The variable object this slot is being created for.
+    """
+    state = self._get_or_create_state(var_list=[variable])
+    state._create_or_restore_slot_variable(  # pylint: disable=protected-access
+        slot_variable_position=slot_variable_position,
+        slot_name=slot_name,
+        variable=variable,
+        optional_op_name=self._name)
+
+  # --------------
+  # Unsupported parent methods
+  # --------------
+  def _slot_dict(self, slot_name):
+    raise NotImplementedError(
+        "_slot_dict() method unsupported in OptimizerV2")
+
+  def _get_or_make_slot(self, var, val, slot_name, op_name):
+    raise NotImplementedError(
+        "_get_or_make_slot() method unsupported in OptimizerV2")
+
+  def _get_or_make_slot_with_initializer(self, var, initializer, shape, dtype,
+                                         slot_name, op_name):
+    raise NotImplementedError(
+        "_get_or_make_slot_with_initializer() method unsupported in "
+        "OptimizerV2")
+
+  def _create_non_slot_variable(self, initial_value, name, colocate_with):
+    raise NotImplementedError(
+        "_create_non_slot_variable() method unsupported in OptimizerV2")
+
+  def _get_non_slot_variable(self, name, graph=None):
+    raise NotImplementedError(
+        "_get_non_slot_variable() method unsupported in OptimizerV2")
+
+  def _non_slot_variables(self):
+    raise NotImplementedError(
+        "_non_slot_variables() method unsupported in OptimizerV2")
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py b/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py
new file mode 100644
index 0000000000..24eada06cc
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2_symbols.py
@@ -0,0 +1,42 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Distribution-aware version of Optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=unused-import
+from tensorflow.contrib.optimizer_v2.adadelta import AdadeltaOptimizer
+from tensorflow.contrib.optimizer_v2.adagrad import AdagradOptimizer
+from tensorflow.contrib.optimizer_v2.adam import AdamOptimizer
+from tensorflow.contrib.optimizer_v2.gradient_descent import GradientDescentOptimizer
+from tensorflow.contrib.optimizer_v2.momentum import MomentumOptimizer
+from tensorflow.contrib.optimizer_v2.optimizer_v2 import OptimizerV2
+from tensorflow.contrib.optimizer_v2.rmsprop import RMSPropOptimizer
+
+from tensorflow.python.util.all_util import remove_undocumented
+
+_allowed_symbols = [
+    'AdadeltaOptimizer',
+    'AdagradOptimizer',
+    'AdamOptimizer',
+    'GradientDescentOptimizer',
+    'MomentumOptimizer',
+    'OptimizerV2',
+    'RMSPropOptimizer',
+]
+
+remove_undocumented(__name__, _allowed_symbols)
diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py b/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
new file mode 100644
index 0000000000..8599af32f6
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2_test.py
@@ -0,0 +1,294 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Functional test for OptimizerV2."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import gradient_descent
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import clip_ops
+from tensorflow.python.ops import gradients_impl
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+
+class OptimizerTest(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testBasic(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      def loss():
+        return 5 * var0 + 3 * var1  # pylint: disable=cell-var-from-loop
+      # Note that for eager execution, minimize expects a function instead of a
+      # Tensor.
+      global_step = resource_variable_ops.ResourceVariable(
+          array_ops.zeros([], dtypes.int64), name='global_step_%d' % i)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+
+      self.evaluate(variables.global_variables_initializer())
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+      # Run 1 step of sgd through optimizer
+      opt_op = sgd_op.minimize(loss, global_step, [var0, var1])
+      self.evaluate(opt_op)
+      # Validate updated params
+      self.assertAllClose([-14., -13.], self.evaluate(var0))
+      self.assertAllClose([-6., -5.], self.evaluate(var1))
+
+  def testAggregationMethod(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        cost = 5 * var0 + 3 * var1
+        global_step = variables.Variable(
+            array_ops.zeros([], dtypes.int64), name='global_step')
+        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+        opt_op = sgd_op.minimize(
+            cost,
+            global_step, [var0, var1],
+            aggregation_method=gradients_impl.AggregationMethod.
+            EXPERIMENTAL_ACCUMULATE_N)
+
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd through optimizer
+        opt_op.run()
+        # Validate updated params
+        self.assertAllClose([-14., -13.], var0.eval())
+        self.assertAllClose([-6., -5.], var1.eval())
+
+  def testPrecomputedGradient(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        cost = 5 * var0 + 3 * var1
+        grad_loss = constant_op.constant([42, -42], dtype=dtype)
+        global_step = variables.Variable(
+            array_ops.zeros([], dtypes.int64), name='global_step')
+        sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+        opt_op = sgd_op.minimize(
+            cost, global_step, [var0, var1], grad_loss=grad_loss)
+
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Run 1 step of sgd through optimizer
+        opt_op.run()
+        # Validate updated params
+        self.assertAllClose([1.0 - 3 * 5 * 42.0, 2.0 - 3 * 5 * (-42.0)],
+                            var0.eval())
+        self.assertAllClose([3.0 - 3 * 3 * 42.0, 4.0 - 3 * 3 * (-42.0)],
+                            var1.eval())
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNoVariables(self):
+    for dtype in [dtypes.half, dtypes.float32, dtypes.float64]:
+      # pylint: disable=cell-var-from-loop
+      def loss():
+        var0 = resource_variable_ops.ResourceVariable(
+            [1.0, 2.0], dtype=dtype, trainable=False, name='a')
+        var1 = resource_variable_ops.ResourceVariable(
+            [3.0, 4.0], dtype=dtype, trainable=False, name='b')
+        return 5 * var0 + var1
+      # pylint: enable=cell-var-from-loop
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError, 'No.*variables'):
+        sgd_op.minimize(loss)
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNoGradients(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b%d' % i)
+      # pylint: disable=cell-var-from-loop
+      def loss():
+        return 5 * var0
+      # pylint: enable=cell-var-from-loop
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError, 'No gradients'):
+        # var1 has no gradient
+        sgd_op.minimize(loss, var_list=[var1])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNoGradientsForAnyVariables_Minimize(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      def loss():
+        return constant_op.constant(5.0)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'No gradients provided for any variable'):
+        sgd_op.minimize(loss, var_list=[var0, var1])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testNoGradientsForAnyVariables_ApplyGradients(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a_%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b_%d' % i)
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'No gradients provided for any variable'):
+        sgd_op.apply_gradients([(None, var0), (None, var1)])
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testGradientsAsVariables(self):
+    for i, dtype in enumerate([dtypes.half, dtypes.float32, dtypes.float64]):
+      # Note that we name the variables uniquely here since the variables don't
+      # seem to be getting deleted at the end of the loop.
+      var0 = resource_variable_ops.ResourceVariable([1.0, 2.0], dtype=dtype,
+                                                    name='a%d' % i)
+      var1 = resource_variable_ops.ResourceVariable([3.0, 4.0], dtype=dtype,
+                                                    name='b%d' % i)
+      def loss():
+        return 5 * var0 + 3 * var1  # pylint: disable=cell-var-from-loop
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      grads_and_vars = sgd_op.compute_gradients(loss, [var0, var1])
+      # Convert gradients to tf.Variables
+      converted_grads = [
+          resource_variable_ops.ResourceVariable(array_ops.zeros([2], dtype),
+                                                 name='c_%d_%d' % (i, j))
+          for j, gv in enumerate(grads_and_vars)
+      ]
+      convert_ops = [
+          state_ops.assign(converted_grads[j], gv[0])
+          for j, gv in enumerate(grads_and_vars)
+      ]
+
+      self.evaluate(variables.global_variables_initializer())
+      # Run convert_ops to achieve the gradietns converting
+      self.evaluate(convert_ops)
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], self.evaluate(var0))
+      self.assertAllClose([3.0, 4.0], self.evaluate(var1))
+
+      # Run 1 step of sgd through optimizer
+      converted_grads_and_vars = list(zip(converted_grads, [var0, var1]))
+      opt_op = sgd_op.apply_gradients(converted_grads_and_vars)
+      self.evaluate(opt_op)
+
+      # Validate updated params
+      self.assertAllClose([-14., -13.], self.evaluate(var0))
+      self.assertAllClose([-6., -5.], self.evaluate(var1))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testComputeGradientsWithTensors(self):
+    x = ops.convert_to_tensor(1.0)
+    def f():
+      return x * x
+    sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+    grads_and_vars = sgd_op.compute_gradients(f, [x])
+    self.assertEqual(1, len(grads_and_vars))
+    grad, x_as_var = grads_and_vars[0]
+    self.assertIs(x, x_as_var)
+    self.assertEqual(2.0, self.evaluate(grad))
+
+    with self.assertRaises(NotImplementedError):
+      sgd_op.apply_gradients(grads_and_vars)
+
+  def testTrainOp(self):
+    with self.test_session():
+      var0 = variables.Variable([1.0, 2.0])
+      var1 = variables.Variable([3.0, 4.0])
+      cost = 5 * var0 + 3 * var1
+      global_step = variables.Variable(
+          array_ops.zeros([], dtypes.int64), name='global_step')
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      opt_op = sgd_op.minimize(cost, global_step, [var0, var1])
+      self.assertTrue(opt_op in ops.get_collection(ops.GraphKeys.TRAIN_OP))
+
+  def testConstraint(self):
+    constraint_01 = lambda x: clip_ops.clip_by_value(x, -0.1, 0.)
+    constraint_0 = lambda x: clip_ops.clip_by_value(x, 0., 1.)
+    with self.test_session():
+      var0 = variables.Variable([1.0, 2.0],
+                                constraint=constraint_01)
+      var1 = variables.Variable([3.0, 4.0],
+                                constraint=constraint_0)
+      cost = 5 * var0 + 3 * var1
+      global_step = variables.Variable(
+          array_ops.zeros([], dtypes.int64), name='global_step')
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      opt_op = sgd_op.minimize(cost, global_step, [var0, var1])
+
+      variables.global_variables_initializer().run()
+      # Fetch params to validate initial values
+      self.assertAllClose([1.0, 2.0], var0.eval())
+      self.assertAllClose([3.0, 4.0], var1.eval())
+      # Run 1 step of sgd through optimizer
+      opt_op.run()
+      # Validate updated params
+      self.assertAllClose([-0.1, -0.1], var0.eval())
+      self.assertAllClose([0., 0.], var1.eval())
+
+  def testStopGradients(self):
+    with self.test_session():
+      var0 = variables.Variable([1.0, 2.0], name='var0')
+      var1 = variables.Variable([3.0, 4.0], name='var1')
+      var0_id = array_ops.identity(var0)
+      cost = 5 * var0_id + 3 * var1
+      sgd_op = gradient_descent.GradientDescentOptimizer(3.0)
+      grads_and_vars = sgd_op.compute_gradients(cost, [var0, var1],
+                                                stop_gradients=[var0_id])
+      grad_dict = {var.op.name: grad for grad, var in grads_and_vars}
+      self.assertIsNone(grad_dict['var0'])
+      self.assertIsNotNone(grad_dict['var1'])
+
+  def testDoNotOverrideCreateSlots(self):
+    class ShouldNotOverrideCreateSlots(optimizer_v2.OptimizerV2):
+
+      def _create_slots(self, var_list):
+        """In OptimizerV2 _create_slots was renamed _create_vars."""
+        return var_list
+
+    with self.assertRaises(RuntimeError):
+      ShouldNotOverrideCreateSlots(True, 'name')
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/contrib/optimizer_v2/rmsprop.py b/tensorflow/contrib/optimizer_v2/rmsprop.py
new file mode 100644
index 0000000000..164ff0ea06
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/rmsprop.py
@@ -0,0 +1,233 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""RMSprop optimizer for Tensorflow.
+
+rmsprop algorithm [tieleman2012rmsprop]
+
+A detailed description of rmsprop.
+
+- maintain a moving (discounted) average of the square of gradients
+- divide gradient by the root of this average
+
+mean_square = decay * mean_square{t-1} + (1-decay) * gradient ** 2
+mom = momentum * mom{t-1} + learning_rate * g_t / sqrt(mean_square + epsilon)
+delta = - mom
+
+This implementation of RMSProp uses plain momentum, not Nesterov momentum.
+
+The centered version additionally maintains a moving (discounted) average of the
+gradients, and uses that average to estimate the variance:
+
+mean_grad = decay * mean_square{t-1} + (1-decay) * gradient
+mean_square = decay * mean_square{t-1} + (1-decay) * gradient ** 2
+mom = momentum * mom{t-1} + learning_rate * g_t /
+    sqrt(mean_square - mean_grad**2 + epsilon)
+delta = - mom
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.optimizer_v2 import optimizer_v2
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import init_ops
+
+from tensorflow.python.training import training_ops
+
+
+class RMSPropOptimizer(optimizer_v2.OptimizerV2):
+  """Optimizer that implements the RMSProp algorithm.
+
+  See the
+  [paper](http://www.cs.toronto.edu/~tijmen/csc321/slides/lecture_slides_lec6.pdf).
+  """
+
+  def __init__(self,
+               learning_rate,
+               decay=0.9,
+               momentum=0.0,
+               epsilon=1e-10,
+               use_locking=False,
+               centered=False,
+               name="RMSProp"):
+    """Construct a new RMSProp optimizer.
+
+    Note that in the dense implementation of this algorithm, variables and their
+    corresponding accumulators (momentum, gradient moving average, square
+    gradient moving average) will be updated even if the gradient is zero
+    (i.e. accumulators will decay, momentum will be applied). The sparse
+    implementation (used when the gradient is an `IndexedSlices` object,
+    typically because of `tf.gather` or an embedding lookup in the forward pass)
+    will not update variable slices or their accumulators unless those slices
+    were used in the forward pass (nor is there an "eventual" correction to
+    account for these omitted updates). This leads to more efficient updates for
+    large embedding lookup tables (where most of the slices are not accessed in
+    a particular graph execution), but differs from the published algorithm.
+
+    Some of the args below are hyperparameters, where a hyperparameter is
+    defined as a scalar Tensor, a regular Python value or a callable (which
+    will be evaluated when `apply_gradients` is called) returning a scalar
+    Tensor or a Python value.
+
+    Args:
+      learning_rate: A float hyperparameter. The learning rate.
+      decay: A float hyperparameter. Discounting factor for the history/coming
+        gradient.
+      momentum: A float hyperparameter.
+      epsilon: A float hyperparameter. Small value to avoid zero denominator.
+      use_locking: If True use locks for update operation.
+      centered: If True, gradients are normalized by the estimated variance of
+        the gradient; if False, by the uncentered second moment. Setting this to
+        True may help with training, but is slightly more expensive in terms of
+        computation and memory. Defaults to False.
+      name: Optional name prefix for the operations created when applying
+        gradients. Defaults to "RMSProp".
+    """
+    super(RMSPropOptimizer, self).__init__(use_locking, name)
+    self._set_hyper("learning_rate", learning_rate)
+    self._set_hyper("decay", decay)
+    self._set_hyper("momentum", momentum)
+    self._set_hyper("epsilon", epsilon)
+
+    self._centered = centered
+
+  def _create_vars(self, var_list, state):
+    for v in var_list:
+      if v.get_shape().is_fully_defined():
+        init_rms = init_ops.ones_initializer(dtype=v.dtype.base_dtype)
+      else:
+        init_rms = array_ops.ones_like(v)
+      state.create_slot_with_initializer(v, init_rms, v.get_shape(),
+                                         v.dtype.base_dtype, "rms")
+      if self._centered:
+        state.zeros_slot(v, "mg")
+      state.zeros_slot(v, "momentum")
+
+  def _apply_dense(self, grad, var, state):
+    rms = state.get_slot(var, "rms")
+    mom = state.get_slot(var, "momentum")
+    if self._centered:
+      mg = state.get_slot(var, "mg")
+      return training_ops.apply_centered_rms_prop(
+          var,
+          mg,
+          rms,
+          mom,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          use_locking=self._use_locking).op
+    else:
+      return training_ops.apply_rms_prop(
+          var,
+          rms,
+          mom,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          use_locking=self._use_locking).op
+
+  def _resource_apply_dense(self, grad, var, state):
+    rms = state.get_slot(var, "rms")
+    mom = state.get_slot(var, "momentum")
+    if self._centered:
+      mg = state.get_slot(var, "mg")
+      return training_ops.resource_apply_centered_rms_prop(
+          var.handle,
+          mg.handle,
+          rms.handle,
+          mom.handle,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_apply_rms_prop(
+          var.handle,
+          rms.handle,
+          mom.handle,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          use_locking=self._use_locking)
+
+  def _apply_sparse(self, grad, var, state):
+    rms = state.get_slot(var, "rms")
+    mom = state.get_slot(var, "momentum")
+    if self._centered:
+      mg = state.get_slot(var, "mg")
+      return training_ops.sparse_apply_centered_rms_prop(
+          var,
+          mg,
+          rms,
+          mom,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad.values,
+          grad.indices,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.sparse_apply_rms_prop(
+          var,
+          rms,
+          mom,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad.values,
+          grad.indices,
+          use_locking=self._use_locking)
+
+  def _resource_apply_sparse(self, grad, var, indices, state):
+    rms = state.get_slot(var, "rms")
+    mom = state.get_slot(var, "momentum")
+    if self._centered:
+      mg = self.get_slot(var, "mg")
+      return training_ops.resource_sparse_apply_centered_rms_prop(
+          var.handle,
+          mg.handle,
+          rms.handle,
+          mom.handle,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          indices,
+          use_locking=self._use_locking)
+    else:
+      return training_ops.resource_sparse_apply_rms_prop(
+          var.handle,
+          rms.handle,
+          mom.handle,
+          state.get_hyper("learning_rate", var.dtype.base_dtype),
+          state.get_hyper("decay", var.dtype.base_dtype),
+          state.get_hyper("momentum", var.dtype.base_dtype),
+          state.get_hyper("epsilon", var.dtype.base_dtype),
+          grad,
+          indices,
+          use_locking=self._use_locking)
diff --git a/tensorflow/contrib/optimizer_v2/rmsprop_test.py b/tensorflow/contrib/optimizer_v2/rmsprop_test.py
new file mode 100644
index 0000000000..ed68f6afbf
--- /dev/null
+++ b/tensorflow/contrib/optimizer_v2/rmsprop_test.py
@@ -0,0 +1,449 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for rmsprop optimizer."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import copy
+import itertools
+import math
+
+import numpy as np
+
+from tensorflow.contrib.optimizer_v2 import rmsprop
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import embedding_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import variables
+from tensorflow.python.platform import test
+
+_DATA_TYPES = [dtypes.half, dtypes.float32]
+
+_TEST_PARAM_VALUES = [
+    # learning_rate, decay, momentum, epsilon, centered, use_resource
+    [0.5, 0.9, 0.0, 1e-3, True, False],
+    [0.5, 0.9, 0.0, 1e-3, False, False],
+    [0.5, 0.9, 0.0, 1e-3, True, True],
+    [0.5, 0.9, 0.0, 1e-3, False, True],
+    [0.1, 0.9, 0.0, 1e-3, True, False],
+    [0.5, 0.95, 0.0, 1e-3, False, False],
+    [0.5, 0.95, 0.0, 1e-5, True, False],
+    [0.5, 0.95, 0.9, 1e-5, True, False],
+]
+
+_TESTPARAMS = [
+    [data_type] + values
+    for data_type, values in itertools.product(_DATA_TYPES, _TEST_PARAM_VALUES)
+]
+
+
+class RMSPropOptimizerTest(test.TestCase):
+
+  def _rmsprop_update_numpy(self, var, g, mg, rms, mom, lr, decay, momentum,
+                            epsilon, centered):
+    rms_t = rms * decay + (1 - decay) * g * g
+    denom_t = rms_t + epsilon
+    if centered:
+      mg_t = mg * decay + (1 - decay) * g
+      denom_t -= mg_t * mg_t
+    else:
+      mg_t = mg
+    mom_t = momentum * mom + lr * g / np.sqrt(denom_t, dtype=denom_t.dtype)
+    var_t = var - mom_t
+    return var_t, mg_t, rms_t, mom_t
+
+  def _sparse_rmsprop_update_numpy(self, var, gindexs, gvalues, mg, rms, mom,
+                                   lr, decay, momentum, epsilon, centered):
+    mg_t = copy.deepcopy(mg)
+    rms_t = copy.deepcopy(rms)
+    mom_t = copy.deepcopy(mom)
+    var_t = copy.deepcopy(var)
+    for i in range(len(gindexs)):
+      gindex = gindexs[i]
+      gvalue = gvalues[i]
+      rms_t[gindex] = rms[gindex] * decay + (1 - decay) * gvalue * gvalue
+      denom_t = rms_t[gindex] + epsilon
+      if centered:
+        mg_t[gindex] = mg_t[gindex] * decay + (1 - decay) * gvalue
+        denom_t -= mg_t[gindex] * mg_t[gindex]
+      mom_t[gindex] = momentum * mom[gindex] + lr * gvalue / np.sqrt(denom_t)
+      var_t[gindex] = var[gindex] - mom_t[gindex]
+    return var_t, mg_t, rms_t, mom_t
+
+  def testDense(self):
+    # TODO(yori): Use ParameterizedTest when available
+    for (dtype, learning_rate, decay, momentum,
+         epsilon, centered, use_resource) in _TESTPARAMS:
+      with self.test_session(use_gpu=True):
+        # Initialize variables for numpy implementation.
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1, 0.2], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01, 0.2], dtype=dtype.as_numpy_dtype)
+
+        if use_resource:
+          var0 = resource_variable_ops.ResourceVariable(var0_np)
+          var1 = resource_variable_ops.ResourceVariable(var1_np)
+        else:
+          var0 = variables.Variable(var0_np)
+          var1 = variables.Variable(var1_np)
+        grads0 = constant_op.constant(grads0_np)
+        grads1 = constant_op.constant(grads1_np)
+        opt = rmsprop.RMSPropOptimizer(
+            learning_rate=learning_rate,
+            decay=decay,
+            momentum=momentum,
+            epsilon=epsilon,
+            centered=centered)
+
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        mg0 = opt.get_slot(var0, "mg")
+        self.assertEqual(mg0 is not None, centered)
+        mg1 = opt.get_slot(var1, "mg")
+        self.assertEqual(mg1 is not None, centered)
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+        rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 4 steps of RMSProp
+        for _ in range(1, 5):
+          update.run()
+
+          var0_np, mg0_np, rms0_np, mom0_np = self._rmsprop_update_numpy(
+              var0_np, grads0_np, mg0_np, rms0_np, mom0_np, learning_rate,
+              decay, momentum, epsilon, centered)
+          var1_np, mg1_np, rms1_np, mom1_np = self._rmsprop_update_numpy(
+              var1_np, grads1_np, mg1_np, rms1_np, mom1_np, learning_rate,
+              decay, momentum, epsilon, centered)
+
+          # Validate updated params
+          if centered:
+            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testMinimizeSparseResourceVariable(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = rmsprop.RMSPropOptimizer(
+            learning_rate=1.0,
+            decay=0.0,
+            momentum=0.0,
+            epsilon=0.0,
+            centered=False).minimize(loss)
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[0., 1.]], var0.eval(), atol=0.01)
+
+  def testMinimizeSparseResourceVariableCentered(self):
+    for dtype in [dtypes.float32, dtypes.float64]:
+      with self.test_session():
+        var0 = resource_variable_ops.ResourceVariable([[1.0, 2.0]], dtype=dtype)
+        x = constant_op.constant([[4.0], [5.0]], dtype=dtype)
+        pred = math_ops.matmul(embedding_ops.embedding_lookup([var0], [0]), x)
+        loss = pred * pred
+        sgd_op = rmsprop.RMSPropOptimizer(
+            learning_rate=1.0,
+            decay=0.0,
+            momentum=0.0,
+            epsilon=1.0,
+            centered=True).minimize(loss)
+        variables.global_variables_initializer().run()
+        # Fetch params to validate initial values
+        self.assertAllCloseAccordingToType([[1.0, 2.0]], var0.eval())
+        # Run 1 step of sgd
+        sgd_op.run()
+        # Validate updated params
+        self.assertAllCloseAccordingToType(
+            [[-111, -138]], var0.eval(), atol=0.01)
+
+  def testSparse(self):
+    # TODO(yori): Use ParameterizedTest when available
+    for (dtype, learning_rate, decay,
+         momentum, epsilon, centered, _) in _TESTPARAMS:
+      with self.test_session(use_gpu=True):
+        # Initialize variables for numpy implementation.
+        var0_np = np.array([1.0, 2.0], dtype=dtype.as_numpy_dtype)
+        grads0_np = np.array([0.1], dtype=dtype.as_numpy_dtype)
+        var1_np = np.array([3.0, 4.0], dtype=dtype.as_numpy_dtype)
+        grads1_np = np.array([0.01], dtype=dtype.as_numpy_dtype)
+
+        var0 = variables.Variable(var0_np)
+        var1 = variables.Variable(var1_np)
+        grads0_np_indices = np.array([0], dtype=np.int32)
+        grads0 = ops.IndexedSlices(
+            constant_op.constant(grads0_np),
+            constant_op.constant(grads0_np_indices), constant_op.constant([1]))
+        grads1_np_indices = np.array([1], dtype=np.int32)
+        grads1 = ops.IndexedSlices(
+            constant_op.constant(grads1_np),
+            constant_op.constant(grads1_np_indices), constant_op.constant([1]))
+        opt = rmsprop.RMSPropOptimizer(
+            learning_rate=learning_rate,
+            decay=decay,
+            momentum=momentum,
+            epsilon=epsilon,
+            centered=centered)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        mg0 = opt.get_slot(var0, "mg")
+        self.assertEqual(mg0 is not None, centered)
+        mg1 = opt.get_slot(var1, "mg")
+        self.assertEqual(mg1 is not None, centered)
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        mg0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mg1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        rms0_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+        rms1_np = np.array([1.0, 1.0], dtype=dtype.as_numpy_dtype)
+        mom0_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+        mom1_np = np.array([0.0, 0.0], dtype=dtype.as_numpy_dtype)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+
+        # Run 4 steps of RMSProp
+        for _ in range(1, 5):
+          update.run()
+
+          var0_np, mg0_np, rms0_np, mom0_np = self._sparse_rmsprop_update_numpy(
+              var0_np, grads0_np_indices, grads0_np, mg0_np, rms0_np, mom0_np,
+              learning_rate, decay, momentum, epsilon, centered)
+          var1_np, mg1_np, rms1_np, mom1_np = self._sparse_rmsprop_update_numpy(
+              var1_np, grads1_np_indices, grads1_np, mg1_np, rms1_np, mom1_np,
+              learning_rate, decay, momentum, epsilon, centered)
+
+          # Validate updated params
+          if centered:
+            self.assertAllCloseAccordingToType(mg0_np, mg0.eval())
+            self.assertAllCloseAccordingToType(mg1_np, mg1.eval())
+          self.assertAllCloseAccordingToType(rms0_np, rms0.eval())
+          self.assertAllCloseAccordingToType(rms1_np, rms1.eval())
+          self.assertAllCloseAccordingToType(mom0_np, mom0.eval())
+          self.assertAllCloseAccordingToType(mom1_np, mom1.eval())
+          self.assertAllCloseAccordingToType(var0_np, var0.eval())
+          self.assertAllCloseAccordingToType(var1_np, var1.eval())
+
+  def testWithoutMomentum(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.test_session(use_gpu=True):
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+        opt = rmsprop.RMSPropOptimizer(
+            learning_rate=2.0, decay=0.9, momentum=0.0, epsilon=1.0)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Step 1: the rms accumulators where 1. So we should see a normal
+        # update: v -= grad * learning_rate
+        update.run()
+        # Check the root mean square accumulators.
+        self.assertAllCloseAccordingToType(
+            np.array([0.901, 0.901]), rms0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.90001, 0.90001]), rms1.eval())
+        # Check the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0))
+            ]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0))
+            ]), var1.eval())
+        # Step 2: the root mean square accumulators contain the previous update.
+        update.run()
+        # Check the rms accumulators.
+        self.assertAllCloseAccordingToType(
+            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+        # Check the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
+                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0)),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1.0)) -
+                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1.0))
+            ]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
+                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0)),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1.0)) -
+                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 1e-5 + 1.0))
+            ]), var1.eval())
+
+  def testWithMomentum(self):
+    for dtype in [dtypes.half, dtypes.float32]:
+      with self.test_session(use_gpu=True):
+        var0 = variables.Variable([1.0, 2.0], dtype=dtype)
+        var1 = variables.Variable([3.0, 4.0], dtype=dtype)
+        grads0 = constant_op.constant([0.1, 0.1], dtype=dtype)
+        grads1 = constant_op.constant([0.01, 0.01], dtype=dtype)
+
+        opt = rmsprop.RMSPropOptimizer(
+            learning_rate=2.0, decay=0.9, momentum=0.5, epsilon=1e-5)
+        update = opt.apply_gradients(zip([grads0, grads1], [var0, var1]))
+        variables.global_variables_initializer().run()
+
+        rms0 = opt.get_slot(var0, "rms")
+        self.assertTrue(rms0 is not None)
+        rms1 = opt.get_slot(var1, "rms")
+        self.assertTrue(rms1 is not None)
+        mom0 = opt.get_slot(var0, "momentum")
+        self.assertTrue(mom0 is not None)
+        mom1 = opt.get_slot(var1, "momentum")
+        self.assertTrue(mom1 is not None)
+
+        # Fetch params to validate initial values
+        self.assertAllClose([1.0, 2.0], var0.eval())
+        self.assertAllClose([3.0, 4.0], var1.eval())
+        # Step 1: rms = 1, mom = 0. So we should see a normal
+        # update: v -= grad * learning_rate
+        update.run()
+        # Check the root mean square accumulators.
+        self.assertAllCloseAccordingToType(
+            np.array([0.901, 0.901]), rms0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.90001, 0.90001]), rms1.eval())
+        # Check the momentum accumulators
+        self.assertAllCloseAccordingToType(
+            np.array([(0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
+                      (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))]), mom0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([(0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
+                      (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))]), mom1.eval())
+
+        # Check that the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5))
+            ]), var0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5))
+            ]), var1.eval())
+
+        # Step 2: the root mean square accumulators contain the previous update.
+        update.run()
+        # Check the rms accumulators.
+        self.assertAllCloseAccordingToType(
+            np.array([0.901 * 0.9 + 0.001, 0.901 * 0.9 + 0.001]), rms0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([0.90001 * 0.9 + 1e-5, 0.90001 * 0.9 + 1e-5]), rms1.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)),
+                0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+                (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))
+            ]), mom0.eval())
+        self.assertAllCloseAccordingToType(
+            np.array([
+                0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)),
+                0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+                (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))
+            ]), mom1.eval())
+
+        # Check the parameters.
+        self.assertAllCloseAccordingToType(
+            np.array([
+                1.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
+                (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5))),
+                2.0 - (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) -
+                (0.5 * (0.1 * 2.0 / math.sqrt(0.901 + 1e-5)) +
+                 (0.1 * 2.0 / math.sqrt(0.901 * 0.9 + 0.001 + 1e-5)))
+            ]), var0.eval())
+
+        self.assertAllCloseAccordingToType(
+            np.array([
+                3.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
+                (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5))),
+                4.0 - (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) -
+                (0.5 * (0.01 * 2.0 / math.sqrt(0.90001 + 1e-5)) +
+                 (0.01 * 2.0 / math.sqrt(0.90001 * 0.9 + 2e-5)))
+            ]), var1.eval())
+
+
+if __name__ == "__main__":
+  test.main()
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index d5106752dd..899fda67fe 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -99,6 +99,7 @@ _update_device = threading.local()
 
 
 def get_update_device():
+  """Get the current device if in a `DistributionStrategy.update()` call."""
   try:
     return _update_device.current
   except AttributeError:
@@ -406,19 +407,19 @@ class DistributionStrategy(object):
     different across devices, and "Mirrored" when the value are the same.
   * Unwrapping and merging: Consider calling a function `fn` on
     multiple devices, like `call_for_each_tower(fn, w)` with an
-    argument `w that is a wrapped value. This means `w` will have a
+    argument `w` that is a wrapped value. This means `w` will have a
     map taking tower device `d0` to `w0`, tower device `d1` to `w1`,
     etc. `call_for_each_tower()` unwraps `w` before calling `fn`, so
     it calls `fn(w0)` on `d0`, `fn(w1)` on `d1`, etc.  It then merges
     the return values from `fn()`, which can possibly result in
     wrapped values. For example, let's say `fn()` returns a tuple with
-    three components: (x, a, v0) from tower 0, (x, b, v1) on tower 1,
+    three components: `(x, a, v0)` from tower 0, `(x, b, v1)` on tower 1,
     etc. If the first component is the same object `x` from every
     tower, then the first component of the merged result will also be
     `x`. If the second component is different (`a`, `b`, ...)  from
     each tower, then the merged value will have a wrapped map from
     tower device to the different values. If the third component is
-    the members of a mirrored variable (`v` maps `d0` to `v0, `d1` to
+    the members of a mirrored variable (`v` maps `d0` to `v0`, `d1` to
     `v1`, etc.), then the merged result will be that mirrored variable
     (`v`).
   * Tower context vs. Cross-tower context: _tower context_ is when we
diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py
index d22a465376..34dd419f15 100644
--- a/tensorflow/tools/docs/generate_lib.py
+++ b/tensorflow/tools/docs/generate_lib.py
@@ -211,7 +211,6 @@ def _get_default_do_not_descend_map():
       'tf': ['cli', 'lib', 'wrappers'],
       'tf.contrib': [
           'compiler',
-          'distribute',
           'grid_rnn',
           # Block contrib.keras to de-clutter the docs
           'keras',
-- 
GitLab


From 79a5ae8ccf1af9e46e10a1e9f8347b33343b06e8 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 29 Mar 2018 15:32:14 -0700
Subject: [PATCH 710/960] Internal Change

PiperOrigin-RevId: 190996815
---
 tensorflow/contrib/distribute/README.md | 143 ++++++++++++++++++++++++
 1 file changed, 143 insertions(+)
 create mode 100644 tensorflow/contrib/distribute/README.md

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
new file mode 100644
index 0000000000..ba9a392c77
--- /dev/null
+++ b/tensorflow/contrib/distribute/README.md
@@ -0,0 +1,143 @@
+# Distribution Strategy
+
+> *NOTE*: This is a experimental feature. The API and performance
+> characteristics are subject to change.
+
+## Overview
+
+[`DistributionStrategy`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/distribute/DistributionStrategy)
+API is an easy way to distribute your training
+across multiple devices/machines. Our goal is to allow users to use existing
+models and training code with minimal changes to enable distributed training.
+Moreover, we've design the API in such a way that it works with both eager and
+graph execution.
+
+Currently we support one type of strategy, called
+[`MirroredStrategy`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/distribute/MirroredStrategy).
+It does in-graph replication with synchronous training
+on many GPUs on one machine. Essentially, we create copies of all variables in
+the model's layers on each device. We then use all-reduce to combine gradients
+across the devices before applying them to the variables to keep them in sync.
+In the future, we intend to support other kinds of training configurations such
+as multi-node, synchronous,
+[asynchronous](https://www.tensorflow.org/deploy/distributed#putting_it_all_together_example_trainer_program),
+parameter servers and model parallelism.
+
+## Example
+
+Let's demonstrate how to use this API with a simple example. We will use the
+[`Estimator`](https://www.tensorflow.org/api_docs/python/tf/estimator/Estimator)
+approach, and show you how to scale your model to run on multiple GPUs on one
+machine using `MirroredStrategy`.
+
+Let's consider a very simple model function which tries to learn a simple
+function.
+
+```python
+def model_fn(features, labels, mode):
+  layer = tf.layers.Dense(1)
+  logits = layer(features)
+
+  if mode == tf.estimator.ModeKeys.PREDICT:
+    predictions = {"logits": logits}
+    return tf.estimator.EstimatorSpec(mode, predictions=predictions)
+
+  loss = tf.losses.mean_squared_error(
+      labels=labels, predictions=tf.reshape(logits, []))
+
+  if mode == tf.estimator.ModeKeys.EVAL:
+    return tf.estimator.EstimatorSpec(mode, loss=loss)
+
+  if mode == tf.estimator.ModeKeys.TRAIN:
+    train_op = tf.train.GradientDescentOptimizer(0.2).minimize(loss_fn())
+    return tf.estimator.EstimatorSpec(mode, loss=loss, train_op=train_op)
+```
+
+Let's also define a simple input function to feed data for training this model.
+Note that we require using
+[`tf.data.Dataset`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset)
+with `DistributionStrategy`.
+
+
+```python
+def input_fn():
+  features = tf.data.Dataset.from_tensors([[1.]]).repeat(100)
+  labels = tf.data.Dataset.from_tensors(1.).repeat(100)
+  return dataset_ops.Dataset.zip((features, labels))
+```
+
+Now that we have a model function and input function defined, we can define the
+estimator. To use `MirroredStrategy`, all we need to do is:
+
+* Create an instance of the `MirroredStrategy` class.
+* Pass it to the
+[`RunConfig`](https://www.tensorflow.org/api_docs/python/tf/estimator/RunConfig)
+parameter of `Estimator`.
+
+
+```python
+distribution = tf.contrib.distribute.MirroredStrategy()
+config = tf.estimator.RunConfig(distribute=distribution)
+classifier = tf.estimator.Estimator(model_fn=model_fn, config=config)
+classifier.train(input_fn=input_fn)
+```
+
+That's it! This change will now configure estimator to run on all GPUs on your
+machine, with the `MirroredStrategy` approach. It will take care of distributing
+the input dataset, replicating layers and variables on each device, and
+combining and applying gradients.
+
+The model and input functions do not have to change because we have changed the
+underlying components of TensorFlow (such as
+optimizer, batch norm and summaries) to become distribution-aware. 
+That means those components learn how to
+combine their state across devices. Further, saving and checkpointing works
+seamlessly, so you can save with one or no distribution strategy and resume with
+another.
+
+Above, we showed the easiest way to use [`MirroredStrategy`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/distribute/MirroredStrategy#__init__).
+There are few things you can customize in practice:
+
+* You can specify a list of specific GPUs (using param `devices`) or the number
+of GPUs (using param `num_gpus`), in case you don't want auto detection.
+* You can specify various parameters for all reduce with the `cross_tower_ops`
+param, such as the all reduce algorithm to use, and gradient repacking.
+
+## Performance Tips
+
+We've tried to make it such that you get the best performance for your existing
+model. We also recommend you follow the tips from
+[Input Pipeline Performance Guide](https://www.tensorflow.org/performance/datasets_performance).
+Specifically, we found using [`map_and_batch`](https://www.tensorflow.org/performance/datasets_performance#map_and_batch)
+and [`dataset.prefetch`](https://www.tensorflow.org/performance/datasets_performance#pipelining)
+in the input function gives a solid boost in performance. When using
+`dataset.prefetch`, use `buffer_size=None` to let it detect optimal buffer size.
+
+## Caveats
+This feature is in early stages and there are a lot of improvements forthcoming:
+
+* Metrics are not yet supported during distributed training.
+* Summaries are currently computed in every tower.
+* Evaluation is not yet distributed.
+* Eager support is in the works; performance can be more challenging with eager
+execution.
+* As mentioned earlier, multi-node and other distributed strategies will be
+introduced in the future.
+* If you are [`batching`](https://www.tensorflow.org/api_docs/python/tf/data/Dataset#batch)
+your input data, we will place one batch on each GPU in each step. So your
+effective batch size will be `num_gpus * batch_size`. Therefore, consider
+adjusting your learning rate or batch size according to the number of GPUs.
+We are working on addressing this limitation by splitting each batch across GPUs
+instead.
+* Dictionaries inside dataset in the input are not supported when prefetching
+on GPUs is turned on. (If you need to use dictionaries in the dataset, turn off
+prefetching on GPUs by passing param `prefetch_on_device=False` to
+`MirroredStrategy`)
+
+## What's next?
+
+Please give distribution strategies a try. This feature is in early stages and
+is evolving, so we welcome your feedback via
+[issues on GitHub](https://github.com/tensorflow/tensorflow/issues/new).
+
+
-- 
GitLab


From 497dab37519a1856a52e6564d8eb1d03382911c3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 15:32:42 -0700
Subject: [PATCH 711/960]   capture_tpu_profile will fallback to old behavior
 if user specify local directories as model directory.

PiperOrigin-RevId: 190996878
---
 tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
index e6811d4ad2..f2003e04dd 100644
--- a/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
+++ b/tensorflow/contrib/tpu/profiler/capture_tpu_profile.cc
@@ -70,8 +70,12 @@ ProfileResponse Profile(const string& service_addr, int duration_ms,
   ProfileRequest request;
   request.set_duration_ms(duration_ms);
   request.set_max_events(kMaxEvents);
-  request.set_repository_root(repository_root);
-  request.set_session_id(session_id);
+  if (tensorflow::str_util::StartsWith(repository_root, "gs://")) {
+    // For backward compatibilities, only generate tracetable etc when the
+    // user provide a GCS path for model directory.
+    request.set_repository_root(repository_root);
+    request.set_session_id(session_id);
+  }
   request.add_tools("input_pipeline");
   request.add_tools("overview_page");
   *request.mutable_opts() = opts;
-- 
GitLab


From af670bdc0e61802778f61778dd1623c87f30e874 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Thu, 29 Mar 2018 15:36:14 -0700
Subject: [PATCH 712/960] Undisables broken list_ops_test

PiperOrigin-RevId: 190997355
---
 tensorflow/core/kernels/list_kernels.h | 16 +++++++---------
 tensorflow/python/kernel_tests/BUILD   |  4 ----
 2 files changed, 7 insertions(+), 13 deletions(-)

diff --git a/tensorflow/core/kernels/list_kernels.h b/tensorflow/core/kernels/list_kernels.h
index 8af48f0a67..f3bbf3b6e3 100644
--- a/tensorflow/core/kernels/list_kernels.h
+++ b/tensorflow/core/kernels/list_kernels.h
@@ -160,15 +160,13 @@ class TensorListFromTensor : public OpKernel {
       tmp_shape.RemoveDim(0);
       OP_REQUIRES(c, tmp.CopyFrom(tmp, tmp_shape),
                   errors::Unknown("Unexpected shape error."));
-      if (tmp.IsAligned() || !DataTypeCanUseMemcpy(DataTypeToEnum<T>::value)) {
-        output_list.tensors.push_back(tmp);
-      } else {
-        Tensor aligned;
-        OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
-        aligned.flat<T>().device(c->eigen_device<Device>()) =
-            tmp.unaligned_flat<T>();
-        output_list.tensors.push_back(aligned);
-      }
+      // TODO(apassos) maybe not always align; but weird compiler bugs seem to
+      // prevent this.
+      Tensor aligned;
+      OP_REQUIRES_OK(c, c->allocate_temp(tmp.dtype(), tmp.shape(), &aligned));
+      aligned.flat<T>().device(c->eigen_device<Device>()) =
+          tmp.unaligned_flat<T>();
+      output_list.tensors.push_back(aligned);
     }
     output_tensor->scalar<Variant>()() = std::move(output_list);
   }
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index 5eceb9f768..ea210346c1 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -96,10 +96,6 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
     ],
     grpc_enabled = True,
-    tags = [
-        "no_gpu",
-        "nogpu",
-    ],
 )
 
 cuda_py_test(
-- 
GitLab


From 2bc52cd2d481a89c9724d20e827097efa4ff3f1e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 15:40:14 -0700
Subject: [PATCH 713/960] - Expose slim arg_scope function to compute keys to
 enable tessting. - Add is_training=None option to mobinenet arg_scopes. This
 allows the users to set is_training from an outer scope.

PiperOrigin-RevId: 190997959
---
 .../contrib/framework/python/ops/arg_scope.py | 28 +++++++++----------
 1 file changed, 14 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/framework/python/ops/arg_scope.py b/tensorflow/contrib/framework/python/ops/arg_scope.py
index 3cad1fee19..5b15033995 100644
--- a/tensorflow/contrib/framework/python/ops/arg_scope.py
+++ b/tensorflow/contrib/framework/python/ops/arg_scope.py
@@ -68,7 +68,7 @@ from tensorflow.python.util import tf_decorator
 
 __all__ = [
     'arg_scope', 'add_arg_scope', 'current_arg_scope', 'has_arg_scope',
-    'arg_scoped_arguments'
+    'arg_scoped_arguments', 'arg_scope_func_key'
 ]
 
 _ARGSTACK = [{}]
@@ -89,7 +89,7 @@ def current_arg_scope():
   return stack[-1]
 
 
-def _key_op(op):
+def arg_scope_func_key(op):
   return getattr(op, '_key_op', str(op))
 
 
@@ -103,9 +103,9 @@ def _kwarg_names(func):
 
 
 def _add_op(op):
-  key_op = _key_op(op)
-  if key_op not in _DECORATED_OPS:
-    _DECORATED_OPS[key_op] = _kwarg_names(op)
+  key = arg_scope_func_key(op)
+  if key not in _DECORATED_OPS:
+    _DECORATED_OPS[key] = _kwarg_names(op)
 
 
 @tf_contextlib.contextmanager
@@ -147,16 +147,16 @@ def arg_scope(list_ops_or_scope, **kwargs):
     try:
       current_scope = current_arg_scope().copy()
       for op in list_ops_or_scope:
-        key_op = _key_op(op)
+        key = arg_scope_func_key(op)
         if not has_arg_scope(op):
           raise ValueError('%s is not decorated with @add_arg_scope',
                            _name_op(op))
-        if key_op in current_scope:
-          current_kwargs = current_scope[key_op].copy()
+        if key in current_scope:
+          current_kwargs = current_scope[key].copy()
           current_kwargs.update(kwargs)
-          current_scope[key_op] = current_kwargs
+          current_scope[key] = current_kwargs
         else:
-          current_scope[key_op] = kwargs.copy()
+          current_scope[key] = kwargs.copy()
       _get_arg_stack().append(current_scope)
       yield current_scope
     finally:
@@ -176,14 +176,14 @@ def add_arg_scope(func):
   def func_with_args(*args, **kwargs):
     current_scope = current_arg_scope()
     current_args = kwargs
-    key_func = _key_op(func)
+    key_func = arg_scope_func_key(func)
     if key_func in current_scope:
       current_args = current_scope[key_func].copy()
       current_args.update(kwargs)
     return func(*args, **current_args)
 
   _add_op(func)
-  setattr(func_with_args, '_key_op', _key_op(func))
+  setattr(func_with_args, '_key_op', arg_scope_func_key(func))
   return tf_decorator.make_decorator(func, func_with_args)
 
 
@@ -196,7 +196,7 @@ def has_arg_scope(func):
   Returns:
     a boolean.
   """
-  return _key_op(func) in _DECORATED_OPS
+  return arg_scope_func_key(func) in _DECORATED_OPS
 
 
 def arg_scoped_arguments(func):
@@ -209,4 +209,4 @@ def arg_scoped_arguments(func):
     a list of kwargs names.
   """
   assert has_arg_scope(func)
-  return _DECORATED_OPS[_key_op(func)]
+  return _DECORATED_OPS[arg_scope_func_key(func)]
-- 
GitLab


From e302cd64afacb5cc9057f03b5fbbee6315a33573 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 16:01:20 -0700
Subject: [PATCH 714/960] Updated eager guide to use tensorflow 1.7. Code
 snippets still work.

PiperOrigin-RevId: 191001008
---
 .../docs_src/programmers_guide/eager.md       | 203 +-----------------
 1 file changed, 5 insertions(+), 198 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
index 9ae1e602f4..8db65737dc 100644
--- a/tensorflow/docs_src/programmers_guide/eager.md
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -29,10 +29,10 @@ problem and share your benchmarks.
 
 ## Setup and basic usage
 
-Install TensorFlow 1.7 to include the updates for eager execution:
+Upgrade to TensorFlow 1.7 to include updates for eager execution:
 
 ```
-$ pip install --pre --upgrade tensorflow
+$ pip install --upgrade tensorflow
 ```
 
 To start eager execution, add `tf.enable_eager_execution()` to the beginning of
@@ -322,14 +322,13 @@ grad_log1pexp(0.)  # => [0.5]
 grad_log1pexp(100.)  # => [nan]
 ```
 
-
 Here, the `log1pexp` function can be analytically simplified with a custom
 gradient. The implementation below reuses the value for `tf.exp(x)` that is
 computed during the forward pass—making it more efficient by eliminating
 redundant calculations:
 
 ```py
-@tfe.custom_gradient
+@tf.custom_gradient
 def log1pexp(x):
   e = tf.exp(x)
   def grad(dy):
@@ -605,7 +604,7 @@ print(x)  # => 2.0
 ```
 
 To save and load models, `tfe.Checkpoint` stores the internal state of objects,
-without requiring hiiden variables. To record the state of a `model`,
+without requiring hidden variables. To record the state of a `model`,
 an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`:
 
 ```py
@@ -649,9 +648,8 @@ inserted during model construction. For example, to record summaries once every
 100 global steps:
 
 ```py
-tf.train.get_or_create_global_step()  # return global step var
 writer = tf.contrib.summary.create_file_writer(logdir)
-global_step=tf.train.get_or_create_global_step()
+global_step=tf.train.get_or_create_global_step()  # return global step var
 
 writer.set_as_default()
 
@@ -733,197 +731,6 @@ But this gap grows larger for models with less computation and there is work to
 be done for optimizing hot code paths for models with lots of small operations.
 
 
-## Work with graphs
-
-While eager execution makes development and debugging more interactive,
-TensorFlow graph execution has advantages for distributed training, performance
-optimizations, and production deployment. However, writing graph code can feel
-different than writing regular Python code and more difficult to debug.
-
-For building and training graph-constructed models, the Python program first
-builds a graph representing the computation, then invokes `Session.run` to send
-the graph for execution on the C++-based runtime.  This provides:
-
-* Automatic differentiation using static autodiff.
-* Simple deployment to a platform independent server.
-* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
-* Compilation and kernel fusion.
-* Automatic distribution and replication (placing nodes on the distributed system).
-
-Deploying code written for eager execution is more difficult: either generate a
-graph from the model, or run the Python runtime and code directly on the server.
-
-### Write compatible code
-
-The same code written for eager execution will also build a graph during graph
-execution. Do this by simply running the same code in a new Python session where
-eager execution is not enabled.
-
-Most TensorFlow operations work during eager execution, but there are some things
-to keep in mind:
-
-* Use `tf.data` for input processing instead of queues. It's faster and easier.
-* Use object-oriented layer APIs—like `tf.keras.layers` and
-  `tf.keras.Model`—since they have explicit storage for variables.
-* Most model code works the same during eager and graph execution, but there are
-  exceptions. (For example, dynamic models using Python control flow to change the
-  computation based on inputs.)
-* Once eager execution is enabled with `tf.enable_eager_execution`, it
-  cannot be turned off. Start a new Python session to return to graph execution.
-
-It's best to write code for both eager execution *and* graph execution. This
-gives you eager's interactive experimentation and debuggability with the
-distributed performance benefits of graph execution.
-
-Write, debug, and iterate in eager execution, then import the model graph for
-production deployment. Use `tfe.Checkpoint` to save and restore model
-variables, this allows movement between eager and graph execution environments.
-See the examples in:
-[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
-
-### Use eager execution in a graph environment
-
-Selectively enable eager execution in a TensorFlow graph environment using
-`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
-been called.
-
-```py
-def my_py_func(x):
-  x = tf.matmul(x, x)  # You can use tf ops
-  print(x)  # but it's eager!
-  return x
-
-with tf.Session() as sess:
-  x = tf.placeholder(dtype=tf.float32)
-  # Call eager function in graph!
-  pf = tfe.py_func(my_py_func, [x], tf.float32)
-  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
-```
-
-
-A `tfe.Checkpoint` stores the complete internal state of the objects passed to it. Nothing else is implicitly included. To record the state of a `model`, an `optimizer`, and a global step pass each one to the checkpoint's constructor:
-
-```py
-model = MyModel()
-optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
-checkpoint_dir = ‘/path/to/model_dir’
-checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
-root = tfe.Checkpoint(optimizer=optimizer,
-                      model=model,
-                      optimizer_step=tf.train.get_or_create_global_step())
-
-root.save(file_prefix=checkpoint_prefix)
-# or
-root.restore(tf.train.latest_checkpoint(checkpoint_dir))
-```
-
-### Object-oriented metrics
-
-`tfe.metrics` are stored as objects. Update a metric by passing the new data to
-the callable, and retrieve the result using the `tfe.metrics.result` method,
-for example:
-
-```py
-m = tfe.metrics.Mean("loss")
-m(0)
-m(5)
-m.result()  # => 2.5
-m([8, 9])
-m.result()  # => 5.5
-```
-
-#### Summaries and TensorBoard
-
-@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
-understanding, debugging and optimizing the model training process. It uses
-summary events that are written while executing the program.
-
-`tf.contrib.summary` is compatible with both eager and graph execution
-environments. Summary operations, such as `tf.contrib.summary.scalar`, are
-inserted during model construction. For example, to record summaries once every
-100 global steps:
-
-```py
-tf.train.get_or_create_global_step()  # return global step var
-writer = tf.contrib.summary.create_file_writer(logdir)
-
-for _ in range(iterations):
-  with writer.as_default():
-    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
-      # your model code goes here
-      tf.contrib.summary.scalar('loss', loss)
-      ...
-```
-
-## Performance
-
-Computation is not automatically offloaded to GPUs during eager execution. To
-explicitly direct a computation to a GPU, enclose it in a
-`tf.device('/gpu:0')` block:
-
-```py
-import time
-
-def measure(x, steps):
-  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
-  tf.matmul(x, x)
-  start = time.time()
-  for i in range(steps):
-    x = tf.matmul(x, x)
-    _ = x.numpy()  # Make sure to execute op and not just enqueue it
-  end = time.time()
-  return end - start
-
-shape = (1000, 1000)
-steps = 200
-print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
-
-# Run on CPU:
-with tf.device("/cpu:0"):
-  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
-
-# Run on GPU, if available:
-if tfe.num_gpus() > 0:
-  with tf.device("/gpu:0"):
-    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
-else:
-  print("GPU: not found")
-```
-
-Output (exact numbers depend on hardware):
-
-```
-Time to multiply a (1000, 1000) matrix by itself 200 times:
-CPU: 4.614904403686523 secs
-GPU: 0.5581181049346924 secs
-```
-
-A `tf.Tensor` object can be copied to a different device to execute its
-operations:
-
-```py
-x = tf.random_normal([10, 10])
-
-x_gpu0 = x.gpu()
-x_cpu = x.cpu()
-
-_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
-_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
-
-if tfe.num_gpus() > 1:
-  x_gpu1 = x.gpu(1)
-  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
-```
-
-### Benchmarks
-
-For compute-heavy models, such as
-[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
-training on a GPU, eager execution performance is comparable to graph execution.
-But this gap grows larger for models with less computation and there is work to
-be done for optimizing hot code paths for models with lots of small operations.
-
-
 ## Work with graphs
 
 While eager execution makes development and debugging more interactive,
-- 
GitLab


From 0f01f076f86882104c4c358b2679cce1ad85057c Mon Sep 17 00:00:00 2001
From: Mark Heffernan <meheff@google.com>
Date: Thu, 29 Mar 2018 16:02:26 -0700
Subject: [PATCH 715/960] Add support for running benchmarks in XLA unit tests.
 In the XLA internal test 'main', parse the --benchmarks flag if it exists and
 runs the specified benchmarks. Previously microbenchmarks defined in unit
 tests were never run.

PiperOrigin-RevId: 191001183
---
 .../xla/tests/xla_internal_test_main.cc       | 33 +++++++++++++++++++
 1 file changed, 33 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/xla_internal_test_main.cc b/tensorflow/compiler/xla/tests/xla_internal_test_main.cc
index 92b2b1ee77..0af40bc15a 100644
--- a/tensorflow/compiler/xla/tests/xla_internal_test_main.cc
+++ b/tensorflow/compiler/xla/tests/xla_internal_test_main.cc
@@ -12,9 +12,12 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
+
 #include "tensorflow/compiler/xla/legacy_flags/debug_options_flags.h"
+#include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
 
 GTEST_API_ int main(int argc, char** argv) {
   std::vector<tensorflow::Flag> flag_list;
@@ -25,7 +28,37 @@ GTEST_API_ int main(int argc, char** argv) {
     return 2;
   }
 
+  // If the --benchmarks flag is passed in then only run the benchmarks, not the
+  // tests.
+  for (int i = 1; i < argc; i++) {
+    tensorflow::StringPiece arg(argv[i]);
+    if (arg == "--benchmarks" || arg.starts_with("--benchmarks=")) {
+      const char* pattern = nullptr;
+      if (arg.starts_with("--benchmarks=")) {
+        pattern = argv[i] + strlen("--benchmarks=");
+      } else {
+        // Handle flag of the form '--benchmarks foo' (no '=').
+        if (i + 1 >= argc ||
+            tensorflow::StringPiece(argv[i + 1]).starts_with("--")) {
+          LOG(ERROR) << "--benchmarks flag requires an argument.";
+          return 2;
+        }
+        pattern = argv[i + 1];
+      }
+      // Unfortunately Google's internal benchmark infrastructure has a
+      // different API than Tensorflow's.
+#if defined(PLATFORM_GOOGLE)
+      base::SetFlag(&FLAGS_benchmarks, pattern);
+      RunSpecifiedBenchmarks();
+#else
+      tensorflow::testing::Benchmark::Run(pattern);
+#endif
+      return 0;
+    }
+  }
+
   testing::InitGoogleTest(&argc, argv);
+
   if (argc > 1) {
     LOG(ERROR) << "Unknown argument " << argv[1] << "\n" << usage;
     return 2;
-- 
GitLab


From 72205dadc2a973b746b3fdb6708429fd882a5d23 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 29 Mar 2018 16:37:05 -0700
Subject: [PATCH 716/960] Minor language change in readme.

PiperOrigin-RevId: 191006151
---
 tensorflow/contrib/distribute/README.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index ba9a392c77..4af51bec1a 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -89,8 +89,8 @@ combining and applying gradients.
 
 The model and input functions do not have to change because we have changed the
 underlying components of TensorFlow (such as
-optimizer, batch norm and summaries) to become distribution-aware. 
-That means those components learn how to
+optimizer, batch norm and summaries) to become distribution-aware.
+That means those components know how to
 combine their state across devices. Further, saving and checkpointing works
 seamlessly, so you can save with one or no distribution strategy and resume with
 another.
@@ -133,6 +133,7 @@ instead.
 on GPUs is turned on. (If you need to use dictionaries in the dataset, turn off
 prefetching on GPUs by passing param `prefetch_on_device=False` to
 `MirroredStrategy`)
+* PartitionedVariables are not supported yet.
 
 ## What's next?
 
-- 
GitLab


From 1ba89338bdb4afb85ae56e64b47acc93a3a28703 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Mar 2018 16:50:34 -0700
Subject: [PATCH 717/960] Fixing a subtle bug where in some cases the post
 cancellation work wasn't being done correctly. This is the scenario in which
 FunctionBufferingResource::Cancel() got called while buffering was being
 done, but then the buffer filled up in which case FillBuffer() wasn't ever
 called and the Cancel() method would get stuck waiting on a notification from
 the condition variable leading to timeouts. This CL fixes this by making sure
 FillBuffer() got called one last time in this case.

Tested by running contrib/data/python/kernel_tests:prefetching_ops_test 500 times and ran contrib/distribute/python:values_test 500 times with no timeouts.

PiperOrigin-RevId: 191007895
---
 tensorflow/contrib/data/kernels/prefetching_kernels.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 2afb8dbbf4..207f2820bf 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -224,6 +224,13 @@ class FunctionBufferingResource : public ResourceBase {
                   if (buffer_.size() < buffer_size_ && !end_of_sequence_) {
                     restart_buffering = true;
                   } else {
+                    // When the buffer is full, we don't want to call
+                    // FillBuffer() unless we're in cancellation phase in which
+                    // case FillBuffer() will do the final cleanup post
+                    // cancellation.
+                    if (cancelled_) {
+                      restart_buffering = true;
+                    }
                     is_buffering_ = false;
                   }
                 }
-- 
GitLab


From 6628001dcc10c429aec33da186ff281d26729ac3 Mon Sep 17 00:00:00 2001
From: Nupur Garg <nupurgarg@google.com>
Date: Thu, 29 Mar 2018 16:56:42 -0700
Subject: [PATCH 718/960] Updating documentation.

PiperOrigin-RevId: 191008662
---
 tensorflow/contrib/lite/toco/README.md        |  33 +-
 .../lite/toco/g3doc/cmdline_examples.md       | 404 ++++++++----------
 .../lite/toco/g3doc/cmdline_reference.md      | 119 ++----
 .../contrib/lite/toco/g3doc/python_api.md     |   7 +
 .../lite/toco/g3doc/toco_landscape.svg        |   1 +
 5 files changed, 258 insertions(+), 306 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/g3doc/toco_landscape.svg

diff --git a/tensorflow/contrib/lite/toco/README.md b/tensorflow/contrib/lite/toco/README.md
index 281b2ea5e4..522e260ad2 100644
--- a/tensorflow/contrib/lite/toco/README.md
+++ b/tensorflow/contrib/lite/toco/README.md
@@ -1,26 +1,27 @@
-# The TensorFlow Lite Optimizing Converter
+# TOCO: TensorFlow Lite Optimizing Converter
 
-The TensorFlow Lite Optimizing Converter's most typical use is converting from the TensorFlow GraphDef to the TensorFlow Lite
-format, but it supports much more than that.
+The TensorFlow Lite Optimizing Converter converts TensorFlow graphs into
+TensorFlow Lite graphs. There are additional usages that are also detailed in
+the usage documentation.
 
 ## Usage documentation
 
 Usage information is given in these documents:
 
+*   [Command-line glossary](g3doc/cmdline_reference.md)
 *   [Command-line examples](g3doc/cmdline_examples.md)
-*   [Command-line reference](g3doc/cmdline_reference.md)
-*   [Python API](g3doc/python_api.md)
-
-## Design documentation
-
-Coming soon!
+*   [Python API examples](g3doc/python_api.md)
 
 ## Where the converter fits in the TensorFlow landscape
 
-In the typical case, an application developer is using TensorFlow to design and
-train models, then uses TensorFlow's freeze_graph.py to generate a frozen
-inference graph, then uses the converter to convert that into a TensorFlow Lite flatbuffer file,
-then ships that file to client devices where the TensorFlow Lite interpreter handles them
-on-device. This is represented in the following diagram:
-
-![drawing](https://storage.googleapis.com/download.tensorflow.org/example_images/tensorflow_landscape.svg)
+Once an application developer has a trained TensorFlow model, TOCO will accept
+that model and generate a TensorFlow Lite
+[FlatBuffer](https://google.github.io/flatbuffers/) file. TOCO currently supports
+[SavedModels](https://www.tensorflow.org/programmers_guide/saved_model#using_savedmodel_with_estimators)
+and frozen graphs (models generated via
+[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)).
+The TensorFlow Lite FlatBuffer file can be shipped to client devices, generally
+mobile devices, where the TensorFlow Lite interpreter handles them on-device.
+This flow is represented in the diagram below.
+
+![drawing](g3doc/toco_landscape.svg)
diff --git a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
index 372c525589..495014c6fc 100644
--- a/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
+++ b/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md
@@ -1,73 +1,72 @@
 # TensorFlow Lite Optimizing Converter command-line examples
 
-This page is a guide to using the TensorFlow Lite Optimizing Converter by
-looking at some example command lines. It is complemented by the following other
-documents:
+This page provides examples on how to use TOCO via command line. It is
+complemented by the following documents:
 
 *   [README](../README.md)
-*   [Command-line reference](cmdline_reference.md)
+*   [Command-line glossary](cmdline_reference.md)
+*   [Python API examples](python_api.md)
 
 Table of contents:
 
-[TOC]
-
-## Convert a TensorFlow GraphDef to TensorFlow Lite for float inference
-
-In this example, we look at the most common task: we have an ordinary TensorFlow
-GraphDef and want to convert it to a TensorFlow Lite flatbuffer to perform
-floating-point inference.
+*   [Convert a TensorFlow SavedModel to TensorFlow Lite](#savedmodel)
+*   [Convert a TensorFlow GraphDef to TensorFlow Lite for float
+    inference](#graphdef-float)
+*   [Quantization](#quantization)
+    *   [Convert a TensorFlow GraphDef to TensorFlow Lite for quantized
+        inference](#graphdef-quant)
+    *   [Use "dummy-quantization" to try out quantized inference on a float
+        graph](#dummy-quant)
+*   [Specifying input and output arrays](#specifying-input-and-output-arrays)
+    *   [Multiple output arrays](#multiple-output-arrays)
+    *   [Multiple input arrays](#multiple-input-arrays)
+    *   [Specifying subgraphs](#specifying-subgraphs)
+*   [Other conversions supported by TOCO](#other-conversions)
+    *   [Optimize a TensorFlow GraphDef](#optimize-graphdef)
+    *   [Convert a TensorFlow Lite FlatBuffer back into TensorFlow GraphDef
+        format](#to-graphdef)
+*   [Logging](#logging)
+    *   [Standard logging](#standard-logging)
+    *   [Verbose logging](#verbose-logging)
+    *   [Graph "video" logging](#graph-video-logging)
+*   [Graph visualizations](#graph-visualizations)
+    *   [Using --output_format=GRAPHVIZ_DOT](#using-output-formatgraphviz-dot)
+    *   [Using --dump_graphviz](#using-dump-graphviz)
+    *   [Legend for the graph visualizations](#graphviz-legend)
+
+## Convert a TensorFlow SavedModel to TensorFlow Lite <a name="savedmodel"></a>
+
+The follow example converts a basic TensorFlow SavedModel into a Tensorflow Lite
+FlatBuffer to perform floating-point inference.
 
 ```
-curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
-  | tar xzv -C /tmp
 bazel run --config=opt \
-  //tensorflow/contrib/lite/toco:toco -- \
-  --input_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \
-  --output_file=/tmp/foo.tflite \
-  --input_format=TENSORFLOW_GRAPHDEF \
-  --output_format=TFLITE \
-  --inference_type=FLOAT \
-  --input_shape=1,128,128,3 \
-  --input_array=input \
-  --output_array=MobilenetV1/Predictions/Reshape_1
+  third_party/tensorflow/contrib/lite/toco:toco -- \
+  --savedmodel_directory=/tmp/saved_model \
+  --output_file=/tmp/foo.tflite
 ```
 
-To explain each of these flags:
-
-*   `--input_format` and `--output_format` determine the formats of the input
-    and output files: here we are converting from `TENSORFLOW_GRAPHDEF` to
-    `TFLITE`.
-*   `--input_file` specifies the path of the input file, to be converted. When
-    `--input_format=TENSORFLOW_GRAPHDEF`, this file should be a
-    *[frozen](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py)*
-    *inference* graph. Being frozen means in particular that the input file is
-    self-contained, and does not reference any external "checkpoint" file. An
-    *inference* graph is a version of a graph meant to be used for inference,
-    typically not the same graph file as was used for training a given model.
-*   `--output_file` specifies the destination to write the converted file to.
-*   `--input_array` specifies the input activations, that is, the input "tensor"
-    in the input TensorFlow GraphDef file. The array designated by
-    `--input_array` is the one that the user will have to provide the contents
-    of as input to the runtime inference code.
-*   `--output_array` specifies the output activations, that is, the output
-    "tensor" in the input TensorFlow GraphDef file. The runtime inference code
-    will store its results in the array designated by `--output_array`.
-*   `--input_shape` specifies the shape of the input array. It is currently
-    required, but the plan is for a future version to no longer require it,
-    allowing to defer the specification of the input shape until runtime. The
-    format of `input_shape` is always a comma-separated list of dimensions,
-    always in TensorFlow convention.
-*   `--inference_type` specifies what type of arithmetic the output file should
-    be relying on. It implies in particular the choice of type of the output
-    arrays in the output file.
-
-## Just optimize a TensorFlow GraphDef
+[SavedModel](https://www.tensorflow.org/programmers_guide/saved_model#using_savedmodel_with_estimators)
+has fewer required flags than frozen graphs (described [below](#graphdef-float))
+due to access to additional data contained within the SavedModel. The values for
+`--input_arrays` and `--output_arrays` are an aggregated, alphabetized list of
+the inputs and outputs in the
+[SignatureDefs](https://www.tensorflow.org/serving/signature_defs) within the
+[MetaGraphDef](https://www.tensorflow.org/programmers_guide/saved_model#apis_to_build_and_load_a_savedmodel)
+specified by `--savedmodel_tagset`. The value for `input_shapes` is
+automatically determined from the MetaGraphDef whenever possible. The default
+value for `--inference_type` for SavedModels is `FLOAT`.
 
-The converter accepts both TENSORFLOW_GRAPHDEF and TFLITE file formats as both
-`--input_format` and `--output_format`. This means that conversion from and to
-any supported format is possible, and in particular, same-format "conversions"
-are possible, and effectively ask the converter to optimize and simplify a
-graph. Example:
+There is currently no support for MetaGraphDefs without a SignatureDef or for
+MetaGraphDefs that use the [`assets/`
+directory](https://www.tensorflow.org/programmers_guide/saved_model#structure_of_a_savedmodel_directory).
+
+## Convert a TensorFlow GraphDef to TensorFlow Lite for float inference <a name="graphdef-float"></a>
+
+The follow example converts a basic TensorFlow GraphDef (frozen by
+[freeze_graph.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/python/tools/freeze_graph.py))
+into a TensorFlow Lite FlatBuffer to perform floating-point inference. Frozen
+graphs contain the variables stored in Checkpoint files as Const ops.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
@@ -75,56 +74,27 @@ curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_
 bazel run --config=opt \
   //tensorflow/contrib/lite/toco:toco -- \
   --input_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \
-  --output_file=/tmp/foo.pb \
-  --input_format=TENSORFLOW_GRAPHDEF \
-  --output_format=TENSORFLOW_GRAPHDEF \
+  --output_file=/tmp/foo.tflite \
+  --inference_type=FLOAT \
   --input_shape=1,128,128,3 \
   --input_array=input \
   --output_array=MobilenetV1/Predictions/Reshape_1
 ```
 
-Here we did not pass `--inference_type` because it is not considered applicable
-to the TensorFlow GraphDef format (as far as we are concerned, TensorFlow
-GraphDefs are technically always float, and the only flavor of "quantized"
-GraphDef that the converter deals with is "FakeQuantized" graphs that are still
-technically float graphs).
+## Quantization
 
-Below in the section about passing arbitrary input/output arrays we give another
-example, using the converter to extract just a sub-graph from a TensorFlow
-GraphDef.
+### Convert a TensorFlow GraphDef to TensorFlow Lite for quantized inference <a name="graphdef-quant"></a>
 
-## Convert a TensorFlow Lite flatbuffer back into TensorFlow GraphDef format
+TOCO is compatible with fixed point quantization models described
+[here](https://www.tensorflow.org/performance/quantization). These are float
+models with
+[`FakeQuant*`](https://www.tensorflow.org/api_guides/python/array_ops#Fake_quantization)
+ops inserted at the boundaries of fused layers to record min-max range
+information. This generates a quantized inference workload that reproduces the
+quantization behavior that was used during training.
 
-As we mentioned that the converter supports file format conversions in any
-direction, let us just give an example of that:
-
-```
-bazel run --config=opt \
-  //tensorflow/contrib/lite/toco:toco -- \
-  --input_file=/tmp/foo.tflite \
-  --output_file=/tmp/foo.pb \
-  --input_format=TFLITE \
-  --output_format=TENSORFLOW_GRAPHDEF \
-  --input_shape=1,128,128,3 \
-  --input_array=input \
-  --output_array=MobilenetV1/Predictions/Reshape_1
-```
-
-## Convert a TensorFlow GraphDef to TensorFlow Lite for quantized inference
-
-Let us now look at a quantized model. As mentioned above, the only flavor of
-quantized TensorFlow GraphDefs that the converter is concerned with, is
-"FakeQuantized" models. These are technically float models, but with special
-`FakeQuant*` ops inserted at the boundaries of fused layers to record min-max
-range information allowing to generate a quantized inference workload that is
-able to reproduce exactly the specific quantization behavior that was used
-during training. Indeed, the whole point of quantized training is to allow for
-both training and inference to perform exactly the same arithmetic, so that the
-way that the training process about around quantization inaccuracy is
-effectively helping the quantized inference process to be more accurate.
-
-Given a quantized TensorFlow GraphDef, generating a quantized TensorFlow Lite
-flatbuffer is done like this:
+The following command generates a quantized TensorFlow Lite FlatBuffer from a
+"quantized" TensorFlow GraphDef.
 
 ```
 bazel run --config=opt \
@@ -141,36 +111,17 @@ bazel run --config=opt \
   --std_value=127
 ```
 
-Here, besides changing `--input_file` to point to a (fake-)quantized GraphDef,
-the only other changes are:
-
-*   To change `--inference_type` to `QUANTIZED_UINT8`. This effectively tells
-    the converter to generate an output file that performs quantized inference
-    on a quantized input.
-*   To pass `--mean_value` and `--std_value` flags to describe how the quantized
-    uint8 input array values are to be interpreted as the mathematical real
-    numbers that the graph is concerned with (keep in mind that even a
-    "fake-quantized" TensorFlow GraphDef is still technically a float graph).
-    The meaning of `--mean_value` and `--std_value` is explained in the
-    command-line reference; it suffices for now to say that they are a property
-    of each model.
+### Use \"dummy-quantization\" to try out quantized inference on a float graph <a name="dummy-quant"></a>
 
-## Use dummy-quantization to try out quantized inference on a float graph
+In order to evaluate the possible benefit of generating a quantized graph, TOCO
+allows "dummy-quantization" on float graphs. The flags `--default_ranges_min`
+and `--default_ranges_max` accept plausable values for the min-max ranges of the
+values in all arrays that do not have min-max information. "Dummy-quantization"
+will produce lower accuracy but will emulate the performance of a correctly
+quantized model.
 
-Sometimes, one only has a plain float graph, and one is curious as to how much
-faster inference might run if one could perform quantized inference instead of
-float inference. Rather than requiring users to first invest in quantizing their
-graphs before they can evaluate a possible benefit, the converter allows to
-simply experiment with what we call "dummy quantization": provide some vaguely
-plausible values for the min-max ranges of values in all arrays that do not have
-min-max information, so that quantization can carry on, certainly producing
-inaccurate results (do not use that in production!) but with performance
-characteristics that should be identical to those of an actually quantized
-flavor of the model.
-
-In the present example, we have a model using Relu6 activation functions almost
-everywhere, so a reasonable guess is that most activation ranges should be
-contained in [0, 6] and roughly comparable to it.
+The example below contains a model using Relu6 activation functions. Therefore,
+a reasonable guess is that most activation ranges should be contained in [0, 6].
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
@@ -191,15 +142,13 @@ bazel run --config=opt \
   --std_value=127.5
 ```
 
-## Multiple output arrays
+## Specifying input and output arrays
 
-Some models have multiple outputs. Even in a model with only one output, you may
-want for the inference code to return the contents of other arrays as well, or
-to perform inference on a subgraph with multiple outputs (see the section below
-on specifying arbitrary arrays as input/output arrays).
+### Multiple output arrays
 
-Either way, using `--output_arrays` instead of `--output_array` allows to
-specify a comma-separated list of output arrays.
+The flag `output_arrays` takes in a comma-separated list of output arrays as
+seen in the example below. This is useful for models or subgraphs with multiple
+outputs.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \
@@ -216,18 +165,11 @@ bazel run --config=opt \
   --output_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu
 ```
 
-## Multiple input arrays
-
-Some models have multiple inputs; even in a model with a single input, you may
-want for the inference code to implement only a subgraph with multiple inputs
-(see the section below on specifying arbitrary arrays as input/output arrays).
+### Multiple input arrays
 
-Either way, multiple input arrays are specified by using `--input_arrays`
-instead of `--input_array` to specify a comma-separated list of input arrays. In
-that case, one also needs to use `--input_shapes` instead of `--input_shape`.
-The syntax for `--input_shapes` is a bit trickier, since already the singular
-`--input_shape` was a comma-separated list of integers! Multiple input shapes
-are delimited by a colon (`:`) in `--input_shapes`.
+The flag `input_arrays` takes in a comma-separated list of input arrays as seen
+in the example below. This is useful for models or subgraphs with multiple
+inputs.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \
@@ -244,54 +186,93 @@ bazel run --config=opt \
   --output_array=InceptionV1/Logits/Predictions/Reshape_1
 ```
 
-## Specifying arbitrary arrays in a graph as input or output arrays
+Note that `input_shapes` is provided as a colon-separated list. Each input shape
+corresponds to the input array at the same position in the respective list.
 
-Any array in the input file can be specified as an input or output array. This
-allows to use the converter to extract a sub-graph out of the input graph file.
-The converter then automatically discards any part of the graph that is not
-needed for the subgraph identified by the specified input and output arrays.
-Another use case for specifying multiple output arrays is to get inference code
-to return the contents of some specified intermediate activations array, not
-just the output activations.
+### Specifying subgraphs
 
-In order to know which array you want to pass as `--input_arrays` /
-`--output_arrays`, it helps to have a visualization of the graph. See the
-section below on graph visualization. When using graph visualization for that
-purpose, make sure to use `--dump_graphviz=` to visualize exactly the graph as
-it is in the actual final form being exported to the output file.
+Any array in the input file can be specified as an input or output array in
+order to extract subgraphs out of an input graph file. TOCO discards the parts
+of the graph outside of the specific subgraph. Use [graph
+visualizations](#graph-visualizations) to identify the input and output arrays
+that make up the desired subgraph.
+
+The follow command shows how to extract a single fused layer out of a TensorFlow
+GraphDef.
+
+```
+curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \
+  | tar xzv -C /tmp
+bazel run --config=opt \
+  //tensorflow/contrib/lite/toco:toco -- \
+  --input_file=/tmp/inception_v1_2016_08_28_frozen.pb \
+  --output_file=/tmp/foo.pb \
+  --input_format=TENSORFLOW_GRAPHDEF \
+  --output_format=TENSORFLOW_GRAPHDEF \
+  --input_shapes=1,28,28,96:1,28,28,16:1,28,28,192:1,28,28,64 \
+  --input_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_3/MaxPool_0a_3x3/MaxPool,InceptionV1/InceptionV1/Mixed_3b/Branch_0/Conv2d_0a_1x1/Relu \
+  --output_array=InceptionV1/InceptionV1/Mixed_3b/concat_v2
+```
 
 Note that the final representation of an on-device inference workload (say, in
-TensorFlow Lite flatbuffers format) tends to have coarser granularity than the
+TensorFlow Lite FlatBuffers format) tends to have coarser granularity than the
 very fine granularity of the TensorFlow GraphDef representation. For example,
 while a fully-connected layer is typically represented as at least four separate
 ops in TensorFlow GraphDef (Reshape, MatMul, BiasAdd, Relu...), it is typically
 represented as a single "fused" op (FullyConnected) in the converter's optimized
 representation and in the final on-device representation (e.g. in TensorFlow
-Lite flatbuffer format). As the level of granularity gets coarser, some
+Lite FlatBuffer format). As the level of granularity gets coarser, some
 intermediate arrays (say, the array between the MatMul and the BiasAdd in the
 TensorFlow GraphDef) are dropped. When specifying intermediate arrays as
-`--input_arrays` / `--output_arrays`, it is generally at least desirable (and
-often required) to specify arrays that are meant to survive in the final form of
-the graph, after fusing. These are typically the outputs of activation functions
-(since everything in each layer until the activation function tends to get
-fused).
+`--input_arrays` / `--output_arrays`, it is desirable (and often required) to
+specify arrays that are meant to survive in the final form of the graph, after
+fusing. These are typically the outputs of activation functions (since
+everything in each layer until the activation function tends to get fused).
+
+## Other conversions supported by TOCO <a name="other-conversions"></a>
+
+The converter accepts both TENSORFLOW_GRAPHDEF and TFLITE file formats as both
+`--input_format` and `--output_format`. This means that conversion to and from
+any supported format is possible.
 
-Here is an example of extracting just a sub-graph, namely just a single fused
-layer, out of a TensorFlow GraphDef, and exporting a TensorFlow GraphDef
-containing just that subgraph:
+### Optimize a TensorFlow GraphDef <a name="optimize-graphdef"></a>
+
+Same-format "conversions" can be used to optimize and simplify a graph or be
+used to [get a subgraph](#specifying-subgraphs) of a graph. The flag
+`--inference_type` is not required because TensorFlow graphs, including those
+containing the
+[`FakeQuant*`](https://www.tensorflow.org/api_guides/python/array_ops#Fake_quantization)
+ops are always float graphs.
 
 ```
-curl https://storage.googleapis.com/download.tensorflow.org/models/inception_v1_2016_08_28_frozen.pb.tar.gz \
+curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
   | tar xzv -C /tmp
 bazel run --config=opt \
   //tensorflow/contrib/lite/toco:toco -- \
-  --input_file=/tmp/inception_v1_2016_08_28_frozen.pb \
+  --input_file=/tmp/mobilenet_v1_0.50_128/frozen_graph.pb \
   --output_file=/tmp/foo.pb \
   --input_format=TENSORFLOW_GRAPHDEF \
   --output_format=TENSORFLOW_GRAPHDEF \
-  --input_shapes=1,28,28,96:1,28,28,16:1,28,28,192:1,28,28,64 \
-  --input_arrays=InceptionV1/InceptionV1/Mixed_3b/Branch_1/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_2/Conv2d_0a_1x1/Relu,InceptionV1/InceptionV1/Mixed_3b/Branch_3/MaxPool_0a_3x3/MaxPool,InceptionV1/InceptionV1/Mixed_3b/Branch_0/Conv2d_0a_1x1/Relu \
-  --output_array=InceptionV1/InceptionV1/Mixed_3b/concat_v2
+  --input_shape=1,128,128,3 \
+  --input_array=input \
+  --output_array=MobilenetV1/Predictions/Reshape_1
+```
+
+### Convert a TensorFlow Lite FlatBuffer back into TensorFlow GraphDef format <a name="to-graphdef"></a>
+
+The converter supports file format conversions from TensorFlow Lite, back into
+TensorFlow GraphDef format.
+
+```
+bazel run --config=opt \
+  //tensorflow/contrib/lite/toco:toco -- \
+  --input_file=/tmp/foo.tflite \
+  --output_file=/tmp/foo.pb \
+  --input_format=TFLITE \
+  --output_format=TENSORFLOW_GRAPHDEF \
+  --input_shape=1,128,128,3 \
+  --input_array=input \
+  --output_array=MobilenetV1/Predictions/Reshape_1
 ```
 
 ## Logging
@@ -299,8 +280,8 @@ bazel run --config=opt \
 ### Standard logging
 
 The converter generates some informative log messages during processing. The
-easiest way to view them is to add `--logtostderr` to command lines. For the
-previous example, that gives:
+easiest way to view them is to add `--logtostderr` to command lines as seen in
+the following example.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
@@ -333,42 +314,34 @@ I1101 21:51:33.309484    5339 toco_tooling.cc:249] Estimated count of arithmetic
 For debugging purposes, the converter supports two levels of verbose logging,
 which can be set by passing a `--v=` flag:
 
-*   At `--v=1`, the converter generates text dumps of the graph at various
-    points during processing, as well as log messages about every graph
-    transformation that did take place, typically answering questions of the
-    form "why was my graph transformed in this way"?
-*   At `--v=2`, the converter additionally generates log messages about graph
-    transformations that were considered but not actually performed, typically
-    answering questions of the form "why was my graph NOT transformed when I
-    expected it would be?".
+*   For `--v=1`, the converter generates text dumps of the graph at various
+    points during processing as well as log messages about every graph
+    transformation that took place.
+*   For `--v=2`, the converter additionally generates log messages about graph
+    transformations that were considered but not performed.
 
 ### Graph "video" logging
 
-When `--dump_graphviz=` is used (see the section on Graph visualizations), one
-may additionally pass `--dump_graphviz_video`, which causes a graph
-visualization to be dumped after each individual graph transformations, often
-resulting in thousands of files. Typically, one would then bisect into these
-files to understand when a given change was introduced in the graph.
+When `--dump_graphviz=` is used (see the section on [graph
+visualizations](#graph-visualizations)), one may additionally pass
+`--dump_graphviz_video`, which causes a graph visualization to be dumped after
+each individual graph transformation. This results in thousands of files.
+Typically, one would then bisect into these files to understand when a given
+change was introduced in the graph.
 
 ## Graph visualizations
 
-The converter is able to export a graph to the GraphViz Dot format, for easy
-visualization. Combined with the converter's ability to transform the graph into
-a simpler, coarser-granularity representation, that makes it a very powerful
-visualization tool.
-
-There are two ways to get the converter to export a GraphViz Dot file,
-corresponding to two separate use cases. Understanding the difference between
-them is key to getting useful graph visualizations.
+TOCO can export a graph to the GraphViz Dot format for easy visualization via
+either the `--output_format` flag or the `--dump_graphviz` flag. The subsections
+below outline the use cases for each.
 
 ### Using `--output_format=GRAPHVIZ_DOT`
 
-The first way to get a graphviz rendering is to pass
-`--output_format=GRAPHVIZ_DOT`, instead of the `--output_format` that you would
-otherwise use. This says: "I just want to get a plausible visualization of that
-graph". The upside is that it makes for very simple command lines, and makes the
-converter very lax about aspects of the graph or the command line that it would
-otherwise complain about. Example:
+The first way to get a graphviz rendering is to pass `GRAPHVIZ_DOT` into
+`--output_format`. This results in a plausable visualization of the graph. This
+reduces the requirements that normally exist during conversion between other
+input and output formats. For example, this may be useful if conversion from
+TENSORFLOW_GRAPHDEF to TFLITE is failing.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
@@ -391,7 +364,7 @@ dot -Tpdf -O /tmp/foo.dot
 ```
 
 And the resulting `.dot.pdf` can be viewed in any PDF viewer, but we suggest one
-with a good ability to pan and zoom across a very large page; Google Chrome does
+with a good ability to pan and zoom across a very large page. Google Chrome does
 well in that respect.
 
 ```
@@ -400,14 +373,14 @@ google-chrome /tmp/foo.dot.pdf
 
 Example PDF files are viewable online in the next section.
 
-### Using `--dump_graphviz=`
+### Using `--dump_graphviz`
 
-The second way to get a graphviz rendering is to pass a `--dump_graphviz=` flag
-specifying a destination directory to dump GraphViz rendering to. Unlike the
-previous approach, this one allows you to keep your real command-line (with your
-real `--output_format` and other flags) unchanged, just appending a
-`--dump_graphviz=` flag to it. This says: "I want visualizations of the actual
-graph during this specific conversion process". Example:
+The second way to get a graphviz rendering is to pass the `--dump_graphviz=`
+flag, specifying a destination directory to dump GraphViz rendering to. Unlike
+the previous approach, this one allows you to keep your real command-line (with
+your real `--output_format` and other flags) unchanged, just appending a
+`--dump_graphviz=` flag to it. This provides a visualization of the actual graph
+during a specific conversion process.
 
 ```
 curl https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_0.50_128_frozen.tgz \
@@ -425,8 +398,8 @@ bazel run --config=opt \
   --dump_graphviz=/tmp
 ```
 
-This generates a few files in the destination directory, here `/tmp`. Most
-important are these two files:
+This generates a few files in the destination directory, here `/tmp`. The two
+most important files are:
 
 ```
 /tmp/toco_AT_IMPORT.dot
@@ -442,8 +415,7 @@ conversion subsequently fails).
 
 `toco_AFTER_TRANSFORMATIONS.dot` represents the graph after all transformations
 were applied to it, just before it was exported to the `--output_file`.
-Typically, this is a much smaller graph, and it conveys much more information
-about each node.
+Typically, this is a much smaller graph with more information about each node.
 
 Again, these can be rendered to PDFs:
 
@@ -451,12 +423,12 @@ Again, these can be rendered to PDFs:
 dot -Tpdf -O /tmp/toco_*.dot
 ```
 
-The resulting files can be seen here:
+Sample output files can be seen here:
 
 *   [toco_AT_IMPORT.dot.pdf](https://storage.googleapis.com/download.tensorflow.org/example_images/toco_AT_IMPORT.dot.pdf)
 *   [toco_AFTER_TRANSFORMATIONS.dot.pdf](https://storage.googleapis.com/download.tensorflow.org/example_images/toco_AFTER_TRANSFORMATIONS.dot.pdf).
 
-### Legend for the graph visualizations
+### Legend for the graph visualizations <a name="graphviz-legend"></a>
 
 *   Operators are red square boxes with the following hues of red:
     *   Most operators are
diff --git a/tensorflow/contrib/lite/toco/g3doc/cmdline_reference.md b/tensorflow/contrib/lite/toco/g3doc/cmdline_reference.md
index 5e07795223..9e99287f82 100644
--- a/tensorflow/contrib/lite/toco/g3doc/cmdline_reference.md
+++ b/tensorflow/contrib/lite/toco/g3doc/cmdline_reference.md
@@ -1,84 +1,47 @@
-# TensorFlow Lite Optimizing Converter command-line reference
+# TensorFlow Lite Optimizing Converter command-line glossary
 
 This page is complete reference of command-line flags. It is complemented by the
 following other documents:
 
 *   [README](../README.md)
 *   [Command-line examples](cmdline_examples.md)
+*   [Python API examples](python_api.md)
 
 Table of contents:
 
-[TOC]
-
-## High-level overview
-
-A full list and detailed specification of all flags is given in the next
-section. For now we focus on a higher-level description of command lines:
-
-```
-toco \
-  --input_format=... \
-  --output_format=... \
-  --input_file=... \
-  --output_file=... \
-  [model flags...] \
-  [transformation flags...] \
-  [logging flags...]
-```
-
-In other words, the converter requires at least the following mandatory flags:
-`--input_format`, `--output_format`, `--input_file`, `--output_file`. Depending
-on the input and output formats, additional flags may be allowed or mandatory:
-
-*   *Model flags* provide additional information about the model stored in the
-    input file.
-    *   `--output_array` or `--output_arrays` specify which arrays in the input
-        file are to be considered the output activations.
-    *   `--input_array` or `--input_arrays` specify which arrays in the input
-        file are to be considered the input activations.
-    *   `--input_shape` or `--input_shapes` specify the shapes of the input
-        arrays.
-    *   `--input_data_type` or `--input_data_types` specify the data types of
-        input arrays, which can be used if the input file does not already
-        specify them.
-    *   `--mean_value` or `--mean_values`, and `--std_value` or `--std_values`,
-        give the dequantization parameters of the input arrays, for the case
-        when the output file will accept quantized input arrays.
-*   *Transformation flags* specify options of the transformations to be applied
-    to the graph, i.e. they specify requested properties that the output file
-    should have.
-    *   `--inference_type` specifies the type of real-numbers arrays in the
-        output file. This only affects arrays of real numbers and allows to
-        control their quantization or dequantization, effectively switching
-        between floating-point and quantized arithmetic for the inference
-        workload, as far as real numbers are concerned. Other data types are
-        unaffected (e.g. plain integers, and strings).
-    *   `--inference_input_type` is like `--inference_type` but specifically
-        controlling input arrays, separately from other arrays. If not
-        specified, then `--inference_type` is used. The use case for specifying
-        `--inference_input_type` is when one wants to perform floating-point
-        inference on a quantized input, as is common in image models operating
-        on bitmap image inputs.
-    *   Some transformation flags allow to carry on with quantization when the
-        input graph is not properly quantized: `--default_ranges_min`,
-        `--default_ranges_max`, `--drop_fake_quant`,
-        `--reorder_across_fake_quant`.
-*   *Logging flags* described below.
-
-## Command-line flags complete reference
-
-### Mandatory flags
-
-*   `--input_format`. Type: string. Specifies the format of the input file.
-    Allowed values:
+*   [High-level flags](#high-level-flags)
+*   [Model flags](#model-flags)
+*   [Transformation flags](#transformation-flags)
+*   [Logging flags](#logging-flags)
+
+## High-level flags
+
+The following high level flags specify the location of the input and output
+files. The flag `--output_file` is always required. Additionally, either
+`--input_file` or `--savedmodel_directory` is required.
+
+*   `--savedmodel_directory`. Type: string. Specifies the full path to the
+    directory containing the SavedModel.
+*   `--savedmodel_tagset`. Type: string. Default:
+    [kSavedModelTagServe](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/cc/saved_model/tag_constants.h).
+    Specifies a comma-separated set of tags identifying the MetaGraphDef within
+    the SavedModel to analyze. All tags in the tag set must be specified.
+*   `--input_file`. Type: string. Specifies the path of the input file. This may
+    be either an absolute or a relative path.
+*   `--output_file`. Type: string. Specifies the path of the output file.
+
+The following high level flags specify the types of the input and output files:
+
+*   `--input_format`. Type: string. Default: `TENSORFLOW_GRAPHDEF`. Specifies
+    the format of the input file. Allowed values:
     *   `TENSORFLOW_GRAPHDEF` &mdash; The TensorFlow GraphDef format. Both
         binary and text proto formats are allowed.
-    *   `TFLITE` &mdash; The TensorFlow Lite flatbuffers format.
-*   `--output_format`. Type: string. Specifies the format of the output file.
-    Allowed values:
+    *   `TFLITE` &mdash; The TensorFlow Lite FlatBuffers format.
+*   `--output_format`. Type: string. Default: `TFLITE`. Specifies the format of
+    the output file. Allowed values:
     *   `TENSORFLOW_GRAPHDEF` &mdash; The TensorFlow GraphDef format. Always
         produces a file in binary (not text) proto format.
-    *   `TFLITE` &mdash; The TensorFlow Lite flatbuffers format.
+    *   `TFLITE` &mdash; The TensorFlow Lite FlatBuffers format.
         *   Whether a float or quantized TensorFlow Lite file will be produced
             depends on the `--inference_type` flag.
     *   `GRAPHVIZ_DOT` &mdash; The GraphViz `.dot` format. This asks the
@@ -95,11 +58,11 @@ on the input and output formats, additional flags may be allowed or mandatory:
             you get in your actual output format as opposed to just a merely
             plausible visualization of a model, consider using `--dump_graphviz`
             instead and keeping your true `--output_format`.
-*   `--input_file`. Type: string. Specifies the path of the input file. This may
-    be either an absolute or a relative path.
-*   `--output_file`. Type: string. Specifies the path of the output file.
 
-### Model flags
+## Model flags
+
+*Model flags* provide additional information about the model stored in the input
+file.
 
 *   `--output_array`. Type: string. Specifies a single array as the output
     activations. Incompatible with `--output_arrays`.
@@ -111,6 +74,10 @@ on the input and output formats, additional flags may be allowed or mandatory:
 *   `--input_arrays`. Type: comma-separated list of strings. Specifies a list of
     arrays as the input activations, for models with multiple inputs.
     Incompatible with `--input_array`.
+*   `--batch_size`. Type: integer. Default: 1. Specifies the batch size for the
+    model. Replaces the first dimension of an input size array if undefined. Use
+    only with SavedModels when neither `--input_shape` nor `input_shapes` flags
+    are specified. Incompatible with GraphDefs.
 
 When `--input_array` is used, the following flags are available to provide
 additional information about the single input array:
@@ -160,7 +127,11 @@ additional information about the multiple input arrays:
     the input arrays specified in `--input_arrays`, in the same order. See
     `--mean_value`, `--std_value` for details.
 
-### Transformation flags
+## Transformation flags
+
+*Transformation flags* specify options of the transformations to be applied to
+the graph, i.e. they specify requested properties that the output file should
+have.
 
 *   `--inference_type`. Type: string. Sets the type of real-number arrays in the
     output file, that is, controls the representation (quantization) of real
@@ -232,7 +203,7 @@ additional information about the multiple input arrays:
     graph transformations on them, at the cost of no longer faithfully matching
     inference and training arithmetic.
 
-### Logging flags
+## Logging flags
 
 The following are standard Google logging flags:
 
diff --git a/tensorflow/contrib/lite/toco/g3doc/python_api.md b/tensorflow/contrib/lite/toco/g3doc/python_api.md
index 36e2d9c372..f0fd638a61 100644
--- a/tensorflow/contrib/lite/toco/g3doc/python_api.md
+++ b/tensorflow/contrib/lite/toco/g3doc/python_api.md
@@ -1,5 +1,12 @@
 # TensorFlow Lite Optimizing Converter (TOCO) Python API reference
 
+This page provides examples on how to use TOCO via the Python API. It is
+complemented by the following documents:
+
+*   [README](../README.md)
+*   [Command-line examples](cmdline_examples.md)
+*   [Command-line glossary](cmdline_reference.md)
+
 ## High-level overview
 
 While the TensorFlow Lite Optimizing Converter can be used from the command
diff --git a/tensorflow/contrib/lite/toco/g3doc/toco_landscape.svg b/tensorflow/contrib/lite/toco/g3doc/toco_landscape.svg
new file mode 100644
index 0000000000..a47c088991
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/g3doc/toco_landscape.svg
@@ -0,0 +1 @@
+<svg version="1.1" viewBox="0.0 0.0 720.0 540.0" fill="none" stroke="none" stroke-linecap="square" stroke-miterlimit="10" xmlns:xlink="http://www.w3.org/1999/xlink" xmlns="http://www.w3.org/2000/svg"><clipPath id="p.0"><path d="m0 0l720.0 0l0 540.0l-720.0 0l0 -540.0z" clip-rule="nonzero"/></clipPath><g clip-path="url(#p.0)"><path fill="#000000" fill-opacity="0.0" d="m0 0l720.0 0l0 540.0l-720.0 0z" fill-rule="evenodd"/><path fill="#f3f3f3" d="m19.375328 28.750656l361.6378 0l0 358.01575l-361.6378 0z" fill-rule="evenodd"/><path stroke="#cccccc" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m19.375328 28.750656l361.6378 0l0 358.01575l-361.6378 0z" fill-rule="evenodd"/><path fill="#434343" d="m338.49512 374.66016q-0.609375 0 -1.171875 -0.140625q-0.546875 -0.15625 -0.96875 -0.421875q-0.25 -0.15625 -0.359375 -0.296875q-0.09375 -0.140625 -0.09375 -0.34375q0 -0.171875 0.09375 -0.28125q0.109375 -0.109375 0.265625 -0.109375q0.171875 0 0.46875 0.1875q0.40625 0.25 0.796875 0.390625q0.390625 0.140625 0.984375 0.140625q0.71875 0 1.109375 -0.25q0.40625 -0.265625 0.40625 -0.734375q0 -0.296875 -0.15625 -0.46875q-0.140625 -0.1875 -0.5 -0.328125q-0.359375 -0.140625 -1.046875 -0.296875q-1.171875 -0.25 -1.6875 -0.671875q-0.5 -0.421875 -0.5 -1.15625q0 -0.578125 0.3125 -1.015625q0.328125 -0.4375 0.890625 -0.6875q0.5625 -0.265625 1.28125 -0.265625q0.53125 0 1.015625 0.140625q0.484375 0.140625 0.859375 0.390625q0.453125 0.328125 0.453125 0.671875q0 0.171875 -0.109375 0.296875q-0.109375 0.125 -0.25 0.125q-0.15625 0 -0.484375 -0.234375q-0.375 -0.234375 -0.703125 -0.359375q-0.328125 -0.140625 -0.828125 -0.140625q-0.625 0 -1.015625 0.28125q-0.375 0.265625 -0.375 0.734375q0 0.296875 0.140625 0.484375q0.140625 0.171875 0.46875 0.3125q0.328125 0.140625 0.9375 0.28125q0.90625 0.1875 1.40625 0.4375q0.5 0.234375 0.703125 0.578125q0.21875 0.34375 0.21875 0.890625q0 0.828125 -0.703125 1.34375q-0.703125 0.515625 -1.859375 0.515625zm9.241241 -1.59375q0.140625 0 0.25 0.125q0.109375 0.109375 0.109375 0.296875q0 0.328125 -0.46875 0.609375q-0.484375 0.28125 -1.015625 0.421875q-0.53125 0.140625 -1.046875 0.140625q-1.5 0 -2.375 -0.890625q-0.875 -0.890625 -0.875 -2.46875q0 -1.0 0.390625 -1.765625q0.390625 -0.765625 1.078125 -1.1875q0.703125 -0.4375 1.59375 -0.4375q1.265625 0 2.015625 0.828125q0.75 0.828125 0.75 2.25q0 0.265625 -0.109375 0.390625q-0.109375 0.109375 -0.34375 0.109375l-4.296875 0q0.125 2.296875 2.171875 2.296875q0.53125 0 0.890625 -0.140625q0.375 -0.140625 0.8125 -0.390625q0.34375 -0.1875 0.46875 -0.1875zm-2.34375 -4.3125q-0.84375 0 -1.359375 0.53125q-0.515625 0.53125 -0.609375 1.515625l3.765625 0q-0.015625 -1.0 -0.484375 -1.515625q-0.46875 -0.53125 -1.3125 -0.53125zm7.5551147 -0.8125q0.546875 -0.03125 0.546875 0.453125q0 0.21875 -0.125 0.34375q-0.109375 0.125 -0.40625 0.15625l-0.390625 0.03125q-0.890625 0.078125 -1.328125 0.640625q-0.4375 0.546875 -0.4375 1.296875l0 3.234375q0 0.265625 -0.15625 0.40625q-0.140625 0.125 -0.375 0.125q-0.234375 0 -0.390625 -0.140625q-0.15625 -0.140625 -0.15625 -0.390625l0 -5.625q0 -0.25 0.15625 -0.390625q0.15625 -0.140625 0.390625 -0.140625q0.21875 0 0.359375 0.140625q0.140625 0.140625 0.140625 0.375l0 0.75q0.28125 -0.578125 0.796875 -0.890625q0.515625 -0.3125 1.1875 -0.359375l0.1875 -0.015625zm6.157959 0.328125q0.15625 -0.3125 0.46875 -0.3125q0.203125 0 0.359375 0.140625q0.15625 0.125 0.15625 0.328125q0 0.109375 -0.046875 0.203125l-2.59375 5.609375q-0.078125 0.171875 -0.25 0.28125q-0.15625 0.09375 -0.34375 0.09375q-0.171875 0 -0.328125 -0.09375q-0.15625 -0.109375 -0.25 -0.28125l-2.59375 -5.609375q-0.046875 -0.09375 -0.046875 -0.1875q0 -0.203125 0.171875 -0.34375q0.1875 -0.15625 0.390625 -0.15625q0.140625 0 0.265625 0.078125q0.125 0.078125 0.1875 0.234375l2.234375 5.0l2.21875 -4.984375zm7.2099915 4.796875q0.140625 0 0.25 0.125q0.109375 0.109375 0.109375 0.296875q0 0.328125 -0.46875 0.609375q-0.484375 0.28125 -1.015625 0.421875q-0.53125 0.140625 -1.046875 0.140625q-1.5 0 -2.375 -0.890625q-0.875 -0.890625 -0.875 -2.46875q0 -1.0 0.390625 -1.765625q0.390625 -0.765625 1.078125 -1.1875q0.703125 -0.4375 1.59375 -0.4375q1.265625 0 2.015625 0.828125q0.75 0.828125 0.75 2.25q0 0.265625 -0.109375 0.390625q-0.109375 0.109375 -0.34375 0.109375l-4.296875 0q0.125 2.296875 2.171875 2.296875q0.53125 0 0.890625 -0.140625q0.375 -0.140625 0.8125 -0.390625q0.34375 -0.1875 0.46875 -0.1875zm-2.34375 -4.3125q-0.84375 0 -1.359375 0.53125q-0.515625 0.53125 -0.609375 1.515625l3.765625 0q-0.015625 -1.0 -0.484375 -1.515625q-0.46875 -0.53125 -1.3125 -0.53125zm7.5551453 -0.8125q0.546875 -0.03125 0.546875 0.453125q0 0.21875 -0.125 0.34375q-0.109375 0.125 -0.40625 0.15625l-0.390625 0.03125q-0.890625 0.078125 -1.328125 0.640625q-0.4375 0.546875 -0.4375 1.296875l0 3.234375q0 0.265625 -0.15625 0.40625q-0.140625 0.125 -0.375 0.125q-0.234375 0 -0.390625 -0.140625q-0.15625 -0.140625 -0.15625 -0.390625l0 -5.625q0 -0.25 0.15625 -0.390625q0.15625 -0.140625 0.390625 -0.140625q0.21875 0 0.359375 0.140625q0.140625 0.140625 0.140625 0.375l0 0.75q0.28125 -0.578125 0.796875 -0.890625q0.515625 -0.3125 1.1875 -0.359375l0.1875 -0.015625z" fill-rule="nonzero"/><path fill="#d9d9d9" d="m25.624672 36.249344l301.88977 0l0 69.98425l-301.88977 0z" fill-rule="evenodd"/><path stroke="#cccccc" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m25.624672 36.249344l301.88977 0l0 69.98425l-301.88977 0z" fill-rule="evenodd"/><path fill="#434343" d="m134.36497 56.831844q-0.234375 0 -0.375 -0.140625q-0.140625 -0.140625 -0.140625 -0.359375l0 -7.1875l-2.578125 0q-0.21875 0 -0.34375 -0.109375q-0.109375 -0.109375 -0.109375 -0.3125q0 -0.203125 0.109375 -0.296875q0.125 -0.109375 0.34375 -0.109375l6.15625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-2.578125 0l0 7.1875q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.34375 0.140625zm9.004181 -1.421875q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.839676 -0.75q2.09375 0 2.09375 2.3125l0 3.25q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.1875q0 -0.8125 -0.328125 -1.1875q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.328125l0 0.609375q0.28125 -0.53125 0.796875 -0.8125q0.53125 -0.28125 1.1875 -0.28125zm5.84729 6.0625q-0.56248474 0 -1.0624847 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.87498474 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0624847 -0.234375 -1.5156097 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.1562347 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.56248474 0 -0.90623474 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84373474 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125zm6.2131653 0q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm7.1288147 -5.25q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm1.970398 6.03125q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.375 0q0.203125 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.125 0.109375 -0.328125 0.109375l-3.90625 0l0 2.90625l3.65625 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.65625 0l0 3.453125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625zm6.5434265 0q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.359375 -0.125q0.203125 0 0.34375 0.125q0.140625 0.125 0.140625 0.34375l0 7.625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125zm4.721527 0.015625q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm12.222534 -4.9375q0.125 -0.28125 0.390625 -0.28125q0.1875 0 0.328125 0.125q0.140625 0.109375 0.140625 0.296875q0 0.078125 -0.03125 0.171875l-1.984375 5.046875q-0.078125 0.15625 -0.21875 0.25q-0.140625 0.078125 -0.296875 0.078125q-0.15625 0 -0.296875 -0.078125q-0.140625 -0.09375 -0.21875 -0.25l-1.65625 -4.21875l-1.640625 4.21875q-0.0625 0.15625 -0.203125 0.25q-0.140625 0.078125 -0.3125 0.078125q-0.15625 0 -0.296875 -0.078125q-0.140625 -0.09375 -0.21875 -0.25l-1.984375 -5.03125q-0.046875 -0.09375 -0.046875 -0.171875q0 -0.1875 0.15625 -0.3125q0.171875 -0.140625 0.359375 -0.140625q0.296875 0 0.40625 0.296875l1.65625 4.421875l1.6875 -4.390625q0.078125 -0.15625 0.203125 -0.234375q0.125 -0.09375 0.265625 -0.09375q0.15625 0 0.28125 0.09375q0.125 0.078125 0.1875 0.234375l1.6875 4.375l1.65625 -4.40625zm12.637604 5.09375q0.046875 0.09375 0.046875 0.203125q0 0.171875 -0.140625 0.296875q-0.140625 0.125 -0.328125 0.125q-0.296875 0 -0.421875 -0.296875l-0.84375 -1.9375l-4.53125 0l-0.859375 1.9375q-0.125 0.296875 -0.421875 0.296875q-0.1875 0 -0.34375 -0.125q-0.140625 -0.125 -0.140625 -0.3125q0 -0.09375 0.046875 -0.1875l3.4375 -7.640625q0.078125 -0.15625 0.21875 -0.234375q0.140625 -0.09375 0.3125 -0.09375q0.171875 0 0.3125 0.09375q0.15625 0.078125 0.21875 0.234375l3.4375 7.640625zm-5.859375 -2.421875l3.8125 0l-1.90625 -4.3125l-1.90625 4.3125zm7.78656 3.046875q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.84375 0q1.328125 0 2.0625 0.65625q0.75 0.640625 0.75 1.828125q0 1.1875 -0.75 1.84375q-0.734375 0.65625 -2.0625 0.65625l-2.359375 0l0 3.03125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm2.765625 -4.34375q1.9375 0 1.9375 -1.6875q0 -1.671875 -1.9375 -1.671875l-2.265625 0l0 3.359375l2.265625 0zm4.9744263 4.34375q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 7.578125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm4.4157715 0.015625q-0.5625 0 -1.0625 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.875 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0625 -0.234375 -1.515625 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.15625 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.5625 0 -0.90625 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84375 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125z" fill-rule="nonzero"/><path fill="#f3f3f3" d="m396.75067 183.75066l249.00787 0l0 203.02364l-249.00787 0z" fill-rule="evenodd"/><path stroke="#cccccc" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m396.75067 183.75066l249.00787 0l0 203.02364l-249.00787 0z" fill-rule="evenodd"/><path fill="#434343" d="m409.42255 374.66803q-0.90625 0 -1.609375 -0.40625q-0.6875 -0.421875 -1.078125 -1.171875q-0.375 -0.765625 -0.375 -1.765625q0 -1.0 0.390625 -1.765625q0.40625 -0.78125 1.109375 -1.203125q0.703125 -0.4375 1.625 -0.4375q0.5 0 1.0 0.140625q0.5 0.140625 0.875 0.40625q0.234375 0.171875 0.328125 0.328125q0.109375 0.140625 0.109375 0.328125q0 0.1875 -0.109375 0.3125q-0.09375 0.109375 -0.25 0.109375q-0.09375 0 -0.203125 -0.046875q-0.09375 -0.046875 -0.171875 -0.09375q-0.078125 -0.0625 -0.09375 -0.078125q-0.359375 -0.234375 -0.671875 -0.359375q-0.3125 -0.140625 -0.765625 -0.140625q-0.96875 0 -1.515625 0.671875q-0.53125 0.65625 -0.53125 1.828125q0 1.171875 0.53125 1.8125q0.546875 0.640625 1.515625 0.640625q0.453125 0 0.78125 -0.125q0.328125 -0.140625 0.65625 -0.375q0.15625 -0.09375 0.28125 -0.15625q0.140625 -0.0625 0.234375 -0.0625q0.140625 0 0.234375 0.125q0.109375 0.109375 0.109375 0.296875q0 0.171875 -0.09375 0.3125q-0.09375 0.140625 -0.34375 0.3125q-0.375 0.25 -0.90625 0.40625q-0.515625 0.15625 -1.0625 0.15625zm4.2591553 -0.03125q-0.234375 0 -0.390625 -0.140625q-0.15625 -0.140625 -0.15625 -0.390625l0 -8.46875q0 -0.25 0.15625 -0.390625q0.15625 -0.140625 0.390625 -0.140625q0.21875 0 0.375 0.140625q0.15625 0.140625 0.15625 0.390625l0 8.46875q0 0.25 -0.15625 0.390625q-0.15625 0.140625 -0.375 0.140625zm3.092102 0q-0.234375 0 -0.390625 -0.140625q-0.15625 -0.140625 -0.15625 -0.390625l0 -5.625q0 -0.25 0.15625 -0.390625q0.15625 -0.140625 0.390625 -0.140625q0.234375 0 0.375 0.140625q0.15625 0.140625 0.15625 0.390625l0 5.625q0 0.265625 -0.15625 0.40625q-0.140625 0.125 -0.375 0.125zm0 -8.09375q-0.3125 0 -0.515625 -0.171875q-0.203125 -0.1875 -0.203125 -0.5q0 -0.296875 0.203125 -0.484375q0.203125 -0.1875 0.515625 -0.1875q0.328125 0 0.515625 0.1875q0.203125 0.1875 0.203125 0.484375q0 0.3125 -0.203125 0.5q-0.1875 0.171875 -0.515625 0.171875zm7.5765076 6.53125q0.140625 0 0.25 0.125q0.109375 0.109375 0.109375 0.296875q0 0.328125 -0.46875 0.609375q-0.484375 0.28125 -1.015625 0.421875q-0.53125 0.140625 -1.046875 0.140625q-1.5 0 -2.375 -0.890625q-0.875 -0.890625 -0.875 -2.46875q0 -1.0 0.390625 -1.765625q0.390625 -0.765625 1.078125 -1.1875q0.703125 -0.4375 1.59375 -0.4375q1.265625 0 2.015625 0.828125q0.75 0.828125 0.75 2.25q0 0.265625 -0.109375 0.390625q-0.109375 0.109375 -0.34375 0.109375l-4.296875 0q0.125 2.296875 2.171875 2.296875q0.53125 0 0.890625 -0.140625q0.375 -0.140625 0.8125 -0.390625q0.34375 -0.1875 0.46875 -0.1875zm-2.34375 -4.3125q-0.84375 0 -1.359375 0.53125q-0.515625 0.53125 -0.609375 1.515625l3.765625 0q-0.015625 -1.0 -0.484375 -1.515625q-0.46875 -0.53125 -1.3125 -0.53125zm7.6020203 -0.84375q2.328125 0 2.328125 2.578125l0 3.609375q0 0.25 -0.140625 0.390625q-0.140625 0.140625 -0.390625 0.140625q-0.25 0 -0.40625 -0.140625q-0.140625 -0.140625 -0.140625 -0.390625l0 -3.546875q0 -0.90625 -0.359375 -1.3125q-0.34375 -0.421875 -1.125 -0.421875q-0.890625 0 -1.421875 0.546875q-0.53125 0.546875 -0.53125 1.484375l0 3.25q0 0.25 -0.140625 0.390625q-0.140625 0.140625 -0.390625 0.140625q-0.25 0 -0.40625 -0.140625q-0.140625 -0.140625 -0.140625 -0.390625l0 -5.625q0 -0.234375 0.140625 -0.375q0.15625 -0.15625 0.40625 -0.15625q0.234375 0 0.375 0.15625q0.140625 0.140625 0.140625 0.359375l0 0.6875q0.328125 -0.609375 0.890625 -0.921875q0.578125 -0.3125 1.3125 -0.3125zm7.304718 5.875q0.46875 0.03125 0.46875 0.421875q0 0.21875 -0.171875 0.34375q-0.171875 0.109375 -0.5 0.078125l-0.359375 -0.015625q-1.0625 -0.09375 -1.578125 -0.640625q-0.5 -0.5625 -0.5 -1.703125l0 -3.34375l-0.890625 0q-0.234375 0 -0.359375 -0.109375q-0.125 -0.109375 -0.125 -0.296875q0 -0.203125 0.125 -0.3125q0.125 -0.125 0.359375 -0.125l0.890625 0l0 -1.515625q0 -0.25 0.140625 -0.390625q0.15625 -0.140625 0.40625 -0.140625q0.234375 0 0.375 0.140625q0.15625 0.140625 0.15625 0.390625l0 1.515625l1.484375 0q0.203125 0 0.328125 0.125q0.140625 0.109375 0.140625 0.3125q0 0.1875 -0.140625 0.296875q-0.125 0.109375 -0.328125 0.109375l-1.484375 0l0 3.40625q0 0.734375 0.296875 1.0625q0.296875 0.3125 0.90625 0.359375l0.359375 0.03125z" fill-rule="nonzero"/><path fill="#f4cccc" d="m206.61942 201.17455l140.47244 0l0 30.992126l-140.47244 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m206.61942 201.17455l140.47244 0l0 30.992126l-140.47244 0z" fill-rule="evenodd"/><path fill="#000000" d="m237.0857 213.5031q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125zm4.248535 1.71875q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm5.861023 4.609375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm8.417801 3.875q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm8.199051 4.46875q0.203125 0 0.296875 0.109375q0.109375 0.09375 0.109375 0.265625q0 0.1875 -0.109375 0.296875q-0.09375 0.09375 -0.296875 0.09375l-4.203125 0q-0.203125 0 -0.34375 -0.125q-0.125 -0.125 -0.125 -0.3125q0 -0.1875 0.140625 -0.359375l3.546875 -4.28125l-3.28125 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l4.0625 0q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.3125q0 0.1875 -0.140625 0.359375l-3.5625 4.28125l3.421875 0zm6.2547913 -0.59375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm3.3865662 5.875q-0.171875 0 -0.28125 -0.09375q-0.109375 -0.09375 -0.109375 -0.21875q0 -0.140625 0.109375 -0.234375q0.109375 -0.09375 0.28125 -0.09375l5.21875 0q0.171875 0 0.28125 0.09375q0.109375 0.09375 0.109375 0.234375q0 0.125 -0.109375 0.21875q-0.109375 0.09375 -0.28125 0.09375l-5.21875 0zm11.2500305 -6.609375q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 5.09375q0 1.296875 -0.671875 1.96875q-0.671875 0.671875 -1.984375 0.671875q-1.28125 0 -2.140625 -0.515625q-0.421875 -0.234375 -0.421875 -0.546875q0 -0.171875 0.078125 -0.28125q0.09375 -0.109375 0.234375 -0.109375q0.125 0 0.4375 0.171875q0.421875 0.21875 0.828125 0.34375q0.40625 0.140625 0.96875 0.140625q0.859375 0 1.28125 -0.453125q0.4375 -0.453125 0.4375 -1.3125l0 -1.03125q-0.25 0.5625 -0.78125 0.859375q-0.515625 0.296875 -1.21875 0.296875q-0.765625 0 -1.359375 -0.359375q-0.59375 -0.359375 -0.9375 -1.015625q-0.328125 -0.65625 -0.328125 -1.515625q0 -0.875 0.328125 -1.53125q0.34375 -0.65625 0.9375 -1.015625q0.59375 -0.359375 1.359375 -0.359375q0.6875 0 1.203125 0.296875q0.515625 0.296875 0.78125 0.84375l0 -0.640625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625zm-2.28125 4.984375q0.84375 0 1.3125 -0.546875q0.484375 -0.5625 0.484375 -1.546875q0 -0.984375 -0.46875 -1.53125q-0.46875 -0.5625 -1.328125 -0.5625q-0.84375 0 -1.34375 0.5625q-0.484375 0.546875 -0.484375 1.53125q0 0.984375 0.484375 1.546875q0.5 0.546875 1.34375 0.546875zm7.4695435 -4.984375q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.720398 -0.015625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.3131714 -5.296875q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm7.20282 -5.265625q1.03125 0 1.546875 0.578125q0.53125 0.578125 0.53125 1.734375l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.78125 -0.328125 -1.15625q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.203125 0.125 -0.328125q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.125q0.125 0.125 0.125 0.34375l0 3.140625q0.28125 -0.53125 0.796875 -0.796875q0.515625 -0.28125 1.1875 -0.28125zm4.331665 6.046875q-0.28125 0 -0.484375 -0.1875q-0.1875 -0.1875 -0.1875 -0.484375q0 -0.296875 0.1875 -0.484375q0.203125 -0.203125 0.484375 -0.203125q0.28125 0 0.46875 0.203125q0.1875 0.1875 0.1875 0.484375q0 0.296875 -0.1875 0.484375q-0.1875 0.1875 -0.46875 0.1875zm5.2167664 -6.046875q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm8.45282 -4.9375q0.140625 -0.296875 0.421875 -0.296875q0.1875 0 0.328125 0.125q0.140625 0.109375 0.140625 0.296875q0 0.109375 -0.046875 0.1875l-3.375 7.28125q-0.0625 0.125 -0.171875 0.1875q-0.109375 0.078125 -0.234375 0.078125q-0.1875 0 -0.328125 -0.109375q-0.125 -0.109375 -0.125 -0.296875q0 -0.09375 0.046875 -0.1875l0.84375 -1.8125l-2.375 -5.140625q-0.046875 -0.078125 -0.046875 -0.171875q0 -0.1875 0.15625 -0.3125q0.15625 -0.140625 0.359375 -0.140625q0.109375 0 0.21875 0.078125q0.125 0.078125 0.1875 0.203125l2.0 4.5l2.0 -4.46875z" fill-rule="nonzero"/><path fill="#f4cccc" d="m154.36745 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m154.36745 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m184.89111 339.47687q-0.234375 0 -0.375 -0.140625q-0.140625 -0.140625 -0.140625 -0.359375l0 -7.1875l-2.578125 0q-0.21875 0 -0.34375 -0.109375q-0.109375 -0.109375 -0.109375 -0.3125q0 -0.203125 0.109375 -0.296875q0.125 -0.109375 0.34375 -0.109375l6.15625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-2.578125 0l0 7.1875q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.34375 0.140625zm8.160431 0.03125q-1.171875 0 -2.046875 -0.515625q-0.859375 -0.53125 -1.328125 -1.5q-0.46875 -0.984375 -0.46875 -2.296875q0 -1.34375 0.453125 -2.3125q0.46875 -0.984375 1.328125 -1.5q0.875 -0.53125 2.0625 -0.53125q1.1875 0 2.0625 0.53125q0.875 0.515625 1.328125 1.5q0.46875 0.96875 0.46875 2.296875q0 1.3125 -0.46875 2.296875q-0.46875 0.984375 -1.34375 1.515625q-0.859375 0.515625 -2.046875 0.515625zm0 -0.84375q1.34375 0 2.09375 -0.90625q0.75 -0.90625 0.75 -2.578125q0 -1.6875 -0.75 -2.578125q-0.734375 -0.90625 -2.09375 -0.90625q-1.34375 0 -2.09375 0.90625q-0.75 0.90625 -0.75 2.578125q0 1.671875 0.75 2.578125q0.75 0.90625 2.09375 0.90625zm9.214935 0.84375q-1.1875 0 -2.0625 -0.515625q-0.875 -0.53125 -1.359375 -1.5q-0.46875 -0.984375 -0.46875 -2.3125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.359375 -1.5q0.875 -0.53125 2.0625 -0.53125q0.8125 0 1.515625 0.265625q0.71875 0.25 1.25 0.734375q0.1875 0.1875 0.1875 0.421875q0 0.171875 -0.09375 0.296875q-0.09375 0.125 -0.21875 0.125q-0.15625 0 -0.359375 -0.140625q-0.609375 -0.46875 -1.109375 -0.65625q-0.5 -0.203125 -1.140625 -0.203125q-1.390625 0 -2.140625 0.90625q-0.75 0.90625 -0.75 2.578125q0 1.671875 0.75 2.578125q0.75 0.90625 2.140625 0.90625q0.640625 0 1.140625 -0.1875q0.5 -0.1875 1.109375 -0.671875q0.203125 -0.125 0.359375 -0.125q0.125 0 0.21875 0.125q0.09375 0.109375 0.09375 0.296875q0 0.234375 -0.1875 0.40625q-0.53125 0.484375 -1.25 0.75q-0.703125 0.25 -1.515625 0.25zm8.077179 0q-1.171875 0 -2.046875 -0.515625q-0.859375 -0.53125 -1.328125 -1.5q-0.46875 -0.984375 -0.46875 -2.296875q0 -1.34375 0.453125 -2.3125q0.46875 -0.984375 1.328125 -1.5q0.875 -0.53125 2.0625 -0.53125q1.1875 0 2.0625 0.53125q0.875 0.515625 1.328125 1.5q0.46875 0.96875 0.46875 2.296875q0 1.3125 -0.46875 2.296875q-0.46875 0.984375 -1.34375 1.515625q-0.859375 0.515625 -2.046875 0.515625zm0 -0.84375q1.34375 0 2.09375 -0.90625q0.75 -0.90625 0.75 -2.578125q0 -1.6875 -0.75 -2.578125q-0.734375 -0.90625 -2.09375 -0.90625q-1.34375 0 -2.09375 0.90625q-0.75 0.90625 -0.75 2.578125q0 1.671875 0.75 2.578125q0.75 0.90625 2.09375 0.90625z" fill-rule="nonzero"/><path fill="#d9ead3" d="m284.12296 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m284.12296 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m314.7006 332.47687q-0.234375 0 -0.375 -0.140625q-0.140625 -0.140625 -0.140625 -0.359375l0 -7.1875l-2.578125 0q-0.21875 0 -0.34375 -0.109375q-0.109375 -0.109375 -0.109375 -0.3125q0 -0.203125 0.109375 -0.296875q0.125 -0.109375 0.34375 -0.109375l6.15625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-2.578125 0l0 7.1875q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.34375 0.140625zm5.113556 0q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.375 0q0.203125 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.125 0.109375 -0.328125 0.109375l-3.90625 0l0 2.90625l3.65625 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.65625 0l0 3.453125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625zm6.6840515 -0.0625q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.328125l0 -7.5625q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 7.171875l3.875 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-4.375 0zm6.3394165 0.0625q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm4.987152 6.515625q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm5.9081726 -0.65625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m303.37402 346.47687q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.375 0q0.203125 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.125 0.109375 -0.328125 0.109375l-3.90625 0l0 2.90625l3.65625 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.65625 0l0 3.453125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625zm6.5434265 0q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.359375 -0.125q0.203125 0 0.34375 0.125q0.140625 0.125 0.140625 0.34375l0 7.625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125zm4.674652 -6.046875q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.0631714 -0.015625q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm4.3300476 -5.28125q0.765625 0 1.34375 0.375q0.59375 0.359375 0.921875 1.046875q0.328125 0.6875 0.328125 1.59375q0 0.90625 -0.328125 1.59375q-0.328125 0.6875 -0.921875 1.078125q-0.578125 0.375 -1.34375 0.375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 0.640625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -7.625q0 -0.203125 0.125 -0.328125q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.125q0.125 0.125 0.125 0.34375l0 3.203125q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.59375q0.46875 -0.59375 0.46875 -1.65625q0 -1.046875 -0.46875 -1.625q-0.46875 -0.578125 -1.328125 -0.578125q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm8.687164 -5.25q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 5.078125q0 0.203125 -0.125 0.34375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.34375 -0.125q-0.125 -0.125 -0.125 -0.328125l0 -0.609375q-0.28125 0.53125 -0.78125 0.8125q-0.5 0.265625 -1.125 0.265625q-1.03125 0 -1.5625 -0.578125q-0.53125 -0.578125 -0.53125 -1.71875l0 -3.265625q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 3.234375q0 0.78125 0.3125 1.15625q0.3125 0.359375 0.984375 0.359375q0.765625 0 1.234375 -0.5q0.46875 -0.5 0.46875 -1.3125l0 -2.9375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625zm4.8726807 -1.71875q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125zm3.9360352 0q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125zm5.873535 6.328125q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.7927856 -0.734375q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625z" fill-rule="nonzero"/><path fill="#f4cccc" d="m413.02625 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m413.02625 319.3983l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m443.6039 332.47687q-0.234375 0 -0.375 -0.140625q-0.140625 -0.140625 -0.140625 -0.359375l0 -7.1875l-2.578125 0q-0.21875 0 -0.34375 -0.109375q-0.109375 -0.109375 -0.109375 -0.3125q0 -0.203125 0.109375 -0.296875q0.125 -0.109375 0.34375 -0.109375l6.15625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-2.578125 0l0 7.1875q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.34375 0.140625zm5.113556 0q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.375 0q0.203125 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.125 0.109375 -0.328125 0.109375l-3.90625 0l0 2.90625l3.65625 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.65625 0l0 3.453125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625zm6.6840515 -0.0625q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.328125l0 -7.5625q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 7.171875l3.875 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-4.375 0zm6.3394165 0.0625q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm4.987152 6.515625q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm5.908142 -0.65625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375z" fill-rule="nonzero"/><path fill="#000000" d="m429.9527 346.47687q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm5.237152 1.234375q2.09375 0 2.09375 2.3125l0 3.25q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.1875q0 -0.8125 -0.328125 -1.1875q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.328125l0 0.609375q0.28125 -0.53125 0.796875 -0.8125q0.53125 -0.28125 1.1875 -0.28125zm6.56604 5.28125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm5.9081726 -0.65625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.7927856 -0.734375q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm4.282898 -0.015625q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm7.14032 -5.25q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm5.861023 4.609375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.5896606 4.53125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm5.9081726 -0.65625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.7927856 -0.734375q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m371.61902 334.89435l41.417297 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m371.61902 334.89435l37.990234 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m409.60925 334.89435l-1.1245728 1.1246033l3.0897522 -1.1246033l-3.0897522 -1.1245728z" fill-rule="evenodd"/><path fill="#c9daf8" d="m548.5407 277.52954l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m548.5407 277.52954l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path fill="#000000" d="m587.0588 293.13934q0.1875 0 0.296875 0.109375q0.109375 0.109375 0.109375 0.296875l0 2.984375q0 0.296875 -0.09375 0.4375q-0.078125 0.140625 -0.328125 0.234375q-0.46875 0.203125 -1.15625 0.328125q-0.6875 0.109375 -1.375 0.109375q-1.25 0 -2.171875 -0.515625q-0.90625 -0.515625 -1.390625 -1.484375q-0.484375 -0.96875 -0.484375 -2.328125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.375 -1.5q0.90625 -0.53125 2.125 -0.53125q0.84375 0 1.5625 0.265625q0.71875 0.25 1.203125 0.734375q0.21875 0.203125 0.21875 0.421875q0 0.171875 -0.109375 0.296875q-0.09375 0.125 -0.234375 0.125q-0.140625 0 -0.328125 -0.140625q-0.625 -0.484375 -1.140625 -0.671875q-0.5 -0.1875 -1.15625 -0.1875q-1.4375 0 -2.203125 0.90625q-0.75 0.890625 -0.75 2.578125q0 1.71875 0.765625 2.609375q0.78125 0.890625 2.28125 0.890625q1.109375 0 2.03125 -0.328125l0 -2.578125l-1.75 0q-0.203125 0 -0.328125 -0.109375q-0.125 -0.109375 -0.125 -0.265625q0 -0.1875 0.125 -0.28125q0.125 -0.109375 0.328125 -0.109375l2.234375 0zm2.8911743 4.46875q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.84375 0q1.328125 0 2.0625 0.65625q0.75 0.640625 0.75 1.828125q0 1.1875 -0.75 1.84375q-0.734375 0.65625 -2.0625 0.65625l-2.359375 0l0 3.03125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm2.765625 -4.34375q1.9375 0 1.9375 -1.6875q0 -1.671875 -1.9375 -1.671875l-2.265625 0l0 3.359375l2.265625 0zm7.7869263 4.375q-1.65625 0 -2.515625 -0.859375q-0.84375 -0.859375 -0.84375 -2.546875l0 -4.703125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.359375l0 4.78125q0 1.25 0.609375 1.875q0.609375 0.609375 1.78125 0.609375q1.171875 0 1.765625 -0.609375q0.609375 -0.625 0.609375 -1.875l0 -4.78125q0 -0.234375 0.140625 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.359375l0 4.703125q0 1.671875 -0.859375 2.546875q-0.859375 0.859375 -2.5 0.859375z" fill-rule="nonzero"/><path fill="#c9daf8" d="m548.5407 319.3983l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m548.5407 319.3983l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path fill="#000000" d="m584.63763 339.50812q-1.1875 0 -2.0625 -0.515625q-0.875 -0.53125 -1.359375 -1.5q-0.46875 -0.984375 -0.46875 -2.3125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.359375 -1.5q0.875 -0.53125 2.0625 -0.53125q0.8125 0 1.515625 0.265625q0.71875 0.25 1.25 0.734375q0.1875 0.1875 0.1875 0.421875q0 0.171875 -0.09375 0.296875q-0.09375 0.125 -0.21875 0.125q-0.15625 0 -0.359375 -0.140625q-0.609375 -0.46875 -1.109375 -0.65625q-0.5 -0.203125 -1.140625 -0.203125q-1.390625 0 -2.140625 0.90625q-0.75 0.90625 -0.75 2.578125q0 1.671875 0.75 2.578125q0.75 0.90625 2.140625 0.90625q0.640625 0 1.140625 -0.1875q0.5 -0.1875 1.109375 -0.671875q0.203125 -0.125 0.359375 -0.125q0.125 0 0.21875 0.125q0.09375 0.109375 0.09375 0.296875q0 0.234375 -0.1875 0.40625q-0.53125 0.484375 -1.25 0.75q-0.703125 0.25 -1.515625 0.25zm5.0302734 -0.03125q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.84375 0q1.328125 0 2.0625 0.65625q0.75 0.640625 0.75 1.828125q0 1.1875 -0.75 1.84375q-0.734375 0.65625 -2.0625 0.65625l-2.359375 0l0 3.03125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm2.765625 -4.34375q1.9375 0 1.9375 -1.6875q0 -1.671875 -1.9375 -1.671875l-2.265625 0l0 3.359375l2.265625 0zm7.7869263 4.375q-1.65625 0 -2.515625 -0.859375q-0.84375 -0.859375 -0.84375 -2.546875l0 -4.703125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.359375l0 4.78125q0 1.25 0.609375 1.875q0.609375 0.609375 1.78125 0.609375q1.171875 0 1.765625 -0.609375q0.609375 -0.625 0.609375 -1.875l0 -4.78125q0 -0.234375 0.140625 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.359375l0 4.703125q0 1.671875 -0.859375 2.546875q-0.859375 0.859375 -2.5 0.859375z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m241.86351 334.89435l42.267715 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m241.86351 334.89435l38.840652 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m280.70413 334.89435l-1.1245728 1.1246033l3.0897827 -1.1246033l-3.0897827 -1.1245728z" fill-rule="evenodd"/><path fill="#d9ead3" d="m413.02625 141.28871l20.53543 0l0 20.53543l-20.53543 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m413.02625 141.28871l20.53543 0l0 20.53543l-20.53543 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m437.52493 135.68242l73.763794 0l0 31.748032l-73.763794 0z" fill-rule="evenodd"/><path fill="#000000" d="m448.0718 156.20241q-0.234375 0 -0.375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -7.5q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.34375 0q2.03125 0 3.140625 1.09375q1.109375 1.09375 1.109375 3.125q0 2.03125 -1.125 3.140625q-1.109375 1.09375 -3.125 1.09375l-2.34375 0zm2.28125 -0.84375q3.28125 0 3.28125 -3.390625q0 -3.390625 -3.28125 -3.390625l-1.796875 0l0 6.78125l1.796875 0zm8.3211975 -5.140625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.0631714 -0.015625q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm3.767517 -5.28125q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm10.15921 0.75q-0.234375 0 -0.375 -0.140625q-0.140625 -0.140625 -0.140625 -0.359375l0 -7.1875l-2.578125 0q-0.21875 0 -0.34375 -0.109375q-0.109375 -0.109375 -0.109375 -0.3125q0 -0.203125 0.109375 -0.296875q0.125 -0.109375 0.34375 -0.109375l6.15625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.109375 0.109375 -0.328125 0.109375l-2.578125 0l0 7.1875q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.34375 0.140625zm8.691681 -5.71875q0.140625 -0.296875 0.421875 -0.296875q0.1875 0 0.328125 0.125q0.140625 0.109375 0.140625 0.296875q0 0.109375 -0.046875 0.1875l-3.375 7.28125q-0.0625 0.125 -0.171875 0.1875q-0.109375 0.078125 -0.234375 0.078125q-0.1875 0 -0.328125 -0.109375q-0.125 -0.109375 -0.125 -0.296875q0 -0.09375 0.046875 -0.1875l0.84375 -1.8125l-2.375 -5.140625q-0.046875 -0.078125 -0.046875 -0.171875q0 -0.1875 0.15625 -0.3125q0.15625 -0.140625 0.359375 -0.140625q0.109375 0 0.21875 0.078125q0.125 0.078125 0.1875 0.203125l2.0 4.5l2.0 -4.46875zm4.902405 -0.328125q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm8.76532 -0.640625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375z" fill-rule="nonzero"/><path fill="#f4cccc" d="m519.9029 141.28871l20.5354 0l0 20.53543l-20.5354 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m519.9029 141.28871l20.5354 0l0 20.53543l-20.5354 0z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m544.40155 135.68242l100.0 0l0 31.748032l-100.0 0z" fill-rule="evenodd"/><path fill="#000000" d="m554.9328 156.26491q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 7.578125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm5.3845215 -6.046875q2.09375 0 2.09375 2.3125l0 3.25q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.1875q0 -0.8125 -0.328125 -1.1875q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.328125l0 0.609375q0.28125 -0.53125 0.796875 -0.8125q0.53125 -0.28125 1.1875 -0.28125zm6.456726 -1.703125q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125zm4.248535 1.71875q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.720398 -0.015625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm6.3444214 0.765625q-0.5625 0 -1.0625 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.875 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0625 -0.234375 -1.515625 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.15625 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.5625 0 -0.90625 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84375 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125zm6.47876 -0.78125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm4.283142 -5.265625q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm5.782898 0q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 5.078125q0 0.203125 -0.125 0.34375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.34375 -0.125q-0.125 -0.125 -0.125 -0.328125l0 -0.609375q-0.28125 0.53125 -0.78125 0.8125q-0.5 0.265625 -1.125 0.265625q-1.03125 0 -1.5625 -0.578125q-0.53125 -0.578125 -0.53125 -1.71875l0 -3.265625q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 3.234375q0 0.78125 0.3125 1.15625q0.3125 0.359375 0.984375 0.359375q0.765625 0 1.234375 -0.5q0.46875 -0.5 0.46875 -1.3125l0 -2.9375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625zm4.7008057 6.046875q-0.8125 0 -1.453125 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.6875 -0.34375 -1.578125q0 -0.90625 0.359375 -1.59375q0.359375 -0.703125 0.984375 -1.078125q0.640625 -0.390625 1.46875 -0.390625q0.453125 0 0.90625 0.125q0.453125 0.125 0.78125 0.359375q0.21875 0.140625 0.3125 0.28125q0.09375 0.140625 0.09375 0.3125q0 0.171875 -0.09375 0.28125q-0.09375 0.09375 -0.234375 0.09375q-0.078125 0 -0.1875 -0.046875q-0.09375 -0.046875 -0.15625 -0.09375q-0.0625 -0.046875 -0.09375 -0.0625q-0.3125 -0.203125 -0.59375 -0.3125q-0.28125 -0.125 -0.6875 -0.125q-0.875 0 -1.359375 0.59375q-0.484375 0.59375 -0.484375 1.65625q0 1.046875 0.484375 1.625q0.484375 0.578125 1.359375 0.578125q0.40625 0 0.703125 -0.109375q0.296875 -0.125 0.59375 -0.328125q0.140625 -0.09375 0.25 -0.15625q0.125 -0.0625 0.203125 -0.0625q0.140625 0 0.21875 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.15625 -0.09375 0.28125q-0.078125 0.125 -0.296875 0.28125q-0.34375 0.234375 -0.8125 0.375q-0.46875 0.125 -0.953125 0.125zm6.029297 -0.78125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm5.830017 -5.265625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 5.078125q0 0.203125 -0.125 0.34375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.34375 -0.125q-0.125 -0.125 -0.125 -0.328125l0 -0.609375q-0.28125 0.53125 -0.78125 0.8125q-0.5 0.265625 -1.125 0.265625q-1.03125 0 -1.5625 -0.578125q-0.53125 -0.578125 -0.53125 -1.71875l0 -3.265625q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.34375l0 3.234375q0 0.78125 0.3125 1.15625q0.3125 0.359375 0.984375 0.359375q0.765625 0 1.234375 -0.5q0.46875 -0.5 0.46875 -1.3125l0 -2.9375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625zm5.1851807 0q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm5.861023 4.609375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375z" fill-rule="nonzero"/><path fill="#d9ead3" d="m78.872284 252.53609l87.49606 0l0 30.992142l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m78.872284 252.53609l87.49606 0l0 30.992142l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m93.328064 272.6459q-0.90625 0 -1.734375 -0.265625q-0.8125 -0.265625 -1.3125 -0.734375q-0.171875 -0.15625 -0.171875 -0.40625q0 -0.171875 0.09375 -0.296875q0.09375 -0.125 0.234375 -0.125q0.15625 0 0.328125 0.125q1.109375 0.859375 2.546875 0.859375q1.03125 0 1.578125 -0.390625q0.5625 -0.390625 0.5625 -1.125q0 -0.421875 -0.265625 -0.671875q-0.265625 -0.265625 -0.703125 -0.421875q-0.4375 -0.15625 -1.15625 -0.328125q-0.984375 -0.21875 -1.625 -0.46875q-0.625 -0.265625 -1.015625 -0.734375q-0.390625 -0.46875 -0.390625 -1.21875q0 -0.71875 0.390625 -1.265625q0.390625 -0.5625 1.09375 -0.875q0.703125 -0.3125 1.59375 -0.3125q0.84375 0 1.5625 0.265625q0.734375 0.25 1.234375 0.734375q0.1875 0.1875 0.1875 0.421875q0 0.171875 -0.09375 0.296875q-0.09375 0.125 -0.234375 0.125q-0.125 0 -0.34375 -0.140625q-0.59375 -0.46875 -1.09375 -0.65625q-0.5 -0.203125 -1.21875 -0.203125q-0.984375 0 -1.546875 0.421875q-0.546875 0.40625 -0.546875 1.15625q0 0.625 0.484375 0.953125q0.484375 0.3125 1.5 0.5625q1.09375 0.25 1.71875 0.484375q0.625 0.21875 1.03125 0.671875q0.421875 0.4375 0.421875 1.171875q0 0.71875 -0.390625 1.265625q-0.390625 0.53125 -1.109375 0.828125q-0.703125 0.296875 -1.609375 0.296875zm6.9353027 -6.078125q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm8.578796 -4.96875q0.140625 -0.296875 0.421875 -0.296875q0.1875 0 0.328125 0.125q0.140625 0.109375 0.140625 0.296875q0 0.109375 -0.046875 0.1875l-2.34375 5.046875q-0.0625 0.15625 -0.21875 0.25q-0.140625 0.078125 -0.3125 0.078125q-0.15625 0 -0.296875 -0.078125q-0.140625 -0.09375 -0.21875 -0.25l-2.328125 -5.046875q-0.046875 -0.078125 -0.046875 -0.171875q0 -0.1875 0.15625 -0.3125q0.15625 -0.140625 0.359375 -0.140625q0.109375 0 0.21875 0.078125q0.125 0.078125 0.1875 0.203125l2.0 4.5l2.0 -4.46875zm6.480545 4.296875q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm8.589676 -3.28125q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.328125l0 7.625q0 0.21875 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.640625q-0.265625 0.546875 -0.78125 0.84375q-0.5 0.296875 -1.1875 0.296875q-0.765625 0 -1.359375 -0.375q-0.578125 -0.390625 -0.90625 -1.078125q-0.328125 -0.6875 -0.328125 -1.59375q0 -0.90625 0.328125 -1.59375q0.328125 -0.6875 0.90625 -1.046875q0.59375 -0.375 1.359375 -0.375q0.6875 0 1.1875 0.296875q0.515625 0.296875 0.78125 0.84375l0 -3.203125q0 -0.21875 0.125 -0.34375q0.125 -0.125 0.359375 -0.125zm-2.25 7.796875q0.84375 0 1.296875 -0.578125q0.46875 -0.59375 0.46875 -1.65625q0 -1.0625 -0.46875 -1.640625q-0.453125 -0.578125 -1.296875 -0.578125q-0.859375 0 -1.34375 0.578125q-0.46875 0.578125 -0.46875 1.625q0 1.0625 0.46875 1.65625q0.484375 0.59375 1.34375 0.59375zm12.202805 -7.796875q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.359375l0 7.59375q0 0.21875 -0.125 0.359375q-0.109375 0.125 -0.328125 0.125q-0.21875 0 -0.328125 -0.125q-0.109375 -0.140625 -0.109375 -0.359375l0 -6.125l-2.59375 4.984375q-0.171875 0.34375 -0.5 0.34375q-0.3125 0 -0.484375 -0.34375l-2.625 -4.921875l0 6.0625q0 0.21875 -0.109375 0.359375q-0.109375 0.125 -0.328125 0.125q-0.21875 0 -0.34375 -0.125q-0.109375 -0.140625 -0.109375 -0.359375l0 -7.59375q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.3125 0 0.484375 0.34375l3.046875 5.84375l3.015625 -5.84375q0.09375 -0.1875 0.203125 -0.265625q0.125 -0.078125 0.28125 -0.078125zm4.8576965 8.59375q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm8.925674 -7.796875q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.328125l0 7.625q0 0.21875 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.640625q-0.265625 0.546875 -0.78125 0.84375q-0.5 0.296875 -1.1875 0.296875q-0.765625 0 -1.359375 -0.375q-0.578125 -0.390625 -0.90625 -1.078125q-0.328125 -0.6875 -0.328125 -1.59375q0 -0.90625 0.328125 -1.59375q0.328125 -0.6875 0.90625 -1.046875q0.59375 -0.375 1.359375 -0.375q0.6875 0 1.1875 0.296875q0.515625 0.296875 0.78125 0.84375l0 -3.203125q0 -0.21875 0.125 -0.34375q0.125 -0.125 0.359375 -0.125zm-2.25 7.796875q0.84375 0 1.296875 -0.578125q0.46875 -0.59375 0.46875 -1.65625q0 -1.0625 -0.46875 -1.640625q-0.453125 -0.578125 -1.296875 -0.578125q-0.859375 0 -1.34375 0.578125q-0.46875 0.578125 -0.46875 1.625q0 1.0625 0.46875 1.65625q0.484375 0.59375 1.34375 0.59375zm9.06218 -0.640625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm4.386551 5.296875q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.359375 -0.125q0.203125 0 0.34375 0.125q0.140625 0.125 0.140625 0.34375l0 7.625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125z" fill-rule="nonzero"/><path fill="#d9ead3" d="m190.14 134.76706l87.49608 0l0 30.992126l-87.49608 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m190.14 134.76706l87.49608 0l0 30.992126l-87.49608 0z" fill-rule="evenodd"/><path fill="#000000" d="m215.10997 150.37688q0.1875 0 0.296875 0.109375q0.109375 0.109375 0.109375 0.296875l0 2.984375q0 0.296875 -0.09375 0.4375q-0.078125 0.140625 -0.328125 0.234375q-0.46875 0.203125 -1.15625 0.328125q-0.6875 0.109375 -1.375 0.109375q-1.25 0 -2.171875 -0.515625q-0.90625 -0.515625 -1.390625 -1.484375q-0.484375 -0.96875 -0.484375 -2.328125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.375 -1.5q0.90625 -0.53125 2.125 -0.53125q0.84375 0 1.5625 0.265625q0.71875 0.25 1.203125 0.734375q0.21875 0.203125 0.21875 0.421875q0 0.171875 -0.109375 0.296875q-0.09375 0.125 -0.234375 0.125q-0.140625 0 -0.328125 -0.140625q-0.625 -0.484375 -1.140625 -0.671875q-0.5 -0.1875 -1.15625 -0.1875q-1.4375 0 -2.203125 0.90625q-0.75 0.890625 -0.75 2.578125q0 1.71875 0.765625 2.609375q0.78125 0.890625 2.28125 0.890625q1.109375 0 2.03125 -0.328125l0 -2.578125l-1.75 0q-0.203125 0 -0.328125 -0.109375q-0.125 -0.109375 -0.125 -0.265625q0 -0.1875 0.125 -0.28125q0.125 -0.109375 0.328125 -0.109375l2.234375 0zm5.1568146 -1.5625q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.720398 -0.015625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.3131714 -5.296875q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm7.2028046 -5.265625q1.03125 0 1.546875 0.578125q0.53125 0.578125 0.53125 1.734375l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.78125 -0.328125 -1.15625q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.203125 0.125 -0.328125q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.125q0.125 0.125 0.125 0.34375l0 3.140625q0.28125 -0.53125 0.796875 -0.796875q0.515625 -0.28125 1.1875 -0.28125zm4.5035553 5.984375q-0.234375 0 -0.375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -7.5q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.34375 0q2.03125 0 3.140625 1.09375q1.109375 1.09375 1.109375 3.125q0 2.03125 -1.125 3.140625q-1.109375 1.09375 -3.125 1.09375l-2.34375 0zm2.28125 -0.84375q3.28125 0 3.28125 -3.390625q0 -3.390625 -3.28125 -3.390625l-1.796875 0l0 6.78125l1.796875 0zm10.461807 -0.515625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.480301 -2.453125q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125z" fill-rule="nonzero"/><path fill="#d9ead3" d="m233.1085 252.53609l87.49608 0l0 30.992142l-87.49608 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m233.1085 252.53609l87.49608 0l0 30.992142l-87.49608 0z" fill-rule="evenodd"/><path fill="#000000" d="m260.00964 265.61465q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.375 0q0.203125 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.203125 -0.125 0.3125q-0.125 0.109375 -0.328125 0.109375l-3.90625 0l0 2.90625l3.65625 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.3125q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.65625 0l0 3.453125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625zm8.9496765 -6.03125q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.767273 6.046875q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm8.535065 -0.046875q0.203125 0 0.296875 0.109375q0.109375 0.09375 0.109375 0.265625q0 0.1875 -0.109375 0.296875q-0.09375 0.09375 -0.296875 0.09375l-4.203125 0q-0.203125 0 -0.34375 -0.125q-0.125 -0.125 -0.125 -0.3125q0 -0.1875 0.140625 -0.359375l3.546875 -4.28125l-3.28125 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l4.0625 0q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.3125q0 0.1875 -0.140625 0.359375l-3.5625 4.28125l3.421875 0zm6.2547913 -0.59375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.8396606 -0.75q2.09375 0 2.09375 2.3125l0 3.25q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.1875q0 -0.8125 -0.328125 -1.1875q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.328125l0 0.609375q0.28125 -0.53125 0.796875 -0.8125q0.53125 -0.28125 1.1875 -0.28125z" fill-rule="nonzero"/><path fill="#000000" d="m258.07846 275.1459q0.1875 0 0.296875 0.109375q0.109375 0.109375 0.109375 0.296875l0 2.984375q0 0.296875 -0.09375 0.4375q-0.078125 0.140625 -0.328125 0.234375q-0.46875 0.203125 -1.15625 0.328125q-0.6875 0.109375 -1.3749847 0.109375q-1.25 0 -2.171875 -0.515625q-0.90625 -0.515625 -1.390625 -1.484375q-0.484375 -0.96875 -0.484375 -2.328125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.375 -1.5q0.90625 -0.53125 2.125 -0.53125q0.84373474 0 1.5624847 0.265625q0.71875 0.25 1.203125 0.734375q0.21875 0.203125 0.21875 0.421875q0 0.171875 -0.109375 0.296875q-0.09375 0.125 -0.234375 0.125q-0.140625 0 -0.328125 -0.140625q-0.625 -0.484375 -1.140625 -0.671875q-0.5 -0.1875 -1.1562347 -0.1875q-1.4375 0 -2.203125 0.90625q-0.75 0.890625 -0.75 2.578125q0 1.71875 0.765625 2.609375q0.78125 0.890625 2.28125 0.890625q1.1093597 0 2.0312347 -0.328125l0 -2.578125l-1.7499847 0q-0.203125 0 -0.328125 -0.109375q-0.125 -0.109375 -0.125 -0.265625q0 -0.1875 0.125 -0.28125q0.125 -0.109375 0.328125 -0.109375l2.2343597 0zm5.15683 -1.5625q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.720398 -0.015625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.3131714 -5.296875q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm7.2027893 -5.265625q1.03125 0 1.546875 0.578125q0.53125 0.578125 0.53125 1.734375l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.78125 -0.328125 -1.15625q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.203125 0.125 -0.328125q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.125q0.125 0.125 0.125 0.34375l0 3.140625q0.28125 -0.53125 0.796875 -0.796875q0.515625 -0.28125 1.1875 -0.28125zm4.5035706 5.984375q-0.234375 0 -0.375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -7.5q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.34375 0q2.03125 0 3.140625 1.09375q1.109375 1.09375 1.109375 3.125q0 2.03125 -1.125 3.140625q-1.109375 1.09375 -3.125 1.09375l-2.34375 0zm2.28125 -0.84375q3.28125 0 3.28125 -3.390625q0 -3.390625 -3.28125 -3.390625l-1.796875 0l0 6.78125l1.796875 0zm10.461792 -0.515625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.480316 -2.453125q-0.640625 0.046875 -0.96875 0.40625q-0.3125 0.34375 -0.3125 1.046875l0 0.390625l1.328125 0q0.203125 0 0.3125 0.109375q0.109375 0.109375 0.109375 0.28125q0 0.1875 -0.109375 0.28125q-0.109375 0.09375 -0.3125 0.09375l-1.328125 0l0 4.65625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -4.65625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -0.21875q0 -1.078125 0.53125 -1.6875q0.546875 -0.625 1.5625 -0.703125l0.3125 -0.015625q0.3125 -0.03125 0.453125 0.0625q0.140625 0.078125 0.140625 0.296875q0 0.34375 -0.421875 0.390625l-0.3125 0.03125z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m276.85565 232.16667l0 20.377945" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m276.85565 232.16667l0 16.950867" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m276.85565 249.11754l-1.1246033 -1.124588l1.1246033 3.0897675l1.1245728 -3.0897675z" fill-rule="evenodd"/><path fill="#f4cccc" d="m31.874016 68.3563l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m31.874016 68.3563l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m67.63894 87.62236q0.171875 0.15625 0.171875 0.359375q0 0.15625 -0.140625 0.296875q-0.140625 0.140625 -0.3125 0.140625q-0.15625 0 -0.328125 -0.140625l-4.484375 -3.921875l0 3.578125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 3.4375l4.28125 -3.796875q0.125 -0.140625 0.3125 -0.140625q0.171875 0 0.296875 0.140625q0.140625 0.140625 0.140625 0.3125q0 0.171875 -0.15625 0.328125l-3.875 3.421875l4.09375 3.5625zm5.8329315 -0.609375q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.792801 -0.734375q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625zm3.720398 -0.015625q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm6.3444214 0.765625q-0.5625 0 -1.0625 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.875 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0625 -0.234375 -1.515625 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.15625 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.5625 0 -0.90625 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84375 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125z" fill-rule="nonzero"/><path fill="#f4cccc" d="m127.74803 68.35761l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m127.74803 68.35761l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m147.45874 88.37367q-0.234375 0 -0.375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -7.5q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l4.484375 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-4.015625 0l0 2.9375l3.78125 0q0.21875 0 0.328125 0.109375q0.125 0.109375 0.125 0.296875q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-3.78125 0l0 3.078125l4.015625 0q0.21875 0 0.328125 0.109375q0.125 0.09375 0.125 0.296875q0 0.1875 -0.125 0.296875q-0.109375 0.109375 -0.328125 0.109375l-4.484375 0zm8.31218 0.078125q-0.5625 0 -1.0625 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.875 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0625 -0.234375 -1.515625 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.15625 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.5625 0 -0.90625 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84375 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125zm6.4787903 -0.78125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm1.8769073 0.765625q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm8.799652 1.234375q1.9375 0 1.9375 2.3125l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.328125 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -3.21875q0 -0.8125 -0.296875 -1.171875q-0.28125 -0.359375 -0.890625 -0.359375q-0.734375 0 -1.15625 0.5q-0.421875 0.484375 -0.421875 1.328125l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.8125 -0.296875 -1.171875q-0.28125 -0.359375 -0.90625 -0.359375q-0.71875 0 -1.140625 0.5q-0.421875 0.484375 -0.421875 1.328125l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.359375 -0.140625q0.203125 0 0.328125 0.125q0.140625 0.125 0.140625 0.34375l0 0.578125q0.265625 -0.515625 0.734375 -0.78125q0.46875 -0.28125 1.078125 -0.28125q1.375 0 1.78125 1.140625q0.265625 -0.515625 0.78125 -0.828125q0.515625 -0.3125 1.171875 -0.3125zm6.0990753 0q2.203125 0 2.203125 2.296875l0 3.265625q0 0.21875 -0.125 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.140625 -0.125 -0.359375l0 -0.578125q-0.21875 0.515625 -0.6875 0.796875q-0.46875 0.28125 -1.078125 0.28125q-0.5625 0 -1.046875 -0.21875q-0.46875 -0.234375 -0.75 -0.640625q-0.265625 -0.40625 -0.265625 -0.90625q0 -0.65625 0.328125 -1.015625q0.34375 -0.375 1.109375 -0.53125q0.765625 -0.15625 2.125 -0.15625l0.265625 0l0 -0.40625q0 -0.71875 -0.296875 -1.046875q-0.28125 -0.34375 -0.953125 -0.34375q-0.8125 0 -1.65625 0.453125q-0.3125 0.203125 -0.453125 0.203125q-0.140625 0 -0.234375 -0.109375q-0.09375 -0.109375 -0.09375 -0.28125q0 -0.171875 0.09375 -0.296875q0.109375 -0.125 0.328125 -0.25q0.421875 -0.25 0.953125 -0.375q0.546875 -0.140625 1.0625 -0.140625zm-0.390625 5.296875q0.71875 0 1.171875 -0.484375q0.46875 -0.484375 0.46875 -1.25l0 -0.34375l-0.21875 0q-1.046875 0 -1.609375 0.09375q-0.546875 0.078125 -0.78125 0.296875q-0.234375 0.203125 -0.234375 0.609375q0 0.46875 0.34375 0.78125q0.34375 0.296875 0.859375 0.296875zm7.0631714 -0.015625q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm3.8144073 0.78125q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm7.1287994 -5.25q0.5 -0.03125 0.5 0.40625q0 0.203125 -0.109375 0.3125q-0.109375 0.109375 -0.375 0.140625l-0.359375 0.03125q-0.796875 0.078125 -1.1875 0.578125q-0.390625 0.484375 -0.390625 1.15625l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.140625 -0.359375q0.140625 -0.125 0.34375 -0.125q0.1875 0 0.3125 0.125q0.140625 0.125 0.140625 0.34375l0 0.671875q0.25 -0.53125 0.71875 -0.796875q0.46875 -0.28125 1.0625 -0.328125l0.171875 -0.015625z" fill-rule="nonzero"/><path fill="#f4cccc" d="m233.1076 68.35761l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m233.1076 68.35761l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m269.00754 88.46742q-0.90625 0 -1.734375 -0.265625q-0.8125 -0.265625 -1.3125 -0.734375q-0.171875 -0.15625 -0.171875 -0.40625q0 -0.171875 0.09375 -0.296875q0.09375 -0.125 0.234375 -0.125q0.15625 0 0.328125 0.125q1.109375 0.859375 2.546875 0.859375q1.03125 0 1.578125 -0.390625q0.5625 -0.390625 0.5625 -1.125q0 -0.421875 -0.265625 -0.671875q-0.265625 -0.265625 -0.703125 -0.421875q-0.4375 -0.15625 -1.15625 -0.328125q-0.984375 -0.21875 -1.625 -0.46875q-0.625 -0.265625 -1.015625 -0.734375q-0.390625 -0.46875 -0.390625 -1.21875q0 -0.71875 0.390625 -1.265625q0.390625 -0.5625 1.09375 -0.875q0.703125 -0.3125 1.59375 -0.3125q0.84375 0 1.5625 0.265625q0.734375 0.25 1.234375 0.734375q0.1875 0.1875 0.1875 0.421875q0 0.171875 -0.09375 0.296875q-0.09375 0.125 -0.234375 0.125q-0.125 0 -0.34375 -0.140625q-0.59375 -0.46875 -1.09375 -0.65625q-0.5 -0.203125 -1.21875 -0.203125q-0.984375 0 -1.546875 0.421875q-0.546875 0.40625 -0.546875 1.15625q0 0.625 0.484375 0.953125q0.484375 0.3125 1.5 0.5625q1.09375 0.25 1.71875 0.484375q0.625 0.21875 1.03125 0.671875q0.421875 0.4375 0.421875 1.171875q0 0.71875 -0.390625 1.265625q-0.390625 0.53125 -1.109375 0.828125q-0.703125 0.296875 -1.609375 0.296875zm5.0446777 -0.03125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.359375 -0.125q0.203125 0 0.34375 0.125q0.140625 0.125 0.140625 0.34375l0 7.625q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125zm2.784027 0q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm8.799652 1.234375q1.9375 0 1.9375 2.3125l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.328125 0.125q-0.21875 0 -0.359375 -0.125q-0.140625 -0.125 -0.140625 -0.359375l0 -3.21875q0 -0.8125 -0.296875 -1.171875q-0.28125 -0.359375 -0.890625 -0.359375q-0.734375 0 -1.15625 0.5q-0.421875 0.484375 -0.421875 1.328125l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.8125 -0.296875 -1.171875q-0.28125 -0.359375 -0.90625 -0.359375q-0.71875 0 -1.140625 0.5q-0.421875 0.484375 -0.421875 1.328125l0 2.921875q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.359375 -0.140625q0.203125 0 0.328125 0.125q0.140625 0.125 0.140625 0.34375l0 0.578125q0.265625 -0.515625 0.734375 -0.78125q0.46875 -0.28125 1.078125 -0.28125q1.375 0 1.78125 1.140625q0.265625 -0.515625 0.78125 -0.828125q0.515625 -0.3125 1.171875 -0.3125z" fill-rule="nonzero"/><path fill="#d9ead3" d="m282.5035 134.76706l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m282.5035 134.76706l87.49606 0l0 30.992126l-87.49606 0z" fill-rule="evenodd"/><path fill="#000000" d="m297.8283 154.87688q-1.1875 0 -2.0625 -0.515625q-0.875 -0.53125 -1.359375 -1.5q-0.46875 -0.984375 -0.46875 -2.3125q0 -1.328125 0.46875 -2.296875q0.484375 -0.984375 1.359375 -1.5q0.875 -0.53125 2.0625 -0.53125q0.8125 0 1.515625 0.265625q0.71875 0.25 1.25 0.734375q0.1875 0.1875 0.1875 0.421875q0 0.171875 -0.09375 0.296875q-0.09375 0.125 -0.21875 0.125q-0.15625 0 -0.359375 -0.140625q-0.609375 -0.46875 -1.109375 -0.65625q-0.5 -0.203125 -1.140625 -0.203125q-1.390625 0 -2.140625 0.90625q-0.75 0.90625 -0.75 2.578125q0 1.671875 0.75 2.578125q0.75 0.90625 2.140625 0.90625q0.640625 0 1.140625 -0.1875q0.5 -0.1875 1.109375 -0.671875q0.203125 -0.125 0.359375 -0.125q0.125 0 0.21875 0.125q0.09375 0.109375 0.09375 0.296875q0 0.234375 -0.1875 0.40625q-0.53125 0.484375 -1.25 0.75q-0.703125 0.25 -1.515625 0.25zm7.358429 -6.078125q1.03125 0 1.546875 0.578125q0.53125 0.578125 0.53125 1.734375l0 3.25q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.21875q0 -0.78125 -0.328125 -1.15625q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.203125 0.125 -0.328125q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.125q0.125 0.125 0.125 0.34375l0 3.140625q0.28125 -0.53125 0.796875 -0.796875q0.515625 -0.28125 1.1875 -0.28125zm8.37854 4.625q0.140625 0 0.234375 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.296875 -0.421875 0.546875q-0.4375 0.25 -0.921875 0.375q-0.46875 0.125 -0.921875 0.125q-1.359375 0 -2.15625 -0.796875q-0.78125 -0.8125 -0.78125 -2.21875q0 -0.90625 0.34375 -1.59375q0.359375 -0.6875 0.984375 -1.0625q0.640625 -0.390625 1.4375 -0.390625q1.140625 0 1.8125 0.75q0.671875 0.734375 0.671875 2.0q0 0.25 -0.09375 0.359375q-0.09375 0.109375 -0.3125 0.109375l-3.859375 0q0.09375 2.0625 1.953125 2.0625q0.46875 0 0.796875 -0.125q0.34375 -0.125 0.71875 -0.34375q0.3125 -0.1875 0.421875 -0.1875zm-2.09375 -3.875q-0.765625 0 -1.234375 0.484375q-0.46875 0.484375 -0.546875 1.359375l3.390625 0q-0.015625 -0.890625 -0.4375 -1.359375q-0.421875 -0.484375 -1.171875 -0.484375zm6.308441 5.3125q-0.8125 0 -1.453125 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.6875 -0.34375 -1.578125q0 -0.90625 0.359375 -1.59375q0.359375 -0.703125 0.984375 -1.078125q0.640625 -0.390625 1.46875 -0.390625q0.453125 0 0.90625 0.125q0.453125 0.125 0.78125 0.359375q0.21875 0.140625 0.3125 0.28125q0.09375 0.140625 0.09375 0.3125q0 0.171875 -0.09375 0.28125q-0.09375 0.09375 -0.234375 0.09375q-0.078125 0 -0.1875 -0.046875q-0.09375 -0.046875 -0.15625 -0.09375q-0.0625 -0.046875 -0.09375 -0.0625q-0.3125 -0.203125 -0.59375 -0.3125q-0.28125 -0.125 -0.6875 -0.125q-0.875 0 -1.359375 0.59375q-0.484375 0.59375 -0.484375 1.65625q0 1.046875 0.484375 1.625q0.484375 0.578125 1.359375 0.578125q0.40625 0 0.703125 -0.109375q0.296875 -0.125 0.59375 -0.328125q0.140625 -0.09375 0.25 -0.15625q0.125 -0.0625 0.203125 -0.0625q0.140625 0 0.21875 0.109375q0.09375 0.109375 0.09375 0.28125q0 0.15625 -0.09375 0.28125q-0.078125 0.125 -0.296875 0.28125q-0.34375 0.234375 -0.8125 0.375q-0.46875 0.125 -0.953125 0.125zm7.998047 -0.84375q0.203125 0.171875 0.203125 0.375q0 0.1875 -0.125 0.328125q-0.125 0.125 -0.3125 0.125q-0.15625 0 -0.328125 -0.140625l-3.125 -2.703125l0 2.359375q0 0.234375 -0.140625 0.359375q-0.140625 0.125 -0.34375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -7.625q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.359375 -0.125q0.203125 0 0.34375 0.125q0.140625 0.125 0.140625 0.34375l0 4.875l2.859375 -2.625q0.15625 -0.140625 0.328125 -0.140625q0.1875 0 0.3125 0.140625q0.140625 0.125 0.140625 0.296875q0 0.203125 -0.171875 0.359375l-2.375 2.109375l2.59375 2.265625zm4.2812805 -5.21875q0.765625 0 1.34375 0.390625q0.59375 0.375 0.921875 1.0625q0.328125 0.6875 0.328125 1.609375q0 0.90625 -0.328125 1.59375q-0.328125 0.671875 -0.90625 1.046875q-0.578125 0.359375 -1.359375 0.359375q-0.6875 0 -1.203125 -0.296875q-0.5 -0.296875 -0.765625 -0.84375l0 2.8125q0 0.21875 -0.125 0.34375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.140625q-0.125 -0.125 -0.125 -0.328125l0 -7.234375q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.125 0.125 0.125 0.34375l0 0.640625q0.265625 -0.546875 0.765625 -0.84375q0.515625 -0.296875 1.203125 -0.296875zm-0.203125 5.265625q0.859375 0 1.328125 -0.578125q0.46875 -0.578125 0.46875 -1.625q0 -1.0625 -0.46875 -1.65625q-0.46875 -0.59375 -1.328125 -0.59375q-0.84375 0 -1.3125 0.578125q-0.453125 0.578125 -0.453125 1.640625q0 1.0625 0.453125 1.65625q0.46875 0.578125 1.3125 0.578125zm6.67157 0.796875q-0.828125 0 -1.46875 -0.359375q-0.625 -0.375 -0.96875 -1.0625q-0.34375 -0.703125 -0.34375 -1.609375q0 -0.90625 0.34375 -1.59375q0.34375 -0.703125 0.96875 -1.0625q0.640625 -0.375 1.46875 -0.375q0.828125 0 1.453125 0.375q0.640625 0.359375 0.984375 1.0625q0.34375 0.6875 0.34375 1.59375q0 0.90625 -0.34375 1.609375q-0.34375 0.6875 -0.984375 1.0625q-0.625 0.359375 -1.453125 0.359375zm0 -0.796875q0.859375 0 1.3125 -0.5625q0.46875 -0.578125 0.46875 -1.671875q0 -1.0625 -0.46875 -1.640625q-0.46875 -0.59375 -1.3125 -0.59375q-0.859375 0 -1.328125 0.59375q-0.46875 0.578125 -0.46875 1.640625q0 1.078125 0.453125 1.65625q0.46875 0.578125 1.34375 0.578125zm4.722534 0.78125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.234375 0.125 -0.359375q0.140625 -0.125 0.359375 -0.125q0.21875 0 0.34375 0.125q0.140625 0.125 0.140625 0.359375l0 5.0625q0 0.234375 -0.140625 0.359375q-0.125 0.125 -0.34375 0.125zm0 -7.28125q-0.296875 0 -0.484375 -0.171875q-0.171875 -0.171875 -0.171875 -0.453125q0 -0.25 0.171875 -0.421875q0.1875 -0.171875 0.484375 -0.171875q0.28125 0 0.453125 0.171875q0.1875 0.171875 0.1875 0.421875q0 0.28125 -0.1875 0.453125q-0.171875 0.171875 -0.453125 0.171875zm5.237152 1.234375q2.09375 0 2.09375 2.3125l0 3.25q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.21875 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -3.1875q0 -0.8125 -0.328125 -1.1875q-0.3125 -0.375 -1.0 -0.375q-0.8125 0 -1.296875 0.5q-0.46875 0.484375 -0.46875 1.328125l0 2.921875q0 0.234375 -0.125 0.359375q-0.125 0.125 -0.359375 0.125q-0.234375 0 -0.359375 -0.125q-0.125 -0.125 -0.125 -0.359375l0 -5.0625q0 -0.21875 0.125 -0.34375q0.125 -0.140625 0.359375 -0.140625q0.21875 0 0.34375 0.140625q0.125 0.125 0.125 0.328125l0 0.609375q0.28125 -0.53125 0.796875 -0.8125q0.53125 -0.28125 1.1875 -0.28125zm6.5660706 5.28125q0.421875 0.03125 0.421875 0.375q0 0.203125 -0.15625 0.3125q-0.140625 0.09375 -0.4375 0.078125l-0.328125 -0.03125q-0.953125 -0.0625 -1.421875 -0.5625q-0.453125 -0.515625 -0.453125 -1.53125l0 -3.015625l-0.796875 0q-0.203125 0 -0.328125 -0.09375q-0.109375 -0.109375 -0.109375 -0.28125q0 -0.171875 0.109375 -0.28125q0.125 -0.109375 0.328125 -0.109375l0.796875 0l0 -1.359375q0 -0.21875 0.125 -0.34375q0.140625 -0.140625 0.375 -0.140625q0.21875 0 0.34375 0.140625q0.140625 0.125 0.140625 0.34375l0 1.359375l1.328125 0q0.1875 0 0.296875 0.109375q0.125 0.109375 0.125 0.28125q0 0.171875 -0.125 0.28125q-0.109375 0.09375 -0.296875 0.09375l-1.328125 0l0 3.0625q0 0.65625 0.265625 0.953125q0.265625 0.296875 0.8125 0.328125l0.3125 0.03125zm3.361267 0.78125q-0.5625 0 -1.0625 -0.125q-0.5 -0.140625 -0.875 -0.375q-0.21875 -0.140625 -0.3125 -0.265625q-0.078125 -0.125 -0.078125 -0.3125q0 -0.15625 0.078125 -0.25q0.09375 -0.109375 0.234375 -0.109375q0.15625 0 0.421875 0.1875q0.359375 0.21875 0.71875 0.34375q0.359375 0.125 0.875 0.125q0.65625 0 1.015625 -0.21875q0.359375 -0.234375 0.359375 -0.671875q0 -0.265625 -0.140625 -0.421875q-0.125 -0.171875 -0.453125 -0.296875q-0.3125 -0.125 -0.9375 -0.25q-1.0625 -0.234375 -1.515625 -0.609375q-0.453125 -0.390625 -0.453125 -1.046875q0 -0.515625 0.28125 -0.90625q0.28125 -0.40625 0.796875 -0.625q0.515625 -0.234375 1.15625 -0.234375q0.46875 0 0.90625 0.125q0.4375 0.125 0.78125 0.34375q0.40625 0.296875 0.40625 0.609375q0 0.15625 -0.09375 0.265625q-0.09375 0.109375 -0.234375 0.109375q-0.140625 0 -0.4375 -0.203125q-0.328125 -0.21875 -0.625 -0.34375q-0.296875 -0.125 -0.75 -0.125q-0.5625 0 -0.90625 0.265625q-0.34375 0.25 -0.34375 0.671875q0 0.25 0.125 0.421875q0.125 0.15625 0.421875 0.28125q0.296875 0.125 0.84375 0.25q0.828125 0.1875 1.265625 0.40625q0.453125 0.203125 0.640625 0.515625q0.203125 0.3125 0.203125 0.796875q0 0.75 -0.640625 1.21875q-0.640625 0.453125 -1.671875 0.453125z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m233.1085 268.03217l-66.74016 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" stroke-dasharray="4.0,3.0" d="m233.10852 268.03217l-63.313095 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m169.79543 268.03217l1.124588 -1.1246033l-3.0897675 1.1246033l3.0897675 1.1245728z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m75.62205 99.34843l0 19.652092l46.992126 0l0 133.54475" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m75.62205 99.34843l0 19.652084l46.992126 0l0 130.11768" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m122.614174 249.1182l-1.124588 -1.124588l1.124588 3.0897675l1.1245804 -3.0897675z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m171.49606 99.34974l0 19.650558l-48.88189 0l0 133.5463" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m171.49606 99.34974l0 19.650558l-48.88189 0l0 130.1192" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m122.614174 249.1195l-1.124588 -1.124588l1.124588 3.0897675l1.1245804 -3.0897675z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m276.85565 99.34974l0 17.70874l-42.960632 0l0 17.724327" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m276.85565 99.34974l0 17.70874l-42.960632 0l0 14.297249" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m233.89502 131.35573l-1.124588 -1.124588l1.124588 3.0897675l1.1245728 -3.0897675z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m276.85565 99.34974l0 17.70874l49.385803 0l0 17.724327" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m276.85565 99.34974l0 17.70874l49.385803 0l0 14.297249" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m326.24146 131.35573l-1.1245728 -1.124588l1.1245728 3.0897675l1.1246033 -3.0897675z" fill-rule="evenodd"/><path fill="#c9daf8" d="m548.5407 235.66077l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m548.5407 235.66077l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path fill="#000000" d="m579.47955 247.1612q0.203125 0 0.328125 0.140625q0.125 0.125 0.125 0.359375l0 7.578125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625q-0.234375 0 -0.390625 -0.203125l-4.984375 -6.65625l0 6.359375q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.34375 0.140625q-0.21875 0 -0.34375 -0.140625q-0.109375 -0.140625 -0.109375 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.40625 0.203125l4.96875 6.65625l0 -6.359375q0 -0.234375 0.125 -0.359375q0.125 -0.140625 0.34375 -0.140625zm8.868103 0q0.203125 0 0.328125 0.140625q0.125 0.125 0.125 0.359375l0 7.578125q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.359375 0.140625q-0.234375 0 -0.390625 -0.203125l-4.984375 -6.65625l0 6.359375q0 0.21875 -0.125 0.359375q-0.125 0.140625 -0.34375 0.140625q-0.21875 0 -0.34375 -0.140625q-0.109375 -0.140625 -0.109375 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.125 -0.140625 0.359375 -0.140625q0.234375 0 0.40625 0.203125l4.96875 6.65625l0 -6.359375q0 -0.234375 0.125 -0.359375q0.125 -0.140625 0.34375 -0.140625zm12.917175 7.953125q0.046875 0.09375 0.046875 0.203125q0 0.171875 -0.140625 0.296875q-0.140625 0.125 -0.328125 0.125q-0.296875 0 -0.421875 -0.296875l-0.84375 -1.9375l-4.53125 0l-0.859375 1.9375q-0.125 0.296875 -0.421875 0.296875q-0.1875 0 -0.34375 -0.125q-0.140625 -0.125 -0.140625 -0.3125q0 -0.09375 0.046875 -0.1875l3.4375 -7.640625q0.078125 -0.15625 0.21875 -0.234375q0.140625 -0.09375 0.3125 -0.09375q0.171875 0 0.3125 0.09375q0.15625 0.078125 0.21875 0.234375l3.4375 7.640625zm-5.859375 -2.421875l3.8125 0l-1.90625 -4.3125l-1.90625 4.3125zm7.78656 3.046875q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.546875q0 -0.21875 0.125 -0.34375q0.140625 -0.125 0.375 -0.125l2.84375 0q1.328125 0 2.0625 0.65625q0.75 0.640625 0.75 1.828125q0 1.1875 -0.75 1.84375q-0.734375 0.65625 -2.0625 0.65625l-2.359375 0l0 3.03125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625zm2.765625 -4.34375q1.9375 0 1.9375 -1.6875q0 -1.671875 -1.9375 -1.671875l-2.265625 0l0 3.359375l2.265625 0zm4.9744263 4.34375q-0.21875 0 -0.359375 -0.140625q-0.125 -0.140625 -0.125 -0.359375l0 -7.578125q0 -0.234375 0.125 -0.359375q0.140625 -0.140625 0.359375 -0.140625q0.234375 0 0.359375 0.140625q0.140625 0.125 0.140625 0.359375l0 7.578125q0 0.21875 -0.140625 0.359375q-0.125 0.140625 -0.359375 0.140625z" fill-rule="nonzero"/><path fill="#c9daf8" d="m548.5407 193.79199l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m548.5407 193.79199l87.49603 0l0 30.992126l-87.49603 0z" fill-rule="evenodd"/><path fill="#000000" d="m589.5417 213.87056q-0.28125 0 -0.484375 -0.1875q-0.1875 -0.1875 -0.1875 -0.484375q0 -0.296875 0.1875 -0.484375q0.203125 -0.203125 0.484375 -0.203125q0.28125 0 0.46875 0.203125q0.1875 0.1875 0.1875 0.484375q0 0.296875 -0.1875 0.484375q-0.1875 0.1875 -0.46875 0.1875zm2.7480469 0q-0.28125 0 -0.484375 -0.1875q-0.1875 -0.1875 -0.1875 -0.484375q0 -0.296875 0.1875 -0.484375q0.203125 -0.203125 0.484375 -0.203125q0.28125 0 0.46875 0.203125q0.1875 0.1875 0.1875 0.484375q0 0.296875 -0.1875 0.484375q-0.1875 0.1875 -0.46875 0.1875zm2.7479858 0q-0.28125 0 -0.484375 -0.1875q-0.1875 -0.1875 -0.1875 -0.484375q0 -0.296875 0.1875 -0.484375q0.203125 -0.203125 0.484375 -0.203125q0.28125 0 0.46875 0.203125q0.1875 0.1875 0.1875 0.484375q0 0.296875 -0.1875 0.484375q-0.1875 0.1875 -0.46875 0.1875z" fill-rule="nonzero"/><path fill="#000000" fill-opacity="0.0" d="m122.620316 283.52823l0 14.9730835l75.49606 0l0 20.90091" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m122.620316 283.52823l0 14.9730835l75.49608 0l0 17.473846" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m198.1164 315.97516l-1.124588 -1.1246033l1.124588 3.0897827l1.1245728 -3.0897827z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m276.85654 283.52823l0 14.9730835l-78.74016 0l0 20.90091" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m276.85654 283.52823l0 14.9730835l-78.74014 0l0 17.473846" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m198.1164 315.97516l-1.124588 -1.1246033l1.124588 3.0897827l1.1245728 -3.0897827z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m500.5223 334.89435l24.009003 0l0 0.06298828l24.022522 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m500.5223 334.89435l24.009003 0l0 0.06298828l20.595398 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m545.1267 334.95734l-1.1245728 1.1246033l3.0897827 -1.1246033l-3.0897827 -1.1245728z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m500.5223 334.89435l24.009003 0l0 -41.858246l24.022522 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m500.5223 334.89435l24.009003 0l0 -41.858246l20.595398 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m545.1267 293.0361l-1.1245728 1.1245728l3.0897827 -1.1245728l-3.0897827 -1.1246033z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m500.5223 334.89435l24.009003 0l0 -83.74802l24.022522 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m500.5223 334.89435l24.009003 0l0 -83.74802l20.595398 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m545.1267 251.14633l-1.1245728 1.1245728l3.0897827 -1.1245728l-3.0897827 -1.124588z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m500.5223 334.89435l24.009003 0l0 -125.60629l24.022522 0" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m500.5223 334.89435l24.009003 0l0 -125.60629l20.595398 0" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m545.1267 209.28806l-1.1245728 1.124588l3.0897827 -1.124588l-3.0897827 -1.124588z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m233.88803 165.75919l0 17.70752l42.960632 0l0 17.694061" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m233.88805 165.75919l0 17.70752l42.960617 0l0 14.266968" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m276.84866 197.73367l-1.1245728 -1.124588l1.1245728 3.0897675l1.1246033 -3.0897675z" fill-rule="evenodd"/><path fill="#000000" fill-opacity="0.0" d="m326.25156 165.75919l0 17.70752l-49.385834 0l0 17.694061" fill-rule="evenodd"/><path stroke="#000000" stroke-width="1.0" stroke-linejoin="round" stroke-linecap="butt" d="m326.25156 165.75919l0 17.70752l-49.385834 0l0 14.266968" fill-rule="evenodd"/><path fill="#000000" stroke="#000000" stroke-width="1.0" stroke-linecap="butt" d="m276.86572 197.73367l-1.1245728 -1.124588l1.1245728 3.0897675l1.1246033 -3.0897675z" fill-rule="evenodd"/></g></svg>
\ No newline at end of file
-- 
GitLab


From 6ccdb724858ac1ac343b47e73b75802e6e8fa004 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 17:15:33 -0700
Subject: [PATCH 719/960] Add details of new mailing lists

PiperOrigin-RevId: 191011187
---
 tensorflow/docs_src/about/uses.md             |  6 ++--
 tensorflow/docs_src/community/contributing.md | 25 +++----------
 tensorflow/docs_src/community/index.md        |  6 +++-
 tensorflow/docs_src/community/leftnav_files   |  1 +
 tensorflow/docs_src/community/lists.md        | 32 ++++++++++++-----
 tensorflow/docs_src/community/swift.md        | 35 +++++++++++++++++++
 6 files changed, 73 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/docs_src/community/swift.md

diff --git a/tensorflow/docs_src/about/uses.md b/tensorflow/docs_src/about/uses.md
index d646880bd3..d3db98203e 100644
--- a/tensorflow/docs_src/about/uses.md
+++ b/tensorflow/docs_src/about/uses.md
@@ -18,9 +18,9 @@ This section describes some of the current uses of the TensorFlow system.
 
 > If you are using TensorFlow for research, for education, or for production
 > usage in some product, we would love to add something about your usage here.
-> Please feel free to email us a brief description of how you're using
-> TensorFlow, or even better, send us a pull request to add an entry to this
-> file.
+> Please feel free to [email us](mailto:usecases@tensorflow.org) a brief
+> description of how you're using TensorFlow, or even better, send us a
+> pull request to add an entry to this file.
 
 * **Deep Speech**
 <ul>
diff --git a/tensorflow/docs_src/community/contributing.md b/tensorflow/docs_src/community/contributing.md
index b0960df435..afbb8bbdd0 100644
--- a/tensorflow/docs_src/community/contributing.md
+++ b/tensorflow/docs_src/community/contributing.md
@@ -22,7 +22,7 @@ inside the organization, including:
 guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).
 
 * Join the
-[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers)
+[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/d/forum/developers)
 mailing list, to coordinate and discuss with others contributing to TensorFlow.
 
 * For coding style conventions, read the @{$style_guide$TensorFlow Style Guide}.
@@ -37,28 +37,13 @@ You may also wish to review our guide to @{$benchmarks$defining and running benc
 To enable focused collaboration on particular areas of TensorFlow, we host
 Special Interest Groups (SIGs). SIGs do their work in public: if you want to
 join and contribute, review the work of the group, and get in touch with the
-relevant SIG leader.
+relevant SIG leader.  Membership policies vary on a per-SIG basis.
 
 * **SIG Build** focuses on issues surrounding building, packaging, and
-  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/build).
+  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/build).
 
 * **SIG TensorBoard** furthers the development and direction of TensorBoard and its plugins.
-  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard).
+  [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/sig-tensorboard).
 
 * **SIG Rust** collaborates on the development of TensorFlow's Rust bindings.
-  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust). 
-
-## Projects developed by the TensorFlow community
-
-The TensorFlow community has created many great projects around TensorFlow, including:
-
-* [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com)
-* [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow)
-* [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials)
-* [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow)
-* [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0))
-* [Operator Vectorization Library](https://github.com/opveclib/opveclib)
-* [Swift language bindings](https://github.com/PerfectlySoft/Perfect-TensorFlow)
-* [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
-* [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
-* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/rust).
diff --git a/tensorflow/docs_src/community/index.md b/tensorflow/docs_src/community/index.md
index c08aeb7a97..eec2e51a87 100644
--- a/tensorflow/docs_src/community/index.md
+++ b/tensorflow/docs_src/community/index.md
@@ -47,6 +47,11 @@ The @{$roadmap$Roadmap} summarizes plans for upcoming additions to TensorFlow.
 For news and updates from around the universe of TensorFlow projects, follow
 [@tensorflow](https://twitter.com/tensorflow) on Twitter.
 
+### Blog
+
+We post regularly to the [TensorFlow Blog](http://blog.tensorflow.org/),
+with content from the TensorFlow team and the best articles from the community.
+
 ### YouTube
 
 Our [YouTube Channel](http://youtube.com/tensorflow/) focuses on machine learing
@@ -56,7 +61,6 @@ and AI with TensorFlow. On it we have a number of new shows, including:
 - Ask TensorFlow: the team answers the best questions tagged #AskTensorFlow from social media 
 - Coding TensorFlow: short bites with tips for success with TensorFlow
 
-
 ## Community Support
 
 ### Mailing Lists
diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files
index 0bd1f14de9..2bae60d9dd 100644
--- a/tensorflow/docs_src/community/leftnav_files
+++ b/tensorflow/docs_src/community/leftnav_files
@@ -6,3 +6,4 @@ groups.md
 documentation.md
 style_guide.md
 benchmarks.md
+swift.md
diff --git a/tensorflow/docs_src/community/lists.md b/tensorflow/docs_src/community/lists.md
index dc9240030e..7450ab36c4 100644
--- a/tensorflow/docs_src/community/lists.md
+++ b/tensorflow/docs_src/community/lists.md
@@ -9,27 +9,43 @@ see [how to get help](/community/#get_help).
 
 ## General TensorFlow lists
 
-* [announce](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce) - Low-volume announcements of new releases.
-* [discuss](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss) - General community discussion around TensorFlow.
-* [developers](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers) - Discussion for developers contributing to TensorFlow.
+* [announce](https://groups.google.com/a/tensorflow.org/d/forum/announce) - Low-volume announcements of new releases.
+* [discuss](https://groups.google.com/a/tensorflow.org/d/forum/discuss) - General community discussion around TensorFlow.
+* [developers](https://groups.google.com/a/tensorflow.org/d/forum/developers) - Discussion for developers contributing to TensorFlow.
 
 ## Project-specific lists
 
 These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
 
-* [tensor2tensor](https://groups.google.com/forum/#!forum/tensor2tensor) - User
+* [hub](https://groups.google.com/a/tensorflow.org/d/forum/hub) -
+  Discussion and collaboration around [TensorFlow Hub](https://github.com/tensorflow/hub).
+* [magenta-discuss](https://groups.google.com/a/tensorflow.org/d/forum/magenta-discuss) -
+  General discussion about [Magenta](https://magenta.tensorflow.org/)
+  development and directions.
+* [swift](https://groups.google.com/a/tensorflow.org/d/forum/swift) -
+  Community and collaboration around Swift for TensorFlow.
+* [tensor2tensor](https://groups.google.com/d/forum/tensor2tensor) - Discussion
   and peer support for Tensor2Tensor.
+* [tfjs-announce](https://groups.google.com/a/tensorflow.org/d/forum/tfjs-announce) -
+  Announcements of new TensorFlow.js releases.
+* [tfjs](https://groups.google.com/a/tensorflow.org/d/forum/tfjs) - Discussion
+  and peer support for TensorFlow.js.
+* [tflite](https://groups.google.com/a/tensorflow.org/d/forum/tflite) - Discussion and
+  peer support for TensorFlow Lite.
+* [tpu-users](https://groups.google.com/a/tensorflow.org/d/forum/tpu-users) - Community discussion
+  and support for TPU users.
 
 ## Special Interest Groups
 
 TensorFlow's [Special Interest
 Groups](/community/contributing#special_interest_groups) (SIGs) support
 community collaboration on particular project focuses. Members of these groups
-work together to build and support TensorFlow related projects.
+work together to build and support TensorFlow related projects. While their
+archives are public, different SIGs have their own membership policies.
 
-* [build](https://groups.google.com/a/tensorflow.org/forum/#!forum/build) -
+* [build](https://groups.google.com/a/tensorflow.org/d/forum/build) -
   Supporting SIG Build, for build, distribution and packaging of TensorFlow.
-* [tensorboard](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard) -
+* [sig-tensorboard](https://groups.google.com/a/tensorflow.org/d/forum/sig-tensorboard) -
   Supporting SIG TensorBoard, for plugin development and other contribution.
-* [rust](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust) -
+* [rust](https://groups.google.com/a/tensorflow.org/d/forum/rust) -
   Supporting SIG Rust, for the Rust language bindings.
diff --git a/tensorflow/docs_src/community/swift.md b/tensorflow/docs_src/community/swift.md
new file mode 100644
index 0000000000..54d9960b23
--- /dev/null
+++ b/tensorflow/docs_src/community/swift.md
@@ -0,0 +1,35 @@
+# Swift Community
+
+Welcome to the Swift for TensorFlow development community!
+
+Swift for TensorFlow is a result of first-principles thinking applied to machine
+learning frameworks, and works quite differently than existing TensorFlow
+language bindings.  Whereas prior solutions are designed within the constraints
+of what can be achieved by a (typically Python or Lua) library, Swift for
+TensorFlow is based on the belief that machine learning is important enough to
+deserve first-class language and compiler support.
+
+First-class language and compiler support allows us to innovate in areas that
+have traditionally been out of bounds for machine learning libraries.  Our
+results provide the performance of TensorFlow graphs with the ease of use of
+define-by-run models, and provides a great user experience - for example, by
+catching more mistakes before you run your code.
+
+## Open Source
+
+As announced at the TensorFlow Developer Summit, we are planning to launch our
+open source project on GitHub in April.  In addition to releasing the code, we
+will be using an open design model, where design discussions happen in public.
+
+Between now and then, we are writing some technical white papers that explain in
+detail the design approach (e.g., the core compiler partitioning technique that
+underlies the whole thing, our approach to automatic differentiation, etc.),
+implementation tradeoffs, and the status of this work.  We can’t wait to engage
+with the broader community, but prefer to start the conversation when these
+white papers are ready.
+
+[Sign up here to join the community Google
+group](https://groups.google.com/a/tensorflow.org/d/forum/swift). We will
+initially use it for announcements, and then open it for general discussion when
+we are ready in April.
+
-- 
GitLab


From 160f5649dd20512d4c23c167634e3ff959bed029 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 17:16:46 -0700
Subject: [PATCH 720/960] Tighten the condition for node removal next to device
 crossings, since the previous changes were observed to cause a performance
 regression for distributed execution.

PiperOrigin-RevId: 191011347
---
 .../optimizers/dependency_optimizer.cc        | 16 +++++++---
 .../optimizers/dependency_optimizer_test.cc   | 32 +++++++++++++++++++
 2 files changed, 43 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index 63bc19630d..e5822c3b42 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -298,6 +298,15 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
       input_nodes.push_back(input_node);
     }
 
+    // Make sure that we don't increase the number of edges that cross
+    // device boundaries.
+    if ((num_inputs == 1 && num_outputs > 1 &&
+         input_nodes[0]->device() != node->device()) ||
+        (num_inputs > 1 && num_outputs == 1 &&
+         output_nodes[0]->device() != node->device())) {
+      return;
+    }
+
     // TODO(rmlarsen): Not all device crossings are equally expensive.
     // Assign a cost to each based on device affinity and compute a
     // cost before and after.
@@ -316,6 +325,8 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
       // unless they only have consumers on the same device as themselves.
       return;
     }
+
+    // Make sure we do not increase the number of device crossings.
     const int num_cross_before = num_cross_in + num_cross_out;
     int num_cross_after = 0;
     for (NodeDef* input_node : input_nodes) {
@@ -325,7 +336,6 @@ void DependencyOptimizer::OptimizeNode(int node_idx,
       }
     }
     if (num_cross_after > num_cross_before) {
-      // Avoid increasing the number of device crossings.
       return;
     }
 
@@ -518,10 +528,6 @@ Status DependencyOptimizer::TransitiveReduction() {
       if (longest_distance[target] > 1) {
         const int input_slot = control_output.second;
         control_edges_to_remove[target].emplace(input_slot, source);
-        //        VLOG(1) << "Removing edge from:\n"
-        //                << optimized_graph_->node(source).DebugString() <<
-        //                "\n\nto:\n\n"
-        //                << optimized_graph_->node(target).DebugString();
       }
     }
   }
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index cc1e142041..57b3118245 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -285,6 +285,38 @@ TEST_F(DependencyOptimizerTest, RemoveNoOps_DeviceBoundaries) {
   VerifyGraphsEqual(item.graph, output, __FUNCTION__);
 }
 
+TEST_F(DependencyOptimizerTest, RemoveIdentityOps_DeviceBoundaries) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::RandomUniform(s.WithOpName("x").WithDevice("/CPU:0"), {1, 2},
+                                DT_FLOAT);
+  Output y = ops::RandomUniform(s.WithOpName("y").WithDevice("/CPU:0"), {1, 2},
+                                DT_FLOAT);
+  // Identity with a single input- and two output dependencies.
+  auto id_a = ops::Identity(s.WithOpName("id_a").WithDevice("/CPU:1"), x);
+  // Identity with a two input- and a single output dependency.
+  auto id_b = ops::Identity(
+      s.WithOpName("id_b").WithControlDependencies(y).WithDevice("/CPU:0"), x);
+
+  Output id =
+      ops::Identity(s.WithControlDependencies(id_a).WithDevice("/CPU:1"), id_b);
+  Output id_1 = ops::Identity(s.WithDevice("/CPU:1"), id_a);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  item.fetch.push_back("Identity");
+  item.fetch.push_back("Identity_1");
+
+  DependencyOptimizer optimizer;
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  // The optimization should be disabled to prevent increasing the number of
+  // nodes crossing device boundaries.
+  TF_CHECK_OK(TopologicalSort(&item.graph));
+  VerifyGraphsEqual(item.graph, output, __FUNCTION__);
+}
+
 TEST_F(DependencyOptimizerTest, RemoveNoOps_SingleInputOrOutput) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output x = ops::RandomUniform(s.WithOpName("x"), {1, 2}, DT_FLOAT);
-- 
GitLab


From 16fb9b6bd7b67c0954ed3249b65ce32e1a03b929 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 17:30:09 -0700
Subject: [PATCH 721/960] * Added a JavaScript tab to DEVELOP. * Added a
 JavaScript file to DEPLOY. * Added a link to JavaScript API Ref.

PiperOrigin-RevId: 191012951
---
 tensorflow/docs_src/deploy/deploy_to_js.md   | 4 ++++
 tensorflow/docs_src/deploy/leftnav_files     | 1 +
 tensorflow/docs_src/javascript/index.md      | 5 +++++
 tensorflow/docs_src/javascript/leftnav_files | 1 +
 4 files changed, 11 insertions(+)
 create mode 100644 tensorflow/docs_src/deploy/deploy_to_js.md
 create mode 100644 tensorflow/docs_src/javascript/index.md
 create mode 100644 tensorflow/docs_src/javascript/leftnav_files

diff --git a/tensorflow/docs_src/deploy/deploy_to_js.md b/tensorflow/docs_src/deploy/deploy_to_js.md
new file mode 100644
index 0000000000..d7ce3ea90b
--- /dev/null
+++ b/tensorflow/docs_src/deploy/deploy_to_js.md
@@ -0,0 +1,4 @@
+# Deploy to JavaScript
+
+You can find details about deploying JavaScript TensorFlow programs
+in the separate [js.tensorflow.org site](https://js.tensorflow.org).
diff --git a/tensorflow/docs_src/deploy/leftnav_files b/tensorflow/docs_src/deploy/leftnav_files
index c682e7add1..93f5bd1ed2 100644
--- a/tensorflow/docs_src/deploy/leftnav_files
+++ b/tensorflow/docs_src/deploy/leftnav_files
@@ -2,3 +2,4 @@ index.md
 distributed.md
 hadoop.md
 s3.md
+deploy_to_js.md
diff --git a/tensorflow/docs_src/javascript/index.md b/tensorflow/docs_src/javascript/index.md
new file mode 100644
index 0000000000..ad63eeb255
--- /dev/null
+++ b/tensorflow/docs_src/javascript/index.md
@@ -0,0 +1,5 @@
+# JavaScript 
+
+You may develop TensorFlow programs in JavaScript, training and deploying
+models right in your browser.  For details, see
+[js.tensorflow.org](https://js.tensorflow.org).
diff --git a/tensorflow/docs_src/javascript/leftnav_files b/tensorflow/docs_src/javascript/leftnav_files
new file mode 100644
index 0000000000..fc0ab8a543
--- /dev/null
+++ b/tensorflow/docs_src/javascript/leftnav_files
@@ -0,0 +1 @@
+index.md
-- 
GitLab


From 5b2a293b1d43c450ed9e08c1ae5cf6e50daa849d Mon Sep 17 00:00:00 2001
From: msofka <msofka@4catalyzer.com>
Date: Thu, 29 Mar 2018 20:33:58 -0400
Subject: [PATCH 722/960] Fix typo in comment (#18105)

---
 tensorflow/contrib/gan/python/losses/python/losses_impl.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index 39588b7219..c1c9cc5d72 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -665,7 +665,7 @@ def least_squares_discriminator_loss(
     loss_collection=ops.GraphKeys.LOSSES,
     reduction=losses.Reduction.SUM_BY_NONZERO_WEIGHTS,
     add_summaries=False):
-  """Least squares generator loss.
+  """Least squares discriminator loss.
 
   This loss comes from `Least Squares Generative Adversarial Networks`
   (https://arxiv.org/abs/1611.04076).
-- 
GitLab


From ec70d93c3035b6d1b031d0ff22b1493e8b2a67c1 Mon Sep 17 00:00:00 2001
From: Andrew Harp <andrewharp@google.com>
Date: Thu, 29 Mar 2018 17:57:27 -0700
Subject: [PATCH 723/960] Port TensorFlow demo to TFLite with three activities:
 MobileNet Classification, MobileNet SSD Object Detection, and Speech hotword
 classification

PiperOrigin-RevId: 191015617
---
 .../lite/examples/android/AndroidManifest.xml |   65 ++
 .../contrib/lite/examples/android/BUILD       |   86 ++
 .../lite/examples/android/assets/BUILD        |   24 +
 .../examples/android/assets/box_priors.txt    |    5 +
 .../android/assets/coco_labels_list.txt       |   91 ++
 .../android/assets/conv_actions_labels.txt    |   12 +
 .../assets/labels_mobilenet_quant_v1_224.txt  | 1001 +++++++++++++++++
 .../lite/examples/android/build.gradle        |   52 +
 .../android/res/animator/color_animation.xml  |   30 +
 .../res/drawable-hdpi/ic_action_info.png      |  Bin 0 -> 1025 bytes
 .../android/res/drawable-hdpi/ic_launcher.png |  Bin 0 -> 4312 bytes
 .../android/res/drawable-hdpi/tile.9.png      |  Bin 0 -> 196 bytes
 .../res/drawable-mdpi/ic_action_info.png      |  Bin 0 -> 665 bytes
 .../android/res/drawable-mdpi/ic_launcher.png |  Bin 0 -> 2265 bytes
 .../res/drawable-xhdpi/ic_action_info.png     |  Bin 0 -> 1355 bytes
 .../res/drawable-xhdpi/ic_launcher.png        |  Bin 0 -> 6683 bytes
 .../res/drawable-xxhdpi/ic_action_info.png    |  Bin 0 -> 2265 bytes
 .../res/drawable-xxhdpi/ic_launcher.png       |  Bin 0 -> 12746 bytes
 .../examples/android/res/drawable/border.xml  |   19 +
 .../android/res/layout/activity_camera.xml    |   22 +
 .../android/res/layout/activity_speech.xml    |   55 +
 .../res/layout/camera_connection_fragment.xml |   38 +
 .../camera_connection_fragment_stylize.xml    |   51 +
 .../camera_connection_fragment_tracking.xml   |   34 +
 .../android/res/layout/list_text_item.xml     |   25 +
 .../res/values-sw600dp/template-dimens.xml    |   24 +
 .../res/values-sw600dp/template-styles.xml    |   25 +
 .../android/res/values-v11/styles.xml         |   24 +
 .../res/values-v11/template-styles.xml        |   22 +
 .../android/res/values-v14/styles.xml         |   12 +
 .../android/res/values-v21/base-colors.xml    |   21 +
 .../res/values-v21/base-template-styles.xml   |   24 +
 .../examples/android/res/values/attrs.xml     |   14 +
 .../android/res/values/base-strings.xml       |   23 +
 .../examples/android/res/values/colors.xml    |   19 +
 .../examples/android/res/values/strings.xml   |   20 +
 .../examples/android/res/values/styles.xml    |   18 +
 .../android/res/values/template-dimens.xml    |   32 +
 .../android/res/values/template-styles.xml    |   42 +
 .../tensorflow/demo/AutoFitTextureView.java   |   74 ++
 .../org/tensorflow/demo/CameraActivity.java   |  450 ++++++++
 .../demo/CameraConnectionFragment.java        |  634 +++++++++++
 .../src/org/tensorflow/demo/Classifier.java   |  107 ++
 .../tensorflow/demo/ClassifierActivity.java   |  197 ++++
 .../org/tensorflow/demo/DetectorActivity.java |  296 +++++
 .../demo/LegacyCameraConnectionFragment.java  |  216 ++++
 .../src/org/tensorflow/demo/OverlayView.java  |   52 +
 .../tensorflow/demo/RecognitionScoreView.java |   67 ++
 .../tensorflow/demo/RecognizeCommands.java    |  186 +++
 .../src/org/tensorflow/demo/ResultsView.java  |   23 +
 .../org/tensorflow/demo/SpeechActivity.java   |  381 +++++++
 .../demo/TFLiteImageClassifier.java           |  209 ++++
 .../demo/TFLiteObjectDetectionAPIModel.java   |  292 +++++
 .../org/tensorflow/demo/env/AssetUtils.java   |   78 ++
 .../org/tensorflow/demo/env/BorderedText.java |  117 ++
 .../org/tensorflow/demo/env/ImageUtils.java   |  344 ++++++
 .../src/org/tensorflow/demo/env/Logger.java   |  190 ++++
 .../src/org/tensorflow/demo/env/Size.java     |  143 +++
 .../org/tensorflow/demo/env/SplitTimer.java   |   50 +
 .../demo/tracking/MultiBoxTracker.java        |  421 +++++++
 .../demo/tracking/ObjectTracker.java          |  661 +++++++++++
 tensorflow/tools/ci_build/ci_sanity.sh        |    3 +-
 tensorflow/workspace.bzl                      |   20 +
 63 files changed, 7140 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/contrib/lite/examples/android/AndroidManifest.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/BUILD
 create mode 100644 tensorflow/contrib/lite/examples/android/assets/BUILD
 create mode 100644 tensorflow/contrib/lite/examples/android/assets/box_priors.txt
 create mode 100644 tensorflow/contrib/lite/examples/android/assets/coco_labels_list.txt
 create mode 100644 tensorflow/contrib/lite/examples/android/assets/conv_actions_labels.txt
 create mode 100644 tensorflow/contrib/lite/examples/android/assets/labels_mobilenet_quant_v1_224.txt
 create mode 100644 tensorflow/contrib/lite/examples/android/build.gradle
 create mode 100644 tensorflow/contrib/lite/examples/android/res/animator/color_animation.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_action_info.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_launcher.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-hdpi/tile.9.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_action_info.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_launcher.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_action_info.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_launcher.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_action_info.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_launcher.png
 create mode 100644 tensorflow/contrib/lite/examples/android/res/drawable/border.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/activity_camera.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/activity_speech.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_stylize.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_tracking.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/layout/list_text_item.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-dimens.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-v11/styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-v11/template-styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-v14/styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-v21/base-colors.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values-v21/base-template-styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/attrs.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/base-strings.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/colors.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/strings.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/template-dimens.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/res/values/template-styles.xml
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/AutoFitTextureView.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraActivity.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/Classifier.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ClassifierActivity.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/DetectorActivity.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/LegacyCameraConnectionFragment.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/OverlayView.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognitionScoreView.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognizeCommands.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ResultsView.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/SpeechActivity.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteImageClassifier.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteObjectDetectionAPIModel.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/AssetUtils.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/BorderedText.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/ImageUtils.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Logger.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Size.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/SplitTimer.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/MultiBoxTracker.java
 create mode 100644 tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/ObjectTracker.java

diff --git a/tensorflow/contrib/lite/examples/android/AndroidManifest.xml b/tensorflow/contrib/lite/examples/android/AndroidManifest.xml
new file mode 100644
index 0000000000..bc9574d646
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/AndroidManifest.xml
@@ -0,0 +1,65 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<manifest xmlns:android="http://schemas.android.com/apk/res/android"
+    package="org.tensorflow.lite.demo">
+
+    <uses-permission android:name="android.permission.CAMERA" />
+    <uses-feature android:name="android.hardware.camera" />
+    <uses-feature android:name="android.hardware.camera.autofocus" />
+    <uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE"/>
+    <uses-permission android:name="android.permission.RECORD_AUDIO" />
+
+    <uses-sdk
+        android:minSdkVersion="21"
+        android:targetSdkVersion="23" />
+
+    <application android:allowBackup="true"
+        android:debuggable="true"
+        android:label="@string/app_name"
+        android:icon="@drawable/ic_launcher"
+        android:theme="@style/MaterialTheme">
+
+        <activity android:name="org.tensorflow.demo.ClassifierActivity"
+                  android:screenOrientation="portrait"
+                  android:label="@string/activity_name_classification">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+        <activity android:name="org.tensorflow.demo.DetectorActivity"
+                  android:screenOrientation="portrait"
+                  android:label="@string/activity_name_detection">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+
+        <activity android:name="org.tensorflow.demo.SpeechActivity"
+            android:screenOrientation="portrait"
+            android:label="@string/activity_name_speech">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+    </application>
+
+</manifest>
diff --git a/tensorflow/contrib/lite/examples/android/BUILD b/tensorflow/contrib/lite/examples/android/BUILD
new file mode 100644
index 0000000000..4928012997
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/BUILD
@@ -0,0 +1,86 @@
+# Description:
+#   TensorFlow camera demo app for Android.
+
+package(default_visibility = ["//visibility:public"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+# Build the demo native demo lib from the original directory to reduce code
+# reuse. Note that the Java counterparts (ObjectTracker.java and
+# ImageUtils.java) are still duplicated.
+cc_library(
+    name = "tensorflow_native_libs",
+    srcs = [
+        "//tensorflow/examples/android:libtensorflow_demo.so",
+    ],
+    tags = [
+        "manual",
+        "notap",
+    ],
+)
+
+android_binary(
+    name = "tflite_demo",
+    srcs = glob([
+        "src/**/*.java",
+    ]),
+    # Package assets from assets dir as well as all model targets.
+    # Remove undesired models (and corresponding Activities in source)
+    # to reduce APK size.
+    assets = [
+        "//tensorflow/contrib/lite/examples/android/assets:labels_mobilenet_quant_v1_224.txt",
+        "@tflite_mobilenet//:mobilenet_quant_v1_224.tflite",
+        "@tflite_conv_actions_frozen//:conv_actions_frozen.tflite",
+        "//tensorflow/contrib/lite/examples/android/assets:conv_actions_labels.txt",
+        "@tflite_mobilenet_ssd//:mobilenet_ssd.tflite",
+        "//tensorflow/contrib/lite/examples/android/assets:box_priors.txt",
+        "//tensorflow/contrib/lite/examples/android/assets:coco_labels_list.txt",
+    ],
+    assets_dir = "",
+    custom_package = "org.tensorflow.lite.demo",
+    inline_constants = 1,
+    manifest = "AndroidManifest.xml",
+    manifest_merger = "android",
+    nocompress_extensions = [
+        ".tflite",
+    ],
+    resource_files = glob(["res/**"]),
+    tags = [
+        "manual",
+        "notap",
+    ],
+    deps = [
+        ":tensorflow_native_libs",
+        "//tensorflow/contrib/lite/java:tensorflowlite",
+    ],
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+            "bin/**",
+            "gen/**",
+            "gradleBuild/**",
+            "libs/**",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+filegroup(
+    name = "java_files",
+    srcs = glob(["src/**/*.java"]),
+)
+
+filegroup(
+    name = "resource_files",
+    srcs = glob(["res/**"]),
+)
+
+exports_files(["AndroidManifest.xml"])
diff --git a/tensorflow/contrib/lite/examples/android/assets/BUILD b/tensorflow/contrib/lite/examples/android/assets/BUILD
new file mode 100644
index 0000000000..dd0cd6c98f
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/assets/BUILD
@@ -0,0 +1,24 @@
+package(default_visibility = ["//visibility:private"])
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(
+    glob(
+        ["**/*"],
+        exclude = [
+            "BUILD",
+        ],
+    ),
+)
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
diff --git a/tensorflow/contrib/lite/examples/android/assets/box_priors.txt b/tensorflow/contrib/lite/examples/android/assets/box_priors.txt
new file mode 100644
index 0000000000..7246b073fe
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/assets/box_priors.txt
@@ -0,0 +1,5 @@
+        0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.02631579 0.02631579 0.026315793 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.078947365 0.07894737 0.078947365 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.13157895 0.13157895 0.13157894 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.18421052 0.18421051 0.18421052 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.23684211 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.28947368 0.28947368 0.28947365 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.34210524 0.34210524 0.3421052 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.92105263 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.97368425 0.9736843 0.97368425 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.049999997 0.049999997 0.049999997 0.05 0.050000012 0.049999997 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.25 0.25 0.25 0.25 0.25000003 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000005 0.35000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.45 0.45000002 0.45000002 0.45000002 0.45000002 0.45000002 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.55 0.55 0.55 0.55 0.54999995 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.099999994 0.1 0.099999994 0.1 0.099999994 0.099999994 0.099999994 0.1 0.099999994 0.1 0.099999994 0.099999994 0.099999994 0.1 0.099999994 0.1 0.099999994 0.099999994 0.099999994 0.1 0.099999994 0.1 0.099999994 0.099999994 0.099999994 0.1 0.099999994 0.1 0.099999994 0.099999994 0.30000004 0.3 0.3 0.3 0.3 0.30000004 0.30000004 0.3 0.3 0.3 0.3 0.30000004 0.30000004 0.3 0.3 0.3 0.3 0.30000004 0.30000004 0.3 0.3 0.3 0.3 0.30000004 0.30000004 0.3 0.3 0.3 0.3 0.30000004 0.49999997 0.5 0.5 0.5 0.5 0.49999997 0.49999997 0.5 0.5 0.5 0.5 0.49999997 0.49999997 0.5 0.5 0.5 0.5 0.49999997 0.49999997 0.5 0.5 0.5 0.5 0.49999997 0.49999997 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.90000004 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.90000004 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.90000004 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.90000004 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.90000004 0.16666667 0.16666667 0.16666666 0.16666667 0.16666669 0.16666667 0.16666667 0.16666667 0.16666666 0.16666667 0.16666669 0.16666667 0.16666667 0.16666667 0.16666666 0.16666667 0.16666669 0.16666667 0.5 0.5 0.49999997 0.5 0.5 0.5 0.5 0.5 0.49999997 0.5 0.5 0.5 0.5 0.5 0.49999997 0.5 0.5 0.5 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.8333334 0.25 0.25 0.25 0.24999999 0.25 0.25 0.25 0.25 0.25 0.24999999 0.25 0.25 0.75 0.75 0.75 0.75 0.74999994 0.75 0.75 0.75 0.75 0.75 0.74999994 0.75 0.5 0.5 0.5 0.5 0.5 0.5 
+        0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.02631579 0.026315793 0.02631579 0.078947365 0.078947365 0.07894737 0.13157895 0.13157894 0.13157895 0.18421052 0.18421052 0.18421051 0.23684211 0.23684211 0.23684211 0.28947368 0.28947365 0.28947368 0.34210524 0.3421052 0.34210524 0.39473683 0.39473683 0.39473683 0.4473684 0.4473684 0.4473684 0.5 0.5 0.5 0.5526316 0.5526316 0.5526316 0.6052632 0.6052632 0.6052632 0.65789473 0.65789473 0.65789473 0.71052635 0.71052635 0.71052635 0.7631579 0.7631579 0.7631579 0.8157895 0.8157895 0.8157895 0.8684211 0.8684211 0.8684211 0.92105263 0.92105263 0.92105263 0.97368425 0.97368425 0.9736843 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.049999997 0.049999997 0.050000004 0.050000012 0.05 0.049999997 0.15 0.14999999 0.15 0.15 0.15 0.15 0.25 0.25 0.25 0.25 0.25 0.25 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.35000002 0.45000002 0.45 0.45000002 0.45000002 0.45 0.45000002 0.55 0.55 0.55 0.55 0.55 0.55 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.65000004 0.75 0.75 0.75 0.75 0.75 0.75 0.85 0.85 0.85 0.85 0.85 0.85 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.95000005 0.10000001 0.099999994 0.1 0.099999994 0.1 0.099999994 0.3 0.3 0.3 0.29999998 0.3 0.30000004 0.5 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.9 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.10000001 0.099999994 0.1 0.099999994 0.1 0.099999994 0.3 0.3 0.3 0.29999998 0.3 0.30000004 0.5 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.9 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.10000001 0.099999994 0.1 0.099999994 0.1 0.099999994 0.3 0.3 0.3 0.29999998 0.3 0.30000004 0.5 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.9 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.10000001 0.099999994 0.1 0.099999994 0.1 0.099999994 0.3 0.3 0.3 0.29999998 0.3 0.30000004 0.5 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.9 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.10000001 0.099999994 0.1 0.099999994 0.1 0.099999994 0.3 0.3 0.3 0.29999998 0.3 0.30000004 0.5 0.5 0.5 0.5 0.5 0.49999997 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.70000005 0.9 0.90000004 0.90000004 0.9 0.90000004 0.90000004 0.16666667 0.16666669 0.16666667 0.16666669 0.16666667 0.16666667 0.49999997 0.5 0.5 0.50000006 0.5 0.5 0.8333334 0.8333334 0.8333334 0.8333333 0.8333334 0.8333334 0.16666667 0.16666669 0.16666667 0.16666669 0.16666667 0.16666667 0.49999997 0.5 0.5 0.50000006 0.5 0.5 0.8333334 0.8333334 0.8333334 0.8333333 0.8333334 0.8333334 0.16666667 0.16666669 0.16666667 0.16666669 0.16666667 0.16666667 0.49999997 0.5 0.5 0.50000006 0.5 0.5 0.8333334 0.8333334 0.8333334 0.8333333 0.8333334 0.8333334 0.25 0.25 0.25 0.25 0.25 0.25 0.75 0.75 0.75 0.75 0.75 0.75 0.25 0.25 0.25 0.25 0.25 0.25 0.75 0.75 0.75 0.75 0.75 0.75 0.5 0.5 0.5 0.5 0.5 0.5 
+        0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.1 0.14142136 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.28284273 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142138 0.2828427 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.099999994 0.14142135 0.28284273 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.10000001 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142138 0.2828427 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142135 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.2828427 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142132 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.100000024 0.14142138 0.28284276 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.2474874 0.4949748 0.20207259 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748738 0.4949748 0.20207258 0.6062481 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.4949748 0.2020726 0.60624814 0.41833 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35000002 0.24748741 0.49497482 0.2020726 0.60624814 0.41832998 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35 0.24748737 0.4949748 0.20207256 0.6062481 0.41833 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497476 0.20207262 0.606248 0.41833004 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000002 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.35000008 0.24748743 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.34999996 0.24748737 0.49497485 0.20207262 0.60624814 0.41832995 0.50000006 0.3535534 0.7071068 0.28867513 0.8660687 0.57008773 0.50000006 0.3535534 0.7071068 0.28867513 0.8660687 0.57008773 0.50000006 0.3535534 0.7071068 0.28867513 0.8660687 0.57008773 0.50000006 0.3535534 0.7071068 0.28867513 0.8660687 0.57008773 0.50000006 0.3535534 0.7071068 0.28867513 0.8660687 0.57008773 0.5000001 0.3535534 0.7071068 0.28867513 0.8660687 0.5700878 0.5000001 0.3535534 0.7071068 0.28867513 0.8660687 0.5700878 0.5000001 0.3535534 0.7071068 0.28867513 0.8660687 0.5700878 0.5000001 0.3535534 0.7071068 0.28867513 0.8660687 0.5700878 0.5000001 0.3535534 0.7071068 0.28867513 0.8660687 0.5700878 0.5 0.3535534 0.7071068 0.2886751 0.8660687 0.5700877 0.5 0.3535534 0.7071068 0.2886751 0.8660687 0.5700877 0.5 0.3535534 0.7071068 0.2886751 0.8660687 0.5700877 0.5 0.3535534 0.7071068 0.2886751 0.8660687 0.5700877 0.5 0.3535534 0.7071068 0.2886751 0.8660687 0.5700877 0.5 0.3535534 0.7071068 0.28867507 0.8660688 0.5700877 0.5 0.3535534 0.7071068 0.28867507 0.8660688 0.5700877 0.5 0.3535534 0.7071068 0.28867507 0.8660688 0.5700877 0.5 0.3535534 0.7071068 0.28867507 0.8660688 0.5700877 0.5 0.3535534 0.7071068 0.28867507 0.8660688 0.5700877 0.5000001 0.3535534 0.70710677 0.2886752 0.8660687 0.5700878 0.5000001 0.3535534 0.70710677 0.2886752 0.8660687 0.5700878 0.5000001 0.3535534 0.70710677 0.2886752 0.8660687 0.5700878 0.5000001 0.3535534 0.70710677 0.2886752 0.8660687 0.5700878 0.5000001 0.3535534 0.70710677 0.2886752 0.8660687 0.5700878 0.65000004 0.45961943 0.91923887 0.37527767 1.1258893 0.7211102 0.65000004 0.45961943 0.91923887 0.37527767 1.1258893 0.7211102 0.65000004 0.45961943 0.91923887 0.37527767 1.1258893 0.7211102 0.6500001 0.4596194 0.9192388 0.37527764 1.1258893 0.7211102 0.6500001 0.4596194 0.9192388 0.37527764 1.1258893 0.7211102 0.6500001 0.4596194 0.9192388 0.37527764 1.1258893 0.7211102 0.6500001 0.45961946 0.9192388 0.3752777 1.1258893 0.72111017 0.6500001 0.45961946 0.9192388 0.3752777 1.1258893 0.72111017 0.6500001 0.45961946 0.9192388 0.3752777 1.1258893 0.72111017 0.8000001 0.5656855 1.131371 0.4618802 1.3857099 0.8717798 0.8000001 0.5656855 1.131371 0.4618802 1.3857099 0.8717798 0.80000013 0.5656855 1.131371 0.4618802 1.3857098 0.87177986 0.80000013 0.5656855 1.131371 0.4618802 1.3857098 0.87177986 0.95000005 0.6717515 1.343503 0.5484828 1.6455305 0.97467947 
+        0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.1 0.28284273 0.14142136 0.099999994 0.28284273 0.14142138 0.099999994 0.2828427 0.14142138 0.099999994 0.28284273 0.14142135 0.099999994 0.28284273 0.14142135 0.10000001 0.2828427 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142138 0.100000024 0.2828427 0.14142138 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142135 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.2828427 0.14142132 0.100000024 0.28284276 0.14142132 0.100000024 0.28284276 0.14142138 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.34999996 0.4949747 0.24748735 0.60621774 0.20206249 0.41833 0.34999996 0.49497467 0.24748737 0.60621774 0.20206249 0.41833 0.34999996 0.49497473 0.24748737 0.60621774 0.20206249 0.41833 0.34999993 0.49497473 0.24748737 0.60621774 0.20206249 0.41832998 0.34999996 0.49497467 0.24748737 0.60621774 0.20206246 0.41833 0.35 0.49497473 0.24748734 0.60621774 0.20206249 0.41833004 0.35 0.49497473 0.2474873 0.60621774 0.20206249 0.41833004 0.3499999 0.49497473 0.2474873 0.6062178 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062177 0.20206249 0.41832995 0.3499999 0.49497467 0.2474873 0.6062178 0.20206255 0.41832995 0.49999997 0.7071067 0.35355335 0.8660254 0.2886607 0.57008773 0.5 0.7071067 0.35355335 0.8660253 0.2886607 0.5700878 0.5 0.7071067 0.35355332 0.86602545 0.28866073 0.5700877 0.5 0.70710665 0.3535533 0.86602545 0.28866076 0.5700877 0.49999994 0.7071067 0.3535534 0.8660253 0.28866065 0.5700878 0.49999997 0.7071067 0.35355335 0.8660254 0.2886607 0.57008773 0.5 0.7071067 0.35355335 0.8660253 0.2886607 0.5700878 0.5 0.7071067 0.35355332 0.86602545 0.28866073 0.5700877 0.5 0.70710665 0.3535533 0.86602545 0.28866076 0.5700877 0.49999994 0.7071067 0.3535534 0.8660253 0.28866065 0.5700878 0.49999997 0.7071067 0.35355335 0.8660254 0.2886607 0.57008773 0.5 0.7071067 0.35355335 0.8660253 0.2886607 0.5700878 0.5 0.7071067 0.35355332 0.86602545 0.28866073 0.5700877 0.5 0.70710665 0.3535533 0.86602545 0.28866076 0.5700877 0.49999994 0.7071067 0.3535534 0.8660253 0.28866065 0.5700878 0.49999997 0.7071067 0.35355335 0.8660254 0.2886607 0.57008773 0.5 0.7071067 0.35355335 0.8660253 0.2886607 0.5700878 0.5 0.7071067 0.35355332 0.86602545 0.28866073 0.5700877 0.5 0.70710665 0.3535533 0.86602545 0.28866076 0.5700877 0.49999994 0.7071067 0.3535534 0.8660253 0.28866065 0.5700878 0.49999997 0.7071067 0.35355335 0.8660254 0.2886607 0.57008773 0.5 0.7071067 0.35355335 0.8660253 0.2886607 0.5700878 0.5 0.7071067 0.35355332 0.86602545 0.28866073 0.5700877 0.5 0.70710665 0.3535533 0.86602545 0.28866076 0.5700877 0.49999994 0.7071067 0.3535534 0.8660253 0.28866065 0.5700878 0.6499999 0.9192387 0.45961934 1.1258329 0.3752589 0.7211102 0.64999986 0.9192387 0.4596193 1.125833 0.37525892 0.7211102 0.64999986 0.91923875 0.45961928 1.1258328 0.37525892 0.72111017 0.6499999 0.9192387 0.45961934 1.1258329 0.3752589 0.7211102 0.64999986 0.9192387 0.4596193 1.125833 0.37525892 0.7211102 0.64999986 0.91923875 0.45961928 1.1258328 0.37525892 0.72111017 0.6499999 0.9192387 0.45961934 1.1258329 0.3752589 0.7211102 0.64999986 0.9192387 0.4596193 1.125833 0.37525892 0.7211102 0.64999986 0.91923875 0.45961928 1.1258328 0.37525892 0.72111017 0.79999995 1.1313708 0.5656854 1.3856406 0.46185714 0.8717798 0.79999995 1.1313708 0.56568533 1.3856406 0.46185708 0.87177986 0.79999995 1.1313708 0.5656854 1.3856406 0.46185714 0.8717798 0.79999995 1.1313708 0.56568533 1.3856406 0.46185708 0.87177986 0.9499999 1.3435028 0.6717514 1.6454482 0.54845536 0.97467947 
+
diff --git a/tensorflow/contrib/lite/examples/android/assets/coco_labels_list.txt b/tensorflow/contrib/lite/examples/android/assets/coco_labels_list.txt
new file mode 100644
index 0000000000..5a70ff82aa
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/assets/coco_labels_list.txt
@@ -0,0 +1,91 @@
+???
+person
+bicycle
+car
+motorcycle
+airplane
+bus
+train
+truck
+boat
+traffic light
+fire hydrant
+???
+stop sign
+parking meter
+bench
+bird
+cat
+dog
+horse
+sheep
+cow
+elephant
+bear
+zebra
+giraffe
+???
+backpack
+umbrella
+???
+???
+handbag
+tie
+suitcase
+frisbee
+skis
+snowboard
+sports ball
+kite
+baseball bat
+baseball glove
+skateboard
+surfboard
+tennis racket
+bottle
+???
+wine glass
+cup
+fork
+knife
+spoon
+bowl
+banana
+apple
+sandwich
+orange
+broccoli
+carrot
+hot dog
+pizza
+donut
+cake
+chair
+couch
+potted plant
+bed
+???
+dining table
+???
+???
+toilet
+???
+tv
+laptop
+mouse
+remote
+keyboard
+cell phone
+microwave
+oven
+toaster
+sink
+refrigerator
+???
+book
+clock
+vase
+scissors
+teddy bear
+hair drier
+toothbrush
diff --git a/tensorflow/contrib/lite/examples/android/assets/conv_actions_labels.txt b/tensorflow/contrib/lite/examples/android/assets/conv_actions_labels.txt
new file mode 100644
index 0000000000..ba416458b0
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/assets/conv_actions_labels.txt
@@ -0,0 +1,12 @@
+_silence_
+_unknown_
+yes
+no
+up
+down
+left
+right
+on
+off
+stop
+go
\ No newline at end of file
diff --git a/tensorflow/contrib/lite/examples/android/assets/labels_mobilenet_quant_v1_224.txt b/tensorflow/contrib/lite/examples/android/assets/labels_mobilenet_quant_v1_224.txt
new file mode 100644
index 0000000000..fe811239d8
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/assets/labels_mobilenet_quant_v1_224.txt
@@ -0,0 +1,1001 @@
+background
+tench
+goldfish
+great white shark
+tiger shark
+hammerhead
+electric ray
+stingray
+cock
+hen
+ostrich
+brambling
+goldfinch
+house finch
+junco
+indigo bunting
+robin
+bulbul
+jay
+magpie
+chickadee
+water ouzel
+kite
+bald eagle
+vulture
+great grey owl
+European fire salamander
+common newt
+eft
+spotted salamander
+axolotl
+bullfrog
+tree frog
+tailed frog
+loggerhead
+leatherback turtle
+mud turtle
+terrapin
+box turtle
+banded gecko
+common iguana
+American chameleon
+whiptail
+agama
+frilled lizard
+alligator lizard
+Gila monster
+green lizard
+African chameleon
+Komodo dragon
+African crocodile
+American alligator
+triceratops
+thunder snake
+ringneck snake
+hognose snake
+green snake
+king snake
+garter snake
+water snake
+vine snake
+night snake
+boa constrictor
+rock python
+Indian cobra
+green mamba
+sea snake
+horned viper
+diamondback
+sidewinder
+trilobite
+harvestman
+scorpion
+black and gold garden spider
+barn spider
+garden spider
+black widow
+tarantula
+wolf spider
+tick
+centipede
+black grouse
+ptarmigan
+ruffed grouse
+prairie chicken
+peacock
+quail
+partridge
+African grey
+macaw
+sulphur-crested cockatoo
+lorikeet
+coucal
+bee eater
+hornbill
+hummingbird
+jacamar
+toucan
+drake
+red-breasted merganser
+goose
+black swan
+tusker
+echidna
+platypus
+wallaby
+koala
+wombat
+jellyfish
+sea anemone
+brain coral
+flatworm
+nematode
+conch
+snail
+slug
+sea slug
+chiton
+chambered nautilus
+Dungeness crab
+rock crab
+fiddler crab
+king crab
+American lobster
+spiny lobster
+crayfish
+hermit crab
+isopod
+white stork
+black stork
+spoonbill
+flamingo
+little blue heron
+American egret
+bittern
+crane
+limpkin
+European gallinule
+American coot
+bustard
+ruddy turnstone
+red-backed sandpiper
+redshank
+dowitcher
+oystercatcher
+pelican
+king penguin
+albatross
+grey whale
+killer whale
+dugong
+sea lion
+Chihuahua
+Japanese spaniel
+Maltese dog
+Pekinese
+Shih-Tzu
+Blenheim spaniel
+papillon
+toy terrier
+Rhodesian ridgeback
+Afghan hound
+basset
+beagle
+bloodhound
+bluetick
+black-and-tan coonhound
+Walker hound
+English foxhound
+redbone
+borzoi
+Irish wolfhound
+Italian greyhound
+whippet
+Ibizan hound
+Norwegian elkhound
+otterhound
+Saluki
+Scottish deerhound
+Weimaraner
+Staffordshire bullterrier
+American Staffordshire terrier
+Bedlington terrier
+Border terrier
+Kerry blue terrier
+Irish terrier
+Norfolk terrier
+Norwich terrier
+Yorkshire terrier
+wire-haired fox terrier
+Lakeland terrier
+Sealyham terrier
+Airedale
+cairn
+Australian terrier
+Dandie Dinmont
+Boston bull
+miniature schnauzer
+giant schnauzer
+standard schnauzer
+Scotch terrier
+Tibetan terrier
+silky terrier
+soft-coated wheaten terrier
+West Highland white terrier
+Lhasa
+flat-coated retriever
+curly-coated retriever
+golden retriever
+Labrador retriever
+Chesapeake Bay retriever
+German short-haired pointer
+vizsla
+English setter
+Irish setter
+Gordon setter
+Brittany spaniel
+clumber
+English springer
+Welsh springer spaniel
+cocker spaniel
+Sussex spaniel
+Irish water spaniel
+kuvasz
+schipperke
+groenendael
+malinois
+briard
+kelpie
+komondor
+Old English sheepdog
+Shetland sheepdog
+collie
+Border collie
+Bouvier des Flandres
+Rottweiler
+German shepherd
+Doberman
+miniature pinscher
+Greater Swiss Mountain dog
+Bernese mountain dog
+Appenzeller
+EntleBucher
+boxer
+bull mastiff
+Tibetan mastiff
+French bulldog
+Great Dane
+Saint Bernard
+Eskimo dog
+malamute
+Siberian husky
+dalmatian
+affenpinscher
+basenji
+pug
+Leonberg
+Newfoundland
+Great Pyrenees
+Samoyed
+Pomeranian
+chow
+keeshond
+Brabancon griffon
+Pembroke
+Cardigan
+toy poodle
+miniature poodle
+standard poodle
+Mexican hairless
+timber wolf
+white wolf
+red wolf
+coyote
+dingo
+dhole
+African hunting dog
+hyena
+red fox
+kit fox
+Arctic fox
+grey fox
+tabby
+tiger cat
+Persian cat
+Siamese cat
+Egyptian cat
+cougar
+lynx
+leopard
+snow leopard
+jaguar
+lion
+tiger
+cheetah
+brown bear
+American black bear
+ice bear
+sloth bear
+mongoose
+meerkat
+tiger beetle
+ladybug
+ground beetle
+long-horned beetle
+leaf beetle
+dung beetle
+rhinoceros beetle
+weevil
+fly
+bee
+ant
+grasshopper
+cricket
+walking stick
+cockroach
+mantis
+cicada
+leafhopper
+lacewing
+dragonfly
+damselfly
+admiral
+ringlet
+monarch
+cabbage butterfly
+sulphur butterfly
+lycaenid
+starfish
+sea urchin
+sea cucumber
+wood rabbit
+hare
+Angora
+hamster
+porcupine
+fox squirrel
+marmot
+beaver
+guinea pig
+sorrel
+zebra
+hog
+wild boar
+warthog
+hippopotamus
+ox
+water buffalo
+bison
+ram
+bighorn
+ibex
+hartebeest
+impala
+gazelle
+Arabian camel
+llama
+weasel
+mink
+polecat
+black-footed ferret
+otter
+skunk
+badger
+armadillo
+three-toed sloth
+orangutan
+gorilla
+chimpanzee
+gibbon
+siamang
+guenon
+patas
+baboon
+macaque
+langur
+colobus
+proboscis monkey
+marmoset
+capuchin
+howler monkey
+titi
+spider monkey
+squirrel monkey
+Madagascar cat
+indri
+Indian elephant
+African elephant
+lesser panda
+giant panda
+barracouta
+eel
+coho
+rock beauty
+anemone fish
+sturgeon
+gar
+lionfish
+puffer
+abacus
+abaya
+academic gown
+accordion
+acoustic guitar
+aircraft carrier
+airliner
+airship
+altar
+ambulance
+amphibian
+analog clock
+apiary
+apron
+ashcan
+assault rifle
+backpack
+bakery
+balance beam
+balloon
+ballpoint
+Band Aid
+banjo
+bannister
+barbell
+barber chair
+barbershop
+barn
+barometer
+barrel
+barrow
+baseball
+basketball
+bassinet
+bassoon
+bathing cap
+bath towel
+bathtub
+beach wagon
+beacon
+beaker
+bearskin
+beer bottle
+beer glass
+bell cote
+bib
+bicycle-built-for-two
+bikini
+binder
+binoculars
+birdhouse
+boathouse
+bobsled
+bolo tie
+bonnet
+bookcase
+bookshop
+bottlecap
+bow
+bow tie
+brass
+brassiere
+breakwater
+breastplate
+broom
+bucket
+buckle
+bulletproof vest
+bullet train
+butcher shop
+cab
+caldron
+candle
+cannon
+canoe
+can opener
+cardigan
+car mirror
+carousel
+carpenter's kit
+carton
+car wheel
+cash machine
+cassette
+cassette player
+castle
+catamaran
+CD player
+cello
+cellular telephone
+chain
+chainlink fence
+chain mail
+chain saw
+chest
+chiffonier
+chime
+china cabinet
+Christmas stocking
+church
+cinema
+cleaver
+cliff dwelling
+cloak
+clog
+cocktail shaker
+coffee mug
+coffeepot
+coil
+combination lock
+computer keyboard
+confectionery
+container ship
+convertible
+corkscrew
+cornet
+cowboy boot
+cowboy hat
+cradle
+crane
+crash helmet
+crate
+crib
+Crock Pot
+croquet ball
+crutch
+cuirass
+dam
+desk
+desktop computer
+dial telephone
+diaper
+digital clock
+digital watch
+dining table
+dishrag
+dishwasher
+disk brake
+dock
+dogsled
+dome
+doormat
+drilling platform
+drum
+drumstick
+dumbbell
+Dutch oven
+electric fan
+electric guitar
+electric locomotive
+entertainment center
+envelope
+espresso maker
+face powder
+feather boa
+file
+fireboat
+fire engine
+fire screen
+flagpole
+flute
+folding chair
+football helmet
+forklift
+fountain
+fountain pen
+four-poster
+freight car
+French horn
+frying pan
+fur coat
+garbage truck
+gasmask
+gas pump
+goblet
+go-kart
+golf ball
+golfcart
+gondola
+gong
+gown
+grand piano
+greenhouse
+grille
+grocery store
+guillotine
+hair slide
+hair spray
+half track
+hammer
+hamper
+hand blower
+hand-held computer
+handkerchief
+hard disc
+harmonica
+harp
+harvester
+hatchet
+holster
+home theater
+honeycomb
+hook
+hoopskirt
+horizontal bar
+horse cart
+hourglass
+iPod
+iron
+jack-o'-lantern
+jean
+jeep
+jersey
+jigsaw puzzle
+jinrikisha
+joystick
+kimono
+knee pad
+knot
+lab coat
+ladle
+lampshade
+laptop
+lawn mower
+lens cap
+letter opener
+library
+lifeboat
+lighter
+limousine
+liner
+lipstick
+Loafer
+lotion
+loudspeaker
+loupe
+lumbermill
+magnetic compass
+mailbag
+mailbox
+maillot
+maillot
+manhole cover
+maraca
+marimba
+mask
+matchstick
+maypole
+maze
+measuring cup
+medicine chest
+megalith
+microphone
+microwave
+military uniform
+milk can
+minibus
+miniskirt
+minivan
+missile
+mitten
+mixing bowl
+mobile home
+Model T
+modem
+monastery
+monitor
+moped
+mortar
+mortarboard
+mosque
+mosquito net
+motor scooter
+mountain bike
+mountain tent
+mouse
+mousetrap
+moving van
+muzzle
+nail
+neck brace
+necklace
+nipple
+notebook
+obelisk
+oboe
+ocarina
+odometer
+oil filter
+organ
+oscilloscope
+overskirt
+oxcart
+oxygen mask
+packet
+paddle
+paddlewheel
+padlock
+paintbrush
+pajama
+palace
+panpipe
+paper towel
+parachute
+parallel bars
+park bench
+parking meter
+passenger car
+patio
+pay-phone
+pedestal
+pencil box
+pencil sharpener
+perfume
+Petri dish
+photocopier
+pick
+pickelhaube
+picket fence
+pickup
+pier
+piggy bank
+pill bottle
+pillow
+ping-pong ball
+pinwheel
+pirate
+pitcher
+plane
+planetarium
+plastic bag
+plate rack
+plow
+plunger
+Polaroid camera
+pole
+police van
+poncho
+pool table
+pop bottle
+pot
+potter's wheel
+power drill
+prayer rug
+printer
+prison
+projectile
+projector
+puck
+punching bag
+purse
+quill
+quilt
+racer
+racket
+radiator
+radio
+radio telescope
+rain barrel
+recreational vehicle
+reel
+reflex camera
+refrigerator
+remote control
+restaurant
+revolver
+rifle
+rocking chair
+rotisserie
+rubber eraser
+rugby ball
+rule
+running shoe
+safe
+safety pin
+saltshaker
+sandal
+sarong
+sax
+scabbard
+scale
+school bus
+schooner
+scoreboard
+screen
+screw
+screwdriver
+seat belt
+sewing machine
+shield
+shoe shop
+shoji
+shopping basket
+shopping cart
+shovel
+shower cap
+shower curtain
+ski
+ski mask
+sleeping bag
+slide rule
+sliding door
+slot
+snorkel
+snowmobile
+snowplow
+soap dispenser
+soccer ball
+sock
+solar dish
+sombrero
+soup bowl
+space bar
+space heater
+space shuttle
+spatula
+speedboat
+spider web
+spindle
+sports car
+spotlight
+stage
+steam locomotive
+steel arch bridge
+steel drum
+stethoscope
+stole
+stone wall
+stopwatch
+stove
+strainer
+streetcar
+stretcher
+studio couch
+stupa
+submarine
+suit
+sundial
+sunglass
+sunglasses
+sunscreen
+suspension bridge
+swab
+sweatshirt
+swimming trunks
+swing
+switch
+syringe
+table lamp
+tank
+tape player
+teapot
+teddy
+television
+tennis ball
+thatch
+theater curtain
+thimble
+thresher
+throne
+tile roof
+toaster
+tobacco shop
+toilet seat
+torch
+totem pole
+tow truck
+toyshop
+tractor
+trailer truck
+tray
+trench coat
+tricycle
+trimaran
+tripod
+triumphal arch
+trolleybus
+trombone
+tub
+turnstile
+typewriter keyboard
+umbrella
+unicycle
+upright
+vacuum
+vase
+vault
+velvet
+vending machine
+vestment
+viaduct
+violin
+volleyball
+waffle iron
+wall clock
+wallet
+wardrobe
+warplane
+washbasin
+washer
+water bottle
+water jug
+water tower
+whiskey jug
+whistle
+wig
+window screen
+window shade
+Windsor tie
+wine bottle
+wing
+wok
+wooden spoon
+wool
+worm fence
+wreck
+yawl
+yurt
+web site
+comic book
+crossword puzzle
+street sign
+traffic light
+book jacket
+menu
+plate
+guacamole
+consomme
+hot pot
+trifle
+ice cream
+ice lolly
+French loaf
+bagel
+pretzel
+cheeseburger
+hotdog
+mashed potato
+head cabbage
+broccoli
+cauliflower
+zucchini
+spaghetti squash
+acorn squash
+butternut squash
+cucumber
+artichoke
+bell pepper
+cardoon
+mushroom
+Granny Smith
+strawberry
+orange
+lemon
+fig
+pineapple
+banana
+jackfruit
+custard apple
+pomegranate
+hay
+carbonara
+chocolate sauce
+dough
+meat loaf
+pizza
+potpie
+burrito
+red wine
+espresso
+cup
+eggnog
+alp
+bubble
+cliff
+coral reef
+geyser
+lakeside
+promontory
+sandbar
+seashore
+valley
+volcano
+ballplayer
+groom
+scuba diver
+rapeseed
+daisy
+yellow lady's slipper
+corn
+acorn
+hip
+buckeye
+coral fungus
+agaric
+gyromitra
+stinkhorn
+earthstar
+hen-of-the-woods
+bolete
+ear
+toilet tissue
diff --git a/tensorflow/contrib/lite/examples/android/build.gradle b/tensorflow/contrib/lite/examples/android/build.gradle
new file mode 100644
index 0000000000..0d4de35815
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/build.gradle
@@ -0,0 +1,52 @@
+apply plugin: 'com.android.application'
+
+android {
+    compileSdkVersion 26
+    buildToolsVersion "26.0.1"
+    defaultConfig {
+        applicationId "org.tensorflow.lite.demo"
+        minSdkVersion 15
+        targetSdkVersion 26
+        versionCode 1
+        versionName "1.0"
+        testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
+
+        // Remove this block.
+        jackOptions {
+            enabled true
+        }
+    }
+    lintOptions {
+        abortOnError false
+    }
+    buildTypes {
+        release {
+            minifyEnabled false
+            proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro'
+        }
+    }
+    aaptOptions {
+        noCompress "tflite"
+    }
+
+    compileOptions {
+        sourceCompatibility JavaVersion.VERSION_1_8
+        targetCompatibility JavaVersion.VERSION_1_8
+    }
+}
+
+repositories {
+    maven {
+        url 'https://google.bintray.com/tensorflow'
+    }
+}
+
+dependencies {
+    compile fileTree(dir: 'libs', include: ['*.jar'])
+    androidTestCompile('com.android.support.test.espresso:espresso-core:2.2.2', {
+        exclude group: 'com.android.support', module: 'support-annotations'
+    })
+    compile 'org.tensorflow:tensorflow-lite:+'
+
+    testCompile 'junit:junit:4.12'
+}
diff --git a/tensorflow/contrib/lite/examples/android/res/animator/color_animation.xml b/tensorflow/contrib/lite/examples/android/res/animator/color_animation.xml
new file mode 100644
index 0000000000..891d8cc1d4
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/animator/color_animation.xml
@@ -0,0 +1,30 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<set xmlns:android="http://schemas.android.com/apk/res/android"
+  android:ordering="sequentially">
+  <objectAnimator
+    android:propertyName="backgroundColor"
+    android:duration="375"
+    android:valueFrom="0x00b3ccff"
+    android:valueTo="0xffb3ccff"
+    android:valueType="colorType"/>
+  <objectAnimator
+    android:propertyName="backgroundColor"
+    android:duration="375"
+    android:valueFrom="0xffb3ccff"
+    android:valueTo="0x00b3ccff"
+    android:valueType="colorType"/>
+</set>
diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_action_info.png b/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_action_info.png
new file mode 100644
index 0000000000000000000000000000000000000000..32bd1aabcabb85ded957230533c00e735183a323
GIT binary patch
literal 1025
zcmeAS@N?(olHy`uVBq!ia0y~yU@!n-4mJh`hH$2z?F<aeCp=voLo%G-&hYn`oG5d=
z*4#ZIpuOYi0{gZSt!HVQnuHXa`b0A_C9f#H3TWrrv>=$BdzG`GOS_eX8|PKS4hItr
z!{x#Lf*uwwlB}#^M%%ysJZW4x`Ag;hvodd6+xNbyt*hU+|Nr^qDWX4<MRxv6Rb}{p
zt67zQip_&dOiKlfq&IBaW;VfZ`SxEm`+VCH*F27H{q@gy`sv!#T1|a@d1(m=kKHDJ
zY<8;4akL-4{im*;Ii`F=Smu=34}W{8O@8U8A~gRFQz7eub?fqy&1T!Sq;7vMdhnCv
zzu5=4HS3l?t-lxkdGkEcr1kf=luyam|NZ;-@wrTeJPVdBTgLdXmxHgPKt$eoXL+N8
z!o};?zdz3BI-0c6v?XH3-s&6COTR5)yk%-)a^)>ci!j4J50xSw)o)ry)qlQKTIc>o
z!(pGt4pXL-=0&eNk3XKfC;jb*jl2p{y>jjjdpR6`w1;SYHJR!2Z<DggesdA7P?qL{
zizSk$a~=`4-jl-^ka&5K&i&r2N0Zjc@447yDAgkP@Jgoo93AEfElZ2Gc%JEg%FfO%
zpK0<vaPF#9{;SV6tC{c|`Xn;1W23&W>+VF;jk?SSyq+(;rI1pTm9;8X+^pqR;uOmT
z4*S&h#p|#}99p?)(<d&YrV|oJcbBi-aZQLJ^tt{afqTbnwK=&TY}ODC^ip9ch+jTI
z#mwZn%H9v(a<yv@2{Ejxlv-xJG-OfDZa=}Fe7>f?`B$aXvodfhzh%A~YL=T=+wyE-
zp98nUG3GTz>~5D5trKpoPJZ>wnqi5}_DyL92UzdkSjf8I(DO+p5i_<O{IFY;)$>%9
z)ZLi&Wt(~?eX$LDVQ?er+uaF$z7lsYwN4acSX#WlB!;19@t;y{uM={`nJw4CPPTrN
zVo33s$oDr*b7mC7jxQWM-xj*aFz`>~JRr&7GV$xQ3AN9}4tPquX-m{kys%M*fn&e0
zk-p+<TL!uR-x<2BTe|Ez&!$d!{NU-)w7;!|ueNZVoS9fw_H9{u>Xl68Vy*+9-PER)
z8hKf;GR);EDT?}>HbsEpr@^Fh>BD#a^yF`yD4H`Pc<$AA7I#_h-q|>L`ShxPH7l#)
zE)+(wSX*51n0~(em(2T3V%7gFE=)M`_VD4up4z^_)?PDaioEd16tFtWY9I8h@xrs{
z&*nmMvae=e4GeNQan|J_-&d}d>DRS<i&{czrazBTecze1@5U_u?xX9(ZW<hnx*6X6
z?1|p3(|21_{f_Z@C5U?NyjipN+;5ZRtMj$LRo(U4t9L~%x^_W;*N%tLOb?D##a)<Y
o^PpAq?gx#^1HE7OEcnZNMxxt)V|eF61_lNOPgg&ebxsLQ0HA={L;wH)

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_launcher.png b/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/ic_launcher.png
new file mode 100644
index 0000000000000000000000000000000000000000..b3113cd15c3255405ee34c622a1e83674e6e5487
GIT binary patch
literal 4312
zcmeAS@N?(olHy`uVBq!ia0y~yVDJE84mJh`hS0a0-5D4J=6kw0hE&A8ol{v56Iym`
zfARb3y&13b-cCyOo~HYdcYVb{Gr0|#WhXZHIWd{KW=&e9qGEh9;AntwLeP>djz}%m
zm5DRGvY1R=CE8AIki2b>yy0k8Pr3AlgWH~jMz4<EyYKru&%4*}Zc9-q_1>1l{%6&j
z`d_-=|J^G;Z~1-BXO|rXJsu5uA0%fUV0EYzDPp|vU)|x14&#rDyH|Ra?km<VtuGLN
zRNbsG^PBaq7r*~EJ~(8{P{kPGawPq*%^Y#Aq%>PT{RXb|>x)mXC_mgg<y{=pGph}{
zj9kvY_zN;$zjyiSo1gH=cRmvbvq!)AiefGe?qkoJZhr3X*4&e+s%j<fyi}dZPe8ye
zOqDA^TBZMR@Xr2IQTN@>hGqv8zfEUQa!Ki*KI<Ik0>&=Rr2E0ef_3?<$N6=eL^Bz!
zuQJS35Mp@wF?E`d_O<_xGdwx`tt(I4yM4UUyW;n^st`SE1_e{OEUVQGn-X{ytUr<$
zZ|_~QF4nb^nLDvH-D&BngE>ttaY2WByNwr2VNP^Y=-QXEc}s8i?0eIS|NZ^DruNcl
zzqy*Ob4?x8c?!P&V%xHRi;HT1tY6soq9up7uKRnk;o@hl1fMmbOBduwJCz;c$b0<g
zSh-8)Yc-QQo4H-5-m=Sdoolv&iLJQmebdaXvL6>d-WgSrJ8$Khc~3>&?+|)<PK@I+
z_adDciJ``a7$US!G2UqU@a0hScfqN#TNsau*LJLWS{-1t^?ix$y&a6Fw2pNvuMbn&
zwms6-F4}A3UhRpm_OK|}dq^qprs^cH876NCo07C4_h9hV!>*IBbv$Y_<?y@RwM*N&
zVHJy@Wpd@!beVu_=VrXP70_65T;%HZR7uxwYn(ovVA>GMGfB;(%ZcrERJxH7pU!{Q
z$mDk#8rrP+SMBsNBiGhUzAiP7bAHk*`SR4mO@X?*ndZ%|PuXc4V{yDs^kDg(V=uF!
z6?fgvbAHq9TJR`U;V=UyuiATKV`EVU?hS#Qo2(cYK0o?X|J*K{>vkKx4g~+>WPQHB
z$FqEWbC0as&8312ehcOF-zRV%oE+NQ*W~@?UGC?j`|5uNcfE^a+@|g5b}d!h!o1f}
z#Brg}w}~2ak`+&638%%Hb0iA%D?Lec-+7wPjpyI5wIN#{sc!jKAlq=5A*og3<7K9j
zOSy|h?#lZf6z{)(sCM^y=0aPJr3vQqE|x7*cz<LQ$0^NLVLu1i`}}1-D~{+C9Jf25
z{$TEPj>Se?O1e!>VO?gAtoo8<Gt?ApVl}evt;<?hA7}RW;`_}Pe{_7jJcH42^F1k+
zf+t(M3*KCv`r>w}>Ftf34B<>l%cb@%`efhiqp_;tvUKa^T9FMo2mBoEot8d#dV81E
zpt)<Wfap<!XeWbJmlc+7Pp#zV^@u-Tz-xNcVNtW!touig?4Ixb_EN4x=KX?;75@&-
z|MX|gZM}mR!=pmdZ>K8ls#aaTD{4UyFM|z3=b^pAj*_MYRfhsDN>~W82W=GgmtpZp
z6-}Ncz$l**%h|&wpQ6DUbaBrVBf*{AUW})jK9o8BDBrw1Z7#>Tcwggpw+*lOa<Lsb
zbL&7=;ghp!%RjG86}el?dUNmQg+aan)AX6$p1d=ER2v(d9d?-0eL<4uiVPNyqsH2a
z2Mkpf+<9Lo?{a~C@ip$nD@87evsr{PXqcZk)97_r<M)-otNYEv_kGx&%rs@Iu^fYA
z`fvW{!CwwMeey-ORdyy@Dl3EH=7dj&(mWh~EDT~=WzQ+v+4wlI{JFsuL!||8SVJDQ
z=WsZ-<w*3VDwH&FZS+rR3S1T9pMGefDSNH?e!Jw5hD}``74C^T{hoX|^vP-0DQ_a$
zO1|4PXy2(g_Vp<14G9i|p9i)vs?Tch(kM8;a8Lej0f%?K9S%!f4VHyCvpt>=YPG1N
z@k8i`U#c55n!8*MOygg***S29>Z{*}uQxAdzOq>_;p@9>L*cjGAtmRzy2^Fhz1kfP
zO-^^VteCkYfhDAC!J{`-%Y&z1`mkirnxNtbZ|?G@<OIY`4aya@ncS_Qtux_1)5$HX
zHLh7peREhUaZ0nX_zA;=SLK^oHkIWasy?oLbTL!P-rvpHEf#g$s<-o7M0s~4C}<=%
zeCD5Wl6#xv8<E4EY72Mq?Vh*tgY)hT{mrKOZ)^5lPH=x|;k{Z_U!`_dPF{ndqr<f9
z>NN{|S9p0^*w#MSx42nn?uwbSztldTx+sv*%dY<JX}6CLMHih2=Luta;&js5;`+Ia
zbj4K<S{@ge%t)z!7{m6%;K1ax<dcf&x5Mmj`*A#Yvq?{2$`pfTzD<Uf9DU^urSIgE
zWX>crUHN}P_jtp*3FV%96@M~BSeOK@n17Q+AR_Egr_h|k)$BXp1PUiDoskj|S8}@e
z)T~>1TmQb7ePOl6HchhQjPN%7`HO0f3QQGC$=ER?^GrkD>+rYMe?QDfIX8!AvupQm
z=_4-{pJ^S~#B*w8%h#W34({I%S1fCC+-PQXr?V~f|Eu}G_dajjHS5Q&<?BkzYAdvl
zA3DqM`_A6EvnypDO1L{1rvK;Lzx$c5@DtS;#j_KysVer^mAAjua+~60$r!<I$9&22
zfK=~X$;6~6$2C@|hab5six$sYx$BRj^}dHoCp27sRkP<E+t;Vlq%CgUX3^YwSwb!-
zu|a6x`<_c4-|dTTdphi#v3L!Kh(V&?$_B%Q5_8_QU%txQ_gdq|R}NNgmG_;I4p%mO
z2;5(OFL=kpg)VAq90IG(8c%-HcV0jA`0e%2l7H_%I&mGtl5_gJj9T&vVO^`z+WwZW
zTPZPhq8!(RYGzHx)H(@a+Xb6McG&Y+IP317D8R8`iD7cFFgt^3%PT)d7jrwA>*Z&1
zCp_M5zfR=0`F1Pzvba@z@1u{+WuLAns^4}(?NQOTOb>@!&bw}Uv_3jz7igNl_x<ht
zJKmmQICanU3ik=Ay(_QiZ#&=|Uf81$a)!B2$fed+W3O<)yF2XihZ#=IEm1nYw>G)7
z=Jqeinu6m-SFi31`C58jb%Po2%qRQzxA;lDyz?$j)94SUNNB!W#;NKM+wEt!e|aBg
zTi#F-7BH=0t<s7OOvhDssPiPBGZA>tR`I35kMXJZyPH9&>#Ls$eUXW3_4sZi`gR`A
z(;pA@MK_&l<M^7A<(~3o^8P=?f-`m;TruNbn*YwP!ta-O_X%H|u<^u(Lf`-PcVx==
z-f796V-Q=QaP4<^M^e&l?!M|f^{+CXcZyHtpYZzVD{=d0`Fp>3&MLltGWXwvi#@w%
zev6EMm$Nbc|4q|x2k&ZI+%p$eS(T!Z*R^T~Z~o1~3Z~gEsjs8&y`CS*>##iWt5Cq<
zREc(0k=h^2)~@<{Ic&o0Lnr2#D?F2s3rMS-Y@V=k$%F!?OD~llJ>J`3T#;|0q%c>|
z_)Ol-JufoTuidwKIe$aM<g2DX1#KB$-;xs8V7|xV_^my^yq@3q$NJ>G-elf;8&*Va
zxM}?R-2Fv~Ki6G8r{N~Q_qE{FeXpaI2>IsC>i=S8$(d=eAcyPV*{qpjOKz%aJ$HFi
zYa8nK`(shWw+VR_5e~P`RdtJq{#@kdeAz8>_n+w>*tRPf{5N0{&sOcdvBzM+l32Ow
zhw7JI9|i2R{QF4x*rMwF3>SP|PBN{1_92=*ZQZu-btbl7?f-wVU0(Zr>6sO(PhaRP
zvW|?@SA3Oo;FN`UgU*DGj)2dOmX%lYWs<9G{=Qbvd;YGkbzQpTqaR!fvDPLw0*yry
zU%%?iIm|bwWnHD~jwN4t1*5jGJr%EKGJMJ5boTF0FU!aP=5S8&S!>Somfuh<Q*xZf
z6RvI_Q@(WXzn}X9+U;gDS;wv^|6J76vSO=5*nXxK*P~b4T|b-ZSBU+8peho&?fA8S
zHQPSly8K=7TDj}Oh824lH*I6P)YFtRxBQ$%*>&01L4E31PNx>fg@vw;Gn2}@waELd
zOG%s3<%OnG>p!p5UVPD+akcTCoca&5Pp|dAw`*>7|2C<s)7-C2R=Qk0AxmtJn%JG+
z+Lyn)aX*_A`z`eEdb^D=YyTeY7wBJehP}hfTOs?k!rsgSoXjR0_Dnv_@NCliGW(tY
zcJ{ROVpqfei+#PGr+sCf$gVYP^QMajJd@qk>w7Qd_UfrGZrWeH_w8GdQU5s;snh*Z
zz2R%NPW^Lc_m_LhH>-9mzB)bT@8;8&R{#IpD9CtrHsj(zC#hJ4nM@wMGoPIDeCgt6
zxp<bI$9=x1*)@TBb>Cc-v@{HDqdZs{tyG_VQ2Y1$xZ+7!NB`ttuIctQJPtJrY&Ap#
zSiK(X+#%z(meoi|`Jkl1`kqA*BDd#Estvmmr@rj&=iu{ec5m;$woim9JEVcF;nMY|
zHkZSbI-D3Y#T8H03NSWZT(P-<#i)d>;Kw_wYnHjkxkY`A+N*WSQf6=b@S9(9^Zwh@
z^jS1mPON<1!x^xTU6e<oTv^sX?3bdJN{^MW+f;U!<!;x7x|#we{_{CxFUCBnmTyn$
z`-_u)%YQk$^Y`2J`U_+<CM&G|Yw+r?z*Do7o$A4-COTSNtSh+1ynMN^M@I4D%hUYW
zE_{-=e=)oAxj91s-;{8NYrhhGG@3Y;9B^asdv2&8C^@I&_c5kZN9-Fs4WdeYH?cdH
z{jJ-~__AQG;_KiAM#t}duY0<KVMTz$cl#H!fBG6+TJE((fOTf-`<c^iKYTy7#=i3T
z>*q&p?)gl*-}|?%!=jCKitkLO%x`ShW{2C>?YI2dmHXUxLB+C13KvaSm+WicT7J;b
z+58yqzOMldUye<^xo}aJ->QsUW3_b^VcY&4bYoF#-Cx%bmF6&GcSDJcmfAtVm&=kL
z*6k|nny4159^7TyI74IYeI2LZ>z!Sa@1>33-)Ni^wRg3=Yku6m@S0EGi={ldc~9)-
zb}`(3=9uSQG3A4Qg*-SW<Q-Yzy-8!U2#-IPs%f-_Noupk{D1pqZ7G<&{-f4jexJVf
zw<i^?@)=T%?XAK&`nU^1jFxw}woaY2^`L%W*ml9jletN+O64n~)}1#^?CZ{QoBK{9
zX2MKk?z`Q*Ar}s$@A&G@ea3L&f`qGLH%(r<zF)Mzin;W2Y4Ak9oqV%X<ljboo?9QM
znQ<XutD)Mfg`TVDeLK7NIj2e7GHrox6%vmoD$Kp!Fk|OrHJv)0plg3p-d_E0lXO#k
zzR<B>zbpAXN}pwY-g)VBgu2zW*Ln=6oZm4m;BGj!YlqSG*nkT=g983*_s6%Dp8w}@
z^miC<xVW?)-zk>hnS7hqub#N<g?GR1)9v%B89Hxhc?73F+`s7BL&ntAE7$aO?_#z8
z>X%d5HtpYS4XKHK`H^#bJ!VbHuD_SE_vf4Wh4JU#7OY{L@NSpI#q_s*i5U)gTFSSN
z3*Ye)p7!TRxNSaj?5F3~X3j2qSS}?lJ>_m*#p3?|UzILy{$C~fGV<8R7bX!a7u^0i
zHS}_V!_94y&fndpmmCtmqvCWnzbgOF`go=b^R?$rwRbJl+EvIrrS*xUQRM83x1Flr
zww32OyuVnPvs_JiZb@`S>T4d3`TxY$%y+!h_VDeZN3+haEIQU+^y}<;iOK8l@hQw&
z-@UlNw&&lOLe?p*OQPeu)~tQ+v$1?l{L434yKla<y5i9ntQ)m#;+MB<-7X<Nn%Xt4
z{91Rh>Th<@xAN^w7kGoMrWX25ne1%&uAzuODebRk=*+FZn^#^t79amIr~bEDNPCyd
zg1M<H-qmjY`YisWYyHp8yaK7y*H13K?v@xn$J8Ox^yTwR*O&U)YHIWLy^c0{erxW&
zZvEdPuEM4>_UM+i`o2Chw=|4#?#{&dd#z&`O7xUoZvI|({J*ET$dvYv{}~-t@?7B3
TEj49eU|{fc^>bP0l+XkKwGk&j

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/tile.9.png b/tensorflow/contrib/lite/examples/android/res/drawable-hdpi/tile.9.png
new file mode 100644
index 0000000000000000000000000000000000000000..135862883e26eddce2b19db021adf62e10357ad0
GIT binary patch
literal 196
zcmeAS@N?(olHy`uVBq!ia0y~yVBlk5U=ZS9V_;y&j(Km$z`!6`;u=vBoS#-wo>-L1
z;Fyx1l&avFo0y&&l$w}QS$HzlhJk@0&C|s(q~cc6pa1{unavKIIKeTE;goUGRQAl2
zq$Hy>1;#(WzrWWB;1_0I$*`oI_3RR37LyqcbN|mTd7aQUVeLozbqPyn_8$0=v}Aqn
zldP13V*V`*C#snR`kmdCz5WP$)t%^KWa44C{6H?Vv+L*@1_lNOPgg&ebxsLQ03X#y
ARsaA1

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_action_info.png b/tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_action_info.png
new file mode 100644
index 0000000000000000000000000000000000000000..8efbbf8b3c44418551699db9388cd77a88362112
GIT binary patch
literal 665
zcmeAS@N?(olHy`uVBq!ia0y~yU{C;I4mJh`hT^KKFANM!37#&FArbCx!@Rwt0|m~V
z?M+B(5>X84><aMMaAA)}jLQedx|SE)xV4?cVyCnPtz<dkD!#}=Trczwf0v5^xA3O`
zM-Gml-YqOkr#w?s?XOc7{8IEzdFO@I4_XZ#o|%)sZ>DkjvjVGLd%xuWyV9BbjhXBA
z?_V9Y_S@n*D{0d&v!DNb|FPoFuA8&6)R_0&ek=FPB9F(f%0(vj+Uu_wR&(#Q?_1T<
zqr_x4fBp8b)q7bh%sJ*b$~nvv<7$n6awNxW|AAZDYj_>`9R6K+{WZJO<&jfgiw>iU
zEyn>#22sW9lTSX`D6-=C<4WegB6YF~o4a!6c--_BT5zK9nmogo%^_O4she-w7YR<h
zm$yCIM=hA6C4P~a*is`!*M{#^d*?sgvo}J=E#h3P-gOPjGbu)aiW?mi5;Uiup6~F8
zEpoQ_YR*6=kG~vd!CK)#E3Ztv(0KUa3#TdETljVx%!%R5l{#+NtzcFlQoZI)r_h0C
zmJI({KdcqL#y-ihq4>msZ8LZa4qe!)*rDIWdBJvlh}M$|Zs~H@1H26UllaW61w^OG
z<{aB-muMhix9Jy)F|$QAztU_sHl-5>EV*x_ZI)ac_FCxWr=K+%(@#&0EIaUuS?uZ|
z^Ms`>W<E?IF*}Y=c3b?hR!1y3Y+n#lZ_=MGyG<vxt{wMu_^ccd)Z*2@V5!m)r-{4U
zd<8Q8NeMOXc58VRoA$L{l_kVfsbE6((Iw?Vkpea?6$W*o+mo2`N+l*1^XOka{9;?W
z#oTk(XJowp@cn=P<K63)zHELH_xj2^PqjDqqins7s;yg9deeNNRtf9u`J5T^8ta1+
XmKvBID7(VIz`)??>gTe~DWM4fgdQT%

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_launcher.png b/tensorflow/contrib/lite/examples/android/res/drawable-mdpi/ic_launcher.png
new file mode 100644
index 0000000000000000000000000000000000000000..51f87ee6507cebec6bff32b1a03b36ffc711689d
GIT binary patch
literal 2265
zcmeAS@N?(olHy`uVBq!ia0y~yU@!n-4mJh`hH$2z?F<YY3p`yMLo%G-&a5m5y()Wr
z|Ni3lwbduQ%sd(drYQso1V}EKwrpubu-2sonjCZansY;~F5kMii1`OQ>&06~3+IHT
zN4#nX7I|?)FfB{BoB48=LQtfTgMx}lV~bAy$=&boeSSS>x{KV&YJGhct49TDKi}Ue
zuABe)_q_W>au3DiOS*6O-)1;zbK(qN%nbvVZF3?$dVf!MvhWLf`2E^J>o*FY(o6&=
zUX5qHBf@X6nNfkgU2gW_f3M^Z>@kvDV$HbCana0JSNRUd&1Q)wCh;?F5fqHqKf%kI
zkhb+6!|rEme;>WcdtT{|fz*<p4lWJ9*xmn&bja>J6gm06*i!#}9bXEccX-6{AK_(A
zNYh^xH$x%1&))0bt9FIMGZlOpI^RipZ^(7}EVpn?^?iqHPwq;@6wd27v#;R`_kzzy
zWEr@g{_9e`5@#-MWFq4fE)=xw`zAZ%8LAwP5|;9(|FFkhceB|j?BKmhZf71t+BE^I
z(1uyL#XH`uKb^>0q#)Dh;#S^ibFhtP8qZC+Cp{mn*EF3f?K`;HpJ7AQZuwfaj(Jtl
zxzlQY@f*87IKnjJ>vzYfxCzUyb?uhRW=`17(O{CX&rCo)sPj`ts#9@z;H?>JPt5zJ
z*Y<BiS4UH4%2XqV<uNfIuJcT9pKj+maoNm<GksB09khI`a<rwt?aVq6VD^SvNXffB
zdlA2riP{EDE$ycU!c|eC&!?>yPd&en^8uIO&AaxmL>G8JpSU^tbdUD$XCJ3B_dDKt
zDQnfgzOapNo5>AE{x{rSoL-8o8~kQE?2!FX`EXahhRJ)^35ku1CTgBb&{RBI_QUJj
zjhmP27B9bB-@v6F`?z-rgV*!7U#q)gOQYqUZ42f<m?g=brtYCKt@}jV)f;TTKcybn
zv+KGk!`y9I&6^W@z8#c!!9M9E*QKU{d0Q)wNFOsdU)Sy0yZ&Zh#k;Mq?#*5lx;Bni
z+k)><fcm;aNq-It<=uFg6A{WTER)v5zU%CQIXo5nrm{Eg`f)#tSs-(RM3i51_PyIb
zJsEc_JC?gW?}wSG{JJF<c1Ciss7!ishxw&>;#RMM;2P#DZ91QC+KDo@-+o+u_i<Zt
z4&N8YR~woYJUqTRu<=Henl?x#tgVPT@bkg5ElU_4=1x5?Avs^abqiaw$9o=~Li4Fp
z*jIm3O7T`!sJ^p)M(vfgIyu|@TMnc}cp5BJ*F4RB{Dw_NCEJ?M>IWAtSQPHQfT8Ie
zyZP@)Oh2xNT<hGjEG9j3cY~l>snEve#o_0ecVA`Q{cN+yWHn=(o$UcT61u<k=XUS-
z8N}JKXYS5*&mZonHSl`9d+l5$hX+fK=3eWpuRMA{=aRRKL0g7ERCI}<*vk#Gq7<VK
zCDaFZ9dqBGVzj5)u;^aWskv*Tt>b5FzxJ~~wYGP$V#~!q?V~+M81{Ov;8xhR;@9a9
z2SV+*RQ_x@H<yt;+B$2K9dFu;d3)!W@4RIF{%rS|H%8xcbvO3CtGi^iYrT-5F|U$a
z7$<|v!@FPCz26@^A;jrA!;v#4Y6c0aJ{=cpj$LAXaa3V1FGu{2XM9JPN`9zFy)-)e
z{h!6zJA0oNRxVAfNl#L#xU|CANo!@J;+C?C&;75;b!yHQo)?zRw`q9T%#-4DZT5o%
ziDR;M2fX+$q_P;lUa|f>*Pqm@IqTL&sp((sQH|fVY`U31#H|lJHfmgI>&mB{yU`ga
zn%=1~=a10(dG!bX{qb=8?jd-cSw-TVpvccBLIUq%t|uO0H>i<clyZ&fe0kiu&gJ_w
zI07W1{;y8Gwp?E4Qu`d~1yk$3%O1D2k8f>EO@B01Fl(-ypu(45!Q9Hnzv)k0>pQvQ
z!OL9<;fymJcX9fEOZQov`1kjRiW{G@BHr*huuPnKeAf9u?N9D`n~aaWHE{d)#-hRD
zyVvI6u*`+q!=_w3P}aV>W-eFp*^^$c<rr4U^87E(?u;#6aA@7H7lnD5CjvXq?C56P
ze6~${v8VQ|PwG-u>Cr#_mOnq~|J{yv;mXUeB{nkf@qankyr4R_O*Y`}j8!J9HH-h0
z^+fHHTlXhQsq2fTT*>S?8^n^*`2FN{xpU%v2EV?uvFNyqqHEemz30li>$n|G9{93<
zN5W2<%7Rq>hL1M|n3W8;E;&y0zMi*IO2SOJKy=QsGbi;G{+`<(c=6A^BTurgy<R&b
z_|k{1Yghi5Rqdqr+Bn{p>A?D!^JkRw8q^YeG@3S>usCTPd{|bJH0iRQr&iT5jr%)R
zZa=0vae9^R^5jLVNAo5LIMgoKV9QeeGa=*MGdb~IQ~fmoYHxzsb5HHf?#=%9ZR*Nx
z6Ye?&vAy`07}T)tq*OxB$u`3iEhjE8K5XFei0E}J@NTY}vti|fcU#wW<!bMn!!TFj
z$>oLOyB%%{-MPkdWA68FVT>Kevgfm|)!+%eUuU>aied7#U0P)|GgVv~s?Kk^y7~E4
z?Q55rFM6aX{lEA8&gbp>{yg2u+{S$CgUJWgt1n)>N>%475_L!|I@@&N^_EF9qh;db
ze{xl_tK~hdI$30H$0f=4V)2#Y4BnImh1om>oB2N7S|i@BVtuza>fDUCQ8(|L{WZ6n
z|ERX+uSqlWwGN0|s`%*W2fOG?Z#}-X;<`RV2s1}~{wJ@f-eq3%4fu~#D5%{rkeMmp
z*=Zm5WX_k#uU=oRwz>UJU#`PU%}_RSRr{1vdu}t>uDa+VwCio-kEU1CTbQ!t>u#;H
z`(v|ZoxBNy%9n=hei@^!m%blp`@d|_+mPp9*c+Jk|G&ud`p5D06Rz6n{Q9$EUQ!i<
zf%78SnTO-Qiq}8>dFSQoKP#oCpW5@9Euh<7y702h+xpCd?^C|22QPf>XEEo>)Ds~)
zdl--M{+beRm)<jLcW=k6V8d1V#lKiDa4uv#!SS!swbf?VbHSJSGuQMNa6inlJ8!wT
ze3!azbbsRItLt?-Z*AUp<92f1e#Rs#v+y79rMrIoxtJMrMNGok`=0r-|G(@Rn#`3R
vf7`wE&2_`ONBZ+;u;(r6^_;%%DdRu(k_Cmi84_urCJTe7tDnm{r-UW|RVPRB

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_action_info.png b/tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_action_info.png
new file mode 100644
index 0000000000000000000000000000000000000000..ba143ea7a80f03b0e850775ad672ccb2d6195e4c
GIT binary patch
literal 1355
zcmeAS@N?(olHy`uVBq!ia0y~yU~m9o4mJh`hE<t`_ZS#hg*;sxLn`9l&Wg^ENfkN1
zduPtJ)j|2aYAr#Aru>RGQ-qs1zBy`h=qfdduh^%dv}gf;*EKfINm8t=@;XXM@o&7f
zJ3P4Uu1ptkJ(M>2?gVzzBn@vv!Sxw;?#5iZ&v}1Je`@BlH)r0RI>*Cfyt#Vc=eeKl
zufLz&`G)I<Am@&MyPX)E_&F??5;<7zFdkBoXmgNYVsu}Sr)Xefv&USh^A6v=-xlAC
z-eqQHc5?lTj*fQKt1tR@sB&e9)?W*AbM^10_E}s9E?l_qRik#p)~%+UlkQ68GCcqE
z>{*(W!LsMipZ~wmbM?YCmWF@-{$($|_(E{X4sO;Ysm4>8j0XJu>$zJG{XBZ~=>EMQ
zR+(JOXHbgxD<EdybD^->NhI>|cKIK<ov+X5=H~9t&B=Mv+V)%GySY=bcK_ZO{t&5t
z_c_O<^&EmfadX^fkdTz@RQ~dq!z@bej?sDX@bK{9oiXp?&6~b|`SRx7n>QkM<;Cyx
zr6)5jxV(Au=Ip$Rw=R#l8IqHe%@gN}Gd>VYcWOEF@lU1I+;i_#j$HlcuQs{)LzO_B
zt=N0fJ#Y5ToH;Xo?}up{=P|BWuxH0}6T{z{hT1ppes=WhEGsFoNwP_bPILIk%Hq$^
zaa;Mc;ptDVEY_{n%WSh_8yP>V&V23p@0Q+tCZ_3{`VSwlHgLTAX~JY-7$5vp*5;Js
zgYP1V-vqC(5=y?xFyZa{#b<1DShN#+`L|Ac|Mu<M2_L<^y;*yYO9;#>bdp$a;r&?U
zs$*>kYr~bF5jtj>Q&(lKDTp%MJ>~RrhJuJ=7gWv$-m|+FA^3iBSMi0GLs53~<<~Fs
zU%|zoRzIclz3JrVGydrM9&-N4tnq&nuSDpRxjt&2_eKk-8YZ=buYL7F)Xm~?*&gTR
zY!0`gy)H<HwSV8Y?TV^Ur}cx6lXP`@FYLaV^Ub=KDM)Ey{XGdK{*Lw8x6@1+bh6|W
z?)6JJP7nO&+k2sH8|%aQPvb-;D09!}SCQJX=fP*G<Zq`fdW!mDKD4&AOEp})c#$(Z
z@iS*r{1u;!%ahK0J7Ih>IIJmDmO<g}BVnTmv4E8!Sx$#NSQ{cQ%`?#3zHQT{qBFVG
zs|CHBZ{NKu`~KCdS%-e@?iH80uCp|hq2b$HAEn+78Ob&ELCR~KwJTY&Ww#%H6so@E
z<Le&xb<cPB&v+hu;_KSzom&_@tgB8K{O{@9S7JTU*WAFMz`45m_w8@_uV24jucM>$
zCyv*iVaNPJ{+S!aC%j%0Z5PF`PNDbo+#^?w^3+$UGPI;M9nsme(l(ft!PEQKzuP{1
zGgK;Lf7vrM#QdI+edoki;UeFiQ@vRk)|oLre!Eup)i;aM>Z@;Ewti_ic<`W~S*Y4s
zndzL26E9y~$aFxjv5`fAhjH~C79-9{%uMlYwV5qh7uc^X7yPtZ^qX+Qu5E{c!v93@
z{N!|q6?bN8kUyO`pK-<eq8t6Zg8LaVPPBD=Y&!m8?|Z|w%d5Q4@H0qlQ&sb-zALh{
zbZ?uk;mz<pMH}Xpwq5%rAO2U{h4-$AsN&xnV&$uIEMpIEKicbXu+C6Pf#vVj8{3o_
z4xNpO+i;Jc?fKIc!m~vkCWuczSYi9FL1E@x-_FICbf&4UO`SLUH#4XA!v}Rr$<ce)
zuYbR*!lv%V`u-(1jQ%f+(3!XT+0*BKOT)IRZWG@8=|A)DnK7yh)_q&3BD&<jPw~gI
zUORiPJ#NypZ<*o+j@B*3Z44QL*=D~kFwZ)>D5}<p|ANxBoG%O6O=i1p{l#*R`5$v~
XZdP7X&8-y-3=9mOu6{1-oD!M<@Kkz~

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_launcher.png b/tensorflow/contrib/lite/examples/android/res/drawable-xhdpi/ic_launcher.png
new file mode 100644
index 0000000000000000000000000000000000000000..6361d792dacd8ce09a14258878b5ce6db5e0debb
GIT binary patch
literal 6683
zcmeAS@N?(olHy`uVBq!ia0y~yU`POA4mJh`hDS5XEf^RiA9=bshE&A8t<5fwxnBCL
zJg?mUym0eD9tI{Bjt2e)7I`I2JC)06o?${KH!2_3Q{TA#(G@jO>yu$m`%ll7pKbr!
z<6C?A^}w4ei-I<8*D6}8>;BPs<J^c%sy7$OP1EcYah&+TS%ssaBAIQ*cc1e6n@g+H
zRxIo|&ch(Mq)Bhm3Z<%D`Sah;E8h3{m12&}Yc0m8>$~&s$VfA+`K)$>A%VG}blL%{
z`*qIDW(*NKS!P5sCopkX2%nc(bTj^+)lzo*wl@VVDKXC<d<#D^?fzf8sJg#O4U!C&
z43j!4o|ebn_P<vub>K|aS*8X)g_i98ax3Hgcb4D$$efe<-TB$?hbns$R2H(YpSb)Z
zkIlu+9AD-fpLRa3CBJwRPfM*jLt!{$z`c)FZt<K8D%7}6=*`@>V}8(dIj6c+{4qI<
znQ{#rI;t~rIc(YepV+Wh`E{K*IQ?PnKBmQ$G8_4iGTwN-R<N9xA@yN3qsh(1_II?>
z>Z*K}u)lx&s7C%qAJYfML(EF4(-_)}jCnkc-~E_!=5U8^QO5D}mQ_z&*M7ddoZ-ys
z`+^g<ZfjfmZS~!Y3{%ebKIAH|Q|S6}`gY2nGqxIi?-jH2*x#HH+i;g9AoIE~=lArN
z4=%}v{r3>KsbI{1cF&=z!>lvbK2O>3-c-Hl$D*5`4jM9WwcLIu|IUNqRMT|>%gjSN
zF0_2Ps(YgM%c(CX>Q42_HHp4&4d!U@VE16oJf*HvS|G{67A<zd=TJ}Y=Iu{e%@(OI
z6_heaVn25A(~3|0TpZ8KezseSGCWPFogv+2wetG+%V+fEgl=xPKk@UV*#@2_6JZ6$
zRW6^*%l;-QOxbeqDtDTIj6rmQI;(9#C$sH-k+u7O@@%Vd+qg86;pc7EL$b^VdSm$|
z9Q4|{<8Ny}_m^LK=t-PKt87BH@th`k1=%GL6QXwQ{$n-Yai+>s#Tf_tcrQI$P<d=&
z`??7?i^LYj<}dpB@N<XU9JUE;0V+z7yF=H%KVu~xfB$H<<?NYLe`<XExpBGD49lI8
z8-AsfeX-N>lWP>c&-6^dW4+j)TWg{XRj1q(>a^+k5c+M~T)}<k%pbnDiT_X*Z)kqs
zoY6$Hcf&pI&0Ea)gSwZ9TP)Y9NzD`4oq6nP{T<!(x_{DB_EqrsExY}<KKz5z>Y{bp
zdu}h1U;T-H8*cz(n$E$-T>YXaH;kixZwycT=EHbmkGR4bZiZB^my80(&UDY7yHfbK
zM&&OK(X_I|p4*#cUA0d?Tb6Wx`qt0dtk1&!Gn{3*sMC_(Vw?MX=feLj*S@a2F7QTP
z^LM-H<+j^g4wc8NC+w4P`V_++wEXwPef!$EgLen`ylI)7%Y1v;?#Hoq`mf^Vf7&3B
z)>&D!F)m=$iq+EgDGS?|EimWVAZK|t?Xjf#j{lR=f6Mpn-p?Fjvi4l`$H4E*>z??T
z%e*<$!~gB3#;5SzDRqzHy!L%nD0<AEa8%l{>vT4giq@1(>$YwzQa-iUi>ceKAy;5I
zqatI_hAXrG70%t-vPHJxv%oQy>f84Pcz4$|KI5O$mK@h!{@P~Nw$*<oEn2Hxbfwhx
z30LcSmMn&o4A0zlKRIFEuUPreIb_ZoX6xAuc?>riijFNgbEa4I>Q*zeMH5(g&(-i1
z?PFXbDWJ%yJBKMVa)Qp4oi*YOM!Bi%YxESJE&RFshOn&5k9pHwpS;lAQvbxua*^H|
z7wK13C!*#r&`MnPrsjauAFt^i^IxiG*U3kotevnk&SIv7L)Urj;$Xp=*w=}PO`>Jf
z_|835+{}7n4)4sHJWn_$Xm%{%U3;_T+LD@|d!(7}bZ}<XDtJu2F(Kh)gNxvyIgNt*
z&ZNCj`n}sP^Wx!rr7Olg2joOgXewk~YuhDPvGZ#f=e+|*It4!EyC<e^Kk>7%d5ZKe
zhEKuY8T=+odrVz){neF&Mbq{F$4O2OGMciKF;jX%rEJkNsR`arsVgGiwan3NQa{dc
zxZ&PiroGlYiGIy{crK});H`LVbxP`?hW*1GhAo0K+jqV+Vc)(_Vb8|v5APO<%JTa$
z`JU3bIwd#OnBml+^#P|ej{Vyi?zg{a$|UCF398$hx&<4Up0-4WzFOc{=4>^CX+p=r
znKEs2W*4rskvUzuXFJ1!l$;j|^VyZ8nl>skt&McDIMsixG{ouUrS>Uq0yzfD9Zj6~
zeJkd8Un^(*?+>fQJ;4Qztc9mHEe&N35Iy+I@PCo<qv_jD9(mpFQn+V+cV}+FrPv8&
zH@1FWw=?j}GQWkp&u244DbMi{Tl4RhR7w@!3GJ$Xea*1~45|iCZ`a0U_}X=>;M85^
zoSiZulEHIp;%lo@w-ip_Qs}<$*I3}+c2E9!r9CG!&tCt@_U+GgMWvv$%!|pP3{!sm
zS)p!StovxD^<CGUopVEcWER}H@OOidZGjQvgk`(^-*erw&DyuzTT|4sNFs;ZBdbMt
zKI{4OOw*DYx65@i1(a&B{t=iRy>PonMlauq?8cZ^>rXLjCwS)_3tjamNN~y7139(L
zTd$=r-TAqv@O}K8h+>9SIeR;QJXKS--qEjC|Aiy9y>F4hMdl2~2?3gqUbRG~%5G&|
zFlp{FHH()@DK>mF=P<rml$|AJoyB<I@0ub<DaY*oLpSbn>Kbq_T-vZJx!owb=<Fme
zyMUBW=RDO*8t1kNo>#vAv)Pn+!Krhrt#WSI{kQ+^`svKxkQtUt+qOB2-F_e<!X9#F
z*&Kt_`PX-!U-<K2AG^`{2O(TW)7iy8HB1fW-??P%3dx2pt6zHsd>sQOEZq8Q!S<hy
zGeV2joK|b(G58rNW~!NfygTQ}vJ*KojphG-`xnX_FxBAmv-P%;pVRd6+UDMAyK})&
z&*yG;Qj)mBABBZG<`r$w-td`eO_@!#LO0LRU7|~$9I42>e!b!f_krHiRlXUsr52XA
zDb{#)G_b7X5q&<L=TXE=j|v5Du^ZnSxD(P}Nd>PkcIKUH`Tn?keyk`%=x#fP1+32T
z_6q6owGES+k3UQ?W4y$EX=?M@zNBoC@T-}JxR(j298ch%&#b<Dx{2v(Gso^&hwF;B
zq7!oX4`-+!kSjlVZqfJ4izZ|=)&AA85M}R@40m0*a=n7oZr#tx@%a(D42(T>tM7X9
z$}hPREx+W^8NY~=ejBbbP18zT7Ibo|#<sNIaZw_TC%MD;Cw4e53r?7Q@TN$@b?Kd9
za>2>len%YBJ#lj5F4_Iw7xt*RIJ9@nxL_r<&{c*bpKn6sbjJ-{^1TLgLgeP0b4=U(
zphj%Nl=@Av3@%&NJ3KUHIPmMFxSWdd#$KDx{1c|`xb8dS<f7}Q=6SMZs&CwOh;{3-
zuQ}8ExcH~d^5=}p%ZiI%p9^vo>y_NyRkVx2vM~0j>e7Vsz1v>f_%KIRbGx*}o?2xp
znU~i&_tUP+tS3r@PJ|nuoitgCIbpW^@2ZSYsR{LuzW?(xPT$>{{)(g0#%_Xvm*~o0
zbFxpC26aB2$i8O#p_%6$j-?-1xiWQbcJ88J29EZxdp8$Zoi9p>DE0c`kazI;+z0F9
z3`}^H#M%lA*iURZaMSZdVd&nN7ZJ$}m*W4O(VVo>pwngc#oJ4=4=!q$ydrw*-7k&z
z7$UDI7*9L>RCIHo>8Go2e`zVM;1Um<x9I!7NprWoE;3@aEc~jG{wyl+#7_m2yH{JP
z7z4I{s8mcc>^Na_!h`8(>*EQh{GyC`PyAVLKV!;HOW~Ct8?N0`*s?-_%TmSZ)LSu!
zJwAS}OZjdDpSqp$c|s-o9POsdo1adbd+S@7Z~D${jFDM)zdL8$`K={tb!?6Idig1x
z8|QkcZ!0vr%sj2jX?o_SOUI;qC#Nwq3T(-cQChL+z3bgiw`MJPbN^3akeXU)`kP5g
zsS6@nzUZF4a9g!SpIyk(!D4&B)}TW^PhXUJylmHzC^)9a*t%3Ke)5dz6=la0PiC5w
z{TE(T`~T7Zwf}!}HF&&RdHkETc$~GGyPi!$(mK&gfn|}l{+&kGnGT&|h_K}jVzB7s
zbnc4{vvybA)?w2&+vD1{BX@KbKbz0G@%G6;u|9@wff=(B{QDY=S58`~`7GQ;H@mG>
zl)+Ex;k@@56Ls&K*8lFDzj{mNw<F&V&6$;9I-~4HdVJN1nNqv_)zr36?z+}Gdw$fq
z^&Bh28<y>3TFc6N_Rm@Q<ta%{`xzas<_TV8>v*Q%vVmjmQFg&^4JMp3PfqdaRaKb0
z*+Fslw`XaN|HGLaOupGTHu~N-t$$v9e%fT~Q=F@1<uz+wum5J1EY5!-<Pk&qq}`VD
zv)t_#zbNVS(qG2=m0^L>^<{4*TXuiC=l*|VYN$?zCc_<WZLwEho$g;_sx1~!dUBxi
zyn~wX`$g%d?;o!c=De^cPkX}b|AkW?M=Oa2UTer=&^kJC)ta^I4SuXPQ#ZVJe!cBq
z1^>9a>W?SO7QB)0Jbr$HYSr6Yx3&InP+>5C>aN1+FF0{tpy0cCX7=B{CcoEZn7TTf
zNl9rz?$w5*T~p?-x8D3?-BPZnKXx(bhIqNB=;_YYiDqgzHTCY6Qx_(%2P!x0Oo`5W
zaJ0_y&$Ze=-&V4xW^Lm>y667c2U?p|UNIcV{W58i$>N5D-!hB-o_{Ut?k~x1|NrT2
z=7w7ay)3o6*&-wz^yK#GujZa-aN}chv6RKr2B|{=ymAWlj9NE0SupOr%Ay-0x8_vt
z&rkcS)Ve-yw|eq*we{04C+z0U-{rpS`7e)KTCE9pKJf2q&r5amdXbkEHYrUkp>4g=
zj+blh?Rd)5x_#c&q}}c-zb{@M&U#>X<b|M~#88G6J#DjZeA+cvbB?*<WtKj5g{Var
zZZ$a+<}$3f#Fm;BvgJ)$`RDufYtHW~_+;>CaeJ!WZS6Nn*5ce<hM5;^o0xbn%zC__
zL91-%%k=ivi9&7VQfY?;SgrkE&wf|yd;9B2MwjZ)?Br=3)7E^`k)C(ojG;$ifh%KP
zbL~&5ig$BA#EJUf<7T+W+A^t?edfvywzfMjSG}9vZxPNWH%&OaDn{<@$4n89i5;pQ
zESH|9>1l1SFZ0a4`FYZ{|MK%LZMQF4Xv0<7ETnh++zuUvnfkS5G3Wd%pB}k7VRfH(
z!80q91y4Lfk1ym9=n1nhmi$m7SirOW3CC+$;V^y%7E$(`zv}8&e(pV>zHeznfbY?d
zm)gqG&++*1n?8%H-POKq{o4{D;op_dekGne`#wRlHmW%0x{`*%m1zy@pQ?V!uk!rt
zet%1ujlq-GKR(QP)wN>z=Bh}Bp#Q(^?fQRIoS7+gYuO~uH*VqF8(LCp9ZKi-DxSQZ
zaZ={B#)?P2Ept|?6l!r>Y&V;Am51S=;==R`o4%?qN_`$Y?{WO!%+U5?U)h|iIhyt+
z5=*!Hzulle&!T1JMO!1`{L8a1x6CkKdAuxE_-u9jrEN0yz2BS{f92Y@sP;6|gx~G@
z|CX9otqkwm|ChmXTGpaZQ=^WA9(|p*s%$meouGX|E19}`6XHzXpS8IDtf{C(?ZnJE
zw<^2WEmdQfV)nAld0#?Uc4qzLsH*y9<^TT~S5CUVH|PAqml=0ubI!!P{M2znz5dS3
z`lr^_U#102wA#J=O!EJqLJX$q)07s8E-#qQs_=F3`nMO2Po|wxFOLjomz%WtakZuB
zdj((f7y&H@ofV9F-UZKF91gYV&svmlJF@G`-1g<px76qDJ}<d<6Ms+8q@0P#nG7Lo
z7C3Jyar|7Za&eBY)w2^9j(hUQPTcoyvQf<g-rC>K*-kSBf7_|HImC9ef&GW%_0Ny&
zXWQtR9Dlx#iSN%&F0UyMvUgoeI=@Ya;mjH(5Bs!Zvp)OR{_L<V`>L>9=y$-GuSIs!
zjdRVlH%I2l1T0+Wx6x#_s*PC5eUVD(HjOKFktTVzhc6yJY}ESf(f)sjmm3&<VJHzl
z^NaCAyJMVu%RzyM9j{KNPV!{aVHXl>nb)LVGqHD><HOs3#H7mRx8>h9i0ThdYs;2%
z)oL!BEHk}$!|6+JqoQ3-?TO?)&03sqx4pNWo&RIBzOB;o^=}_s$>9;=Rk-G|SRr?J
zz|msQ98pyb?K2;AOmpm%&F<GTMa$b|=Ga-yvi{BV@x;%+=g!OhSN8L%W0|#ZnRGU@
zL+KXH;>Aq+BiLOmSkCG&mA!g)=1uQI?&EFGP9NxqdEU0ypS%9KXT|n-hl<w{PV-DI
z{GI1_K!^R%CaD7MbU8T=u}7T}kCbhTp4^Fh{_A#o#h*`w=hB^&QY3Hj`MMtD<N42F
zTDE`P(O-T6hp&C^GdId=`}O$6z4`us|J-)em9>9l^ZUQu@z23u85ex|v`{T$BHM%j
z)k8HanQy)2Sm_rWGRH1t!IKqMj1tBl+}dy4E}RuDUl+z*%Bxh9vB_ptRI&Z@BB!;_
zznd}#u(lNWEM0bc`usaN@phZ<?yURB;NSmkqW<MorJVXd4~akMog>csHr@8_e#4G^
z?pIqnH_goZsPOZPxs-3D&8+%ov*$05Vtmo%^y7I<x0{;dkG&0Nbj+DQ_*pOPinzIL
z;qS=;`vR{q=R5wmFR<_0<Q>m9nxFFBYHGN&ie=B8ou$kH&rZK8ewCluCf>2^|2KcW
zpt_XR*Iu<`JU<?9!vA0W>6Z|b3Ss@QZ6-gzm@AbsUXW^DCY58oSD?+SR(sdlvKm|W
z)`=SrpD{D!5<19uX?DHJw?EVK`IbI^Y-`J}#a<iT|7+gESnu60_ww3ToMF#C(;e2|
z`0n@R8=nGQf9Gvnug&OSdcDg{zvrG?h9Uc*^}G1Hv{@Wj8|+S2T`60({F%(@A2aX$
zu8u#QQKPr#`}t?r4sE>F{9qNsihY5i5?Xc|sw>1Qv`=<11ekxyxS*uAf>EGX>dQ8U
z6CrD7l=)uP&ujJjymK?lrUwSgjf0jw=hksL-`QOMEA;a7YUaK3XYS2rTp{{ZlsRB$
zW7lD6TZ3(8jx5}H?6Y4z419k#qC(GUXPRll;%40@ofi{c@>tk$d7M_N|6E*hUwnU(
zVI23n-*3LKF>F;>`6aA(ax2HJXAEclg$bxSY;jmM%VvT|qfS*qymffiyr9qJTHP^>
z5B}-bT%Gjj2^+&!j{U!6LU(gXTu`#(xN0G4yuh@Y@f_2ttAdUe#heBAm{#9ATk`1H
zr4?Irz8#%+-R^^U{Kk|ADFye{%}f@=@n}5{cHxXKHUI6fk13(zz=vH=15+k*Z@0>0
z2#A&mu-4ED_r6fYB+zK1FkQy4(Dy<c+ZBP1tE>%K*13rtY+tJuq|Q2K#t@Os8}Lxk
z`G`(HbC>wElX^J}8~!ouF!}O9d|y^4uh<qd$Hci(8wA;}JYo)czTi~Y?x$Yc_tzF~
z-=M<~x-{?^)1Qm;rTG8!|6lz3%;pyr78?aSbXB4*b64;zm{H;$_HfE`9x<k*K!#P>
zmkz0&4NkbE_MqQ(ZS4>5`uS%*ZH>6U@e3ORH-p2)v<b6Pw%lV=mp!rN)xOm#DfhoN
zE@jZzyX;J(x{3a*ZT&%8{%hVWdmw)F+~a&Rladv;^6tLZUom%~Zq2h>H||=6ZJZP%
zV4B~Q!Pi(J5TnubnQ4yjgEb6YPQ1B$H#f~@2za%K@2ej3*~aCH-f535KbiA)c${Z2
zeQ&+?yIle6KfSMu>_Ttved{`NH+#c0g$}m6HT<hX{QMl2nS5W)-4tu&{c7$`k*x}c
z%uYO?GK;Zgp2}f!--F*0vtHE)mdI9z-v4Tyzp}5b$TUZ^c1r1+eXU!nOXb)9zpM>z
zh)s28b#dH!V6|Kjuf_^BUU3F7URTe$mm8j)4bJ$SnRWMnQ~CGh^K>he7B71^-!WzN
zERmHvHRAtIzV@wicM!8y|Fd18tWB=#BAvnvo_A<w+*W!tSLp1<(@IzVX*I277J0cV
z<IS(X*&N%n4z4c$wdVG^^=;4Au{~JUG$Cn4<)k@*f8FmDX60C|Eq!aJ_{+ZX(2L57
z9iPIq%g=nj|M~Un>xWo$SKV1<FMpya=l%IF(`@gX*S+=rAGYoDtG%|{_dba07Oc9T
z(eQgimP50p@}j->r}IYde_sDZdX;&|)urz}9<Em1|DR#P9I=V--UlrByZhPlc{Shs
z#QyyjX_(#q<y!OY>q%94B9mTOR_dOs&98OHRrbpMD3e{<`+XtjtqTpR=EvLE?&!?p
z<&Dse-gM}74#NhggKMuIzsBE?^fBY#&RbrS^31kqiEpV;*!Hkx`dtZ8`32&|?sK2-
z|7tmJ_OG)~*-XqBBEr^eoxgx##_xm4iO#p$pM3VOiCdOt@3kvZsyHgeYnd5C#NOX$
z0`uQIOL#7<bm3R*`{ngN{<z6(U@-gh<#D4=Sy{Wy$0zBQqW^V0wlqihuHOII=)IP~
zG`0tBD+7(T@VuO7$hFmeUF+?-B%5DvFP8^B%3YXSb-T7<_9jhUm*1EAYu^dZllyDo
z&u@REPgJd`>b~#G*&kjp9?&)2#A<Bgbn#yN%eC$Q;_UxT_xf>q%dxBLcD}zl^EGS3
z?fY};AMG!-c=?8pap&!wbswE=%y;%bPqSXI^L^QFL#FE0Y$3<um#W;2Z`g0?deB#Y
z^7^OYN*Dg@lDk*S$8Kl4dDhRn-@P8Z-}i18LqH_U>il?yCJm7&qx=8ffB*cV)p^d0
zTc?lK@UN*6DRcbEw&<Ce&{{=~!nnOBr**b4FN`**N&aBQ8=U+-^x~!eN=feLByQWU
z*>~~KT=(6V&z|3<^7!6f=2dP9&etAv$A0>CVehq*^7SEpe0L9@yvgBMv2v;J<)~%_
zR)$YvEfZDqAKT0-`{(r9=3i6ckMn-ho?o1D^5lKqrugkCcbrzm{=9VV$<AWO%ToP`
zDFwTYi!%4mk+M@*-p(I*LSD>Qjq`k{?+@nb7FP<rvhV&inz?rR+<@~dc7C6n^&!eP
zea)}v-n{8AcP?gFVdS}V`&0(S*FV*_-w2O;zu8^?u5;DG$$N}E9;P_49BLN1YndST
z{}IEU+^m=&UgmqE&z|)qI(om|w8>24{*ttbe_u&|>+=6wy5F`obF#&!&dv99f45z{
zvG`ZTyB+J*<2%2tSuGrQvgv;Q{F@93a|Q3BO_-!}uKE1-zrA{VmGNdNS3w2_1_n=8
KKbLh*2~7ZzzqWk<

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_action_info.png b/tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_action_info.png
new file mode 100644
index 0000000000000000000000000000000000000000..394eb7e534905e36fd24c3defac92c09b403ee39
GIT binary patch
literal 2265
zcmeAS@N?(olHy`uVBq!ia0y~yU`POA4mJh`hDS5XEf^R$7I?ZihE&A8os*d(8Y*(U
zGFBtRyY<F2-G-<;yDqwF%TLMTm^CAV=@`osW)nk4)<@!s3W_@z823rIsLo=L7g)^b
z<l{3XL!%~uJNCu1X;KF_T$Q>Q)|h8gzkKF(AN9|_Zkd-q+qe9?+T`!QtcuU=d|p4T
z`uyHv=f^&mS2a!ud)M~Ug1I40*?>WU!6e(kxcV_88^c+t3uo?CG%z}BW;5ArE|<V?
zVGZNiH@)mwlq7tS($LrU-(UIpS?<CBjoHhM*xA^Ep1$0&Wy`+;i<-Hw*H4);#s2l{
z*T3ETL*LKke9O3W=~7FLQa!GDy_?b(-ii6QPjss5WNRLadge>{cRp)hw_Nw_*s)`N
zd$an^**;`&c)2++$ugYt)VyU*g$(7K0kalHwwH4la9$Q-Z3u{ryt!8+gK?JNQT{fu
ztGvIM87>}qezL8tO@4uV*xBSYU$!s@1O*u}bI0}sak+EoCch0|^ZTI1om0N<9RAaj
z?!I~ce=B1^K!Aa&vhw3htMHRkje^t}pG!^qZTvmv*{dh-SJ%Wn-W)#9dFJDvC!Q1=
z&9{uX%<xt3-?<*g@`h@^?_a;lW@ly!&T2H*Wk~p1J~?9b%Rf^xY@gM~YW(22UM?#u
z`&L{pW`~{F+cR@4i<9Ew;_MEHeu!Z$QFE=H_y6z3`}g;+U%B$-@{Jz?3iB^8v^+aI
zd%M7Ek<;R<OS7N<{3zn37=JB3_@Zaq`k4%h3N}1@Zw6+3WBO(2bSeB%h}Mt3Hn#)@
zfudu=Mf2QOOwwqY>&!TRVZ*YE9CGh%dp3D7Jo213U%1sR*x&y=dv0!S^TV_6#W)y#
z{`@H!85yblvSOxpdzIwldM^(v!7D6YY9$SSB{ta{o%(J{=YP|Z^78lj`T6fXs~C=a
z%DkF)>iKp)p$7+_J$okq_uUHVzmFb0s=jpj^7O?YY|I6!s;j^Me9Ku^-O%$szJ%Ml
zWY@OMn?GNOn7le`flB1_Cr@+^OQe`Q%ke!Ue($TX`VTIKk7bIo)z@xn3q6{-IsN>*
z$KM&Ra`jG*(`pMkyP1DS#Ub@4OZ6Wwk^XFWpJBqnw5+UGQ7RH<w*;Sl{@u9XQLPVe
zgn2||Vk<Ytq{U^wO<lgSHY{4S=;V>VU(XfpYmr^^(t#;*pYZ&UJ!P%NGAC^sYc?G2
zKRHXE;lVlWmp5yw->W>lv~6X_PoeG0{pLosrSgfr{;=Y7__~<P0N2@^cig5Q`D6Oi
zp~pQln6KeVnM6=tk;s)_Gcw<WncVVJjak$DPnNHtUUALaN89VxOHH~cSUcf^rHRW{
zuFCus>tr?A8N}n7H@Nbt${eiP_hFSprCfve#vkH$G7gsOE10Z!&@67xH&>M}>g<=~
zey(4FQeKP>FMcchF`7R2ou$ZT@6bz2rv$ocCeNHL^r5SNZBSFXY@}zUY|ERa+?s)V
zSUK)jrzHpma4}rlf6DTqlji#sI{FJ{7-g*MVmk2q!>Zt=GOKewPMSOK%%q1K!$tX1
z85Ycl<k)&p@b~#--RMwG?UGfiR$bb)YggRU*F5ZZyBqfVPF?Pr!F)*Vh1Y$y*-x14
zwylne3}$0^m~Pl)`*6dw=7`gpQyDnco;r2P(>(7^MTd(e?*ZqYHa*`N1`luFx^+wM
z>K=v%f>ZY0ar|D)^LhnW+$n|?D^}b%q>{bsY|yTipL5cleD3FREWMcEQo{Xp(RD^E
zZKdWlhS#rN%?(tXEX2TJzg_u(S^L-d2Y%THDK5#AlDHn!^Ez<1OvO>RLl@r1r3N!G
z<XOo#rhH1xuP(Tpww}pq{`8}wPUo8QN+jpa3)|X}yxhM+_u2cae9X7|8*3kLtzX3u
z!NuS#rj)(oOqKbH88fY{Ic{8D`2N+uiicuXg;gKs>*nWeUvKz1wzFNEk%4>S$D%ry
z$W4zKT;F`llFd8+n!({o!~3X)>2swgUh8Xr|K*nZd4>}kCR|-C`%UHO_s0wcH&z#L
zvK=sH*r1^#e1CW3)9Co%;KwV!Jd|X(Y1d(puXIB|SLmAJtG}lIRT*N^9^8nmn0!6A
ze;%WP=Ba<PzdEWUXp6ASV{Ta=t#s@|*@Q2t`-ATIvNE_bY&fQP(C<-l^_`&J4-3}G
zFfdGdta6>#l%YbJA!^T;%f<|OS3MbSc4eRXC->A>oZ;HCM9oGX)3(F!HwImMv2^l2
z1_evU(6vWyS~52{D?ecnZnD|Ykj`)58Br`$CfzoP*-nLh&X)v<K>4c@O=1F3^EY!&
zlQ<k;EO^;QaPzSf%R~g1@2)<_cxd{Dgs#F_j2vrZ)+)2BI9o=&ay)#kE_7<PaH8CU
zF9j1d7mB60D6}nJzwpb#>s~+a9ldln?JBp5W3@w&j%QS-#RS)%3m83Sx&B<RZ7B;c
z!^ikn_Rh063zbE8nM!@CO%9vDpmOz?jyBKFeWr#Q3~SbQuD`I}^4m@Q`@-efi+K0~
zE@VA_7ISJ_yXI}V{^AujZ#FJq@HlmL=OWGh+n0-ri9P$c>-C1#i5yOQ;&yG`x^?Se
z?F*iJZSQiq?%TJ3v44w(*S9SWdM0aiHBSC+YAZRkXOT&jd`J*?QS$U2{g)G0hZ$6s
z?9-fCqyN&C!E@R5keg*eZx_{AcB)#&2XA#S(9b&F;kK`J0Y`js$iWjFKlmCpn75a@
zuqz5MwCuH9cTBeB07J(Umg0oV1qw}U2@`&2^y@snX62@q`%x}(+U(iZ(*w`^y<q4*
z%~JNlbLZd7nD5N}Ca!gdsbR{rozMT5XUvb4ytZ+wcR-uWI`Jv)r{>KI+jdcYsYgZO
zm(O8e%h|3zjF>dLIXXF0W5xgI-4mwD#rBlP9((lkNKA3b$B2e0x&0dFWbWCDE&W-!
z{_l^fHI+*44|t|6eS7?RdU|^N*I)Oh&95!E<Tph|AZ5OA*K|e)V;MFE=7u!hjI?~4
z0}KlynP=UU<~Lx-xYm&N?HDr;!z|MUGxvVPspLOX9b^3(?$;MX7#J8BJYD@<);T3K
F0RSdUDRBS*

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_launcher.png b/tensorflow/contrib/lite/examples/android/res/drawable-xxhdpi/ic_launcher.png
new file mode 100644
index 0000000000000000000000000000000000000000..2e27bec9785d4d51fe597bced7f04508994aa10c
GIT binary patch
literal 12746
zcmeAS@N?(olHy`uVBq!ia0y~yV3+{H9Bd2>4A0#j?O<R~VDNNt45^5FJ2$f@<$CG;
z|KIm*{kH1ex_j4eTFT@JsYZTMi{7Ymoa?cMkj&}M$0~iBl{gaWIfPi;8fpq2=yD3&
z=-Yl{)7%4Hn<E8sBes}LQabADe0IX&nJ-_ytbM<(w(|DggFDx+TE6V2@!hMjw%n2Z
z8fOGP$6a0hyRP)zO@8O^FV-Cp3~P{mc<bHLyP^)_3-nmrqZ;)b+@torI~v=ti$^At
zbr*wd=FeT?<&3Y43YLhzU@BSi^mW`F))l7@OblDVeL;4bzt>5|3*PJri|r=6ROWeI
zxhCiM?-6@M?$=C<Y}*gpk`s=xt!aF%V^kzrd;ho6F*|GC1q=%~4rK39VK^zV*#5q$
zyZD0G{va>UY^Mhq3_Mffx)&5^KAygPv(>A~N-O54Zk@`_!PS50rr$wrw*y~gdcMrL
zVtm=g>}T$gWBE-lXJ1g7z2Vm>i60fL4g3lW3KGeW?m7jCFE}<ir@{HZ`-2HrHZrJe
z`14f9?EjxNE7s3zh@X?-_*TWmfT4qt<9bs8Ps8;CJvKWwvYw4gs;c1(OXItCqep&z
z%r*78B2EWUhqa2mf6BRC-d8q69pK!)rgyu}ja2SXm4&lwHTE0oneG1=yJlaR+_`TO
zO_wAj)H8StjxiilXk}pW6qk=#-|TLl_$_ww#gpchC2yqK>V8Na*mm%&O_C#T{?5`l
z_WnXI7v8^g?=pkw1I0||Aa$o*edgcUo%fWKMigIne~|Cbx!&wl(<=$4bcRFTj#Zis
z*$H#jWv-jgvZP@OQ_s(YJzRPDPMfbUnHJGJG465MH>pX>^QzY5pM6oCC3%ta*0nju
zuDRX5<-AgQZ#rYv0q6Vob8^Bx89dMYIcFJi&qn{$=YE6qpG<#lDJbYVOuBOMz!rfy
z%f4`KKk(t9x2J-qg1`ioi8I)B!&>L9ICbL0X%G9dK&ii%9pAIxGkd&z8{-tl9V~*$
zJt{Yx%l9m2$Z~LIxNzF3@@(1mxHqOptHiFh3O;<|{xH|qV82P%0;`q<Mm!IfD<n2t
zG|f8wNYSRm<bMu>OT9+N3;iu?H!_?lIOr=L^+fH<52nk?+)mzSi&mE(?4C2rxXLfD
zLAA@n)=g;T$`pH(OD}Ah1G-C7-+0KB>x!<aeaf=>+&tHhZ$GcuS0z`sQr^*a*%H2F
zwF6&nFU%Abs%E(IC;eta*UGiMOM+ybt|cDXmXH#@bnafI$^G@-$K+>TTm0AC!*<`r
ztWWkj(YJO>Ehvu&YxNeicr9rmZR?m?=JfgV;X`-jTBB#A)HN>QduiqAalc`Tl?1<N
z_A%$wpAX!fn@m(E1x>73nvoqQX%p6vrthU(DQaVOx-_}(4nxPh+Xp)5-^$9#-m-P#
zPI;RX23t34u*dIXe{x1FJwc=NQ~vV5pT2U2#m-52>g{h-{#a*^6aOA3u?Z@rNvFOm
z&%Ql1=Y`ew+;H}vW!o8cm72YD73!&EC}xafOOd-1HPP)8+na~lO0^GH)qRS;lVPDS
zd!KTVi`M>&%^a6@3oT%?Sn_;^#?%~<{dc-gZT+`ko8_55r#w4eseCSaE)b&j@cJSS
zt-^-MOE|yI32@A_`^C35Fxb2)M8Wabghuvd&lkE*UpjLSr>*hfw+m+*EIPg6*R@2x
zCx*W|a}_)<b;R+{>ASM<sO5?NPe0CQZd`N1<a*#r6~W7sXZ-%7`Q)C3e%9*cGq#&@
z?zdx6tDbUIm6<t6qE~lS=pz|+2j1Df2UQf9jefsoSZ%QEx$4i(9eXv`Hm>gQJ@?_o
z3mLchGaD^k6er4dzjeHIO|GgTYRcoZJEsm@l2^NXC#HX8XI+7r!YS|PGq!)@*z-r_
z&LssGTi#_aUoG`1x>Q(JvC}<UY4)TQnV%;AKf!!G%fRUJr8o1Rx@C!NT$w5HG1SF<
znxG0-hnlyCy{BC6B)9Le_PgA>uU5X?e7f03VZuCTE1xU-f6Z6_dg^NMqJ4Ip`%O-!
z)QU7tZ~b!FL7-WrY=y%1Sx>$*Dx`kX&hGu^RHw}7b}(F-eVzD%quXZO+^n|d46mg1
z_hnm-i+y?W!YJsyy1bC3;?za8_J@`DRULFs8M23!tnoXp;2r4j&2g7)(&O3B1EOns
zc1o*15M8FQSI%R`sp*Sl8hs0HZCYdXUppfwWBZ&5Ug68u>ITfq)%Ef_!PRrdRC0FB
z-00fN7lO?NCF7bV>NW6Bxmeugpyc{{UyG}pRzuXb)oB|pyM1)KbiX36$8z=IsISj7
zA3jsB3^?hv-;Cv9Ducdkn`Q*Zxjyloxu;)MFy8v;YBn{Ot2<*!%H?cnpZL~1eQ(>X
z9E<CtmYnnHKI8s;FT?GPZz{cH7v7NC`)=jdZx<L!xaUPO3y5VYH6HL@yl>9YZ`u7C
ztUHbVTHIM%%wZ#aY3o#_vnIDzX_Q@Er=K35;1vBg%wmmmcjakr*{EzGqYsL;{5F%C
zXGTsh`!DL^7c=GM^7~v1bZ0Hw+V}6GR>8J=9IGd2>{b_A>3qNP{#Ls$oV{<m-c2r%
z5x%{CNkIHd?p5Z;BWhEb^744jebeAR)%-#G@Pu;=OD;%H3swH}q3_1kh3}VtbvmhG
zlh*dCyWpTx`Psn!X$Hsol#T9O+Y!iW?0w-)XvRd=lW$owE=uMtdoF0dxgq#}4l9GB
z%lfo`7dJM2abVt}UFLf6(~Rw3dj$_i&Jy~$+JD2R-sKPS<~C0EKCu4C6nXc^urFI@
z{hM?9$dN$#ce&TSb=fX>S2zSPP0nJsY+IZfEzG#IuVTU_*6Umbo5E94en|P-{n9%3
z@6OUTVWay2n>@{xvftiNs8V3u#XILx&>p>0p0!`~+ahxlgSd+pA6nmMyx)X1GJi?r
zm0Xr}TZO9>8m!*%O!+eJ4$}<ImuwT{-f9VY*mD_hF58{3i$AZ5;rrD28=QqaTYpN2
zwB5_$>h$=<5Z9cx-avO3n{uJB+k2-2$)_*L&AYDrSK7mFZ|gknrEOc6CA`{l^qT|Y
zmeVH7O}Gzh`lOo%%01DJuM&B>T0Y_2F^fHi1eRRpKDLNqJL5{nRVE?J{@u2;dhQ!`
zk=K^<<YR%URw+jtS40R*<(RRq(Ooj)O1s$lCdpC<i-bw@BGvn38xEOsUv_I*Br-{6
zVyVGyj=ZT$t`>`KygWTMC&%q{T?*St_3P4pC$8n3=yUxo&(Tn(x?%Q|w8o|9`42^&
zWSeq9b*^W{QQ<TC^IGFiJ6#Ey;K<XXqy6-J=0}C{W%uGgdNfvKOlWdsi^&xIa>C_q
zYl%%qme8hoArrOO*w1`8ut=^Vq{-cR#Q_-uu3FQ(wsRkNYF^XLR2E{K@l4BoiCApN
z&zo*Z6Qgar4K^2V*zCQb?fSy6xr_fht4y3Fc6;LXZNIq!yk9gfVb?sFVtF(m&LjDq
zxqio~vzrUdmd}`N&)vZzacP1PlgFo*FT7seIdwMEZdPB7y61<CK#xqP9a2k_6;4~8
z_*Ax|JK)@m`}~q$oWpES912_EF<V9_(^=lnS-PX?qmqTX_H2<ieHu5{Mn^eI1)3>m
zep+zBgEb*FV<S^fh@VvLb;0>Hlhx11xTjQz+1}rKEnw5hdKbSF%Bq`6Q#RiI8@Si`
zQO-Uw=cl(b_wSl|)pe(QU1q?QY3GgB7ckXTO4OC|Ofa*!F8y+emp5Ph+#<HNeDzQ}
z+t%X}OZ+DYh)XSb?HpF*F!jsjV}43)3NJ&86wJaHuQQtP>ZCcp_IEse?10ay?duyS
zx^!K5-%|Id@7EGOncbJ%q&jn^EIzx|YKsSNRZie5&8H1ds#2uV-ZLcypMLT8hFVe;
z?~8?NeJ|F|$oOx|zCx9MrV3wVSm7GG>+)0XNu)oW8N5Pw{*$^yC$_KqE(mI9e3_8(
zqbTZ<XE}>s`#$E#KT;~)PD;zRX->S~vL?K-A;ysDL#)Drw=eh@ESK~dI9xxFBdn9v
z;99a_1z)WOo31qDV^N1E2d5O4Jb64fQ~BYwYbV;CEXkP8Fpb^VZ05|FqG|7C{Ns0Y
zd^{i1eBslEc8&iXE9|D2?{@C4|K56I>W!rf3pBf4MI9<GvgP^8csEkYv9!|b+VyjX
zZZG*Xv(O{aiS1b~=Ns8Fwzk#&Vf*<wnp(<Oq=VcPvr9YPik^tlyb}I>f`r$vEnW&t
zE50gx)sbTCc3IGWWFc=5ZxU~eHuEg`RS~OPdYlquCwh1t<rB@lm9BY7jOCQz$t_$x
zSG3N2QR?Q|-}mFC_CA5tjbFrdwc{GTE>vo}=fCsj_BuuO2fuGH-U?meE_A%;`h01V
zZ0+PlC0|)qaX+6qUEVR^oom2r=07Sf4xI0L@1$;Zl{MS=dY$(I9c{VxD?tKJl~nd-
ztmaaAuyaBl?}w}W#pl1@6j#_)T;#2(y;iYzl30ZrlkUL-!9kk$1WFBVUMt>x`}?lL
z+uS(+MX_G+esE0c@i8e&#nyY(z0o(bU;KVi+ViKN)o_|MgSqu`8Eprz>y>X!dEcA+
zh?H`lbLtN-ztm=KuJ`Jv@rpC^li$4({@~5bu}Y9*`l2Vg(?YXn9lf}AbwR51RSPYz
z1B#Y2pJlTA)a3|jd7XNoTq-DC{^YLtEVmvPP4jEw|KfD|#f2)>8E+duR%NW5Xz%H{
zq|bA4z{7oKlY)z`1s<JHl@vAWrnbSQ`(od2b}Jc$Zt<&scJ?>lg<Qw;cY0#W#jA4y
zPqW<CpL6j}ujj*O?mOe<raRUOG#WF#JkEH^SuBz1_u5S}ue~oyxiR&K;`xmx3a|K<
zJZ_sO67O^^d*MFks=if%cm6BB({B7$CA5uIth>S8L_<H!iT(VI2b{h?`**l~SQRqA
zY~JJ8bGbhz%n}Hkw0}b5C+m!FRuwg_9WP2XgRiS>)_TVO=lcIk=h)?cF5FUg<L${<
zZMJD*N_`Q2(T<rW6V6H8uKU`2=57U3ULHf<I(eT-M-;1fGM?=_^72IOU9mSy{wwNl
za+}A%y(pqdBU~Z-=xv?VWn%9G818<XoRBb+<;%yqrSgTRwr@YdwR}rv=#Eu!vXMQ}
z^H-|~?cB_`R&Qz8WTRa?g7IPoGOs7s|1rN6+1z*Mww`Uf%?ZJrKdw@*&-X9dWuVz~
zilxQ)VACs=7IUtLDhhKdBG!~gFO2(q&$>8%mD2H+DN<A4wN7d)k-5%2w{abJ#n%s|
z9gj~i2wo24))8g*k4y+-$m<SJTB>Uh>KFeq#=|9bOU9%+hY4*0S2yWvx$;+~al6S+
zjjO&E|NFgsR=Q^2p9N)46@_i4h|c&neZR@M>;L~~hNRs;zvryxCza;F7qez-r7gXr
zWA*m*&cCa@<pQIpark(&JBMTo-1cstX0cm*n=r%MJ?gscuFLlGKEA^6<)ocKPE+FE
z#82<{FTMBoV6e{lh7>cEpUJJd&1YRZwnm<O>@2F{zUo3w;{O-&Jokkgmi&q|*Ro+e
z9r-8e*s6V&0$0r0X3my>>{#it>nJD3B)hM6EPi1#ReU;ImxW$s*b#j0fI)iCB>t1l
zYnbLOd0w&5dBP$7O<@ctcD*`qF|a}EpYja0E&Et1{CUnS5>eplE%;z@Q*7R)`r98B
zJwv54ow@iT*E?6GT%CNo$usbr^R#WM-aS_;i}vX8M&CI9CvMx^rIu4zvOHvqST1$@
zytA10@43Y%ae2X~KQEt&tYdL3l{nPx%vNZSHTBS%4JFGGr`<1`F3Z#*)-5ey#HRUJ
z;HZpCo$RJKPUc%@)uc|XFaEN4F~h~TCbMb}JkWADJt^>mk4)5j&e{`9mo+`2S(KGu
zHL$bRPrdiNkY%Uu({&Bc8(7z=d>3Gf-*jZh@k_#b#!82NUC9q-d*0Z;jd9YK%W*MR
z4(z-3y7H=BT&(nUmr56l*~tY(!DsaQSBPmGI=kYP-wLkt(`=qR^OF4`_r5AAg4ttj
z4G+hq%ozWtlP1asg?MY#v2LHa;`+?<Is3Xrnm4`lm?E7$VY^Ic?)umB%H8}<REXGQ
zI=nVKpjoV<bRdT@E#S`-olmzfyUp^<Tk`%Ozd|h+o3z)bjv~fQVVM!L?wy-=V)A1>
zZ~ux5i!c4Ul0VVG;={#n;!}gv0%}elc*d>HY+m0Ly3YRKo!-wMw3#_hNwUPnc&dng
zSn{{!%%^Ky=lz(DIB=^~wsjc_JTaegg;Tt*E;jB3$D0X9Dwp#uVVIPtlheEKGQ-3;
zmyM>AJlG<0MEE=#ze_zZ<iAsN?_mqi<U5lD`uGD&6<3F<Hmd&ZxH7Bd=dV*=?^#3?
zok@P_+5a-I@D0Px`j7eZn^bx7o}4p|vp@f5t^332=Nv3De{AJY$n%sDef?%jrk_wd
zi}ccnfFil(CA!`-IJjm{U=e7%EVS9`!}*sE>i15aotUK2Aj6lkl5gX_RVS;~O5WkE
zpJ6?diM{G^t$)%<iHdl(J;z!4e?AD5KQrk;+G3-Qz?C!kR{n`u^1gDyRil)>B3ADC
z)_Jx3OdZSBXA5a;eBZle&V_9(tNx$7|1<JfjdOR-+{bB8Wh9;&a|khpz2L}ED|@@e
z;U?qWxzf8Y?_8KF+Wy7TWhq<aYDcR+qmxUYw>;W8GyiHQf6ejgJ4I(a%^o%MUUS`d
z>glOTn%SnlPc=Ta8C_0n2$^urh_QQ#(19ejFBj!9BD+i{zwg+#QpTk)+Ee-~Gxygy
zwo5+LGOhUbyZ-g}li6z|rPX&t&Pv)Q*1B?b&*ieG+tXOK@jj0`Z|(DDQLI6aq4(8Y
z23{-81*USPY!&)<DCX15JKc*Vr4+17OwRXo)&vx^#zbBWSbZy~!(#HKNm|}PEhqEV
zx~nf(w0(&~?vW-tQ<hu<<`<;~XB##09G-<s-+Qxr?eSWs6`$tU{b6;;=y|@$XWFJH
zv!qY=RvatoH;zo|y`-Xg!1|+ws^*DVYi}&6SZs83Zk0x<jsM<59}X9)ZrShrnKRit
zh(k<vuIrmGa|+%VZOdM#Aric3Pc4(*TJd?wN*_+NJ+l^CIr+AfV~WIsy_p|McyyxJ
zHm{!W#aU@}@%?}^f0<s0*Chpd9b6jSaX{B?>%tk!_|+x5EqXRHmFflV-J@Y3vFqXO
z(gxG@9DjQj9@(fE!xlN6_jHzU!MmwPZgML)o_x#n@!|W7zx{RW&rWzLY`T5$Q&pSp
zzP~LRxqYXrEE}tOb)>pAeO)JUH}AXh_IRL*(B8SSp|_>{pB|h%Wo^}k$90Tfyz9%I
zyBbbjYEwNOZPuMRM_u>WW2=fuOf5YhEtYDwINtvzblQ`B=@Px7lE_nv8Xm`2PAT8S
zteX1w;`$zur@xClI2|p`f)s9jmwIpeb=L;><vVMOq*@!gdv|gM-jSO6^iRhc5w(=P
zRvgRCBwn>YxppJ-SwV^G??jdrzgFAFsw~yH5oL7r+onc^0Pcf9oX6OIFZddDVbhe5
zca5HtgckBY%k%ZfXSJ0U6Z=`>A`|4ISfnj5n|q<sjcq2@e1~Q+bTX9vN>y=-zc#7)
zzL~q-AFg}f{XJP!{=Toty-@%8b^70@FOP5BRuDF2+c~8Vi-K4LPlfX^pSZ%g-__o#
zN9C1*!}l{i>J2Fm3^#7g*}}wfJIO|nx%=e<!-k`l6_;52gGx6tue-15+gM}CTVB0j
z<rXf-e<HuE_hzY{@cMV7#`3Ll8Mi{(^7y2c8>hEk{cYtvzvkq>gZ2I=Bc>JqJi27Z
z$4d?ytn80<MJKPge$2(Rp6TnW9VM?NcU<LZ`80Xr)ot2|>w2wU2{?SeGO@zFAw%(m
zk-~QVzY}|$q+hq2O<Gi>njyBdZIalXU0*NU3JeMlQ7vtK_pNklcg<<e8ybyS$+7+4
zuZVG`ERB1#a^v)txizLjud5m7|KQD<vhtIFVfDpzKmB6F)ATQIiTkc!|6e&N_upTo
z?N)t{e(m6Cl6btzK5&KfU3bnU3|_t!2JafxcYRuIAG@giHy`JgYQ`MlJ?fnEI-8>7
zBqV$$x)_Dd=PBSSFl(B7%6DPA==`fnDQoY1m{F#+a@Ayyw~P}e)`VtV6kjy|pUu=W
zCb}02uDQ*aD^R@eLw9=fR%!JVMRUch*2x904;Rcee7W1RqExu?zQ?k~4xbt4PMABv
zLF9%{pIOt&iI4f0$w@3oD{WfIyo@cvs^?@}#^TI^$tH~ssSIqaOv{)oOg&DWRt*2m
zXv}2YdGz4@?D&R|jrZ1VJ5=@dY5L_^(SE1y9NV!X{QQZtv-?l<9$oHgX|8knr^K99
zYpcXHy?dAZWMn!smr27gCh_D(t`h4F^<Vhi`S*#nf4C|mvEW&(#)Ogs7Z&m^UHDb~
zKoU>N%8BQlR5u4}X)cYBiT<%rQ#>PoYv_@<+R~?UmT;6?bEcNFuh{ynCj6nwQ?Hrl
z%Ht#^OueHqdw-2l#N0|B5wl|fr@o$w($ucC*G-*}(lo_h@Rj=}#fjfODQ;yA@UArZ
zx$%VaA>R3chV?lGy-SaUB_5Ia?OyjdnP1G|p*Vm1G`_STJ_F|W>ai!Y8=M*aLxV4$
z+q=D9&(wWGHRIevwokMcY-66?aQjXr-=}Hp>??leZD0E9+!>ebI?Z*tx?c;_?H{Z;
z*MI8IEtR6x=`MLsO-{9X{^(=9pQ~~&@c6`IO7FsiZ?SqbI)B$!wct)r%Rb{39$E9l
z=={9hI%j=u{yK5~?;ppoM0dL_#YS^)Z)3Ujm^nhf+04gMUu5}8`T4C&y`DRFM_%1F
z+3V2cfV4%TX08pJnJl_<b)0`Qyk%OsQhMbgulJWeNACRaY{mA1w_EPaux)*GjWcld
zmxHTQ1x&iNR+`3dE-yX3wy5{+&51X(3TMg1`Yk<w<-}%=vpNTTazAI3HLJ4SPpb9b
zb2Rzprn-m6|1Ie~nqhzELjdCvR|aJ+7O^F(SKEn&9QPBfxptX(UI)WHt_OFe8(tjT
zvv^JQx2Y?Ni_KI&e~z4aUyA3|RK*;<c@v-h;SAwc-V`S{-RXu;YLQtrqsf=We;I;!
zZ9-FK>hT^Z?SEOuS(eu3Uj2N@)S1s;ew04sv+l!}ADc`r^_Bjbd_4MEak%lnh8J@4
zS+agTJ2bO!%75>db3W*JxVNuhnecN_kyn;7^MhK?1M!T1s@X5-Ws3YVjFR%+k<Fd?
z{pS5W4ZSywi>jY4(aE)E5=@!FacR=7(_E`=OjVgEvE$qfu2hK!FPS2m{>-p0Z9Vn>
zphP+AW4^~#t&$UEdVa8Z_ryNiyL|oUFF$<Wo}9mPnbG&dc2}<0cl@h+u=s7mb7tNN
zZjqapC<GtACgCzwO36a2jcxwwI~f)xsSS6_8=@St=AO+;%00Wz=KoK}oOLM=!;UYB
zw(j(LsIuy4@rr4`T7vx!eP8*Oy?I$76UVfZ918bXRxGuPdll9<v$Hf!ZQ2aqz0JR#
z&6qmt`N@l-wx*kUH_yMDGxzMlD;jOh^0~|*Kd07DSRTJ2WMNNeFqb`#z_y1R){9tn
zt-YhYC1}O|kcdNzg;HIFCTMoVa3*Q9Unx@!4t!MAt|BmLvA*)8D%WGb;yUUcnP20(
z&6u<PS=5ak7vEoBefS-B{jV7@HNTyewp%H3M>7|!vQ0RX7Gs=vOJ>@^ttE0gfoVM-
z9gO^(Ewi#d%$1B!_}D0^YgiV_SO3BEb?yHNY)rx3f|D%PGB@&Hy}Pxxq3)yl^^^z3
z6Qes%+?mDec<W^@@4RnEw;MCGF5VroZuztp7niM(6Ggq=8Ze$>UKi-4es%l%B9AgI
z1?#^?{=NHk&U{(Dp(?JU=GFBRbF72D|G0hY_u4f9?~hNZf4NhzzVus*#=8DauR@-w
zk3{C2TJhy>ok5y`UdcSoqG>vXX-Bd*1c|P67vCv5e`-}t)aJ+Lu1m@rd2J2z=l$oL
zWDzoL(JyA%uXVTCR{Y_fAKk!|p{vIIdW!cgqXjAPWlo8uCfaKh8&52Ee%qwPb4)?S
z+gT^yX-%MCiSqaFe%p&f8Z@r@2N;-sHVS%tzv=F8t)yms7rTF_SI>Rre1l0y^L*f@
z8-Gs6%TImXt^RkXw?B(__<W(GYcHQ!eLHB!w0pr9B37;1oMCoc=<#hG!<vb_c0ajZ
zSAN+vPhy7iH$kR$nYdN4w-QyKzEjPWUl9NEcRknr%SVqrT5)E5ztayh<}C*IU7y;&
zpS|pYoWt6!cY~a|+0J<VQb}yul)XUDL{pik@np(MuE=$*Pp!*8Ys#`J6dee(WO>fm
z|1~Df_Hh2DCl6jr{M`4vck+DY+&Mu4FRs@<Icc@~c<9ypb((8)*M2|wG&}j=vp+2w
z&)r-ySIfs)F8ylNlxlNOxmH@oa_cdnzr}9vZ_Yk3C(_MmZ|@PwTbpLBt>@nw`}S*f
zj{Ji7-_!py#+{wweYyYConMQM3RZsWp2DB?takOg*Lqfgw;5LQ94J_!+x?}&$nH^-
zTHbmwor$Yw6xis+dQHlUSo@pv&|;<~HyL>@KR-E7a_#o|Hg$Ksu7mPbaq_<>Mly&@
zRPQMJ(tWt(U)EnP>%J35U4;wpJDGay_?k6q{rRuI@@>suKApz)HE#F0&D*qv9<Tn@
z`95AmXXf^v6h@smN3+#!zvaH#Suq8C{r>N4zu|SmBhUVLM(WHF`Fd%`m%ENX^1OEC
zH3e3>NW|`2TvVIhP`3N_$>OYE7kv0<&1}xgHL#9OzNlz_Vkf8Jb_<>D+p3q!XK-lz
z|Iy&Sn_;C)itL%Lpv@XuAzTMqA8gxuF8LeV=A_M+-t0ZJ_DjuzqvEYMH|H(!&t3T6
zokNZ3<%!MvzDa$4a3;R1TYUcn(d^BcH|0Nn`PU+TWI5NLFUGeg&YPHKbZz67@44$r
zZVRpWwf4X9F{YK7PN%sl{vKSqet-NLeH%?K>B$;-Js#)1SD3V_xm0^B+RY>~Yr&I4
z3u-+Kv;&*(20N-)pRMxWe{jvhO%6<-j$OU@;MD5(!n3oNADsF{`T6;SecKq6DohS+
zIcUs$!6=e5aGH!O;|{&Yjf;2H8!l$B-KO&P;dX(!d(KT>o}cih*qAqax#jEE0l81U
zeUNSaxA<mc?q}=zoG-cSGbFaE2q_CqENM7y|Au+_p=Gm=-$;phxtTrsV|2RbtN&tF
z`!yC!Dfs_n`uS?@>*r3cy{b87+j6Gsk||CpihnGa&Ie?2ZSOz9_Wtn&Hw%Y_JoE1y
z*5Y$xw9B$|I;Zdd%FVB6@rkuh)HeS%6Of+qkl8<`(elCJ#K8ZL%+FWm@Bgy*#M)Q}
z#?>qJ<k_wHm;^LmsI^+JI4r@unB|)bm(9cpK4Gd&MoV7@$-WXe&A=eo3R(-`=Vh^|
z^VXZ9V(zM}uvsfB^U|I%Cntn$HqzeE9xizH*#S9irOAq!OO8dzxlYVA{9SJE{*~FL
z@9iq(=oukTPjT6p=HLB!N_>;j_ABj=j?E~^THf^NS!URcJqp=N69aW!X9h2Px2pHn
z9D_|h427cHi@UmZ@%&yF(9qBBvpHsEq^a1;oyQ(Kn<uJ2-*V-O$(k#>I{McC3bkJq
zn)UacbbRgp8UMZ=4>)DZ>3dz&h255qX@VrHcUow@vtQ~FE8g=9=k|Ot^hyX4UEjNk
z@zVJf;l`1cSFT^za_D}xWMbUIiT_qRzWHOS>!n*}zk{WE!jiyX_g59qe>CekE<cwY
z&9w5a>#1ej^ZqTlcTR2HzFDQ8x`hw^xjH%N<E_aWIldjA%J^3ujbl+_lvJ5=QlItt
zUYQz~#R;oi9t2!xW{+8IkSCTfX}9ghqmP`!E6(4{oF_N)x}TkK^AUlcFSXB^|Bs)q
zljHMa`u<;nmeoHWZwji}rOdGMT$M`hAx3*GmkA!tH^p|y@d@+nKL7se>4NWH9N#%_
zesy7S<%^e@nO`qoIVgQcZbtL8h<3{zT_y2?OII4TwY~d&VaMVFB^nJ;3S5`X8P}G)
z-MZuD)JsXFs@;DsNqmY{t}D#tIr>&J^42_utrm*4*HaqPKg6-wIeop?*zzHh%U3R}
zrqi`5Eg>y0t8?P)Lp{2mZ(NuqD1G8_plL-|r0LiFe-<-dIi-Fi`h0!4LG?@FO}7*q
zJRU`C40z;{V=!4Q@t0Qdq{3ITytO1bXYRGCHes6f@XJBB+o67U`Clc}NMCsA?znm1
zwH;TjWmV+sgS~t76|J^3-DsT~x#D})ZA&hL-Ro^l9I9u`d^qJvfRbwA<DmSvZ^hM*
zygHMq6K^m4#QfZ;C`;Du(=JpkJ=SiSu6CBup;Pi=sMzb>2ZPnuZMgUO`WOCx#X)wN
z7jj;|$lkjB`n<nu6WQJ`ssH)&z3ArrTBdf^j1mK$Wl7vT*$H7TODhUqwK>Kbnp`@6
zD|>0t>Z5s!n`KH319Hn!8EhQ|L>H*8JALAWYo1El0{1+J?bj1}^OC=8{!-h&M7&g}
zr+HtvJ42R-O3<3a1{POO>c{l%y1YL9PH$`7+8uTpr;aM_+-M*fvhDbmEj<~`m!52V
zp|<FzWJIUxu{EM{FC)3C&gVq={}VCYpVzl5?A^Cx;z_cs84Ao|g$EX^Xv9p_TD&Vk
z;z<mD`L(iHmRs8nIeTvkT(aKkcyMz@^Q->t2eycwJj43oNNJ|(y3C86+YA0ZKiU3d
ztMnP07tMC-K3lO}kXx3PJ&S2;jpvoR=kc)|&(_UJ?W}70&Jw6=yybv?&9~COUyPp|
zT*@H-(z4p>!K{}H=Q4Hl^EgPqU|e3dyOwRn4L0r*xx##IFV7s>vg>f_f*9eT9}^BS
z8;7^do0sPHs#H$$x0K)CH?At~t9^PI>nr-st+~AJ{cdgX87h-bvFrVRxBuPr)z_;l
zW1_0`eb%NgkeE}KcGc=9r_lYg4EN_LHuN#McoiwFvueovEB57J$E?hQXQOUBb$V$v
z*JJ(NrxKwlQjA+V^lc(VMDv}!KJ!kK_`B)EzYE^e@^T;iNiXJ~tXHDL@OGiRX|3ql
zW3$e_?VA^L`C;kxzxx+6T)Z1_<J^S>{KpscA3gYp$;OeZIAc*c+k$d0p_NBjzxXI<
znV!FYZ;f)o^ocWm7+e0Rjc;9g)_vZO<ELXAzD+ow8?;igK(%95Sof<V1>0L+TR4c!
zyATm!>#bX=9JF`EL-VUYgM%k56D_-;>zmuMYWa$FMVnUM=Kj1yV4i;J-e;$aZcaV3
z<4pBA8|`IvFSLV<pDBs|yLYKB`=df`!EwFDcVALx&%4t6@0Jx`z~^IAd{i<;R{g&f
zAoD9j$mZnLX>OCZA5PSGE5B;H;M3INS&tStS6Btynx?d$>E#ucmiLD{;?wp%pMP}b
z{(mw$pO^hT@my>d!`A)Re#aH&Fl=pd(R91=WOw_S$|AL@gl#PSvmcaMyzXAf_o>`^
z#aV;M!n7CWOT!L_6t0$<`+38)`BL?4ehZ4a&fG|yS^Fij?%Cz@2d|{>VtCtq=%JrQ
zS;MTTSy>sLpZCp}m3Azc?aM=zEj+4{SFXH1=f2pMxAb>1r^o#Ujht1}yLeZ7wm*qi
za=m+>u_IN!TI_lK7t7hNjbC^zILx@|e&~0W1=nUP^iMl_@DbY=Mk&{XrrCTmJcBx`
zL?=yA6c_bWIn|@m=^T4SLwb@_ebfZGW{uon<M+pU?uhA|SxU5WERn3AoUUKP`P}~V
zoU?BwZ9ctxC$>$Vr~T*mYQ7782YGW-*(@A|mrKUWMDgbqo2a)%UAyo^MLXx&Tds~I
zN7JK0nOj$}%Ab9_pk`fcUZb8v`@shvuHPwcD7(Mv*ntpnv#ejatqg(J1*IgG&&-ms
zcxkn02LEi=6>fs%l1kTEUwkl-d3NeIpMa;zrTz>p<E=NhnyU6zbj@-P5HZbJxjO2@
z#6D+Pw|Cbb?BTwUXkhWj^t<eWxzYQ#ihi_X-gBOXXOfA@v`l;MwNHPSGg{g3wD0`n
z$kMS?qGDFnhF@GtQ@<5nSj>IFF?^Nrb6X3ByT9yo-ZdX>j%E(vn<wn~aoal4Ymz2>
z-}GV|__q0d|F!0D#f$e}ZC9Ot%~TRUt@b|uTc$1Vh3@`r@ykgRIhAnJeZH&tl)@JE
z!UpgAYlS=*uhc*E{<U-e&pGQq<{5lt(2;pHrH^@uR&T`OzI|7#tab$C`7`ZR-q~X$
zQaj`0`wee?>c0}V`#$HK@@I?BE2rj}{0r!~^LNoXrYkRGyai03XTGiZW5_J>%+uRc
zBF;-Z?GNu4yKnVh`Tw1n&U-T4;OoZT#Ue6thcDg#s>{DG>ie&AWwmkcj7kNIu4G(h
z`kdL8;kh(AKp<rAp<Cx__ka8+l(GHF!bhHmr_PFfK5L)u&W&C|9JcBE#VY^E?`>ML
zulLL3*5+p(Z>yWXKWO<I{!;GF5l6d4ix~nxCM@IE5oG@3yCFe#liN<KD{SjIFaGl`
zPH2$&d{pC};#!wZ_szFAT)ynVxBr}Sb9h9_k5wN}EYDMNyqxLwI<|q&$z}HQ5)p;N
zET{RJmQQYoVB{9dV|=p3VT*F_w`m^lSu(<dmAEDrO07MkadQ_-RzhZ~Sw&Gvr>fh!
z4B`8~H%a_@uqN@?b-}m6b8T+?lg!Rtcjx20egDtDsj&Zl;C)>VL!u-P_Z3_ANezV>
z%iA4=JWBogL$<Qs%l5u4ydbSykmKMCF%R2EPW_f={Pv$+d{g>R&-v`mMroaU`&E9O
z)1P(ovgMISq6*$mZQkEKbclI!!3p7h)&)L7J-W??ryg?gn+BXvt&m7@4OY<H+o6!f
z<DNI~^y9`moT)A>2d1(FwN991*q&J>>bB{K@=T>kkJo=ZxLoh!kq^GPUtVUaEL*|X
zr<EW2Ug6k{W{b$Q`^&%Ho90l>=+ZyYh)eVJ6<*Z|lic)}_q4fW>1cPGGIFW(vj|D$
zhR)JAU3_HC4=(dy?hQfSI@4mC?$%HKK7ZzQ<-fBfYO<I=I-csaEqc7NcK!#$`4?C-
zrnB6d=)#-1*lK4;<_B5!4DZ#2E7Pt`KA^}c@>x~M=|jhdr3Yg(!bGcT<{oWdzv18K
z`-Vo#*Jr)|otgD4HY4O-P*u`qWq0*e;{B2nckiziU$C3eCEMNCa>^IQyB*AWlNa_K
zIiReRP^KVo^|X!b-K4S<(XdBqJy)2z)1GZTa%9at>z!@x|G!mvKh;&`mG$0#=(l5#
z_yTK&Ex!{r-m2ZXw?w1x1lMFAal0Oc2c1W><RxaO)h6d}Iw@asujb2f&qovboY!AV
zyH~C|Z;w2K%va;x3|pB6e)g}lb&?g*W9d1;w&n#}x5~20`wv|2S=Zg2^W%f*oL?rU
zGRyT+z69OSbU$!cjbHw&=e^~xK{^90PR_g2$kZrebg6o(>hblPGTyuHeSYQhuM<Bj
zdFGr}uAC?GB+dU>`P?g+sav@(Ff#7inz3qb?2VOnllGptq^_qlt#&G}-Ph9BFY7nm
z$uL@%Hf6uG>bX3}>#Q#f7&u>9xiNg*)AZ!X>W*)F4kax<f4lyvc*Ye*%@esQ?)R7P
zUG6M@oAreT!_Ak|CrZ>xhW!%FREzJh`+R%#d)105an}_E`v2y>o2p**U%%-0iPLin
z{W*@;l{4O|XMKNKl3&Zg#`N2yjG5B+SzqjXu<?I;8RM<`-gn0(@9MtMu$nNnYNdAL
zJN9kuKhCUUu{yHoZnO6;p%vPVeJzpV3-laf8*a&MyX2F3S6M#e|J6Nrd)~it_`UuH
zbD>)}dzFIRVe7m_yIV`QZ&052vw-PIwff)f{RV57?{1aloYih8CI4gV<UglQ-+z35
z@6VIYz16RnzR0n?=-zL4gz3FY!PB4bKR&<rPbKT<@|kYTzS7G2?~U?m(y#Kd26&$p
zcYW;o;IQAmBa@W%J(d5i>R2A%@%QTe*!OKlR>#tw-}mr3x%&TWdp?#!GY?(OujLn8
zR~HoGEuXL}(@D7@LruHX=f-mjE{9d8Oy20RX@B&cCi83AmZ0@+YmRhRJnQFJ&~6u4
zxBqFi#&&~UnW>$%AE#Gq%Slh19{(p`L($*IE5hv#?3l(7p))5`e<kyex7iz{S8Zds
z6?)(AZG>c2!X7r|e@sHo`X4-g@m@F7SF8Kezik)8*5>=RcHxX)uIeAjS~N{)`o1d1
zHD~JQubDc3nqMwgqxF#`3-nqahGe?Qib%PuGXza?7yVpeuvpq4N!nm?{KMMd_~f*2
zZjSwWDfYiUPt|kyF40wHzs|7ij>g~Z_2vtoi_H_6dgyUd_1D5k{kcsaCI_(;@~!k{
zoNT}~o8c|f%XLB-;kCPGeO>bEb}i=>iR;~G&Yk)A<AF~1w|d{6`aiC(Kb{X#Q#yFr
zZT|hT{Xd&4|NZ-Qh`H?k684KZ*FOFKx0L68oMdCL{M4(jJYQN0Z`xGqc1>G7V(%=G
ztwn7Y7wFx(=EPazZEwZlaMsQ)=KIW<D-L&vEft%4{mqjt7Zz`hGAW)c`7--g=lfX)
zuNa#YUaVZNd7<T)Gkd_~=WQyB0%ry8c&A`Cy=~R`f4{zWZ=Lym(roQ!rTu27RCEf~
znJQ#8vAkgq5SG(qxt!~MjLo~_+xm@Hj;y}*#C`wCHD`M4ejMR><5L`>{?yTa%GS5;
zQ~CLg9nNl?$r8wP+C2E2`M=|t`PG$OqT7z8Y_(8*JvH!|<>5nXm(F}XBZwtsWub)F
z<G;-bwz_9Uqg4A=<!|RQIXy4BGt}kR^pq|6k{1`wzI7q^a@*W{M_wq;c9><AwW71+
zz3JU_cWn!;#_ykhxz+SIT-vxZbJ4yjEyo|7JL|0;aBl9$=l5!2^7L#Jy`z;k1@#;%
z)-mvH&MdrV&oa%^=-4#X<j*G8o4RIrKHA~5JvME*_|_{Y^OcJqt(s<68p^XL|6s>0
zO9Sief%+CV-~4RfB<sC={&i>V#;9#ZzxY@L_vV-?{#u*Q^JC{#!SZL9Ja=v_iF#&!
zJn(FK>e^fCw%hsVsBYP(Goj1)Sh(cnX%Z&N=XXRlNX90o)x_Sg?Y1$k41I5Fxc+NQ
z#E~ofIcBeJ&vtEoKkL`G#SULDFmF-!xP5tI)=GiP+doS_yjs1nqBdxa{v45=OM{b=
zKL$R#>K0*jRA9TrWpmTCHN|Pm^FBrV=jrZ`bNV(>+om?j@WLA2)7!c{=B!!pT%`1B
zz~1_M+OEy)jIU#vFX&A;A?}h~qP!%fdi|f93+B`pABmnlcf#x9^d-TviGj*|vu~f#
zv5Yc_pPp9b=go6gsrJz>%dZzDeQKLj_)Hs=3OvqNUca_oo@I*l+|TQnS3KO9mzTLP
z`QxY4hUrhej5I_qtq@_`8GWMb)vXmATDCT+xgY)=(R%#MYJ2eOL+A1$C;LA&{1SL;
z)vO15(x*Q8`=BbEX^Xei#d4JiC!&;PHJ{$CS$w_zC%<1}bs>YZy7*WA`n0eNw{xfZ
zo;SE?TfFc5_3f|1&fEL<uV%D5b*N;$?G3IHMynfErRR4Y<|<))wWtuwo`s3i)>jvb
hFVJJg*zEAnUd)5h#bR#u9R>yl22WQ%mvv4FO#nnWO+o+w

literal 0
HcmV?d00001

diff --git a/tensorflow/contrib/lite/examples/android/res/drawable/border.xml b/tensorflow/contrib/lite/examples/android/res/drawable/border.xml
new file mode 100644
index 0000000000..dd1d64d1d6
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/drawable/border.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<shape xmlns:android="http://schemas.android.com/apk/res/android" android:shape="rectangle" >
+  <solid android:color="#00000000" />
+  <stroke android:width="1dip" android:color="#cccccc" />
+</shape>
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/activity_camera.xml b/tensorflow/contrib/lite/examples/android/res/layout/activity_camera.xml
new file mode 100644
index 0000000000..1a22d4b33e
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/activity_camera.xml
@@ -0,0 +1,22 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:id="@+id/container"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:background="#000"
+    tools:context="org.tensorflow.demo.CameraActivity" />
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/activity_speech.xml b/tensorflow/contrib/lite/examples/android/res/layout/activity_speech.xml
new file mode 100644
index 0000000000..2fe1338da5
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/activity_speech.xml
@@ -0,0 +1,55 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FrameLayout
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    xmlns:app="http://schemas.android.com/apk/res-auto"
+    xmlns:tools="http://schemas.android.com/tools"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    tools:context="org.tensorflow.demo.SpeechActivity">
+
+    <TextView
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="Say one of the words below!"
+        android:id="@+id/textView"
+        android:textAlignment="center"
+        android:layout_gravity="top"
+        android:textSize="24dp"
+        android:layout_marginTop="10dp"
+        android:layout_marginLeft="10dp"
+        />
+
+    <ListView
+        android:id="@+id/list_view"
+        android:layout_width="240dp"
+        android:layout_height="wrap_content"
+        android:background="@drawable/border"
+        android:layout_gravity="top|center_horizontal"
+        android:textAlignment="center"
+        android:layout_marginTop="100dp"
+        />
+
+    <Button
+        android:id="@+id/quit"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:text="Quit"
+        android:layout_gravity="bottom|center_horizontal"
+        android:layout_marginBottom="10dp"
+        />
+
+</FrameLayout>
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment.xml b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment.xml
new file mode 100644
index 0000000000..a1bbdf1702
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment.xml
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+    <org.tensorflow.demo.AutoFitTextureView
+        android:id="@+id/texture"
+        android:layout_width="wrap_content"
+        android:layout_height="wrap_content"
+        android:layout_alignParentBottom="true" />
+
+    <org.tensorflow.demo.RecognitionScoreView
+        android:id="@+id/results"
+        android:layout_width="match_parent"
+        android:layout_height="112dp"
+        android:layout_alignParentTop="true" />
+
+    <org.tensorflow.demo.OverlayView
+        android:id="@+id/debug_overlay"
+        android:layout_width="match_parent"
+        android:layout_height="match_parent"
+        android:layout_alignParentBottom="true" />
+
+</RelativeLayout>
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_stylize.xml b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_stylize.xml
new file mode 100644
index 0000000000..1cdb24cab0
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_stylize.xml
@@ -0,0 +1,51 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
+      android:orientation="vertical"
+      android:layout_width="match_parent"
+      android:layout_height="match_parent">
+  <org.tensorflow.demo.AutoFitTextureView
+    android:id="@+id/texture"
+    android:layout_width="wrap_content"
+    android:layout_height="wrap_content"
+    android:layout_alignParentTop="true" />
+
+  <RelativeLayout
+    android:id="@+id/black"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:background="#FF000000" />
+
+  <GridView
+    android:id="@+id/grid_layout"
+    android:numColumns="7"
+    android:stretchMode="columnWidth"
+    android:layout_alignParentBottom="true"
+    android:layout_width="match_parent"
+    android:layout_height="wrap_content" />
+
+  <org.tensorflow.demo.OverlayView
+      android:id="@+id/overlay"
+      android:layout_width="match_parent"
+      android:layout_height="match_parent"
+      android:layout_alignParentTop="true" />
+
+  <org.tensorflow.demo.OverlayView
+    android:id="@+id/debug_overlay"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent"
+    android:layout_alignParentTop="true" />
+</RelativeLayout>
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_tracking.xml b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_tracking.xml
new file mode 100644
index 0000000000..ca18ea075d
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/camera_connection_fragment_tracking.xml
@@ -0,0 +1,34 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<FrameLayout xmlns:android="http://schemas.android.com/apk/res/android"
+    android:layout_width="match_parent"
+    android:layout_height="match_parent">
+
+      <org.tensorflow.demo.AutoFitTextureView
+          android:id="@+id/texture"
+          android:layout_width="wrap_content"
+          android:layout_height="wrap_content"/>
+
+      <org.tensorflow.demo.OverlayView
+          android:id="@+id/tracking_overlay"
+          android:layout_width="match_parent"
+          android:layout_height="match_parent"/>
+
+      <org.tensorflow.demo.OverlayView
+          android:id="@+id/debug_overlay"
+          android:layout_width="match_parent"
+          android:layout_height="match_parent"/>
+</FrameLayout>
diff --git a/tensorflow/contrib/lite/examples/android/res/layout/list_text_item.xml b/tensorflow/contrib/lite/examples/android/res/layout/list_text_item.xml
new file mode 100644
index 0000000000..526017fbb2
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/layout/list_text_item.xml
@@ -0,0 +1,25 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<TextView
+    xmlns:android="http://schemas.android.com/apk/res/android"
+    android:id="@+id/list_text_item"
+    android:layout_width="match_parent"
+    android:layout_height="wrap_content"
+    android:text="TextView"
+    android:textSize="24dp"
+    android:textAlignment="center"
+    android:gravity="center_horizontal"
+    />
diff --git a/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-dimens.xml b/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-dimens.xml
new file mode 100644
index 0000000000..820eda0e55
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-dimens.xml
@@ -0,0 +1,24 @@
+<!--
+  Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<resources>
+
+    <!-- Semantic definitions -->
+
+    <dimen name="horizontal_page_margin">@dimen/margin_huge</dimen>
+    <dimen name="vertical_page_margin">@dimen/margin_medium</dimen>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-styles.xml b/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-styles.xml
new file mode 100644
index 0000000000..09303314e9
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-sw600dp/template-styles.xml
@@ -0,0 +1,25 @@
+<!--
+  Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<resources>
+
+    <style name="Widget.SampleMessage">
+        <item name="android:textAppearance">?android:textAppearanceLarge</item>
+        <item name="android:lineSpacingMultiplier">1.2</item>
+        <item name="android:shadowDy">-6.5</item>
+    </style>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-v11/styles.xml b/tensorflow/contrib/lite/examples/android/res/values-v11/styles.xml
new file mode 100644
index 0000000000..c2d1babc12
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-v11/styles.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="utf-8"?>
+<resources>
+
+  <!--
+        Base application theme for API 11+. This theme completely replaces
+        AppBaseTheme from res/values/styles.xml on API 11+ devices.
+  -->
+  <style name="AppBaseTheme" parent="android:Theme.Holo.Light">
+    <!-- API 11 theme customizations can go here. -->
+  </style>
+
+  <style name="FullscreenTheme" parent="android:Theme.Holo">
+    <item name="android:actionBarStyle">@style/FullscreenActionBarStyle</item>
+    <item name="android:windowActionBarOverlay">true</item>
+    <item name="android:windowBackground">@null</item>
+    <item name="metaButtonBarStyle">?android:attr/buttonBarStyle</item>
+    <item name="metaButtonBarButtonStyle">?android:attr/buttonBarButtonStyle</item>
+  </style>
+
+  <style name="FullscreenActionBarStyle" parent="android:Widget.Holo.ActionBar">
+    <!--  <item name="android:background">@color/black_overlay</item>  -->
+  </style>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-v11/template-styles.xml b/tensorflow/contrib/lite/examples/android/res/values-v11/template-styles.xml
new file mode 100644
index 0000000000..1ad048439c
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-v11/template-styles.xml
@@ -0,0 +1,22 @@
+<!--
+  Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<resources>
+
+    <!-- Activity themes -->
+    <style name="Theme.Base" parent="android:Theme.Holo.Light" />
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-v14/styles.xml b/tensorflow/contrib/lite/examples/android/res/values-v14/styles.xml
new file mode 100644
index 0000000000..cc370849c0
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-v14/styles.xml
@@ -0,0 +1,12 @@
+<resources>
+
+  <!--
+        Base application theme for API 14+. This theme completely replaces
+        AppBaseTheme from BOTH res/values/styles.xml and
+        res/values-v11/styles.xml on API 14+ devices.
+  -->
+  <style name="AppBaseTheme" parent="android:Theme.Holo.Light.DarkActionBar">
+    <!-- API 14 theme customizations can go here. -->
+  </style>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-v21/base-colors.xml b/tensorflow/contrib/lite/examples/android/res/values-v21/base-colors.xml
new file mode 100644
index 0000000000..c16da7c51c
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-v21/base-colors.xml
@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources>
+
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values-v21/base-template-styles.xml b/tensorflow/contrib/lite/examples/android/res/values-v21/base-template-styles.xml
new file mode 100644
index 0000000000..8890d2f4a5
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values-v21/base-template-styles.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources>
+
+    <!-- Activity themes -->
+    <style name="Theme.Base" parent="android:Theme.Material.Light">
+    </style>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/attrs.xml b/tensorflow/contrib/lite/examples/android/res/values/attrs.xml
new file mode 100644
index 0000000000..56e5beae76
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/attrs.xml
@@ -0,0 +1,14 @@
+<resources>
+
+  <!--
+         Declare custom theme attributes that allow changing which styles are
+         used for button bars depending on the API level.
+         ?android:attr/buttonBarStyle is new as of API 11 so this is
+         necessary to support previous API levels.
+  -->
+  <declare-styleable name="ButtonBarContainerTheme">
+    <attr name="metaButtonBarStyle" format="reference" />
+    <attr name="metaButtonBarButtonStyle" format="reference" />
+  </declare-styleable>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/base-strings.xml b/tensorflow/contrib/lite/examples/android/res/values/base-strings.xml
new file mode 100644
index 0000000000..ebc5dc8423
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/base-strings.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<resources>
+    <string name="app_name">TFLite Demo</string>
+    <string name="activity_name_classification">TFL Classify</string>
+    <string name="activity_name_detection">TFL Detect</string>
+    <string name="activity_name_speech">TFL Speech</string>
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/colors.xml b/tensorflow/contrib/lite/examples/android/res/values/colors.xml
new file mode 100644
index 0000000000..584ed6052d
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/colors.xml
@@ -0,0 +1,19 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+ Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+    <color name="control_background">#cc4285f4</color>
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/strings.xml b/tensorflow/contrib/lite/examples/android/res/values/strings.xml
new file mode 100644
index 0000000000..ea20ee78e0
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/strings.xml
@@ -0,0 +1,20 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+    <string name="description_info">Info</string>
+    <string name="request_permission">This sample needs camera permission.</string>
+    <string name="camera_error">This device doesn\'t support Camera2 API.</string>
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/styles.xml b/tensorflow/contrib/lite/examples/android/res/values/styles.xml
new file mode 100644
index 0000000000..dd1d973e9b
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/styles.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="utf-8"?><!--
+ Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<resources>
+    <style name="MaterialTheme" parent="android:Theme.Material.Light.NoActionBar.Fullscreen" />
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/template-dimens.xml b/tensorflow/contrib/lite/examples/android/res/values/template-dimens.xml
new file mode 100644
index 0000000000..069977b6a6
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/template-dimens.xml
@@ -0,0 +1,32 @@
+<!--
+  Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<resources>
+
+    <!-- Define standard dimensions to comply with Holo-style grids and rhythm. -->
+
+    <dimen name="margin_tiny">4dp</dimen>
+    <dimen name="margin_small">8dp</dimen>
+    <dimen name="margin_medium">16dp</dimen>
+    <dimen name="margin_large">32dp</dimen>
+    <dimen name="margin_huge">64dp</dimen>
+
+    <!-- Semantic definitions -->
+
+    <dimen name="horizontal_page_margin">@dimen/margin_medium</dimen>
+    <dimen name="vertical_page_margin">@dimen/margin_medium</dimen>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/res/values/template-styles.xml b/tensorflow/contrib/lite/examples/android/res/values/template-styles.xml
new file mode 100644
index 0000000000..1b87714a49
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/res/values/template-styles.xml
@@ -0,0 +1,42 @@
+<!--
+  Copyright 2013 The TensorFlow Authors. All Rights Reserved.
+
+  Licensed under the Apache License, Version 2.0 (the "License");
+  you may not use this file except in compliance with the License.
+  You may obtain a copy of the License at
+
+      http://www.apache.org/licenses/LICENSE-2.0
+
+  Unless required by applicable law or agreed to in writing, software
+  distributed under the License is distributed on an "AS IS" BASIS,
+  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+  See the License for the specific language governing permissions and
+  limitations under the License.
+  -->
+
+<resources>
+
+    <!-- Activity themes -->
+
+    <style name="Theme.Base" parent="android:Theme.Light" />
+
+    <style name="Theme.Sample" parent="Theme.Base" />
+
+    <style name="AppTheme" parent="Theme.Sample" />
+    <!-- Widget styling -->
+
+    <style name="Widget" />
+
+    <style name="Widget.SampleMessage">
+        <item name="android:textAppearance">?android:textAppearanceMedium</item>
+        <item name="android:lineSpacingMultiplier">1.1</item>
+    </style>
+
+    <style name="Widget.SampleMessageTile">
+        <item name="android:background">@drawable/tile</item>
+        <item name="android:shadowColor">#7F000000</item>
+        <item name="android:shadowDy">-3.5</item>
+        <item name="android:shadowRadius">2</item>
+    </style>
+
+</resources>
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/AutoFitTextureView.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/AutoFitTextureView.java
new file mode 100644
index 0000000000..eff24afdba
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/AutoFitTextureView.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.content.Context;
+import android.util.AttributeSet;
+import android.view.TextureView;
+
+/**
+ * A {@link TextureView} that can be adjusted to a specified aspect ratio.
+ */
+public class AutoFitTextureView extends TextureView {
+  private int ratioWidth = 0;
+  private int ratioHeight = 0;
+
+  public AutoFitTextureView(final Context context) {
+    this(context, null);
+  }
+
+  public AutoFitTextureView(final Context context, final AttributeSet attrs) {
+    this(context, attrs, 0);
+  }
+
+  public AutoFitTextureView(final Context context, final AttributeSet attrs, final int defStyle) {
+    super(context, attrs, defStyle);
+  }
+
+  /**
+   * Sets the aspect ratio for this view. The size of the view will be measured based on the ratio
+   * calculated from the parameters. Note that the actual sizes of parameters don't matter, that
+   * is, calling setAspectRatio(2, 3) and setAspectRatio(4, 6) make the same result.
+   *
+   * @param width  Relative horizontal size
+   * @param height Relative vertical size
+   */
+  public void setAspectRatio(final int width, final int height) {
+    if (width < 0 || height < 0) {
+      throw new IllegalArgumentException("Size cannot be negative.");
+    }
+    ratioWidth = width;
+    ratioHeight = height;
+    requestLayout();
+  }
+
+  @Override
+  protected void onMeasure(final int widthMeasureSpec, final int heightMeasureSpec) {
+    super.onMeasure(widthMeasureSpec, heightMeasureSpec);
+    final int width = MeasureSpec.getSize(widthMeasureSpec);
+    final int height = MeasureSpec.getSize(heightMeasureSpec);
+    if (0 == ratioWidth || 0 == ratioHeight) {
+      setMeasuredDimension(width, height);
+    } else {
+      if (width < height * ratioWidth / ratioHeight) {
+        setMeasuredDimension(width, width * ratioHeight / ratioWidth);
+      } else {
+        setMeasuredDimension(height * ratioWidth / ratioHeight, height);
+      }
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraActivity.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraActivity.java
new file mode 100644
index 0000000000..15d5456f02
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraActivity.java
@@ -0,0 +1,450 @@
+/*
+ * Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.Manifest;
+import android.app.Activity;
+import android.app.Fragment;
+import android.content.Context;
+import android.content.pm.PackageManager;
+import android.hardware.Camera;
+import android.hardware.camera2.CameraAccessException;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraManager;
+import android.hardware.camera2.params.StreamConfigurationMap;
+import android.media.Image;
+import android.media.Image.Plane;
+import android.media.ImageReader;
+import android.media.ImageReader.OnImageAvailableListener;
+import android.os.Build;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.os.Trace;
+import android.util.Size;
+import android.view.KeyEvent;
+import android.view.Surface;
+import android.view.WindowManager;
+import android.widget.Toast;
+import java.nio.ByteBuffer;
+import org.tensorflow.demo.env.ImageUtils;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+public abstract class CameraActivity extends Activity
+    implements OnImageAvailableListener, Camera.PreviewCallback {
+  private static final Logger LOGGER = new Logger();
+
+  private static final int PERMISSIONS_REQUEST = 1;
+
+  private static final String PERMISSION_CAMERA = Manifest.permission.CAMERA;
+  private static final String PERMISSION_STORAGE = Manifest.permission.WRITE_EXTERNAL_STORAGE;
+
+  private boolean debug = false;
+
+  private Handler handler;
+  private HandlerThread handlerThread;
+  private boolean useCamera2API;
+  private boolean isProcessingFrame = false;
+  private byte[][] yuvBytes = new byte[3][];
+  private int[] rgbBytes = null;
+  private int yRowStride;
+
+  protected int previewWidth = 0;
+  protected int previewHeight = 0;
+
+  private Runnable postInferenceCallback;
+  private Runnable imageConverter;
+
+  @Override
+  protected void onCreate(final Bundle savedInstanceState) {
+    LOGGER.d("onCreate " + this);
+    super.onCreate(null);
+    getWindow().addFlags(WindowManager.LayoutParams.FLAG_KEEP_SCREEN_ON);
+
+    setContentView(R.layout.activity_camera);
+
+    if (hasPermission()) {
+      setFragment();
+    } else {
+      requestPermission();
+    }
+  }
+
+
+  protected int[] getRgbBytes() {
+    imageConverter.run();
+    return rgbBytes;
+  }
+
+  protected int getLuminanceStride() {
+    return yRowStride;
+  }
+
+  protected byte[] getLuminance() {
+    return yuvBytes[0];
+  }
+
+  /**
+   * Callback for android.hardware.Camera API
+   */
+  @Override
+  public void onPreviewFrame(final byte[] bytes, final Camera camera) {
+    if (isProcessingFrame) {
+      LOGGER.w("Dropping frame!");
+      return;
+    }
+
+    try {
+      // Initialize the storage bitmaps once when the resolution is known.
+      if (rgbBytes == null) {
+        Camera.Size previewSize = camera.getParameters().getPreviewSize();
+        previewHeight = previewSize.height;
+        previewWidth = previewSize.width;
+        rgbBytes = new int[previewWidth * previewHeight];
+        onPreviewSizeChosen(new Size(previewSize.width, previewSize.height), 90);
+      }
+    } catch (final Exception e) {
+      LOGGER.e(e, "Exception!");
+      return;
+    }
+
+    isProcessingFrame = true;
+    yuvBytes[0] = bytes;
+    yRowStride = previewWidth;
+
+    imageConverter =
+        new Runnable() {
+          @Override
+          public void run() {
+            ImageUtils.convertYUV420SPToARGB8888(bytes, previewWidth, previewHeight, rgbBytes);
+          }
+        };
+
+    postInferenceCallback =
+        new Runnable() {
+          @Override
+          public void run() {
+            camera.addCallbackBuffer(bytes);
+            isProcessingFrame = false;
+          }
+        };
+    processImage();
+  }
+
+  /**
+   * Callback for Camera2 API
+   */
+  @Override
+  public void onImageAvailable(final ImageReader reader) {
+    //We need wait until we have some size from onPreviewSizeChosen
+    if (previewWidth == 0 || previewHeight == 0) {
+      return;
+    }
+    if (rgbBytes == null) {
+      rgbBytes = new int[previewWidth * previewHeight];
+    }
+    try {
+      final Image image = reader.acquireLatestImage();
+
+      if (image == null) {
+        return;
+      }
+
+      if (isProcessingFrame) {
+        image.close();
+        return;
+      }
+      isProcessingFrame = true;
+      Trace.beginSection("imageAvailable");
+      final Plane[] planes = image.getPlanes();
+      fillBytes(planes, yuvBytes);
+      yRowStride = planes[0].getRowStride();
+      final int uvRowStride = planes[1].getRowStride();
+      final int uvPixelStride = planes[1].getPixelStride();
+
+      imageConverter =
+          new Runnable() {
+            @Override
+            public void run() {
+              ImageUtils.convertYUV420ToARGB8888(
+                  yuvBytes[0],
+                  yuvBytes[1],
+                  yuvBytes[2],
+                  previewWidth,
+                  previewHeight,
+                  yRowStride,
+                  uvRowStride,
+                  uvPixelStride,
+                  rgbBytes);
+            }
+          };
+
+      postInferenceCallback =
+          new Runnable() {
+            @Override
+            public void run() {
+              image.close();
+              isProcessingFrame = false;
+            }
+          };
+
+      processImage();
+    } catch (final Exception e) {
+      LOGGER.e(e, "Exception!");
+      Trace.endSection();
+      return;
+    }
+    Trace.endSection();
+  }
+
+  @Override
+  public synchronized void onStart() {
+    LOGGER.d("onStart " + this);
+    super.onStart();
+  }
+
+  @Override
+  public synchronized void onResume() {
+    LOGGER.d("onResume " + this);
+    super.onResume();
+
+    handlerThread = new HandlerThread("inference");
+    handlerThread.start();
+    handler = new Handler(handlerThread.getLooper());
+  }
+
+  @Override
+  public synchronized void onPause() {
+    LOGGER.d("onPause " + this);
+
+    if (!isFinishing()) {
+      LOGGER.d("Requesting finish");
+      finish();
+    }
+
+    handlerThread.quitSafely();
+    try {
+      handlerThread.join();
+      handlerThread = null;
+      handler = null;
+    } catch (final InterruptedException e) {
+      LOGGER.e(e, "Exception!");
+    }
+
+    super.onPause();
+  }
+
+  @Override
+  public synchronized void onStop() {
+    LOGGER.d("onStop " + this);
+    super.onStop();
+  }
+
+  @Override
+  public synchronized void onDestroy() {
+    LOGGER.d("onDestroy " + this);
+    super.onDestroy();
+  }
+
+  protected synchronized void runInBackground(final Runnable r) {
+    if (handler != null) {
+      handler.post(r);
+    }
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      final int requestCode, final String[] permissions, final int[] grantResults) {
+    if (requestCode == PERMISSIONS_REQUEST) {
+      if (grantResults.length > 0
+          && grantResults[0] == PackageManager.PERMISSION_GRANTED
+          && grantResults[1] == PackageManager.PERMISSION_GRANTED) {
+        setFragment();
+      } else {
+        requestPermission();
+      }
+    }
+  }
+
+  private boolean hasPermission() {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+      return checkSelfPermission(PERMISSION_CAMERA) == PackageManager.PERMISSION_GRANTED &&
+          checkSelfPermission(PERMISSION_STORAGE) == PackageManager.PERMISSION_GRANTED;
+    } else {
+      return true;
+    }
+  }
+
+  private void requestPermission() {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+      if (shouldShowRequestPermissionRationale(PERMISSION_CAMERA) ||
+          shouldShowRequestPermissionRationale(PERMISSION_STORAGE)) {
+        Toast.makeText(CameraActivity.this,
+            "Camera AND storage permission are required for this demo", Toast.LENGTH_LONG).show();
+      }
+      requestPermissions(new String[] {PERMISSION_CAMERA, PERMISSION_STORAGE}, PERMISSIONS_REQUEST);
+    }
+  }
+
+  // Returns true if the device supports the required hardware level, or better.
+  private boolean isHardwareLevelSupported(
+      CameraCharacteristics characteristics, int requiredLevel) {
+    int deviceLevel = characteristics.get(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL);
+    if (deviceLevel == CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY) {
+      return requiredLevel == deviceLevel;
+    }
+    // deviceLevel is not LEGACY, can use numerical sort
+    return requiredLevel <= deviceLevel;
+  }
+
+  private String chooseCamera() {
+    final CameraManager manager = (CameraManager) getSystemService(Context.CAMERA_SERVICE);
+    try {
+      for (final String cameraId : manager.getCameraIdList()) {
+        final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
+
+        // We don't use a front facing camera in this sample.
+        final Integer facing = characteristics.get(CameraCharacteristics.LENS_FACING);
+        if (facing != null && facing == CameraCharacteristics.LENS_FACING_FRONT) {
+          continue;
+        }
+
+        final StreamConfigurationMap map =
+            characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+
+        if (map == null) {
+          continue;
+        }
+
+        // Fallback to camera1 API for internal cameras that don't have full support.
+        // This should help with legacy situations where using the camera2 API causes
+        // distorted or otherwise broken previews.
+        useCamera2API = (facing == CameraCharacteristics.LENS_FACING_EXTERNAL)
+            || isHardwareLevelSupported(characteristics, 
+                                        CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_FULL);
+        LOGGER.i("Camera API lv2?: %s", useCamera2API);
+        return cameraId;
+      }
+    } catch (CameraAccessException e) {
+      LOGGER.e(e, "Not allowed to access camera");
+    }
+
+    return null;
+  }
+
+  protected void setFragment() {
+    String cameraId = chooseCamera();
+
+    Fragment fragment;
+    if (useCamera2API) {
+      CameraConnectionFragment camera2Fragment =
+          CameraConnectionFragment.newInstance(
+              new CameraConnectionFragment.ConnectionCallback() {
+                @Override
+                public void onPreviewSizeChosen(final Size size, final int rotation) {
+                  previewHeight = size.getHeight();
+                  previewWidth = size.getWidth();
+                  CameraActivity.this.onPreviewSizeChosen(size, rotation);
+                }
+              },
+              this,
+              getLayoutId(),
+              getDesiredPreviewFrameSize());
+
+      camera2Fragment.setCamera(cameraId);
+      fragment = camera2Fragment;
+    } else {
+      fragment =
+          new LegacyCameraConnectionFragment(this, getLayoutId(), getDesiredPreviewFrameSize());
+    }
+
+    getFragmentManager()
+        .beginTransaction()
+        .replace(R.id.container, fragment)
+        .commit();
+  }
+
+  protected void fillBytes(final Plane[] planes, final byte[][] yuvBytes) {
+    // Because of the variable row stride it's not possible to know in
+    // advance the actual necessary dimensions of the yuv planes.
+    for (int i = 0; i < planes.length; ++i) {
+      final ByteBuffer buffer = planes[i].getBuffer();
+      if (yuvBytes[i] == null) {
+        LOGGER.d("Initializing buffer %d at size %d", i, buffer.capacity());
+        yuvBytes[i] = new byte[buffer.capacity()];
+      }
+      buffer.get(yuvBytes[i]);
+    }
+  }
+
+  public boolean isDebug() {
+    return debug;
+  }
+
+  public void requestRender() {
+    final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay);
+    if (overlay != null) {
+      overlay.postInvalidate();
+    }
+  }
+
+  public void addCallback(final OverlayView.DrawCallback callback) {
+    final OverlayView overlay = (OverlayView) findViewById(R.id.debug_overlay);
+    if (overlay != null) {
+      overlay.addCallback(callback);
+    }
+  }
+
+  public void onSetDebug(final boolean debug) {}
+
+  @Override
+  public boolean onKeyDown(final int keyCode, final KeyEvent event) {
+    if (keyCode == KeyEvent.KEYCODE_VOLUME_DOWN || keyCode == KeyEvent.KEYCODE_VOLUME_UP) {
+      debug = !debug;
+      requestRender();
+      onSetDebug(debug);
+      return true;
+    }
+    return super.onKeyDown(keyCode, event);
+  }
+
+  protected void readyForNextImage() {
+    if (postInferenceCallback != null) {
+      postInferenceCallback.run();
+    }
+  }
+
+  protected int getScreenOrientation() {
+    switch (getWindowManager().getDefaultDisplay().getRotation()) {
+      case Surface.ROTATION_270:
+        return 270;
+      case Surface.ROTATION_180:
+        return 180;
+      case Surface.ROTATION_90:
+        return 90;
+      default:
+        return 0;
+    }
+  }
+
+  protected abstract void processImage();
+
+  protected abstract void onPreviewSizeChosen(final Size size, final int rotation);
+  protected abstract int getLayoutId();
+  protected abstract Size getDesiredPreviewFrameSize();
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java
new file mode 100644
index 0000000000..51a1adb538
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/CameraConnectionFragment.java
@@ -0,0 +1,634 @@
+/*
+ * Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.app.Activity;
+import android.app.AlertDialog;
+import android.app.Dialog;
+import android.app.DialogFragment;
+import android.app.Fragment;
+import android.content.Context;
+import android.content.DialogInterface;
+import android.content.res.Configuration;
+import android.graphics.ImageFormat;
+import android.graphics.Matrix;
+import android.graphics.RectF;
+import android.graphics.SurfaceTexture;
+import android.hardware.camera2.CameraAccessException;
+import android.hardware.camera2.CameraCaptureSession;
+import android.hardware.camera2.CameraCharacteristics;
+import android.hardware.camera2.CameraDevice;
+import android.hardware.camera2.CameraManager;
+import android.hardware.camera2.CaptureRequest;
+import android.hardware.camera2.CaptureResult;
+import android.hardware.camera2.TotalCaptureResult;
+import android.hardware.camera2.params.StreamConfigurationMap;
+import android.media.ImageReader;
+import android.media.ImageReader.OnImageAvailableListener;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.text.TextUtils;
+import android.util.Size;
+import android.util.SparseIntArray;
+import android.view.LayoutInflater;
+import android.view.Surface;
+import android.view.TextureView;
+import android.view.View;
+import android.view.ViewGroup;
+import android.widget.Toast;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.List;
+import java.util.concurrent.Semaphore;
+import java.util.concurrent.TimeUnit;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+public class CameraConnectionFragment extends Fragment {
+  private static final Logger LOGGER = new Logger();
+
+  /**
+   * The camera preview size will be chosen to be the smallest frame by pixel size capable of
+   * containing a DESIRED_SIZE x DESIRED_SIZE square.
+   */
+  private static final int MINIMUM_PREVIEW_SIZE = 320;
+
+  /**
+   * Conversion from screen rotation to JPEG orientation.
+   */
+  private static final SparseIntArray ORIENTATIONS = new SparseIntArray();
+  private static final String FRAGMENT_DIALOG = "dialog";
+
+  static {
+    ORIENTATIONS.append(Surface.ROTATION_0, 90);
+    ORIENTATIONS.append(Surface.ROTATION_90, 0);
+    ORIENTATIONS.append(Surface.ROTATION_180, 270);
+    ORIENTATIONS.append(Surface.ROTATION_270, 180);
+  }
+
+  /**
+   * {@link android.view.TextureView.SurfaceTextureListener} handles several lifecycle events on a
+   * {@link TextureView}.
+   */
+  private final TextureView.SurfaceTextureListener surfaceTextureListener =
+      new TextureView.SurfaceTextureListener() {
+        @Override
+        public void onSurfaceTextureAvailable(
+            final SurfaceTexture texture, final int width, final int height) {
+          openCamera(width, height);
+        }
+
+        @Override
+        public void onSurfaceTextureSizeChanged(
+            final SurfaceTexture texture, final int width, final int height) {
+          configureTransform(width, height);
+        }
+
+        @Override
+        public boolean onSurfaceTextureDestroyed(final SurfaceTexture texture) {
+          return true;
+        }
+
+        @Override
+        public void onSurfaceTextureUpdated(final SurfaceTexture texture) {}
+      };
+
+  /**
+   * Callback for Activities to use to initialize their data once the
+   * selected preview size is known.
+   */
+  public interface ConnectionCallback {
+    void onPreviewSizeChosen(Size size, int cameraRotation);
+  }
+
+  /**
+   * ID of the current {@link CameraDevice}.
+   */
+  private String cameraId;
+
+  /**
+   * An {@link AutoFitTextureView} for camera preview.
+   */
+  private AutoFitTextureView textureView;
+
+  /**
+   * A {@link CameraCaptureSession } for camera preview.
+   */
+  private CameraCaptureSession captureSession;
+
+  /**
+   * A reference to the opened {@link CameraDevice}.
+   */
+  private CameraDevice cameraDevice;
+
+  /**
+   * The rotation in degrees of the camera sensor from the display.
+   */
+  private Integer sensorOrientation;
+
+  /**
+   * The {@link android.util.Size} of camera preview.
+   */
+  private Size previewSize;
+
+  /**
+   * {@link android.hardware.camera2.CameraDevice.StateCallback}
+   * is called when {@link CameraDevice} changes its state.
+   */
+  private final CameraDevice.StateCallback stateCallback =
+      new CameraDevice.StateCallback() {
+        @Override
+        public void onOpened(final CameraDevice cd) {
+          // This method is called when the camera is opened.  We start camera preview here.
+          cameraOpenCloseLock.release();
+          cameraDevice = cd;
+          createCameraPreviewSession();
+        }
+
+        @Override
+        public void onDisconnected(final CameraDevice cd) {
+          cameraOpenCloseLock.release();
+          cd.close();
+          cameraDevice = null;
+        }
+
+        @Override
+        public void onError(final CameraDevice cd, final int error) {
+          cameraOpenCloseLock.release();
+          cd.close();
+          cameraDevice = null;
+          final Activity activity = getActivity();
+          if (null != activity) {
+            activity.finish();
+          }
+        }
+      };
+
+  /**
+   * An additional thread for running tasks that shouldn't block the UI.
+   */
+  private HandlerThread backgroundThread;
+
+  /**
+   * A {@link Handler} for running tasks in the background.
+   */
+  private Handler backgroundHandler;
+
+  /**
+   * An {@link ImageReader} that handles preview frame capture.
+   */
+  private ImageReader previewReader;
+
+  /**
+   * {@link android.hardware.camera2.CaptureRequest.Builder} for the camera preview
+   */
+  private CaptureRequest.Builder previewRequestBuilder;
+
+  /**
+   * {@link CaptureRequest} generated by {@link #previewRequestBuilder}
+   */
+  private CaptureRequest previewRequest;
+
+  /**
+   * A {@link Semaphore} to prevent the app from exiting before closing the camera.
+   */
+  private final Semaphore cameraOpenCloseLock = new Semaphore(1);
+
+  /**
+   * A {@link OnImageAvailableListener} to receive frames as they are available.
+   */
+  private final OnImageAvailableListener imageListener;
+
+  /** The input size in pixels desired by TensorFlow (width and height of a square bitmap). */
+  private final Size inputSize;
+
+  /**
+   * The layout identifier to inflate for this Fragment.
+   */
+  private final int layout;
+
+
+  private final ConnectionCallback cameraConnectionCallback;
+
+  private CameraConnectionFragment(
+      final ConnectionCallback connectionCallback,
+      final OnImageAvailableListener imageListener,
+      final int layout,
+      final Size inputSize) {
+    this.cameraConnectionCallback = connectionCallback;
+    this.imageListener = imageListener;
+    this.layout = layout;
+    this.inputSize = inputSize;
+  }
+
+  /**
+   * Shows a {@link Toast} on the UI thread.
+   *
+   * @param text The message to show
+   */
+  private void showToast(final String text) {
+    final Activity activity = getActivity();
+    if (activity != null) {
+      activity.runOnUiThread(
+          new Runnable() {
+            @Override
+            public void run() {
+              Toast.makeText(activity, text, Toast.LENGTH_SHORT).show();
+            }
+          });
+    }
+  }
+
+  /**
+   * Given {@code choices} of {@code Size}s supported by a camera, chooses the smallest one whose
+   * width and height are at least as large as the minimum of both, or an exact match if possible.
+   *
+   * @param choices The list of sizes that the camera supports for the intended output class
+   * @param width The minimum desired width
+   * @param height The minimum desired height
+   * @return The optimal {@code Size}, or an arbitrary one if none were big enough
+   */
+  protected static Size chooseOptimalSize(final Size[] choices, final int width, final int height) {
+    final int minSize = Math.max(Math.min(width, height), MINIMUM_PREVIEW_SIZE);
+    final Size desiredSize = new Size(width, height);
+
+    // Collect the supported resolutions that are at least as big as the preview Surface
+    boolean exactSizeFound = false;
+    final List<Size> bigEnough = new ArrayList<Size>();
+    final List<Size> tooSmall = new ArrayList<Size>();
+    for (final Size option : choices) {
+      if (option.equals(desiredSize)) {
+        // Set the size but don't return yet so that remaining sizes will still be logged.
+        exactSizeFound = true;
+      }
+
+      if (option.getHeight() >= minSize && option.getWidth() >= minSize) {
+        bigEnough.add(option);
+      } else {
+        tooSmall.add(option);
+      }
+    }
+
+    LOGGER.i("Desired size: " + desiredSize + ", min size: " + minSize + "x" + minSize);
+    LOGGER.i("Valid preview sizes: [" + TextUtils.join(", ", bigEnough) + "]");
+    LOGGER.i("Rejected preview sizes: [" + TextUtils.join(", ", tooSmall) + "]");
+
+    if (exactSizeFound) {
+      LOGGER.i("Exact size match found.");
+      return desiredSize;
+    }
+
+    // Pick the smallest of those, assuming we found any
+    if (bigEnough.size() > 0) {
+      final Size chosenSize = Collections.min(bigEnough, new CompareSizesByArea());
+      LOGGER.i("Chosen size: " + chosenSize.getWidth() + "x" + chosenSize.getHeight());
+      return chosenSize;
+    } else {
+      LOGGER.e("Couldn't find any suitable preview size");
+      return choices[0];
+    }
+  }
+
+  public static CameraConnectionFragment newInstance(
+      final ConnectionCallback callback,
+      final OnImageAvailableListener imageListener,
+      final int layout,
+      final Size inputSize) {
+    return new CameraConnectionFragment(callback, imageListener, layout, inputSize);
+  }
+
+  @Override
+  public View onCreateView(
+      final LayoutInflater inflater, final ViewGroup container, final Bundle savedInstanceState) {
+    return inflater.inflate(layout, container, false);
+  }
+
+  @Override
+  public void onViewCreated(final View view, final Bundle savedInstanceState) {
+    textureView = (AutoFitTextureView) view.findViewById(R.id.texture);
+  }
+
+  @Override
+  public void onActivityCreated(final Bundle savedInstanceState) {
+    super.onActivityCreated(savedInstanceState);
+  }
+
+  @Override
+  public void onResume() {
+    super.onResume();
+    startBackgroundThread();
+
+    // When the screen is turned off and turned back on, the SurfaceTexture is already
+    // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open
+    // a camera and start preview from here (otherwise, we wait until the surface is ready in
+    // the SurfaceTextureListener).
+    if (textureView.isAvailable()) {
+      openCamera(textureView.getWidth(), textureView.getHeight());
+    } else {
+      textureView.setSurfaceTextureListener(surfaceTextureListener);
+    }
+  }
+
+  @Override
+  public void onPause() {
+    closeCamera();
+    stopBackgroundThread();
+    super.onPause();
+  }
+
+  public void setCamera(String cameraId) {
+    this.cameraId = cameraId;
+  }
+
+  /**
+   * Sets up member variables related to camera.
+   */
+  private void setUpCameraOutputs() {
+    final Activity activity = getActivity();
+    final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
+    try {
+      final CameraCharacteristics characteristics = manager.getCameraCharacteristics(cameraId);
+
+      final StreamConfigurationMap map =
+          characteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP);
+
+      sensorOrientation = characteristics.get(CameraCharacteristics.SENSOR_ORIENTATION);
+
+      // Danger, W.R.! Attempting to use too large a preview size could  exceed the camera
+      // bus' bandwidth limitation, resulting in gorgeous previews but the storage of
+      // garbage capture data.
+      previewSize =
+          chooseOptimalSize(map.getOutputSizes(SurfaceTexture.class),
+              inputSize.getWidth(),
+              inputSize.getHeight());
+
+      // We fit the aspect ratio of TextureView to the size of preview we picked.
+      final int orientation = getResources().getConfiguration().orientation;
+      if (orientation == Configuration.ORIENTATION_LANDSCAPE) {
+        textureView.setAspectRatio(previewSize.getWidth(), previewSize.getHeight());
+      } else {
+        textureView.setAspectRatio(previewSize.getHeight(), previewSize.getWidth());
+      }
+    } catch (final CameraAccessException e) {
+      LOGGER.e(e, "Exception!");
+    } catch (final NullPointerException e) {
+      // Currently an NPE is thrown when the Camera2API is used but not supported on the
+      // device this code runs.
+      // TODO(andrewharp): abstract ErrorDialog/RuntimeException handling out into new method and
+      // reuse throughout app.
+      ErrorDialog.newInstance(getString(R.string.camera_error))
+          .show(getChildFragmentManager(), FRAGMENT_DIALOG);
+      throw new RuntimeException(getString(R.string.camera_error));
+    }
+
+    cameraConnectionCallback.onPreviewSizeChosen(previewSize, sensorOrientation);
+  }
+
+  /**
+   * Opens the camera specified by {@link CameraConnectionFragment#cameraId}.
+   */
+  private void openCamera(final int width, final int height) {
+    setUpCameraOutputs();
+    configureTransform(width, height);
+    final Activity activity = getActivity();
+    final CameraManager manager = (CameraManager) activity.getSystemService(Context.CAMERA_SERVICE);
+    try {
+      if (!cameraOpenCloseLock.tryAcquire(2500, TimeUnit.MILLISECONDS)) {
+        throw new RuntimeException("Time out waiting to lock camera opening.");
+      }
+      manager.openCamera(cameraId, stateCallback, backgroundHandler);
+    } catch (final CameraAccessException e) {
+      LOGGER.e(e, "Exception!");
+    } catch (final InterruptedException e) {
+      throw new RuntimeException("Interrupted while trying to lock camera opening.", e);
+    }
+  }
+
+  /**
+   * Closes the current {@link CameraDevice}.
+   */
+  private void closeCamera() {
+    try {
+      cameraOpenCloseLock.acquire();
+      if (null != captureSession) {
+        captureSession.close();
+        captureSession = null;
+      }
+      if (null != cameraDevice) {
+        cameraDevice.close();
+        cameraDevice = null;
+      }
+      if (null != previewReader) {
+        previewReader.close();
+        previewReader = null;
+      }
+    } catch (final InterruptedException e) {
+      throw new RuntimeException("Interrupted while trying to lock camera closing.", e);
+    } finally {
+      cameraOpenCloseLock.release();
+    }
+  }
+
+  /**
+   * Starts a background thread and its {@link Handler}.
+   */
+  private void startBackgroundThread() {
+    backgroundThread = new HandlerThread("ImageListener");
+    backgroundThread.start();
+    backgroundHandler = new Handler(backgroundThread.getLooper());
+  }
+
+  /**
+   * Stops the background thread and its {@link Handler}.
+   */
+  private void stopBackgroundThread() {
+    backgroundThread.quitSafely();
+    try {
+      backgroundThread.join();
+      backgroundThread = null;
+      backgroundHandler = null;
+    } catch (final InterruptedException e) {
+      LOGGER.e(e, "Exception!");
+    }
+  }
+
+  private final CameraCaptureSession.CaptureCallback captureCallback =
+      new CameraCaptureSession.CaptureCallback() {
+        @Override
+        public void onCaptureProgressed(
+            final CameraCaptureSession session,
+            final CaptureRequest request,
+            final CaptureResult partialResult) {}
+
+        @Override
+        public void onCaptureCompleted(
+            final CameraCaptureSession session,
+            final CaptureRequest request,
+            final TotalCaptureResult result) {}
+      };
+
+  /**
+   * Creates a new {@link CameraCaptureSession} for camera preview.
+   */
+  private void createCameraPreviewSession() {
+    try {
+      final SurfaceTexture texture = textureView.getSurfaceTexture();
+      assert texture != null;
+
+      // We configure the size of default buffer to be the size of camera preview we want.
+      texture.setDefaultBufferSize(previewSize.getWidth(), previewSize.getHeight());
+
+      // This is the output Surface we need to start preview.
+      final Surface surface = new Surface(texture);
+
+      // We set up a CaptureRequest.Builder with the output Surface.
+      previewRequestBuilder = cameraDevice.createCaptureRequest(CameraDevice.TEMPLATE_PREVIEW);
+      previewRequestBuilder.addTarget(surface);
+
+      LOGGER.i("Opening camera preview: " + previewSize.getWidth() + "x" + previewSize.getHeight());
+
+      // Create the reader for the preview frames.
+      previewReader =
+          ImageReader.newInstance(
+              previewSize.getWidth(), previewSize.getHeight(), ImageFormat.YUV_420_888, 2);
+
+      previewReader.setOnImageAvailableListener(imageListener, backgroundHandler);
+      previewRequestBuilder.addTarget(previewReader.getSurface());
+
+      // Here, we create a CameraCaptureSession for camera preview.
+      cameraDevice.createCaptureSession(
+          Arrays.asList(surface, previewReader.getSurface()),
+          new CameraCaptureSession.StateCallback() {
+
+            @Override
+            public void onConfigured(final CameraCaptureSession cameraCaptureSession) {
+              // The camera is already closed
+              if (null == cameraDevice) {
+                return;
+              }
+
+              // When the session is ready, we start displaying the preview.
+              captureSession = cameraCaptureSession;
+              try {
+                // Auto focus should be continuous for camera preview.
+                previewRequestBuilder.set(
+                    CaptureRequest.CONTROL_AF_MODE,
+                    CaptureRequest.CONTROL_AF_MODE_CONTINUOUS_PICTURE);
+                // Flash is automatically enabled when necessary.
+                previewRequestBuilder.set(
+                    CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON_AUTO_FLASH);
+
+                // Finally, we start displaying the camera preview.
+                previewRequest = previewRequestBuilder.build();
+                captureSession.setRepeatingRequest(
+                    previewRequest, captureCallback, backgroundHandler);
+              } catch (final CameraAccessException e) {
+                LOGGER.e(e, "Exception!");
+              }
+            }
+
+            @Override
+            public void onConfigureFailed(final CameraCaptureSession cameraCaptureSession) {
+              showToast("Failed");
+            }
+          },
+          null);
+    } catch (final CameraAccessException e) {
+      LOGGER.e(e, "Exception!");
+    }
+  }
+
+  /**
+   * Configures the necessary {@link android.graphics.Matrix} transformation to `mTextureView`.
+   * This method should be called after the camera preview size is determined in
+   * setUpCameraOutputs and also the size of `mTextureView` is fixed.
+   *
+   * @param viewWidth  The width of `mTextureView`
+   * @param viewHeight The height of `mTextureView`
+   */
+  private void configureTransform(final int viewWidth, final int viewHeight) {
+    final Activity activity = getActivity();
+    if (null == textureView || null == previewSize || null == activity) {
+      return;
+    }
+    final int rotation = activity.getWindowManager().getDefaultDisplay().getRotation();
+    final Matrix matrix = new Matrix();
+    final RectF viewRect = new RectF(0, 0, viewWidth, viewHeight);
+    final RectF bufferRect = new RectF(0, 0, previewSize.getHeight(), previewSize.getWidth());
+    final float centerX = viewRect.centerX();
+    final float centerY = viewRect.centerY();
+    if (Surface.ROTATION_90 == rotation || Surface.ROTATION_270 == rotation) {
+      bufferRect.offset(centerX - bufferRect.centerX(), centerY - bufferRect.centerY());
+      matrix.setRectToRect(viewRect, bufferRect, Matrix.ScaleToFit.FILL);
+      final float scale =
+          Math.max(
+              (float) viewHeight / previewSize.getHeight(),
+              (float) viewWidth / previewSize.getWidth());
+      matrix.postScale(scale, scale, centerX, centerY);
+      matrix.postRotate(90 * (rotation - 2), centerX, centerY);
+    } else if (Surface.ROTATION_180 == rotation) {
+      matrix.postRotate(180, centerX, centerY);
+    }
+    textureView.setTransform(matrix);
+  }
+
+  /**
+   * Compares two {@code Size}s based on their areas.
+   */
+  static class CompareSizesByArea implements Comparator<Size> {
+    @Override
+    public int compare(final Size lhs, final Size rhs) {
+      // We cast here to ensure the multiplications won't overflow
+      return Long.signum(
+          (long) lhs.getWidth() * lhs.getHeight() - (long) rhs.getWidth() * rhs.getHeight());
+    }
+  }
+
+  /**
+   * Shows an error message dialog.
+   */
+  public static class ErrorDialog extends DialogFragment {
+    private static final String ARG_MESSAGE = "message";
+
+    public static ErrorDialog newInstance(final String message) {
+      final ErrorDialog dialog = new ErrorDialog();
+      final Bundle args = new Bundle();
+      args.putString(ARG_MESSAGE, message);
+      dialog.setArguments(args);
+      return dialog;
+    }
+
+    @Override
+    public Dialog onCreateDialog(final Bundle savedInstanceState) {
+      final Activity activity = getActivity();
+      return new AlertDialog.Builder(activity)
+          .setMessage(getArguments().getString(ARG_MESSAGE))
+          .setPositiveButton(
+              android.R.string.ok,
+              new DialogInterface.OnClickListener() {
+                @Override
+                public void onClick(final DialogInterface dialogInterface, final int i) {
+                  activity.finish();
+                }
+              })
+          .create();
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/Classifier.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/Classifier.java
new file mode 100644
index 0000000000..07995febaf
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/Classifier.java
@@ -0,0 +1,107 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import android.graphics.Bitmap;
+import android.graphics.RectF;
+import java.util.List;
+
+/**
+ * Generic interface for interacting with different recognition engines.
+ */
+public interface Classifier {
+  /**
+   * An immutable result returned by a Classifier describing what was recognized.
+   */
+  public class Recognition {
+    /**
+     * A unique identifier for what has been recognized. Specific to the class, not the instance of
+     * the object.
+     */
+    private final String id;
+
+    /**
+     * Display name for the recognition.
+     */
+    private final String title;
+
+    /**
+     * A sortable score for how good the recognition is relative to others. Higher should be better.
+     */
+    private final Float confidence;
+
+    /** Optional location within the source image for the location of the recognized object. */
+    private RectF location;
+
+    public Recognition(
+        final String id, final String title, final Float confidence, final RectF location) {
+      this.id = id;
+      this.title = title;
+      this.confidence = confidence;
+      this.location = location;
+    }
+
+    public String getId() {
+      return id;
+    }
+
+    public String getTitle() {
+      return title;
+    }
+
+    public Float getConfidence() {
+      return confidence;
+    }
+
+    public RectF getLocation() {
+      return new RectF(location);
+    }
+
+    public void setLocation(RectF location) {
+      this.location = location;
+    }
+
+    @Override
+    public String toString() {
+      String resultString = "";
+      if (id != null) {
+        resultString += "[" + id + "] ";
+      }
+
+      if (title != null) {
+        resultString += title + " ";
+      }
+
+      if (confidence != null) {
+        resultString += String.format("(%.1f%%) ", confidence * 100.0f);
+      }
+
+      if (location != null) {
+        resultString += location + " ";
+      }
+
+      return resultString.trim();
+    }
+  }
+
+  List<Recognition> recognizeImage(Bitmap bitmap);
+
+  void enableStatLogging(final boolean debug);
+
+  String getStatString();
+
+  void close();
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ClassifierActivity.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ClassifierActivity.java
new file mode 100644
index 0000000000..dcbbefbeab
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ClassifierActivity.java
@@ -0,0 +1,197 @@
+/*
+ * Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.graphics.Bitmap;
+import android.graphics.Bitmap.Config;
+import android.graphics.Canvas;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import android.graphics.Typeface;
+import android.media.ImageReader.OnImageAvailableListener;
+import android.os.SystemClock;
+import android.util.Size;
+import android.util.TypedValue;
+import java.util.List;
+import java.util.Vector;
+import org.tensorflow.demo.OverlayView.DrawCallback;
+import org.tensorflow.demo.env.BorderedText;
+import org.tensorflow.demo.env.ImageUtils;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+public class ClassifierActivity extends CameraActivity implements OnImageAvailableListener {
+  private static final Logger LOGGER = new Logger();
+
+  protected static final boolean SAVE_PREVIEW_BITMAP = false;
+
+  private ResultsView resultsView;
+
+  private Bitmap rgbFrameBitmap = null;
+  private Bitmap croppedBitmap = null;
+  private Bitmap cropCopyBitmap = null;
+
+  private long lastProcessingTimeMs;
+
+  // These are the settings for the original v1 Inception model. If you want to
+  // use a model that's been produced from the TensorFlow for Poets codelab,
+  // you'll need to set IMAGE_SIZE = 299, IMAGE_MEAN = 128, IMAGE_STD = 128,
+  // INPUT_NAME = "Mul", and OUTPUT_NAME = "final_result".
+  // You'll also need to update the MODEL_FILE and LABEL_FILE paths to point to
+  // the ones you produced.
+  //
+  // To use v3 Inception model, strip the DecodeJpeg Op from your retrained
+  // model first:
+  //
+  // python strip_unused.py \
+  // --input_graph=<retrained-pb-file> \
+  // --output_graph=<your-stripped-pb-file> \
+  // --input_node_names="Mul" \
+  // --output_node_names="final_result" \
+  // --input_binary=true
+  private static final int INPUT_SIZE = 224;
+
+  private static final String MODEL_FILE = "mobilenet_quant_v1_224.tflite";
+  private static final String LABEL_FILE = "labels_mobilenet_quant_v1_224.txt";
+
+  private static final boolean MAINTAIN_ASPECT = true;
+
+  private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480);
+
+
+  private Integer sensorOrientation;
+  private Classifier classifier;
+  private Matrix frameToCropTransform;
+  private Matrix cropToFrameTransform;
+
+  private BorderedText borderedText;
+
+  @Override
+  protected int getLayoutId() {
+    return R.layout.camera_connection_fragment;
+  }
+
+  @Override
+  protected Size getDesiredPreviewFrameSize() {
+    return DESIRED_PREVIEW_SIZE;
+  }
+
+  private static final float TEXT_SIZE_DIP = 10;
+
+  @Override
+  public void onPreviewSizeChosen(final Size size, final int rotation) {
+    final float textSizePx = TypedValue.applyDimension(
+        TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics());
+    borderedText = new BorderedText(textSizePx);
+    borderedText.setTypeface(Typeface.MONOSPACE);
+
+    classifier = TFLiteImageClassifier.create(getAssets(), MODEL_FILE, LABEL_FILE, INPUT_SIZE);
+
+    previewWidth = size.getWidth();
+    previewHeight = size.getHeight();
+
+    sensorOrientation = rotation - getScreenOrientation();
+    LOGGER.i("Camera orientation relative to screen canvas: %d", sensorOrientation);
+
+    LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight);
+    rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888);
+    croppedBitmap = Bitmap.createBitmap(INPUT_SIZE, INPUT_SIZE, Config.ARGB_8888);
+
+    frameToCropTransform = ImageUtils.getTransformationMatrix(
+        previewWidth, previewHeight,
+        INPUT_SIZE, INPUT_SIZE,
+        sensorOrientation, MAINTAIN_ASPECT);
+
+    cropToFrameTransform = new Matrix();
+    frameToCropTransform.invert(cropToFrameTransform);
+
+    addCallback(
+        new DrawCallback() {
+          @Override
+          public void drawCallback(final Canvas canvas) {
+            renderDebug(canvas);
+          }
+        });
+  }
+
+  @Override
+  protected void processImage() {
+    rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight);
+    final Canvas canvas = new Canvas(croppedBitmap);
+    canvas.drawBitmap(rgbFrameBitmap, frameToCropTransform, null);
+
+    // For examining the actual TF input.
+    if (SAVE_PREVIEW_BITMAP) {
+      ImageUtils.saveBitmap(croppedBitmap);
+    }
+    runInBackground(
+        new Runnable() {
+          @Override
+          public void run() {
+            final long startTime = SystemClock.uptimeMillis();
+            final List<Classifier.Recognition> results = classifier.recognizeImage(croppedBitmap);
+            lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime;
+            LOGGER.i("Detect: %s", results);
+            cropCopyBitmap = Bitmap.createBitmap(croppedBitmap);
+            if (resultsView == null) {
+              resultsView = (ResultsView) findViewById(R.id.results);
+            }
+            resultsView.setResults(results);
+            requestRender();
+            readyForNextImage();
+          }
+        });
+  }
+
+  @Override
+  public void onSetDebug(boolean debug) {
+    classifier.enableStatLogging(debug);
+  }
+
+  private void renderDebug(final Canvas canvas) {
+    if (!isDebug()) {
+      return;
+    }
+    final Bitmap copy = cropCopyBitmap;
+    if (copy != null) {
+      final Matrix matrix = new Matrix();
+      final float scaleFactor = 2;
+      matrix.postScale(scaleFactor, scaleFactor);
+      matrix.postTranslate(
+          canvas.getWidth() - copy.getWidth() * scaleFactor,
+          canvas.getHeight() - copy.getHeight() * scaleFactor);
+      canvas.drawBitmap(copy, matrix, new Paint());
+
+      final Vector<String> lines = new Vector<String>();
+      if (classifier != null) {
+        String statString = classifier.getStatString();
+        String[] statLines = statString.split("\n");
+        for (String line : statLines) {
+          lines.add(line);
+        }
+      }
+
+      lines.add("Frame: " + previewWidth + "x" + previewHeight);
+      lines.add("Crop: " + copy.getWidth() + "x" + copy.getHeight());
+      lines.add("View: " + canvas.getWidth() + "x" + canvas.getHeight());
+      lines.add("Rotation: " + sensorOrientation);
+      lines.add("Inference time: " + lastProcessingTimeMs + "ms");
+
+      borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines);
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/DetectorActivity.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/DetectorActivity.java
new file mode 100644
index 0000000000..de997e454a
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/DetectorActivity.java
@@ -0,0 +1,296 @@
+/*
+ * Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.graphics.Bitmap;
+import android.graphics.Bitmap.Config;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import android.graphics.Paint.Style;
+import android.graphics.RectF;
+import android.graphics.Typeface;
+import android.media.ImageReader.OnImageAvailableListener;
+import android.os.SystemClock;
+import android.util.Size;
+import android.util.TypedValue;
+import android.widget.Toast;
+import java.io.IOException;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Vector;
+import org.tensorflow.demo.OverlayView.DrawCallback;
+import org.tensorflow.demo.env.BorderedText;
+import org.tensorflow.demo.env.ImageUtils;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.demo.tracking.MultiBoxTracker;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+/**
+ * An activity that uses a TensorFlowMultiBoxDetector and ObjectTracker to detect and then track
+ * objects.
+ */
+public class DetectorActivity extends CameraActivity implements OnImageAvailableListener {
+  private static final Logger LOGGER = new Logger();
+
+  // Configuration values for the prepackaged SSD model.
+  private static final int TF_OD_API_INPUT_SIZE = 300;
+  private static final String TF_OD_API_MODEL_FILE = "mobilenet_ssd.tflite";
+  private static final String TF_OD_API_LABELS_FILE = "file:///android_asset/coco_labels_list.txt";
+
+  // Which detection model to use: by default uses Tensorflow Object Detection API frozen
+  // checkpoints.
+  private enum DetectorMode {
+    TF_OD_API;
+  }
+
+  private static final DetectorMode MODE = DetectorMode.TF_OD_API;
+
+  // Minimum detection confidence to track a detection.
+  private static final float MINIMUM_CONFIDENCE_TF_OD_API = 0.6f;
+
+  private static final boolean MAINTAIN_ASPECT = false;
+
+  private static final Size DESIRED_PREVIEW_SIZE = new Size(640, 480);
+
+  private static final boolean SAVE_PREVIEW_BITMAP = false;
+  private static final float TEXT_SIZE_DIP = 10;
+
+  private Integer sensorOrientation;
+
+  private Classifier detector;
+
+  private long lastProcessingTimeMs;
+  private Bitmap rgbFrameBitmap = null;
+  private Bitmap croppedBitmap = null;
+  private Bitmap cropCopyBitmap = null;
+
+  private boolean computingDetection = false;
+
+  private long timestamp = 0;
+
+  private Matrix frameToCropTransform;
+  private Matrix cropToFrameTransform;
+
+  private MultiBoxTracker tracker;
+
+  private byte[] luminanceCopy;
+
+  private BorderedText borderedText;
+  @Override
+  public void onPreviewSizeChosen(final Size size, final int rotation) {
+    final float textSizePx =
+        TypedValue.applyDimension(
+            TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics());
+    borderedText = new BorderedText(textSizePx);
+    borderedText.setTypeface(Typeface.MONOSPACE);
+
+    tracker = new MultiBoxTracker(this);
+
+    int cropSize = TF_OD_API_INPUT_SIZE;
+
+    try {
+      detector =
+          TFLiteObjectDetectionAPIModel.create(
+              getAssets(), TF_OD_API_MODEL_FILE, TF_OD_API_LABELS_FILE, TF_OD_API_INPUT_SIZE);
+      cropSize = TF_OD_API_INPUT_SIZE;
+    } catch (final IOException e) {
+      LOGGER.e("Exception initializing classifier!", e);
+      Toast toast =
+          Toast.makeText(
+              getApplicationContext(), "Classifier could not be initialized", Toast.LENGTH_SHORT);
+      toast.show();
+      finish();
+    }
+
+
+    previewWidth = size.getWidth();
+    previewHeight = size.getHeight();
+
+    sensorOrientation = rotation - getScreenOrientation();
+    LOGGER.i("Camera orientation relative to screen canvas: %d", sensorOrientation);
+
+    LOGGER.i("Initializing at size %dx%d", previewWidth, previewHeight);
+    rgbFrameBitmap = Bitmap.createBitmap(previewWidth, previewHeight, Config.ARGB_8888);
+    croppedBitmap = Bitmap.createBitmap(cropSize, cropSize, Config.ARGB_8888);
+
+    frameToCropTransform =
+        ImageUtils.getTransformationMatrix(
+            previewWidth, previewHeight,
+            cropSize, cropSize,
+            sensorOrientation, MAINTAIN_ASPECT);
+
+    cropToFrameTransform = new Matrix();
+    frameToCropTransform.invert(cropToFrameTransform);
+
+    trackingOverlay = (OverlayView) findViewById(R.id.tracking_overlay);
+    trackingOverlay.addCallback(
+        new DrawCallback() {
+          @Override
+          public void drawCallback(final Canvas canvas) {
+            tracker.draw(canvas);
+            if (isDebug()) {
+              tracker.drawDebug(canvas);
+            }
+          }
+        });
+
+    addCallback(
+        new DrawCallback() {
+          @Override
+          public void drawCallback(final Canvas canvas) {
+            if (!isDebug()) {
+              return;
+            }
+            final Bitmap copy = cropCopyBitmap;
+            if (copy == null) {
+              return;
+            }
+
+            final int backgroundColor = Color.argb(100, 0, 0, 0);
+            canvas.drawColor(backgroundColor);
+
+            final Matrix matrix = new Matrix();
+            final float scaleFactor = 2;
+            matrix.postScale(scaleFactor, scaleFactor);
+            matrix.postTranslate(
+                canvas.getWidth() - copy.getWidth() * scaleFactor,
+                canvas.getHeight() - copy.getHeight() * scaleFactor);
+            canvas.drawBitmap(copy, matrix, new Paint());
+
+            final Vector<String> lines = new Vector<String>();
+            if (detector != null) {
+              final String statString = detector.getStatString();
+              final String[] statLines = statString.split("\n");
+              for (final String line : statLines) {
+                lines.add(line);
+              }
+            }
+            lines.add("");
+
+            lines.add("Frame: " + previewWidth + "x" + previewHeight);
+            lines.add("Crop: " + copy.getWidth() + "x" + copy.getHeight());
+            lines.add("View: " + canvas.getWidth() + "x" + canvas.getHeight());
+            lines.add("Rotation: " + sensorOrientation);
+            lines.add("Inference time: " + lastProcessingTimeMs + "ms");
+
+            borderedText.drawLines(canvas, 10, canvas.getHeight() - 10, lines);
+          }
+        });
+  }
+
+  OverlayView trackingOverlay;
+
+  @Override
+  protected void processImage() {
+    ++timestamp;
+    final long currTimestamp = timestamp;
+    byte[] originalLuminance = getLuminance();
+    tracker.onFrame(
+        previewWidth,
+        previewHeight,
+        getLuminanceStride(),
+        sensorOrientation,
+        originalLuminance,
+        timestamp);
+    trackingOverlay.postInvalidate();
+
+    // No mutex needed as this method is not reentrant.
+    if (computingDetection) {
+      readyForNextImage();
+      return;
+    }
+    computingDetection = true;
+    LOGGER.i("Preparing image " + currTimestamp + " for detection in bg thread.");
+
+    rgbFrameBitmap.setPixels(getRgbBytes(), 0, previewWidth, 0, 0, previewWidth, previewHeight);
+
+    if (luminanceCopy == null) {
+      luminanceCopy = new byte[originalLuminance.length];
+    }
+    System.arraycopy(originalLuminance, 0, luminanceCopy, 0, originalLuminance.length);
+    readyForNextImage();
+
+    final Canvas canvas = new Canvas(croppedBitmap);
+    canvas.drawBitmap(rgbFrameBitmap, frameToCropTransform, null);
+    // For examining the actual TF input.
+    if (SAVE_PREVIEW_BITMAP) {
+      ImageUtils.saveBitmap(croppedBitmap);
+    }
+
+    runInBackground(
+        new Runnable() {
+          @Override
+          public void run() {
+            LOGGER.i("Running detection on image " + currTimestamp);
+            final long startTime = SystemClock.uptimeMillis();
+            final List<Classifier.Recognition> results = detector.recognizeImage(croppedBitmap);
+            lastProcessingTimeMs = SystemClock.uptimeMillis() - startTime;
+
+            cropCopyBitmap = Bitmap.createBitmap(croppedBitmap);
+            final Canvas canvas = new Canvas(cropCopyBitmap);
+            final Paint paint = new Paint();
+            paint.setColor(Color.RED);
+            paint.setStyle(Style.STROKE);
+            paint.setStrokeWidth(2.0f);
+
+            float minimumConfidence = MINIMUM_CONFIDENCE_TF_OD_API;
+            switch (MODE) {
+              case TF_OD_API:
+                minimumConfidence = MINIMUM_CONFIDENCE_TF_OD_API;
+                break;
+            }
+
+            final List<Classifier.Recognition> mappedRecognitions =
+                new LinkedList<Classifier.Recognition>();
+
+            for (final Classifier.Recognition result : results) {
+              final RectF location = result.getLocation();
+              if (location != null && result.getConfidence() >= minimumConfidence) {
+                canvas.drawRect(location, paint);
+
+                cropToFrameTransform.mapRect(location);
+                result.setLocation(location);
+                mappedRecognitions.add(result);
+              }
+            }
+
+            tracker.trackResults(mappedRecognitions, luminanceCopy, currTimestamp);
+            trackingOverlay.postInvalidate();
+
+            requestRender();
+            computingDetection = false;
+          }
+        });
+  }
+
+  @Override
+  protected int getLayoutId() {
+    return R.layout.camera_connection_fragment_tracking;
+  }
+
+  @Override
+  protected Size getDesiredPreviewFrameSize() {
+    return DESIRED_PREVIEW_SIZE;
+  }
+
+  @Override
+  public void onSetDebug(final boolean debug) {
+    detector.enableStatLogging(debug);
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/LegacyCameraConnectionFragment.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/LegacyCameraConnectionFragment.java
new file mode 100644
index 0000000000..fd83029753
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/LegacyCameraConnectionFragment.java
@@ -0,0 +1,216 @@
+package org.tensorflow.demo;
+
+/*
+ * Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import android.app.Fragment;
+import android.graphics.SurfaceTexture;
+import android.hardware.Camera;
+import android.hardware.Camera.CameraInfo;
+import android.os.Bundle;
+import android.os.Handler;
+import android.os.HandlerThread;
+import android.util.Size;
+import android.util.SparseIntArray;
+import android.view.LayoutInflater;
+import android.view.Surface;
+import android.view.TextureView;
+import android.view.View;
+import android.view.ViewGroup;
+import java.io.IOException;
+import java.util.List;
+import org.tensorflow.demo.env.ImageUtils;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+public class LegacyCameraConnectionFragment extends Fragment {
+  private Camera camera;
+  private static final Logger LOGGER = new Logger();
+  private Camera.PreviewCallback imageListener;
+  private Size desiredSize;
+
+  /**
+   * The layout identifier to inflate for this Fragment.
+   */
+  private int layout;
+
+  public LegacyCameraConnectionFragment(
+      final Camera.PreviewCallback imageListener, final int layout, final Size desiredSize) {
+    this.imageListener = imageListener;
+    this.layout = layout;
+    this.desiredSize = desiredSize;
+  }
+
+  /**
+   * Conversion from screen rotation to JPEG orientation.
+   */
+  private static final SparseIntArray ORIENTATIONS = new SparseIntArray();
+
+  static {
+    ORIENTATIONS.append(Surface.ROTATION_0, 90);
+    ORIENTATIONS.append(Surface.ROTATION_90, 0);
+    ORIENTATIONS.append(Surface.ROTATION_180, 270);
+    ORIENTATIONS.append(Surface.ROTATION_270, 180);
+  }
+
+  /**
+   * {@link android.view.TextureView.SurfaceTextureListener} handles several lifecycle events on a
+   * {@link TextureView}.
+   */
+  private final TextureView.SurfaceTextureListener surfaceTextureListener =
+      new TextureView.SurfaceTextureListener() {
+        @Override
+        public void onSurfaceTextureAvailable(
+            final SurfaceTexture texture, final int width, final int height) {
+
+          int index = getCameraId();
+          camera = Camera.open(index);
+
+          try {
+            Camera.Parameters parameters = camera.getParameters();
+            List<String> focusModes = parameters.getSupportedFocusModes();
+            if (focusModes != null
+                && focusModes.contains(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE)) {
+              parameters.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_PICTURE);
+            }
+            List<Camera.Size> cameraSizes = parameters.getSupportedPreviewSizes();
+            Size[] sizes = new Size[cameraSizes.size()];
+            int i = 0;
+            for (Camera.Size size : cameraSizes) {
+              sizes[i++] = new Size(size.width, size.height);
+            }
+            Size previewSize =
+                CameraConnectionFragment.chooseOptimalSize(
+                    sizes, desiredSize.getWidth(), desiredSize.getHeight());
+            parameters.setPreviewSize(previewSize.getWidth(), previewSize.getHeight());
+            camera.setDisplayOrientation(90);
+            camera.setParameters(parameters);
+            camera.setPreviewTexture(texture);
+          } catch (IOException exception) {
+            camera.release();
+          }
+
+          camera.setPreviewCallbackWithBuffer(imageListener);
+          Camera.Size s = camera.getParameters().getPreviewSize();
+          camera.addCallbackBuffer(new byte[ImageUtils.getYUVByteSize(s.height, s.width)]);
+
+          textureView.setAspectRatio(s.height, s.width);
+
+          camera.startPreview();
+        }
+
+        @Override
+        public void onSurfaceTextureSizeChanged(
+            final SurfaceTexture texture, final int width, final int height) {}
+
+        @Override
+        public boolean onSurfaceTextureDestroyed(final SurfaceTexture texture) {
+          return true;
+        }
+
+        @Override
+        public void onSurfaceTextureUpdated(final SurfaceTexture texture) {}
+      };
+
+  /**
+   * An {@link AutoFitTextureView} for camera preview.
+   */
+  private AutoFitTextureView textureView;
+
+  /**
+   * An additional thread for running tasks that shouldn't block the UI.
+   */
+  private HandlerThread backgroundThread;
+
+  @Override
+  public View onCreateView(
+      final LayoutInflater inflater, final ViewGroup container, final Bundle savedInstanceState) {
+    return inflater.inflate(layout, container, false);
+  }
+
+  @Override
+  public void onViewCreated(final View view, final Bundle savedInstanceState) {
+    textureView = (AutoFitTextureView) view.findViewById(R.id.texture);
+  }
+
+  @Override
+  public void onActivityCreated(final Bundle savedInstanceState) {
+    super.onActivityCreated(savedInstanceState);
+  }
+
+  @Override
+  public void onResume() {
+    super.onResume();
+    startBackgroundThread();
+    // When the screen is turned off and turned back on, the SurfaceTexture is already
+    // available, and "onSurfaceTextureAvailable" will not be called. In that case, we can open
+    // a camera and start preview from here (otherwise, we wait until the surface is ready in
+    // the SurfaceTextureListener).
+
+    if (textureView.isAvailable()) {
+      camera.startPreview();
+    } else {
+      textureView.setSurfaceTextureListener(surfaceTextureListener);
+    }
+  }
+
+  @Override
+  public void onPause() {
+    stopCamera();
+    stopBackgroundThread();
+    super.onPause();
+  }
+
+  /**
+   * Starts a background thread and its {@link Handler}.
+   */
+  private void startBackgroundThread() {
+    backgroundThread = new HandlerThread("CameraBackground");
+    backgroundThread.start();
+  }
+
+  /**
+   * Stops the background thread and its {@link Handler}.
+   */
+  private void stopBackgroundThread() {
+    backgroundThread.quitSafely();
+    try {
+      backgroundThread.join();
+      backgroundThread = null;
+    } catch (final InterruptedException e) {
+      LOGGER.e(e, "Exception!");
+    }
+  }
+
+  protected void stopCamera() {
+    if (camera != null) {
+      camera.stopPreview();
+      camera.setPreviewCallback(null);
+      camera.release();
+      camera = null;
+    }
+  }
+
+  private int getCameraId() {
+    CameraInfo ci = new CameraInfo();
+    for (int i = 0; i < Camera.getNumberOfCameras(); i++) {
+      Camera.getCameraInfo(i, ci);
+      if (ci.facing == CameraInfo.CAMERA_FACING_BACK)
+        return i;
+    }
+    return -1; // No camera found
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/OverlayView.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/OverlayView.java
new file mode 100644
index 0000000000..0f8d109fb4
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/OverlayView.java
@@ -0,0 +1,52 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import android.content.Context;
+import android.graphics.Canvas;
+import android.util.AttributeSet;
+import android.view.View;
+import java.util.LinkedList;
+import java.util.List;
+
+/**
+ * A simple View providing a render callback to other classes.
+ */
+public class OverlayView extends View {
+  private final List<DrawCallback> callbacks = new LinkedList<DrawCallback>();
+
+  public OverlayView(final Context context, final AttributeSet attrs) {
+    super(context, attrs);
+  }
+
+  /**
+   * Interface defining the callback for client classes.
+   */
+  public interface DrawCallback {
+    public void drawCallback(final Canvas canvas);
+  }
+
+  public void addCallback(final DrawCallback callback) {
+    callbacks.add(callback);
+  }
+
+  @Override
+  public synchronized void draw(final Canvas canvas) {
+    for (final DrawCallback callback : callbacks) {
+      callback.drawCallback(canvas);
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognitionScoreView.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognitionScoreView.java
new file mode 100644
index 0000000000..31a4b07c83
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognitionScoreView.java
@@ -0,0 +1,67 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import android.content.Context;
+import android.graphics.Canvas;
+import android.graphics.Paint;
+import android.util.AttributeSet;
+import android.util.TypedValue;
+import android.view.View;
+import java.util.List;
+import org.tensorflow.demo.Classifier.Recognition;
+
+public class RecognitionScoreView extends View implements ResultsView {
+  private static final float TEXT_SIZE_DIP = 24;
+  private List<Recognition> results;
+  private final float textSizePx;
+  private final Paint fgPaint;
+  private final Paint bgPaint;
+
+  public RecognitionScoreView(final Context context, final AttributeSet set) {
+    super(context, set);
+
+    textSizePx =
+        TypedValue.applyDimension(
+            TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, getResources().getDisplayMetrics());
+    fgPaint = new Paint();
+    fgPaint.setTextSize(textSizePx);
+
+    bgPaint = new Paint();
+    bgPaint.setColor(0xcc4285f4);
+  }
+
+  @Override
+  public void setResults(final List<Recognition> results) {
+    this.results = results;
+    postInvalidate();
+  }
+
+  @Override
+  public void onDraw(final Canvas canvas) {
+    final int x = 10;
+    int y = (int) (fgPaint.getTextSize() * 1.5f);
+
+    canvas.drawPaint(bgPaint);
+
+    if (results != null) {
+      for (final Recognition recog : results) {
+        canvas.drawText(recog.getTitle() + ": " + recog.getConfidence(), x, y, fgPaint);
+        y += (int) (fgPaint.getTextSize() * 1.5f);
+      }
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognizeCommands.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognizeCommands.java
new file mode 100644
index 0000000000..9e91aea7ef
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/RecognizeCommands.java
@@ -0,0 +1,186 @@
+/*
+ * Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.tensorflow.demo;
+
+import android.util.Log;
+import android.util.Pair;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Deque;
+import java.util.List;
+
+/** Reads in results from an instantaneous audio recognition model and smoothes them over time. */
+public class RecognizeCommands {
+  // Configuration settings.
+  private List<String> labels = new ArrayList<String>();
+  private long averageWindowDurationMs;
+  private float detectionThreshold;
+  private int suppressionMs;
+  private int minimumCount;
+  private long minimumTimeBetweenSamplesMs;
+
+  // Working variables.
+  private Deque<Pair<Long, float[]>> previousResults = new ArrayDeque<Pair<Long, float[]>>();
+  private String previousTopLabel;
+  private int labelsCount;
+  private long previousTopLabelTime;
+  private float previousTopLabelScore;
+
+  private static final String SILENCE_LABEL = "_silence_";
+  private static final long MINIMUM_TIME_FRACTION = 4;
+
+  public RecognizeCommands(
+      List<String> inLabels,
+      long inAverageWindowDurationMs,
+      float inDetectionThreshold,
+      int inSuppressionMS,
+      int inMinimumCount,
+      long inMinimumTimeBetweenSamplesMS) {
+    labels = inLabels;
+    averageWindowDurationMs = inAverageWindowDurationMs;
+    detectionThreshold = inDetectionThreshold;
+    suppressionMs = inSuppressionMS;
+    minimumCount = inMinimumCount;
+    labelsCount = inLabels.size();
+    previousTopLabel = SILENCE_LABEL;
+    previousTopLabelTime = Long.MIN_VALUE;
+    previousTopLabelScore = 0.0f;
+    minimumTimeBetweenSamplesMs = inMinimumTimeBetweenSamplesMS;
+  }
+
+  /** Holds information about what's been recognized. */
+  public static class RecognitionResult {
+    public final String foundCommand;
+    public final float score;
+    public final boolean isNewCommand;
+
+    public RecognitionResult(String inFoundCommand, float inScore, boolean inIsNewCommand) {
+      foundCommand = inFoundCommand;
+      score = inScore;
+      isNewCommand = inIsNewCommand;
+    }
+  }
+
+  private static class ScoreForSorting implements Comparable<ScoreForSorting> {
+    public final float score;
+    public final int index;
+
+    public ScoreForSorting(float inScore, int inIndex) {
+      score = inScore;
+      index = inIndex;
+    }
+
+    @Override
+    public int compareTo(ScoreForSorting other) {
+      if (this.score > other.score) {
+        return -1;
+      } else if (this.score < other.score) {
+        return 1;
+      } else {
+        return 0;
+      }
+    }
+  }
+
+  public RecognitionResult processLatestResults(float[] currentResults, long currentTimeMS) {
+    if (currentResults.length != labelsCount) {
+      throw new RuntimeException(
+          "The results for recognition should contain "
+              + labelsCount
+              + " elements, but there are "
+              + currentResults.length);
+    }
+
+    if ((!previousResults.isEmpty()) && (currentTimeMS < previousResults.getFirst().first)) {
+      throw new RuntimeException(
+          "You must feed results in increasing time order, but received a timestamp of "
+              + currentTimeMS
+              + " that was earlier than the previous one of "
+              + previousResults.getFirst().first);
+    }
+
+    final int howManyResults = previousResults.size();
+    // Ignore any results that are coming in too frequently.
+    if (howManyResults > 1) {
+      final long timeSinceMostRecent = currentTimeMS - previousResults.getLast().first;
+      if (timeSinceMostRecent < minimumTimeBetweenSamplesMs) {
+        return new RecognitionResult(previousTopLabel, previousTopLabelScore, false);
+      }
+    }
+
+    // Add the latest results to the head of the queue.
+    previousResults.addLast(new Pair<Long, float[]>(currentTimeMS, currentResults));
+
+    // Prune any earlier results that are too old for the averaging window.
+    final long timeLimit = currentTimeMS - averageWindowDurationMs;
+    while (previousResults.getFirst().first < timeLimit) {
+      previousResults.removeFirst();
+    }
+
+    // If there are too few results, assume the result will be unreliable and
+    // bail.
+    final long earliestTime = previousResults.getFirst().first;
+    final long samplesDuration = currentTimeMS - earliestTime;
+    if ((howManyResults < minimumCount)
+        || (samplesDuration < (averageWindowDurationMs / MINIMUM_TIME_FRACTION))) {
+      Log.v("RecognizeResult", "Too few results");
+      return new RecognitionResult(previousTopLabel, 0.0f, false);
+    }
+
+    // Calculate the average score across all the results in the window.
+    float[] averageScores = new float[labelsCount];
+    for (Pair<Long, float[]> previousResult : previousResults) {
+      final float[] scoresTensor = previousResult.second;
+      int i = 0;
+      while (i < scoresTensor.length) {
+        averageScores[i] += scoresTensor[i] / howManyResults;
+        ++i;
+      }
+    }
+
+    // Sort the averaged results in descending score order.
+    ScoreForSorting[] sortedAverageScores = new ScoreForSorting[labelsCount];
+    for (int i = 0; i < labelsCount; ++i) {
+      sortedAverageScores[i] = new ScoreForSorting(averageScores[i], i);
+    }
+    Arrays.sort(sortedAverageScores);
+
+    // See if the latest top score is enough to trigger a detection.
+    final int currentTopIndex = sortedAverageScores[0].index;
+    final String currentTopLabel = labels.get(currentTopIndex);
+    final float currentTopScore = sortedAverageScores[0].score;
+    // If we've recently had another label trigger, assume one that occurs too
+    // soon afterwards is a bad result.
+    long timeSinceLastTop;
+    if (previousTopLabel.equals(SILENCE_LABEL) || (previousTopLabelTime == Long.MIN_VALUE)) {
+      timeSinceLastTop = Long.MAX_VALUE;
+    } else {
+      timeSinceLastTop = currentTimeMS - previousTopLabelTime;
+    }
+    boolean isNewCommand;
+    if ((currentTopScore > detectionThreshold) && (timeSinceLastTop > suppressionMs)) {
+      previousTopLabel = currentTopLabel;
+      previousTopLabelTime = currentTimeMS;
+      previousTopLabelScore = currentTopScore;
+      isNewCommand = true;
+    } else {
+      isNewCommand = false;
+    }
+    return new RecognitionResult(currentTopLabel, currentTopScore, isNewCommand);
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ResultsView.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ResultsView.java
new file mode 100644
index 0000000000..211d7e66fb
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/ResultsView.java
@@ -0,0 +1,23 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import java.util.List;
+import org.tensorflow.demo.Classifier.Recognition;
+
+public interface ResultsView {
+  public void setResults(final List<Recognition> results);
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/SpeechActivity.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/SpeechActivity.java
new file mode 100644
index 0000000000..9c9c30bc09
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/SpeechActivity.java
@@ -0,0 +1,381 @@
+/*
+ * Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *       http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* Demonstrates how to run an audio recognition model in Android.
+
+This example loads a simple speech recognition model trained by the tutorial at
+https://www.tensorflow.org/tutorials/audio_training
+
+The model files should be downloaded automatically from the TensorFlow website,
+but if you have a custom model you can update the LABEL_FILENAME and
+MODEL_FILENAME constants to point to your own files.
+
+The example application displays a list view with all of the known audio labels,
+and highlights each one when it thinks it has detected one through the
+microphone. The averaging of results to give a more reliable signal happens in
+the RecognizeCommands helper class.
+*/
+
+package org.tensorflow.demo;
+
+import android.animation.ValueAnimator;
+import android.app.Activity;
+import android.content.pm.PackageManager;
+import android.content.res.AssetFileDescriptor;
+import android.content.res.AssetManager;
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.MediaRecorder;
+import android.os.Build;
+import android.os.Bundle;
+import android.util.Log;
+import android.view.View;
+import android.widget.ArrayAdapter;
+import android.widget.Button;
+import android.widget.ListView;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.ReentrantLock;
+import org.tensorflow.lite.Interpreter;
+import org.tensorflow.lite.demo.R; // Explicit import needed for internal Google builds.
+
+/**
+ * An activity that listens for audio and then uses a TensorFlow model to detect particular classes,
+ * by default a small set of action words.
+ */
+public class SpeechActivity extends Activity {
+
+  // Constants that control the behavior of the recognition code and model
+  // settings. See the audio recognition tutorial for a detailed explanation of
+  // all these, but you should customize them to match your training settings if
+  // you are running your own model.
+  private static final int SAMPLE_RATE = 16000;
+  private static final int SAMPLE_DURATION_MS = 1000;
+  private static final int RECORDING_LENGTH = (int) (SAMPLE_RATE * SAMPLE_DURATION_MS / 1000);
+  private static final long AVERAGE_WINDOW_DURATION_MS = 500;
+  private static final float DETECTION_THRESHOLD = 0.70f;
+  private static final int SUPPRESSION_MS = 1500;
+  private static final int MINIMUM_COUNT = 3;
+  private static final long MINIMUM_TIME_BETWEEN_SAMPLES_MS = 30;
+  private static final String LABEL_FILENAME = "file:///android_asset/conv_actions_labels.txt";
+  private static final String MODEL_FILENAME = "file:///android_asset/conv_actions_frozen.tflite";
+
+  // UI elements.
+  private static final int REQUEST_RECORD_AUDIO = 13;
+  private Button quitButton;
+  private ListView labelsListView;
+  private static final String LOG_TAG = SpeechActivity.class.getSimpleName();
+
+  // Working variables.
+  short[] recordingBuffer = new short[RECORDING_LENGTH];
+  int recordingOffset = 0;
+  boolean shouldContinue = true;
+  private Thread recordingThread;
+  boolean shouldContinueRecognition = true;
+  private Thread recognitionThread;
+  private final ReentrantLock recordingBufferLock = new ReentrantLock();
+
+  private List<String> labels = new ArrayList<String>();
+  private List<String> displayedLabels = new ArrayList<>();
+  private RecognizeCommands recognizeCommands = null;
+
+  private Interpreter tfLite;
+
+  /** Memory-map the model file in Assets. */
+  private static MappedByteBuffer loadModelFile(AssetManager assets, String modelFilename)
+      throws IOException {
+    AssetFileDescriptor fileDescriptor = assets.openFd(modelFilename);
+    FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
+    FileChannel fileChannel = inputStream.getChannel();
+    long startOffset = fileDescriptor.getStartOffset();
+    long declaredLength = fileDescriptor.getDeclaredLength();
+    return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
+  }
+
+  @Override
+  protected void onCreate(Bundle savedInstanceState) {
+    // Set up the UI.
+    super.onCreate(savedInstanceState);
+    setContentView(R.layout.activity_speech);
+    quitButton = (Button) findViewById(R.id.quit);
+    quitButton.setOnClickListener(
+        new View.OnClickListener() {
+          @Override
+          public void onClick(View view) {
+            moveTaskToBack(true);
+            android.os.Process.killProcess(android.os.Process.myPid());
+            System.exit(1);
+          }
+        });
+    labelsListView = (ListView) findViewById(R.id.list_view);
+
+    // Load the labels for the model, but only display those that don't start
+    // with an underscore.
+    String actualLabelFilename = LABEL_FILENAME.split("file:///android_asset/", -1)[1];
+    Log.i(LOG_TAG, "Reading labels from: " + actualLabelFilename);
+    BufferedReader br = null;
+    try {
+      br = new BufferedReader(new InputStreamReader(getAssets().open(actualLabelFilename)));
+      String line;
+      while ((line = br.readLine()) != null) {
+        labels.add(line);
+        if (line.charAt(0) != '_') {
+          displayedLabels.add(line.substring(0, 1).toUpperCase() + line.substring(1));
+        }
+      }
+      br.close();
+    } catch (IOException e) {
+      throw new RuntimeException("Problem reading label file!", e);
+    }
+
+    // Build a list view based on these labels.
+    ArrayAdapter<String> arrayAdapter =
+        new ArrayAdapter<String>(this, R.layout.list_text_item, displayedLabels);
+    labelsListView.setAdapter(arrayAdapter);
+
+    // Set up an object to smooth recognition results to increase accuracy.
+    recognizeCommands =
+        new RecognizeCommands(
+            labels,
+            AVERAGE_WINDOW_DURATION_MS,
+            DETECTION_THRESHOLD,
+            SUPPRESSION_MS,
+            MINIMUM_COUNT,
+            MINIMUM_TIME_BETWEEN_SAMPLES_MS);
+
+    String actualModelFilename = MODEL_FILENAME.split("file:///android_asset/", -1)[1];
+    try {
+      tfLite = new Interpreter(loadModelFile(getAssets(), actualModelFilename));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+
+    tfLite.resizeInput(0, new int[] {RECORDING_LENGTH, 1});
+    tfLite.resizeInput(1, new int[] {1});
+
+    // Start the recording and recognition threads.
+    requestMicrophonePermission();
+    startRecording();
+    startRecognition();
+  }
+
+  private void requestMicrophonePermission() {
+    if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.M) {
+      requestPermissions(
+          new String[]{android.Manifest.permission.RECORD_AUDIO}, REQUEST_RECORD_AUDIO);
+    }
+  }
+
+  @Override
+  public void onRequestPermissionsResult(
+      int requestCode, String[] permissions, int[] grantResults) {
+    if (requestCode == REQUEST_RECORD_AUDIO
+        && grantResults.length > 0
+        && grantResults[0] == PackageManager.PERMISSION_GRANTED) {
+      startRecording();
+      startRecognition();
+    }
+  }
+
+  public synchronized void startRecording() {
+    if (recordingThread != null) {
+      return;
+    }
+    shouldContinue = true;
+    recordingThread =
+        new Thread(
+            new Runnable() {
+              @Override
+              public void run() {
+                record();
+              }
+            });
+    recordingThread.start();
+  }
+
+  public synchronized void stopRecording() {
+    if (recordingThread == null) {
+      return;
+    }
+    shouldContinue = false;
+    recordingThread = null;
+  }
+
+  private void record() {
+    android.os.Process.setThreadPriority(android.os.Process.THREAD_PRIORITY_AUDIO);
+
+    // Estimate the buffer size we'll need for this device.
+    int bufferSize =
+        AudioRecord.getMinBufferSize(
+            SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT);
+    if (bufferSize == AudioRecord.ERROR || bufferSize == AudioRecord.ERROR_BAD_VALUE) {
+      bufferSize = SAMPLE_RATE * 2;
+    }
+    short[] audioBuffer = new short[bufferSize / 2];
+
+    AudioRecord record =
+        new AudioRecord(
+            MediaRecorder.AudioSource.DEFAULT,
+            SAMPLE_RATE,
+            AudioFormat.CHANNEL_IN_MONO,
+            AudioFormat.ENCODING_PCM_16BIT,
+            bufferSize);
+
+    if (record.getState() != AudioRecord.STATE_INITIALIZED) {
+      Log.e(LOG_TAG, "Audio Record can't initialize!");
+      return;
+    }
+
+    record.startRecording();
+
+    Log.v(LOG_TAG, "Start recording");
+
+    // Loop, gathering audio data and copying it to a round-robin buffer.
+    while (shouldContinue) {
+      int numberRead = record.read(audioBuffer, 0, audioBuffer.length);
+      int maxLength = recordingBuffer.length;
+      int newRecordingOffset = recordingOffset + numberRead;
+      int secondCopyLength = Math.max(0, newRecordingOffset - maxLength);
+      int firstCopyLength = numberRead - secondCopyLength;
+      // We store off all the data for the recognition thread to access. The ML
+      // thread will copy out of this buffer into its own, while holding the
+      // lock, so this should be thread safe.
+      recordingBufferLock.lock();
+      try {
+        System.arraycopy(audioBuffer, 0, recordingBuffer, recordingOffset, firstCopyLength);
+        System.arraycopy(audioBuffer, firstCopyLength, recordingBuffer, 0, secondCopyLength);
+        recordingOffset = newRecordingOffset % maxLength;
+      } finally {
+        recordingBufferLock.unlock();
+      }
+    }
+
+    record.stop();
+    record.release();
+  }
+
+  public synchronized void startRecognition() {
+    if (recognitionThread != null) {
+      return;
+    }
+    shouldContinueRecognition = true;
+    recognitionThread =
+        new Thread(
+            new Runnable() {
+              @Override
+              public void run() {
+                recognize();
+              }
+            });
+    recognitionThread.start();
+  }
+
+  public synchronized void stopRecognition() {
+    if (recognitionThread == null) {
+      return;
+    }
+    shouldContinueRecognition = false;
+    recognitionThread = null;
+  }
+
+  private void recognize() {
+    Log.v(LOG_TAG, "Start recognition");
+
+    short[] inputBuffer = new short[RECORDING_LENGTH];
+    float[][] floatInputBuffer = new float[RECORDING_LENGTH][1];
+    float[][] outputScores = new float[1][labels.size()];
+    int[] sampleRateList = new int[] {SAMPLE_RATE};
+
+    // Loop, grabbing recorded data and running the recognition model on it.
+    while (shouldContinueRecognition) {
+      // The recording thread places data in this round-robin buffer, so lock to
+      // make sure there's no writing happening and then copy it to our own
+      // local version.
+      recordingBufferLock.lock();
+      try {
+        int maxLength = recordingBuffer.length;
+        int firstCopyLength = maxLength - recordingOffset;
+        int secondCopyLength = recordingOffset;
+        System.arraycopy(recordingBuffer, recordingOffset, inputBuffer, 0, firstCopyLength);
+        System.arraycopy(recordingBuffer, 0, inputBuffer, firstCopyLength, secondCopyLength);
+      } finally {
+        recordingBufferLock.unlock();
+      }
+
+      // We need to feed in float values between -1.0f and 1.0f, so divide the
+      // signed 16-bit inputs.
+      for (int i = 0; i < RECORDING_LENGTH; ++i) {
+        floatInputBuffer[i][0] = inputBuffer[i] / 32767.0f;
+      }
+
+      Object[] inputArray = {floatInputBuffer, sampleRateList};
+      Map<Integer, Object> outputMap = new HashMap<>();
+      outputMap.put(0, outputScores);
+
+      // Run the model.
+      tfLite.runForMultipleInputsOutputs(inputArray, outputMap);
+
+      // Use the smoother to figure out if we've had a real recognition event.
+      long currentTime = System.currentTimeMillis();
+      final RecognizeCommands.RecognitionResult result =
+          recognizeCommands.processLatestResults(outputScores[0], currentTime);
+
+      runOnUiThread(
+          new Runnable() {
+            @Override
+            public void run() {
+              // If we do have a new command, highlight the right list entry.
+              if (!result.foundCommand.startsWith("_") && result.isNewCommand) {
+                int labelIndex = -1;
+                for (int i = 0; i < labels.size(); ++i) {
+                  if (labels.get(i).equals(result.foundCommand)) {
+                    labelIndex = i;
+                  }
+                }
+                final View labelView = (View) labelsListView.getChildAt(labelIndex - 2);
+                ValueAnimator colorAnimation =
+                    ValueAnimator.ofArgb(0x00b3ccff, 0xffb3ccff, 0x00b3ccff);
+                colorAnimation.setDuration(750);
+                colorAnimation.addUpdateListener(
+                    new ValueAnimator.AnimatorUpdateListener() {
+                      @Override
+                      public void onAnimationUpdate(ValueAnimator animator) {
+                        labelView.setBackgroundColor((int) animator.getAnimatedValue());
+                      }
+                    });
+                colorAnimation.start();
+              }
+            }
+          });
+      try {
+        // We don't need to run too frequently, so snooze for a bit.
+        Thread.sleep(MINIMUM_TIME_BETWEEN_SAMPLES_MS);
+      } catch (InterruptedException e) {
+        // Ignore
+      }
+    }
+
+    Log.v(LOG_TAG, "End recognition");
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteImageClassifier.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteImageClassifier.java
new file mode 100644
index 0000000000..d75c3ceada
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteImageClassifier.java
@@ -0,0 +1,209 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import android.content.res.AssetFileDescriptor;
+import android.content.res.AssetManager;
+import android.graphics.Bitmap;
+import android.os.SystemClock;
+import android.os.Trace;
+import android.util.Log;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStreamReader;
+import java.nio.ByteBuffer;
+import java.nio.ByteOrder;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.List;
+import java.util.PriorityQueue;
+import java.util.Vector;
+import org.tensorflow.lite.Interpreter;
+
+/** A classifier specialized to label images using TensorFlow. */
+public class TFLiteImageClassifier implements Classifier {
+  private static final String TAG = "TFLiteImageClassifier";
+
+  // Only return this many results with at least this confidence.
+  private static final int MAX_RESULTS = 3;
+
+  private Interpreter tfLite;
+
+  /** Dimensions of inputs. */
+  private static final int DIM_BATCH_SIZE = 1;
+
+  private static final int DIM_PIXEL_SIZE = 3;
+
+  private static final int DIM_IMG_SIZE_X = 224;
+  private static final int DIM_IMG_SIZE_Y = 224;
+
+  byte[][] labelProb;
+
+  // Pre-allocated buffers.
+  private Vector<String> labels = new Vector<String>();
+  private int[] intValues;
+  private ByteBuffer imgData = null;
+
+  private TFLiteImageClassifier() {}
+
+  /** Memory-map the model file in Assets. */
+  private static MappedByteBuffer loadModelFile(AssetManager assets, String modelFilename)
+      throws IOException {
+    AssetFileDescriptor fileDescriptor = assets.openFd(modelFilename);
+    FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
+    FileChannel fileChannel = inputStream.getChannel();
+    long startOffset = fileDescriptor.getStartOffset();
+    long declaredLength = fileDescriptor.getDeclaredLength();
+    return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
+  }
+
+  /**
+   * Initializes a native TensorFlow session for classifying images.
+   *
+   * @param assetManager The asset manager to be used to load assets.
+   * @param modelFilename The filepath of the model GraphDef protocol buffer.
+   * @param labelFilename The filepath of label file for classes.
+   * @param inputSize The input size. A square image of inputSize x inputSize is assumed.
+   * @throws IOException
+   */
+  public static Classifier create(
+      AssetManager assetManager, String modelFilename, String labelFilename, int inputSize) {
+    TFLiteImageClassifier c = new TFLiteImageClassifier();
+
+    // Read the label names into memory.
+    // TODO(andrewharp): make this handle non-assets.
+    Log.i(TAG, "Reading labels from: " + labelFilename);
+    BufferedReader br = null;
+    try {
+      br = new BufferedReader(new InputStreamReader(assetManager.open(labelFilename)));
+      String line;
+      while ((line = br.readLine()) != null) {
+        c.labels.add(line);
+      }
+      br.close();
+    } catch (IOException e) {
+      throw new RuntimeException("Problem reading label file!" , e);
+    }
+
+    c.imgData =
+        ByteBuffer.allocateDirect(
+            DIM_BATCH_SIZE * DIM_IMG_SIZE_X * DIM_IMG_SIZE_Y * DIM_PIXEL_SIZE);
+
+    c.imgData.order(ByteOrder.nativeOrder());
+    try {
+      c.tfLite = new Interpreter(loadModelFile(assetManager, modelFilename));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+
+    // The shape of the output is [N, NUM_CLASSES], where N is the batch size.
+    Log.i(TAG, "Read " + c.labels.size() + " labels");
+
+    // Pre-allocate buffers.
+    c.intValues = new int[inputSize * inputSize];
+
+    c.labelProb = new byte[1][c.labels.size()];
+
+    return c;
+  }
+
+  /** Writes Image data into a {@code ByteBuffer}. */
+  private void convertBitmapToByteBuffer(Bitmap bitmap) {
+    if (imgData == null) {
+      return;
+    }
+    imgData.rewind();
+    bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
+    // Convert the image to floating point.
+    int pixel = 0;
+    long startTime = SystemClock.uptimeMillis();
+    for (int i = 0; i < DIM_IMG_SIZE_X; ++i) {
+      for (int j = 0; j < DIM_IMG_SIZE_Y; ++j) {
+        final int val = intValues[pixel++];
+        imgData.put((byte) ((val >> 16) & 0xFF));
+        imgData.put((byte) ((val >> 8) & 0xFF));
+        imgData.put((byte) (val & 0xFF));
+      }
+    }
+    long endTime = SystemClock.uptimeMillis();
+    Log.d(TAG, "Timecost to put values into ByteBuffer: " + Long.toString(endTime - startTime));
+  }
+
+  @Override
+  public List<Recognition> recognizeImage(final Bitmap bitmap) {
+    // Log this method so that it can be analyzed with systrace.
+    Trace.beginSection("recognizeImage");
+
+    Trace.beginSection("preprocessBitmap");
+
+    long startTime;
+    long endTime;
+    startTime = SystemClock.uptimeMillis();
+
+    convertBitmapToByteBuffer(bitmap);
+
+    // Run the inference call.
+    Trace.beginSection("run");
+    startTime = SystemClock.uptimeMillis();
+    tfLite.run(imgData, labelProb);
+    endTime = SystemClock.uptimeMillis();
+    Log.i(TAG, "Inf time: " + (endTime - startTime));
+    Trace.endSection();
+
+    // Find the best classifications.
+    PriorityQueue<Recognition> pq =
+        new PriorityQueue<Recognition>(
+            3,
+            new Comparator<Recognition>() {
+              @Override
+              public int compare(Recognition lhs, Recognition rhs) {
+                // Intentionally reversed to put high confidence at the head of the queue.
+                return Float.compare(rhs.getConfidence(), lhs.getConfidence());
+              }
+            });
+    for (int i = 0; i < labels.size(); ++i) {
+      pq.add(
+          new Recognition(
+              "" + i,
+              labels.size() > i ? labels.get(i) : "unknown",
+              (float) labelProb[0][i],
+              null));
+    }
+    final ArrayList<Recognition> recognitions = new ArrayList<Recognition>();
+    int recognitionsSize = Math.min(pq.size(), MAX_RESULTS);
+    for (int i = 0; i < recognitionsSize; ++i) {
+      recognitions.add(pq.poll());
+    }
+    Trace.endSection(); // "recognizeImage"
+    return recognitions;
+  }
+
+  @Override
+  public void enableStatLogging(boolean logStats) {
+  }
+
+  @Override
+  public String getStatString() {
+    return "";
+  }
+
+  @Override
+  public void close() {
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteObjectDetectionAPIModel.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteObjectDetectionAPIModel.java
new file mode 100644
index 0000000000..bfb4a0a04b
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/TFLiteObjectDetectionAPIModel.java
@@ -0,0 +1,292 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo;
+
+import android.content.res.AssetFileDescriptor;
+import android.content.res.AssetManager;
+import android.graphics.Bitmap;
+import android.graphics.RectF;
+import android.os.Trace;
+import java.io.BufferedReader;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.MappedByteBuffer;
+import java.nio.channels.FileChannel;
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.PriorityQueue;
+import java.util.StringTokenizer;
+import java.util.Vector;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.lite.Interpreter;
+
+/**
+ * Wrapper for frozen detection models trained using the Tensorflow Object Detection API:
+ * github.com/tensorflow/models/tree/master/research/object_detection
+ */
+public class TFLiteObjectDetectionAPIModel implements Classifier {
+  private static final Logger LOGGER = new Logger();
+
+  // Only return this many results.
+  private static final int NUM_RESULTS = 1917;
+  private static final int NUM_CLASSES = 91;
+
+  private static final float Y_SCALE = 10.0f;
+  private static final float X_SCALE = 10.0f;
+  private static final float H_SCALE = 5.0f;
+  private static final float W_SCALE = 5.0f;
+
+  // Config values.
+  private int inputSize;
+
+  private final float[][] boxPriors = new float[4][NUM_RESULTS];
+
+  // Pre-allocated buffers.
+  private Vector<String> labels = new Vector<String>();
+  private int[] intValues;
+  private float[][][] outputLocations;
+  private float[][][] outputClasses;
+
+  float[][][][] img;
+
+  private Interpreter tfLite;
+
+  private float expit(final float x) {
+    return (float) (1. / (1. + Math.exp(-x)));
+  }
+
+  /** Memory-map the model file in Assets. */
+  private static MappedByteBuffer loadModelFile(AssetManager assets, String modelFilename)
+      throws IOException {
+    AssetFileDescriptor fileDescriptor = assets.openFd(modelFilename);
+    FileInputStream inputStream = new FileInputStream(fileDescriptor.getFileDescriptor());
+    FileChannel fileChannel = inputStream.getChannel();
+    long startOffset = fileDescriptor.getStartOffset();
+    long declaredLength = fileDescriptor.getDeclaredLength();
+    return fileChannel.map(FileChannel.MapMode.READ_ONLY, startOffset, declaredLength);
+  }
+
+  private void loadCoderOptions(
+      final AssetManager assetManager, final String locationFilename, final float[][] boxPriors)
+      throws IOException {
+    // Try to be intelligent about opening from assets or sdcard depending on prefix.
+    final String assetPrefix = "file:///android_asset/";
+    InputStream is;
+    if (locationFilename.startsWith(assetPrefix)) {
+      is = assetManager.open(locationFilename.split(assetPrefix, -1)[1]);
+    } else {
+      is = new FileInputStream(locationFilename);
+    }
+
+    final BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+
+    for (int lineNum = 0; lineNum < 4; ++lineNum) {
+      String line = reader.readLine();
+      final StringTokenizer st = new StringTokenizer(line, ", ");
+      int priorIndex = 0;
+      while (st.hasMoreTokens()) {
+        final String token = st.nextToken();
+        try {
+          final float number = Float.parseFloat(token);
+          boxPriors[lineNum][priorIndex++] = number;
+        } catch (final NumberFormatException e) {
+          // Silently ignore.
+        }
+      }
+      if (priorIndex != NUM_RESULTS) {
+        throw new RuntimeException(
+            "BoxPrior length mismatch: " + priorIndex + " vs " + NUM_RESULTS);
+      }
+    }
+
+    LOGGER.i("Loaded box priors!");
+  }
+
+  void decodeCenterSizeBoxes(float[][][] predictions) {
+    for (int i = 0; i < NUM_RESULTS; ++i) {
+      float ycenter = predictions[0][i][0] / Y_SCALE * boxPriors[2][i] + boxPriors[0][i];
+      float xcenter = predictions[0][i][1] / X_SCALE * boxPriors[3][i] + boxPriors[1][i];
+      float h = (float) Math.exp(predictions[0][i][2] / H_SCALE) * boxPriors[2][i];
+      float w = (float) Math.exp(predictions[0][i][3] / W_SCALE) * boxPriors[3][i];
+
+      float ymin = ycenter - h / 2.f;
+      float xmin = xcenter - w / 2.f;
+      float ymax = ycenter + h / 2.f;
+      float xmax = xcenter + w / 2.f;
+
+      predictions[0][i][0] = ymin;
+      predictions[0][i][1] = xmin;
+      predictions[0][i][2] = ymax;
+      predictions[0][i][3] = xmax;
+    }
+  }
+
+  /**
+   * Initializes a native TensorFlow session for classifying images.
+   *
+   * @param assetManager The asset manager to be used to load assets.
+   * @param modelFilename The filepath of the model GraphDef protocol buffer.
+   * @param labelFilename The filepath of label file for classes.
+   */
+  public static Classifier create(
+      final AssetManager assetManager,
+      final String modelFilename,
+      final String labelFilename,
+      final int inputSize) throws IOException {
+    final TFLiteObjectDetectionAPIModel d = new TFLiteObjectDetectionAPIModel();
+
+    d.loadCoderOptions(assetManager, "file:///android_asset/box_priors.txt", d.boxPriors);
+
+    InputStream labelsInput = null;
+    String actualFilename = labelFilename.split("file:///android_asset/")[1];
+    labelsInput = assetManager.open(actualFilename);
+    BufferedReader br = null;
+    br = new BufferedReader(new InputStreamReader(labelsInput));
+    String line;
+    while ((line = br.readLine()) != null) {
+      LOGGER.w(line);
+      d.labels.add(line);
+    }
+    br.close();
+
+    d.inputSize = inputSize;
+
+    try {
+      d.tfLite = new Interpreter(loadModelFile(assetManager, modelFilename));
+    } catch (Exception e) {
+      throw new RuntimeException(e);
+    }
+
+    // Pre-allocate buffers.
+    d.img = new float[1][inputSize][inputSize][3];
+
+    d.intValues = new int[d.inputSize * d.inputSize];
+    d.outputLocations = new float[1][NUM_RESULTS][4];
+    d.outputClasses = new float[1][NUM_RESULTS][NUM_CLASSES];
+    return d;
+  }
+
+  private TFLiteObjectDetectionAPIModel() {}
+
+  @Override
+  public List<Recognition> recognizeImage(final Bitmap bitmap) {
+    // Log this method so that it can be analyzed with systrace.
+    Trace.beginSection("recognizeImage");
+
+    Trace.beginSection("preprocessBitmap");
+    // Preprocess the image data from 0-255 int to normalized float based
+    // on the provided parameters.
+    bitmap.getPixels(intValues, 0, bitmap.getWidth(), 0, 0, bitmap.getWidth(), bitmap.getHeight());
+
+    for (int i = 0; i < inputSize; ++i) {
+      for (int j = 0; j < inputSize; ++j) {
+        int pixel = intValues[j * inputSize + i];
+        img[0][j][i][2] = (float) (pixel & 0xFF) / 128.0f - 1.0f;
+        img[0][j][i][1] = (float) ((pixel >> 8) & 0xFF) / 128.0f - 1.0f;
+        img[0][j][i][0] = (float) ((pixel >> 16) & 0xFF) / 128.0f - 1.0f;
+      }
+    }
+    Trace.endSection(); // preprocessBitmap
+
+    // Copy the input data into TensorFlow.
+    Trace.beginSection("feed");
+    outputLocations = new float[1][NUM_RESULTS][4];
+    outputClasses = new float[1][NUM_RESULTS][NUM_CLASSES];
+
+    Object[] inputArray = {img};
+    Map<Integer, Object> outputMap = new HashMap<>();
+    outputMap.put(0, outputLocations);
+    outputMap.put(1, outputClasses);
+    Trace.endSection();
+
+    // Run the inference call.
+    Trace.beginSection("run");
+    tfLite.runForMultipleInputsOutputs(inputArray, outputMap);
+    Trace.endSection();
+
+    decodeCenterSizeBoxes(outputLocations);
+
+    // Find the best detections.
+    final PriorityQueue<Recognition> pq =
+        new PriorityQueue<Recognition>(
+            1,
+            new Comparator<Recognition>() {
+              @Override
+              public int compare(final Recognition lhs, final Recognition rhs) {
+                // Intentionally reversed to put high confidence at the head of the queue.
+                return Float.compare(rhs.getConfidence(), lhs.getConfidence());
+              }
+            });
+
+    // Scale them back to the input size.
+    for (int i = 0; i < NUM_RESULTS; ++i) {
+      float topClassScore = -1000f;
+      int topClassScoreIndex = -1;
+
+      // Skip the first catch-all class.
+      for (int j = 1; j < NUM_CLASSES; ++j) {
+        float score = expit(outputClasses[0][i][j]);
+
+        if (score > topClassScore) {
+          topClassScoreIndex = j;
+          topClassScore = score;
+        }
+      }
+
+      if (topClassScore > 0.001f) {
+        final RectF detection =
+            new RectF(
+                outputLocations[0][i][1] * inputSize,
+                outputLocations[0][i][0] * inputSize,
+                outputLocations[0][i][3] * inputSize,
+                outputLocations[0][i][2] * inputSize);
+
+        pq.add(
+            new Recognition(
+                "" + i,
+                labels.get(topClassScoreIndex),
+                outputClasses[0][i][topClassScoreIndex],
+                detection));
+      }
+    }
+
+    final ArrayList<Recognition> recognitions = new ArrayList<Recognition>();
+    for (int i = 0; i < Math.min(pq.size(), 10); ++i) {
+      Recognition recog = pq.poll();
+      recognitions.add(recog);
+    }
+    Trace.endSection(); // "recognizeImage"
+    return recognitions;
+  }
+
+  @Override
+  public void enableStatLogging(final boolean logStats) {
+  }
+
+  @Override
+  public String getStatString() {
+    return "";
+  }
+
+  @Override
+  public void close() {
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/AssetUtils.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/AssetUtils.java
new file mode 100644
index 0000000000..c50efdf889
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/AssetUtils.java
@@ -0,0 +1,78 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.content.Context;
+import android.content.res.AssetManager;
+import android.util.Log;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+
+/** Utilities for dealing with assets. */
+public class AssetUtils {
+
+  private static final String TAG = AssetUtils.class.getSimpleName();
+
+  private static final int BYTE_BUF_SIZE = 2048;
+
+  /**
+   * Copies a file from assets.
+   *
+   * @param context application context used to discover assets.
+   * @param assetName the relative file name within assets.
+   * @param targetName the target file name, always over write the existing file.
+   * @throws IOException if operation fails.
+   */
+  public static void copy(Context context, String assetName, String targetName) throws IOException {
+
+    Log.d(TAG, "creating file " + targetName + " from " + assetName);
+
+    File targetFile = null;
+    InputStream inputStream = null;
+    FileOutputStream outputStream = null;
+
+    try {
+      AssetManager assets = context.getAssets();
+      targetFile = new File(targetName);
+      inputStream = assets.open(assetName);
+      // TODO(kanlig): refactor log messages to make them more useful.
+      Log.d(TAG, "Creating outputstream");
+      outputStream = new FileOutputStream(targetFile, false /* append */);
+      copy(inputStream, outputStream);
+    } finally {
+      if (outputStream != null) {
+        outputStream.close();
+      }
+      if (inputStream != null) {
+        inputStream.close();
+      }
+    }
+  }
+
+  private static void copy(InputStream from, OutputStream to) throws IOException {
+    byte[] buf = new byte[BYTE_BUF_SIZE];
+    while (true) {
+      int r = from.read(buf);
+      if (r == -1) {
+        break;
+      }
+      to.write(buf, 0, r);
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/BorderedText.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/BorderedText.java
new file mode 100644
index 0000000000..decfc3d879
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/BorderedText.java
@@ -0,0 +1,117 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Paint;
+import android.graphics.Paint.Align;
+import android.graphics.Paint.Style;
+import android.graphics.Rect;
+import android.graphics.Typeface;
+import java.util.Vector;
+
+/**
+ * A class that encapsulates the tedious bits of rendering legible, bordered text onto a canvas.
+ */
+public class BorderedText {
+  private final Paint interiorPaint;
+  private final Paint exteriorPaint;
+
+  private final float textSize;
+
+  /**
+   * Creates a left-aligned bordered text object with a white interior, and a black exterior with
+   * the specified text size.
+   *
+   * @param textSize text size in pixels
+   */
+  public BorderedText(final float textSize) {
+    this(Color.WHITE, Color.BLACK, textSize);
+  }
+
+  /**
+   * Create a bordered text object with the specified interior and exterior colors, text size and
+   * alignment.
+   *
+   * @param interiorColor the interior text color
+   * @param exteriorColor the exterior text color
+   * @param textSize text size in pixels
+   */
+  public BorderedText(final int interiorColor, final int exteriorColor, final float textSize) {
+    interiorPaint = new Paint();
+    interiorPaint.setTextSize(textSize);
+    interiorPaint.setColor(interiorColor);
+    interiorPaint.setStyle(Style.FILL);
+    interiorPaint.setAntiAlias(false);
+    interiorPaint.setAlpha(255);
+
+    exteriorPaint = new Paint();
+    exteriorPaint.setTextSize(textSize);
+    exteriorPaint.setColor(exteriorColor);
+    exteriorPaint.setStyle(Style.FILL_AND_STROKE);
+    exteriorPaint.setStrokeWidth(textSize / 8);
+    exteriorPaint.setAntiAlias(false);
+    exteriorPaint.setAlpha(255);
+
+    this.textSize = textSize;
+  }
+
+  public void setTypeface(Typeface typeface) {
+    interiorPaint.setTypeface(typeface);
+    exteriorPaint.setTypeface(typeface);
+  }
+
+  public void drawText(final Canvas canvas, final float posX, final float posY, final String text) {
+    canvas.drawText(text, posX, posY, exteriorPaint);
+    canvas.drawText(text, posX, posY, interiorPaint);
+  }
+
+  public void drawLines(Canvas canvas, final float posX, final float posY, Vector<String> lines) {
+    int lineNum = 0;
+    for (final String line : lines) {
+      drawText(canvas, posX, posY - getTextSize() * (lines.size() - lineNum - 1), line);
+      ++lineNum;
+    }
+  }
+
+  public void setInteriorColor(final int color) {
+    interiorPaint.setColor(color);
+  }
+
+  public void setExteriorColor(final int color) {
+    exteriorPaint.setColor(color);
+  }
+
+  public float getTextSize() {
+    return textSize;
+  }
+
+  public void setAlpha(final int alpha) {
+    interiorPaint.setAlpha(alpha);
+    exteriorPaint.setAlpha(alpha);
+  }
+
+  public void getTextBounds(
+      final String line, final int index, final int count, final Rect lineBounds) {
+    interiorPaint.getTextBounds(line, index, count, lineBounds);
+  }
+
+  public void setTextAlign(final Align align) {
+    interiorPaint.setTextAlign(align);
+    exteriorPaint.setTextAlign(align);
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/ImageUtils.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/ImageUtils.java
new file mode 100644
index 0000000000..e02c655917
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/ImageUtils.java
@@ -0,0 +1,344 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.graphics.Bitmap;
+import android.graphics.Matrix;
+import android.os.Environment;
+import java.io.File;
+import java.io.FileOutputStream;
+
+/**
+ * Utility class for manipulating images.
+ **/
+public class ImageUtils {
+  @SuppressWarnings("unused")
+  private static final Logger LOGGER = new Logger();
+
+  static {
+    try {
+      System.loadLibrary("tensorflow_demo");
+    } catch (UnsatisfiedLinkError e) {
+      LOGGER.w("Native library not found, native RGB -> YUV conversion may be unavailable.");
+    }
+  }
+
+  /**
+   * Utility method to compute the allocated size in bytes of a YUV420SP image
+   * of the given dimensions.
+   */
+  public static int getYUVByteSize(final int width, final int height) {
+    // The luminance plane requires 1 byte per pixel.
+    final int ySize = width * height;
+
+    // The UV plane works on 2x2 blocks, so dimensions with odd size must be rounded up.
+    // Each 2x2 block takes 2 bytes to encode, one each for U and V.
+    final int uvSize = ((width + 1) / 2) * ((height + 1) / 2) * 2;
+
+    return ySize + uvSize;
+  }
+
+  /**
+   * Saves a Bitmap object to disk for analysis.
+   *
+   * @param bitmap The bitmap to save.
+   */
+  public static void saveBitmap(final Bitmap bitmap) {
+    saveBitmap(bitmap, "preview.png");
+  }
+
+  /**
+   * Saves a Bitmap object to disk for analysis.
+   *
+   * @param bitmap The bitmap to save.
+   * @param filename The location to save the bitmap to.
+   */
+  public static void saveBitmap(final Bitmap bitmap, final String filename) {
+    final String root =
+        Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator + "tensorflow";
+    LOGGER.i("Saving %dx%d bitmap to %s.", bitmap.getWidth(), bitmap.getHeight(), root);
+    final File myDir = new File(root);
+
+    if (!myDir.mkdirs()) {
+      LOGGER.i("Make dir failed");
+    }
+
+    final String fname = filename;
+    final File file = new File(myDir, fname);
+    if (file.exists()) {
+      file.delete();
+    }
+    try {
+      final FileOutputStream out = new FileOutputStream(file);
+      bitmap.compress(Bitmap.CompressFormat.PNG, 99, out);
+      out.flush();
+      out.close();
+    } catch (final Exception e) {
+      LOGGER.e(e, "Exception!");
+    }
+  }
+
+  // This value is 2 ^ 18 - 1, and is used to clamp the RGB values before their ranges
+  // are normalized to eight bits.
+  static final int kMaxChannelValue = 262143;
+
+  // Always prefer the native implementation if available.
+  private static boolean useNativeConversion = false;
+
+  public static void convertYUV420SPToARGB8888(
+      byte[] input,
+      int width,
+      int height,
+      int[] output) {
+    if (useNativeConversion) {
+      try {
+        ImageUtils.convertYUV420SPToARGB8888(input, output, width, height, false);
+        return;
+      } catch (UnsatisfiedLinkError e) {
+        LOGGER.w(
+            "Native YUV420SP -> RGB implementation not found, falling back to Java implementation");
+        useNativeConversion = false;
+      }
+    }
+
+    // Java implementation of YUV420SP to ARGB8888 converting
+    final int frameSize = width * height;
+    for (int j = 0, yp = 0; j < height; j++) {
+      int uvp = frameSize + (j >> 1) * width;
+      int u = 0;
+      int v = 0;
+
+      for (int i = 0; i < width; i++, yp++) {
+        int y = 0xff & input[yp];
+        if ((i & 1) == 0) {
+          v = 0xff & input[uvp++];
+          u = 0xff & input[uvp++];
+        }
+
+        output[yp] = YUV2RGB(y, u, v);
+      }
+    }
+  }
+
+  private static int YUV2RGB(int y, int u, int v) {
+    // Adjust and check YUV values
+    y = (y - 16) < 0 ? 0 : (y - 16);
+    u -= 128;
+    v -= 128;
+
+    // This is the floating point equivalent. We do the conversion in integer
+    // because some Android devices do not have floating point in hardware.
+    // nR = (int)(1.164 * nY + 2.018 * nU);
+    // nG = (int)(1.164 * nY - 0.813 * nV - 0.391 * nU);
+    // nB = (int)(1.164 * nY + 1.596 * nV);
+    int y1192 = 1192 * y;
+    int r = (y1192 + 1634 * v);
+    int g = (y1192 - 833 * v - 400 * u);
+    int b = (y1192 + 2066 * u);
+
+    // Clipping RGB values to be inside boundaries [ 0 , kMaxChannelValue ]
+    r = r > kMaxChannelValue ? kMaxChannelValue : (r < 0 ? 0 : r);
+    g = g > kMaxChannelValue ? kMaxChannelValue : (g < 0 ? 0 : g);
+    b = b > kMaxChannelValue ? kMaxChannelValue : (b < 0 ? 0 : b);
+
+    return 0xff000000 | ((r << 6) & 0xff0000) | ((g >> 2) & 0xff00) | ((b >> 10) & 0xff);
+  }
+
+
+  public static void convertYUV420ToARGB8888(
+      byte[] yData,
+      byte[] uData,
+      byte[] vData,
+      int width,
+      int height,
+      int yRowStride,
+      int uvRowStride,
+      int uvPixelStride,
+      int[] out) {
+    if (useNativeConversion) {
+      try {
+        convertYUV420ToARGB8888(
+            yData, uData, vData, out, width, height, yRowStride, uvRowStride, uvPixelStride, false);
+        return;
+      } catch (UnsatisfiedLinkError e) {
+        LOGGER.w(
+            "Native YUV420 -> RGB implementation not found, falling back to Java implementation");
+        useNativeConversion = false;
+      }
+    }
+
+    int yp = 0;
+    for (int j = 0; j < height; j++) {
+      int pY = yRowStride * j;
+      int pUV = uvRowStride * (j >> 1);
+
+      for (int i = 0; i < width; i++) {
+        int uv_offset = pUV + (i >> 1) * uvPixelStride;
+
+        out[yp++] = YUV2RGB(
+            0xff & yData[pY + i],
+            0xff & uData[uv_offset],
+            0xff & vData[uv_offset]);
+      }
+    }
+  }
+
+
+  /**
+   * Converts YUV420 semi-planar data to ARGB 8888 data using the supplied width and height. The
+   * input and output must already be allocated and non-null. For efficiency, no error checking is
+   * performed.
+   *
+   * @param input The array of YUV 4:2:0 input data.
+   * @param output A pre-allocated array for the ARGB 8:8:8:8 output data.
+   * @param width The width of the input image.
+   * @param height The height of the input image.
+   * @param halfSize If true, downsample to 50% in each dimension, otherwise not.
+   */
+  private static native void convertYUV420SPToARGB8888(
+      byte[] input, int[] output, int width, int height, boolean halfSize);
+
+  /**
+   * Converts YUV420 semi-planar data to ARGB 8888 data using the supplied width
+   * and height. The input and output must already be allocated and non-null.
+   * For efficiency, no error checking is performed.
+   *
+   * @param y
+   * @param u
+   * @param v
+   * @param uvPixelStride
+   * @param width The width of the input image.
+   * @param height The height of the input image.
+   * @param halfSize If true, downsample to 50% in each dimension, otherwise not.
+   * @param output A pre-allocated array for the ARGB 8:8:8:8 output data.
+   */
+  private static native void convertYUV420ToARGB8888(
+      byte[] y,
+      byte[] u,
+      byte[] v,
+      int[] output,
+      int width,
+      int height,
+      int yRowStride,
+      int uvRowStride,
+      int uvPixelStride,
+      boolean halfSize);
+
+  /**
+   * Converts YUV420 semi-planar data to RGB 565 data using the supplied width
+   * and height. The input and output must already be allocated and non-null.
+   * For efficiency, no error checking is performed.
+   *
+   * @param input The array of YUV 4:2:0 input data.
+   * @param output A pre-allocated array for the RGB 5:6:5 output data.
+   * @param width The width of the input image.
+   * @param height The height of the input image.
+   */
+  private static native void convertYUV420SPToRGB565(
+      byte[] input, byte[] output, int width, int height);
+
+  /**
+   * Converts 32-bit ARGB8888 image data to YUV420SP data.  This is useful, for
+   * instance, in creating data to feed the classes that rely on raw camera
+   * preview frames.
+   *
+   * @param input An array of input pixels in ARGB8888 format.
+   * @param output A pre-allocated array for the YUV420SP output data.
+   * @param width The width of the input image.
+   * @param height The height of the input image.
+   */
+  private static native void convertARGB8888ToYUV420SP(
+      int[] input, byte[] output, int width, int height);
+
+  /**
+   * Converts 16-bit RGB565 image data to YUV420SP data.  This is useful, for
+   * instance, in creating data to feed the classes that rely on raw camera
+   * preview frames.
+   *
+   * @param input An array of input pixels in RGB565 format.
+   * @param output A pre-allocated array for the YUV420SP output data.
+   * @param width The width of the input image.
+   * @param height The height of the input image.
+   */
+  private static native void convertRGB565ToYUV420SP(
+      byte[] input, byte[] output, int width, int height);
+
+  /**
+   * Returns a transformation matrix from one reference frame into another.
+   * Handles cropping (if maintaining aspect ratio is desired) and rotation.
+   *
+   * @param srcWidth Width of source frame.
+   * @param srcHeight Height of source frame.
+   * @param dstWidth Width of destination frame.
+   * @param dstHeight Height of destination frame.
+   * @param applyRotation Amount of rotation to apply from one frame to another.
+   *  Must be a multiple of 90.
+   * @param maintainAspectRatio If true, will ensure that scaling in x and y remains constant,
+   * cropping the image if necessary.
+   * @return The transformation fulfilling the desired requirements.
+   */
+  public static Matrix getTransformationMatrix(
+      final int srcWidth,
+      final int srcHeight,
+      final int dstWidth,
+      final int dstHeight,
+      final int applyRotation,
+      final boolean maintainAspectRatio) {
+    final Matrix matrix = new Matrix();
+
+    if (applyRotation != 0) {
+      if (applyRotation % 90 != 0) {
+        LOGGER.w("Rotation of %d % 90 != 0", applyRotation);
+      }
+
+      // Translate so center of image is at origin.
+      matrix.postTranslate(-srcWidth / 2.0f, -srcHeight / 2.0f);
+
+      // Rotate around origin.
+      matrix.postRotate(applyRotation);
+    }
+
+    // Account for the already applied rotation, if any, and then determine how
+    // much scaling is needed for each axis.
+    final boolean transpose = (Math.abs(applyRotation) + 90) % 180 == 0;
+
+    final int inWidth = transpose ? srcHeight : srcWidth;
+    final int inHeight = transpose ? srcWidth : srcHeight;
+
+    // Apply scaling if necessary.
+    if (inWidth != dstWidth || inHeight != dstHeight) {
+      final float scaleFactorX = dstWidth / (float) inWidth;
+      final float scaleFactorY = dstHeight / (float) inHeight;
+
+      if (maintainAspectRatio) {
+        // Scale by minimum factor so that dst is filled completely while
+        // maintaining the aspect ratio. Some image may fall off the edge.
+        final float scaleFactor = Math.max(scaleFactorX, scaleFactorY);
+        matrix.postScale(scaleFactor, scaleFactor);
+      } else {
+        // Scale exactly to fill dst from src.
+        matrix.postScale(scaleFactorX, scaleFactorY);
+      }
+    }
+
+    if (applyRotation != 0) {
+      // Translate back from origin centered reference to destination frame.
+      matrix.postTranslate(dstWidth / 2.0f, dstHeight / 2.0f);
+    }
+
+    return matrix;
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Logger.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Logger.java
new file mode 100644
index 0000000000..0d984096a0
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Logger.java
@@ -0,0 +1,190 @@
+/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.util.Log;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * Wrapper for the platform log function, allows convenient message prefixing and log disabling.
+ */
+public final class Logger {
+  private static final String DEFAULT_TAG = "tensorflow";
+  private static final int DEFAULT_MIN_LOG_LEVEL = Log.DEBUG;
+
+  // Classes to be ignored when examining the stack trace
+  private static final Set<String> IGNORED_CLASS_NAMES;
+
+  static {
+    IGNORED_CLASS_NAMES = new HashSet<String>(3);
+    IGNORED_CLASS_NAMES.add("dalvik.system.VMStack");
+    IGNORED_CLASS_NAMES.add("java.lang.Thread");
+    IGNORED_CLASS_NAMES.add(Logger.class.getCanonicalName());
+  }
+
+  private final String tag;
+  private final String messagePrefix;
+  private int minLogLevel = DEFAULT_MIN_LOG_LEVEL;
+
+  /**
+   * Creates a Logger using the class name as the message prefix.
+   *
+   * @param clazz the simple name of this class is used as the message prefix.
+   */
+  public Logger(final Class<?> clazz) {
+    this(clazz.getSimpleName());
+  }
+
+  /**
+   * Creates a Logger using the specified message prefix.
+   *
+   * @param messagePrefix is prepended to the text of every message.
+   */
+  public Logger(final String messagePrefix) {
+    this(DEFAULT_TAG, messagePrefix);
+  }
+
+  /**
+   * Creates a Logger with a custom tag and a custom message prefix. If the message prefix
+   * is set to <pre>null</pre>, the caller's class name is used as the prefix.
+   *
+   * @param tag identifies the source of a log message.
+   * @param messagePrefix prepended to every message if non-null. If null, the name of the caller is
+   *                      being used
+   */
+  public Logger(final String tag, final String messagePrefix) {
+    this.tag = tag;
+    final String prefix = messagePrefix == null ? getCallerSimpleName() : messagePrefix;
+    this.messagePrefix = (prefix.length() > 0) ? prefix + ": " : prefix;
+  }
+
+  /**
+   * Creates a Logger using the caller's class name as the message prefix.
+   */
+  public Logger() {
+    this(DEFAULT_TAG, null);
+  }
+
+  /**
+   * Creates a Logger using the caller's class name as the message prefix.
+   */
+  public Logger(final int minLogLevel) {
+    this(DEFAULT_TAG, null);
+    this.minLogLevel = minLogLevel;
+  }
+
+  public void setMinLogLevel(final int minLogLevel) {
+    this.minLogLevel = minLogLevel;
+  }
+
+  public boolean isLoggable(final int logLevel) {
+    return logLevel >= minLogLevel || Log.isLoggable(tag, logLevel);
+  }
+
+  /**
+   * Return caller's simple name.
+   *
+   * Android getStackTrace() returns an array that looks like this:
+   *     stackTrace[0]: dalvik.system.VMStack
+   *     stackTrace[1]: java.lang.Thread
+   *     stackTrace[2]: com.google.android.apps.unveil.env.UnveilLogger
+   *     stackTrace[3]: com.google.android.apps.unveil.BaseApplication
+   *
+   * This function returns the simple version of the first non-filtered name.
+   *
+   * @return caller's simple name
+   */
+  private static String getCallerSimpleName() {
+    // Get the current callstack so we can pull the class of the caller off of it.
+    final StackTraceElement[] stackTrace = Thread.currentThread().getStackTrace();
+
+    for (final StackTraceElement elem : stackTrace) {
+      final String className = elem.getClassName();
+      if (!IGNORED_CLASS_NAMES.contains(className)) {
+        // We're only interested in the simple name of the class, not the complete package.
+        final String[] classParts = className.split("\\.");
+        return classParts[classParts.length - 1];
+      }
+    }
+
+    return Logger.class.getSimpleName();
+  }
+
+  private String toMessage(final String format, final Object... args) {
+    return messagePrefix + (args.length > 0 ? String.format(format, args) : format);
+  }
+
+  public void v(final String format, final Object... args) {
+    if (isLoggable(Log.VERBOSE)) {
+      Log.v(tag, toMessage(format, args));
+    }
+  }
+
+  public void v(final Throwable t, final String format, final Object... args) {
+    if (isLoggable(Log.VERBOSE)) {
+      Log.v(tag, toMessage(format, args), t);
+    }
+  }
+
+  public void d(final String format, final Object... args) {
+    if (isLoggable(Log.DEBUG)) {
+      Log.d(tag, toMessage(format, args));
+    }
+  }
+
+  public void d(final Throwable t, final String format, final Object... args) {
+    if (isLoggable(Log.DEBUG)) {
+      Log.d(tag, toMessage(format, args), t);
+    }
+  }
+
+  public void i(final String format, final Object... args) {
+    if (isLoggable(Log.INFO)) {
+      Log.i(tag, toMessage(format, args));
+    }
+  }
+
+  public void i(final Throwable t, final String format, final Object... args) {
+    if (isLoggable(Log.INFO)) {
+      Log.i(tag, toMessage(format, args), t);
+    }
+  }
+
+  public void w(final String format, final Object... args) {
+    if (isLoggable(Log.WARN)) {
+      Log.w(tag, toMessage(format, args));
+    }
+  }
+
+  public void w(final Throwable t, final String format, final Object... args) {
+    if (isLoggable(Log.WARN)) {
+      Log.w(tag, toMessage(format, args), t);
+    }
+  }
+
+  public void e(final String format, final Object... args) {
+    if (isLoggable(Log.ERROR)) {
+      Log.e(tag, toMessage(format, args));
+    }
+  }
+
+  public void e(final Throwable t, final String format, final Object... args) {
+    if (isLoggable(Log.ERROR)) {
+      Log.e(tag, toMessage(format, args), t);
+    }
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Size.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Size.java
new file mode 100644
index 0000000000..ef15d14daa
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/Size.java
@@ -0,0 +1,143 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.graphics.Bitmap;
+import android.text.TextUtils;
+import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Size class independent of a Camera object.
+ */
+public class Size implements Comparable<Size>, Serializable {
+
+  // 1.4 went out with this UID so we'll need to maintain it to preserve pending queries when
+  // upgrading.
+  public static final long serialVersionUID = 7689808733290872361L;
+
+  public final int width;
+  public final int height;
+
+  public Size(final int width, final int height) {
+    this.width = width;
+    this.height = height;
+  }
+
+  public Size(final Bitmap bmp) {
+    this.width = bmp.getWidth();
+    this.height = bmp.getHeight();
+  }
+
+  /**
+   * Rotate a size by the given number of degrees.
+   * @param size Size to rotate.
+   * @param rotation Degrees {0, 90, 180, 270} to rotate the size.
+   * @return Rotated size.
+   */
+  public static Size getRotatedSize(final Size size, final int rotation) {
+    if (rotation % 180 != 0) {
+      // The phone is portrait, therefore the camera is sideways and frame should be rotated.
+      return new Size(size.height, size.width);
+    }
+    return size;
+  }
+
+  public static Size parseFromString(String sizeString) {
+    if (TextUtils.isEmpty(sizeString)) {
+      return null;
+    }
+
+    sizeString = sizeString.trim();
+
+    // The expected format is "<width>x<height>".
+    final String[] components = sizeString.split("x");
+    if (components.length == 2) {
+      try {
+        final int width = Integer.parseInt(components[0]);
+        final int height = Integer.parseInt(components[1]);
+        return new Size(width, height);
+      } catch (final NumberFormatException e) {
+        return null;
+      }
+    } else {
+      return null;
+    }
+  }
+
+  public static List<Size> sizeStringToList(final String sizes) {
+    final List<Size> sizeList = new ArrayList<Size>();
+    if (sizes != null) {
+      final String[] pairs = sizes.split(",");
+      for (final String pair : pairs) {
+        final Size size = Size.parseFromString(pair);
+        if (size != null) {
+          sizeList.add(size);
+        }
+      }
+    }
+    return sizeList;
+  }
+
+  public static String sizeListToString(final List<Size> sizes) {
+    String sizesString = "";
+    if (sizes != null && sizes.size() > 0) {
+      sizesString = sizes.get(0).toString();
+      for (int i = 1; i < sizes.size(); i++) {
+        sizesString += "," + sizes.get(i).toString();
+      }
+    }
+    return sizesString;
+  }
+
+  public final float aspectRatio() {
+    return (float) width / (float) height;
+  }
+
+  @Override
+  public int compareTo(final Size other) {
+    return width * height - other.width * other.height;
+  }
+
+  @Override
+  public boolean equals(final Object other) {
+    if (other == null) {
+      return false;
+    }
+
+    if (!(other instanceof Size)) {
+      return false;
+    }
+
+    final Size otherSize = (Size) other;
+    return (width == otherSize.width && height == otherSize.height);
+  }
+
+  @Override
+  public int hashCode() {
+    return width * 32713 + height;
+  }
+
+  @Override
+  public String toString() {
+    return dimensionsAsString(width, height);
+  }
+
+  public static final String dimensionsAsString(final int width, final int height) {
+    return width + "x" + height;
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/SplitTimer.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/SplitTimer.java
new file mode 100644
index 0000000000..459b0a0d4d
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/env/SplitTimer.java
@@ -0,0 +1,50 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.env;
+
+import android.os.SystemClock;
+
+/**
+ * A simple utility timer for measuring CPU time and wall-clock splits.
+ */
+public class SplitTimer {
+  private final Logger logger;
+
+  private long lastWallTime;
+  private long lastCpuTime;
+
+  public SplitTimer(final String name) {
+    logger = new Logger(name);
+    newSplit();
+  }
+
+  public void newSplit() {
+    lastWallTime = SystemClock.uptimeMillis();
+    lastCpuTime = SystemClock.currentThreadTimeMillis();
+  }
+
+  public void endSplit(final String splitName) {
+    final long currWallTime = SystemClock.uptimeMillis();
+    final long currCpuTime = SystemClock.currentThreadTimeMillis();
+
+    logger.i(
+        "%s: cpu=%dms wall=%dms",
+        splitName, currCpuTime - lastCpuTime, currWallTime - lastWallTime);
+
+    lastWallTime = currWallTime;
+    lastCpuTime = currCpuTime;
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/MultiBoxTracker.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/MultiBoxTracker.java
new file mode 100644
index 0000000000..af6af2bc8f
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/MultiBoxTracker.java
@@ -0,0 +1,421 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.tracking;
+
+import android.content.Context;
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import android.graphics.Paint.Cap;
+import android.graphics.Paint.Join;
+import android.graphics.Paint.Style;
+import android.graphics.RectF;
+import android.text.TextUtils;
+import android.util.Pair;
+import android.util.TypedValue;
+import android.widget.Toast;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Queue;
+import org.tensorflow.demo.Classifier.Recognition;
+import org.tensorflow.demo.env.BorderedText;
+import org.tensorflow.demo.env.ImageUtils;
+import org.tensorflow.demo.env.Logger;
+
+/**
+ * A tracker wrapping ObjectTracker that also handles non-max suppression and matching existing
+ * objects to new detections.
+ */
+public class MultiBoxTracker {
+  private final Logger logger = new Logger();
+
+  private static final float TEXT_SIZE_DIP = 18;
+
+  // Maximum percentage of a box that can be overlapped by another box at detection time. Otherwise
+  // the lower scored box (new or old) will be removed.
+  private static final float MAX_OVERLAP = 0.2f;
+
+  private static final float MIN_SIZE = 16.0f;
+
+  // Allow replacement of the tracked box with new results if
+  // correlation has dropped below this level.
+  private static final float MARGINAL_CORRELATION = 0.75f;
+
+  // Consider object to be lost if correlation falls below this threshold.
+  private static final float MIN_CORRELATION = 0.3f;
+
+  private static final int[] COLORS = {
+    Color.BLUE, Color.RED, Color.GREEN, Color.YELLOW, Color.CYAN, Color.MAGENTA, Color.WHITE,
+    Color.parseColor("#55FF55"), Color.parseColor("#FFA500"), Color.parseColor("#FF8888"),
+    Color.parseColor("#AAAAFF"), Color.parseColor("#FFFFAA"), Color.parseColor("#55AAAA"),
+    Color.parseColor("#AA33AA"), Color.parseColor("#0D0068")
+  };
+
+  private final Queue<Integer> availableColors = new LinkedList<Integer>();
+
+  public ObjectTracker objectTracker;
+
+  final List<Pair<Float, RectF>> screenRects = new LinkedList<Pair<Float, RectF>>();
+
+  private static class TrackedRecognition {
+    ObjectTracker.TrackedObject trackedObject;
+    RectF location;
+    float detectionConfidence;
+    int color;
+    String title;
+  }
+
+  private final List<TrackedRecognition> trackedObjects = new LinkedList<TrackedRecognition>();
+
+  private final Paint boxPaint = new Paint();
+
+  private final float textSizePx;
+  private final BorderedText borderedText;
+
+  private Matrix frameToCanvasMatrix;
+
+  private int frameWidth;
+  private int frameHeight;
+
+  private int sensorOrientation;
+  private Context context;
+
+  public MultiBoxTracker(final Context context) {
+    this.context = context;
+    for (final int color : COLORS) {
+      availableColors.add(color);
+    }
+
+    boxPaint.setColor(Color.RED);
+    boxPaint.setStyle(Style.STROKE);
+    boxPaint.setStrokeWidth(12.0f);
+    boxPaint.setStrokeCap(Cap.ROUND);
+    boxPaint.setStrokeJoin(Join.ROUND);
+    boxPaint.setStrokeMiter(100);
+
+    textSizePx =
+        TypedValue.applyDimension(
+            TypedValue.COMPLEX_UNIT_DIP, TEXT_SIZE_DIP, context.getResources().getDisplayMetrics());
+    borderedText = new BorderedText(textSizePx);
+  }
+
+  private Matrix getFrameToCanvasMatrix() {
+    return frameToCanvasMatrix;
+  }
+
+  public synchronized void drawDebug(final Canvas canvas) {
+    final Paint textPaint = new Paint();
+    textPaint.setColor(Color.WHITE);
+    textPaint.setTextSize(60.0f);
+
+    final Paint boxPaint = new Paint();
+    boxPaint.setColor(Color.RED);
+    boxPaint.setAlpha(200);
+    boxPaint.setStyle(Style.STROKE);
+
+    for (final Pair<Float, RectF> detection : screenRects) {
+      final RectF rect = detection.second;
+      canvas.drawRect(rect, boxPaint);
+      canvas.drawText("" + detection.first, rect.left, rect.top, textPaint);
+      borderedText.drawText(canvas, rect.centerX(), rect.centerY(), "" + detection.first);
+    }
+
+    if (objectTracker == null) {
+      return;
+    }
+
+    // Draw correlations.
+    for (final TrackedRecognition recognition : trackedObjects) {
+      final ObjectTracker.TrackedObject trackedObject = recognition.trackedObject;
+
+      final RectF trackedPos = trackedObject.getTrackedPositionInPreviewFrame();
+
+      if (getFrameToCanvasMatrix().mapRect(trackedPos)) {
+        final String labelString = String.format("%.2f", trackedObject.getCurrentCorrelation());
+        borderedText.drawText(canvas, trackedPos.right, trackedPos.bottom, labelString);
+      }
+    }
+
+    final Matrix matrix = getFrameToCanvasMatrix();
+    objectTracker.drawDebug(canvas, matrix);
+  }
+
+  public synchronized void trackResults(
+      final List<Recognition> results, final byte[] frame, final long timestamp) {
+    logger.i("Processing %d results from %d", results.size(), timestamp);
+    processResults(timestamp, results, frame);
+  }
+
+  public synchronized void draw(final Canvas canvas) {
+    final boolean rotated = sensorOrientation % 180 == 90;
+    final float multiplier =
+        Math.min(canvas.getHeight() / (float) (rotated ? frameWidth : frameHeight),
+                 canvas.getWidth() / (float) (rotated ? frameHeight : frameWidth));
+    frameToCanvasMatrix =
+        ImageUtils.getTransformationMatrix(
+            frameWidth,
+            frameHeight,
+            (int) (multiplier * (rotated ? frameHeight : frameWidth)),
+            (int) (multiplier * (rotated ? frameWidth : frameHeight)),
+            sensorOrientation,
+            false);
+    for (final TrackedRecognition recognition : trackedObjects) {
+      final RectF trackedPos =
+          (objectTracker != null)
+              ? recognition.trackedObject.getTrackedPositionInPreviewFrame()
+              : new RectF(recognition.location);
+
+      getFrameToCanvasMatrix().mapRect(trackedPos);
+      boxPaint.setColor(recognition.color);
+
+      final float cornerSize = Math.min(trackedPos.width(), trackedPos.height()) / 8.0f;
+      canvas.drawRoundRect(trackedPos, cornerSize, cornerSize, boxPaint);
+
+      final String labelString =
+          !TextUtils.isEmpty(recognition.title)
+              ? String.format("%s %.2f", recognition.title, recognition.detectionConfidence)
+              : String.format("%.2f", recognition.detectionConfidence);
+      borderedText.drawText(canvas, trackedPos.left + cornerSize, trackedPos.bottom, labelString);
+    }
+  }
+
+  private boolean initialized = false;
+
+  public synchronized void onFrame(
+      final int w,
+      final int h,
+      final int rowStride,
+      final int sensorOrientation,
+      final byte[] frame,
+      final long timestamp) {
+    if (objectTracker == null && !initialized) {
+      ObjectTracker.clearInstance();
+
+      logger.i("Initializing ObjectTracker: %dx%d", w, h);
+      objectTracker = ObjectTracker.getInstance(w, h, rowStride, true);
+      frameWidth = w;
+      frameHeight = h;
+      this.sensorOrientation = sensorOrientation;
+      initialized = true;
+
+      if (objectTracker == null) {
+        String message =
+            "Object tracking support not found. "
+                + "See tensorflow/examples/android/README.md for details.";
+        Toast.makeText(context, message, Toast.LENGTH_LONG).show();
+        logger.e(message);
+      }
+    }
+
+    if (objectTracker == null) {
+      return;
+    }
+
+    objectTracker.nextFrame(frame, null, timestamp, null, true);
+
+    // Clean up any objects not worth tracking any more.
+    final LinkedList<TrackedRecognition> copyList =
+        new LinkedList<TrackedRecognition>(trackedObjects);
+    for (final TrackedRecognition recognition : copyList) {
+      final ObjectTracker.TrackedObject trackedObject = recognition.trackedObject;
+      final float correlation = trackedObject.getCurrentCorrelation();
+      if (correlation < MIN_CORRELATION) {
+        logger.v("Removing tracked object %s because NCC is %.2f", trackedObject, correlation);
+        trackedObject.stopTracking();
+        trackedObjects.remove(recognition);
+
+        availableColors.add(recognition.color);
+      }
+    }
+  }
+
+  private void processResults(
+      final long timestamp, final List<Recognition> results, final byte[] originalFrame) {
+    final List<Pair<Float, Recognition>> rectsToTrack = new LinkedList<Pair<Float, Recognition>>();
+
+    screenRects.clear();
+    final Matrix rgbFrameToScreen = new Matrix(getFrameToCanvasMatrix());
+
+    for (final Recognition result : results) {
+      if (result.getLocation() == null) {
+        continue;
+      }
+      final RectF detectionFrameRect = new RectF(result.getLocation());
+
+      final RectF detectionScreenRect = new RectF();
+      rgbFrameToScreen.mapRect(detectionScreenRect, detectionFrameRect);
+
+      logger.v(
+          "Result! Frame: " + result.getLocation() + " mapped to screen:" + detectionScreenRect);
+
+      screenRects.add(new Pair<Float, RectF>(result.getConfidence(), detectionScreenRect));
+
+      if (detectionFrameRect.width() < MIN_SIZE || detectionFrameRect.height() < MIN_SIZE) {
+        logger.w("Degenerate rectangle! " + detectionFrameRect);
+        continue;
+      }
+
+      rectsToTrack.add(new Pair<Float, Recognition>(result.getConfidence(), result));
+    }
+
+    if (rectsToTrack.isEmpty()) {
+      logger.v("Nothing to track, aborting.");
+      return;
+    }
+
+    if (objectTracker == null) {
+      trackedObjects.clear();
+      for (final Pair<Float, Recognition> potential : rectsToTrack) {
+        final TrackedRecognition trackedRecognition = new TrackedRecognition();
+        trackedRecognition.detectionConfidence = potential.first;
+        trackedRecognition.location = new RectF(potential.second.getLocation());
+        trackedRecognition.trackedObject = null;
+        trackedRecognition.title = potential.second.getTitle();
+        trackedRecognition.color = COLORS[trackedObjects.size()];
+        trackedObjects.add(trackedRecognition);
+
+        if (trackedObjects.size() >= COLORS.length) {
+          break;
+        }
+      }
+      return;
+    }
+
+    logger.i("%d rects to track", rectsToTrack.size());
+    for (final Pair<Float, Recognition> potential : rectsToTrack) {
+      handleDetection(originalFrame, timestamp, potential);
+    }
+  }
+
+  private void handleDetection(
+      final byte[] frameCopy, final long timestamp, final Pair<Float, Recognition> potential) {
+    final ObjectTracker.TrackedObject potentialObject =
+        objectTracker.trackObject(potential.second.getLocation(), timestamp, frameCopy);
+
+    final float potentialCorrelation = potentialObject.getCurrentCorrelation();
+    logger.v(
+        "Tracked object went from %s to %s with correlation %.2f",
+        potential.second, potentialObject.getTrackedPositionInPreviewFrame(), potentialCorrelation);
+
+    if (potentialCorrelation < MARGINAL_CORRELATION) {
+      logger.v("Correlation too low to begin tracking %s.", potentialObject);
+      potentialObject.stopTracking();
+      return;
+    }
+
+    final List<TrackedRecognition> removeList = new LinkedList<TrackedRecognition>();
+
+    float maxIntersect = 0.0f;
+
+    // This is the current tracked object whose color we will take. If left null we'll take the
+    // first one from the color queue.
+    TrackedRecognition recogToReplace = null;
+
+    // Look for intersections that will be overridden by this object or an intersection that would
+    // prevent this one from being placed.
+    for (final TrackedRecognition trackedRecognition : trackedObjects) {
+      final RectF a = trackedRecognition.trackedObject.getTrackedPositionInPreviewFrame();
+      final RectF b = potentialObject.getTrackedPositionInPreviewFrame();
+      final RectF intersection = new RectF();
+      final boolean intersects = intersection.setIntersect(a, b);
+
+      final float intersectArea = intersection.width() * intersection.height();
+      final float totalArea = a.width() * a.height() + b.width() * b.height() - intersectArea;
+      final float intersectOverUnion = intersectArea / totalArea;
+
+      // If there is an intersection with this currently tracked box above the maximum overlap
+      // percentage allowed, either the new recognition needs to be dismissed or the old
+      // recognition needs to be removed and possibly replaced with the new one.
+      if (intersects && intersectOverUnion > MAX_OVERLAP) {
+        if (potential.first < trackedRecognition.detectionConfidence
+            && trackedRecognition.trackedObject.getCurrentCorrelation() > MARGINAL_CORRELATION) {
+          // If track for the existing object is still going strong and the detection score was
+          // good, reject this new object.
+          potentialObject.stopTracking();
+          return;
+        } else {
+          removeList.add(trackedRecognition);
+
+          // Let the previously tracked object with max intersection amount donate its color to
+          // the new object.
+          if (intersectOverUnion > maxIntersect) {
+            maxIntersect = intersectOverUnion;
+            recogToReplace = trackedRecognition;
+          }
+        }
+      }
+    }
+
+    // If we're already tracking the max object and no intersections were found to bump off,
+    // pick the worst current tracked object to remove, if it's also worse than this candidate
+    // object.
+    if (availableColors.isEmpty() && removeList.isEmpty()) {
+      for (final TrackedRecognition candidate : trackedObjects) {
+        if (candidate.detectionConfidence < potential.first) {
+          if (recogToReplace == null
+              || candidate.detectionConfidence < recogToReplace.detectionConfidence) {
+            // Save it so that we use this color for the new object.
+            recogToReplace = candidate;
+          }
+        }
+      }
+      if (recogToReplace != null) {
+        logger.v("Found non-intersecting object to remove.");
+        removeList.add(recogToReplace);
+      } else {
+        logger.v("No non-intersecting object found to remove");
+      }
+    }
+
+    // Remove everything that got intersected.
+    for (final TrackedRecognition trackedRecognition : removeList) {
+      logger.v(
+          "Removing tracked object %s with detection confidence %.2f, correlation %.2f",
+          trackedRecognition.trackedObject,
+          trackedRecognition.detectionConfidence,
+          trackedRecognition.trackedObject.getCurrentCorrelation());
+      trackedRecognition.trackedObject.stopTracking();
+      trackedObjects.remove(trackedRecognition);
+      if (trackedRecognition != recogToReplace) {
+        availableColors.add(trackedRecognition.color);
+      }
+    }
+
+    if (recogToReplace == null && availableColors.isEmpty()) {
+      logger.e("No room to track this object, aborting.");
+      potentialObject.stopTracking();
+      return;
+    }
+
+    // Finally safe to say we can track this object.
+    logger.v(
+        "Tracking object %s (%s) with detection confidence %.2f at position %s",
+        potentialObject,
+        potential.second.getTitle(),
+        potential.first,
+        potential.second.getLocation());
+    final TrackedRecognition trackedRecognition = new TrackedRecognition();
+    trackedRecognition.detectionConfidence = potential.first;
+    trackedRecognition.trackedObject = potentialObject;
+    trackedRecognition.title = potential.second.getTitle();
+
+    // Use the color from a replaced object before taking one from the color queue.
+    trackedRecognition.color =
+        recogToReplace != null ? recogToReplace.color : availableColors.poll();
+    trackedObjects.add(trackedRecognition);
+  }
+}
diff --git a/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/ObjectTracker.java b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/ObjectTracker.java
new file mode 100644
index 0000000000..8b4248d8fb
--- /dev/null
+++ b/tensorflow/contrib/lite/examples/android/src/org/tensorflow/demo/tracking/ObjectTracker.java
@@ -0,0 +1,661 @@
+/* Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+package org.tensorflow.demo.tracking;
+
+import android.graphics.Canvas;
+import android.graphics.Color;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import android.graphics.PointF;
+import android.graphics.RectF;
+import android.graphics.Typeface;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Vector;
+import javax.microedition.khronos.opengles.GL10;
+import org.tensorflow.demo.env.Logger;
+import org.tensorflow.demo.env.Size;
+
+/**
+ * True object detector/tracker class that tracks objects across consecutive preview frames.
+ * It provides a simplified Java interface to the analogous native object defined by
+ * jni/client_vision/tracking/object_tracker.*.
+ *
+ * Currently, the ObjectTracker is a singleton due to native code restrictions, and so must
+ * be allocated by ObjectTracker.getInstance(). In addition, release() should be called
+ * as soon as the ObjectTracker is no longer needed, and before a new one is created.
+ *
+ * nextFrame() should be called as new frames become available, preferably as often as possible.
+ *
+ * After allocation, new TrackedObjects may be instantiated via trackObject(). TrackedObjects
+ * are associated with the ObjectTracker that created them, and are only valid while that
+ * ObjectTracker still exists.
+ */
+public class ObjectTracker {
+  private static final Logger LOGGER = new Logger();
+
+  private static boolean libraryFound = false;
+
+  static {
+    try {
+      System.loadLibrary("tensorflow_demo");
+      libraryFound = true;
+    } catch (UnsatisfiedLinkError e) {
+      LOGGER.e("libtensorflow_demo.so not found, tracking unavailable");
+    }
+  }
+
+  private static final boolean DRAW_TEXT = false;
+
+  /**
+   * How many history points to keep track of and draw in the red history line.
+   */
+  private static final int MAX_DEBUG_HISTORY_SIZE = 30;
+
+  /**
+   * How many frames of optical flow deltas to record.
+   * TODO(andrewharp): Push this down to the native level so it can be polled
+   * efficiently into a an array for upload, instead of keeping a duplicate
+   * copy in Java.
+   */
+  private static final int MAX_FRAME_HISTORY_SIZE = 200;
+
+  private static final int DOWNSAMPLE_FACTOR = 2;
+
+  private final byte[] downsampledFrame;
+
+  protected static ObjectTracker instance;
+
+  private final Map<String, TrackedObject> trackedObjects;
+
+  private long lastTimestamp;
+
+  private FrameChange lastKeypoints;
+
+  private final Vector<PointF> debugHistory;
+
+  private final LinkedList<TimestampedDeltas> timestampedDeltas;
+
+  protected final int frameWidth;
+  protected final int frameHeight;
+  private final int rowStride;
+  protected final boolean alwaysTrack;
+
+  private static class TimestampedDeltas {
+    final long timestamp;
+    final byte[] deltas;
+
+    public TimestampedDeltas(final long timestamp, final byte[] deltas) {
+      this.timestamp = timestamp;
+      this.deltas = deltas;
+    }
+  }
+
+  /**
+   * A simple class that records keypoint information, which includes
+   * local location, score and type. This will be used in calculating
+   * FrameChange.
+   */
+  public static class Keypoint {
+    public final float x;
+    public final float y;
+    public final float score;
+    public final int type;
+
+    public Keypoint(final float x, final float y) {
+      this.x = x;
+      this.y = y;
+      this.score = 0;
+      this.type = -1;
+    }
+
+    public Keypoint(final float x, final float y, final float score, final int type) {
+      this.x = x;
+      this.y = y;
+      this.score = score;
+      this.type = type;
+    }
+
+    Keypoint delta(final Keypoint other) {
+      return new Keypoint(this.x - other.x, this.y - other.y);
+    }
+  }
+
+  /**
+   * A simple class that could calculate Keypoint delta.
+   * This class will be used in calculating frame translation delta
+   * for optical flow.
+   */
+  public static class PointChange {
+    public final Keypoint keypointA;
+    public final Keypoint keypointB;
+    Keypoint pointDelta;
+    private final boolean wasFound;
+
+    public PointChange(final float x1, final float y1,
+                       final float x2, final float y2,
+                       final float score, final int type,
+                       final boolean wasFound) {
+      this.wasFound = wasFound;
+
+      keypointA = new Keypoint(x1, y1, score, type);
+      keypointB = new Keypoint(x2, y2);
+    }
+
+    public Keypoint getDelta() {
+      if (pointDelta == null) {
+        pointDelta = keypointB.delta(keypointA);
+      }
+      return pointDelta;
+    }
+  }
+
+  /** A class that records a timestamped frame translation delta for optical flow. */
+  public static class FrameChange {
+    public static final int KEYPOINT_STEP = 7;
+
+    public final Vector<PointChange> pointDeltas;
+
+    private final float minScore;
+    private final float maxScore;
+
+    public FrameChange(final float[] framePoints) {
+      float minScore = 100.0f;
+      float maxScore = -100.0f;
+
+      pointDeltas = new Vector<PointChange>(framePoints.length / KEYPOINT_STEP);
+
+      for (int i = 0; i < framePoints.length; i += KEYPOINT_STEP) {
+        final float x1 = framePoints[i + 0] * DOWNSAMPLE_FACTOR;
+        final float y1 = framePoints[i + 1] * DOWNSAMPLE_FACTOR;
+
+        final boolean wasFound = framePoints[i + 2] > 0.0f;
+
+        final float x2 = framePoints[i + 3] * DOWNSAMPLE_FACTOR;
+        final float y2 = framePoints[i + 4] * DOWNSAMPLE_FACTOR;
+        final float score = framePoints[i + 5];
+        final int type = (int) framePoints[i + 6];
+
+        minScore = Math.min(minScore, score);
+        maxScore = Math.max(maxScore, score);
+
+        pointDeltas.add(new PointChange(x1, y1, x2, y2, score, type, wasFound));
+      }
+
+      this.minScore = minScore;
+      this.maxScore = maxScore;
+    }
+  }
+
+  public static synchronized ObjectTracker getInstance(
+      final int frameWidth, final int frameHeight, final int rowStride, final boolean alwaysTrack) {
+    if (!libraryFound) {
+      LOGGER.e(
+          "Native object tracking support not found. "
+              + "See tensorflow/examples/android/README.md for details.");
+      return null;
+    }
+
+    if (instance == null) {
+      instance = new ObjectTracker(frameWidth, frameHeight, rowStride, alwaysTrack);
+      instance.init();
+    } else {
+      throw new RuntimeException(
+          "Tried to create a new objectracker before releasing the old one!");
+    }
+    return instance;
+  }
+
+  public static synchronized void clearInstance() {
+    if (instance != null) {
+      instance.release();
+    }
+  }
+
+  protected ObjectTracker(
+      final int frameWidth, final int frameHeight, final int rowStride, final boolean alwaysTrack) {
+    this.frameWidth = frameWidth;
+    this.frameHeight = frameHeight;
+    this.rowStride = rowStride;
+    this.alwaysTrack = alwaysTrack;
+    this.timestampedDeltas = new LinkedList<TimestampedDeltas>();
+
+    trackedObjects = new HashMap<String, TrackedObject>();
+
+    debugHistory = new Vector<PointF>(MAX_DEBUG_HISTORY_SIZE);
+
+    downsampledFrame =
+        new byte
+            [(frameWidth + DOWNSAMPLE_FACTOR - 1)
+                / DOWNSAMPLE_FACTOR
+                * (frameWidth + DOWNSAMPLE_FACTOR - 1)
+                / DOWNSAMPLE_FACTOR];
+  }
+
+  protected void init() {
+    // The native tracker never sees the full frame, so pre-scale dimensions
+    // by the downsample factor.
+    initNative(frameWidth / DOWNSAMPLE_FACTOR, frameHeight / DOWNSAMPLE_FACTOR, alwaysTrack);
+  }
+
+  private final float[] matrixValues = new float[9];
+
+  private long downsampledTimestamp;
+
+  @SuppressWarnings("unused")
+  public synchronized void drawOverlay(final GL10 gl,
+      final Size cameraViewSize, final Matrix matrix) {
+    final Matrix tempMatrix = new Matrix(matrix);
+    tempMatrix.preScale(DOWNSAMPLE_FACTOR, DOWNSAMPLE_FACTOR);
+    tempMatrix.getValues(matrixValues);
+    drawNative(cameraViewSize.width, cameraViewSize.height, matrixValues);
+  }
+
+  public synchronized void nextFrame(
+      final byte[] frameData, final byte[] uvData,
+      final long timestamp, final float[] transformationMatrix,
+      final boolean updateDebugInfo) {
+    if (downsampledTimestamp != timestamp) {
+      ObjectTracker.downsampleImageNative(
+          frameWidth, frameHeight, rowStride, frameData, DOWNSAMPLE_FACTOR, downsampledFrame);
+      downsampledTimestamp = timestamp;
+    }
+
+    // Do Lucas Kanade using the fullframe initializer.
+    nextFrameNative(downsampledFrame, uvData, timestamp, transformationMatrix);
+
+    timestampedDeltas.add(new TimestampedDeltas(timestamp, getKeypointsPacked(DOWNSAMPLE_FACTOR)));
+    while (timestampedDeltas.size() > MAX_FRAME_HISTORY_SIZE) {
+      timestampedDeltas.removeFirst();
+    }
+
+    for (final TrackedObject trackedObject : trackedObjects.values()) {
+      trackedObject.updateTrackedPosition();
+    }
+
+    if (updateDebugInfo) {
+      updateDebugHistory();
+    }
+
+    lastTimestamp = timestamp;
+  }
+
+  public synchronized void release() {
+    releaseMemoryNative();
+    synchronized (ObjectTracker.class) {
+      instance = null;
+    }
+  }
+
+  private void drawHistoryDebug(final Canvas canvas) {
+    drawHistoryPoint(
+        canvas, frameWidth * DOWNSAMPLE_FACTOR / 2, frameHeight * DOWNSAMPLE_FACTOR / 2);
+  }
+
+  private void drawHistoryPoint(final Canvas canvas, final float startX, final float startY) {
+    final Paint p = new Paint();
+    p.setAntiAlias(false);
+    p.setTypeface(Typeface.SERIF);
+
+    p.setColor(Color.RED);
+    p.setStrokeWidth(2.0f);
+
+    // Draw the center circle.
+    p.setColor(Color.GREEN);
+    canvas.drawCircle(startX, startY, 3.0f, p);
+
+    p.setColor(Color.RED);
+
+    // Iterate through in backwards order.
+    synchronized (debugHistory) {
+      final int numPoints = debugHistory.size();
+      float lastX = startX;
+      float lastY = startY;
+      for (int keypointNum = 0; keypointNum < numPoints; ++keypointNum) {
+        final PointF delta = debugHistory.get(numPoints - keypointNum - 1);
+        final float newX = lastX + delta.x;
+        final float newY = lastY + delta.y;
+        canvas.drawLine(lastX, lastY, newX, newY, p);
+        lastX = newX;
+        lastY = newY;
+      }
+    }
+  }
+
+  private static int floatToChar(final float value) {
+    return Math.max(0, Math.min((int) (value * 255.999f), 255));
+  }
+
+  private void drawKeypointsDebug(final Canvas canvas) {
+    final Paint p = new Paint();
+    if (lastKeypoints == null) {
+      return;
+    }
+    final int keypointSize = 3;
+
+    final float minScore = lastKeypoints.minScore;
+    final float maxScore = lastKeypoints.maxScore;
+
+    for (final PointChange keypoint : lastKeypoints.pointDeltas) {
+      if (keypoint.wasFound) {
+        final int r =
+            floatToChar((keypoint.keypointA.score - minScore) / (maxScore - minScore));
+        final int b =
+            floatToChar(1.0f - (keypoint.keypointA.score - minScore) / (maxScore - minScore));
+
+        final int color = 0xFF000000 | (r << 16) | b;
+        p.setColor(color);
+
+        final float[] screenPoints = {keypoint.keypointA.x, keypoint.keypointA.y,
+                                      keypoint.keypointB.x, keypoint.keypointB.y};
+        canvas.drawRect(screenPoints[2] - keypointSize,
+                        screenPoints[3] - keypointSize,
+                        screenPoints[2] + keypointSize,
+                        screenPoints[3] + keypointSize, p);
+        p.setColor(Color.CYAN);
+        canvas.drawLine(screenPoints[2], screenPoints[3],
+                        screenPoints[0], screenPoints[1], p);
+
+        if (DRAW_TEXT) {
+          p.setColor(Color.WHITE);
+          canvas.drawText(keypoint.keypointA.type + ": " + keypoint.keypointA.score,
+              keypoint.keypointA.x, keypoint.keypointA.y, p);
+        }
+      } else {
+        p.setColor(Color.YELLOW);
+        final float[] screenPoint = {keypoint.keypointA.x, keypoint.keypointA.y};
+        canvas.drawCircle(screenPoint[0], screenPoint[1], 5.0f, p);
+      }
+    }
+  }
+
+  private synchronized PointF getAccumulatedDelta(final long timestamp, final float positionX,
+      final float positionY, final float radius) {
+    final RectF currPosition = getCurrentPosition(timestamp,
+        new RectF(positionX - radius, positionY - radius, positionX + radius, positionY + radius));
+    return new PointF(currPosition.centerX() - positionX, currPosition.centerY() - positionY);
+  }
+
+  private synchronized RectF getCurrentPosition(final long timestamp, final RectF
+      oldPosition) {
+    final RectF downscaledFrameRect = downscaleRect(oldPosition);
+
+    final float[] delta = new float[4];
+    getCurrentPositionNative(timestamp, downscaledFrameRect.left, downscaledFrameRect.top,
+        downscaledFrameRect.right, downscaledFrameRect.bottom, delta);
+
+    final RectF newPosition = new RectF(delta[0], delta[1], delta[2], delta[3]);
+
+    return upscaleRect(newPosition);
+  }
+
+  private void updateDebugHistory() {
+    lastKeypoints = new FrameChange(getKeypointsNative(false));
+
+    if (lastTimestamp == 0) {
+      return;
+    }
+
+    final PointF delta =
+        getAccumulatedDelta(
+            lastTimestamp, frameWidth / DOWNSAMPLE_FACTOR, frameHeight / DOWNSAMPLE_FACTOR, 100);
+
+    synchronized (debugHistory) {
+      debugHistory.add(delta);
+
+      while (debugHistory.size() > MAX_DEBUG_HISTORY_SIZE) {
+        debugHistory.remove(0);
+      }
+    }
+  }
+
+  public synchronized void drawDebug(final Canvas canvas, final Matrix frameToCanvas) {
+    canvas.save();
+    canvas.setMatrix(frameToCanvas);
+
+    drawHistoryDebug(canvas);
+    drawKeypointsDebug(canvas);
+
+    canvas.restore();
+  }
+
+  public Vector<String> getDebugText() {
+    final Vector<String> lines = new Vector<String>();
+
+    if (lastKeypoints != null) {
+      lines.add("Num keypoints " + lastKeypoints.pointDeltas.size());
+      lines.add("Min score: " + lastKeypoints.minScore);
+      lines.add("Max score: " + lastKeypoints.maxScore);
+    }
+
+    return lines;
+  }
+
+  public synchronized List<byte[]> pollAccumulatedFlowData(final long endFrameTime) {
+    final List<byte[]> frameDeltas = new ArrayList<byte[]>();
+    while (timestampedDeltas.size() > 0) {
+      final TimestampedDeltas currentDeltas = timestampedDeltas.peek();
+      if (currentDeltas.timestamp <= endFrameTime) {
+        frameDeltas.add(currentDeltas.deltas);
+        timestampedDeltas.removeFirst();
+      } else {
+        break;
+      }
+    }
+
+    return frameDeltas;
+  }
+
+  private RectF downscaleRect(final RectF fullFrameRect) {
+    return new RectF(
+        fullFrameRect.left / DOWNSAMPLE_FACTOR,
+        fullFrameRect.top / DOWNSAMPLE_FACTOR,
+        fullFrameRect.right / DOWNSAMPLE_FACTOR,
+        fullFrameRect.bottom / DOWNSAMPLE_FACTOR);
+  }
+
+  private RectF upscaleRect(final RectF downsampledFrameRect) {
+    return new RectF(
+        downsampledFrameRect.left * DOWNSAMPLE_FACTOR,
+        downsampledFrameRect.top * DOWNSAMPLE_FACTOR,
+        downsampledFrameRect.right * DOWNSAMPLE_FACTOR,
+        downsampledFrameRect.bottom * DOWNSAMPLE_FACTOR);
+  }
+
+  /**
+   * A TrackedObject represents a native TrackedObject, and provides access to the
+   * relevant native tracking information available after every frame update. They may
+   * be safely passed around and accessed externally, but will become invalid after
+   * stopTracking() is called or the related creating ObjectTracker is deactivated.
+   *
+   * @author andrewharp@google.com (Andrew Harp)
+   */
+  public class TrackedObject {
+    private final String id;
+
+    private long lastExternalPositionTime;
+
+    private RectF lastTrackedPosition;
+    private boolean visibleInLastFrame;
+
+    private boolean isDead;
+
+    TrackedObject(final RectF position, final long timestamp, final byte[] data) {
+      isDead = false;
+
+      id = Integer.toString(this.hashCode());
+
+      lastExternalPositionTime = timestamp;
+
+      synchronized (ObjectTracker.this) {
+        registerInitialAppearance(position, data);
+        setPreviousPosition(position, timestamp);
+        trackedObjects.put(id, this);
+      }
+    }
+
+    public void stopTracking() {
+      checkValidObject();
+
+      synchronized (ObjectTracker.this) {
+        isDead = true;
+        forgetNative(id);
+        trackedObjects.remove(id);
+      }
+    }
+
+    public float getCurrentCorrelation() {
+      checkValidObject();
+      return ObjectTracker.this.getCurrentCorrelation(id);
+    }
+
+    void registerInitialAppearance(final RectF position, final byte[] data) {
+      final RectF externalPosition = downscaleRect(position);
+      registerNewObjectWithAppearanceNative(id,
+            externalPosition.left, externalPosition.top,
+            externalPosition.right, externalPosition.bottom,
+            data);
+    }
+
+    synchronized void setPreviousPosition(final RectF position, final long timestamp) {
+      checkValidObject();
+      synchronized (ObjectTracker.this) {
+        if (lastExternalPositionTime > timestamp) {
+          LOGGER.w("Tried to use older position time!");
+          return;
+        }
+        final RectF externalPosition = downscaleRect(position);
+        lastExternalPositionTime = timestamp;
+
+        setPreviousPositionNative(id,
+            externalPosition.left, externalPosition.top,
+            externalPosition.right, externalPosition.bottom,
+            lastExternalPositionTime);
+
+        updateTrackedPosition();
+      }
+    }
+
+    void setCurrentPosition(final RectF position) {
+      checkValidObject();
+      final RectF downsampledPosition = downscaleRect(position);
+      synchronized (ObjectTracker.this) {
+        setCurrentPositionNative(id,
+            downsampledPosition.left, downsampledPosition.top,
+            downsampledPosition.right, downsampledPosition.bottom);
+      }
+    }
+
+    private synchronized void updateTrackedPosition() {
+      checkValidObject();
+
+      final float[] delta = new float[4];
+      getTrackedPositionNative(id, delta);
+      lastTrackedPosition = new RectF(delta[0], delta[1], delta[2], delta[3]);
+
+      visibleInLastFrame = isObjectVisible(id);
+    }
+
+    public synchronized RectF getTrackedPositionInPreviewFrame() {
+      checkValidObject();
+
+      if (lastTrackedPosition == null) {
+        return null;
+      }
+      return upscaleRect(lastTrackedPosition);
+    }
+
+    synchronized long getLastExternalPositionTime() {
+      return lastExternalPositionTime;
+    }
+
+    public synchronized boolean visibleInLastPreviewFrame() {
+      return visibleInLastFrame;
+    }
+
+    private void checkValidObject() {
+      if (isDead) {
+        throw new RuntimeException("TrackedObject already removed from tracking!");
+      } else if (ObjectTracker.this != instance) {
+        throw new RuntimeException("TrackedObject created with another ObjectTracker!");
+      }
+    }
+  }
+
+  public synchronized TrackedObject trackObject(
+      final RectF position, final long timestamp, final byte[] frameData) {
+    if (downsampledTimestamp != timestamp) {
+      ObjectTracker.downsampleImageNative(
+          frameWidth, frameHeight, rowStride, frameData, DOWNSAMPLE_FACTOR, downsampledFrame);
+      downsampledTimestamp = timestamp;
+    }
+    return new TrackedObject(position, timestamp, downsampledFrame);
+  }
+
+  public synchronized TrackedObject trackObject(final RectF position, final byte[] frameData) {
+    return new TrackedObject(position, lastTimestamp, frameData);
+  }
+
+  /** ********************* NATIVE CODE ************************************ */
+
+  /** This will contain an opaque pointer to the native ObjectTracker */
+  private long nativeObjectTracker;
+
+  private native void initNative(int imageWidth, int imageHeight, boolean alwaysTrack);
+
+  protected native void registerNewObjectWithAppearanceNative(
+      String objectId, float x1, float y1, float x2, float y2, byte[] data);
+
+  protected native void setPreviousPositionNative(
+      String objectId, float x1, float y1, float x2, float y2, long timestamp);
+
+  protected native void setCurrentPositionNative(
+      String objectId, float x1, float y1, float x2, float y2);
+
+  protected native void forgetNative(String key);
+
+  protected native String getModelIdNative(String key);
+
+  protected native boolean haveObject(String key);
+  protected native boolean isObjectVisible(String key);
+  protected native float getCurrentCorrelation(String key);
+
+  protected native float getMatchScore(String key);
+
+  protected native void getTrackedPositionNative(String key, float[] points);
+
+  protected native void nextFrameNative(
+      byte[] frameData, byte[] uvData, long timestamp, float[] frameAlignMatrix);
+
+  protected native void releaseMemoryNative();
+
+  protected native void getCurrentPositionNative(long timestamp,
+      final float positionX1, final float positionY1,
+      final float positionX2, final float positionY2,
+      final float[] delta);
+
+  protected native byte[] getKeypointsPacked(float scaleFactor);
+
+  protected native float[] getKeypointsNative(boolean onlyReturnCorrespondingKeypoints);
+
+  protected native void drawNative(int viewWidth, int viewHeight, float[] frameToCanvas);
+
+  protected static native void downsampleImageNative(
+      int width, int height, int rowStride, byte[] input, int factor, byte[] output);
+}
diff --git a/tensorflow/tools/ci_build/ci_sanity.sh b/tensorflow/tools/ci_build/ci_sanity.sh
index aeac085d30..9627475d84 100755
--- a/tensorflow/tools/ci_build/ci_sanity.sh
+++ b/tensorflow/tools/ci_build/ci_sanity.sh
@@ -431,7 +431,8 @@ cmd_status(){
 # out by default in TF WORKSPACE file.
 do_bazel_nobuild() {
   BUILD_TARGET="//tensorflow/..."
-  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/contrib/lite/java/demo/app/src/main/..."
+  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/contrib/lite/java/demo/app/..."
+  BUILD_TARGET="${BUILD_TARGET} -//tensorflow/contrib/lite/examples/android/..."
   BUILD_TARGET="${BUILD_TARGET} -//tensorflow/contrib/lite/schema/..."
   BUILD_CMD="bazel build --nobuild ${BAZEL_FLAGS} -- ${BUILD_TARGET}"
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index ac6380dd3e..fe6b9407d6 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -706,6 +706,26 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       build_file = clean_dep("//third_party:tflite_mobilenet.BUILD"),
   )
 
+  tf_http_archive(
+      name = "tflite_mobilenet_ssd",
+      sha256 = "767057f2837a46d97882734b03428e8dd640b93236052b312b2f0e45613c1cf0",
+      urls = [
+          "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_ssd_tflite_v1.zip",
+          "https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_ssd_tflite_v1.zip",
+      ],
+      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
+  )
+
+  tf_http_archive(
+      name = "tflite_conv_actions_frozen",
+      sha256 = "d947b38cba389b5e2d0bfc3ea6cc49c784e187b41a071387b3742d1acac7691e",
+      urls = [
+          "https://mirror.bazel.build/storage.googleapis.com/download.tensorflow.org/models/tflite/conv_actions_tflite.zip",
+          "https://storage.googleapis.com/download.tensorflow.org/models/tflite/conv_actions_tflite.zip",
+      ],
+      build_file = str(Label("//third_party:tflite_mobilenet.BUILD")),
+  )
+
   tf_http_archive(
       name = "tflite_smartreply",
       sha256 = "8980151b85a87a9c1a3bb1ed4748119e4a85abd3cb5744d83da4d4bd0fbeef7c",
-- 
GitLab


From a8a95bf470877a9f70dfc7e587b30071ebc4ca5d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 18:11:09 -0700
Subject: [PATCH 724/960] Add distribution support for incrementing the global
 step. Don't require Dataset as input to eval and predict even when using a
 DistributionStrategy.

PiperOrigin-RevId: 191017191
---
 .../python/estimator/canned/baseline_test.py  | 10 +++++-----
 .../python/estimator/canned/boosted_trees.py  |  6 +++---
 .../estimator/canned/dnn_linear_combined.py   |  5 ++---
 .../estimator/canned/dnn_testing_utils.py     |  6 +++---
 .../estimator/canned/linear_testing_utils.py  | 10 +++++-----
 tensorflow/python/estimator/estimator.py      |  3 ++-
 tensorflow/python/training/distribute.py      | 20 +++++++++++++++++++
 7 files changed, 40 insertions(+), 20 deletions(-)

diff --git a/tensorflow/python/estimator/canned/baseline_test.py b/tensorflow/python/estimator/canned/baseline_test.py
index 7833df2052..7bf2e62da9 100644
--- a/tensorflow/python/estimator/canned/baseline_test.py
+++ b/tensorflow/python/estimator/canned/baseline_test.py
@@ -42,13 +42,13 @@ from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import input as input_lib
 from tensorflow.python.training import optimizer
 from tensorflow.python.training import queue_runner
@@ -482,7 +482,7 @@ class BaselineRegressorTrainingTest(test.TestCase):
       self.assertEquals(0, loss.shape.ndims)
       if expected_loss is None:
         if global_step is not None:
-          return state_ops.assign_add(global_step, 1).op
+          return distribute_lib.increment_var(global_step)
         return control_flow_ops.no_op()
       assert_loss = assert_close(
           math_ops.to_float(expected_loss, name='expected'),
@@ -490,7 +490,7 @@ class BaselineRegressorTrainingTest(test.TestCase):
           name='assert_loss')
       with ops.control_dependencies((assert_loss,)):
         if global_step is not None:
-          return state_ops.assign_add(global_step, 1).op
+          return distribute_lib.increment_var(global_step)
         return control_flow_ops.no_op()
 
     mock_optimizer = test.mock.NonCallableMock(
@@ -685,13 +685,13 @@ class BaselineClassifierTrainingTest(test.TestCase):
       # Verify loss. We can't check the value directly, so we add an assert op.
       self.assertEquals(0, loss.shape.ndims)
       if expected_loss is None:
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
       assert_loss = assert_close(
           math_ops.to_float(expected_loss, name='expected'),
           loss,
           name='assert_loss')
       with ops.control_dependencies((assert_loss,)):
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
 
     mock_optimizer = test.mock.NonCallableMock(
         spec=optimizer.Optimizer,
diff --git a/tensorflow/python/estimator/canned/boosted_trees.py b/tensorflow/python/estimator/canned/boosted_trees.py
index 7f1bcc31f2..500ea03ea7 100644
--- a/tensorflow/python/estimator/canned/boosted_trees.py
+++ b/tensorflow/python/estimator/canned/boosted_trees.py
@@ -32,10 +32,10 @@ from tensorflow.python.ops import data_flow_ops
 from tensorflow.python.ops import gradients_impl
 from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.summary import summary
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import session_run_hook
 from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
@@ -425,7 +425,7 @@ def _bt_model_fn(
         return grow_op
 
       if train_in_memory and is_single_machine:
-        train_op.append(state_ops.assign_add(global_step, 1))
+        train_op.append(distribute_lib.increment_var(global_step))
         train_op.append(grow_tree_from_stats_summaries(stats_summary_list))
       else:
         summary_accumulator = data_flow_ops.ConditionalAccumulator(
@@ -445,7 +445,7 @@ def _bt_model_fn(
           return grow_op
 
         with ops.control_dependencies([apply_grad]):
-          train_op.append(state_ops.assign_add(global_step, 1))
+          train_op.append(distribute_lib.increment_var(global_step))
           if config.is_chief:
             train_op.append(
                 control_flow_ops.cond(
diff --git a/tensorflow/python/estimator/canned/dnn_linear_combined.py b/tensorflow/python/estimator/canned/dnn_linear_combined.py
index 6d0fb96057..f47706db2f 100644
--- a/tensorflow/python/estimator/canned/dnn_linear_combined.py
+++ b/tensorflow/python/estimator/canned/dnn_linear_combined.py
@@ -31,10 +31,10 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import partitioned_variables
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses
 from tensorflow.python.summary import summary
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import sync_replicas_optimizer
 from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
@@ -215,8 +215,7 @@ def _dnn_linear_combined_model_fn(features,
 
     train_op = control_flow_ops.group(*train_ops)
     with ops.control_dependencies([train_op]):
-      with ops.colocate_with(global_step):
-        return state_ops.assign_add(global_step, 1)
+      return distribute_lib.increment_var(global_step)
 
   return head.create_estimator_spec(
       features=features,
diff --git a/tensorflow/python/estimator/canned/dnn_testing_utils.py b/tensorflow/python/estimator/canned/dnn_testing_utils.py
index 44545c058c..62b13c3200 100644
--- a/tensorflow/python/estimator/canned/dnn_testing_utils.py
+++ b/tensorflow/python/estimator/canned/dnn_testing_utils.py
@@ -44,13 +44,13 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import partitioned_variables
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import test
 from tensorflow.python.summary import summary as summary_lib
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import monitored_session
 from tensorflow.python.training import optimizer as optimizer_lib
@@ -196,7 +196,7 @@ def mock_optimizer(testcase, hidden_units, expected_loss=None):
     testcase.assertEquals(0, loss.shape.ndims)
     if expected_loss is None:
       if global_step is not None:
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
       return control_flow_ops.no_op()
     assert_loss = assert_close(
         math_ops.to_float(expected_loss, name='expected'),
@@ -204,7 +204,7 @@ def mock_optimizer(testcase, hidden_units, expected_loss=None):
         name='assert_loss')
     with ops.control_dependencies((assert_loss,)):
       if global_step is not None:
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
       return control_flow_ops.no_op()
 
   optimizer_mock = test.mock.NonCallableMagicMock(
diff --git a/tensorflow/python/estimator/canned/linear_testing_utils.py b/tensorflow/python/estimator/canned/linear_testing_utils.py
index da3ce86999..0e6436b421 100644
--- a/tensorflow/python/estimator/canned/linear_testing_utils.py
+++ b/tensorflow/python/estimator/canned/linear_testing_utils.py
@@ -47,13 +47,13 @@ from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import parsing_ops
 from tensorflow.python.ops import partitioned_variables
-from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
 from tensorflow.python.training import checkpoint_utils
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import gradient_descent
 from tensorflow.python.training import input as input_lib
 from tensorflow.python.training import optimizer as optimizer_lib
@@ -682,7 +682,7 @@ class BaseLinearRegressorTrainingTest(object):
       self.assertEquals(0, loss.shape.ndims)
       if expected_loss is None:
         if global_step is not None:
-          return state_ops.assign_add(global_step, 1).op
+          return distribute_lib.increment_var(global_step)
         return control_flow_ops.no_op()
       assert_loss = assert_close(
           math_ops.to_float(expected_loss, name='expected'),
@@ -690,7 +690,7 @@ class BaseLinearRegressorTrainingTest(object):
           name='assert_loss')
       with ops.control_dependencies((assert_loss,)):
         if global_step is not None:
-          return state_ops.assign_add(global_step, 1).op
+          return distribute_lib.increment_var(global_step)
         return control_flow_ops.no_op()
 
     mock_optimizer = test.mock.NonCallableMock(
@@ -905,13 +905,13 @@ class BaseLinearClassifierTrainingTest(object):
       # Verify loss. We can't check the value directly, so we add an assert op.
       self.assertEquals(0, loss.shape.ndims)
       if expected_loss is None:
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
       assert_loss = assert_close(
           math_ops.to_float(expected_loss, name='expected'),
           loss,
           name='assert_loss')
       with ops.control_dependencies((assert_loss,)):
-        return state_ops.assign_add(global_step, 1).op
+        return distribute_lib.increment_var(global_step)
 
     mock_optimizer = test.mock.NonCallableMock(
         spec=optimizer_lib.Optimizer,
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 2fe521b063..6ce935c730 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -693,7 +693,8 @@ class Estimator(object):
     # using any input is alright in that case. There is also a
     # has_dataset_or_queue_runner function that we may want to extend and use.
     if (self._distribution is not None and
-        not isinstance(result, dataset_ops.Dataset)):
+        not isinstance(result, dataset_ops.Dataset) and
+        mode == model_fn_lib.ModeKeys.TRAIN):
       raise ValueError('input_fn() must return a tf.data.Dataset when using a '
                        'DistributionStrategy.')
     input_hooks = []
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 899fda67fe..d51ef341a0 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -23,6 +23,8 @@ import threading
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import resource_variable_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses_impl
 from tensorflow.python.training import device_util
@@ -1166,6 +1168,24 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     raise RuntimeError("worker_device_index() method unsupported by "
                        "_DefaultDistributionStrategy.")
 
+# ------------------------------------------------------------------------------
+# Common operations
+
+
+def increment_var(v, amount=1):
+  """`v += amount`, distributed-aware version."""
+  def update(vu):
+    if isinstance(vu, resource_variable_ops.ResourceVariable):
+      return vu.assign_add(amount, read_value=False)
+    else:
+      return state_ops.assign_add(vu, amount)
+
+  def merge_fn(dist, vm):
+    return dist.group(dist.update(vm, update))
+
+  tower_context = get_tower_context()
+  return tower_context.merge_call(merge_fn, v)
+
 
 # ------------------------------------------------------------------------------
 # Singletons
-- 
GitLab


From ac39aec50ff359ae915d13a011fdb905b8c48af0 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Thu, 29 Mar 2018 18:15:34 -0700
Subject: [PATCH 725/960] Move the simple_estimator_example to the examples/
 dir.

PiperOrigin-RevId: 191017560
---
 tensorflow/contrib/cmake/python_modules.txt   |  1 +
 tensorflow/contrib/distribute/python/BUILD    | 15 ------
 .../contrib/distribute/python/examples/BUILD  | 19 ++++++++
 .../simple_estimator_example.py               | 48 ++++++++-----------
 4 files changed, 39 insertions(+), 44 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/examples/BUILD
 rename tensorflow/contrib/distribute/python/{ => examples}/simple_estimator_example.py (57%)

diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index b10538d6d6..02c456c199 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -162,6 +162,7 @@ tensorflow/contrib/decision_trees/proto
 tensorflow/contrib/deprecated
 tensorflow/contrib/distribute
 tensorflow/contrib/distribute/python
+tensorflow/contrib/distribute/python/examples
 tensorflow/contrib/distributions
 tensorflow/contrib/distributions/python
 tensorflow/contrib/distributions/python/ops
diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 4dfd3f7228..e805569a4d 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -340,21 +340,6 @@ py_test(
     ],
 )
 
-py_binary(
-    name = "simple_estimator_example",
-    srcs = ["simple_estimator_example.py"],
-    deps = [
-        ":mirrored_strategy",
-        "//tensorflow/python:array_ops",
-        "//tensorflow/python:constant_op",
-        "//tensorflow/python:layers",
-        "//tensorflow/python:training",
-        "//tensorflow/python/data/ops:dataset_ops",
-        "//tensorflow/python/estimator:estimator_py",
-        "//tensorflow/python/estimator:model_fn",
-    ],
-)
-
 py_library(
     name = "cross_tower_utils",
     srcs = ["cross_tower_utils.py"],
diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD
new file mode 100644
index 0000000000..27eb3c079d
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/examples/BUILD
@@ -0,0 +1,19 @@
+# Example TensorFlow models that use DistributionStrategy for training.
+
+package(
+    default_visibility = [
+        "//tensorflow:internal",
+    ],
+)
+
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+py_binary(
+    name = "simple_estimator_example",
+    srcs = ["simple_estimator_example.py"],
+    deps = [
+        "//tensorflow:tensorflow_py",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/simple_estimator_example.py b/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
similarity index 57%
rename from tensorflow/contrib/distribute/python/simple_estimator_example.py
rename to tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
index 7095d801ad..5d6e02b4b9 100644
--- a/tensorflow/contrib/distribute/python/simple_estimator_example.py
+++ b/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
@@ -20,63 +20,53 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-from tensorflow.contrib.distribute.python import mirrored_strategy
-from tensorflow.python.data.ops import dataset_ops
-from tensorflow.python.estimator import estimator as estimator_lib
-from tensorflow.python.estimator import model_fn as model_fn_lib
-from tensorflow.python.estimator import run_config
-from tensorflow.python.framework import constant_op
-from tensorflow.python.layers import core
-from tensorflow.python.ops import array_ops
-from tensorflow.python.platform import app
-from tensorflow.python.training import gradient_descent
-from tensorflow.python.training import training_util
+import tensorflow as tf
 
 
 def build_model_fn_optimizer():
   """Simple model_fn with optimizer."""
   # TODO(anjalisridhar): Move this inside the model_fn once OptimizerV2 is
   # done?
-  optimizer = gradient_descent.GradientDescentOptimizer(0.2)
+  optimizer = tf.train.GradientDescentOptimizer(0.2)
 
   def model_fn(features, labels, mode):  # pylint: disable=unused-argument
     """model_fn which uses a single unit Dense layer."""
     # You can also use the Flatten layer if you want to test a model without any
     # weights.
-    layer = core.Dense(1, use_bias=True)
+    layer = tf.layers.Dense(1, use_bias=True)
     logits = layer(features)
 
-    if mode == model_fn_lib.ModeKeys.PREDICT:
+    if mode == tf.estimator.ModeKeys.PREDICT:
       predictions = {"logits": logits}
-      return model_fn_lib.EstimatorSpec(mode, predictions=predictions)
+      return tf.estimator.EstimatorSpec(mode, predictions=predictions)
 
     def loss_fn():
-      y = array_ops.reshape(logits, []) - constant_op.constant(1.)
+      y = tf.reshape(logits, []) - tf.constant(1.)
       return y * y
 
-    if mode == model_fn_lib.ModeKeys.EVAL:
-      return model_fn_lib.EstimatorSpec(mode, loss=loss_fn())
+    if mode == tf.estimator.ModeKeys.EVAL:
+      return tf.estimator.EstimatorSpec(mode, loss=loss_fn())
 
-    assert mode == model_fn_lib.ModeKeys.TRAIN
+    assert mode == tf.estimator.ModeKeys.TRAIN
 
-    global_step = training_util.get_global_step()
+    global_step = tf.train.get_global_step()
     train_op = optimizer.minimize(loss_fn(), global_step=global_step)
-    return model_fn_lib.EstimatorSpec(mode, loss=loss_fn(), train_op=train_op)
+    return tf.estimator.EstimatorSpec(mode, loss=loss_fn(), train_op=train_op)
 
   return model_fn
 
 
 def main(_):
-  distribution = mirrored_strategy.MirroredStrategy(
+  distribution = tf.contrib.distribute.MirroredStrategy(
       ["/device:GPU:0", "/device:GPU:1"])
-  config = run_config.RunConfig(distribute=distribution)
+  config = tf.estimator.RunConfig(distribute=distribution)
 
   def input_fn():
-    features = dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
-    labels = dataset_ops.Dataset.from_tensors([1.]).repeat(10)
-    return dataset_ops.Dataset.zip((features, labels))
+    features = tf.data.Dataset.from_tensors([[1.]]).repeat(10)
+    labels = tf.data.Dataset.from_tensors([1.]).repeat(10)
+    return tf.data.Dataset.zip((features, labels))
 
-  estimator = estimator_lib.Estimator(
+  estimator = tf.estimator.Estimator(
       model_fn=build_model_fn_optimizer(), config=config)
   estimator.train(input_fn=input_fn, steps=10)
 
@@ -84,7 +74,7 @@ def main(_):
   print("Eval result: {}".format(eval_result))
 
   def predict_input_fn():
-    predict_features = dataset_ops.Dataset.from_tensors([[1.]]).repeat(10)
+    predict_features = tf.data.Dataset.from_tensors([[1.]]).repeat(10)
     return predict_features
 
   predictions = estimator.predict(input_fn=predict_input_fn)
@@ -94,4 +84,4 @@ def main(_):
 
 
 if __name__ == "__main__":
-  app.run(main)
+  tf.app.run()
-- 
GitLab


From 36c2d9a2808daea7ea84d2bc1e5eb9fcf06999f7 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Mar 2018 18:18:28 -0700
Subject: [PATCH 726/960] Getting rid of the threadpool from
 FunctionBufferingResource. It wasn't really serving much purpose apart from
 moving all the function logic execution onto those threads and making the
 destruction of the resource quite complicated.

PiperOrigin-RevId: 191017836
---
 .../data/kernels/prefetching_kernels.cc       | 30 ++-------------
 tensorflow/contrib/data/ops/dataset_ops.cc    |  2 -
 .../kernel_tests/prefetching_ops_test.py      |  1 -
 .../data/python/ops/prefetching_ops.py        |  5 +--
 .../distribute/python/prefetching_ops_v2.py   |  3 +-
 tensorflow/contrib/eager/python/datasets.py   |  1 -
 tensorflow/core/BUILD                         | 12 ++++++
 .../core/common_runtime/direct_session.cc     | 15 +-------
 .../core/common_runtime/eager/context.cc      |  9 +++--
 .../core/common_runtime/eager/context.h       |  3 ++
 tensorflow/core/common_runtime/function.cc    |  9 +++--
 .../core/common_runtime/process_util.cc       | 14 +++++++
 tensorflow/core/common_runtime/process_util.h |  7 ++++
 .../core/common_runtime/process_util_test.cc  | 38 +++++++++++++++++++
 14 files changed, 92 insertions(+), 57 deletions(-)
 create mode 100644 tensorflow/core/common_runtime/process_util_test.cc

diff --git a/tensorflow/contrib/data/kernels/prefetching_kernels.cc b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
index 207f2820bf..a2bfce0362 100644
--- a/tensorflow/contrib/data/kernels/prefetching_kernels.cc
+++ b/tensorflow/contrib/data/kernels/prefetching_kernels.cc
@@ -40,8 +40,7 @@ class FunctionBufferingResource : public ResourceBase {
                             const NameAttrList& func, int64 buffer_size,
                             const string& source_device,
                             const string& target_device,
-                            const std::vector<Tensor>& func_args,
-                            int64 thread_pool_size)
+                            const std::vector<Tensor>& func_args)
       : lib_(lib),
         pflr_(std::move(pflr)),
         func_(func),
@@ -52,22 +51,10 @@ class FunctionBufferingResource : public ResourceBase {
         handle_(kInvalidHandle),
         is_buffering_(false),
         end_of_sequence_(false),
-        cancelled_(false) {
-    if (thread_pool_size > 0) {
-      thread_pool_ = new thread::ThreadPool(Env::Default(), ThreadOptions(),
-                                            "buffer_resource", thread_pool_size,
-                                            false /* low_latency_hint */);
-      runner_ = [this](std::function<void()> c) {
-        thread_pool_->Schedule(std::move(c));
-      };
-    }
-  }
+        cancelled_(false) {}
 
   ~FunctionBufferingResource() override {
     Cancel();
-    if (thread_pool_ != nullptr) {
-      delete thread_pool_;
-    }
   }
 
   string DebugString() override {
@@ -179,17 +166,12 @@ class FunctionBufferingResource : public ResourceBase {
       for (int i = 0; i < cancellation_callbacks.size(); ++i) {
         cancellation_callbacks[i](cancellation_buffer_elements[i]);
       }
-      // We only wait on cond_var_ in the destructor, so there would atmost be
-      // one waiter to notify.
-      cond_var_.notify_one();
+      cond_var_.notify_all();
       return;
     }
     FunctionLibraryRuntime::Options opts;
     // Copied from CapturedFunction::generate_step_id();
     opts.step_id = -std::abs(static_cast<int64>(random::New64()));
-    if (runner_ != nullptr) {
-      opts.runner = &runner_;
-    }
     opts.source_device = source_device_;
     AllocatorAttributes arg_alloc_attr;
     arg_alloc_attr.set_on_host(true);
@@ -251,11 +233,9 @@ class FunctionBufferingResource : public ResourceBase {
   const string source_device_;
   const string target_device_;
   const std::vector<Tensor> func_args_;
-  thread::ThreadPool* thread_pool_ = nullptr;
   FunctionLibraryRuntime::Handle handle_ GUARDED_BY(mu_);
   std::deque<BufferElement> buffer_ GUARDED_BY(mu_);
   std::deque<FunctionBufferCallback> requests_ GUARDED_BY(mu_);
-  std::function<void(std::function<void()>)> runner_ = nullptr;
   bool is_buffering_ GUARDED_BY(mu_);
   bool end_of_sequence_ GUARDED_BY(mu_);
   bool cancelled_ GUARDED_BY(mu_);
@@ -270,7 +250,6 @@ class FunctionBufferResourceHandleOp : public OpKernel {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("buffer_size", &buffer_size_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("container", &container_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("shared_name", &name_));
-    OP_REQUIRES_OK(ctx, ctx->GetAttr("thread_pool_size", &thread_pool_size_));
   }
 
   ~FunctionBufferResourceHandleOp() override {
@@ -318,7 +297,7 @@ class FunctionBufferResourceHandleOp : public OpKernel {
                this](FunctionBufferingResource** ptr) {
                 *ptr = new FunctionBufferingResource(
                     clone_lib, std::move(pflr), func_, buffer_size_,
-                    source_device, target_device, func_args, thread_pool_size_);
+                    source_device, target_device, func_args);
                 return Status::OK();
               }));
       core::ScopedUnref s(buffer);
@@ -340,7 +319,6 @@ class FunctionBufferResourceHandleOp : public OpKernel {
   int64 buffer_size_;
   string container_;
   string name_;
-  int64 thread_pool_size_;
 };
 
 REGISTER_KERNEL_BUILDER(Name("FunctionBufferingResource")
diff --git a/tensorflow/contrib/data/ops/dataset_ops.cc b/tensorflow/contrib/data/ops/dataset_ops.cc
index bd96448d64..cf0a8bbccb 100644
--- a/tensorflow/contrib/data/ops/dataset_ops.cc
+++ b/tensorflow/contrib/data/ops/dataset_ops.cc
@@ -53,7 +53,6 @@ REGISTER_OP("FunctionBufferingResource")
     .Attr("container: string")
     .Attr("f: func")
     .Attr("buffer_size: int")
-    .Attr("thread_pool_size: int")
     .SetShapeFn(shape_inference::UnknownShape)
     .Doc(R"doc(
 Creates a resource that fills up a buffer by making function calls.
@@ -63,7 +62,6 @@ target_device: Target device to execute the function on.
 resource: Handle to the resource created.
 f: Function to be executed.
 buffer_size: Size of the buffer.
-thread_pool_size: Size of the threadpool doing the prefetching.
 container: If non-empty, this resource is placed in the given container.
   Otherwise, a default container is used.
 shared_name: If non-empty, this resource will be shared under the given name
diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index a14736ac09..676959a900 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -70,7 +70,6 @@ class StagingAreaOpsTest(test.TestCase):
           target_device=target,
           string_arg=ds_iterator_handle,
           buffer_size=3,
-          thread_pool_size=2,
           shared_name=buffer_name)
 
     with ops.device(device1):
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 1438b5426f..98651bb568 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -36,7 +36,6 @@ def function_buffering_resource(string_arg,
                                 target_device,
                                 f,
                                 buffer_size,
-                                thread_pool_size=0,
                                 container="",
                                 shared_name=None,
                                 name=None):
@@ -48,7 +47,6 @@ def function_buffering_resource(string_arg,
       shared_name=shared_name,
       f=f,
       buffer_size=buffer_size,
-      thread_pool_size=thread_pool_size,
       container=container,
       name=name)
 
@@ -90,8 +88,7 @@ class _PrefetchToDeviceIterator(object):
           target_device=gen_dataset_ops.iterator_get_device(
               input_iterator._iterator_resource),
           string_arg=input_iterator_handle,
-          buffer_size=buffer_size,
-          thread_pool_size=0)
+          buffer_size=buffer_size)
 
   def get_next(self, name=None):
     """See @{tf.data.Iterator.get_next}."""
diff --git a/tensorflow/contrib/distribute/python/prefetching_ops_v2.py b/tensorflow/contrib/distribute/python/prefetching_ops_v2.py
index b9ffd2f266..e1ddf3cece 100644
--- a/tensorflow/contrib/distribute/python/prefetching_ops_v2.py
+++ b/tensorflow/contrib/distribute/python/prefetching_ops_v2.py
@@ -59,8 +59,7 @@ class _PrefetchToDeviceIterator(object):
             f=_prefetch_fn,
             target_device=target_device,
             string_arg=input_iterator_handle,
-            buffer_size=buffer_size,
-            thread_pool_size=0)
+            buffer_size=buffer_size)
         self._buffering_resources.append(buffer_resource_handle)
 
   def get_next(self, name=None):
diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index a4c3283dac..60453006f4 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -98,7 +98,6 @@ class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
             f=remote_fn,
             target_device=target,
             buffer_size=10,
-            thread_pool_size=1,
             container="",
             shared_name=_generate_shared_name("function_buffer_resource"))
         self._buffer_resource_deleter = resource_variable_ops.EagerResourceDeleter(  # pylint: disable=line-too-long
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 614e06cf83..21f7866abd 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -3273,6 +3273,18 @@ tf_cc_test(
     ],
 )
 
+tf_cc_test(
+    name = "common_runtime_process_util_test",
+    size = "small",
+    srcs = ["common_runtime/process_util_test.cc"],
+    linkstatic = tf_kernel_tests_linkstatic(),
+    deps = [
+        ":core_cpu_internal",
+        ":test",
+        ":test_main",
+    ],
+)
+
 tf_cc_test(
     name = "common_runtime_rendezvous_util_test",
     size = "small",
diff --git a/tensorflow/core/common_runtime/direct_session.cc b/tensorflow/core/common_runtime/direct_session.cc
index 25cfb9e524..0479061daf 100644
--- a/tensorflow/core/common_runtime/direct_session.cc
+++ b/tensorflow/core/common_runtime/direct_session.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/graph_optimizer.h"
 #include "tensorflow/core/common_runtime/memory_types.h"
 #include "tensorflow/core/common_runtime/optimization_registry.h"
+#include "tensorflow/core/common_runtime/process_util.h"
 #include "tensorflow/core/common_runtime/step_stats_collector.h"
 #include "tensorflow/core/framework/function.h"
 #include "tensorflow/core/framework/graph.pb_text.h"
@@ -69,20 +70,6 @@ auto* direct_session_runs = monitoring::Counter<0>::New(
     "/tensorflow/core/direct_session_runs",
     "The number of times DirectSession::Run() has been called.");
 
-int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
-  const int32 t = options.config.inter_op_parallelism_threads();
-  if (t != 0) return t;
-  // Default to using the number of cores available in the process.
-  return port::NumSchedulableCPUs();
-}
-
-thread::ThreadPool* NewThreadPoolFromSessionOptions(
-    const SessionOptions& options) {
-  const int32 num_threads = NumInterOpThreadsFromSessionOptions(options);
-  VLOG(1) << "Direct session inter op parallelism threads: " << num_threads;
-  return new thread::ThreadPool(options.env, "Compute", num_threads);
-}
-
 Status NewThreadPoolFromThreadPoolOptions(
     const SessionOptions& options,
     const ThreadPoolOptionProto& thread_pool_options, int pool_number,
diff --git a/tensorflow/core/common_runtime/eager/context.cc b/tensorflow/core/common_runtime/eager/context.cc
index 9c47ad6187..d3fe6a7ede 100644
--- a/tensorflow/core/common_runtime/eager/context.cc
+++ b/tensorflow/core/common_runtime/eager/context.cc
@@ -15,6 +15,8 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/eager/context.h"
 
+#include "tensorflow/core/common_runtime/process_util.h"
+
 namespace tensorflow {
 
 EagerContext::EagerContext(const SessionOptions& opts,
@@ -25,9 +27,10 @@ EagerContext::EagerContext(const SessionOptions& opts,
       device_manager_(std::move(device_mgr)),
       devices_(device_manager_->ListDevices()),
       rendezvous_(rendezvous),
-      pflr_(new ProcessFunctionLibraryRuntime(device_manager_.get(), opts.env,
-                                              TF_GRAPH_DEF_VERSION,
-                                              &func_lib_def_, {})),
+      thread_pool_(NewThreadPoolFromSessionOptions(opts)),
+      pflr_(new ProcessFunctionLibraryRuntime(
+          device_manager_.get(), opts.env, TF_GRAPH_DEF_VERSION, &func_lib_def_,
+          {}, thread_pool_.get())),
       log_device_placement_(opts.config.log_device_placement()),
       async_default_(async) {
   if (async_default_) {
diff --git a/tensorflow/core/common_runtime/eager/context.h b/tensorflow/core/common_runtime/eager/context.h
index a88fa5eaa4..6665df27d0 100644
--- a/tensorflow/core/common_runtime/eager/context.h
+++ b/tensorflow/core/common_runtime/eager/context.h
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/common_runtime/rendezvous_mgr.h"
 #include "tensorflow/core/framework/rendezvous.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
@@ -160,6 +161,8 @@ class EagerContext {
   FunctionLibraryDefinition func_lib_def_ GUARDED_BY(functions_mu_){
       OpRegistry::Global(), {}};
 
+  std::unique_ptr<thread::ThreadPool> thread_pool_;
+
   // One FunctionLibraryRuntime per device.
   // func_libs[i] is the FunctionLibraryRuntime corresponding to
   // session->devices[i].
diff --git a/tensorflow/core/common_runtime/function.cc b/tensorflow/core/common_runtime/function.cc
index 37c59a16f5..d310520ebd 100644
--- a/tensorflow/core/common_runtime/function.cc
+++ b/tensorflow/core/common_runtime/function.cc
@@ -796,16 +796,17 @@ void FunctionLibraryRuntimeImpl::Run(const Options& opts, Handle handle,
       done(status);
     };
   }
-  if (!parent_->IsInstantiatedOnDevice(device_name_, handle)) {
-    parent_->Run(run_opts, handle, args, rets, done);
-    return;
-  }
 
   if (run_opts.runner == nullptr) {
     run_opts.runner = &default_runner_;
   }
   DCHECK(run_opts.runner != nullptr);
 
+  if (!parent_->IsInstantiatedOnDevice(device_name_, handle)) {
+    parent_->Run(run_opts, handle, args, rets, done);
+    return;
+  }
+
   Executor::Args* exec_args = new Executor::Args;
   // Inherit the step_id from the caller.
   exec_args->step_id = run_opts.step_id;
diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc
index d738fef7be..d5bd7f8b98 100644
--- a/tensorflow/core/common_runtime/process_util.cc
+++ b/tensorflow/core/common_runtime/process_util.cc
@@ -46,6 +46,20 @@ thread::ThreadPool* ComputePool(const SessionOptions& options) {
   return compute_pool;
 }
 
+int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
+  const int32 t = options.config.inter_op_parallelism_threads();
+  if (t != 0) return t;
+  // Default to using the number of cores available in the process.
+  return port::NumSchedulableCPUs();
+}
+
+thread::ThreadPool* NewThreadPoolFromSessionOptions(
+    const SessionOptions& options) {
+  const int32 num_threads = NumInterOpThreadsFromSessionOptions(options);
+  VLOG(1) << "Direct session inter op parallelism threads: " << num_threads;
+  return new thread::ThreadPool(options.env, "Compute", num_threads);
+}
+
 void SchedClosure(std::function<void()> closure) {
   if (port::Tracing::IsActive()) {
     const uint64 id = port::Tracing::UniqueId();
diff --git a/tensorflow/core/common_runtime/process_util.h b/tensorflow/core/common_runtime/process_util.h
index fc3a262fe1..5d92666716 100644
--- a/tensorflow/core/common_runtime/process_util.h
+++ b/tensorflow/core/common_runtime/process_util.h
@@ -30,6 +30,13 @@ namespace tensorflow {
 // using 'options'.  Caller does not take ownership over threadpool.
 thread::ThreadPool* ComputePool(const SessionOptions& options);
 
+// Returns number of inter op threads.
+int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options);
+
+// Creates a thread pool with number of inter op threads.
+thread::ThreadPool* NewThreadPoolFromSessionOptions(
+    const SessionOptions& options);
+
 // Schedule "closure" in the default thread queue.
 void SchedClosure(std::function<void()> closure);
 
diff --git a/tensorflow/core/common_runtime/process_util_test.cc b/tensorflow/core/common_runtime/process_util_test.cc
new file mode 100644
index 0000000000..46672ac92e
--- /dev/null
+++ b/tensorflow/core/common_runtime/process_util_test.cc
@@ -0,0 +1,38 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include "tensorflow/core/common_runtime/process_util.h"
+
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace {
+
+TEST(ProcessUtilTest, NumThreads) {
+  SessionOptions opts;
+  opts.config.set_inter_op_parallelism_threads(10);
+  EXPECT_EQ(10, NumInterOpThreadsFromSessionOptions(opts));
+}
+
+TEST(ProcessUtilTest, ThreadPool) {
+  SessionOptions opts;
+  opts.config.set_inter_op_parallelism_threads(10);
+
+  thread::ThreadPool* pool = NewThreadPoolFromSessionOptions(opts);
+  EXPECT_EQ(10, pool->NumThreads());
+  delete pool;
+}
+
+}  // anonymous namespace
+}  // namespace tensorflow
-- 
GitLab


From 53c003f7b9ec8e2f98b7df14154a3320383f2a9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 18:37:21 -0700
Subject: [PATCH 727/960] Fixed typo in PReLU.

PiperOrigin-RevId: 191019421
---
 tensorflow/contrib/lite/toco/tflite/operator.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index 0989bfe5a3..0cb348bda5 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -855,7 +855,7 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
   ops.emplace_back(
       new SimpleOperator<Relu6Operator>("RELU6", OperatorType::kRelu6));
   ops.emplace_back(
-      new SimpleOperator<Relu1Operator>("PRELU", OperatorType::kPRelu));
+      new SimpleOperator<PReluOperator>("PRELU", OperatorType::kPRelu));
   ops.emplace_back(new SimpleOperator<LogisticOperator>(
       "LOGISTIC", OperatorType::kLogistic));
   ops.emplace_back(
-- 
GitLab


From 17955b4ff561517cd0e767cf99b333178d211a99 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Thu, 29 Mar 2018 18:44:12 -0700
Subject: [PATCH 728/960] Disable shape inference for functions since it
 doesn't seem to work when functions are instantiated inside a while loop.

PiperOrigin-RevId: 191019870
---
 tensorflow/core/grappler/costs/graph_properties.cc      | 1 -
 tensorflow/core/grappler/costs/graph_properties_test.cc | 3 +++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index a5fd79447d..5103098f27 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -926,7 +926,6 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) {
   ShapeRefiner shape_refiner(graph.versions(), graph.op_registry());
   shape_refiner.set_require_shape_inference_fns(false);
   shape_refiner.set_disable_constant_propagation(true);
-  shape_refiner.set_function_library_for_shape_inference(&function_library);
   ImportGraphDefOptions options;
   // Graph optimization happens at the late stage of graph execution,
   // when colocation constraints are already validated previously and
diff --git a/tensorflow/core/grappler/costs/graph_properties_test.cc b/tensorflow/core/grappler/costs/graph_properties_test.cc
index 284d9d409b..db4dae96de 100644
--- a/tensorflow/core/grappler/costs/graph_properties_test.cc
+++ b/tensorflow/core/grappler/costs/graph_properties_test.cc
@@ -742,6 +742,8 @@ TEST_F(GraphPropertiesTest, InferRestoreOpShape_WithTwoNodesShareSameOutput) {
   EXPECT_EQ("float: [128,256]", PropToString(prop));
 }
 
+#if 0
+// Disabled for now since this doesnt' seem to work when functions are instantiated inside while loops. It's also unclear whether it's correct when the same function is instantiated twice.
 TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) {
   // Test graph produced in python using:
   /*
@@ -775,6 +777,7 @@ TEST_F(GraphPropertiesTest, FunctionStaticShapeInference) {
   EXPECT_TRUE(shape.IsFullyDefined());
   EXPECT_FALSE(shape.unknown_rank());
 }
+#endif
 
 TEST_F(GraphPropertiesTest, SymbolicShapes) {
   // Build a simple graph with placeholders of unknown dimensions. These
-- 
GitLab


From 0567f169e6910587e7dcca547aa5baeaaffbc03d Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 29 Mar 2018 18:50:05 -0700
Subject: [PATCH 729/960] Internal change.

PiperOrigin-RevId: 191020351
---
 tensorflow/contrib/distribute/python/values.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index c1ba22ed5a..b5b7ec8381 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -243,7 +243,7 @@ def _tensor_conversion(var, dtype=None, name=None, as_ref=False):
 
 
 ops.register_tensor_conversion_function(DistributedVariable, _tensor_conversion)
-# TODO(josh11b): ops.register_dense_tensor_like_type(DistributedVariable)?
+ops.register_dense_tensor_like_type(DistributedVariable)
 
 
 class _MirroredSaveable(saver.BaseSaverBuilder.ResourceVariableSaveable):
-- 
GitLab


From 3344714a2886287377c2255906d6004f213e3bc0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 18:57:52 -0700
Subject: [PATCH 730/960] Prevent infinite loop in constant folding when
 materializing broadcast gradient nodes that are subsequently constant folded.

PiperOrigin-RevId: 191020868
---
 .../grappler/optimizers/constant_folding.cc   | 28 ++++---
 .../optimizers/constant_folding_test.cc       | 73 +++++++++++++++++++
 2 files changed, 92 insertions(+), 9 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index c3f8a1ce22..7de544de52 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -483,6 +483,11 @@ Status ConstantFolding::MaterializeBroadcastGradientArgs(
     return Status::OK();
   }
 
+  // Don't optimize this again if it was already optimized and folded.
+  if (OptimizedNodeExists(node, "-folded-1") ||
+      OptimizedNodeExists(node, "-folded-2")) {
+    return Status::OK();
+  }
   int64 min_id = 0;
   BCast::Vec shape1;
   if (!ExtractShape(*shape_node1, properties, &shape1, &min_id)) {
@@ -547,6 +552,7 @@ Status ConstantFolding::MaterializeBroadcastGradientArgs(
 
   const DataType type = node.attr().at("T").type();
   NodeDef* out[2];
+  bool created_const = false;
   for (int j = 0; j < 2; ++j) {
     int reduction_indices = reduce_dims[j].size();
     Tensor value(type, TensorShape({reduction_indices}));
@@ -570,17 +576,20 @@ Status ConstantFolding::MaterializeBroadcastGradientArgs(
           AddControlDependency(node.name(), graph_, node_map_.get());
       *out[j]->add_input() = ctrl_dep;
       node_map_->AddOutput(NodeName(ctrl_dep), const_name);
+      created_const = true;
     }
   }
 
-  const std::set<NodeDef*> outputs = node_map_->GetOutputs(node.name());
-  for (NodeDef* output : outputs) {
-    for (int k = 0; k < output->input_size(); ++k) {
-      int port;
-      string node_name = ParseNodeName(output->input(k), &port);
-      if (node_name == node.name() && port >= 0 && port < 2 && out[port]) {
-        *output->mutable_input(k) = out[port]->name();
-        node_map_->UpdateInput(output->name(), node_name, out[port]->name());
+  if (created_const) {
+    const std::set<NodeDef*> outputs = node_map_->GetOutputs(node.name());
+    for (NodeDef* output : outputs) {
+      for (int k = 0; k < output->input_size(); ++k) {
+        int port;
+        string node_name = ParseNodeName(output->input(k), &port);
+        if (node_name == node.name() && port >= 0 && port < 2 && out[port]) {
+          *output->mutable_input(k) = out[port]->name();
+          node_map_->UpdateInput(output->name(), node_name, out[port]->name());
+        }
       }
     }
   }
@@ -1237,7 +1246,8 @@ Status ConstantFolding::FoldGraph(GraphDef* output) {
     Status s = FoldNode(node, output);
     processed_nodes.insert(node->name());
     if (!s.ok()) {
-      VLOG(1) << "Failed to fold node " << node->name() << ": " << s;
+      VLOG(1) << "Failed to fold node " << node->DebugString()
+              << "\nError message: " << s;
     } else {
       for (auto& output : fanout) {
         if (IsFoldable(*output)) {
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 16a19ba8ce..0548e53e03 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1754,6 +1754,79 @@ TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs) {
   EXPECT_EQ(6, found);
 }
 
+TEST_F(ConstantFoldingTest, MaterializeBroadcastGradientArgs_InfiniteLoop) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output a =
+      ops::Placeholder(s.WithOpName("a"), DT_FLOAT,
+                       ops::Placeholder::Shape(PartialTensorShape({2, 2})));
+  Output b = ops::Square(s.WithOpName("b"), a);
+  Output c = ops::Mul(s.WithOpName("c"), a, b);
+  Output d = ops::Shape(s.WithOpName("d"), a);
+  Output e = ops::Shape(s.WithOpName("e"), b);
+
+  auto f = ops::internal::BroadcastGradientArgs(s.WithOpName("f"), d, e);
+  Output o1 = ops::Identity(s.WithOpName("o1"), f.r0);
+  Output o2 = ops::Identity(s.WithOpName("o2"), f.r1);
+
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  ConstantFolding optimizer(nullptr /* cpu_device */);
+  GraphDef output;
+  Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  // Run a second time to make sure the optimization is idempotent.
+  item.graph.Swap(&output);
+  status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+
+  EXPECT_EQ(11, output.node_size());
+  int found = 0;
+  for (const auto& node : output.node()) {
+    if (node.name() == "ConstantFolding/f-folded-1") {
+      ++found;
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("^a", node.input(0));
+      EXPECT_EQ("^b", node.input(1));
+    } else if (node.name() == "d") {
+      ++found;
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^a", node.input(0));
+    } else if (node.name() == "e") {
+      ++found;
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^b", node.input(0));
+    } else if (node.name() == "o1") {
+      ++found;
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("ConstantFolding/f-bcastargs-0", node.input(0));
+    } else if (node.name() == "o2") {
+      ++found;
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("ConstantFolding/f-bcastargs-1", node.input(0));
+    } else if (node.name() == "ConstantFolding/f-bcastargs-0") {
+      ++found;
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^ConstantFolding/f-folded-1", node.input(0));
+      EXPECT_EQ(0, TensorShape(node.attr().at("value").tensor().tensor_shape())
+                       .num_elements());
+    } else if (node.name() == "ConstantFolding/f-bcastargs-1") {
+      ++found;
+      EXPECT_EQ("Const", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("^ConstantFolding/f-folded-1", node.input(0));
+      EXPECT_EQ(0, TensorShape(node.attr().at("value").tensor().tensor_shape())
+                       .num_elements());
+    }
+  }
+  EXPECT_EQ(7, found);
+}
+
 TEST_F(ConstantFoldingTest, MaterializeReductionIndices) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   Output input =
-- 
GitLab


From 28b95b35482d665a9483803ae9fb7b081b4f8a75 Mon Sep 17 00:00:00 2001
From: Rui Zhao <rzhao@google.com>
Date: Thu, 29 Mar 2018 19:02:29 -0700
Subject: [PATCH 731/960] Add output info into op_context otherwise the
 estimator crashes when OOM.

PiperOrigin-RevId: 191021184
---
 .../grappler/optimizers/static_schedule.cc    |  6 ++++
 .../optimizers/static_schedule_test.cc        | 32 +++++++++----------
 2 files changed, 22 insertions(+), 16 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/static_schedule.cc b/tensorflow/core/grappler/optimizers/static_schedule.cc
index 450e853407..5206e9957d 100644
--- a/tensorflow/core/grappler/optimizers/static_schedule.cc
+++ b/tensorflow/core/grappler/optimizers/static_schedule.cc
@@ -40,6 +40,12 @@ static Costs::NanoSeconds PredictExecutionTime(
     op_context.op_info.add_inputs()->Swap(&input);
   }
 
+  std::vector<OpInfo::TensorProperties> outputs =
+      properties.GetOutputProperties(node.name());
+  for (auto& output : outputs) {
+    op_context.op_info.add_outputs()->Swap(&output);
+  }
+
   DeviceProperties device = placer.get_device(node);
   op_context.op_info.mutable_device()->Swap(&device);
 
diff --git a/tensorflow/core/grappler/optimizers/static_schedule_test.cc b/tensorflow/core/grappler/optimizers/static_schedule_test.cc
index 08580d9284..d632e460e7 100644
--- a/tensorflow/core/grappler/optimizers/static_schedule_test.cc
+++ b/tensorflow/core/grappler/optimizers/static_schedule_test.cc
@@ -64,17 +64,17 @@ TEST_F(StaticScheduleTest, BasicGraph) {
     if (time.first->name() == "Const/Const") {
       EXPECT_EQ(Costs::NanoSeconds(1), time.second);
     } else if (time.first->name() == "x") {
-      EXPECT_EQ(Costs::NanoSeconds(250001), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(1500001), time.second);
     } else if (time.first->name() == "Square") {
-      EXPECT_EQ(Costs::NanoSeconds(1500004), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(4000004), time.second);
     } else if (time.first->name() == "Square_1") {
-      EXPECT_EQ(Costs::NanoSeconds(2750007), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(6500007), time.second);
     } else if (time.first->name() == "Square_2") {
-      EXPECT_EQ(Costs::NanoSeconds(4000010), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(9000010), time.second);
     } else if (time.first->name() == "Square_3") {
-      EXPECT_EQ(Costs::NanoSeconds(5250013), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(11500013), time.second);
     } else if (time.first->name() == "y") {
-      EXPECT_EQ(Costs::NanoSeconds(6500013), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(14000013), time.second);
     }
   }
 }
@@ -110,13 +110,13 @@ TEST_F(StaticScheduleTest, BasicGraphWithCtrlDependencies) {
     if (time.first->name() == "a") {
       EXPECT_EQ(Costs::NanoSeconds(1), time.second);
     } else if (time.first->name() == "b") {
-      EXPECT_EQ(Costs::NanoSeconds(12500001), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(25000001), time.second);
     } else if (time.first->name() == "c") {
-      EXPECT_EQ(Costs::NanoSeconds(12500002), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(25000002), time.second);
     } else if (time.first->name() == "d") {
-      EXPECT_EQ(Costs::NanoSeconds(12500003), time.second);
-    } else if (time.first->name() == "e") {
       EXPECT_EQ(Costs::NanoSeconds(25000003), time.second);
+    } else if (time.first->name() == "e") {
+      EXPECT_EQ(Costs::NanoSeconds(50000003), time.second);
     }
   }
 }
@@ -142,17 +142,17 @@ TEST_F(StaticScheduleTest, RequiredTimes) {
 
   for (auto time : required_times) {
     if (time.first->name() == "Const/Const") {
-      EXPECT_EQ(Costs::NanoSeconds(-6500012), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-14000012), time.second);
     } else if (time.first->name() == "x") {
-      EXPECT_EQ(Costs::NanoSeconds(-6250012), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-12500012), time.second);
     } else if (time.first->name() == "Square") {
-      EXPECT_EQ(Costs::NanoSeconds(-5000009), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-10000009), time.second);
     } else if (time.first->name() == "Square_1") {
-      EXPECT_EQ(Costs::NanoSeconds(-3750006), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-7500006), time.second);
     } else if (time.first->name() == "Square_2") {
-      EXPECT_EQ(Costs::NanoSeconds(-2500003), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-5000003), time.second);
     } else if (time.first->name() == "Square_3") {
-      EXPECT_EQ(Costs::NanoSeconds(-1250000), time.second);
+      EXPECT_EQ(Costs::NanoSeconds(-2500000), time.second);
     } else if (time.first->name() == "y") {
       EXPECT_EQ(Costs::NanoSeconds(0), time.second);
     }
-- 
GitLab


From 28dec7f4669e8ed5af4a3abebf9888d3fdffe5fd Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 29 Mar 2018 19:24:58 -0700
Subject: [PATCH 732/960] Add basic serialization support to
 DistributedVariable (by using the underlying primary variable's
 serialization). Also, throw an exception when trying to de-serialize as we
 haven't implemented that yet.

PiperOrigin-RevId: 191022884
---
 tensorflow/contrib/distribute/python/values.py |  3 +++
 tensorflow/python/training/distribute.py       | 17 +++++++++++++++++
 2 files changed, 20 insertions(+)

diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index b5b7ec8381..9acb6a9db9 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -216,6 +216,9 @@ class DistributedVariable(DistributedDelegate):
   def get_shape(self):
     return self._primary_var.get_shape()
 
+  def to_proto(self, export_scope=None):
+    return self._primary_var.to_proto(export_scope=export_scope)
+
   @property
   def op(self):
     # We want cross-tower code that does some var.op.X calls
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index d51ef341a0..38b0580cd8 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1194,3 +1194,20 @@ _default_distribution_strategy = _DefaultDistributionStrategy()
 _default_tower_context = TowerContext(
     _default_distribution_strategy, tower_id=0)
 _default_tower_mode = _DefaultTowerThreadMode()
+
+
+# ------------------------------------------------------------------------------
+# We haven't yet implemented deserialization for DistributedVariables.
+# So here we catch any attempts to deserialize variables
+# when using distribution strategies.
+# pylint: disable=protected-access
+def _from_proto_fn(v, import_scope=None):
+  if has_distribution_strategy():
+    raise NotImplementedError(
+        "Deserialization of variables is not yet supported when using"
+        "distributed strategies.")
+  else:
+    resource_variable_ops._from_proto_fn(v, import_scope=import_scope)
+
+resource_variable_ops._from_proto_fn = _from_proto_fn
+# pylint: enable=protected-access
-- 
GitLab


From 8712a7fa24da7803a79c10501f8fd65d358a62b5 Mon Sep 17 00:00:00 2001
From: Igor Saprykin <isaprykin@google.com>
Date: Thu, 29 Mar 2018 19:29:14 -0700
Subject: [PATCH 733/960] Internal change.

PiperOrigin-RevId: 191023160
---
 tensorflow/contrib/optimizer_v2/adagrad.py | 24 +++++++++++-----------
 1 file changed, 12 insertions(+), 12 deletions(-)

diff --git a/tensorflow/contrib/optimizer_v2/adagrad.py b/tensorflow/contrib/optimizer_v2/adagrad.py
index e54f990cca..c333d1e089 100644
--- a/tensorflow/contrib/optimizer_v2/adagrad.py
+++ b/tensorflow/contrib/optimizer_v2/adagrad.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.optimizer_v2 import optimizer_v2
-from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.ops import init_ops
@@ -65,17 +64,18 @@ class AdagradOptimizer(optimizer_v2.OptimizerV2):
 
   def _create_vars(self, var_list, state):
     for v in var_list:
-      with ops.colocate_with(v):
-        dtype = v.dtype.base_dtype
-        if v.get_shape().is_fully_defined():
-          init = init_ops.constant_initializer(self._initial_accumulator_value,
-                                               dtype=dtype)
-        else:
-          # Use a Tensor instead of initializer if variable does not have static
-          # shape.
-          init_constant = gen_array_ops.fill(
-              array_ops.shape(v), self._initial_accumulator_value)
-          init = math_ops.cast(init_constant, dtype)
+      # TODO(isaprykin): Delete colocate_with(v) from other optimizers and
+      # confirm that colocation will happen anyway.
+      dtype = v.dtype.base_dtype
+      if v.get_shape().is_fully_defined():
+        init = init_ops.constant_initializer(self._initial_accumulator_value,
+                                             dtype=dtype)
+      else:
+        # Use a Tensor instead of initializer if variable does not have static
+        # shape.
+        init_constant = gen_array_ops.fill(
+            array_ops.shape(v), self._initial_accumulator_value)
+        init = math_ops.cast(init_constant, dtype)
       state.create_slot_with_initializer(v, init, v.get_shape(), dtype,
                                          "accumulator")
 
-- 
GitLab


From 9d8201cad77f6892c8da059bf23348c7eb16e3be Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Thu, 29 Mar 2018 19:29:21 -0700
Subject: [PATCH 734/960] BUGFIX:  sample_stats.percentile:  Fractions for
 interpolation were done in float, which caused the error d - 1 == d for large
 'd'.  Fix is to do in double.  Also clipping index values to [0,..., d - 1]
 in case double isn't enough for some huge array PiperOrigin-RevId: 191023164

---
 .../python/kernel_tests/sample_stats_test.py  | 11 ++++++++++
 .../distributions/python/ops/sample_stats.py  | 20 ++++++++++++++-----
 2 files changed, 26 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py b/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py
index 4186cf129d..ea04e8c29a 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/sample_stats_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.contrib.distributions.python.ops import sample_stats
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import spectral_ops_test_util
 from tensorflow.python.platform import test
 
@@ -455,6 +456,16 @@ class PercentileTestWithNearestInterpolation(test.TestCase):
       with self.assertRaisesOpError("rank"):
         pct.eval(feed_dict={q_ph: [0.5]})
 
+  def test_finds_max_of_long_array(self):
+    # d - 1 == d in float32 and d = 3e7.
+    # So this test only passes if we use double for the percentile indices.
+    # If float is used, it fails with InvalidArgumentError about an index out of
+    # bounds.
+    x = math_ops.linspace(0., 3e7, num=int(3e7))
+    with self.test_session():
+      minval = sample_stats.percentile(x, q=0, validate_args=True)
+      self.assertAllEqual(0, minval.eval())
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distributions/python/ops/sample_stats.py b/tensorflow/contrib/distributions/python/ops/sample_stats.py
index dfc8133619..f5aaa5cf34 100644
--- a/tensorflow/contrib/distributions/python/ops/sample_stats.py
+++ b/tensorflow/contrib/distributions/python/ops/sample_stats.py
@@ -25,6 +25,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
+from tensorflow.python.ops import clip_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
@@ -301,13 +302,16 @@ def percentile(x,
 
   with ops.name_scope(name, [x, q]):
     x = ops.convert_to_tensor(x, name="x")
-    q = math_ops.to_float(q, name="q")
+    # Double is needed here and below, else we get the wrong index if the array
+    # is huge along axis.
+    q = math_ops.to_double(q, name="q")
     _get_static_ndims(q, expect_ndims=0)
 
     if validate_args:
       q = control_flow_ops.with_dependencies([
-          check_ops.assert_rank(q, 0), check_ops.assert_greater_equal(q, 0.),
-          check_ops.assert_less_equal(q, 100.)
+          check_ops.assert_rank(q, 0),
+          check_ops.assert_greater_equal(q, math_ops.to_double(0.)),
+          check_ops.assert_less_equal(q, math_ops.to_double(100.))
       ], q)
 
     if axis is None:
@@ -332,7 +336,7 @@ def percentile(x,
       y = _move_dims_to_flat_end(x, axis, x_ndims)
 
     frac_at_q_or_above = 1. - q / 100.
-    d = math_ops.to_float(array_ops.shape(y)[-1])
+    d = math_ops.to_double(array_ops.shape(y)[-1])
 
     if interpolation == "lower":
       index = math_ops.ceil((d - 1) * frac_at_q_or_above)
@@ -341,12 +345,18 @@ def percentile(x,
     elif interpolation == "nearest":
       index = math_ops.round((d - 1) * frac_at_q_or_above)
 
+    # If d is gigantic, then we would have d == d - 1, even in double... So
+    # let's use max/min to avoid out of bounds errors.
+    d = array_ops.shape(y)[-1]
+    # d - 1 will be distinct from d in int32.
+    index = clip_ops.clip_by_value(math_ops.to_int32(index), 0, d - 1)
+
     # Sort everything, not just the top 'k' entries, which allows multiple calls
     # to sort only once (under the hood) and use CSE.
     sorted_y = _sort_tensor(y)
 
     # result.shape = B
-    result = sorted_y[..., math_ops.to_int32(index)]
+    result = sorted_y[..., index]
     result.set_shape(y.get_shape()[:-1])
 
     if keep_dims:
-- 
GitLab


From 566f9041e19831a4eb8904654ddd365fd8f234c0 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Thu, 29 Mar 2018 19:45:17 -0700
Subject: [PATCH 735/960] Internal change

PiperOrigin-RevId: 191024085
---
 tensorflow/contrib/data/python/kernel_tests/BUILD | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index 0f4c9e48cf..c8699e0d5a 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -477,11 +477,6 @@ py_test(
     size = "small",
     srcs = ["prefetching_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "manual",
-        "no_oss",
-        "notap",
-    ],
     deps = [
         "//tensorflow/contrib/data/python/ops:prefetching_ops",
         "//tensorflow/core:protos_all_py",
-- 
GitLab


From 9451c12d62b272789947f475554601295ada4083 Mon Sep 17 00:00:00 2001
From: Yuefeng Zhou <yuefengz@google.com>
Date: Thu, 29 Mar 2018 19:56:47 -0700
Subject: [PATCH 736/960] Internal change

PiperOrigin-RevId: 191024677
---
 tensorflow/contrib/distribute/python/BUILD    |   2 +
 .../distribute/python/cross_tower_ops.py      | 381 +++++++++++++-----
 .../distribute/python/cross_tower_ops_test.py |  66 ++-
 .../distribute/python/cross_tower_utils.py    | 186 +++++++++
 .../distribute/python/mirrored_strategy.py    |  25 +-
 tensorflow/python/BUILD                       |   1 +
 tensorflow/python/client/device_lib.i         |  41 +-
 tensorflow/python/client/device_lib.py        |  10 +-
 tensorflow/python/client/device_lib_test.py   |   5 +
 tensorflow/python/estimator/estimator.py      |   1 +
 tensorflow/python/training/distribute.py      |   4 +
 11 files changed, 591 insertions(+), 131 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index e805569a4d..6d4cac71ed 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -360,8 +360,10 @@ py_library(
         ":cross_tower_utils",
         ":values",
         "//tensorflow/python:array_ops",
+        "//tensorflow/python:device_lib",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:platform",
         "//tensorflow/python:training",
         "//tensorflow/python/eager:context",
         "@six_archive//:six",
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py
index cb98351735..68f202ea62 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py
@@ -22,6 +22,7 @@ import six
 
 from tensorflow.contrib.distribute.python import cross_tower_utils
 from tensorflow.contrib.distribute.python import values as value_lib
+from tensorflow.python.client import device_lib
 from tensorflow.python.eager import context
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -231,7 +232,7 @@ class ReductionToOneDeviceCrossTowerOps(CrossTowerOps):
 def _group_value_by_device(per_device_values):
   """Group values into sublists by their devices.
 
-  This grouping is needed to call the allreduce library.
+  This grouping is needed to call the all-reduce library.
 
   Args:
     per_device_values: a list of PerDevice obejcts.
@@ -251,10 +252,20 @@ def _group_value_by_device(per_device_values):
 
 
 def _ungroup_and_make_mirrored(grouped_reduced, destinations, method_string):
-  """Ungroup results from allreduce and make Mirrored objects.
+  """Ungroup results from all-reduce and make Mirrored objects.
 
-  Each allreduce result would be divided by the number of destinations before
+  Each all-reduce result will be divided by the number of destinations before
   Mirrored objects are created if method_string is "mean".
+
+  Args:
+    grouped_reduced: a list of lists, each sublist has components for each
+      device, paired with a None. It is the result from
+      cross_tower_utils.aggregate_gradients_using*.
+    destinations: a list of device strings for returned Mirrored objects.
+    method_string: "mean" or "sum".
+
+  Returns:
+    a list of Mirrored objects.
   """
   index = [{} for _ in range(len(grouped_reduced[0]))]
   for d, per_device_reduced in enumerate(grouped_reduced):
@@ -266,23 +277,171 @@ def _ungroup_and_make_mirrored(grouped_reduced, destinations, method_string):
   return [value_lib.Mirrored(v) for v in index]
 
 
+class ConcatAndSplitPacker(object):
+  """Concatenate and split tensors for reduction."""
+
+  def __init__(self, num_packs=1):
+    """Initialize the ConcatAndSplitPacker object.
+
+    Args:
+      num_packs: specifies the number of split packs that will be
+        formed.
+
+    Raises:
+      ValueError: if num_packs is not greater than 0.
+    """
+    if num_packs <= 0:
+      raise ValueError("num_packs must be greater than zero.")
+    self.num_packs = num_packs
+
+  def pack(self, grouped_grads_and_vars):
+    """Pack tensors."""
+    self.grouped_grads_and_vars = grouped_grads_and_vars
+    self.all_tower_shapes = []
+    self.all_tower_sizes = []
+
+    device_grad_packs = []
+    for tower_grads_and_vars in grouped_grads_and_vars:
+      with ops.colocate_with(tower_grads_and_vars[0][0]):
+        # Flatten all the grads.
+        flat_grads = [
+            array_ops.reshape(g, [-1]) for g, _ in tower_grads_and_vars
+        ]
+        # Remember the original shape of all the grads.
+        tower_shapes = [array_ops.shape(g) for g, _ in tower_grads_and_vars]
+        # Remember the original sizes of all the grads.
+        tower_sizes = [array_ops.size(g) for g, _ in tower_grads_and_vars]
+        # Concat all the flat grads into a big flat tensor.
+        concat_grads = array_ops.concat(flat_grads, 0)
+
+        # Split the big tensor into num_splits packs. In cases where the
+        # total size is not divisible num_splits, the last pack gets
+        # more elements.
+        # TODO(zhengxq): it is also possible to optimize away all the concat
+        # as well.
+        num_splits = self.num_packs
+        total_grad_size = array_ops.size(concat_grads)
+        split_size = total_grad_size // num_splits
+        split_size_last = total_grad_size - split_size * (num_splits - 1)
+        split_sizes = [split_size] * (num_splits - 1) + [split_size_last]
+        grad_packs = array_ops.split(concat_grads, split_sizes)
+
+        # Ready to aggregate the repacked gradients, with fake variables.
+        # TODO(zhengxq): It is hacky to have to use fake variables.
+        # We should remove the need for variables in
+        # aggregate_gradients_using*.
+        device_grad_packs.append(zip(grad_packs, [None] * num_splits))
+        self.all_tower_shapes.append(tower_shapes)
+        self.all_tower_sizes.append(tower_sizes)
+
+    return device_grad_packs
+
+  def unpack(self, summed_device_grad_packs):
+    """Reverse the pack."""
+    aggregated_device_grads = []
+    for (summed_tower_grad_packs,
+         tower_grads_and_vars, tower_shapes, tower_sizes) in zip(
+             summed_device_grad_packs, self.grouped_grads_and_vars,
+             self.all_tower_shapes, self.all_tower_sizes):
+      # pylint: enable=line-too-long
+      # Reverse the packing operations in the previous steps. Form the
+      # summed gradients back into their original shapes.
+      with ops.colocate_with(summed_tower_grad_packs[0][0]):
+        # Form a list of the summed grad packs.
+        device_grad_packs = [g for g, _ in summed_tower_grad_packs]
+
+        # Concat them back into a big flat tensor.
+        device_grads_concat = array_ops.concat(device_grad_packs, 0)
+
+        # Split the tensors back into their original sizes.
+        grads_with_sizes = array_ops.split(device_grads_concat, tower_sizes)
+
+        # Reshape the tensors back into their original shapes.
+        grads_with_shapes = [
+            array_ops.reshape(grad, shape)
+            for shape, grad in zip(tower_shapes, grads_with_sizes)
+        ]
+
+        # Form the list with the original list of variables.
+        summed_tower_grads = [
+            (g, v) for g, (_, v) in zip(grads_with_shapes, tower_grads_and_vars)
+        ]
+        aggregated_device_grads.append(summed_tower_grads)
+    return aggregated_device_grads
+
+
+class AggregateSmallTensorPacker(object):
+  """Concatenate small gradient tensors together for reduction."""
+
+  def __init__(self,
+               agg_small_grads_max_bytes=1048576,
+               agg_small_grads_max_group=16):
+    """Initialize the AggregateSmallTensorPacker object.
+
+    Args:
+      agg_small_grads_max_bytes: largest tensor eligible for aggregation,
+        in number of bytes.
+      agg_small_grads_max_group: largest permitted aggregation of small
+        tensors.
+
+    Raises:
+      ValueError: if `agg_small_grads_max_bytes` or `agg_small_grads_max_group`
+        is not greater than 0.
+    """
+    if agg_small_grads_max_bytes <= 0 or agg_small_grads_max_group <= 0:
+      raise ValueError("agg_small_grads_max_bytes and agg_small_grads_max_group"
+                       " should both be greater than zero.")
+    self.agg_small_grads_max_bytes = agg_small_grads_max_bytes
+    self.agg_small_grads_max_group = agg_small_grads_max_group
+
+  def pack(self, grouped_grads_and_vars):
+    """Aggregate small tensors."""
+    if (self.agg_small_grads_max_bytes > 0 and
+        self.agg_small_grads_max_group > 0):
+      tower_grads, self.packing = cross_tower_utils.pack_small_tensors(
+          grouped_grads_and_vars,
+          max_bytes=self.agg_small_grads_max_bytes,
+          max_group=self.agg_small_grads_max_group)
+    return tower_grads
+
+  def unpack(self, summed_device_grad_packs):
+    """Reverse the aggregation process."""
+    return cross_tower_utils.unpack_small_tensors(summed_device_grad_packs,
+                                                  self.packing)
+
+
 class AllReduceCrossTowerOps(CrossTowerOps):
   """Reduction using all reduce."""
 
-  def __init__(self, all_reduce_alg="nccl", gradient_repacking=1):
-    """Initialize this subclass of CrossTowerOps with allreduce.
-
-    Gradients would be repacked for more efficient cross-device transportation.
+  def __init__(self,
+               all_reduce_alg="nccl",
+               num_packs=1,
+               agg_small_grads_max_bytes=0,
+               agg_small_grads_max_group=10):
+    """All-reduce implementation of CrossTowerOps.
+
+    Before performing all-reduce, tensors will be repacked or aggregated for
+    more efficient cross-device transportation:
+      1) If `num_packs` is non-zero, pack values into
+        `num_packs` splits.
+      2) Otherwise, if `agg_small_grads_max_bytes` > 0 and
+        `agg_small_grads_max_group` > 0, aggregate values smaller than
+        `agg_small_grads_max_bytes` into groups with at most
+        `agg_small_grads_max_group` values.
+      3) Otherwise, no repacking or grouping will happen.
 
     Args:
-      all_reduce_alg: the allreduce algorithm to use, currently only "nccl" or
+      all_reduce_alg: the all-reduce algorithm to use, currently only "nccl" or
         "hierarchical_copy" are supported.
-      gradient_repacking: If zero, no gradient repacking would be done. If
-        non-zero value it specifies the number of split packs that will be
-        formed.
+      num_packs: see above.
+      agg_small_grads_max_bytes: see above.
+      agg_small_grads_max_group: see above.
+        tensors.
     """
     self.all_reduce_alg = all_reduce_alg
-    self.gradient_repacking = gradient_repacking
+    self.num_packs = num_packs
+    self.agg_small_grads_max_bytes = agg_small_grads_max_bytes
+    self.agg_small_grads_max_group = agg_small_grads_max_group
     super(AllReduceCrossTowerOps, self).__init__()
 
   def _reduce(self, method_string, per_device_value, destinations):
@@ -312,99 +471,115 @@ class AllReduceCrossTowerOps(CrossTowerOps):
 
   def _batch_all_reduce(self, method_string, per_device_values):
     """All reduce algorithm in a batch."""
-    logging.info("batch_all_reduce invoked for batches size = %d with algorithm"
-                 " = %s and gradient repacking = %d", len(per_device_values),
-                 self.all_reduce_alg, self.gradient_repacking)
     destinations = per_device_values[0].devices
     grouped = _group_value_by_device(per_device_values)
-    if self.gradient_repacking == 0:
-      if self.all_reduce_alg == "nccl":
-        reduced = cross_tower_utils.aggregate_gradients_using_nccl(grouped)
-      else:
-        # TODO(yuefengz): check that gpu ids in `destinations` are in ascending
-        # order.
-        reduced = (
-            cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
-                destinations, grouped))
+    if self.num_packs > 0:
+      logging.info(
+          "batch_all_reduce invoked for batches size = %d with "
+          "algorithm = %s and num_packs = %d", len(per_device_values),
+          self.all_reduce_alg, self.num_packs)
+      tensor_packer = ConcatAndSplitPacker(self.num_packs)
+      device_grad_packs = tensor_packer.pack(grouped)
+    elif (self.agg_small_grads_max_bytes > 0 and
+          self.agg_small_grads_max_group > 0):
+      logging.info(
+          "batch_all_reduce invoked for batches size = %d with "
+          "algorithm = %s, agg_small_grads_max_bytes = %d and "
+          "agg_small_grads_max_group = %d", len(per_device_values),
+          self.all_reduce_alg, self.agg_small_grads_max_bytes,
+          self.agg_small_grads_max_group)
+      tensor_packer = AggregateSmallTensorPacker(100, 10)
+      device_grad_packs = tensor_packer.pack(grouped)
     else:
-      device_grad_packs = []
-      all_tower_shapes = []
-      all_tower_sizes = []
-      for tower_grads_and_vars in grouped:
-        with ops.colocate_with(tower_grads_and_vars[0][0]):
-          # Flatten all the grads.
-          flat_grads = [
-              array_ops.reshape(g, [-1]) for g, _ in tower_grads_and_vars
-          ]
-          # Remember the original shape of all the grads.
-          tower_shapes = [array_ops.shape(g) for g, _ in tower_grads_and_vars]
-          # Remember the original sizes of all the grads.
-          tower_sizes = [array_ops.size(g) for g, _ in tower_grads_and_vars]
-          # Concat all the flat grads into a big flat tensor.
-          concat_grads = array_ops.concat(flat_grads, 0)
-
-          # Split the big tensor into num_splits packs. In cases where the
-          # total size is not divisible num_splits, the last pack gets
-          # more elements.
-          # TODO(zhengxq): it is possible to optimize away the additional
-          # data movement by copying along the original variable boundary.
-          # TODO(zhengxq): it is also possible to optimize away all the concat
-          # as well.
-          num_splits = self.gradient_repacking
-          total_grad_size = array_ops.size(concat_grads)
-          split_size = total_grad_size // num_splits
-          split_size_last = total_grad_size - split_size * (num_splits - 1)
-          split_sizes = [split_size] * (num_splits - 1) + [split_size_last]
-          grad_packs = array_ops.split(concat_grads, split_sizes)
-
-          # Ready to aggregate the repacked gradients, with fake variables.
-          # TODO(zhengxq): It is hacky to have to use fake variables.
-          # We should remove the need for variables in
-          # aggregate_gradients_using*.
-          device_grad_packs.append(zip(grad_packs, [None] * num_splits))
-          all_tower_shapes.append(tower_shapes)
-          all_tower_sizes.append(tower_sizes)
-
-      # The actual aggregation of the repacked gradients. Note that they are
-      # sharded among different aggregation trees. So it is important to
-      # strike the balance on num_splits.
-      if self.all_reduce_alg == "nccl":
-        summed_device_grad_packs = (
-            cross_tower_utils.aggregate_gradients_using_nccl(device_grad_packs))
-      else:
-        summed_device_grad_packs = (
-            cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
-                destinations, device_grad_packs))
-
-      aggregated_device_grads = []
-      for (summed_tower_grad_packs, tower_grads_and_vars, tower_shapes,
-           tower_sizes) in zip(summed_device_grad_packs, grouped,
-                               all_tower_shapes, all_tower_sizes):
-        # pylint: enable=line-too-long
-        # Reverse the packing operations in the previous steps. Form the
-        # summed gradients back into their original shapes.
-        with ops.colocate_with(summed_tower_grad_packs[0][0]):
-          # Form a list of the summed grad packs.
-          device_grad_packs = [g for g, _ in summed_tower_grad_packs]
-
-          # Concat them back into a big flat tensor.
-          device_grads_concat = array_ops.concat(device_grad_packs, 0)
-
-          # Split the tensors back into their original sizes.
-          grads_with_sizes = array_ops.split(device_grads_concat, tower_sizes)
-
-          # Reshape the tensors back into their original shapes.
-          grads_with_shapes = [
-              array_ops.reshape(grad, shape)
-              for shape, grad in zip(tower_shapes, grads_with_sizes)
-          ]
-
-          # Form the list with the original list of variables.
-          summed_tower_grads = [
-              (g, v)
-              for g, (_, v) in zip(grads_with_shapes, tower_grads_and_vars)
-          ]
-          aggregated_device_grads.append(summed_tower_grads)
-      reduced = aggregated_device_grads
+      logging.info(
+          "batch_all_reduce invoked for batches size = %d with algorithm = %s",
+          len(per_device_values), self.all_reduce_alg)
+      tensor_packer = None
+      device_grad_packs = grouped
+
+    # The actual aggregation of the repacked gradients. Note that they are
+    # sharded among different aggregation trees. So it is important to strike
+    # the balance on num_splits.
+    if self.all_reduce_alg == "nccl":
+      reduced = cross_tower_utils.aggregate_gradients_using_nccl(
+          device_grad_packs)
+    else:
+      # TODO(yuefengz): check that gpu ids in `destinations` are in ascending
+      # order.
+      reduced = (
+          cross_tower_utils.aggregate_gradients_using_hierarchical_copy(
+              destinations, device_grad_packs))
+
+    if tensor_packer:
+      reduced = tensor_packer.unpack(reduced)
+
     return _ungroup_and_make_mirrored(reduced, per_device_values[0].devices,
                                       method_string)
+
+
+_dgx1_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7],
+               [0, 5, 6, 7], [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6]]
+
+
+def _has_dgx1_like_links(gpu_links):
+  if not gpu_links:
+    return False
+  # TODO(yuefengz): figure out the right topology for hierarchial copy if
+  # number of gpus are less than 8.
+  if len(gpu_links) < 8:
+    return False
+  for i, (gpu_link, dgx1_link) in enumerate(zip(gpu_links, _dgx1_links)):
+    if (set(gpu_link) != set(dgx1_link) and
+        set(gpu_link) != set(dgx1_link + [i])):
+      return False
+  return True
+
+
+def _choose_all_reduce_algorithm(device_links):
+  if _has_dgx1_like_links(device_links):
+    logging.info("Configured hierarchical_copy with num_packs=%d",
+                 len(device_links))
+    return AllReduceCrossTowerOps(
+        "hierarchical_copy", num_packs=len(device_links))
+  else:
+    logging.info("Configured nccl all-reduce.")
+    return AllReduceCrossTowerOps("nccl", num_packs=1)
+
+
+def choose_the_best(devices, session_config=None):
+  """Find the best subclass of CrossTowerOps given a tensorflow session.
+
+  Args:
+    devices: a list of devices passed for distribute strategy.
+    session_config: a tensorflow session config or None. If None, it will make
+      deciesion based on all local devices.
+
+  Returns:
+    a subclass of CrossTowerOps.
+  """
+  requested_devices = set([device_util.canonicalize(d) for d in devices])
+  machine_devices = device_lib.list_local_devices(session_config=session_config)
+  using_devices = []
+  for d in machine_devices:
+    if device_util.canonicalize(d.name) in requested_devices:
+      using_devices.append(d)
+    else:
+      logging.info(
+          "Device is available but not used by distribute strategy: %s", d.name)
+
+  if len(using_devices) != len(requested_devices):
+    logging.warning("Not all devices in distribute strategy are visible by "
+                    "TensorFlow sessions.")
+    return ReductionToOneDeviceCrossTowerOps()
+
+  if any([d.device_type.lower() != "gpu" for d in using_devices]):
+    logging.warning("Not all devices in DistributionStrategy are visible to "
+                    "TensorFlow session.")
+    return ReductionToOneDeviceCrossTowerOps()
+
+  device_links = [[] for _ in range(len(using_devices))]
+  for i, device in enumerate(using_devices):
+    for link in device.locality.links.link:
+      device_links[i].append(link.device_id)
+
+  return _choose_all_reduce_algorithm(device_links)
diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
index bb43147f5e..7c7b087088 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops_test.py
@@ -101,22 +101,22 @@ class CrossTowerOpsTest(test.TestCase, parameterized.TestCase):
       mode=["graph", "eager"])
   allreduce_combinations = combinations.combine(
       cross_tower_ops=[
-          combinations.NamedObject("AllReduce",
-                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
-                                       "nccl", 1)),
-          combinations.NamedObject("HierarchicalCopy",
-                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
-                                       "hierarchical_copy", 8)),
-          combinations.NamedObject("AllReduceNoGradientRepacking",
-                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
-                                       "nccl", 0)),
-          combinations.NamedObject("HierarchicalCopyNoGradientRepacking",
-                                   cross_tower_ops_lib.AllReduceCrossTowerOps(
-                                       "hierarchical_copy", 0))
-      ],
-      distribution=[
-          combinations.mirrored_strategy_with_two_gpus
+          combinations.NamedObject(
+              "AllReduce",
+              cross_tower_ops_lib.AllReduceCrossTowerOps("nccl", 1, 0, 0)),
+          combinations.NamedObject(
+              "HierarchicalCopy",
+              cross_tower_ops_lib.AllReduceCrossTowerOps(
+                  "hierarchical_copy", 8, 0, 0)),
+          combinations.NamedObject(
+              "AllReduceNoGradientRepacking",
+              cross_tower_ops_lib.AllReduceCrossTowerOps("nccl", 0, 0, 0)),
+          combinations.NamedObject(
+              "HierarchicalCopyAggregateSmallTensors",
+              cross_tower_ops_lib.AllReduceCrossTowerOps(
+                  "hierarchical_copy", 0, 100, 10))
       ],
+      distribution=[combinations.mirrored_strategy_with_two_gpus],
       mode=["graph", "eager"])
 
   @combinations.generate(reduction_to_one_combinations + allreduce_combinations)
@@ -180,6 +180,42 @@ class CrossTowerOpsTest(test.TestCase, parameterized.TestCase):
             cross_tower_ops.broadcast(constant_op.constant(1.), destinations),
             _fake_mirrored(1., destinations))
 
+  def testChooseAlgorithm(self):
+    device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7],
+                    [0, 5, 6, 7], [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6]]
+    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertTrue(
+        isinstance(result, cross_tower_ops_lib.AllReduceCrossTowerOps))
+    self.assertEqual(result.all_reduce_alg, "hierarchical_copy")
+    self.assertEqual(result.num_packs, 8)
+
+    # if there are only 4 devices
+    device_links = [[1, 2, 3, 4], [0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7]]
+    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertTrue(
+        isinstance(result, cross_tower_ops_lib.AllReduceCrossTowerOps))
+    self.assertEqual(result.all_reduce_alg, "nccl")
+    self.assertEqual(result.num_packs, 1)
+
+    # if devices links contain each device itself
+    device_links = [[0, 1, 2, 3, 4], [0, 1, 2, 3, 5], [0, 1, 2, 3, 6],
+                    [0, 1, 2, 3, 7], [0, 4, 5, 6, 7], [1, 4, 5, 6, 7],
+                    [2, 4, 5, 6, 7], [3, 4, 5, 6, 7]]
+    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertTrue(
+        isinstance(result, cross_tower_ops_lib.AllReduceCrossTowerOps))
+    self.assertEqual(result.all_reduce_alg, "hierarchical_copy")
+    self.assertEqual(result.num_packs, 8)
+
+    # if not dgx1-like links
+    device_links = [[0, 2, 3, 5], [0, 1, 3, 6], [0, 1, 2, 7], [0, 5, 6, 7],
+                    [1, 4, 6, 7], [2, 4, 5, 7], [3, 4, 5, 6], [1, 2, 3, 4]]
+    result = cross_tower_ops_lib._choose_all_reduce_algorithm(device_links)
+    self.assertTrue(
+        isinstance(result, cross_tower_ops_lib.AllReduceCrossTowerOps))
+    self.assertEqual(result.all_reduce_alg, "nccl")
+    self.assertEqual(result.num_packs, 1)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py
index 93acd835d7..0dc6b8db6b 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py
@@ -18,7 +18,10 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import collections as pycoll
+
 from tensorflow.contrib import nccl
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
@@ -151,3 +154,186 @@ def aggregate_single_gradient_using_copy(grad_and_vars, use_mean,
     return (grad, v), has_nan_or_inf
   else:
     return (grad, v), None
+
+
+def extract_ranges(index_list, range_size_limit=32):
+  """Extract consecutive ranges and singles from index_list.
+
+  Args:
+    index_list: List of monotone increasing non-negative integers.
+    range_size_limit: Largest size range to return.  If a larger
+      consecutive range exists, it will be returned as multiple
+      ranges.
+
+  Returns:
+    (ranges, singles) where ranges is a list of [first, last] pairs of
+      consecutive elements in index_list, and singles is all of the
+      other elements, in original order.
+  """
+  if not index_list:
+    return [], []
+  first = index_list[0]
+  last = first
+  ranges = []
+  singles = []
+  for i in index_list[1:]:
+    if i == last + 1 and (last - first) <= range_size_limit:
+      last = i
+    else:
+      if last > first:
+        ranges.append([first, last])
+      else:
+        singles.append(first)
+      first = i
+      last = i
+  if last > first:
+    ranges.append([first, last])
+  else:
+    singles.append(first)
+  return ranges, singles
+
+
+GradPackTuple = pycoll.namedtuple('GradPackTuple', 'indices vars shapes')
+
+
+def pack_range(key, packing, grad_vars, rng):
+  """Form the concatenation of a specified range of gradient tensors.
+
+  Args:
+    key: Value under which to store meta-data in packing that will be used
+      later to restore the grad_var list structure.
+    packing: Dict holding data describing packed ranges of small tensors.
+    grad_vars: List of (grad, var) pairs for one tower.
+    rng: A pair of integers giving the first, last indices of a consecutive
+      range of tensors to be packed.
+
+  Returns:
+    A tensor that is the concatenation of all the specified small tensors.
+  """
+  to_pack = grad_vars[rng[0]:rng[1] + 1]
+  members = []
+  variables = []
+  restore_shapes = []
+  with ops.name_scope('pack'):
+    for g, v in to_pack:
+      variables.append(v)
+      restore_shapes.append(g.shape)
+      with ops.device(g.device):
+        members.append(array_ops.reshape(g, [-1]))
+    packing[key] = GradPackTuple(
+        indices=range(rng[0], rng[1] + 1),
+        vars=variables,
+        shapes=restore_shapes)
+    with ops.device(members[0].device):
+      return array_ops.concat(members, 0)
+
+
+def unpack_grad_tuple(gv, gpt):
+  """Unpack a previously packed collection of gradient tensors.
+
+  Args:
+    gv: A (grad, var) pair to be unpacked.
+    gpt: A GradPackTuple describing the packing operation that produced gv.
+
+  Returns:
+    A list of (grad, var) pairs corresponding to the values that were
+     originally packed into gv, maybe following subsequent operations like
+     reduction.
+  """
+  elt_widths = [x.num_elements() for x in gpt.shapes]
+  with ops.device(gv[0][0].device):
+    with ops.name_scope('unpack'):
+      splits = array_ops.split(gv[0], elt_widths)
+      unpacked_gv = []
+      for idx, s in enumerate(splits):
+        unpacked_gv.append((array_ops.reshape(s, gpt.shapes[idx]),
+                            gpt.vars[idx]))
+  return unpacked_gv
+
+
+def pack_small_tensors(tower_grads, max_bytes=0, max_group=0):
+  """Concatenate small gradient tensors together for reduction.
+
+  Args:
+    tower_grads: List of lists of (gradient, variable) tuples.
+    max_bytes: Int giving max number of bytes in a tensor that
+      may be considered small.
+    max_group: Int giving max number of small tensors that may be
+      concatenated into one new tensor.
+
+  Returns:
+    new_tower_grads, packing where new_tower_grads is identical to
+      tower_grads except that all feasible small_tensors have been removed
+      from their places and concatenated into larger tensors that are
+      now in the front of the list for each tower, and packing contains
+      the data necessary to restore the tower_grads structure.
+
+  Look through the first tower for gradients of the same type (float),
+  and small size, that are all sequential.  For each such group,
+  replace by a new tensor that is a flattened concatenation.  Note
+  that the corresponding variable will be absent, which doesn't matter
+  because it isn't used during all-reduce.
+
+  Requires:
+    Every gv_list in towers must have isomorphic structure including identical
+      tensor sizes and types.
+  """
+  small_indices = []
+  large_indices = []
+  for idx, (g, _) in enumerate(tower_grads[0]):
+    if g.dtype == dtypes.float32 and (4 * g.shape.num_elements()) <= max_bytes:
+      small_indices.append(idx)
+    else:
+      large_indices.append(idx)
+  small_ranges, small_singles = extract_ranges(
+      small_indices, range_size_limit=max_group)
+  large_indices = sorted(large_indices + small_singles)
+  num_gv = len(tower_grads[0])
+  packing = {}
+  if small_ranges:
+    new_tower_grads = []
+    for dev_idx, gv_list in enumerate(tower_grads):
+      assert len(gv_list) == num_gv
+      new_gv_list = []
+      for r in small_ranges:
+        key = '%d:%d' % (dev_idx, len(new_gv_list))
+        new_gv_list.append((pack_range(key, packing, gv_list, r),
+                            'packing_var_placeholder'))
+      for i in large_indices:
+        new_gv_list.append(gv_list[i])
+      new_tower_grads.append(new_gv_list)
+    return new_tower_grads, packing
+  else:
+    return tower_grads, None
+
+
+def unpack_small_tensors(tower_grads, packing):
+  """Undo the structure alterations to tower_grads done by pack_small_tensors.
+
+  Args:
+    tower_grads: List of List of (grad, var) tuples.
+    packing: A dict generated by pack_small_tensors describing the changes
+      it made to tower_grads.
+
+  Returns:
+    new_tower_grads: identical to tower_grads except that concatentations
+      of small tensors have been split apart and returned to their original
+      positions, paired with their original variables.
+  """
+  if not packing:
+    return tower_grads
+  new_tower_grads = []
+  num_devices = len(tower_grads)
+  num_packed = len(packing.keys()) // num_devices
+  for dev_idx, gv_list in enumerate(tower_grads):
+    gv_list = list(gv_list)
+    new_gv_list = gv_list[num_packed:]
+    for i in xrange(0, num_packed):
+      k = '%d:%d' % (dev_idx, i)
+      gpt = packing[k]
+      gv = unpack_grad_tuple(gv_list[i], gpt)
+      for gi, idx in enumerate(gpt.indices):
+        assert idx == gpt.indices[gi]
+        new_gv_list.insert(idx, gv[gi])
+    new_tower_grads.append(new_gv_list)
+  return new_tower_grads
diff --git a/tensorflow/contrib/distribute/python/mirrored_strategy.py b/tensorflow/contrib/distribute/python/mirrored_strategy.py
index 8cf83c52d8..eb0edb3a11 100644
--- a/tensorflow/contrib/distribute/python/mirrored_strategy.py
+++ b/tensorflow/contrib/distribute/python/mirrored_strategy.py
@@ -78,9 +78,7 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
         [device_util.canonicalize(d) for d in devices])
     self._device_index = values.PerDevice(
         dict((d, i) for i, d in enumerate(devices)))
-    self.cross_tower_ops = (
-        cross_tower_ops or
-        cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps())
+    self._cross_tower_ops = cross_tower_ops
     self._prefetch_on_device = prefetch_on_device
 
   def _create_variable(self, next_creator, *args, **kwargs):
@@ -149,7 +147,8 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
 
   def _broadcast(self, tensor, destinations):
     # TODO(josh11b): In eager mode, use one thread per device, or async mode.
-    return self.cross_tower_ops.broadcast(tensor, destinations or self._devices)
+    return self._get_cross_tower_ops().broadcast(tensor, destinations or
+                                                 self._devices)
 
   def _call_for_each_tower(self, fn, *args, **kwargs):
     """Run `fn` in separate threads, once per tower/worker device.
@@ -272,16 +271,28 @@ class MirroredStrategy(distribute_lib.DistributionStrategy):
     # in addition to PerDevice data.
     return values.PerDevice({k: values.MapOutput(v) for k, v in index.items()})
 
+  def configure(self, session_config=None):
+    if self._cross_tower_ops is None:
+      self._cross_tower_ops = cross_tower_ops_lib.choose_the_best(
+          self._devices, session_config=session_config)
+
+  def _get_cross_tower_ops(self):
+    if self._cross_tower_ops is None:
+      self._cross_tower_ops = (
+          cross_tower_ops_lib.ReductionToOneDeviceCrossTowerOps())
+    return self._cross_tower_ops
+
   def _reduce(self, method_string, value, destinations):
     if len(self._devices) == 1 and not isinstance(value, values.PerDevice):
       value = values.PerDevice({self._devices[0]: value})
     assert isinstance(value, values.PerDevice)
-    return self.cross_tower_ops.reduce(
+
+    return self._get_cross_tower_ops().reduce(
         method_string, value, destinations=destinations)
 
   def _batch_reduce(self, method_string, value_destination_pairs):
-    return self.cross_tower_ops.batch_reduce(method_string,
-                                             value_destination_pairs)
+    return self._get_cross_tower_ops().batch_reduce(method_string,
+                                                    value_destination_pairs)
 
   def _update(self, var, fn, *args, **kwargs):
     # TODO(josh11b): Also support TowerLocalVariables here? If so, args and
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b05d87635f..fa63a0525d 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3253,6 +3253,7 @@ cuda_py_tests(
         ":client_testlib",
         ":framework_test_lib",
         ":platform_test",
+        "//tensorflow/core:protos_all_py",
     ],
 )
 
diff --git a/tensorflow/python/client/device_lib.i b/tensorflow/python/client/device_lib.i
index 51c04584a5..944e855cee 100644
--- a/tensorflow/python/client/device_lib.i
+++ b/tensorflow/python/client/device_lib.i
@@ -15,19 +15,39 @@ limitations under the License.
 
 %include "tensorflow/python/platform/base.i"
 
+%typemap(in) const tensorflow::ConfigProto& (tensorflow::ConfigProto temp) {
+  char* c_string;
+  Py_ssize_t py_size;
+  if (PyBytes_AsStringAndSize($input, &c_string, &py_size) == -1) {
+    // Python has raised an error (likely TypeError or UnicodeEncodeError).
+    SWIG_fail;
+  }
+
+  if (!temp.ParseFromString(string(c_string, py_size))) {
+    PyErr_SetString(
+        PyExc_TypeError,
+        "The ConfigProto could not be parsed as a valid protocol buffer");
+    SWIG_fail;
+  }
+  $1 = &temp;
+}
+
 %{
 #include "tensorflow/core/common_runtime/device.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
 #include "tensorflow/core/framework/device_attributes.pb.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/protobuf/config.pb.h"
 #include "tensorflow/core/public/session_options.h"
 
 namespace tensorflow {
 namespace swig {
 
-static std::vector<string> ListDevices(TF_Status* out_status) {
+static std::vector<string> ListDevicesWithSessionConfig(
+    const tensorflow::ConfigProto& config, TF_Status* out_status) {
   std::vector<string> output;
   SessionOptions options;
+  options.config = config;
   std::vector<Device*> devices;
   Status status = DeviceFactory::AddDevices(
       options, "" /* name_prefix */, &devices);
@@ -35,7 +55,8 @@ static std::vector<string> ListDevices(TF_Status* out_status) {
     Set_TF_Status_from_Status(out_status, status);
   }
 
-  std::vector<std::unique_ptr<Device>> device_holder(devices.begin(), devices.end());
+  std::vector<std::unique_ptr<Device>> device_holder(devices.begin(),
+                                                     devices.end());
 
   for (const Device* device : devices) {
     const DeviceAttributes& attr = device->attributes();
@@ -53,6 +74,11 @@ static std::vector<string> ListDevices(TF_Status* out_status) {
   return output;
 }
 
+std::vector<string> ListDevices(TF_Status* out_status) {
+  tensorflow::ConfigProto session_config;
+  return ListDevicesWithSessionConfig(session_config, out_status);
+}
+
 }  // namespace swig
 }  // namespace tensorflow
 
@@ -62,21 +88,28 @@ static std::vector<string> ListDevices(TF_Status* out_status) {
 
 %unignore tensorflow;
 %unignore tensorflow::swig;
+%unignore tensorflow::swig::ListDevicesWithSessionConfig;
 %unignore tensorflow::swig::ListDevices;
 
 // Wrap this function
 namespace tensorflow {
 namespace swig {
 std::vector<string> ListDevices(TF_Status* out_status);
+static std::vector<string> ListDevicesWithSessionConfig(
+    const tensorflow::ConfigProto& config, TF_Status* out_status);
 }  // namespace swig
 }  // namespace tensorflow
 
 %insert("python") %{
-def list_devices():
+def list_devices(session_config=None):
   from tensorflow.python.framework import errors
 
   with errors.raise_exception_on_not_ok_status() as status:
-    return ListDevices(status)
+    if session_config:
+      return ListDevicesWithSessionConfig(session_config.SerializeToString(),
+                                          status)
+    else:
+      return ListDevices(status)
 %}
 
 %unignoreall
diff --git a/tensorflow/python/client/device_lib.py b/tensorflow/python/client/device_lib.py
index ad430cbae5..9d90d5395e 100644
--- a/tensorflow/python/client/device_lib.py
+++ b/tensorflow/python/client/device_lib.py
@@ -22,9 +22,12 @@ from tensorflow.core.framework import device_attributes_pb2
 from tensorflow.python import pywrap_tensorflow
 
 
-def list_local_devices():
+def list_local_devices(session_config=None):
   """List the available devices available in the local process.
 
+  Args:
+    session_config: a session config proto or None to use the default config.
+
   Returns:
     A list of `DeviceAttribute` protocol buffers.
   """
@@ -33,4 +36,7 @@ def list_local_devices():
     m.ParseFromString(pb_str)
     return m
 
-  return [_convert(s) for s in pywrap_tensorflow.list_devices()]
+  return [
+      _convert(s)
+      for s in pywrap_tensorflow.list_devices(session_config=session_config)
+  ]
diff --git a/tensorflow/python/client/device_lib_test.py b/tensorflow/python/client/device_lib_test.py
index aaf41626ab..fec41f50b6 100644
--- a/tensorflow/python/client/device_lib_test.py
+++ b/tensorflow/python/client/device_lib_test.py
@@ -18,6 +18,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import device_lib
 from tensorflow.python.framework import test_util
 from tensorflow.python.platform import googletest
@@ -31,6 +32,10 @@ class DeviceLibTest(test_util.TensorFlowTestCase):
     self.assertGreater(len(devices), 0)
     self.assertEqual(devices[0].device_type, "CPU")
 
+    devices = device_lib.list_local_devices(config_pb2.ConfigProto())
+    self.assertGreater(len(devices), 0)
+    self.assertEqual(devices[0].device_type, "CPU")
+
     # GPU test
     if test.is_gpu_available():
       self.assertGreater(len(devices), 1)
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index 6ce935c730..ab69a093a2 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -859,6 +859,7 @@ class Estimator(object):
                                              saving_listeners)
 
   def _train_model_distributed(self, input_fn, hooks, saving_listeners):
+    self._distribution.configure(self._session_config)
     worker_hooks = []
     with ops.Graph().as_default() as g:
       with self._distribution.scope():
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index 38b0580cd8..bfab9d7be2 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -1105,6 +1105,10 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     # in contrib.
     return dataset.make_one_shot_iterator()
 
+  def configure(self, session_config=None):
+    """Find the best configuration given a tensorflow session config."""
+    del session_config
+
   def _broadcast(self, tensor, destinations):
     if destinations is None:
       return tensor
-- 
GitLab


From df847112f0a8805dab02cc5581870a8460032ef3 Mon Sep 17 00:00:00 2001
From: Anjali Sridhar <anjalisridhar@google.com>
Date: Thu, 29 Mar 2018 19:57:27 -0700
Subject: [PATCH 737/960] Internal Change.

PiperOrigin-RevId: 191024708
---
 .../contrib/distribute/python/examples/BUILD  | 11 ++++
 .../python/examples/simple_tfkeras_example.py | 62 +++++++++++++++++++
 .../python/keras/_impl/keras/optimizers.py    | 29 +++++++--
 3 files changed, 96 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py

diff --git a/tensorflow/contrib/distribute/python/examples/BUILD b/tensorflow/contrib/distribute/python/examples/BUILD
index 27eb3c079d..cbfd178502 100644
--- a/tensorflow/contrib/distribute/python/examples/BUILD
+++ b/tensorflow/contrib/distribute/python/examples/BUILD
@@ -17,3 +17,14 @@ py_binary(
         "//tensorflow:tensorflow_py",
     ],
 )
+
+py_binary(
+    name = "simple_tfkeras_example",
+    srcs = [
+        "simple_tfkeras_example.py",
+    ],
+    deps = [
+        "//tensorflow:tensorflow_py",
+        "//third_party/py/numpy",
+    ],
+)
diff --git a/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py b/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py
new file mode 100644
index 0000000000..e714255f69
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py
@@ -0,0 +1,62 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""An example tf.keras model that is trained using MirroredStrategy."""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+from sys import argv
+import numpy as np
+import tensorflow as tf
+
+
+def input_fn():
+  x = np.random.random((1024, 10))
+  y = np.random.randint(2, size=(1024, 1))
+  x = tf.cast(x, tf.float32)
+  dataset = tf.data.Dataset.from_tensor_slices((x, y))
+  dataset = dataset.repeat(10)
+  dataset = dataset.batch(32)
+  return dataset
+
+
+def main(args):
+  if len(args) < 2:
+    print('You must specify  model_dir for checkpoints such as'
+          ' /tmp/tfkeras_example./')
+    return
+
+  print('Using %s to store checkpoints.' % args[1])
+
+  strategy = tf.contrib.distribute.MirroredStrategy(
+      ['/device:GPU:0', '/device:GPU:1'])
+  config = tf.estimator.RunConfig(distribute=strategy)
+  optimizer = tf.train.GradientDescentOptimizer(0.2)
+
+  model = tf.keras.Sequential()
+  model.add(tf.keras.layers.Dense(16, activation='relu', input_shape=(10,)))
+  model.add(tf.keras.layers.Dense(1, activation='sigmoid'))
+
+  model.compile(loss='binary_crossentropy', optimizer=optimizer)
+  model.summary()
+  tf.keras.backend.set_learning_phase(True)
+  keras_estimator = tf.keras.estimator.model_to_estimator(
+      keras_model=model, config=config, model_dir=args[1])
+
+  keras_estimator.train(input_fn=input_fn, steps=10)
+  eval_result = keras_estimator.evaluate(input_fn=input_fn)
+  print('Eval result: {}'.format(eval_result))
+
+if __name__ == '__main__':
+  tf.app.run(argv=argv)
diff --git a/tensorflow/python/keras/_impl/keras/optimizers.py b/tensorflow/python/keras/_impl/keras/optimizers.py
index b715d722b9..acbb9091d3 100644
--- a/tensorflow/python/keras/_impl/keras/optimizers.py
+++ b/tensorflow/python/keras/_impl/keras/optimizers.py
@@ -31,7 +31,9 @@ from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import optimizer as tf_optimizer_module
+from tensorflow.python.training import training_util
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -728,12 +730,27 @@ class TFOptimizer(Optimizer):
     return self.optimizer.compute_gradients(loss, params)
 
   def get_updates(self, loss, params):
-    self.updates = [K.update_add(self.iterations, 1)]
-    if not params:
-      return self.updates
-    grads = self.optimizer.compute_gradients(loss, params)
-    opt_update = self.optimizer.apply_gradients(
-        grads, global_step=self.iterations)
+    if distribute_lib.has_distribution_strategy():
+      self.updates = []
+
+      if not params:
+        # After the model vars have been created, the second call to get_updates
+        # is called with params as an empty list. This ensures that we call
+        # compute_gradients with params=None.
+        grads = self.optimizer.compute_gradients(loss)
+      else:
+        grads = self.optimizer.compute_gradients(loss, params)
+      global_step = training_util.get_global_step()
+      opt_update = self.optimizer.apply_gradients(grads, global_step)
+    else:
+      self.updates = [K.update_add(self.iterations, 1)]
+      if not params:
+        return self.updates
+
+      grads = self.optimizer.compute_gradients(loss, params)
+      opt_update = self.optimizer.apply_gradients(
+          grads, global_step=self.iterations)
+
     self.updates.append(opt_update)
     return self.updates
 
-- 
GitLab


From bf170839d2a8be1b16e0a6c6a74ac2f0dc427f96 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Thu, 29 Mar 2018 19:57:49 -0700
Subject: [PATCH 738/960] Add summaries from only the first tower in
 distributed strategy.

PiperOrigin-RevId: 191024726
---
 tensorflow/python/BUILD                  |  1 +
 tensorflow/python/ops/summary_op_util.py | 11 +++++++++++
 tensorflow/python/ops/summary_ops.py     |  3 +++
 tensorflow/python/summary/summary.py     | 11 +++++++++++
 4 files changed, 26 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index fa63a0525d..aa0acd243c 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4174,6 +4174,7 @@ py_library(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
+        ":constant_op",
         ":errors",
         ":framework",
         ":framework_for_generated_wrappers",
diff --git a/tensorflow/python/ops/summary_op_util.py b/tensorflow/python/ops/summary_op_util.py
index 37b80d5e20..a793f634bd 100644
--- a/tensorflow/python/ops/summary_op_util.py
+++ b/tensorflow/python/ops/summary_op_util.py
@@ -23,6 +23,7 @@ import re
 
 from tensorflow.python.framework import ops
 from tensorflow.python.platform import tf_logging
+from tensorflow.python.training import distribute
 
 
 def collect(val, collections, default_collections):
@@ -42,6 +43,16 @@ def collect(val, collections, default_collections):
 _INVALID_TAG_CHARACTERS = re.compile(r'[^-/\w\.]')
 
 
+def skip_summary():
+  # If using multiple towers in distributed strategy, skip summaries on all
+  # towers except the first one (tower_id=0).
+  # TODO(priyag): Add a new optional argument that will provide multiple
+  # alternatives to override default behavior. (e.g. run on last tower,
+  # compute sum or mean across towers).
+  tower_context = distribute.get_tower_context()
+  return tower_context and tower_context.tower_id > 0
+
+
 def clean_tag(name):
   """Cleans a tag. Removes illegal characters for instance.
 
diff --git a/tensorflow/python/ops/summary_ops.py b/tensorflow/python/ops/summary_ops.py
index 037bc9845a..ec4d4a6e92 100644
--- a/tensorflow/python/ops/summary_ops.py
+++ b/tensorflow/python/ops/summary_ops.py
@@ -18,6 +18,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.core.framework import summary_pb2
+from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import gen_logging_ops
 from tensorflow.python.ops import summary_op_util
@@ -71,6 +72,8 @@ def tensor_summary(name,
 
   serialized_summary_metadata = summary_metadata.SerializeToString()
 
+  if summary_op_util.skip_summary():
+    return constant_op.constant("")
   with summary_op_util.summary_scope(
       name, family, values=[tensor]) as (tag, scope):
     val = gen_logging_ops.tensor_summary_v2(
diff --git a/tensorflow/python/summary/summary.py b/tensorflow/python/summary/summary.py
index 97f2ddfdfc..1286ed6703 100644
--- a/tensorflow/python/summary/summary.py
+++ b/tensorflow/python/summary/summary.py
@@ -50,6 +50,7 @@ from tensorflow.core.util.event_pb2 import TaggedRunMetadata
 
 
 from tensorflow.python.eager import context as _context
+from tensorflow.python.framework import constant_op as _constant_op
 from tensorflow.python.framework import dtypes as _dtypes
 from tensorflow.python.framework import ops as _ops
 from tensorflow.python.ops import gen_logging_ops as _gen_logging_ops
@@ -98,6 +99,8 @@ def scalar(name, tensor, collections=None, family=None):
   Raises:
     ValueError: If tensor has the wrong shape or type.
   """
+  if _summary_op_util.skip_summary():
+    return _constant_op.constant('')
   with _summary_op_util.summary_scope(
       name, family, values=[tensor]) as (tag, scope):
     val = _gen_logging_ops.scalar_summary(tags=tag, values=tensor, name=scope)
@@ -151,6 +154,8 @@ def image(name, tensor, max_outputs=3, collections=None, family=None):
     A scalar `Tensor` of type `string`. The serialized `Summary` protocol
     buffer.
   """
+  if _summary_op_util.skip_summary():
+    return _constant_op.constant('')
   with _summary_op_util.summary_scope(
       name, family, values=[tensor]) as (tag, scope):
     val = _gen_logging_ops.image_summary(
@@ -189,6 +194,8 @@ def histogram(name, values, collections=None, family=None):
     A scalar `Tensor` of type `string`. The serialized `Summary` protocol
     buffer.
   """
+  if _summary_op_util.skip_summary():
+    return _constant_op.constant('')
   with _summary_op_util.summary_scope(
       name, family, values=[values],
       default_name='HistogramSummary') as (tag, scope):
@@ -234,6 +241,8 @@ def audio(name, tensor, sample_rate, max_outputs=3, collections=None,
     A scalar `Tensor` of type `string`. The serialized `Summary` protocol
     buffer.
   """
+  if _summary_op_util.skip_summary():
+    return _constant_op.constant('')
   with _summary_op_util.summary_scope(
       name, family=family, values=[tensor]) as (tag, scope):
     sample_rate = _ops.convert_to_tensor(
@@ -282,6 +291,8 @@ def merge(inputs, collections=None, name=None):
     raise RuntimeError(
         'Merging tf.summary.* ops is not compatible with eager execution. '
         'Use tf.contrib.summary instead.')
+  if _summary_op_util.skip_summary():
+    return _constant_op.constant('')
   name = _summary_op_util.clean_tag(name)
   with _ops.name_scope(name, 'Merge', inputs):
     val = _gen_logging_ops.merge_summary(inputs=inputs, name=name)
-- 
GitLab


From 9db93a8db8fafee8092b30ae33a26248867e9b26 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 21:20:50 -0700
Subject: [PATCH 739/960] Merged commit includes the following changes:
 191029891  by xiejw:

    Fix python script file.

--
191029336  by isaprykin:

    Add .configure method to the whole hierarchy of DistributionStrategies.

--
191026971  by blamb:

    Updates get_started nav and renames the beginner guides to include eager.

--
191025863  by mrry:

    Add private Python API for accessing the C++ Session::*Callable API.

--
191025795  by mikecase:

    Internal Change.

--
191024780  by isaprykin:

    Internal change.

--

PiperOrigin-RevId: 191029891
---
 tensorflow/contrib/distribute/python/BUILD    |  26 +++
 .../contrib/distribute/python/combinations.py |   4 +
 .../python/estimator_integration_test.py      | 127 +++++++++++++++
 tensorflow/docs_src/get_started/eager.md      |   3 +
 .../get_started/get_started_for_beginners.md  |   2 +-
 tensorflow/docs_src/get_started/index.md      |  12 +-
 tensorflow/docs_src/get_started/leftnav_files |   3 +-
 .../get_started/premade_estimators.md         |   3 +-
 tensorflow/python/client/session.py           |  57 +++++++
 tensorflow/python/client/session_test.py      |  12 ++
 tensorflow/python/client/tf_session.i         |  30 ++++
 tensorflow/python/client/tf_session_helper.cc | 150 ++++++++++++++++++
 tensorflow/python/client/tf_session_helper.h  |  25 +++
 tensorflow/python/feature_column/BUILD        |   1 +
 .../python/feature_column/feature_column.py   |   4 +-
 tensorflow/python/training/distribute.py      |  28 +++-
 16 files changed, 471 insertions(+), 16 deletions(-)
 create mode 100644 tensorflow/contrib/distribute/python/estimator_integration_test.py
 create mode 100644 tensorflow/docs_src/get_started/eager.md

diff --git a/tensorflow/contrib/distribute/python/BUILD b/tensorflow/contrib/distribute/python/BUILD
index 6d4cac71ed..78b2b0054a 100644
--- a/tensorflow/contrib/distribute/python/BUILD
+++ b/tensorflow/contrib/distribute/python/BUILD
@@ -260,6 +260,32 @@ cuda_py_test(
     ],
 )
 
+cuda_py_test(
+    name = "estimator_integration_test",
+    srcs = ["estimator_integration_test.py"],
+    additional_deps = [
+        ":combinations",
+        "@absl_py//absl/testing:parameterized",
+        "//third_party/py/numpy",
+        "//tensorflow/contrib/optimizer_v2:training",
+        "//tensorflow/python/data/ops:dataset_ops",
+        "//tensorflow/python/eager:test",
+        "//tensorflow/python/estimator:dnn_linear_combined",
+        "//tensorflow/python/estimator:export_export",
+        "//tensorflow/python/estimator:numpy_io",
+        "//tensorflow/python/estimator:prediction_keys",
+        "//tensorflow/python/estimator:run_config",
+        "//tensorflow/python/feature_column",
+        "//tensorflow/python:framework_ops",
+        "//tensorflow/python:platform",
+        "//tensorflow/python:summary",
+    ],
+    tags = [
+        "multi_and_single_gpu",
+        "no_pip",
+    ],
+)
+
 py_library(
     name = "single_loss_example",
     srcs = ["single_loss_example.py"],
diff --git a/tensorflow/contrib/distribute/python/combinations.py b/tensorflow/contrib/distribute/python/combinations.py
index dd8e7c4376..02b1e7ef9f 100644
--- a/tensorflow/contrib/distribute/python/combinations.py
+++ b/tensorflow/contrib/distribute/python/combinations.py
@@ -255,6 +255,10 @@ one_device_strategy = NamedDistribution(
 mirrored_strategy_with_gpu_and_cpu = NamedDistribution(
     "MirroredCPUAndGPU",
     mirrored_strategy.MirroredStrategy(["/gpu:0", "/cpu:0"]), 1)
+mirrored_strategy_without_prefetch = NamedDistribution(
+    "MirroredCPUAndGPUNoPrefetch",
+    mirrored_strategy.MirroredStrategy(
+        ["/gpu:0", "/cpu:0"], prefetch_on_device=False), 1)
 mirrored_strategy_with_two_gpus = NamedDistribution(
     "Mirrored2GPUs",
     mirrored_strategy.MirroredStrategy(["/gpu:0", "/gpu:1"]), 2)
diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py
new file mode 100644
index 0000000000..9be186a724
--- /dev/null
+++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py
@@ -0,0 +1,127 @@
+# Copyright 2016 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests that show that DistributionStrategy works with canned Estimator."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import shutil
+import tempfile
+from absl.testing import parameterized
+import numpy as np
+import six
+
+from tensorflow.contrib.distribute.python import combinations
+from tensorflow.contrib.optimizer_v2 import adagrad
+from tensorflow.python.data.ops import dataset_ops
+from tensorflow.python.eager import test
+from tensorflow.python.estimator import run_config
+from tensorflow.python.estimator.canned import dnn_linear_combined
+from tensorflow.python.estimator.canned import prediction_keys
+from tensorflow.python.estimator.export import export
+from tensorflow.python.estimator.inputs import numpy_io
+from tensorflow.python.feature_column import feature_column
+from tensorflow.python.framework import ops
+from tensorflow.python.platform import gfile
+from tensorflow.python.summary.writer import writer_cache
+
+
+class DNNLinearCombinedClassifierIntegrationTest(test.TestCase,
+                                                 parameterized.TestCase):
+
+  def setUp(self):
+    self._model_dir = tempfile.mkdtemp()
+
+  def dataset_input_fn(self, x, y, batch_size, shuffle):
+
+    def input_fn():
+      dataset = dataset_ops.Dataset.from_tensor_slices((x, y))
+      if shuffle:
+        dataset = dataset.shuffle(batch_size)
+      dataset = dataset.repeat(10).batch(batch_size)
+      return dataset
+
+    return input_fn
+
+  @combinations.generate(
+      combinations.combine(
+          mode=['graph'],
+          distribution=[
+              combinations.one_device_strategy,
+              combinations.mirrored_strategy_without_prefetch
+          ]))
+  def test_complete_flow_with_mode(self, distribution):
+    label_dimension = 2
+    input_dimension = label_dimension
+    batch_size = 10
+    data = np.linspace(0., 2., batch_size * label_dimension, dtype=np.float32)
+    data = data.reshape(batch_size, label_dimension)
+    train_input_fn = self.dataset_input_fn(
+        x={'x': data},
+        y=data,
+        batch_size=batch_size // len(distribution.worker_devices),
+        shuffle=True)
+    eval_input_fn = numpy_io.numpy_input_fn(
+        x={'x': data}, y=data, batch_size=batch_size, shuffle=False)
+    predict_input_fn = numpy_io.numpy_input_fn(
+        x={'x': data}, batch_size=batch_size, shuffle=False)
+
+    linear_feature_columns = [
+        feature_column.numeric_column('x', shape=(input_dimension,))
+    ]
+    dnn_feature_columns = [
+        feature_column.numeric_column('x', shape=(input_dimension,))
+    ]
+    feature_columns = linear_feature_columns + dnn_feature_columns
+    estimator = dnn_linear_combined.DNNLinearCombinedRegressor(
+        linear_feature_columns=linear_feature_columns,
+        dnn_hidden_units=(2, 2),
+        dnn_feature_columns=dnn_feature_columns,
+        label_dimension=label_dimension,
+        model_dir=self._model_dir,
+        # TODO(isaprykin): Work around the colocate_with error.
+        dnn_optimizer=adagrad.AdagradOptimizer(0.001),
+        linear_optimizer=adagrad.AdagradOptimizer(0.001),
+        config=run_config.RunConfig(distribute=distribution))
+
+    num_steps = 10
+    estimator.train(train_input_fn, steps=num_steps)
+
+    scores = estimator.evaluate(eval_input_fn)
+    self.assertEqual(num_steps, scores[ops.GraphKeys.GLOBAL_STEP])
+    self.assertIn('loss', six.iterkeys(scores))
+
+    predictions = np.array([
+        x[prediction_keys.PredictionKeys.PREDICTIONS]
+        for x in estimator.predict(predict_input_fn)
+    ])
+    self.assertAllEqual((batch_size, label_dimension), predictions.shape)
+
+    feature_spec = feature_column.make_parse_example_spec(feature_columns)
+    serving_input_receiver_fn = export.build_parsing_serving_input_receiver_fn(
+        feature_spec)
+    export_dir = estimator.export_savedmodel(tempfile.mkdtemp(),
+                                             serving_input_receiver_fn)
+    self.assertTrue(gfile.Exists(export_dir))
+
+  def tearDown(self):
+    if self._model_dir:
+      writer_cache.FileWriterCache.clear()
+      shutil.rmtree(self._model_dir)
+
+
+if __name__ == '__main__':
+  test.main()
diff --git a/tensorflow/docs_src/get_started/eager.md b/tensorflow/docs_src/get_started/eager.md
new file mode 100644
index 0000000000..ad89f0154c
--- /dev/null
+++ b/tensorflow/docs_src/get_started/eager.md
@@ -0,0 +1,3 @@
+# Get Started with Eager Execution
+
+[Colab notebook](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/eager.ipynb)
diff --git a/tensorflow/docs_src/get_started/get_started_for_beginners.md b/tensorflow/docs_src/get_started/get_started_for_beginners.md
index f59cebe6c4..fbe0ed74f8 100644
--- a/tensorflow/docs_src/get_started/get_started_for_beginners.md
+++ b/tensorflow/docs_src/get_started/get_started_for_beginners.md
@@ -1,4 +1,4 @@
-# Getting Started for ML Beginners
+# Get Started with Graph Execution
 
 This document explains how to use machine learning to classify (categorize)
 Iris flowers by species.  This document dives deeply into the TensorFlow
diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md
index fb83a770a5..9c58b3b900 100644
--- a/tensorflow/docs_src/get_started/index.md
+++ b/tensorflow/docs_src/get_started/index.md
@@ -1,4 +1,4 @@
-# Getting Started
+# Get Started
 
 If you are new to machine learning, we recommend taking the following online
 course prior to diving into TensorFlow documentation:
@@ -13,10 +13,12 @@ functionality, TensorFlow is mainly designed for deep neural network models.
 TensorFlow provides many APIs. This section focuses on the high-level APIs.
 If you are new to TensorFlow, begin by reading one of the following documents:
 
-  * @{$get_started/get_started_for_beginners}, which is aimed at readers
-    new to machine learning.
-  * @{$get_started/premade_estimators}, which is aimed at readers who have
-    experience in machine learning.
+  * @{$get_started/eager} is for machine learning beginners and uses
+    @{$programmers_guide/eager}.
+  * @{$get_started/get_started_for_beginners} is also for machine learning
+    beginners and uses @{$programmers_guide/graphs}.
+  * @{$get_started/premade_estimators} assumes some machine learning background
+    and uses an @{tf.estimator.Estimator$Estimator}.
 
 Then, read the following documents, which demonstrate the key features
 in the high-level APIs:
diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files
index 437791d6a3..17bc209e46 100644
--- a/tensorflow/docs_src/get_started/leftnav_files
+++ b/tensorflow/docs_src/get_started/leftnav_files
@@ -1,6 +1,7 @@
 index.md
 
-### Getting Started
+### Beginners
+eager.md
 get_started_for_beginners.md
 premade_estimators.md
 
diff --git a/tensorflow/docs_src/get_started/premade_estimators.md b/tensorflow/docs_src/get_started/premade_estimators.md
index e50d2f5420..aa4f85f6ce 100644
--- a/tensorflow/docs_src/get_started/premade_estimators.md
+++ b/tensorflow/docs_src/get_started/premade_estimators.md
@@ -1,5 +1,4 @@
-
-# Getting Started with TensorFlow
+# Get Started with Estimators
 
 This document introduces the TensorFlow programming environment and shows you
 how to solve the Iris classification problem in TensorFlow.
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index da5dc6f599..5c9ed9ccaf 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -1431,6 +1431,63 @@ class BaseSession(SessionInterface):
         return tf_session.TF_PRun(
             self._session, handle, feed_dict, fetch_list, status)
 
+  # pylint: disable=protected-access
+  class _Callable(object):
+    """Experimental wrapper for the C++ `Session::MakeCallable()` API."""
+
+    def __init__(self, session, callable_options):
+      self._session = session
+      self._handle = None
+      options_ptr = tf_session.TF_NewBufferFromString(
+          compat.as_bytes(callable_options.SerializeToString()))
+      try:
+        with errors.raise_exception_on_not_ok_status() as status:
+          if session._created_with_new_api:
+            self._handle = tf_session.TF_SessionMakeCallable(
+                session._session, options_ptr, status)
+          else:
+            self._handle = tf_session.TF_DeprecatedSessionMakeCallable(
+                session._session, options_ptr, status)
+      finally:
+        tf_session.TF_DeleteBuffer(options_ptr)
+
+    def __call__(self, *args):
+      # TODO(b/74355905): Support argument and return value nested structures,
+      # and tensor-like objects such as SparseTensors.
+      with errors.raise_exception_on_not_ok_status() as status:
+        if self._session._created_with_new_api:
+          return tf_session.TF_SessionRunCallable(
+              self._session._session, self._handle, args, status, None)
+        else:
+          return tf_session.TF_DeprecatedSessionRunCallable(
+              self._session._session, self._handle, args, status, None)
+
+    def __del__(self):
+      if self._handle is not None:
+        with errors.raise_exception_on_not_ok_status() as status:
+          if self._session._created_with_new_api:
+            tf_session.TF_SessionReleaseCallable(
+                self._session._session, self._handle, status)
+          else:
+            tf_session.TF_DeprecatedSessionReleaseCallable(
+                self._session._session, self._handle, status)
+  # pylint: enable=protected-access
+
+  # TODO(b/74355905): Reimplement `Session.make_callable()` using this method
+  # where possible.
+  def _make_callable_from_options(self, callable_options):
+    """Returns a handle to a "callable" with the given options.
+
+    Args:
+      callable_options: A `CallableOptions` protocol buffer message describing
+        the computation that will be performed by the callable.
+
+    Returns:
+      A handle to the new callable.
+    """
+    self._extend_graph()
+    return BaseSession._Callable(self, callable_options)
+
 
 @tf_export('Session')
 class Session(BaseSession):
diff --git a/tensorflow/python/client/session_test.py b/tensorflow/python/client/session_test.py
index 6e2640efd1..92497272c6 100644
--- a/tensorflow/python/client/session_test.py
+++ b/tensorflow/python/client/session_test.py
@@ -1371,6 +1371,18 @@ class SessionTest(test_util.TensorFlowTestCase):
                               run_metadata=run_metadata))
       self.assertGreater(len(run_metadata.step_stats.dev_stats), 0)
 
+  def testOptimizedMakeCallable(self):
+    with session.Session() as sess:
+      ph = array_ops.placeholder(dtypes.float32)
+      a = math_ops.add(ph, 1.0)
+      callable_opts = config_pb2.CallableOptions()
+      callable_opts.feed.append(ph.name)
+      callable_opts.fetch.append(a.name)
+      for _ in range(3):
+        callable_fn = sess._make_callable_from_options(callable_opts)
+        for _ in range(5):
+          self.assertEqual([2.0], callable_fn(np.array(1.0, dtype=np.float32)))
+
   def testFeedError(self):
     with session.Session() as sess:
       feed_t = array_ops.placeholder(dtype=dtypes.float32)
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 70a3d032f4..77ce9195ee 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -419,6 +419,25 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
   $result = new_result;
 }
 
+%typemap(in, numinputs=0) int64_t* out_handle (int64_t out_handle) {
+  $1 = &out_handle;
+}
+
+%typemap(argout) int64_t* out_handle {
+  $result = PyLong_FromLongLong(*$1);
+}
+
+%typemap(in) int64_t handle {
+  if (!PyLong_Check($input)) {
+    SWIG_exception_fail(
+        SWIG_TypeError,
+        tensorflow::strings::Printf(
+            "Expected a python long for conversion to callable handle but got %s",
+            Py_TYPE($input)->tp_name).c_str());
+  }
+  $1 = PyLong_AsLongLong($input);
+}
+
 // TODO(skyewm): SWIG emits a warning for the const char* in TF_WhileParams,
 // skip for now
 %ignore TF_WhileParams;
@@ -452,6 +471,17 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
 // See comment for "%noexception TF_SessionRun_wrapper;"
 %noexception TF_SessionPRun_wrapper;
 
+%unignore TF_DeprecatedSessionMakeCallable;
+%unignore TF_SessionMakeCallable;
+%unignore TF_DeprecatedSessionRunCallable;
+%unignore TF_SessionRunCallable;
+%unignore TF_DeprecatedSessionReleaseCallable;
+%unignore TF_SessionReleaseCallable;
+
+// See comment for "%noexception TF_SessionRun_wrapper;"
+%noexception TF_DeprecatedSessionRunCallable;
+%noexception TF_SessionRunCallable;
+
 %rename("_TF_SetTarget") TF_SetTarget;
 %rename("_TF_SetConfig") TF_SetConfig;
 %rename("_TF_NewSessionOptions") TF_NewSessionOptions;
diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc
index a8ab91749a..ca57abd712 100644
--- a/tensorflow/python/client/tf_session_helper.cc
+++ b/tensorflow/python/client/tf_session_helper.cc
@@ -155,6 +155,156 @@ void TF_Run_wrapper(TF_DeprecatedSession* session, const TF_Buffer* run_options,
   ClearDecrefCache();
 }
 
+namespace {
+void MakeCallableHelper(tensorflow::Session* session,
+                        const TF_Buffer* callable_options, int64_t* out_handle,
+                        TF_Status* out_status) {
+  tensorflow::CallableOptions callable_options_proto;
+  if (callable_options != nullptr &&
+      !callable_options_proto.ParseFromArray(callable_options->data,
+                                             callable_options->length)) {
+    Set_TF_Status_from_Status(
+        out_status,
+        errors::InvalidArgument("Unparseable CallableOptions proto"));
+    return;
+  }
+  tensorflow::Session::CallableHandle handle;
+  Status s = session->MakeCallable(callable_options_proto, &handle);
+  if (!s.ok()) {
+    Set_TF_Status_from_Status(out_status, s);
+    return;
+  }
+  *out_handle = handle;
+}
+}  // namespace
+
+void TF_DeprecatedSessionMakeCallable(TF_DeprecatedSession* session,
+                                      const TF_Buffer* callable_options,
+                                      int64_t* out_handle,
+                                      TF_Status* out_status) {
+  MakeCallableHelper(session->session, callable_options, out_handle,
+                     out_status);
+}
+void TF_SessionMakeCallable(TF_Session* session,
+                            const TF_Buffer* callable_options,
+                            int64_t* out_handle, TF_Status* out_status) {
+  MakeCallableHelper(session->session, callable_options, out_handle,
+                     out_status);
+}
+
+namespace {
+void RunCallableHelper(tensorflow::Session* session, int64_t handle,
+                       PyObject* feed_values, TF_Status* out_status,
+                       PyObjectVector* out_values, TF_Buffer* run_metadata) {
+  // Convert feed values to a vector of tensorflow::Tensor objects.
+  std::vector<Tensor> input_tensors;
+  Status s;
+  {
+    feed_values =
+        PySequence_Fast(feed_values, "feed_values must be a sequence");
+    if (feed_values == nullptr) return;
+    Safe_PyObjectPtr feed_values_holder(make_safe(feed_values));
+    Py_ssize_t len = PySequence_Fast_GET_SIZE(feed_values);
+    input_tensors.reserve(len);
+    for (Py_ssize_t i = 0; i < len; ++i) {
+      PyObject* elem = PySequence_Fast_GET_ITEM(feed_values, i);
+      if (!elem) {
+        Set_TF_Status_from_Status(
+            out_status, errors::Internal("Could not get feed value ", i));
+        return;
+      }
+      Tensor t;
+      s = NdarrayToTensor(elem, &t);
+      if (!s.ok()) {
+        Set_TF_Status_from_Status(out_status, s);
+        return;
+      }
+      input_tensors.push_back(std::move(t));
+    }
+  }
+
+  // Allocate a RunMetadata protobuf object to receive the metadata,
+  // if the caller is expecting any.
+  std::unique_ptr<RunMetadata> run_metadata_proto;
+  if (run_metadata != nullptr) {
+    run_metadata_proto.reset(new RunMetadata);
+  }
+
+  // Run the callable.
+  std::vector<Tensor> output_tensors;
+  Py_BEGIN_ALLOW_THREADS;
+  s = session->RunCallable(handle, input_tensors, &output_tensors,
+                           run_metadata_proto.get());
+  Py_END_ALLOW_THREADS;
+
+  if (!s.ok()) {
+    Set_TF_Status_from_Status(out_status, s);
+    return;
+  }
+
+  // If requested, serialize the RunMetadata to pass it back to the caller.
+  if (run_metadata != nullptr) {
+    s = MessageToBuffer(*run_metadata_proto, run_metadata);
+    if (!s.ok()) {
+      Set_TF_Status_from_Status(out_status, s);
+      return;
+    }
+  }
+
+  // Convert results to NumPy arrays. Since this can fail, stage the
+  // results via a safe container that takes care of decreasing the
+  // reference count on failure.
+  std::vector<Safe_PyObjectPtr> py_outputs_safe;
+  py_outputs_safe.reserve(output_tensors.size());
+  for (const Tensor& output : output_tensors) {
+    PyObject* py_array;
+    s = TensorToNdarray(output, &py_array);
+    if (!s.ok()) {
+      Set_TF_Status_from_Status(out_status, s);
+      return;
+    }
+    py_outputs_safe.push_back(make_safe(py_array));
+  }
+
+  // If we reach this point, we have successfully built a list of objects
+  // so we can release them from the safe container.
+  out_values->reserve(py_outputs_safe.size());
+  for (auto& output : py_outputs_safe) {
+    out_values->push_back(output.release());
+  }
+}
+}  // namespace
+
+void TF_DeprecatedSessionRunCallable(TF_DeprecatedSession* session,
+                                     int64_t handle, PyObject* feed_values,
+                                     TF_Status* out_status,
+                                     PyObjectVector* out_values,
+                                     TF_Buffer* run_metadata) {
+  RunCallableHelper(session->session, handle, feed_values, out_status,
+                    out_values, run_metadata);
+  ClearDecrefCache();
+}
+void TF_SessionRunCallable(TF_Session* session, int64_t handle,
+                           PyObject* feed_values, TF_Status* out_status,
+                           PyObjectVector* out_values,
+                           TF_Buffer* run_metadata) {
+  RunCallableHelper(session->session, handle, feed_values, out_status,
+                    out_values, run_metadata);
+  ClearDecrefCache();
+}
+
+void TF_DeprecatedSessionReleaseCallable(TF_DeprecatedSession* session,
+                                         int64_t handle,
+                                         TF_Status* out_status) {
+  Set_TF_Status_from_Status(out_status,
+                            session->session->ReleaseCallable(handle));
+}
+void TF_SessionReleaseCallable(TF_Session* session, int64_t handle,
+                               TF_Status* out_status) {
+  Set_TF_Status_from_Status(out_status,
+                            session->session->ReleaseCallable(handle));
+}
+
 // Wrapper for TF_PRunSetup that converts the arguments to appropriate types.
 // If *out_status is OK, the caller becomes the owner of *out_handle.
 void TF_PRunSetup_wrapper(TF_DeprecatedSession* session,
diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h
index 83318dc178..603d03e315 100644
--- a/tensorflow/python/client/tf_session_helper.h
+++ b/tensorflow/python/client/tf_session_helper.h
@@ -59,6 +59,31 @@ void TF_Run_wrapper(TF_DeprecatedSession* session, const TF_Buffer* run_options,
                     const NameVector& target_nodes, TF_Status* out_status,
                     PyObjectVector* out_values, TF_Buffer* run_outputs);
 
+// Python wrappers for the `Session::MakeCallable()` API.
+void TF_DeprecatedSessionMakeCallable(TF_DeprecatedSession* session,
+                                      const TF_Buffer* callable_options,
+                                      int64_t* out_handle,
+                                      TF_Status* out_status);
+void TF_SessionMakeCallable(TF_Session* session,
+                            const TF_Buffer* callable_options,
+                            int64_t* out_handle, TF_Status* out_status);
+
+// Python wrappers for the `Session::RunCallable()` API.
+void TF_DeprecatedSessionRunCallable(TF_DeprecatedSession* session,
+                                     int64_t handle, PyObject* feed_values,
+                                     TF_Status* out_status,
+                                     PyObjectVector* out_values,
+                                     TF_Buffer* run_metadata);
+void TF_SessionRunCallable(TF_Session* session, int64_t handle,
+                           PyObject* feed_values, TF_Status* out_status,
+                           PyObjectVector* out_values, TF_Buffer* run_metadata);
+
+// Python wrappers for the `Session::ReleaseCallable()` API.
+void TF_DeprecatedSessionReleaseCallable(TF_DeprecatedSession* session,
+                                         int64_t handle, TF_Status* out_status);
+void TF_SessionReleaseCallable(TF_Session* session, int64_t handle,
+                               TF_Status* out_status);
+
 // Set up the graph with the intended feeds and fetches for partial run.
 // *out_handle is owned by the caller.
 //
diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 0ae9900a1d..219105d386 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -33,6 +33,7 @@ py_library(
         "//tensorflow/python:nn_ops",
         "//tensorflow/python:parsing_ops",
         "//tensorflow/python:platform",
+        "//tensorflow/python:resource_variable_ops",
         "//tensorflow/python:sparse_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 7d99fcb3e7..92c6ff21c4 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -148,6 +148,7 @@ from tensorflow.python.ops import lookup_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn_ops
 from tensorflow.python.ops import parsing_ops
+from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.ops import template
@@ -451,7 +452,8 @@ def linear_model(features,
     if cols_to_vars is not None:
       # Add the bias to cols_to_vars as well, converting the Variable or
       # PartitionedVariable to a list of Variable's.
-      if isinstance(bias, variables.Variable):
+      if (isinstance(bias, variables.Variable) or
+          resource_variable_ops.is_resource_variable(bias)):
         cols_to_vars['bias'] = [bias]
       else:  # Must be a PartitionedVariable.
         cols_to_vars['bias'] = list(bias)
diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index bfab9d7be2..b11412cac7 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -27,6 +27,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops.losses import losses_impl
+from tensorflow.python.platform import tf_logging
 from tensorflow.python.training import device_util
 from tensorflow.python.util import nest
 
@@ -287,17 +288,22 @@ def _require_distribution_strategy_scope(distribution_strategy):
 class _CurrentDistributionContext(object):
   """Context manager for setting the `DistributionStrategy` and var creator."""
 
-  def __init__(self, distribution_strategy, var_creator_scope):
+  def __init__(self, distribution_strategy, var_creator_scope, var_scope=None):
     self._context = _CrossTowerThreadMode(distribution_strategy)
     self._var_creator_scope = var_creator_scope
+    self._var_scope = var_scope
 
   def __enter__(self):
     _push_per_thread_mode(self._context)
+    if self._var_scope:
+      self._var_scope.__enter__()
     self._var_creator_scope.__enter__()
     return self._context.distribution_strategy
 
   def __exit__(self, exception_type, exception_value, traceback):
     self._var_creator_scope.__exit__(exception_type, exception_value, traceback)
+    if self._var_scope:
+      self._var_scope.__exit__(exception_type, exception_value, traceback)
     _pop_per_thread_mode()
 
 
@@ -556,8 +562,18 @@ class DistributionStrategy(object):
       kwargs["use_resource"] = True
       return self._create_variable(*args, **kwargs)
 
+    def disable_partitioned_variables(getter, *args, **kwargs):
+      if kwargs.pop("partitioner", None) is not None:
+        tf_logging.log_first_n(
+            tf_logging.WARN, "Partitioned variables are disabled when using "
+            "DistributionStrategy.", 1)
+      return getter(*args, **kwargs)
+
     return _CurrentDistributionContext(
-        self, variable_scope.variable_creator_scope(creator_with_resource_vars))
+        self, variable_scope.variable_creator_scope(creator_with_resource_vars),
+        variable_scope.variable_scope(
+            variable_scope.get_variable_scope(),
+            custom_getter=disable_partitioned_variables))
 
   def _create_variable(self, next_creator, *args, **kwargs):
     # Note: should support "colocate_with" argument.
@@ -954,6 +970,10 @@ class DistributionStrategy(object):
   def _worker_device_index(self):
     raise NotImplementedError("must be implemented in descendants")
 
+  def configure(self, session_config=None):
+    """Find the best configuration given a tensorflow session config."""
+    del session_config
+
 
 # A note about the difference between the context managers
 # `TowerContext` (defined here) and `_CurrentDistributionContext`
@@ -1105,10 +1125,6 @@ class _DefaultDistributionStrategy(DistributionStrategy):
     # in contrib.
     return dataset.make_one_shot_iterator()
 
-  def configure(self, session_config=None):
-    """Find the best configuration given a tensorflow session config."""
-    del session_config
-
   def _broadcast(self, tensor, destinations):
     if destinations is None:
       return tensor
-- 
GitLab


From da0b3295c92d38a514123ec92086d807d207d647 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 28 Mar 2018 15:31:19 -0700
Subject: [PATCH 740/960] Refresh Community pages to surface new resources,
 SIGs and mailing lists.

PiperOrigin-RevId: 190845545
---
 tensorflow/docs_src/community/contributing.md | 64 +++++++++++++
 tensorflow/docs_src/community/groups.md       | 17 ++++
 tensorflow/docs_src/community/index.md        | 92 ++++++++++++++++---
 tensorflow/docs_src/community/leftnav_files   |  4 +-
 tensorflow/docs_src/community/lists.md        | 35 +++++++
 5 files changed, 198 insertions(+), 14 deletions(-)
 create mode 100644 tensorflow/docs_src/community/contributing.md
 create mode 100644 tensorflow/docs_src/community/groups.md
 create mode 100644 tensorflow/docs_src/community/lists.md

diff --git a/tensorflow/docs_src/community/contributing.md b/tensorflow/docs_src/community/contributing.md
new file mode 100644
index 0000000000..b0960df435
--- /dev/null
+++ b/tensorflow/docs_src/community/contributing.md
@@ -0,0 +1,64 @@
+# Contributing to TensorFlow
+
+TensorFlow is an open-source project, and we welcome your participation
+and contribution. This page describes how to get involved.
+
+## Repositories
+
+The code for TensorFlow is hosted in the [TensorFlow GitHub
+organization](https://github.com/tensorflow). Multiple projects are located
+inside the organization, including:
+
+* [TensorFlow](https://github.com/tensorflow/tensorflow)
+* [Models](https://github.com/tensorflow/models)
+* [TensorBoard](https://github.com/tensorflow/tensorboard)
+* [TensorFlow.js](https://github.com/tensorflow/tfjs)
+* [TensorFlow Serving](https://github.com/tensorflow/serving)
+* [TensorFlow Documentation](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/docs_src)
+
+## Contributor checklist
+
+* Before contributing to TensorFlow source code, please review the [contribution
+guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).
+
+* Join the
+[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers)
+mailing list, to coordinate and discuss with others contributing to TensorFlow.
+
+* For coding style conventions, read the @{$style_guide$TensorFlow Style Guide}.
+
+* Finally, review @{$documentation$Writing TensorFlow Documentation}, which
+  explains documentation conventions.
+
+You may also wish to review our guide to @{$benchmarks$defining and running benchmarks}.
+
+## Special Interest Groups
+
+To enable focused collaboration on particular areas of TensorFlow, we host
+Special Interest Groups (SIGs). SIGs do their work in public: if you want to
+join and contribute, review the work of the group, and get in touch with the
+relevant SIG leader.
+
+* **SIG Build** focuses on issues surrounding building, packaging, and
+  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/build).
+
+* **SIG TensorBoard** furthers the development and direction of TensorBoard and its plugins.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard).
+
+* **SIG Rust** collaborates on the development of TensorFlow's Rust bindings.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust). 
+
+## Projects developed by the TensorFlow community
+
+The TensorFlow community has created many great projects around TensorFlow, including:
+
+* [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com)
+* [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow)
+* [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials)
+* [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow)
+* [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0))
+* [Operator Vectorization Library](https://github.com/opveclib/opveclib)
+* [Swift language bindings](https://github.com/PerfectlySoft/Perfect-TensorFlow)
+* [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
+* [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
+* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
diff --git a/tensorflow/docs_src/community/groups.md b/tensorflow/docs_src/community/groups.md
new file mode 100644
index 0000000000..d92f5775fa
--- /dev/null
+++ b/tensorflow/docs_src/community/groups.md
@@ -0,0 +1,17 @@
+# User Groups
+
+TensorFlow has communities around the world.
+
+## Asia
+
+* [TensorFlow Korea (TF-KR) User Group](https://www.facebook.com/groups/TensorFlowKR/) _(Korean language)_
+* [TensorFlow User Group Tokyo](https://tfug-tokyo.connpass.com/) _(Japanese Language)_
+* [Soleil Data Dojo](https://soleildatadojo.connpass.com/) _(Japanese language)_
+* [TensorFlow User Group Utsunomiya](https://tfug-utsunomiya.connpass.com/)
+
+
+## Europe
+
+* [TensorFlow Barcelona](https://www.meetup.com/Barcelona-Machine-Learning-Meetup/)
+* [TensorFlow Madrid](https://www.meetup.com/TensorFlow-Madrid/)
+
diff --git a/tensorflow/docs_src/community/index.md b/tensorflow/docs_src/community/index.md
index b706d9b204..c08aeb7a97 100644
--- a/tensorflow/docs_src/community/index.md
+++ b/tensorflow/docs_src/community/index.md
@@ -1,15 +1,81 @@
 # Community
 
-This section contains the following documents:
-
-  * @{$welcome$Welcome to the TensorFlow Community}, which explains how
-    you can get involved, where to report issues, and where to join
-    like-minded TensorFlow enthusiasts online.
-  * @{$roadmap$Roadmap}, which summarizes upcoming additions to TensorFlow.
-  * @{$documentation$Writing TensorFlow Documentation}, which explains
-    TensorFlow's documentation conventions.  If you are modifying
-    TensorFlow source code or documentation, please read this guide.
-  * @{$style_guide$TensorFlow Style Guide}, which identifies coding style
-    conventions that TensorFlow developers and users should follow.
-  * @{$community/benchmarks$Benchmarks}, Benchmarks, a guide for defining and
-    running a TensorFlow benchmark.
+Welcome to the TensorFlow community! This page explains where to get help, and
+different ways to be part of the community. We are committed to fostering an
+open and welcoming environment, and request that you review our [code of
+conduct](https://github.com/tensorflow/tensorflow/blob/master/CODE_OF_CONDUCT.md).
+
+## Get Help
+
+### Technical Questions
+
+To ask or answer technical questions about TensorFlow, use [Stack
+Overflow](https://stackoverflow.com/questions/tagged/tensorflow). For example,
+ask or search about a particular error message you encountered during
+installation.
+
+### Bugs and Feature Requests
+
+To report bugs or make feature requests, file an issue on GitHub. Please choose
+the appropriate repository for the project. Major repositories include:
+
+  * [TensorFlow](https://github.com/tensorflow/tensorflow/issues)
+  * [TensorBoard](https://github.com/tensorflow/tensorboard/issues)
+  * [TensorFlow models](https://github.com/tensorflow/models/issues)
+  
+### Security
+
+Before using TensorFlow, please take a look at our security model, list of
+recent security announcements, and ways you can report security issues to the
+TensorFlow team at the
+[Using TensorFlow Securely](https://github.com/tensorflow/tensorflow/blob/master/SECURITY.md) page on GitHub.
+
+## Stay Informed
+
+### Announcements Mailing List
+
+All major releases and important announcements are sent to
+[announce@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce).
+We recommend that you join this list if you depend on TensorFlow in any way.
+
+### Development Roadmap
+
+The @{$roadmap$Roadmap} summarizes plans for upcoming additions to TensorFlow.
+
+### Social Media
+
+For news and updates from around the universe of TensorFlow projects, follow
+[@tensorflow](https://twitter.com/tensorflow) on Twitter.
+
+### YouTube
+
+Our [YouTube Channel](http://youtube.com/tensorflow/) focuses on machine learing
+and AI with TensorFlow. On it we have a number of new shows, including:
+
+- TensorFlow Meets: meet with community contributors to learn and share what they're doing
+- Ask TensorFlow: the team answers the best questions tagged #AskTensorFlow from social media 
+- Coding TensorFlow: short bites with tips for success with TensorFlow
+
+
+## Community Support
+
+### Mailing Lists
+
+For general discussion about TensorFlow development and direction, please join
+the [TensorFlow discuss mailing
+list](https://groups.google.com/a/tensorflow.org/d/forum/discuss).
+
+A number of other mailing lists exist, focused on different project areas, which
+can be found at @{$lists$TensorFlow Mailing Lists}.
+
+### User Groups
+
+To meet with like-minded people local to you, check out the many
+@{$groups$TensorFlow user groups} around the world.
+
+
+## Contributing To TensorFlow
+
+We welcome contributions and collaboration on TensorFlow. For more information,
+please read [Contributing to TensorFlow](contributing.md).
+
diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files
index fab35024ad..0bd1f14de9 100644
--- a/tensorflow/docs_src/community/leftnav_files
+++ b/tensorflow/docs_src/community/leftnav_files
@@ -1,6 +1,8 @@
 index.md
-welcome.md
 roadmap.md
+contributing.md
+lists.md
+groups.md
 documentation.md
 style_guide.md
 benchmarks.md
diff --git a/tensorflow/docs_src/community/lists.md b/tensorflow/docs_src/community/lists.md
new file mode 100644
index 0000000000..dc9240030e
--- /dev/null
+++ b/tensorflow/docs_src/community/lists.md
@@ -0,0 +1,35 @@
+# Mailing Lists
+
+As a community, we do much of our collaboration on public mailing lists.
+Please note that if you're looking for help using TensorFlow, [Stack
+Overflow](https://stackoverflow.com/questions/tagged/tensorflow) and
+[GitHub issues](https://github.com/tensorflow/tensorflow/issues)
+are the best initial places to look. For more information,
+see [how to get help](/community/#get_help).
+
+## General TensorFlow lists
+
+* [announce](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce) - Low-volume announcements of new releases.
+* [discuss](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss) - General community discussion around TensorFlow.
+* [developers](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers) - Discussion for developers contributing to TensorFlow.
+
+## Project-specific lists
+
+These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
+
+* [tensor2tensor](https://groups.google.com/forum/#!forum/tensor2tensor) - User
+  and peer support for Tensor2Tensor.
+
+## Special Interest Groups
+
+TensorFlow's [Special Interest
+Groups](/community/contributing#special_interest_groups) (SIGs) support
+community collaboration on particular project focuses. Members of these groups
+work together to build and support TensorFlow related projects.
+
+* [build](https://groups.google.com/a/tensorflow.org/forum/#!forum/build) -
+  Supporting SIG Build, for build, distribution and packaging of TensorFlow.
+* [tensorboard](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard) -
+  Supporting SIG TensorBoard, for plugin development and other contribution.
+* [rust](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust) -
+  Supporting SIG Rust, for the Rust language bindings.
-- 
GitLab


From be554c147c1cb5a53e3d891342dcdeeeaa7b16ce Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 13:28:05 -0700
Subject: [PATCH 741/960] Docs: Move TFLite docs into tensorflow.org

PiperOrigin-RevId: 190977057
---
 tensorflow/contrib/lite/README.md             | 237 +-----------------
 .../lite/g3doc/TFLite-Architecture.jpg        | Bin 48710 -> 0 bytes
 tensorflow/docs_src/mobile/leftnav_files      |   1 +
 .../docs_src/mobile/tflite/demo_android.md    | 156 ++++++++++--
 tensorflow/docs_src/mobile/tflite/demo_ios.md |   2 +-
 tensorflow/docs_src/mobile/tflite/devguide.md | 224 +++++++++++++++++
 tensorflow/docs_src/mobile/tflite/index.md    |   4 +-
 7 files changed, 363 insertions(+), 261 deletions(-)
 delete mode 100644 tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg
 create mode 100644 tensorflow/docs_src/mobile/tflite/devguide.md

diff --git a/tensorflow/contrib/lite/README.md b/tensorflow/contrib/lite/README.md
index df8c1c623c..a676b705f1 100644
--- a/tensorflow/contrib/lite/README.md
+++ b/tensorflow/contrib/lite/README.md
@@ -1,235 +1,8 @@
 # TensorFlow Lite
-TensorFlow Lite is TensorFlow's lightweight solution for mobile and embedded devices. It enables low-latency inference of on-device machine learning models with a small binary size and fast performance supporting hardware acceleration.
 
-TensorFlow Lite uses many techniques for achieving low latency like optimizing the kernels for specific mobile apps, pre-fused activations, quantized kernels that allow smaller and faster (fixed-point math) models, and in the future, leverage specialized machine learning hardware to get the best possible performance for a particular model on a particular device.
+TensorFlow Lite is TensorFlow's lightweight solution for mobile and embedded
+devices. It enables low-latency inference of on-device machine learning models
+with a small binary size and fast performance supporting hardware acceleration.
 
-![image](g3doc/TFLite-Architecture.jpg)
-# Getting Started with an Android Demo App
-
-This section contains an example application using TensorFlow Lite for Android devices. The demo is a sample camera app that classifies images continuously using either a quantized Mobilenet model or a floating point Inception-v3 model. A device running Android 5.0 ( API 21) or higher is required to run the demo.
-
-There are 3 ways to get the demo app to your device
- - Download the prebuilt binary or
- - Use Android Studio to build the application or
- - Download the source code for TensorFlow Lite and the demo and build it using bazel
-
-## Description
-In the demo app, inference is done using the TensorFlow Lite Java API. The demo app classifies frames in real-time, displaying the top most probable classifications. It also displays the time taken to detect the object.
-
-## Downloading the pre-built binary
-The fastest path to trying the demo, is to download the pre-built binary
-[TfLiteCameraDemo.apk](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
-
-Once the apk is installed, click the app icon to start the app. The first-time the app is opened, the app asks for runtime permissions to access the device camera. The demo app opens the back-camera of the device and recognizes the objects in the camera's field of view. At the bottom of the image (or at the left of the image if the device is in landscape mode), it shows the latency of classification and the top three objects classified.
-
-## Building in Android Studio using TensorFlow Lite AAR from JCenter
-The simplest way to compile the demo app, and try out changes to the project code is to use AndroidStudio.
-
- - Install the latest version of Android Studio 3 as specified [here](https://developer.android.com/studio/index.html).
- - Make sure the Android SDK version is greater than 26 and NDK version is greater than 14 (in the Android Studio Settings).
- - Import the `tensorflow/contrib/lite/java/demo` directory as a new Android Studio project.
- - Click through installing all the Gradle extensions it requests.
- - Either
-     - Download the quantized Mobilenet TensorFlow Lite model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip)
-         - unzip and copy mobilenet_quant_v1_224.tflite to the assets directory:
-           `tensorflow/contrib/lite/java/demo/app/src/main/assets/`
-     - Or download the floating point Inception-v3 model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
-         - unzip and copy inceptionv3_non_slim_2015.tflite to the assets directory
-         - change the chosen classifier in [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java) from
-         `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`
-         to
-         `classifier = new ImageClassifierFloatInception(getActivity());`
- - Build and run the demo app
-
-## Building TensorFlow Lite and the demo app from source
-
-### Clone the TensorFlow repo
-- git clone
-  [https://github.com/tensorflow/tensorflow](https://github.com/tensorflow/tensorflow)
-
-### Install Bazel
-If bazel is not installed on your system, install it now by following [these directions](https://bazel.build/versions/master/docs/install.html)
-
-NOTE: Bazel does not fully support building Android on Windows yet. Full support for Gradle/CMake builds is coming soon, but in the meantime Windows users should download the [prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk) instead.
-
-### Install Android NDK and SDK
-Bazel is the primary build system for TensorFlow. Bazel and the Android NDK and SDK must be installed on your system.
- - Install the latest version of Bazel as per the instructions on the [Bazel website](https://bazel.build/versions/master/docs/install.html)
- - The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The current recommended version is 14b, which can be found [here](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads).
- - The Android SDK and build tools may be obtained [here](https://developer.android.com/tools/revisions/build-tools.html), or alternatively as part of [Android Studio](https://developer.android.com/studio/index.html). Build tools API >= 23 is required to build the TF Android demo (though it will run on API >= 21 devices).
- - In the root of the TensorFlow repository update the `WORKSPACE` file with the `api_level` and location of the SDK and NDK. If you installed it with AndroidStudio the SDK path can be found in the SDK manager, and the default NDK path is:`{SDK path}/ndk-bundle.`
-
-```
-android_sdk_repository (
-    name = "androidsdk",
-    api_level = 23,
-    build_tools_version = "23.0.2",
-    path = "/home/xxxx/android-sdk-linux/",
-)
-
-android_ndk_repository(
-    name = "androidndk",
-    path = "/home/xxxx/android-ndk-r10e/",
-    api_level = 19,
-)
-```
-
-Additional details on building with Android can be found [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
-
-### Build the source code
-Run bazel with the following command to build the demo.
-
-Build the demo app:
-
-```
-bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
-```
-
-### Note
-
-Currently, we only support building the Android demo app within a Python 2
-environment (due to a Bazel bug).
-
-### More about the demo
-The demo is resizing each camera image frame to (224 width * 224 height) to match the quantized Mobilenet model being used (299 * 299 for Inception-v3). The resized image is converted into a ByteBuffer row by row of size 1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch. 224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents three colors of a pixel. This demo uses the TensorFlow Lite Java inference API for models which take a single input and provide a single output. This outputs a two-dimensional array, with the first dimension being the category index and the second dimension being the confidence of classification. Both models have 1001 unique categories and the app sorts the probabilities of all the categories and displays the top three. The model file must be downloaded and bundled within the assets directory of the app.
-
-# iOS Demo App
-
-Similar to the Android demo app, there's an iOS camera app that uses exactly the same model (224 * 224 quantized Mobilenet).
-
-This demo app requires a camera so it doesn't work with simulators. It need to be executed on a real iOS device. Follow the instructions to build and run the demo app:
-
-1.   Run `third_party/tensorflow/contrib/lite/examples/ios/download_models.sh` to download the model files used by the demo app.
-1.   Install [CocoaPods](https://cocoapods.org/) if it wasn't installed yet: `sudo gem install cocoapods`.
-1.   Run `pod install` in `tensorflow/contrib/lite/examples/ios/camera` to generate the workspace file.
-1.   Open the project by running `open tflite_camera_example.xcworkspace`, and build the app in XCode.
-
-# TensorFlow Lite Quick Start
-
-## Step 1. Decide which GraphDef to use
- Depending on the use case, the developer may choose to use one of the popular
- open-sourced models such as InceptionV3 or MobileNets, re-train these models
- with their own custom data set or even build their own custom model.
-
-### Using a pre-trained model
-
-[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html) is a family of mobile-first computer vision models for [TensorFlow](https://www.tensorflow.org/) designed to effectively maximize accuracy while being mindful of the restricted resources for an on-device or embedded application. MobileNets are small, low-latency, low-power models parameterized to meet the resource constraints of a variety of use cases. They can be built upon for classification, detection, embeddings and segmentation similar to how other popular large scale models, such as [Inception](https://arxiv.org/pdf/1602.07261.pdf), are used. Google provides 16 pre-trained [ImageNet](http://www.image-net.org/challenges/LSVRC/)  classification checkpoints for MobileNets for use in mobile projects of all sizes.
-
-[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model which achieves fairly high accuracy in recognizing general objects with 1000 classes, like "Zebra", "Dalmatian", and "Dishwasher". The model extracts general features from input images using a convolutional neural network and classifies them based on those features with fully-connected and softmax layers.
-
-[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)  is an on-device model which provides one-touch replies for an incoming text message by suggesting contextually relevant messages. The model is built specifically for memory constrained devices such as watches & phones and it has been successfully used to surface [Smart Replies on Android Wear](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html). Note that this model only works on Android as of now.
-
-These pre-trained models can be downloaded from [here](g3doc/models.md).
-
-### Retrain Inception-V3 or MobileNet for a custom data set
-The above pre-trained models have been trained on the ImageNet data set, which consists of 1000 predefined classes. A model will need to be re-trained if these classes are not relevant or useful for a given use case. This technique is called transfer learning, which starts with a model that has been already trained on a problem and will then be retrained on a similar problem. Deep learning from scratch can take days, but transfer learning can be done fairly quickly. In order to do this, a developer will need to generate their custom data set labeled with the relevant classes.
-
-The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/) codelab walks through this process step-by-step. The retraining code supports retraining for both floating point and quantized inference.
-
-
-### Train a custom model
-A developer may choose to train a custom model using Tensorflow. TensorFlow documentation has [several tutorials](https://www.tensorflow.org/tutorials/) for building and training models. If the user has written a model using TensorFlow's Slim Framework the first step is to export this to a GraphDef file. This is necessary because Slim does not store the model structure outside the code, so to communicate with other parts of the framework it needs to be exported. Documentation for the export can be found [here](https://github.com/tensorflow/models/tree/master/research/slim#Export). The output of this step will be a .pb file for the custom model.
-
-TensorFlow Lite currently supports a subset of TensorFlow operators. Please refer to [this document](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for details of supported operators and their usage. This
-set will continue to expand in future releases of Tensorflow Lite.
-
-
-## Step 2. Model format conversion
-
-The model generated in Step 1 is a standard Tensorflow model. After the completion of Step 1 a user should have a standard .pb or .pbtxt GraphDef file. If the application developer is using a pre-trained model (as defined in Step 1 above), they can download a ready to use, already converted model for use from [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/models.md). Models generated using retraining (aka transfer learning) or custom models will need to be converted using the steps mentioned below.
-
-A prerequisite to converting the model to the Tensorflow Lite format is to freeze the graph.
-
-Since we employ several formats, the following definitions may be useful:
- - GraphDef (.pb) - a protobuf that represents the TensorFlow training and or computation graph. This contains operators, tensors, and variables definitions.
-
- - CheckPoint (.ckpt) - Serialized variables from a TensorFlow graph. Note, this does not contain the graph structure, so alone it cannot typically be interpreted.
-
- - FrozenGraphDef - a subclass of GraphDef that contains no variables. A GraphDef can be converted to a frozen graphdef by taking a checkpoint and a graphdef and converting every variable into a constant with the value looked up in the checkpoint.
-
- - SavedModel - A collection of GraphDef and CheckPoint together with a signature that labels input and output arguments to a model. A GraphDef and Checkpoint can be extracted from a saved model.
-
- - TensorFlow lite model (.tflite) - a serialized flatbuffer, containing TensorFlow lite operators and Tensors for the TensorFlow lite interpreter. This is most analogous to TensorFlow frozen GraphDefs.
-
-### Freeze Graph
-To use this .pb GraphDef file within TensorFlow Lite, the application developer will need checkpoints containing trained weight parameters. The .pb contains only the structure of the graph. The process of merging the checkpoint values with the graph structure is known as "freezing" the graph.
-
-The developer should know where the checkpoints folder is present or checkpoints can also be downloaded for a pre-trained model (Example: Here is a link to the [MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
-
-Graph freezing can be done using the command below (and modifying the arguments appropriately)
-
-```
-bazel build tensorflow/python/tools:freeze_graph
-
-bazel-bin/tensorflow/python/tools/freeze_graph\
-    --input_graph=/tmp/mobilenet_v1_224.pb \
-    --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
-    --input_binary=true --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
-    --output_node_names=MobileNet/Predictions/Reshape_1
-```
-
-The user has to first build the freeze_graph script using bazel and then run the script.  The input_binary flag has to be enabled to ensure that the protobuf is read and written in binary format.  The user has to input the .pb and the .ckpt files to freeze the graph The output_node_names may not be obvious outside of the code that built the model. The easiest way to find them is to visualize the graph, either with
-graphviz, or [in tensorboard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3).
-
-This frozen Graphdef is now ready to be converted to flatbuffer format (.tflite) for use on Android or iOS.  On Android users have the flexibility to use either the float or quantized versions of the frozen graphdef, if available, using the Tensorflow Optimizing Converter tool.
-
-Here is a sample command line to convert the frozen Graphdef to '.tflite' format for  The Tensorflow Optimizing Converter supports both float and quantized models, however, different configuration parameters are needed depending on whether a FLOAT or QUANTIZED mode is being used.
-(Here is a link to the pb [file](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)).
-
-```
-bazel build tensorflow/contrib/lite/toco:toco
-
-bazel-bin/tensorflow/contrib/lite/toco/toco \
-  --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
-  --input_format=TENSORFLOW_GRAPHDEF  --output_format=TFLITE \
-  --output_file=/tmp/mobilenet_v1_1.0_224.tflite --inference_type=FLOAT \
-  --input_type=FLOAT --input_arrays=input \
-  --output_arrays=MobilenetV1/Predictions/Reshape_1 --input_shapes=1,224,224,3
-```
-
-- The input_file argument should point to the frozen GraphDef file that holds the model architecture.
-- The output_file argument should point to where the TensorFlow Lite model file should be generated.
-- The input_type and inference_type arguments should be set to FLOAT, unless converted a [quantized](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/) model.
-- Setting the input_array, output_array and input_shape arguments are a bit trickier. The easiest way to find these values is to explore the graph in tensorboard .  The user should reuse the arguments that were used for specifying the output nodes for inference in the `freeze_graph`step.
-
-Note, it is also possible to use the Tensorflow Optimizing Converter through protos either from Python or from the command line see the
-documentation [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py). A developer can then integrate the conversion step into their model design workflow to ensure that a model will be easily convertible to a mobile inference graph. For example,
-
-```python
-import tensorflow as tf
-
-img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
-val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
-out = tf.identity(val, name="out")
-with tf.Session() as sess:
-  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
-  open("converteds_model.tflite", "wb").write(tflite_model)
-
-```
-For detailed instructions on how to use the Tensorflow Optimizing Converter, please see [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
-
-You may refer to the [Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md) for troubleshooting help. If that doesn't help, please file an [issue](https://github.com/tensorflow/tensorflow/issues).
-
-If you would like to see a visual description of your TensorFlow Lite model after conversion, you can use tensorflow/contrib/lite/tools/visualize.py by running
-```sh
-bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
-```
-and then visualize the resulting HTML file in a browser.
-
-## Step 3. Use the TensorFlow Lite model for inference in a mobile app
-
-After completion of Step 2 the developer should have a .tflite model.
-
-### For Android
-Because Android apps need to be written in Java, and core TensorFlow is in C++, a JNI library is provided to interface between the two. Its interface is aimed only at inference, so it provides the ability to load a graph, set up inputs, and run the model to calculate particular outputs. The full documentation for the set of methods can be seen [here](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/). The demo app is also open sourced on [github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
-
-The [demo app](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app)  uses this interface, so it's a good place to look for example usage. You can also download the prebuilt binary [here](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
-
-Note that you'd need to follow instructions for installing TensorFlow on Android, setting up bazel and Android Studio outlined [here](https://www.tensorflow.org/mobile/android_build).
-
-### For iOS
-Follow the documentation [here](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md) to get integrate a TFLite model into your app.
-
-## Core ML support
-
-Core ML is a machine learning framework used across Apple products. In addition to using Tensorflow Lite models directly in their applications, developers have the option to convert their trained Tensorflow models to the [CoreML](https://developer.apple.com/machine-learning/) format for use on Apple devices. For information on how to use the converter please refer to the [Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
+See the documentation: https://www.tensorflow.org/mobile/tflite/
+Documentation edits can be made here: [tensorflow/docs_src/mobile/tflite](../../docs_src/mobile/tflite)
diff --git a/tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg b/tensorflow/contrib/lite/g3doc/TFLite-Architecture.jpg
deleted file mode 100644
index bc83946647c6a923a8a0bd3a041b42e4febe6a31..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 48710
zcmex=<NpH&0WUXCHwH!~1_nk3Mh1rew;7xnIM~?O*;qN)+1WWcIk<R4czL+Fc_f8|
z`9)-<<mF_gWMmXn^wbrUbd+UeG|V-13=B<7Oyt!qZ7qy!^o&i6K!z}Ka&q!;^GNXW
zN*F21C>oIr{vTiv<Y2nMe1MrziGfLwky()O{}Bdx1_nk}MlfK20!Aig7FITP4o)ua
z|3?_M3NSD+GBY!=Ftf6<urM$%)-p0NGq4D<3Mm>ovIz$!vMUve7&T5@$f4}C@t|nX
z#SbdRNkvVZTw>x9l2WQ_>Kd9_CZ=ZQ7M51dF0O9w9-dyoA)#U65s^{JDXD4c8JStd
zC8cHM6_r)ZEv;?s9i3g1CQq3<ZTgIvvlcC0vUJ(<6)RV5+Pr1!w(UE1?mBe%$kAiR
zPn<k;>GGAU*RJ2VdF$b$$4{O<d;a3(tB;>PfBE|D`;VW$K>lK6U<UgNL_j=-<}X18
zCPpR}7G@T9kiQt2${84$m<3r_6%E;h90S=C3x$=88aYIqCNA7~kW<+>=!0ld(M2vX
z6_bamA3<IN`;0h`HId~rxW^Fwy2Zf5%*enb$SlZU&+z@1#-au;5PrC)*!03xlhe`J
zJ9S&0XPeYkUoVc)@2!85bkY9)e+DUrm46#o)L%GU{gcangDL+niTrxS=q~|m?BCcH
z{%3IJeg8z`-+?RkFB13uY0barAR&K4s{JpI`Thxue=`UCXIR+w{gdc^SB8Z8ge>_l
z2KW9s^}peD{?{gb|3uhd5yr+*!$*T<G_{OoqJbz|G&f~LHMyPr&%nw1o?%|oE~&oy
zuR`^^>ZdX|KG2@sy;|tI(UwiTr}yrfW3cYnj`{8NpZM~>&;QThBzil5(NzQ$BbS-)
zR=0j!|EDKP{r7){#|~0c0$mwIsD?vIyT4vGS>3((s`yj6wYM7d1LMr)wyro{u(@{E
zgQ>YUjJDZpeLc;+W&ehzSN6M=*#1;^F>D08R+4?;q6RJ}rzTGZz9knpZ27XZ*Z=r3
z|KHB}PcoP+zG_P@5j<L%ue{wgl2QG`*{JoZAJ+fr`4<0v{*&x$MT`d(0+=pGUgY38
zwe#iVp3AE{Kbp6d#`=f<_T9W@qEQ30-Q5}g8EjWDZ(Fn~^04l_=^{ZVpPd$oNdKx3
zuDImK>41ILa^I?UG0IHJwGMgy^8$M**i8xwUzDabaA_=Ik!uNHy69T)>wZuEm%B-i
zE@><<Ja&NTRPlxC3+#oi?#wfF9~WPpA{+z@b|Lm4RtAwkM}<<01x#HU)WM7A|8<%F
zZvNAtkmCOgD<=QF{AUHz{KNZyvYY=t|DRzY*W1;dKg$0zC_MYmQ2C$Xi(>3njF=TY
z^yB(JH(lYs{~126d$l88>OaGYql@bI|1)7+3HCBt!XZ~IM!x^6?j^_NZg=$Gs~lgS
z_~-S`Ss_jAvz(ZerX^J~#+j~&+xl<0>64ju$<O8Pwm)@{-YzF^_(M<n?fm`spKX_U
zV^iO_ww~ctT<V@%HNBtR%8!4st&e{mS@h#kWX`OMTYlGTF8=g!$CqQ;!p>Ltrmwzz
zm-*A`#N(^x@iJ8Tcgs#bT<+5o+91>1d?kGD%k5Ho4|`@Ns=WKmV|LAwq59i;$B)VE
zANF3>JT*0lf5nM)$q&E(Y1ONL-TzgDarML1()amq$KQF?5oFNVeoorrT$R+br&mhr
zMPAKMZ*vPx@;b66G_=-#ZF!tFPmO8ycFS9w$CtSl*)eFD&TKsPnDh3}E{2sWUi>=W
zE~>ZhLfN!qKVm-AzFz;~-?m@pnhP#Jek-ujUHQk32iNXCzU7|h`?AG(;(TlA;vZKd
zo?L#iao4}|*-1>Y|J-c(jQfu@*sS;b9sg|&)A4_H-GcHPmDlEE{gb)2CU<tg&Yi-!
z2a7fdoqiwkNBcj6vhDQW&womN@fWT?vAy;`L(Kj|4F)y5Yh&*2xh8I8GA}B>cgB;(
zIfC<#ZQPStXj7*=H7HZGOrPrr-&FO6`X{#Pe+%-TY3=X*&+sI5`ro7fLKrW_XPVDC
z-d=Jzw@As*dFN@vpPSg17R;($vVGPRvyC1?k@8vF{Mq(Kd;MpaSakj0-u6%HO+P%h
zQ{VKTVcq@54zk~7^X!xQ@a1j2`JCjm$qJJvI2?F>Z^!;^*>>7j&KP<fD00cV9VmWp
zx4cxloz(1n-&fHuXFaU3&tG*X#%q@Efwq7J+s|*A=peTK%iQ$CYSs&joP>KnS{|$A
zFuh$-94obL!gb-dk%>Q3kNeF$TWlUD|1tKBn)K?a({!F^pFLKuwR=fMRNLf5uU;1j
z8y%j)aq|WLj|*&Si`QkvDP3h<JA0SQ+ieyT-k$iH{OeMdob0s{@BD2)|1)SkF6?ri
zy6lI}ijO+h8&7Ss+p*qvXNK6_TQ80&GC%)pxc%c<hQeERODiAca=rJ<Y`UiqyJ?=e
zUiIVs@@ors&7HGNYRb0FCsZaGF<YoS@A|r0vSg=9V#2kh2Y&A9-FbYU_F0=yP+YzL
z{rD#r!y6mph4)^zMR(6$I?XbXF~rc~{k<K3rJj|2)LpgZlc@27rsuNRS91R|Jdqdw
zTO@yK&vMuQ3=?Nv{<r%-Llnc8zd2JcyziRC(Rf1nMRwDN*dv!Jx=bsVq<*^HmXy8s
zy8RR8`u_}Z@Bj2xes8jWlG6Wg=T?S@k4KMOeEmBp-1XD+)+F1kC1r9-*K;4u6bf$@
zQAwSnp}=?3=5JQLET~-g&yX|y*Y)Cu?|+J~sej#dfh}zBI`0K9u1)*+`0XY)cAGZ`
zB<<%GeUsAL`bgu}hdXmy9@t!xpYfY(^Wz8q8Eywm{_dlk|18s_n_t*}*^yth54>|~
zR;eDd=LulVFx8DdbZh<1cPizs$~8H+r?#(H@7t8Jqsel@Q(0%`H`(Xj{ZW6U{nBN7
zkh{)|1L@P}gqJUg_D)}NJNkUTk;S2Hd{qIgw?I7vN-?Zf^$a}!Z;}6Z|4$3J>iPaN
zJdwHo-u_>U_O^BE5AXkJp8fave})UKWmgyiT}4oEIQQfDpAuXAzyD`AwtLmXn(Y4!
z%3_`W{xkfzz_wH*(1DdoSi(;Ccg`-ytsA#b3ZJ=c-@!ThRYyLQx2HF~s-Ej~W!Y7(
zZmqBCE7v{iVrYI-w*T{6{=dIP5C7PGc)C$mcJcCCCY?$(k4%3rssFk$_OA%ziAt-f
z*Bh5?o^>g)T!KSjy$+*<(&`Frul*aYq*t$wObqL1*m2A+d&i;%<yj&-8Xrz9S6&$=
zcWlqI&xfUsb=H5IYv8GqRW!-<d&3u{y;@xipS5S$1g{rqdc--C@tiD^hTZa|WmbAE
zCb>e<H9sexTDnAep~VaBqArHd@q8Jx=C1G1RhxG#<58IE%;}8>*F8QW;9b1no->#6
ziH{FMt0o`K40MnYZ#J3hbUboW^lHz<o<kwV>)494<i%dx>V3Cg<=VAkZ%J3isYO=d
z3okl|Y*`U!D|JmWWc3=YHOp5#+VkrB;akThuaaDw`*dR9cUK+f@;Zz4L5ms;OhWcu
zHSz8!-EqHU$vVlhxjX(dNIiM?M}N(W>-RG*?>QOj7Zl-thJB&@lklY%9pnnW%Nbl*
z{M)eaVQ3pChw|e)A7iIACYC2IS-SVjqfGX^-QmA?zK_}xz&<zYp3oJO*Z&!|zVy1}
zB3+eZZ<Buc-hv%=+SioDCLb=i_<TZuf_?DOs#Foi#)tL%bA_(nO5-gyGk^PHL5b*j
zP&8e7)nA**?JKiDjPt7hrQN?Ym?wRhdu*rbVdoV~PUp%*_#4^oX)-*2X@lv0o~$sb
z#djhOt8AElU;60D;5m<FvoC8b;I}sXu|0BYg__WDhTz>rEqC9(%Dg9ZajUd$rP|KT
zvox9v?wmeaB`U&rpyIg475%N}t*)JmO#XBwFZ9HD)9Wt^PUbI)JgOx!QHgQNSGFx0
z3wYuMUvRFA(g@=$l=I1}s^u*#`?gQxg6!h$Z3Ro7v;@4U{e69Tpo3g>e(IyWao;4<
zCuNuRZfCDih+ovV^#zAya=`Io#*@dRN>MWmdIlnv4{!6V@`B}AhG4YjPN6+=*2Qde
zA+>i*?;_{i22+hiT?|yk>&lNswO*UM#8r36F-^II*7lf$`7dLBc|Ouq4wpK}*j9gD
zv^U39OVg_F*p?L`t3PPZYjv&sR^$4g!Ey4xdajrs-T#`^|NIj@|95HYzvBEAkL;&>
z;nF_;`H}uFp821D1kM+%zmW2uVg8~1w>tWt+piSLpQ>UNeg5;K_+K{vpMNyYckF**
zRDb@V`?qQ0KM%jM=znU<)b;s4!;cmJ86XmWLh8Rr+CTrW__sUvpNC&%+&|50bgldc
zcY-m@s=td{{}ue|Q~v4iuxQVJ20Nwy3=2;FXRvRTgjx3Q;-UWx7Panzf0i!@jH`#c
ztq1B0k^0QU`Z;!sJN^Y<&{%H|bDJC7F8eJ7`_IV-9;w%SAre0S`H^^t%Mez{-?E5*
z&QGOK_yc#=QFmx~qdGwNkpll8KmMP8w9^k(O?|g8w@hE%*rbSINncgimqL@(@i%wh
zIJTB+%0AQN6^pvGixn1iX)sU&uXz5SAyEDI@}D77W&Sg~Qu)tN@}EJAVflym{~XW$
z`~08bh12e=M|Gg;*>eBA`oA3ExtLW<!9LUf3=_Lu{xj77$=<b2{Mi0ah8gqU|7YlB
z$Rako5vh*xRK<UWe-0DsuOIl&a3N}S^nZrqTjJmGKMkv?|IeV{VgLF;{R`dDwf`9&
z-_rli{%Jk4&5wj?OSexdi@LO*^B2IfoVL5)hAq20wtPR${O*90@%(F>SAMKt=l{Jm
z*YRjb%&8aWwxm4x8Qc4NK3`OnfZCO-+&m2@d;V?h%E>Di*!F4DSN6mwRoC8k|7W-!
zUB4si-sTnOm%qH0oflHs*>>uA57*c1#1%WwMo)FSon1XIR>w6}!uqpi`6o+{w|Q#5
zA03rfyj$h5dRJ`g-q&Zg3wcU9P7tl1TN*esci&Nw&7wMmPY;Tps_lH<f2zC0<+^@X
zgR#)$<8@gNrykPN-TmvApv|EoU)6m7;12J|yUy-B>y|taSn_PPj?sbNs=X8UN+j*-
z$V=P&p{Cg3b_$P!#W}m=aFvMneiKa(wWa^;tDd{_U+MI3DXL3S*Y&?Q{dZwb>{D6O
z_`>7T-c$R}WLL7(-_t7e?c!X=DRAhM;PtG@UGG(Pdc0rK<v#Pu`d<tA!sT`z4fxEv
zynX3x-x$jYb6QU_moLAxDl$arpL%Nb_X{(g1y9<p_icN!bl|1L&6jfX!s};iGB%ez
zn00*J>LdOtvqWc~F;q;5?kV1KT{|YS*CKfBY1Y(~vV?TaOM7;2Sk$H6@&-H>tFfp-
zaNo;c!FBTg8CLB7&#<of*X7yo<}WpW_@Cid_kV_8FYMRW^0|Kd=iWX0t?=eWUD_}F
z1z_<@DXnGspW%w<znA|)Ro(4>b=u#xKh>c5$NE3RgyQ-={~3M>th+U6#z0&dissq>
zXPBTO|Mi3Y7qL~>{xf`hEB?FrXXt$S{|qOD_<#M7|HU1eUH{`-_uqqmRv$QTCn0w(
za8IYkYB8rPcN#0sDIT}By6#}oEAsN$?=w3;u?Nqq*9q@;X16-IW9#y=s?v>ti{7f7
zi|e{L*+2U9q+KCJr#a#xj~Uc$W!aVcM4Xdlg3*cRT(<R6=OU_BhpgMZw!2Sh_T8J`
zs;6eGm474a&r^5mzW$CCx?66aK6HpfcH12D94UYPQ0t=oJL>LDU4N!)%Yvy2eT&w_
zM8rm&+UU!?dOMfloWEM-7fmG|evZ2Ot#O`#|5K;MVZU5o@NL!BKAGiMUSC{u>bg+A
zmVpIR_gbfCUzPhR%hx1x8L(-7`xW~3j#7j|x>MoAu4|hvtaY1jdTc>z@1rCcMH$9e
zlNCwX@ujt;maoDN2YILG9p(7rxYCuQH_t+rGr-p%dsf5DrNV#SER1*)=l(Q2dHY4x
zT}{@R>SdK8qJ@n;p98XX<=stvo3l{;DT~(UsB;o44$t)ab=$pI!=LBrsfT&H7EQdO
z*W0n#=-&gGx3_GIuTAsydKZz{xy)Me`1A(GnCi>#N@l-y;Wyo_HP84;klplZnTswL
zWIz2qIPq+;X=jx6x7VMn!mrL-b-yv;M|-egW!mFy=DCt9a=upI7C0o?C-J0BQu6CE
zz3Ek-y@d|ke)RjXa#wEat~;*;4yrlrPI$aFXrAgh^St*i-ffbuH?{MQ$G)0A{o<N(
z<NQ!d<_wvCp+VbA-}332Ni5&47ch72j^}byvko6F`<$6wTyTj~(wl*gL3Xug!bVZ8
zQ$kmlUO8v6$JWZLFwA$(QPI51K701D+&*w_@qWj^)2YR*2bw%$Q*6a$>!u!iam9P~
z)ElouS8mUK<>*=xW!I><Gk^PSdF?be|N29lj?FGCt+XlIV60^+cX)o(tq7})U6Z4~
zyh@dc)$~2~yHw+1)U;VK8()3<c721<>d6ORRD2bc+nOu7Y^9FDx+@X~{%EPT&B^@j
zzHFP8MTgg-<+3F^=iD#(tGS>5Kf{&y{|wh1|7M>3?tf|ahyM(JUH>!uEs?*rcTV@)
zKmEq$x75>JMZz!4Qy{X2LFsK_(=)^xhEHv<R*b9dkc*6MPip7s)i=~@eJDP1Co%cP
zDcL!l*~|O?xy-cN|J$_ar>MfN8O~2S@1L?ixa0As_0qi2H;dJ_huQl0>SrxZ-@NnM
z3u$SG(%#4=cWzHJyY>9zuaL{9{J&M%ZaMk;;nt-I{*TXP_cpzXPQTByR=R3+(p=TA
zXD;bXiaBv^f5|jUd9N>;&%8_?%Y`pIY{OhCvvW&&jGo$mhL)`hKgS$>`uFUek}r$y
zoLX5kKi%j+T3%xPoZ^=;4SRM7SKiN<eD;|6d&@sBllJdya;tck5|%to@w|PM@{A>4
zyYKGb6Lk7^l<B5Fe5YI@^54jL-Z<vTDYN6wa+zY+JylKF--Ue6f79KQx-UM5>(R5F
zvmW^yMb5obqI2`8*R83Kk3O}%vu4II?pf>KYTWy?HoHDPu+}MTyV0%l{}``u_bWT>
zn|1Zuqp%z9zpu=SeZ0Kwo!%0rbe<#a|Mcc&PZBp;qb6wF(I0a6;2+JPXDJ;=HhU=B
z23Ruu6P^F9JIL>>cJ#B%udJsYmwerS;#PCm@4BU_@;4LPa}I85o*aE)%D=n&0~}@?
zn_c|H^x%><bH2XN>k}LgimKmwX3W|1Nz(n}bjv@lG}44xa!t#NRwpEzzq8hsP3?+x
zUJ@@GIy;k{$?fFJu$(-_>7LuKs6KtWIPu40_MhR>wtvrFYMuA+3G-%iQK`E=8zsWh
z1?5&qytQx5mN`B3rP$S7?@~T}S6wsp{pF3f<InR%xgJ|6&RZ<ie_TTT+)iKdf6ik2
z-*>IbOUrQ4HJ;J#qEps?XvQRyJV}|=d_8;|>#Ba7&yEtcEQ#golJHOW3)r{jy}Dui
zz1WwVOAhV&&+v8Shr2qL?mM-AFMA=t__}%i=eIHHxAkn^Tfa5>Ipf&h=k{}zV$Vw^
zO&8Mp7tdFhzqa&0L)PoM@az8=Pye0uHZTA6%BzBf`fa+?eRWh=@&)I}*I!#dy;uEm
z(Az0@a}_4P-ah}@jt>2Gy2VfYJ~m&^4p!MQDYNFLYFhn!v1bd@0;5*!e!1M&TI2)o
z?qcn%KVPl0HNLW5*SlBrpCL{8$;qF!8#Hsy<S&cS@p^YC{LV(RYv*1)z8$(RfpO35
z#>RG~MQxudWKN2_>#Le@w{C0UnMbp{bk~;O5472pn|0m)z?-GBOOMU!&(}%u{IsP-
z_`=t9<wxp2NzeZ9FVyk<x|FG@)iZgzFK(CVU+SuIUi<wtw=!FAnORSdJ{HX=Qu%IW
zu*2Og%Xe+tcIPk4{@DL#2(16l5X=5`sg6OVtfmd)zh(a!{_Wy_9lKL%^ZXM}&)%w8
z4k_3)uoY6|hz+rV?cB936{Wisb4~gs!ucc(=RD|t9bx~WA)0Si>-i_z>laP-d}ng&
z#uNL8jNAS*ENDIdMB_h03)Ag?P2%<w*6r(x+4x50{LT~ghZuAJGdOb3KN0w!frEAX
zUlzT3#n@VrBKHkHm)?0||BwNzZcF7GF<{-yNV<9P>t=)74bgq<<b!C7#rkT?tu$Ww
zF|4qED0IpGh1q|G@0YIoKH~q$6jRIoc^%gd!-)CGdZ8j!KX{mqTrMukxc;A^#oOaQ
z!-CuY8SZ-rAAOLmIk_fk&b#Hi9&{aC_xP~<CszOOw|`zb^r&{r&zK-NYyF6<m{-0t
z<v%psuD{^){F6xiA(*Sf<6Wy{HomEpx%93GhrG<hH+t!K1EL0^IU0-MaLw4`z%u!U
z<^19ah<I7Mmfx}2sV(B+)F(z}&-A;lE}G?;!@GS)psUCcYE2r7B6V~p!KSzzME)W7
zabx_Er@2;ObhVNDxaFZpb6g8}y(k?aCQARR>(~9$^U@-uQsegsd}3d#X|26!vAFuF
zu6>IC878P6zff-&BU(~5z2o7&4b$!_O!R&{$zA5UWt^DR5-nc~>-phR16I0C?sd&|
z+TG7ne?rv$e*W_q+Y%}N4#mrB#O^+r*pXDkaol7>sIuSECvVzk1<uWq*>d&nxkFvc
zm)|%2V^@Cjo{D|R&foXFcE-I)S+cHw@*Ek})4GScG*+ol+V{eze`xi~L_(`?|Bkid
zw|bY_W%HJG_k6E&{~21=yZmRE|K&f!KK~y9Y-dG#|1)sLb^d3tfBB!GX8wo8|Ah7m
z|Es_J@2CAk2WgWq^FOLp>VN(-EdTqd{!sr9uP^>T|1<die)6B8l_BS1#QhIbU(Emf
zZ~pg_{~5UKA8KE;fBx70{p5cljGJAP>kox5sz3j${=WJjP~k_s#olhLw)uw-evi4S
zCcXNI_3!=<>q46NI1b9L-iGRdFtM$b3+}P)*tV}X{goRB|F6>Z=HkuIzQ24|y|XU!
zbXMuSxsu(ndYum+tKFV`^-!rte|>S}FVl1Fd9Fv_zDW!Go_tWT_H0Q!vM=VSalgIt
zyZel>_r1B|pUoH;7}yw=uE|I7x;(OmuO)UhHy3T(kK}t~pEIX*9Yb|>AZm~=;GLx{
zjTG$e$U)v<mVwW122_hh>y8I)zOpz|=XqqPpR{Z*LliYe>0o9Iz0tZFhyM%{WFP|*
zYgU-77HcvyJK-7NoAm?xFkYNoX;49rWo&=6R!iE`GwZ_to!s!BVSdQs-)e{d)h4f=
z65jZA*8Ow;zE#H^n*M41j-`JmC;n%!Tj~DIxBbs=_t>!R>YPi_H%~5iR7^WKaf<Mt
zRrl5xX6IM@s<DdQZM}l)p@XG;Yx%Lcp26R`Hov`IcbsRHo2l*Wr9X2D<8PYDe|{}1
z51~q%4_$i8alN3s$w=vaK-7<_?2cvSGZhUj!}x4%U&+dpzAgVgZS{j&`({<j+y`0y
zYbjXgtEm_7Rn$dSJe;IC>7i=Eg7q@<rdobD+<*7)Hu){Ncb_IrTHLogyuRY^HldR*
zzssyg)f$;0d}qldZ>8uXEi2Q%%4X)(*qiRTz^LUZaz->IuG!S?y!7d_<=^{nC0gz*
z`Q4Wf2^C1-1nv6I@Wbuj@j~IV#lroj?9;cuXF3ozZ9!*tfzW;Dwi4}02ifCK?MiGt
zTC_cAaoE@Ti7g*3>QWZ6ynnIZ`22r{O*$UuZwmA;TmlL5h}XLsr{(eo7w1fn+5Yap
z@BJEUPUrKU*4<O=Uo5+S=6BIwvpO#v&3yCh?$z%ZE5Z-IJ}<I%dGSn$W@vl`?TYwg
zS#bS))+Z&SWnV2$B)$#k<$L_w%=^*-7L}aiD$G6C;?7Io{rha+b<U^XB_IJfr9S0G
z{rOd4&z3H=n>As}q}3*itsS0(O`Dy~GEZ{%%!j>8f^H-&nE2zOpT$hSkUO_B<I6P9
zwInQCG%N46OglI{kAsuLucg^9yP|`)2Z&61+$Je{S#NTF`P%c}=BqyqVDUUwA#?TN
z%17CUXBk)DE(~Ak*0bipwoexJZ)WrV{OV=DDaZc#RqbVV^3}aw8M#mUPx*O2Z~gY`
zO8G63Y18&+c2&GnlrdJ%>I^=%<Y{^8GmEQ-woRRSf8x3S3}D9xLHrPUOMCij)BSV*
zJ`dlp^v~*d8gtbSv;P&f3f<rS$FYvx-=JREFMn6vl>6=f8D8oCXL!Bf-`00ewyN9J
zrPuZ=n=Y4F@oBZ}T#>-IkB_M}ugeM>&Rha7OafT#kymqt3!@DFW6V-%BQ58Oy^1{c
zpTQhNt))j$bNGUf|7ZBb()@29`@g2EQTPAZF8a?<P_MQ5@qdP&OpgB<;+g+5IA*QQ
z|IZ+Iv3`gB)C$Sxhna7lGM}g_5*Tk$-FjE?nr+z4xlJqTJS~{Elv<w%KJxL->Q3dL
zB~vZeNNKN|rxmdL#PsMnR}D^;THbz{yl2we)R}p??{bb<JiU|J{yK8mH_eh-|C*1I
zvbPyz@?Kc5zl>Qcdei1w&~MxS48oHvllpI2RlG}(liucBwBoha<mPEH<}qOlCaFuW
zd#Cek`|TrhrFMk&KCy63vpa0kB2eIFc;h0^xx(<m7t`+upMI>qWY?Wn4<=m>ytwT0
z!yOTKtaCnF_dE%HvTN(cZI|EJU6^F$v2=!<2=lBb87m@Zdq2GXMf<mH{G-do3$CtH
zntbHjr8TQAHp@krIXu6%^ds9woA49c!s~Z`I#wp&R=F<w<DyN|>~ekG{nCpg9xRws
zx%Fw7m9O0=<A-k@e2pqEiCH8W9T!b|Wnp^T+r8_ZMNfQf#Do6NT~Aj03Y&N)s;T91
zY5s}M1aadzv2)LDdmYhG(R9I8X-fu|b>E!dSC3y^D<15v@KHLwx7*Ck$p1gXx*b_s
zk7tMYzTCb450hM^?~?VGf2D2jcz5k}XufW<yHslspKHd8^(?bkzO647@q6QN??AYr
zYw_h{7T2amYJZkK&oNij)3*18$%&HRr>mP++&&@ZIAM*w-PVP&cX$2R=E-w%<0CE!
zbIC_LrL;YguUcyrY}$Kb9#`<SqJ7h>wK*Sb*t2`<v`4y^&1dw^S#R6xyy}i8-&7O-
z!qm@mI35L``19+!x@*X0^EY=ya{n{1v`*J6waY)Cv8e0Qc@MYW%U}BbsQ<SJ(#gii
zm0v8yZ~v6j%irR6u1jP6LK_EIl}srO&xONw0?rnB0<J^@GVh&TXxq0vV5gn@hOJ%a
zO!)pYRNhE8b$a)Xw={QApV0I(4r$zt@55C0T;94{BIw<N+vmk=N>@C&vgNE@bm*oI
zkBjE#HDZJeyS}eF>3R2TwRx`ny5*gIdiJTmU2EO{GaRx_NH<IRx$ow7>jJ}n^-@ij
z4}bLJy&UUzQ9@>=LgDeM{omFqWTl6Sw``368u8rXXt;0Mt!Q2s>y}C7s=_i88`wXs
znyRA`dn-0~rPealwbCzK?OM~eeTY3+svPa{FtXzbQ{k-OhfDumkB&F|(=7M*?<?i{
z-2V(5n{Ia6*GXF&)bE{jF<#&{uY|DruY~Rd3Cr{T|Mohy@dfSn%$Wa&>8VagS?;Ur
ze}tyU$!#y4RhcK7JF#me_p}_D^v{dp`IPQOttmE+_MafpBX}&URz7mORK(_K#kpF~
zb7q`)?Oz*qQ~%B3EM6mJv58#_>@0lE_tpj-+*O#Vo8!E)yXRK!M7!Rlk#~>FY(JBF
z&(4SW3ro_HYhQL(_qyHVP7*4Q3(ynKw>M463NJXl-t>y^KHXP0zaG<0`y8?%*6DUo
zSH0oXnr%INh38ETrb!jfk@AeKJ?drp=%MwtH~&7VeK@;1zfF53?<t-M;ZOg?Ut6^8
z!{5!Bk6yicy6~XluY`n0Tb5m2>uvISwSkjGXF!VlsZH-ntt<Cf<(}ESPV&28-+r^Y
zsiBz<9+pii^S{s$a@6bO_Wj9150(3#xpaO%#wWVh)!kL}^QZFMd2d1|Il77jNq~m3
zdeH{=(MSKYUli$Y|6~($|IOUFB7yNv6^-OBBt_Q;Tc))V-t!D$^haKzb@d49v>;?7
z334YMd6CxjQskx43|AIWX&xVH4)c_cKh&LD8V~G{{Lip2YUhhR?r9U1B%YsI!OmrW
zY_Cp(8wX2JW#-?be^$>HZdu;x8S<he;K;ewpH}k<|1)svPXF_H{=c8{kKO;MY>WSV
zuKxFv`ra!0Y4h*IpX%4O-?4vc;eUqYhr8|fy!>~0{a@vC_77Dz{b!h0{GZ{z@}G?V
z3@y`6|EoOzpP{Dz<9hc`mw$Kv41Bu$yZEQYzvX#v{(bm&dHgT;IrWFUQ~xvAJpa$I
z&;7?mWNYL4A*xUNztex}{>@H$^Y6pI{q}$9w?Hv^@jszx{h!b6|9-MRwpSja7~;k^
zHRkEx+rQ1Ne<??tf8=lc$UFa@|E<-3hD$b&k<9J?;hD4l`JDd@-%tE!kb3tu^6a#^
zx<@=_Uza%b@!HP%AD-*$pHGqhezf@7fd?zCJ&m{X21|GJo9HGRV;C|8YRKjP3@x+1
z|EqNT&rlQYx#-T;e{(jUnfPAE{nVa9uW8(@VPRe6Jgz=@ax$N{_9<f+19QuldT>n0
zMb#J9YrXt;{BT))<xw(2w*J6pRF9%WX!gZFTmC)z7XnT&E5J5Y)MsA*H}Sdrhe<c<
z&s+Xyct7!<2}*okU;Go4yjDOW|8IZWn|~k3Pb;(M3*8iOpAL@ct^edxZpWMP@ieo4
zSa%#8>s}0j4Gh1+!=%~0dF#`!JYV0nGVab_)<s>~Yo5W@5Kv6RR(}ol6`S~a2vN_{
zJL;cs7JiXG6V4j@zQZ2Ij#Eqi<5KJ9y;S^_=$(q>e-11E{jRrq_<F~O<9`~1|9<;d
zvgtp=qd#5e!hhSJxv{8A`}WRH3tdHm8K4CYP3WQ4Cc-*p9Gd-F<New*L1Mvwbe6OF
zMlPB%Y0_pbqr&p&dDSZ;@80l9+L4xOU%L4)*W$Rz<(a&1cRO3<Isa^arQLqDq;t#3
zeK~t{>O2|#^6#yfu}5d+zKmD>D^GvsxE3?%QvA%hVQ+HRtxpq(e)s+0f%sy-%Oxkj
zYJPJuXnw!*#iHG_r8!J~7m8^A`z*Jn_Eo`-$dH=zE@6rNtsI|pQ;hvCFTVWXhQvw5
z-{<c7#xCmmt}H5=QnTdQbG<Z?K-WM;^cs+w{2^AG&i$y~@%&<*?t<;xBIhOepYiyk
zZTwH@ZvMZ9mp^Bv_trl#U;4NHTVj~@$uotY51rD|`fe-3$7Rv6U45hYAJtv=|1w_w
zY*K$HdhLIP3$pgcK^No7zdenA8sB;M?_}@YO8*&JwnzVGSYTC`*8ju1bpJ2r<<GLB
zH2&RL{b%(@{x`E?Z(=hgJm=lM<3*b*J=GY0#-xOY^S&v{FnT4Qw4HC}&$5Ebb$85C
ztzPb4a&5QqWWmN>u{E>(FFgFu@KE>Ke})%)^JgxGd*H>Q&9#;DJ~{Hv-IJWDW$}1c
z&ObSgYwvkaon<*E;<0zL^RY*_p0x-&Y}k?VGhJ=w`Tq=XpSZ((XL_ankiPh>DE{s1
z=WZ`rcCYaL*|4o2exJ~jdz9Ly5wY6Lcux7^*A-VTJ&CW-vEyC+c{4NI{=R(StnUI_
zf?iDAWpI$)^upZ?BX=F9TPbtZEy84y?_8byaK>p9!~1U6y>=Qtntt7I)2?qB_2Gw(
zo%NkFcYDXzYzr|76OXIx3vI64|1j-!{TFusGuIuHA3oc)by8-!z>GVq*<=l`&$@A0
z=h3#y?_4-VZ!ezPvch_Y=vp2(tuQ|=t^3zn!k4y(XZiU&Jl<`3sK6ve{Z(f4(W+N-
z%z765lIeOKSADrSYr0g5*{jP7?kz}PYaV*($yL+ueMW*o29f<)sy*vZS7`6Poz~eW
zH{EpEo3;0ztN&-<%-jB-!J*{OS$VjlobB7Ezq#a_UN)Uc=RHG_p~pSBe5;Gevx{PQ
z-W|XD*mUx-<orCBtPMB(pQqbnHNk10O#Rkam(RH#A_leltCcpKknUh#njCI&<kF7^
z6W8jn$*ujVf;F`SU5xn8kQe$d^vA6K3@Ml1o!E~hB~0tG->zM+^~d?Q+VnC<^&Zn1
zi_>e>OhYAACLb-iCAaL>zNw;*k140Ah_Wj;cRelD|H*czen)ox!TL|Sj{iQ_TYdfX
zCuL(m(WI^Y=F)9_a-ObtUT7@p`ogC`t!iSZRSArTe_a3PTor%)SJ%eSKiU5o6!+@C
z{_7gCe%5*WPI>3Mr|dXcMFQgl->`}V`sj&XxPABb`j)5LGbi{Ix2VtCdcGws=+u(N
zD~*>XZC@A_l6UED#G$Pk*EZ#+7cc+tZ8=L-&9q5A8%|FOoKsL7eCL&u+P1FfA1Z%e
zP3Ks^z<93s)=nwo*n%_9rLx3sJewEGd8K&Y^MLEu_J5eBdtjcu`srhTKF$f*c71~M
z>dkD6HlDa7v|5eJeE(0=6OX&%wmhEc$M<J%^zW0?;yJa8OEzAQeHB%^_V1DdM?N!m
z&Qm<b@t;Aesy_Ypt4Xi!=l4#%-f{4`{gZH6_p16^Mf)~HzkG7|x0Tr?|IJp(E2DiE
z#y2aine1_9g}=q4nM>mj=Rf>C<KKmi=R7sc6L*!ZE1It&bcAt|dB9zBp0vfkofc2u
zvfaD>p#H0nE0y}Yca~ib*fBj|n|Zk2#U-nbSAVd49m{b|XL8gW$;qc?nKC&PF-oqp
zUS3t%7TV*LoY1~Fx&Pd<HmkK3H?_ZBDcw-=%KF}2zIp5$FMRpERJ_yr^xUrbeww$^
zwz61>zgP9&;<Pl-^U1Gpb=ORpQ-$TrrB&9X>##^|dAjyp>GY~ALO&|IPR-An%4F1h
ze&IU5`A<D=JM{Lw6Yty7FMZ^8iqjfvgQLZdHi{bZ`eo<x^_X%5?CQRgD-sx!9uVj%
z!WbU(pJ77o`M<@cTOaKIDYoE0L*@RtNw4BR@$UT3ur6)e9`XMSN>}9Hn}1%v?8JYD
zR{h`uXZLr?J1<z&rG3X{!f3uBkZ*)<{AW1fbpP_%_~%jov_pN}6c#W(<(nTI@vU#;
zJ*RgCtCc3zvSly6$~SRU%#yP!I=(Ht!+N%=B13CN<U!Nwwdd^x^?Y1XE^nR5*3(xa
z{Y<FUfzRzgb?EH3_P)MM=8+=xWdVD39`{<GTb!7_^p)7kGWiO-NabliGtAAx>!iEx
zuA8R0@Oh@pB<t`R+eu5`+@1aOU7`QnE5*H+_U9`qez>E_<M!#rQJwvzqQ%XTQq$(E
z%`QtQ<2@GIUKs0fZS8yAMJfBE-Q)MIYk2;7U2)Xz=-r=ATFw-8YSUsZPF|U~(ABDX
zu7uT&&d{yCeT&`NBuzMi0(MLgeSY6`=7N{=+!mLr9+1vG={T=t-hYPapO?9>a^!Dc
z&}G?vE4koy^@p3QtJ-#E>Xf}??(tOGa;Jp(+U}hn-X03d`sn?<xcAtm;-0WcIcaN`
zZd<%=iQz;k9mlyFFHa7YKm6^pqj5ys&r41PCtTxm%TC$m?3iF9<RY*`zND%C*rKNA
zeixH>PLtkPsB*tleEZL{sa6%4r-G)=eP3JlHtAuoAjgW;U$)KuzSHZzQ)7<uuKl||
z*&P1VVEkpRZrQ>2%PzlNq&(H$%;Lat)A+Ymb*mTMK3G_$`=nWtrHUuyyVcz(=GTH;
z)n`g6D^IXwcs&2otTri=J0^iOdrOmw*K2h}eb)Y^wD?CFo6zz6invFHlh*7I`F(Nw
z!D|nC1P;ibx%zUe2jiratJ6;>PVG%*dA4bN{@Z6C?ig@7&M}KDJRj_}rQpj$`}EY3
zUqAb%tiCd(^6O^j8jXcpEtwhIr}*0kMor%?b;PbR+QO#wj<Za9{`#n6AD%uDi)ubu
zsNiBaaq{blq5ApDr@eceduFY$b*}U|!}(s%mUg!4sD$2q_svS{;nQGYU6W0BUd&Y$
z%sBdlXPN5(?!K$h0hg9s`QvJmbx<(W+swF6^HNq>8m|WDb%R|q?QcF>(jT*>J$A>0
zlQU97nH;PHV>gv(=|!IQUVq|A*IM?otZU~#3*2=<QLHE?@~&jf`h9D5tdV-Hzg4C0
zRPC-+XU-XPOkDpYd-8g9gH{dSYWw+L%c?3Qug;0zQh)u9NT6%w^TN?$NCa!|3^b^S
z2a{5L`#q3Yyj3KKVewMYa*yrNYoqRo|6aPO_T$-1FTN#L#Dlavo%H+{id-`n`}Xm?
zmWZd_yQ-@NOZyq-FIa0Y5Mz35z0styqn|Xs9L@aU6TSKDr;YO^EnjZi^{}-y#(iV+
zruc=YJ_;>w|F$VS-8Mhl_Rb*-<xJ@dWvh!bs(SbM>P&kY=;nFC;&1lSi?jVA*GFAU
z4U69w{qbd>Yr%9Wu_YI6)=H@Gn|9i=mO1M#ee+&3_SKh9mZ|-#&u1Jv?fd6kP_%bP
zlIN1vYv#Y#xJUIgEoVvieLY^qVCk|+*M+=Z#1`y4Z}DjM+MCh$Uw!*_a;JLgX)_DX
z^RFXAPD}MZ_?`JGljrcgE!S&n`P|m{CKl%BiyoOcUn_8G;)!*0eiR*YR^6p~$YZfm
z*zYA*W2MY?Y`S7{@|;Vh_M7^@NlQ~Nbw8aS@;2}Ff_rvGF&AF$Uq10{LZIhc@3`bJ
zTh(<7A^pxE&<F$5x~2aa6mHM|^(;FZ-sU`dW&acD;{OaWXLBp`|GBQPznA|!-gn}E
z2Clf}jkD{8?Aje&MZ$MfQmsQf#5%9-AJ_kB`?7xhSJ9-^;Kpb;s4*IEIu8^yd#BV0
zacL~tqkJ7cKDKjd%-VU&S>Duz=GC=#EqBv;dG?T(_d?^QIWh6U)j`{~Z@<3h_13g4
zGv4!TIjbME>GIdiyDm*PuV+m@DIaxm(zD7$kGQam%Tt3YL`6&Yr0e=sWv~5u|DVK@
z2=B{JR{V<o_1Q|=*57}_u04iZ){2}mo3sZsylS;Jc~!LIqPw^Dy*CO|EwFf@E!Pzv
zxALOi{*Hf3{n!4gV|)8-bFS{Rc@x|9^G==NIA9@~^gU|B?;NStofDVGPo2=iyXW85
zv%#@f*<zdeg9@W`4(+Z|t+gsseUPy__HEtXbK7$BeakZcU9R5I|JCaV&lRa)X<4)H
zfwL7(A4%-jjVYP6wI{ynDgUNF_fIXFe>Ct^r&#*TfJ6~K*3!$XJ6HX@Bm3s<t6y_A
zFLjzz8S<riR!CS-mekbaf2uQuw^fM+{}Z)}tXTB;-HoFhU(@6t$9y<d7<4z{#kAD+
zzRo*)UBlvee@u$sUz;z!&i>b-rSWW8LG|md-KbWY|8ec(iPx%6O6<8c-{qvHIpdwW
zl7|l*?!PXLZu`amZ?8=Sv!DHIy?3`?oU&h<wMqS2ip|}UPR3cPua)>$bLHR4{m<}x
zPyP2Bt5-kJm95%)xi&@qLdg6N%^CYYpUeOKBkP*0{PRB?|E|}cO3E@9*fzI-<>I7(
z2W+DI5v@^^3T0RMPsf7)GvxZ_+_-W3<+je;oYth62g0+yY=|&z?d~eyd9drKjmwl-
zTf0pHUCZV`^Caxh5%TGwl$93sJ>y(b@px+F<@Xx4T2Y?MH#`*=Pb<2oeNq3b_G<6h
zqV5@tr#2_gX^wcaPo?>9-Xxi-<L5uA?PcpawoPhk{v+*KmyU!i{a`IuaBJ%w`;Q-&
z#w=fX@IM1r){nmHi;o-0m&!kmo9t7ucuJDT#pA!7O^@p==GANWI<t{`<@3{zcy0GT
zUCh5~*3{Cuw!321KUz}n_2$0U<t4wrnLYn^v#ars_2T{aF0cL1z;gW4nt7hii_=z{
zt=aDJ^gqK^9sfUjUH>!G2K9QK__C|}-!IV%^EqGSR9?RPp-leTj2gv-@;?iiUn<x4
zx_^_^f0kOGedS+B!d$;2r69#3msj23|5?@l!Sq5uM~3si7{*^G{xdud(r^9Z@^ZOL
z-A?;67dBbNpE>+HZuzIhQ)`R;<A3Jwez9zUozj+b%6*yqIr{O}cCYx7Yb5`*OziZ3
zhMc1@9*=vHOhV<l45QrD4zInPR&+Ld@0!Y2@mb;WAKxy%V+P4jH!9{mU%JHX!S^!x
zN7;K9?Mj^XaPc=D)xIO%Z)HO21h1(-dmkPmf44*UpJ<*u<I4waJ9qA!yY~e1pZcg*
zE7q;JxSYRLrTEm6r@IXweAw2V-@DClr|gr{^;VbKuHCznB%$0>zhObhm9Q4<1Lh&Y
zv5t8sHl0`z^U%@OSM5@Ga$r%J%a%Y_L+t8FJaS>sFTY`!z~=Aens7?eL!qb1`bv2J
z)0;m38SXmOdp+4wFJRh|%X=VG=~%FA$ggnQ7?(_6w^+YZ5|-DOJ>_dpR$0lDtI{Vb
zKIzB@?MYYvxP5-tbY@ET!n(I#zN~rc9lqnrlAq^pwBPM1k$)Wd_1UAsb#-sdY}fFG
zN|nvcjn6ysZTjTMJ8M2{laZ_a+gm;>Z>iYQ9F_id8LM)(J)gFgXyniOnX*}etFA16
zcfWJwgO5TjKg)e?8y&p7+JEkD(<|>@Eb98+KN()3A<w}QU~ApKe|Y(A$)$Fu_TIS8
zT3c)S_q4C(>x<PIU)I_S9<I^n+0I;ItE~Oq;Xi}a;r0B|aYfVBPRfT)eW-bP`CIe+
zm3x;|bVd7axbw`TitXM0$FU{G?uA09h1=a;IIcHdy{m3z;j+`a)J_S1R(z59aO0)+
z-L?T6cgAGhEBP|<MM~qfiIK6+nk;gTqQ_s|Q#PM=ns?Kdti#`1zWxgLTw?cE?Z)$e
zk&SUvZG5zRkJ<8b?c-W1<+apTTJNH(=%vC|N)G%O^r?|c>h#Yl*VFYITr0np+o{yf
zGTHxDz&`5X>Z51k-wWBzR}EWqe@38d_K*5EYJYTZH68yO_kQ>Ms)x(@DmCsMUm3N0
z)3WE;H>P`j__Dn<Z6}+BRpZ_Lhjw!)blno|65;5+z%{As7nejuVBEXsCz3X9H8lFZ
z?TtvFYl%!Er1?ow700a7tJb@CqO)=-OSt;#irfcxZ71l92g{fKs=6NUni$Q`b?VZc
zSBD<GN_<*jYb_Uld9CQ)N5M&YO$Qdl@PBQss{i$xZC~9QOSQ+<H-2-8)~+?Rj(Qk-
zzdgRLn}4J7*L6a>?YVa5YEJ*XN$_v|+OpL5-<Qvwv*g2n2II0WjrBP{6DS!q8WJO{
zJ*yt`ENovWxgzS=#eaX-=zi^yn%Cv0`K~K?N?Do4w%+CT(H8p|yIS{HFEc!U^Mu_w
z_pfWcv}}9LzXh&3axL2P?_;GNSE&U$PCZP=)#kg3F1m1Z+ua+1t|A=Jbx9bj9r1~4
z%9byTnK<vkgG(13)RT28s(pDP4n)jbI%odtV7=bu>fRRpSBw9=`YJkWv2X2^oR6mS
zZ`D`E=Hv>$i#o91<G{+RqHLm0{SJoSu0dVV%I22dWfV`q(W_y#FYia=imdPTLhDO@
zm|AyCuUe9&ni?u0+Zyoiz(@VR>`VVM9O%?we#NEz^gqKJexd&i7pmg_a7^k)VmH_=
zShO!5Sv@j)RPV3|E5H4-<1gsvZhvjA_oiIa^L~$f)vsm8+-Cc4n|LnvQCRXT4-v*B
z=xSf$4=!A}%HzXJ^G(N&#wwHrSqm)N7k*jg`q!J!KYg7&CB-u7JO8=5DU4s|ihlaf
z@QuH-{)MgnKNe5-Z|pFpgM2_>y#9ZNgo^3^nlJrlc;GUn9*NEPl@+WNu6|VOPz|Q3
z%Rk%>y83pX^6kveufl~DbtY+homt!XB7^w__&hgytnQIFt=Rp~TlV_j^7y|j;jS_7
zyZ1lYc(?v~>3@a`t|cLO1}N<`Qs#yBN!;?3ZdhgH|ImSF(UMu5YnK}O3m;W@wQh<0
zi7O=xfv&a>rztG+xXaA5?y0f~!zr!oYVYptnz<~S4J4IJbR7O@sRgYGZDf%VsFq^D
zw>{BfzI6Qwx8r{=|2bM~zwAH53BAkzcBubr(tgH|T|$HDDv7O6k*)K8GT#5saDB`C
zU!3b_+Hn78kT>7&`fux(dz=3=C|Lhz`1Nh_e+Ea>&+3i!kDq;ivHx0wbrjCl`|^`K
z8<WrUURG8vnmEycXVDzxqSixZ7ym>&n(e?U@~ro&vynmYg^353UBEb^7QKBJ^Piz{
z|C0@l{~2QCe}+4va0A#cl?1vn&}rwWUDn6)Khvz&|9ZFo7stA;o%hA-PbTmB&#><G
ze})BJFEDqbuHNM~@7J=mxv3H-ryiQ0cyDER>I3P$>fXC{UNMzk@pz(ej>gTC>MLJ5
z81J~gWw&nd;qO(p3*xs#x-E5(X=0Z@wZoqGhtu`Qi|f|ETYv9rLXz|o76x{e>yAtR
zxNn*rp<TG__AOUKCGVw{N!IConW<N|NZSg>Z`pb!v3_e_ML_zb!^{1n7>-{3!=EcQ
zm2L6j`&*v`omklxVB~qs{c-dz`{pfYxuY+i`E{&R=e<Dg!@aHPkw3cqHh+*^yZx5n
zdZoE42E~(@oBym%di0;+u=%g2{T+VWCYpG$PdNQq__powr)~!vSN`zg7y2<jbXxYY
z?Vp$uT3KY8uYX!IqlR_)y*Q6=vrgH~<w}cg2;G<WAi42q-r+~JqNTEB&MRDmN*o2B
zKRYyUPPp}hx#fP_FXxxk`sHsvUi>}Pmd%aBZc0Pi@&~6wF8QrqlvSU*y?0;1DI;5t
zwflu%Ouu#J7H|KpiGNO<7W>rnhJj}uziCvhzsrZ~t?8F?OhvD#?@01q?s|@`){1Ly
z<)go9_d_l`?|PZmr!mX2i^->wA#H|ujsAknD7B5Hvh!Bt<xP^^>~tWtO6Fr+@T&R~
zQJ??qw*S|XeMM*8ALFI3ukYIQdZWkn?js2)`hJgJzKzRNXnG%C`&ldcbV0G^3z6+n
zR{aZK++*3i^<s^csq}|c*Eg+FOuR9N@yD#^ruAIw|EQl{Ha%nQy8Bf|JrCseug%Yn
zv$|CBWy{758@F9~`^d5BuRtH05x;4fZMNq}*?;YoG9Qnxa94}sD`J#TUVh>7E8U5`
z{~5TxyJz1ldGYnQRYUA+LB|8%WnV3u{Xjdj<o4>d?t$6P*~adQ99#{C=c=v;mYL{k
zAN?+M_1?P9f{Aa<judf9ocQ`^_VSph*ZYL7KHo9>&9>{tb}7o%;tWi#9y0kId*t(%
zy%fy4mHKh_rX)rk24$P{&+FXwG9Q_>ZsPJER<g`~d!ui1vM%v!NLTsK@OAb17YvKK
zG#ij6X&8_Exc$$`%>Lc{XWB3Sn*VbwoB#Eo{TIP_g9@zT4q_`7k#@-5(l!4XPVoI_
z__ue-e})UbH76g$|M<E0m;TrF%f4bSx3)0ex`=i{C(Ok!K0njr<8!ULw{o#;@R2Ir
zt$$A2aC3VExNo;OE}MN);E}8PmGGsn;|w-zyKd5bTf22<pT?QumRF3&*V;9y?*6zs
z^2eHtixIbbZB+$~-ah$u;_daVI<xFLm;P~TIXB@r%ZHy6l^<xGo_}iLtow(=XFa%n
zZcB$bXWX40q5J1Q91Ym<(XV}V#m1#q-m=Up=y4U^VXw*lUhw@aXRF=6_a>ct^!n{B
zAM;<1!5g9(&av;X%|4zl#lCmu+Qk_U(p3d_t@yG&@KXL3fBw_9mwqg`6S?8<-Ff9X
zzj&U15q&Ohbo*7*@6$*3Zu$IV@<ow(`==_rsFH73^L9%v!>jV!q3cybd=GE5ke{!%
z^KJg27yGRH{dtnM?Wuk7%8JRT@W-`pn;-Y8`*vMA?!S54m9>u@Pk&h9?vqlwI&*EN
zo${`WdXp+1T~V5@c7INT<B6A{yIE4^x6d-%H`y<?@a&J+)1Y+ubyZY;!`i1C_UyQ_
zRcwpdW+M-$rX8}cSM81E_x)IN>+HRC6YsdoUCwd0wd#-4vNd~kJ@&)4tu~*fv!lD)
z&$Of`oH%GQ?Lgr_g%71$+q&h}FO8pl+DySP!N~Z2*AwjvrGLXcF55`TH7>ue`=-b3
zrSJmDllFhJKEHAh33N5U7+-q+hxb2&Y7_sj7ylVf?cze=HrP!Db;{}0xn}$vj?%UW
zJ@*5t<q^H?mhQiWyqgUJl@lgdg`D#eEUH#qa(k0jVC?ok3fKNKeEzNe_XB9BxYYfB
z^(y}v_OVvpzb)ba><{1kRiSSRO~OHYTuk@>`RDncp@z%;A=~!<4D(Ogzi)K^c1-x^
ze#Y9M^~s%!cIW?aT>GEl`6vJ14?u@%AT(?<n1A*M`&F%P7PGXs{by)_TO<FWIr=}t
z`6u(gA6Wc5ne)&62Es<|C)TKcE5kP)xd`*(PjoNJ-<F7f_J{SW*WNz0uDAP<JOv38
zu!es(4*h2^e;^;S`o^P-b+`WsWY_=vGx^^Sk@`ct`TrU0C)M9)>VMm#|GAF&*V4G-
z!mefak-UVYJSVY!_6Kor5Nut3|NPSXM=Nted<E|;ZF4Q1k4uaF><|2_Lf;haTDQD#
zP5j=sH=T_iTZc3EXKl@WWb1U@WwuA};@>Ja?{L-J`6cxNIbIN6(5%mX2sH;!MEVmO
z`c#Za>_)jJRy4czyG;}EjPf@S^5#hvUCVQvKdNRz)h}TiTMxZV<uf+9{}~je<-hxX
z7X9*X_J0P2-TuEG%>Tu)eg-a?b&u)PV%1*pA$RH4JJXvU-P3exy}g$?dHveb`8R&$
z|7Ct#|Nhc{hSo}pHMb<H*3IsHQ+nypsU?+@ZPs;t^?y@q{;%=d{_ijA5A`Flcm8MC
zzVbhVw#l!a-Sfg-tCoMWoBf}`;dlJ+FZ&<5qjJ95KTZ59S&q-R{<p8{PaS?`^sX{i
z^c9M$<dK~yfBVXRhN<mWQty1)rG3?Y<Cput*l+!3xW5#1)C4LI>^lAxr*}Rr313y8
z{Br*n{;mHR_Iv*mLFMEJ{|nP6PMJM%%H#v57oU1{w)M-c+%M+Sp4QyxdwR#nIBxpi
z{$>9eF4$fF_gDW%AS!3C|Idh8x7hO*SGPvbP5b*^@0;WdhB7^N(X{JF%D>H0`Q_tg
zzv+x;L9p)6>$Rv_Q4HPd|5E~Fj`pIx!N28w|1&J8UH|u2{D;M;oT~YsHVlb)`_F(J
zar1B7%K!5t>OaH%h5tnAK@kTI>eur>nE#o^VgGvZe+I4BUsqg`O?qW<F;L8jX~hB2
z6BW|`yjkSGUi{B6wYuc_c~R9X4<~Kj_+_q0V3@)HRf1*vWdAcv_`3f0*Q3bY@a%{8
zfAUBDXV@Q+RNrjxmg)W2efdRK(GRx%6&h=`S8F`p)Z}qxdQJMAE9zdm1>b-A&%o1j
zJ>)~^Vr|vBT@vnsZ0oB<UaY)#DW$>Z^d~vvS;Y}C>!<(TGyV4hsq?R{=I4dQZ*4Ey
zDw-E-_DbUWo*KR#mPbQYtn@0myKIfcIiqVj8f$0UYH8*iJ~`j8QTv?M+}1_jH3>6h
zc%I9Ihini{dNBXijgv_V@1F!M+UB}$)_1oXI!%5v&x_nJIpG|$W@fVdUCX1He`kij
z+47&^0r%^`Z7WVJYBDHvtvvG*qk)30`~0Y;_&<Yu*!*|qpG9wn3qPL!<JXq|42An=
ztoPUW&rlt2JYD>cTE+X`{ZEvW|GoT|v~T|9`X}O<{~2P0|1&s>erCrd^Ub}F)H6?*
zrs3-Q@0)$C{s}wiR)I_PFVyy>e9-?>bMLSCuM6y(D#ZWhe*eAlKf~tm^qBO<`fK;=
zcg_E8T9!Py>&Km|@#3m4swKj+G`m)v{~ogaLfQn-@ipJqo<7+(GoPjG#exk_CWhvz
z_XkB>ve;OBr;l;P>ZTUkbM>_)+n)%oSAQSvY}R$XR&n2f9sPgRZT?&poj27uhTqQq
zF3<C4OFrFtm1TA(TiEaH{baMQ-L;9Td-@(4^1Wf6)pm}#xMbqDbN-ihK5bhmweRFw
z-_VF1pO-IW<9MY3x>ZN}$K#z_rdj;^qqgu;d9PKl)VqS?8Fo9?eV=-3`{rktuJ1hm
zb$#)yV~Q-LEP>5E53W0Qrf-=2_M+AVzFX@n*uSZ;RCR?K-HX_{GGppDnZoeQ=(Rmr
z8jJQUUW0F*0$J|U+v<PL)8~J^a9#F6{!bl&`s?ohLT|jj|4C-ge};8!JKi(bpG;`}
zx6A(1`u&IhGjQ3pHwHgW2MvGZIMC^UxSy+!-v1QwZvX3D{~2B=?T+XB&v2sZ`rluV
z_Qd{YQ0;sDZ}<9tP1;*Oy#Ld_w*LMaL;HjC7n>~oxcKs`KUUc>QI!`xB>yw)4ADHP
ze5|PNRBfO3BMrYfDSwyb^=B<fxxVX7evCrIn+NGP9`w!KvAS^C?Y6(Slk<Na`g#6g
z%xY`nyfl?xll_g)IIx9V{AbA88<BRQJAcZP3QN<STT@OK<moPXAv5vIS^b&cdZSNA
za;?d3V6QsF_pqBS!N2AbTf*&T*?T+JOv`+COwZc3mh0ED^})+Rl%#u(1X~`9w%+@C
z-j;?tu`0<spUq02x$D&Pf2=-XT188mU-71P9Xqt=p<b-=kLg}5E0?X`dGTyvrs$ch
z%SKyjSeg5ecV#rFZP>K_PO1Ie;w958RVKg6d=$xMUva1U+I!QUxleiLUYvhbD#|$Q
zaj@9;AeQa@6&ClZE`PZ>D<f~ogg<BhiFSFat30{1deh%)$}h4do&|rKbLn=ht#?Jl
zr^q~$uZz00&s5D#L?0Ey#*gvutAG6J^50|sl6Geymm@Eq%D>NkR`lpUL(6&DkK6yW
z<ka7_KNEiWSN@--xAw1_>R%}BOTZz1;^XO%Kv#_WgYX^t7KFWQxPH0n{GW#R{~2E2
z^8d?n{meYp{|x=J?=Sk#5XJD#pXoot+I{~SO6|X8z4^(KwBtOo!0X4x={Yg8KH1Ju
z|1fdG=bNv@KCbyRV~N-L%EDYJm9pe#ahn*MBY!N~JkPGUJ7(4U^?}FvMZRC&xZu`o
zY1S5#rhv+tD`nF^Ia}?S<nh94UDZ`9EjhEzlV(p_vD8T{>-IFgck>>FTi-sduITwP
ztYmk0wd<tF=<hQQt<$#ZnpLs#(E4pv8>UL^C@L|_{W>MHW15Of-Wqlj*Ye!_+=7Xx
zPaA3;T$81}Fm3AEwX?6s2b;IFJ4NcH&q#b)Cf9QH_UsiYmln0g8|fJtWSez8Z_`>5
zGWq&!=g1)6^$Lr+v}asFE(Y+ihhP!GUw@(uTtdVnl@OaQ3NF2-dr>9h#&*6PC;q&a
z@&Aze;`H*5i?6I$bN|ih^}o1d|1;dz{LgUY;@_F2{}~*!_n)-?G3!TtI{$OqbLlrO
zE!teaS#SRf-SVI6{~3hczolOP%ewAAgUWw~N14?Y_4dzXF1^VNjNE^7di)o$-G3(k
zXXsq~+vxSbrt9^S>OY!5l$_tV)HUrtLt1S8g{b$R{Qs%-L#zh%p^sdA^YG7qh8e})
z#3-^mHdS;svP<oeUAMXY^S{KidW6-S`7IR^mXGY2jDKf#{byLP_5PFje|+W6$)Eq(
zZnB#QMaBetZ$Tx04L;X|9{KV7pWRyfJNeJFmq0>qFF5py>h!lyy7%acXQ}S7&<!bE
zhL<mJeJNVum=qke@41=g{<JSP(~sT%B)AcL(Sl*T|I7WKxVQXgSjPz(+yB{&TkhB6
zle(~>MIvcN@5uiQ6J-B0{H^x<&#++D&&db&fBaniCH{51?`!){%=`Z{tk1OnA{=j2
zA^y)Uzpndl2J=lD?tj~U|Gn~`;pXbi>x>WVf1SVouK(XzyRCklx&8N@`{tH^;Dy7N
zm*6Wx2+0k>B8KQz$csf=b#fD<-FNSZeNYnhqsLAz^!ceR8{dYkSg`KQF71H5t#@}%
z=w|8uUims8^5xqinT;v8j4WP9on|#xmCE({+Yx$o_N1*{AtCd8W`BE=xBu>H^Ig?Z
zZ*E0?=b2{}rp+IC^sKI6-Wt8B-!%i>x8~h?xuLexWtGi!mgaj~4n0)w%ynOAnyr;-
zmKYc%mAlFI_fexfE`GMPi*EXsUAv#G<>vM<I`H)KAT2-7*xS!sK5FS*Gt^kSLPsHf
ztF}h&{E~Th!gLcm4xMZAi}lsK8Y;crf#}LZyYu7uKebWyJMNze&qZn_?6q0^_ifF&
znD`GfAMZ2!&oH64|M&8rUBBvm|1(U8UH-Sg|1V4U897`sH{~p|NL(Sn6@{w_@GBoz
zo^X&m`5^x5_WLeKNq=AGpIak7<-;pi)aLj1Kkd1o=6C$T>5%64>!hQPcV3=%VTY%6
z+Qt{_yACJGa&FIFzVOY}<!?oMotGWX&ifSWW&WRmBRDwX)$(MWcU__dbHX%LzN%b|
z6+Hf1Q{BxhcK5HDd+e>Y+`YN@@5bV1?|$0az0cZQzv|Db4ORw;p9?Q-%vcfGr#vYj
zzwmF^e50}(8&d`5y%pUdc~Wg#`u;86TX|%*)G+ajo)7fr@0-d|bTI4gJ5hsIOV-`I
zxTbK~3iYtqb?r9aSD9Pwd1=zo$i~N<JnPh=Evskx9-lVr>9zk1l>&#Byw~-eYZQC!
zy3gBbc}4zF#};)xUEfif$g%aqKea6{i#+!|bQK9V+(1krjNGvX-54VCpW*n``M-Pr
z9IdK!|9$D_oE!Te8XvDAc^q^7CYvOBOugGq22}u%T<{oNGrR!xu}eSB-2VGYe>&$M
z70@=b<sQU@1r9ZgPj7wy)2?s-df__j1NT2gDEwzw*Z*hr#_Ri^SXTaLh~+K3$M&CL
z(n0pWCG}6^>)Zc{)bcYeKX!h}i$z^G8U|`iDN5&L{_5MO)!ug9w02bgBzo+}e#8oo
z_$B`tPWZ{MP43vfY3gsO!0;6lU&w!$7*T)u_}?G-pc8(j`F~aXXV}mCPt>XF$2~9m
z(?9+*9I%`G>e9cY{|tXr%;dj({LfG)4L-HE?eM>O_U{kZw+1ZggR10T?frBA`aSzW
zm-V*)wW+_~1{#rlsJP)j!}9*$AMQU4(O5jG|I<JF56rJF{mXy-UjIMCA?L*Z4F2-p
zAMQWY|HC84{^jF;f6RY`a%ntS{AYjvABJCB_Rs$H`}BW?4-;-+cc`Vte+K=>{}~wU
z*6Kg|x9`)Sb?MgYhp+1_Y}xRj=d1n0$k3IoOY<|A{`t@FK+wudUZe}B7t{X<o?id+
zHUG2vL;XM8_4YqsV?Xmh1K07_Gvxj=^#5V9J^%a<*L@t}gQR&nk~!-VszK3k=U@GS
z)i|RoZ2v=<UH=*8ZTOcCj`E_{|EdoEod03z*E9U}$Nw=@&p-b|v>qvb-I3CW*7ZNi
z)8jv1lRxvHp;i7vUu^yPh<f9HBL8+az6T{M{=5Gf>>q^wXJ~<^Bzc4%R>vcB)E}}e
z{m)>V_~-nOKuT<`EoQ#|@jpX@-1E=>SmP0i4Rq`>FFd_&{m;Ph^!lH#?Vs%*dLpsC
zp~21Y-G0tmlpuRr6J5{x=i6DfKS~qI7smTXy-E4-_e`Kon@7Q_hzIlTw-($}`Onbb
zVSm^D+}@md>*UrcTE*J@XLum<W5@cWJE_H|_mE!FuCocEOG%r3$f`w*zbyT;UBQeS
zt-I1MgoA=h6dqhpYD}*^|Hu3{yKY~&QLlODbfZ81)$ER^x-zH#XLzUt&-y4yqgC|q
zN%x=i@;}(auP*&_|N6c7{|qf%umS}s8LScsgeYYEnptoD^*hK+L~(*@rbtoXzxd<-
z7;2;9&;G4_`k&#4#~NsH0&+4Uh8McZOsSs^N(u5|(?8e8SJwX%f|q<E^++Ysia>W&
z`KN#CAFv~wi7mBab0#jYV{s;|)I)P-SWeE8{WFeAYfad_>Bz7Ck1OYIXo*>}C9eJh
zs}`TuI~UX}uMID|k+M8;dGYMmGxq(Ua`M=Pe?{B>sDvSx=J3Fe{Ljz=%jb(R3Jmsl
z_4W_KP$LUof#ArqK`?v4<<x$Z3J2lnY$Uso3c3221K&a6zw^(3hQ{!JLU0Wt^@m*H
zHOE@yA_6JTx?eZ2{m&r(gAwFxP*DT-CQ`kJ!`Wy>4N7#vgAP|AhpVUwLQY+4|1%&;
zwovW2R<Sk<CQjY{T-tN>@{WU7rt6s2?f<~GSkl@poptq@=~7$IY<!+=sjSSlQy)GW
zzHW6OsN-_{_p_bxaKZJv7ff0%U1nFeYRgT~99PtY0a+rr-}67iggNr>!xbOB|EW_R
z|MhPD7s1%N$^RJ?E$r{DDU?4neX-fXKOVmpZ(Kbwb9(TNy^k~umX*u=3blIm<@`_K
zb^jS2gnqgaWw=^x>CMeAK87y!(mcM)(rc20>W(R{T;jzSxL&7c>)IVU<ebQ}R)Jv>
z`|?X`)`VS)Q|L_G=KA@vrTn4V)vGLvcKOBb7oHW_ynVOae}={*-ol5q-L<BcS}1&5
z7M?L}+pT$f3(~gxM1SmmKl#q(_c=Q6bW%SjY-<ZSKP@R|{_&;i(@rke*lWdfCMeU(
zYL2Vu%~$$~%Wg(*+|YY3^jKM+#nRo=PWyEonQJG>bAYLU&Ei*3WmS~rTKm8?)n!}c
zc4rlqgcr^7of36XiN)}wU~Wqa^V`rh4=;bsb!@u1rtsF2+z0HI@4M=6ZrbJN^``!o
z;npj$6OUi*)LECb-MdpUwe#O=xwhThv!yN9sv6yna#yK2{&as#)d$U8tw)L~>hAxQ
zH8c~^PdQ)c`}CCNXYXY#^%*-&Ps=SAb}c`>EGlwIz}mH&7xUk<S=;u<;@RrRm)iv5
z;*%tvwuVMJK9?<G<=39IeNWqFi!Fbz6^4Yq4SL6YdcwDFk5~eG7w-*RZ&esL+4Ig9
zMRkp3S7(Hmi~pMPpP?k2GxkpWtKh#E<M@iV*wrsQ+ILI&#i9j!_H4WSR(1ED9ewxi
zDeinawP{s<QT8Fv?J1Q{zJwbt`{VLq+WNX}W>HV~OxG)0WRP;FmN9|R-s|OSuPs^^
zYqlmm`g3)^q1wEIt|DR1;JHFAvEAFew`|FLEUvfon$qEFjaLq@``k@U&1T(U>e5&O
zS~|;OUjOmA{omsJr*WQ-{D1st|9kn*>uc}+Gdw<5|LfWPQ+p;Kng8)4|L^5LuV4Q#
z|KpOF{|tARf3n$)dprx@Sl3^kPu_Hx4fyKMTe|Z<L&JZDsQYX95$yNz+B5z$?Abr@
z%g*}WyYyl+r<)bHo19|u{O*@EFMH>nF3ag|eXcxfD%tm+cG7LmE_%8D=7-i#-^Gsy
zAOCUf^6ABAGfv&LD>G<IRF$tXuljpjns@HeBmOFe#*1nnS%-+;dD$<zciBRovKL}^
zXV%~Ub+uO8e_J*C{@*6NCOwg1&MkrS=4#fbYZk3IeWm^uZ?UDjU%RYp*`=4;{OeN<
zx<t#8RC_tJ>VgCJ?5fW8T&~)Zoc(0m#pdHny-&}&=UpwfS>#*yt>@CNWiS8EH(P5r
zt>i^h>$wkBv%2nGJ9XG&$E@s-*(awqoRGBn=>Dzxu5WZ~=?ne0qT99i&F?&Rd7_l3
zs6bwt1w*;%jhENTri)F=^3LR}<UciOj@Gv?%g?gT-FkNErRamttvl`tZepwMFYH?J
zbloYxsJf<UNs|i$uYa|8-JQ8M_0#j7#g8paMay>e7`&2GY&#bbp0(~<;nnt~5!3xQ
zPB@)!x^+z<V@qm*wEgpYvrkLsPm9uR*{m|@S>KumE7qM@+ctCI;bPTUeutm3XPo@L
z&M7@oy*%{Rnj@`W*4em8-?(XBc5aQG`fc;nlwCb*>-p+WIQjp2Qa@u&*Yyw2|Fo$5
zXISU{v+L8x`|mD)F*#Dt<?in)5`JQyMqotL`<^}44_6B5#)mfD3YTt4*ZJ}F)`AJP
zub%a;&x}*tVjkuD$Go7q&2V1GFTpGFYjZw)oob);;Ai#C+wVoMzR9@gD%x=fJRZf#
zAmqoeV(q%M?{lwJM?Rh<rlK=(L#l;^h4LM#va%gQ7hOdfKqH~r+(+iWTKjMLudug?
zKPLYT-T(Xeukf<yf1ibO%s(bo`Yry?peWt{{cO=bL{B$1Y$+De`1Ke5$<*>|ZT~30
z5?tZTY^BQ<PX1XabFb?!)jv^Z|9$lY@`4ITub2P(Y8m$rOJAH_Q7>e-;!SqY?a1s}
zrC1vW3D5cS%YA+)ZF;eO!t$VyyN}MVT<K>jx}?|pZ0fqVxe1?~eo8zqEABtEeD;wd
z?-kK^yGw%wZf`Fu=RdVkV{P5`u<tvkU)7Ad_HX8%dn%gs-&R+L^cNgi<yU^CJnHG5
z4W2SmC4naMYx_;RUCn#fTwSNyylBnajVo7Xvj=tEQ|9PD`Ph2%{7buJy@R)H+OpB;
z*3#prI8Odv!u4pSr}|2@brQEG2Gz#DaQP50T`Ni@Z++*pe;2BkH5h1a3@&5pTAaMv
zH?iis;0nc)ciI<9uKA^{-nHgd?whO)TP9`deldG*db;xMQ;(7tiz@4W3(pdd4_>M}
z=UTU-<0rd!?Hkv4Y0Wd*oO@?#^P;J1O1K}nOcRN|E%8ppf7g>M1{JLfC%rwos(uNB
z!P<uXOOr18KF;2IV;Wc2zJ^N6MP1qxUU9v9^q(PU{=2S++Pk~|de!gJK60+C{rndT
z!`m*kmE}j@e>v-a@R#Y~O)t0Knh?E!kD1{FgZhiNaf>7DbU)0S|5#T3x809DsWP5C
zA7yO(tOWmwK7YESBk^Ov>fqG+sh7il=2cGiJ$B;hKFuH5&g~D+e)*XBZQ7=rTDDsz
z99eOqe8mCfjqDQt8MuyhAGxP@W$S+i?u;_2C1v%uta!|WPwVbF6?*RPC5B3sRvF>G
z?8txOuXb(RzV)Mi&w5i|JvJU)7A9pitNh#}^Pj)dwm&K*`}F;>UMXFpSGk4@FaBpJ
zEah>WrnFX1WaWdN!hYwtXJX4_FMku7wc^s8b(b?U)-IWPZ`FJMJ?A{{ecQ<W$LabX
z-P}ie^&@7<dIw$-Q`@@hQ%jPgVu1OokH@}R{Ob9>VvqIXvq}1b83i#Ft)(|i%>-ke
zOdd0CE<1L2?Up=Ai$DBpOTNW(zm4|3u6XUM=2q*!>G`+vZ8l2AF4x>x$;{4vxxuh&
z<Dv@vm<?Cfs9pJ!SXrC7e4&SKQu~Q;i36X#Hs0^y+c!Vv%*M$4&9gi$tD^qBihIB7
z?fcmG^1B_L&b`?BP&fQooW@7CWzS>U?)zNY>fX8Nw~^R9gZr;eCpR#>zW&UvY3JJF
z<NuT{B~K1)K4w<cdZau$-hb2O@3+=eKB#=^{`vggs`j=&l46Z~JGt*|J(YDm$nQsg
zVvn6;KvD7a^`fWD4@tkQF}f!+`RJto3@OJor`;3X<Ph4O{yg_)-rD0bbG{w5k~p*X
zk;?N^3w8#@>|R}cKXv#0i{GjuWYTlFlrD!)|JJB0wr=C(l+AN)Yo6Wr!N)$xzJINE
z;<}34$HJuccWlYcyYI7M(=3m-Ilik-K4)S0^QJPU$*TN~)%C#o2h}(C<R(AL<}J8+
zR)jOZSp24Q=f!9hCjZmBmlxS)o7K0OfBW>x>=%F6I+MNXK@*qk+$45djl1Bs#Ds9=
zKI`PQUdIyEB|dxEoHC3$_A>JKsXOb#)~2tI4Lv-~<M7@e?T>Vm{Xh83(zy^B^ExOd
z`p)VjvdpWM*d$Lfq-<mSX1VlZe1}c=!|1izzE?|scj)x+e12_qE6eklSL}vXe=erC
zFVA1!9`T=nPbYcyq5X-ws;leyt#`R4-%HmEnG~(euI2CdwfKmg_|;O~C|!S#kE+LJ
zYaL0PbkF18#eAiC|Dpw$tABINcx=G6M498L`@Ox7*B{l3-WORrZI|i&TzQ=}<`4gE
zw%hvfvhBI8sjE*++SENe<4k(K1YZm9+8<T5*{jzjAFSi+J|tG&{cYRs-FxSpFSehi
z6qbB5pkVRj<NF`R+E&cC?iJCwOYh^pS<xRh{h7YVr1$Mg-ur(*ho;Be{uw^0V)8#b
zMeyXH)z{toe|(<*Z|C+;>s3Cg|EY2Q_wrxqmpiI|GNsP{y;ZMF&S=JkewD4*)^y+z
zW7SRn=TQEi;p<!Ve@)rXKr1MYpM8I^er>HeuI&;F_(MKikDmX7|LfXc2jHxqF{l4N
zbHAMWpW)q$JCl#(JUjDu2LEQ6{`v0<`~5UK*Kd05zvAy4p_ALw{>`<1d~4&1HE~9p
zukJVYUlkb3yIWsCI=g%Oq#K^Hr^PgPFV$V&;li!Mo7JE3WTIHzi!Z;zWJRu-ZQ45T
zP=%HL!fm&{WS8gahzTB>ke=0i^2G6&pqJYheLnKC=#}z4&tt`X+I)Ret{E+!RCHCn
z&_>o&Z1LVTrMKGli}o!nP~7ac<M+O@s=wLtzl)rgdZ&D3Gcrt8d-vq|(v->bwgj*4
zU2XPt(;KY{tpKlAZ?0W7J-%^!cILw9?y`Suk40i%Yii$~U2U--BD$#Y-lmKdzwYLR
za=+;QwN_oPIxFw)kDjxIyl2c-iykVid0y~k`PoMnD*Wo-a-1GsXFYlT%euO*`q;JW
zc2EAQWtF*cW$~J=$0mz5HfT8UeYjiCb$U_Pv8A3N<rDW>1uk1FfV0{2;CTJ+^B3m0
ze^^rTCD65|^U*x-57+%SUAGB-^fx}VNqeuWDO+pJ7yhRTIt&b7UiL4#BD|zd;>YDm
zzK+g(-fR6T-vwP5TqlNhSXc3d`v2khb=`Y)jn$5P!I<oQQXg4<$-ce+r&h~FwJ-Sf
zmWk)<{`^Y0xbz2~J%8lus<qD++O1uCFY3gkdp<j?%l=-CnZw6kcUWTm@`^h)%dPfZ
zdQ})0u=(PP{fAStSKCi)*S>XJG5wLe{M|p6a>i1BS40OLxj9?q=ppS1-rJLJGw`v;
z%$gthV{-M%*HyKx%Qo$k+fr+#ee+%2vAj8Zx+L7h*&8H^kJnXkt$r|{G3(ZUhQqt9
zBW+yElb6PNye(foJ5^=sgumA<v^W9^Pn@@Xz5U8Rd8=37s<yY4mR-!xeJgfizG9lE
zVgOUql0G+fyTg}uX<t44GD($l#a+u=7dQ1?^S_(<_1dKklV(TB#NI!$U!cG3(knr~
z?p0A)-5J-p^O)`&3z={*qHs<X-`ri{u4~ReI?rEG8Xx*`-puU6;xg`-(^<wm6_Oqk
zH1FL!A9tYSZ0waf>9eaU%xt+n9|@ms+`X|}_35FRoC<<lL$;sV^W|*DkJAS)Rm`hZ
z`*u0ls&3)ZGpXCB98!%ddm^fm{;vPpPC0#sS8fJ@fzKxAPx}>F^j>!MeqXC~twpDf
z&%60!e%!S!`=v7c*GZpvTXMOO^~<e_CO03ZSIG(qMw1rIb1Tx^rQT7xpQ}ds;;c*W
zv}dmkdlH@bQbjTBidfE*{{1J;+wJ`ryKe4>chR-ckylD(%c6glc8hJ<rFJ*jQi1n$
za|43}vwUyxwjU91A6$NT)<162D-G4ilAnvV3z{zGJ>KW{W}2=4<!x4~eW!0dxjcOq
z@4DE(x2>yJuUntJ`exQeo6U;8@ki_UKen|m%l>u7>gu#jXN7ibi-_Z7v&wyCbo%=J
z<yCP4*K7(OW%s)M$rkwA?Xz#1VV3UX(sOJssT)7_?>-+t<MV-a&0ZgPKYSFOxAl>~
zpzqo{5t`8^X0N&2{67kCp8Wnzwm{qc&^=y%-ps1C{kNaaE#+RR$J4P)t-q?VRbleP
zK7P@Y;o>d5mPIR0EuFNhuRpw6ZR+0J+BYl9HQmBL*_?L$(Y*HT%4ogikFH$)%3QeZ
zxzCl7gCdGXb)gd-6AJsH<>M^BM}78Q-+F7>%3QXYt1Gruzm=2Rb5LemS02YnYo8;N
z?|l9hzLY`KL59IVd$!r?)>RL6Y<JyNO?~<;`NzGxH$$#Wdvr~{)BbAoe}+BvOY_1I
z?4qap-y2$exPP?ssa5}<mOT6Wrl)=&R#cZpVG*p`@BE|ii_NV2N9vY?mT6|C4fZjp
zMP1sD`PYI+9<Z%U28khByvzIrBG+UsRXf2@d-0#?lV@A9LIa<KZ#}1zYU!&NTAb8%
z5;RD9{`cg1=nW)YP<Hn5Uwz6yC69is+J7oLFw*|R#K`{)Mfty<)c49GvjdORD~oSf
zdE?rmP4y3zHvfBaAAIA7Jrdh)amPP5`S8J~=ECm(4E>h>8D7t>*ZSlyRDb-4{k!?k
z<NEFPW~%O)xXO2$6I)2ReAMTDm{ff5p)XlMYp%Lxw`LW!NyVJHwV0e%#6FuMy3C0L
zACAqde_(%Y`)g)6>uIt6_nGp()BiK<2A!4M@D^u6LLLY>vHF+qn=o@<orPUfv_JKH
zp7>AnY){K_r@*e=8>gl`^<7f>CCvYiN?84!+0grMV9a>quQKkRj8FdB8~@aF#hUp)
zJl6bY*lGUvll`%NB=-C(h4Lp3-w1g_D6R1)25(&Jn)09FkZbzillvhTsKA-E?t*`M
zY+L<~Z55sTpMg_!{&zL_H8}_l$VCR<M6IQ+;Bd4%Nv?^gQ~A#jn*Z<Pzp3VkFo>+$
zw^Z~$!xSYxrgZBz?e&bUYk6E!HEv1=EDj9uU3~Vd{!flO_4~4?BJBrxwN)Nd@NZoE
zhsQ4}w(jrv>kpc>zj8*)@;}2B&wnrfg{s1aFB&vauTk1DzrFqwU;g*`{~4S_Z(|<A
zgnW(CyVXcn5;{mtfvzL2xr#g-o;PFNtsW<{(3%esDdDQfGHt#885ASzw)Uys>X|=r
zEo;@rxOD#~S@NGh=Kp7?>;G{PiGB5z_M?9r^-jODTYqES`}Z0@rP}}eiT=-UfAK$)
zN(9@qs>=?fia%-h{ymG4^rM=k@$1NL{T;Sl@BV%2pSYs_{L%e?f7w5FM=--zEPgHY
zX5WKv-n)OFvU6MbpTS=2Kg0JI{~3DaP<U~R`qy}G`rUj}51*r^hR%5yvhfSEiH^&?
zPW>IWQ^Lc)PqFEB{#U2_pW*$*{|r(noIMp5%IrH#0>5^a#TQho-u?SjUOC`D!~E|0
z-(TvF^&_#@2e|QZfSvjfzq(b{O`#Hv*O?~G-#$O}JNu^o^6yh>a5`YGtNt4QO~2dU
zOedgBK4Ln^wAlOi0zaAZ|NOE2Kf`|aKN(2utgj-qNwL%K$ek>&uXI0g#s2xD`~UvR
ze_V`UuD-HcI_>ZB{B;%!!)M>r-w}7t_Wr#9Br~v?XP-h$3{N65hW(IZIN|k4D}Ro8
zF2!3OCr$jDy=Gndk^3J%v;S`Yd41v3Yeh$vT>9N7b7)pC+q?5RI9Ao;T5Z4L=cOS~
zfP-2Bc=6(`OzGk)>rc2n{`)y*edB)k@g1Sq1aAd`#!e@${^S0tR^3%3Y*`|GPRn9`
zwjSjKk2fayKe-P6`&#%<??1z&<E8%@Vy}btRNct9|4Fp@@2kV2e<Xhy=h%NtepR+5
z(6vVM>qA!&@Hxl~t7pZ|+<PUdU0$!~zSr_t{`PmeCci7+J)3HCCo%c%XTjJCt-`sf
z=e|VExv{HHWWL^er*FH{a|@U4nYN<w?QPyI$*&{Ls<>h@Pgm;RI@I!cN6A`m-l}8e
z(<F5tZ}Vi-Il3eHedxPoTe}ZmigGU&-ch`TO;XwP_0=}1)umgq?#-GPYI%Lh-f1iE
ztSo<cQ)$|r=rgLGeWJVSOxG{X`0(=Abn#A=Q}azKo`htty5*R;T;5XRc*6sksgFfX
zoDS(Ger3C&c%=K#JJFDg#D%67QU7*l_(olQ$}YJ*w<<4vUF+Y4rgsnS5Y#$zpvhuI
z(uWu2Q_tSgT~kxJd~fvc((T{E7R%3<d#ReVxLij|*67)_&n~rQnb$atC!9TC+P~?r
z<%*lJtKVPWxM^iaK;AF4X?LwBoL*!*=h}4xzsU~o7;;LUt<9Phdu;LAHP(+M`c<bE
zHu}$ccj?%vTL<-o!i6-`WZ5lEQ@3WDUiYp%l{`sz_Q4bEri&K0c^>)|n^|n19lO;!
zcFEfXf3^MFPH`XYdYb;2xAD}hrzS^vS4C?xI8G?2+j=?jZ?kXoo>zjFvsHMyo|Rnp
z*s7gg#<f~OW!|0ZcKXwd7Ok1O=lzxYyk{PV?#<ipyyE(oiskL+w)KWZ>`V>p=~}%u
z;^(znC1D@6PO7mp%kI8(_jTdz*RM^2I(2U{?Ylhj-kPkaqqE{7&dRJ*QkFf)SHQ9E
z*8H#)%ic{}X}{-b`}TIV>37$EGQNG7%dt~`;<~1HhL2XBUSwnCxVO?HHEEem$#dE4
zrPBj@cFwc)x|yiF>_XHE*W%ojFFxJcd#BJVW>R!R<+;L$D~A$euas}zw*P~7o4W9x
zZM*i(|G@M3;inBZckcMl(BH&<_vOS{(;gr8<`i7}CRbB|#isD}wJpo{Zt698G3lMT
zzz!Z~J_d=iX|XYp-j7;mg}ZNmtb3(o#me?W2cu4hO?!H1pHlAYP5Ft>4;yYMY5OZz
z60<33>5cG%>o|TkKlsmZKXl#nZx?+PtAFRKPXC^<L*h@iSoZ1ShbtclyQLZliI`j5
zjrQy1GMwBt%h=-T+R&x8rS6ulBH?G}{eUen3^#sM|Kn%#-@|`iFI$&><Ua$n&|E&{
zKf_-AnFnXq@7{bG*A8*qd&k$Ww@VomV`mzRx*k|NfoGsJ77%}C8dPoHmB3q##@$!;
zKT$6H_j6r1(wfUv*aY8d{Ab{^Yd>-IpYm7GVHVoK1_QO|Qp)^KYr=nqsvZ9suokF^
zUH^FgN^X<?vGFVD%&2`LzdkJL(u5zc^-y}sJ;80i9(ReE7nCWR*|KGCzOZNh5BK)D
zTNINxO)_U?XI{F$P_lQr#=Wa=+_j5(Qs(t$KdCG_QyDWkYWeb+M+0L;=e06NN$);3
zqupN1Wy!4V`HOBvyqd7tdXwkBwUO1oq>f3Zrl)=Jxfa>0TCz3c$%=KGp8H&0Uz)j)
zSLu{^VW@<tPes9hh7^<k4EDDe?e({1dh4z&Kb_Vtb7<;x)615>v=>i4JW=|F?&66D
zHTMNobJ=e+dA(((PF{+!^YwCP@84$oC)8&IezrU{^R1q8V|v95m%9>uy!oz+GB3WI
z7#ruWQoXC8`dZJff-5E7(G~H6?<AA|-nup~Jjiopey-qLpJSZsLYIC`7j?~F*PmRt
zs<!V|xsriTk6)#1tN1Us`Mg{DMK-%?rLfLOu)nmSvNkSU*W|P5udmuGUaxHQ6Fd=K
z;dcA)n?jGS*hA}8<|OUjp}2kf)^5)l@kvVl*Y|&3-h6P6>)kigpWmq16Te}T*V{Qt
zi+3K<OnzhRmys5Fvh%=Kt?Y##-tLaho-SbURHXk(*xt=qx%Ec(+~sflJg89eHu~b}
zkOxcEE|<UD+N1UOj@p%=tm|Jb68b_mxM^8U(A;_>T1VOYYKuZ;;{80ksmHJDY4g4j
zDcyUn$4vEF(WPr#8jE(E+W<Om1%#2azHiL=&i@RLpYea^|GeH`^FKpXym7P9e}?k(
zY1q#W!gZ!ld|b^*`fNs6`EcEK`w#N3qrW!7S)X=J|9z%^+3EibcP|9GiXJE<`|w}c
z6+^x4q>rWT^cuXE-x+JuX;GJ<gY<+~(@s9zzDdvJxrK!Nxpgn^DK}&;lirr7wT_Wt
zPS?{F>wct(F1Wf&TPFCx<yqyg-j#}3J>C5B(XGrU+e=<5@jMAJ&iZzGWz;Tzn~N?*
zXa4P5wDXqMywu2}>KCR?zvR1l(T*!QiPx3(Z=XBIT!^PcVs-kFZT}f=9ca(r^yc7C
z|M;avA;EpAQTDvkwsWR_Su*wAmyH*e?U%~lbZ5fZyw{r*irCls&1##xtb69$lu40k
z@+Ey!PhSj{IkxJq+Slz<zsqb8xp#Ky&)Ma-mhL^)%DZW&v36?P<@e!AA9>7n#cm1M
zmg>a3GWha-(=&mdX`)l^Tzjkfo%cbe#R}W?#fz)&O1#Q0lXBcs^V?iA?CYatd#~y2
z-@LT>t$x`#j)2p>TTcgu{#_hA^{A`%<iGR%UT*uMpD7e|aD~ctA18HXE<fLF-$%>E
zm!EDk=Ez}Fc&D^;m$p{Ktt|@{@|x~k>!WJ5`T4x%VY`man6-0nqy&3+HgEg7Y^Kiv
zJLXlE-8q$*roww%;#a7Z=Cxy2SIdPjU$8Ra<)!lbSMFRc)y~cpdUY#IDbjdLuFc1Y
zlE8?o>$N8Kq<-_w4D7Y^@?H_>D%!XKN97Ld5RKODp!PuDl|YAi$A?Cb=syF}p@uIP
z&hoAd^O&8z-Z`?)`a^$L{3F(PE0?_#7Z=>U`lw3gPJf25@^9rw>VGlD)<3XSn;e$x
zyS${#{`Rc)Ki~f|9Cy|K_H5yQ2HXD(A1}t_BNWQtyf65l;RS2ye};yy?N>GGm45fu
zZI1pq^JaZsmi*`Y{~2UOz^sq|8Tz^&*KgjZ{GZ_h*X#cbj4%G3`_ccO!9jcdANIws
zrr1xcJ@|Fj{WIrI|1J*r&oJ*l!;cq>5zKYwzpWqH|7BRW|ASoroA(|68NM*>`p<Bn
z%5J6ePw_kQk!$~)d9#1JDgV#+Ae-#rEE4RT-~Xp1@bAtQ_2=vVl!QzFo%>P#Uqg8O
z4}Nzf4~H!L)BTQr&9#4NW%2OPkq5E<)IVOd<36(6?2*0uYbpPe_=4KB?VpV|{kwC;
z{`q5g;DCZj^w@uf^cwg73=2fB|6{(0WH~rgn_rvmpE-9rA{N}iOndnsFNhDCAE1~7
zyA>R#h5M0W?@K+B?O(m*Pwac}>umnB^i)`YKtksDKP&BC<d{Q>mQ?}&T)(ql%dR)J
z)_;4}`Cs*ahR2Kg;Ve?ZX8%7c4R|1t5Hz6hpfJ^d1Bry7ISzB`eiDMF&i>;?NDSi5
zQeSF=+CS;<umk0&8~+(VSqhwy))rUVKGyd8eX57K>*;?6<Ya~vH>=v@rpP`PnQ62(
z$m`PNj8izqurhXk=<h47ULp0~arY_N1wXAFSiY}cwq{X=PVU3Pc8{siuh_JcjjHet
z6fJ5%pC!=DM%@4{{m)LnuKREH%eP+QJ1)I@(3AS4^`iSqg~oW}8$Y~{N4Z?Co$k2j
z^c+KjcQOI&xw)0L%ib$~d-wWS*70v=|1&^Jp~C+RIFjtOpoM;ZpSpIg$qwn-G<oBV
z`ked!89@0s@bBh&Six0~l#h|j`D!1w_|J;^h}D12ys_Vw|DWMGIQiY&4=sk)*&`)#
z<os>7*8OKdZPNPB#vA^@RWJUX_MZWg_ILbeKvs@g9Bh?;>iD&5|Cw`#5gPj6&i>B;
zDZw7&jMl}kuGCLuzozw{!PuG~p#ko4?S7;*uo%hds{{Xq$dCFGM?Eqcmkf^=(LBXk
zS*m2FWXe39@nqKX8l+=AC;zs7WDcp`x7Nx1@IET=<KOMzD_^?1E2eBdsnWT%QgoMR
zMo;v<HQ#@MN+K#BE54_g_u^HP)2p*R^<3DCb!w|Gu+0^1{JVce{e`{NKfCO=f93xr
zzP{ejLHbJ=`?r4!|1&s!eg913-~B80FXHz8Im(cKF+%?KulB#(*Z0p@{JTEjKf}VR
z@1I2&_q!(4=f9Hwq96A!ssHU?=YOrQ@1F@^`#Wm*Xt0c?meEW^zp~}4%zuVgD&Uo`
zT1Vgomzi&QK*i1ahw^)$2wn+b>p#5zC%gIY^ZywZa=l&M_d#^)(Y2l4GiGV>J-cz&
PEh0SG0d`57{QsK(<sPdF

diff --git a/tensorflow/docs_src/mobile/leftnav_files b/tensorflow/docs_src/mobile/leftnav_files
index 4cf134cc3c..585470d5f0 100644
--- a/tensorflow/docs_src/mobile/leftnav_files
+++ b/tensorflow/docs_src/mobile/leftnav_files
@@ -1,6 +1,7 @@
 index.md
 ### TensorFlow Lite
 tflite/index.md
+tflite/devguide.md
 tflite/demo_android.md
 tflite/demo_ios.md
 >>>
diff --git a/tensorflow/docs_src/mobile/tflite/demo_android.md b/tensorflow/docs_src/mobile/tflite/demo_android.md
index c94b5597a6..7f2f8882a2 100644
--- a/tensorflow/docs_src/mobile/tflite/demo_android.md
+++ b/tensorflow/docs_src/mobile/tflite/demo_android.md
@@ -1,42 +1,144 @@
-# TensorFlow Lite Demo for Android
+# Android Demo App
 
-The TensorFlow Lite demo is a camera app that continuously classifies whatever
-it sees from your device's back camera, using a quantized MobileNet model.
+An example Android application using TensorFLow Lite is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
+The demo is a sample camera app that classifies images continuously
+using either a quantized Mobilenet model or a floating point Inception-v3 model.
+To run the demo, a device running Android 5.0 ( API 21) or higher is required.
 
-You'll need an Android device running Android 5.0 or higher to run the demo.
+In the demo app, inference is done using the TensorFlow Lite Java API. The demo
+app classifies frames in real-time, displaying the top most probable
+classifications. It also displays the time taken to detect the object.
 
-To get you started working with TensorFlow Lite on Android, we'll walk you
-through building and deploying our TensorFlow demo app in Android Studio.
+There are three ways to get the demo app to your device:
 
-Note: For a more detailed guide see the
-[TFLite Codelab](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2-tflite/index.html#0)
+* Download the [prebuilt binary APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+* Use Android Studio to build the application.
+* Download the source code for TensorFlow Lite and the demo and build it using
+  bazel.
 
-It's also possible to build the demo app with Bazel, but we only recommend
-this for advanced users who are very familiar with the Bazel build
-environment. For more information on that, see our page [on Github](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite#building-tensorflow-lite-and-the-demo-app-from-source).
 
-## Build and deploy with Android Studio
+## Download the pre-built binary
 
-1. Clone the TensorFlow repository from GitHub if you haven't already:
+The easiest way to try the demo is to download the
+[pre-built binary APK](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk)
 
-        git clone https://github.com/tensorflow/tensorflow
+Once the APK is installed, click the app icon to start the program. The first
+time the app is opened, it asks for runtime permissions to access the device
+camera. The demo app opens the back-camera of the device and recognizes objects
+in the camera's field of view. At the bottom of the image (or at the left
+of the image if the device is in landscape mode), it displays top three objects
+classified and the classification latency.
 
-2. Install the latest version of Android Studio from [here](https://developer.android.com/studio/index.html).
 
-3. From the **Welcome to Android Studio** screen, use the **Import Project
-   (Gradle, Eclipse ADT, etc)** option to import the
-   `tensorflow/contrib/lite/java/demo` directory as an existing Android Studio
-   Project.
+## Build in Android Studio with TensorFlow Lite AAR from JCenter
 
-    Android Studio may prompt you to install Gradle upgrades and other tool
-    versions; you should accept these upgrades.
+Use Android Studio to try out changes in the project code and compile the demo
+app:
 
-4. Download the TensorFlow Lite MobileNet model from [here](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip).
+* Install the latest version of
+  [Android Studio](https://developer.android.com/studio/index.html).
+* Make sure the Android SDK version is greater than 26 and NDK version is greater
+  than 14 (in the Android Studio settings).
+* Import the `tensorflow/contrib/lite/java/demo` directory as a new
+  Android Studio project.
+* Install all the Gradle extensions it requests.
 
-    Unzip this and copy the `mobilenet_quant_v1_224.tflite` file to the assets
-    directory: `tensorflow/contrib/lite/java/demo/app/src/main/assets/`
+To get a model, either:
 
-5. Build and run the app in Android Studio.
+* Download the quantized [Mobilenet TensorFlow Lite model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/mobilenet_v1_224_android_quant_2017_11_08.zip)
+  and unzip and copy `mobilenet_quant_v1_224.tflite` to the assets directory:
+  `tensorflow/contrib/lite/java/demo/app/src/main/assets/`.
+* Or, download the floating point [Inception-v3 model](https://storage.googleapis.com/download.tensorflow.org/models/tflite/inception_v3_slim_2016_android_2017_11_10.zip)
+  and unzip and copy `inceptionv3_non_slim_2015.tflite` to the assets
+  directory. Change the chosen classifier in
+  [Camera2BasicFragment.java](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/java/demo/app/src/main/java/com/example/android/tflitecamerademo/Camera2BasicFragment.java)<br>
+  from: `classifier = new ImageClassifierQuantizedMobileNet(getActivity());`<br>
+  to: `classifier = new ImageClassifierFloatInception(getActivity());`.
 
-You'll have to grant permissions for the app to use the device's camera. Point
-the camera at various objects and enjoy seeing how the model classifies things!
+Now you can build and run the demo app.
+
+
+## Build TensorFlow Lite and the demo app from source
+
+### Clone the TensorFlow repo
+
+```sh
+git clone https://github.com/tensorflow/tensorflow
+```
+
+### Install Bazel
+
+If `bazel` is not installed on your system, see
+[Installing Bazel](https://bazel.build/versions/master/docs/install.html).
+
+Note: Bazel does not currently support Android builds on Windows. Windows users
+should download the
+[prebuilt binary](https://storage.googleapis.com/download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+
+### Install Android NDK and SDK
+
+The Android NDK is required to build the native (C/C++) TensorFlow Lite code. The
+current recommended version is *14b* and can be found on the
+[NDK Archives](https://developer.android.com/ndk/downloads/older_releases.html#ndk-14b-downloads)
+page.
+
+The Android SDK and build tools can be
+[downloaded separately](https://developer.android.com/tools/revisions/build-tools.html)
+or used as part of
+[Android Studio](https://developer.android.com/studio/index.html). To build the
+TensorFlow Lite Android demo, build tools require API >= 23 (but it will run on
+devices with API >= 21).
+
+In the root of the TensorFlow repository, update the `WORKSPACE` file with the
+`api_level` and location of the SDK and NDK. If you installed it with
+Android Studio, the SDK path can be found in the SDK manager. The default NDK
+path is:`{SDK path}/ndk-bundle.` For example:
+
+```
+android_sdk_repository (
+    name = "androidsdk",
+    api_level = 23,
+    build_tools_version = "23.0.2",
+    path = "/home/xxxx/android-sdk-linux/",
+)
+
+android_ndk_repository(
+    name = "androidndk",
+    path = "/home/xxxx/android-ndk-r10e/",
+    api_level = 19,
+)
+```
+
+Some additional details are available on the
+[TF Lite Android App page](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/README.md).
+
+### Build the source code
+
+To build the demo app, run `bazel`:
+
+```
+bazel build --cxxopt=--std=c++11 //tensorflow/contrib/lite/java/demo/app/src/main:TfLiteCameraDemo
+```
+
+Caution: Because of an bazel bug, we only support building the Android demo app
+within a Python 2 environment.
+
+
+## About the demo
+
+The demo app is resizing each camera image frame (224 width * 224 height) to
+match the quantized MobileNets model (299 * 299 for Inception-v3). The resized
+image is converted—row by row—into a
+[ByteBuffer](https://developer.android.com/reference/java/nio/ByteBuffer.html).
+Its size is  1 * 224 * 224 * 3 bytes, where 1 is the number of images in a batch.
+224 * 224 (299 * 299) is the width and height of the image. 3 bytes represents
+the 3 colors of a pixel.
+
+This demo uses the TensorFlow Lite Java inference API
+for models which take a single input and provide a single output. This outputs a
+two-dimensional array, with the first dimension being the category index and the
+second dimension being the confidence of classification. Both models have 1001
+unique categories and the app sorts the probabilities of all the categories and
+displays the top three. The model file must be downloaded and bundled within the
+assets directory of the app.
diff --git a/tensorflow/docs_src/mobile/tflite/demo_ios.md b/tensorflow/docs_src/mobile/tflite/demo_ios.md
index 3ee9b1cbca..3be21da89f 100644
--- a/tensorflow/docs_src/mobile/tflite/demo_ios.md
+++ b/tensorflow/docs_src/mobile/tflite/demo_ios.md
@@ -1,4 +1,4 @@
-# TensorFlow Lite Demo for iOS
+# iOS Demo App
 
 The TensorFlow Lite demo is a camera app that continuously classifies whatever
 it sees from your device's back camera, using a quantized MobileNet model. These
diff --git a/tensorflow/docs_src/mobile/tflite/devguide.md b/tensorflow/docs_src/mobile/tflite/devguide.md
new file mode 100644
index 0000000000..5b521dca7b
--- /dev/null
+++ b/tensorflow/docs_src/mobile/tflite/devguide.md
@@ -0,0 +1,224 @@
+# Developer Guide
+
+Using a TensorFlow Lite model in your mobile app requires multiple
+considerations: you must choose a pre-trained or custom model, convert the model
+to a TensorFLow Lite format, and finally, integrate the model in your app.
+
+## 1. Choose a model
+
+Depending on the use case, you can choose one of the popular open-sourced models,
+such as *InceptionV3* or *MobileNets*, and re-train these models with a custom
+data set or even build your own custom model.
+
+### Use a pre-trained model
+
+[MobileNets](https://research.googleblog.com/2017/06/mobilenets-open-source-models-for.html)
+is a family of mobile-first computer vision models for TensorFlow designed to
+effectively maximize accuracy, while taking into consideration the restricted
+resources for on-device or embedded applications. MobileNets are small,
+low-latency, low-power models parameterized to meet the resource constraints for
+a variety of uses. They can be used for classification, detection, embeddings, and
+segmentation—similar to other popular large scale models, such as
+[Inception](https://arxiv.org/pdf/1602.07261.pdf). Google provides 16 pre-trained
+[ImageNet](http://www.image-net.org/challenges/LSVRC/) classification checkpoints
+for MobileNets that can be used in mobile projects of all sizes.
+
+[Inception-v3](https://arxiv.org/abs/1512.00567) is an image recognition model
+that achieves fairly high accuracy recognizing general objects with 1000 classes,
+for example, "Zebra", "Dalmatian", and "Dishwasher". The model extracts general
+features from input images using a convolutional neural network and classifies
+them based on those features with fully-connected and softmax layers.
+
+[On Device Smart Reply](https://research.googleblog.com/2017/02/on-device-machine-intelligence.html)
+is an on-device model that provides one-touch replies for incoming text messages
+by suggesting contextually relevant messages. The model is built specifically for
+memory constrained devices, such as watches and phones, and has been successfully
+used in Smart Replies on Android Wear. Currently, this model is Android-specific.
+
+These pre-trained models are [available for download](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/g3doc/models.md)
+
+### Re-train Inception-V3 or MobileNet for a custom data set
+
+These pre-trained models were trained on the *ImageNet* data set which contains
+1000 predefined classes. If these classes are not sufficient for your use case,
+the model will need to be re-trained. This technique is called
+*transfer learning* and starts with a model that has been already trained on a
+problem, then retrains the model on a similar problem. Deep learning from
+scratch can take days, but transfer learning is fairly quick. In order to do
+this, you need to generate a custom data set labeled with the relevant classes.
+
+The [TensorFlow for Poets](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets/)
+codelab walks through the re-training process step-by-step. The code supports
+both floating point and quantized inference.
+
+### Train a custom model
+
+A developer may choose to train a custom model using Tensorflow (see the
+@{$tutorials} for examples of building and training models). If you have already
+written a model, the first step is to export this to a @{tf.GraphDef} file. This
+is required because some formats do not store the model structure outside the
+code, and we must communicate with other parts of the framework. See
+[Exporting the Inference Graph](https://github.com/tensorflow/models/blob/master/research/slim/README.md)
+to create .pb file for the custom model.
+
+TensorFlow Lite currently supports a subset of TensorFlow operators. Refer to the
+[TensorFlow Lite & TensorFlow Compatibility Guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for supported operators and their usage. This set of operators will continue to
+grow in future Tensorflow Lite releases.
+
+
+## 2. Convert the model format
+
+The model generated (or downloaded) in the previous step is a *standard*
+Tensorflow model and you should now have a .pb or .pbtxt @{tf.GraphDef} file.
+Models generated with transfer learning (re-training) or custom models must be
+converted—but, we must first freeze the graph to convert the model to the
+Tensorflow Lite format. This process uses several model formats:
+
+* @{tf.GraphDef} (.pb) —A protobuf that represents the TensorFlow training or
+  computation graph. It contains operators, tensors, and variables definitions.
+* *CheckPoint* (.ckpt) —Serialized variables from a TensorFlow graph. Since this
+  does not contain a graph structure, it cannot be interpreted by itself.
+* `FrozenGraphDef` —A subclass of `GraphDef` that does not contain
+  variables. A `GraphDef` can be converted to a `FrozenGraphDef` by taking a
+  CheckPoint and a `GraphDef`, and converting each variable into a constant
+  using the value retrieved from the CheckPoint.
+* `SavedModel` —A `GraphDef` and CheckPoint with a signature that labels
+  input and output arguments to a model. A `GraphDef` and CheckPoint can be
+  extracted from a `SavedModel`.
+* *TensorFlow Lite model* (.tflite) —A serialized
+  [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
+  Lite operators and tensors for the TensorFlow Lite interpreter, similiar to a
+  `FrozenGraphDef`.
+
+### Freeze Graph
+
+To use the `GraphDef` .pb file with TensorFlow Lite, you must have checkpoints
+that contain trained weight parameters. The .pb file only contains the structure
+of the graph. The process of merging the checkpoint values with the graph
+structure is called *freezing the graph*.
+
+You should have a checkpoints folder or download them for a pre-trained model
+(for example,
+[MobileNets](https://github.com/tensorflow/models/blob/master/research/slim/nets/mobilenet_v1.md)).
+
+To freeze the graph, use the following command (changing the arguments):
+
+```
+freeze_graph --input_graph=/tmp/mobilenet_v1_224.pb \
+  --input_checkpoint=/tmp/checkpoints/mobilenet-10202.ckpt \
+  --input_binary=true \
+  --output_graph=/tmp/frozen_mobilenet_v1_224.pb \
+  --output_node_names=MobileNetV1/Predictions/Reshape_1
+```
+
+The `input_binary` flag must be enabled so the protobuf is read and written in
+a binary format. Set the `input_graph` and `input_checkpoint` files.
+
+The `output_node_names` may not be obvious outside of the code that built the
+model. The easiest way to find them is to visualize the graph, either with
+[TensorBoard](https://codelabs.developers.google.com/codelabs/tensorflow-for-poets-2/#3)
+or `graphviz`.
+
+The frozen `GraphDef` is now ready for conversion to the `FlatBuffer` format
+(.tflite) for use on Android or iOS devices. For Android, the Tensorflow
+Optimizing Converter tool supports both float and quantized models. To convert
+the frozen `GraphDef` to the .tflite format:
+
+```
+toco --input_file=$(pwd)/mobilenet_v1_1.0_224/frozen_graph.pb \
+  --input_format=TENSORFLOW_GRAPHDEF \
+  --output_format=TFLITE \
+  --output_file=/tmp/mobilenet_v1_1.0_224.tflite \
+  --inference_type=FLOAT \
+  --input_type=FLOAT \
+  --input_arrays=input \
+  --output_arrays=MobilenetV1/Predictions/Reshape_1 \
+  --input_shapes=1,224,224,3
+```
+
+The `input_file` argument should reference the frozen `GraphDef` file
+containing the model architecture. The [frozen_graph.pb](https://storage.googleapis.com/download.tensorflow.org/models/mobilenet_v1_1.0_224_frozen.tgz)
+file used here is available for download. `output_file` is where the TensorFlow
+Lite model will get generated. The `input_type` and `inference_type`
+arguments should be set to `FLOAT`, unless converting a
+@{$performance/quantization$quantized model}. Setting the `input_array`,
+`output_array`, and `input_shape` arguments are not as straightforward. The
+easiest way to find these values is to explore the graph using Tensorboard. Reuse
+the arguments for specifying the output nodes for inference in the
+`freeze_graph` step.
+
+It is also possible to use the Tensorflow Optimizing Converter with protobufs
+from either Python or from the command line (see the 
+[toco_from_protos.py](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/python/toco_from_protos.py)
+example). This allows you to integrate the conversion step into the model design
+workflow, ensuring the model is easily convertible to a mobile inference graph.
+For example:
+
+```python
+import tensorflow as tf
+
+img = tf.placeholder(name="img", dtype=tf.float32, shape=(1, 64, 64, 3))
+val = img + tf.constant([1., 2., 3.]) + tf.constant([1., 4., 4.])
+out = tf.identity(val, name="out")
+
+with tf.Session() as sess:
+  tflite_model = tf.contrib.lite.toco_convert(sess.graph_def, [img], [out])
+  open("converteds_model.tflite", "wb").write(tflite_model)
+```
+
+For usage, see the Tensorflow Optimizing Converter
+[command-line examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/toco/g3doc/cmdline_examples.md).
+
+Refer to the
+[Ops compatibility guide](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/tf_ops_compatibility.md)
+for troubleshooting help, and if that doesn't help, please
+[file an issue](https://github.com/tensorflow/tensorflow/issues).
+
+The [development repo](https://github.com/tensorflow/tensorflow) contains a tool
+to visualize TensorFlow Lite models after conversion. To build the
+[visualize.py](https://github.com/tensorflow/tensorflow/blob/master/tensorflow/contrib/lite/tools/visualize.py)
+tool:
+
+```sh
+bazel run tensorflow/contrib/lite/tools:visualize -- model.tflite model_viz.html
+```
+
+This generates an interactive HTML page listing subgraphs, operations, and a
+graph visualization.
+
+
+## 3. Use the TensorFlow Lite model for inference in a mobile app
+
+After completing the prior steps, you should now have a .tflite model file.
+
+### Android
+
+Since Android apps are written in Java and the core TensorFlow library is in C++,
+a JNI library is provided as an interface. This is only meant for inference—it
+provides the ability to load a graph, set up inputs, and run the model to
+calculate outputs.
+
+The open source Android demo app uses the JNI interface and is available
+[on GitHub](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/java/demo/app).
+You can also download a
+[prebuilt APK](http://download.tensorflow.org/deps/tflite/TfLiteCameraDemo.apk).
+See the @{$tflite/demo_android} guide for details.
+
+The @{$mobile/android_build} guide has instructions for installing TensorFlow on
+Android and setting up `bazel` and Android Studio.
+
+### iOS
+
+To integrate a TensorFlow model in an iOS app, see the
+[TensorFlow Lite for iOS](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/lite/g3doc/ios.md)
+guide and @{$tflite/demo_ios} guide.
+
+#### Core ML support
+
+Core ML is a machine learning framework used in Apple products. In addition to
+using Tensorflow Lite models directly in your applications, you can convert
+trained Tensorflow models to the
+[CoreML](https://developer.apple.com/machine-learning/) format for use on Apple
+devices. To use the converter, refer to the
+[Tensorflow-CoreML converter documentation](https://github.com/tf-coreml/tf-coreml).
diff --git a/tensorflow/docs_src/mobile/tflite/index.md b/tensorflow/docs_src/mobile/tflite/index.md
index beb24794fc..11f11ea4dc 100644
--- a/tensorflow/docs_src/mobile/tflite/index.md
+++ b/tensorflow/docs_src/mobile/tflite/index.md
@@ -155,7 +155,9 @@ retraining for both floating point and quantized inference.
 
 The following diagram shows the architectural design of TensorFlow Lite:
 
-![tensorflow lite architecture](https://www.tensorflow.org/images/tflite-architecture.jpg)
+<img src="/images/tflite-architecture.jpg"
+     alt="TensorFlow Lite architecture diagram"
+     style="max-width:600px;">
 
 Starting with a trained TensorFlow model on disk, you'll convert that model to
 the TensorFlow Lite file format (`.tflite`) using the TensorFlow Lite
-- 
GitLab


From 17d05bb44cecea741e8a4f7adb8c107af06893b4 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Thu, 29 Mar 2018 23:35:30 -0700
Subject: [PATCH 742/960] Internal change.

PiperOrigin-RevId: 191037166
---
 tensorflow/compiler/tests/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index edabdc218a..204a2a2f90 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -375,6 +375,7 @@ tf_xla_py_test(
     name = "momentum_test",
     size = "small",
     srcs = ["momentum_test.py"],
+    tags = ["no_oss"],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
-- 
GitLab


From 266d701ea642394771d238566d9f3c00eab9ea19 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 13:31:23 -0700
Subject: [PATCH 743/960] Docs: Add Eager Execution guide to Programmer's
 Guide.

PiperOrigin-RevId: 190977505
---
 tensorflow/contrib/eager/README.md            |  20 +-
 .../contrib/eager/python/g3doc/guide.md       | 906 +---------------
 .../docs_src/programmers_guide/eager.md       | 992 ++++++++++++++++++
 .../docs_src/programmers_guide/leftnav_files  |   3 +-
 4 files changed, 1015 insertions(+), 906 deletions(-)
 create mode 100644 tensorflow/docs_src/programmers_guide/eager.md

diff --git a/tensorflow/contrib/eager/README.md b/tensorflow/contrib/eager/README.md
index 9d2ca07c3a..9a3b780af8 100644
--- a/tensorflow/contrib/eager/README.md
+++ b/tensorflow/contrib/eager/README.md
@@ -1,12 +1,8 @@
 # Eager Execution
 
-> *WARNING*: This is a preview/pre-alpha version. The API and performance
-> characteristics are subject to change.
-
-Eager execution is an experimental interface to TensorFlow that provides an
-imperative programming style (à la [NumPy](http://www.numpy.org)). When you
-enable eager execution, TensorFlow operations execute immediately; you do not
-execute a pre-constructed graph with
+Eager execution provides an imperative interface to TensorFlow (similiar to
+[NumPy](http://www.numpy.org)). When you enable eager execution, TensorFlow
+operations execute immediately; you do not execute a pre-constructed graph with
 [`Session.run()`](https://www.tensorflow.org/api_docs/python/tf/Session).
 
 For example, consider a simple computation in TensorFlow:
@@ -33,7 +29,7 @@ print(m)
 ## Caveats
 
 This feature is in early stages and work remains to be done in terms of smooth
-support for distributed and multi-GPU training and CPU performance.
+support for distributed and multi-GPU training and performance.
 
 - [Known issues](https://github.com/tensorflow/tensorflow/issues?q=is%3Aissue%20is%3Aopen%20label%3Acomp%3Aeager)
 - Feedback is welcome, please consider
@@ -41,21 +37,23 @@ support for distributed and multi-GPU training and CPU performance.
 
 ## Installation
 
-Eager execution is included in TensorFlow versions 1.5 and above.
+Eager execution is included in TensorFlow versions 1.7 and above.
 Installation instructions at https://www.tensorflow.org/install/
 
 ## Documentation
 
 For an introduction to eager execution in TensorFlow, see:
 
-- [User Guide](python/g3doc/guide.md)
+- [User Guide](https://www.tensorflow.org/programmers_guide/eager) ([source](../../docs_src/programmers_guide/eager.md))
 - Notebook: [Basic Usage](python/examples/notebooks/1_basics.ipynb)
 - Notebook: [Gradients](python/examples/notebooks/2_gradients.ipynb)
 - Notebook: [Importing Data](python/examples/notebooks/3_datasets.ipynb)
 
 ## Changelog
 
-- 2017/10/31: Initial preview release.
+- 2017/10/31: Initial preview release (in TensorFlow 1.5)
 - 2017/12/01: Example of dynamic neural network:
   [SPINN: Stack-augmented Parser-Interpreter Neural Network](https://arxiv.org/abs/1603.06021).
   See [README.md](python/examples/spinn/README.md) for details.
+- 2017/03: Core functionality moved out of the experimental tf.contrib namespace
+  in TensorFlow 1.7.
diff --git a/tensorflow/contrib/eager/python/g3doc/guide.md b/tensorflow/contrib/eager/python/g3doc/guide.md
index ebb05051f2..2d2aba6908 100644
--- a/tensorflow/contrib/eager/python/g3doc/guide.md
+++ b/tensorflow/contrib/eager/python/g3doc/guide.md
@@ -1,900 +1,18 @@
-# TensorFlow Eager Execution
-
-## What is this?
+# Eager execution
 
 Eager execution is a feature that makes TensorFlow execute operations
-immediately: concrete values are returned, instead of a computational graph to
-be executed later.
-
-As a result, enabling eager execution provides:
-
--   A [NumPy](http://www.numpy.org/)-like library for numerical computation with
-    support for GPU acceleration and automatic differentiation.
--   A flexible platform for machine learning research and experimentation.
-
-Eager execution is under active development. This guide walks through an
-alpha/preview release. In particular, not all TensorFlow APIs currently work
-with eager execution enabled, and some models may be slow to execute, compared
-to models defined without using eager execution.
-
-## Installation
-
-Eager execution is included in TensorFlow versions 1.5 and above.
-Installation instructions at https://www.tensorflow.org/install/
-
-The contents of this guide are compatible with TensorFlow 1.5. However, if you
-run into bugs that are fixed in source but not the release, you may want to
-either [build from source](https://www.tensorflow.org/install/install_sources)
-or try a nightly build. The nightly builds are available as:
-
-- [`pip` packages](https://github.com/tensorflow/tensorflow/blob/master/README.md#installation) and
-
-- [docker](https://hub.docker.com/r/tensorflow/tensorflow/) images.
-
-For example, to run the latest nightly docker image:
-
-```sh
-# If you have a GPU, use https://github.com/NVIDIA/nvidia-docker
-docker pull tensorflow/tensorflow:nightly-gpu
-docker run --runtime=nvidia -it -p 8888:8888 tensorflow/tensorflow:nightly-gpu
-
-# If you do not have a GPU, use the CPU-only image
-docker pull tensorflow/tensorflow:nightly
-docker run -it -p 8888:8888 tensorflow/tensorflow:nightly
-```
-
-And then visit http://localhost:8888 in your browser for a Jupyter notebook
-environment.
-
-## Getting Started
-
-With TensorFlow installed, eager execution is enabled via a single call:
-
-```python
-import tensorflow as tf
-
-import tensorflow.contrib.eager as tfe
-
-tfe.enable_eager_execution()
-```
-
-Enabling eager execution changes how TensorFlow functions behave (in particular,
-`Tensor` objects will reference concrete values instead of being symbolic
-handles to nodes in a computational graph). As a result, eager execution should
-be enabled at the beginning of a program and cannot be disabled afterwards in
-the same program.
-
-Code examples in the rest of this guide assume that eager execution has been
-enabled.
-
-## A library for numerical computation
-
-A significant fraction of the [TensorFlow
-API](https://www.tensorflow.org/api_docs/python/) consists of numerical
-operations:
-[arithmetic operations](https://www.tensorflow.org/api_guides/python/math_ops#Arithmetic_Operators),
-[matrix operations](https://www.tensorflow.org/api_guides/python/math_ops#Matrix_Math_Functions),
-[linear algebra operations](https://www.tensorflow.org/versions/master/api_docs/python/tf/linalg),
-etc.
-
-With eager execution enabled, these operations consume and return
-multi-dimensional arrays as `Tensor` objects, similar to NumPy
-[`ndarray`s](https://docs.scipy.org/doc/numpy-1.13.0/reference/generated/numpy.ndarray.html).
-For example:
-
-```python
-# Multiply two 2x2 matrices
-x = tf.matmul([[1, 2],
-               [3, 4]],
-              [[4, 5],
-               [6, 7]])
-# Add one to each element
-# (tf.add supports broadcasting)
-y = tf.add(x, 1)
-
-# Create a random random 5x3 matrix
-z = tf.random_uniform([5, 3])
-
-print(x)
-print(y)
-print(z)
-```
-
-Output:
-
-```
-tf.Tensor(
-[[16 19]
- [36 43]], shape=(2, 2), dtype=int32)
-tf.Tensor(
-[[17 20]
- [37 44]], shape=(2, 2), dtype=int32)
-tf.Tensor(
-[[ 0.25058532  0.0929395   0.54113817]
- [ 0.3108716   0.93350542  0.84909797]
- [ 0.53081679  0.12788558  0.01767385]
- [ 0.29725885  0.33540785  0.83588314]
- [ 0.38877153  0.39720535  0.78914213]], shape=(5, 3), dtype=float32)
-```
-
-For convenience, these operations can also be triggered via operator overloading
-of the `Tensor` object. For example, the `+` operator is equivalent to `tf.add`,
-`-` to `tf.subtract`, `*` to `tf.multiply`, etc.:
-
-```python
-x = (tf.ones([1], dtype=tf.float32) + 1) * 2 - 1
-print(x)
-```
-
-Output:
-
-```
-tf.Tensor([ 3.], shape=(1,), dtype=float32)
-```
-
-### Converting to and from NumPy
-
-The operations above automatically convert Python objects (like lists of
-numbers) and NumPy arrays to `Tensor` objects. `Tensor` objects can also be used
-as NumPy arrays by numpy operations.
-
-```python
-import numpy as np
-
-x = tf.add(1, 1)                     # tf.Tensor with a value of 2
-y = tf.add(np.array(1), np.array(1)) # tf.Tensor with a value of 2
-z = np.multiply(x, y)                # numpy.int64 with a value of 4
-```
-
-Alternatively, they can be explicitly converted using
-[`tf.constant`](https://www.tensorflow.org/api_docs/python/tf/constant), as
-shown in the next example.
-
-Conversely, you can call the `numpy()` method of a `Tensor` object' to obtain
-its NumPy `ndarray` value. For example:
-
-```python
-import numpy as np
-
-np_x = np.array(2., dtype=np.float32)
-x = tf.constant(np_x)
-
-py_y = 3.
-y = tf.constant(py_y)
-
-z = x + y + 1
-
-print(z)
-print(z.numpy())
-```
-
-Output:
-
-```
-tf.Tensor(6.0, shape=(), dtype=float32)
-6.0
-```
-
-### GPU acceleration
-
-Many TensorFlow operations support GPU acceleration. With eager execution
-enabled, [computation is *not* automatically
-offloaded](https://www.tensorflow.org/tutorials/using_gpu) to GPUs. Instead, you
-must explicitly specify when GPUs should be used.
-
-The simplest way to do this is to enclose your computation in a `with
-tf.device('/gpu:0')` block. Also of interest is the `tfe.num_gpus()` function,
-which returns the number of available GPUs.
-
-For example, consider this snippet to measure the time to multiply two 1000x1000
-matrices on CPU:
-
-```python
-import time
-
-def measure(x):
-  # The very first time a GPU is used by TensorFlow, it is initialized.
-  # So exclude the first run from timing.
-  tf.matmul(x, x)
-
-  start = time.time()
-  for i in range(10):
-    tf.matmul(x, x)
-  end = time.time()
-
-  return "Took %s seconds to multiply a %s matrix by itself 10 times" % (end - start, x.shape)
-
-# Run on CPU:
-with tf.device("/cpu:0"):
-  print("CPU: %s" % measure(tf.random_normal([1000, 1000])))
-
-# If a GPU is available, run on GPU:
-if tfe.num_gpus() > 0:
-  with tf.device("/gpu:0"):
-    print("GPU: %s" % measure(tf.random_normal([1000, 1000])))
-```
-
-Output (exact numbers will depend on the characteristics of the hardware):
-
-```python
-CPU: Took 0.145531892776 seconds to multiply a (1000, 1000) matrix by itself 10 times
-GPU: Took 0.000458955764771 seconds to multiply a (1000, 1000) matrix by itself 10 times
-```
-
-Alternatively, methods on the `Tensor` object can be used to explicitly copy the
-`Tensor` to a different device. Operations are typically executed on the device
-on which the inputs are placed. For example:
-
-```python
-x = tf.random_normal([10, 10])
-
-x_gpu0 = x.gpu()
-x_cpu = x.cpu()
-
-_ = tf.matmul(x_cpu, x_cpu)  # Runs on CPU
-_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
-
-if tfe.num_gpus() > 1:
-  x_gpu1 = x.gpu(1)
-  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
-```
-
-### Automatic Differentiation
-
-[Automatic
-differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation) is
-very useful when implementing many machine learning algorithms (e.g.,
-[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training
-neural networks). For this purpose, TensorFlow eager execution provides an
-[autograd](https://github.com/HIPS/autograd)-style API for automatic
-differentiation. Specifically, the functions:
-
--   `tfe.gradients_function(f)`: Returns a Python function that computes the
-    derivatives of the Python function `f` with respect to its arguments. `f`
-    must return a scalar value. When the returned function is invoked, it
-    returns a list of `Tensor` objects (one element for each argument of `f`).
--   `tfe.value_and_gradients_function(f)`: Similar to `tfe.gradients_function`,
-    except that when the returned function is invoked, it returns the value of
-    `f` in addition to the list of derivatives of `f` with respect to its
-    arguments.
-
-These functions naturally apply to higher order differentiation as well. For
-example:
-
-```python
-def f(x):
-  return tf.multiply(x, x)  # Or x * x
-assert 9 == f(3.).numpy()
-
-df = tfe.gradients_function(f)
-assert 6 == df(3.)[0].numpy()
-
-# Second order deriviative.
-d2f = tfe.gradients_function(lambda x: df(x)[0])
-assert 2 == d2f(3.)[0].numpy()
-
-# Third order derivative.
-d3f = tfe.gradients_function(lambda x : d2f(x)[0])
-assert 0 == d3f(3.)[0].numpy()
-```
-
-These functions can be used to train models. For example, consider the following
-simple linear regression model:
-
-```python
-def prediction(input, weight, bias):
-  return input * weight + bias
-
-# A toy dataset of points around 3 * x + 2
-NUM_EXAMPLES = 1000
-training_inputs = tf.random_normal([NUM_EXAMPLES])
-noise = tf.random_normal([NUM_EXAMPLES])
-training_outputs = training_inputs * 3 + 2 + noise
-
-# A loss function: Mean-squared error
-def loss(weight, bias):
-  error = prediction(training_inputs, weight, bias) - training_outputs
-  return tf.reduce_mean(tf.square(error))
-
-# Function that returns the derivative of loss with respect to
-# weight and bias
-grad = tfe.gradients_function(loss)
-
-# Train for 200 steps (starting from some random choice for W and B, on the same
-# batch of data).
-W = 5.
-B = 10.
-learning_rate = 0.01
-print("Initial loss: %f" % loss(W, B).numpy())
-for i in range(200):
-  (dW, dB) = grad(W, B)
-  W -= dW * learning_rate
-  B -= dB * learning_rate
-  if i % 20 == 0:
-    print("Loss at step %d: %f" % (i, loss(W, B).numpy()))
-print("Final loss: %f" % loss(W, B).numpy())
-print("W, B = %f, %f" % (W.numpy(), B.numpy()))
-```
-
-Output: (the exact numbers may vary depending on the randomness in noise)
-
-```
-Initial loss: 66.730003
-Loss at step 0: 64.200096
-Loss at step 20: 29.872814
-Loss at step 40: 14.233772
-Loss at step 60: 7.090570
-Loss at step 80: 3.819887
-Loss at step 100: 2.318821
-Loss at step 120: 1.628385
-Loss at step 140: 1.310142
-Loss at step 160: 1.163167
-Loss at step 180: 1.095162
-Final loss: 1.064711
-W, B = 3.094944, 2.161383
-```
-
-To utilize the GPU, place the code above within a `with tf.device("/gpu:0"):`
-block. (However, this particular model, with only two floating point parameters,
-is unlikely to benefit from GPU acceleration.)
-
-### Customizing gradients
-
-One may want to define custom gradients for an operation, or for a function.
-This may be useful for multiple reasons, including providing a more efficient
-or more [numerically stable](https://en.wikipedia.org/wiki/Numerical_stability)
-gradient for a sequence of operations.
-
-For example, consider the function `log(1 + e^x)`, which commonly occurs in the
-computation of cross entropy and log likelihoods.
-
-```python
-def log1pexp(x):
-  return tf.log(1 + tf.exp(x))
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# Works fine at x = 0.
-assert 0.5 == float(grad_log1pexp(0.)[0])
-
-# Returns a `nan` at x = 100 due to numerical instability.
-import math
-assert math.isnan(float(grad_log1pexp(100.)[0]))
-```
-
-We can define a custom gradient for the above function that analytically
-simplifies the gradient expression.
-
-```python
-@tfe.custom_gradient
-def log1pexp(x):
-  e = tf.exp(x)
-  def grad(dy):
-    return dy * (1 - 1 / (1 + e))
-  return tf.log(1 + e), grad
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# Works as before at x = 0.
-assert 0.5 == float(grad_log1pexp(0.)[0])
-
-# But now works at x = 100 as well.
-assert 1.0 == float(grad_log1pexp(100.)[0])
-```
-Also notice how the gradient function implementation reuses an expression
-(`tf.exp(x)`) computed during the forward pass, hence making the gradient
-computation more efficient by avoiding redundant computation.
-
-## Building and training models
-
-In practice, your computation may have many parameters to be optimized (by
-computing derivatives). Encapsulating them into re-usable classes/objects
-makes the code easier to follow than writing a single top-level function with
-many arguments.
-
-In fact, eager execution encourages use of the [Keras](https://keras.io)-style
-"Layer" classes in the
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers)
-module.
-
-Furthermore, you may want to apply more sophisticated techniques to compute
-parameter updates, such as those in
-[`tf.train.Optimizer`](https://www.tensorflow.org/api_guides/python/train#Optimizers)
-implementations.
-
-This next section walks through using the same `Optimizer` and `Layer` APIs used
-to build trainable TensorFlow graphs in an environment where eager execution is
-enabled.
-
-### Variables and Optimizers
-
-`tfe.Variable` objects store mutable `Tensor` values that can be accessed during
-training, making automatic differentiation easier. In particular, parameters of
-a model can be encapsulated in Python classes as variables.
-
-`tfe.gradients_function(f)` introduced earlier computes the derivatives of `f`
-with respect to its arguments. However, it requires all parameters of interest
-to be arguments of `f`, which becomes cumbersome when `f` depends on a large
-number of trainable parameters.
-
-`tfe.implicit_gradients` is an alternative function with some useful properties:
-
--   It computes the derivatives of `f` with respect to all the `tfe.Variable`s
-    used by `f`.
--   When the returned function is invoked, it returns a list of
-    (gradient value, Variable object) tuples.
-
-Representing model parameters as `Variable` objects, along with the use of
-`tfe.implicit_gradients`, typically results in better encapsulation. For
-example, the linear regression model described above can be written into a
-class:
-
-```python
-class Model(object):
-  def __init__(self):
-    self.W = tfe.Variable(5., name='weight')
-    self.B = tfe.Variable(10., name='bias')
-
-  def predict(self, inputs):
-    return inputs * self.W + self.B
-
-
-# The loss function to be optimized
-def loss(model, inputs, targets):
-  error = model.predict(inputs) - targets
-  return tf.reduce_mean(tf.square(error))
-
-# A toy dataset of points around 3 * x + 2
-NUM_EXAMPLES = 1000
-training_inputs = tf.random_normal([NUM_EXAMPLES])
-noise = tf.random_normal([NUM_EXAMPLES])
-training_outputs = training_inputs * 3 + 2 + noise
-
-# Define:
-# 1. A model
-# 2. Derivatives of a loss function with respect to model parameters
-# 3. A strategy for updating the variables based on the derivatives
-model = Model()
-grad = tfe.implicit_gradients(loss)
-optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
-
-# The training loop
-print("Initial loss: %f" %
-      loss(model, training_inputs, training_outputs).numpy())
-for i in range(201):
-  optimizer.apply_gradients(grad(model, training_inputs, training_outputs))
-  if i % 20 == 0:
-    print("Loss at step %d: %f" %
-          (i, loss(model, training_inputs, training_outputs).numpy()))
-print("Final loss: %f" % loss(model, training_inputs, training_outputs).numpy())
-print("W, B = %s, %s" % (model.W.numpy(), model.B.numpy()))
-```
-
-Output:
-
-```
-Initial loss: 69.693184
-Loss at step 0: 66.987854
-Loss at step 20: 30.553387
-Loss at step 40: 14.250237
-Loss at step 60: 6.955020
-Loss at step 80: 3.690550
-Loss at step 100: 2.229739
-Loss at step 120: 1.576032
-Loss at step 140: 1.283496
-Loss at step 160: 1.152584
-Loss at step 180: 1.093999
-Final loss: 1.067780
-W, B = 3.0114281, 2.0865183
-```
-
-Using `implicit_gradients` avoids the need to provide all the trainable
-parameters of the model as arguments to the `loss` function.
-
-### Using Keras and the Layers API
-
-[Keras](https://keras.io) is a popular API for defining model structures. The
-[`tf.keras.layers`](https://www.tensorflow.org/api_docs/python/tf/keras/layers)
-module provides a set of building blocks for models and is implemented using the
-`tf.layers.Layer` subclasses in the
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers)
-module. We encourage the use of these same building blocks when using
-TensorFlow's eager execution feature. For example, the very same linear
-regression model can be built using `tf.layers.Dense`:
-
-```python
-class Model(object):
-  def __init__(self):
-    self.layer = tf.layers.Dense(1)
-
-  def predict(self, inputs):
-    return self.layer(inputs)
-```
-
-The `tf.layers` API makes it more convenient to define more sophisticated
-models. For example, the following will train an MNIST model:
-
-```python
-class MNISTModel(object):
-  def __init__(self, data_format):
-    # 'channels_first' is typically faster on GPUs
-    # while 'channels_last' is typically faster on CPUs.
-    # See: https://www.tensorflow.org/performance/performance_guide#data_formats
-    if data_format == 'channels_first':
-      self._input_shape = [-1, 1, 28, 28]
-    else:
-      self._input_shape = [-1, 28, 28, 1]
-    self.conv1 = tf.layers.Conv2D(32, 5,
-                                  padding='same',
-                                  activation=tf.nn.relu,
-                                  data_format=data_format)
-    self.max_pool2d = tf.layers.MaxPooling2D(
-        (2, 2), (2, 2), padding='same', data_format=data_format)
-    self.conv2 = tf.layers.Conv2D(64, 5,
-                                  padding='same',
-                                  activation=tf.nn.relu,
-                                  data_format=data_format)
-    self.dense1 = tf.layers.Dense(1024, activation=tf.nn.relu)
-    self.dropout = tf.layers.Dropout(0.5)
-    self.dense2 = tf.layers.Dense(10)
-
-  def predict(self, inputs):
-    x = tf.reshape(inputs, self._input_shape)
-    x = self.max_pool2d(self.conv1(x))
-    x = self.max_pool2d(self.conv2(x))
-    x = tf.layers.flatten(x)
-    x = self.dropout(self.dense1(x))
-    return self.dense2(x)
-
-def loss(model, inputs, targets):
-  return tf.reduce_mean(
-      tf.nn.softmax_cross_entropy_with_logits(
-          logits=model.predict(inputs), labels=targets))
-
-
-# Load the training and validation data
-from tensorflow.examples.tutorials.mnist import input_data
-data = input_data.read_data_sets("./mnist_data", one_hot=True)
-
-# Train
-device = "gpu:0" if tfe.num_gpus() else "cpu:0"
-model = MNISTModel('channels_first' if tfe.num_gpus() else 'channels_last')
-optimizer = tf.train.AdamOptimizer(learning_rate=1e-4)
-grad = tfe.implicit_gradients(loss)
-for i in range(20001):
-  with tf.device(device):
-    (inputs, targets) = data.train.next_batch(50)
-    optimizer.apply_gradients(grad(model, inputs, targets))
-    if i % 100 == 0:
-      print("Step %d: Loss on training set : %f" %
-            (i, loss(model, inputs, targets).numpy()))
-print("Loss on test set: %f" % loss(model, data.test.images, data.test.labels).numpy())
-```
-
-For a more complete example, see [the example in the tensorflow/models
-repository](https://github.com/tensorflow/models/tree/master/official/mnist/mnist_eager.py).
-
-### Checkpointing trained variables
-
-TensorFlow Variables (`tfe.Variable`) provides a way to represent shared,
-persistent state of your model. The `tfe.Saver` class (which is a thin wrapper
-over the
-[`tf.train.Saver`](https://www.tensorflow.org/api_docs/python/tf/train/Saver)
-class) provides a means to save and restore variables to and from _checkpoints_.
-
-For example:
-
-```python
-# Create variables.
-x = tfe.Variable(10., name='x')
-y = tfe.Variable(5., name='y')
-
-# Create a Saver.
-saver = tfe.Saver([x, y])
-
-# Assign new values to the variables and save.
-x.assign(2.)
-saver.save('/tmp/ckpt')
-
-# Change the variable after saving.
-x.assign(11.)
-assert 16. == (x + y).numpy()  # 11 + 5
-
-# Restore the values in the checkpoint.
-saver.restore('/tmp/ckpt')
-
-assert 7. == (x + y).numpy()  # 2 + 5
-```
-
-### `tfe.Network`
-
-You may often want to organize your models using classes, like the `MNISTModel`
-class described above. We recommend inheriting from the `tfe.Network` class as
-it provides conveniences like keeping track of all model variables and methods
-to save and restore from checkpoints.
-
-Sub-classes of `tfe.Network` may register `Layer`s (like classes in
-[`tf.layers`](https://www.tensorflow.org/api_docs/python/tf/layers),
-or [Keras
-layers](https://www.tensorflow.org/api_docs/python/tf/keras/layers))
-using a call to `self.track_layer()` and define the computation in an
-implementation of `call()`.
-
-Note that `tf.layers.Layer` objects (like `tf.layers.Dense`) create variables
-lazily, when the first input is encountered.
-
-For example, consider the following two-layer neural network:
-
-```python
-class TwoLayerNet(tfe.Network):
-  def __init__(self):
-    super(TwoLayerNet, self).__init__()
-    self.layer1 = self.track_layer(
-      tf.layers.Dense(2, activation=tf.nn.relu, use_bias=False))
-    self.layer2 = self.track_layer(tf.layers.Dense(3, use_bias=False))
-
-  def call(self, x):
-    return self.layer2(self.layer1(x))
-
-net = TwoLayerNet()
-
-# No variables created yet
-assert 0 == len(net.variables)
-
-# They are created on first input:
-inp = tf.constant([[1.]])
-
-# Since input is a 1x1 matrix, net.l1 has 2 units and net.l2 has 3 units,
-# the output is the product of a 1x1 matrix with a 1x2 matrix with a 2x3
-# matrix.
-assert [1, 3] == net(inp).shape.as_list()  # Invoke net; get output shape.
-assert 1 == len(net.layer1.variables)
-assert 1 == len(net.layer2.variables)
-assert 2 == len(net.variables)  # weights for each layer.
-assert [1, 2] == net.variables[0].shape.as_list()  # weights of layer1.
-assert [2, 3] == net.variables[1].shape.as_list()  # weights of layer2.
-```
-
-The `tfe.Network` class is itself a sub-class of `tf.layers.Layer`. This allows
-instances of `tfe.Network` to be embedded in other networks. For example:
-
-```python
-class ThreeLayerNet(tfe.Network):
-  def __init__(self):
-    super(ThreeLayerNet, self).__init__()
-    self.a = self.track_layer(TwoLayerNet())
-    self.b = self.track_layer(tf.layers.Dense(4, use_bias=False))
-
-  def call(self, x):
-    return self.b(self.a(x))
-
-net = ThreeLayerNet()
-
-assert [1, 4] == net(inp).shape.as_list()
-assert 3 == len(net.variables)
-assert [1, 2] == net.variables[0].shape.as_list()
-assert [2, 3] == net.variables[1].shape.as_list()
-assert [3, 4] == net.variables[2].shape.as_list()
-```
-
-See more examples in
-[`tensorflow/contrib/eager/python/examples`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples).
-
-`tfe.Saver` in combination with `tfe.restore_variables_on_create` provides a
-convenient way to save and load checkpoints without changing the program once
-the checkpoint has been created. For example, we can set an objective for the
-output of our network, choose an optimizer, and a location for the checkpoint:
-
-```python
-objective = tf.constant([[2., 3., 4., 5.]])
-optimizer = tf.train.AdamOptimizer(0.01)
-checkpoint_directory = '/tmp/tfe_example'
-checkpoint_prefix = os.path.join(checkpoint_directory, 'ckpt')
-net = ThreeLayerNet()
-```
-
-Note that variables have not been created yet. We want them to be restored from
-a checkpoint, if one exists, so we create them inside a
-`tfe.restore_variables_on_create` context manager. Then our training loop is the
-same whether starting training or resuming from a previous checkpoint:
-
-```python
-with tfe.restore_variables_on_create(
-    tf.train.latest_checkpoint(checkpoint_directory)):
-  global_step = tf.train.get_or_create_global_step()
-  for _ in range(100):
-    loss_fn = lambda: tf.norm(net(inp) - objective)
-    optimizer.minimize(loss_fn, global_step=global_step)
-    if tf.equal(global_step % 20, 0):
-      print("Step %d, output %s" % (global_step.numpy(),
-                                    net(inp).numpy()))
-      all_variables = (
-          net.variables
-          + optimizer.variables()
-          + [global_step])
-      # Save the checkpoint.
-      tfe.Saver(all_variables).save(checkpoint_prefix, global_step=global_step)
-```
-
-The first time it runs, `Network` variables are initialized randomly. Then the
-output is trained to match the objective we've set:
-
-```
-Step 20, output [[ 0.03575622  0.29863232  0.03474367  0.24735749]]
-Step 40, output [[ 0.40646029  0.9856872   0.46851286  0.95358551]]
-Step 60, output [[ 1.74541104  2.800704    1.79055595  2.74783421]]
-Step 80, output [[ 2.14977384  3.44340849  3.96120024  5.16242075]]
-Step 100, output [[ 1.99943113  3.02364397  3.93500996  4.9610076 ]]
-```
-
-In subsequent iterations, variables are initialized with the values read from
-the latest checkpoint. Running the same code again, we continue from where we
-left off:
-
-```
-Step 120, output [[ 1.99234128  3.0271616   3.98732996  4.96401167]]
-Step 140, output [[ 2.00133467  3.01270437  4.00616646  5.00406504]]
-Step 160, output [[ 1.99647415  2.9956708   3.99064088  4.99632359]]
-Step 180, output [[ 2.00699997  3.00904822  4.00706148  5.01193142]]
-Step 200, output [[ 1.98334622  2.98249531  3.97375059  4.97123432]]
-```
-
-
-### Summaries, metrics and TensorBoard
-
-[TensorBoard](https://www.tensorflow.org/get_started/summaries_and_tensorboard)
-is a popular tool for understanding, debugging and optimizing the model training
-process. To benefit from the visualizations offered by TensorBoard, summary
-events need to be written during the course of execution of your program. You
-might find many Tensorflow programs that include the
-[`tf.summary`](https://www.tensorflow.org/api_guides/python/summary) operations
-during graph construction.
-
-`tf.summary` operations are *not* compatible with eager execution, but an
-equivalent alternative exists in
-[`tf.contrib.summary`](https://www.tensorflow.org/versions/master/api_docs/python/tf/contrib/summary)
-that is compatible with both eager execution and graph construction.
-
-During model construction simply insert summary operations like
-`tf.contrib.summary.scalar`. These operations do nothing by default, unless a
-summary writer is currently active and a writing policy is set.
-
-For example, to record summaries once every 100 global steps, use:
-
-```python
-tf.train.get_or_create_global_step()  # Ensuring the global step variable exists
-writer = tf.contrib.summary.create_file_writer(logdir)
-
-for _ in range(iterations):
-  with writer.as_default():
-    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
-      # your model code goes here
-      tf.contrib.summary.scalar('loss', loss)
-      # ...
-```
-
-See the full mnist example in
-[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
-for a full model using `tf.contrib.summary`.
-
-Similarly to summaries, the metrics in `tf.metrics` are currently not compatible
-with eager execution. We instead provide object-oriented metrics in the
-`tfe.metrics` package, which are compatible with graph construction as well.
-
-Metrics in the `tfe.metrics`, such as `tfe.metrics.Mean` and
-`tfe.Metrics.Accuracy`, all implement an intuitive object-oriented
-interface. Here's an example of how to use the `tfe.metrics.Mean` metric:
-
-```python
-# Metrics are objects, which can be created and destroyed.
-my_mean = tfe.metrics.Mean(name='my_mean')
-# While a metric is active, you can call it as a function to accumulate into its
-# internal state.
-my_mean(0.0)
-my_mean(10.0)
-# Once you've finished updating the metric, you can get its result. In this case
-# a simple average over all the calls to it. If a summary writer is active the
-# metric will write the appropriate summaries using the metric name.
-assert 5.0 == my_mean.result().numpy()
-```
-
-For a full example of a model using metrics for evaluation, see the mnist
-example in
-[`tensorflow/contrib/eager/python/examples/mnist`](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist).
-
-### Input Pipelines
-
-The discussion above has been centered around the computation executed by your
-model. The
-[`tf.data`](https://www.tensorflow.org/api_docs/python/tf/data)
-module provides APIs to build complex input pipelines from simple, reusable
-pieces.
-
-If you're familiar with constructing `tf.data.Dataset` objects when building
-TensorFlow graphs, the same API calls are used when eager execution is enabled.
-However, the process of iterating over elements of the dataset differs between
-eager execution and graph construction. When eager execution is enabled, the
-discussion on iterator creation using `make_one_shot_iterator()` and
-`get_next()` in the
-[Programmer's Guide](https://www.tensorflow.org/programmers_guide/datasets) is
-*not* applicable. Instead, a more Pythonic `Iterator` class is available.
-
-For example:
-
-```python
-# Create a source Dataset from in-memory numpy arrays.
-# For reading from files on disk, you may want to use other Dataset classes
-# like the TextLineDataset or the TFRecordDataset.
-dataset = tf.data.Dataset.from_tensor_slices([1, 2, 3, 4, 5, 6])
-
-# Apply transformations, shuffling, batching etc.
-dataset = dataset.map(tf.square).shuffle(2).batch(2)
-
-# Use tfe.Iterator to iterate over the dataset.
-for x in tfe.Iterator(dataset):
-  print(x)
-```
-
-Output:
-
-```
-tf.Tensor([4 9], shape=(2,), dtype=int32)
-tf.Tensor([16 25], shape=(2,), dtype=int32)
-tf.Tensor([36  1], shape=(2,), dtype=int32)
-```
-
-## Interoperating with Graphs
-
-Eager execution improves the process of model development in Python; however,
-because it is in its earliest stages, it does not yet support some features
-available to [TensorFlow
-graphs](https://www.tensorflow.org/get_started/get_started#the_computational_graph)
-that are desirable when deploying models in production. In particular, eager
-execution does not yet support distributed training, exporting models (to other
-[programming languages](https://www.tensorflow.org/api_docs/), [TensorFlow
-serving](https://www.tensorflow.org/serving/), and mobile applications), and
-various memory and computation optimizations that are applied to TensorFlow's
-dataflow graphs.
-
-That said, the APIs used to build modes are exactly the same whether executing
-eagerly or constructing graphs. This means that you can iteratively develop your
-model with eager execution enabled and later, if needed, use the same code to
-reap the benefits of representing models as computational graphs.
-
-For example, the same model definition used to construct a graph in
-[mnist.py`](https://github.com/tensorflow/models/tree/master/official/mnist/mnist.py)
-can be trained with eager execution enabled as in [`mnist_eager.py`](https://github.com/tensorflow/models/tree/master/official/mnist/mnist_eager.py).
-
-Other models in the [examples
-directory](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/)
-demonstrate this as well.
-
-Some differences worth noting:
-
--   There is no notion of a `tf.placeholder` or a `tf.Session` when eager
-    execution is enabled.
--   Many properties on the `tf.Tensor` object, like `tf.Tensor.name`,
-    `tf.Tensor.op`, `tf.Tensor.inputs` are not meaningful when eager execution
-    is enabled and their use will raise an `AttributeError`.
--   To use `tfe.implicit_gradients` in graph construction, variables must be
-    created with [`use_resource=True`] provided to
-    [`tf.get_variable()`](https://www.tensorflow.org/api_docs/python/tf/get_variable)
-    or
-    [`tf.variable_scope()`](https://www.tensorflow.org/api_docs/python/tf/variable_scope).
--   Some API calls (such as the functional-style `tf.layers.dense`,
-    `tf.layers.conv2d`) are not compatible with eager execution. Use of such
-    methods should raise an error indicating the alternative (e.g., the
-    `tf.layers.Dense` and `tf.layers.Conv2D` classes).
-
-## What next?
+immediately: concrete values are returned, instead of creating a computational
+graph that is executed later.
 
-Please give eager execution a spin. This feature is in early stages and is
-evolving, so we welcome your feedback via issues on GitHub (see [known
-issues](https://github.com/tensorflow/tensorflow/labels/comp:eager)).
+A user guide is available: https://www.tensorflow.org/programmers_guide/eager
+([source file](../../../../docs_src/programmers_guide/eager.md))
 
-You may want to browse through some sample code, including benchmarks for some:
+We welcome feedback through [GitHub issues](https://github.com/tensorflow/tensorflow/labels/comp:eager).
 
--   [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression)
--   [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
--   [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50)
--   [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot)
--   [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb)
+Sample code is available, including benchmarks for some:
 
+- [Linear Regression](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/linear_regression)
+- [MNIST handwritten digit classifier](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/mnist)
+- [ResNet50 image classification](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/resnet50)
+- [RNN to generate colors](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_colorbot)
+- [RNN language model](https://www.tensorflow.org/code/tensorflow/contrib/eager/python/examples/rnn_ptb)
diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
new file mode 100644
index 0000000000..9ae1e602f4
--- /dev/null
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -0,0 +1,992 @@
+# Eager Execution
+
+TensorFlow's eager execution is an imperative programming environment that
+evaluates operations immediately, without an extra graph-building step.
+Operations return concrete values instead of constructing a computational graph
+to run later. This makes it easy to get started with TensorFlow, debug models,
+reduce boilerplate code, and is fun! To follow along with this guide, run the
+code samples below in an interactive `python` interpreter.
+
+Eager execution supports most TensorFlow operations and GPU acceleration.
+Automatic differentiation uses a dynamically-constructed tape instead of a static
+graph to compute gradients. Eager execution is a flexible machine learning
+platform for research and experimentation that provides:
+
+* *An intuitive interface* —Structure your code naturally and use Python data
+  structures. Quickly iterate on small models and small data.
+* *Easier debugging* —Call ops directly to inspect running models and test
+  changes. Use standard Python debugging tools for immediate error reporting.
+* *Natural control flow* —Use Python control flow instead of graph control flow,
+  including support for dynamic models.
+
+For a collection of examples running in eager execution, see:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+Note: Some models may experience increased overhead with eager execution enabled.
+Performance improvements are ongoing, but please
+[file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a
+problem and share your benchmarks.
+
+## Setup and basic usage
+
+Install TensorFlow 1.7 to include the updates for eager execution:
+
+```
+$ pip install --pre --upgrade tensorflow
+```
+
+To start eager execution, add `tf.enable_eager_execution()` to the beginning of
+the program or console session. Do not add this operation to other modules that
+the program calls.
+
+```py
+from __future__ import absolute_import, division, print_function
+
+import tensorflow as tf
+
+tf.enable_eager_execution()
+```
+
+Now you can run TensorFlow operations and the results will return immediately:
+
+```py
+tf.executing_eagerly()        # => True
+
+x = [[2.]]
+m = tf.matmul(x, x)
+print("hello, {}".format(m))  # => "hello, [[4.]]"
+```
+
+Enabling eager execution changes how TensorFlow operations behave—now they
+immediately evaluate and return their values to Python. `tf.Tensor` objects
+reference concrete values instead of symbolic handles to nodes in a computational
+graph. Since there isn't a computational graph to build and run later in a
+session, it's easy to inspect results using `print()` or a debugger. Evaluating,
+printing, and checking tensor values does not break the flow for computing
+gradients.
+
+Eager execution works nicely with [NumPy](http://www.numpy.org/). NumPy
+operations accept `tf.Tensor` arguments. TensorFlow
+[math operations](https://www.tensorflow.org/api_guides/python/math_ops) convert
+Python objects and NumPy arrays to `tf.Tensor` objects. The
+`tf.Tensor.numpy` method returns the object's value as a NumPy `ndarray`.
+
+```py
+a = tf.constant([[1, 2],
+                 [3, 4]])
+print(a)
+# => tf.Tensor([[1 2]
+#               [3 4]], shape=(2, 2), dtype=int32)
+
+# Broadcasting support
+b = tf.add(a, 1)
+print(b)
+# => tf.Tensor([[2 3]
+#               [4 5]], shape=(2, 2), dtype=int32)
+
+# Operator overloading is supported
+print(a * b)
+# => tf.Tensor([[ 2  6]
+#               [12 20]], shape=(2, 2), dtype=int32)
+
+# Use NumPy values
+import numpy as np
+
+c = np.multiply(a, b)
+print(c)
+# => [[ 2  6]
+#     [12 20]]
+
+# Obtain numpy value from a tensor:
+print(a.numpy())
+# => [[1 2]
+#     [3 4]]
+```
+
+The `tfe` module contains symbols available to both eager and graph execution
+environments and is useful for writing code to [work with graphs](#work_with_graphs):
+
+```py
+import tensorflow.contrib.eager as tfe
+```
+
+## Eager training
+
+### Automatic differentiation
+
+[Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)
+is useful for implementing machine learning algorithms such as
+[backpropagation](https://en.wikipedia.org/wiki/Backpropagation) for training
+neural networks. During eager execution, use `tfe.GradientTape` to trace
+operations for computing gradients later.
+
+`tfe.GradientTape` is an opt-in feature to provide maximal performance when
+not tracing. Since different operations can occur during each call, all
+forward-pass operations get recorded to a "tape". To compute the gradient, play
+the tape backwards and then discard. A particular `tfe.GradientTape` can only
+be computed once, subsequent calls throw a runtime error.
+
+```py
+w = tfe.Variable([[1.0]])
+with tfe.GradientTape() as tape:
+  loss = w * w
+
+grad = tape.gradient(loss, [w])
+print(grad)  # => [tf.Tensor([[ 2.]], shape=(1, 1), dtype=float32)]
+```
+
+Here's an example of `tfe.GradientTape` that records forward-pass operations
+to train a simple model:
+
+```py
+# A toy dataset of points around 3 * x + 2
+NUM_EXAMPLES = 1000
+training_inputs = tf.random_normal([NUM_EXAMPLES])
+noise = tf.random_normal([NUM_EXAMPLES])
+training_outputs = training_inputs * 3 + 2 + noise
+
+def prediction(input, weight, bias):
+  return input * weight + bias
+
+# A loss function using mean-squared error
+def loss(weights, biases):
+  error = prediction(training_inputs, weights, biases) - training_outputs
+  return tf.reduce_mean(tf.square(error))
+
+# Return the derivative of loss with respect to weight and bias
+def grad(weights, biases):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(weights, biases) 
+  return tape.gradient(loss_value, [weights, biases])
+
+train_steps = 200
+learning_rate = 0.01
+# Start with arbitrary values for W and B on the same batch of data
+W = tfe.Variable(5.)
+B = tfe.Variable(10.)
+
+print("Initial loss: {:.3f}".format(loss(W, B)))
+
+for i in range(train_steps):
+  dW, dB = grad(W, B)
+  W.assign_sub(dW * learning_rate)
+  B.assign_sub(dB * learning_rate)
+  if i % 20 == 0:
+    print("Loss at step {:03d}: {:.3f}".format(i, loss(W, B)))
+
+print("Final loss: {:.3f}".format(loss(W, B)))
+print("W = {}, B = {}".format(W.numpy(), B.numpy()))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 71.204
+Loss at step 000: 68.333
+Loss at step 020: 30.222
+Loss at step 040: 13.691
+Loss at step 060: 6.508
+Loss at step 080: 3.382
+Loss at step 100: 2.018
+Loss at step 120: 1.422
+Loss at step 140: 1.161
+Loss at step 160: 1.046
+Loss at step 180: 0.996
+Final loss: 0.974
+W = 3.01582956314, B = 2.1191945076
+```
+
+Replay the `tfe.GradientTape` to compute the gradients and apply them in a
+training loop. This is demonstrated in an excerpt from the
+[mnist_eager.py](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_eager.py)
+example:
+
+```py
+dataset = tf.data.Dataset.from_tensor_slices((data.train.images,
+                                              data.train.labels))
+...
+for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)):
+  ...
+  with tfe.GradientTape() as tape:
+    logits = model(images, training=True)
+    loss_value = loss(logits, labels)
+  ...
+  grads = tape.gradient(loss_value, model.variables)
+  optimizer.apply_gradients(zip(grads, model.variables),
+                            global_step=tf.train.get_or_create_global_step())
+```
+
+#### Dynamic models
+
+`tfe.GradientTape` can also be used in dynamic models. This example for a
+[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search)
+algorithm looks like normal NumPy code, except there are gradients and is
+differentiable, despite the complex control flow:
+
+```py
+def line_search_step(fn, init_x, rate=1.0):
+  with tfe.GradientTape() as tape:
+    # Variables are automatically recorded, but manually watch a tensor
+    tape.watch(init_x)
+    value = fn(init_x)
+  grad, = tape.gradient(value, [init_x])
+  grad_norm = tf.reduce_sum(grad * grad)
+  init_value = value
+  while value > init_value - rate * grad_norm:
+    x = init_x - rate * grad
+    value = fn(x)
+    rate /= 2.0
+  return x, value
+```
+
+#### Additional functions to compute gradients
+
+`tfe.GradientTape` is a powerful interface for computing gradients, but there
+is another [Autograd](https://github.com/HIPS/autograd)-style API available for
+automatic differentiation. These functions are useful if writing math code with
+only tensors and gradient functions, and without `tfe.Variables`:
+
+* `tfe.gradients_function` —Returns a function that computes the derivatives
+  of its input function parameter with respect to its arguments. The input
+  function parameter must return a scalar value. When the returned function is
+  invoked, it returns a list of `tf.Tensor` objects: one element for each
+  argument of the input function. Since anything of interest must be passed as a
+  function parameter, this becomes unwieldy if there's a dependency on many
+  trainable parameters.
+* `tfe.value_and_gradients_function` —Similar to
+  `tfe.gradients_function`, but when the returned function is invoked, it
+  returns the value from the input function in addition to the list of
+  derivatives of the input function with respect to its arguments.
+
+In the following example, `tfe.gradients_function` takes the `square`
+function as an argument and returns a function that computes the partial
+derivatives of `square` with respect to its inputs. To calculate the derivative
+of `square` at `3`, `grad(3.0)` returns `6`.
+
+```py
+def square(x):
+  return tf.multiply(x, x)
+
+grad = tfe.gradients_function(square)
+
+square(3.)  # => 9.0
+grad(3.)    # => [6.0]
+
+# The second-order derivative of square:
+gradgrad = tfe.gradients_function(lambda x: grad(x)[0])
+gradgrad(3.)  # => [2.0]
+
+# The third-order derivative is None:
+gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])
+gradgradgrad(3.)  # => [None]
+
+
+# With flow control:
+def abs(x):
+  return x if x > 0. else -x
+
+grad = tfe.gradients_function(abs)
+
+grad(3.)   # => [1.0]
+grad(-3.)  # => [-1.0]
+```
+
+### Custom gradients
+
+Custom gradients are an easy way to override gradients in eager and graph
+execution. Within the forward function, define the gradient with respect to the
+inputs, outputs, or intermediate results. For example, here's an easy way to clip
+the norm of the gradients in the backward pass:
+
+```py
+@tf.custom_gradient
+def clip_gradient_by_norm(x, norm):
+  y = tf.identity(x)
+  def grad_fn(dresult):
+    return [tf.clip_by_norm(dresult, norm), None]
+  return y, grad_fn
+```
+
+Custom gradients are commonly used to provide a numerically stable gradient for a
+sequence of operations:
+
+```py
+def log1pexp(x):
+  return tf.log(1 + tf.exp(x))
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# The gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# However, x = 100 fails because of numerical instability.
+grad_log1pexp(100.)  # => [nan]
+```
+
+
+Here, the `log1pexp` function can be analytically simplified with a custom
+gradient. The implementation below reuses the value for `tf.exp(x)` that is
+computed during the forward pass—making it more efficient by eliminating
+redundant calculations:
+
+```py
+@tfe.custom_gradient
+def log1pexp(x):
+  e = tf.exp(x)
+  def grad(dy):
+    return dy * (1 - 1 / (1 + e))
+  return tf.log(1 + e), grad
+
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# As before, the gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# And the gradient computation also works at x = 100.
+grad_log1pexp(100.)  # => [1.0]
+```
+
+
+## Build and train models
+
+There are many parameters to optimize when calculating derivatives. TensorFlow
+code is easier to read when structured into reusable classes and objects instead
+of a single top-level function. Eager execution encourages the use of the
+Keras-style layer classes in the `tf.keras.layers` module. Additionally, the
+`tf.train.Optimizer` classes provide sophisticated techniques to calculate
+parameter updates.
+
+The following example creates a multi-layer model that classifies the standard
+[MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It
+demonstrates the optimizer and layer APIs to build trainable graphs in an eager
+execution environment.
+
+### Build a model
+
+The `tf.keras.Sequential` model is a linear stack of layers. It is easy to
+use for basic models:
+
+```py
+model = tf.keras.Sequential([
+  tf.keras.layers.Dense(10, input_shape=(784,)),  # must declare input shape
+  tf.keras.layers.Dense(10)
+])
+```
+
+Alternatively, organize models in classes by inheriting from `tf.keras.Model`.
+This is a container for layers that is a layer itself, allowing `tf.keras.Model`
+objects to contain other `tf.keras.Model` objects.
+
+```py
+class MNISTModel(tf.keras.Model):
+  def __init__(self):
+    super(MNISTModel, self).__init__()
+    self.dense1 = tf.keras.layers.Dense(units=10)
+    self.dense2 = tf.keras.layers.Dense(units=10)
+
+  def call(self, input):
+    """Run the model."""
+    result = self.dense1(input)
+    result = self.dense2(result)
+    result = self.dense2(result)  # reuse variables from dense2 layer
+    return result
+
+model = MNISTModel()
+```
+
+It's not required to set an input shape for the `tf.keras.Model` class since
+the parameters are set the first time input is passed to the layer.
+
+`tf.keras.layers` classes create and contain their own model variables that
+are tied to the lifetime of their layer objects. To share layer variables, share
+their objects.
+
+### Train a model
+
+Even without training, call the model and inspect the output in eager execution:
+
+```py
+# Create a tensor representing a blank image
+batch = tf.zeros([1, 1, 784])
+print(batch.shape)  # => (1, 1, 784)
+
+result = model(batch)
+# => tf.Tensor([[[ 0.  0., ..., 0.]]], shape=(1, 1, 10), dtype=float32)
+```
+
+This example uses the
+[dataset.py module](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py)
+from the
+[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist),
+download this file to your local directory. Run the following to download the
+MNIST data files to your working directory and prepare a `tf.data.Dataset`
+for training:
+
+```py
+import dataset  # download dataset.py file
+dataset_train = dataset.train('./datasets').shuffle(60000).repeat(4).batch(32)
+```
+
+To train a model, define a loss function to optimize and then calculate
+gradients. Use an optimizer to update the variables:
+
+```py
+def loss(model, x, y):
+  prediction = model(x)
+  return tf.losses.sparse_softmax_cross_entropy(labels=y, logits=prediction)
+
+def grad(model, inputs, targets):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(model, inputs, targets)
+  return tape.gradient(loss_value, model.variables)
+
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.001)
+
+x, y = tfe.Iterator(dataset_train).next()
+print("Initial loss: {:.3f}".format(loss(model, x, y)))
+
+# Training loop
+for (i, (x, y)) in enumerate(tfe.Iterator(dataset_train)):
+  # Calculate derivatives of the input function with respect to its parameters.
+  grads = grad(model, x, y)
+  # Apply the gradient to the model
+  optimizer.apply_gradients(zip(grads, model.variables),
+                            global_step=tf.train.get_or_create_global_step())
+  if i % 200 == 0:
+    print("Loss at step {:04d}: {:.3f}".format(i, loss(model, x, y)))
+
+print("Final loss: {:.3f}".format(loss(model, x, y)))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 2.674
+Loss at step 0000: 2.593
+Loss at step 0200: 2.143
+Loss at step 0400: 2.009
+Loss at step 0600: 2.103
+Loss at step 0800: 1.621
+Loss at step 1000: 1.695
+...
+Loss at step 6600: 0.602
+Loss at step 6800: 0.557
+Loss at step 7000: 0.499
+Loss at step 7200: 0.744
+Loss at step 7400: 0.681
+Final loss: 0.670
+```
+
+And for faster training, move the computation to a GPU:
+
+```py
+with tf.device("/gpu:0"):
+  for (i, (x, y)) in enumerate(tfe.Iterator(dataset_train)):
+    # minimize() is equivalent to the grad() and apply_gradients() calls.
+    optimizer.minimize(lambda: loss(model, x, y),
+                       global_step=tf.train.get_or_create_global_step())
+```
+
+### Variables and optimizers
+
+`tfe.Variable` objects store mutable `tf.Tensor` values accessed during
+training to make automatic differentiation easier. The parameters of a model can
+be encapsulated in classes as variables.
+
+Better encapsulate model parameters by using `tfe.Variable` with
+`tfe.GradientTape`. For example, the automatic differentiation example above
+can be rewritten:
+
+```py
+class Model(tf.keras.Model):
+  def __init__(self):
+    super(Model, self).__init__()
+    self.W = tfe.Variable(5., name='weight')
+    self.B = tfe.Variable(10., name='bias')
+  def predict(self, inputs):
+    return inputs * self.W + self.B
+
+# A toy dataset of points around 3 * x + 2
+NUM_EXAMPLES = 2000
+training_inputs = tf.random_normal([NUM_EXAMPLES])
+noise = tf.random_normal([NUM_EXAMPLES])
+training_outputs = training_inputs * 3 + 2 + noise
+
+# The loss function to be optimized
+def loss(model, inputs, targets):
+  error = model.predict(inputs) - targets
+  return tf.reduce_mean(tf.square(error))
+
+def grad(model, inputs, targets):
+  with tfe.GradientTape() as tape:
+    loss_value = loss(model, inputs, targets)
+  return tape.gradient(loss_value, [model.W, model.B])
+
+# Define:
+# 1. A model.
+# 2. Derivatives of a loss function with respect to model parameters.
+# 3. A strategy for updating the variables based on the derivatives.
+model = Model()
+optimizer = tf.train.GradientDescentOptimizer(learning_rate=0.01)
+
+print("Initial loss: {:.3f}".format(loss(model, training_inputs, training_outputs)))
+
+# Training loop
+for i in range(300):
+  grads = grad(model, training_inputs, training_outputs)
+  optimizer.apply_gradients(zip(grads, [model.W, model.B]),
+                            global_step=tf.train.get_or_create_global_step())
+  if i % 20 == 0:
+    print("Loss at step {:03d}: {:.3f}".format(i, loss(model, training_inputs, training_outputs)))
+
+print("Final loss: {:.3f}".format(loss(model, training_inputs, training_outputs)))
+print("W = {}, B = {}".format(model.W.numpy(), model.B.numpy()))
+```
+
+Output (exact numbers may vary):
+
+```
+Initial loss: 69.066
+Loss at step 000: 66.368
+Loss at step 020: 30.107
+Loss at step 040: 13.959
+Loss at step 060: 6.769
+Loss at step 080: 3.567
+Loss at step 100: 2.141
+Loss at step 120: 1.506
+Loss at step 140: 1.223
+Loss at step 160: 1.097
+Loss at step 180: 1.041
+Loss at step 200: 1.016
+Loss at step 220: 1.005
+Loss at step 240: 1.000
+Loss at step 260: 0.998
+Loss at step 280: 0.997
+Final loss: 0.996
+W = 2.99431324005, B = 2.02129220963
+```
+
+## Use objects for state during eager execution
+
+With graph execution, program state (such as the variables) is stored in global
+collections and their lifetime is managed by the `tf.Session` object. In
+contrast, during eager execution the lifetime of state objects is determined by
+the lifetime of their corresponding Python object.
+
+### Variables are objects
+
+During eager execution, variables persist until the last reference to the object
+is removed, and is then deleted.
+
+```py
+with tf.device("gpu:0"):
+  v = tfe.Variable(tf.random_normal([1000, 1000]))
+  v = None  # v no longer takes up GPU memory
+```
+
+### Object-based saving
+
+`tfe.Checkpoint` can save and restore `tfe.Variable`s to and from
+checkpoints:
+
+```py
+x = tfe.Variable(10.)
+
+checkpoint = tfe.Checkpoint(x=x)  # save as "x"
+
+x.assign(2.)   # Assign a new value to the variables and save.
+save_path = checkpoint.save('./ckpt/')
+
+x.assign(11.)  # Change the variable after saving.
+
+# Restore values from the checkpoint
+checkpoint.restore(save_path)
+
+print(x)  # => 2.0
+```
+
+To save and load models, `tfe.Checkpoint` stores the internal state of objects,
+without requiring hiiden variables. To record the state of a `model`,
+an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`:
+
+```py
+model = MyModel()
+optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
+checkpoint_dir = ‘/path/to/model_dir’
+checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
+root = tfe.Checkpoint(optimizer=optimizer,
+                      model=model,
+                      optimizer_step=tf.train.get_or_create_global_step())
+
+root.save(file_prefix=checkpoint_prefix)
+# or
+root.restore(tf.train.latest_checkpoint(checkpoint_dir))
+```
+
+### Object-oriented metrics
+
+`tfe.metrics` are stored as objects. Update a metric by passing the new data to
+the callable, and retrieve the result using the `tfe.metrics.result` method,
+for example:
+
+```py
+m = tfe.metrics.Mean("loss")
+m(0)
+m(5)
+m.result()  # => 2.5
+m([8, 9])
+m.result()  # => 5.5
+```
+
+#### Summaries and TensorBoard
+
+@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
+understanding, debugging and optimizing the model training process. It uses
+summary events that are written while executing the program.
+
+`tf.contrib.summary` is compatible with both eager and graph execution
+environments. Summary operations, such as `tf.contrib.summary.scalar`, are
+inserted during model construction. For example, to record summaries once every
+100 global steps:
+
+```py
+tf.train.get_or_create_global_step()  # return global step var
+writer = tf.contrib.summary.create_file_writer(logdir)
+global_step=tf.train.get_or_create_global_step()
+
+writer.set_as_default()
+
+for _ in range(iterations):
+  global_step.assign_add(1)
+  # Must include a record_summaries method
+  with tf.contrib.summary.record_summaries_every_n_global_steps(100):
+    # your model code goes here
+    tf.contrib.summary.scalar('loss', loss)
+     ...
+```
+
+## Performance
+
+Computation is not automatically offloaded to GPUs during eager execution. To
+explicitly direct a computation to a GPU, enclose it in a
+`tf.device('/gpu:0')` block:
+
+```py
+import time
+
+def measure(x, steps):
+  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
+  tf.matmul(x, x)
+  start = time.time()
+  for i in range(steps):
+    x = tf.matmul(x, x)
+    _ = x.numpy()  # Make sure to execute op and not just enqueue it
+  end = time.time()
+  return end - start
+
+shape = (1000, 1000)
+steps = 200
+print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
+
+# Run on CPU:
+with tf.device("/cpu:0"):
+  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+
+# Run on GPU, if available:
+if tfe.num_gpus() > 0:
+  with tf.device("/gpu:0"):
+    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+else:
+  print("GPU: not found")
+```
+
+Output (exact numbers depend on hardware):
+
+```
+Time to multiply a (1000, 1000) matrix by itself 200 times:
+CPU: 4.614904403686523 secs
+GPU: 0.5581181049346924 secs
+```
+
+A `tf.Tensor` object can be copied to a different device to execute its
+operations:
+
+```py
+x = tf.random_normal([10, 10])
+
+x_gpu0 = x.gpu()
+x_cpu = x.cpu()
+
+_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
+_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
+
+if tfe.num_gpus() > 1:
+  x_gpu1 = x.gpu(1)
+  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
+```
+
+### Benchmarks
+
+For compute-heavy models, such as
+[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
+training on a GPU, eager execution performance is comparable to graph execution.
+But this gap grows larger for models with less computation and there is work to
+be done for optimizing hot code paths for models with lots of small operations.
+
+
+## Work with graphs
+
+While eager execution makes development and debugging more interactive,
+TensorFlow graph execution has advantages for distributed training, performance
+optimizations, and production deployment. However, writing graph code can feel
+different than writing regular Python code and more difficult to debug.
+
+For building and training graph-constructed models, the Python program first
+builds a graph representing the computation, then invokes `Session.run` to send
+the graph for execution on the C++-based runtime.  This provides:
+
+* Automatic differentiation using static autodiff.
+* Simple deployment to a platform independent server.
+* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
+* Compilation and kernel fusion.
+* Automatic distribution and replication (placing nodes on the distributed system).
+
+Deploying code written for eager execution is more difficult: either generate a
+graph from the model, or run the Python runtime and code directly on the server.
+
+### Write compatible code
+
+The same code written for eager execution will also build a graph during graph
+execution. Do this by simply running the same code in a new Python session where
+eager execution is not enabled.
+
+Most TensorFlow operations work during eager execution, but there are some things
+to keep in mind:
+
+* Use `tf.data` for input processing instead of queues. It's faster and easier.
+* Use object-oriented layer APIs—like `tf.keras.layers` and
+  `tf.keras.Model`—since they have explicit storage for variables.
+* Most model code works the same during eager and graph execution, but there are
+  exceptions. (For example, dynamic models using Python control flow to change the
+  computation based on inputs.)
+* Once eager execution is enabled with `tf.enable_eager_execution`, it
+  cannot be turned off. Start a new Python session to return to graph execution.
+
+It's best to write code for both eager execution *and* graph execution. This
+gives you eager's interactive experimentation and debuggability with the
+distributed performance benefits of graph execution.
+
+Write, debug, and iterate in eager execution, then import the model graph for
+production deployment. Use `tfe.Checkpoint` to save and restore model
+variables, this allows movement between eager and graph execution environments.
+See the examples in:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+### Use eager execution in a graph environment
+
+Selectively enable eager execution in a TensorFlow graph environment using
+`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
+been called.
+
+```py
+def my_py_func(x):
+  x = tf.matmul(x, x)  # You can use tf ops
+  print(x)  # but it's eager!
+  return x
+
+with tf.Session() as sess:
+  x = tf.placeholder(dtype=tf.float32)
+  # Call eager function in graph!
+  pf = tfe.py_func(my_py_func, [x], tf.float32)
+  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
+```
+
+
+A `tfe.Checkpoint` stores the complete internal state of the objects passed to it. Nothing else is implicitly included. To record the state of a `model`, an `optimizer`, and a global step pass each one to the checkpoint's constructor:
+
+```py
+model = MyModel()
+optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
+checkpoint_dir = ‘/path/to/model_dir’
+checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
+root = tfe.Checkpoint(optimizer=optimizer,
+                      model=model,
+                      optimizer_step=tf.train.get_or_create_global_step())
+
+root.save(file_prefix=checkpoint_prefix)
+# or
+root.restore(tf.train.latest_checkpoint(checkpoint_dir))
+```
+
+### Object-oriented metrics
+
+`tfe.metrics` are stored as objects. Update a metric by passing the new data to
+the callable, and retrieve the result using the `tfe.metrics.result` method,
+for example:
+
+```py
+m = tfe.metrics.Mean("loss")
+m(0)
+m(5)
+m.result()  # => 2.5
+m([8, 9])
+m.result()  # => 5.5
+```
+
+#### Summaries and TensorBoard
+
+@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
+understanding, debugging and optimizing the model training process. It uses
+summary events that are written while executing the program.
+
+`tf.contrib.summary` is compatible with both eager and graph execution
+environments. Summary operations, such as `tf.contrib.summary.scalar`, are
+inserted during model construction. For example, to record summaries once every
+100 global steps:
+
+```py
+tf.train.get_or_create_global_step()  # return global step var
+writer = tf.contrib.summary.create_file_writer(logdir)
+
+for _ in range(iterations):
+  with writer.as_default():
+    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
+      # your model code goes here
+      tf.contrib.summary.scalar('loss', loss)
+      ...
+```
+
+## Performance
+
+Computation is not automatically offloaded to GPUs during eager execution. To
+explicitly direct a computation to a GPU, enclose it in a
+`tf.device('/gpu:0')` block:
+
+```py
+import time
+
+def measure(x, steps):
+  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
+  tf.matmul(x, x)
+  start = time.time()
+  for i in range(steps):
+    x = tf.matmul(x, x)
+    _ = x.numpy()  # Make sure to execute op and not just enqueue it
+  end = time.time()
+  return end - start
+
+shape = (1000, 1000)
+steps = 200
+print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
+
+# Run on CPU:
+with tf.device("/cpu:0"):
+  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+
+# Run on GPU, if available:
+if tfe.num_gpus() > 0:
+  with tf.device("/gpu:0"):
+    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
+else:
+  print("GPU: not found")
+```
+
+Output (exact numbers depend on hardware):
+
+```
+Time to multiply a (1000, 1000) matrix by itself 200 times:
+CPU: 4.614904403686523 secs
+GPU: 0.5581181049346924 secs
+```
+
+A `tf.Tensor` object can be copied to a different device to execute its
+operations:
+
+```py
+x = tf.random_normal([10, 10])
+
+x_gpu0 = x.gpu()
+x_cpu = x.cpu()
+
+_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
+_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
+
+if tfe.num_gpus() > 1:
+  x_gpu1 = x.gpu(1)
+  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
+```
+
+### Benchmarks
+
+For compute-heavy models, such as
+[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
+training on a GPU, eager execution performance is comparable to graph execution.
+But this gap grows larger for models with less computation and there is work to
+be done for optimizing hot code paths for models with lots of small operations.
+
+
+## Work with graphs
+
+While eager execution makes development and debugging more interactive,
+TensorFlow graph execution has advantages for distributed training, performance
+optimizations, and production deployment. However, writing graph code can feel
+different than writing regular Python code and more difficult to debug.
+
+For building and training graph-constructed models, the Python program first
+builds a graph representing the computation, then invokes `Session.run` to send
+the graph for execution on the C++-based runtime.  This provides:
+
+* Automatic differentiation using static autodiff.
+* Simple deployment to a platform independent server.
+* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
+* Compilation and kernel fusion.
+* Automatic distribution and replication (placing nodes on the distributed system).
+
+Deploying code written for eager execution is more difficult: either generate a
+graph from the model, or run the Python runtime and code directly on the server.
+
+### Write compatible code
+
+The same code written for eager execution will also build a graph during graph
+execution. Do this by simply running the same code in a new Python session where
+eager execution is not enabled.
+
+Most TensorFlow operations work during eager execution, but there are some things
+to keep in mind:
+
+* Use `tf.data` for input processing instead of queues. It's faster and easier.
+* Use object-oriented layer APIs—like `tf.keras.layers` and
+  `tf.keras.Model`—since they have explicit storage for variables.
+* Most model code works the same during eager and graph execution, but there are
+  exceptions. (For example, dynamic models using Python control flow to change the
+  computation based on inputs.)
+* Once eager execution is enabled with `tf.enable_eager_execution`, it
+  cannot be turned off. Start a new Python session to return to graph execution.
+
+It's best to write code for both eager execution *and* graph execution. This
+gives you eager's interactive experimentation and debuggability with the
+distributed performance benefits of graph execution.
+
+Write, debug, and iterate in eager execution, then import the model graph for
+production deployment. Use `tfe.Checkpoint` to save and restore model
+variables, this allows movement between eager and graph execution environments.
+See the examples in:
+[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
+
+### Use eager execution in a graph environment
+
+Selectively enable eager execution in a TensorFlow graph environment using
+`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
+been called.
+
+```py
+def my_py_func(x):
+  x = tf.matmul(x, x)  # You can use tf ops
+  print(x)  # but it's eager!
+  return x
+
+with tf.Session() as sess:
+  x = tf.placeholder(dtype=tf.float32)
+  # Call eager function in graph!
+  pf = tfe.py_func(my_py_func, [x], tf.float32)
+  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
+```
diff --git a/tensorflow/docs_src/programmers_guide/leftnav_files b/tensorflow/docs_src/programmers_guide/leftnav_files
index 3fe4cb2dda..7ac63bf2e0 100644
--- a/tensorflow/docs_src/programmers_guide/leftnav_files
+++ b/tensorflow/docs_src/programmers_guide/leftnav_files
@@ -1,8 +1,9 @@
 index.md
 
 ### High Level APIs
-estimators.md
+eager.md
 datasets.md
+estimators.md
 
 ### Low Level APIs
 low_level_intro.md
-- 
GitLab


From 02783a09b2e06d03a9d7356167458b3a87a5b64d Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Thu, 29 Mar 2018 16:01:20 -0700
Subject: [PATCH 744/960] Updated eager guide to use tensorflow 1.7. Code
 snippets still work.

PiperOrigin-RevId: 191001008
---
 .../docs_src/programmers_guide/eager.md       | 203 +-----------------
 1 file changed, 5 insertions(+), 198 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
index 9ae1e602f4..8db65737dc 100644
--- a/tensorflow/docs_src/programmers_guide/eager.md
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -29,10 +29,10 @@ problem and share your benchmarks.
 
 ## Setup and basic usage
 
-Install TensorFlow 1.7 to include the updates for eager execution:
+Upgrade to TensorFlow 1.7 to include updates for eager execution:
 
 ```
-$ pip install --pre --upgrade tensorflow
+$ pip install --upgrade tensorflow
 ```
 
 To start eager execution, add `tf.enable_eager_execution()` to the beginning of
@@ -322,14 +322,13 @@ grad_log1pexp(0.)  # => [0.5]
 grad_log1pexp(100.)  # => [nan]
 ```
 
-
 Here, the `log1pexp` function can be analytically simplified with a custom
 gradient. The implementation below reuses the value for `tf.exp(x)` that is
 computed during the forward pass—making it more efficient by eliminating
 redundant calculations:
 
 ```py
-@tfe.custom_gradient
+@tf.custom_gradient
 def log1pexp(x):
   e = tf.exp(x)
   def grad(dy):
@@ -605,7 +604,7 @@ print(x)  # => 2.0
 ```
 
 To save and load models, `tfe.Checkpoint` stores the internal state of objects,
-without requiring hiiden variables. To record the state of a `model`,
+without requiring hidden variables. To record the state of a `model`,
 an `optimizer`, and a global step, pass them to a `tfe.Checkpoint`:
 
 ```py
@@ -649,9 +648,8 @@ inserted during model construction. For example, to record summaries once every
 100 global steps:
 
 ```py
-tf.train.get_or_create_global_step()  # return global step var
 writer = tf.contrib.summary.create_file_writer(logdir)
-global_step=tf.train.get_or_create_global_step()
+global_step=tf.train.get_or_create_global_step()  # return global step var
 
 writer.set_as_default()
 
@@ -733,197 +731,6 @@ But this gap grows larger for models with less computation and there is work to
 be done for optimizing hot code paths for models with lots of small operations.
 
 
-## Work with graphs
-
-While eager execution makes development and debugging more interactive,
-TensorFlow graph execution has advantages for distributed training, performance
-optimizations, and production deployment. However, writing graph code can feel
-different than writing regular Python code and more difficult to debug.
-
-For building and training graph-constructed models, the Python program first
-builds a graph representing the computation, then invokes `Session.run` to send
-the graph for execution on the C++-based runtime.  This provides:
-
-* Automatic differentiation using static autodiff.
-* Simple deployment to a platform independent server.
-* Graph-based optimizations (common subexpression elimination, constant-folding, etc.).
-* Compilation and kernel fusion.
-* Automatic distribution and replication (placing nodes on the distributed system).
-
-Deploying code written for eager execution is more difficult: either generate a
-graph from the model, or run the Python runtime and code directly on the server.
-
-### Write compatible code
-
-The same code written for eager execution will also build a graph during graph
-execution. Do this by simply running the same code in a new Python session where
-eager execution is not enabled.
-
-Most TensorFlow operations work during eager execution, but there are some things
-to keep in mind:
-
-* Use `tf.data` for input processing instead of queues. It's faster and easier.
-* Use object-oriented layer APIs—like `tf.keras.layers` and
-  `tf.keras.Model`—since they have explicit storage for variables.
-* Most model code works the same during eager and graph execution, but there are
-  exceptions. (For example, dynamic models using Python control flow to change the
-  computation based on inputs.)
-* Once eager execution is enabled with `tf.enable_eager_execution`, it
-  cannot be turned off. Start a new Python session to return to graph execution.
-
-It's best to write code for both eager execution *and* graph execution. This
-gives you eager's interactive experimentation and debuggability with the
-distributed performance benefits of graph execution.
-
-Write, debug, and iterate in eager execution, then import the model graph for
-production deployment. Use `tfe.Checkpoint` to save and restore model
-variables, this allows movement between eager and graph execution environments.
-See the examples in:
-[tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
-
-### Use eager execution in a graph environment
-
-Selectively enable eager execution in a TensorFlow graph environment using
-`tfe.py_func`. This is used when `tf.enable_eager_execution()` has *not*
-been called.
-
-```py
-def my_py_func(x):
-  x = tf.matmul(x, x)  # You can use tf ops
-  print(x)  # but it's eager!
-  return x
-
-with tf.Session() as sess:
-  x = tf.placeholder(dtype=tf.float32)
-  # Call eager function in graph!
-  pf = tfe.py_func(my_py_func, [x], tf.float32)
-  sess.run(pf, feed_dict={x: [[2.0]]})  # [[4.0]]
-```
-
-
-A `tfe.Checkpoint` stores the complete internal state of the objects passed to it. Nothing else is implicitly included. To record the state of a `model`, an `optimizer`, and a global step pass each one to the checkpoint's constructor:
-
-```py
-model = MyModel()
-optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
-checkpoint_dir = ‘/path/to/model_dir’
-checkpoint_prefix = os.path.join(checkpoint_dir, "ckpt")
-root = tfe.Checkpoint(optimizer=optimizer,
-                      model=model,
-                      optimizer_step=tf.train.get_or_create_global_step())
-
-root.save(file_prefix=checkpoint_prefix)
-# or
-root.restore(tf.train.latest_checkpoint(checkpoint_dir))
-```
-
-### Object-oriented metrics
-
-`tfe.metrics` are stored as objects. Update a metric by passing the new data to
-the callable, and retrieve the result using the `tfe.metrics.result` method,
-for example:
-
-```py
-m = tfe.metrics.Mean("loss")
-m(0)
-m(5)
-m.result()  # => 2.5
-m([8, 9])
-m.result()  # => 5.5
-```
-
-#### Summaries and TensorBoard
-
-@{$summaries_and_tensorboard$TensorBoard} is a visualization tool for
-understanding, debugging and optimizing the model training process. It uses
-summary events that are written while executing the program.
-
-`tf.contrib.summary` is compatible with both eager and graph execution
-environments. Summary operations, such as `tf.contrib.summary.scalar`, are
-inserted during model construction. For example, to record summaries once every
-100 global steps:
-
-```py
-tf.train.get_or_create_global_step()  # return global step var
-writer = tf.contrib.summary.create_file_writer(logdir)
-
-for _ in range(iterations):
-  with writer.as_default():
-    with tf.contrib.summary.record_summaries_every_n_global_steps(100):
-      # your model code goes here
-      tf.contrib.summary.scalar('loss', loss)
-      ...
-```
-
-## Performance
-
-Computation is not automatically offloaded to GPUs during eager execution. To
-explicitly direct a computation to a GPU, enclose it in a
-`tf.device('/gpu:0')` block:
-
-```py
-import time
-
-def measure(x, steps):
-  # TensorFlow initializes a GPU the first time it's used, exclude from timing.
-  tf.matmul(x, x)
-  start = time.time()
-  for i in range(steps):
-    x = tf.matmul(x, x)
-    _ = x.numpy()  # Make sure to execute op and not just enqueue it
-  end = time.time()
-  return end - start
-
-shape = (1000, 1000)
-steps = 200
-print("Time to multiply a {} matrix by itself {} times:".format(shape, steps))
-
-# Run on CPU:
-with tf.device("/cpu:0"):
-  print("CPU: {} secs".format(measure(tf.random_normal(shape), steps)))
-
-# Run on GPU, if available:
-if tfe.num_gpus() > 0:
-  with tf.device("/gpu:0"):
-    print("GPU: {} secs".format(measure(tf.random_normal(shape), steps)))
-else:
-  print("GPU: not found")
-```
-
-Output (exact numbers depend on hardware):
-
-```
-Time to multiply a (1000, 1000) matrix by itself 200 times:
-CPU: 4.614904403686523 secs
-GPU: 0.5581181049346924 secs
-```
-
-A `tf.Tensor` object can be copied to a different device to execute its
-operations:
-
-```py
-x = tf.random_normal([10, 10])
-
-x_gpu0 = x.gpu()
-x_cpu = x.cpu()
-
-_ = tf.matmul(x_cpu, x_cpu)    # Runs on CPU
-_ = tf.matmul(x_gpu0, x_gpu0)  # Runs on GPU:0
-
-if tfe.num_gpus() > 1:
-  x_gpu1 = x.gpu(1)
-  _ = tf.matmul(x_gpu1, x_gpu1)  # Runs on GPU:1
-```
-
-### Benchmarks
-
-For compute-heavy models, such as
-[ResNet50](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples/resnet50)
-training on a GPU, eager execution performance is comparable to graph execution.
-But this gap grows larger for models with less computation and there is work to
-be done for optimizing hot code paths for models with lots of small operations.
-
-
 ## Work with graphs
 
 While eager execution makes development and debugging more interactive,
-- 
GitLab


From 857dfa321d3689a87af9b6bc64c488a444ae80e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 17:15:33 -0700
Subject: [PATCH 745/960] Add details of new mailing lists

PiperOrigin-RevId: 191011187
---
 tensorflow/docs_src/about/uses.md             |  6 ++--
 tensorflow/docs_src/community/contributing.md | 25 +++----------
 tensorflow/docs_src/community/index.md        |  6 +++-
 tensorflow/docs_src/community/leftnav_files   |  1 +
 tensorflow/docs_src/community/lists.md        | 32 ++++++++++++-----
 tensorflow/docs_src/community/swift.md        | 35 +++++++++++++++++++
 6 files changed, 73 insertions(+), 32 deletions(-)
 create mode 100644 tensorflow/docs_src/community/swift.md

diff --git a/tensorflow/docs_src/about/uses.md b/tensorflow/docs_src/about/uses.md
index d646880bd3..d3db98203e 100644
--- a/tensorflow/docs_src/about/uses.md
+++ b/tensorflow/docs_src/about/uses.md
@@ -18,9 +18,9 @@ This section describes some of the current uses of the TensorFlow system.
 
 > If you are using TensorFlow for research, for education, or for production
 > usage in some product, we would love to add something about your usage here.
-> Please feel free to email us a brief description of how you're using
-> TensorFlow, or even better, send us a pull request to add an entry to this
-> file.
+> Please feel free to [email us](mailto:usecases@tensorflow.org) a brief
+> description of how you're using TensorFlow, or even better, send us a
+> pull request to add an entry to this file.
 
 * **Deep Speech**
 <ul>
diff --git a/tensorflow/docs_src/community/contributing.md b/tensorflow/docs_src/community/contributing.md
index b0960df435..afbb8bbdd0 100644
--- a/tensorflow/docs_src/community/contributing.md
+++ b/tensorflow/docs_src/community/contributing.md
@@ -22,7 +22,7 @@ inside the organization, including:
 guidelines](https://github.com/tensorflow/tensorflow/blob/master/CONTRIBUTING.md).
 
 * Join the
-[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers)
+[developers@tensorflow.org](https://groups.google.com/a/tensorflow.org/d/forum/developers)
 mailing list, to coordinate and discuss with others contributing to TensorFlow.
 
 * For coding style conventions, read the @{$style_guide$TensorFlow Style Guide}.
@@ -37,28 +37,13 @@ You may also wish to review our guide to @{$benchmarks$defining and running benc
 To enable focused collaboration on particular areas of TensorFlow, we host
 Special Interest Groups (SIGs). SIGs do their work in public: if you want to
 join and contribute, review the work of the group, and get in touch with the
-relevant SIG leader.
+relevant SIG leader.  Membership policies vary on a per-SIG basis.
 
 * **SIG Build** focuses on issues surrounding building, packaging, and
-  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/build).
+  distribution of TensorFlow. [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/build).
 
 * **SIG TensorBoard** furthers the development and direction of TensorBoard and its plugins.
-  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard).
+  [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/sig-tensorboard).
 
 * **SIG Rust** collaborates on the development of TensorFlow's Rust bindings.
-  [Mailing list](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust). 
-
-## Projects developed by the TensorFlow community
-
-The TensorFlow community has created many great projects around TensorFlow, including:
-
-* [Machine Learning with TensorFlow (Book & Code)](http://tensorflowbook.com)
-* [@jtoy's awesome "Awesome TensorFlow" list of awesome things](https://github.com/jtoy/awesome-tensorflow)
-* [TensorFlow tutorials](https://github.com/pkmital/tensorflow_tutorials)
-* [Caffe to TensorFlow model converter](https://github.com/ethereon/caffe-tensorflow)
-* [Bitfusion's` GPU-enabled AWS EC2 TensorFlow AMI](https://github.com/bitfusionio/amis/tree/master/awsmrkt-bfboost-ubuntu14-cuda75-tensorflow) ([Launch AMI](https://aws.amazon.com/marketplace/pp/B01EYKBEQ0))
-* [Operator Vectorization Library](https://github.com/opveclib/opveclib)
-* [Swift language bindings](https://github.com/PerfectlySoft/Perfect-TensorFlow)
-* [Sublime Tensorflow - A plugin for Sublime Text](https://github.com/baptisteArnaud/Sublime-Tensorflow)
-* [GPflow - Gaussian processes in TensorFlow](https://github.com/GPflow/GPflow)
-* [CS 20SI: Tensorflow for Deep Learning Research](https://web.stanford.edu/class/cs20si/) - please note, this course was designed with TensorFlow v0.12, so some of the notes may be out of date - but it's still a great resource.
+  [Mailing list](https://groups.google.com/a/tensorflow.org/d/forum/rust).
diff --git a/tensorflow/docs_src/community/index.md b/tensorflow/docs_src/community/index.md
index c08aeb7a97..eec2e51a87 100644
--- a/tensorflow/docs_src/community/index.md
+++ b/tensorflow/docs_src/community/index.md
@@ -47,6 +47,11 @@ The @{$roadmap$Roadmap} summarizes plans for upcoming additions to TensorFlow.
 For news and updates from around the universe of TensorFlow projects, follow
 [@tensorflow](https://twitter.com/tensorflow) on Twitter.
 
+### Blog
+
+We post regularly to the [TensorFlow Blog](http://blog.tensorflow.org/),
+with content from the TensorFlow team and the best articles from the community.
+
 ### YouTube
 
 Our [YouTube Channel](http://youtube.com/tensorflow/) focuses on machine learing
@@ -56,7 +61,6 @@ and AI with TensorFlow. On it we have a number of new shows, including:
 - Ask TensorFlow: the team answers the best questions tagged #AskTensorFlow from social media 
 - Coding TensorFlow: short bites with tips for success with TensorFlow
 
-
 ## Community Support
 
 ### Mailing Lists
diff --git a/tensorflow/docs_src/community/leftnav_files b/tensorflow/docs_src/community/leftnav_files
index 0bd1f14de9..2bae60d9dd 100644
--- a/tensorflow/docs_src/community/leftnav_files
+++ b/tensorflow/docs_src/community/leftnav_files
@@ -6,3 +6,4 @@ groups.md
 documentation.md
 style_guide.md
 benchmarks.md
+swift.md
diff --git a/tensorflow/docs_src/community/lists.md b/tensorflow/docs_src/community/lists.md
index dc9240030e..7450ab36c4 100644
--- a/tensorflow/docs_src/community/lists.md
+++ b/tensorflow/docs_src/community/lists.md
@@ -9,27 +9,43 @@ see [how to get help](/community/#get_help).
 
 ## General TensorFlow lists
 
-* [announce](https://groups.google.com/a/tensorflow.org/forum/#!forum/announce) - Low-volume announcements of new releases.
-* [discuss](https://groups.google.com/a/tensorflow.org/forum/#!forum/discuss) - General community discussion around TensorFlow.
-* [developers](https://groups.google.com/a/tensorflow.org/forum/#!forum/developers) - Discussion for developers contributing to TensorFlow.
+* [announce](https://groups.google.com/a/tensorflow.org/d/forum/announce) - Low-volume announcements of new releases.
+* [discuss](https://groups.google.com/a/tensorflow.org/d/forum/discuss) - General community discussion around TensorFlow.
+* [developers](https://groups.google.com/a/tensorflow.org/d/forum/developers) - Discussion for developers contributing to TensorFlow.
 
 ## Project-specific lists
 
 These projects inside the TensorFlow GitHub organization have lists dedicated to their communities:
 
-* [tensor2tensor](https://groups.google.com/forum/#!forum/tensor2tensor) - User
+* [hub](https://groups.google.com/a/tensorflow.org/d/forum/hub) -
+  Discussion and collaboration around [TensorFlow Hub](https://github.com/tensorflow/hub).
+* [magenta-discuss](https://groups.google.com/a/tensorflow.org/d/forum/magenta-discuss) -
+  General discussion about [Magenta](https://magenta.tensorflow.org/)
+  development and directions.
+* [swift](https://groups.google.com/a/tensorflow.org/d/forum/swift) -
+  Community and collaboration around Swift for TensorFlow.
+* [tensor2tensor](https://groups.google.com/d/forum/tensor2tensor) - Discussion
   and peer support for Tensor2Tensor.
+* [tfjs-announce](https://groups.google.com/a/tensorflow.org/d/forum/tfjs-announce) -
+  Announcements of new TensorFlow.js releases.
+* [tfjs](https://groups.google.com/a/tensorflow.org/d/forum/tfjs) - Discussion
+  and peer support for TensorFlow.js.
+* [tflite](https://groups.google.com/a/tensorflow.org/d/forum/tflite) - Discussion and
+  peer support for TensorFlow Lite.
+* [tpu-users](https://groups.google.com/a/tensorflow.org/d/forum/tpu-users) - Community discussion
+  and support for TPU users.
 
 ## Special Interest Groups
 
 TensorFlow's [Special Interest
 Groups](/community/contributing#special_interest_groups) (SIGs) support
 community collaboration on particular project focuses. Members of these groups
-work together to build and support TensorFlow related projects.
+work together to build and support TensorFlow related projects. While their
+archives are public, different SIGs have their own membership policies.
 
-* [build](https://groups.google.com/a/tensorflow.org/forum/#!forum/build) -
+* [build](https://groups.google.com/a/tensorflow.org/d/forum/build) -
   Supporting SIG Build, for build, distribution and packaging of TensorFlow.
-* [tensorboard](https://groups.google.com/a/tensorflow.org/forum/#!forum/tensorboard) -
+* [sig-tensorboard](https://groups.google.com/a/tensorflow.org/d/forum/sig-tensorboard) -
   Supporting SIG TensorBoard, for plugin development and other contribution.
-* [rust](https://groups.google.com/a/tensorflow.org/forum/#!forum/rust) -
+* [rust](https://groups.google.com/a/tensorflow.org/d/forum/rust) -
   Supporting SIG Rust, for the Rust language bindings.
diff --git a/tensorflow/docs_src/community/swift.md b/tensorflow/docs_src/community/swift.md
new file mode 100644
index 0000000000..54d9960b23
--- /dev/null
+++ b/tensorflow/docs_src/community/swift.md
@@ -0,0 +1,35 @@
+# Swift Community
+
+Welcome to the Swift for TensorFlow development community!
+
+Swift for TensorFlow is a result of first-principles thinking applied to machine
+learning frameworks, and works quite differently than existing TensorFlow
+language bindings.  Whereas prior solutions are designed within the constraints
+of what can be achieved by a (typically Python or Lua) library, Swift for
+TensorFlow is based on the belief that machine learning is important enough to
+deserve first-class language and compiler support.
+
+First-class language and compiler support allows us to innovate in areas that
+have traditionally been out of bounds for machine learning libraries.  Our
+results provide the performance of TensorFlow graphs with the ease of use of
+define-by-run models, and provides a great user experience - for example, by
+catching more mistakes before you run your code.
+
+## Open Source
+
+As announced at the TensorFlow Developer Summit, we are planning to launch our
+open source project on GitHub in April.  In addition to releasing the code, we
+will be using an open design model, where design discussions happen in public.
+
+Between now and then, we are writing some technical white papers that explain in
+detail the design approach (e.g., the core compiler partitioning technique that
+underlies the whole thing, our approach to automatic differentiation, etc.),
+implementation tradeoffs, and the status of this work.  We can’t wait to engage
+with the broader community, but prefer to start the conversation when these
+white papers are ready.
+
+[Sign up here to join the community Google
+group](https://groups.google.com/a/tensorflow.org/d/forum/swift). We will
+initially use it for announcements, and then open it for general discussion when
+we are ready in April.
+
-- 
GitLab


From f77d1b63df9d62ba4bbadeb159b75c52ec9b4e31 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 29 Mar 2018 17:30:09 -0700
Subject: [PATCH 746/960] * Added a JavaScript tab to DEVELOP. * Added a
 JavaScript file to DEPLOY. * Added a link to JavaScript API Ref.

PiperOrigin-RevId: 191012951
---
 tensorflow/docs_src/deploy/deploy_to_js.md   | 4 ++++
 tensorflow/docs_src/deploy/leftnav_files     | 1 +
 tensorflow/docs_src/javascript/index.md      | 5 +++++
 tensorflow/docs_src/javascript/leftnav_files | 1 +
 4 files changed, 11 insertions(+)
 create mode 100644 tensorflow/docs_src/deploy/deploy_to_js.md
 create mode 100644 tensorflow/docs_src/javascript/index.md
 create mode 100644 tensorflow/docs_src/javascript/leftnav_files

diff --git a/tensorflow/docs_src/deploy/deploy_to_js.md b/tensorflow/docs_src/deploy/deploy_to_js.md
new file mode 100644
index 0000000000..d7ce3ea90b
--- /dev/null
+++ b/tensorflow/docs_src/deploy/deploy_to_js.md
@@ -0,0 +1,4 @@
+# Deploy to JavaScript
+
+You can find details about deploying JavaScript TensorFlow programs
+in the separate [js.tensorflow.org site](https://js.tensorflow.org).
diff --git a/tensorflow/docs_src/deploy/leftnav_files b/tensorflow/docs_src/deploy/leftnav_files
index c682e7add1..93f5bd1ed2 100644
--- a/tensorflow/docs_src/deploy/leftnav_files
+++ b/tensorflow/docs_src/deploy/leftnav_files
@@ -2,3 +2,4 @@ index.md
 distributed.md
 hadoop.md
 s3.md
+deploy_to_js.md
diff --git a/tensorflow/docs_src/javascript/index.md b/tensorflow/docs_src/javascript/index.md
new file mode 100644
index 0000000000..ad63eeb255
--- /dev/null
+++ b/tensorflow/docs_src/javascript/index.md
@@ -0,0 +1,5 @@
+# JavaScript 
+
+You may develop TensorFlow programs in JavaScript, training and deploying
+models right in your browser.  For details, see
+[js.tensorflow.org](https://js.tensorflow.org).
diff --git a/tensorflow/docs_src/javascript/leftnav_files b/tensorflow/docs_src/javascript/leftnav_files
new file mode 100644
index 0000000000..fc0ab8a543
--- /dev/null
+++ b/tensorflow/docs_src/javascript/leftnav_files
@@ -0,0 +1 @@
+index.md
-- 
GitLab


From 71ed936432ba96a3db1e8b742f58600243f9beb3 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 30 Mar 2018 00:18:18 -0700
Subject: [PATCH 747/960] add eager stub, fix titles and leftnav

---
 tensorflow/docs_src/get_started/eager.md                 | 4 ++++
 .../docs_src/get_started/get_started_for_beginners.md    | 2 +-
 tensorflow/docs_src/get_started/index.md                 | 9 +++++++--
 tensorflow/docs_src/get_started/leftnav_files            | 9 +++++----
 tensorflow/docs_src/get_started/premade_estimators.md    | 2 +-
 5 files changed, 18 insertions(+), 8 deletions(-)
 create mode 100644 tensorflow/docs_src/get_started/eager.md

diff --git a/tensorflow/docs_src/get_started/eager.md b/tensorflow/docs_src/get_started/eager.md
new file mode 100644
index 0000000000..3c60a08b40
--- /dev/null
+++ b/tensorflow/docs_src/get_started/eager.md
@@ -0,0 +1,4 @@
+# Get Started
+
+This file is a place-holder for this
+[Colab notebook.](https://colab.research.google.com/github/tensorflow/models/blob/master/samples/core/get_started/eager.ipynb)
diff --git a/tensorflow/docs_src/get_started/get_started_for_beginners.md b/tensorflow/docs_src/get_started/get_started_for_beginners.md
index b88483be69..1e61c9b05a 100644
--- a/tensorflow/docs_src/get_started/get_started_for_beginners.md
+++ b/tensorflow/docs_src/get_started/get_started_for_beginners.md
@@ -1,4 +1,4 @@
-# Getting Started for ML Beginners
+# Premade Estimators for ML Beginners
 
 This document explains how to use machine learning to classify (categorize)
 Iris flowers by species.  This document dives deeply into the TensorFlow
diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md
index b7bd1286e3..30a10696bf 100644
--- a/tensorflow/docs_src/get_started/index.md
+++ b/tensorflow/docs_src/get_started/index.md
@@ -3,8 +3,13 @@
 TensorFlow is a tool for machine learning. While it contains a wide range of
 functionality, TensorFlow is mainly designed for deep neural network models.
 
-TensorFlow provides many APIs. This section focuses on the high-level APIs.
-If you are new to TensorFlow, begin by reading one of the following documents:
+The easiest way to get started with tensorflow is using Eager Execution.
+
+  * @{$get_started/eager}, is for anyone new to  machine learning or TensorFlow.
+
+TensorFlow provides many APIs. The remainder of this section focuses on the
+Estimator API which provide scalable, high-performance models.
+To get started with Estimators begin by reading one of the following documents:
 
   * @{$get_started/get_started_for_beginners}, which is aimed at readers
     new to machine learning.
diff --git a/tensorflow/docs_src/get_started/leftnav_files b/tensorflow/docs_src/get_started/leftnav_files
index 437791d6a3..c63a9b4b63 100644
--- a/tensorflow/docs_src/get_started/leftnav_files
+++ b/tensorflow/docs_src/get_started/leftnav_files
@@ -1,10 +1,11 @@
 index.md
 
-### Getting Started
-get_started_for_beginners.md
-premade_estimators.md
+eager.md
 
-### Details
+### Estimators
+get_started_for_beginners.md: For Beginners
+premade_estimators.md: Premade Estimators
+>>>
 checkpoints.md
 feature_columns.md
 datasets_quickstart.md
diff --git a/tensorflow/docs_src/get_started/premade_estimators.md b/tensorflow/docs_src/get_started/premade_estimators.md
index 6bffd2e065..601f145df2 100644
--- a/tensorflow/docs_src/get_started/premade_estimators.md
+++ b/tensorflow/docs_src/get_started/premade_estimators.md
@@ -1,5 +1,5 @@
 
-# Getting Started with TensorFlow
+# Premade Estimators
 
 This document introduces the TensorFlow programming environment and shows you
 how to solve the Iris classification problem in TensorFlow.
-- 
GitLab


From 36b4744977d75b75bc969dfd25f8951f1141caa6 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 02:41:35 -0700
Subject: [PATCH 748/960] Before this change, a Tensor contained a device
 pointer and a TensorInfoManager datastructure contained a mapping from device
 pointer to XlaTensorInfo object. This TensorInfoManager needed to be an
 Allocator too, so it could be informed when a Tensor is released.

After this change, a Tensor on an XlaDevice contains an XlaTensor object. The XlaTensor object is the equivalent of the old XlaTensorInfo object.

This has advantages and drawbacks:
  + We don't need yet another allocator wrapper, as there is no side-band data to manage.
  + No hashtable lookups are required.
  - As XlaLocalLaunchOp could either be on an XlaDevice or a TF-classic device, we need some way to distinguish whether a Tensor is a TF-classic tensor (holds a device pointer) or an XlaTensor (we use a tagged pointer).

As part of this, allocate ShapedBuffers using the XLA backend's allocator directly instead of a roundabout route where we:
  Wrapped the XLA allocator in an XlaDeviceAllocator
  Then wrapped the XlaDeviceAllocator in an XlaAllocator

This leaves less to go wrong. Ideally we'd actually use StreamExecutor's allocator here, but this is less useful than XLA's as it doesn't provide helpful OOM messages (just returns nullptr).

PiperOrigin-RevId: 191048184
---
 tensorflow/compiler/jit/BUILD                 |  12 +-
 .../compiler/jit/kernels/xla_launch_op.cc     |  31 ++--
 .../compiler/jit/xla_compile_on_demand_op.cc  |  19 +--
 tensorflow/compiler/jit/xla_device.cc         |  28 +---
 tensorflow/compiler/jit/xla_device.h          |  14 +-
 tensorflow/compiler/jit/xla_device_context.cc |  61 ++++----
 tensorflow/compiler/jit/xla_device_context.h  |  24 ++-
 tensorflow/compiler/jit/xla_launch_util.cc    | 147 +++++++++---------
 tensorflow/compiler/jit/xla_launch_util.h     |  21 ++-
 tensorflow/compiler/jit/xla_tensor.cc         |  98 ++++++++++++
 .../jit/{xla_tensor_info.h => xla_tensor.h}   |  89 +++++------
 tensorflow/compiler/jit/xla_tensor_info.cc    |  56 -------
 12 files changed, 310 insertions(+), 290 deletions(-)
 create mode 100644 tensorflow/compiler/jit/xla_tensor.cc
 rename tensorflow/compiler/jit/{xla_tensor_info.h => xla_tensor.h} (50%)
 delete mode 100644 tensorflow/compiler/jit/xla_tensor_info.cc

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 9ea246ffdc..c2283bab3d 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -120,11 +120,13 @@ cc_library(
 )
 
 cc_library(
-    name = "xla_tensor_info",
-    srcs = ["xla_tensor_info.cc"],
-    hdrs = ["xla_tensor_info.h"],
+    name = "xla_tensor",
+    srcs = ["xla_tensor.cc"],
+    hdrs = ["xla_tensor.h"],
     deps = [
         ":common",
+        "//tensorflow/compiler/tf2xla:common",
+        "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/service:shaped_buffer",
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:core_cpu_internal",
@@ -154,7 +156,7 @@ cc_library(
         ":common",
         ":jit_compilation_passes",
         ":xla_launch_util",
-        ":xla_tensor_info",
+        ":xla_tensor",
         "//tensorflow/compiler/jit/ops:xla_ops",
         "//tensorflow/compiler/tf2xla:common",
         "//tensorflow/compiler/tf2xla:dump_graph",
@@ -201,7 +203,7 @@ cc_library(
     deps = [
         ":common",
         ":xla_compilation_cache",
-        ":xla_tensor_info",
+        ":xla_tensor",
         "//tensorflow/compiler/tf2xla:xla_compiler",
         "//tensorflow/compiler/xla:status_macros",
         "//tensorflow/compiler/xla:statusor",
diff --git a/tensorflow/compiler/jit/kernels/xla_launch_op.cc b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
index 8a8e8bb8df..2d6511a45b 100644
--- a/tensorflow/compiler/jit/kernels/xla_launch_op.cc
+++ b/tensorflow/compiler/jit/kernels/xla_launch_op.cc
@@ -116,11 +116,7 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
 
   const XlaDevice::Metadata* metadata;
   Status s = XlaDevice::GetMetadata(ctx, &metadata);
-
-  XlaTensorInfoManager* tensor_info_manager = nullptr;
-  if (s.ok()) {
-    tensor_info_manager = &metadata->tensor_info_manager();
-  }
+  bool allocate_xla_tensors = s.ok();
 
   // Get the platform_id_ for XLA_* devices.
   if (platform_id_ == nullptr) {
@@ -134,8 +130,23 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
 
   xla::LocalClient* client = static_cast<xla::LocalClient*>(cache->client());
 
-  // Builds an XLA allocator for the device.
-  XlaAllocator xla_allocator(client->platform(), ctx);
+  XlaAllocator local_xla_allocator(client->backend().platform(),
+                                   ctx->device()->GetAllocator({}));
+  xla::DeviceMemoryAllocator* xla_allocator;
+  // If we are on an XlaDevice, use the underlying XLA platform's allocator
+  // directly. We could use the StreamExecutor's allocator which may
+  // theoretically be more correct, but XLA returns a nice OOM message in a
+  // Status and StreamExecutor does not.
+  //
+  // Importantly we can't use ctx->device()->GetAllocator() as the allocator
+  // (which local_xla_allocator above uses) as on an XlaDevice, this is a
+  // dummy allocator that returns XlaTensor objects. The XlaCompiler needs a
+  // real allocator to allocate real buffers.
+  if (allocate_xla_tensors) {
+    xla_allocator = client->backend().memory_allocator();
+  } else {
+    xla_allocator = &local_xla_allocator;
+  }
 
   XlaCompiler::Options options;
   options.client = client;
@@ -143,7 +154,7 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
   options.flib_def = ctx->function_library()->GetFunctionLibraryDefinition();
   options.graph_def_version = ctx->function_library()->graph_def_version();
   options.allow_cpu_custom_calls = (platform_id_ == gpu::host::kHostPlatformId);
-  options.device_allocator = &xla_allocator;
+  options.device_allocator = xla_allocator;
 
   const XlaCompiler::CompilationResult* kernel;
   xla::LocalExecutable* executable;
@@ -159,14 +170,14 @@ void XlaLocalLaunchOp::Compute(OpKernelContext* ctx) {
   VLOG(1) << "Executing XLA Computation...";
 
   XlaComputationLaunchContext launch_context(
-      num_resource_args_, client, &xla_allocator, tensor_info_manager);
+      num_resource_args_, client, xla_allocator, allocate_xla_tensors);
   launch_context.PopulateInputs(ctx, kernel, variables);
 
   // Execute the computation.
   VLOG(2) << "Executing computation.";
   xla::ExecutableRunOptions run_options;
   run_options.set_stream(stream);
-  run_options.set_allocator(&xla_allocator);
+  run_options.set_allocator(xla_allocator);
   run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device());
   Env* env = Env::Default();
   auto start_time = env->NowMicros();
diff --git a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
index 915b9ce84a..682d6ea8cc 100644
--- a/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
+++ b/tensorflow/compiler/jit/xla_compile_on_demand_op.cc
@@ -51,12 +51,10 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx,
   int64 num_resource_args = variables.size();
 
   xla::LocalClient* client = metadata.client();
-  XlaTensorInfoManager* tensor_info_manager = &metadata.tensor_info_manager();
 
   // Builds an XLA allocator for the device.
-  XlaAllocator xla_allocator(client->platform(), ctx);
   XlaComputationLaunchContext launch_context(
-      num_resource_args, client, &xla_allocator, tensor_info_manager);
+      num_resource_args, client, client->backend().memory_allocator(), true);
 
   launch_context.PopulateInputs(ctx, result, variables);
 
@@ -67,7 +65,7 @@ Status XlaCompileOnDemandOp::Run(OpKernelContext* ctx,
   VLOG(2) << "Executing computation.";
   xla::ExecutableRunOptions run_options;
   run_options.set_stream(stream);
-  run_options.set_allocator(&xla_allocator);
+  run_options.set_allocator(client->backend().memory_allocator());
   run_options.set_intra_op_thread_pool(&ctx->eigen_cpu_device());
 
   auto run_result = executable->Run(launch_context.arguments(), run_options);
@@ -106,16 +104,13 @@ Status XlaCompileOnDemandOp::Compile(
     OpKernelContext* ctx, const XlaDevice::Metadata& metadata,
     const XlaCompiler::CompilationResult** result,
     xla::LocalExecutable** executable) {
-  XlaTensorInfoManager* tensor_info_manager = &metadata.tensor_info_manager();
-
   std::map<int, Tensor> constant_arguments;
   for (int64 i = 0; i < ctx->num_inputs(); ++i) {
     const Tensor& device_tensor = ctx->input(i);
-    if (const XlaTensorInfo* tensor_info =
-            tensor_info_manager->GetTensorInfo(device_tensor)) {
-      if (tensor_info->has_host_tensor() &&
+    if (const XlaTensor* xla_tensor = XlaTensor::FromTensor(&device_tensor)) {
+      if (xla_tensor->has_host_tensor() &&
           ShouldArgumentBeConstant(&ctx->op_kernel(), i)) {
-        constant_arguments[i] = tensor_info->host_tensor();
+        constant_arguments[i] = xla_tensor->host_tensor();
       }
     }
     if (constant_arguments.count(i) == 0 &&
@@ -123,8 +118,10 @@ Status XlaCompileOnDemandOp::Compile(
       // Slow path; the argument is not available as a host constant so we must
       // fetch it synchronously.
       Tensor host_tensor;
+      AllocatorAttributes attrs;
+      attrs.set_on_host(true);
       TF_RETURN_IF_ERROR(ctx->allocate_temp(
-          device_tensor.dtype(), device_tensor.shape(), &host_tensor));
+          device_tensor.dtype(), device_tensor.shape(), &host_tensor, attrs));
       Notification n;
       ctx->op_device_context()->CopyDeviceTensorToCPU(
           &device_tensor, "ConstantArgument",
diff --git a/tensorflow/compiler/jit/xla_device.cc b/tensorflow/compiler/jit/xla_device.cc
index 82048f5d78..12f471735f 100644
--- a/tensorflow/compiler/jit/xla_device.cc
+++ b/tensorflow/compiler/jit/xla_device.cc
@@ -100,7 +100,7 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
   }
 
   std::unique_ptr<XlaDeviceAllocator> alloc =
-      xla::MakeUnique<XlaDeviceAllocator>(backend, device_ordinal);
+      xla::MakeUnique<XlaDeviceAllocator>();
   XlaDeviceAllocator* alloc_ptr = alloc.get();
   state.allocators_[{backend, device_ordinal}] = std::move(alloc);
   return alloc_ptr;
@@ -136,13 +136,11 @@ XlaDeviceAllocator* XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
   return Status::OK();
 }
 
-XlaDevice::Metadata::Metadata(
-    int device_ordinal, se::Platform* platform, const DeviceType& device_type,
-    std::unique_ptr<XlaTensorInfoManager>* tensor_info_manager)
+XlaDevice::Metadata::Metadata(int device_ordinal, se::Platform* platform,
+                              const DeviceType& device_type)
     : device_ordinal_(device_ordinal),
       device_type_(device_type),
-      platform_(platform),
-      tensor_info_manager_(*tensor_info_manager) {}
+      platform_(platform) {}
 
 int XlaDevice::Metadata::device_ordinal() const { return device_ordinal_; }
 
@@ -157,12 +155,9 @@ const DeviceType& XlaDevice::Metadata::jit_device_type() const {
   return device_type_;
 }
 
-XlaTensorInfoManager& XlaDevice::Metadata::tensor_info_manager() const {
-  return *tensor_info_manager_;
-}
-
 /* static */ Status XlaDevice::GetMetadata(OpKernelContext* ctx,
                                            const Metadata** metadata) {
+  *metadata = nullptr;
   XlaDevice* xla_device =
       dynamic_cast<XlaDevice*>(ctx->device()->UnderlyingDevice());
   if (xla_device == nullptr) {
@@ -181,15 +176,11 @@ XlaDevice::XlaDevice(const SessionOptions& options,
                      const DeviceType& jit_device_name, se::Platform* platform,
                      bool transfer_as_literal)
     : LocalDevice(options, attrs),
-      xla_metadata_(
-          device_ordinal, platform, jit_device_name,
-          // Pass tensor_info_manager_ by reference as it is initialized lazily.
-          &tensor_info_manager_),
+      xla_metadata_(device_ordinal, platform, jit_device_name),
       device_ordinal_(device_ordinal),
       jit_device_name_(jit_device_name),
       xla_allocator_(nullptr),
       platform_(platform),
-      tensor_info_manager_(nullptr),
       transfer_as_literal_(transfer_as_literal) {}
 
 XlaDevice::~XlaDevice() {}
@@ -215,7 +206,6 @@ Allocator* XlaDevice::GetAllocator(AllocatorAttributes attr) {
     xla::Backend* backend = client()->mutable_backend();
     xla_allocator_ = XlaDeviceAllocatorState::GetOrCreateXlaDeviceAllocator(
         backend, device_ordinal_);
-    tensor_info_manager_.reset(new XlaTensorInfoManager(xla_allocator_));
   }
   return xla_allocator_;
 }
@@ -236,8 +226,7 @@ Status XlaDevice::FillContextMap(const Graph* graph,
   // Call GetAllocator for the side-effect of ensuring the allocator and
   // XlaTensorInfoManager is created.
   (void)GetAllocator({});
-  auto ctx = new XlaDeviceContext(stream, tensor_info_manager_.get(),
-                                  transfer_as_literal_);
+  auto ctx = new XlaDeviceContext(stream, client(), transfer_as_literal_);
   for (Node* n : graph->nodes()) {
     VLOG(2) << n->id() << " : " << n->type_string() << " : " << n->name();
     ctx->Ref();
@@ -285,8 +274,7 @@ Status XlaDevice::MakeTensorFromProto(const TensorProto& tensor_proto,
     Tensor copy(GetAllocator(alloc_attrs), parsed.dtype(), parsed.shape());
     Notification n;
     TF_ASSIGN_OR_RETURN(se::Stream * stream, GetStream());
-    XlaTransferManager manager(stream, tensor_info_manager_.get(),
-                               transfer_as_literal_);
+    XlaTransferManager manager(stream, client(), transfer_as_literal_);
     manager.CopyCPUTensorToDevice(&parsed, this, &copy,
                                   [&n, &status](const Status& s) {
                                     status = s;
diff --git a/tensorflow/compiler/jit/xla_device.h b/tensorflow/compiler/jit/xla_device.h
index 9cd9167e52..4fe7dd8c9f 100644
--- a/tensorflow/compiler/jit/xla_device.h
+++ b/tensorflow/compiler/jit/xla_device.h
@@ -26,7 +26,7 @@ limitations under the License.
 #ifndef TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_
 #define TENSORFLOW_COMPILER_JIT_XLA_DEVICE_H_
 
-#include "tensorflow/compiler/jit/xla_tensor_info.h"
+#include "tensorflow/compiler/jit/xla_tensor.h"
 #include "tensorflow/compiler/tf2xla/xla_op_registry.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/common_runtime/device_factory.h"
@@ -50,8 +50,7 @@ class XlaDevice : public LocalDevice {
   class Metadata {
    public:
     Metadata(int device_ordinal, perftools::gputools::Platform* platform,
-             const DeviceType& device_type,
-             std::unique_ptr<XlaTensorInfoManager>* tensor_info_manager);
+             const DeviceType& device_type);
 
     // The index of the device on this host.
     int device_ordinal() const;
@@ -59,13 +58,11 @@ class XlaDevice : public LocalDevice {
     perftools::gputools::Platform* platform() const;
     xla::LocalClient* client() const;
     const DeviceType& jit_device_type() const;
-    XlaTensorInfoManager& tensor_info_manager() const;
 
    private:
     const int device_ordinal_;
     const DeviceType device_type_;
     perftools::gputools::Platform* platform_;  // Not owned.
-    std::unique_ptr<XlaTensorInfoManager>& tensor_info_manager_;
 
     TF_DISALLOW_COPY_AND_ASSIGN(Metadata);
   };
@@ -123,13 +120,6 @@ class XlaDevice : public LocalDevice {
   // copying back and forth between CPU and the device, and
   // computations enqueued by XLA.
   xla::Backend::StreamPtr stream_;
-  // Manages sideband data about tensors, in particular the on-device shape tree
-  // if the tensor requires multiple device buffers to represent (for example,
-  // tuple shapes).
-  // This is a unique_ptr because XlaTensorInfoManager is non-copy-constructible
-  // and we need to initialize this lazily (as we also lazily initialize the
-  // underlying allocator).
-  std::unique_ptr<XlaTensorInfoManager> tensor_info_manager_;
   // Must we use XLA's transfer manager for correct host<->device transfers? if
   // false, we can use ThenMemcpy() instead.
   bool transfer_as_literal_;
diff --git a/tensorflow/compiler/jit/xla_device_context.cc b/tensorflow/compiler/jit/xla_device_context.cc
index 88f7c15f0b..6a57831cde 100644
--- a/tensorflow/compiler/jit/xla_device_context.cc
+++ b/tensorflow/compiler/jit/xla_device_context.cc
@@ -27,37 +27,32 @@ namespace se = ::perftools::gputools;
 namespace tensorflow {
 
 // The allocator used for Tensors assigned to the XLA device.
-XlaDeviceAllocator::XlaDeviceAllocator(const xla::Backend* backend,
-                                       int device_ordinal)
-    : backend_(backend), device_ordinal_(device_ordinal) {}
-
+XlaDeviceAllocator::XlaDeviceAllocator() {}
 XlaDeviceAllocator::~XlaDeviceAllocator() = default;
 
 string XlaDeviceAllocator::Name() { return "xla"; }
 
 void* XlaDeviceAllocator::AllocateRaw(size_t alignment, size_t num_bytes) {
-  se::DeviceMemoryBase dmem =
-      backend_->memory_allocator()
-          ->Allocate(device_ordinal_, num_bytes, /*retry_on_failure=*/false)
-          .ValueOrDie();
-  VLOG(2) << "Allocated XLA device tensor " << dmem.opaque() << "(" << num_bytes
-          << ")";
-  return dmem.opaque();
+  // We always return an empty XlaTensor object, encoded as an opaque tagged
+  // pointer. We can return an empty object and ignore num_bytes here because we
+  // have control over all of the uses of this device tensor, and can lazily
+  // allocate memory when used. This allows us to also know the shape of the
+  // allocated Tensor, which is useful if the device's tensor representation
+  // differs from the host.
+  return XlaTensor::ToOpaquePointer(new XlaTensor());
 }
 
 void XlaDeviceAllocator::DeallocateRaw(void* ptr) {
-  se::DeviceMemoryBase dmem(ptr);
-  TF_CHECK_OK(backend_->memory_allocator()->Deallocate(device_ordinal_, &dmem));
-  VLOG(2) << "Deallocated XLA device tensor " << ptr;
+  delete XlaTensor::FromOpaquePointer(ptr);
 }
 
 void XlaDeviceAllocator::GetStats(AllocatorStats* stats) { stats->Clear(); }
 
-XlaTransferManager::XlaTransferManager(
-    se::Stream* stream, XlaTensorInfoManager* tensor_info_manager,
-    bool transfer_as_literal)
+XlaTransferManager::XlaTransferManager(se::Stream* stream,
+                                       xla::LocalClient* client,
+                                       bool transfer_as_literal)
     : stream_(stream),
-      tensor_info_manager_(tensor_info_manager),
+      client_(client),
       transfer_as_literal_(transfer_as_literal) {}
 
 void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
@@ -74,9 +69,21 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
 
     void* src_ptr = const_cast<void*>(DMAHelper::base(cpu_tensor));
     const int64 total_bytes = cpu_tensor->TotalBytes();
-    void* dst_ptr = DMAHelper::base(device_tensor);
-    se::DeviceMemoryBase dev_dst_ptr(dst_ptr, total_bytes);
 
+    XlaTensor* xla_tensor = XlaTensor::FromTensor(device_tensor);
+    CHECK(xla_tensor);
+    if (!xla_tensor->has_shaped_buffer()) {
+      Status s = xla_tensor->AllocateShapedBuffer(
+          device_tensor->dtype(), device_tensor->shape(), client_,
+          stream_->parent()->device_ordinal());
+      if (!s.ok()) {
+        done(s);
+        return;
+      }
+    }
+
+    se::DeviceMemoryBase dev_dst_ptr =
+        XlaTensor::DeviceMemoryFromTensor(*device_tensor);
     Status status;
     if (transfer_as_literal_) {
       status = xla::Unimplemented(
@@ -92,10 +99,7 @@ void XlaTransferManager::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
             block_status.error_message().c_str());
       }
     }
-
-    XlaTensorInfo* tensor_info =
-        tensor_info_manager_->GetOrCreateTensorInfo(*device_tensor);
-    tensor_info->set_host_tensor(*cpu_tensor);
+    xla_tensor->set_host_tensor(*cpu_tensor);
 
     done(status);
     return;
@@ -119,8 +123,8 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor,
             << device_tensor->NumElements();
 
     const int64 total_bytes = cpu_tensor->TotalBytes();
-    void* src_ptr = const_cast<void*>(DMAHelper::base(device_tensor));
-    se::DeviceMemoryBase dev_src_ptr(src_ptr, total_bytes);
+    se::DeviceMemoryBase dev_src_ptr =
+        XlaTensor::DeviceMemoryFromTensor(*device_tensor);
     void* dst_ptr = DMAHelper::base(cpu_tensor);
 
     Status status;
@@ -147,10 +151,9 @@ void XlaTransferManager::CopyDeviceTensorToCPU(const Tensor* device_tensor,
   done(Status::OK());
 }
 
-XlaDeviceContext::XlaDeviceContext(se::Stream* stream,
-                                   XlaTensorInfoManager* tensor_info_manager,
+XlaDeviceContext::XlaDeviceContext(se::Stream* stream, xla::LocalClient* client,
                                    bool transfer_as_literal)
-    : manager_(stream, tensor_info_manager, transfer_as_literal) {}
+    : manager_(stream, client, transfer_as_literal) {}
 
 void XlaDeviceContext::CopyCPUTensorToDevice(const Tensor* cpu_tensor,
                                              Device* device,
diff --git a/tensorflow/compiler/jit/xla_device_context.h b/tensorflow/compiler/jit/xla_device_context.h
index df02f4eac4..a8ad511fbd 100644
--- a/tensorflow/compiler/jit/xla_device_context.h
+++ b/tensorflow/compiler/jit/xla_device_context.h
@@ -18,7 +18,7 @@ limitations under the License.
 
 #include <memory>
 
-#include "tensorflow/compiler/jit/xla_tensor_info.h"
+#include "tensorflow/compiler/jit/xla_tensor.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/framework/allocator.h"
@@ -27,11 +27,12 @@ limitations under the License.
 
 namespace tensorflow {
 
-// The allocator used for Tensors assigned to the XLA device. It uses
-// XLA backend's allocator.
+// The allocator used for Tensors assigned to the XLA device. The allocator
+// ignores the alignment and size of the request and always returns a new,
+// empty, XlaTensor.
 class XlaDeviceAllocator : public Allocator {
  public:
-  XlaDeviceAllocator(const xla::Backend* backend, int device_ordinal);
+  XlaDeviceAllocator();
   ~XlaDeviceAllocator() override;
 
   string Name() override;
@@ -39,19 +40,13 @@ class XlaDeviceAllocator : public Allocator {
   void* AllocateRaw(size_t alignment, size_t num_bytes) override;
   void DeallocateRaw(void* ptr) override;
   void GetStats(AllocatorStats* stats) override;
-
- private:
-  // Which backend in the client this allocator belongs to.
-  const xla::Backend* backend_;
-  // Which hardware device in the client's backend this allocator belongs to.
-  const int device_ordinal_;
 };
 
 // Helper class for managing data transfers between host and XLA devices.
 class XlaTransferManager {
  public:
   explicit XlaTransferManager(perftools::gputools::Stream* stream,
-                              XlaTensorInfoManager* tensor_info_manager,
+                              xla::LocalClient* client,
                               bool transfer_as_literal);
 
   void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device,
@@ -65,8 +60,8 @@ class XlaTransferManager {
   // Stream obtained from a Device, used to transfer tensors between
   // CPU and device.
   perftools::gputools::Stream* stream_;
-  // The tensor info manager, for access to sideband information about tensors.
-  XlaTensorInfoManager* tensor_info_manager_;
+  // For the underlying memory allocator and XLA's TransferManager.
+  xla::LocalClient* client_;
   // True if we must use XLA's TransferManager for correct device transfers.
   bool transfer_as_literal_;
 };
@@ -77,8 +72,7 @@ class XlaTransferManager {
 class XlaDeviceContext : public DeviceContext {
  public:
   explicit XlaDeviceContext(perftools::gputools::Stream* stream,
-                            XlaTensorInfoManager* tensor_info_manager,
-                            bool transfer_as_literal);
+                            xla::LocalClient* client, bool transfer_as_literal);
 
   void CopyCPUTensorToDevice(const Tensor* cpu_tensor, Device* device,
                              Tensor* device_tensor,
diff --git a/tensorflow/compiler/jit/xla_launch_util.cc b/tensorflow/compiler/jit/xla_launch_util.cc
index bb7316c60c..354be1e1b5 100644
--- a/tensorflow/compiler/jit/xla_launch_util.cc
+++ b/tensorflow/compiler/jit/xla_launch_util.cc
@@ -52,60 +52,66 @@ std::map<int, OptionalTensor> SnapshotResourceVariables(OpKernelContext* ctx,
   return snapshot;
 }
 
-XlaAllocator::XlaAllocator(const gpu::Platform* platform,
-                           OpKernelContext* op_context)
-    : xla::DeviceMemoryAllocator(platform), op_context_(op_context) {}
+XlaAllocator::XlaAllocator(const gpu::Platform* platform, Allocator* wrapped)
+    : xla::DeviceMemoryAllocator(platform), wrapped_(wrapped) {}
 
-XlaAllocator::~XlaAllocator() { CHECK(allocated_.empty()); }
+XlaAllocator::~XlaAllocator() {}
 
 xla::StatusOr<gpu::DeviceMemoryBase> XlaAllocator::Allocate(
     int device_ordinal, uint64 size, bool retry_on_failure) {
-  void* data = op_context_->device()->GetAllocator({})->AllocateRaw(
-      Allocator::kAllocatorAlignment, size);
-  allocated_.insert(data);
-  return gpu::DeviceMemoryBase(data, size);
+  void* data = wrapped_->AllocateRaw(Allocator::kAllocatorAlignment, size);
+  if (data == nullptr) {
+    return errors::ResourceExhausted("Out of memory while trying to allocate ",
+                                     size, " bytes.");
+  } else {
+    return gpu::DeviceMemoryBase(data, size);
+  }
 }
 
-void XlaAllocator::Release(void* ptr) { allocated_.erase(ptr); }
-
 Status XlaAllocator::Deallocate(int device_ordinal,
                                 gpu::DeviceMemoryBase* mem) {
-  if (allocated_.count(mem->opaque())) {
-    op_context_->device()->GetAllocator({})->DeallocateRaw(mem->opaque());
-    allocated_.erase(mem->opaque());
-  }
+  wrapped_->DeallocateRaw(mem->opaque());
   return Status::OK();
 }
 
 namespace {
-// Return the 'index''th subtree of the given ShapedBuffer as a ShapedBuffer.
-xla::ShapedBuffer ExtractSubShapedBuffer(const xla::ShapedBuffer& shaped_buffer,
-                                         int index) {
+// Return the 'index''th subtree of the given ShapedBuffer as a
+// ScopedShapedBuffer. The returned ScopedShapedBuffer takes ownership of the
+// subtree, and sets the input's buffer pointers to nullptr for the subtree.
+std::unique_ptr<xla::ScopedShapedBuffer> ExtractSubShapedBuffer(
+    xla::ShapedBuffer* shaped_buffer, int index,
+    xla::DeviceMemoryAllocator* allocator) {
   xla::Shape on_host_shape = xla::ShapeUtil::GetTupleElementShape(
-      shaped_buffer.on_host_shape(), index);
+      shaped_buffer->on_host_shape(), index);
   xla::Shape on_device_shape = xla::ShapeUtil::GetTupleElementShape(
-      shaped_buffer.on_device_shape(), index);
+      shaped_buffer->on_device_shape(), index);
 
   xla::ShapedBuffer sub_shaped_buffer(on_host_shape, on_device_shape,
-                                      shaped_buffer.platform(),
-                                      shaped_buffer.device_ordinal());
+                                      shaped_buffer->platform(),
+                                      shaped_buffer->device_ordinal());
 
-  auto& shape_tree = shaped_buffer.buffers();
+  auto& shape_tree = shaped_buffer->buffers();
   auto& sub_shape_tree = sub_shaped_buffer.buffers();
   sub_shape_tree.CopySubtreeFrom(shape_tree,
                                  /*source_base_index=*/{index},
                                  /*target_base_index=*/{});
-  return sub_shaped_buffer;
+  for (auto& index_to_buffer : shape_tree) {
+    if (!index_to_buffer.first.empty() && index_to_buffer.first[0] == index) {
+      index_to_buffer.second = gpu::DeviceMemoryBase(nullptr, 0);
+    }
+  }
+  return xla::ScopedShapedBuffer::MakeScoped(&sub_shaped_buffer, allocator)
+      .ValueOrDie();
 }
 }  // namespace
 
 XlaComputationLaunchContext::XlaComputationLaunchContext(
     int64 num_resource_args, xla::LocalClient* client,
-    XlaAllocator* xla_allocator, XlaTensorInfoManager* tensor_info_manager)
+    xla::DeviceMemoryAllocator* xla_allocator, bool allocate_xla_tensors)
     : num_resource_args_(num_resource_args),
       client_(client),
       xla_allocator_(xla_allocator),
-      tensor_info_manager_(tensor_info_manager) {}
+      allocate_xla_tensors_(allocate_xla_tensors) {}
 
 void XlaComputationLaunchContext::PopulateInputs(
     OpKernelContext* ctx, const XlaCompiler::CompilationResult* kernel,
@@ -130,20 +136,17 @@ void XlaComputationLaunchContext::PopulateInputs(
     const xla::Shape on_device_shape =
         client_->backend().transfer_manager()->HostShapeToDeviceShape(shape);
     if (xla::ShapeUtil::IsTuple(on_device_shape)) {
-      CHECK(tensor_info_manager_);
-      const XlaTensorInfo* tensor_info =
-          tensor_info_manager_->GetTensorInfo(*t);
-      CHECK(tensor_info && tensor_info->has_shaped_buffer());
+      const XlaTensor* xla_tensor = XlaTensor::FromTensor(t);
+      CHECK(xla_tensor && xla_tensor->has_shaped_buffer());
       arg_ptrs_[i] =
-          const_cast<xla::ShapedBuffer*>(&tensor_info->shaped_buffer());
+          const_cast<xla::ShapedBuffer*>(&xla_tensor->shaped_buffer());
     } else {
       CHECK(xla::ShapeUtil::Equal(shape, on_device_shape))
           << "On-device shape "
           << xla::ShapeUtil::HumanStringWithLayout(on_device_shape)
           << " not the same as on-host shape "
           << xla::ShapeUtil::HumanStringWithLayout(shape);
-      gpu::DeviceMemoryBase dmem = gpu::DeviceMemoryBase(
-          const_cast<char*>(t->tensor_data().data()), t->tensor_data().size());
+      gpu::DeviceMemoryBase dmem = XlaTensor::DeviceMemoryFromTensor(*t);
       arg_buffers_[i] = xla::MakeUnique<xla::ShapedBuffer>(
           /*on_host_shape=*/shape, /*on_device_shape=*/shape,
           client_->platform(), client_->default_device_ordinal());
@@ -168,11 +171,7 @@ void XlaComputationLaunchContext::PopulateOutputs(
   // Copy XLA results to the OpOutputList.
   int output_num = 0;
   for (int i = 0; i < ctx->num_outputs(); ++i) {
-    AllocatorAttributes alloc_attrs = ctx->output_alloc_attr(i);
     Allocator* allocator = ctx->device()->GetAllocator({});
-    if (tensor_info_manager_ && !alloc_attrs.on_host()) {
-      allocator = tensor_info_manager_;
-    }
     if (kernel->outputs[i].is_constant) {
       // Output is a constant.
       const Tensor& const_tensor = kernel->outputs[i].constant_value;
@@ -182,47 +181,49 @@ void XlaComputationLaunchContext::PopulateOutputs(
         // Copy host -> device. (Empty tensors don't have backing buffers.)
         VLOG(1) << "Constant output tensor on device";
 
-        TF_CHECK_OK(
-            ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
+        OP_REQUIRES_OK(
+            ctx, ctx->allocate_output(i, const_tensor.shape(), &output_tensor));
+        if (XlaTensor* xla_tensor = XlaTensor::FromTensor(output_tensor)) {
+          OP_REQUIRES_OK(ctx, xla_tensor->AllocateShapedBuffer(
+                                  const_tensor.dtype(), const_tensor.shape(),
+                                  client_, stream->parent()->device_ordinal()));
+        }
 
         const void* src_ptr = DMAHelper::base(&const_tensor);
-        void* dst_ptr = DMAHelper::base(output_tensor);
-        gpu::DeviceMemoryBase gpu_dst_ptr(dst_ptr, total_bytes);
+        gpu::DeviceMemoryBase dst_ptr =
+            XlaTensor::DeviceMemoryFromTensor(*output_tensor);
         // Memcpying asynchronously is safe for the GPU, but the CPU uses a
         // shared allocator so hold a reference to the copied-to buffer until
         // complete.
         TensorReference ref(*output_tensor);
-        stream->ThenMemcpy(&gpu_dst_ptr, src_ptr, total_bytes);
+        stream->ThenMemcpy(&dst_ptr, src_ptr, total_bytes);
         stream->ThenDoHostCallback([ref] { ref.Unref(); });
       } else {
         // No copy required.
         ctx->set_output(i, const_tensor);
         output_tensor = ctx->mutable_output(i);
       }
-      if (tensor_info_manager_) {
-        XlaTensorInfo* tensor_info =
-            tensor_info_manager_->GetOrCreateTensorInfo(*output_tensor);
-        tensor_info->set_host_tensor(const_tensor);
+      if (XlaTensor* xla_tensor = XlaTensor::FromTensor(output_tensor)) {
+        xla_tensor->set_host_tensor(const_tensor);
       }
     } else {
       const TensorShape& shape = kernel->outputs[i].shape;
       VLOG(2) << "Retval " << i << " shape " << shape.DebugString();
 
       gpu::DeviceMemoryBase buffer = output->buffer({output_num});
-      Tensor output_tensor = XlaTensorBuffer::MakeTensor(
-          ctx->expected_output_dtype(i), shape, buffer, allocator);
-      xla_allocator_->Release(buffer.opaque());
-
-      xla::Shape output_shape = xla::ShapeUtil::GetTupleElementShape(
-          output->on_device_shape(), output_num);
-      if (xla::ShapeUtil::IsTuple(output_shape)) {
-        CHECK(tensor_info_manager_);
-        XlaTensorInfo* tensor_info =
-            tensor_info_manager_->GetOrCreateTensorInfo(output_tensor);
-        tensor_info->set_shaped_buffer(
-            ExtractSubShapedBuffer(*output, output_num));
+      if (allocate_xla_tensors_) {
+        Tensor* output_tensor;
+        OP_REQUIRES_OK(ctx, ctx->allocate_output(i, shape, &output_tensor));
+        XlaTensor* xla_tensor = XlaTensor::FromTensor(output_tensor);
+        CHECK(xla_tensor);
+        xla_tensor->set_shaped_buffer(
+            ExtractSubShapedBuffer(output.get(), output_num, xla_allocator_));
+      } else {
+        Tensor output_tensor = XlaTensorBuffer::MakeTensor(
+            ctx->expected_output_dtype(i), shape, buffer, allocator);
+        output->set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num});
+        ctx->set_output(i, output_tensor);
       }
-      ctx->set_output(i, output_tensor);
       ++output_num;
     }
 
@@ -235,9 +236,6 @@ void XlaComputationLaunchContext::PopulateOutputs(
   VLOG(2) << "Applying variable updates";
   for (int i = 0; i < kernel->resource_updates.size(); ++i) {
     Allocator* allocator = ctx->device()->GetAllocator({});
-    if (tensor_info_manager_) {
-      allocator = tensor_info_manager_;
-    }
     const XlaCompiler::ResourceUpdate& write = kernel->resource_updates[i];
     OP_REQUIRES(ctx,
                 write.input_index >= 0 && write.input_index < ctx->num_inputs(),
@@ -260,18 +258,21 @@ void XlaComputationLaunchContext::PopulateOutputs(
     mutex_lock ml(*variable->mu());
     OP_REQUIRES(ctx, variable->tensor()->dtype() == write.type,
                 errors::Internal("Mismatched type in variable write"));
-    *variable->tensor() =
-        XlaTensorBuffer::MakeTensor(write.type, write.shape, buffer, allocator);
-    xla_allocator_->Release(buffer.opaque());
-
-    xla::Shape output_shape = xla::ShapeUtil::GetTupleElementShape(
-        output->on_device_shape(), output_num);
-    if (xla::ShapeUtil::IsTuple(output_shape)) {
-      CHECK(tensor_info_manager_);
-      XlaTensorInfo* tensor_info =
-          tensor_info_manager_->GetOrCreateTensorInfo(*variable->tensor());
-      tensor_info->set_shaped_buffer(
-          ExtractSubShapedBuffer(*output, output_num));
+
+    if (allocate_xla_tensors_) {
+      Tensor output_tensor;
+      OP_REQUIRES_OK(
+          ctx, ctx->allocate_temp(write.type, write.shape, &output_tensor));
+      XlaTensor* xla_tensor = XlaTensor::FromTensor(&output_tensor);
+      CHECK(xla_tensor);
+      xla_tensor->set_shaped_buffer(
+          ExtractSubShapedBuffer(output.get(), output_num, xla_allocator_));
+      *variable->tensor() = output_tensor;
+    } else {
+      Tensor output_tensor = XlaTensorBuffer::MakeTensor(
+          write.type, write.shape, buffer, allocator);
+      output->set_buffer(gpu::DeviceMemoryBase(nullptr, 0), {output_num});
+      *variable->tensor() = output_tensor;
     }
     ++output_num;
   }
diff --git a/tensorflow/compiler/jit/xla_launch_util.h b/tensorflow/compiler/jit/xla_launch_util.h
index 8694f6ce58..14f70fe358 100644
--- a/tensorflow/compiler/jit/xla_launch_util.h
+++ b/tensorflow/compiler/jit/xla_launch_util.h
@@ -19,7 +19,7 @@ limitations under the License.
 #define TENSORFLOW_COMPILER_JIT_XLA_LAUNCH_UTIL_H_
 
 #include "tensorflow/compiler/jit/xla_compilation_cache.h"
-#include "tensorflow/compiler/jit/xla_tensor_info.h"
+#include "tensorflow/compiler/jit/xla_tensor.h"
 #include "tensorflow/compiler/tf2xla/xla_compiler.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/core/framework/allocation_description.pb.h"
@@ -47,16 +47,13 @@ std::map<int, OptionalTensor> SnapshotResourceVariables(OpKernelContext* ctx,
 class XlaAllocator : public xla::DeviceMemoryAllocator {
  public:
   XlaAllocator(const perftools::gputools::Platform* platform,
-               OpKernelContext* op_context);
+               Allocator* wrapped);
   ~XlaAllocator() override;
   xla::StatusOr<perftools::gputools::DeviceMemoryBase> Allocate(
       int device_ordinal, uint64 size, bool retry_on_failure) override;
   Status Deallocate(int device_ordinal,
                     perftools::gputools::DeviceMemoryBase* mem) override;
 
-  // Un-track 'ptr' - do not delete it on destruction.
-  void Release(void* ptr);
-
   // The Tensorflow BFC allocator used on GPU allows host-side deallocation
   // before GPU execution takes place. Tensorflow uses the ordering of the main
   // compute stream to enforce a happens-before relationship between a memory
@@ -67,17 +64,19 @@ class XlaAllocator : public xla::DeviceMemoryAllocator {
   bool AllowsAsynchronousDeallocation() const override { return true; }
 
  private:
-  OpKernelContext* const op_context_;
-  std::unordered_set<void*> allocated_;
+  Allocator* wrapped_;
 };
 
 // Helper class to perform the marshalling of TensorFlow inputs and outputs to
 // ShapedBuffers suitable for passing to an XLA computation.
 class XlaComputationLaunchContext {
  public:
+  // Create a new launch context. 'allocate_xla_tensors' is true if allocated
+  // output tensors and variables are always XlaTensors. If false they are
+  // assumed to be "normal" device pointers.
   XlaComputationLaunchContext(int64 num_resource_args, xla::LocalClient* client,
-                              XlaAllocator* xla_allocator,
-                              XlaTensorInfoManager* tensor_info_manager);
+                              xla::DeviceMemoryAllocator* xla_allocator,
+                              bool allocate_xla_tensors);
 
   // Add all inputs within `ctx` as XLA arguments (returned by arguments()).
   // `variables` is a map from TensorFlow argument number to resource variable.
@@ -97,8 +96,8 @@ class XlaComputationLaunchContext {
  private:
   int64 num_resource_args_;
   xla::LocalClient* client_;
-  XlaAllocator* xla_allocator_;
-  XlaTensorInfoManager* tensor_info_manager_;
+  xla::DeviceMemoryAllocator* xla_allocator_;
+  bool allocate_xla_tensors_;
   std::vector<std::unique_ptr<xla::ShapedBuffer>> arg_buffers_;
   std::vector<xla::ShapedBuffer*> arg_ptrs_;
 };
diff --git a/tensorflow/compiler/jit/xla_tensor.cc b/tensorflow/compiler/jit/xla_tensor.cc
new file mode 100644
index 0000000000..956328e675
--- /dev/null
+++ b/tensorflow/compiler/jit/xla_tensor.cc
@@ -0,0 +1,98 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/compiler/jit/xla_tensor.h"
+#include "tensorflow/compiler/tf2xla/shape_util.h"
+
+namespace tensorflow {
+
+/*static*/ XlaTensor* XlaTensor::FromTensor(Tensor* tensor) {
+  if (tensor->NumElements() == 0) {
+    return nullptr;
+  }
+  XlaTensor* xla_tensor =
+      FromOpaquePointer(const_cast<char*>(tensor->tensor_data().data()));
+  return xla_tensor;
+}
+
+/*static*/ const XlaTensor* XlaTensor::FromTensor(const Tensor* tensor) {
+  return FromTensor(const_cast<Tensor*>(tensor));
+}
+
+/*static*/ perftools::gputools::DeviceMemoryBase
+XlaTensor::DeviceMemoryFromTensor(const Tensor& tensor) {
+  const XlaTensor* xla_tensor = FromTensor(&tensor);
+  if (xla_tensor) {
+    CHECK(xla_tensor->has_shaped_buffer());
+    return xla_tensor->shaped_buffer().root_buffer();
+  } else {
+    return perftools::gputools::DeviceMemoryBase(
+        const_cast<char*>(tensor.tensor_data().data()),
+        tensor.tensor_data().size());
+  }
+}
+
+Status XlaTensor::AllocateShapedBuffer(DataType dtype, const TensorShape& shape,
+                                       xla::LocalClient* client,
+                                       int device_ordinal) {
+  xla::Shape on_host_shape;
+  TF_RETURN_IF_ERROR(TensorShapeToXLAShape(dtype, shape, &on_host_shape));
+  xla::Shape on_device_shape =
+      client->backend().transfer_manager()->HostShapeToDeviceShape(
+          on_host_shape);
+
+  xla::ShapedBuffer buffer(on_host_shape, on_device_shape, client->platform(),
+                           device_ordinal);
+  for (auto& index_to_buffer : buffer.buffers()) {
+    xla::Shape subshape =
+        xla::ShapeUtil::GetSubshape(on_device_shape, index_to_buffer.first);
+    uint64 size =
+        client->backend().transfer_manager()->GetByteSizeRequirement(subshape);
+    TF_ASSIGN_OR_RETURN(index_to_buffer.second,
+                        client->backend().memory_allocator()->Allocate(
+                            device_ordinal, size, /*retry_on_failure=*/false));
+  }
+
+  TF_ASSIGN_OR_RETURN(auto scoped_buffer,
+                      xla::ScopedShapedBuffer::MakeScoped(
+                          &buffer, client->backend().memory_allocator()));
+  set_shaped_buffer(std::move(scoped_buffer));
+  return Status::OK();
+}
+
+// The pointer tag, OR-ed into the XlaTensor's address to distinguish it from
+// device-side tensors, which are either CPU or GPU memory pointers. This works
+// because we're guaranteed that CPU and GPU pointers are aligned to > 1 bits.
+namespace {
+constexpr uintptr_t kTag = 0x1ULL;
+}
+
+/*static*/ XlaTensor* XlaTensor::FromOpaquePointer(void* ptr) {
+  uintptr_t value = reinterpret_cast<uintptr_t>(ptr);
+  if (value & kTag) {
+    return reinterpret_cast<XlaTensor*>(value & ~kTag);
+  } else {
+    return nullptr;
+  }
+}
+
+/*static*/ void* XlaTensor::ToOpaquePointer(XlaTensor* tensor) {
+  uintptr_t value = reinterpret_cast<uintptr_t>(tensor);
+  CHECK_EQ(value & kTag, 0);
+  value |= kTag;
+  return reinterpret_cast<XlaTensor*>(value);
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/compiler/jit/xla_tensor_info.h b/tensorflow/compiler/jit/xla_tensor.h
similarity index 50%
rename from tensorflow/compiler/jit/xla_tensor_info.h
rename to tensorflow/compiler/jit/xla_tensor.h
index a02f642c37..5ff2fb08f0 100644
--- a/tensorflow/compiler/jit/xla_tensor_info.h
+++ b/tensorflow/compiler/jit/xla_tensor.h
@@ -13,9 +13,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
-#ifndef TENSORFLOW_COMPILER_JIT_XLA_TENSOR_INFO_H_
-#define TENSORFLOW_COMPILER_JIT_XLA_TENSOR_INFO_H_
+#ifndef TENSORFLOW_COMPILER_JIT_XLA_TENSOR_H_
+#define TENSORFLOW_COMPILER_JIT_XLA_TENSOR_H_
 
+#include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/service/shaped_buffer.h"
 #include "tensorflow/core/framework/allocator.h"
 #include "tensorflow/core/framework/device_base.h"
@@ -24,11 +25,32 @@ limitations under the License.
 
 namespace tensorflow {
 
-// Information about a tensor. The XlaTensorInfoManager can maintain one of
-// these per device Tensor.
-class XlaTensorInfo {
+// The implementation of a Tensor for an XlaDevice. All device tensors are
+// actually one of these.
+//
+// To distinguish between "normal" device tensors and XlaTensors, the raw
+// pointer data stored in the TensorBuffer is a tagged pointer.
+class XlaTensor {
  public:
-  XlaTensorInfo() {}
+  // Downcast from a Tensor to an XlaTensor. Return nullptr if the downcast
+  // fails.
+  static XlaTensor* FromTensor(Tensor* tensor);
+  // Downcast from a Tensor to an XlaTensor. Return nullptr if the downcast
+  // fails.
+  static const XlaTensor* FromTensor(const Tensor* tensor);
+
+  // Create a DeviceMemoryBase from a Tensor. The Tensor can be an XlaTensor, in
+  // which case the returned value is shaped_buffer()->root_buffer(), or a
+  // normal Tensor in which case the returned value is
+  // {tensor.tensor_data().data(), tensor.tensor_data().size}.
+  static perftools::gputools::DeviceMemoryBase DeviceMemoryFromTensor(
+      const Tensor& tensor);
+
+  // Assign the internal ShapedBuffer to new memory for the given dtype and
+  // shape. If a ShapedBuffer exists already (has_shaped_buffer() == true), it
+  // is replaced and the managed memory deallocated.
+  Status AllocateShapedBuffer(DataType dtype, const TensorShape& shape,
+                              xla::LocalClient* client, int device_ordinal);
 
   // Some Tensors can have complex on-device shapes, including tuple shapes. To
   // manage the memory for these tensors a ShapedBuffer may be required.
@@ -37,10 +59,14 @@ class XlaTensorInfo {
   bool has_shaped_buffer() const { return shaped_buffer_ != nullptr; }
   // Return the contained ShapedBuffer.
   // REQUIRES: has_shaped_buffer()
-  const xla::ShapedBuffer& shaped_buffer() const { return *shaped_buffer_; }
+  const xla::ShapedBuffer& shaped_buffer() const {
+    CHECK(has_shaped_buffer());
+    return *shaped_buffer_;
+  }
   // Mutates the TensorInfo to set the ShapedBuffer.
-  void set_shaped_buffer(xla::ShapedBuffer shaped_buffer) {
-    shaped_buffer_.reset(new xla::ShapedBuffer(std::move(shaped_buffer)));
+  void set_shaped_buffer(
+      std::unique_ptr<xla::ScopedShapedBuffer> shaped_buffer) {
+    shaped_buffer_ = std::move(shaped_buffer);
   }
 
   // Some tensors on the device may have known values on the host. We use these
@@ -57,51 +83,18 @@ class XlaTensorInfo {
     host_tensor_.reset(new Tensor(tensor));
   }
 
+  // Convert from a raw pointer to an XlaTensor, removing the pointer tag.
+  static XlaTensor* FromOpaquePointer(void* ptr);
+  // Convert to a raw pointer from an XlaTensor, adding the pointer tag.
+  static void* ToOpaquePointer(XlaTensor* tensor);
+
  private:
   // The optional contained ShapedBuffer.
-  std::unique_ptr<xla::ShapedBuffer> shaped_buffer_;
+  std::unique_ptr<xla::ScopedShapedBuffer> shaped_buffer_;
   // An optional host tensor value.
   std::unique_ptr<Tensor> host_tensor_;
 };
 
-// Manages XlaTensorInfo objects. This class is also an Allocator, so that
-// XlaTensorInfo objects can be deleted when their Tensor is deallocated.
-class XlaTensorInfoManager : public AllocatorWrapper {
- public:
-  // Creates a new XlaTensorInfoManager, delegating all DeallocateRaw calls to
-  // allocator.
-  XlaTensorInfoManager(Allocator* allocator) : AllocatorWrapper(allocator) {}
-  ~XlaTensorInfoManager() {
-    // Destroy the tensor info hashtable under the lock, to ensure all accesses
-    // to the hashtable are properly sequenced.
-    mutex_lock lock(lock_);
-    tensor_infos_.clear();
-  }
-
-  // Returns the XlaTensorInfo for the given device memory pointer or nullptr if
-  // none exists.
-  const XlaTensorInfo* GetTensorInfo(const void* device_ptr) const;
-  // Returns the XlaTensorInfo for the device memory pointer extracted from
-  // tensor or nullptr if none exists.
-  const XlaTensorInfo* GetTensorInfo(const Tensor& tensor);
-
-  // Returns the XlaTensorInfo for the given device memory pointer, creating one
-  // if necessary.
-  XlaTensorInfo* GetOrCreateTensorInfo(const Tensor& tensor);
-  // Returns the XlaTensorInfo for the device memory pointer extracted from
-  // tensor, creating one if necessary.
-  XlaTensorInfo* GetOrCreateTensorInfo(const void* device_ptr);
-
-  // Allocator interface
-  void DeallocateRaw(void* ptr) override;
-
- private:
-  mutable mutex lock_;
-  // The managed tensor infos. The mapped value is a unique_ptr so that returned
-  // references are stable over rehashes.
-  std::unordered_map<const void*, std::unique_ptr<XlaTensorInfo>> tensor_infos_
-      GUARDED_BY(lock_);
-};
 }  // namespace tensorflow
 
 #endif
diff --git a/tensorflow/compiler/jit/xla_tensor_info.cc b/tensorflow/compiler/jit/xla_tensor_info.cc
deleted file mode 100644
index 0ce18c27cb..0000000000
--- a/tensorflow/compiler/jit/xla_tensor_info.cc
+++ /dev/null
@@ -1,56 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#include "tensorflow/compiler/jit/xla_tensor_info.h"
-
-namespace tensorflow {
-
-const XlaTensorInfo* XlaTensorInfoManager::GetTensorInfo(
-    const void* device_ptr) const {
-  mutex_lock lock(lock_);
-  auto iterator = tensor_infos_.find(device_ptr);
-  return (iterator == tensor_infos_.end()) ? nullptr
-                                           : tensor_infos_.at(device_ptr).get();
-}
-
-XlaTensorInfo* XlaTensorInfoManager::GetOrCreateTensorInfo(
-    const void* device_ptr) {
-  mutex_lock lock(lock_);
-  auto iterator = tensor_infos_.find(device_ptr);
-  if (iterator != tensor_infos_.end()) {
-    return iterator->second.get();
-  }
-  auto iterator_and_inserted =
-      tensor_infos_.emplace(device_ptr, MakeUnique<XlaTensorInfo>());
-  CHECK(iterator_and_inserted.second);
-  return iterator_and_inserted.first->second.get();
-}
-
-const XlaTensorInfo* XlaTensorInfoManager::GetTensorInfo(const Tensor& tensor) {
-  return GetTensorInfo(tensor.tensor_data().data());
-}
-
-XlaTensorInfo* XlaTensorInfoManager::GetOrCreateTensorInfo(
-    const Tensor& tensor) {
-  return GetOrCreateTensorInfo(tensor.tensor_data().data());
-}
-
-void XlaTensorInfoManager::DeallocateRaw(void* ptr) {
-  wrapped()->DeallocateRaw(ptr);
-  mutex_lock lock(lock_);
-  tensor_infos_.erase(ptr);
-}
-
-}  // namespace tensorflow
-- 
GitLab


From 2261679a3b778ac6ddd868cf9c4906c84a979828 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 07:24:14 -0700
Subject: [PATCH 749/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 191063815
---
 tensorflow/compiler/jit/BUILD                     |  1 +
 .../jit/encapsulate_subgraphs_pass_test.cc        |  5 +++--
 tensorflow/compiler/jit/graph_to_functiondef.cc   |  3 ++-
 .../jit/mark_for_compilation_pass_test.cc         | 11 ++++++-----
 tensorflow/compiler/tf2xla/BUILD                  |  1 +
 tensorflow/compiler/tf2xla/kernels/softmax_op.cc  |  3 ++-
 tensorflow/compiler/tf2xla/sharding_util.cc       |  5 +++--
 tensorflow/compiler/tf2xla/tf2xla_util_test.cc    |  3 ++-
 tensorflow/compiler/tf2xla/xla_compiler_test.cc   | 15 +++++++++------
 tensorflow/compiler/xla/shape_util.cc             |  6 +++---
 tensorflow/compiler/xla/text_literal_reader.cc    |  4 ++--
 tensorflow/compiler/xla/util.cc                   |  4 ++--
 .../grappler/optimizers/arithmetic_optimizer.cc   |  5 +++--
 .../grappler/optimizers/constant_folding_test.cc  |  5 +++--
 .../grappler/optimizers/dependency_optimizer.cc   |  3 ++-
 .../core/grappler/optimizers/layout_optimizer.cc  |  4 ++--
 tensorflow/core/platform/env_test.cc              |  5 ++---
 tensorflow/core/platform/file_system.cc           |  4 ++--
 tensorflow/core/platform/file_system_test.cc      |  5 +++--
 19 files changed, 53 insertions(+), 39 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index c2283bab3d..24aa203c00 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -361,6 +361,7 @@ tf_cc_test(
         "//tensorflow/core:core_cpu",
         "//tensorflow/core:framework",
         "//tensorflow/core:framework_internal",
+        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
         "//tensorflow/core:testlib",
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 7899b5d72d..56efe98fdb 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/util/equal_graph_def.h"
 
@@ -771,7 +772,7 @@ TEST(EncapsulateSubgraphsWithGuaranteeConstOpTest, Simple) {
         Graph* graph = graph_ptr->get();
         for (const Node* n : graph->nodes()) {
           if (n->type_string() == "_Arg" &&
-              StringPiece(n->name()).starts_with("const")) {
+              str_util::StartsWith(n->name(), "const")) {
             ++guaranteed_consts;
             EXPECT_TRUE(HasGuaranteeConstAttr(*n));
           } else {
@@ -816,7 +817,7 @@ TEST(EncapsulateSubgraphsWithGuaranteeConstOpTest, Add) {
         Graph* graph = graph_ptr->get();
         for (const Node* n : graph->nodes()) {
           if (n->type_string() == "_Arg" &&
-              StringPiece(n->name()).starts_with("const")) {
+              str_util::StartsWith(n->name(), "const")) {
             ++guaranteed_consts;
             EXPECT_TRUE(HasGuaranteeConstAttr(*n));
           } else {
diff --git a/tensorflow/compiler/jit/graph_to_functiondef.cc b/tensorflow/compiler/jit/graph_to_functiondef.cc
index 6fa21fa620..8f5e11dfa4 100644
--- a/tensorflow/compiler/jit/graph_to_functiondef.cc
+++ b/tensorflow/compiler/jit/graph_to_functiondef.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/graph/graph.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -229,7 +230,7 @@ Status GraphToFunctionDef(const Graph& graph, const string& name,
   for (int n_index = 0; n_index < fdef->node_def_size(); ++n_index) {
     NodeDef* node_def = fdef->mutable_node_def(n_index);
     for (int i = 0; i < node_def->input_size(); ++i) {
-      if (StringPiece(node_def->input(i)).starts_with("^")) {
+      if (str_util::StartsWith(node_def->input(i), "^")) {
         // Control input
         const string normalized =
             node_names.Renormalize(node_def->input(i).substr(1));
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index 1a8858ccce..381c0205fd 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -27,6 +27,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/graph/graph_def_builder_util.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -519,11 +520,11 @@ TEST(XlaCompilationTest, IllegalCycle_UsefulErrorMessage) {
 
   Status status = MarkForCompilation(&graph);
   EXPECT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.ToString())
-                  .contains("Edge from c to a would create a cycle.\n"
-                            "+-> a\n"
-                            "|   b\n"
-                            "+-- c\n"));
+  EXPECT_TRUE(str_util::StrContains(status.ToString(),
+                                    "Edge from c to a would create a cycle.\n"
+                                    "+-> a\n"
+                                    "|   b\n"
+                                    "+-- c\n"));
 }
 
 TEST(XlaCompilationTest, Retval) {
diff --git a/tensorflow/compiler/tf2xla/BUILD b/tensorflow/compiler/tf2xla/BUILD
index 8c33bf179c..e7daf4e01c 100644
--- a/tensorflow/compiler/tf2xla/BUILD
+++ b/tensorflow/compiler/tf2xla/BUILD
@@ -332,6 +332,7 @@ tf_cc_test(
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/core:core_cpu_internal",
         "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:tensor_testutil",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
index aa47cb799f..463788b8b4 100644
--- a/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
+++ b/tensorflow/compiler/tf2xla/kernels/softmax_op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 namespace {
@@ -28,7 +29,7 @@ namespace {
 class SoftmaxOp : public XlaOpKernel {
  public:
   explicit SoftmaxOp(OpKernelConstruction* ctx) : XlaOpKernel(ctx) {
-    log_ = StringPiece(type_string()).starts_with("Log");
+    log_ = str_util::StartsWith(type_string(), "Log");
   }
 
   void Compile(XlaOpKernelContext* ctx) override {
diff --git a/tensorflow/compiler/tf2xla/sharding_util.cc b/tensorflow/compiler/tf2xla/sharding_util.cc
index 1a0e09758f..5759c72af3 100644
--- a/tensorflow/compiler/tf2xla/sharding_util.cc
+++ b/tensorflow/compiler/tf2xla/sharding_util.cc
@@ -16,6 +16,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
@@ -65,8 +66,8 @@ ParseShardingFromDevice(
   if (explicit_sharding.has_value()) {
     return explicit_sharding;
   } else if (!parsed_device.has_type || !parsed_device.has_id ||
-             !StringPiece(parsed_device.type)
-                  .contains(kDeviceSuffixReplicatedCore)) {
+             !str_util::StrContains(parsed_device.type,
+                                    kDeviceSuffixReplicatedCore)) {
     return tensorflow::gtl::optional<xla::OpSharding>();
   } else {
     const int core = parsed_device.id;
diff --git a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
index ed10d80609..ae51446204 100644
--- a/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
+++ b/tensorflow/compiler/tf2xla/tf2xla_util_test.cc
@@ -25,6 +25,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -33,7 +34,7 @@ namespace {
 
 void ExpectErrorContains(const Status& status, StringPiece str) {
   EXPECT_NE(Status::OK(), status);
-  EXPECT_TRUE(StringPiece(status.error_message()).contains(str))
+  EXPECT_TRUE(str_util::StrContains(status.error_message(), str))
       << "expected error: " << status.error_message() << " to contain: " << str;
 }
 
diff --git a/tensorflow/compiler/tf2xla/xla_compiler_test.cc b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
index a18eeacd41..096dc7160b 100644
--- a/tensorflow/compiler/tf2xla/xla_compiler_test.cc
+++ b/tensorflow/compiler/tf2xla/xla_compiler_test.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph.h"
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/version.h"
 
@@ -257,10 +258,10 @@ TEST_F(XlaCompilerTest, HasSaneErrorOnNonCompileTimeConstantInputToReshape) {
                             std::move(graph), args, &result);
   EXPECT_FALSE(status.ok());
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("depends on a parameter"))
+      str_util::StrContains(status.error_message(), "depends on a parameter"))
       << status.error_message();
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("[[Node: C = Reshape"))
+      str_util::StrContains(status.error_message(), "[[Node: C = Reshape"))
       << status.error_message();
 }
 
@@ -597,7 +598,8 @@ TEST_F(XlaCompilerTest, UndefinedFunctionFails) {
       compiler.CompileFunction(XlaCompiler::CompileOptions(), name_attr,
                                /*args=*/{}, &result);
   EXPECT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("is not defined."))
+  EXPECT_TRUE(str_util::StrContains(StringPiece(status.error_message()),
+                                    "is not defined."))
       << status.error_message();
 }
 
@@ -676,11 +678,12 @@ TEST_F(XlaCompilerTest, LocalFunctionWithWrongArgumentsFail) {
 
   ASSERT_FALSE(status.ok());
   // Flib lookup failure.
-  EXPECT_TRUE(StringPiece(status.error_message()).contains("is not defined."))
+  EXPECT_TRUE(str_util::StrContains(StringPiece(status.error_message()),
+                                    "is not defined."))
       << status.error_message();
   // Local flib lookup failure.
-  EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Attr T is not found"))
+  EXPECT_TRUE(str_util::StrContains(StringPiece(status.error_message()),
+                                    "Attr T is not found"))
       << status.error_message();
 }
 
diff --git a/tensorflow/compiler/xla/shape_util.cc b/tensorflow/compiler/xla/shape_util.cc
index 4f604e6f7c..6825d24765 100644
--- a/tensorflow/compiler/xla/shape_util.cc
+++ b/tensorflow/compiler/xla/shape_util.cc
@@ -502,11 +502,11 @@ namespace {
 StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
   tensorflow::str_util::RemoveLeadingWhitespace(s);
 
-  if (s->Consume("(")) {  // Tuple.
+  if (tensorflow::str_util::ConsumePrefix(s, "(")) {  // Tuple.
     std::vector<Shape> shapes;
     bool must_end = false;
     while (true) {
-      if (s->Consume(")")) {
+      if (tensorflow::str_util::ConsumePrefix(s, ")")) {
         break;
       } else if (must_end) {
         return InvalidArgument("Expected end of tuple; got: \"%s\"",
@@ -515,7 +515,7 @@ StatusOr<Shape> ParseShapeStringInternal(tensorflow::StringPiece* s) {
       shapes.emplace_back();
       TF_ASSIGN_OR_RETURN(shapes.back(), ParseShapeStringInternal(s));
       tensorflow::str_util::RemoveLeadingWhitespace(s);
-      must_end = !s->Consume(",");
+      must_end = !tensorflow::str_util::ConsumePrefix(s, ",");
     }
     return ShapeUtil::MakeTupleShape(shapes);
   }
diff --git a/tensorflow/compiler/xla/text_literal_reader.cc b/tensorflow/compiler/xla/text_literal_reader.cc
index 6fa4c48e11..44f874cd2a 100644
--- a/tensorflow/compiler/xla/text_literal_reader.cc
+++ b/tensorflow/compiler/xla/text_literal_reader.cc
@@ -38,7 +38,7 @@ namespace xla {
 
 StatusOr<std::unique_ptr<Literal>> TextLiteralReader::ReadPath(
     tensorflow::StringPiece path) {
-  CHECK(!path.ends_with(".gz"))
+  CHECK(!tensorflow::str_util::EndsWith(path, ".gz"))
       << "TextLiteralReader no longer supports reading .gz files";
   std::unique_ptr<tensorflow::RandomAccessFile> file;
   Status s =
@@ -115,7 +115,7 @@ StatusOr<std::unique_ptr<Literal>> TextLiteralReader::ReadAllLines() {
     tensorflow::StringPiece value_string = pieces[1];
     tensorflow::str_util::RemoveWhitespaceContext(&coordinates_string);
     tensorflow::str_util::RemoveWhitespaceContext(&value_string);
-    if (!coordinates_string.Consume("(")) {
+    if (!tensorflow::str_util::ConsumePrefix(&coordinates_string, "(")) {
       return InvalidArgument(
           "expected '(' at the beginning of coordinates: \"%s\"", line.c_str());
     }
diff --git a/tensorflow/compiler/xla/util.cc b/tensorflow/compiler/xla/util.cc
index dc4f7a1cb4..e43498e381 100644
--- a/tensorflow/compiler/xla/util.cc
+++ b/tensorflow/compiler/xla/util.cc
@@ -243,8 +243,8 @@ string HumanReadableNumOps(double flops, double nanoseconds,
       static_cast<int64>(nano_flops * 1e9));
   tensorflow::StringPiece sp(throughput);
   // Use the more common "G(FLOPS)", rather than "B(FLOPS)"
-  if (sp.ends_with("B") ||  // Ends in 'B', ignoring case
-      sp.ends_with("b")) {
+  if (tensorflow::str_util::EndsWith(sp, "B") ||  // Ends in 'B', ignoring case
+      tensorflow::str_util::EndsWith(sp, "b")) {
     *throughput.rbegin() = 'G';
   }
   throughput += tensorflow::strings::StrCat(op_prefix, "OP/s");
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 36b26c18f9..d155e0b289 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/tensor_coding.h"
 #include "tensorflow/core/util/device_name_utils.h"
@@ -1303,8 +1304,8 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     // with image.type than with dst_type.
     if (DeviceNameUtils::SplitDeviceName(transpose->device(), &dontcare,
                                          &device) &&
-        (StringPiece(device).contains(DEVICE_CPU) ||
-         StringPiece(device).contains(DEVICE_GPU))) {
+        (str_util::StrContains(device, DEVICE_CPU) ||
+         str_util::StrContains(device, DEVICE_GPU))) {
       const NodeDef* cast = node_map_->GetNode(transpose->input(0));
       if (cast->op() == "Cast") {
         const NodeDef* input = node_map_->GetNode(cast->input(0));
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 0548e53e03..1db4fb9de7 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -2050,7 +2051,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_AssociativeAndCommutative) {
         EXPECT_EQ("ConstantFolding/acc6_partial_split_2", node.input(1));
         EXPECT_EQ("y", node.input(2));
       }
-      if (StringPiece(node.name()).starts_with("ConstantFolding/")) {
+      if (str_util::StartsWith(node.name(), "ConstantFolding/")) {
         EXPECT_EQ("Const", node.op());
       }
     }
@@ -2135,7 +2136,7 @@ TEST_F(ConstantFoldingTest, PartialFolding_Concat) {
       EXPECT_EQ("x", node.input(1));
       EXPECT_EQ("y", node.input(2));
       EXPECT_EQ("axis", node.input(3));
-    } else if (StringPiece(node.name()).starts_with("ConstantFolding/")) {
+    } else if (str_util::StartsWith(node.name(), "ConstantFolding/")) {
       EXPECT_EQ("Const", node.op());
     } else {
       EXPECT_EQ(item.graph.node(i).DebugString(), node.DebugString());
diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
index e5822c3b42..ed9bce439c 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
@@ -87,7 +88,7 @@ bool DependencyOptimizer::SafeToRemoveIdentity(const NodeDef& node) {
     // Don't turn Identity nodes following Switch into NoOp or remove them
     // if it requires anchoring a control dependencies the Switch node, which
     // is not valid.
-    if (StringPiece(node.name()).starts_with(kConstantFoldingCtrl)) {
+    if (str_util::StartsWith(node.name(), kConstantFoldingCtrl)) {
       // TODO(rmlarsen): Try to remove this artificial contraint.
       return false;
     }
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 18e63f823b..254c1edf7b 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -547,8 +547,8 @@ class NodeProcessor : public GraphProcessor {
     string device;
     string not_used;
     if (DeviceNameUtils::SplitDeviceName(device_name, &not_used, &device) &&
-        (StringPiece(str_util::Lowercase(device)))
-            .contains(str_util::Lowercase(DEVICE_GPU))) {
+        str_util::StrContains(str_util::Lowercase(device),
+                              str_util::Lowercase(DEVICE_GPU))) {
       return true;
     }
     return false;
diff --git a/tensorflow/core/platform/env_test.cc b/tensorflow/core/platform/env_test.cc
index 9a6ff48069..a70a417e6a 100644
--- a/tensorflow/core/platform/env_test.cc
+++ b/tensorflow/core/platform/env_test.cc
@@ -373,9 +373,8 @@ TEST_F(DefaultEnvTest, CreateUniqueFileName) {
 
   EXPECT_TRUE(env->CreateUniqueFileName(&filename, suffix));
 
-  StringPiece str(filename);
-  EXPECT_TRUE(str.starts_with(prefix));
-  EXPECT_TRUE(str.ends_with(suffix));
+  EXPECT_TRUE(str_util::StartsWith(filename, prefix));
+  EXPECT_TRUE(str_util::EndsWith(filename, suffix));
 }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/platform/file_system.cc b/tensorflow/core/platform/file_system.cc
index 5bc8606e28..a2f42f44ac 100644
--- a/tensorflow/core/platform/file_system.cc
+++ b/tensorflow/core/platform/file_system.cc
@@ -134,7 +134,7 @@ Status FileSystem::GetMatchingPaths(const string& pattern,
               const string child_path = io::JoinPath(current_dir, children[i]);
               // In case the child_path doesn't start with the fixed_prefix then
               // we don't need to explore this path.
-              if (!StringPiece(child_path).starts_with(fixed_prefix)) {
+              if (!str_util::StartsWith(child_path, fixed_prefix)) {
                 children_dir_status[i] = Status(tensorflow::error::CANCELLED,
                                                 "Operation not needed");
               } else {
@@ -241,7 +241,7 @@ Status FileSystem::RecursivelyCreateDir(const string& dirname) {
       return status;
     }
     // Basename returns "" for / ending dirs.
-    if (!remaining_dir.ends_with("/")) {
+    if (!str_util::EndsWith(remaining_dir, "/")) {
       sub_dirs.push_back(io::Basename(remaining_dir));
     }
     remaining_dir = io::Dirname(remaining_dir);
diff --git a/tensorflow/core/platform/file_system_test.cc b/tensorflow/core/platform/file_system_test.cc
index e07aad55cb..f261b8f576 100644
--- a/tensorflow/core/platform/file_system_test.cc
+++ b/tensorflow/core/platform/file_system_test.cc
@@ -124,7 +124,7 @@ class InterPlanetaryFileSystem : public NullFileSystem {
     io::ParseURI(name, &scheme, &host, &path);
     ASSERT_EQ(scheme, "ipfs");
     ASSERT_EQ(host, "solarsystem");
-    path.Consume("/");
+    str_util::ConsumePrefix(&path, "/");
     *parsed_path = path.ToString();
   }
 
@@ -160,7 +160,8 @@ string Match(InterPlanetaryFileSystem* ipfs, const string& suffix_pattern) {
     std::sort(results.begin(), results.end());
     for (const string& result : results) {
       StringPiece trimmed_result(result);
-      EXPECT_TRUE(trimmed_result.Consume(strings::StrCat(kPrefix, "/")));
+      EXPECT_TRUE(str_util::ConsumePrefix(&trimmed_result,
+                                          strings::StrCat(kPrefix, "/")));
       trimmed_results.push_back(trimmed_result);
     }
     return str_util::Join(trimmed_results, ",");
-- 
GitLab


From 330c2a831dfff5640ebc2e2811749c6557f6198a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 08:12:16 -0700
Subject: [PATCH 750/960] Update test for DebugStripper to construct graph in
 scope.

PiperOrigin-RevId: 191067139
---
 tensorflow/core/grappler/optimizers/BUILD     |  1 +
 .../optimizers/debug_stripper_test.cc         | 57 ++++++++++---------
 2 files changed, 31 insertions(+), 27 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 2c365c467c..f865d0c159 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -630,6 +630,7 @@ cc_library(
 
 tf_cuda_cc_test(
     name = "debug_stripper_test",
+    size = "small",
     srcs = ["debug_stripper_test.cc"],
     deps = [
         ":debug_stripper",
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
index aacd55f136..c79c36841d 100644
--- a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
+++ b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
@@ -16,7 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/debug_stripper.h"
 
 #include "tensorflow/cc/ops/standard_ops.h"
-#include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/utils/grappler_test.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
@@ -29,14 +29,13 @@ namespace {
 class DebugStripperTest : public GrapplerTest {};
 
 TEST_F(DebugStripperTest, OutputEqualToInput) {
-  constexpr char device[] = "/device:CPU:0";
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({}));
+  Output y = ops::Placeholder(s, DT_FLOAT, ops::Placeholder::Shape({}));
+  Output add = ops::Add(s, x, y);
+  Output result = ops::Identity(s, add);
   GrapplerItem item;
-  item.graph = test::function::GDef(
-      {test::function::NDef("x", "Placeholder", {}, {{"dtype", DT_FLOAT}},
-                            device),
-       test::function::NDef("y", "XTimesTwo", {"x"}, {}, device),
-       test::function::NDef("z", "Identity", {"y"}, {{"T", DT_FLOAT}}, device)},
-      {});
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
   DebugStripper optimizer;
   GraphDef output;
@@ -45,19 +44,17 @@ TEST_F(DebugStripperTest, OutputEqualToInput) {
 }
 
 TEST_F(DebugStripperTest, StripAssertFromGraph) {
-  constexpr char device[] = "/device:CPU:0";
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  auto greaterequal = ops::GreaterEqual(s.WithOpName("GreaterEqual"), x, y);
+  auto assert = ops::Assert(s.WithOpName("Assert"), greaterequal, {x, y});
+  Output add = ops::Add(
+      s.WithOpName("z").WithControlDependencies({assert.operation}), x, y);
   GrapplerItem item;
-  item.graph = test::function::GDef(
-      {test::function::NDef("x", "Placeholder", {}, {{"dtype", DT_FLOAT}},
-                            device),
-       test::function::NDef("y", "Placeholder", {}, {{"dtype", DT_FLOAT}},
-                            device),
-       test::function::NDef("GreaterEqual", "GreaterEqual", {"x", "y"},
-                            {{"T", DT_FLOAT}}, device),
-       test::function::NDef("Assert", "Assert", {"GreaterEqual"},
-                            {{"T", DT_FLOAT}}, device),
-       test::function::NDef("z", "Add", {"x", "y", "^Assert"}, {}, device)},
-      {});
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
   DebugStripper optimizer;
   GraphDef output;
@@ -68,31 +65,27 @@ TEST_F(DebugStripperTest, StripAssertFromGraph) {
     if (node.name() == "x") {
       count++;
       EXPECT_EQ("Placeholder", node.op());
-      EXPECT_EQ(device, node.device());
       EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "y") {
       count++;
       EXPECT_EQ("Placeholder", node.op());
-      EXPECT_EQ(device, node.device());
       EXPECT_EQ(0, node.input_size());
     } else if (node.name() == "GreaterEqual") {
       count++;
       EXPECT_EQ("GreaterEqual", node.op());
-      EXPECT_EQ(device, node.device());
       EXPECT_EQ(2, node.input_size());
       EXPECT_EQ("x", node.input(0));
       EXPECT_EQ("y", node.input(1));
     } else if (node.name() == "Assert") {
       count++;
       EXPECT_EQ("NoOp", node.op());
-      EXPECT_EQ(device, node.device());
-      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ(3, node.input_size());
       EXPECT_EQ("^GreaterEqual", node.input(0));
-      EXPECT_EQ(0, node.attr_size());
+      EXPECT_EQ("^x", node.input(1));
+      EXPECT_EQ("^y", node.input(2));
     } else if (node.name() == "z") {
       count++;
       EXPECT_EQ("Add", node.op());
-      EXPECT_EQ(device, node.device());
       EXPECT_EQ(3, node.input_size());
       EXPECT_EQ("x", node.input(0));
       EXPECT_EQ("y", node.input(1));
@@ -100,6 +93,16 @@ TEST_F(DebugStripperTest, StripAssertFromGraph) {
     }
   }
   EXPECT_EQ(5, count);
+
+  Tensor x_t(DT_FLOAT, TensorShape({}));
+  Tensor y_t(DT_FLOAT, TensorShape({}));
+  x_t.flat<float>()(0) = 1.0f;
+  y_t.flat<float>()(0) = 0.5f;
+  std::vector<Tensor> expected =
+      EvaluateNodes(item.graph, {"z"}, {{"x", x_t}, {"y", y_t}});
+  std::vector<Tensor> optimized =
+      EvaluateNodes(output, {"z"}, {{"x", x_t}, {"y", y_t}});
+  test::ExpectTensorEqual<float>(expected[0], optimized[0]);
 }
 
 }  // namespace
-- 
GitLab


From ddbb2c52db5cfab02b80b2ef563d8d6251dcfe77 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 08:23:30 -0700
Subject: [PATCH 751/960] Fix a crash in Quantize() when
 tf.contrib.framework.get_name_scope() == None.

PiperOrigin-RevId: 191068059
---
 .../contrib/quantize/python/quantize.py       |  4 +++-
 .../contrib/quantize/python/quantize_test.py  | 21 +++++++++++++++++++
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/quantize/python/quantize.py b/tensorflow/contrib/quantize/python/quantize.py
index 2889016a84..d53d4d7b10 100644
--- a/tensorflow/contrib/quantize/python/quantize.py
+++ b/tensorflow/contrib/quantize/python/quantize.py
@@ -416,7 +416,9 @@ def _InsertQuantOp(context,
   # name_prefix starts with 'TPUReplicate/loop/'; without dropping it
   # variables are created as TPUReplicate/loop/TPUReplicate/loop/..., which
   # breaks things later.
-  name_prefix = common.DropStringPrefix(name_prefix, ops.get_name_scope() + '/')
+  name_scope = ops.get_name_scope()
+  if name_scope:
+    name_prefix = common.DropStringPrefix(name_prefix, name_scope + '/')
 
   inputs = producer.outputs[0]
   # Prevent ops from being quantized multiple times. Bypass ops can sometimes
diff --git a/tensorflow/contrib/quantize/python/quantize_test.py b/tensorflow/contrib/quantize/python/quantize_test.py
index 98f05c8bfc..8d057d3710 100644
--- a/tensorflow/contrib/quantize/python/quantize_test.py
+++ b/tensorflow/contrib/quantize/python/quantize_test.py
@@ -247,6 +247,27 @@ class QuantizeTest(test_util.TensorFlowTestCase):
       self.assertTrue(not op.name.startswith('name_scope/name_scope/'),
                       'Broken op: %s' % op.name)
 
+  def testWithNullNameScope(self):
+    self._RunTestOverParameters(self._TestWithNullNameScope)
+
+  def _TestWithNullNameScope(self, is_training):
+    graph = ops.Graph()
+    with graph.as_default():
+      with graph.name_scope(None):
+        batch_size, height, width, depth = 5, 128, 128, 3
+        input1 = array_ops.zeros((batch_size, height, width, depth))
+        _ = conv2d(
+            input1,
+            32, [5, 5],
+            stride=2,
+            padding='SAME',
+            weights_initializer=self._WeightInit(0.09),
+            activation_fn=None,
+            scope='test')
+
+        quantize.Quantize(graph, is_training, weight_bits=8, activation_bits=8)
+        # Passes if Quantize() does not crash.
+
   def _WeightInit(self, stddev):
     """Returns truncated normal variable initializer.
 
-- 
GitLab


From e995271f8550bf6b62fb9089ea702f2114783904 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Fri, 30 Mar 2018 09:41:29 -0700
Subject: [PATCH 752/960] Fix the index to match the left nav.

PiperOrigin-RevId: 191075496
---
 tensorflow/docs_src/programmers_guide/index.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/docs_src/programmers_guide/index.md b/tensorflow/docs_src/programmers_guide/index.md
index e8c2fa6990..017db0e8cb 100644
--- a/tensorflow/docs_src/programmers_guide/index.md
+++ b/tensorflow/docs_src/programmers_guide/index.md
@@ -5,6 +5,7 @@ works. The units are as follows:
 
 ## High Level APIs
 
+  * @{$programmers_guide/eager}, which is the easiest way to use tensorflow.
   * @{$programmers_guide/estimators}, which introduces a high-level
     TensorFlow API that greatly simplifies ML programming.
   * @{$programmers_guide/datasets}, which explains how to
-- 
GitLab


From 3d9c27742693f9859e2fb75de57fe108520de712 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 09:56:29 -0700
Subject: [PATCH 753/960] Fix several data races by acquiring locks.

(The racy accesses were detected by the thread safety annotations.)

PiperOrigin-RevId: 191077376
---
 tensorflow/core/common_runtime/executor.cc                  | 6 ++++--
 .../core/common_runtime/process_function_library_runtime.h  | 5 ++++-
 tensorflow/core/kernels/queue_op.h                          | 1 +
 3 files changed, 9 insertions(+), 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index b06b75d658..195803fd7f 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -1022,7 +1022,8 @@ class ExecutorState {
     int total_input_tensors = 0;
     std::vector<const Node*>* nodes = nullptr;
 
-    // Lock ordering: ExecutorState.mu_ < mu.
+    // Lock ordering: ExecutorState.mu_ < mu;
+    // during structured traversal: parent_frame->mu < mu.
     mutex mu;
 
     void InitializeFrameInfo(const string& enter_name) {
@@ -2333,8 +2334,9 @@ void ExecutorState::DeleteFrame(FrameState* frame, TaggedNodeSeq* ready) {
   FrameState* parent_frame = frame->parent_frame;
   const int64 parent_iter = frame->parent_iter;
   if (parent_frame != nullptr) {
-    mutex_lock paranet_frame_lock(parent_frame->mu);
+    mutex_lock parent_frame_lock(parent_frame->mu);
     // Propagate all the dead exits to the parent frame.
+    mutex_lock this_frame_lock(frame->mu);
     for (const Node* node : frame->dead_exits) {
       auto parent_iter_state = parent_frame->GetIteration(parent_iter);
       for (const Edge* e : node->out_edges()) {
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime.h b/tensorflow/core/common_runtime/process_function_library_runtime.h
index d69e8bc2a0..c7b8259f78 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime.h
+++ b/tensorflow/core/common_runtime/process_function_library_runtime.h
@@ -155,7 +155,10 @@ class ProcessFunctionLibraryRuntime {
 
     string target_device() { return target_device_; }
 
-    FunctionLibraryRuntime::LocalHandle local_handle() { return local_handle_; }
+    FunctionLibraryRuntime::LocalHandle local_handle() {
+      mutex_lock l(mu_);
+      return local_handle_;
+    }
 
     // Initializes the FunctionData object by potentially making an Initialize
     // call to the DistributedFunctionLibraryRuntime.
diff --git a/tensorflow/core/kernels/queue_op.h b/tensorflow/core/kernels/queue_op.h
index ad606803ee..6c19f9841c 100644
--- a/tensorflow/core/kernels/queue_op.h
+++ b/tensorflow/core/kernels/queue_op.h
@@ -43,6 +43,7 @@ class QueueOp : public ResourceOpKernel<QueueInterface> {
 
   void Compute(OpKernelContext* context) override {
     ResourceOpKernel<QueueInterface>::Compute(context);
+    mutex_lock l(mu_);
     if (resource_ && context->track_allocations()) {
       context->record_persistent_memory_allocation(resource_->MemoryUsed());
     }
-- 
GitLab


From e8c0e68d12cc959bf041782e2fdc909be663b2b8 Mon Sep 17 00:00:00 2001
From: Alex Wiltschko <alex.bw@gmail.com>
Date: Fri, 30 Mar 2018 10:00:31 -0700
Subject: [PATCH 754/960] Adding optional generated code printing in the demo
 notebook (#18122)

---
 .../notebooks/dev_summit_2018_demo.ipynb      | 177 +++++++-----------
 1 file changed, 63 insertions(+), 114 deletions(-)

diff --git a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
index 3129a39a4b..d62390494b 100644
--- a/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
+++ b/tensorflow/contrib/autograph/examples/notebooks/dev_summit_2018_demo.ipynb
@@ -167,9 +167,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 413
@@ -191,7 +189,7 @@
       "source": [
         "print(autograph.to_code(g))"
       ],
-      "execution_count": 4,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -243,9 +241,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 53
@@ -277,7 +273,7 @@
         "  print('g(9) = %s' % g(9))\n",
         "  print('tf_g(9) = %s' % tf_g_result)"
       ],
-      "execution_count": 5,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -323,9 +319,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 35
@@ -360,9 +354,12 @@
         "  with tf.Session() as sess:\n",
         "    result = sess.run(tf_sum_even(tf.constant([10, 12, 15, 20])))\n",
         "\n",
-        "  print('Sum of even numbers: %s' % result)"
+        "  print('Sum of even numbers: %s' % result)\n",
+        "  \n",
+        "# Uncomment the line below to print the generated graph code\n",
+        "# print(autograph.to_code(sum_even))"
       ],
-      "execution_count": 6,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -413,9 +410,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 53
@@ -445,9 +440,12 @@
         "    try:\n",
         "      print(sess.run(tf_f(tf.constant(0))))\n",
         "    except tf.errors.InvalidArgumentError as e:\n",
-        "      print('Got error message: %s' % e.message)"
+        "      print('Got error message: %s' % e.message)\n",
+        "      \n",
+        "# Uncomment the line below to print the generated graph code\n",
+        "# print(autograph.to_code(f))"
       ],
-      "execution_count": 7,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -479,9 +477,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 35
@@ -512,9 +508,12 @@
         "tf_print_sign = autograph.to_graph(print_sign)\n",
         "with tf.Graph().as_default():\n",
         "  with tf.Session() as sess:\n",
-        "    sess.run(tf_print_sign(tf.constant(1)))"
+        "    sess.run(tf_print_sign(tf.constant(1)))\n",
+        "    \n",
+        "# Uncomment the line below to print the generated graph code\n",
+        "# print(autograph.to_code(print_sign))"
       ],
-      "execution_count": 8,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -545,9 +544,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 35
@@ -579,9 +576,12 @@
         "tf_f = autograph.to_graph(f)\n",
         "with tf.Graph().as_default():\n",
         "  with tf.Session() as sess:\n",
-        "    print(sess.run(tf_f(tf.constant(5))))"
+        "    print(sess.run(tf_f(tf.constant(5))))\n",
+        "    \n",
+        "# Uncomment the line below to print the generated graph code\n",
+        "# print(autograph.to_code(f))"
       ],
-      "execution_count": 9,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -612,9 +612,7 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 1
-            }
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 53
@@ -661,9 +659,12 @@
         "with tf.Graph().as_default():  \n",
         "  with tf.Session() as sess:\n",
         "    n = tf.constant(50)\n",
-        "    sess.run(tf_print_primes(n))"
+        "    sess.run(tf_print_primes(n))\n",
+        "    \n",
+        "# Uncomment the line below to print the generated graph code\n",
+        "# print(autograph.to_code(print_primes))"
       ],
-      "execution_count": 10,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -978,15 +979,9 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 13
-            },
-            {
-              "item_id": 14
-            },
-            {
-              "item_id": 15
-            }
+            {},
+            {},
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 988
@@ -1037,7 +1032,7 @@
         "    plt.ylabel('Accuracy')\n",
         "    plt.show()"
       ],
-      "execution_count": 17,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -1419,75 +1414,29 @@
             "wait_interval": 0
           },
           "output_extras": [
-            {
-              "item_id": 12
-            },
-            {
-              "item_id": 13
-            },
-            {
-              "item_id": 14
-            },
-            {
-              "item_id": 15
-            },
-            {
-              "item_id": 16
-            },
-            {
-              "item_id": 17
-            },
-            {
-              "item_id": 18
-            },
-            {
-              "item_id": 19
-            },
-            {
-              "item_id": 20
-            },
-            {
-              "item_id": 21
-            },
-            {
-              "item_id": 23
-            },
-            {
-              "item_id": 24
-            },
-            {
-              "item_id": 25
-            },
-            {
-              "item_id": 26
-            },
-            {
-              "item_id": 27
-            },
-            {
-              "item_id": 28
-            },
-            {
-              "item_id": 29
-            },
-            {
-              "item_id": 30
-            },
-            {
-              "item_id": 31
-            },
-            {
-              "item_id": 32
-            },
-            {
-              "item_id": 33
-            },
-            {
-              "item_id": 34
-            },
-            {
-              "item_id": 35
-            }
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {},
+            {}
           ],
           "base_uri": "https://localhost:8080/",
           "height": 668
@@ -1553,7 +1502,7 @@
         "    # Run the inference loop.\n",
         "    run_input_loop(sess, inference_ops, color_name_placeholder)"
       ],
-      "execution_count": 22,
+      "execution_count": 0,
       "outputs": [
         {
           "output_type": "stream",
@@ -1967,4 +1916,4 @@
       ]
     }
   ]
-}
\ No newline at end of file
+}
-- 
GitLab


From 8328d84fb54c59b93161950e50709b401576bcc3 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Sat, 31 Mar 2018 01:00:49 +0800
Subject: [PATCH 755/960] Fix several formatting issues in python api guides
 (#18121)

* Fix several formatting in python api guides

* Fix minor format
---
 .../api_guides/python/contrib.graph_editor.md  | 18 +++++++++---------
 .../docs_src/api_guides/python/io_ops.md       |  4 ++--
 tensorflow/docs_src/api_guides/python/nn.md    | 18 +++++++++---------
 3 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md b/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md
index de4f126507..20fe88a799 100644
--- a/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md
+++ b/tensorflow/docs_src/api_guides/python/contrib.graph_editor.md
@@ -61,21 +61,21 @@ A subgraph can be created in several ways:
 
 * using a list of ops:
 
-```python
-my_sgv = ge.sgv(ops)
-```
+  ```python
+  my_sgv = ge.sgv(ops)
+  ```
 
 * from a name scope:
 
-```python
-my_sgv = ge.sgv_scope("foo/bar", graph=tf.get_default_graph())
-```
+  ```python
+  my_sgv = ge.sgv_scope("foo/bar", graph=tf.get_default_graph())
+  ```
 
 * using regular expression:
 
-```python
-my_sgv = ge.sgv("foo/.*/.*read$", graph=tf.get_default_graph())
-```
+  ```python
+  my_sgv = ge.sgv("foo/.*/.*read$", graph=tf.get_default_graph())
+  ```
 
 Note that the Graph Editor is meant to manipulate several graphs at the same
 time, typically during transform or copy operation. For that reason,
diff --git a/tensorflow/docs_src/api_guides/python/io_ops.md b/tensorflow/docs_src/api_guides/python/io_ops.md
index 94cf0de32a..86b4b39409 100644
--- a/tensorflow/docs_src/api_guides/python/io_ops.md
+++ b/tensorflow/docs_src/api_guides/python/io_ops.md
@@ -8,7 +8,7 @@ Note: Functions taking `Tensor` arguments can also take anything accepted by
 ## Placeholders
 
 TensorFlow provides a placeholder operation that must be fed with data
-on execution.  For more info, see the section on @{$reading_data#feeding$Feeding data}.
+on execution.  For more info, see the section on @{$reading_data#Feeding$Feeding data}.
 
 *   @{tf.placeholder}
 *   @{tf.placeholder_with_default}
@@ -42,7 +42,7 @@ formats into tensors.
 
 ### Example protocol buffer
 
-TensorFlow's @{$reading_data#standard-tensorflow-format$recommended format for training examples}
+TensorFlow's @{$reading_data#standard_tensorflow_format$recommended format for training examples}
 is serialized `Example` protocol buffers, [described
 here](https://www.tensorflow.org/code/tensorflow/core/example/example.proto).
 They contain `Features`, [described
diff --git a/tensorflow/docs_src/api_guides/python/nn.md b/tensorflow/docs_src/api_guides/python/nn.md
index 8e6fd1cff9..8d8daaae19 100644
--- a/tensorflow/docs_src/api_guides/python/nn.md
+++ b/tensorflow/docs_src/api_guides/python/nn.md
@@ -89,7 +89,7 @@ bottom. Note that this is different from existing libraries such as cuDNN and
 Caffe, which explicitly specify the number of padded pixels and always pad the
 same number of pixels on both sides.
 
-For the `'VALID`' scheme, the output height and width are computed as:
+For the `'VALID'` scheme, the output height and width are computed as:
 
     out_height = ceil(float(in_height - filter_height + 1) / float(strides[1]))
     out_width  = ceil(float(in_width - filter_width + 1) / float(strides[2]))
@@ -98,10 +98,10 @@ and no padding is used.
 
 Given the output size and the padding, the output can be computed as
 
-    output[b, i, j, :] =
-        sum_{di, dj} input[b, strides[1] * i + di - pad_top,
-                           strides[2] * j + dj - pad_left, ...] *
-                     filter[di, dj, ...]
+$$    output[b, i, j, :] =
+        sum_{d_i, d_j} input[b, strides[1] * i + d_i - pad_{top},\
+                           strides[2] * j + d_j - pad_{left}, ...] *
+                     filter[d_i, d_j,\ ...]$$
 
 where any value outside the original input image region are considered zero (
 i.e. we pad zero values around the border of the image).
@@ -161,12 +161,12 @@ Morphological operators are non-linear filters used in image processing.
 ](https://en.wikipedia.org/wiki/Dilation_(morphology))
 is the max-sum counterpart of standard sum-product convolution:
 
-    output[b, y, x, c] =
+$$    output[b, y, x, c] =
         max_{dy, dx} input[b,
                            strides[1] * y + rates[1] * dy,
                            strides[2] * x + rates[2] * dx,
                            c] +
-                     filter[dy, dx, c]
+                     filter[dy, dx, c]$$
 
 The `filter` is usually called structuring function. Max-pooling is a special
 case of greyscale morphological dilation when the filter assumes all-zero
@@ -176,12 +176,12 @@ values (a.k.a. flat structuring function).
 ](https://en.wikipedia.org/wiki/Erosion_(morphology))
 is the min-sum counterpart of standard sum-product convolution:
 
-    output[b, y, x, c] =
+$$    output[b, y, x, c] =
         min_{dy, dx} input[b,
                            strides[1] * y - rates[1] * dy,
                            strides[2] * x - rates[2] * dx,
                            c] -
-                     filter[dy, dx, c]
+                     filter[dy, dx, c]$$
 
 Dilation and erosion are dual to each other. The dilation of the input signal
 `f` by the structuring signal `g` is equal to the negation of the erosion of
-- 
GitLab


From 0ecaf4d775f96b6c050de478f17dd647ff7e80e3 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 10:15:11 -0700
Subject: [PATCH 756/960] [XLA] Add IsConstant to the local Python client.

PiperOrigin-RevId: 191080094
---
 .../xla/python/local_computation_builder.cc   |  11 ++
 .../xla/python/local_computation_builder.h    |   7 +
 .../xla/python/local_computation_builder.i    | 164 ++++++++++--------
 tensorflow/compiler/xla/python/xla_client.py  |  14 ++
 .../compiler/xla/python/xla_client_test.py    |  11 ++
 5 files changed, 130 insertions(+), 77 deletions(-)

diff --git a/tensorflow/compiler/xla/python/local_computation_builder.cc b/tensorflow/compiler/xla/python/local_computation_builder.cc
index b21ab3044f..2bacc6a914 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.cc
+++ b/tensorflow/compiler/xla/python/local_computation_builder.cc
@@ -521,6 +521,17 @@ ComputationDataHandle LocalComputationBuilder::Conditional(
                               false_computation.computation());
 }
 
+StatusOr<bool> LocalComputationBuilder::IsConstant(
+    const ComputationDataHandle& operand, int64 num_parameters) {
+  return builder_.IsConstant(operand, num_parameters);
+}
+
+StatusOr<std::unique_ptr<Literal>> LocalComputationBuilder::ComputeConstant(
+    const ComputationDataHandle& operand, const Layout* output_layout,
+    tensorflow::gtl::ArraySlice<Literal> parameters) {
+  return builder_.ComputeConstant(operand, output_layout, parameters);
+}
+
 #define _FORWARD(method_name, return_sig, args_sig, args)    \
   return_sig LocalComputationBuilder::method_name args_sig { \
     return builder_.method_name args;                        \
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.h b/tensorflow/compiler/xla/python/local_computation_builder.h
index a7375c8965..31046e60f1 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.h
+++ b/tensorflow/compiler/xla/python/local_computation_builder.h
@@ -268,6 +268,13 @@ class LocalComputationBuilder {
                                     const ComputationDataHandle& false_operand,
                                     const LocalComputation& false_computation);
 
+  StatusOr<bool> IsConstant(const ComputationDataHandle& operand,
+                            int64 num_parameters);
+
+  StatusOr<std::unique_ptr<Literal> > ComputeConstant(
+      const ComputationDataHandle& operand, const Layout* output_layout,
+      tensorflow::gtl::ArraySlice<Literal> parameters);
+
 #define _FORWARD(method_name, return_sig, args_sig) \
   return_sig method_name args_sig;
 
diff --git a/tensorflow/compiler/xla/python/local_computation_builder.i b/tensorflow/compiler/xla/python/local_computation_builder.i
index 8f231d1a12..ac792e8189 100644
--- a/tensorflow/compiler/xla/python/local_computation_builder.i
+++ b/tensorflow/compiler/xla/python/local_computation_builder.i
@@ -182,7 +182,7 @@ tensorflow::ImportNumpy();
 %typemap(in) const ComputationDataHandle& (ComputationDataHandle temp) {
   const int64 handle = numpy::PyIntOrPyLongToLong($input);
   if (handle == -1 && PyErr_Occurred()) {
-    return NULL;
+    SWIG_fail;
   }
   temp.set_handle(handle);
   $1 = &temp;
@@ -201,7 +201,7 @@ tensorflow::ImportNumpy();
     }
   } else {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
 }
 
@@ -211,7 +211,7 @@ tensorflow::ImportNumpy();
     $result = numpy::PyObjectFromXlaLiteral(*value);
   } else {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
 }
 
@@ -224,7 +224,7 @@ tensorflow::ImportNumpy();
     }
   } else {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
 }
 
@@ -233,7 +233,16 @@ tensorflow::ImportNumpy();
     $result = numpy::PyShapeInfoFromXlaShape($1.ConsumeValueOrDie());
   } else {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
+  }
+}
+
+%typemap(out) StatusOr<bool> {
+  if ($1.ok()) {
+    $result = PyBool_FromLong($1.ConsumeValueOrDie());
+  } else {
+    PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
+    SWIG_fail;
   }
 }
 
@@ -241,7 +250,7 @@ tensorflow::ImportNumpy();
   if (!$1.ok()) {
     PyErr_SetString(
         PyExc_RuntimeError, $1.ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
   Py_INCREF(Py_None);
   $result = Py_None;
@@ -253,7 +262,7 @@ tensorflow::ImportNumpy();
     (std::vector<int64> temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   temps.resize(size);
@@ -265,13 +274,13 @@ tensorflow::ImportNumpy();
           PyExc_TypeError,
           "Argument sequence element cannot be converted to int");
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     temps[i] = numpy::PyIntOrPyLongToLong(py_int);
     if (temps[i] == -1 && PyErr_Occurred()) {
       Py_DECREF(py_int);
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     Py_DECREF(py_int);
     Py_DECREF(o);
@@ -285,7 +294,7 @@ tensorflow::ImportNumpy();
     (std::vector<ComputationDataHandle> temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   temps.resize(size);
@@ -296,13 +305,13 @@ tensorflow::ImportNumpy();
       PyErr_SetString(
           PyExc_TypeError,
           "Argument sequence element cannot be converted to int");
-      return NULL;
+      SWIG_fail;
     }
     const int64 handle = numpy::PyIntOrPyLongToLong(py_int);
     if (handle == -1 && PyErr_Occurred()) {
       Py_DECREF(py_int);
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     temps[i].set_handle(handle);
     Py_DECREF(py_int);
@@ -317,7 +326,7 @@ tensorflow::ImportNumpy();
     (std::vector<LocalShapedBuffer*> temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   temps.reserve(size);
@@ -326,7 +335,7 @@ tensorflow::ImportNumpy();
     LocalShapedBuffer* lsbp;
     if ((SWIG_ConvertPtr(o, (void**) &lsbp, $descriptor(xla::swig::LocalShapedBuffer*),
                          SWIG_POINTER_EXCEPTION)) == -1) {
-      return NULL;
+      SWIG_fail;
     }
     temps.push_back(lsbp);
     Py_DECREF(o);
@@ -340,7 +349,7 @@ tensorflow::ImportNumpy();
   literal_status = numpy::XlaLiteralFromPyObject($input);
   if (!literal_status.ok()) {
     PyErr_SetString(PyExc_RuntimeError, literal_status.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
   $1 = literal_status.ValueOrDie().get();
 }
@@ -352,7 +361,7 @@ tensorflow::ImportNumpy();
 %typemap(out) StatusOr< std::unique_ptr<Literal> > {
   if (!$1.ok()) {
     PyErr_SetString(PyExc_RuntimeError, $1.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
   $result = numpy::PyObjectFromXlaLiteral(*$1.ValueOrDie());
 }
@@ -360,7 +369,7 @@ tensorflow::ImportNumpy();
 %typemap(in) const std::vector<Literal>& (std::vector<Literal> temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   for (int i = 0; i < size; ++i) {
@@ -369,7 +378,7 @@ tensorflow::ImportNumpy();
     if (!literal_status.ok()) {
       PyErr_SetString(PyExc_RuntimeError, literal_status.status().ToString().c_str());
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     temps.push_back(std::move(*literal_status.ConsumeValueOrDie()));
     Py_DECREF(o);
@@ -383,7 +392,7 @@ tensorflow::ImportNumpy();
   StatusOr<OpMetadata> statusor = numpy::OpMetadataFromPyObject($input);
   if (!statusor.ok()) {
     PyErr_SetString(PyExc_RuntimeError, statusor.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
   temp = std::move(statusor).ValueOrDie();
   $1 = &temp;
@@ -395,7 +404,7 @@ tensorflow::ImportNumpy();
   StatusOr<Shape> statusor = numpy::XlaShapeFromPyShape($input);
   if (!statusor.ok()) {
     PyErr_SetString(PyExc_RuntimeError, statusor.status().ToString().c_str());
-    return NULL;
+    SWIG_fail;
   }
   temp = std::move(statusor).ValueOrDie();
   $1 = &temp;
@@ -410,7 +419,7 @@ tensorflow::ImportNumpy();
     StatusOr<Shape> statusor = numpy::XlaShapeFromPyShape($input);
     if (!statusor.ok()) {
       PyErr_SetString(PyExc_RuntimeError, statusor.status().ToString().c_str());
-      return NULL;
+      SWIG_fail;
     }
     temp = std::move(statusor).ValueOrDie();
     $1 = &temp;
@@ -424,7 +433,7 @@ tensorflow::ImportNumpy();
 %typemap(in) const std::vector<Shape>& (std::vector<Shape> temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   for (int i = 0; i < size; ++i) {
@@ -433,7 +442,7 @@ tensorflow::ImportNumpy();
     Py_DECREF(o);
     if (!statusor.ok()) {
       PyErr_SetString(PyExc_RuntimeError, statusor.status().ToString().c_str());
-      return NULL;
+      SWIG_fail;
     }
     temps.push_back(statusor.ConsumeValueOrDie());
   }
@@ -444,7 +453,7 @@ tensorflow::ImportNumpy();
     std::vector<tensorflow::gtl::optional<Shape> > temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   for (int i = 0; i < size; ++i) {
@@ -456,7 +465,7 @@ tensorflow::ImportNumpy();
       Py_DECREF(o);
       if (!statusor.ok()) {
         PyErr_SetString(PyExc_RuntimeError, statusor.status().ToString().c_str());
-        return NULL;
+        SWIG_fail;
       }
       temps.push_back(statusor.ConsumeValueOrDie());
     }
@@ -470,18 +479,18 @@ tensorflow::ImportNumpy();
   PyObject* py_int = numpy::PyNumberToPyInt($input);
   if (!py_int) {
     PyErr_SetString(PyExc_TypeError, "Argument cannot be converted to int");
-    return NULL;
+    SWIG_fail;
   }
   const long value = numpy::PyIntOrPyLongToLong(py_int);
   if (value == -1 && PyErr_Occurred()) {
     Py_DECREF(py_int);
-    return NULL;
+    SWIG_fail;
   }
   if (!PrimitiveType_IsValid(value)) {
     PyErr_SetString(
         PyExc_TypeError, "Argument not valid for PrimitiveType enum");
     Py_DECREF(py_int);
-    return NULL;
+    SWIG_fail;
   }
   $1 = static_cast<PrimitiveType>(value);
 }
@@ -492,19 +501,19 @@ tensorflow::ImportNumpy();
     (std::vector<std::pair<int64, int64> > temps) {
   if (!PySequence_Check($input)) {
     PyErr_SetString(PyExc_TypeError, "Argument is not a sequence");
-    return NULL;
+    SWIG_fail;
   }
   const int size = PySequence_Size($input);
   temps.reserve(size);
   for (int i = 0; i < size; ++i) {
     PyObject* o = PySequence_GetItem($input, i);
     if (!o) {
-      return NULL;
+      SWIG_fail;
     }
     PyObject* first = PyTuple_GetItem(o, 0);
     if (!first) {
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     PyObject* first_pyint = numpy::PyNumberToPyInt(first);
     if (!first_pyint) {
@@ -512,13 +521,13 @@ tensorflow::ImportNumpy();
           PyExc_TypeError,
           "First pair item cannot be converted to int");
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     PyObject* second = PyTuple_GetItem(o, 1);
     if (!second) {
       Py_DECREF(o);
       Py_DECREF(first_pyint);
-      return NULL;
+      SWIG_fail;
     }
     PyObject* second_pyint = numpy::PyNumberToPyInt(second);
     if (!second_pyint) {
@@ -527,21 +536,21 @@ tensorflow::ImportNumpy();
           "Second pair item cannot be converted to int");
       Py_DECREF(o);
       Py_DECREF(first_pyint);
-      return NULL;
+      SWIG_fail;
     }
     const int64 first_value = numpy::PyIntOrPyLongToLong(first_pyint);
     if (first_value == -1 && PyErr_Occurred()) {
       Py_DECREF(o);
       Py_DECREF(first_pyint);
       Py_DECREF(second_pyint);
-      return NULL;
+      SWIG_fail;
     }
     const int64 second_value = numpy::PyIntOrPyLongToLong(second_pyint);
     if (second_value == -1 && PyErr_Occurred()) {
       Py_DECREF(o);
       Py_DECREF(first_pyint);
       Py_DECREF(second_pyint);
-      return NULL;
+      SWIG_fail;
     }
     temps.push_back(std::make_pair(first_value, second_value));
     Py_DECREF(o);
@@ -559,26 +568,26 @@ tensorflow::ImportNumpy();
   PyObject* lhs_contracting_dimensions = PyObject_GetAttrString(
       $input, "lhs_contracting_dimensions");
   if (!lhs_contracting_dimensions) {
-    return NULL;
+    SWIG_fail;
   }
 
   length = PySequence_Size(lhs_contracting_dimensions);
   if (length == -1) {
     Py_DECREF(lhs_contracting_dimensions);
-    return NULL;
+    SWIG_fail;
   }
 
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(lhs_contracting_dimensions, i);
     if (!item) {
       Py_DECREF(lhs_contracting_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(lhs_contracting_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_lhs_contracting_dimensions(dimension);
     Py_DECREF(item);
@@ -589,26 +598,26 @@ tensorflow::ImportNumpy();
   PyObject* rhs_contracting_dimensions = PyObject_GetAttrString(
       $input, "rhs_contracting_dimensions");
   if (!lhs_contracting_dimensions) {
-    return NULL;
+    SWIG_fail;
   }
 
   length = PySequence_Size(rhs_contracting_dimensions);
   if (length == -1) {
     Py_DECREF(rhs_contracting_dimensions);
-    return NULL;
+    SWIG_fail;
   }
 
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(rhs_contracting_dimensions, i);
     if (!item) {
       Py_DECREF(rhs_contracting_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(rhs_contracting_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_rhs_contracting_dimensions(dimension);
     Py_DECREF(item);
@@ -619,26 +628,26 @@ tensorflow::ImportNumpy();
   PyObject* lhs_batch_dimensions = PyObject_GetAttrString(
       $input, "lhs_batch_dimensions");
   if (!lhs_batch_dimensions) {
-    return NULL;
+    SWIG_fail;
   }
 
   length = PySequence_Size(lhs_batch_dimensions);
   if (length == -1) {
     Py_DECREF(lhs_batch_dimensions);
-    return NULL;
+    SWIG_fail;
   }
 
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(lhs_batch_dimensions, i);
     if (!item) {
       Py_DECREF(lhs_batch_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(lhs_batch_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_lhs_batch_dimensions(dimension);
     Py_DECREF(item);
@@ -649,26 +658,26 @@ tensorflow::ImportNumpy();
   PyObject* rhs_batch_dimensions = PyObject_GetAttrString(
       $input, "rhs_batch_dimensions");
   if (!rhs_batch_dimensions) {
-    return NULL;
+    SWIG_fail;
   }
 
   length = PySequence_Size(rhs_batch_dimensions);
   if (length == -1) {
     Py_DECREF(rhs_batch_dimensions);
-    return NULL;
+    SWIG_fail;
   }
 
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(rhs_batch_dimensions, i);
     if (!item) {
       Py_DECREF(rhs_batch_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(rhs_batch_dimensions);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_rhs_batch_dimensions(dimension);
     Py_DECREF(item);
@@ -684,20 +693,20 @@ tensorflow::ImportNumpy();
     (PaddingConfig padding_config) {
   PyObject* dimensions = PyObject_GetAttrString($input, "dimensions");
   if (!dimensions) {
-    return NULL;
+    SWIG_fail;
   }
 
   int length = PySequence_Size(dimensions);
   if (length == -1) {
     Py_DECREF(dimensions);
-    return NULL;
+    SWIG_fail;
   }
 
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(dimensions, i);
     if (!item) {
       Py_DECREF(dimensions);
-      return NULL;
+      SWIG_fail;
     }
     int64 edge_padding_low, edge_padding_high, interior_padding;
     if (!GetIntAttr(item, "edge_padding_low", &edge_padding_low)
@@ -705,7 +714,7 @@ tensorflow::ImportNumpy();
         || !GetIntAttr(item, "interior_padding", &interior_padding)) {
       Py_DECREF(item);
       Py_DECREF(dimensions);
-      return NULL;
+      SWIG_fail;
     }
     Py_DECREF(item);
 
@@ -727,32 +736,32 @@ tensorflow::ImportNumpy();
   int64 value;
 
   if (!GetIntAttr($input, "input_batch_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_input_batch_dimension(value);
 
   if (!GetIntAttr($input, "input_feature_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_input_feature_dimension(value);
 
   if (!GetIntAttr($input, "output_batch_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_output_batch_dimension(value);
 
   if (!GetIntAttr($input, "output_feature_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_output_feature_dimension(value);
 
   if (!GetIntAttr($input, "kernel_output_feature_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_kernel_output_feature_dimension(value);
 
   if (!GetIntAttr($input, "kernel_input_feature_dimension", &value)) {
-    return NULL;
+    SWIG_fail;
   }
   dimension_numbers.set_kernel_input_feature_dimension(value);
 
@@ -761,24 +770,24 @@ tensorflow::ImportNumpy();
 
   o = PyObject_GetAttrString($input, "input_spatial_dimensions");
   if (!o) {
-    return NULL;
+    SWIG_fail;
   }
   length = PySequence_Size(o);
   if (length == -1) {
     Py_DECREF(o);
-    return NULL;
+    SWIG_fail;
   }
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(o, i);
     if (!item) {
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_input_spatial_dimensions(dimension);
     Py_DECREF(item);
@@ -787,24 +796,24 @@ tensorflow::ImportNumpy();
 
   o = PyObject_GetAttrString($input, "kernel_spatial_dimensions");
   if (!o) {
-    return NULL;
+    SWIG_fail;
   }
   length = PySequence_Size(o);
   if (length == -1) {
     Py_DECREF(o);
-    return NULL;
+    SWIG_fail;
   }
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(o, i);
     if (!item) {
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_kernel_spatial_dimensions(dimension);
     Py_DECREF(item);
@@ -813,24 +822,24 @@ tensorflow::ImportNumpy();
 
   o = PyObject_GetAttrString($input, "output_spatial_dimensions");
   if (!o) {
-    return NULL;
+    SWIG_fail;
   }
   length = PySequence_Size(o);
   if (length == -1) {
     Py_DECREF(o);
-    return NULL;
+    SWIG_fail;
   }
   for (int i = 0; i < length; ++i) {
     PyObject* item = PySequence_GetItem(o, i);
     if (!item) {
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     const int64 dimension = numpy::PyIntOrPyLongToLong(item);
     if (dimension == -1 && PyErr_Occurred()) {
       Py_DECREF(item);
       Py_DECREF(o);
-      return NULL;
+      SWIG_fail;
     }
     dimension_numbers.add_output_spatial_dimensions(dimension);
     Py_DECREF(item);
@@ -865,12 +874,12 @@ tensorflow::ImportNumpy();
 
     PyObject* o = PyObject_GetAttrString($input, "hlo_profile");
     if (o == NULL) {
-      return NULL;
+      SWIG_fail;
     }
     if (o != Py_None) {
       if (!PyBool_Check(o)) {
         PyErr_SetString(PyExc_TypeError, "ExecutableBuildOptions.hlo_profile must be a bool or None.");
-        return NULL;
+        SWIG_fail;
       }
       build_options.set_hlo_profile(o == Py_True);
     }
@@ -885,7 +894,7 @@ tensorflow::ImportNumpy();
       if (!statusor.ok()) {
         PyErr_SetString(PyExc_TypeError, tensorflow::strings::StrCat("ExecutableBuildOptions.result_shape could not be created from Python shape value: ", statusor.status().ToString()).c_str());
         Py_DECREF(o);
-        return NULL;
+        SWIG_fail;
       }
       build_options.set_result_layout(statusor.ValueOrDie());
     }
@@ -951,6 +960,7 @@ tensorflow::ImportNumpy();
 %unignore xla::swig::LocalComputationBuilder::RngBernoulli;
 %unignore xla::swig::LocalComputationBuilder::While;
 %unignore xla::swig::LocalComputationBuilder::Conditional;
+%unignore xla::swig::LocalComputationBuilder::IsConstant;
 %unignore xla::swig::LocalComputationBuilder::Eq;
 %unignore xla::swig::LocalComputationBuilder::Ne;
 %unignore xla::swig::LocalComputationBuilder::Ge;
diff --git a/tensorflow/compiler/xla/python/xla_client.py b/tensorflow/compiler/xla/python/xla_client.py
index e548d420f4..9c81f6439d 100644
--- a/tensorflow/compiler/xla/python/xla_client.py
+++ b/tensorflow/compiler/xla/python/xla_client.py
@@ -1028,6 +1028,20 @@ class ComputationBuilder(object):
             _unwrap_data_handle(false_operand),
             false_computation.c_local_computation))
 
+  def IsConstant(self, operand, num_parameters=0):
+    """Enqueues an IsConstant operation onto the computation.
+
+    Args:
+      operand: a ComputationDataHandle to test.
+      num_parameters: optional int, number of computation parameters to treat as
+        constant (default 0).
+
+    Returns: bool indicating whether `operand` is a compile-time constant,
+      meaning its value does not depend on parameters with index greater than or
+      equal to `num_parameters`.
+    """
+    return self._client.IsConstant(_unwrap_data_handle(operand), num_parameters)
+
   def Dot(self, lhs, rhs):
     """Enqueues a dot operation onto the computation.
 
diff --git a/tensorflow/compiler/xla/python/xla_client_test.py b/tensorflow/compiler/xla/python/xla_client_test.py
index 4c16c1f8b0..d97264ea64 100644
--- a/tensorflow/compiler/xla/python/xla_client_test.py
+++ b/tensorflow/compiler/xla/python/xla_client_test.py
@@ -855,6 +855,17 @@ class SingleOpTest(LocalComputationTest):
     self.assertTrue(np.all(lo <= result))
     self.assertTrue(np.all(result < hi))
 
+  def testIsConstant(self):
+    c = self._NewComputation()
+    a = c.ConstantS32Scalar(3)
+    b = c.ConstantS32Scalar(1)
+    x = c.ParameterFromNumpy(NumpyArrayS32(0))
+    const_expr = c.Sub(b, a)
+    non_const_expr = c.Mul(const_expr, x)
+    self.assertTrue(c.IsConstant(const_expr))
+    self.assertFalse(c.IsConstant(non_const_expr))
+    # self.assertTrue(c.IsConstant(c.Sub(c.Add(x, a), x)))  # TODO(b/77245564)
+
 
 class EmbeddedComputationsTest(LocalComputationTest):
   """Tests for XLA graphs with embedded computations (such as maps)."""
-- 
GitLab


From 65669c0e12ba7aaa43a34db3798b8ac8fd97ba3e Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Fri, 30 Mar 2018 10:41:48 -0700
Subject: [PATCH 757/960] Fix issue with Luong attention when scale=True and
 dtype of tf.float16/tf.float64 (#18106)

* Fix issue with Luong attention when scale=True and dtype=tf.float16/tf.float64

This fix tries to address the issue raised in 18099 where
Luong throws a ValueError when scale=True and dtype is not tf.float32.

This fix addresses the issue with the additional test case added.

This fix fixes 18099.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix pylint issue

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for Luong attention with scale=True and dtype=float16/float64

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add assertEqual to confirm the dtypes of the output

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../kernel_tests/attention_wrapper_test.py    | 36 +++++++++++++++++++
 .../seq2seq/python/ops/attention_wrapper.py   |  3 +-
 2 files changed, 38 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
index 07b3ad71d4..d508cf3f9d 100644
--- a/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
+++ b/tensorflow/contrib/seq2seq/python/kernel_tests/attention_wrapper_test.py
@@ -353,6 +353,42 @@ class AttentionWrapperTest(test.TestCase):
         attention_mechanism_depth=9,
         name='testLuongNotNormalized')
 
+  def testLuongScaledDType(self):
+    # Test case for GitHub issue 18099
+    for dtype in [np.float16, np.float32, np.float64]:
+      num_units = 128
+      encoder_outputs = array_ops.placeholder(dtype, shape=[64, None, 256])
+      encoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64])
+      decoder_inputs = array_ops.placeholder(dtype, shape=[64, None, 128])
+      decoder_sequence_length = array_ops.placeholder(dtypes.int32, shape=[64])
+      batch_size = 64
+      attention_mechanism = wrapper.LuongAttention(
+          num_units=num_units,
+          memory=encoder_outputs,
+          memory_sequence_length=encoder_sequence_length,
+          scale=True,
+          dtype=dtype,
+      )
+      cell = rnn_cell.LSTMCell(num_units)
+      cell = wrapper.AttentionWrapper(cell, attention_mechanism)
+
+      helper = helper_py.TrainingHelper(decoder_inputs,
+                                        decoder_sequence_length)
+      my_decoder = basic_decoder.BasicDecoder(
+          cell=cell,
+          helper=helper,
+          initial_state=cell.zero_state(
+              dtype=dtype, batch_size=batch_size))
+
+      final_outputs, final_state, _ = decoder.dynamic_decode(my_decoder)
+      self.assertTrue(
+          isinstance(final_outputs, basic_decoder.BasicDecoderOutput))
+      self.assertEqual(final_outputs.rnn_output.dtype, dtype)
+      self.assertTrue(
+          isinstance(final_state, wrapper.AttentionWrapperState))
+      self.assertTrue(
+          isinstance(final_state.cell_state, rnn_cell.LSTMStateTuple))
+
   def testLuongScaled(self):
     create_attention_mechanism = functools.partial(
         wrapper.LuongAttention, scale=True)
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index be53779826..9e0d69593f 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -339,7 +339,8 @@ def _luong_score(query, keys, scale):
   if scale:
     # Scalar used in weight scaling
     g = variable_scope.get_variable(
-        "attention_g", dtype=dtype, initializer=1.)
+        "attention_g", dtype=dtype,
+        initializer=init_ops.ones_initializer, shape=())
     score = g * score
   return score
 
-- 
GitLab


From a1aa918af9c5c1c3f777abcc7dc9ab2929bb1001 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 10:47:58 -0700
Subject: [PATCH 758/960] Check for errors returned by ExecuteOnStream before
 calling BlockHostUntilDone. If ExecuteOnStream fails, BlockHostUntilDone will
 also fail in a way that makes debugging much harder.

RELNOTES: n/a
PiperOrigin-RevId: 191085159
---
 tensorflow/compiler/xla/service/executable.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/service/executable.cc b/tensorflow/compiler/xla/service/executable.cc
index be92b1629a..471d2fd6ce 100644
--- a/tensorflow/compiler/xla/service/executable.cc
+++ b/tensorflow/compiler/xla/service/executable.cc
@@ -80,6 +80,7 @@ StatusOr<std::unique_ptr<ShapedBuffer>> Executable::ExecuteOnStreamWrapper(
 
   StatusOr<std::unique_ptr<ShapedBuffer>> return_value =
       ExecuteOnStream(run_options, arguments, profile_ptr.get());
+  TF_RETURN_IF_ERROR(return_value.status());
 
   if (profile != nullptr) {
     VLOG(1) << "enqueueing 'stop timer' and blocking host until done...";
-- 
GitLab


From 528c64665f2c3220acb59031926274403b96dddb Mon Sep 17 00:00:00 2001
From: Sourabh Bajaj <sourabhbajaj@google.com>
Date: Fri, 30 Mar 2018 10:50:18 -0700
Subject: [PATCH 759/960] Create a wrapper for bfloat16 scope so that users
 don't need the custom getter.

PiperOrigin-RevId: 191085552
---
 tensorflow/contrib/tpu/BUILD                  | 12 +++
 tensorflow/contrib/tpu/__init__.py            |  1 +
 tensorflow/contrib/tpu/python/tpu/bfloat16.py | 78 +++++++++++++++++++
 .../contrib/tpu/python/tpu/bfloat16_test.py   | 52 +++++++++++++
 4 files changed, 143 insertions(+)
 create mode 100644 tensorflow/contrib/tpu/python/tpu/bfloat16.py
 create mode 100644 tensorflow/contrib/tpu/python/tpu/bfloat16_test.py

diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index 3e32a7a85c..b267cceef1 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -159,6 +159,7 @@ py_library(
     name = "tpu_lib",
     srcs = [
         "python/tpu/__init__.py",
+        "python/tpu/bfloat16.py",
         "python/tpu/device_assignment.py",
         "python/tpu/topology.py",
         "python/tpu/tpu.py",
@@ -240,6 +241,17 @@ tf_py_test(
     ],
 )
 
+tf_py_test(
+    name = "bfloat16_test",
+    size = "small",
+    srcs = ["python/tpu/bfloat16_test.py"],
+    additional_deps = [
+        ":tpu",
+        "//tensorflow/python:client_testlib",
+        "//tensorflow/python:framework",
+    ],
+)
+
 tf_py_test(
     name = "tpu_infeed_test",
     size = "small",
diff --git a/tensorflow/contrib/tpu/__init__.py b/tensorflow/contrib/tpu/__init__.py
index ea6e874f2d..bb60f3e2d7 100644
--- a/tensorflow/contrib/tpu/__init__.py
+++ b/tensorflow/contrib/tpu/__init__.py
@@ -53,6 +53,7 @@ from __future__ import print_function
 # pylint: disable=wildcard-import,unused-import
 from tensorflow.contrib.tpu.python import profiler
 from tensorflow.contrib.tpu.python.ops.tpu_ops import *
+from tensorflow.contrib.tpu.python.tpu.bfloat16 import *
 from tensorflow.contrib.tpu.python.tpu.device_assignment import *
 from tensorflow.contrib.tpu.python.tpu.topology import *
 from tensorflow.contrib.tpu.python.tpu.tpu import *
diff --git a/tensorflow/contrib/tpu/python/tpu/bfloat16.py b/tensorflow/contrib/tpu/python/tpu/bfloat16.py
new file mode 100644
index 0000000000..929d1824c3
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/bfloat16.py
@@ -0,0 +1,78 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+"""Helper context for running models with bfloat16."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+from tensorflow.python.platform import tf_logging as logging
+from tensorflow.python.util import tf_contextlib
+
+
+def _get_custom_getter():
+  """Returns a custom getter that this class's methods must be called under.
+
+  All methods of this class must be called under a variable scope that was
+  passed this custom getter. Example:
+
+  ```python
+  network = ConvNetBuilder(...)
+  with tf.variable_scope('cg', custom_getter=network.get_custom_getter()):
+    network.conv(...)
+    # Call more methods of network here
+  ```
+
+  Currently, this custom getter only does anything if self.use_tf_layers is
+  True. In that case, it causes variables to be stored as dtype
+  self.variable_type, then casted to the requested dtype, instead of directly
+  storing the variable as the requested dtype.
+  """
+
+  def inner_custom_getter(getter, *args, **kwargs):
+    """Custom getter that forces variables to have type self.variable_type."""
+    cast_to_bfloat16 = False
+    requested_dtype = kwargs['dtype']
+    if requested_dtype == dtypes.bfloat16:
+      # Only change the variable dtype if doing so does not decrease variable
+      # precision.
+      kwargs['dtype'] = dtypes.float32
+      cast_to_bfloat16 = True
+    var = getter(*args, **kwargs)
+    # This if statement is needed to guard the cast, because batch norm
+    # assigns directly to the return value of this custom getter. The cast
+    # makes the return value not a variable so it cannot be assigned. Batch
+    # norm variables are always in fp32 so this if statement is never
+    # triggered for them.
+    if cast_to_bfloat16:
+      var = math_ops.cast(var, dtypes.bfloat16)
+    return var
+
+  return inner_custom_getter
+
+
+@tf_contextlib.contextmanager
+def bfloat16_scope():
+  """Scope class for bfloat16 variables so that the model uses custom getter.
+
+  This enables variables to be read as bfloat16 type when using get_variable.
+  """
+  with variable_scope.variable_scope(
+      'bfloat16', custom_getter=_get_custom_getter()) as varscope:
+    yield varscope
diff --git a/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py b/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py
new file mode 100644
index 0000000000..fda4331e89
--- /dev/null
+++ b/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py
@@ -0,0 +1,52 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# =============================================================================
+
+"""Tests for bfloat16 helper."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.contrib.tpu.python.tpu import bfloat16
+from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import variable_scope
+
+from tensorflow.python.platform import test
+
+
+class BFloat16ScopeTest(test.TestCase):
+
+  def testScopeName(self):
+    """Test if name for the variable scope is propogated correctly.
+    """
+    with bfloat16.bfloat16_scope() as bf:
+      self.assertEqual(bf.name, "bfloat16")
+
+  def testRequestedDType(self):
+    """Test if requested dtype is honored in the getter.
+    """
+    with bfloat16.bfloat16_scope() as scope:
+      v1 = variable_scope.get_variable("v1", [])
+      self.assertEqual(v1.dtype.base_dtype, dtypes.float32)
+      v2 = variable_scope.get_variable("v2", [], dtype=dtypes.bfloat16)
+      self.assertEqual(v2.dtype.base_dtype, dtypes.bfloat16)
+      self.assertEqual([dtypes.float32, dtypes.float32],
+                       [v.dtype.base_dtype for v in scope.global_variables()])
+
+
+if __name__ == "__main__":
+  test.main()
-- 
GitLab


From c4acdccbb7284c6a63e6824a7ee45ce7a86606b9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 11:12:24 -0700
Subject: [PATCH 760/960] Rename distributed_apply to _distributed_apply in
 OptimizerV2 to match the Optimizer base class.

PiperOrigin-RevId: 191089407
---
 tensorflow/contrib/optimizer_v2/optimizer_v2.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/optimizer_v2/optimizer_v2.py b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
index 471992fdac..25d19578ea 100644
--- a/tensorflow/contrib/optimizer_v2/optimizer_v2.py
+++ b/tensorflow/contrib/optimizer_v2/optimizer_v2.py
@@ -866,7 +866,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       raise ValueError("No gradients provided for any variable: %s." %
                        ([str(v) for _, v in grads_and_vars],))
     return distribute_lib.get_tower_context().merge_call(
-        self.distributed_apply, filtered, global_step=global_step, name=name)
+        self._distributed_apply, filtered, global_step=global_step, name=name)
 
   def _get_or_create_state(self, var_list=None):
     """Either looks up or creates `_OptimizerV2State`.
@@ -899,7 +899,7 @@ class OptimizerV2(optimizer_v1.Optimizer):
       self._per_graph_state[graph_key] = per_graph_state
     return per_graph_state
 
-  def distributed_apply(self, distribution, grads_and_vars, global_step, name):
+  def _distributed_apply(self, distribution, grads_and_vars, global_step, name):
     """`apply_gradients` for use with a `DistributionStrategy`."""
     reduced_grads = distribution.batch_reduce("sum", grads_and_vars)
     var_list = [v for _, v in grads_and_vars]
-- 
GitLab


From ff451cae700d0f0f17ff9d2dde32299344b21fb1 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Fri, 30 Mar 2018 11:21:31 -0700
Subject: [PATCH 761/960] Internal change.

PiperOrigin-RevId: 191090993
---
 .../main/java/org/tensorflow/lite/Interpreter.java  | 13 +++++++++++++
 .../tensorflow/lite/NativeInterpreterWrapper.java   | 12 ++++++------
 2 files changed, 19 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
index 14f461f5f9..a33959dca4 100644
--- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
+++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/Interpreter.java
@@ -67,6 +67,19 @@ public final class Interpreter implements AutoCloseable {
     wrapper = new NativeInterpreterWrapper(modelFile.getAbsolutePath());
   }
 
+  /**
+   * Initializes a {@code Interpreter} and specifies the number of threads used for inference.
+   *
+   * @param modelFile: a file of a pre-trained TF Lite model
+   * @param numThreads: number of threads to use for inference
+   */
+  public Interpreter(@NonNull File modelFile, int numThreads) {
+    if (modelFile == null) {
+      return;
+    }
+    wrapper = new NativeInterpreterWrapper(modelFile.getAbsolutePath(), numThreads);
+  }
+
   /**
    * Initializes a {@code Interpreter} with a {@code MappedByteBuffer} to the model file.
    *
diff --git a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
index dbf8f8f7cc..fc8187acfe 100644
--- a/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
+++ b/tensorflow/contrib/lite/java/src/main/java/org/tensorflow/lite/NativeInterpreterWrapper.java
@@ -32,9 +32,13 @@ import java.util.Map;
 final class NativeInterpreterWrapper implements AutoCloseable {
 
   NativeInterpreterWrapper(String modelPath) {
+    this(modelPath, /* numThreads= */ -1);
+  }
+
+  NativeInterpreterWrapper(String modelPath, int numThreads) {
     errorHandle = createErrorReporter(ERROR_BUFFER_SIZE);
     modelHandle = createModel(modelPath, errorHandle);
-    interpreterHandle = createInterpreter(modelHandle, errorHandle, /* numThreads= */ -1);
+    interpreterHandle = createInterpreter(modelHandle, errorHandle, numThreads);
     isMemoryAllocated = true;
   }
 
@@ -44,11 +48,7 @@ final class NativeInterpreterWrapper implements AutoCloseable {
    * NativeInterpreterWrapper}.
    */
   NativeInterpreterWrapper(MappedByteBuffer mappedByteBuffer) {
-    modelByteBuffer = mappedByteBuffer;
-    errorHandle = createErrorReporter(ERROR_BUFFER_SIZE);
-    modelHandle = createModelWithBuffer(modelByteBuffer, errorHandle);
-    interpreterHandle = createInterpreter(modelHandle, errorHandle, /* numThreads= */ -1);
-    isMemoryAllocated = true;
+    this(mappedByteBuffer, /* numThreads= */ -1);
   }
 
   /**
-- 
GitLab


From bdd48ec56baefdd2eea3b167904457f7181ea1c6 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Sat, 31 Mar 2018 02:31:31 +0800
Subject: [PATCH 762/960]  Fix some formatting issue in cudnn_rnn and
 factorization docstrings (#18125)

* Fix some formatting issue in cudnn_rnn and factorization docstrings

* fix minor intent <=80 chars

* Add more minor formatting fixes

* fix minor intent

* fix sigma with \sigma
---
 .../cudnn_rnn/python/ops/cudnn_rnn_ops.py        | 16 ++++++++++------
 .../factorization/python/ops/clustering_ops.py   | 11 ++++++-----
 .../contrib/factorization/python/ops/gmm_ops.py  |  4 ++--
 .../contrib/factorization/python/ops/kmeans.py   |  8 ++++----
 .../contrib/factorization/python/ops/wals.py     |  6 +++---
 5 files changed, 25 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
index 622241a177..2ac9442406 100644
--- a/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
+++ b/tensorflow/contrib/cudnn_rnn/python/ops/cudnn_rnn_ops.py
@@ -86,19 +86,23 @@ class CudnnCompatibleGRUCell(rnn_cell_impl.GRUCell):
 
   Cudnn compatible GRU (from Cudnn library user guide):
   ```python
-  r_t = sigma(x_t * W_r + h_t-1 * R_h + b_Wr + b_Rr)  # reset gate
-  u_t = sigma(x_t * W_u + h_t-1 * R_u + b_Wu + b_Ru)  # update gate
-  h'_t = tanh(x_t * W_h + r_t .* (h_t-1 * R_h + b_Rh) + b_Wh)  # new memory gate
-  h_t = (1 - u_t) .* h'_t + u_t .* h_t-1
+  # reset gate
+  $$r_t = \sigma(x_t * W_r + h_t-1 * R_h + b_{Wr} + b_{Rr})$$
+  # update gate
+  $$u_t = \sigma(x_t * W_u + h_t-1 * R_u + b_{Wu} + b_{Ru})$$
+  # new memory gate
+  $$h'_t = tanh(x_t * W_h + r_t .* (h_t-1 * R_h + b_{Rh}) + b_{Wh})$$
+  $$h_t = (1 - u_t) .* h'_t + u_t .* h_t-1$$
   ```
 
   Other GRU (see @{tf.nn.rnn_cell.GRUCell} and @{tf.contrib.rnn.GRUBlockCell}):
   ```python
-  h'_t = tanh(x_t * W_h + (r_t .* h_t-1) * R_h + b_Wh)  # new memory gate
+  # new memory gate
+  \\(h'_t = tanh(x_t * W_h + (r_t .* h_t-1) * R_h + b_{Wh})\\)
   ```
   which is not equivalent to Cudnn GRU: in addition to the extra bias term b_Rh,
   ```python
-  r .* (h * R) != (r .* h) * R
+  \\(r .* (h * R) != (r .* h) * R\\)
   ```
   """
 
diff --git a/tensorflow/contrib/factorization/python/ops/clustering_ops.py b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
index 23137e0a97..84e80791f4 100644
--- a/tensorflow/contrib/factorization/python/ops/clustering_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/clustering_ops.py
@@ -41,11 +41,12 @@ from tensorflow.python.platform import resource_loader
 _clustering_ops = loader.load_op_library(
     resource_loader.get_path_to_datafile('_clustering_ops.so'))
 
-# Euclidean distance between vectors U and V is defined as ||U - V||_F which is
-# the square root of the sum of the absolute squares of the elements difference.
+# Euclidean distance between vectors U and V is defined as \\(||U - V||_F\\)
+# which is the square root of the sum of the absolute squares of the elements
+# difference.
 SQUARED_EUCLIDEAN_DISTANCE = 'squared_euclidean'
 # Cosine distance between vectors U and V is defined as
-# 1 - (U \dot V) / (||U||_F ||V||_F)
+# \\(1 - (U \dot V) / (||U||_F ||V||_F)\\)
 COSINE_DISTANCE = 'cosine'
 
 RANDOM_INIT = 'random'
@@ -472,8 +473,8 @@ class KMeans(object):
         # Locally compute the sum of inputs mapped to each id.
         # For a cluster with old cluster value x, old count n, and with data
         # d_1,...d_k newly assigned to it, we recompute the new value as
-        # x += (sum_i(d_i) - k * x) / (n + k).
-        # Compute sum_i(d_i), see comment above.
+        # \\(x += (sum_i(d_i) - k * x) / (n + k)\\).
+        # Compute \\(sum_i(d_i)\\), see comment above.
         cluster_center_updates = math_ops.unsorted_segment_sum(
             inp, unique_idx, num_unique_cluster_idx)
         # Shape to enable broadcasting count_updates and learning_rate to inp.
diff --git a/tensorflow/contrib/factorization/python/ops/gmm_ops.py b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
index 14d4c733e3..5d77bc77e1 100644
--- a/tensorflow/contrib/factorization/python/ops/gmm_ops.py
+++ b/tensorflow/contrib/factorization/python/ops/gmm_ops.py
@@ -357,8 +357,8 @@ class GmmAlgorithm(object):
     # Shape broadcasting.
     probs = array_ops.expand_dims(self._probs[shard_id], 0)
     # Membership weights are computed as:
-    # w_{ik} = \frac{\alpha_k f(\mathbf{y_i}|\mathbf{\theta}_k)}
-    #               {\sum_{m=1}^{K}\alpha_mf(\mathbf{y_i}|\mathbf{\theta}_m)}
+    # $$w_{ik} = \frac{\alpha_k f(\mathbf{y_i}|\mathbf{\theta}_k)}$$
+    # $$            {\sum_{m=1}^{K}\alpha_mf(\mathbf{y_i}|\mathbf{\theta}_m)}$$
     # where "i" is the i-th example, "k" is the k-th mixture, theta are
     # the model parameters and y_i the observations.
     # These are defined for each shard.
diff --git a/tensorflow/contrib/factorization/python/ops/kmeans.py b/tensorflow/contrib/factorization/python/ops/kmeans.py
index 38faca119d..bfe338c9f9 100644
--- a/tensorflow/contrib/factorization/python/ops/kmeans.py
+++ b/tensorflow/contrib/factorization/python/ops/kmeans.py
@@ -374,11 +374,11 @@ class KMeansClustering(estimator.Estimator):
               than `num_clusters`, a TensorFlow runtime error occurs.
       distance_metric: The distance metric used for clustering. One of:
         * `KMeansClustering.SQUARED_EUCLIDEAN_DISTANCE`: Euclidean distance
-             between vectors `u` and `v` is defined as `||u - v||_2` which is
-             the square root of the sum of the absolute squares of the elements'
-             difference.
+             between vectors `u` and `v` is defined as `\\(||u - v||_2\\)`
+             which is the square root of the sum of the absolute squares of
+             the elements' difference.
         * `KMeansClustering.COSINE_DISTANCE`: Cosine distance between vectors
-             `u` and `v` is defined as `1 - (u . v) / (||u||_2 ||v||_2)`.
+             `u` and `v` is defined as `\\(1 - (u . v) / (||u||_2 ||v||_2)\\)`.
       random_seed: Python integer. Seed for PRNG used to initialize centers.
       use_mini_batch: A boolean specifying whether to use the mini-batch k-means
         algorithm. See explanation above.
diff --git a/tensorflow/contrib/factorization/python/ops/wals.py b/tensorflow/contrib/factorization/python/ops/wals.py
index 62db3bb4c4..ca46c39baa 100644
--- a/tensorflow/contrib/factorization/python/ops/wals.py
+++ b/tensorflow/contrib/factorization/python/ops/wals.py
@@ -216,7 +216,7 @@ def _wals_factorization_model_function(features, labels, mode, params):
         name=WALSMatrixFactorization.LOSS,
         collections=[ops.GraphKeys.GLOBAL_VARIABLES])
     # The root weighted squared error =
-    #   \sqrt( \sum_{i,j} w_ij * (a_ij - r_ij)^2 / \sum_{i,j} w_ij )
+    #   \\(\sqrt( \sum_{i,j} w_ij * (a_ij - r_ij)^2 / \sum_{i,j} w_ij )\\)
     rwse_var = variable_scope.variable(
         0.,
         trainable=False,
@@ -490,11 +490,11 @@ class WALSMatrixFactorization(estimator.Estimator):
           and the problem simplifies to ALS. Note that, in this case,
           col_weights must also be set to "None".
         - List of lists of non-negative scalars, of the form
-          [[w_0, w_1, ...], [w_k, ... ], [...]],
+          \\([[w_0, w_1, ...], [w_k, ... ], [...]]\\),
           where the number of inner lists equal to the number of row factor
           shards and the elements in each inner list are the weights for the
           rows of that shard. In this case,
-          w_ij = unonbserved_weight + row_weights[i] * col_weights[j].
+          \\(w_ij = unonbserved_weight + row_weights[i] * col_weights[j]\\).
         - A non-negative scalar: This value is used for all row weights.
           Note that it is allowed to have row_weights as a list and col_weights
           as a scalar, or vice-versa.
-- 
GitLab


From baa67e733e2a0a34e64214fd0b5347d017e99ebd Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Sat, 31 Mar 2018 02:31:43 +0800
Subject: [PATCH 763/960] Fix the tpu related broken link especially for
 imagenet_to_gcs.py (#18126)

---
 tensorflow/contrib/tpu/tpu_estimator.md            |  2 +-
 tensorflow/docs_src/programmers_guide/using_tpu.md | 10 +++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/contrib/tpu/tpu_estimator.md b/tensorflow/contrib/tpu/tpu_estimator.md
index 4ef8f9eebd..639e708169 100644
--- a/tensorflow/contrib/tpu/tpu_estimator.md
+++ b/tensorflow/contrib/tpu/tpu_estimator.md
@@ -172,7 +172,7 @@ It is always recommended to port a small, simple model first to make sure that
 you are familiar with the basic concepts of `TPUEstimator` and test end-to-end
 behavior. Once your simple model runs, gradually add more functionality.
 In addition, there are several sample models, available at
-[github.com/tensorflow/tpu-demos](https://github.com/tensorflow/tpu-demos).
+[github.com/tensorflow/tpu](https://github.com/tensorflow/tpu).
 
 To convert your code from the vanilla `Estimator` class to use TPUs, change the
 following (note some of the details may change over time):
diff --git a/tensorflow/docs_src/programmers_guide/using_tpu.md b/tensorflow/docs_src/programmers_guide/using_tpu.md
index a9c2cb3e33..cb0d86fc4c 100644
--- a/tensorflow/docs_src/programmers_guide/using_tpu.md
+++ b/tensorflow/docs_src/programmers_guide/using_tpu.md
@@ -11,7 +11,7 @@ This doc is aimed at users who:
   using an existing model.
 * Have, perhaps, skimmed the code of an example TPU model
   [[1]](https://github.com/tensorflow/models/blob/master/official/mnist/mnist_tpu.py)
-  [[2]](https://github.com/tensorflow/tpu-demos/tree/master/cloud_tpu/models).
+  [[2]](https://github.com/tensorflow/tpu/tree/master/models).
 * Are interested in porting an existing `Estimator` model to
   run on Cloud TPUs
 
@@ -288,7 +288,7 @@ If shape inference has failed, but the shape is known it is possible to
 impose the correct shape using `tf.set_shape()`. 
 
 In the example below the shape
-inference algorithm fails, but it is corrected using `set_shape`:
+inference algorithm fails, but it is correctly using `set_shape`:
 
 ```
 >>> x = tf.zeros(tf.constant([1,2,3])+1)
@@ -371,10 +371,10 @@ in bytes. A minimum of a few MB (`buffer_size=8*1024*1024`) is recommended so
 that data is available when needed.
 
 The TPU-demos repo includes
-[a script](https://github.com/tensorflow/tpu-demos/blob/master/cloud_tpu/datasets/imagenet_to_gcs.py)
+[a script](https://github.com/tensorflow/tpu/blob/master/tools/datasets/imagenet_to_gcs.py)
 for downloading the imagenet dataset and converting it to an appropriate format.
 This together with the imagenet
-[models](https://github.com/tensorflow/tpu-demos/tree/master/cloud_tpu/models)
+[models](https://github.com/tensorflow/tpu/tree/master/models)
 included in the repo demonstrate all of these best-practices.
 
 
@@ -387,7 +387,7 @@ For details on how to actually set up and run a Cloud TPU see:
 This document is by no means exhaustive. The best source of more detail on how
 to make a Cloud TPU compatible model are the example models published in:
 
- * The [TPU Demos Repository.](https://github.com/tensorflow/tpu-demos/)
+ * The [TPU Demos Repository.](https://github.com/tensorflow/tpu)
 
 For more information about tuning TensorFlow code for performance see:
 
-- 
GitLab


From 5eef3a21a0df996c407a78cdfbdcdd11ce4f6f34 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 11:32:58 -0700
Subject: [PATCH 764/960] Break FileSystem's dependency on ThreadPool.

PiperOrigin-RevId: 191092932
---
 .../android/asset_manager_filesystem.cc       |   6 +
 .../android/asset_manager_filesystem.h        |   3 +
 .../contrib/makefile/proto_text_cc_files.txt  |   1 +
 tensorflow/core/BUILD                         |   1 +
 tensorflow/core/platform/file_system.cc       |  93 -------------
 tensorflow/core/platform/file_system.h        |   4 +-
 .../core/platform/file_system_helper.cc       | 126 ++++++++++++++++++
 tensorflow/core/platform/file_system_helper.h |  51 +++++++
 .../platform/hadoop/hadoop_file_system.cc     |   6 +
 .../core/platform/hadoop/hadoop_file_system.h |   3 +
 tensorflow/core/platform/null_file_system.h   |   6 +
 .../core/platform/posix/posix_file_system.cc  |   6 +
 .../core/platform/posix/posix_file_system.h   |   3 +
 tensorflow/core/platform/s3/s3_file_system.cc |   6 +
 tensorflow/core/platform/s3/s3_file_system.h  |   3 +
 .../platform/windows/windows_file_system.cc   |   4 +-
 tensorflow/core/util/memmapped_file_system.cc |   6 +
 tensorflow/core/util/memmapped_file_system.h  |   2 +
 18 files changed, 233 insertions(+), 97 deletions(-)
 create mode 100644 tensorflow/core/platform/file_system_helper.cc
 create mode 100644 tensorflow/core/platform/file_system_helper.h

diff --git a/tensorflow/contrib/android/asset_manager_filesystem.cc b/tensorflow/contrib/android/asset_manager_filesystem.cc
index 380a652435..fe2d13e636 100644
--- a/tensorflow/contrib/android/asset_manager_filesystem.cc
+++ b/tensorflow/contrib/android/asset_manager_filesystem.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 
 namespace tensorflow {
 namespace {
@@ -243,6 +244,11 @@ bool AssetManagerFileSystem::DirectoryExists(const std::string& fname) {
   return AAssetDir_getNextFileName(dir.get()) != NULL;
 }
 
+Status AssetManagerFileSystem::GetMatchingPaths(const string& pattern,
+                                                std::vector<string>* results) {
+  return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
 Status AssetManagerFileSystem::NewWritableFile(
     const string& fname, std::unique_ptr<WritableFile>* result) {
   return errors::Unimplemented("Asset storage is read only.");
diff --git a/tensorflow/contrib/android/asset_manager_filesystem.h b/tensorflow/contrib/android/asset_manager_filesystem.h
index 665304b5ee..a87ff42ae2 100644
--- a/tensorflow/contrib/android/asset_manager_filesystem.h
+++ b/tensorflow/contrib/android/asset_manager_filesystem.h
@@ -66,6 +66,9 @@ class AssetManagerFileSystem : public FileSystem {
   Status DeleteDir(const string& d) override;
   Status RenameFile(const string& s, const string& t) override;
 
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override;
+
  private:
   string RemoveAssetPrefix(const string& name);
 
diff --git a/tensorflow/contrib/makefile/proto_text_cc_files.txt b/tensorflow/contrib/makefile/proto_text_cc_files.txt
index 77c936d8c5..76428bc1d4 100644
--- a/tensorflow/contrib/makefile/proto_text_cc_files.txt
+++ b/tensorflow/contrib/makefile/proto_text_cc_files.txt
@@ -12,6 +12,7 @@ tensorflow/core/platform/posix/env.cc
 tensorflow/core/platform/posix/load_library.cc
 tensorflow/core/platform/posix/env_time.cc
 tensorflow/core/platform/file_system.cc
+tensorflow/core/platform/file_system_helper.cc
 tensorflow/core/platform/env.cc
 tensorflow/core/platform/env_time.cc
 tensorflow/core/platform/setround.cc
diff --git a/tensorflow/core/BUILD b/tensorflow/core/BUILD
index 21f7866abd..7d5ae1c5b5 100644
--- a/tensorflow/core/BUILD
+++ b/tensorflow/core/BUILD
@@ -349,6 +349,7 @@ cc_library(
         "platform/env.h",
         "platform/env_time.h",
         "platform/file_system.h",
+        "platform/file_system_helper.h",
         "platform/fingerprint.h",
         "platform/init_main.h",
         "platform/logging.h",
diff --git a/tensorflow/core/platform/file_system.cc b/tensorflow/core/platform/file_system.cc
index a2f42f44ac..b55e94d552 100644
--- a/tensorflow/core/platform/file_system.cc
+++ b/tensorflow/core/platform/file_system.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <deque>
 
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
@@ -28,28 +27,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-namespace {
-
-constexpr int kNumThreads = 8;
-
-// Run a function in parallel using a ThreadPool, but skip the ThreadPool
-// on the iOS platform due to its problems with more than a few threads.
-void ForEach(int first, int last, const std::function<void(int)>& f) {
-#if TARGET_OS_IPHONE
-  for (int i = first; i < last; i++) {
-    f(i);
-  }
-#else
-  int num_threads = std::min(kNumThreads, last - first);
-  thread::ThreadPool threads(Env::Default(), "ForEach", num_threads);
-  for (int i = first; i < last; i++) {
-    threads.Schedule([f, i] { f(i); });
-  }
-#endif
-}
-
-}  // anonymous namespace
-
 FileSystem::~FileSystem() {}
 
 string FileSystem::TranslateName(const string& name) const {
@@ -94,76 +71,6 @@ bool FileSystem::FilesExist(const std::vector<string>& files,
   return result;
 }
 
-Status FileSystem::GetMatchingPaths(const string& pattern,
-                                    std::vector<string>* results) {
-  results->clear();
-  // Find the fixed prefix by looking for the first wildcard.
-  string fixed_prefix = pattern.substr(0, pattern.find_first_of("*?[\\"));
-  string eval_pattern = pattern;
-  std::vector<string> all_files;
-  string dir = io::Dirname(fixed_prefix).ToString();
-  // If dir is empty then we need to fix up fixed_prefix and eval_pattern to
-  // include . as the top level directory.
-  if (dir.empty()) {
-    dir = ".";
-    fixed_prefix = io::JoinPath(dir, fixed_prefix);
-    eval_pattern = io::JoinPath(dir, pattern);
-  }
-
-  // Setup a BFS to explore everything under dir.
-  std::deque<string> dir_q;
-  dir_q.push_back(dir);
-  Status ret;  // Status to return.
-  // children_dir_status holds is_dir status for children. It can have three
-  // possible values: OK for true; FAILED_PRECONDITION for false; CANCELLED
-  // if we don't calculate IsDirectory (we might do that because there isn't
-  // any point in exploring that child path).
-  std::vector<Status> children_dir_status;
-  while (!dir_q.empty()) {
-    string current_dir = dir_q.front();
-    dir_q.pop_front();
-    std::vector<string> children;
-    Status s = GetChildren(current_dir, &children);
-    ret.Update(s);
-    if (children.empty()) continue;
-    // This IsDirectory call can be expensive for some FS. Parallelizing it.
-    children_dir_status.resize(children.size());
-    ForEach(0, children.size(),
-            [this, &current_dir, &children, &fixed_prefix,
-             &children_dir_status](int i) {
-              const string child_path = io::JoinPath(current_dir, children[i]);
-              // In case the child_path doesn't start with the fixed_prefix then
-              // we don't need to explore this path.
-              if (!str_util::StartsWith(child_path, fixed_prefix)) {
-                children_dir_status[i] = Status(tensorflow::error::CANCELLED,
-                                                "Operation not needed");
-              } else {
-                children_dir_status[i] = IsDirectory(child_path);
-              }
-            });
-    for (int i = 0; i < children.size(); ++i) {
-      const string child_path = io::JoinPath(current_dir, children[i]);
-      // If the IsDirectory call was cancelled we bail.
-      if (children_dir_status[i].code() == tensorflow::error::CANCELLED) {
-        continue;
-      }
-      // If the child is a directory add it to the queue.
-      if (children_dir_status[i].ok()) {
-        dir_q.push_back(child_path);
-      }
-      all_files.push_back(child_path);
-    }
-  }
-
-  // Match all obtained files to the input pattern.
-  for (const auto& f : all_files) {
-    if (Env::Default()->MatchPath(f, eval_pattern)) {
-      results->push_back(f);
-    }
-  }
-  return ret;
-}
-
 Status FileSystem::DeleteRecursively(const string& dirname,
                                      int64* undeleted_files,
                                      int64* undeleted_dirs) {
diff --git a/tensorflow/core/platform/file_system.h b/tensorflow/core/platform/file_system.h
index 8f99766e15..077b1d79cf 100644
--- a/tensorflow/core/platform/file_system.h
+++ b/tensorflow/core/platform/file_system.h
@@ -138,10 +138,8 @@ class FileSystem {
   ///  * OK - no errors
   ///  * UNIMPLEMENTED - Some underlying functions (like GetChildren) are not
   ///                    implemented
-  /// The default implementation uses a combination of GetChildren, MatchPath
-  /// and IsDirectory.
   virtual Status GetMatchingPaths(const string& pattern,
-                                  std::vector<string>* results);
+                                  std::vector<string>* results) = 0;
 
   /// \brief Obtains statistics for the given path.
   virtual Status Stat(const string& fname, FileStatistics* stat) = 0;
diff --git a/tensorflow/core/platform/file_system_helper.cc b/tensorflow/core/platform/file_system_helper.cc
new file mode 100644
index 0000000000..22c5057281
--- /dev/null
+++ b/tensorflow/core/platform/file_system_helper.cc
@@ -0,0 +1,126 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/platform/file_system_helper.h"
+
+#include <deque>
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/platform.h"
+
+namespace tensorflow {
+namespace internal {
+
+namespace {
+
+constexpr int kNumThreads = 8;
+
+// Run a function in parallel using a ThreadPool, but skip the ThreadPool
+// on the iOS platform due to its problems with more than a few threads.
+void ForEach(int first, int last, const std::function<void(int)>& f) {
+#if TARGET_OS_IPHONE
+  for (int i = first; i < last; i++) {
+    f(i);
+  }
+#else
+  int num_threads = std::min(kNumThreads, last - first);
+  thread::ThreadPool threads(Env::Default(), "ForEach", num_threads);
+  for (int i = first; i < last; i++) {
+    threads.Schedule([f, i] { f(i); });
+  }
+#endif
+}
+
+}  // namespace
+
+Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern,
+                        std::vector<string>* results) {
+  results->clear();
+  // Find the fixed prefix by looking for the first wildcard.
+  string fixed_prefix = pattern.substr(0, pattern.find_first_of("*?[\\"));
+  string eval_pattern = pattern;
+  std::vector<string> all_files;
+  string dir = io::Dirname(fixed_prefix).ToString();
+  // If dir is empty then we need to fix up fixed_prefix and eval_pattern to
+  // include . as the top level directory.
+  if (dir.empty()) {
+    dir = ".";
+    fixed_prefix = io::JoinPath(dir, fixed_prefix);
+    eval_pattern = io::JoinPath(dir, pattern);
+  }
+
+  // Setup a BFS to explore everything under dir.
+  std::deque<string> dir_q;
+  dir_q.push_back(dir);
+  Status ret;  // Status to return.
+  // children_dir_status holds is_dir status for children. It can have three
+  // possible values: OK for true; FAILED_PRECONDITION for false; CANCELLED
+  // if we don't calculate IsDirectory (we might do that because there isn't
+  // any point in exploring that child path).
+  std::vector<Status> children_dir_status;
+  while (!dir_q.empty()) {
+    string current_dir = dir_q.front();
+    dir_q.pop_front();
+    std::vector<string> children;
+    Status s = fs->GetChildren(current_dir, &children);
+    ret.Update(s);
+    if (children.empty()) continue;
+    // This IsDirectory call can be expensive for some FS. Parallelizing it.
+    children_dir_status.resize(children.size());
+    ForEach(0, children.size(),
+            [fs, &current_dir, &children, &fixed_prefix,
+             &children_dir_status](int i) {
+              const string child_path = io::JoinPath(current_dir, children[i]);
+              // In case the child_path doesn't start with the fixed_prefix then
+              // we don't need to explore this path.
+              if (!str_util::StartsWith(child_path, fixed_prefix)) {
+                children_dir_status[i] = Status(tensorflow::error::CANCELLED,
+                                                "Operation not needed");
+              } else {
+                children_dir_status[i] = fs->IsDirectory(child_path);
+              }
+            });
+    for (int i = 0; i < children.size(); ++i) {
+      const string child_path = io::JoinPath(current_dir, children[i]);
+      // If the IsDirectory call was cancelled we bail.
+      if (children_dir_status[i].code() == tensorflow::error::CANCELLED) {
+        continue;
+      }
+      // If the child is a directory add it to the queue.
+      if (children_dir_status[i].ok()) {
+        dir_q.push_back(child_path);
+      }
+      all_files.push_back(child_path);
+    }
+  }
+
+  // Match all obtained files to the input pattern.
+  for (const auto& f : all_files) {
+    if (env->MatchPath(f, eval_pattern)) {
+      results->push_back(f);
+    }
+  }
+  return ret;
+}
+
+}  // namespace internal
+}  // namespace tensorflow
diff --git a/tensorflow/core/platform/file_system_helper.h b/tensorflow/core/platform/file_system_helper.h
new file mode 100644
index 0000000000..8d812b0e38
--- /dev/null
+++ b/tensorflow/core/platform/file_system_helper.h
@@ -0,0 +1,51 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
+#define TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
+
+#include <string>
+#include <vector>
+
+#include "tensorflow/core/lib/core/status.h"
+
+namespace tensorflow {
+
+class FileSystem;
+class Env;
+
+namespace internal {
+
+// Given a pattern, stores in 'results' the set of paths (in the given file
+// system) that match that pattern.
+//
+// This helper may be used by implementations of FileSystem::GetMatchingPaths()
+// in order to provide parallel scanning of subdirectories (except on iOS).
+//
+// Arguments:
+//   fs: may not be null and will be used to identify directories and list
+//       their contents.
+//   env: may not be null and will be used to check if a match has been found.
+//   pattern: see FileSystem::GetMatchingPaths() for details.
+//   results: will be cleared and may not be null.
+//
+// Returns an error status if any call to 'fs' failed.
+Status GetMatchingPaths(FileSystem* fs, Env* env, const string& pattern,
+                        std::vector<string>* results);
+
+}  // namespace internal
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_PLATFORM_FILE_SYSTEM_HELPER_H_
diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.cc b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
index 74863293a3..9a71fbe2b7 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/posix/error.h"
@@ -396,6 +397,11 @@ Status HadoopFileSystem::GetChildren(const string& dir,
   return Status::OK();
 }
 
+Status HadoopFileSystem::GetMatchingPaths(const string& pattern,
+                                          std::vector<string>* results) {
+  return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
 Status HadoopFileSystem::DeleteFile(const string& fname) {
   hdfsFS fs = nullptr;
   TF_RETURN_IF_ERROR(Connect(fname, &fs));
diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system.h b/tensorflow/core/platform/hadoop/hadoop_file_system.h
index 5f2b222622..6af7a698ff 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system.h
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system.h
@@ -49,6 +49,9 @@ class HadoopFileSystem : public FileSystem {
 
   Status GetChildren(const string& dir, std::vector<string>* result) override;
 
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override;
+
   Status DeleteFile(const string& fname) override;
 
   Status CreateDir(const string& name) override;
diff --git a/tensorflow/core/platform/null_file_system.h b/tensorflow/core/platform/null_file_system.h
index 008e6d54d0..420abc1ada 100644
--- a/tensorflow/core/platform/null_file_system.h
+++ b/tensorflow/core/platform/null_file_system.h
@@ -22,6 +22,7 @@ limitations under the License.
 
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/file_system.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 
 namespace tensorflow {
 
@@ -65,6 +66,11 @@ class NullFileSystem : public FileSystem {
     return errors::Unimplemented("GetChildren unimplemented");
   }
 
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override {
+    return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+  }
+
   Status DeleteFile(const string& fname) override {
     return errors::Unimplemented("DeleteFile unimplemented");
   }
diff --git a/tensorflow/core/platform/posix/posix_file_system.cc b/tensorflow/core/platform/posix/posix_file_system.cc
index 9a8021565c..47bfa020ce 100644
--- a/tensorflow/core/platform/posix/posix_file_system.cc
+++ b/tensorflow/core/platform/posix/posix_file_system.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/posix/error.h"
 #include "tensorflow/core/platform/posix/posix_file_system.h"
@@ -225,6 +226,11 @@ Status PosixFileSystem::GetChildren(const string& dir,
   return Status::OK();
 }
 
+Status PosixFileSystem::GetMatchingPaths(const string& pattern,
+                                         std::vector<string>* results) {
+  return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
 Status PosixFileSystem::DeleteFile(const string& fname) {
   Status result;
   if (unlink(TranslateName(fname).c_str()) != 0) {
diff --git a/tensorflow/core/platform/posix/posix_file_system.h b/tensorflow/core/platform/posix/posix_file_system.h
index 98ffa43b8a..e8898d0a97 100644
--- a/tensorflow/core/platform/posix/posix_file_system.h
+++ b/tensorflow/core/platform/posix/posix_file_system.h
@@ -47,6 +47,9 @@ class PosixFileSystem : public FileSystem {
 
   Status Stat(const string& fname, FileStatistics* stats) override;
 
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override;
+
   Status DeleteFile(const string& fname) override;
 
   Status CreateDir(const string& name) override;
diff --git a/tensorflow/core/platform/s3/s3_file_system.cc b/tensorflow/core/platform/s3/s3_file_system.cc
index 301fcb9dbf..ee423699b2 100644
--- a/tensorflow/core/platform/s3/s3_file_system.cc
+++ b/tensorflow/core/platform/s3/s3_file_system.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/platform/s3/s3_file_system.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/strings/str_util.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/s3/aws_logging.h"
 #include "tensorflow/core/platform/s3/s3_crypto.h"
@@ -497,6 +498,11 @@ Status S3FileSystem::Stat(const string& fname, FileStatistics* stats) {
   return Status::OK();
 }
 
+Status S3FileSystem::GetMatchingPaths(const string& pattern,
+                                      std::vector<string>* results) {
+  return internal::GetMatchingPaths(this, Env::Default(), pattern, results);
+}
+
 Status S3FileSystem::DeleteFile(const string& fname) {
   string bucket, object;
   TF_RETURN_IF_ERROR(ParseS3Path(fname, false, &bucket, &object));
diff --git a/tensorflow/core/platform/s3/s3_file_system.h b/tensorflow/core/platform/s3/s3_file_system.h
index 31264be621..5d0565b378 100644
--- a/tensorflow/core/platform/s3/s3_file_system.h
+++ b/tensorflow/core/platform/s3/s3_file_system.h
@@ -46,6 +46,9 @@ class S3FileSystem : public FileSystem {
 
   Status Stat(const string& fname, FileStatistics* stat) override;
 
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override;
+
   Status DeleteFile(const string& fname) override;
 
   Status CreateDir(const string& name) override;
diff --git a/tensorflow/core/platform/windows/windows_file_system.cc b/tensorflow/core/platform/windows/windows_file_system.cc
index 682e46e0fc..dc2efbeaf5 100644
--- a/tensorflow/core/platform/windows/windows_file_system.cc
+++ b/tensorflow/core/platform/windows/windows_file_system.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
+#include "tensorflow/core/platform/file_system_helper.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/posix/error.h"
 #include "tensorflow/core/platform/windows/error.h"
@@ -494,7 +495,8 @@ Status WindowsFileSystem::GetMatchingPaths(const string& pattern,
   // but no code appears to rely on this behavior.
   string converted_pattern(pattern);
   std::replace(converted_pattern.begin(), converted_pattern.end(), '\\', '/');
-  TF_RETURN_IF_ERROR(FileSystem::GetMatchingPaths(converted_pattern, results));
+  TF_RETURN_IF_ERROR(internal::GetMatchingPaths(this, Env::Default(),
+                                                converted_pattern, results));
   for (string& result : *results) {
     std::replace(result.begin(), result.end(), '/', '\\');
   }
diff --git a/tensorflow/core/util/memmapped_file_system.cc b/tensorflow/core/util/memmapped_file_system.cc
index a0f43d2d4a..ea0a381f4f 100644
--- a/tensorflow/core/util/memmapped_file_system.cc
+++ b/tensorflow/core/util/memmapped_file_system.cc
@@ -157,6 +157,12 @@ Status MemmappedFileSystem::GetChildren(const string& filename,
   return errors::Unimplemented("memmapped format doesn't support GetChildren");
 }
 
+Status MemmappedFileSystem::GetMatchingPaths(const string& pattern,
+                                             std::vector<string>* results) {
+  return errors::Unimplemented(
+      "memmapped format doesn't support GetMatchingPaths");
+}
+
 Status MemmappedFileSystem::DeleteFile(const string& filename) {
   return errors::Unimplemented("memmapped format doesn't support DeleteFile");
 }
diff --git a/tensorflow/core/util/memmapped_file_system.h b/tensorflow/core/util/memmapped_file_system.h
index 541587aeab..76cc4911f5 100644
--- a/tensorflow/core/util/memmapped_file_system.h
+++ b/tensorflow/core/util/memmapped_file_system.h
@@ -85,6 +85,8 @@ class MemmappedFileSystem : public FileSystem {
   Status NewAppendableFile(const string& fname,
                            std::unique_ptr<WritableFile>* result) override;
   Status GetChildren(const string& dir, std::vector<string>* r) override;
+  Status GetMatchingPaths(const string& pattern,
+                          std::vector<string>* results) override;
   Status DeleteFile(const string& f) override;
   Status CreateDir(const string& d) override;
   Status DeleteDir(const string& d) override;
-- 
GitLab


From 8cd922f95241b2bd6ce96c399c34e3ad137c705e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 11:43:05 -0700
Subject: [PATCH 765/960] Enable writing of summaries in ExamplesPerSecondHook

PiperOrigin-RevId: 191094585
---
 tensorflow/contrib/tpu/python/tpu/tpu_estimator.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
index fa56708f44..6834600b79 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_estimator.py
@@ -2019,7 +2019,8 @@ class TPUEstimator(estimator_lib.Estimator):
                   host_ops,
                   run_infeed_loop_on_coordinator=(
                       run_infeed_loop_on_coordinator)),
-              ExamplesPerSecondHook(ctx.global_batch_size),
+              ExamplesPerSecondHook(ctx.global_batch_size,
+                                    output_dir=self.model_dir),
               InstallSignalHandlerHook(),
               training.LoggingTensorHook(
                   {
-- 
GitLab


From 6d4623b6a901ed3ec34c47a61de69389d93abb6b Mon Sep 17 00:00:00 2001
From: Lukas Geiger <lgeiger@users.noreply.github.com>
Date: Fri, 30 Mar 2018 21:05:08 +0200
Subject: [PATCH 766/960] [tfgan] Add one sided wasserstein gradient penalty
 (#18098)

* [tfgan] Add one sided wasserstein gradient penalty

* [tfgan] Switch gradient penalty mode to boolean

* [tfgan] Fix lint errors and boolean comparison
---
 .../gan/python/losses/python/losses_impl.py   | 12 ++++++---
 .../python/losses/python/losses_impl_test.py  | 22 ++++++++++++++++
 tensorflow/contrib/gan/python/train.py        |  4 +++
 tensorflow/contrib/gan/python/train_test.py   | 25 +++++++++++++++++--
 4 files changed, 58 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl.py b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
index c1c9cc5d72..1ba3a64167 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl.py
@@ -306,6 +306,7 @@ def wasserstein_gradient_penalty(
     discriminator_scope,
     epsilon=1e-10,
     target=1.0,
+    one_sided=False,
     weights=1.0,
     scope=None,
     loss_collection=ops.GraphKeys.LOSSES,
@@ -327,6 +328,8 @@ def wasserstein_gradient_penalty(
       computing the gradient norm.
     target: Optional Python number or `Tensor` indicating the target value of
       gradient norm. Defaults to 1.0.
+    one_sided: If `True`, penalty proposed in https://arxiv.org/abs/1709.08894
+      is used. Defaults to `False`.
     weights: Optional `Tensor` whose rank is either 0, or the same rank as
       `real_data` and `generated_data`, and must be broadcastable to
       them (i.e., all dimensions must be either `1`, or the same as the
@@ -377,10 +380,13 @@ def wasserstein_gradient_penalty(
     # For numerical stability, add epsilon to the sum before taking the square
     # root. Note tf.norm does not add epsilon.
     slopes = math_ops.sqrt(gradient_squares + epsilon)
-    penalties = math_ops.square(slopes / target - 1.0)
+    penalties = slopes / target - 1.0
+    if one_sided:
+      penalties = math_ops.maximum(0., penalties)
+    penalties_squared = math_ops.square(penalties)
     penalty = losses.compute_weighted_loss(
-        penalties, weights, scope=scope, loss_collection=loss_collection,
-        reduction=reduction)
+        penalties_squared, weights, scope=scope,
+        loss_collection=loss_collection, reduction=reduction)
 
     if add_summaries:
       summary.scalar('gradient_penalty_loss', penalty)
diff --git a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
index dbaa624ae9..2889e93743 100644
--- a/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
+++ b/tensorflow/contrib/gan/python/losses/python/losses_impl_test.py
@@ -481,6 +481,28 @@ class GradientPenaltyTest(test.TestCase, _PenaltyTest):
                       })
       self.assertAlmostEqual(self._expected_loss, loss, 5)
 
+  def test_loss_using_one_sided_mode(self):
+    generated_data = array_ops.placeholder(dtypes.float32, shape=(None, None))
+    real_data = array_ops.placeholder(dtypes.float32, shape=(None, None))
+
+    loss = tfgan_losses.wasserstein_gradient_penalty(
+        generated_data,
+        real_data,
+        self._kwargs['generator_inputs'],
+        self._kwargs['discriminator_fn'],
+        self._kwargs['discriminator_scope'],
+        one_sided=True)
+    self.assertEqual(generated_data.dtype, loss.dtype)
+
+    with self.test_session() as sess:
+      variables.global_variables_initializer().run()
+      loss = sess.run(loss,
+                      feed_dict={
+                          generated_data: self._generated_data_np,
+                          real_data: self._real_data_np,
+                      })
+      self.assertAlmostEqual(self._expected_loss, loss, 5)
+
   def test_loss_with_gradient_norm_target(self):
     """Test loss value with non default gradient norm target."""
     generated_data = array_ops.placeholder(dtypes.float32, shape=(None, None))
diff --git a/tensorflow/contrib/gan/python/train.py b/tensorflow/contrib/gan/python/train.py
index 776eb11ecb..73acd05b60 100644
--- a/tensorflow/contrib/gan/python/train.py
+++ b/tensorflow/contrib/gan/python/train.py
@@ -461,6 +461,7 @@ def gan_loss(
     gradient_penalty_weight=None,
     gradient_penalty_epsilon=1e-10,
     gradient_penalty_target=1.0,
+    gradient_penalty_one_sided=False,
     mutual_information_penalty_weight=None,
     aux_cond_generator_weight=None,
     aux_cond_discriminator_weight=None,
@@ -485,6 +486,8 @@ def gan_loss(
     gradient_penalty_target: If `gradient_penalty_weight` is not None, a Python
       number or `Tensor` indicating the target value of gradient norm. See the
       CIFAR10 section of https://arxiv.org/abs/1710.10196. Defaults to 1.0.
+    gradient_penalty_one_sided: If `True`, penalty proposed in
+      https://arxiv.org/abs/1709.08894 is used. Defaults to `False`.
     mutual_information_penalty_weight: If not `None`, must be a non-negative
       Python number or Tensor indicating how much to weight the mutual
       information penalty. See https://arxiv.org/abs/1606.03657 for more
@@ -546,6 +549,7 @@ def gan_loss(
         model,
         epsilon=gradient_penalty_epsilon,
         target=gradient_penalty_target,
+        one_sided=gradient_penalty_one_sided,
         add_summaries=add_summaries)
     dis_loss += gradient_penalty_weight * gp_loss
   if _use_aux_loss(mutual_information_penalty_weight):
diff --git a/tensorflow/contrib/gan/python/train_test.py b/tensorflow/contrib/gan/python/train_test.py
index f9bdaa74c9..3ebbe55d05 100644
--- a/tensorflow/contrib/gan/python/train_test.py
+++ b/tensorflow/contrib/gan/python/train_test.py
@@ -359,10 +359,12 @@ class GANLossTest(test.TestCase):
     self.assertGreater(len(ops.get_collection(ops.GraphKeys.SUMMARIES)), 0)
 
   # Test gradient penalty option.
-  def _test_grad_penalty_helper(self, create_gan_model_fn):
+  def _test_grad_penalty_helper(self, create_gan_model_fn, one_sided=False):
     model = create_gan_model_fn()
     loss = train.gan_loss(model)
-    loss_gp = train.gan_loss(model, gradient_penalty_weight=1.0)
+    loss_gp = train.gan_loss(model,
+                             gradient_penalty_weight=1.0,
+                             gradient_penalty_one_sided=one_sided)
     self.assertTrue(isinstance(loss_gp, namedtuples.GANLoss))
 
     # Check values.
@@ -394,6 +396,25 @@ class GANLossTest(test.TestCase):
   def test_grad_penalty_callable_acgan(self):
     self._test_grad_penalty_helper(create_callable_acgan_model)
 
+  def test_grad_penalty_one_sided_gan(self):
+    self._test_grad_penalty_helper(create_gan_model, one_sided=True)
+
+  def test_grad_penalty_one_sided_callable_gan(self):
+    self._test_grad_penalty_helper(create_callable_gan_model, one_sided=True)
+
+  def test_grad_penalty_one_sided_infogan(self):
+    self._test_grad_penalty_helper(create_infogan_model, one_sided=True)
+
+  def test_grad_penalty_one_sided_callable_infogan(self):
+    self._test_grad_penalty_helper(
+        create_callable_infogan_model, one_sided=True)
+
+  def test_grad_penalty_one_sided_acgan(self):
+    self._test_grad_penalty_helper(create_acgan_model, one_sided=True)
+
+  def test_grad_penalty_one_sided_callable_acgan(self):
+    self._test_grad_penalty_helper(create_callable_acgan_model, one_sided=True)
+
   # Test mutual information penalty option.
   def _test_mutual_info_penalty_helper(self, create_gan_model_fn):
     train.gan_loss(create_gan_model_fn(),
-- 
GitLab


From 918bd556b880e3d372b9c19d96ac6cfee0e3a852 Mon Sep 17 00:00:00 2001
From: James Keeling <jtkeeling@google.com>
Date: Tue, 27 Mar 2018 09:36:52 -0700
Subject: [PATCH 767/960] Prevent warning every time someone imports
 contrib.learn.datasets.base

Everything in contrib/learn/python/learn/datasets/base.py has been deprecated. One of the function in there is a decorator, retry. Because another function in that file is decorated with retry, the function is called upon import, which prints a warning.

I have fixed this by adding a private function, _internal_retry, which is used internally, and redefining retry to simply call this. That way, using retry in user-code will still print the deprecated warning, but it's not printed upon every import.

I also cleaned up the docstrings slightly.

PiperOrigin-RevId: 190626717
---
 .../learn/python/learn/datasets/base.py       | 35 ++++++++++++++++++-
 1 file changed, 34 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/learn/python/learn/datasets/base.py b/tensorflow/contrib/learn/python/learn/datasets/base.py
index 3b5c9b97c0..4676eedb20 100644
--- a/tensorflow/contrib/learn/python/learn/datasets/base.py
+++ b/tensorflow/contrib/learn/python/learn/datasets/base.py
@@ -139,15 +139,48 @@ def retry(initial_delay,
 
   Args:
     initial_delay: the initial delay.
+    max_delay: the maximum delay allowed (actual max is
+        max_delay * (1 + jitter).
     factor: each subsequent retry, the delay is multiplied by this value.
         (must be >= 1).
     jitter: to avoid lockstep, the returned delay is multiplied by a random
         number between (1-jitter) and (1+jitter). To add a 20% jitter, set
         jitter = 0.2. Must be < 1.
+    is_retriable: (optional) a function that takes an Exception as an argument
+        and returns true if retry should be applied.
+
+  Returns:
+    A function that wraps another function to automatically retry it.
+  """
+  return _internal_retry(
+      initial_delay=initial_delay,
+      max_delay=max_delay,
+      factor=factor,
+      jitter=jitter,
+      is_retriable=is_retriable)
+
+
+def _internal_retry(initial_delay,
+                    max_delay,
+                    factor=2.0,
+                    jitter=0.25,
+                    is_retriable=None):
+  """Simple decorator for wrapping retriable functions, for internal use only.
+
+  Args:
+    initial_delay: the initial delay.
     max_delay: the maximum delay allowed (actual max is
         max_delay * (1 + jitter).
+    factor: each subsequent retry, the delay is multiplied by this value.
+        (must be >= 1).
+    jitter: to avoid lockstep, the returned delay is multiplied by a random
+        number between (1-jitter) and (1+jitter). To add a 20% jitter, set
+        jitter = 0.2. Must be < 1.
     is_retriable: (optional) a function that takes an Exception as an argument
         and returns true if retry should be applied.
+
+  Returns:
+    A function that wraps another function to automatically retry it.
   """
   if factor < 1:
     raise ValueError('factor must be >= 1; was %f' % (factor,))
@@ -195,7 +228,7 @@ def _is_retriable(e):
 
 
 @deprecated(None, 'Please use urllib or similar directly.')
-@retry(initial_delay=1.0, max_delay=16.0, is_retriable=_is_retriable)
+@_internal_retry(initial_delay=1.0, max_delay=16.0, is_retriable=_is_retriable)
 def urlretrieve_with_retry(url, filename=None):
   return urllib.request.urlretrieve(url, filename)
 
-- 
GitLab


From f270e43f886cc39bc4c0fb11e147b9c38853b3a4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 12:26:21 -0700
Subject: [PATCH 768/960] Disable tsan for
 tensorflow/python/estimator:replicate_model_fn_test

It gets flaky failures.

PiperOrigin-RevId: 191100692
---
 tensorflow/python/estimator/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/python/estimator/BUILD b/tensorflow/python/estimator/BUILD
index f93bc221cc..5d8b19223f 100644
--- a/tensorflow/python/estimator/BUILD
+++ b/tensorflow/python/estimator/BUILD
@@ -966,5 +966,6 @@ cuda_py_test(
     tags = [
         "multi_gpu",
         "noasan",  # flaky time outs
+        "notsan",  # flaky
     ],
 )
-- 
GitLab


From 90df8ab4ab127fac239597ce0708577a335613bf Mon Sep 17 00:00:00 2001
From: fo40225 <fo40225@users.noreply.github.com>
Date: Sat, 31 Mar 2018 04:16:16 +0800
Subject: [PATCH 769/960] fix test error (#18128)

---
 tensorflow/contrib/cmake/python_modules.txt | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 02c456c199..f273c7e550 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -335,6 +335,7 @@ tensorflow/contrib/nccl/kernels
 tensorflow/contrib/nccl/ops
 tensorflow/contrib/nccl/python
 tensorflow/contrib/nccl/python/ops
+tensorflow/contrib/nearest_neighbor
 tensorflow/contrib/nearest_neighbor/kernels
 tensorflow/contrib/nearest_neighbor/ops
 tensorflow/contrib/nearest_neighbor/python
-- 
GitLab


From 42b42fcab0031e36105cb1db105b70cbcfa6b125 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Fri, 30 Mar 2018 14:00:02 -0700
Subject: [PATCH 770/960] Remove unused imports

PiperOrigin-RevId: 191112880
---
 tensorflow/contrib/tpu/python/tpu/bfloat16.py      | 1 -
 tensorflow/contrib/tpu/python/tpu/bfloat16_test.py | 2 --
 2 files changed, 3 deletions(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/bfloat16.py b/tensorflow/contrib/tpu/python/tpu/bfloat16.py
index 929d1824c3..5e49af6408 100644
--- a/tensorflow/contrib/tpu/python/tpu/bfloat16.py
+++ b/tensorflow/contrib/tpu/python/tpu/bfloat16.py
@@ -22,7 +22,6 @@ from __future__ import print_function
 from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
-from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_contextlib
 
 
diff --git a/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py b/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py
index fda4331e89..48a01c7308 100644
--- a/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py
+++ b/tensorflow/contrib/tpu/python/tpu/bfloat16_test.py
@@ -20,9 +20,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.contrib.tpu.python.tpu import bfloat16
-from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variable_scope
 
 from tensorflow.python.platform import test
-- 
GitLab


From 5bc8b00f3c62549ff6954fa8929afce85c4da46b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 14:17:49 -0700
Subject: [PATCH 771/960] tf.maximum: Correctly calculate output shape when
 broadcast is necessary.

PiperOrigin-RevId: 191115726
---
 tensorflow/contrib/lite/kernels/maximum.cc     | 18 +++++++++++++++---
 .../contrib/lite/kernels/maximum_test.cc       | 14 ++++++++++++++
 2 files changed, 29 insertions(+), 3 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/maximum.cc b/tensorflow/contrib/lite/kernels/maximum.cc
index 9fdf2b47ea..13c40603ce 100644
--- a/tensorflow/contrib/lite/kernels/maximum.cc
+++ b/tensorflow/contrib/lite/kernels/maximum.cc
@@ -52,9 +52,21 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   MaximumContext op_context(context, node);
   TF_LITE_ENSURE_EQ(context, op_context.input1->type, op_context.input2->type);
-  TfLiteIntArray* output_dims = TfLiteIntArrayCopy(op_context.input2->dims);
-  op_context.output->type = op_context.input2->type;
-  return context->ResizeTensor(context, op_context.output, output_dims);
+  op_context.output->type = op_context.input1->type;
+
+  bool requires_broadcast =
+      !HaveSameShapes(op_context.input1, op_context.input2);
+
+  TfLiteIntArray* output_size = nullptr;
+  if (requires_broadcast) {
+    TF_LITE_ENSURE_OK(
+        context, CalculateShapeForBroadcast(context, op_context.input1,
+                                            op_context.input2, &output_size));
+  } else {
+    output_size = TfLiteIntArrayCopy(op_context.input1->dims);
+  }
+
+  return context->ResizeTensor(context, op_context.output, output_size);
 }
 
 template <KernelType kernel_type>
diff --git a/tensorflow/contrib/lite/kernels/maximum_test.cc b/tensorflow/contrib/lite/kernels/maximum_test.cc
index b3fd7d4e6f..df2bf29c20 100644
--- a/tensorflow/contrib/lite/kernels/maximum_test.cc
+++ b/tensorflow/contrib/lite/kernels/maximum_test.cc
@@ -71,6 +71,20 @@ TEST(MaximumOpTest, FloatTest) {
       ElementsAreArray(ArrayFloatNear({1.0, 0.0, 1.0, 12.0, -2.0, -1.43})));
 }
 
+TEST(MaximumOpTest, FloatWithBroadcastTest) {
+  std::initializer_list<float> data1 = {1.0, 0.0, -1.0, -2.0, -1.44, 11.0};
+  std::initializer_list<float> data2 = {0.5, 2.0};
+  MaximumOpModel m({TensorType_FLOAT32, {3, 1, 2}}, {TensorType_FLOAT32, {2}},
+                   TensorType_FLOAT32);
+  m.SetInput1<float>(data1);
+  m.SetInput2<float>(data2);
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 1, 2}));
+  EXPECT_THAT(
+      m.GetOutput<float>(),
+      ElementsAreArray(ArrayFloatNear({1.0, 2.0, 0.5, 2.0, 0.5, 11.0})));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From 79d38e6a570c351366481ab6d27a1e310da49046 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 30 Mar 2018 14:34:56 -0700
Subject: [PATCH 772/960] Updating release notes: changing minimum Cudnn
 supported version and adding tensorrt note.

---
 RELEASE.md | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/RELEASE.md b/RELEASE.md
index c63d9f20c9..886541572f 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -9,6 +9,8 @@
 * Distributed Mutex / CriticalSection added to `tf.contrib.framework.CriticalSection`.
 * Better text processing with `tf.regex_replace`.
 * Easy, efficient sequence input with `tf.contrib.data.bucket_by_sequence_length`
+* Initial support for `tf.contrib.tensorrt` that enables native TensorRT in
+  TensorFlow.
 
 ## Bug Fixes and Other Changes
 * Accelerated Linear Algebra (XLA):
@@ -50,6 +52,12 @@
   * Support `float16` `dtype` in `tf.linalg.*`.
   * Add `tf.estimator.export.TensorServingInputReceiver` that allows `tf.estimator.Estimator.export_savedmodel` to pass raw tensors to model functions.
 
+## Deprecations
+
+* TensorFlow 1.7 may be the last time we support cuDNN versions below 6.0.
+  Starting with TensorFlow 1.8 release, 6.0 will be the minimum supported
+  version.
+
 ## Thanks to our Contributors
 
 This release contains contributions from many people at Google, as well as:
-- 
GitLab


From 57de6ef5b168ea4d9ef81eb2d187d74aa7617ff6 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Fri, 30 Mar 2018 14:37:34 -0700
Subject: [PATCH 773/960] Also adding a note about Cuda supported version

---
 RELEASE.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/RELEASE.md b/RELEASE.md
index 886541572f..e845953174 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -54,6 +54,9 @@
 
 ## Deprecations
 
+* TensorFlow 1.7 may be the last time we support Cuda versions below 8.0.
+  Starting with TensorFlow 1.8 release, 8.0 will be the minimum supported
+  version.
 * TensorFlow 1.7 may be the last time we support cuDNN versions below 6.0.
   Starting with TensorFlow 1.8 release, 6.0 will be the minimum supported
   version.
-- 
GitLab


From 15c10899c9c0e1717251b380330cc248b2c76c9c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 14:45:21 -0700
Subject: [PATCH 774/960]  show breakdown of total execution time with compute
 and memory time

PiperOrigin-RevId: 191119550
---
 tensorflow/core/grappler/costs/virtual_scheduler.cc | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/core/grappler/costs/virtual_scheduler.cc b/tensorflow/core/grappler/costs/virtual_scheduler.cc
index 3ac3ae0f8f..0e5c654acf 100644
--- a/tensorflow/core/grappler/costs/virtual_scheduler.cc
+++ b/tensorflow/core/grappler/costs/virtual_scheduler.cc
@@ -44,6 +44,8 @@ Costs CombineCosts(const Costs& left, const Costs& right) {
 
   Costs result = left;
   result.execution_time += right.execution_time;
+  result.compute_time += right.compute_time;
+  result.memory_time += right.memory_time;
   if (right.inaccurate) {
     result.inaccurate = true;
   }
@@ -841,6 +843,8 @@ bool VirtualScheduler::MarkCurrNodeExecuted(const Costs& node_costs) {
 Costs VirtualScheduler::Summary() const {
   // Print out basic execution summary.
   VLOG(1) << "Expected execution time: " << graph_costs_.execution_time.count();
+  VLOG(1) << "Expected compute time: " << graph_costs_.compute_time.count();
+  VLOG(1) << "Expected memory time: " << graph_costs_.memory_time.count();
   VLOG(1) << "Expected max memory: " << graph_costs_.max_memory;
   VLOG(1) << "Expected max per-op buffers: " << graph_costs_.max_per_op_buffers;
   VLOG(1) << "Expected max per-op streaming buffers: "
-- 
GitLab


From 97731cb122f53552bd15351e046a256f78cca444 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Fri, 30 Mar 2018 14:56:08 -0700
Subject: [PATCH 775/960] Raise exception in SWIG on bad TF_Status from C API.

This change provides an alternative mechanism to
tf.raise_exception_on_not_ok_status(), which is inefficient and
error-prone (people often use the status multiple times in the with
block, but it's only checked when the context manager exits). Instead,
it uses SWIG to automatically raise an exception when a C API method
fails. Note that this removes the status argument from affected
methods.

For now, I've only applied this typemap to C API methods. It would be
good to expand this to all uses of raise_exception_on_not_ok_status.

PiperOrigin-RevId: 191121016
---
 tensorflow/c/c_api.h                          |  8 +-
 tensorflow/contrib/cmake/tf_python.cmake      |  2 +
 tensorflow/python/BUILD                       | 13 +++
 tensorflow/python/client/session.py           | 91 +++++++++----------
 .../client/session_list_devices_test.py       | 19 +---
 tensorflow/python/client/tf_session.i         | 33 ++++++-
 tensorflow/python/client/tf_session_helper.h  | 12 +--
 tensorflow/python/eager/context.py            |  8 +-
 tensorflow/python/eager/function.py           | 38 +++-----
 tensorflow/python/framework/errors_impl.py    |  3 +
 tensorflow/python/framework/function.py       | 34 +++----
 tensorflow/python/framework/importer.py       |  5 +-
 tensorflow/python/framework/load_library.py   |  7 +-
 tensorflow/python/framework/ops.py            | 87 +++++++-----------
 tensorflow/python/framework/smart_cond.py     |  6 +-
 .../python/lib/core/py_exception_registry.cc  | 50 ++++++++++
 .../python/lib/core/py_exception_registry.h   | 73 +++++++++++++++
 .../python/lib/core/py_exception_registry.i   | 28 ++++++
 tensorflow/python/tensorflow.i                |  2 +-
 19 files changed, 324 insertions(+), 195 deletions(-)
 create mode 100644 tensorflow/python/lib/core/py_exception_registry.cc
 create mode 100644 tensorflow/python/lib/core/py_exception_registry.h
 create mode 100644 tensorflow/python/lib/core/py_exception_registry.i

diff --git a/tensorflow/c/c_api.h b/tensorflow/c/c_api.h
index b32f574628..fe85f8ee0e 100644
--- a/tensorflow/c/c_api.h
+++ b/tensorflow/c/c_api.h
@@ -1496,7 +1496,8 @@ TF_CAPI_EXPORT extern int TF_DeviceListCount(const TF_DeviceList* list);
 // If index is out of bounds, an error code will be set in the status object,
 // and a null pointer will be returned.
 TF_CAPI_EXPORT extern const char* TF_DeviceListName(const TF_DeviceList* list,
-                                                    int index, TF_Status*);
+                                                    int index,
+                                                    TF_Status* status);
 
 // Retrieves the type of the device at the given index.
 //
@@ -1506,14 +1507,15 @@ TF_CAPI_EXPORT extern const char* TF_DeviceListName(const TF_DeviceList* list,
 // If index is out of bounds, an error code will be set in the status object,
 // and a null pointer will be returned.
 TF_CAPI_EXPORT extern const char* TF_DeviceListType(const TF_DeviceList* list,
-                                                    int index, TF_Status*);
+                                                    int index,
+                                                    TF_Status* status);
 
 // Retrieve the amount of memory associated with a given device.
 //
 // If index is out of bounds, an error code will be set in the status object,
 // and -1 will be returned.
 TF_CAPI_EXPORT extern int64_t TF_DeviceListMemoryBytes(
-    const TF_DeviceList* list, int index, TF_Status*);
+    const TF_DeviceList* list, int index, TF_Status* status);
 
 // --------------------------------------------------------------------------
 // Load plugins containing custom ops and kernels
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index b776307924..fae45ead5c 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -474,6 +474,8 @@ set (pywrap_tensorflow_internal_src
     "${tensorflow_source_dir}/tensorflow/python/lib/core/ndarray_tensor_bridge.cc"
     "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.h"
     "${tensorflow_source_dir}/tensorflow/python/lib/core/py_func.cc"
+    "${tensorflow_source_dir}/tensorflow/python/lib/core/py_exception_registry.h"
+    "${tensorflow_source_dir}/tensorflow/python/lib/core/py_exception_registry.cc"
     "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.h"
     "${tensorflow_source_dir}/tensorflow/python/lib/core/py_seq_tensor.cc"
     "${tensorflow_source_dir}/tensorflow/python/lib/core/py_util.h"
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index aa0acd243c..c502a3a42b 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -283,6 +283,17 @@ cc_library(
     ],
 )
 
+cc_library(
+    name = "py_exception_registry",
+    srcs = ["lib/core/py_exception_registry.cc"],
+    hdrs = ["lib/core/py_exception_registry.h"],
+    deps = [
+        "//tensorflow/c:c_api",
+        "//tensorflow/core:lib",
+        "//util/python:python_headers",
+    ],
+)
+
 cc_library(
     name = "kernel_registry",
     srcs = ["util/kernel_registry.cc"],
@@ -3313,6 +3324,7 @@ tf_py_wrap_cc(
         "grappler/model_analyzer.i",
         "grappler/tf_optimizer.i",
         "lib/core/bfloat16.i",
+        "lib/core/py_exception_registry.i",
         "lib/core/py_func.i",
         "lib/core/strings.i",
         "lib/io/file_io.i",
@@ -3344,6 +3356,7 @@ tf_py_wrap_cc(
         ":kernel_registry",
         ":numpy_lib",
         ":safe_ptr",
+        ":py_exception_registry",
         ":py_func_lib",
         ":py_record_reader_lib",
         ":py_record_writer_lib",
diff --git a/tensorflow/python/client/session.py b/tensorflow/python/client/session.py
index 5c9ed9ccaf..4c84d78f2e 100644
--- a/tensorflow/python/client/session.py
+++ b/tensorflow/python/client/session.py
@@ -27,7 +27,6 @@ import numpy as np
 
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow as tf_session
-from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import device
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
@@ -629,14 +628,12 @@ class BaseSession(SessionInterface):
     self._session = None
     opts = tf_session.TF_NewSessionOptions(target=self._target, config=config)
     try:
-      with errors.raise_exception_on_not_ok_status() as status:
-        if self._created_with_new_api:
-          # pylint: disable=protected-access
-          self._session = tf_session.TF_NewSession(self._graph._c_graph, opts,
-                                                   status)
-          # pylint: enable=protected-access
-        else:
-          self._session = tf_session.TF_NewDeprecatedSession(opts, status)
+      if self._created_with_new_api:
+        # pylint: disable=protected-access
+        self._session = tf_session.TF_NewSession(self._graph._c_graph, opts)
+        # pylint: enable=protected-access
+      else:
+        self._session = tf_session.TF_NewDeprecatedSession(opts)
     finally:
       tf_session.TF_DeleteSessionOptions(opts)
 
@@ -663,22 +660,20 @@ class BaseSession(SessionInterface):
     Returns:
       A list of devices in the session.
     """
-    with errors.raise_exception_on_not_ok_status() as status:
-      if self._created_with_new_api:
-        raw_device_list = tf_session.TF_SessionListDevices(
-            self._session, status)
-      else:
-        raw_device_list = tf_session.TF_DeprecatedSessionListDevices(
-            self._session, status)
-      device_list = []
-      size = tf_session.TF_DeviceListCount(raw_device_list)
-      for i in range(size):
-        name = tf_session.TF_DeviceListName(raw_device_list, i, status)
-        device_type = tf_session.TF_DeviceListType(raw_device_list, i, status)
-        memory = tf_session.TF_DeviceListMemoryBytes(raw_device_list, i, status)
-        device_list.append(_DeviceAttributes(name, device_type, memory))
-      tf_session.TF_DeleteDeviceList(raw_device_list)
-      return device_list
+    if self._created_with_new_api:
+      raw_device_list = tf_session.TF_SessionListDevices(self._session)
+    else:
+      raw_device_list = tf_session.TF_DeprecatedSessionListDevices(
+          self._session)
+    device_list = []
+    size = tf_session.TF_DeviceListCount(raw_device_list)
+    for i in range(size):
+      name = tf_session.TF_DeviceListName(raw_device_list, i)
+      device_type = tf_session.TF_DeviceListType(raw_device_list, i)
+      memory = tf_session.TF_DeviceListMemoryBytes(raw_device_list, i)
+      device_list.append(_DeviceAttributes(name, device_type, memory))
+    tf_session.TF_DeleteDeviceList(raw_device_list)
+    return device_list
 
   def close(self):
     """Closes this session.
@@ -692,15 +687,13 @@ class BaseSession(SessionInterface):
     if self._created_with_new_api:
       if self._session and not self._closed:
         self._closed = True
-        with errors.raise_exception_on_not_ok_status() as status:
-          tf_session.TF_CloseSession(self._session, status)
+        tf_session.TF_CloseSession(self._session)
 
     else:
       with self._extend_lock:
         if self._opened and not self._closed:
           self._closed = True
-          with errors.raise_exception_on_not_ok_status() as status:
-            tf_session.TF_CloseDeprecatedSession(self._session, status)
+          tf_session.TF_CloseDeprecatedSession(self._session)
 
   def __del__(self):
     # cleanly ignore all exceptions
@@ -710,11 +703,10 @@ class BaseSession(SessionInterface):
       pass
     if self._session is not None:
       try:
-        status = c_api_util.ScopedTFStatus()
         if self._created_with_new_api:
-          tf_session.TF_DeleteSession(self._session, status)
+          tf_session.TF_DeleteSession(self._session)
         else:
-          tf_session.TF_DeleteDeprecatedSession(self._session, status)
+          tf_session.TF_DeleteDeprecatedSession(self._session)
       except AttributeError:
         # At shutdown, `c_api_util` or `tf_session` may have been garbage
         # collected, causing the above method calls to fail. In this case,
@@ -1031,11 +1023,11 @@ class BaseSession(SessionInterface):
     # Set up a graph with feeds and fetches for partial run.
     def _setup_fn(session, feed_list, fetch_list, target_list):
       self._extend_graph()
-      with errors.raise_exception_on_not_ok_status() as status:
-        if self._created_with_new_api:
-          return tf_session.TF_SessionPRunSetup_wrapper(
-              session, feed_list, fetch_list, target_list, status)
-        else:
+      if self._created_with_new_api:
+        return tf_session.TF_SessionPRunSetup_wrapper(
+            session, feed_list, fetch_list, target_list)
+      else:
+        with errors.raise_exception_on_not_ok_status() as status:
           return tf_session.TF_PRunSetup(session, feed_list, fetch_list,
                                          target_list, status)
 
@@ -1345,8 +1337,7 @@ class BaseSession(SessionInterface):
   def _extend_graph(self):
     if self._created_with_new_api:
       with self._graph._lock:  # pylint: disable=protected-access
-        with errors.raise_exception_on_not_ok_status() as status:
-          tf_session.ExtendSession(self._session, status)
+        tf_session.ExtendSession(self._session)
     else:
       # Ensure any changes to the graph are reflected in the runtime.
       with self._extend_lock:
@@ -1412,22 +1403,22 @@ class BaseSession(SessionInterface):
 
   def _call_tf_sessionrun(self, options, feed_dict, fetch_list, target_list,
                           run_metadata):
-    with errors.raise_exception_on_not_ok_status() as status:
-      if self._created_with_new_api:
-        return tf_session.TF_SessionRun_wrapper(
-            self._session, options, feed_dict, fetch_list, target_list,
-            run_metadata, status)
-      else:
+    if self._created_with_new_api:
+      return tf_session.TF_SessionRun_wrapper(
+          self._session, options, feed_dict, fetch_list, target_list,
+          run_metadata)
+    else:
+      with errors.raise_exception_on_not_ok_status() as status:
         return tf_session.TF_Run(
             self._session, options, feed_dict, fetch_list, target_list,
             status, run_metadata)
 
   def _call_tf_sessionprun(self, handle, feed_dict, fetch_list):
-    with errors.raise_exception_on_not_ok_status() as status:
-      if self._created_with_new_api:
-        return tf_session.TF_SessionPRun_wrapper(
-            self._session, handle, feed_dict, fetch_list, status)
-      else:
+    if self._created_with_new_api:
+      return tf_session.TF_SessionPRun_wrapper(
+          self._session, handle, feed_dict, fetch_list)
+    else:
+      with errors.raise_exception_on_not_ok_status() as status:
         return tf_session.TF_PRun(
             self._session, handle, feed_dict, fetch_list, status)
 
diff --git a/tensorflow/python/client/session_list_devices_test.py b/tensorflow/python/client/session_list_devices_test.py
index 5a7413c12e..38a3acb2dc 100644
--- a/tensorflow/python/client/session_list_devices_test.py
+++ b/tensorflow/python/client/session_list_devices_test.py
@@ -23,7 +23,6 @@ from tensorflow.core.protobuf import cluster_pb2
 from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow as tf_session
 from tensorflow.python.client import session
-from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import test_util
@@ -42,21 +41,13 @@ class SessionListDevicesTestMethods(object):
 
   def testInvalidDeviceNumber(self):
     opts = tf_session.TF_NewSessionOptions()
-    with errors.raise_exception_on_not_ok_status() as status:
-      c_session = tf_session.TF_NewSession(
-          ops.get_default_graph()._c_graph, opts, status)
-      raw_device_list = tf_session.TF_SessionListDevices(
-          c_session, status)
+    c_session = tf_session.TF_NewSession(ops.get_default_graph()._c_graph, opts)
+    raw_device_list = tf_session.TF_SessionListDevices(c_session)
     size = tf_session.TF_DeviceListCount(raw_device_list)
-    # Test that invalid device numbers return -1 rather than a Swig-wrapped
-    # pointer.
-    status_no_exception = c_api_util.ScopedTFStatus()
-    memory = tf_session.TF_DeviceListMemoryBytes(
-        raw_device_list, size, status_no_exception)
-    self.assertEqual(memory, -1)
+    with self.assertRaises(errors.InvalidArgumentError):
+      tf_session.TF_DeviceListMemoryBytes(raw_device_list, size)
     tf_session.TF_DeleteDeviceList(raw_device_list)
-    with errors.raise_exception_on_not_ok_status() as status:
-      tf_session.TF_CloseSession(c_session, status)
+    tf_session.TF_CloseSession(c_session)
 
   def testListDevicesGrpcSession(self):
     server = server_lib.Server.create_local_server()
diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 77ce9195ee..5dcd0c192e 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -18,11 +18,12 @@ limitations under the License.
 %{
 
 #include "tensorflow/c/python_api.h"
-#include "tensorflow/python/client/tf_session_helper.h"
 #include "tensorflow/core/framework/session_state.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/public/version.h"
+#include "tensorflow/python/client/tf_session_helper.h"
+#include "tensorflow/python/lib/core/py_exception_registry.h"
 
 // Helper function to convert a Python list of Tensors to a C++ vector of
 // TF_Outputs.
@@ -352,6 +353,27 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
       reinterpret_cast<const char*>($1.data), $1.length);
 }
 
+// Typemaps to automatically raise a Python exception from bad output TF_Status.
+// TODO(b/77295559): expand this to all TF_Status* output params and deprecate
+// raise_exception_on_not_ok_status (currently it only affects the C API).
+%typemap(in, numinputs=0) TF_Status* status (TF_Status* status) {
+  status = TF_NewStatus();
+  $1 = status;
+}
+
+%typemap(argout) TF_Status* status {
+  TF_Code code = TF_GetCode($1);
+  if (code != TF_OK) {
+    PyObject* exc = tensorflow::PyExceptionRegistry::Lookup(code);
+    // Arguments to OpError.
+    PyObject* exc_args = Py_BuildValue("sss", nullptr, nullptr, TF_Message($1));
+    TF_DeleteStatus($1);
+    SWIG_SetErrorObj(exc, exc_args);
+    SWIG_fail;
+  }
+  TF_DeleteStatus($1);
+}
+
 // Converts input Python list of wrapped TF_Outputs into a single array
 %typemap(in) (const TF_Output* inputs, int num_inputs)
     (std::vector<TF_Output> inputs) {
@@ -499,9 +521,8 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
       _TF_SetTarget(opts, target)
     if config is not None:
       from tensorflow.python.framework import errors
-      with errors.raise_exception_on_not_ok_status() as status:
-        config_str = config.SerializeToString()
-        _TF_SetConfig(opts, config_str, status)
+      config_str = config.SerializeToString()
+      _TF_SetConfig(opts, config_str)
     return opts
 %}
 
@@ -758,3 +779,7 @@ def TF_Reset(target, containers=None, config=None):
 %include "tensorflow/python/client/tf_session_helper.h"
 
 %unignoreall
+
+// Clear "TF_Status* status" typemap so it doesn't affect other modules and
+// unexpectedly remove the TF_Status* argument from wrappers.
+%clear TF_Status* status;
diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h
index 603d03e315..5416d41376 100644
--- a/tensorflow/python/client/tf_session_helper.h
+++ b/tensorflow/python/client/tf_session_helper.h
@@ -136,8 +136,7 @@ string EqualAttrValueWrapper(const string& actual, const string& expected);
 //
 // If shape is unknown, sets unknown_shape to true.
 tensorflow::gtl::InlinedVector<int64_t, 6> TF_GraphGetTensorShapeHelper(
-    TF_Graph* graph, TF_Output output, TF_Status* out_status,
-    bool* unknown_shape);
+    TF_Graph* graph, TF_Output output, TF_Status* status, bool* unknown_shape);
 
 // Runs the graph associated with the session starting with the supplied inputs.
 // On success, `py_outputs` is populated with a numpy ndarray for each output
@@ -149,7 +148,7 @@ void TF_SessionRun_wrapper(TF_Session* session, const TF_Buffer* run_options,
                            const std::vector<PyObject*>& input_ndarrays,
                            const std::vector<TF_Output>& outputs,
                            const std::vector<TF_Operation*>& targets,
-                           TF_Buffer* run_metadata, TF_Status* out_status,
+                           TF_Buffer* run_metadata, TF_Status* status,
                            std::vector<PyObject*>* py_outputs);
 
 // Set up the graph with the intended feeds (inputs) and fetches (output) for
@@ -165,8 +164,7 @@ void TF_SessionPRunSetup_wrapper(TF_Session* session,
                                  const std::vector<TF_Output>& inputs,
                                  const std::vector<TF_Output>& outputs,
                                  const std::vector<TF_Operation*>& targets,
-                                 const char** out_handle,
-                                 TF_Status* out_status);
+                                 const char** out_handle, TF_Status* status);
 
 // Continue to run the graph with additional feeds and fetches. The
 // execution state is uniquely identified by the handle.
@@ -182,7 +180,7 @@ void TF_SessionPRun_wrapper(TF_Session* session, const char* handle,
                             const std::vector<TF_Output>& inputs,
                             const std::vector<PyObject*>& input_ndarrays,
                             const std::vector<TF_Output>& outputs,
-                            TF_Status* out_status,
+                            TF_Status* status,
                             std::vector<PyObject*>* py_outputs);
 
 // Retrieves the inputs of this operation.
@@ -204,7 +202,7 @@ TF_Function* TF_GraphToFunction_wrapper(
     const std::vector<TF_Operation*>* opers,
     const std::vector<TF_Output>& inputs, const std::vector<TF_Output>& outputs,
     const NameVector& output_names, const TF_FunctionOptions* opts,
-    const char* description, TF_Status* out_status);
+    const char* description, TF_Status* status);
 
 // Set the shapes and types for the output's handle.
 //
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 8c1bb06bc3..6ad9e0d88f 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -244,13 +244,9 @@ class Context(object):
       try:
         self._num_gpus = 0
         for i in range(pywrap_tensorflow.TF_DeviceListCount(device_list)):
-          with errors.raise_exception_on_not_ok_status() as status:
-            dev_name = pywrap_tensorflow.TF_DeviceListName(
-                device_list, i, status)
+          dev_name = pywrap_tensorflow.TF_DeviceListName(device_list, i)
           self._context_devices.append(pydev.canonical_name(dev_name))
-          with errors.raise_exception_on_not_ok_status() as status:
-            dev_type = pywrap_tensorflow.TF_DeviceListType(
-                device_list, i, status)
+          dev_type = pywrap_tensorflow.TF_DeviceListType(device_list, i)
           if dev_type == "GPU":
             self._num_gpus += 1
 
diff --git a/tensorflow/python/eager/function.py b/tensorflow/python/eager/function.py
index 343012e552..711eddcec1 100644
--- a/tensorflow/python/eager/function.py
+++ b/tensorflow/python/eager/function.py
@@ -34,7 +34,6 @@ from tensorflow.python.eager.graph_only_ops import graph_placeholder
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_module
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
@@ -79,14 +78,10 @@ def capture_value(tensor_map, value, dtype, name):
         ranks = [len(s.dim) if not s.unknown_rank else -1 for s in shapes]
         shapes = [[d.size for d in s.dim]
                   if not s.unknown_rank else None for s in shapes]
-        with errors.raise_exception_on_not_ok_status() as status:
-          pywrap_tensorflow.TF_GraphSetOutputHandleShapesAndTypes_wrapper(
-              captured_value._op._graph._c_graph,  # pylint: disable=protected-access
-              captured_value._as_tf_output(),  # pylint: disable=protected-access
-              shapes,
-              ranks,
-              types,
-              status)
+        pywrap_tensorflow.TF_GraphSetOutputHandleShapesAndTypes_wrapper(
+            captured_value._op._graph._c_graph,  # pylint: disable=protected-access
+            captured_value._as_tf_output(),  # pylint: disable=protected-access
+            shapes, ranks, types)
 
     tensor_map[ops.tensor_id(value)] = (value, captured_value)
   else:
@@ -275,23 +270,20 @@ class _EagerDefinedFunction(object):
       inputs: the tensors in the graph to be used as inputs to the function
       outputs: the tensors in the graph which will be outputs to the function
     """
-    with errors.raise_exception_on_not_ok_status() as status:
-      fn = pywrap_tensorflow.TF_GraphToFunction_wrapper(
-          graph._c_graph,  # pylint: disable=protected-access
-          compat.as_str(name),
-          False,
-          [o._c_op for o in operations],  # pylint: disable=protected-access
-          [t._as_tf_output() for t in inputs],  # pylint: disable=protected-access
-          [t._as_tf_output() for t in outputs],  # pylint: disable=protected-access
-          [],
-          None,
-          compat.as_str(""),
-          status)
+    fn = pywrap_tensorflow.TF_GraphToFunction_wrapper(
+        graph._c_graph,  # pylint: disable=protected-access
+        compat.as_str(name),
+        False,
+        [o._c_op for o in operations],  # pylint: disable=protected-access
+        [t._as_tf_output() for t in inputs],  # pylint: disable=protected-access
+        [t._as_tf_output() for t in outputs],  # pylint: disable=protected-access
+        [],
+        None,
+        compat.as_str(""))
     # TODO(apassos) avoid creating a FunctionDef (specially to grab the
     # signature, but also in general it's nice not to depend on it.
     with c_api_util.tf_buffer() as buffer_:
-      with errors.raise_exception_on_not_ok_status() as status:
-        pywrap_tensorflow.TF_FunctionToFunctionDef(fn, buffer_, status)
+      pywrap_tensorflow.TF_FunctionToFunctionDef(fn, buffer_)
       proto_data = pywrap_tensorflow.TF_GetBuffer(buffer_)
     function_def = function_pb2.FunctionDef()
     function_def.ParseFromString(compat.as_bytes(proto_data))
diff --git a/tensorflow/python/framework/errors_impl.py b/tensorflow/python/framework/errors_impl.py
index 2a40316d51..84106c32c6 100644
--- a/tensorflow/python/framework/errors_impl.py
+++ b/tensorflow/python/framework/errors_impl.py
@@ -473,6 +473,8 @@ _CODE_TO_EXCEPTION_CLASS = {
     DATA_LOSS: DataLossError,
 }
 
+c_api.PyExceptionRegistry_Init(_CODE_TO_EXCEPTION_CLASS)
+
 _EXCEPTION_CLASS_TO_CODE = dict((
     (class_, code) for (code, class_) in _CODE_TO_EXCEPTION_CLASS.items()))
 
@@ -499,6 +501,7 @@ def _make_specific_exception(node_def, op, message, error_code):
 # Named like a function for backwards compatibility with the
 # @tf_contextlib.contextmanager version, which was switched to a class to avoid
 # some object creation overhead.
+# TODO(b/77295559): expand use of TF_Status* SWIG typemap and deprecate this.
 @tf_export("errors.raise_exception_on_not_ok_status")  # pylint: disable=invalid-name
 class raise_exception_on_not_ok_status(object):
   """Context manager to check for C API status."""
diff --git a/tensorflow/python/framework/function.py b/tensorflow/python/framework/function.py
index 82dd2a3356..c5caf9ebc0 100644
--- a/tensorflow/python/framework/function.py
+++ b/tensorflow/python/framework/function.py
@@ -30,7 +30,6 @@ from tensorflow.python import pywrap_tensorflow as c_api
 from tensorflow.python.eager import context
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import graph_to_function_def
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -275,8 +274,7 @@ class _DefinedFunction(object):
     self._create_definition_if_needed()
     if self._c_func:
       with c_api_util.tf_buffer() as buf:
-        with errors.raise_exception_on_not_ok_status() as status:
-          c_api.TF_FunctionToFunctionDef(self._c_func, buf, status)
+        c_api.TF_FunctionToFunctionDef(self._c_func, buf)
         fdef = function_pb2.FunctionDef()
         proto_data = c_api.TF_GetBuffer(buf)
         fdef.ParseFromString(compat.as_bytes(proto_data))
@@ -399,18 +397,16 @@ class _DefinedFunction(object):
                       if self._out_names else [])
       description = self._func.__doc__ or None
       # pylint: disable=protected-access
-      with errors.raise_exception_on_not_ok_status() as status:
-        self._c_func = c_api.TF_GraphToFunction_wrapper(
-            temp_graph._c_graph,
-            base_func_name,
-            self._func_name is None,  # append_hash_to_fn_name
-            None,  # opers
-            [t._as_tf_output() for t in inputs],
-            [t._as_tf_output() for t in outputs],
-            output_names,
-            None,  # opts
-            description,
-            status)
+      self._c_func = c_api.TF_GraphToFunction_wrapper(
+          temp_graph._c_graph,
+          base_func_name,
+          self._func_name is None,  # append_hash_to_fn_name
+          None,  # opers
+          [t._as_tf_output() for t in inputs],
+          [t._as_tf_output() for t in outputs],
+          output_names,
+          None,  # opts
+          description)
       # pylint: enable=protected-access
       self._set_c_attrs(kwargs_attr)
 
@@ -433,9 +429,8 @@ class _DefinedFunction(object):
       serialized = attr_value.SerializeToString()
       # TODO(skyewm): this creates and deletes a new TF_Status for every attr.
       # It might be worth creating a convenient way to re-use the same status.
-      with errors.raise_exception_on_not_ok_status() as status:
-        c_api.TF_FunctionSetAttrValueProto(self._c_func, compat.as_str(name),
-                                           serialized, status)
+      c_api.TF_FunctionSetAttrValueProto(self._c_func, compat.as_str(name),
+                                         serialized)
 
   def _create_hash_str(self, input_arg, output_arg, node_def):
     """Creates an 8-character string unique to this input.
@@ -830,8 +825,7 @@ def _from_definition(fdef, grad_func=None):
   # pylint: disable=protected-access
   if ops._USE_C_API:
     serialized = fdef.SerializeToString()
-    with errors.raise_exception_on_not_ok_status() as status:
-      result._c_func = c_api.TF_FunctionImportFunctionDef(serialized, status)
+    result._c_func = c_api.TF_FunctionImportFunctionDef(serialized)
     result._extra_inputs = []
   else:
     result._definition = fdef
diff --git a/tensorflow/python/framework/importer.py b/tensorflow/python/framework/importer.py
index 4ea34d7bb2..23f529b988 100644
--- a/tensorflow/python/framework/importer.py
+++ b/tensorflow/python/framework/importer.py
@@ -485,9 +485,8 @@ def import_graph_def(graph_def,
     with graph._lock:  # pylint: disable=protected-access
       with c_api_util.tf_buffer(graph_def.SerializeToString()) as serialized:
         try:
-          with errors.raise_exception_on_not_ok_status() as status:
-            results = c_api.TF_GraphImportGraphDefWithResults(
-                graph._c_graph, serialized, options, status)  # pylint: disable=protected-access
+          results = c_api.TF_GraphImportGraphDefWithResults(
+              graph._c_graph, serialized, options)  # pylint: disable=protected-access
         except errors.InvalidArgumentError as e:
           # Convert to ValueError for backwards compatibility.
           raise ValueError(str(e))
diff --git a/tensorflow/python/framework/load_library.py b/tensorflow/python/framework/load_library.py
index 1f2aa264c1..535c6017f5 100644
--- a/tensorflow/python/framework/load_library.py
+++ b/tensorflow/python/framework/load_library.py
@@ -26,7 +26,6 @@ import threading  # pylint: disable=unused-import
 from tensorflow.core.framework import op_def_pb2
 from tensorflow.core.lib.core import error_codes_pb2  # pylint: disable=unused-import
 from tensorflow.python import pywrap_tensorflow as py_tf
-from tensorflow.python.framework import errors_impl
 from tensorflow.python.util import compat
 from tensorflow.python.util.tf_export import tf_export
 
@@ -54,8 +53,7 @@ def load_op_library(library_filename):
   Raises:
     RuntimeError: when unable to load the library or get the python wrappers.
   """
-  with errors_impl.raise_exception_on_not_ok_status() as status:
-    lib_handle = py_tf.TF_LoadLibrary(library_filename, status)
+  lib_handle = py_tf.TF_LoadLibrary(library_filename)
 
   op_list_str = py_tf.TF_GetOpList(lib_handle)
   op_list = op_def_pb2.OpList()
@@ -99,5 +97,4 @@ def load_file_system_library(library_filename):
   Raises:
     RuntimeError: when unable to load the library.
   """
-  with errors_impl.raise_exception_on_not_ok_status() as status:
-    lib_handle = py_tf.TF_LoadLibrary(library_filename, status)
+  py_tf.TF_LoadLibrary(library_filename)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 6930737a0c..7ca0b836dd 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -373,15 +373,12 @@ class Tensor(_TensorLike):
     """
     graph = self._op._graph._c_graph # pylint: disable=protected-access
     if graph and _USE_C_SHAPES:
-      with errors.raise_exception_on_not_ok_status() as status:
-        num_dims = c_api.TF_GraphGetTensorNumDims(graph, self._as_tf_output(),
-                                                  status)
+      num_dims = c_api.TF_GraphGetTensorNumDims(graph, self._as_tf_output())
       if num_dims == -1:
         dim_list = None
       else:
-        with errors.raise_exception_on_not_ok_status() as status:
-          dim_list = c_api.TF_GraphGetTensorShape_wrapper(
-              graph, self._as_tf_output(), num_dims, status)
+        dim_list = c_api.TF_GraphGetTensorShape_wrapper(
+            graph, self._as_tf_output(), num_dims)
         dim_list = [None if i == -1 else i for i in dim_list]
       return tensor_shape.TensorShape(dim_list)
     return self._shape_val
@@ -489,13 +486,11 @@ class Tensor(_TensorLike):
         else:
           dim_list.append(dim.value)
     try:
-      with errors.raise_exception_on_not_ok_status() as status:
-        c_api.TF_GraphSetTensorShape_wrapper(
-            self._op._graph._c_graph,  # pylint: disable=protected-access
-            self._as_tf_output(),
-            dim_list,
-            unknown_shape,
-            status)
+      c_api.TF_GraphSetTensorShape_wrapper(
+          self._op._graph._c_graph,  # pylint: disable=protected-access
+          self._as_tf_output(),
+          dim_list,
+          unknown_shape)
     except errors.InvalidArgumentError as e:
       # Convert to ValueError for backwards compatibility.
       raise ValueError(str(e))
@@ -1514,13 +1509,10 @@ def _create_c_op(graph, node_def, inputs, control_inputs):
     serialized = attr_value.SerializeToString()
     # TODO(skyewm): this creates and deletes a new TF_Status for every attr.
     # It might be worth creating a convenient way to re-use the same status.
-    with errors.raise_exception_on_not_ok_status() as status:
-      c_api.TF_SetAttrValueProto(op_desc,
-                                 compat.as_str(name), serialized, status)
+    c_api.TF_SetAttrValueProto(op_desc, compat.as_str(name), serialized)
 
   try:
-    with errors.raise_exception_on_not_ok_status() as status:
-      c_op = c_api.TF_FinishOperation(op_desc, status)
+    c_op = c_api.TF_FinishOperation(op_desc)
   except errors.InvalidArgumentError as e:
     # Convert to ValueError for backwards compatibility.
     raise ValueError(str(e))
@@ -1943,12 +1935,10 @@ class Operation(object):
     if self._c_op:
       # Reset cached inputs.
       self._inputs_val = None
-      with errors.raise_exception_on_not_ok_status() as status:
-        c_api.UpdateEdge(
-            self._graph._c_graph,  # pylint: disable=protected-access
-            tensor._as_tf_output(),  # pylint: disable=protected-access
-            self._tf_input(index),
-            status)
+      c_api.UpdateEdge(
+          self._graph._c_graph,  # pylint: disable=protected-access
+          tensor._as_tf_output(),  # pylint: disable=protected-access
+          self._tf_input(index))
     else:
       self._inputs_val[index].consumers().remove(self)
       self._inputs_val[index] = tensor
@@ -2169,8 +2159,7 @@ class Operation(object):
     # pylint: enable=line-too-long
     if self._c_op:
       with c_api_util.tf_buffer() as buf:
-        with errors.raise_exception_on_not_ok_status() as status:
-          c_api.TF_OperationToNodeDef(self._c_op, buf, status)
+        c_api.TF_OperationToNodeDef(self._c_op, buf)
         data = c_api.TF_GetBuffer(buf)
       node_def = node_def_pb2.NodeDef()
       node_def.ParseFromString(compat.as_bytes(data))
@@ -2228,11 +2217,9 @@ class Operation(object):
       buf = c_api.TF_NewBufferFromString(
           compat.as_bytes(attr_value.SerializeToString()))
       try:
-        with errors.raise_exception_on_not_ok_status() as status:
-          # pylint: disable=protected-access
-          c_api.SetAttr(self._graph._c_graph, self._c_op, attr_name, buf,
-                        status)
-          # pylint: enable=protected-access
+        # pylint: disable=protected-access
+        c_api.SetAttr(self._graph._c_graph, self._c_op, attr_name, buf)
+        # pylint: enable=protected-access
       finally:
         c_api.TF_DeleteBuffer(buf)
     else:
@@ -2254,8 +2241,7 @@ class Operation(object):
     if self._c_op:
       try:
         with c_api_util.tf_buffer() as buf:
-          with errors.raise_exception_on_not_ok_status() as status:
-            c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf, status)
+          c_api.TF_OperationGetAttrValueProto(self._c_op, name, buf)
           data = c_api.TF_GetBuffer(buf)
       except errors.InvalidArgumentError as e:
         # Convert to ValueError for backwards compatibility.
@@ -2469,11 +2455,10 @@ def _set_shapes_for_outputs_c_api(op):
   # The C API computes the shapes when the TF_Operation is created. Fetch the
   # output shapes from the C object.
   for output in op.outputs:
-    with errors.raise_exception_on_not_ok_status() as status:
-      # pylint: disable=protected-access
-      shape_vector, unknown_shape = c_api.TF_GraphGetTensorShapeHelper(
-          op._graph._c_graph, output._as_tf_output(), status)
-      # pylint: enable=protected-access
+    # pylint: disable=protected-access
+    shape_vector, unknown_shape = c_api.TF_GraphGetTensorShapeHelper(
+        op._graph._c_graph, output._as_tf_output())
+    # pylint: enable=protected-access
     if unknown_shape:
       output.set_shape(tensor_shape.unknown_shape())
     elif not shape_vector:
@@ -2994,8 +2979,7 @@ class Graph(object):
     # pylint: enable=line-too-long
     if self._c_graph:
       with c_api_util.tf_buffer() as buf:
-        with errors.raise_exception_on_not_ok_status() as status:
-          c_api.TF_GraphVersions(self._c_graph, buf, status)
+        c_api.TF_GraphVersions(self._c_graph, buf)
         data = c_api.TF_GetBuffer(buf)
       version_def = versions_pb2.VersionDef()
       version_def.ParseFromString(compat.as_bytes(data))
@@ -3098,8 +3082,7 @@ class Graph(object):
     if self._c_graph:
       with self._lock:
         with c_api_util.tf_buffer() as buf:
-          with errors.raise_exception_on_not_ok_status() as status:
-            c_api.TF_GraphToGraphDef(self._c_graph, buf, status)
+          c_api.TF_GraphToGraphDef(self._c_graph, buf)
           data = c_api.TF_GetBuffer(buf)
         graph = graph_pb2.GraphDef()
         graph.ParseFromString(compat.as_bytes(data))
@@ -3208,14 +3191,10 @@ class Graph(object):
       # remove this when all functions are generated using the C API by default
       # as this will be unnecessary.
       if not function._c_func:
-        with errors.raise_exception_on_not_ok_status() as status:
-          serialized = function.definition.SerializeToString()
-          function._c_func = c_api.TF_FunctionImportFunctionDef(
-              serialized, status)
-      with errors.raise_exception_on_not_ok_status() as status:
-        gradient = function._grad_func._c_func if function._grad_func else None
-        c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient,
-                                   status)
+        serialized = function.definition.SerializeToString()
+        function._c_func = c_api.TF_FunctionImportFunctionDef(serialized)
+      gradient = function._grad_func._c_func if function._grad_func else None
+      c_api.TF_GraphCopyFunction(self._c_graph, function._c_func, gradient)
     else:
       # If there is already a function with the same name, raise an error
       # if bodies are different. Else, do nothing. The C API version above
@@ -3732,11 +3711,9 @@ class Graph(object):
     """Returns the `OpDef` proto for `type`. `type` is a string."""
     if self._c_graph:
       with c_api_util.tf_buffer() as buf:
-        with errors.raise_exception_on_not_ok_status() as status:
-          # pylint: disable=protected-access
-          c_api.TF_GraphGetOpDef(self._c_graph,
-                                 compat.as_bytes(type), buf, status)
-          # pylint: enable=protected-access
+        # pylint: disable=protected-access
+        c_api.TF_GraphGetOpDef(self._c_graph, compat.as_bytes(type), buf)
+        # pylint: enable=protected-access
         data = c_api.TF_GetBuffer(buf)
       op_def = op_def_pb2.OpDef()
       op_def.ParseFromString(compat.as_bytes(data))
diff --git a/tensorflow/python/framework/smart_cond.py b/tensorflow/python/framework/smart_cond.py
index c7ff23e4ff..48a834392b 100644
--- a/tensorflow/python/framework/smart_cond.py
+++ b/tensorflow/python/framework/smart_cond.py
@@ -19,7 +19,6 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python import pywrap_tensorflow as c_api
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.ops import control_flow_ops
@@ -83,9 +82,8 @@ def smart_constant_value(pred):
     # wanted to limit the change hidden behind _USE_C_API).
     # pylint: disable=protected-access
     if pred_value is None and ops._USE_C_API:
-      with errors.raise_exception_on_not_ok_status() as status:
-        pred_value = c_api.TF_TryEvaluateConstant_wrapper(
-            pred.graph._c_graph, pred._as_tf_output(), status)
+      pred_value = c_api.TF_TryEvaluateConstant_wrapper(pred.graph._c_graph,
+                                                        pred._as_tf_output())
     # pylint: enable=protected-access
 
   else:
diff --git a/tensorflow/python/lib/core/py_exception_registry.cc b/tensorflow/python/lib/core/py_exception_registry.cc
new file mode 100644
index 0000000000..6637de632b
--- /dev/null
+++ b/tensorflow/python/lib/core/py_exception_registry.cc
@@ -0,0 +1,50 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/python/lib/core/py_exception_registry.h"
+
+#include <Python.h>
+
+namespace tensorflow {
+
+PyExceptionRegistry* PyExceptionRegistry::singleton_ = nullptr;
+
+void PyExceptionRegistry::Init(PyObject* code_to_exc_type_map) {
+  DCHECK(singleton_ == nullptr) << "PyExceptionRegistry::Init() already called";
+  singleton_ = new PyExceptionRegistry;
+
+  DCHECK(PyDict_Check(code_to_exc_type_map));
+  PyObject* key;
+  PyObject* value;
+  Py_ssize_t pos = 0;
+  while (PyDict_Next(code_to_exc_type_map, &pos, &key, &value)) {
+    TF_Code code = static_cast<TF_Code>(PyLong_AsLong(key));
+    singleton_->exc_types_[code] = value;
+    // The exception classes should also have the lifetime of the process, but
+    // incref just in case.
+    Py_INCREF(value);
+  }
+}
+
+PyObject* PyExceptionRegistry::Lookup(TF_Code code) {
+  DCHECK(singleton_ != nullptr) << "Must call PyExceptionRegistry::Init() "
+                                   "before PyExceptionRegistry::Lookup()";
+  DCHECK_NE(code, TF_OK);
+  DCHECK(singleton_->exc_types_.find(code) != singleton_->exc_types_.end())
+      << "Unknown error code passed to PyExceptionRegistry::Lookup: " << code;
+  return singleton_->exc_types_[code];
+}
+
+}  // namespace tensorflow
diff --git a/tensorflow/python/lib/core/py_exception_registry.h b/tensorflow/python/lib/core/py_exception_registry.h
new file mode 100644
index 0000000000..2b0f23b548
--- /dev/null
+++ b/tensorflow/python/lib/core/py_exception_registry.h
@@ -0,0 +1,73 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#ifndef TENSORFLOW_PYTHON_LIB_CORE_PY_EXCEPTION_REGISTRY_H_
+#define TENSORFLOW_PYTHON_LIB_CORE_PY_EXCEPTION_REGISTRY_H_
+
+#include <map>
+
+#include "tensorflow/c/c_api.h"
+#include "tensorflow/core/platform/logging.h"
+
+#ifndef PyObject_HEAD
+struct _object;
+typedef _object PyObject;
+#endif
+
+namespace tensorflow {
+
+// Global registry mapping C API error codes to the corresponding custom Python
+// exception type. This is used to expose the exception types to C extension
+// code (i.e. so we can raise custom exceptions via SWIG).
+//
+// Init() must be called exactly once at the beginning of the process before
+// Lookup() can be used.
+//
+// Example usage:
+//   TF_Status* status = TF_NewStatus();
+//   TF_Foo(..., status);
+//
+//   if (TF_GetCode(status) != TF_OK) {
+//     PyObject* exc_type = PyExceptionRegistry::Lookup(TF_GetCode(status));
+//     // Arguments to OpError base class. Set `node_def` and `op` to None.
+//     PyObject* args =
+//       Py_BuildValue("sss", nullptr, nullptr, TF_Message(status));
+//     PyErr_SetObject(exc_type, args);
+//     Py_DECREF(args);
+//     TF_DeleteStatus(status);
+//     return NULL;
+//   }
+class PyExceptionRegistry {
+ public:
+  // Initializes the process-wide registry. Should be called exactly once near
+  // the beginning of the process. The arguments are the various Python
+  // exception types (e.g. `cancelled_exc` corresponds to
+  // errors.CancelledError).
+  static void Init(PyObject* code_to_exc_type_map);
+
+  // Returns the Python exception type corresponding to `code`. Init() must be
+  // called before using this function. `code` should not be TF_OK.
+  static PyObject* Lookup(TF_Code code);
+
+ private:
+  static PyExceptionRegistry* singleton_;
+  PyExceptionRegistry() = default;
+
+  // Maps error codes to the corresponding Python exception type.
+  std::map<TF_Code, PyObject*> exc_types_;
+};
+
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_PYTHON_LIB_CORE_PY_EXCEPTION_REGISTRY_H_
diff --git a/tensorflow/python/lib/core/py_exception_registry.i b/tensorflow/python/lib/core/py_exception_registry.i
new file mode 100644
index 0000000000..e872b74985
--- /dev/null
+++ b/tensorflow/python/lib/core/py_exception_registry.i
@@ -0,0 +1,28 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+%include "tensorflow/python/platform/base.i"
+
+%{
+#include "tensorflow/python/lib/core/py_exception_registry.h"
+%}
+
+%ignoreall
+
+%unignore tensorflow::PyExceptionRegistry;
+%unignore tensorflow::PyExceptionRegistry::Init;
+
+%include "tensorflow/python/lib/core/py_exception_registry.h"
+%unignoreall
diff --git a/tensorflow/python/tensorflow.i b/tensorflow/python/tensorflow.i
index 82b908ac0e..26e8acd897 100644
--- a/tensorflow/python/tensorflow.i
+++ b/tensorflow/python/tensorflow.i
@@ -25,6 +25,7 @@ limitations under the License.
 %include "tensorflow/python/util/tfprof.i"
 
 %include "tensorflow/python/lib/core/py_func.i"
+%include "tensorflow/python/lib/core/py_exception_registry.i"
 
 %include "tensorflow/python/lib/io/py_record_reader.i"
 %include "tensorflow/python/lib/io/py_record_writer.i"
@@ -54,4 +55,3 @@ limitations under the License.
 %include "tensorflow/python/grappler/tf_optimizer.i"
 %include "tensorflow/python/grappler/cost_analyzer.i"
 %include "tensorflow/python/grappler/model_analyzer.i"
-
-- 
GitLab


From 32133ec21ff01f829bc2f2ac19a5f78632bc07dc Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Fri, 30 Mar 2018 15:00:41 -0700
Subject: [PATCH 776/960] Make tfe.Iterator work with async mode.

PiperOrigin-RevId: 191121622
---
 tensorflow/contrib/eager/python/datasets.py   | 24 +++++++-----
 .../data/kernel_tests/iterator_ops_test.py    |  9 +++++
 tensorflow/python/data/ops/iterator_ops.py    | 38 ++++++++++---------
 3 files changed, 44 insertions(+), 27 deletions(-)

diff --git a/tensorflow/contrib/eager/python/datasets.py b/tensorflow/contrib/eager/python/datasets.py
index 60453006f4..99b1e098d5 100644
--- a/tensorflow/contrib/eager/python/datasets.py
+++ b/tensorflow/contrib/eager/python/datasets.py
@@ -107,16 +107,20 @@ class Iterator(iterator_ops.EagerIterator, checkpointable.CheckpointableBase):
   def _next_internal(self):
     """Returns a nested structure of `tf.Tensor`s containing the next element.
     """
-    if self._buffer_resource_handle is not None:
-      with ops.device(self._device):
-        ret = prefetching_ops.function_buffering_resource_get_next(
-            function_buffer_resource=self._buffer_resource_handle,
-            output_types=self._flat_output_types)
-      return sparse.deserialize_sparse_tensors(
-          nest.pack_sequence_as(self._output_types, ret), self._output_types,
-          self._output_shapes, self._output_classes)
-    else:
-      return super(Iterator, self)._next_internal()
+    # This runs in sync mode as iterators use an error status to communicate
+    # that there is no more data to iterate over.
+    # TODO(b/77291417): Fix
+    with context.execution_mode(context.SYNC):
+      if self._buffer_resource_handle is not None:
+        with ops.device(self._device):
+          ret = prefetching_ops.function_buffering_resource_get_next(
+              function_buffer_resource=self._buffer_resource_handle,
+              output_types=self._flat_output_types)
+        return sparse.deserialize_sparse_tensors(
+            nest.pack_sequence_as(self._output_types, ret), self._output_types,
+            self._output_shapes, self._output_classes)
+      else:
+        return super(Iterator, self)._next_internal()
 
   # TODO(shivaniagrawal): Expose checkpointable stateful objects from dataset
   # attributes(potential).
diff --git a/tensorflow/python/data/kernel_tests/iterator_ops_test.py b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
index 4a14a915bd..0af282a024 100644
--- a/tensorflow/python/data/kernel_tests/iterator_ops_test.py
+++ b/tensorflow/python/data/kernel_tests/iterator_ops_test.py
@@ -28,6 +28,7 @@ from tensorflow.python.client import session
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.ops import iterator_ops
 from tensorflow.python.data.ops import readers
+from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
@@ -717,6 +718,14 @@ class IteratorTest(test.TestCase):
       self.assertTrue(
           iterator_ops.GET_NEXT_CALL_WARNING_MESSAGE in str(warning.message))
 
+  def testEagerIteratorAsync(self):
+    with context.eager_mode(), context.execution_mode(context.ASYNC):
+      val = 0
+      dataset = dataset_ops.Dataset.range(10)
+      for foo in dataset:
+        self.assertEqual(val, foo.numpy())
+        val += 1
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/data/ops/iterator_ops.py b/tensorflow/python/data/ops/iterator_ops.py
index d79b9d6011..0c76afd29d 100644
--- a/tensorflow/python/data/ops/iterator_ops.py
+++ b/tensorflow/python/data/ops/iterator_ops.py
@@ -488,23 +488,27 @@ class EagerIterator(object):
   def _next_internal(self):
     """Returns a nested structure of `tf.Tensor`s containing the next element.
     """
-    with ops.device(self._device):
-      # TODO(ashankar): Consider removing this ops.device() contextmanager
-      # and instead mimic ops placement in graphs: Operations on resource
-      # handles execute on the same device as where the resource is placed.
-      # NOTE(mrry): Here we use the "_sync" variant of `iterator_get_next`
-      # because in eager mode this code will run synchronously on the calling
-      # thread. Therefore we do not need to make a defensive context switch
-      # to a background thread, and can achieve a small constant performance
-      # boost by invoking the iterator synchronously.
-      ret = gen_dataset_ops.iterator_get_next_sync(
-          self._resource,
-          output_types=self._flat_output_types,
-          output_shapes=self._flat_output_shapes)
-
-    return sparse.deserialize_sparse_tensors(
-        nest.pack_sequence_as(self._output_types, ret), self._output_types,
-        self._output_shapes, self._output_classes)
+    # This runs in sync mode as iterators use an error status to communicate
+    # that there is no more data to iterate over.
+    # TODO(b/77291417): Fix
+    with context.execution_mode(context.SYNC):
+      with ops.device(self._device):
+        # TODO(ashankar): Consider removing this ops.device() contextmanager
+        # and instead mimic ops placement in graphs: Operations on resource
+        # handles execute on the same device as where the resource is placed.
+        # NOTE(mrry): Here we use the "_sync" variant of `iterator_get_next`
+        # because in eager mode this code will run synchronously on the calling
+        # thread. Therefore we do not need to make a defensive context switch
+        # to a background thread, and can achieve a small constant performance
+        # boost by invoking the iterator synchronously.
+        ret = gen_dataset_ops.iterator_get_next_sync(
+            self._resource,
+            output_types=self._flat_output_types,
+            output_shapes=self._flat_output_shapes)
+
+      return sparse.deserialize_sparse_tensors(
+          nest.pack_sequence_as(self._output_types, ret), self._output_types,
+          self._output_shapes, self._output_classes)
 
   def next(self):
     """Returns a nested structure of `tf.Tensor`s containing the next element.
-- 
GitLab


From ef20c62273a5c20d5fce28fd779aa47fb2382ac5 Mon Sep 17 00:00:00 2001
From: Dimitris Vardoulakis <dimvar@google.com>
Date: Fri, 30 Mar 2018 15:15:31 -0700
Subject: [PATCH 777/960] Improve the precision of Reduce in the interpreter
 when adding floats, by accumulating in a double.

Also, speed it up by not creating intermediate Literals. The microbenchmark now runs in < 1 ns, as opposed to 30 sec before.

PiperOrigin-RevId: 191123983
---
 .../compiler/xla/service/hlo_evaluator.cc     | 46 +++++++++---
 .../xla/service/hlo_evaluator_test.cc         | 75 +++++++++++++++++++
 2 files changed, 110 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_evaluator.cc b/tensorflow/compiler/xla/service/hlo_evaluator.cc
index 693004d364..9d7251b6ae 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator.cc
@@ -1520,14 +1520,12 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
       arg_dim_counts[dim] = arg_dimensions[dim];
     }
 
-    // Create mapping from result index to arg index.
-    const int64 result_rank = ShapeUtil::Rank(result->shape());
-    int64 result_dim = 0;
-    std::vector<int64> result_to_arg_index(result_rank);
+    // Map each dimension in the result to a dimension in arg that isn't
+    // being reduced.
+    std::vector<int64> result_to_arg_index;
     for (int64 i = 0; i < arg_dimensions.size(); ++i) {
       if (arg_dim_steps[i] == 0) {
-        result_to_arg_index[result_dim] = i;
-        ++result_dim;
+        result_to_arg_index.push_back(i);
       }
     }
 
@@ -1542,6 +1540,20 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
             base[result_to_arg_index[i]] = multi_index[i];
           }
 
+          // When the reduction is addition of floats, accumulate in a double
+          // for better precision. Also, avoid creating Literals for the
+          // intermediate results; it's much faster.
+          if (ShapeUtil::ElementIsFloating(init_literal.shape()) &&
+              IsScalarAdd(function)) {
+            double computed_result = 0;
+            auto func = [&](ArraySlice<int64> input_index) {
+              computed_result += arg_literal.Get<float>(input_index);
+              return true;
+            };
+            ShapeUtil::ForEachIndex(arg_literal.shape(), base, arg_dim_counts,
+                                    arg_dim_steps, func);
+            return static_cast<ReturnT>(computed_result);
+          }
           auto func = [&](ArraySlice<int64> input_index) {
             auto curr_val = arg_literal.Get<ReturnT>(input_index);
 
@@ -1554,19 +1566,17 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
             std::unique_ptr<Literal> computed_result =
                 embedded_evaluator.Evaluate<const Literal*>(*function, args)
                     .ConsumeValueOrDie();
-            // Clear visit states so that the we can use the evaluate again on
+            // Clear visit states so that we can use the evaluator again on
             // the same computation.
             embedded_evaluator.ResetVisitStates();
-
             // Assign computed result to result_val.
             result_val = computed_result->Get<ReturnT>({});
-
             return true;
           };
-
+          // Computes one element of the result, reducing all dimensions that
+          // contribute to that element.
           ShapeUtil::ForEachIndex(arg_literal.shape(), base, arg_dim_counts,
                                   arg_dim_steps, func);
-
           return result_val;
         }));
 
@@ -1574,6 +1584,20 @@ class HloEvaluator::TypedVisitor : public DfsHloVisitorWithDefault {
     return Status::OK();
   }
 
+  bool IsScalarAdd(HloComputation* computation) {
+    HloInstruction* instruction = computation->root_instruction();
+    if (instruction->opcode() == HloOpcode::kAdd &&
+        computation->num_parameters() == 2) {
+      const HloInstruction* lhs = instruction->operand(0);
+      const HloInstruction* rhs = instruction->operand(1);
+      return lhs->opcode() == HloOpcode::kParameter &&
+             ShapeUtil::IsScalar(lhs->shape()) &&
+             rhs->opcode() == HloOpcode::kParameter &&
+             ShapeUtil::IsScalar(rhs->shape()) && lhs != rhs;
+    }
+    return false;
+  }
+
   Status HandleSelectAndScatter(HloInstruction* select_and_scatter) override {
     auto operand = select_and_scatter->operand(0);
     auto source = select_and_scatter->operand(1);
diff --git a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
index 685cacd7f7..dd14dd3853 100644
--- a/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
+++ b/tensorflow/compiler/xla/service/hlo_evaluator_test.cc
@@ -40,6 +40,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/platform/test.h"
+#include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/platform/types.h"
 
 namespace xla {
@@ -1205,6 +1206,80 @@ TEST_P(HloEvaluatorTest,
   LiteralTestUtil::ExpectEqual(*expected, *result);
 }
 
+class HloEvaluatorPreciseReduceTest : public HloVerifiedTestBase {};
+
+// Tests that Reduce doesn't lose precision when adding many numbers (because
+// it accumulates its result in a double).
+TEST_F(HloEvaluatorPreciseReduceTest, AddReductionPrecisionTest) {
+  HloComputation::Builder b(TestName());
+
+  constexpr int kNumElements = 1 << 25;  // float += 1 saturates at 1<<24
+  std::vector<float> v(kNumElements, 1.0f);
+  HloInstruction* arg_instruction = b.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR1<float>(v)));
+  HloInstruction* init_value = b.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(0.f)));
+
+  HloComputation::Builder add_computation("add");
+  Shape scalar_shape = ShapeUtil::MakeShape(F32, {});
+  auto param_lhs = add_computation.AddInstruction(
+      HloInstruction::CreateParameter(0, scalar_shape, "lhs"));
+  auto param_rhs = add_computation.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
+  add_computation.AddInstruction(HloInstruction::CreateBinary(
+      scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs));
+  auto add_func = module().AddEmbeddedComputation(add_computation.Build());
+
+  HloInstruction* reduce_instruction = b.AddInstruction(
+      HloInstruction::CreateReduce(scalar_shape, arg_instruction, init_value,
+                                   /*dimensions_to_reduce=*/{0}, add_func));
+  module().AddEntryComputation(b.Build());
+
+  HloEvaluator hlo_eval;
+  std::unique_ptr<Literal> result =
+      hlo_eval.Evaluate(reduce_instruction).ConsumeValueOrDie();
+  LiteralTestUtil::ExpectR0Equal<float>(kNumElements, *result);
+}
+
+// Reducing many numbers should be fast because it doesn't create
+// intermediate Literals; the microbenchmark should finish in < 1 msec.
+void BM_ReducePrecisely(int num_iters) {
+  tensorflow::testing::StopTiming();
+  HloComputation::Builder b("BM_ReducePrecisely");
+  HloModuleConfig config;
+  config.set_debug_options(legacy_flags::GetDebugOptionsFromFlags());
+  HloModule module("BM_ReducePrecisely", VersionedComputationHandle(), config);
+
+  constexpr int kNumElements = 1 << 25;  // float += 1 saturates at 1<<24
+  std::vector<float> v(kNumElements, 1.0f);
+  HloInstruction* arg_instruction = b.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR1<float>(v)));
+  auto init_value = b.AddInstruction(
+      HloInstruction::CreateConstant(Literal::CreateR0<float>(0.f)));
+
+  HloComputation::Builder add_computation("add");
+  Shape scalar_shape = ShapeUtil::MakeShape(F32, {});
+  auto param_lhs = add_computation.AddInstruction(
+      HloInstruction::CreateParameter(0, scalar_shape, "lhs"));
+  auto param_rhs = add_computation.AddInstruction(
+      HloInstruction::CreateParameter(1, scalar_shape, "rhs"));
+  add_computation.AddInstruction(HloInstruction::CreateBinary(
+      scalar_shape, HloOpcode::kAdd, param_lhs, param_rhs));
+  auto add_func = module.AddEmbeddedComputation(add_computation.Build());
+
+  HloInstruction* reduce_instruction = b.AddInstruction(
+      HloInstruction::CreateReduce(scalar_shape, arg_instruction, init_value,
+                                   /*dimensions_to_reduce=*/{0}, add_func));
+  module.AddEntryComputation(b.Build());
+
+  HloEvaluator hlo_eval;
+  tensorflow::testing::StartTiming();
+  hlo_eval.Evaluate(reduce_instruction).ConsumeValueOrDie();
+  tensorflow::testing::StopTiming();
+}
+
+BENCHMARK(BM_ReducePrecisely);
+
 TEST_P(HloEvaluatorTest, ReduceAdd) {
   HloComputation::Builder b(TestName());
 
-- 
GitLab


From 954cffeb889671104dcbd53298c555a1bdb3639a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 15:18:18 -0700
Subject: [PATCH 778/960] Add attr when rewriter adds an XlaHostCompute Op
 indicating which outside_compilation_subgraph it corresponds to.

PiperOrigin-RevId: 191124345
---
 .../jit/encapsulate_subgraphs_pass.cc         |  1 +
 .../jit/encapsulate_subgraphs_pass_test.cc    | 30 ++++++++++++-------
 2 files changed, 21 insertions(+), 10 deletions(-)

diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
index 53ec6c1e60..b04b333141 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass.cc
@@ -825,6 +825,7 @@ Status Encapsulator::Subgraph::AddHostComputes(
       builder.Attr("key",
                    strings::StrCat("host_compute_channel_", subgraph_name, "_",
                                    oc_subgraph_name));
+      builder.Attr("_outside_compilation_subgraph", oc_subgraph_name);
       Status s = builder.Finalize(&host_compute_def);
       if (!s.ok()) return s;
 
diff --git a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
index 56efe98fdb..8599a7038a 100644
--- a/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
+++ b/tensorflow/compiler/jit/encapsulate_subgraphs_pass_test.cc
@@ -902,7 +902,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionOneOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph",
              "_outside_compilation_shape_inference_F1_O1"},
-            {"shapes", gtl::ArraySlice<DataType>({})}},
+            {"shapes", gtl::ArraySlice<DataType>({})},
+            {"_outside_compilation_subgraph", "O1"}},
            {"c"}},
       },
       {{"f_0_retval", "F:o:0"}});
@@ -1046,7 +1047,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O2"},
             {"shape_inference_graph",
              "_outside_compilation_shape_inference_F1_O2"},
-            {"shapes", gtl::ArraySlice<DataType>({})}},
+            {"shapes", gtl::ArraySlice<DataType>({})},
+            {"_outside_compilation_subgraph", "O2"}},
            {"F"}},
           {{"outside_compilation_O1_host_compute"},
            "XlaHostCompute",
@@ -1056,7 +1058,8 @@ TEST(EncapsulateSubgraphsTest, OneFunctionTwoOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph",
              "_outside_compilation_shape_inference_F1_O1"},
-            {"shapes", gtl::ArraySlice<DataType>({})}},
+            {"shapes", gtl::ArraySlice<DataType>({})},
+            {"_outside_compilation_subgraph", "O1"}},
            {"D"}},
       },
       {{"i_0_retval", "I:o:0"}});
@@ -1193,7 +1196,8 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph",
              "_outside_compilation_shape_inference_F1_O1"},
-            {"shapes", gtl::ArraySlice<DataType>({})}},
+            {"shapes", gtl::ArraySlice<DataType>({})},
+            {"_outside_compilation_subgraph", "O1"}},
            {"D"}},
       },
       {{"d_0_retval", "D:o:0"}, {"f_0_retval", "F:o:0"}});
@@ -1214,7 +1218,8 @@ TEST(EncapsulateSubgraphsTest, TwoFunctionsTwoOutside) {
             {"key", "host_compute_channel_F2_O1"},
             {"shape_inference_graph", ""},
             {"shapes",
-             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})}}},
+             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})},
+            {"_outside_compilation_subgraph", "O1"}}},
       },
       {{"g_0_retval", "G:o:0"}, {"i_0_retval", "I:o:0"}});
 
@@ -1321,7 +1326,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoInputs) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", ""},
             {"shapes",
-             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})}}},
+             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})},
+            {"_outside_compilation_subgraph", "O1"}}},
       },
       {{"f_0_retval", "F:o:0"}});
 
@@ -1403,7 +1409,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlInput) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", ""},
             {"shapes",
-             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})}},
+             gtl::ArraySlice<TensorShapeProto>({shape_proto_expected})},
+            {"_outside_compilation_subgraph", "O1"}},
            {"D"}},
       },
       {{"f_0_retval", "F:o:0"}});
@@ -1482,7 +1489,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationNoOutputs) {
             {"Toutputs", gtl::ArraySlice<DataType>({})},
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", ""},
-            {"shapes", gtl::ArraySlice<TensorShapeProto>({})}}},
+            {"shapes", gtl::ArraySlice<TensorShapeProto>({})},
+            {"_outside_compilation_subgraph", "O1"}}},
       },
       {{"f_0_retval", "F:o:0"}});
 
@@ -1561,7 +1569,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationControlOutput) {
             {"Toutputs", gtl::ArraySlice<DataType>({})},
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph", ""},
-            {"shapes", gtl::ArraySlice<TensorShapeProto>({})}}},
+            {"shapes", gtl::ArraySlice<TensorShapeProto>({})},
+            {"_outside_compilation_subgraph", "O1"}}},
       },
       {{"f_0_retval", "F:o:0"}});
 
@@ -1725,7 +1734,8 @@ TEST(EncapsulateSubgraphsTest, OutsideCompilationShapeInference) {
             {"key", "host_compute_channel_F1_O1"},
             {"shape_inference_graph",
              "_outside_compilation_shape_inference_F1_O1"},
-            {"shapes", gtl::ArraySlice<DataType>({})}},
+            {"shapes", gtl::ArraySlice<DataType>({})},
+            {"_outside_compilation_subgraph", "O1"}},
            {"c"}},
       },
       {{"f_0_retval", "F:o:0"}});
-- 
GitLab


From d0e883aceb21c611cadd6712f139164d18989568 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 15:18:48 -0700
Subject: [PATCH 779/960]   Add the waiting time and cross-replica-sum time to
 StepInfoResult.

PiperOrigin-RevId: 191124408
---
 tensorflow/contrib/tpu/profiler/tf_op_stats.proto | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
index 590db2c376..2a15875627 100644
--- a/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
+++ b/tensorflow/contrib/tpu/profiler/tf_op_stats.proto
@@ -79,6 +79,10 @@ message StepInfoResult {
   optional uint64 infeed_duration_ps = 3;
   // The start time of this step in picoseconds.
   optional uint64 begin_ps = 4;
+  // The waiting time within this step in picoseconds.
+  optional uint64 wait_duration_ps = 5;
+  // The time spent on cross-replica-sum in picoseconds.
+  optional uint64 crs_duration_ps = 6;
 }
 
 // Result proto for a sequence of steps.
-- 
GitLab


From 03b07d7549dc0ee6f90c206c64a3d64906669975 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 15:35:14 -0700
Subject: [PATCH 780/960] [XLA] Add a reduce-window test. REL_NOTES:n/a
 PiperOrigin-RevId: 191126542

---
 .../compiler/xla/tests/reduce_window_test.cc  | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index 9c317fe579..d6f580a6f9 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -252,6 +252,31 @@ TEST_P(ReduceWindowTest, AmongMajor2DimsMediumSize) {
                            DefaultErrorSpec());
 }
 
+// Tests the super windowing logic w.r.t handling prime number of windows in a
+// major dimension with reduction.
+TEST_P(ReduceWindowTest, PrimeWindowsInReductionDimension) {
+  Array4D<float> input_array(15, 15, 4, 128);
+  input_array.FillRandom(2.f, 4.f);
+
+  int win_len = 3;
+  int win_stride = 2;
+
+  const auto input_data_handle =
+      CreateConstantFromArray(input_array, &builder_);
+
+  Padding padding = Padding::kSame;
+  // Reduce only along the x and y dimensions, according to the win_len.
+  ReduceWindowAdd(input_data_handle, {win_len, win_len, 1, 1},
+                  {win_stride, win_stride, 1, 1}, padding);
+
+  auto result = ReferenceUtil::ReduceWindow4DAdd(
+      input_array, 0.0f, {win_len, win_len, 1, 1},
+      {win_stride, win_stride, 1, 1}, padding);
+
+  ComputeAndCompareLiteral(&builder_, *Literal::CreateFromArray(*result), {},
+                           DefaultErrorSpec());
+}
+
 // Tests a reduction function that is not a simple add/min/max/etc.
 XLA_TEST_P(ReduceWindowTest, NonstandardReduceFunction) {
   Array4D<float> input_array(1, 2, 2, 1);
-- 
GitLab


From 9bcecca7304473aeeba191776eeb6c15e96ad335 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 15:41:02 -0700
Subject: [PATCH 781/960] Restore definitions of static members in
 MklCpuAllocator.

These were removed in #17396 which made the static member variables of
MklCpuAllocator into inline variables, which are a C++17 feature, and not
properly restored in #18006 which reverted the inline declarations, leading to
an ODR violation that is apparently ignored with some compilers.

PiperOrigin-RevId: 191127281
---
 tensorflow/core/common_runtime/mkl_cpu_allocator.cc | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 829c19204a..43a909466e 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,6 +19,9 @@ limitations under the License.
 
 namespace tensorflow {
 
+constexpr const char* MklCPUAllocator::kMaxLimitStr;
+constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
+
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
-- 
GitLab


From a90a47f86082878c911d18529728f81bbd50d33c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 15:51:12 -0700
Subject: [PATCH 782/960] Doc string clean-ups for class DistributionStrategy.

PiperOrigin-RevId: 191128500
---
 tensorflow/python/training/distribute.py | 11 ++++++++---
 1 file changed, 8 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/training/distribute.py b/tensorflow/python/training/distribute.py
index b11412cac7..c44627eadb 100644
--- a/tensorflow/python/training/distribute.py
+++ b/tensorflow/python/training/distribute.py
@@ -376,7 +376,9 @@ class DistributionStrategy(object):
     update. Allreduce is an algorithm for performing a reduction on
     values from multiple devices and making the result available on
     all of those devices.
-  * TODO(josh11b): Future: partitioned variables
+  * In the future we will have support for TensorFlows' partitioned
+    variables, where a single variable is split across multiple
+    devices.
 
   We have then a few approaches we want to support:
   * Code written (as if) with no knowledge of class `DistributionStrategy`.
@@ -390,7 +392,6 @@ class DistributionStrategy(object):
     ```
     with my_distribution.scope():
       iterator = my_distribution.distribute_dataset(dataset)
-      # TODO(josh11b): iterator = dataset.make_one_shot_iterator()
       tower_train_ops = my_distribution.call_for_each_tower(
           tower_fn, iterator.get_next())
       train_op = tf.group(my_distribution.unwrap(tower_train_ops))
@@ -402,6 +403,10 @@ class DistributionStrategy(object):
     using `my_distribution`'s policy, and library functions called by
     `tower_fn` can use the `get_tower_context()` API to get enhanced
     behavior in this case.
+
+    Note that in the future we will add support for initializable
+    Dataset iterators, at which point this example code will change.
+
   * If you want to write a distributed algorithm, you may use any of
     the `DistributionStrategy` APIs inside a
     `with my_distribution.scope():` block of code.
@@ -514,7 +519,7 @@ class DistributionStrategy(object):
 
   Steps 3 and 4 are done automatically by class `Optimizer` if you call
   its `apply_gradients` method in a tower context. Otherwise you can
-  manually call its `distributed_apply` method in a cross-tower context.
+  manually call its `_distributed_apply` method in a cross-tower context.
 
   Another thing you might want to do in the middle of your tower function
   is an all-reduce of some intermediate value, using `d.reduce()` or
-- 
GitLab


From 36fef8baaaa461dc7bcb65d9c7c7f8796fc80c21 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 16:16:13 -0700
Subject: [PATCH 783/960] Implement strip CheckNumerics in DebugStripper.

PiperOrigin-RevId: 191131935
---
 tensorflow/core/grappler/op_types.cc          |  4 ++
 tensorflow/core/grappler/op_types.h           |  1 +
 .../grappler/optimizers/debug_stripper.cc     |  5 ++
 .../optimizers/debug_stripper_test.cc         | 59 +++++++++++++++++++
 4 files changed, 69 insertions(+)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index c31ac9b59c..e0ee49d157 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -68,6 +68,10 @@ bool IsBitcast(const NodeDef& node) { return node.op() == "Bitcast"; }
 
 bool IsCast(const NodeDef& node) { return node.op() == "Cast"; }
 
+bool IsCheckNumerics(const NodeDef& node) {
+  return node.op() == "CheckNumerics";
+}
+
 bool IsComplex(const NodeDef& node) { return node.op() == "Complex"; }
 
 bool IsComplexAbs(const NodeDef& node) { return node.op() == "ComplexAbs"; }
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index 39affcbc24..aa6750d5c3 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -37,6 +37,7 @@ bool IsBiasAdd(const NodeDef& node);
 bool IsBiasAddGrad(const NodeDef& node);
 bool IsBitcast(const NodeDef& node);
 bool IsCast(const NodeDef& node);
+bool IsCheckNumerics(const NodeDef& node);
 bool IsComplex(const NodeDef& node);
 bool IsComplexAbs(const NodeDef& node);
 bool IsConj(const NodeDef& node);
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper.cc b/tensorflow/core/grappler/optimizers/debug_stripper.cc
index 0e058e3435..8bd10171f1 100644
--- a/tensorflow/core/grappler/optimizers/debug_stripper.cc
+++ b/tensorflow/core/grappler/optimizers/debug_stripper.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/grappler/optimizers/debug_stripper.h"
 
+#include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/grappler/clusters/cluster.h"
 #include "tensorflow/core/grappler/grappler_item.h"
@@ -39,6 +40,10 @@ Status DebugStripper::Optimize(Cluster* cluster, const GrapplerItem& item,
           inp = AsControlDependency(inp);
         }
       }
+    } else if (IsCheckNumerics(node)) {
+      // Replace with Identity op which will be pruned later.
+      node.set_op("Identity");
+      node.mutable_attr()->erase("message");
     }
   }
   return Status::OK();
diff --git a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
index c79c36841d..3f11febc64 100644
--- a/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
+++ b/tensorflow/core/grappler/optimizers/debug_stripper_test.cc
@@ -105,6 +105,65 @@ TEST_F(DebugStripperTest, StripAssertFromGraph) {
   test::ExpectTensorEqual<float>(expected[0], optimized[0]);
 }
 
+TEST_F(DebugStripperTest, StripCheckNumericsFromGraph) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  auto check1 = ops::CheckNumerics(s.WithOpName("CheckNumerics1"), x, "foo");
+  auto check2 = ops::CheckNumerics(s.WithOpName("CheckNumerics2"), y, "foo");
+  Output add = ops::Add(s.WithOpName("z"), check1, check2);
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  DebugStripper optimizer;
+  GraphDef output;
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int count = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "x") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "y") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "CheckNumerics1") {
+      count++;
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ(1, node.attr_size());
+    } else if (node.name() == "CheckNumerics2") {
+      count++;
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("y", node.input(0));
+      EXPECT_EQ(1, node.attr_size());
+    } else if (node.name() == "z") {
+      count++;
+      EXPECT_EQ("Add", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("CheckNumerics1", node.input(0));
+      EXPECT_EQ("CheckNumerics2", node.input(1));
+    }
+  }
+  EXPECT_EQ(5, count);
+
+  Tensor x_t(DT_FLOAT, TensorShape({}));
+  Tensor y_t(DT_FLOAT, TensorShape({}));
+  x_t.flat<float>()(0) = 1.0f;
+  y_t.flat<float>()(0) = 0.5f;
+  std::vector<Tensor> expected =
+      EvaluateNodes(item.graph, {"z"}, {{"x", x_t}, {"y", y_t}});
+  std::vector<Tensor> optimized =
+      EvaluateNodes(output, {"z"}, {{"x", x_t}, {"y", y_t}});
+  test::ExpectTensorEqual<float>(expected[0], optimized[0]);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From 5f6f3198dcba3fcf38aab4a44093fc8c02c49bbc Mon Sep 17 00:00:00 2001
From: Neal Wu <wun@google.com>
Date: Fri, 30 Mar 2018 16:31:31 -0700
Subject: [PATCH 784/960] Simple fixes for documentation on
 tf.nn.softmax_cross_entropy_with_logits (and v2).

PiperOrigin-RevId: 191134015
---
 tensorflow/python/ops/nn_ops.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/ops/nn_ops.py b/tensorflow/python/ops/nn_ops.py
index 0c55386241..07ca32953f 100644
--- a/tensorflow/python/ops/nn_ops.py
+++ b/tensorflow/python/ops/nn_ops.py
@@ -1808,7 +1808,7 @@ def softmax_cross_entropy_with_logits_v2(
   or `float64`).
 
   Backpropagation will happen into both `logits` and `labels`.  To disallow
-  backpropagation into `labels`, pass label tensors through a `stop_gradients`
+  backpropagation into `labels`, pass label tensors through @{tf.stop_gradient}
   before feeding it to this function.
 
   **Note that to avoid confusion, it is required to pass only named arguments to
@@ -1895,7 +1895,7 @@ _XENT_DEPRECATION = """
 Future major versions of TensorFlow will allow gradients to flow
 into the labels input on backprop by default.
 
-See tf.nn.softmax_cross_entropy_with_logits_v2.
+See @{tf.nn.softmax_cross_entropy_with_logits_v2}.
 """
 
 
-- 
GitLab


From e75f554199f13d13775cdde8394fdf8d0683b6af Mon Sep 17 00:00:00 2001
From: Saurabh Saxena <srbs@google.com>
Date: Fri, 30 Mar 2018 17:22:41 -0700
Subject: [PATCH 785/960] Support int64 slice spec for StridedSliceGrad.
 Currently this doesn't work since _StridedSliceGrad always passes a int32
 shape which restricts indices values to be of the same type. Add a test that
 fails before and passes after the change.

PiperOrigin-RevId: 191140718
---
 tensorflow/python/kernel_tests/array_ops_test.py | 8 ++++++++
 tensorflow/python/ops/array_grad.py              | 7 ++++++-
 2 files changed, 14 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 64c1760d5e..78bdb7eda7 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -780,6 +780,14 @@ class StridedSliceGradTest(test_util.TensorFlowTestCase):
       grad = GradSliceChecker(self, sess, var, np.array(8))
       _ = grad[tuple()]
 
+  def testInt64Indices(self):
+    with self.test_session(use_gpu=True) as sess:
+      a = math_ops.range(3)
+      index = constant_op.constant(1, dtype=dtypes.int64)
+      b = 2 * a[index]
+      grad, = gradients_impl.gradients(b, a)
+      self.assertAllEqual(sess.run(grad), [0, 2, 0])
+
 
 class StridedSliceGradTypeTest(test_util.TensorFlowTestCase):
   """Test varied index types and host located memory."""
diff --git a/tensorflow/python/ops/array_grad.py b/tensorflow/python/ops/array_grad.py
index 3c6a5c9e56..57d2657838 100644
--- a/tensorflow/python/ops/array_grad.py
+++ b/tensorflow/python/ops/array_grad.py
@@ -255,10 +255,15 @@ def _SliceGrad(op, grad):
 @ops.RegisterGradient("StridedSlice")
 def _StridedSliceGrad(op, grad):
   """Gradient for StridedSlice op."""
-  x = array_ops.shape(op.inputs[0])
   begin = op.inputs[1]
   end = op.inputs[2]
   strides = op.inputs[3]
+  # StridedSliceGrad requires `x`, `begin`, `end` and `strides` to be of the
+  # same dtype so we build a shape of the same type as other args.
+  # Note that the choice of `begin` for specifying `out_type` is arbitrary.
+  # We could choose any of {begin|end|strides}.dtype since they are required to
+  # be the same.
+  x = array_ops.shape(op.inputs[0], out_type=begin.dtype)
 
   return array_ops.strided_slice_grad(
       x,
-- 
GitLab


From 7690f8e3a3fdc9dd3da64a41093aac0acbbf1fdc Mon Sep 17 00:00:00 2001
From: Russell Power <power@google.com>
Date: Fri, 30 Mar 2018 20:36:03 -0700
Subject: [PATCH 786/960] Log large allocations and total memory usage for the
 CPU device.

PiperOrigin-RevId: 191152060
---
 tensorflow/core/framework/allocator.cc     | 38 +++++++++++++++++++++-
 tensorflow/core/grappler/clusters/utils.cc |  2 +-
 tensorflow/core/platform/mem.h             |  2 +-
 tensorflow/core/platform/posix/port.cc     |  2 +-
 tensorflow/core/platform/windows/port.cc   |  2 +-
 5 files changed, 41 insertions(+), 5 deletions(-)

diff --git a/tensorflow/core/framework/allocator.cc b/tensorflow/core/framework/allocator.cc
index a382b8be95..6182f95f28 100644
--- a/tensorflow/core/framework/allocator.cc
+++ b/tensorflow/core/framework/allocator.cc
@@ -61,6 +61,26 @@ static bool cpu_allocator_collect_stats = false;
 // If true, cpu allocator collects full stats.
 static bool cpu_allocator_collect_full_stats = false;
 
+// Individual allocations large than this amount will trigger a warning.
+static const double kLargeAllocationWarningThreshold = 0.1;
+
+// If cpu_allocator_collect_stats is true, warn when the total allocated memory
+// exceeds this threshold.
+static const double kTotalAllocationWarningThreshold = 0.5;
+
+// Cache first invocation to port::AvailableRam, as it can be expensive.
+static int64_t LargeAllocationWarningBytes() {
+  static int64_t value = static_cast<int64>(port::AvailableRam() *
+                                            kLargeAllocationWarningThreshold);
+  return value;
+}
+
+static int64_t TotalAllocationWarningBytes() {
+  static int64_t value = static_cast<int64>(port::AvailableRam() *
+                                            kTotalAllocationWarningThreshold);
+  return value;
+}
+
 void EnableCPUAllocatorStats(bool enable) {
   cpu_allocator_collect_stats = enable;
 }
@@ -70,7 +90,8 @@ void EnableCPUAllocatorFullStats(bool enable) {
 
 class CPUAllocator : public VisitableAllocator {
  public:
-  CPUAllocator() : allocation_begun_(false) {}
+  CPUAllocator()
+      : total_allocation_warning_triggered_(false), allocation_begun_(false) {}
 
   ~CPUAllocator() override {}
 
@@ -81,6 +102,12 @@ class CPUAllocator : public VisitableAllocator {
       allocation_begun_ = true;
     }
 
+    if (num_bytes > LargeAllocationWarningBytes()) {
+      LOG(WARNING) << "Allocation of " << num_bytes << " exceeds "
+                   << 100 * kLargeAllocationWarningThreshold
+                   << "% of system memory.";
+    }
+
     void* p = port::AlignedMalloc(num_bytes, alignment);
     if (cpu_allocator_collect_stats) {
       const std::size_t alloc_size = port::MallocExtension_GetAllocatedSize(p);
@@ -91,6 +118,14 @@ class CPUAllocator : public VisitableAllocator {
           std::max<int64>(stats_.max_bytes_in_use, stats_.bytes_in_use);
       stats_.max_alloc_size =
           std::max<int64>(stats_.max_alloc_size, alloc_size);
+
+      if (stats_.bytes_in_use > TotalAllocationWarningBytes() &&
+          !total_allocation_warning_triggered_) {
+        LOG(WARNING) << "Total allocated memory " << stats_.bytes_in_use
+                     << "exceeds " << 100 * kTotalAllocationWarningThreshold
+                     << "% of system memory";
+        total_allocation_warning_triggered_ = true;
+      }
     }
 
     // visit each Visitor in alloc_visitors_
@@ -162,6 +197,7 @@ class CPUAllocator : public VisitableAllocator {
  private:
   mutex mu_;
   AllocatorStats stats_ GUARDED_BY(mu_);
+  bool total_allocation_warning_triggered_ GUARDED_BY(mu_);
 
   // visitor_mutex_ protects write access to alloc_visitors_ and free_visitors_.
   // While write access is mutually exclusive, reads may happen concurrently.
diff --git a/tensorflow/core/grappler/clusters/utils.cc b/tensorflow/core/grappler/clusters/utils.cc
index b54b34959a..50d6e6468f 100644
--- a/tensorflow/core/grappler/clusters/utils.cc
+++ b/tensorflow/core/grappler/clusters/utils.cc
@@ -54,7 +54,7 @@ DeviceProperties GetLocalCPUInfo() {
 
   int64 free_mem = port::AvailableRam();
   if (free_mem < INT64_MAX) {
-    device.set_memory_size(free_mem * 1024);
+    device.set_memory_size(free_mem);
   }
 
   (*device.mutable_environment())["cpu_instruction_set"] =
diff --git a/tensorflow/core/platform/mem.h b/tensorflow/core/platform/mem.h
index 7bb9fc264f..fca3a2332d 100644
--- a/tensorflow/core/platform/mem.h
+++ b/tensorflow/core/platform/mem.h
@@ -59,7 +59,7 @@ void MallocExtension_ReleaseToSystem(std::size_t num_bytes);
 // routine, this routine returns 0.
 std::size_t MallocExtension_GetAllocatedSize(const void* p);
 
-// Returns the amount of RAM available in kB, or INT64_MAX if unknown.
+// Returns the amount of RAM available in bytes, or INT64_MAX if unknown.
 int64 AvailableRam();
 
 }  // namespace port
diff --git a/tensorflow/core/platform/posix/port.cc b/tensorflow/core/platform/posix/port.cc
index 494acde803..8e316472fe 100644
--- a/tensorflow/core/platform/posix/port.cc
+++ b/tensorflow/core/platform/posix/port.cc
@@ -177,7 +177,7 @@ int64 AvailableRam() {
   struct sysinfo info;
   int err = sysinfo(&info);
   if (err == 0) {
-    return info.freeram / 1024;
+    return info.freeram;
   }
 #endif
   return INT64_MAX;
diff --git a/tensorflow/core/platform/windows/port.cc b/tensorflow/core/platform/windows/port.cc
index f3b27ea394..174f41a993 100644
--- a/tensorflow/core/platform/windows/port.cc
+++ b/tensorflow/core/platform/windows/port.cc
@@ -166,7 +166,7 @@ int64 AvailableRam() {
   MEMORYSTATUSEX statex;
   statex.dwLength = sizeof(statex);
   if (GlobalMemoryStatusEx(&statex)) {
-    return statex.ullAvailPhys / 1024;
+    return statex.ullAvailPhys;
   }
   return INT64_MAX;
 }
-- 
GitLab


From 10878ede0688fbf6d720bef1d132a1537f9ea72c Mon Sep 17 00:00:00 2001
From: brett koonce <koonce@hello.com>
Date: Fri, 30 Mar 2018 21:51:21 -0700
Subject: [PATCH 787/960] Distribute sp (#18138)

* contrib/distribute: minor spelling tweaks

* contrib/distributions: minor spelling tweaks
---
 tensorflow/contrib/distribute/python/cross_tower_ops.py   | 4 ++--
 tensorflow/contrib/distribute/python/cross_tower_utils.py | 2 +-
 .../contrib/distribute/python/shared_variable_creator.py  | 2 +-
 .../kernel_tests/bijectors/kumaraswamy_bijector_test.py   | 2 +-
 tensorflow/contrib/distributions/python/ops/estimator.py  | 2 +-
 .../contrib/distributions/python/ops/independent.py       | 2 +-
 .../distributions/python/ops/onehot_categorical.py        | 4 ++--
 .../contrib/distributions/python/ops/relaxed_bernoulli.py | 8 ++++----
 .../python/ops/relaxed_onehot_categorical.py              | 2 +-
 .../contrib/distributions/python/ops/vector_student_t.py  | 2 +-
 10 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/cross_tower_ops.py b/tensorflow/contrib/distribute/python/cross_tower_ops.py
index 68f202ea62..bbe5e877d5 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_ops.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_ops.py
@@ -12,7 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 # ==============================================================================
-"""Classes for different algortihms of reduction and broadcasting."""
+"""Classes for different algorithms of reduction and broadcasting."""
 
 from __future__ import absolute_import
 from __future__ import division
@@ -155,7 +155,7 @@ class CrossTowerOps(object):
     Args:
       method_string: either 'sum' or 'mean' specifying the reduction method.
       value_destination_pairs: a list or a tuple of tuples of PerDevice objects
-        and destinations. If a destionation is None, then the destinations
+        and destinations. If a destination is None, then the destinations
         are set to match the devices of the input PerDevice object.
 
     Returns:
diff --git a/tensorflow/contrib/distribute/python/cross_tower_utils.py b/tensorflow/contrib/distribute/python/cross_tower_utils.py
index 0dc6b8db6b..fc04e2195f 100644
--- a/tensorflow/contrib/distribute/python/cross_tower_utils.py
+++ b/tensorflow/contrib/distribute/python/cross_tower_utils.py
@@ -316,7 +316,7 @@ def unpack_small_tensors(tower_grads, packing):
       it made to tower_grads.
 
   Returns:
-    new_tower_grads: identical to tower_grads except that concatentations
+    new_tower_grads: identical to tower_grads except that concatenations
       of small tensors have been split apart and returned to their original
       positions, paired with their original variables.
   """
diff --git a/tensorflow/contrib/distribute/python/shared_variable_creator.py b/tensorflow/contrib/distribute/python/shared_variable_creator.py
index aca9c7af05..a7083e279f 100644
--- a/tensorflow/contrib/distribute/python/shared_variable_creator.py
+++ b/tensorflow/contrib/distribute/python/shared_variable_creator.py
@@ -46,7 +46,7 @@ def make_fn(shared_variable_store, device_id):
   error.
   Additionally, we de-uniquify variable names before checking for matches. This
   helps re-use variables which are intended to be the same but have different
-  names due to variable uniquificaton happening upstream. Since this might
+  names due to variable uniquification happening upstream. Since this might
   mean we may have multiple variables with the same canonical name, we store
   them in a list per canonical name and return them in the same order as well.
 
diff --git a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py
index ad11d9f248..074b5f275d 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/bijectors/kumaraswamy_bijector_test.py
@@ -69,7 +69,7 @@ class KumaraswamyBijectorTest(test.TestCase):
       bijector = Kumaraswamy(
           concentration1=concentration1,
           concentration0=concentration0, validate_args=True)
-      # Omitting the endpoints 0 and 1, since idlj will be inifinity at these
+      # Omitting the endpoints 0 and 1, since idlj will be infinity at these
       # endpoints.
       y = np.linspace(.01, 0.99, num=10).astype(np.float32)
       x = 1 - (1 - y ** concentration1) ** concentration0
diff --git a/tensorflow/contrib/distributions/python/ops/estimator.py b/tensorflow/contrib/distributions/python/ops/estimator.py
index 6b53338c45..98edd337fe 100644
--- a/tensorflow/contrib/distributions/python/ops/estimator.py
+++ b/tensorflow/contrib/distributions/python/ops/estimator.py
@@ -75,7 +75,7 @@ def estimator_head_distribution_regression(make_distribution_fn,
 
 
 class _DistributionRegressionHead(_RegressionHead):
-  """Creates a _RegressionHead instance from an arbitray `Distribution`."""
+  """Creates a _RegressionHead instance from an arbitrary `Distribution`."""
 
   def __init__(self,
                make_distribution_fn,
diff --git a/tensorflow/contrib/distributions/python/ops/independent.py b/tensorflow/contrib/distributions/python/ops/independent.py
index 7dcb3e3ac4..b1bacb91b0 100644
--- a/tensorflow/contrib/distributions/python/ops/independent.py
+++ b/tensorflow/contrib/distributions/python/ops/independent.py
@@ -36,7 +36,7 @@ class Independent(distribution_lib.Distribution):
 
   This distribution is useful for regarding a collection of independent,
   non-identical distributions as a single random variable. For example, the
-  `Indpendent` distribution composed of a collection of `Bernoulli`
+  `Independent` distribution composed of a collection of `Bernoulli`
   distributions might define a distribution over an image (where each
   `Bernoulli` is a distribution over each pixel).
 
diff --git a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
index 46c2cc8b7a..e3e40b2e9c 100644
--- a/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/onehot_categorical.py
@@ -52,7 +52,7 @@ class OneHotCategorical(distribution.Distribution):
 
   #### Examples
 
-  Creates a 3-class distiribution, with the 2nd class, the most likely to be
+  Creates a 3-class distribution, with the 2nd class, the most likely to be
   drawn from.
 
   ```python
@@ -60,7 +60,7 @@ class OneHotCategorical(distribution.Distribution):
   dist = OneHotCategorical(probs=p)
   ```
 
-  Creates a 3-class distiribution, with the 2nd class the most likely to be
+  Creates a 3-class distribution, with the 2nd class the most likely to be
   drawn from, using logits.
 
   ```python
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
index b525809015..e454a53c62 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_bernoulli.py
@@ -35,10 +35,10 @@ class RelaxedBernoulli(transformed_distribution.TransformedDistribution):
 
   The RelaxedBernoulli is a distribution over the unit interval (0,1), which
   continuously approximates a Bernoulli. The degree of approximation is
-  controlled by a temperature: as the temperaturegoes to 0 the RelaxedBernoulli
-  becomes discrete with a distribution described by the `logits` or `probs`
-  parameters, as the temperature goes to infinity the RelaxedBernoulli
-  becomes the constant distribution that is identically 0.5.
+  controlled by a temperature: as the temperature goes to 0 the
+  RelaxedBernoulli becomes discrete with a distribution described by the
+  `logits` or `probs` parameters, as the temperature goes to infinity the
+  RelaxedBernoulli becomes the constant distribution that is identically 0.5.
 
   The RelaxedBernoulli distribution is a reparameterized continuous
   distribution that is the binary special case of the RelaxedOneHotCategorical
diff --git a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
index ff33f327c7..f56ba07816 100644
--- a/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
+++ b/tensorflow/contrib/distributions/python/ops/relaxed_onehot_categorical.py
@@ -303,7 +303,7 @@ class RelaxedOneHotCategorical(
   The RelaxedOneHotCategorical is a distribution over random probability
   vectors, vectors of positive real values that sum to one, which continuously
   approximates a OneHotCategorical. The degree of approximation is controlled by
-  a temperature: as the temperaturegoes to 0 the RelaxedOneHotCategorical
+  a temperature: as the temperature goes to 0 the RelaxedOneHotCategorical
   becomes discrete with a distribution described by the `logits` or `probs`
   parameters, as the temperature goes to infinity the RelaxedOneHotCategorical
   becomes the constant distribution that is identically the constant vector of
diff --git a/tensorflow/contrib/distributions/python/ops/vector_student_t.py b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
index 8c67647a61..887981d64e 100644
--- a/tensorflow/contrib/distributions/python/ops/vector_student_t.py
+++ b/tensorflow/contrib/distributions/python/ops/vector_student_t.py
@@ -66,7 +66,7 @@ class _VectorStudentT(transformed_distribution.TransformedDistribution):
   This distribution is an Affine transformation of iid
   [Student's t-distributions](
   https://en.wikipedia.org/wiki/Student%27s_t-distribution)
-  and should not be confused with the [Multivate Student's t-distribution](
+  and should not be confused with the [Multivariate Student's t-distribution](
   https://en.wikipedia.org/wiki/Multivariate_t-distribution). The
   traditional Multivariate Student's t-distribution is type of
   [elliptical distribution](
-- 
GitLab


From d2a4eee2012b3f5b0a221d00384971dc7b5e425b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Fri, 30 Mar 2018 22:55:58 -0700
Subject: [PATCH 788/960] Add a unit test to verify that dependency optimizer
 could remove noop and greaterequal node in a simple graph.

PiperOrigin-RevId: 191157450
---
 .../optimizers/dependency_optimizer_test.cc   | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
index 57b3118245..6a297da52d 100644
--- a/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/dependency_optimizer_test.cc
@@ -678,6 +678,50 @@ TEST_F(DependencyOptimizerTest, Identity_DeviceCrossing_ConsumerOnSameDevice) {
   }
 }
 
+TEST_F(DependencyOptimizerTest, RemoveGreaterEqualWithNoOp) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+  Output x = ops::Placeholder(s.WithOpName("x"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  Output y = ops::Placeholder(s.WithOpName("y"), DT_FLOAT,
+                              ops::Placeholder::Shape({}));
+  auto greaterequal = ops::GreaterEqual(s.WithOpName("GreaterEqual"), x, y);
+  auto noop =
+      ops::NoOp(s.WithOpName("NoOp").WithControlDependencies(greaterequal));
+  Output add = ops::Add(
+      s.WithOpName("z").WithControlDependencies({noop.operation}), x, y);
+  GrapplerItem item;
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  DependencyOptimizer optimizer;
+  GraphDef output;
+  item.fetch.push_back("z");
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int count = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "x") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "y") {
+      count++;
+      EXPECT_EQ("Placeholder", node.op());
+      EXPECT_EQ(0, node.input_size());
+    } else if (node.name() == "GreaterEqual") {
+      count++;
+    } else if (node.name() == "NoOp") {
+      count++;
+    } else if (node.name() == "z") {
+      count++;
+      EXPECT_EQ("Add", node.op());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("x", node.input(0));
+      EXPECT_EQ("y", node.input(1));
+    }
+  }
+  EXPECT_EQ(3, count);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
-- 
GitLab


From d2c9c95566ad1945c7ac24da41953effc6f3cf68 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <blamb@google.com>
Date: Fri, 30 Mar 2018 23:37:48 -0700
Subject: [PATCH 789/960] Fix typo

PiperOrigin-RevId: 191159820
---
 tensorflow/docs_src/mobile/tflite/devguide.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/mobile/tflite/devguide.md b/tensorflow/docs_src/mobile/tflite/devguide.md
index 5b521dca7b..96392a3c9b 100644
--- a/tensorflow/docs_src/mobile/tflite/devguide.md
+++ b/tensorflow/docs_src/mobile/tflite/devguide.md
@@ -88,7 +88,7 @@ Tensorflow Lite format. This process uses several model formats:
   extracted from a `SavedModel`.
 * *TensorFlow Lite model* (.tflite) —A serialized
   [FlatBuffer](https://google.github.io/flatbuffers/) that contains TensorFlow
-  Lite operators and tensors for the TensorFlow Lite interpreter, similiar to a
+  Lite operators and tensors for the TensorFlow Lite interpreter, similar to a
   `FrozenGraphDef`.
 
 ### Freeze Graph
-- 
GitLab


From f16b50e388543d2c05698926aa1ef265090dda6b Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Sun, 1 Apr 2018 06:57:00 +0800
Subject: [PATCH 790/960] Support a single Tensor in StagingArea.put() (#17863)

* Support a single Tensor in a StagingArea.put()

* Fix the input values parameter could also be dictionary type
---
 tensorflow/python/ops/data_flow_ops.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/ops/data_flow_ops.py b/tensorflow/python/ops/data_flow_ops.py
index d2cc87555f..cb725199a8 100644
--- a/tensorflow/python/ops/data_flow_ops.py
+++ b/tensorflow/python/ops/data_flow_ops.py
@@ -1769,7 +1769,9 @@ class StagingArea(BaseStagingArea):
     its capacity.
 
     Args:
-      values: Tensor (or a tuple of Tensors) to place into the staging area.
+      values: A single tensor, a list or tuple of tensors, or a dictionary with
+        tensor values. The number of elements must match the length of the
+        list provided to the dtypes argument when creating the StagingArea.
       name: A name for the operation (optional).
 
     Returns:
@@ -1780,11 +1782,12 @@ class StagingArea(BaseStagingArea):
     """
     with ops.name_scope(name, "%s_put" % self._name,
                         self._scope_vals(values)) as scope:
+      
+      if not isinstance(values, (list, tuple, dict)):
+        values = [values]
 
       # Hard-code indices for this staging area
-      indices = (
-          list(six.moves.range(len(values)))
-          if isinstance(values, (list, tuple)) else None)
+      indices = list(six.moves.range(len(values)))
       vals, _ = self._check_put_dtypes(values, indices)
 
       with ops.colocate_with(self._coloc_op):
-- 
GitLab


From aae991b259f1acd91dc6fa21ea74bba4c4710530 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 1 Apr 2018 01:15:35 -0700
Subject: [PATCH 791/960] [XLA] Add a ReduceWindow test to reduce along the
 lane dimension.

PiperOrigin-RevId: 191214828
---
 .../compiler/xla/tests/reduce_window_test.cc    | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/reduce_window_test.cc b/tensorflow/compiler/xla/tests/reduce_window_test.cc
index d6f580a6f9..8dd24f1237 100644
--- a/tensorflow/compiler/xla/tests/reduce_window_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_window_test.cc
@@ -277,6 +277,23 @@ TEST_P(ReduceWindowTest, PrimeWindowsInReductionDimension) {
                            DefaultErrorSpec());
 }
 
+TEST_P(ReduceWindowTest, ReduceAlongLaneDimension) {
+  Array4D<float> input_array(19, 17, 8, 256);
+  input_array.FillWithMinorDimNum();
+
+  const auto input_data_handle =
+      CreateConstantFromArray(input_array, &builder_);
+
+  Padding padding = Padding::kSame;
+  ReduceWindowAdd(input_data_handle, {1, 1, 1, 11}, {1, 1, 1, 1}, padding);
+
+  auto result = ReferenceUtil::ReduceWindow4DAdd(
+      input_array, 0.0f, {1, 1, 1, 11}, {1, 1, 1, 1}, padding);
+
+  ComputeAndCompareLiteral(&builder_, *Literal::CreateFromArray(*result), {},
+                           DefaultErrorSpec());
+}
+
 // Tests a reduction function that is not a simple add/min/max/etc.
 XLA_TEST_P(ReduceWindowTest, NonstandardReduceFunction) {
   Array4D<float> input_array(1, 2, 2, 1);
-- 
GitLab


From 926bd44844d36bbefcbd620eb65ba0019e0a6dde Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Sun, 1 Apr 2018 16:01:20 -0700
Subject: [PATCH 792/960] [XLA] Redesign: implement and test concat.

PiperOrigin-RevId: 191244047
---
 .../xla/client/xla_client/xla_builder.cc      |  38 +++--
 tensorflow/compiler/xla/tests/BUILD           |   4 +-
 tensorflow/compiler/xla/tests/concat_test.cc  | 145 +++++++++---------
 3 files changed, 103 insertions(+), 84 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index e51a8b14c0..7a9ff0c441 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -52,6 +52,16 @@ bool CanBeRoot(HloOpcode opcode) {
   }
 }
 
+StatusOr<std::vector<Shape>> GetOperandShapes(
+    tensorflow::gtl::ArraySlice<XlaOp> operands) {
+  std::vector<Shape> operand_shapes;
+  for (const XlaOp& operand : operands) {
+    TF_ASSIGN_OR_RETURN(const Shape& shape, operand.GetShape());
+    operand_shapes.push_back(shape);
+  }
+  return operand_shapes;
+}
+
 }  // namespace
 
 StatusOr<Shape> XlaBuilder::GetShape(const XlaOp& op) const {
@@ -362,11 +372,7 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
   return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     std::vector<const Shape*> operand_shape_ptrs;
-    std::vector<Shape> operand_shapes;
-    for (const auto& operand : operands) {
-      TF_ASSIGN_OR_RETURN(const Shape& shape, operand.GetShape());
-      operand_shapes.push_back(shape);
-    }
+    TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(operands));
     c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                 [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape,
@@ -457,7 +463,21 @@ XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
 
 XlaOp XlaBuilder::ConcatInDim(tensorflow::gtl::ArraySlice<XlaOp> operands,
                               int64 dimension) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    std::vector<const Shape*> operand_shape_ptrs;
+    TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(operands));
+    c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
+                [](const Shape& shape) { return &shape; });
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferConcatOpShape(operand_shape_ptrs, dimension));
+
+    instr.add_dimensions(dimension);
+
+    return AddInstruction(std::move(instr), HloOpcode::kConcatenate, operands);
+  }());
 }
 
 XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value,
@@ -508,11 +528,7 @@ XlaOp XlaBuilder::Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements) {
   return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
     HloInstructionProto instr;
     std::vector<const Shape*> operand_shape_ptrs;
-    std::vector<Shape> operand_shapes;
-    for (const XlaOp& e : elements) {
-      TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(e));
-      operand_shapes.push_back(shape);
-    }
+    TF_ASSIGN_OR_RETURN(const auto& operand_shapes, GetOperandShapes(elements));
     c_transform(operand_shapes, std::back_inserter(operand_shape_ptrs),
                 [](const Shape& shape) { return &shape; });
     TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index e337669aeb..283efbb707 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1444,9 +1444,9 @@ xla_test(
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:test",
         "//tensorflow/compiler/xla:test_helpers",
-        "//tensorflow/compiler/xla/client:computation",
-        "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/concat_test.cc b/tensorflow/compiler/xla/tests/concat_test.cc
index fb0e9c724a..a4c8a83eb1 100644
--- a/tensorflow/compiler/xla/tests/concat_test.cc
+++ b/tensorflow/compiler/xla/tests/concat_test.cc
@@ -18,9 +18,9 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/array3d.h"
-#include "tensorflow/compiler/xla/client/computation.h"
-#include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/reference_util.h"
 #include "tensorflow/compiler/xla/statusor.h"
 #include "tensorflow/compiler/xla/test.h"
@@ -38,9 +38,9 @@ using ::testing::HasSubstr;
 
 // Concatenate expects at least one argument.
 XLA_TEST_F(ConcatTest, Concat_Nothing) {
-  ComputationBuilder builder(client_, TestName());
-  auto concatenated = builder.ConcatInDim({}, 0);
-  StatusOr<Computation> computation_status = builder.Build();
+  XlaBuilder builder(TestName());
+  builder.ConcatInDim({}, 0);
+  StatusOr<XlaComputation> computation_status = builder.Build();
   ASSERT_FALSE(computation_status.ok());
   EXPECT_THAT(computation_status.status().ToString(),
               HasSubstr("Concatenate expects at least one argument"));
@@ -48,18 +48,18 @@ XLA_TEST_F(ConcatTest, Concat_Nothing) {
 
 // Concatenate with one argument works.
 XLA_TEST_F(ConcatTest, Concat_R1_With_Nothing) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0, 64.0});
-  auto concatenated = builder.ConcatInDim({a}, 0);
+  builder.ConcatInDim({a}, 0);
 
   std::vector<float> expected = {42, 64};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_L0_With_Nothing) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
-  auto concatenated = builder.ConcatInDim({a}, 0);
+  builder.ConcatInDim({a}, 0);
 
   std::vector<float> expected = {};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -68,51 +68,51 @@ XLA_TEST_F(ConcatTest, Concat_R1_L0_With_Nothing) {
 // Show that we can't concatenate R0 with R0 because we can't name the dimension
 // to concatenate on.
 XLA_TEST_F(ConcatTest, CannotConcatR0WithR0) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR0<float>(42.0);
   auto b = builder.ConstantR0<float>(64.0);
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
-  StatusOr<Computation> computation_status = builder.Build();
+  builder.ConcatInDim({a, b}, 0);
+  StatusOr<XlaComputation> computation_status = builder.Build();
   ASSERT_FALSE(computation_status.ok());
   EXPECT_THAT(computation_status.status().ToString(),
               HasSubstr("out of bounds: 0"));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_L0_With_R1_L0) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({});
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   std::vector<float> expected = {};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_L0_With_R1_L1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({});
   auto b = builder.ConstantR1<float>({256.0});
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   std::vector<float> expected = {256};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_L2_With_R1_L0) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0, 64.0});
   auto b = builder.ConstantR1<float>({});
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   std::vector<float> expected = {42, 64};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_L2_With_R1_L1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0, 64.0});
   auto b = builder.ConstantR1<float>({256.0});
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   std::vector<float> expected = {42, 64, 256};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -129,20 +129,20 @@ XLA_TEST_F(ConcatTest, Concat_R1_L253_With_R1_L7) {
     expected[253 + i] = rhs[i] = 253 + i + 1;
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>(lhs);
   auto b = builder.ConstantR1<float>(rhs);
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_0x0_With_0x0) {
   for (int dim : {0, 1}) {
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto a = builder.ConstantR2FromArray2D(Array2D<float>(0, 0));
     auto b = builder.ConstantR2FromArray2D(Array2D<float>(0, 0));
-    auto concatenated = builder.ConcatInDim({a, b}, dim);
+    builder.ConcatInDim({a, b}, dim);
 
     ComputeAndCompareR2<float>(&builder, Array2D<float>(0, 0), {},
                                ErrorSpec(0.0001));
@@ -150,26 +150,27 @@ XLA_TEST_F(ConcatTest, Concat_0x0_With_0x0) {
 }
 
 XLA_TEST_F(ConcatTest, Concat_1x1_With_1x1_InDim0) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a_array = CreatePatternedMatrix(1, 1);
   auto b_array = CreatePatternedMatrix(1, 1, /*offset=*/64.0);
   auto a = builder.ConstantR2FromArray2D(*a_array);
   auto b = builder.ConstantR2FromArray2D(*b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   Array2D<float> expected({
-      {0}, {64},
+      {0},
+      {64},
   });
   ComputeAndCompareR2<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_1x1_With_1x1_InDim1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a_array = CreatePatternedMatrix(1, 1);
   auto b_array = CreatePatternedMatrix(1, 1, /*offset=*/64.0);
   auto a = builder.ConstantR2FromArray2D(*a_array);
   auto b = builder.ConstantR2FromArray2D(*b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 1);
+  builder.ConcatInDim({a, b}, 1);
 
   Array2D<float> expected({
       {0, 64},
@@ -178,22 +179,22 @@ XLA_TEST_F(ConcatTest, Concat_1x1_With_1x1_InDim1) {
 }
 
 XLA_TEST_F(ConcatTest, Concat2x0With2x5) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto b_array = CreatePatternedMatrix(2, 5, /*offset=*/64.0);
   auto a = builder.ConstantR2FromArray2D(Array2D<float>(2, 0));
   auto b = builder.ConstantR2FromArray2D(*b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 1);
+  builder.ConcatInDim({a, b}, 1);
 
   ComputeAndCompareR2<float>(&builder, *b_array, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat2x3With2x5) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a_array = CreatePatternedMatrix(2, 3);
   auto b_array = CreatePatternedMatrix(2, 5, /*offset=*/64.0);
   auto a = builder.ConstantR2FromArray2D(*a_array);
   auto b = builder.ConstantR2FromArray2D(*b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 1);
+  builder.ConcatInDim({a, b}, 1);
 
   Array2D<float> expected({
       {0, 1, 2, 64, 65, 66, 67, 68},
@@ -203,22 +204,22 @@ XLA_TEST_F(ConcatTest, Concat2x3With2x5) {
 }
 
 XLA_TEST_F(ConcatTest, Concat3x2With0x2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a_array = CreatePatternedMatrix(3, 2);
   auto a = builder.ConstantR2FromArray2D(*a_array);
   auto b = builder.ConstantR2FromArray2D(Array2D<float>(0, 2));
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   ComputeAndCompareR2<float>(&builder, *a_array, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat3x2With5x2) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a_array = CreatePatternedMatrix(3, 2);
   auto b_array = CreatePatternedMatrix(5, 2, /*offset=*/64.0);
   auto a = builder.ConstantR2FromArray2D(*a_array);
   auto b = builder.ConstantR2FromArray2D(*b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 0);
+  builder.ConcatInDim({a, b}, 0);
 
   Array2D<float> expected({
       {0, 1},
@@ -234,16 +235,16 @@ XLA_TEST_F(ConcatTest, Concat3x2With5x2) {
 }
 
 XLA_TEST_F(ConcatTest, Concat_R3_3x0x2_3x0x1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR3FromArray3D(Array3D<float>(3, 0, 2));
   auto b = builder.ConstantR3FromArray3D(Array3D<float>(3, 0, 1));
-  auto concatenated = builder.ConcatInDim({a, b}, 2);
+  builder.ConcatInDim({a, b}, 2);
   ComputeAndCompareR3<float>(&builder, Array3D<float>(3, 0, 3), {},
                              ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R3_3x1x2_3x1x1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array3D<float> a_array({
       // 3x1x2
       {{0, 1}},
@@ -258,27 +259,29 @@ XLA_TEST_F(ConcatTest, Concat_R3_3x1x2_3x1x1) {
   });
   auto a = builder.ConstantR3FromArray3D(a_array);
   auto b = builder.ConstantR3FromArray3D(b_array);
-  auto concatenated = builder.ConcatInDim({a, b}, 2);
+  builder.ConcatInDim({a, b}, 2);
 
   Array3D<float> expected({
-      {{0, 1, 6}}, {{2, 3, 7}}, {{4, 5, 8}},
+      {{0, 1, 6}},
+      {{2, 3, 7}},
+      {{4, 5, 8}},
   });
   ComputeAndCompareR3<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R1_1x1_1x1_1x1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0});
   auto b = builder.ConstantR1<float>({64.0});
   auto c = builder.ConstantR1<float>({256.0});
-  auto concatenated = builder.ConcatInDim({a, b, c}, 0);
+  builder.ConcatInDim({a, b, c}, 0);
 
   std::vector<float> expected = {42, 64, 256};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, Concat_R3_3x1x2_3x1x1_3x1x1) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   Array3D<float> a_array({
       // 3x1x2
       {{0, 1}},
@@ -300,35 +303,35 @@ XLA_TEST_F(ConcatTest, Concat_R3_3x1x2_3x1x1_3x1x1) {
   auto a = builder.ConstantR3FromArray3D(a_array);
   auto b = builder.ConstantR3FromArray3D(b_array);
   auto c = builder.ConstantR3FromArray3D(c_array);
-  auto concatenated = builder.ConcatInDim({a, b, c}, 2);
+  builder.ConcatInDim({a, b, c}, 2);
 
   Array3D<float> expected({
-      {{0, 1, 2, 3}}, {{4, 5, 6, 7}}, {{8, 9, 10, 11}},
+      {{0, 1, 2, 3}},
+      {{4, 5, 6, 7}},
+      {{8, 9, 10, 11}},
   });
   ComputeAndCompareR3<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, DoubleConcatLeftAssociative) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0});
   auto b = builder.ConstantR1<float>({64.0});
   auto c = builder.ConstantR1<float>({256.0});
   // concatenated = (a concat b) concat c
-  auto concatenated =
-      builder.ConcatInDim({builder.ConcatInDim({a, b}, 0), c}, 0);
+  builder.ConcatInDim({builder.ConcatInDim({a, b}, 0), c}, 0);
 
   std::vector<float> expected = {42, 64, 256};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
 
 XLA_TEST_F(ConcatTest, DoubleConcatRightAssociative) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR1<float>({42.0});
   auto b = builder.ConstantR1<float>({64.0});
   auto c = builder.ConstantR1<float>({256.0});
   // concatenated = a concat (b concat c)
-  auto concatenated =
-      builder.ConcatInDim({a, builder.ConcatInDim({b, c}, 0)}, 0);
+  builder.ConcatInDim({a, builder.ConcatInDim({b, c}, 0)}, 0);
 
   std::vector<float> expected = {42, 64, 256};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -342,7 +345,7 @@ XLA_TEST_F(ConcatTest, Concat_1x1024_With_1x1024_InDim0) {
     rhs(0, i) = i + 1024;
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2FromArray2D<float>(lhs);
   auto b = builder.ConstantR2FromArray2D<float>(rhs);
   builder.ConcatInDim({a, b}, 0);
@@ -363,7 +366,7 @@ XLA_TEST_F(ConcatTest, Concat_1x1024_With_1x1024_InDim1) {
     rhs(0, i) = i + 1024;
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2FromArray2D<float>(lhs);
   auto b = builder.ConstantR2FromArray2D<float>(rhs);
   builder.ConcatInDim({a, b}, 1);
@@ -388,7 +391,7 @@ XLA_TEST_F(ConcatTest, Concat_64x64_With_64x2) {
     }
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2FromArray2D<float>(lhs);
   auto b = builder.ConstantR2FromArray2D<float>(rhs);
   builder.ConcatInDim({a, b}, 1);
@@ -404,13 +407,13 @@ XLA_TEST_F(ConcatTest, Concat_64x64_With_64x2) {
 
 // Show that we can't concatenate with an opaques.
 XLA_TEST_F(ConcatTest, CannotConcatOpaques) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto opaque_shape = ShapeUtil::MakeOpaqueShape();
   auto r1f32 = xla::ShapeUtil::MakeShape(xla::F32, {1});
   auto x = builder.Parameter(0, r1f32, "x");
   auto y = builder.Parameter(1, opaque_shape, "y");
-  auto concatenated = builder.ConcatInDim({x, y}, 0);
-  StatusOr<Computation> computation_status = builder.Build();
+  builder.ConcatInDim({x, y}, 0);
+  StatusOr<XlaComputation> computation_status = builder.Build();
   ASSERT_FALSE(computation_status.ok());
   EXPECT_THAT(
       computation_status.status().ToString(),
@@ -418,23 +421,23 @@ XLA_TEST_F(ConcatTest, CannotConcatOpaques) {
 }
 
 XLA_TEST_F(ConcatTest, ConcatSeveralBoxedPredicates) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto p0 = builder.ConstantR1<bool>({true});
   auto p1 = builder.ConstantR1<bool>({false});
   auto p2 = builder.ConstantR1<bool>({true});
-  auto concatenated = builder.ConcatInDim({p0, p1, p2}, 0);
+  builder.ConcatInDim({p0, p1, p2}, 0);
 
   bool expected[] = {true, false, true};
   ComputeAndCompareR1<bool>(&builder, expected, {});
 }
 
 XLA_TEST_F(ConcatTest, ConcatSeveralR1S32s) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a0 = builder.ConstantR1<int32>({1});
   auto a1 = builder.ConstantR1<int32>({2, 3});
   auto a2 = builder.ConstantR1<int32>({4, 5, 6});
   auto a3 = builder.ConstantR1<int32>({7, 8, 9, 10});
-  auto concatenated = builder.ConcatInDim({a0, a1, a2, a3}, 0);
+  builder.ConcatInDim({a0, a1, a2, a3}, 0);
 
   std::vector<int32> expected(10);
   std::iota(expected.begin(), expected.end(), 1);
@@ -442,7 +445,7 @@ XLA_TEST_F(ConcatTest, ConcatSeveralR1S32s) {
 }
 
 XLA_TEST_F(ConcatTest, ConcatR3WeirdDims) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
 
   Array3D<float> arr0(9, 17, 1);
   arr0.Fill(1);
@@ -462,14 +465,14 @@ XLA_TEST_F(ConcatTest, ConcatR3WeirdDims) {
     }
   }
 
-  ComputationDataHandle h0;
+  XlaOp h0;
   auto p0 = CreateR3Parameter<float>(arr0, /*parameter_number=*/0, "p0",
                                      &builder, &h0);
-  ComputationDataHandle h1;
+  XlaOp h1;
   auto p1 = CreateR3Parameter<float>(arr1, /*parameter_number=*/1, "p1",
                                      &builder, &h1);
 
-  auto concatenated = builder.ConcatInDim({h0, h1}, 2);
+  builder.ConcatInDim({h0, h1}, 2);
 
   ComputeAndCompareR3<float>(&builder, expected, {p0.get(), p1.get()});
 }
@@ -495,7 +498,7 @@ TEST_P(ConcatR2BinaryTest, DoIt) {
   Array2D<int32> rhs(spec.rhs_dim0, spec.rhs_dim1);
   rhs.FillUnique(1000);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a0 = builder.ConstantR2FromArray2D<int32>(lhs);
   auto a1 = builder.ConstantR2FromArray2D<int32>(rhs);
   builder.ConcatInDim({a0, a1}, spec.concat_dimension);
@@ -521,7 +524,7 @@ XLA_TEST_F(ConcatTest, ConcatOperandsOfSameOperand) {
   auto x_data = client_->TransferToServer(*x_literal).ConsumeValueOrDie();
   auto y_data = client_->TransferToServer(*y_literal).ConsumeValueOrDie();
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.Parameter(0, f32_scalar, "x");
   auto y = builder.Parameter(1, f32_scalar, "y");
   auto mul = builder.Mul(x, y);
@@ -545,7 +548,7 @@ XLA_TEST_F(ConcatTest, ConcatBroadcastArgument) {
   auto y_data = client_->TransferToServer(*y_literal).ConsumeValueOrDie();
   auto z_data = client_->TransferToServer(*z_literal).ConsumeValueOrDie();
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.Parameter(0, x_literal->shape(), "x");
   auto y = builder.Parameter(1, f32_scalar, "y");
   auto z = builder.Parameter(2, f32_scalar, "z");
@@ -573,7 +576,7 @@ XLA_TEST_F(ConcatTest, ConcatBroadcastArgumentR3) {
   auto y_data = client_->TransferToServer(*y_literal).ConsumeValueOrDie();
   auto z_data = client_->TransferToServer(*z_literal).ConsumeValueOrDie();
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto x = builder.Parameter(0, x_literal->shape(), "x");
   auto y = builder.Parameter(1, f32_scalar, "y");
   auto z = builder.Parameter(2, f32_scalar, "y");
-- 
GitLab


From 1fbad5034a8ea531e496b0ecbf9e2c3839b62311 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Sun, 1 Apr 2018 22:46:11 -0700
Subject: [PATCH 793/960] Make batch_sequences_with_states_test.py work with
 the C API enabled.

The C API improves static shape inference, making more errors caught
at graph construction time instead of runtime.

PiperOrigin-RevId: 191260634
---
 .../training/batch_sequences_with_states_test.py    | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py b/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
index dbdbb08a82..16c260edb0 100644
--- a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
+++ b/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
@@ -526,10 +526,15 @@ class PaddingTest(test.TestCase):
           "key_2": constant_op.constant([1.5, 2.5])  # length 2
       }
 
-      _, padded_seq = sqss._padding(sequences, 2)
-      with self.assertRaisesOpError(
-          ".*All sequence lengths must match, but received lengths.*"):
-        padded_seq["key_1"].eval()
+      if ops._USE_C_API:
+        with self.assertRaisesRegexp(
+            ValueError, "Fill dimensions must be >= 0"):
+          _, padded_seq = sqss._padding(sequences, 2)
+      else:
+        _, padded_seq = sqss._padding(sequences, 2)
+        with self.assertRaisesOpError(
+            ".*All sequence lengths must match, but received lengths.*"):
+          padded_seq["key_1"].eval()
 
   def testPadding(self):
     with ops.Graph().as_default() as g, self.test_session(graph=g):
-- 
GitLab


From 3be80b5d31ecadd4f4a883b1ebe159d005408c1d Mon Sep 17 00:00:00 2001
From: Chikanaga Tomoyuki <nagachika@ruby-lang.org>
Date: Mon, 2 Apr 2018 17:59:05 +0900
Subject: [PATCH 794/960] Remove warnings for copy_op_to_graph.

---
 tensorflow/contrib/copy_graph/python/util/copy_elements.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/copy_graph/python/util/copy_elements.py b/tensorflow/contrib/copy_graph/python/util/copy_elements.py
index b806799202..102bc460fd 100644
--- a/tensorflow/contrib/copy_graph/python/util/copy_elements.py
+++ b/tensorflow/contrib/copy_graph/python/util/copy_elements.py
@@ -201,7 +201,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, scope=''):
     #An instance of tensorflow.core.framework.node_def_pb2.NodeDef, it
     #stores String-based info such as name, device and type of the op.
     #Unique to every Operation instance.
-    new_node_def = deepcopy(op._node_def)
+    new_node_def = deepcopy(op.node_def)
     #Change the name
     new_node_def.name = new_name
 
@@ -211,7 +211,7 @@ def copy_op_to_graph(org_instance, to_graph, variables, scope=''):
 
     #Make a copy of the op_def too.
     #Its unique to every _type_ of Operation.
-    op_def = deepcopy(op._op_def)
+    op_def = deepcopy(op.op_def)
 
     #Initialize a new Operation instance
     new_op = ops.Operation(new_node_def, to_graph, new_inputs, output_types,
-- 
GitLab


From be571938196fb191f260a2c45176d406e6c19a13 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 08:00:03 -0700
Subject: [PATCH 795/960] Adding support for RandomUniform. Basic support for
 op import/export of RandomUniform, and constant resolution with static seeds.

PiperOrigin-RevId: 191293897
---
 tensorflow/contrib/lite/toco/BUILD            |   1 +
 .../contrib/lite/toco/export_tensorflow.cc    |  21 +++
 .../graph_transformations.h                   |   1 +
 .../propagate_array_data_types.cc             | 164 +++++++++++-------
 .../propagate_fixed_sizes.cc                  |  10 +-
 .../resolve_constant_random_uniform.cc        | 116 +++++++++++++
 .../contrib/lite/toco/import_tensorflow.cc    |  23 ++-
 tensorflow/contrib/lite/toco/model.h          |   8 +
 tensorflow/contrib/lite/toco/toco_tooling.cc  |   1 +
 tensorflow/contrib/lite/toco/tooling_util.cc  |   1 +
 10 files changed, 274 insertions(+), 72 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_random_uniform.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index d552de313c..2dd689ad4c 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -259,6 +259,7 @@ cc_library(
         "graph_transformations/resolve_constant_fake_quant.cc",
         "graph_transformations/resolve_constant_fill.cc",
         "graph_transformations/resolve_constant_gather.cc",
+        "graph_transformations/resolve_constant_random_uniform.cc",
         "graph_transformations/resolve_constant_range.cc",
         "graph_transformations/resolve_constant_shape_or_rank.cc",
         "graph_transformations/resolve_constant_stack.cc",
diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index 22a23357b3..e88357f7dd 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -1711,6 +1711,23 @@ void ConvertTopKV2Operator(const Model& model, const TopKV2Operator& src_op,
   (*topk_op->mutable_attr())["sorted"].set_b(true);
 }
 
+void ConvertRandomUniformOperator(const Model& model,
+                                  const RandomUniformOperator& src_op,
+                                  GraphDef* tensorflow_graph) {
+  CHECK(tensorflow_graph != nullptr);
+  auto* new_op = tensorflow_graph->add_node();
+  new_op->set_op("RandomUniform");
+  CHECK_EQ(src_op.inputs.size(), 1);
+  new_op->set_name(src_op.outputs[0]);
+  *new_op->add_input() = src_op.inputs[0];
+  const auto shape_type = GetTensorFlowDataType(model, src_op.inputs[0]);
+  (*new_op->mutable_attr())["T"].set_type(shape_type);
+  (*new_op->mutable_attr())["dtype"].set_type(
+      GetTensorFlowDataType(src_op.dtype));
+  (*new_op->mutable_attr())["seed"].set_i(src_op.seed);
+  (*new_op->mutable_attr())["seed2"].set_i(src_op.seed2);
+}
+
 void ConvertOperator(const Model& model, const Operator& src_op,
                      GraphDef* tensorflow_graph) {
   if (src_op.fused_activation_function != FusedActivationFunctionType::kNone) {
@@ -1897,6 +1914,10 @@ void ConvertOperator(const Model& model, const Operator& src_op,
     ConvertTransposeConvOperator(
         model, static_cast<const TransposeConvOperator&>(src_op),
         tensorflow_graph);
+  } else if (src_op.type == OperatorType::kRandomUniform) {
+    ConvertRandomUniformOperator(
+        model, static_cast<const RandomUniformOperator&>(src_op),
+        tensorflow_graph);
   } else {
     LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(src_op.type);
   }
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 640afc7c74..76ec02aa07 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -173,6 +173,7 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveStridedSliceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveSliceAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveMeanAttributes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveTransposeAttributes)
+DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRandomUniform)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantRange)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantShapeOrRank)
 DECLARE_GRAPH_TRANSFORMATION(ResolveConstantStack)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
index 778da39bf1..89ad58f887 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_array_data_types.cc
@@ -50,78 +50,108 @@ bool PropagateArrayDataTypes::Run(Model* model, std::size_t op_index) {
     old_output_data_types[output] = model->GetArray(output).data_type;
   }
   // Do the actual output data types propagation.
-  if (op->type == OperatorType::kDequantize ||
-      op->type == OperatorType::kResizeBilinear) {
-    // These operators unconditionally produce float outputs
-    SetDataTypeForAllOutputs(model, op, ArrayDataType::kFloat);
-  } else if (op->type == OperatorType::kTensorFlowLess ||
-             op->type == OperatorType::kTensorFlowLessEqual ||
-             op->type == OperatorType::kTensorFlowGreater ||
-             op->type == OperatorType::kTensorFlowGreaterEqual) {
-    // These operators unconditionally produce bool outputs
-    SetDataTypeForAllOutputs(model, op, ArrayDataType::kBool);
-  } else if (op->type == OperatorType::kRank ||
-             op->type == OperatorType::kTensorFlowShape) {
-    // These operators only produce int32 outputs.
-    SetDataTypeForAllOutputs(model, op, ArrayDataType::kInt32);
-  } else if (op->type == OperatorType::kTensorFlowSplit ||
-             op->type == OperatorType::kTensorFlowConcat ||
-             op->type == OperatorType::kFill) {
-    // These operators produce an output with the same type as their 2nd input
-    CHECK_GE(op->inputs.size(), 2);
-    const ArrayDataType data_type = model->GetArray(op->inputs[1]).data_type;
-    SetDataTypeForAllOutputs(model, op, data_type);
-  } else if (op->type == OperatorType::kTransposeConv) {
-    // These operators produce an output with the same type as their 3rd input
-    CHECK_GE(op->inputs.size(), 3);
-    const ArrayDataType data_type = model->GetArray(op->inputs[2]).data_type;
-    SetDataTypeForAllOutputs(model, op, data_type);
-  } else if (op->type == OperatorType::kCast) {
-    // Data type of the Cast op is specified.
-    CHECK_EQ(op->outputs.size(), 1);
-    auto* cast_op = static_cast<CastOperator*>(op);
-    model->GetArray(op->outputs[0]).data_type = cast_op->dst_data_type;
-  } else if (op->type == OperatorType::kArgMax) {
-    // Data type of the ArgMax op is specified.
-    CHECK_EQ(op->outputs.size(), 1);
-    auto* argmax_op = static_cast<ArgMaxOperator*>(op);
-    model->GetArray(op->outputs[0]).data_type = argmax_op->output_data_type;
-  } else if (op->type == OperatorType::kRange) {
-    auto* range_op = static_cast<RangeOperator*>(op);
-    // Output type of the Range op can be set via an attribute
-    ArrayDataType data_type;
-    if (range_op->dtype != ArrayDataType::kNone) {
-      // Use the type if specified
-      data_type = range_op->dtype;
-    } else {
-      // Otherwise use the first input
-      CHECK_GE(op->inputs.size(), 1);
-      data_type = model->GetArray(op->inputs[0]).data_type;
+  switch (op->type) {
+    case OperatorType::kDequantize:
+    case OperatorType::kResizeBilinear:
+      // These operators unconditionally produce float outputs
+      SetDataTypeForAllOutputs(model, op, ArrayDataType::kFloat);
+      break;
+    case OperatorType::kTensorFlowLess:
+    case OperatorType::kTensorFlowLessEqual:
+    case OperatorType::kTensorFlowGreater:
+    case OperatorType::kTensorFlowGreaterEqual:
+      // These operators unconditionally produce bool outputs
+      SetDataTypeForAllOutputs(model, op, ArrayDataType::kBool);
+      break;
+    case OperatorType::kRank:
+    case OperatorType::kTensorFlowShape:
+      // These operators only produce int32 outputs.
+      SetDataTypeForAllOutputs(model, op, ArrayDataType::kInt32);
+      break;
+    case OperatorType::kTensorFlowSplit:
+    case OperatorType::kTensorFlowConcat:
+    case OperatorType::kFill: {
+      // These operators produce an output with the same type as their 2nd input
+      CHECK_GE(op->inputs.size(), 2);
+      const ArrayDataType data_type = model->GetArray(op->inputs[1]).data_type;
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
     }
-    CHECK_EQ(op->outputs.size(), 1);
-    SetDataTypeForAllOutputs(model, op, data_type);
-  } else if (op->type == OperatorType::kTensorFlowUnsupported) {
-    auto* unsupported_op = static_cast<TensorFlowUnsupportedOperator*>(op);
-    // Some output tensors from the op could be eliminated by optimization.
-    // This can make unsupported_op->output_data_types have more elements than
-    // op->outputs.
-    if (unsupported_op->output_data_types.size() < op->outputs.size()) {
+    case OperatorType::kTransposeConv: {
+      // These operators produce an output with the same type as their 3rd input
+      CHECK_GE(op->inputs.size(), 3);
+      const ArrayDataType data_type = model->GetArray(op->inputs[2]).data_type;
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
+    case OperatorType::kCast: {
+      // Data type of the Cast op is specified.
+      CHECK_EQ(op->outputs.size(), 1);
+      auto* cast_op = static_cast<CastOperator*>(op);
+      model->GetArray(op->outputs[0]).data_type = cast_op->dst_data_type;
+      break;
+    }
+    case OperatorType::kArgMax: {
+      // Data type of the ArgMax op is specified.
+      CHECK_EQ(op->outputs.size(), 1);
+      auto* argmax_op = static_cast<ArgMaxOperator*>(op);
+      model->GetArray(op->outputs[0]).data_type = argmax_op->output_data_type;
+      break;
+    }
+    case OperatorType::kRange: {
+      auto* range_op = static_cast<RangeOperator*>(op);
+      // Output type of the Range op can be set via an attribute
+      ArrayDataType data_type;
+      if (range_op->dtype != ArrayDataType::kNone) {
+        // Use the type if specified
+        data_type = range_op->dtype;
+      } else {
+        // Otherwise use the first input
+        CHECK_GE(op->inputs.size(), 1);
+        data_type = model->GetArray(op->inputs[0]).data_type;
+      }
+      CHECK_EQ(op->outputs.size(), 1);
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
+    }
+    case OperatorType::kRandomUniform: {
+      auto* rand_op = static_cast<RandomUniformOperator*>(op);
+      // The output type of RandomUniform is specified with an attribute
+      if (rand_op->dtype == ArrayDataType::kNone) {
+        return false;
+      }
+      CHECK_EQ(op->outputs.size(), 1);
+      SetDataTypeForAllOutputs(model, op, rand_op->dtype);
+      break;
+    }
+    case OperatorType::kTensorFlowUnsupported: {
+      auto* unsupported_op = static_cast<TensorFlowUnsupportedOperator*>(op);
+      // Some output tensors from the op could be eliminated by optimization.
+      // This can make unsupported_op->output_data_types have more elements than
+      // op->outputs.
+      if (unsupported_op->output_data_types.size() < op->outputs.size()) {
+        return false;
+      }
+      for (int i = 0; i < op->outputs.size(); ++i) {
+        auto output = op->outputs[i];
+        auto data_type = unsupported_op->output_data_types[i];
+        model->GetArray(output).data_type = data_type;
+      }
+      break;
+    }
+    case OperatorType::kExpandDims: {
+      // Yield on ExpandDim until it is converted to Reshape
       return false;
     }
-    for (int i = 0; i < op->outputs.size(); ++i) {
-      auto output = op->outputs[i];
-      auto data_type = unsupported_op->output_data_types[i];
-      model->GetArray(output).data_type = data_type;
+    default: {
+      // These operators produce outputs with the same type as their 1st input
+      CHECK_GT(op->inputs.size(), 0);
+      const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
+      SetDataTypeForAllOutputs(model, op, data_type);
+      break;
     }
-  } else if (op->type == OperatorType::kExpandDims) {
-    // Yield on ExpandDim until it is converted to Reshape
-    return false;
-  } else {
-    // These operators produce outputs with the same type as their 1st input
-    CHECK_GT(op->inputs.size(), 0);
-    const ArrayDataType data_type = model->GetArray(op->inputs[0]).data_type;
-    SetDataTypeForAllOutputs(model, op, data_type);
   }
+
   // Return true if any output data type changed, false if none changed.
   for (const auto& output : op->outputs) {
     if (old_output_data_types[output] != model->GetArray(output).data_type) {
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index 676736cfc5..b96d698675 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -392,8 +392,7 @@ void ProcessSpaceToDepthOperator(Model* model, SpaceToDepthOperator* op) {
                          depth * block_size * block_size}));
 }
 
-void ProcessFillOperator(Model* model, FillOperator* op) {
-  CHECK_EQ(op->inputs.size(), 2);
+void ProcessOpWithShapeInput(Model* model, Operator* op) {
   CHECK_EQ(op->outputs.size(), 1);
   auto& output_array = model->GetArray(op->outputs[0]);
   if (output_array.has_shape()) {
@@ -1529,7 +1528,8 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
                                   static_cast<SpaceToDepthOperator*>(op));
       break;
     case OperatorType::kFill:
-      ProcessFillOperator(model, static_cast<FillOperator*>(op));
+      CHECK_EQ(op->inputs.size(), 2);
+      ProcessOpWithShapeInput(model, op);
       break;
     case OperatorType::kFullyConnected:
       ProcessFullyConnectedOperator(model,
@@ -1659,6 +1659,10 @@ bool PropagateFixedSizes::Run(Model* model, std::size_t op_index) {
       // transforms that remove them, so we avoid propagating shapes through
       // them and let things settle once they've been removed.
       break;
+    case OperatorType::kRandomUniform:
+      CHECK_EQ(op->inputs.size(), 1);
+      ProcessOpWithShapeInput(model, op);
+      break;
     default:
       // Unimplemented, another graph transformation should drop it.
       LOG(FATAL) << "Unhandled operator type " << OperatorTypeName(op->type);
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_random_uniform.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_random_uniform.cc
new file mode 100644
index 0000000000..88d06d7dc7
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_constant_random_uniform.cc
@@ -0,0 +1,116 @@
+/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <algorithm>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+#include "tensorflow/core/lib/random/philox_random.h"
+#include "tensorflow/core/lib/random/random_distributions.h"
+
+namespace toco {
+
+template <ArrayDataType Type>
+bool ComputeRandomUniformArray(Model* model, RandomUniformOperator* op) {
+  typedef tensorflow::random::UniformDistribution<
+      tensorflow::random::PhiloxRandom, DataType<Type>>
+      Distribution;
+
+  // Allocate output
+  auto& output_array = model->GetArray(op->outputs[0]);
+  CHECK(output_array.data_type == Type);
+  std::vector<DataType<Type>>& data =
+      output_array.GetMutableBuffer<Type>().data;
+  data.resize(RequiredBufferSizeForShape(output_array.shape()));
+
+  // We use the same random number generator and distribution as TensorFlow to
+  // produce the exact same values given the same seeds. See
+  // tensorflow::functor::FillPhiloxRandomTask<Distribution, false> in
+  // //third_party/tensorflow/core/kernels/random_op.cc for the implementation.
+  tensorflow::random::PhiloxRandom generator(op->seed, op->seed2);
+  Distribution dist;
+
+  // The generator creates Distribution::kResultElementCount samples at a time.
+  size_t offset = 0;
+  size_t num_samples = Distribution::kResultElementCount;
+  while (offset < data.size()) {
+    const typename Distribution::ResultType samples = dist(&generator);
+    std::copy(&samples[0],
+              &samples[0] + std::min(num_samples, data.size() - offset),
+              &data[0] + offset);
+    offset += num_samples;
+  }
+
+  return true;
+}
+
+bool ResolveConstantRandomUniform::Run(Model* model, std::size_t op_index) {
+  const auto it = model->operators.begin() + op_index;
+  auto* base_op = it->get();
+  if (base_op->type != OperatorType::kRandomUniform) {
+    return false;
+  }
+  auto* op = static_cast<RandomUniformOperator*>(base_op);
+
+  CHECK_EQ(op->inputs.size(), 1);
+  CHECK_EQ(op->outputs.size(), 1);
+
+  auto& output_array = model->GetArray(op->outputs[0]);
+  if (output_array.data_type == ArrayDataType::kNone) {
+    // Yield until the output type has been set by PropagateArrayDataTypes
+    return false;
+  }
+
+  if (!output_array.has_shape()) {
+    // Yield until the output shape has been set by PropagateFixedShapes
+    return false;
+  }
+
+  if ((op->seed == 0) && (op->seed2 == 0)) {
+    LOG(WARNING) << "RandomUniform op outputting \"" << op->outputs[0]
+                 << "\" is truly random (using /dev/random system entropy). "
+                    "Therefore, cannot resolve as constant. Set \"seed\" or "
+                    "\"seed2\" attr non-zero to fix this";
+    return false;
+  }
+
+  switch (output_array.data_type) {
+    case ArrayDataType::kFloat:
+      if (!ComputeRandomUniformArray<ArrayDataType::kFloat>(model, op)) {
+        return false;
+      }
+      break;
+    // For future support of double or half.
+    // case ArrayDataType::kDouble...
+    default:
+      LOG(FATAL)
+          << "Unsupported data type given to RandomUniform op with output \""
+          << op->outputs[0] << "\"";
+      break;
+  }
+
+  // Erase input arrays if no longer used
+  toco::DeleteArrayIfUsedOnce(op->inputs[0], model);
+
+  // Erase the operator
+  model->operators.erase(it);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/import_tensorflow.cc b/tensorflow/contrib/lite/toco/import_tensorflow.cc
index c26e4bddff..876479079b 100644
--- a/tensorflow/contrib/lite/toco/import_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/import_tensorflow.cc
@@ -74,7 +74,7 @@ const string& GetStringAttr(const NodeDef& node, const string& attr_name) {
   return attr.s();
 }
 
-int GetIntAttr(const NodeDef& node, const string& attr_name) {
+int64 GetIntAttr(const NodeDef& node, const string& attr_name) {
   CHECK(HasAttr(node, attr_name)) << attr_name << " not found in:\n"
                                   << node.DebugString();
   const auto& attr = node.attr().at(attr_name);
@@ -569,6 +569,23 @@ void ConvertBiasAddOperator(const NodeDef& node,
   model->operators.emplace_back(biasadd);
 }
 
+void ConvertRandomUniform(const NodeDef& node,
+                          const TensorFlowImportFlags& tf_import_flags,
+                          Model* model) {
+  CHECK_EQ(node.op(), "RandomUniform");
+  CheckInputsCount(node, tf_import_flags, 1);
+
+  CHECK_EQ(GetDataTypeAttr(node, "T"), DT_INT32);
+  auto op = absl::make_unique<RandomUniformOperator>();
+  op->inputs.push_back(node.input(0));
+  op->outputs.push_back(node.name());
+  op->dtype = ConvertDataType(GetDataTypeAttr(node, "dtype"));
+  op->seed = GetIntAttr(node, "seed");
+  op->seed2 = GetIntAttr(node, "seed2");
+  CHECK(model != nullptr);
+  model->operators.emplace_back(std::move(op));
+}
+
 void ConvertReluOperator(const NodeDef& node,
                          const TensorFlowImportFlags& tf_import_flags,
                          Model* model) {
@@ -1931,7 +1948,7 @@ void ConvertTopKV2Operator(const NodeDef& node,
   // K can be encoded as attr (TopK) convert it to a const.
   if (HasAttr(node, "k")) {
     string k_array = CreateConstArray<ArrayDataType::kInt32>(
-        model, node.name() + "k", {GetIntAttr(node, "k")});
+        model, node.name() + "k", {static_cast<int32>(GetIntAttr(node, "k"))});
     op->inputs.push_back(k_array);
   } else {
     CheckInputsCount(node, tf_import_flags, 2);
@@ -2168,6 +2185,8 @@ std::unique_ptr<Model> ImportTensorFlowGraphDef(
     } else if (node.op() == "DynamicStitch" ||
                node.op() == "ParallelDynamicStitch") {
       ConvertDynamicStitchOperator(node, tf_import_flags, model);
+    } else if (node.op() == "RandomUniform") {
+      ConvertRandomUniform(node, tf_import_flags, model);
     } else {
       ConvertUnsupportedOperator(node, tf_import_flags, model);
     }
diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 5199e292e1..64269d369d 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -60,6 +60,7 @@ enum class OperatorType {
   kMaxPool,
   kFakeQuant,
   kMul,
+  kRandomUniform,
   kRange,
   kRank,
   kRelu,
@@ -946,6 +947,13 @@ struct FloorModOperator : Operator {
   FloorModOperator() : Operator(OperatorType::kFloorMod) {}
 };
 
+struct RandomUniformOperator : Operator {
+  RandomUniformOperator() : Operator(OperatorType::kRandomUniform) {}
+  ArrayDataType dtype = ArrayDataType::kNone;
+  int64 seed;
+  int64 seed2;
+};
+
 // Creates a sequence of numbers that begins at start and extends by increments
 // of delta up to but not including limit.
 //
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 30dd6fab9e..0c52f50e39 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -79,6 +79,7 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveConstantBinaryOperator);
   transformations->Add(new ResolveConstantFill);
   transformations->Add(new ResolveConstantGather);
+  transformations->Add(new ResolveConstantRandomUniform);
   transformations->Add(new ResolveConstantRange);
   transformations->Add(new ResolveConstantStack);
   transformations->Add(new ResolveConstantStridedSlice);
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index f3f50487ff..060c52e9e3 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -297,6 +297,7 @@ const char* OperatorTypeName(OperatorType type) {
     HANDLE_OPERATORTYPENAME_CASE(L2Pool)
     HANDLE_OPERATORTYPENAME_CASE(FakeQuant)
     HANDLE_OPERATORTYPENAME_CASE(Mul)
+    HANDLE_OPERATORTYPENAME_CASE(RandomUniform)
     HANDLE_OPERATORTYPENAME_CASE(Relu)
     HANDLE_OPERATORTYPENAME_CASE(Relu1)
     HANDLE_OPERATORTYPENAME_CASE(Relu6)
-- 
GitLab


From f4672ca59b259436dd1cb60b9e12ba9c523e17f6 Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Mon, 2 Apr 2018 08:29:42 -0700
Subject: [PATCH 796/960] Update tests to enable flipping on bfloat16 types.

Skip individual tests that are currently failing with bfloat16.

PiperOrigin-RevId: 191296618
---
 tensorflow/compiler/tests/binary_ops_test.py  | 43 +++++++++++--------
 .../compiler/tests/spacetobatch_op_test.py    |  7 +++
 .../compiler/tests/variable_ops_test.py       | 12 ++++--
 3 files changed, 40 insertions(+), 22 deletions(-)

diff --git a/tensorflow/compiler/tests/binary_ops_test.py b/tensorflow/compiler/tests/binary_ops_test.py
index ba7b9bacd2..d1d7379c0a 100644
--- a/tensorflow/compiler/tests/binary_ops_test.py
+++ b/tensorflow/compiler/tests/binary_ops_test.py
@@ -190,19 +190,24 @@ class BinaryOpsTest(XLATestCase):
           ],
           equality_test=self.ListsAreClose)
 
-      self._testBinary(
-          gen_nn_ops.sparse_softmax_cross_entropy_with_logits,
-          np.array([[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8],
-                    [0.9, 1.0, 1.1, 1.2]], dtype=dtype),
-          np.array([2, 1, 7], dtype=np.int32),
-          expected=[
-              np.array([1.342536, 1.442536, np.nan], dtype=dtype),
-              np.array([[0.213838, 0.236328, -0.738817, 0.288651],
-                        [0.213838, -0.763672, 0.261183, 0.288651],
-                        [np.nan, np.nan, np.nan, np.nan]],
-                       dtype=dtype),
-          ],
-          equality_test=self.ListsAreClose)
+      # TODO(b/68813416): Fails with bfloat16.
+      if dtype != dtypes.bfloat16.as_numpy_dtype:
+        self._testBinary(
+            gen_nn_ops.sparse_softmax_cross_entropy_with_logits,
+            np.array(
+                [[0.1, 0.2, 0.3, 0.4], [0.5, 0.6, 0.7, 0.8],
+                 [0.9, 1.0, 1.1, 1.2]],
+                dtype=dtype),
+            np.array([2, 1, 7], dtype=np.int32),
+            expected=[
+                np.array([1.342536, 1.442536, np.nan], dtype=dtype),
+                np.array(
+                    [[0.213838, 0.236328, -0.738817, 0.288651], [
+                        0.213838, -0.763672, 0.261183, 0.288651
+                    ], [np.nan, np.nan, np.nan, np.nan]],
+                    dtype=dtype),
+            ],
+            equality_test=self.ListsAreClose)
 
   def testIntOps(self):
     for dtype in self.int_types:
@@ -260,12 +265,6 @@ class BinaryOpsTest(XLATestCase):
           np.array([[1], [2]], dtype=dtype),
           dtype(7),
           expected=np.array([[8], [9]], dtype=dtype))
-      self._testBinary(
-          math_ops.add,
-          np.array([0xffffffff, 0xfffffffff, 1, 1], dtype=np.int64),
-          np.array([1, 1, 0xffffffff, 0xfffffffff], dtype=np.int64),
-          expected=np.array(
-              [1 << 32, 1 << 36, 1 << 32, 1 << 36], dtype=np.int64))
 
       self._testBinary(
           math_ops.subtract,
@@ -361,6 +360,12 @@ class BinaryOpsTest(XLATestCase):
           np.array([2, -1], dtype=dtype),
           expected=np.array([[[[3, 1], [5, 3]]]], dtype=dtype))
 
+    self._testBinary(
+        math_ops.add,
+        np.array([0xffffffff, 0xfffffffff, 1, 1], dtype=np.int64),
+        np.array([1, 1, 0xffffffff, 0xfffffffff], dtype=np.int64),
+        expected=np.array([1 << 32, 1 << 36, 1 << 32, 1 << 36], dtype=np.int64))
+
   def testComplexOps(self):
     for dtype in self.complex_types:
       ctypes = {np.complex64: np.float32}
diff --git a/tensorflow/compiler/tests/spacetobatch_op_test.py b/tensorflow/compiler/tests/spacetobatch_op_test.py
index 92518aadc4..6083981493 100644
--- a/tensorflow/compiler/tests/spacetobatch_op_test.py
+++ b/tensorflow/compiler/tests/spacetobatch_op_test.py
@@ -21,6 +21,7 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.compiler.tests.xla_test import XLATestCase
+from tensorflow.python.framework import dtypes
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import gen_array_ops
 from tensorflow.python.platform import test
@@ -156,6 +157,12 @@ class SpaceToBatchNDTest(XLATestCase):
     paddings = np.array(paddings).reshape((len(block_shape), 2))
     with self.test_session() as sess, self.test_scope():
       for dtype in self.float_types:
+        # TODO(b/68813416): Skip bfloat16's as the input type for direct is
+        # float32 and results in a mismatch, while making testDirect provide the
+        # correctly typed input results in 'no fill-function for data-type'
+        # error.
+        if dtype == dtypes.bfloat16.as_numpy_dtype:
+          continue
         placeholder = array_ops.placeholder(dtype)
         # outputs = space_to_batch(inputs)
         x_tf = array_ops.space_to_batch_nd(placeholder, block_shape, paddings)
diff --git a/tensorflow/compiler/tests/variable_ops_test.py b/tensorflow/compiler/tests/variable_ops_test.py
index b08d6ab21e..8ecad00f6e 100644
--- a/tensorflow/compiler/tests/variable_ops_test.py
+++ b/tensorflow/compiler/tests/variable_ops_test.py
@@ -230,7 +230,10 @@ class SliceAssignTest(XLATestCase):
       # shrink shape changes
       checker[1:2, 1] = [66]
       checker[1, 1:2] = [66]
-      checker[1, 1] = 66
+      if dtype != dtypes.bfloat16.as_numpy_dtype:
+        # TODO(b/68813416): valnp call above results in an ndarray and not a
+        # number for bfloat16s.
+        checker[1, 1] = 66
       # newaxis shape changes
       checker[:, None, :] = [[[10, 20, 30]], [[40, 50, 50]]]
       # shrink and newaxis
@@ -243,8 +246,11 @@ class SliceAssignTest(XLATestCase):
 
       # Assign vector to scalar (rank-0) using newaxis
       checker2 = StridedSliceAssignChecker(self, 222, dtype=dtype)
-      checker2[()] = 6  # no indices
-      checker2[...] = 6  # ellipsis
+      if dtype != dtypes.bfloat16.as_numpy_dtype:
+        # TODO(b/68813416): valnp call above results in an ndarray and not a
+        # number for bfloat16s.
+        checker2[()] = 6  # no indices
+        checker2[...] = 6  # ellipsis
       checker2[None] = [6]  # new axis
 
   def testUninitialized(self):
-- 
GitLab


From 2addad8411efaa45cf4ef1ebefdd6f0590bfbd76 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 2 Apr 2018 08:35:56 -0700
Subject: [PATCH 797/960] Validate the shape of count for RepeatDataset
 (#18150)

* Validate the shape of count for RepeatDataset

The count in RepeatDataset requires a scalar though
the validation is only done in ops, not in shape function.

This fix validates the shape of count for RepeatDataset.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Sanitize with clang-foramt -i

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for shape validation of count with RepeatDataset

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../data/python/kernel_tests/sequence_dataset_op_test.py   | 6 ++++++
 tensorflow/core/ops/dataset_ops.cc                         | 7 +++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
index 36ddf30042..b044ff1775 100644
--- a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
@@ -100,6 +100,12 @@ class SequenceDatasetSerializationTest(
     # Test repeat empty dataset
     self.run_core_tests(lambda: self._build_repeat_dataset(-1, 0), None, 0)
 
+  def testInvalidRepeat(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Shape must be rank 0 but is rank 1'):
+      self.run_core_tests(lambda: self._build_repeat_dataset([1, 2], 0),
+                          None, 0)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index e2453b9712..2852c49e19 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -105,8 +105,11 @@ REGISTER_OP("RepeatDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);  // TODO(mrry): Validate the
-                                                // shape of `count`.
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle count_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &count_shape));
+      return shape_inference::ScalarShape(c);
+    });
 
 REGISTER_OP("TakeDataset")
     .Input("input_dataset: variant")
-- 
GitLab


From c7a04561fb8972fb64907acc5f10f3c6d4cef9f2 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Mon, 2 Apr 2018 08:50:20 -0700
Subject: [PATCH 798/960] Replace deprecated
 tf.contrib.metrics.streaming_accuracy with tf.metrics.accuracy (#18154)

* Replace deprecated tf.contrib.metrics.streaming_accuracy with tf.metrics.accuracy

As tf.contrib.metrics.streaming_accuracy (tf.contrib.metrics.streaming_mean_squared_error) has been deprecated and
has been replaced with tf.metrics.accuracy, this fix
replaces deprecated tf.contrib.metrics.streaming_accuracy with tf.metrics.accuracy
in tensorflow/contrib/training/python/training/evaluation.py.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Replace streaming_accuracy with tf.metrics.accuracy in tests

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix additional WARNING in evaluation_test.py

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Fix pylint issue

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../training/python/training/evaluation.py       | 10 ++++------
 .../training/python/training/evaluation_test.py  | 16 +++++++++-------
 2 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/evaluation.py b/tensorflow/contrib/training/python/training/evaluation.py
index 1a5fb45be0..4bb53e8678 100644
--- a/tensorflow/contrib/training/python/training/evaluation.py
+++ b/tensorflow/contrib/training/python/training/evaluation.py
@@ -36,9 +36,8 @@ out the metrics values to stdout:
 
   # Choose the metrics to compute:
   names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({
-      "accuracy": tf.contrib.metrics.streaming_accuracy(predictions, labels),
-      "mse": tf.contrib.metrics.streaming_mean_squared_error(
-        predictions, labels),
+      "accuracy": tf.metrics.accuracy(labels, predictions),
+      "mse": tf.metrics.mean_squared_error(labels, predictions),
   })
 
   # Define the summaries to write:
@@ -81,9 +80,8 @@ more summaries and call the evaluate_repeatedly method:
 
   # Choose the metrics to compute:
   names_to_values, names_to_updates = tf.contrib.metrics.aggregate_metric_map({
-      "accuracy": tf.contrib.metrics.streaming_accuracy(predictions, labels),
-      "mse": tf.contrib.metrics.streaming_mean_squared_error(
-          predictions, labels),
+      "accuracy": tf.metrics.accuracy(labels, predictions),
+      "mse": tf.metrics.mean_squared_error(labels, predictions),
   })
 
   # Define the summaries to write:
diff --git a/tensorflow/contrib/training/python/training/evaluation_test.py b/tensorflow/contrib/training/python/training/evaluation_test.py
index b07039916c..c36d00e842 100644
--- a/tensorflow/contrib/training/python/training/evaluation_test.py
+++ b/tensorflow/contrib/training/python/training/evaluation_test.py
@@ -27,7 +27,6 @@ import numpy as np
 from tensorflow.contrib.framework.python.ops import variables
 from tensorflow.contrib.layers.python.layers import layers
 from tensorflow.contrib.losses.python.losses import loss_ops
-from tensorflow.contrib.metrics.python.ops import metric_ops
 from tensorflow.contrib.training.python.training import evaluation
 from tensorflow.contrib.training.python.training import training
 from tensorflow.core.protobuf import config_pb2
@@ -38,6 +37,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import metrics
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variables as variables_lib
 from tensorflow.python.platform import gfile
@@ -196,7 +196,8 @@ class EvaluateOnceTest(test.TestCase):
     logits = logistic_classifier(inputs)
     predictions = math_ops.round(logits)
 
-    accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels)
+    accuracy, update_op = metrics.accuracy(
+        predictions=predictions, labels=labels)
 
     checkpoint_path = evaluation.wait_for_new_checkpoint(checkpoint_dir)
 
@@ -311,7 +312,8 @@ class EvaluateRepeatedlyTest(test.TestCase):
     logits = logistic_classifier(inputs)
     predictions = math_ops.round(logits)
 
-    accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels)
+    accuracy, update_op = metrics.accuracy(
+        predictions=predictions, labels=labels)
 
     final_values = evaluation.evaluate_repeatedly(
         checkpoint_dir=checkpoint_dir,
@@ -365,7 +367,8 @@ class EvaluateRepeatedlyTest(test.TestCase):
     logits = logistic_classifier(inputs)
     predictions = math_ops.round(logits)
 
-    accuracy, update_op = metric_ops.streaming_accuracy(predictions, labels)
+    accuracy, update_op = metrics.accuracy(
+        predictions=predictions, labels=labels)
 
     timeout_fn_calls = [0]
     def timeout_fn():
@@ -417,9 +420,8 @@ class EvaluateRepeatedlyTest(test.TestCase):
     self.assertEqual(final_values['my_var'], expected_value)
 
   def _create_names_to_metrics(self, predictions, labels):
-    accuracy0, update_op0 = metric_ops.streaming_accuracy(predictions, labels)
-    accuracy1, update_op1 = metric_ops.streaming_accuracy(
-        predictions + 1, labels)
+    accuracy0, update_op0 = metrics.accuracy(labels, predictions)
+    accuracy1, update_op1 = metrics.accuracy(labels, predictions + 1)
 
     names_to_values = {'Accuracy': accuracy0, 'Another_accuracy': accuracy1}
     names_to_updates = {'Accuracy': update_op0, 'Another_accuracy': update_op1}
-- 
GitLab


From 43c6dd98f1a69c0515f0769b997cfac576a195e5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 09:25:33 -0700
Subject: [PATCH 799/960] Add CycleGAN specific summary.

PiperOrigin-RevId: 191302480
---
 .../gan/python/eval/python/summaries_impl.py  | 64 ++++++++++++++-----
 .../gan/python/eval/python/summaries_test.py  | 20 +++---
 2 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
index 0d1afad72d..508f487722 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries_impl.py
@@ -31,6 +31,7 @@ __all__ = [
     'add_image_comparison_summaries',
     'add_gan_model_summaries',
     'add_regularization_loss_summaries',
+    'add_cyclegan_image_summaries',
 ]
 
 
@@ -51,14 +52,9 @@ def add_gan_model_image_summaries(gan_model, grid_size=4, model_summaries=True):
     ValueError: If real and generated data aren't images.
   """
   if isinstance(gan_model, namedtuples.CycleGANModel):
-    saved_params = locals()
-    saved_params.pop('gan_model', None)
-    with ops.name_scope('cyclegan_x2y_image_summaries'):
-      add_gan_model_image_summaries(gan_model.model_x2y, **saved_params)
-    with ops.name_scope('cyclegan_y2x_image_summaries'):
-      add_gan_model_image_summaries(gan_model.model_y2x, **saved_params)
-    return
-
+    raise ValueError(
+        '`add_gan_model_image_summaries` does not take CycleGANModels. Please '
+        'use `add_cyclegan_image_summaries` instead.')
   _assert_is_image(gan_model.real_data)
   _assert_is_image(gan_model.generated_data)
 
@@ -89,6 +85,49 @@ def add_gan_model_image_summaries(gan_model, grid_size=4, model_summaries=True):
     add_gan_model_summaries(gan_model)
 
 
+def add_cyclegan_image_summaries(cyclegan_model):
+  """Adds image summaries for CycleGAN.
+
+  There are two summaries, one for each generator. The first image is the
+  generator input, the second is the generator output, and the third is G(F(x)).
+
+  Args:
+    cyclegan_model: A CycleGANModel tuple.
+
+  Raises:
+    ValueError: If `cyclegan_model` isn't a CycleGANModel.
+    ValueError: If generated data, generator inputs, and reconstructions aren't
+      images.
+    ValueError: If the generator input, generated data, and reconstructions
+      aren't all the same size.
+  """
+  if not isinstance(cyclegan_model, namedtuples.CycleGANModel):
+    raise ValueError('`cyclegan_model` was not a CycleGANModel. Instead, was '
+                     '%s' % type(cyclegan_model))
+
+  _assert_is_image(cyclegan_model.model_x2y.generator_inputs)
+  _assert_is_image(cyclegan_model.model_x2y.generated_data)
+  _assert_is_image(cyclegan_model.reconstructed_x)
+  _assert_is_image(cyclegan_model.model_y2x.generator_inputs)
+  _assert_is_image(cyclegan_model.model_y2x.generated_data)
+  _assert_is_image(cyclegan_model.reconstructed_y)
+
+  def _add_comparison_summary(gan_model, reconstructions):
+    image_list = (array_ops.unstack(gan_model.generator_inputs[:1]) +
+                  array_ops.unstack(gan_model.generated_data[:1]) +
+                  array_ops.unstack(reconstructions[:1]))
+    summary.image(
+        'image_comparison', eval_utils.image_reshaper(
+            image_list, num_cols=len(image_list)), max_outputs=1)
+
+  with ops.name_scope('x2y_image_comparison_summaries'):
+    _add_comparison_summary(
+        cyclegan_model.model_x2y, cyclegan_model.reconstructed_x)
+  with ops.name_scope('y2x_image_comparison_summaries'):
+    _add_comparison_summary(
+        cyclegan_model.model_y2x, cyclegan_model.reconstructed_y)
+
+
 def add_image_comparison_summaries(gan_model, num_comparisons=2,
                                    display_diffs=False):
   """Adds image summaries to compare triplets of images.
@@ -109,15 +148,6 @@ def add_image_comparison_summaries(gan_model, num_comparisons=2,
     ValueError: If the generator input, real, and generated data aren't all the
       same size.
   """
-  if isinstance(gan_model, namedtuples.CycleGANModel):
-    saved_params = locals()
-    saved_params.pop('gan_model', None)
-    with ops.name_scope('cyclegan_x2y_image_comparison_summaries'):
-      add_image_comparison_summaries(gan_model.model_x2y, **saved_params)
-    with ops.name_scope('cyclegan_y2x_image_comparison_summaries'):
-      add_image_comparison_summaries(gan_model.model_y2x, **saved_params)
-    return
-
   _assert_is_image(gan_model.generator_inputs)
   _assert_is_image(gan_model.generated_data)
   _assert_is_image(gan_model.real_data)
diff --git a/tensorflow/contrib/gan/python/eval/python/summaries_test.py b/tensorflow/contrib/gan/python/eval/python/summaries_test.py
index 45eb108586..33d51bfc21 100644
--- a/tensorflow/contrib/gan/python/eval/python/summaries_test.py
+++ b/tensorflow/contrib/gan/python/eval/python/summaries_test.py
@@ -65,15 +65,14 @@ def get_cyclegan_model():
   return namedtuples.CycleGANModel(
       model_x2y=model_x2y,
       model_y2x=model_y2x,
-      reconstructed_x=array_ops.zeros([3, 30, 35, 6]),
-      reconstructed_y=array_ops.zeros([3, 30, 35, 6]))
+      reconstructed_x=array_ops.zeros([4, 32, 32, 3]),
+      reconstructed_y=array_ops.zeros([4, 32, 32, 3]))
 
 
 class SummariesTest(test.TestCase):
 
-  def _test_add_gan_model_image_summaries_impl(self, get_model_fn,
-                                               expected_num_summary_ops,
-                                               model_summaries):
+  def _test_add_gan_model_image_summaries_impl(
+      self, get_model_fn, expected_num_summary_ops, model_summaries):
     summaries.add_gan_model_image_summaries(get_model_fn(), grid_size=2,
                                             model_summaries=model_summaries)
 
@@ -89,8 +88,9 @@ class SummariesTest(test.TestCase):
   def test_add_gan_model_image_summaries_no_model(self):
     self._test_add_gan_model_image_summaries_impl(get_gan_model, 2, False)
 
-  def test_add_gan_model_image_summaries_for_cyclegan(self):
-    self._test_add_gan_model_image_summaries_impl(get_cyclegan_model, 10, True)
+  def test_cyclegan_image_summaries_dont_work(self):
+    with self.assertRaises(ValueError):
+      summaries.add_gan_model_image_summaries(get_cyclegan_model())
 
   def _test_add_gan_model_summaries_impl(self, get_model_fn,
                                          expected_num_summary_ops):
@@ -137,7 +137,11 @@ class SummariesTest(test.TestCase):
     self._test_add_image_comparison_summaries_impl(get_gan_model, 1)
 
   def test_add_image_comparison_summaries_for_cyclegan(self):
-    self._test_add_image_comparison_summaries_impl(get_cyclegan_model, 2)
+    summaries.add_cyclegan_image_summaries(get_cyclegan_model())
+
+    self.assertEquals(2, len(ops.get_collection(ops.GraphKeys.SUMMARIES)))
+    with self.test_session(use_gpu=True):
+      summary.merge_all().eval()
 
 
 if __name__ == '__main__':
-- 
GitLab


From 7ca4101eac5cb580400d48ef683d812b44bc45bd Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Mon, 2 Apr 2018 09:29:33 -0700
Subject: [PATCH 800/960] Only test types supported and change log_eps for
 bfloat16.

PiperOrigin-RevId: 191302894
---
 tensorflow/compiler/tests/cholesky_op_test.py             | 7 +++++++
 .../compiler/tests/matrix_triangular_solve_op_test.py     | 8 ++++++++
 tensorflow/compiler/tests/unary_ops_test.py               | 5 ++++-
 3 files changed, 19 insertions(+), 1 deletion(-)

diff --git a/tensorflow/compiler/tests/cholesky_op_test.py b/tensorflow/compiler/tests/cholesky_op_test.py
index 5010fe5e21..1a8989d7c2 100644
--- a/tensorflow/compiler/tests/cholesky_op_test.py
+++ b/tensorflow/compiler/tests/cholesky_op_test.py
@@ -34,6 +34,13 @@ from tensorflow.python.platform import test
 
 class CholeskyOpTest(XLATestCase):
 
+  # Cholesky defined for float64, float32, complex64, complex128
+  # (https://www.tensorflow.org/api_docs/python/tf/cholesky)
+  @property
+  def float_types(self):
+    return set(super(CholeskyOpTest, self).float_types).intersection(
+        (np.float64, np.float32, np.complex64, np.complex128))
+
   def _verifyCholeskyBase(self, sess, placeholder, x, chol, verification, atol):
     chol_np, verification_np = sess.run([chol, verification], {placeholder: x})
     self.assertAllClose(x, verification_np, atol=atol)
diff --git a/tensorflow/compiler/tests/matrix_triangular_solve_op_test.py b/tensorflow/compiler/tests/matrix_triangular_solve_op_test.py
index cccb7f5789..5819b2bf2b 100644
--- a/tensorflow/compiler/tests/matrix_triangular_solve_op_test.py
+++ b/tensorflow/compiler/tests/matrix_triangular_solve_op_test.py
@@ -37,6 +37,14 @@ def MakePlaceholder(x):
 
 class MatrixTriangularSolveOpTest(XLATestCase):
 
+  #  MatrixTriangularSolve defined for float64, float32, complex64, complex128
+  # (https://www.tensorflow.org/api_docs/python/tf/matrix_triangular_solve)
+  @property
+  def float_types(self):
+    return set(super(MatrixTriangularSolveOpTest,
+                     self).float_types).intersection(
+                         (np.float64, np.float32, np.complex64, np.complex128))
+
   def _VerifyTriangularSolveBase(self, sess, placeholder_a, placeholder_ca,
                                  placeholder_b, a, clean_a, b, verification,
                                  atol):
diff --git a/tensorflow/compiler/tests/unary_ops_test.py b/tensorflow/compiler/tests/unary_ops_test.py
index a8ab235378..17149aa1c8 100644
--- a/tensorflow/compiler/tests/unary_ops_test.py
+++ b/tensorflow/compiler/tests/unary_ops_test.py
@@ -793,7 +793,10 @@ class UnaryOpsTest(XLATestCase):
       self._assertSoftplusMatchesExpected([[-2, 0, 8]], dtype)
       self._assertSoftplusMatchesExpected(
           [[-9, 7, -5, 3, -1], [1, -3, 5, -7, 9]], dtype)
-      log_eps = np.log(np.finfo(dtype).eps)
+      if dtype == dtypes.bfloat16.as_numpy_dtype:
+        log_eps = np.log(np.finfo(np.float32).eps)
+      else:
+        log_eps = np.log(np.finfo(dtype).eps)
       one = dtype(1)
       ten = dtype(10)
       self._assertSoftplusMatchesExpected([
-- 
GitLab


From 861f7a3ecc3c5ecbc22d2fa731956f4ff0469c25 Mon Sep 17 00:00:00 2001
From: Sergio Guadarrama <sguada@google.com>
Date: Mon, 2 Apr 2018 09:49:35 -0700
Subject: [PATCH 801/960] Automated g4 rollback of changelist 191127281

PiperOrigin-RevId: 191305220
---
 tensorflow/core/common_runtime/mkl_cpu_allocator.cc | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
index 43a909466e..829c19204a 100644
--- a/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
+++ b/tensorflow/core/common_runtime/mkl_cpu_allocator.cc
@@ -19,9 +19,6 @@ limitations under the License.
 
 namespace tensorflow {
 
-constexpr const char* MklCPUAllocator::kMaxLimitStr;
-constexpr const size_t MklCPUAllocator::kDefaultMaxLimit;
-
 }  // namespace tensorflow
 
 #endif  // INTEL_MKL
-- 
GitLab


From 136b6095c42f6591c70a9f640598ff8398348b40 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 10:14:35 -0700
Subject: [PATCH 802/960] Additional arg scope test that demonstrate how nested
 arg_scope objects behave.

PiperOrigin-RevId: 191308666
---
 .../framework/python/ops/arg_scope_test.py    | 24 +++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/tensorflow/contrib/framework/python/ops/arg_scope_test.py b/tensorflow/contrib/framework/python/ops/arg_scope_test.py
index 7ba9d4ffa9..4c3879d4fc 100644
--- a/tensorflow/contrib/framework/python/ops/arg_scope_test.py
+++ b/tensorflow/contrib/framework/python/ops/arg_scope_test.py
@@ -170,6 +170,30 @@ class ArgScopeTest(test.TestCase):
         self.assertTupleEqual(args, func1_args)
         self.assertDictEqual(kwargs, func1_kwargs)
 
+  def testNestedArgScopeObjectCreatedOutsideScopeOverridesArgScope(self):
+
+    def get_scope_object():
+      with arg_scope([func1], a=1, b=None, c=[1]) as sc:
+        return sc
+
+    scope_object = get_scope_object()
+    with arg_scope([func1], b=2, d=10):
+      with arg_scope(scope_object):
+        args, kwargs = func1(0)
+        self.assertTupleEqual(args, (0,))
+        self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1]})
+
+  def testArgScopeObjectCreatedWithinScopeInheritsArgScope(self):
+    def get_scope_object():
+      with arg_scope([func1], a=1, b=None, c=[1]) as sc:
+        return sc
+
+    with arg_scope([func1], b=2, d=10):
+      with arg_scope(get_scope_object()):
+        args, kwargs = func1(0)
+        self.assertTupleEqual(args, (0,))
+        self.assertDictEqual(kwargs, {'a': 1, 'b': None, 'c': [1], 'd': 10})
+
   def testSharedArgScope(self):
     func1_args = (0,)
     func1_kwargs = {'a': 1, 'b': None, 'c': [1]}
-- 
GitLab


From 3edab0abb1213f88507692042a320abc695ff674 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 10:27:12 -0700
Subject: [PATCH 803/960] Remove reshape of sparse tensor indices in for
 maybe_batch.

PiperOrigin-RevId: 191310753
---
 tensorflow/python/training/input.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/tensorflow/python/training/input.py b/tensorflow/python/training/input.py
index 44f00a96de..caa26581e8 100644
--- a/tensorflow/python/training/input.py
+++ b/tensorflow/python/training/input.py
@@ -515,8 +515,7 @@ def _store_sparse_tensors(tensor_list, enqueue_many, keep_input,
     def _sparse_values_to_keep(t, keep_input):
       """Convert a per-row `keep_input` vector to a per-value one."""
       # Get the rows of every value in the sparse Tensor.
-      row_values = array_ops.reshape(
-          t.indices, [array_ops.shape(t.indices)[0], -1])[:, 0]
+      row_values = t.indices[:, 0]
       # The value should be kept iff the row should be kept.
       return array_ops.gather(keep_input, row_values)
     if keep_input.shape.ndims == 1:
-- 
GitLab


From 82bb63b958f5e39fb57ad45147b7d2c662c9b06c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 10:50:09 -0700
Subject: [PATCH 804/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 191314576
---
 tensorflow/core/framework/attr_value_util.cc  |  26 +--
 .../core/framework/common_shape_fns_test.cc   |  18 +-
 tensorflow/core/framework/dataset.h           |   2 +-
 tensorflow/core/framework/function.cc         |   5 +-
 tensorflow/core/framework/function_test.cc    |   2 +-
 tensorflow/core/framework/graph_def_util.cc   |   3 +-
 .../core/framework/node_def_builder_test.cc   |   5 +-
 tensorflow/core/framework/node_def_util.cc    |   7 +-
 .../core/framework/node_def_util_test.cc      |   5 +-
 tensorflow/core/framework/op.cc               |   3 +-
 .../core/framework/op_compatibility_test.cc   |   7 +-
 tensorflow/core/framework/op_def_builder.cc   |  49 ++--
 tensorflow/core/framework/op_def_util.cc      |  12 +-
 tensorflow/core/framework/op_def_util_test.cc |   2 +-
 tensorflow/core/framework/op_gen_lib.cc       |  21 +-
 tensorflow/core/framework/op_kernel.cc        |   2 +-
 tensorflow/core/framework/op_kernel_test.cc   |  19 +-
 .../core/framework/resource_mgr_test.cc       |   3 +-
 .../core/framework/shape_inference_test.cc    | 209 ++++++++----------
 .../framework/shape_inference_testutil.cc     |   8 +-
 .../core/framework/shape_inference_testutil.h |  23 +-
 .../shape_inference_testutil_test.cc          |  14 +-
 tensorflow/core/framework/types.cc            |   2 +-
 tensorflow/core/framework/types_test.cc       |   6 +-
 .../core/framework/variant_op_copy_test.cc    |  10 +-
 .../framework/variant_op_registry_test.cc     |  11 +-
 26 files changed, 242 insertions(+), 232 deletions(-)

diff --git a/tensorflow/core/framework/attr_value_util.cc b/tensorflow/core/framework/attr_value_util.cc
index ebb56d525e..87c1ddd15d 100644
--- a/tensorflow/core/framework/attr_value_util.cc
+++ b/tensorflow/core/framework/attr_value_util.cc
@@ -186,7 +186,7 @@ Status AttrValueHasType(const AttrValue& attr_value, StringPiece type) {
   // check if has_list is false and some other field in attr_value is
   // set to flag the error.  This test can be made more strict once
   // support for GraphDef versions <= 4 is dropped.
-  if (StringPiece(type).starts_with("list(") && !attr_value.has_list()) {
+  if (str_util::StartsWith(type, "list(") && !attr_value.has_list()) {
     if (num_set) {
       return errors::InvalidArgument(
           "AttrValue missing value with expected type '", type, "'");
@@ -197,7 +197,7 @@ Status AttrValueHasType(const AttrValue& attr_value, StringPiece type) {
   }
 
   // Okay to have an empty list, but not to be missing a non-list value.
-  if (num_set == 0 && !StringPiece(type).starts_with("list(")) {
+  if (num_set == 0 && !str_util::StartsWith(type, "list(")) {
     return errors::InvalidArgument(
         "AttrValue missing value with expected type '", type, "'");
   }
@@ -241,29 +241,29 @@ Status AttrValueHasType(const AttrValue& attr_value, StringPiece type) {
 bool ParseAttrValue(StringPiece type, StringPiece text, AttrValue* out) {
   // Parse type.
   string field_name;
-  bool is_list = type.Consume("list(");
-  if (type.Consume("string")) {
+  bool is_list = str_util::ConsumePrefix(&type, "list(");
+  if (str_util::ConsumePrefix(&type, "string")) {
     field_name = "s";
-  } else if (type.Consume("int")) {
+  } else if (str_util::ConsumePrefix(&type, "int")) {
     field_name = "i";
-  } else if (type.Consume("float")) {
+  } else if (str_util::ConsumePrefix(&type, "float")) {
     field_name = "f";
-  } else if (type.Consume("bool")) {
+  } else if (str_util::ConsumePrefix(&type, "bool")) {
     field_name = "b";
-  } else if (type.Consume("type")) {
+  } else if (str_util::ConsumePrefix(&type, "type")) {
     field_name = "type";
-  } else if (type.Consume("shape")) {
+  } else if (str_util::ConsumePrefix(&type, "shape")) {
     field_name = "shape";
-  } else if (type.Consume("tensor")) {
+  } else if (str_util::ConsumePrefix(&type, "tensor")) {
     field_name = "tensor";
-  } else if (type.Consume("func")) {
+  } else if (str_util::ConsumePrefix(&type, "func")) {
     field_name = "func";
-  } else if (type.Consume("placeholder")) {
+  } else if (str_util::ConsumePrefix(&type, "placeholder")) {
     field_name = "placeholder";
   } else {
     return false;
   }
-  if (is_list && !type.Consume(")")) {
+  if (is_list && !str_util::ConsumePrefix(&type, ")")) {
     return false;
   }
 
diff --git a/tensorflow/core/framework/common_shape_fns_test.cc b/tensorflow/core/framework/common_shape_fns_test.cc
index 5f3e5ad457..13d429b895 100644
--- a/tensorflow/core/framework/common_shape_fns_test.cc
+++ b/tensorflow/core/framework/common_shape_fns_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/shape_inference_testutil.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -140,9 +141,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) {
                        {}, {}, {});
     auto s = MatMulShape(&c);
     EXPECT_FALSE(s.ok());
-    EXPECT_TRUE(
-        StringPiece(s.ToString())
-            .contains("Invalid argument: Shape must be rank 2 but is rank 1"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.ToString(), "Invalid argument: Shape must be rank 2 but is rank 1"));
   }
 
   {
@@ -161,10 +161,9 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) {
                        {S({2, 5}), S({3, 4})}, {}, {}, {});
     auto s = MatMulShape(&c);
     EXPECT_FALSE(s.ok());
-    EXPECT_TRUE(
-        StringPiece(s.ToString())
-            .contains(
-                "Invalid argument: Dimensions must be equal, but are 5 and 3"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.ToString(),
+        "Invalid argument: Dimensions must be equal, but are 5 and 3"));
   }
 
   {
@@ -173,9 +172,8 @@ TEST(CommonShapeFnsTest, MatMulShapeTest) {
                        {S({2, 5, 3}), S({3, 5, 4})}, {}, {}, {});
     auto s = MatMulShape(&c);
     EXPECT_FALSE(s.ok());
-    EXPECT_TRUE(
-        StringPiece(s.ToString())
-            .contains("Invalid argument: Shape must be rank 2 but is rank 3"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.ToString(), "Invalid argument: Shape must be rank 2 but is rank 3"));
   }
 
   {
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index beaf0adbc5..fb1fe9c51f 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -201,7 +201,7 @@ class GraphDefBuilderWrapper {
   // Also looks up the `op_def->name` in the global
   // `WhitelistedStatefulOpRegistry`.
   bool IsOpWhitelisted(const OpDef* op_def) const {
-    return (StringPiece(op_def->name()).ends_with("Dataset") &&
+    return (str_util::EndsWith(op_def->name(), "Dataset") &&
             op_def->output_arg_size() == 1 &&
             op_def->output_arg(0).type() == DT_VARIANT) ||
            dataset::WhitelistedStatefulOpRegistry::Global()->Contains(
diff --git a/tensorflow/core/framework/function.cc b/tensorflow/core/framework/function.cc
index 3e7b89d4eb..bdc1af9fda 100644
--- a/tensorflow/core/framework/function.cc
+++ b/tensorflow/core/framework/function.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/util/equal_graph_def.h"
 
 namespace tensorflow {
@@ -278,7 +279,7 @@ class FunctionInstantiationHelper {
       auto it = index_.lower_bound(node_name);
       while (it != index_.end() && it->first <= node_colon_bound) {
         if (it->first == node_name ||
-            tensorflow::StringPiece(it->first).starts_with(node_colon)) {
+            tensorflow::str_util::StartsWith(it->first, node_colon)) {
           nid = it->second.nid;
           break;
         }
@@ -502,7 +503,7 @@ string Print(const NodeDef& n) {
   std::vector<StringPiece> dat;
   std::vector<string> dep;
   for (StringPiece s : n.input()) {
-    if (s.Consume("^")) {
+    if (str_util::ConsumePrefix(&s, "^")) {
       dep.push_back(s.ToString());
     } else {
       dat.push_back(s);
diff --git a/tensorflow/core/framework/function_test.cc b/tensorflow/core/framework/function_test.cc
index 23685e9c53..44e1383719 100644
--- a/tensorflow/core/framework/function_test.cc
+++ b/tensorflow/core/framework/function_test.cc
@@ -496,7 +496,7 @@ MySelect(x:float) -> (z:float) {
 }
 
 static void HasError(const Status& s, const string& substr) {
-  EXPECT_TRUE(StringPiece(s.ToString()).contains(substr))
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), substr))
       << ">>" << s << "<<, expected substring >>" << substr << "<<";
 }
 
diff --git a/tensorflow/core/framework/graph_def_util.cc b/tensorflow/core/framework/graph_def_util.cc
index 896cb3cd7f..f7539d37be 100644
--- a/tensorflow/core/framework/graph_def_util.cc
+++ b/tensorflow/core/framework/graph_def_util.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/framework/versions.pb_text.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 
 namespace tensorflow {
@@ -94,7 +95,7 @@ static Status RemoveNewDefaultAttrsFromNodeDef(
   std::vector<string> to_remove;
   for (const auto& attr : node_def->attr()) {
     // If the attr is not in consumer_op_def and doesn't start with '_'...
-    if (!StringPiece(attr.first).starts_with("_") &&
+    if (!str_util::StartsWith(attr.first, "_") &&
         FindAttr(attr.first, *consumer_op_def) == nullptr) {
       const OpDef::AttrDef* producer_attr_def =
           FindAttr(attr.first, *producer_op_def);
diff --git a/tensorflow/core/framework/node_def_builder_test.cc b/tensorflow/core/framework/node_def_builder_test.cc
index e836873f66..cc583df348 100644
--- a/tensorflow/core/framework/node_def_builder_test.cc
+++ b/tensorflow/core/framework/node_def_builder_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_def_builder.h"
 #include "tensorflow/core/framework/op_def_util.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -82,7 +83,7 @@ class NodeDefBuilderTest : public ::testing::Test {
     EXPECT_FALSE(status.ok()) << SummarizeNodeDef(node_def);
     if (status.ok()) return;
     for (const string& message : messages) {
-      EXPECT_TRUE(StringPiece(status.error_message()).contains(message))
+      EXPECT_TRUE(str_util::StrContains(status.error_message(), message))
           << status << ", " << message;
     }
   }
@@ -103,7 +104,7 @@ class NodeDefBuilderTest : public ::testing::Test {
     }
     EXPECT_FALSE(status.ok()) << SummarizeNodeDef(node_def);
     if (status.ok()) return;
-    EXPECT_TRUE(StringPiece(status.error_message()).contains(message))
+    EXPECT_TRUE(str_util::StrContains(status.error_message(), message))
         << "Actual error: " << status.error_message()
         << "\nDoes not contain: " << message;
   }
diff --git a/tensorflow/core/framework/node_def_util.cc b/tensorflow/core/framework/node_def_util.cc
index 95fb386314..bad92ca9b3 100644
--- a/tensorflow/core/framework/node_def_util.cc
+++ b/tensorflow/core/framework/node_def_util.cc
@@ -31,6 +31,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
 #include "tensorflow/core/lib/strings/scanner.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -131,7 +132,7 @@ Status AttrSlice::Find(StringPiece attr_name,
   // Skip AttachDef for internal attrs since it is a little bit
   // expensive and it is common for them to correctly not be included
   // in a NodeDef.
-  if (!attr_name.starts_with("_") && ndef_ != nullptr) {
+  if (!str_util::StartsWith(attr_name, "_") && ndef_ != nullptr) {
     s = AttachDef(s, *ndef_);
   }
   return s;
@@ -399,7 +400,7 @@ Status ValidateNodeDef(const NodeDef& node_def, const OpDef& op_def) {
   size_t num_inputs = 0;
   // TODO(josh11b): Unify the input field validation.
   for (const string& input : node_def.input()) {
-    if (StringPiece(input).starts_with("^")) {
+    if (str_util::StartsWith(input, "^")) {
       seen_control = true;
       if (input.find(':') != string::npos) {
         return errors::InvalidArgument(
@@ -425,7 +426,7 @@ Status ValidateNodeDef(const NodeDef& node_def, const OpDef& op_def) {
   }
   for (const auto& attr : node_def.attr()) {
     // Allow internal optional attributes with names starting with "_".
-    if (StringPiece(attr.first).starts_with("_")) {
+    if (str_util::StartsWith(attr.first, "_")) {
       continue;
     }
     auto iter = op_attrs.find(attr.first);
diff --git a/tensorflow/core/framework/node_def_util_test.cc b/tensorflow/core/framework/node_def_util_test.cc
index ae3a93eafe..2a49425dba 100644
--- a/tensorflow/core/framework/node_def_util_test.cc
+++ b/tensorflow/core/framework/node_def_util_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_def_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -65,7 +66,7 @@ void ExpectFailure(const NodeDef& bad, const OpDef& op_def,
       << "; OpDef: " << SummarizeOpDef(op_def);
 
   LOG(INFO) << "Message: " << status.error_message();
-  EXPECT_TRUE(StringPiece(status.ToString()).contains(message))
+  EXPECT_TRUE(str_util::StrContains(status.ToString(), message))
       << "NodeDef: " << SummarizeNodeDef(bad)
       << "; OpDef: " << SummarizeOpDef(op_def) << "\nActual error: " << status
       << "\nDoes not contain: " << message;
@@ -265,7 +266,7 @@ void ExpectInvalidSyntax(const NodeDef& bad, const string& message) {
   EXPECT_TRUE(errors::IsInvalidArgument(status))
       << status << "; NodeDef: " << SummarizeNodeDef(bad);
 
-  EXPECT_TRUE(StringPiece(status.ToString()).contains(message))
+  EXPECT_TRUE(str_util::StrContains(StringPiece(status.ToString()), message))
       << "NodeDef: " << SummarizeNodeDef(bad) << ", " << status << ", "
       << message;
 }
diff --git a/tensorflow/core/framework/op.cc b/tensorflow/core/framework/op.cc
index fc5467b3c8..5f68c59fe9 100644
--- a/tensorflow/core/framework/op.cc
+++ b/tensorflow/core/framework/op.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/map_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/host_info.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/mutex.h"
@@ -142,7 +143,7 @@ void OpRegistry::Export(bool include_internal, OpList* ops) const {
   out->Reserve(sorted.size());
 
   for (const auto& item : sorted) {
-    if (include_internal || !StringPiece(item.first).starts_with("_")) {
+    if (include_internal || !str_util::StartsWith(item.first, "_")) {
       *out->Add() = item.second->op_def;
     }
   }
diff --git a/tensorflow/core/framework/op_compatibility_test.cc b/tensorflow/core/framework/op_compatibility_test.cc
index b57bdcb841..c782480f1f 100644
--- a/tensorflow/core/framework/op_compatibility_test.cc
+++ b/tensorflow/core/framework/op_compatibility_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/kernels/ops_testutil.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -96,7 +97,7 @@ class OpCompatibilityTest : public OpsTestBase {
       ADD_FAILURE() << SummarizeOpDef(old_op_def) << " vs. "
                     << SummarizeOpDef(new_op_def);
     } else {
-      EXPECT_TRUE(StringPiece(status.error_message()).contains(error))
+      EXPECT_TRUE(str_util::StrContains(status.error_message(), error))
           << status << " does not contain " << error;
     }
   }
@@ -118,7 +119,7 @@ class OpCompatibilityTest : public OpsTestBase {
       ADD_FAILURE() << SummarizeNodeDef(*node_def());
     } else {
       EXPECT_TRUE(
-          StringPiece(status.error_message()).contains(validation_error))
+          str_util::StrContains(status.error_message(), validation_error))
           << status << " does not contain " << validation_error;
     }
 
@@ -179,7 +180,7 @@ class OpCompatibilityTest : public OpsTestBase {
                     << SummarizeOpDef(*new_op_def);
     } else {
       EXPECT_TRUE(
-          StringPiece(status.error_message()).contains(compatibility_error))
+          str_util::StrContains(status.error_message(), compatibility_error))
           << status << " does not contain " << compatibility_error;
     }
   }
diff --git a/tensorflow/core/framework/op_def_builder.cc b/tensorflow/core/framework/op_def_builder.cc
index 962bc11ccb..403bd0b5e2 100644
--- a/tensorflow/core/framework/op_def_builder.cc
+++ b/tensorflow/core/framework/op_def_builder.cc
@@ -112,9 +112,11 @@ bool ConsumeAttrNumber(StringPiece* sp, int64* out) {
 
 bool ConsumeCompoundAttrType(StringPiece* sp, StringPiece* out) {
   auto capture_begin = sp->begin();
-  if (sp->Consume("numbertype") || sp->Consume("numerictype") ||
-      sp->Consume("quantizedtype") || sp->Consume("realnumbertype") ||
-      sp->Consume("realnumberictype")) {
+  if (str_util::ConsumePrefix(sp, "numbertype") ||
+      str_util::ConsumePrefix(sp, "numerictype") ||
+      str_util::ConsumePrefix(sp, "quantizedtype") ||
+      str_util::ConsumePrefix(sp, "realnumbertype") ||
+      str_util::ConsumePrefix(sp, "realnumberictype")) {
     *out = StringPiece(capture_begin, sp->begin() - capture_begin);
     return true;
   }
@@ -155,32 +157,32 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
   bool is_list = ConsumeListPrefix(&spec);
   string type;
   StringPiece type_string;  // Used if type == "type"
-  if (spec.Consume("string")) {
+  if (str_util::ConsumePrefix(&spec, "string")) {
     type = "string";
-  } else if (spec.Consume("int")) {
+  } else if (str_util::ConsumePrefix(&spec, "int")) {
     type = "int";
-  } else if (spec.Consume("float")) {
+  } else if (str_util::ConsumePrefix(&spec, "float")) {
     type = "float";
-  } else if (spec.Consume("bool")) {
+  } else if (str_util::ConsumePrefix(&spec, "bool")) {
     type = "bool";
-  } else if (spec.Consume("type")) {
+  } else if (str_util::ConsumePrefix(&spec, "type")) {
     type = "type";
-  } else if (spec.Consume("shape")) {
+  } else if (str_util::ConsumePrefix(&spec, "shape")) {
     type = "shape";
-  } else if (spec.Consume("tensor")) {
+  } else if (str_util::ConsumePrefix(&spec, "tensor")) {
     type = "tensor";
-  } else if (spec.Consume("func")) {
+  } else if (str_util::ConsumePrefix(&spec, "func")) {
     type = "func";
   } else if (ConsumeCompoundAttrType(&spec, &type_string)) {
     type = "type";
     AttrValue* allowed = attr->mutable_allowed_values();
     VERIFY(ProcessCompoundType(type_string, allowed),
            "Expected to see a compound type, saw: ", type_string);
-  } else if (spec.Consume("{")) {
+  } else if (str_util::ConsumePrefix(&spec, "{")) {
     // e.g. "{ int32, float, bool }" or "{ \"foo\", \"bar\" }"
     AttrValue* allowed = attr->mutable_allowed_values();
     str_util::RemoveLeadingWhitespace(&spec);
-    if (spec.starts_with("\"") || spec.starts_with("'")) {
+    if (str_util::StartsWith(spec, "\"") || str_util::StartsWith(spec, "'")) {
       type = "string";  // "{ \"foo\", \"bar\" }" or "{ 'foo', 'bar' }"
       while (true) {
         StringPiece escaped_string;
@@ -193,11 +195,12 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
                "Trouble unescaping \"", escaped_string,
                "\", got error: ", error);
         allowed->mutable_list()->add_s(unescaped);
-        if (spec.Consume(",")) {
+        if (str_util::ConsumePrefix(&spec, ",")) {
           str_util::RemoveLeadingWhitespace(&spec);
-          if (spec.Consume("}")) break;  // Allow ending with ", }".
+          if (str_util::ConsumePrefix(&spec, "}"))
+            break;  // Allow ending with ", }".
         } else {
-          VERIFY(spec.Consume("}"),
+          VERIFY(str_util::ConsumePrefix(&spec, "}"),
                  "Expected , or } after strings in list, not: '", spec, "'");
           break;
         }
@@ -215,11 +218,12 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
                  "Unrecognized type string '", type_string, "'");
           allowed->mutable_list()->add_type(dt);
         }
-        if (spec.Consume(",")) {
+        if (str_util::ConsumePrefix(&spec, ",")) {
           str_util::RemoveLeadingWhitespace(&spec);
-          if (spec.Consume("}")) break;  // Allow ending with ", }".
+          if (str_util::ConsumePrefix(&spec, "}"))
+            break;  // Allow ending with ", }".
         } else {
-          VERIFY(spec.Consume("}"),
+          VERIFY(str_util::ConsumePrefix(&spec, "}"),
                  "Expected , or } after types in list, not: '", spec, "'");
           break;
         }
@@ -232,7 +236,8 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
 
   // Write the type into *attr.
   if (is_list) {
-    VERIFY(spec.Consume(")"), "Expected ) to close 'list(', not: '", spec, "'");
+    VERIFY(str_util::ConsumePrefix(&spec, ")"),
+           "Expected ) to close 'list(', not: '", spec, "'");
     str_util::RemoveLeadingWhitespace(&spec);
     attr->set_type(strings::StrCat("list(", type, ")"));
   } else {
@@ -240,7 +245,7 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
   }
 
   // Read optional minimum constraint at the end.
-  if ((is_list || type == "int") && spec.Consume(">=")) {
+  if ((is_list || type == "int") && str_util::ConsumePrefix(&spec, ">=")) {
     int64 min_limit = -999;
     VERIFY(ConsumeAttrNumber(&spec, &min_limit),
            "Could not parse integer lower limit after '>=', found '", spec,
@@ -250,7 +255,7 @@ void FinalizeAttr(StringPiece spec, OpDef* op_def,
   }
 
   // Parse default value, if present.
-  if (spec.Consume("=")) {
+  if (str_util::ConsumePrefix(&spec, "=")) {
     str_util::RemoveLeadingWhitespace(&spec);
     VERIFY(ParseAttrValue(attr->type(), spec, attr->mutable_default_value()),
            "Could not parse default value '", spec, "'");
diff --git a/tensorflow/core/framework/op_def_util.cc b/tensorflow/core/framework/op_def_util.cc
index c80802aad3..9be0dc69d2 100644
--- a/tensorflow/core/framework/op_def_util.cc
+++ b/tensorflow/core/framework/op_def_util.cc
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/lib/hash/hash.h"
 #include "tensorflow/core/lib/strings/proto_serialization.h"
 #include "tensorflow/core/lib/strings/scanner.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -239,7 +240,7 @@ static Status ValidateArg(const OpDef::ArgDef& arg, const OpDef& op_def,
 Status ValidateOpDef(const OpDef& op_def) {
   using ::tensorflow::strings::Scanner;
 
-  if (!StringPiece(op_def.name()).starts_with("_")) {
+  if (!str_util::StartsWith(op_def.name(), "_")) {
     VALIDATE(Scanner(op_def.name())
                  .One(Scanner::UPPERLETTER)
                  .Any(Scanner::LETTER_DIGIT)
@@ -259,11 +260,11 @@ Status ValidateOpDef(const OpDef& op_def) {
 
     // Validate type
     StringPiece type(attr.type());
-    bool is_list = type.Consume("list(");
+    bool is_list = str_util::ConsumePrefix(&type, "list(");
     bool found = false;
     for (StringPiece valid : {"string", "int", "float", "bool", "type", "shape",
                               "tensor", "func"}) {
-      if (type.Consume(valid)) {
+      if (str_util::ConsumePrefix(&type, valid)) {
         found = true;
         break;
       }
@@ -271,8 +272,9 @@ Status ValidateOpDef(const OpDef& op_def) {
     VALIDATE(found, "Unrecognized type '", type, "' in attr '", attr.name(),
              "'");
     if (is_list) {
-      VALIDATE(type.Consume(")"), "'list(' is missing ')' in attr ",
-               attr.name(), "'s type ", attr.type());
+      VALIDATE(str_util::ConsumePrefix(&type, ")"),
+               "'list(' is missing ')' in attr ", attr.name(), "'s type ",
+               attr.type());
     }
     VALIDATE(type.empty(), "Extra '", type, "' at the end of attr ",
              attr.name(), "'s type ", attr.type());
diff --git a/tensorflow/core/framework/op_def_util_test.cc b/tensorflow/core/framework/op_def_util_test.cc
index 2b9812d4fc..4514d92e38 100644
--- a/tensorflow/core/framework/op_def_util_test.cc
+++ b/tensorflow/core/framework/op_def_util_test.cc
@@ -57,7 +57,7 @@ class ValidateOpDefTest : public ::testing::Test {
     EXPECT_FALSE(status.ok()) << "Did not see error with: " << message;
     if (!status.ok()) {
       LOG(INFO) << "message: " << status;
-      EXPECT_TRUE(StringPiece(status.ToString()).contains(message))
+      EXPECT_TRUE(str_util::StrContains(status.ToString(), message))
           << "Actual: " << status << "\nExpected to contain: " << message;
     }
   }
diff --git a/tensorflow/core/framework/op_gen_lib.cc b/tensorflow/core/framework/op_gen_lib.cc
index 5f2eb9d99a..7f23272871 100644
--- a/tensorflow/core/framework/op_gen_lib.cc
+++ b/tensorflow/core/framework/op_gen_lib.cc
@@ -50,10 +50,10 @@ string WordWrap(StringPiece prefix, StringPiece str, int width) {
     StringPiece to_append = str.substr(0, space);
     str.remove_prefix(space + 1);
     // Remove spaces at break.
-    while (to_append.ends_with(" ")) {
+    while (str_util::EndsWith(to_append, " ")) {
       to_append.remove_suffix(1);
     }
-    while (str.Consume(" ")) {
+    while (str_util::ConsumePrefix(&str, " ")) {
     }
 
     // Go on to the next line.
@@ -65,8 +65,9 @@ string WordWrap(StringPiece prefix, StringPiece str, int width) {
 }
 
 bool ConsumeEquals(StringPiece* description) {
-  if (description->Consume("=")) {
-    while (description->Consume(" ")) {  // Also remove spaces after "=".
+  if (str_util::ConsumePrefix(description, "=")) {
+    while (str_util::ConsumePrefix(description,
+                                   " ")) {  // Also remove spaces after "=".
     }
     return true;
   }
@@ -98,7 +99,7 @@ static bool StartsWithFieldName(StringPiece line,
                                 const std::vector<string>& multi_line_fields) {
   StringPiece up_to_colon;
   if (!SplitAt(':', &line, &up_to_colon)) return false;
-  while (up_to_colon.Consume(" "))
+  while (str_util::ConsumePrefix(&up_to_colon, " "))
     ;  // Remove leading spaces.
   for (const auto& field : multi_line_fields) {
     if (up_to_colon == field) {
@@ -119,9 +120,9 @@ static bool ConvertLine(StringPiece line,
   StringPiece up_to_colon;
   StringPiece after_colon = line;
   SplitAt(':', &after_colon, &up_to_colon);
-  while (after_colon.Consume(" "))
+  while (str_util::ConsumePrefix(&after_colon, " "))
     ;  // Remove leading spaces.
-  if (!after_colon.Consume("\"")) {
+  if (!str_util::ConsumePrefix(&after_colon, "\"")) {
     // We only convert string fields, so don't convert this line.
     return false;
   }
@@ -181,9 +182,9 @@ string PBTxtToMultiline(StringPiece pbtxt,
 static bool FindMultiline(StringPiece line, size_t colon, string* end) {
   if (colon == StringPiece::npos) return false;
   line.remove_prefix(colon + 1);
-  while (line.Consume(" ")) {
+  while (str_util::ConsumePrefix(&line, " ")) {
   }
-  if (line.Consume("<<")) {
+  if (str_util::ConsumePrefix(&line, "<<")) {
     *end = line.ToString();
     return true;
   }
@@ -228,7 +229,7 @@ string PBTxtFromMultiline(StringPiece multiline_pbtxt) {
     string suffix;
     while (!multiline_pbtxt.empty()) {
       SplitAt('\n', &multiline_pbtxt, &line);
-      if (line.Consume(end)) break;
+      if (str_util::ConsumePrefix(&line, end)) break;
       if (first) {
         first = false;
       } else {
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 9ec1c213c3..6ba196cc34 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -96,7 +96,7 @@ OpKernel::OpKernel(OpKernelConstruction* context,
       output_memory_types_(context->output_memory_types().begin(),
                            context->output_memory_types().end()),
       graph_def_version_(context->graph_def_version()),
-      is_internal_(StringPiece(type_string()).starts_with("_")),
+      is_internal_(str_util::StartsWith(type_string(), "_")),
       input_name_map_(context->num_inputs()),
       output_name_map_(context->num_outputs()) {
   OP_REQUIRES_OK(context,
diff --git a/tensorflow/core/framework/op_kernel_test.cc b/tensorflow/core/framework/op_kernel_test.cc
index b53b877f28..bcd409e5c5 100644
--- a/tensorflow/core/framework/op_kernel_test.cc
+++ b/tensorflow/core/framework/op_kernel_test.cc
@@ -546,9 +546,9 @@ TEST_F(OpKernelBuilderTest, BuilderTypeListAttr) {
                                             {"T|list(type)|[DT_FLOAT]"}));
 
   ExpectFailure("BuildTypeListAttr", DEVICE_CPU, {}, error::INVALID_ARGUMENT);
-  EXPECT_TRUE(
-      StringPiece(GetKernelClassName("BuildTypeListAttr", DEVICE_CPU, {}))
-          .contains("Invalid argument: "));
+  EXPECT_TRUE(str_util::StrContains(
+      GetKernelClassName("BuildTypeListAttr", DEVICE_CPU, {}),
+      "Invalid argument: "));
 
   ExpectFailure("BuildTypeListAttr", DEVICE_CPU, {"T|int|7"},
                 error::INVALID_ARGUMENT);
@@ -565,8 +565,8 @@ TEST_F(OpKernelBuilderTest, DuplicateKernel) {
   DeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Multiple OpKernel registrations match NodeDef"));
+  EXPECT_TRUE(str_util::StrContains(
+      status.error_message(), "Multiple OpKernel registrations match NodeDef"));
 
   ExpectFailure("DuplicateKernel", DEVICE_CPU, {}, error::INVALID_ARGUMENT);
 }
@@ -585,8 +585,8 @@ TEST_F(OpKernelBuilderTest, DuplicateKernelForT) {
   DeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("Multiple OpKernel registrations match NodeDef"));
+  EXPECT_TRUE(str_util::StrContains(
+      status.error_message(), "Multiple OpKernel registrations match NodeDef"));
 
   ExpectFailure("DuplicateKernelForT", DEVICE_CPU, {"T|type|DT_FLOAT"},
                 error::INVALID_ARGUMENT);
@@ -606,8 +606,9 @@ TEST_F(OpKernelBuilderTest, BadConstraint) {
   DeviceTypeVector devs;
   Status status = SupportedDeviceTypesForNode(DeviceTypes(), ndef, &devs);
   ASSERT_FALSE(status.ok());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("OpKernel 'BadConstraint' has constraint on attr "
+  EXPECT_TRUE(
+      str_util::StrContains(status.error_message(),
+                            "OpKernel 'BadConstraint' has constraint on attr "
                             "'T' not in NodeDef"));
 
   ExpectFailure("BadConstraint", DEVICE_CPU, {"dtype|type|DT_FLOAT"},
diff --git a/tensorflow/core/framework/resource_mgr_test.cc b/tensorflow/core/framework/resource_mgr_test.cc
index 07272e2374..798220d4c3 100644
--- a/tensorflow/core/framework/resource_mgr_test.cc
+++ b/tensorflow/core/framework/resource_mgr_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -71,7 +72,7 @@ string LookupOrCreate(ResourceMgr* rm, const string& container,
 }
 
 static void HasError(const Status& s, const string& substr) {
-  EXPECT_TRUE(StringPiece(s.ToString()).contains(substr))
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), substr))
       << s << ", expected substring " << substr;
 }
 
diff --git a/tensorflow/core/framework/shape_inference_test.cc b/tensorflow/core/framework/shape_inference_test.cc
index f48a7b9c47..da103bfec9 100644
--- a/tensorflow/core/framework/shape_inference_test.cc
+++ b/tensorflow/core/framework/shape_inference_test.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -152,10 +153,9 @@ TEST_F(ShapeInferenceTest, Run) {
     };
     Status s = c.Run(fn);
     // Extra error message is attached when Run fails.
-    EXPECT_TRUE(StringPiece(s.ToString())
-                    .contains("Shape must be at most rank 0 but "
-                              "is rank 1 for 'foo' (op: "
-                              "'foo_op')"))
+    EXPECT_TRUE(str_util::StrContains(
+        s.ToString(),
+        "Shape must be at most rank 0 but is rank 1 for 'foo' (op: 'foo_op')"))
         << s;
   }
 }
@@ -367,10 +367,9 @@ TEST_F(ShapeInferenceTest, WithRankAtMost) {
 
   // WithRankAtMost on shape with known dimensionality.
   s1 = in1;
-  EXPECT_TRUE(
-      StringPiece(c.WithRankAtMost(in1, 2, &s1).ToString())
-          .contains(
-              "Invalid argument: Shape must be at most rank 2 but is rank 3"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.WithRankAtMost(in1, 2, &s1).ToString(),
+      "Invalid argument: Shape must be at most rank 2 but is rank 3"));
 
   EXPECT_FALSE(IsSet(s1));
   EXPECT_TRUE(c.WithRankAtMost(in1, 3, &s1).ok());
@@ -406,10 +405,9 @@ TEST_F(ShapeInferenceTest, WithRankAtLeast) {
 
   // WithRankAtLeast on shape with known dimensionality.
   s1 = in1;
-  EXPECT_TRUE(
-      StringPiece(c.WithRankAtLeast(in1, 4, &s1).ToString())
-          .contains(
-              "Invalid argument: Shape must be at least rank 4 but is rank 3"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.WithRankAtLeast(in1, 4, &s1).ToString(),
+      "Invalid argument: Shape must be at least rank 4 but is rank 3"));
 
   EXPECT_FALSE(IsSet(s1));
   EXPECT_TRUE(c.WithRankAtLeast(in1, 3, &s1).ok());
@@ -449,12 +447,14 @@ TEST_F(ShapeInferenceTest, WithValue) {
   // WithValue on dimension with known size.
   out1 = d0;
 
-  EXPECT_TRUE(StringPiece(c.WithValue(d0, 0, &out1).ToString())
-                  .contains("Invalid argument: Dimension must be 0 but is 1"));
+  EXPECT_TRUE(
+      str_util::StrContains(c.WithValue(d0, 0, &out1).ToString(),
+                            "Invalid argument: Dimension must be 0 but is 1"));
   EXPECT_FALSE(IsSet(out1));
   out1 = d0;
-  EXPECT_TRUE(StringPiece(c.WithValue(d0, 2, &out1).ToString())
-                  .contains("Invalid argument: Dimension must be 2 but is 1"));
+  EXPECT_TRUE(
+      str_util::StrContains(c.WithValue(d0, 2, &out1).ToString(),
+                            "Invalid argument: Dimension must be 2 but is 1"));
 
   EXPECT_FALSE(IsSet(out1));
   EXPECT_TRUE(c.WithValue(d0, 1, &out1).ok());
@@ -513,16 +513,14 @@ TEST_F(ShapeInferenceTest, MergeDim) {
   EXPECT_EQ(3, merged_dims.size());
 
   // Merging unequal values is an error.
-  EXPECT_TRUE(
-      StringPiece(c.Merge(d2, d1, &out).ToString())
-          .contains(
-              "Invalid argument: Dimensions must be equal, but are 2 and 1"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Merge(d2, d1, &out).ToString(),
+      "Invalid argument: Dimensions must be equal, but are 2 and 1"));
 
   EXPECT_FALSE(IsSet(out));
-  EXPECT_TRUE(
-      StringPiece(c.Merge(d1, d2, &out).ToString())
-          .contains(
-              "Invalid argument: Dimensions must be equal, but are 1 and 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Merge(d1, d2, &out).ToString(),
+      "Invalid argument: Dimensions must be equal, but are 1 and 2"));
 
   EXPECT_FALSE(IsSet(out));
 
@@ -729,26 +727,23 @@ TEST_F(ShapeInferenceTest, MergeShape) {
 
   // Incompatible merges give errors and set out to nullptr.
   out = s_unknown;
-  EXPECT_TRUE(
-      StringPiece(c.Merge(s_u_2, s_1_3, &out).ToString())
-          .contains(
-              "Invalid argument: Dimension 1 in both shapes must be equal, but "
-              "are 2 and 3"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Merge(s_u_2, s_1_3, &out).ToString(),
+      "Invalid argument: Dimension 1 in both shapes must be equal, but "
+      "are 2 and 3"));
 
   EXPECT_FALSE(IsSet(out));
   out = s_unknown;
-  EXPECT_TRUE(
-      StringPiece(c.Merge(s_1_3, s_u_2, &out).ToString())
-          .contains(
-              "Invalid argument: Dimension 1 in both shapes must be equal, but "
-              "are 3 and 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Merge(s_1_3, s_u_2, &out).ToString(),
+      "Invalid argument: Dimension 1 in both shapes must be equal, but "
+      "are 3 and 2"));
 
   EXPECT_FALSE(IsSet(out));
   out = s_unknown;
-  EXPECT_TRUE(
-      StringPiece(c.Merge(s_1, s_1_2, &out).ToString())
-          .contains(
-              "Invalid argument: Shapes must be equal rank, but are 1 and 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Merge(s_1, s_1_2, &out).ToString(),
+      "Invalid argument: Shapes must be equal rank, but are 1 and 2"));
 
   EXPECT_FALSE(IsSet(out));
 
@@ -795,22 +790,18 @@ TEST_F(ShapeInferenceTest, MergePrefix) {
   // Incompatible merges give errors and set outs to nullptr.
   s_out = s_unknown;
   s_prefix_out = s_unknown;
-  EXPECT_TRUE(
-      StringPiece(
-          c.MergePrefix(s_1_u_3, s_2_4, &s_out, &s_prefix_out).ToString())
-          .contains(
-              "Invalid argument: Dimensions must be equal, but are 1 and 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.MergePrefix(s_1_u_3, s_2_4, &s_out, &s_prefix_out).ToString(),
+      "Invalid argument: Dimensions must be equal, but are 1 and 2"));
 
   EXPECT_FALSE(IsSet(s_out));
   EXPECT_FALSE(IsSet(s_prefix_out));
 
   s_out = s_unknown;
   s_prefix_out = s_unknown;
-  EXPECT_TRUE(
-      StringPiece(
-          c.MergePrefix(s_2_4, s_1_u_3, &s_out, &s_prefix_out).ToString())
-          .contains(
-              "Invalid argument: Shape must be at least rank 3 but is rank 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.MergePrefix(s_2_4, s_1_u_3, &s_out, &s_prefix_out).ToString(),
+      "Invalid argument: Shape must be at least rank 3 but is rank 2"));
   EXPECT_FALSE(IsSet(s_out));
   EXPECT_FALSE(IsSet(s_prefix_out));
 }
@@ -868,24 +859,21 @@ TEST_F(ShapeInferenceTest, Subshape) {
 
   // Errors.
   out = unknown;
-  EXPECT_TRUE(StringPiece(c.Subshape(in0, 6, -3, &out).ToString())
-                  .contains("Invalid argument: Subshape must have computed "
-                            "start <= end, but is 5 "
-                            "and 2 (computed from start 6 and end -3 over "
-                            "shape with rank 5)"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Subshape(in0, 6, -3, &out).ToString(),
+      "Invalid argument: Subshape must have computed start <= end, but is 5 "
+      "and 2 (computed from start 6 and end -3 over shape with rank 5)"));
   EXPECT_FALSE(IsSet(out));
   out = unknown;
-  EXPECT_TRUE(StringPiece(c.Subshape(in0, -50, 100, &out).ToString())
-                  .contains("Invalid argument: Subshape start out of "
-                            "bounds: -50, for shape with "
-                            "rank 5"));
+  EXPECT_TRUE(str_util::StrContains(c.Subshape(in0, -50, 100, &out).ToString(),
+                                    "Invalid argument: Subshape start out of "
+                                    "bounds: -50, for shape with rank 5"));
 
   EXPECT_FALSE(IsSet(out));
   out = unknown;
-  EXPECT_TRUE(StringPiece(c.Subshape(in0, 0, -50, &out).ToString())
-                  .contains("Invalid argument: Subshape end out of bounds: "
-                            "-50, for shape with rank "
-                            "5"));
+  EXPECT_TRUE(str_util::StrContains(c.Subshape(in0, 0, -50, &out).ToString(),
+                                    "Invalid argument: Subshape end out of "
+                                    "bounds: -50, for shape with rank 5"));
 
   EXPECT_FALSE(IsSet(out));
 }
@@ -1094,27 +1082,26 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeTensor) {
   EXPECT_EQ("[]", create(&t));
 
   t = ::tensorflow::test::AsTensor<float>({1, 2, 3});
-  EXPECT_TRUE(
-      StringPiece(create(&t))
-          .contains("Input tensor must be int32 or int64, but was float"));
+  EXPECT_TRUE(str_util::StrContains(
+      create(&t), "Input tensor must be int32 or int64, but was float"));
 
   t = ::tensorflow::test::AsScalar<int32>(1);
-  EXPECT_TRUE(StringPiece(create(&t))
-                  .contains("Input tensor must be rank 1, but was rank 0"));
+  EXPECT_TRUE(str_util::StrContains(
+      create(&t), "Input tensor must be rank 1, but was rank 0"));
 
   t = ::tensorflow::test::AsTensor<int32>({1, 2}, TensorShape{2, 1});
-  EXPECT_TRUE(StringPiece(create(&t))
-                  .contains("Input tensor must be rank 1, but was rank 2"));
+  EXPECT_TRUE(str_util::StrContains(
+      create(&t), "Input tensor must be rank 1, but was rank 2"));
 
   // Test negative values for the dims.
   t = ::tensorflow::test::AsTensor<int64>({3, -2, 1});
-  EXPECT_TRUE(StringPiece(create(&t))
-                  .contains("Invalid value in tensor used for shape: -2"));
+  EXPECT_TRUE(str_util::StrContains(
+      create(&t), "Invalid value in tensor used for shape: -2"));
 
   // Test negative values for the dims.
   t = ::tensorflow::test::AsTensor<int32>({3, -2, 1});
-  EXPECT_TRUE(StringPiece(create(&t))
-                  .contains("Invalid value in tensor used for shape: -2"));
+  EXPECT_TRUE(str_util::StrContains(
+      create(&t), "Invalid value in tensor used for shape: -2"));
 
   // Test when the input shape is wrong.
   {
@@ -1172,9 +1159,9 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeProto) {
   EXPECT_TRUE(c.MakeShapeFromShapeProto(proto, &out).ok());
   EXPECT_EQ("?", c.DebugString(out));
   proto.add_dim()->set_size(0);
-  EXPECT_TRUE(
-      StringPiece(c.MakeShapeFromShapeProto(proto, &out).error_message())
-          .contains("An unknown shape must not have any dimensions set."));
+  EXPECT_TRUE(str_util::StrContains(
+      c.MakeShapeFromShapeProto(proto, &out).error_message(),
+      "An unknown shape must not have any dimensions set."));
   EXPECT_FALSE(IsSet(out));
 
   // With known rank.
@@ -1188,10 +1175,10 @@ TEST_F(ShapeInferenceTest, MakeShapeFromShapeProto) {
 
   // With invalid dimension value.
   proto.add_dim()->set_size(-2);
-  EXPECT_TRUE(
-      StringPiece(c.MakeShapeFromShapeProto(proto, &out).error_message())
-          .contains("Shape [0,?,1000,-2] has dimensions with values below -1 "
-                    "(where -1 means unknown)"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.MakeShapeFromShapeProto(proto, &out).error_message(),
+      "Shape [0,?,1000,-2] has dimensions with values below -1 "
+      "(where -1 means unknown)"));
 
   EXPECT_FALSE(IsSet(out));
 }
@@ -1257,9 +1244,10 @@ TEST_F(ShapeInferenceTest, MakeDimForScalarInput) {
   EXPECT_TRUE(c.MakeDimForScalarInput(0, &d).ok());
   EXPECT_EQ("20", c.DebugString(d));
 
-  EXPECT_TRUE(StringPiece(c.MakeDimForScalarInput(1, &d).error_message())
-                  .contains("Dimension size, given by scalar input 1, must "
-                            "be non-negative but is -1"));
+  EXPECT_TRUE(
+      str_util::StrContains(c.MakeDimForScalarInput(1, &d).error_message(),
+                            "Dimension size, given by scalar input 1, must be "
+                            "non-negative but is -1"));
 
   // Same tests, with int64 values.
   t1 = tensorflow::test::AsScalar<int64>(20);
@@ -1267,9 +1255,10 @@ TEST_F(ShapeInferenceTest, MakeDimForScalarInput) {
   EXPECT_TRUE(c.MakeDimForScalarInput(0, &d).ok());
   EXPECT_EQ("20", c.DebugString(d));
 
-  EXPECT_TRUE(StringPiece(c.MakeDimForScalarInput(1, &d).error_message())
-                  .contains("Dimension size, given by scalar input 1, must "
-                            "be non-negative but is -1"));
+  EXPECT_TRUE(
+      str_util::StrContains(c.MakeDimForScalarInput(1, &d).error_message(),
+                            "Dimension size, given by scalar input 1, must be "
+                            "non-negative but is -1"));
 }
 
 TEST_F(ShapeInferenceTest, GetAttr) {
@@ -1322,33 +1311,33 @@ TEST_F(ShapeInferenceTest, Divide) {
   EXPECT_TRUE(c.Divide(d_6, d_2, evenly_divisible, &out).ok());
   EXPECT_EQ("3", c.DebugString(out));
 
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, 5, evenly_divisible, &out).error_message())
-          .contains("Dimension size must be evenly divisible by 5 but is 6"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, 5, evenly_divisible, &out).error_message(),
+      "Dimension size must be evenly divisible by 5 but is 6"));
 
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, 0, evenly_divisible, &out).error_message())
-          .contains("Divisor must be positive but is 0"));
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, d_0, evenly_divisible, &out).error_message())
-          .contains("Divisor must be positive but is 0"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, 0, evenly_divisible, &out).error_message(),
+      "Divisor must be positive but is 0"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, d_0, evenly_divisible, &out).error_message(),
+      "Divisor must be positive but is 0"));
 
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, -1, evenly_divisible, &out).error_message())
-          .contains("Divisor must be positive but is -1"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, -1, evenly_divisible, &out).error_message(),
+      "Divisor must be positive but is -1"));
 
   // Repeat error cases above with evenly_divisible=false.
   evenly_divisible = false;
   EXPECT_TRUE(c.Divide(d_6, 5, evenly_divisible, &out).ok());
   EXPECT_EQ("1", c.DebugString(out));
 
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, 0, evenly_divisible, &out).error_message())
-          .contains("Divisor must be positive but is 0"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, 0, evenly_divisible, &out).error_message(),
+      "Divisor must be positive but is 0"));
 
-  EXPECT_TRUE(
-      StringPiece(c.Divide(d_6, -1, evenly_divisible, &out).error_message())
-          .contains("Divisor must be positive but is -1"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Divide(d_6, -1, evenly_divisible, &out).error_message(),
+      "Divisor must be positive but is -1"));
 }
 
 TEST_F(ShapeInferenceTest, Add) {
@@ -1396,11 +1385,9 @@ TEST_F(ShapeInferenceTest, Add) {
   EXPECT_TRUE(c.Add(d_0, d_6, &out).ok());
   EXPECT_TRUE(SameHandle(out, d_6));
 
-  EXPECT_TRUE(
-      StringPiece(c.Add(d_6, std::numeric_limits<int64>::max() - 5, &out)
-                      .error_message())
-          .contains(
-              "Dimension size overflow from adding 6 and 9223372036854775802"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Add(d_6, std::numeric_limits<int64>::max() - 5, &out).error_message(),
+      "Dimension size overflow from adding 6 and 9223372036854775802"));
 }
 
 TEST_F(ShapeInferenceTest, Subtract) {
@@ -1448,9 +1435,9 @@ TEST_F(ShapeInferenceTest, Subtract) {
   EXPECT_TRUE(c.Subtract(d_6, d_0, &out).ok());
   EXPECT_TRUE(SameHandle(out, d_6));
 
-  EXPECT_TRUE(
-      StringPiece(c.Subtract(d_5, d_6, &out).error_message())
-          .contains("Negative dimension size caused by subtracting 6 from 5"));
+  EXPECT_TRUE(str_util::StrContains(
+      c.Subtract(d_5, d_6, &out).error_message(),
+      "Negative dimension size caused by subtracting 6 from 5"));
 }
 
 TEST_F(ShapeInferenceTest, Multiply) {
diff --git a/tensorflow/core/framework/shape_inference_testutil.cc b/tensorflow/core/framework/shape_inference_testutil.cc
index b4765ab0b2..b54dd220ab 100644
--- a/tensorflow/core/framework/shape_inference_testutil.cc
+++ b/tensorflow/core/framework/shape_inference_testutil.cc
@@ -100,7 +100,7 @@ Status ShapeInferenceTestutil::InferShapes(ShapeInferenceTestOp op,
       }
     }
 
-    if (expected.starts_with("in")) {
+    if (str_util::StartsWith(expected, "in")) {
       if (in_index == -1) {
         return Unknown(err_prefix,
                        " should have matched an input shape by "
@@ -135,7 +135,9 @@ Status ShapeInferenceTestutil::InferShapes(ShapeInferenceTestOp op,
     }
 
     // Verify the dimensions.
-    CHECK(expected.starts_with("[") && expected.ends_with("]")) << expected;
+    CHECK(str_util::StartsWith(expected, "[") &&
+          str_util::EndsWith(expected, "]"))
+        << expected;
     expected.remove_prefix(1);
     expected.remove_suffix(1);
 
@@ -176,7 +178,7 @@ Status ShapeInferenceTestutil::InferShapes(ShapeInferenceTestOp op,
           return Unknown(err_prefix, " expected to be unknown but was ",
                          c.Value(out_dim), err_suffix);
         }
-      } else if (expected_dim.starts_with("d")) {
+      } else if (str_util::StartsWith(expected_dim, "d")) {
         // Compare the dimension values.
         auto v = str_util::Split(expected_dim, '|');
         if (in_dim_idx.first == -1) {
diff --git a/tensorflow/core/framework/shape_inference_testutil.h b/tensorflow/core/framework/shape_inference_testutil.h
index 7977841482..2a99af7659 100644
--- a/tensorflow/core/framework/shape_inference_testutil.h
+++ b/tensorflow/core/framework/shape_inference_testutil.h
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/shape_inference.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/core/public/version.h"
 
@@ -83,17 +84,17 @@ class ShapeInferenceTestutil {
       "", ::tensorflow::shape_inference::ShapeInferenceTestutil::InferShapes( \
               op, i, o)                                                       \
               .error_message())
-#define INFER_ERROR(error_substring, op, i)                                 \
-  {                                                                         \
-    string error_message =                                                  \
-        ::tensorflow::shape_inference::ShapeInferenceTestutil::InferShapes( \
-            op, i, "e")                                                     \
-            .error_message();                                               \
-    const string& substring = error_substring;                              \
-    EXPECT_NE("", error_message);                                           \
-    EXPECT_TRUE(StringPiece(error_message).contains(substring))             \
-        << "Expected to see '" << substring << "' in '" << error_message    \
-        << "'";                                                             \
+#define INFER_ERROR(error_substring, op, i)                                    \
+  {                                                                            \
+    string error_message =                                                     \
+        ::tensorflow::shape_inference::ShapeInferenceTestutil::InferShapes(    \
+            op, i, "e")                                                        \
+            .error_message();                                                  \
+    const string& substring = error_substring;                                 \
+    EXPECT_NE("", error_message);                                              \
+    EXPECT_TRUE(::tensorflow::str_util::StrContains(error_message, substring)) \
+        << "Expected to see '" << substring << "' in '" << error_message       \
+        << "'";                                                                \
   }
 
 }  // namespace tensorflow
diff --git a/tensorflow/core/framework/shape_inference_testutil_test.cc b/tensorflow/core/framework/shape_inference_testutil_test.cc
index 20a6807064..a4405b502c 100644
--- a/tensorflow/core/framework/shape_inference_testutil_test.cc
+++ b/tensorflow/core/framework/shape_inference_testutil_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def_builder.h"
 #include "tensorflow/core/framework/op.h"
 #include "tensorflow/core/framework/shape_inference.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -25,10 +26,11 @@ namespace shape_inference {
 
 namespace {
 
-#define EXPECT_CONTAINS(str, substr)                                 \
-  do {                                                               \
-    string s = (str);                                                \
-    EXPECT_TRUE(StringPiece(s).contains(substr)) << "String: " << s; \
+#define EXPECT_CONTAINS(str, substr)                            \
+  do {                                                          \
+    string s = (str);                                           \
+    EXPECT_TRUE(::tensorflow::str_util::StrContains(s, substr)) \
+        << "String: " << s;                                     \
   } while (false)
 
 static OpShapeInferenceFn* global_fn_ptr = nullptr;
@@ -97,8 +99,8 @@ TEST(ShapeInferenceTestutilTest, Failures) {
   auto error_message = ShapeInferenceTestutil::InferShapes(
                            ShapeInferenceTestOp("NoSuchOp"), "", "")
                            .error_message();
-  EXPECT_TRUE(StringPiece(error_message)
-                  .starts_with("Op type not registered 'NoSuchOp'"));
+  EXPECT_TRUE(
+      str_util::StartsWith(error_message, "Op type not registered 'NoSuchOp'"));
 
   // Wrong shape error messages.
   EXPECT_CONTAINS(RunInferShapes(op, "[1];[2];[1]", "?", fn_copy_input_0),
diff --git a/tensorflow/core/framework/types.cc b/tensorflow/core/framework/types.cc
index adf4e1bae3..2280114de5 100644
--- a/tensorflow/core/framework/types.cc
+++ b/tensorflow/core/framework/types.cc
@@ -114,7 +114,7 @@ string DataTypeString(DataType dtype) {
 }
 
 bool DataTypeFromString(StringPiece sp, DataType* dt) {
-  if (sp.ends_with("_ref")) {
+  if (str_util::EndsWith(sp, "_ref")) {
     sp.remove_suffix(4);
     DataType non_ref;
     if (DataTypeFromString(sp, &non_ref) && !IsRefType(non_ref)) {
diff --git a/tensorflow/core/framework/types_test.cc b/tensorflow/core/framework/types_test.cc
index 60f2b4135a..16b069c70a 100644
--- a/tensorflow/core/framework/types_test.cc
+++ b/tensorflow/core/framework/types_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 
 #include "tensorflow/core/framework/type_traits.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -140,9 +141,8 @@ TEST(TypesTest, ComplexTypes) {
 TEST(TypesTest, IntegerTypes) {
   for (auto dt : AllTypes()) {
     const string name = DataTypeString(dt);
-    const StringPiece n = name;
-    EXPECT_EQ(DataTypeIsInteger(dt),
-              n.starts_with("int") || n.starts_with("uint"))
+    EXPECT_EQ(DataTypeIsInteger(dt), str_util::StartsWith(name, "int") ||
+                                         str_util::StartsWith(name, "uint"))
         << "DataTypeInteger failed for " << name;
   }
 }
diff --git a/tensorflow/core/framework/variant_op_copy_test.cc b/tensorflow/core/framework/variant_op_copy_test.cc
index 85e014f804..60fa7bd559 100644
--- a/tensorflow/core/framework/variant_op_copy_test.cc
+++ b/tensorflow/core/framework/variant_op_copy_test.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/framework/variant_tensor_data.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/util/port.h"
 
@@ -259,8 +260,8 @@ TEST(VariantOpCopyTest, CreateConstOnGPUFailsGracefully) {
   ClientSession session(root);
   std::vector<Tensor> outputs;
   Status s = session.Run({create_const}, &outputs);
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("GPU copy from non-DMA string tensor"))
+  EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                    "GPU copy from non-DMA string tensor"))
       << s.ToString();
 }
 
@@ -365,8 +366,9 @@ TEST(VariantOpCopyTest, CreateCopyCPUToGPUStringFailsSafely) {
   std::vector<Tensor> outputs;
   Status err = session.Run({create_op, identity}, &outputs);
   EXPECT_EQ(err.code(), errors::Code::INVALID_ARGUMENT);
-  EXPECT_TRUE(StringPiece(err.error_message())
-                  .contains("During Variant Host->Device Copy: non-DMA-copy "
+  EXPECT_TRUE(
+      str_util::StrContains(err.error_message(),
+                            "During Variant Host->Device Copy: non-DMA-copy "
                             "attempted of tensor type: string"))
       << err.error_message();
 }
diff --git a/tensorflow/core/framework/variant_op_registry_test.cc b/tensorflow/core/framework/variant_op_registry_test.cc
index 06ca211c76..7055e62c0e 100644
--- a/tensorflow/core/framework/variant_op_registry_test.cc
+++ b/tensorflow/core/framework/variant_op_registry_test.cc
@@ -14,6 +14,7 @@ limitations under the License.
 ==============================================================================*/
 
 #include <memory>
+#include "tensorflow/core/lib/strings/str_util.h"
 
 #define EIGEN_USE_THREADS
 
@@ -130,7 +131,7 @@ TEST(VariantOpShapeRegistryTest, TestBasic) {
   Variant v = vv_early_exit;
   Status s0 = (*shape_fn)(v, &shape);
   EXPECT_FALSE(s0.ok());
-  EXPECT_TRUE(StringPiece(s0.error_message()).contains("early exit!"));
+  EXPECT_TRUE(str_util::StrContains(s0.error_message(), "early exit!"));
 
   VariantValue vv_ok{false /* early_exit */};
   v = vv_ok;
@@ -229,7 +230,7 @@ TEST(VariantOpZerosLikeRegistryTest, TestBasicCPU) {
                                         ZEROS_LIKE_VARIANT_UNARY_OP, v, &v_out);
   EXPECT_FALSE(s0.ok());
   EXPECT_TRUE(
-      StringPiece(s0.error_message()).contains("early exit zeros_like"));
+      str_util::StrContains(s0.error_message(), "early exit zeros_like"));
 
   VariantValue vv_ok{false /* early_exit */, 0 /* value */};
   v = vv_ok;
@@ -254,7 +255,7 @@ TEST(VariantOpUnaryOpRegistryTest, TestBasicGPU) {
                                         ZEROS_LIKE_VARIANT_UNARY_OP, v, &v_out);
   EXPECT_FALSE(s0.ok());
   EXPECT_TRUE(
-      StringPiece(s0.error_message()).contains("early exit zeros_like"));
+      str_util::StrContains(s0.error_message(), "early exit zeros_like"));
 
   VariantValue vv_ok{false /* early_exit */, 0 /* value */};
   v = vv_ok;
@@ -299,7 +300,7 @@ TEST(VariantOpAddRegistryTest, TestBasicCPU) {
   Status s0 = BinaryOpVariants<CPUDevice>(
       null_context_pointer, ADD_VARIANT_BINARY_OP, v_a, v_b, &v_out);
   EXPECT_FALSE(s0.ok());
-  EXPECT_TRUE(StringPiece(s0.error_message()).contains("early exit add"));
+  EXPECT_TRUE(str_util::StrContains(s0.error_message(), "early exit add"));
 
   VariantValue vv_ok{false /* early_exit */, 3 /* value */};
   v_a = vv_ok;
@@ -325,7 +326,7 @@ TEST(VariantOpAddRegistryTest, TestBasicGPU) {
   Status s0 = BinaryOpVariants<GPUDevice>(
       null_context_pointer, ADD_VARIANT_BINARY_OP, v_a, v_b, &v_out);
   EXPECT_FALSE(s0.ok());
-  EXPECT_TRUE(StringPiece(s0.error_message()).contains("early exit add"));
+  EXPECT_TRUE(str_util::StrContains(s0.error_message(), "early exit add"));
 
   VariantValue vv_ok{false /* early_exit */, 3 /* value */};
   v_a = vv_ok;
-- 
GitLab


From 5617c9b4be759b62f3cbd29b0a58c41a43ac47a0 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Mon, 2 Apr 2018 10:51:13 -0700
Subject: [PATCH 805/960] Improves the documentation of control_dependencies.

PiperOrigin-RevId: 191314766
---
 tensorflow/python/framework/ops.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 7ca0b836dd..22b621e4cb 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -4489,6 +4489,22 @@ class Graph(object):
         return tf.matmul(tensor, tensor)
     ```
 
+    Also note that though execution of ops created under this scope will trigger
+    execution of the dependencies, the ops created under this scope might still
+    be pruned from a normal tensorflow graph. For example, in the following
+    snippet of code the dependencies are never executed:
+
+    ```python
+      loss = model.loss()
+      with tf.control_dependencies(dependencies):
+        loss = loss + tf.constant(1)  # note: dependencies ignored in the
+                                      # backward pass
+      return tf.gradients(loss, model.variables)
+    ```
+
+    This is because evaluating the gradient graph does not require evaluating
+    the constant(1) op created in the forward pass.
+
     Args:
       control_inputs: A list of `Operation` or `Tensor` objects which
         must be executed or computed before running the operations
-- 
GitLab


From 4118b7c8432ca770191ca22fc01fd435e4e4571e Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 11:03:18 -0700
Subject: [PATCH 806/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 191316903
---
 .../common_runtime/direct_session_test.cc     |  25 ++--
 .../core/common_runtime/function_test.cc      |   9 +-
 .../function_threadpool_test.cc               |   3 +-
 tensorflow/core/common_runtime/placer.cc      |   4 +-
 tensorflow/core/common_runtime/placer_test.cc | 110 +++++++++---------
 .../process_function_library_runtime_test.cc  |   3 +-
 .../core/common_runtime/session_test.cc       |  16 +--
 .../core/common_runtime/shape_refiner_test.cc |   9 +-
 tensorflow/core/graph/graph_constructor.cc    |  15 +--
 .../core/graph/graph_constructor_test.cc      |   4 +-
 tensorflow/core/graph/graph_partition.cc      |   3 +-
 tensorflow/core/graph/graph_partition_test.cc |   5 +-
 tensorflow/core/graph/graph_test.cc           |   3 +-
 .../core/graph/quantize_training_test.cc      |   5 +-
 tensorflow/core/graph/subgraph_test.cc        |   4 +-
 tensorflow/core/graph/tensor_id.cc            |   3 +-
 tensorflow/core/graph/validate_test.cc        |   7 +-
 17 files changed, 122 insertions(+), 106 deletions(-)

diff --git a/tensorflow/core/common_runtime/direct_session_test.cc b/tensorflow/core/common_runtime/direct_session_test.cc
index ee38960618..f95cecfc66 100644
--- a/tensorflow/core/common_runtime/direct_session_test.cc
+++ b/tensorflow/core/common_runtime/direct_session_test.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
@@ -155,22 +156,22 @@ TEST_F(DirectSessionMinusAXTest, RunSimpleNetwork_Callable) {
 
     Status s = session->RunCallable(handle, {}, nullptr, nullptr);
     EXPECT_TRUE(errors::IsInvalidArgument(s));
-    EXPECT_TRUE(StringPiece(s.error_message())
-                    .contains("`fetch_tensors` must be provided"));
+    EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                      "`fetch_tensors` must be provided"));
 
     TF_ASSERT_OK(session->ReleaseCallable(handle));
 
     std::vector<Tensor> outputs;
     s = session->RunCallable(handle, {}, &outputs, nullptr);
     EXPECT_TRUE(errors::IsInvalidArgument(s));
-    EXPECT_TRUE(
-        StringPiece(s.error_message())
-            .contains("Attempted to run callable after handle was released"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.error_message(),
+        "Attempted to run callable after handle was released"));
 
     s = session->RunCallable(handle + 1, {}, &outputs, nullptr);
     EXPECT_TRUE(errors::IsInvalidArgument(s));
     EXPECT_TRUE(
-        StringPiece(s.error_message()).contains("No such callable handle"));
+        str_util::StrContains(s.error_message(), "No such callable handle"));
   }
 }
 
@@ -567,7 +568,7 @@ TEST(DirectSessionTest, MultipleFeedTest) {
       {first_identity->name() + ":0", second_identity->name() + ":0"}, {},
       &outputs);
   EXPECT_TRUE(errors::IsInvalidArgument(s));
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("fed more than once"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "fed more than once"));
 }
 
 TEST(DirectSessionTest, MultipleFeedTest_Callable) {
@@ -650,7 +651,7 @@ TEST(DirectSessionTest, MultipleFeedTest_Callable) {
           {first_identity->name() + ":0", second_identity->name() + ":0"}, {}),
       &handle);
   EXPECT_TRUE(errors::IsInvalidArgument(s));
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("fed more than once"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "fed more than once"));
 }
 
 TEST(DirectSessionTest, FetchMultipleTimes) {
@@ -845,8 +846,8 @@ TEST(DirectSessionTest, PartialRunMissingFeed) {
   s = session->PRun(handle, {{first_const->name(), value_11}},
                     {third_identity->name() + ":0"}, &outputs);
   ASSERT_TRUE(errors::IsInvalidArgument(s));
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("can't be computed from the feeds"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                    "can't be computed from the feeds"));
 }
 
 TEST(DirectSessionTest, PartialRunMultiOutputFeed) {
@@ -875,8 +876,8 @@ TEST(DirectSessionTest, PartialRunMultiOutputFeed) {
   // Fetch fourth_identity without feeds.
   s = session->PRun(handle, {}, {fourth_identity->name() + ":0"}, &outputs);
   ASSERT_TRUE(errors::IsInvalidArgument(s));
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("can't be computed from the feeds"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                    "can't be computed from the feeds"));
 
   // Feed switch_node:1 and fetch fourth_identity.
   s = session->PRun(handle, {{switch_node->name() + ":1", bool_value}},
diff --git a/tensorflow/core/common_runtime/function_test.cc b/tensorflow/core/common_runtime/function_test.cc
index d17ef4d459..61b2f0e60f 100644
--- a/tensorflow/core/common_runtime/function_test.cc
+++ b/tensorflow/core/common_runtime/function_test.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
@@ -53,8 +54,8 @@ Status GetOpSig(const string& op, const OpDef** sig) {
   return OpRegistry::Global()->LookUpOpDef(op, sig);
 }
 
-void HasError(const Status& s, const string& substr) {
-  EXPECT_TRUE(StringPiece(s.ToString()).contains(substr))
+void HasError(const Status& s, StringPiece substr) {
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), substr))
       << s << ", expected substring " << substr;
 }
 
@@ -240,7 +241,7 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     Status status2 = Run(flr, handle, opts, args, std::move(rets));
     EXPECT_TRUE(errors::IsInvalidArgument(status2));
     EXPECT_TRUE(
-        StringPiece(status2.error_message()).contains("remote execution."));
+        str_util::StrContains(status2.error_message(), "remote execution."));
 
     return status;
   }
@@ -310,7 +311,7 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     Status status2 = Run(flr, handle, opts, args, std::move(rets));
     EXPECT_TRUE(errors::IsInvalidArgument(status2));
     EXPECT_TRUE(
-        StringPiece(status2.error_message()).contains("remote execution."));
+        str_util::StrContains(status2.error_message(), "remote execution."));
 
     return status;
   }
diff --git a/tensorflow/core/common_runtime/function_threadpool_test.cc b/tensorflow/core/common_runtime/function_threadpool_test.cc
index 6223a4e648..2d09e83d01 100644
--- a/tensorflow/core/common_runtime/function_threadpool_test.cc
+++ b/tensorflow/core/common_runtime/function_threadpool_test.cc
@@ -39,6 +39,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
@@ -153,7 +154,7 @@ class FunctionLibraryRuntimeTest : public ::testing::Test {
     Status status2 = Run(flr, handle, opts, args, std::move(rets));
     EXPECT_TRUE(errors::IsInvalidArgument(status2));
     EXPECT_TRUE(
-        StringPiece(status2.error_message()).contains("remote execution."));
+        str_util::StrContains(status2.error_message(), "remote execution."));
 
     return status;
   }
diff --git a/tensorflow/core/common_runtime/placer.cc b/tensorflow/core/common_runtime/placer.cc
index e128b9257f..86851c2c07 100644
--- a/tensorflow/core/common_runtime/placer.cc
+++ b/tensorflow/core/common_runtime/placer.cc
@@ -29,6 +29,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 
@@ -151,7 +152,8 @@ class ColocationGraph {
       if (attr_value != nullptr && attr_value->has_list()) {
         for (const string& class_spec : attr_value->list().s()) {
           StringPiece spec(class_spec);
-          if (spec.Consume(kColocationGroupPrefixStringPiece)) {
+          if (str_util::ConsumePrefix(&spec,
+                                      kColocationGroupPrefixStringPiece)) {
             found_spec = true;
             TF_RETURN_IF_ERROR(
                 ColocateNodeToGroup(&colocation_group_root, node, spec));
diff --git a/tensorflow/core/common_runtime/placer_test.cc b/tensorflow/core/common_runtime/placer_test.cc
index 098024d219..5ad251c892 100644
--- a/tensorflow/core/common_runtime/placer_test.cc
+++ b/tensorflow/core/common_runtime/placer_test.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -262,9 +263,9 @@ class PlacerTest : public ::testing::Test {
                 ->attributes()                                          \
                 .device_type())
 
-#define EXPECT_DEVICE_CONTAINS(g, name, device_substr)                        \
-  EXPECT_TRUE(StringPiece(GetNodeByName((g), (name))->assigned_device_name()) \
-                  .contains(device_substr))
+#define EXPECT_DEVICE_CONTAINS(g, name, device_substr) \
+  EXPECT_TRUE(::tensorflow::str_util::StrContains(     \
+      GetNodeByName((g), (name))->assigned_device_name(), device_substr))
 
 // Test that a graph with no constraints will successfully assign nodes to the
 // "best available" device (i.e. prefer GPU over CPU).
@@ -488,11 +489,10 @@ TEST_F(PlacerTest, TestAssignedGpuDeviceToCpuDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INTERNAL, s.code());
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains(
-              "Assigned device '/job:a/replica:0/task:0/device:fakegpu:0' "
-              "does not have registered OpKernel support for TestInput"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "Assigned device '/job:a/replica:0/task:0/device:fakegpu:0' "
+      "does not have registered OpKernel support for TestInput"));
 }
 
 // Test that graphs with reference connections are correctly placed.
@@ -541,15 +541,15 @@ TEST_F(PlacerTest, TestReferenceConnection) {
   {
     Status s = ReferenceTestHelper("VariableCPU", "AssignGPU", "FakeCPU");
     EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-    EXPECT_TRUE(StringPiece(s.error_message())
-                    .contains("no device type supports both of those nodes"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.error_message(), "no device type supports both of those nodes"));
   }
   TF_EXPECT_OK(ReferenceTestHelper("VariableGPU", "TestAssign", "FakeGPU"));
   {
     Status s = ReferenceTestHelper("VariableGPU", "AssignCPU", "FakeCPU");
     EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-    EXPECT_TRUE(StringPiece(s.error_message())
-                    .contains("no device type supports both of those nodes"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.error_message(), "no device type supports both of those nodes"));
   }
   TF_EXPECT_OK(ReferenceTestHelper("VariableGPU", "AssignGPU", "FakeGPU"));
 }
@@ -760,8 +760,9 @@ TEST_F(PlacerTest, TestInvalidMultipleColocationGroups) {
   }
 
   Status s = Place(&g);
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("Cannot colocate nodes 'foo' and 'in' because no "
+  EXPECT_TRUE(
+      str_util::StrContains(s.error_message(),
+                            "Cannot colocate nodes 'foo' and 'in' because no "
                             "device type supports both of those nodes and the "
                             "other nodes colocated with them"));
 }
@@ -824,11 +825,11 @@ TEST_F(PlacerTest, TestColocationGroupWithUnsatisfiableReferenceConnections) {
   }
 
   Status s = Place(&g);
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains("Cannot colocate nodes 'var3' and 'assign3' because no "
-                    "device type supports both of those nodes and the other "
-                    "nodes colocated with them."));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "Cannot colocate nodes 'var3' and 'assign3' because no "
+      "device type supports both of those nodes and the other "
+      "nodes colocated with them."));
 }
 
 TEST_F(PlacerTest, TestColocationAndReferenceConnections) {
@@ -888,7 +889,7 @@ TEST_F(PlacerTest, TestEmptyDeviceSet) {
 
   Status s = Place(&g, &empty);
   EXPECT_TRUE(
-      StringPiece(s.error_message()).contains("No devices are registered"));
+      str_util::StrContains(s.error_message(), "No devices are registered"));
 }
 
 // Test that placement fails when the requested device forces an
@@ -913,16 +914,17 @@ TEST_F(PlacerTest, TestHeterogeneousDeviceSetFailure) {
   heterogeneous.AddDevice(cpu.get());
   Status s = Place(&g, &heterogeneous);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("colocated with a group of nodes that required "
+  EXPECT_TRUE(
+      str_util::StrContains(s.error_message(),
+                            "colocated with a group of nodes that required "
                             "incompatible device"));
 
   // The error message should contain information that indicates which
   // op types have which registered device types.
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("VariableGPU: FakeGPU"))
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "VariableGPU: FakeGPU"))
       << s;
   EXPECT_TRUE(
-      StringPiece(s.error_message()).contains("TestAssign: FakeGPU FakeCPU"))
+      str_util::StrContains(s.error_message(), "TestAssign: FakeGPU FakeCPU"))
       << s;
 }
 
@@ -937,7 +939,7 @@ TEST_F(PlacerTest, TestUnknownDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "/job:foo"));
 }
 
 // Test that placement fails when the combination of partial
@@ -952,7 +954,7 @@ TEST_F(PlacerTest, TestUnknownMergedDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("/job:foo"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "/job:foo"));
 }
 
 // Test that placement fails when the previously-assigned device for a
@@ -969,9 +971,9 @@ TEST_F(PlacerTest, TestUnknownAssignedDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INTERNAL, s.code());
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains("Assigned device '/job:foo' does not match any device"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "Assigned device '/job:foo' does not match any device"));
 }
 
 // Test that placement fails when an op with no registered kernels is
@@ -986,12 +988,11 @@ TEST_F(PlacerTest, TestNoKernelsRegistered) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "No OpKernel was registered to support Op 'VariableNoKernels'"));
   EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains(
-              "No OpKernel was registered to support Op 'VariableNoKernels'"));
-  EXPECT_TRUE(
-      StringPiece(s.error_message()).contains("<no registered kernels>"));
+      str_util::StrContains(s.error_message(), "<no registered kernels>"));
 }
 
 // Test that placement fails when a kernel is registered but no known
@@ -1011,10 +1012,10 @@ TEST_F(PlacerTest, TestNoDevicesRegistered) {
 
   Status s = Place(&g, &cpu_only);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("No OpKernel was registered to support "
-                            "Op 'VariableGPU'"));
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("device='FakeGPU'"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "No OpKernel was registered to support Op 'VariableGPU'"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "device='FakeGPU'"));
 }
 
 // Test that placement fails when a requested device is malformed.
@@ -1028,8 +1029,8 @@ TEST_F(PlacerTest, TestMalformedDeviceSpecification) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("Malformed device specification '/foo:bar'"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(), "Malformed device specification '/foo:bar'"));
 }
 
 // Test that placement fails when a previously-assigned device is malformed.
@@ -1045,8 +1046,8 @@ TEST_F(PlacerTest, TestMalformedAssignedDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INTERNAL, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("Malformed assigned device '/foo:bar'"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(),
+                                    "Malformed assigned device '/foo:bar'"));
 }
 
 // Test that placement fails when a device was previously assigned to
@@ -1063,9 +1064,8 @@ TEST_F(PlacerTest, TestNonUniqueAssignedDevice) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INTERNAL, s.code());
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains("Assigned device '/job:a' does not match any device"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(), "Assigned device '/job:a' does not match any device"));
 }
 
 // Test that ops request to be placed on non-existent devices will be relocated
@@ -1099,7 +1099,7 @@ TEST_F(PlacerTest, TestNonexistentGpuNoAllowSoftPlacement) {
   SessionOptions options;
   Status s = Place(&g, &options);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakegpu:11"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "/device:fakegpu:11"));
 }
 
 // Test that placement fails when a node requests an explicit device that is not
@@ -1116,10 +1116,10 @@ TEST_F(PlacerTest, TestUnsupportedDeviceNoAllowSoftPlacement) {
   SessionOptions options;
   Status s = Place(&g, &options);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("/device:fakecpu:0"));
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains("no supported kernel for fakecpu devices is available"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "/device:fakecpu:0"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "no supported kernel for fakecpu devices is available"));
 }
 
 // Test that placement fails when a node requests an explicit device that is not
@@ -1137,9 +1137,9 @@ TEST_F(PlacerTest, TestNonExistentDevice) {
   Status s = Place(&g, &options);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
   LOG(WARNING) << s.error_message();
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("was explicitly assigned to /job:foo/replica:17 "
-                            "but available devices"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "was explicitly assigned to /job:foo/replica:17 but available devices"));
 }
 
 TEST_F(PlacerTest, TestUnsupportedDeviceAllowSoftPlacement) {
@@ -1205,8 +1205,8 @@ TEST_F(PlacerTest, TestUnsatisfiableConstraintWithReferenceConnections) {
 
   Status s = Place(&g);
   EXPECT_EQ(error::INVALID_ARGUMENT, s.code());
-  EXPECT_TRUE(StringPiece(s.error_message())
-                  .contains("Cannot colocate nodes 'var' and 'assign'"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(), "Cannot colocate nodes 'var' and 'assign'"));
 }
 
 // Test that a generator node follows its consumers (where there are several
diff --git a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
index 2da67b084a..4fbf2abc67 100644
--- a/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
+++ b/tensorflow/core/common_runtime/process_function_library_runtime_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session_options.h"
 #include "tensorflow/core/public/version.h"
@@ -132,7 +133,7 @@ class ProcessFunctionLibraryRuntimeTest : public ::testing::Test {
                    });
     done2.WaitForNotification();
     EXPECT_TRUE(errors::IsNotFound(status));
-    EXPECT_TRUE(StringPiece(status.error_message()).contains("not found."));
+    EXPECT_TRUE(str_util::StrContains(status.error_message(), "not found."));
 
     return Status::OK();
   }
diff --git a/tensorflow/core/common_runtime/session_test.cc b/tensorflow/core/common_runtime/session_test.cc
index a074154450..feaf29c7bb 100644
--- a/tensorflow/core/common_runtime/session_test.cc
+++ b/tensorflow/core/common_runtime/session_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/public/session.h"
 
 #include "tensorflow/core/common_runtime/session_factory.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session_options.h"
 
@@ -31,10 +32,9 @@ TEST(SessionTest, InvalidTargetReturnsNull) {
   Session* session;
   Status s = tensorflow::NewSession(options, &session);
   EXPECT_EQ(s.code(), error::NOT_FOUND);
-  EXPECT_TRUE(
-      StringPiece(s.error_message())
-          .contains(
-              "No session factory registered for the given session options"));
+  EXPECT_TRUE(str_util::StrContains(
+      s.error_message(),
+      "No session factory registered for the given session options"));
 }
 
 // Register a fake session factory to test error handling paths in
@@ -44,7 +44,7 @@ class FakeSessionFactory : public SessionFactory {
   FakeSessionFactory() {}
 
   bool AcceptsOptions(const SessionOptions& options) override {
-    return StringPiece(options.target).starts_with("fake");
+    return str_util::StartsWith(options.target, "fake");
   }
 
   Session* NewSession(const SessionOptions& options) override {
@@ -68,9 +68,9 @@ TEST(SessionTest, MultipleFactoriesForTarget) {
   Status s = tensorflow::NewSession(options, &session);
   EXPECT_EQ(s.code(), error::INTERNAL);
   EXPECT_TRUE(
-      StringPiece(s.error_message()).contains("Multiple session factories"));
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("FAKE_SESSION_1"));
-  EXPECT_TRUE(StringPiece(s.error_message()).contains("FAKE_SESSION_2"));
+      str_util::StrContains(s.error_message(), "Multiple session factories"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "FAKE_SESSION_1"));
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), "FAKE_SESSION_2"));
 }
 
 }  // namespace
diff --git a/tensorflow/core/common_runtime/shape_refiner_test.cc b/tensorflow/core/common_runtime/shape_refiner_test.cc
index adf5a9afff..f48638afc0 100644
--- a/tensorflow/core/common_runtime/shape_refiner_test.cc
+++ b/tensorflow/core/common_runtime/shape_refiner_test.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/graph/testlib.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/version.h"
 
@@ -143,8 +144,8 @@ TEST_F(ShapeRefinerTest, BadShapes) {
   // an error.
   Status s = m.AddNode(mm.node());
   ASSERT_FALSE(s.ok());
-  ASSERT_TRUE(StringPiece(s.error_message())
-                  .contains("Dimensions must be equal, but are 1 and 2"));
+  ASSERT_TRUE(str_util::StrContains(
+      s.error_message(), "Dimensions must be equal, but are 1 and 2"));
 }
 
 TEST_F(ShapeRefinerTest, SetShape) {
@@ -1032,8 +1033,8 @@ TEST_F(ShapeRefinerTest, ConstantValueAsShape_PackInvalidInput) {
     TF_ASSERT_OK(m.AddNode(input.node()));
   }
   TF_ASSERT_OK(m.AddNode(pack.node()));
-  EXPECT_TRUE(
-      StringPiece(m.AddNode(result).error_message()).contains("but is rank 2"));
+  EXPECT_TRUE(str_util::StrContains(m.AddNode(result).error_message(),
+                                    "but is rank 2"));
 }
 
 TEST_F(ShapeRefinerTest, ConstantValueAsShape_Concat) {
diff --git a/tensorflow/core/graph/graph_constructor.cc b/tensorflow/core/graph/graph_constructor.cc
index 76ee88e684..f15e2ce9fa 100644
--- a/tensorflow/core/graph/graph_constructor.cc
+++ b/tensorflow/core/graph/graph_constructor.cc
@@ -37,6 +37,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/strings/scanner.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/public/version.h"
 
@@ -73,7 +74,7 @@ class GraphConstructor {
     Options(const ImportGraphDefOptions& in)  // NOLINT(runtime/explicit)
         : allow_internal_ops(false),
           expect_device_spec(false),
-          prefix(in.prefix.empty() || StringPiece(in.prefix).ends_with("/")
+          prefix(in.prefix.empty() || str_util::EndsWith(in.prefix, "/")
                      ? in.prefix
                      : in.prefix + "/"),
           uniquify_names(in.uniquify_names),
@@ -436,7 +437,7 @@ Status GraphConstructor::BuildNodeIndex() {
     bool in_control_dependence = false;
     for (int i = 0; i < node_def.input_size(); ++i) {
       StringPiece input_name = node_def.input(i);
-      if (!input_name.empty() && input_name.starts_with("^")) {
+      if (!input_name.empty() && str_util::StartsWith(input_name, "^")) {
         in_control_dependence = true;
       } else if (in_control_dependence) {
         return errors::InvalidArgument(
@@ -484,7 +485,7 @@ Status GraphConstructor::InitFromEdges() {
       bool has_loop_back_edge = false;
       for (int i = 0; i < node_def.input_size(); ++i) {
         StringPiece input_name(node_def.input(i));
-        if (input_name.starts_with("^")) {
+        if (str_util::StartsWith(input_name, "^")) {
           num_control_edges++;
         } else {
           TensorId id(ParseTensorName(input_name));
@@ -534,7 +535,7 @@ Status GraphConstructor::ValidateColocationConstraints(
   if (iter == node_def.attr().end()) return Status::OK();
   for (const string& c : iter->second.list().s()) {
     StringPiece s(c);
-    if (s.Consume(kColocationGroupPrefix) &&
+    if (str_util::ConsumePrefix(&s, kColocationGroupPrefix) &&
         gdef_nodes_.find(s) == gdef_nodes_.end()) {
       return errors::InvalidArgument(
           "Node '", node_def.name(),
@@ -764,7 +765,7 @@ void GraphConstructor::AddPrefixToNodeDef(
     // Skip remapped inputs (which already exist in g_ and are not being
     // imported).
     if (input_already_exists[i]) continue;
-    if (input.Consume("^")) {
+    if (str_util::ConsumePrefix(&input, "^")) {
       node_def->set_input(i, strings::StrCat("^", prefix_, input));
     } else {
       node_def->set_input(i, strings::StrCat(prefix_, input));
@@ -776,7 +777,7 @@ void GraphConstructor::AddPrefixToNodeDef(
         node_def->mutable_attr()->at(kColocationAttrName).mutable_list();
     for (int i = 0; i < list->s_size(); ++i) {
       StringPiece v(list->s(i));
-      if (v.Consume(kColocationGroupPrefix)) {
+      if (str_util::ConsumePrefix(&v, kColocationGroupPrefix)) {
         list->set_s(i, strings::StrCat(kColocationGroupPrefix, prefix_, v));
       }
     }
@@ -819,7 +820,7 @@ void GraphConstructor::UpdateUniquifiedColocationNames() {
     bool updated = false;
     for (int i = 0; i < coloc_values.size(); ++i) {
       StringPiece val(coloc_values[i]);
-      if (val.Consume(kColocationGroupPrefix)) {
+      if (str_util::ConsumePrefix(&val, kColocationGroupPrefix)) {
         const auto& name_pair = uniquified_names_.find(val.ToString());
         if (name_pair == uniquified_names_.end()) continue;
         updated = true;
diff --git a/tensorflow/core/graph/graph_constructor_test.cc b/tensorflow/core/graph/graph_constructor_test.cc
index 963c1dc024..c18ccf6ce4 100644
--- a/tensorflow/core/graph/graph_constructor_test.cc
+++ b/tensorflow/core/graph/graph_constructor_test.cc
@@ -156,7 +156,9 @@ class GraphConstructorTest : public ::testing::Test {
       return "";
     }
     StringPiece loc(value[0]);
-    return loc.Consume(kColocationGroupPrefix) ? loc.ToString() : "";
+    return str_util::ConsumePrefix(&loc, kColocationGroupPrefix)
+               ? loc.ToString()
+               : "";
   }
 
   string GraphDebugString() const {
diff --git a/tensorflow/core/graph/graph_partition.cc b/tensorflow/core/graph/graph_partition.cc
index 17a174101b..877e4f1b44 100644
--- a/tensorflow/core/graph/graph_partition.cc
+++ b/tensorflow/core/graph/graph_partition.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/graph/tensor_id.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/device_name_utils.h"
 
@@ -372,7 +373,7 @@ string ControlLoopName(const string& name) {
 
 bool IsControlLoop(const Node* node) {
   const string& name = node->name();
-  return StringPiece(name).starts_with("_cloop");
+  return str_util::StartsWith(name, "_cloop");
 }
 
 // An enter node for control flow.
diff --git a/tensorflow/core/graph/graph_partition_test.cc b/tensorflow/core/graph/graph_partition_test.cc
index 6841f29149..83b24cafe2 100644
--- a/tensorflow/core/graph/graph_partition_test.cc
+++ b/tensorflow/core/graph/graph_partition_test.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
@@ -120,7 +121,7 @@ void CheckLoopConstruction(const GraphDef& graph_def) {
       if (ndef.op() == "_Recv") {
         bool has_control = false;
         for (const string& input_name : ndef.input()) {
-          if (StringPiece(input_name).starts_with("^")) {
+          if (str_util::StartsWith(input_name, "^")) {
             has_control = true;
             break;
           }
@@ -128,7 +129,7 @@ void CheckLoopConstruction(const GraphDef& graph_def) {
         EXPECT_TRUE(has_control);
       }
       // Must have a control loop
-      if (StringPiece(ndef.name()).starts_with("_cloop")) {
+      if (str_util::StartsWith(ndef.name(), "_cloop")) {
         if (ndef.op() == "Enter") {
           has_control_enter = true;
         }
diff --git a/tensorflow/core/graph/graph_test.cc b/tensorflow/core/graph/graph_test.cc
index e2ce0ba046..c8c2b225fe 100644
--- a/tensorflow/core/graph/graph_test.cc
+++ b/tensorflow/core/graph/graph_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
@@ -408,7 +409,7 @@ TEST_F(GraphTest, NewName) {
   EXPECT_NE(a1, a2);
   EXPECT_NE(a1, b1);
   EXPECT_NE(a2, b1);
-  EXPECT_TRUE(StringPiece(a1).starts_with("A")) << a1;
+  EXPECT_TRUE(str_util::StartsWith(a1, "A")) << a1;
 }
 
 TEST_F(GraphTest, IsValidNode) {
diff --git a/tensorflow/core/graph/quantize_training_test.cc b/tensorflow/core/graph/quantize_training_test.cc
index 2ad69dbd0c..e46f92bc24 100644
--- a/tensorflow/core/graph/quantize_training_test.cc
+++ b/tensorflow/core/graph/quantize_training_test.cc
@@ -32,6 +32,7 @@ limitations under the License.
 #include "tensorflow/core/graph/testlib.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session.h"
@@ -215,7 +216,7 @@ TEST_F(QuantizeTrainingTest, WithBackwardNodes_QuantizeAndDequantize) {
   Node* found_node;
   Status s = FindNode(g, strings::StrCat(d->name(), "/QuantizeAndDequantizeV2"),
                       &found_node);
-  EXPECT_TRUE(StringPiece(s.ToString()).contains("not found")) << s;
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), "not found")) << s;
 
   // Ensure that m1 and m2's inputs were quantized.
   TF_ASSERT_OK(
@@ -269,7 +270,7 @@ TEST_F(QuantizeTrainingTest, WithBackwardNodes_FakeQuant) {
   Node* found_node;
   Status s = FindNode(g, strings::StrCat(d->name(), "/FakeQuantWithMinMaxVars"),
                       &found_node);
-  EXPECT_TRUE(StringPiece(s.ToString()).contains("not found")) << s;
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), "not found")) << s;
 
   // Ensure that m1 and m2's inputs were quantized.
   TF_ASSERT_OK(
diff --git a/tensorflow/core/graph/subgraph_test.cc b/tensorflow/core/graph/subgraph_test.cc
index 7219d9812f..6c014a8d44 100644
--- a/tensorflow/core/graph/subgraph_test.cc
+++ b/tensorflow/core/graph/subgraph_test.cc
@@ -312,8 +312,8 @@ TEST_F(SubgraphTest, ChainOfFools) {
   EXPECT_TRUE(HasEdge("e", 0, "_send_e_0", 0));
 }
 
-static bool HasSubstr(const string& base, const string& substr) {
-  bool ok = StringPiece(base).contains(substr);
+static bool HasSubstr(StringPiece base, StringPiece substr) {
+  bool ok = str_util::StrContains(base, substr);
   EXPECT_TRUE(ok) << base << ", expected substring " << substr;
   return ok;
 }
diff --git a/tensorflow/core/graph/tensor_id.cc b/tensorflow/core/graph/tensor_id.cc
index 089ea5e527..8af1936d64 100644
--- a/tensorflow/core/graph/tensor_id.cc
+++ b/tensorflow/core/graph/tensor_id.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 
@@ -45,7 +46,7 @@ TensorId ParseTensorName(StringPiece name) {
   if (p > base && *p == ':' && mul > 1) {
     id.first = StringPiece(base, p - base);
     id.second = index;
-  } else if (name.starts_with("^")) {
+  } else if (str_util::StartsWith(name, "^")) {
     // Control edge
     id.first = StringPiece(base + 1);
     id.second = Graph::kControlSlot;
diff --git a/tensorflow/core/graph/validate_test.cc b/tensorflow/core/graph/validate_test.cc
index cb6d107cad..d58cdc3c5b 100644
--- a/tensorflow/core/graph/validate_test.cc
+++ b/tensorflow/core/graph/validate_test.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/kernels/ops_util.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -60,7 +61,7 @@ TEST(ValidateGraphDefTest, GraphWithUnspecifiedDefaultAttr) {
   CHECK(parser.MergeFromString(graph_def_str, &graph_def)) << graph_def_str;
   Status s = graph::ValidateGraphDef(graph_def, *OpRegistry::Global());
   EXPECT_FALSE(s.ok());
-  EXPECT_TRUE(StringPiece(s.ToString()).contains("NodeDef missing attr"));
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), "NodeDef missing attr"));
 
   // Add the defaults.
   TF_ASSERT_OK(AddDefaultAttrsToGraphDef(&graph_def, *OpRegistry::Global(), 0));
@@ -83,7 +84,7 @@ TEST(ValidateGraphDefTest, GraphWithUnspecifiedRequiredAttr) {
   CHECK(parser.MergeFromString(graph_def_str, &graph_def)) << graph_def_str;
   Status s = graph::ValidateGraphDef(graph_def, *OpRegistry::Global());
   EXPECT_FALSE(s.ok());
-  EXPECT_TRUE(StringPiece(s.ToString()).contains("NodeDef missing attr"));
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), "NodeDef missing attr"));
 
   // Add the defaults.
   TF_ASSERT_OK(AddDefaultAttrsToGraphDef(&graph_def, *OpRegistry::Global(), 0));
@@ -91,7 +92,7 @@ TEST(ValidateGraphDefTest, GraphWithUnspecifiedRequiredAttr) {
   // Validation should still fail.
   s = graph::ValidateGraphDef(graph_def, *OpRegistry::Global());
   EXPECT_FALSE(s.ok());
-  EXPECT_TRUE(StringPiece(s.ToString()).contains("NodeDef missing attr"));
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), "NodeDef missing attr"));
 }
 
 TEST(ValidateGraphDefAgainstOpListTest, GraphWithOpOnlyInOpList) {
-- 
GitLab


From 6fa7b699dc5d09d38c2706ef794a3874bc76e979 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 11:17:56 -0700
Subject: [PATCH 807/960] Switched to using TensorFlow's true dilated
 convolution support now that it is implemented, and removed emulated dilated
 convolution support.

PiperOrigin-RevId: 191319505
---
 .../contrib/lite/toco/export_tensorflow.cc    | 95 ++-----------------
 .../convert_pure_conv_to_depthwise.cc         |  5 +
 2 files changed, 15 insertions(+), 85 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/export_tensorflow.cc b/tensorflow/contrib/lite/toco/export_tensorflow.cc
index e88357f7dd..5d51431005 100644
--- a/tensorflow/contrib/lite/toco/export_tensorflow.cc
+++ b/tensorflow/contrib/lite/toco/export_tensorflow.cc
@@ -357,6 +357,14 @@ void ConvertConvOperator(const Model& model, const ConvOperator& src_op,
   strides.mutable_list()->add_i(src_op.stride_height);
   strides.mutable_list()->add_i(src_op.stride_width);
   strides.mutable_list()->add_i(1);
+  if ((src_op.dilation_width_factor != 1) ||
+      (src_op.dilation_height_factor != 1)) {
+    auto& dilations = (*conv2d_op->mutable_attr())["dilations"];
+    dilations.mutable_list()->add_i(1);
+    dilations.mutable_list()->add_i(src_op.dilation_height_factor);
+    dilations.mutable_list()->add_i(src_op.dilation_width_factor);
+    dilations.mutable_list()->add_i(1);
+  }
   string padding;
   if (src_op.padding.type == PaddingType::kSame) {
     padding = "SAME";
@@ -391,84 +399,6 @@ void ConvertConvOperator(const Model& model, const ConvOperator& src_op,
   }
 }
 
-void ConvertDilatedConvOperator(const Model& model, const ConvOperator& src_op,
-                                GraphDef* tensorflow_graph) {
-  CHECK((src_op.dilation_width_factor > 1) ||
-        (src_op.dilation_height_factor > 1))
-      << "Conv operator must have height or width dilation factor > 1. "
-         "Otherwise, use regular conv op.";
-  CHECK_EQ(src_op.stride_width, 1)
-      << "Dilated AND strided convolution is unsupported";
-  CHECK_EQ(src_op.stride_height, 1)
-      << "Dilated AND strided convolution is unsupported";
-
-  // Emulate dilated convolution with a chain of SpaceToBatchND -> Conv ->
-  // BatchToSpaceND ops.
-
-  // Compute padding
-  const auto& input_array = model.GetArray(src_op.inputs[0]);
-  const auto& input_shape = input_array.shape();
-  CHECK_EQ(input_shape.dimensions_count(), 4);
-  int height_mod_dilation = input_shape.dims(1) % src_op.dilation_height_factor;
-  int pad_height;
-  if (height_mod_dilation) {
-    pad_height = src_op.dilation_height_factor - height_mod_dilation;
-  } else {
-    pad_height = 0;
-  }
-  int pad_width;
-  int width_mod_dilation = input_shape.dims(2) % src_op.dilation_width_factor;
-  if (width_mod_dilation) {
-    pad_width = src_op.dilation_width_factor - width_mod_dilation;
-  } else {
-    pad_width = 0;
-  }
-
-  // SpaceToBatchND op "collapses" the spatially separated elements together
-  string stb_output = src_op.outputs[0] + "/dilated_conv_SpaceToBatch";
-  auto* stb_op = tensorflow_graph->add_node();
-  stb_op->set_op("SpaceToBatchND");
-  stb_op->set_name(stb_output);
-  *stb_op->add_input() = src_op.inputs[0];
-  (*stb_op->mutable_attr())["T"].set_type(DT_FLOAT);
-  string block_shape = src_op.outputs[0] + "/dilated_conv_block_shape";
-  CreateIntTensorConst(
-      block_shape,
-      {src_op.dilation_height_factor, src_op.dilation_width_factor}, {2},
-      tensorflow_graph);
-  *stb_op->add_input() = block_shape;
-  (*stb_op->mutable_attr())["Tblock_shape"].set_type(DT_INT32);
-  string stb_paddings = src_op.outputs[0] + "/dilated_conv_paddings";
-  CreateIntTensorConst(stb_paddings, {0, pad_height, pad_width, 0}, {2, 2},
-                       tensorflow_graph);
-  *stb_op->add_input() = stb_paddings;
-  (*stb_op->mutable_attr())["Tpaddings"].set_type(DT_INT32);
-
-  // Perform a regular conv on the "collapsed" elements
-  ConvOperator conv_op;
-  string conv_output = src_op.outputs[0] + "/dilated_conv_Conv2D";
-  conv_op.inputs = src_op.inputs;
-  conv_op.inputs[0] = stb_output;
-  conv_op.outputs = {conv_output};
-  conv_op.padding.type = src_op.padding.type;
-  conv_op.stride_width = src_op.stride_width;
-  conv_op.stride_height = src_op.stride_height;
-  conv_op.dilation_width_factor = 1;
-  conv_op.dilation_height_factor = 1;
-  ConvertConvOperator(model, conv_op, tensorflow_graph);
-
-  // BatchToSpaceND op restores elements to their original layout
-  auto* bts_op = tensorflow_graph->add_node();
-  bts_op->set_op("BatchToSpaceND");
-  bts_op->set_name(src_op.outputs[0]);
-  *bts_op->add_input() = conv_output;
-  (*bts_op->mutable_attr())["T"].set_type(DT_FLOAT);
-  *bts_op->add_input() = block_shape;
-  (*bts_op->mutable_attr())["Tblock_shape"].set_type(DT_INT32);
-  *bts_op->add_input() = stb_paddings;
-  (*bts_op->mutable_attr())["Tcrops"].set_type(DT_INT32);
-}
-
 void ConvertDepthwiseConvOperator(const Model& model,
                                   const DepthwiseConvOperator& src_op,
                                   GraphDef* tensorflow_graph) {
@@ -1736,13 +1666,8 @@ void ConvertOperator(const Model& model, const Operator& src_op,
   }
 
   if (src_op.type == OperatorType::kConv) {
-    const ConvOperator& conv_op = static_cast<const ConvOperator&>(src_op);
-    if ((conv_op.dilation_width_factor != 1) ||
-        (conv_op.dilation_height_factor != 1)) {
-      return ConvertDilatedConvOperator(model, conv_op, tensorflow_graph);
-    } else {
-      ConvertConvOperator(model, conv_op, tensorflow_graph);
-    }
+    ConvertConvOperator(model, static_cast<const ConvOperator&>(src_op),
+                        tensorflow_graph);
   } else if (src_op.type == OperatorType::kDepthwiseConv) {
     ConvertDepthwiseConvOperator(
         model, static_cast<const DepthwiseConvOperator&>(src_op),
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/convert_pure_conv_to_depthwise.cc b/tensorflow/contrib/lite/toco/graph_transformations/convert_pure_conv_to_depthwise.cc
index d38db85280..0fffab574d 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/convert_pure_conv_to_depthwise.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/convert_pure_conv_to_depthwise.cc
@@ -33,6 +33,11 @@ bool ConvertPureConvToDepthwise::Run(Model* model, std::size_t op_index) {
   if (conv_op->stride_width != conv_op->stride_height) {
     return false;
   }
+  if ((conv_op->dilation_width_factor != 1) ||
+      (conv_op->dilation_height_factor != 1)) {
+    // Depthwise conv does not support dilation
+    return false;
+  }
   auto& weights_array = model->GetArray(conv_op->inputs[1]);
   if (!weights_array.buffer) {
     // Yield until the weights are resolved as a constant array.
-- 
GitLab


From 5d81b72b9c1a7edd1a84c13b1dc753b310545e56 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 11:35:00 -0700
Subject: [PATCH 808/960] [XLA] Redesign: improve error handling: - For every
 op creation method, check whether there's any existing error, if so, don't do
 anything and returns an empty op. To do this efficiently, make the
 NoteErrorOrReturn method accept a lambda, and check first_error_ before
 evaluating the lambda. - Return error instead of TF_CHECK_RET, because the
 second seems to always print ERROR logs.

PiperOrigin-RevId: 191322082
---
 .../xla/client/xla_client/xla_builder.cc      | 76 ++++++++++++++-----
 .../xla/client/xla_client/xla_builder.h       |  8 +-
 2 files changed, 59 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 7a9ff0c441..04091ecb11 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -15,6 +15,7 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 
+#include <functional>
 #include <numeric>
 #include <string>
 #include <utility>
@@ -65,12 +66,17 @@ StatusOr<std::vector<Shape>> GetOperandShapes(
 }  // namespace
 
 StatusOr<Shape> XlaBuilder::GetShape(const XlaOp& op) const {
+  TF_RETURN_IF_ERROR(first_error_);
+
   TF_ASSIGN_OR_RETURN(auto instr, LookUpInstruction(op));
   return instr->shape();
 }
 
 StatusOr<Shape> XlaOp::GetShape() const {
-  TF_RET_CHECK(builder_ != nullptr);
+  if (builder_ == nullptr) {
+    return InvalidArgument(
+        "cannot GetShape for an invalid XlaOp with handle %lld", handle());
+  }
   return builder_->GetShape(*this);
 }
 
@@ -91,7 +97,22 @@ void XlaBuilder::NoteError(const Status& error) {
   }
 }
 
+XlaOp XlaBuilder::NoteErrorOrReturn(
+    const std::function<StatusOr<XlaOp>()>& op_creator) {
+  if (!first_error_.ok()) {
+    return {};
+  }
+  auto op = op_creator();
+  if (!op.ok()) {
+    NoteError(op.status());
+    return {};
+  }
+  return op.ConsumeValueOrDie();
+}
+
 StatusOr<ProgramShape> XlaBuilder::GetProgramShape(int64* root_id) {
+  TF_RETURN_IF_ERROR(first_error_);
+
   TF_RET_CHECK(root_id != nullptr);
   ProgramShape program_shape;
 
@@ -197,6 +218,8 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
 StatusOr<XlaOp> XlaBuilder::InDimBroadcast(
     const Shape& shape, const XlaOp& operand,
     tensorflow::gtl::ArraySlice<int64> broadcast_dimensions) {
+  TF_RETURN_IF_ERROR(first_error_);
+
   HloInstructionProto instr;
   *instr.mutable_shape() = shape;
   for (int64 dim : broadcast_dimensions) {
@@ -207,6 +230,8 @@ StatusOr<XlaOp> XlaBuilder::InDimBroadcast(
 
 StatusOr<XlaOp> XlaBuilder::AddBroadcastSequence(const Shape& output_shape,
                                                  const XlaOp& operand) {
+  TF_RETURN_IF_ERROR(first_error_);
+
   TF_ASSIGN_OR_RETURN(const Shape& operand_shape, operand.GetShape());
 
   CHECK(ShapeUtil::IsScalar(operand_shape) ||
@@ -250,7 +275,7 @@ XlaOp XlaBuilder::UnaryOp(HloOpcode unop, const XlaOp& operand) {
     TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
                         ShapeInference::InferUnaryOpShape(unop, operand_shape));
     return AddInstruction(std::move(instr), unop, {operand});
-  }());
+  });
 }
 
 XlaOp XlaBuilder::BinaryOp(
@@ -307,7 +332,7 @@ XlaOp XlaBuilder::BinaryOp(
     }
 
     return AddInstruction(std::move(instr), binop, {updated_lhs, updated_rhs});
-  }());
+  });
 }
 
 XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
@@ -345,7 +370,7 @@ XlaOp XlaBuilder::TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
     }
     return AddInstruction(std::move(instr), triop,
                           {updated_lhs, updated_rhs, updated_ehs});
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Add(const XlaOp& lhs, const XlaOp& rhs,
@@ -364,7 +389,7 @@ XlaOp XlaBuilder::ConstantLiteral(const Literal& literal) {
     *instr.mutable_shape() = literal.shape();
     *instr.mutable_literal() = literal.ToProto();
     return AddInstruction(std::move(instr), HloOpcode::kConstant);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Call(const XlaComputation& computation,
@@ -390,7 +415,7 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
     }
 
     return AddInstruction(std::move(instr), HloOpcode::kCall, operands);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
@@ -406,7 +431,7 @@ XlaOp XlaBuilder::Parameter(int64 parameter_number, const Shape& shape,
     instr.set_name(name);
     *instr.mutable_shape() = shape;
     return AddInstruction(std::move(instr), HloOpcode::kParameter);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Broadcast(
@@ -430,10 +455,12 @@ XlaOp XlaBuilder::Broadcast(
       dimensions[i] = i + ShapeUtil::Rank(shape) - operand_rank;
     }
     return InDimBroadcast(shape, operand, dimensions);
-  }());
+  });
 }
 
 StatusOr<XlaOp> XlaBuilder::Reshape(const Shape& shape, const XlaOp& operand) {
+  TF_RETURN_IF_ERROR(first_error_);
+
   HloInstructionProto instr;
   *instr.mutable_shape() = shape;
   return AddInstruction(std::move(instr), HloOpcode::kReshape, {operand});
@@ -477,7 +504,7 @@ XlaOp XlaBuilder::ConcatInDim(tensorflow::gtl::ArraySlice<XlaOp> operands,
     instr.add_dimensions(dimension);
 
     return AddInstruction(std::move(instr), HloOpcode::kConcatenate, operands);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Pad(const XlaOp& operand, const XlaOp& padding_value,
@@ -497,7 +524,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand,
                            ? operand
                            : Transpose(operand, dimensions);
     return Reshape(shape, transposed);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Reshape(const XlaOp& operand,
@@ -507,7 +534,7 @@ XlaOp XlaBuilder::Reshape(const XlaOp& operand,
     std::vector<int64> dimensions(shape.dimensions_size());
     std::iota(dimensions.begin(), dimensions.end(), 0);
     return Reshape(operand, dimensions, new_sizes);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Collapse(const XlaOp& operand,
@@ -535,7 +562,7 @@ XlaOp XlaBuilder::Tuple(tensorflow::gtl::ArraySlice<XlaOp> elements) {
                         ShapeInference::InferVariadicOpShape(
                             HloOpcode::kTuple, operand_shape_ptrs));
     return AddInstruction(std::move(instr), HloOpcode::kTuple, elements);
-  }());
+  });
 }
 
 XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
@@ -554,7 +581,7 @@ XlaOp XlaBuilder::GetTupleElement(const XlaOp& tuple_data, int64 index) {
 
     return AddInstruction(std::move(instr), HloOpcode::kGetTupleElement,
                           {tuple_data});
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Eq(const XlaOp& lhs, const XlaOp& rhs,
@@ -804,7 +831,7 @@ XlaOp XlaBuilder::Transpose(const XlaOp& operand,
       instr.add_dimensions(dim);
     }
     return AddInstruction(std::move(instr), HloOpcode::kTranspose, {operand});
-  }());
+  });
 }
 
 XlaOp XlaBuilder::Rev(const XlaOp& operand,
@@ -1071,6 +1098,8 @@ XlaBuilder::CreateDefaultConvDimensionNumbers(int num_spatial_dims) {
 StatusOr<XlaOp> XlaBuilder::AddInstruction(
     HloInstructionProto&& instr, HloOpcode opcode,
     tensorflow::gtl::ArraySlice<XlaOp> operands) {
+  TF_RETURN_IF_ERROR(first_error_);
+
   const int64 handle = instructions_.size();
   instr.set_id(handle);
   instr.set_opcode(HloOpcodeString(opcode));
@@ -1081,10 +1110,15 @@ StatusOr<XlaOp> XlaBuilder::AddInstruction(
     instr.set_name(StrCat(instr.name(), ".", handle));
   }
   for (const auto& operand : operands) {
-    TF_RET_CHECK(operand.builder_ != nullptr);
-    TF_RET_CHECK(operand.builder_ == this)
-        << "Do not add XlaOp from builder " << operand.builder_->name()
-        << " to builder " << this->name();
+    if (operand.builder_ == nullptr) {
+      return InvalidArgument("invalid XlaOp with handle %lld",
+                             operand.handle());
+    }
+    if (operand.builder_ != this) {
+      return InvalidArgument("Do not add XlaOp from builder %s to builder %s",
+                             operand.builder_->name().c_str(),
+                             this->name().c_str());
+    }
     instr.add_operand_ids(operand.handle());
   }
 
@@ -1101,6 +1135,12 @@ StatusOr<XlaOp> XlaBuilder::AddInstruction(
 
 StatusOr<const HloInstructionProto*> XlaBuilder::LookUpInstruction(
     const XlaOp& op) const {
+  TF_RETURN_IF_ERROR(first_error_);
+
+  if (op.builder_ != this) {
+    return InvalidArgument("invalid XlaOp with handle %lld", op.handle());
+  }
+
   TF_RET_CHECK(op.builder_ == this);
   if (op.handle() >= instructions_.size() || op.handle() < 0) {
     return InvalidArgument("no XlaOp value %lld", op.handle());
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index f66feb93ce..f43101db34 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -809,13 +809,7 @@ class XlaBuilder {
   // * dying if die_immediately_on_error_ is true
   void NoteError(const Status& error);
 
-  XlaOp NoteErrorOrReturn(StatusOr<XlaOp>&& op) {
-    if (!op.ok()) {
-      NoteError(op.status());
-      return XlaOp();
-    }
-    return op.ConsumeValueOrDie();
-  }
+  XlaOp NoteErrorOrReturn(const std::function<StatusOr<XlaOp>()>& op_creator);
 
   // Helper method that creates an empty op and notes error.
   XlaOp UnimplementedOp();
-- 
GitLab


From 6c4095c7353666c4b75ce189e68860be1159b40a Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 2 Apr 2018 11:53:37 -0700
Subject: [PATCH 809/960] TFTS: Clean up the cold start SignatureDef.

Removes state where it wasn't used.

PiperOrigin-RevId: 191324834
---
 .../timeseries/python/timeseries/estimators_test.py  |  7 +++++++
 .../contrib/timeseries/python/timeseries/head.py     | 12 +++++++++++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
index 51d0c0ca3f..9f161c1695 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators_test.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 import tempfile
 
 import numpy
+import six
 
 from tensorflow.contrib.timeseries.python.timeseries import ar_model
 from tensorflow.contrib.timeseries.python.timeseries import estimators
@@ -127,6 +128,12 @@ class TimeSeriesRegressorTest(test.TestCase):
             session=sess)
 
         # Test cold starting
+        six.assertCountEqual(
+            self,
+            [feature_keys.FilteringFeatures.TIMES,
+             feature_keys.FilteringFeatures.VALUES],
+            signatures.signature_def[
+                feature_keys.SavedModelLabels.COLD_START_FILTER].inputs.keys())
         batch_numpy_times = numpy.tile(
             numpy.arange(30, dtype=numpy.int64)[None, :], (10, 1))
         batch_numpy_values = numpy.ones([10, 30, 1])
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py
index 4cf6bbcfd4..71085f9de8 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head.py
@@ -58,6 +58,16 @@ def time_series_regression_head(model,
                                    input_statistics_generator)
 
 
+class _NoStatePredictOutput(export_lib.PredictOutput):
+
+  def as_signature_def(self, receiver_tensors):
+    no_state_receiver_tensors = {
+        key: value for key, value in receiver_tensors.items()
+        if not key.startswith(feature_keys.State.STATE_PREFIX)}
+    return super(_NoStatePredictOutput, self).as_signature_def(
+        receiver_tensors=no_state_receiver_tensors)
+
+
 class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-access
   """See `time_series_regression_head`."""
 
@@ -167,7 +177,7 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
                 export_lib.PredictOutput(
                     state_to_dictionary(filtering_outputs.end_state)),
             feature_keys.SavedModelLabels.COLD_START_FILTER:
-                export_lib.PredictOutput(
+                _NoStatePredictOutput(
                     state_to_dictionary(cold_filtering_outputs.end_state))
         },
         # Likely unused, but it is necessary to return `predictions` to satisfy
-- 
GitLab


From 46df4a1afd50f69966e63245e7758cc0d5656c4e Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 2 Apr 2018 12:01:32 -0700
Subject: [PATCH 810/960] Fix #18180

tf.size() was not respecting the `out_type` argument when eager execution was
enabled.

PiperOrigin-RevId: 191326039
---
 .../python/kernel_tests/array_ops_test.py     | 48 +++++++++++--------
 tensorflow/python/ops/array_ops.py            |  5 +-
 2 files changed, 32 insertions(+), 21 deletions(-)

diff --git a/tensorflow/python/kernel_tests/array_ops_test.py b/tensorflow/python/kernel_tests/array_ops_test.py
index 78bdb7eda7..5a20eebbc5 100644
--- a/tensorflow/python/kernel_tests/array_ops_test.py
+++ b/tensorflow/python/kernel_tests/array_ops_test.py
@@ -1007,30 +1007,38 @@ class SliceAssignTest(test_util.TensorFlowTestCase):
 
 class ShapeSizeRankTest(test_util.TensorFlowTestCase):
 
+  @test_util.run_in_graph_and_eager_modes()
   def testDenseShape(self):
-    with self.test_session():
-      t_value = [[0, 42], [24, 0]]
-      self.assertAllEqual((2, 2), array_ops.shape(t_value).eval())
-      self.assertEqual(4, array_ops.size(t_value).eval())
-      self.assertEqual(2, array_ops.rank(t_value).eval())
+    t_value = [[0, 42], [24, 0]]
+    self.assertAllEqual((2, 2), self.evaluate(array_ops.shape(t_value)))
+    self.assertEqual(4, self.evaluate(array_ops.size(t_value)))
+    self.assertEqual(2, self.evaluate(array_ops.rank(t_value)))
 
-      t = constant_op.constant(t_value)
-      self.assertAllEqual((2, 2), array_ops.shape(t).eval())
-      self.assertEqual(4, array_ops.size(t).eval())
-      self.assertEqual(2, array_ops.rank(t).eval())
+    t = constant_op.constant(t_value)
+    self.assertAllEqual((2, 2), self.evaluate(array_ops.shape(t)))
+    self.assertEqual(4, self.evaluate(array_ops.size(t)))
+    self.assertEqual(2, self.evaluate(array_ops.rank(t)))
 
+  @test_util.run_in_graph_and_eager_modes()
   def testSparseShape(self):
-    with self.test_session():
-      sp_value = sparse_tensor.SparseTensorValue(
-          indices=((0, 1), (1, 0)), values=(42, 24), dense_shape=(2, 2))
-      self.assertAllEqual((2, 2), array_ops.shape(sp_value).eval())
-      self.assertEqual(4, array_ops.size(sp_value).eval())
-      self.assertEqual(2, array_ops.rank(sp_value).eval())
-
-      sp = sparse_tensor.SparseTensor.from_value(sp_value)
-      self.assertAllEqual((2, 2), array_ops.shape(sp).eval())
-      self.assertEqual(4, array_ops.size(sp).eval())
-      self.assertEqual(2, array_ops.rank(sp).eval())
+    sp_value = sparse_tensor.SparseTensorValue(
+        indices=((0, 1), (1, 0)), values=(42, 24), dense_shape=(2, 2))
+    self.assertAllEqual((2, 2), self.evaluate(array_ops.shape(sp_value)))
+    self.assertEqual(4, self.evaluate(array_ops.size(sp_value)))
+    self.assertEqual(2, self.evaluate(array_ops.rank(sp_value)))
+
+    sp = sparse_tensor.SparseTensor.from_value(sp_value)
+    self.assertAllEqual((2, 2), self.evaluate(array_ops.shape(sp)))
+    self.assertEqual(4, self.evaluate(array_ops.size(sp)))
+    self.assertEqual(2, self.evaluate(array_ops.rank(sp)))
+
+  @test_util.run_in_graph_and_eager_modes()
+  def testSizeDtype(self):
+    tensor = [1]
+    self.assertEqual(dtypes.int32, self.evaluate(array_ops.size(tensor)).dtype)
+    self.assertEqual(
+        dtypes.int64,
+        self.evaluate(array_ops.size(tensor, out_type=dtypes.int64)).dtype)
 
 
 @test_util.with_c_api
diff --git a/tensorflow/python/ops/array_ops.py b/tensorflow/python/ops/array_ops.py
index 207866610b..68d446602e 100644
--- a/tensorflow/python/ops/array_ops.py
+++ b/tensorflow/python/ops/array_ops.py
@@ -387,7 +387,10 @@ def size_internal(input, name=None, optimize=True, out_type=dtypes.int32):
   """
   if context.executing_eagerly() and not isinstance(
       input, (sparse_tensor.SparseTensor, sparse_tensor.SparseTensorValue)):
-    return np.prod(ops.convert_to_tensor(input)._shape_tuple())  # pylint: disable=protected-access
+    input = ops.convert_to_tensor(input)
+    np_out_type = out_type.as_numpy_dtype
+    num_elements = np.prod(input._shape_tuple(), dtype=np_out_type)  # pylint: disable=protected-acces:
+    return ops.convert_to_tensor(num_elements, dtype=out_type)
   with ops.name_scope(name, "Size", [input]) as name:
     if isinstance(input, (sparse_tensor.SparseTensor,
                           sparse_tensor.SparseTensorValue)):
-- 
GitLab


From 43f5b27f6064b64b7dbcfcae865829e3617a7112 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 2 Apr 2018 12:07:16 -0700
Subject: [PATCH 811/960] Switch to the TensorFlow API generation based on
 ApiDef's and tf_export decorators.

PiperOrigin-RevId: 191326767
---
 tensorflow/BUILD                              |  8 --
 tensorflow/__init__.py                        |  7 +-
 tensorflow/contrib/cmake/python_modules.txt   |  2 +
 tensorflow/contrib/cmake/tf_python.cmake      | 91 +++++++++++++++----
 tensorflow/experimental_api.py                | 38 --------
 tensorflow/python/framework/dtypes.py         |  2 +-
 tensorflow/tools/api/generator/BUILD          |  2 +
 .../tools/api/generator/create_python_api.py  | 21 ++++-
 tensorflow/tools/api/tests/BUILD              |  1 -
 .../tools/api/tests/api_compatibility_test.py | 48 +---------
 .../ci_build/windows/cpu/cmake/run_py.bat     |  6 +-
 11 files changed, 105 insertions(+), 121 deletions(-)
 delete mode 100644 tensorflow/experimental_api.py

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index 3d5737a9d7..cfafffdd13 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -540,14 +540,6 @@ py_library(
     srcs = ["__init__.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
-    deps = ["//tensorflow/python"],
-)
-
-py_library(
-    name = "experimental_tensorflow_py",
-    srcs = ["experimental_api.py"],
-    srcs_version = "PY2AND3",
-    visibility = ["//tensorflow/tools/api/tests:__subpackages__"],
     deps = [
         "//tensorflow/python",
         "//tensorflow/tools/api/generator:python_api",
diff --git a/tensorflow/__init__.py b/tensorflow/__init__.py
index 78ad6aec19..c8683e3976 100644
--- a/tensorflow/__init__.py
+++ b/tensorflow/__init__.py
@@ -20,14 +20,19 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+# pylint: disable=g-bad-import-order
+from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
 # pylint: disable=wildcard-import
-from tensorflow.python import *  # pylint: disable=redefined-builtin
+from tensorflow.tools.api.generator.api import *  # pylint: disable=redefined-builtin
 # pylint: enable=wildcard-import
 
 from tensorflow.python.util.lazy_loader import LazyLoader
 contrib = LazyLoader('contrib', globals(), 'tensorflow.contrib')
 del LazyLoader
 
+from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
+app.flags = flags  # pylint: disable=undefined-variable
+
 del absolute_import
 del division
 del print_function
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index 02c456c199..d5cf42b641 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -104,6 +104,8 @@ tensorflow/python/user_ops
 tensorflow/python/util
 tensorflow/python/util/protobuf
 tensorflow/tools
+tensorflow/tools/api
+tensorflow/tools/api/generator
 tensorflow/tools/graph_transforms
 tensorflow/contrib
 tensorflow/contrib/all_reduce
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index fae45ead5c..20eeded8d3 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -686,6 +686,77 @@ AddUserOps(TARGET _beam_search_ops
     DEPENDS pywrap_tensorflow_internal tf_python_ops
     DISTCOPY ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/seq2seq/python/ops/)
 
+if(WIN32)
+  if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
+    add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.dll
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
+  else()
+    add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.dll
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.lib
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
+  endif()
+else()
+  add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal.so
+                                     ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.so)
+endif()
+
+
+########################################################
+# Generate API __init__.py files.
+########################################################
+
+# Parse tensorflow/tools/api/generator/BUILD to get list of generated files.
+FILE(READ ${tensorflow_source_dir}/tensorflow/tools/api/generator/BUILD api_generator_BUILD_text)
+STRING(REGEX MATCH "# BEGIN GENERATED FILES.*# END GENERATED FILES" api_init_files_text ${api_generator_BUILD_text})
+string(REPLACE "# BEGIN GENERATED FILES" "" api_init_files_text ${api_init_files_text})
+string(REPLACE "# END GENERATED FILES" "" api_init_files_text ${api_init_files_text})
+string(REPLACE "," ";" api_init_files_list ${api_init_files_text})
+
+set(api_init_files "")
+foreach(api_init_file ${api_init_files_list})
+    string(STRIP "${api_init_file}" api_init_file)
+    if(api_init_file)
+        string(REPLACE "\"" "" api_init_file "${api_init_file}")  # Remove quotes
+        list(APPEND api_init_files "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/${api_init_file}")
+    endif()
+endforeach(api_init_file)
+set(api_init_list_file "${tensorflow_source_dir}/api_init_files_list.txt")
+file(WRITE "${api_init_list_file}" "${api_init_files}")
+
+# Run create_python_api.py to generate __init__.py files.
+add_custom_command(
+      OUTPUT ${api_init_files}
+      DEPENDS tf_python_ops tf_python_copy_scripts_to_destination pywrap_tensorflow_internal tf_python_touchup_modules tf_extension_ops
+
+      # tensorflow/__init__.py depends on files generated in this step. So, remove it while
+      # this step is running since the files aren't there yet.
+      COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+                                         ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/final.__init__.py
+      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+
+      # Run create_python_api.py to generate API init files.
+      COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python ${PYTHON_EXECUTABLE}
+              "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/create_python_api.py" "${api_init_list_file}"
+
+      # Re-add tensorflow/__init__.py back.
+      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+      COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/final.__init__.py
+                                         ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
+
+      COMMENT "Generating __init__.py files for Python API."
+      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tf_python"
+)
+
+add_custom_target(tf_python_api SOURCES ${api_init_files})
+add_dependencies(tf_python_api tf_python_ops)
+
+
 ############################################################
 # Build a PIP package containing the TensorFlow runtime.
 ############################################################
@@ -695,6 +766,7 @@ add_dependencies(tf_python_build_pip_package
     tf_python_copy_scripts_to_destination
     tf_python_touchup_modules
     tf_python_ops
+    tf_python_api
     tf_extension_ops)
 
 # Fix-up Python files that were not included by the add_python_module() macros.
@@ -707,25 +779,6 @@ add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/testing/python/framework/util_test.py
                                    ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/testing/python/framework/)
 
-if(WIN32)
-  if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
-    add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.dll
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
-  else()
-    add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.dll
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.lib
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
-  endif()
-else()
-  add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal.so
-                                     ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.so)
-endif()
 add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/tools/pip_package/README
                                    ${CMAKE_CURRENT_BINARY_DIR}/tf_python/)
diff --git a/tensorflow/experimental_api.py b/tensorflow/experimental_api.py
deleted file mode 100644
index 63a8aa9cb1..0000000000
--- a/tensorflow/experimental_api.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-# ==============================================================================
-
-# Bring in all of the public TensorFlow interface into this
-# module.
-
-from __future__ import absolute_import
-from __future__ import division
-from __future__ import print_function
-
-# pylint: disable=g-bad-import-order
-from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
-# pylint: disable=wildcard-import
-from tensorflow.tools.api.generator.api import *  # pylint: disable=redefined-builtin
-# pylint: enable=wildcard-import
-
-from tensorflow.python.util.lazy_loader import LazyLoader
-contrib = LazyLoader('contrib', globals(), 'tensorflow.contrib')
-del LazyLoader
-
-from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-app.flags = flags  # pylint: disable=undefined-variable
-
-del absolute_import
-del division
-del print_function
diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index 0edae92fd4..a31c424263 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -345,7 +345,7 @@ tf_export("uint16").export_constant(__name__, "uint16")
 uint32 = DType(types_pb2.DT_UINT32)
 tf_export("uint32").export_constant(__name__, "uint32")
 uint64 = DType(types_pb2.DT_UINT64)
-tf_export("uint64").export_constant(__name__, "uint32")
+tf_export("uint64").export_constant(__name__, "uint64")
 int16 = DType(types_pb2.DT_INT16)
 tf_export("int16").export_constant(__name__, "int16")
 int8 = DType(types_pb2.DT_INT8)
diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD
index 6722536358..f8063ae0fb 100644
--- a/tensorflow/tools/api/generator/BUILD
+++ b/tensorflow/tools/api/generator/BUILD
@@ -32,6 +32,7 @@ genrule(
     # api/module1/module2/__init__.py and api/module3/__init__.py.
     # keep sorted
     outs = [
+        # BEGIN GENERATED FILES
         "api/__init__.py",
         "api/app/__init__.py",
         "api/bitwise/__init__.py",
@@ -116,6 +117,7 @@ genrule(
         "api/train/__init__.py",
         "api/train/queue_runner/__init__.py",
         "api/user_ops/__init__.py",
+        # END GENERATED FILES
     ],
     cmd = "$(location create_python_api) $(OUTS)",
     tools = ["create_python_api"],
diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py
index 183c4731b8..1505dc69b9 100644
--- a/tensorflow/tools/api/generator/create_python_api.py
+++ b/tensorflow/tools/api/generator/create_python_api.py
@@ -195,16 +195,19 @@ def create_api_files(output_files):
   """
   module_name_to_file_path = {}
   for output_file in output_files:
+    # Convert path separators to '/' for easier parsing below.
+    normalized_output_file = output_file.replace(os.sep, '/')
     if _API_DIR not in output_file:
       raise ValueError(
           'Output files must be in api/ directory, found %s.' % output_file)
     # Get the module name that corresponds to output_file.
     # First get module directory under _API_DIR.
     module_dir = os.path.dirname(
-        output_file[output_file.rfind(_API_DIR)+len(_API_DIR):])
+        normalized_output_file[
+            normalized_output_file.rfind(_API_DIR)+len(_API_DIR):])
     # Convert / to .
     module_name = module_dir.replace('/', '.').strip('.')
-    module_name_to_file_path[module_name] = output_file
+    module_name_to_file_path[module_name] = os.path.normpath(output_file)
 
   # Create file for each expected output in genrule.
   for module, file_path in module_name_to_file_path.items():
@@ -241,6 +244,16 @@ if __name__ == '__main__':
   parser = argparse.ArgumentParser()
   parser.add_argument(
       'outputs', metavar='O', type=str, nargs='+',
-      help='Python files that we expect this script to output.')
+      help='If a single file is passed in, then we we assume it contains a '
+      'semicolon-separated list of Python files that we expect this script to '
+      'output. If multiple files are passed in, then we assume output files '
+      'are listed directly as arguments.')
   args = parser.parse_args()
-  main(args.outputs)
+  if len(args.outputs) == 1:
+    # If we only get a single argument, then it must be a file containing
+    # list of outputs.
+    with open(args.outputs[0]) as output_list_file:
+      outputs = [line.strip() for line in output_list_file.read().split(';')]
+  else:
+    outputs = args.outputs
+  main(outputs)
diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD
index 0dc154b6d2..724b12cd47 100644
--- a/tensorflow/tools/api/tests/BUILD
+++ b/tensorflow/tools/api/tests/BUILD
@@ -23,7 +23,6 @@ py_test(
     ],
     srcs_version = "PY2AND3",
     deps = [
-        "//tensorflow:experimental_tensorflow_py",
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:lib",
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 603b2a4327..26d5bca637 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -34,7 +34,6 @@ import sys
 import unittest
 
 import tensorflow as tf
-from tensorflow import experimental_api as api
 
 from google.protobuf import text_format
 
@@ -47,8 +46,6 @@ from tensorflow.tools.api.lib import python_object_to_proto_visitor
 from tensorflow.tools.common import public_api
 from tensorflow.tools.common import traverse
 
-if hasattr(tf, 'experimental_api'):
-  del tf.experimental_api
 
 # FLAGS defined at the bottom:
 FLAGS = None
@@ -205,51 +202,12 @@ class ApiCompatibilityTest(test.TestCase):
     # Extract all API stuff.
     visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
 
-    public_api_visitor = public_api.PublicAPIVisitor(visitor)
-    public_api_visitor.do_not_descend_map['tf'].append('contrib')
-    public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
-    traverse.traverse(tf, public_api_visitor)
-
-    proto_dict = visitor.GetProtos()
-
-    # Read all golden files.
-    expression = os.path.join(
-        resource_loader.get_root_dir_with_all_resources(),
-        _KeyToFilePath('*'))
-    golden_file_list = file_io.get_matching_files(expression)
-
-    def _ReadFileToProto(filename):
-      """Read a filename, create a protobuf from its contents."""
-      ret_val = api_objects_pb2.TFAPIObject()
-      text_format.Merge(file_io.read_file_to_string(filename), ret_val)
-      return ret_val
-
-    golden_proto_dict = {
-        _FileNameToKey(filename): _ReadFileToProto(filename)
-        for filename in golden_file_list
-    }
-
-    # Diff them. Do not fail if called with update.
-    # If the test is run to update goldens, only report diffs but do not fail.
-    self._AssertProtoDictEquals(
-        golden_proto_dict,
-        proto_dict,
-        verbose=FLAGS.verbose_diffs,
-        update_goldens=FLAGS.update_goldens)
-
-  @unittest.skipUnless(
-      sys.version_info.major == 2,
-      'API compabitility test goldens are generated using python2.')
-  def testNewAPIBackwardsCompatibility(self):
-    # Extract all API stuff.
-    visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
-
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.do_not_descend_map['tf'].append('contrib')
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
     # TODO(annarev): Make slide_dataset available in API.
     public_api_visitor.private_map['tf'] = ['slide_dataset']
-    traverse.traverse(api, public_api_visitor)
+    traverse.traverse(tf, public_api_visitor)
 
     proto_dict = visitor.GetProtos()
 
@@ -276,9 +234,7 @@ class ApiCompatibilityTest(test.TestCase):
         golden_proto_dict,
         proto_dict,
         verbose=FLAGS.verbose_diffs,
-        update_goldens=False,
-        additional_missing_object_message=
-        'Check if tf_export decorator/call is missing for this symbol.')
+        update_goldens=FLAGS.update_goldens)
 
 
 if __name__ == '__main__':
diff --git a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
index 3c3b223a00..30554a084c 100644
--- a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
+++ b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
@@ -28,6 +28,9 @@ IF DEFINED TF_NIGHTLY (ECHO TF_NIGHTLY is set to %TF_NIGHTLY%) ELSE (SET TF_NIGH
 :: Set pip binary location. Do not override if it is set already.
 IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Program Files\Anaconda3\Scripts\pip.exe")
 
+:: Install absl-py.
+%PIP_EXE% install --upgrade absl-py
+
 :: Run the CMAKE build to build the pip package.
 CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\cpu\cmake\run_build.bat
 if %errorlevel% neq 0 exit /b %errorlevel%
@@ -37,9 +40,6 @@ DIR %REPO_ROOT%\%BUILD_DIR%\tf_python\dist\ /S /B > wheel_filename_file
 set /p WHEEL_FILENAME=<wheel_filename_file
 del wheel_filename_file
 
-:: Install absl-py.
-%PIP_EXE% install --upgrade absl-py
-
 :: Install the pip package.
 echo Installing PIP package...
 %PIP_EXE% install --upgrade --no-deps %WHEEL_FILENAME% -v -v
-- 
GitLab


From fc34c057d9d1118477b3e02870b97305c2d1af86 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 12:36:08 -0700
Subject: [PATCH 812/960] Fix a bug in AvgPoolGrad op cost in extracting input
 x's shape. AvgPoolGrad takes a shape tensor; hence, a value should be parsed
 from inputs(0) to extract correct shape of x.

PiperOrigin-RevId: 191330762
---
 .../grappler/costs/op_level_cost_estimator.cc | 29 ++++++-
 .../costs/op_level_cost_estimator_test.cc     | 85 ++++++++++---------
 2 files changed, 70 insertions(+), 44 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 0f6307cfdf..75258d0547 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -817,6 +817,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropInputOperations(
   }
   if (!shape_found) {
     // Set the minimum filter size that's feasible.
+    input_shape.Clear();
     for (int i = 0; i < 4; ++i) {
       input_shape.add_dim()->set_size(1);
     }
@@ -859,6 +860,7 @@ int64 OpLevelCostEstimator::CountConv2DBackpropFilterOperations(
   }
   if (!shape_found) {
     // Set the minimum filter size that's feasible.
+    filter_shape.Clear();
     for (int i = 0; i < 4; ++i) {
       filter_shape.add_dim()->set_size(1);
     }
@@ -1242,10 +1244,31 @@ Costs OpLevelCostEstimator::PredictAvgPoolGrad(
     const OpContext& op_context) const {
   bool found_unknown_shapes = false;
   const auto& op_info = op_context.op_info;
-  // x: op_info.inputs(0)
+  // x's shape: op_info.inputs(0)
   // y_grad: op_info.inputs(1)
-  ConvolutionDimensions dims = OpDimensionsFromInputs(
-      op_info.inputs(0).shape(), op_info, &found_unknown_shapes);
+
+  // Extract x_shape from op_info.inputs(0).value() or op_info.outputs(0).
+  bool shape_found = false;
+  TensorShapeProto x_shape;
+  if (op_info.inputs_size() >= 1 && op_info.inputs(0).has_value()) {
+    const TensorProto& value = op_info.inputs(0).value();
+    shape_found = GetTensorShapeProtoFromTensorProto(value, &x_shape);
+  }
+  if (!shape_found && op_info.outputs_size() > 0) {
+    x_shape = op_info.outputs(0).shape();
+    shape_found = true;
+  }
+  if (!shape_found) {
+    // Set the minimum shape that's feasible.
+    x_shape.Clear();
+    for (int i = 0; i < 4; ++i) {
+      x_shape.add_dim()->set_size(1);
+    }
+    found_unknown_shapes = true;
+  }
+
+  ConvolutionDimensions dims =
+      OpDimensionsFromInputs(x_shape, op_info, &found_unknown_shapes);
 
   int64 ops = 0;
   if (dims.kx <= dims.sx && dims.ky <= dims.sy) {
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 56915ed821..4758bbfee7 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -217,6 +217,39 @@ std::vector<int> GetPoolingOutputSize(const std::vector<int>& input,
   return output;
 }
 
+// Helper functions for testing GetTensorShapeProtoFromTensorProto().
+void GetTensorProto(const DataType dtype, const std::vector<int64>& shape,
+                    const std::vector<int64> values, const bool tensor_content,
+                    TensorProto* tensor_proto) {
+  tensor_proto->Clear();
+  TensorProto temp_tensor_proto;
+  temp_tensor_proto.set_dtype(dtype);
+  for (const auto& x : shape) {
+    temp_tensor_proto.mutable_tensor_shape()->add_dim()->set_size(x);
+  }
+  for (const auto& x : values) {
+    if (dtype == DT_INT64) {
+      temp_tensor_proto.add_int64_val(x);
+    } else if (dtype == DT_INT32 || dtype == DT_INT16 || dtype == DT_INT8 ||
+               dtype == DT_UINT8) {
+      temp_tensor_proto.add_int_val(x);
+    } else if (dtype == DT_UINT32) {
+      temp_tensor_proto.add_uint32_val(x);
+    } else if (dtype == DT_UINT64) {
+      temp_tensor_proto.add_uint64_val(x);
+    } else {
+      CHECK(false) << "Unsupported dtype: " << dtype;
+    }
+  }
+  Tensor tensor(dtype);
+  CHECK(tensor.FromProto(temp_tensor_proto));
+  if (tensor_content) {
+    tensor.AsProtoTensorContent(tensor_proto);
+  } else {
+    tensor.AsProtoField(tensor_proto);
+  }
+}
+
 OpContext DescribePoolingOp(const string& op_name, const std::vector<int>& x,
                             const std::vector<int>& ksize,
                             const std::vector<int>& strides,
@@ -233,8 +266,11 @@ OpContext DescribePoolingOp(const string& op_name, const std::vector<int>& x,
     DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
     DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_outputs());
   } else if (op_name == "AvgPoolGrad") {
-    // input: x, y_grad, output: x_grad.
-    DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_inputs());
+    // input: x's shape, y_grad, output: x_grad.
+    DescribeArbitraryRankInput({4}, DT_INT32, &op_info);
+    auto* tensor_proto = op_info.mutable_inputs(0)->mutable_value();
+    GetTensorProto(DT_INT32, {4}, {x[0], x[1], x[2], x[3]},
+                   /*tensor_content=*/false, tensor_proto);
     DescribeTensor4D(y[0], y[1], y[2], y[3], op_info.add_inputs());
     DescribeTensor4D(x[0], x[1], x[2], x[3], op_info.add_outputs());
   } else if (op_name == "MaxPoolGrad") {
@@ -510,39 +546,6 @@ TEST_F(OpLevelCostEstimatorTest, BatchMatMul) {
   EXPECT_NE(matmul_inaccurate, batch_matmul_inaccurate);
 }
 
-// Helper functions for testing GetTensorShapeProtoFromTensorProto().
-void GetTensorProto(const DataType dtype, const std::vector<int64>& shape,
-                    const std::vector<int64> values, const bool tensor_content,
-                    TensorProto* tensor_proto) {
-  tensor_proto->Clear();
-  TensorProto temp_tensor_proto;
-  temp_tensor_proto.set_dtype(dtype);
-  for (const auto& x : shape) {
-    temp_tensor_proto.mutable_tensor_shape()->add_dim()->set_size(x);
-  }
-  for (const auto& x : values) {
-    if (dtype == DT_INT64) {
-      temp_tensor_proto.add_int64_val(x);
-    } else if (dtype == DT_INT32 || dtype == DT_INT16 || dtype == DT_INT8 ||
-               dtype == DT_UINT8) {
-      temp_tensor_proto.add_int_val(x);
-    } else if (dtype == DT_UINT32) {
-      temp_tensor_proto.add_uint32_val(x);
-    } else if (dtype == DT_UINT64) {
-      temp_tensor_proto.add_uint64_val(x);
-    } else {
-      CHECK(false) << "Unsupported dtype: " << dtype;
-    }
-  }
-  Tensor tensor(dtype);
-  CHECK(tensor.FromProto(temp_tensor_proto));
-  if (tensor_content) {
-    tensor.AsProtoTensorContent(tensor_proto);
-  } else {
-    tensor.AsProtoField(tensor_proto);
-  }
-}
-
 void ExpectTensorShape(const std::vector<int64>& expected,
                        const TensorShapeProto& tensor_shape_proto) {
   TensorShape tensor_shape_expected(expected);
@@ -746,25 +749,25 @@ TEST_F(OpLevelCostEstimatorTest, PredictAvgPoolGrad) {
   {
     // Typical 3xz3 window with 2x2 stride.
     auto costs = predict_avg_pool_grad(10, 20, 384, 3, 2, "SAME");
-    EXPECT_EQ(Costs::Duration(1920000), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(1305602), costs.execution_time);
     EXPECT_EQ(Costs::Duration(537600), costs.compute_time);
-    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_EQ(Costs::Duration(768002), costs.memory_time);
     EXPECT_FALSE(costs.inaccurate);
   }
   {
     // 1x1 window with 2x2 stride: used for shortcut in resnet-50.
     auto costs = predict_avg_pool_grad(10, 20, 384, 1, 2, "SAME");
-    EXPECT_EQ(Costs::Duration(1574400), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(960002), costs.execution_time);
     EXPECT_EQ(Costs::Duration(192000), costs.compute_time);
-    EXPECT_EQ(Costs::Duration(1382400), costs.memory_time);
+    EXPECT_EQ(Costs::Duration(768002), costs.memory_time);
     EXPECT_FALSE(costs.inaccurate);
   }
   {
     // 2x2 window with 3x3 stride.
     auto costs = predict_avg_pool_grad(10, 20, 384, 2, 3, "VALID");
-    EXPECT_EQ(Costs::Duration(1476480), costs.execution_time);
+    EXPECT_EQ(Costs::Duration(862082), costs.execution_time);
     EXPECT_EQ(Costs::Duration(172416), costs.compute_time);
-    EXPECT_EQ(Costs::Duration(1304064), costs.memory_time);
+    EXPECT_EQ(Costs::Duration(689666), costs.memory_time);
     EXPECT_FALSE(costs.inaccurate);
   }
 }
-- 
GitLab


From c8064f1ac3c42951aa1593260346b75d306ffe95 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 12:41:59 -0700
Subject: [PATCH 813/960] Rewrite Add/AddN subgraph, minimizing number of
 required broadcasts.

1) Collect to AddOpsGroup inputs of symbolically defined
   shapes, that can be broadcasted to the root shape
2) Rewrite equal shapes with AddN(s)
3) Build Add tree from aggegations of different shapes,
   minimizing the cost of broadcast

PiperOrigin-RevId: 191331566
---
 tensorflow/core/grappler/optimizers/BUILD     |   2 +
 .../optimizers/arithmetic_optimizer.cc        | 297 +++++++-----
 .../optimizers/arithmetic_optimizer.h         |   8 +-
 .../optimizers/arithmetic_optimizer_test.cc   | 441 ++++++++++++------
 .../grappler/optimizers/symbolic_shapes.h     |   4 +-
 5 files changed, 500 insertions(+), 252 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index f865d0c159..4ce3e73911 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -251,6 +251,7 @@ cc_library(
         ":constant_folding",
         ":graph_optimizer",
         ":graph_optimizer_stage",
+        ":symbolic_shapes",
         "//tensorflow/core:framework",
         "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
@@ -260,6 +261,7 @@ cc_library(
         "//tensorflow/core/grappler:utils",
         "//tensorflow/core/grappler/costs:graph_properties",
         "//tensorflow/core/grappler/utils:frame",
+        "//tensorflow/core/grappler/utils:topological_sort",
     ],
 )
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index d155e0b289..882e4d9a40 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/arithmetic_optimizer.h"
 
 #include <algorithm>
+#include <deque>
 #include <limits>
 #include <unordered_map>
 #include <unordered_set>
@@ -31,8 +32,10 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/optimizers/constant_folding.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer_stage.h"
+#include "tensorflow/core/grappler/optimizers/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/grappler/utils/frame.h"
+#include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/strings/str_util.h"
@@ -197,39 +200,6 @@ void SetSourceDataType(DataType dtype, NodeDef* node) {
 
 bool IsNumberType(DataType dtype) { return kNumberTypes.Contains(dtype); }
 
-// Shape is symbolically defined if it has a known rank, and each dimension is
-// defined, or is an unknown symbol (dim.size <= -2).
-bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape) {
-  return !shape.unknown_rank() &&
-         std::all_of(
-             shape.dim().begin(), shape.dim().end(),
-             [](const TensorShapeProto::Dim& dim) { return dim.size() != -1; });
-}
-
-bool ShapeIsSymbolicallyDefined(const OpInfo::TensorProperties& properties) {
-  return ShapeIsSymbolicallyDefined(properties.shape());
-}
-
-bool ShapesSymbolicallyEqual(const TensorShapeProto& left,
-                             const TensorShapeProto& right) {
-  if (left.unknown_rank() || right.unknown_rank() ||
-      left.dim_size() != right.dim_size()) {
-    return false;
-  }
-  for (int i = 0; i < left.dim_size(); ++i) {
-    if (left.dim(i).size() == -1 || right.dim(i).size() == -1 ||
-        left.dim(i).size() != right.dim(i).size()) {
-      return false;
-    }
-  }
-  return true;
-}
-
-bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
-                             const OpInfo::TensorProperties& right) {
-  return ShapesSymbolicallyEqual(left.shape(), right.shape());
-}
-
 // Returns whether `reshape` is an identity op. The tensor that `reshape`
 // reshapes is the `output_pos`-th output of node `input`.
 bool ReshapeIsIdentity(const NodeDef& reshape, const NodeDef& input,
@@ -348,17 +318,30 @@ class ArithmeticOptimizerStage : public GraphOptimizerStage<string> {
 // Rewrite a tree of Add/AddN with a single AddN operation, consuming all the
 // original inputs of absorbed nodes.
 //
-// All nodes in a Add/AddN subgraph must have symbolically equal shape. All
-// nodes must have the same device placement.
+// 1) All nodes must have the same device placement.
+//
+// 2) If All nodes in a Add/AddN subgraph have symbolically equal shape, tree is
+//    optimized to a single AddN node.
 //
-// Example:
 //                AddN_1
 //             /    |    \
-//          Add_1   z   Add_2       -> AddN(z, y, z, w, q, e)
+//          Add_1   z   Add_2       -> AddN(x, y, z, w, q, e)
 //          /  \        /  \
 //         x    y      w    Add_3
 //                          / \
 //                         q   e
+//
+// 3) If some nodes have different shape (it needs to be broadcastable to the
+//    shape of a "root), tree is optimized to AddNs for symbolically equal
+//    shapes, and a tree of Add ops, that minimize broadcasts.
+//
+//                AddN_1                                 Add
+//             /    |    \                              /  \
+//          Add_1   z   Add_2       ->               Add    w
+//          /  \        /  \                        /   \
+//         x    y      w    Add_3      AddN(x, y, q, e)  z
+//                          / \
+//                         q   e
 class AddOpsRewriteStage : public ArithmeticOptimizerStage {
  public:
   explicit AddOpsRewriteStage(const GraphOptimizerContext& ctx,
@@ -379,7 +362,7 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     OpInfo::TensorProperties properties;
     Status has_properties = GetTensorProperties(node->name(), &properties);
     return has_properties.ok() && ShapeIsSymbolicallyDefined(properties) &&
-           HasAllInputsOfSymbolicallyEqualShape(*node, properties);
+           HasAllInputsOfBroadcastableShape(*node, properties);
   }
 
   Status TrySimplify(NodeDef* node, string* simplified_node_name) override {
@@ -387,7 +370,7 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     AddOpsGroup group;
     TF_RETURN_IF_ERROR(CreateAddOpsGroup(node, &group));
 
-    if (!group.absorbed_nodes.empty() && !IsRewritten(group)) {
+    if (!group.absorbed_nodes.empty()) {
       *simplified_node_name = RewriteAddOpsGroup(group);
     }
 
@@ -395,6 +378,14 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
   }
 
  private:
+  // Input name with a statically inferred shape from GraphProperties
+  struct InputAndShape {
+    InputAndShape(const string& input, const TensorShapeProto& shape)
+        : input(input), shape(shape) {}
+    string input;
+    TensorShapeProto shape;
+  };
+
   // Holds together an add ops subgraph that we want to rewrite together.
   //
   // For the graph above the AddOpsGroup will be:
@@ -406,12 +397,12 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     TensorShapeProto root_shape;
     // Add/AddN operations below the root level that were absorbed by this group
     std::vector<NodeDef*> absorbed_nodes;
-    // Inputs of absorbed nodes that will be forwarded to rewritten AddN node
-    std::vector<string> inputs;
+    // Inputs of absorbed nodes that will be forwarded to optimized AddN ops
+    std::vector<InputAndShape> inputs;
   };
 
-  // Check if all inputs have symbolically equal shapes
-  bool HasAllInputsOfSymbolicallyEqualShape(
+  // Check if all inputs can be broadcasted to the same shape
+  bool HasAllInputsOfBroadcastableShape(
       const NodeDef& node, const OpInfo::TensorProperties& properties) const {
     const AddOpsRewriteStage* self = this;
     return std::all_of(
@@ -421,7 +412,7 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
           Status has_input_properties =
               self->GetTensorProperties(input, &input_properties);
           return has_input_properties.ok() &&
-                 ShapesSymbolicallyEqual(properties, input_properties);
+                 ShapesBroadcastable(properties, input_properties);
         });
   }
 
@@ -467,11 +458,11 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     if (node->device() != group.root_node->device()) {
       return false;
     }
-    // All input shapes must be symbolically defined and equal to the node shape
+    // All input shapes must be broadcastable to the node shape
     OpInfo::TensorProperties properties;
     Status has_properties = GetTensorProperties(name, &properties);
     return has_properties.ok() &&
-           HasAllInputsOfSymbolicallyEqualShape(*node, properties);
+           HasAllInputsOfBroadcastableShape(*node, properties);
   }
 
   // Node requirements both for a root node and an absorbed node
@@ -490,18 +481,16 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     if (rewritten_nodes_.find(node->name()) != rewritten_nodes_.end()) {
       return false;
     }
+    // it must not be created by this stage at any of previous optimization runs
+    if (StringPiece(node->name()).contains(stage_name_)) {
+      return false;
+    }
     // should not drive or be driven by control dependency
     // TODO(ezhulenev): relax this condition for root node
     return !(IsDrivenByControlDependency(*node) ||
              DrivesControlDependency(*node));
   }
 
-  // Check that optimized group node name doesn't exists. It might happen if
-  // graph optimized multiple times without pruning between invocations.
-  bool IsRewritten(const AddOpsGroup& group) const {
-    return ctx_.node_map->NodeExists(AddOpsGroupName(group));
-  }
-
   // Create an AddOpsGroup with a root in a given node
   Status CreateAddOpsGroup(const NodeDef* root_node, AddOpsGroup* group) {
     OpInfo::TensorProperties root_node_output_properties;
@@ -513,7 +502,10 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
 
     group->absorbed_nodes.reserve(root_node->input_size());
     for (int i = 0; i < root_node->input_size(); ++i) {
-      TF_RETURN_IF_ERROR(AbsorbInputByAddOpsGroup(root_node->input(i), group));
+      const string& input_i = root_node->input(i);
+      if (!IsControlInput(input_i)) {
+        TF_RETURN_IF_ERROR(AbsorbInputByAddOpsGroup(input_i, group));
+      }
     }
 
     return Status::OK();
@@ -526,71 +518,159 @@ class AddOpsRewriteStage : public ArithmeticOptimizerStage {
     if (IsAbsorbableByAddOpsGroup(input, *group)) {
       group->absorbed_nodes.push_back(node);
       for (int i = 0; i < node->input_size(); ++i) {
-        TF_RETURN_IF_ERROR(AbsorbInputByAddOpsGroup(node->input(i), group));
+        const string& input_i = node->input(i);
+        if (!IsControlInput(input)) {
+          TF_RETURN_IF_ERROR(AbsorbInputByAddOpsGroup(input_i, group));
+        }
       }
     } else {
       // If node can't be absorbed, add it to AddOpsGroup input
-      group->inputs.push_back(input);
+      OpInfo::TensorProperties properties;
+      TF_RETURN_IF_ERROR(GetTensorProperties(input, &properties));
+      group->inputs.emplace_back(input, properties.shape());
     }
     return Status::OK();
   }
 
-  // New node for AddOpsGroup is added to the same scope as a root_node. All
-  // absorbed nodes are stripped of their scope, and only names are used in a
-  // new node name.
-  //
-  // Example: AddOpsGroup(root="a/b/c/Add_2", absorbed=["d/Add_1", "e/Add"])
-  //          node_name="a/b/c/AddOpsGroup_Add_2_Add_1_Add
-  string AddOpsGroupName(const AddOpsGroup& group) const {
-    CHECK_NOTNULL(group.root_node);
-
-    auto root = ParseNodeScopeAndName(group.root_node->name());
+  // Rewrite an add ops group into a single AddN if all input shapes are
+  // symbolically equal. If not, create AddN for equal shapes first, and then
+  // build an Add tree, minimizing the cost of broadcasts.
+  string RewriteAddOpsGroup(const AddOpsGroup& group) {
+    // all new nodes will be placed under the scope of a root node
+    auto root_scope_and_name = ParseNodeScopeAndName(group.root_node->name());
+
+    auto shape_sig = [](const TensorShapeProto& shape) {
+      string name = strings::StrCat("r:", shape.dim_size(), ":d");
+      for (int i = 0; i < shape.dim_size(); ++i)
+        strings::StrAppend(&name, ":", shape.dim(i).size());
+      return name;
+    };
+
+    // Find what shapes are present in the inputs of absorbed nodes
+    std::unordered_map<string, std::vector<InputAndShape>> shape_sig_to_inputs;
+    for (const auto& input : group.inputs) {
+      shape_sig_to_inputs[shape_sig(input.shape)].push_back(input);
+    }
 
-    std::vector<string> absorbed_node_names(group.absorbed_nodes.size());
-    std::transform(group.absorbed_nodes.begin(), group.absorbed_nodes.end(),
-                   absorbed_node_names.begin(),
-                   [](const NodeDef* node) { return node->name(); });
+    // Collect all the shapes from representative elements
+    std::vector<TensorShapeProto> shapes;
+    shapes.reserve(shape_sig_to_inputs.size());
+    for (const auto& el : shape_sig_to_inputs)
+      shapes.push_back(el.second[0].shape);
+
+    // If all inputs have the same shape, rewrite whole group with a single AddN
+    if (shapes.size() == 1) {
+      string node_name = OptimizedNodeName(root_scope_and_name);
+      AddInputsOfSymbolicallyEqualShape(*group.root_node, node_name,
+                                        group.inputs);
+      // keep track of nodes that were created or absorbed as a part of rewrite
+      rewritten_nodes_.insert(node_name);
+      return node_name;
+    }
 
-    return OptimizedNodeName(root, absorbed_node_names);
-  }
+    // For inputs of different shapes:
+    // 1. Rewrite inputs of the same shape using AddN (leaf nodes)
+    // 2. Build a tree of Add nodes, minimizing cost of broadcast
+    std::sort(shapes.begin(), shapes.end(),
+              [](const TensorShapeProto& left, const TensorShapeProto& right) {
+                return CompareSymbolicallyShapedTensorSizes(left, right);
+              });
+
+    // optimized name for leaf AddN nodes
+    auto leaf_node_name = [&root_scope_and_name, this](int i) {
+      return OptimizedNodeName(root_scope_and_name,
+                               strings::StrCat("Leaf_", i));
+    };
+    // optimized name for internal nodes of a tree built up from AddN leaves
+    auto internal_node_name = [&root_scope_and_name, this](int i) {
+      return OptimizedNodeName(root_scope_and_name,
+                               strings::StrCat("Internal_", i));
+    };
+
+    // Add/AddN nodes that must be added to the tree
+    std::deque<InputAndShape> add_ops;
+
+    // Prepare leaf AddN nodes for inputs of equal shape
+    for (int i = 0; i < shapes.size(); ++i) {
+      const auto node_name = leaf_node_name(i);
+      const auto& inputs = shape_sig_to_inputs[shape_sig(shapes[i])];
+      add_ops.push_back(AddInputsOfSymbolicallyEqualShape(*group.root_node,
+                                                          node_name, inputs));
+    }
 
-  // Create a new node for a AddOpsGroup and return it's name.
-  string RewriteAddOpsGroup(const AddOpsGroup& group) {
-    CHECK_GT(group.absorbed_nodes.size(), 0)
-        << "AddOpsGroup must have non empty absorbed nodes";
+    // Build up a tree of Add ops
+    int internal_nodes = 0;
+    do {
+      const InputAndShape lhs = add_ops.front();
+      add_ops.pop_front();
+      const InputAndShape rhs = add_ops.front();
+      add_ops.pop_front();
+      string name = add_ops.empty() ? OptimizedNodeName(root_scope_and_name)
+                                    : internal_node_name(internal_nodes++);
+      InputAndShape add = AddAggregatedInputs(*group.root_node, name, lhs, rhs);
+      add_ops.push_front(add);
+    } while (add_ops.size() > 1);
+
+    InputAndShape optimized_root_node = add_ops.front();
+    return optimized_root_node.input;
+  }
+
+  // Add 'AddN' node to aggregate inputs of symbolically equal shape
+  InputAndShape AddInputsOfSymbolicallyEqualShape(
+      const NodeDef& root_node, const string& node_name,
+      const std::vector<InputAndShape>& inputs) {
+    CHECK(!inputs.empty()) << "Inputs must be non-empty";
+
+    // Do not create redundant AddN nodes
+    if (inputs.size() == 1) {
+      return inputs[0];
+    }
 
-    // name for a new node constructed from AddOpsGroup
-    string node_name = AddOpsGroupName(group);
+    // get shape from representative element
+    auto shape = inputs[0].shape;
 
     // copy attributes from a root node
-    DataType dtype = group.root_node->attr().at("T").type();
+    DataType dtype = root_node.attr().at("T").type();
 
     // add new AddN node
-    NodeDef* added_node = AddEmptyNode(node_name);
-    added_node->set_op("AddN");
-    added_node->set_device(group.root_node->device());
-    (*added_node->mutable_attr())["T"].set_type(dtype);
-    (*added_node->mutable_attr())["N"].set_i(group.inputs.size());
-
-    // all inputs of absorbed nodes are added to the new node
-    for (const string& input : group.inputs) {
-      ctx_.node_map->AddOutput(input, node_name);
-      added_node->add_input(input);
+    NodeDef* node = AddEmptyNode(node_name);
+    node->set_op("AddN");
+    node->set_device(root_node.device());
+    (*node->mutable_attr())["T"].set_type(dtype);
+    (*node->mutable_attr())["N"].set_i(inputs.size());
+
+    for (const auto& inputAndShape : inputs) {
+      ctx_.node_map->AddOutput(inputAndShape.input, node_name);
+      node->add_input(inputAndShape.input);
     }
 
-    // Add frame dependencies that the original node might have had.
-    AddFrameControlDeps(group.root_node, {added_node}, "", {});
+    rewritten_nodes_.insert(node_name);
+    return InputAndShape(node_name, shape);
+  }
 
-    VLOG(1) << "Absorbed " << group.absorbed_nodes.size()
-            << " Add/AddN nodes from the graph";
+  // Add a single 'Add' node to sum two inputs
+  InputAndShape AddAggregatedInputs(const NodeDef& root_node,
+                                    const string& node_name,
+                                    const InputAndShape& left,
+                                    const InputAndShape& right) {
+    // copy attributes from a root node
+    DataType dtype = root_node.attr().at("T").type();
 
-    // keep track of nodes that were created or absorbed as a part of rewrite
-    rewritten_nodes_.insert(node_name);
-    for (const NodeDef* absorbed : group.absorbed_nodes) {
-      rewritten_nodes_.insert(absorbed->name());
-    }
+    // add new Add node
+    NodeDef* node = AddEmptyNode(node_name);
+    node->set_op("Add");
+    node->set_device(root_node.device());
+    (*node->mutable_attr())["T"].set_type(dtype);
+
+    ctx_.node_map->AddOutput(left.input, node_name);
+    ctx_.node_map->AddOutput(right.input, node_name);
 
-    return node_name;
+    node->add_input(left.input);
+    node->add_input(right.input);
+
+    rewritten_nodes_.insert(node_name);
+    return InputAndShape(
+        node_name, TensorShapeProto());  // shape is not important at this point
   }
 
   // keep nodes that were added or absorbed as a part of AddOpsGroup rewrite
@@ -1686,24 +1766,33 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
 Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
                                      const GrapplerItem& item,
                                      GraphDef* optimized_graph) {
-  optimized_graph_ = optimized_graph;
-  *optimized_graph_ = item.graph;
+  GrapplerItem optimized_item(item);
+  optimized_graph_ = &optimized_item.graph;
 
   // Set up helper data structures.
   nodes_to_preserve_ = item.NodesToPreserve();
   fetch_nodes_known_ = !item.fetch.empty();
   node_map_.reset(new NodeMap(optimized_graph_));
-  int num_frames;
-  TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
-                                               &frame_map_, &num_frames));
+
+  DedupComputations();
+
+  // Perform topological sort on the graph in order to help AddOpsRewrite to
+  // optimize larger subgraphs starting from the roots with more inputs.
+  TF_RETURN_IF_ERROR(TopologicalSort(optimized_graph_));
+
   // Shapes are only needed in aggressive mode.
   graph_properties_.reset(new GraphProperties(item));
   TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false));
 
+  // Identify loop frames
+  int num_frames;
+  TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
+                                               &frame_map_, &num_frames));
+
   // Perform the optimizations.
-  DedupComputations();
   TF_RETURN_IF_ERROR(SimplifyArithmeticOps());
 
+  optimized_graph->Swap(optimized_graph_);
   return Status::OK();
 }
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 965f0e9ea2..63a7b55893 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -69,7 +69,13 @@ class ArithmeticOptimizer : public GraphOptimizer {
     // optimization level by default.
     static ArithmeticOptimizerOptions Default(
         RewriterConfig::Toggle opt_level) {
-      return ArithmeticOptimizerOptions();
+      ArithmeticOptimizerOptions options;
+      // TODO(ezhulenev): enable combine_add_to_addn by default after 1.8
+      // release cut
+      if (opt_level == RewriterConfig::AGGRESSIVE) {
+        options.combine_add_to_addn = true;
+      }
+      return options;
     }
   };
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index ad3edc144a..ef3ed35fa6 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -156,25 +156,23 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   item.fetch = {"div"};
 
-  ArithmeticOptimizer optimizer;
-  GraphDef output;
   auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {});
   EXPECT_EQ(1, tensors_expected.size());
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
 
+  ArithmeticOptimizer optimizer;
+  GraphDef output;
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
   EXPECT_EQ(2, output.node_size());
-  const NodeDef& new_c1 = output.node(0);
-  EXPECT_EQ("c1", new_c1.name());
-  const NodeDef& new_div = output.node(1);
-  EXPECT_EQ("div", new_div.name());
-  EXPECT_EQ(2, new_div.input_size());
-  EXPECT_EQ("c1", new_div.input(0));
-  EXPECT_EQ("c1", new_div.input(1));
+
+  const NodeDef* new_c1 = node_map.GetNode("c1");
+  ASSERT_NE(new_c1, nullptr);
+
+  const NodeDef* new_div = node_map.GetNode("div");
+  ASSERT_NE(new_div, nullptr);
+  EXPECT_EQ(2, new_div->input_size());
+  EXPECT_EQ("c1", new_div->input(0));
+  EXPECT_EQ("c1", new_div->input(1));
 
   auto tensors = EvaluateNodes(output, item.fetch, {});
   EXPECT_EQ(1, tensors.size());
@@ -198,20 +196,18 @@ TEST_F(ArithmeticOptimizerTest, OpDeduppingAssertAndCheckNumerics) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(5, output.node_size());
-  const NodeDef& new_div = output.node(3);
-  EXPECT_EQ(4, new_div.input_size());
-  EXPECT_EQ("check1", new_div.input(0));
-  EXPECT_EQ("check1", new_div.input(1));
-  EXPECT_EQ("^assert1", new_div.input(2));
-  EXPECT_EQ("^assert1", new_div.input(3));
+  const NodeDef* new_div = node_map.GetNode("div");
+  ASSERT_NE(new_div, nullptr);
+  EXPECT_EQ(4, new_div->input_size());
+  EXPECT_EQ("check1", new_div->input(0));
+  EXPECT_EQ("check1", new_div->input(1));
+  EXPECT_EQ("^assert1", new_div->input(2));
+  EXPECT_EQ("^assert1", new_div->input(3));
 }
 
 TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) {
@@ -227,28 +223,24 @@ TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(4, output.node_size());
-  const NodeDef& new_c1 = output.node(0);
-  EXPECT_EQ("c1", new_c1.name());
-  const NodeDef& new_c2 = output.node(1);
-  EXPECT_EQ("c2", new_c2.name());
-  const NodeDef& new_mul1 = output.node(2);
-  EXPECT_EQ("mul1", new_mul1.name());
-  EXPECT_EQ(2, new_mul1.input_size());
-  EXPECT_EQ("c1", new_mul1.input(0));
-  EXPECT_EQ("c2", new_mul1.input(1));
-  const NodeDef& new_div1 = output.node(3);
-  EXPECT_EQ("div1", new_div1.name());
-  EXPECT_EQ(2, new_div1.input_size());
-  EXPECT_EQ("mul1", new_div1.input(0));
-  EXPECT_EQ("mul1", new_div1.input(1));
+  const NodeDef* new_c1 = node_map.GetNode("c1");
+  ASSERT_NE(new_c1, nullptr);
+  const NodeDef* new_c2 = node_map.GetNode("c2");
+  ASSERT_NE(new_c2, nullptr);
+  const NodeDef* new_mul1 = node_map.GetNode("mul1");
+  ASSERT_NE(new_mul1, nullptr);
+  EXPECT_EQ(2, new_mul1->input_size());
+  EXPECT_EQ("c1", new_mul1->input(0));
+  EXPECT_EQ("c2", new_mul1->input(1));
+  const NodeDef* new_div1 = node_map.GetNode("div1");
+  ASSERT_NE(new_div1, nullptr);
+  EXPECT_EQ(2, new_div1->input_size());
+  EXPECT_EQ("mul1", new_div1->input(0));
+  EXPECT_EQ("mul1", new_div1->input(1));
 }
 
 TEST_F(ArithmeticOptimizerTest, MulToSquare) {
@@ -364,26 +356,25 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimple) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(5, output.node_size());
-  const NodeDef& new_const = output.node(3);
-  EXPECT_EQ(OptimizedName("add_const"), new_const.name());
-  EXPECT_EQ("^x", new_const.input(0));
+
+  const NodeDef* new_const = node_map.GetNode(OptimizedName("add_const"));
+  ASSERT_NE(new_const, nullptr);
+  EXPECT_EQ("^x", new_const->input(0));
   EXPECT_EQ(std::string("\0\0\0@", 4),
-            new_const.attr().at("value").tensor().tensor_content());
-  const NodeDef& new_mul = output.node(4);
-  EXPECT_EQ(OptimizedName("add_mul"), new_mul.name());
-  EXPECT_EQ(OptimizedName("add_const"), new_mul.input(0));
-  EXPECT_EQ("x", new_mul.input(1));
-  const NodeDef& new_id = output.node(2);
-  EXPECT_EQ("id", new_id.name());
-  EXPECT_EQ(OptimizedName("add_mul"), new_id.input(0));
+            new_const->attr().at("value").tensor().tensor_content());
+
+  const NodeDef* new_mul = node_map.GetNode(OptimizedName("add_mul"));
+  ASSERT_NE(new_mul, nullptr);
+  EXPECT_EQ(OptimizedName("add_const"), new_mul->input(0));
+  EXPECT_EQ("x", new_mul->input(1));
+
+  const NodeDef* new_id = node_map.GetNode("id");
+  ASSERT_NE(new_id, nullptr);
+  EXPECT_EQ(OptimizedName("add_mul"), new_id->input(0));
 }
 
 TEST_F(ArithmeticOptimizerTest, TrivialSumsSimpleWithControlDep) {
@@ -398,27 +389,26 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimpleWithControlDep) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(6, output.node_size());
-  const NodeDef& new_const = output.node(4);
-  EXPECT_EQ(OptimizedName("add_const"), new_const.name());
-  EXPECT_EQ("^x", new_const.input(0));
+
+  const NodeDef* new_const = node_map.GetNode(OptimizedName("add_const"));
+  ASSERT_NE(new_const, nullptr);
+  EXPECT_EQ("^x", new_const->input(0));
   EXPECT_EQ(std::string("\0\0\0@", 4),
-            new_const.attr().at("value").tensor().tensor_content());
-  const NodeDef& new_mul = output.node(5);
-  EXPECT_EQ(OptimizedName("add_mul"), new_mul.name());
-  EXPECT_EQ(OptimizedName("add_const"), new_mul.input(0));
-  EXPECT_EQ("x", new_mul.input(1));
-  EXPECT_EQ("^y", new_mul.input(2));
-  const NodeDef& new_id = output.node(3);
-  EXPECT_EQ("id", new_id.name());
-  EXPECT_EQ(OptimizedName("add_mul"), new_id.input(0));
+            new_const->attr().at("value").tensor().tensor_content());
+
+  const NodeDef* new_mul = node_map.GetNode(OptimizedName("add_mul"));
+  ASSERT_NE(new_mul, nullptr);
+  EXPECT_EQ(OptimizedName("add_const"), new_mul->input(0));
+  EXPECT_EQ("x", new_mul->input(1));
+  EXPECT_EQ("^y", new_mul->input(2));
+
+  const NodeDef* new_id = node_map.GetNode("id");
+  ASSERT_NE(new_id, nullptr);
+  EXPECT_EQ(OptimizedName("add_mul"), new_id->input(0));
 }
 
 TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
@@ -458,25 +448,25 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
   EXPECT_EQ(17, output.node_size());
 
   const NodeDef* id_node = node_map.GetNode("id");
-  ASSERT_TRUE(id_node != nullptr);
+  ASSERT_NE(id_node, nullptr);
   EXPECT_EQ(1, id_node->input_size());
   EXPECT_EQ(HoistMulName("Add_6"), id_node->input(0));
 
   const NodeDef* mul_node = node_map.GetNode(HoistMulName("Add_6"));
-  ASSERT_TRUE(mul_node != nullptr);
+  ASSERT_NE(mul_node, nullptr);
   EXPECT_EQ(2, mul_node->input_size());
   EXPECT_EQ("Placeholder", mul_node->input(0));
   EXPECT_EQ(HoistAddName("Add_6"), mul_node->input(1));
 
   const NodeDef* add_6_node = node_map.GetNode(HoistAddName("Add_6"));
-  ASSERT_TRUE(add_6_node != nullptr);
+  ASSERT_NE(add_6_node, nullptr);
   EXPECT_EQ(3, add_6_node->input_size());
   EXPECT_EQ(HoistAddName("Add_4"), add_6_node->input(0));
   EXPECT_EQ(HoistAddName("Add_5"), add_6_node->input(1));
   EXPECT_EQ("^Placeholder", add_6_node->input(2));
 
   const NodeDef* add_4_node = node_map.GetNode(HoistAddName("Add_4"));
-  ASSERT_TRUE(add_4_node != nullptr);
+  ASSERT_NE(add_4_node, nullptr);
   EXPECT_EQ("Add", add_4_node->op());
   EXPECT_EQ(3, add_4_node->input_size());
   EXPECT_EQ(OptimizedName("Add_const"), add_4_node->input(0));
@@ -484,7 +474,7 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
   EXPECT_EQ("^Placeholder", add_4_node->input(2));
 
   const NodeDef* add_5_node = node_map.GetNode(HoistAddName("Add_5"));
-  ASSERT_TRUE(add_5_node != nullptr);
+  ASSERT_NE(add_5_node, nullptr);
   EXPECT_EQ("Add", add_5_node->op());
   EXPECT_EQ(3, add_5_node->input_size());
   EXPECT_EQ(OptimizedName("Add_const"), add_5_node->input(0));
@@ -492,14 +482,14 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
   EXPECT_EQ("^Placeholder", add_5_node->input(2));
 
   const NodeDef* add_const_node = node_map.GetNode(OptimizedName("Add_const"));
-  ASSERT_TRUE(add_const_node != nullptr);
+  ASSERT_NE(add_const_node, nullptr);
   EXPECT_EQ("Const", add_const_node->op());
   EXPECT_EQ(1, add_const_node->input_size());
   EXPECT_EQ("^Placeholder", add_const_node->input(0));
 
   const NodeDef* add_1_const_node =
       node_map.GetNode(OptimizedName("Add_1_const"));
-  ASSERT_TRUE(add_1_const_node != nullptr);
+  ASSERT_NE(add_1_const_node, nullptr);
   EXPECT_EQ("Const", add_1_const_node->op());
   EXPECT_EQ(1, add_1_const_node->input_size());
   EXPECT_EQ("^Placeholder", add_1_const_node->input(0));
@@ -550,17 +540,17 @@ TEST_F(ArithmeticOptimizerTest, HoistFactor) {
         EXPECT_EQ(9, output.node_size());
 
         const NodeDef* new_add_node = node_map.GetNode(HoistAddName("add"));
-        ASSERT_TRUE(new_add_node != nullptr) << "Hoisted Add node not found";
+        ASSERT_NE(new_add_node, nullptr) << "Hoisted Add node not found";
         EXPECT_EQ("y1", new_add_node->input(0));
         EXPECT_EQ("y2", new_add_node->input(1));
 
         const NodeDef* new_mul_node = node_map.GetNode(HoistMulName("add"));
-        ASSERT_TRUE(new_mul_node != nullptr) << "Hoisted Mul node not found";
+        ASSERT_NE(new_mul_node, nullptr) << "Hoisted Mul node not found";
         EXPECT_EQ("x", new_mul_node->input(0));
         EXPECT_EQ(new_add_node->name(), new_mul_node->input(1));
 
         const NodeDef* id_node = node_map.GetNode("id");
-        ASSERT_TRUE(id_node != nullptr) << "Id node not found";
+        ASSERT_NE(id_node, nullptr) << "Id node not found";
         EXPECT_EQ("id", id_node->name());
         EXPECT_EQ(HoistMulName("add"), id_node->input(0));
       }
@@ -581,18 +571,17 @@ TEST_F(ArithmeticOptimizerTest, FuseConjAndTranspose) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(7, output.node_size());
-  EXPECT_EQ(OptimizedName("trans_fused"), output.node(6).name());
-  EXPECT_EQ("ConjugateTranspose", output.node(6).op());
-  EXPECT_EQ("z", output.node(6).input(0));
-  EXPECT_EQ("perm", output.node(6).input(1));
+
+  const NodeDef* trans_fused_node =
+      node_map.GetNode(OptimizedName("trans_fused"));
+  ASSERT_NE(trans_fused_node, nullptr);
+  EXPECT_EQ("ConjugateTranspose", trans_fused_node->op());
+  EXPECT_EQ("z", trans_fused_node->input(0));
+  EXPECT_EQ("perm", trans_fused_node->input(1));
 }
 
 TEST_F(ArithmeticOptimizerTest, FuseConjAndConjugateTranspose) {
@@ -609,14 +598,16 @@ TEST_F(ArithmeticOptimizerTest, FuseConjAndConjugateTranspose) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(7, output.node_size());
-  EXPECT_EQ(OptimizedName("conjugate_trans_fused"), output.node(6).name());
-  EXPECT_EQ("Transpose", output.node(6).op());
-  EXPECT_EQ("z", output.node(6).input(0));
-  EXPECT_EQ("perm", output.node(6).input(1));
+
+  const NodeDef* conjugate_trans_fused_node =
+      node_map.GetNode(OptimizedName("conjugate_trans_fused"));
+  EXPECT_EQ("Transpose", conjugate_trans_fused_node->op());
+  EXPECT_EQ("z", conjugate_trans_fused_node->input(0));
+  EXPECT_EQ("perm", conjugate_trans_fused_node->input(1));
 }
 
 TEST_F(ArithmeticOptimizerTest, FuseTransposeAndConj) {
@@ -632,18 +623,16 @@ TEST_F(ArithmeticOptimizerTest, FuseTransposeAndConj) {
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
-  Status status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
-  // Run the optimizer twice to make sure the rewrite is idempotent.
-  item.graph.Swap(&output);
-  status = optimizer.Optimize(nullptr, item, &output);
-  TF_EXPECT_OK(status);
+  OptimizeTwice(&optimizer, &item, &output);
+  NodeMap node_map(&output);
 
   EXPECT_EQ(7, output.node_size());
-  EXPECT_EQ(OptimizedName("conj_fused"), output.node(6).name());
-  EXPECT_EQ("ConjugateTranspose", output.node(6).op());
-  EXPECT_EQ("z", output.node(6).input(0));
-  EXPECT_EQ("perm", output.node(6).input(1));
+
+  const NodeDef* conj_fused_node =
+      node_map.GetNode(OptimizedName("conj_fused"));
+  EXPECT_EQ("ConjugateTranspose", conj_fused_node->op());
+  EXPECT_EQ("z", conj_fused_node->input(0));
+  EXPECT_EQ("perm", conj_fused_node->input(1));
 }
 
 TEST_F(ArithmeticOptimizerTest, FoldTransposeIntoMatMul) {
@@ -668,23 +657,22 @@ TEST_F(ArithmeticOptimizerTest, FoldTransposeIntoMatMul) {
 
     ArithmeticOptimizer optimizer;
     GraphDef output;
-    Status status = optimizer.Optimize(nullptr, item, &output);
-    TF_EXPECT_OK(status);
-    // Run the optimizer twice to make sure the rewrite is idempotent.
-    item.graph.Swap(&output);
-    status = optimizer.Optimize(nullptr, item, &output);
-    TF_EXPECT_OK(status);
+    OptimizeTwice(&optimizer, &item, &output);
+    NodeMap node_map(&output);
 
     EXPECT_EQ(7, output.node_size());
-    EXPECT_EQ(OptimizedName("matmul_fused"), output.node(6).name());
-    EXPECT_EQ("a", output.node(6).input(0));
-    EXPECT_EQ("b", output.node(6).input(1));
+
+    const NodeDef* matmul_fused_node =
+        node_map.GetNode(OptimizedName("matmul_fused"));
+    ASSERT_NE(matmul_fused_node, nullptr);
+    EXPECT_EQ("a", matmul_fused_node->input(0));
+    EXPECT_EQ("b", matmul_fused_node->input(1));
     if (matmul_type == "BatchMatMul") {
-      EXPECT_TRUE(output.node(6).attr().at("adj_x").b());
-      EXPECT_TRUE(output.node(6).attr().at("adj_y").b());
+      EXPECT_TRUE(matmul_fused_node->attr().at("adj_x").b());
+      EXPECT_TRUE(matmul_fused_node->attr().at("adj_y").b());
     } else {
-      EXPECT_TRUE(output.node(6).attr().at("transpose_a").b());
-      EXPECT_TRUE(output.node(6).attr().at("transpose_b").b());
+      EXPECT_TRUE(matmul_fused_node->attr().at("transpose_a").b());
+      EXPECT_TRUE(matmul_fused_node->attr().at("transpose_b").b());
     }
   }
 }
@@ -1322,8 +1310,8 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfIdenticalShape) {
 
   // check add tree was replaced with AddN
   const NodeDef* collapsed_add =
-      node_map.GetNode("y/ArithmeticOptimizer/AddOpsRewrite_Add_abc_Add_ab");
-  ASSERT_TRUE(collapsed_add != nullptr);
+      node_map.GetNode("y/ArithmeticOptimizer/AddOpsRewrite_Add_abc");
+  ASSERT_NE(collapsed_add, nullptr);
 
   EXPECT_EQ("AddN", collapsed_add->op());
   EXPECT_EQ(3, collapsed_add->input_size());
@@ -1333,7 +1321,7 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfIdenticalShape) {
 
   // check output was re-wired to new node
   const NodeDef* updated_outputs = node_map.GetNode("outputs");
-  ASSERT_TRUE(updated_outputs != nullptr);
+  ASSERT_NE(updated_outputs, nullptr);
 
   EXPECT_EQ(collapsed_add->name(), updated_outputs->input(0));
 }
@@ -1381,8 +1369,8 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MultiplePasses) {
 
   // check left Add subtree replaced with AddN
   const NodeDef* collapsed_left =
-      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_abc_Add_ab");
-  ASSERT_TRUE(collapsed_left != nullptr);
+      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_abc");
+  ASSERT_NE(collapsed_left, nullptr);
 
   EXPECT_EQ("AddN", collapsed_left->op());
   EXPECT_EQ(3, collapsed_left->input_size());
@@ -1392,8 +1380,8 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MultiplePasses) {
 
   // check right Add subtree replaced with AddN
   const NodeDef* collapsed_right =
-      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_xyz_Add_xy");
-  ASSERT_TRUE(collapsed_right != nullptr);
+      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_xyz");
+  ASSERT_NE(collapsed_right, nullptr);
 
   EXPECT_EQ("AddN", collapsed_right->op());
   EXPECT_EQ(3, collapsed_right->input_size());
@@ -1403,7 +1391,7 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MultiplePasses) {
 
   // check that Mul inputs re-wired to new Nodes
   const NodeDef* updated_mul = node_map.GetNode("Mul");
-  ASSERT_TRUE(updated_mul != nullptr);
+  ASSERT_NE(updated_mul, nullptr);
 
   EXPECT_EQ("Mul", updated_mul->op());
   EXPECT_EQ(2, updated_mul->input_size());
@@ -1444,9 +1432,9 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddInputMultipleTimes) {
   NodeMap node_map(&output);
 
   // check Add tree replaced with AddN
-  const NodeDef* collapsed_add = node_map.GetNode(
-      "ArithmeticOptimizer/AddOpsRewrite_Add_all_Add_ab_Add_bc");
-  ASSERT_TRUE(collapsed_add != nullptr);
+  const NodeDef* collapsed_add =
+      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_all");
+  ASSERT_NE(collapsed_add, nullptr);
 
   EXPECT_EQ("AddN", collapsed_add->op());
   EXPECT_EQ(4, collapsed_add->input_size());
@@ -1496,8 +1484,8 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfSymbolicallyEqualShape) {
 
   // check add tree was replaced with AddN
   const NodeDef* collapsed_add =
-      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_abc_Add_ab");
-  ASSERT_TRUE(collapsed_add != nullptr);
+      node_map.GetNode("ArithmeticOptimizer/AddOpsRewrite_Add_abc");
+  ASSERT_NE(collapsed_add, nullptr);
   EXPECT_EQ("AddN", collapsed_add->op());
   EXPECT_EQ(3, collapsed_add->input_size());
   EXPECT_EQ("a", collapsed_add->input(0));
@@ -1506,10 +1494,173 @@ TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_AddOpsOfSymbolicallyEqualShape) {
 
   // check output was re-wired to new node
   const NodeDef* updated_outputs = node_map.GetNode("outputs");
-  ASSERT_TRUE(updated_outputs != nullptr);
+  ASSERT_NE(updated_outputs, nullptr);
   EXPECT_EQ(collapsed_add->name(), updated_outputs->input(0));
 }
 
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MinimizeBCast) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  auto a = ops::Variable(s.WithOpName("a"), {32}, DT_FLOAT);
+  auto b = ops::Variable(s.WithOpName("b"), {32, 32}, DT_FLOAT);
+  auto c = ops::Variable(s.WithOpName("c"), {32, 32, 32}, DT_FLOAT);
+  auto add_ab = ops::Add(s.WithOpName("Add_ab"), a, b);
+  auto add_abc = ops::Add(s.WithOpName("Add_abc"), add_ab, c);
+
+  auto x = ops::Variable(s.WithOpName("x"), {32}, DT_FLOAT);
+  auto y = ops::Variable(s.WithOpName("y"), {32, 32}, DT_FLOAT);
+  auto z = ops::Variable(s.WithOpName("z"), {32, 32, 32}, DT_FLOAT);
+  auto add_xy = ops::Add(s.WithOpName("Add_xy"), x, y);
+  auto add_xyz = ops::Add(s.WithOpName("Add_xyz"), add_xy, z);
+
+  auto add_all = ops::Add(s.WithOpName("AddAll"), add_abc, add_xyz);
+  auto outputs = ops::Identity(s.WithOpName("outputs"), add_all);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyAddToAddNCombining(&optimizer);
+
+  OptimizeAndPrune(&optimizer, &item, &output);
+
+  // We expect the following rewrite(s) to occur:
+  //  1) [a, x], [b, y], [c, z] - aggregate same shapes first
+  //  2) Build an aggregation tree minimizing cost of broadcast
+  //
+  //         +                              +
+  //      /     \                       /       \
+  //     +       +                     +       AddN(c, z)
+  //    / \     / \                 /     \
+  //   +   c   x   + -->    AddN(a, x)  AddN(b, y)
+  //  / \         / \
+  // a   b       y   z
+  EXPECT_EQ(12, output.node_size());
+  NodeMap node_map(&output);
+
+  // expected names of outer and inner nodes
+  string outer_add_name = "ArithmeticOptimizer/AddOpsRewrite_AddAll";
+  string outer_0_add_name =
+      "ArithmeticOptimizer/AddOpsRewrite_Internal_0_AddAll";
+  string inner_0_add_name = "ArithmeticOptimizer/AddOpsRewrite_Leaf_0_AddAll";
+  string inner_1_add_name = "ArithmeticOptimizer/AddOpsRewrite_Leaf_1_AddAll";
+  string inner_2_add_name = "ArithmeticOptimizer/AddOpsRewrite_Leaf_2_AddAll";
+
+  // Add [a, x] first
+  const NodeDef* add_ax_node = node_map.GetNode(inner_0_add_name);
+  ASSERT_NE(add_ax_node, nullptr);
+  EXPECT_EQ("AddN", add_ax_node->op());
+  EXPECT_EQ(2, add_ax_node->input_size());
+  EXPECT_EQ("a", add_ax_node->input(0));
+  EXPECT_EQ("x", add_ax_node->input(1));
+
+  // Then add [b, y]
+  const NodeDef* add_by_node = node_map.GetNode(inner_1_add_name);
+  ASSERT_NE(add_by_node, nullptr);
+  EXPECT_EQ("AddN", add_by_node->op());
+  EXPECT_EQ(2, add_by_node->input_size());
+  EXPECT_EQ("b", add_by_node->input(0));
+  EXPECT_EQ("y", add_by_node->input(1));
+
+  // Then add [c, z]
+  const NodeDef* add_cz_node = node_map.GetNode(inner_2_add_name);
+  ASSERT_NE(add_cz_node, nullptr);
+  EXPECT_EQ("AddN", add_cz_node->op());
+  EXPECT_EQ(2, add_cz_node->input_size());
+  EXPECT_EQ("c", add_cz_node->input(0));
+  EXPECT_EQ("z", add_cz_node->input(1));
+
+  // Then add results together starting from smaller shapes [a, x] + [b, y]
+  const NodeDef* outer_0_node = node_map.GetNode(outer_0_add_name);
+  ASSERT_NE(outer_0_node, nullptr);
+  EXPECT_EQ("Add", outer_0_node->op());
+  EXPECT_EQ(2, outer_0_node->input_size());
+  EXPECT_EQ(inner_0_add_name, outer_0_node->input(0));
+  EXPECT_EQ(inner_1_add_name, outer_0_node->input(1));
+
+  // And finally top level Add node
+  const NodeDef* outer_node = node_map.GetNode(outer_add_name);
+  ASSERT_NE(outer_node, nullptr);
+  EXPECT_EQ("Add", outer_node->op());
+  EXPECT_EQ(2, outer_node->input_size());
+  EXPECT_EQ(outer_0_add_name, outer_node->input(0));
+  EXPECT_EQ(inner_2_add_name, outer_node->input(1));
+
+  // And outputs reading new top level Add node
+  const NodeDef* updated_outputs = node_map.GetNode("outputs");
+  ASSERT_NE(updated_outputs, nullptr);
+  EXPECT_EQ(outer_add_name, updated_outputs->input(0));
+}
+
+TEST_F(ArithmeticOptimizerTest, AddOpsRewrite_MinimizeBCastWithSymbolicShapes) {
+  tensorflow::Scope s = tensorflow::Scope::NewRootScope();
+
+  // We have a small input with one unknown dimension
+  auto small = ops::Variable(s.WithOpName("small"), {-1, 1, 1}, DT_FLOAT);
+
+  // And second input which is larger, but has the same unknown dimension
+  // device spec prevents this node from rewriting
+  auto d = "/job:do_not_rewrite_me";
+  auto v = ops::Variable(s.WithOpName("v"), {1, 32, 32}, DT_FLOAT);
+  auto large = ops::Add(s.WithOpName("large").WithDevice(d), small, v);
+
+  // [a, c] have {?, 1, 1} shape, [b] has {?, 32, 32}
+  auto a = ops::Sqrt(s.WithOpName("a"), small);
+  auto b = ops::Square(s.WithOpName("b"), large);
+  auto c = ops::Round(s.WithOpName("c"), small);
+
+  // [add_ab, add_abc] shape must be inferred from inputs
+  auto add_ab = ops::Add(s.WithOpName("Add_ab"), a, b);
+  auto add_abc = ops::Add(s.WithOpName("Add_abc"), add_ab, c);
+
+  auto outputs = ops::Identity(s.WithOpName("outputs"), add_abc);
+
+  GrapplerItem item;
+  item.fetch = {"outputs"};
+  TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
+  GraphDef output;
+  ArithmeticOptimizer optimizer;
+  EnableOnlyAddToAddNCombining(&optimizer);
+  OptimizeAndPrune(&optimizer, &item, &output);
+
+  // We expect the following rewrite(s) to occur: it's much cheaper to add small
+  // tensors, and do the broadcast just once
+  //
+  //     +                  +
+  //    / \                / \
+  //   +   c      -->     +   b
+  //  / \                / \
+  // a   b              a   c
+  EXPECT_EQ(9, output.node_size());
+  NodeMap node_map(&output);
+
+  // expected names of outer and inner nodes
+  string outer_add_name = "ArithmeticOptimizer/AddOpsRewrite_Add_abc";
+  string inner_add_name = "ArithmeticOptimizer/AddOpsRewrite_Leaf_0_Add_abc";
+
+  // outer Add node
+  const NodeDef* outer_add = node_map.GetNode(outer_add_name);
+  ASSERT_NE(outer_add, nullptr);
+  EXPECT_EQ("Add", outer_add->op());
+  EXPECT_EQ(inner_add_name, outer_add->input(0));
+  EXPECT_EQ("b", outer_add->input(1));
+
+  // inner AddN node
+  const NodeDef* inner_add = node_map.GetNode(inner_add_name);
+  ASSERT_NE(inner_add, nullptr);
+  EXPECT_EQ(2, inner_add->input_size());
+  EXPECT_EQ("a", inner_add->input(0));
+  EXPECT_EQ("c", inner_add->input(1));
+
+  // check output was re-wired to new node
+  const NodeDef* updated_outputs = node_map.GetNode("outputs");
+  ASSERT_NE(updated_outputs, nullptr);
+  EXPECT_EQ(outer_add_name, updated_outputs->input(0));
+}
+
 TEST_F(ArithmeticOptimizerTest, RemoveNegation) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
   auto x = ops::Variable(s.WithOpName("x"), {2, 2}, DT_FLOAT);
diff --git a/tensorflow/core/grappler/optimizers/symbolic_shapes.h b/tensorflow/core/grappler/optimizers/symbolic_shapes.h
index a9dcf44e23..eb79bab314 100644
--- a/tensorflow/core/grappler/optimizers/symbolic_shapes.h
+++ b/tensorflow/core/grappler/optimizers/symbolic_shapes.h
@@ -31,8 +31,8 @@ bool IsUnknown(const TensorShapeProto::Dim& dim);
 bool ShapeIsSymbolicallyDefined(const TensorShapeProto& shape);
 bool ShapeIsSymbolicallyDefined(const OpInfo::TensorProperties& properties);
 
-// Shapes are symbolically equal, if they have the same rank, they are
-// they are known or symbolically defined, and have matching dimensions.
+// Shapes are symbolically equal, if they have the same rank, they are known or
+// symbolically defined, and have matching dimensions.
 bool ShapesSymbolicallyEqual(const TensorShapeProto& left,
                              const TensorShapeProto& right);
 bool ShapesSymbolicallyEqual(const OpInfo::TensorProperties& left,
-- 
GitLab


From 5531482360f803030564d768752aa38800b39636 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 2 Apr 2018 12:58:52 -0700
Subject: [PATCH 814/960] eager: Tweak error message.

Motivated by
https://stackoverflow.com/questions/49616532/a-tensorflow-eager-gpu-error/49617069

PiperOrigin-RevId: 191334050
---
 tensorflow/c/eager/c_api.cc | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/tensorflow/c/eager/c_api.cc b/tensorflow/c/eager/c_api.cc
index bb1492fca2..c96a38dec3 100644
--- a/tensorflow/c/eager/c_api.cc
+++ b/tensorflow/c/eager/c_api.cc
@@ -496,9 +496,11 @@ tensorflow::Status ValidateInputTypeAndPlacement(
               expected_device->name(), " but is actually on ",
               actual_device->name(), " (operation running on ",
               op_device->name(), ")",
-              " Tensors can be copied explicitly using .gpu() or .cpu(),"
-              " or transparently copied by using tfe.enable_eager_execution("
-              "tfe.DEVICE_PLACEMENT_SILENT). Copying tensors between devices"
+              " Tensors can be copied explicitly using .gpu() or .cpu() "
+              "methods,"
+              " or transparently copied by using tf.enable_eager_execution("
+              "device_policy=tfe.DEVICE_PLACEMENT_SILENT). Copying tensors "
+              "between devices"
               " may slow down your model");
         case tensorflow::DEVICE_PLACEMENT_WARN:
           LOG(WARNING) << "before computing " << op->name << " input #" << i
-- 
GitLab


From dbb11a8593570ad4d7af96da9a7e314f0209844c Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Mon, 2 Apr 2018 13:17:14 -0700
Subject: [PATCH 815/960] Reduce overhead for eager ops

- Call _context_handle in the fastpath. Fall back to slow path if it is not
  initialized.
  A better fix would be to not initialize handle and devices lazily (and not
  have to pay that function call in the slow path either), but that
  seems to break all GPU/TPU tests. I'm not as yet really familiar with how
  devices are recognized, but I'd be happy to hear any ideas you may have to
  fix this.
- context.context() is monkey patched to remove the "is None" check once we
  know the context is correctly initialized. Ideally we would be able to remove
  this function call as well.
- Maintain is_eager instead of doing the comparison every time. Also, in the
  fastpath, inline the check directly instead of paying the function call cost.
- Inline _eager_context.device_name instead of get the device_name property to
  not pay the function call cost

gen_array_ops.identity Old: 216706.923837 examples/sec (4.61452722549)
gen_array_ops.identity New: 290819.129714 examples/sec (3.43856334686)

PiperOrigin-RevId: 191336857
---
 tensorflow/python/eager/context.py            | 25 +++++++++++--------
 .../python/eager/python_eager_op_gen.cc       |  8 +++---
 tensorflow/python/eager/pywrap_tfe_src.cc     |  9 +++++++
 tensorflow/python/framework/ops.py            |  4 +++
 4 files changed, 32 insertions(+), 14 deletions(-)

diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 6ad9e0d88f..99ec895b54 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -86,6 +86,7 @@ class _EagerContext(threading.local):
     self.device_spec = pydev.DeviceSpec.from_string("")
     self.device_name = self.device_spec.to_string()
     self.mode = _default_mode
+    self.is_eager = _default_mode == EAGER_MODE
     self.scope_name = ""
     self.recording_summaries = False
     self.summary_writer_resource = None
@@ -283,9 +284,12 @@ class Context(object):
 
   @tf_contextlib.contextmanager
   def _mode(self, mode):
+    """A context manager to allow setting the mode to EAGER/GRAPH."""
     ctx = self._eager_context
     old_mode = ctx.mode
+    old_is_eager = ctx.is_eager
     ctx.mode = mode
+    ctx.is_eager = mode == EAGER_MODE
     if mode == EAGER_MODE:
       # Entering graph mode does not provide us with sufficient information to
       # record a context switch; graph-based context switches are only logged
@@ -294,13 +298,14 @@ class Context(object):
     try:
       yield
     finally:
+      ctx.is_eager = old_is_eager
       ctx.mode = old_mode
       if mode == EAGER_MODE:
         self.context_switches.pop()
 
   def executing_eagerly(self):
     """Returns True if current thread has eager executing enabled."""
-    return self._eager_context.mode == EAGER_MODE
+    return self._eager_context.is_eager
 
   def scalar_cache(self):
     """Per-device cache for scalars."""
@@ -508,23 +513,19 @@ class Context(object):
     To retrieve the accumulated metadata call context.export_run_metadata()
     and to stop tracing call context.disable_run_metadata().
     """
-    if not self._context_handle:
-      self._initialize_handle_and_devices()
-    pywrap_tensorflow.TFE_ContextEnableRunMetadata(self._context_handle)
+    pywrap_tensorflow.TFE_ContextEnableRunMetadata(self._handle)
 
   @tf_contextlib.contextmanager
   def device_policy(self, policy):
-    if not self._context_handle:
-      self._initialize_handle_and_devices()
-    old = pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy(
-        self._context_handle)
+    handle = self._handle
+    old = pywrap_tensorflow.TFE_ContextGetDevicePlacementPolicy(handle)
     pywrap_tensorflow.TFE_ContextSetThreadLocalDevicePlacementPolicy(
-        self._handle, policy)
+        handle, policy)
     try:
       yield
     finally:
       pywrap_tensorflow.TFE_ContextSetThreadLocalDevicePlacementPolicy(
-          self._handle, old)
+          handle, old)
 
   def disable_run_metadata(self):
     """Disables tracing of op execution via RunMetadata."""
@@ -575,6 +576,10 @@ def context():
   return _context
 
 
+def context_safe():
+  return _context
+
+
 # TODO(agarwal): remove this.
 def get_default_context():
   """Same as context."""
diff --git a/tensorflow/python/eager/python_eager_op_gen.cc b/tensorflow/python/eager/python_eager_op_gen.cc
index c2ce8efd7f..06185904e7 100644
--- a/tensorflow/python/eager/python_eager_op_gen.cc
+++ b/tensorflow/python/eager/python_eager_op_gen.cc
@@ -367,7 +367,7 @@ void GenEagerPythonOp::HandleGraphMode(const string& function_setup) {
   // Handle graph-mode case
   strings::StrAppend(&result_,
                      "  _ctx = _context.context()\n"
-                     "  if not _ctx.executing_eagerly():\n",
+                     "  if not _ctx._eager_context.is_eager:\n",
                      function_setup,
                      "    _, _, _op = _op_def_lib._apply_op_helper(\n");
   AddBodyNoReturn("        ");
@@ -712,9 +712,9 @@ bool GenEagerPythonOp::AddEagerFallbackCode(
 }
 
 void GenEagerPythonOp::AddEagerFastPathExecute() {
-  string fastpath_execute_params =
-      strings::StrCat("_ctx._handle, _ctx.device_name, \"", op_def_.name(),
-                      "\", ", "name, _ctx._post_execution_callbacks");
+  string fastpath_execute_params = strings::StrCat(
+      "_ctx._context_handle, _ctx._eager_context.device_name, \"",
+      op_def_.name(), "\", ", "name, _ctx._post_execution_callbacks");
   string fallback_params;
 
   for (int i = 0; i < api_def_.in_arg_size(); i++) {
diff --git a/tensorflow/python/eager/pywrap_tfe_src.cc b/tensorflow/python/eager/pywrap_tfe_src.cc
index 8a398f6447..d99bd0b0ff 100644
--- a/tensorflow/python/eager/pywrap_tfe_src.cc
+++ b/tensorflow/python/eager/pywrap_tfe_src.cc
@@ -1844,6 +1844,15 @@ PyObject* TFE_Py_FastPathExecute_C(PyObject*, PyObject* args) {
 
   op_exec_info.ctx = reinterpret_cast<TFE_Context*>(
       PyCapsule_GetPointer(PyTuple_GET_ITEM(args, 0), nullptr));
+
+  if (op_exec_info.ctx == nullptr) {
+    // The context hasn't been initialized. It will be in the slow path.
+    RaiseFallbackException(
+        "This function does not handle the case of the path where "
+        "all inputs are not already EagerTensors.");
+    return nullptr;
+  }
+
   op_exec_info.device_name = GetDeviceName(PyTuple_GET_ITEM(args, 1));
   op_exec_info.op_name = PyTuple_GET_ITEM(args, 2);
   op_exec_info.op_def = GetOpDef(op_exec_info.op_name);
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 22b621e4cb..c0baeb98ac 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -5343,6 +5343,10 @@ def enable_eager_execution(config=None, device_policy=None,
     raise ValueError(
         "tf.enable_eager_execution must be called at program startup.")
 
+  # Monkey patch to get rid of an unnecessary conditional since the context is
+  # now initialized.
+  context.context = context.context_safe
+
 
 def eager_run(main=None, argv=None):
   """Runs the program with an optional main function and argv list.
-- 
GitLab


From 2f64599f850f1560e4c6ef2c5869156f39c2ad44 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 13:41:26 -0700
Subject: [PATCH 816/960] Verify that a %send and a %recv on the same channel,
 don't end up landing on the same device. Also verify that send/send-done and
 recv/recv-done are on the same device/module.

PiperOrigin-RevId: 191340724
---
 .../xla/service/hlo_module_group_metadata.cc  | 21 +++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
index fa5dcb0b36..54c34ce116 100644
--- a/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
+++ b/tensorflow/compiler/xla/service/hlo_module_group_metadata.cc
@@ -313,6 +313,27 @@ Status HloModuleGroupMetadata::VerifyChannelInstructions() {
     if (!ShapeUtil::Compatible(send_shape, recv_shape)) {
       return FailedPrecondition("send/recv shapes do not match");
     }
+    const HloModule* send_module = channel.send->parent()->parent();
+    const HloModule* send_done_module = channel.send_done->parent()->parent();
+    if (send_module != send_done_module) {
+      return FailedPrecondition(
+          "send and send-done (channel=%lld) must be on the same device: %lld "
+          "vs. %lld",
+          channel.id, GetModuleId(send_module), GetModuleId(send_done_module));
+    }
+    const HloModule* recv_module = channel.recv->parent()->parent();
+    const HloModule* recv_done_module = channel.recv_done->parent()->parent();
+    if (recv_module != recv_done_module) {
+      return FailedPrecondition(
+          "recv and recv-done (channel=%lld) must be on the same device: %lld "
+          "vs. %lld",
+          channel.id, GetModuleId(recv_module), GetModuleId(recv_done_module));
+    }
+    if (send_module == recv_module) {
+      return FailedPrecondition(
+          "send and recv (channel=%lld) must be on different devices: %lld",
+          channel.id, GetModuleId(send_module));
+    }
   }
 
   // Check if channel instructions are used only in allowed computations.
-- 
GitLab


From cf7d4f0ecfbdd7002d3c89a7da75d534b1c7d806 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 2 Apr 2018 13:53:52 -0700
Subject: [PATCH 817/960] Don't bypass reshape nodes that anchor control
 dependencies

PiperOrigin-RevId: 191342646
---
 .../grappler/optimizers/arithmetic_optimizer.cc     | 13 ++++++-------
 tensorflow/core/grappler/utils.cc                   |  8 ++++++++
 tensorflow/core/grappler/utils.h                    |  3 +++
 3 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 882e4d9a40..6e27259998 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -1344,19 +1344,18 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
     int output_pos = 0;
     string input_node_name = ParseNodeName(node->input(0), &output_pos);
     const NodeDef* input = node_map_->GetNode(input_node_name);
-    if (input->op() == "Reshape") {
+    if (input->op() == "Reshape" && !HasControlInputs(*input)) {
       reshape->set_input(0, input->input(0));
       node_map_->UpdateInput(reshape->name(), input->name(), input->input(0));
       nodes_to_simplify->PushBack(reshape);
       return reshape->name();
     }
 
-    // If the reshape is a no-op, forward its input to its consumers. This is
-    // considered aggressive, because users may state that the placeholder
-    // outputs tensors of shape [M, N] while feeding it with tensors of shape
-    // [M*N] (or worse). The reshape nodes are then necessary to update the
-    // tensor metadata to the required shape.
-    if (ReshapeIsIdentity(*reshape, *input, output_pos, *graph_properties_)) {
+    // If the reshape is a no-op, forward its input to its consumers, unless it
+    // anchors a control dependency since we want to make sure that control
+    // dependency is triggered.
+    if (ReshapeIsIdentity(*reshape, *input, output_pos, *graph_properties_) &&
+        !HasControlInputs(*reshape)) {
       return reshape->input(0);
     }
   }
diff --git a/tensorflow/core/grappler/utils.cc b/tensorflow/core/grappler/utils.cc
index 86a6d5000d..5893f286ed 100644
--- a/tensorflow/core/grappler/utils.cc
+++ b/tensorflow/core/grappler/utils.cc
@@ -255,6 +255,14 @@ int NumOutputs(const NodeDef& node, GraphDef* graph) {
   return num_outputs;
 }
 
+bool HasControlInputs(const NodeDef& node) {
+  int num_inputs = node.input_size();
+  if (num_inputs > 0 && IsControlInput(node.input(num_inputs - 1))) {
+    return true;
+  }
+  return false;
+}
+
 int NumNonControlInputs(const NodeDef& node) {
   int num_inputs = node.input_size();
   for (const string& input : node.input()) {
diff --git a/tensorflow/core/grappler/utils.h b/tensorflow/core/grappler/utils.h
index 7aa31939f5..11555d712a 100644
--- a/tensorflow/core/grappler/utils.h
+++ b/tensorflow/core/grappler/utils.h
@@ -138,6 +138,9 @@ string AsControlDependency(const string& node);
 // some of the outputs may be unconnected.
 int NumOutputs(const NodeDef& node, GraphDef* graph);
 
+// Returns true iff the node has at least one control input.
+bool HasControlInputs(const NodeDef& node);
+
 // Number of connected non-control inputs.
 int NumNonControlInputs(const NodeDef& node);
 
-- 
GitLab


From e4c5a755967a6b5442d164e43f03f0672610c998 Mon Sep 17 00:00:00 2001
From: Patrick Nguyen <drpng@google.com>
Date: Mon, 2 Apr 2018 14:03:53 -0700
Subject: [PATCH 818/960] Export the rest of If, While, and For.

We keep _If and _While. This moves the tests and python generators.
The operators are not part of the public tensorflow API.

PiperOrigin-RevId: 191344237
---
 .../core/api_def/base_api/api_def_For.pbtxt   |  29 ++
 .../core/api_def/base_api/api_def_If.pbtxt    |  40 +++
 .../core/api_def/base_api/api_def_While.pbtxt |  33 +++
 .../core/api_def/python_api/api_def_For.pbtxt |   1 +
 .../core/api_def/python_api/api_def_If.pbtxt  |   1 +
 .../api_def/python_api/api_def_While.pbtxt    |   1 +
 tensorflow/core/kernels/functional_ops.cc     | 189 ++++++++++--
 tensorflow/core/ops/functional_ops.cc         |  41 ++-
 .../kernel_tests/functional_ops_test.py       | 272 ++++++++++++++++++
 tensorflow/python/ops/functional_ops.py       | 260 ++++++++++++++++-
 10 files changed, 838 insertions(+), 29 deletions(-)
 create mode 100644 tensorflow/core/api_def/base_api/api_def_For.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_If.pbtxt
 create mode 100644 tensorflow/core/api_def/base_api/api_def_While.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_For.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_If.pbtxt
 create mode 100644 tensorflow/core/api_def/python_api/api_def_While.pbtxt

diff --git a/tensorflow/core/api_def/base_api/api_def_For.pbtxt b/tensorflow/core/api_def/base_api/api_def_For.pbtxt
new file mode 100644
index 0000000000..a7cd8e1a26
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_For.pbtxt
@@ -0,0 +1,29 @@
+op {
+  graph_op_name: "For"
+  in_arg { name: "start" description: "The lower bound. An int32" }
+  in_arg { name: "limit" description: "The upper bound. An int32" }
+  in_arg { name: "delta" description: "The increment. An int32" }
+  in_arg {
+    name: "input"
+    description: "A list of input tensors whose types are T."
+  }
+  out_arg {
+    name: "output"
+    description: "A list of output tensors whose types are T."
+  }
+  attr { name: "T"  description: "A list of dtypes." }
+  attr {
+    name: "body"
+    description: <<END
+    A function that takes a list of tensors (int32, T) and returns another
+    list of tensors (T).
+END
+  }
+  summary: <<END
+  ```python
+   output = input;
+   for i in range(start, limit, delta)
+     output = body(i, output);
+  ```
+END
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_If.pbtxt b/tensorflow/core/api_def/base_api/api_def_If.pbtxt
new file mode 100644
index 0000000000..7ba5a3f37e
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_If.pbtxt
@@ -0,0 +1,40 @@
+op {
+  graph_op_name: "If"
+  in_arg { name: "cond"  description: "The predicate." }
+  in_arg {
+    name: "cond"
+    description: <<END
+      A Tensor. If the tensor is a scalar of non-boolean type, the
+      scalar is converted to a boolean according to the
+      following rule: if the scalar is a numerical value, non-zero means
+      `True` and zero means False; if the scalar is a string, non-empty
+      means `True` and empty means `False`. If the tensor is not a scalar,
+      being empty means False and being non-empty means True.
+END
+  }
+  in_arg {
+    name: "input"
+    description: "A list of input tensors."
+  }
+  out_arg {
+    name: "output"
+    description: "A list of return values."
+  }
+  attr { name: "Tin"  description: "A list of input types." }
+  attr { name: "Tout"  description: "A list of output types." }
+  attr {
+    name: "then_branch"
+    description: <<END
+      A function that takes 'inputs' and returns a list of tensors, whose
+      types are the same as what else_branch returns.
+END
+  }
+  attr {
+    name: "else_branch"
+    description: <<END
+    A function that takes 'inputs' and returns a list of tensors, whose
+    types are the same as what then_branch returns.
+END
+  }
+  summary: "output = cond ? then_branch(input) : else_branch(input)"
+}
diff --git a/tensorflow/core/api_def/base_api/api_def_While.pbtxt b/tensorflow/core/api_def/base_api/api_def_While.pbtxt
new file mode 100644
index 0000000000..95a19c6dff
--- /dev/null
+++ b/tensorflow/core/api_def/base_api/api_def_While.pbtxt
@@ -0,0 +1,33 @@
+op {
+  graph_op_name: "While"
+  in_arg {
+    name: "input"
+    description: "A list of input tensors whose types are T."
+  }
+  out_arg {
+    name: "output"
+    description: "A list of output tensors whose types are T."
+  }
+  attr { name: "T"  description: "dtype in use." }
+  attr {
+    name: "cond"
+    description: <<END
+      A function takes 'input' and returns a tensor.  If the tensor is
+      a scalar of non-boolean, the scalar is converted to a boolean
+      according to the following rule: if the scalar is a numerical
+      value, non-zero means True and zero means False; if the scalar is
+      a string, non-empty means True and empty means False. If the
+      tensor is not a scalar, non-emptiness means True and False
+      otherwise.
+END
+  }
+  attr {
+    name: "body"
+    description: <<END
+      A function that takes a list of tensors and returns another
+      list of tensors. Both lists have the same types as specified
+      by T.
+END
+  }
+  summary: "output = input; While (Cond(output)) { output = Body(output) }"
+}
diff --git a/tensorflow/core/api_def/python_api/api_def_For.pbtxt b/tensorflow/core/api_def/python_api/api_def_For.pbtxt
new file mode 100644
index 0000000000..a58ddf56fe
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_For.pbtxt
@@ -0,0 +1 @@
+op { graph_op_name: "For" visibility: HIDDEN }
diff --git a/tensorflow/core/api_def/python_api/api_def_If.pbtxt b/tensorflow/core/api_def/python_api/api_def_If.pbtxt
new file mode 100644
index 0000000000..a44db5da08
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_If.pbtxt
@@ -0,0 +1 @@
+op { graph_op_name: "If" visibility: HIDDEN }
diff --git a/tensorflow/core/api_def/python_api/api_def_While.pbtxt b/tensorflow/core/api_def/python_api/api_def_While.pbtxt
new file mode 100644
index 0000000000..f47a9b0fce
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_While.pbtxt
@@ -0,0 +1 @@
+op { graph_op_name: "While" visibility: HIDDEN }
diff --git a/tensorflow/core/kernels/functional_ops.cc b/tensorflow/core/kernels/functional_ops.cc
index b687088db1..911aa3a78f 100644
--- a/tensorflow/core/kernels/functional_ops.cc
+++ b/tensorflow/core/kernels/functional_ops.cc
@@ -1,4 +1,4 @@
-/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -12,7 +12,6 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #define EIGEN_USE_THREADS
 
 #include "third_party/eigen3/unsupported/Eigen/CXX11/Tensor"
@@ -21,10 +20,12 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/framework/tensor_shape.h"
 #include "tensorflow/core/lib/core/threadpool.h"
-#include "tensorflow/core/platform/mutex.h"
 
-namespace tensorflow {
+#if GOOGLE_CUDA
+#include "tensorflow/stream_executor/stream.h"
+#endif  // GOOGLE_CUDA
 
+namespace tensorflow {
 typedef Eigen::GpuDevice GPUDevice;
 typedef Eigen::ThreadPoolDevice CPUDevice;
 typedef FunctionLibraryRuntime::Handle FHandle;
@@ -106,11 +107,9 @@ void SetRunOptions(OpKernelContext* ctx, FunctionLibraryRuntime::Options* opts,
   opts->runner = ctx->runner();
 }
 
-}  // end namespace
-
-class FunctionalIf : public AsyncOpKernel {
+class IfOp : public AsyncOpKernel {
  public:
-  explicit FunctionalIf(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
+  explicit IfOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
     auto lib = ctx->function_library();
     OP_REQUIRES(ctx, lib != nullptr, errors::Internal("No function library"));
     const NameAttrList* func;
@@ -120,7 +119,7 @@ class FunctionalIf : public AsyncOpKernel {
     OP_REQUIRES_OK(ctx, Instantiate(lib, *func, &else_handle_));
   }
 
-  ~FunctionalIf() override {}
+  ~IfOp() override {}
 
   void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
     bool cond;
@@ -134,8 +133,7 @@ class FunctionalIf : public AsyncOpKernel {
 
   class State {
    public:
-    State(FunctionalIf* kernel, OpKernelContext* ctx, bool cond,
-          DoneCallback done)
+    State(IfOp* kernel, OpKernelContext* ctx, bool cond, DoneCallback done)
         : kernel_(kernel),
           ctx_(ctx),
           cond_(cond),
@@ -168,7 +166,7 @@ class FunctionalIf : public AsyncOpKernel {
     }
 
    private:
-    FunctionalIf* const kernel_;
+    IfOp* const kernel_;
     OpKernelContext* const ctx_;
     const bool cond_;
     const DoneCallback done_;
@@ -179,18 +177,22 @@ class FunctionalIf : public AsyncOpKernel {
   };
 };
 
-REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_CPU), FunctionalIf);
+// TODO(drpng): remove this.
+REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_CPU), IfOp);
 REGISTER_KERNEL_BUILDER(Name("_If").Device(DEVICE_GPU).HostMemory("cond"),
-                        FunctionalIf);
+                        IfOp);
+
+REGISTER_KERNEL_BUILDER(Name("If").Device(DEVICE_CPU), IfOp);
+REGISTER_KERNEL_BUILDER(Name("If").Device(DEVICE_GPU).HostMemory("cond"), IfOp);
 
-class FunctionalWhile : public AsyncOpKernel {
+class WhileOp : public AsyncOpKernel {
  public:
-  explicit FunctionalWhile(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
+  explicit WhileOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
     OP_REQUIRES_OK(ctx, ctx->GetAttr("cond", &cond_func_));
     OP_REQUIRES_OK(ctx, ctx->GetAttr("body", &body_func_));
   }
 
-  ~FunctionalWhile() override {}
+  ~WhileOp() override {}
 
   void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
     auto lib = ctx->function_library();
@@ -234,7 +236,7 @@ class FunctionalWhile : public AsyncOpKernel {
 
   class State {
    public:
-    State(FunctionalWhile* kernel, OpKernelContext* ctx, FHandle cond_handle,
+    State(WhileOp* kernel, OpKernelContext* ctx, FHandle cond_handle,
           FHandle body_handle, DoneCallback done)
         : kernel_(kernel),
           ctx_(ctx),
@@ -253,7 +255,7 @@ class FunctionalWhile : public AsyncOpKernel {
     void Start() { EvalCond(); }
 
    private:
-    FunctionalWhile* const kernel_;
+    WhileOp* const kernel_;
     OpKernelContext* const ctx_;
     const FHandle cond_handle_;
     const FHandle body_handle_;
@@ -316,7 +318,152 @@ class FunctionalWhile : public AsyncOpKernel {
     }
   };
 };
-REGISTER_KERNEL_BUILDER(Name("_While").Device(DEVICE_CPU), FunctionalWhile);
-REGISTER_KERNEL_BUILDER(Name("_While").Device(DEVICE_GPU), FunctionalWhile);
+// TODO(drpng): remove these.
+REGISTER_KERNEL_BUILDER(Name("_While").Device(DEVICE_CPU), WhileOp);
+REGISTER_KERNEL_BUILDER(Name("_While").Device(DEVICE_GPU), WhileOp);
+
+REGISTER_KERNEL_BUILDER(Name("While").Device(DEVICE_CPU), WhileOp);
+REGISTER_KERNEL_BUILDER(Name("While").Device(DEVICE_GPU), WhileOp);
+
+Status GetScalar(OpKernelContext* ctx, int index, int32* value,
+                 const char* label) {
+  Tensor t = ctx->input(index);
+  if (!TensorShapeUtils::IsScalar(t.shape())) {
+    return errors::InvalidArgument(label, " must be a scalar, but ",
+                                   t.shape().DebugString());
+  }
+  *value = t.scalar<int32>()();
+  return Status::OK();
+}
+
+class ForOp : public AsyncOpKernel {
+ public:
+  explicit ForOp(OpKernelConstruction* ctx) : AsyncOpKernel(ctx) {
+    auto lib = ctx->function_library();
+    OP_REQUIRES(ctx, lib != nullptr, errors::Internal("No function library"));
+    const NameAttrList* func;
+    OP_REQUIRES_OK(ctx, ctx->GetAttr("body", &func));
+    OP_REQUIRES_OK(ctx, Instantiate(lib, *func, &body_handle_));
+  }
+
+  ~ForOp() override {}
+
+  void ComputeAsync(OpKernelContext* ctx, DoneCallback done) override {
+    (new State(this, ctx, done))->Start();
+  }
+
+ private:
+  FHandle body_handle_;
+
+  class State {
+   public:
+    State(ForOp* kernel, OpKernelContext* ctx, DoneCallback done)
+        : kernel_(kernel),
+          ctx_(ctx),
+          done_(std::move(done)),
+          lib_(CHECK_NOTNULL(ctx_->function_library())),
+          args_(1 + ctx_->num_inputs() - 3) {
+      args_[0] = Tensor(DT_INT32, {});
+      iter_ = &args_[0].scalar<int32>()();
+
+      const int32 num_loop_inputs = ctx_->num_inputs() - 3;
+      rets_.reserve(num_loop_inputs);
+      for (int i = 0; i < num_loop_inputs; ++i) {
+        rets_.push_back(ctx_->input(3 + i));
+      }
+    }
+
+    ~State() {}
+
+    void Start() {
+      Status s = StartLoop();
+      if (!s.ok()) Finish(s);
+    }
+
+   private:
+    ForOp* const kernel_;
+    OpKernelContext* const ctx_;
+    const DoneCallback done_;
+    FunctionLibraryRuntime* const lib_;
+    FunctionLibraryRuntime::Options opts_;
+    TensorVec args_;
+    TensorVec rets_;
+
+    int32* iter_;  // points to args_[0].
+    int32 limit_;
+    int32 delta_;
+
+    // If an error e is returned, caller must call Finish(e).
+    // If OK is returned, the async loop execution has been started.
+    Status StartLoop() {
+      SetRunOptions(ctx_, &opts_, false /* always_collect_stats */);
+
+      TF_RETURN_IF_ERROR(GetScalar(ctx_, 0, iter_, "start"));
+      TF_RETURN_IF_ERROR(GetScalar(ctx_, 1, &limit_, "limit"));
+      TF_RETURN_IF_ERROR(GetScalar(ctx_, 2, &delta_, "delta"));
+
+      if ((delta_ > 0 && *iter_ <= limit_) ||
+          (delta_ < 0 && *iter_ >= limit_) ||
+          (delta_ == 0 && *iter_ == limit_)) {
+        RunNext();
+        return Status::OK();
+      } else {
+        return errors::InvalidArgument("Invalid start/limit/delta: ", *iter_,
+                                       " ", limit_, " ", delta_);
+      }
+    }
+
+    void RunNext() {
+      bool done_loop;
+      if (delta_ > 0) {
+        done_loop = *iter_ >= limit_;
+      } else {
+        done_loop = *iter_ <= limit_;
+      }
+      if (done_loop) {
+        Finish(Status::OK());
+        return;
+      }
+
+      if (rets_.size() >= args_.size()) {
+        Finish(errors::InvalidArgument(
+            "For loop body returned ", rets_.size(),
+            " arguments. Expected: ", args_.size() - 1));
+        return;
+      }
+      for (int i = 0; i < rets_.size(); ++i) {
+        args_[1 + i] = std::move(rets_[i]);
+      }
+      rets_.clear();
+      lib_->Run(opts_, kernel_->body_handle_, args_, &rets_,
+                [this](const Status& s) {
+                  if (s.ok()) {
+                    *iter_ += delta_;
+                    RunNext();
+                  } else {
+                    Finish(s);
+                  }
+                });
+    }
+
+    void Finish(Status s) {
+      if (s.ok()) {
+        s = SetOutputs(kernel_, ctx_, rets_);
+      }
+      ctx_->SetStatus(s);
+      done_();
+      delete this;
+    }
+  };
+};
+
+REGISTER_KERNEL_BUILDER(Name("For").Device(DEVICE_CPU), ForOp);
+REGISTER_KERNEL_BUILDER(Name("For")
+                            .Device(DEVICE_GPU)
+                            .HostMemory("start")
+                            .HostMemory("limit")
+                            .HostMemory("delta"),
+                        ForOp);
 
+}  // namespace
 }  // namespace tensorflow
diff --git a/tensorflow/core/ops/functional_ops.cc b/tensorflow/core/ops/functional_ops.cc
index 4b21fac80a..792686cae1 100644
--- a/tensorflow/core/ops/functional_ops.cc
+++ b/tensorflow/core/ops/functional_ops.cc
@@ -50,6 +50,7 @@ REGISTER_OP("RemoteCall")
     .SetIsStateful()
     .SetShapeFn(shape_inference::UnknownShape);
 
+// TODO(drpng): remove this.
 REGISTER_OP("_If")
     .Input("cond: Tcond")
     .Input("input: Tin")
@@ -76,8 +77,18 @@ else_branch: A function that takes 'inputs' and returns a list of
     tensors.  whose types are the same as what then_branch returns.
 )doc");
 
-// TODO(b/37549631) setting the While Op to always be stateful is too
-// conservative.
+REGISTER_OP("If")
+    .Input("cond: Tcond")
+    .Input("input: Tin")
+    .Output("output: Tout")
+    .Attr("Tcond: type")
+    .Attr("Tin: list(type)")
+    .Attr("Tout: list(type)")
+    .Attr("then_branch: func")
+    .Attr("else_branch: func")
+    .SetShapeFn(shape_inference::UnknownShape);
+
+// TODO(drpng): remove this.
 REGISTER_OP("_While")
     .Input("input: T")
     .Output("output: T")
@@ -108,4 +119,30 @@ body: A function that takes a list of tensors and returns another
       by T.
 )doc");
 
+// TODO(b/37549631) setting the While Op to always be stateful is too
+// conservative.
+REGISTER_OP("While")
+    .Input("input: T")
+    .Output("output: T")
+    .Attr("T: list(type) >= 0")
+    .Attr("cond: func")
+    .Attr("body: func")
+    .SetIsStateful()
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      for (int i = 0; i < c->num_outputs(); ++i) {
+        c->set_output(i, c->input(i));
+      }
+      return Status::OK();
+    });
+
+REGISTER_OP("For")
+    .Input("start: int32")
+    .Input("limit: int32")
+    .Input("delta: int32")
+    .Input("input: T")
+    .Output("output: T")
+    .Attr("T: list(type) >= 0")
+    .Attr("body: func")
+    .SetShapeFn(shape_inference::UnknownShape);
+
 }  // end namespace tensorflow
diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 1301ef9d19..10aea89173 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -24,6 +24,7 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python.client import session
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import errors
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
@@ -39,6 +40,7 @@ import tensorflow.python.ops.tensor_array_grad  # pylint: disable=unused-import
 from tensorflow.python.platform import test
 
 
+# pylint: disable=invalid-name
 def simple_scoped_fn(a, x):
   """Simple function: (a, x) -> 2(x+a), but with "2" as a variable in scope."""
   with variable_scope.variable_scope("body"):
@@ -607,6 +609,276 @@ class FunctionalOpsTest(test.TestCase):
       mul = sess.run(remote_op)
       self.assertEqual(mul, 9)
 
+  def testIf(self):
+
+    @function.Defun(dtypes.float32)
+    def Twice(x):
+      return x * 2
+
+    @function.Defun(dtypes.float32)
+    def Thrice(x):
+      return x * 3 + 1
+
+    with self.test_session(use_gpu=False) as sess:
+
+      def Run(x):
+        return sess.run(
+            functional_ops.If(math_ops.greater(x, 0), [x], Twice, Thrice))[0]
+
+      self.assertAllEqual(Run(9.), 18.)
+      self.assertAllEqual(Run(-8.), -23.)
+      self.assertAllEqual(Run(0.), 1.)
+
+  def testWhile(self):
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def Cond(n, unused_x):
+      return n > 0
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def Body(n, x):
+      return n - 1, x + n
+
+    # TODO(b/65752372): Set `use_gpu=False` because
+    # `functional_ops.While()` does not reliably work on GPU (apparently
+    # because the result of evaluating the condition may be in device
+    # memory, but it is read on the host).
+    with self.test_session(use_gpu=False) as sess:
+
+      def Run(n):
+        return sess.run(functional_ops.While([n, 0.], Cond, Body))[1]
+
+      self.assertAllEqual(Run(20.), 210.)
+      self.assertAllEqual(Run(100.), 5050.)
+
+  def testWhileError(self):
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def Cond(n, unused_x):
+      return n > 0
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def CondReturnsTooManyArgs(n, x):
+      return n > 0, x
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def Body(n, x):
+      return n - 1, x + n
+
+    @function.Defun(*[dtypes.float32] * 2)
+    def BodyReturnsTooManyArgs(n, x):
+      return n - 1, x + n, x
+
+    # TODO(b/65752372): Set `use_gpu=False` because
+    # `functional_ops.While()` does not reliably work on GPU (apparently
+    # because the result of evaluating the condition may be in device
+    # memory, but it is read on the host).
+    with self.test_session(use_gpu=False):
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "Expected a single scalar.*got 2 tensors."):
+        functional_ops.While([5., 0.], CondReturnsTooManyArgs, Body)[0].eval()
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "While loop body returned 3 arguments. Expected: 2"):
+        functional_ops.While([5., 0.], Cond, BodyReturnsTooManyArgs)[0].eval()
+
+  def testWhileInMultipleSubgraphs(self):
+
+    @function.Defun(* [dtypes.float32] * 2)
+    def Cond(n, x):  # pylint: disable=unused-argument
+      return n > 0
+
+    @function.Defun(* [dtypes.float32] * 2)
+    def Body(n, x):
+      return n - 1, x + n
+
+    # TODO(b/65752372): Set `use_gpu=False` because
+    # `functional_ops.While()` does not reliably work on GPU (apparently
+    # because the result of evaluating the condition may be in device
+    # memory, but it is read on the host).
+    with self.test_session(use_gpu=False) as sess:
+      n = array_ops.placeholder(dtypes.float32)
+      _, result = functional_ops.While([n, 0.], Cond, Body)
+      c = constant_op.constant(37.)
+
+      self.assertAllEqual(210., sess.run(result, feed_dict={n: 20.}))
+      self.assertAllEqual(5050., sess.run(result, feed_dict={n: 100.}))
+      # Test that the result is the same when we run a different subgraph.
+      self.assertAllEqual(5050., sess.run([result, c], feed_dict={n: 100.})[0])
+
+  def _tfSum(self, rewrite_with_while):
+    # On GPU, don't rewrite using a while loop.
+    use_gpu = not rewrite_with_while
+    with self.test_session(use_gpu=use_gpu) as sess:
+
+      @function.Defun(dtypes.int32, dtypes.float32)
+      def Body(n, x):
+        return x + math_ops.to_float(n)
+
+      xs = [
+          # 1 + 2  + ... + 20
+          functional_ops.For(
+              1, 21, 1, [0.], Body, rewrite_with_while=rewrite_with_while)[0],
+          # 100 + 99 + ... + 1
+          functional_ops.For(
+              100, 0, -1, [0.], Body, rewrite_with_while=rewrite_with_while)[0],
+      ]
+      xvals = sess.run(xs)
+    self.assertAllEqual(210, xvals[0])
+    self.assertAllEqual(5050, xvals[1])
+
+  def testFor(self):
+    self._tfSum(False)
+
+  def testForWithWhile(self):
+    self._tfSum(True)
+
+  def testForWithWhileNaming(self):
+    g = ops.Graph()
+    with g.as_default():
+
+      @function.Defun(dtypes.int32, dtypes.float32, func_name="TestBody")
+      def TestBody(n, x):
+        return x + math_ops.to_float(n)
+
+      _ = functional_ops.For(
+          1, 21, 1, [0.], TestBody, rewrite_with_while=True)[0]
+
+    names = []
+    for func in g.as_graph_def().library.function:
+      names.append(func.signature.name)
+    self.assertTrue("TestBody" in names)
+    self.assertTrue("TestBody_Cond" in names)
+    self.assertTrue("TestBody_Body" in names)
+
+  def testForCapturedInputs(self):
+    v = variables.Variable(1.0)
+
+    @function.Defun(dtypes.int32)
+    def TestNullary(n):
+      v + math_ops.to_float(n)  # pylint: disable=expression-not-assigned
+
+    @function.Defun(dtypes.int32, dtypes.float32)
+    def TestUnary(n, x):
+      return x + math_ops.to_float(n) + v
+
+    @function.Defun(dtypes.int32, dtypes.float32, dtypes.float32)
+    def TestBinary(n, x, x2):
+      return x + math_ops.to_float(n) + v, x2 + v
+
+    for rewrite_with_while in (True, False):
+      # TODO(b/65752372): Set `use_gpu=False` because
+      # `functional_ops.While()` does not reliably work on GPU (apparently
+      # because the result of evaluating the condition may be in device
+      # memory, but it is read on the host).
+      use_gpu = not rewrite_with_while
+      with self.test_session(use_gpu=use_gpu) as sess:
+        result_nullary = functional_ops.For(
+            1, 10, 1, [], TestNullary,
+            rewrite_with_while=rewrite_with_while)
+        result_unary = functional_ops.For(
+            1, 10, 1, [0.], TestUnary,
+            rewrite_with_while=rewrite_with_while)
+        result_binary = functional_ops.For(
+            1, 10, 1, [0., 0.], TestBinary,
+            rewrite_with_while=rewrite_with_while)
+        sess.run(variables.global_variables_initializer())
+        assert not result_nullary
+        # The nullary variant doesn't return anything so we can't easily run it.
+        # As a total hack, fetch the operation by name and run it.
+        sess.run(ops.get_default_graph().get_operation_by_name(
+            "While" if rewrite_with_while else "For"))
+        assert len(result_unary) == 1
+        self.assertEqual([54.0], sess.run(result_unary))
+        assert len(result_binary) == 2
+        self.assertEqual([54.0, 9.0], sess.run(result_binary))
+
+  def _tfMLP(self, xval, wsval, bsval, rewrite_with_while):
+    # On GPU, don't rewrite using a while loop.
+    use_gpu = not rewrite_with_while
+    with self.test_session(use_gpu=use_gpu):
+
+      @function.Defun(dtypes.int32, *[dtypes.float64] * 3)
+      def MLP(i, a, ws, bs):
+        a = math_ops.tanh(math_ops.matmul(a, ws[i, :]) + bs[i, :])
+        return a, ws, bs
+
+      ret = functional_ops.For(
+          0,
+          wsval.shape[0],
+          1, [xval, wsval, bsval],
+          MLP,
+          rewrite_with_while=rewrite_with_while)[0]
+
+      return ret.eval()
+
+  def _npMLP(self, xval, wsval, bsval):
+    for i in range(wsval.shape[0]):
+      xval = np.tanh(np.dot(xval, wsval[i, :]) + bsval[i, :])
+    return xval
+
+  def _testForMLP(self, rewrite_with_while):
+    # We construct a 5-layer Multi-Layer Perceptron network here.
+    # Each layer have the same number of hidden unites (3), and the
+    # activation function is tanh().  We feed the input (xval) with
+    # batch size 2.
+    xval = np.random.normal(size=(2, 3))
+    wsval = np.random.normal(size=(5, 3, 3))
+    bsval = np.random.normal(size=(5, 3))
+    np_ans = self._npMLP(xval, wsval, bsval)
+    tf_for_ans = self._tfMLP(xval, wsval, bsval, rewrite_with_while)
+    self.assertAllClose(np_ans, tf_for_ans)
+
+  def testForMLP(self):
+    self._testForMLP(False)
+
+  def testForMLPWhile(self):
+    self._testForMLP(True)
+
+  def testForError(self):
+
+    @function.Defun(dtypes.int32, dtypes.float32)
+    def Foo(i, v):
+      return math_ops.to_float(i) + v
+
+    @function.Defun(dtypes.int32, dtypes.float32)
+    def ReturnsTooManyArgs(unused_i, v):
+      return v, v
+
+    with self.test_session(use_gpu=True):
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "must be a scalar"):
+        functional_ops.For([0], 10, 1, [0.0], Foo)[0].eval()
+      with self.assertRaisesRegexp(errors.InvalidArgumentError,
+                                   "Invalid start/limit/delta"):
+        functional_ops.For(0, 10, -1, [0.0], Foo)[0].eval()
+      with self.assertRaisesRegexp(
+          errors.InvalidArgumentError,
+          "For loop body returned 2 arguments. Expected: 1"):
+        functional_ops.For(0, 10, 1, [0.0], ReturnsTooManyArgs)[0].eval()
+
+  def testGradient(self):
+
+    @function.Defun(dtypes.float32)
+    def Poly(x):
+      # y = 2x^3+3x^2+4x+8
+      return 2 * x * x * x + 3 * x * x + 4 * x + 8
+
+    @function.Defun(dtypes.float32)
+    def Grad(x):
+      # dy/dx = dy/dy * dy/dx = 1.0 * (6x^2+6x+4)
+      return functional_ops.Gradient([x, 1.0], Poly)[0]
+
+    with self.test_session(use_gpu=False) as sess:
+      a = constant_op.constant(0.)
+      avals = [Poly(a), Grad(a)]
+      b = constant_op.constant(1.)
+      bvals = [Poly(b), Grad(b)]
+      self.assertAllEqual(sess.run(avals), [8., 4.])
+      self.assertAllEqual(sess.run(bvals), [17., 16.])
+
 
 if __name__ == "__main__":
   test.main()
+
+# pylint: enable=invalid-name
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index a840b1eddf..4d95ca262c 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -1,4 +1,4 @@
-# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -27,22 +27,24 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.core.framework import attr_value_pb2
 from tensorflow.python.eager import context
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import gen_functional_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import tensor_array_ops
 from tensorflow.python.ops import variable_scope as vs
-# go/tf-wildcard-import
-# pylint: disable=wildcard-import
-from tensorflow.python.ops.gen_functional_ops import *
-# pylint: enable=wildcard-import
 # pylint: disable=unused-import
-from tensorflow.python.ops.gen_functional_ops import symbolic_gradient
+from tensorflow.python.ops.gen_functional_ops import remote_call
 # pylint: enable=unused-import
+from tensorflow.python.ops.gen_functional_ops import symbolic_gradient
 from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
@@ -634,3 +636,249 @@ def scan(fn, elems, initializer=None, parallel_iterations=10, back_prop=True,
       varscope.set_caching_device(None)
 
     return output_pack(results_flat)
+
+
+# pylint: disable=invalid-name
+def If(cond, inputs, then_branch, else_branch, name=None):
+  r"""output = Cond(inputs) ? then_branch(inputs) : else_branch(inputs).
+
+  Args:
+    cond: A `Tensor`. A scalar. If the scalar is not a boolean, the scalar is
+      converted to a boolean according to the following rule: if the
+      scalar is a numerical value, non-zero means True and zero means
+      False; if the scalar is a string, non-empty means True and empty
+      means False.
+    inputs: A list of input tensors.
+    then_branch: A function takes 'inputs' and returns a list of tensors,
+        whose types are the same as what else_branch returns.
+    else_branch: A function takes 'inputs' and returns a list of tensors.
+        whose types are the same as what then_branch returns.
+    name: A name for the operation (optional).
+
+  Returns:
+    A list of tensors returned by either then_branch(inputs)
+    or else_branch(inputs).
+  """
+  # pylint: disable=protected-access
+  return gen_functional_ops._if(
+      cond,
+      inputs, [_.type for _ in then_branch.definition.signature.output_arg],
+      then_branch,
+      else_branch,
+      name=name)
+
+
+def Gradient(inputs, f, name=None):
+  r"""Computes the gradient function for function f via backpropagation.
+
+  Args:
+    inputs: A list of tensors of size N + M.
+    f: The function we want to compute the gradient for.
+
+      The function 'f' must be a numerical function which takes N inputs and
+      produces M outputs. Its gradient function 'g', which is  a function
+      taking N + M inputs and produces N outputs.
+
+      I.e. if we have
+         (y1, y2, ..., yM) = f(x1, x2, ..., xN),
+      then, g is
+         (dL/dx1, dL/dx2, ..., dL/dxN) = g(x1, x2, ..., xN,
+                                           dL/dy1, dL/dy2, ..., dL/dyM),
+
+      where L is a scalar-value function of (x1, x2, ..., xN) (e.g., the
+      loss function). dL/dxi is the partial derivative of L with respect
+      to xi.
+
+    name: A name for the operation (optional).
+
+  Returns:
+    A list of tensors of size N.
+  """
+  # TODO(zhifengc): Pretty-print the above spec in latex.
+  # TODO(zhfiengc): Needs some math expert to say the comment above better.
+  tlist = [_.type for _ in f.definition.signature.input_arg]
+  return symbolic_gradient(input=inputs, Tout=tlist, f=f, name=name)
+
+
+# pylint: disable=invalid-name,protected-access
+def While(input_, cond, body, name=None, hostmem=None):
+  r"""output = input; While (Cond(output)) { output = Body(output) }.
+
+  Args:
+    input_: A list of `Tensor` objects.
+      A list of input tensors whose types are T.
+    cond: . A function takes 'input' and returns a tensor.  If the tensor is
+      a scalar of non-boolean, the scalar is converted to a boolean
+      according to the following rule: if the scalar is a numerical
+      value, non-zero means True and zero means False; if the scalar is
+      a string, non-empty means True and empty means False. If the
+      tensor is not a scalar, non-emptiness means True and False
+      otherwise.
+    body: . A funcion takes a list of tensors and returns another
+      list tensors. Both lists have the same types as specified
+      by T.
+    name: A name for the operation (optional).
+    hostmem: A list of integer. If i is in the list, input[i] is a
+      host memory tensor.
+
+  Returns:
+    A list of `Tensor` objects. Has the same type as `input`.
+    A list of output tensors whose types are T.
+  """
+  ret = gen_functional_ops._while(input_, cond, body, name=name)
+  if hostmem:
+    input_attr = attr_value_pb2.AttrValue()
+    input_attr.list.i.extend(hostmem)
+    ret[0].op._set_attr("_input_hostmem", input_attr)  # pylint: disable=protected-access
+
+    output_attr = attr_value_pb2.AttrValue()
+    output_attr.list.i.extend(hostmem)
+    ret[0].op._set_attr("_output_hostmem", output_attr)  # pylint: disable=protected-access
+  return ret
+
+
+# b/36459430
+#
+# Ideally, we do not need this rewrite For loop into a While loop.
+# However, today, if a While runs on GPU and the condition returns a
+# boolean, the While kernel crashes. Even if we fix the crash, the
+# bool needs to be copied between GPU and CPU. So, a for loop is much
+# preferred when running on GPU.
+#
+# On the other hand, For op has no directly XLA kernel. So, when we run
+# a for loop, we need to rewrite it using a While op.
+#
+# It should be possible and probably better to write a XLA C++ kernel
+# implementing the logic in _ForUsingWhile.
+def _ForUsingWhile(start,
+                   limit,
+                   delta,
+                   inputs,
+                   forbody,
+                   name=None,
+                   hostmem=None):
+  """Helper to implement a For loop using a While."""
+  # To support negative delta (e.g., range(100, 0, -3)), we iterate
+  # over the range(n) and use iter * delta + start as the real
+  # iteration index. (e.g., for i in range(34): iter = i * (-3) +
+  # 100).
+  d = math_ops.abs(delta)
+  # XLA on TPUs doesn't support integer division
+  n = math_ops.cast(
+      math_ops.cast((math_ops.abs(limit - start) + d - 1), dtypes.float32) /
+      math_ops.cast(d, dtypes.float32), dtypes.int32)
+
+  # Carried loop variables ("extra_args") are implicitly added to the input list
+  # of the WhileBody function. WhileCond does not call forbody, and so does not
+  # depend on any of forbody's extra_args. Since WhileCond and WhileBody
+  # must have identical inputs, we have to augment the cond signature to take
+  # the same types as the carried loop variables.
+  body_sig = [dtypes.int32] * 4 + list(forbody.declared_input_types)[1:]
+  cond_sig = body_sig + [t.dtype for t in forbody.captured_inputs]
+
+  cond_name = "%s_Cond" % forbody.name
+
+  @function.Defun(*cond_sig, func_name=cond_name)
+  def WhileCond(i, n, *args):
+    del args
+    return i < n
+
+  body_name = "%s_Body" % forbody.name
+
+  @function.Defun(*body_sig, func_name=body_name)
+  def WhileBody(i, n, start, delta, *args):
+    """A While wrapper for forbody that handles loop-carried captured inputs."""
+    for_result = forbody(start + i * delta, *args)
+    # Nullary functions return an Operation. Normal functions can't do this
+    # because their return values are converted to Tensors.
+    if isinstance(for_result, ops.Operation):
+      for_result = ()
+    # Unary functions return a single Tensor value.
+    elif isinstance(for_result, ops.Tensor):
+      for_result = (for_result,)
+    extra_args = tuple(function.get_extra_args())
+    return (i + 1, n, start, delta) + tuple(for_result) + extra_args
+
+  if hostmem is not None:
+    hostmem = [(4 + _) for _ in hostmem]
+
+  results = While(
+      input_=[0, n, start, delta] + inputs + WhileBody.captured_inputs,
+      cond=WhileCond,
+      body=WhileBody,
+      name=name,
+      hostmem=hostmem)
+  # Slice off the loop-carried captured inputs.
+  return list(results[4:len(results) - len(WhileBody.captured_inputs)])
+
+
+def For(start,
+        limit,
+        delta,
+        inputs,
+        body,
+        name=None,
+        hostmem=None,
+        rewrite_with_while=None):
+  r"""out = input; for i in range(start, limit, delta) out = body(i, out).
+
+  Args:
+    start: A `Tensor` of type `int32`.
+    limit: A `Tensor` of type `int32`.
+    delta: A `Tensor` of type `int32`.
+    inputs: A list of `Tensor` objects.
+      A list of input tensors whose types are T.
+    body: A function takes a list of tensors and returns another
+      list of tensors. Both lists have the same types as (int32, T...).
+    name: A name for the operation (optional).
+    hostmem: A list of integer. If i is in the list, inputs[i] is a
+      host memory tensor. In other words, (i+1)-th argument of the body
+      function is expecting a host memory.
+    rewrite_with_while: If True, using While op to implement the For.
+
+  Returns:
+    A list of `Tensor` objects. Has the same type as `input`.
+    A list of output tensors whose types are T.
+  """
+  if rewrite_with_while:
+    return _ForUsingWhile(start, limit, delta, inputs, body, name, hostmem)
+  if body.captured_inputs:
+    wrapper_name = "%s_BodyWrapper" % body.name
+
+    @function.Defun(*body.declared_input_types, func_name=wrapper_name)
+    def BodyWrapper(*args):
+      """A wrapper for body that handles loop-carried captured inputs."""
+      body_result = body(*args)
+      extra_args = tuple(function.get_extra_args())
+      # Nullary functions return an Operation. Normal functions can't do this
+      # because their return values are converted to Tensors.
+      if isinstance(body_result, ops.Operation):
+        return extra_args
+      # Unary functions return a single Tensor value.
+      elif not isinstance(body_result, tuple):
+        return (body_result,) + extra_args
+      # N-ary functions return a tuple of Tensors.
+      else:
+        return body_result + extra_args
+
+    inputs += BodyWrapper.captured_inputs
+    ret = gen_functional_ops._for(
+        start, limit, delta, inputs, BodyWrapper, name=name)
+    # Slice off the loop-carried captured inputs.
+    ret = ret[:-len(BodyWrapper.captured_inputs)]
+  else:
+    ret = gen_functional_ops._for(start, limit, delta, inputs, body, name=name)
+  if hostmem:
+    num_for_params = 3  # start/limit/delta
+
+    input_attr = attr_value_pb2.AttrValue()
+    input_attr.list.i.extend([num_for_params + i for i in hostmem])
+    ret[0].op._set_attr("_input_hostmem", input_attr)  # pylint: disable=protected-access
+
+    output_attr = attr_value_pb2.AttrValue()
+    output_attr.list.i.extend(hostmem)
+    ret[0].op._set_attr("_output_hostmem", output_attr)  # pylint: disable=protected-access
+  return ret
+
+
+# pylint: enable=invalid-name,protected-access
-- 
GitLab


From e0488c7f4ee1d3d996c6e87076f4683a41883fd3 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 2 Apr 2018 14:19:42 -0700
Subject: [PATCH 819/960] Export "VERSION" as "__version__" as well

PiperOrigin-RevId: 191346647
---
 tensorflow/python/framework/versions.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/versions.py b/tensorflow/python/framework/versions.py
index 06955b8858..d08b4bf48a 100644
--- a/tensorflow/python/framework/versions.py
+++ b/tensorflow/python/framework/versions.py
@@ -29,7 +29,7 @@ __cxx11_abi_flag__ = pywrap_tensorflow.__cxx11_abi_flag__
 __monolithic_build__ = pywrap_tensorflow.__monolithic_build__
 
 VERSION = __version__
-tf_export("VERSION").export_constant(__name__, "VERSION")
+tf_export("VERSION", "__version__").export_constant(__name__, "VERSION")
 GIT_VERSION = __git_version__
 tf_export("GIT_VERSION").export_constant(__name__, "GIT_VERSION")
 COMPILER_VERSION = __compiler_version__
-- 
GitLab


From 817882c28fd6f0dbbbf35b6ac0764ccbd38430d0 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 2 Apr 2018 14:40:37 -0700
Subject: [PATCH 820/960] ResourceHandleShapeAndType returns bytes, not
 unicode.

This could cause failures when enabling the C API with python3.

PiperOrigin-RevId: 191350031
---
 tensorflow/python/client/tf_session.i | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 5dcd0c192e..ee76e29c05 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -460,6 +460,11 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
   $1 = PyLong_AsLongLong($input);
 }
 
+// Override default py3 behavior of attempting to encode into Unicode.
+%typemap(out) std::string tensorflow::ResourceHandleShapeAndType {
+  $result = PyBytes_FromStringAndSize($1.data(), $1.size());
+}
+
 // TODO(skyewm): SWIG emits a warning for the const char* in TF_WhileParams,
 // skip for now
 %ignore TF_WhileParams;
-- 
GitLab


From 8f543ed7e3e2775aedb5c953f7f5cbff2139663a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 14:46:13 -0700
Subject: [PATCH 821/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 191350894
---
 tensorflow/c/c_api_test.cc                        |  2 +-
 tensorflow/cc/saved_model/loader_test.cc          | 15 ++++++++-------
 tensorflow/cc/tutorials/example_trainer.cc        |  6 ++++--
 .../compiler/xla/service/llvm_ir/llvm_util.cc     |  3 ++-
 .../compiler/xla/tools/parser/hlo_parser_test.cc  |  3 ++-
 .../contrib/android/asset_manager_filesystem.cc   |  5 ++---
 tensorflow/contrib/nccl/kernels/nccl_rewrite.cc   |  3 ++-
 .../tensorboard/db/summary_file_writer_test.cc    |  3 ++-
 .../distributed_runtime/base_rendezvous_mgr.cc    |  4 ++--
 .../core/distributed_runtime/rpc/grpc_session.cc  |  3 ++-
 tensorflow/core/grappler/op_types.cc              |  3 ++-
 tensorflow/core/lib/wav/wav_io_test.cc            |  3 ++-
 tensorflow/core/ops/math_grad_test.cc             |  3 ++-
 tensorflow/core/ops/math_ops_test.cc              | 14 ++++++++------
 .../platform/hadoop/hadoop_file_system_test.cc    |  3 ++-
 .../internal/advisor/tfprof_advisor_test.cc       | 14 ++++++++------
 .../core/util/tensor_bundle/tensor_bundle_test.cc | 13 +++++++------
 tensorflow/examples/label_image/main.cc           |  7 ++++---
 tensorflow/examples/multibox_detector/main.cc     |  7 ++++---
 tensorflow/python/eager/python_eager_op_gen.cc    |  4 ++--
 tensorflow/python/lib/core/py_seq_tensor.cc       |  7 ++++---
 21 files changed, 72 insertions(+), 53 deletions(-)

diff --git a/tensorflow/c/c_api_test.cc b/tensorflow/c/c_api_test.cc
index 028f146be3..ca80db23ed 100644
--- a/tensorflow/c/c_api_test.cc
+++ b/tensorflow/c/c_api_test.cc
@@ -53,7 +53,7 @@ Status TF_TensorToTensor(const TF_Tensor* src, Tensor* dst);
 namespace {
 
 static void ExpectHasSubstr(StringPiece s, StringPiece expected) {
-  EXPECT_TRUE(StringPiece(s).contains(expected))
+  EXPECT_TRUE(str_util::StrContains(s, expected))
       << "'" << s << "' does not contain '" << expected << "'";
 }
 
diff --git a/tensorflow/cc/saved_model/loader_test.cc b/tensorflow/cc/saved_model/loader_test.cc
index 4c64d2cfe3..72b8bc1871 100644
--- a/tensorflow/cc/saved_model/loader_test.cc
+++ b/tensorflow/cc/saved_model/loader_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -133,9 +134,9 @@ TEST_F(LoaderTest, NoTagMatch) {
   Status st = LoadSavedModel(session_options, run_options, export_dir,
                              {"missing-tag"}, &bundle);
   EXPECT_FALSE(st.ok());
-  EXPECT_TRUE(StringPiece(st.error_message())
-                  .contains("Could not find meta graph def matching supplied "
-                            "tags: { missing-tag }"))
+  EXPECT_TRUE(str_util::StrContains(
+      st.error_message(),
+      "Could not find meta graph def matching supplied tags: { missing-tag }"))
       << st.error_message();
 }
 
@@ -149,9 +150,9 @@ TEST_F(LoaderTest, NoTagMatchMultiple) {
   Status st = LoadSavedModel(session_options, run_options, export_dir,
                              {kSavedModelTagServe, "missing-tag"}, &bundle);
   EXPECT_FALSE(st.ok());
-  EXPECT_TRUE(
-      StringPiece(st.error_message())
-          .contains("Could not find meta graph def matching supplied tags: "))
+  EXPECT_TRUE(str_util::StrContains(
+      st.error_message(),
+      "Could not find meta graph def matching supplied tags: "))
       << st.error_message();
 }
 
@@ -169,7 +170,7 @@ TEST_F(LoaderTest, SessionCreationFailure) {
   Status st = LoadSavedModel(session_options, run_options, export_dir,
                              {kSavedModelTagServe}, &bundle);
   EXPECT_FALSE(st.ok());
-  EXPECT_TRUE(StringPiece(st.error_message()).contains(kInvalidTarget))
+  EXPECT_TRUE(str_util::StrContains(st.error_message(), kInvalidTarget))
       << st.error_message();
 }
 
diff --git a/tensorflow/cc/tutorials/example_trainer.cc b/tensorflow/cc/tutorials/example_trainer.cc
index 3675d72ee3..5dbc4f5f6a 100644
--- a/tensorflow/cc/tutorials/example_trainer.cc
+++ b/tensorflow/cc/tutorials/example_trainer.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/graph/default_device.h"
 #include "tensorflow/core/graph/graph_def_builder.h"
 #include "tensorflow/core/lib/core/threadpool.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
@@ -166,7 +167,8 @@ namespace {
 
 bool ParseInt32Flag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                     int32* dst) {
-  if (arg.Consume(flag) && arg.Consume("=")) {
+  if (tensorflow::str_util::ConsumePrefix(&arg, flag) &&
+      tensorflow::str_util::ConsumePrefix(&arg, "=")) {
     char extra;
     return (sscanf(arg.data(), "%d%c", dst, &extra) == 1);
   }
@@ -176,7 +178,7 @@ bool ParseInt32Flag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
 
 bool ParseBoolFlag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                    bool* dst) {
-  if (arg.Consume(flag)) {
+  if (tensorflow::str_util::ConsumePrefix(&arg, flag)) {
     if (arg.empty()) {
       *dst = true;
       return true;
diff --git a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
index 2a282f3be7..ec04239b4f 100644
--- a/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
+++ b/tensorflow/compiler/xla/service/llvm_ir/llvm_util.cc
@@ -34,6 +34,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/casts.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
@@ -762,7 +763,7 @@ void InitializeLLVMCommandLineOptions(const HloModuleConfig& config) {
     fake_argv_storage.push_back("");
     for (const auto& it : options) {
       // Skip options the XLA backend itself consumes.
-      if (!tensorflow::StringPiece(it.first).starts_with("xla_")) {
+      if (!tensorflow::str_util::StartsWith(it.first, "xla_")) {
         if (it.second.empty()) {
           fake_argv_storage.push_back(it.first);
         } else {
diff --git a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
index 863081d654..adc8b1d620 100644
--- a/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
+++ b/tensorflow/compiler/xla/tools/parser/hlo_parser_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include <string>
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace xla {
@@ -894,7 +895,7 @@ class HloParserTest : public ::testing::Test,
                       public ::testing::WithParamInterface<TestData> {
  protected:
   static void ExpectHasSubstr(StringPiece s, StringPiece expected) {
-    EXPECT_TRUE(StringPiece(s).contains(expected))
+    EXPECT_TRUE(tensorflow::str_util::StrContains(s, expected))
         << "'" << s << "' does not contain '" << expected << "'";
   }
 
diff --git a/tensorflow/contrib/android/asset_manager_filesystem.cc b/tensorflow/contrib/android/asset_manager_filesystem.cc
index fe2d13e636..513d519eab 100644
--- a/tensorflow/contrib/android/asset_manager_filesystem.cc
+++ b/tensorflow/contrib/android/asset_manager_filesystem.cc
@@ -229,9 +229,8 @@ string AssetManagerFileSystem::NormalizeDirectoryPath(const string& fname) {
 }
 
 string AssetManagerFileSystem::RemoveAssetPrefix(const string& name) {
-  string output(name);
-  StringPiece piece(output);
-  piece.Consume(prefix_);
+  StringPiece piece(name);
+  str_util::ConsumePrefix(&piece, prefix_);
   return piece.ToString();
 }
 
diff --git a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
index a4de46a93f..4676e937e5 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
+++ b/tensorflow/contrib/nccl/kernels/nccl_rewrite.cc
@@ -13,6 +13,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include "tensorflow/core/lib/strings/str_util.h"
 #if GOOGLE_CUDA
 
 #include <forward_list>
@@ -254,7 +255,7 @@ class NcclReplacePass : public GraphOptimizationPass {
     // Find reduction and broadcast ops and replace them with Send/Recv ops.
     for (Node* node : graph->op_nodes()) {
       StringPiece type = node->type_string();
-      if (!type.starts_with("Nccl")) {
+      if (!str_util::StartsWith(type, "Nccl")) {
         continue;
       }
       if (type == "NcclReduce") {
diff --git a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
index c61b465596..cd3f712256 100644
--- a/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
+++ b/tensorflow/contrib/tensorboard/db/summary_file_writer_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/refcount.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/record_reader.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/util/event.pb.h"
@@ -58,7 +59,7 @@ class SummaryFileWriterTest : public ::testing::Test {
     TF_CHECK_OK(env_.GetChildren(testing::TmpDir(), &files));
     bool found = false;
     for (const string& f : files) {
-      if (StringPiece(f).contains(test_name)) {
+      if (str_util::StrContains(f, test_name)) {
         if (found) {
           return errors::Unknown("Found more than one file for ", test_name);
         }
diff --git a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc
index 049eec347c..bafd9bfc68 100644
--- a/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc
+++ b/tensorflow/core/distributed_runtime/base_rendezvous_mgr.cc
@@ -144,9 +144,9 @@ BaseRemoteRendezvous::~BaseRemoteRendezvous() {
 // Returns true if "device_name" is a valid full name of local device
 // of the "worker".  This helper is purely based on the worker name
 // and device name and does no lookups in the worker->device_mgr.
-static bool IsLocalDevice(const string& worker_name,
+static bool IsLocalDevice(const StringPiece worker_name,
                           const StringPiece device_name) {
-  return device_name.starts_with(worker_name);
+  return str_util::StartsWith(device_name, worker_name);
 }
 
 Status BaseRemoteRendezvous::Initialize(WorkerSession* session) {
diff --git a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
index 120a33f17b..3e79a40683 100644
--- a/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
+++ b/tensorflow/core/distributed_runtime/rpc/grpc_session.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/framework/attr_value.pb.h"
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/mutex.h"
 #include "tensorflow/core/protobuf/master.pb.h"
 
@@ -402,7 +403,7 @@ Status GrpcSession::Reset(const SessionOptions& options,
 class GrpcSessionFactory : public SessionFactory {
  public:
   bool AcceptsOptions(const SessionOptions& options) override {
-    return StringPiece(options.target).starts_with(kSchemePrefix);
+    return str_util::StartsWith(options.target, kSchemePrefix);
   }
 
   Session* NewSession(const SessionOptions& options) override {
diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e0ee49d157..e12e432a33 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -21,6 +21,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/op_types.h"
 #include "tensorflow/core/grappler/utils.h"
 #include "tensorflow/core/lib/core/status.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -409,7 +410,7 @@ bool ModifiesInputsInPlace(const NodeDef& node) {
   // Some nodes do in-place updates on regular tensor inputs.
   string op_name = node.op();
   std::transform(op_name.begin(), op_name.end(), op_name.begin(), ::tolower);
-  if (StringPiece(op_name).contains("inplace")) {
+  if (str_util::StrContains(op_name, "inplace")) {
     return true;
   }
   return GetBoolAttr(node, "in_place") || GetBoolAttr(node, "inplace");
diff --git a/tensorflow/core/lib/wav/wav_io_test.cc b/tensorflow/core/lib/wav/wav_io_test.cc
index d8a83fc464..9e41da6a20 100644
--- a/tensorflow/core/lib/wav/wav_io_test.cc
+++ b/tensorflow/core/lib/wav/wav_io_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 
 #include "tensorflow/core/lib/core/error_codes.pb.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/types.h"
 
@@ -203,7 +204,7 @@ TEST(WavIO, ChunkSizeOverflow) {
       wav_data_string, &decoded_audio, &decoded_sample_count,
       &decoded_channel_count, &decoded_sample_rate);
   EXPECT_FALSE(decode_status.ok());
-  EXPECT_TRUE(StringPiece(decode_status.error_message()).contains("too large"))
+  EXPECT_TRUE(str_util::StrContains(decode_status.error_message(), "too large"))
       << decode_status.error_message();
 }
 
diff --git a/tensorflow/core/ops/math_grad_test.cc b/tensorflow/core/ops/math_grad_test.cc
index 8dcd3e815f..da38a6bc24 100644
--- a/tensorflow/core/ops/math_grad_test.cc
+++ b/tensorflow/core/ops/math_grad_test.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/framework/function_testlib.h"
 #include "tensorflow/core/framework/op_kernel.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/public/session.h"
 
@@ -362,7 +363,7 @@ class MathGradTest : public ::testing::Test {
 };
 
 void HasError(const Status& s, const string& substr) {
-  EXPECT_TRUE(StringPiece(s.ToString()).contains(substr))
+  EXPECT_TRUE(str_util::StrContains(s.ToString(), substr))
       << s << ", expected substring " << substr;
 }
 
diff --git a/tensorflow/core/ops/math_ops_test.cc b/tensorflow/core/ops/math_ops_test.cc
index ca3772e6f8..8f974d5367 100644
--- a/tensorflow/core/ops/math_ops_test.cc
+++ b/tensorflow/core/ops/math_ops_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -239,20 +240,21 @@ TEST(MathOpsTest, Select_ShapeFn) {
 
   // Expect an error when the shapes can't be merged.
   handle_data[2]->at(0).first = shape_proto({2, 2});
-  EXPECT_TRUE(StringPiece(run_inference_for_handles().error_message())
-                  .contains("must be equal, but are 1 and 2"));
+  EXPECT_TRUE(str_util::StrContains(run_inference_for_handles().error_message(),
+                                    "must be equal, but are 1 and 2"));
   handle_data[2]->at(0).first = i1;  // restore to valid
 
   // Expect an error when the types can't be merged.
   handle_data[2]->at(1).second = DT_INT64;
-  EXPECT_TRUE(StringPiece(run_inference_for_handles().error_message())
-                  .contains("pointing to different dtypes"));
+  EXPECT_TRUE(str_util::StrContains(run_inference_for_handles().error_message(),
+                                    "pointing to different dtypes"));
   handle_data[2]->at(1).second = DT_INT32;  // restore to valid
 
   // Expect an error when different numbers of tensors are merged.
   handle_data[2]->push_back({i1, DT_FLOAT});
-  EXPECT_TRUE(StringPiece(run_inference_for_handles().error_message())
-                  .contains("pointing to different numbers of tensors"));
+  EXPECT_TRUE(
+      str_util::StrContains(run_inference_for_handles().error_message(),
+                            "pointing to different numbers of tensors"));
   handle_data[2]->pop_back();  // restore to valid.
 }
 
diff --git a/tensorflow/core/platform/hadoop/hadoop_file_system_test.cc b/tensorflow/core/platform/hadoop/hadoop_file_system_test.cc
index 6ba2f04d0f..b207d34749 100644
--- a/tensorflow/core/platform/hadoop/hadoop_file_system_test.cc
+++ b/tensorflow/core/platform/hadoop/hadoop_file_system_test.cc
@@ -18,6 +18,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/gtl/stl_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/file_system.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -197,7 +198,7 @@ TEST_F(HadoopFileSystemTest, WriteWhileReading) {
   // Skip the test if we're not testing on HDFS. Hadoop's local filesystem
   // implementation makes no guarantees that writable files are readable while
   // being written.
-  if (!StringPiece(fname).starts_with("hdfs://")) {
+  if (!str_util::StartsWith(fname, "hdfs://")) {
     return;
   }
 
diff --git a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc
index e968b9c97e..96b6cc30bd 100644
--- a/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc
+++ b/tensorflow/core/profiler/internal/advisor/tfprof_advisor_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/profiler/internal/advisor/tfprof_advisor.h"
 
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -82,8 +83,8 @@ TEST_F(TFProfAdvisorTest, OperationChecker) {
   (*options.mutable_checkers())[kCheckers[1]];
   AdviceProto advice = advisor_->Advise(options);
   EXPECT_EQ(advice.checkers().at(kCheckers[1]).reports_size(), 1);
-  EXPECT_TRUE(StringPiece(advice.checkers().at(kCheckers[1]).reports(0))
-                  .contains("NCHW"));
+  EXPECT_TRUE(str_util::StrContains(
+      advice.checkers().at(kCheckers[1]).reports(0), "NCHW"));
 }
 
 TEST_F(TFProfAdvisorTest, UtilizationChecker) {
@@ -91,16 +92,17 @@ TEST_F(TFProfAdvisorTest, UtilizationChecker) {
   (*options.mutable_checkers())[kCheckers[0]];
   AdviceProto advice = advisor_->Advise(options);
   EXPECT_EQ(advice.checkers().at(kCheckers[0]).reports_size(), 1);
-  EXPECT_TRUE(StringPiece(advice.checkers().at(kCheckers[0]).reports(0))
-                  .contains("low utilization"));
+  EXPECT_TRUE(str_util::StrContains(
+      advice.checkers().at(kCheckers[0]).reports(0), "low utilization"));
 }
 
 TEST_F(TFProfAdvisorTest, ExpensiveOperationChecker) {
   AdvisorOptionsProto options;
   (*options.mutable_checkers())[kCheckers[2]];
   AdviceProto advice = advisor_->Advise(options);
-  EXPECT_TRUE(StringPiece(advice.checkers().at(kCheckers[2]).reports(0))
-                  .contains("top 1 operation type: Conv2D"));
+  EXPECT_TRUE(
+      str_util::StrContains(advice.checkers().at(kCheckers[2]).reports(0),
+                            "top 1 operation type: Conv2D"));
 }
 
 }  // namespace tfprof
diff --git a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
index 08f1aa7125..7f166f0ec0 100644
--- a/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
+++ b/tensorflow/core/util/tensor_bundle/tensor_bundle_test.cc
@@ -26,6 +26,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
 #include "tensorflow/core/lib/io/table_builder.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
@@ -293,7 +294,7 @@ void VersionTest(const VersionDef& version, StringPiece expected_error) {
   BundleReader reader(Env::Default(), path);
   EXPECT_TRUE(errors::IsInvalidArgument(reader.status()));
   EXPECT_TRUE(
-      StringPiece(reader.status().error_message()).starts_with(expected_error));
+      str_util::StartsWith(reader.status().error_message(), expected_error));
 }
 
 }  // namespace
@@ -588,7 +589,7 @@ TEST(TensorBundleTest, Error) {
     TF_EXPECT_OK(writer.Add("foo", Constant_2x3(1.f)));
     EXPECT_FALSE(writer.Add("foo", Constant_2x3(2.f)).ok());
     EXPECT_TRUE(
-        StringPiece(writer.status().ToString()).contains("duplicate key"));
+        str_util::StrContains(writer.status().ToString(), "duplicate key"));
     EXPECT_FALSE(writer.Finish().ok());
   }
   {  // Double finish
@@ -598,7 +599,7 @@ TEST(TensorBundleTest, Error) {
   }
   {  // Not found.
     BundleReader reader(Env::Default(), Prefix("nonexist"));
-    EXPECT_TRUE(StringPiece(reader.status().ToString()).contains("Not found"));
+    EXPECT_TRUE(str_util::StrContains(reader.status().ToString(), "Not found"));
   }
 }
 
@@ -629,7 +630,7 @@ TEST(TensorBundleTest, Checksum) {
     BundleReader reader(Env::Default(), Prefix(prefix));
     Status status = reader.Lookup(key, &val);
     EXPECT_TRUE(errors::IsDataLoss(status));
-    EXPECT_TRUE(StringPiece(status.ToString()).contains(expected_msg));
+    EXPECT_TRUE(str_util::StrContains(status.ToString(), expected_msg));
   };
 
   // Corrupts a float tensor.
@@ -680,8 +681,8 @@ TEST(TensorBundleTest, Endianness) {
 
   BundleReader reader(Env::Default(), Prefix("end"));
   EXPECT_TRUE(errors::IsUnimplemented(reader.status()));
-  EXPECT_TRUE(StringPiece(reader.status().ToString())
-                  .contains("different endianness from the reader"));
+  EXPECT_TRUE(str_util::StrContains(reader.status().ToString(),
+                                    "different endianness from the reader"));
 }
 
 TEST(TensorBundleTest, TruncatedTensorContents) {
diff --git a/tensorflow/examples/label_image/main.cc b/tensorflow/examples/label_image/main.cc
index 63bc39de6c..baa65d3243 100644
--- a/tensorflow/examples/label_image/main.cc
+++ b/tensorflow/examples/label_image/main.cc
@@ -49,6 +49,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/init_main.h"
@@ -137,15 +138,15 @@ Status ReadTensorFromImageFile(const string& file_name, const int input_height,
   // Now try to figure out what kind of file it is and decode it.
   const int wanted_channels = 3;
   tensorflow::Output image_reader;
-  if (tensorflow::StringPiece(file_name).ends_with(".png")) {
+  if (tensorflow::str_util::EndsWith(file_name, ".png")) {
     image_reader = DecodePng(root.WithOpName("png_reader"), file_reader,
                              DecodePng::Channels(wanted_channels));
-  } else if (tensorflow::StringPiece(file_name).ends_with(".gif")) {
+  } else if (tensorflow::str_util::EndsWith(file_name, ".gif")) {
     // gif decoder returns 4-D tensor, remove the first dim
     image_reader =
         Squeeze(root.WithOpName("squeeze_first_dim"),
                 DecodeGif(root.WithOpName("gif_reader"), file_reader));
-  } else if (tensorflow::StringPiece(file_name).ends_with(".bmp")) {
+  } else if (tensorflow::str_util::EndsWith(file_name, ".bmp")) {
     image_reader = DecodeBmp(root.WithOpName("bmp_reader"), file_reader);
   } else {
     // Assume if it's neither a PNG nor a GIF then it must be a JPEG.
diff --git a/tensorflow/examples/multibox_detector/main.cc b/tensorflow/examples/multibox_detector/main.cc
index e38704fd98..96ea525a4e 100644
--- a/tensorflow/examples/multibox_detector/main.cc
+++ b/tensorflow/examples/multibox_detector/main.cc
@@ -30,6 +30,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/platform/logging.h"
@@ -84,10 +85,10 @@ Status ReadTensorFromImageFile(const string& file_name, const int input_height,
   // Now try to figure out what kind of file it is and decode it.
   const int wanted_channels = 3;
   tensorflow::Output image_reader;
-  if (tensorflow::StringPiece(file_name).ends_with(".png")) {
+  if (tensorflow::str_util::EndsWith(file_name, ".png")) {
     image_reader = DecodePng(root.WithOpName("png_reader"), file_reader,
                              DecodePng::Channels(wanted_channels));
-  } else if (tensorflow::StringPiece(file_name).ends_with(".gif")) {
+  } else if (tensorflow::str_util::EndsWith(file_name, ".gif")) {
     image_reader = DecodeGif(root.WithOpName("gif_reader"), file_reader);
   } else {
     // Assume if it's neither a PNG nor a GIF then it must be a JPEG.
@@ -131,7 +132,7 @@ Status ReadTensorFromImageFile(const string& file_name, const int input_height,
 
 Status SaveImage(const Tensor& tensor, const string& file_path) {
   LOG(INFO) << "Saving image to " << file_path;
-  CHECK(tensorflow::StringPiece(file_path).ends_with(".png"))
+  CHECK(tensorflow::str_util::EndsWith(file_path, ".png"))
       << "Only saving of png files is supported.";
 
   auto root = tensorflow::Scope::NewRootScope();
diff --git a/tensorflow/python/eager/python_eager_op_gen.cc b/tensorflow/python/eager/python_eager_op_gen.cc
index 06185904e7..15d20bdd1a 100644
--- a/tensorflow/python/eager/python_eager_op_gen.cc
+++ b/tensorflow/python/eager/python_eager_op_gen.cc
@@ -117,7 +117,7 @@ class GenEagerPythonOp : public python_op_gen_internal::GenPythonOp {
                    const string& function_name)
       : python_op_gen_internal::GenPythonOp(op_def, api_def, function_name) {
     op_name_ = function_name_;
-    op_name_.Consume("_");
+    str_util::ConsumePrefix(&op_name_, "_");
   }
   ~GenEagerPythonOp() override {}
 
@@ -492,7 +492,7 @@ bool GenEagerPythonOp::GetEagerFunctionSetup(const string& indentation,
       strings::StrAppend(function_setup, indentation, "  ", attr_api_name,
                          " = ", default_value, "\n");
     }
-    if (attr_type.starts_with("list(")) {
+    if (str_util::StartsWith(attr_type, "list(")) {
       ExpectListArg(indentation, attr_api_name, function_setup);
     }
 
diff --git a/tensorflow/python/lib/core/py_seq_tensor.cc b/tensorflow/python/lib/core/py_seq_tensor.cc
index 8247d354db..32ea737a99 100644
--- a/tensorflow/python/lib/core/py_seq_tensor.cc
+++ b/tensorflow/python/lib/core/py_seq_tensor.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/core/framework/types.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/types.h"
 #include "tensorflow/python/lib/core/numpy.h"
 #include "tensorflow/python/lib/core/py_util.h"
@@ -77,9 +78,9 @@ string PyRepr(PyObject* obj) {
 bool IsPyDimension(PyObject* obj) {
   const char* tp_name = obj->ob_type->tp_name;
   if (strcmp(tp_name, "Dimension") != 0) return false;
-  bool ret =
-      StringPiece(PyRepr(PyType(obj)))
-          .ends_with("tensorflow.python.framework.tensor_shape.Dimension'>");
+  bool ret = str_util::EndsWith(
+      PyRepr(PyType(obj)),
+      "tensorflow.python.framework.tensor_shape.Dimension'>");
   return ret;
 }
 
-- 
GitLab


From 1ec02d2a1cf74ab7c0889b6fc6c678c31274d659 Mon Sep 17 00:00:00 2001
From: Eugene Brevdo <ebrevdo@google.com>
Date: Mon, 2 Apr 2018 14:48:21 -0700
Subject: [PATCH 822/960] [TF] Copy-on-write for Resource Variant assign op.

PiperOrigin-RevId: 191351293
---
 .../core/kernels/resource_variable_ops.cc     | 27 ++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index d1675f27dd..082b57b8e2 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -252,6 +252,7 @@ class AssignVariableOp : public OpKernel {
     // tensor, we can just adopt the input tensor's buffer instead.
     std::unique_ptr<Tensor> input_alias =
         context->forward_input(1, dtype_, value.shape(), DEVICE_MEMORY, attr);
+
     mutex_lock ml(*variable->mu());
     variable->is_initialized = true;
     if (input_alias) {
@@ -363,9 +364,33 @@ class AssignVariableOp<Device, Variant> : public OpKernel {
                     DataTypeString(variable->tensor()->dtype()), " got ",
                     DataTypeString(DT_VARIANT)));
 
+    AllocatorAttributes attr;
+    attr.set_on_host(true);
+
+    // Copying is unnecessary if we are the last user of the value
+    // tensor, we can just adopt the input tensor's buffer instead.
+    // Note that Variant objects themselves always reside on host.
+    std::unique_ptr<Tensor> input_alias =
+        context->forward_input(1, DT_VARIANT, value.shape(), HOST_MEMORY, attr);
+
     mutex_lock ml(*variable->mu());
     variable->is_initialized = true;
-    *variable->tensor() = Tensor(DT_VARIANT, value.shape());
+    if (input_alias) {
+      *variable->tensor() = *input_alias;
+      return;
+    }
+
+    // Need to copy, but maybe we can re-use variable's buffer?
+    if (!variable->tensor()->RefCountIsOne() ||
+        !variable->tensor()->shape().IsSameSize(value.shape())) {
+      PersistentTensor unused;
+      Tensor* tmp;
+      OP_REQUIRES_OK(context,
+                     context->allocate_persistent(DT_VARIANT, value.shape(),
+                                                  &unused, &tmp, attr));
+      *variable->tensor() = *tmp;
+    }
+
     const auto elements_in = value.flat<Variant>();
     auto elements_out = variable->tensor()->flat<Variant>();
     auto copy_fn = std::bind(&VariantCopyFn<Device>, context,
-- 
GitLab


From 2ded65fbbb5724b6ecfca05fb6727f27bb29a14a Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 2 Apr 2018 15:00:40 -0700
Subject: [PATCH 823/960] Fix assertion error in
 Graph._create_op_from_tf_operation.

_create_op_from_tf_operation would raise an assertion error if a node
was imported with the same name as an unused name_scope. This isn't
really an error, so this change removes the assert.

Note that with this change, the affected node will have a different
name with the C API enabled, but this seems ok especially considering
this is an edge case.

PiperOrigin-RevId: 191353116
---
 tensorflow/python/framework/importer_test.py | 17 +++++++++++++++++
 tensorflow/python/framework/ops.py           |  8 ++++++--
 2 files changed, 23 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/framework/importer_test.py b/tensorflow/python/framework/importer_test.py
index 369669c2e6..2c913d1e02 100644
--- a/tensorflow/python/framework/importer_test.py
+++ b/tensorflow/python/framework/importer_test.py
@@ -219,6 +219,23 @@ class ImportGraphDefTest(test.TestCase):
       self.assertEqual(outer_inner.name, "outer/inner_1")
       self.assertEqual(outer_inner_c.name, "outer/inner/c_1")
 
+  def testEmptyNameScope(self):
+    with ops.Graph().as_default():
+      # Create name scope but don't create any ops with it
+      with ops.name_scope("foo"):
+        pass
+
+      # Import graph def that uses name scope name
+      op, = importer.import_graph_def(
+          self._MakeGraphDef("node { name: 'foo' op: 'IntOutput' }"),
+          return_elements=["foo"],
+          name="")
+
+      if ops._USE_C_API:
+        self.assertEqual(op.name, "foo")
+      else:
+        self.assertEqual(op.name, "foo_1")
+
   def testInputMap(self):
     with ops.Graph().as_default():
       feed_a_0 = constant_op.constant(0, dtype=dtypes.int32)
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index c0baeb98ac..be0fe5ee44 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -3344,8 +3344,12 @@ class Graph(object):
     """
     self._check_not_finalized()
     ret = Operation(c_op, self)
-    assert ret.name not in self._names_in_use
-    self._names_in_use[ret.name] = 1
+    # If a name_scope was created with ret.name but no nodes were created in it,
+    # the name will still appear in _names_in_use even though the name hasn't
+    # been used. This is ok, just leave _names_in_use as-is in this case.
+    # TODO(skyewm): make the C API guarantee no name conflicts.
+    if ret.name not in self._names_in_use:
+      self._names_in_use[ret.name] = 1
     self._create_op_helper(ret, compute_device=compute_device)
     return ret
 
-- 
GitLab


From 5bb819f64deaa9a641abd95b17c00a843dcb3ce8 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 15:17:59 -0700
Subject: [PATCH 824/960] Update ops-related pbtxt files.

PiperOrigin-RevId: 191356007
---
 .../core/ops/compat/ops_history.v1.pbtxt      | 96 +++++++++++++++++++
 tensorflow/core/ops/ops.pbtxt                 | 96 +++++++++++++++++++
 2 files changed, 192 insertions(+)

diff --git a/tensorflow/core/ops/compat/ops_history.v1.pbtxt b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
index 7cdf36f423..10b24c2d34 100644
--- a/tensorflow/core/ops/compat/ops_history.v1.pbtxt
+++ b/tensorflow/core/ops/compat/ops_history.v1.pbtxt
@@ -20671,6 +20671,38 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "For"
+  input_arg {
+    name: "start"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "limit"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "delta"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+}
 op {
   name: "FractionalAvgPool"
   input_arg {
@@ -22754,6 +22786,45 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "If"
+  input_arg {
+    name: "cond"
+    type_attr: "Tcond"
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tcond"
+    type: "type"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "then_branch"
+    type: "func"
+  }
+  attr {
+    name: "else_branch"
+    type: "func"
+  }
+}
 op {
   name: "Igamma"
   input_arg {
@@ -68075,6 +68146,31 @@ op {
     }
   }
 }
+op {
+  name: "While"
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "cond"
+    type: "func"
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+  is_stateful: true
+}
 op {
   name: "WholeFileReader"
   output_arg {
diff --git a/tensorflow/core/ops/ops.pbtxt b/tensorflow/core/ops/ops.pbtxt
index 42a68cb712..5764976aee 100644
--- a/tensorflow/core/ops/ops.pbtxt
+++ b/tensorflow/core/ops/ops.pbtxt
@@ -9779,6 +9779,38 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "For"
+  input_arg {
+    name: "start"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "limit"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "delta"
+    type: DT_INT32
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+}
 op {
   name: "FractionalAvgPool"
   input_arg {
@@ -11183,6 +11215,45 @@ op {
   }
   is_stateful: true
 }
+op {
+  name: "If"
+  input_arg {
+    name: "cond"
+    type_attr: "Tcond"
+  }
+  input_arg {
+    name: "input"
+    type_list_attr: "Tin"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "Tout"
+  }
+  attr {
+    name: "Tcond"
+    type: "type"
+  }
+  attr {
+    name: "Tin"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "Tout"
+    type: "list(type)"
+    has_minimum: true
+    minimum: 1
+  }
+  attr {
+    name: "then_branch"
+    type: "func"
+  }
+  attr {
+    name: "else_branch"
+    type: "func"
+  }
+}
 op {
   name: "Igamma"
   input_arg {
@@ -32936,6 +33007,31 @@ op {
     }
   }
 }
+op {
+  name: "While"
+  input_arg {
+    name: "input"
+    type_list_attr: "T"
+  }
+  output_arg {
+    name: "output"
+    type_list_attr: "T"
+  }
+  attr {
+    name: "T"
+    type: "list(type)"
+    has_minimum: true
+  }
+  attr {
+    name: "cond"
+    type: "func"
+  }
+  attr {
+    name: "body"
+    type: "func"
+  }
+  is_stateful: true
+}
 op {
   name: "WholeFileReader"
   output_arg {
-- 
GitLab


From 11c0faed23ec32c3f1532f5154dd3c7bb38847d5 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 15:27:24 -0700
Subject: [PATCH 825/960] [XLA] Set trace for the operand of a trace
 instruction when creating the instruction directly or creating from proto.
 Also implement XlaBuidler::Trace.

PiperOrigin-RevId: 191357376
---
 tensorflow/compiler/xla/client/xla_client/xla_builder.cc | 7 ++++++-
 tensorflow/compiler/xla/service/hlo_instruction.cc       | 8 ++++++++
 tensorflow/compiler/xla/service/user_computation.cc      | 1 -
 3 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index 04091ecb11..ec2362179e 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -543,7 +543,12 @@ XlaOp XlaBuilder::Collapse(const XlaOp& operand,
 }
 
 void XlaBuilder::Trace(const string& tag, const XlaOp& operand) {
-  UnimplementedOp();
+  NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    *instr.mutable_shape() = ShapeUtil::MakeNil();
+    *instr.mutable_literal() = Literal::CreateR1U8(tag)->ToProto();
+    return AddInstruction(std::move(instr), HloOpcode::kTrace, {operand});
+  });
 }
 
 XlaOp XlaBuilder::Select(const XlaOp& pred, const XlaOp& on_true,
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.cc b/tensorflow/compiler/xla/service/hlo_instruction.cc
index a2a2c1e615..fcf9ebf5f7 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.cc
+++ b/tensorflow/compiler/xla/service/hlo_instruction.cc
@@ -98,6 +98,13 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
     }
   }
 
+  if (instruction->opcode() == HloOpcode::kTrace) {
+    TF_RET_CHECK(instruction->operands().size() == 1)
+        << "Trace instruction should have 1 operand but sees "
+        << instruction->operands().size();
+    instruction->mutable_operand(0)->set_tracing(instruction.get());
+  }
+
   TF_RET_CHECK(!proto.name().empty());
   instruction->name_ = proto.name();
 
@@ -170,6 +177,7 @@ StatusOr<std::unique_ptr<HloInstruction>> HloInstruction::CreateFromProto(
       WrapUnique(new HloInstruction(HloOpcode::kTrace, ShapeUtil::MakeNil()));
   instruction->operands_.push_back(operand);
   instruction->literal_ = Literal::CreateR1U8(tag);
+  operand->set_tracing(instruction.get());
   return instruction;
 }
 
diff --git a/tensorflow/compiler/xla/service/user_computation.cc b/tensorflow/compiler/xla/service/user_computation.cc
index fcdb2e01fb..532f7fd5bf 100644
--- a/tensorflow/compiler/xla/service/user_computation.cc
+++ b/tensorflow/compiler/xla/service/user_computation.cc
@@ -3491,7 +3491,6 @@ void ComputationLowerer::Visit(
       HloInstruction* operand = lookup_instruction(trace_request.operand());
       hlo_instruction = add_instruction(
           HloInstruction::CreateTrace(trace_request.tag(), operand));
-      operand->set_tracing(hlo_instruction);
       break;
     }
 
-- 
GitLab


From 84fc3becd9c18b3f93ec60f35a3b474a8f192bd9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 15:39:35 -0700
Subject: [PATCH 826/960] Add tf.math.polyval that evaluates an element-wise
 polynomial using Horner's method. This is equivalent to numpy.polyval.

PiperOrigin-RevId: 191359241
---
 .../python/kernel_tests/cwise_ops_test.py     | 42 ++++++++++
 tensorflow/python/ops/math_ops.py             | 83 ++++++++++++++-----
 tensorflow/tools/api/generator/BUILD          |  1 +
 .../tools/api/golden/tensorflow.math.pbtxt    |  7 ++
 tensorflow/tools/api/golden/tensorflow.pbtxt  |  4 +
 5 files changed, 115 insertions(+), 22 deletions(-)
 create mode 100644 tensorflow/tools/api/golden/tensorflow.math.pbtxt

diff --git a/tensorflow/python/kernel_tests/cwise_ops_test.py b/tensorflow/python/kernel_tests/cwise_ops_test.py
index 8db0bb6f0d..34e7751243 100644
--- a/tensorflow/python/kernel_tests/cwise_ops_test.py
+++ b/tensorflow/python/kernel_tests/cwise_ops_test.py
@@ -2165,5 +2165,47 @@ class AccumulateTest(test.TestCase):
         math_ops.accumulate_n([a], tensor_dtype=np.int32)
 
 
+class PolyvalTest(test.TestCase):
+
+  def _runtest(self, dtype, degree):
+    x = np.random.rand(2, 2).astype(dtype)
+    coeffs = [np.random.rand(2, 2).astype(dtype) for _ in range(degree + 1)]
+    np_val = np.polyval(coeffs, x)
+    with self.test_session():
+      tf_val = math_ops.polyval(coeffs, x)
+      self.assertAllClose(np_val, tf_val.eval())
+
+  def testSimple(self):
+    for dtype in [
+        np.int32, np.float32, np.float64, np.complex64, np.complex128
+    ]:
+      for degree in range(5):
+        self._runtest(dtype, degree)
+
+  def testBroadcast(self):
+    dtype = np.float32
+    degree = 3
+    shapes = [(1,), (2, 1), (1, 2), (2, 2)]
+    for x_shape in shapes:
+      for coeff_shape in shapes:
+        x = np.random.rand(*x_shape).astype(dtype)
+        coeffs = [
+            np.random.rand(*coeff_shape).astype(dtype)
+            for _ in range(degree + 1)
+        ]
+        np_val = np.polyval(coeffs, x)
+        with self.test_session():
+          tf_val = math_ops.polyval(coeffs, x)
+          self.assertAllClose(np_val, tf_val.eval())
+
+  def testEmpty(self):
+    x = np.random.rand(2, 2).astype(np.float32)
+    coeffs = []
+    np_val = np.polyval(coeffs, x)
+    with self.test_session():
+      tf_val = math_ops.polyval(coeffs, x)
+      self.assertAllClose(np_val, tf_val.eval())
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 276897ab99..1c20d004cb 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -71,6 +71,7 @@ See the @{$python/math_ops} guide.
 @@igammac
 @@zeta
 @@polygamma
+@@polyval
 @@betainc
 @@rint
 @@diag
@@ -174,6 +175,7 @@ from tensorflow.python.ops.gen_math_ops import *
 # pylint: enable=wildcard-import
 from tensorflow.python.util import compat
 from tensorflow.python.util import deprecation
+from tensorflow.python.util import nest
 from tensorflow.python.util.tf_export import tf_export
 
 # Aliases for some automatically-generated names.
@@ -184,7 +186,6 @@ arg_min = deprecation.deprecated(None, "Use `argmin` instead")(arg_min)  # pylin
 tf_export("arg_max")(arg_max)
 tf_export("arg_min")(arg_min)
 
-
 # This is set by resource_variable_ops.py. It is included in this way since
 # there is a circular dependency between math_ops and resource_variable_ops
 _resource_variable_type = None
@@ -1343,8 +1344,7 @@ def _ReductionDims(x, axis, reduction_indices):
   else:
     # Fast path: avoid creating Rank and Range ops if ndims is known.
     if isinstance(x, ops.Tensor) and x._rank() is not None:  # pylint: disable=protected-access
-      return constant_op.constant(
-          np.arange(x._rank()), dtype=dtypes.int32)  # pylint: disable=protected-access
+      return constant_op.constant(np.arange(x._rank()), dtype=dtypes.int32)  # pylint: disable=protected-access
     if (isinstance(x, sparse_tensor.SparseTensor) and
         x.dense_shape.get_shape().is_fully_defined()):
       rank = x.dense_shape.get_shape()[0].value  # sparse.dense_shape is 1-D.
@@ -2273,10 +2273,11 @@ def accumulate_n(inputs, shape=None, tensor_dtype=None, name=None):
     ValueError: If `inputs` don't all have same shape and dtype or the shape
     cannot be inferred.
   """
+
   def _input_error():
-    return ValueError(
-        "inputs must be a list of at least one Tensor with the "
-        "same dtype and shape")
+    return ValueError("inputs must be a list of at least one Tensor with the "
+                      "same dtype and shape")
+
   if not inputs or not isinstance(inputs, (list, tuple)):
     raise _input_error()
   inputs = ops.convert_n_to_tensor_or_indexed_slices(inputs)
@@ -2294,8 +2295,8 @@ def accumulate_n(inputs, shape=None, tensor_dtype=None, name=None):
 
   # tensor_dtype is for safety only; operator's output type computed in C++
   if tensor_dtype is not None and tensor_dtype != inputs[0].dtype:
-    raise TypeError("tensor_dtype is {}, but input is of type {}"
-                    .format(tensor_dtype, inputs[0].dtype))
+    raise TypeError("tensor_dtype is {}, but input is of type {}".format(
+        tensor_dtype, inputs[0].dtype))
 
   if len(inputs) == 1 and name is None:
     return inputs[0]
@@ -2761,14 +2762,14 @@ def sparse_segment_sum(data, indices, segment_ids, name=None,
         name=name)
   else:
     return gen_math_ops.sparse_segment_sum(
-        data=data,
-        indices=indices,
-        segment_ids=segment_ids,
-        name=name)
+        data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
 @tf_export("sparse_segment_mean")
-def sparse_segment_mean(data, indices, segment_ids, name=None,
+def sparse_segment_mean(data,
+                        indices,
+                        segment_ids,
+                        name=None,
                         num_segments=None):
   r"""Computes the mean along sparse segments of a tensor.
 
@@ -2805,14 +2806,14 @@ def sparse_segment_mean(data, indices, segment_ids, name=None,
         name=name)
   else:
     return gen_math_ops.sparse_segment_mean(
-        data=data,
-        indices=indices,
-        segment_ids=segment_ids,
-        name=name)
+        data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
 @tf_export("sparse_segment_sqrt_n")
-def sparse_segment_sqrt_n(data, indices, segment_ids, name=None,
+def sparse_segment_sqrt_n(data,
+                          indices,
+                          segment_ids,
+                          name=None,
                           num_segments=None):
   r"""Computes the sum along sparse segments of a tensor divided by the sqrt(N).
 
@@ -2842,10 +2843,7 @@ def sparse_segment_sqrt_n(data, indices, segment_ids, name=None,
         name=name)
   else:
     return gen_math_ops.sparse_segment_sqrt_n(
-        data=data,
-        indices=indices,
-        segment_ids=segment_ids,
-        name=name)
+        data=data, indices=indices, segment_ids=segment_ids, name=name)
 
 
 @tf_export("tensordot", "linalg.tensordot")
@@ -3016,6 +3014,47 @@ def tensordot(a, b, axes, name=None):
       return product
 
 
+@tf_export("math.polyval")
+def polyval(coeffs, x, name=None):
+  r"""Computes the elementwise value of a polynomial.
+
+  If `x` is a tensor and `coeffs` is a list n + 1 tensors, this function returns
+  the value of the n-th order polynomial
+
+     p(x) = coeffs[n-1] + coeffs[n-2] * x + ...  + coeffs[0] * x**(n-1)
+
+  evaluated using Horner's method, i.e.
+
+     p(x) = coeffs[n-1] + x * (coeffs[n-2] + ... + x * (coeffs[1] +
+            x * coeffs[0]))
+
+  Args:
+    coeffs: A list of `Tensor` representing the coefficients of the polynomial.
+    x: A `Tensor` representing the variable of the polynomial.
+    name: A name for the operation (optional).
+
+  Returns:
+    A `tensor` of the shape as the expression p(x) with usual broadcasting rules
+    for element-wise addition and multiplication applied.
+
+  @compatibility(numpy)
+  Equivalent to numpy.polyval.
+  @end_compatibility
+  """
+
+  with ops.name_scope(name, "polyval", nest.flatten(coeffs) + [x]) as name:
+    x = ops.convert_to_tensor(x, name="x")
+    if len(coeffs) < 1:
+      return array_ops.zeros_like(x, name=name)
+    coeffs = [
+        ops.convert_to_tensor(coeff, name=("coeff_%d" % index))
+        for index, coeff in enumerate(coeffs)
+    ]
+    p = coeffs[0]
+    for c in coeffs[1:]:
+      p = c + p * x
+    return p
+
 # FFT ops were moved to tf.spectral. tf.fft symbols were part of the TensorFlow
 # 1.0 API so we leave these here for backwards compatibility.
 fft = gen_spectral_ops.fft
diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD
index f8063ae0fb..a1c569951e 100644
--- a/tensorflow/tools/api/generator/BUILD
+++ b/tensorflow/tools/api/generator/BUILD
@@ -94,6 +94,7 @@ genrule(
         "api/logging/__init__.py",
         "api/losses/__init__.py",
         "api/manip/__init__.py",
+        "api/math/__init__.py",
         "api/metrics/__init__.py",
         "api/nn/__init__.py",
         "api/nn/rnn_cell/__init__.py",
diff --git a/tensorflow/tools/api/golden/tensorflow.math.pbtxt b/tensorflow/tools/api/golden/tensorflow.math.pbtxt
new file mode 100644
index 0000000000..897718c05e
--- /dev/null
+++ b/tensorflow/tools/api/golden/tensorflow.math.pbtxt
@@ -0,0 +1,7 @@
+path: "tensorflow.math"
+tf_module {
+  member_method {
+    name: "polyval"
+    argspec: "args=[\'coeffs\', \'x\', \'name\'], varargs=None, keywords=None, defaults=[\'None\'], "
+  }
+}
diff --git a/tensorflow/tools/api/golden/tensorflow.pbtxt b/tensorflow/tools/api/golden/tensorflow.pbtxt
index 937044aece..afa3b78eb7 100644
--- a/tensorflow/tools/api/golden/tensorflow.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.pbtxt
@@ -404,6 +404,10 @@ tf_module {
     name: "manip"
     mtype: "<type \'module\'>"
   }
+  member {
+    name: "math"
+    mtype: "<type \'module\'>"
+  }
   member {
     name: "metrics"
     mtype: "<type \'module\'>"
-- 
GitLab


From bf813f8ecf03b0a550593a530e2fec05b491cf9b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 15:46:34 -0700
Subject: [PATCH 827/960] Go: Update generated wrapper functions for TensorFlow
 ops. PiperOrigin-RevId: 191360220

---
 tensorflow/go/op/wrappers.go | 250 +++++++++++++++++------------------
 1 file changed, 125 insertions(+), 125 deletions(-)

diff --git a/tensorflow/go/op/wrappers.go b/tensorflow/go/op/wrappers.go
index a33703ad6f..0fd2177df7 100644
--- a/tensorflow/go/op/wrappers.go
+++ b/tensorflow/go/op/wrappers.go
@@ -1720,6 +1720,131 @@ func Size(scope *Scope, input tf.Output, optional ...SizeAttr) (output tf.Output
 	return op.Output(0)
 }
 
+// Returns the rank of a tensor.
+//
+// This operation returns an integer representing the rank of `input`.
+//
+// For example:
+//
+// ```
+// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
+// # shape of tensor 't' is [2, 2, 3]
+// rank(t) ==> 3
+// ```
+//
+// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
+// of a tensor is the number of indices required to uniquely select each element
+// of the tensor. Rank is also known as "order", "degree", or "ndims."
+func Rank(scope *Scope, input tf.Output) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	opspec := tf.OpSpec{
+		Type: "Rank",
+		Input: []tf.Input{
+			input,
+		},
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
+// ReverseSequenceAttr is an optional argument to ReverseSequence.
+type ReverseSequenceAttr func(optionalAttr)
+
+// ReverseSequenceBatchDim sets the optional batch_dim attribute to value.
+//
+// value: The dimension along which reversal is performed.
+// If not specified, defaults to 0
+func ReverseSequenceBatchDim(value int64) ReverseSequenceAttr {
+	return func(m optionalAttr) {
+		m["batch_dim"] = value
+	}
+}
+
+// Reverses variable length slices.
+//
+// This op first slices `input` along the dimension `batch_dim`, and for each
+// slice `i`, reverses the first `seq_lengths[i]` elements along
+// the dimension `seq_dim`.
+//
+// The elements of `seq_lengths` must obey `seq_lengths[i] <= input.dims[seq_dim]`,
+// and `seq_lengths` must be a vector of length `input.dims[batch_dim]`.
+//
+// The output slice `i` along dimension `batch_dim` is then given by input
+// slice `i`, with the first `seq_lengths[i]` slices along dimension
+// `seq_dim` reversed.
+//
+// For example:
+//
+// ```
+// # Given this:
+// batch_dim = 0
+// seq_dim = 1
+// input.dims = (4, 8, ...)
+// seq_lengths = [7, 2, 3, 5]
+//
+// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
+// output[0, 0:7, :, ...] = input[0, 7:0:-1, :, ...]
+// output[1, 0:2, :, ...] = input[1, 2:0:-1, :, ...]
+// output[2, 0:3, :, ...] = input[2, 3:0:-1, :, ...]
+// output[3, 0:5, :, ...] = input[3, 5:0:-1, :, ...]
+//
+// # while entries past seq_lens are copied through:
+// output[0, 7:, :, ...] = input[0, 7:, :, ...]
+// output[1, 2:, :, ...] = input[1, 2:, :, ...]
+// output[2, 3:, :, ...] = input[2, 3:, :, ...]
+// output[3, 2:, :, ...] = input[3, 2:, :, ...]
+// ```
+//
+// In contrast, if:
+//
+// ```
+// # Given this:
+// batch_dim = 2
+// seq_dim = 0
+// input.dims = (8, ?, 4, ...)
+// seq_lengths = [7, 2, 3, 5]
+//
+// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
+// output[0:7, :, 0, :, ...] = input[7:0:-1, :, 0, :, ...]
+// output[0:2, :, 1, :, ...] = input[2:0:-1, :, 1, :, ...]
+// output[0:3, :, 2, :, ...] = input[3:0:-1, :, 2, :, ...]
+// output[0:5, :, 3, :, ...] = input[5:0:-1, :, 3, :, ...]
+//
+// # while entries past seq_lens are copied through:
+// output[7:, :, 0, :, ...] = input[7:, :, 0, :, ...]
+// output[2:, :, 1, :, ...] = input[2:, :, 1, :, ...]
+// output[3:, :, 2, :, ...] = input[3:, :, 2, :, ...]
+// output[2:, :, 3, :, ...] = input[2:, :, 3, :, ...]
+// ```
+//
+// Arguments:
+//	input: The input to reverse.
+//	seq_lengths: 1-D with length `input.dims(batch_dim)` and
+// `max(seq_lengths) <= input.dims(seq_dim)`
+//	seq_dim: The dimension which is partially reversed.
+//
+// Returns The partially reversed input. It has the same shape as `input`.
+func ReverseSequence(scope *Scope, input tf.Output, seq_lengths tf.Output, seq_dim int64, optional ...ReverseSequenceAttr) (output tf.Output) {
+	if scope.Err() != nil {
+		return
+	}
+	attrs := map[string]interface{}{"seq_dim": seq_dim}
+	for _, a := range optional {
+		a(attrs)
+	}
+	opspec := tf.OpSpec{
+		Type: "ReverseSequence",
+		Input: []tf.Input{
+			input, seq_lengths,
+		},
+		Attrs: attrs,
+	}
+	op := scope.AddOperation(opspec)
+	return op.Output(0)
+}
+
 // Returns the complex conjugate of a complex number.
 //
 // Given a tensor `input` of complex numbers, this operation returns a tensor of
@@ -5128,102 +5253,6 @@ func RsqrtGrad(scope *Scope, y tf.Output, dy tf.Output) (z tf.Output) {
 	return op.Output(0)
 }
 
-// ReverseSequenceAttr is an optional argument to ReverseSequence.
-type ReverseSequenceAttr func(optionalAttr)
-
-// ReverseSequenceBatchDim sets the optional batch_dim attribute to value.
-//
-// value: The dimension along which reversal is performed.
-// If not specified, defaults to 0
-func ReverseSequenceBatchDim(value int64) ReverseSequenceAttr {
-	return func(m optionalAttr) {
-		m["batch_dim"] = value
-	}
-}
-
-// Reverses variable length slices.
-//
-// This op first slices `input` along the dimension `batch_dim`, and for each
-// slice `i`, reverses the first `seq_lengths[i]` elements along
-// the dimension `seq_dim`.
-//
-// The elements of `seq_lengths` must obey `seq_lengths[i] <= input.dims[seq_dim]`,
-// and `seq_lengths` must be a vector of length `input.dims[batch_dim]`.
-//
-// The output slice `i` along dimension `batch_dim` is then given by input
-// slice `i`, with the first `seq_lengths[i]` slices along dimension
-// `seq_dim` reversed.
-//
-// For example:
-//
-// ```
-// # Given this:
-// batch_dim = 0
-// seq_dim = 1
-// input.dims = (4, 8, ...)
-// seq_lengths = [7, 2, 3, 5]
-//
-// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
-// output[0, 0:7, :, ...] = input[0, 7:0:-1, :, ...]
-// output[1, 0:2, :, ...] = input[1, 2:0:-1, :, ...]
-// output[2, 0:3, :, ...] = input[2, 3:0:-1, :, ...]
-// output[3, 0:5, :, ...] = input[3, 5:0:-1, :, ...]
-//
-// # while entries past seq_lens are copied through:
-// output[0, 7:, :, ...] = input[0, 7:, :, ...]
-// output[1, 2:, :, ...] = input[1, 2:, :, ...]
-// output[2, 3:, :, ...] = input[2, 3:, :, ...]
-// output[3, 2:, :, ...] = input[3, 2:, :, ...]
-// ```
-//
-// In contrast, if:
-//
-// ```
-// # Given this:
-// batch_dim = 2
-// seq_dim = 0
-// input.dims = (8, ?, 4, ...)
-// seq_lengths = [7, 2, 3, 5]
-//
-// # then slices of input are reversed on seq_dim, but only up to seq_lengths:
-// output[0:7, :, 0, :, ...] = input[7:0:-1, :, 0, :, ...]
-// output[0:2, :, 1, :, ...] = input[2:0:-1, :, 1, :, ...]
-// output[0:3, :, 2, :, ...] = input[3:0:-1, :, 2, :, ...]
-// output[0:5, :, 3, :, ...] = input[5:0:-1, :, 3, :, ...]
-//
-// # while entries past seq_lens are copied through:
-// output[7:, :, 0, :, ...] = input[7:, :, 0, :, ...]
-// output[2:, :, 1, :, ...] = input[2:, :, 1, :, ...]
-// output[3:, :, 2, :, ...] = input[3:, :, 2, :, ...]
-// output[2:, :, 3, :, ...] = input[2:, :, 3, :, ...]
-// ```
-//
-// Arguments:
-//	input: The input to reverse.
-//	seq_lengths: 1-D with length `input.dims(batch_dim)` and
-// `max(seq_lengths) <= input.dims(seq_dim)`
-//	seq_dim: The dimension which is partially reversed.
-//
-// Returns The partially reversed input. It has the same shape as `input`.
-func ReverseSequence(scope *Scope, input tf.Output, seq_lengths tf.Output, seq_dim int64, optional ...ReverseSequenceAttr) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	attrs := map[string]interface{}{"seq_dim": seq_dim}
-	for _, a := range optional {
-		a(attrs)
-	}
-	opspec := tf.OpSpec{
-		Type: "ReverseSequence",
-		Input: []tf.Input{
-			input, seq_lengths,
-		},
-		Attrs: attrs,
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // DepthwiseConv2dNativeAttr is an optional argument to DepthwiseConv2dNative.
 type DepthwiseConv2dNativeAttr func(optionalAttr)
 
@@ -5808,35 +5837,6 @@ func FusedBatchNormV2(scope *Scope, x tf.Output, scale tf.Output, offset tf.Outp
 	return op.Output(0), op.Output(1), op.Output(2), op.Output(3), op.Output(4)
 }
 
-// Returns the rank of a tensor.
-//
-// This operation returns an integer representing the rank of `input`.
-//
-// For example:
-//
-// ```
-// # 't' is [[[1, 1, 1], [2, 2, 2]], [[3, 3, 3], [4, 4, 4]]]
-// # shape of tensor 't' is [2, 2, 3]
-// rank(t) ==> 3
-// ```
-//
-// **Note**: The rank of a tensor is not the same as the rank of a matrix. The rank
-// of a tensor is the number of indices required to uniquely select each element
-// of the tensor. Rank is also known as "order", "degree", or "ndims."
-func Rank(scope *Scope, input tf.Output) (output tf.Output) {
-	if scope.Err() != nil {
-		return
-	}
-	opspec := tf.OpSpec{
-		Type: "Rank",
-		Input: []tf.Input{
-			input,
-		},
-	}
-	op := scope.AddOperation(opspec)
-	return op.Output(0)
-}
-
 // Transforms a Tensor into a serialized TensorProto proto.
 //
 // Arguments:
-- 
GitLab


From da92e74fcb28f31c2a4163c58e6e585f561b1c33 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 2 Apr 2018 15:48:04 -0700
Subject: [PATCH 828/960] Add support for resource variables

PiperOrigin-RevId: 191360477
---
 tensorflow/core/grappler/op_types.cc | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index e12e432a33..584008b0c1 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -409,6 +409,16 @@ bool IsFreeOfSideEffect(const NodeDef& node) {
 bool ModifiesInputsInPlace(const NodeDef& node) {
   // Some nodes do in-place updates on regular tensor inputs.
   string op_name = node.op();
+
+  // Ops that modify resource variables effectively modify one of their inputs.
+  if (op_name == "AssignVariableOp" || op_name == "AssignAddVariableOp" ||
+      op_name == "AssignSubVariableOp" || op_name == "ResourceScatterUpdate" ||
+      op_name == "ResourceScatterAdd" || op_name == "ResourceScatterSub" ||
+      op_name == "ResourceScatterMul" || op_name == "ResourceScatterDiv" ||
+      op_name == "ResourceScatterMin" || op_name == "ResourceScatterMax") {
+    return false;
+  }
+
   std::transform(op_name.begin(), op_name.end(), op_name.begin(), ::tolower);
   if (str_util::StrContains(op_name, "inplace")) {
     return true;
-- 
GitLab


From 0042e54608eedf1f10d761bc9ca68f030feeed04 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 2 Apr 2018 15:51:21 -0700
Subject: [PATCH 829/960] Automated g4 rollback of changelist 191326767

PiperOrigin-RevId: 191360905
---
 tensorflow/BUILD                              |  8 ++
 tensorflow/__init__.py                        |  7 +-
 tensorflow/contrib/cmake/python_modules.txt   |  2 -
 tensorflow/contrib/cmake/tf_python.cmake      | 91 ++++---------------
 tensorflow/experimental_api.py                | 38 ++++++++
 tensorflow/python/framework/dtypes.py         |  2 +-
 tensorflow/tools/api/generator/BUILD          |  2 -
 .../tools/api/generator/create_python_api.py  | 21 +----
 tensorflow/tools/api/tests/BUILD              |  1 +
 .../tools/api/tests/api_compatibility_test.py | 48 +++++++++-
 .../ci_build/windows/cpu/cmake/run_py.bat     |  6 +-
 11 files changed, 121 insertions(+), 105 deletions(-)
 create mode 100644 tensorflow/experimental_api.py

diff --git a/tensorflow/BUILD b/tensorflow/BUILD
index cfafffdd13..3d5737a9d7 100644
--- a/tensorflow/BUILD
+++ b/tensorflow/BUILD
@@ -540,6 +540,14 @@ py_library(
     srcs = ["__init__.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
+    deps = ["//tensorflow/python"],
+)
+
+py_library(
+    name = "experimental_tensorflow_py",
+    srcs = ["experimental_api.py"],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow/tools/api/tests:__subpackages__"],
     deps = [
         "//tensorflow/python",
         "//tensorflow/tools/api/generator:python_api",
diff --git a/tensorflow/__init__.py b/tensorflow/__init__.py
index c8683e3976..78ad6aec19 100644
--- a/tensorflow/__init__.py
+++ b/tensorflow/__init__.py
@@ -20,19 +20,14 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-# pylint: disable=g-bad-import-order
-from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
 # pylint: disable=wildcard-import
-from tensorflow.tools.api.generator.api import *  # pylint: disable=redefined-builtin
+from tensorflow.python import *  # pylint: disable=redefined-builtin
 # pylint: enable=wildcard-import
 
 from tensorflow.python.util.lazy_loader import LazyLoader
 contrib = LazyLoader('contrib', globals(), 'tensorflow.contrib')
 del LazyLoader
 
-from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
-app.flags = flags  # pylint: disable=undefined-variable
-
 del absolute_import
 del division
 del print_function
diff --git a/tensorflow/contrib/cmake/python_modules.txt b/tensorflow/contrib/cmake/python_modules.txt
index d5cf42b641..02c456c199 100644
--- a/tensorflow/contrib/cmake/python_modules.txt
+++ b/tensorflow/contrib/cmake/python_modules.txt
@@ -104,8 +104,6 @@ tensorflow/python/user_ops
 tensorflow/python/util
 tensorflow/python/util/protobuf
 tensorflow/tools
-tensorflow/tools/api
-tensorflow/tools/api/generator
 tensorflow/tools/graph_transforms
 tensorflow/contrib
 tensorflow/contrib/all_reduce
diff --git a/tensorflow/contrib/cmake/tf_python.cmake b/tensorflow/contrib/cmake/tf_python.cmake
index 20eeded8d3..fae45ead5c 100755
--- a/tensorflow/contrib/cmake/tf_python.cmake
+++ b/tensorflow/contrib/cmake/tf_python.cmake
@@ -686,77 +686,6 @@ AddUserOps(TARGET _beam_search_ops
     DEPENDS pywrap_tensorflow_internal tf_python_ops
     DISTCOPY ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/seq2seq/python/ops/)
 
-if(WIN32)
-  if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
-    add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.dll
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
-  else()
-    add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.dll
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
-      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.lib
-                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
-  endif()
-else()
-  add_custom_command(TARGET pywrap_tensorflow_internal POST_BUILD
-    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal.so
-                                     ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.so)
-endif()
-
-
-########################################################
-# Generate API __init__.py files.
-########################################################
-
-# Parse tensorflow/tools/api/generator/BUILD to get list of generated files.
-FILE(READ ${tensorflow_source_dir}/tensorflow/tools/api/generator/BUILD api_generator_BUILD_text)
-STRING(REGEX MATCH "# BEGIN GENERATED FILES.*# END GENERATED FILES" api_init_files_text ${api_generator_BUILD_text})
-string(REPLACE "# BEGIN GENERATED FILES" "" api_init_files_text ${api_init_files_text})
-string(REPLACE "# END GENERATED FILES" "" api_init_files_text ${api_init_files_text})
-string(REPLACE "," ";" api_init_files_list ${api_init_files_text})
-
-set(api_init_files "")
-foreach(api_init_file ${api_init_files_list})
-    string(STRIP "${api_init_file}" api_init_file)
-    if(api_init_file)
-        string(REPLACE "\"" "" api_init_file "${api_init_file}")  # Remove quotes
-        list(APPEND api_init_files "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/${api_init_file}")
-    endif()
-endforeach(api_init_file)
-set(api_init_list_file "${tensorflow_source_dir}/api_init_files_list.txt")
-file(WRITE "${api_init_list_file}" "${api_init_files}")
-
-# Run create_python_api.py to generate __init__.py files.
-add_custom_command(
-      OUTPUT ${api_init_files}
-      DEPENDS tf_python_ops tf_python_copy_scripts_to_destination pywrap_tensorflow_internal tf_python_touchup_modules tf_extension_ops
-
-      # tensorflow/__init__.py depends on files generated in this step. So, remove it while
-      # this step is running since the files aren't there yet.
-      COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
-                                         ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/final.__init__.py
-      COMMAND ${CMAKE_COMMAND} -E touch ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
-
-      # Run create_python_api.py to generate API init files.
-      COMMAND ${CMAKE_COMMAND} -E env PYTHONPATH=${CMAKE_CURRENT_BINARY_DIR}/tf_python ${PYTHON_EXECUTABLE}
-              "${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/tools/api/generator/create_python_api.py" "${api_init_list_file}"
-
-      # Re-add tensorflow/__init__.py back.
-      COMMAND ${CMAKE_COMMAND} -E remove -f ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
-      COMMAND ${CMAKE_COMMAND} -E rename ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/final.__init__.py
-                                         ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/__init__.py
-
-      COMMENT "Generating __init__.py files for Python API."
-      WORKING_DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/tf_python"
-)
-
-add_custom_target(tf_python_api SOURCES ${api_init_files})
-add_dependencies(tf_python_api tf_python_ops)
-
-
 ############################################################
 # Build a PIP package containing the TensorFlow runtime.
 ############################################################
@@ -766,7 +695,6 @@ add_dependencies(tf_python_build_pip_package
     tf_python_copy_scripts_to_destination
     tf_python_touchup_modules
     tf_python_ops
-    tf_python_api
     tf_extension_ops)
 
 # Fix-up Python files that were not included by the add_python_module() macros.
@@ -779,6 +707,25 @@ add_custom_command(TARGET tf_python_copy_scripts_to_destination PRE_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/contrib/testing/python/framework/util_test.py
                                    ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/contrib/testing/python/framework/)
 
+if(WIN32)
+  if(${CMAKE_GENERATOR} MATCHES "Visual Studio.*")
+    add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.dll
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/$(Configuration)/pywrap_tensorflow_internal.lib
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
+  else()
+    add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.dll
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.pyd
+      COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/pywrap_tensorflow_internal.lib
+                                       ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/)
+  endif()
+else()
+  add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${CMAKE_CURRENT_BINARY_DIR}/libpywrap_tensorflow_internal.so
+                                     ${CMAKE_CURRENT_BINARY_DIR}/tf_python/tensorflow/python/_pywrap_tensorflow_internal.so)
+endif()
 add_custom_command(TARGET tf_python_build_pip_package POST_BUILD
   COMMAND ${CMAKE_COMMAND} -E copy ${tensorflow_source_dir}/tensorflow/tools/pip_package/README
                                    ${CMAKE_CURRENT_BINARY_DIR}/tf_python/)
diff --git a/tensorflow/experimental_api.py b/tensorflow/experimental_api.py
new file mode 100644
index 0000000000..63a8aa9cb1
--- /dev/null
+++ b/tensorflow/experimental_api.py
@@ -0,0 +1,38 @@
+# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+# Bring in all of the public TensorFlow interface into this
+# module.
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+# pylint: disable=g-bad-import-order
+from tensorflow.python import pywrap_tensorflow  # pylint: disable=unused-import
+# pylint: disable=wildcard-import
+from tensorflow.tools.api.generator.api import *  # pylint: disable=redefined-builtin
+# pylint: enable=wildcard-import
+
+from tensorflow.python.util.lazy_loader import LazyLoader
+contrib = LazyLoader('contrib', globals(), 'tensorflow.contrib')
+del LazyLoader
+
+from tensorflow.python.platform import flags  # pylint: disable=g-import-not-at-top
+app.flags = flags  # pylint: disable=undefined-variable
+
+del absolute_import
+del division
+del print_function
diff --git a/tensorflow/python/framework/dtypes.py b/tensorflow/python/framework/dtypes.py
index a31c424263..0edae92fd4 100644
--- a/tensorflow/python/framework/dtypes.py
+++ b/tensorflow/python/framework/dtypes.py
@@ -345,7 +345,7 @@ tf_export("uint16").export_constant(__name__, "uint16")
 uint32 = DType(types_pb2.DT_UINT32)
 tf_export("uint32").export_constant(__name__, "uint32")
 uint64 = DType(types_pb2.DT_UINT64)
-tf_export("uint64").export_constant(__name__, "uint64")
+tf_export("uint64").export_constant(__name__, "uint32")
 int16 = DType(types_pb2.DT_INT16)
 tf_export("int16").export_constant(__name__, "int16")
 int8 = DType(types_pb2.DT_INT8)
diff --git a/tensorflow/tools/api/generator/BUILD b/tensorflow/tools/api/generator/BUILD
index a1c569951e..9f1bdd8aae 100644
--- a/tensorflow/tools/api/generator/BUILD
+++ b/tensorflow/tools/api/generator/BUILD
@@ -32,7 +32,6 @@ genrule(
     # api/module1/module2/__init__.py and api/module3/__init__.py.
     # keep sorted
     outs = [
-        # BEGIN GENERATED FILES
         "api/__init__.py",
         "api/app/__init__.py",
         "api/bitwise/__init__.py",
@@ -118,7 +117,6 @@ genrule(
         "api/train/__init__.py",
         "api/train/queue_runner/__init__.py",
         "api/user_ops/__init__.py",
-        # END GENERATED FILES
     ],
     cmd = "$(location create_python_api) $(OUTS)",
     tools = ["create_python_api"],
diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py
index 1505dc69b9..183c4731b8 100644
--- a/tensorflow/tools/api/generator/create_python_api.py
+++ b/tensorflow/tools/api/generator/create_python_api.py
@@ -195,19 +195,16 @@ def create_api_files(output_files):
   """
   module_name_to_file_path = {}
   for output_file in output_files:
-    # Convert path separators to '/' for easier parsing below.
-    normalized_output_file = output_file.replace(os.sep, '/')
     if _API_DIR not in output_file:
       raise ValueError(
           'Output files must be in api/ directory, found %s.' % output_file)
     # Get the module name that corresponds to output_file.
     # First get module directory under _API_DIR.
     module_dir = os.path.dirname(
-        normalized_output_file[
-            normalized_output_file.rfind(_API_DIR)+len(_API_DIR):])
+        output_file[output_file.rfind(_API_DIR)+len(_API_DIR):])
     # Convert / to .
     module_name = module_dir.replace('/', '.').strip('.')
-    module_name_to_file_path[module_name] = os.path.normpath(output_file)
+    module_name_to_file_path[module_name] = output_file
 
   # Create file for each expected output in genrule.
   for module, file_path in module_name_to_file_path.items():
@@ -244,16 +241,6 @@ if __name__ == '__main__':
   parser = argparse.ArgumentParser()
   parser.add_argument(
       'outputs', metavar='O', type=str, nargs='+',
-      help='If a single file is passed in, then we we assume it contains a '
-      'semicolon-separated list of Python files that we expect this script to '
-      'output. If multiple files are passed in, then we assume output files '
-      'are listed directly as arguments.')
+      help='Python files that we expect this script to output.')
   args = parser.parse_args()
-  if len(args.outputs) == 1:
-    # If we only get a single argument, then it must be a file containing
-    # list of outputs.
-    with open(args.outputs[0]) as output_list_file:
-      outputs = [line.strip() for line in output_list_file.read().split(';')]
-  else:
-    outputs = args.outputs
-  main(outputs)
+  main(args.outputs)
diff --git a/tensorflow/tools/api/tests/BUILD b/tensorflow/tools/api/tests/BUILD
index 724b12cd47..0dc154b6d2 100644
--- a/tensorflow/tools/api/tests/BUILD
+++ b/tensorflow/tools/api/tests/BUILD
@@ -23,6 +23,7 @@ py_test(
     ],
     srcs_version = "PY2AND3",
     deps = [
+        "//tensorflow:experimental_tensorflow_py",
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:lib",
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 26d5bca637..603b2a4327 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -34,6 +34,7 @@ import sys
 import unittest
 
 import tensorflow as tf
+from tensorflow import experimental_api as api
 
 from google.protobuf import text_format
 
@@ -46,6 +47,8 @@ from tensorflow.tools.api.lib import python_object_to_proto_visitor
 from tensorflow.tools.common import public_api
 from tensorflow.tools.common import traverse
 
+if hasattr(tf, 'experimental_api'):
+  del tf.experimental_api
 
 # FLAGS defined at the bottom:
 FLAGS = None
@@ -205,8 +208,6 @@ class ApiCompatibilityTest(test.TestCase):
     public_api_visitor = public_api.PublicAPIVisitor(visitor)
     public_api_visitor.do_not_descend_map['tf'].append('contrib')
     public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
-    # TODO(annarev): Make slide_dataset available in API.
-    public_api_visitor.private_map['tf'] = ['slide_dataset']
     traverse.traverse(tf, public_api_visitor)
 
     proto_dict = visitor.GetProtos()
@@ -236,6 +237,49 @@ class ApiCompatibilityTest(test.TestCase):
         verbose=FLAGS.verbose_diffs,
         update_goldens=FLAGS.update_goldens)
 
+  @unittest.skipUnless(
+      sys.version_info.major == 2,
+      'API compabitility test goldens are generated using python2.')
+  def testNewAPIBackwardsCompatibility(self):
+    # Extract all API stuff.
+    visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
+
+    public_api_visitor = public_api.PublicAPIVisitor(visitor)
+    public_api_visitor.do_not_descend_map['tf'].append('contrib')
+    public_api_visitor.do_not_descend_map['tf.GPUOptions'] = ['Experimental']
+    # TODO(annarev): Make slide_dataset available in API.
+    public_api_visitor.private_map['tf'] = ['slide_dataset']
+    traverse.traverse(api, public_api_visitor)
+
+    proto_dict = visitor.GetProtos()
+
+    # Read all golden files.
+    expression = os.path.join(
+        resource_loader.get_root_dir_with_all_resources(),
+        _KeyToFilePath('*'))
+    golden_file_list = file_io.get_matching_files(expression)
+
+    def _ReadFileToProto(filename):
+      """Read a filename, create a protobuf from its contents."""
+      ret_val = api_objects_pb2.TFAPIObject()
+      text_format.Merge(file_io.read_file_to_string(filename), ret_val)
+      return ret_val
+
+    golden_proto_dict = {
+        _FileNameToKey(filename): _ReadFileToProto(filename)
+        for filename in golden_file_list
+    }
+
+    # Diff them. Do not fail if called with update.
+    # If the test is run to update goldens, only report diffs but do not fail.
+    self._AssertProtoDictEquals(
+        golden_proto_dict,
+        proto_dict,
+        verbose=FLAGS.verbose_diffs,
+        update_goldens=False,
+        additional_missing_object_message=
+        'Check if tf_export decorator/call is missing for this symbol.')
+
 
 if __name__ == '__main__':
   parser = argparse.ArgumentParser()
diff --git a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
index 30554a084c..3c3b223a00 100644
--- a/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
+++ b/tensorflow/tools/ci_build/windows/cpu/cmake/run_py.bat
@@ -28,9 +28,6 @@ IF DEFINED TF_NIGHTLY (ECHO TF_NIGHTLY is set to %TF_NIGHTLY%) ELSE (SET TF_NIGH
 :: Set pip binary location. Do not override if it is set already.
 IF DEFINED PIP_EXE (ECHO PIP_EXE is set to %PIP_EXE%) ELSE (SET PIP_EXE="C:\Program Files\Anaconda3\Scripts\pip.exe")
 
-:: Install absl-py.
-%PIP_EXE% install --upgrade absl-py
-
 :: Run the CMAKE build to build the pip package.
 CALL %REPO_ROOT%\tensorflow\tools\ci_build\windows\cpu\cmake\run_build.bat
 if %errorlevel% neq 0 exit /b %errorlevel%
@@ -40,6 +37,9 @@ DIR %REPO_ROOT%\%BUILD_DIR%\tf_python\dist\ /S /B > wheel_filename_file
 set /p WHEEL_FILENAME=<wheel_filename_file
 del wheel_filename_file
 
+:: Install absl-py.
+%PIP_EXE% install --upgrade absl-py
+
 :: Install the pip package.
 echo Installing PIP package...
 %PIP_EXE% install --upgrade --no-deps %WHEEL_FILENAME% -v -v
-- 
GitLab


From b5b87738dc03015939add7820da3d5dd2a744c84 Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Mon, 2 Apr 2018 15:55:01 -0700
Subject: [PATCH 830/960] Java: Update to 1.7.0

PiperOrigin-RevId: 191361364
---
 tensorflow/java/maven/libtensorflow/pom.xml         | 2 +-
 tensorflow/java/maven/libtensorflow_jni/pom.xml     | 2 +-
 tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml | 2 +-
 tensorflow/java/maven/pom.xml                       | 2 +-
 tensorflow/java/maven/proto/pom.xml                 | 2 +-
 tensorflow/java/maven/tensorflow/pom.xml            | 2 +-
 6 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/java/maven/libtensorflow/pom.xml b/tensorflow/java/maven/libtensorflow/pom.xml
index 0b69a8cbe5..c99d04869a 100644
--- a/tensorflow/java/maven/libtensorflow/pom.xml
+++ b/tensorflow/java/maven/libtensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc1</version>
+    <version>1.7.0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni/pom.xml b/tensorflow/java/maven/libtensorflow_jni/pom.xml
index 541876f7f5..4561c2c8ad 100644
--- a/tensorflow/java/maven/libtensorflow_jni/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc1</version>
+    <version>1.7.0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni</artifactId>
diff --git a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
index d8933e5238..82a2b8e769 100644
--- a/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
+++ b/tensorflow/java/maven/libtensorflow_jni_gpu/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc1</version>
+    <version>1.7.0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>libtensorflow_jni_gpu</artifactId>
diff --git a/tensorflow/java/maven/pom.xml b/tensorflow/java/maven/pom.xml
index 6286fd73df..4c1ec0cc80 100644
--- a/tensorflow/java/maven/pom.xml
+++ b/tensorflow/java/maven/pom.xml
@@ -6,7 +6,7 @@
   <modelVersion>4.0.0</modelVersion>
   <groupId>org.tensorflow</groupId>
   <artifactId>parentpom</artifactId>
-  <version>1.7.0-rc1</version>
+  <version>1.7.0</version>
   <packaging>pom</packaging>
 
   <url>https://www.tensorflow.org</url>
diff --git a/tensorflow/java/maven/proto/pom.xml b/tensorflow/java/maven/proto/pom.xml
index 4e881f5a63..fcd8236bad 100644
--- a/tensorflow/java/maven/proto/pom.xml
+++ b/tensorflow/java/maven/proto/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc1</version>
+    <version>1.7.0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>proto</artifactId>
diff --git a/tensorflow/java/maven/tensorflow/pom.xml b/tensorflow/java/maven/tensorflow/pom.xml
index d512a7eda9..241581713a 100644
--- a/tensorflow/java/maven/tensorflow/pom.xml
+++ b/tensorflow/java/maven/tensorflow/pom.xml
@@ -6,7 +6,7 @@
   <parent>
     <groupId>org.tensorflow</groupId>
     <artifactId>parentpom</artifactId>
-    <version>1.7.0-rc1</version>
+    <version>1.7.0</version>
     <relativePath>../</relativePath>
   </parent>
   <artifactId>tensorflow</artifactId>
-- 
GitLab


From 817eddd18b7c1c569bb6d284f13bd9d496b415a9 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 2 Apr 2018 16:21:37 -0700
Subject: [PATCH 831/960] Clarify ResourceVariable specification

PiperOrigin-RevId: 191365224
---
 tensorflow/python/ops/resource_variable_ops.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/resource_variable_ops.py b/tensorflow/python/ops/resource_variable_ops.py
index 2f39ea2e7d..07e25e540c 100644
--- a/tensorflow/python/ops/resource_variable_ops.py
+++ b/tensorflow/python/ops/resource_variable_ops.py
@@ -171,7 +171,9 @@ class ResourceVariable(variables.Variable):
   to see all modifications to the value of the variable which happen in any
   operation on which the read_value depends on (either directly, indirectly, or
   via a control dependency) and guaranteed to not see any modification to the
-  value of the variable on which the read_value operation does not depend on.
+  value of the variable from operations that depend on the read_value operation.
+  Updates from operations that have no dependency relationship to the read_value
+  operation might or might not be visible to read_value.
 
   For example, if there is more than one assignment to a ResourceVariable in
   a single session.run call there is a well-defined value for each operation
-- 
GitLab


From 19f7990d06b672e9a8f5085b42bb6822e4877a8b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 16:45:16 -0700
Subject: [PATCH 832/960] Add a config option to run Grappler optimizers more
 than once. Don't crash in layout optimizer if no cluster is given. Clean up
 Cluster::DisableOptimizer() so it actually turns all current optimizers off.

PiperOrigin-RevId: 191368433
---
 tensorflow/core/grappler/clusters/cluster.cc  |  4 +
 .../grappler/optimizers/layout_optimizer.cc   |  4 +
 .../grappler/optimizers/meta_optimizer.cc     | 82 +++++++++++--------
 .../optimizers/meta_optimizer_test.cc         | 14 ++++
 .../core/protobuf/rewriter_config.proto       | 12 +++
 5 files changed, 80 insertions(+), 36 deletions(-)

diff --git a/tensorflow/core/grappler/clusters/cluster.cc b/tensorflow/core/grappler/clusters/cluster.cc
index 39bfca244e..8d8c6084ec 100644
--- a/tensorflow/core/grappler/clusters/cluster.cc
+++ b/tensorflow/core/grappler/clusters/cluster.cc
@@ -62,6 +62,10 @@ void Cluster::DisableOptimizer(bool disable) {
         options_.config.mutable_graph_options()->mutable_rewrite_options();
     rewriter_config->set_layout_optimizer(RewriterConfig::OFF);
     rewriter_config->set_disable_model_pruning(true);
+    rewriter_config->set_function_optimization(RewriterConfig::OFF);
+    rewriter_config->set_arithmetic_optimization(RewriterConfig::OFF);
+    rewriter_config->set_loop_optimization(RewriterConfig::OFF);
+    rewriter_config->set_dependency_optimization(RewriterConfig::OFF);
     rewriter_config->set_constant_folding(RewriterConfig::OFF);
     rewriter_config->set_memory_optimization(RewriterConfig::NO_MEM_OPT);
     rewriter_config->mutable_auto_parallel()->set_enable(false);
diff --git a/tensorflow/core/grappler/optimizers/layout_optimizer.cc b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
index 254c1edf7b..308eecd420 100644
--- a/tensorflow/core/grappler/optimizers/layout_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/layout_optimizer.cc
@@ -2119,6 +2119,10 @@ Status LayoutOptimizer::Tune(const GrapplerItem& item,
 
 Status LayoutOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                  GraphDef* output) {
+  if (cluster == nullptr) {
+    return errors::InvalidArgument("cluster == nullptr");
+  }
+
   if (GetNumGPUs(*cluster) < 1) {
     // LayoutOptimizer is currently only tuned for GPU.
     *output = item.graph;
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index ad655db727..ce27d3d95c 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -44,16 +44,15 @@ int64 NumEdges(const GraphDef& graph) {
 }
 
 string PrintSizesBeforeAfter(const GraphDef& before, const GraphDef& after) {
-  return strings::StrCat("Graph size before: ", before.node_size(), " nodes, ",
-                         NumEdges(before),
-                         " edges. Graph size after: ", after.node_size(),
-                         " nodes, ", NumEdges(after), " edges.");
+  return strings::StrCat("Graph size after: ", after.node_size(), " nodes (",
+                         after.node_size() - before.node_size(), "), ",
+                         NumEdges(after), " edges (",
+                         NumEdges(after) - NumEdges(before), ")");
 }
 }  // namespace
 
 std::unique_ptr<GraphOptimizer> MetaOptimizer::NewOptimizer(
     const string& optimizer) {
-  VLOG(1) << "Adding graph optimization pass: " << optimizer;
   std::unique_ptr<GraphOptimizer> graph_optimizer;
   if (optimizer == "pruning") {
     graph_optimizer.reset(new ModelPruner());
@@ -171,41 +170,52 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     return Status::OK();
   }
 
+  // Some optimizers should be run only once.
+  const std::set<string> run_once_optimizers = {"layout"};
   bool already_optimized = false;
-  for (const auto& optimizer : optimizers) {
-    if (!already_optimized) {
-      Status status = optimizer->Optimize(cluster, item, optimized_graph);
-      string result;
-      if (!status.ok()) {
-        VLOG(1) << "Not able to apply optimizer " << optimizer->name()
-                << ". Return status: " << status.ToString();
-        result = status.ToString();
-      } else {
-        already_optimized = true;
-        result = strings::StrCat(
-            "OK. ", PrintSizesBeforeAfter(item.graph, *optimized_graph));
+  const int num_iterations =
+      cfg_.meta_optimizer_iterations() == RewriterConfig::DEFAULT_NUM_ITERS
+          ? 1
+          : cfg_.meta_optimizer_iterations();
+  for (int iteration = 0; iteration < num_iterations; ++iteration) {
+    VLOG(1) << "Starting optimization iteration " << iteration + 1;
+    for (const auto& optimizer : optimizers) {
+      if (iteration > 0 && run_once_optimizers.count(optimizer->name())) {
+        continue;
       }
-      result_.push_back(std::make_pair(optimizer->name(), result));
-      VLOG(1) << "Optimizer " << optimizer->name()
-              << " return status: " << result;
-    } else {
-      GrapplerItem optimized_item(item, std::move(*optimized_graph));
-      Status status =
-          optimizer->Optimize(cluster, optimized_item, optimized_graph);
-      string result;
-      if (!status.ok()) {
-        VLOG(1) << "Not able to apply optimizer " << optimizer->name()
-                << ". Return status: " << status.ToString();
-        optimized_graph->Swap(&optimized_item.graph);
-        result = status.ToString();
+      if (!already_optimized) {
+        Status status = optimizer->Optimize(cluster, item, optimized_graph);
+        string result;
+        if (!status.ok()) {
+          VLOG(1) << "Not able to apply optimizer " << optimizer->name()
+                  << ". Return status: " << status.ToString();
+          result = status.ToString();
+        } else {
+          already_optimized = true;
+          result = strings::StrCat(
+              "OK. ", PrintSizesBeforeAfter(item.graph, *optimized_graph));
+        }
+        result_.push_back(std::make_pair(optimizer->name(), result));
+        VLOG(1) << "Optimizer " << optimizer->name()
+                << " return status: " << result;
       } else {
-        result = strings::StrCat(
-            "OK. ",
-            PrintSizesBeforeAfter(optimized_item.graph, *optimized_graph));
+        GrapplerItem optimized_item(item, std::move(*optimized_graph));
+        Status status =
+            optimizer->Optimize(cluster, optimized_item, optimized_graph);
+        string result;
+        if (!status.ok()) {
+          VLOG(1) << "Not able to apply optimizer " << optimizer->name() << ": "
+                  << status.ToString();
+          optimized_graph->Swap(&optimized_item.graph);
+          result = status.ToString();
+        } else {
+          result = strings::StrCat(
+              optimizer->name(), ": ",
+              PrintSizesBeforeAfter(optimized_item.graph, *optimized_graph));
+        }
+        result_.push_back(std::make_pair(optimizer->name(), result));
+        VLOG(1) << result;
       }
-      result_.push_back(std::make_pair(optimizer->name(), result));
-      VLOG(1) << "Optimizer " << optimizer->name()
-              << " return status: " << result;
     }
   }
 
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
index 536347d834..d9a386b9be 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer_test.cc
@@ -72,6 +72,20 @@ TEST(MetaOptimizerTest, RunsCustomOptimizer) {
   EXPECT_TRUE(TestOptimizer::IsOptimized());
 }
 
+TEST(MetaOptimizerTest, RunOptimizersTwice) {
+  TrivialTestGraphInputYielder fake_input(4, 1, 10, false, {"CPU:0"});
+  GrapplerItem item;
+  CHECK(fake_input.NextItem(&item));
+
+  RewriterConfig rewriter_config;
+  rewriter_config.set_meta_optimizer_iterations(RewriterConfig::TWO);
+
+  MetaOptimizer optimizer(nullptr, rewriter_config);
+  GraphDef output;
+  const Status status = optimizer.Optimize(nullptr, item, &output);
+  TF_EXPECT_OK(status);
+}
+
 }  // namespace
 }  // namespace grappler
 }  // namespace tensorflow
diff --git a/tensorflow/core/protobuf/rewriter_config.proto b/tensorflow/core/protobuf/rewriter_config.proto
index bb772460b0..9b6202e7b4 100644
--- a/tensorflow/core/protobuf/rewriter_config.proto
+++ b/tensorflow/core/protobuf/rewriter_config.proto
@@ -29,6 +29,14 @@ message RewriterConfig {
     AGGRESSIVE = 3;
   }
 
+  // Enum controling the number of times to run optimizers. The default is to
+  // run them once.
+  enum NumIterationsType {
+    DEFAULT_NUM_ITERS = 0;
+    ONE = 1;
+    TWO = 2;
+  }
+
   // Optimize tensor layouts (default is ON)
   // e.g. This will try to use NCHW layout on GPU which is faster.
   Toggle layout_optimizer = 1;
@@ -51,6 +59,10 @@ message RewriterConfig {
   // If true, don't remove unnecessary ops from the graph
   bool disable_model_pruning = 2;
 
+  // Controls how many times we run the optimizers in meta optimizer (default
+  // is once).
+  NumIterationsType meta_optimizer_iterations = 12;
+
   enum MemOptType {
     // The default setting (SCHEDULING and SWAPPING HEURISTICS only)
     DEFAULT_MEM_OPT = 0;
-- 
GitLab


From 7e9113ab912caff9ad15195b15771ff20bde6080 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 17:08:27 -0700
Subject: [PATCH 833/960] [XLA] Redesign: implement ExecuteGraphParallel.

PiperOrigin-RevId: 191371793
---
 tensorflow/compiler/xla/client/client.cc     |  34 ++-
 tensorflow/compiler/xla/service/service.cc   | 229 ++++++++++++++++---
 tensorflow/compiler/xla/service/service.h    |  22 ++
 tensorflow/compiler/xla/tests/BUILD          |   2 +
 tensorflow/compiler/xla/tests/client_test.cc |  10 +-
 5 files changed, 259 insertions(+), 38 deletions(-)

diff --git a/tensorflow/compiler/xla/client/client.cc b/tensorflow/compiler/xla/client/client.cc
index c4c8894374..3f45167fcb 100644
--- a/tensorflow/compiler/xla/client/client.cc
+++ b/tensorflow/compiler/xla/client/client.cc
@@ -324,8 +324,38 @@ StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
 
 StatusOr<std::vector<std::unique_ptr<GlobalData>>> Client::ExecuteParallel(
     tensorflow::gtl::ArraySlice<XlaComputationInstance> computations) {
-  return Unimplemented(
-      "ExecuteParallel is not yet implemented for XlaComputation.");
+  ExecuteGraphParallelRequest request;
+
+  for (const XlaComputationInstance& computation : computations) {
+    ExecuteGraphRequest single_request;
+    *single_request.mutable_computation() = computation.computation.proto();
+    for (GlobalData* argument : computation.arguments) {
+      *single_request.add_arguments() = argument->handle();
+    }
+    *single_request.mutable_execution_options() = computation.execution_options;
+    *request.add_requests() = single_request;
+  }
+
+  ExecuteParallelResponse response;
+  VLOG(1) << "making execute-graph-parallel request: "
+          << request.ShortDebugString();
+  tensorflow::Status s = stub_->ExecuteGraphParallel(&request, &response);
+  VLOG(1) << "done with request";
+
+  if (!s.ok()) {
+    return s;
+  }
+
+  std::vector<std::unique_ptr<GlobalData>> outputs;
+  for (size_t i = 0; i < computations.size(); ++i) {
+    outputs.push_back(
+        MakeUnique<GlobalData>(stub_, response.responses(i).output()));
+    if (computations[i].execution_profile != nullptr) {
+      *computations[i].execution_profile = response.responses(i).profile();
+    }
+  }
+
+  return std::move(outputs);
 }
 
 StatusOr<std::vector<DeviceHandle>> Client::GetDeviceHandles(
diff --git a/tensorflow/compiler/xla/service/service.cc b/tensorflow/compiler/xla/service/service.cc
index ca8071b7bb..ec883a6cf3 100644
--- a/tensorflow/compiler/xla/service/service.cc
+++ b/tensorflow/compiler/xla/service/service.cc
@@ -409,6 +409,37 @@ StatusOr<std::vector<std::unique_ptr<Executable>>> Service::BuildExecutables(
   return std::move(executables);
 }
 
+StatusOr<std::vector<std::unique_ptr<Executable>>> Service::BuildExecutables(
+    const std::vector<const HloModuleProto*>& module_protos,
+    std::vector<std::unique_ptr<HloModuleConfig>> module_configs,
+    Backend* backend,
+    std::vector<std::vector<perftools::gputools::StreamExecutor*>> executors,
+    DeviceMemoryAllocator* device_allocator) {
+  VLOG(1) << Printf("BuildExecutable on service %p", this);
+
+  VLOG(1) << "Computations:";
+  for (const HloModuleProto* proto : module_protos) {
+    VLOG(1) << proto->name();
+  }
+
+  CHECK_EQ(module_protos.size(), module_configs.size());
+  std::vector<std::unique_ptr<HloModule>> modules;
+  for (int64 i = 0; i < module_protos.size(); ++i) {
+    const HloModuleProto* proto = module_protos[i];
+    const HloModuleConfig& config = *module_configs[i];
+    TF_ASSIGN_OR_RETURN(auto module,
+                        HloModule::CreateFromProto(*proto, config));
+    modules.push_back(std::move(module));
+  }
+
+  TF_ASSIGN_OR_RETURN(
+      std::vector<std::unique_ptr<Executable>> executables,
+      backend->compiler()->Compile(std::move(modules), std::move(executors),
+                                   device_allocator));
+
+  return std::move(executables);
+}
+
 StatusOr<std::unique_ptr<Executable>> Service::BuildExecutable(
     const VersionedComputationHandle& versioned_handle,
     std::unique_ptr<HloModuleConfig> module_config, Backend* backend,
@@ -703,6 +734,47 @@ tensorflow::Status Service::SetReturnValue(const SetReturnValueRequest* arg,
   return computation->SetReturnValue(arg->operand());
 }
 
+StatusOr<std::vector<perftools::gputools::StreamExecutor*>>
+Service::GetExecutors(const ExecutionOptions& execution_options,
+                      int64 requests_size, int64 request_index) const {
+  if (execution_options.device_handles().empty()) {
+    return FailedPrecondition(
+        "device handles must be given to execute parallel computations");
+  }
+  if (requests_size > 1 && execution_options.device_handles_size() > 1) {
+    return InvalidArgument(
+        "Parallel requests with multiple device handles is not supported. "
+        "Found %lld parallel requests, with request %lld containing %d device "
+        "handles.",
+        requests_size, request_index, execution_options.device_handles_size());
+  }
+  std::vector<perftools::gputools::StreamExecutor*> executors;
+  for (const auto& device_handle : execution_options.device_handles()) {
+    TF_ASSIGN_OR_RETURN(auto replicas,
+                        Replicas(*execute_backend_, device_handle));
+    se::StreamExecutor* executor = replicas[0];
+    CHECK(executor != nullptr);
+    executors.push_back(executor);
+  }
+  return executors;
+}
+
+StatusOr<std::vector<std::vector<const ShapedBuffer*>>> Service::GetArguments(
+    const ExecutionOptions& execution_options,
+    tensorflow::gtl::ArraySlice<const GlobalDataHandle*> arguments) {
+  // Resolve the allocations for the arguments of the computation, and create
+  // a vector of device memory offsets for the arguments from the allocations.
+  // In the case of partitioned computations, assume all arguments go on the
+  // zeroth core.
+  TF_ASSIGN_OR_RETURN(
+      auto replicas,
+      Replicas(*execute_backend_, execution_options.device_handles(0)));
+  TF_ASSIGN_OR_RETURN(
+      std::vector<std::vector<const ShapedBuffer*>> replicated_arguments,
+      ResolveAndValidateArguments(arguments, replicas));
+  return replicated_arguments;
+}
+
 tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
                                             ExecuteParallelResponse* result) {
   VLOG(1) << "running execute-parallel request: " << arg->ShortDebugString();
@@ -731,26 +803,10 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
     // is one of the executors to run the replicated computation.
     const ExecutionOptions& execution_options =
         arg->requests(i).execution_options();
-    if (execution_options.device_handles().empty()) {
-      return FailedPrecondition(
-          "device handles must be given to execute parallel computations");
-    }
-    if (arg->requests_size() > 1 &&
-        execution_options.device_handles_size() > 1) {
-      return InvalidArgument(
-          "Parallel requests with multiple device handles is not supported. "
-          "Found %d parallel requests, with request %lld containing %d device "
-          "handles.",
-          arg->requests_size(), i, execution_options.device_handles_size());
-    }
-    std::vector<perftools::gputools::StreamExecutor*> executors;
-    for (const auto& device_handle : execution_options.device_handles()) {
-      TF_ASSIGN_OR_RETURN(auto replicas,
-                          Replicas(*execute_backend_, device_handle));
-      se::StreamExecutor* executor = replicas[0];
-      CHECK(executor != nullptr);
-      executors.push_back(executor);
-    }
+
+    // Get the executors.
+    TF_ASSIGN_OR_RETURN(auto executors, GetExecutors(execution_options,
+                                                     arg->requests_size(), i));
 
     // Resolve the UserComputation object associated with the requested
     // computation and compute the program shape.
@@ -767,16 +823,9 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
         std::shared_ptr<const ProgramShape> program_shape,
         user_computation->ComputeProgramShape(versioned_handle.version));
 
-    // Resolve the allocations for the arguments of the computation, and create
-    // a vector of device memory offsets for the arguments from the allocations.
-    // In the case of partitioned computations, assume all arguments go on the
-    // zeroth core.
-    TF_ASSIGN_OR_RETURN(
-        auto replicas,
-        Replicas(*execute_backend_, execution_options.device_handles(0)));
-    TF_ASSIGN_OR_RETURN(
-        std::vector<std::vector<const ShapedBuffer*>> replicated_arguments,
-        ResolveAndValidateArguments(request.arguments(), replicas));
+    // Get the replicated arguments.
+    TF_ASSIGN_OR_RETURN(auto replicated_arguments,
+                        GetArguments(execution_options, request.arguments()));
 
     // Create an HloModuleConfig object for the computation, given the shape of
     // the program and the argument allocations. Here, we care only about the
@@ -839,7 +888,103 @@ tensorflow::Status Service::ExecuteParallel(const ExecuteParallelRequest* arg,
 
 tensorflow::Status Service::ExecuteGraphParallel(
     const ExecuteGraphParallelRequest* arg, ExecuteParallelResponse* result) {
-  return Unimplemented("execute-graph-parallel is not yet implemented");
+  VLOG(1) << "running execute-graph-parallel request";
+
+  std::vector<std::vector<std::vector<const ShapedBuffer*>>> all_arguments;
+  std::vector<std::vector<perftools::gputools::StreamExecutor*>> all_executors;
+  std::vector<const HloModuleProto*> module_protos;
+  std::vector<std::unique_ptr<HloModuleConfig>> module_configs;
+  std::vector<string> computation_names;
+  std::vector<DeviceHandle> device_handles;
+
+  int num_requested_devices =
+      std::accumulate(arg->requests().begin(), arg->requests().end(), 0,
+                      [](int a, const ExecuteGraphRequest& r) -> int {
+                        return a + r.execution_options().device_handles_size();
+                      });
+  if (num_requested_devices * options_.number_of_replicas() >
+      execute_backend_->device_count()) {
+    return FailedPrecondition(
+        "there are not enough stream executors to execute %d computations",
+        num_requested_devices);
+  }
+
+  for (int64 i = 0; i < arg->requests_size(); ++i) {
+    // Get the stream executor for the i'th computation. This stream executor
+    // is one of the executors to run the replicated computation.
+    const ExecutionOptions& execution_options =
+        arg->requests(i).execution_options();
+    const ExecuteGraphRequest& request = arg->requests(i);
+    TF_RET_CHECK(request.has_computation()) << "computations may not be empty";
+    TF_RET_CHECK(request.computation().has_program_shape())
+        << "programe shape may not be empty";
+
+    // Get the executors.
+    TF_ASSIGN_OR_RETURN(auto executors, GetExecutors(execution_options,
+                                                     arg->requests_size(), i));
+
+    // Get the replicated arguments.
+    TF_ASSIGN_OR_RETURN(auto replicated_arguments,
+                        GetArguments(execution_options, request.arguments()));
+
+    // Create an HloModuleConfig object for the computation, given the shape of
+    // the program and the argument allocations. Here, we care only about the
+    // shapes of the arguments, so, it is sufficient to use the arguments of
+    // replica 0.
+    TF_ASSIGN_OR_RETURN(
+        std::unique_ptr<HloModuleConfig> module_config,
+        CreateModuleConfig(request.computation().program_shape(),
+                           replicated_arguments.front(),
+                           request.execution_options(),
+                           /*user_computation=*/nullptr));
+    VLOG(3)
+        << "ExecuteGraphParallel created HloModuleConfig computation layout: "
+        << module_config->entry_computation_layout().ToString();
+
+    // Adds to the vectors to build and execute the computations after the loop.
+    all_arguments.push_back(replicated_arguments);
+    all_arguments.insert(all_arguments.end(), executors.size() - 1, {{}});
+    module_protos.push_back(&request.computation());
+    module_configs.push_back(std::move(module_config));
+    computation_names.insert(computation_names.end(), executors.size(),
+                             request.computation().name());
+    all_executors.push_back(executors);
+    device_handles.insert(device_handles.end(),
+                          execution_options.device_handles().begin(),
+                          execution_options.device_handles().end());
+  }
+
+  // Build the HloModules and compile to generate the executables.
+  //
+  // TODO(jlebar): There's currently no way to pass a device allocator to
+  // ExecuteGraphParallel, so we have to pass a null device_allocator below.
+  TF_ASSIGN_OR_RETURN(std::vector<std::unique_ptr<Executable>> executables,
+                      BuildExecutables(module_protos, std::move(module_configs),
+                                       execute_backend_.get(), all_executors,
+                                       /*device_allocator=*/nullptr));
+  std::vector<Executable*> executable_ptrs;
+  executable_ptrs.reserve(executables.size());
+  for (const auto& executable : executables) {
+    executable_ptrs.push_back(executable.get());
+  }
+
+  // Execute the generated executables in parallel and return the device
+  // handles for each computation's output.
+  ExecutionProfile profile;
+  TF_ASSIGN_OR_RETURN(
+      std::vector<GlobalDataHandle> outputs,
+      ExecuteParallelAndRegisterResult(executable_ptrs, all_arguments,
+                                       execute_backend_.get(), device_handles,
+                                       computation_names, &profile));
+  for (const GlobalDataHandle& output : outputs) {
+    ExecuteResponse response;
+    *response.mutable_output() = output;
+    *response.mutable_profile() = profile;
+    *result->add_responses() = response;
+  }
+
+  VLOG(1) << "successfully completed 'execute-graph-parallel' request";
+  return tensorflow::Status::OK();
 }
 
 tensorflow::Status Service::GetDeviceHandles(const GetDeviceHandlesRequest* arg,
@@ -872,6 +1017,20 @@ tensorflow::Status Service::ExecuteOneToN(const ExecuteRequest* arg,
   *parallel_arg.add_requests() = *arg;
   ExecuteParallelResponse parallel_result;
   TF_RETURN_IF_ERROR(ExecuteParallel(&parallel_arg, &parallel_result));
+  return PickParallelResponse(parallel_result, result);
+}
+
+tensorflow::Status Service::ExecuteOneToN(const ExecuteGraphRequest* arg,
+                                          ExecuteResponse* result) {
+  ExecuteGraphParallelRequest parallel_arg;
+  *parallel_arg.add_requests() = *arg;
+  ExecuteParallelResponse parallel_result;
+  TF_RETURN_IF_ERROR(ExecuteGraphParallel(&parallel_arg, &parallel_result));
+  return PickParallelResponse(parallel_result, result);
+}
+
+tensorflow::Status Service::PickParallelResponse(
+    const ExecuteParallelResponse& parallel_result, ExecuteResponse* result) {
   // The "result device" selection is a bit hacky, but better than assuming it
   // is device 0. We have b/76035356 for restructuring the client API to clean
   // up the current asymmetries and support more functionalities.
@@ -999,8 +1158,14 @@ tensorflow::Status Service::ExecuteGraph(const ExecuteGraphRequest* arg,
   if (!arg->has_computation()) {
     return InvalidArgument("computations may not be empty");
   }
+  if (!arg->computation().has_program_shape()) {
+    return InvalidArgument("programe shape may not be empty");
+  }
 
-  // TODO(b/74197823): Handle partitioning.
+  // If we received multiple device handles, we must partition the module.
+  if (arg->execution_options().device_handles_size() > 1) {
+    return ExecuteOneToN(arg, result);
+  }
 
   TF_ASSIGN_OR_RETURN(auto replicas, Replicas(*execute_backend_,
                                               SingleComputationDeviceHandle()));
diff --git a/tensorflow/compiler/xla/service/service.h b/tensorflow/compiler/xla/service/service.h
index ebe4a2e043..e09d58bbe7 100644
--- a/tensorflow/compiler/xla/service/service.h
+++ b/tensorflow/compiler/xla/service/service.h
@@ -278,6 +278,20 @@ class Service : public ServiceInterface {
       const ExecutionOptions& execution_options,
       const UserComputation* user_computation = nullptr);
 
+  // Picks a parallel response and fills the result.
+  Status PickParallelResponse(const ExecuteParallelResponse& parallel_result,
+                              ExecuteResponse* result);
+
+  // Prepare the executors for executing parallel.
+  StatusOr<std::vector<perftools::gputools::StreamExecutor*>> GetExecutors(
+      const ExecutionOptions& execution_options, int64 requests_size,
+      int64 request_index) const;
+
+  // Prepare the arguments for executing parallel.
+  StatusOr<std::vector<std::vector<const ShapedBuffer*>>> GetArguments(
+      const ExecutionOptions& execution_options,
+      tensorflow::gtl::ArraySlice<const GlobalDataHandle*> arguments);
+
  protected:
   friend class LocalExecutable;
 
@@ -334,6 +348,12 @@ class Service : public ServiceInterface {
       Backend* backend,
       std::vector<std::vector<perftools::gputools::StreamExecutor*>> executors,
       DeviceMemoryAllocator* device_allocator);
+  StatusOr<std::vector<std::unique_ptr<Executable>>> BuildExecutables(
+      const std::vector<const HloModuleProto*>& module_protos,
+      std::vector<std::unique_ptr<HloModuleConfig>> module_configs,
+      Backend* backend,
+      std::vector<std::vector<perftools::gputools::StreamExecutor*>> executors,
+      DeviceMemoryAllocator* device_allocator);
 
   // Similar to BuildExecutable, but look in the compilation cache for the
   // executable first. If the executable is not in the cache, it is built and
@@ -378,6 +398,8 @@ class Service : public ServiceInterface {
   // will be the result of this computation.
   tensorflow::Status ExecuteOneToN(const ExecuteRequest* arg,
                                    ExecuteResponse* result);
+  tensorflow::Status ExecuteOneToN(const ExecuteGraphRequest* arg,
+                                   ExecuteResponse* result);
 
   // Convenience function which checks whether the given shape_with_layout
   // (presumably passed by the client to set the result layout) is valid for the
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 283efbb707..9cead12eba 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -1566,6 +1566,8 @@ xla_test(
         "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:global_data",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:test_utils",
diff --git a/tensorflow/compiler/xla/tests/client_test.cc b/tensorflow/compiler/xla/tests/client_test.cc
index 045148cdd1..32e2f2c084 100644
--- a/tensorflow/compiler/xla/tests/client_test.cc
+++ b/tensorflow/compiler/xla/tests/client_test.cc
@@ -19,6 +19,8 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/global_data.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/shape_util.h"
 #include "tensorflow/compiler/xla/status_macros.h"
 #include "tensorflow/compiler/xla/statusor.h"
@@ -109,14 +111,14 @@ XLA_TEST_F(ClientTest, ExecuteWithTupleLayout) {
 
 XLA_TEST_F(ClientTest,
         DISABLED_ON_CPU_PARALLEL(DISABLED_ON_GPU(ExecuteParallel))) {
-  Computation add_with_one_arg, mul_with_two_args, dot_with_one_arg;
+  XlaComputation add_with_one_arg, mul_with_two_args, dot_with_one_arg;
   Shape shape = ShapeUtil::MakeShape(S32, {2, 2});
 
   TF_ASSERT_OK_AND_ASSIGN(
       std::unique_ptr<GlobalData> const_arg,
       client_->TransferToServer(*Literal::CreateR2<int32>({{5, 6}, {7, 8}})));
 
-  ComputationBuilder b(client_, TestName() + ".add");
+  XlaBuilder b(TestName() + ".add");
   b.Add(b.Parameter(0, shape, "param_0"),
         b.ConstantR2<int32>({{1, 2}, {3, 4}}));
   TF_ASSERT_OK_AND_ASSIGN(add_with_one_arg, b.Build());
@@ -124,14 +126,14 @@ XLA_TEST_F(ClientTest,
   // We can't really test parallel execution on CPU since all of the cores in a
   // CPU are presented as a single device.  So for now we test "parallel"
   // execution on a single device.
-  std::vector<Client::ComputationInstance> computation_instances;
+  std::vector<Client::XlaComputationInstance> computation_instances;
   TF_ASSERT_OK_AND_ASSIGN(std::vector<xla::DeviceHandle> devices,
                           client_->GetDeviceHandles(1));
   ASSERT_EQ(devices.size(), 1);
 
   ExecutionOptions options = execution_options_;
   *options.add_device_handles() = devices[0];
-  computation_instances.push_back(Client::ComputationInstance(
+  computation_instances.push_back(Client::XlaComputationInstance(
       add_with_one_arg, {const_arg.get()}, options, nullptr));
 
   TF_ASSERT_OK_AND_ASSIGN(auto results,
-- 
GitLab


From 03613455c1c1c3957aedc4edcedd96a21bf9a514 Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Mon, 2 Apr 2018 17:21:41 -0700
Subject: [PATCH 834/960] TFTS: Add a OneShotPredictionHead with no model state
 in its serving signature.

PiperOrigin-RevId: 191373516
---
 .../timeseries/python/timeseries/BUILD        | 15 +++-
 .../python/timeseries/estimators.py           |  8 +-
 .../timeseries/python/timeseries/head.py      | 84 +++++++++++++-----
 .../timeseries/python/timeseries/head_test.py | 86 ++++++++++++++++++-
 4 files changed, 166 insertions(+), 27 deletions(-)

diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 55a25e39fe..86022f46ce 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -88,10 +88,14 @@ py_library(
         "//tensorflow/python:array_ops",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
+        "//tensorflow/python:parsing_ops",
         "//tensorflow/python:tensor_shape",
+        "//tensorflow/python:tensor_util",
         "//tensorflow/python:training",
+        "//tensorflow/python:util",
         "//tensorflow/python/estimator:estimator_py",
         "//tensorflow/python/estimator:export",
+        "//tensorflow/python/feature_column",
     ],
 )
 
@@ -132,7 +136,6 @@ py_library(
     srcs_version = "PY2AND3",
     deps = [
         ":feature_keys",
-        "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/layers:layers_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
@@ -141,6 +144,7 @@ py_library(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:state_ops",
         "//tensorflow/python:summary",
+        "//tensorflow/python:training",
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python/estimator:estimator_py",
@@ -160,19 +164,28 @@ py_test(
         "no_pip_gpu",  # b/63391119
     ],
     deps = [
+        ":estimators",
         ":feature_keys",
         ":head",
+        ":input_pipeline",
         ":model",
         ":state_management",
+        "//tensorflow/contrib/timeseries/examples:lstm",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:client_testlib",
         "//tensorflow/python:dtypes",
         "//tensorflow/python:framework_ops",
         "//tensorflow/python:math_ops",
         "//tensorflow/python:metrics",
+        "//tensorflow/python:session",
         "//tensorflow/python:training",
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator:estimator_py",
+        "//tensorflow/python/feature_column",
+        "//tensorflow/python/saved_model:loader",
+        "//tensorflow/python/saved_model:tag_constants",
+        "//third_party/py/numpy",
+        "@six_archive//:six",
     ],
 )
 
diff --git a/tensorflow/contrib/timeseries/python/timeseries/estimators.py b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
index 469cea4fd2..886e1846e2 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/estimators.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/estimators.py
@@ -44,7 +44,7 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
   """An Estimator to fit and evaluate a time series model."""
 
   def __init__(self, model, state_manager=None, optimizer=None, model_dir=None,
-               config=None):
+               config=None, head_type=ts_head_lib.TimeSeriesRegressionHead):
     """Initialize the Estimator.
 
     Args:
@@ -55,6 +55,8 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
           from tf.train.Optimizer. Defaults to Adam with step size 0.02.
       model_dir: See `Estimator`.
       config: See `Estimator`.
+      head_type: The kind of head to use for the model (inheriting from
+          `TimeSeriesRegressionHead`).
     """
     input_statistics_generator = math_utils.InputStatisticsFromMiniBatch(
         dtype=model.dtype, num_features=model.num_features)
@@ -63,8 +65,8 @@ class TimeSeriesRegressor(estimator_lib.Estimator):
     if optimizer is None:
       optimizer = train.AdamOptimizer(0.02)
     self._model = model
-    ts_regression_head = ts_head_lib.time_series_regression_head(
-        model, state_manager, optimizer,
+    ts_regression_head = head_type(
+        model=model, state_manager=state_manager, optimizer=optimizer,
         input_statistics_generator=input_statistics_generator)
     model_fn = ts_regression_head.create_estimator_spec
     super(TimeSeriesRegressor, self).__init__(
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head.py b/tensorflow/contrib/timeseries/python/timeseries/head.py
index 71085f9de8..a28a5872b8 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head.py
@@ -39,25 +39,6 @@ from tensorflow.python.util import nest
 from tensorflow.python.summary import summary
 
 
-def time_series_regression_head(model,
-                                state_manager,
-                                optimizer,
-                                input_statistics_generator=None):
-  """Creates a `_Head` for time series regression.
-
-  Args:
-    model: A model for time series regression.
-    state_manager: A state manager.
-    optimizer: An optimizer.
-    input_statistics_generator: A input statistics generator.
-
-  Returns:
-    An instance of `_Head` for time series regression.
-  """
-  return _TimeSeriesRegressionHead(model, state_manager, optimizer,
-                                   input_statistics_generator)
-
-
 class _NoStatePredictOutput(export_lib.PredictOutput):
 
   def as_signature_def(self, receiver_tensors):
@@ -68,8 +49,8 @@ class _NoStatePredictOutput(export_lib.PredictOutput):
         receiver_tensors=no_state_receiver_tensors)
 
 
-class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-access
-  """See `time_series_regression_head`."""
+class TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-access
+  """Determines input and output signatures for a time series model."""
 
   def __init__(self,
                model,
@@ -77,6 +58,15 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
                optimizer,
                input_statistics_generator=None,
                name=None):
+    """Creates a `_Head` for time series regression.
+
+    Args:
+      model: A model for time series regression.
+      state_manager: A state manager.
+      optimizer: An optimizer.
+      input_statistics_generator: A input statistics generator.
+      name: An optional name for the model.
+    """
     self.model = model
     self.state_manager = state_manager
     self.optimizer = optimizer
@@ -265,6 +255,58 @@ class _TimeSeriesRegressionHead(head_lib._Head):  # pylint:disable=protected-acc
         return self._serving_ops(features)
 
 
+class OneShotPredictionHead(TimeSeriesRegressionHead):
+  """A time series head which exports a single stateless serving signature.
+
+  The serving default signature exported by this head expects `times`, `values`,
+  and any exogenous features, but no state. `values` has shape `[batch_size,
+  filter_length, num_features]` and `times` has shape `[batch_size,
+  total_length]`, where `total_length > filter_length`. Any exogenous features
+  must have their shapes prefixed by the shape of the `times` feature.
+
+  When serving, first performs filtering on the series up to `filter_length`
+  starting from the default start state for the model, then computes predictions
+  on the remainder of the series, returning them.
+
+  Model state is neither accepted nor returned, so filtering must be performed
+  each time predictions are requested when using this head.
+  """
+
+  def _serving_ops(self, features):
+    """Add ops for serving to the graph."""
+    with variable_scope.variable_scope("model", use_resource=True):
+      filtering_features = {}
+      prediction_features = {}
+      values_length = array_ops.shape(
+          features[feature_keys.FilteringFeatures.VALUES])[1]
+      for key, value in features.items():
+        if key == feature_keys.State.STATE_TUPLE:
+          # Ignore state input. The model's default start state is replicated
+          # across the batch.
+          continue
+        if key == feature_keys.FilteringFeatures.VALUES:
+          filtering_features[key] = value
+        else:
+          filtering_features[key] = value[:, :values_length]
+          prediction_features[key] = value[:, values_length:]
+      cold_filtering_outputs = self.model.define_loss(
+          features=filtering_features, mode=estimator_lib.ModeKeys.EVAL)
+      prediction_features[feature_keys.State.STATE_TUPLE] = (
+          cold_filtering_outputs.end_state)
+    with variable_scope.variable_scope("model", reuse=True):
+      prediction_outputs = self.model.predict(
+          features=prediction_features)
+    return estimator_lib.EstimatorSpec(
+        mode=estimator_lib.ModeKeys.PREDICT,
+        export_outputs={
+            feature_keys.SavedModelLabels.PREDICT:
+                _NoStatePredictOutput(prediction_outputs),
+        },
+        # Likely unused, but it is necessary to return `predictions` to satisfy
+        # the Estimator's error checking.
+        predictions={})
+
+
 def _check_feature_shapes_compatible_with(features,
                                           compatible_with_name,
                                           compatible_with_value,
diff --git a/tensorflow/contrib/timeseries/python/timeseries/head_test.py b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
index 3415061cfd..c606db76a6 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/head_test.py
+++ b/tensorflow/contrib/timeseries/python/timeseries/head_test.py
@@ -18,12 +18,20 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import numpy
+import six
+
+from tensorflow.contrib.timeseries.examples import lstm as lstm_example
+from tensorflow.contrib.timeseries.python.timeseries import estimators as ts_estimators
 from tensorflow.contrib.timeseries.python.timeseries import feature_keys
 from tensorflow.contrib.timeseries.python.timeseries import head as ts_head_lib
+from tensorflow.contrib.timeseries.python.timeseries import input_pipeline
 from tensorflow.contrib.timeseries.python.timeseries import model
 from tensorflow.contrib.timeseries.python.timeseries import state_management
 
+from tensorflow.python.client import session as session_lib
 from tensorflow.python.estimator import estimator_lib
+from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
@@ -31,6 +39,9 @@ from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import metrics
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
+from tensorflow.python.saved_model import loader
+from tensorflow.python.saved_model import tag_constants
+from tensorflow.python.training import adam
 from tensorflow.python.training import coordinator as coordinator_lib
 from tensorflow.python.training import queue_runner_impl
 from tensorflow.python.training import training as train
@@ -90,7 +101,7 @@ class EvaluationMetricsTests(test.TestCase):
                       .count_up_to(10),
                       dtype=dtypes.float32), (1, 1, 1))
       }
-      model_fn = ts_head_lib.time_series_regression_head(
+      model_fn = ts_head_lib.TimeSeriesRegressionHead(
           model=_TickerModel(),
           state_manager=state_management.PassthroughStateManager(),
           optimizer=train.GradientDescentOptimizer(0.001)).create_estimator_spec
@@ -127,7 +138,7 @@ class _StubModel(object):
 
 
 def _stub_model_fn():
-  return ts_head_lib.time_series_regression_head(
+  return ts_head_lib.TimeSeriesRegressionHead(
       model=_StubModel(),
       state_manager=state_management.PassthroughStateManager(),
       optimizer=train.AdamOptimizer(0.001)).create_estimator_spec
@@ -263,5 +274,76 @@ class PredictFeatureCheckingTests(test.TestCase):
           mode=estimator_lib.ModeKeys.PREDICT)
 
 
+class OneShotTests(test.TestCase):
+
+  def test_one_shot_prediction_head_export(self):
+    model_dir = self.get_temp_dir()
+    categorical_column = feature_column.categorical_column_with_hash_bucket(
+        key="categorical_exogenous_feature", hash_bucket_size=16)
+    exogenous_feature_columns = [
+        feature_column.numeric_column(
+            "2d_exogenous_feature", shape=(2,)),
+        feature_column.embedding_column(
+            categorical_column=categorical_column, dimension=10)]
+    estimator = ts_estimators.TimeSeriesRegressor(
+        model=lstm_example._LSTMModel(
+            num_features=5, num_units=128,
+            exogenous_feature_columns=exogenous_feature_columns),
+        optimizer=adam.AdamOptimizer(0.001),
+        config=estimator_lib.RunConfig(tf_random_seed=4),
+        state_manager=state_management.ChainingStateManager(),
+        head_type=ts_head_lib.OneShotPredictionHead,
+        model_dir=model_dir)
+    train_features = {
+        feature_keys.TrainEvalFeatures.TIMES: numpy.arange(
+            20, dtype=numpy.int64),
+        feature_keys.TrainEvalFeatures.VALUES: numpy.tile(numpy.arange(
+            20, dtype=numpy.float32)[:, None], [1, 5]),
+        "2d_exogenous_feature": numpy.ones([20, 2]),
+        "categorical_exogenous_feature": numpy.array(
+            ["strkey"] * 20)[:, None]
+    }
+    train_input_fn = input_pipeline.RandomWindowInputFn(
+        input_pipeline.NumpyReader(train_features), shuffle_seed=2,
+        num_threads=1, batch_size=16, window_size=16)
+    estimator.train(input_fn=train_input_fn, steps=5)
+    input_receiver_fn = estimator.build_raw_serving_input_receiver_fn()
+    export_location = estimator.export_savedmodel(self.get_temp_dir(),
+                                                  input_receiver_fn)
+    graph = ops.Graph()
+    with graph.as_default():
+      with session_lib.Session() as session:
+        signatures = loader.load(
+            session, [tag_constants.SERVING], export_location)
+        self.assertEqual([feature_keys.SavedModelLabels.PREDICT],
+                         list(signatures.signature_def.keys()))
+        predict_signature = signatures.signature_def[
+            feature_keys.SavedModelLabels.PREDICT]
+        six.assertCountEqual(
+            self,
+            [feature_keys.FilteringFeatures.TIMES,
+             feature_keys.FilteringFeatures.VALUES,
+             "2d_exogenous_feature",
+             "categorical_exogenous_feature"],
+            predict_signature.inputs.keys())
+        features = {
+            feature_keys.TrainEvalFeatures.TIMES: numpy.tile(
+                numpy.arange(35, dtype=numpy.int64)[None, :], [2, 1]),
+            feature_keys.TrainEvalFeatures.VALUES: numpy.tile(numpy.arange(
+                20, dtype=numpy.float32)[None, :, None], [2, 1, 5]),
+            "2d_exogenous_feature": numpy.ones([2, 35, 2]),
+            "categorical_exogenous_feature": numpy.tile(numpy.array(
+                ["strkey"] * 35)[None, :, None], [2, 1, 1])
+        }
+        feeds = {
+            graph.as_graph_element(input_value.name): features[input_key]
+            for input_key, input_value in predict_signature.inputs.items()}
+        fetches = {output_key: graph.as_graph_element(output_value.name)
+                   for output_key, output_value
+                   in predict_signature.outputs.items()}
+        output = session.run(fetches, feed_dict=feeds)
+        self.assertAllEqual((2, 15, 5), output["mean"].shape)
+
+
 if __name__ == "__main__":
   test.main()
-- 
GitLab


From ce1e141e5bbe0b6894a0bf600c3835c532145ba0 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 2 Apr 2018 17:31:47 -0700
Subject: [PATCH 835/960] Internal change.

PiperOrigin-RevId: 191374719
---
 tensorflow/python/ops/math_ops.py                    |  1 -
 tensorflow/tools/api/tests/api_compatibility_test.py | 10 ++++++++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 1c20d004cb..39f40882db 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -71,7 +71,6 @@ See the @{$python/math_ops} guide.
 @@igammac
 @@zeta
 @@polygamma
-@@polyval
 @@betainc
 @@rint
 @@diag
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index 603b2a4327..7eeae05847 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -145,6 +145,9 @@ class ApiCompatibilityTest(test.TestCase):
       verbose_diff_message = ''
       # First check if the key is not found in one or the other.
       if key in only_in_expected:
+        # TODO(annarev): remove once we switch to using tf_export decorators.
+        if key == 'tensorflow.math':
+          continue
         diff_message = 'Object %s expected but not found (removed). %s' % (
             key, additional_missing_object_message)
         verbose_diff_message = diff_message
@@ -229,6 +232,13 @@ class ApiCompatibilityTest(test.TestCase):
         for filename in golden_file_list
     }
 
+    # TODO(annarev): remove once we switch to using tf_export decorators.
+    tf_module = golden_proto_dict['tensorflow'].tf_module
+    for i in range(len(tf_module.member)):
+      if tf_module.member[i].name == 'math':
+        del tf_module.member[i]
+        break
+
     # Diff them. Do not fail if called with update.
     # If the test is run to update goldens, only report diffs but do not fail.
     self._AssertProtoDictEquals(
-- 
GitLab


From e1f8206f8d2704132d546175c29c0693911ec240 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Mon, 2 Apr 2018 17:50:12 -0700
Subject: [PATCH 836/960] Don't use session context manager when we need
 session to outlive the context block.

PiperOrigin-RevId: 191376772
---
 tensorflow/python/keras/_impl/keras/estimator.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/estimator.py b/tensorflow/python/keras/_impl/keras/estimator.py
index 8426d84df9..5d370ebbb5 100644
--- a/tensorflow/python/keras/_impl/keras/estimator.py
+++ b/tensorflow/python/keras/_impl/keras/estimator.py
@@ -466,8 +466,8 @@ def model_to_estimator(keras_model=None,
       keras_model_fn, model_dir=model_dir, config=config)
 
   # Pass the config into keras backend's default session.
-  with session.Session(config=estimator._session_config) as sess:
-    K.set_session(sess)
+  sess = session.Session(config=estimator._session_config)
+  K.set_session(sess)
 
   keras_weights = keras_model.get_weights()
   if keras_model._is_graph_network:
-- 
GitLab


From 556b4055a42eaaee248239e90b35f90cf2e89dcf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 18:18:11 -0700
Subject: [PATCH 837/960] [XLA] Redesign: migrate while_test to use XlaBuilder,
 and implement the ops needed to pass the tests, including: While, Slice,
 DynamicUpdateSlice, Dot, DotGeneral, ConvertElementType, Rng, and Reduce.

Also, when a module has embedded computaitons, the service side would complain if the instruction names are not unique in the scope of the module. To ensure instruction names are unique in module, use both the computation id and instruction id as suffix.

PiperOrigin-RevId: 191379697
---
 tensorflow/compiler/xla/client/lib/BUILD      |   2 +
 .../compiler/xla/client/lib/arithmetic.cc     |  95 ++++-
 .../compiler/xla/client/lib/arithmetic.h      |  44 +++
 .../xla/client/xla_client/xla_builder.cc      | 163 ++++++--
 .../xla/client/xla_client/xla_builder.h       |  10 +-
 tensorflow/compiler/xla/tests/BUILD           |   4 +-
 tensorflow/compiler/xla/tests/reduce_test.cc  |  69 ++--
 tensorflow/compiler/xla/tests/while_test.cc   | 368 +++++++++---------
 8 files changed, 513 insertions(+), 242 deletions(-)

diff --git a/tensorflow/compiler/xla/client/lib/BUILD b/tensorflow/compiler/xla/client/lib/BUILD
index d02972f2c0..f4673a8204 100644
--- a/tensorflow/compiler/xla/client/lib/BUILD
+++ b/tensorflow/compiler/xla/client/lib/BUILD
@@ -24,6 +24,8 @@ cc_library(
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:computation",
         "//tensorflow/compiler/xla/client:computation_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/core:lib",
     ],
 )
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.cc b/tensorflow/compiler/xla/client/lib/arithmetic.cc
index 24048a1e5a..63df449e0b 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.cc
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.cc
@@ -26,6 +26,7 @@ limitations under the License.
 
 namespace xla {
 namespace {
+
 using InstructionGenerator =
     ComputationDataHandle (*)(ComputationBuilder*, const ComputationDataHandle&,
                               const ComputationDataHandle&);
@@ -47,6 +48,27 @@ Computation CreateScalarComputation(const string& name, PrimitiveType type,
   generator(b.get(), lhs, rhs);
   return b->BuildAndNoteError();
 }
+
+using XlaOpGenerator = XlaOp (*)(XlaBuilder*, const XlaOp&, const XlaOp&);
+
+XlaComputation CreateScalarComputation(const string& name, PrimitiveType type,
+                                       XlaBuilder* builder,
+                                       XlaOpGenerator generator) {
+  std::unique_ptr<XlaBuilder> b;
+  if (type == PRED) {
+    b = builder->CreateSubBuilder(name);
+  } else {
+    b = builder->CreateSubBuilder(
+        tensorflow::strings::StrCat(name, "_", PrimitiveType_Name(type)));
+  }
+
+  const Shape scalar = ShapeUtil::MakeShape(type, {});
+  auto lhs = b->Parameter(0, scalar, "lhs");
+  auto rhs = b->Parameter(1, scalar, "rhs");
+  generator(b.get(), lhs, rhs);
+  return b->BuildAndNoteError();
+}
+
 }  // namespace
 
 Computation CreateScalarAddComputation(PrimitiveType type,
@@ -60,7 +82,7 @@ Computation CreateScalarAddComputation(PrimitiveType type,
 Computation CreateScalarMultiplyComputation(PrimitiveType type,
                                             ComputationBuilder* builder) {
   return CreateScalarComputation(
-      "add", type, builder,
+      "mul", type, builder,
       [](ComputationBuilder* b, const ComputationDataHandle& lhs,
          const ComputationDataHandle& rhs) { return b->Mul(lhs, rhs); });
 }
@@ -114,4 +136,75 @@ StatusOr<ComputationDataHandle> Any(const ComputationDataHandle& predicates,
   return builder->Reduce(predicates, f, logical_or, all_dimensions);
 }
 
+XlaComputation CreateScalarAddComputation(PrimitiveType type,
+                                          XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "add", type, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Add(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarMultiplyComputation(PrimitiveType type,
+                                               XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "mul", type, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Mul(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarGeComputation(PrimitiveType type,
+                                         XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "ge", type, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Ge(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarMaxComputation(PrimitiveType type,
+                                          XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "max", type, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Max(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarMinComputation(PrimitiveType type,
+                                          XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "min", type, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Min(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarAndComputation(XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "and", PRED, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->And(lhs, rhs);
+      });
+}
+
+XlaComputation CreateScalarOrComputation(XlaBuilder* builder) {
+  return CreateScalarComputation(
+      "or", PRED, builder,
+      [](XlaBuilder* b, const XlaOp& lhs, const XlaOp& rhs) {
+        return b->Or(lhs, rhs);
+      });
+}
+
+StatusOr<XlaOp> Any(const XlaOp& predicates, XlaBuilder* builder) {
+  auto f = builder->ConstantR0<bool>(false);
+  XlaComputation logical_or = CreateScalarOrComputation(builder);
+  TF_ASSIGN_OR_RETURN(const Shape& predicates_shape,
+                      builder->GetShape(predicates));
+  std::vector<int64> all_dimensions(ShapeUtil::Rank(predicates_shape));
+  std::iota(all_dimensions.begin(), all_dimensions.end(), 0);
+  return builder->Reduce(predicates, f, logical_or, all_dimensions);
+}
+
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/client/lib/arithmetic.h b/tensorflow/compiler/xla/client/lib/arithmetic.h
index ae89784bc2..f4d3fc8015 100644
--- a/tensorflow/compiler/xla/client/lib/arithmetic.h
+++ b/tensorflow/compiler/xla/client/lib/arithmetic.h
@@ -20,6 +20,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/client/computation.h"
 #include "tensorflow/compiler/xla/client/computation_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/xla_data.pb.h"
 
 namespace xla {
@@ -56,6 +58,48 @@ Computation CreateScalarOrComputation(ComputationBuilder* builder);
 StatusOr<ComputationDataHandle> Any(const ComputationDataHandle& predicates,
                                     ComputationBuilder* builder);
 
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar add computation and returns it.
+XlaComputation CreateScalarAddComputation(PrimitiveType type,
+                                          XlaBuilder* builder);
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar multiply computation and returns it.
+XlaComputation CreateScalarMultiplyComputation(PrimitiveType type,
+                                               XlaBuilder* builder);
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar ge computation and returns it.
+XlaComputation CreateScalarGeComputation(PrimitiveType type,
+                                         XlaBuilder* builder);
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar max computation and returns it.
+XlaComputation CreateScalarMaxComputation(PrimitiveType type,
+                                          XlaBuilder* builder);
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar min computation and returns it.
+XlaComputation CreateScalarMinComputation(PrimitiveType type,
+                                          XlaBuilder* builder);
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar logical AND computation and returns it.
+XlaComputation CreateScalarAndComputation(XlaBuilder* builder);
+
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Creates a scalar logical OR computation and returns it.
+XlaComputation CreateScalarOrComputation(XlaBuilder* builder);
+
+// TODO(b/74197823): This is a part of a NOT YET ready refactor.
+//
+// Returns whether any predicate in "predicates" is set.
+//
+// Note: if predicates is zero-sized, Any() vacuously returns false.
+StatusOr<XlaOp> Any(const XlaOp& predicates, XlaBuilder* builder);
+
 }  // namespace xla
 
 #endif  // TENSORFLOW_COMPILER_XLA_CLIENT_LIB_ARITHMETIC_H_
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index ec2362179e..c2e661cb3d 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -81,7 +81,7 @@ StatusOr<Shape> XlaOp::GetShape() const {
 }
 
 XlaBuilder::XlaBuilder(const string& computation_name)
-    : name_(computation_name) {}
+    : name_(computation_name), unique_id_(GetUniqueId()) {}
 
 XlaBuilder::~XlaBuilder() {}
 
@@ -179,7 +179,6 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
   }
 
   HloComputationProto entry;
-  entry.set_name(name_);
 
   {
     int64 root_id;
@@ -193,9 +192,9 @@ StatusOr<XlaComputation> XlaBuilder::Build() {
     entry.add_instructions()->Swap(&instruction);
   }
 
-  const int64 id = GetUniqueId();
-  entry.set_id(id);
-  XlaComputation computation(id);
+  entry.set_id(unique_id_);
+  entry.set_name(StrCat(name_, entry.id()));  // Ensure that the name is unique.
+  XlaComputation computation(entry.id());
   HloModuleProto* module = computation.mutable_proto();
   module->set_name(entry.name());
   module->set_id(entry.id());
@@ -407,12 +406,7 @@ XlaOp XlaBuilder::Call(const XlaComputation& computation,
         ShapeInference::InferCallShape(operand_shape_ptrs,
                                        /*to_apply=*/called_program_shape));
 
-    // Add called computation.
-    instr.add_called_computation_ids(
-        computation.proto().entry_computation_id());
-    for (const HloComputationProto& e : computation.proto().computations()) {
-      embedded_.insert({e.id(), e});
-    }
+    AddCalledComputation(computation, &instr);
 
     return AddInstruction(std::move(instr), HloOpcode::kCall, operands);
   });
@@ -470,7 +464,22 @@ XlaOp XlaBuilder::Slice(const XlaOp& operand,
                         tensorflow::gtl::ArraySlice<int64> start_indices,
                         tensorflow::gtl::ArraySlice<int64> limit_indices,
                         tensorflow::gtl::ArraySlice<int64> strides) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferSliceShape(operand_shape, start_indices,
+                                        limit_indices, strides));
+    for (int i = 0; i < start_indices.size(); i++) {
+      auto* slice_config = instr.add_slice_dimensions();
+      slice_config->set_start(start_indices[i]);
+      slice_config->set_limit(limit_indices[i]);
+      slice_config->set_stride(strides[i]);
+    }
+
+    return AddInstruction(std::move(instr), HloOpcode::kSlice, {operand});
+  });
 }
 
 XlaOp XlaBuilder::SliceInDim(const XlaOp& operand, int64 start_index,
@@ -485,7 +494,20 @@ XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
 
 XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
                                      const XlaOp& start_indices) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(const Shape& update_shape, GetShape(update));
+    TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
+                        GetShape(start_indices));
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferDynamicUpdateSliceShape(
+                            operand_shape, update_shape, start_indices_shape));
+
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicUpdateSlice,
+                          {operand, update, start_indices});
+  });
 }
 
 XlaOp XlaBuilder::ConcatInDim(tensorflow::gtl::ArraySlice<XlaOp> operands,
@@ -620,12 +642,29 @@ XlaOp XlaBuilder::Lt(const XlaOp& lhs, const XlaOp& rhs,
 }
 
 XlaOp XlaBuilder::Dot(const XlaOp& lhs, const XlaOp& rhs) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
+
+    DotDimensionNumbers dimension_numbers;
+    dimension_numbers.add_lhs_contracting_dimensions(
+        lhs_shape.dimensions_size() == 1 ? 0 : 1);
+    dimension_numbers.add_rhs_contracting_dimensions(0);
+    return DotGeneral(lhs, rhs, dimension_numbers);
+  });
 }
 
 XlaOp XlaBuilder::DotGeneral(const XlaOp& lhs, const XlaOp& rhs,
                              const DotDimensionNumbers& dimension_numbers) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& lhs_shape, GetShape(lhs));
+    TF_ASSIGN_OR_RETURN(const Shape& rhs_shape, GetShape(rhs));
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferDotOpShape(lhs_shape, rhs_shape,
+                                                        dimension_numbers));
+    *instr.mutable_dot_dimension_numbers() = dimension_numbers;
+    return AddInstruction(std::move(instr), HloOpcode::kDot, {lhs, rhs});
+  });
 }
 
 XlaOp XlaBuilder::Conv(const XlaOp& lhs, const XlaOp& rhs,
@@ -860,7 +899,14 @@ XlaOp XlaBuilder::Pow(const XlaOp& lhs, const XlaOp& rhs,
 
 XlaOp XlaBuilder::ConvertElementType(const XlaOp& operand,
                                      PrimitiveType new_element_type) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferConvertShape(operand_shape, new_element_type));
+    return AddInstruction(std::move(instr), HloOpcode::kConvert, {operand});
+  });
 }
 
 XlaOp XlaBuilder::BitcastConvertType(const XlaOp& operand,
@@ -894,19 +940,64 @@ XlaOp XlaBuilder::Map(tensorflow::gtl::ArraySlice<XlaOp> operands,
   return UnimplementedOp();
 }
 
+XlaOp XlaBuilder::RngOp(RandomDistribution distribution,
+                        tensorflow::gtl::ArraySlice<XlaOp> parameters,
+                        const Shape& shape) {
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    // Check the number of parameters per RNG distribution.
+    switch (distribution) {
+      case RandomDistribution::RNG_NORMAL:
+      case RandomDistribution::RNG_UNIFORM:
+        if (parameters.size() != 2) {
+          return InvalidArgument(
+              "RNG distribution (%s) expects 2 parameters, but got %ld",
+              RandomDistribution_Name(distribution).c_str(), parameters.size());
+        }
+        break;
+      default:
+        LOG(FATAL) << "unhandled distribution " << distribution;
+    }
+
+    TF_RETURN_IF_ERROR(ShapeUtil::ValidateShapeWithOptionalLayout(shape));
+    *instr.mutable_shape() = shape;
+
+    instr.set_distribution(distribution);
+
+    return AddInstruction(std::move(instr), HloOpcode::kRng, parameters);
+  });
+}
+
 XlaOp XlaBuilder::RngNormal(const XlaOp& mu, const XlaOp& sigma,
                             const Shape& shape) {
-  return UnimplementedOp();
+  return RngOp(RandomDistribution::RNG_NORMAL, {mu, sigma}, shape);
 }
 
 XlaOp XlaBuilder::RngUniform(const XlaOp& a, const XlaOp& b,
                              const Shape& shape) {
-  return UnimplementedOp();
+  return RngOp(RandomDistribution::RNG_UNIFORM, {a, b}, shape);
 }
 
 XlaOp XlaBuilder::While(const XlaComputation& condition,
                         const XlaComputation& body, const XlaOp& init) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    // Infer shape.
+    TF_ASSIGN_OR_RETURN(const auto& body_program_shape, body.GetProgramShape());
+    TF_ASSIGN_OR_RETURN(const auto& condition_program_shape,
+                        condition.GetProgramShape());
+    TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init));
+    TF_ASSIGN_OR_RETURN(
+        *instr.mutable_shape(),
+        ShapeInference::InferWhileShape(condition_program_shape,
+                                        body_program_shape, init_shape));
+    // Body comes before condition computation in the vector.
+    AddCalledComputation(body, &instr);
+    AddCalledComputation(condition, &instr);
+    return AddInstruction(std::move(instr), HloOpcode::kWhile, {init});
+  });
 }
 
 XlaOp XlaBuilder::Gather(const XlaOp& input, const XlaOp& gather_indices,
@@ -926,7 +1017,27 @@ XlaOp XlaBuilder::Reduce(
     const XlaOp& operand, const XlaOp& init_value,
     const XlaComputation& computation,
     tensorflow::gtl::ArraySlice<int64> dimensions_to_reduce) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(const Shape& init_shape, GetShape(init_value));
+    TF_ASSIGN_OR_RETURN(const ProgramShape& called_program_shape,
+                        computation.GetProgramShape());
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferReduceShape(
+                            operand_shape, init_shape, dimensions_to_reduce,
+                            called_program_shape));
+
+    for (int64 dim : dimensions_to_reduce) {
+      instr.add_dimensions(dim);
+    }
+
+    AddCalledComputation(computation, &instr);
+
+    return AddInstruction(std::move(instr), HloOpcode::kReduce,
+                          {operand, init_value});
+  });
 }
 
 XlaOp XlaBuilder::ReduceAll(const XlaOp& operand, const XlaOp& init_value,
@@ -1109,10 +1220,10 @@ StatusOr<XlaOp> XlaBuilder::AddInstruction(
   instr.set_id(handle);
   instr.set_opcode(HloOpcodeString(opcode));
   if (instr.name().empty()) {
-    instr.set_name(StrCat(instr.opcode(), ".", handle));
+    instr.set_name(StrCat(instr.opcode(), ".", unique_id_, ".", handle));
   } else {
     // Append the handle to make sure the name is unique.
-    instr.set_name(StrCat(instr.name(), ".", handle));
+    instr.set_name(StrCat(instr.name(), ".", unique_id_, ".", handle));
   }
   for (const auto& operand : operands) {
     if (operand.builder_ == nullptr) {
@@ -1138,6 +1249,14 @@ StatusOr<XlaOp> XlaBuilder::AddInstruction(
   return op;
 }
 
+void XlaBuilder::AddCalledComputation(const XlaComputation& computation,
+                                      HloInstructionProto* instr) {
+  instr->add_called_computation_ids(computation.proto().entry_computation_id());
+  for (const HloComputationProto& e : computation.proto().computations()) {
+    embedded_.insert({e.id(), e});
+  }
+}
+
 StatusOr<const HloInstructionProto*> XlaBuilder::LookUpInstruction(
     const XlaOp& op) const {
   TF_RETURN_IF_ERROR(first_error_);
diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.h b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
index f43101db34..0673b86646 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.h
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.h
@@ -803,6 +803,9 @@ class XlaBuilder {
       HloInstructionProto&& instr, HloOpcode opcode,
       tensorflow::gtl::ArraySlice<XlaOp> operands = {});
 
+  void AddCalledComputation(const XlaComputation& computation,
+                            HloInstructionProto* instr);
+
   // Notes that the error occurred by:
   // * storing it internally and capturing a backtrace if it's the first error
   //   (this deferred value will be produced on the call to Build())
@@ -829,6 +832,10 @@ class XlaBuilder {
   XlaOp TernaryOp(HloOpcode triop, const XlaOp& lhs, const XlaOp& rhs,
                   const XlaOp& ehs);
 
+  XlaOp RngOp(RandomDistribution distribution,
+              tensorflow::gtl::ArraySlice<XlaOp> parameters,
+              const Shape& shape);
+
   StatusOr<XlaOp> InDimBroadcast(
       const Shape& shape, const XlaOp& operand,
       tensorflow::gtl::ArraySlice<int64> broadcast_dimensions);
@@ -846,7 +853,8 @@ class XlaBuilder {
   // computation and fills the root_id in the pointer.
   StatusOr<ProgramShape> GetProgramShape(int64* root_id);
 
-  string name_;  // Name to use for the built computation.
+  string name_;      // Name to use for the built computation.
+  int64 unique_id_;  // The unique id for the built computation.
 
   // The first error encountered while building the computation.
   // This is OK until the first error is encountered.
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 9cead12eba..aba61fbca4 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -347,10 +347,10 @@ xla_test(
         "//tensorflow/compiler/xla:statusor",
         "//tensorflow/compiler/xla:xla_data_proto",
         "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:computation",
-        "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
         "//tensorflow/compiler/xla/client/lib:arithmetic",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
+        "//tensorflow/compiler/xla/client/xla_client:xla_computation",
         "//tensorflow/compiler/xla/service:platform_util",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
diff --git a/tensorflow/compiler/xla/tests/reduce_test.cc b/tensorflow/compiler/xla/tests/reduce_test.cc
index 3a097a01ab..d24927d22b 100644
--- a/tensorflow/compiler/xla/tests/reduce_test.cc
+++ b/tensorflow/compiler/xla/tests/reduce_test.cc
@@ -57,6 +57,11 @@ limitations under the License.
 namespace xla {
 namespace {
 
+using FuncGeneratorForType = Computation (*)(PrimitiveType,
+                                             ComputationBuilder*);
+
+using FuncGenerator = Computation (*)(ComputationBuilder*);
+
 class ReduceTest : public ClientLibraryTestBase {
  protected:
   ReduceTest() {
@@ -755,53 +760,57 @@ XLA_TEST_F(ReduceTest, ReduceR3AmongDim2) {
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_Add) {
-  RunVectorizedReduceTest(CreateScalarAddComputation,
-                          [](float a, float b) { return a + b; },
-                          [](int32 a, int32 b) {
-                            return static_cast<int32>(static_cast<uint32>(a) +
-                                                      static_cast<uint32>(b));
-                          },
-                          [](uint32 a, uint32 b) { return a + b; }, 0.0, 0, 0);
+  RunVectorizedReduceTest(
+      static_cast<FuncGeneratorForType>(CreateScalarAddComputation),
+      [](float a, float b) { return a + b; },
+      [](int32 a, int32 b) {
+        return static_cast<int32>(static_cast<uint32>(a) +
+                                  static_cast<uint32>(b));
+      },
+      [](uint32 a, uint32 b) { return a + b; }, 0.0, 0, 0);
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_Multiply) {
-  RunVectorizedReduceTest(CreateScalarMultiplyComputation,
-                          [](float a, float b) { return a * b; },
-                          [](int32 a, int32 b) {
-                            return static_cast<int32>(static_cast<uint32>(a) *
-                                                      static_cast<uint32>(b));
-                          },
-                          [](uint32 a, uint32 b) { return a * b; }, 1.0, 1, 1);
+  RunVectorizedReduceTest(
+      static_cast<FuncGeneratorForType>(CreateScalarMultiplyComputation),
+      [](float a, float b) { return a * b; },
+      [](int32 a, int32 b) {
+        return static_cast<int32>(static_cast<uint32>(a) *
+                                  static_cast<uint32>(b));
+      },
+      [](uint32 a, uint32 b) { return a * b; }, 1.0, 1, 1);
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_Max) {
-  RunVectorizedReduceTest(CreateScalarMaxComputation,
-                          [](float a, float b) { return std::max(a, b); },
-                          [](int32 a, int32 b) { return std::max(a, b); },
-                          [](uint32 a, uint32 b) { return std::max(a, b); },
-                          std::numeric_limits<float>::min(),
-                          std::numeric_limits<int32>::min(),
-                          std::numeric_limits<uint32>::min());
+  RunVectorizedReduceTest(
+      static_cast<FuncGeneratorForType>(CreateScalarMaxComputation),
+      [](float a, float b) { return std::max(a, b); },
+      [](int32 a, int32 b) { return std::max(a, b); },
+      [](uint32 a, uint32 b) { return std::max(a, b); },
+      std::numeric_limits<float>::min(), std::numeric_limits<int32>::min(),
+      std::numeric_limits<uint32>::min());
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_Min) {
-  RunVectorizedReduceTest(CreateScalarMinComputation,
-                          [](float a, float b) { return std::min(a, b); },
-                          [](int32 a, int32 b) { return std::min(a, b); },
-                          [](uint32 a, uint32 b) { return std::min(a, b); },
-                          std::numeric_limits<float>::max(),
-                          std::numeric_limits<int32>::max(),
-                          std::numeric_limits<uint32>::max());
+  RunVectorizedReduceTest(
+      static_cast<FuncGeneratorForType>(CreateScalarMinComputation),
+      [](float a, float b) { return std::min(a, b); },
+      [](int32 a, int32 b) { return std::min(a, b); },
+      [](uint32 a, uint32 b) { return std::min(a, b); },
+      std::numeric_limits<float>::max(), std::numeric_limits<int32>::max(),
+      std::numeric_limits<uint32>::max());
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanAnd) {
   RunVectorizedReduceTestForType<bool>(
-      CreateScalarAndComputation, [](bool a, bool b) { return a && b; }, true);
+      static_cast<FuncGenerator>(CreateScalarAndComputation),
+      [](bool a, bool b) { return a && b; }, true);
 }
 
 XLA_TEST_F(ReduceTest, VectorizedReduce_BooleanOr) {
   RunVectorizedReduceTestForType<bool>(
-      CreateScalarOrComputation, [](bool a, bool b) { return a || b; }, false);
+      static_cast<FuncGenerator>(CreateScalarOrComputation),
+      [](bool a, bool b) { return a || b; }, false);
 }
 
 class ReduceR3ToR2Test : public ReduceTest,
diff --git a/tensorflow/compiler/xla/tests/while_test.cc b/tensorflow/compiler/xla/tests/while_test.cc
index 33d457c70b..89ce2ce797 100644
--- a/tensorflow/compiler/xla/tests/while_test.cc
+++ b/tensorflow/compiler/xla/tests/while_test.cc
@@ -18,10 +18,10 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/compiler/xla/client/computation.h"
-#include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/lib/arithmetic.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_computation.h"
 #include "tensorflow/compiler/xla/literal_util.h"
 #include "tensorflow/compiler/xla/service/platform_util.h"
 #include "tensorflow/compiler/xla/shape_util.h"
@@ -54,29 +54,28 @@ TEST_F(WhileTest, WhileWithScalarS32Result) {
   auto result_shape = ShapeUtil::MakeShape(S32, {});
 
   // Create a computation for the condition: repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     builder.Gt(builder.ConstantR0<int32>(5), prev);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body: add 1 to the result variable.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR0<int32>(1);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.ConstantR0<int32>(0);
-  auto result = builder.While(condition, body, init);
-  auto shape = builder.GetShape(result).ConsumeValueOrDie();
+  builder.While(condition, body, init);
 
   ComputeAndCompareR0<int32>(&builder, 5, {});
 }
@@ -91,29 +90,28 @@ TEST_F(WhileTest, WhileWithScalarS64Result) {
   auto result_shape = ShapeUtil::MakeShape(S64, {});
 
   // Create a computation for the condition: repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     builder.Gt(builder.ConstantR0<int64>(5), prev);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body: add 1 to the result variable.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR0<int64>(1);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.ConstantR0<int64>(0);
-  auto result = builder.While(condition, body, init);
-  auto shape = builder.GetShape(result).ConsumeValueOrDie();
+  builder.While(condition, body, init);
 
   ComputeAndCompareR0<int64>(&builder, 5, {});
 }
@@ -123,31 +121,30 @@ TEST_F(WhileTest, WhileWithScalarResultNonConstInit) {
   auto orig_shape = ShapeUtil::MakeShape(S32, {2});
 
   // Create a computation for the condition: repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     builder.Gt(builder.ConstantR0<int32>(5), prev);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body: add 1 to the result variable.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR0<int32>(1);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.Reduce(builder.ConstantR1<int32>(2, 1),
                              builder.ConstantR0<int32>(0),
                              CreateScalarAddComputation(S32, &builder), {0});
-  auto result = builder.While(condition, body, init);
-  auto shape = builder.GetShape(result).ConsumeValueOrDie();
+  builder.While(condition, body, init);
 
   ComputeAndCompareR0<int32>(&builder, 5, {});
 }
@@ -156,28 +153,28 @@ TEST_F(WhileTest, WhileWithPredicateResult) {
   auto result_shape = ShapeUtil::MakeShape(PRED, {});
 
   // Create a computation for the condition: run until condition is true.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     builder.Ne(builder.ConstantR0<bool>(true), prev);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body: or condition with true.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
-    auto result = builder.Or(prev, builder.ConstantR0<bool>(true));
+    builder.Or(prev, builder.ConstantR0<bool>(true));
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.Ne(builder.ConstantR0<bool>(false),
                          builder.ConstantR0<bool>(true));
-  auto result = builder.While(condition, body, init);
+  builder.While(condition, body, init);
 
   ComputeAndCompareR0<bool>(&builder, true, {});
 }
@@ -194,9 +191,9 @@ TEST_F(WhileTest, DISABLED_WhileWithEmptyVectorResult) {
   Shape result_shape = ShapeUtil::MakeShape(F32, {0});
 
   // Create a computation for the reduction.
-  Computation add;
+  XlaComputation add;
   {
-    ComputationBuilder builder(client_, "add");
+    XlaBuilder builder("add");
     auto x = builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x");
     auto y = builder.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y");
     builder.Add(x, y);
@@ -205,33 +202,34 @@ TEST_F(WhileTest, DISABLED_WhileWithEmptyVectorResult) {
 
   // Create a computation for the condition.
   // Repeat until the sum of the result vector is less than 15.5f.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto sum = builder.Reduce(prev, builder.ConstantR0<float>(0.0f), add,
                               /*dimensions_to_reduce=*/{0});
-    auto test = builder.Gt(builder.ConstantR0<float>(15.5f), sum);
+    builder.Gt(builder.ConstantR0<float>(15.5f), sum);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body.
   // Add a constant vector of 1.f to the result vector.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR1<float>({});
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.ConstantR1<float>({});
   auto result = builder.While(condition, body, init);
-  VLOG(2) << "while = " << ShapeUtil::HumanString(
-                               *builder.GetShape(result).ConsumeValueOrDie());
+  VLOG(2) << "while = "
+          << ShapeUtil::HumanString(
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   ComputeAndCompareR1<float>(&builder, {}, {}, ErrorSpec(0.0001));
 }
@@ -247,9 +245,9 @@ TEST_F(WhileTest, WhileWithVectorResult) {
   Shape result_shape = ShapeUtil::MakeShape(F32, {8});
 
   // Create a computation for the reduction.
-  Computation add;
+  XlaComputation add;
   {
-    ComputationBuilder builder(client_, "add");
+    XlaBuilder builder("add");
     auto x = builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x");
     auto y = builder.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y");
     builder.Add(x, y);
@@ -258,33 +256,34 @@ TEST_F(WhileTest, WhileWithVectorResult) {
 
   // Create a computation for the condition.
   // Repeat until the sum of the result vector is less than 5.5f.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto sum = builder.Reduce(prev, builder.ConstantR0<float>(0.0f), add,
                               /*dimensions_to_reduce=*/{0});
-    auto test = builder.Gt(builder.ConstantR0<float>(15.5f), sum);
+    builder.Gt(builder.ConstantR0<float>(15.5f), sum);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body.
   // Add a constant vector of 1.f to the result vector.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR1<float>(8, 0.125f);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.ConstantR1<float>(8, 0.f);
   auto result = builder.While(condition, body, init);
-  VLOG(2) << "while = " << ShapeUtil::HumanString(
-                               *builder.GetShape(result).ConsumeValueOrDie());
+  VLOG(2) << "while = "
+          << ShapeUtil::HumanString(
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   // Individual elements with increase by 1/8 each time through the loop, so
   // the sum will increase by 1.0.  It will first be >15.5 when the elements
@@ -306,9 +305,9 @@ TEST_F(WhileTest, WhileWithVectorResultIntoTuple) {
   Shape result_shape = ShapeUtil::MakeShape(F32, {8});
 
   // Create a computation for the reduction.
-  Computation add;
+  XlaComputation add;
   {
-    ComputationBuilder builder(client_, "add");
+    XlaBuilder builder("add");
     auto x = builder.Parameter(0, ShapeUtil::MakeShape(F32, {}), "x");
     auto y = builder.Parameter(1, ShapeUtil::MakeShape(F32, {}), "y");
     builder.Add(x, y);
@@ -317,34 +316,34 @@ TEST_F(WhileTest, WhileWithVectorResultIntoTuple) {
 
   // Create a computation for the condition.
   // Repeat until the sum of the result vector is less than 5.5f.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto sum = builder.Reduce(prev, builder.ConstantR0<float>(0.0f), add,
                               /*dimensions_to_reduce=*/{0});
-    auto test = builder.Gt(builder.ConstantR0<float>(15.5f), sum);
+    builder.Gt(builder.ConstantR0<float>(15.5f), sum);
     condition = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a computation for the body.
   // Add a constant vector of 1.f to the result vector.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR1<float>(8, 0.125f);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.ConstantR1<float>(8, 0.f);
   auto result = builder.While(condition, body, init);
   VLOG(2) << "while = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
   builder.Tuple({result});
 
   // Individual elements with increase by 1/8 each time through the loop, so
@@ -366,9 +365,9 @@ TEST_F(WhileTest, WhileWithPermutationAndTupleResult) {
   // Create a computation for the condition.
   // Repeat for N iterations.
   const int N = 2;
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(N), iteration);
@@ -377,28 +376,28 @@ TEST_F(WhileTest, WhileWithPermutationAndTupleResult) {
 
   // Create a computation for the body.
   // Add 1 to the iteration variable and permute the weights.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto w1 = builder.GetTupleElement(prev, 1);
     auto w2 = builder.GetTupleElement(prev, 2);
     auto w3 = builder.GetTupleElement(prev, 3);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), w3, w1, w2});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(3, 1.f),
        builder.ConstantR1<float>(3, 2.f), builder.ConstantR1<float>(3, 3.f)});
   auto result = builder.While(condition, body, init);
   VLOG(2) << "result = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   auto expected_counter = Literal::CreateR0<int32>(N);
   auto expected_w1 = Literal::CreateR1<float>({1.0f, 1.0f, 1.0f});
@@ -419,9 +418,9 @@ TEST_F(WhileTest, WhileWithPermutationAndVectorResult) {
   // Create a computation for the condition.
   // Repeat for N iterations.
   const int N = 2;
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(N), iteration);
@@ -430,21 +429,21 @@ TEST_F(WhileTest, WhileWithPermutationAndVectorResult) {
 
   // Create a computation for the body.
   // Add 1 to the iteration variable permute the weights.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto w1 = builder.GetTupleElement(prev, 1);
     auto w2 = builder.GetTupleElement(prev, 2);
     auto w3 = builder.GetTupleElement(prev, 3);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), w3, w1, w2});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(3, 1.f),
        builder.ConstantR1<float>(3, 2.f), builder.ConstantR1<float>(3, 3.f)});
@@ -455,7 +454,7 @@ TEST_F(WhileTest, WhileWithPermutationAndVectorResult) {
   auto result = builder.Add(add12, builder.GetTupleElement(xla_while, 3));
   VLOG(2) << "result = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
   std::vector<float> expected = {6.f, 6.f, 6.f};
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
 }
@@ -474,9 +473,9 @@ TEST_F(WhileTest, WhileWithTupleResult) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(5), iteration);
@@ -486,26 +485,27 @@ TEST_F(WhileTest, WhileWithTupleResult) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and add a constant vector of 1.0f to
   // the weight variable, both of which are tuple elements.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto weights = builder.GetTupleElement(prev, 1);
     auto input = builder.ConstantR1<float>(10, 1.f);
     auto new_weights = builder.Add(weights, input);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_weights});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(10, 0.f)});
   auto result = builder.While(condition, body, init);
-  VLOG(2) << "while = " << ShapeUtil::HumanString(
-                               *builder.GetShape(result).ConsumeValueOrDie());
+  VLOG(2) << "while = "
+          << ShapeUtil::HumanString(
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   auto expected_counter = Literal::CreateR0<int32>(5);
   auto expected_data = Literal::CreateR1<float>(
@@ -523,9 +523,9 @@ TEST_F(WhileTest, WhileWithPredicateTupleResult) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(5), iteration);
@@ -534,27 +534,27 @@ TEST_F(WhileTest, WhileWithPredicateTupleResult) {
 
   // Create a computation for the body.
   // Add 1 to the iteration variable and or the predicate with true
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto pred = builder.GetTupleElement(prev, 1);
     auto new_pred = builder.Or(pred, builder.ConstantR0<bool>(true));
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_pred});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple({builder.ConstantR0<int32>(0),
                              builder.Ne(builder.ConstantR0<bool>(false),
                                         builder.ConstantR0<bool>(true))});
   auto result = builder.While(condition, body, init);
   VLOG(2) << "while = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   auto expected_counter = Literal::CreateR0<int32>(5);
   auto expected_predicate = Literal::CreateR0<bool>(true);
@@ -570,9 +570,9 @@ TEST_F(WhileTest, WhileWithTupleConstantScalarResult) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(5), iteration);
@@ -582,25 +582,24 @@ TEST_F(WhileTest, WhileWithTupleConstantScalarResult) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and set the other tuple element to a
   // constant.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
-    auto result =
-        builder.Tuple({builder.Add(iteration, builder.ConstantR0<int32>(1)),
-                       builder.ConstantR0<int32>(7)});
+    builder.Tuple({builder.Add(iteration, builder.ConstantR0<int32>(1)),
+                   builder.ConstantR0<int32>(7)});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR0<int32>(7)});
   auto result = builder.While(condition, body, init);
   VLOG(2) << "while = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   auto expected_counter = Literal::CreateR0<int32>(5);
   auto expected_data = Literal::CreateR0<int32>(7);
@@ -631,20 +630,20 @@ TEST_F(WhileTest, TwoWhileWithTupleResult) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   const int c1 = 5;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c1));
     TF_ASSERT_OK_AND_ASSIGN(condition, builder.Build());
   }
 
-  Computation condition2;
+  XlaComputation condition2;
   const int c2 = 7;
   {
-    ComputationBuilder builder(client_, "condition2");
+    XlaBuilder builder("condition2");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c2));
@@ -654,34 +653,34 @@ TEST_F(WhileTest, TwoWhileWithTupleResult) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and add a constant vector of 1.0f to
   // the weight variable, both of which are tuple elements.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto weights = builder.GetTupleElement(prev, 1);
     auto input = builder.ConstantR1<float>(10, 1.f);
     auto new_weights = builder.Add(weights, input);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_weights});
     TF_ASSERT_OK_AND_ASSIGN(body, builder.Build());
   }
 
-  Computation body2;
+  XlaComputation body2;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto weights = builder.GetTupleElement(prev, 1);
     auto input = builder.ConstantR1<float>(10, 1.f);
     auto new_weights = builder.Add(weights, input);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_weights});
     TF_ASSERT_OK_AND_ASSIGN(body2, builder.Build());
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(10, 0.f)});
   auto while1 = builder.While(condition, body, init);
@@ -692,11 +691,11 @@ TEST_F(WhileTest, TwoWhileWithTupleResult) {
   auto while_result2 = builder.GetTupleElement(while2, 1);
   VLOG(2) << "while_result2 = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(while_result2).ConsumeValueOrDie());
+                 builder.GetShape(while_result2).ConsumeValueOrDie());
   auto result = builder.Add(while_result1, while_result2);
   VLOG(2) << "result = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
   const float sum = c1 + c2;
   std::vector<float> expected(10, sum);
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -710,20 +709,20 @@ TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   const int c1 = 5;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c1));
     TF_ASSERT_OK_AND_ASSIGN(condition, builder.Build());
   }
 
-  Computation condition2;
+  XlaComputation condition2;
   const int c2 = 7;
   {
-    ComputationBuilder builder(client_, "condition2");
+    XlaBuilder builder("condition2");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c2));
@@ -733,21 +732,21 @@ TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and add a constant vector of 1.0f to
   // the weight variable, both of which are tuple elements.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto weights = builder.GetTupleElement(prev, 1);
     auto input = builder.ConstantR1<float>(10, 1.f);
     auto new_weights = builder.Add(weights, input);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_weights});
     TF_ASSERT_OK_AND_ASSIGN(body, builder.Build());
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(10, 0.f)});
   auto while1 = builder.While(condition, body, init);
@@ -758,11 +757,11 @@ TEST_F(WhileTest, TwoWhileLoopsAndSharedBody) {
   auto while_result2 = builder.GetTupleElement(while2, 1);
   VLOG(2) << "while_result2 = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(while_result2).ConsumeValueOrDie());
+                 builder.GetShape(while_result2).ConsumeValueOrDie());
   auto result = builder.Add(while_result1, while_result2);
   VLOG(2) << "result = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
   const float sum = c1 + c2;
   std::vector<float> expected(10, sum);
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -777,20 +776,20 @@ TEST_F(WhileTest, DISABLED_ON_GPU(WhileLoopsWithSharedBodyAndInit)) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   const int c1 = 5;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c1));
     TF_ASSERT_OK_AND_ASSIGN(condition, builder.Build());
   }
 
-  Computation condition2;
+  XlaComputation condition2;
   const int c2 = 7;
   {
-    ComputationBuilder builder(client_, "condition2");
+    XlaBuilder builder("condition2");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(c2));
@@ -800,21 +799,21 @@ TEST_F(WhileTest, DISABLED_ON_GPU(WhileLoopsWithSharedBodyAndInit)) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and add a constant vector of 1.0f to
   // the weight variable, both of which are tuple elements.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     auto weights = builder.GetTupleElement(prev, 1);
     auto input = builder.ConstantR1<float>(10, 1.f);
     auto new_weights = builder.Add(weights, input);
-    auto result = builder.Tuple(
+    builder.Tuple(
         {builder.Add(iteration, builder.ConstantR0<int32>(1)), new_weights});
     TF_ASSERT_OK_AND_ASSIGN(body, builder.Build());
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(10, 0.f)});
   auto while1 = builder.While(condition, body, init);
@@ -824,11 +823,11 @@ TEST_F(WhileTest, DISABLED_ON_GPU(WhileLoopsWithSharedBodyAndInit)) {
   auto while_result2 = builder.GetTupleElement(while2, 1);
   VLOG(2) << "while_result2 = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(while_result2).ConsumeValueOrDie());
+                 builder.GetShape(while_result2).ConsumeValueOrDie());
   auto result = builder.Add(while_result1, while_result2);
   VLOG(2) << "result = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
   const float sum = c1 + c2;
   std::vector<float> expected(10, sum);
   ComputeAndCompareR1<float>(&builder, expected, {}, ErrorSpec(0.0001));
@@ -844,9 +843,9 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
 
   // Create a computation for the condition.
   // Repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Gt(builder.ConstantR0<int32>(5), iteration);
@@ -856,9 +855,9 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
   // Create a computation for the body.
   // Add 1 to the iteration variable and add a constant vector of 1.0f to
   // the weight variable, both of which are tuple elements.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     // TupleElement 0
     auto iteration = builder.GetTupleElement(prev, 0);
@@ -873,18 +872,18 @@ XLA_TEST_F(WhileTest, WhileWithDynamicUpdateSlice) {
     // UpdateSlice.
     auto out1 = builder.DynamicUpdateSlice(input, update, starts);
 
-    auto result = builder.Tuple({out0, out1});
+    builder.Tuple({out0, out1});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, "while");
+  XlaBuilder builder("while");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), builder.ConstantR1<float>(10, 0.f)});
   auto result = builder.While(condition, body, init);
   VLOG(2) << "while = "
           << ShapeUtil::HumanString(
-                 *builder.GetShape(result).ConsumeValueOrDie());
+                 builder.GetShape(result).ConsumeValueOrDie());
 
   auto expected_counter = Literal::CreateR0<int32>(5);
   auto expected_data = Literal::CreateR1<float>(
@@ -915,18 +914,18 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) {
 
   // Create a computation for the condition: repeat for count iterations.
   auto build_condition = [this, v6s32](int count) {
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto prev = builder.Reshape(
         builder.Slice(builder.Parameter(0, v6s32, "prev"), {0}, {1}, {1}), {0},
-          {});
+        {});
     builder.Gt(builder.ConstantR0<int32>(count), prev);
     return builder.Build().ConsumeValueOrDie();
   };
 
   // Create a computation for the body: add 1 to the result variable.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, v6s32, "prev");
     auto inc = builder.ConcatInDim(
         {builder.ConstantR1<int32>({1}),
@@ -934,16 +933,15 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithPrngScalarResult)) {
                             builder.ConstantR0<int32>(100),
                             ShapeUtil::MakeShape(S32, {5}))},
         0);
-    auto result = builder.Add(inc, prev);
+    builder.Add(inc, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
   auto while_loop = [this, &body, build_condition](int count) {
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto init = builder.ConstantR1<int32>({0, 0, 0, 0, 0, 0});
-    auto result = builder.While(build_condition(count), body, init);
-    auto shape = builder.GetShape(result).ConsumeValueOrDie();
+    builder.While(build_condition(count), body, init);
     return builder.Build();
   };
 
@@ -1107,9 +1105,9 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) {
   auto inner_result_shape = ShapeUtil::MakeTupleShape(
       {ShapeUtil::MakeShape(S32, {}), ShapeUtil::MakeShape(S32, {})});
 
-  Computation inner_condition;
+  XlaComputation inner_condition;
   {
-    ComputationBuilder builder(client_, "inner_condition");
+    XlaBuilder builder("inner_condition");
     auto params = builder.Parameter(0, inner_result_shape, "prev");
     auto i = builder.GetTupleElement(params, 0);
     builder.Lt(i, builder.ConstantR0<int32>(7));
@@ -1118,9 +1116,9 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) {
 
   // Creates a computation for the outer loop condition:
   // repeat while result < 30.
-  Computation outer_condition;
+  XlaComputation outer_condition;
   {
-    ComputationBuilder builder(client_, "outer_condition");
+    XlaBuilder builder("outer_condition");
     auto prev = builder.Parameter(0, outer_result_shape, "prev");
     builder.Lt(prev, builder.ConstantR0<int32>(30));
     outer_condition = builder.Build().ConsumeValueOrDie();
@@ -1128,34 +1126,33 @@ XLA_TEST_F(WhileTest, NestedWhileWithScalarResult) {
 
   // Creates a computation for the inner loop body: add 1 to `i`, and add 2 to
   // `result`.
-  Computation inner_body;
+  XlaComputation inner_body;
   {
-    ComputationBuilder builder(client_, "inner_body");
+    XlaBuilder builder("inner_body");
     auto params = builder.Parameter(0, inner_result_shape, "prev");
     auto i = builder.GetTupleElement(params, 0);
     auto result = builder.GetTupleElement(params, 1);
     i = builder.Add(builder.ConstantR0<int32>(1), i);
     result = builder.Add(builder.ConstantR0<int32>(2), result);
-    auto output = builder.Tuple({i, result});
+    builder.Tuple({i, result});
     inner_body = builder.Build().ConsumeValueOrDie();
   }
 
   // Creates a computation for the outer loop: run the inner loop with i = 0.
-  Computation outer_body;
+  XlaComputation outer_body;
   {
-    ComputationBuilder builder(client_, "outer_body");
+    XlaBuilder builder("outer_body");
     auto prev = builder.Parameter(0, outer_result_shape, "prev");
     auto init = builder.Tuple({builder.ConstantR0<int32>(0), prev});
     auto result = builder.While(inner_condition, inner_body, init);
-    auto output = builder.GetTupleElement(result, 1);
+    builder.GetTupleElement(result, 1);
     outer_body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.ConstantR0<int32>(0);
-  auto result = builder.While(outer_condition, outer_body, init);
-  auto shape = builder.GetShape(result).ConsumeValueOrDie();
+  builder.While(outer_condition, outer_body, init);
 
   ComputeAndCompareR0<int32>(&builder, 42, {});
 }
@@ -1170,18 +1167,18 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) {
   auto result_shape = ShapeUtil::MakeShape(S32, {});
 
   // Create a computation for the condition: repeat for 5 iterations.
-  Computation condition_callee;
+  XlaComputation condition_callee;
   {
-    ComputationBuilder builder(client_, "condition_callee");
+    XlaBuilder builder("condition_callee");
     auto prev = builder.Parameter(0, result_shape, "prev");
     builder.Tuple({builder.Gt(builder.ConstantR0<int32>(5), prev)});
 
     condition_callee = builder.Build().ConsumeValueOrDie();
   }
 
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto result = builder.Call(condition_callee, {prev});
     builder.GetTupleElement(result, 0);
@@ -1189,20 +1186,19 @@ TEST_F(WhileTest, DISABLED_ON_INTERPRETER(WhileWithCallInsideCondition)) {
   }
 
   // Create a computation for the body: add 1 to the result variable.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, result_shape, "prev");
     auto input = builder.ConstantR0<int32>(1);
-    auto result = builder.Add(input, prev);
+    builder.Add(input, prev);
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While node with computations for the condition and the body.
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto init = builder.ConstantR0<int32>(0);
-  auto result = builder.While(condition, body, init);
-  auto shape = builder.GetShape(result).ConsumeValueOrDie();
+  builder.While(condition, body, init);
 
   ComputeAndCompareR0<int32>(&builder, 5, {});
 }
@@ -1214,28 +1210,28 @@ TEST_F(WhileTest, WhileWithLoopInvariantOperation) {
       {scalar_s32, matrix_shape, matrix_shape, matrix_shape});
 
   // Create a computation for the condition: repeat for 5 iterations.
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client_, "condition");
+    XlaBuilder builder("condition");
     auto state = builder.Parameter(0, while_shape, "state");
     builder.Gt(builder.ConstantR0<int32>(5), builder.GetTupleElement(state, 0));
     TF_ASSERT_OK_AND_ASSIGN(condition, builder.Build());
   }
 
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client_, "body");
+    XlaBuilder builder("body");
     auto state = builder.Parameter(0, while_shape, "state");
     auto indvar = builder.GetTupleElement(state, 0);
     auto input_0 = builder.GetTupleElement(state, 1);
     auto input_1 = builder.GetTupleElement(state, 2);
     auto output = builder.Tanh(builder.Dot(input_0, input_1));
     auto indvar_next = builder.Add(indvar, builder.ConstantR0<int32>(1));
-    auto tuple_result = builder.Tuple({indvar_next, input_0, input_1, output});
+    builder.Tuple({indvar_next, input_0, input_1, output});
     TF_ASSERT_OK_AND_ASSIGN(body, builder.Build());
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto matrix_input = builder.Parameter(0, matrix_shape, "matrix");
   auto init = builder.Tuple(
       {builder.ConstantR0<int32>(0), matrix_input, matrix_input, matrix_input});
@@ -1268,9 +1264,9 @@ void BM_WhileLoop(int num_iters) {
 
   // Create while condition computation with 'loop_limit'.
   const int32 loop_limit = 100;
-  Computation condition;
+  XlaComputation condition;
   {
-    ComputationBuilder builder(client, "condition");
+    XlaBuilder builder("condition");
     auto prev = builder.Parameter(0, loop_state_shape, "prev");
     auto iteration = builder.GetTupleElement(prev, 0);
     builder.Lt(iteration, builder.ConstantR0<int32>(loop_limit));
@@ -1278,9 +1274,9 @@ void BM_WhileLoop(int num_iters) {
   }
 
   // Create while body computation with unit loop increment.
-  Computation body;
+  XlaComputation body;
   {
-    ComputationBuilder builder(client, "body");
+    XlaBuilder builder("body");
     auto prev = builder.Parameter(0, loop_state_shape, "prev");
     // TupleElement 0
     auto iteration = builder.GetTupleElement(prev, 0);
@@ -1294,12 +1290,12 @@ void BM_WhileLoop(int num_iters) {
     auto starts = builder.ConstantR1<int32>({0, 0, 0});
     // UpdateSlice.
     auto out1 = builder.DynamicUpdateSlice(input, update, starts);
-    auto result = builder.Tuple({out0, out1});
+    builder.Tuple({out0, out1});
     body = builder.Build().ConsumeValueOrDie();
   }
 
   // Create a While instruction.
-  ComputationBuilder builder(client, "while");
+  XlaBuilder builder("while");
   auto zero = builder.ConstantR0<float>(0.0);
   auto input = builder.Broadcast(zero, {seq_len, 1024, 1024});
   auto init = builder.Tuple({builder.ConstantR0<int32>(0), input});
-- 
GitLab


From 9ba97cf96e8e2bd7a43db660b56932927d65d5bf Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 18:33:11 -0700
Subject: [PATCH 838/960] Add int64 to Variant mutable hash table variants.

PiperOrigin-RevId: 191380970
---
 tensorflow/core/kernels/lookup_table_op.cc | 32 +++++++++++-----------
 tensorflow/core/kernels/lookup_table_op.h  | 17 ++++++------
 2 files changed, 25 insertions(+), 24 deletions(-)

diff --git a/tensorflow/core/kernels/lookup_table_op.cc b/tensorflow/core/kernels/lookup_table_op.cc
index e3872fee0e..57b7798ba0 100644
--- a/tensorflow/core/kernels/lookup_table_op.cc
+++ b/tensorflow/core/kernels/lookup_table_op.cc
@@ -22,6 +22,7 @@ limitations under the License.
 
 #include "tensorflow/core/framework/register_types.h"
 #include "tensorflow/core/framework/types.h"
+#include "tensorflow/core/framework/variant.h"
 #include "tensorflow/core/kernels/initializable_lookup_table.h"
 #include "tensorflow/core/lib/gtl/inlined_vector.h"
 #include "tensorflow/core/lib/hash/hash.h"
@@ -62,8 +63,7 @@ class MutableHashTableOfScalars final : public LookupInterface {
     mutex_lock l(mu_);
     for (int64 i = 0; i < key_values.size(); ++i) {
       value_values(i) = gtl::FindWithDefault(
-          table_, SubtleMustCopyUnlessStringOrFloat(key_values(i)),
-          default_val);
+          table_, SubtleMustCopyIfIntegral(key_values(i)), default_val);
     }
 
     return Status::OK();
@@ -78,9 +78,8 @@ class MutableHashTableOfScalars final : public LookupInterface {
       table_.clear();
     }
     for (int64 i = 0; i < key_values.size(); ++i) {
-      gtl::InsertOrUpdate(&table_,
-                          SubtleMustCopyUnlessStringOrFloat(key_values(i)),
-                          SubtleMustCopyUnlessStringOrFloat(value_values(i)));
+      gtl::InsertOrUpdate(&table_, SubtleMustCopyIfIntegral(key_values(i)),
+                          SubtleMustCopyIfIntegral(value_values(i)));
     }
     return Status::OK();
   }
@@ -172,8 +171,8 @@ class MutableHashTableOfTensors final : public LookupInterface {
 
     mutex_lock l(mu_);
     for (int64 i = 0; i < key_values.size(); ++i) {
-      ValueArray* value_vec = gtl::FindOrNull(
-          table_, SubtleMustCopyUnlessStringOrFloat(key_values(i)));
+      ValueArray* value_vec =
+          gtl::FindOrNull(table_, SubtleMustCopyIfIntegral(key_values(i)));
       if (value_vec != nullptr) {
         for (int64 j = 0; j < value_dim; j++) {
           value_values(i, j) = value_vec->at(j);
@@ -203,8 +202,8 @@ class MutableHashTableOfTensors final : public LookupInterface {
         V value = value_values(i, j);
         value_vec.push_back(value);
       }
-      gtl::InsertOrUpdate(
-          &table_, SubtleMustCopyUnlessStringOrFloat(key_values(i)), value_vec);
+      gtl::InsertOrUpdate(&table_, SubtleMustCopyIfIntegral(key_values(i)),
+                          value_vec);
     }
     return Status::OK();
   }
@@ -379,15 +378,14 @@ class MutableDenseHashTable final : public LookupInterface {
           for (int64 j = 0; j < value_size; ++j) {
             // TODO(andreasst): check if we can get rid of SubtleMustCopy
             // here and elsewhere in this file.
-            value_matrix(i, j) = SubtleMustCopyUnlessStringOrFloat(
-                value_buckets_matrix(bucket_index, j));
+            value_matrix(i, j) =
+                SubtleMustCopyIfIntegral(value_buckets_matrix(bucket_index, j));
           }
           break;
         }
         if (IsEqualKey(key_buckets_matrix, bucket_index, empty_key_matrix, 0)) {
           for (int64 j = 0; j < value_size; ++j) {
-            value_matrix(i, j) =
-                SubtleMustCopyUnlessStringOrFloat(default_flat(j));
+            value_matrix(i, j) = SubtleMustCopyIfIntegral(default_flat(j));
           }
           break;
         }
@@ -531,7 +529,7 @@ class MutableDenseHashTable final : public LookupInterface {
         if (IsEqualKey(key_buckets_matrix, bucket_index, key_matrix, i)) {
           for (int64 j = 0; j < value_size; ++j) {
             value_buckets_matrix(bucket_index, j) =
-                SubtleMustCopyUnlessStringOrFloat(value_matrix(i, j));
+                SubtleMustCopyIfIntegral(value_matrix(i, j));
           }
           break;
         }
@@ -539,11 +537,11 @@ class MutableDenseHashTable final : public LookupInterface {
           ++num_entries_;
           for (int64 j = 0; j < key_size; ++j) {
             key_buckets_matrix(bucket_index, j) =
-                SubtleMustCopyUnlessStringOrFloat(key_matrix(i, j));
+                SubtleMustCopyIfIntegral(key_matrix(i, j));
           }
           for (int64 j = 0; j < value_size; ++j) {
             value_buckets_matrix(bucket_index, j) =
-                SubtleMustCopyUnlessStringOrFloat(value_matrix(i, j));
+                SubtleMustCopyIfIntegral(value_matrix(i, j));
           }
           break;
         }
@@ -849,6 +847,7 @@ REGISTER_KERNEL(string, int64);
 REGISTER_KERNEL(int64, string);
 REGISTER_KERNEL(string, bool);
 REGISTER_KERNEL(int64, float);
+REGISTER_KERNEL(int64, Variant);
 
 #undef REGISTER_KERNEL
 
@@ -899,6 +898,7 @@ REGISTER_KERNEL(int64, double);
 REGISTER_KERNEL(string, float);
 REGISTER_KERNEL(string, bool);
 REGISTER_KERNEL(int64, bool);
+REGISTER_KERNEL(int64, Variant);
 
 #undef REGISTER_KERNEL
 
diff --git a/tensorflow/core/kernels/lookup_table_op.h b/tensorflow/core/kernels/lookup_table_op.h
index 3657fd5b6a..29a0cc91fe 100644
--- a/tensorflow/core/kernels/lookup_table_op.h
+++ b/tensorflow/core/kernels/lookup_table_op.h
@@ -125,19 +125,21 @@ namespace lookup {
 // integral types. However non-integer variables are not allowed and therefore
 // the local copy is unnecessary.
 template <typename T>
-T SubtleMustCopyUnlessStringOrFloat(const T& value) {
+T SubtleMustCopyIfIntegral(const T& value) {
   return internal::SubtleMustCopy(value);
 }
 
-inline const string& SubtleMustCopyUnlessStringOrFloat(const string& value) {
+inline const string& SubtleMustCopyIfIntegral(const string& value) {
   return value;
 }
 
-inline const float SubtleMustCopyUnlessStringOrFloat(const float value) {
+inline const float SubtleMustCopyIfIntegral(const float value) { return value; }
+
+inline const double SubtleMustCopyIfIntegral(const double value) {
   return value;
 }
 
-inline const double SubtleMustCopyUnlessStringOrFloat(const double value) {
+inline const Variant& SubtleMustCopyIfIntegral(const Variant& value) {
   return value;
 }
 
@@ -204,8 +206,8 @@ class HashTable : public InitializableLookupTable {
     const auto key_values = keys.flat<K>();
     const auto value_values = values.flat<V>();
     for (int64 i = 0; i < key_values.size(); ++i) {
-      const K key = SubtleMustCopyUnlessStringOrFloat(key_values(i));
-      const V value = SubtleMustCopyUnlessStringOrFloat(value_values(i));
+      const K key = SubtleMustCopyIfIntegral(key_values(i));
+      const V value = SubtleMustCopyIfIntegral(value_values(i));
       const V& previous_value = gtl::LookupOrInsert(table_.get(), key, value);
       if (previous_value != value) {
         return errors::FailedPrecondition(
@@ -224,8 +226,7 @@ class HashTable : public InitializableLookupTable {
 
     for (int64 i = 0; i < key_values.size(); ++i) {
       value_values(i) = gtl::FindWithDefault(
-          *table_, SubtleMustCopyUnlessStringOrFloat(key_values(i)),
-          default_val);
+          *table_, SubtleMustCopyIfIntegral(key_values(i)), default_val);
     }
     return Status::OK();
   }
-- 
GitLab


From ec16dd449482f38100083505314cb4179f7d1cb8 Mon Sep 17 00:00:00 2001
From: Anna R <annarev@google.com>
Date: Mon, 2 Apr 2018 19:20:53 -0700
Subject: [PATCH 839/960] Automated g4 rollback of changelist 191037166

PiperOrigin-RevId: 191385075
---
 tensorflow/compiler/tests/BUILD | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tensorflow/compiler/tests/BUILD b/tensorflow/compiler/tests/BUILD
index 204a2a2f90..edabdc218a 100644
--- a/tensorflow/compiler/tests/BUILD
+++ b/tensorflow/compiler/tests/BUILD
@@ -375,7 +375,6 @@ tf_xla_py_test(
     name = "momentum_test",
     size = "small",
     srcs = ["momentum_test.py"],
-    tags = ["no_oss"],
     deps = [
         ":xla_test",
         "//tensorflow/python:array_ops",
-- 
GitLab


From fa2e70d7fae8466c4006bd29334a3cc440ee6d3a Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Mon, 2 Apr 2018 19:32:45 -0700
Subject: [PATCH 840/960] Deleted a special case

PiperOrigin-RevId: 191385909
---
 tensorflow/core/grappler/optimizers/function_optimizer.cc | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 2a6b8a325f..6cdc51f1bd 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -286,12 +286,6 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (func.attr().count("_noinline") != 0) {
       continue;
     }
-    // Don't touch anything marked XLA to prevent XLA failures further down the
-    // road.
-    if (func.attr().count("_XlaCompile") > 0 &&
-        func.attr().at("_XlaCompile").b()) {
-      continue;
-    }
     // Can't create IdentityN nodes with no input or output: skip these
     // functions for now.
     if (func.signature().input_arg_size() == 0 ||
-- 
GitLab


From 38e0139329482d8e44629dea2e87853808eacd0d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 20:50:39 -0700
Subject: [PATCH 841/960] Windows: Enable tensorflow/contrib in Bazel build
 (Second try)

This reverts commit 4e108ef30d7cd7ae5e1c550ec5ae27e79b8c6e39.

PiperOrigin-RevId: 191391075
---
 tensorflow/contrib/BUILD                      |   8 +-
 tensorflow/contrib/__init__.py                |   7 +-
 .../boosted_trees/lib/utils/batch_features.h  |   6 +-
 tensorflow/contrib/distributions/BUILD        |   2 +
 .../python/examples/linear_regression/BUILD   |   1 +
 tensorflow/contrib/gan/BUILD                  |   1 +
 .../contrib/kfac/python/kernel_tests/BUILD    |   1 +
 tensorflow/contrib/labeled_tensor/BUILD       |   1 +
 tensorflow/contrib/layers/BUILD               |   2 +
 tensorflow/contrib/learn/BUILD                |   5 +
 tensorflow/contrib/lookup/BUILD               |   1 +
 .../contrib/remote_fused_graph/pylib/BUILD    |   1 -
 tensorflow/contrib/saved_model/BUILD          |   1 +
 tensorflow/contrib/session_bundle/BUILD       |   1 +
 .../contrib/slim/python/slim/data/BUILD       |   1 +
 tensorflow/contrib/tensor_forest/BUILD        |   1 -
 tensorflow/contrib/tensorboard/BUILD          |   1 +
 tensorflow/contrib/timeseries/examples/BUILD  |   5 +-
 .../timeseries/python/timeseries/BUILD        |   5 +-
 .../timeseries/state_space_models/BUILD       |   1 +
 tensorflow/contrib/tpu/BUILD                  |   1 +
 tensorflow/contrib/util/loader.py             |   7 +-
 tensorflow/core/framework/dataset.h           |   4 +-
 tensorflow/core/lib/core/stringpiece.cc       |   2 -
 tensorflow/core/lib/core/stringpiece.h        |   2 +-
 tensorflow/core/platform/abi.cc               |   8 +-
 tensorflow/core/platform/cpu_info.h           |   2 +-
 tensorflow/core/platform/tracing.h            |   4 +-
 tensorflow/python/BUILD                       |  77 ++++++--
 tensorflow/python/debug/BUILD                 |   1 +
 tensorflow/python/keras/BUILD                 |   5 +-
 tensorflow/python/kernel_tests/BUILD          |   4 -
 tensorflow/tensorflow.bzl                     |  34 +++-
 .../windows/cpu/pip/build_tf_windows.sh       |   3 +-
 tensorflow/tools/def_file_filter/BUILD        |   9 +
 tensorflow/tools/def_file_filter/BUILD.tpl    |  15 ++
 .../def_file_filter/def_file_filter.py.tpl    | 168 ++++++++++++++++++
 .../def_file_filter_configure.bzl             |  56 ++++++
 tensorflow/tools/pip_package/BUILD            | 130 ++++++--------
 tensorflow/workspace.bzl                      |   6 +
 40 files changed, 469 insertions(+), 121 deletions(-)
 create mode 100644 tensorflow/tools/def_file_filter/BUILD
 create mode 100644 tensorflow/tools/def_file_filter/BUILD.tpl
 create mode 100644 tensorflow/tools/def_file_filter/def_file_filter.py.tpl
 create mode 100644 tensorflow/tools/def_file_filter/def_file_filter_configure.bzl

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 0cebb49afb..bf69144ad8 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -8,6 +8,7 @@ package(default_visibility = ["//tensorflow:__subpackages__"])
 load("//third_party/mpi:mpi.bzl", "if_mpi")
 load("@local_config_cuda//cuda:build_defs.bzl", "if_cuda")
 load("@local_config_tensorrt//:build_defs.bzl", "if_tensorrt")
+load("//tensorflow:tensorflow.bzl", "if_not_windows")
 
 py_library(
     name = "contrib_py",
@@ -40,7 +41,6 @@ py_library(
         "//tensorflow/contrib/estimator:estimator_py",
         "//tensorflow/contrib/factorization:factorization_py",
         "//tensorflow/contrib/feature_column:feature_column_py",
-        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
         "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/contrib/fused_conv:fused_conv_py",
         "//tensorflow/contrib/gan",
@@ -63,7 +63,6 @@ py_library(
         "//tensorflow/contrib/linalg:linalg_py",
         "//tensorflow/contrib/linear_optimizer:sdca_estimator_py",
         "//tensorflow/contrib/linear_optimizer:sdca_ops_py",
-        "//tensorflow/contrib/lite/python:lite",
         "//tensorflow/contrib/lookup:lookup_py",
         "//tensorflow/contrib/losses:losses_py",
         "//tensorflow/contrib/losses:metric_learning_py",
@@ -117,7 +116,10 @@ py_library(
             "//tensorflow/contrib/kafka",
         ],
         "//conditions:default": [],
-    }),
+    }) + if_not_windows([
+        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
+        "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
+    ]),
 )
 
 cc_library(
diff --git a/tensorflow/contrib/__init__.py b/tensorflow/contrib/__init__.py
index a8e05df708..1c5b00f92e 100644
--- a/tensorflow/contrib/__init__.py
+++ b/tensorflow/contrib/__init__.py
@@ -1,3 +1,4 @@
+# pylint: disable=g-import-not-at-top
 # Copyright 2015 The TensorFlow Authors. All Rights Reserved.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
@@ -18,6 +19,8 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+import os
+
 # Add projects here, they will show up under tf.contrib.
 from tensorflow.contrib import batching
 from tensorflow.contrib import bayesflow
@@ -84,7 +87,8 @@ from tensorflow.contrib import tpu
 from tensorflow.contrib import training
 from tensorflow.contrib import util
 from tensorflow.contrib.eager.python import tfe as eager
-from tensorflow.contrib.lite.python import lite
+if os.name != "nt":
+  from tensorflow.contrib.lite.python import lite
 from tensorflow.contrib.optimizer_v2 import optimizer_v2_symbols as optimizer_v2
 from tensorflow.contrib.receptive_field import receptive_field_api as receptive_field
 from tensorflow.contrib.remote_fused_graph import pylib as remote_fused_graph
@@ -94,6 +98,7 @@ from tensorflow.contrib.summary import summary
 from tensorflow.python.util.lazy_loader import LazyLoader
 ffmpeg = LazyLoader("ffmpeg", globals(),
                     "tensorflow.contrib.ffmpeg")
+del os
 del LazyLoader
 
 del absolute_import
diff --git a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
index da5e744851..a3b1b013e3 100644
--- a/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
+++ b/tensorflow/contrib/boosted_trees/lib/utils/batch_features.h
@@ -48,9 +48,9 @@ class BatchFeatures {
   Status GetFeatureColumnSizes(int64* const num_dense_float_features,
                                int64* const num_sparse_float_features,
                                int64* const num_sparse_int_features) const {
-    QCHECK_NE(num_dense_float_features, nullptr);
-    QCHECK_NE(num_sparse_float_features, nullptr);
-    QCHECK_NE(num_sparse_int_features, nullptr);
+    QCHECK_NE(num_dense_float_features, static_cast<int64*>(nullptr));
+    QCHECK_NE(num_sparse_float_features, static_cast<int64*>(nullptr));
+    QCHECK_NE(num_sparse_int_features, static_cast<int64*>(nullptr));
     *num_dense_float_features = dense_float_feature_columns_.size();
     *num_sparse_float_features = sparse_float_feature_columns_.size();
     *num_sparse_int_features = sparse_int_feature_columns_.size();
diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index de08eb491b..514638ecbb 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -454,6 +454,7 @@ cuda_py_test(
         "//tensorflow/python:framework_test_lib",
         "//tensorflow/python:platform_test",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
@@ -1128,6 +1129,7 @@ cuda_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:platform_test",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
index f86331af6f..2f6cfdf31e 100644
--- a/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
+++ b/tensorflow/contrib/eager/python/examples/linear_regression/BUILD
@@ -22,6 +22,7 @@ cuda_py_test(
         ":linear_regression",
         "//tensorflow:tensorflow_py",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 cuda_py_test(
diff --git a/tensorflow/contrib/gan/BUILD b/tensorflow/contrib/gan/BUILD
index 9e56d3c039..461066bbb4 100644
--- a/tensorflow/contrib/gan/BUILD
+++ b/tensorflow/contrib/gan/BUILD
@@ -354,6 +354,7 @@ py_test(
     name = "classifier_metrics_test",
     srcs = ["python/eval/python/classifier_metrics_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":classifier_metrics",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/kfac/python/kernel_tests/BUILD b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
index f73c24f8fb..2477d2bfc1 100644
--- a/tensorflow/contrib/kfac/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/kfac/python/kernel_tests/BUILD
@@ -114,6 +114,7 @@ py_test(
     name = "utils_test",
     srcs = ["utils_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         "//tensorflow/contrib/kfac/python/ops:utils",
         "//tensorflow/contrib/tpu",
diff --git a/tensorflow/contrib/labeled_tensor/BUILD b/tensorflow/contrib/labeled_tensor/BUILD
index 18b265ae80..c8812d4b23 100644
--- a/tensorflow/contrib/labeled_tensor/BUILD
+++ b/tensorflow/contrib/labeled_tensor/BUILD
@@ -70,6 +70,7 @@ py_test(
         "python/ops/core_test.py",
     ],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":_typecheck",
         ":core",
diff --git a/tensorflow/contrib/layers/BUILD b/tensorflow/contrib/layers/BUILD
index 4be55468db..d5b3b279a1 100644
--- a/tensorflow/contrib/layers/BUILD
+++ b/tensorflow/contrib/layers/BUILD
@@ -188,6 +188,7 @@ py_test(
     size = "small",
     srcs = ["python/layers/normalization_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/contrib/framework:framework_py",
@@ -353,6 +354,7 @@ py_test(
     size = "small",
     srcs = ["python/ops/sparse_ops_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":layers_py",
         "//tensorflow/python:array_ops",
diff --git a/tensorflow/contrib/learn/BUILD b/tensorflow/contrib/learn/BUILD
index ba55365c14..d665fc9335 100644
--- a/tensorflow/contrib/learn/BUILD
+++ b/tensorflow/contrib/learn/BUILD
@@ -117,6 +117,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/data_feeder_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/python:client_testlib",
@@ -172,6 +173,7 @@ tf_py_test(
         "//tensorflow/python:variables",
         "//tensorflow/python/estimator",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
@@ -190,6 +192,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/graph_actions_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/framework:framework_py",
@@ -591,6 +594,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/learn_io/io_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/learn/python/learn/datasets",
@@ -820,6 +824,7 @@ py_test(
     size = "small",
     srcs = ["python/learn/utils/saved_model_export_utils_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":learn",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/lookup/BUILD b/tensorflow/contrib/lookup/BUILD
index 02b4f80252..f616207d46 100644
--- a/tensorflow/contrib/lookup/BUILD
+++ b/tensorflow/contrib/lookup/BUILD
@@ -46,4 +46,5 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     grpc_enabled = True,
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
diff --git a/tensorflow/contrib/remote_fused_graph/pylib/BUILD b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
index 996b55f9b8..3aa8a14f44 100644
--- a/tensorflow/contrib/remote_fused_graph/pylib/BUILD
+++ b/tensorflow/contrib/remote_fused_graph/pylib/BUILD
@@ -38,7 +38,6 @@ py_test(
     size = "small",
     srcs = ["python/ops/remote_fused_graph_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":remote_fused_graph_ops_py",
         "//tensorflow/core:protos_all_py",
diff --git a/tensorflow/contrib/saved_model/BUILD b/tensorflow/contrib/saved_model/BUILD
index faad40d335..e431c464ef 100644
--- a/tensorflow/contrib/saved_model/BUILD
+++ b/tensorflow/contrib/saved_model/BUILD
@@ -53,6 +53,7 @@ py_test(
     size = "small",
     srcs = ["python/saved_model/reader_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":saved_model_py",
diff --git a/tensorflow/contrib/session_bundle/BUILD b/tensorflow/contrib/session_bundle/BUILD
index 31717305e7..9c08859180 100644
--- a/tensorflow/contrib/session_bundle/BUILD
+++ b/tensorflow/contrib/session_bundle/BUILD
@@ -151,6 +151,7 @@ py_test(
     name = "gc_test",
     srcs = ["gc_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     visibility = ["//visibility:private"],
     deps = [
         ":gc",
diff --git a/tensorflow/contrib/slim/python/slim/data/BUILD b/tensorflow/contrib/slim/python/slim/data/BUILD
index dc12e67fc6..eef043e832 100644
--- a/tensorflow/contrib/slim/python/slim/data/BUILD
+++ b/tensorflow/contrib/slim/python/slim/data/BUILD
@@ -61,6 +61,7 @@ py_test(
     name = "dataset_data_provider_test",
     srcs = ["dataset_data_provider_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":dataset",
         ":dataset_data_provider",
diff --git a/tensorflow/contrib/tensor_forest/BUILD b/tensorflow/contrib/tensor_forest/BUILD
index 11a59ec22b..136856c015 100644
--- a/tensorflow/contrib/tensor_forest/BUILD
+++ b/tensorflow/contrib/tensor_forest/BUILD
@@ -539,7 +539,6 @@ py_test(
     srcs = ["client/random_forest_test.py"],
     srcs_version = "PY2AND3",
     tags = [
-        "no_windows",
         "nomac",  # b/63258195
         "notsan",
     ],
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index f4efd9717d..c955b13244 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -9,6 +9,7 @@ exports_files(["LICENSE"])
 
 # For platform specific build config
 load("//tensorflow/core:platform/default/build_config.bzl", "tf_proto_library")
+load("//tensorflow:tensorflow.bzl", "py_test")
 
 tf_proto_library(
     name = "protos_all",
diff --git a/tensorflow/contrib/timeseries/examples/BUILD b/tensorflow/contrib/timeseries/examples/BUILD
index 40cf9147b3..32e948a009 100644
--- a/tensorflow/contrib/timeseries/examples/BUILD
+++ b/tensorflow/contrib/timeseries/examples/BUILD
@@ -25,7 +25,10 @@ py_test(
     srcs = ["predict_test.py"],
     data = ["data/period_trend.csv"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],  # b/67513579
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",  # b/67513579
+    ],
     deps = [
         ":predict",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/BUILD b/tensorflow/contrib/timeseries/python/timeseries/BUILD
index 86022f46ce..af572d8124 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/BUILD
@@ -160,9 +160,7 @@ py_test(
         "head_test.py",
     ],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_pip_gpu",  # b/63391119
-    ],
+    tags = ["no_pip_gpu"],  # b/63391119
     deps = [
         ":estimators",
         ":feature_keys",
@@ -440,6 +438,7 @@ py_test(
     srcs_version = "PY2AND3",
     tags = [
         "no_pip_gpu",  # b/63391119
+        "no_windows",  # TODO: needs investigation on Windows
     ],
     deps = [
         ":feature_keys",
diff --git a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
index ca25ccd2b8..5d33e23a42 100644
--- a/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
+++ b/tensorflow/contrib/timeseries/python/timeseries/state_space_models/BUILD
@@ -40,6 +40,7 @@ py_test(
     timeout = "long",  # Moderate but for asan
     srcs = ["state_space_model_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
     deps = [
         ":state_space_model",
         "//tensorflow/contrib/layers:layers_py",
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index b267cceef1..d4830b6bcf 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -228,6 +228,7 @@ tf_py_test(
         "//tensorflow/python:framework",
         "//tensorflow/python:layers",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 tf_py_test(
diff --git a/tensorflow/contrib/util/loader.py b/tensorflow/contrib/util/loader.py
index f4283cd9ed..dca01d26f4 100644
--- a/tensorflow/contrib/util/loader.py
+++ b/tensorflow/contrib/util/loader.py
@@ -42,9 +42,10 @@ def load_op_library(path):
     plugin.
   """
   if os.name == 'nt':
-    # To avoid makeing every user_ops aware of windows, re-write
-    # the file extension from .so to .dll.
-    path = re.sub(r'\.so$', '.dll', path)
+    # To avoid making every user_ops aware of windows, re-write
+    # the file extension from .so to .dll if .so file doesn't exist.
+    if not os.path.exists(path):
+      path = re.sub(r'\.so$', '.dll', path)
 
     # Currently we have only some user_ops as dlls on windows - don't try
     # to load them if the dll is not found.
diff --git a/tensorflow/core/framework/dataset.h b/tensorflow/core/framework/dataset.h
index fb1fe9c51f..9e7ffe6c0b 100644
--- a/tensorflow/core/framework/dataset.h
+++ b/tensorflow/core/framework/dataset.h
@@ -474,11 +474,11 @@ class GraphDatasetBase : public DatasetBase {
   }
 
   // Key for storing the Dataset graph in the serialized format.
-  static const char kDatasetGraphKey[];
+  TF_EXPORT static const char kDatasetGraphKey[];
 
   // Key for storing the output node of the Dataset graph in the serialized
   // format.
-  static const char kDatasetGraphOutputNodeKey[];
+  TF_EXPORT static const char kDatasetGraphOutputNodeKey[];
 
  private:
   Status Serialize(OpKernelContext* ctx, string* serialized_graph_def,
diff --git a/tensorflow/core/lib/core/stringpiece.cc b/tensorflow/core/lib/core/stringpiece.cc
index 5bd79778a6..0b006fa2b4 100644
--- a/tensorflow/core/lib/core/stringpiece.cc
+++ b/tensorflow/core/lib/core/stringpiece.cc
@@ -55,6 +55,4 @@ StringPiece StringPiece::substr(size_t pos, size_t n) const {
   return StringPiece(data_ + pos, n);
 }
 
-const StringPiece::size_type StringPiece::npos = size_type(-1);
-
 }  // namespace tensorflow
diff --git a/tensorflow/core/lib/core/stringpiece.h b/tensorflow/core/lib/core/stringpiece.h
index 79409cce4b..835b938cbf 100644
--- a/tensorflow/core/lib/core/stringpiece.h
+++ b/tensorflow/core/lib/core/stringpiece.h
@@ -65,7 +65,7 @@ class StringPiece {
   iterator begin() const { return data_; }
   iterator end() const { return data_ + size_; }
 
-  static const size_t npos;
+  static const size_t npos = size_type(-1);
 
   // Return the ith byte in the referenced data.
   // REQUIRES: n < size()
diff --git a/tensorflow/core/platform/abi.cc b/tensorflow/core/platform/abi.cc
index 4df62734e9..e597a490d6 100644
--- a/tensorflow/core/platform/abi.cc
+++ b/tensorflow/core/platform/abi.cc
@@ -15,7 +15,7 @@ limitations under the License.
 
 #include "tensorflow/core/platform/abi.h"
 
-#if defined(PLATFORM_WINDOWS)
+#if defined(_MSC_VER)
 #include <windows.h>
 #include <cstring>
 #else
@@ -26,19 +26,19 @@ limitations under the License.
 #include <memory>
 #include <string>
 
-#if defined(PLATFORM_WINDOWS)
+#if defined(_MSC_VER)
 
 extern "C" char* __unDName(char* output_string, const char* name,
                            int max_string_length, void* (*p_alloc)(std::size_t),
                            void (*p_free)(void*), unsigned short disable_flags);
 
-#endif  // defined(PLATFORM_WINDOWS)
+#endif  // defined(_MSC_VER)
 
 namespace tensorflow {
 namespace port {
 
 std::string MaybeAbiDemangle(const char* name) {
-#if defined(PLATFORM_WINDOWS)
+#if defined(_MSC_VER)
   std::unique_ptr<char> demangled{__unDName(nullptr, name, 0, std::malloc,
                                             std::free,
                                             static_cast<unsigned short>(0))};
diff --git a/tensorflow/core/platform/cpu_info.h b/tensorflow/core/platform/cpu_info.h
index 331f3e5251..bb77650e26 100644
--- a/tensorflow/core/platform/cpu_info.h
+++ b/tensorflow/core/platform/cpu_info.h
@@ -18,7 +18,7 @@ limitations under the License.
 
 #include <string>
 
-#if defined(PLATFORM_WINDOWS)
+#if defined(_MSC_VER)
 #include "tensorflow/core/platform/windows/cpu_info.h"
 #endif
 
diff --git a/tensorflow/core/platform/tracing.h b/tensorflow/core/platform/tracing.h
index 8f7bff1bb0..3c6e7b0db5 100644
--- a/tensorflow/core/platform/tracing.h
+++ b/tensorflow/core/platform/tracing.h
@@ -103,7 +103,9 @@ class Tracing {
   friend class ScopedAnnotation;
   friend class TraceMe;
 
-  static std::atomic<Tracing::Engine*> tracing_engine_;
+  // TODO: TF_EXPORT is for building //tensorflow/contrib/data:_dataset_ops.so
+  //       on Windows. Figure out a way to remove TF_EXPORT here.
+  TF_EXPORT static std::atomic<Tracing::Engine*> tracing_engine_;
   static Tracing::Engine* engine() {
     return tracing_engine_.load(std::memory_order_acquire);
   }
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index c502a3a42b..9d1e9bdc7e 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -28,6 +28,8 @@ load("//tensorflow:tensorflow.bzl", "py_tests")
 load("//tensorflow:tensorflow.bzl", "tf_py_build_info_genrule")
 load("//tensorflow:tensorflow.bzl", "tf_py_wrap_cc")
 load("//tensorflow:tensorflow.bzl", "tf_cc_shared_object")
+load("//tensorflow:tensorflow.bzl", "tf_native_cc_binary")
+load("//tensorflow:tensorflow.bzl", "tf_custom_op_library_additional_deps_impl")
 load("//tensorflow:tensorflow.bzl", "cuda_py_test")
 load("//tensorflow:tensorflow.bzl", "cuda_py_tests")
 load("//tensorflow/core:platform/default/build_config.bzl", "pyx_library")
@@ -58,9 +60,10 @@ py_library(
         "//tensorflow/tools/api/generator:__pkg__",
         "//tensorflow/tools/quantization:__pkg__",  # TODO(b/34059704): remove when fixed
     ],
-    deps = [":no_contrib"] + if_not_windows([
+    deps = [
+        ":no_contrib",
         "//tensorflow/contrib:contrib_py",
-    ]),
+    ],
 )
 
 py_library(
@@ -971,7 +974,6 @@ py_test(
     srcs = ["framework/contrib_test.py"],
     main = "framework/contrib_test.py",
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         "//tensorflow:tensorflow_py",
         "//tensorflow/python:client_testlib",
@@ -1341,7 +1343,6 @@ py_test(
     srcs = ["framework/dtypes_test.py"],
     main = "framework/dtypes_test.py",
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":framework_for_generated_wrappers",
         ":framework_test_lib",
@@ -1717,7 +1718,6 @@ py_test(
     size = "small",
     srcs = ["ops/clip_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":clip_ops",
@@ -2786,7 +2786,6 @@ cuda_py_test(
     ],
     data = ["//tensorflow/core:image_testdata"],
     shard_count = 5,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -3391,6 +3390,65 @@ tf_py_wrap_cc(
          tf_additional_gdr_deps()),
 )
 
+# ** Targets for Windows build (start) **
+# We need the following targets to expose symbols from _pywrap_tensorflow.dll
+
+# Build a cc_binary from tf_custom_op_library_additional_deps_impl,
+# it contains all object code from its dependencies.
+tf_native_cc_binary(
+    name = "tf_custom_op_library_additional_deps.so",
+    linkshared = 1,
+    linkstatic = 1,
+    deps = tf_custom_op_library_additional_deps_impl(),
+)
+
+# Get a DEF file generated by parsing all object files
+# of tf_custom_op_library_additional_deps.so
+filegroup(
+    name = "pywrap_tensorflow_def_file",
+    srcs = [":tf_custom_op_library_additional_deps.so"],
+    output_group = "def_file",
+)
+
+# Filter the DEF file to reduce the number of symbols to 64K or less.
+# Note that we also write the name of the pyd file into DEF file so that
+# the dynamic libraries of custom ops can find it at runtime.
+genrule(
+    name = "pywrap_tensorflow_filtered_def_file",
+    srcs = [":pywrap_tensorflow_def_file"],
+    outs = ["pywrap_tensorflow_filtered_def_file.def"],
+    cmd = select({
+        "//tensorflow:windows": """
+              $(location @local_config_def_file_filter//:def_file_filter) \\
+              --input $(location :pywrap_tensorflow_def_file) \\
+              --output $@ \\
+              --target _pywrap_tensorflow_internal.pyd
+          """,
+        "//conditions:default": "touch $@",  # Just a placeholder for Unix platforms
+    }),
+    tools = ["@local_config_def_file_filter//:def_file_filter"],
+)
+
+# Get the import library of  _pywrap_tensorflow_internal.dll
+filegroup(
+    name = "pywrap_tensorflow_import_lib_file",
+    srcs = [":_pywrap_tensorflow_internal.so"],
+    output_group = "interface_library",
+)
+
+# Create a cc_import rule for the import library of _pywrap_tensorflow_internal.dll
+# so that custom ops' dynamic libraries can link against it.
+cc_import(
+    name = "pywrap_tensorflow_import_lib",
+    interface_library = select({
+        "//tensorflow:windows": ":pywrap_tensorflow_import_lib_file",
+        "//conditions:default": "not_exsiting_on_unix.lib",  # Just a placeholder for Unix platforms
+    }),
+    system_provided = 1,
+)
+
+# ** Targets for Windows build (end) **
+
 py_library(
     name = "lib",
     srcs = [
@@ -3763,7 +3821,6 @@ py_test(
     size = "small",
     srcs = ["lib/core/bfloat16_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":client_testlib",
         ":lib",
@@ -4071,7 +4128,6 @@ py_test(
     size = "small",
     srcs = ["training/checkpoint_ops_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["no_windows"],
     deps = [
         ":checkpoint_ops_gen",
         ":client",
@@ -4112,10 +4168,7 @@ py_test(
     size = "medium",
     srcs = ["training/monitored_session_test.py"],
     srcs_version = "PY2AND3",
-    tags = [
-        "no_windows",
-        "notsan",  # b/67945581
-    ],
+    tags = ["notsan"],  # b/67945581
     deps = [
         ":array_ops",
         ":client_testlib",
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index 4195586313..b81aa3745c 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -913,6 +913,7 @@ cuda_py_test(
         "//tensorflow/python:util",
         "//tensorflow/python:variables",
     ],
+    tags = ["no_windows"],  # TODO: needs investigation on Windows
 )
 
 py_test(
diff --git a/tensorflow/python/keras/BUILD b/tensorflow/python/keras/BUILD
index 2a06907f49..57f5097639 100755
--- a/tensorflow/python/keras/BUILD
+++ b/tensorflow/python/keras/BUILD
@@ -637,7 +637,10 @@ py_test(
     size = "small",
     srcs = ["_impl/keras/utils/io_utils_test.py"],
     srcs_version = "PY2AND3",
-    tags = ["notsan"],
+    tags = [
+        "no_windows",  # TODO: needs investigation on Windows
+        "notsan",
+    ],
     deps = [
         ":keras",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index ea210346c1..d6f97fc4c3 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -295,7 +295,6 @@ tf_py_test(
         "//tensorflow/python:nn_grad",
     ],
     data = ["//tensorflow/core:image_testdata"],
-    tags = ["no_windows"],
 )
 
 tf_py_test(
@@ -1142,7 +1141,6 @@ tf_py_test(
         "//tensorflow/python:variables",
     ],
     data = ["//tensorflow/core:lmdb_testdata"],
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2332,7 +2330,6 @@ cuda_py_test(
         "//tensorflow/python:variables",
     ],
     shard_count = 4,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -2463,7 +2460,6 @@ cuda_py_test(
         "//tensorflow/python/eager:context",
     ],
     shard_count = 10,
-    tags = ["no_windows"],
 )
 
 cuda_py_test(
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index fcc57d506e..e9d2f279cd 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -342,6 +342,22 @@ register_extension_info(
     label_regex_for_dep = "{extension_name}.*",
 )
 
+# A simple wrap around native.cc_binary rule.
+# When using this rule, you should realize it doesn't link to any tensorflow
+# dependencies by default.
+def tf_native_cc_binary(name,
+                        copts=tf_copts(),
+                        **kwargs):
+  native.cc_binary(
+      name=name,
+      copts=copts,
+      **kwargs)
+
+register_extension_info(
+    extension_name = "tf_native_cc_binary",
+    label_regex_for_dep = "{extension_name}.*",
+)
+
 def tf_gen_op_wrapper_cc(name,
                          out_ops_file,
                          pkg="",
@@ -1178,6 +1194,20 @@ def tf_custom_op_library_additional_deps():
       "@protobuf_archive//:protobuf_headers",
       clean_dep("//third_party/eigen3"),
       clean_dep("//tensorflow/core:framework_headers_lib"),
+  ] + if_windows(["//tensorflow/python:pywrap_tensorflow_import_lib"])
+
+# A list of targets that contains the implemenation of
+# tf_custom_op_library_additional_deps. It's used to generate a DEF file for
+# exporting symbols from _pywrap_tensorflow.dll on Windows.
+def tf_custom_op_library_additional_deps_impl():
+  return [
+      "@protobuf_archive//:protobuf",
+      "@nsync//:nsync_cpp",
+      # for //third_party/eigen3
+      clean_dep("//third_party/eigen3"),
+      # for //tensorflow/core:framework_headers_lib
+      clean_dep("//tensorflow/core:framework"),
+      clean_dep("//tensorflow/core:reader_base"),
   ]
 
 # Traverse the dependency graph along the "deps" attribute of the
@@ -1264,6 +1294,7 @@ def tf_custom_op_library(name, srcs=[], gpu_srcs=[], deps=[], linkopts=[]):
       deps=deps + if_cuda(cuda_deps),
       data=[name + "_check_deps"],
       copts=tf_copts(is_external=True),
+      features = ["windows_export_all_symbols"],
       linkopts=linkopts + select({
           "//conditions:default": [
               "-lm",
@@ -1410,7 +1441,8 @@ def tf_py_wrap_cc(name,
       ]) + tf_extension_copts()),
       linkopts=tf_extension_linkopts() + extra_linkopts,
       linkstatic=1,
-      deps=deps + extra_deps)
+      deps=deps + extra_deps,
+      **kwargs)
   native.genrule(
       name="gen_" + cc_library_pyd_name,
       srcs=[":" + cc_library_name],
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 8b8ba31a0d..40189a6d1b 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -65,4 +65,5 @@ bazel test -c opt $BUILD_OPTS -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
-  //${PY_TEST_DIR}/tensorflow/python/...
+  //${PY_TEST_DIR}/tensorflow/python/... \
+  //${PY_TEST_DIR}/tensorflow/contrib/...
diff --git a/tensorflow/tools/def_file_filter/BUILD b/tensorflow/tools/def_file_filter/BUILD
new file mode 100644
index 0000000000..e390e0fb05
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/BUILD
@@ -0,0 +1,9 @@
+# Description:
+# Tools for filtering DEF file for TensorFlow on Windows
+#
+# On Windows, we use a DEF file generated by Bazel to export
+# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll).
+# The maximum number of symbols that can be exported per DLL is 64K,
+# so we have to filter some useless symbols through this python script.
+
+package(default_visibility = ["//visibility:public"])
diff --git a/tensorflow/tools/def_file_filter/BUILD.tpl b/tensorflow/tools/def_file_filter/BUILD.tpl
new file mode 100644
index 0000000000..3cb72f4979
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/BUILD.tpl
@@ -0,0 +1,15 @@
+# Description:
+# Tools for filtering DEF file for TensorFlow on Windows
+#
+# On Windows, we use a DEF file generated by Bazel to export
+# symbols from the tensorflow dynamic library(_pywrap_tensorflow.dll).
+# The maximum number of symbols that can be exported per DLL is 64K,
+# so we have to filter some useless symbols through this python script.
+
+package(default_visibility = ["//visibility:public"])
+
+py_binary(
+    name = "def_file_filter",
+    srcs = ["def_file_filter.py"],
+    srcs_version = "PY2AND3",
+)
diff --git a/tensorflow/tools/def_file_filter/def_file_filter.py.tpl b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
new file mode 100644
index 0000000000..8bdc03eb0f
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/def_file_filter.py.tpl
@@ -0,0 +1,168 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+"""def_file_filter.py - tool to filter a windows def file.
+
+The def file can be used to export symbols from the tensorflow dll to enable
+tf.load_library().
+
+Because the linker allows only 64K symbols to be exported per dll
+we filter the symbols down to the essentials. The regular expressions
+we use for this are specific to tensorflow.
+
+TODO: this works fine but there is an issue with exporting
+'const char * const' and importing it from a user_ops. The problem is
+on the importing end and using __declspec(dllimport) works around it.
+"""
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import argparse
+import io
+import os
+import re
+import subprocess
+import sys
+import tempfile
+
+# External tools we use that come with visual studio sdk
+UNDNAME = "%{undname_bin_path}"
+
+# Exclude if matched
+EXCLUDE_RE = re.compile(r"RTTI|deleting destructor|::internal::")
+
+# Include if matched before exclude
+INCLUDEPRE_RE = re.compile(r"google::protobuf::internal::ExplicitlyConstructed|"
+                           r"google::protobuf::internal::ArenaImpl::AllocateAligned|" # for contrib/data/_prefetching_ops
+                           r"google::protobuf::internal::ArenaImpl::AddCleanup|" # for contrib/data/_prefetching_ops
+                           r"google::protobuf::Arena::OnArenaAllocation|" # for contrib/data/_prefetching_ops
+                           r"tensorflow::internal::LogMessage|"
+                           r"tensorflow::internal::LogString|"
+                           r"tensorflow::internal::CheckOpMessageBuilder|"
+                           r"tensorflow::internal::MakeCheckOpValueString|"
+                           r"tensorflow::internal::PickUnusedPortOrDie|"
+                           r"tensorflow::internal::ValidateDevice|"
+                           r"tensorflow::ops::internal::Enter|"
+                           r"tensorflow::strings::internal::AppendPieces|"
+                           r"tensorflow::strings::internal::CatPieces|"
+                           r"tensorflow::io::internal::JoinPathImpl")
+
+# Include if matched after exclude
+INCLUDE_RE = re.compile(r"^(TF_\w*)$|"
+                        r"^(TFE_\w*)$|"
+                        r"nsync::|"
+                        r"tensorflow::|"
+                        r"functor::|"
+                        r"perftools::gputools")
+
+# We want to identify data members explicitly in the DEF file, so that no one
+# can implicitly link against the DLL if they use one of the variables exported
+# from the DLL and the header they use does not decorate the symbol with
+# __declspec(dllimport). It is easier to detect what a data symbol does
+# NOT look like, so doing it with the below regex.
+DATA_EXCLUDE_RE = re.compile(r"[)(]|"
+                             r"vftable|"
+                             r"vbtable|"
+                             r"vcall|"
+                             r"RTTI|"
+                             r"protobuf::internal::ExplicitlyConstructed")
+
+def get_args():
+  """Parse command line."""
+  filename_list = lambda x: x.split(";")
+  parser = argparse.ArgumentParser()
+  parser.add_argument("--input", type=filename_list,
+                      help="paths to input def file",
+                      required=True)
+  parser.add_argument("--output", help="output deffile", required=True)
+  parser.add_argument("--target", help="name of the target", required=True)
+  args = parser.parse_args()
+  return args
+
+
+def main():
+  """main."""
+  args = get_args()
+
+  # Pipe dumpbin to extract all linkable symbols from libs.
+  # Good symbols are collected in candidates and also written to
+  # a temp file.
+  candidates = []
+  tmpfile = tempfile.NamedTemporaryFile(mode="w", delete=False)
+  for def_file_path in args.input:
+    def_file = open(def_file_path, 'r')
+    for line in def_file:
+      cols = line.split()
+      sym = cols[0]
+      tmpfile.file.write(sym + "\n")
+      candidates.append(sym)
+  tmpfile.file.close()
+
+  # Run the symbols through undname to get their undecorated name
+  # so we can filter on something readable.
+  with open(args.output, "w") as def_fp:
+    # track dupes
+    taken = set()
+
+    # Header for the def file.
+    def_fp.write("LIBRARY " + args.target + "\n")
+    def_fp.write("EXPORTS\n")
+    def_fp.write("\t ??1OpDef@tensorflow@@UEAA@XZ\n")
+
+    # Each symbols returned by undname matches the same position in candidates.
+    # We compare on undname but use the decorated name from candidates.
+    dupes = 0
+    proc = subprocess.Popen([UNDNAME, tmpfile.name], stdout=subprocess.PIPE)
+    for idx, line in enumerate(io.TextIOWrapper(proc.stdout, encoding="utf-8")):
+      decorated = candidates[idx]
+      if decorated in taken:
+        # Symbol is already in output, done.
+        dupes += 1
+        continue
+
+      if not INCLUDEPRE_RE.search(line):
+        if EXCLUDE_RE.search(line):
+          continue
+        if not INCLUDE_RE.search(line):
+          continue
+
+      if "deleting destructor" in line:
+        # Some of the symbols convered by INCLUDEPRE_RE export deleting
+        # destructor symbols, which is a bad idea.
+        # So we filter out such symbols here.
+        continue
+
+      if DATA_EXCLUDE_RE.search(line):
+        def_fp.write("\t" + decorated + "\n")
+      else:
+        def_fp.write("\t" + decorated + " DATA\n")
+      taken.add(decorated)
+    def_fp.close()
+
+  exit_code = proc.wait()
+  if exit_code != 0:
+    print("{} failed, exit={}".format(UNDNAME, exit_code))
+    return exit_code
+
+  os.unlink(tmpfile.name)
+
+  print("symbols={}, taken={}, dupes={}"
+        .format(len(candidates), len(taken), dupes))
+  return 0
+
+
+if __name__ == "__main__":
+  sys.exit(main())
diff --git a/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
new file mode 100644
index 0000000000..47539b2423
--- /dev/null
+++ b/tensorflow/tools/def_file_filter/def_file_filter_configure.bzl
@@ -0,0 +1,56 @@
+"""Repository rule for def file filter autoconfiguration.
+
+This repository reuses Bazel's VC detect mechanism to find undname.exe,
+which is a tool used in def_file_filter.py.
+
+def_file_filter.py is for filtering the DEF file for TensorFlow on Windows.
+On Windows, we use a DEF file generated by Bazel to export symbols from the
+tensorflow dynamic library(_pywrap_tensorflow.dll). The maximum number of
+symbols that can be exported per DLL is 64K, so we have to filter some useless
+symbols through this python script.
+
+`def_file_filter_config` depends on the following environment variables:
+  * `BAZEL_VC`
+  * `BAZEL_VS`
+  * `VS90COMNTOOLS`
+  * `VS100COMNTOOLS`
+  * `VS110COMNTOOLS`
+  * `VS120COMNTOOLS`
+  * `VS140COMNTOOLS`
+"""
+
+load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_vc_path")
+load("@bazel_tools//tools/cpp:windows_cc_configure.bzl", "find_msvc_tool")
+load("@bazel_tools//tools/cpp:lib_cc_configure.bzl", "auto_configure_fail")
+
+def _def_file_filter_configure_impl(repository_ctx):
+  if repository_ctx.os.name.lower().find("windows") == -1:
+    repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
+    repository_ctx.file("def_file_filter.py", "")
+    return
+  vc_path = find_vc_path(repository_ctx)
+  if vc_path == "visual-studio-not-found":
+    auto_configure_fail("Visual C++ build tools not found on your machine")
+  undname_bin_path = find_msvc_tool(repository_ctx, vc_path, "undname.exe").replace("\\", "\\\\")
+
+  repository_ctx.template(
+    "def_file_filter.py",
+    Label("//tensorflow/tools/def_file_filter:def_file_filter.py.tpl"),
+    {
+      "%{undname_bin_path}": undname_bin_path,
+    })
+  repository_ctx.symlink(Label("//tensorflow/tools/def_file_filter:BUILD.tpl"), "BUILD")
+
+
+def_file_filter_configure = repository_rule(
+    implementation = _def_file_filter_configure_impl,
+    environ = [
+        "BAZEL_VC",
+        "BAZEL_VS",
+        "VS90COMNTOOLS",
+        "VS100COMNTOOLS",
+        "VS110COMNTOOLS",
+        "VS120COMNTOOLS",
+        "VS140COMNTOOLS"
+    ],
+)
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 62fec2c402..4a70f666b6 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -48,36 +48,66 @@ py_binary(
     deps = ["//tensorflow:tensorflow_py"],
 )
 
+COMMON_PIP_DEPS = [
+    ":licenses",
+    "MANIFEST.in",
+    "README",
+    "setup.py",
+    ":included_headers",
+    "//tensorflow:tensorflow_py",
+    "//tensorflow/contrib/autograph:autograph",
+    "//tensorflow/contrib/autograph/converters:converters",
+    "//tensorflow/contrib/autograph/converters:test_lib",
+    "//tensorflow/contrib/autograph/impl:impl",
+    "//tensorflow/contrib/autograph/pyct:pyct",
+    "//tensorflow/contrib/autograph/pyct/static_analysis:static_analysis",
+    "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
+    "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
+    "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
+    "//tensorflow/contrib/data/python/ops:contrib_op_loader",
+    "//tensorflow/contrib/eager/python/examples:examples_pip",
+    "//tensorflow/contrib/eager/python:checkpointable_utils",
+    "//tensorflow/contrib/eager/python:evaluator",
+    "//tensorflow/contrib/gan:gan",
+    "//tensorflow/contrib/graph_editor:graph_editor_pip",
+    "//tensorflow/contrib/keras:keras",
+    "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
+    "//tensorflow/contrib/nn:nn_py",
+    "//tensorflow/contrib/predictor:predictor_pip",
+    "//tensorflow/contrib/receptive_field:receptive_field_pip",
+    "//tensorflow/contrib/session_bundle:session_bundle_pip",
+    "//tensorflow/contrib/signal:signal_py",
+    "//tensorflow/contrib/signal:test_util",
+    "//tensorflow/contrib/slim:slim",
+    "//tensorflow/contrib/slim/python/slim/data:data_pip",
+    "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
+    "//tensorflow/contrib/specs:specs",
+    "//tensorflow/contrib/summary:summary_test_util",
+    "//tensorflow/contrib/tensor_forest:init_py",
+    "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
+    "//tensorflow/contrib/timeseries:timeseries_pip",
+    "//tensorflow/contrib/tpu",
+    "//tensorflow/examples/tutorials/mnist:package",
+    "//tensorflow/python:distributed_framework_test_lib",
+    "//tensorflow/python:meta_graph_testdata",
+    "//tensorflow/python:spectral_ops_test_util",
+    "//tensorflow/python:util_example_parser_configuration",
+    "//tensorflow/python/debug:debug_pip",
+    "//tensorflow/python/eager:eager_pip",
+    "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files",
+    "//tensorflow/python/saved_model:saved_model",
+    "//tensorflow/python/tools:tools_pip",
+    "//tensorflow/python:test_ops",
+    "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
+]
+
 # On Windows, python binary is a zip file of runfiles tree.
 # Add everything to its data dependency for generating a runfiles tree
 # for building the pip package on Windows.
 py_binary(
     name = "simple_console_for_windows",
     srcs = ["simple_console_for_windows.py"],
-    data = [
-        "MANIFEST.in",
-        "README",
-        "setup.py",
-        ":included_headers",
-        "//tensorflow/contrib/nn:nn_py",
-        "//tensorflow/contrib/session_bundle:session_bundle_pip",
-        "//tensorflow/contrib/signal:signal_py",
-        "//tensorflow/contrib/slim/python/slim/data:data_pip",
-        "//tensorflow/python:util_example_parser_configuration",
-        "//tensorflow/python/debug:debug_pip",
-        "//tensorflow/python/saved_model",
-        "//tensorflow/python:spectral_ops_test_util",
-        "//tensorflow/python/tools:tools_pip",
-        "//tensorflow/python/eager:eager_pip",
-        "//tensorflow/contrib/summary:summary_test_util",
-        # These targets don't build on Windows yet. Exclude them for now.
-        # "//tensorflow/contrib/slim",
-        # "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-        # "//tensorflow/contrib/specs",
-        # "//tensorflow/contrib/tensor_forest:init_py",
-        # "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-        # "//tensorflow/examples/tutorials/mnist:package",
-    ],
+    data = COMMON_PIP_DEPS,
     srcs_version = "PY2AND3",
     deps = ["//tensorflow:tensorflow_py"],
 )
@@ -138,63 +168,13 @@ sh_binary(
     data = select({
         "//tensorflow:windows": [":simple_console_for_windows"],
         "//tensorflow:windows_msvc": [":simple_console_for_windows"],
-        "//conditions:default": [
-            ":licenses",
-            "MANIFEST.in",
-            "README",
-            "setup.py",
-            ":included_headers",
+        "//conditions:default": COMMON_PIP_DEPS + [
             ":simple_console",
-            "//tensorflow:tensorflow_py",
-            "//tensorflow/contrib/boosted_trees:boosted_trees_pip",
-            "//tensorflow/contrib/cluster_resolver:cluster_resolver_pip",
-            "//tensorflow/contrib/data/python/kernel_tests:dataset_serialization_test",
-            "//tensorflow/contrib/data/python/ops:contrib_op_loader",
-            "//tensorflow/contrib/eager/python/examples:examples_pip",
-            "//tensorflow/contrib/eager/python:checkpointable_utils",
-            "//tensorflow/contrib/eager/python:evaluator",
-            "//tensorflow/contrib/gan:gan",
-            "//tensorflow/contrib/graph_editor:graph_editor_pip",
-            "//tensorflow/contrib/keras:keras",
-            "//tensorflow/contrib/labeled_tensor:labeled_tensor_pip",
             "//tensorflow/contrib/lite/python:interpreter_test_data",
             "//tensorflow/contrib/lite/python:tf_lite_py_pip",
             "//tensorflow/contrib/lite/toco:toco",
             "//tensorflow/contrib/lite/toco/python:toco_wrapper",
             "//tensorflow/contrib/lite/toco/python:toco_from_protos",
-            "//tensorflow/contrib/nn:nn_py",
-            "//tensorflow/contrib/predictor:predictor_pip",
-            "//tensorflow/contrib/autograph:autograph",
-            "//tensorflow/contrib/autograph/converters:converters",
-            "//tensorflow/contrib/autograph/converters:test_lib",
-            "//tensorflow/contrib/autograph/impl:impl",
-            "//tensorflow/contrib/autograph/pyct:pyct",
-            "//tensorflow/contrib/autograph/pyct/static_analysis:static_analysis",
-            "//tensorflow/contrib/receptive_field:receptive_field_pip",
-            "//tensorflow/contrib/session_bundle:session_bundle_pip",
-            "//tensorflow/contrib/signal:signal_py",
-            "//tensorflow/contrib/signal:test_util",
-            "//tensorflow/contrib/slim:slim",
-            "//tensorflow/contrib/slim/python/slim/data:data_pip",
-            "//tensorflow/contrib/slim/python/slim/nets:nets_pip",
-            "//tensorflow/contrib/specs:specs",
-            "//tensorflow/contrib/summary:summary_test_util",
-            "//tensorflow/contrib/tensor_forest:init_py",
-            "//tensorflow/contrib/tensor_forest/hybrid:hybrid_pip",
-            "//tensorflow/contrib/timeseries:timeseries_pip",
-            "//tensorflow/contrib/tpu",
-            "//tensorflow/examples/tutorials/mnist:package",
-            "//tensorflow/python:distributed_framework_test_lib",
-            "//tensorflow/python:meta_graph_testdata",
-            "//tensorflow/python:spectral_ops_test_util",
-            "//tensorflow/python:util_example_parser_configuration",
-            "//tensorflow/python/debug:debug_pip",
-            "//tensorflow/python/eager:eager_pip",
-            "//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files",
-            "//tensorflow/python/saved_model:saved_model",
-            "//tensorflow/python/tools:tools_pip",
-            "//tensorflow/python:test_ops",
-            "//tensorflow/tools/dist_test/server:grpc_tensorflow_server",
         ],
     }) + if_mkl(["//third_party/mkl:intel_binary_blob"]) + if_tensorrt([
         "//tensorflow/contrib/tensorrt:init_py",
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index fe6b9407d6..5dd27bcda0 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -13,6 +13,8 @@ load("//third_party:repo.bzl", "tf_http_archive")
 load("//third_party/clang_toolchain:cc_configure_clang.bzl", "cc_download_clang_toolchain")
 load("@io_bazel_rules_closure//closure/private:java_import_external.bzl", "java_import_external")
 load("@io_bazel_rules_closure//closure:defs.bzl", "filegroup_external")
+load("//tensorflow/tools/def_file_filter:def_file_filter_configure.bzl",
+     "def_file_filter_configure")
 
 
 # Sanitize a dependency so that it works correctly from code that includes
@@ -33,6 +35,10 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   sycl_configure(name="local_config_sycl")
   python_configure(name="local_config_python")
 
+  # For windows bazel build
+  # TODO: Remove def file filter when TensorFlow can export symbols properly on Windows.
+  def_file_filter_configure(name = "local_config_def_file_filter")
+
   # Point //external/local_config_arm_compiler to //external/arm_compiler
   arm_compiler_configure(
       name="local_config_arm_compiler",
-- 
GitLab


From 3027f580046866cb74d5edf4e41c9406e007234c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Mon, 2 Apr 2018 21:02:40 -0700
Subject: [PATCH 842/960] BUG_FIX: Allow Uniform pdf to work on float64 inputs.
 PiperOrigin-RevId: 191391778

---
 .../kernel_tests/distributions/uniform_test.py   | 16 ++++++++++++++++
 tensorflow/python/ops/distributions/uniform.py   |  3 ++-
 2 files changed, 18 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/distributions/uniform_test.py b/tensorflow/python/kernel_tests/distributions/uniform_test.py
index df99a0ed25..a8def95b14 100644
--- a/tensorflow/python/kernel_tests/distributions/uniform_test.py
+++ b/tensorflow/python/kernel_tests/distributions/uniform_test.py
@@ -281,6 +281,22 @@ class UniformTest(test.TestCase):
       expected_pdf = [1.0, 0.1]
       self.assertAllClose(expected_pdf, pdf.eval())
 
+  def testUniformFloat64(self):
+    uniform = uniform_lib.Uniform(
+        low=np.float64(0.), high=np.float64(1.))
+
+    self.assertAllClose(
+        [1., 1.],
+        self.evaluate(uniform.prob(np.array([0.5, 0.6], dtype=np.float64))))
+
+    self.assertAllClose(
+        [0.5, 0.6],
+        self.evaluate(uniform.cdf(np.array([0.5, 0.6], dtype=np.float64))))
+
+    self.assertAllClose(0.5, self.evaluate(uniform.mean()))
+    self.assertAllClose(1 / 12., self.evaluate(uniform.variance()))
+    self.assertAllClose(0., self.evaluate(uniform.entropy()))
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/python/ops/distributions/uniform.py b/tensorflow/python/ops/distributions/uniform.py
index ec623b55eb..0891bffdd5 100644
--- a/tensorflow/python/ops/distributions/uniform.py
+++ b/tensorflow/python/ops/distributions/uniform.py
@@ -166,7 +166,8 @@ class Uniform(distribution.Distribution):
     return self.low + self.range() * samples
 
   def _prob(self, x):
-    broadcasted_x = x * array_ops.ones(self.batch_shape_tensor())
+    broadcasted_x = x * array_ops.ones(
+        self.batch_shape_tensor(), dtype=x.dtype)
     return array_ops.where(
         math_ops.is_nan(broadcasted_x),
         broadcasted_x,
-- 
GitLab


From 53eeeb7ac4a876a59ae975a8d6dd8a48f645b7b7 Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Mon, 2 Apr 2018 21:03:06 -0700
Subject: [PATCH 843/960] Re-enable Gather and Slice estimators with output
 size check.

PiperOrigin-RevId: 191391805
---
 .../grappler/costs/op_level_cost_estimator.cc | 10 ++-
 .../costs/op_level_cost_estimator_test.cc     | 75 +++++++++++--------
 2 files changed, 51 insertions(+), 34 deletions(-)

diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
index 75258d0547..14e46ecdd9 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator.cc
@@ -202,12 +202,9 @@ OpLevelCostEstimator::OpLevelCostEstimator() {
 
       {kNoOp, wrap(&OpLevelCostEstimator::PredictNoOp)},
 
-      // TODO(76227186): re-enable with output size check & test
-      /*
       {kGather, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
       {kGatherV2, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
       {kSlice, wrap(&OpLevelCostEstimator::PredictGatherOrSlice)},
-      */
 
       {kPlaceholder, wrap(&OpLevelCostEstimator::PredictIdentity)},
       {kIdentity, wrap(&OpLevelCostEstimator::PredictIdentity)},
@@ -1058,6 +1055,13 @@ Costs OpLevelCostEstimator::PredictGatherOrSlice(
   // part of it. For these op the size of the output determines the memory cost.
   const auto& op_info = op_context.op_info;
 
+  const int inputs_needed = op_info.op() == "Slice" ? 3 : 2;
+  if (op_info.outputs_size() == 0 || op_info.inputs_size() < inputs_needed) {
+    Costs costs = Costs::ZeroCosts();
+    costs.inaccurate = true;
+    return costs;
+  }
+
   bool unknown_shapes = false;
 
   // Each output element is a copy of some element from input.
diff --git a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
index 4758bbfee7..d797a8a8c1 100644
--- a/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
+++ b/tensorflow/core/grappler/costs/op_level_cost_estimator_test.cc
@@ -401,43 +401,56 @@ class OpLevelCostEstimatorTest : public ::testing::Test {
   OpLevelCostEstimator estimator_;
 };
 
-// TODO(76227186): re-enable with output size check & test
-/*
 TEST_F(OpLevelCostEstimatorTest, TestGatherCosts) {
-OpContext op_context;
-SetCpuDevice(&op_context.op_info);
-op_context.op_info.set_op("Gather");
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  op_context.op_info.set_op("Gather");
 
-// Huge first input shouldn't affect Gather execution and memory costs.
-DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
-DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
-DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
+  // Huge first input shouldn't affect Gather execution and memory costs.
+  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+  DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankOutput({16, 10}, DT_FLOAT, &op_context.op_info);
 
-auto cost = estimator_.PredictCosts(op_context);
-EXPECT_EQ(Costs::Duration(130), cost.memory_time);
-EXPECT_EQ(Costs::Duration(16), cost.compute_time);
-EXPECT_EQ(Costs::Duration(146), cost.execution_time);
-EXPECT_FALSE(cost.inaccurate);
+  auto cost = estimator_.PredictCosts(op_context);
+  EXPECT_EQ(Costs::Duration(130), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(16), cost.compute_time);
+  EXPECT_EQ(Costs::Duration(146), cost.execution_time);
+  EXPECT_FALSE(cost.inaccurate);
+}
+
+TEST_F(OpLevelCostEstimatorTest, TestGatherCostsWithoutOutput) {
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  op_context.op_info.set_op("Gather");
+
+  // Huge first input shouldn't affect Gather execution and memory costs.
+  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+  DescribeArbitraryRankInput({16}, DT_INT64, &op_context.op_info);
+
+  auto cost = estimator_.PredictCosts(op_context);
+  EXPECT_EQ(Costs::Duration(0), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(0), cost.compute_time);
+  EXPECT_EQ(Costs::Duration(0), cost.execution_time);
+  EXPECT_TRUE(cost.inaccurate);
 }
 
 TEST_F(OpLevelCostEstimatorTest, TestSliceCosts) {
-OpContext op_context;
-SetCpuDevice(&op_context.op_info);
-op_context.op_info.set_op("Slice");
-
-// Huge first input shouldn't affect Slice execution and memory costs.
-DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
-DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
-DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
-DescribeArbitraryRankOutput({10, 10}, DT_FLOAT, &op_context.op_info);
-
-auto cost = estimator_.PredictCosts(op_context);
-EXPECT_EQ(Costs::Duration(81), cost.memory_time);
-EXPECT_EQ(Costs::Duration(10), cost.compute_time);
-EXPECT_EQ(Costs::Duration(91), cost.execution_time);
-EXPECT_FALSE(cost.inaccurate);
-}
-*/
+  OpContext op_context;
+  SetCpuDevice(&op_context.op_info);
+  op_context.op_info.set_op("Slice");
+
+  // Huge first input shouldn't affect Slice execution and memory costs.
+  DescribeArbitraryRankInput({10000000, 10}, DT_FLOAT, &op_context.op_info);
+  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankInput({2}, DT_INT64, &op_context.op_info);
+  DescribeArbitraryRankOutput({10, 10}, DT_FLOAT, &op_context.op_info);
+
+  auto cost = estimator_.PredictCosts(op_context);
+  EXPECT_EQ(Costs::Duration(81), cost.memory_time);
+  EXPECT_EQ(Costs::Duration(10), cost.compute_time);
+  EXPECT_EQ(Costs::Duration(91), cost.execution_time);
+  EXPECT_FALSE(cost.inaccurate);
+}
 
 TEST_F(OpLevelCostEstimatorTest, BiasAddExecutionTime) {
   auto cost = PredictCosts(DescribeBiasAdd(1000, 10));
-- 
GitLab


From 89f86f518dc0317b9e4e7b2932d73209e636fa72 Mon Sep 17 00:00:00 2001
From: Igor Ganichev <iga@google.com>
Date: Mon, 2 Apr 2018 22:24:14 -0700
Subject: [PATCH 844/960] Clarify OpDef.is_stateful flag definition

PiperOrigin-RevId: 191396824
---
 tensorflow/core/framework/op_def.proto | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/core/framework/op_def.proto b/tensorflow/core/framework/op_def.proto
index ba545a1994..ca0e5e7133 100644
--- a/tensorflow/core/framework/op_def.proto
+++ b/tensorflow/core/framework/op_def.proto
@@ -126,6 +126,12 @@ message OpDef {
   // -------------------------------------------------------------------------
   // Optimization constraints.
 
+  // Ops are marked as stateful if their behavior depends on some state beyond
+  // their input tensors (e.g. variable reading op) or if they have
+  // a side-effect (e.g. printing or asserting ops). Equivalently, stateless ops
+  // must always produce the same output for the same input and have
+  // no side-effects.
+  //
   // By default Ops may be moved between devices.  Stateful ops should
   // either not be moved, or should only be moved if that state can also
   // be moved (e.g. via some sort of save / restore).
-- 
GitLab


From 4e6d0fa3969f93415175b1bf85a0068da9210c45 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Mon, 2 Apr 2018 23:44:24 -0700
Subject: [PATCH 845/960] Automated g4 rollback of changelist 191385909

PiperOrigin-RevId: 191401933
---
 tensorflow/core/grappler/optimizers/function_optimizer.cc | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 6cdc51f1bd..2a6b8a325f 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -286,6 +286,12 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
     if (func.attr().count("_noinline") != 0) {
       continue;
     }
+    // Don't touch anything marked XLA to prevent XLA failures further down the
+    // road.
+    if (func.attr().count("_XlaCompile") > 0 &&
+        func.attr().at("_XlaCompile").b()) {
+      continue;
+    }
     // Can't create IdentityN nodes with no input or output: skip these
     // functions for now.
     if (func.signature().input_arg_size() == 0 ||
-- 
GitLab


From 7dbb2b52d67ad584e7bb85d61e82c4a136ec5de0 Mon Sep 17 00:00:00 2001
From: Gunhan Gulsoy <gunan@google.com>
Date: Tue, 3 Apr 2018 01:01:16 -0700
Subject: [PATCH 846/960] Remove "-lpthread" when building tests on macos. In
 most cases it seems to be not used.

PiperOrigin-RevId: 191407383
---
 tensorflow/tensorflow.bzl | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index e9d2f279cd..098ae7e6e3 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -638,9 +638,12 @@ def tf_cc_test(name,
       linkopts=select({
         clean_dep("//tensorflow:android"): [
             "-pie",
-          ],
+        ],
         clean_dep("//tensorflow:windows"): [],
         clean_dep("//tensorflow:windows_msvc"): [],
+        clean_dep("//tensorflow:darwin"): [
+            "-lm",
+        ],
         "//conditions:default": [
             "-lpthread",
             "-lm"
-- 
GitLab


From 62f61fbdb992eb6257f9002c7580373f8c08e758 Mon Sep 17 00:00:00 2001
From: Silver Chan <chenchuanyinuestc@gmail.com>
Date: Tue, 3 Apr 2018 19:05:06 +0800
Subject: [PATCH 847/960] fix problem: ld libgrpc symbols for MacOS

fix problem: ld libgrpc symbols for MacOS when cmake build
---
 tensorflow/contrib/cmake/external/grpc.cmake | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/contrib/cmake/external/grpc.cmake b/tensorflow/contrib/cmake/external/grpc.cmake
index abfc69243e..1fefb731a7 100644
--- a/tensorflow/contrib/cmake/external/grpc.cmake
+++ b/tensorflow/contrib/cmake/external/grpc.cmake
@@ -35,6 +35,7 @@ else()
   set(grpc_STATIC_LIBRARIES
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc++_unsecure.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgrpc_unsecure.a
+      ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libaddress_sorting.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/third_party/cares/cares/lib/libcares.a
       ${CMAKE_CURRENT_BINARY_DIR}/grpc/src/grpc/libgpr.a)
 endif()
-- 
GitLab


From c3821d7f4f72dd6f03942b776a2887c44e12710b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 05:52:03 -0700
Subject: [PATCH 848/960] Update LLVM API usage to match upstream change.

PiperOrigin-RevId: 191428965
---
 tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc | 1 -
 tensorflow/compiler/xla/service/cpu/simple_orc_jit.h  | 1 -
 2 files changed, 2 deletions(-)

diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 80c24eaccf..4198260a22 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -87,7 +87,6 @@ SimpleOrcJIT::SimpleOrcJIT(const llvm::TargetOptions& target_options,
                                 /*MAttrs=*/DetectMachineAttributes()))),
       disassembler_(*target_machine_),
       data_layout_(target_machine_->createDataLayout()),
-      execution_session_(string_pool_),
       symbol_resolver_(llvm::orc::createLegacyLookupResolver(
           [this](const std::string& name) -> llvm::JITSymbol {
             return this->ResolveRuntimeSymbol(name);
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
index aaeff2de87..f4260a95bc 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.h
@@ -102,7 +102,6 @@ class SimpleOrcJIT {
   std::unique_ptr<llvm::TargetMachine> target_machine_;
   const Disassembler disassembler_;
   const llvm::DataLayout data_layout_;
-  llvm::orc::SymbolStringPool string_pool_;
   llvm::orc::ExecutionSession execution_session_;
   std::shared_ptr<llvm::orc::SymbolResolver> symbol_resolver_;
   ObjLayerT object_layer_;
-- 
GitLab


From b036452ca347b7ab809711f71041d0457b0062d0 Mon Sep 17 00:00:00 2001
From: Benjamin Kramer <kramerb@google.com>
Date: Tue, 3 Apr 2018 07:28:40 -0700
Subject: [PATCH 849/960] [TF:XLA] Bump open source llvm revision to r329057

DataTypes.h is no longer a generated header. X86DisassemblerDecoderCommon.h is now part of :support.

PiperOrigin-RevId: 191438031
---
 tensorflow/workspace.bzl    |  8 ++++----
 third_party/llvm/llvm.BUILD | 12 +-----------
 2 files changed, 5 insertions(+), 15 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 5dd27bcda0..0bb297e72e 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -460,11 +460,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "llvm",
       urls = [
-          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/1c3cdea2f181d8e14ee184466c5fb237f1b4cda8.tar.gz",
-          "https://github.com/llvm-mirror/llvm/archive/1c3cdea2f181d8e14ee184466c5fb237f1b4cda8.tar.gz",
+          "https://mirror.bazel.build/github.com/llvm-mirror/llvm/archive/7e78daafdd22f3f17720a103d29d89590534004e.tar.gz",
+          "https://github.com/llvm-mirror/llvm/archive/7e78daafdd22f3f17720a103d29d89590534004e.tar.gz",
       ],
-      sha256 = "1efbb9b05af88368be984d2f6526061d4a857181ef10f8841889a3a46869bb01",
-      strip_prefix = "llvm-1c3cdea2f181d8e14ee184466c5fb237f1b4cda8",
+      sha256 = "a6d94bd9de23515a1e3792a830421e3885977ea43d03427cdbe68f98cb7e0045",
+      strip_prefix = "llvm-7e78daafdd22f3f17720a103d29d89590534004e",
       build_file = clean_dep("//third_party/llvm:llvm.BUILD"),
   )
 
diff --git a/third_party/llvm/llvm.BUILD b/third_party/llvm/llvm.BUILD
index 28293a3659..075b46896e 100644
--- a/third_party/llvm/llvm.BUILD
+++ b/third_party/llvm/llvm.BUILD
@@ -162,13 +162,6 @@ all_cmake_vars = select({
 })
 
 # Performs CMake variable substitutions on configuration header files.
-expand_cmake_vars(
-    name = "datatypes_gen",
-    src = "include/llvm/Support/DataTypes.h.cmake",
-    cmake_vars = all_cmake_vars,
-    dst = "include/llvm/Support/DataTypes.h",
-)
-
 expand_cmake_vars(
     name = "config_gen",
     src = "include/llvm/Config/config.h.cmake",
@@ -305,9 +298,7 @@ cc_binary(
     srcs = glob([
         "utils/TableGen/*.cpp",
         "utils/TableGen/*.h",
-    ]) + [
-        "lib/Target/X86/Disassembler/X86DisassemblerDecoderCommon.h",
-    ],
+    ]),
     linkopts = [
         "-lm",
         "-ldl",
@@ -2014,7 +2005,6 @@ cc_library(
         "include/llvm/Support/WasmRelocs/*.def",
     ]) + [
         "include/llvm/BinaryFormat/MachO.def",
-        "include/llvm/Support/DataTypes.h",
         "include/llvm/Support/VCSRevision.h",
         "include/llvm/ExecutionEngine/ObjectMemoryBuffer.h",
     ],
-- 
GitLab


From caf6706067718e5b8856036893977085b3b18fcb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 08:54:01 -0700
Subject: [PATCH 850/960] Changes loss_reduction default to SUM_OVER_BATCH_SIZE
 for regression_head and poisson_regression_head.

PiperOrigin-RevId: 191446787
---
 tensorflow/contrib/estimator/BUILD                 |  3 +++
 .../python/estimator/dnn_linear_combined_test.py   |  9 +++++++--
 .../contrib/estimator/python/estimator/dnn_test.py |  5 ++++-
 .../contrib/estimator/python/estimator/head.py     | 12 ++++++++----
 .../estimator/python/estimator/head_test.py        |  4 ++--
 .../estimator/python/estimator/linear_test.py      |  5 ++++-
 .../estimator/python/estimator/multi_head_test.py  | 14 +++++++-------
 7 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/tensorflow/contrib/estimator/BUILD b/tensorflow/contrib/estimator/BUILD
index 2be62c9438..bec0329ebb 100644
--- a/tensorflow/contrib/estimator/BUILD
+++ b/tensorflow/contrib/estimator/BUILD
@@ -89,6 +89,7 @@ py_test(
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
         "//tensorflow/python/feature_column",
+        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -129,6 +130,7 @@ py_test(
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
         "//tensorflow/python/feature_column",
+        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
@@ -266,6 +268,7 @@ py_test(
         "//tensorflow/python/estimator:numpy_io",
         "//tensorflow/python/estimator:prediction_keys",
         "//tensorflow/python/feature_column",
+        "//tensorflow/python/ops/losses",
         "//third_party/py/numpy",
         "@six_archive//:six",
     ],
diff --git a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined_test.py b/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined_test.py
index b5e4d34dc7..dd009a6753 100644
--- a/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/dnn_linear_combined_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import nn
+from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
@@ -52,7 +53,9 @@ def _dnn_only_estimator_fn(
     config=None):
   return dnn_linear_combined.DNNLinearCombinedEstimator(
       head=head_lib.regression_head(
-          weight_column=weight_column, label_dimension=label_dimension),
+          weight_column=weight_column, label_dimension=label_dimension,
+          # Tests in core (from which this test inherits) test the sum loss.
+          loss_reduction=losses.Reduction.SUM),
       model_dir=model_dir,
       dnn_feature_columns=feature_columns,
       dnn_optimizer=optimizer,
@@ -100,7 +103,9 @@ def _linear_only_estimator_fn(
     partitioner=None):
   return dnn_linear_combined.DNNLinearCombinedEstimator(
       head=head_lib.regression_head(
-          weight_column=weight_column, label_dimension=label_dimension),
+          weight_column=weight_column, label_dimension=label_dimension,
+          # Tests in core (from which this test inherits) test the sum loss.
+          loss_reduction=losses.Reduction.SUM),
       model_dir=model_dir,
       linear_feature_columns=feature_columns,
       linear_optimizer=optimizer,
diff --git a/tensorflow/contrib/estimator/python/estimator/dnn_test.py b/tensorflow/contrib/estimator/python/estimator/dnn_test.py
index 71f810acec..75e3107670 100644
--- a/tensorflow/contrib/estimator/python/estimator/dnn_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/dnn_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.estimator.export import export
 from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import ops
+from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
@@ -41,7 +42,9 @@ def _dnn_estimator_fn(weight_column=None, label_dimension=1, *args, **kwargs):
   """Returns a DNNEstimator that uses regression_head."""
   return dnn.DNNEstimator(
       head=head_lib.regression_head(
-          weight_column=weight_column, label_dimension=label_dimension),
+          weight_column=weight_column, label_dimension=label_dimension,
+          # Tests in core (from which this test inherits) test the sum loss.
+          loss_reduction=losses.Reduction.SUM),
       *args, **kwargs)
 
 
diff --git a/tensorflow/contrib/estimator/python/estimator/head.py b/tensorflow/contrib/estimator/python/estimator/head.py
index 74da2cbb3f..85ef3291ba 100644
--- a/tensorflow/contrib/estimator/python/estimator/head.py
+++ b/tensorflow/contrib/estimator/python/estimator/head.py
@@ -178,7 +178,7 @@ def binary_classification_head(
 
 def regression_head(weight_column=None,
                     label_dimension=1,
-                    loss_reduction=losses.Reduction.SUM,
+                    loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE,
                     loss_fn=None,
                     inverse_link_fn=None,
                     name=None):
@@ -218,7 +218,9 @@ def regression_head(weight_column=None,
       of the last dimension of the labels `Tensor` (typically, this has shape
       `[batch_size, label_dimension]`).
     loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
-      reduce training loss over batch. Defaults to `SUM`.
+      reduce training loss over batch and label dimension. Defaults to
+      `SUM_OVER_BATCH_SIZE`, namely weighted sum of losses divided by
+      `batch size * label_dimension`. See `tf.losses.Reduction`.
     loss_fn: Optional loss function. Defaults to `mean_squared_error`.
     inverse_link_fn: Optional inverse link function, also known as 'mean
       function'. Defaults to identity.
@@ -243,7 +245,7 @@ def regression_head(weight_column=None,
 def poisson_regression_head(
     weight_column=None,
     label_dimension=1,
-    loss_reduction=losses.Reduction.SUM,
+    loss_reduction=losses.Reduction.SUM_OVER_BATCH_SIZE,
     compute_full_loss=True,
     name=None):
   """Creates a `_Head` for poisson regression using `tf.nn.log_poisson_loss`.
@@ -275,7 +277,9 @@ def poisson_regression_head(
       of the last dimension of the labels `Tensor` (typically, this has shape
       `[batch_size, label_dimension]`).
     loss_reduction: One of `tf.losses.Reduction` except `NONE`. Describes how to
-      reduce training loss over batch. Defaults to `SUM`.
+      reduce training loss over batch and label dimension. Defaults to
+      `SUM_OVER_BATCH_SIZE`, namely weighted sum of losses divided by
+      `batch size * label_dimension`. See `tf.losses.Reduction`.
     compute_full_loss: Whether to include the constant `log(z!)` term in
       computing the poisson loss. See `tf.nn.log_poisson_loss` for the full
       documentation.
diff --git a/tensorflow/contrib/estimator/python/estimator/head_test.py b/tensorflow/contrib/estimator/python/estimator/head_test.py
index 8837dfdc6c..98962ca427 100644
--- a/tensorflow/contrib/estimator/python/estimator/head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/head_test.py
@@ -1162,8 +1162,8 @@ class PoissonRegressionHead(test.TestCase):
     #         exp(-1) - 2 * (-1) + 2*ln(2) - 2 + 0.5*ln(2*pi*2),
     #         exp(1) - 3 * 1 + 3*ln(3) - 3 + 0.5*ln(2*pi*3)]
     #      = [1.0, 3.020, 1.482]
-    # sum_loss = 5.502
-    expected_loss = 5.502
+    # training_loss = (1.0 + 3.020 + 1.482) / 3
+    expected_loss = 1.834
     atol = 0.001
     expected_train_result = b'my_train_op'
     def _train_op_fn(loss):
diff --git a/tensorflow/contrib/estimator/python/estimator/linear_test.py b/tensorflow/contrib/estimator/python/estimator/linear_test.py
index c63514eb68..c41996b9c6 100644
--- a/tensorflow/contrib/estimator/python/estimator/linear_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/linear_test.py
@@ -32,6 +32,7 @@ from tensorflow.python.estimator.export import export
 from tensorflow.python.estimator.inputs import numpy_io
 from tensorflow.python.feature_column import feature_column
 from tensorflow.python.framework import ops
+from tensorflow.python.ops.losses import losses
 from tensorflow.python.platform import gfile
 from tensorflow.python.platform import test
 from tensorflow.python.summary.writer import writer_cache
@@ -42,7 +43,9 @@ def _linear_estimator_fn(
   """Returns a LinearEstimator that uses regression_head."""
   return linear.LinearEstimator(
       head=head_lib.regression_head(
-          weight_column=weight_column, label_dimension=label_dimension),
+          weight_column=weight_column, label_dimension=label_dimension,
+          # Tests in core (from which this test inherits) test the sum loss.
+          loss_reduction=losses.Reduction.SUM),
       *args, **kwargs)
 
 
diff --git a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
index 74d3d6d728..d9e5aca295 100644
--- a/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
+++ b/tensorflow/contrib/estimator/python/estimator/multi_head_test.py
@@ -483,14 +483,14 @@ class MultiHeadTest(test.TestCase):
                            [[2., 2., 0.], [2., 2., 0.]]], dtype=np.float32),
     }
     # Loss for the first head:
-    # loss1 = (1+1)^2 + (0-1)^2 + (1+1)^2 + (0-1)^2 +
-    #         (1.5+1.5)^2 + (1.5-1.5)^2 + (1.5+1.5)^2 + (1.5-1.5)^2
-    #       = 28
+    # loss1 = ((1+1)^2 + (0-1)^2 + (1+1)^2 + (0-1)^2 +
+    #          (1.5+1.5)^2 + (1.5-1.5)^2 + (1.5+1.5)^2 + (1.5-1.5)^2) / 8
+    #       = 3.5
     # Loss for the second head:
-    # loss2 = (0-2)^2 + (1+2)^2 + (0-2)^2 + (0-2)^2 + (1+2)^2 + (0-2)^2 +
-    #         (2+2)^2 + (2-2)^2 + (0+2)^2 + (2+2)^2 + (2-2)^2 + (0+2)^2
-    #       = 74
-    expected_training_loss = 28. + 74.
+    # loss2 = ((0-2)^2 + (1+2)^2 + (0-2)^2 + (0-2)^2 + (1+2)^2 + (0-2)^2 +
+    #          (2+2)^2 + (2-2)^2 + (0+2)^2 + (2+2)^2 + (2-2)^2 + (0+2)^2) / 12
+    #       = 6.167
+    expected_training_loss = 3.5 + 6.167
 
     training_loss = multi_head.create_loss(
         features={},
-- 
GitLab


From 5d1086ae98ccfe691161ff50c93036d432866741 Mon Sep 17 00:00:00 2001
From: Ian Langmore <langmore@google.com>
Date: Tue, 3 Apr 2018 08:59:08 -0700
Subject: [PATCH 851/960] cholesky_solve_with_broadcast,
 matrix_solve_with_broadcast and matrix_triangular_solve_with_broadcast added
 to linear_operator_util.py

PiperOrigin-RevId: 191447378
---
 .../linalg/linear_operator_util_test.py       | 136 ++++++++++++++++--
 .../python/ops/linalg/linear_operator_util.py |  85 +++++++++--
 2 files changed, 202 insertions(+), 19 deletions(-)

diff --git a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
index e1edffc3d9..7b291e29de 100644
--- a/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
+++ b/tensorflow/python/kernel_tests/linalg/linear_operator_util_test.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import random_seed
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops.linalg import linear_operator_util
 from tensorflow.python.platform import test
@@ -94,8 +95,8 @@ class AssertNoEntriesWithModulusZeroTest(test.TestCase):
 class BroadcastMatrixBatchDimsTest(test.TestCase):
 
   def test_zero_batch_matrices_returned_as_empty_list(self):
-    self.assertAllEqual(
-        [], linear_operator_util.broadcast_matrix_batch_dims([]))
+    self.assertAllEqual([],
+                        linear_operator_util.broadcast_matrix_batch_dims([]))
 
   def test_one_batch_matrix_returned_after_tensor_conversion(self):
     arr = rng.rand(2, 3, 4)
@@ -194,6 +195,44 @@ class BroadcastMatrixBatchDimsTest(test.TestCase):
       linear_operator_util.broadcast_matrix_batch_dims([y, x])
 
 
+class CholeskySolveWithBroadcastTest(test.TestCase):
+
+  def test_static_dims_broadcast(self):
+    # batch_shape = [2]
+    chol = rng.rand(3, 3)
+    rhs = rng.rand(2, 3, 7)
+    chol_broadcast = chol + np.zeros((2, 1, 1))
+
+    with self.test_session():
+      result = linear_operator_util.cholesky_solve_with_broadcast(chol, rhs)
+      self.assertAllEqual((2, 3, 7), result.get_shape())
+      expected = linalg_ops.cholesky_solve(chol_broadcast, rhs)
+      self.assertAllEqual(expected.eval(), result.eval())
+
+  def test_dynamic_dims_broadcast_64bit(self):
+    # batch_shape = [2, 2]
+    chol = rng.rand(2, 3, 3)
+    rhs = rng.rand(2, 1, 3, 7)
+    chol_broadcast = chol + np.zeros((2, 2, 1, 1))
+    rhs_broadcast = rhs + np.zeros((2, 2, 1, 1))
+
+    chol_ph = array_ops.placeholder(dtypes.float64)
+    rhs_ph = array_ops.placeholder(dtypes.float64)
+
+    with self.test_session() as sess:
+      result, expected = sess.run(
+          [
+              linear_operator_util.cholesky_solve_with_broadcast(
+                  chol_ph, rhs_ph),
+              linalg_ops.cholesky_solve(chol_broadcast, rhs_broadcast)
+          ],
+          feed_dict={
+              chol_ph: chol,
+              rhs_ph: rhs,
+          })
+      self.assertAllEqual(expected, result)
+
+
 class MatmulWithBroadcastTest(test.TestCase):
 
   def test_static_dims_broadcast(self):
@@ -209,7 +248,7 @@ class MatmulWithBroadcastTest(test.TestCase):
       expected = math_ops.matmul(x, y_broadcast)
       self.assertAllEqual(expected.eval(), result.eval())
 
-  def test_dynamic_dims_broadcast_32bit(self):
+  def test_dynamic_dims_broadcast_64bit(self):
     # batch_shape = [2]
     # for each batch member, we have a 1x3 matrix times a 3x7 matrix ==> 1x7
     x = rng.rand(2, 1, 3)
@@ -221,9 +260,90 @@ class MatmulWithBroadcastTest(test.TestCase):
 
     with self.test_session() as sess:
       result, expected = sess.run(
-          [linear_operator_util.matmul_with_broadcast(x_ph, y_ph),
-           math_ops.matmul(x, y_broadcast)],
-          feed_dict={x_ph: x, y_ph: y})
+          [
+              linear_operator_util.matmul_with_broadcast(x_ph, y_ph),
+              math_ops.matmul(x, y_broadcast)
+          ],
+          feed_dict={
+              x_ph: x,
+              y_ph: y
+          })
+      self.assertAllEqual(expected, result)
+
+
+class MatrixSolveWithBroadcastTest(test.TestCase):
+
+  def test_static_dims_broadcast(self):
+    # batch_shape = [2]
+    matrix = rng.rand(3, 3)
+    rhs = rng.rand(2, 3, 7)
+    matrix_broadcast = matrix + np.zeros((2, 1, 1))
+
+    with self.test_session():
+      result = linear_operator_util.matrix_solve_with_broadcast(matrix, rhs)
+      self.assertAllEqual((2, 3, 7), result.get_shape())
+      expected = linalg_ops.matrix_solve(matrix_broadcast, rhs)
+      self.assertAllEqual(expected.eval(), result.eval())
+
+  def test_dynamic_dims_broadcast_64bit(self):
+    # batch_shape = [2, 2]
+    matrix = rng.rand(2, 3, 3)
+    rhs = rng.rand(2, 1, 3, 7)
+    matrix_broadcast = matrix + np.zeros((2, 2, 1, 1))
+    rhs_broadcast = rhs + np.zeros((2, 2, 1, 1))
+
+    matrix_ph = array_ops.placeholder(dtypes.float64)
+    rhs_ph = array_ops.placeholder(dtypes.float64)
+
+    with self.test_session() as sess:
+      result, expected = sess.run(
+          [
+              linear_operator_util.matrix_solve_with_broadcast(
+                  matrix_ph, rhs_ph),
+              linalg_ops.matrix_solve(matrix_broadcast, rhs_broadcast)
+          ],
+          feed_dict={
+              matrix_ph: matrix,
+              rhs_ph: rhs,
+          })
+      self.assertAllEqual(expected, result)
+
+
+class MatrixTriangularSolveWithBroadcastTest(test.TestCase):
+
+  def test_static_dims_broadcast(self):
+    # batch_shape = [2]
+    matrix = rng.rand(2, 3, 3)
+    rhs = rng.rand(3, 7)
+    rhs_broadcast = rhs + np.zeros((2, 1, 1))
+
+    with self.test_session():
+      result = linear_operator_util.matrix_triangular_solve_with_broadcast(
+          matrix, rhs)
+      self.assertAllEqual((2, 3, 7), result.get_shape())
+      expected = linalg_ops.matrix_triangular_solve(matrix, rhs_broadcast)
+      self.assertAllEqual(expected.eval(), result.eval())
+
+  def test_dynamic_dims_broadcast_64bit(self):
+    # batch_shape = [2]
+    matrix = rng.rand(2, 3, 3)
+    rhs = rng.rand(3, 7)
+    rhs_broadcast = rhs + np.zeros((2, 1, 1))
+
+    matrix_ph = array_ops.placeholder(dtypes.float64)
+    rhs_ph = array_ops.placeholder(dtypes.float64)
+
+    with self.test_session() as sess:
+      result, expected = sess.run(
+          [
+              linear_operator_util.matrix_triangular_solve_with_broadcast(
+                  matrix_ph, rhs_ph),
+              linalg_ops.matrix_triangular_solve(matrix, rhs_broadcast)
+          ],
+          feed_dict={
+              matrix_ph: matrix,
+              rhs_ph: rhs,
+          })
       self.assertAllEqual(expected, result)
 
 
@@ -244,7 +364,7 @@ class AssertCompatibleMatrixDimensionsTest(test.TestCase):
       operator = DomainDimensionStubOperator(3)
       # Should not raise
       linear_operator_util.assert_compatible_matrix_dimensions(
-          operator, x).run()
+          operator, x).run()  # pyformat: disable
 
   def test_incompatible_dimensions_raise(self):
     with self.test_session():
@@ -252,7 +372,7 @@ class AssertCompatibleMatrixDimensionsTest(test.TestCase):
       operator = DomainDimensionStubOperator(3)
       with self.assertRaisesOpError("Incompatible matrix dimensions"):
         linear_operator_util.assert_compatible_matrix_dimensions(
-            operator, x).run()
+            operator, x).run()  # pyformat: disable
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/python/ops/linalg/linear_operator_util.py b/tensorflow/python/ops/linalg/linear_operator_util.py
index 427bd1e890..9dd40765c2 100644
--- a/tensorflow/python/ops/linalg/linear_operator_util.py
+++ b/tensorflow/python/ops/linalg/linear_operator_util.py
@@ -23,6 +23,7 @@ from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import linalg_ops
 from tensorflow.python.ops import math_ops
 
 
@@ -102,6 +103,22 @@ def assert_is_batch_matrix(tensor):
         "%s" % tensor)
 
 
+def shape_tensor(shape, name=None):
+  """Convert Tensor using default type, unless empty list or tuple."""
+  # Works just like random_ops._ShapeTensor.
+  if isinstance(shape, (tuple, list)) and not shape:
+    dtype = dtypes.int32
+  else:
+    dtype = None
+  return ops.convert_to_tensor(shape, dtype=dtype, name=name)
+
+
+################################################################################
+# Broadcasting versions of common linear algebra functions.
+# TODO(b/77519145) Do this more efficiently in some special cases.
+################################################################################
+
+
 def broadcast_matrix_batch_dims(batch_matrices, name=None):
   """Broadcast leading dimensions of zero or more [batch] matrices.
 
@@ -170,7 +187,8 @@ def broadcast_matrix_batch_dims(batch_matrices, name=None):
     bcast_batch_shape = batch_matrices[0].get_shape()[:-2]
     for mat in batch_matrices[1:]:
       bcast_batch_shape = array_ops.broadcast_static_shape(
-          bcast_batch_shape, mat.get_shape()[:-2])
+          bcast_batch_shape,
+          mat.get_shape()[:-2])
     if bcast_batch_shape.is_fully_defined():
       # The [1, 1] at the end will broadcast with anything.
       bcast_shape = bcast_batch_shape.concatenate([1, 1])
@@ -183,7 +201,8 @@ def broadcast_matrix_batch_dims(batch_matrices, name=None):
     bcast_batch_shape = array_ops.shape(batch_matrices[0])[:-2]
     for mat in batch_matrices[1:]:
       bcast_batch_shape = array_ops.broadcast_dynamic_shape(
-          bcast_batch_shape, array_ops.shape(mat)[:-2])
+          bcast_batch_shape,
+          array_ops.shape(mat)[:-2])
     bcast_shape = array_ops.concat([bcast_batch_shape, [1, 1]], axis=0)
     for i, mat in enumerate(batch_matrices):
       batch_matrices[i] = _broadcast_to_shape(mat, bcast_shape)
@@ -195,6 +214,13 @@ def _broadcast_to_shape(x, shape):
   return x + array_ops.zeros(shape=shape, dtype=x.dtype)
 
 
+def cholesky_solve_with_broadcast(chol, rhs, name=None):
+  """Solve systems of linear equations."""
+  with ops.name_scope(name, "CholeskySolveWithBroadcast", [chol, rhs]):
+    chol, rhs = broadcast_matrix_batch_dims([chol, rhs])
+    return linalg_ops.cholesky_solve(chol, rhs)
+
+
 def matmul_with_broadcast(a,
                           b,
                           transpose_a=False,
@@ -206,6 +232,11 @@ def matmul_with_broadcast(a,
                           name=None):
   """Multiplies matrix `a` by matrix `b`, producing `a @ b`.
 
+  Works identically to `tf.matmul`, but broadcasts batch dims
+  of `a` and `b` (by replicating) if they are determined statically to be
+  different, or if static shapes are not fully defined.  Thus, this may result
+  in an inefficient replication of data.
+
   The inputs must be matrices (or tensors of rank > 2, representing batches of
   matrices).
 
@@ -276,7 +307,7 @@ def matmul_with_broadcast(a,
     ValueError: If transpose_a and adjoint_a, or transpose_b and adjoint_b
       are both set to True.
   """
-  with ops.name_scope(name, "MatMulWithBroadcast", [a, b]) as name:
+  with ops.name_scope(name, "MatMulWithBroadcast", [a, b]):
     a, b = broadcast_matrix_batch_dims([a, b])
     return math_ops.matmul(
         a,
@@ -289,11 +320,43 @@ def matmul_with_broadcast(a,
         b_is_sparse=b_is_sparse)
 
 
-def shape_tensor(shape, name=None):
-  """Convert Tensor using default type, unless empty list or tuple."""
-  # Works just like random_ops._ShapeTensor.
-  if isinstance(shape, (tuple, list)) and not shape:
-    dtype = dtypes.int32
-  else:
-    dtype = None
-  return ops.convert_to_tensor(shape, dtype=dtype, name=name)
+def matrix_solve_with_broadcast(matrix, rhs, adjoint=False, name=None):
+  """Solve systems of linear equations."""
+  with ops.name_scope(name, "MatrixSolveWithBroadcast", [matrix, rhs]):
+    matrix, rhs = broadcast_matrix_batch_dims([matrix, rhs])
+    return linalg_ops.matrix_solve(matrix, rhs, adjoint=adjoint)
+
+
+def matrix_triangular_solve_with_broadcast(matrix,
+                                           rhs,
+                                           lower=True,
+                                           adjoint=False,
+                                           name=None):
+  """Solves triangular systems of linear equations with by backsubstitution.
+
+  Works identically to `tf.matrix_triangular_solve`, but broadcasts batch dims
+  of `matrix` and `rhs` (by replicating) if they are determined statically to be
+  different, or if static shapes are not fully defined.  Thus, this may result
+  in an inefficient replication of data.
+
+  Args:
+    matrix: A Tensor. Must be one of the following types:
+      `float64`, `float32`, `complex64`, `complex128`. Shape is `[..., M, M]`.
+    rhs: A `Tensor`. Must have the same `dtype` as `matrix`.
+      Shape is `[..., M, K]`.
+    lower: An optional `bool`. Defaults to `True`. Indicates whether the
+      innermost matrices in `matrix` are lower or upper triangular.
+    adjoint: An optional `bool`. Defaults to `False`. Indicates whether to solve
+      with matrix or its (block-wise) adjoint.
+    name: A name for the operation (optional).
+
+  Returns:
+    `Tensor` with same `dtype` as `matrix` and shape `[..., M, K]`.
+  """
+  with ops.name_scope(name, "MatrixTriangularSolve", [matrix, rhs]):
+    matrix, rhs = broadcast_matrix_batch_dims([matrix, rhs])
+    return linalg_ops.matrix_triangular_solve(
+        matrix,
+        rhs,
+        lower=lower,
+        adjoint=adjoint)
-- 
GitLab


From 4657e5336160e019379c373a369e3a9b199bc680 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 3 Apr 2018 09:09:13 -0700
Subject: [PATCH 852/960] Make batch_sequences_with_states_test.py work with
 the C API enabled, take 2.

It turns out the error can depend on what sequence comes first in the
input dict. This change internally sorts the input to make the error
predictable (this is useful for this test, as well as any users who
may run into this).

PiperOrigin-RevId: 191449214
---
 .../training/batch_sequences_with_states_test.py  | 15 ++++++---------
 .../training/sequence_queueing_state_saver.py     |  5 +++--
 2 files changed, 9 insertions(+), 11 deletions(-)

diff --git a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py b/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
index 16c260edb0..f305197c19 100644
--- a/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
+++ b/tensorflow/contrib/training/python/training/batch_sequences_with_states_test.py
@@ -27,6 +27,7 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors_impl
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import random_ops
@@ -517,6 +518,7 @@ class BatchSequencesWithStatesTestWithCApi(BatchSequencesWithStatesTest):
     ops._USE_C_API = self._prev_value
 
 
+@test_util.with_c_api
 class PaddingTest(test.TestCase):
 
   def testPaddingInvalidLengths(self):
@@ -526,15 +528,10 @@ class PaddingTest(test.TestCase):
           "key_2": constant_op.constant([1.5, 2.5])  # length 2
       }
 
-      if ops._USE_C_API:
-        with self.assertRaisesRegexp(
-            ValueError, "Fill dimensions must be >= 0"):
-          _, padded_seq = sqss._padding(sequences, 2)
-      else:
-        _, padded_seq = sqss._padding(sequences, 2)
-        with self.assertRaisesOpError(
-            ".*All sequence lengths must match, but received lengths.*"):
-          padded_seq["key_1"].eval()
+      _, padded_seq = sqss._padding(sequences, 2)
+      with self.assertRaisesOpError(
+          ".*All sequence lengths must match, but received lengths.*"):
+        padded_seq["key_1"].eval()
 
   def testPadding(self):
     with ops.Graph().as_default() as g, self.test_session(graph=g):
diff --git a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py
index 7223194885..99d486b183 100644
--- a/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py
+++ b/tensorflow/contrib/training/python/training/sequence_queueing_state_saver.py
@@ -1574,8 +1574,9 @@ def _padding(sequences, num_unroll):
   if not sequences:
     return 0, {}
 
-  sequences_dict = {}
-  for key, value in sequences.items():
+  # Sort 'sequences_dict' so 'length' will have a predictable value below.
+  sequences_dict = collections.OrderedDict()
+  for key, value in sorted(sequences.items()):
     if not (isinstance(value, sparse_tensor.SparseTensor) or
             isinstance(value, sparse_tensor.SparseTensorValue)):
       sequences_dict[key] = ops.convert_to_tensor(value)
-- 
GitLab


From 27c762c336bb11c8f74694e3d3ea5c8c47a28003 Mon Sep 17 00:00:00 2001
From: Shashi Shekhar <shashishekhar@google.com>
Date: Tue, 3 Apr 2018 09:47:18 -0700
Subject: [PATCH 853/960] Enable C++ warnings on a few targets.

PiperOrigin-RevId: 191454435
---
 tensorflow/contrib/lite/arena_planner.h   | 2 +-
 tensorflow/contrib/lite/nnapi_delegate.cc | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/arena_planner.h b/tensorflow/contrib/lite/arena_planner.h
index f84b3dad95..e9d0fbc5a9 100644
--- a/tensorflow/contrib/lite/arena_planner.h
+++ b/tensorflow/contrib/lite/arena_planner.h
@@ -25,7 +25,7 @@ limitations under the License.
 
 namespace tflite {
 
-class AllocationInfo;
+struct AllocationInfo;
 
 // A memory planner that makes all the allocations using arenas.
 //
diff --git a/tensorflow/contrib/lite/nnapi_delegate.cc b/tensorflow/contrib/lite/nnapi_delegate.cc
index decaf9f160..bc13444dc7 100644
--- a/tensorflow/contrib/lite/nnapi_delegate.cc
+++ b/tensorflow/contrib/lite/nnapi_delegate.cc
@@ -162,7 +162,7 @@ void AddOpsAndParams(tflite::Interpreter* interpreter,
     };
 
     auto duplicate_state_tensor_float32 =
-        [interpreter, &nn_model, &augmented_inputs, &next_id](int tensor_id) {
+        [interpreter, &nn_model, &augmented_inputs](int tensor_id) {
           const TfLiteTensor* tensor = interpreter->tensor(tensor_id);
           CHECK_NN(ANeuralNetworksModel_setOperandValue(
               nn_model, tensor_id, tensor->data.raw, tensor->bytes));
-- 
GitLab


From cf8c504688c5f5813c8772eb107ed3d4a1385888 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 10:00:00 -0700
Subject: [PATCH 854/960] Bug Fix: If num_uses > 0 the the inputs tensor need
 not be a list but can be reshaped to [batch_size*num_uses, input_size].
 `num_uses` should be incremented by one in this case.'

PiperOrigin-RevId: 191456184
---
 .../kfac/python/ops/layer_collection.py       | 23 ++++++++++++-------
 1 file changed, 15 insertions(+), 8 deletions(-)

diff --git a/tensorflow/contrib/kfac/python/ops/layer_collection.py b/tensorflow/contrib/kfac/python/ops/layer_collection.py
index 586a004f88..19608aca47 100644
--- a/tensorflow/contrib/kfac/python/ops/layer_collection.py
+++ b/tensorflow/contrib/kfac/python/ops/layer_collection.py
@@ -990,9 +990,11 @@ class LayerCollection(object):
                                                    num_uses=num_uses),
                                 reuse=reuse)
     block.register_additional_tower(inputs, outputs)
-
-    assert len(inputs) == len(outputs)
-    self._add_uses(params, len(inputs))
+    if isinstance(inputs, (tuple, list)):
+      assert len(inputs) == len(outputs)
+      self._add_uses(params, len(inputs))
+    else:
+      self._add_uses(params, 1)
 
   def register_conv2d_multi(self,
                             params,
@@ -1066,9 +1068,11 @@ class LayerCollection(object):
         reuse=reuse)
 
     block.register_additional_tower(inputs, outputs)
-
-    assert len(inputs) == len(outputs)
-    self._add_uses(params, len(inputs))
+    if isinstance(inputs, (tuple, list)):
+      assert len(inputs) == len(outputs)
+      self._add_uses(params, len(inputs))
+    else:
+      self._add_uses(params, 1)
 
   # TODO(b/74108452): change the loss registration functions names to refer
   # to "loss functions" instead of distributions.  Following naming convention
@@ -1088,7 +1092,7 @@ class LayerCollection(object):
       inputs: A list of Tensors, each of shape [batch_size, input_size] and
         dtype int32. Indices into embedding matrix. The list indexes each use
         in the graph (which might correspond to a "time-step" in an RNN).
-        OR, can be single Tensor, of shape [num_uses, batch_size, input_size],
+        OR, can be single Tensor, of shape [num_uses*batch_size, input_size],
         which is a reshaped version of a Tensor of shape [num_uses, batch_size,
         input_size].
       outputs: A list of Tensors, each of shape [batch_size, embedding_size].
@@ -1129,7 +1133,10 @@ class LayerCollection(object):
         params, block_type(self, vocab_size, num_uses=num_uses), reuse=reuse)
     block.register_additional_tower(inputs, outputs)
 
-    self._add_uses(params, len(inputs))
+    if isinstance(inputs, (tuple, list)):
+      self._add_uses(params, len(inputs))
+    else:
+      self._add_uses(params, 1)
 
   def register_categorical_predictive_distribution(self,
                                                    logits,
-- 
GitLab


From cfc886ac6064a04c71dd6c52e8c21ebec91eae50 Mon Sep 17 00:00:00 2001
From: Derek Murray <mrry@google.com>
Date: Tue, 3 Apr 2018 10:00:02 -0700
Subject: [PATCH 855/960] [tf.data] Fix handling of nested structures in
 `tf.contrib.data.prefetch_to_device()`.

PiperOrigin-RevId: 191456191
---
 .../kernel_tests/prefetching_ops_test.py      | 32 +++++++++++++++++++
 .../data/python/ops/prefetching_ops.py        | 17 +++++++++-
 2 files changed, 48 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index 676959a900..f2c57f92e2 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -231,6 +231,37 @@ class StagingAreaOpsTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  def testPrefetchDictToDevice(self):
+    host_dataset = dataset_ops.Dataset.range(10).map(lambda x: {"a": x})
+    device_dataset = host_dataset.apply(
+        prefetching_ops.prefetch_to_device("/cpu:1"))
+
+    # NOTE(mrry): This device block creates the "host" dataset and iterator on
+    # /cpu:0, and ensures that the prefetching is across devices. In typical use
+    # this would not be necessary, because the GPU device would not support any
+    # of the dataset-related ops.
+    with ops.device("/cpu:0"):
+      iterator = device_dataset.make_one_shot_iterator()
+
+    self.assertEqual(host_dataset.output_types, device_dataset.output_types)
+    self.assertEqual(host_dataset.output_types, iterator.output_types)
+    self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
+    self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
+    self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
+    self.assertEqual(host_dataset.output_classes, iterator.output_classes)
+
+    next_element = iterator.get_next()
+    self.assertEqual(dtypes.int64, next_element["a"].dtype)
+    self.assertEqual([], next_element["a"].shape)
+
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+    with self.test_session(config=worker_config) as sess:
+      for i in range(10):
+        self.assertEqual({"a": i}, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
   def testPrefetchToDeviceGpu(self):
     if not test_util.is_gpu_available():
       self.skipTest("No GPU available")
@@ -248,5 +279,6 @@ class StagingAreaOpsTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 98651bb568..554bfaa2cf 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -28,6 +28,7 @@ from tensorflow.python.data.util import sparse
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import function
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 
 
 # TODO(rohanj): Add a python class that constructs resource in the __init__
@@ -77,10 +78,24 @@ class _PrefetchToDeviceIterator(object):
 
     @function.Defun(dtypes.string)
     def _prefetch_fn(handle):
+      """Prefetches one element from `input_iterator`."""
       remote_iterator = iterator_ops.Iterator.from_string_handle(
           handle, input_iterator.output_types, input_iterator.output_shapes,
           input_iterator.output_classes)
-      return remote_iterator.get_next()
+      ret = remote_iterator.get_next()
+
+      # Convert any `SparseTensorValue`s to `SparseTensor`s.
+      ret = nest.pack_sequence_as(ret, [
+          sparse_tensor_lib.SparseTensor.from_value(t)
+          if sparse_tensor_lib.is_sparse(t) else t for t in nest.flatten(ret)
+      ])
+
+      # Serialize any sparse tensors and convert result to tensors.
+      ret = nest.pack_sequence_as(ret, [
+          ops.convert_to_tensor(t)
+          for t in nest.flatten(sparse.serialize_sparse_tensors(ret))
+      ])
+      return nest.flatten(ret)
 
     with ops.device(device):
       self._buffering_resource = function_buffering_resource(
-- 
GitLab


From 655b2663e7d2609b0f578b9ef3c1401de22dc5c2 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 3 Apr 2018 10:04:47 -0700
Subject: [PATCH 856/960] Apply "Raise exception in SWIG on bad TF_Status" to
 base.i

Minor fixes to make this work.

PiperOrigin-RevId: 191457070
---
 tensorflow/python/client/tf_session.i      | 26 ----------
 tensorflow/python/eager/backprop.py        | 11 ++---
 tensorflow/python/eager/context.py         | 56 +++++++++-------------
 tensorflow/python/eager/imperative_grad.py |  6 +--
 tensorflow/python/grappler/item.py         |  4 +-
 tensorflow/python/lib/io/tf_record.py      |  3 +-
 tensorflow/python/platform/base.i          | 22 +++++++++
 tensorflow/python/pywrap_tfe.i             |  2 +
 8 files changed, 54 insertions(+), 76 deletions(-)

diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index ee76e29c05..68768f2b4c 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -23,7 +23,6 @@ limitations under the License.
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/public/version.h"
 #include "tensorflow/python/client/tf_session_helper.h"
-#include "tensorflow/python/lib/core/py_exception_registry.h"
 
 // Helper function to convert a Python list of Tensors to a C++ vector of
 // TF_Outputs.
@@ -353,27 +352,6 @@ TF_ImportGraphDefResultsMissingUnusedInputMappings_wrapper{
       reinterpret_cast<const char*>($1.data), $1.length);
 }
 
-// Typemaps to automatically raise a Python exception from bad output TF_Status.
-// TODO(b/77295559): expand this to all TF_Status* output params and deprecate
-// raise_exception_on_not_ok_status (currently it only affects the C API).
-%typemap(in, numinputs=0) TF_Status* status (TF_Status* status) {
-  status = TF_NewStatus();
-  $1 = status;
-}
-
-%typemap(argout) TF_Status* status {
-  TF_Code code = TF_GetCode($1);
-  if (code != TF_OK) {
-    PyObject* exc = tensorflow::PyExceptionRegistry::Lookup(code);
-    // Arguments to OpError.
-    PyObject* exc_args = Py_BuildValue("sss", nullptr, nullptr, TF_Message($1));
-    TF_DeleteStatus($1);
-    SWIG_SetErrorObj(exc, exc_args);
-    SWIG_fail;
-  }
-  TF_DeleteStatus($1);
-}
-
 // Converts input Python list of wrapped TF_Outputs into a single array
 %typemap(in) (const TF_Output* inputs, int num_inputs)
     (std::vector<TF_Output> inputs) {
@@ -784,7 +762,3 @@ def TF_Reset(target, containers=None, config=None):
 %include "tensorflow/python/client/tf_session_helper.h"
 
 %unignoreall
-
-// Clear "TF_Status* status" typemap so it doesn't affect other modules and
-// unexpectedly remove the TF_Status* argument from wrappers.
-%clear TF_Status* status;
diff --git a/tensorflow/python/eager/backprop.py b/tensorflow/python/eager/backprop.py
index 209b012621..92774d4d50 100644
--- a/tensorflow/python/eager/backprop.py
+++ b/tensorflow/python/eager/backprop.py
@@ -31,7 +31,6 @@ from tensorflow.python.eager import imperative_grad
 from tensorflow.python.eager import tape
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
-from tensorflow.python.framework import errors
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
@@ -50,12 +49,10 @@ def op_attr_type(op_type, attr_name):
   try:
     return _op_attr_type_cache[(op_type, attr_name)]
   except KeyError:
-    with errors.raise_exception_on_not_ok_status() as status:
-      h = context.context()._handle  # pylint: disable=protected-access
-      attr_type = pywrap_tensorflow.TFE_OpNameGetAttrType(
-          h, op_type, attr_name, status)
-    _op_attr_type_cache[(op_type, attr_name)] = attr_type
-    return attr_type
+    h = context.context()._handle  # pylint: disable=protected-access
+    attr_type = pywrap_tensorflow.TFE_OpNameGetAttrType(h, op_type, attr_name)
+  _op_attr_type_cache[(op_type, attr_name)] = attr_type
+  return attr_type
 
 
 def make_attr(attr_type, value):
diff --git a/tensorflow/python/eager/context.py b/tensorflow/python/eager/context.py
index 99ec895b54..9e146f021e 100644
--- a/tensorflow/python/eager/context.py
+++ b/tensorflow/python/eager/context.py
@@ -28,7 +28,6 @@ from tensorflow.core.protobuf import config_pb2
 from tensorflow.python import pywrap_tensorflow
 from tensorflow.python.framework import c_api_util
 from tensorflow.python.framework import device as pydev
-from tensorflow.python.framework import errors
 from tensorflow.python.util import compat
 from tensorflow.python.util import is_in_graph_mode
 from tensorflow.python.util import tf_contextlib
@@ -224,24 +223,21 @@ class Context(object):
       assert self._context_devices is None
       opts = pywrap_tensorflow.TFE_NewContextOptions()
       try:
-        with errors.raise_exception_on_not_ok_status() as status:
-          if self._config is not None:
-            config_str = self._config.SerializeToString()
-            pywrap_tensorflow.TFE_ContextOptionsSetConfig(
-                opts, config_str, len(config_str), status)
-          if self._device_policy is not None:
-            pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy(
-                opts, self._device_policy)
-          if self._execution_mode == ASYNC:
-            pywrap_tensorflow.TFE_ContextOptionsSetAsync(opts, True)
-          self._context_handle = pywrap_tensorflow.TFE_NewContext(opts, status)
+        if self._config is not None:
+          config_str = self._config.SerializeToString()
+          pywrap_tensorflow.TFE_ContextOptionsSetConfig(opts, config_str)
+        if self._device_policy is not None:
+          pywrap_tensorflow.TFE_ContextOptionsSetDevicePlacementPolicy(
+              opts, self._device_policy)
+        if self._execution_mode == ASYNC:
+          pywrap_tensorflow.TFE_ContextOptionsSetAsync(opts, True)
+        self._context_handle = pywrap_tensorflow.TFE_NewContext(opts)
       finally:
         pywrap_tensorflow.TFE_DeleteContextOptions(opts)
       # Store list of devices
       self._context_devices = []
-      with errors.raise_exception_on_not_ok_status() as status:
-        device_list = pywrap_tensorflow.TFE_ContextListDevices(
-            self._context_handle, status)
+      device_list = pywrap_tensorflow.TFE_ContextListDevices(
+          self._context_handle)
       try:
         self._num_gpus = 0
         for i in range(pywrap_tensorflow.TF_DeviceListCount(device_list)):
@@ -412,9 +408,7 @@ class Context(object):
     if mode is None:
       mode = SYNC
     self._eager_context.execution_mode = mode
-    with errors.raise_exception_on_not_ok_status() as status:
-      pywrap_tensorflow.TFE_ContextSetAsyncForThread(self._handle,
-                                                     mode == ASYNC, status)
+    pywrap_tensorflow.TFE_ContextSetAsyncForThread(self._handle, mode == ASYNC)
 
   @tf_contextlib.contextmanager
   def execution_mode(self, mode):
@@ -428,8 +422,7 @@ class Context(object):
 
   def async_wait(self):
     """Waits for ops dispatched in ASYNC mode to finish."""
-    with errors.raise_exception_on_not_ok_status() as status:
-      pywrap_tensorflow.TFE_ContextAsyncWait(self._handle, status)
+    pywrap_tensorflow.TFE_ContextAsyncWait(self._handle)
 
   def async_clear_error(self):
     """Clears errors raised during ASYNC execution."""
@@ -449,11 +442,9 @@ class Context(object):
     Args:
       fn: A wrapped TF_Function (returned from TF_GraphToFunction_wrapper).
     """
-    with errors.raise_exception_on_not_ok_status() as status:
-      pywrap_tensorflow.TFE_ContextAddFunction(
-          self._handle,  # pylint: disable=protected-access
-          fn,
-          status)
+    pywrap_tensorflow.TFE_ContextAddFunction(
+        self._handle,  # pylint: disable=protected-access
+        fn)
 
   def add_function_def(self, fdef):
     """Add a function definition to the context.
@@ -465,12 +456,10 @@ class Context(object):
       fdef: A FunctionDef protocol buffer message.
     """
     fdef_string = fdef.SerializeToString()
-    with errors.raise_exception_on_not_ok_status() as status:
-      pywrap_tensorflow.TFE_ContextAddFunctionDef(
-          self._handle,  # pylint: disable=protected-access
-          fdef_string,
-          len(fdef_string),
-          status)
+    pywrap_tensorflow.TFE_ContextAddFunctionDef(
+        self._handle,  # pylint: disable=protected-access
+        fdef_string,
+        len(fdef_string))
 
   def add_post_execution_callback(self, callback):
     """Add a post-execution callback to the context.
@@ -545,9 +534,8 @@ class Context(object):
     if not self._context_handle:
       return None
     with c_api_util.tf_buffer() as buffer_:
-      with errors.raise_exception_on_not_ok_status() as status:
-        pywrap_tensorflow.TFE_ContextExportRunMetadata(
-            self._context_handle, buffer_, status)
+      pywrap_tensorflow.TFE_ContextExportRunMetadata(
+          self._context_handle, buffer_)
       proto_data = pywrap_tensorflow.TF_GetBuffer(buffer_)
     run_metadata = config_pb2.RunMetadata()
     run_metadata.ParseFromString(compat.as_bytes(proto_data))
diff --git a/tensorflow/python/eager/imperative_grad.py b/tensorflow/python/eager/imperative_grad.py
index 837cad974a..000152855d 100644
--- a/tensorflow/python/eager/imperative_grad.py
+++ b/tensorflow/python/eager/imperative_grad.py
@@ -21,7 +21,6 @@ from __future__ import print_function
 import collections
 
 from tensorflow.python import pywrap_tensorflow
-from tensorflow.python.framework import errors
 
 
 VSpace = collections.namedtuple(
@@ -60,6 +59,5 @@ def imperative_grad(
      or if only non-differentiable functions of the source were used in the
      computation of target.
   """
-  with errors.raise_exception_on_not_ok_status() as status:
-    return pywrap_tensorflow.TFE_Py_TapeGradient(
-        tape._tape, vspace, target, sources, output_gradients, status)  # pylint: disable=protected-access
+  return pywrap_tensorflow.TFE_Py_TapeGradient(
+      tape._tape, vspace, target, sources, output_gradients)  # pylint: disable=protected-access
diff --git a/tensorflow/python/grappler/item.py b/tensorflow/python/grappler/item.py
index 4a083849bd..1748efdd13 100644
--- a/tensorflow/python/grappler/item.py
+++ b/tensorflow/python/grappler/item.py
@@ -51,9 +51,7 @@ class Item(object):
     self._BuildTFItem()
 
   def IdentifyImportantOps(self, sort_topologically=False):
-    with errors.raise_exception_on_not_ok_status() as status:
-      return tf_item.TF_IdentifyImportantOps(self.tf_item, sort_topologically,
-                                             status)
+    return tf_item.TF_IdentifyImportantOps(self.tf_item, sort_topologically)
 
   def GetOpProperties(self):
     ret_from_swig = tf_item.TF_GetOpProperties(self.tf_item)
diff --git a/tensorflow/python/lib/io/tf_record.py b/tensorflow/python/lib/io/tf_record.py
index 6fcf9c91d8..bf2d6f68b5 100644
--- a/tensorflow/python/lib/io/tf_record.py
+++ b/tensorflow/python/lib/io/tf_record.py
@@ -78,8 +78,7 @@ def tf_record_iterator(path, options=None):
   try:
     while True:
       try:
-        with errors.raise_exception_on_not_ok_status() as status:
-          reader.GetNext(status)
+        reader.GetNext()
       except errors.OutOfRangeError:
         break
       yield reader.record()
diff --git a/tensorflow/python/platform/base.i b/tensorflow/python/platform/base.i
index dbefca2be9..478dd46f7e 100644
--- a/tensorflow/python/platform/base.i
+++ b/tensorflow/python/platform/base.i
@@ -229,3 +229,25 @@ _COPY_TYPEMAPS(unsigned int, mode_t);
 %define final %enddef
 %define override %enddef
 #endif
+
+// Typemaps to automatically raise a Python exception from bad output TF_Status.
+// TODO(b/77295559): expand this to all TF_Status* output params and deprecate
+// raise_exception_on_not_ok_status (currently it only affects the C API).
+%typemap(in, numinputs=0) TF_Status* status (TF_Status* status) {
+  $1 = TF_NewStatus();
+}
+
+%typemap(freearg) (TF_Status* status) {
+ TF_DeleteStatus($1);
+}
+
+%typemap(argout) TF_Status* status {
+  TF_Code code = TF_GetCode($1);
+  if (code != TF_OK) {
+    PyObject* exc = tensorflow::PyExceptionRegistry::Lookup(code);
+    // Arguments to OpError.
+    PyObject* exc_args = Py_BuildValue("sss", nullptr, nullptr, TF_Message($1));
+    SWIG_SetErrorObj(exc, exc_args);
+    SWIG_fail;
+  }
+}
diff --git a/tensorflow/python/pywrap_tfe.i b/tensorflow/python/pywrap_tfe.i
index 39fabb9c1b..7acb8eeb1a 100644
--- a/tensorflow/python/pywrap_tfe.i
+++ b/tensorflow/python/pywrap_tfe.i
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+%include "tensorflow/python/platform/base.i"
+
 %ignore "";
 
 %rename("%s") TFE_NewContext;
-- 
GitLab


From 63a3be25482121e34ad3ac399e1ece4b24656318 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Tue, 3 Apr 2018 10:06:32 -0700
Subject: [PATCH 857/960] Don't call context function either in the fastpath.

Also add a slowpath benchmark for identity.

PiperOrigin-RevId: 191457407
---
 tensorflow/python/eager/benchmarks_test.py     |  3 +++
 tensorflow/python/eager/python_eager_op_gen.cc | 11 +++++++----
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/eager/benchmarks_test.py b/tensorflow/python/eager/benchmarks_test.py
index 9ca5041c38..7ad37058fd 100644
--- a/tensorflow/python/eager/benchmarks_test.py
+++ b/tensorflow/python/eager/benchmarks_test.py
@@ -201,6 +201,9 @@ class MicroBenchmarks(test.Benchmark):
     m = self._m_2
     self._run(lambda: gen_array_ops.identity(m), 30000)
 
+  def benchmark_slowpath_tf_identity(self):
+    self._run(lambda: gen_array_ops.identity(1), 30000)
+
   def benchmark_tfe_py_execute_identity(self):
     m = self._m_2
     ctx_handle = context.context()._handle
diff --git a/tensorflow/python/eager/python_eager_op_gen.cc b/tensorflow/python/eager/python_eager_op_gen.cc
index 15d20bdd1a..9afab0077b 100644
--- a/tensorflow/python/eager/python_eager_op_gen.cc
+++ b/tensorflow/python/eager/python_eager_op_gen.cc
@@ -366,8 +366,8 @@ string GenEagerPythonOp::Code() {
 void GenEagerPythonOp::HandleGraphMode(const string& function_setup) {
   // Handle graph-mode case
   strings::StrAppend(&result_,
-                     "  _ctx = _context.context()\n"
-                     "  if not _ctx._eager_context.is_eager:\n",
+                     "  _ctx = _context._context\n"
+                     "  if _ctx is None or not _ctx._eager_context.is_eager:\n",
                      function_setup,
                      "    _, _, _op = _op_def_lib._apply_op_helper(\n");
   AddBodyNoReturn("        ");
@@ -683,13 +683,14 @@ bool GenEagerPythonOp::AddEagerFallbackCode(
     return true;
   }
 
-  AddDefLine(strings::StrCat(function_name_, kEagerFallbackSuffix), parameters);
+  AddDefLine(strings::StrCat(function_name_, kEagerFallbackSuffix),
+             strings::StrCat(parameters, ", ctx=None"));
   strings::StrAppend(
       &result_, "  r\"\"\"This is the slowpath function for Eager mode.\n");
   strings::StrAppend(&result_, "  This is for function ", function_name_,
                      "\n  \"\"\"\n");
 
-  strings::StrAppend(&result_, "  _ctx = _context.context()\n");
+  strings::StrAppend(&result_, "  _ctx = ctx if ctx else _context.context()\n");
 
   string function_setup;
   if (!GetEagerFunctionSetup("  ", &function_setup)) {
@@ -755,6 +756,8 @@ void GenEagerPythonOp::AddEagerFastPathExecute() {
   strings::StrAppend(&result_, "      ", "return _result\n");
 
   // Handle fallback.
+  if (!fallback_params.empty()) strings::StrAppend(&fallback_params, ", ");
+  strings::StrAppend(&fallback_params, "ctx=_ctx");
   strings::StrAppend(&result_, "    ", "except _core._FallbackException:\n");
   strings::StrAppend(
       &result_, "      ", "return ", function_name_, kEagerFallbackSuffix,
-- 
GitLab


From 1846881d8010d41e7c5a866bd2bb1b01ec220a13 Mon Sep 17 00:00:00 2001
From: Mark Daoust <markdaoust@google.com>
Date: Tue, 3 Apr 2018 10:17:33 -0700
Subject: [PATCH 858/960] Use r1.7 version.

---
 tensorflow/docs_src/get_started/index.md | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tensorflow/docs_src/get_started/index.md b/tensorflow/docs_src/get_started/index.md
index e09d2dcbde..b28cb9df75 100644
--- a/tensorflow/docs_src/get_started/index.md
+++ b/tensorflow/docs_src/get_started/index.md
@@ -18,12 +18,10 @@ TensorFlow provides many APIs. The remainder of this section focuses on the
 Estimator API which provide scalable, high-performance models.
 To get started with Estimators begin by reading one of the following documents:
 
-  * @{$get_started/eager} is for machine learning beginners and uses
-    @{$programmers_guide/eager}.
-  * @{$get_started/get_started_for_beginners} is also for machine learning
-    beginners and uses @{$programmers_guide/graphs}.
-  * @{$get_started/premade_estimators} assumes some machine learning background
-    and uses an @{tf.estimator.Estimator$Estimator}.
+  * @{$get_started/get_started_for_beginners}, which is aimed at readers
+    new to machine learning.
+  * @{$get_started/premade_estimators}, which is aimed at readers who have
+    experience in machine learning.
 
 Then, read the following documents, which demonstrate the key features
 in the high-level APIs:
-- 
GitLab


From f005999d571536b859229229e4487cf749d9a786 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 10:32:57 -0700
Subject: [PATCH 859/960] Add 8bit strided slice op to tflite

PiperOrigin-RevId: 191461696
---
 .../contrib/lite/kernels/strided_slice.cc     |   5 +
 .../lite/kernels/strided_slice_test.cc        | 113 ++++++++++--------
 2 files changed, 69 insertions(+), 49 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/strided_slice.cc b/tensorflow/contrib/lite/kernels/strided_slice.cc
index eb374d9031..e6d5c300dc 100644
--- a/tensorflow/contrib/lite/kernels/strided_slice.cc
+++ b/tensorflow/contrib/lite/kernels/strided_slice.cc
@@ -228,6 +228,11 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
         TF_LITE_STRIDED_SLICE(reference_ops, int64_t);
       }
       break;
+    case kTfLiteUInt8:
+      if (kernel_type == kReference) {
+        TF_LITE_STRIDED_SLICE(reference_ops, uint8_t);
+      }
+      break;
     default:
       context->ReportError(context,
                            "Type is currently not supported "
diff --git a/tensorflow/contrib/lite/kernels/strided_slice_test.cc b/tensorflow/contrib/lite/kernels/strided_slice_test.cc
index 5c98c5f431..22d7b097cb 100644
--- a/tensorflow/contrib/lite/kernels/strided_slice_test.cc
+++ b/tensorflow/contrib/lite/kernels/strided_slice_test.cc
@@ -24,6 +24,8 @@ namespace {
 using ::int32;
 using ::testing::ElementsAreArray;
 
+template <typename input_type = float,
+          TensorType tensor_input_type = TensorType_FLOAT32>
 class StridedSliceOpModel : public SingleOpModel {
  public:
   StridedSliceOpModel(std::initializer_list<int> input_shape,
@@ -32,11 +34,11 @@ class StridedSliceOpModel : public SingleOpModel {
                       std::initializer_list<int> strides_shape, int begin_mask,
                       int end_mask, int ellipsis_mask, int new_axis_mask,
                       int shrink_axis_mask) {
-    input_ = AddInput(TensorType_FLOAT32);
+    input_ = AddInput(tensor_input_type);
     begin_ = AddInput(TensorType_INT32);
     end_ = AddInput(TensorType_INT32);
     strides_ = AddInput(TensorType_INT32);
-    output_ = AddOutput(TensorType_FLOAT32);
+    output_ = AddOutput(tensor_input_type);
     SetBuiltinOp(
         BuiltinOperator_STRIDED_SLICE, BuiltinOptions_StridedSliceOptions,
         CreateStridedSliceOptions(builder_, begin_mask, end_mask, ellipsis_mask,
@@ -45,8 +47,8 @@ class StridedSliceOpModel : public SingleOpModel {
     BuildInterpreter({input_shape, begin_shape, end_shape, strides_shape});
   }
 
-  void SetInput(std::initializer_list<float> data) {
-    PopulateTensor<float>(input_, data);
+  void SetInput(std::initializer_list<input_type> data) {
+    PopulateTensor<input_type>(input_, data);
   }
   void SetBegin(std::initializer_list<int32> data) {
     PopulateTensor<int32>(begin_, data);
@@ -58,7 +60,9 @@ class StridedSliceOpModel : public SingleOpModel {
     PopulateTensor<int32>(strides_, data);
   }
 
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  std::vector<input_type> GetOutput() {
+    return ExtractVector<input_type>(output_);
+  }
   std::vector<int> GetOutputShape() { return GetTensorShape(output_); }
 
  private:
@@ -71,19 +75,19 @@ class StridedSliceOpModel : public SingleOpModel {
 
 TEST(StridedSliceOpTest, UnsupportedInputSize) {
   EXPECT_DEATH(
-      StridedSliceOpModel({2, 2, 2, 2, 2}, {5}, {5}, {5}, 0, 0, 0, 0, 0),
+      StridedSliceOpModel<>({2, 2, 2, 2, 2}, {5}, {5}, {5}, 0, 0, 0, 0, 0),
       "StridedSlice op only supports 1D-4D input arrays.");
 }
 
 TEST(StridedSliceOpTest, UnssupportedArgs) {
-  EXPECT_DEATH(StridedSliceOpModel({3, 2}, {2}, {2}, {2}, 0, 0, 1, 0, 0),
+  EXPECT_DEATH(StridedSliceOpModel<>({3, 2}, {2}, {2}, {2}, 0, 0, 1, 0, 0),
                "ellipsis_mask is not implemented yet.");
-  EXPECT_DEATH(StridedSliceOpModel({3, 2}, {2}, {2}, {2}, 0, 0, 0, 1, 0),
+  EXPECT_DEATH(StridedSliceOpModel<>({3, 2}, {2}, {2}, {2}, 0, 0, 0, 1, 0),
                "new_axis_mask is not implemented yet.");
 }
 
 TEST(StridedSliceOpTest, In1D) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({3});
@@ -94,7 +98,7 @@ TEST(StridedSliceOpTest, In1D) {
 }
 
 TEST(StridedSliceOpTest, In1D_EmptyOutput) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({10});
   m.SetEnd({3});
@@ -104,7 +108,7 @@ TEST(StridedSliceOpTest, In1D_EmptyOutput) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegativeBegin) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-3});
   m.SetEnd({3});
@@ -115,7 +119,7 @@ TEST(StridedSliceOpTest, In1D_NegativeBegin) {
 }
 
 TEST(StridedSliceOpTest, In1D_OutOfRangeBegin) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-5});
   m.SetEnd({3});
@@ -126,7 +130,7 @@ TEST(StridedSliceOpTest, In1D_OutOfRangeBegin) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegativeEnd) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({-2});
@@ -137,7 +141,7 @@ TEST(StridedSliceOpTest, In1D_NegativeEnd) {
 }
 
 TEST(StridedSliceOpTest, In1D_OutOfRangeEnd) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-3});
   m.SetEnd({5});
@@ -148,7 +152,7 @@ TEST(StridedSliceOpTest, In1D_OutOfRangeEnd) {
 }
 
 TEST(StridedSliceOpTest, In1D_BeginMask) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 1, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 1, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({3});
@@ -159,7 +163,7 @@ TEST(StridedSliceOpTest, In1D_BeginMask) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegativeBeginNegativeStride) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-2});
   m.SetEnd({-3});
@@ -170,7 +174,7 @@ TEST(StridedSliceOpTest, In1D_NegativeBeginNegativeStride) {
 }
 
 TEST(StridedSliceOpTest, In1D_OutOfRangeBeginNegativeStride) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({5});
   m.SetEnd({2});
@@ -181,7 +185,7 @@ TEST(StridedSliceOpTest, In1D_OutOfRangeBeginNegativeStride) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegativeEndNegativeStride) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({2});
   m.SetEnd({-4});
@@ -192,7 +196,7 @@ TEST(StridedSliceOpTest, In1D_NegativeEndNegativeStride) {
 }
 
 TEST(StridedSliceOpTest, In1D_OutOfRangeEndNegativeStride) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-3});
   m.SetEnd({-5});
@@ -203,7 +207,7 @@ TEST(StridedSliceOpTest, In1D_OutOfRangeEndNegativeStride) {
 }
 
 TEST(StridedSliceOpTest, In1D_EndMask) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 1, 0, 0, 0);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 1, 0, 0, 0);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({3});
@@ -214,7 +218,7 @@ TEST(StridedSliceOpTest, In1D_EndMask) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegStride) {
-  StridedSliceOpModel m({3}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({3}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3});
   m.SetBegin({-1});
   m.SetEnd({-4});
@@ -225,7 +229,7 @@ TEST(StridedSliceOpTest, In1D_NegStride) {
 }
 
 TEST(StridedSliceOpTest, In1D_EvenLenStride2) {
-  StridedSliceOpModel m({2}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2});
   m.SetBegin({0});
   m.SetEnd({2});
@@ -236,7 +240,7 @@ TEST(StridedSliceOpTest, In1D_EvenLenStride2) {
 }
 
 TEST(StridedSliceOpTest, In1D_OddLenStride2) {
-  StridedSliceOpModel m({3}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({3}, {1}, {1}, {1}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3});
   m.SetBegin({0});
   m.SetEnd({3});
@@ -247,7 +251,7 @@ TEST(StridedSliceOpTest, In1D_OddLenStride2) {
 }
 
 TEST(StridedSliceOpTest, In2D_Identity) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({0, 0});
   m.SetEnd({2, 3});
@@ -258,7 +262,7 @@ TEST(StridedSliceOpTest, In2D_Identity) {
 }
 
 TEST(StridedSliceOpTest, In2D) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, 0});
   m.SetEnd({2, 2});
@@ -269,7 +273,7 @@ TEST(StridedSliceOpTest, In2D) {
 }
 
 TEST(StridedSliceOpTest, In2D_Stride2) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({0, 0});
   m.SetEnd({2, 3});
@@ -280,7 +284,7 @@ TEST(StridedSliceOpTest, In2D_Stride2) {
 }
 
 TEST(StridedSliceOpTest, In2D_NegStride) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, -1});
   m.SetEnd({2, -4});
@@ -291,7 +295,7 @@ TEST(StridedSliceOpTest, In2D_NegStride) {
 }
 
 TEST(StridedSliceOpTest, In2D_BeginMask) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 1, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 1, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, 0});
   m.SetEnd({2, 2});
@@ -302,7 +306,7 @@ TEST(StridedSliceOpTest, In2D_BeginMask) {
 }
 
 TEST(StridedSliceOpTest, In2D_EndMask) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 2, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 2, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, 0});
   m.SetEnd({2, 2});
@@ -313,7 +317,7 @@ TEST(StridedSliceOpTest, In2D_EndMask) {
 }
 
 TEST(StridedSliceOpTest, In2D_NegStrideBeginMask) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 2, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 2, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, -2});
   m.SetEnd({2, -4});
@@ -324,7 +328,7 @@ TEST(StridedSliceOpTest, In2D_NegStrideBeginMask) {
 }
 
 TEST(StridedSliceOpTest, In2D_NegStrideEndMask) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 2, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 2, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({1, -2});
   m.SetEnd({2, -3});
@@ -335,7 +339,7 @@ TEST(StridedSliceOpTest, In2D_NegStrideEndMask) {
 }
 
 TEST(StridedSliceOpTest, In3D_Identity) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -347,7 +351,7 @@ TEST(StridedSliceOpTest, In3D_Identity) {
 }
 
 TEST(StridedSliceOpTest, In3D_NegStride) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({-1, -1, -1});
   m.SetEnd({-3, -4, -3});
@@ -359,7 +363,7 @@ TEST(StridedSliceOpTest, In3D_NegStride) {
 }
 
 TEST(StridedSliceOpTest, In3D_Strided2) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 0);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -370,7 +374,7 @@ TEST(StridedSliceOpTest, In3D_Strided2) {
 }
 
 TEST(StridedSliceOpTest, In1D_ShrinkAxisMask1) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({3});
@@ -381,7 +385,7 @@ TEST(StridedSliceOpTest, In1D_ShrinkAxisMask1) {
 }
 
 TEST(StridedSliceOpTest, In1D_EmptyOutputShrinkAxisMask1) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({2});
   m.SetEnd({1});
@@ -392,7 +396,7 @@ TEST(StridedSliceOpTest, In1D_EmptyOutputShrinkAxisMask1) {
 }
 
 TEST(StridedSliceOpTest, In1D_BeginMaskShrinkAxisMask1) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 1, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 1, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({1});
   m.SetEnd({3});
@@ -403,7 +407,7 @@ TEST(StridedSliceOpTest, In1D_BeginMaskShrinkAxisMask1) {
 }
 
 TEST(StridedSliceOpTest, In1D_NegativeBeginNegativeStrideShrinkAxisMask1) {
-  StridedSliceOpModel m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({4}, {1}, {1}, {1}, 0, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4});
   m.SetBegin({-2});
   m.SetEnd({-3});
@@ -414,7 +418,7 @@ TEST(StridedSliceOpTest, In1D_NegativeBeginNegativeStrideShrinkAxisMask1) {
 }
 
 TEST(StridedSliceOpTest, In2D_ShrinkAxisMask1) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({0, 0});
   m.SetEnd({2, 3});
@@ -425,7 +429,7 @@ TEST(StridedSliceOpTest, In2D_ShrinkAxisMask1) {
 }
 
 TEST(StridedSliceOpTest, In2D_ShrinkAxisMask2) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 2);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 2);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({0, 0});
   m.SetEnd({2, 3});
@@ -436,7 +440,7 @@ TEST(StridedSliceOpTest, In2D_ShrinkAxisMask2) {
 }
 
 TEST(StridedSliceOpTest, In2D_ShrinkAxisMask3) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 3);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 0, 0, 0, 0, 3);
   m.SetInput({1, 2, 3, 4, 5, 6});
   m.SetBegin({0, 0});
   m.SetEnd({2, 3});
@@ -447,7 +451,7 @@ TEST(StridedSliceOpTest, In2D_ShrinkAxisMask3) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis1) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 1);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 1);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -458,7 +462,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis1) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis2) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 2);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 2);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -469,7 +473,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis2) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis3) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 3);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 3);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -480,7 +484,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis3) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis4) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 4);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 4);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -491,7 +495,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis4) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis5) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 5);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 5);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -502,7 +506,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis5) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis6) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 6);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 6);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -513,7 +517,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis6) {
 }
 
 TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis7) {
-  StridedSliceOpModel m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 7);
+  StridedSliceOpModel<> m({2, 3, 2}, {3}, {3}, {3}, 0, 0, 0, 0, 7);
   m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
   m.SetBegin({0, 0, 0});
   m.SetEnd({2, 3, 2});
@@ -525,7 +529,7 @@ TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis7) {
 
 // This tests catches a very subtle bug that was fixed by cl/188403234.
 TEST(StridedSliceOpTest, RunTwice) {
-  StridedSliceOpModel m({2, 3}, {2}, {2}, {2}, 1, 0, 0, 0, 0);
+  StridedSliceOpModel<> m({2, 3}, {2}, {2}, {2}, 1, 0, 0, 0, 0);
 
   auto setup_inputs = [&m]() {
     m.SetInput({1, 2, 3, 4, 5, 6});
@@ -544,6 +548,17 @@ TEST(StridedSliceOpTest, RunTwice) {
   EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 2, 4, 5}));
 }
 
+TEST(StridedSliceOpTest, In3D_IdentityShrinkAxis1Uint8) {
+  StridedSliceOpModel<uint8, TensorType_UINT8> m({2, 3, 2}, {3}, {3}, {3}, 0, 0,
+                                                 0, 0, 1);
+  m.SetInput({1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12});
+  m.SetBegin({0, 0, 0});
+  m.SetEnd({2, 3, 2});
+  m.SetStrides({1, 1, 1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutputShape(), ElementsAreArray({3, 2}));
+  EXPECT_THAT(m.GetOutput(), ElementsAreArray({1, 2, 3, 4, 5, 6}));
+}
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From 895566e3af6e3e44eb22d0c2d20b4890e42982fc Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 10:35:00 -0700
Subject: [PATCH 860/960] Three operators are added that can be used to
 decrease the number of transpose/reshape occurences. These operators are: -
 Swap elementwise - Swap reshape-transpose - Fuse transpose-reshape

Swap elementwise groups operations that operate on value, and move values. This
allows all movement operations (reshape, transpose, etc) to group at one
portion of the graph while value manipulations group on the other end.

Swap reshape/transpose finds cases where the reshape-transpose can be reorderd.
This allows the reshape-transpose-reshape case to be transformed to
transpose-reshape-reshape, which can then merge the reshape-reshape.

Finally the transpose-reshape merge allows cases where, when reshape maintains
the same dimensions and does not affect memory ordering, the transpose and
reshape can be merged into a single transpose operator.

PiperOrigin-RevId: 191462038
---
 tensorflow/contrib/lite/toco/BUILD            |   4 +-
 .../graph_transformations.h                   |   4 +-
 .../merge_reshape_into_preceding_transpose.cc | 190 ++++++++++++++
 .../reorder_activation_functions.cc           | 137 ----------
 .../reorder_elementwise_unary.cc              | 153 +++++++++++
 .../reorder_reshape_transpose.cc              | 248 ++++++++++++++++++
 .../resolve_tensorflow_matmul.cc              |  14 +-
 tensorflow/contrib/lite/toco/toco_tooling.cc  |   4 +-
 tensorflow/contrib/lite/toco/tooling_util.cc  |  29 ++
 tensorflow/contrib/lite/toco/tooling_util.h   |  13 +
 10 files changed, 649 insertions(+), 147 deletions(-)
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/merge_reshape_into_preceding_transpose.cc
 delete mode 100644 tensorflow/contrib/lite/toco/graph_transformations/reorder_activation_functions.cc
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/reorder_elementwise_unary.cc
 create mode 100644 tensorflow/contrib/lite/toco/graph_transformations/reorder_reshape_transpose.cc

diff --git a/tensorflow/contrib/lite/toco/BUILD b/tensorflow/contrib/lite/toco/BUILD
index 2dd689ad4c..8a35fb9034 100644
--- a/tensorflow/contrib/lite/toco/BUILD
+++ b/tensorflow/contrib/lite/toco/BUILD
@@ -234,6 +234,7 @@ cc_library(
         "graph_transformations/identify_relu1.cc",
         "graph_transformations/lstm_utils.cc",
         "graph_transformations/make_initial_dequantize_operator.cc",
+        "graph_transformations/merge_reshape_into_preceding_transpose.cc",
         "graph_transformations/propagate_activation_function_into_constants.cc",
         "graph_transformations/propagate_array_data_types.cc",
         "graph_transformations/propagate_fixed_sizes.cc",
@@ -251,7 +252,8 @@ cc_library(
         "graph_transformations/remove_trivial_reshape.cc",
         "graph_transformations/remove_trivial_slice.cc",
         "graph_transformations/remove_unused_op.cc",
-        "graph_transformations/reorder_activation_functions.cc",
+        "graph_transformations/reorder_elementwise_unary.cc",
+        "graph_transformations/reorder_reshape_transpose.cc",
         "graph_transformations/resolve_batch_normalization.cc",
         "graph_transformations/resolve_batch_to_space_nd_attributes.cc",
         "graph_transformations/resolve_constant_binary.cc",
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
index 76ec02aa07..27c5044bb3 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
+++ b/tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h
@@ -128,6 +128,7 @@ DECLARE_GRAPH_TRANSFORMATION(IdentifyL2Pool)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyLstmCell)
 DECLARE_GRAPH_TRANSFORMATION(SplitLstmCellInputs)
 DECLARE_GRAPH_TRANSFORMATION(MergeLstmCellInputs)
+DECLARE_GRAPH_TRANSFORMATION(MergeReshapeIntoPrecedingTranspose)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyRelu1)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyPRelu)
 DECLARE_GRAPH_TRANSFORMATION(IdentifyDilatedConv)
@@ -152,7 +153,8 @@ DECLARE_GRAPH_TRANSFORMATION(ResolveConstantUnaryOperator)
 DECLARE_GRAPH_TRANSFORMATION(CreateIm2colArrays)
 DECLARE_GRAPH_TRANSFORMATION(DropIm2colArrays)
 DECLARE_GRAPH_TRANSFORMATION(ReadFakeQuantMinMax)
-DECLARE_GRAPH_TRANSFORMATION(ReorderActivationFunctions)
+DECLARE_GRAPH_TRANSFORMATION(ReorderElementwiseUnary)
+DECLARE_GRAPH_TRANSFORMATION(ReorderReshapeTranspose)
 DECLARE_GRAPH_TRANSFORMATION(ResolveReorderAxes)
 DECLARE_GRAPH_TRANSFORMATION(ResolveTensorFlowConcat)
 DECLARE_GRAPH_TRANSFORMATION(ResolveTensorFlowMatMul)
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/merge_reshape_into_preceding_transpose.cc b/tensorflow/contrib/lite/toco/graph_transformations/merge_reshape_into_preceding_transpose.cc
new file mode 100644
index 0000000000..5065004093
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/merge_reshape_into_preceding_transpose.cc
@@ -0,0 +1,190 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <algorithm>
+#include <memory>
+#include <string>
+#include <unordered_map>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/graph_transformations/remove_trivial_passthrough.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/runtime/types.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+namespace {
+
+bool OperatorReady(const Model& model, const Operator* op) {
+  if (!model.HasArray(op->inputs[0]) || !model.HasArray(op->inputs[1]) ||
+      !model.HasArray(op->outputs[0])) {
+    // Arrays are missing.
+    return false;
+  }
+
+  if (!model.GetArray(op->inputs[0]).has_shape() ||
+      !model.GetArray(op->outputs[0]).has_shape()) {
+    // Input and output needs the shape.
+    return false;
+  }
+
+  if (!model.GetArray(op->inputs[1]).buffer) {
+    // Buffer needs to be a constant.
+    return false;
+  }
+
+  return true;
+}
+
+// Returns whether the reshape could be a transpose.
+std::vector<int32> ReshapeToTranspose(const Model& model,
+                                      const TensorFlowReshapeOperator* op) {
+  CHECK(!op->shape.empty());
+  CHECK(model.HasArray(op->inputs[0]));
+  CHECK(model.HasArray(op->outputs[0]));
+
+  const auto& input_array = model.GetArray(op->inputs[0]);
+  const auto& output_array = model.GetArray(op->outputs[0]);
+
+  CHECK(input_array.has_shape());
+  CHECK(output_array.has_shape());
+
+  std::vector<int> in_shape = input_array.shape().dims();
+  std::vector<int> out_shape = output_array.shape().dims();
+
+  std::vector<int> one_indices;
+  std::vector<int> not_one_indices;
+
+  // Separate into one indices and not one indices.
+  for (int i = 0; i < in_shape.size(); i++) {
+    if (in_shape[i] == 1) {
+      one_indices.push_back(i);
+    } else {
+      not_one_indices.push_back(i);
+    }
+  }
+
+  // Reorder the vertices.
+  std::vector<int> perm;
+  perm.reserve(in_shape.size());
+  int one_index = 0;
+  int not_one_index = 0;
+  for (const auto val : out_shape) {
+    if (val == 1) {
+      perm.push_back(one_indices[one_index]);
+      one_index++;
+    } else {
+      perm.push_back(not_one_indices[not_one_index]);
+      not_one_index++;
+    }
+  }
+
+  return perm;
+}
+
+}  // namespace
+
+// When a transpose is fed into a reshape, it is possible for the two operators
+// to be merged if the reshape does not affect memory ordering and does not
+// affects the number of dimensions. This only occurs when only unary dimensions
+// are shifting position.
+bool MergeReshapeIntoPrecedingTranspose::Run(Model* model,
+                                             std::size_t op_index) {
+  auto it = model->operators.begin() + op_index;
+  auto* reshape_op = ConvertOperator<TensorFlowReshapeOperator*>(
+      it->get(), OperatorType::kTensorFlowReshape);
+
+  if (reshape_op == nullptr) {
+    return false;
+  }
+
+  if (!OperatorReady(*model, reshape_op) || reshape_op->shape.empty()) {
+    return false;
+  }
+
+  const string intermediate_name = reshape_op->inputs[0];
+  const string output_name = reshape_op->outputs[0];
+
+  // Guarantee the input is only consume by the reshape.
+  if (CountOpsWithInput(*model, intermediate_name) != 1) {
+    return false;
+  }
+
+  // Check for the parent operator.
+  const auto& transpose_it = FindOpWithOutput(*model, intermediate_name);
+  if (transpose_it == model->operators.end()) {
+    return false;
+  }
+
+  // Find the parent operator and guarantee it is a transpose.
+  TransposeOperator* transpose_op = ConvertOperator<TransposeOperator*>(
+      transpose_it->get(), OperatorType::kTranspose);
+
+  if (transpose_op == nullptr) {
+    return false;
+  }
+
+  if (!OperatorReady(*model, transpose_op) || transpose_op->perm.empty()) {
+    return false;
+  }
+
+  if (!ReshapeIsEquivalentToTranspose(*model, reshape_op,
+                                      false /*allow_extra_unary_dimensions*/)) {
+    return false;
+  }
+
+  // Check that the intermediate is not an output array.
+  if (!IsDiscardableArray(*model, intermediate_name)) {
+    AddMessageF(
+        "Cannot fuse %s and %s as it would invalidate the transpose "
+        "output array.",
+        LogName(*transpose_op), LogName(*reshape_op));
+    return false;
+  }
+
+  AddMessageF("Merging operations %s and %s", LogName(*transpose_op),
+              LogName(*reshape_op));
+
+  // const auto& intermediate_array = model->GetArray(intermediate_name);
+  // const auto& output_array = model->GetArray(output_name);
+
+  auto merged_perm = ReshapeToTranspose(*model, reshape_op);
+
+  // Combine the permutations.
+  const auto& transpose_perm = transpose_op->perm;
+  for (int i = 0; i < merged_perm.size(); i++) {
+    merged_perm[i] = transpose_perm[merged_perm[i]];
+  }
+
+  // Remove the reshape as passthrough operation.
+  if (!RemoveTrivialPassthroughOp(this, model, op_index)) {
+    return false;
+  }
+
+  // Update transpose_op's constant buffer to contain the new permutation.
+  model->GetArray(transpose_op->inputs[1])
+      .GetMutableBuffer<ArrayDataType::kInt32>()
+      .data = merged_perm;
+  transpose_op->perm = merged_perm;
+
+  // transpose_ops's shape will likely has changed.
+  model->GetArray(transpose_op->outputs[0]).clear_shape();
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/reorder_activation_functions.cc b/tensorflow/contrib/lite/toco/graph_transformations/reorder_activation_functions.cc
deleted file mode 100644
index 9852c86c21..0000000000
--- a/tensorflow/contrib/lite/toco/graph_transformations/reorder_activation_functions.cc
+++ /dev/null
@@ -1,137 +0,0 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-#include <memory>
-#include <string>
-#include <unordered_map>
-#include <vector>
-
-#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
-#include "tensorflow/contrib/lite/toco/model.h"
-#include "tensorflow/contrib/lite/toco/runtime/types.h"
-#include "tensorflow/contrib/lite/toco/tooling_util.h"
-#include "tensorflow/core/platform/logging.h"
-
-namespace toco {
-
-bool ReorderActivationFunctions::Run(Model* model, std::size_t op_index) {
-  const auto ac_it = model->operators.begin() + op_index;
-  std::unique_ptr<Operator>& ac_op = *ac_it;
-  DCHECK(ac_op);
-
-  if (ac_op->type != OperatorType::kRelu6 &&
-      ac_op->type != OperatorType::kRelu1 &&
-      ac_op->type != OperatorType::kRelu) {
-    return false;
-  }
-
-  auto exchange_it = FindOpWithOutput(*model, ac_op->inputs[0]);
-  if (exchange_it == model->operators.end()) return false;
-  // Find the op producing the array passed to this activation function
-  std::unique_ptr<Operator>& exchange_op = *exchange_it;
-  DCHECK(exchange_op);
-
-  // Allow activation functions to move up over any operator that does not
-  // change the values.
-  switch (exchange_op->type) {
-    case OperatorType::kExpandDims:
-    case OperatorType::kSqueeze:
-    case OperatorType::kTensorFlowReshape:
-    case OperatorType::kTranspose:
-      break;
-    default:
-      return false;
-  }
-
-  DCHECK_EQ(exchange_op->outputs[0], ac_op->inputs[0]);
-  const auto exchange_op_input = exchange_op->inputs[0];
-  const auto intermediate_array = exchange_op->outputs[0];
-  const auto ac_op_output = ac_op->outputs[0];
-
-  int count_ops_consuming_output =
-      CountOpsWithInput(*model, intermediate_array);
-  DCHECK_GE(count_ops_consuming_output, 1);
-  if (count_ops_consuming_output > 1) {
-    AddMessageF(
-        "Not exchanging activation function with %s because it is consumed by "
-        "more than 1 other operator",
-        LogName(*exchange_op));
-    return false;
-  }
-
-  // If the ac_op was originally producing an output_array we can't trivially
-  // reorder as otherwise the output array name would change and break
-  // downstream assumptions. To work around that we perform some renaming below
-  // in that case at the cost of a bit more confusing array names in this rare
-  // case.
-  bool is_ac_op_output =
-      std::find(model->flags.output_arrays().begin(),
-                model->flags.output_arrays().end(),
-                ac_op_output) != model->flags.output_arrays().end();
-  if (is_ac_op_output) {
-    // To preserve the output array name of the activation function we need to
-    // create a temporary to use to pass between ac->ex.
-    //
-    // Original:
-    //  (a) -> EX -> (b) -> AC -> (c)
-    // Now:
-    //  (a) -> AC -> (c') -> EX -> (c)
-    AddMessageF(
-        "Exchanging activation function %s with %s but renaming to preserve "
-        "output array %s",
-        LogName(*ac_op), LogName(*exchange_op), ac_op->outputs[0]);
-
-    auto renamed_ac_op_output =
-        AvailableArrayName(*model, ac_op_output + "_exchange");
-    ac_op->inputs[0] = exchange_op_input;
-    ac_op->outputs[0] = renamed_ac_op_output;
-    model->EraseArray(exchange_op->outputs[0]);
-    exchange_op->inputs[0] = renamed_ac_op_output;
-    exchange_op->outputs[0] = ac_op_output;
-  } else {
-    // Simply swap the order and update consumers to use the exchange_op output
-    // array (b).
-    //
-    // Original:
-    //  (a) -> EX -> (b) -> AC -> (c)
-    // Now:
-    //  (a) -> AC -> (c) -> EX -> (b)
-    AddMessageF("Exchanging activation function %s with %s", LogName(*ac_op),
-                LogName(*exchange_op));
-
-    Operator* consumer = GetFirstOpWithInput(*model, ac_op_output);
-    while (consumer) {
-      for (int i = 0; i < consumer->inputs.size(); ++i) {
-        if (consumer->inputs[i] == ac_op_output) {
-          consumer->inputs[i] = intermediate_array;
-        }
-      }
-      consumer = GetFirstOpWithInput(*model, ac_op_output);
-    }
-    ac_op->inputs[0] = exchange_op_input;
-    exchange_op->inputs[0] = ac_op_output;
-  }
-
-  // Clear shapes; this will allow shape propagation to fix the sizes for us.
-  model->GetOrCreateArray(ac_op->outputs[0]).clear_shape();
-  model->GetOrCreateArray(exchange_op->outputs[0]).clear_shape();
-
-  // Finally, reorder operators.  Note that this only works when there are no
-  // other direct descendents of the exchange_op.
-  ac_op.swap(exchange_op);
-
-  return true;
-}
-
-}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/reorder_elementwise_unary.cc b/tensorflow/contrib/lite/toco/graph_transformations/reorder_elementwise_unary.cc
new file mode 100644
index 0000000000..9f5b7920cb
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/reorder_elementwise_unary.cc
@@ -0,0 +1,153 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <iterator>
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+namespace {
+
+bool IsElementwiseOperator(OperatorType optype) {
+  switch (optype) {
+    case OperatorType::kCast:
+    case OperatorType::kExp:
+    case OperatorType::kFloor:
+    case OperatorType::kNeg:
+    case OperatorType::kRelu:
+    case OperatorType::kRelu1:
+    case OperatorType::kRelu6:
+    case OperatorType::kTanh:
+    case OperatorType::kTensorFlowSqrt:
+    case OperatorType::kTensorFlowSquare:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool IsMoveOperator(OperatorType optype) {
+  switch (optype) {
+    case OperatorType::kDepthToSpace:
+    case OperatorType::kExpandDims:
+    case OperatorType::kSpaceToDepth:
+    case OperatorType::kSqueeze:
+    case OperatorType::kTensorFlowReshape:
+    case OperatorType::kTranspose:
+      return true;
+    default:
+      return false;
+  }
+}
+
+}  // namespace
+
+// Swap elementwise operators such that all value operators occur before all
+// element move operators, e.g. negation then transpose.
+bool ReorderElementwiseUnary::Run(Model* model, std::size_t op_index) {
+  const auto element_op_it = model->operators.begin() + op_index;
+  std::unique_ptr<Operator>& element_op = *element_op_it;
+  if (!IsElementwiseOperator(element_op->type)) {
+    return false;
+  }
+
+  const string intermediate_name = element_op->inputs[0];
+  auto it = FindOpWithOutput(*model, intermediate_name);
+  if (it == model->operators.end()) {
+    AddMessageF("No preceding operator");
+    return false;
+  }
+
+  std::unique_ptr<Operator>& move_op = *it;
+  if (!IsMoveOperator(move_op->type)) {
+    AddMessageF("Preceding operator is not a move operator");
+    return false;
+  }
+
+  if (CountOpsWithInput(*model, intermediate_name) != 1) {
+    AddMessageF("Input %s used elsewhere", intermediate_name);
+    return false;
+  }
+
+  // Check that the intermediate is discardable.
+  if (!IsDiscardableArray(*model, intermediate_name)) {
+    AddMessageF(
+        "Cannot swap elementwise as it would invalidate %s which is "
+        "an output array.",
+        intermediate_name);
+    return false;
+  }
+
+  // op->inputs may change so we need to keep a value by copy.
+  const string input_name = move_op->inputs[0];
+  const string output_name = element_op->outputs[0];
+
+  AddMessageF("Swapping around operators with %s and %s", LogName(*element_op),
+              LogName(*move_op));
+
+  // If the output array is an exit node for the graph then we need to retain
+  // the name as an output node. This makes the naming scheme a little confusing
+  // but is required in this rare case.
+  if (!IsDiscardableArray(*model, output_name)) {
+    // The output name of the sequence needs to stay static, so create a new
+    // array new use for the intermediate.
+    const auto new_intermediate_name =
+        AvailableArrayName(*model, element_op->outputs[0] + "_reorder");
+    AddMessageF("Adding new array %s to preserve output array name %s",
+                new_intermediate_name, output_name);
+
+    element_op->inputs[0] = input_name;
+    element_op->outputs[0] = new_intermediate_name;
+    model->EraseArray(intermediate_name);
+    move_op->inputs[0] = new_intermediate_name;
+    move_op->outputs[0] = output_name;
+  } else {
+    // The intermediate array is now the output array.
+    for (int i = 0; i < model->operators.size(); i++) {
+      Operator* consumer = model->operators[i].get();
+      for (int j = 0; j < consumer->inputs.size(); j++) {
+        if (consumer->inputs[j] == output_name) {
+          consumer->inputs[j] = intermediate_name;
+        }
+      }
+    }
+
+    element_op->inputs[0] = input_name;
+    move_op->inputs[0] = output_name;
+  }
+
+  // Reset both arrays as shape, type, min/max, etc can all change because of
+  // the position swap.
+  model->EraseArray(element_op->outputs[0]);
+  model->EraseArray(move_op->outputs[0]);
+
+  // Reconstruct.
+  model->GetOrCreateArray(element_op->outputs[0]);
+  model->GetOrCreateArray(move_op->outputs[0]);
+
+  // Swap the order of the operators.
+  element_op.swap(move_op);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/reorder_reshape_transpose.cc b/tensorflow/contrib/lite/toco/graph_transformations/reorder_reshape_transpose.cc
new file mode 100644
index 0000000000..9e7fe1b1cc
--- /dev/null
+++ b/tensorflow/contrib/lite/toco/graph_transformations/reorder_reshape_transpose.cc
@@ -0,0 +1,248 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+#include <iterator>
+#include <memory>
+#include <string>
+#include <unordered_set>
+#include <vector>
+
+#include "tensorflow/contrib/lite/toco/graph_transformations/graph_transformations.h"
+#include "tensorflow/contrib/lite/toco/model.h"
+#include "tensorflow/contrib/lite/toco/tooling_util.h"
+#include "tensorflow/core/platform/logging.h"
+
+namespace toco {
+
+namespace {
+
+bool OperatorReady(const Model& model, const Operator* op) {
+  if (!model.HasArray(op->inputs[0]) || !model.HasArray(op->inputs[1]) ||
+      !model.HasArray(op->outputs[0])) {
+    return false;
+  }
+
+  if (!model.GetArray(op->inputs[0]).has_shape() ||
+      !model.GetArray(op->outputs[0]).has_shape()) {
+    // Input and output needs the shape.
+    return false;
+  }
+
+  if (!model.GetArray(op->inputs[1]).buffer) {
+    // Buffer needs to be a constant.
+    return false;
+  }
+
+  return true;
+}
+
+// Utility function to filter out a value.
+void Filter(std::vector<int>* vec, int value) {
+  vec->erase(std::remove(vec->begin(), vec->end(), value), vec->end());
+}
+
+// Computes a new permutation used to swap a reshape-transpose to a
+// transpose-reshape. In this case the permutation operates on the intermediate
+// shape.
+std::vector<int> ComputeNewPerm(std::vector<int> input_dims,
+                                std::vector<int> intermediate_dims,
+                                std::vector<int> perm) {
+  // These are the major axis of the input.
+  std::vector<int> input_indices;
+  for (int i = 0; i < input_dims.size(); i++) {
+    if (input_dims[i] != 1) {
+      input_indices.push_back(i);
+    }
+  }
+
+  // This maps which indices of the input produced the intermediate indices for
+  // non-unary dimensions.
+  std::unordered_map<int, int> intermediate_to_input_indices_map;
+  for (int i = 0; i < intermediate_dims.size(); i++) {
+    if (intermediate_dims[i] != 1) {
+      intermediate_to_input_indices_map[i] =
+          input_indices[intermediate_to_input_indices_map.size()];
+    }
+  }
+
+  // Translate the transpose permutation to a new permutation starting with the
+  // major indices.
+  std::vector<int> new_perm;
+  new_perm.reserve(input_dims.size());
+  for (int i = 0; i < perm.size(); i++) {
+    if (intermediate_dims[perm[i]] == 1) continue;
+
+    new_perm.push_back(intermediate_to_input_indices_map[perm[i]]);
+  }
+
+  // Fill the rest of the transpose in with the ones.
+  for (int index = 0; index < input_dims.size(); index++) {
+    if (input_dims[index] == 1) {
+      new_perm.push_back(index);
+    }
+  }
+
+  CHECK_EQ(new_perm.size(), input_dims.size());
+  return new_perm;
+}
+
+}  // namespace
+
+// Swaps reshape-transpose to transpose-reshape whenever possible. This is
+// possible when the reshape does not affect memory ordering.
+bool ReorderReshapeTranspose::Run(Model* model, std::size_t op_index) {
+  auto transpose_it = model->operators.begin() + op_index;
+
+  TransposeOperator* transpose_op = ConvertOperator<TransposeOperator*>(
+      transpose_it->get(), OperatorType::kTranspose);
+
+  if (transpose_op == nullptr) {
+    return false;
+  }
+
+  if (!OperatorReady(*model, transpose_op) || transpose_op->perm.empty()) {
+    // Wait for values to propagate.
+    return false;
+  }
+
+  // Find the operator that produces the transpose op.
+  auto reshape_it = FindOpWithOutput(*model, transpose_op->inputs[0]);
+  if (reshape_it == model->operators.end()) {
+    return false;
+  }
+
+  TensorFlowReshapeOperator* reshape_op =
+      ConvertOperator<TensorFlowReshapeOperator*>(
+          reshape_it->get(), OperatorType::kTensorFlowReshape);
+  if (reshape_op == nullptr) {
+    return false;
+  }
+
+  // Ignore if the reshape is uninitialized.
+  if (!OperatorReady(*model, reshape_op) || reshape_op->shape.empty()) {
+    return false;
+  }
+
+  // Need to copy to keep static if permutated.
+  const string input_name = reshape_op->inputs[0];
+  const string intermediate_name = reshape_op->outputs[0];
+  const string output_name = transpose_op->outputs[0];
+
+  // Intermediate should not be consumed by any other operators.
+  if (CountOpsWithInput(*model, intermediate_name) != 1) {
+    AddMessageF("Input %s used elsewhere", intermediate_name);
+    return false;
+  }
+
+  // Check that the intermediate is not an output array.
+  if (!IsDiscardableArray(*model, intermediate_name)) {
+    AddMessageF(
+        "Cannot reorder reshape-transpose as it would invalidate %s which is "
+        "an output array.",
+        intermediate_name);
+    return false;
+  }
+
+  // Get the arrays.
+  const auto& input_array = model->GetArray(input_name);
+  const auto& intermediate_array = model->GetArray(intermediate_name);
+  const auto& output_array = model->GetArray(output_name);
+
+  // Get the shapes of each array.
+  Shape input_shape = input_array.shape();
+  Shape intermediate_shape = intermediate_array.shape();
+  Shape output_shape = output_array.shape();
+
+  // Assign ids to non-unary indices.
+  std::vector<int> input_dims = input_shape.dims();
+  std::vector<int> intermediate_dims = intermediate_shape.dims();
+  std::vector<int> output_dims = output_shape.dims();
+
+  // If the reshape is equivalent to a transpose with fewer/more unary
+  // dimensions then it can be moved between the transpose.
+  if (!ReshapeIsEquivalentToTranspose(*model, reshape_op,
+                                      true /*allow_extra_unary_dims*/)) {
+    return false;
+  }
+
+  if (!IsDiscardableArray(*model, output_name)) {
+    // The output name of the sequence needs to stay static, so create a new
+    // array new use for the intermediate.
+    const auto new_intermediate_name =
+        AvailableArrayName(*model, transpose_op->outputs[0] + "_exchange");
+    AddMessageF("Adding new array %s to preserve output array name %s",
+                new_intermediate_name, transpose_op->outputs[0]);
+    transpose_op->inputs[0] = input_name;
+    transpose_op->outputs[0] = new_intermediate_name;
+    reshape_op->inputs[0] = new_intermediate_name;
+    reshape_op->outputs[0] = output_name;
+    model->EraseArray(intermediate_name);
+  } else {
+    // The intermediate array is now the output array.
+    for (int i = 0; i < model->operators.size(); i++) {
+      Operator* consumer = model->operators[i].get();
+      for (int j = 0; j < consumer->inputs.size(); j++) {
+        if (consumer->inputs[j] == output_name) {
+          consumer->inputs[j] = intermediate_name;
+        }
+      }
+    }
+
+    transpose_op->inputs[0] = input_name;
+    reshape_op->inputs[0] = output_name;
+  }
+
+  // If transposes constant buffer is used elsewhere, make a new copy.
+  if (CountOpsWithInput(*model, transpose_op->inputs[1]) != 1) {
+    transpose_op->inputs[1] =
+        AvailableArrayName(*model, transpose_op->inputs[1] + "_copy");
+  }
+
+  // Make the new transpose permutation.
+  const std::vector<int> new_perm =
+      ComputeNewPerm(input_dims, intermediate_dims, transpose_op->perm);
+  CHECK_EQ(input_dims.size(), new_perm.size());
+
+  auto& transpose_array = model->GetOrCreateArray(transpose_op->inputs[1]);
+  transpose_array.GetMutableBuffer<ArrayDataType::kInt32>().data = new_perm;
+  *(transpose_array.mutable_shape()->mutable_dims()) = {
+      static_cast<int>(new_perm.size())};
+  transpose_op->perm = new_perm;
+
+  // If the reshape's constant buffer is reused, create a new one.
+  if (CountOpsWithInput(*model, reshape_op->inputs[1]) != 1) {
+    reshape_op->inputs[1] =
+        AvailableArrayName(*model, reshape_op->inputs[1] + "_copy");
+  }
+
+  // We need to modify the reshape input array to target the new output size.
+  auto& reshape_array = model->GetOrCreateArray(reshape_op->inputs[1]);
+  reshape_array.GetMutableBuffer<ArrayDataType::kInt32>().data = output_dims;
+  *(reshape_array.mutable_shape()->mutable_dims()) = {
+      static_cast<int>(output_shape.dimensions_count())};
+  reshape_op->shape.clear();
+
+  AddMessageF("Swapping around operators between %s and %s", input_name,
+              output_name);
+
+  model->GetOrCreateArray(transpose_op->outputs[0]).clear_shape();
+  model->GetOrCreateArray(reshape_op->outputs[0]).clear_shape();
+
+  // Swap the order of the operators.
+  transpose_it->swap(*reshape_it);
+
+  return true;
+}
+
+}  // namespace toco
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/resolve_tensorflow_matmul.cc b/tensorflow/contrib/lite/toco/graph_transformations/resolve_tensorflow_matmul.cc
index f38203c80f..2a236d3f98 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/resolve_tensorflow_matmul.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/resolve_tensorflow_matmul.cc
@@ -60,6 +60,13 @@ bool ResolveTensorFlowMatMul::Run(Model* model, std::size_t op_index) {
   string input_lhs = matmul_op->inputs[0];
   string input_rhs = transpose_op->outputs[0];
 
+  // Construct the new FullyConnectedOperator.
+  auto* fc_op = new FullyConnectedOperator;
+  fc_op->outputs = matmul_op->outputs;
+
+  // Insert the newly constructed FullyConnectedOperator.
+  model->operators.emplace(matmul_it, fc_op) + 1;
+
   // Find the op producing the array passed to this MatMul
   auto previous_op_it = model->operators.begin();
   bool found = false;
@@ -76,13 +83,6 @@ bool ResolveTensorFlowMatMul::Run(Model* model, std::size_t op_index) {
   }
   Operator* previous_op = (found) ? previous_op_it->get() : nullptr;
 
-  // Construct the new FullyConnectedOperator.
-  auto* fc_op = new FullyConnectedOperator;
-  fc_op->outputs = matmul_op->outputs;
-
-  // Insert the newly constructed FullyConnectedOperator.
-  model->operators.emplace(matmul_it, fc_op) + 1;
-
   // Refresh iterator.
   matmul_it = model->operators.begin();
   for (; matmul_it != model->operators.end(); ++matmul_it) {
diff --git a/tensorflow/contrib/lite/toco/toco_tooling.cc b/tensorflow/contrib/lite/toco/toco_tooling.cc
index 0c52f50e39..76e9a27aef 100644
--- a/tensorflow/contrib/lite/toco/toco_tooling.cc
+++ b/tensorflow/contrib/lite/toco/toco_tooling.cc
@@ -74,7 +74,9 @@ void MakeGeneralGraphTransformationsSet(
   transformations->Add(new ResolveTensorFlowMatMul);
   transformations->Add(new FuseBinaryIntoPrecedingAffine);
   transformations->Add(new FuseBinaryIntoFollowingAffine);
-  transformations->Add(new ReorderActivationFunctions);
+  transformations->Add(new MergeReshapeIntoPrecedingTranspose);
+  transformations->Add(new ReorderElementwiseUnary);
+  transformations->Add(new ReorderReshapeTranspose);
   transformations->Add(new ResolveBatchNormalization);
   transformations->Add(new ResolveConstantBinaryOperator);
   transformations->Add(new ResolveConstantFill);
diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 060c52e9e3..c1eaba7f4c 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -1921,6 +1921,35 @@ bool IsDiscardableArray(const Model& model, const string& array_name) {
   return true;
 }
 
+bool ReshapeIsEquivalentToTranspose(const Model& model,
+                                    const TensorFlowReshapeOperator* op,
+                                    bool allow_extra_unary_dims) {
+  CHECK(!op->shape.empty());
+  CHECK(model.HasArray(op->inputs[0]));
+  CHECK(model.HasArray(op->outputs[0]));
+
+  const auto& input_array = model.GetArray(op->inputs[0]);
+  const auto& output_array = model.GetArray(op->outputs[0]);
+
+  CHECK(input_array.has_shape());
+  CHECK(output_array.has_shape());
+
+  std::vector<int> in_shape = input_array.shape().dims();
+  std::vector<int> out_shape = output_array.shape().dims();
+
+  // If the reshape changes the number of dimensions so it cannot be interpreted
+  // as a transpose.
+  if (!allow_extra_unary_dims && in_shape.size() != out_shape.size()) {
+    return false;
+  }
+
+  in_shape.erase(std::remove(in_shape.begin(), in_shape.end(), 1),
+                 in_shape.end());
+  out_shape.erase(std::remove(out_shape.begin(), out_shape.end(), 1),
+                  out_shape.end());
+  return in_shape == out_shape;
+}
+
 void CheckFinalDataTypesSatisfied(const Model& model) {
   for (const auto& array_entry : model.GetArrayMap()) {
     const auto& array = *array_entry.second;
diff --git a/tensorflow/contrib/lite/toco/tooling_util.h b/tensorflow/contrib/lite/toco/tooling_util.h
index d3b7224fe3..259ee7fbd0 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.h
+++ b/tensorflow/contrib/lite/toco/tooling_util.h
@@ -169,10 +169,23 @@ void GetQuantizationParamsFromMinMax(const MinMax& minmax,
       ::tflite::ChooseQuantizationParams<Integer>(rmin, rmax);
 }
 
+template <typename T>
+T ConvertOperator(Operator* o, OperatorType type) {
+  if (o != nullptr && o->type == type) {
+    return static_cast<T>(o);
+  }
+
+  return nullptr;
+}
+
 void CheckIsReadyForQuantization(const Model& model);
 void UseDefaultMinMaxRangeValues(Model* model, double default_ranges_min,
                                  double default_ranges_max);
 
+bool ReshapeIsEquivalentToTranspose(const Model& model,
+                                    const TensorFlowReshapeOperator* op,
+                                    bool allow_extra_unary_dims);
+
 inline int Offset(const Shape& shape, const std::vector<int>& indices) {
   DCHECK_EQ(shape.dimensions_count(), indices.size());
   const int dims_count = shape.dimensions_count();
-- 
GitLab


From 5e93a0795b57a22c4c08424fb10fe3ca7e182707 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 3 Apr 2018 10:47:37 -0700
Subject: [PATCH 861/960] [TF:XLA] Add DT_HALF to the supported data types for
 the CPU and GPU devices.

Add a test case to compilation_passes_test for DT_HALF data type.

PiperOrigin-RevId: 191464288
---
 .../jit/mark_for_compilation_pass_test.cc     | 22 ++++++++++++++++++-
 tensorflow/compiler/jit/xla_cpu_device.cc     |  4 ++--
 tensorflow/compiler/jit/xla_gpu_device.cc     |  4 ++--
 tensorflow/compiler/tf2xla/xla_op_registry.h  |  8 +++----
 4 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index 381c0205fd..af1919278c 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -138,7 +138,7 @@ TEST(XlaCompilationTest, CompilableCycles) {
   EXPECT_EQ(clusters["A"], clusters["C"]);
 }
 
-TEST(XlaCompilationTest, UnsupportedTypes) {
+TEST(XlaCompilationTest, Complex128Unsupported) {
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   GraphDef graphdef;
   {
@@ -158,6 +158,26 @@ TEST(XlaCompilationTest, UnsupportedTypes) {
   EXPECT_TRUE(clusters.empty());
 }
 
+TEST(XlaCompilationTest, HalfSupported) {
+  std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
+  GraphDef graphdef;
+  {
+    GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
+    Node* a = ops::SourceOp(
+        "Const", builder.opts()
+                     .WithName("A")
+                     .WithAttr("dtype", DT_HALF)
+                     .WithAttr("value", Tensor(DT_HALF, TensorShape())));
+    Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B"));
+    ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C"));
+    TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
+  }
+
+  TF_ASSERT_OK(MarkForCompilation(&graph));
+  auto clusters = GetClusters(*graph);
+  EXPECT_FALSE(clusters.empty());
+}
+
 TEST(XlaCompilationTest, ConcatWithConstArg) {
   std::unique_ptr<Graph> graph(new Graph(OpRegistry::Global()));
   GraphDef graphdef;
diff --git a/tensorflow/compiler/jit/xla_cpu_device.cc b/tensorflow/compiler/jit/xla_cpu_device.cc
index d2dfdeea68..bc07dbd7bd 100644
--- a/tensorflow/compiler/jit/xla_cpu_device.cc
+++ b/tensorflow/compiler/jit/xla_cpu_device.cc
@@ -62,8 +62,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_CPU, XlaCpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 6> kAllXlaCpuTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+constexpr std::array<DataType, 7> kAllXlaCpuTypes = {
+    {DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_CPU, XlaLocalLaunchOp, kAllXlaCpuTypes);
 REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_CPU, kAllXlaCpuTypes);
diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 5a1db81774..84c0d5f51f 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -62,8 +62,8 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_GPU, XlaGpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 6> kAllXlaGpuTypes = {
-    {DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+constexpr std::array<DataType, 7> kAllXlaGpuTypes = {
+    {DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_GPU, XlaLocalLaunchOp, kAllXlaGpuTypes);
 REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_GPU, kAllXlaGpuTypes);
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index ff7453194a..da2a6c3e28 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -51,12 +51,12 @@ constexpr std::array<DataType, 9> kNumericTypes = {
     {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
      DT_COMPLEX64, DT_BFLOAT16}};
 
-constexpr std::array<DataType, 8> kCpuAllTypes = {
-    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE,
+constexpr std::array<DataType, 9> kCpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
      DT_COMPLEX64, DT_BOOL}};
 
-constexpr std::array<DataType, 8> kGpuAllTypes = {
-    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_FLOAT, DT_DOUBLE,
+constexpr std::array<DataType, 9> kGpuAllTypes = {
+    {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
      DT_COMPLEX64, DT_BOOL}};
 
 // Class that manages registrations of operators and devices for the XLA JIT.
-- 
GitLab


From f654b0d15af364d6f43d22a179fa05d20650fe9a Mon Sep 17 00:00:00 2001
From: Asim Shankar <ashankar@google.com>
Date: Tue, 3 Apr 2018 10:51:57 -0700
Subject: [PATCH 862/960] tf.map_fn: Improve error messaging when elems
 consists of scalars.

Fixes #17694
Prior to this change, when tf.map_fn was provided with scalars, the error would
be something like:

Traceback (most recent call last):
  File "/tensorflow/python/kernel_tests/functional_ops_test.py", line 165, in testMapOverScalarErrors
    functional_ops.map_fn(lambda x: x, [1, 2])
  File "/tensorflow/python/ops/functional_ops.py", line 368, in map_fn
    n = elems_flat[0].shape[0].value or array_ops.shape(elems_flat[0])[0]
  File "/tensorflow/python/framework/tensor_shape.py", line 609, in __getitem__
    return self._dims[key]
IndexError: list index out of range
PiperOrigin-RevId: 191465183
---
 tensorflow/python/kernel_tests/functional_ops_test.py |  7 +++++++
 tensorflow/python/ops/functional_ops.py               | 10 +++++++++-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/functional_ops_test.py b/tensorflow/python/kernel_tests/functional_ops_test.py
index 10aea89173..34fb655035 100644
--- a/tensorflow/python/kernel_tests/functional_ops_test.py
+++ b/tensorflow/python/kernel_tests/functional_ops_test.py
@@ -160,6 +160,13 @@ class FunctionalOpsTest(test.TestCase):
                 values=constant_op.constant([0, 1, 2]),
                 dense_shape=[2, 2]))
 
+  @test_util.run_in_graph_and_eager_modes()
+  def testMapOverScalarErrors(self):
+    with self.assertRaisesRegexp(ValueError, "not scalars"):
+      functional_ops.map_fn(lambda x: x, [1, 2])
+    with self.assertRaisesRegexp(ValueError, "not a scalar"):
+      functional_ops.map_fn(lambda x: x, 1)
+
   def testMap_Scoped(self):
     with self.test_session() as sess:
 
diff --git a/tensorflow/python/ops/functional_ops.py b/tensorflow/python/ops/functional_ops.py
index 4d95ca262c..161f6f3659 100644
--- a/tensorflow/python/ops/functional_ops.py
+++ b/tensorflow/python/ops/functional_ops.py
@@ -367,7 +367,15 @@ def map_fn(fn, elems, dtype=None, parallel_iterations=10, back_prop=True,
     dtype_flat = output_flatten(dtype)
 
     # Convert elems to tensor array. n may be known statically.
-    n = elems_flat[0].shape[0].value or array_ops.shape(elems_flat[0])[0]
+    static_shape = elems_flat[0].shape
+    if static_shape.ndims is not None and static_shape.ndims < 1:
+      if len(elems_flat) == 1:
+        raise ValueError("elems must be a 1+ dimensional Tensor, not a scalar")
+      else:
+        raise ValueError(
+            "elements in elems must be 1+ dimensional Tensors, not scalars"
+        )
+    n = static_shape[0].value or array_ops.shape(elems_flat[0])[0]
 
     # TensorArrays are always flat
     elems_ta = [
-- 
GitLab


From 86235e48fe39f2b9318f01e963499a555ea88084 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 11:15:29 -0700
Subject: [PATCH 863/960] Turn no-op split/splitv operators into identity.

PiperOrigin-RevId: 191469655
---
 tensorflow/core/grappler/op_types.cc          |  2 +
 tensorflow/core/grappler/op_types.h           |  1 +
 .../grappler/optimizers/constant_folding.cc   | 10 +++
 .../optimizers/constant_folding_test.cc       | 76 +++++++++++++++++++
 4 files changed, 89 insertions(+)

diff --git a/tensorflow/core/grappler/op_types.cc b/tensorflow/core/grappler/op_types.cc
index 584008b0c1..a24d2dbd9f 100644
--- a/tensorflow/core/grappler/op_types.cc
+++ b/tensorflow/core/grappler/op_types.cc
@@ -365,6 +365,8 @@ bool IsTruncateDiv(const NodeDef& node) { return node.op() == "TruncateDiv"; }
 
 bool IsTruncateMod(const NodeDef& node) { return node.op() == "TruncateMod"; }
 
+bool IsUnpack(const NodeDef& node) { return node.op() == "Unpack"; }
+
 bool IsVariable(const NodeDef& node) {
   const auto& op = node.op();
   return op == "Variable" || op == "VariableV2" || op == "AutoReloadVariable" ||
diff --git a/tensorflow/core/grappler/op_types.h b/tensorflow/core/grappler/op_types.h
index aa6750d5c3..8667f72c7e 100644
--- a/tensorflow/core/grappler/op_types.h
+++ b/tensorflow/core/grappler/op_types.h
@@ -140,6 +140,7 @@ bool IsTile(const NodeDef& node);
 bool IsTranspose(const NodeDef& node);
 bool IsTruncateDiv(const NodeDef& node);
 bool IsTruncateMod(const NodeDef& node);
+bool IsUnpack(const NodeDef& node);
 bool IsVariable(const NodeDef& node);
 bool IsZeta(const NodeDef& node);
 
diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 7de544de52..87052c7ba0 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -1542,6 +1542,16 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
   for (int i = 0; i < optimized_graph->node_size(); ++i) {
     NodeDef* node = optimized_graph->mutable_node(i);
 
+    if (IsSplit(*node) && node->attr().at("num_split").i() == 1) {
+      ReplaceOperationWithIdentity(1, node, optimized_graph);
+      continue;
+    }
+
+    if (IsSplitV(*node) && node->attr().at("num_split").i() == 1) {
+      ReplaceOperationWithIdentity(0, node, optimized_graph);
+      continue;
+    }
+
     // Remove Shuffle or Reverse op over scalar values.
     if (use_shape_info &&
         !properties->GetInputProperties(node->name()).empty() &&
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 1db4fb9de7..7faa68a657 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1335,6 +1335,82 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   EXPECT_EQ(2, out_idx.flat<int32>()(0));
 }
 
+TEST_F(ConstantFoldingTest, SplitRemoval) {
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+  Output in1 =
+      ops::Variable(scope.WithOpName("in1"), TensorShape({2}), DT_FLOAT);
+  Output in2 =
+      ops::Variable(scope.WithOpName("in2"), TensorShape({4}), DT_FLOAT);
+  auto split_dim = ops::Const(scope.WithOpName("split_dim"), {0}, {});
+  ops::Split s1(scope.WithOpName("s1"), split_dim, in1, 1);
+  ops::Split s2(scope.WithOpName("s2"), split_dim, in2, 2);
+
+  ops::Add out(scope.WithOpName("out"), s1[0], s2[0]);
+
+  GrapplerItem item;
+  item.fetch = {"out"};
+  TF_CHECK_OK(scope.ToGraphDef(&item.graph));
+
+  ConstantFolding optimizer(nullptr /* cpu_device */);
+  GraphDef got;
+  Status status = optimizer.Optimize(nullptr, item, &got);
+  TF_EXPECT_OK(status);
+
+  GraphDef want;
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("split_dim", "Const", {}, {}, &want);
+  AddNode("s1", "Identity", {"in1", AsControlDependency("split_dim")}, {},
+          &want);
+  AddNode("s2", "Split", {"in2", "split_dim"}, {}, &want);
+  AddNode("out", "Add", {"s1", "s2"}, {}, &want);
+
+  CompareGraphs(want, got);
+}
+
+TEST_F(ConstantFoldingTest, SplitVRemoval) {
+  tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
+
+  Output in1 =
+      ops::Variable(scope.WithOpName("in1"), TensorShape({2}), DT_FLOAT);
+  Output in2 =
+      ops::Variable(scope.WithOpName("in2"), TensorShape({5}), DT_FLOAT);
+  auto split_dim = ops::Const(scope.WithOpName("split_dim"), {0}, {});
+  auto size_splits1 = ops::Const(scope.WithOpName("size_splits1"), {2}, {1});
+  auto size_splits2 = ops::Const(scope.WithOpName("size_splits2"), {2, 3}, {2});
+  ops::SplitV s1(scope.WithOpName("s1"), in1, size_splits1, split_dim, 1);
+  ops::SplitV s2(scope.WithOpName("s2"), in2, size_splits2, split_dim, 2);
+
+  LOG(INFO) << s1.output.size();
+  LOG(INFO) << s2.output.size();
+  ops::Add out(scope.WithOpName("out"), s1[0], s2[0]);
+
+  GrapplerItem item;
+  item.fetch = {"out"};
+  TF_CHECK_OK(scope.ToGraphDef(&item.graph));
+
+  ConstantFolding optimizer(nullptr /* cpu_device */);
+  GraphDef got;
+  Status status = optimizer.Optimize(nullptr, item, &got);
+  TF_EXPECT_OK(status);
+
+  GraphDef want;
+  AddNode("in1", "VariableV2", {}, {}, &want);
+  AddNode("in2", "VariableV2", {}, {}, &want);
+  AddNode("split_dim", "Const", {}, {}, &want);
+  AddNode("size_splits1", "Const", {}, {}, &want);
+  AddNode("size_splits2", "Const", {}, {}, &want);
+  AddNode("s1", "Identity",
+          {"in1", AsControlDependency("size_splits1"),
+           AsControlDependency("split_dim")},
+          {}, &want);
+  AddNode("s2", "SplitV", {"in2", "size_splits2", "split_dim"}, {}, &want);
+  AddNode("out", "Add", {"s1", "s2"}, {}, &want);
+
+  CompareGraphs(want, got);
+}
+
 TEST_F(ConstantFoldingTest, ShuffleReverseOnScalarRemoval) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
 
-- 
GitLab


From fe2d7600bd6ecfcf3a083b7c01b2c114ba7c63ff Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Tue, 3 Apr 2018 11:52:23 -0700
Subject: [PATCH 864/960] Add convert f32 to s64 test.

PiperOrigin-RevId: 191476199
---
 tensorflow/compiler/xla/tests/convert_test.cc | 36 +++++++++++++++++++
 1 file changed, 36 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/convert_test.cc b/tensorflow/compiler/xla/tests/convert_test.cc
index 9a899b7914..8718fa5066 100644
--- a/tensorflow/compiler/xla/tests/convert_test.cc
+++ b/tensorflow/compiler/xla/tests/convert_test.cc
@@ -230,6 +230,42 @@ XLA_TEST_F(ConvertTest, ConvertR1S32ToR1S64) {
   ComputeAndCompareR1<int64>(&builder, expected, {arg_data.get()});
 }
 
+XLA_TEST_F(ConvertTest, ConvertR1F32ToR1S64) {
+  ComputationBuilder builder(client_, TestName());
+  // Test cases from compiler_rt library.
+  std::vector<float> arg{0.0f,
+                         0.5f,
+                         0.99f,
+                         1.0f,
+                         1.5f,
+                         1.99f,
+                         2.0f,
+                         2.01f,
+                         -0.5f,
+                         -0.99f,
+                         -1.0f,
+                         -1.5f,
+                         -1.99f,
+                         -2.0f,
+                         -2.01f,
+                         0x1.FFFFFEp+62F,
+                         0x1.FFFFFCp+62F,
+                         -0x1.FFFFFEp+62F,
+                         -0x1.FFFFFCp+62F};
+  std::unique_ptr<Literal> arg_literal = Literal::CreateR1<float>({arg});
+  auto arg_param = builder.Parameter(0, arg_literal->shape(), "arg_param");
+  std::unique_ptr<GlobalData> arg_data =
+      client_->TransferToServer(*arg_literal).ConsumeValueOrDie();
+
+  builder.ConvertElementType(arg_param, S64);
+
+  std::vector<int64> expected(arg.size());
+  for (int64 i = 0; i < arg.size(); ++i) {
+    expected[i] = static_cast<int64>(arg[i]);
+  }
+  ComputeAndCompareR1<int64>(&builder, expected, {arg_data.get()});
+}
+
 XLA_TEST_F(ConvertTest, ConvertR1U8ToR1F32) {
   ComputationBuilder builder(client_, TestName());
   auto a = builder.ConstantR1<uint8_t>({32, 64});
-- 
GitLab


From d5ebc299ece050d3349feadc20339d636a9d84a1 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 3 Apr 2018 12:38:54 -0700
Subject: [PATCH 865/960] Use PyLong_AsLongLong instead of PyInt_AsLong to
 guarantee 64-bit output.

A C long is 32 bits on some platforms, which can cause the
PyInt_AsLong call in PyInt64ListToVector to
overflow. large_concat_op_test exposes this bug on such platforms.

PiperOrigin-RevId: 191484167
---
 tensorflow/python/client/tf_session.i                  | 2 +-
 tensorflow/python/kernel_tests/large_concat_op_test.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 68768f2b4c..0c18d973a7 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -72,7 +72,7 @@ void PyInt64ListToVector(PyObject* py_int_seq, std::vector<int64_t>* vec) {
   int size = PySequence_Fast_GET_SIZE(py_int_seq);
   for (int i = 0; i < size; ++i) {
     PyObject* item = PySequence_Fast_GET_ITEM(py_int_seq, i);
-    vec->push_back(PyInt_AsLong(item));
+    vec->push_back(PyLong_AsLongLong(item));
   }
 }
 
diff --git a/tensorflow/python/kernel_tests/large_concat_op_test.py b/tensorflow/python/kernel_tests/large_concat_op_test.py
index 66afb6ec01..184d1dde2a 100644
--- a/tensorflow/python/kernel_tests/large_concat_op_test.py
+++ b/tensorflow/python/kernel_tests/large_concat_op_test.py
@@ -19,10 +19,12 @@ from __future__ import print_function
 
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import test_util
 from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import test
 
 
+@test_util.with_c_api
 class LargeConcatOpTest(test.TestCase):
   """Tests that belong in concat_op_test.py, but run over large tensors."""
 
-- 
GitLab


From 1c16317d2d63d2a61812961e96be0d840fa81186 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 12:40:51 -0700
Subject: [PATCH 866/960] Only create a min/max from extra arrays info if
 provided.

PiperOrigin-RevId: 191484470
---
 tensorflow/contrib/lite/toco/tooling_util.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index c1eaba7f4c..668cf51619 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -2006,9 +2006,9 @@ void UseArraysExtraInfo(Model* model) {
       continue;
     }
     auto& array = model->GetArray(entry.name());
-    auto& minmax = array.GetOrCreateMinMax();
     if (entry.has_min() || entry.has_max()) {
       CHECK_EQ(entry.has_min(), entry.has_max());
+      auto& minmax = array.GetOrCreateMinMax();
       minmax.min = entry.min();
       minmax.max = entry.max();
     }
-- 
GitLab


From b888bae170e38007be39dc047d0590457ad5c069 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 13:28:37 -0700
Subject: [PATCH 867/960] [XLA] Redesign: migrate slice_test.

PiperOrigin-RevId: 191491425
---
 tensorflow/compiler/xla/tests/BUILD         |  2 +-
 tensorflow/compiler/xla/tests/slice_test.cc | 30 ++++++++++-----------
 2 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index aba61fbca4..5dcd02a1a4 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -937,8 +937,8 @@ xla_test(
     deps = [
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:reference_util",
-        "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/tests:client_library_test_base",
         "//tensorflow/compiler/xla/tests:literal_test_util",
         "//tensorflow/compiler/xla/tests:xla_internal_test_main",
diff --git a/tensorflow/compiler/xla/tests/slice_test.cc b/tensorflow/compiler/xla/tests/slice_test.cc
index a14a365bd0..8d9a9c7b73 100644
--- a/tensorflow/compiler/xla/tests/slice_test.cc
+++ b/tensorflow/compiler/xla/tests/slice_test.cc
@@ -19,8 +19,8 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/compiler/xla/array2d.h"
-#include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/reference_util.h"
 #include "tensorflow/compiler/xla/tests/client_library_test_base.h"
 #include "tensorflow/compiler/xla/tests/literal_test_util.h"
@@ -41,7 +41,7 @@ TEST_F(SliceTest, Slice3x3x3_To_3x3x1_F32) {
   Array3D<float> values(3, 3, 3);
   values.FillIota(0);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR3FromArray3D<float>(values);
   builder.Slice(original, {0, 0, 0}, {3, 3, 1}, {1, 1, 1});
 
@@ -54,7 +54,7 @@ TEST_F(SliceTest, Slice3x3x3_To_3x1x3_F32) {
   Array3D<float> values(3, 3, 3);
   values.FillIota(0);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR3FromArray3D<float>(values);
   builder.Slice(original, {0, 0, 0}, {3, 1, 3}, {1, 1, 1});
 
@@ -67,7 +67,7 @@ TEST_F(SliceTest, Slice3x3x3_To_1x3x3_F32) {
   Array3D<float> values(3, 3, 3);
   values.FillIota(0);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR3FromArray3D<float>(values);
   builder.Slice(original, {0, 0, 0}, {1, 3, 3}, {1, 1, 1});
 
@@ -77,7 +77,7 @@ TEST_F(SliceTest, Slice3x3x3_To_1x3x3_F32) {
 }
 
 XLA_TEST_F(SliceTest, Slice0x0to0x0F32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(Array2D<float>(0, 0));
   builder.Slice(original, {0, 0}, {0, 0}, {1, 1});
 
@@ -85,7 +85,7 @@ XLA_TEST_F(SliceTest, Slice0x0to0x0F32) {
 }
 
 XLA_TEST_F(SliceTest, Slice0x20to0x5F32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(Array2D<float>(0, 20));
   builder.Slice(original, {0, 15}, {0, 20}, {1, 1});
 
@@ -93,7 +93,7 @@ XLA_TEST_F(SliceTest, Slice0x20to0x5F32) {
 }
 
 XLA_TEST_F(SliceTest, Slice3x0to2x0F32) {
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(Array2D<float>(3, 0));
   builder.Slice(original, {1, 0}, {3, 0}, {1, 1});
 
@@ -108,7 +108,7 @@ XLA_TEST_F(SliceTest, SliceQuadrantOf256x256) {
     }
   }
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(values);
   builder.Slice(original, {128, 128}, {256, 256}, {1, 1});
 
@@ -126,7 +126,7 @@ TEST_F(SliceTest, Slice_1x4096_To_1x1024) {
   Array2D<float> values(1, 4096);
   std::iota(values.data(), values.data() + 4096, 0.0);
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(values);
   builder.Slice(original, {0, 3072}, {1, 4096}, {1, 1});
 
@@ -147,7 +147,7 @@ TEST_F(SliceTest, Slice_16x4_To_16x2) {
       }
     }
   }
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR2FromArray2D<float>(values);
   builder.Slice(original, {0, 0}, {16, 2}, {1, 1});
   ComputeAndCompareR2<float>(&builder, expected, {}, ErrorSpec(0.000001));
@@ -159,7 +159,7 @@ TEST_F(SliceTest, SliceR4ThreeDimsMiddleMinor) {
   values.FillRandom(3.14f);
   auto expected = ReferenceUtil::Slice4D(
       values, {{1, 0, 8, 0}}, {{2, 2, 16, 128}}, /*strides=*/{{1, 1, 1, 1}});
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR4FromArray4D(values);
   builder.Slice(original, {1, 0, 8, 0}, {2, 2, 16, 128}, {1, 1, 1, 1});
   ComputeAndCompareR4(&builder, *expected, {}, ErrorSpec(0.000001));
@@ -172,7 +172,7 @@ XLA_TEST_F(SliceTest, StridedSliceR4WithOutputLayout) {
                                          /*strides=*/{{1, 1, 2, 1}});
   auto expected_literal = Literal::CreateR4FromArray4DWithLayout(
       *expected, LayoutUtil::MakeLayout({0, 1, 2, 3}));
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto original = builder.ConstantR4FromArray4D(values);
   builder.Slice(original, {0, 0, 0, 0}, {2, 4, 6, 8}, {1, 1, 2, 1});
   ComputeAndCompareLiteral(&builder, *expected_literal, {}, ErrorSpec(0.000001),
@@ -198,7 +198,7 @@ class SliceR1Test : public ClientLibraryTestBase,
     tensorflow::gtl::InlinedVector<NativeT, 1> input(spec.input_dim0);
     std::iota(input.begin(), input.end(), NativeT());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto original = builder.ConstantR1<NativeT>(input);
     builder.Slice(original, {spec.slice_start}, {spec.slice_limit},
                   {spec.slice_stride});
@@ -339,7 +339,7 @@ XLA_TEST_P(SliceR2Test, DoIt) {
   Array2D<int32> input(spec.input_dim0, spec.input_dim1);
   input.FillUnique();
 
-  ComputationBuilder builder(client_, TestName());
+  XlaBuilder builder(TestName());
   auto a = builder.ConstantR2FromArray2DWithLayout<int32>(
       input, LayoutUtil::MakeLayout(spec.layout));
   builder.Slice(a, spec.slice_starts, spec.slice_limits, spec.slice_strides);
@@ -429,7 +429,7 @@ class SliceR4Test : public ClientLibraryTestBase,
     values.FillRandom(3.14f);
     auto expected = ReferenceUtil::Slice4D(
         values, spec.slice_starts, spec.slice_limits, spec.slice_strides);
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     auto literal = Literal::CreateR4FromArray4DWithLayout(
         values, LayoutUtil::MakeLayout(spec.input_layout));
     auto parameter = builder.Parameter(0, literal->shape(), "p0");
-- 
GitLab


From bf1d69a69819c9435c7819f40f17adb64024cedd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 13:43:05 -0700
Subject: [PATCH 868/960] Use double for arrays extra info min/max to match
 toco type.

PiperOrigin-RevId: 191493450
---
 tensorflow/contrib/lite/toco/model_flags.proto | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/model_flags.proto b/tensorflow/contrib/lite/toco/model_flags.proto
index 42e0f54826..835dea49eb 100644
--- a/tensorflow/contrib/lite/toco/model_flags.proto
+++ b/tensorflow/contrib/lite/toco/model_flags.proto
@@ -98,8 +98,8 @@ message ArraysExtraInfo {
   message Entry {
     // Next ID to use: 7.
     optional string name = 1;
-    optional float min = 2;
-    optional float max = 3;
+    optional double min = 2;
+    optional double max = 3;
     optional IODataType data_type = 4;
     optional InputArrayShape shape = 5;
     optional float constant_float_value = 6;
-- 
GitLab


From b9b90965de4e475ccff8a571de016026447ee1df Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 13:53:00 -0700
Subject: [PATCH 869/960] Disabled some tests on Windows

PiperOrigin-RevId: 191494857
---
 tensorflow/contrib/BUILD                                  | 2 +-
 tensorflow/contrib/autograph/converters/BUILD             | 3 +++
 tensorflow/contrib/autograph/impl/BUILD                   | 2 ++
 tensorflow/contrib/autograph/pyct/BUILD                   | 1 +
 tensorflow/contrib/autograph/pyct/static_analysis/BUILD   | 2 ++
 tensorflow/contrib/autograph/utils/BUILD                  | 2 ++
 tensorflow/contrib/data/BUILD                             | 8 +++++++-
 tensorflow/contrib/lite/python/BUILD                      | 1 +
 tensorflow/contrib/stat_summarizer/BUILD                  | 1 +
 tensorflow/contrib/tensorboard/BUILD                      | 1 +
 tensorflow/contrib/tpu/BUILD                              | 1 +
 tensorflow/python/BUILD                                   | 2 ++
 tensorflow/python/debug/BUILD                             | 1 +
 tensorflow/python/kernel_tests/BUILD                      | 1 +
 .../tools/ci_build/windows/cpu/pip/build_tf_windows.sh    | 1 +
 15 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index bf69144ad8..6964cbd564 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -117,7 +117,7 @@ py_library(
         ],
         "//conditions:default": [],
     }) + if_not_windows([
-        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
+        # for :contrib_py
         "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
     ]),
 )
diff --git a/tensorflow/contrib/autograph/converters/BUILD b/tensorflow/contrib/autograph/converters/BUILD
index 608bd82722..c5a0dc1095 100644
--- a/tensorflow/contrib/autograph/converters/BUILD
+++ b/tensorflow/contrib/autograph/converters/BUILD
@@ -61,6 +61,7 @@ py_test(
     name = "asserts_test",
     srcs = ["asserts_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":test_lib",
         "//tensorflow/python:client_testlib",
@@ -81,6 +82,7 @@ py_test(
     name = "builtin_functions_test",
     srcs = ["builtin_functions_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":test_lib",
         "//tensorflow/python:client_testlib",
@@ -92,6 +94,7 @@ py_test(
     size = "large",
     srcs = ["call_trees_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":test_lib",
         "//tensorflow/contrib/autograph/impl",
diff --git a/tensorflow/contrib/autograph/impl/BUILD b/tensorflow/contrib/autograph/impl/BUILD
index e468176da1..0de479741a 100644
--- a/tensorflow/contrib/autograph/impl/BUILD
+++ b/tensorflow/contrib/autograph/impl/BUILD
@@ -38,6 +38,7 @@ py_test(
     name = "api_test",
     srcs = ["api_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":impl",
         "//tensorflow/contrib/autograph/utils",
@@ -50,6 +51,7 @@ py_test(
     name = "conversion_test",
     srcs = ["conversion_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":impl",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/autograph/pyct/BUILD b/tensorflow/contrib/autograph/pyct/BUILD
index edec5f7712..c483ff68c4 100644
--- a/tensorflow/contrib/autograph/pyct/BUILD
+++ b/tensorflow/contrib/autograph/pyct/BUILD
@@ -66,6 +66,7 @@ py_test(
     name = "compiler_test",
     srcs = ["compiler_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":pyct",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD
index d192bc7aab..83f3bafc42 100644
--- a/tensorflow/contrib/autograph/pyct/static_analysis/BUILD
+++ b/tensorflow/contrib/autograph/pyct/static_analysis/BUILD
@@ -34,6 +34,7 @@ py_test(
     name = "activity_test",
     srcs = ["activity_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":static_analysis",
         "//tensorflow/contrib/autograph/pyct",
@@ -46,6 +47,7 @@ py_test(
     name = "live_values_test",
     srcs = ["live_values_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":static_analysis",
         "//tensorflow/contrib/autograph/pyct",
diff --git a/tensorflow/contrib/autograph/utils/BUILD b/tensorflow/contrib/autograph/utils/BUILD
index b53fbb5c18..d3a1b94688 100644
--- a/tensorflow/contrib/autograph/utils/BUILD
+++ b/tensorflow/contrib/autograph/utils/BUILD
@@ -44,6 +44,7 @@ py_test(
     name = "builtins_test",
     srcs = ["builtins_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":utils",
         "//tensorflow/python:client_testlib",
@@ -84,6 +85,7 @@ py_test(
     name = "py_func_test",
     srcs = ["py_func_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":utils",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/data/BUILD b/tensorflow/contrib/data/BUILD
index 35312f06b3..7bb0dc1c0f 100644
--- a/tensorflow/contrib/data/BUILD
+++ b/tensorflow/contrib/data/BUILD
@@ -8,6 +8,7 @@ load(
     "//tensorflow:tensorflow.bzl",
     "tf_custom_op_library",
     "tf_gen_op_libs",
+    "if_not_windows",
 )
 load(
     "//tensorflow/core:platform/default/build_config_root.bzl",
@@ -31,12 +32,17 @@ py_library(
     ],
 )
 
+cc_library(
+    name = "lib_proto_parsing_for_dataset_ops",
+    deps = if_not_windows(["//tensorflow/core:lib_proto_parsing"]),
+)
+
 tf_custom_op_library(
     name = "_dataset_ops.so",
     srcs = ["ops/dataset_ops.cc"],
     deps = ["//tensorflow/contrib/data/kernels:dataset_kernels"] +
            if_static(
-               extra_deps = ["//tensorflow/core:lib_proto_parsing"],
+               extra_deps = [":lib_proto_parsing_for_dataset_ops"],
                otherwise = [],
            ),
 )
diff --git a/tensorflow/contrib/lite/python/BUILD b/tensorflow/contrib/lite/python/BUILD
index e70aa51298..e735062a7f 100644
--- a/tensorflow/contrib/lite/python/BUILD
+++ b/tensorflow/contrib/lite/python/BUILD
@@ -101,6 +101,7 @@ py_test(
     name = "convert_saved_model_test",
     srcs = ["convert_saved_model_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     visibility = ["//visibility:public"],
     deps = [
         ":convert_saved_model",
diff --git a/tensorflow/contrib/stat_summarizer/BUILD b/tensorflow/contrib/stat_summarizer/BUILD
index d4096751c4..30be14c10c 100644
--- a/tensorflow/contrib/stat_summarizer/BUILD
+++ b/tensorflow/contrib/stat_summarizer/BUILD
@@ -31,4 +31,5 @@ tf_py_test(
         "//tensorflow/python:math_ops",
         "//tensorflow/python:variables",
     ],
+    tags = ["no_windows"],
 )
diff --git a/tensorflow/contrib/tensorboard/BUILD b/tensorflow/contrib/tensorboard/BUILD
index c955b13244..2b6a2b2f3c 100644
--- a/tensorflow/contrib/tensorboard/BUILD
+++ b/tensorflow/contrib/tensorboard/BUILD
@@ -82,6 +82,7 @@ py_test(
     size = "small",
     srcs = ["plugins/trace/trace_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":trace",
         "//tensorflow/python:client_testlib",
diff --git a/tensorflow/contrib/tpu/BUILD b/tensorflow/contrib/tpu/BUILD
index d4830b6bcf..4de09dd988 100644
--- a/tensorflow/contrib/tpu/BUILD
+++ b/tensorflow/contrib/tpu/BUILD
@@ -215,6 +215,7 @@ tf_py_test(
         ":datasets",
     ],
     grpc_enabled = True,
+    tags = ["no_windows"],
 )
 
 tf_py_test(
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 9d1e9bdc7e..b5f4387efd 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3782,6 +3782,7 @@ cuda_py_test(
         ":math_ops",
         "//tensorflow/core:protos_all_py",
     ],
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
@@ -4013,6 +4014,7 @@ py_test(
     srcs = ["training/saver_large_partitioned_variable_test.py"],
     srcs_version = "PY2AND3",
     tags = [
+        "no_windows",
         "noasan",  # http://b/30782289
         "notsan",  # http://b/30782289
     ],
diff --git a/tensorflow/python/debug/BUILD b/tensorflow/python/debug/BUILD
index b81aa3745c..250b4b1b6a 100644
--- a/tensorflow/python/debug/BUILD
+++ b/tensorflow/python/debug/BUILD
@@ -921,6 +921,7 @@ py_test(
     size = "small",
     srcs = ["cli/profile_analyzer_cli_test.py"],
     srcs_version = "PY2AND3",
+    tags = ["no_windows"],
     deps = [
         ":debugger_cli_common",
         ":profile_analyzer_cli",
diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index d6f97fc4c3..a544e4fa6e 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2722,6 +2722,7 @@ cuda_py_test(
     ],
     data = ["//tensorflow/python/kernel_tests/testdata:self_adjoint_eig_op_test_files"],
     shard_count = 20,
+    tags = ["no_windows"],
 )
 
 cuda_py_test(
diff --git a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
index 40189a6d1b..438c5d52f6 100644
--- a/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
+++ b/tensorflow/tools/ci_build/windows/cpu/pip/build_tf_windows.sh
@@ -65,5 +65,6 @@ bazel test -c opt $BUILD_OPTS -k --test_output=errors \
   --define=no_tensorflow_py_deps=true --test_lang_filters=py \
   --test_tag_filters=-no_pip,-no_windows,-no_oss \
   --build_tag_filters=-no_pip,-no_windows,-no_oss --build_tests_only \
+  --flaky_test_attempts=3 \
   //${PY_TEST_DIR}/tensorflow/python/... \
   //${PY_TEST_DIR}/tensorflow/contrib/...
-- 
GitLab


From 86b3f351dd98db9cdfc2fc68a2a4328e90b36035 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 14:42:31 -0700
Subject: [PATCH 870/960] [XLA] Redesign: implement and test dynamic slice.

PiperOrigin-RevId: 191502312
---
 .../xla/client/xla_client/xla_builder.cc      | 18 +++++++-
 tensorflow/compiler/xla/tests/BUILD           |  3 +-
 .../compiler/xla/tests/dynamic_ops_test.cc    | 41 +++++++++----------
 3 files changed, 37 insertions(+), 25 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index c2e661cb3d..fe8ae77683 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -489,7 +489,23 @@ XlaOp XlaBuilder::SliceInDim(const XlaOp& operand, int64 start_index,
 
 XlaOp XlaBuilder::DynamicSlice(const XlaOp& operand, const XlaOp& start_indices,
                                tensorflow::gtl::ArraySlice<int64> slice_sizes) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    TF_ASSIGN_OR_RETURN(const Shape& operand_shape, GetShape(operand));
+    TF_ASSIGN_OR_RETURN(const Shape& start_indices_shape,
+                        GetShape(start_indices));
+    TF_ASSIGN_OR_RETURN(*instr.mutable_shape(),
+                        ShapeInference::InferDynamicSliceShape(
+                            operand_shape, start_indices_shape, slice_sizes));
+
+    for (int64 size : slice_sizes) {
+      instr.add_dynamic_slice_sizes(size);
+    }
+
+    return AddInstruction(std::move(instr), HloOpcode::kDynamicSlice,
+                          {operand, start_indices});
+  });
 }
 
 XlaOp XlaBuilder::DynamicUpdateSlice(const XlaOp& operand, const XlaOp& update,
diff --git a/tensorflow/compiler/xla/tests/BUILD b/tensorflow/compiler/xla/tests/BUILD
index 5dcd02a1a4..6f58c20f34 100644
--- a/tensorflow/compiler/xla/tests/BUILD
+++ b/tensorflow/compiler/xla/tests/BUILD
@@ -977,9 +977,8 @@ xla_test(
         "//tensorflow/compiler/xla:reference_util",
         "//tensorflow/compiler/xla:test_helpers",
         "//tensorflow/compiler/xla/client:client_library",
-        "//tensorflow/compiler/xla/client:computation",
-        "//tensorflow/compiler/xla/client:computation_builder",
         "//tensorflow/compiler/xla/client:local_client",
+        "//tensorflow/compiler/xla/client/xla_client:xla_builder",
         "//tensorflow/compiler/xla/service:computation_placer",
         "//tensorflow/compiler/xla/service:device_memory_allocator",
         "//tensorflow/compiler/xla/service:local_service",
diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
index 4f354e6aef..c0a16ad288 100644
--- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
@@ -18,9 +18,8 @@ limitations under the License.
 
 #include "tensorflow/compiler/xla/array2d.h"
 #include "tensorflow/compiler/xla/client/client_library.h"
-#include "tensorflow/compiler/xla/client/computation.h"
-#include "tensorflow/compiler/xla/client/computation_builder.h"
 #include "tensorflow/compiler/xla/client/local_client.h"
+#include "tensorflow/compiler/xla/client/xla_client/xla_builder.h"
 #include "tensorflow/compiler/xla/reference_util.h"
 #include "tensorflow/compiler/xla/service/device_memory_allocator.h"
 #include "tensorflow/compiler/xla/service/local_service.h"
@@ -112,10 +111,8 @@ class DynamicSliceTest : public ClientLibraryTestBase {
   void TestR3Wrap() {
     // Slice at dimension boundaries, but with sizes that cause indices to wrap.
     RunR3<IndexT, DataT>(
-      {{{1, 2}, {3, 4}, {5, 6}},
-       {{7, 8}, {9, 10}, {11, 12}}},
-      {0, 2, 1}, {2, 1, 2},
-      {{{6, 5}}, {{12, 11}}});
+        {{{1, 2}, {3, 4}, {5, 6}}, {{7, 8}, {9, 10}, {11, 12}}}, {0, 2, 1},
+        {2, 1, 2}, {{{6, 5}}, {{12, 11}}});
   }
 
   template <typename IndexT, typename DataT>
@@ -137,9 +134,9 @@ class DynamicSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -163,9 +160,9 @@ class DynamicSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -189,9 +186,9 @@ class DynamicSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -359,9 +356,9 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -390,9 +387,9 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -421,9 +418,9 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
                        ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
                        .ValueOrDie());
 
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer dynamic slice start indices parameter.
-    ComputationDataHandle starts;
+    XlaOp starts;
     std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
         slice_starts, 0, "slice_starts", &builder, &starts);
     // Build dynamic slice computation.
@@ -474,13 +471,13 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
     }
 
     // Build dynamic slice computation.
-    ComputationBuilder builder(client_, TestName());
+    XlaBuilder builder(TestName());
     // Initialize and transfer input parameter.
-    ComputationDataHandle input;
+    XlaOp input;
     std::unique_ptr<GlobalData> input_data =
         CreateR3Parameter<T>(input_values, 0, "input_values", &builder, &input);
     // Initialize and transfer update parameter.
-    ComputationDataHandle update;
+    XlaOp update;
     std::unique_ptr<GlobalData> update_data = CreateR3Parameter<T>(
         update_values, 1, "update_values", &builder, &update);
     auto starts = builder.ConstantR1<int32>({index, 0, 0});
@@ -672,7 +669,7 @@ void BM_DynamicSlice(int num_iters) {
       TransferManager::GetForPlatform(platform).ValueOrDie();
   int device_ordinal = client->default_device_ordinal();
 
-  ComputationBuilder builder(client, "DynamicSlice");
+  XlaBuilder builder("DynamicSlice");
 
   // Create input as a constant: shape [1, 2, 3, 4]
   auto input_literal = Literal::CreateR4(
-- 
GitLab


From 5dcbb54070e9c4d7d2bb5ce4b1f488ac1ef22333 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 15:01:09 -0700
Subject: [PATCH 871/960] Internal change

PiperOrigin-RevId: 191505262
---
 tensorflow/contrib/BUILD | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/BUILD b/tensorflow/contrib/BUILD
index 6964cbd564..bf69144ad8 100644
--- a/tensorflow/contrib/BUILD
+++ b/tensorflow/contrib/BUILD
@@ -117,7 +117,7 @@ py_library(
         ],
         "//conditions:default": [],
     }) + if_not_windows([
-        # for :contrib_py
+        "//tensorflow/contrib/ffmpeg:ffmpeg_ops_py",
         "//tensorflow/contrib/lite/python:lite",  # unix dependency, need to fix code
     ]),
 )
-- 
GitLab


From a764216776465a5385596ca83af6edf3da72c504 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 15:03:44 -0700
Subject: [PATCH 872/960] Accept toco ModelFlags protos on the command line.

PiperOrigin-RevId: 191505886
---
 tensorflow/contrib/lite/toco/args.h           |  1 +
 .../contrib/lite/toco/model_cmdline_flags.cc  | 24 ++++++++++++++++++-
 2 files changed, 24 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/args.h b/tensorflow/contrib/lite/toco/args.h
index 52c789293c..39e49bc347 100644
--- a/tensorflow/contrib/lite/toco/args.h
+++ b/tensorflow/contrib/lite/toco/args.h
@@ -211,6 +211,7 @@ struct ParsedModelFlags {
   Arg<bool> allow_nonexistent_arrays = Arg<bool>(false);
   Arg<bool> allow_nonascii_arrays = Arg<bool>(false);
   Arg<string> arrays_extra_info_file;
+  Arg<string> model_flags_file;
 };
 
 // Flags that describe the operation you would like to do (what conversion
diff --git a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
index 4264f21c76..245eb52444 100644
--- a/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
+++ b/tensorflow/contrib/lite/toco/model_cmdline_flags.cc
@@ -160,6 +160,11 @@ bool ParseModelFlagsFromCommandLineFlags(
           "Path to an optional file containing a serialized ArraysExtraInfo "
           "proto allowing to pass extra information about arrays not specified "
           "in the input model file, such as extra MinMax information."),
+      Flag("model_flags_file", parsed_flags.model_flags_file.bind(),
+           parsed_flags.model_flags_file.default_value(),
+           "Path to an optional file containing a serialized ModelFlags proto. "
+           "Options specified on the command line will override the values in "
+           "the proto."),
   };
   bool asked_for_help =
       *argc == 2 && (!strcmp(argv[1], "--help") || !strcmp(argv[1], "-help"));
@@ -182,7 +187,24 @@ void ReadModelFlagsFromCommandLineFlags(
     const ParsedModelFlags& parsed_model_flags, ModelFlags* model_flags) {
   toco::port::CheckInitGoogleIsDone("InitGoogle is not done yet");
 
-// "batch" flag only exists internally
+  // Load proto containing the initial model flags.
+  // Additional flags specified on the command line will overwrite the values.
+  if (parsed_model_flags.model_flags_file.specified()) {
+    string model_flags_file_contents;
+    QCHECK(port::file::GetContents(parsed_model_flags.model_flags_file.value(),
+                                   &model_flags_file_contents,
+                                   port::file::Defaults())
+               .ok())
+        << "Specified --model_flags_file="
+        << parsed_model_flags.model_flags_file.value()
+        << " was not found or could not be read";
+    QCHECK(ParseFromStringEitherTextOrBinary(model_flags_file_contents,
+                                             model_flags))
+        << "Specified --model_flags_file="
+        << parsed_model_flags.model_flags_file.value()
+        << " could not be parsed";
+  }
+
 #ifdef PLATFORM_GOOGLE
   CHECK(!((base::SpecifiedOnCommandLine("batch") &&
            parsed_model_flags.variable_batch.specified())))
-- 
GitLab


From 4681562607bf4001ecd61492f1e7567be9212c6f Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Tue, 3 Apr 2018 15:08:51 -0700
Subject: [PATCH 873/960] Implementing make_initializatable_iterator for the
 PrefetchToDeviceDataset.

PiperOrigin-RevId: 191506754
---
 .../kernel_tests/prefetching_ops_test.py      | 61 +++++++++++++++-
 .../data/python/ops/prefetching_ops.py        | 73 +++++++++++++++----
 2 files changed, 120 insertions(+), 14 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
index f2c57f92e2..4b50260670 100644
--- a/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/prefetching_ops_test.py
@@ -33,7 +33,7 @@ from tensorflow.python.ops import resource_variable_ops
 from tensorflow.python.platform import test
 
 
-class StagingAreaOpsTest(test.TestCase):
+class PrefetchingKernelsOpsTest(test.TestCase):
 
   def setUp(self):
     self._event = threading.Event()
@@ -200,6 +200,9 @@ class StagingAreaOpsTest(test.TestCase):
 
       sess.run(destroy_op)
 
+
+class PrefetchToDeviceTest(test.TestCase):
+
   def testPrefetchToDevice(self):
     host_dataset = dataset_ops.Dataset.range(10)
     device_dataset = host_dataset.apply(
@@ -279,6 +282,62 @@ class StagingAreaOpsTest(test.TestCase):
       with self.assertRaises(errors.OutOfRangeError):
         sess.run(next_element)
 
+  def testPrefetchToDeviceWithReInit(self):
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops.prefetch_to_device("/cpu:1"))
+
+    # NOTE(mrry): This device block creates the "host" dataset and iterator on
+    # /cpu:0, and ensures that the prefetching is across devices. In typical use
+    # this would not be necessary, because the GPU device would not support any
+    # of the dataset-related ops.
+    with ops.device("/cpu:0"):
+      iterator = device_dataset.make_initializable_iterator()
+
+    self.assertEqual(host_dataset.output_types, device_dataset.output_types)
+    self.assertEqual(host_dataset.output_types, iterator.output_types)
+    self.assertEqual(host_dataset.output_shapes, device_dataset.output_shapes)
+    self.assertEqual(host_dataset.output_shapes, iterator.output_shapes)
+    self.assertEqual(host_dataset.output_classes, device_dataset.output_classes)
+    self.assertEqual(host_dataset.output_classes, iterator.output_classes)
+
+    next_element = iterator.get_next()
+    self.assertEqual(dtypes.int64, next_element.dtype)
+    self.assertEqual([], next_element.shape)
+
+    worker_config = config_pb2.ConfigProto()
+    worker_config.device_count["CPU"] = 2
+    with self.test_session(config=worker_config) as sess:
+      sess.run(iterator.initializer)
+      for i in range(5):
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
+      for i in range(10):
+        self.assertEqual(i, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
+  def testPrefetchToDeviceGpuWithReInit(self):
+    if not test_util.is_gpu_available():
+      self.skipTest("No GPU available")
+
+    host_dataset = dataset_ops.Dataset.range(10)
+    device_dataset = host_dataset.apply(
+        prefetching_ops.prefetch_to_device("/gpu:0"))
+
+    iterator = device_dataset.make_initializable_iterator()
+    next_element = iterator.get_next()
+
+    with self.test_session() as sess:
+      sess.run(iterator.initializer)
+      for i in range(5):
+        self.assertEqual(i, sess.run(next_element))
+      sess.run(iterator.initializer)
+      for i in range(10):
+        self.assertEqual(i, sess.run(next_element))
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(next_element)
+
 
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/prefetching_ops.py b/tensorflow/contrib/data/python/ops/prefetching_ops.py
index 554bfaa2cf..77e23d0319 100644
--- a/tensorflow/contrib/data/python/ops/prefetching_ops.py
+++ b/tensorflow/contrib/data/python/ops/prefetching_ops.py
@@ -68,20 +68,48 @@ def function_buffering_resource_reset(function_buffer_resource, name=None):
 
 # pylint: disable=protected-access
 class _PrefetchToDeviceIterator(object):
-  """A replacement for @{tf.data.Iterator} that prefetches to another device."""
+  """A replacement for @{tf.data.Iterator} that prefetches to another device.
 
-  def __init__(self, input_dataset, device, buffer_size):
+  Args:
+    input_dataset: The input dataset
+    one_shot: If true, we make a one shot iterator that's already initialized.
+    device: A fully specified device string where we want to prefetch to
+    buffer_size: Size of the prefetching buffer.
+    shared_name: (Optional.) If non-empty, the returned iterator will be
+        shared under the given name across multiple sessions that share the
+        same devices (e.g. when using a remote server).
+
+  Returns:
+    An Iterator type object.
+  """
+
+  def __init__(self,
+               input_dataset,
+               one_shot,
+               device,
+               buffer_size,
+               shared_name=None):
     self._input_dataset = input_dataset
     self._get_next_call_count = 0
-    input_iterator = input_dataset.make_one_shot_iterator()
-    input_iterator_handle = input_iterator.string_handle()
+    self._one_shot = one_shot
+    if shared_name is None:
+      shared_name = ""
+
+    if self._one_shot:
+      self._input_iterator = input_dataset.make_one_shot_iterator()
+    else:
+      self._input_iterator = iterator_ops.Iterator.from_structure(
+          self._input_dataset.output_types, self._input_dataset.output_shapes,
+          shared_name, self._input_dataset.output_classes)
+    input_iterator_handle = self._input_iterator.string_handle()
 
     @function.Defun(dtypes.string)
     def _prefetch_fn(handle):
       """Prefetches one element from `input_iterator`."""
       remote_iterator = iterator_ops.Iterator.from_string_handle(
-          handle, input_iterator.output_types, input_iterator.output_shapes,
-          input_iterator.output_classes)
+          handle, self._input_iterator.output_types,
+          self._input_iterator.output_shapes,
+          self._input_iterator.output_classes)
       ret = remote_iterator.get_next()
 
       # Convert any `SparseTensorValue`s to `SparseTensor`s.
@@ -101,9 +129,16 @@ class _PrefetchToDeviceIterator(object):
       self._buffering_resource = function_buffering_resource(
           f=_prefetch_fn,
           target_device=gen_dataset_ops.iterator_get_device(
-              input_iterator._iterator_resource),
+              self._input_iterator._iterator_resource),
           string_arg=input_iterator_handle,
-          buffer_size=buffer_size)
+          buffer_size=buffer_size,
+          shared_name=shared_name)
+
+    if not self._one_shot:
+      reset_op = function_buffering_resource_reset(self._buffering_resource)
+      with ops.control_dependencies([reset_op]):
+        self._initializer = self._input_iterator.make_initializer(
+            self._input_dataset)
 
   def get_next(self, name=None):
     """See @{tf.data.Iterator.get_next}."""
@@ -127,6 +162,12 @@ class _PrefetchToDeviceIterator(object):
 
     return ret
 
+  @property
+  def initializer(self):
+    if self._one_shot:
+      raise NotImplementedError("Can't initialize a one_shot_iterator")
+    return self._initializer
+
   @property
   def output_classes(self):
     return self._input_dataset.output_classes
@@ -150,13 +191,19 @@ class _PrefetchToDeviceDataset(dataset_ops.Dataset):
     self._buffer_size = buffer_size if buffer_size is not None else 1
 
   def make_one_shot_iterator(self):
-    return _PrefetchToDeviceIterator(self._input_dataset, self._device,
-                                     self._buffer_size)
+    return _PrefetchToDeviceIterator(
+        self._input_dataset,
+        one_shot=True,
+        device=self._device,
+        buffer_size=self._buffer_size)
 
   def make_initializable_iterator(self, shared_name=None):
-    raise NotImplementedError("`prefetch_to_device()` is not currently "
-                              "compatible with initializable iterators. Use "
-                              "`make_one_shot_iterator()` instead.")
+    return _PrefetchToDeviceIterator(
+        self._input_dataset,
+        one_shot=False,
+        device=self._device,
+        buffer_size=self._buffer_size,
+        shared_name=shared_name)
 
   def _as_variant_tensor(self):
     # TODO(mrry): Raise this error earlier (e.g. when one of the Dataset
-- 
GitLab


From 97a7df890632b349d0b179f418a2d17ae812f8ce Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 15:20:58 -0700
Subject: [PATCH 874/960] Remove unused #includes

PiperOrigin-RevId: 191508478
---
 tensorflow/core/platform/default/tracing_impl.h | 1 -
 tensorflow/core/platform/denormal.cc            | 3 ++-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/platform/default/tracing_impl.h b/tensorflow/core/platform/default/tracing_impl.h
index e813e4a17a..7834548896 100644
--- a/tensorflow/core/platform/default/tracing_impl.h
+++ b/tensorflow/core/platform/default/tracing_impl.h
@@ -22,7 +22,6 @@ limitations under the License.
 // IWYU pragma: friend third_party/tensorflow/core/platform/tracing.h
 
 #include "tensorflow/core/lib/core/status.h"
-#include "tensorflow/core/lib/core/threadpool.h"
 #include "tensorflow/core/lib/random/random.h"
 #include "tensorflow/core/platform/tracing.h"
 
diff --git a/tensorflow/core/platform/denormal.cc b/tensorflow/core/platform/denormal.cc
index 3631d9ddf9..82cbc43b4f 100644
--- a/tensorflow/core/platform/denormal.cc
+++ b/tensorflow/core/platform/denormal.cc
@@ -13,8 +13,9 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <tuple>
+
 #include "tensorflow/core/platform/denormal.h"
-#include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/platform/cpu_info.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/platform.h"
-- 
GitLab


From 1b20395c19199aaf124b2468da0e1d935c659d7b Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Tue, 3 Apr 2018 15:25:16 -0700
Subject: [PATCH 875/960] Add bfloat16 to GPU types.

PiperOrigin-RevId: 191509090
---
 tensorflow/compiler/jit/xla_gpu_device.cc    | 5 +++--
 tensorflow/compiler/tests/build_defs.bzl     | 3 +--
 tensorflow/compiler/tf2xla/xla_op_registry.h | 4 ++--
 3 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/jit/xla_gpu_device.cc b/tensorflow/compiler/jit/xla_gpu_device.cc
index 84c0d5f51f..ac60423d95 100644
--- a/tensorflow/compiler/jit/xla_gpu_device.cc
+++ b/tensorflow/compiler/jit/xla_gpu_device.cc
@@ -62,8 +62,9 @@ REGISTER_LOCAL_DEVICE_FACTORY(DEVICE_XLA_GPU, XlaGpuDeviceFactory);
 
 // Kernel registrations
 
-constexpr std::array<DataType, 7> kAllXlaGpuTypes = {
-    {DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL}};
+constexpr std::array<DataType, 8> kAllXlaGpuTypes = {
+    {DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE, DT_COMPLEX64, DT_BOOL,
+     DT_BFLOAT16}};
 
 REGISTER_XLA_LAUNCH_KERNEL(DEVICE_XLA_GPU, XlaLocalLaunchOp, kAllXlaGpuTypes);
 REGISTER_XLA_DEVICE_KERNELS(DEVICE_XLA_GPU, kAllXlaGpuTypes);
diff --git a/tensorflow/compiler/tests/build_defs.bzl b/tensorflow/compiler/tests/build_defs.bzl
index 0528a5415d..a9db1c173d 100644
--- a/tensorflow/compiler/tests/build_defs.bzl
+++ b/tensorflow/compiler/tests/build_defs.bzl
@@ -56,7 +56,7 @@ def tf_xla_py_test(name, srcs=[], deps=[], tags=[], data=[], main=None,
     elif backend == "gpu":
       backend_args += [
           "--test_device=XLA_GPU",
-          "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64"
+          "--types=DT_FLOAT,DT_DOUBLE,DT_INT32,DT_INT64,DT_BOOL,DT_COMPLEX64,DT_BFLOAT16"
       ]
       backend_tags += ["requires-gpu-sm35"]
     elif backend in plugins:
@@ -89,4 +89,3 @@ def generate_backend_suites(backends=[]):
     backends = all_backends()
   for backend in backends:
     native.test_suite(name="%s_tests" % backend, tags=["tf_xla_%s" % backend])
-
diff --git a/tensorflow/compiler/tf2xla/xla_op_registry.h b/tensorflow/compiler/tf2xla/xla_op_registry.h
index da2a6c3e28..e255b01dd7 100644
--- a/tensorflow/compiler/tf2xla/xla_op_registry.h
+++ b/tensorflow/compiler/tf2xla/xla_op_registry.h
@@ -55,9 +55,9 @@ constexpr std::array<DataType, 9> kCpuAllTypes = {
     {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
      DT_COMPLEX64, DT_BOOL}};
 
-constexpr std::array<DataType, 9> kGpuAllTypes = {
+constexpr std::array<DataType, 10> kGpuAllTypes = {
     {DT_UINT32, DT_UINT64, DT_INT32, DT_INT64, DT_HALF, DT_FLOAT, DT_DOUBLE,
-     DT_COMPLEX64, DT_BOOL}};
+     DT_COMPLEX64, DT_BOOL, DT_BFLOAT16}};
 
 // Class that manages registrations of operators and devices for the XLA JIT.
 // Not thread-safe.
-- 
GitLab


From bfb63e0a6458ff66e67a79467beab854d7c9d69d Mon Sep 17 00:00:00 2001
From: Allen Lavoie <allenl@google.com>
Date: Tue, 3 Apr 2018 15:28:27 -0700
Subject: [PATCH 876/960] Checkpointable: Utility to split a dependency for
 saving/loading

Useful when a single op produces Tensors which should each be saved under
different objects (or when Tensors saved with many different objects need to be
restored together as inputs to a single op).

I plan to use this for cuDNN RNN saving, which currently relies heavily on name
matching to allow cuDNN cells to be swapped out for CPU-compatible RNN
cells. This change has just the utility and some tests for it.

PiperOrigin-RevId: 191509664
---
 .../eager/python/checkpointable_utils.py      | 113 ++++++++++++++++++
 .../eager/python/checkpointable_utils_test.py |  82 +++++++++++++
 2 files changed, 195 insertions(+)

diff --git a/tensorflow/contrib/eager/python/checkpointable_utils.py b/tensorflow/contrib/eager/python/checkpointable_utils.py
index 91a7aded11..34cb8d0e08 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils.py
@@ -19,6 +19,7 @@ from __future__ import print_function
 
 import abc
 import collections
+import functools
 import weakref
 
 from tensorflow.contrib.eager.proto import checkpointable_object_graph_pb2
@@ -867,3 +868,115 @@ class Checkpoint(core_checkpointable.Checkpointable):
     # initialization when executing eagerly.
     self._maybe_create_save_counter()
     return status
+
+
+class _CallbackSaveable(saver_lib.BaseSaverBuilder.SaveableObject):
+  """Wraps save and restore callbacks as a `SaveableObject`."""
+
+  def __init__(self, name, dtype, save_callback, restore_callback):
+    self._restore_callback = restore_callback
+    spec = saver_lib.BaseSaverBuilder.SaveSpec(
+        tensor=save_callback,
+        slice_spec="",
+        name=name,
+        dtype=dtype)
+    super(_CallbackSaveable, self).__init__(
+        save_callback, [spec], name)
+
+  def restore(self, restored_tensors, restored_shapes):
+    """Restore the same value into both variables."""
+    tensor, = restored_tensors
+    return self._restore_callback(tensor)
+
+
+class _SplitDependency(core_checkpointable.CheckpointableBase):
+  """Looks like a regular variable while synchronizing save/restores."""
+
+  def __init__(self, save_buffer, restore_buffer, name, dtype, num_components,
+               fill_save_buffer_fn, consume_restore_buffer_fn):
+    self._save_buffer = save_buffer
+    self._restore_buffer = restore_buffer
+    self._name = name
+    self._dtype = dtype
+    self._num_components = num_components
+    self._fill_save_buffer_fn = fill_save_buffer_fn
+    self._consume_restore_buffer_fn = consume_restore_buffer_fn
+
+  def _save(self):
+    """Pull from the shared buffer, populating it if necessary."""
+    if self._name not in self._save_buffer:
+      if self._save_buffer:
+        raise AssertionError(
+            ("Split dependency %s (%s) unsynchronized. Split dependencies must "
+             "be saved together.") % (self._name, self))
+      self._fill_save_buffer_fn(self._save_buffer)
+    return self._save_buffer.pop(self._name)
+
+  def _restore(self, tensor):
+    """Push into the shared buffer, flushing it if necessary."""
+    if self._name in self._restore_buffer:
+      raise AssertionError(
+          ("Split dependency %s (%s) unsynchronized. Split dependencies must "
+           "be restored together.") % (self._name, self))
+    self._restore_buffer[self._name] = tensor
+    if len(self._restore_buffer) == self._num_components:
+      op = self._consume_restore_buffer_fn(self._restore_buffer)
+      self._restore_buffer.clear()
+      return op
+    else:
+      return control_flow_ops.no_op()
+
+  def _gather_saveables_for_checkpoint(self):
+    """Looks to Checkpointable like a regular variable."""
+    return {
+        core_checkpointable.VARIABLE_VALUE_KEY:
+        functools.partial(_CallbackSaveable,
+                          dtype=self._dtype,
+                          save_callback=self._save,
+                          restore_callback=self._restore)
+    }
+
+
+def split_dependency(component_names, component_dtypes,
+                     fill_save_buffer_fn, consume_restore_buffer_fn):
+  """Creates multiple dependencies with a synchronized save/restore.
+
+  Useful when a single op produces `Tensor`s which should each be saved under
+  different objects, or when `Tensor`s saved with many different objects need to
+  be restored together as inputs to a single op (i.e. an object which uses a
+  single fused op may be swapped out for a subgraph of objects, and these two
+  programs are checkpoint compatible).
+
+  Args:
+    component_names: A sequence of names for the split
+      dependencies. `fill_save_buffer_fn` must add these keys to the dictionary
+      it is passed, and `consume_restore_buffer_fn` will receive a dictionary
+      with these keys.
+    component_dtypes: Data types for the `Tensor`s being saved and restored, a
+      sequence corresponding to `component_names`.
+    fill_save_buffer_fn: A function which takes an empty dictionary as an
+      argument and adds `Tensor`s with `component_names` as keys. These
+      `Tensor`s will be saved as if they were individual variables.
+    consume_restore_buffer_fn: A function which takes a dictionary with
+      `component_names` as keys mapping to restored individual `Tensor`s and
+      returns a restore op (or if executing eagerly, runs the restoration and
+      may return `None`).
+
+  Returns:
+    A dictionary mapping from names to Checkpointable objects. If one is
+    reachable from an object as a dependency, the others should be too; adding
+    dependencies on some but not all of the objects will result in errors.
+  """
+  save_buffer = {}
+  restore_buffer = {}
+  split_dependencies = {}
+  for name, dtype in zip(component_names, component_dtypes):
+    split_dependencies[name] = _SplitDependency(
+        save_buffer=save_buffer,
+        restore_buffer=restore_buffer,
+        name=name,
+        dtype=dtype,
+        num_components=len(component_names),
+        fill_save_buffer_fn=fill_save_buffer_fn,
+        consume_restore_buffer_fn=consume_restore_buffer_fn)
+  return split_dependencies
diff --git a/tensorflow/contrib/eager/python/checkpointable_utils_test.py b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
index 5e1b64728a..891c093a0f 100644
--- a/tensorflow/contrib/eager/python/checkpointable_utils_test.py
+++ b/tensorflow/contrib/eager/python/checkpointable_utils_test.py
@@ -33,6 +33,7 @@ from tensorflow.python.framework import test_util
 from tensorflow.python.keras._impl.keras.engine import sequential
 from tensorflow.python.keras._impl.keras.engine import training
 from tensorflow.python.layers import core
+from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import init_ops
 from tensorflow.python.ops import resource_variable_ops
@@ -69,6 +70,87 @@ class MyModel(training.Model):
     return ret
 
 
+def _split_variable_closure(variable):
+  def _fill_save_buffer_fn(save_buffer):
+    save_buffer["first_half"] = variable[:2]
+    save_buffer["second_half"] = variable[2:]
+  return _fill_save_buffer_fn
+
+
+def _combine_variable_closure(variable):
+  def _consume_restore_buffer_fn(restore_buffer):
+    return variable.assign(
+        array_ops.concat([restore_buffer["first_half"],
+                          restore_buffer["second_half"]],
+                         axis=0))
+  return _consume_restore_buffer_fn
+
+
+class SaveTensorSlicesAsDeps(checkpointable.CheckpointableBase):
+
+  def __init__(self):
+    self.combined = resource_variable_ops.ResourceVariable([0., 0., 0., 0.])
+    split_dependencies = checkpointable_utils.split_dependency(
+        component_names=("first_half", "second_half"),
+        component_dtypes=(self.combined.dtype,) * 2,
+        fill_save_buffer_fn=_split_variable_closure(
+            self.combined),
+        consume_restore_buffer_fn=_combine_variable_closure(
+            self.combined))
+    for name, dep in split_dependencies.items():
+      self._track_checkpointable(dep, name=name)
+
+
+class HasRegularDeps(checkpointable.Checkpointable):
+
+  def __init__(self):
+    self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
+    self.second_half = resource_variable_ops.ResourceVariable([0., 0.])
+
+
+class OnlyOneDep(checkpointable.Checkpointable):
+
+  def __init__(self):
+    self.first_half = resource_variable_ops.ResourceVariable([0., 0.])
+
+
+class SplitTests(test.TestCase):
+
+  @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
+  def testSaveRestoreSplitDep(self):
+    save_checkpoint = checkpointable_utils.Checkpoint(
+        dep=SaveTensorSlicesAsDeps())
+    self.evaluate(save_checkpoint.dep.combined.assign([1., 2., 3., 4.]))
+    checkpoint_directory = self.get_temp_dir()
+    checkpoint_prefix = os.path.join(checkpoint_directory, "ckpt")
+    save_path = save_checkpoint.save(checkpoint_prefix)
+
+    regular_deps = HasRegularDeps()
+    regular_restore_checkpoint = checkpointable_utils.Checkpoint(
+        dep=regular_deps)
+    regular_restore_checkpoint.restore(
+        save_path).assert_consumed().run_restore_ops()
+    self.assertAllEqual([1., 2.], self.evaluate(regular_deps.first_half))
+    self.assertAllEqual([3., 4.], self.evaluate(regular_deps.second_half))
+
+    one_dep = OnlyOneDep()
+    one_dep_restore_checkpoint = checkpointable_utils.Checkpoint(dep=one_dep)
+    status = one_dep_restore_checkpoint.restore(save_path)
+    with self.assertRaises(AssertionError):
+      # Missing the second dependency.
+      status.assert_consumed()
+    status.run_restore_ops()
+    self.assertAllEqual([1., 2.], self.evaluate(one_dep.first_half))
+
+    restore_checkpoint = checkpointable_utils.Checkpoint()
+    status = restore_checkpoint.restore(save_path)
+    restore_checkpoint.dep = SaveTensorSlicesAsDeps()
+    status.assert_consumed().run_restore_ops()
+    self.assertAllEqual(
+        [1., 2., 3., 4.],
+        self.evaluate(restore_checkpoint.dep.combined))
+
+
 class InterfaceTests(test.TestCase):
 
   @test_util.run_in_graph_and_eager_modes(assert_no_eager_garbage=True)
-- 
GitLab


From 1948b74779e34e5ac608ef427b3409ca0a98c5f5 Mon Sep 17 00:00:00 2001
From: Francois Chollet <fchollet@google.com>
Date: Tue, 3 Apr 2018 15:39:02 -0700
Subject: [PATCH 877/960] Add ability to pass symbolic tensors as inputs and
 targets in calls to Model training and evaluation methods.

This also works for eager tensors, but due to a slicing behavior difference between eager tensors and Numpy arrays, we have to implement a workaround (with a performance cost).

PiperOrigin-RevId: 191511215
---
 .../python/keras/_impl/keras/backend.py       |  18 ++-
 .../keras/_impl/keras/engine/training.py      |   3 +
 .../_impl/keras/engine/training_eager.py      |  70 ++++++++---
 .../_impl/keras/engine/training_eager_test.py | 104 ++++++++++++++--
 .../keras/_impl/keras/engine/training_test.py | 115 ++++++++++++++++++
 .../_impl/keras/engine/training_utils.py      |  37 ++++--
 6 files changed, 312 insertions(+), 35 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/backend.py b/tensorflow/python/keras/_impl/keras/backend.py
index 7baf27642a..3aac6a9065 100644
--- a/tensorflow/python/keras/_impl/keras/backend.py
+++ b/tensorflow/python/keras/_impl/keras/backend.py
@@ -34,6 +34,7 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes as dtypes_module
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_util
 from tensorflow.python.layers import base as tf_base_layers
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import clip_ops
@@ -2795,6 +2796,8 @@ class Function(object):
     else:
       feed_dict = {}
 
+    session = get_session()
+    data_tensors_to_feed = []
     for tensor, value in zip(self.inputs, inputs):
       if value is None:
         continue
@@ -2803,9 +2806,20 @@ class Function(object):
         indices = np.concatenate((np.expand_dims(sparse_coo.row, 1),
                                   np.expand_dims(sparse_coo.col, 1)), 1)
         value = (indices, sparse_coo.data, sparse_coo.shape)
-      feed_dict[tensor] = value
+      elif tensor_util.is_tensor(value):
+        data_tensors_to_feed.append((tensor, value))
+      else:
+        feed_dict[tensor] = value
+
+    if data_tensors_to_feed:
+      # This is a *temporary* workaround (i.e. hack) to feed a symbolic tensor
+      # to `feed_dict`. It is very inefficient. It will be removed as soon
+      # as it becomes possible to pass symbolic tensors to `feed_dict`.
+      data_tensor_values = session.run([x[1] for x in data_tensors_to_feed])
+      for i, v in enumerate(data_tensor_values):
+        feed_dict[data_tensors_to_feed[i][0]] = v
+
     fetches = self.outputs + [self.updates_op] + self.fetches
-    session = get_session()
     updated = session.run(
         fetches=fetches, feed_dict=feed_dict, **self.session_kwargs)
     return updated[:len(self.outputs)]
diff --git a/tensorflow/python/keras/_impl/keras/engine/training.py b/tensorflow/python/keras/_impl/keras/engine/training.py
index 971245c162..71de657da8 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training.py
@@ -1181,6 +1181,9 @@ class Model(Network):
           batch_size=batch_size)
 
     elif validation_split and 0. < validation_split < 1.:
+      if training_utils.has_symbolic_tensors(x):
+        raise ValueError('If your data is in the form of symbolic tensors, '
+                         'you cannot use `validation_split`.')
       if hasattr(x[0], 'shape'):
         split_at = int(x[0].shape[0] * (1. - validation_split))
       else:
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_eager.py b/tensorflow/python/keras/_impl/keras/engine/training_eager.py
index 67858a578c..4cdb5f108a 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_eager.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_eager.py
@@ -31,9 +31,8 @@ from tensorflow.python.keras._impl.keras import callbacks as cbks
 from tensorflow.python.keras._impl.keras import losses
 from tensorflow.python.keras._impl.keras import metrics as metrics_module
 from tensorflow.python.keras._impl.keras.engine import training_utils
-from tensorflow.python.keras._impl.keras.utils.generic_utils import make_batches
-from tensorflow.python.keras._impl.keras.utils.generic_utils import Progbar
-from tensorflow.python.keras._impl.keras.utils.generic_utils import slice_arrays
+from tensorflow.python.keras._impl.keras.utils import generic_utils
+from tensorflow.python.ops import array_ops
 from tensorflow.python.platform import tf_logging as logging
 
 
@@ -173,6 +172,41 @@ def _model_loss(model, inputs, targets, sample_weights=None, training=False):
   return outs, total_loss, loss_metrics
 
 
+def slice_arrays(arrays, indices, contiguous=True):
+  """Slices batches out of provided arrays (workaround for eager tensors).
+
+  Unfortunately eager tensors don't have the same slicing behavior as
+  Numpy arrays (they folow  the same slicing behavior as symbolic TF tensors),
+  hence we cannot use `generic_utils.slice_arrays` directly
+  and we have to implement this workaround based on `concat`. This has a
+  performance cost.
+
+  Arguments:
+    arrays: Single array or list of arrays.
+    indices: List of indices in the array that should be included in the output
+      batch.
+    contiguous: Boolean flag indicating whether the indices are contiguous.
+
+  Returns:
+    Slice of data (either single array or list of arrays).
+  """
+  if any(tensor_util.is_tensor(x) for x in arrays):
+    converted_to_list = False
+    if not isinstance(arrays, list):
+      converted_to_list = True
+      arrays = [arrays]
+    if not contiguous:
+      entries = [[x[i:i + 1] for i in indices] for x in arrays]
+      slices = [array_ops.concat(x, axis=0) for x in entries]
+    else:
+      slices = [x[indices[0]:indices[-1] + 1] for x in arrays]
+    if converted_to_list:
+      slices = slices[0]
+    return slices
+  else:
+    return generic_utils.slice_arrays(arrays, indices)
+
+
 def _process_single_batch(model,
                           inputs,
                           targets,
@@ -270,9 +304,8 @@ def test_on_batch(model, inputs, targets, sample_weights=None):
       model, inputs, targets, sample_weights=sample_weights, training=False)
   if not isinstance(outs, list):
     outs = [outs]
-  metric_names, metrics_results = _eager_metrics_fn(
+  _, metrics_results = _eager_metrics_fn(
       model, outs, targets)
-  model.metrics_names.append(metric_names)
   if not isinstance(loss, list):
     loss = [loss]
   return loss + loss_metrics + metrics_results
@@ -328,6 +361,12 @@ def fit_loop(
   Raises:
     ValueError: In case of invalid argument values.
   """
+  if not batch_size:
+    raise ValueError('With eager execution, `batch_size` should be specified.')
+  if steps_per_epoch or validation_steps:
+    raise ValueError('With eager execution, `steps_per_epoch` and '
+                     '`validation_steps` are not valid arguments '
+                     '(set `batch_size` instead).')
   # Required for Eager mode
   with backend.learning_phase_scope(1):
     do_validation = False
@@ -410,15 +449,18 @@ def fit_loop(
       elif shuffle:
         np.random.shuffle(index_array)
 
-      batches = make_batches(num_train_samples, batch_size)
+      batches = generic_utils.make_batches(num_train_samples, batch_size)
 
       for batch_index, (batch_start, batch_end) in enumerate(batches):
         batch_ids = index_array[batch_start:batch_end]
         try:
-          inputs_batch = slice_arrays(inputs, batch_ids)
-          targets_batch = slice_arrays(targets, batch_ids)
+          inputs_batch = slice_arrays(inputs, batch_ids,
+                                      contiguous=not shuffle)
+          targets_batch = slice_arrays(targets, batch_ids,
+                                       contiguous=not shuffle)
           if sample_weights:
-            sample_weights_batch = slice_arrays(sample_weights, batch_ids)
+            sample_weights_batch = slice_arrays(sample_weights, batch_ids,
+                                                contiguous=not shuffle)
           else:
             sample_weights_batch = None
         except TypeError:
@@ -539,8 +581,8 @@ def test_loop(model, inputs, targets,
         feed_data, batch_size=batch_size, steps=steps, steps_name='steps')
     outs = []
     if verbose == 1:
-      progbar = Progbar(target=num_samples)
-    batches = make_batches(num_samples, batch_size)
+      progbar = generic_utils.Progbar(target=num_samples)
+    batches = generic_utils.make_batches(num_samples, batch_size)
     index_array = np.arange(num_samples)
     for batch_index, (batch_start, batch_end) in enumerate(batches):
       batch_ids = index_array[batch_start:batch_end]
@@ -620,12 +662,12 @@ def predict_loop(model, inputs,
         inputs, batch_size, steps, 'steps')
     if verbose == 1:
       if steps is not None:
-        progbar = Progbar(target=steps)
+        progbar = generic_utils.Progbar(target=steps)
       else:
-        progbar = Progbar(target=num_samples)
+        progbar = generic_utils.Progbar(target=num_samples)
 
     outs = []
-    batches = make_batches(num_samples, batch_size)
+    batches = generic_utils.make_batches(num_samples, batch_size)
     index_array = np.arange(num_samples)
     for batch_index, (batch_start, batch_end) in enumerate(batches):
       batch_ids = index_array[batch_start:batch_end]
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py b/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py
index 8848b393d5..6cdb6b0753 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_eager_test.py
@@ -18,7 +18,6 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import os
 import numpy as np
 
 from tensorflow.python.framework import ops
@@ -308,6 +307,100 @@ class TrainingTest(test.TestCase):
       model.compile(loss=None,
                     optimizer='rms')
 
+  def test_model_methods_with_eager_tensors_multi_io(self):
+    a = keras.layers.Input(shape=(3,), name='input_a')
+    b = keras.layers.Input(shape=(3,), name='input_b')
+
+    dense = keras.layers.Dense(4, name='dense')
+    c = dense(a)
+    d = dense(b)
+    e = keras.layers.Dropout(0.5, name='dropout')(c)
+
+    model = keras.models.Model([a, b], [d, e])
+
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    loss_weights = [1., 0.5]
+    metrics = ['mae']
+    model.compile(
+        optimizer,
+        loss,
+        metrics=metrics,
+        loss_weights=loss_weights,
+        sample_weight_mode=None)
+
+    input_a = keras.backend.zeros(shape=(10, 3))
+    input_b = keras.backend.zeros(shape=(10, 3))
+    target_d = keras.backend.zeros(shape=(10, 4))
+    target_e = keras.backend.zeros(shape=(10, 4))
+
+    model.fit(
+        [input_a, input_b], [target_d, target_e],
+        epochs=1,
+        batch_size=5,
+        verbose=0)
+    # Test: no shuffle.
+    model.fit(
+        [input_a, input_b], [target_d, target_e],
+        epochs=1,
+        batch_size=5,
+        verbose=0,
+        shuffle=False)
+    # Test: validation data.
+    model.fit([input_a, input_b], [target_d, target_e],
+              epochs=1, batch_size=2, verbose=0,
+              validation_data=([input_a, input_b], [target_d, target_e]))
+    model.train_on_batch([input_a, input_b], [target_d, target_e])
+    model.predict([input_a, input_b], batch_size=5)
+    model.evaluate([input_a, input_b], [target_d, target_e],
+                   batch_size=2, verbose=0)
+    model.test_on_batch([input_a, input_b], [target_d, target_e])
+
+    # Test: mix np and tensors.
+    input_b = np.zeros(shape=(10, 3)).astype('float32')
+    target_e = np.zeros(shape=(10, 4)).astype('float32')
+    model.fit(
+        [input_a, input_b], [target_d, target_e],
+        epochs=1,
+        batch_size=5,
+        verbose=0)
+    model.fit([input_a, input_b], [target_d, target_e],
+              epochs=1, batch_size=2, verbose=0,
+              validation_data=([input_a, input_b], [target_d, target_e]))
+    model.fit(
+        [input_a, input_b], [target_d, target_e],
+        epochs=1,
+        batch_size=5,
+        verbose=0,
+        shuffle=False)
+    model.train_on_batch([input_a, input_b], [target_d, target_e])
+    model.predict([input_a, input_b], batch_size=5)
+    model.evaluate([input_a, input_b], [target_d, target_e],
+                   batch_size=2, verbose=0)
+    model.test_on_batch([input_a, input_b], [target_d, target_e])
+
+  def test_model_methods_with_eager_tensors_single_io(self):
+    x = keras.layers.Input(shape=(3,), name='input')
+    y = keras.layers.Dense(4, name='dense')(x)
+    model = keras.Model(x, y)
+
+    optimizer = RMSPropOptimizer(learning_rate=0.001)
+    loss = 'mse'
+    metrics = ['mae']
+    model.compile(optimizer, loss, metrics=metrics)
+
+    inputs = keras.backend.zeros(shape=(10, 3))
+    targets = keras.backend.zeros(shape=(10, 4))
+
+    model.fit(inputs, targets, epochs=1, batch_size=2, verbose=0)
+    model.fit(inputs, targets, epochs=1, batch_size=3, verbose=0, shuffle=False)
+    model.fit(inputs, targets, epochs=1, batch_size=4, verbose=0,
+              validation_data=(inputs, targets))
+    model.evaluate(inputs, targets, batch_size=2, verbose=0)
+    model.predict(inputs, batch_size=2)
+    model.train_on_batch(inputs, targets)
+    model.test_on_batch(inputs, targets)
+
 
 class LossWeightingTest(test.TestCase):
 
@@ -533,14 +626,5 @@ class LossWeightingTest(test.TestCase):
 
 
 if __name__ == '__main__':
-  # Bazel sets these environment variables to very long paths.
-  # Tempfile uses them to create long paths, and in turn multiprocessing
-  # library tries to create sockets named after paths. Delete whatever bazel
-  # writes to these to avoid tests failing due to socket addresses being too
-  # long.
-  for var in ('TMPDIR', 'TMP', 'TEMP'):
-    if var in os.environ:
-      del os.environ[var]
-
   ops.enable_eager_execution()
   test.main()
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_test.py b/tensorflow/python/keras/_impl/keras/engine/training_test.py
index fd91dbba52..08fd26dd18 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_test.py
@@ -1117,6 +1117,121 @@ class TestTrainingUtils(test.TestCase):
 
 class TestTrainingWithDataTensors(test.TestCase):
 
+  def test_training_and_eval_methods_on_symbolic_tensors_single_io(self):
+    with self.test_session():
+      x = keras.layers.Input(shape=(3,), name='input')
+      y = keras.layers.Dense(4, name='dense')(x)
+      model = keras.Model(x, y)
+
+      optimizer = 'rmsprop'
+      loss = 'mse'
+      metrics = ['mae']
+      model.compile(optimizer, loss, metrics=metrics)
+
+      inputs = keras.backend.zeros(shape=(10, 3))
+      targets = keras.backend.zeros(shape=(10, 4))
+
+      model.fit(inputs, targets, epochs=1, steps_per_epoch=2, verbose=0)
+      model.evaluate(inputs, targets, steps=2, verbose=0)
+      model.predict(inputs, steps=2)
+      model.train_on_batch(inputs, targets)
+      model.test_on_batch(inputs, targets)
+      model.fit(inputs, targets,
+                epochs=1, steps_per_epoch=2, verbose=0,
+                validation_data=(inputs, targets), validation_steps=2)
+
+  def test_training_and_eval_methods_on_symbolic_tensors_multi_io(self):
+    with self.test_session():
+      a = keras.layers.Input(shape=(3,), name='input_a')
+      b = keras.layers.Input(shape=(3,), name='input_b')
+
+      dense = keras.layers.Dense(4, name='dense')
+      c = dense(a)
+      d = dense(b)
+      e = keras.layers.Dropout(0.5, name='dropout')(c)
+
+      model = keras.models.Model([a, b], [d, e])
+
+      optimizer = 'rmsprop'
+      loss = 'mse'
+      loss_weights = [1., 0.5]
+      metrics = ['mae']
+      model.compile(optimizer, loss, metrics=metrics, loss_weights=loss_weights)
+
+      input_a_tf = keras.backend.zeros(shape=(10, 3))
+      input_b_tf = keras.backend.zeros(shape=(10, 3))
+
+      output_d_tf = keras.backend.zeros(shape=(10, 4))
+      output_e_tf = keras.backend.zeros(shape=(10, 4))
+
+      model.fit(
+          [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+          epochs=1,
+          steps_per_epoch=2,
+          verbose=0)
+      with self.assertRaisesRegexp(ValueError,
+                                   'should specify the `steps_per_epoch`'):
+        model.fit(
+            [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+            epochs=1,
+            batch_size=5,
+            verbose=0)
+      model.train_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
+
+      # Test with dictionary inputs
+      model.fit(
+          {'input_a': input_a_tf,
+           'input_b': input_b_tf},
+          {'dense': output_d_tf,
+           'dropout': output_e_tf},
+          epochs=1,
+          steps_per_epoch=2,
+          verbose=0)
+      model.fit(
+          {'input_a': input_a_tf,
+           'input_b': input_b_tf},
+          {'dense': output_d_tf,
+           'dropout': output_e_tf},
+          validation_data=({'input_a': input_a_tf,
+                            'input_b': input_b_tf},
+                           {'dense': output_d_tf,
+                            'dropout': output_e_tf}),
+          epochs=1,
+          steps_per_epoch=2,
+          validation_steps=2,
+          verbose=0)
+      model.train_on_batch(
+          {'input_a': input_a_tf,
+           'input_b': input_b_tf},
+          {'dense': output_d_tf,
+           'dropout': output_e_tf})
+
+      # Test with validation data
+      model.fit(
+          [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+          validation_data=([input_a_tf, input_b_tf],
+                           [output_d_tf, output_e_tf]),
+          epochs=1,
+          steps_per_epoch=2,
+          validation_steps=2,
+          verbose=0)
+      # Test with validation split
+      with self.assertRaisesRegexp(ValueError,
+                                   'you cannot use `validation_split`'):
+        model.fit(
+            [input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+            epochs=2,
+            steps_per_epoch=2,
+            verbose=0,
+            validation_split=0.2,
+            validation_steps=2)
+
+      # Test evaluation / prediction methods
+      model.evaluate([input_a_tf, input_b_tf], [output_d_tf, output_e_tf],
+                     steps=2, verbose=0)
+      model.predict([input_a_tf, input_b_tf], steps=2)
+      model.test_on_batch([input_a_tf, input_b_tf], [output_d_tf, output_e_tf])
+
   def test_model_with_input_feed_tensor(self):
     """We test building a model with a TF variable as input.
 
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_utils.py b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
index 105638ce10..76537b735f 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
@@ -22,6 +22,7 @@ import copy
 
 import numpy as np
 
+from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras import losses
@@ -64,15 +65,29 @@ def check_num_samples(ins,
     if batch_size is not None:
       raise ValueError(
           'If ' + steps_name + ' is set, the `batch_size` must be None.')
-  elif ins and hasattr(ins[0], 'shape'):
-    num_samples = ins[0].shape[0]
-  else:
+  if has_symbolic_tensors(ins) and steps is None:
+    raise ValueError('If your data is in the form of symbolic tensors, '
+                     'you should specify the `' + steps_name + '` argument '
+                     '(instead of the `batch_size` argument).')
+  if ins and hasattr(ins[0], 'shape'):
+    num_samples = int(ins[0].shape[0])
+  elif steps is None:
     raise ValueError(
         'Either the input data should have '
         'a defined shape, or ' + steps_name + ' should be specified.')
   return num_samples
 
 
+def standardize_single_array(x):
+  if x is None:
+    return None
+  elif tensor_util.is_tensor(x):
+    return x
+  elif x.ndim == 1:
+    x = np.expand_dims(x, 1)
+  return x
+
+
 def standardize_input_data(data,
                            names,
                            shapes=None,
@@ -130,9 +145,7 @@ def standardize_input_data(data,
   else:
     data = data.values if data.__class__.__name__ == 'DataFrame' else data
     data = [data]
-  data = [
-      np.expand_dims(x, 1) if x is not None and x.ndim == 1 else x for x in data
-  ]
+  data = [standardize_single_array(x) for x in data]
 
   if len(data) != len(names):
     if data and hasattr(data[0], 'shape'):
@@ -158,7 +171,7 @@ def standardize_input_data(data,
   # Check shapes compatibility.
   if shapes:
     for i in range(len(names)):
-      if shapes[i] is not None:
+      if shapes[i] is not None and not tensor_util.is_tensor(data[i]):
         data_shape = data[i].shape
         shape = shapes[i]
         if data[i].ndim != len(shape):
@@ -245,12 +258,13 @@ def check_array_lengths(inputs, targets, weights=None):
   """
 
   def set_of_lengths(x):
-    # return a set with the variation between
+    # Returns a set with the variation between
     # different shapes, with None => 0
     if x is None:
       return {}
     else:
-      return set([y.shape[0] for y in x if y is not None])
+      return set([y.shape[0] for y in x
+                  if y is not None and not tensor_util.is_tensor(y)])
 
   set_x = set_of_lengths(inputs)
   set_y = set_of_lengths(targets)
@@ -532,3 +546,8 @@ def standardize_weights(y,
     return weights
   else:
     return None
+
+
+def has_symbolic_tensors(ls):
+  return (any(tensor_util.is_tensor(v) for v in ls)
+          and not context.executing_eagerly())
-- 
GitLab


From fa047c0e5976bb92d91afedb5961a697cb3be13b Mon Sep 17 00:00:00 2001
From: Jon Shlens <shlens@google.com>
Date: Tue, 3 Apr 2018 15:55:59 -0700
Subject: [PATCH 878/960] Add Group Normalization to tf.contrib.layers.

# Example usage: NHWC
outputs = tf.contrib.layers.group_norm(inputs, groups=32, channels_axis=-1, reduction_axes=[-3, -2])
# Example usage: NCHW
outputs = tf.contrib.layers.group_norm(inputs, groups=32, channels_axis=-3, reduction_axes=[-2, -1])
PiperOrigin-RevId: 191513496
---
 tensorflow/contrib/layers/__init__.py         |   2 +
 .../layers/python/layers/normalization.py     | 195 +++++++++++++++
 .../python/layers/normalization_test.py       | 226 ++++++++++++++++++
 3 files changed, 423 insertions(+)

diff --git a/tensorflow/contrib/layers/__init__.py b/tensorflow/contrib/layers/__init__.py
index 337c9e06b8..00f03a111a 100644
--- a/tensorflow/contrib/layers/__init__.py
+++ b/tensorflow/contrib/layers/__init__.py
@@ -104,6 +104,7 @@ See the @{$python/contrib.layers} guide.
 @@infer_real_valued_columns
 @@sequence_input_from_feature_columns
 
+@@group_norm
 @@instance_norm
 """
 
@@ -122,6 +123,7 @@ _allowed_symbols = ['bias_add',
                     'conv3d',
                     'elu',
                     'feature_column',
+                    'group_norm',
                     'instance_norm',
                     'legacy_fully_connected',
                     'legacy_linear',
diff --git a/tensorflow/contrib/layers/python/layers/normalization.py b/tensorflow/contrib/layers/python/layers/normalization.py
index e7d4080ff7..c807ab0f2e 100644
--- a/tensorflow/contrib/layers/python/layers/normalization.py
+++ b/tensorflow/contrib/layers/python/layers/normalization.py
@@ -24,11 +24,13 @@ from tensorflow.contrib.layers.python.layers import utils
 from tensorflow.python.framework import ops
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import init_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import nn
 from tensorflow.python.ops import variable_scope
 
 
 __all__ = [
+    'group_norm',
     'instance_norm',
 ]
 
@@ -158,3 +160,196 @@ def instance_norm(inputs,
     if activation_fn is not None:
       outputs = activation_fn(outputs)
     return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
+
+
+@add_arg_scope
+def group_norm(inputs,
+               groups=32,
+               channels_axis=-1,
+               reduction_axes=(-3, -2),
+               center=True,
+               scale=True,
+               epsilon=1e-6,
+               activation_fn=None,
+               param_initializers=None,
+               reuse=None,
+               variables_collections=None,
+               outputs_collections=None,
+               trainable=True,
+               scope=None):
+  """Functional interface for the group normalization layer.
+
+  Reference: https://arxiv.org/abs/1803.08494.
+
+    "Group Normalization", Yuxin Wu, Kaiming He
+
+  Args:
+    inputs: A Tensor with at least 2 dimensions one which is channels. All
+     shape dimensions must be fully defined.
+    groups: Integer. Divide the channels into this number of groups over which
+      normalization statistics are computed. This number must be commensurate
+      with the number of channels in `inputs`.
+    channels_axis: An integer. Specifies index of channels axis which will be
+      broken into `groups`, each of which whose statistics will be computed
+      across. Must be mutually exclusive with `reduction_axes`. Preferred usage
+      is to specify negative integers to be agnostic as to whether a batch
+      dimension is included.
+    reduction_axes: Tuple of integers. Specifies dimensions over which
+       statistics will be accumulated. Must be mutually exclusive with
+       `channels_axis`. Statistics will not be accumulated across axes not
+       specified in `reduction_axes` nor `channel_axis`. Preferred usage is to
+       specify negative integers to be agnostic to whether a batch dimension is
+       included.
+
+      Some sample usage cases:
+        NHWC format: channels_axis=-1, reduction_axes=[-3, -2]
+        NCHW format: channels_axis=-3, reduction_axes=[-2, -1]
+
+    center: If True, add offset of `beta` to normalized tensor. If False, `beta`
+      is ignored.
+    scale: If True, multiply by `gamma`. If False, `gamma` is
+      not used. When the next layer is linear (also e.g. `nn.relu`), this can be
+      disabled since the scaling can be done by the next layer.
+    epsilon: Small float added to variance to avoid dividing by zero.
+    activation_fn: Activation function, default set to None to skip it and
+      maintain a linear activation.
+    param_initializers: Optional initializers for beta, gamma, moving mean and
+      moving variance.
+    reuse: Whether or not the layer and its variables should be reused. To be
+      able to reuse the layer scope must be given.
+    variables_collections: Optional collections for the variables.
+    outputs_collections: Collections to add the outputs.
+    trainable: If `True` also add variables to the graph collection
+      `GraphKeys.TRAINABLE_VARIABLES` (see `tf.Variable`).
+    scope: Optional scope for `variable_scope`.
+
+  Returns:
+    A `Tensor` representing the output of the operation.
+
+  Raises:
+    ValueError: If the rank of `inputs` is undefined.
+    ValueError: If rank or channels dimension of `inputs` is undefined.
+    ValueError: If number of groups is not commensurate with number of channels.
+    ValueError: If reduction_axes or channels_axis are out of bounds.
+    ValueError: If reduction_axes are not mutually exclusive with channels_axis.
+  """
+  # TODO(shlens): Support partially defined shapes for the inputs.
+  inputs = ops.convert_to_tensor(inputs)
+  original_shape = inputs.shape
+
+  if inputs.shape.ndims is None:
+    raise ValueError('Inputs %s has undefined rank.' % inputs.name)
+  if channels_axis > (inputs.shape.ndims - 1):
+    raise ValueError('Axis is out of bounds.')
+
+  # Standardize the channels_axis to be positive and identify # of channels.
+  if channels_axis < 0:
+    channels_axis = inputs.shape.ndims + channels_axis
+  channels = inputs.shape[channels_axis].value
+
+  if channels is None:
+    raise ValueError('Inputs %s has undefined channel dimension: %d.' % (
+        inputs.name, channels_axis))
+
+  # Standardize the reduction_axes to be positive.
+  reduction_axes = list(reduction_axes)
+  for i in range(len(reduction_axes)):
+    if reduction_axes[i] < 0:
+      reduction_axes[i] += inputs.shape.ndims
+
+  for a in reduction_axes:
+    if a > inputs.shape.ndims:
+      raise ValueError('Axis is out of bounds.')
+    if inputs.shape[a].value is None:
+      raise ValueError('Inputs %s has undefined dimensions %d.' % (
+          inputs.name, a))
+    if channels_axis == a:
+      raise ValueError('reduction_axis must be mutually exclusive '
+                       'with channels_axis')
+  if groups > channels:
+    raise ValueError('Invalid groups %d for %d channels.' % (groups, channels))
+  if channels % groups != 0:
+    raise ValueError('%d channels is not commensurate with %d groups.' %
+                     (channels, groups))
+
+  # Determine axes before channels. Some examples of common image formats:
+  #  'NCHW': before = [N], after = [HW]
+  #  'NHWC': before = [NHW], after = []
+  axes_before_channels = inputs.shape.as_list()[:channels_axis]
+  axes_after_channels = inputs.shape.as_list()[channels_axis+1:]
+
+  # Manually broadcast the parameters to conform to the number of groups.
+  params_shape_broadcast = ([1] * len(axes_before_channels) +
+                            [groups, channels // groups] +
+                            [1] * len(axes_after_channels))
+
+  # Reshape the input by the group within the channel dimension.
+  inputs_shape = (axes_before_channels + [groups, channels // groups] +
+                  axes_after_channels)
+  inputs = array_ops.reshape(inputs, inputs_shape)
+
+  # Determine the dimensions across which moments are calculated.
+  moments_axes = [channels_axis + 1]
+  for a in reduction_axes:
+    if a > channels_axis:
+      moments_axes.append(a + 1)
+    else:
+      moments_axes.append(a)
+
+  with variable_scope.variable_scope(
+      scope, 'GroupNorm', [inputs], reuse=reuse) as sc:
+    # Note that the params_shape is the number of channels always.
+    params_shape = [channels]
+
+    # Allocate parameters for the beta and gamma of the normalization.
+    beta, gamma = None, None
+    dtype = inputs.dtype.base_dtype
+    if param_initializers is None:
+      param_initializers = {}
+    if center:
+      beta_collections = utils.get_variable_collections(
+          variables_collections, 'beta')
+      beta_initializer = param_initializers.get(
+          'beta', init_ops.zeros_initializer())
+      beta = variables.model_variable('beta',
+                                      shape=params_shape,
+                                      dtype=dtype,
+                                      initializer=beta_initializer,
+                                      collections=beta_collections,
+                                      trainable=trainable)
+      beta = array_ops.reshape(beta, params_shape_broadcast)
+
+    if scale:
+      gamma_collections = utils.get_variable_collections(
+          variables_collections, 'gamma')
+      gamma_initializer = param_initializers.get(
+          'gamma', init_ops.ones_initializer())
+      gamma = variables.model_variable('gamma',
+                                       shape=params_shape,
+                                       dtype=dtype,
+                                       initializer=gamma_initializer,
+                                       collections=gamma_collections,
+                                       trainable=trainable)
+      gamma = array_ops.reshape(gamma, params_shape_broadcast)
+
+    # Calculate the moments.
+    mean, variance = nn.moments(inputs, moments_axes, keep_dims=True)
+
+    # Compute normalization.
+    # TODO(shlens): Fix nn.batch_normalization to handle the 5-D Tensor
+    # appropriately so that this operation may be faster.
+    gain = math_ops.rsqrt(variance + epsilon)
+    offset = -mean * gain
+    if gamma is not None:
+      gain *= gamma
+      offset *= gamma
+    if beta is not None:
+      offset += beta
+    outputs = inputs * gain + offset
+
+    # Collapse the groups into the channel dimension.
+    outputs = array_ops.reshape(outputs, original_shape)
+
+    if activation_fn is not None:
+      outputs = activation_fn(outputs)
+    return utils.collect_named_outputs(outputs_collections, sc.name, outputs)
diff --git a/tensorflow/contrib/layers/python/layers/normalization_test.py b/tensorflow/contrib/layers/python/layers/normalization_test.py
index 5cff1bf0eb..b6e96350db 100644
--- a/tensorflow/contrib/layers/python/layers/normalization_test.py
+++ b/tensorflow/contrib/layers/python/layers/normalization_test.py
@@ -166,5 +166,231 @@ class InstanceNormTest(test.TestCase):
   def testOutputBigInput5DNCHW(self):
     self.doOutputTest((1, 100, 100, 1, 1), 'NCHW', tol=1e-3)
 
+
+class GroupNormTest(test.TestCase):
+
+  def testInvalidGroupSize(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(5, 2, 10, 10))
+    with self.assertRaisesRegexp(ValueError,
+                                 'Invalid groups 10 for 2 channels.'):
+      normalization.group_norm(inputs, groups=10,
+                               reduction_axes=[-2, -1], channels_axis=-3)
+
+  def testBadCommensurateGroup(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(5, 4, 10, 10))
+    with self.assertRaisesRegexp(ValueError,
+                                 '4 channels is not commensurate with '
+                                 '3 groups.'):
+      normalization.group_norm(inputs, groups=3,
+                               reduction_axes=[-2, -1], channels_axis=-3)
+
+  def testAxisIsBad(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(1, 2, 4, 5))
+    with self.assertRaisesRegexp(ValueError,
+                                 'Axis is out of bounds.'):
+      normalization.group_norm(inputs, channels_axis=5)
+    with self.assertRaisesRegexp(ValueError,
+                                 'Axis is out of bounds.'):
+      normalization.group_norm(inputs, reduction_axes=[1, 5])
+
+  def testNotMutuallyExclusiveAxis(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(10, 32, 32, 32))
+    # Specify axis with negative values.
+    with self.assertRaisesRegexp(ValueError, 'mutually exclusive'):
+      normalization.group_norm(inputs, channels_axis=-2, reduction_axes=[-2])
+    # Specify axis with positive values.
+    with self.assertRaisesRegexp(ValueError, 'mutually exclusive'):
+      normalization.group_norm(inputs, channels_axis=1, reduction_axes=[1, 3])
+    # Specify axis with mixed positive and negative values.
+    with self.assertRaisesRegexp(ValueError, 'mutually exclusive'):
+      normalization.group_norm(inputs, channels_axis=-2, reduction_axes=[2])
+
+  def testUnknownShape(self):
+    inputs = array_ops.placeholder(dtypes.float32)
+    with self.assertRaisesRegexp(ValueError, 'undefined rank'):
+      normalization.group_norm(inputs)
+
+  def testParamsShapeNotFullyDefinedReductionAxes(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(1, 32, None, 4))
+    with self.assertRaisesRegexp(ValueError, 'undefined dimensions'):
+      normalization.group_norm(inputs)
+
+  def testParamsShapeNotFullyDefinedChannelsAxis(self):
+    inputs = array_ops.placeholder(dtypes.float32, shape=(1, 3, 4, None))
+    with self.assertRaisesRegexp(ValueError, 'undefined channel dimension'):
+      normalization.group_norm(inputs, channels_axis=-1,
+                               reduction_axes=[-3, -2])
+
+  def testCreateOp(self):
+    height, width, groups = 3, 3, 4
+    images = random_ops.random_uniform((5, height, width, 2*groups), seed=1)
+    output = normalization.group_norm(images, groups=groups, channels_axis=-1,
+                                      reduction_axes=[-3, -2])
+    print('name: ', output.op.name)
+    self.assertListEqual([5, height, width, 2*groups], output.shape.as_list())
+
+  def testCreateOpFloat64(self):
+    height, width, groups = 3, 3, 5
+    images = random_ops.random_uniform(
+        (5, height, width, 4*groups), dtype=dtypes.float64, seed=1)
+    output = normalization.group_norm(images, groups=groups)
+    self.assertEqual(dtypes.float64, output.dtype)
+    self.assertListEqual([5, height, width, 4*groups], output.shape.as_list())
+
+  def testCreateOpNoScaleCenter(self):
+    height, width, groups = 3, 3, 7
+    images = random_ops.random_uniform(
+        (5, height, width, 3*groups), dtype=dtypes.float32, seed=1)
+    output = normalization.group_norm(images, groups=groups, center=False,
+                                      scale=False)
+    self.assertListEqual([5, height, width, 3*groups], output.shape.as_list())
+    self.assertEqual(0, len(contrib_variables.get_variables_by_name('beta')))
+    self.assertEqual(0, len(contrib_variables.get_variables_by_name('gamma')))
+
+  def testCreateVariables_NHWC(self):
+    height, width = 3, 3
+    images = random_ops.random_uniform((5, height, width, 8), seed=1)
+    normalization.group_norm(images, groups=4,
+                             channels_axis=-1, reduction_axes=(-3, -2),
+                             center=True, scale=True)
+    beta = contrib_variables.get_variables_by_name('beta')[0]
+    gamma = contrib_variables.get_variables_by_name('gamma')[0]
+    self.assertEqual('GroupNorm/beta', beta.op.name)
+    self.assertEqual('GroupNorm/gamma', gamma.op.name)
+
+  def testCreateVariables_NCHW(self):
+    height, width, groups = 3, 3, 4
+    images = random_ops.random_uniform((5, 2*groups, height, width), seed=1)
+    normalization.group_norm(images, groups=4,
+                             channels_axis=-3, reduction_axes=(-2, -1),
+                             center=True, scale=True)
+    beta = contrib_variables.get_variables_by_name('beta')[0]
+    gamma = contrib_variables.get_variables_by_name('gamma')[0]
+    self.assertEqual('GroupNorm/beta', beta.op.name)
+    self.assertEqual('GroupNorm/gamma', gamma.op.name)
+
+  def testReuseVariables(self):
+    height, width = 3, 3
+    images = random_ops.random_uniform((5, height, width, 4), seed=1)
+    normalization.group_norm(images, groups=2, scale=True, scope='IN')
+    normalization.group_norm(images, groups=2, scale=True, scope='IN',
+                             reuse=True)
+    beta = contrib_variables.get_variables_by_name('beta')
+    gamma = contrib_variables.get_variables_by_name('gamma')
+    self.assertEqual(1, len(beta))
+    self.assertEqual(1, len(gamma))
+
+  def testValueCorrectWithReuseVars(self):
+    height, width = 3, 3
+    image_shape = (10, height, width, 4)
+    images = random_ops.random_uniform(image_shape, seed=1)
+    output_train = normalization.group_norm(images, groups=2, scope='IN')
+    output_eval = normalization.group_norm(images, groups=2, scope='IN',
+                                           reuse=True)
+    with self.test_session() as sess:
+      sess.run(variables.global_variables_initializer())
+      # output_train and output_eval should be the same.
+      train_np, eval_np = sess.run([output_train, output_eval])
+      self.assertAllClose(train_np, eval_np)
+
+  def doOutputTest(self, input_shape, channels_axis=None, reduction_axes=None,
+                   groups=2, tol=1e-2):
+    # Select the axis for the channel and the dimensions along which statistics
+    # are accumulated.
+    if channels_axis < 0:
+      channels_axis += len(input_shape)
+    reduced_axes = [channels_axis + 1]
+    for a in reduction_axes:
+      if a < 0:
+        a += len(input_shape)
+      if a < channels_axis:
+        reduced_axes.append(a)
+      else:
+        reduced_axes.append(a+1)
+    reduced_axes = tuple(reduced_axes)
+
+    # Calculate the final shape for the output Tensor.
+    axes_before_channels = input_shape[:channels_axis]
+    axes_after_channels = input_shape[channels_axis+1:]
+    channels = input_shape[channels_axis]
+    outputs_shape = (axes_before_channels + [groups, channels // groups] +
+                     axes_after_channels)
+
+    # Calculate the final shape for the output statistics.
+    reduced_shape = []
+    for i, a in enumerate(outputs_shape):
+      if i not in reduced_axes:
+        reduced_shape.append(a)
+
+    for mu in (0.0, 1e2):
+      for sigma in (1.0, 0.1):
+        # Determine shape of Tensor after normalization.
+        expected_mean = np.zeros(reduced_shape)
+        expected_var = np.ones(reduced_shape)
+
+        inputs = random_ops.random_uniform(input_shape, seed=0) * sigma + mu
+        output_op = normalization.group_norm(
+            inputs, groups=groups, center=False, scale=False,
+            channels_axis=channels_axis,
+            reduction_axes=reduction_axes)
+        with self.test_session() as sess:
+          sess.run(variables.global_variables_initializer())
+          outputs = sess.run(output_op)
+          # Make sure that there are no NaNs
+          self.assertFalse(np.isnan(outputs).any())
+
+          outputs = np.reshape(outputs, outputs_shape)
+          mean = np.mean(outputs, axis=reduced_axes)
+          var = np.var(outputs, axis=reduced_axes)
+          # The mean and variance of each example should be close to 0 and 1
+          # respectively.
+          self.assertAllClose(expected_mean, mean, rtol=tol, atol=tol)
+          self.assertAllClose(expected_var, var, rtol=tol, atol=tol)
+
+  def testOutputSmallInput4D_NHWC(self):
+    input_shape = [10, 10, 10, 30]
+    # Specify axes with positive values.
+    self.doOutputTest(input_shape, channels_axis=3, reduction_axes=[1, 2])
+    # Specify axes with negative values.
+    self.doOutputTest(input_shape, channels_axis=-1, reduction_axes=[-3, -2])
+
+  def testOutputSmallInput3D_NHWC(self):
+    input_shape = [10, 10, 30]
+    # Specify axes with positive values.
+    self.doOutputTest(input_shape, channels_axis=2, reduction_axes=[0, 1])
+    # Specify axes with negative values.
+    self.doOutputTest(input_shape, channels_axis=-1, reduction_axes=[-3, -2])
+
+  def testOutputSmallInput4D_NCHW(self):
+    input_shape = [10, 10, 10, 30]
+    # Specify axes with positive values.
+    self.doOutputTest(input_shape, channels_axis=1, reduction_axes=[2, 3])
+    # Specify axes with negative values.
+    self.doOutputTest(input_shape, channels_axis=-3, reduction_axes=[-2, -1])
+
+  def testOutputSmallInput3D_NCHW(self):
+    input_shape = [10, 10, 30]
+    # Specify axes with positive values.
+    self.doOutputTest(input_shape, channels_axis=0, reduction_axes=[1, 2])
+    # Specify axes with negative values.
+    self.doOutputTest(input_shape, channels_axis=-3, reduction_axes=[-2, -1])
+
+  def testOutputBigInput4D_NHWC(self):
+    self.doOutputTest([5, 100, 100, 1], channels_axis=3, reduction_axes=[1, 2],
+                      groups=1)
+
+  def testOutputBigInput4D_NCHW(self):
+    self.doOutputTest([1, 100, 100, 4], channels_axis=1, reduction_axes=[2, 3],
+                      groups=4)
+
+  def testOutputSmallInput2D_NC(self):
+    self.doOutputTest([10, 7*100], channels_axis=1, reduction_axes=[], groups=7)
+
+  def testOutputSmallInput5D_NCXXX(self):
+    self.doOutputTest([10, 10, 20, 40, 5],
+                      channels_axis=1,
+                      reduction_axes=[2, 3, 4],
+                      groups=5)
+
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From 9a0f96d0ba8770e8089bce9916de10d117b082e9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 15:58:06 -0700
Subject: [PATCH 879/960] Add forward_from_array to OpKernelContext::Params.

This is an optional reservation on the forwarding of a particular
input tensor to a particular output that will be used by future
optimizations.

PiperOrigin-RevId: 191513782
---
 tensorflow/core/common_runtime/executor.cc    | 34 ++++++---
 tensorflow/core/framework/op_kernel.cc        | 72 ++++++++++++++-----
 tensorflow/core/framework/op_kernel.h         | 42 +++++++++--
 tensorflow/core/kernels/assign_op.h           |  3 +-
 .../core/kernels/resource_variable_ops.cc     | 13 ++--
 5 files changed, 129 insertions(+), 35 deletions(-)

diff --git a/tensorflow/core/common_runtime/executor.cc b/tensorflow/core/common_runtime/executor.cc
index 195803fd7f..0c461a9ee9 100644
--- a/tensorflow/core/common_runtime/executor.cc
+++ b/tensorflow/core/common_runtime/executor.cc
@@ -258,6 +258,13 @@ struct NodeItem {
   // Return array of per-output allocator attributes.
   const AllocatorAttributes* output_attrs() const { return output_attr_base(); }
 
+  // Return array of expected input index from which each output should
+  // be forwarded:
+  // kNeverForward (-2) for DO NOT FORWARD (must allocate).
+  // kNoReservation (-1) for no expected forwarding.
+  // 0... for forward from that input.
+  const int* forward_from() const { return forward_from_base(); }
+
  private:
   friend class GraphView;
 
@@ -267,6 +274,7 @@ struct NodeItem {
   //   AllocatorAttributes output_attr[num_outputs];
   //   uint8               input_type[num_inputs];
   //   uint8               output_type[num_outputs];
+  //   int                 forward_from[num_outputs];
 
   // Return pointer to variable length section.
   char* var() const {
@@ -292,6 +300,13 @@ struct NodeItem {
         sizeof(AllocatorAttributes) * num_outputs + sizeof(uint8) * num_inputs);
   }
 
+  int* forward_from_base() const {
+    return reinterpret_cast<int*>(var() + sizeof(EdgeInfo) * num_output_edges +
+                                  sizeof(AllocatorAttributes) * num_outputs +
+                                  sizeof(uint8) * num_inputs +
+                                  sizeof(uint8) * num_outputs);
+  }
+
   TF_DISALLOW_COPY_AND_ASSIGN(NodeItem);
 };
 
@@ -466,7 +481,8 @@ size_t GraphView::NodeItemBytes(const Node* n) {
       + num_output_edges * sizeof(EdgeInfo)        // output_edges[...]
       + num_outputs * sizeof(AllocatorAttributes)  // output_attr[...]
       + num_inputs * sizeof(uint8)                 // input_type[num_inputs]
-      + num_outputs * sizeof(uint8);               // output_type[num_outputs]
+      + num_outputs * sizeof(uint8)                // output_type[num_outputs]
+      + num_outputs * sizeof(int);                 // forward_from[num_outputs]
   static constexpr size_t kItemAlignment = sizeof(NodeItem*);
   static_assert(kItemAlignment % alignof(NodeItem) == 0,
                 "NodeItem must be aligned with kItemAlignment");
@@ -737,8 +753,8 @@ Status InferAllocAttr(const Node* n, const Node* dst,
       VLOG(2) << "node " << n->name() << " is the sink of an RPC in";
     } else if ((local_dev_name.type == "CPU" || n->IsHostRecv()) &&
                parsed_src_name.type != "CPU") {
-      // Value is going to be the sink of a local DMA from GPU to CPU (or other
-      // types of accelerators).
+      // Value is going to be the sink of a local DMA from GPU to CPU (or
+      // other types of accelerators).
       attr->set_gpu_compatible(true);
       VLOG(2) << "node " << n->name() << " is the sink of a gpu->cpu copy";
     } else {
@@ -1091,7 +1107,8 @@ class ExecutorState {
     void ActivateLoopInvs(const GraphView* gview, int64 iter,
                           TaggedNodeSeq* ready) EXCLUSIVE_LOCKS_REQUIRED(mu);
 
-    // Add a new loop invariant and make it available to all active iterations.
+    // Add a new loop invariant and make it available to all active
+    // iterations.
     void AddLoopInv(const NodeItem* item, const Entry& value,
                     TaggedNodeSeq* ready) EXCLUSIVE_LOCKS_REQUIRED(mu);
 
@@ -1148,8 +1165,8 @@ class ExecutorState {
         if (front_index_ == ready_.size()) {
           ready_.clear();
         } else {
-          // Lots of unused entries at beginning of vector: move everything down
-          // to start of vector.
+          // Lots of unused entries at beginning of vector: move everything
+          // down to start of vector.
           ready_.erase(ready_.begin(), ready_.begin() + front_index_);
         }
         front_index_ = 0;
@@ -1597,6 +1614,7 @@ void ExecutorState::Process(TaggedNode tagged_node, int64 scheduled_usec) {
       params.frame_iter = FrameAndIter(input_frame->frame_id, input_iter);
       params.is_input_dead = is_input_dead;
       params.output_attr_array = item.output_attrs();
+      params.forward_from_array = nullptr;  // later: item.forward_from();
 
       if (item.kernel_is_async) {
         // Asynchronous computes.
@@ -2605,7 +2623,7 @@ void ExecutorImpl::RunAsync(const Args& args, DoneCallback done) {
   (new ExecutorState(args, this))->RunAsync(std::move(done));
 }
 
-}  // end namespace
+}  // namespace
 
 Status NewLocalExecutor(const LocalExecutorParams& params,
                         std::unique_ptr<const Graph> graph,
@@ -2631,4 +2649,4 @@ Status CreateNonCachedKernel(Device* device, FunctionLibraryRuntime* flib,
 
 void DeleteNonCachedKernel(OpKernel* kernel) { delete kernel; }
 
-}  // end namespace tensorflow
+}  // namespace tensorflow
diff --git a/tensorflow/core/framework/op_kernel.cc b/tensorflow/core/framework/op_kernel.cc
index 6ba196cc34..cfde1e8ea3 100644
--- a/tensorflow/core/framework/op_kernel.cc
+++ b/tensorflow/core/framework/op_kernel.cc
@@ -96,7 +96,7 @@ OpKernel::OpKernel(OpKernelConstruction* context,
       output_memory_types_(context->output_memory_types().begin(),
                            context->output_memory_types().end()),
       graph_def_version_(context->graph_def_version()),
-      is_internal_(str_util::StartsWith(type_string(), "_")),
+      is_internal_(StringPiece(type_string()).starts_with("_")),
       input_name_map_(context->num_inputs()),
       output_name_map_(context->num_outputs()) {
   OP_REQUIRES_OK(context,
@@ -365,7 +365,7 @@ Status OpKernelContext::input_ref_mutex(StringPiece name, mutex** out_mutex) {
 
 const Tensor& OpKernelContext::input(int index) {
   DCHECK_GE(index, 0);
-  DCHECK_LT(index, num_inputs());
+  DCHECK_LT(index, num_inputs()) << " name: " << op_kernel().name();
   DCHECK(!input_is_ref(index));
   const Tensor& tensor = *((*params_->inputs)[index].tensor);
   record_tensor_reference(tensor);
@@ -420,8 +420,8 @@ bool OpKernelContext::forward_input_to_output_with_shape(
                                ? AllocatorAttributes()
                                : output_alloc_attr(output_index);
   std::unique_ptr<Tensor> new_tensor = forward_input(
-      input_index, expected_output_dtype(output_index), output_shape,
-      output_memory_type(output_index), output_attr);
+      input_index, output_index, expected_output_dtype(output_index),
+      output_shape, output_memory_type(output_index), output_attr);
   if (new_tensor != nullptr) {
     // Transfer ownership to the output slot in OpKernelContext.
     outputs_[output_index] = TensorValue(new_tensor.release());
@@ -461,35 +461,66 @@ Status OpKernelContext::forward_input_to_output_with_shape(
 }
 
 std::unique_ptr<Tensor> OpKernelContext::forward_input(
-    int input_index, DataType output_dtype, const TensorShape& output_shape,
-    MemoryType output_memory_type, const AllocatorAttributes& output_attr) {
+    int input_index, int output_index, DataType output_dtype,
+    const TensorShape& output_shape, MemoryType output_memory_type,
+    const AllocatorAttributes& output_attr) {
   DCHECK_GE(input_index, 0);
   DCHECK_LT(input_index, num_inputs());
   const TensorValue& input = (*params_->inputs)[input_index];
-  // Check that input tensor exists, is not a ref, and has no other consumers.
-  if (input.tensor == nullptr || input.is_ref() || !input->RefCountIsOne()) {
+  // Check whether at graph construction time this output was marked
+  // either for no forwarding or with a reservation for this input.
+  // If it's reserved for this input we'll skip the refcount and
+  // AllocatorAttribute checks.
+  // TODO(tucker): Maybe we should skip all of the checks?
+  bool never_forward =
+      (params_->forward_from_array != nullptr && output_index >= 0 &&
+       params_->forward_from_array[output_index] == Params::kNeverForward);
+  if (never_forward) return nullptr;
+  bool forward_expected =
+      (params_->forward_from_array != nullptr && output_index >= 0 &&
+       params_->forward_from_array[output_index] == input_index);
+  if (!forward_expected && params_->forward_from_array != nullptr) {
+    // Check for possibly conflicting forward.
+    for (int i = 0; i < num_outputs(); ++i) {
+      if (params_->forward_from_array[i] == input_index) {
+        // This input is reserved for output i.
+        return nullptr;
+      }
+    }
+  }
+  // Check that input tensor exists and is not a ref.
+  if (input.tensor == nullptr || input.is_ref()) {
+    CHECK(!forward_expected);
     return nullptr;
   }
   // Check that input type matches.
   if (input_dtype(input_index) != output_dtype) {
+    CHECK(!forward_expected);
     return nullptr;
   }
   // Check that the input and output sizes are compatible.
   if (input.tensor->shape().num_elements() != output_shape.num_elements()) {
+    CHECK(!forward_expected);
     return nullptr;
   }
   // Check that input and output memory types match, i.e.
   // that they either both live in host or both live in device memory.
   if (input_memory_type(input_index) != output_memory_type) {
+    CHECK(!forward_expected);
     return nullptr;
   }
-  // Check that output allocator attributes are not more restrictive than
-  // input allocator attributes.
-  const auto input_attr = params_->input_alloc_attrs == nullptr
-                              ? AllocatorAttributes()
-                              : input_alloc_attr(input_index);
-  if (!output_attr.IsEqualOrLessRestrictiveThan(input_attr)) {
-    return nullptr;
+  if (!forward_expected) {
+    if (!input->RefCountIsOne()) {
+      return nullptr;
+    }
+    // Check that output allocator attributes are not more restrictive than
+    // input allocator attributes.
+    const auto input_attr = params_->input_alloc_attrs == nullptr
+                                ? AllocatorAttributes()
+                                : input_alloc_attr(input_index);
+    if (!output_attr.IsEqualOrLessRestrictiveThan(input_attr)) {
+      return nullptr;
+    }
   }
   // TODO(rmlarsen): Use MakeUnique here. There is already a copy in
   // tensorflow/compiler/xla/ptr_util.h. Perhaps this should be part of
@@ -505,7 +536,8 @@ Status OpKernelContext::forward_input_or_allocate_temp(
     Tensor* out_temp) {
   for (int input_index : candidate_input_indices) {
     std::unique_ptr<Tensor> new_tensor =
-        forward_input(input_index, type, shape, DEVICE_MEMORY, allocator_attr);
+        forward_input(input_index, Params::kNoReservation /*output_index*/,
+                      type, shape, DEVICE_MEMORY, allocator_attr);
     if (new_tensor != nullptr) {
       *out_temp = std::move(*new_tensor);
       return Status::OK();
@@ -595,6 +627,14 @@ Status OpKernelContext::allocate_output(int index, const TensorShape& shape,
                                         Tensor** output) {
   DCHECK_GE(index, 0);
   DCHECK_LT(index, num_outputs());
+  bool forward_expected =
+      (params_->forward_from_array != nullptr && index >= 0 &&
+       params_->forward_from_array[index] >= 0);
+  if (forward_expected) {
+    return errors::Internal(
+        "Explicit allocate_output call where input forwarding required.  Try "
+        "turning off the ScopedAllocator optimizer.");
+  }
   AllocatorAttributes attr = output_alloc_attr(index);
   return allocate_output(index, shape, output, attr);
 }
diff --git a/tensorflow/core/framework/op_kernel.h b/tensorflow/core/framework/op_kernel.h
index 2d97160830..67943377b9 100644
--- a/tensorflow/core/framework/op_kernel.h
+++ b/tensorflow/core/framework/op_kernel.h
@@ -64,10 +64,11 @@ class AsyncOpKernel;
 class CallFrameInterface;
 class FunctionLibraryRuntime;
 class OpKernelConstruction;  // declared below
-class OpKernelContext;       // declared below
+class OpKernelContext;       // declared below,
 class OpRegistryInterface;
 class ResourceMgr;
 class ScopedStepContainer;
+class CollectiveExecutor;
 class StepStatsCollector;
 
 class OpKernel {
@@ -532,6 +533,10 @@ class OpKernelContext {
     // computations running on other devices.
     Rendezvous* rendezvous = nullptr;
 
+    // Mechanism for executing a collective op that needs to coordinate
+    // with parallel instances runing on other devices.
+    CollectiveExecutor* collective_executor = nullptr;
+
     // The session state for this op.
     SessionState* session_state = nullptr;
 
@@ -565,6 +570,12 @@ class OpKernelContext {
 
     // TensorSliceReaderCache support.
     checkpoint::TensorSliceReaderCacheWrapper* slice_reader_cache = nullptr;
+
+    // Support for forwarding reservations (used by ScopedAllocator).
+    static const int kNeverForward = -2;
+    static const int kNoReservation = -1;
+    // Values in [0,...) represent reservations for the indexed output.
+    const int* forward_from_array = nullptr;
   };
 
   // params must outlive the OpKernelContext.
@@ -707,14 +718,31 @@ class OpKernelContext {
   //     input[input_index] are compatible with those given in dtype, shape,
   //     memory_type, and attr,
   //   * refcount on the underlying buffer is one.
+  //   * Either there is no forwarding reservation for either input_index
+  //     or output_index or the specified input is reserved for the specified
+  //     output. More precisely:
+  //
+  //     These cases mean neither input nor output has a reservation:
+  //        forward_from_array = nullptr
+  //     OR (input_index is not in forward_from_array AND
+  //         (output_index == kNoReservation OR
+  //          forward_from_array[output_index] == kNoReservation))
+  //
+  //     This case means that input_index is reserved for output_index:
+  //        forward_from_array[output_index] == input_index
+  //
+  //     This case means the output is reserved to always be allocated,
+  //     never assigned a forwarded input:
+  //        forward_from_array[output_index] == kNeverForward
+  //
   // Otherwise returns nullptr.
   // NOTE: For Cuda kernels that read inputs using the __ldg() intrinsic,
   // forwarding is only safe if there are no reads via __ldg() after writes
   // to the same address.
   std::unique_ptr<Tensor> forward_input(
-      int input_index, DataType dtype, const TensorShape& shape,
-      MemoryType memory_type,
-      const AllocatorAttributes& attr) TF_MUST_USE_RESULT;
+      int input_index, int output_index, DataType output_dtype,
+      const TensorShape& output_shape, MemoryType output_memory_type,
+      const AllocatorAttributes& output_attr) TF_MUST_USE_RESULT;
 
   // Tries to forward one of the inputs given in input_indices to
   // output[output_index]. If none of the given inputs can be forwarded, calls
@@ -934,6 +962,10 @@ class OpKernelContext {
   // Rendezvous Send() and Recv().
   Rendezvous* rendezvous() const { return params_->rendezvous; }
 
+  CollectiveExecutor* collective_executor() const {
+    return params_->collective_executor;
+  }
+
   // An op kernel can access the session state it belongs to.
   SessionState* session_state() const { return params_->session_state; }
 
@@ -1102,7 +1134,7 @@ class OpKernelContext {
 
   Status status_;
   friend class CollectiveExecutor;  // for access to params_
-  Params* params_;    // not owned
+  Params* params_;                  // not owned
   mutable mutex mu_;  // mutable so const accessors can acquire the lock
   gtl::InlinedVector<WrappedAllocator, 4> wrapped_allocators_ GUARDED_BY(mu_);
   gtl::InlinedVector<TensorValue, 4> outputs_;
diff --git a/tensorflow/core/kernels/assign_op.h b/tensorflow/core/kernels/assign_op.h
index a312e8e8a4..2ed1628bf1 100644
--- a/tensorflow/core/kernels/assign_op.h
+++ b/tensorflow/core/kernels/assign_op.h
@@ -77,7 +77,8 @@ class AssignOp : public OpKernel {
 
       // 1. Try to reuse the rhs.
       std::unique_ptr<Tensor> input_alias = context->forward_input(
-          1, old_lhs.dtype(), old_lhs.shape(), DEVICE_MEMORY, attr);
+          1, OpKernelContext::Params::kNoReservation /*output_index*/,
+          old_lhs.dtype(), old_lhs.shape(), DEVICE_MEMORY, attr);
       if (input_alias != nullptr) {
         // Transfer ownership to the ref.
         context->replace_ref_input(0, *input_alias.release(),
diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 082b57b8e2..5c54609ee6 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -250,9 +250,9 @@ class AssignVariableOp : public OpKernel {
 
     // Copying is unnecessary if we are the last user of the value
     // tensor, we can just adopt the input tensor's buffer instead.
-    std::unique_ptr<Tensor> input_alias =
-        context->forward_input(1, dtype_, value.shape(), DEVICE_MEMORY, attr);
-
+    std::unique_ptr<Tensor> input_alias = context->forward_input(
+        1, OpKernelContext::Params::kNoReservation /*output_index*/, dtype_,
+        value.shape(), DEVICE_MEMORY, attr);
     mutex_lock ml(*variable->mu());
     variable->is_initialized = true;
     if (input_alias) {
@@ -370,11 +370,14 @@ class AssignVariableOp<Device, Variant> : public OpKernel {
     // Copying is unnecessary if we are the last user of the value
     // tensor, we can just adopt the input tensor's buffer instead.
     // Note that Variant objects themselves always reside on host.
-    std::unique_ptr<Tensor> input_alias =
-        context->forward_input(1, DT_VARIANT, value.shape(), HOST_MEMORY, attr);
+    std::unique_ptr<Tensor> input_alias = context->forward_input(
+        1, OpKernelContext::Params::kNoReservation /*output_index*/, DT_VARIANT,
+        value.shape(), HOST_MEMORY, attr);
 
     mutex_lock ml(*variable->mu());
     variable->is_initialized = true;
+    *variable->tensor() = Tensor(DT_VARIANT, value.shape());
+
     if (input_alias) {
       *variable->tensor() = *input_alias;
       return;
-- 
GitLab


From b3b96493e7ff08d7c026926f92f97c51ecaf9aa3 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 3 Apr 2018 16:18:19 -0700
Subject: [PATCH 880/960] [TF]Disable memory sanitizer for
 compilation_passes_test while the problem is being

PiperOrigin-RevId: 191516643
---
 tensorflow/compiler/jit/BUILD | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index 24aa203c00..e7d18e8351 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -347,6 +347,7 @@ tf_cc_test(
         "encapsulate_subgraphs_pass_test.cc",
         "mark_for_compilation_pass_test.cc",
     ],
+    tags = ["nomsan"],  # TODO: b/77543571
     deps = [
         ":common",
         ":compilation_passes",
-- 
GitLab


From 10055506747ced8773863f4695c78661e6059e32 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 3 Apr 2018 16:20:15 -0700
Subject: [PATCH 881/960] Fold more constants

PiperOrigin-RevId: 191516861
---
 tensorflow/core/grappler/optimizers/constant_folding.cc | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index 87052c7ba0..dd522aa228 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -747,10 +747,6 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const {
   if (op.find("Quantized") != string::npos || op.find("Sparse") == 0) {
     return false;
   }
-  if (node.attr().count("_XlaCompile") > 0 &&
-      node.attr().at("_XlaCompile").b()) {
-    return false;
-  }
 
   const OpDef* op_def = nullptr;
   Status status = OpRegistry::Global()->LookUpOpDef(node.op(), &op_def);
-- 
GitLab


From fa2a9473923bb6f20454623810a1b23cbeb7de1c Mon Sep 17 00:00:00 2001
From: Akshay Agrawal <akshayka@google.com>
Date: Tue, 3 Apr 2018 16:35:49 -0700
Subject: [PATCH 882/960] Minor touch-ups to the eager programming guide.

PiperOrigin-RevId: 191518771
---
 .../docs_src/programmers_guide/eager.md       | 43 +++++++++----------
 1 file changed, 21 insertions(+), 22 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
index 8db65737dc..414653c280 100644
--- a/tensorflow/docs_src/programmers_guide/eager.md
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -1,35 +1,34 @@
 # Eager Execution
 
 TensorFlow's eager execution is an imperative programming environment that
-evaluates operations immediately, without an extra graph-building step.
-Operations return concrete values instead of constructing a computational graph
-to run later. This makes it easy to get started with TensorFlow, debug models,
-reduce boilerplate code, and is fun! To follow along with this guide, run the
-code samples below in an interactive `python` interpreter.
-
-Eager execution supports most TensorFlow operations and GPU acceleration.
-Automatic differentiation uses a dynamically-constructed tape instead of a static
-graph to compute gradients. Eager execution is a flexible machine learning
-platform for research and experimentation that provides:
-
-* *An intuitive interface* —Structure your code naturally and use Python data
+evaluates operations immediately, without building graphs: operations return
+concrete values instead of constructing a computational graph to run later. This
+makes it easy to get started with TensorFlow and debug models, and it
+reduces boilerplate as well. To follow along with this guide, run the code
+samples below in an interactive `python` interpreter.
+
+Eager execution is a flexible machine learning platform for research and
+experimentation, providing:
+
+* *An intuitive interface*—Structure your code naturally and use Python data
   structures. Quickly iterate on small models and small data.
-* *Easier debugging* —Call ops directly to inspect running models and test
+* *Easier debugging*—Call ops directly to inspect running models and test
   changes. Use standard Python debugging tools for immediate error reporting.
-* *Natural control flow* —Use Python control flow instead of graph control flow,
-  including support for dynamic models.
+* *Natural control flow*—Use Python control flow instead of graph control
+  flow, simplifying the specification of dynamic models.
 
-For a collection of examples running in eager execution, see:
+Eager execution supports most TensorFlow operations and GPU acceleration. For a
+collection of examples running in eager execution, see:
 [tensorflow/contrib/eager/python/examples](https://github.com/tensorflow/tensorflow/tree/master/tensorflow/contrib/eager/python/examples).
 
-Note: Some models may experience increased overhead with eager execution enabled.
-Performance improvements are ongoing, but please
+Note: Some models may experience increased overhead with eager execution
+enabled. Performance improvements are ongoing, but please
 [file a bug](https://github.com/tensorflow/tensorflow/issues) if you find a
 problem and share your benchmarks.
 
 ## Setup and basic usage
 
-Upgrade to TensorFlow 1.7 to include updates for eager execution:
+Upgrade to the latest version of TensorFlow:
 
 ```
 $ pip install --upgrade tensorflow
@@ -110,7 +109,7 @@ environments and is useful for writing code to [work with graphs](#work_with_gra
 import tensorflow.contrib.eager as tfe
 ```
 
-## Eager training
+## Updating model parameters
 
 ### Automatic differentiation
 
@@ -124,7 +123,7 @@ operations for computing gradients later.
 not tracing. Since different operations can occur during each call, all
 forward-pass operations get recorded to a "tape". To compute the gradient, play
 the tape backwards and then discard. A particular `tfe.GradientTape` can only
-be computed once, subsequent calls throw a runtime error.
+compute one gradient; subsequent calls throw a runtime error.
 
 ```py
 w = tfe.Variable([[1.0]])
@@ -415,7 +414,7 @@ result = model(batch)
 This example uses the
 [dataset.py module](https://github.com/tensorflow/models/blob/master/official/mnist/dataset.py)
 from the
-[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist),
+[TensorFlow MNIST example](https://github.com/tensorflow/models/tree/master/official/mnist);
 download this file to your local directory. Run the following to download the
 MNIST data files to your working directory and prepare a `tf.data.Dataset`
 for training:
-- 
GitLab


From 0520c62b0955af879c48abf87df56f07c4c37b54 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 16:46:29 -0700
Subject: [PATCH 883/960] This is a cosmetic change to logging, which makes it
 easier to understand this message when TPU nodes have a huge number of
 devices attached.

PiperOrigin-RevId: 191520116
---
 tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
index eea57ed336..3ae350c7bb 100644
--- a/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
+++ b/tensorflow/contrib/tpu/python/tpu/tpu_system_metadata.py
@@ -120,7 +120,8 @@ def _query_tpu_system_metadata(master_address, run_config,
     logging.info('*** Num TPU Workers: %d', metadata.num_hosts)
     logging.info('*** Num TPU Cores Per Worker: %d',
                  metadata.num_of_cores_per_host)
-    logging.info('*** Available Devices: %s', metadata.devices)
+    for device in metadata.devices:
+      logging.info('*** Available Device: %s', device)
   else:
     logging.info('Failed to find TPU: %s', metadata)
   return metadata
-- 
GitLab


From 41f3f5707b105bab91683edc3cdd8497a8e0c71d Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 16:56:25 -0700
Subject: [PATCH 884/960] [TF:XLA] Add test for scalar dynamic-update-slice.

PiperOrigin-RevId: 191521429
---
 .../compiler/xla/tests/dynamic_ops_test.cc    | 43 +++++++++++++++++++
 1 file changed, 43 insertions(+)

diff --git a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
index c0a16ad288..5f00c34002 100644
--- a/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
+++ b/tensorflow/compiler/xla/tests/dynamic_ops_test.cc
@@ -278,6 +278,15 @@ XLA_TEST_F(DynamicSliceTest, Int32R3Pred) {
 
 class DynamicUpdateSliceTest : public ClientLibraryTestBase {
  protected:
+  template <typename IndexT, typename DataT>
+  void TestR0() {
+    // Disable algebraic simplifier, otherwise the op will be replaced by a
+    // constant.
+    execution_options_.mutable_debug_options()->add_xla_disable_hlo_passes(
+        "algsimp");
+    RunR0<IndexT, DataT>(0, 123, {}, 123);
+  }
+
   template <typename IndexT, typename DataT>
   void TestR1() {
     // Slice at dimension start.
@@ -338,6 +347,35 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
         {1, 2, 1}, {{{1, 2}, {3, 4}, {5, 6}}, {{7, 15}, {9, 10}, {11, 13}}});
   }
 
+  template <typename IndexT, typename DataT>
+  void RunR0(int input_value_int, int update_value_int,
+             const std::vector<IndexT> slice_starts, int expected_value_int) {
+    Literal input_value =
+        std::move(*Literal::CreateR0(input_value_int)
+                       ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
+                       .ValueOrDie());
+    Literal update_value =
+        std::move(*Literal::CreateR0(update_value_int)
+                       ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
+                       .ValueOrDie());
+    Literal expected_value =
+        std::move(*Literal::CreateR0(expected_value_int)
+                       ->Convert(primitive_util::NativeToPrimitiveType<DataT>())
+                       .ValueOrDie());
+
+    ComputationBuilder builder(client_, TestName());
+    // Initialize and transfer dynamic slice start indices parameter.
+    ComputationDataHandle starts;
+    std::unique_ptr<GlobalData> start_data = CreateR1Parameter<IndexT>(
+        slice_starts, 0, "slice_starts", &builder, &starts);
+    // Build dynamic slice computation.
+    auto input = builder.ConstantLiteral(input_value);
+    auto update = builder.ConstantLiteral(update_value);
+    builder.DynamicUpdateSlice(input, update, starts);
+    // Run computation and compare against expected values.
+    ComputeAndCompareLiteral(&builder, expected_value, {start_data.get()});
+  }
+
   template <typename IndexT, typename DataT>
   void RunR1(tensorflow::gtl::ArraySlice<int> input_values_int,
              tensorflow::gtl::ArraySlice<int> update_values_int,
@@ -497,6 +535,11 @@ class DynamicUpdateSliceTest : public ClientLibraryTestBase {
   }
 };
 
+XLA_TEST_F(DynamicUpdateSliceTest, Int32R0BF16) { TestR0<int32, bfloat16>(); }
+XLA_TEST_F(DynamicUpdateSliceTest, Int32R0) { TestR0<int32, float>(); }
+XLA_TEST_F(DynamicUpdateSliceTest, Int64R0) { TestR0<int64, float>(); }
+XLA_TEST_F(DynamicUpdateSliceTest, UInt64R0) { TestR0<uint64, float>(); }
+
 // TODO(b/71820067): The CPU parallel backend failed for this on 2018-01-10.
 XLA_TEST_F(DynamicUpdateSliceTest, DISABLED_ON_CPU_PARALLEL(Int32R1BF16)) {
   TestR1<int32, bfloat16>();
-- 
GitLab


From df2229540e9a1607193dcb8c83d5f3d7cf5d1a56 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 17:08:19 -0700
Subject: [PATCH 885/960] [XLA] Redesign: implement and test Send, Recv.

PiperOrigin-RevId: 191523125
---
 .../xla/client/xla_client/xla_builder.cc      | 37 ++++++++++++++++++-
 .../xla/tests/client_library_test_base.cc     |  6 +++
 .../xla/tests/client_library_test_base.h      | 13 +++++++
 3 files changed, 54 insertions(+), 2 deletions(-)

diff --git a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
index fe8ae77683..2d587cc3b9 100644
--- a/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
+++ b/tensorflow/compiler/xla/client/xla_client/xla_builder.cc
@@ -45,6 +45,7 @@ int64 GetUniqueId() {
 bool CanBeRoot(HloOpcode opcode) {
   switch (opcode) {
     case HloOpcode::kSend:
+    case HloOpcode::kSendDone:
     case HloOpcode::kOutfeed:
     case HloOpcode::kTrace:
       return false;
@@ -1127,11 +1128,43 @@ XlaOp XlaBuilder::ReducePrecision(const XlaOp& operand, const int exponent_bits,
 }
 
 void XlaBuilder::Send(const XlaOp& operand, const ChannelHandle& handle) {
-  UnimplementedOp();
+  NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    // Send instruction produces a tuple of {aliased operand, U32 context}.
+    TF_ASSIGN_OR_RETURN(const Shape& shape, GetShape(operand));
+    *instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeShape(U32, {})});
+    instr.set_channel_id(handle.handle());
+    TF_ASSIGN_OR_RETURN(
+        XlaOp send,
+        AddInstruction(std::move(instr), HloOpcode::kSend, {operand}));
+
+    HloInstructionProto send_done_instr;
+    *send_done_instr.mutable_shape() = ShapeUtil::MakeNil();
+    send_done_instr.set_channel_id(handle.handle());
+    return AddInstruction(std::move(send_done_instr), HloOpcode::kSendDone,
+                          {send});
+  });
 }
 
 XlaOp XlaBuilder::Recv(const Shape& shape, const ChannelHandle& handle) {
-  return UnimplementedOp();
+  return NoteErrorOrReturn([&]() -> StatusOr<XlaOp> {
+    HloInstructionProto instr;
+
+    // Recv instruction produces a tuple of {receive buffer, U32 context}.
+    *instr.mutable_shape() =
+        ShapeUtil::MakeTupleShape({shape, ShapeUtil::MakeShape(U32, {})});
+    instr.set_channel_id(handle.handle());
+    TF_ASSIGN_OR_RETURN(XlaOp recv,
+                        AddInstruction(std::move(instr), HloOpcode::kRecv, {}));
+
+    HloInstructionProto recv_done_instr;
+    *recv_done_instr.mutable_shape() = shape;
+    recv_done_instr.set_channel_id(handle.handle());
+    return AddInstruction(std::move(recv_done_instr), HloOpcode::kRecvDone,
+                          {recv});
+  });
 }
 
 StatusOr<bool> XlaBuilder::IsConstant(const XlaOp& operand,
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.cc b/tensorflow/compiler/xla/tests/client_library_test_base.cc
index 4a9faef1dc..17c6a83c1a 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.cc
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.cc
@@ -601,6 +601,12 @@ ComputationDataHandle ClientLibraryTestBase::CreateConstantFromLiteral(
       use_bfloat16_ ? *LiteralTestUtil::ConvertF32ToBF16(literal) : literal);
 }
 
+XlaOp ClientLibraryTestBase::CreateConstantFromLiteral(const Literal& literal,
+                                                       XlaBuilder* builder) {
+  return builder->ConstantLiteral(
+      use_bfloat16_ ? *LiteralTestUtil::ConvertF32ToBF16(literal) : literal);
+}
+
 template void ClientLibraryTestBase::ComputeAndCompareLiteral(
     ComputationBuilder* builder, const Literal& expected,
     tensorflow::gtl::ArraySlice<GlobalData*> arguments,
diff --git a/tensorflow/compiler/xla/tests/client_library_test_base.h b/tensorflow/compiler/xla/tests/client_library_test_base.h
index be90f14c8e..52f31b0669 100644
--- a/tensorflow/compiler/xla/tests/client_library_test_base.h
+++ b/tensorflow/compiler/xla/tests/client_library_test_base.h
@@ -312,6 +312,7 @@ class ClientLibraryTestBase : public ::testing::Test {
   // will be converted to BF16s.
   ComputationDataHandle CreateConstantFromLiteral(const Literal& literal,
                                                   ComputationBuilder* builder);
+  XlaOp CreateConstantFromLiteral(const Literal& literal, XlaBuilder* builder);
 
   // Creates a constant instruction with the given array. When the use_bfloat16
   // flag is set but the array has float elements, the elements will be
@@ -322,6 +323,12 @@ class ClientLibraryTestBase : public ::testing::Test {
     return CreateConstantFromLiteral(*Literal::CreateFromArray(array), builder);
   }
 
+  template <typename NativeT>
+  XlaOp CreateConstantFromArray(const Array<NativeT>& array,
+                                XlaBuilder* builder) {
+    return CreateConstantFromLiteral(*Literal::CreateFromArray(array), builder);
+  }
+
   // Same as CreateConstantFromArray, but for scalars.
   template <typename NativeT>
   ComputationDataHandle CreateConstantFromScalar(NativeT value,
@@ -330,6 +337,12 @@ class ClientLibraryTestBase : public ::testing::Test {
                                      builder);
   }
 
+  template <typename NativeT>
+  XlaOp CreateConstantFromScalar(NativeT value, XlaBuilder* builder) {
+    return CreateConstantFromLiteral(*Literal::CreateR0<NativeT>(value),
+                                     builder);
+  }
+
   // Creates a parameter instruction that wraps a given value and then stores
   // into "data_handle" the global handle for that parameter.
   //
-- 
GitLab


From 467f195a2dd87257e3719576637774ebcf7a4590 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 17:08:57 -0700
Subject: [PATCH 886/960] Add max_constant_size_in_bytes parameter for
 ConstantFolding transform that sets the maximum size of each created
 constant.

PiperOrigin-RevId: 191523208
---
 .../graph_transforms/fold_constants_lib.cc    |  4 ++
 .../graph_transforms/fold_constants_test.cc   | 44 +++++++++++++++++++
 2 files changed, 48 insertions(+)

diff --git a/tensorflow/tools/graph_transforms/fold_constants_lib.cc b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
index 250f54e20f..85660f94a8 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_lib.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_lib.cc
@@ -283,6 +283,10 @@ Status FoldConstants(const GraphDef& input_graph_def,
     };
   }
 
+  TF_RETURN_IF_ERROR(context.GetOneInt64Parameter(
+      "max_constant_size_in_bytes", cf_opts.max_constant_size_in_bytes,
+      &cf_opts.max_constant_size_in_bytes));
+
   // Constant folding.
   bool was_mutated;
   TF_RETURN_IF_ERROR(ConstantFold(cf_opts, nullptr, Env::Default(), nullptr,
diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc
index 41106de008..6bfdfe43f5 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc
@@ -370,6 +370,46 @@ class ConstantFoldingTest : public ::testing::Test {
     EXPECT_EQ(0, node_map.count("b"));
     EXPECT_EQ(1, node_map.count("c"));
   }
+
+  void TestMaxConstantSizeInBytes() {
+    auto root = tensorflow::Scope::NewRootScope();
+
+    const int width = 100;
+
+    Tensor a_data(DT_FLOAT, TensorShape({width}));
+    test::FillIota<float>(&a_data, 1.0f);
+    Output a_const = ::tensorflow::ops::Const(
+        root.WithOpName("a_expect_remains"), Input::Initializer(a_data));
+
+    Tensor b_data(DT_FLOAT, TensorShape({width}));
+    test::FillIota<float>(&b_data, 1.0f);
+    Output b_const = ::tensorflow::ops::Const(
+        root.WithOpName("b_expect_remains"), Input::Initializer(b_data));
+
+    Output add = ::tensorflow::ops::Add(root.WithOpName("add_expect_remains"),
+                                        a_const, b_const);
+
+    Output placeholder = ::tensorflow::ops::Placeholder(
+        root.WithOpName("placeholder_expect_remains"), DT_FLOAT);
+
+    Output mul = ::tensorflow::ops::Mul(
+        root.WithOpName("output_expect_remains"), add, placeholder);
+
+    GraphDef graph_def;
+    TF_ASSERT_OK(root.ToGraphDef(&graph_def));
+
+    Tensor placeholder_tensor(DT_FLOAT, TensorShape({width}));
+    test::FillIota<float>(&placeholder_tensor, 1.0f);
+
+    // Setting the maximum constant size to 10 bytes should stop the constant
+    // folding at add(a, b) that would have yielded a constant of
+    // 100*sizeof(float) bytes.
+    graph_transforms::TransformFuncContext context;
+    context.params["max_constant_size_in_bytes"] = {"10"};
+    TestConstantFolding(graph_def,
+                        {{"placeholder_expect_remains", placeholder_tensor}},
+                        {}, {"output_expect_remains"}, context);
+  }
 };
 
 TEST_F(ConstantFoldingTest, TestSimpleAdd) { TestSimpleAdd(); }
@@ -394,5 +434,9 @@ TEST_F(ConstantFoldingTest, TestRemoveUnusedNodesMultipleOutputs) {
   TestRemoveUnusedNodesMultipleOutputs();
 }
 
+TEST_F(ConstantFoldingTest, TestMaxConstantSizeInBytes) {
+  TestMaxConstantSizeInBytes();
+}
+
 }  // namespace graph_transforms
 }  // namespace tensorflow
-- 
GitLab


From 4a20926c555a2c8da47f4138f436417bfe7db6d0 Mon Sep 17 00:00:00 2001
From: Reed Wanderman-Milne <reedwm@google.com>
Date: Tue, 3 Apr 2018 17:26:16 -0700
Subject: [PATCH 887/960] Fix Conv3D shape inference.

Before, the stride rows and columns were mixed up, causing shape inference to output the wrong shape.

PiperOrigin-RevId: 191525254
---
 tensorflow/core/framework/common_shape_fns.cc      | 4 ++--
 tensorflow/python/kernel_tests/conv_ops_3d_test.py | 3 +++
 2 files changed, 5 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/framework/common_shape_fns.cc b/tensorflow/core/framework/common_shape_fns.cc
index 2fb17c2b02..72eeda7a43 100644
--- a/tensorflow/core/framework/common_shape_fns.cc
+++ b/tensorflow/core/framework/common_shape_fns.cc
@@ -504,8 +504,8 @@ Status Conv3DShape(shape_inference::InferenceContext* c) {
     input_shape =
         c->MakeShape({{dim('N'), dim('0'), dim('1'), dim('2'), dim('C')}});
     stride_planes = strides[2];
-    stride_cols = strides[3];
-    stride_rows = strides[4];
+    stride_rows = strides[3];
+    stride_cols = strides[4];
   } else {
     stride_planes = strides[1];
     stride_rows = strides[2];
diff --git a/tensorflow/python/kernel_tests/conv_ops_3d_test.py b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
index ec8ac74163..f4616fd661 100644
--- a/tensorflow/python/kernel_tests/conv_ops_3d_test.py
+++ b/tensorflow/python/kernel_tests/conv_ops_3d_test.py
@@ -25,6 +25,7 @@ import numpy as np
 
 from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.framework import test_util
 from tensorflow.python.ops import gradient_checker
 from tensorflow.python.ops import nn_ops
@@ -344,6 +345,8 @@ class Conv3DTest(test.TestCase):
         if data_format == "NCDHW":
           conv = test_util.NCHWToNHWC(conv)
 
+        self.assertEqual(conv.shape, tensor_shape.TensorShape(output_shape))
+
         if test_input:
           jacob_t, jacob_n = gradient_checker.compute_gradient(
               orig_input_tensor, input_shape, conv, output_shape)
-- 
GitLab


From 68a2716ba9c553b0442bb6ecca8742c0ad5d37f8 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Tue, 3 Apr 2018 17:35:54 -0700
Subject: [PATCH 888/960] Refactors the eager guide to be more
 researcher-friendly.

 * Shows how to build layers
 * Orders the topics as they'd be ordered in a normal model

PiperOrigin-RevId: 191526275
---
 .../docs_src/programmers_guide/eager.md       | 414 ++++++++++--------
 1 file changed, 232 insertions(+), 182 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/eager.md b/tensorflow/docs_src/programmers_guide/eager.md
index 414653c280..dc5b403428 100644
--- a/tensorflow/docs_src/programmers_guide/eager.md
+++ b/tensorflow/docs_src/programmers_guide/eager.md
@@ -109,9 +109,106 @@ environments and is useful for writing code to [work with graphs](#work_with_gra
 import tensorflow.contrib.eager as tfe
 ```
 
-## Updating model parameters
+## Dynamic control flow
 
-### Automatic differentiation
+A major benefit of eager execution is that all the functionality of the host
+language is available while your model is executing. So, for example,
+it is easy to write [fizzbuzz](https://en.wikipedia.org/wiki/Fizz_buzz):
+
+```py
+def fizzbuzz(max_num):
+  counter = tf.constant(0)
+  for num in range(max_num):
+    num = tf.constant(num)
+    if num % 3 == 0 and num % 5 == 0:
+      print('FizzBuzz')
+    elif num % 3 == 0:
+      print('Fizz')
+    elif num % 5 == 0:
+      print('Buzz')
+    else:
+      print(num)
+    counter += 1
+  return counter
+```
+
+This has conditionals that depend on tensor values and it prints these values
+at runtime.
+
+## Build a model
+
+Many machine learning models are represented by composing layers. When
+using TensorFlow with eager execution you can either write your own layers or
+use a layer provided in the `tf.keras.layers` package.
+
+While you can use any Python object to represent a layer,
+TensorFlow has `tf.keras.layers.Layer` as a convenient base class. Inherit from
+it to implement your own layer:
+
+```py
+class MySimpleLayer(tf.keras.layers.Layer):
+  def __init__(self, output_units):
+    self.output_units = output_units
+
+  def build(self, input):
+    # The build method gets called the first time your layer is used.
+    # Creating variables on build() allows you to make their shape depend
+    # on the input shape and hence remove the need for the user to specify
+    # full shapes. It is possible to create variables during __init__() if
+    # you already know their full shapes.
+    self.kernel = self.add_variable(
+      "kernel", [input.shape[-1], self.output_units])
+
+  def call(self, input):
+    # Override call() instead of __call__ so we can perform some bookkeeping.
+    return tf.matmul(input, self.kernel)
+```
+
+Use `tf.keras.layers.Dense` layer instead  of `MySimpleLayer` above as it has
+a superset of its functionality (it can also add a bias).
+
+When composing layers into models you can use `tf.keras.Sequential` to represent
+models which are a linear stack of layers. It is easy to use for basic models:
+
+```py
+model = tf.keras.Sequential([
+  tf.keras.layers.Dense(10, input_shape=(784,)),  # must declare input shape
+  tf.keras.layers.Dense(10)
+])
+```
+
+Alternatively, organize models in classes by inheriting from `tf.keras.Model`.
+This is a container for layers that is a layer itself, allowing `tf.keras.Model`
+objects to contain other `tf.keras.Model` objects.
+
+```py
+class MNISTModel(tf.keras.Model):
+  def __init__(self):
+    super(MNISTModel, self).__init__()
+    self.dense1 = tf.keras.layers.Dense(units=10)
+    self.dense2 = tf.keras.layers.Dense(units=10)
+
+  def call(self, input):
+    """Run the model."""
+    result = self.dense1(input)
+    result = self.dense2(result)
+    result = self.dense2(result)  # reuse variables from dense2 layer
+    return result
+
+model = MNISTModel()
+```
+
+It's not required to set an input shape for the `tf.keras.Model` class since
+the parameters are set the first time input is passed to the layer.
+
+`tf.keras.layers` classes create and contain their own model variables that
+are tied to the lifetime of their layer objects. To share layer variables, share
+their objects.
+
+
+## Eager training
+
+### Computing gradients
 
 [Automatic differentiation](https://en.wikipedia.org/wiki/Automatic_differentiation)
 is useful for implementing machine learning algorithms such as
@@ -215,189 +312,12 @@ for (batch, (images, labels)) in enumerate(tfe.Iterator(dataset)):
                             global_step=tf.train.get_or_create_global_step())
 ```
 
-#### Dynamic models
-
-`tfe.GradientTape` can also be used in dynamic models. This example for a
-[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search)
-algorithm looks like normal NumPy code, except there are gradients and is
-differentiable, despite the complex control flow:
-
-```py
-def line_search_step(fn, init_x, rate=1.0):
-  with tfe.GradientTape() as tape:
-    # Variables are automatically recorded, but manually watch a tensor
-    tape.watch(init_x)
-    value = fn(init_x)
-  grad, = tape.gradient(value, [init_x])
-  grad_norm = tf.reduce_sum(grad * grad)
-  init_value = value
-  while value > init_value - rate * grad_norm:
-    x = init_x - rate * grad
-    value = fn(x)
-    rate /= 2.0
-  return x, value
-```
-
-#### Additional functions to compute gradients
-
-`tfe.GradientTape` is a powerful interface for computing gradients, but there
-is another [Autograd](https://github.com/HIPS/autograd)-style API available for
-automatic differentiation. These functions are useful if writing math code with
-only tensors and gradient functions, and without `tfe.Variables`:
-
-* `tfe.gradients_function` —Returns a function that computes the derivatives
-  of its input function parameter with respect to its arguments. The input
-  function parameter must return a scalar value. When the returned function is
-  invoked, it returns a list of `tf.Tensor` objects: one element for each
-  argument of the input function. Since anything of interest must be passed as a
-  function parameter, this becomes unwieldy if there's a dependency on many
-  trainable parameters.
-* `tfe.value_and_gradients_function` —Similar to
-  `tfe.gradients_function`, but when the returned function is invoked, it
-  returns the value from the input function in addition to the list of
-  derivatives of the input function with respect to its arguments.
-
-In the following example, `tfe.gradients_function` takes the `square`
-function as an argument and returns a function that computes the partial
-derivatives of `square` with respect to its inputs. To calculate the derivative
-of `square` at `3`, `grad(3.0)` returns `6`.
-
-```py
-def square(x):
-  return tf.multiply(x, x)
-
-grad = tfe.gradients_function(square)
-
-square(3.)  # => 9.0
-grad(3.)    # => [6.0]
-
-# The second-order derivative of square:
-gradgrad = tfe.gradients_function(lambda x: grad(x)[0])
-gradgrad(3.)  # => [2.0]
-
-# The third-order derivative is None:
-gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])
-gradgradgrad(3.)  # => [None]
-
-
-# With flow control:
-def abs(x):
-  return x if x > 0. else -x
-
-grad = tfe.gradients_function(abs)
-
-grad(3.)   # => [1.0]
-grad(-3.)  # => [-1.0]
-```
-
-### Custom gradients
-
-Custom gradients are an easy way to override gradients in eager and graph
-execution. Within the forward function, define the gradient with respect to the
-inputs, outputs, or intermediate results. For example, here's an easy way to clip
-the norm of the gradients in the backward pass:
-
-```py
-@tf.custom_gradient
-def clip_gradient_by_norm(x, norm):
-  y = tf.identity(x)
-  def grad_fn(dresult):
-    return [tf.clip_by_norm(dresult, norm), None]
-  return y, grad_fn
-```
-
-Custom gradients are commonly used to provide a numerically stable gradient for a
-sequence of operations:
-
-```py
-def log1pexp(x):
-  return tf.log(1 + tf.exp(x))
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# The gradient computation works fine at x = 0.
-grad_log1pexp(0.)  # => [0.5]
-
-# However, x = 100 fails because of numerical instability.
-grad_log1pexp(100.)  # => [nan]
-```
-
-Here, the `log1pexp` function can be analytically simplified with a custom
-gradient. The implementation below reuses the value for `tf.exp(x)` that is
-computed during the forward pass—making it more efficient by eliminating
-redundant calculations:
-
-```py
-@tf.custom_gradient
-def log1pexp(x):
-  e = tf.exp(x)
-  def grad(dy):
-    return dy * (1 - 1 / (1 + e))
-  return tf.log(1 + e), grad
-
-grad_log1pexp = tfe.gradients_function(log1pexp)
-
-# As before, the gradient computation works fine at x = 0.
-grad_log1pexp(0.)  # => [0.5]
-
-# And the gradient computation also works at x = 100.
-grad_log1pexp(100.)  # => [1.0]
-```
-
-
-## Build and train models
-
-There are many parameters to optimize when calculating derivatives. TensorFlow
-code is easier to read when structured into reusable classes and objects instead
-of a single top-level function. Eager execution encourages the use of the
-Keras-style layer classes in the `tf.keras.layers` module. Additionally, the
-`tf.train.Optimizer` classes provide sophisticated techniques to calculate
-parameter updates.
 
 The following example creates a multi-layer model that classifies the standard
 [MNIST handwritten digits](https://www.tensorflow.org/tutorials/layers). It
 demonstrates the optimizer and layer APIs to build trainable graphs in an eager
 execution environment.
 
-### Build a model
-
-The `tf.keras.Sequential` model is a linear stack of layers. It is easy to
-use for basic models:
-
-```py
-model = tf.keras.Sequential([
-  tf.keras.layers.Dense(10, input_shape=(784,)),  # must declare input shape
-  tf.keras.layers.Dense(10)
-])
-```
-
-Alternatively, organize models in classes by inheriting from `tf.keras.Model`.
-This is a container for layers that is a layer itself, allowing `tf.keras.Model`
-objects to contain other `tf.keras.Model` objects.
-
-```py
-class MNISTModel(tf.keras.Model):
-  def __init__(self):
-    super(MNISTModel, self).__init__()
-    self.dense1 = tf.keras.layers.Dense(units=10)
-    self.dense2 = tf.keras.layers.Dense(units=10)
-
-  def call(self, input):
-    """Run the model."""
-    result = self.dense1(input)
-    result = self.dense2(result)
-    result = self.dense2(result)  # reuse variables from dense2 layer
-    return result
-
-model = MNISTModel()
-```
-
-It's not required to set an input shape for the `tf.keras.Model` class since
-the parameters are set the first time input is passed to the layer.
-
-`tf.keras.layers` classes create and contain their own model variables that
-are tied to the lifetime of their layer objects. To share layer variables, share
-their objects.
-
 ### Train a model
 
 Even without training, call the model and inspect the output in eager execution:
@@ -661,11 +581,141 @@ for _ in range(iterations):
      ...
 ```
 
+## Advanced automatic differentiation topics
+
+### Dynamic models
+
+`tfe.GradientTape` can also be used in dynamic models. This example for a
+[backtracking line search](https://wikipedia.org/wiki/Backtracking_line_search)
+algorithm looks like normal NumPy code, except there are gradients and is
+differentiable, despite the complex control flow:
+
+```py
+def line_search_step(fn, init_x, rate=1.0):
+  with tfe.GradientTape() as tape:
+    # Variables are automatically recorded, but manually watch a tensor
+    tape.watch(init_x)
+    value = fn(init_x)
+  grad, = tape.gradient(value, [init_x])
+  grad_norm = tf.reduce_sum(grad * grad)
+  init_value = value
+  while value > init_value - rate * grad_norm:
+    x = init_x - rate * grad
+    value = fn(x)
+    rate /= 2.0
+  return x, value
+```
+
+### Additional functions to compute gradients
+
+`tfe.GradientTape` is a powerful interface for computing gradients, but there
+is another [Autograd](https://github.com/HIPS/autograd)-style API available for
+automatic differentiation. These functions are useful if writing math code with
+only tensors and gradient functions, and without `tfe.Variables`:
+
+* `tfe.gradients_function` —Returns a function that computes the derivatives
+  of its input function parameter with respect to its arguments. The input
+  function parameter must return a scalar value. When the returned function is
+  invoked, it returns a list of `tf.Tensor` objects: one element for each
+  argument of the input function. Since anything of interest must be passed as a
+  function parameter, this becomes unwieldy if there's a dependency on many
+  trainable parameters.
+* `tfe.value_and_gradients_function` —Similar to
+  `tfe.gradients_function`, but when the returned function is invoked, it
+  returns the value from the input function in addition to the list of
+  derivatives of the input function with respect to its arguments.
+
+In the following example, `tfe.gradients_function` takes the `square`
+function as an argument and returns a function that computes the partial
+derivatives of `square` with respect to its inputs. To calculate the derivative
+of `square` at `3`, `grad(3.0)` returns `6`.
+
+```py
+def square(x):
+  return tf.multiply(x, x)
+
+grad = tfe.gradients_function(square)
+
+square(3.)  # => 9.0
+grad(3.)    # => [6.0]
+
+# The second-order derivative of square:
+gradgrad = tfe.gradients_function(lambda x: grad(x)[0])
+gradgrad(3.)  # => [2.0]
+
+# The third-order derivative is None:
+gradgradgrad = tfe.gradients_function(lambda x: gradgrad(x)[0])
+gradgradgrad(3.)  # => [None]
+
+
+# With flow control:
+def abs(x):
+  return x if x > 0. else -x
+
+grad = tfe.gradients_function(abs)
+
+grad(3.)   # => [1.0]
+grad(-3.)  # => [-1.0]
+```
+
+### Custom gradients
+
+Custom gradients are an easy way to override gradients in eager and graph
+execution. Within the forward function, define the gradient with respect to the
+inputs, outputs, or intermediate results. For example, here's an easy way to clip
+the norm of the gradients in the backward pass:
+
+```py
+@tf.custom_gradient
+def clip_gradient_by_norm(x, norm):
+  y = tf.identity(x)
+  def grad_fn(dresult):
+    return [tf.clip_by_norm(dresult, norm), None]
+  return y, grad_fn
+```
+
+Custom gradients are commonly used to provide a numerically stable gradient for a
+sequence of operations:
+
+```py
+def log1pexp(x):
+  return tf.log(1 + tf.exp(x))
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# The gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# However, x = 100 fails because of numerical instability.
+grad_log1pexp(100.)  # => [nan]
+```
+
+Here, the `log1pexp` function can be analytically simplified with a custom
+gradient. The implementation below reuses the value for `tf.exp(x)` that is
+computed during the forward pass—making it more efficient by eliminating
+redundant calculations:
+
+```py
+@tf.custom_gradient
+def log1pexp(x):
+  e = tf.exp(x)
+  def grad(dy):
+    return dy * (1 - 1 / (1 + e))
+  return tf.log(1 + e), grad
+
+grad_log1pexp = tfe.gradients_function(log1pexp)
+
+# As before, the gradient computation works fine at x = 0.
+grad_log1pexp(0.)  # => [0.5]
+
+# And the gradient computation also works at x = 100.
+grad_log1pexp(100.)  # => [1.0]
+```
+
 ## Performance
 
-Computation is not automatically offloaded to GPUs during eager execution. To
-explicitly direct a computation to a GPU, enclose it in a
-`tf.device('/gpu:0')` block:
+Computation is automatically offloaded to GPUs during eager execution. If you
+want control over where a computation runs you can enclose it in a
+`tf.device('/gpu:0')` block (or the CPU equivalent):
 
 ```py
 import time
-- 
GitLab


From e61eb9b62a6b7b3a00cccd143499f91e7255d7fd Mon Sep 17 00:00:00 2001
From: Jacques Pienaar <jpienaar@google.com>
Date: Tue, 3 Apr 2018 17:36:10 -0700
Subject: [PATCH 889/960] Add testcase for convert.

PiperOrigin-RevId: 191526297
---
 tensorflow/compiler/xla/tests/convert_test.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/compiler/xla/tests/convert_test.cc b/tensorflow/compiler/xla/tests/convert_test.cc
index 8718fa5066..0842a8918b 100644
--- a/tensorflow/compiler/xla/tests/convert_test.cc
+++ b/tensorflow/compiler/xla/tests/convert_test.cc
@@ -241,6 +241,7 @@ XLA_TEST_F(ConvertTest, ConvertR1F32ToR1S64) {
                          1.99f,
                          2.0f,
                          2.01f,
+                         2147483648.f,
                          -0.5f,
                          -0.99f,
                          -1.0f,
-- 
GitLab


From 5b652f57709d30d883570a82ac500051d8bfe1e6 Mon Sep 17 00:00:00 2001
From: Tony Wang <tonywy@google.com>
Date: Tue, 3 Apr 2018 17:45:18 -0700
Subject: [PATCH 890/960] [TF:XLA] Add INTEL_MKL_ML MatMul method to XLA/CPU
 backend

The INTEL GEMM API provides 32-bit and 64-bit MatMul. With INTEL_MKL flag set,
XLA backend emits runtime call to INTEL GEMM MatMul instead of Eigen.

PiperOrigin-RevId: 191527251
---
 .../xla/legacy_flags/debug_options_flags.cc   |   7 +
 tensorflow/compiler/xla/service/cpu/BUILD     |  25 ++++
 .../compiler/xla/service/cpu/cpu_runtime.cc   |   8 ++
 .../compiler/xla/service/cpu/cpu_runtime.h    |   4 +
 .../xla/service/cpu/cpu_runtime_test.cc       |  82 ++++++++++-
 .../xla/service/cpu/dot_op_emitter.cc         |  23 ++--
 .../xla/service/cpu/runtime_matmul_mkl.cc     | 129 ++++++++++++++++++
 .../xla/service/cpu/runtime_matmul_mkl.h      |  80 +++++++++++
 .../xla/service/cpu/simple_orc_jit.cc         |   5 +
 tensorflow/compiler/xla/xla.proto             |   3 +
 10 files changed, 351 insertions(+), 15 deletions(-)
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
 create mode 100644 tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h

diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
index c8ed3e3a2b..f037663e3f 100644
--- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
@@ -40,6 +40,9 @@ void SetDebugOptionsDefaults(DebugOptions* flags) {
   flags->set_xla_cpu_multi_thread_eigen(true);
   flags->set_xla_gpu_cuda_data_dir("./cuda_sdk_lib");
   flags->set_xla_eliminate_hlo_implicit_broadcast(true);
+#ifdef INTEL_MKL
+  flags->set_xla_cpu_use_mkl_dnn(true);
+#endif  // INTEL_MKL
 
   // Set cudnn batchnorm off by default; it does not provide a performance win
   // on average.
@@ -288,6 +291,10 @@ void AllocateFlags() {
           flag_values->xla_gpu_use_cudnn_batchnorm(),
           "Allows the GPU backend to implement batchnorm HLOs using cudnn, "
           "rather than expanding them to a soup of HLOs."),
+      tensorflow::Flag("xla_cpu_use_mkl_dnn",
+                       bool_setter_for(&DebugOptions::set_xla_cpu_use_mkl_dnn),
+                       flag_values->xla_cpu_use_mkl_dnn(),
+                       "Generate calls to MKL-DNN in the CPU backend."),
   });
   ParseFlagsFromEnv(*flag_objects);
 }
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index 966e2d0fc5..d22c135249 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -18,6 +18,10 @@ load(":build_defs.bzl", "runtime_copts")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
+load(
+    "//third_party/mkl:build_defs.bzl",
+    "if_mkl",
+)
 
 # Filegroup used to collect source files for dependency checking.
 filegroup(
@@ -170,6 +174,7 @@ cc_library(
         ":runtime_fft",
         ":runtime_fork_join",
         ":runtime_matmul",
+        ":runtime_matmul_mkl",
         ":runtime_single_threaded_conv2d",
         ":runtime_single_threaded_matmul",
         "@llvm//:execution_engine",
@@ -534,10 +539,28 @@ cc_library(
         ":runtime_matvec",
         "//tensorflow/compiler/xla:executable_run_options",
         "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
         "//third_party/eigen3",
     ],
 )
 
+cc_library(
+    name = "runtime_matmul_mkl",
+    srcs = ["runtime_matmul_mkl.cc"],
+    hdrs = ["runtime_matmul_mkl.h"],
+    copts = runtime_copts(),
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/compiler/xla:executable_run_options",
+        "//tensorflow/core:framework_lite",
+        "//tensorflow/core:lib",
+        "//third_party/eigen3",
+    ] + if_mkl([
+        "//third_party/mkl:intel_binary_blob",
+        "@mkl_dnn",
+    ]),
+)
+
 cc_library(
     name = "runtime_single_threaded_conv2d",
     srcs = [
@@ -584,10 +607,12 @@ cc_library(
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
+    shard_count = 10,
     tags = ["optonly"],
     deps = [
         ":cpu_runtime",
         ":runtime_matmul",
+        ":runtime_matmul_mkl",
         ":runtime_single_threaded_matmul",
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index 9a3bd68c80..872b0be1f8 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -37,6 +37,14 @@ extern const char* const kEigenMatMulF32SymbolName =
     "__xla_cpu_runtime_EigenMatMulF32";
 extern const char* const kEigenMatMulF64SymbolName =
     "__xla_cpu_runtime_EigenMatMulF64";
+extern const char* const kMKLMatMulF32SymbolName =
+    "__xla_cpu_runtime_MKLMatMulF32";
+extern const char* const kMKLMatMulF64SymbolName =
+    "__xla_cpu_runtime_MKLMatMulF64";
+extern const char* const kMKLSingleThreadedMatMulF32SymbolName =
+    "__xla_cpu_runtime_MKLSingleThreadedMatMulF32";
+extern const char* const kMKLSingleThreadedMatMulF64SymbolName =
+    "__xla_cpu_runtime_MKLSingleThreadedMatMulF64";
 extern const char* const kEigenConvF16SymbolName =
     "__xla_cpu_runtime_EigenConvF16";
 extern const char* const kEigenConvF32SymbolName =
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index e61d6ea28b..e392e231b4 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -44,6 +44,10 @@ namespace runtime {
 extern const char* const kEigenMatMulF16SymbolName;
 extern const char* const kEigenMatMulF32SymbolName;
 extern const char* const kEigenMatMulF64SymbolName;
+extern const char* const kMKLMatMulF32SymbolName;
+extern const char* const kMKLMatMulF64SymbolName;
+extern const char* const kMKLSingleThreadedMatMulF32SymbolName;
+extern const char* const kMKLSingleThreadedMatMulF64SymbolName;
 extern const char* const kEigenConvF16SymbolName;
 extern const char* const kEigenConvF32SymbolName;
 extern const char* const kEigenFftSymbolName;
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
index f385829cdf..9e04307295 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
+#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/common_runtime/eigen_thread_pool.h"
@@ -130,21 +131,19 @@ MatMulShape MatMulShapes[] = {
 // * transpose_lhs
 // * transpose_rhs
 // * single_threaded
-using EigenMatMulTestParam = std::tuple<MatMulShape, bool, bool, bool>;
+using MatMulTestParam = std::tuple<MatMulShape, bool, bool, bool>;
 
-class EigenMatMulTest
-    : public CpuRuntimeTest,
-      public ::testing::WithParamInterface<EigenMatMulTestParam> {
+class EigenMatMulTest : public CpuRuntimeTest,
+                        public ::testing::WithParamInterface<MatMulTestParam> {
  public:
-  static string Name(
-      const ::testing::TestParamInfo<EigenMatMulTestParam>& info) {
+  static string Name(const ::testing::TestParamInfo<MatMulTestParam>& info) {
     MatMulShape shape = std::get<0>(info.param);
     bool transpose_lhs = std::get<1>(info.param);
     bool transpose_rhs = std::get<2>(info.param);
     bool single_threaded = std::get<3>(info.param);
 
     return tensorflow::strings::Printf(
-        "MatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
+        "EigenMatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
         transpose_lhs ? "Tlhs_" : "", transpose_rhs ? "Trhs_" : "",
         single_threaded ? "single" : "multi");
   }
@@ -169,5 +168,74 @@ INSTANTIATE_TEST_CASE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
                                            ::testing::Bool()),
                         EigenMatMulTest::Name);
 
+#ifdef INTEL_MKL
+class MKLMatMulTest : public CpuRuntimeTest,
+                      public ::testing::WithParamInterface<MatMulTestParam> {
+ public:
+  static string Name(const ::testing::TestParamInfo<MatMulTestParam>& info) {
+    MatMulShape shape = std::get<0>(info.param);
+    bool transpose_lhs = std::get<1>(info.param);
+    bool transpose_rhs = std::get<2>(info.param);
+    bool single_threaded = std::get<3>(info.param);
+
+    return tensorflow::strings::Printf(
+        "MKLMatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
+        transpose_lhs ? "Tlhs_" : "", transpose_rhs ? "Trhs_" : "",
+        single_threaded ? "single" : "multi");
+  }
+};
+
+std::unique_ptr<Array2D<float>> MKLMatrixMultiply(const Array2D<float>& a,
+                                                  const Array2D<float>& b,
+                                                  bool transpose_lhs,
+                                                  bool transpose_rhs,
+                                                  bool single_threaded) {
+  CHECK_EQ(a.width(), b.height());
+  int64 m = a.height();
+  int64 n = b.width();
+  int64 k = a.width();
+
+  // The MKL matmul runtime function expects the matrix to be in column major
+  // order and array2d is in row-major order. Create transposes of a and b. The
+  // 'data' buffer in the transposed array is the original array in column major
+  // order.
+  auto a_transpose = MaybeTransposeArray2D(a, !transpose_lhs);
+  auto b_transpose = MaybeTransposeArray2D(b, !transpose_rhs);
+
+  // Since we're going to transpose c before returning it, swap the order of the
+  // dimension sizes to ensure the returned array is properly dimensioned.
+  auto c_transpose = MakeUnique<Array2D<float>>(n, m);
+  if (single_threaded) {
+    __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
+        nullptr, c_transpose->data(), a_transpose->data(), b_transpose->data(),
+        m, n, k, transpose_lhs, transpose_rhs);
+  } else {
+    __xla_cpu_runtime_MKLMatMulF32(nullptr, c_transpose->data(),
+                                   a_transpose->data(), b_transpose->data(), m,
+                                   n, k, transpose_lhs, transpose_rhs);
+  }
+  return MaybeTransposeArray2D(*c_transpose, true);
+}
+
+TEST_P(MKLMatMulTest, DoIt) {
+  MatMulShape shape = std::get<0>(GetParam());
+  bool transpose_lhs = std::get<1>(GetParam());
+  bool transpose_rhs = std::get<2>(GetParam());
+  bool single_threaded = std::get<3>(GetParam());
+
+  auto a = MakeLinspaceArray2D(0.0, 1.0, shape.m, shape.k);
+  auto b = MakeLinspaceArray2D(-2.0, 2.0, shape.k, shape.n);
+  auto c =
+      MKLMatrixMultiply(*a, *b, transpose_lhs, transpose_rhs, single_threaded);
+  CheckMatrixMultiply(*a, *b, *c);
+}
+
+INSTANTIATE_TEST_CASE_P(MKLMatMulTestInstantiaion, MKLMatMulTest,
+                        ::testing::Combine(::testing::ValuesIn(MatMulShapes),
+                                           ::testing::Bool(), ::testing::Bool(),
+                                           ::testing::Bool()),
+                        MKLMatMulTest::Name);
+#endif  // INTEL_MKL
+
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 8b1e20d79e..29afd8ea5f 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -918,28 +918,35 @@ tensorflow::Status DotOpEmitter::EmitCallToRuntime() {
   // The two transpose_... parameters are actually booleans, but we use int32
   // to avoid target-dependent calling convention details.
 
-  bool multi_threaded_eigen =
+  bool multi_threaded =
       hlo_module_config_.debug_options().xla_cpu_multi_thread_eigen();
+  bool use_mkl_dnn = hlo_module_config_.debug_options().xla_cpu_use_mkl_dnn();
   PrimitiveType type = target_array_.GetShape().element_type();
   llvm::Type* float_type;
   const char* fn_name;
   switch (type) {
     case F16:
-      fn_name = multi_threaded_eigen
+      fn_name = multi_threaded
                     ? runtime::kEigenMatMulF16SymbolName
                     : runtime::kEigenSingleThreadedMatMulF16SymbolName;
       float_type = ir_builder_->getHalfTy();
       break;
     case F32:
-      fn_name = multi_threaded_eigen
-                    ? runtime::kEigenMatMulF32SymbolName
-                    : runtime::kEigenSingleThreadedMatMulF32SymbolName;
+      fn_name = multi_threaded
+                    ? (use_mkl_dnn ? runtime::kMKLMatMulF32SymbolName
+                                   : runtime::kEigenMatMulF32SymbolName)
+                    : (use_mkl_dnn
+                           ? runtime::kMKLSingleThreadedMatMulF32SymbolName
+                           : runtime::kEigenSingleThreadedMatMulF32SymbolName);
       float_type = ir_builder_->getFloatTy();
       break;
     case F64:
-      fn_name = multi_threaded_eigen
-                    ? runtime::kEigenMatMulF64SymbolName
-                    : runtime::kEigenSingleThreadedMatMulF64SymbolName;
+      fn_name = multi_threaded
+                    ? (use_mkl_dnn ? runtime::kMKLMatMulF64SymbolName
+                                   : runtime::kEigenMatMulF64SymbolName)
+                    : (use_mkl_dnn
+                           ? runtime::kMKLSingleThreadedMatMulF64SymbolName
+                           : runtime::kEigenSingleThreadedMatMulF64SymbolName);
       float_type = ir_builder_->getDoubleTy();
       break;
     default:
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
new file mode 100644
index 0000000000..729a4e7f5b
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
@@ -0,0 +1,129 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifdef INTEL_MKL
+#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
+#include "third_party/intel_mkl_ml/include/mkl_cblas.h"
+#include "third_party/intel_mkl_ml/include/mkl_service.h"
+
+#include "tensorflow/compiler/xla/executable_run_options.h"
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+
+#define EIGEN_USE_THREADS
+#include "third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool"
+
+using tensorflow::int32;
+using tensorflow::int64;
+
+namespace {
+// BLAS GEMM API for 32-bit Matrix Multiplication.
+
+// MatMul function is defined as: c = alpha * op(a) * op(b) + beta * c.
+// Since XLA MatMul does not used alpha, beta, we set them to 1.0 and 0.0.
+// Matrix lhs, rhs and out are all colum-major.
+void MatMulF32(const void* run_options_ptr, float* out, float* lhs, float* rhs,
+               int64 m, int64 n, int64 k, int32 transpose_lhs,
+               int32 transpose_rhs) {
+  const float alpha = 1.0f, beta = 0.0f;
+  // lda, ldb, and ldc are the leading dimensions of matrices a, b, and c,
+  // respectively. For column-major matrices, the leading dimension is the
+  // stride between consecutive columns (which equals the number of rows). If
+  // the matrix is transposed, the leading dimension is the stride between
+  // consecutive rows (which equals the number of columns).
+  int lda = transpose_lhs ? k : m;
+  int ldb = transpose_rhs ? n : k;
+  int ldc = m;
+  cblas_sgemm(CblasColMajor, transpose_lhs ? CblasTrans : CblasNoTrans,
+              transpose_rhs ? CblasTrans : CblasNoTrans, m, n, k, alpha, lhs,
+              lda, rhs, ldb, beta, out, ldc);
+}
+
+// BLAS GEMM API for 64-bit Matrix Multiplication.
+
+// MatMul function is defined as: c = alpha * op(a) * op(b) + beta * c.
+// Since XLA MatMul does not used alpha, beta, we set them to 1.0 and 0.0.
+// Matrix lhs, rhs and out are all colum-major.
+void MatMulF64(const void* run_options_ptr, double* out, double* lhs,
+               double* rhs, int64 m, int64 n, int64 k, int32 transpose_lhs,
+               int32 transpose_rhs) {
+  const float alpha = 1.0f, beta = 0.0f;
+  // lda, ldb, and ldc are the leading dimensions of matrices a, b, and c,
+  // respectively. For a column-major matrix, the leading dimension is the
+  // stride between consecutive columns (which equals the number of rows). If
+  // the matrix is transposed, the leading dimension is the stride between
+  // consecutive rows (which equals the number of columns).
+  int lda = transpose_lhs ? k : m;
+  int ldb = transpose_rhs ? n : k;
+  int ldc = m;
+  cblas_dgemm(CblasColMajor, transpose_lhs ? CblasTrans : CblasNoTrans,
+              transpose_rhs ? CblasTrans : CblasNoTrans, m, n, k, alpha, lhs,
+              lda, rhs, ldb, beta, out, ldc);
+}
+
+}  // namespace
+
+void __xla_cpu_runtime_MKLMatMulF32(const void* run_options_ptr, float* out,
+                                    float* lhs, float* rhs, int64 m, int64 n,
+                                    int64 k, int32 transpose_lhs,
+                                    int32 transpose_rhs) {
+  const xla::ExecutableRunOptions* run_options =
+      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
+  // BLAS GEMM MatMul uses OpenMP for parallelization, so we pass the thread
+  // number specified in intra_op_thread_pool to MKL.
+  int prev_num_threads = mkl_set_num_threads_local(
+      run_options->intra_op_thread_pool()->numThreads());
+  MatMulF32(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
+  // Set thread number back to the previous number.
+  mkl_set_num_threads_local(prev_num_threads);
+}
+// BLAS GEMM API for 64-bit Matrix Multiplication
+void __xla_cpu_runtime_MKLMatMulF64(const void* run_options_ptr, double* out,
+                                    double* lhs, double* rhs, int64 m, int64 n,
+                                    int64 k, int32 transpose_lhs,
+                                    int32 transpose_rhs) {
+  const xla::ExecutableRunOptions* run_options =
+      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
+  // BLAS GEMM MatMul uses OpenMP for parallelization, so we pass the thread
+  // number specified in intra_op_thread_pool to MKL.
+  int prev_num_threads = mkl_set_num_threads_local(
+      run_options->intra_op_thread_pool()->numThreads());
+  MatMulF64(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
+  // Set thread number back to the previous number.
+  mkl_set_num_threads_local(prev_num_threads);
+}
+void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(const void* run_options_ptr,
+                                                  float* out, float* lhs,
+                                                  float* rhs, int64 m, int64 n,
+                                                  int64 k, int32 transpose_lhs,
+                                                  int32 transpose_rhs) {
+  // Set the thread number to 1 for single threaded excution.
+  int prev_num_threads = mkl_set_num_threads_local(1);
+  MatMulF32(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
+  // Set thread number back to the previous number.
+  mkl_set_num_threads_local(prev_num_threads);
+}
+void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(const void* run_options_ptr,
+                                                  double* out, double* lhs,
+                                                  double* rhs, int64 m, int64 n,
+                                                  int64 k, int32 transpose_lhs,
+                                                  int32 transpose_rhs) {
+  // Set the thread number to 1 for single threaded excution.
+  int prev_num_threads = mkl_set_num_threads_local(1);
+  MatMulF64(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
+  // Set thread number back to the previous number.
+  mkl_set_num_threads_local(prev_num_threads);
+}
+#endif  // INTEL_MKL
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h
new file mode 100644
index 0000000000..9dbc506c08
--- /dev/null
+++ b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h
@@ -0,0 +1,80 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
+#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
+
+#include "tensorflow/core/platform/logging.h"
+#include "tensorflow/core/platform/types.h"
+#ifdef INTEL_MKL
+#include "third_party/intel_mkl_ml/include/mkl_cblas.h"
+
+extern void __xla_cpu_runtime_MKLMatMulF32(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
+    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs);
+extern void __xla_cpu_runtime_MKLMatMulF64(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
+    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs);
+extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
+    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs);
+extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
+    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs);
+
+#else
+extern void __xla_cpu_runtime_MKLMatMulF32(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
+    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs) {
+  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
+                "INTEL_MKL. Add --config=mkl to build with MKL.";
+}
+extern void __xla_cpu_runtime_MKLMatMulF64(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
+    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs) {
+  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
+                "INTEL_MKL. Add --config=mkl to build with MKL.";
+}
+extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
+    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs) {
+  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
+                "INTEL_MKL. Add --config=mkl to build with MKL.";
+}
+extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(
+    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
+    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
+    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
+    tensorflow::int32 transpose_rhs) {
+  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
+                "INTEL_MKL. Add --config=mkl to build with MKL.";
+}
+
+#endif  // INTEL_MKL
+#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index 4198260a22..b7ce5bbe47 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -35,6 +35,7 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_fp16.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
+#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/windows_compatibility.h"
@@ -183,6 +184,10 @@ bool RegisterKnownJITSymbols() {
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF16);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(MKLMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(MKLMatMulF64);
+  REGISTER_CPU_RUNTIME_SYMBOL(MKLSingleThreadedMatMulF32);
+  REGISTER_CPU_RUNTIME_SYMBOL(MKLSingleThreadedMatMulF64);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF16);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF16);
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index 5cb18113e5..f9943f71d3 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -189,6 +189,9 @@ message DebugOptions {
   // directory.
   string xla_dump_per_pass_hlo_proto_to = 96;
 
+  // Generate calls to MKL-DNN in the CPU backend.
+  bool xla_cpu_use_mkl_dnn = 97;
+
   // Extra options to pass to the compilation backend; specific interpretation
   // of these values is left to the backend.
   map<string, string> xla_backend_extra_options = 500;
-- 
GitLab


From bde05fdf247ea6311414677f55260f7e8085718f Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Tue, 3 Apr 2018 17:52:53 -0700
Subject: [PATCH 891/960] Fix a shape inference bug.

PiperOrigin-RevId: 191528009
---
 .../core/common_runtime/shape_refiner.cc      |  5 ++
 tensorflow/python/BUILD                       |  1 +
 .../python/grappler/tf_optimizer_test.py      | 47 ++++++++++++++++++-
 3 files changed, 52 insertions(+), 1 deletion(-)

diff --git a/tensorflow/core/common_runtime/shape_refiner.cc b/tensorflow/core/common_runtime/shape_refiner.cc
index cef50be3b1..1b7e3138ee 100644
--- a/tensorflow/core/common_runtime/shape_refiner.cc
+++ b/tensorflow/core/common_runtime/shape_refiner.cc
@@ -351,6 +351,11 @@ Status ShapeRefiner::UpdateNode(const Node* node, bool relax, bool* refined) {
         }
       }
     }
+    if (node_context->requested_input_tensor_as_partial_shape(dst_input)) {
+      // The input value may have changed. Since we have no way to know if
+      // that's indeed the case, err on the safe side.
+      *refined = true;
+    }
 
     // Also propagate handle shape and dtype of edges which are carrying
     // resource handles.
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index b5f4387efd..57b0b78c82 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4782,6 +4782,7 @@ py_test(
         ":client_testlib",
         ":framework_for_generated_wrappers",
         ":math_ops",
+        ":tf_item",
         ":tf_optimizer",
         "//tensorflow/core:protos_all_py",
         "//third_party/py/numpy",
diff --git a/tensorflow/python/grappler/tf_optimizer_test.py b/tensorflow/python/grappler/tf_optimizer_test.py
index 3ee4d7807e..1c0f072dd3 100644
--- a/tensorflow/python/grappler/tf_optimizer_test.py
+++ b/tensorflow/python/grappler/tf_optimizer_test.py
@@ -17,12 +17,16 @@
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
-
 from tensorflow.core.protobuf import rewriter_config_pb2
 from tensorflow.python.framework import constant_op
+from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import meta_graph
 from tensorflow.python.framework import ops
+from tensorflow.python.framework import tensor_shape
+from tensorflow.python.grappler import item as gitem
 from tensorflow.python.grappler import tf_optimizer
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
 from tensorflow.python.ops import variables
 from tensorflow.python.platform import test
@@ -74,6 +78,47 @@ class PyWrapOptimizeGraphTest(test.TestCase):
     self.assertEqual(a2.op.name, optimized_graph.node[3].name)
     self.assertEqual('Variable/Assign', optimized_graph.node[4].name)
 
+  def testLoops(self):
+    g = ops.Graph()
+    with g.as_default():
+
+      def _Cond(_, counter):
+        return counter < end
+
+      def _Body(buf, counter):
+        buf = array_ops.concat([buf, [counter]], 0)
+        counter += 1
+        return [buf, counter]
+
+      start = array_ops.placeholder(shape=[], dtype=dtypes.int32)
+      end = array_ops.placeholder(shape=[], dtype=dtypes.int32)
+      init_buf = array_ops.zeros(shape=[0], dtype=dtypes.int32)
+      loop_vars = [init_buf, start]
+      shape_inv = [
+          tensor_shape.TensorShape([None]),
+          tensor_shape.TensorShape([])
+      ]
+      buf, _ = control_flow_ops.while_loop(_Cond, _Body, loop_vars, shape_inv)
+
+      f = -array_ops.ones_like(buf, optimize=False)
+      buf_shape = array_ops.shape(buf)
+      f_shape = array_ops.shape(f)
+      ops.add_to_collection('train_op', buf_shape)
+      ops.add_to_collection('train_op', f_shape)
+
+    # Optimize the graph.
+    mg = meta_graph.create_meta_graph_def(graph=g)
+    rewriter_config = rewriter_config_pb2.RewriterConfig()
+    optimized_graph = tf_optimizer.OptimizeGraph(rewriter_config, mg)
+    mg.graph_def.CopyFrom(optimized_graph)
+
+    # Check that the nodes referenced in various collections have been preserved
+    item = gitem.Item(mg)
+    props = item.GetOpProperties()
+    buf_prop = props[buf.op.name]
+    f_prop = props[f.op.name]
+    self.assertEqual(buf_prop, f_prop)
+
 
 if __name__ == '__main__':
   test.main()
-- 
GitLab


From f5f7fdb28dd798c6e6b39ab5e55fd33a63ae8733 Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Tue, 3 Apr 2018 17:56:14 -0700
Subject: [PATCH 892/960] Replace trivial backend calls with calls to
 underlying TensorFlow functions - Part 1

PiperOrigin-RevId: 191528321
---
 .../python/keras/_impl/keras/activations.py   |  3 +-
 .../keras/applications/imagenet_utils.py      |  6 +-
 .../python/keras/_impl/keras/constraints.py   | 12 ++-
 .../keras/_impl/keras/engine/topology_test.py |  2 +-
 .../_impl/keras/engine/training_utils.py      |  5 +-
 .../keras/layers/advanced_activations.py      |  3 +-
 .../keras/layers/convolutional_recurrent.py   | 14 +--
 .../python/keras/_impl/keras/layers/core.py   | 12 ++-
 .../keras/_impl/keras/layers/embeddings.py    |  6 +-
 .../python/keras/_impl/keras/layers/merge.py  |  9 +-
 .../python/keras/_impl/keras/layers/noise.py  | 10 +-
 .../keras/_impl/keras/layers/recurrent.py     | 30 +++---
 .../_impl/keras/layers/recurrent_test.py      | 15 +--
 .../keras/_impl/keras/layers/wrappers.py      |  3 +-
 tensorflow/python/keras/_impl/keras/losses.py |  7 +-
 .../python/keras/_impl/keras/metrics.py       | 12 ++-
 .../python/keras/_impl/keras/metrics_test.py  | 18 ++--
 .../python/keras/_impl/keras/optimizers.py    | 99 ++++++++++---------
 .../python/keras/_impl/keras/regularizers.py  |  5 +-
 .../keras/_impl/keras/utils/layer_utils.py    |  5 +-
 20 files changed, 154 insertions(+), 122 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/activations.py b/tensorflow/python/keras/_impl/keras/activations.py
index 236e17653e..74ec373ea5 100644
--- a/tensorflow/python/keras/_impl/keras/activations.py
+++ b/tensorflow/python/keras/_impl/keras/activations.py
@@ -23,6 +23,7 @@ import six
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.layers.base import Layer
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
@@ -46,7 +47,7 @@ def softmax(x, axis=-1):
     return K.softmax(x)
   elif ndim > 2:
     e = K.exp(x - K.max(x, axis=axis, keepdims=True))
-    s = K.sum(e, axis=axis, keepdims=True)
+    s = math_ops.reduce_sum(e, axis=axis, keepdims=True)
     return e / s
   else:
     raise ValueError('Cannot apply softmax to a tensor that is 1D')
diff --git a/tensorflow/python/keras/_impl/keras/applications/imagenet_utils.py b/tensorflow/python/keras/_impl/keras/applications/imagenet_utils.py
index c26a28ed40..d928a7afdc 100644
--- a/tensorflow/python/keras/_impl/keras/applications/imagenet_utils.py
+++ b/tensorflow/python/keras/_impl/keras/applications/imagenet_utils.py
@@ -22,8 +22,10 @@ import json
 
 import numpy as np
 
+from tensorflow.python.framework import constant_op
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.data_utils import get_file
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
@@ -151,11 +153,11 @@ def _preprocess_symbolic_input(x, data_format, mode):
     std = None
 
   if _IMAGENET_MEAN is None:
-    _IMAGENET_MEAN = K.constant(-np.array(mean))
+    _IMAGENET_MEAN = constant_op.constant(-np.array(mean), dtype=K.floatx())
 
   # Zero-center by mean pixel
   if K.dtype(x) != K.dtype(_IMAGENET_MEAN):
-    x = K.bias_add(x, K.cast(_IMAGENET_MEAN, K.dtype(x)), data_format)
+    x = K.bias_add(x, math_ops.cast(_IMAGENET_MEAN, K.dtype(x)), data_format)
   else:
     x = K.bias_add(x, _IMAGENET_MEAN, data_format)
   if std is not None:
diff --git a/tensorflow/python/keras/_impl/keras/constraints.py b/tensorflow/python/keras/_impl/keras/constraints.py
index 271fbbb63d..aac4d0f1e9 100644
--- a/tensorflow/python/keras/_impl/keras/constraints.py
+++ b/tensorflow/python/keras/_impl/keras/constraints.py
@@ -24,6 +24,7 @@ import six
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -65,7 +66,8 @@ class MaxNorm(Constraint):
     self.axis = axis
 
   def __call__(self, w):
-    norms = K.sqrt(K.sum(K.square(w), axis=self.axis, keepdims=True))
+    norms = K.sqrt(
+        math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True))
     desired = K.clip(norms, 0, self.max_value)
     return w * (desired / (K.epsilon() + norms))
 
@@ -79,7 +81,7 @@ class NonNeg(Constraint):
   """
 
   def __call__(self, w):
-    return w * K.cast(K.greater_equal(w, 0.), K.floatx())
+    return w * math_ops.cast(K.greater_equal(w, 0.), K.floatx())
 
 
 @tf_export('keras.constraints.UnitNorm', 'keras.constraints.unit_norm')
@@ -105,7 +107,8 @@ class UnitNorm(Constraint):
 
   def __call__(self, w):
     return w / (
-        K.epsilon() + K.sqrt(K.sum(K.square(w), axis=self.axis, keepdims=True)))
+        K.epsilon() + K.sqrt(
+            math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True)))
 
   def get_config(self):
     return {'axis': self.axis}
@@ -148,7 +151,8 @@ class MinMaxNorm(Constraint):
     self.axis = axis
 
   def __call__(self, w):
-    norms = K.sqrt(K.sum(K.square(w), axis=self.axis, keepdims=True))
+    norms = K.sqrt(
+        math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True))
     desired = (
         self.rate * K.clip(norms, self.min_value, self.max_value) +
         (1 - self.rate) * norms)
diff --git a/tensorflow/python/keras/_impl/keras/engine/topology_test.py b/tensorflow/python/keras/_impl/keras/engine/topology_test.py
index b50277c8ff..9ab4b6fdcf 100644
--- a/tensorflow/python/keras/_impl/keras/engine/topology_test.py
+++ b/tensorflow/python/keras/_impl/keras/engine/topology_test.py
@@ -783,7 +783,7 @@ class TopologyConstructionTest(test.TestCase):
   def test_activity_regularization_with_model_composition(self):
 
     def reg(x):
-      return keras.backend.sum(x)
+      return math_ops.reduce_sum(x)
 
     net_a_input = keras.Input((2,))
     net_a = net_a_input
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_utils.py b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
index 76537b735f..58d2c78aad 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
@@ -26,6 +26,7 @@ from tensorflow.python.eager import context
 from tensorflow.python.framework import tensor_util
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras import losses
+from tensorflow.python.ops import math_ops
 
 
 def check_num_samples(ins,
@@ -436,7 +437,7 @@ def weighted_masked_objective(fn):
     score_array = fn(y_true, y_pred)
     if mask is not None:
       # Cast the mask to floatX to avoid float64 upcasting in theano
-      mask = K.cast(mask, K.floatx())
+      mask = math_ops.cast(mask, K.floatx())
       # mask should have the same shape as score_array
       score_array *= mask
       #  the loss per batch should be proportional
@@ -450,7 +451,7 @@ def weighted_masked_objective(fn):
       weight_ndim = K.ndim(weights)
       score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))
       score_array *= weights
-      score_array /= K.mean(K.cast(K.not_equal(weights, 0), K.floatx()))
+      score_array /= K.mean(math_ops.cast(K.not_equal(weights, 0), K.floatx()))
     return K.mean(score_array)
 
   return weighted
diff --git a/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py b/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
index c40ee109aa..45b0c6c91a 100644
--- a/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
@@ -26,6 +26,7 @@ from tensorflow.python.keras._impl.keras import regularizers
 from tensorflow.python.keras._impl.keras.engine import InputSpec
 from tensorflow.python.keras._impl.keras.engine import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -232,7 +233,7 @@ class ThresholdedReLU(Layer):
     self.theta = K.cast_to_floatx(theta)
 
   def call(self, inputs, mask=None):
-    return inputs * K.cast(K.greater(inputs, self.theta), K.floatx())
+    return inputs * math_ops.cast(K.greater(inputs, self.theta), K.floatx())
 
   def get_config(self):
     config = {'theta': float(self.theta)}
diff --git a/tensorflow/python/keras/_impl/keras/layers/convolutional_recurrent.py b/tensorflow/python/keras/_impl/keras/layers/convolutional_recurrent.py
index d95a094245..b78962d66a 100644
--- a/tensorflow/python/keras/_impl/keras/layers/convolutional_recurrent.py
+++ b/tensorflow/python/keras/_impl/keras/layers/convolutional_recurrent.py
@@ -29,6 +29,8 @@ from tensorflow.python.keras._impl.keras.engine import InputSpec
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
 from tensorflow.python.keras._impl.keras.layers.recurrent import Recurrent
 from tensorflow.python.keras._impl.keras.utils import conv_utils
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -438,9 +440,9 @@ class ConvLSTM2D(ConvRecurrent2D):
 
   def get_initial_state(self, inputs):
     # (samples, timesteps, rows, cols, filters)
-    initial_state = K.zeros_like(inputs)
+    initial_state = array_ops.zeros_like(inputs)
     # (samples, rows, cols, filters)
-    initial_state = K.sum(initial_state, axis=1)
+    initial_state = math_ops.reduce_sum(initial_state, axis=1)
     shape = list(self.kernel_shape)
     shape[-1] = self.filters
     initial_state = self.input_conv(
@@ -483,8 +485,8 @@ class ConvLSTM2D(ConvRecurrent2D):
   def get_constants(self, inputs, training=None):
     constants = []
     if self.implementation == 0 and 0 < self.dropout < 1:
-      ones = K.zeros_like(inputs)
-      ones = K.sum(ones, axis=1)
+      ones = array_ops.zeros_like(inputs)
+      ones = math_ops.reduce_sum(ones, axis=1)
       ones += 1
 
       def dropped_inputs():
@@ -501,8 +503,8 @@ class ConvLSTM2D(ConvRecurrent2D):
     if 0 < self.recurrent_dropout < 1:
       shape = list(self.kernel_shape)
       shape[-1] = self.filters
-      ones = K.zeros_like(inputs)
-      ones = K.sum(ones, axis=1)
+      ones = array_ops.zeros_like(inputs)
+      ones = math_ops.reduce_sum(ones, axis=1)
       ones = self.input_conv(ones, K.zeros(shape), padding=self.padding)
       ones += 1.
 
diff --git a/tensorflow/python/keras/_impl/keras/layers/core.py b/tensorflow/python/keras/_impl/keras/layers/core.py
index 73e4f15f7e..a709a079fd 100644
--- a/tensorflow/python/keras/_impl/keras/layers/core.py
+++ b/tensorflow/python/keras/_impl/keras/layers/core.py
@@ -37,6 +37,8 @@ from tensorflow.python.keras._impl.keras.utils.generic_utils import func_dump
 from tensorflow.python.keras._impl.keras.utils.generic_utils import func_load
 from tensorflow.python.keras._impl.keras.utils.generic_utils import has_arg
 from tensorflow.python.layers import core as tf_core_layers
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -80,7 +82,7 @@ class Masking(Layer):
   def call(self, inputs):
     boolean_mask = K.any(
         K.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
-    return inputs * K.cast(boolean_mask, inputs.dtype)
+    return inputs * math_ops.cast(boolean_mask, inputs.dtype)
 
   def compute_output_shape(self, input_shape):
     return input_shape
@@ -170,7 +172,7 @@ class SpatialDropout1D(Dropout):
     self.input_spec = InputSpec(ndim=3)
 
   def _get_noise_shape(self, inputs):
-    input_shape = K.shape(inputs)
+    input_shape = array_ops.shape(inputs)
     noise_shape = (input_shape[0], 1, input_shape[2])
     return noise_shape
 
@@ -222,7 +224,7 @@ class SpatialDropout2D(Dropout):
     self.input_spec = InputSpec(ndim=4)
 
   def _get_noise_shape(self, inputs):
-    input_shape = K.shape(inputs)
+    input_shape = array_ops.shape(inputs)
     if self.data_format == 'channels_first':
       return (input_shape[0], input_shape[1], 1, 1)
     elif self.data_format == 'channels_last':
@@ -275,7 +277,7 @@ class SpatialDropout3D(Dropout):
     self.input_spec = InputSpec(ndim=5)
 
   def _get_noise_shape(self, inputs):
-    input_shape = K.shape(inputs)
+    input_shape = array_ops.shape(inputs)
     if self.data_format == 'channels_first':
       return (input_shape[0], input_shape[1], 1, 1, 1)
     elif self.data_format == 'channels_last':
@@ -414,7 +416,7 @@ class Reshape(Layer):
     return tensor_shape.TensorShape(output_shape)
 
   def call(self, inputs):
-    return K.reshape(inputs, (K.shape(inputs)[0],) + self.target_shape)
+    return K.reshape(inputs, (array_ops.shape(inputs)[0],) + self.target_shape)
 
   def get_config(self):
     config = {'target_shape': self.target_shape}
diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings.py b/tensorflow/python/keras/_impl/keras/layers/embeddings.py
index 006ecd3135..a0fd7a9637 100644
--- a/tensorflow/python/keras/_impl/keras/layers/embeddings.py
+++ b/tensorflow/python/keras/_impl/keras/layers/embeddings.py
@@ -24,6 +24,8 @@ from tensorflow.python.keras._impl.keras import initializers
 from tensorflow.python.keras._impl.keras import regularizers
 from tensorflow.python.keras._impl.keras.engine import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -152,8 +154,8 @@ class Embedding(Layer):
 
   def call(self, inputs):
     if K.dtype(inputs) != 'int32':
-      inputs = K.cast(inputs, 'int32')
-    out = K.gather(self.embeddings, inputs)
+      inputs = math_ops.cast(inputs, 'int32')
+    out = array_ops.gather(self.embeddings, inputs)
     return out
 
   def get_config(self):
diff --git a/tensorflow/python/keras/_impl/keras/layers/merge.py b/tensorflow/python/keras/_impl/keras/layers/merge.py
index c660cbd449..6290db29a7 100644
--- a/tensorflow/python/keras/_impl/keras/layers/merge.py
+++ b/tensorflow/python/keras/_impl/keras/layers/merge.py
@@ -23,6 +23,7 @@ from __future__ import print_function
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.engine.base_layer import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
+from tensorflow.python.ops import array_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -137,7 +138,7 @@ class _Merge(Layer):
         for x in inputs:
           x_ndim = K.ndim(x)
           if x_ndim is None:
-            x_shape = K.shape(x)
+            x_shape = array_ops.shape(x)
             batch_size = x_shape[0]
             new_shape = K.concatenate([x_shape[1:], K.expand_dims(batch_size)])
             x_transposed = K.reshape(x,
@@ -159,8 +160,8 @@ class _Merge(Layer):
         if transposed:
           # If inputs have been transposed, we have to transpose the output too.
           if y_ndim is None:
-            y_shape = K.shape(y)
-            y_ndim = K.shape(y_shape)[0]
+            y_shape = array_ops.shape(y)
+            y_ndim = array_ops.shape(y_shape)[0]
             batch_size = y_shape[y_ndim - 1]
             new_shape = K.concatenate(
                 [K.expand_dims(batch_size), y_shape[:y_ndim - 1]])
@@ -418,7 +419,7 @@ class Concatenate(_Merge):
     for input_i, mask_i in zip(inputs, mask):
       if mask_i is None:
         # Input is unmasked. Append all 1s to masks,
-        masks.append(K.ones_like(input_i, dtype='bool'))
+        masks.append(array_ops.ones_like(input_i, dtype='bool'))
       elif K.ndim(mask_i) < K.ndim(input_i):
         # Mask is smaller than the input, expand it
         masks.append(K.expand_dims(mask_i))
diff --git a/tensorflow/python/keras/_impl/keras/layers/noise.py b/tensorflow/python/keras/_impl/keras/layers/noise.py
index e309d160e5..4366b654f2 100644
--- a/tensorflow/python/keras/_impl/keras/layers/noise.py
+++ b/tensorflow/python/keras/_impl/keras/layers/noise.py
@@ -23,6 +23,8 @@ import numpy as np
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.engine import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -58,7 +60,7 @@ class GaussianNoise(Layer):
 
     def noised():
       return inputs + K.random_normal(
-          shape=K.shape(inputs), mean=0., stddev=self.stddev)
+          shape=array_ops.shape(inputs), mean=0., stddev=self.stddev)
 
     return K.in_train_phase(noised, inputs, training=training)
 
@@ -104,7 +106,7 @@ class GaussianDropout(Layer):
       def noised():
         stddev = np.sqrt(self.rate / (1.0 - self.rate))
         return inputs * K.random_normal(
-            shape=K.shape(inputs), mean=1.0, stddev=stddev)
+            shape=array_ops.shape(inputs), mean=1.0, stddev=stddev)
 
       return K.in_train_phase(noised, inputs, training=training)
     return inputs
@@ -153,7 +155,7 @@ class AlphaDropout(Layer):
     self.supports_masking = True
 
   def _get_noise_shape(self, inputs):
-    return self.noise_shape if self.noise_shape else K.shape(inputs)
+    return self.noise_shape if self.noise_shape else array_ops.shape(inputs)
 
   def call(self, inputs, training=None):
     if 0. < self.rate < 1.:
@@ -166,7 +168,7 @@ class AlphaDropout(Layer):
 
         kept_idx = K.greater_equal(
             K.random_uniform(noise_shape, seed=seed), rate)
-        kept_idx = K.cast(kept_idx, K.floatx())
+        kept_idx = math_ops.cast(kept_idx, K.floatx())
 
         # Get affine transformation params
         a = ((1 - rate) * (1 + rate * alpha_p**2))**-0.5
diff --git a/tensorflow/python/keras/_impl/keras/layers/recurrent.py b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
index 791f9b3113..bd7c42e63e 100644
--- a/tensorflow/python/keras/_impl/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
@@ -33,6 +33,9 @@ from tensorflow.python.keras._impl.keras.engine import InputSpec
 from tensorflow.python.keras._impl.keras.engine import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
 from tensorflow.python.keras._impl.keras.utils.generic_utils import has_arg
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
@@ -503,8 +506,10 @@ class RNN(Layer):
 
   def get_initial_state(self, inputs):
     # build an all-zero tensor of shape (samples, output_dim)
-    initial_state = K.zeros_like(inputs)  # (samples, timesteps, input_dim)
-    initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
+    initial_state = array_ops.zeros_like(inputs)
+    # shape of initial_state = (samples, timesteps, input_dim)
+    initial_state = math_ops.reduce_sum(initial_state, axis=(1, 2))
+    # shape of initial_state = (samples,)
     initial_state = K.expand_dims(initial_state)  # (samples, 1)
     if hasattr(self.cell.state_size, '__len__'):
       return [K.tile(initial_state, [1, dim]) for dim in self.cell.state_size]
@@ -631,7 +636,7 @@ class RNN(Layer):
     if self.stateful:
       updates = []
       for i in range(len(states)):
-        updates.append(K.update(self.states[i], states[i]))
+        updates.append(state_ops.assign(self.states[i], states[i]))
       self.add_update(updates, inputs)
 
     if self.return_sequences:
@@ -907,8 +912,7 @@ class SimpleRNNCell(Layer):
     prev_output = states[0]
     if 0 < self.dropout < 1 and self._dropout_mask is None:
       self._dropout_mask = _generate_dropout_mask(
-          _generate_dropout_ones(inputs,
-                                 K.shape(inputs)[-1]),
+          _generate_dropout_ones(inputs, array_ops.shape(inputs)[-1]),
           self.dropout,
           training=training)
     if (0 < self.recurrent_dropout < 1 and
@@ -1309,8 +1313,7 @@ class GRUCell(Layer):
 
     if 0 < self.dropout < 1 and self._dropout_mask is None:
       self._dropout_mask = _generate_dropout_mask(
-          _generate_dropout_ones(inputs,
-                                 K.shape(inputs)[-1]),
+          _generate_dropout_ones(inputs, array_ops.shape(inputs)[-1]),
           self.dropout,
           training=training,
           count=3)
@@ -1793,8 +1796,7 @@ class LSTMCell(Layer):
   def call(self, inputs, states, training=None):
     if 0 < self.dropout < 1 and self._dropout_mask is None:
       self._dropout_mask = _generate_dropout_mask(
-          _generate_dropout_ones(inputs,
-                                 K.shape(inputs)[-1]),
+          _generate_dropout_ones(inputs, array_ops.shape(inputs)[-1]),
           self.dropout,
           training=training,
           count=4)
@@ -2176,7 +2178,7 @@ class LSTM(RNN):
 
 
 def _generate_dropout_ones(inputs, dims):
-  return K.ones((K.shape(inputs)[0], dims))
+  return K.ones((array_ops.shape(inputs)[0], dims))
 
 
 def _generate_dropout_mask(ones, rate, training=None, count=1):
@@ -2351,8 +2353,10 @@ class Recurrent(Layer):
 
   def get_initial_state(self, inputs):
     # build an all-zero tensor of shape (samples, output_dim)
-    initial_state = K.zeros_like(inputs)  # (samples, timesteps, input_dim)
-    initial_state = K.sum(initial_state, axis=(1, 2))  # (samples,)
+    initial_state = array_ops.zeros_like(inputs)
+    # shape of initial_state = (samples, timesteps, input_dim)
+    initial_state = math_ops.reduce_sum(initial_state, axis=(1, 2))
+    # shape of initial_state = (samples,)
     initial_state = K.expand_dims(initial_state)  # (samples, 1)
     initial_state = K.tile(initial_state, [1,
                                            self.units])  # (samples, output_dim)
@@ -2456,7 +2460,7 @@ class Recurrent(Layer):
     if self.stateful:
       updates = []
       for i in range(len(states)):
-        updates.append(K.update(self.states[i], states[i]))
+        updates.append(state_ops.assign(self.states[i], states[i]))
       self.add_update(updates, inputs)
 
     # Properly set learning phase
diff --git a/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py b/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py
index de022153f6..fb743b617f 100644
--- a/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py
+++ b/tensorflow/python/keras/_impl/keras/layers/recurrent_test.py
@@ -24,6 +24,9 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.keras._impl import keras
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import test
 
 
@@ -395,8 +398,8 @@ class RNNTest(test.TestCase):
 
     # Test `get_losses_for` and `losses`
     x = keras.Input((None, 1))
-    loss_1 = keras.backend.sum(x)
-    loss_2 = keras.backend.sum(cells[0].kernel)
+    loss_1 = math_ops.reduce_sum(x)
+    loss_2 = math_ops.reduce_sum(cells[0].kernel)
     cells[0].add_loss(loss_1, inputs=x)
     cells[0].add_loss(loss_2)
     self.assertEqual(len(layer.losses), 2)
@@ -410,10 +413,10 @@ class RNNTest(test.TestCase):
     layer.build((None, None, 1))
 
     x = keras.Input((None, 1))
-    update_1 = keras.backend.update_add(
-        cells[0].kernel, x[0, 0, 0] * cells[0].kernel)
-    update_2 = keras.backend.update_add(
-        cells[0].kernel, keras.backend.ones_like(cells[0].kernel))
+    update_1 = state_ops.assign_add(cells[0].kernel,
+                                    x[0, 0, 0] * cells[0].kernel)
+    update_2 = state_ops.assign_add(cells[0].kernel,
+                                    array_ops.ones_like(cells[0].kernel))
     cells[0].add_update(update_1, inputs=x)
     cells[0].add_update(update_2)
     self.assertEqual(len(layer.updates), 2)
diff --git a/tensorflow/python/keras/_impl/keras/layers/wrappers.py b/tensorflow/python/keras/_impl/keras/layers/wrappers.py
index 76ddd9299d..12f33614e2 100644
--- a/tensorflow/python/keras/_impl/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/_impl/keras/layers/wrappers.py
@@ -28,6 +28,7 @@ from tensorflow.python.keras._impl.keras.engine import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
 from tensorflow.python.keras._impl.keras.utils.generic_utils import has_arg
 from tensorflow.python.layers import utils as tf_layers_util
+from tensorflow.python.ops import array_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -209,7 +210,7 @@ class TimeDistributed(Wrapper):
       # We can go with reshape-based implementation for performance.
       input_length = input_shape[1]
       if not input_length:
-        input_length = K.shape(inputs)[1]
+        input_length = array_ops.shape(inputs)[1]
       # Shape: (num_samples * timesteps, ...). And track the
       # transformation in self._input_map.
       input_uid = tf_layers_util.object_list_uid(inputs)
diff --git a/tensorflow/python/keras/_impl/keras/losses.py b/tensorflow/python/keras/_impl/keras/losses.py
index 1576ed7b99..859bda0c9d 100644
--- a/tensorflow/python/keras/_impl/keras/losses.py
+++ b/tensorflow/python/keras/_impl/keras/losses.py
@@ -24,6 +24,7 @@ import six
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -66,7 +67,7 @@ def hinge(y_true, y_pred):
 
 @tf_export('keras.losses.categorical_hinge')
 def categorical_hinge(y_true, y_pred):
-  pos = K.sum(y_true * y_pred, axis=-1)
+  pos = math_ops.reduce_sum(y_true * y_pred, axis=-1)
   neg = K.max((1. - y_true) * y_pred, axis=-1)
   return K.maximum(0., neg - pos + 1.)
 
@@ -117,7 +118,7 @@ def binary_crossentropy(y_true, y_pred):
 def kullback_leibler_divergence(y_true, y_pred):
   y_true = K.clip(y_true, K.epsilon(), 1)
   y_pred = K.clip(y_pred, K.epsilon(), 1)
-  return K.sum(y_true * K.log(y_true / y_pred), axis=-1)
+  return math_ops.reduce_sum(y_true * K.log(y_true / y_pred), axis=-1)
 
 
 @tf_export('keras.metrics.poisson', 'keras.losses.poisson')
@@ -129,7 +130,7 @@ def poisson(y_true, y_pred):
 def cosine_proximity(y_true, y_pred):
   y_true = K.l2_normalize(y_true, axis=-1)
   y_pred = K.l2_normalize(y_pred, axis=-1)
-  return -K.sum(y_true * y_pred, axis=-1)
+  return -math_ops.reduce_sum(y_true * y_pred, axis=-1)
 
 
 # Aliases.
diff --git a/tensorflow/python/keras/_impl/keras/metrics.py b/tensorflow/python/keras/_impl/keras/metrics.py
index 82778a3dc4..24192cf5a1 100644
--- a/tensorflow/python/keras/_impl/keras/metrics.py
+++ b/tensorflow/python/keras/_impl/keras/metrics.py
@@ -37,6 +37,7 @@ from tensorflow.python.keras._impl.keras.losses import sparse_categorical_crosse
 from tensorflow.python.keras._impl.keras.losses import squared_hinge
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -47,15 +48,15 @@ def binary_accuracy(y_true, y_pred):
 
 @tf_export('keras.metrics.categorical_accuracy')
 def categorical_accuracy(y_true, y_pred):
-  return K.cast(
+  return math_ops.cast(
       K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx())
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
-  return K.cast(
+  return math_ops.cast(
       K.equal(
-          K.max(y_true, axis=-1), K.cast(K.argmax(y_pred, axis=-1),
-                                         K.floatx())), K.floatx())
+          K.max(y_true, axis=-1),
+          math_ops.cast(K.argmax(y_pred, axis=-1), K.floatx())), K.floatx())
 
 
 @tf_export('keras.metrics.top_k_categorical_accuracy')
@@ -66,7 +67,8 @@ def top_k_categorical_accuracy(y_true, y_pred, k=5):
 @tf_export('keras.metrics.sparse_top_k_categorical_accuracy')
 def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5):
   return K.mean(
-      K.in_top_k(y_pred, K.cast(K.max(y_true, axis=-1), 'int32'), k), axis=-1)
+      K.in_top_k(y_pred, math_ops.cast(K.max(y_true, axis=-1), 'int32'), k),
+      axis=-1)
 
 
 # Aliases
diff --git a/tensorflow/python/keras/_impl/keras/metrics_test.py b/tensorflow/python/keras/_impl/keras/metrics_test.py
index 44289ea02a..2b73e0c16f 100644
--- a/tensorflow/python/keras/_impl/keras/metrics_test.py
+++ b/tensorflow/python/keras/_impl/keras/metrics_test.py
@@ -21,6 +21,8 @@ from __future__ import print_function
 import numpy as np
 
 from tensorflow.python.keras._impl import keras
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.platform import test
 
 
@@ -104,16 +106,16 @@ class KerasMetricsTest(test.TestCase):
             The total number of true positives seen this epoch at the
                 completion of the batch.
         """
-        y_true = keras.backend.cast(y_true, 'int32')
-        y_pred = keras.backend.cast(keras.backend.round(y_pred), 'int32')
-        correct_preds = keras.backend.cast(
+        y_true = math_ops.cast(y_true, 'int32')
+        y_pred = math_ops.cast(keras.backend.round(y_pred), 'int32')
+        correct_preds = math_ops.cast(
             keras.backend.equal(y_pred, y_true), 'int32')
-        true_pos = keras.backend.cast(
-            keras.backend.sum(correct_preds * y_true), 'int32')
+        true_pos = math_ops.cast(
+            math_ops.reduce_sum(correct_preds * y_true), 'int32')
         current_true_pos = self.true_positives * 1
-        self.add_update(keras.backend.update_add(self.true_positives,
-                                                 true_pos),
-                        inputs=[y_true, y_pred])
+        self.add_update(
+            state_ops.assign_add(self.true_positives, true_pos),
+            inputs=[y_true, y_pred])
         return current_true_pos + true_pos
 
     metric_fn = BinaryTruePositives()
diff --git a/tensorflow/python/keras/_impl/keras/optimizers.py b/tensorflow/python/keras/_impl/keras/optimizers.py
index acbb9091d3..dc0e472b88 100644
--- a/tensorflow/python/keras/_impl/keras/optimizers.py
+++ b/tensorflow/python/keras/_impl/keras/optimizers.py
@@ -31,6 +31,7 @@ from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import control_flow_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import state_ops
 from tensorflow.python.training import distribute as distribute_lib
 from tensorflow.python.training import optimizer as tf_optimizer_module
 from tensorflow.python.training import training_util
@@ -118,7 +119,7 @@ class Optimizer(object):
                        'Common ops without gradient: '
                        'K.argmax, K.round, K.eval.')
     if hasattr(self, 'clipnorm') and self.clipnorm > 0:
-      norm = K.sqrt(sum([K.sum(K.square(g)) for g in grads]))
+      norm = K.sqrt(sum([math_ops.reduce_sum(K.square(g)) for g in grads]))
       grads = [clip_norm(g, self.clipnorm, norm) for g in grads]
     if hasattr(self, 'clipvalue') and self.clipvalue > 0:
       grads = [K.clip(g, -self.clipvalue, self.clipvalue) for g in grads]
@@ -204,20 +205,20 @@ class SGD(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
     # momentum
     shapes = [K.int_shape(p) for p in params]
     moments = [K.zeros(shape) for shape in shapes]
     self.weights = [self.iterations] + moments
     for p, g, m in zip(params, grads, moments):
       v = self.momentum * m - lr * g  # velocity
-      self.updates.append(K.update(m, v))
+      self.updates.append(state_ops.assign(m, v))
 
       if self.nesterov:
         new_p = p + self.momentum * v - lr * g
@@ -228,7 +229,7 @@ class SGD(Optimizer):
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -277,25 +278,25 @@ class RMSprop(Optimizer):
     grads = self.get_gradients(loss, params)
     accumulators = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
     self.weights = accumulators
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
 
     for p, g, a in zip(params, grads, accumulators):
       # update accumulator
       new_a = self.rho * a + (1. - self.rho) * K.square(g)
-      self.updates.append(K.update(a, new_a))
+      self.updates.append(state_ops.assign(a, new_a))
       new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
 
       # Apply constraints.
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -339,24 +340,24 @@ class Adagrad(Optimizer):
     shapes = [K.int_shape(p) for p in params]
     accumulators = [K.zeros(shape) for shape in shapes]
     self.weights = accumulators
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
 
     for p, g, a in zip(params, grads, accumulators):
       new_a = a + K.square(g)  # update accumulator
-      self.updates.append(K.update(a, new_a))
+      self.updates.append(state_ops.assign(a, new_a))
       new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
 
       # Apply constraints.
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -403,18 +404,18 @@ class Adadelta(Optimizer):
     accumulators = [K.zeros(shape) for shape in shapes]
     delta_accumulators = [K.zeros(shape) for shape in shapes]
     self.weights = accumulators + delta_accumulators
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
 
     for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
       # update accumulator
       new_a = self.rho * a + (1. - self.rho) * K.square(g)
-      self.updates.append(K.update(a, new_a))
+      self.updates.append(state_ops.assign(a, new_a))
 
       # use the new accumulator and the *old* delta_accumulator
       update = g * K.sqrt(d_a + self.epsilon) / K.sqrt(new_a + self.epsilon)
@@ -424,11 +425,11 @@ class Adadelta(Optimizer):
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
 
       # update delta_accumulator
       new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
-      self.updates.append(K.update(d_a, new_d_a))
+      self.updates.append(state_ops.assign(d_a, new_d_a))
     return self.updates
 
   def get_config(self):
@@ -483,15 +484,15 @@ class Adam(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
 
-    t = K.cast(self.iterations, K.floatx()) + 1
+    t = math_ops.cast(self.iterations, K.floatx()) + 1
     lr_t = lr * (
         K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
 
@@ -509,19 +510,19 @@ class Adam(Optimizer):
       if self.amsgrad:
         vhat_t = K.maximum(vhat, v_t)
         p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
-        self.updates.append(K.update(vhat, vhat_t))
+        self.updates.append(state_ops.assign(vhat, vhat_t))
       else:
         p_t = p - lr_t * m_t / (K.sqrt(v_t) + self.epsilon)
 
-      self.updates.append(K.update(m, m_t))
-      self.updates.append(K.update(v, v_t))
+      self.updates.append(state_ops.assign(m, m_t))
+      self.updates.append(state_ops.assign(v, v_t))
       new_p = p_t
 
       # Apply constraints.
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -573,15 +574,15 @@ class Adamax(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
     lr = self.lr
     if self.initial_decay > 0:
-      lr = lr * (1. /  # pylint: disable=g-no-augmented-assignment
-                 (1. + self.decay * K.cast(self.iterations,
-                                           K.dtype(self.decay))))
+      lr = lr * (  # pylint: disable=g-no-augmented-assignment
+          1. / (1. + self.decay * math_ops.cast(self.iterations,
+                                                K.dtype(self.decay))))
 
-    t = K.cast(self.iterations, K.floatx()) + 1
+    t = math_ops.cast(self.iterations, K.floatx()) + 1
     lr_t = lr / (1. - K.pow(self.beta_1, t))
 
     shapes = [K.int_shape(p) for p in params]
@@ -597,15 +598,15 @@ class Adamax(Optimizer):
       u_t = K.maximum(self.beta_2 * u, K.abs(g))
       p_t = p - lr_t * m_t / (u_t + self.epsilon)
 
-      self.updates.append(K.update(m, m_t))
-      self.updates.append(K.update(u, u_t))
+      self.updates.append(state_ops.assign(m, m_t))
+      self.updates.append(state_ops.assign(u, u_t))
       new_p = p_t
 
       # Apply constraints.
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -659,9 +660,9 @@ class Nadam(Optimizer):
 
   def get_updates(self, loss, params):
     grads = self.get_gradients(loss, params)
-    self.updates = [K.update_add(self.iterations, 1)]
+    self.updates = [state_ops.assign_add(self.iterations, 1)]
 
-    t = K.cast(self.iterations, K.floatx()) + 1
+    t = math_ops.cast(self.iterations, K.floatx()) + 1
 
     # Due to the recommendations in [2], i.e. warming momentum schedule
     momentum_cache_t = self.beta_1 * (
@@ -689,8 +690,8 @@ class Nadam(Optimizer):
       m_t_bar = (
           1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
 
-      self.updates.append(K.update(m, m_t))
-      self.updates.append(K.update(v, v_t))
+      self.updates.append(state_ops.assign(m, m_t))
+      self.updates.append(state_ops.assign(v, v_t))
 
       p_t = p - self.lr * m_t_bar / (K.sqrt(v_t_prime) + self.epsilon)
       new_p = p_t
@@ -699,7 +700,7 @@ class Nadam(Optimizer):
       if getattr(p, 'constraint', None) is not None:
         new_p = p.constraint(new_p)
 
-      self.updates.append(K.update(p, new_p))
+      self.updates.append(state_ops.assign(p, new_p))
     return self.updates
 
   def get_config(self):
@@ -743,7 +744,7 @@ class TFOptimizer(Optimizer):
       global_step = training_util.get_global_step()
       opt_update = self.optimizer.apply_gradients(grads, global_step)
     else:
-      self.updates = [K.update_add(self.iterations, 1)]
+      self.updates = [state_ops.assign_add(self.iterations, 1)]
       if not params:
         return self.updates
 
diff --git a/tensorflow/python/keras/_impl/keras/regularizers.py b/tensorflow/python/keras/_impl/keras/regularizers.py
index 2c30844647..fdb9d33810 100644
--- a/tensorflow/python/keras/_impl/keras/regularizers.py
+++ b/tensorflow/python/keras/_impl/keras/regularizers.py
@@ -23,6 +23,7 @@ import six
 from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
+from tensorflow.python.ops import math_ops
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -55,9 +56,9 @@ class L1L2(Regularizer):
   def __call__(self, x):
     regularization = 0.
     if self.l1:
-      regularization += K.sum(self.l1 * K.abs(x))
+      regularization += math_ops.reduce_sum(self.l1 * K.abs(x))
     if self.l2:
-      regularization += K.sum(self.l2 * K.square(x))
+      regularization += math_ops.reduce_sum(self.l2 * K.square(x))
     return regularization
 
   def get_config(self):
diff --git a/tensorflow/python/keras/_impl/keras/utils/layer_utils.py b/tensorflow/python/keras/_impl/keras/utils/layer_utils.py
index 4c8009dfd8..902972ecbb 100644
--- a/tensorflow/python/keras/_impl/keras/utils/layer_utils.py
+++ b/tensorflow/python/keras/_impl/keras/utils/layer_utils.py
@@ -35,7 +35,7 @@ def count_params(weights):
   Returns:
       The total number of scalars composing the weights
   """
-  return int(np.sum([K.count_params(p) for p in set(weights)]))
+  return int(np.sum([np.prod(p.get_shape().as_list()) for p in set(weights)]))
 
 
 def print_summary(model, line_length=None, positions=None, print_fn=None):
@@ -193,8 +193,7 @@ def print_summary(model, line_length=None, positions=None, print_fn=None):
   else:
     trainable_count = count_params(model.trainable_weights)
 
-  non_trainable_count = int(
-      np.sum([K.count_params(p) for p in set(model.non_trainable_weights)]))
+  non_trainable_count = count_params(model.non_trainable_weights)
 
   print_fn('Total params: {:,}'.format(trainable_count + non_trainable_count))
   print_fn('Trainable params: {:,}'.format(trainable_count))
-- 
GitLab


From 6d05c781d71b09f10246b2d15039e1b1df899f62 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 3 Apr 2018 17:57:08 -0700
Subject: [PATCH 893/960] Fix bug where name generated by Graph::NewName can
 conflict with generated Send node name.

I fixed this very locally to avoid having to fix a bunch of tests
(this is currently blocking enabling the C API), but this should be
fixed more generally in the future.

PiperOrigin-RevId: 191528409
---
 tensorflow/core/graph/quantize_training.cc | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/graph/quantize_training.cc b/tensorflow/core/graph/quantize_training.cc
index cb0fc8a154..3b6e8cc233 100644
--- a/tensorflow/core/graph/quantize_training.cc
+++ b/tensorflow/core/graph/quantize_training.cc
@@ -259,8 +259,14 @@ Status AddRestoreVariableSubgraphs(Graph* graph, Node* save_op,
   const string restore_op_name = strings::StrCat(name_prefix, "/RestoreV2");
   const string assign_op_name = strings::StrCat(name_prefix, "/Assign");
   for (Node* var : variables) {
-    string new_restore_op_name = graph->NewName(restore_op_name);
-    string new_assign_op_name = graph->NewName(assign_op_name);
+    // Add an extra prefix after calling graph->NewName because the "unique"
+    // name may conflict with names generated for Send nodes.
+    // TODO(b/77547936): fix this more generally and get rid of the extra prefix
+    // here.
+    string new_restore_op_name =
+        strings::StrCat(graph->NewName(restore_op_name), "_qt");
+    string new_assign_op_name =
+        strings::StrCat(graph->NewName(assign_op_name), "_qt");
     string tensor_names_op_name =
         strings::StrCat(new_restore_op_name, "/tensor_names");
     string shape_and_slices_op_name =
-- 
GitLab


From 2615b467def240e6a90f309aaa56311677104d63 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 3 Apr 2018 18:08:21 -0700
Subject: [PATCH 894/960] [XLA] Reshape mover should only transform a
 reshape/transpose instruction if all the users of the instruction can be
 transformed.

This is because if only part of the users of a reshape/transpose instruction are
reshape-move transformed, the original reshape/transpose instruction can't be
eliminated while a modified clone copy of the instruction is added to support
the transformation. As a result, the transformation increases the number of
reshape/transpose instructions in the kerenel and can potentially increase
memory consumption.

Add two test cases.

PiperOrigin-RevId: 191529681
---
 .../compiler/xla/service/reshape_mover.cc     | 306 ++++++++++--------
 .../xla/service/reshape_mover_test.cc         |  80 ++++-
 2 files changed, 255 insertions(+), 131 deletions(-)

diff --git a/tensorflow/compiler/xla/service/reshape_mover.cc b/tensorflow/compiler/xla/service/reshape_mover.cc
index f15117f45c..49ec38eb62 100644
--- a/tensorflow/compiler/xla/service/reshape_mover.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover.cc
@@ -53,16 +53,8 @@ bool IsReshapeOrTranspose(const HloInstruction* instruction) {
          instruction->opcode() == HloOpcode::kTranspose;
 }
 
-// Returns true if `a` is a broadcast instruction to target shape `shape` and
-// its operand is a scalar.
-bool IsBroadcastScalarToShape(const HloInstruction* a, const Shape& shape) {
-  return a->opcode() == HloOpcode::kBroadcast &&
-         ShapeUtil::SameDimensions(a->shape(), shape) &&
-         ShapeUtil::IsScalar(a->operand(0)->shape());
-}
-
-// Returns true iff `instruction` can change its shape simply by adjusting
-// metadata.
+// Returns true if `instruction` can change its shape simply by adjusting
+// metadata or if `instruction` is a broadcast of a scalar value.
 bool CanTriviallyChangeShape(const HloInstruction* instruction) {
   // NOTE: Technically a sequence of reshape(reshape(constant)) is also
   // trivially reshapable, so we might be tempted to simply recurse if
@@ -97,19 +89,30 @@ bool CanTriviallyChangeShape(const HloInstruction* instruction) {
     return true;
   }
 
+  // A broadcase of scalar can trivially change its shape.
+  if (instruction->opcode() == HloOpcode::kBroadcast &&
+      ShapeUtil::IsScalar(instruction->operand(0)->shape())) {
+    return true;
+  }
+
   return false;
 }
 
-// Finds the first non-scalar operand of an instruction that is a non-trivial
-// reshape or transpose. Returns the operand if it is found or nullptr if not
-// found.
+// Returns true iff `instruction` is a reshape/transpose instruction for which
+// a shape change is nontrivial.
+bool IsNontrivialReshape(const HloInstruction* instruction) {
+  return !ShapeUtil::IsScalar(instruction->shape()) &&
+         IsReshapeOrTranspose(instruction) &&
+         !CanTriviallyChangeShape(instruction->operand(0));
+}
+
+// Finds the first operand of an instruction that is a non-trivial reshape or
+// transpose. Returns such an operand or nullptr if not found.
 HloInstruction* FirstNonScalarAndNonTrivialReshapeOperand(
     const HloInstruction* hlo) {
   for (HloInstruction* operand : hlo->operands()) {
-    if (!ShapeUtil::IsScalar(operand->shape()) &&
-        IsReshapeOrTranspose(operand) &&
-        !CanTriviallyChangeShape(operand->operand(0))) {
-      VLOG(5) << "Found first non-scalar and non-trivial reshape operand of "
+    if (IsNontrivialReshape(operand)) {
+      VLOG(5) << "Found first non-trivial reshape operand of "
               << hlo->ToString(HloPrintOptions().set_print_metadata(false))
               << ":\n\t"
               << operand->ToString(HloPrintOptions().set_print_metadata(false));
@@ -119,7 +122,7 @@ HloInstruction* FirstNonScalarAndNonTrivialReshapeOperand(
   return nullptr;
 }
 
-// Returns whether `a` and `b` are equivalent for the purposes of this pass.
+// Returns whether `a` and `b` are equivalent reshapes/transposes.
 bool AreEquivalentReshapes(const HloInstruction* a, const HloInstruction* b) {
   if (a->opcode() != b->opcode() ||
       !ShapeUtil::SameDimensions(a->shape(), b->shape())) {
@@ -136,85 +139,14 @@ bool AreEquivalentReshapes(const HloInstruction* a, const HloInstruction* b) {
   }
 }
 
-// Returns true if all operands of `instruction` can easily change shape.
-// Operands can easily change shape if they are all reshapes/transposes to and
-// from the same shape. Additionally, operands like constant, rng, and any
-// scalar change shape with only an adjustment of metadata.
-bool AllOperandsHaveEasyShapeChanges(
-    const HloInstruction* instruction,
-    const HloInstruction* first_reshape_operand) {
-  auto print_no_metadata = HloPrintOptions().set_print_metadata(false);
-  VLOG(3) << "** Checking whether all operands have easy shape changes: "
-          << instruction->ToString(print_no_metadata);
-  // Check whether all operands:
-  //    0. Have the same dimensions as the output -- if not, it may be
-  //       implicitly broadcast, which can confound the movement's
-  //       correctness.
-  //
-  // And one of the following:
-  //    1. Are reshapes or transposes that have the same input and
-  //       output shapes as all other reshaped or transposed operands.
-  //     or
-  //    2. Are one of kConstant, kRng, and scalars that can change shape
-  //    trivially,
-  //     or
-  //    3. Are broadcast with a scalar operand.
-  for (const HloInstruction* operand : instruction->operands()) {
-    if (!ShapeUtil::SameDimensions(operand->shape(), instruction->shape())) {
-      VLOG(5) << "Operand shape differs from output shape; may be "
-                 "implicitly broadcast, so preventing "
-                 "movement\n\toperand: "
-              << operand->ToString(print_no_metadata) << "\n\tinstruction: "
-              << instruction->ToString(print_no_metadata);
-      return false;
-    }
-
-    // Skip the rest checks if the current operand is first_reshape_operand
-    // itself.
-    if (first_reshape_operand == operand) {
-      continue;
-    }
-
-    if (AreEquivalentReshapes(first_reshape_operand, operand)) {
-      VLOG(5) << "Are equivalent reshapes:\n\tfirst_reshape_operand: "
-              << first_reshape_operand->ToString(print_no_metadata)
-              << "\n\toperand: " << operand->ToString(print_no_metadata);
-      continue;
-    }
-
-    if (CanTriviallyChangeShape(operand)) {
-      VLOG(5) << "Operand can trivially change shape: "
-              << operand->ToString(print_no_metadata);
-      continue;
-    }
-
-    if (IsBroadcastScalarToShape(operand, first_reshape_operand->shape())) {
-      VLOG(5) << "Broadcast scalar to shape: "
-              << operand->ToString(print_no_metadata);
-      continue;
-    }
-
-    // TODO(someone): Look into supporting general ops for the operands as
-    // well.
-    VLOG(5) << "Operand is neither equalivant to the first Reshape operand"
-               "nor can trivially change shape: "
-            << operand->ToString(print_no_metadata);
-    return false;
-  }
-
-  VLOG(3) << "All operands have easy shape changes: "
-          << instruction->ToString(print_no_metadata);
-  return true;
-}
-
 // This function is called once we've decided to sink reshape/transpose operands
 // across an instruction. It returns an updated `operand` with a shape that
 // plays nicely with `new_operand_shape`; either it has the same shape (of the
 // correct type), or it is a scalar that may be implicitly broadcast.
-HloInstruction* UpdateOperand(HloComputation* computation,
-                              const HloInstruction* first_reshape_operand,
+HloInstruction* UpdateOperand(const HloInstruction* first_reshape_operand,
                               const Shape& new_operand_shape,
                               HloInstruction* operand) {
+  HloComputation* computation = operand->parent();
   const PrimitiveType element_type = operand->shape().element_type();
   const Shape new_shape =
       ShapeUtil::ChangeElementType(new_operand_shape, element_type);
@@ -245,42 +177,24 @@ HloInstruction* UpdateOperand(HloComputation* computation,
       VLOG(5) << "Using existing operand of kReshape or kTranspose";
       return operand->mutable_operand(0);
     }
-    case HloOpcode::kBroadcast:
-      CHECK(IsBroadcastScalarToShape(operand, first_reshape_operand->shape()));
-      VLOG(5) << "Changing broadcast";
-      return computation->AddInstruction(
+    case HloOpcode::kBroadcast: {
+      CHECK(ShapeUtil::IsScalar(operand->operand(0)->shape()));
+      HloInstruction* inst = computation->AddInstruction(
           operand->CloneWithNewOperands(new_shape, operand->operands()));
+      VLOG(5) << "Changing broadcast from " << operand->ToString() << " to "
+              << inst->ToString();
+      return inst;
+    }
 
     default:
       LOG(FATAL) << "Unexpected operand opcode during update: " << operand;
   }
 }
 
-// Try to sink any reshape or transpose operands of `instruction` across it. We
-// do so if `instruction` is elementwise and all operands are either equivalent
-// reshapes/transposes or are trivially reshapable.
-StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
-                                         HloInstruction* instruction) {
-  // Only perform sinks for live elementwise instructions with operands.
-  const bool is_dead = instruction->user_count() == 0 &&
-                       instruction != computation->root_instruction();
-  if (!instruction->IsElementwise() || instruction->operands().empty() ||
-      is_dead) {
-    return false;
-  }
-
-  // Only perform sinks if there are any nontrivial reshape/transpose operands.
-  const HloInstruction* first_reshape_operand =
-      FirstNonScalarAndNonTrivialReshapeOperand(instruction);
-  if (!first_reshape_operand) {
-    return false;
-  }
-
-  // Only perform sinks if all operands can easily change shape.
-  if (!AllOperandsHaveEasyShapeChanges(instruction, first_reshape_operand)) {
-    return false;
-  }
-
+// Actually performs the reshape-move transformation -- that is, sinks the
+// reshape or transpose operands of `instruction` across it.
+StatusOr<bool> PerformSinkReshapeOrTranspose(
+    HloInstruction* instruction, const HloInstruction* first_reshape_operand) {
   auto print_no_metadata = HloPrintOptions().set_print_metadata(false);
   // At this point we've decided to sink reshape/transpose operands.
   const Shape& new_operand_shape = first_reshape_operand->operand(0)->shape();
@@ -301,8 +215,8 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
     }
     VLOG(3) << "Updating operand #" << i << ": "
             << operands[i]->ToString(print_no_metadata);
-    operands[i] = UpdateOperand(computation, first_reshape_operand,
-                                new_operand_shape, operands[i]);
+    operands[i] =
+        UpdateOperand(first_reshape_operand, new_operand_shape, operands[i]);
   }
   if (HloOpcode::kFusion == instruction->opcode()) {
     // Here we already know `instruction` is elementwise, and no operand is
@@ -314,6 +228,7 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
       *shape->mutable_layout() = new_operand_shape.layout();
     }
   }
+  HloComputation* computation = instruction->parent();
   HloInstruction* new_elementwise =
       computation->AddInstruction(instruction->CloneWithNewOperands(
           // `instruction` may change the element type, e.g., from
@@ -348,6 +263,141 @@ StatusOr<bool> TrySinkReshapeOrTranspose(HloComputation* computation,
   return true;
 }
 
+// Returns true if the instruction is a reshape-move candidate.
+//
+// An instruction is a reshape-move candidate if the instruction is elementwise,
+// has at least one nontrivial reshape/transpose operand, and its operands are
+// either trivially reshapable or are equivalent nontrivial reshapes/transposes.
+bool IsReshapeMoveCandidate(HloInstruction* instruction) {
+  auto print_no_metadata = HloPrintOptions().set_print_metadata(false);
+  VLOG(5) << "** Checking instruction: "
+          << instruction->ToString(print_no_metadata);
+
+  // Only perform reshape-move for live elementwise instructions with operands.
+  const bool is_dead = instruction->user_count() == 0 &&
+                       instruction != instruction->parent()->root_instruction();
+  if (!instruction->IsElementwise() || instruction->operands().empty() ||
+      is_dead) {
+    return false;
+  }
+
+  // Check whether all operands:
+  //    0. Have the same dimensions as the output -- if not, they may be
+  //       implicitly broadcast, which can confound the movement's
+  //       correctness.
+  //
+  // And one of the following:
+  //    1. Are reshapes or transposes that have the same input and
+  //       output shapes as all other reshaped or transposed operands.
+  //     or
+  //    2. Are one of kConstant, kRng, broadcast of a scalar value, and scalars
+  //     that can change shape trivially.
+  const HloInstruction* first_reshape_operand = nullptr;
+  for (const HloInstruction* operand : instruction->operands()) {
+    if (!ShapeUtil::SameDimensions(operand->shape(), instruction->shape())) {
+      VLOG(5) << "Operand shape differs from output shape; may be "
+                 "implicitly broadcast, so preventing "
+                 "movement\n\toperand: "
+              << operand->ToString(print_no_metadata) << "\n\tinstruction: "
+              << instruction->ToString(print_no_metadata);
+      return false;
+    }
+
+    if (CanTriviallyChangeShape(operand)) {
+      VLOG(5) << "Operand can trivially change shape: "
+              << operand->ToString(print_no_metadata);
+      continue;
+    }
+
+    if (!IsNontrivialReshape(operand)) {
+      VLOG(5) << "Operand can't trivially change shape: "
+              << operand->ToString(print_no_metadata);
+      return false;
+    }
+
+    if (first_reshape_operand == nullptr) {
+      first_reshape_operand = operand;
+      VLOG(5) << "First reshape operand "
+              << operand->ToString(print_no_metadata);
+    } else if (AreEquivalentReshapes(first_reshape_operand, operand)) {
+      VLOG(5)
+          << "Operand is an equivalent reshape of the first reshape operand "
+          << operand->ToString(print_no_metadata);
+    } else {
+      // TODO(someone): Look into supporting general ops for the operands as
+      // well.
+      VLOG(5) << "Operand is a reshape but is not equivalent to the first "
+                 "Reshape operand"
+              << operand->ToString(print_no_metadata);
+      return false;
+    }
+  }
+
+  if (first_reshape_operand) {
+    VLOG(5) << "All operands have easy shape changes: "
+            << instruction->ToString(print_no_metadata);
+  }
+
+  return first_reshape_operand != nullptr;
+}
+
+// Reshape-moves all qualifying instructions in reshape_candidates.  Returns
+// true if it makes changes.
+//
+// `reshape_candidates` is a set of HloInstructions with nontrivial reshape
+// operands, and a instruction in the set can be reshape-moved iff all the users
+// of its nontrivial reshape operands can also be reshaped-moved.
+//
+// The algorithm here iteratively finds the nontrivial operands with users that
+// are outside the set of `reshape_candidates`, and removes their users from
+// `reshape_candidates`, until either `reshape_candidates` becomes empty or none
+// of the remaining nontrivial operands have users outside `reshape_candidates`.
+// In the later case, all the remaining instructions in `reshape_candidates`
+// are reshape-moved and the routine returns true.
+StatusOr<bool> TryReshapeMoveOnCandidates(
+    HloInstructionSet* reshape_candidates) {
+  bool removed = true;
+  while (!reshape_candidates->empty() && removed) {
+    if (VLOG_IS_ON(5)) {
+      for (const HloInstruction* instruction : *reshape_candidates) {
+        VLOG(5) << "candidate " << instruction->ToString();
+      }
+    }
+    ConstHloInstructionSet nontrivial_operands;
+    for (const HloInstruction* instruction : *reshape_candidates) {
+      for (const auto* operand : instruction->operands()) {
+        if (IsNontrivialReshape(operand)) {
+          nontrivial_operands.insert(operand);
+        }
+      }
+    }
+
+    removed = false;
+    for (auto operand : nontrivial_operands) {
+      if (c_any_of(operand->users(), [&](HloInstruction* user) {
+            return !reshape_candidates->count(user);
+          })) {
+        for (auto* user : operand->users()) {
+          removed |= reshape_candidates->erase(user) > 0;
+        }
+      }
+    }
+  }
+
+  if (reshape_candidates->empty()) {
+    return false;
+  }
+  for (HloInstruction* instruction : *reshape_candidates) {
+    const HloInstruction* first_reshape_operand =
+        FirstNonScalarAndNonTrivialReshapeOperand(instruction);
+    TF_ASSIGN_OR_RETURN(
+        bool did_change,
+        PerformSinkReshapeOrTranspose(instruction, first_reshape_operand));
+    CHECK(did_change);
+  }
+  return true;
+}
+
 }  // namespace
 
 StatusOr<bool> ReshapeMover::Run(HloModule* module) {
@@ -355,11 +405,15 @@ StatusOr<bool> ReshapeMover::Run(HloModule* module) {
   VLOG(2) << "Pre ReshapeMover HLO:";
   XLA_VLOG_LINES(2, module->ToString());
   for (auto* comp : module->MakeNonfusionComputations()) {
-    for (HloInstruction* instruction : comp->MakeInstructionPostOrder()) {
-      TF_ASSIGN_OR_RETURN(bool did_change,
-                          TrySinkReshapeOrTranspose(comp, instruction));
-      changed |= did_change;
+    HloInstructionSet reshape_candidates;
+    for (HloInstruction* instruction : comp->instructions()) {
+      if (IsReshapeMoveCandidate(instruction)) {
+        reshape_candidates.insert(instruction);
+      }
     }
+    TF_ASSIGN_OR_RETURN(bool did_change,
+                        TryReshapeMoveOnCandidates(&reshape_candidates));
+    changed |= did_change;
   }
   VLOG(2) << "Post ReshapeMover HLO:";
   XLA_VLOG_LINES(2, module->ToString());
diff --git a/tensorflow/compiler/xla/service/reshape_mover_test.cc b/tensorflow/compiler/xla/service/reshape_mover_test.cc
index 4e0a0a8832..094f7319f4 100644
--- a/tensorflow/compiler/xla/service/reshape_mover_test.cc
+++ b/tensorflow/compiler/xla/service/reshape_mover_test.cc
@@ -564,15 +564,15 @@ TEST_F(ReshapeMoverTest, SinkTransposeAcrossBroadcastScalar) {
   const string hlo_string = R"(
     HloModule TransposeMulInversedTransposeModule
     ENTRY TransposeMulInversedTranspose {
-      src0 = f32[1,20,8,32]{3,2,1,0} parameter(0)
-      transpose0 = f32[1,8,20,32]{3,2,1,0} transpose(src0), dimensions={0,2,1,3}
+      src0 = f32[20,8]{1,0} parameter(0)
+      transpose0 = f32[8,20]{1,0} transpose(src0), dimensions={1,0}
       src1 = f32[] parameter(1)
-      broadcast0 = f32[1,8,20,32]{3,2,1,0} broadcast(src1), dimensions={}
-      ROOT multiply0 = f32[1,8,20,32]{3,2,1,0} multiply(transpose0, broadcast0)
+      broadcast0 = f32[8,20]{1,0} broadcast(src1), dimensions={}
+      ROOT multiply0 = f32[8,20]{1,0} multiply(transpose0, broadcast0)
     }
   )";
 
-  ParseAndVerifyModule(hlo_string.c_str());
+  ParseAndVerifyModule(hlo_string);
   TF_ASSERT_OK_AND_ASSIGN(bool changed, ReshapeMover().Run(&module()));
   EXPECT_TRUE(changed);
 
@@ -580,5 +580,75 @@ TEST_F(ReshapeMoverTest, SinkTransposeAcrossBroadcastScalar) {
               op::Transpose(op::Multiply()));
 }
 
+TEST_F(ReshapeMoverTest, ReshapeWithUsersOutsideCandidatesNotSink) {
+  const string hlo_string = R"(
+    HloModule ReshapeWithUsersOutsideCandidates
+    ENTRY ReshapeWithMultipleUsers {
+      param0 = f32[20,8]{1,0} parameter(0)
+      reshape0 = f32[8,20]{1,0} reshape(param0)
+      param1 = f32[] parameter(1)
+      broadcast0 = f32[8,20]{1,0} broadcast(param1), dimensions={}
+      param2 = f32[20,8]{1,0} parameter(2)
+      reshape1 = f32[8,20]{1,0} reshape(param2)
+      param3 = f32[20,8]{1,0} parameter(3)
+      reshape2 = f32[8,20]{1,0} reshape(param3)
+      param4 = f32[8,20]{1,0} parameter(4)
+      add0 = f32[8,20]{1,0} add(reshape0, broadcast0)
+      add1 = f32[8,20]{1,0} add(reshape0, reshape1)
+      add2 = f32[8,20]{1,0} add(reshape1, param4)
+      ROOT tuple = (f32[8,20]{1,0},f32[8,20]{1,0},
+        f32[8,20]{1,0}) tuple(add0, add1, add2)
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string);
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, ReshapeMover().Run(&module()));
+  EXPECT_FALSE(changed);
+}
+
+TEST_F(ReshapeMoverTest, ReshapeNoUsersOutsideCandidatesSink1) {
+  const string hlo_string = R"(
+    HloModule ReshapeNoUsersOutsideCandidates1
+    ENTRY ReshapeWithMultipleUsers1 {
+      param0 = f32[20,8]{1,0} parameter(0)
+      reshape0 = f32[8,20]{1,0} reshape(param0)
+      param1 = f32[] parameter(1)
+      broadcast0 = f32[8,20]{1,0} broadcast(param1), dimensions={}
+      param2 = f32[20,8]{1,0} parameter(2)
+      reshape1 = f32[8,20]{1,0} reshape(param2)
+      param3 = f32[20,8]{1,0} parameter(3)
+      reshape2 = f32[8,20]{1,0} reshape(param3)
+      add0 = f32[8,20]{1,0} add(reshape0, broadcast0)
+      add1 = f32[8,20]{1,0} add(reshape0, reshape1)
+      add2 = f32[8,20]{1,0} add(reshape1, reshape2)
+      ROOT tuple = (f32[8,20]{1,0},f32[8,20]{1,0},
+        f32[8,20]{1,0}) tuple(add0, add1, add2)
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string);
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, ReshapeMover().Run(&module()));
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module().entry_computation()->root_instruction(),
+              op::Tuple(op::Reshape(), op::Reshape(), op::Reshape()));
+}
+
+TEST_F(ReshapeMoverTest, ReshapeNoUsersOutsideCandidatesSink2) {
+  const string hlo_string = R"(
+    HloModule ReshapeNoUsersOutsideCandidates2
+    ENTRY ReshapeWithMultipleUsers2 {
+      param0 = f32[20,8]{1,0} parameter(0)
+      reshape0 = f32[8,20]{1,0} reshape(param0)
+      ROOT add0 = f32[8,20]{1,0} add(reshape0, reshape0)
+    }
+  )";
+
+  ParseAndVerifyModule(hlo_string);
+  TF_ASSERT_OK_AND_ASSIGN(bool changed, ReshapeMover().Run(&module()));
+  EXPECT_TRUE(changed);
+  EXPECT_THAT(module().entry_computation()->root_instruction(),
+              op::Reshape(op::Add()));
+}
+
 }  // namespace
 }  // namespace xla
-- 
GitLab


From 763ebd29fec7d3f8b26ead26f8d7c571672d2fb4 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Tue, 3 Apr 2018 18:16:54 -0700
Subject: [PATCH 895/960] Fix error message that was displaying wrong shape

PiperOrigin-RevId: 191530602
---
 tensorflow/core/kernels/sparse_cross_op.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/sparse_cross_op.cc b/tensorflow/core/kernels/sparse_cross_op.cc
index 4b5df7aff0..4ebb7fbcc7 100644
--- a/tensorflow/core/kernels/sparse_cross_op.cc
+++ b/tensorflow/core/kernels/sparse_cross_op.cc
@@ -419,7 +419,7 @@ class SparseCrossOp : public OpKernel {
           context, TensorShapeUtils::IsMatrix(dense_list_in[i].shape()),
           errors::InvalidArgument(
               "Dense inputs should be a matrix but received shape ",
-              indices_list_in[i].shape().DebugString(), " at position ", i));
+              dense_list_in[i].shape().DebugString(), " at position ", i));
       OP_REQUIRES(context, dense_list_in[i].dim_size(0) == batch_size,
                   errors::InvalidArgument("Expected batch size ", batch_size,
                                           " got ", dense_list_in[i].dim_size(0),
-- 
GitLab


From be6b0edf02bb691e7e1b1ff5bb05c18f4830f94c Mon Sep 17 00:00:00 2001
From: Max Galkin <maxgalkin@google.com>
Date: Tue, 3 Apr 2018 18:47:46 -0700
Subject: [PATCH 896/960] Extra logging to help debug shape inference logic in
 optimizers.

PiperOrigin-RevId: 191533338
---
 tensorflow/core/grappler/costs/graph_properties.cc | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tensorflow/core/grappler/costs/graph_properties.cc b/tensorflow/core/grappler/costs/graph_properties.cc
index 5103098f27..8fe154dbf3 100644
--- a/tensorflow/core/grappler/costs/graph_properties.cc
+++ b/tensorflow/core/grappler/costs/graph_properties.cc
@@ -1011,6 +1011,7 @@ Status GraphProperties::InferStatically(bool assume_valid_feeds) {
     }
     // Skip any information that comes from fed nodes.
     if (fed_ports.find(node->name()) != fed_ports.end()) {
+      VLOG(2) << "Skipping feed node shape: " << node->name();
       continue;
     }
     for (const auto& merged_shapes : node_ctx->MergedShapes()) {
-- 
GitLab


From 62d547aa53dd0e7f53e8544b3c41d9274727d333 Mon Sep 17 00:00:00 2001
From: Rui Zhao <rzhao@google.com>
Date: Tue, 3 Apr 2018 18:49:53 -0700
Subject: [PATCH 897/960] Attach an id for each cost node in virtual_cluster.

PiperOrigin-RevId: 191533511
---
 tensorflow/core/grappler/clusters/virtual_cluster.cc | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/tensorflow/core/grappler/clusters/virtual_cluster.cc b/tensorflow/core/grappler/clusters/virtual_cluster.cc
index ae70c98608..abfa7bc48e 100644
--- a/tensorflow/core/grappler/clusters/virtual_cluster.cc
+++ b/tensorflow/core/grappler/clusters/virtual_cluster.cc
@@ -66,6 +66,7 @@ Status VirtualCluster::Run(const GraphDef& graph,
   }
 
   Costs node_costs;
+  int node_id = 0;
   do {
     OpContext op_context = scheduler.GetCurrNode();
     node_costs = node_estimator_->PredictCosts(op_context);
@@ -73,6 +74,7 @@ Status VirtualCluster::Run(const GraphDef& graph,
       CostGraphDef::Node* cost_node =
           metadata->mutable_cost_graph()->add_node();
       const string& op_name = op_context.name;
+      cost_node->set_id(node_id++);
       cost_node->set_name(op_name);
       cost_node->set_device(op_context.device_name);
       cost_node->set_compute_cost(
-- 
GitLab


From ede6e1ff31531cae98844676af4981a821760188 Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Tue, 3 Apr 2018 19:20:44 -0700
Subject: [PATCH 898/960] [TF:XLA] Add half precision support to test_utils.

PiperOrigin-RevId: 191535944
---
 tensorflow/compiler/xla/tests/test_utils.cc | 25 ++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/tensorflow/compiler/xla/tests/test_utils.cc b/tensorflow/compiler/xla/tests/test_utils.cc
index 0bc7df2a65..821432ef7d 100644
--- a/tensorflow/compiler/xla/tests/test_utils.cc
+++ b/tensorflow/compiler/xla/tests/test_utils.cc
@@ -23,14 +23,14 @@ namespace xla {
 
 namespace {
 
-template <typename FloatT>
-void PopulateWithRandomFloatingPointData(Literal* literal,
-                                         std::minstd_rand0* engine) {
+template <typename FloatT, typename GeneratorT>
+void PopulateWithRandomFloatingPointDataImpl(Literal* literal,
+                                             std::minstd_rand0* engine) {
   CHECK_EQ(literal->shape().element_type(),
            primitive_util::NativeToPrimitiveType<FloatT>());
   // Create uniform numbers between 1 and 1.125 to avoid creating denormal
   // numbers.
-  std::uniform_real_distribution<FloatT> generator(1.0f, 1.125f);
+  std::uniform_real_distribution<GeneratorT> generator(1.0f, 1.125f);
   const bool should_index_bias = ShapeUtil::ElementsIn(literal->shape()) > 1000;
   TF_CHECK_OK(literal->Populate<FloatT>(
       [&](tensorflow::gtl::ArraySlice<int64> indices) {
@@ -52,10 +52,22 @@ void PopulateWithRandomFloatingPointData(Literal* literal,
         FloatT index_bias =
             static_cast<FloatT>(index_product % 113 - negative_bias) /
             static_cast<FloatT>(256.0f);
-        return (generator(*engine) - 1.0625) + index_bias;
+        return static_cast<FloatT>(generator(*engine) - 1.0625f) + index_bias;
       }));
 }
 
+template <typename FloatT>
+void PopulateWithRandomFloatingPointData(Literal* literal,
+                                         std::minstd_rand0* engine) {
+  PopulateWithRandomFloatingPointDataImpl<FloatT, FloatT>(literal, engine);
+}
+
+template <>
+void PopulateWithRandomFloatingPointData<half>(Literal* literal,
+                                               std::minstd_rand0* engine) {
+  PopulateWithRandomFloatingPointDataImpl<half, float>(literal, engine);
+}
+
 // The standard library does not have a case for bfloat16, unsurprisingly, so we
 // handle that one specially.
 template <>
@@ -100,6 +112,9 @@ StatusOr<std::unique_ptr<Literal>> MakeFakeLiteralInternal(
     case BF16:
       PopulateWithRandomFloatingPointData<bfloat16>(literal.get(), engine);
       break;
+    case F16:
+      PopulateWithRandomFloatingPointData<half>(literal.get(), engine);
+      break;
     case F32:
       PopulateWithRandomFloatingPointData<float>(literal.get(), engine);
       break;
-- 
GitLab


From 0f60a7d10102048625e45c07a59c71c0c4375529 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Tue, 3 Apr 2018 19:29:53 -0700
Subject: [PATCH 899/960] Enable calling the C API by default.

Enabling the C API makes TensorFlow's Python code to call the TensorFlow C API for low-level graph construction. This should cause no noticeable changes for the most part. One known difference is improved static shape inference, meaning some shape errors will be surfaced during graph construction instead of at runtime.

Note that this can be disabled by setting the environment variable TF_C_API_GRAPH_CONSTRUCTION=0 (for now, eventually this option will be removed).

PiperOrigin-RevId: 191536563
---
 tensorflow/python/framework/ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index be0fe5ee44..0215501b56 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -63,7 +63,7 @@ from tensorflow.python.util.tf_export import tf_export
 # calls to the C API. Currently disabled by default but can be manually enabled
 # in code or via the environment variable. This will be removed once all
 # functionality is supported and there's no performance penalty with it enabled.
-_USE_C_API = os.getenv("TF_C_API_GRAPH_CONSTRUCTION", "0") is not "0"
+_USE_C_API = os.getenv("TF_C_API_GRAPH_CONSTRUCTION", "1") is not "0"
 _USE_C_SHAPES = os.getenv("TF_C_API_GRAPH_CONSTRUCTION_SHAPES", "0") is not "0"
 
 
-- 
GitLab


From 1e38be65609a3fcdbc764a8a6d26a3f13f0efbbc Mon Sep 17 00:00:00 2001
From: Olivia Nordquist <nolivia@google.com>
Date: Tue, 3 Apr 2018 20:28:58 -0700
Subject: [PATCH 900/960] removing an old TODO

PiperOrigin-RevId: 191540512
---
 tensorflow/python/ops/math_ops_test.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/tensorflow/python/ops/math_ops_test.py b/tensorflow/python/ops/math_ops_test.py
index 9f85188b35..05bcee8801 100644
--- a/tensorflow/python/ops/math_ops_test.py
+++ b/tensorflow/python/ops/math_ops_test.py
@@ -155,9 +155,7 @@ class RoundTest(test_util.TensorFlowTestCase):
 
   @test_util.run_in_graph_and_eager_modes()
   def testRounding(self):
-    x = [0.49, 0.7, -0.3, -0.8]
-    # TODO(nolivia): Remove this when RoundOp is forwards compatible
-    # x = np.arange(-5.0, 5.0, .25)
+    x = np.arange(-5.0, 5.0, .25)
     for dtype in [np.float32, np.double, np.int32]:
       x_np = np.array(x, dtype=dtype)
       with test_util.device(use_gpu=True):
-- 
GitLab


From 65631f2d1af62f05b56a8a59b3e223883daa4b80 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Tue, 3 Apr 2018 21:25:20 -0700
Subject: [PATCH 901/960] Help build_pip_package work without ./configure

If the ./configure tool isn't run this script won't exist. It should
be safe to ignore this file if it isn't present since the variable
expansion below will just default to whatever python is on the PATH.

PiperOrigin-RevId: 191543995
---
 tensorflow/tools/pip_package/build_pip_package.sh | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/pip_package/build_pip_package.sh b/tensorflow/tools/pip_package/build_pip_package.sh
index dc31e4c5f7..e2d212a0db 100755
--- a/tensorflow/tools/pip_package/build_pip_package.sh
+++ b/tensorflow/tools/pip_package/build_pip_package.sh
@@ -160,7 +160,9 @@ function main() {
 
   # Before we leave the top-level directory, make sure we know how to
   # call python.
-  source tools/python_bin_path.sh
+  if [[ -e tools/python_bin_path.sh ]]; then
+    source tools/python_bin_path.sh
+  fi
 
   pushd ${TMPDIR}
   rm -f MANIFEST
-- 
GitLab


From 437fe4535c5447e192430763cb6e94b699a6a635 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Tue, 3 Apr 2018 21:33:58 -0700
Subject: [PATCH 902/960] Fix formatting of MKL URLs

The github.com was missing in the mirror.bazel.build path.

PiperOrigin-RevId: 191544461
---
 tensorflow/workspace.bzl | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 0bb297e72e..2510d369fc 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -48,7 +48,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   mkl_repository(
       name = "mkl_linux",
       urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
           "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_lnx_2018.0.1.20171227.tgz",
       ],
       sha256 = "feacc3d82565c1231470359b42c696236fae873704e0b013436afba5fd4fd30f",
@@ -58,7 +58,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   mkl_repository(
       name = "mkl_windows",
       urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip",
           "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_win_2018.0.1.20171227.zip"
       ],
       sha256 = "24bae8d7b22b431a654acadea43f2243c46ae6b1e5a73a4a936825f31d284ee4",
@@ -68,7 +68,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   mkl_repository(
       name = "mkl_darwin",
       urls = [
-          "https://mirror.bazel.build/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
+          "https://mirror.bazel.build/github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz",
           "https://github.com/intel/mkl-dnn/releases/download/v0.12/mklml_mac_2018.0.1.20171227.tgz"
       ],
       sha256 = "0e954ec6fd3dc5e37f64c4043f6b5613dd687558da3df1028b3b7c29ff5cf77f",
-- 
GitLab


From 1f5324ca69bc1017972eef8e418691cff9a86dd7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 00:33:14 -0700
Subject: [PATCH 903/960] Fix PR-AUC calculation, namely the incorrect use of
 linear interpolation for Precision - see Section 4 of Davis & Goadrich 2006
 (https://www.biostat.wisc.edu/~page/rocpr.pdf) Also, modify the name of the
 "trapezoidal" summation method to reflect the fact that the proper
 interpolation method in this case isn't quite the trapezoidal one.

PiperOrigin-RevId: 191555707
---
 .../python/kernel_tests/metrics_test.py       | 63 +++++++++++++--
 tensorflow/python/ops/metrics_impl.py         | 77 +++++++++++++++++--
 2 files changed, 127 insertions(+), 13 deletions(-)

diff --git a/tensorflow/python/kernel_tests/metrics_test.py b/tensorflow/python/kernel_tests/metrics_test.py
index ad802f7e1f..55653489af 100644
--- a/tensorflow/python/kernel_tests/metrics_test.py
+++ b/tensorflow/python/kernel_tests/metrics_test.py
@@ -1124,40 +1124,91 @@ class AUCTest(test.TestCase):
 
       self.assertAlmostEqual(0.7, auc.eval(), 5)
 
-  def testAUCPRSpecialCase(self):
+  # Regarding the AUC-PR tests: note that the preferred method when
+  # calculating AUC-PR is summation_method='careful_interpolation'.
+  def testCorrectAUCPRSpecialCase(self):
     with self.test_session() as sess:
       predictions = constant_op.constant(
           [0.1, 0.4, 0.35, 0.8], shape=(1, 4), dtype=dtypes_lib.float32)
       labels = constant_op.constant([0, 0, 1, 1], shape=(1, 4))
-      auc, update_op = metrics.auc(labels, predictions, curve='PR')
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='careful_interpolation')
+
+      sess.run(variables.local_variables_initializer())
+      # expected ~= 0.79726744594
+      expected = 1 - math.log(1.5) / 2
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
+
+  def testCorrectAnotherAUCPRSpecialCase(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [0.1, 0.4, 0.35, 0.8, 0.1, 0.135, 0.81],
+          shape=(1, 7),
+          dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 0, 1, 0, 1, 0, 1], shape=(1, 7))
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='careful_interpolation')
+
+      sess.run(variables.local_variables_initializer())
+      # expected ~= 0.61350593198
+      expected = (2.5 - 2 * math.log(4./3) - 0.25 * math.log(7./5)) / 3
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
+
+  def testThirdCorrectAUCPRSpecialCase(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [0.0, 0.1, 0.2, 0.33, 0.3, 0.4, 0.5],
+          shape=(1, 7),
+          dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 0, 0, 0, 1, 1, 1], shape=(1, 7))
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='careful_interpolation')
+
+      sess.run(variables.local_variables_initializer())
+      # expected ~= 0.90410597584
+      expected = 1 - math.log(4./3) / 3
+      self.assertAlmostEqual(expected, sess.run(update_op), delta=1e-3)
+      self.assertAlmostEqual(expected, auc.eval(), delta=1e-3)
+
+  def testIncorrectAUCPRSpecialCase(self):
+    with self.test_session() as sess:
+      predictions = constant_op.constant(
+          [0.1, 0.4, 0.35, 0.8], shape=(1, 4), dtype=dtypes_lib.float32)
+      labels = constant_op.constant([0, 0, 1, 1], shape=(1, 4))
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='trapezoidal')
 
       sess.run(variables.local_variables_initializer())
       self.assertAlmostEqual(0.79166, sess.run(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.79166, auc.eval(), delta=1e-3)
 
-  def testAnotherAUCPRSpecialCase(self):
+  def testAnotherIncorrectAUCPRSpecialCase(self):
     with self.test_session() as sess:
       predictions = constant_op.constant(
           [0.1, 0.4, 0.35, 0.8, 0.1, 0.135, 0.81],
           shape=(1, 7),
           dtype=dtypes_lib.float32)
       labels = constant_op.constant([0, 0, 1, 0, 1, 0, 1], shape=(1, 7))
-      auc, update_op = metrics.auc(labels, predictions, curve='PR')
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='trapezoidal')
 
       sess.run(variables.local_variables_initializer())
       self.assertAlmostEqual(0.610317, sess.run(update_op), delta=1e-3)
 
       self.assertAlmostEqual(0.610317, auc.eval(), delta=1e-3)
 
-  def testThirdAUCPRSpecialCase(self):
+  def testThirdIncorrectAUCPRSpecialCase(self):
     with self.test_session() as sess:
       predictions = constant_op.constant(
           [0.0, 0.1, 0.2, 0.33, 0.3, 0.4, 0.5],
           shape=(1, 7),
           dtype=dtypes_lib.float32)
       labels = constant_op.constant([0, 0, 0, 0, 1, 1, 1], shape=(1, 7))
-      auc, update_op = metrics.auc(labels, predictions, curve='PR')
+      auc, update_op = metrics.auc(labels, predictions, curve='PR',
+                                   summation_method='trapezoidal')
 
       sess.run(variables.local_variables_initializer())
       self.assertAlmostEqual(0.90277, sess.run(update_op), delta=1e-3)
diff --git a/tensorflow/python/ops/metrics_impl.py b/tensorflow/python/ops/metrics_impl.py
index 9ec4954579..47eea6ef6b 100644
--- a/tensorflow/python/ops/metrics_impl.py
+++ b/tensorflow/python/ops/metrics_impl.py
@@ -33,6 +33,7 @@ from tensorflow.python.ops import sparse_ops
 from tensorflow.python.ops import state_ops
 from tensorflow.python.ops import variable_scope
 from tensorflow.python.ops import weights_broadcast_ops
+from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.deprecation import deprecated
 from tensorflow.python.util.tf_export import tf_export
 
@@ -626,10 +627,16 @@ def auc(labels,
     curve: Specifies the name of the curve to be computed, 'ROC' [default] or
       'PR' for the Precision-Recall-curve.
     name: An optional variable_scope name.
-    summation_method: Specifies the Riemann summation method used, 'trapezoidal'
-      [default] that applies the trapezoidal rule, 'minoring' that applies
-      left summation for increasing intervals and right summation for decreasing
-      intervals or 'majoring' that applies the opposite.
+    summation_method: Specifies the Riemann summation method used
+      (https://en.wikipedia.org/wiki/Riemann_sum): 'trapezoidal' [default] that
+      applies the trapezoidal rule; 'careful_interpolation', a variant of it
+      differing only by a more correct interpolation scheme for PR-AUC -
+      interpolating (true/false) positives but not the ratio that is precision;
+      'minoring' that applies left summation for increasing intervals and right
+      summation for decreasing intervals; 'majoring' that does the opposite.
+      Note that 'careful_interpolation' is strictly preferred to 'trapezoidal'
+      (to be deprecated soon) as it applies the same method for ROC, and a
+      better one (see Davis & Goadrich 2006 for details) for the PR curve.
 
   Returns:
     auc: A scalar `Tensor` representing the current area-under-curve.
@@ -664,8 +671,62 @@ def auc(labels,
     # Add epsilons to avoid dividing by 0.
     epsilon = 1.0e-6
 
+    def interpolate_pr_auc(tp, fp, fn):
+      """Interpolation formula inspired by section 4 of Davis & Goadrich 2006.
+
+      Note here we derive & use a closed formula not present in the paper
+      - as follows:
+      Modeling all of TP (true positive weight),
+      FP (false positive weight) and their sum P = TP + FP (positive weight)
+      as varying linearly within each interval [A, B] between successive
+      thresholds, we get
+        Precision = (TP_A + slope * (P - P_A)) / P
+      with slope = dTP / dP = (TP_B - TP_A) / (P_B - P_A).
+      The area within the interval is thus (slope / total_pos_weight) times
+        int_A^B{Precision.dP} = int_A^B{(TP_A + slope * (P - P_A)) * dP / P}
+        int_A^B{Precision.dP} = int_A^B{slope * dP + intercept * dP / P}
+      where intercept = TP_A - slope * P_A = TP_B - slope * P_B, resulting in
+        int_A^B{Precision.dP} = TP_B - TP_A + intercept * log(P_B / P_A)
+      Bringing back the factor (slope / total_pos_weight) we'd put aside, we get
+         slope * [dTP + intercept *  log(P_B / P_A)] / total_pos_weight
+      where dTP == TP_B - TP_A.
+      Note that when P_A == 0 the above calculation simplifies into
+        int_A^B{Precision.dTP} = int_A^B{slope * dTP} = slope * (TP_B - TP_A)
+      which is really equivalent to imputing constant precision throughout the
+      first bucket having >0 true positives.
+
+      Args:
+        tp: true positive counts
+        fp: false positive counts
+        fn: false negative counts
+      Returns:
+        pr_auc: an approximation of the area under the P-R curve.
+      """
+      dtp = tp[:num_thresholds - 1] - tp[1:]
+      p = tp + fp
+      prec_slope = _safe_div(dtp, p[:num_thresholds - 1] - p[1:], 'prec_slope')
+      intercept = tp[1:] - math_ops.multiply(prec_slope, p[1:])
+      safe_p_ratio = array_ops.where(
+          math_ops.logical_and(p[:num_thresholds - 1] > 0, p[1:] > 0),
+          _safe_div(p[:num_thresholds - 1], p[1:], 'recall_relative_ratio'),
+          array_ops.ones_like(p[1:]))
+      return math_ops.reduce_sum(
+          _safe_div(
+              prec_slope * (dtp + intercept * math_ops.log(safe_p_ratio)),
+              tp[1:] + fn[1:],
+              name='pr_auc_increment'),
+          name='interpolate_pr_auc')
+
     def compute_auc(tp, fn, tn, fp, name):
       """Computes the roc-auc or pr-auc based on confusion counts."""
+      if curve == 'PR':
+        if summation_method == 'trapezoidal':
+          logging.warning(
+              'Trapezoidal rule is known to produce incorrect PR-AUCs; '
+              'please switch to "careful_interpolation" instead.')
+        elif summation_method == 'careful_interpolation':
+          # This one is a bit tricky and is handled separately.
+          return interpolate_pr_auc(tp, fp, fn)
       rec = math_ops.div(tp + epsilon, tp + fn + epsilon)
       if curve == 'ROC':
         fp_rate = math_ops.div(fp, fp + tn + epsilon)
@@ -675,7 +736,9 @@ def auc(labels,
         prec = math_ops.div(tp + epsilon, tp + fp + epsilon)
         x = rec
         y = prec
-      if summation_method == 'trapezoidal':
+      if summation_method in ('trapezoidal', 'careful_interpolation'):
+        # Note that the case ('PR', 'careful_interpolation') has been handled
+        # above.
         return math_ops.reduce_sum(
             math_ops.multiply(x[:num_thresholds - 1] - x[1:],
                               (y[:num_thresholds - 1] + y[1:]) / 2.),
@@ -923,8 +986,8 @@ def mean_per_class_accuracy(labels,
         weights = array_ops.reshape(weights, [-1])
       weights = math_ops.to_float(weights)
 
-      is_correct = is_correct * weights
-      ones = ones * weights
+      is_correct *= weights
+      ones *= weights
 
     update_total_op = state_ops.scatter_add(total, labels, ones)
     update_count_op = state_ops.scatter_add(count, labels, is_correct)
-- 
GitLab


From 71b19430e8484b136e0b872f6a543aff8a242587 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 05:31:48 -0700
Subject: [PATCH 904/960] Sync replicas distributed training example with two
 strategies: 1) Interleave covariance and inverse update ops with training op.
 2) Run the inverse and covariance ops on separate dedicated workers.

PiperOrigin-RevId: 191579634
---
 tensorflow/contrib/kfac/examples/BUILD        |  24 +-
 tensorflow/contrib/kfac/examples/convnet.py   | 315 +++++++++++++++---
 .../convnet_mnist_distributed_main.py         |  62 ++++
 .../convnet_mnist_multi_tower_main.py         |  48 +++
 ...t_main.py => convnet_mnist_single_main.py} |  32 +-
 .../kfac/examples/tests/convnet_test.py       |  17 +-
 6 files changed, 411 insertions(+), 87 deletions(-)
 create mode 100644 tensorflow/contrib/kfac/examples/convnet_mnist_distributed_main.py
 create mode 100644 tensorflow/contrib/kfac/examples/convnet_mnist_multi_tower_main.py
 rename tensorflow/contrib/kfac/examples/{convnet_mnist_main.py => convnet_mnist_single_main.py} (57%)

diff --git a/tensorflow/contrib/kfac/examples/BUILD b/tensorflow/contrib/kfac/examples/BUILD
index 7dd40c19c5..8186fa1c62 100644
--- a/tensorflow/contrib/kfac/examples/BUILD
+++ b/tensorflow/contrib/kfac/examples/BUILD
@@ -28,8 +28,28 @@ py_library(
 )
 
 py_binary(
-    name = "convnet_mnist_main",
-    srcs = ["convnet_mnist_main.py"],
+    name = "convnet_mnist_single_main",
+    srcs = ["convnet_mnist_single_main.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":convnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "convnet_mnist_multi_tower_main",
+    srcs = ["convnet_mnist_multi_tower_main.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":convnet",
+        "//tensorflow:tensorflow_py",
+    ],
+)
+
+py_binary(
+    name = "convnet_mnist_distributed_main",
+    srcs = ["convnet_mnist_distributed_main.py"],
     srcs_version = "PY2AND3",
     deps = [
         ":convnet",
diff --git a/tensorflow/contrib/kfac/examples/convnet.py b/tensorflow/contrib/kfac/examples/convnet.py
index 39d80addaa..e8e3353091 100644
--- a/tensorflow/contrib/kfac/examples/convnet.py
+++ b/tensorflow/contrib/kfac/examples/convnet.py
@@ -37,6 +37,8 @@ import tensorflow as tf
 
 from tensorflow.contrib.kfac.examples import mlp
 from tensorflow.contrib.kfac.examples import mnist
+from tensorflow.contrib.kfac.python.ops import optimizer as opt
+
 
 lc = tf.contrib.kfac.layer_collection
 oq = tf.contrib.kfac.op_queue
@@ -48,12 +50,18 @@ __all__ = [
     "linear_layer",
     "build_model",
     "minimize_loss_single_machine",
-    "minimize_loss_distributed",
+    "distributed_grads_only_and_ops_chief_worker",
+    "distributed_grads_and_ops_dedicated_workers",
     "train_mnist_single_machine",
-    "train_mnist_distributed",
+    "train_mnist_distributed_sync_replicas",
+    "train_mnist_multitower"
 ]
 
 
+# Inverse update ops will be run every _INVERT_EVRY iterations.
+_INVERT_EVERY = 10
+
+
 def conv_layer(layer_id, inputs, kernel_size, out_channels):
   """Builds a convolutional layer with ReLU non-linearity.
 
@@ -161,8 +169,9 @@ def build_model(examples, labels, num_labels, layer_collection):
   accuracy = tf.reduce_mean(
       tf.cast(tf.equal(labels, tf.argmax(logits, axis=1)), dtype=tf.float32))
 
-  tf.summary.scalar("loss", loss)
-  tf.summary.scalar("accuracy", accuracy)
+  with tf.device("/cpu:0"):
+    tf.summary.scalar("loss", loss)
+    tf.summary.scalar("accuracy", accuracy)
 
   # Register parameters. K-FAC needs to know about the inputs, outputs, and
   # parameters of each conv/fully connected layer and the logits powering the
@@ -181,41 +190,59 @@ def build_model(examples, labels, num_labels, layer_collection):
 def minimize_loss_single_machine(loss,
                                  accuracy,
                                  layer_collection,
+                                 device="/gpu:0",
                                  session_config=None):
   """Minimize loss with K-FAC on a single machine.
 
-  A single Session is responsible for running all of K-FAC's ops.
+  A single Session is responsible for running all of K-FAC's ops. The covariance
+  and inverse update ops are placed on `device`. All model variables are on CPU.
 
   Args:
     loss: 0-D Tensor. Loss to be minimized.
     accuracy: 0-D Tensor. Accuracy of classifier on current minibatch.
     layer_collection: LayerCollection instance describing model architecture.
       Used by K-FAC to construct preconditioner.
+    device: string, Either '/cpu:0' or '/gpu:0'. The covaraince and invserse
+      update ops are run on this device.
     session_config: None or tf.ConfigProto. Configuration for tf.Session().
 
   Returns:
     final value for 'accuracy'.
   """
   # Train with K-FAC.
-  global_step = tf.train.get_or_create_global_step()
+  g_step = tf.train.get_or_create_global_step()
   optimizer = opt.KfacOptimizer(
       learning_rate=0.0001,
       cov_ema_decay=0.95,
       damping=0.001,
       layer_collection=layer_collection,
+      placement_strategy="round_robin",
+      cov_devices=[device],
+      inv_devices=[device],
       momentum=0.9)
-  train_op = optimizer.minimize(loss, global_step=global_step)
+  (cov_update_thunks,
+   inv_update_thunks) = optimizer.make_vars_and_create_op_thunks()
+
+  with tf.device(device):
+    train_op = optimizer.minimize(loss, global_step=g_step)
+
+  def make_update_op(update_thunks):
+    update_op = [thunk() for thunk in update_thunks]
+    return tf.group(*update_op)
+
+  cov_update_op = make_update_op(cov_update_thunks)
+  with tf.control_dependencies([train_op, cov_update_op]):
+    inverse_op = tf.cond(
+        tf.equal(tf.mod(g_step + 1, _INVERT_EVERY), 0),
+        lambda: make_update_op(inv_update_thunks), tf.no_op)
 
   tf.logging.info("Starting training.")
   with tf.train.MonitoredTrainingSession(config=session_config) as sess:
     while not sess.should_stop():
-      global_step_, loss_, accuracy_, _, _ = sess.run(
-          [global_step, loss, accuracy, train_op, optimizer.cov_update_op])
-
-      if global_step_ % 100 == 0:
-        sess.run(optimizer.inv_update_op)
+      global_step_, loss_, accuracy_, _ = sess.run(
+          [g_step, loss, accuracy, inverse_op])
 
-      if global_step_ % 100 == 0:
+      if (global_step_ + 1) % _INVERT_EVERY == 0:
         tf.logging.info("global_step: %d | loss: %f | accuracy: %s",
                         global_step_, loss_, accuracy_)
 
@@ -250,16 +277,62 @@ def _num_gradient_tasks(num_tasks):
   return int(np.ceil(0.6 * num_tasks))
 
 
-def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master,
-                              checkpoint_dir, loss, accuracy, layer_collection):
-  """Minimize loss with an synchronous implementation of K-FAC.
+def _make_distributed_train_op(
+    task_id,
+    num_worker_tasks,
+    num_ps_tasks,
+    layer_collection
+):
+  """Creates optimizer and distributed training op.
 
-  Different tasks are responsible for different parts of K-FAC's Ops. The first
-  60% of tasks update weights; the next 20% accumulate covariance statistics;
-  the last 20% invert the matrices used to precondition gradients.
+  Constructs KFAC optimizer and wraps it in `sync_replicas` optimizer. Makes
+  the train op.
+
+  Args:
+   task_id: int. Integer in [0, num_worker_tasks). ID for this worker.
+    num_worker_tasks: int. Number of workers in this distributed training setup.
+    num_ps_tasks: int. Number of parameter servers holding variables. If 0,
+      parameter servers are not used.
+    layer_collection: LayerCollection instance describing model architecture.
+      Used by K-FAC to construct preconditioner.
+
+  Returns:
+    sync_optimizer: `tf.train.SyncReplicasOptimizer` instance which wraps KFAC
+      optimizer.
+    optimizer: Instance of `opt.KfacOptimizer`.
+    global_step: `tensor`, Global step.
+  """
+  tf.logging.info("Task id : %d", task_id)
+  with tf.device(tf.train.replica_device_setter(num_ps_tasks)):
+    global_step = tf.train.get_or_create_global_step()
+    optimizer = opt.KfacOptimizer(
+        learning_rate=0.0001,
+        cov_ema_decay=0.95,
+        damping=0.001,
+        layer_collection=layer_collection,
+        momentum=0.9)
+    sync_optimizer = tf.train.SyncReplicasOptimizer(
+        opt=optimizer,
+        replicas_to_aggregate=_num_gradient_tasks(num_worker_tasks),
+        total_num_replicas=num_worker_tasks)
+    return sync_optimizer, optimizer, global_step
+
+
+def distributed_grads_only_and_ops_chief_worker(
+    task_id, is_chief, num_worker_tasks, num_ps_tasks, master, checkpoint_dir,
+    loss, accuracy, layer_collection, invert_every=10):
+  """Minimize loss with a synchronous implementation of K-FAC.
+
+  All workers perform gradient computation. Chief worker applies gradient after
+  averaging the gradients obtained from all the workers. All workers block
+  execution untill the update is applied. Chief worker runs covariance and
+  inverse update ops. Covariance and inverse matrices are placed on parameter
+  servers in a round robin manner. For further details on synchronous
+  distributed optimization check `tf.train.SyncReplicasOptimizer`.
 
   Args:
     task_id: int. Integer in [0, num_worker_tasks). ID for this worker.
+    is_chief: `boolean`, `True` if the worker is chief worker.
     num_worker_tasks: int. Number of workers in this distributed training setup.
     num_ps_tasks: int. Number of parameter servers holding variables. If 0,
       parameter servers are not used.
@@ -271,6 +344,7 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master,
       run with each step.
     layer_collection: LayerCollection instance describing model architecture.
       Used by K-FAC to construct preconditioner.
+    invert_every: `int`, Number of steps between update the inverse.
 
   Returns:
     final value for 'accuracy'.
@@ -278,19 +352,80 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master,
   Raises:
     ValueError: if task_id >= num_worker_tasks.
   """
-  with tf.device(tf.train.replica_device_setter(num_ps_tasks)):
-    global_step = tf.train.get_or_create_global_step()
-    optimizer = opt.KfacOptimizer(
-        learning_rate=0.0001,
-        cov_ema_decay=0.95,
-        damping=0.001,
-        layer_collection=layer_collection,
-        momentum=0.9)
-    inv_update_queue = oq.OpQueue(optimizer.inv_update_ops)
-    sync_optimizer = tf.train.SyncReplicasOptimizer(
-        opt=optimizer,
-        replicas_to_aggregate=_num_gradient_tasks(num_worker_tasks))
-    train_op = sync_optimizer.minimize(loss, global_step=global_step)
+
+  sync_optimizer, optimizer, global_step = _make_distributed_train_op(
+      task_id, num_worker_tasks, num_ps_tasks, layer_collection)
+  (cov_update_thunks,
+   inv_update_thunks) = optimizer.make_vars_and_create_op_thunks()
+  train_op = sync_optimizer.minimize(loss, global_step=global_step)
+
+  tf.logging.info("Starting training.")
+  hooks = [sync_optimizer.make_session_run_hook(is_chief)]
+
+  def make_update_op(update_thunks):
+    update_op = [thunk() for thunk in update_thunks]
+    return tf.group(*update_op)
+
+  if is_chief:
+    cov_update_op = make_update_op(cov_update_thunks)
+    with tf.control_dependencies([train_op, cov_update_op]):
+      update_op = tf.cond(
+          tf.equal(tf.mod(global_step + 1, invert_every), 0),
+          lambda: make_update_op(inv_update_thunks),
+          tf.no_op)
+  else:
+    update_op = train_op
+
+  with tf.train.MonitoredTrainingSession(
+      master=master,
+      is_chief=is_chief,
+      checkpoint_dir=checkpoint_dir,
+      hooks=hooks,
+      stop_grace_period_secs=0) as sess:
+    while not sess.should_stop():
+      global_step_, loss_, accuracy_, _ = sess.run(
+          [global_step, loss, accuracy, update_op])
+      tf.logging.info("global_step: %d | loss: %f | accuracy: %s", global_step_,
+                      loss_, accuracy_)
+  return accuracy_
+
+
+def distributed_grads_and_ops_dedicated_workers(
+    task_id, is_chief, num_worker_tasks, num_ps_tasks, master, checkpoint_dir,
+    loss, accuracy, layer_collection):
+  """Minimize loss with a synchronous implementation of K-FAC.
+
+  Different workers are responsible for different parts of K-FAC's Ops. The
+  first 60% of tasks compute gradients; the next 20% accumulate covariance
+  statistics; the last 20% invert the matrices used to precondition gradients.
+  The chief worker applies the gradient .
+
+  Args:
+    task_id: int. Integer in [0, num_worker_tasks). ID for this worker.
+    is_chief: `boolean`, `True` if the worker is chief worker.
+    num_worker_tasks: int. Number of workers in this distributed training setup.
+    num_ps_tasks: int. Number of parameter servers holding variables. If 0,
+      parameter servers are not used.
+    master: string. IP and port of TensorFlow runtime process. Set to empty
+      string to run locally.
+    checkpoint_dir: string or None. Path to store checkpoints under.
+    loss: 0-D Tensor. Loss to be minimized.
+    accuracy: dict mapping strings to 0-D Tensors. Additional accuracy to
+      run with each step.
+    layer_collection: LayerCollection instance describing model architecture.
+      Used by K-FAC to construct preconditioner.
+
+  Returns:
+    final value for 'accuracy'.
+
+  Raises:
+    ValueError: if task_id >= num_worker_tasks.
+  """
+  sync_optimizer, optimizer, global_step = _make_distributed_train_op(
+      task_id, num_worker_tasks, num_ps_tasks, layer_collection)
+  _, cov_update_op, inv_update_ops, _, _, _ = optimizer.make_ops_and_vars()
+  train_op = sync_optimizer.minimize(loss, global_step=global_step)
+  inv_update_queue = oq.OpQueue(inv_update_ops)
 
   tf.logging.info("Starting training.")
   is_chief = (task_id == 0)
@@ -306,7 +441,7 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master,
       if _is_gradient_task(task_id, num_worker_tasks):
         learning_op = train_op
       elif _is_cov_update_task(task_id, num_worker_tasks):
-        learning_op = optimizer.cov_update_op
+        learning_op = cov_update_op
       elif _is_inv_update_task(task_id, num_worker_tasks):
         # TODO(duckworthd): Running this op before cov_update_op has been run a
         # few times can result in "InvalidArgumentError: Cholesky decomposition
@@ -324,13 +459,18 @@ def minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks, master,
   return accuracy_
 
 
-def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False):
+def train_mnist_single_machine(data_dir,
+                               num_epochs,
+                               use_fake_data=False,
+                               device="/gpu:0"):
   """Train a ConvNet on MNIST.
 
   Args:
     data_dir: string. Directory to read MNIST examples from.
     num_epochs: int. Number of passes to make over the training set.
     use_fake_data: bool. If True, generate a synthetic dataset.
+    device: string, Either '/cpu:0' or '/gpu:0'. The covaraince and inverse
+      update ops are run on this device.
 
   Returns:
     accuracy of model on the final minibatch of training data.
@@ -350,22 +490,38 @@ def train_mnist_single_machine(data_dir, num_epochs, use_fake_data=False):
       examples, labels, num_labels=10, layer_collection=layer_collection)
 
   # Fit model.
-  return minimize_loss_single_machine(loss, accuracy, layer_collection)
+  return minimize_loss_single_machine(
+      loss, accuracy, layer_collection, device=device)
 
 
 def train_mnist_multitower(data_dir, num_epochs, num_towers,
-                           use_fake_data=True):
+                           use_fake_data=True, devices=None):
   """Train a ConvNet on MNIST.
 
+  Training data is split equally among the towers. Each tower computes loss on
+  its own batch of data and the loss is aggregated on the CPU. The model
+  variables are placed on first tower. The covariance and inverse update ops
+  and variables are placed on GPUs in a round robin manner.
+
   Args:
     data_dir: string. Directory to read MNIST examples from.
     num_epochs: int. Number of passes to make over the training set.
     num_towers: int. Number of CPUs to split inference across.
     use_fake_data: bool. If True, generate a synthetic dataset.
+    devices: string, Either list of CPU or GPU. The covaraince and inverse
+      update ops are run on this device.
 
   Returns:
     accuracy of model on the final minibatch of training data.
   """
+  if devices:
+    device_count = {"GPU": num_towers}
+  else:
+    device_count = {"CPU": num_towers}
+
+  devices = devices or [
+      "/cpu:{}".format(tower_id) for tower_id in range(num_towers)
+  ]
   # Load a dataset.
   tf.logging.info("Loading MNIST into memory.")
   tower_batch_size = 128
@@ -388,7 +544,7 @@ def train_mnist_multitower(data_dir, num_epochs, num_towers,
   layer_collection = lc.LayerCollection()
   tower_results = []
   for tower_id in range(num_towers):
-    with tf.device("/cpu:%d" % tower_id):
+    with tf.device(devices[tower_id]):
       with tf.name_scope("tower%d" % tower_id):
         with tf.variable_scope(tf.get_variable_scope(), reuse=(tower_id > 0)):
           tf.logging.info("Building tower %d." % tower_id)
@@ -402,34 +558,79 @@ def train_mnist_multitower(data_dir, num_epochs, num_towers,
   accuracy = tf.reduce_mean(accuracies)
 
   # Fit model.
+
   session_config = tf.ConfigProto(
-      allow_soft_placement=False, device_count={
-          "CPU": num_towers
-      })
-  return minimize_loss_single_machine(
-      loss, accuracy, layer_collection, session_config=session_config)
+      allow_soft_placement=False,
+      device_count=device_count,
+  )
+
+  g_step = tf.train.get_or_create_global_step()
+  optimizer = opt.KfacOptimizer(
+      learning_rate=0.0001,
+      cov_ema_decay=0.95,
+      damping=0.001,
+      layer_collection=layer_collection,
+      placement_strategy="round_robin",
+      cov_devices=devices,
+      inv_devices=devices,
+      momentum=0.9)
+  (cov_update_thunks,
+   inv_update_thunks) = optimizer.make_vars_and_create_op_thunks()
 
+  train_op = optimizer.minimize(loss, global_step=g_step)
 
-def train_mnist_distributed(task_id,
-                            num_worker_tasks,
-                            num_ps_tasks,
-                            master,
-                            data_dir,
-                            num_epochs,
-                            use_fake_data=False):
-  """Train a ConvNet on MNIST.
+  def make_update_op(update_thunks):
+    update_op = [thunk() for thunk in update_thunks]
+    return tf.group(*update_op)
+
+  cov_update_op = make_update_op(cov_update_thunks)
+  with tf.control_dependencies([train_op, cov_update_op]):
+    inverse_op = tf.cond(
+        tf.equal(tf.mod(g_step + 1, _INVERT_EVERY), 0),
+        lambda: make_update_op(inv_update_thunks), tf.no_op)
+
+  tf.logging.info("Starting training.")
+  with tf.train.MonitoredTrainingSession(config=session_config) as sess:
+    while not sess.should_stop():
+      global_step_, loss_, accuracy_, _ = sess.run(
+          [g_step, loss, accuracy, inverse_op])
+
+      if (global_step_ + 1) % _INVERT_EVERY == 0:
+        tf.logging.info("global_step: %d | loss: %f | accuracy: %s",
+                        global_step_, loss_, accuracy_)
+
+
+def train_mnist_distributed_sync_replicas(task_id,
+                                          is_chief,
+                                          num_worker_tasks,
+                                          num_ps_tasks,
+                                          master,
+                                          data_dir,
+                                          num_epochs,
+                                          op_strategy,
+                                          use_fake_data=False):
+  """Train a ConvNet on MNIST using Sync replicas optimizer.
 
   Args:
     task_id: int. Integer in [0, num_worker_tasks). ID for this worker.
+    is_chief: `boolean`, `True` if the worker is chief worker.
     num_worker_tasks: int. Number of workers in this distributed training setup.
     num_ps_tasks: int. Number of parameter servers holding variables.
     master: string. IP and port of TensorFlow runtime process.
     data_dir: string. Directory to read MNIST examples from.
     num_epochs: int. Number of passes to make over the training set.
+    op_strategy: `string`, Strategy to run the covariance and inverse
+      ops. If op_strategy == `chief_worker` then covaraiance and inverse
+      update ops are run on chief worker otherwise they are run on dedicated
+      workers.
+
     use_fake_data: bool. If True, generate a synthetic dataset.
 
   Returns:
     accuracy of model on the final minibatch of training data.
+
+  Raises:
+    ValueError: If `op_strategy` not in ["chief_worker", "dedicated_workers"].
   """
   # Load a dataset.
   tf.logging.info("Loading MNIST into memory.")
@@ -448,9 +649,17 @@ def train_mnist_distributed(task_id,
 
   # Fit model.
   checkpoint_dir = None if data_dir is None else os.path.join(data_dir, "kfac")
-  return minimize_loss_distributed(task_id, num_worker_tasks, num_ps_tasks,
-                                   master, checkpoint_dir, loss, accuracy,
-                                   layer_collection)
+  if op_strategy == "chief_worker":
+    return distributed_grads_only_and_ops_chief_worker(
+        task_id, is_chief, num_worker_tasks, num_ps_tasks, master,
+        checkpoint_dir, loss, accuracy, layer_collection)
+  elif op_strategy == "dedicated_workers":
+    return distributed_grads_and_ops_dedicated_workers(
+        task_id, is_chief, num_worker_tasks, num_ps_tasks, master,
+        checkpoint_dir, loss, accuracy, layer_collection)
+  else:
+    raise ValueError("Only supported op strategies are : {}, {}".format(
+        "chief_worker", "dedicated_workers"))
 
 
 if __name__ == "__main__":
diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_distributed_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_distributed_main.py
new file mode 100644
index 0000000000..b4c2d4a9e9
--- /dev/null
+++ b/tensorflow/contrib/kfac/examples/convnet_mnist_distributed_main.py
@@ -0,0 +1,62 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Train a ConvNet on MNIST using K-FAC.
+
+Distributed training with sync replicas optimizer. See
+`convnet.train_mnist_distributed_sync_replicas` for details.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from absl import flags
+import tensorflow as tf
+
+from tensorflow.contrib.kfac.examples import convnet
+
+FLAGS = flags.FLAGS
+flags.DEFINE_integer("task", -1, "Task identifier")
+flags.DEFINE_string("data_dir", "/tmp/mnist", "local mnist dir")
+flags.DEFINE_string(
+    "cov_inv_op_strategy", "chief_worker",
+    "In dist training mode run the cov, inv ops on chief or dedicated workers."
+)
+flags.DEFINE_string("master", "local", "Session master.")
+flags.DEFINE_integer("ps_tasks", 2,
+                     "Number of tasks in the parameter server job.")
+flags.DEFINE_integer("replicas_to_aggregate", 5,
+                     "Number of replicas to aggregate.")
+flags.DEFINE_integer("worker_replicas", 5, "Number of replicas in worker job.")
+flags.DEFINE_integer("num_epochs", None, "Number of epochs.")
+
+
+def _is_chief():
+  """Determines whether a job is the chief worker."""
+  if "chief_worker" in FLAGS.brain_jobs:
+    return FLAGS.brain_job_name == "chief_worker"
+  else:
+    return FLAGS.task == 0
+
+
+def main(unused_argv):
+  _ = unused_argv
+  convnet.train_mnist_distributed_sync_replicas(
+      FLAGS.task, _is_chief(), FLAGS.worker_replicas, FLAGS.ps_tasks,
+      FLAGS.master, FLAGS.data_dir, FLAGS.num_epochs, FLAGS.cov_inv_op_strategy)
+
+if __name__ == "__main__":
+  tf.app.run(main=main)
diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_multi_tower_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_multi_tower_main.py
new file mode 100644
index 0000000000..4249bf8a8d
--- /dev/null
+++ b/tensorflow/contrib/kfac/examples/convnet_mnist_multi_tower_main.py
@@ -0,0 +1,48 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+r"""Train a ConvNet on MNIST using K-FAC.
+
+Multi tower training mode. See `convnet.train_mnist_multitower` for details.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+
+from absl import flags
+import tensorflow as tf
+
+from tensorflow.contrib.kfac.examples import convnet
+
+FLAGS = flags.FLAGS
+flags.DEFINE_string("data_dir", "/tmp/multitower_1/mnist", "local mnist dir")
+flags.DEFINE_integer("num_towers", 2,
+                     "Number of towers for multi tower training.")
+
+
+def main(unused_argv):
+  _ = unused_argv
+  assert FLAGS.num_towers > 1
+  devices = ["/gpu:{}".format(tower_id) for tower_id in range(FLAGS.num_towers)]
+  convnet.train_mnist_multitower(
+      FLAGS.data_dir,
+      num_epochs=200,
+      num_towers=FLAGS.num_towers,
+      devices=devices)
+
+
+if __name__ == "__main__":
+  tf.app.run(main=main)
diff --git a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py b/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py
similarity index 57%
rename from tensorflow/contrib/kfac/examples/convnet_mnist_main.py
rename to tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py
index b0c6fbde19..3aa52aff19 100644
--- a/tensorflow/contrib/kfac/examples/convnet_mnist_main.py
+++ b/tensorflow/contrib/kfac/examples/convnet_mnist_single_main.py
@@ -14,44 +14,26 @@
 # ==============================================================================
 r"""Train a ConvNet on MNIST using K-FAC.
 
-See convnet.py for details.
+Train on single machine. See `convnet.train_mnist_single_machine` for details.
 """
 
 from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
-import argparse
-import sys
 
+from absl import flags
 import tensorflow as tf
 
 from tensorflow.contrib.kfac.examples import convnet
 
-FLAGS = None
+FLAGS = flags.FLAGS
+flags.DEFINE_string("data_dir", "/tmp/mnist", "local mnist dir")
 
 
-def main(argv):
-  _ = argv
-
-  if FLAGS.num_towers > 1:
-    convnet.train_mnist_multitower(
-        FLAGS.data_dir, num_epochs=200, num_towers=FLAGS.num_towers)
-  else:
-    convnet.train_mnist_single_machine(FLAGS.data_dir, num_epochs=200)
+def main(unused_argv):
+  convnet.train_mnist_single_gpu(FLAGS.data_dir, num_epochs=200)
 
 
 if __name__ == "__main__":
-  parser = argparse.ArgumentParser()
-  parser.add_argument(
-      "--data_dir",
-      type=str,
-      default="/tmp/mnist",
-      help="Directory to store dataset in.")
-  parser.add_argument(
-      "--num_towers",
-      type=int,
-      default=1,
-      help="Number of CPUs to split minibatch across.")
-  FLAGS, unparsed = parser.parse_known_args()
-  tf.app.run(main=main, argv=[sys.argv[0]] + unparsed)
+  tf.app.run(main=main)
diff --git a/tensorflow/contrib/kfac/examples/tests/convnet_test.py b/tensorflow/contrib/kfac/examples/tests/convnet_test.py
index 8d86c2bb51..6de775cc79 100644
--- a/tensorflow/contrib/kfac/examples/tests/convnet_test.py
+++ b/tensorflow/contrib/kfac/examples/tests/convnet_test.py
@@ -112,15 +112,16 @@ class ConvNetTest(tf.test.TestCase):
   def testMinimizeLossSingleMachine(self):
     with tf.Graph().as_default():
       loss, accuracy, layer_collection = self._build_toy_problem()
-      accuracy_ = convnet.minimize_loss_single_machine(loss, accuracy,
-                                                       layer_collection)
-      self.assertLess(accuracy_, 1.0)
+      accuracy_ = convnet.minimize_loss_single_machine(
+          loss, accuracy, layer_collection, device="/cpu:0")
+      self.assertLess(accuracy_, 2.0)
 
   def testMinimizeLossDistributed(self):
     with tf.Graph().as_default():
       loss, accuracy, layer_collection = self._build_toy_problem()
-      accuracy_ = convnet.minimize_loss_distributed(
+      accuracy_ = convnet.distributed_grads_only_and_ops_chief_worker(
           task_id=0,
+          is_chief=True,
           num_worker_tasks=1,
           num_ps_tasks=0,
           master="",
@@ -128,7 +129,7 @@ class ConvNetTest(tf.test.TestCase):
           loss=loss,
           accuracy=accuracy,
           layer_collection=layer_collection)
-      self.assertLess(accuracy_, 1.0)
+      self.assertLess(accuracy_, 2.0)
 
   def testTrainMnistSingleMachine(self):
     with tf.Graph().as_default():
@@ -138,7 +139,7 @@ class ConvNetTest(tf.test.TestCase):
       # but there are too few parameters for the model to effectively memorize
       # the training set the way an MLP can.
       convnet.train_mnist_single_machine(
-          data_dir=None, num_epochs=1, use_fake_data=True)
+          data_dir=None, num_epochs=1, use_fake_data=True, device="/cpu:0")
 
   def testTrainMnistMultitower(self):
     with tf.Graph().as_default():
@@ -149,13 +150,15 @@ class ConvNetTest(tf.test.TestCase):
   def testTrainMnistDistributed(self):
     with tf.Graph().as_default():
       # Ensure model training doesn't crash.
-      convnet.train_mnist_distributed(
+      convnet.train_mnist_distributed_sync_replicas(
           task_id=0,
+          is_chief=True,
           num_worker_tasks=1,
           num_ps_tasks=0,
           master="",
           data_dir=None,
           num_epochs=1,
+          op_strategy="chief_worker",
           use_fake_data=True)
 
 
-- 
GitLab


From 3fbec92fa4997fc834807b57b229fa9ada179f6c Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 07:45:31 -0700
Subject: [PATCH 905/960] Turn Cast into a proper builtin operator.

PiperOrigin-RevId: 191590230
---
 tensorflow/contrib/lite/BUILD                 |  1 +
 tensorflow/contrib/lite/builtin_op_data.h     |  7 ++
 tensorflow/contrib/lite/kernels/cast.cc       |  9 +++
 tensorflow/contrib/lite/model.cc              | 70 ++++++++++++-------
 tensorflow/contrib/lite/schema/schema.fbs     |  2 +
 .../contrib/lite/schema/schema_generated.h    | 38 +++++++++-
 .../contrib/lite/toco/tflite/operator.cc      | 26 ++++---
 .../contrib/lite/toco/tflite/operator_test.cc |  2 +-
 8 files changed, 117 insertions(+), 38 deletions(-)

diff --git a/tensorflow/contrib/lite/BUILD b/tensorflow/contrib/lite/BUILD
index ac269d540a..9c4533079c 100644
--- a/tensorflow/contrib/lite/BUILD
+++ b/tensorflow/contrib/lite/BUILD
@@ -89,6 +89,7 @@ cc_library(
     hdrs = [
         "builtin_op_data.h",
     ],
+    deps = [":context"],
 )
 
 cc_library(
diff --git a/tensorflow/contrib/lite/builtin_op_data.h b/tensorflow/contrib/lite/builtin_op_data.h
index 5fc8954743..2b6c24768c 100644
--- a/tensorflow/contrib/lite/builtin_op_data.h
+++ b/tensorflow/contrib/lite/builtin_op_data.h
@@ -17,6 +17,8 @@ limitations under the License.
 
 #include <stdint.h>
 
+#include "tensorflow/contrib/lite/context.h"
+
 #ifdef __cplusplus
 extern "C" {
 #endif  // __cplusplus
@@ -174,6 +176,11 @@ typedef struct {
   int block_size;
 } TfLiteSpaceToDepthParams;
 
+typedef struct {
+  TfLiteType in_data_type;
+  TfLiteType out_data_type;
+} TfLiteCastParams;
+
 typedef enum {
   kTfLiteCombinerTypeSum = 0,
   kTfLiteCombinerTypeMean = 1,
diff --git a/tensorflow/contrib/lite/kernels/cast.cc b/tensorflow/contrib/lite/kernels/cast.cc
index 19942de7bc..17ef2c572e 100644
--- a/tensorflow/contrib/lite/kernels/cast.cc
+++ b/tensorflow/contrib/lite/kernels/cast.cc
@@ -34,6 +34,15 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
   TF_LITE_ENSURE_EQ(context, NumOutputs(node), 1);
   TfLiteTensor* input = GetInput(context, node, kInputTensor);
   TfLiteTensor* output = GetOutput(context, node, kOutputTensor);
+
+  // TODO(ahentz): these two checks would make the new implementation
+  // incompatible with some existing models, where params is not specified. It
+  // is OK not to have them because toco would have set input and output types
+  // to match the parameters.
+  // auto* params = reinterpret_cast<TfLiteCastParams*>(node->builtin_data);
+  // TF_LITE_ENSURE_EQ(context, input->type, params->in_data_type);
+  // TF_LITE_ENSURE_EQ(context, output->type, params->out_data_type);
+
   return context->ResizeTensor(context, output,
                                TfLiteIntArrayCopy(input->dims));
 }
diff --git a/tensorflow/contrib/lite/model.cc b/tensorflow/contrib/lite/model.cc
index 791d1378f3..606f4a5635 100644
--- a/tensorflow/contrib/lite/model.cc
+++ b/tensorflow/contrib/lite/model.cc
@@ -32,6 +32,32 @@ namespace tflite {
 
 const char* kEmptyTensorName = "";
 
+TfLiteStatus ConvertTensorType(TensorType tensor_type, TfLiteType* type,
+                               ErrorReporter* error_reporter) {
+  switch (tensor_type) {
+    case TensorType_FLOAT32:
+      *type = kTfLiteFloat32;
+      break;
+    case TensorType_INT32:
+      *type = kTfLiteInt32;
+      break;
+    case TensorType_UINT8:
+      *type = kTfLiteUInt8;
+      break;
+    case TensorType_INT64:
+      *type = kTfLiteInt64;
+      break;
+    case TensorType_STRING:
+      *type = kTfLiteString;
+      break;
+    default:
+      error_reporter->Report("Unimplemented data type %s (%d) in tensor\n",
+                             EnumNameTensorType(tensor_type), tensor_type);
+      return kTfLiteError;
+  }
+  return kTfLiteOk;
+}
+
 // Loads a model from `filename`. If `mmap_file` is true then use mmap,
 // otherwise make a copy of the model in a buffer.
 std::unique_ptr<Allocation> GetAllocationFromFile(const char* filename,
@@ -307,10 +333,25 @@ void* ParseOpData(const Operator* op, BuiltinOperator op_type,
     case BuiltinOperator_EXP:
     case BuiltinOperator_TOPK_V2:
     case BuiltinOperator_LOG_SOFTMAX:
-    case BuiltinOperator_CAST:
     case BuiltinOperator_DEQUANTIZE:
     case BuiltinOperator_PRELU:
       break;
+    case BuiltinOperator_CAST: {
+      TfLiteCastParams* params = MallocPOD<TfLiteCastParams>();
+      if (auto* schema_params = op->builtin_options_as_CastOptions()) {
+        auto in_status =
+            ConvertTensorType(schema_params->in_data_type(),
+                              &params->in_data_type, error_reporter);
+        auto out_status =
+            ConvertTensorType(schema_params->out_data_type(),
+                              &params->out_data_type, error_reporter);
+        if (in_status != kTfLiteOk || out_status != kTfLiteOk) {
+          break;
+        }
+      }
+      builtin_data = reinterpret_cast<void*>(params);
+      break;
+    }
     case BuiltinOperator_LSH_PROJECTION: {
       TfLiteLSHProjectionParams* params =
           MallocPOD<TfLiteLSHProjectionParams>();
@@ -707,29 +748,10 @@ TfLiteStatus InterpreterBuilder::ParseTensors(
     }
 
     TfLiteType type;
-    switch (tensor->type()) {
-      case TensorType_FLOAT32:
-        type = kTfLiteFloat32;
-        break;
-      case TensorType_INT32:
-        type = kTfLiteInt32;
-        break;
-      case TensorType_UINT8:
-        type = kTfLiteUInt8;
-        break;
-      case TensorType_INT64:
-        type = kTfLiteInt64;
-        break;
-      case TensorType_STRING:
-        type = kTfLiteString;
-        break;
-      default:
-        // tensorType = ArrayType::NONE;
-        error_reporter_->Report("Unimplemented data type %s (%d) in tensor\n",
-                                EnumNameTensorType(tensor->type()),
-                                tensor->type());
-        status = kTfLiteError;
-        continue;
+    if (ConvertTensorType(tensor->type(), &type, error_reporter_) !=
+        kTfLiteOk) {
+      status = kTfLiteError;
+      continue;
     }
     auto get_readonly_data = [&](const char** buffer_data,
                                  size_t* buffer_size) {
diff --git a/tensorflow/contrib/lite/schema/schema.fbs b/tensorflow/contrib/lite/schema/schema.fbs
index 7d2e00fe32..c63bfb28cc 100644
--- a/tensorflow/contrib/lite/schema/schema.fbs
+++ b/tensorflow/contrib/lite/schema/schema.fbs
@@ -381,6 +381,8 @@ table LogSoftmaxOptions {
 }
 
 table CastOptions {
+  in_data_type: TensorType;
+  out_data_type: TensorType;
 }
 
 table DequantizeOptions {
diff --git a/tensorflow/contrib/lite/schema/schema_generated.h b/tensorflow/contrib/lite/schema/schema_generated.h
index 66a97a1460..0735be5c8f 100755
--- a/tensorflow/contrib/lite/schema/schema_generated.h
+++ b/tensorflow/contrib/lite/schema/schema_generated.h
@@ -3702,14 +3702,30 @@ flatbuffers::Offset<LogSoftmaxOptions> CreateLogSoftmaxOptions(flatbuffers::Flat
 
 struct CastOptionsT : public flatbuffers::NativeTable {
   typedef CastOptions TableType;
-  CastOptionsT() {
+  TensorType in_data_type;
+  TensorType out_data_type;
+  CastOptionsT()
+      : in_data_type(TensorType_FLOAT32),
+        out_data_type(TensorType_FLOAT32) {
   }
 };
 
 struct CastOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
   typedef CastOptionsT NativeTableType;
+  enum {
+    VT_IN_DATA_TYPE = 4,
+    VT_OUT_DATA_TYPE = 6
+  };
+  TensorType in_data_type() const {
+    return static_cast<TensorType>(GetField<int8_t>(VT_IN_DATA_TYPE, 0));
+  }
+  TensorType out_data_type() const {
+    return static_cast<TensorType>(GetField<int8_t>(VT_OUT_DATA_TYPE, 0));
+  }
   bool Verify(flatbuffers::Verifier &verifier) const {
     return VerifyTableStart(verifier) &&
+           VerifyField<int8_t>(verifier, VT_IN_DATA_TYPE) &&
+           VerifyField<int8_t>(verifier, VT_OUT_DATA_TYPE) &&
            verifier.EndTable();
   }
   CastOptionsT *UnPack(const flatbuffers::resolver_function_t *_resolver = nullptr) const;
@@ -3720,6 +3736,12 @@ struct CastOptions FLATBUFFERS_FINAL_CLASS : private flatbuffers::Table {
 struct CastOptionsBuilder {
   flatbuffers::FlatBufferBuilder &fbb_;
   flatbuffers::uoffset_t start_;
+  void add_in_data_type(TensorType in_data_type) {
+    fbb_.AddElement<int8_t>(CastOptions::VT_IN_DATA_TYPE, static_cast<int8_t>(in_data_type), 0);
+  }
+  void add_out_data_type(TensorType out_data_type) {
+    fbb_.AddElement<int8_t>(CastOptions::VT_OUT_DATA_TYPE, static_cast<int8_t>(out_data_type), 0);
+  }
   explicit CastOptionsBuilder(flatbuffers::FlatBufferBuilder &_fbb)
         : fbb_(_fbb) {
     start_ = fbb_.StartTable();
@@ -3733,8 +3755,12 @@ struct CastOptionsBuilder {
 };
 
 inline flatbuffers::Offset<CastOptions> CreateCastOptions(
-    flatbuffers::FlatBufferBuilder &_fbb) {
+    flatbuffers::FlatBufferBuilder &_fbb,
+    TensorType in_data_type = TensorType_FLOAT32,
+    TensorType out_data_type = TensorType_FLOAT32) {
   CastOptionsBuilder builder_(_fbb);
+  builder_.add_out_data_type(out_data_type);
+  builder_.add_in_data_type(in_data_type);
   return builder_.Finish();
 }
 
@@ -5727,6 +5753,8 @@ inline CastOptionsT *CastOptions::UnPack(const flatbuffers::resolver_function_t
 inline void CastOptions::UnPackTo(CastOptionsT *_o, const flatbuffers::resolver_function_t *_resolver) const {
   (void)_o;
   (void)_resolver;
+  { auto _e = in_data_type(); _o->in_data_type = _e; };
+  { auto _e = out_data_type(); _o->out_data_type = _e; };
 }
 
 inline flatbuffers::Offset<CastOptions> CastOptions::Pack(flatbuffers::FlatBufferBuilder &_fbb, const CastOptionsT* _o, const flatbuffers::rehasher_function_t *_rehasher) {
@@ -5737,8 +5765,12 @@ inline flatbuffers::Offset<CastOptions> CreateCastOptions(flatbuffers::FlatBuffe
   (void)_rehasher;
   (void)_o;
   struct _VectorArgs { flatbuffers::FlatBufferBuilder *__fbb; const CastOptionsT* __o; const flatbuffers::rehasher_function_t *__rehasher; } _va = { &_fbb, _o, _rehasher}; (void)_va;
+  auto _in_data_type = _o->in_data_type;
+  auto _out_data_type = _o->out_data_type;
   return tflite::CreateCastOptions(
-      _fbb);
+      _fbb,
+      _in_data_type,
+      _out_data_type);
 }
 
 inline DequantizeOptionsT *DequantizeOptions::UnPack(const flatbuffers::resolver_function_t *_resolver) const {
diff --git a/tensorflow/contrib/lite/toco/tflite/operator.cc b/tensorflow/contrib/lite/toco/tflite/operator.cc
index 0cb348bda5..f991529569 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator.cc
@@ -204,17 +204,22 @@ class BatchToSpaceND
                    TocoOperator* op) const override {}
 };
 
-class Cast : public CustomOperator<CastOperator> {
+class Cast : public BuiltinOperator<CastOperator, ::tflite::CastOptions,
+                                    ::tflite::BuiltinOptions_CastOptions> {
  public:
-  using CustomOperator::CustomOperator;
-  void WriteOptions(const TocoOperator& op,
-                    flexbuffers::Builder* fbb) const override {
-    fbb->Int("src_data_type", DataType::Serialize(op.src_data_type));
-    fbb->Int("dst_data_type", DataType::Serialize(op.dst_data_type));
+  using BuiltinOperator::BuiltinOperator;
+  flatbuffers::Offset<TfLiteOptions> WriteOptions(
+      const TocoOperator& op,
+      flatbuffers::FlatBufferBuilder* builder) const override {
+    return ::tflite::CreateCastOptions(*builder,
+                                       DataType::Serialize(op.src_data_type),
+                                       DataType::Serialize(op.dst_data_type));
   }
-  void ReadOptions(const flexbuffers::Map& m, TocoOperator* op) const override {
-    op->src_data_type = DataType::Deserialize(m["src_data_type"].AsInt64());
-    op->dst_data_type = DataType::Deserialize(m["dst_data_type"].AsInt64());
+
+  void ReadOptions(const TfLiteOptions& options,
+                   TocoOperator* op) const override {
+    op->src_data_type = DataType::Deserialize(options.in_data_type());
+    op->dst_data_type = DataType::Deserialize(options.out_data_type());
   }
 };
 
@@ -827,9 +832,10 @@ std::vector<std::unique_ptr<BaseOperator>> BuildOperatorList() {
       new TopK_V2(::tflite::BuiltinOperator_TOPK_V2, OperatorType::kTopK_V2));
   ops.emplace_back(
       new Lstm(::tflite::BuiltinOperator_LSTM, OperatorType::kLstmCell));
+  ops.emplace_back(
+      new Cast(::tflite::BuiltinOperator_CAST, OperatorType::kCast));
 
   // Custom Operators.
-  ops.emplace_back(new Cast("CAST", OperatorType::kCast));
   ops.emplace_back(
       new DepthToSpace("DEPTH_TO_SPACE", OperatorType::kDepthToSpace));
   ops.emplace_back(new FakeQuant("FAKE_QUANT", OperatorType::kFakeQuant));
diff --git a/tensorflow/contrib/lite/toco/tflite/operator_test.cc b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
index f7a213ecfc..4783843b7f 100644
--- a/tensorflow/contrib/lite/toco/tflite/operator_test.cc
+++ b/tensorflow/contrib/lite/toco/tflite/operator_test.cc
@@ -131,7 +131,7 @@ TEST_F(OperatorTest, BuiltinMean) {
   EXPECT_EQ(op.keep_dims, output_toco_op->keep_dims);
 }
 
-TEST_F(OperatorTest, CustomCast) {
+TEST_F(OperatorTest, BuiltinCast) {
   CastOperator op;
   op.src_data_type = ArrayDataType::kFloat;
   op.dst_data_type = ArrayDataType::kUint8;
-- 
GitLab


From cb8cdf261f932186d34bdd43b86e887eec450213 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 09:31:25 -0700
Subject: [PATCH 906/960] Restructuring the HLO partitioner to fit host
 computation and handle kCall. Pre process the input module to reassign
 reserved devices (like the host compute one) to new sequentially increasing
 device numbers, and track those in the GlobalState. This avoids having many
 places where we need to spread the is-special-device logic, within the HLO
 partitioner and its related components. Added handling for kCall, which was
 missing from previous implementation.

PiperOrigin-RevId: 191601831
---
 .../compiler/xla/service/hlo_computation.cc    | 18 +++++++-----------
 .../compiler/xla/service/hlo_instruction.h     |  7 +++++++
 tensorflow/compiler/xla/service/hlo_sharding.h |  4 ++++
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/tensorflow/compiler/xla/service/hlo_computation.cc b/tensorflow/compiler/xla/service/hlo_computation.cc
index 6f983d0b95..594413e88f 100644
--- a/tensorflow/compiler/xla/service/hlo_computation.cc
+++ b/tensorflow/compiler/xla/service/hlo_computation.cc
@@ -304,19 +304,15 @@ void ComputeComputationPostOrder(
     HloComputation* computation,
     tensorflow::gtl::FlatSet<HloComputation*>* visited,
     std::list<HloComputation*>* post_order) {
-  if (visited->count(computation) > 0) {
-    return;
-  }
-
-  for (auto* instruction : computation->instructions()) {
-    for (HloComputation* called_computation :
-         instruction->called_computations()) {
-      ComputeComputationPostOrder(called_computation, visited, post_order);
+  if (visited->insert(computation).second) {
+    for (auto* instruction : computation->instructions()) {
+      for (HloComputation* called_computation :
+           instruction->called_computations()) {
+        ComputeComputationPostOrder(called_computation, visited, post_order);
+      }
     }
+    post_order->push_back(computation);
   }
-
-  visited->insert(computation);
-  post_order->push_back(computation);
 }
 
 }  // namespace
diff --git a/tensorflow/compiler/xla/service/hlo_instruction.h b/tensorflow/compiler/xla/service/hlo_instruction.h
index a94ba145df..80f8408244 100644
--- a/tensorflow/compiler/xla/service/hlo_instruction.h
+++ b/tensorflow/compiler/xla/service/hlo_instruction.h
@@ -928,6 +928,13 @@ class HloInstruction {
   const HloSharding& sharding_or_default(const HloSharding& default_) const {
     return sharding_ ? *sharding_ : default_;
   }
+  // Returns the sharding unique device, if any.
+  tensorflow::gtl::optional<int64> sharding_unique_device() const {
+    if (sharding_ == nullptr || !sharding_->HasUniqueDevice()) {
+      return tensorflow::gtl::optional<int64>();
+    }
+    return sharding_->UniqueDevice().ValueOrDie();
+  }
   // Sets the sharding of this operator. Should only be called by HloModule or
   // HloComputation methods.
   void set_sharding(const HloSharding& sharding) {
diff --git a/tensorflow/compiler/xla/service/hlo_sharding.h b/tensorflow/compiler/xla/service/hlo_sharding.h
index 18d406f370..06204acbca 100644
--- a/tensorflow/compiler/xla/service/hlo_sharding.h
+++ b/tensorflow/compiler/xla/service/hlo_sharding.h
@@ -94,6 +94,10 @@ class HloSharding {
   // Create a new sharding from a protobuf OpSharding.
   static StatusOr<HloSharding> FromProto(const OpSharding& proto);
 
+  // Checks whether device is a reserved device number. A reserved device number
+  // has usually a special meaning, with dedicated handling logic.
+  static bool IsReservedDevice(int64 device) { return device < 0; }
+
   OpSharding ToProto() const;
   string ToString() const;
 
-- 
GitLab


From 8badd11d875a826bd318ed439909d5c47a7fb811 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 09:51:55 -0700
Subject: [PATCH 907/960] Check arguments of ComputeConvSizes that should be
 positive.

PiperOrigin-RevId: 191604311
---
 .../graph_transformations/propagate_fixed_sizes.cc     | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
index b96d698675..68d6f21cf8 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/propagate_fixed_sizes.cc
@@ -38,6 +38,16 @@ void ComputeConvSizes(const Shape& input_shape, int output_depth, int kwidth,
   const int input_height = input_shape.dims(1);
   const int batch = input_shape.dims(0);
 
+  CHECK_GE(input_width, 1);
+  CHECK_GE(input_height, 1);
+  CHECK_GE(batch, 1);
+  CHECK_GE(kwidth, 1);
+  CHECK_GE(kheight, 1);
+  CHECK_GE(stride_width, 1);
+  CHECK_GE(stride_height, 1);
+  CHECK_GE(dilation_width_factor, 1);
+  CHECK_GE(dilation_height_factor, 1);
+
   int dilated_kwidth = dilation_width_factor * (kwidth - 1) + 1;
   int dilated_kheight = dilation_height_factor * (kheight - 1) + 1;
 
-- 
GitLab


From f4dcfcaae4e85bbb727eb1f5bfc14f6fa3a055ed Mon Sep 17 00:00:00 2001
From: Tony Wang <tonywy@google.com>
Date: Wed, 4 Apr 2018 10:00:39 -0700
Subject: [PATCH 908/960] Automated g4 rollback of changelist 191527251

PiperOrigin-RevId: 191605505
---
 .../xla/legacy_flags/debug_options_flags.cc   |   7 -
 tensorflow/compiler/xla/service/cpu/BUILD     |  25 ----
 .../compiler/xla/service/cpu/cpu_runtime.cc   |   8 --
 .../compiler/xla/service/cpu/cpu_runtime.h    |   4 -
 .../xla/service/cpu/cpu_runtime_test.cc       |  82 +----------
 .../xla/service/cpu/dot_op_emitter.cc         |  23 ++--
 .../xla/service/cpu/runtime_matmul_mkl.cc     | 129 ------------------
 .../xla/service/cpu/runtime_matmul_mkl.h      |  80 -----------
 .../xla/service/cpu/simple_orc_jit.cc         |   5 -
 tensorflow/compiler/xla/xla.proto             |   3 -
 10 files changed, 15 insertions(+), 351 deletions(-)
 delete mode 100644 tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
 delete mode 100644 tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h

diff --git a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
index f037663e3f..c8ed3e3a2b 100644
--- a/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
+++ b/tensorflow/compiler/xla/legacy_flags/debug_options_flags.cc
@@ -40,9 +40,6 @@ void SetDebugOptionsDefaults(DebugOptions* flags) {
   flags->set_xla_cpu_multi_thread_eigen(true);
   flags->set_xla_gpu_cuda_data_dir("./cuda_sdk_lib");
   flags->set_xla_eliminate_hlo_implicit_broadcast(true);
-#ifdef INTEL_MKL
-  flags->set_xla_cpu_use_mkl_dnn(true);
-#endif  // INTEL_MKL
 
   // Set cudnn batchnorm off by default; it does not provide a performance win
   // on average.
@@ -291,10 +288,6 @@ void AllocateFlags() {
           flag_values->xla_gpu_use_cudnn_batchnorm(),
           "Allows the GPU backend to implement batchnorm HLOs using cudnn, "
           "rather than expanding them to a soup of HLOs."),
-      tensorflow::Flag("xla_cpu_use_mkl_dnn",
-                       bool_setter_for(&DebugOptions::set_xla_cpu_use_mkl_dnn),
-                       flag_values->xla_cpu_use_mkl_dnn(),
-                       "Generate calls to MKL-DNN in the CPU backend."),
   });
   ParseFlagsFromEnv(*flag_objects);
 }
diff --git a/tensorflow/compiler/xla/service/cpu/BUILD b/tensorflow/compiler/xla/service/cpu/BUILD
index d22c135249..966e2d0fc5 100644
--- a/tensorflow/compiler/xla/service/cpu/BUILD
+++ b/tensorflow/compiler/xla/service/cpu/BUILD
@@ -18,10 +18,6 @@ load(":build_defs.bzl", "runtime_copts")
 load("//tensorflow:tensorflow.bzl", "tf_cc_test")
 load("//tensorflow:tensorflow.bzl", "tf_cc_binary")
 load("//tensorflow/compiler/xla:xla.bzl", "ORC_JIT_MEMORY_MAPPER_TARGETS")
-load(
-    "//third_party/mkl:build_defs.bzl",
-    "if_mkl",
-)
 
 # Filegroup used to collect source files for dependency checking.
 filegroup(
@@ -174,7 +170,6 @@ cc_library(
         ":runtime_fft",
         ":runtime_fork_join",
         ":runtime_matmul",
-        ":runtime_matmul_mkl",
         ":runtime_single_threaded_conv2d",
         ":runtime_single_threaded_matmul",
         "@llvm//:execution_engine",
@@ -539,28 +534,10 @@ cc_library(
         ":runtime_matvec",
         "//tensorflow/compiler/xla:executable_run_options",
         "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib",
         "//third_party/eigen3",
     ],
 )
 
-cc_library(
-    name = "runtime_matmul_mkl",
-    srcs = ["runtime_matmul_mkl.cc"],
-    hdrs = ["runtime_matmul_mkl.h"],
-    copts = runtime_copts(),
-    visibility = ["//visibility:public"],
-    deps = [
-        "//tensorflow/compiler/xla:executable_run_options",
-        "//tensorflow/core:framework_lite",
-        "//tensorflow/core:lib",
-        "//third_party/eigen3",
-    ] + if_mkl([
-        "//third_party/mkl:intel_binary_blob",
-        "@mkl_dnn",
-    ]),
-)
-
 cc_library(
     name = "runtime_single_threaded_conv2d",
     srcs = [
@@ -607,12 +584,10 @@ cc_library(
 tf_cc_test(
     name = "cpu_runtime_test",
     srcs = ["cpu_runtime_test.cc"],
-    shard_count = 10,
     tags = ["optonly"],
     deps = [
         ":cpu_runtime",
         ":runtime_matmul",
-        ":runtime_matmul_mkl",
         ":runtime_single_threaded_matmul",
         "//tensorflow/compiler/xla:array2d",
         "//tensorflow/compiler/xla:types",
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
index 872b0be1f8..9a3bd68c80 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.cc
@@ -37,14 +37,6 @@ extern const char* const kEigenMatMulF32SymbolName =
     "__xla_cpu_runtime_EigenMatMulF32";
 extern const char* const kEigenMatMulF64SymbolName =
     "__xla_cpu_runtime_EigenMatMulF64";
-extern const char* const kMKLMatMulF32SymbolName =
-    "__xla_cpu_runtime_MKLMatMulF32";
-extern const char* const kMKLMatMulF64SymbolName =
-    "__xla_cpu_runtime_MKLMatMulF64";
-extern const char* const kMKLSingleThreadedMatMulF32SymbolName =
-    "__xla_cpu_runtime_MKLSingleThreadedMatMulF32";
-extern const char* const kMKLSingleThreadedMatMulF64SymbolName =
-    "__xla_cpu_runtime_MKLSingleThreadedMatMulF64";
 extern const char* const kEigenConvF16SymbolName =
     "__xla_cpu_runtime_EigenConvF16";
 extern const char* const kEigenConvF32SymbolName =
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
index e392e231b4..e61d6ea28b 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime.h
@@ -44,10 +44,6 @@ namespace runtime {
 extern const char* const kEigenMatMulF16SymbolName;
 extern const char* const kEigenMatMulF32SymbolName;
 extern const char* const kEigenMatMulF64SymbolName;
-extern const char* const kMKLMatMulF32SymbolName;
-extern const char* const kMKLMatMulF64SymbolName;
-extern const char* const kMKLSingleThreadedMatMulF32SymbolName;
-extern const char* const kMKLSingleThreadedMatMulF64SymbolName;
 extern const char* const kEigenConvF16SymbolName;
 extern const char* const kEigenConvF32SymbolName;
 extern const char* const kEigenFftSymbolName;
diff --git a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
index 9e04307295..f385829cdf 100644
--- a/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
+++ b/tensorflow/compiler/xla/service/cpu/cpu_runtime_test.cc
@@ -24,7 +24,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/client/local_client.h"
 #include "tensorflow/compiler/xla/ptr_util.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
-#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
 #include "tensorflow/compiler/xla/types.h"
 #include "tensorflow/core/common_runtime/eigen_thread_pool.h"
@@ -131,19 +130,21 @@ MatMulShape MatMulShapes[] = {
 // * transpose_lhs
 // * transpose_rhs
 // * single_threaded
-using MatMulTestParam = std::tuple<MatMulShape, bool, bool, bool>;
+using EigenMatMulTestParam = std::tuple<MatMulShape, bool, bool, bool>;
 
-class EigenMatMulTest : public CpuRuntimeTest,
-                        public ::testing::WithParamInterface<MatMulTestParam> {
+class EigenMatMulTest
+    : public CpuRuntimeTest,
+      public ::testing::WithParamInterface<EigenMatMulTestParam> {
  public:
-  static string Name(const ::testing::TestParamInfo<MatMulTestParam>& info) {
+  static string Name(
+      const ::testing::TestParamInfo<EigenMatMulTestParam>& info) {
     MatMulShape shape = std::get<0>(info.param);
     bool transpose_lhs = std::get<1>(info.param);
     bool transpose_rhs = std::get<2>(info.param);
     bool single_threaded = std::get<3>(info.param);
 
     return tensorflow::strings::Printf(
-        "EigenMatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
+        "MatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
         transpose_lhs ? "Tlhs_" : "", transpose_rhs ? "Trhs_" : "",
         single_threaded ? "single" : "multi");
   }
@@ -168,74 +169,5 @@ INSTANTIATE_TEST_CASE_P(EigenMatMulTestInstantiaion, EigenMatMulTest,
                                            ::testing::Bool()),
                         EigenMatMulTest::Name);
 
-#ifdef INTEL_MKL
-class MKLMatMulTest : public CpuRuntimeTest,
-                      public ::testing::WithParamInterface<MatMulTestParam> {
- public:
-  static string Name(const ::testing::TestParamInfo<MatMulTestParam>& info) {
-    MatMulShape shape = std::get<0>(info.param);
-    bool transpose_lhs = std::get<1>(info.param);
-    bool transpose_rhs = std::get<2>(info.param);
-    bool single_threaded = std::get<3>(info.param);
-
-    return tensorflow::strings::Printf(
-        "MKLMatMul_%lld_%lld_%lld_%s%s%s_threaded", shape.m, shape.k, shape.n,
-        transpose_lhs ? "Tlhs_" : "", transpose_rhs ? "Trhs_" : "",
-        single_threaded ? "single" : "multi");
-  }
-};
-
-std::unique_ptr<Array2D<float>> MKLMatrixMultiply(const Array2D<float>& a,
-                                                  const Array2D<float>& b,
-                                                  bool transpose_lhs,
-                                                  bool transpose_rhs,
-                                                  bool single_threaded) {
-  CHECK_EQ(a.width(), b.height());
-  int64 m = a.height();
-  int64 n = b.width();
-  int64 k = a.width();
-
-  // The MKL matmul runtime function expects the matrix to be in column major
-  // order and array2d is in row-major order. Create transposes of a and b. The
-  // 'data' buffer in the transposed array is the original array in column major
-  // order.
-  auto a_transpose = MaybeTransposeArray2D(a, !transpose_lhs);
-  auto b_transpose = MaybeTransposeArray2D(b, !transpose_rhs);
-
-  // Since we're going to transpose c before returning it, swap the order of the
-  // dimension sizes to ensure the returned array is properly dimensioned.
-  auto c_transpose = MakeUnique<Array2D<float>>(n, m);
-  if (single_threaded) {
-    __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
-        nullptr, c_transpose->data(), a_transpose->data(), b_transpose->data(),
-        m, n, k, transpose_lhs, transpose_rhs);
-  } else {
-    __xla_cpu_runtime_MKLMatMulF32(nullptr, c_transpose->data(),
-                                   a_transpose->data(), b_transpose->data(), m,
-                                   n, k, transpose_lhs, transpose_rhs);
-  }
-  return MaybeTransposeArray2D(*c_transpose, true);
-}
-
-TEST_P(MKLMatMulTest, DoIt) {
-  MatMulShape shape = std::get<0>(GetParam());
-  bool transpose_lhs = std::get<1>(GetParam());
-  bool transpose_rhs = std::get<2>(GetParam());
-  bool single_threaded = std::get<3>(GetParam());
-
-  auto a = MakeLinspaceArray2D(0.0, 1.0, shape.m, shape.k);
-  auto b = MakeLinspaceArray2D(-2.0, 2.0, shape.k, shape.n);
-  auto c =
-      MKLMatrixMultiply(*a, *b, transpose_lhs, transpose_rhs, single_threaded);
-  CheckMatrixMultiply(*a, *b, *c);
-}
-
-INSTANTIATE_TEST_CASE_P(MKLMatMulTestInstantiaion, MKLMatMulTest,
-                        ::testing::Combine(::testing::ValuesIn(MatMulShapes),
-                                           ::testing::Bool(), ::testing::Bool(),
-                                           ::testing::Bool()),
-                        MKLMatMulTest::Name);
-#endif  // INTEL_MKL
-
 }  // namespace
 }  // namespace xla
diff --git a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
index 29afd8ea5f..8b1e20d79e 100644
--- a/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
+++ b/tensorflow/compiler/xla/service/cpu/dot_op_emitter.cc
@@ -918,35 +918,28 @@ tensorflow::Status DotOpEmitter::EmitCallToRuntime() {
   // The two transpose_... parameters are actually booleans, but we use int32
   // to avoid target-dependent calling convention details.
 
-  bool multi_threaded =
+  bool multi_threaded_eigen =
       hlo_module_config_.debug_options().xla_cpu_multi_thread_eigen();
-  bool use_mkl_dnn = hlo_module_config_.debug_options().xla_cpu_use_mkl_dnn();
   PrimitiveType type = target_array_.GetShape().element_type();
   llvm::Type* float_type;
   const char* fn_name;
   switch (type) {
     case F16:
-      fn_name = multi_threaded
+      fn_name = multi_threaded_eigen
                     ? runtime::kEigenMatMulF16SymbolName
                     : runtime::kEigenSingleThreadedMatMulF16SymbolName;
       float_type = ir_builder_->getHalfTy();
       break;
     case F32:
-      fn_name = multi_threaded
-                    ? (use_mkl_dnn ? runtime::kMKLMatMulF32SymbolName
-                                   : runtime::kEigenMatMulF32SymbolName)
-                    : (use_mkl_dnn
-                           ? runtime::kMKLSingleThreadedMatMulF32SymbolName
-                           : runtime::kEigenSingleThreadedMatMulF32SymbolName);
+      fn_name = multi_threaded_eigen
+                    ? runtime::kEigenMatMulF32SymbolName
+                    : runtime::kEigenSingleThreadedMatMulF32SymbolName;
       float_type = ir_builder_->getFloatTy();
       break;
     case F64:
-      fn_name = multi_threaded
-                    ? (use_mkl_dnn ? runtime::kMKLMatMulF64SymbolName
-                                   : runtime::kEigenMatMulF64SymbolName)
-                    : (use_mkl_dnn
-                           ? runtime::kMKLSingleThreadedMatMulF64SymbolName
-                           : runtime::kEigenSingleThreadedMatMulF64SymbolName);
+      fn_name = multi_threaded_eigen
+                    ? runtime::kEigenMatMulF64SymbolName
+                    : runtime::kEigenSingleThreadedMatMulF64SymbolName;
       float_type = ir_builder_->getDoubleTy();
       break;
     default:
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
deleted file mode 100644
index 729a4e7f5b..0000000000
--- a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.cc
+++ /dev/null
@@ -1,129 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifdef INTEL_MKL
-#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
-#include "third_party/intel_mkl_ml/include/mkl_cblas.h"
-#include "third_party/intel_mkl_ml/include/mkl_service.h"
-
-#include "tensorflow/compiler/xla/executable_run_options.h"
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/types.h"
-
-#define EIGEN_USE_THREADS
-#include "third_party/eigen3/unsupported/Eigen/CXX11/ThreadPool"
-
-using tensorflow::int32;
-using tensorflow::int64;
-
-namespace {
-// BLAS GEMM API for 32-bit Matrix Multiplication.
-
-// MatMul function is defined as: c = alpha * op(a) * op(b) + beta * c.
-// Since XLA MatMul does not used alpha, beta, we set them to 1.0 and 0.0.
-// Matrix lhs, rhs and out are all colum-major.
-void MatMulF32(const void* run_options_ptr, float* out, float* lhs, float* rhs,
-               int64 m, int64 n, int64 k, int32 transpose_lhs,
-               int32 transpose_rhs) {
-  const float alpha = 1.0f, beta = 0.0f;
-  // lda, ldb, and ldc are the leading dimensions of matrices a, b, and c,
-  // respectively. For column-major matrices, the leading dimension is the
-  // stride between consecutive columns (which equals the number of rows). If
-  // the matrix is transposed, the leading dimension is the stride between
-  // consecutive rows (which equals the number of columns).
-  int lda = transpose_lhs ? k : m;
-  int ldb = transpose_rhs ? n : k;
-  int ldc = m;
-  cblas_sgemm(CblasColMajor, transpose_lhs ? CblasTrans : CblasNoTrans,
-              transpose_rhs ? CblasTrans : CblasNoTrans, m, n, k, alpha, lhs,
-              lda, rhs, ldb, beta, out, ldc);
-}
-
-// BLAS GEMM API for 64-bit Matrix Multiplication.
-
-// MatMul function is defined as: c = alpha * op(a) * op(b) + beta * c.
-// Since XLA MatMul does not used alpha, beta, we set them to 1.0 and 0.0.
-// Matrix lhs, rhs and out are all colum-major.
-void MatMulF64(const void* run_options_ptr, double* out, double* lhs,
-               double* rhs, int64 m, int64 n, int64 k, int32 transpose_lhs,
-               int32 transpose_rhs) {
-  const float alpha = 1.0f, beta = 0.0f;
-  // lda, ldb, and ldc are the leading dimensions of matrices a, b, and c,
-  // respectively. For a column-major matrix, the leading dimension is the
-  // stride between consecutive columns (which equals the number of rows). If
-  // the matrix is transposed, the leading dimension is the stride between
-  // consecutive rows (which equals the number of columns).
-  int lda = transpose_lhs ? k : m;
-  int ldb = transpose_rhs ? n : k;
-  int ldc = m;
-  cblas_dgemm(CblasColMajor, transpose_lhs ? CblasTrans : CblasNoTrans,
-              transpose_rhs ? CblasTrans : CblasNoTrans, m, n, k, alpha, lhs,
-              lda, rhs, ldb, beta, out, ldc);
-}
-
-}  // namespace
-
-void __xla_cpu_runtime_MKLMatMulF32(const void* run_options_ptr, float* out,
-                                    float* lhs, float* rhs, int64 m, int64 n,
-                                    int64 k, int32 transpose_lhs,
-                                    int32 transpose_rhs) {
-  const xla::ExecutableRunOptions* run_options =
-      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
-  // BLAS GEMM MatMul uses OpenMP for parallelization, so we pass the thread
-  // number specified in intra_op_thread_pool to MKL.
-  int prev_num_threads = mkl_set_num_threads_local(
-      run_options->intra_op_thread_pool()->numThreads());
-  MatMulF32(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
-  // Set thread number back to the previous number.
-  mkl_set_num_threads_local(prev_num_threads);
-}
-// BLAS GEMM API for 64-bit Matrix Multiplication
-void __xla_cpu_runtime_MKLMatMulF64(const void* run_options_ptr, double* out,
-                                    double* lhs, double* rhs, int64 m, int64 n,
-                                    int64 k, int32 transpose_lhs,
-                                    int32 transpose_rhs) {
-  const xla::ExecutableRunOptions* run_options =
-      static_cast<const xla::ExecutableRunOptions*>(run_options_ptr);
-  // BLAS GEMM MatMul uses OpenMP for parallelization, so we pass the thread
-  // number specified in intra_op_thread_pool to MKL.
-  int prev_num_threads = mkl_set_num_threads_local(
-      run_options->intra_op_thread_pool()->numThreads());
-  MatMulF64(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
-  // Set thread number back to the previous number.
-  mkl_set_num_threads_local(prev_num_threads);
-}
-void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(const void* run_options_ptr,
-                                                  float* out, float* lhs,
-                                                  float* rhs, int64 m, int64 n,
-                                                  int64 k, int32 transpose_lhs,
-                                                  int32 transpose_rhs) {
-  // Set the thread number to 1 for single threaded excution.
-  int prev_num_threads = mkl_set_num_threads_local(1);
-  MatMulF32(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
-  // Set thread number back to the previous number.
-  mkl_set_num_threads_local(prev_num_threads);
-}
-void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(const void* run_options_ptr,
-                                                  double* out, double* lhs,
-                                                  double* rhs, int64 m, int64 n,
-                                                  int64 k, int32 transpose_lhs,
-                                                  int32 transpose_rhs) {
-  // Set the thread number to 1 for single threaded excution.
-  int prev_num_threads = mkl_set_num_threads_local(1);
-  MatMulF64(nullptr, out, lhs, rhs, m, n, k, transpose_lhs, transpose_rhs);
-  // Set thread number back to the previous number.
-  mkl_set_num_threads_local(prev_num_threads);
-}
-#endif  // INTEL_MKL
diff --git a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h b/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h
deleted file mode 100644
index 9dbc506c08..0000000000
--- a/tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-==============================================================================*/
-
-#ifndef TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
-#define TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
-
-#include "tensorflow/core/platform/logging.h"
-#include "tensorflow/core/platform/types.h"
-#ifdef INTEL_MKL
-#include "third_party/intel_mkl_ml/include/mkl_cblas.h"
-
-extern void __xla_cpu_runtime_MKLMatMulF32(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
-    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs);
-extern void __xla_cpu_runtime_MKLMatMulF64(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
-    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs);
-extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
-    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs);
-extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
-    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs);
-
-#else
-extern void __xla_cpu_runtime_MKLMatMulF32(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
-    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs) {
-  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
-                "INTEL_MKL. Add --config=mkl to build with MKL.";
-}
-extern void __xla_cpu_runtime_MKLMatMulF64(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
-    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs) {
-  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
-                "INTEL_MKL. Add --config=mkl to build with MKL.";
-}
-extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF32(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, float* out,
-    float* lhs, float* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs) {
-  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
-                "INTEL_MKL. Add --config=mkl to build with MKL.";
-}
-extern void __xla_cpu_runtime_MKLSingleThreadedMatMulF64(
-    const void* /* xla::ExecutableRunOptions* */ run_options_ptr, double* out,
-    double* lhs, double* rhs, tensorflow::int64 m, tensorflow::int64 n,
-    tensorflow::int64 k, tensorflow::int32 transpose_lhs,
-    tensorflow::int32 transpose_rhs) {
-  LOG(FATAL) << "Attempt to call MKL MatMul runtime library without defining "
-                "INTEL_MKL. Add --config=mkl to build with MKL.";
-}
-
-#endif  // INTEL_MKL
-#endif  // TENSORFLOW_COMPILER_XLA_SERVICE_CPU_RUNTIME_MATMUL_MKL_H_
diff --git a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
index b7ce5bbe47..4198260a22 100644
--- a/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
+++ b/tensorflow/compiler/xla/service/cpu/simple_orc_jit.cc
@@ -35,7 +35,6 @@ limitations under the License.
 #include "tensorflow/compiler/xla/service/cpu/runtime_fork_join.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_fp16.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_matmul.h"
-#include "tensorflow/compiler/xla/service/cpu/runtime_matmul_mkl.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_conv2d.h"
 #include "tensorflow/compiler/xla/service/cpu/runtime_single_threaded_matmul.h"
 #include "tensorflow/compiler/xla/service/cpu/windows_compatibility.h"
@@ -184,10 +183,6 @@ bool RegisterKnownJITSymbols() {
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF16);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF32);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenMatMulF64);
-  REGISTER_CPU_RUNTIME_SYMBOL(MKLMatMulF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(MKLMatMulF64);
-  REGISTER_CPU_RUNTIME_SYMBOL(MKLSingleThreadedMatMulF32);
-  REGISTER_CPU_RUNTIME_SYMBOL(MKLSingleThreadedMatMulF64);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF16);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedConvF32);
   REGISTER_CPU_RUNTIME_SYMBOL(EigenSingleThreadedMatMulF16);
diff --git a/tensorflow/compiler/xla/xla.proto b/tensorflow/compiler/xla/xla.proto
index f9943f71d3..5cb18113e5 100644
--- a/tensorflow/compiler/xla/xla.proto
+++ b/tensorflow/compiler/xla/xla.proto
@@ -189,9 +189,6 @@ message DebugOptions {
   // directory.
   string xla_dump_per_pass_hlo_proto_to = 96;
 
-  // Generate calls to MKL-DNN in the CPU backend.
-  bool xla_cpu_use_mkl_dnn = 97;
-
   // Extra options to pass to the compilation backend; specific interpretation
   // of these values is left to the backend.
   map<string, string> xla_backend_extra_options = 500;
-- 
GitLab


From 0fef384fb94f83abd9c787b3c8ab7abc1f7ade95 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 11:15:56 -0700
Subject: [PATCH 909/960] Guard against out-of-bounds dims accesses, even in
 optimized, no-asserts builds.

PiperOrigin-RevId: 191617948
---
 tensorflow/contrib/lite/toco/model.h | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/tensorflow/contrib/lite/toco/model.h b/tensorflow/contrib/lite/toco/model.h
index 64269d369d..9bd72e7de1 100644
--- a/tensorflow/contrib/lite/toco/model.h
+++ b/tensorflow/contrib/lite/toco/model.h
@@ -1507,7 +1507,14 @@ class Shape {
 
   // We still have that one convenience accessor to avoid
   // the awkward double bracket issue:  shape.dims()[i].
-  int dims(int i) const { return dims_[i]; }
+  int dims(int i) const {
+    // Always check for out-of-bounds accesses, even in optimized builds where
+    // standard assertions are disabled. Out-of-bounds access here is a common
+    // occurence.
+    CHECK_GE(i, 0);
+    CHECK_GT(dims_.size(), i);
+    return dims_[i];
+  }
 
   bool operator==(const Shape& comp) const {
     return (this->dims_ == comp.dims());
-- 
GitLab


From 73b0540f81d6a5b88557b69df1bc3da25ca81a1b Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 4 Apr 2018 11:35:05 -0700
Subject: [PATCH 910/960] Also registers the gpu resourcegather kernel for half
 (no idea why it was missing)

PiperOrigin-RevId: 191621111
---
 tensorflow/core/kernels/resource_variable_ops.cc | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/core/kernels/resource_variable_ops.cc b/tensorflow/core/kernels/resource_variable_ops.cc
index 5c54609ee6..f49a05c70a 100644
--- a/tensorflow/core/kernels/resource_variable_ops.cc
+++ b/tensorflow/core/kernels/resource_variable_ops.cc
@@ -605,7 +605,7 @@ TF_CALL_QUANTIZED_TYPES(REGISTER_GATHER_CPU);
 #if GOOGLE_CUDA
 #define REGISTER_GATHER_GPU(type) REGISTER_GATHER_ALL_INDICES(GPU, type)
 
-TF_CALL_GPU_NUMBER_TYPES_NO_HALF(REGISTER_GATHER_GPU);
+TF_CALL_GPU_NUMBER_TYPES(REGISTER_GATHER_GPU);
 
 #endif  // GOOGLE_CUDA
 
-- 
GitLab


From bd540c8c45bf66a9c14af38a272840b47731b91a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 12:00:38 -0700
Subject: [PATCH 911/960] Warning when tweaking a minmax to contain 0.

PiperOrigin-RevId: 191625723
---
 .../toco/graph_transformations/read_fake_quant_min_max.cc  | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc b/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc
index 11f8d4b6ee..bdcca5b7ca 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/read_fake_quant_min_max.cc
@@ -72,6 +72,13 @@ bool ReadFakeQuantMinMax::Run(Model* model, std::size_t op_index) {
     minmax.min = min_array.GetBuffer<ArrayDataType::kFloat>().data[0];
     minmax.max = max_array.GetBuffer<ArrayDataType::kFloat>().data[0];
     // We always want [min, max] to contain 0.
+    if (minmax.min > 0 || minmax.max < 0) {
+      LOG(ERROR) << "For " << LogName(*fakequant_op) << " the MinMax range "
+                 << "[" << minmax.min << ", " << minmax.max
+                 << "] does not contain 0. "
+                 << "Proceeding by tweaking it to contain 0, which will result "
+                    "in poor accuracy.";
+    }
     minmax.min = std::min(minmax.min, 0.);
     minmax.max = std::max(minmax.max, 0.);
 
-- 
GitLab


From 13e103b8f0dcc89673dd0d3d589b976c05c37a09 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 12:08:49 -0700
Subject: [PATCH 912/960] Replaced calls to deprecated tensorflow::StringPiece
 methods with their tensorflow::str_util equivalents.

This will allow the deprecated methods to be removed.

PiperOrigin-RevId: 191627087
---
 tensorflow/core/lib/strings/numbers.cc        |  5 ++--
 .../core/lib/strings/ordered_code_test.cc     |  3 +-
 tensorflow/core/lib/strings/scanner.h         |  5 ++--
 tensorflow/core/platform/cloud/BUILD          |  2 ++
 .../core/platform/cloud/gcs_file_system.cc    |  9 +++---
 .../platform/cloud/gcs_file_system_test.cc    | 17 ++++++-----
 .../cloud/retrying_file_system_test.cc        | 29 ++++++++++---------
 .../platform/cloud/retrying_utils_test.cc     |  8 ++---
 tensorflow/core/util/command_line_flags.cc    | 20 +++++++++----
 .../core/util/device_name_utils_test.cc       |  2 +-
 tensorflow/core/util/equal_graph_def.cc       |  5 ++--
 tensorflow/core/util/memmapped_file_system.cc |  3 +-
 tensorflow/core/util/reporter_test.cc         |  2 +-
 .../core/util/tensor_slice_reader_test.cc     |  3 +-
 .../core/util/tensor_slice_writer_test.cc     |  9 +++---
 .../tools/graph_transforms/backports_test.cc  |  3 +-
 .../graph_transforms/fold_constants_test.cc   |  5 ++--
 .../freeze_requantization_ranges.cc           |  9 +++---
 .../tools/graph_transforms/insert_logging.cc  |  3 +-
 .../tools/graph_transforms/sparsify_gather.cc |  7 +++--
 .../graph_transforms/transform_graph_test.cc  |  8 ++---
 .../tools/graph_transforms/transform_utils.cc |  7 ++---
 22 files changed, 94 insertions(+), 70 deletions(-)

diff --git a/tensorflow/core/lib/strings/numbers.cc b/tensorflow/core/lib/strings/numbers.cc
index 516decc3c0..8f34baa7de 100644
--- a/tensorflow/core/lib/strings/numbers.cc
+++ b/tensorflow/core/lib/strings/numbers.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include <locale>
 #include <unordered_map>
 
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/macros.h"
@@ -203,7 +204,7 @@ bool safe_strto64(StringPiece str, int64* value) {
 
   int64 vlimit = kint64max;
   int sign = 1;
-  if (str.Consume("-")) {
+  if (str_util::ConsumePrefix(&str, "-")) {
     sign = -1;
     // Different limit for positive and negative integers.
     vlimit = kint64min;
@@ -265,7 +266,7 @@ bool safe_strto32(StringPiece str, int32* value) {
 
   int64 vmax = kint32max;
   int sign = 1;
-  if (str.Consume("-")) {
+  if (str_util::ConsumePrefix(&str, "-")) {
     sign = -1;
     // Different max for positive and negative integers.
     ++vmax;
diff --git a/tensorflow/core/lib/strings/ordered_code_test.cc b/tensorflow/core/lib/strings/ordered_code_test.cc
index fee8a6f93e..ede9f4d390 100644
--- a/tensorflow/core/lib/strings/ordered_code_test.cc
+++ b/tensorflow/core/lib/strings/ordered_code_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/random/simple_philox.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
@@ -128,7 +129,7 @@ void TestWriteAppends(T first, U second) {
   string encoded_first_only = encoded;
   OCWriteToString<U>(&encoded, second);
   EXPECT_NE(encoded, encoded_first_only);
-  EXPECT_TRUE(StringPiece(encoded).starts_with(encoded_first_only));
+  EXPECT_TRUE(str_util::StartsWith(encoded, encoded_first_only));
 }
 
 template <typename T>
diff --git a/tensorflow/core/lib/strings/scanner.h b/tensorflow/core/lib/strings/scanner.h
index d3b63357ee..c82e771368 100644
--- a/tensorflow/core/lib/strings/scanner.h
+++ b/tensorflow/core/lib/strings/scanner.h
@@ -18,6 +18,7 @@ limitations under the License.
 
 #include <string>
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/macros.h"
 
 namespace tensorflow {
@@ -75,14 +76,14 @@ class Scanner {
   // Consume the next s.size() characters of the input, if they match <s>. If
   // they don't match <s>, this is a no-op.
   Scanner& ZeroOrOneLiteral(StringPiece s) {
-    cur_.Consume(s);
+    str_util::ConsumePrefix(&cur_, s);
     return *this;
   }
 
   // Consume the next s.size() characters of the input, if they match <s>. If
   // they don't match <s>, then GetResult will ultimately return false.
   Scanner& OneLiteral(StringPiece s) {
-    if (!cur_.Consume(s)) {
+    if (!str_util::ConsumePrefix(&cur_, s)) {
       error_ = true;
     }
     return *this;
diff --git a/tensorflow/core/platform/cloud/BUILD b/tensorflow/core/platform/cloud/BUILD
index 3ee7be3c4e..be84316c48 100644
--- a/tensorflow/core/platform/cloud/BUILD
+++ b/tensorflow/core/platform/cloud/BUILD
@@ -85,6 +85,7 @@ cc_library(
         ":retrying_utils",
         ":time_util",
         "//tensorflow/core:framework_headers_lib",
+        "//tensorflow/core:lib",
         "//tensorflow/core:lib_internal",
         "@jsoncpp_git//:jsoncpp",
     ],
@@ -263,6 +264,7 @@ tf_cc_test(
     deps = [
         ":gcs_file_system",
         ":http_request_fake",
+        "//tensorflow/core:lib",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
     ],
diff --git a/tensorflow/core/platform/cloud/gcs_file_system.cc b/tensorflow/core/platform/cloud/gcs_file_system.cc
index 1691826483..3c0dc13d75 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system.cc
@@ -172,7 +172,7 @@ Status ParseGcsPath(StringPiece fname, bool empty_object_ok, string* bucket,
     return errors::InvalidArgument("GCS path doesn't contain a bucket name: ",
                                    fname);
   }
-  objectp.Consume("/");
+  str_util::ConsumePrefix(&objectp, "/");
   *object = objectp.ToString();
   if (!empty_object_ok && object->empty()) {
     return errors::InvalidArgument("GCS path doesn't contain an object name: ",
@@ -535,7 +535,8 @@ class GcsWritableFile : public WritableFile {
       *uploaded = 0;
     } else {
       StringPiece range_piece(received_range);
-      range_piece.Consume("bytes=");  // May or may not be present.
+      str_util::ConsumePrefix(&range_piece,
+                              "bytes=");  // May or may not be present.
       std::vector<int64> range_parts;
       if (!str_util::SplitAndParseAsInts(range_piece, '-', &range_parts) ||
           range_parts.size() != 2) {
@@ -1172,7 +1173,7 @@ Status GcsFileSystem::GetChildrenBounded(const string& dirname,
         // 'object_prefix', which is part of 'dirname', should be removed from
         // the beginning of 'name'.
         StringPiece relative_path(name);
-        if (!relative_path.Consume(object_prefix)) {
+        if (!str_util::ConsumePrefix(&relative_path, object_prefix)) {
           return errors::Internal(strings::StrCat(
               "Unexpected response: the returned file name ", name,
               " doesn't match the prefix ", object_prefix));
@@ -1201,7 +1202,7 @@ Status GcsFileSystem::GetChildrenBounded(const string& dirname,
         }
         const string& prefix_str = prefix.asString();
         StringPiece relative_path(prefix_str);
-        if (!relative_path.Consume(object_prefix)) {
+        if (!str_util::ConsumePrefix(&relative_path, object_prefix)) {
           return errors::Internal(
               "Unexpected response: the returned folder name ", prefix_str,
               " doesn't match the prefix ", object_prefix);
diff --git a/tensorflow/core/platform/cloud/gcs_file_system_test.cc b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
index 8516421614..2fbde9b6a7 100644
--- a/tensorflow/core/platform/cloud/gcs_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/gcs_file_system_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/platform/cloud/gcs_file_system.h"
 #include <fstream>
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/cloud/http_request_fake.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -584,8 +585,9 @@ TEST(GcsFileSystemTest, NewWritableFile_ResumeUploadAllAttemptsFail) {
   TF_EXPECT_OK(file->Append("content2"));
   const auto& status = file->Close();
   EXPECT_EQ(errors::Code::ABORTED, status.code());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("All 10 retry attempts failed. The last failure: "
+  EXPECT_TRUE(
+      str_util::StrContains(status.error_message(),
+                            "All 10 retry attempts failed. The last failure: "
                             "Unavailable: important HTTP error 503"))
       << status;
 }
@@ -641,13 +643,12 @@ TEST(GcsFileSystemTest, NewWritableFile_UploadReturns410) {
   const auto& status = file->Close();
   EXPECT_EQ(errors::Code::UNAVAILABLE, status.code());
   EXPECT_TRUE(
-      StringPiece(status.error_message())
-          .contains(
-              "Upload to gs://bucket/path/writeable.txt failed, caused by: "
-              "Not found: important HTTP error 410"))
+      str_util::StrContains(status.error_message(),
+                            "Upload to gs://bucket/path/writeable.txt failed, "
+                            "caused by: Not found: important HTTP error 410"))
       << status;
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("when uploading gs://bucket/path/writeable.txt"))
+  EXPECT_TRUE(str_util::StrContains(
+      status.error_message(), "when uploading gs://bucket/path/writeable.txt"))
       << status;
 }
 
diff --git a/tensorflow/core/platform/cloud/retrying_file_system_test.cc b/tensorflow/core/platform/cloud/retrying_file_system_test.cc
index d3f763bb3c..ee6886fef7 100644
--- a/tensorflow/core/platform/cloud/retrying_file_system_test.cc
+++ b/tensorflow/core/platform/cloud/retrying_file_system_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/platform/cloud/retrying_file_system.h"
 #include <fstream>
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 
 namespace tensorflow {
@@ -245,7 +246,7 @@ TEST(RetryingFileSystemTest, NewRandomAccessFile_AllRetriesFailed) {
   char scratch[10];
   const auto& status = random_access_file->Read(0, 10, &result, scratch);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -399,7 +400,7 @@ TEST(RetryingFileSystemTest, NewWritableFile_AllRetriesFailed) {
   // Use it and check the results.
   const auto& status = writable_file->Sync();
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -428,7 +429,7 @@ TEST(RetryingFileSystemTest, NewReadOnlyMemoryRegionFromFile_AllRetriesFailed) {
   const auto& status =
       fs.NewReadOnlyMemoryRegionFromFile("filename.txt", &result);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -454,7 +455,7 @@ TEST(RetryingFileSystemTest, GetChildren_AllRetriesFailed) {
   std::vector<string> result;
   const auto& status = fs.GetChildren("gs://path", &result);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -481,7 +482,7 @@ TEST(RetryingFileSystemTest, GetMatchingPaths_AllRetriesFailed) {
   std::vector<string> result;
   const auto& status = fs.GetMatchingPaths("gs://path/dir", &result);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -506,7 +507,7 @@ TEST(RetryingFileSystemTest, DeleteFile_AllRetriesFailed) {
   std::vector<string> result;
   const auto& status = fs.DeleteFile("gs://path/file.txt");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -531,7 +532,7 @@ TEST(RetryingFileSystemTest, CreateDir_AllRetriesFailed) {
   std::vector<string> result;
   const auto& status = fs.CreateDir("gs://path/newdir");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -556,7 +557,7 @@ TEST(RetryingFileSystemTest, DeleteDir_AllRetriesFailed) {
   std::vector<string> result;
   const auto& status = fs.DeleteDir("gs://path/dir");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -582,7 +583,7 @@ TEST(RetryingFileSystemTest, GetFileSize_AllRetriesFailed) {
   uint64 size;
   const auto& status = fs.GetFileSize("gs://path/file.txt", &size);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -605,7 +606,7 @@ TEST(RetryingFileSystemTest, RenameFile_AllRetriesFailed) {
 
   const auto& status = fs.RenameFile("old_name", "new_name");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -630,7 +631,7 @@ TEST(RetryingFileSystemTest, Stat_AllRetriesFailed) {
   FileStatistics stat;
   const auto& status = fs.Stat("file_name", &stat);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -642,7 +643,7 @@ TEST(RetryingFileSystemTest, FileExists_AllRetriesFailed) {
 
   const auto& status = fs.FileExists("file_name");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -677,7 +678,7 @@ TEST(RetryingFileSystemTest, IsDirectory_AllRetriesFailed) {
 
   const auto& status = fs.IsDirectory("gs://path/dir");
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
@@ -706,7 +707,7 @@ TEST(RetryingFileSystemTest, DeleteRecursively_AllRetriesFailed) {
   const auto& status =
       fs.DeleteRecursively("gs://path/dir", &undeleted_files, &undeleted_dirs);
   EXPECT_TRUE(
-      StringPiece(status.error_message()).contains("Retriable error #10"))
+      str_util::StrContains(status.error_message(), "Retriable error #10"))
       << status;
 }
 
diff --git a/tensorflow/core/platform/cloud/retrying_utils_test.cc b/tensorflow/core/platform/cloud/retrying_utils_test.cc
index 6eb340e094..1b6527618a 100644
--- a/tensorflow/core/platform/cloud/retrying_utils_test.cc
+++ b/tensorflow/core/platform/cloud/retrying_utils_test.cc
@@ -16,6 +16,7 @@ limitations under the License.
 #include "tensorflow/core/platform/cloud/retrying_utils.h"
 #include <fstream>
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/test.h"
 
@@ -31,10 +32,9 @@ TEST(RetryingUtilsTest, CallWithRetries_RetryDelays) {
 
   const auto& status = RetryingUtils::CallWithRetries(f, 500000L, sleep);
   EXPECT_EQ(errors::Code::ABORTED, status.code());
-  EXPECT_TRUE(StringPiece(status.error_message())
-                  .contains("All 10 retry attempts "
-                            "failed. The last failure: "
-                            "Unavailable: Failed."))
+  EXPECT_TRUE(str_util::StrContains(
+      status.error_message(),
+      "All 10 retry attempts failed. The last failure: Unavailable: Failed."))
       << status;
 
   EXPECT_EQ(10, requested_delays.size());
diff --git a/tensorflow/core/util/command_line_flags.cc b/tensorflow/core/util/command_line_flags.cc
index 3efc703faf..480ce94fca 100644
--- a/tensorflow/core/util/command_line_flags.cc
+++ b/tensorflow/core/util/command_line_flags.cc
@@ -17,6 +17,7 @@ limitations under the License.
 #include <vector>
 
 #include "tensorflow/core/lib/core/stringpiece.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/stringprintf.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -28,7 +29,9 @@ bool ParseStringFlag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                      const std::function<bool(string)>& hook,
                      bool* value_parsing_ok) {
   *value_parsing_ok = true;
-  if (arg.Consume("--") && arg.Consume(flag) && arg.Consume("=")) {
+  if (str_util::ConsumePrefix(&arg, "--") &&
+      str_util::ConsumePrefix(&arg, flag) &&
+      str_util::ConsumePrefix(&arg, "=")) {
     *value_parsing_ok = hook(arg.ToString());
     return true;
   }
@@ -40,7 +43,9 @@ bool ParseInt32Flag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                     const std::function<bool(int32)>& hook,
                     bool* value_parsing_ok) {
   *value_parsing_ok = true;
-  if (arg.Consume("--") && arg.Consume(flag) && arg.Consume("=")) {
+  if (str_util::ConsumePrefix(&arg, "--") &&
+      str_util::ConsumePrefix(&arg, flag) &&
+      str_util::ConsumePrefix(&arg, "=")) {
     char extra;
     int32 parsed_int32;
     if (sscanf(arg.data(), "%d%c", &parsed_int32, &extra) != 1) {
@@ -60,7 +65,9 @@ bool ParseInt64Flag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                     const std::function<bool(int64)>& hook,
                     bool* value_parsing_ok) {
   *value_parsing_ok = true;
-  if (arg.Consume("--") && arg.Consume(flag) && arg.Consume("=")) {
+  if (str_util::ConsumePrefix(&arg, "--") &&
+      str_util::ConsumePrefix(&arg, flag) &&
+      str_util::ConsumePrefix(&arg, "=")) {
     char extra;
     int64 parsed_int64;
     if (sscanf(arg.data(), "%lld%c", &parsed_int64, &extra) != 1) {
@@ -80,7 +87,8 @@ bool ParseBoolFlag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                    const std::function<bool(bool)>& hook,
                    bool* value_parsing_ok) {
   *value_parsing_ok = true;
-  if (arg.Consume("--") && arg.Consume(flag)) {
+  if (str_util::ConsumePrefix(&arg, "--") &&
+      str_util::ConsumePrefix(&arg, flag)) {
     if (arg.empty()) {
       *value_parsing_ok = hook(true);
       return true;
@@ -107,7 +115,9 @@ bool ParseFloatFlag(tensorflow::StringPiece arg, tensorflow::StringPiece flag,
                     const std::function<bool(float)>& hook,
                     bool* value_parsing_ok) {
   *value_parsing_ok = true;
-  if (arg.Consume("--") && arg.Consume(flag) && arg.Consume("=")) {
+  if (str_util::ConsumePrefix(&arg, "--") &&
+      str_util::ConsumePrefix(&arg, flag) &&
+      str_util::ConsumePrefix(&arg, "=")) {
     char extra;
     float parsed_float;
     if (sscanf(arg.data(), "%f%c", &parsed_float, &extra) != 1) {
diff --git a/tensorflow/core/util/device_name_utils_test.cc b/tensorflow/core/util/device_name_utils_test.cc
index c1bc0f3378..ff9c108f10 100644
--- a/tensorflow/core/util/device_name_utils_test.cc
+++ b/tensorflow/core/util/device_name_utils_test.cc
@@ -408,7 +408,7 @@ static void MergeDevNamesError(const string& name_a, const string& name_b,
   DeviceNameUtils::ParsedName target_a = Name(name_a);
   Status s = DeviceNameUtils::MergeDevNames(&target_a, Name(name_b));
   EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-  EXPECT_TRUE(StringPiece(s.error_message()).contains(expected_error_substr))
+  EXPECT_TRUE(str_util::StrContains(s.error_message(), expected_error_substr))
       << s;
 }
 
diff --git a/tensorflow/core/util/equal_graph_def.cc b/tensorflow/core/util/equal_graph_def.cc
index f1ec497a67..b87dce0dff 100644
--- a/tensorflow/core/util/equal_graph_def.cc
+++ b/tensorflow/core/util/equal_graph_def.cc
@@ -23,6 +23,7 @@ limitations under the License.
 #include "tensorflow/core/framework/node_def.pb.h"
 #include "tensorflow/core/framework/node_def_util.h"
 #include "tensorflow/core/lib/hash/hash.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/protobuf.h"
 
@@ -144,7 +145,7 @@ bool EqualNodeDef(const NodeDef& actual, const NodeDef& expected, string* diff,
 
   int first_control_input = actual.input_size();
   for (int i = 0; i < actual.input_size(); ++i) {
-    if (StringPiece(actual.input(i)).starts_with("^")) {
+    if (str_util::StartsWith(actual.input(i), "^")) {
       first_control_input = i;
       break;
     }
@@ -240,7 +241,7 @@ uint64 NodeDefHash(const NodeDef& ndef, const EqualGraphDefOptions& options) {
   // Normal inputs. Order important.
   int first_control_input = ndef.input_size();
   for (int i = 0; i < ndef.input_size(); ++i) {
-    if (StringPiece(ndef.input(i)).starts_with("^")) {
+    if (str_util::StartsWith(ndef.input(i), "^")) {
       first_control_input = i;
       break;
     }
diff --git a/tensorflow/core/util/memmapped_file_system.cc b/tensorflow/core/util/memmapped_file_system.cc
index ea0a381f4f..1fa6b8bec0 100644
--- a/tensorflow/core/util/memmapped_file_system.cc
+++ b/tensorflow/core/util/memmapped_file_system.cc
@@ -15,6 +15,7 @@ limitations under the License.
 #include "tensorflow/core/util/memmapped_file_system.h"
 
 #include "tensorflow/core/lib/core/errors.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/util/memmapped_file_system.pb.h"
 
@@ -242,7 +243,7 @@ Status MemmappedFileSystem::InitializeFromFile(Env* env,
 }
 
 bool MemmappedFileSystem::IsMemmappedPackageFilename(const string& filename) {
-  return StringPiece(filename).starts_with(kMemmappedPackagePrefix);
+  return str_util::StartsWith(filename, kMemmappedPackagePrefix);
 }
 
 namespace {
diff --git a/tensorflow/core/util/reporter_test.cc b/tensorflow/core/util/reporter_test.cc
index 575c27d4ef..90ea09876e 100644
--- a/tensorflow/core/util/reporter_test.cc
+++ b/tensorflow/core/util/reporter_test.cc
@@ -29,7 +29,7 @@ namespace {
 
 // Tests of all the error paths in log_reader.cc follow:
 static void ExpectHasSubstr(StringPiece s, StringPiece expected) {
-  EXPECT_TRUE(StringPiece(s).contains(expected))
+  EXPECT_TRUE(str_util::StrContains(s, expected))
       << s << " does not contain " << expected;
 }
 
diff --git a/tensorflow/core/util/tensor_slice_reader_test.cc b/tensorflow/core/util/tensor_slice_reader_test.cc
index 010cc36823..3c9590e488 100644
--- a/tensorflow/core/util/tensor_slice_reader_test.cc
+++ b/tensorflow/core/util/tensor_slice_reader_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/lib/strings/strcat.h"
 #include "tensorflow/core/platform/env.h"
 #include "tensorflow/core/platform/logging.h"
@@ -422,7 +423,7 @@ static void VersionTest(const VersionDef& versions, const string& error) {
   // Read it back in and verify that we get the expected error
   TensorSliceReader reader(path, OpenTableTensorSliceReader);
   EXPECT_TRUE(reader.status().code() == error::INVALID_ARGUMENT &&
-              StringPiece(reader.status().error_message()).starts_with(error))
+              str_util::StartsWith(reader.status().error_message(), error))
       << "Expected error starting with '" << errors::InvalidArgument(error)
       << "', got '" << reader.status() << "'";
 }
diff --git a/tensorflow/core/util/tensor_slice_writer_test.cc b/tensorflow/core/util/tensor_slice_writer_test.cc
index ff5bfd65ae..31397f11b6 100644
--- a/tensorflow/core/util/tensor_slice_writer_test.cc
+++ b/tensorflow/core/util/tensor_slice_writer_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/logging.h"
 #include "tensorflow/core/platform/protobuf.h"
 #include "tensorflow/core/platform/test.h"
@@ -333,8 +334,8 @@ TEST(TensorSliceWriteTest, SizeErrors) {
     const std::vector<int8> data(300000000, -1);
     Status s = writer.Add("test1", shape, slice, data.data());
     EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-    EXPECT_TRUE(StringPiece(s.error_message())
-                    .contains("Tensor slice is too large to serialize"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.error_message(), "Tensor slice is too large to serialize"));
   }
 
   // Add a large string tensor slice, which will fail.
@@ -344,8 +345,8 @@ TEST(TensorSliceWriteTest, SizeErrors) {
     const std::vector<string> data(256 * 1024, std::string(8192, 'f'));
     Status s = writer.Add("test2", shape, slice, data.data());
     EXPECT_EQ(s.code(), error::INVALID_ARGUMENT);
-    EXPECT_TRUE(StringPiece(s.error_message())
-                    .contains("Tensor slice is too large to serialize"));
+    EXPECT_TRUE(str_util::StrContains(
+        s.error_message(), "Tensor slice is too large to serialize"));
   }
 }
 
diff --git a/tensorflow/tools/graph_transforms/backports_test.cc b/tensorflow/tools/graph_transforms/backports_test.cc
index ab9a61afa7..80a954e062 100644
--- a/tensorflow/tools/graph_transforms/backports_test.cc
+++ b/tensorflow/tools/graph_transforms/backports_test.cc
@@ -20,6 +20,7 @@ limitations under the License.
 #include "tensorflow/cc/ops/standard_ops.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/public/session.h"
@@ -191,7 +192,7 @@ TEST(BackportTensorArrayV3Test, TestBackportTensorArrayV3Subtypes) {
     std::map<string, const NodeDef*> node_lookup;
     MapNamesToNodes(result, &node_lookup);
     ASSERT_EQ(1, node_lookup.count("v3_node"));
-    EXPECT_TRUE(StringPiece(node_lookup.at("v3_node")->op()).ends_with("V2"));
+    EXPECT_TRUE(str_util::EndsWith(node_lookup.at("v3_node")->op(), "V2"));
   }
 }
 
diff --git a/tensorflow/tools/graph_transforms/fold_constants_test.cc b/tensorflow/tools/graph_transforms/fold_constants_test.cc
index 6bfdfe43f5..a082399a87 100644
--- a/tensorflow/tools/graph_transforms/fold_constants_test.cc
+++ b/tensorflow/tools/graph_transforms/fold_constants_test.cc
@@ -24,6 +24,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_shape.pb.h"
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/public/session.h"
@@ -209,10 +210,10 @@ class ConstantFoldingTest : public ::testing::Test {
     for (const NodeDef& node : graph_def.node()) {
       const StringPiece name(node.name());
       const int occurrence_count = folded_node_map.count(node.name());
-      if (name.ends_with("expect_removed")) {
+      if (str_util::EndsWith(name, "expect_removed")) {
         EXPECT_EQ(0, occurrence_count) << "node.name()=" << node.name();
       }
-      if (name.ends_with("expect_remains")) {
+      if (str_util::EndsWith(name, "expect_remains")) {
         EXPECT_EQ(1, occurrence_count) << "node.name()=" << node.name();
       }
     }
diff --git a/tensorflow/tools/graph_transforms/freeze_requantization_ranges.cc b/tensorflow/tools/graph_transforms/freeze_requantization_ranges.cc
index 2436c7e4a2..f401723808 100644
--- a/tensorflow/tools/graph_transforms/freeze_requantization_ranges.cc
+++ b/tensorflow/tools/graph_transforms/freeze_requantization_ranges.cc
@@ -40,8 +40,8 @@ Status ExtractMinMaxRecords(const string& log_file_name,
   for (const string& file_line : file_lines) {
     // We expect to find a line with components separated by semicolons, so to
     // start make sure that the basic structure is in place/
-    StringPiece line(file_line);
-    if (!line.contains(print_suffix + ";" + requant_prefix)) {
+    if (!str_util::StrContains(file_line,
+                               print_suffix + ";" + requant_prefix)) {
       continue;
     }
     std::vector<string> line_parts = str_util::Split(file_line, ';');
@@ -53,8 +53,7 @@ Status ExtractMinMaxRecords(const string& log_file_name,
     bool min_max_found = false;
     int min_max_index;
     for (int i = 1; i < line_parts.size(); ++i) {
-      StringPiece line_part(line_parts[i]);
-      if (line_part.starts_with(requant_prefix)) {
+      if (str_util::StartsWith(line_parts[i], requant_prefix)) {
         min_max_found = true;
         min_max_index = i;
       }
@@ -90,7 +89,7 @@ Status ExtractMinMaxRecords(const string& log_file_name,
       continue;
     }
     StringPiece name_string = line_parts[min_max_index - 1];
-    if (!name_string.ends_with(print_suffix)) {
+    if (!str_util::EndsWith(name_string, print_suffix)) {
       continue;
     }
     string name =
diff --git a/tensorflow/tools/graph_transforms/insert_logging.cc b/tensorflow/tools/graph_transforms/insert_logging.cc
index e1ee2b420b..377665448c 100644
--- a/tensorflow/tools/graph_transforms/insert_logging.cc
+++ b/tensorflow/tools/graph_transforms/insert_logging.cc
@@ -19,6 +19,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/graph/subgraph.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -101,7 +102,7 @@ Status InsertLogging(const GraphDef& input_graph_def,
     const bool op_matches = (ops.count(node.op()) > 0);
     bool prefix_matches = false;
     for (const string& prefix : prefixes) {
-      if (StringPiece(node.name()).starts_with(prefix)) {
+      if (str_util::StartsWith(node.name(), prefix)) {
         prefix_matches = true;
       }
     }
diff --git a/tensorflow/tools/graph_transforms/sparsify_gather.cc b/tensorflow/tools/graph_transforms/sparsify_gather.cc
index 701e350fc3..cc82100148 100644
--- a/tensorflow/tools/graph_transforms/sparsify_gather.cc
+++ b/tensorflow/tools/graph_transforms/sparsify_gather.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/graph/graph_constructor.h"
 #include "tensorflow/core/graph/node_builder.h"
 #include "tensorflow/core/graph/subgraph.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/init_main.h"
 #include "tensorflow/core/public/session.h"
 #include "tensorflow/core/util/command_line_flags.h"
@@ -88,7 +89,7 @@ void CreateConstNode(const Tensor& tensor, const string& name,
 
 string GetMonolithicTensorKey(const string& tensor_slice_name) {
   std::vector<string> names = Split(tensor_slice_name, "/");
-  if (StringPiece(names[names.size() - 1]).starts_with("part_")) {
+  if (str_util::StartsWith(names[names.size() - 1], "part_")) {
     CHECK_GE(names.size(), 2);
     names.pop_back();
   }
@@ -102,8 +103,8 @@ Status ObtainTensorSlice(const GraphDef& input_graph_def,
   for (const auto& node : input_graph_def.node()) {
     std::vector<string> node_name_parts = Split(node.name(), "/");
     if (node_name_parts.size() == 2 &&
-        StringPiece(node_name_parts[0]).starts_with("save") &&
-        StringPiece(node_name_parts[1]).starts_with("Assign") &&
+        str_util::StartsWith(node_name_parts[0], "save") &&
+        str_util::StartsWith(node_name_parts[1], "Assign") &&
         node.input(0) == target_name) {
       restore_node_name = node.input(1);
       break;
diff --git a/tensorflow/tools/graph_transforms/transform_graph_test.cc b/tensorflow/tools/graph_transforms/transform_graph_test.cc
index bc2412fcbd..b276229aa4 100644
--- a/tensorflow/tools/graph_transforms/transform_graph_test.cc
+++ b/tensorflow/tools/graph_transforms/transform_graph_test.cc
@@ -22,6 +22,7 @@ limitations under the License.
 #include "tensorflow/core/framework/tensor_testutil.h"
 #include "tensorflow/core/lib/core/status_test_util.h"
 #include "tensorflow/core/lib/io/path.h"
+#include "tensorflow/core/lib/strings/str_util.h"
 #include "tensorflow/core/platform/test.h"
 #include "tensorflow/core/platform/test_benchmark.h"
 #include "tensorflow/core/public/session.h"
@@ -112,12 +113,11 @@ class TransformGraphTest : public ::testing::Test {
     graph_transforms::MapNamesToNodes(out_graph_def, &out_node_map);
 
     for (const NodeDef& node : out_graph_def.node()) {
-      const StringPiece name(node.name());
       const int occurrence_count = out_node_map.count(node.name());
-      if (name.ends_with("expect_removed")) {
+      if (str_util::EndsWith(node.name(), "expect_removed")) {
         EXPECT_EQ(0, occurrence_count) << "node.name()=" << node.name();
       }
-      if (name.ends_with("expect_remains")) {
+      if (str_util::EndsWith(node.name(), "expect_remains")) {
         EXPECT_EQ(1, occurrence_count) << "node.name()=" << node.name();
       }
     }
@@ -139,7 +139,7 @@ class TransformGraphTest : public ::testing::Test {
     Status no_such_status =
         TransformGraph({}, {}, {{"test_no_such_transform", {}}}, &graph_def);
     EXPECT_TRUE(
-        StringPiece(no_such_status.ToString()).contains("not recognized"));
+        str_util::StrContains(no_such_status.ToString(), "not recognized"));
   }
 
   void TestParseTransformParameters() {
diff --git a/tensorflow/tools/graph_transforms/transform_utils.cc b/tensorflow/tools/graph_transforms/transform_utils.cc
index 55f28a9e1d..367048965d 100644
--- a/tensorflow/tools/graph_transforms/transform_utils.cc
+++ b/tensorflow/tools/graph_transforms/transform_utils.cc
@@ -88,7 +88,7 @@ void NodeNamePartsFromInput(const string& input_name, string* prefix,
     *suffix = ":" + input_parts[1];
   }
   StringPiece node_name_piece(input_parts[0]);
-  if (node_name_piece.Consume("^")) {
+  if (str_util::ConsumePrefix(&node_name_piece, "^")) {
     *prefix = "^";
   } else {
     *prefix = "";
@@ -200,8 +200,7 @@ Status SortByExecutionOrder(const GraphDef& input_graph_def,
       // for merge only wait for one non-control input.
       int32 num_control_edges = 0;
       for (int i = 0; i < node_def.input_size(); ++i) {
-        StringPiece input_name(node_def.input(i));
-        if (input_name.starts_with("^")) {
+        if (str_util::StartsWith(node_def.input(i), "^")) {
           num_control_edges++;
         }
       }
@@ -504,7 +503,7 @@ Status RenameNodeInputs(const GraphDef& input_graph_def,
           const string& dest_name = input_to_rename.second;
           bool is_match;
           string match_name;
-          if (StringPiece(source_name).ends_with(":*")) {
+          if (str_util::EndsWith(source_name, ":*")) {
             is_match = true;
             string prefix;
             string unused_node_name;
-- 
GitLab


From 2a82ae63091ba39f84f0dac6d0333e2e0600f067 Mon Sep 17 00:00:00 2001
From: Adria Puigdomenech <adriap@google.com>
Date: Wed, 4 Apr 2018 13:02:27 -0700
Subject: [PATCH 913/960] Expose scatter_add for resource variables.

PiperOrigin-RevId: 191634030
---
 .../python_api/api_def_ScatterAdd.pbtxt       |  4 ++
 .../resource_variable_ops_test.py             |  6 +++
 tensorflow/python/ops/state_ops.py            | 52 +++++++++++++++++++
 3 files changed, 62 insertions(+)
 create mode 100644 tensorflow/core/api_def/python_api/api_def_ScatterAdd.pbtxt

diff --git a/tensorflow/core/api_def/python_api/api_def_ScatterAdd.pbtxt b/tensorflow/core/api_def/python_api/api_def_ScatterAdd.pbtxt
new file mode 100644
index 0000000000..4f5b6decf6
--- /dev/null
+++ b/tensorflow/core/api_def/python_api/api_def_ScatterAdd.pbtxt
@@ -0,0 +1,4 @@
+op {
+  graph_op_name: "ScatterAdd"
+  visibility: HIDDEN
+}
diff --git a/tensorflow/python/kernel_tests/resource_variable_ops_test.py b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
index c31d5a1f91..edc63264a3 100644
--- a/tensorflow/python/kernel_tests/resource_variable_ops_test.py
+++ b/tensorflow/python/kernel_tests/resource_variable_ops_test.py
@@ -802,6 +802,12 @@ class ResourceVariableOpsTest(test_util.TensorFlowTestCase):
       state_ops.scatter_update(v, [1], [3.0])
       self.assertAllEqual([1.0, 3.0], v.numpy())
 
+  def testScatterAddStateOps(self):
+    with context.eager_mode():
+      v = resource_variable_ops.ResourceVariable([1.0, 2.0], name="add")
+      state_ops.scatter_add(v, [1], [3])
+      self.assertAllEqual([1.0, 5.0], v.numpy())
+
   def testScatterUpdateCast(self):
     with context.eager_mode():
       v = resource_variable_ops.ResourceVariable([1.0, 2.0], name="update")
diff --git a/tensorflow/python/ops/state_ops.py b/tensorflow/python/ops/state_ops.py
index 01fc3182bc..f6a11ca625 100644
--- a/tensorflow/python/ops/state_ops.py
+++ b/tensorflow/python/ops/state_ops.py
@@ -423,3 +423,55 @@ def scatter_nd_update(ref, indices, updates, use_locking=True, name=None):
       ref.handle, indices, ops.convert_to_tensor(updates, dtype=ref.dtype),
       use_locking, name)]):
     return ref.read_value()
+
+
+@tf_export("scatter_add")
+def scatter_add(ref, indices, updates, use_locking=False, name=None):
+  # pylint: disable=line-too-long
+  r"""Adds sparse updates to the variable referenced by `resource`.
+
+  This operation computes
+
+  ```python
+      # Scalar indices
+      ref[indices, ...] += updates[...]
+
+      # Vector indices (for each i)
+      ref[indices[i], ...] += updates[i, ...]
+
+      # High rank indices (for each i, ..., j)
+      ref[indices[i, ..., j], ...] += updates[i, ..., j, ...]
+  ```
+
+  This operation outputs `ref` after the update is done.
+  This makes it easier to chain operations that need to use the updated value.
+  Duplicate entries are handled correctly: if multiple `indices` reference
+  the same location, their contributions add.
+
+  Requires `updates.shape = indices.shape + ref.shape[1:]`.
+
+  <div style="width:70%; margin:auto; margin-bottom:10px; margin-top:20px;">
+  <img style="width:100%" src='https://www.tensorflow.org/images/ScatterAdd.png' alt>
+  </div>
+
+  Args:
+    ref: A `Variable`.
+    indices: A `Tensor`. Must be one of the following types: `int32`, `int64`.
+      A tensor of indices into the first dimension of `ref`.
+    updates: A `Tensor`. Must have the same type as `ref`.
+      A tensor of updated values to store in `ref`.
+    use_locking: An optional `bool`. Defaults to `True`.
+      If True, the assignment will be protected by a lock;
+      otherwise the behavior is undefined, but may exhibit less contention.
+    name: A name for the operation (optional).
+
+  Returns:
+    Same as `ref`.  Returned as a convenience for operations that want
+    to use the updated values after the update is done.
+  """
+  if ref.dtype._is_ref_dtype:
+    return gen_state_ops.scatter_add(ref, indices, updates,
+                                     use_locking=use_locking, name=name)
+  return ref._lazy_read(gen_resource_variable_ops.resource_scatter_add(  # pylint: disable=protected-access
+      ref.handle, indices, ops.convert_to_tensor(updates, ref.dtype),
+      name=name))
-- 
GitLab


From ab4ba5e4ba85aa6cd0e5b3e430e80eef39174ffa Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 13:24:06 -0700
Subject: [PATCH 914/960] Fix up output array min/max post-quantization if the
 range was overridden.

PiperOrigin-RevId: 191637143
---
 .../graph_transformations/hardcode_min_max.cc |  7 ++-
 .../toco/graph_transformations/quantize.cc    | 49 ++++++++++++++++++-
 2 files changed, 52 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc b/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
index 7c97ef0d31..23c9e3246b 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/hardcode_min_max.cc
@@ -223,8 +223,11 @@ bool PropagateMinMaxAmongArrays(Model* model,
     if (array.minmax) {
       CHECK(*array.minmax == *reference_minmax)
           << "Both the following arrays have minmax, and they disagree: "
-          << reference_array_name << " and " << array_name
-          << ". Expected that either only one of them would have minmax, or at "
+          << reference_array_name << " (" << reference_minmax->min << ","
+          << reference_minmax->max << ") and " << array_name << " ("
+          << array.minmax->min << "," << array.minmax->max
+          << "). Expected that either only one of them would have minmax, or "
+             "at "
              "least that they would agree.";
     } else {
       array.GetOrCreateMinMax() = *reference_minmax;
diff --git a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
index 9fcc95e1fe..7784558b22 100644
--- a/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
+++ b/tensorflow/contrib/lite/toco/graph_transformations/quantize.cc
@@ -472,6 +472,44 @@ bool ChooseQuantizationForOperatorOutput(
 
   return true;
 }
+
+// Fixes array minmax info to match the quantization parameters.
+// This is required for when quantization parameters change for an array during
+// quantization (such as ChooseQuantizationForOperatorOutput).
+void FixMinMaxPostQuantization(ArrayDataType quantized_data_type,
+                               const QuantizationParams& quantization_params,
+                               MinMax* minmax) {
+  double qmin, qmax;
+  switch (quantized_data_type) {
+    case ArrayDataType::kUint8:
+      qmin = 0;
+      qmax = 255;
+      break;
+    case ArrayDataType::kInt16:
+      qmin = -32768;
+      qmax = 32767;
+      break;
+    default:
+      // No update required.
+      return;
+  }
+
+  // Compute new minmax values.
+  double min =
+      (qmin - quantization_params.zero_point) * quantization_params.scale;
+  double max =
+      (qmax - quantization_params.zero_point) * quantization_params.scale;
+
+  // If we are close to the existing minmax values don't bother changing them.
+  // This prevents propagating small floating point precision errors.
+  constexpr double kMinMaxThreshold = 1e-5;
+  const double width = max - min;
+  if (std::abs(min - minmax->min) > kMinMaxThreshold * width ||
+      std::abs(max - minmax->max) > kMinMaxThreshold * width) {
+    minmax->min = min;
+    minmax->max = max;
+  }
+}
 }  // namespace
 
 bool Quantize::Run(Model* model, std::size_t op_index) {
@@ -618,12 +656,19 @@ bool Quantize::Run(Model* model, std::size_t op_index) {
                                             &quantization_params)) {
       changed = true;
       const auto& output = op.outputs[output_index];
+      auto& output_array = model->GetArray(output);
+
+      // Fix up the min/max information on the output array to match the chosen
+      // quantization parameters.
+      auto& output_minmax = output_array.GetMinMax();
+      FixMinMaxPostQuantization(quantized_data_type, quantization_params,
+                                &output_minmax);
+
       QuantizeArray(this, model, output, quantized_data_type,
                     quantization_params);
+
       const auto& dequantized_output =
           AvailableArrayName(*model, output + "_dequantized");
-      const auto& output_array = model->GetArray(output);
-      const auto& output_minmax = output_array.GetMinMax();
       auto& dequantized_output_array =
           model->GetOrCreateArray(dequantized_output);
       dequantized_output_array.data_type = ArrayDataType::kFloat;
-- 
GitLab


From 712fc6252228748a72bbc015be55bd20ba811cbb Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 13:38:20 -0700
Subject: [PATCH 915/960] Permit use of ArraysExtraInfo/constant_float_value
 when target model is quantized.

PiperOrigin-RevId: 191639289
---
 tensorflow/contrib/lite/toco/tooling_util.cc | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/lite/toco/tooling_util.cc b/tensorflow/contrib/lite/toco/tooling_util.cc
index 668cf51619..56fa8f4b69 100644
--- a/tensorflow/contrib/lite/toco/tooling_util.cc
+++ b/tensorflow/contrib/lite/toco/tooling_util.cc
@@ -2027,11 +2027,12 @@ void UseArraysExtraInfo(Model* model) {
     }
     if (entry.has_constant_float_value()) {
       CHECK(array.has_shape());
-      CHECK(array.data_type == ArrayDataType::kFloat);
-      auto& data = array.GetMutableBuffer<ArrayDataType::kFloat>().data;
-      data.resize(RequiredBufferSizeForShape(array.shape()));
-      for (float& f : data) {
-        f = entry.constant_float_value();
+      if (array.data_type == ArrayDataType::kFloat) {
+        auto& data = array.GetMutableBuffer<ArrayDataType::kFloat>().data;
+        data.resize(RequiredBufferSizeForShape(array.shape()));
+        for (float& f : data) {
+          f = entry.constant_float_value();
+        }
       }
     }
   }
-- 
GitLab


From 86d06ff37f9f1cb2bb5cd0c2d594286bbf023491 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 14:28:22 -0700
Subject: [PATCH 916/960] run evaluate nodes on parts of arithmetic optimizer
 tests.

PiperOrigin-RevId: 191647386
---
 tensorflow/core/grappler/optimizers/BUILD     |   5 +
 .../optimizers/arithmetic_optimizer_test.cc   | 143 ++++++++++++++++--
 .../optimizers/constant_folding_test.cc       |  18 +--
 .../core/grappler/utils/grappler_test.h       |   9 ++
 4 files changed, 146 insertions(+), 29 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 4ce3e73911..0c6549d940 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -274,6 +274,11 @@ tf_cuda_cc_test(
         ":constant_folding",
         ":model_pruner",
         "//tensorflow/cc:cc_ops",
+        "//tensorflow/cc:cc_ops_internal",
+        "//tensorflow/core:all_kernels",
+        "//tensorflow/core:core_cpu",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core:test",
         "//tensorflow/core:test_main",
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index ef3ed35fa6..48f1dd5aa1 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -156,7 +156,7 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   item.fetch = {"div"};
 
-  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, {});
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
   EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
@@ -164,7 +164,6 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   OptimizeTwice(&optimizer, &item, &output);
   NodeMap node_map(&output);
   EXPECT_EQ(2, output.node_size());
-
   const NodeDef* new_c1 = node_map.GetNode("c1");
   ASSERT_NE(new_c1, nullptr);
 
@@ -174,7 +173,7 @@ TEST_F(ArithmeticOptimizerTest, OpDedupping) {
   EXPECT_EQ("c1", new_div->input(0));
   EXPECT_EQ("c1", new_div->input(1));
 
-  auto tensors = EvaluateNodes(output, item.fetch, {});
+  auto tensors = EvaluateNodes(output, item.fetch);
   EXPECT_EQ(1, tensors.size());
   test::ExpectTensorNear<double>(tensors_expected[0], tensors[0], 1e-6);
 }
@@ -193,6 +192,11 @@ TEST_F(ArithmeticOptimizerTest, OpDeduppingAssertAndCheckNumerics) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
   item.fetch = {"div"};
+  Tensor bool_t(DT_BOOL, TensorShape({}));
+  bool_t.scalar<bool>().setConstant(true);
+  auto tensors_expected =
+      EvaluateNodes(item.graph, item.fetch, {{"Placeholder", bool_t}});
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -208,6 +212,10 @@ TEST_F(ArithmeticOptimizerTest, OpDeduppingAssertAndCheckNumerics) {
   EXPECT_EQ("check1", new_div->input(1));
   EXPECT_EQ("^assert1", new_div->input(2));
   EXPECT_EQ("^assert1", new_div->input(3));
+
+  auto tensors = EvaluateNodes(output, item.fetch, {{"Placeholder", bool_t}});
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<double>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) {
@@ -219,7 +227,9 @@ TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) {
   Output div1 = ops::Div(s.WithOpName("div1"), mul1, mul2);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-  item.fetch = {"div"};
+  item.fetch = {"div1"};
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -241,6 +251,10 @@ TEST_F(ArithmeticOptimizerTest, OpDedupCommutative) {
   EXPECT_EQ(2, new_div1->input_size());
   EXPECT_EQ("mul1", new_div1->input(0));
   EXPECT_EQ("mul1", new_div1->input(1));
+
+  auto tensors = EvaluateNodes(output, item.fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, MulToSquare) {
@@ -251,6 +265,9 @@ TEST_F(ArithmeticOptimizerTest, MulToSquare) {
   Output id = ops::Identity(s.WithOpName("id"), mul);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"id"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -265,6 +282,10 @@ TEST_F(ArithmeticOptimizerTest, MulToSquare) {
   EXPECT_EQ(2, output.node(4).input_size());
   EXPECT_EQ("c", output.node(4).input(0));
   EXPECT_EQ("^d", output.node(4).input(1));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsReal) {
@@ -277,6 +298,9 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsReal) {
   Output id = ops::Identity(s.WithOpName("id"), recip2);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"id"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -287,6 +311,10 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsReal) {
   EXPECT_EQ("c", output.node(1).input(0));
   EXPECT_EQ("c", output.node(3).input(0));
   EXPECT_EQ("c", output.node(5).input(0));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithChain) {
@@ -299,6 +327,9 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithChain) {
   Output id2 = ops::Identity(s.WithOpName("id2"), recip2);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"id2"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -312,6 +343,10 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithChain) {
   EXPECT_EQ(6, output.node_size());
   EXPECT_EQ("squeeze", output.node(5).input(0));
   EXPECT_EQ("c", output.node(2).input(0));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithControlChain) {
@@ -326,6 +361,10 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithControlChain) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
+  std::vector<string> fetch = {"id2"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
   ArithmeticOptimizer optimizer;
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
@@ -343,6 +382,10 @@ TEST_F(ArithmeticOptimizerTest, SimplifyInvolutionsWithControlChain) {
       EXPECT_EQ(original.input(j), optimized.input(j));
     }
   }
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, TrivialSumsSimple) {
@@ -354,6 +397,10 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimple) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
+  std::vector<string> fetch = {"id"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
   ArithmeticOptimizer optimizer;
   GraphDef output;
   OptimizeTwice(&optimizer, &item, &output);
@@ -375,6 +422,10 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimple) {
   const NodeDef* new_id = node_map.GetNode("id");
   ASSERT_NE(new_id, nullptr);
   EXPECT_EQ(OptimizedName("add_mul"), new_id->input(0));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, TrivialSumsSimpleWithControlDep) {
@@ -387,6 +438,10 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimpleWithControlDep) {
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
 
+  std::vector<string> fetch = {"id"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
+
   ArithmeticOptimizer optimizer;
   GraphDef output;
   OptimizeTwice(&optimizer, &item, &output);
@@ -409,6 +464,10 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsSimpleWithControlDep) {
   const NodeDef* new_id = node_map.GetNode("id");
   ASSERT_NE(new_id, nullptr);
   EXPECT_EQ(OptimizedName("add_mul"), new_id->input(0));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
@@ -424,6 +483,7 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
 
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+
   const std::vector<string> devices{
       "/device:CPU:0", "/device:GPU:0", "/device:CPU:0", "/device:GPU:1",
       "/device:CPU:0", "/device:CPU:0", "/device:CPU:0",
@@ -515,7 +575,8 @@ TEST_F(ArithmeticOptimizerTest, HoistFactor) {
       GrapplerItem item;
       item.fetch = {"id"};
       TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
+      auto tensors_expected = EvaluateNodes(item.graph, item.fetch);
+      EXPECT_EQ(1, tensors_expected.size());
       ArithmeticOptimizer optimizer;
       EnableOnlyHoistCommonFactor(&optimizer);
 
@@ -554,21 +615,26 @@ TEST_F(ArithmeticOptimizerTest, HoistFactor) {
         EXPECT_EQ("id", id_node->name());
         EXPECT_EQ(HoistMulName("add"), id_node->input(0));
       }
+      auto tensors = EvaluateNodes(output, item.fetch);
+      EXPECT_EQ(1, tensors.size());
+      test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
     }
   }
 }
 
 TEST_F(ArithmeticOptimizerTest, FuseConjAndTranspose) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  Output re = ops::Const(s.WithOpName("re"), {1.0, 2.0, 3.0, 4.0}, {2, 2});
-  Output im = ops::Const(s.WithOpName("im"), {5.0, 6.0, 7.0, 8.0}, {2, 2});
+  Output re = ops::Const(s.WithOpName("re"), {1.0f, 2.0f, 3.0f, 4.0f}, {2, 2});
+  Output im = ops::Const(s.WithOpName("im"), {5.0f, 6.0f, 7.0f, 8.0f}, {2, 2});
   Output z = ops::Complex(s.WithOpName("z"), re, im);
   Output perm = ops::Const(s.WithOpName("perm"), {1, 0}, {2});
   Output conj = ops::Conj(s.WithOpName("conj"), z);
   Output transp = ops::Transpose(s.WithOpName("trans"), conj, perm);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
+  std::vector<string> fetch = {"trans"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
   ArithmeticOptimizer optimizer;
   GraphDef output;
   OptimizeTwice(&optimizer, &item, &output);
@@ -582,12 +648,16 @@ TEST_F(ArithmeticOptimizerTest, FuseConjAndTranspose) {
   EXPECT_EQ("ConjugateTranspose", trans_fused_node->op());
   EXPECT_EQ("z", trans_fused_node->input(0));
   EXPECT_EQ("perm", trans_fused_node->input(1));
+
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<complex64>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ArithmeticOptimizerTest, FuseConjAndConjugateTranspose) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  Output re = ops::Const(s.WithOpName("re"), {1.0, 2.0, 3.0, 4.0}, {2, 2});
-  Output im = ops::Const(s.WithOpName("im"), {5.0, 6.0, 7.0, 8.0}, {2, 2});
+  Output re = ops::Const(s.WithOpName("re"), {1.0f, 2.0f, 3.0f, 4.0f}, {2, 2});
+  Output im = ops::Const(s.WithOpName("im"), {5.0f, 6.0f, 7.0f, 8.0f}, {2, 2});
   Output z = ops::Complex(s.WithOpName("z"), re, im);
   Output perm = ops::Const(s.WithOpName("perm"), {1, 0}, {2});
   Output conj = ops::Conj(s.WithOpName("conj"), z);
@@ -595,6 +665,9 @@ TEST_F(ArithmeticOptimizerTest, FuseConjAndConjugateTranspose) {
       ops::ConjugateTranspose(s.WithOpName("conjugate_trans"), conj, perm);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"conjugate_trans"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -608,18 +681,24 @@ TEST_F(ArithmeticOptimizerTest, FuseConjAndConjugateTranspose) {
   EXPECT_EQ("Transpose", conjugate_trans_fused_node->op());
   EXPECT_EQ("z", conjugate_trans_fused_node->input(0));
   EXPECT_EQ("perm", conjugate_trans_fused_node->input(1));
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<complex64>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ArithmeticOptimizerTest, FuseTransposeAndConj) {
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
-  Output re = ops::Const(s.WithOpName("re"), {1.0, 2.0, 3.0, 4.0}, {2, 2});
-  Output im = ops::Const(s.WithOpName("im"), {5.0, 6.0, 7.0, 8.0}, {2, 2});
+  Output re = ops::Const(s.WithOpName("re"), {1.0f, 2.0f, 3.0f, 4.0f}, {2, 2});
+  Output im = ops::Const(s.WithOpName("im"), {5.0f, 6.0f, 7.0f, 8.0f}, {2, 2});
   Output z = ops::Complex(s.WithOpName("z"), re, im);
   Output perm = ops::Const(s.WithOpName("perm"), {1, 0}, {2});
   Output trans = ops::Transpose(s.WithOpName("trans"), z, perm);
   Output conj = ops::Conj(s.WithOpName("conj"), trans);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"conj"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -633,6 +712,9 @@ TEST_F(ArithmeticOptimizerTest, FuseTransposeAndConj) {
   EXPECT_EQ("ConjugateTranspose", conj_fused_node->op());
   EXPECT_EQ("z", conj_fused_node->input(0));
   EXPECT_EQ("perm", conj_fused_node->input(1));
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<complex64>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ArithmeticOptimizerTest, FoldTransposeIntoMatMul) {
@@ -654,6 +736,9 @@ TEST_F(ArithmeticOptimizerTest, FoldTransposeIntoMatMul) {
     }
     GrapplerItem item;
     TF_CHECK_OK(s.ToGraphDef(&item.graph));
+    std::vector<string> fetch = {"matmul"};
+    auto tensors_expected = EvaluateNodes(item.graph, fetch);
+    EXPECT_EQ(1, tensors_expected.size());
 
     ArithmeticOptimizer optimizer;
     GraphDef output;
@@ -674,6 +759,9 @@ TEST_F(ArithmeticOptimizerTest, FoldTransposeIntoMatMul) {
       EXPECT_TRUE(matmul_fused_node->attr().at("transpose_a").b());
       EXPECT_TRUE(matmul_fused_node->attr().at("transpose_b").b());
     }
+    auto tensors = EvaluateNodes(output, fetch);
+    EXPECT_EQ(1, tensors.size());
+    test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
   }
 }
 
@@ -695,6 +783,9 @@ TEST_F(ArithmeticOptimizerTest, FoldConjugateTransposeIntoBatchMatMul) {
   Output matmul = ops::BatchMatMul(s.WithOpName("matmul"), trans_a, trans_b);
   GrapplerItem item;
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
+  std::vector<string> fetch = {"matmul"};
+  auto tensors_expected = EvaluateNodes(item.graph, fetch);
+  EXPECT_EQ(1, tensors_expected.size());
 
   ArithmeticOptimizer optimizer;
   GraphDef output;
@@ -707,6 +798,9 @@ TEST_F(ArithmeticOptimizerTest, FoldConjugateTransposeIntoBatchMatMul) {
   EXPECT_EQ("b", output.node(10).input(1));
   EXPECT_TRUE(output.node(10).attr().at("adj_x").b());
   EXPECT_TRUE(output.node(10).attr().at("adj_y").b());
+  auto tensors = EvaluateNodes(output, fetch);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<complex64>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, IdentityReshape) {
@@ -727,7 +821,10 @@ TEST_F(ArithmeticOptimizerTest, IdentityReshape) {
   GrapplerItem item;
   item.fetch = {"outputs"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
+  auto x_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({3, 3, 28, 28}));
+  auto tensors_expected =
+      EvaluateNodes(item.graph, item.fetch, {{"Placeholder", x_t}});
+  EXPECT_EQ(1, tensors_expected.size());
   GraphDef output;
   TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
 
@@ -735,6 +832,9 @@ TEST_F(ArithmeticOptimizerTest, IdentityReshape) {
   TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
 
   EXPECT_EQ(0, CountOpNodes(output, "Reshape"));
+  auto tensors = EvaluateNodes(output, item.fetch, {{"Placeholder", x_t}});
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, NotIdentityReshape) {
@@ -749,7 +849,10 @@ TEST_F(ArithmeticOptimizerTest, NotIdentityReshape) {
   GrapplerItem item;
   item.fetch = {"outputs"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
+  auto x_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({8, 3, 28, 28}));
+  item.feed = {{"Placeholder", x_t}};
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors_expected.size());
   GraphDef output;
   TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
 
@@ -757,6 +860,9 @@ TEST_F(ArithmeticOptimizerTest, NotIdentityReshape) {
   TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
 
   EXPECT_EQ(1, CountOpNodes(output, "Reshape"));
+  auto tensors = EvaluateNodes(output, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorNear<float>(tensors_expected[0], tensors[0], 1e-6);
 }
 
 TEST_F(ArithmeticOptimizerTest, NotIdentityReshapeTooManyUnknownDimSizes) {
@@ -769,7 +875,6 @@ TEST_F(ArithmeticOptimizerTest, NotIdentityReshapeTooManyUnknownDimSizes) {
   GrapplerItem item;
   item.fetch = {"outputs"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
   GraphDef output;
   TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
 
@@ -800,7 +905,10 @@ TEST_F(ArithmeticOptimizerTest, CombineReshapes) {
   GrapplerItem item;
   item.fetch = {"outputs"};
   TF_CHECK_OK(s.ToGraphDef(&item.graph));
-
+  auto x_t = GenerateRandomTensor<DT_INT8>(TensorShape({8, 3, 28, 28, 4}));
+  item.feed = {{"nchw_vect_c", x_t}};
+  auto tensors_expected = EvaluateNodes(item.graph, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors_expected.size());
   GraphDef output;
   TF_EXPECT_OK(ArithmeticOptimizer().Optimize(nullptr, item, &output));
 
@@ -808,6 +916,9 @@ TEST_F(ArithmeticOptimizerTest, CombineReshapes) {
   TF_EXPECT_OK(ModelPruner().Optimize(nullptr, item, &output));
 
   EXPECT_EQ(1, CountOpNodes(output, "Reshape"));
+  auto tensors = EvaluateNodes(output, item.fetch, item.feed);
+  EXPECT_EQ(1, tensors.size());
+  test::ExpectTensorEqual<int8>(tensors_expected[0], tensors[0]);
 }
 
 TEST_F(ArithmeticOptimizerTest, ReorderTransposeCast) {
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 7faa68a657..8d146637a6 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -83,14 +83,6 @@ class ConstantFoldingTest : public GrapplerTest {
   }
 };
 
-template <DataType DTYPE>
-Tensor GetRandomTensor(const TensorShape& shape) {
-  typedef typename EnumToDataType<DTYPE>::Type T;
-  Tensor tensor(DTYPE, shape);
-  tensor.flat<T>() = tensor.flat<T>().random();
-  return tensor;
-}
-
 TEST_F(ConstantFoldingTest, SimpleFolding) {
   // Build a simple graph with a few trivially prunable ops.
   tensorflow::Scope s = tensorflow::Scope::NewRootScope();
@@ -380,11 +372,11 @@ TEST_F(ConstantFoldingTest, NeutralElement) {
         EXPECT_EQ(2, t.tensor_shape().dim(1).size());
       }
     }
-    auto a_t = GetRandomTensor<DT_FLOAT>(TensorShape({3, 2}));
-    auto b_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 3}));
-    auto x_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
-    auto y_t = GetRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
-    auto bias_t = GetRandomTensor<DT_FLOAT>(TensorShape({2}));
+    auto a_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({3, 2}));
+    auto b_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({2, 3}));
+    auto x_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
+    auto y_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({2, 2}));
+    auto bias_t = GenerateRandomTensor<DT_FLOAT>(TensorShape({2}));
 
     auto tensors_expected = EvaluateNodes(
         item.graph, item.fetch,
diff --git a/tensorflow/core/grappler/utils/grappler_test.h b/tensorflow/core/grappler/utils/grappler_test.h
index 3bc7bea454..e1394b9c35 100644
--- a/tensorflow/core/grappler/utils/grappler_test.h
+++ b/tensorflow/core/grappler/utils/grappler_test.h
@@ -57,6 +57,15 @@ class GrapplerTest : public ::testing::Test {
   // Count nodes of the given op-type in a graph.
   int CountOpNodes(const GraphDef& graph, const string& op);
 
+  // Get a random tansor with given shape.
+  template <DataType DTYPE>
+  Tensor GenerateRandomTensor(const TensorShape& shape) const {
+    typedef typename EnumToDataType<DTYPE>::Type T;
+    Tensor tensor(DTYPE, shape);
+    tensor.flat<T>() = tensor.flat<T>().random();
+    return tensor;
+  }
+
  private:
   SessionOptions options_;
 };
-- 
GitLab


From c7d947ebea70d48e5fe0b6d268f1ada0709c42fe Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Wed, 4 Apr 2018 14:29:03 -0700
Subject: [PATCH 917/960] Adding a _create_state method to FeatureColumn so
 that we can decouple variable (more generally, state) creation from getting
 dense tensors etc. This lays the groundwork to create a Layer type interface
 (which is more eager friendly) on top of feature columns where variable
 creation is separated from getting the tensors out.

PiperOrigin-RevId: 191647517
---
 .../python/feature_column/feature_column.py   | 149 ++++++++++++++----
 .../feature_column/feature_column_test.py     | 136 ++++++++++++++++
 2 files changed, 256 insertions(+), 29 deletions(-)

diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index 92c6ff21c4..e116739bc0 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -1643,6 +1643,19 @@ class _FeatureColumn(object):
     """
     pass
 
+  def _create_state(self, weight_collections=None, creator=None):
+    """Returns an object that captures the state of the column.
+
+    Args:
+      weight_collections: Collections to add the variable to
+      creator: Variable creator method called, if provided.
+
+    Returns:
+      An object that encapsulates the state of the column. Can return None.
+    """
+    del weight_collections, creator  # Unused
+    return None
+
 
 class _DenseColumn(_FeatureColumn):
   """Represents a column which can be represented as `Tensor`.
@@ -1662,7 +1675,11 @@ class _DenseColumn(_FeatureColumn):
     pass
 
   @abc.abstractmethod
-  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
+  def _get_dense_tensor(self,
+                        inputs,
+                        weight_collections=None,
+                        trainable=None,
+                        state=None):
     """Returns a `Tensor`.
 
     The output of this function will be used by model-builder-functions. For
@@ -1680,6 +1697,9 @@ class _DenseColumn(_FeatureColumn):
         will be created) are added.
       trainable: If `True` also add variables to the graph collection
         `GraphKeys.TRAINABLE_VARIABLES` (see @{tf.Variable}).
+      state: An object encapsulating the state of the column. Columns that
+        create state using the _create_state method would have that state
+        passed in to this method.
 
     Returns:
       `Tensor` of shape [batch_size] + `_variable_shape`.
@@ -1687,13 +1707,13 @@ class _DenseColumn(_FeatureColumn):
     pass
 
 
-def _create_weighted_sum(
-    column,
-    builder,
-    units,
-    sparse_combiner,
-    weight_collections,
-    trainable):
+def _create_weighted_sum(column,
+                         builder,
+                         units,
+                         sparse_combiner,
+                         weight_collections,
+                         trainable,
+                         state=None):
   """Creates a weighted sum for a dense or sparse column for linear_model."""
   if isinstance(column, _CategoricalColumn):
     return _create_categorical_column_weighted_sum(
@@ -1709,16 +1729,28 @@ def _create_weighted_sum(
         builder=builder,
         units=units,
         weight_collections=weight_collections,
-        trainable=trainable)
+        trainable=trainable,
+        state=state)
 
 
-def _create_dense_column_weighted_sum(
-    column, builder, units, weight_collections, trainable):
+def _create_dense_column_weighted_sum(column,
+                                      builder,
+                                      units,
+                                      weight_collections,
+                                      trainable,
+                                      state=None):
   """Create a weighted sum of a dense column for linear_model."""
-  tensor = column._get_dense_tensor(  # pylint: disable=protected-access
-      builder,
-      weight_collections=weight_collections,
-      trainable=trainable)
+  if state is not None:
+    tensor = column._get_dense_tensor(  # pylint: disable=protected-access
+        builder,
+        weight_collections=weight_collections,
+        trainable=trainable,
+        state=state)
+  else:
+    tensor = column._get_dense_tensor(  # pylint: disable=protected-access
+        builder,
+        weight_collections=weight_collections,
+        trainable=trainable)
   num_elements = column._variable_shape.num_elements()  # pylint: disable=protected-access
   batch_size = array_ops.shape(tensor)[0]
   tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
@@ -2195,8 +2227,33 @@ class _EmbeddingColumn(
       self._shape = tensor_shape.vector(self.dimension)
     return self._shape
 
-  def _get_dense_tensor_internal(
-      self, inputs, weight_collections=None, trainable=None):
+  def _create_state(self, weight_collections=None, creator=None):
+    variables_map = {}
+    embedding_shape = (self.categorical_column._num_buckets, self.dimension)  # pylint: disable=protected-access
+    if creator is not None:
+      embedding_weights = creator(
+          name='embedding_weights',
+          shape=embedding_shape,
+          dtype=dtypes.float32,
+          initializer=self.initializer,
+          trainable=self.trainable)
+      ops.add_to_collections(weight_collections, embedding_weights)
+    else:
+      embedding_weights = variable_scope.get_variable(
+          name='embedding_weights',
+          shape=embedding_shape,
+          dtype=dtypes.float32,
+          initializer=self.initializer,
+          trainable=self.trainable,
+          collections=weight_collections)
+    variables_map['embedding_weights'] = embedding_weights
+    return variables_map
+
+  def _get_dense_tensor_internal(self,
+                                 inputs,
+                                 weight_collections=None,
+                                 trainable=None,
+                                 state=None):
     """Private method that follows the signature of _get_dense_tensor."""
     # Get sparse IDs and weights.
     sparse_tensors = self.categorical_column._get_sparse_tensors(  # pylint: disable=protected-access
@@ -2204,14 +2261,10 @@ class _EmbeddingColumn(
     sparse_ids = sparse_tensors.id_tensor
     sparse_weights = sparse_tensors.weight_tensor
 
-    embedding_shape = (self.categorical_column._num_buckets, self.dimension)  # pylint: disable=protected-access
-    embedding_weights = variable_scope.get_variable(
-        name='embedding_weights',
-        shape=embedding_shape,
-        dtype=dtypes.float32,
-        initializer=self.initializer,
-        trainable=self.trainable and trainable,
-        collections=weight_collections)
+    if state is None:
+      state = self._create_state(weight_collections)
+    embedding_weights = state['embedding_weights']
+
     if self.ckpt_to_load_from is not None:
       to_restore = embedding_weights
       if isinstance(to_restore, variables.PartitionedVariable):
@@ -2229,7 +2282,11 @@ class _EmbeddingColumn(
         name='%s_weights' % self.name,
         max_norm=self.max_norm)
 
-  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
+  def _get_dense_tensor(self,
+                        inputs,
+                        weight_collections=None,
+                        trainable=None,
+                        state=None):
     if isinstance(self.categorical_column, _SequenceCategoricalColumn):
       raise ValueError(
           'In embedding_column: {}. '
@@ -2242,8 +2299,10 @@ class _EmbeddingColumn(
               self.name, type(self.categorical_column),
               self.categorical_column))
     return self._get_dense_tensor_internal(
-        inputs=inputs, weight_collections=weight_collections,
-        trainable=trainable)
+        inputs=inputs,
+        weight_collections=weight_collections,
+        trainable=trainable,
+        state=state)
 
   def _get_sequence_dense_tensor(
       self, inputs, weight_collections=None, trainable=None):
@@ -2299,7 +2358,39 @@ class _SharedEmbeddingColumn(
       self._shape = tensor_shape.vector(self.dimension)
     return self._shape
 
-  def _get_dense_tensor(self, inputs, weight_collections=None, trainable=None):
+  def _create_state(self, weight_collections=None, creator=None):
+    variables_map = {}
+    shared_embedding_collection = ops.get_collection(
+        self.shared_embedding_collection_name)
+    if not shared_embedding_collection:
+      embedding_shape = (self.categorical_column._num_buckets, self.dimension)  # pylint: disable=protected-access
+      if creator is not None:
+        embedding_weights = creator(
+            name='embedding_weights',
+            shape=embedding_shape,
+            dtype=dtypes.float32,
+            initializer=self.initializer,
+            trainable=self.trainable)
+        ops.add_to_collections(weight_collections, embedding_weights)
+      else:
+        embedding_weights = variable_scope.get_variable(
+            name='embedding_weights',
+            shape=embedding_shape,
+            dtype=dtypes.float32,
+            initializer=self.initializer,
+            trainable=self.trainable,
+            collections=weight_collections)
+      ops.add_to_collection(self.shared_embedding_collection_name,
+                            embedding_weights)
+      variables_map['embedding_weights'] = embedding_weights
+
+    return variables_map
+
+  def _get_dense_tensor(self,
+                        inputs,
+                        weight_collections=None,
+                        trainable=None,
+                        state=None):
     # This method is called from a variable_scope with name _var_scope_name,
     # which is shared among all shared embeddings. Open a name_scope here, so
     # that the ops for different columns have distinct names.
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 6f366e7722..4006a76bb4 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -3733,6 +3733,70 @@ class EmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(embedding_values, global_vars[0].eval())
       self.assertAllEqual(expected_lookups, embedding_lookup.eval())
 
+  def test_get_dense_tensor_with_state(self):
+    # Inputs.
+    vocabulary_size = 3
+    sparse_input = sparse_tensor.SparseTensorValue(
+        # example 0, ids [2]
+        # example 1, ids [0, 1]
+        # example 2, ids []
+        # example 3, ids [1]
+        indices=((0, 0), (1, 0), (1, 4), (3, 0)),
+        values=(2, 0, 1, 1),
+        dense_shape=(4, 5))
+
+    # Embedding variable.
+    embedding_dimension = 2
+    embedding_values = (
+        (1., 2.),  # id 0
+        (3., 5.),  # id 1
+        (7., 11.)  # id 2
+    )
+    def _initializer(shape, dtype, partition_info):
+      self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
+      self.assertEqual(dtypes.float32, dtype)
+      self.assertIsNone(partition_info)
+      return embedding_values
+
+    # Expected lookup result, using combiner='mean'.
+    expected_lookups = (
+        # example 0, ids [2], embedding = [7, 11]
+        (7., 11.),
+        # example 1, ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5]
+        (2., 3.5),
+        # example 2, ids [], embedding = [0, 0]
+        (0., 0.),
+        # example 3, ids [1], embedding = [3, 5]
+        (3., 5.),
+    )
+
+    # Build columns.
+    categorical_column = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=vocabulary_size)
+    embedding_column = fc.embedding_column(
+        categorical_column, dimension=embedding_dimension,
+        initializer=_initializer)
+
+    # Create embedding_weights variable.
+    weight_collections = [ops.GraphKeys.GLOBAL_VARIABLES,
+                          ops.GraphKeys.MODEL_VARIABLES]
+    state = embedding_column._create_state(weight_collections)
+
+    # Provide sparse input and get dense result.
+    embedding_lookup = embedding_column._get_dense_tensor(
+        _LazyBuilder({
+            'aaa': sparse_input
+        }),
+        state=state)
+
+    # Assert expected embedding variable and lookups.
+    global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
+    self.assertItemsEqual(
+        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
+    with _initialized_session():
+      self.assertAllEqual(embedding_values, global_vars[0].eval())
+      self.assertAllEqual(expected_lookups, embedding_lookup.eval())
+
   def test_get_dense_tensor_3d(self):
     # Inputs.
     vocabulary_size = 4
@@ -4453,6 +4517,78 @@ class SharedEmbeddingColumnTest(test.TestCase):
       self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval())
       self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval())
 
+  def test_get_dense_tensor_with_state(self):
+    # Inputs.
+    vocabulary_size = 3
+    # -1 values are ignored.
+    input_a = np.array(
+        [[2, -1, -1],  # example 0, ids [2]
+         [0, 1, -1]])  # example 1, ids [0, 1]
+    input_b = np.array(
+        [[0, -1, -1],  # example 0, ids [0]
+         [-1, -1, -1]])  # example 1, ids []
+    input_features = {
+        'aaa': input_a,
+        'bbb': input_b
+    }
+
+    # Embedding variable.
+    embedding_dimension = 2
+    embedding_values = (
+        (1., 2.),  # id 0
+        (3., 5.),  # id 1
+        (7., 11.)  # id 2
+    )
+    def _initializer(shape, dtype, partition_info):
+      self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
+      self.assertEqual(dtypes.float32, dtype)
+      self.assertIsNone(partition_info)
+      return embedding_values
+
+    # Expected lookup result, using combiner='mean'.
+    expected_lookups_a = (
+        # example 0:
+        (7., 11.),  # ids [2], embedding = [7, 11]
+        # example 1:
+        (2., 3.5),  # ids [0, 1], embedding = mean([1, 2] + [3, 5]) = [2, 3.5]
+    )
+    expected_lookups_b = (
+        # example 0:
+        (1., 2.),  # ids [0], embedding = [1, 2]
+        # example 1:
+        (0., 0.),  # ids [], embedding = [0, 0]
+    )
+
+    # Build columns.
+    categorical_column_a = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=vocabulary_size)
+    categorical_column_b = fc.categorical_column_with_identity(
+        key='bbb', num_buckets=vocabulary_size)
+    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+        [categorical_column_a, categorical_column_b],
+        dimension=embedding_dimension, initializer=_initializer)
+
+    # Create state.
+    weight_collections = [ops.GraphKeys.GLOBAL_VARIABLES,
+                          ops.GraphKeys.MODEL_VARIABLES]
+    state = embedding_column_a._create_state(weight_collections)
+
+    # Provide sparse input and get dense result.
+    embedding_lookup_a = embedding_column_a._get_dense_tensor(
+        _LazyBuilder(input_features), state=state)
+    embedding_lookup_b = embedding_column_b._get_dense_tensor(
+        _LazyBuilder(input_features), state=state)
+
+    # Assert expected embedding variable and lookups.
+    global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
+    self.assertItemsEqual(
+        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
+    embedding_var = global_vars[0]
+    with _initialized_session():
+      self.assertAllEqual(embedding_values, embedding_var.eval())
+      self.assertAllEqual(expected_lookups_a, embedding_lookup_a.eval())
+      self.assertAllEqual(expected_lookups_b, embedding_lookup_b.eval())
+
   def test_get_dense_tensor_placeholder_inputs(self):
     # Inputs.
     vocabulary_size = 3
-- 
GitLab


From 733c7b38089120e6a7490c0c2d3c1006b9f91aa9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 14:31:29 -0700
Subject: [PATCH 918/960] Inline nested function calls.

PiperOrigin-RevId: 191647899
---
 .../grappler/optimizers/function_optimizer.cc | 240 +++++++++++-------
 .../optimizers/function_optimizer_test.cc     | 137 ++++++++--
 2 files changed, 273 insertions(+), 104 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/function_optimizer.cc b/tensorflow/core/grappler/optimizers/function_optimizer.cc
index 2a6b8a325f..f1da469a6c 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer.cc
@@ -32,16 +32,129 @@ limitations under the License.
 
 namespace tensorflow {
 namespace grappler {
+namespace {
+
+class FunctionInliningContext {
+ public:
+  explicit FunctionInliningContext(const GrapplerItem& item)
+      : library_(&item.graph.library()), functions_(InliningCandidates(item)) {}
+
+  const FunctionDefLibrary& Library() const { return *library_; }
+
+  bool HasInlinedFunctions() const { return !functions_.empty(); }
+
+  // Find inlining candidate by name. Return nullptr if not found.
+  const FunctionDef* FindInlinedFunction(const string& name) const {
+    auto it = functions_.find(name);
+    if (it != functions_.end()) {
+      return it->second;
+    } else {
+      return nullptr;
+    }
+  }
+
+ private:
+  std::unordered_map<string, const FunctionDef*> InliningCandidates(
+      const GrapplerItem& item) const {
+    std::unordered_map<string, const FunctionDef*> functions;
+    for (const FunctionDef& func : item.graph.library().function()) {
+      // Don't inline functions marked as noinline
+      if (func.attr().count("_noinline") != 0) {
+        continue;
+      }
+      // Don't touch anything marked XLA to prevent XLA failures further down
+      // the road.
+      if (func.attr().count("_XlaCompile") > 0 &&
+          func.attr().at("_XlaCompile").b()) {
+        continue;
+      }
+      // Can't create IdentityN nodes with no input or output: skip these
+      // functions for now.
+      if (func.signature().input_arg_size() == 0 ||
+          func.signature().output_arg_size() == 0) {
+        continue;
+      }
+      functions[func.signature().name()] = &func;
+    }
+    return functions;
+  }
+
+  const FunctionDefLibrary* library_;
+  std::unordered_map<string, const FunctionDef*> functions_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(FunctionInliningContext);
+};
+
+// Copy input/output argument type to the type_list. Return error if argument
+// type is not explicitly defined, and not specified in function attributes.
+Status CopyArgType(const NodeDef& func_node,
+                   const std::unordered_map<string, AttrValue>& func_attr,
+                   const string& arg_kind, const OpDef::ArgDef& arg,
+                   AttrValue::ListValue* type_list) {
+  if (arg.type() != DT_INVALID) {
+    type_list->add_type(arg.type());
+  } else {
+    auto it = func_attr.find(arg.type_attr());
+    if (it == func_attr.end() || it->second.type() == DT_INVALID) {
+      return errors::InvalidArgument(
+          "Invalid ", arg_kind, " argument ", arg.name(), " for function ",
+          func_node.op(), " instantiated by ", func_node.name());
+    }
+    type_list->add_type(it->second.type());
+  }
+  return Status::OK();
+}
+
+// Add an IdentityN op to hook the function inputs to: this ensures that
+// they're all evaluated before the evaluation of the function body starts.
+Status HookInlinedFunctionInputs(
+    const NodeDef& func_node, const FunctionDef& func,
+    const std::unordered_map<string, AttrValue>& func_attr, NodeDef* inputs) {
+  inputs->set_name(strings::StrCat(func_node.name(), "/", "inlined_inputs"));
+  inputs->set_op("IdentityN");
+  inputs->set_device(func_node.device());
+  *inputs->mutable_input() = func_node.input();
+  AttrValue::ListValue* type_list =
+      (*inputs->mutable_attr())["T"].mutable_list();
+  for (const OpDef::ArgDef& arg : func.signature().input_arg()) {
+    TF_RETURN_IF_ERROR(
+        CopyArgType(func_node, func_attr, "input", arg, type_list));
+  }
+  return Status::OK();
+}
+
+// Add an IdentityN op to hook the function outputs to: this ensures that the
+// function body is fully evaluated before its fanout gets scheduled.
+Status HookInlinedFunctionOutputs(
+    const NodeDef& func_node, const FunctionDef& func,
+    const std::unordered_map<string, AttrValue>& func_attr,
+    const gtl::ArraySlice<string> fetch, NodeDef* outputs) {
+  outputs->set_name(func_node.name());
+  outputs->set_op("IdentityN");
+  outputs->set_device(func_node.device());
+  AttrValue::ListValue* type_list =
+      (*outputs->mutable_attr())["T"].mutable_list();
+  for (int i = 0; i < func.signature().output_arg_size(); ++i) {
+    const OpDef::ArgDef& arg = func.signature().output_arg(i);
+    TF_RETURN_IF_ERROR(
+        CopyArgType(func_node, func_attr, "output", arg, type_list));
+    // Use the fetch names since they take into account the output mapping.
+    outputs->add_input(strings::StrCat(func_node.name(), "/", fetch[i]));
+  }
+  return Status::OK();
+}
+
+Status InlineFunction(const NodeDef& func_node, const FunctionDef& func,
+                      const FunctionInliningContext& ctx,
+                      GraphDef* optimized_graph) {
+  const std::unordered_map<string, AttrValue> func_attr(
+      func_node.attr().begin(), func_node.attr().end());
 
-Status InlineFunction(const NodeDef& node, const FunctionDef& func,
-                      const FunctionDefLibrary& library, GraphDef* graph) {
-  const std::unordered_map<string, AttrValue> attr(node.attr().begin(),
-                                                   node.attr().end());
   std::unique_ptr<GrapplerItem> item =
-      GrapplerItemFromFunctionDef(func, attr, library);
+      GrapplerItemFromFunctionDef(func, func_attr, ctx.Library());
   if (!item) {
-    return errors::InvalidArgument("Failed to inline function ", node.op(),
-                                   " instantiated by ", node.name());
+    return errors::InvalidArgument("Failed to inline function ", func_node.op(),
+                                   " instantiated by ", func_node.name());
   }
 
   std::unordered_map<string, int> input_nodes;
@@ -50,43 +163,25 @@ Status InlineFunction(const NodeDef& node, const FunctionDef& func,
     input_nodes[arg.name()] = i;
   }
 
-  // Add an IdentityN op to hook the function inputs to: this ensures that
-  // they're all evaluated before the evaluation of the function body starts.
-  NodeDef* func_inputs = graph->add_node();
-  func_inputs->set_name(strings::StrCat(node.name(), "/", "inlined_inputs"));
-  func_inputs->set_op("IdentityN");
-  func_inputs->set_device(node.device());
-  *func_inputs->mutable_input() = node.input();
-  AttrValue::ListValue* type_list =
-      (*func_inputs->mutable_attr())["T"].mutable_list();
-  for (const OpDef::ArgDef& arg : func.signature().input_arg()) {
-    if (arg.type() != DT_INVALID) {
-      type_list->add_type(arg.type());
-    } else {
-      auto it = attr.find(arg.type_attr());
-      if (it == attr.end()) {
-        return errors::InvalidArgument("Invalid input argument ", arg.name(),
-                                       " for function ", node.op(),
-                                       " instantiated by ", node.name());
-      }
-      type_list->add_type(it->second.type());
-    }
-  }
+  // Hook inlined function inputs to IdentityN node
+  NodeDef* func_inputs = optimized_graph->add_node();
+  TF_RETURN_IF_ERROR(
+      HookInlinedFunctionInputs(func_node, func, func_attr, func_inputs));
 
   for (NodeDef& func_body_node : *item->graph.mutable_node()) {
     if (input_nodes.find(func_body_node.name()) != input_nodes.end()) {
+      CHECK_EQ(0, func_body_node.input_size());
       // Turn input placeholders into identity nodes
       if (IsPlaceholder(func_body_node)) {
         func_body_node.set_op("Identity");
       }
-      CHECK_EQ(0, func_body_node.input_size());
       int input_id = input_nodes[func_body_node.name()];
       func_body_node.add_input(
           strings::StrCat(func_inputs->name(), ":", input_id));
     } else {
       // Update the input names if any.
       for (string& input : *func_body_node.mutable_input()) {
-        input = AddPrefixToNodeName(input, node.name());
+        input = AddPrefixToNodeName(input, /*prefix=*/func_node.name());
       }
       // If the node has no input, make hook it up to the func_inputs node to
       // ensure it runs in the same frame as the other nodes of the function
@@ -98,39 +193,29 @@ Status InlineFunction(const NodeDef& node, const FunctionDef& func,
 
     // Add the node name as a prefix to avoid collisions after inlining
     func_body_node.set_name(
-        strings::StrCat(node.name(), "/", func_body_node.name()));
+        strings::StrCat(func_node.name(), "/", func_body_node.name()));
 
     // Make sure the node is placed
-    func_body_node.set_device(node.device());
-
-    // Move the node to the main graph
-    graph->add_node()->Swap(&func_body_node);
-  }
-
-  // Add an IdentityN op to hook the function outputs to: this ensures that the
-  // function body is fully evaluated before its fanout gets scheduled.
-  NodeDef* func_outputs = graph->add_node();
-  func_outputs->set_name(node.name());
-  func_outputs->set_op("IdentityN");
-  func_outputs->set_device(node.device());
-  type_list = (*func_outputs->mutable_attr())["T"].mutable_list();
-  for (int i = 0; i < func.signature().output_arg_size(); ++i) {
-    const OpDef::ArgDef& arg = func.signature().output_arg(i);
-    if (arg.type() != DT_INVALID) {
-      type_list->add_type(arg.type());
+    func_body_node.set_device(func_node.device());
+
+    // Check if a body node is itself a function
+    const FunctionDef* func_body_node_func =
+        ctx.FindInlinedFunction(func_body_node.op());
+    if (func_body_node_func != nullptr) {
+      // Recursively inline function calls
+      TF_RETURN_IF_ERROR(InlineFunction(func_body_node, *func_body_node_func,
+                                        ctx, optimized_graph));
     } else {
-      auto it = attr.find(arg.type_attr());
-      if (it == attr.end()) {
-        return errors::InvalidArgument("Invalid output argument ", arg.name(),
-                                       " for function ", node.op(),
-                                       " instantiated by ", node.name());
-      }
-      type_list->add_type(it->second.type());
+      // Move the node to the main graph
+      optimized_graph->add_node()->Swap(&func_body_node);
     }
-    // Use the fetch names since they take into account the output mapping.
-    func_outputs->add_input(strings::StrCat(node.name(), "/", item->fetch[i]));
   }
 
+  // Hook inlined function outputs to IdentityN node
+  NodeDef* func_outputs = optimized_graph->add_node();
+  TF_RETURN_IF_ERROR(HookInlinedFunctionOutputs(func_node, func, func_attr,
+                                                item->fetch, func_outputs));
+
   return Status::OK();
 }
 
@@ -278,31 +363,14 @@ Status InlineSymbolicGradient(const NodeDef& node, SymbolicGradientEnv* env,
   return Status::OK();
 }
 
+}  // namespace
+
 Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
                                    GraphDef* optimized_graph) {
-  std::unordered_map<string, const FunctionDef*> functions;
-  for (const FunctionDef& func : item.graph.library().function()) {
-    // Don't inline functions marked as noinline
-    if (func.attr().count("_noinline") != 0) {
-      continue;
-    }
-    // Don't touch anything marked XLA to prevent XLA failures further down the
-    // road.
-    if (func.attr().count("_XlaCompile") > 0 &&
-        func.attr().at("_XlaCompile").b()) {
-      continue;
-    }
-    // Can't create IdentityN nodes with no input or output: skip these
-    // functions for now.
-    if (func.signature().input_arg_size() == 0 ||
-        func.signature().output_arg_size() == 0) {
-      continue;
-    }
-    functions[func.signature().name()] = &func;
-  }
+  FunctionInliningContext function_inlining_ctx(item);
 
-  // Nothing to do.
-  if (functions.empty()) {
+  // Nothing to do here.
+  if (!function_inlining_ctx.HasInlinedFunctions()) {
     *optimized_graph = item.graph;
     return Status::OK();
   }
@@ -315,12 +383,14 @@ Status FunctionOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
       TF_RETURN_IF_ERROR(InlineSymbolicGradient(node, &env, optimized_graph));
       continue;
     }
-    auto it = functions.find(node.op());
-    if (it == functions.end()) {
-      *optimized_graph->add_node() = node;
+
+    const FunctionDef* func =
+        function_inlining_ctx.FindInlinedFunction(node.op());
+    if (func != nullptr) {
+      TF_RETURN_IF_ERROR(
+          InlineFunction(node, *func, function_inlining_ctx, optimized_graph));
     } else {
-      TF_RETURN_IF_ERROR(InlineFunction(node, *it->second, item.graph.library(),
-                                        optimized_graph));
+      *optimized_graph->add_node() = node;
     }
   }
 
diff --git a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
index deb2fabded..c804d75756 100644
--- a/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/function_optimizer_test.cc
@@ -26,7 +26,22 @@ namespace tensorflow {
 namespace grappler {
 namespace {
 
-class FunctionOptimizerTest : public GrapplerTest {};
+constexpr char kDevice[] = "/device:CPU:0";
+
+class FunctionOptimizerTest : public GrapplerTest {
+ protected:
+  Tensor MakeScalarTensor(float value) {
+    Tensor tensor(DT_FLOAT, {});
+    tensor.scalar<float>()() = value;
+    return tensor;
+  }
+
+  Tensor MakeScalarTensor(int value) {
+    Tensor tensor(DT_INT32, {});
+    tensor.scalar<int>()() = value;
+    return tensor;
+  }
+};
 
 TEST_F(FunctionOptimizerTest, SimpleFunction) {
   // Build a graph to compute y = XTimesTwo(x)
@@ -94,9 +109,8 @@ TEST_F(FunctionOptimizerTest, SimpleFunction) {
   }
   EXPECT_EQ(7, count);
 
+  Tensor pi = MakeScalarTensor(3.14f);
   item.fetch = {"z"};
-  Tensor pi(DT_FLOAT, {});
-  pi.flat<float>()(0) = 3.14f;
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
   GrapplerItem optimized(item, std::move(output));
@@ -183,9 +197,8 @@ TEST_F(FunctionOptimizerTest, FixedTypeFunction) {
   }
   EXPECT_EQ(6, count);
 
+  Tensor pi = MakeScalarTensor(3.14f);
   item.fetch = {"z"};
-  Tensor pi(DT_FLOAT, {});
-  pi.flat<float>()(0) = 3.14f;
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
   GrapplerItem optimized(item, std::move(output));
@@ -268,9 +281,8 @@ TEST_F(FunctionOptimizerTest, FunctionWithOutputMapping) {
   }
   EXPECT_EQ(6, count);
 
+  Tensor pi = MakeScalarTensor(3.14f);
   item.fetch = {"z"};
-  Tensor pi(DT_FLOAT, {});
-  pi.flat<float>()(0) = 3.14f;
   item.feed.emplace_back("x", pi);
   auto tensors_expected = EvaluateFetchNodes(item);
   GrapplerItem optimized(item, std::move(output));
@@ -325,18 +337,11 @@ TEST_F(FunctionOptimizerTest, FunctionWithInputForwarding) {
   TF_EXPECT_OK(status);
 
   item.fetch = {"z0", "z1", "z2"};
-  Tensor in(DT_FLOAT, {});
-  in.flat<float>()(0) = 3.14f;
-  item.feed.emplace_back("x0", in);
-  in.flat<float>()(0) = 2.7f;
-  item.feed.emplace_back("x1", in);
-  in.flat<float>()(0) = 1.0f;
-  item.feed.emplace_back("x2", in);
-  in.flat<float>()(0) = -1.0f;
-  item.feed.emplace_back("x4", in);
-  Tensor in_int(DT_INT32, {});
-  in_int.flat<int>()(0) = 1234;
-  item.feed.emplace_back("x3", in_int);
+  item.feed.emplace_back("x0", MakeScalarTensor(3.14f));
+  item.feed.emplace_back("x1", MakeScalarTensor(2.7f));
+  item.feed.emplace_back("x2", MakeScalarTensor(1.0f));
+  item.feed.emplace_back("x4", MakeScalarTensor(-1.0f));
+  item.feed.emplace_back("x3", MakeScalarTensor(1234));
   auto tensors_expected = EvaluateFetchNodes(item);
   GrapplerItem optimized(item, std::move(output));
   auto tensors = EvaluateFetchNodes(optimized);
@@ -379,6 +384,100 @@ TEST_F(FunctionOptimizerTest, FunctionWithoutInput) {
   EXPECT_EQ(item.graph.DebugString(), output.DebugString());
 }
 
+TEST_F(FunctionOptimizerTest, InlineFunctionWithNestedFunctionCall) {
+  // Define square via function library:
+  //   MySquare(x) = MyMul(x, x)
+
+  FunctionDef mul_func = FunctionDefHelper::Create(
+      "MyMul", {"x:T", "y:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"output"}, "Mul", {"x", "y"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "output:z:0"}});
+
+  FunctionDef square_func = FunctionDefHelper::Create(
+      "MySquare", {"x:T"}, {"z:T"}, {"T: {float, double}"},
+      {{{"output"}, "MyMul", {"x", "x"}, {{"T", "$T"}}}},
+      /* Mapping between function returns and function node outputs. */
+      {{"z", "output:z:0"}});
+
+  GrapplerItem item;
+  item.graph = test::function::GDef(
+      {test::function::NDef("a", "Placeholder", {}, {{"dtype", DT_FLOAT}},
+                            kDevice),
+       test::function::NDef("square", "MySquare", {"a"}, {{"T", DT_FLOAT}},
+                            kDevice),
+       test::function::NDef("outputs", "Identity", {"square:0"},
+                            {{"T", DT_FLOAT}}, kDevice)},
+      // FunctionLib
+      {mul_func, square_func});
+
+  GraphDef output;
+  FunctionOptimizer optimizer(RewriterConfig::AGGRESSIVE);
+  TF_EXPECT_OK(optimizer.Optimize(nullptr, item, &output));
+
+  int count = 0;
+  for (const NodeDef& node : output.node()) {
+    if (node.name() == "square/inlined_inputs" && count++) {
+      EXPECT_EQ("IdentityN", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("a", node.input(0));
+    } else if (node.name() == "square/x" && count++) {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square/inlined_inputs:0", node.input(0));
+    } else if (node.name() == "square/output/inlined_inputs" && count++) {
+      EXPECT_EQ("IdentityN", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("square/x", node.input(0));
+      EXPECT_EQ("square/x", node.input(1));
+    } else if (node.name() == "square/output/x" && count++) {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square/output/inlined_inputs:0", node.input(0));
+    } else if (node.name() == "square/output/y" && count++) {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square/output/inlined_inputs:1", node.input(0));
+    } else if (node.name() == "square/output/output" && count++) {
+      EXPECT_EQ("Mul", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(2, node.input_size());
+      EXPECT_EQ("square/output/x", node.input(0));
+      EXPECT_EQ("square/output/y", node.input(1));
+    } else if (node.name() == "square/output" && count++) {
+      EXPECT_EQ("IdentityN", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square/output/output:0", node.input(0));
+    } else if (node.name() == "square" && count++) {
+      EXPECT_EQ("IdentityN", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square/output:0", node.input(0));
+    } else if (node.name() == "outputs" && count++) {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(kDevice, node.device());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("square:0", node.input(0));
+    }
+  }
+  EXPECT_EQ(9, count);
+
+  item.fetch = {"outputs"};
+  item.feed.emplace_back("a", MakeScalarTensor(2.0f));
+  auto tensors_expected = EvaluateFetchNodes(item);
+
+  GrapplerItem optimized(item, std::move(output));
+  auto tensors = EvaluateFetchNodes(optimized);
+
+  test::ExpectTensorEqual<float>(tensors_expected[0], tensors[0]);
+}
+
 TEST_F(FunctionOptimizerTest, SymbolicGradients) {
   tensorflow::Scope scope = tensorflow::Scope::NewRootScope();
 
-- 
GitLab


From f338cf1fce7be5d5d0aad0a41415a03b1f92817b Mon Sep 17 00:00:00 2001
From: Shanqing Cai <cais@google.com>
Date: Wed, 4 Apr 2018 14:39:40 -0700
Subject: [PATCH 919/960] Add link to tensorflow.js

PiperOrigin-RevId: 191649295
---
 tensorflow/docs_src/extend/index.md | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/tensorflow/docs_src/extend/index.md b/tensorflow/docs_src/extend/index.md
index bdff60b39e..1ab0340ad9 100644
--- a/tensorflow/docs_src/extend/index.md
+++ b/tensorflow/docs_src/extend/index.md
@@ -16,9 +16,10 @@ TensorFlow:
     for your own file and record formats.
 
 Python is currently the only language supported by TensorFlow's API stability
-promises.  However, TensorFlow also provides functionality in C++, Java, and Go,
+promises. However, TensorFlow also provides functionality in C++, Go, Java and
+[JavaScript](https://js.tensorflow.org),
 plus community support for [Haskell](https://github.com/tensorflow/haskell) and
-[Rust](https://github.com/tensorflow/rust).  If you'd like to create or
+[Rust](https://github.com/tensorflow/rust). If you'd like to create or
 develop TensorFlow features in a language other than these languages, read the
 following guide:
 
-- 
GitLab


From 2948be3af67b7ec124942654dc7f734eec346f55 Mon Sep 17 00:00:00 2001
From: Akshay Modi <nareshmodi@google.com>
Date: Wed, 4 Apr 2018 14:41:59 -0700
Subject: [PATCH 920/960] Check that the c_api module is not destroyed

PiperOrigin-RevId: 191649662
---
 tensorflow/python/framework/c_api_util.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/python/framework/c_api_util.py b/tensorflow/python/framework/c_api_util.py
index 6c522de452..4356a534b4 100644
--- a/tensorflow/python/framework/c_api_util.py
+++ b/tensorflow/python/framework/c_api_util.py
@@ -33,7 +33,7 @@ class ScopedTFStatus(object):
   def __del__(self):
     # Note: when we're destructing the global context (i.e when the process is
     # terminating) we can have already deleted other modules.
-    if c_api.TF_DeleteStatus is not None:
+    if c_api is not None and c_api.TF_DeleteStatus is not None:
       c_api.TF_DeleteStatus(self.status)
 
 
@@ -46,7 +46,7 @@ class ScopedTFGraph(object):
   def __del__(self):
     # Note: when we're destructing the global context (i.e when the process is
     # terminating) we can have already deleted other modules.
-    if c_api.TF_DeleteGraph is not None:
+    if c_api is not None and c_api.TF_DeleteGraph is not None:
       c_api.TF_DeleteGraph(self.graph)
 
 
@@ -59,7 +59,7 @@ class ScopedTFImportGraphDefOptions(object):
   def __del__(self):
     # Note: when we're destructing the global context (i.e when the process is
     # terminating) we can have already deleted other modules.
-    if c_api.TF_DeleteImportGraphDefOptions is not None:
+    if c_api is not None and c_api.TF_DeleteImportGraphDefOptions is not None:
       c_api.TF_DeleteImportGraphDefOptions(self.options)
 
 
-- 
GitLab


From e8882f768127b71e03efbf193a9c3152ab84802a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 14:45:05 -0700
Subject: [PATCH 921/960] GraphOptimizerStagePipeline to pass through multiple
 optimizer stages, skipping stages that return error.

PiperOrigin-RevId: 191650182
---
 .../optimizers/arithmetic_optimizer.cc        | 63 +++++++------------
 .../optimizers/graph_optimizer_stage.h        | 61 ++++++++++++++++++
 2 files changed, 83 insertions(+), 41 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 6e27259998..919f23fd98 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -1667,34 +1667,24 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
                                   &frame_map_);
   const ArithmeticOptimizerContext ctx_ext(&nodes_to_simplify);
 
-  std::vector<std::unique_ptr<ArithmeticOptimizerStage>> stages;
-
-  if (options_.combine_add_to_addn) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new AddOpsRewriteStage(ctx, ctx_ext)));
-  }
-  if (options_.hoist_common_factor_out_of_aggregation) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new HoistCommonFactorOutOfAggregation(ctx, ctx_ext)));
-  }
-  if (options_.remove_identity_transpose) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveIdentityTranspose(ctx, ctx_ext)));
-  }
-  if (options_.remove_redundant_bitcast) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveRedundantBitcastStage(ctx, ctx_ext)));
-  }
-  if (options_.remove_redundant_cast) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveRedundantCastStage(ctx, ctx_ext)));
-  }
-  if (options_.remove_negation) {
-    stages.push_back(std::unique_ptr<ArithmeticOptimizerStage>(
-        new RemoveNegationStage(ctx, ctx_ext)));
-  }
-
-  VLOG(1) << "Simplify arithmetic ops using " << stages.size()
+  // Stop pipeline after first stage returning non-empty simplified tensor name.
+  const auto stop = [](const string& result) { return !result.empty(); };
+  GraphOptimizerStagePipeline<string> pipeline(stop);
+
+  if (options_.combine_add_to_addn)
+    pipeline.AddStage<AddOpsRewriteStage>(ctx, ctx_ext);
+  if (options_.hoist_common_factor_out_of_aggregation)
+    pipeline.AddStage<HoistCommonFactorOutOfAggregation>(ctx, ctx_ext);
+  if (options_.remove_identity_transpose)
+    pipeline.AddStage<RemoveIdentityTranspose>(ctx, ctx_ext);
+  if (options_.remove_redundant_bitcast)
+    pipeline.AddStage<RemoveRedundantBitcastStage>(ctx, ctx_ext);
+  if (options_.remove_redundant_cast)
+    pipeline.AddStage<RemoveRedundantCastStage>(ctx, ctx_ext);
+  if (options_.remove_negation)
+    pipeline.AddStage<RemoveNegationStage>(ctx, ctx_ext);
+
+  VLOG(1) << "Simplify arithmetic ops using " << pipeline.NumStages()
           << " arithmetic optimization stages";
 
   while (!nodes_to_simplify.Empty()) {
@@ -1707,22 +1697,13 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
     }
 
     // if it was not simplified try to run it through all configured stages
-    if (simplified_tensor.empty()) {
-      for (auto& stage : stages) {
-        if (stage->IsSupported(node)) {
-          TF_RETURN_IF_ERROR(stage->TrySimplify(node, &simplified_tensor));
-          if (!simplified_tensor.empty()) {
-            break;
-          }
-        }
+    if (!stop(simplified_tensor)) {
+      bool optimized = pipeline.PassThroughAllStages(node, &simplified_tensor);
+      if (!optimized) {
+        continue;
       }
     }
 
-    // if it's still empty go to the next Node
-    if (simplified_tensor.empty()) {
-      continue;
-    }
-
     // re-wire consumers of an old node to the new one
     if (NodeName(simplified_tensor) != node->name()) {
       // Always consider simplified_tensor for further optimizations.
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
index be95c00d2d..8d3e965c57 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -117,6 +117,9 @@ class GraphOptimizerStage {
       : optimizer_name_(optimizer_name), stage_name_(stage_name), ctx_(ctx) {}
   virtual ~GraphOptimizerStage() = default;
 
+  const string& stage_name() const { return stage_name_; }
+  const string& optimizer_name() const { return optimizer_name_; }
+
   // Check if we should try to simplify node. Returning true doesn't
   // guarantee that node will be simplified.
   //
@@ -179,6 +182,64 @@ class GraphOptimizerStage {
   const GraphOptimizerContext ctx_;
 };
 
+template <typename Result>
+class GraphOptimizerStagePipeline {
+ public:
+  // Break predicate specifies if a pipeline should stop early, and not pass
+  // a node to the next registered optimizer stage, typically that should be the
+  // case when a stage successfully optimized a node, and it wants to yield
+  // control to the optimizer.
+  explicit GraphOptimizerStagePipeline(
+      const std::function<bool(const Result&)> break_predicate)
+      : break_predicate_(break_predicate) {}
+
+  // Add a stage to the pipeline. It should be called with the arguments for the
+  // stage constructor:
+  //
+  //   pipeline.AddStage<FooStage>(constructor_arg1, constructor_arg2);
+  //
+  // Returns a reference to the added stage.
+  template <typename T, typename... Args>
+  T& AddStage(Args&&... args) {
+    auto stage = new T(std::forward<Args>(args)...);
+    stages_.push_back(std::unique_ptr<T>(stage));
+    return *stage;
+  }
+
+  // Pass a node through all registered optimizer stages, until break predicate
+  // is true.
+  //
+  // Return true, if pipeline exited after a break predicate was evaluated as
+  // 'true', which typically means that a node was optimized by one of the
+  // registered stages.
+  //
+  // Return false, if node was not optimized by any of registered stages.
+  bool PassThroughAllStages(NodeDef* node, Result* result) {
+    for (auto& stage : stages_) {
+      if (stage->IsSupported(node)) {
+        const Status stage_status = stage->TrySimplify(node, result);
+        // Each stage must be "error safe" (just like exception safe). In
+        // case of any error it must leave optimized graph unmodified.
+        if (!stage_status.ok()) {
+          LOG(WARNING) << "Failed to run optimizer " << stage->optimizer_name()
+                       << ", stage " << stage->stage_name()
+                       << ". Error: " << stage_status.error_message();
+        }
+        if (break_predicate_(*result)) return true;
+      }
+    }
+    return false;
+  }
+
+  std::size_t NumStages() { return stages_.size(); }
+
+ private:
+  std::vector<std::unique_ptr<GraphOptimizerStage<Result>>> stages_;
+  std::function<bool(const Result&)> break_predicate_;
+
+  TF_DISALLOW_COPY_AND_ASSIGN(GraphOptimizerStagePipeline);
+};
+
 }  // end namespace grappler
 }  // end namespace tensorflow
 
-- 
GitLab


From a9cb0e19b9d96935c653f9cf89cebb6407564e5b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 14:55:16 -0700
Subject: [PATCH 922/960] Add quantized uint8 L2Normalization Kernel.

PiperOrigin-RevId: 191652174
---
 tensorflow/contrib/lite/kernels/l2norm.cc     | 22 ++++++++-
 .../contrib/lite/kernels/l2norm_test.cc       | 49 ++++++++++++++++---
 2 files changed, 62 insertions(+), 9 deletions(-)

diff --git a/tensorflow/contrib/lite/kernels/l2norm.cc b/tensorflow/contrib/lite/kernels/l2norm.cc
index ee8bfe56d9..e67f4e06f3 100644
--- a/tensorflow/contrib/lite/kernels/l2norm.cc
+++ b/tensorflow/contrib/lite/kernels/l2norm.cc
@@ -45,10 +45,15 @@ TfLiteStatus Prepare(TfLiteContext* context, TfLiteNode* node) {
 
   TF_LITE_ENSURE(context, NumDimensions(input) <= 4);
 
-  // TODO(ahentz): Our current implementations only support float32.
-  TF_LITE_ENSURE_EQ(context, output->type, kTfLiteFloat32);
+  TF_LITE_ENSURE(
+      context, output->type == kTfLiteFloat32 || output->type == kTfLiteUInt8);
   TF_LITE_ENSURE_EQ(context, input->type, output->type);
 
+  if (output->type == kTfLiteUInt8) {
+    TF_LITE_ENSURE_EQ(context, output->params.scale, (1. / 128.));
+    TF_LITE_ENSURE_EQ(context, output->params.zero_point, 128);
+  }
+
   // TODO(ahentz): For some reason our implementations don't support
   // activations.
   TF_LITE_ENSURE_EQ(context, params->activation, kTfLiteActNone);
@@ -75,6 +80,19 @@ TfLiteStatus Eval(TfLiteContext* context, TfLiteNode* node) {
       TF_LITE_L2NORM(optimized_ops);
     }
 #undef TF_LITE_L2NORM
+  } else if (output->type == kTfLiteUInt8) {
+#define TF_LITE_L2NORM(type)                                               \
+  type::L2Normalization(GetTensorData<uint8>(input), GetTensorDims(input), \
+                        input->params.zero_point,                          \
+                        GetTensorData<uint8>(output), GetTensorDims(output))
+
+    if (kernel_type == kReference) {
+      TF_LITE_L2NORM(reference_ops);
+    }
+    if (kernel_type == kGenericOptimized) {
+      TF_LITE_L2NORM(optimized_ops);
+    }
+#undef TF_LITE_L2NORM
   } else {
     context->ReportError(context, "Inputs and outputs not all float types.");
     return kTfLiteError;
diff --git a/tensorflow/contrib/lite/kernels/l2norm_test.cc b/tensorflow/contrib/lite/kernels/l2norm_test.cc
index 30e103f330..042314ccf5 100644
--- a/tensorflow/contrib/lite/kernels/l2norm_test.cc
+++ b/tensorflow/contrib/lite/kernels/l2norm_test.cc
@@ -25,10 +25,22 @@ using ::testing::ElementsAreArray;
 
 class L2NormOpModel : public SingleOpModel {
  public:
-  L2NormOpModel(std::initializer_list<int> input_shape,
-                ActivationFunctionType activation_type) {
-    input_ = AddInput(TensorType_FLOAT32);
-    output_ = AddOutput(TensorType_FLOAT32);
+  L2NormOpModel(const std::initializer_list<int> input_shape,
+                const TensorType tensor_type,
+                const ActivationFunctionType activation_type) {
+    TensorData data = TensorData{tensor_type};
+    if (tensor_type != TensorType_FLOAT32) {
+      data.min = -2.0;
+      data.max = 2.0;
+      data.scale = 2.0;
+      data.zero_point = 128;
+    }
+    input_ = AddInput(data);
+    if (tensor_type != TensorType_FLOAT32) {
+      data.min = -1.0;
+      data.max = 127.0 / 128.0;
+    }
+    output_ = AddOutput(data);
     SetBuiltinOp(BuiltinOperator_L2_NORMALIZATION, BuiltinOptions_L2NormOptions,
                  CreateL2NormOptions(builder_, activation_type).Union());
     BuildInterpreter({input_shape});
@@ -38,7 +50,17 @@ class L2NormOpModel : public SingleOpModel {
     PopulateTensor(input_, data);
   }
 
-  std::vector<float> GetOutput() { return ExtractVector<float>(output_); }
+  template <typename T>
+  std::vector<T> GetOutput() {
+    return ExtractVector<T>(output_);
+  }
+
+  std::vector<float> GetDequantizedOutput() {
+    return Dequantize<uint8_t>(ExtractVector<uint8_t>(output_),
+                               GetScale(output_), GetZeroPoint(output_));
+  }
+
+  int input() const { return input_; }
 
  private:
   int input_;
@@ -46,13 +68,26 @@ class L2NormOpModel : public SingleOpModel {
 };
 
 TEST(L2NormOpTest, SimpleTest) {
-  L2NormOpModel m({1, 1, 1, 6}, ActivationFunctionType_NONE);
+  L2NormOpModel m({1, 1, 1, 6}, TensorType_FLOAT32,
+                  ActivationFunctionType_NONE);
   m.SetInput({-1.1, 0.6, 0.7, 1.2, -0.7, 0.1});
   m.Invoke();
-  EXPECT_THAT(m.GetOutput(),
+  EXPECT_THAT(m.GetOutput<float>(),
               ElementsAreArray({-0.55, 0.3, 0.35, 0.6, -0.35, 0.05}));
 }
 
+TEST(L2NormOpTest, SimpleUint8Test) {
+  L2NormOpModel m({1, 1, 1, 6}, TensorType_UINT8, ActivationFunctionType_NONE);
+
+  m.QuantizeAndPopulate<uint8_t>(m.input(), {-1.1, 0.6, 0.7, 1.2, -0.7, 0.1});
+  m.Invoke();
+  EXPECT_THAT(m.GetOutput<uint8_t>(),
+              ElementsAreArray({58, 166, 173, 205, 83, 134}));
+  EXPECT_THAT(m.GetDequantizedOutput(),
+              ElementsAreArray(
+                  ArrayFloatNear({-0.55, 0.3, 0.35, 0.6, -0.35, 0.05}, 0.1)));
+}
+
 }  // namespace
 }  // namespace tflite
 
-- 
GitLab


From 91bf5524560c5bc0783b43717156c7dbb6f798f5 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 4 Apr 2018 15:06:08 -0700
Subject: [PATCH 923/960] Rename `distribute` to `train_distribute` parameter
 in `RunConfig` to clarify that its purpose is only for training.

PiperOrigin-RevId: 191654161
---
 tensorflow/contrib/distribute/README.md        |  2 +-
 .../python/estimator_integration_test.py       |  2 +-
 .../examples/simple_estimator_example.py       |  2 +-
 .../python/examples/simple_tfkeras_example.py  |  2 +-
 .../python/learn/estimators/run_config.py      |  2 +-
 tensorflow/python/estimator/estimator.py       |  2 +-
 tensorflow/python/estimator/run_config.py      | 18 +++++++++---------
 .../tensorflow.estimator.-run-config.pbtxt     | 10 +++++-----
 8 files changed, 20 insertions(+), 20 deletions(-)

diff --git a/tensorflow/contrib/distribute/README.md b/tensorflow/contrib/distribute/README.md
index 4af51bec1a..28483f4c88 100644
--- a/tensorflow/contrib/distribute/README.md
+++ b/tensorflow/contrib/distribute/README.md
@@ -77,7 +77,7 @@ parameter of `Estimator`.
 
 ```python
 distribution = tf.contrib.distribute.MirroredStrategy()
-config = tf.estimator.RunConfig(distribute=distribution)
+config = tf.estimator.RunConfig(train_distribute=distribution)
 classifier = tf.estimator.Estimator(model_fn=model_fn, config=config)
 classifier.train(input_fn=input_fn)
 ```
diff --git a/tensorflow/contrib/distribute/python/estimator_integration_test.py b/tensorflow/contrib/distribute/python/estimator_integration_test.py
index 9be186a724..2b49b8f4ef 100644
--- a/tensorflow/contrib/distribute/python/estimator_integration_test.py
+++ b/tensorflow/contrib/distribute/python/estimator_integration_test.py
@@ -95,7 +95,7 @@ class DNNLinearCombinedClassifierIntegrationTest(test.TestCase,
         # TODO(isaprykin): Work around the colocate_with error.
         dnn_optimizer=adagrad.AdagradOptimizer(0.001),
         linear_optimizer=adagrad.AdagradOptimizer(0.001),
-        config=run_config.RunConfig(distribute=distribution))
+        config=run_config.RunConfig(train_distribute=distribution))
 
     num_steps = 10
     estimator.train(train_input_fn, steps=num_steps)
diff --git a/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py b/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
index 5d6e02b4b9..00c25c7a24 100644
--- a/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
+++ b/tensorflow/contrib/distribute/python/examples/simple_estimator_example.py
@@ -59,7 +59,7 @@ def build_model_fn_optimizer():
 def main(_):
   distribution = tf.contrib.distribute.MirroredStrategy(
       ["/device:GPU:0", "/device:GPU:1"])
-  config = tf.estimator.RunConfig(distribute=distribution)
+  config = tf.estimator.RunConfig(train_distribute=distribution)
 
   def input_fn():
     features = tf.data.Dataset.from_tensors([[1.]]).repeat(10)
diff --git a/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py b/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py
index e714255f69..b87224251c 100644
--- a/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py
+++ b/tensorflow/contrib/distribute/python/examples/simple_tfkeras_example.py
@@ -41,7 +41,7 @@ def main(args):
 
   strategy = tf.contrib.distribute.MirroredStrategy(
       ['/device:GPU:0', '/device:GPU:1'])
-  config = tf.estimator.RunConfig(distribute=strategy)
+  config = tf.estimator.RunConfig(train_distribute=strategy)
   optimizer = tf.train.GradientDescentOptimizer(0.2)
 
   model = tf.keras.Sequential()
diff --git a/tensorflow/contrib/learn/python/learn/estimators/run_config.py b/tensorflow/contrib/learn/python/learn/estimators/run_config.py
index f3500bf56f..8c85c431be 100644
--- a/tensorflow/contrib/learn/python/learn/estimators/run_config.py
+++ b/tensorflow/contrib/learn/python/learn/estimators/run_config.py
@@ -298,7 +298,7 @@ class RunConfig(ClusterConfig, core_run_config.RunConfig):
     #   core_run_config.RunConfig.__init__(self)
     # so instead of breaking compatibility with that assumption, we
     # just manually initialize this field:
-    self._distribute = None
+    self._train_distribute = None
 
     gpu_options = config_pb2.GPUOptions(
         per_process_gpu_memory_fraction=gpu_memory_fraction)
diff --git a/tensorflow/python/estimator/estimator.py b/tensorflow/python/estimator/estimator.py
index ab69a093a2..4d3eff71ad 100644
--- a/tensorflow/python/estimator/estimator.py
+++ b/tensorflow/python/estimator/estimator.py
@@ -188,7 +188,7 @@ class Estimator(object):
       self._config = config
 
     # The distribute field contains an instance of DistributionStrategy.
-    self._distribution = self._config.distribute
+    self._distribution = self._config.train_distribute
 
     # Model directory.
     model_dir = compat_internal.path_to_str(model_dir)
diff --git a/tensorflow/python/estimator/run_config.py b/tensorflow/python/estimator/run_config.py
index 41415b89e9..f62c9cece6 100644
--- a/tensorflow/python/estimator/run_config.py
+++ b/tensorflow/python/estimator/run_config.py
@@ -44,7 +44,7 @@ _DEFAULT_REPLACEABLE_LIST = [
     'keep_checkpoint_max',
     'keep_checkpoint_every_n_hours',
     'log_step_count_steps',
-    'distribute'
+    'train_distribute'
 ]
 
 _SAVE_CKPT_ERR = (
@@ -302,7 +302,7 @@ class RunConfig(object):
                keep_checkpoint_max=5,
                keep_checkpoint_every_n_hours=10000,
                log_step_count_steps=100,
-               distribute=None):
+               train_distribute=None):
     """Constructs a RunConfig.
 
     All distributed training related properties `cluster_spec`, `is_chief`,
@@ -426,10 +426,10 @@ class RunConfig(object):
         the feature.
       log_step_count_steps: The frequency, in number of global steps, that the
         global step/sec and the loss will be logged during training.
-      distribute: an optional instance of
+      train_distribute: an optional instance of
         `tf.contrib.distribute.DistributionStrategy`. If specified,
-        then Estimator will distribute the user's model according to the policy
-        specified by that strategy.
+        then Estimator will distribute the user's model during training,
+        according to the policy specified by that strategy.
 
     Raises:
       ValueError: If both `save_checkpoints_steps` and `save_checkpoints_secs`
@@ -466,7 +466,7 @@ class RunConfig(object):
         keep_checkpoint_max=keep_checkpoint_max,
         keep_checkpoint_every_n_hours=keep_checkpoint_every_n_hours,
         log_step_count_steps=log_step_count_steps,
-        distribute=distribute)
+        train_distribute=train_distribute)
 
     self._init_distributed_setting_from_environment_var(tf_config)
 
@@ -678,10 +678,10 @@ class RunConfig(object):
     return self._service
 
   @property
-  def distribute(self):
+  def train_distribute(self):
     """Returns the optional `tf.contrib.distribute.DistributionStrategy` object.
     """
-    return self._distribute
+    return self._train_distribute
 
   def replace(self, **kwargs):
     """Returns a new instance of `RunConfig` replacing specified properties.
@@ -697,7 +697,7 @@ class RunConfig(object):
       - `keep_checkpoint_max`,
       - `keep_checkpoint_every_n_hours`,
       - `log_step_count_steps`,
-      - `distribute`.
+      - `train_distribute`.
 
     In addition, either `save_checkpoints_steps` or `save_checkpoints_secs`
     can be set (should not be both).
diff --git a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
index 759ff752b0..05e603efb7 100644
--- a/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
+++ b/tensorflow/tools/api/golden/tensorflow.estimator.-run-config.pbtxt
@@ -6,10 +6,6 @@ tf_class {
     name: "cluster_spec"
     mtype: "<type \'property\'>"
   }
-  member {
-    name: "distribute"
-    mtype: "<type \'property\'>"
-  }
   member {
     name: "evaluation_master"
     mtype: "<type \'property\'>"
@@ -82,9 +78,13 @@ tf_class {
     name: "tf_random_seed"
     mtype: "<type \'property\'>"
   }
+  member {
+    name: "train_distribute"
+    mtype: "<type \'property\'>"
+  }
   member_method {
     name: "__init__"
-    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\', \'distribute\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\', \'None\'], "
+    argspec: "args=[\'self\', \'model_dir\', \'tf_random_seed\', \'save_summary_steps\', \'save_checkpoints_steps\', \'save_checkpoints_secs\', \'session_config\', \'keep_checkpoint_max\', \'keep_checkpoint_every_n_hours\', \'log_step_count_steps\', \'train_distribute\'], varargs=None, keywords=None, defaults=[\'None\', \'None\', \'100\', \'<object object instance>\', \'<object object instance>\', \'None\', \'5\', \'10000\', \'100\', \'None\'], "
   }
   member_method {
     name: "replace"
-- 
GitLab


From bf8ad8277258bdf352ddd1df5200e61ba625f7a2 Mon Sep 17 00:00:00 2001
From: Rohan Jain <rohanj@google.com>
Date: Wed, 4 Apr 2018 15:14:19 -0700
Subject: [PATCH 924/960] Creates a LinearModel (inherits from
 keras.training.Model) that creates a linear model.

Had to modify the __call__ method in the base layer class so that it could work with feature style inputs in which case we lazily convert the inputs to tensors instead of providing tensors as inputs upfront.

PiperOrigin-RevId: 191655445
---
 tensorflow/python/feature_column/BUILD        |    1 +
 .../python/feature_column/feature_column.py   |  195 ++-
 .../feature_column/feature_column_test.py     | 1538 +++++++++++++++--
 tensorflow/python/layers/base.py              |    3 +-
 4 files changed, 1545 insertions(+), 192 deletions(-)

diff --git a/tensorflow/python/feature_column/BUILD b/tensorflow/python/feature_column/BUILD
index 219105d386..295d4ca094 100644
--- a/tensorflow/python/feature_column/BUILD
+++ b/tensorflow/python/feature_column/BUILD
@@ -43,6 +43,7 @@ py_library(
         "//tensorflow/python:util",
         "//tensorflow/python:variable_scope",
         "//tensorflow/python:variables",
+        "//tensorflow/python/keras",
         "@six_archive//:six",
     ],
 )
diff --git a/tensorflow/python/feature_column/feature_column.py b/tensorflow/python/feature_column/feature_column.py
index e116739bc0..3a315e5c2e 100644
--- a/tensorflow/python/feature_column/feature_column.py
+++ b/tensorflow/python/feature_column/feature_column.py
@@ -139,6 +139,8 @@ from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import ops
 from tensorflow.python.framework import sparse_tensor as sparse_tensor_lib
 from tensorflow.python.framework import tensor_shape
+from tensorflow.python.keras._impl.keras.engine import training
+from tensorflow.python.layers import base
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import check_ops
 from tensorflow.python.ops import control_flow_ops
@@ -460,6 +462,154 @@ def linear_model(features,
     return predictions
 
 
+class _FCLinearWrapper(base.Layer):
+  """Wraps a _FeatureColumn in a layer for use in a linear model.
+
+  See `linear_model` above.
+  """
+
+  def __init__(self,
+               feature_column,
+               units=1,
+               sparse_combiner='sum',
+               weight_collections=None,
+               trainable=True,
+               name=None,
+               **kwargs):
+    super(_FCLinearWrapper, self).__init__(
+        trainable=trainable, name=name, **kwargs)
+    self._feature_column = feature_column
+    self._units = units
+    self._sparse_combiner = sparse_combiner
+    self._weight_collections = weight_collections
+    self._state = {}
+
+  def build(self, _):
+    self._state = self._feature_column._create_state(  # pylint: disable=protected-access
+        self._weight_collections, self.add_variable)
+
+    if isinstance(self._feature_column, _CategoricalColumn):
+      weight = self.add_variable(
+          name='weights',
+          shape=(self._feature_column._num_buckets, self._units),  # pylint: disable=protected-access
+          initializer=init_ops.zeros_initializer(),
+          trainable=self.trainable)
+    else:
+      num_elements = self._feature_column._variable_shape.num_elements()  # pylint: disable=protected-access
+      weight = self.add_variable(
+          name='weights',
+          shape=[num_elements, self._units],
+          initializer=init_ops.zeros_initializer(),
+          trainable=self.trainable)
+    ops.add_to_collections(self._weight_collections, weight)
+    self._weight_var = weight
+    self.built = True
+
+  def call(self, builder):
+    weighted_sum = _create_weighted_sum(
+        column=self._feature_column,
+        builder=builder,
+        units=self._units,
+        sparse_combiner=self._sparse_combiner,
+        weight_collections=self._weight_collections,
+        trainable=self.trainable,
+        weight_var=self._weight_var,
+        state=self._state)
+    return weighted_sum
+
+
+class _BiasLayer(base.Layer):
+  """A layer for the bias term.
+  """
+
+  def __init__(self,
+               units=1,
+               trainable=True,
+               weight_collections=None,
+               name=None,
+               **kwargs):
+    super(_BiasLayer, self).__init__(trainable=trainable, name=name, **kwargs)
+    self._units = units
+    self._weight_collections = weight_collections
+
+  def build(self, _):
+    self._bias_variable = self.add_variable(
+        'bias_weights',
+        shape=[self._units],
+        initializer=init_ops.zeros_initializer(),
+        trainable=self.trainable)
+    ops.add_to_collections(self._weight_collections, self._bias_variable)
+    self.built = True
+
+  def call(self, _):
+    return self._bias_variable
+
+
+class _LinearModel(training.Model):
+  """Creates a linear model using feature columns.
+  """
+
+  def __init__(self,
+               feature_columns,
+               units=1,
+               sparse_combiner='sum',
+               weight_collections=None,
+               trainable=True,
+               name=None,
+               **kwargs):
+    super(_LinearModel, self).__init__(name=name, **kwargs)
+    self._feature_columns = _clean_feature_columns(feature_columns)
+    self._weight_collections = list(weight_collections or [])
+    if ops.GraphKeys.MODEL_VARIABLES not in self._weight_collections:
+      self._weight_collections.append(ops.GraphKeys.MODEL_VARIABLES)
+
+    column_layers = {}
+    for column in sorted(self._feature_columns, key=lambda x: x.name):
+      with variable_scope.variable_scope(
+          None, default_name=column._var_scope_name) as vs:  # pylint: disable=protected-access
+        column_name = vs.name
+      column_layer = _FCLinearWrapper(column, units, sparse_combiner,
+                                      self._weight_collections, trainable,
+                                      column_name, **kwargs)
+      column_layers[column_name] = column_layer
+    self._column_layers = self._add_layers(column_layers)
+    self._bias_layer = _BiasLayer(
+        units=units,
+        trainable=trainable,
+        weight_collections=self._weight_collections,
+        name='bias_layer',
+        **kwargs)
+
+  def call(self, features):
+    for column in self._feature_columns:
+      if not isinstance(column, (_DenseColumn, _CategoricalColumn)):
+        raise ValueError(
+            'Items of feature_columns must be either a '
+            '_DenseColumn or _CategoricalColumn. Given: {}'.format(column))
+    weighted_sums = []
+    ordered_columns = []
+    builder = _LazyBuilder(features)
+    for layer in sorted(self._column_layers.values(), key=lambda x: x.name):
+      ordered_columns.append(layer._feature_column)  # pylint: disable=protected-access
+      weighted_sum = layer(builder)
+      weighted_sums.append(weighted_sum)
+
+    _verify_static_batch_size_equality(weighted_sums, ordered_columns)
+    predictions_no_bias = math_ops.add_n(
+        weighted_sums, name='weighted_sum_no_bias')
+    predictions = nn_ops.bias_add(
+        predictions_no_bias, self._bias_layer(builder), name='weighted_sum')  # pylint: disable=not-callable
+    return predictions
+
+  def _add_layers(self, layers):
+    # "Magic" required for keras.Model classes to track all the variables in
+    # a list of layers.Layer objects.
+    # TODO(ashankar): Figure out API so user code doesn't have to do this.
+    for name, layer in layers.items():
+      setattr(self, 'layer-%s' % name, layer)
+    return layers
+
+
 def _transform_features(features, feature_columns):
   """Returns transformed features based on features columns passed in.
 
@@ -1713,6 +1863,7 @@ def _create_weighted_sum(column,
                          sparse_combiner,
                          weight_collections,
                          trainable,
+                         weight_var=None,
                          state=None):
   """Creates a weighted sum for a dense or sparse column for linear_model."""
   if isinstance(column, _CategoricalColumn):
@@ -1722,7 +1873,8 @@ def _create_weighted_sum(column,
         units=units,
         sparse_combiner=sparse_combiner,
         weight_collections=weight_collections,
-        trainable=trainable)
+        trainable=trainable,
+        weight_var=weight_var)
   else:
     return _create_dense_column_weighted_sum(
         column=column,
@@ -1730,6 +1882,7 @@ def _create_weighted_sum(column,
         units=units,
         weight_collections=weight_collections,
         trainable=trainable,
+        weight_var=weight_var,
         state=state)
 
 
@@ -1738,6 +1891,7 @@ def _create_dense_column_weighted_sum(column,
                                       units,
                                       weight_collections,
                                       trainable,
+                                      weight_var=None,
                                       state=None):
   """Create a weighted sum of a dense column for linear_model."""
   if state is not None:
@@ -1754,12 +1908,15 @@ def _create_dense_column_weighted_sum(column,
   num_elements = column._variable_shape.num_elements()  # pylint: disable=protected-access
   batch_size = array_ops.shape(tensor)[0]
   tensor = array_ops.reshape(tensor, shape=(batch_size, num_elements))
-  weight = variable_scope.get_variable(
-      name='weights',
-      shape=[num_elements, units],
-      initializer=init_ops.zeros_initializer(),
-      trainable=trainable,
-      collections=weight_collections)
+  if weight_var is not None:
+    weight = weight_var
+  else:
+    weight = variable_scope.get_variable(
+        name='weights',
+        shape=[num_elements, units],
+        initializer=init_ops.zeros_initializer(),
+        trainable=trainable,
+        collections=weight_collections)
   return math_ops.matmul(tensor, weight, name='weighted_sum')
 
 
@@ -1809,8 +1966,13 @@ class _CategoricalColumn(_FeatureColumn):
     pass
 
 
-def _create_categorical_column_weighted_sum(
-    column, builder, units, sparse_combiner, weight_collections, trainable):
+def _create_categorical_column_weighted_sum(column,
+                                            builder,
+                                            units,
+                                            sparse_combiner,
+                                            weight_collections,
+                                            trainable,
+                                            weight_var=None):
   """Create a weighted sum of a categorical column for linear_model."""
   sparse_tensors = column._get_sparse_tensors(  # pylint: disable=protected-access
       builder,
@@ -1824,12 +1986,15 @@ def _create_categorical_column_weighted_sum(
     weight_tensor = sparse_ops.sparse_reshape(
         weight_tensor, [array_ops.shape(weight_tensor)[0], -1])
 
-  weight = variable_scope.get_variable(
-      name='weights',
-      shape=(column._num_buckets, units),  # pylint: disable=protected-access
-      initializer=init_ops.zeros_initializer(),
-      trainable=trainable,
-      collections=weight_collections)
+  if weight_var is not None:
+    weight = weight_var
+  else:
+    weight = variable_scope.get_variable(
+        name='weights',
+        shape=(column._num_buckets, units),  # pylint: disable=protected-access
+        initializer=init_ops.zeros_initializer(),
+        trainable=trainable,
+        collections=weight_collections)
   return _safe_embedding_lookup_sparse(
       weight,
       id_tensor,
diff --git a/tensorflow/python/feature_column/feature_column_test.py b/tensorflow/python/feature_column/feature_column_test.py
index 4006a76bb4..07588af37e 100644
--- a/tensorflow/python/feature_column/feature_column_test.py
+++ b/tensorflow/python/feature_column/feature_column_test.py
@@ -34,6 +34,7 @@ from tensorflow.python.feature_column.feature_column import _CategoricalColumn
 from tensorflow.python.feature_column.feature_column import _DenseColumn
 from tensorflow.python.feature_column.feature_column import _FeatureColumn
 from tensorflow.python.feature_column.feature_column import _LazyBuilder
+from tensorflow.python.feature_column.feature_column import _LinearModel
 from tensorflow.python.feature_column.feature_column import _transform_features
 from tensorflow.python.feature_column.feature_column import InputLayer
 from tensorflow.python.framework import constant_op
@@ -339,6 +340,20 @@ class NumericColumnTest(test.TestCase):
         sess.run(price_var.assign([[10.]]))
         self.assertAllClose([[10.], [50.]], predictions.eval())
 
+  def test_keras_linear_model(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default():
+      features = {'price': [[1.], [5.]]}
+      predictions = get_keras_linear_model_predictions(features, [price])
+      bias = get_keras_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([[0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], predictions.eval())
+        sess.run(price_var.assign([[10.]]))
+        self.assertAllClose([[10.], [50.]], predictions.eval())
+
 
 class BucketizedColumnTest(test.TestCase):
 
@@ -561,6 +576,62 @@ class BucketizedColumnTest(test.TestCase):
         sess.run(bias.assign([1.]))
         self.assertAllClose([[81.], [141.]], predictions.eval())
 
+  def test_keras_linear_model_one_input_value(self):
+    """Tests _LinearModel for input with shape=[1]."""
+    price = fc.numeric_column('price', shape=[1])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    with ops.Graph().as_default():
+      features = {'price': [[-1.], [1.], [5.], [6.]]}
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [bucketized_price])
+      bias = get_keras_linear_model_bias()
+      bucketized_price_var = get_linear_model_column_var(bucketized_price)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        # One weight variable per bucket, all initialized to zero.
+        self.assertAllClose([[0.], [0.], [0.], [0.], [0.]],
+                            bucketized_price_var.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], predictions.eval())
+        sess.run(
+            bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.]]))
+        # price -1. is in the 0th bucket, whose weight is 10.
+        # price 1. is in the 1st bucket, whose weight is 20.
+        # price 5. is in the 3rd bucket, whose weight is 40.
+        # price 6. is in the 4th bucket, whose weight is 50.
+        self.assertAllClose([[10.], [20.], [40.], [50.]], predictions.eval())
+        sess.run(bias.assign([1.]))
+        self.assertAllClose([[11.], [21.], [41.], [51.]], predictions.eval())
+
+  def test_keras_linear_model_two_input_values(self):
+    """Tests _LinearModel for input with shape=[2]."""
+    price = fc.numeric_column('price', shape=[2])
+    bucketized_price = fc.bucketized_column(price, boundaries=[0, 2, 4, 6])
+    with ops.Graph().as_default():
+      features = {'price': [[-1., 1.], [5., 6.]]}
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [bucketized_price])
+      bias = get_keras_linear_model_bias()
+      bucketized_price_var = get_linear_model_column_var(bucketized_price)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        # One weight per bucket per input column, all initialized to zero.
+        self.assertAllClose(
+            [[0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.], [0.]],
+            bucketized_price_var.eval())
+        self.assertAllClose([[0.], [0.]], predictions.eval())
+        sess.run(
+            bucketized_price_var.assign([[10.], [20.], [30.], [40.], [50.],
+                                         [60.], [70.], [80.], [90.], [100.]]))
+        # 1st example:
+        #   price -1. is in the 0th bucket, whose weight is 10.
+        #   price 1. is in the 6th bucket, whose weight is 70.
+        # 2nd example:
+        #   price 5. is in the 3rd bucket, whose weight is 40.
+        #   price 6. is in the 9th bucket, whose weight is 100.
+        self.assertAllClose([[80.], [140.]], predictions.eval())
+        sess.run(bias.assign([1.]))
+        self.assertAllClose([[81.], [141.]], predictions.eval())
+
 
 class HashedCategoricalColumnTest(test.TestCase):
 
@@ -767,6 +838,28 @@ class HashedCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
         self.assertAllClose(((4.,), (6.,)), predictions.eval())
 
+  def test_keras_linear_model(self):
+    wire_column = fc.categorical_column_with_hash_bucket('wire', 4)
+    self.assertEqual(4, wire_column._num_buckets)
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          wire_column.name:
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=('marlo', 'skywalker', 'omar'),
+                  dense_shape=(2, 2))
+      }, (wire_column,))
+      bias = get_keras_linear_model_bias()
+      wire_var = get_linear_model_column_var(wire_column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
+        # 'marlo' -> 3: wire_var[3] = 4
+        # 'skywalker' -> 2, 'omar' -> 2: wire_var[2] + wire_var[2] = 3+3 = 6
+        self.assertAllClose(((4.,), (6.,)), predictions.eval())
+
 
 class CrossedColumnTest(test.TestCase):
 
@@ -1028,60 +1121,800 @@ class CrossedColumnTest(test.TestCase):
       def _num_buckets(self):
         return 5
 
-      def _transform_feature(self, inputs):
-        return (inputs.get(self.name),
-                inputs.get('{}_weights'.format(self.name)))
+      def _transform_feature(self, inputs):
+        return (inputs.get(self.name),
+                inputs.get('{}_weights'.format(self.name)))
+
+      def _get_sparse_tensors(self, inputs, weight_collections=None,
+                              trainable=None):
+        """Populates both id_tensor and weight_tensor."""
+        ids_and_weights = inputs.get(self)
+        return _CategoricalColumn.IdWeightPair(
+            id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
+
+    t = _TestColumnWithWeights()
+    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    with ops.Graph().as_default():
+      with self.assertRaisesRegexp(
+          ValueError,
+          'crossed_column does not support weight_tensor.*{}'.format(t.name)):
+        fc.linear_model({
+            t.name: sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=[0, 1, 2],
+                dense_shape=(2, 2)),
+            '{}_weights'.format(t.name): sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=[1., 10., 2.],
+                dense_shape=(2, 2)),
+            'c': sparse_tensor.SparseTensor(
+                indices=((0, 0), (1, 0), (1, 1)),
+                values=['cA', 'cB', 'cC'],
+                dense_shape=(2, 2)),
+        }, (crossed,))
+
+  def test_keras_linear_model(self):
+    """Tests _LinearModel.
+
+    Uses data from test_get_sparse_tesnsors_simple.
+    """
+    a = fc.numeric_column('a', dtype=dtypes.int32, shape=(2,))
+    b = fc.bucketized_column(a, boundaries=(0, 1))
+    crossed = fc.crossed_column([b, 'c'], hash_bucket_size=5, hash_key=5)
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          'a':
+              constant_op.constant(((-1., .5), (.5, 1.))),
+          'c':
+              sparse_tensor.SparseTensor(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=['cA', 'cB', 'cC'],
+                  dense_shape=(2, 2)),
+      }, (crossed,))
+      bias = get_keras_linear_model_bias()
+      crossed_var = get_linear_model_column_var(crossed)
+      with _initialized_session() as sess:
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,), (0.,)),
+                            crossed_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        sess.run(crossed_var.assign(((1.,), (2.,), (3.,), (4.,), (5.,))))
+        # Expected ids after cross = (1, 0, 1, 3, 4, 2)
+        self.assertAllClose(((3.,), (14.,)), predictions.eval())
+        sess.run(bias.assign((.1,)))
+        self.assertAllClose(((3.1,), (14.1,)), predictions.eval())
+
+  def test_keras_linear_model_with_weights(self):
+
+    class _TestColumnWithWeights(_CategoricalColumn):
+      """Produces sparse IDs and sparse weights."""
+
+      @property
+      def name(self):
+        return 'test_column'
+
+      @property
+      def _parse_example_spec(self):
+        return {
+            self.name:
+                parsing_ops.VarLenFeature(dtypes.int32),
+            '{}_weights'.format(self.name):
+                parsing_ops.VarLenFeature(dtypes.float32),
+        }
+
+      @property
+      def _num_buckets(self):
+        return 5
+
+      def _transform_feature(self, inputs):
+        return (inputs.get(self.name),
+                inputs.get('{}_weights'.format(self.name)))
+
+      def _get_sparse_tensors(self,
+                              inputs,
+                              weight_collections=None,
+                              trainable=None):
+        """Populates both id_tensor and weight_tensor."""
+        ids_and_weights = inputs.get(self)
+        return _CategoricalColumn.IdWeightPair(
+            id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
+
+    t = _TestColumnWithWeights()
+    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
+    with ops.Graph().as_default():
+      with self.assertRaisesRegexp(
+          ValueError,
+          'crossed_column does not support weight_tensor.*{}'.format(t.name)):
+        get_keras_linear_model_predictions({
+            t.name:
+                sparse_tensor.SparseTensor(
+                    indices=((0, 0), (1, 0), (1, 1)),
+                    values=[0, 1, 2],
+                    dense_shape=(2, 2)),
+            '{}_weights'.format(t.name):
+                sparse_tensor.SparseTensor(
+                    indices=((0, 0), (1, 0), (1, 1)),
+                    values=[1., 10., 2.],
+                    dense_shape=(2, 2)),
+            'c':
+                sparse_tensor.SparseTensor(
+                    indices=((0, 0), (1, 0), (1, 1)),
+                    values=['cA', 'cB', 'cC'],
+                    dense_shape=(2, 2)),
+        }, (crossed,))
+
+
+def get_linear_model_bias():
+  with variable_scope.variable_scope('linear_model', reuse=True):
+    return variable_scope.get_variable('bias_weights')
+
+
+def get_linear_model_column_var(column):
+  return ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES,
+                            'linear_model/' + column.name)[0]
+
+
+def get_keras_linear_model_bias():
+  with variable_scope.variable_scope('linear_model', reuse=True):
+    with variable_scope.variable_scope('bias_layer', reuse=True):
+      return variable_scope.get_variable('bias_weights')
+
+
+def get_keras_linear_model_predictions(features,
+                                       feature_columns,
+                                       units=1,
+                                       sparse_combiner='sum',
+                                       weight_collections=None,
+                                       trainable=True):
+  keras_linear_model = _LinearModel(
+      feature_columns,
+      units,
+      sparse_combiner,
+      weight_collections,
+      trainable,
+      name='linear_model')
+  return keras_linear_model(features)  # pylint: disable=not-callable
+
+
+@test_util.with_c_api
+class LinearModelTest(test.TestCase):
+
+  def test_raises_if_empty_feature_columns(self):
+    with self.assertRaisesRegexp(ValueError,
+                                 'feature_columns must not be empty'):
+      fc.linear_model(features={}, feature_columns=[])
+
+  def test_should_be_feature_column(self):
+    with self.assertRaisesRegexp(ValueError, 'must be a _FeatureColumn'):
+      fc.linear_model(features={'a': [[0]]}, feature_columns='NotSupported')
+
+  def test_should_be_dense_or_categorical_column(self):
+
+    class NotSupportedColumn(_FeatureColumn):
+
+      @property
+      def name(self):
+        return 'NotSupportedColumn'
+
+      def _transform_feature(self, cache):
+        pass
+
+      @property
+      def _parse_example_spec(self):
+        pass
+
+    with self.assertRaisesRegexp(
+        ValueError, 'must be either a _DenseColumn or _CategoricalColumn'):
+      fc.linear_model(
+          features={'a': [[0]]}, feature_columns=[NotSupportedColumn()])
+
+  def test_does_not_support_dict_columns(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Expected feature_columns to be iterable, found dict.'):
+      fc.linear_model(
+          features={'a': [[0]]}, feature_columns={'a': fc.numeric_column('a')})
+
+  def test_raises_if_duplicate_name(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Duplicate feature column name found for columns'):
+      fc.linear_model(
+          features={'a': [[0]]},
+          feature_columns=[fc.numeric_column('a'),
+                           fc.numeric_column('a')])
+
+  def test_dense_bias(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default():
+      features = {'price': [[1.], [5.]]}
+      predictions = fc.linear_model(features, [price])
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        sess.run(price_var.assign([[10.]]))
+        sess.run(bias.assign([5.]))
+        self.assertAllClose([[15.], [55.]], predictions.eval())
+
+  def test_sparse_bias(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default():
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
+          indices=[[0, 0], [1, 0], [1, 1]],
+          dense_shape=[2, 2])
+      features = {'wire_cast': wire_tensor}
+      predictions = fc.linear_model(features, [wire_cast])
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([[0.], [0.], [0.], [0.]], wire_cast_var.eval())
+        sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
+        sess.run(bias.assign([5.]))
+        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+
+  def test_dense_and_sparse_bias(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default():
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
+          indices=[[0, 0], [1, 0], [1, 1]],
+          dense_shape=[2, 2])
+      features = {'wire_cast': wire_tensor, 'price': [[1.], [5.]]}
+      predictions = fc.linear_model(features, [wire_cast, price])
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
+        sess.run(bias.assign([5.]))
+        sess.run(price_var.assign([[10.]]))
+        self.assertAllClose([[1015.], [10065.]], predictions.eval())
+
+  def test_dense_and_sparse_column(self):
+    """When the column is both dense and sparse, uses sparse tensors."""
+
+    class _DenseAndSparseColumn(_DenseColumn, _CategoricalColumn):
+
+      @property
+      def name(self):
+        return 'dense_and_sparse_column'
+
+      @property
+      def _parse_example_spec(self):
+        return {self.name: parsing_ops.VarLenFeature(self.dtype)}
+
+      def _transform_feature(self, inputs):
+        return inputs.get(self.name)
+
+      @property
+      def _variable_shape(self):
+        raise ValueError('Should not use this method.')
+
+      def _get_dense_tensor(self, inputs, weight_collections=None,
+                            trainable=None):
+        raise ValueError('Should not use this method.')
+
+      @property
+      def _num_buckets(self):
+        return 4
+
+      def _get_sparse_tensors(self, inputs, weight_collections=None,
+                              trainable=None):
+        sp_tensor = sparse_tensor.SparseTensor(
+            indices=[[0, 0], [1, 0], [1, 1]],
+            values=[2, 0, 3],
+            dense_shape=[2, 2])
+        return _CategoricalColumn.IdWeightPair(sp_tensor, None)
+
+    dense_and_sparse_column = _DenseAndSparseColumn()
+    with ops.Graph().as_default():
+      sp_tensor = sparse_tensor.SparseTensor(
+          values=['omar', 'stringer', 'marlo'],
+          indices=[[0, 0], [1, 0], [1, 1]],
+          dense_shape=[2, 2])
+      features = {dense_and_sparse_column.name: sp_tensor}
+      predictions = fc.linear_model(features, [dense_and_sparse_column])
+      bias = get_linear_model_bias()
+      dense_and_sparse_column_var = get_linear_model_column_var(
+          dense_and_sparse_column)
+      with _initialized_session() as sess:
+        sess.run(dense_and_sparse_column_var.assign(
+            [[10.], [100.], [1000.], [10000.]]))
+        sess.run(bias.assign([5.]))
+        self.assertAllClose([[1005.], [10015.]], predictions.eval())
+
+  def test_dense_multi_output(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default():
+      features = {'price': [[1.], [5.]]}
+      predictions = fc.linear_model(features, [price], units=3)
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose(np.zeros((3,)), bias.eval())
+        self.assertAllClose(np.zeros((1, 3)), price_var.eval())
+        sess.run(price_var.assign([[10., 100., 1000.]]))
+        sess.run(bias.assign([5., 6., 7.]))
+        self.assertAllClose([[15., 106., 1007.], [55., 506., 5007.]],
+                            predictions.eval())
+
+  def test_sparse_multi_output(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default():
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
+          indices=[[0, 0], [1, 0], [1, 1]],
+          dense_shape=[2, 2])
+      features = {'wire_cast': wire_tensor}
+      predictions = fc.linear_model(features, [wire_cast], units=3)
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      with _initialized_session() as sess:
+        self.assertAllClose(np.zeros((3,)), bias.eval())
+        self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
+        sess.run(
+            wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [
+                1000., 1100., 1200.
+            ], [10000., 11000., 12000.]]))
+        sess.run(bias.assign([5., 6., 7.]))
+        self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
+                            predictions.eval())
+
+  def test_dense_multi_dimension(self):
+    price = fc.numeric_column('price', shape=2)
+    with ops.Graph().as_default():
+      features = {'price': [[1., 2.], [5., 6.]]}
+      predictions = fc.linear_model(features, [price])
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose([[0.], [0.]], price_var.eval())
+        sess.run(price_var.assign([[10.], [100.]]))
+        self.assertAllClose([[210.], [650.]], predictions.eval())
+
+  def test_sparse_multi_rank(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default():
+      wire_tensor = array_ops.sparse_placeholder(dtypes.string)
+      wire_value = sparse_tensor.SparseTensorValue(
+          values=['omar', 'stringer', 'marlo', 'omar'],  # hashed = [2, 0, 3, 2]
+          indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 1]],
+          dense_shape=[2, 2, 2])
+      features = {'wire_cast': wire_tensor}
+      predictions = fc.linear_model(features, [wire_cast])
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      with _initialized_session() as sess:
+        self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
+        self.assertAllClose(
+            np.zeros((2, 1)),
+            predictions.eval(feed_dict={wire_tensor: wire_value}))
+        sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
+        self.assertAllClose(
+            [[1010.], [11000.]],
+            predictions.eval(feed_dict={wire_tensor: wire_value}))
+
+  def test_sparse_combiner(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default():
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar', 'stringer', 'marlo'],  # hashed to = [2, 0, 3]
+          indices=[[0, 0], [1, 0], [1, 1]],
+          dense_shape=[2, 2])
+      features = {'wire_cast': wire_tensor}
+      predictions = fc.linear_model(
+          features, [wire_cast], sparse_combiner='mean')
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      with _initialized_session() as sess:
+        sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
+        sess.run(bias.assign([5.]))
+        self.assertAllClose([[1005.], [5010.]], predictions.eval())
+
+  def test_dense_multi_dimension_multi_output(self):
+    price = fc.numeric_column('price', shape=2)
+    with ops.Graph().as_default():
+      features = {'price': [[1., 2.], [5., 6.]]}
+      predictions = fc.linear_model(features, [price], units=3)
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose(np.zeros((3,)), bias.eval())
+        self.assertAllClose(np.zeros((2, 3)), price_var.eval())
+        sess.run(price_var.assign([[1., 2., 3.], [10., 100., 1000.]]))
+        sess.run(bias.assign([2., 3., 4.]))
+        self.assertAllClose([[23., 205., 2007.], [67., 613., 6019.]],
+                            predictions.eval())
+
+  def test_raises_if_shape_mismatch(self):
+    price = fc.numeric_column('price', shape=2)
+    with ops.Graph().as_default():
+      features = {'price': [[1.], [5.]]}
+      if ops._USE_C_API:
+        with self.assertRaisesRegexp(
+            Exception,
+            r'Cannot reshape a tensor with 2 elements to shape \[2,2\]'):
+          predictions = fc.linear_model(features, [price])
+      else:
+        predictions = fc.linear_model(features, [price])
+        with _initialized_session():
+          with self.assertRaisesRegexp(Exception, 'requested shape has 4'):
+            predictions.eval()
+
+  def test_dense_reshaping(self):
+    price = fc.numeric_column('price', shape=[1, 2])
+    with ops.Graph().as_default():
+      features = {'price': [[[1., 2.]], [[5., 6.]]]}
+      predictions = fc.linear_model(features, [price])
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([[0.], [0.]], price_var.eval())
+        self.assertAllClose([[0.], [0.]], predictions.eval())
+        sess.run(price_var.assign([[10.], [100.]]))
+        self.assertAllClose([[210.], [650.]], predictions.eval())
+
+  def test_dense_multi_column(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[1., 2.], [5., 6.]],
+          'price2': [[3.], [4.]]
+      }
+      predictions = fc.linear_model(features, [price1, price2])
+      bias = get_linear_model_bias()
+      price1_var = get_linear_model_column_var(price1)
+      price2_var = get_linear_model_column_var(price2)
+      with _initialized_session() as sess:
+        self.assertAllClose([0.], bias.eval())
+        self.assertAllClose([[0.], [0.]], price1_var.eval())
+        self.assertAllClose([[0.]], price2_var.eval())
+        self.assertAllClose([[0.], [0.]], predictions.eval())
+        sess.run(price1_var.assign([[10.], [100.]]))
+        sess.run(price2_var.assign([[1000.]]))
+        sess.run(bias.assign([7.]))
+        self.assertAllClose([[3217.], [4657.]], predictions.eval())
+
+  def test_fills_cols_to_vars(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
+      cols_to_vars = {}
+      fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
+      bias = get_linear_model_bias()
+      price1_var = get_linear_model_column_var(price1)
+      price2_var = get_linear_model_column_var(price2)
+      self.assertAllEqual(cols_to_vars['bias'], [bias])
+      self.assertAllEqual(cols_to_vars[price1], [price1_var])
+      self.assertAllEqual(cols_to_vars[price2], [price2_var])
+
+  def test_fills_cols_to_vars_partitioned_variables(self):
+    price1 = fc.numeric_column('price1', shape=2)
+    price2 = fc.numeric_column('price2', shape=3)
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[1., 2.], [6., 7.]],
+          'price2': [[3., 4., 5.], [8., 9., 10.]]
+      }
+      cols_to_vars = {}
+      with variable_scope.variable_scope(
+          'linear',
+          partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)):
+        fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
+      with _initialized_session():
+        self.assertEqual([0.], cols_to_vars['bias'][0].eval())
+        # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
+        self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval())
+        # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
+        # a [1, 1] Variable.
+        self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval())
+        self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
+
+  def test_dense_collection(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default() as g:
+      features = {'price': [[1.], [5.]]}
+      fc.linear_model(features, [price], weight_collections=['my-vars'])
+      my_vars = g.get_collection('my-vars')
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      self.assertIn(bias, my_vars)
+      self.assertIn(price_var, my_vars)
+
+  def test_sparse_collection(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default() as g:
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
+      features = {'wire_cast': wire_tensor}
+      fc.linear_model(
+          features, [wire_cast], weight_collections=['my-vars'])
+      my_vars = g.get_collection('my-vars')
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      self.assertIn(bias, my_vars)
+      self.assertIn(wire_cast_var, my_vars)
+
+  def test_dense_trainable_default(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default() as g:
+      features = {'price': [[1.], [5.]]}
+      fc.linear_model(features, [price])
+      bias = get_linear_model_bias()
+      price_var = get_linear_model_column_var(price)
+      trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      self.assertIn(bias, trainable_vars)
+      self.assertIn(price_var, trainable_vars)
+
+  def test_sparse_trainable_default(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default() as g:
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
+      features = {'wire_cast': wire_tensor}
+      fc.linear_model(features, [wire_cast])
+      trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      bias = get_linear_model_bias()
+      wire_cast_var = get_linear_model_column_var(wire_cast)
+      self.assertIn(bias, trainable_vars)
+      self.assertIn(wire_cast_var, trainable_vars)
+
+  def test_dense_trainable_false(self):
+    price = fc.numeric_column('price')
+    with ops.Graph().as_default() as g:
+      features = {'price': [[1.], [5.]]}
+      fc.linear_model(features, [price], trainable=False)
+      trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      self.assertEqual([], trainable_vars)
+
+  def test_sparse_trainable_false(self):
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default() as g:
+      wire_tensor = sparse_tensor.SparseTensor(
+          values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
+      features = {'wire_cast': wire_tensor}
+      fc.linear_model(features, [wire_cast], trainable=False)
+      trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      self.assertEqual([], trainable_vars)
+
+  def test_column_order(self):
+    price_a = fc.numeric_column('price_a')
+    price_b = fc.numeric_column('price_b')
+    wire_cast = fc.categorical_column_with_hash_bucket('wire_cast', 4)
+    with ops.Graph().as_default() as g:
+      features = {
+          'price_a': [[1.]],
+          'price_b': [[3.]],
+          'wire_cast':
+              sparse_tensor.SparseTensor(
+                  values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
+      }
+      fc.linear_model(
+          features, [price_a, wire_cast, price_b],
+          weight_collections=['my-vars'])
+      my_vars = g.get_collection('my-vars')
+      self.assertIn('price_a', my_vars[0].name)
+      self.assertIn('price_b', my_vars[1].name)
+      self.assertIn('wire_cast', my_vars[2].name)
+
+    with ops.Graph().as_default() as g:
+      features = {
+          'price_a': [[1.]],
+          'price_b': [[3.]],
+          'wire_cast':
+              sparse_tensor.SparseTensor(
+                  values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
+      }
+      fc.linear_model(
+          features, [wire_cast, price_b, price_a],
+          weight_collections=['my-vars'])
+      my_vars = g.get_collection('my-vars')
+      self.assertIn('price_a', my_vars[0].name)
+      self.assertIn('price_b', my_vars[1].name)
+      self.assertIn('wire_cast', my_vars[2].name)
+
+  def test_static_batch_size_mismatch(self):
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': [[1.], [5.], [7.]],  # batchsize = 3
+          'price2': [[3.], [4.]]  # batchsize = 2
+      }
+    with self.assertRaisesRegexp(
+        ValueError,
+        'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+      fc.linear_model(features, [price1, price2])
+
+  def test_subset_of_static_batch_size_mismatch(self):
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    price3 = fc.numeric_column('price3')
+    with ops.Graph().as_default():
+      features = {
+          'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
+          'price2': [[3.], [4.]],  # batchsize = 2
+          'price3': [[3.], [4.], [5.]]  # batchsize = 3
+      }
+      with self.assertRaisesRegexp(
+          ValueError,
+          'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
+        fc.linear_model(features, [price1, price2, price3])
+
+  def test_runtime_batch_size_mismatch(self):
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
+          'price2': [[3.], [4.]]  # batchsize = 2
+      }
+      predictions = fc.linear_model(features, [price1, price2])
+      with _initialized_session() as sess:
+        with self.assertRaisesRegexp(errors.OpError,
+                                     'must have the same size and shape'):
+          sess.run(
+              predictions, feed_dict={features['price1']: [[1.], [5.], [7.]]})
+
+  def test_runtime_batch_size_matches(self):
+    price1 = fc.numeric_column('price1')
+    price2 = fc.numeric_column('price2')
+    with ops.Graph().as_default():
+      features = {
+          'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
+          'price2': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
+      }
+      predictions = fc.linear_model(features, [price1, price2])
+      with _initialized_session() as sess:
+        sess.run(
+            predictions,
+            feed_dict={
+                features['price1']: [[1.], [5.]],
+                features['price2']: [[1.], [5.]],
+            })
+
+  def test_with_numpy_input_fn(self):
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    body_style = fc.categorical_column_with_vocabulary_list(
+        'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
+
+    input_fn = numpy_io.numpy_input_fn(
+        x={
+            'price': np.array([-1., 2., 13., 104.]),
+            'body-style': np.array(['sedan', 'hardtop', 'wagon', 'sedan']),
+        },
+        batch_size=2,
+        shuffle=False)
+    features = input_fn()
+    net = fc.linear_model(features, [price_buckets, body_style])
+    # self.assertEqual(1 + 3 + 5, net.shape[1])
+    with _initialized_session() as sess:
+      coord = coordinator.Coordinator()
+      threads = queue_runner_impl.start_queue_runners(sess, coord=coord)
+
+      bias = get_linear_model_bias()
+      price_buckets_var = get_linear_model_column_var(price_buckets)
+      body_style_var = get_linear_model_column_var(body_style)
+
+      sess.run(price_buckets_var.assign([[10.], [100.], [1000.], [10000.]]))
+      sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
+      sess.run(bias.assign([5.]))
+
+      self.assertAllClose([[10 - 1000 + 5.], [100 - 10 + 5.]], sess.run(net))
+
+      coord.request_stop()
+      coord.join(threads)
+
+  def test_with_1d_sparse_tensor(self):
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    body_style = fc.categorical_column_with_vocabulary_list(
+        'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
+
+    # Provides 1-dim tensor and dense tensor.
+    features = {
+        'price': constant_op.constant([-1., 12.,]),
+        'body-style': sparse_tensor.SparseTensor(
+            indices=((0,), (1,)),
+            values=('sedan', 'hardtop'),
+            dense_shape=(2,)),
+    }
+    self.assertEqual(1, features['price'].shape.ndims)
+    self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0])
+
+    net = fc.linear_model(features, [price_buckets, body_style])
+    with _initialized_session() as sess:
+      bias = get_linear_model_bias()
+      price_buckets_var = get_linear_model_column_var(price_buckets)
+      body_style_var = get_linear_model_column_var(body_style)
+
+      sess.run(price_buckets_var.assign([[10.], [100.], [1000.], [10000.]]))
+      sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
+      sess.run(bias.assign([5.]))
+
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]], sess.run(net))
+
+  def test_with_1d_unknown_shape_sparse_tensor(self):
+    price = fc.numeric_column('price')
+    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    body_style = fc.categorical_column_with_vocabulary_list(
+        'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
+    country = fc.categorical_column_with_vocabulary_list(
+        'country', vocabulary_list=['US', 'JP', 'CA'])
+
+    # Provides 1-dim tensor and dense tensor.
+    features = {
+        'price': array_ops.placeholder(dtypes.float32),
+        'body-style': array_ops.sparse_placeholder(dtypes.string),
+        'country': array_ops.placeholder(dtypes.string),
+    }
+    self.assertIsNone(features['price'].shape.ndims)
+    self.assertIsNone(features['body-style'].get_shape().ndims)
 
-      def _get_sparse_tensors(self, inputs, weight_collections=None,
-                              trainable=None):
-        """Populates both id_tensor and weight_tensor."""
-        ids_and_weights = inputs.get(self)
-        return _CategoricalColumn.IdWeightPair(
-            id_tensor=ids_and_weights[0], weight_tensor=ids_and_weights[1])
+    price_data = np.array([-1., 12.])
+    body_style_data = sparse_tensor.SparseTensorValue(
+        indices=((0,), (1,)),
+        values=('sedan', 'hardtop'),
+        dense_shape=(2,))
+    country_data = np.array(['US', 'CA'])
 
-    t = _TestColumnWithWeights()
-    crossed = fc.crossed_column([t, 'c'], hash_bucket_size=5, hash_key=5)
-    with ops.Graph().as_default():
-      with self.assertRaisesRegexp(
-          ValueError,
-          'crossed_column does not support weight_tensor.*{}'.format(t.name)):
-        fc.linear_model({
-            t.name: sparse_tensor.SparseTensor(
-                indices=((0, 0), (1, 0), (1, 1)),
-                values=[0, 1, 2],
-                dense_shape=(2, 2)),
-            '{}_weights'.format(t.name): sparse_tensor.SparseTensor(
-                indices=((0, 0), (1, 0), (1, 1)),
-                values=[1., 10., 2.],
-                dense_shape=(2, 2)),
-            'c': sparse_tensor.SparseTensor(
-                indices=((0, 0), (1, 0), (1, 1)),
-                values=['cA', 'cB', 'cC'],
-                dense_shape=(2, 2)),
-        }, (crossed,))
+    net = fc.linear_model(features, [price_buckets, body_style, country])
+    bias = get_linear_model_bias()
+    price_buckets_var = get_linear_model_column_var(price_buckets)
+    body_style_var = get_linear_model_column_var(body_style)
+    with _initialized_session() as sess:
+      sess.run(price_buckets_var.assign([[10.], [100.], [1000.], [10000.]]))
+      sess.run(body_style_var.assign([[-10.], [-100.], [-1000.]]))
+      sess.run(bias.assign([5.]))
 
+      self.assertAllClose([[10 - 1000 + 5.], [1000 - 10 + 5.]],
+                          sess.run(
+                              net,
+                              feed_dict={
+                                  features['price']: price_data,
+                                  features['body-style']: body_style_data,
+                                  features['country']: country_data
+                              }))
 
-def get_linear_model_bias():
-  with variable_scope.variable_scope('linear_model', reuse=True):
-    return variable_scope.get_variable('bias_weights')
+  def test_with_rank_0_feature(self):
+    price = fc.numeric_column('price')
+    features = {
+        'price': constant_op.constant(0),
+    }
+    self.assertEqual(0, features['price'].shape.ndims)
 
+    # Static rank 0 should fail
+    with self.assertRaisesRegexp(ValueError, 'Feature .* cannot have rank 0'):
+      fc.linear_model(features, [price])
 
-def get_linear_model_column_var(column):
-  return ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES,
-                            'linear_model/' + column.name)[0]
+    # Dynamic rank 0 should fail
+    features = {
+        'price': array_ops.placeholder(dtypes.float32),
+    }
+    net = fc.linear_model(features, [price])
+    self.assertEqual(1, net.shape[1])
+    with _initialized_session() as sess:
+      with self.assertRaisesOpError('Feature .* cannot have rank 0'):
+        sess.run(net, feed_dict={features['price']: np.array(1)})
 
 
 @test_util.with_c_api
-class LinearModelTest(test.TestCase):
+class _LinearModelTest(test.TestCase):
 
   def test_raises_if_empty_feature_columns(self):
     with self.assertRaisesRegexp(ValueError,
                                  'feature_columns must not be empty'):
-      fc.linear_model(features={}, feature_columns=[])
+      get_keras_linear_model_predictions(features={}, feature_columns=[])
 
   def test_should_be_feature_column(self):
     with self.assertRaisesRegexp(ValueError, 'must be a _FeatureColumn'):
-      fc.linear_model(features={'a': [[0]]}, feature_columns='NotSupported')
+      get_keras_linear_model_predictions(
+          features={'a': [[0]]}, feature_columns='NotSupported')
 
   def test_should_be_dense_or_categorical_column(self):
 
@@ -1100,7 +1933,7 @@ class LinearModelTest(test.TestCase):
 
     with self.assertRaisesRegexp(
         ValueError, 'must be either a _DenseColumn or _CategoricalColumn'):
-      fc.linear_model(
+      get_keras_linear_model_predictions(
           features={'a': [[0]]}, feature_columns=[NotSupportedColumn()])
 
   def test_does_not_support_dict_columns(self):
@@ -1112,7 +1945,7 @@ class LinearModelTest(test.TestCase):
   def test_raises_if_duplicate_name(self):
     with self.assertRaisesRegexp(
         ValueError, 'Duplicate feature column name found for columns'):
-      fc.linear_model(
+      get_keras_linear_model_predictions(
           features={'a': [[0]]},
           feature_columns=[fc.numeric_column('a'),
                            fc.numeric_column('a')])
@@ -1121,8 +1954,8 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
-      predictions = fc.linear_model(features, [price])
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(features, [price])
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
         self.assertAllClose([0.], bias.eval())
@@ -1138,8 +1971,8 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0], [1, 0], [1, 1]],
           dense_shape=[2, 2])
       features = {'wire_cast': wire_tensor}
-      predictions = fc.linear_model(features, [wire_cast])
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(features, [wire_cast])
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
         self.assertAllClose([0.], bias.eval())
@@ -1157,8 +1990,9 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0], [1, 0], [1, 1]],
           dense_shape=[2, 2])
       features = {'wire_cast': wire_tensor, 'price': [[1.], [5.]]}
-      predictions = fc.linear_model(features, [wire_cast, price])
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [wire_cast, price])
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
@@ -1187,7 +2021,9 @@ class LinearModelTest(test.TestCase):
       def _variable_shape(self):
         raise ValueError('Should not use this method.')
 
-      def _get_dense_tensor(self, inputs, weight_collections=None,
+      def _get_dense_tensor(self,
+                            inputs,
+                            weight_collections=None,
                             trainable=None):
         raise ValueError('Should not use this method.')
 
@@ -1195,7 +2031,9 @@ class LinearModelTest(test.TestCase):
       def _num_buckets(self):
         return 4
 
-      def _get_sparse_tensors(self, inputs, weight_collections=None,
+      def _get_sparse_tensors(self,
+                              inputs,
+                              weight_collections=None,
                               trainable=None):
         sp_tensor = sparse_tensor.SparseTensor(
             indices=[[0, 0], [1, 0], [1, 1]],
@@ -1210,13 +2048,15 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0], [1, 0], [1, 1]],
           dense_shape=[2, 2])
       features = {dense_and_sparse_column.name: sp_tensor}
-      predictions = fc.linear_model(features, [dense_and_sparse_column])
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(
+          features, [dense_and_sparse_column])
+      bias = get_keras_linear_model_bias()
       dense_and_sparse_column_var = get_linear_model_column_var(
           dense_and_sparse_column)
       with _initialized_session() as sess:
-        sess.run(dense_and_sparse_column_var.assign(
-            [[10.], [100.], [1000.], [10000.]]))
+        sess.run(
+            dense_and_sparse_column_var.assign([[10.], [100.], [1000.],
+                                                [10000.]]))
         sess.run(bias.assign([5.]))
         self.assertAllClose([[1005.], [10015.]], predictions.eval())
 
@@ -1224,8 +2064,9 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price')
     with ops.Graph().as_default():
       features = {'price': [[1.], [5.]]}
-      predictions = fc.linear_model(features, [price], units=3)
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(
+          features, [price], units=3)
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
         self.assertAllClose(np.zeros((3,)), bias.eval())
@@ -1243,16 +2084,17 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0], [1, 0], [1, 1]],
           dense_shape=[2, 2])
       features = {'wire_cast': wire_tensor}
-      predictions = fc.linear_model(features, [wire_cast], units=3)
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(
+          features, [wire_cast], units=3)
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
         self.assertAllClose(np.zeros((3,)), bias.eval())
         self.assertAllClose(np.zeros((4, 3)), wire_cast_var.eval())
         sess.run(
-            wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.], [
-                1000., 1100., 1200.
-            ], [10000., 11000., 12000.]]))
+            wire_cast_var.assign([[10., 11., 12.], [100., 110., 120.],
+                                  [1000., 1100.,
+                                   1200.], [10000., 11000., 12000.]]))
         sess.run(bias.assign([5., 6., 7.]))
         self.assertAllClose([[1005., 1106., 1207.], [10015., 11017., 12019.]],
                             predictions.eval())
@@ -1261,7 +2103,7 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
-      predictions = fc.linear_model(features, [price])
+      predictions = get_keras_linear_model_predictions(features, [price])
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
         self.assertAllClose([[0.], [0.]], price_var.eval())
@@ -1277,7 +2119,7 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0, 0], [0, 1, 0], [1, 0, 0], [1, 0, 1]],
           dense_shape=[2, 2, 2])
       features = {'wire_cast': wire_tensor}
-      predictions = fc.linear_model(features, [wire_cast])
+      predictions = get_keras_linear_model_predictions(features, [wire_cast])
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
         self.assertAllClose(np.zeros((4, 1)), wire_cast_var.eval())
@@ -1297,9 +2139,9 @@ class LinearModelTest(test.TestCase):
           indices=[[0, 0], [1, 0], [1, 1]],
           dense_shape=[2, 2])
       features = {'wire_cast': wire_tensor}
-      predictions = fc.linear_model(
+      predictions = get_keras_linear_model_predictions(
           features, [wire_cast], sparse_combiner='mean')
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       with _initialized_session() as sess:
         sess.run(wire_cast_var.assign([[10.], [100.], [1000.], [10000.]]))
@@ -1310,8 +2152,9 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price', shape=2)
     with ops.Graph().as_default():
       features = {'price': [[1., 2.], [5., 6.]]}
-      predictions = fc.linear_model(features, [price], units=3)
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(
+          features, [price], units=3)
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
         self.assertAllClose(np.zeros((3,)), bias.eval())
@@ -1329,9 +2172,9 @@ class LinearModelTest(test.TestCase):
         with self.assertRaisesRegexp(
             Exception,
             r'Cannot reshape a tensor with 2 elements to shape \[2,2\]'):
-          predictions = fc.linear_model(features, [price])
+          predictions = get_keras_linear_model_predictions(features, [price])
       else:
-        predictions = fc.linear_model(features, [price])
+        predictions = get_keras_linear_model_predictions(features, [price])
         with _initialized_session():
           with self.assertRaisesRegexp(Exception, 'requested shape has 4'):
             predictions.eval()
@@ -1340,8 +2183,8 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price', shape=[1, 2])
     with ops.Graph().as_default():
       features = {'price': [[[1., 2.]], [[5., 6.]]]}
-      predictions = fc.linear_model(features, [price])
-      bias = get_linear_model_bias()
+      predictions = get_keras_linear_model_predictions(features, [price])
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       with _initialized_session() as sess:
         self.assertAllClose([0.], bias.eval())
@@ -1354,12 +2197,10 @@ class LinearModelTest(test.TestCase):
     price1 = fc.numeric_column('price1', shape=2)
     price2 = fc.numeric_column('price2')
     with ops.Graph().as_default():
-      features = {
-          'price1': [[1., 2.], [5., 6.]],
-          'price2': [[3.], [4.]]
-      }
-      predictions = fc.linear_model(features, [price1, price2])
-      bias = get_linear_model_bias()
+      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [price1, price2])
+      bias = get_keras_linear_model_bias()
       price1_var = get_linear_model_column_var(price1)
       price2_var = get_linear_model_column_var(price2)
       with _initialized_session() as sess:
@@ -1372,50 +2213,14 @@ class LinearModelTest(test.TestCase):
         sess.run(bias.assign([7.]))
         self.assertAllClose([[3217.], [4657.]], predictions.eval())
 
-  def test_fills_cols_to_vars(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2')
-    with ops.Graph().as_default():
-      features = {'price1': [[1., 2.], [5., 6.]], 'price2': [[3.], [4.]]}
-      cols_to_vars = {}
-      fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
-      bias = get_linear_model_bias()
-      price1_var = get_linear_model_column_var(price1)
-      price2_var = get_linear_model_column_var(price2)
-      self.assertAllEqual(cols_to_vars['bias'], [bias])
-      self.assertAllEqual(cols_to_vars[price1], [price1_var])
-      self.assertAllEqual(cols_to_vars[price2], [price2_var])
-
-  def test_fills_cols_to_vars_partitioned_variables(self):
-    price1 = fc.numeric_column('price1', shape=2)
-    price2 = fc.numeric_column('price2', shape=3)
-    with ops.Graph().as_default():
-      features = {
-          'price1': [[1., 2.], [6., 7.]],
-          'price2': [[3., 4., 5.], [8., 9., 10.]]
-      }
-      cols_to_vars = {}
-      with variable_scope.variable_scope(
-          'linear',
-          partitioner=partitioned_variables.fixed_size_partitioner(2, axis=0)):
-        fc.linear_model(features, [price1, price2], cols_to_vars=cols_to_vars)
-      with _initialized_session():
-        self.assertEqual([0.], cols_to_vars['bias'][0].eval())
-        # Partitioning shards the [2, 1] price1 var into 2 [1, 1] Variables.
-        self.assertAllEqual([[0.]], cols_to_vars[price1][0].eval())
-        self.assertAllEqual([[0.]], cols_to_vars[price1][1].eval())
-        # Partitioning shards the [3, 1] price2 var into a [2, 1] Variable and
-        # a [1, 1] Variable.
-        self.assertAllEqual([[0.], [0.]], cols_to_vars[price2][0].eval())
-        self.assertAllEqual([[0.]], cols_to_vars[price2][1].eval())
-
   def test_dense_collection(self):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
-      fc.linear_model(features, [price], weight_collections=['my-vars'])
+      get_keras_linear_model_predictions(
+          features, [price], weight_collections=['my-vars'])
       my_vars = g.get_collection('my-vars')
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       self.assertIn(bias, my_vars)
       self.assertIn(price_var, my_vars)
@@ -1426,10 +2231,10 @@ class LinearModelTest(test.TestCase):
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
       features = {'wire_cast': wire_tensor}
-      fc.linear_model(
+      get_keras_linear_model_predictions(
           features, [wire_cast], weight_collections=['my-vars'])
       my_vars = g.get_collection('my-vars')
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       self.assertIn(bias, my_vars)
       self.assertIn(wire_cast_var, my_vars)
@@ -1438,8 +2243,8 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
-      fc.linear_model(features, [price])
-      bias = get_linear_model_bias()
+      get_keras_linear_model_predictions(features, [price])
+      bias = get_keras_linear_model_bias()
       price_var = get_linear_model_column_var(price)
       trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
       self.assertIn(bias, trainable_vars)
@@ -1451,9 +2256,9 @@ class LinearModelTest(test.TestCase):
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
       features = {'wire_cast': wire_tensor}
-      fc.linear_model(features, [wire_cast])
+      get_keras_linear_model_predictions(features, [wire_cast])
       trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       wire_cast_var = get_linear_model_column_var(wire_cast)
       self.assertIn(bias, trainable_vars)
       self.assertIn(wire_cast_var, trainable_vars)
@@ -1462,7 +2267,7 @@ class LinearModelTest(test.TestCase):
     price = fc.numeric_column('price')
     with ops.Graph().as_default() as g:
       features = {'price': [[1.], [5.]]}
-      fc.linear_model(features, [price], trainable=False)
+      get_keras_linear_model_predictions(features, [price], trainable=False)
       trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
       self.assertEqual([], trainable_vars)
 
@@ -1472,7 +2277,7 @@ class LinearModelTest(test.TestCase):
       wire_tensor = sparse_tensor.SparseTensor(
           values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
       features = {'wire_cast': wire_tensor}
-      fc.linear_model(features, [wire_cast], trainable=False)
+      get_keras_linear_model_predictions(features, [wire_cast], trainable=False)
       trainable_vars = g.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
       self.assertEqual([], trainable_vars)
 
@@ -1488,7 +2293,7 @@ class LinearModelTest(test.TestCase):
               sparse_tensor.SparseTensor(
                   values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
       }
-      fc.linear_model(
+      get_keras_linear_model_predictions(
           features, [price_a, wire_cast, price_b],
           weight_collections=['my-vars'])
       my_vars = g.get_collection('my-vars')
@@ -1504,7 +2309,7 @@ class LinearModelTest(test.TestCase):
               sparse_tensor.SparseTensor(
                   values=['omar'], indices=[[0, 0]], dense_shape=[1, 1])
       }
-      fc.linear_model(
+      get_keras_linear_model_predictions(
           features, [wire_cast, price_b, price_a],
           weight_collections=['my-vars'])
       my_vars = g.get_collection('my-vars')
@@ -1523,7 +2328,7 @@ class LinearModelTest(test.TestCase):
     with self.assertRaisesRegexp(
         ValueError,
         'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
-      fc.linear_model(features, [price1, price2])
+      get_keras_linear_model_predictions(features, [price1, price2])
 
   def test_subset_of_static_batch_size_mismatch(self):
     price1 = fc.numeric_column('price1')
@@ -1538,7 +2343,7 @@ class LinearModelTest(test.TestCase):
       with self.assertRaisesRegexp(
           ValueError,
           'Batch size \(first dimension\) of each feature must be same.'):  # pylint: disable=anomalous-backslash-in-string
-        fc.linear_model(features, [price1, price2, price3])
+        get_keras_linear_model_predictions(features, [price1, price2, price3])
 
   def test_runtime_batch_size_mismatch(self):
     price1 = fc.numeric_column('price1')
@@ -1548,7 +2353,8 @@ class LinearModelTest(test.TestCase):
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 3
           'price2': [[3.], [4.]]  # batchsize = 2
       }
-      predictions = fc.linear_model(features, [price1, price2])
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [price1, price2])
       with _initialized_session() as sess:
         with self.assertRaisesRegexp(errors.OpError,
                                      'must have the same size and shape'):
@@ -1563,7 +2369,8 @@ class LinearModelTest(test.TestCase):
           'price1': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
           'price2': array_ops.placeholder(dtype=dtypes.int64),  # batchsize = 2
       }
-      predictions = fc.linear_model(features, [price1, price2])
+      predictions = get_keras_linear_model_predictions(features,
+                                                       [price1, price2])
       with _initialized_session() as sess:
         sess.run(
             predictions,
@@ -1574,7 +2381,12 @@ class LinearModelTest(test.TestCase):
 
   def test_with_numpy_input_fn(self):
     price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    price_buckets = fc.bucketized_column(
+        price, boundaries=[
+            0.,
+            10.,
+            100.,
+        ])
     body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
@@ -1586,13 +2398,14 @@ class LinearModelTest(test.TestCase):
         batch_size=2,
         shuffle=False)
     features = input_fn()
-    net = fc.linear_model(features, [price_buckets, body_style])
+    net = get_keras_linear_model_predictions(features,
+                                             [price_buckets, body_style])
     # self.assertEqual(1 + 3 + 5, net.shape[1])
     with _initialized_session() as sess:
       coord = coordinator.Coordinator()
       threads = queue_runner_impl.start_queue_runners(sess, coord=coord)
 
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       price_buckets_var = get_linear_model_column_var(price_buckets)
       body_style_var = get_linear_model_column_var(body_style)
 
@@ -1607,24 +2420,35 @@ class LinearModelTest(test.TestCase):
 
   def test_with_1d_sparse_tensor(self):
     price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    price_buckets = fc.bucketized_column(
+        price, boundaries=[
+            0.,
+            10.,
+            100.,
+        ])
     body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
 
     # Provides 1-dim tensor and dense tensor.
     features = {
-        'price': constant_op.constant([-1., 12.,]),
-        'body-style': sparse_tensor.SparseTensor(
-            indices=((0,), (1,)),
-            values=('sedan', 'hardtop'),
-            dense_shape=(2,)),
+        'price':
+            constant_op.constant([
+                -1.,
+                12.,
+            ]),
+        'body-style':
+            sparse_tensor.SparseTensor(
+                indices=((0,), (1,)),
+                values=('sedan', 'hardtop'),
+                dense_shape=(2,)),
     }
     self.assertEqual(1, features['price'].shape.ndims)
     self.assertEqual(1, features['body-style'].dense_shape.get_shape()[0])
 
-    net = fc.linear_model(features, [price_buckets, body_style])
+    net = get_keras_linear_model_predictions(features,
+                                             [price_buckets, body_style])
     with _initialized_session() as sess:
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       price_buckets_var = get_linear_model_column_var(price_buckets)
       body_style_var = get_linear_model_column_var(body_style)
 
@@ -1636,7 +2460,12 @@ class LinearModelTest(test.TestCase):
 
   def test_with_1d_unknown_shape_sparse_tensor(self):
     price = fc.numeric_column('price')
-    price_buckets = fc.bucketized_column(price, boundaries=[0., 10., 100.,])
+    price_buckets = fc.bucketized_column(
+        price, boundaries=[
+            0.,
+            10.,
+            100.,
+        ])
     body_style = fc.categorical_column_with_vocabulary_list(
         'body-style', vocabulary_list=['hardtop', 'wagon', 'sedan'])
     country = fc.categorical_column_with_vocabulary_list(
@@ -1653,13 +2482,12 @@ class LinearModelTest(test.TestCase):
 
     price_data = np.array([-1., 12.])
     body_style_data = sparse_tensor.SparseTensorValue(
-        indices=((0,), (1,)),
-        values=('sedan', 'hardtop'),
-        dense_shape=(2,))
+        indices=((0,), (1,)), values=('sedan', 'hardtop'), dense_shape=(2,))
     country_data = np.array(['US', 'CA'])
 
-    net = fc.linear_model(features, [price_buckets, body_style, country])
-    bias = get_linear_model_bias()
+    net = get_keras_linear_model_predictions(
+        features, [price_buckets, body_style, country])
+    bias = get_keras_linear_model_bias()
     price_buckets_var = get_linear_model_column_var(price_buckets)
     body_style_var = get_linear_model_column_var(body_style)
     with _initialized_session() as sess:
@@ -1685,13 +2513,13 @@ class LinearModelTest(test.TestCase):
 
     # Static rank 0 should fail
     with self.assertRaisesRegexp(ValueError, 'Feature .* cannot have rank 0'):
-      fc.linear_model(features, [price])
+      get_keras_linear_model_predictions(features, [price])
 
     # Dynamic rank 0 should fail
     features = {
         'price': array_ops.placeholder(dtypes.float32),
     }
-    net = fc.linear_model(features, [price])
+    net = get_keras_linear_model_predictions(features, [price])
     self.assertEqual(1, net.shape[1])
     with _initialized_session() as sess:
       with self.assertRaisesOpError('Feature .* cannot have rank 0'):
@@ -2715,6 +3543,32 @@ class VocabularyFileCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
+  def test_keras_linear_model(self):
+    wire_column = fc.categorical_column_with_vocabulary_file(
+        key='wire',
+        vocabulary_file=self._wire_vocabulary_file_name,
+        vocabulary_size=self._wire_vocabulary_size,
+        num_oov_buckets=1)
+    self.assertEqual(4, wire_column._num_buckets)
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          wire_column.name:
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=('marlo', 'skywalker', 'omar'),
+                  dense_shape=(2, 2))
+      }, (wire_column,))
+      bias = get_keras_linear_model_bias()
+      wire_var = get_linear_model_column_var(wire_column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
+        # 'marlo' -> 2: wire_var[2] = 3
+        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+
 
 class VocabularyListCategoricalColumnTest(test.TestCase):
 
@@ -3082,6 +3936,31 @@ class VocabularyListCategoricalColumnTest(test.TestCase):
         # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
         self.assertAllClose(((3.,), (5.,)), predictions.eval())
 
+  def test_keras_linear_model(self):
+    wire_column = fc.categorical_column_with_vocabulary_list(
+        key='aaa',
+        vocabulary_list=('omar', 'stringer', 'marlo'),
+        num_oov_buckets=1)
+    self.assertEqual(4, wire_column._num_buckets)
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          wire_column.name:
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=('marlo', 'skywalker', 'omar'),
+                  dense_shape=(2, 2))
+      }, (wire_column,))
+      bias = get_keras_linear_model_bias()
+      wire_var = get_linear_model_column_var(wire_column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,), (0.,)), wire_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        wire_var.assign(((1.,), (2.,), (3.,), (4.,))).eval()
+        # 'marlo' -> 2: wire_var[2] = 3
+        # 'skywalker' -> 3, 'omar' -> 0: wire_var[3] + wire_var[0] = 4+1 = 5
+        self.assertAllClose(((3.,), (5.,)), predictions.eval())
+
 
 class IdentityCategoricalColumnTest(test.TestCase):
 
@@ -3285,17 +4164,39 @@ class IdentityCategoricalColumnTest(test.TestCase):
               input_shape: (2, 2),
           }))
 
-  def test_linear_model(self):
+  def test_linear_model(self):
+    column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
+    self.assertEqual(3, column._num_buckets)
+    with ops.Graph().as_default():
+      predictions = fc.linear_model({
+          column.name: sparse_tensor.SparseTensorValue(
+              indices=((0, 0), (1, 0), (1, 1)),
+              values=(0, 2, 1),
+              dense_shape=(2, 2))
+      }, (column,))
+      bias = get_linear_model_bias()
+      weight_var = get_linear_model_column_var(column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        weight_var.assign(((1.,), (2.,), (3.,))).eval()
+        # weight_var[0] = 1
+        # weight_var[2] + weight_var[1] = 3+2 = 5
+        self.assertAllClose(((1.,), (5.,)), predictions.eval())
+
+  def test_keras_linear_model(self):
     column = fc.categorical_column_with_identity(key='aaa', num_buckets=3)
     self.assertEqual(3, column._num_buckets)
     with ops.Graph().as_default():
-      predictions = fc.linear_model({
-          column.name: sparse_tensor.SparseTensorValue(
-              indices=((0, 0), (1, 0), (1, 1)),
-              values=(0, 2, 1),
-              dense_shape=(2, 2))
+      predictions = get_keras_linear_model_predictions({
+          column.name:
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=(0, 2, 1),
+                  dense_shape=(2, 2))
       }, (column,))
-      bias = get_linear_model_bias()
+      bias = get_keras_linear_model_bias()
       weight_var = get_linear_model_column_var(column)
       with _initialized_session():
         self.assertAllClose((0.,), bias.eval())
@@ -3537,6 +4438,25 @@ class IndicatorColumnTest(test.TestCase):
         weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
         self.assertAllClose([[2. + 3.]], predictions.eval())
 
+  def test_keras_linear_model(self):
+    animal = fc.indicator_column(
+        fc.categorical_column_with_identity('animal', num_buckets=4))
+    with ops.Graph().as_default():
+      features = {
+          'animal':
+              sparse_tensor.SparseTensor(
+                  indices=[[0, 0], [0, 1]], values=[1, 2], dense_shape=[1, 2])
+      }
+
+      predictions = get_keras_linear_model_predictions(features, [animal])
+      weight_var = get_linear_model_column_var(animal)
+      with _initialized_session():
+        # All should be zero-initialized.
+        self.assertAllClose([[0.], [0.], [0.], [0.]], weight_var.eval())
+        self.assertAllClose([[0.]], predictions.eval())
+        weight_var.assign([[1.], [2.], [3.], [4.]]).eval()
+        self.assertAllClose([[2. + 3.]], predictions.eval())
+
   def test_input_layer(self):
     animal = fc.indicator_column(
         fc.categorical_column_with_identity('animal', num_buckets=4))
@@ -3727,8 +4647,8 @@ class EmbeddingColumnTest(test.TestCase):
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
+    self.assertItemsEqual(('embedding_weights:0',),
+                          tuple([v.name for v in global_vars]))
     with _initialized_session():
       self.assertAllEqual(embedding_values, global_vars[0].eval())
       self.assertAllEqual(expected_lookups, embedding_lookup.eval())
@@ -3752,6 +4672,7 @@ class EmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -3774,20 +4695,21 @@ class EmbeddingColumnTest(test.TestCase):
     categorical_column = fc.categorical_column_with_identity(
         key='aaa', num_buckets=vocabulary_size)
     embedding_column = fc.embedding_column(
-        categorical_column, dimension=embedding_dimension,
+        categorical_column,
+        dimension=embedding_dimension,
         initializer=_initializer)
 
     # Create embedding_weights variable.
-    weight_collections = [ops.GraphKeys.GLOBAL_VARIABLES,
-                          ops.GraphKeys.MODEL_VARIABLES]
+    weight_collections = [
+        ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES
+    ]
     state = embedding_column._create_state(weight_collections)
 
     # Provide sparse input and get dense result.
     embedding_lookup = embedding_column._get_dense_tensor(
         _LazyBuilder({
             'aaa': sparse_input
-        }),
-        state=state)
+        }), state=state)
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
@@ -4087,6 +5009,82 @@ class EmbeddingColumnTest(test.TestCase):
         # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
         self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
 
+  def test_keras_linear_model(self):
+    # Inputs.
+    batch_size = 4
+    vocabulary_size = 3
+    sparse_input = sparse_tensor.SparseTensorValue(
+        # example 0, ids [2]
+        # example 1, ids [0, 1]
+        # example 2, ids []
+        # example 3, ids [1]
+        indices=((0, 0), (1, 0), (1, 4), (3, 0)),
+        values=(2, 0, 1, 1),
+        dense_shape=(batch_size, 5))
+
+    # Embedding variable.
+    embedding_dimension = 2
+    embedding_shape = (vocabulary_size, embedding_dimension)
+    zeros_embedding_values = np.zeros(embedding_shape)
+
+    def _initializer(shape, dtype, partition_info):
+      self.assertAllEqual(embedding_shape, shape)
+      self.assertEqual(dtypes.float32, dtype)
+      self.assertIsNone(partition_info)
+      return zeros_embedding_values
+
+    # Build columns.
+    categorical_column = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=vocabulary_size)
+    embedding_column = fc.embedding_column(
+        categorical_column,
+        dimension=embedding_dimension,
+        initializer=_initializer)
+
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          categorical_column.name: sparse_input
+      }, (embedding_column,))
+      expected_var_names = (
+          'linear_model/bias_layer/bias_weights:0',
+          'linear_model/aaa_embedding/weights:0',
+          'linear_model/aaa_embedding/embedding_weights:0',
+      )
+      self.assertItemsEqual(
+          expected_var_names,
+          [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
+      trainable_vars = {
+          v.name: v
+          for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      }
+      self.assertItemsEqual(expected_var_names, trainable_vars.keys())
+      bias = trainable_vars['linear_model/bias_layer/bias_weights:0']
+      embedding_weights = trainable_vars[
+          'linear_model/aaa_embedding/embedding_weights:0']
+      linear_weights = trainable_vars['linear_model/aaa_embedding/weights:0']
+      with _initialized_session():
+        # Predictions with all zero weights.
+        self.assertAllClose(np.zeros((1,)), bias.eval())
+        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), linear_weights.eval())
+        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+
+        # Predictions with all non-zero weights.
+        embedding_weights.assign((
+            (1., 2.),  # id 0
+            (3., 5.),  # id 1
+            (7., 11.)  # id 2
+        )).eval()
+        linear_weights.assign(((4.,), (6.,))).eval()
+        # example 0, ids [2], embedding[0] = [7, 11]
+        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+        # example 2, ids [], embedding[2] = [0, 0]
+        # example 3, ids [1], embedding[3] = [3, 5]
+        # sum(embeddings * linear_weights)
+        # = [4*7 + 6*11, 4*2 + 6*3.5, 4*0 + 6*0, 4*3 + 6*5] = [94, 29, 0, 42]
+        self.assertAllClose(((94.,), (29.,), (0.,), (42.,)), predictions.eval())
+
   def test_input_layer(self):
     # Inputs.
     vocabulary_size = 3
@@ -4509,8 +5507,8 @@ class SharedEmbeddingColumnTest(test.TestCase):
 
     # Assert expected embedding variable and lookups.
     global_vars = ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)
-    self.assertItemsEqual(
-        ('embedding_weights:0',), tuple([v.name for v in global_vars]))
+    self.assertItemsEqual(('embedding_weights:0',),
+                          tuple([v.name for v in global_vars]))
     embedding_var = global_vars[0]
     with _initialized_session():
       self.assertAllEqual(embedding_values, embedding_var.eval())
@@ -4521,16 +5519,15 @@ class SharedEmbeddingColumnTest(test.TestCase):
     # Inputs.
     vocabulary_size = 3
     # -1 values are ignored.
-    input_a = np.array(
-        [[2, -1, -1],  # example 0, ids [2]
-         [0, 1, -1]])  # example 1, ids [0, 1]
-    input_b = np.array(
-        [[0, -1, -1],  # example 0, ids [0]
-         [-1, -1, -1]])  # example 1, ids []
-    input_features = {
-        'aaa': input_a,
-        'bbb': input_b
-    }
+    input_a = np.array([
+        [2, -1, -1],  # example 0, ids [2]
+        [0, 1, -1]
+    ])  # example 1, ids [0, 1]
+    input_b = np.array([
+        [0, -1, -1],  # example 0, ids [0]
+        [-1, -1, -1]
+    ])  # example 1, ids []
+    input_features = {'aaa': input_a, 'bbb': input_b}
 
     # Embedding variable.
     embedding_dimension = 2
@@ -4539,6 +5536,7 @@ class SharedEmbeddingColumnTest(test.TestCase):
         (3., 5.),  # id 1
         (7., 11.)  # id 2
     )
+
     def _initializer(shape, dtype, partition_info):
       self.assertAllEqual((vocabulary_size, embedding_dimension), shape)
       self.assertEqual(dtypes.float32, dtype)
@@ -4566,11 +5564,13 @@ class SharedEmbeddingColumnTest(test.TestCase):
         key='bbb', num_buckets=vocabulary_size)
     embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
         [categorical_column_a, categorical_column_b],
-        dimension=embedding_dimension, initializer=_initializer)
+        dimension=embedding_dimension,
+        initializer=_initializer)
 
     # Create state.
-    weight_collections = [ops.GraphKeys.GLOBAL_VARIABLES,
-                          ops.GraphKeys.MODEL_VARIABLES]
+    weight_collections = [
+        ops.GraphKeys.GLOBAL_VARIABLES, ops.GraphKeys.MODEL_VARIABLES
+    ]
     state = embedding_column_a._create_state(weight_collections)
 
     # Provide sparse input and get dense result.
@@ -4731,6 +5731,97 @@ class SharedEmbeddingColumnTest(test.TestCase):
         # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
         self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
 
+  def test_keras_linear_model(self):
+    # Inputs.
+    batch_size = 2
+    vocabulary_size = 3
+    # -1 values are ignored.
+    input_a = np.array([
+        [2, -1, -1],  # example 0, ids [2]
+        [0, 1, -1]
+    ])  # example 1, ids [0, 1]
+    input_b = np.array([
+        [0, -1, -1],  # example 0, ids [0]
+        [-1, -1, -1]
+    ])  # example 1, ids []
+
+    # Embedding variable.
+    embedding_dimension = 2
+    embedding_shape = (vocabulary_size, embedding_dimension)
+    zeros_embedding_values = np.zeros(embedding_shape)
+
+    def _initializer(shape, dtype, partition_info):
+      self.assertAllEqual(embedding_shape, shape)
+      self.assertEqual(dtypes.float32, dtype)
+      self.assertIsNone(partition_info)
+      return zeros_embedding_values
+
+    # Build columns.
+    categorical_column_a = fc.categorical_column_with_identity(
+        key='aaa', num_buckets=vocabulary_size)
+    categorical_column_b = fc.categorical_column_with_identity(
+        key='bbb', num_buckets=vocabulary_size)
+    embedding_column_a, embedding_column_b = fc.shared_embedding_columns(
+        [categorical_column_a, categorical_column_b],
+        dimension=embedding_dimension,
+        initializer=_initializer)
+
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          categorical_column_a.name: input_a,
+          categorical_column_b.name: input_b,
+      }, (embedding_column_a, embedding_column_b))
+      # Linear weights do not follow the column name. But this is a rare use
+      # case, and fixing it would add too much complexity to the code.
+      expected_var_names = (
+          'linear_model/bias_layer/bias_weights:0',
+          'linear_model/aaa_bbb_shared_embedding/weights:0',
+          'linear_model/aaa_bbb_shared_embedding/embedding_weights:0',
+          'linear_model/aaa_bbb_shared_embedding_1/weights:0',
+      )
+      self.assertItemsEqual(
+          expected_var_names,
+          [v.name for v in ops.get_collection(ops.GraphKeys.GLOBAL_VARIABLES)])
+      trainable_vars = {
+          v.name: v
+          for v in ops.get_collection(ops.GraphKeys.TRAINABLE_VARIABLES)
+      }
+      self.assertItemsEqual(expected_var_names, trainable_vars.keys())
+      bias = trainable_vars['linear_model/bias_layer/bias_weights:0']
+      embedding_weights = trainable_vars[
+          'linear_model/aaa_bbb_shared_embedding/embedding_weights:0']
+      linear_weights_a = trainable_vars[
+          'linear_model/aaa_bbb_shared_embedding/weights:0']
+      linear_weights_b = trainable_vars[
+          'linear_model/aaa_bbb_shared_embedding_1/weights:0']
+      with _initialized_session():
+        # Predictions with all zero weights.
+        self.assertAllClose(np.zeros((1,)), bias.eval())
+        self.assertAllClose(zeros_embedding_values, embedding_weights.eval())
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), linear_weights_a.eval())
+        self.assertAllClose(
+            np.zeros((embedding_dimension, 1)), linear_weights_b.eval())
+        self.assertAllClose(np.zeros((batch_size, 1)), predictions.eval())
+
+        # Predictions with all non-zero weights.
+        embedding_weights.assign((
+            (1., 2.),  # id 0
+            (3., 5.),  # id 1
+            (7., 11.)  # id 2
+        )).eval()
+        linear_weights_a.assign(((4.,), (6.,))).eval()
+        # example 0, ids [2], embedding[0] = [7, 11]
+        # example 1, ids [0, 1], embedding[1] = mean([1, 2] + [3, 5]) = [2, 3.5]
+        # sum(embeddings * linear_weights)
+        # = [4*7 + 6*11, 4*2 + 6*3.5] = [94, 29]
+        linear_weights_b.assign(((3.,), (5.,))).eval()
+        # example 0, ids [0], embedding[0] = [1, 2]
+        # example 1, ids [], embedding[1] = 0, 0]
+        # sum(embeddings * linear_weights)
+        # = [3*1 + 5*2, 3*0 +5*0] = [13, 0]
+        self.assertAllClose([[94. + 13.], [29.]], predictions.eval())
+
   def _test_input_layer(self, trainable=True):
     # Inputs.
     vocabulary_size = 3
@@ -5016,6 +6107,101 @@ class WeightedCategoricalColumnTest(test.TestCase):
               dense_shape=(2, 2)),
           weight_tensor.eval())
 
+  def test_keras_linear_model(self):
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
+            key='ids', num_buckets=3),
+        weight_feature_key='values')
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          'ids':
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=(0, 2, 1),
+                  dense_shape=(2, 2)),
+          'values':
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=(.5, 1., .1),
+                  dense_shape=(2, 2))
+      }, (column,))
+      bias = get_keras_linear_model_bias()
+      weight_var = get_linear_model_column_var(column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        weight_var.assign(((1.,), (2.,), (3.,))).eval()
+        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+        # = 3*1 + 2*.1 = 3+.2 = 3.2
+        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+
+  def test_keras_linear_model_mismatched_shape(self):
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
+            key='ids', num_buckets=3),
+        weight_feature_key='values')
+    with ops.Graph().as_default():
+      with self.assertRaisesRegexp(ValueError,
+                                   r'Dimensions.*are not compatible'):
+        get_keras_linear_model_predictions({
+            'ids':
+                sparse_tensor.SparseTensorValue(
+                    indices=((0, 0), (1, 0), (1, 1)),
+                    values=(0, 2, 1),
+                    dense_shape=(2, 2)),
+            'values':
+                sparse_tensor.SparseTensorValue(
+                    indices=((0, 0), (0, 1), (1, 0), (1, 1)),
+                    values=(.5, 11., 1., .1),
+                    dense_shape=(2, 2))
+        }, (column,))
+
+  def test_keras_linear_model_mismatched_dense_values(self):
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
+            key='ids', num_buckets=3),
+        weight_feature_key='values')
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          'ids':
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=(0, 2, 1),
+                  dense_shape=(2, 2)),
+          'values': ((.5,), (1.,))
+      }, (column,))
+      with _initialized_session():
+        with self.assertRaisesRegexp(errors.OpError, 'Incompatible shapes'):
+          predictions.eval()
+
+  def test_keras_linear_model_mismatched_dense_shape(self):
+    column = fc.weighted_categorical_column(
+        categorical_column=fc.categorical_column_with_identity(
+            key='ids', num_buckets=3),
+        weight_feature_key='values')
+    with ops.Graph().as_default():
+      predictions = get_keras_linear_model_predictions({
+          'ids':
+              sparse_tensor.SparseTensorValue(
+                  indices=((0, 0), (1, 0), (1, 1)),
+                  values=(0, 2, 1),
+                  dense_shape=(2, 2)),
+          'values': ((.5,), (1.,), (.1,))
+      }, (column,))
+      bias = get_keras_linear_model_bias()
+      weight_var = get_linear_model_column_var(column)
+      with _initialized_session():
+        self.assertAllClose((0.,), bias.eval())
+        self.assertAllClose(((0.,), (0.,), (0.,)), weight_var.eval())
+        self.assertAllClose(((0.,), (0.,)), predictions.eval())
+        weight_var.assign(((1.,), (2.,), (3.,))).eval()
+        # weight_var[0] * weights[0, 0] = 1 * .5 = .5
+        # weight_var[2] * weights[1, 0] + weight_var[1] * weights[1, 1]
+        # = 3*1 + 2*.1 = 3+.2 = 3.2
+        self.assertAllClose(((.5,), (3.2,)), predictions.eval())
+
   def test_linear_model(self):
     column = fc.weighted_categorical_column(
         categorical_column=fc.categorical_column_with_identity(
diff --git a/tensorflow/python/layers/base.py b/tensorflow/python/layers/base.py
index 242cdff6f3..ec741d3265 100644
--- a/tensorflow/python/layers/base.py
+++ b/tensorflow/python/layers/base.py
@@ -694,7 +694,8 @@ class Layer(checkpointable.CheckpointableBase):
               self._dtype = input_list[0].dtype.base_dtype.name
             except AttributeError:
               pass
-          input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs)
+          if all(hasattr(x, 'get_shape') for x in input_list):
+            input_shapes = nest.map_structure(lambda x: x.get_shape(), inputs)
           self.build(input_shapes)
         try:
           # Note: not all sub-classes of Layer call Layer.__init__ (especially
-- 
GitLab


From 1093a54cb79be1dd606eee9ff27b718006ba9d63 Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 4 Apr 2018 15:15:32 -0700
Subject: [PATCH 925/960] Iterate over a copy of dictionary keys when closing
 variable subscopes. Otherwise, we run into a "dictionary changed size during
 iteration" once in a while, as we are modifying the values in the dictionary
 during the iteration.

PiperOrigin-RevId: 191655599
---
 tensorflow/python/ops/variable_scope.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/variable_scope.py b/tensorflow/python/ops/variable_scope.py
index c35735ca65..e33085ba62 100644
--- a/tensorflow/python/ops/variable_scope.py
+++ b/tensorflow/python/ops/variable_scope.py
@@ -1164,7 +1164,7 @@ class _VariableScopeStore(threading.local):
       self.variable_scopes_count[scope_name] = 1
 
   def close_variable_subscopes(self, scope_name):
-    for k in self.variable_scopes_count:
+    for k in list(self.variable_scopes_count.keys()):
       if not scope_name or k.startswith(scope_name + "/"):
         self.variable_scopes_count[k] = 0
 
-- 
GitLab


From b3081c19c7ae4966400cf5073043bfb65ed4f2b8 Mon Sep 17 00:00:00 2001
From: Daniel Zheng <danielzheng@google.com>
Date: Wed, 4 Apr 2018 15:28:01 -0700
Subject: [PATCH 926/960] Fix typo in `tf.reduce_mean` documentation.

The range of valid values for the `axis` argument should be `[-rank(input), rank(input))`, just like other reduction ops.

PiperOrigin-RevId: 191657767
---
 tensorflow/python/ops/math_ops.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/python/ops/math_ops.py b/tensorflow/python/ops/math_ops.py
index 39f40882db..b460ce5b95 100644
--- a/tensorflow/python/ops/math_ops.py
+++ b/tensorflow/python/ops/math_ops.py
@@ -1521,7 +1521,7 @@ def reduce_mean(input_tensor,
     input_tensor: The tensor to reduce. Should have numeric type.
     axis: The dimensions to reduce. If `None` (the default),
       reduces all dimensions. Must be in the range
-      `[-rank(input_tensor), rank(input_tensor)]`.
+      `[-rank(input_tensor), rank(input_tensor))`.
     keepdims: If true, retains reduced dimensions with length 1.
     name: A name for the operation (optional).
     reduction_indices: The old (deprecated) name for axis.
-- 
GitLab


From e7ad6ec4267f1f79ee7d9f558c8a008746682959 Mon Sep 17 00:00:00 2001
From: Smit Hinsu <hinsu@google.com>
Date: Wed, 4 Apr 2018 15:36:52 -0700
Subject: [PATCH 927/960] Remove dependency on absl to fix tf-nightly-gpu
 windows GPU build

PiperOrigin-RevId: 191659091
---
 tensorflow/stream_executor/BUILD                | 1 -
 tensorflow/stream_executor/cuda/cuda_dnn.cc     | 7 +++----
 tensorflow/stream_executor/cuda/cudnn_version.h | 7 ++++---
 3 files changed, 7 insertions(+), 8 deletions(-)

diff --git a/tensorflow/stream_executor/BUILD b/tensorflow/stream_executor/BUILD
index 27cdb860fe..1913fc20ee 100644
--- a/tensorflow/stream_executor/BUILD
+++ b/tensorflow/stream_executor/BUILD
@@ -75,7 +75,6 @@ cc_library(
         ":stream_executor",
         "//tensorflow/core:lib",
         "//tensorflow/core/kernels:ops_util",
-        "@com_google_absl//absl/strings",
         "@local_config_cuda//cuda:cuda_headers",
     ] + if_cuda_is_configured([
         "//tensorflow/core:cuda",
diff --git a/tensorflow/stream_executor/cuda/cuda_dnn.cc b/tensorflow/stream_executor/cuda/cuda_dnn.cc
index 1aea0485fd..f408c06f46 100644
--- a/tensorflow/stream_executor/cuda/cuda_dnn.cc
+++ b/tensorflow/stream_executor/cuda/cuda_dnn.cc
@@ -18,7 +18,6 @@ limitations under the License.
 #include <functional>
 #include <memory>
 
-#include "absl/strings/str_cat.h"
 #include "third_party/eigen3/Eigen/Core"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/util/env_var.h"
@@ -113,7 +112,7 @@ string ToString(libraryPropertyType type) {
     case PATCH_LEVEL:
       return "PATCH_LEVEL";
     default:
-      return absl::StrCat(
+      return port::StrCat(
           "<unknown libraryPropertyType: ", static_cast<int>(type), ">");
   }
 }
@@ -375,7 +374,7 @@ port::Status GetCudnnProperty(libraryPropertyType type, int* value) {
   cudnnStatus_t status = cudnnGetProperty(type, value);
   if (status != CUDNN_STATUS_SUCCESS) {
     const string error =
-        absl::StrCat("cudnnGetProperty failed for type: ", ToString(type),
+        port::StrCat("cudnnGetProperty failed for type: ", ToString(type),
                      " with status: ", ToString(status));
     LOG(ERROR) << error;
     return port::Status{port::error::INTERNAL, error};
@@ -419,7 +418,7 @@ port::Status CudnnSupport::Init() {
     CudnnVersion loaded_version;
     TF_RETURN_IF_ERROR(GetLoadedCudnnVersion(&loaded_version));
     if (!IsSourceCompatibleWithCudnnLibrary(source_version, loaded_version)) {
-      const tensorflow::string error = absl::StrCat(
+      const tensorflow::string error = port::StrCat(
           "Loaded runtime CuDNN library: ", loaded_version.ToString(),
           " but source was compiled with: ", source_version.ToString(),
           ".  CuDNN library major and minor version needs to match or have "
diff --git a/tensorflow/stream_executor/cuda/cudnn_version.h b/tensorflow/stream_executor/cuda/cudnn_version.h
index 058cc87bfa..2ed02e1700 100644
--- a/tensorflow/stream_executor/cuda/cudnn_version.h
+++ b/tensorflow/stream_executor/cuda/cudnn_version.h
@@ -18,7 +18,7 @@ limitations under the License.
 
 #include <string>
 
-#include "absl/strings/str_join.h"
+#include "tensorflow/core/lib/strings/strcat.h"
 
 namespace perftools {
 namespace gputools {
@@ -30,8 +30,9 @@ struct CudnnVersion {
   CudnnVersion(int major, int minor, int patch)
       : major_version(major), minor_version(minor), patch_level(patch) {}
 
-  std::string ToString() const {
-    return absl::StrJoin({major_version, minor_version, patch_level}, ".");
+  tensorflow::string ToString() const {
+    return tensorflow::strings::StrCat(major_version, ".", minor_version, ".",
+                                       patch_level);
   }
 
   int major_version;
-- 
GitLab


From 4cfb393b087dc50c150054531186ccb71882e2d0 Mon Sep 17 00:00:00 2001
From: Alexandre Passos <apassos@google.com>
Date: Wed, 4 Apr 2018 15:42:14 -0700
Subject: [PATCH 928/960] Adding Operation._control_outputs

PiperOrigin-RevId: 191659944
---
 tensorflow/python/client/tf_session.i         | 19 +++++++++++++++
 tensorflow/python/client/tf_session_helper.cc |  9 +++++++
 tensorflow/python/client/tf_session_helper.h  |  4 ++++
 tensorflow/python/framework/ops.py            | 24 +++++++++++++++++++
 tensorflow/python/framework/ops_test.py       |  2 ++
 5 files changed, 58 insertions(+)

diff --git a/tensorflow/python/client/tf_session.i b/tensorflow/python/client/tf_session.i
index 0c18d973a7..b82182d5d3 100644
--- a/tensorflow/python/client/tf_session.i
+++ b/tensorflow/python/client/tf_session.i
@@ -157,6 +157,25 @@ tensorflow::ImportNumpy();
   }
 }
 
+// We use TF_OperationGetControlOutputs_wrapper instead of
+// TF_OperationGetControlOutputs
+%ignore TF_OperationGetControlOutputs;
+%unignore TF_OperationGetControlOutputs_wrapper;
+// See comment for "%noexception TF_SessionRun_wrapper;"
+%noexception TF_OperationGetControlOutputs_wrapper;
+
+// Build a Python list of TF_Operation* and return it.
+%typemap(out) std::vector<TF_Operation*> tensorflow::TF_OperationGetControlOutputs_wrapper {
+  $result = PyList_New($1.size());
+  if (!$result) {
+    SWIG_exception_fail(SWIG_MemoryError, "$symname: couldn't create list");
+  }
+
+  for (size_t i = 0; i < $1.size(); ++i) {
+    PyList_SET_ITEM($result, i, CreateWrappedTFOperation($1[i]));
+  }
+}
+
 %ignore TF_OperationOutputConsumers;
 %unignore TF_OperationOutputConsumers_wrapper;
 // See comment for "%noexception TF_SessionRun_wrapper;"
diff --git a/tensorflow/python/client/tf_session_helper.cc b/tensorflow/python/client/tf_session_helper.cc
index ca57abd712..b48d758e4a 100644
--- a/tensorflow/python/client/tf_session_helper.cc
+++ b/tensorflow/python/client/tf_session_helper.cc
@@ -550,6 +550,15 @@ std::vector<TF_Operation*> TF_OperationGetControlInputs_wrapper(
   return control_inputs;
 }
 
+std::vector<TF_Operation*> TF_OperationGetControlOutputs_wrapper(
+    TF_Operation* oper) {
+  std::vector<TF_Operation*> control_outputs(
+      TF_OperationNumControlOutputs(oper));
+  TF_OperationGetControlOutputs(oper, control_outputs.data(),
+                                control_outputs.size());
+  return control_outputs;
+}
+
 std::vector<const char*> TF_OperationOutputConsumers_wrapper(
     TF_Output oper_out) {
   int num_consumers = TF_OperationOutputNumConsumers(oper_out);
diff --git a/tensorflow/python/client/tf_session_helper.h b/tensorflow/python/client/tf_session_helper.h
index 5416d41376..d2b4abc476 100644
--- a/tensorflow/python/client/tf_session_helper.h
+++ b/tensorflow/python/client/tf_session_helper.h
@@ -190,6 +190,10 @@ std::vector<TF_Output> GetOperationInputs(TF_Operation* oper);
 std::vector<TF_Operation*> TF_OperationGetControlInputs_wrapper(
     TF_Operation* oper);
 
+// Retrieves the control outputs of this operation.
+std::vector<TF_Operation*> TF_OperationGetControlOutputs_wrapper(
+    TF_Operation* oper);
+
 // Retrieves the op names of the consumers of `oper_out`. The returned strings
 // have the lifetime of the underlying TF_Graph.
 std::vector<const char*> TF_OperationOutputConsumers_wrapper(
diff --git a/tensorflow/python/framework/ops.py b/tensorflow/python/framework/ops.py
index 0215501b56..2d55f98a1c 100644
--- a/tensorflow/python/framework/ops.py
+++ b/tensorflow/python/framework/ops.py
@@ -2113,6 +2113,30 @@ class Operation(object):
     else:
       return self._control_inputs_val
 
+  @property
+  def _control_outputs(self):
+    """The `Operation` objects which have a control dependency on this op.
+
+    Before any of the ops in self._control_outputs can execute tensorflow will
+    ensure self has finished executing.
+
+    Returns:
+      A list of `Operation` objects.
+
+    """
+    if self._c_op:
+      control_c_ops = c_api.TF_OperationGetControlOutputs_wrapper(self._c_op)
+      # pylint: disable=protected-access
+      return [
+          self.graph._get_operation_by_name_unsafe(
+              c_api.TF_OperationName(c_op)) for c_op in control_c_ops
+      ]
+      # pylint: enable=protected-access
+    else:
+      # TODO(apassos) this should be less inefficient.
+      return [o for o in self._graph.get_operations()
+              if self in o.control_inputs]
+
   @property
   def _control_inputs(self):
     logging.warning("Operation._control_inputs is private, use "
diff --git a/tensorflow/python/framework/ops_test.py b/tensorflow/python/framework/ops_test.py
index aa51391871..58bead91ed 100644
--- a/tensorflow/python/framework/ops_test.py
+++ b/tensorflow/python/framework/ops_test.py
@@ -473,6 +473,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(z.control_inputs, [x, x])
     z._add_control_inputs([x, y, y])  # pylint: disable=protected-access
     self.assertEqual(z.control_inputs, [x, x, x, y, y])
+    self.assertEqual(x._control_outputs, [z])
 
   def testAddControlInputC(self):
     # The C API dedups redundant control edges, pure Python does not
@@ -487,6 +488,7 @@ class OperationTest(test_util.TensorFlowTestCase):
     self.assertEqual(z.control_inputs, [x])
     z._add_control_inputs([x, y, y])  # pylint: disable=protected-access
     self.assertEqual(z.control_inputs, [x, y])
+    self.assertEqual(x._control_outputs, [z])
 
   def testRemoveAllControlInputs(self):
     a = constant_op.constant(1)
-- 
GitLab


From c8e3d2b43e4cbf9a9e32567a2e59597916f5b0b9 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 15:45:16 -0700
Subject: [PATCH 929/960] Adds commandline option (toco_compatible, bool) that
 makes the optimize_for_inference script only use TOCO friendly ops. In
 particular, FusedResizeAndPadConv2D is not supported by TOCO.

This change does not alter existing behavior (the boolean is set to false by default).

PiperOrigin-RevId: 191660378
---
 tensorflow/python/tools/optimize_for_inference.py    | 12 +++++++++++-
 .../python/tools/optimize_for_inference_lib.py       |  9 ++++++---
 2 files changed, 17 insertions(+), 4 deletions(-)

diff --git a/tensorflow/python/tools/optimize_for_inference.py b/tensorflow/python/tools/optimize_for_inference.py
index 902748d55e..dac6a06a89 100644
--- a/tensorflow/python/tools/optimize_for_inference.py
+++ b/tensorflow/python/tools/optimize_for_inference.py
@@ -87,7 +87,9 @@ def main(unused_args):
   output_graph_def = optimize_for_inference_lib.optimize_for_inference(
       input_graph_def,
       FLAGS.input_names.split(","),
-      FLAGS.output_names.split(","), FLAGS.placeholder_type_enum)
+      FLAGS.output_names.split(","),
+      FLAGS.placeholder_type_enum,
+      FLAGS.toco_compatible)
 
   if FLAGS.frozen_graph:
     f = gfile.FastGFile(FLAGS.output, "w")
@@ -138,6 +140,14 @@ def parse_args():
       type=int,
       default=dtypes.float32.as_datatype_enum,
       help="The AttrValue enum to use for placeholders.")
+  parser.add_argument(
+      "--toco_compatible",
+      type=bool,
+      default=False,
+      help="""\
+      If true, only use ops compatible with Tensorflow
+      Lite Optimizing Converter.\
+      """)
   return parser.parse_known_args()
 
 
diff --git a/tensorflow/python/tools/optimize_for_inference_lib.py b/tensorflow/python/tools/optimize_for_inference_lib.py
index 9c19271222..bb90d1cd6e 100644
--- a/tensorflow/python/tools/optimize_for_inference_lib.py
+++ b/tensorflow/python/tools/optimize_for_inference_lib.py
@@ -87,7 +87,7 @@ EPSILON_ATTR = {
 
 
 def optimize_for_inference(input_graph_def, input_node_names, output_node_names,
-                           placeholder_type_enum):
+                           placeholder_type_enum, toco_compatible=False):
   """Applies a series of inference optimizations on the input graph.
 
   Args:
@@ -98,6 +98,8 @@ def optimize_for_inference(input_graph_def, input_node_names, output_node_names,
       results.
     placeholder_type_enum: The AttrValue enum for the placeholder data type, or
         a list that specifies one value per input node name.
+    toco_compatible: Boolean, if True, only runs optimizations that result in
+      TOCO compatible graph operations (default=False).
 
   Returns:
     An optimized version of the input graph.
@@ -110,8 +112,9 @@ def optimize_for_inference(input_graph_def, input_node_names, output_node_names,
   optimized_graph_def = graph_util.remove_training_nodes(
       optimized_graph_def, output_node_names)
   optimized_graph_def = fold_batch_norms(optimized_graph_def)
-  optimized_graph_def = fuse_resize_and_conv(optimized_graph_def,
-                                             output_node_names)
+  if not toco_compatible:
+    optimized_graph_def = fuse_resize_and_conv(optimized_graph_def,
+                                               output_node_names)
   ensure_graph_is_valid(optimized_graph_def)
   return optimized_graph_def
 
-- 
GitLab


From 1bba94af33aca56c3a2240302d80f44d65e6aa17 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 15:45:20 -0700
Subject: [PATCH 930/960] Compile TensorFlow with /arch:AVX on Windows

/arch:AVX is the correpsonding option in MSVC for gcc's --march=native

PiperOrigin-RevId: 191660389
---
 configure.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/configure.py b/configure.py
index 0f52c0ec99..26eff5767e 100644
--- a/configure.py
+++ b/configure.py
@@ -484,6 +484,8 @@ def set_cc_opt_flags(environ_cp):
   if is_ppc64le():
     # gcc on ppc64le does not support -march, use mcpu instead
     default_cc_opt_flags = '-mcpu=native'
+  elif is_windows():
+    default_cc_opt_flags = '/arch:AVX'
   else:
     default_cc_opt_flags = '-march=native'
   question = ('Please specify optimization flags to use during compilation when'
@@ -494,7 +496,7 @@ def set_cc_opt_flags(environ_cp):
   for opt in cc_opt_flags.split():
     write_to_bazelrc('build:opt --copt=%s' % opt)
   # It should be safe on the same build host.
-  if not is_ppc64le():
+  if not is_ppc64le() and not is_windows():
     write_to_bazelrc('build:opt --host_copt=-march=native')
   write_to_bazelrc('build:opt --define with_default_optimizations=true')
   # TODO(mikecase): Remove these default defines once we are able to get
-- 
GitLab


From f034e6c457ab283dbe42b1bf561943c9fa5dffe7 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 15:46:26 -0700
Subject: [PATCH 931/960] Internal change.

PiperOrigin-RevId: 191660588
---
 tensorflow/contrib/distributions/BUILD | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/tensorflow/contrib/distributions/BUILD b/tensorflow/contrib/distributions/BUILD
index 514638ecbb..9799901483 100644
--- a/tensorflow/contrib/distributions/BUILD
+++ b/tensorflow/contrib/distributions/BUILD
@@ -502,12 +502,6 @@ cuda_py_test(
         "//tensorflow/python:client_testlib",
     ],
     shard_count = 4,
-    tags = [
-        "manual",
-        "noasan",
-        "noguitar",
-        "optonly",
-    ],
 )
 
 cuda_py_test(
-- 
GitLab


From 7cee71e28e98bac613623feea19c4a51439e9a0a Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 16:05:08 -0700
Subject: [PATCH 932/960] Enable constant propagation across Enter nodes, but
 only if is_constant is true. Don't propagate constants with control
 dependencies through Merge nodes.

PiperOrigin-RevId: 191663396
---
 .../grappler/optimizers/constant_folding.cc   | 35 +++++-----
 .../optimizers/constant_folding_test.cc       | 50 ++++++++++++--
 tensorflow/python/BUILD                       | 23 +++++++
 .../python/grappler/constant_folding_test.py  | 69 +++++++++++++++++++
 4 files changed, 153 insertions(+), 24 deletions(-)
 create mode 100644 tensorflow/python/grappler/constant_folding_test.py

diff --git a/tensorflow/core/grappler/optimizers/constant_folding.cc b/tensorflow/core/grappler/optimizers/constant_folding.cc
index dd522aa228..d941a0b3f9 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding.cc
@@ -773,7 +773,7 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const {
   // the case of a merge node that propagate the first inputs that becomes
   // available, and therefore only requires a single constant input to be
   // foldable.
-  bool has_constant_input = false;
+  bool merge_has_constant_input = false;
   const bool is_merge = IsMerge(node);
   for (const auto& input : node.input()) {
     if (IsControlInput(input)) {
@@ -784,21 +784,20 @@ bool ConstantFolding::IsFoldable(const NodeDef& node) const {
       return false;
     }
     bool is_const = IsReallyConstant(*input_node);
-    if (!is_const && !is_merge) {
-      return false;
-    }
-    // Don't fold strings constants for now since this causes problems with
-    // checkpointing.
-    if (is_const && input_node->attr().at("dtype").type() == DT_STRING) {
+    if (is_const) {
+      // Don't fold strings constants for now since this causes problems with
+      // checkpointing.
+      if (input_node->attr().at("dtype").type() == DT_STRING) {
+        return false;
+      }
+      // Special case: If a Merge node has at least one constant input that
+      // does not depend on a control input, we can fold it.
+      merge_has_constant_input |= !HasControlInputs(*input_node);
+    } else if (!is_merge) {
       return false;
     }
-    has_constant_input |= is_const;
-  }
-  if (is_merge) {
-    return has_constant_input;
   }
-
-  return true;
+  return !is_merge || merge_has_constant_input;
 }
 
 namespace {
@@ -1714,9 +1713,11 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
     }
 
     // Move constants past Enter.
-    // TODO(rmlarsen): Reenable when we fix the root cause of b/76008022
-    if (opt_level_ == RewriterConfig::AGGRESSIVE && IsEnter(*node) &&
-        node->input_size() > 0) {
+    if (IsEnter(*node) && node->input_size() > 0) {
+      if (node->attr().count("is_constant") == 0 ||
+          !node->attr().at("is_constant").b()) {
+        continue;
+      }
       const string& node_name = node->name();
       const NodeDef* input = node_map_->GetNode(node->input(0));
       if (input != nullptr && IsReallyConstant(*input) &&
@@ -1745,7 +1746,7 @@ Status ConstantFolding::SimplifyGraph(GraphDef* optimized_graph,
           node_map_->AddOutput(node_name, new_node->name());
           for (NodeDef* consumer : consumers) {
             for (int i = 0; i < consumer->input_size(); ++i) {
-              if (consumer->input(i) == node_name) {
+              if (NodeName(consumer->input(i)) == node_name) {
                 node_map_->UpdateInput(consumer->name(), node_name,
                                        new_node->name());
                 consumer->set_input(i, new_node->name());
diff --git a/tensorflow/core/grappler/optimizers/constant_folding_test.cc b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
index 8d146637a6..71ee81dfde 100644
--- a/tensorflow/core/grappler/optimizers/constant_folding_test.cc
+++ b/tensorflow/core/grappler/optimizers/constant_folding_test.cc
@@ -1256,6 +1256,10 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   ops::Merge m1(scope.WithOpName("m1"), {x, const1, const2});
   ops::Merge m2(scope.WithOpName("m2"), {const1, const3});
   ops::Merge m3(scope.WithOpName("m3"), {x, y});
+  // m4 is not foldable because the only constant input
+  // has a control input, so we cannot know if it will be
+  // triggered.
+  ops::Merge m4(scope.WithOpName("m4"), {x, const1});
 
   ops::Identity out1(scope.WithOpName("out1"), m1.output);
   ops::Identity idx1(scope.WithOpName("idx1"), m1.value_index);
@@ -1263,9 +1267,11 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   ops::Identity idx2(scope.WithOpName("idx2"), m2.value_index);
   ops::Identity out3(scope.WithOpName("out3"), m3.output);
   ops::Identity idx3(scope.WithOpName("idx3"), m3.value_index);
+  ops::Identity out4(scope.WithOpName("out4"), m4.output);
+  ops::Identity idx4(scope.WithOpName("idx4"), m4.value_index);
 
   GrapplerItem item;
-  item.fetch = {"out1", "idx1", "out2", "idx2", "out3", "idx3"};
+  item.fetch = {"out1", "idx1", "out2", "idx2", "out3", "idx3", "out4", "idx4"};
   TF_CHECK_OK(scope.ToGraphDef(&item.graph));
 
   ConstantFolding optimizer(nullptr /* cpu_device */);
@@ -1273,6 +1279,7 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
+  EXPECT_EQ(19, output.node_size());
   int found_nodes = 0;
   for (const auto& node : output.node()) {
     if (node.name() == "out1") {
@@ -1309,10 +1316,18 @@ TEST_F(ConstantFoldingTest, MergeNodes) {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("m3:1", node.input(0));
       ++found_nodes;
+    } else if (node.name() == "out4") {
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("m4", node.input(0));
+      ++found_nodes;
+    } else if (node.name() == "idx4") {
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("m4:1", node.input(0));
+      ++found_nodes;
     }
   }
   // Make sure the graph contains all the nodes we're expecting.
-  EXPECT_EQ(6, found_nodes);
+  EXPECT_EQ(8, found_nodes);
 
   std::vector<string> fetch = {"out1", "idx1"};
   auto tensors = EvaluateNodes(output, fetch);
@@ -2320,6 +2335,10 @@ TEST_F(ConstantFoldingTest, Enter) {
   GrapplerItem item;
   AttrValue frame_name;
   frame_name.set_s("foo");
+  AttrValue is_constant_true;
+  is_constant_true.set_b(true);
+  AttrValue is_constant_false;
+  is_constant_false.set_b(false);
   AttrValue type;
   type.set_type(DT_FLOAT);
   AttrValue value;
@@ -2330,19 +2349,31 @@ TEST_F(ConstantFoldingTest, Enter) {
   GraphDef& graph = item.graph;
   AddNode("x", "Placeholder", {}, {{"T", type}}, &graph);
   AddNode("c1", "Const", {"^x"}, {{"value", value}, {"dtype", type}}, &graph);
-  AddNode("enter1", "Enter", {"x"}, {{"T", type}, {"frame_name", frame_name}},
+  AddNode("enter1", "Enter", {"x"},
+          {{"T", type},
+           {"frame_name", frame_name},
+           {"is_constant", is_constant_true}},
+          &graph);
+  AddNode("enter2", "Enter", {"c1"},
+          {{"T", type},
+           {"frame_name", frame_name},
+           {"is_constant", is_constant_true}},
           &graph);
-  AddNode("enter2", "Enter", {"c1"}, {{"T", type}, {"frame_name", frame_name}},
+  AddNode("enter3", "Enter", {"c1"},
+          {{"T", type},
+           {"frame_name", frame_name},
+           {"is_constant", is_constant_false}},
           &graph);
   AddNode("id1", "Identity", {"enter1"}, {{"T", type}}, &graph);
   AddNode("id2", "Identity", {"enter2"}, {{"T", type}}, &graph);
   AddNode("id3", "Identity", {"enter2"}, {{"T", type}}, &graph);
+  AddNode("id4", "Identity", {"enter3"}, {{"T", type}}, &graph);
   item.fetch.push_back("id1");
   item.fetch.push_back("id2");
   item.fetch.push_back("id3");
+  item.fetch.push_back("id4");
 
-  ConstantFolding optimizer(RewriterConfig::AGGRESSIVE,
-                            nullptr /* cpu_device */);
+  ConstantFolding optimizer(nullptr /* cpu_device */);
   GraphDef output;
   Status status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
@@ -2351,7 +2382,7 @@ TEST_F(ConstantFoldingTest, Enter) {
   status = optimizer.Optimize(nullptr, item, &output);
   TF_EXPECT_OK(status);
 
-  EXPECT_EQ(7, output.node_size());
+  EXPECT_EQ(9, output.node_size());
   for (const NodeDef& node : output.node()) {
     if (node.name() == "id1") {
       EXPECT_EQ("Identity", node.op());
@@ -2363,6 +2394,11 @@ TEST_F(ConstantFoldingTest, Enter) {
       EXPECT_EQ(1, node.input_size());
       EXPECT_EQ("^enter2", node.input(0));
     }
+    if (node.name() == "id4") {
+      EXPECT_EQ("Identity", node.op());
+      EXPECT_EQ(1, node.input_size());
+      EXPECT_EQ("enter3", node.input(0));
+    }
   }
 }
 
diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 57b0b78c82..a9363608b5 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -4843,6 +4843,29 @@ py_test(
     ],
 )
 
+cuda_py_test(
+    name = "constant_folding_test",
+    size = "medium",
+    srcs = [
+        "grappler/constant_folding_test.py",
+    ],
+    additional_deps = [
+        ":client_testlib",
+        ":framework_for_generated_wrappers",
+        ":array_ops",
+        ":control_flow_ops",
+        ":dtypes",
+        ":functional_ops",
+        ":math_ops",
+        ":ops",
+        "//third_party/py/numpy",
+        "//tensorflow/core:protos_all_py",
+    ],
+    tags = [
+        "grappler",
+    ],
+)
+
 cuda_py_test(
     name = "layout_optimizer_test",
     size = "medium",
diff --git a/tensorflow/python/grappler/constant_folding_test.py b/tensorflow/python/grappler/constant_folding_test.py
new file mode 100644
index 0000000000..ab1d0ed25b
--- /dev/null
+++ b/tensorflow/python/grappler/constant_folding_test.py
@@ -0,0 +1,69 @@
+# Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for Grappler Constant Folding."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+import numpy as np
+
+from tensorflow.python.client import session
+from tensorflow.python.framework import dtypes
+from tensorflow.python.framework import ops
+from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import control_flow_ops
+from tensorflow.python.ops import functional_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.platform import test
+
+
+class ConstantFoldingTest(test.TestCase):
+
+  # See b/76008022.
+  def testScanInsideWhile(self):
+
+    def loop_cond(idx_step, *unused_args):
+      return idx_step < 1
+
+    def loop_body(idx_step, y):
+      x = array_ops.zeros([10, 20, 30], dtype=dtypes.float32)
+      x = functional_ops.scan(
+          math_ops.add,
+          x,
+          initializer=array_ops.zeros([20, 30], dtype=dtypes.float32),
+          back_prop=False,
+          parallel_iterations=1)
+
+      with ops.device('/cpu:0'):
+        y = array_ops.identity(x)
+
+        return idx_step + 1, y
+
+    if test.is_gpu_available(cuda_only=True):
+      init_y = array_ops.zeros([10, 20, 30], dtype=dtypes.float32)
+      _, y = control_flow_ops.while_loop(
+          loop_cond,
+          loop_body,
+          loop_vars=[0, init_y],
+          back_prop=False,
+          parallel_iterations=1)
+      with session.Session() as sess:
+        y_v = sess.run(y)
+        self.assertAllEqual(np.zeros([10, 20, 30]), y_v)
+
+
+if __name__ == '__main__':
+  test.main()
-- 
GitLab


From e98c13c55e519cb70ede110cd8941f8cb75ab718 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 16:17:05 -0700
Subject: [PATCH 933/960] Running sparse_ops_test only in opt mode since the
 test is flaky (times out) in fastbuild mode.

PiperOrigin-RevId: 191665014
---
 tensorflow/python/kernel_tests/BUILD | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/tensorflow/python/kernel_tests/BUILD b/tensorflow/python/kernel_tests/BUILD
index a544e4fa6e..6c34ea1816 100644
--- a/tensorflow/python/kernel_tests/BUILD
+++ b/tensorflow/python/kernel_tests/BUILD
@@ -2519,7 +2519,10 @@ cuda_py_test(
         "//tensorflow/python:sparse_ops",
     ],
     shard_count = 5,
-    tags = ["noasan"],
+    tags = [
+        "noasan",
+        "optonly",  # b/77589990
+    ],
 )
 
 cuda_py_test(
-- 
GitLab


From f8acfb01792886274778d9ad7a9d990cbef14141 Mon Sep 17 00:00:00 2001
From: Benoit Steiner <bsteiner@google.com>
Date: Wed, 4 Apr 2018 16:17:46 -0700
Subject: [PATCH 934/960] Fixed handling of control dependencies in the
 arithmethic optimizer

PiperOrigin-RevId: 191665098
---
 .../optimizers/arithmetic_optimizer.cc        | 110 ++++++++----------
 .../optimizers/arithmetic_optimizer.h         |  12 +-
 .../optimizers/arithmetic_optimizer_test.cc   |   9 +-
 .../optimizers/graph_optimizer_stage.h        |  10 +-
 .../optimizers/graph_optimizer_stage_test.cc  |  12 +-
 5 files changed, 64 insertions(+), 89 deletions(-)

diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
index 919f23fd98..59a5695af0 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.cc
@@ -34,7 +34,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/graph_optimizer_stage.h"
 #include "tensorflow/core/grappler/optimizers/symbolic_shapes.h"
 #include "tensorflow/core/grappler/utils.h"
-#include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/errors.h"
 #include "tensorflow/core/lib/core/stringpiece.h"
@@ -290,21 +289,16 @@ class ArithmeticOptimizerStage : public GraphOptimizerStage<string> {
 
   // TODO(ezhulenev): remove this method from ArithmeticOptimizer when all
   // optimizations will be migrated to stages
-  void AddFrameControlDeps(const NodeDef* old_node,
-                           const std::vector<NodeDef*>& new_nodes,
-                           const string& source_for_ctrl_dep,
-                           const std::vector<NodeDef*>& sinks_for_control_dep) {
-    const auto frame_it = ctx_.frame_map->find(old_node);
-    if (frame_it != ctx_.frame_map->end()) {
-      for (auto node : new_nodes) {
-        ctx_.frame_map->emplace(node, frame_it->second);
-      }
-      if (!source_for_ctrl_dep.empty() && !sinks_for_control_dep.empty()) {
-        const string ctrl_dep = ConstantFolding::AddControlDependency(
-            source_for_ctrl_dep, ctx_.optimized_graph, ctx_.node_map);
-        for (auto node : sinks_for_control_dep) {
-          MaybeAddControlInput(ctrl_dep, node, ctx_.optimized_graph,
-                               ctx_.node_map);
+  void ForwardControlDependencies(
+      NodeDef* target_node, const std::vector<const NodeDef*>& src_nodes) {
+    for (const auto& src : src_nodes) {
+      for (int i = src->input_size() - 1; i >= 0; --i) {
+        if (IsControlInput(src->input(i))) {
+          *target_node->add_input() = src->input(i);
+          ctx_.node_map->AddOutput(NodeName(src->input(i)),
+                                   target_node->name());
+        } else {
+          break;
         }
       }
     }
@@ -703,7 +697,8 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
     CHECK(IsSupported(node));
 
     std::set<string> common_factors;
-    TF_RETURN_IF_ERROR(GetCommonFactors(node, &common_factors));
+    std::vector<string> ctrl_deps;
+    TF_RETURN_IF_ERROR(GetCommonFactors(node, &common_factors, &ctrl_deps));
 
     if (common_factors.size() == 1) {
       const string& common_factor = *common_factors.begin();
@@ -735,9 +730,11 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
           new_add_node->set_input(i, unique_factors[i]);
         }
 
-        // Add frame dependencies that the original node might have had.
-        AddFrameControlDeps(node, {new_add_node, new_mul_node}, common_factor,
-                            {new_add_node});
+        // Add control deps on add node
+        for (const string& ctrl_dep : ctrl_deps) {
+          *new_add_node->add_input() = ctrl_dep;
+          ctx_.node_map->AddOutput(NodeName(ctrl_dep), new_add_node->name());
+        }
 
         // optimize new inner aggregation node
         AddToOptimizationQueue(new_add_node);
@@ -763,14 +760,16 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
   }
 
   // Determine the set of common factors if the input nodes are all Mul nodes.
-  Status GetCommonFactors(const NodeDef* node,
-                          std::set<string>* common_factors) const {
+  Status GetCommonFactors(const NodeDef* node, std::set<string>* common_factors,
+                          std::vector<string>* ctrl_deps) const {
     CHECK(common_factors->empty());
 
     for (int i = 0; i < node->input_size(); ++i) {
       if (i > 0 && common_factors->empty()) break;
-      if (IsControlInput(node->input(i))) break;
-
+      if (IsControlInput(node->input(i))) {
+        ctrl_deps->push_back(node->input(i));
+        continue;
+      }
       NodeDef* input;
       TF_RETURN_IF_ERROR(GetInputNode(node->input(i), &input));
 
@@ -790,6 +789,9 @@ class HoistCommonFactorOutOfAggregation : public ArithmeticOptimizerStage {
             std::inserter(intersection, intersection.begin()));
         std::swap(*common_factors, intersection);
       }
+      for (int i = 2; i < input->input_size(); ++i) {
+        ctrl_deps->push_back(input->input(i));
+      }
     }
     return Status::OK();
   }
@@ -1275,20 +1277,15 @@ void ArithmeticOptimizer::DedupComputations() {
   }
 }
 
-void ArithmeticOptimizer::AddFrameControlDeps(
-    const NodeDef* old_node, const std::vector<NodeDef*>& new_nodes,
-    const string& source_for_ctrl_dep,
-    const std::vector<NodeDef*>& sinks_for_control_dep) {
-  const auto frame_it = frame_map_.find(old_node);
-  if (frame_it != frame_map_.end()) {
-    for (auto node : new_nodes) {
-      frame_map_.emplace(node, frame_it->second);
-    }
-    if (!source_for_ctrl_dep.empty() && !sinks_for_control_dep.empty()) {
-      const string ctrl_dep = ConstantFolding::AddControlDependency(
-          source_for_ctrl_dep, optimized_graph_, node_map_.get());
-      for (auto node : sinks_for_control_dep) {
-        MaybeAddControlInput(ctrl_dep, node, optimized_graph_, node_map_.get());
+void ArithmeticOptimizer::ForwardControlDependencies(
+    NodeDef* target_node, const std::vector<const NodeDef*>& src_nodes) {
+  for (const auto& src : src_nodes) {
+    for (int i = src->input_size() - 1; i >= 0; --i) {
+      if (IsControlInput(src->input(i))) {
+        *target_node->add_input() = src->input(i);
+        node_map_->AddOutput(NodeName(src->input(i)), target_node->name());
+      } else {
+        break;
       }
     }
   }
@@ -1408,10 +1405,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
           node_map_->AddOutput(new_transpose->name(), new_cast->name());
 
           nodes_to_simplify->PushBack(new_transpose);
-          //  Add frame dependencies that the original node might have had.
-          AddFrameControlDeps(node, {new_transpose, new_cast},
-                              new_transpose->input(0), {new_transpose});
-
+          ForwardControlDependencies(new_transpose, {cast, node});
           return new_cast->name();
         }
       }
@@ -1485,7 +1479,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
             node_map_->AddOutput(weights->name(), scaled_weights->name());
             scaled_weights->add_input(mul->input(1));
             node_map_->AddOutput(scale->name(), scaled_weights->name());
-            AddFrameControlDeps(node, {scaled_weights}, "", {});
+            ForwardControlDependencies(scaled_weights, {source});
 
             // Update `conv`'s weights to `scaled_weights`.
             conv->set_input(1, scaled_weights->name());
@@ -1521,7 +1515,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
   }
 
   if (IsAggregate(*node) && NumNonControlInputs(*node) > 0) {
-    // Discard aggregate nodes with a single input.
+    // Discard aggregate nodes with a single input and no control dependencies.
     if (node->input_size() == 1) {
       return node->input(0);
     }
@@ -1567,6 +1561,8 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
         return "";
       }
       new_const_node->set_device(node->device());
+      MaybeAddControlInput(NodeName(node->input(0)), new_const_node,
+                           optimized_graph_, node_map_.get());
       nodes_to_simplify->PushBack(new_const_node);
 
       // 2. Replace the aggregate node with Mul(Const(N), x).
@@ -1579,9 +1575,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
       new_mul_node->add_input(node->input(0));
       node_map_->AddOutput(node->input(0), new_mul_node->name());
 
-      CopyControlInputs(*node, new_mul_node, optimized_graph_, node_map_.get());
-      AddFrameControlDeps(node, {new_const_node, new_mul_node}, node->input(0),
-                          {new_const_node});
+      ForwardControlDependencies(new_mul_node, {node});
       return new_mul_node->name();
     }
   }
@@ -1614,7 +1608,6 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
         FlipBooleanAttr(attr_a, new_op);
         new_op->set_input(0, a->input(0));
         node_map_->UpdateInput(new_op->name(), a->name(), a->input(0));
-        AddFrameControlDeps(node, {new_op}, a->input(0), {new_op});
       }
       if (b_is_foldable) {
         const string attr_b =
@@ -1622,10 +1615,15 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
         FlipBooleanAttr(attr_b, new_op);
         new_op->set_input(1, b->input(0));
         node_map_->UpdateInput(new_op->name(), b->name(), b->input(0));
-        if (!a_is_foldable) {
-          AddFrameControlDeps(node, {new_op}, b->input(0), {new_op});
-        }
       }
+      std::vector<const NodeDef*> deps_to_forward({node});
+      if (a_is_foldable) {
+        deps_to_forward.push_back(a);
+      }
+      if (b_is_foldable) {
+        deps_to_forward.push_back(b);
+      }
+      ForwardControlDependencies(new_op, deps_to_forward);
     }
   }
 
@@ -1647,7 +1645,7 @@ string ArithmeticOptimizer::TrySimplifyAndReplaceUses(
                                                        : "Transpose");
       new_op->set_input(0, input->input(0));
       node_map_->UpdateInput(new_op->name(), node->name(), input->input(0));
-      AddFrameControlDeps(node, {new_op}, "", {});
+      ForwardControlDependencies(new_op, {node, input});
       return new_op->name();
     }
   }
@@ -1663,8 +1661,7 @@ Status ArithmeticOptimizer::SimplifyArithmeticOps() {
   }
 
   const GraphOptimizerContext ctx(&nodes_to_preserve_, optimized_graph_,
-                                  graph_properties_.get(), node_map_.get(),
-                                  &frame_map_);
+                                  graph_properties_.get(), node_map_.get());
   const ArithmeticOptimizerContext ctx_ext(&nodes_to_simplify);
 
   // Stop pipeline after first stage returning non-empty simplified tensor name.
@@ -1764,11 +1761,6 @@ Status ArithmeticOptimizer::Optimize(Cluster* /*cluster*/,
   graph_properties_.reset(new GraphProperties(item));
   TF_RETURN_IF_ERROR(graph_properties_->InferStatically(false));
 
-  // Identify loop frames
-  int num_frames;
-  TF_RETURN_IF_ERROR(IdentifyFramesWithNodeMap(*optimized_graph_, *node_map_,
-                                               &frame_map_, &num_frames));
-
   // Perform the optimizations.
   TF_RETURN_IF_ERROR(SimplifyArithmeticOps());
 
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
index 63a7b55893..7e81ed0a1f 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer.h
@@ -20,7 +20,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/optimizers/graph_optimizer.h"
 #include "tensorflow/core/grappler/utils.h"
-#include "tensorflow/core/grappler/utils/frame.h"
 #include "tensorflow/core/protobuf/rewriter_config.pb.h"
 
 namespace tensorflow {
@@ -100,13 +99,9 @@ class ArithmeticOptimizer : public GraphOptimizer {
   // Dedup redundant nodes in the graph.
   void DedupComputations();
 
-  // Fix frame dependencies by adding control dependencies from old_input to
-  // nodes in new_nodes_for_control_dep, and update frame_map for all nodes in
-  // new_nodes.
-  void AddFrameControlDeps(const NodeDef* old_node,
-                           const std::vector<NodeDef*>& new_nodes,
-                           const string& source_for_ctrl_dep,
-                           const std::vector<NodeDef*>& sinks_for_control_dep);
+  // Forward the control dependencies anchored on src_nodes to the target_nodes.
+  void ForwardControlDependencies(NodeDef* target_node,
+                                  const std::vector<const NodeDef*>& src_nodes);
 
   // Runs peep-hole optimizations on `optimized_graph`, e.g., removing inverse
   // transposes.
@@ -135,7 +130,6 @@ class ArithmeticOptimizer : public GraphOptimizer {
   bool fetch_nodes_known_ = false;
   std::unordered_set<string> nodes_to_preserve_;
   std::unique_ptr<NodeMap> node_map_;
-  FrameMap frame_map_;
   std::unique_ptr<GraphProperties> graph_properties_;
   GraphDef* optimized_graph_ = nullptr;  // Not owned.
 };
diff --git a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
index 48f1dd5aa1..e117341ba3 100644
--- a/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
+++ b/tensorflow/core/grappler/optimizers/arithmetic_optimizer_test.cc
@@ -520,26 +520,23 @@ TEST_F(ArithmeticOptimizerTest, TrivialSumsRepeatedAdd) {
 
   const NodeDef* add_6_node = node_map.GetNode(HoistAddName("Add_6"));
   ASSERT_NE(add_6_node, nullptr);
-  EXPECT_EQ(3, add_6_node->input_size());
+  EXPECT_EQ(2, add_6_node->input_size());
   EXPECT_EQ(HoistAddName("Add_4"), add_6_node->input(0));
   EXPECT_EQ(HoistAddName("Add_5"), add_6_node->input(1));
-  EXPECT_EQ("^Placeholder", add_6_node->input(2));
 
   const NodeDef* add_4_node = node_map.GetNode(HoistAddName("Add_4"));
   ASSERT_NE(add_4_node, nullptr);
   EXPECT_EQ("Add", add_4_node->op());
-  EXPECT_EQ(3, add_4_node->input_size());
+  EXPECT_EQ(2, add_4_node->input_size());
   EXPECT_EQ(OptimizedName("Add_const"), add_4_node->input(0));
   EXPECT_EQ(OptimizedName("Add_1_const"), add_4_node->input(1));
-  EXPECT_EQ("^Placeholder", add_4_node->input(2));
 
   const NodeDef* add_5_node = node_map.GetNode(HoistAddName("Add_5"));
   ASSERT_NE(add_5_node, nullptr);
   EXPECT_EQ("Add", add_5_node->op());
-  EXPECT_EQ(3, add_5_node->input_size());
+  EXPECT_EQ(2, add_5_node->input_size());
   EXPECT_EQ(OptimizedName("Add_const"), add_5_node->input(0));
   EXPECT_EQ(OptimizedName("Add_1_const"), add_5_node->input(1));
-  EXPECT_EQ("^Placeholder", add_5_node->input(2));
 
   const NodeDef* add_const_node = node_map.GetNode(OptimizedName("Add_const"));
   ASSERT_NE(add_const_node, nullptr);
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
index 8d3e965c57..7ed0474861 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage.h
@@ -21,7 +21,6 @@ limitations under the License.
 #include "tensorflow/core/grappler/costs/graph_properties.h"
 #include "tensorflow/core/grappler/grappler_item.h"
 #include "tensorflow/core/grappler/utils.h"
-#include "tensorflow/core/grappler/utils/frame.h"
 
 namespace tensorflow {
 namespace grappler {
@@ -45,21 +44,16 @@ const NodeScopeAndName ParseNodeScopeAndName(const string& node_name);
 struct GraphOptimizerContext {
   GraphOptimizerContext(const std::unordered_set<string>* nodes_to_preserve,
                         GraphDef* optimized_graph,
-                        GraphProperties* graph_properties, NodeMap* node_map,
-                        FrameMap* frame_map)
+                        GraphProperties* graph_properties, NodeMap* node_map)
       : nodes_to_preserve(nodes_to_preserve),
         optimized_graph(optimized_graph),
         graph_properties(graph_properties),
-        node_map(node_map),
-        frame_map(frame_map) {}
+        node_map(node_map) {}
 
   const std::unordered_set<string>* nodes_to_preserve;
   GraphDef* optimized_graph;
   GraphProperties* graph_properties;
   NodeMap* node_map;
-  // TODO(ezhulenev): it seems that frame_map is only relevant for loop
-  // optimizer? Move it to loop-optimizer specific context extension.
-  FrameMap* frame_map;
 };
 
 Status GetInputNode(const GraphOptimizerContext& ctx, const string& input,
diff --git a/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc b/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc
index 416327e622..3f5ab87a5a 100644
--- a/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc
+++ b/tensorflow/core/grappler/optimizers/graph_optimizer_stage_test.cc
@@ -58,8 +58,8 @@ TEST_F(GraphOptimizerStageTest, ParseNodeNameAndScope_InScope) {
 TEST_F(GraphOptimizerStageTest, OptimizedNodeName) {
   GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
                             /*optimized_graph*/ nullptr,
-                            /*graph_properties*/ nullptr, /*node_name*/ nullptr,
-                            /*frame_map*/ nullptr);
+                            /*graph_properties*/ nullptr,
+                            /*node_name*/ nullptr);
   FakeOptimizerStage stage("my_opt", "my_stg", ctx);
 
   const auto node = ParseNodeScopeAndName("a/b/c/Add");
@@ -94,8 +94,7 @@ TEST_F(GraphOptimizerStageTest, GetInputNodeAndProperties) {
   GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
                             /*optimized_graph*/ &item.graph,
                             /*graph_properties*/ &properties,
-                            /*node_name*/ &node_map,
-                            /*frame_map*/ nullptr);
+                            /*node_name*/ &node_map);
   FakeOptimizerStage stage("my_opt", "my_stg", ctx);
 
   NodeDef* add_node;
@@ -134,8 +133,7 @@ TEST_F(GraphOptimizerStageTest, AddNodes) {
   GraphOptimizerContext ctx(/*nodes_to_preserve*/ nullptr,
                             /*optimized_graph*/ &item.graph,
                             /*graph_properties*/ &properties,
-                            /*node_name*/ &node_map,
-                            /*frame_map*/ nullptr);
+                            /*node_name*/ &node_map);
   FakeOptimizerStage stage("my_opt", "my_stg", ctx);
 
   NodeDef* add_node;
@@ -165,4 +163,4 @@ TEST_F(GraphOptimizerStageTest, AddNodes) {
 
 }  // namespace
 }  // end namespace grappler
-}  // end namespace tensorflow
\ No newline at end of file
+}  // end namespace tensorflow
-- 
GitLab


From d107fee1e4a9a4462f01564798d345802acc2aef Mon Sep 17 00:00:00 2001
From: Frank Chen <frankchn@google.com>
Date: Wed, 4 Apr 2018 16:26:25 -0700
Subject: [PATCH 935/960] Check that n + kBlockTrailerSize does not overflow
 before reading a block

PiperOrigin-RevId: 191666300
---
 tensorflow/core/lib/io/format.cc | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/tensorflow/core/lib/io/format.cc b/tensorflow/core/lib/io/format.cc
index 64852943ad..0c24c660a2 100644
--- a/tensorflow/core/lib/io/format.cc
+++ b/tensorflow/core/lib/io/format.cc
@@ -13,6 +13,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
 
+#include <limits>
+
 #include "tensorflow/core/lib/io/format.h"
 
 #include "tensorflow/core/lib/core/coding.h"
@@ -84,6 +86,11 @@ Status ReadBlock(RandomAccessFile* file, const BlockHandle& handle,
   // Read the block contents as well as the type/crc footer.
   // See table_builder.cc for the code that built this structure.
   size_t n = static_cast<size_t>(handle.size());
+
+  if (kBlockTrailerSize > std::numeric_limits<size_t>::max() - n) {
+    return errors::DataLoss("handle.size() too big");
+  }
+
   char* buf = new char[n + kBlockTrailerSize];
   StringPiece contents;
   Status s = file->Read(handle.offset(), n + kBlockTrailerSize, &contents, buf);
-- 
GitLab


From 2af88f3e2114ce28fddd2f512477db020d34407a Mon Sep 17 00:00:00 2001
From: Priya Gupta <priyag@google.com>
Date: Wed, 4 Apr 2018 16:34:54 -0700
Subject: [PATCH 936/960] Address bug in distributed strategies `Monitor` to
 allow running for >1 step.

PiperOrigin-RevId: 191667378
---
 tensorflow/contrib/distribute/python/monitor.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/tensorflow/contrib/distribute/python/monitor.py b/tensorflow/contrib/distribute/python/monitor.py
index fe80bb4df5..7644acedc9 100644
--- a/tensorflow/contrib/distribute/python/monitor.py
+++ b/tensorflow/contrib/distribute/python/monitor.py
@@ -19,6 +19,7 @@ from __future__ import division
 from __future__ import print_function
 
 from tensorflow.python.eager import context
+from tensorflow.python.framework import errors
 from tensorflow.python.ops import variables
 
 
@@ -55,7 +56,9 @@ class Monitor(object):
 
   def run_steps(self, num_steps=None):
     step = 0
-    done = False
-    while done is not None and (num_steps is None or step < num_steps):
-      done = self._run_step()
-      step += 1
+    while num_steps is None or step < num_steps:
+      try:
+        self._run_step()
+        step += 1
+      except errors.OutOfRangeError:
+        break
-- 
GitLab


From faeebb7daef6d1fdd0e4eb3a3e0afedcd2d3350d Mon Sep 17 00:00:00 2001
From: Pavithra Vijay <psv@google.com>
Date: Wed, 4 Apr 2018 16:53:51 -0700
Subject: [PATCH 937/960] Replace trivial backend calls with calls to
 underlying TensorFlow functions - Part 2

PiperOrigin-RevId: 191669725
---
 .../python/keras/_impl/keras/activations.py   | 13 +++---
 .../python/keras/_impl/keras/constraints.py   |  9 ++--
 .../_impl/keras/engine/training_utils.py      |  3 +-
 .../keras/layers/advanced_activations.py      |  5 +-
 .../python/keras/_impl/keras/layers/core.py   |  9 ++--
 .../keras/_impl/keras/layers/core_test.py     |  4 +-
 .../keras/_impl/keras/layers/embeddings.py    |  2 +-
 .../python/keras/_impl/keras/layers/merge.py  | 46 +++++++++++--------
 .../python/keras/_impl/keras/layers/noise.py  |  2 +-
 .../keras/_impl/keras/layers/recurrent.py     |  6 ++-
 .../keras/_impl/keras/layers/wrappers.py      |  4 +-
 tensorflow/python/keras/_impl/keras/losses.py | 33 +++++++------
 .../python/keras/_impl/keras/metrics.py       | 22 +++++----
 .../python/keras/_impl/keras/metrics_test.py  |  5 +-
 .../python/keras/_impl/keras/optimizers.py    | 31 +++++++------
 .../python/keras/_impl/keras/regularizers.py  |  4 +-
 16 files changed, 111 insertions(+), 87 deletions(-)

diff --git a/tensorflow/python/keras/_impl/keras/activations.py b/tensorflow/python/keras/_impl/keras/activations.py
index 74ec373ea5..b518898ad8 100644
--- a/tensorflow/python/keras/_impl/keras/activations.py
+++ b/tensorflow/python/keras/_impl/keras/activations.py
@@ -24,6 +24,7 @@ from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.layers.base import Layer
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util.tf_export import tf_export
 
@@ -44,9 +45,9 @@ def softmax(x, axis=-1):
   """
   ndim = K.ndim(x)
   if ndim == 2:
-    return K.softmax(x)
+    return nn.softmax(x)
   elif ndim > 2:
-    e = K.exp(x - K.max(x, axis=axis, keepdims=True))
+    e = math_ops.exp(x - math_ops.reduce_max(x, axis=axis, keepdims=True))
     s = math_ops.reduce_sum(e, axis=axis, keepdims=True)
     return e / s
   else:
@@ -80,12 +81,12 @@ def selu(x):
 
 @tf_export('keras.activations.softplus')
 def softplus(x):
-  return K.softplus(x)
+  return nn.softplus(x)
 
 
 @tf_export('keras.activations.softsign')
 def softsign(x):
-  return K.softsign(x)
+  return nn.softsign(x)
 
 
 @tf_export('keras.activations.relu')
@@ -95,12 +96,12 @@ def relu(x, alpha=0., max_value=None):
 
 @tf_export('keras.activations.tanh')
 def tanh(x):
-  return K.tanh(x)
+  return nn.tanh(x)
 
 
 @tf_export('keras.activations.sigmoid')
 def sigmoid(x):
-  return K.sigmoid(x)
+  return nn.sigmoid(x)
 
 
 @tf_export('keras.activations.hard_sigmoid')
diff --git a/tensorflow/python/keras/_impl/keras/constraints.py b/tensorflow/python/keras/_impl/keras/constraints.py
index aac4d0f1e9..abe95d8e0c 100644
--- a/tensorflow/python/keras/_impl/keras/constraints.py
+++ b/tensorflow/python/keras/_impl/keras/constraints.py
@@ -67,7 +67,7 @@ class MaxNorm(Constraint):
 
   def __call__(self, w):
     norms = K.sqrt(
-        math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True))
+        math_ops.reduce_sum(math_ops.square(w), axis=self.axis, keepdims=True))
     desired = K.clip(norms, 0, self.max_value)
     return w * (desired / (K.epsilon() + norms))
 
@@ -81,7 +81,7 @@ class NonNeg(Constraint):
   """
 
   def __call__(self, w):
-    return w * math_ops.cast(K.greater_equal(w, 0.), K.floatx())
+    return w * math_ops.cast(math_ops.greater_equal(w, 0.), K.floatx())
 
 
 @tf_export('keras.constraints.UnitNorm', 'keras.constraints.unit_norm')
@@ -108,7 +108,8 @@ class UnitNorm(Constraint):
   def __call__(self, w):
     return w / (
         K.epsilon() + K.sqrt(
-            math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True)))
+            math_ops.reduce_sum(
+                math_ops.square(w), axis=self.axis, keepdims=True)))
 
   def get_config(self):
     return {'axis': self.axis}
@@ -152,7 +153,7 @@ class MinMaxNorm(Constraint):
 
   def __call__(self, w):
     norms = K.sqrt(
-        math_ops.reduce_sum(K.square(w), axis=self.axis, keepdims=True))
+        math_ops.reduce_sum(math_ops.square(w), axis=self.axis, keepdims=True))
     desired = (
         self.rate * K.clip(norms, self.min_value, self.max_value) +
         (1 - self.rate) * norms)
diff --git a/tensorflow/python/keras/_impl/keras/engine/training_utils.py b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
index 58d2c78aad..a3fc8ef2a0 100644
--- a/tensorflow/python/keras/_impl/keras/engine/training_utils.py
+++ b/tensorflow/python/keras/_impl/keras/engine/training_utils.py
@@ -451,7 +451,8 @@ def weighted_masked_objective(fn):
       weight_ndim = K.ndim(weights)
       score_array = K.mean(score_array, axis=list(range(weight_ndim, ndim)))
       score_array *= weights
-      score_array /= K.mean(math_ops.cast(K.not_equal(weights, 0), K.floatx()))
+      score_array /= K.mean(
+          math_ops.cast(math_ops.not_equal(weights, 0), K.floatx()))
     return K.mean(score_array)
 
   return weighted
diff --git a/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py b/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
index 45b0c6c91a..11ca89d625 100644
--- a/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
+++ b/tensorflow/python/keras/_impl/keras/layers/advanced_activations.py
@@ -147,7 +147,7 @@ class PReLU(Layer):
     if K.backend() == 'theano':
       neg = (
           K.pattern_broadcast(self.alpha, self.param_broadcast) *
-          (inputs - K.abs(inputs)) * 0.5)
+          (inputs - math_ops.abs(inputs)) * 0.5)
     else:
       neg = -self.alpha * K.relu(-inputs)
     return pos + neg
@@ -233,7 +233,8 @@ class ThresholdedReLU(Layer):
     self.theta = K.cast_to_floatx(theta)
 
   def call(self, inputs, mask=None):
-    return inputs * math_ops.cast(K.greater(inputs, self.theta), K.floatx())
+    return inputs * math_ops.cast(
+        math_ops.greater(inputs, self.theta), K.floatx())
 
   def get_config(self):
     config = {'theta': float(self.theta)}
diff --git a/tensorflow/python/keras/_impl/keras/layers/core.py b/tensorflow/python/keras/_impl/keras/layers/core.py
index a709a079fd..c74fc1e4c0 100644
--- a/tensorflow/python/keras/_impl/keras/layers/core.py
+++ b/tensorflow/python/keras/_impl/keras/layers/core.py
@@ -77,11 +77,11 @@ class Masking(Layer):
     self.mask_value = mask_value
 
   def compute_mask(self, inputs, mask=None):
-    return K.any(K.not_equal(inputs, self.mask_value), axis=-1)
+    return K.any(math_ops.not_equal(inputs, self.mask_value), axis=-1)
 
   def call(self, inputs):
     boolean_mask = K.any(
-        K.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
+        math_ops.not_equal(inputs, self.mask_value), axis=-1, keepdims=True)
     return inputs * math_ops.cast(boolean_mask, inputs.dtype)
 
   def compute_output_shape(self, input_shape):
@@ -416,7 +416,8 @@ class Reshape(Layer):
     return tensor_shape.TensorShape(output_shape)
 
   def call(self, inputs):
-    return K.reshape(inputs, (array_ops.shape(inputs)[0],) + self.target_shape)
+    return array_ops.reshape(inputs,
+                             (array_ops.shape(inputs)[0],) + self.target_shape)
 
   def get_config(self):
     config = {'target_shape': self.target_shape}
@@ -469,7 +470,7 @@ class Permute(Layer):
     return tensor_shape.TensorShape(output_shape)
 
   def call(self, inputs):
-    return K.permute_dimensions(inputs, (0,) + self.dims)
+    return array_ops.transpose(inputs, perm=(0,) + self.dims)
 
   def get_config(self):
     config = {'dims': self.dims}
diff --git a/tensorflow/python/keras/_impl/keras/layers/core_test.py b/tensorflow/python/keras/_impl/keras/layers/core_test.py
index 2ca816adbd..551d1b1c3a 100644
--- a/tensorflow/python/keras/_impl/keras/layers/core_test.py
+++ b/tensorflow/python/keras/_impl/keras/layers/core_test.py
@@ -23,6 +23,7 @@ import numpy as np
 from tensorflow.python.framework import test_util as tf_test_util
 from tensorflow.python.keras._impl import keras
 from tensorflow.python.keras._impl.keras import testing_utils
+from tensorflow.python.ops import math_ops
 from tensorflow.python.platform import test
 
 
@@ -159,7 +160,7 @@ class CoreLayersTest(test.TestCase):
 
     # test with lambda
     ld = keras.layers.Lambda(
-        lambda x: keras.backend.concatenate([keras.backend.square(x), x]))
+        lambda x: keras.backend.concatenate([math_ops.square(x), x]))
     config = ld.get_config()
     ld = keras.layers.Lambda.from_config(config)
 
@@ -235,4 +236,3 @@ class CoreLayersTest(test.TestCase):
 
 if __name__ == '__main__':
   test.main()
-
diff --git a/tensorflow/python/keras/_impl/keras/layers/embeddings.py b/tensorflow/python/keras/_impl/keras/layers/embeddings.py
index a0fd7a9637..540e2d945c 100644
--- a/tensorflow/python/keras/_impl/keras/layers/embeddings.py
+++ b/tensorflow/python/keras/_impl/keras/layers/embeddings.py
@@ -128,7 +128,7 @@ class Embedding(Layer):
     if not self.mask_zero:
       return None
     else:
-      return K.not_equal(inputs, 0)
+      return math_ops.not_equal(inputs, 0)
 
   @shape_type_conversion
   def compute_output_shape(self, input_shape):
diff --git a/tensorflow/python/keras/_impl/keras/layers/merge.py b/tensorflow/python/keras/_impl/keras/layers/merge.py
index 6290db29a7..7c87e6c067 100644
--- a/tensorflow/python/keras/_impl/keras/layers/merge.py
+++ b/tensorflow/python/keras/_impl/keras/layers/merge.py
@@ -24,6 +24,8 @@ from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.engine.base_layer import Layer
 from tensorflow.python.keras._impl.keras.engine.base_layer import shape_type_conversion
 from tensorflow.python.ops import array_ops
+from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
 from tensorflow.python.util.tf_export import tf_export
 
 
@@ -128,7 +130,7 @@ class _Merge(Layer):
         for x in inputs:
           x_ndim = K.ndim(x)
           for _ in range(max_ndim - x_ndim):
-            x = K.expand_dims(x, 1)
+            x = array_ops.expand_dims(x, axis=1)
           reshaped_inputs.append(x)
         return self._merge_function(reshaped_inputs)
       else:
@@ -140,17 +142,20 @@ class _Merge(Layer):
           if x_ndim is None:
             x_shape = array_ops.shape(x)
             batch_size = x_shape[0]
-            new_shape = K.concatenate([x_shape[1:], K.expand_dims(batch_size)])
-            x_transposed = K.reshape(x,
-                                     K.stack([batch_size,
-                                              K.prod(x_shape[1:])]))
-            x_transposed = K.permute_dimensions(x_transposed, (1, 0))
-            x_transposed = K.reshape(x_transposed, new_shape)
+            new_shape = K.concatenate(
+                [x_shape[1:],
+                 array_ops.expand_dims(batch_size, axis=-1)])
+            x_transposed = array_ops.reshape(
+                x,
+                array_ops.stack(
+                    [batch_size, math_ops.reduce_prod(x_shape[1:])], axis=0))
+            x_transposed = array_ops.transpose(x_transposed, perm=(1, 0))
+            x_transposed = array_ops.reshape(x_transposed, new_shape)
             reshaped_inputs.append(x_transposed)
             transposed = True
           elif x_ndim > 1:
             dims = list(range(1, x_ndim)) + [0]
-            reshaped_inputs.append(K.permute_dimensions(x, dims))
+            reshaped_inputs.append(array_ops.transpose(x, perm=dims))
             transposed = True
           else:
             # We don't transpose inputs if they are 1D vectors or scalars.
@@ -163,14 +168,15 @@ class _Merge(Layer):
             y_shape = array_ops.shape(y)
             y_ndim = array_ops.shape(y_shape)[0]
             batch_size = y_shape[y_ndim - 1]
-            new_shape = K.concatenate(
-                [K.expand_dims(batch_size), y_shape[:y_ndim - 1]])
-            y = K.reshape(y, (-1, batch_size))
-            y = K.permute_dimensions(y, (1, 0))
-            y = K.reshape(y, new_shape)
+            new_shape = K.concatenate([
+                array_ops.expand_dims(batch_size, axis=-1), y_shape[:y_ndim - 1]
+            ])
+            y = array_ops.reshape(y, (-1, batch_size))
+            y = array_ops.transpose(y, perm=(1, 0))
+            y = array_ops.reshape(y, new_shape)
           elif y_ndim > 1:
             dims = [y_ndim - 1] + list(range(y_ndim - 1))
-            y = K.permute_dimensions(y, dims)
+            y = array_ops.transpose(y, perm=dims)
         return y
     else:
       return self._merge_function(inputs)
@@ -208,7 +214,7 @@ class _Merge(Layer):
                        'should have the same length.')
     if all([m is None for m in mask]):
       return None
-    masks = [K.expand_dims(m, 0) for m in mask if m is not None]
+    masks = [array_ops.expand_dims(m, axis=0) for m in mask if m is not None]
     return K.all(K.concatenate(masks, axis=0), axis=0, keepdims=False)
 
 
@@ -326,7 +332,7 @@ class Maximum(_Merge):
   def _merge_function(self, inputs):
     output = inputs[0]
     for i in range(1, len(inputs)):
-      output = K.maximum(output, inputs[i])
+      output = math_ops.maximum(output, inputs[i])
     return output
 
 
@@ -341,7 +347,7 @@ class Minimum(_Merge):
   def _merge_function(self, inputs):
     output = inputs[0]
     for i in range(1, len(inputs)):
-      output = K.minimum(output, inputs[i])
+      output = math_ops.minimum(output, inputs[i])
     return output
 
 
@@ -422,7 +428,7 @@ class Concatenate(_Merge):
         masks.append(array_ops.ones_like(input_i, dtype='bool'))
       elif K.ndim(mask_i) < K.ndim(input_i):
         # Mask is smaller than the input, expand it
-        masks.append(K.expand_dims(mask_i))
+        masks.append(array_ops.expand_dims(mask_i, axis=-1))
       else:
         masks.append(mask_i)
     concatenated = K.concatenate(masks, axis=self.axis)
@@ -512,8 +518,8 @@ class Dot(_Merge):
         else:
           axes.append(self.axes[i])
     if self.normalize:
-      x1 = K.l2_normalize(x1, axis=axes[0])
-      x2 = K.l2_normalize(x2, axis=axes[1])
+      x1 = nn.l2_normalize(x1, axis=axes[0])
+      x2 = nn.l2_normalize(x2, axis=axes[1])
     output = K.batch_dot(x1, x2, axes)
     return output
 
diff --git a/tensorflow/python/keras/_impl/keras/layers/noise.py b/tensorflow/python/keras/_impl/keras/layers/noise.py
index 4366b654f2..72dc7a1ff8 100644
--- a/tensorflow/python/keras/_impl/keras/layers/noise.py
+++ b/tensorflow/python/keras/_impl/keras/layers/noise.py
@@ -166,7 +166,7 @@ class AlphaDropout(Layer):
         scale = 1.0507009873554804934193349852946
         alpha_p = -alpha * scale
 
-        kept_idx = K.greater_equal(
+        kept_idx = math_ops.greater_equal(
             K.random_uniform(noise_shape, seed=seed), rate)
         kept_idx = math_ops.cast(kept_idx, K.floatx())
 
diff --git a/tensorflow/python/keras/_impl/keras/layers/recurrent.py b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
index bd7c42e63e..7f9f77c296 100644
--- a/tensorflow/python/keras/_impl/keras/layers/recurrent.py
+++ b/tensorflow/python/keras/_impl/keras/layers/recurrent.py
@@ -510,7 +510,8 @@ class RNN(Layer):
     # shape of initial_state = (samples, timesteps, input_dim)
     initial_state = math_ops.reduce_sum(initial_state, axis=(1, 2))
     # shape of initial_state = (samples,)
-    initial_state = K.expand_dims(initial_state)  # (samples, 1)
+    initial_state = array_ops.expand_dims(initial_state, axis=-1)
+    # shape of initial_state = (samples, 1)
     if hasattr(self.cell.state_size, '__len__'):
       return [K.tile(initial_state, [1, dim]) for dim in self.cell.state_size]
     else:
@@ -2357,7 +2358,8 @@ class Recurrent(Layer):
     # shape of initial_state = (samples, timesteps, input_dim)
     initial_state = math_ops.reduce_sum(initial_state, axis=(1, 2))
     # shape of initial_state = (samples,)
-    initial_state = K.expand_dims(initial_state)  # (samples, 1)
+    initial_state = array_ops.expand_dims(initial_state, axis=-1)
+    # shape of initial_state = (samples, 1)
     initial_state = K.tile(initial_state, [1,
                                            self.units])  # (samples, output_dim)
     initial_state = [initial_state for _ in range(len(self.states))]
diff --git a/tensorflow/python/keras/_impl/keras/layers/wrappers.py b/tensorflow/python/keras/_impl/keras/layers/wrappers.py
index 12f33614e2..c510e464ae 100644
--- a/tensorflow/python/keras/_impl/keras/layers/wrappers.py
+++ b/tensorflow/python/keras/_impl/keras/layers/wrappers.py
@@ -214,7 +214,7 @@ class TimeDistributed(Wrapper):
       # Shape: (num_samples * timesteps, ...). And track the
       # transformation in self._input_map.
       input_uid = tf_layers_util.object_list_uid(inputs)
-      inputs = K.reshape(inputs, (-1,) + input_shape[2:])
+      inputs = array_ops.reshape(inputs, (-1,) + input_shape[2:])
       self._input_map[input_uid] = inputs
       # (num_samples * timesteps, ...)
       y = self.layer.call(inputs, **kwargs)
@@ -222,7 +222,7 @@ class TimeDistributed(Wrapper):
         uses_learning_phase = y._uses_learning_phase
       # Shape: (num_samples, timesteps, ...)
       output_shape = self.compute_output_shape(input_shape).as_list()
-      y = K.reshape(y, (-1, input_length) + tuple(output_shape[2:]))
+      y = array_ops.reshape(y, (-1, input_length) + tuple(output_shape[2:]))
 
     # Apply activity regularizer if any:
     if (hasattr(self.layer, 'activity_regularizer') and
diff --git a/tensorflow/python/keras/_impl/keras/losses.py b/tensorflow/python/keras/_impl/keras/losses.py
index 859bda0c9d..1d634d3801 100644
--- a/tensorflow/python/keras/_impl/keras/losses.py
+++ b/tensorflow/python/keras/_impl/keras/losses.py
@@ -25,51 +25,54 @@ from tensorflow.python.keras._impl.keras import backend as K
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
 from tensorflow.python.util.tf_export import tf_export
 
 
 @tf_export('keras.metrics.mean_squared_error',
            'keras.losses.mean_squared_error')
 def mean_squared_error(y_true, y_pred):
-  return K.mean(K.square(y_pred - y_true), axis=-1)
+  return K.mean(math_ops.square(y_pred - y_true), axis=-1)
 
 
 @tf_export('keras.metrics.mean_absolute_error',
            'keras.losses.mean_absolute_error')
 def mean_absolute_error(y_true, y_pred):
-  return K.mean(K.abs(y_pred - y_true), axis=-1)
+  return K.mean(math_ops.abs(y_pred - y_true), axis=-1)
 
 
 @tf_export('keras.metrics.mean_absolute_percentage_error',
            'keras.losses.mean_absolute_percentage_error')
 def mean_absolute_percentage_error(y_true, y_pred):
-  diff = K.abs((y_true - y_pred) / K.clip(K.abs(y_true), K.epsilon(), None))
+  diff = math_ops.abs(
+      (y_true - y_pred) / K.clip(math_ops.abs(y_true), K.epsilon(), None))
   return 100. * K.mean(diff, axis=-1)
 
 
 @tf_export('keras.metrics.mean_squared_logarithmic_error',
            'keras.losses.mean_squared_logarithmic_error')
 def mean_squared_logarithmic_error(y_true, y_pred):
-  first_log = K.log(K.clip(y_pred, K.epsilon(), None) + 1.)
-  second_log = K.log(K.clip(y_true, K.epsilon(), None) + 1.)
-  return K.mean(K.square(first_log - second_log), axis=-1)
+  first_log = math_ops.log(K.clip(y_pred, K.epsilon(), None) + 1.)
+  second_log = math_ops.log(K.clip(y_true, K.epsilon(), None) + 1.)
+  return K.mean(math_ops.square(first_log - second_log), axis=-1)
 
 
 @tf_export('keras.metrics.squared_hinge', 'keras.losses.squared_hinge')
 def squared_hinge(y_true, y_pred):
-  return K.mean(K.square(K.maximum(1. - y_true * y_pred, 0.)), axis=-1)
+  return K.mean(
+      math_ops.square(math_ops.maximum(1. - y_true * y_pred, 0.)), axis=-1)
 
 
 @tf_export('keras.metrics.hinge', 'keras.losses.hinge')
 def hinge(y_true, y_pred):
-  return K.mean(K.maximum(1. - y_true * y_pred, 0.), axis=-1)
+  return K.mean(math_ops.maximum(1. - y_true * y_pred, 0.), axis=-1)
 
 
 @tf_export('keras.losses.categorical_hinge')
 def categorical_hinge(y_true, y_pred):
   pos = math_ops.reduce_sum(y_true * y_pred, axis=-1)
-  neg = K.max((1. - y_true) * y_pred, axis=-1)
-  return K.maximum(0., neg - pos + 1.)
+  neg = math_ops.reduce_max((1. - y_true) * y_pred, axis=-1)
+  return math_ops.maximum(0., neg - pos + 1.)
 
 
 @tf_export('keras.losses.logcosh')
@@ -90,7 +93,7 @@ def logcosh(y_true, y_pred):
   """
 
   def _logcosh(x):
-    return x + K.softplus(-2. * x) - K.log(2.)
+    return x + nn.softplus(-2. * x) - math_ops.log(2.)
 
   return K.mean(_logcosh(y_pred - y_true), axis=-1)
 
@@ -118,18 +121,18 @@ def binary_crossentropy(y_true, y_pred):
 def kullback_leibler_divergence(y_true, y_pred):
   y_true = K.clip(y_true, K.epsilon(), 1)
   y_pred = K.clip(y_pred, K.epsilon(), 1)
-  return math_ops.reduce_sum(y_true * K.log(y_true / y_pred), axis=-1)
+  return math_ops.reduce_sum(y_true * math_ops.log(y_true / y_pred), axis=-1)
 
 
 @tf_export('keras.metrics.poisson', 'keras.losses.poisson')
 def poisson(y_true, y_pred):
-  return K.mean(y_pred - y_true * K.log(y_pred + K.epsilon()), axis=-1)
+  return K.mean(y_pred - y_true * math_ops.log(y_pred + K.epsilon()), axis=-1)
 
 
 @tf_export('keras.metrics.cosine_proximity', 'keras.losses.cosine_proximity')
 def cosine_proximity(y_true, y_pred):
-  y_true = K.l2_normalize(y_true, axis=-1)
-  y_pred = K.l2_normalize(y_pred, axis=-1)
+  y_true = nn.l2_normalize(y_true, axis=-1)
+  y_pred = nn.l2_normalize(y_pred, axis=-1)
   return -math_ops.reduce_sum(y_true * y_pred, axis=-1)
 
 
diff --git a/tensorflow/python/keras/_impl/keras/metrics.py b/tensorflow/python/keras/_impl/keras/metrics.py
index 24192cf5a1..747c3e6515 100644
--- a/tensorflow/python/keras/_impl/keras/metrics.py
+++ b/tensorflow/python/keras/_impl/keras/metrics.py
@@ -38,39 +38,45 @@ from tensorflow.python.keras._impl.keras.losses import squared_hinge
 from tensorflow.python.keras._impl.keras.utils.generic_utils import deserialize_keras_object
 from tensorflow.python.keras._impl.keras.utils.generic_utils import serialize_keras_object
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import nn
 from tensorflow.python.util.tf_export import tf_export
 
 
 @tf_export('keras.metrics.binary_accuracy')
 def binary_accuracy(y_true, y_pred):
-  return K.mean(K.equal(y_true, K.round(y_pred)), axis=-1)
+  return K.mean(math_ops.equal(y_true, math_ops.round(y_pred)), axis=-1)
 
 
 @tf_export('keras.metrics.categorical_accuracy')
 def categorical_accuracy(y_true, y_pred):
   return math_ops.cast(
-      K.equal(K.argmax(y_true, axis=-1), K.argmax(y_pred, axis=-1)), K.floatx())
+      math_ops.equal(
+          math_ops.argmax(y_true, axis=-1), math_ops.argmax(y_pred, axis=-1)),
+      K.floatx())
 
 
 def sparse_categorical_accuracy(y_true, y_pred):
   return math_ops.cast(
-      K.equal(
-          K.max(y_true, axis=-1),
-          math_ops.cast(K.argmax(y_pred, axis=-1), K.floatx())), K.floatx())
+      math_ops.equal(
+          math_ops.reduce_max(y_true, axis=-1),
+          math_ops.cast(math_ops.argmax(y_pred, axis=-1), K.floatx())),
+      K.floatx())
 
 
 @tf_export('keras.metrics.top_k_categorical_accuracy')
 def top_k_categorical_accuracy(y_true, y_pred, k=5):
-  return K.mean(K.in_top_k(y_pred, K.argmax(y_true, axis=-1), k), axis=-1)
+  return K.mean(
+      nn.in_top_k(y_pred, math_ops.argmax(y_true, axis=-1), k), axis=-1)
 
 
 @tf_export('keras.metrics.sparse_top_k_categorical_accuracy')
 def sparse_top_k_categorical_accuracy(y_true, y_pred, k=5):
   return K.mean(
-      K.in_top_k(y_pred, math_ops.cast(K.max(y_true, axis=-1), 'int32'), k),
+      nn.in_top_k(y_pred,
+                  math_ops.cast(math_ops.reduce_max(y_true, axis=-1), 'int32'),
+                  k),
       axis=-1)
 
-
 # Aliases
 
 mse = MSE = mean_squared_error
diff --git a/tensorflow/python/keras/_impl/keras/metrics_test.py b/tensorflow/python/keras/_impl/keras/metrics_test.py
index 2b73e0c16f..9deaab0c05 100644
--- a/tensorflow/python/keras/_impl/keras/metrics_test.py
+++ b/tensorflow/python/keras/_impl/keras/metrics_test.py
@@ -107,9 +107,8 @@ class KerasMetricsTest(test.TestCase):
                 completion of the batch.
         """
         y_true = math_ops.cast(y_true, 'int32')
-        y_pred = math_ops.cast(keras.backend.round(y_pred), 'int32')
-        correct_preds = math_ops.cast(
-            keras.backend.equal(y_pred, y_true), 'int32')
+        y_pred = math_ops.cast(math_ops.round(y_pred), 'int32')
+        correct_preds = math_ops.cast(math_ops.equal(y_pred, y_true), 'int32')
         true_pos = math_ops.cast(
             math_ops.reduce_sum(correct_preds * y_true), 'int32')
         current_true_pos = self.true_positives * 1
diff --git a/tensorflow/python/keras/_impl/keras/optimizers.py b/tensorflow/python/keras/_impl/keras/optimizers.py
index dc0e472b88..9f383deb72 100644
--- a/tensorflow/python/keras/_impl/keras/optimizers.py
+++ b/tensorflow/python/keras/_impl/keras/optimizers.py
@@ -119,7 +119,8 @@ class Optimizer(object):
                        'Common ops without gradient: '
                        'K.argmax, K.round, K.eval.')
     if hasattr(self, 'clipnorm') and self.clipnorm > 0:
-      norm = K.sqrt(sum([math_ops.reduce_sum(K.square(g)) for g in grads]))
+      norm = K.sqrt(
+          sum([math_ops.reduce_sum(math_ops.square(g)) for g in grads]))
       grads = [clip_norm(g, self.clipnorm, norm) for g in grads]
     if hasattr(self, 'clipvalue') and self.clipvalue > 0:
       grads = [K.clip(g, -self.clipvalue, self.clipvalue) for g in grads]
@@ -288,7 +289,7 @@ class RMSprop(Optimizer):
 
     for p, g, a in zip(params, grads, accumulators):
       # update accumulator
-      new_a = self.rho * a + (1. - self.rho) * K.square(g)
+      new_a = self.rho * a + (1. - self.rho) * math_ops.square(g)
       self.updates.append(state_ops.assign(a, new_a))
       new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
 
@@ -349,7 +350,7 @@ class Adagrad(Optimizer):
                                                 K.dtype(self.decay))))
 
     for p, g, a in zip(params, grads, accumulators):
-      new_a = a + K.square(g)  # update accumulator
+      new_a = a + math_ops.square(g)  # update accumulator
       self.updates.append(state_ops.assign(a, new_a))
       new_p = p - lr * g / (K.sqrt(new_a) + self.epsilon)
 
@@ -414,7 +415,7 @@ class Adadelta(Optimizer):
 
     for p, g, a, d_a in zip(params, grads, accumulators, delta_accumulators):
       # update accumulator
-      new_a = self.rho * a + (1. - self.rho) * K.square(g)
+      new_a = self.rho * a + (1. - self.rho) * math_ops.square(g)
       self.updates.append(state_ops.assign(a, new_a))
 
       # use the new accumulator and the *old* delta_accumulator
@@ -428,7 +429,7 @@ class Adadelta(Optimizer):
       self.updates.append(state_ops.assign(p, new_p))
 
       # update delta_accumulator
-      new_d_a = self.rho * d_a + (1 - self.rho) * K.square(update)
+      new_d_a = self.rho * d_a + (1 - self.rho) * math_ops.square(update)
       self.updates.append(state_ops.assign(d_a, new_d_a))
     return self.updates
 
@@ -494,7 +495,8 @@ class Adam(Optimizer):
 
     t = math_ops.cast(self.iterations, K.floatx()) + 1
     lr_t = lr * (
-        K.sqrt(1. - K.pow(self.beta_2, t)) / (1. - K.pow(self.beta_1, t)))
+        K.sqrt(1. - math_ops.pow(self.beta_2, t)) /
+        (1. - math_ops.pow(self.beta_1, t)))
 
     ms = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
     vs = [K.zeros(K.int_shape(p), dtype=K.dtype(p)) for p in params]
@@ -506,9 +508,9 @@ class Adam(Optimizer):
 
     for p, g, m, v, vhat in zip(params, grads, ms, vs, vhats):
       m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
-      v_t = (self.beta_2 * v) + (1. - self.beta_2) * K.square(g)
+      v_t = (self.beta_2 * v) + (1. - self.beta_2) * math_ops.square(g)
       if self.amsgrad:
-        vhat_t = K.maximum(vhat, v_t)
+        vhat_t = math_ops.maximum(vhat, v_t)
         p_t = p - lr_t * m_t / (K.sqrt(vhat_t) + self.epsilon)
         self.updates.append(state_ops.assign(vhat, vhat_t))
       else:
@@ -583,7 +585,7 @@ class Adamax(Optimizer):
                                                 K.dtype(self.decay))))
 
     t = math_ops.cast(self.iterations, K.floatx()) + 1
-    lr_t = lr / (1. - K.pow(self.beta_1, t))
+    lr_t = lr / (1. - math_ops.pow(self.beta_1, t))
 
     shapes = [K.int_shape(p) for p in params]
     # zero init of 1st moment
@@ -595,7 +597,7 @@ class Adamax(Optimizer):
     for p, g, m, u in zip(params, grads, ms, us):
 
       m_t = (self.beta_1 * m) + (1. - self.beta_1) * g
-      u_t = K.maximum(self.beta_2 * u, K.abs(g))
+      u_t = math_ops.maximum(self.beta_2 * u, math_ops.abs(g))
       p_t = p - lr_t * m_t / (u_t + self.epsilon)
 
       self.updates.append(state_ops.assign(m, m_t))
@@ -666,10 +668,11 @@ class Nadam(Optimizer):
 
     # Due to the recommendations in [2], i.e. warming momentum schedule
     momentum_cache_t = self.beta_1 * (
-        1. - 0.5 * (K.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
+        1. - 0.5 *
+        (math_ops.pow(K.cast_to_floatx(0.96), t * self.schedule_decay)))
     momentum_cache_t_1 = self.beta_1 * (
         1. - 0.5 *
-        (K.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
+        (math_ops.pow(K.cast_to_floatx(0.96), (t + 1) * self.schedule_decay)))
     m_schedule_new = self.m_schedule * momentum_cache_t
     m_schedule_next = self.m_schedule * momentum_cache_t * momentum_cache_t_1
     self.updates.append((self.m_schedule, m_schedule_new))
@@ -685,8 +688,8 @@ class Nadam(Optimizer):
       g_prime = g / (1. - m_schedule_new)
       m_t = self.beta_1 * m + (1. - self.beta_1) * g
       m_t_prime = m_t / (1. - m_schedule_next)
-      v_t = self.beta_2 * v + (1. - self.beta_2) * K.square(g)
-      v_t_prime = v_t / (1. - K.pow(self.beta_2, t))
+      v_t = self.beta_2 * v + (1. - self.beta_2) * math_ops.square(g)
+      v_t_prime = v_t / (1. - math_ops.pow(self.beta_2, t))
       m_t_bar = (
           1. - momentum_cache_t) * g_prime + momentum_cache_t_1 * m_t_prime
 
diff --git a/tensorflow/python/keras/_impl/keras/regularizers.py b/tensorflow/python/keras/_impl/keras/regularizers.py
index fdb9d33810..74c37d370e 100644
--- a/tensorflow/python/keras/_impl/keras/regularizers.py
+++ b/tensorflow/python/keras/_impl/keras/regularizers.py
@@ -56,9 +56,9 @@ class L1L2(Regularizer):
   def __call__(self, x):
     regularization = 0.
     if self.l1:
-      regularization += math_ops.reduce_sum(self.l1 * K.abs(x))
+      regularization += math_ops.reduce_sum(self.l1 * math_ops.abs(x))
     if self.l2:
-      regularization += math_ops.reduce_sum(self.l2 * K.square(x))
+      regularization += math_ops.reduce_sum(self.l2 * math_ops.square(x))
     return regularization
 
   def get_config(self):
-- 
GitLab


From 1d0f3581f7350f5c666a1b9869637d6feac3d4df Mon Sep 17 00:00:00 2001
From: Bixia Zheng <bixia@google.com>
Date: Wed, 4 Apr 2018 17:14:11 -0700
Subject: [PATCH 938/960] [XLA]: Enable Memory sanitizer for
 compilation_passes_test.

Initialize the buffer for a scalar tensor to avoid uninitialized
accesses.

PiperOrigin-RevId: 191672257
---
 tensorflow/compiler/jit/BUILD                         |  1 -
 .../compiler/jit/mark_for_compilation_pass_test.cc    | 11 ++++++-----
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/tensorflow/compiler/jit/BUILD b/tensorflow/compiler/jit/BUILD
index e7d18e8351..24aa203c00 100644
--- a/tensorflow/compiler/jit/BUILD
+++ b/tensorflow/compiler/jit/BUILD
@@ -347,7 +347,6 @@ tf_cc_test(
         "encapsulate_subgraphs_pass_test.cc",
         "mark_for_compilation_pass_test.cc",
     ],
-    tags = ["nomsan"],  # TODO: b/77543571
     deps = [
         ":common",
         ":compilation_passes",
diff --git a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
index af1919278c..2e362e0a63 100644
--- a/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
+++ b/tensorflow/compiler/jit/mark_for_compilation_pass_test.cc
@@ -163,11 +163,12 @@ TEST(XlaCompilationTest, HalfSupported) {
   GraphDef graphdef;
   {
     GraphDefBuilder builder(GraphDefBuilder::kFailImmediately);
-    Node* a = ops::SourceOp(
-        "Const", builder.opts()
-                     .WithName("A")
-                     .WithAttr("dtype", DT_HALF)
-                     .WithAttr("value", Tensor(DT_HALF, TensorShape())));
+    Tensor t(DT_HALF, TensorShape());
+    t.scalar<Eigen::half>()() = static_cast<Eigen::half>(0.0f);
+    Node* a = ops::SourceOp("Const", builder.opts()
+                                         .WithName("A")
+                                         .WithAttr("dtype", DT_HALF)
+                                         .WithAttr("value", t));
     Node* b = ops::UnaryOp("Neg", a, builder.opts().WithName("B"));
     ops::BinaryOp("MatMul", a, b, builder.opts().WithName("C"));
     TF_EXPECT_OK(GraphDefBuilderToGraph(builder, graph.get()));
-- 
GitLab


From 18862fb82f317a930a2d1cde51bca9c47924c882 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 17:18:43 -0700
Subject: [PATCH 939/960] Exclude some gcc options in Windows build

PiperOrigin-RevId: 191672761
---
 tensorflow/python/BUILD   | 1 +
 tensorflow/tensorflow.bzl | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index a9363608b5..a8f1318509 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -427,6 +427,7 @@ tf_cc_shared_object(
             "-lm",
         ],
         "//tensorflow:darwin": [],
+        "//tensorflow:windows": [],
     }),
     deps = [
         "//tensorflow/core:framework_headers_lib",
diff --git a/tensorflow/tensorflow.bzl b/tensorflow/tensorflow.bzl
index 098ae7e6e3..fd44b0eb3b 100644
--- a/tensorflow/tensorflow.bzl
+++ b/tensorflow/tensorflow.bzl
@@ -304,6 +304,7 @@ def tf_cc_shared_object(
           clean_dep("//tensorflow:darwin"): [
               "-Wl,-install_name,@rpath/" + name.split("/")[-1],
           ],
+          clean_dep("//tensorflow:windows"): [],
           "//conditions:default": [
               "-Wl,-soname," + name.split("/")[-1],
           ],
@@ -929,6 +930,7 @@ def tf_cuda_library(deps=None, cuda_deps=None, copts=tf_copts(), **kwargs):
   if 'linkstatic' not in kwargs or kwargs['linkstatic'] != 1:
     enable_text_relocation_linkopt = select({
           clean_dep("//tensorflow:darwin"): [],
+          clean_dep("//tensorflow:windows"): [],
           "//conditions:default": ['-Wl,-z,notext'],})
     if 'linkopts' in kwargs:
       kwargs['linkopts'] += enable_text_relocation_linkopt
-- 
GitLab


From 4b563ed0008953519a0ad9ec09a3261f1d3759dd Mon Sep 17 00:00:00 2001
From: "Joshua V. Dillon" <jvdillon@google.com>
Date: Wed, 4 Apr 2018 17:28:41 -0700
Subject: [PATCH 940/960] BUGFIX: Detect when broadcasting is required and
 raise NotImplementedError.

PiperOrigin-RevId: 191673876
---
 .../python/kernel_tests/batch_reshape_test.py |  37 ++++
 .../distributions/python/ops/batch_reshape.py | 192 +++++++++++++-----
 2 files changed, 174 insertions(+), 55 deletions(-)

diff --git a/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py b/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
index 4d2f40e27f..c6c8d2cf6e 100644
--- a/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
+++ b/tensorflow/contrib/distributions/python/kernel_tests/batch_reshape_test.py
@@ -22,6 +22,7 @@ import numpy as np
 
 from tensorflow.contrib.distributions.python.ops import batch_reshape as batch_reshape_lib
 from tensorflow.contrib.distributions.python.ops import mvn_diag as mvn_lib
+from tensorflow.contrib.distributions.python.ops import poisson as poisson_lib
 from tensorflow.contrib.distributions.python.ops import wishart as wishart_lib
 from tensorflow.python.framework import constant_op
 from tensorflow.python.ops import array_ops
@@ -514,6 +515,42 @@ class _BatchReshapeTest(object):
               batch_shape=new_batch_shape_ph,
               validate_args=True).sample().eval()
 
+  def test_broadcasting_explicitly_unsupported(self):
+    old_batch_shape = [4]
+    new_batch_shape = [1, 4, 1]
+    rate_ = self.dtype([1, 10, 2, 20])
+
+    rate = array_ops.placeholder_with_default(
+        rate_,
+        shape=old_batch_shape if self.is_static_shape else None)
+    poisson_4 = poisson_lib.Poisson(rate)
+    new_batch_shape_ph = (
+        constant_op.constant(np.int32(new_batch_shape)) if self.is_static_shape
+        else array_ops.placeholder_with_default(
+            np.int32(new_batch_shape), shape=None))
+    poisson_141_reshaped = batch_reshape_lib.BatchReshape(
+        poisson_4, new_batch_shape_ph, validate_args=True)
+
+    x_4 = self.dtype([2, 12, 3, 23])
+    x_114 = self.dtype([2, 12, 3, 23]).reshape(1, 1, 4)
+
+    if self.is_static_shape:
+      with self.assertRaisesRegexp(NotImplementedError,
+                                   "too few event dims"):
+        poisson_141_reshaped.log_prob(x_4)
+      with self.assertRaisesRegexp(NotImplementedError,
+                                   "unexpected batch and event shape"):
+        poisson_141_reshaped.log_prob(x_114)
+      return
+
+    with self.assertRaisesOpError("too few event dims"):
+      with self.test_session():
+        poisson_141_reshaped.log_prob(x_4).eval()
+
+    with self.assertRaisesOpError("unexpected batch and event shape"):
+      with self.test_session():
+        poisson_141_reshaped.log_prob(x_114).eval()
+
 
 class BatchReshapeStaticTest(_BatchReshapeTest, test.TestCase):
 
diff --git a/tensorflow/contrib/distributions/python/ops/batch_reshape.py b/tensorflow/contrib/distributions/python/ops/batch_reshape.py
index c7ee9b2117..3e6c35e0d6 100644
--- a/tensorflow/contrib/distributions/python/ops/batch_reshape.py
+++ b/tensorflow/contrib/distributions/python/ops/batch_reshape.py
@@ -115,7 +115,7 @@ class BatchReshape(distribution_lib.Distribution):
       self._batch_shape_static = tensor_util.constant_value(self._batch_shape_)
       if self._batch_shape_static is not None:
         self._batch_shape_static = np.int32(self._batch_shape_static)
-      self._runtime_assertions = make_runtime_assertions(
+      self._runtime_assertions = validate_init_args(
           self._distribution,
           self._batch_shape_,
           validate_args,
@@ -229,7 +229,8 @@ class BatchReshape(distribution_lib.Distribution):
 
   def _call_reshape_input_output(self, fn, x):
     """Calls `fn`, appropriately reshaping its input `x` and output."""
-    with ops.control_dependencies(self._runtime_assertions):
+    with ops.control_dependencies(
+        self._runtime_assertions + self._validate_sample_arg(x)):
       sample_shape, static_sample_shape = self._sample_shape(x)
       old_shape = array_ops.concat([
           sample_shape,
@@ -273,61 +274,142 @@ class BatchReshape(distribution_lib.Distribution):
         result.set_shape(static_shape)
       return result
 
-
-def make_runtime_assertions(
+  def _validate_sample_arg(self, x):
+    """Helper which validates sample arg, e.g., input to `log_prob`."""
+    with ops.name_scope(name="validate_sample_arg", values=[x]):
+      x_ndims = (array_ops.rank(x) if x.shape.ndims is None else x.shape.ndims)
+      event_ndims = (array_ops.size(self.event_shape_tensor())
+                     if self.event_shape.ndims is None
+                     else self.event_shape.ndims)
+      batch_ndims = (array_ops.size(self.batch_shape_tensor())
+                     if self.batch_shape.ndims is None
+                     else self.batch_shape.ndims)
+      expected_batch_event_ndims = batch_ndims + event_ndims
+
+      if (isinstance(x_ndims, int) and
+          isinstance(expected_batch_event_ndims, int)):
+        if x_ndims < expected_batch_event_ndims:
+          raise NotImplementedError(
+              "Broadcasting is not supported; too few event dims "
+              "(expected at least {}, saw {}).".format(
+                  expected_batch_event_ndims, x_ndims))
+        ndims_assertion = []
+      elif self.validate_args:
+        ndims_assertion = [
+            check_ops.assert_greater_equal(
+                x_ndims,
+                expected_batch_event_ndims,
+                message="Broadcasting is not supported; too few event dims.",
+                name="assert_batch_and_event_ndims_large_enough"),
+        ]
+
+      if (self.batch_shape.is_fully_defined() and
+          self.event_shape.is_fully_defined()):
+        expected_batch_event_shape = np.int32(self.batch_shape.concatenate(
+            self.event_shape).as_list())
+      else:
+        expected_batch_event_shape = array_ops.concat([
+            self.batch_shape_tensor(),
+            self.event_shape_tensor(),
+        ], axis=0)
+
+      sample_ndims = x_ndims - expected_batch_event_ndims
+      if isinstance(sample_ndims, int):
+        sample_ndims = max(sample_ndims, 0)
+      if (isinstance(sample_ndims, int) and
+          x.shape[sample_ndims:].is_fully_defined()):
+        actual_batch_event_shape = np.int32(x.shape[sample_ndims:].as_list())
+      else:
+        sample_ndims = math_ops.maximum(sample_ndims, 0)
+        actual_batch_event_shape = array_ops.shape(x)[sample_ndims:]
+
+      if (isinstance(expected_batch_event_shape, np.ndarray) and
+          isinstance(actual_batch_event_shape, np.ndarray)):
+        if any(expected_batch_event_shape != actual_batch_event_shape):
+          raise NotImplementedError("Broadcasting is not supported; "
+                                    "unexpected batch and event shape "
+                                    "(expected {}, saw {}).".format(
+                                        expected_batch_event_shape,
+                                        actual_batch_event_shape))
+        # We need to set the final runtime-assertions to `ndims_assertion` since
+        # its possible this assertion was created. We could add a condition to
+        # only do so if `self.validate_args == True`, however this is redundant
+        # as `ndims_assertion` already encodes this information.
+        runtime_assertions = ndims_assertion
+      elif self.validate_args:
+        # We need to make the `ndims_assertion` a control dep because otherwise
+        # TF itself might raise an exception owing to this assertion being
+        # ill-defined, ie, one cannot even compare different rank Tensors.
+        with ops.control_dependencies(ndims_assertion):
+          shape_assertion = check_ops.assert_equal(
+              expected_batch_event_shape,
+              actual_batch_event_shape,
+              message=("Broadcasting is not supported; "
+                       "unexpected batch and event shape."),
+              name="assert_batch_and_event_shape_same")
+        runtime_assertions = [shape_assertion]
+      else:
+        runtime_assertions = []
+
+      return runtime_assertions
+
+
+def validate_init_args(
     distribution,
     batch_shape,
     validate_args,
     batch_shape_static):
   """Helper to __init__ which makes or raises assertions."""
-  runtime_assertions = []
-
-  if batch_shape.shape.ndims is not None:
-    if batch_shape.shape.ndims != 1:
-      raise ValueError("`batch_shape` must be a vector "
-                       "(saw rank: {}).".format(
-                           batch_shape.shape.ndims))
-  elif validate_args:
-    runtime_assertions += [
-        check_ops.assert_rank(
-            batch_shape,
-            1,
-            message="`batch_shape` must be a vector.",
-            name="assert_batch_shape_is_vector"),
-    ]
-
-  batch_size_static = np.prod(batch_shape_static)
-  dist_batch_size_static = (
-      None if not distribution.batch_shape.is_fully_defined()
-      else np.prod(distribution.batch_shape).value)
-
-  if batch_size_static is not None and dist_batch_size_static is not None:
-    if batch_size_static != dist_batch_size_static:
-      raise ValueError("`batch_shape` size ({}) must match "
-                       "`distribution.batch_shape` size ({}).".format(
-                           batch_size_static,
-                           dist_batch_size_static))
-  elif validate_args:
-    runtime_assertions += [
-        check_ops.assert_equal(
-            math_ops.reduce_prod(batch_shape),
-            math_ops.reduce_prod(distribution.batch_shape_tensor()),
-            message=("`batch_shape` size must match "
-                     "`distributions.batch_shape` size."),
-            name="assert_batch_size"),
-    ]
-
-  if batch_shape_static is not None:
-    if np.any(batch_shape_static < 1):
-      raise ValueError("`batch_shape` elements must be positive "
-                       "(i.e., larger than zero).")
-  elif validate_args:
-    runtime_assertions += [
-        check_ops.assert_positive(
-            batch_shape,
-            message=("`batch_shape` elements must be positive "
-                     "(i.e., larger than zero)."),
-            name="assert_batch_shape_positive")
-    ]
-
-  return runtime_assertions
+  with ops.name_scope(name="validate_init_args",
+                      values=[batch_shape] + distribution._graph_parents):  # pylint: disable=protected-access
+    runtime_assertions = []
+
+    if batch_shape.shape.ndims is not None:
+      if batch_shape.shape.ndims != 1:
+        raise ValueError("`batch_shape` must be a vector "
+                         "(saw rank: {}).".format(
+                             batch_shape.shape.ndims))
+    elif validate_args:
+      runtime_assertions += [
+          check_ops.assert_rank(
+              batch_shape,
+              1,
+              message="`batch_shape` must be a vector.",
+              name="assert_batch_shape_is_vector"),
+      ]
+
+    batch_size_static = np.prod(batch_shape_static)
+    dist_batch_size_static = (
+        None if not distribution.batch_shape.is_fully_defined()
+        else np.prod(distribution.batch_shape).value)
+
+    if batch_size_static is not None and dist_batch_size_static is not None:
+      if batch_size_static != dist_batch_size_static:
+        raise ValueError("`batch_shape` size ({}) must match "
+                         "`distribution.batch_shape` size ({}).".format(
+                             batch_size_static,
+                             dist_batch_size_static))
+    elif validate_args:
+      runtime_assertions += [
+          check_ops.assert_equal(
+              math_ops.reduce_prod(batch_shape),
+              math_ops.reduce_prod(distribution.batch_shape_tensor()),
+              message=("`batch_shape` size must match "
+                       "`distributions.batch_shape` size."),
+              name="assert_batch_size"),
+      ]
+
+    if batch_shape_static is not None:
+      if np.any(batch_shape_static < 1):
+        raise ValueError("`batch_shape` elements must be positive "
+                         "(i.e., larger than zero).")
+    elif validate_args:
+      runtime_assertions += [
+          check_ops.assert_positive(
+              batch_shape,
+              message=("`batch_shape` elements must be positive "
+                       "(i.e., larger than zero)."),
+              name="assert_batch_shape_positive")
+      ]
+
+    return runtime_assertions
-- 
GitLab


From 8abde65d3c7813a36082acfc341d22b0c5e76e02 Mon Sep 17 00:00:00 2001
From: Skye Wanderman-Milne <skyewm@google.com>
Date: Wed, 4 Apr 2018 18:00:27 -0700
Subject: [PATCH 941/960] Sort control inputs alphabetically in
 ToGraphDefSubRange.

PiperOrigin-RevId: 191677358
---
 tensorflow/core/graph/graph.cc | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tensorflow/core/graph/graph.cc b/tensorflow/core/graph/graph.cc
index a7af5e2312..fb8a6c39e6 100644
--- a/tensorflow/core/graph/graph.cc
+++ b/tensorflow/core/graph/graph.cc
@@ -567,6 +567,11 @@ void Graph::ToGraphDefSubRange(GraphDef* graph_def, int from_node_id) const {
         inputs[edge->dst_input()] = edge;
       }
     }
+    // Sort the control inputs for more predictable serialization.
+    std::sort(inputs.begin() + node->num_inputs(), inputs.end(),
+              [](const Edge* a, const Edge* b) -> bool {
+                return a->src()->name() < b->src()->name();
+              });
     node_def->clear_input();
     node_def->mutable_input()->Reserve(inputs.size());
 
-- 
GitLab


From 2194f66f0a905940327d05f2d63c1c7137e47574 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 18:20:36 -0700
Subject: [PATCH 942/960] Add a helper function to re-assign colocation in a
 graph.

PiperOrigin-RevId: 191679495
---
 tensorflow/core/grappler/optimizers/BUILD     |   1 +
 .../grappler/optimizers/meta_optimizer.cc     |   4 +-
 tensorflow/core/grappler/utils/BUILD          |  25 +++
 tensorflow/core/grappler/utils/colocation.cc  | 122 ++++++++++++
 tensorflow/core/grappler/utils/colocation.h   |  39 ++++
 .../core/grappler/utils/colocation_test.cc    | 183 ++++++++++++++++++
 6 files changed, 373 insertions(+), 1 deletion(-)
 create mode 100644 tensorflow/core/grappler/utils/colocation.cc
 create mode 100644 tensorflow/core/grappler/utils/colocation.h
 create mode 100644 tensorflow/core/grappler/utils/colocation_test.cc

diff --git a/tensorflow/core/grappler/optimizers/BUILD b/tensorflow/core/grappler/optimizers/BUILD
index 0c6549d940..122fd48584 100644
--- a/tensorflow/core/grappler/optimizers/BUILD
+++ b/tensorflow/core/grappler/optimizers/BUILD
@@ -508,6 +508,7 @@ cc_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:protos_all_cc",
         "//tensorflow/core/grappler:grappler_item",
+        "//tensorflow/core/grappler/utils:colocation",
         "//tensorflow/core/grappler/utils:topological_sort",
     ],
 )
diff --git a/tensorflow/core/grappler/optimizers/meta_optimizer.cc b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
index ce27d3d95c..5723e397ab 100644
--- a/tensorflow/core/grappler/optimizers/meta_optimizer.cc
+++ b/tensorflow/core/grappler/optimizers/meta_optimizer.cc
@@ -1,4 +1,4 @@
-/* Copyright 2017 The TensorFlow Authors. All Rights Reserved.
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -28,6 +28,7 @@ limitations under the License.
 #include "tensorflow/core/grappler/optimizers/loop_optimizer.h"
 #include "tensorflow/core/grappler/optimizers/memory_optimizer.h"
 #include "tensorflow/core/grappler/optimizers/model_pruner.h"
+#include "tensorflow/core/grappler/utils/colocation.h"
 #include "tensorflow/core/grappler/utils/topological_sort.h"
 #include "tensorflow/core/lib/core/status.h"
 
@@ -221,6 +222,7 @@ Status MetaOptimizer::Optimize(Cluster* cluster, const GrapplerItem& item,
 
   if (already_optimized) {
     TF_RETURN_IF_ERROR(TopologicalSort(optimized_graph));
+    ReassignColocation(optimized_graph);
     // Make sure that the optimizers preserved the graph version and library.
     DCHECK_GE(optimized_graph->library().function_size(),
               item.graph.library().function_size());
diff --git a/tensorflow/core/grappler/utils/BUILD b/tensorflow/core/grappler/utils/BUILD
index baf24c2505..7419c26dff 100644
--- a/tensorflow/core/grappler/utils/BUILD
+++ b/tensorflow/core/grappler/utils/BUILD
@@ -181,3 +181,28 @@ tf_cc_test(
         "//tensorflow/core:testlib",
     ],
 )
+
+cc_library(
+    name = "colocation",
+    srcs = ["colocation.cc"],
+    hdrs = ["colocation.h"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core/grappler:utils",
+    ],
+)
+
+tf_cc_test(
+    name = "colocation_test",
+    size = "small",
+    srcs = ["colocation_test.cc"],
+    deps = [
+        ":colocation",
+        "//tensorflow/core:framework",
+        "//tensorflow/core:protos_all_cc",
+        "//tensorflow/core:test",
+        "//tensorflow/core:test_main",
+        "//tensorflow/core:testlib",
+    ],
+)
diff --git a/tensorflow/core/grappler/utils/colocation.cc b/tensorflow/core/grappler/utils/colocation.cc
new file mode 100644
index 0000000000..0573e0a830
--- /dev/null
+++ b/tensorflow/core/grappler/utils/colocation.cc
@@ -0,0 +1,122 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/utils/colocation.h"
+
+#include <cstring>
+#include "tensorflow/core/framework/attr_value.pb.h"
+#include "tensorflow/core/framework/node_def.pb.h"
+#include "tensorflow/core/grappler/utils.h"
+
+namespace tensorflow {
+namespace grappler {
+
+namespace {
+
+// Find root node of the colocation group.
+// The map is mapping from one node name to its parent. node_name is the
+// starting node to search. By iteratively following the path from child to
+// parent, we can find the root node for the colocation group that node_name
+// belongs to.
+string GetColocationGroupRoot(std::unordered_map<string, string>* map,
+                              const string& node_name) {
+  if (map->find(node_name) == map->end()) {
+    // If node_name is not in the map, we create a new root node which points
+    // to itself.
+    map->insert({node_name, node_name});
+    return node_name;
+  }
+  string cur = node_name;
+  while ((*map)[cur] != cur) {
+    // Backtracing the map until we reach the root node.
+    cur = (*map)[cur];
+  }
+  return cur;
+}
+
+// Merge two colocation groups into one.
+// left and right is the root node of two colocation groups respectively.
+void MergeColocationGroup(std::unordered_map<string, string>* map,
+                          const string& left, const string& right) {
+  // Do nothing if left or right node is not in the map.
+  if (map->find(left) == map->end() || map->find(right) == map->end()) {
+    return;
+  }
+  if (left != right) {
+    // Make the right node a child of the left node, which merges the two
+    // groups.
+    map->at(right) = left;
+  }
+}
+}  // namespace
+
+// Use of disjoint set algorithm to build the colocation groups from the input
+// graph. The core data structure in use is a hash map from one node to its
+// parent node. Whenever we see two nodes colocate with each other, we merge
+// their colocation groups together. After we traverse all colocation pairs
+// in the graph, we will have several disjoint sets. Then we pick the root node
+// of each disjoint set as the representative node, and let all other nodes in
+// the group colocate with the representative node.
+void ReassignColocation(GraphDef* graph) {
+  constexpr char kClassAttr[] = "_class";
+  constexpr char kColocPrefix[] = "loc:@";
+
+  // A hashmap that maps from a node name to its parent node name.
+  std::unordered_map<string, string> coloc_groups;
+  NodeMap node_map(graph);
+  for (const auto& node : graph->node()) {
+    auto iter = node.attr().find(kClassAttr);
+    if (iter != node.attr().end() && iter->second.has_list()) {
+      for (const auto& str : iter->second.list().s()) {
+        size_t pos = str.find(kColocPrefix);
+        if (pos == 0) {
+          // After we find a colocation, update the colocation groups.
+          string colocate_node = str.substr(pos + strlen(kColocPrefix));
+          MergeColocationGroup(
+              &coloc_groups, GetColocationGroupRoot(&coloc_groups, node.name()),
+              GetColocationGroupRoot(&coloc_groups, colocate_node));
+        }
+      }
+    }
+  }
+
+  // We use the root node of each colocation groups as its representative
+  // node. For each node in one group, colocate with the representative node
+  // if the node is in the graph.
+  for (const auto& pair : coloc_groups) {
+    if (pair.first != pair.second) {
+      // This is a child node.
+      NodeDef* node = node_map.GetNode(pair.first);
+      if (node) {
+        // Colocate this node with the root node.
+        AttrValue new_value;
+        new_value.mutable_list()->add_s(
+            kColocPrefix + GetColocationGroupRoot(&coloc_groups, pair.first));
+        node->mutable_attr()->erase(kClassAttr);
+        node->mutable_attr()->insert({kClassAttr, new_value});
+      }
+    } else {
+      // This is a root node. Clear the _class attribute.
+      NodeDef* node = node_map.GetNode(pair.first);
+      if (node) {  // root node should always exist in the graph as guaranteed
+                   // by order of merging. Just put check here to ensure safety.
+        node->mutable_attr()->erase(kClassAttr);
+      }
+    }
+  }
+}
+
+}  // namespace grappler
+}  // namespace tensorflow
diff --git a/tensorflow/core/grappler/utils/colocation.h b/tensorflow/core/grappler/utils/colocation.h
new file mode 100644
index 0000000000..6062db6102
--- /dev/null
+++ b/tensorflow/core/grappler/utils/colocation.h
@@ -0,0 +1,39 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#ifndef TENSORFLOW_CORE_GRAPPLER_UTILS_COLOCATION_H_
+#define TENSORFLOW_CORE_GRAPPLER_UTILS_COLOCATION_H_
+
+#include <unordered_map>
+#include "tensorflow/core/framework/graph.pb.h"
+
+namespace tensorflow {
+namespace grappler {
+
+// Evaluates the colocation relation in the graph and rewrites the new
+// colocation relation in the graph. We scan the graph nodes sequentially, and
+// builds a disjoint-sets of nodes (within each disjoint-set the nodes are
+// colocated with each other). We then select the root node of each set as a
+// representative node, and then colocate each node within the set (should also
+// exist in graph) with the representative node.
+// Note that there is current one situation this function can't handle:
+// Node A colocates with X, node B colocates with Y, X colocates with Y but
+// X, Y are removed from graph. In this case we can't know A colocates with B.
+void ReassignColocation(GraphDef* graph);
+
+}  // namespace grappler
+}  // namespace tensorflow
+
+#endif  // TENSORFLOW_CORE_GRAPPLER_UTILS_COLOCATION_H_
diff --git a/tensorflow/core/grappler/utils/colocation_test.cc b/tensorflow/core/grappler/utils/colocation_test.cc
new file mode 100644
index 0000000000..6638364240
--- /dev/null
+++ b/tensorflow/core/grappler/utils/colocation_test.cc
@@ -0,0 +1,183 @@
+/* Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+==============================================================================*/
+
+#include "tensorflow/core/grappler/utils/colocation.h"
+
+#include "tensorflow/core/framework/function_testlib.h"
+#include "tensorflow/core/framework/node_def_builder.h"
+#include "tensorflow/core/lib/core/status_test_util.h"
+#include "tensorflow/core/platform/test.h"
+
+namespace tensorflow {
+namespace grappler {
+
+class ColocationTest : public ::testing::Test {};
+
+bool VerifyNodeHasColocation(const NodeDef& ndef, const string& coloc) {
+  if (ndef.attr().empty()) {
+    return false;
+  }
+  if (ndef.attr().find("_class") == ndef.attr().end()) {
+    return false;
+  }
+  return ndef.attr().at("_class").list().s(0) == coloc;
+}
+
+TEST(ColocationTest, ReassignColocation_SingleNode) {
+  // Node A colocates with B, but node B is not in the graph.
+  //   A
+  //   |
+  //   |
+  //  [B]
+
+  NodeDef ndef;
+  const Status status =
+      NodeDefBuilder("A", "Const").Attr("_class", {"loc:@B"}).Finalize(&ndef);
+  TF_EXPECT_OK(status);
+  GraphDef gdef = test::function::GDef({ndef});
+
+  EXPECT_EQ(1, gdef.node_size());
+  EXPECT_EQ(1, gdef.node(0).attr_size());
+
+  ReassignColocation(&gdef);
+
+  // Validates that node A's colocation info is cleared.
+  EXPECT_EQ(1, gdef.node_size());
+  EXPECT_EQ(0, gdef.node(0).attr_size());
+}
+
+TEST(ColocationTest, ReassignColocation_MultiNode_SingleGroup) {
+  // Node A, B, C colocate with X. D colocates with C. E colocates with D.
+  // Node X is not in the graph.
+  //  A   B   C---D---E
+  //  |   |   |
+  //  |   |   |
+  //  +--[X]--+
+  // After re-assign of colocation, A, B, C, D should colocate with E.
+  // A   B   C   D
+  // |   |   |   |
+  // |   |   |   |
+  // +---+-E-+---+
+
+  NodeDef ndef_a, ndef_b, ndef_c, ndef_d, ndef_e;
+  Status status =
+      NodeDefBuilder("A", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_a);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("B", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_b);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("C", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_c);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("D", "Const").Attr("_class", {"loc:@C"}).Finalize(&ndef_d);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("E", "Const").Attr("_class", {"loc:@D"}).Finalize(&ndef_e);
+  TF_EXPECT_OK(status);
+  GraphDef gdef =
+      test::function::GDef({ndef_a, ndef_b, ndef_c, ndef_d, ndef_e});
+
+  EXPECT_EQ(5, gdef.node_size());
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(0), "loc:@X"));  // A
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(1), "loc:@X"));  // B
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(2), "loc:@X"));  // C
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(3), "loc:@C"));  // D
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(4), "loc:@D"));  // E
+
+  ReassignColocation(&gdef);
+
+  EXPECT_EQ(5, gdef.node_size());
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(0), "loc:@E"));  // A
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(1), "loc:@E"));  // B
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(2), "loc:@E"));  // C
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(3), "loc:@E"));  // D
+  EXPECT_EQ(0, gdef.node(4).attr_size());                        // E
+}
+
+TEST(ColocationTest, ReassignColocation_MultiNode_MultiGroup) {
+  // Before re-assign:
+  // Node A, B, C colocate with X. D colocates with C. E colocates with D.
+  // Node U, V colocates with W. Node X, W are not in the graph:
+  //  A   B   C---D---E
+  //  |   |   |
+  //  |   |   |
+  //  +--[X]--+
+  //
+  //  U       V
+  //  |       |
+  //  |       |
+  //  +--[W]--+
+  //
+  // After re-assign:
+  // A, B, C, D should colocate with E. U should colocate with V.
+  // A   B   C   D
+  // |   |   |   |
+  // |   |   |   |
+  // +---+-E-+---+
+  //
+  // U
+  // |
+  // |
+  // V
+
+  NodeDef ndef_a, ndef_b, ndef_c, ndef_d, ndef_e, ndef_u, ndef_v;
+  Status status =
+      NodeDefBuilder("A", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_a);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("B", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_b);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("C", "Const").Attr("_class", {"loc:@X"}).Finalize(&ndef_c);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("D", "Const").Attr("_class", {"loc:@C"}).Finalize(&ndef_d);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("E", "Const").Attr("_class", {"loc:@D"}).Finalize(&ndef_e);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("U", "Const").Attr("_class", {"loc:@W"}).Finalize(&ndef_u);
+  TF_EXPECT_OK(status);
+  status =
+      NodeDefBuilder("V", "Const").Attr("_class", {"loc:@W"}).Finalize(&ndef_v);
+  TF_EXPECT_OK(status);
+  GraphDef gdef = test::function::GDef(
+      {ndef_a, ndef_b, ndef_c, ndef_d, ndef_e, ndef_u, ndef_v});
+
+  EXPECT_EQ(7, gdef.node_size());
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(0), "loc:@X"));  // A
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(1), "loc:@X"));  // B
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(2), "loc:@X"));  // C
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(3), "loc:@C"));  // D
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(4), "loc:@D"));  // E
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(5), "loc:@W"));  // U
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(6), "loc:@W"));  // V
+
+  ReassignColocation(&gdef);
+
+  EXPECT_EQ(7, gdef.node_size());
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(0), "loc:@E"));  // A
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(1), "loc:@E"));  // B
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(2), "loc:@E"));  // C
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(3), "loc:@E"));  // D
+  EXPECT_EQ(0, gdef.node(4).attr_size());                        // E
+  EXPECT_TRUE(VerifyNodeHasColocation(gdef.node(5), "loc:@V"));  // U
+  EXPECT_EQ(0, gdef.node(6).attr_size());                        // V
+}
+
+}  // namespace grappler
+}  // namespace tensorflow
-- 
GitLab


From aa890776f062f3429bcedb0a080b712ebb97793b Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Wed, 4 Apr 2018 22:37:35 -0700
Subject: [PATCH 943/960] Support arbitrary permutations for DataFormatDimMap.

PiperOrigin-RevId: 191696203
---
 tensorflow/core/kernels/data_format_ops.cc | 40 ++++++++++++++--------
 tensorflow/core/kernels/data_format_ops.h  | 18 +++++++---
 tensorflow/python/ops/nn_test.py           | 36 +++++++++++++++++++
 3 files changed, 76 insertions(+), 18 deletions(-)

diff --git a/tensorflow/core/kernels/data_format_ops.cc b/tensorflow/core/kernels/data_format_ops.cc
index 39ef8ee3ac..4485152e96 100644
--- a/tensorflow/core/kernels/data_format_ops.cc
+++ b/tensorflow/core/kernels/data_format_ops.cc
@@ -37,25 +37,37 @@ class DataFormatDimMapOp : public OpKernel {
     OP_REQUIRES_OK(context, context->GetAttr("src_format", &src_format));
     string dst_format;
     OP_REQUIRES_OK(context, context->GetAttr("dst_format", &dst_format));
+    OP_REQUIRES(context, src_format.size() == 4,
+                errors::InvalidArgument(strings::StrCat(
+                    "Source format must of length 4, received src_format = ",
+                    src_format)));
     OP_REQUIRES(
-        context, src_format == "NHWC",
+        context, dst_format.size() == 4,
         errors::InvalidArgument(strings::StrCat(
-            "Current implementation doesn't support source data format ",
-            src_format)));
-    OP_REQUIRES(context, dst_format == "NCHW",
-                errors::InvalidArgument(strings::StrCat(
-                    "Current implementation doesn't support dst data format ",
-                    dst_format)));
+            "Destination format must of length 4, received dst_format = ",
+            dst_format)));
+    dst_idx_ = Tensor(DT_INT32, {static_cast<int64>(src_format.size())});
+    for (int i = 0; i < src_format.size(); ++i) {
+      for (int j = 0; j < dst_format.size(); ++j) {
+        if (dst_format[j] == src_format[i]) {
+          dst_idx_.vec<int>()(i) = j;
+          break;
+        }
+      }
+    }
   }
 
   void Compute(OpKernelContext* context) override {
     const Tensor& input = context->input(0);
-    Tensor* output = nullptr;
+    Tensor* output;
     OP_REQUIRES_OK(context,
                    context->allocate_output(0, input.shape(), &output));
     functor::DataFormatDimMap<Device, T>()(context->eigen_device<Device>(),
-                                           input.flat<T>(), output->flat<T>());
+                                           input.flat<T>(), output->flat<T>(),
+                                           dst_idx_.vec<int>());
   }
+
+  Tensor dst_idx_;
 };
 
 template <typename Device, typename T>
@@ -147,11 +159,11 @@ TF_CALL_int64(REGISTER_KERNEL);
 #if GOOGLE_CUDA
 // Forward declarations of the functor specializations for GPU.
 namespace functor {
-#define DECLARE_GPU_SPEC(T)                                \
-  template <>                                              \
-  void DataFormatDimMap<GPUDevice, T>::operator()(         \
-      const GPUDevice& d, typename TTypes<T>::ConstFlat x, \
-      typename TTypes<T>::Flat y);                         \
+#define DECLARE_GPU_SPEC(T)                                    \
+  template <>                                                  \
+  void DataFormatDimMap<GPUDevice, T>::operator()(             \
+      const GPUDevice& d, typename TTypes<T>::ConstFlat x,     \
+      typename TTypes<T>::Flat y, const TTypes<int>::Vec dst); \
   extern template struct DataFormatDimMap<GPUDevice, T>;
 #define DECLARE_GPU_SPECS(T) DECLARE_GPU_SPEC(T);
 TF_CALL_int32(DECLARE_GPU_SPECS);
diff --git a/tensorflow/core/kernels/data_format_ops.h b/tensorflow/core/kernels/data_format_ops.h
index 2ccc919586..1ca144cb40 100644
--- a/tensorflow/core/kernels/data_format_ops.h
+++ b/tensorflow/core/kernels/data_format_ops.h
@@ -27,15 +27,25 @@ namespace functor {
 template <typename Device, typename T>
 struct DataFormatDimMap {
   void operator()(const Device& d, typename TTypes<T>::ConstFlat x,
-                  typename TTypes<T>::Flat y) {
+                  typename TTypes<T>::Flat y, const TTypes<int>::Vec dst) {
     auto zero = x.constant(0);
     auto one = x.constant(1);
-    auto three = x.constant(3);
+    auto two = x.constant(2);
+
+    auto f_zero = x.constant(dst(0));
+    auto f_one = x.constant(dst(1));
+    auto f_two = x.constant(dst(2));
+    auto f_three = x.constant(dst(3));
+
     auto four = x.constant(4);
     auto x_mod = (x + four) % 4;
+
     auto is_zero = (x_mod == zero);
-    auto is_three = (x_mod == three);
-    y.device(d) = is_zero.select(zero, is_three.select(one, x_mod + one));
+    auto is_one = (x_mod == one);
+    auto is_two = (x_mod == two);
+
+    y.device(d) = is_zero.select(
+        f_zero, is_one.select(f_one, is_two.select(f_two, f_three)));
   }
 };
 
diff --git a/tensorflow/python/ops/nn_test.py b/tensorflow/python/ops/nn_test.py
index da86d5f6ca..46a5f4fae6 100644
--- a/tensorflow/python/ops/nn_test.py
+++ b/tensorflow/python/ops/nn_test.py
@@ -1081,6 +1081,42 @@ class DataFormatDimMapTest(test_lib.TestCase):
     self._test([1, -3, -2], [2, 2, 3])
     self._test([[1, -3], [1, -1]], [[2, 2], [2, 1]])
 
+  def testNHWCtoNCHW(self):
+    x_val = [1, -3, -2]
+    y_val_expected = [2, 2, 3]
+    x = constant_op.constant(x_val)
+    y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="NCHW")
+    with self.test_session(use_gpu=test_lib.is_gpu_available()) as sess:
+      y_val = sess.run(y)
+      self.assertAllEqual(y_val, y_val_expected)
+
+  def testNHWCtoHWNC(self):
+    x_val = [-4, -3, -2, -1, 0, 1, 2, 3]
+    y_val_expected = [2, 0, 1, 3, 2, 0, 1, 3]
+    x = constant_op.constant(x_val)
+    y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="HWNC")
+    with self.test_session(use_gpu=test_lib.is_gpu_available()) as sess:
+      y_val = sess.run(y)
+      self.assertAllEqual(y_val, y_val_expected)
+
+  def testNHWCtoWHCN(self):
+    x_val = [-4, -3, -2, -1, 0, 1, 2, 3]
+    y_val_expected = [3, 1, 0, 2, 3, 1, 0, 2]
+    x = constant_op.constant(x_val)
+    y = nn_ops.data_format_dim_map(x, src_format="NHWC", dst_format="WHCN")
+    with self.test_session(use_gpu=test_lib.is_gpu_available()) as sess:
+      y_val = sess.run(y)
+      self.assertAllEqual(y_val, y_val_expected)
+
+  def testArbitraryASCII(self):
+    x_val = [-4, -3, -2, -1, 0, 1, 2, 3]
+    y_val_expected = [3, 2, 1, 0, 3, 2, 1, 0]
+    x = constant_op.constant(x_val)
+    y = nn_ops.data_format_dim_map(x, src_format="qwer", dst_format="rewq")
+    with self.test_session(use_gpu=test_lib.is_gpu_available()) as sess:
+      y_val = sess.run(y)
+      self.assertAllEqual(y_val, y_val_expected)
+
 
 class DataFormatVectorPermuteTest(test_lib.TestCase):
 
-- 
GitLab


From 007e3e6c3d72ea5edca361eb908fb7aa66ac6d6d Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Wed, 4 Apr 2018 22:37:42 -0700
Subject: [PATCH 944/960] Upgrade zlib to latest version

PiperOrigin-RevId: 191696213
---
 tensorflow/workspace.bzl |  8 ++++----
 third_party/zlib.BUILD   | 16 ++--------------
 2 files changed, 6 insertions(+), 18 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 2510d369fc..cd8b6f83a9 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -503,11 +503,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "zlib_archive",
       urls = [
-          "https://mirror.bazel.build/zlib.net/zlib-1.2.8.tar.gz",
-          "http://zlib.net/fossils/zlib-1.2.8.tar.gz",
+          "https://mirror.bazel.build/zlib.net/zlib-1.2.11.tar.gz",
+          "https://zlib.net/zlib-1.2.11.tar.gz",
       ],
-      sha256 = "36658cb768a54c1d4dec43c3116c27ed893e88b02ecfcb44f2166f9c0b7f2a0d",
-      strip_prefix = "zlib-1.2.8",
+      sha256 = "c3e5e9fdd5004dcb542feda5ee4f0ff0744628baf8ed2dd5d66f8ca1197cb1a1",
+      strip_prefix = "zlib-1.2.11",
       build_file = clean_dep("//third_party:zlib.BUILD"),
   )
 
diff --git a/third_party/zlib.BUILD b/third_party/zlib.BUILD
index d164ee719c..e8048dd98a 100644
--- a/third_party/zlib.BUILD
+++ b/third_party/zlib.BUILD
@@ -2,18 +2,6 @@ package(default_visibility = ["//visibility:public"])
 
 licenses(["notice"])  # BSD/MIT-like license (for zlib)
 
-config_setting(
-    name = "windows",
-    values = {"cpu": "x64_windows"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "windows_msvc",
-    values = {"cpu": "x64_windows_msvc"},
-    visibility = ["//visibility:public"],
-)
-
 cc_library(
     name = "zlib",
     srcs = [
@@ -45,8 +33,8 @@ cc_library(
     ],
     hdrs = ["zlib.h"],
     copts = select({
-        ":windows": [],
-        ":windows_msvc": [],
+        "@org_tensorflow//tensorflow:windows": [],
+        "@org_tensorflow//tensorflow:windows_msvc": [],
         "//conditions:default": [
             "-Wno-shift-negative-value",
             "-DZ_HAVE_UNISTD_H",
-- 
GitLab


From dfa9921e6343727b05f42f8d4a918b19528ff994 Mon Sep 17 00:00:00 2001
From: Justine Tunney <jart@google.com>
Date: Thu, 5 Apr 2018 00:53:51 -0700
Subject: [PATCH 945/960] Upgrade Snappy to latest version

They added big endian preprocessor macros in recent releases. Hopefully
this should do the right thing on IBM mainframes.

PiperOrigin-RevId: 191705207
---
 tensorflow/workspace.bzl |  8 ++--
 third_party/snappy.BUILD | 96 +++++++++++++++++++++++++++++-----------
 2 files changed, 73 insertions(+), 31 deletions(-)

diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index cd8b6f83a9..23f6d3c1d9 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -524,11 +524,11 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   tf_http_archive(
       name = "snappy",
       urls = [
-          "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.4.tar.gz",
-          "https://github.com/google/snappy/archive/1.1.4.tar.gz",
+          "https://mirror.bazel.build/github.com/google/snappy/archive/1.1.7.tar.gz",
+          "https://github.com/google/snappy/archive/1.1.7.tar.gz",
       ],
-      sha256 = "2f7504c73d85bac842e893340333be8cb8561710642fc9562fccdd9d2c3fcc94",
-      strip_prefix = "snappy-1.1.4",
+      sha256 = "3dfa02e873ff51a11ee02b9ca391807f0c8ea0529a4924afa645fbf97163f9d4",
+      strip_prefix = "snappy-1.1.7",
       build_file = clean_dep("//third_party:snappy.BUILD"),
   )
 
diff --git a/third_party/snappy.BUILD b/third_party/snappy.BUILD
index fd48ed8941..cc11f52d0e 100644
--- a/third_party/snappy.BUILD
+++ b/third_party/snappy.BUILD
@@ -4,25 +4,12 @@ licenses(["notice"])  # BSD 3-Clause
 
 exports_files(["COPYING"])
 
-config_setting(
-    name = "windows",
-    values = {"cpu": "x64_windows"},
-    visibility = ["//visibility:public"],
-)
-
-config_setting(
-    name = "windows_msvc",
-    values = {"cpu": "x64_windows_msvc"},
-    visibility = ["//visibility:public"],
-)
-
 cc_library(
     name = "snappy",
     srcs = [
+        "config.h",
         "snappy.cc",
         "snappy.h",
-        "snappy-c.cc",
-        "snappy-c.h",
         "snappy-internal.h",
         "snappy-sinksource.cc",
         "snappy-sinksource.h",
@@ -32,30 +19,85 @@ cc_library(
     ],
     hdrs = ["snappy.h"],
     copts = select({
-        ":windows": [],
-        ":windows_msvc": [],
+        "@org_tensorflow//tensorflow:windows": [
+            "/DHAVE_CONFIG_H",
+            "/EHsc",
+        ],
+        "@org_tensorflow//tensorflow:windows_msvc": [
+            "/DHAVE_CONFIG_H",
+            "/EHsc",
+        ],
         "//conditions:default": [
+            "-DHAVE_CONFIG_H",
+            "-fno-exceptions",
+            "-Wno-sign-compare",
             "-Wno-shift-negative-value",
             "-Wno-implicit-function-declaration",
         ],
     }),
 )
 
+genrule(
+    name = "config_h",
+    outs = ["config.h"],
+    cmd = "\n".join([
+        "cat <<'EOF' >$@",
+        "#define HAVE_STDDEF_H 1",
+        "#define HAVE_STDINT_H 1",
+        "",
+        "#ifdef __has_builtin",
+        "#  if !defined(HAVE_BUILTIN_EXPECT) && __has_builtin(__builtin_expect)",
+        "#    define HAVE_BUILTIN_EXPECT 1",
+        "#  endif",
+        "#  if !defined(HAVE_BUILTIN_CTZ) && __has_builtin(__builtin_ctzll)",
+        "#    define HAVE_BUILTIN_CTZ 1",
+        "#  endif",
+        "#elif defined(__GNUC__) && (__GNUC__ > 3 || __GNUC__ == 3 && __GNUC_MINOR__ >= 4)",
+        "#  ifndef HAVE_BUILTIN_EXPECT",
+        "#    define HAVE_BUILTIN_EXPECT 1",
+        "#  endif",
+        "#  ifndef HAVE_BUILTIN_CTZ",
+        "#    define HAVE_BUILTIN_CTZ 1",
+        "#  endif",
+        "#endif",
+        "",
+        "#ifdef __has_include",
+        "#  if !defined(HAVE_BYTESWAP_H) && __has_include(<byteswap.h>)",
+        "#    define HAVE_BYTESWAP_H 1",
+        "#  endif",
+        "#  if !defined(HAVE_UNISTD_H) && __has_include(<unistd.h>)",
+        "#    define HAVE_UNISTD_H 1",
+        "#  endif",
+        "#  if !defined(HAVE_SYS_ENDIAN_H) && __has_include(<sys/endian.h>)",
+        "#    define HAVE_SYS_ENDIAN_H 1",
+        "#  endif",
+        "#  if !defined(HAVE_SYS_MMAN_H) && __has_include(<sys/mman.h>)",
+        "#    define HAVE_SYS_MMAN_H 1",
+        "#  endif",
+        "#  if !defined(HAVE_SYS_UIO_H) && __has_include(<sys/uio.h>)",
+        "#    define HAVE_SYS_UIO_H 1",
+        "#  endif",
+        "#endif",
+        "",
+        "#ifndef SNAPPY_IS_BIG_ENDIAN",
+        "#  ifdef __s390x__",
+        "#    define SNAPPY_IS_BIG_ENDIAN 1",
+        "#  elif defined(__BYTE_ORDER__) && defined(__ORDER_BIG_ENDIAN__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__",
+        "#    define SNAPPY_IS_BIG_ENDIAN 1",
+        "#  endif",
+        "#endif",
+        "EOF",
+    ]),
+)
+
 genrule(
     name = "snappy_stubs_public_h",
     srcs = ["snappy-stubs-public.h.in"],
     outs = ["snappy-stubs-public.h"],
     cmd = ("sed " +
-           "-e 's/@ac_cv_have_stdint_h@/1/g' " +
-           "-e 's/@ac_cv_have_stddef_h@/1/g' " +
-           "-e 's/@ac_cv_have_stdint_h@/1/g' " +
-           select({
-               "@org_tensorflow//tensorflow:windows": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ",
-               "@org_tensorflow//tensorflow:windows_msvc": "-e 's/@ac_cv_have_sys_uio_h@/0/g' ",
-               "//conditions:default": "-e 's/@ac_cv_have_sys_uio_h@/1/g' ",
-           }) +
-           "-e 's/@SNAPPY_MAJOR@/1/g' " +
-           "-e 's/@SNAPPY_MINOR@/1/g' " +
-           "-e 's/@SNAPPY_PATCHLEVEL@/4/g' " +
+           "-e 's/$${\\(.*\\)_01}/\\1/g' " +
+           "-e 's/$${SNAPPY_MAJOR}/1/g' " +
+           "-e 's/$${SNAPPY_MINOR}/1/g' " +
+           "-e 's/$${SNAPPY_PATCHLEVEL}/4/g' " +
            "$< >$@"),
 )
-- 
GitLab


From 1fda7645d132b71b9084b01945795e97e582adcd Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Apr 2018 03:09:27 -0700
Subject: [PATCH 946/960] Add support for NCCL2. The configure script asks for
 what version of NCCL to use. The default is still NCCL 1 from GitHub. If the
 user chooses NCCL 2, it asks for the install directory.

The nccl_configure.bzl generates two different BUILD files based on the chose NCCL version. For NCCL 1, it aliases to the existing 'nccl_archive' http_repo on GitHub. For NCCL 2, it creates a target containing the NCCL 2 library and headers from the chosen install directory.

PiperOrigin-RevId: 191718007
---
 configure.py                                  |  77 +++++++
 tensorflow/contrib/nccl/BUILD                 |   6 +-
 .../contrib/nccl/kernels/nccl_manager.h       |   2 +-
 tensorflow/contrib/nccl/kernels/nccl_ops.cc   |   2 +-
 tensorflow/tools/pip_package/BUILD            |   3 +-
 tensorflow/workspace.bzl                      |   4 +-
 third_party/nccl/LICENSE                      | 203 ++++++++++++++++++
 .../{nccl.BUILD => nccl/nccl_archive.BUILD}   |   2 +
 third_party/nccl/nccl_configure.bzl           | 172 +++++++++++++++
 9 files changed, 463 insertions(+), 8 deletions(-)
 create mode 100644 third_party/nccl/LICENSE
 rename third_party/{nccl.BUILD => nccl/nccl_archive.BUILD} (95%)
 create mode 100644 third_party/nccl/nccl_configure.bzl

diff --git a/configure.py b/configure.py
index 26eff5767e..da3f97ab30 100644
--- a/configure.py
+++ b/configure.py
@@ -35,6 +35,7 @@ except ImportError:
 
 _DEFAULT_CUDA_VERSION = '9.0'
 _DEFAULT_CUDNN_VERSION = '7'
+_DEFAULT_NCCL_VERSION = '1.3'
 _DEFAULT_CUDA_COMPUTE_CAPABILITIES = '3.5,5.2'
 _DEFAULT_CUDA_PATH = '/usr/local/cuda'
 _DEFAULT_CUDA_PATH_LINUX = '/opt/cuda'
@@ -1105,6 +1106,81 @@ def set_tf_tensorrt_install_path(environ_cp):
   write_action_env_to_bazelrc('TF_TENSORRT_VERSION', tf_tensorrt_version)
 
 
+def set_tf_nccl_install_path(environ_cp):
+  """Set NCCL_INSTALL_PATH and TF_NCCL_VERSION.
+
+  Args:
+    environ_cp: copy of the os.environ.
+
+  Raises:
+    ValueError: if this method was called under non-Linux platform.
+    UserInputError: if user has provided invalid input multiple times.
+  """
+  if not is_linux():
+    raise ValueError('Currently NCCL is only supported on Linux platforms.')
+
+  ask_nccl_version = (
+      'Please specify the NCCL version you want to use. '
+      '[Leave empty to default to NCCL %s]: ') % _DEFAULT_NCCL_VERSION
+
+  for _ in range(_DEFAULT_PROMPT_ASK_ATTEMPTS):
+    tf_nccl_version = get_from_env_or_user_or_default(
+        environ_cp, 'TF_NCCL_VERSION', ask_nccl_version, _DEFAULT_NCCL_VERSION)
+    tf_nccl_version = reformat_version_sequence(str(tf_nccl_version), 1)
+
+    if tf_nccl_version == '1':
+      break  # No need to get install path, NCCL 1 is a GitHub repo.
+
+    # TODO(csigg): Look with ldconfig first if we can find the library in paths
+    # like /usr/lib/x86_64-linux-gnu and the header file in the corresponding
+    # include directory. This is where the NCCL .deb packages install them.
+    # Then ask the user if we should use that. Instead of a single
+    # NCCL_INSTALL_PATH, pass separate NCCL_LIB_PATH and NCCL_HDR_PATH to
+    # nccl_configure.bzl
+    default_nccl_path = environ_cp.get('CUDA_TOOLKIT_PATH')
+    ask_nccl_path = (r'Please specify the location where NCCL %s library is '
+                     'installed. Refer to README.md for more details. [Default '
+                     'is %s]:') % (tf_nccl_version, default_nccl_path)
+    nccl_install_path = get_from_env_or_user_or_default(
+        environ_cp, 'NCCL_INSTALL_PATH', ask_nccl_path, default_nccl_path)
+
+    # Result returned from "read" will be used unexpanded. That make "~"
+    # unusable. Going through one more level of expansion to handle that.
+    nccl_install_path = os.path.realpath(os.path.expanduser(nccl_install_path))
+    if is_windows() or is_cygwin():
+      nccl_install_path = cygpath(nccl_install_path)
+
+    if is_windows():
+      nccl_lib_path = 'lib/x64/nccl.lib'
+    elif is_linux():
+      nccl_lib_path = 'lib/libnccl.so.%s' % tf_nccl_version
+    elif is_macos():
+      nccl_lib_path = 'lib/libnccl.%s.dylib' % tf_nccl_version
+
+    nccl_lib_path = os.path.join(nccl_install_path, nccl_lib_path)
+    nccl_hdr_path = os.path.join(nccl_install_path, 'include/nccl.h')
+    if os.path.exists(nccl_lib_path) and os.path.exists(nccl_hdr_path):
+      # Set NCCL_INSTALL_PATH
+      environ_cp['NCCL_INSTALL_PATH'] = nccl_install_path
+      write_action_env_to_bazelrc('NCCL_INSTALL_PATH', nccl_install_path)
+      break
+
+    # Reset and Retry
+    print('Invalid path to NCCL %s toolkit, %s or %s not found. Please use the '
+          'O/S agnostic package of NCCL 2' % (tf_nccl_version, nccl_lib_path,
+                                              nccl_hdr_path))
+
+    environ_cp['TF_NCCL_VERSION'] = ''
+  else:
+    raise UserInputError('Invalid TF_NCCL setting was provided %d '
+                         'times in a row. Assuming to be a scripting mistake.' %
+                         _DEFAULT_PROMPT_ASK_ATTEMPTS)
+
+  # Set TF_NCCL_VERSION
+  environ_cp['TF_NCCL_VERSION'] = tf_nccl_version
+  write_action_env_to_bazelrc('TF_NCCL_VERSION', tf_nccl_version)
+
+
 def get_native_cuda_compute_capabilities(environ_cp):
   """Get native cuda compute capabilities.
 
@@ -1441,6 +1517,7 @@ def main():
     set_tf_cudnn_version(environ_cp)
     if is_linux():
       set_tf_tensorrt_install_path(environ_cp)
+    set_tf_nccl_install_path(environ_cp)
     set_tf_cuda_compute_capabilities(environ_cp)
     if 'LD_LIBRARY_PATH' in environ_cp and environ_cp.get(
         'LD_LIBRARY_PATH') != '1':
diff --git a/tensorflow/contrib/nccl/BUILD b/tensorflow/contrib/nccl/BUILD
index 6cbfd03881..334e70318d 100644
--- a/tensorflow/contrib/nccl/BUILD
+++ b/tensorflow/contrib/nccl/BUILD
@@ -31,7 +31,7 @@ tf_custom_op_library(
         "kernels/nccl_ops.cc",
     ],
     deps = if_cuda([
-        "@nccl_archive//:nccl",
+        "@local_config_nccl//:nccl",
         "//tensorflow/core:gpu_headers_lib",
     ]),
 )
@@ -61,7 +61,7 @@ tf_cuda_cc_test(
             "//tensorflow/core:test",
             "//tensorflow/core:test_main",
             "//tensorflow/core:testlib",
-            "@nccl_archive//:nccl",
+            "@local_config_nccl//:nccl",
         ],
 )
 
@@ -80,7 +80,7 @@ tf_kernel_library(
         "//tensorflow/core:lib",
         "//tensorflow/core:proto_text",
         "//tensorflow/core:stream_executor",
-        "@nccl_archive//:nccl",
+        "@local_config_nccl//:nccl",
     ],
     alwayslink = 1,
 )
diff --git a/tensorflow/contrib/nccl/kernels/nccl_manager.h b/tensorflow/contrib/nccl/kernels/nccl_manager.h
index bb219e0edc..6ff8cea84e 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_manager.h
+++ b/tensorflow/contrib/nccl/kernels/nccl_manager.h
@@ -20,7 +20,7 @@ limitations under the License.
 #include <unordered_map>
 #include <vector>
 
-#include "src/nccl.h"
+#include "third_party/nccl/nccl.h"
 #include "tensorflow/core/common_runtime/gpu/gpu_event_mgr.h"
 #include "tensorflow/core/framework/tensor.h"
 #include "tensorflow/core/platform/mutex.h"
diff --git a/tensorflow/contrib/nccl/kernels/nccl_ops.cc b/tensorflow/contrib/nccl/kernels/nccl_ops.cc
index 266d4f6f0d..c2b76caef3 100644
--- a/tensorflow/contrib/nccl/kernels/nccl_ops.cc
+++ b/tensorflow/contrib/nccl/kernels/nccl_ops.cc
@@ -17,7 +17,7 @@ limitations under the License.
 
 #include <vector>
 
-#include "src/nccl.h"
+#include "third_party/nccl/nccl.h"
 #include "tensorflow/contrib/nccl/kernels/nccl_manager.h"
 #include "tensorflow/core/framework/op_kernel.h"
 
diff --git a/tensorflow/tools/pip_package/BUILD b/tensorflow/tools/pip_package/BUILD
index 4a70f666b6..376644718f 100644
--- a/tensorflow/tools/pip_package/BUILD
+++ b/tensorflow/tools/pip_package/BUILD
@@ -141,6 +141,7 @@ filegroup(
         "@kafka//:LICENSE",
         "@libxsmm_archive//:LICENSE",
         "@lmdb//:LICENSE",
+        "@local_config_nccl//:LICENSE",
         "@local_config_sycl//sycl:LICENSE.text",
         "@grpc//third_party/nanopb:LICENSE.txt",
         "@grpc//third_party/address_sorting:LICENSE",
@@ -157,8 +158,6 @@ filegroup(
         "@org_python_pypi_backports_weakref//:LICENSE",
     ] + if_mkl([
         "//third_party/mkl:LICENSE",
-    ]) + if_not_windows([
-        "@nccl_archive//:LICENSE.txt",
     ]) + tf_additional_license_deps(),
 )
 
diff --git a/tensorflow/workspace.bzl b/tensorflow/workspace.bzl
index 23f6d3c1d9..ace0d411b9 100644
--- a/tensorflow/workspace.bzl
+++ b/tensorflow/workspace.bzl
@@ -2,6 +2,7 @@
 
 load("//third_party/gpus:cuda_configure.bzl", "cuda_configure")
 load("//third_party/tensorrt:tensorrt_configure.bzl", "tensorrt_configure")
+load("//third_party:nccl/nccl_configure.bzl", "nccl_configure")
 load("//third_party/mkl:build_defs.bzl", "mkl_repository")
 load("//third_party/git:git_configure.bzl", "git_configure")
 load("//third_party/py:python_configure.bzl", "python_configure")
@@ -31,6 +32,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
   cc_download_clang_toolchain(name="local_config_download_clang")
   cuda_configure(name="local_config_cuda")
   tensorrt_configure(name="local_config_tensorrt")
+  nccl_configure(name="local_config_nccl")
   git_configure(name="local_config_git")
   sycl_configure(name="local_config_sycl")
   python_configure(name="local_config_python")
@@ -540,7 +542,7 @@ def tf_workspace(path_prefix="", tf_repo_name=""):
       ],
       sha256 = "2ca86fb6179ecbff789cc67c836139c1bbc0324ed8c04643405a30bf26325176",
       strip_prefix = "nccl-03d856977ecbaac87e598c0c4bafca96761b9ac7",
-      build_file = clean_dep("//third_party:nccl.BUILD"),
+      build_file = clean_dep("//third_party:nccl/nccl_archive.BUILD"),
   )
 
   tf_http_archive(
diff --git a/third_party/nccl/LICENSE b/third_party/nccl/LICENSE
new file mode 100644
index 0000000000..146d9b765c
--- /dev/null
+++ b/third_party/nccl/LICENSE
@@ -0,0 +1,203 @@
+Copyright 2018 The TensorFlow Authors.  All rights reserved.
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright 2018, The TensorFlow Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
diff --git a/third_party/nccl.BUILD b/third_party/nccl/nccl_archive.BUILD
similarity index 95%
rename from third_party/nccl.BUILD
rename to third_party/nccl/nccl_archive.BUILD
index b2b8e18824..a05899e38d 100644
--- a/third_party/nccl.BUILD
+++ b/third_party/nccl/nccl_archive.BUILD
@@ -43,6 +43,7 @@ cc_library(
         "-Iexternal/nccl_archive/src",
         "-O3",
     ] + cuda_default_copts(),
+    include_prefix = "third_party/nccl",
     linkopts = select({
         "@org_tensorflow//tensorflow:android": [
             "-pie",
@@ -61,6 +62,7 @@ cc_library(
             "-lrt",
         ],
     }),
+    strip_include_prefix = "src",
     visibility = ["//visibility:public"],
     deps = ["@local_config_cuda//cuda:cuda_headers"],
 )
diff --git a/third_party/nccl/nccl_configure.bzl b/third_party/nccl/nccl_configure.bzl
new file mode 100644
index 0000000000..9dfcb18369
--- /dev/null
+++ b/third_party/nccl/nccl_configure.bzl
@@ -0,0 +1,172 @@
+# -*- Python -*-
+"""Repository rule for NCCL configuration.
+
+`nccl_configure` depends on the following environment variables:
+
+  * `TF_NCCL_VERSION`: The NCCL version.
+  * `NCCL_INSTALL_PATH`: The installation path of the NCCL library.
+"""
+
+load(
+    "//third_party/gpus:cuda_configure.bzl",
+    "auto_configure_fail",
+    "find_cuda_define",
+    "matches_version",
+)
+
+_NCCL_INSTALL_PATH = "NCCL_INSTALL_PATH"
+_TF_NCCL_VERSION = "TF_NCCL_VERSION"
+
+_DEFINE_NCCL_MAJOR = "#define NCCL_MAJOR"
+_DEFINE_NCCL_MINOR = "#define NCCL_MINOR"
+_DEFINE_NCCL_PATCH = "#define NCCL_PATCH"
+
+_NCCL_DUMMY_BUILD_CONTENT = """
+filegroup(
+  name = "LICENSE",
+  visibility = ["//visibility:public"],
+)
+
+cc_library(
+  name = "nccl",
+  visibility = ["//visibility:public"],
+)
+"""
+
+_NCCL_ARCHIVE_BUILD_CONTENT = """
+filegroup(
+  name = "LICENSE",
+  data = ["@nccl_archive//:LICENSE.txt"],
+  visibility = ["//visibility:public"],
+)
+
+alias(
+  name = "nccl",
+  actual = "@nccl_archive//:nccl",
+  visibility = ["//visibility:public"],
+)
+"""
+
+_NCCL_LOCAL_BUILD_TEMPLATE = """
+filegroup(
+  name = "LICENSE",
+  data = ["nccl/NCCL-SLA.txt"],
+  visibility = ["//visibility:public"],
+)
+
+cc_library(
+  name = "nccl",
+  srcs = ["nccl/lib/libnccl.so.%s"],
+  hdrs = ["nccl/include/nccl.h"],
+  include_prefix = "third_party/nccl",
+  strip_include_prefix = "nccl/include",
+  deps = [
+      "@local_config_cuda//cuda:cuda_headers",
+  ],
+  visibility = ["//visibility:public"],
+)
+"""
+
+
+def _find_nccl_header(repository_ctx, nccl_install_path):
+  """Finds the NCCL header on the system.
+
+  Args:
+    repository_ctx: The repository context.
+    nccl_install_path: The NCCL library install directory.
+
+  Returns:
+    The path to the NCCL header.
+  """
+  header_path = repository_ctx.path("%s/include/nccl.h" % nccl_install_path)
+  if not header_path.exists:
+    auto_configure_fail("Cannot find %s" % str(header_path))
+  return header_path
+
+
+def _check_nccl_version(repository_ctx, nccl_install_path, nccl_version):
+  """Checks whether the header file matches the specified version of NCCL.
+
+  Args:
+    repository_ctx: The repository context.
+    nccl_install_path: The NCCL library install directory.
+    nccl_version: The expected NCCL version.
+
+  Returns:
+    A string containing the library version of NCCL.
+  """
+  header_path = _find_nccl_header(repository_ctx, nccl_install_path)
+  header_dir = str(header_path.realpath.dirname)
+  major_version = find_cuda_define(repository_ctx, header_dir, "nccl.h",
+                                   _DEFINE_NCCL_MAJOR)
+  minor_version = find_cuda_define(repository_ctx, header_dir, "nccl.h",
+                                   _DEFINE_NCCL_MINOR)
+  patch_version = find_cuda_define(repository_ctx, header_dir, "nccl.h",
+                                   _DEFINE_NCCL_PATCH)
+  header_version = "%s.%s.%s" % (major_version, minor_version, patch_version)
+  if not matches_version(nccl_version, header_version):
+    auto_configure_fail(
+        ("NCCL library version detected from %s/nccl.h (%s) does not match " +
+         "TF_NCCL_VERSION (%s). To fix this rerun configure again.") %
+        (header_dir, header_version, nccl_version))
+
+
+def _find_nccl_lib(repository_ctx, nccl_install_path, nccl_version):
+  """Finds the given NCCL library on the system.
+
+  Args:
+    repository_ctx: The repository context.
+    nccl_install_path: The NCCL library installation directory.
+    nccl_version: The version of NCCL library files as returned
+      by _nccl_version.
+
+  Returns:
+    The path to the NCCL library.
+  """
+  lib_path = repository_ctx.path("%s/lib/libnccl.so.%s" % (nccl_install_path,
+                                                           nccl_version))
+  if not lib_path.exists:
+    auto_configure_fail("Cannot find NCCL library %s" % str(lib_path))
+  return lib_path
+
+
+def _nccl_configure_impl(repository_ctx):
+  """Implementation of the nccl_configure repository rule."""
+  if _TF_NCCL_VERSION not in repository_ctx.os.environ:
+    # Add a dummy build file to make bazel query happy.
+    repository_ctx.file("BUILD", _NCCL_DUMMY_BUILD_CONTENT)
+    return
+
+  nccl_version = repository_ctx.os.environ[_TF_NCCL_VERSION].strip()
+  if matches_version("1", nccl_version):
+    # Alias to GitHub target from @nccl_archive.
+    if not matches_version(nccl_version, "1.3"):
+      auto_configure_fail(
+          "NCCL from GitHub must use version 1.3 (got %s)" % nccl_version)
+    repository_ctx.file("BUILD", _NCCL_ARCHIVE_BUILD_CONTENT)
+  else:
+    # Create target for locally installed NCCL.
+    nccl_install_path = repository_ctx.os.environ[_NCCL_INSTALL_PATH].strip()
+    _check_nccl_version(repository_ctx, nccl_install_path, nccl_version)
+    repository_ctx.symlink(nccl_install_path, "nccl")
+    repository_ctx.file("BUILD", _NCCL_LOCAL_BUILD_TEMPLATE % nccl_version)
+
+
+nccl_configure = repository_rule(
+    implementation=_nccl_configure_impl,
+    environ=[
+        _NCCL_INSTALL_PATH,
+        _TF_NCCL_VERSION,
+    ],
+)
+"""Detects and configures the NCCL configuration.
+
+Add the following to your WORKSPACE FILE:
+
+```python
+nccl_configure(name = "local_config_nccl")
+```
+
+Args:
+  name: A unique name for this workspace rule.
+"""
-- 
GitLab


From 361a13cf0c2b65d26f6e2b5b68875adfcea98dd0 Mon Sep 17 00:00:00 2001
From: "A. Unique TensorFlower" <gardener@tensorflow.org>
Date: Thu, 5 Apr 2018 05:39:21 -0700
Subject: [PATCH 947/960] Create a separate operators module that is to contain
 all Python constructs that we override: control flow, builtins, operators,
 etc.

PiperOrigin-RevId: 191729654
---
 tensorflow/contrib/autograph/impl/BUILD       |  1 +
 tensorflow/contrib/autograph/impl/config.py   | 17 ++++++++-----
 tensorflow/contrib/autograph/operators/BUILD  | 25 +++++++++++++++++++
 .../contrib/autograph/operators/__init__.py   | 24 ++++++++++++++++++
 4 files changed, 61 insertions(+), 6 deletions(-)
 create mode 100644 tensorflow/contrib/autograph/operators/BUILD
 create mode 100644 tensorflow/contrib/autograph/operators/__init__.py

diff --git a/tensorflow/contrib/autograph/impl/BUILD b/tensorflow/contrib/autograph/impl/BUILD
index 0de479741a..54424e2647 100644
--- a/tensorflow/contrib/autograph/impl/BUILD
+++ b/tensorflow/contrib/autograph/impl/BUILD
@@ -26,6 +26,7 @@ py_library(
     visibility = ["//tensorflow:__subpackages__"],
     deps = [
         "//tensorflow/contrib/autograph/converters",
+        "//tensorflow/contrib/autograph/operators",
         "//tensorflow/contrib/autograph/pyct",
         "//tensorflow/contrib/autograph/pyct/static_analysis",
         "//tensorflow/contrib/autograph/utils",
diff --git a/tensorflow/contrib/autograph/impl/config.py b/tensorflow/contrib/autograph/impl/config.py
index 543c1486e6..26326465e2 100644
--- a/tensorflow/contrib/autograph/impl/config.py
+++ b/tensorflow/contrib/autograph/impl/config.py
@@ -41,10 +41,15 @@ DEFAULT_UNCOMPILED_MODULES = set((
 
 NO_SIDE_EFFECT_CONSTRUCTORS = set(('tensorflow',))
 
-# TODO(mdan): Also allow controlling the generated names (for testability).
+# TODO(mdan): Also allow controlling the generated names.
+# TODO(mdan); Consolidate all internal imports into a single __ag module.
 COMPILED_IMPORT_STATEMENTS = (
-    'from __future__ import print_function', 'import tensorflow as tf',
-    'from tensorflow.contrib.autograph.impl import api as '
-    'autograph_api',
-    'from tensorflow.contrib.autograph import utils as '
-    'autograph_utils')
+    'from __future__ import print_function',
+    'import tensorflow as tf',
+    'from tensorflow.contrib.autograph.impl import api'
+    ' as autograph_api',
+    'from tensorflow.contrib.autograph import utils'
+    ' as autograph_utils',
+    'from tensorflow.contrib.autograph import operators'
+    ' as __ops',
+)
diff --git a/tensorflow/contrib/autograph/operators/BUILD b/tensorflow/contrib/autograph/operators/BUILD
new file mode 100644
index 0000000000..7856c253bd
--- /dev/null
+++ b/tensorflow/contrib/autograph/operators/BUILD
@@ -0,0 +1,25 @@
+licenses(["notice"])  # Apache 2.0
+
+exports_files(["LICENSE"])
+
+filegroup(
+    name = "all_files",
+    srcs = glob(
+        ["**/*"],
+        exclude = [
+            "**/METADATA",
+            "**/OWNERS",
+        ],
+    ),
+    visibility = ["//tensorflow:__subpackages__"],
+)
+
+py_library(
+    name = "operators",
+    srcs = [
+        "__init__.py",
+    ],
+    srcs_version = "PY2AND3",
+    visibility = ["//tensorflow:__subpackages__"],
+    deps = [],
+)
diff --git a/tensorflow/contrib/autograph/operators/__init__.py b/tensorflow/contrib/autograph/operators/__init__.py
new file mode 100644
index 0000000000..c3f4cab69e
--- /dev/null
+++ b/tensorflow/contrib/autograph/operators/__init__.py
@@ -0,0 +1,24 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""This module implements operators that we overload.
+
+Note that "operator" is used loosely here, and includes control structures like
+conditionals and loops, implemented in functional form, using for example
+closures for the body.
+"""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
-- 
GitLab


From 435e3ba8d3898c94f114aaa99cbeb1e741985e58 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sebasti=C3=A1n=20Ram=C3=ADrez?= <tiangolo@gmail.com>
Date: Thu, 5 Apr 2018 20:11:52 +0400
Subject: [PATCH 948/960] Docker Jupyter: Update deprecated
 softmax_cross_entropy_with_logits (#17412)

* Docker Jupyter Notebooks: Update deprecated softmax_cross_entropy_with_logits

* Docker Jupyter Notebooks: Revert removing collapsed: false
---
 .../tools/docker/notebooks/3_mnist_from_scratch.ipynb       | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb b/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb
index 5585ebdcd3..824fe14560 100644
--- a/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb
+++ b/tensorflow/tools/docker/notebooks/3_mnist_from_scratch.ipynb
@@ -1207,7 +1207,7 @@
    "source": [
     "# Training computation: logits + cross-entropy loss.\n",
     "logits = model(train_data_node, True)\n",
-    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits(\n",
+    "loss = tf.reduce_mean(tf.nn.softmax_cross_entropy_with_logits_v2(\n",
     "  labels=train_labels_node, logits=logits))\n",
     "\n",
     "# L2 regularization for the fully connected parameters.\n",
@@ -2031,7 +2031,7 @@
    "views": {}
   },
   "kernelspec": {
-   "display_name": "Python [default]",
+   "display_name": "Python 3",
    "language": "python",
    "name": "python3"
   },
@@ -2049,5 +2049,5 @@
   }
  },
  "nbformat": 4,
- "nbformat_minor": 0
+ "nbformat_minor": 1
 }
-- 
GitLab


From 14241b17aae754e2a64c8a350caf63e6572fe9cd Mon Sep 17 00:00:00 2001
From: Jayaram Bobba <jayaram.bobba@intel.com>
Date: Thu, 5 Apr 2018 10:28:59 -0700
Subject: [PATCH 949/960] [Intel MKL] Change inter op defaults when built with
 MKL  (#17931)

* Change inter op defaults when built with MKL to avoid thread oversubscription

* Bump up default mkl inter_op to be less conservative
---
 .../core/common_runtime/process_util.cc       | 21 +++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/tensorflow/core/common_runtime/process_util.cc b/tensorflow/core/common_runtime/process_util.cc
index d5bd7f8b98..7ff360ee26 100644
--- a/tensorflow/core/common_runtime/process_util.cc
+++ b/tensorflow/core/common_runtime/process_util.cc
@@ -15,6 +15,9 @@ limitations under the License.
 
 #include "tensorflow/core/common_runtime/process_util.h"
 
+#ifdef INTEL_MKL
+#include <omp.h>
+#endif
 #include <string.h>
 
 #include "tensorflow/core/lib/core/threadpool.h"
@@ -47,10 +50,24 @@ thread::ThreadPool* ComputePool(const SessionOptions& options) {
 }
 
 int32 NumInterOpThreadsFromSessionOptions(const SessionOptions& options) {
-  const int32 t = options.config.inter_op_parallelism_threads();
-  if (t != 0) return t;
+  const int32 inter_op = options.config.inter_op_parallelism_threads();
+  if (inter_op != 0) return inter_op;
+#ifdef INTEL_MKL
+  // MKL library executes ops in parallel using OMP threads
+  // Set inter_op conservatively to avoid thread oversubscription that could 
+  // lead to severe perf degradations and OMP resource exhaustion
+  const int mkl_intra_op = omp_get_max_threads();
+  CHECK_GE(mkl_intra_op, 1);
+  const int32 mkl_inter_op = std::max(
+          (port::NumSchedulableCPUs() + mkl_intra_op - 1) / mkl_intra_op, 2);
+  VLOG(0) << "Creating new thread pool with default inter op setting: "
+          << mkl_inter_op
+          << ". Tune using inter_op_parallelism_threads for best performance.";
+  return mkl_inter_op;
+#else
   // Default to using the number of cores available in the process.
   return port::NumSchedulableCPUs();
+#endif
 }
 
 thread::ThreadPool* NewThreadPoolFromSessionOptions(
-- 
GitLab


From de61d322391a824c9dd97b5b4913b45f8a12539d Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Fri, 6 Apr 2018 01:29:32 +0800
Subject: [PATCH 950/960] Fix some rendering format in contrib doc strings
 (#18148)

* Fix some rendering format in contrib doc strings

* Fix line too long pylint error
---
 .../contrib/kernel_methods/python/losses.py   |  6 +-
 .../python/mappers/random_fourier_features.py | 16 ++--
 .../mappers/random_fourier_features_test.py   |  2 +-
 .../contrib/kfac/python/ops/fisher_blocks.py  | 82 +++++++++----------
 .../seq2seq/python/ops/attention_wrapper.py   |  4 +-
 tensorflow/contrib/sparsemax/__init__.py      |  2 +-
 .../contrib/sparsemax/python/ops/sparsemax.py |  2 +-
 7 files changed, 57 insertions(+), 57 deletions(-)

diff --git a/tensorflow/contrib/kernel_methods/python/losses.py b/tensorflow/contrib/kernel_methods/python/losses.py
index f182fef067..4ef0a66a52 100644
--- a/tensorflow/contrib/kernel_methods/python/losses.py
+++ b/tensorflow/contrib/kernel_methods/python/losses.py
@@ -43,10 +43,10 @@ def sparse_multiclass_hinge_loss(
 
   This is a generalization of standard (binary) hinge loss. For a given instance
   with correct label c*, the loss is given by:
-    loss = max_{c != c*} logits_c - logits_{c*} + 1.
+    $$loss = max_{c != c*} logits_c - logits_{c*} + 1.$$
   or equivalently
-    loss = max_c { logits_c - logits_{c*} + I_{c != c*} }
-  where I_{c != c*} = 1 if c != c* and 0 otherwise.
+    $$loss = max_c { logits_c - logits_{c*} + I_{c != c*} }$$
+  where \\(I_{c != c*} = 1\ \text{if}\ c != c*\\) and 0 otherwise.
 
   Args:
     labels: `Tensor` of shape [batch_size] or [batch_size, 1]. Corresponds to
diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py
index 9dc01124ab..091f0a1098 100644
--- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py
+++ b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features.py
@@ -35,23 +35,23 @@ class RandomFourierFeatureMapper(dkm.DenseKernelMapper):
 
   The RFFM mapping is used to approximate the Gaussian (RBF) kernel:
   ```
-  exp(-||x-y||_2^2 / (2 * sigma^2))
+  $$(exp(-||x-y||_2^2 / (2 * \sigma^2))$$
   ```
 
   The implementation of RFFM is based on the following paper:
   "Random Features for Large-Scale Kernel Machines" by Ali Rahimi and Ben Recht.
   (link: https://people.eecs.berkeley.edu/~brecht/papers/07.rah.rec.nips.pdf)
 
-  The mapping uses a matrix `Omega \in R^{d x D}` and a bias vector `b \in R^D`
-  where `d` is the input dimension (number of dense input features) and `D` is
-  the output dimension (i.e., dimension of the feature space the input is mapped
-  to). Each entry of `Omega` is sampled i.i.d. from a (scaled) Gaussian
-  distribution and each entry of `b` is sampled independently and uniformly from
-  [0, 2 * pi].
+  The mapping uses a matrix `\\(Omega \in R^{d x D}\\)` and a bias vector
+  `\\(b \in R^D\\)` where `d` is the input dimension (number of dense input
+  features) and `D` is the output dimension (i.e., dimension of the feature
+  space the input is mapped to). Each entry of `Omega` is sampled i.i.d. from a
+  (scaled) Gaussian distribution and each entry of `b` is sampled independently
+  and uniformly from [0, \\(2 * pi\\)].
 
   For a single input feature vector x in R^d, its RFFM is defined as:
   ```
-      sqrt(2/D) * cos(x * Omega + b)
+      $$sqrt(2/D) * cos(x * Omega + b)$$
   ```
   where `cos` is the element-wise cosine function and `x, b` are represented as
   row vectors. The aforementioned paper shows that the linear kernel of
diff --git a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py
index 6f4a264485..91929184a2 100644
--- a/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py
+++ b/tensorflow/contrib/kernel_methods/python/mappers/random_fourier_features_test.py
@@ -34,7 +34,7 @@ def _inner_product(x, y):
   """Inner product between tensors x and y.
 
   The input tensors are assumed to be in ROW representation, that is, the method
-  returns x * y^T.
+  returns \\(x * y^T\\).
 
   Args:
     x: input tensor in row format
diff --git a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
index e0d9cb5ea9..00b3673a74 100644
--- a/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
+++ b/tensorflow/contrib/kfac/python/ops/fisher_blocks.py
@@ -19,11 +19,11 @@ Information matrix. Suppose one has a model that parameterizes a posterior
 distribution over 'y' given 'x' with parameters 'params', p(y | x, params). Its
 Fisher Information matrix is given by,
 
-  F(params) = E[ v(x, y, params) v(x, y, params)^T ]
+  $$F(params) = E[ v(x, y, params) v(x, y, params)^T ]$$
 
 where,
 
-  v(x, y, params) = (d / d params) log p(y | x, params)
+  $$v(x, y, params) = (d / d params) log p(y | x, params)$$
 
 and the expectation is taken with respect to the data's distribution for 'x' and
 the model's posterior distribution for 'y',
@@ -85,7 +85,7 @@ def normalize_damping(damping, num_replications):
 def compute_pi_tracenorm(left_cov, right_cov):
   """Computes the scalar constant pi for Tikhonov regularization/damping.
 
-  pi = sqrt( (trace(A) / dim(A)) / (trace(B) / dim(B)) )
+  $$\pi = \sqrt{ (trace(A) / dim(A)) / (trace(B) / dim(B)) }$$
   See section 6.3 of https://arxiv.org/pdf/1503.05671.pdf for details.
 
   Args:
@@ -462,14 +462,14 @@ class FullyConnectedDiagonalFB(InputOutputMultiTower, FisherBlock):
   Let 'params' be a vector parameterizing a model and 'i' an arbitrary index
   into it. We are interested in Fisher(params)[i, i]. This is,
 
-    Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
-                         = E[ v(x, y, params)[i] ^ 2 ]
+    $$Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
+                         = E[ v(x, y, params)[i] ^ 2 ]$$
 
   Consider fully connected layer in this model with (unshared) weight matrix
   'w'. For an example 'x' that produces layer inputs 'a' and output
   preactivations 's',
 
-    v(x, y, w) = vec( a (d loss / d s)^T )
+    $$v(x, y, w) = vec( a (d loss / d s)^T )$$
 
   This FisherBlock tracks Fisher(params)[i, i] for all indices 'i' corresponding
   to the layer's parameters 'w'.
@@ -532,14 +532,14 @@ class ConvDiagonalFB(InputOutputMultiTower, FisherBlock):
   Let 'params' be a vector parameterizing a model and 'i' an arbitrary index
   into it. We are interested in Fisher(params)[i, i]. This is,
 
-    Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
-                         = E[ v(x, y, params)[i] ^ 2 ]
+    $$Fisher(params)[i, i] = E[ v(x, y, params) v(x, y, params)^T ][i, i]
+                         = E[ v(x, y, params)[i] ^ 2 ]$$
 
   Consider a convoluational layer in this model with (unshared) filter matrix
   'w'. For an example image 'x' that produces layer inputs 'a' and output
   preactivations 's',
 
-    v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T )
+    $$v(x, y, w) = vec( sum_{loc} a_{loc} (d loss / d s_{loc})^T )$$
 
   where 'loc' is a single (x, y) location in an image.
 
@@ -805,12 +805,12 @@ class ConvKFCBasicFB(InputOutputMultiTower, KroneckerProductFB):
   'w'. For a minibatch that produces inputs 'a' and output preactivations 's',
   this FisherBlock estimates,
 
-    F(w) = #locations * kronecker(E[flat(a) flat(a)^T],
-                                  E[flat(ds) flat(ds)^T])
+    $$F(w) = \#locations * kronecker(E[flat(a) flat(a)^T],
+                                  E[flat(ds) flat(ds)^T])$$
 
   where
 
-    ds = (d / ds) log p(y | x, w)
+    $$ds = (d / ds) log p(y | x, w)$$
     #locations = number of (x, y) locations where 'w' is applied.
 
   where the expectation is taken over all examples and locations and flat()
@@ -1567,7 +1567,7 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse,
 
     if self._option == SeriesFBApproximation.option1:
 
-      # Note that L_A = A0^(-1/2) * U_A and L_G = G0^(-1/2) * U_G.
+      # Note that \\(L_A = A0^{-1/2} * U_A and L_G = G0^{-1/2} * U_G.\\)
       L_A, psi_A = self._input_factor.get_option1quants(
           self._input_damping_func)
       L_G, psi_G = self._output_factor.get_option1quants(
@@ -1581,33 +1581,33 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse,
         T = self._num_timesteps
         return (1 - x)**2 / (T * (1 - x**2) - 2 * x * (1 - x**T))
 
-      # Y = gamma( psi_G*psi_A^T ) (computed element-wise)
+      # \\(Y = \gamma( psi_G*psi_A^T )\\) (computed element-wise)
       # Even though Y is Z-independent we are recomputing it from the psi's
       # each since Y depends on both A and G quantities, and it is relatively
       # cheap to compute.
       Y = gamma(array_ops.reshape(psi_G, [int(psi_G.shape[0]), -1]) * psi_A)
 
-      # Z = L_G^T * Z * L_A
+      # \\(Z = L_G^T * Z * L_A\\)
       # This is equivalent to the following computation from the original
       # pseudo-code:
-      # Z = G0^(-1/2) * Z * A0^(-1/2)
-      # Z = U_G^T * Z * U_A
+      # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\)
+      # \\(Z = U_G^T * Z * U_A\\)
       Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A), transpose_a=True)
 
-      # Z = Z .* Y
+      # \\(Z = Z .* Y\\)
       Z *= Y
 
-      # Z = L_G * Z * L_A^T
+      # \\(Z = L_G * Z * L_A^T\\)
       # This is equivalent to the following computation from the original
       # pseudo-code:
-      # Z = U_G * Z * U_A^T
-      # Z = G0^(-1/2) * Z * A0^(-1/2)
+      # \\(Z = U_G * Z * U_A^T\\)
+      # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\)
       Z = math_ops.matmul(L_G, math_ops.matmul(Z, L_A, transpose_b=True))
 
     elif self._option == SeriesFBApproximation.option2:
 
-      # Note that P_A = A_1^T * A_0^(-1) and P_G = G_1^T * G_0^(-1),
-      # and K_A = A_0^(-1/2) * E_A and K_G = G_0^(-1/2) * E_G.
+      # Note that \\(P_A = A_1^T * A_0^{-1} and P_G = G_1^T * G_0^{-1}\\),
+      # and \\(K_A = A_0^{-1/2} * E_A\ and\ K_G = G_0^{-1/2} * E_G.\\)
       P_A, K_A, mu_A = self._input_factor.get_option2quants(
           self._input_damping_func)
       P_G, K_G, mu_G = self._output_factor.get_option2quants(
@@ -1616,26 +1616,26 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse,
       # Our approach differs superficially from the pseudo-code in the paper
       # in order to reduce the total number of matrix-matrix multiplies.
       # In particular, the first three computations in the pseudo code are
-      # Z = G0^(-1/2) * Z * A0^(-1/2)
-      # Z = Z - hPsi_G^T * Z * hPsi_A
-      # Z = E_G^T * Z * E_A
-      # Noting that hPsi = C0^(-1/2) * C1 * C0^(-1/2), so that
-      # C0^(-1/2) * hPsi = C0^(-1) * C1 * C0^(-1/2) = P^T * C0^(-1/2)
+      # \\(Z = G0^{-1/2} * Z * A0^{-1/2}\\)
+      # \\(Z = Z - hPsi_G^T * Z * hPsi_A\\)
+      # \\(Z = E_G^T * Z * E_A\\)
+      # Noting that hPsi = C0^{-1/2} * C1 * C0^{-1/2}\\), so that
+      # \\(C0^{-1/2} * hPsi = C0^{-1} * C1 * C0^{-1/2} = P^T * C0^{-1/2}\\)
       # the entire computation can be written as
-      # Z = E_G^T * (G0^(-1/2) * Z * A0^(-1/2)
-      #     - hPsi_G^T * G0^(-1/2) * Z * A0^(-1/2) * hPsi_A) * E_A
-      #   = E_G^T * (G0^(-1/2) * Z * A0^(-1/2)
-      #     - G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2)) * E_A
-      #   = E_G^T * G0^(-1/2) * Z * A0^(-1/2) * E_A
-      #     -  E_G^T* G0^(-1/2) * P_G * Z * P_A^T * A0^(-1/2) * E_A
-      #   = K_G^T * Z * K_A  -  K_G^T * P_G * Z * P_A^T * K_A
+      # \\(Z = E_G^T * (G0^{-1/2} * Z * A0^{-1/2}\\)
+      # \\(    - hPsi_G^T * G0^{-1/2} * Z * A0^{-1/2} * hPsi_A) * E_A\\)
+      # \\(  = E_G^T * (G0^{-1/2} * Z * A0^{-1/2}\\)
+      # \\(    - G0^{-1/2} * P_G * Z * P_A^T * A0^{-1/2}) * E_A\\)
+      # \\(  = E_G^T * G0^{-1/2} * Z * A0^{-1/2} * E_A\\)
+      # \\(    -  E_G^T* G0^{-1/2} * P_G * Z * P_A^T * A0^{-1/2} * E_A\\)
+      # \\(  = K_G^T * Z * K_A  -  K_G^T * P_G * Z * P_A^T * K_A\\)
       # This final expression is computed by the following two lines:
-      # Z = Z - P_G * Z * P_A^T
+      # \\(Z = Z - P_G * Z * P_A^T\\)
       Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A, transpose_b=True))
-      # Z = K_G^T * Z * K_A
+      # \\(Z = K_G^T * Z * K_A\\)
       Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A), transpose_a=True)
 
-      # Z = Z ./ (1*1^T - mu_G*mu_A^T)
+      # \\(Z = Z ./ (1*1^T - mu_G*mu_A^T)\\)
       # Be careful with the outer product.  We don't want to accidentally
       # make it an inner-product instead.
       tmp = 1.0 - array_ops.reshape(mu_G, [int(mu_G.shape[0]), -1]) * mu_A
@@ -1646,13 +1646,13 @@ class FullyConnectedSeriesFB(InputOutputMultiTowerMultiUse,
       # We now perform the transpose/reverse version of the operations
       # derived above, whose derivation from the original pseudo-code is
       # analgous.
-      # Z = K_G * Z * K_A^T
+      # \\(Z = K_G * Z * K_A^T\\)
       Z = math_ops.matmul(K_G, math_ops.matmul(Z, K_A, transpose_b=True))
 
-      # Z = Z - P_G^T * Z * P_A
+      # \\(Z = Z - P_G^T * Z * P_A\\)
       Z -= math_ops.matmul(P_G, math_ops.matmul(Z, P_A), transpose_a=True)
 
-      # Z = normalize (1/E[T]) * Z
+      # \\(Z = normalize (1/E[T]) * Z\\)
       # Note that this normalization is done because we compute the statistics
       # by averaging, not summing, over time. (And the gradient is presumably
       # summed over time, not averaged, and thus their scales are different.)
diff --git a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
index 9e0d69593f..f0f143ddfc 100644
--- a/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
+++ b/tensorflow/contrib/seq2seq/python/ops/attention_wrapper.py
@@ -610,8 +610,8 @@ def monotonic_attention(p_choose_i, previous_attention, mode):
   addition, once an input sequence element is attended to at a given output
   timestep, elements occurring before it cannot be attended to at subsequent
   output timesteps.  This function generates attention distributions according
-  to these assumptions.  For more information, see ``Online and Linear-Time
-  Attention by Enforcing Monotonic Alignments''.
+  to these assumptions.  For more information, see `Online and Linear-Time
+  Attention by Enforcing Monotonic Alignments`.
 
   Args:
     p_choose_i: Probability of choosing input sequence/memory element i.  Should
diff --git a/tensorflow/contrib/sparsemax/__init__.py b/tensorflow/contrib/sparsemax/__init__.py
index 19d213fb3e..7bc726f4a8 100644
--- a/tensorflow/contrib/sparsemax/__init__.py
+++ b/tensorflow/contrib/sparsemax/__init__.py
@@ -14,7 +14,7 @@
 # ==============================================================================
 """Module that implements sparsemax and sparsemax loss, see [1].
 
-[1] https://arxiv.org/abs/1602.02068
+[1]: https://arxiv.org/abs/1602.02068
 
 ## Sparsemax
 
diff --git a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py b/tensorflow/contrib/sparsemax/python/ops/sparsemax.py
index 890ca20f4c..e617af2ff1 100644
--- a/tensorflow/contrib/sparsemax/python/ops/sparsemax.py
+++ b/tensorflow/contrib/sparsemax/python/ops/sparsemax.py
@@ -31,7 +31,7 @@ def sparsemax(logits, name=None):
   """Computes sparsemax activations [1].
 
   For each batch `i` and class `j` we have
-    sparsemax[i, j] = max(logits[i, j] - tau(logits[i, :]), 0)
+    $$sparsemax[i, j] = max(logits[i, j] - tau(logits[i, :]), 0)$$
 
   [1]: https://arxiv.org/abs/1602.02068
 
-- 
GitLab


From 3d8a0538c1d75142c381a5c169ef0696cb95b4ec Mon Sep 17 00:00:00 2001
From: Alan Yee <alyee@ucsd.edu>
Date: Thu, 5 Apr 2018 12:32:03 -0700
Subject: [PATCH 951/960] Update metrics_ops.py (#18155)

* Update metrics_ops.py

Add deprecation notes

* Update metrics_ops.py

Fix styling for linter
---
 .../contrib/metrics/python/ops/metric_ops.py  | 37 +++++++++++--------
 1 file changed, 21 insertions(+), 16 deletions(-)

diff --git a/tensorflow/contrib/metrics/python/ops/metric_ops.py b/tensorflow/contrib/metrics/python/ops/metric_ops.py
index 81f05e7ce5..088319a557 100644
--- a/tensorflow/contrib/metrics/python/ops/metric_ops.py
+++ b/tensorflow/contrib/metrics/python/ops/metric_ops.py
@@ -62,7 +62,8 @@ def _safe_div(numerator, denominator, name):
       0,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.true_positives. Note that the '
+            'order of the labels and predictions arguments has been switched.')
 def streaming_true_positives(predictions,
                              labels,
                              weights=None,
@@ -106,7 +107,8 @@ def streaming_true_positives(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.true_negatives. Note that the '
+            'order of the labels and predictions arguments has been switched.')
 def streaming_true_negatives(predictions,
                              labels,
                              weights=None,
@@ -150,7 +152,8 @@ def streaming_true_negatives(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.false_positives. Note that the '
+            'order of the labels and predictions arguments has been switched.')
 def streaming_false_positives(predictions,
                               labels,
                               weights=None,
@@ -194,7 +197,8 @@ def streaming_false_positives(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.false_negatives. Note that the '
+            'order of the labels and predictions arguments has been switched.')
 def streaming_false_negatives(predictions,
                               labels,
                               weights=None,
@@ -237,7 +241,7 @@ def streaming_false_negatives(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.mean')
 def streaming_mean(values,
                    weights=None,
                    metrics_collections=None,
@@ -286,7 +290,7 @@ def streaming_mean(values,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.mean_tensor')
 def streaming_mean_tensor(values,
                           weights=None,
                           metrics_collections=None,
@@ -340,9 +344,8 @@ def streaming_mean_tensor(values,
       name=name)
 
 
-@deprecated(None,
-            'Please switch to tf.metrics.accuracy. Note that the order of the '
-            'labels and predictions arguments has been switched.')
+@deprecated(None, 'Please switch to tf.metrics.accuracy. Note that the order '
+                  'of the labels and predictions arguments has been switched.')
 def streaming_accuracy(predictions,
                        labels,
                        weights=None,
@@ -399,7 +402,8 @@ def streaming_accuracy(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.precision. Note that the order '
+                  'of the labels and predictions arguments has been switched.')
 def streaming_precision(predictions,
                         labels,
                         weights=None,
@@ -455,7 +459,8 @@ def streaming_precision(predictions,
       updates_collections=updates_collections,
       name=name)
 
-
+@deprecated(None, 'Please switch to tf.metrics.recall. Note that the order '
+                  'of the labels and predictions arguments has been switched.')
 def streaming_recall(predictions,
                      labels,
                      weights=None,
@@ -975,8 +980,8 @@ def streaming_curve_points(labels=None,
     return points, update_op
 
 
-@deprecated(None, 'Please switch to tf.metrics.auc. Note that the order of the '
-            'labels and predictions arguments has been switched.')
+@deprecated(None, 'Please switch to tf.metrics.auc. Note that the order of '
+                  'the labels and predictions arguments has been switched.')
 def streaming_auc(predictions,
                   labels,
                   weights=None,
@@ -1797,9 +1802,9 @@ def streaming_sensitivity_at_specificity(predictions,
       name=name)
 
 
-@deprecated(
-    None, 'Please switch to tf.metrics.precision_at_thresholds. Note that the '
-    'order of the labels and predictions arguments has been switched.')
+@deprecated(None,
+            'Please switch to tf.metrics.precision_at_thresholds. Note that '
+            'the order of the labels and predictions arguments are switched.')
 def streaming_precision_at_thresholds(predictions,
                                       labels,
                                       thresholds,
-- 
GitLab


From af0790fdc841092e14dd2ed0c753f088edb660b6 Mon Sep 17 00:00:00 2001
From: Yun Peng <pcloudy@google.com>
Date: Thu, 5 Apr 2018 23:32:05 +0200
Subject: [PATCH 952/960] Add win_def_file attribute for
 tensorflow/python:pywrap_tensorflow_internal

This attribute is somehow missing when pushing for internal to github.

This should fix the TensorFlow Bazel postsubmit for github.
---
 tensorflow/python/BUILD | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tensorflow/python/BUILD b/tensorflow/python/BUILD
index 6ec8a1cdab..a8f1318509 100644
--- a/tensorflow/python/BUILD
+++ b/tensorflow/python/BUILD
@@ -3343,6 +3343,10 @@ tf_py_wrap_cc(
         "util/transform_graph.i",
         "util/util.i",
     ],
+    win_def_file = select({
+        "//tensorflow:windows": ":pywrap_tensorflow_filtered_def_file",
+        "//conditions:default": None,
+    }),
     deps = [
         ":bfloat16_lib",
         ":cost_analyzer_lib",
-- 
GitLab


From ffe24e657bc9dc365f98de17e7118d94d88c3705 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 5 Apr 2018 16:49:06 -0700
Subject: [PATCH 953/960] Fix warning in rnn_cell.py (#18164)

This fix fixes the warning in rnn_cell.py caused by l2_normalize with dim:
```
rnn_cell.py:2894: calling l2_normalize (from tensorflow.python.ops.nn_impl) with dim is deprecated and will be removed in a future version.
```

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/rnn/python/ops/rnn_cell.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/rnn/python/ops/rnn_cell.py b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
index 2f6ae9f367..b12e2cd5ed 100644
--- a/tensorflow/contrib/rnn/python/ops/rnn_cell.py
+++ b/tensorflow/contrib/rnn/python/ops/rnn_cell.py
@@ -2891,7 +2891,7 @@ class WeightNormLSTMCell(rnn_cell_impl.RNNCell):
 
     output_size = weight.get_shape().as_list()[1]
     g = vs.get_variable(name, [output_size], dtype=weight.dtype)
-    return nn_impl.l2_normalize(weight, dim=0) * g
+    return nn_impl.l2_normalize(weight, axis=0) * g
 
   def _linear(self,
               args,
-- 
GitLab


From d5fc41a8ca69d37696c7c324e19a3a724b70192b Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 5 Apr 2018 16:49:29 -0700
Subject: [PATCH 954/960] Fix `TypeError: 'dict_keys'` in contrib.distribute
 with python 3 (#18212)

This fix tries to fix the issue raised in 18205 where
```
TypeError: 'dict_keys' object does not support indexing
```
was thrown when using contrib.distribute in python 3.

The issue is that DistributedValues.devices returned `self._index.keys()`
which is a `dict_keys` and is not a list in python 3.

This fix converts the `dict_keys` to list for python 3
to fix the issue.

This fix fixes 18205.
This fix als fixes 18188.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/contrib/distribute/python/values.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/contrib/distribute/python/values.py b/tensorflow/contrib/distribute/python/values.py
index 9acb6a9db9..87bf059038 100644
--- a/tensorflow/contrib/distribute/python/values.py
+++ b/tensorflow/contrib/distribute/python/values.py
@@ -73,7 +73,7 @@ class DistributedValues(object):
 
   @property
   def devices(self):
-    return self._index.keys()
+    return list(self._index.keys())
 
   def __str__(self):
     return "%s:%s" % (self.__class__.__name__, self._index)
-- 
GitLab


From 73f25fc34c69878c83ee2eeb8f030cb79a76472f Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 5 Apr 2018 16:52:07 -0700
Subject: [PATCH 955/960] Fix an issue caused by (#18183)

While trying to run on my machine (Ubuntu 16.04 Python 2.7)
```
bazel test -s --config=opt --cache_test_results=no //tensorflow/tools/api/tests:api_compatibility_test
```

The following error was encountered:
```
  ......
  File "/home/ubuntu/.cache/bazel/_bazel_ubuntu/ad1e09741bb4109fbc70ef8216b59ee2/execroot/org_tensorflow/bazel-out/host/bin/tensorflow/tools/api/generator/create_python_api.runfiles/org_tensorflow/tensorflow/tools/api/generator/create_python_api.py", line 125, in get_api_imports
    if not module or 'tensorflow.' not in module.__name__:
  File "/usr/lib/python2.7/dist-packages/py/_apipkg.py", line 171, in __getattribute__
    return getattr(getmod(), name)
  File "/usr/lib/python2.7/dist-packages/py/_error.py", line 43, in __getattr__
    raise AttributeError(name)
AttributeError: __name__
```

The issue is that `<AliasModule 'py.error' for 'py._error.error'>` does not
have a `__name__` attribute (See similiar issue in https://github.com/pytest-dev/py/issues/73).

This fix tries to address the issue by adding an `hasattr()` check so
that AttributeError is not thrown.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 tensorflow/tools/api/generator/create_python_api.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/tensorflow/tools/api/generator/create_python_api.py b/tensorflow/tools/api/generator/create_python_api.py
index 183c4731b8..70f9776b08 100644
--- a/tensorflow/tools/api/generator/create_python_api.py
+++ b/tensorflow/tools/api/generator/create_python_api.py
@@ -122,7 +122,8 @@ def get_api_imports():
   # we want to traverse over TensorFlow Python modules.
   for module in sys.modules.values():
     # Only look at tensorflow modules.
-    if not module or 'tensorflow.' not in module.__name__:
+    if (not module or not hasattr(module, "__name__") or
+        'tensorflow.' not in module.__name__):
       continue
     # Do not generate __init__.py files for contrib modules for now.
     if '.contrib.' in module.__name__ or module.__name__.endswith('.contrib'):
-- 
GitLab


From 8b9509c2ae41e2fc1c925ec14903b09a14b84803 Mon Sep 17 00:00:00 2001
From: ImSheridan <xiaoyudong0512@gmail.com>
Date: Fri, 6 Apr 2018 07:54:33 +0800
Subject: [PATCH 956/960] Fix minor typos for programer guide documents
 (#17486)

* Fix minor typo fix for proggramer guide doccuments

* revert typo in dataset.md

* revert wrong typo fix

* revert wrong typo fix
---
 tensorflow/docs_src/get_started/custom_estimators.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tensorflow/docs_src/get_started/custom_estimators.md b/tensorflow/docs_src/get_started/custom_estimators.md
index 941c3e1690..275cda12bc 100644
--- a/tensorflow/docs_src/get_started/custom_estimators.md
+++ b/tensorflow/docs_src/get_started/custom_estimators.md
@@ -546,7 +546,7 @@ In brief, here's what the three graphs tell you:
 
 * accuracy: The accuracy is recorded by the following two lines:
 
-    * `eval_metric_ops={'my_accuracy': accuracy})`, during evaluation.
+    * `eval_metric_ops={'my_accuracy': accuracy}`, during evaluation.
     * `tf.summary.scalar('accuracy', accuracy[1])`, during training.
 
 These tensorboard graphs are one of the main reasons it's important to pass a
-- 
GitLab


From 261222e69e4e0ecab044c2fd74531b574a66d812 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A9dric=20Deltheil?=
 <355031+deltheil@users.noreply.github.com>
Date: Fri, 6 Apr 2018 01:56:20 +0200
Subject: [PATCH 957/960] contrib/lite: avoid building benchmark_model in
 build_ios_universal_lib.sh (#17796)

It is not needed part of this script. In addition, since f0633ec benchmark_model
now depends on //tensorflow/core library which is not taken into account by the
Makefile from contrib/lite, and thus causes failure:

$ ./build_ios_universal_lib.sh
...
In file included from tensorflow/contrib/lite/tools/benchmark_model.cc:29:
In file included from ./tensorflow/core/platform/env.h:24:
In file included from ./tensorflow/core/lib/core/errors.h:21:
./tensorflow/core/lib/core/status.h:23:10: fatal error: 'tensorflow/core/lib/core/error_codes.pb.h' file not found
#include "tensorflow/core/lib/core/error_codes.pb.h"
         ^~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
1 error generated.
---
 .../contrib/lite/build_ios_universal_lib.sh       | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/tensorflow/contrib/lite/build_ios_universal_lib.sh b/tensorflow/contrib/lite/build_ios_universal_lib.sh
index 4a9023ff33..9f398f4a9f 100755
--- a/tensorflow/contrib/lite/build_ios_universal_lib.sh
+++ b/tensorflow/contrib/lite/build_ios_universal_lib.sh
@@ -19,11 +19,16 @@ set -e
 SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 cd "$SCRIPT_DIR/../../.."
 
-make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=x86_64 -j 8
-make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=i386 -j 8
-make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7 -j 8
-make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7s -j 8
-make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=arm64 -j 8
+make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=x86_64 -j 8 \
+$SCRIPT_DIR/gen/lib/ios_x86_64/libtensorflow-lite.a
+make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=i386 -j 8 \
+$SCRIPT_DIR/gen/lib/ios_i386/libtensorflow-lite.a
+make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7 -j 8 \
+$SCRIPT_DIR/gen/lib/ios_armv7/libtensorflow-lite.a
+make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=armv7s -j 8 \
+$SCRIPT_DIR/gen/lib/ios_armv7s/libtensorflow-lite.a
+make -f tensorflow/contrib/lite/Makefile TARGET=IOS IOS_ARCH=arm64 -j 8 \
+$SCRIPT_DIR/gen/lib/ios_arm64/libtensorflow-lite.a
 
 lipo \
 tensorflow/contrib/lite/gen/lib/ios_x86_64/libtensorflow-lite.a \
-- 
GitLab


From cde06a39592a849a2bc0ba022e858e6366c87cc5 Mon Sep 17 00:00:00 2001
From: Billy Lamberta <b@lamberta.org>
Date: Thu, 5 Apr 2018 17:14:29 -0700
Subject: [PATCH 958/960] Docs: Fix formatting in programmers_guide/debugger
 (#18281)

Fix formating for code block in debugger.md
Fixes #17946
---
 .../docs_src/programmers_guide/debugger.md    | 61 +++++++++----------
 1 file changed, 30 insertions(+), 31 deletions(-)

diff --git a/tensorflow/docs_src/programmers_guide/debugger.md b/tensorflow/docs_src/programmers_guide/debugger.md
index d1cd7e7c06..f5a0eb0a20 100644
--- a/tensorflow/docs_src/programmers_guide/debugger.md
+++ b/tensorflow/docs_src/programmers_guide/debugger.md
@@ -4,29 +4,28 @@
 
 [TOC]
 
-TensorFlow debugger (**tfdbg**) is a specialized debugger for TensorFlow. It
-lets you view the internal structure and states of running TensorFlow graphs
-during training and inference, which is difficult to debug with general-purpose
-debuggers such as Python's `pdb` due to TensorFlow's computation-graph paradigm.
-
-> NOTE: TensorFlow debugger uses a
-> [curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based
-> text user interface. On Mac OS X, the `ncurses` library is required and can
-> be installed with `brew install homebrew/dupes/ncurses`. On Windows, curses
-> isn't as well supported, so a
-> [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based interface can
-> be used with tfdbg by installing `pyreadline` with pip.
-> If you use Anaconda3, you can install it with a command
-> such as `"C:\Program Files\Anaconda3\Scripts\pip.exe" install pyreadline`.
-> Unofficial Windows curses packages can be downloaded
-> [here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#curses), then subsequently
-> installed using `pip install <your_version>.whl`, however curses on Windows
-> may not work as reliably as curses on Linux or Mac.
-
-> NOTE: This guide focuses on the command-line interface (CLI) of tfdbg. For
-> guide on how to use the graphical user interface (GUI) of tfdbg, i.e., the
-> **TensorBoard Debugger Plugin**, please visit
-> [its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md).
+`tfdbg` is a specialized debugger for TensorFlow. It lets you view the internal
+structure and states of running TensorFlow graphs during training and inference,
+which is difficult to debug with general-purpose debuggers such as Python's `pdb`
+due to TensorFlow's computation-graph paradigm.
+
+This guide focuses on the command-line interface (CLI) of `tfdbg`. For guide on
+how to use the graphical user interface (GUI) of tfdbg, i.e., the
+**TensorBoard Debugger Plugin**, please visit
+[its README](https://github.com/tensorflow/tensorboard/blob/master/tensorboard/plugins/debugger/README.md).
+
+Note: The TensorFlow debugger uses a
+[curses](https://en.wikipedia.org/wiki/Curses_\(programming_library\))-based text
+user interface. On Mac OS X, the `ncurses` library is required and can be
+installed with `brew install homebrew/dupes/ncurses`. On Windows, curses isn't as
+well supported, so a [readline](https://en.wikipedia.org/wiki/GNU_Readline)-based
+interface can be used with tfdbg by installing `pyreadline` with `pip`. If you
+use Anaconda3, you can install it with a command such as
+`"C:\Program Files\Anaconda3\Scripts\pip.exe" install pyreadline`. Unofficial
+Windows curses packages can be downloaded
+[here](https://www.lfd.uci.edu/~gohlke/pythonlibs/#curses), then subsequently
+installed using `pip install <your_version>.whl`, however curses on Windows may
+not work as reliably as curses on Linux or Mac.
 
 This tutorial demonstrates how to use the **tfdbg** CLI to debug the appearance
 of [`nan`s](https://en.wikipedia.org/wiki/NaN)
@@ -748,16 +747,16 @@ There are three possible workarounds or solutions:
    to which tfdbg dumps the debug data. You can use it to let tfdbg dump the
    debug data on a disk with larger free space. For example:
 
-   ``` python
-   # For LocalCLIDebugWrapperSession
-   sess = tf_debug.LocalCLIDebugWrapperSession(dump_root="/with/lots/of/space")
-
-   # For LocalCLIDebugHook
-   hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")]
-   ```
+```python
+# For LocalCLIDebugWrapperSession
+sess = tf_debug.LocalCLIDebugWrapperSession(dump_root="/with/lots/of/space")
 
+# For LocalCLIDebugHook
+hooks = [tf_debug.LocalCLIDebugHook(dump_root="/with/lots/of/space")]
+```
    Make sure that the directory pointed to by dump_root is empty or nonexistent.
-   tfdbg cleans up the dump directories before exiting.
+   `tfdbg` cleans up the dump directories before exiting.
+
 *  Reduce the batch size used during the runs.
 *  Use the filtering options of tfdbg's `run` command to watch only specific
    nodes in the graph. For example:
-- 
GitLab


From a1fbdf8268d8043190fe7fa0b3c5796a455ea529 Mon Sep 17 00:00:00 2001
From: Yong Tang <yong.tang.github@outlook.com>
Date: Thu, 5 Apr 2018 17:17:25 -0700
Subject: [PATCH 959/960] Validate the shape of TakeDataset and SkipDataset
 (#18267)

* Validate the shape of TakeDataset and SkipDataset

The `count` inputs of the TakeDataset and SkipDataset require
scalar. That was not validated before though.

This fix validates the shape of count for TakeDataset and SkipDataset.

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for invalide `count` of TakeDataset

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Validate shape of count for SkipDataset

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>

* Add test case for invalid count shape for SkipDataset

Signed-off-by: Yong Tang <yong.tang.github@outlook.com>
---
 .../python/kernel_tests/sequence_dataset_op_test.py  | 10 ++++++++++
 tensorflow/core/ops/dataset_ops.cc                   | 12 ++++++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
index b044ff1775..b13ad9ba4e 100644
--- a/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/sequence_dataset_op_test.py
@@ -47,6 +47,11 @@ class SequenceDatasetSerializationTest(
     # Skip nothing
     self.run_core_tests(lambda: self._build_skip_dataset(0), None, 10)
 
+  def testInvalidSkip(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Shape must be rank 0 but is rank 1'):
+      self.run_core_tests(lambda: self._build_skip_dataset([1, 2]), None, 0)
+
   def _build_take_dataset(self, count):
     components = (np.arange(10),)
     return dataset_ops.Dataset.from_tensor_slices(components).take(count)
@@ -69,6 +74,11 @@ class SequenceDatasetSerializationTest(
     # Take nothing
     self.run_core_tests(lambda: self._build_take_dataset(0), None, 0)
 
+  def testInvalidTake(self):
+    with self.assertRaisesRegexp(
+        ValueError, 'Shape must be rank 0 but is rank 1'):
+      self.run_core_tests(lambda: self._build_take_dataset([1, 2]), None, 0)
+
   def _build_repeat_dataset(self, count, take_count=3):
     components = (np.arange(10),)
     return dataset_ops.Dataset.from_tensor_slices(components).take(
diff --git a/tensorflow/core/ops/dataset_ops.cc b/tensorflow/core/ops/dataset_ops.cc
index 2852c49e19..b25abbcc67 100644
--- a/tensorflow/core/ops/dataset_ops.cc
+++ b/tensorflow/core/ops/dataset_ops.cc
@@ -117,7 +117,11 @@ REGISTER_OP("TakeDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle count_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &count_shape));
+      return shape_inference::ScalarShape(c);
+    });
 
 REGISTER_OP("SkipDataset")
     .Input("input_dataset: variant")
@@ -125,7 +129,11 @@ REGISTER_OP("SkipDataset")
     .Output("handle: variant")
     .Attr("output_types: list(type) >= 1")
     .Attr("output_shapes: list(shape) >= 1")
-    .SetShapeFn(shape_inference::ScalarShape);
+    .SetShapeFn([](shape_inference::InferenceContext* c) {
+      shape_inference::ShapeHandle count_shape;
+      TF_RETURN_IF_ERROR(c->WithRank(c->input(1), 0, &count_shape));
+      return shape_inference::ScalarShape(c);
+    });
 
 REGISTER_OP("BytesProducedStatsDataset")
     .Input("input_dataset: variant")
-- 
GitLab


From d82b2f71b60d5fff48884c20c7b85e517330e91f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Yan=20Facai=20=28=E9=A2=9C=E5=8F=91=E6=89=8D=29?=
 <facai.yan@gmail.com>
Date: Fri, 6 Apr 2018 10:51:12 +0800
Subject: [PATCH 960/960] add assert_element_shape method for tf.contrib.data
 (#17480)

* ENH: add assert_element_shape method

* CLN: add indentation

* ENH: raise exception when wrong shape is given

* CLN: fix too long line
---
 tensorflow/contrib/data/__init__.py           |  2 +
 .../contrib/data/python/kernel_tests/BUILD    |  1 +
 .../kernel_tests/batch_dataset_op_test.py     | 70 +++++++++++++++++++
 tensorflow/contrib/data/python/ops/BUILD      |  1 +
 .../contrib/data/python/ops/batching.py       | 40 +++++++++++
 5 files changed, 114 insertions(+)

diff --git a/tensorflow/contrib/data/__init__.py b/tensorflow/contrib/data/__init__.py
index 17048314a4..125260b4c1 100644
--- a/tensorflow/contrib/data/__init__.py
+++ b/tensorflow/contrib/data/__init__.py
@@ -25,6 +25,7 @@ See the @{$datasets$Importing Data} Programmer's Guide for an overview.
 @@Counter
 @@SqlDataset
 
+@@assert_element_shape
 @@batch_and_drop_remainder
 @@bucket_by_sequence_length
 @@dense_to_sparse_batch
@@ -55,6 +56,7 @@ from __future__ import print_function
 
 # pylint: disable=unused-import
 
+from tensorflow.contrib.data.python.ops.batching import assert_element_shape
 from tensorflow.contrib.data.python.ops.batching import batch_and_drop_remainder
 from tensorflow.contrib.data.python.ops.batching import dense_to_sparse_batch
 from tensorflow.contrib.data.python.ops.batching import map_and_batch
diff --git a/tensorflow/contrib/data/python/kernel_tests/BUILD b/tensorflow/contrib/data/python/kernel_tests/BUILD
index c8699e0d5a..7270d533c6 100644
--- a/tensorflow/contrib/data/python/kernel_tests/BUILD
+++ b/tensorflow/contrib/data/python/kernel_tests/BUILD
@@ -22,6 +22,7 @@ py_test(
         "//tensorflow/python:dtypes",
         "//tensorflow/python:errors",
         "//tensorflow/python:math_ops",
+        "//tensorflow/python:script_ops",
         "//tensorflow/python:sparse_tensor",
         "//tensorflow/python:string_ops",
         "//tensorflow/python:tensor_shape",
diff --git a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
index 75482f67da..413d873797 100644
--- a/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
+++ b/tensorflow/contrib/data/python/kernel_tests/batch_dataset_op_test.py
@@ -28,8 +28,10 @@ from tensorflow.python.framework import constant_op
 from tensorflow.python.framework import dtypes
 from tensorflow.python.framework import errors
 from tensorflow.python.framework import sparse_tensor
+from tensorflow.python.framework import tensor_shape
 from tensorflow.python.ops import array_ops
 from tensorflow.python.ops import math_ops
+from tensorflow.python.ops import script_ops
 from tensorflow.python.ops import string_ops
 from tensorflow.python.platform import test
 
@@ -579,5 +581,73 @@ class PaddedBatchDatasetSerializationTest(
                         lambda: build_dataset(seq_lens2), 8)
 
 
+class RestructuredDatasetTest(test.TestCase):
+
+  def test_assert_element_shape(self):
+
+    def create_unknown_shape_dataset(x):
+      return script_ops.py_func(lambda _: (np.ones(2, dtype=np.float32),
+                                           np.zeros((3, 4), dtype=np.int32)),
+                                [x],
+                                [dtypes.float32, dtypes.int32])
+
+    dataset = dataset_ops.Dataset.range(5).map(create_unknown_shape_dataset)
+    unknown_shapes = (tensor_shape.TensorShape(None),
+                      tensor_shape.TensorShape(None))
+    self.assertEqual(unknown_shapes, dataset.output_shapes)
+
+    expected_shapes = (tensor_shape.TensorShape(2),
+                       tensor_shape.TensorShape((3, 4)))
+    result = dataset.apply(batching.assert_element_shape(expected_shapes))
+    self.assertEqual(expected_shapes, result.output_shapes)
+
+    iterator = result.make_initializable_iterator()
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      for _ in range(5):
+        sess.run(get_next)
+      with self.assertRaises(errors.OutOfRangeError):
+        sess.run(get_next)
+
+  def test_assert_wrong_element_shape(self):
+
+    def create_dataset(_):
+      return (array_ops.ones(2, dtype=dtypes.float32),
+              array_ops.zeros((3, 4), dtype=dtypes.int32))
+
+    dataset = dataset_ops.Dataset.range(3).map(create_dataset)
+    wrong_shapes = (tensor_shape.TensorShape(2),
+                    tensor_shape.TensorShape((3, 10)))
+    with self.assertRaises(ValueError):
+      dataset.apply(batching.assert_element_shape(wrong_shapes))
+
+  def test_assert_wrong_element_shape_on_unknown_shape_dataset(self):
+
+    def create_unknown_shape_dataset(x):
+      return script_ops.py_func(lambda _: (np.ones(2, dtype=np.float32),
+                                           np.zeros((3, 4), dtype=np.int32)),
+                                [x],
+                                [dtypes.float32, dtypes.int32])
+
+    dataset = dataset_ops.Dataset.range(3).map(create_unknown_shape_dataset)
+    unknown_shapes = (tensor_shape.TensorShape(None),
+                      tensor_shape.TensorShape(None))
+    self.assertEqual(unknown_shapes, dataset.output_shapes)
+
+    wrong_shapes = (tensor_shape.TensorShape(2),
+                    tensor_shape.TensorShape((3, 10)))
+    iterator = (
+        dataset.apply(batching.assert_element_shape(wrong_shapes))
+        .make_initializable_iterator())
+    init_op = iterator.initializer
+    get_next = iterator.get_next()
+    with self.test_session() as sess:
+      sess.run(init_op)
+      with self.assertRaises(errors.InvalidArgumentError):
+        sess.run(get_next)
+
+
 if __name__ == "__main__":
   test.main()
diff --git a/tensorflow/contrib/data/python/ops/BUILD b/tensorflow/contrib/data/python/ops/BUILD
index 236792bb98..a1a5c9ed05 100644
--- a/tensorflow/contrib/data/python/ops/BUILD
+++ b/tensorflow/contrib/data/python/ops/BUILD
@@ -119,6 +119,7 @@ py_library(
     deps = [
         ":contrib_op_loader",
         ":gen_dataset_ops",
+        "//tensorflow/contrib/framework:framework_py",
         "//tensorflow/python:array_ops",
         "//tensorflow/python:control_flow_ops",
         "//tensorflow/python:dataset_ops_gen",
diff --git a/tensorflow/contrib/data/python/ops/batching.py b/tensorflow/contrib/data/python/ops/batching.py
index a212adf6cf..1eba010b56 100644
--- a/tensorflow/contrib/data/python/ops/batching.py
+++ b/tensorflow/contrib/data/python/ops/batching.py
@@ -17,6 +17,7 @@ from __future__ import absolute_import
 from __future__ import division
 from __future__ import print_function
 
+from tensorflow.contrib.framework import with_shape
 from tensorflow.python.data.ops import dataset_ops
 from tensorflow.python.data.util import nest
 from tensorflow.python.data.util import sparse
@@ -345,6 +346,45 @@ class _RestructuredDataset(dataset_ops.Dataset):
     return self._output_shapes
 
 
+def assert_element_shape(expected_shapes):
+  """Assert the shape of this `Dataset`.
+
+  ```python
+  shapes = [tf.TensorShape([16, 256]), tf.TensorShape(None)]
+  result = dataset.apply(tf.contrib.data.assert_element_shape(shapes))
+  print(result.output_shapes)  # ==> "((16, 256), <unknown>)"
+  ```
+
+  If dataset shapes and expected_shape, are fully defined, assert they match.
+  Otherwise, add assert op that will validate the shapes when tensors are
+  evaluated, and set shapes on tensors, respectively.
+
+  Args:
+    expected_shapes: A nested structure of `tf.TensorShape` objects.
+
+  Returns:
+    A `Dataset` transformation function, which can be passed to
+    @{tf.data.Dataset.apply}
+  """
+
+  def _check_shape(*elements):
+    flatten_tensors = nest.flatten(elements)
+    flatten_shapes = nest.flatten(expected_shapes)
+    checked_tensors = [with_shape(shape, tensor)
+                       for shape, tensor in zip(flatten_shapes,
+                                                flatten_tensors)]
+    return nest.pack_sequence_as(elements, checked_tensors)
+
+  def _apply_fn(dataset):
+    return _RestructuredDataset(
+        dataset.map(_check_shape),
+        dataset.output_types,
+        output_shapes=expected_shapes,
+        output_classes=dataset.output_classes)
+
+  return _apply_fn
+
+
 class _MapAndBatchDataset(dataset_ops.MapDataset):
   """A `Dataset` that maps a function over a batch of elements."""
 
-- 
GitLab